| 20260507 13:50:41.567 | INFO | /opt/pacbio/bin/smrt-basecaller --process-log-interval=5000 --process-log=NONE --config=traceSaver.roi=[[2048,768,1,64],[2048,2304,1,64],[2048,3840,1,64],[2048,5376,1,64]]@bazWriter.samplingParams.filenames[0]=/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1_basic_preview.baz@bazWriter.samplingParams.filenames[1]=/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1_full_preview.baz@bazWriter.samplingParams.presets[0]=BasicPreview@bazWriter.samplingParams.presets[1]=FullPreview@realTimeMetrics.jsonOutputFile=/data/nrta/2/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.rtmetrics.json@/data/icc/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.basecaller.config.json@source.WXIPCDataSourceConfig.sraIndex=0@maskFraction=1.0000000@bazIO.MaxMiBToWrite=12874602@dataSource.darkCalFileName=/data/icc/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.darkcal.h5@dataSource.imagePsfKernel=[[0.0008,0.003,0.0166,0.0035,0.0011],[0.0027,0.013,0.0727,0.014,0.0032],[0.0118,0.0545,0.5944,0.0653,0.0124],[0.0032,0.0131,0.0727,0.0141,0.0031],[0.001,0.0029,0.0168,0.0034,0.0009]]@system.analyzerHardware=A100 --statusfd=2 --outputbazfile=/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1.bazfilelist.txt --outputtrcfile=/data/icc/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.trc.h5 --numWorkerThreads=0 --nop=0 --maxFrames=8640000 --log-level=INFO --log-file=/data/icc/m84031_260507_084733_s1/metadata/m84031_260507_084733_s1.basecaller.log,/var/log/pacbio/basecaller/m84031_260507_084733_s1.basecaller --log-directory-max-size=1073741820 | 20260507 13:50:41.567 | INFO | Version 26.1.0 | 20260507 13:50:41.567 | INFO | git branch: release/kestrel-13.5 | 20260507 13:50:41.567 | INFO | git commit hash: 2dc54f5a66f39b94662202d5514157d553e25f7d | 20260507 13:50:41.567 | INFO | git commit date: 2026-01-13 11:49:20 +0100 | 20260507 13:50:41.568 | INFO | { | 20260507 13:50:41.569 | INFO | "algorithm" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "Metrics" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "Method" : "Gpu", | 20260507 13:50:41.569 | INFO | "framesPerHFMetricBlock" : 4096, | 20260507 13:50:41.569 | INFO | "generateFeedbackMetrics" : true, | 20260507 13:50:41.569 | INFO | "realtimeActivityLabels" : true, | 20260507 13:50:41.569 | INFO | "sandwichTolerance" : 0 | 20260507 13:50:41.569 | INFO | }, | 20260507 13:50:41.569 | INFO | "baselineStatsAggregatorConfig" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "Method" : "Gpu" | 20260507 13:50:41.569 | INFO | }, | 20260507 13:50:41.569 | INFO | "baselinerConfig" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "Filter" : "MultiScaleLarge", | 20260507 13:50:41.569 | INFO | "JumpSuppression" : 0, | 20260507 13:50:41.569 | INFO | "MeanBiasAdjust" : 0, | 20260507 13:50:41.569 | INFO | "MeanEmaScaleStrides" : 0, | 20260507 13:50:41.569 | INFO | "Method" : "DeviceMultiScale", | 20260507 13:50:41.569 | INFO | "SigmaBiasAdjust" : 0, | 20260507 13:50:41.569 | INFO | "SigmaEmaScaleStrides" : 512 | 20260507 13:50:41.569 | INFO | }, | 20260507 13:50:41.569 | INFO | "dmeConfig" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "AddPrelimEstimateConfidence" : true, | 20260507 13:50:41.569 | INFO | "AmplitudeFeedbackDampingFactor" : 0.89999997615814209, | 20260507 13:50:41.569 | INFO | "AmplitudeFeedbackInitSampleCount" : 2000, | 20260507 13:50:41.569 | INFO | "AnalogMixFractionThreshold" : | 20260507 13:50:41.569 | INFO | [ | 20260507 13:50:41.569 | INFO | 0.013000000268220901, | 20260507 13:50:41.569 | INFO | 0.039000000804662704 | 20260507 13:50:41.569 | INFO | ], | 20260507 13:50:41.569 | INFO | "ConfidenceHalfLife" : | 20260507 13:50:41.569 | INFO | [ | 20260507 13:50:41.569 | INFO | 4096, | 20260507 13:50:41.569 | INFO | 12000 | 20260507 13:50:41.569 | INFO | ], | 20260507 13:50:41.569 | INFO | "ConfidenceHalfLifePauseEnhance" : 0, | 20260507 13:50:41.569 | INFO | "ConvergeCoeff" : 2.9999999242136255e-05, | 20260507 13:50:41.569 | INFO | "EmIterationLimit" : 20, | 20260507 13:50:41.569 | INFO | "EnableAmplitudeRatioFeedback" : true, | 20260507 13:50:41.569 | INFO | "EnableFeedbackMultiloadAdjust" : false, | 20260507 13:50:41.569 | INFO | "EnablePulseVarianceFeedback" : true, | 20260507 13:50:41.569 | INFO | "FeedbackExcessNoiseCoeffDampFactor" : 19, | 20260507 13:50:41.569 | INFO | "FirstAttemptsIterMultiple" : 2, | 20260507 13:50:41.569 | INFO | "GTestStatFactor" : -1, | 20260507 13:50:41.569 | INFO | "GofLogChiSqrThresh1" : 111, | 20260507 13:50:41.569 | INFO | "GofLogChiSqrThresh2" : 8, | 20260507 13:50:41.569 | INFO | "HybridAtol" : 8, | 20260507 13:50:41.569 | INFO | "HybridRtol" : 48, | 20260507 13:50:41.569 | INFO | "IterateToLimit" : false, | 20260507 13:50:41.569 | INFO | "LaserPowerChangeReduceConfidence" : -1, | 20260507 13:50:41.569 | INFO | "MaxFeedbackVarRatio" : 2.5, | 20260507 13:50:41.569 | INFO | "Method" : "EmDevice", | 20260507 13:50:41.569 | INFO | "MinAnalogSnrThresh0" : 2, | 20260507 13:50:41.569 | INFO | "MinAnalogSnrThresh1" : 4, | 20260507 13:50:41.569 | INFO | "MinFeedbackBasesOverThreshold" : 4, | 20260507 13:50:41.569 | INFO | "MinFramesForEstimate" : 4000, | 20260507 13:50:41.569 | INFO | "ModelUpdateWeightMax" : 0.5, | 20260507 13:50:41.569 | INFO | "NumBinsMin" : 500, | 20260507 13:50:41.569 | INFO | "NumFramesPreAccumStats" : 1000, | 20260507 13:50:41.569 | INFO | "PulseAmpRegularization" : 0, | 20260507 13:50:41.569 | INFO | "PureUpdate" : false, | 20260507 13:50:41.569 | INFO | "RtalFeedbackLabelHistorySize" : 5, | 20260507 13:50:41.569 | INFO | "ScaleSnrConfTol" : 1, | 20260507 13:50:41.569 | INFO | "SimModel" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "baselineMean" : 200, | 20260507 13:50:41.569 | INFO | "baselineVar" : 33, | 20260507 13:50:41.569 | INFO | "useSimulatedBaselineParams" : false | 20260507 13:50:41.569 | INFO | }, | 20260507 13:50:41.569 | INFO | "SnrDropThresh" : 1, | 20260507 13:50:41.569 | INFO | "SuccessConfidenceThresh" : 0.10000000149011612, | 20260507 13:50:41.569 | INFO | "ThreshAnalogSNR" : 2.1489999294281006, | 20260507 13:50:41.569 | INFO | "ThreshFractionFeedbackPulses" : 0.12999999523162842, | 20260507 13:50:41.569 | INFO | "XsnArrayCVRatios" : | 20260507 13:50:41.569 | INFO | [ | 20260507 13:50:41.569 | INFO | 0, | 20260507 13:50:41.569 | INFO | 0, | 20260507 13:50:41.569 | INFO | 0, | 20260507 13:50:41.569 | INFO | 0 | 20260507 13:50:41.569 | INFO | ], | 20260507 13:50:41.569 | INFO | "XsnCoeffLstSqWeightPower" : 2, | 20260507 13:50:41.569 | INFO | "XsnCoeffRatioCeil" : 10, | 20260507 13:50:41.569 | INFO | "XsnCoeffRatioFloor" : 0.0099999997764825821 | 20260507 13:50:41.569 | INFO | }, | 20260507 13:50:41.569 | INFO | "frameLabelerConfig" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "Method" : "Device", | 20260507 13:50:41.569 | INFO | "roi" : | 20260507 13:50:41.569 | INFO | { | 20260507 13:50:41.569 | INFO | "filterType" : "Default", | 20260507 13:50:41.569 | INFO | "lowerThreshold" : 2, | 20260507 13:50:41.569 | INFO | "upperThreshold" : 6 | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "viterbi" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "alpha" : 1, | 20260507 13:50:41.570 | INFO | "beta" : 1, | 20260507 13:50:41.570 | INFO | "gamma" : 1 | 20260507 13:50:41.570 | INFO | } | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "modelEstimationMode" : "DynamicEstimations", | 20260507 13:50:41.570 | INFO | "pulseAccumConfig" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "Method" : "GpuPulses", | 20260507 13:50:41.570 | INFO | "XspAmpThresh" : 0.69999999999999996, | 20260507 13:50:41.570 | INFO | "XspWidthThresh" : 3.5, | 20260507 13:50:41.570 | INFO | "maxPulsePerSec" : 20, | 20260507 13:50:41.570 | INFO | "simConfig" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "basecalls" : | 20260507 13:50:41.570 | INFO | [ | 20260507 13:50:41.570 | INFO | "A", | 20260507 13:50:41.570 | INFO | "C", | 20260507 13:50:41.570 | INFO | "G", | 20260507 13:50:41.570 | INFO | "T" | 20260507 13:50:41.570 | INFO | ], | 20260507 13:50:41.570 | INFO | "ipdVars" : [], | 20260507 13:50:41.570 | INFO | "ipds" : | 20260507 13:50:41.570 | INFO | [ | 20260507 13:50:41.570 | INFO | 6 | 20260507 13:50:41.570 | INFO | ], | 20260507 13:50:41.570 | INFO | "maxSignals" : | 20260507 13:50:41.570 | INFO | [ | 20260507 13:50:41.570 | INFO | 21, | 20260507 13:50:41.570 | INFO | 11, | 20260507 13:50:41.570 | INFO | 17, | 20260507 13:50:41.570 | INFO | 9 | 20260507 13:50:41.570 | INFO | ], | 20260507 13:50:41.570 | INFO | "maxSignalsVars" : [], | 20260507 13:50:41.570 | INFO | "meanSignals" : | 20260507 13:50:41.570 | INFO | [ | 20260507 13:50:41.570 | INFO | 20, | 20260507 13:50:41.570 | INFO | 10, | 20260507 13:50:41.570 | INFO | 16, | 20260507 13:50:41.570 | INFO | 8 | 20260507 13:50:41.570 | INFO | ], | 20260507 13:50:41.570 | INFO | "meanSignalsVars" : [], | 20260507 13:50:41.570 | INFO | "midSignals" : | 20260507 13:50:41.570 | INFO | [ | 20260507 13:50:41.570 | INFO | 21, | 20260507 13:50:41.570 | INFO | 11, | 20260507 13:50:41.570 | INFO | 17, | 20260507 13:50:41.570 | INFO | 9 | 20260507 13:50:41.570 | INFO | ], | 20260507 13:50:41.570 | INFO | "midSignalsVars" : [], | 20260507 13:50:41.570 | INFO | "pwVars" : [], | 20260507 13:50:41.570 | INFO | "pws" : | 20260507 13:50:41.570 | INFO | [ | 20260507 13:50:41.570 | INFO | 5 | 20260507 13:50:41.570 | INFO | ], | 20260507 13:50:41.570 | INFO | "seed" : 0 | 20260507 13:50:41.570 | INFO | } | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "pulseSerlzConfig" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "Method" : "GpuPulses", | 20260507 13:50:41.570 | INFO | "PulseCountCoeff" : 2 | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "simulatedFaults" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "negativeDyeSpectrumCounter" : 0 | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "staticDetModelConfig" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "baselineMean" : 0, | 20260507 13:50:41.570 | INFO | "baselineVariance" : 33 | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "traceHistogramConfig" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "BaselineStatMinFrameCount" : 50, | 20260507 13:50:41.570 | INFO | "BinSizeCoeff" : 0.20000000298023224, | 20260507 13:50:41.570 | INFO | "BinSizeLowBoundCoeff" : 1, | 20260507 13:50:41.570 | INFO | "FallBackBaselineSigma" : 10, | 20260507 13:50:41.570 | INFO | "Method" : "Gpu" | 20260507 13:50:41.570 | INFO | } | 20260507 13:50:41.570 | INFO | }, | 20260507 13:50:41.570 | INFO | "bazIO" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.570 | INFO | "BufferSizeMiBytes" : 4, | 20260507 13:50:41.570 | INFO | "MaxMiBPerSecondWrite" : 450, | 20260507 13:50:41.570 | INFO | "MaxMiBToWrite" : 12874602, | 20260507 13:50:41.570 | INFO | "SyncSizeMiBytes" : 32, | 20260507 13:50:41.570 | INFO | "compression" : | 20260507 13:50:41.570 | INFO | { | 20260507 13:50:41.571 | INFO | "blockSize" : 131072, | 20260507 13:50:41.571 | INFO | "enabled" : true, | 20260507 13:50:41.571 | INFO | "level" : 1 | 20260507 13:50:41.571 | INFO | } | 20260507 13:50:41.571 | INFO | }, | 20260507 13:50:41.571 | INFO | "bazWriter" : | 20260507 13:50:41.571 | INFO | { | 20260507 13:50:41.571 | INFO | "maxChunksQueued" : 2, | 20260507 13:50:41.571 | INFO | "samplingParams" : | 20260507 13:50:41.571 | INFO | { | 20260507 13:50:41.571 | INFO | "filenames" : | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | "/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1_basic_preview.baz", | 20260507 13:50:41.571 | INFO | "/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1_full_preview.baz" | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | "presets" : | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | "BasicPreview", | 20260507 13:50:41.571 | INFO | "FullPreview" | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | "roiList" : | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 384, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 640, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 896, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 1152, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 1408, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 1664, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 1920, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 2176, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 2432, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 2688, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.571 | INFO | 64 | 20260507 13:50:41.571 | INFO | ], | 20260507 13:50:41.571 | INFO | [ | 20260507 13:50:41.571 | INFO | 2944, | 20260507 13:50:41.571 | INFO | 128, | 20260507 13:50:41.571 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 3200, | 20260507 13:50:41.572 | INFO | 128, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 3456, | 20260507 13:50:41.572 | INFO | 128, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 3712, | 20260507 13:50:41.572 | INFO | 128, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 3968, | 20260507 13:50:41.572 | INFO | 128, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 128, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 640, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 896, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 1152, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 1408, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 1664, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 1920, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 2176, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 2432, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.572 | INFO | 1, | 20260507 13:50:41.572 | INFO | 64 | 20260507 13:50:41.572 | INFO | ], | 20260507 13:50:41.572 | INFO | [ | 20260507 13:50:41.572 | INFO | 2688, | 20260507 13:50:41.572 | INFO | 384, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 2944, | 20260507 13:50:41.573 | INFO | 384, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 3200, | 20260507 13:50:41.573 | INFO | 384, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 3456, | 20260507 13:50:41.573 | INFO | 384, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 3712, | 20260507 13:50:41.573 | INFO | 384, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 3968, | 20260507 13:50:41.573 | INFO | 384, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 128, | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 384, | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 896, | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 1152, | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.573 | INFO | 1408, | 20260507 13:50:41.573 | INFO | 640, | 20260507 13:50:41.573 | INFO | 1, | 20260507 13:50:41.573 | INFO | 64 | 20260507 13:50:41.573 | INFO | ], | 20260507 13:50:41.573 | INFO | [ | 20260507 13:50:41.574 | INFO | 1664, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 1920, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 2176, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 2432, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 2688, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 2944, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 3200, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 3456, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 3712, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 3968, | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 128, | 20260507 13:50:41.574 | INFO | 896, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 384, | 20260507 13:50:41.574 | INFO | 896, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 640, | 20260507 13:50:41.574 | INFO | 896, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 896, | 20260507 13:50:41.574 | INFO | 896, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.574 | INFO | [ | 20260507 13:50:41.574 | INFO | 1152, | 20260507 13:50:41.574 | INFO | 896, | 20260507 13:50:41.574 | INFO | 1, | 20260507 13:50:41.574 | INFO | 64 | 20260507 13:50:41.574 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 1408, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 1664, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 1920, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 2176, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 2432, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 2688, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 2944, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 3200, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 3456, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 3712, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 3968, | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 128, | 20260507 13:50:41.575 | INFO | 1152, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 384, | 20260507 13:50:41.575 | INFO | 1152, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 640, | 20260507 13:50:41.575 | INFO | 1152, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 896, | 20260507 13:50:41.575 | INFO | 1152, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.575 | INFO | ], | 20260507 13:50:41.575 | INFO | [ | 20260507 13:50:41.575 | INFO | 1152, | 20260507 13:50:41.575 | INFO | 1152, | 20260507 13:50:41.575 | INFO | 1, | 20260507 13:50:41.575 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 1408, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 1664, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 1920, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 2176, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 2432, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 2688, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 2944, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 3200, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 3456, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 3712, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 3968, | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 128, | 20260507 13:50:41.576 | INFO | 1408, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 384, | 20260507 13:50:41.576 | INFO | 1408, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 640, | 20260507 13:50:41.576 | INFO | 1408, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 896, | 20260507 13:50:41.576 | INFO | 1408, | 20260507 13:50:41.576 | INFO | 1, | 20260507 13:50:41.576 | INFO | 64 | 20260507 13:50:41.576 | INFO | ], | 20260507 13:50:41.576 | INFO | [ | 20260507 13:50:41.576 | INFO | 1152, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 1664, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 1920, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 2176, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 2432, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 2688, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 2944, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 3200, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 3456, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 3712, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 3968, | 20260507 13:50:41.577 | INFO | 1408, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 128, | 20260507 13:50:41.577 | INFO | 1664, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 384, | 20260507 13:50:41.577 | INFO | 1664, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 640, | 20260507 13:50:41.577 | INFO | 1664, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.577 | INFO | 896, | 20260507 13:50:41.577 | INFO | 1664, | 20260507 13:50:41.577 | INFO | 1, | 20260507 13:50:41.577 | INFO | 64 | 20260507 13:50:41.577 | INFO | ], | 20260507 13:50:41.577 | INFO | [ | 20260507 13:50:41.578 | INFO | 1152, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 1408, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 1920, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 2176, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 2432, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 2688, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 2944, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 3200, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 3456, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 3712, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 3968, | 20260507 13:50:41.578 | INFO | 1664, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 128, | 20260507 13:50:41.578 | INFO | 1920, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 384, | 20260507 13:50:41.578 | INFO | 1920, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 640, | 20260507 13:50:41.578 | INFO | 1920, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.578 | INFO | [ | 20260507 13:50:41.578 | INFO | 896, | 20260507 13:50:41.578 | INFO | 1920, | 20260507 13:50:41.578 | INFO | 1, | 20260507 13:50:41.578 | INFO | 64 | 20260507 13:50:41.578 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 1152, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 1408, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 1664, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 2176, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 2432, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 2688, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 2944, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 3200, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 3456, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 3712, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 3968, | 20260507 13:50:41.579 | INFO | 1920, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 128, | 20260507 13:50:41.579 | INFO | 2176, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 384, | 20260507 13:50:41.579 | INFO | 2176, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.579 | INFO | 640, | 20260507 13:50:41.579 | INFO | 2176, | 20260507 13:50:41.579 | INFO | 1, | 20260507 13:50:41.579 | INFO | 64 | 20260507 13:50:41.579 | INFO | ], | 20260507 13:50:41.579 | INFO | [ | 20260507 13:50:41.580 | INFO | 896, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 1152, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 1408, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 1664, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 1920, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 2432, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 2688, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 2944, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 3200, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 3456, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 3712, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 3968, | 20260507 13:50:41.580 | INFO | 2176, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 128, | 20260507 13:50:41.580 | INFO | 2432, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 384, | 20260507 13:50:41.580 | INFO | 2432, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.580 | INFO | 64 | 20260507 13:50:41.580 | INFO | ], | 20260507 13:50:41.580 | INFO | [ | 20260507 13:50:41.580 | INFO | 640, | 20260507 13:50:41.580 | INFO | 2432, | 20260507 13:50:41.580 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 896, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 1152, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 1408, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 1664, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 1920, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 2176, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 2688, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 2944, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 3200, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 3456, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 3712, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 3968, | 20260507 13:50:41.581 | INFO | 2432, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 128, | 20260507 13:50:41.581 | INFO | 2688, | 20260507 13:50:41.581 | INFO | 1, | 20260507 13:50:41.581 | INFO | 64 | 20260507 13:50:41.581 | INFO | ], | 20260507 13:50:41.581 | INFO | [ | 20260507 13:50:41.581 | INFO | 384, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 640, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 896, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 1152, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 1408, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 1664, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 1920, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 2176, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 2432, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 2944, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 3200, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 3456, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 3712, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 3968, | 20260507 13:50:41.582 | INFO | 2688, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.582 | INFO | 64 | 20260507 13:50:41.582 | INFO | ], | 20260507 13:50:41.582 | INFO | [ | 20260507 13:50:41.582 | INFO | 128, | 20260507 13:50:41.582 | INFO | 2944, | 20260507 13:50:41.582 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 384, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 640, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 896, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 1152, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 1408, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 1664, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 1920, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 2176, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 2432, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 2688, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 3200, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 3456, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 3712, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.583 | INFO | 3968, | 20260507 13:50:41.583 | INFO | 2944, | 20260507 13:50:41.583 | INFO | 1, | 20260507 13:50:41.583 | INFO | 64 | 20260507 13:50:41.583 | INFO | ], | 20260507 13:50:41.583 | INFO | [ | 20260507 13:50:41.584 | INFO | 128, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 384, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 640, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 896, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 1152, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 1408, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 1664, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 1920, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 2176, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 2432, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 2688, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 2944, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 3456, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 3712, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.584 | INFO | [ | 20260507 13:50:41.584 | INFO | 3968, | 20260507 13:50:41.584 | INFO | 3200, | 20260507 13:50:41.584 | INFO | 1, | 20260507 13:50:41.584 | INFO | 64 | 20260507 13:50:41.584 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 128, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 384, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 640, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 896, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 1152, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 1408, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 1664, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 1920, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 2176, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 2432, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 2688, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 2944, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 3200, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.585 | INFO | 3712, | 20260507 13:50:41.585 | INFO | 3456, | 20260507 13:50:41.585 | INFO | 1, | 20260507 13:50:41.585 | INFO | 64 | 20260507 13:50:41.585 | INFO | ], | 20260507 13:50:41.585 | INFO | [ | 20260507 13:50:41.586 | INFO | 3968, | 20260507 13:50:41.586 | INFO | 3456, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 128, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 384, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 640, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 896, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 1152, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 1408, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 1664, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 1920, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 2176, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 2432, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 2688, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 2944, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 3200, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 3456, | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.586 | INFO | 1, | 20260507 13:50:41.586 | INFO | 64 | 20260507 13:50:41.586 | INFO | ], | 20260507 13:50:41.586 | INFO | [ | 20260507 13:50:41.586 | INFO | 3712, | 20260507 13:50:41.587 | INFO | 3712, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 3712, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 128, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 384, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 640, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 896, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 1152, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 1408, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 1664, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 1920, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 2176, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 2432, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 2688, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 2944, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 3200, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.587 | INFO | ], | 20260507 13:50:41.587 | INFO | [ | 20260507 13:50:41.587 | INFO | 3456, | 20260507 13:50:41.587 | INFO | 3968, | 20260507 13:50:41.587 | INFO | 1, | 20260507 13:50:41.587 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 3712, | 20260507 13:50:41.588 | INFO | 3968, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 3968, | 20260507 13:50:41.588 | INFO | 3968, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 128, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 384, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 640, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 896, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 1152, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 1408, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 1664, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 1920, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 2176, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 2432, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 2688, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 2944, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 3200, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.588 | INFO | 1, | 20260507 13:50:41.588 | INFO | 64 | 20260507 13:50:41.588 | INFO | ], | 20260507 13:50:41.588 | INFO | [ | 20260507 13:50:41.588 | INFO | 3456, | 20260507 13:50:41.588 | INFO | 4224, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 3712, | 20260507 13:50:41.589 | INFO | 4224, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 3968, | 20260507 13:50:41.589 | INFO | 4224, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 128, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 384, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 640, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 896, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 1152, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 1408, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 1664, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 1920, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 2176, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 2432, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 2688, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 2944, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.589 | INFO | 3200, | 20260507 13:50:41.589 | INFO | 4480, | 20260507 13:50:41.589 | INFO | 1, | 20260507 13:50:41.589 | INFO | 64 | 20260507 13:50:41.589 | INFO | ], | 20260507 13:50:41.589 | INFO | [ | 20260507 13:50:41.590 | INFO | 3456, | 20260507 13:50:41.590 | INFO | 4480, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 3712, | 20260507 13:50:41.590 | INFO | 4480, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 3968, | 20260507 13:50:41.590 | INFO | 4480, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 128, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 384, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 640, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 896, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 1152, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 1408, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 1664, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 1920, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 2176, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 2432, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 2688, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 2944, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.590 | INFO | [ | 20260507 13:50:41.590 | INFO | 3200, | 20260507 13:50:41.590 | INFO | 4736, | 20260507 13:50:41.590 | INFO | 1, | 20260507 13:50:41.590 | INFO | 64 | 20260507 13:50:41.590 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 3456, | 20260507 13:50:41.591 | INFO | 4736, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 3712, | 20260507 13:50:41.591 | INFO | 4736, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 3968, | 20260507 13:50:41.591 | INFO | 4736, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 128, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 384, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 640, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 896, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 1152, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 1408, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 1664, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 1920, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 2176, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 2432, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 2688, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 2944, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.591 | INFO | ], | 20260507 13:50:41.591 | INFO | [ | 20260507 13:50:41.591 | INFO | 3200, | 20260507 13:50:41.591 | INFO | 4992, | 20260507 13:50:41.591 | INFO | 1, | 20260507 13:50:41.591 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 3456, | 20260507 13:50:41.592 | INFO | 4992, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 3712, | 20260507 13:50:41.592 | INFO | 4992, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 3968, | 20260507 13:50:41.592 | INFO | 4992, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 128, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 384, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 640, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 896, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 1152, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 1408, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 1664, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 1920, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 2176, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 2432, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 2688, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.592 | INFO | [ | 20260507 13:50:41.592 | INFO | 2944, | 20260507 13:50:41.592 | INFO | 5248, | 20260507 13:50:41.592 | INFO | 1, | 20260507 13:50:41.592 | INFO | 64 | 20260507 13:50:41.592 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 3200, | 20260507 13:50:41.593 | INFO | 5248, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 3456, | 20260507 13:50:41.593 | INFO | 5248, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 3712, | 20260507 13:50:41.593 | INFO | 5248, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 3968, | 20260507 13:50:41.593 | INFO | 5248, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 128, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 384, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 640, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 896, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 1152, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 1408, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 1664, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 1920, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 2176, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 2432, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.593 | INFO | [ | 20260507 13:50:41.593 | INFO | 2688, | 20260507 13:50:41.593 | INFO | 5504, | 20260507 13:50:41.593 | INFO | 1, | 20260507 13:50:41.593 | INFO | 64 | 20260507 13:50:41.593 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 2944, | 20260507 13:50:41.594 | INFO | 5504, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 3200, | 20260507 13:50:41.594 | INFO | 5504, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 3456, | 20260507 13:50:41.594 | INFO | 5504, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 3712, | 20260507 13:50:41.594 | INFO | 5504, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 3968, | 20260507 13:50:41.594 | INFO | 5504, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 128, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 384, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 640, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 896, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 1152, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 1408, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 1664, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 1920, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 2176, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 2432, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.594 | INFO | 1, | 20260507 13:50:41.594 | INFO | 64 | 20260507 13:50:41.594 | INFO | ], | 20260507 13:50:41.594 | INFO | [ | 20260507 13:50:41.594 | INFO | 2688, | 20260507 13:50:41.594 | INFO | 5760, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 2944, | 20260507 13:50:41.595 | INFO | 5760, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 3200, | 20260507 13:50:41.595 | INFO | 5760, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 3456, | 20260507 13:50:41.595 | INFO | 5760, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 3712, | 20260507 13:50:41.595 | INFO | 5760, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 3968, | 20260507 13:50:41.595 | INFO | 5760, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 128, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 384, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 640, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 896, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 1152, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 1408, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 1664, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 1920, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 2176, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.595 | INFO | 64 | 20260507 13:50:41.595 | INFO | ], | 20260507 13:50:41.595 | INFO | [ | 20260507 13:50:41.595 | INFO | 2432, | 20260507 13:50:41.595 | INFO | 6016, | 20260507 13:50:41.595 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 2688, | 20260507 13:50:41.596 | INFO | 6016, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 2944, | 20260507 13:50:41.596 | INFO | 6016, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 3200, | 20260507 13:50:41.596 | INFO | 6016, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 3456, | 20260507 13:50:41.596 | INFO | 6016, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 3712, | 20260507 13:50:41.596 | INFO | 6016, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 3968, | 20260507 13:50:41.596 | INFO | 6016, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ] | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 409, | 20260507 13:50:41.596 | INFO | 384, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 1228, | 20260507 13:50:41.596 | INFO | 384, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 2047, | 20260507 13:50:41.596 | INFO | 384, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 2866, | 20260507 13:50:41.596 | INFO | 384, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 3685, | 20260507 13:50:41.596 | INFO | 384, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 409, | 20260507 13:50:41.596 | INFO | 1152, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 1228, | 20260507 13:50:41.596 | INFO | 1152, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 2047, | 20260507 13:50:41.596 | INFO | 1152, | 20260507 13:50:41.596 | INFO | 1, | 20260507 13:50:41.596 | INFO | 64 | 20260507 13:50:41.596 | INFO | ], | 20260507 13:50:41.596 | INFO | [ | 20260507 13:50:41.596 | INFO | 2866, | 20260507 13:50:41.596 | INFO | 1152, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 3685, | 20260507 13:50:41.597 | INFO | 1152, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 409, | 20260507 13:50:41.597 | INFO | 1920, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 1228, | 20260507 13:50:41.597 | INFO | 1920, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 2047, | 20260507 13:50:41.597 | INFO | 1920, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 2866, | 20260507 13:50:41.597 | INFO | 1920, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 3685, | 20260507 13:50:41.597 | INFO | 1920, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 409, | 20260507 13:50:41.597 | INFO | 2688, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 1228, | 20260507 13:50:41.597 | INFO | 2688, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 2047, | 20260507 13:50:41.597 | INFO | 2688, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 2866, | 20260507 13:50:41.597 | INFO | 2688, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 3685, | 20260507 13:50:41.597 | INFO | 2688, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 409, | 20260507 13:50:41.597 | INFO | 3456, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 1228, | 20260507 13:50:41.597 | INFO | 3456, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 2047, | 20260507 13:50:41.597 | INFO | 3456, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.597 | INFO | 2866, | 20260507 13:50:41.597 | INFO | 3456, | 20260507 13:50:41.597 | INFO | 1, | 20260507 13:50:41.597 | INFO | 64 | 20260507 13:50:41.597 | INFO | ], | 20260507 13:50:41.597 | INFO | [ | 20260507 13:50:41.598 | INFO | 3685, | 20260507 13:50:41.598 | INFO | 3456, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 409, | 20260507 13:50:41.598 | INFO | 4224, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 1228, | 20260507 13:50:41.598 | INFO | 4224, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 2047, | 20260507 13:50:41.598 | INFO | 4224, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 2866, | 20260507 13:50:41.598 | INFO | 4224, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 3685, | 20260507 13:50:41.598 | INFO | 4224, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 409, | 20260507 13:50:41.598 | INFO | 4992, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 1228, | 20260507 13:50:41.598 | INFO | 4992, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 2047, | 20260507 13:50:41.598 | INFO | 4992, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 2866, | 20260507 13:50:41.598 | INFO | 4992, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 3685, | 20260507 13:50:41.598 | INFO | 4992, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 409, | 20260507 13:50:41.598 | INFO | 5760, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 1228, | 20260507 13:50:41.598 | INFO | 5760, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.598 | INFO | 64 | 20260507 13:50:41.598 | INFO | ], | 20260507 13:50:41.598 | INFO | [ | 20260507 13:50:41.598 | INFO | 2047, | 20260507 13:50:41.598 | INFO | 5760, | 20260507 13:50:41.598 | INFO | 1, | 20260507 13:50:41.599 | INFO | 64 | 20260507 13:50:41.599 | INFO | ], | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 2866, | 20260507 13:50:41.599 | INFO | 5760, | 20260507 13:50:41.599 | INFO | 1, | 20260507 13:50:41.599 | INFO | 64 | 20260507 13:50:41.599 | INFO | ], | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 3685, | 20260507 13:50:41.599 | INFO | 5760, | 20260507 13:50:41.599 | INFO | 1, | 20260507 13:50:41.599 | INFO | 64 | 20260507 13:50:41.599 | INFO | ] | 20260507 13:50:41.599 | INFO | ] | 20260507 13:50:41.599 | INFO | ] | 20260507 13:50:41.599 | INFO | } | 20260507 13:50:41.599 | INFO | }, | 20260507 13:50:41.599 | INFO | "dataSource" : | 20260507 13:50:41.599 | INFO | { | 20260507 13:50:41.599 | INFO | "crosstalkFilterKernel" : [], | 20260507 13:50:41.599 | INFO | "darkCalFileName" : "/data/icc/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.darkcal.h5", | 20260507 13:50:41.599 | INFO | "imagePsfKernel" : | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 0.00080000000000000004, | 20260507 13:50:41.599 | INFO | 0.0030000000000000001, | 20260507 13:50:41.599 | INFO | 0.0166, | 20260507 13:50:41.599 | INFO | 0.0035000000000000001, | 20260507 13:50:41.599 | INFO | 0.0011000000000000001 | 20260507 13:50:41.599 | INFO | ], | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 0.0027000000000000001, | 20260507 13:50:41.599 | INFO | 0.012999999999999999, | 20260507 13:50:41.599 | INFO | 0.072700000000000001, | 20260507 13:50:41.599 | INFO | 0.014, | 20260507 13:50:41.599 | INFO | 0.0032000000000000002 | 20260507 13:50:41.599 | INFO | ], | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 0.0118, | 20260507 13:50:41.599 | INFO | 0.0545, | 20260507 13:50:41.599 | INFO | 0.59440000000000004, | 20260507 13:50:41.599 | INFO | 0.065299999999999997, | 20260507 13:50:41.599 | INFO | 0.0124 | 20260507 13:50:41.599 | INFO | ], | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 0.0032000000000000002, | 20260507 13:50:41.599 | INFO | 0.013100000000000001, | 20260507 13:50:41.599 | INFO | 0.072700000000000001, | 20260507 13:50:41.599 | INFO | 0.0141, | 20260507 13:50:41.599 | INFO | 0.0030999999999999999 | 20260507 13:50:41.599 | INFO | ], | 20260507 13:50:41.599 | INFO | [ | 20260507 13:50:41.599 | INFO | 0.001, | 20260507 13:50:41.599 | INFO | 0.0028999999999999998, | 20260507 13:50:41.599 | INFO | 0.016799999999999999, | 20260507 13:50:41.599 | INFO | 0.0033999999999999998, | 20260507 13:50:41.599 | INFO | 0.00089999999999999998 | 20260507 13:50:41.599 | INFO | ] | 20260507 13:50:41.599 | INFO | ] | 20260507 13:50:41.599 | INFO | }, | 20260507 13:50:41.599 | INFO | "internalMode" : false, | 20260507 13:50:41.599 | INFO | "layout" : | 20260507 13:50:41.599 | INFO | { | 20260507 13:50:41.599 | INFO | "framesPerChunk" : 512, | 20260507 13:50:41.599 | INFO | "lanesPerPool" : 4096, | 20260507 13:50:41.599 | INFO | "zmwsPerLane" : 64 | 20260507 13:50:41.599 | INFO | }, | 20260507 13:50:41.599 | INFO | "maskFraction" : 1, | 20260507 13:50:41.599 | INFO | "monitoringReportInterval" : 8192, | 20260507 13:50:41.599 | INFO | "multipleBazFiles" : true, | 20260507 13:50:41.599 | INFO | "prelimHQ" : | 20260507 13:50:41.599 | INFO | { | 20260507 13:50:41.599 | INFO | "enablePreHQ" : true, | 20260507 13:50:41.599 | INFO | "expectedFrameRate" : 100, | 20260507 13:50:41.599 | INFO | "expectedPulsesPerZmw" : 164, | 20260507 13:50:41.599 | INFO | "fracMultiBlocks" : 0.20000000298023224, | 20260507 13:50:41.599 | INFO | "fracSingleBlocks" : 0.5, | 20260507 13:50:41.599 | INFO | "hqBufferGrowthFactor" : 2, | 20260507 13:50:41.599 | INFO | "hqThrottleFraction" : 1, | 20260507 13:50:41.599 | INFO | "maxPreHQBuffers" : 1, | 20260507 13:50:41.599 | INFO | "maxSustainedPulseRate" : 4, | 20260507 13:50:41.599 | INFO | "preHQBufferGrowthFactor" : 1, | 20260507 13:50:41.599 | INFO | "requiredActivityLabels" : 10, | 20260507 13:50:41.599 | INFO | "zmwOutputStride" : 1 | 20260507 13:50:41.599 | INFO | }, | 20260507 13:50:41.599 | INFO | "realTimeMetrics" : | 20260507 13:50:41.599 | INFO | { | 20260507 13:50:41.599 | INFO | "csvOutputFile" : "", | 20260507 13:50:41.599 | INFO | "diagOutputFile" : "", | 20260507 13:50:41.599 | INFO | "jsonOutputFile" : "/data/nrta/2/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.rtmetrics.json", | 20260507 13:50:41.599 | INFO | "minConfidenceScore" : 0, | 20260507 13:50:41.600 | INFO | "regions" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | { | 20260507 13:50:41.600 | INFO | "featuresForFilter" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "PorSequencing" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "medianIntraLaneStride" : 8, | 20260507 13:50:41.600 | INFO | "metrics" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "PulseRate", | 20260507 13:50:41.600 | INFO | "PulseWidth", | 20260507 13:50:41.600 | INFO | "BaseRate", | 20260507 13:50:41.600 | INFO | "BaseWidth", | 20260507 13:50:41.600 | INFO | "Baseline", | 20260507 13:50:41.600 | INFO | "BaselineStd", | 20260507 13:50:41.600 | INFO | "Pkmid", | 20260507 13:50:41.600 | INFO | "Snr" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.600 | INFO | "name" : "FullChip", | 20260507 13:50:41.600 | INFO | "roi" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | 0, | 20260507 13:50:41.600 | INFO | 0, | 20260507 13:50:41.600 | INFO | 4096, | 20260507 13:50:41.600 | INFO | 6144 | 20260507 13:50:41.600 | INFO | ] | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.600 | INFO | }, | 20260507 13:50:41.600 | INFO | { | 20260507 13:50:41.600 | INFO | "featuresForFilter" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "PorSequencing" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.600 | INFO | "metrics" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "Baseline", | 20260507 13:50:41.600 | INFO | "BaselineStd", | 20260507 13:50:41.600 | INFO | "Pkmid", | 20260507 13:50:41.600 | INFO | "Snr" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.600 | INFO | "name" : "TopStrip", | 20260507 13:50:41.600 | INFO | "roi" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | 200, | 20260507 13:50:41.600 | INFO | 64, | 20260507 13:50:41.600 | INFO | 64, | 20260507 13:50:41.600 | INFO | 6016 | 20260507 13:50:41.600 | INFO | ] | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.600 | INFO | }, | 20260507 13:50:41.600 | INFO | { | 20260507 13:50:41.600 | INFO | "featuresForFilter" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "PorSequencing" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.600 | INFO | "metrics" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "Baseline", | 20260507 13:50:41.600 | INFO | "BaselineStd", | 20260507 13:50:41.600 | INFO | "Pkmid", | 20260507 13:50:41.600 | INFO | "Snr" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.600 | INFO | "name" : "MidStrip", | 20260507 13:50:41.600 | INFO | "roi" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | 2016, | 20260507 13:50:41.600 | INFO | 64, | 20260507 13:50:41.600 | INFO | 64, | 20260507 13:50:41.600 | INFO | 6016 | 20260507 13:50:41.600 | INFO | ] | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.600 | INFO | }, | 20260507 13:50:41.600 | INFO | { | 20260507 13:50:41.600 | INFO | "featuresForFilter" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "PorSequencing" | 20260507 13:50:41.600 | INFO | ], | 20260507 13:50:41.600 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.600 | INFO | "metrics" : | 20260507 13:50:41.600 | INFO | [ | 20260507 13:50:41.600 | INFO | "Baseline", | 20260507 13:50:41.600 | INFO | "BaselineStd", | 20260507 13:50:41.600 | INFO | "Pkmid", | 20260507 13:50:41.601 | INFO | "Snr" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.601 | INFO | "name" : "BotStrip", | 20260507 13:50:41.601 | INFO | "roi" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | 3832, | 20260507 13:50:41.601 | INFO | 64, | 20260507 13:50:41.601 | INFO | 64, | 20260507 13:50:41.601 | INFO | 6016 | 20260507 13:50:41.601 | INFO | ] | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.601 | INFO | }, | 20260507 13:50:41.601 | INFO | { | 20260507 13:50:41.601 | INFO | "featuresForFilter" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | "LaserPower2p0x" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.601 | INFO | "metrics" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | "Baseline", | 20260507 13:50:41.601 | INFO | "BaselineStd", | 20260507 13:50:41.601 | INFO | "Pkmid", | 20260507 13:50:41.601 | INFO | "Snr" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.601 | INFO | "name" : "FullChip_LP2P0X", | 20260507 13:50:41.601 | INFO | "roi" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | 0, | 20260507 13:50:41.601 | INFO | 0, | 20260507 13:50:41.601 | INFO | 4096, | 20260507 13:50:41.601 | INFO | 6144 | 20260507 13:50:41.601 | INFO | ] | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.601 | INFO | }, | 20260507 13:50:41.601 | INFO | { | 20260507 13:50:41.601 | INFO | "featuresForFilter" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | "LaserPower1p5x" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.601 | INFO | "metrics" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | "Baseline", | 20260507 13:50:41.601 | INFO | "BaselineStd", | 20260507 13:50:41.601 | INFO | "Pkmid", | 20260507 13:50:41.601 | INFO | "Snr" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.601 | INFO | "name" : "FullChip_LP1P5X", | 20260507 13:50:41.601 | INFO | "roi" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | 0, | 20260507 13:50:41.601 | INFO | 0, | 20260507 13:50:41.601 | INFO | 4096, | 20260507 13:50:41.601 | INFO | 6144 | 20260507 13:50:41.601 | INFO | ] | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.601 | INFO | }, | 20260507 13:50:41.601 | INFO | { | 20260507 13:50:41.601 | INFO | "featuresForFilter" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | "LaserPower0p5x" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.601 | INFO | "metrics" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | "Baseline", | 20260507 13:50:41.601 | INFO | "BaselineStd", | 20260507 13:50:41.601 | INFO | "Pkmid", | 20260507 13:50:41.601 | INFO | "Snr" | 20260507 13:50:41.601 | INFO | ], | 20260507 13:50:41.601 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.601 | INFO | "name" : "FullChip_LP0P5X", | 20260507 13:50:41.601 | INFO | "roi" : | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | [ | 20260507 13:50:41.601 | INFO | 0, | 20260507 13:50:41.601 | INFO | 0, | 20260507 13:50:41.601 | INFO | 4096, | 20260507 13:50:41.601 | INFO | 6144 | 20260507 13:50:41.601 | INFO | ] | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.602 | INFO | }, | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "featuresForFilter" : | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | "LaserPower0p0x" | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.602 | INFO | "metrics" : | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | "Baseline", | 20260507 13:50:41.602 | INFO | "BaselineStd", | 20260507 13:50:41.602 | INFO | "Pkmid", | 20260507 13:50:41.602 | INFO | "Snr" | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.602 | INFO | "name" : "FullChip_LP0P0X", | 20260507 13:50:41.602 | INFO | "roi" : | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | 0, | 20260507 13:50:41.602 | INFO | 0, | 20260507 13:50:41.602 | INFO | 4096, | 20260507 13:50:41.602 | INFO | 6144 | 20260507 13:50:41.602 | INFO | ] | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "useSingleActivityLabels" : true | 20260507 13:50:41.602 | INFO | }, | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "featuresForFilter" : | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | "LaserScatter" | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "medianIntraLaneStride" : 1, | 20260507 13:50:41.602 | INFO | "metrics" : | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | "Baseline", | 20260507 13:50:41.602 | INFO | "BaselineStd" | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "minSampleSize" : 1000, | 20260507 13:50:41.602 | INFO | "name" : "FullChip_LS", | 20260507 13:50:41.602 | INFO | "roi" : | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | [ | 20260507 13:50:41.602 | INFO | 0, | 20260507 13:50:41.602 | INFO | 0, | 20260507 13:50:41.602 | INFO | 4096, | 20260507 13:50:41.602 | INFO | 6144 | 20260507 13:50:41.602 | INFO | ] | 20260507 13:50:41.602 | INFO | ], | 20260507 13:50:41.602 | INFO | "useSingleActivityLabels" : false | 20260507 13:50:41.602 | INFO | } | 20260507 13:50:41.602 | INFO | ] | 20260507 13:50:41.602 | INFO | }, | 20260507 13:50:41.602 | INFO | "reportChecking" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "activeFractionThreshold" : 0.97999999999999998, | 20260507 13:50:41.602 | INFO | "growingQueueSizeThreshold" : 2, | 20260507 13:50:41.602 | INFO | "queueSizeThreshold" : 20, | 20260507 13:50:41.602 | INFO | "totalTimeMsThreshold" : 1000000 | 20260507 13:50:41.602 | INFO | }, | 20260507 13:50:41.602 | INFO | "smrtMask" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "smrtMaskScoreFilePath" : "" | 20260507 13:50:41.602 | INFO | }, | 20260507 13:50:41.602 | INFO | "source" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "WXIPCDataSourceConfig" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "acqConfig" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "A" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "baseLabel" : "A", | 20260507 13:50:41.602 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260507 13:50:41.602 | INFO | "interPulseDistance" : 0.079999998211860657, | 20260507 13:50:41.602 | INFO | "ipd2SlowStepRatio" : 0, | 20260507 13:50:41.602 | INFO | "pulseWidth" : 0.16599999368190765, | 20260507 13:50:41.602 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260507 13:50:41.602 | INFO | "relAmplitude" : 0.67000001668930054 | 20260507 13:50:41.602 | INFO | }, | 20260507 13:50:41.602 | INFO | "C" : | 20260507 13:50:41.602 | INFO | { | 20260507 13:50:41.602 | INFO | "baseLabel" : "C", | 20260507 13:50:41.602 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260507 13:50:41.603 | INFO | "interPulseDistance" : 0.070000000298023224, | 20260507 13:50:41.603 | INFO | "ipd2SlowStepRatio" : 0, | 20260507 13:50:41.603 | INFO | "pulseWidth" : 0.20900000631809235, | 20260507 13:50:41.603 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260507 13:50:41.603 | INFO | "relAmplitude" : 1 | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "G" : | 20260507 13:50:41.603 | INFO | { | 20260507 13:50:41.603 | INFO | "baseLabel" : "G", | 20260507 13:50:41.603 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260507 13:50:41.603 | INFO | "interPulseDistance" : 0.070000000298023224, | 20260507 13:50:41.603 | INFO | "ipd2SlowStepRatio" : 0, | 20260507 13:50:41.603 | INFO | "pulseWidth" : 0.19300000369548798, | 20260507 13:50:41.603 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260507 13:50:41.603 | INFO | "relAmplitude" : 0.25999999046325684 | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "T" : | 20260507 13:50:41.603 | INFO | { | 20260507 13:50:41.603 | INFO | "baseLabel" : "T", | 20260507 13:50:41.603 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260507 13:50:41.603 | INFO | "interPulseDistance" : 0.079999998211860657, | 20260507 13:50:41.603 | INFO | "ipd2SlowStepRatio" : 0, | 20260507 13:50:41.603 | INFO | "pulseWidth" : 0.16300000250339508, | 20260507 13:50:41.603 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260507 13:50:41.603 | INFO | "relAmplitude" : 0.44499999284744263 | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "chipLayoutName" : "KestrelRTO2", | 20260507 13:50:41.603 | INFO | "photoelectronSensitivity" : 6.25, | 20260507 13:50:41.603 | INFO | "refSnr" : 16.700000762939453 | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "autoConnect" : true, | 20260507 13:50:41.603 | INFO | "canonicalFrameRate" : 100, | 20260507 13:50:41.603 | INFO | "chunkValidationLevel" : 0, | 20260507 13:50:41.603 | INFO | "daemonUrl" : "http://localhost:23602", | 20260507 13:50:41.603 | INFO | "dataMode" : "FRAME", | 20260507 13:50:41.603 | INFO | "dataPath" : "Normal", | 20260507 13:50:41.603 | INFO | "loopback" : false, | 20260507 13:50:41.603 | INFO | "maskCardinalityFraction" : 1, | 20260507 13:50:41.603 | INFO | "maskEncoding" : "COMPRESSEDBASE64_V2", | 20260507 13:50:41.603 | INFO | "maxPopLoops" : 10, | 20260507 13:50:41.603 | INFO | "pedestal" : -32768, | 20260507 13:50:41.603 | INFO | "pointerReplacementSafetyFactor" : 0.80000000000000004, | 20260507 13:50:41.603 | INFO | "primerScaler" : 1, | 20260507 13:50:41.603 | INFO | "simConfig" : | 20260507 13:50:41.603 | INFO | { | 20260507 13:50:41.603 | INFO | "numCols" : 0, | 20260507 13:50:41.603 | INFO | "numRows" : 0, | 20260507 13:50:41.603 | INFO | "numSubSensors" : 2, | 20260507 13:50:41.603 | INFO | "platformOverride" : "DONT_CARE", | 20260507 13:50:41.603 | INFO | "regionNumSlotsPerTrack" : 128, | 20260507 13:50:41.603 | INFO | "regionNumTracks" : 8, | 20260507 13:50:41.603 | INFO | "regionSize" : 1024 | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "simulatedFrameRate" : 100, | 20260507 13:50:41.603 | INFO | "simulatedInputFile" : "CONSTANT/123", | 20260507 13:50:41.603 | INFO | "simulationCondensePixels" : true, | 20260507 13:50:41.603 | INFO | "simulationNumRowsToReplicate" : 20, | 20260507 13:50:41.603 | INFO | "simulationRoi" : [], | 20260507 13:50:41.603 | INFO | "sleepDebug" : 0, | 20260507 13:50:41.603 | INFO | "smrtMaskMode" : "DEFAULT", | 20260507 13:50:41.603 | INFO | "sraIndex" : 0, | 20260507 13:50:41.603 | INFO | "tilePoolFactor" : 3, | 20260507 13:50:41.603 | INFO | "timeouts" : | 20260507 13:50:41.603 | INFO | { | 20260507 13:50:41.603 | INFO | "armedTimeoutSeconds" : 400, | 20260507 13:50:41.603 | INFO | "doneTimeoutSeconds" : 10, | 20260507 13:50:41.603 | INFO | "firstChunkTimeoutSeconds" : 400, | 20260507 13:50:41.603 | INFO | "httpTimeoutSeconds" : 10, | 20260507 13:50:41.603 | INFO | "notifiedTimeoutSeconds" : 15, | 20260507 13:50:41.603 | INFO | "resetTimeoutSeconds" : 15, | 20260507 13:50:41.603 | INFO | "subsequentChunkTimeoutSeconds" : -1, | 20260507 13:50:41.603 | INFO | "wxDaemonTimeoutSeconds" : 60 | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "verbosity" : 0 | 20260507 13:50:41.603 | INFO | } | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.603 | INFO | "system" : | 20260507 13:50:41.603 | INFO | { | 20260507 13:50:41.603 | INFO | "analyzerHardware" : "A100", | 20260507 13:50:41.603 | INFO | "basecallerBatchConcurrency" : 1, | 20260507 13:50:41.603 | INFO | "basecallerConcurrency" : 3, | 20260507 13:50:41.603 | INFO | "basecallerDeviceConcurrency" : 3, | 20260507 13:50:41.603 | INFO | "basecallerGpuDevices" : | 20260507 13:50:41.603 | INFO | [ | 20260507 13:50:41.603 | INFO | 0 | 20260507 13:50:41.603 | INFO | ], | 20260507 13:50:41.603 | INFO | "ioConcurrency" : 5, | 20260507 13:50:41.603 | INFO | "maxPermGpuDataMB" : 10000, | 20260507 13:50:41.603 | INFO | "numWorkerThreads" : 12, | 20260507 13:50:41.603 | INFO | "nvidiaSmiRunTimeSeconds" : 30, | 20260507 13:50:41.603 | INFO | "serialGpuCompute" : true | 20260507 13:50:41.603 | INFO | }, | 20260507 13:50:41.604 | INFO | "traceSaver" : | 20260507 13:50:41.604 | INFO | { | 20260507 13:50:41.604 | INFO | "bufferedChunksAllowed" : 5, | 20260507 13:50:41.604 | INFO | "frameChunking" : 512, | 20260507 13:50:41.604 | INFO | "outFormat" : "Natural", | 20260507 13:50:41.604 | INFO | "roi" : | 20260507 13:50:41.604 | INFO | [ | 20260507 13:50:41.604 | INFO | [ | 20260507 13:50:41.604 | INFO | 2048, | 20260507 13:50:41.604 | INFO | 768, | 20260507 13:50:41.604 | INFO | 1, | 20260507 13:50:41.604 | INFO | 64 | 20260507 13:50:41.604 | INFO | ], | 20260507 13:50:41.604 | INFO | [ | 20260507 13:50:41.604 | INFO | 2048, | 20260507 13:50:41.604 | INFO | 2304, | 20260507 13:50:41.604 | INFO | 1, | 20260507 13:50:41.604 | INFO | 64 | 20260507 13:50:41.604 | INFO | ], | 20260507 13:50:41.604 | INFO | [ | 20260507 13:50:41.604 | INFO | 2048, | 20260507 13:50:41.604 | INFO | 3840, | 20260507 13:50:41.604 | INFO | 1, | 20260507 13:50:41.604 | INFO | 64 | 20260507 13:50:41.604 | INFO | ], | 20260507 13:50:41.604 | INFO | [ | 20260507 13:50:41.604 | INFO | 2048, | 20260507 13:50:41.604 | INFO | 5376, | 20260507 13:50:41.604 | INFO | 1, | 20260507 13:50:41.604 | INFO | 64 | 20260507 13:50:41.604 | INFO | ] | 20260507 13:50:41.604 | INFO | ], | 20260507 13:50:41.604 | INFO | "zmwChunking" : 2048 | 20260507 13:50:41.604 | INFO | } | 20260507 13:50:41.604 | INFO | } | 20260507 13:50:41.604 | INFO | Output Target: /data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1.bazfilelist.txt | 20260507 13:50:41.655 | INFO | Found 1 CUDA devices | 20260507 13:50:41.655 | INFO | CUDA GPU device: 0 | 20260507 13:50:41.655 | INFO | Device Name:NVIDIA A100 80GB PCIe | 20260507 13:50:41.655 | INFO | UUID:GPU-ce26be3f-889e-2795-9302-85bf090c57ae | 20260507 13:50:41.655 | INFO | Global memory:85095874560 | 20260507 13:50:41.655 | INFO | sharedMemPerBlock:49152 | 20260507 13:50:41.655 | INFO | major/minor:8/0 | 20260507 13:50:41.655 | INFO | No message | 20260507 13:50:41.655 | INFO | CpuSchedulerCanary: starting | 20260507 13:50:41.655 | INFO | Starting CPU scheduling canary | 20260507 13:50:41.661 | INFO | Waiting for wx-daemon. Current boot state:"Daemon ready" | 20260507 13:50:41.661 | INFO | Connecting to shared memory key: 0x50420051 at 0x00500000000 | 20260507 13:50:45.914 | WARN | Precalculated crosstalk filter will not be used, one of the dimensions is zero | 20260507 13:50:45.914 | INFO | PSF will be passed to the DataSource for processing. | 20260507 13:50:45.916 | INFO | WXIPCDataSource expectedMaskedZmws:25165824 | 20260507 13:50:45.916 | INFO | WXIPCDataSource expectedSensorZmws:25165824 | 20260507 13:50:45.916 | WARN | PSF:[ [ 0.00079984 0.0029994 0.0165967 0.0034993 0.00109978] [ 0.00269946 0.0129974 0.0726855 0.0139972 0.00319936] [ 0.0117976 0.0544891 0.594281 0.0652869 0.0123975] [ 0.00319936 0.0130974 0.0726855 0.0140972 0.00309938] [ 0.0009998 0.00289942 0.0167966 0.00339932 0.00089982] ] | 20260507 13:50:45.916 | WARN | bNorm:0.811438 | 20260507 13:50:45.916 | WARN | PSF is not that close to the identity PSF. Norm of deviation = 0.811438 | 20260507 13:50:45.917 | INFO | Based on platform 6 the smrtMaskMode default is set to DISABLED | 20260507 13:50:45.917 | INFO | Connecting to wx-daemon http://localhost:23602 | 20260507 13:50:45.917 | INFO | platform:Kestrel | 20260507 13:50:45.918 | INFO | wxShortVersion: 26.1.0 wxLongVersion: 26.1.0.a8206dc561f8e1cdf6683a2ad782923bb7106e43 wxIpcVersion: 3 wxPersonality: 1.8.2-0 dataSourceShortVersion: 26.1.0 dataSourceLongVersion: 26.1.0.0 dataSourceIpcVersion: 3 | 20260507 13:50:45.919 | INFO | Waiting for wx-daemon. Current boot state:"Daemon ready" | 20260507 13:50:45.920 | INFO | Connecting IPC | 20260507 13:50:45.921 | INFO | Reconfiguring:{ "cameraConfig" : 0, "cardinalityFraction" : 1, "crosstalkKernel" : [ [ 0.00014065283436721698, 0.00024733493649707156, -0.02646965601969033, 1.2938141044737791e-05, -0.000209726572126589 ], [ 2.8209306317858476e-05, -0.011227287975635325, -0.16249877620543987, -0.009973250094704407, -0.00028787427166992058 ], [ -0.019849015808807709, -0.11953626080922616, 1.7245260129463982, -0.14535734185743521, -0.018293397259077079 ], [ -0.0011299207645029945, -0.011351514333903183, -0.16237503050695248, -0.010276039796743052, -7.8888800056879383e-05 ], [ -0.00019585354582084264, 0.00063817726959822143, -0.026998977274235791, 0.00028844750081735671, 0.00022703896098751053 ] ], "darkCalFileName" : "/data/icc/m84031_260507_084733_s1/pb_internal/m84031_260507_084733_s1.darkcal.h5", "encoding" : "UINT8", "expectedZmws" : 0, "layoutDims" : [ 4096, 512, 64 ], "layoutType" : "BLOCK_LAYOUT_DENSE", "memoryPoolIndex" : -1, "mode" : "AUTO", "mosi_socket_address" : "tcp://127.0.0.1:43159", "numFrames" : 8640000, "pedestal" : -32768, "requireAuroraUp" : true, "smrtMaskType" : "" } | 20260507 13:50:45.922 | INFO | Reconfigure returned:OK "ok" | 20260507 13:50:46.023 | INFO | Connecting to SRA:0 using tcp://127.0.0.1:43197 | 20260507 13:50:46.024 | INFO | wx-daemon requests 192 pointers | 20260507 13:50:46.024 | INFO | wxipc is sending requests 192 pointers | 20260507 13:50:46.026 | INFO | Pushing 192 free packets, with packetSize 134217728 | 20260507 13:50:46.026 | INFO | Done, Pushed 192 free packets | 20260507 13:50:46.026 | INFO | Sent first free packet to tcp://127.0.0.1:43197 | 20260507 13:50:46.026 | INFO | bursted 192 free packets | 20260507 13:50:46.027 | INFO | Reconfigure_continue returned:OK "ok" | 20260507 13:50:52.463 | INFO | Polled wx-daemon for pedestal and cached 20 | 20260507 13:50:52.463 | INFO | Chip layout: KestrelRTO2 | 20260507 13:50:52.463 | INFO | Frame rate: 100 | 20260507 13:50:52.463 | INFO | Photo-electron sensitivity: 6.25 | 20260507 13:50:52.463 | INFO | Reference SNR: 16.7 | 20260507 13:50:52.463 | INFO | Analog properties: [C, RA 1, exnCV 0.1, ipd 0.07, pw 0.209, pw2ssr 3.2, ipd2ssr 0] | 20260507 13:50:52.463 | INFO | Analog properties: [A, RA 0.67, exnCV 0.1, ipd 0.08, pw 0.166, pw2ssr 3.2, ipd2ssr 0] | 20260507 13:50:52.463 | INFO | Analog properties: [T, RA 0.445, exnCV 0.1, ipd 0.08, pw 0.163, pw2ssr 3.2, ipd2ssr 0] | 20260507 13:50:52.463 | INFO | Analog properties: [G, RA 0.26, exnCV 0.1, ipd 0.07, pw 0.193, pw2ssr 3.2, ipd2ssr 0] | 20260507 13:50:52.463 | INFO | Number of analysis zmwLanes (after smrt masking) = 393216 | 20260507 13:50:52.463 | INFO | Number of analysis chunks = 16875 | 20260507 13:50:52.463 | INFO | Instantiating TrivialRepacker | 20260507 13:50:52.463 | WARN | WXIPCDataSource returning dummy instrument name WXIPCDataSource | 20260507 13:50:52.465 | INFO | OOM score went from -1 to 666 at CreateTracePrepper checkpoint 1 | 20260507 13:50:52.465 | INFO | memory usage change: vm:26471824 RSS:25872 shared:22471 text:4978 data:23696 at CreateTracePrepper checkpoint 1 | 20260507 13:50:52.465 | INFO | Added 2 lanes to the ROI | 20260507 13:50:52.465 | INFO | Added 2 lanes to the ROI | 20260507 13:50:52.466 | INFO | Added 2 lanes to the ROI | 20260507 13:50:52.466 | INFO | Added 2 lanes to the ROI | 20260507 13:50:53.192 | INFO | Creating cached UnitCellProperties for dataSource 0x4e92fa0 | 20260507 13:50:53.712 | INFO | Done creating cached UnitCellProperties for dataSource 0x4e92fa0 | 20260507 13:50:53.712 | INFO | memory usage change: vm:26618852 RSS:173455 shared:22520 text:4978 data:170724 at SmrtBasecaller::CreateTraceSaver checkpoint 2 | 20260507 13:50:53.714 | INFO | TraceSaverBody created | 20260507 13:50:53.740 | INFO | memory usage change: vm:26570371 RSS:124785 shared:22614 text:4978 data:122243 at SmrtBasecaller::RunAnalyzer checkpoint 1 | 20260507 13:50:53.740 | INFO | Resource: 0, Device: 0, Assigning 3 CudaStreams | 20260507 13:50:53.740 | INFO | TraceHistogramAccumulator: BinSizeCoeff = 0.2. | 20260507 13:50:53.740 | INFO | TraceHistogramAccumulator: BaselineStatMinFrameCount = 50. | 20260507 13:50:53.740 | INFO | TraceHistogramAccumulator: FallBackBaselineSigma = 10. | 20260507 13:50:53.740 | INFO | DetectionModelEstimator: NumFramesPreAccumStats = 1000. | 20260507 13:50:53.740 | INFO | DetectionModelEstimator: MinFramesForEstimate= 4000. | 20260507 13:50:53.740 | INFO | Pulse signal variance shot noise coefficient = 1.77377. | 20260507 13:50:54.302 | INFO | OOM score went from 666 to 667 at BasecallerBody checkpoint 3 | 20260507 13:50:54.303 | INFO | memory usage change: vm:68517105 RSS:338970 shared:232900 text:4978 data:125937 at BasecallerBody checkpoint 3 | 20260507 13:51:13.053 | INFO | OOM score went from 667 to 684 at SmrtBasecaller::RunAnalyzer checkpoint 2 | 20260507 13:51:13.053 | INFO | memory usage change: vm:75587989 RSS:6877392 shared:6770120 text:4978 data:127125 at SmrtBasecaller::RunAnalyzer checkpoint 2 | 20260507 13:51:13.054 | INFO | CreateRealTimeMetrics: 9 regions | 20260507 13:51:13.054 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260507 13:51:13.085 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:13.136 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:13.251 | INFO | memory usage change: vm:75593766 RSS:6883134 shared:6770149 text:4978 data:132902 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260507 13:51:13.299 | INFO | memory usage change: vm:75618343 RSS:6907752 shared:6770149 text:4978 data:157479 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260507 13:51:13.380 | INFO | memory usage change: vm:75642920 RSS:6932304 shared:6770149 text:4978 data:182056 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260507 13:51:13.394 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:200,64,64,6016 | 20260507 13:51:13.394 | INFO | memory usage change: vm:75618343 RSS:6907762 shared:6770149 text:4978 data:157479 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260507 13:51:13.394 | INFO | Added 3008 lanes to the ROI | 20260507 13:51:13.395 | INFO | Added 3008 lanes to the ROI | 20260507 13:51:13.396 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:2016,64,64,6016 | 20260507 13:51:13.397 | INFO | Added 3008 lanes to the ROI | 20260507 13:51:13.397 | INFO | Added 3008 lanes to the ROI | 20260507 13:51:13.399 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:3832,64,64,6016 | 20260507 13:51:13.400 | INFO | Added 3008 lanes to the ROI | 20260507 13:51:13.400 | INFO | Added 3008 lanes to the ROI | 20260507 13:51:13.401 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260507 13:51:13.426 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:13.471 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:13.587 | INFO | memory usage change: vm:75619874 RSS:6909279 shared:6770149 text:4978 data:159010 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260507 13:51:13.636 | INFO | memory usage change: vm:75644451 RSS:6933831 shared:6770149 text:4978 data:183587 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260507 13:51:13.717 | INFO | memory usage change: vm:75669028 RSS:6958383 shared:6770149 text:4978 data:208164 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260507 13:51:13.731 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260507 13:51:13.731 | INFO | memory usage change: vm:75644451 RSS:6933870 shared:6770149 text:4978 data:183587 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260507 13:51:13.756 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:13.799 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:13.922 | INFO | memory usage change: vm:75645219 RSS:6934595 shared:6770149 text:4978 data:184355 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260507 13:51:13.973 | INFO | memory usage change: vm:75669796 RSS:6959213 shared:6770149 text:4978 data:208932 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260507 13:51:14.059 | INFO | memory usage change: vm:75694373 RSS:6983765 shared:6770149 text:4978 data:233509 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260507 13:51:14.073 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260507 13:51:14.073 | INFO | memory usage change: vm:75669796 RSS:6959215 shared:6770149 text:4978 data:208932 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260507 13:51:14.100 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:14.153 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:14.269 | INFO | memory usage change: vm:75670180 RSS:6959544 shared:6770149 text:4978 data:209316 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260507 13:51:14.318 | INFO | memory usage change: vm:75694757 RSS:6984162 shared:6770149 text:4978 data:233893 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260507 13:51:14.401 | INFO | memory usage change: vm:75719334 RSS:7008714 shared:6770149 text:4978 data:258470 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260507 13:51:14.418 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260507 13:51:14.418 | INFO | memory usage change: vm:75694757 RSS:6984176 shared:6770149 text:4978 data:233893 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260507 13:51:14.468 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:14.545 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:14.729 | INFO | memory usage change: vm:75695141 RSS:6984505 shared:6770149 text:4978 data:234277 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260507 13:51:14.814 | INFO | memory usage change: vm:75719718 RSS:7009123 shared:6770149 text:4978 data:258854 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260507 13:51:14.954 | INFO | memory usage change: vm:75744295 RSS:7033675 shared:6770149 text:4978 data:283431 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260507 13:51:14.972 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260507 13:51:14.972 | INFO | memory usage change: vm:75719718 RSS:7009137 shared:6770149 text:4978 data:258854 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260507 13:51:15.014 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:15.090 | INFO | Added 196608 lanes to the ROI | 20260507 13:51:15.259 | INFO | memory usage change: vm:75720102 RSS:7009466 shared:6770149 text:4978 data:259238 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260507 13:51:15.345 | INFO | memory usage change: vm:75744679 RSS:7034084 shared:6770149 text:4978 data:283815 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260507 13:51:15.486 | INFO | memory usage change: vm:75769256 RSS:7058636 shared:6770149 text:4978 data:308392 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260507 13:51:15.505 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1139: properties size:9 | 20260507 13:51:15.505 | INFO | memory usage change: vm:75744679 RSS:7034098 shared:6770149 text:4978 data:283815 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 8 | 20260507 13:51:15.869 | INFO | memory usage change: vm:75799977 RSS:7089339 shared:6770149 text:4978 data:339113 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 9 | 20260507 13:51:17.777 | INFO | memory usage change: vm:75641921 RSS:6931404 shared:6770213 text:4978 data:181057 at SmrtBasecaller::RunAnalyzer checkpoint 4 | 20260507 13:51:22.241 | INFO | OOM score went from 684 to 687 at SmrtBasecaller::RunAnalyzer checkpoint 6 | 20260507 13:51:22.241 | INFO | memory usage change: vm:80072422 RSS:8385992 shared:6770229 text:4978 data:4611558 at SmrtBasecaller::RunAnalyzer checkpoint 6 | 20260507 13:51:22.241 | INFO | AddNode CreateBazSaver started | 20260507 13:51:22.989 | INFO | OOM score went from 687 to 688 at SmrtBasecaller::CreateBazSaver checkpoint 3 | 20260507 13:51:22.989 | INFO | memory usage change: vm:80154511 RSS:8449544 shared:6770229 text:4978 data:4693647 at SmrtBasecaller::CreateBazSaver checkpoint 3 | 20260507 13:51:27.080 | INFO | memory usage change: vm:80351135 RSS:8812778 shared:6770229 text:4978 data:4890271 at SmrtBasecaller::CreateBazSaver checkpoint 4 | 20260507 13:51:27.081 | INFO | Enabling real-time activity labels in BAZ file. | 20260507 13:51:27.081 | INFO | Opening 96 BAZ files for writing with filename prefix: /data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1 zmws: 25165824 | 20260507 13:51:27.081 | INFO | Generating BAZ file list to file: /data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1.bazfilelist.txt | 20260507 13:51:30.972 | INFO | Opened 1 baz files so far | 20260507 13:51:34.097 | INFO | Opened 2 baz files so far | 20260507 13:51:34.152 | INFO | Opened 3 baz files so far | 20260507 13:51:34.223 | INFO | Opened 4 baz files so far | 20260507 13:51:34.266 | INFO | Opened 5 baz files so far | 20260507 13:51:34.334 | INFO | Opened 6 baz files so far | 20260507 13:51:34.464 | INFO | Opened 7 baz files so far | 20260507 13:51:34.469 | INFO | Opened 8 baz files so far | 20260507 13:51:34.472 | INFO | Opened 9 baz files so far | 20260507 13:51:34.482 | INFO | Opened 10 baz files so far | 20260507 13:51:34.483 | INFO | Opened 11 baz files so far | 20260507 13:51:34.487 | INFO | Opened 12 baz files so far | 20260507 13:51:34.500 | INFO | Opened 13 baz files so far | 20260507 13:51:37.298 | INFO | Opened 14 baz files so far | 20260507 13:51:37.379 | INFO | Opened 15 baz files so far | 20260507 13:51:37.416 | INFO | Opened 16 baz files so far | 20260507 13:51:37.450 | INFO | Opened 17 baz files so far | 20260507 13:51:37.468 | INFO | Opened 18 baz files so far | 20260507 13:51:37.784 | INFO | Opened 19 baz files so far | 20260507 13:51:37.788 | INFO | Opened 20 baz files so far | 20260507 13:51:37.789 | INFO | Opened 21 baz files so far | 20260507 13:51:37.790 | INFO | Opened 22 baz files so far | 20260507 13:51:37.790 | INFO | Opened 23 baz files so far | 20260507 13:51:37.793 | INFO | Opened 24 baz files so far | 20260507 13:51:37.840 | INFO | Opened 25 baz files so far | 20260507 13:51:40.281 | INFO | Opened 26 baz files so far | 20260507 13:51:40.569 | INFO | Opened 27 baz files so far | 20260507 13:51:40.660 | INFO | Opened 28 baz files so far | 20260507 13:51:40.691 | INFO | Opened 29 baz files so far | 20260507 13:51:40.714 | INFO | Opened 30 baz files so far | 20260507 13:51:40.996 | INFO | Opened 31 baz files so far | 20260507 13:51:40.997 | INFO | Opened 32 baz files so far | 20260507 13:51:40.999 | INFO | Opened 33 baz files so far | 20260507 13:51:41.001 | INFO | Opened 34 baz files so far | 20260507 13:51:41.003 | INFO | Opened 35 baz files so far | 20260507 13:51:41.005 | INFO | Opened 36 baz files so far | 20260507 13:51:41.033 | INFO | Opened 37 baz files so far | 20260507 13:51:43.211 | INFO | Opened 38 baz files so far | 20260507 13:51:43.663 | INFO | Opened 39 baz files so far | 20260507 13:51:43.802 | INFO | Opened 40 baz files so far | 20260507 13:51:43.841 | INFO | Opened 41 baz files so far | 20260507 13:51:43.873 | INFO | Opened 42 baz files so far | 20260507 13:51:44.137 | INFO | Opened 43 baz files so far | 20260507 13:51:44.144 | INFO | Opened 44 baz files so far | 20260507 13:51:44.148 | INFO | Opened 45 baz files so far | 20260507 13:51:44.158 | INFO | Opened 46 baz files so far | 20260507 13:51:44.158 | INFO | Opened 47 baz files so far | 20260507 13:51:44.165 | INFO | Opened 48 baz files so far | 20260507 13:51:44.198 | INFO | Opened 49 baz files so far | 20260507 13:51:45.955 | INFO | Opened 50 baz files so far | 20260507 13:51:46.429 | INFO | Opened 51 baz files so far | 20260507 13:51:46.570 | INFO | Opened 52 baz files so far | 20260507 13:51:46.708 | INFO | Opened 53 baz files so far | 20260507 13:51:46.755 | INFO | Opened 54 baz files so far | 20260507 13:51:47.166 | INFO | Opened 55 baz files so far | 20260507 13:51:47.166 | INFO | Opened 56 baz files so far | 20260507 13:51:47.172 | INFO | Opened 57 baz files so far | 20260507 13:51:47.186 | INFO | Opened 58 baz files so far | 20260507 13:51:47.191 | INFO | Opened 59 baz files so far | 20260507 13:51:47.199 | INFO | Opened 60 baz files so far | 20260507 13:51:47.203 | INFO | Opened 61 baz files so far | 20260507 13:51:48.442 | INFO | Opened 62 baz files so far | 20260507 13:51:49.219 | INFO | Opened 63 baz files so far | 20260507 13:51:49.389 | INFO | Opened 64 baz files so far | 20260507 13:51:49.624 | INFO | Opened 65 baz files so far | 20260507 13:51:49.841 | INFO | Opened 66 baz files so far | 20260507 13:51:50.180 | INFO | Opened 67 baz files so far | 20260507 13:51:50.206 | INFO | Opened 68 baz files so far | 20260507 13:51:50.212 | INFO | Opened 69 baz files so far | 20260507 13:51:50.216 | INFO | Opened 70 baz files so far | 20260507 13:51:50.221 | INFO | Opened 71 baz files so far | 20260507 13:51:50.230 | INFO | Opened 72 baz files so far | 20260507 13:51:50.279 | INFO | Opened 73 baz files so far | 20260507 13:51:51.105 | INFO | Opened 74 baz files so far | 20260507 13:51:52.166 | INFO | Opened 75 baz files so far | 20260507 13:51:52.275 | INFO | Opened 76 baz files so far | 20260507 13:51:52.541 | INFO | Opened 77 baz files so far | 20260507 13:51:52.841 | INFO | Opened 78 baz files so far | 20260507 13:51:53.357 | INFO | Opened 79 baz files so far | 20260507 13:51:53.363 | INFO | Opened 80 baz files so far | 20260507 13:51:53.370 | INFO | Opened 81 baz files so far | 20260507 13:51:53.381 | INFO | Opened 82 baz files so far | 20260507 13:51:53.383 | INFO | Opened 83 baz files so far | 20260507 13:51:53.402 | INFO | Opened 84 baz files so far | 20260507 13:51:53.498 | INFO | Opened 85 baz files so far | 20260507 13:51:53.950 | INFO | Opened 86 baz files so far | 20260507 13:51:55.319 | INFO | Opened 87 baz files so far | 20260507 13:51:55.355 | INFO | Opened 88 baz files so far | 20260507 13:51:55.583 | INFO | Opened 89 baz files so far | 20260507 13:51:55.937 | INFO | Opened 90 baz files so far | 20260507 13:51:56.421 | INFO | Opened 91 baz files so far | 20260507 13:51:56.588 | INFO | Opened 92 baz files so far | 20260507 13:51:56.589 | INFO | Opened 93 baz files so far | 20260507 13:51:56.591 | INFO | Opened 94 baz files so far | 20260507 13:51:56.605 | INFO | Opened 95 baz files so far | 20260507 13:51:56.633 | INFO | Opened 96 baz files so far | 20260507 13:51:56.650 | INFO | Finished opening a total of 96 baz files | 20260507 13:51:56.651 | INFO | OOM score went from 688 to 692 at BazWriterBody checkpoint 3 | 20260507 13:51:56.651 | INFO | memory usage change: vm:81592021 RSS:10041791 shared:6770350 text:4978 data:6131146 at BazWriterBody checkpoint 3 | 20260507 13:51:57.880 | INFO | AddNode CreateBazSaver done | 20260507 13:51:57.880 | INFO | OOM score went from 692 to 691 at SmrtBasecaller::RunAnalyzer checkpoint 7 | 20260507 13:51:57.880 | INFO | memory usage change: vm:81346959 RSS:9815192 shared:6770350 text:4978 data:5886084 at SmrtBasecaller::RunAnalyzer checkpoint 7 | 20260507 13:51:57.882 | INFO | Added 192 lanes to the ROI | 20260507 13:51:57.882 | INFO | Added 192 lanes to the ROI | 20260507 13:51:57.882 | INFO | Added 20 lanes to the ROI | 20260507 13:51:57.882 | INFO | Added 20 lanes to the ROI | 20260507 13:52:04.301 | INFO | memory usage change: vm:81346959 RSS:9815209 shared:6770366 text:4978 data:5886084 at AddNode CreateSampledBazSaver done | 20260507 13:52:04.352 | INFO | OOM score went from 691 to 690 at SmrtBasecaller::RunAnalyzer checkpoint 9 | 20260507 13:52:04.352 | INFO | memory usage change: vm:81248654 RSS:9716904 shared:6770366 text:4978 data:5787779 at SmrtBasecaller::RunAnalyzer checkpoint 9 | 20260507 13:52:04.353 | INFO | Entered sourceRunning loop, awaiting data from datasource ... | 20260507 13:52:10.353 | WARN | No incoming data for 6.00006 seconds, receive wait 6.00006 seconds, last deliver duration 0 seconds | 20260507 13:52:11.362 | INFO | Base Frame index set to 17638665 | 20260507 13:52:11.363 | INFO | Movie started at frame:0, expectedEnd Frame set to:8640000 | 20260507 13:52:11.363 | INFO | Analyzing chunk frames = [0,512) | 20260507 13:52:11.363 | INFO | MemInfo: { | 20260507 13:52:11.363 | INFO | "gpu" : " 11 GB", | 20260507 13:52:11.363 | INFO | "huge" : "", | 20260507 13:52:11.363 | INFO | "huge_shared" : " 25 GB", | 20260507 13:52:11.363 | INFO | "malloc" : " 15 GB", | 20260507 13:52:11.363 | INFO | "pinned" : " 27 GB" | 20260507 13:52:11.363 | INFO | } | 20260507 13:52:11.363 | INFO | Perf report for the last 78.898 seconds: | 20260507 13:52:11.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:11.363 | INFO | REPACKER : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | PREP_TRACE : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | SAVE_TRACE : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | ANALYSIS : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | PREP_RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | PRE_HQ : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | BAZWRITER : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:11.363 | INFO | BAZ_SAMPLING : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:16.502 | INFO | Analyzing chunk frames = [512,1024) | 20260507 13:52:16.502 | INFO | MemInfo: { | 20260507 13:52:16.502 | INFO | "gpu" : " 13 GB", | 20260507 13:52:16.502 | INFO | "huge" : "", | 20260507 13:52:16.502 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:16.502 | INFO | "malloc" : " 15 GB", | 20260507 13:52:16.502 | INFO | "pinned" : " 28 GB" | 20260507 13:52:16.502 | INFO | } | 20260507 13:52:16.503 | INFO | Perf report for the last 5.13949 seconds: | 20260507 13:52:16.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:16.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:16.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:16.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:16.503 | INFO | ANALYSIS : 100.0%, 3.0, 179.8ms, 9 | 20260507 13:52:16.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:16.503 | INFO | RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:16.503 | INFO | PRE_HQ : 6.8%, 1.0, 4.2ms, 0 | 20260507 13:52:16.503 | INFO | BAZWRITER : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:16.503 | INFO | BAZ_SAMPLING : 0.0%, 0.0, 0.0ms, 0 | 20260507 13:52:17.123 | INFO | Uploaded 36GiB to GPU in 1.74931s (20.5796GiB/s) | 20260507 13:52:17.123 | INFO | Downloaded 24GiB to GPU in 1.10846s (21.6516GiB/s) | 20260507 13:52:21.642 | INFO | Analyzing chunk frames = [1024,1536) | 20260507 13:52:21.642 | INFO | MemInfo: { | 20260507 13:52:21.642 | INFO | "gpu" : " 13 GB", | 20260507 13:52:21.642 | INFO | "huge" : "", | 20260507 13:52:21.642 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:21.642 | INFO | "malloc" : " 15 GB", | 20260507 13:52:21.642 | INFO | "pinned" : " 28 GB" | 20260507 13:52:21.642 | INFO | } | 20260507 13:52:21.642 | INFO | Perf report for the last 5.13953 seconds: | 20260507 13:52:21.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:21.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:21.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:21.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:21.642 | INFO | ANALYSIS : 70.6%, 2.9, 52.4ms, 0 | 20260507 13:52:21.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:21.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:21.642 | INFO | PRE_HQ : 8.5%, 1.0, 2.1ms, 0 | 20260507 13:52:21.642 | INFO | BAZWRITER : 0.0%, 1.1, 0.0ms, 0 | 20260507 13:52:21.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:21.643 | INFO | Uploaded 36GiB to GPU in 1.53788s (23.4089GiB/s) | 20260507 13:52:21.643 | INFO | Downloaded 24GiB to GPU in 1.05247s (22.8036GiB/s) | 20260507 13:52:26.781 | INFO | Analyzing chunk frames = [1536,2048) | 20260507 13:52:26.781 | INFO | MemInfo: { | 20260507 13:52:26.781 | INFO | "gpu" : " 13 GB", | 20260507 13:52:26.781 | INFO | "huge" : "", | 20260507 13:52:26.781 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:26.781 | INFO | "malloc" : " 15 GB", | 20260507 13:52:26.781 | INFO | "pinned" : " 28 GB" | 20260507 13:52:26.781 | INFO | } | 20260507 13:52:26.781 | INFO | Perf report for the last 5.13904 seconds: | 20260507 13:52:26.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:26.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 13:52:26.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | PRE_HQ : 8.0%, 1.0, 4.2ms, 0 | 20260507 13:52:26.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:26.782 | INFO | Uploaded 36GiB to GPU in 1.52626s (23.587GiB/s) | 20260507 13:52:26.782 | INFO | Downloaded 24GiB to GPU in 1.03904s (23.0982GiB/s) | 20260507 13:52:31.921 | INFO | Analyzing chunk frames = [2048,2560) | 20260507 13:52:31.921 | INFO | MemInfo: { | 20260507 13:52:31.921 | INFO | "gpu" : " 13 GB", | 20260507 13:52:31.921 | INFO | "huge" : "", | 20260507 13:52:31.921 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:31.921 | INFO | "malloc" : " 15 GB", | 20260507 13:52:31.921 | INFO | "pinned" : " 28 GB" | 20260507 13:52:31.921 | INFO | } | 20260507 13:52:31.921 | INFO | Perf report for the last 5.13967 seconds: | 20260507 13:52:31.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:31.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260507 13:52:31.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | PRE_HQ : 8.1%, 1.0, 4.2ms, 0 | 20260507 13:52:31.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:31.921 | INFO | Uploaded 36GiB to GPU in 1.52629s (23.5867GiB/s) | 20260507 13:52:31.921 | INFO | Downloaded 24GiB to GPU in 1.03977s (23.082GiB/s) | 20260507 13:52:37.060 | INFO | Analyzing chunk frames = [2560,3072) | 20260507 13:52:37.060 | INFO | MemInfo: { | 20260507 13:52:37.060 | INFO | "gpu" : " 13 GB", | 20260507 13:52:37.060 | INFO | "huge" : "", | 20260507 13:52:37.060 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:37.060 | INFO | "malloc" : " 15 GB", | 20260507 13:52:37.060 | INFO | "pinned" : " 28 GB" | 20260507 13:52:37.060 | INFO | } | 20260507 13:52:37.060 | INFO | Perf report for the last 5.13935 seconds: | 20260507 13:52:37.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:37.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.060 | INFO | ANALYSIS : 63.0%, 3.0, 98.3ms, 0 | 20260507 13:52:37.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.061 | INFO | PRE_HQ : 8.0%, 1.0, 4.2ms, 0 | 20260507 13:52:37.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:37.061 | INFO | Uploaded 36GiB to GPU in 1.52583s (23.5937GiB/s) | 20260507 13:52:37.061 | INFO | Downloaded 24GiB to GPU in 1.03929s (23.0927GiB/s) | 20260507 13:52:42.199 | INFO | Analyzing chunk frames = [3072,3584) | 20260507 13:52:42.199 | INFO | MemInfo: { | 20260507 13:52:42.199 | INFO | "gpu" : " 13 GB", | 20260507 13:52:42.199 | INFO | "huge" : "", | 20260507 13:52:42.199 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:42.199 | INFO | "malloc" : " 15 GB", | 20260507 13:52:42.199 | INFO | "pinned" : " 28 GB" | 20260507 13:52:42.199 | INFO | } | 20260507 13:52:42.200 | INFO | Perf report for the last 5.1393 seconds: | 20260507 13:52:42.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:42.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | ANALYSIS : 63.7%, 3.0, 99.1ms, 0 | 20260507 13:52:42.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | PRE_HQ : 8.3%, 1.0, 4.3ms, 0 | 20260507 13:52:42.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:42.200 | INFO | Uploaded 36GiB to GPU in 1.52665s (23.5811GiB/s) | 20260507 13:52:42.200 | INFO | Downloaded 24GiB to GPU in 1.03935s (23.0913GiB/s) | 20260507 13:52:47.340 | INFO | Analyzing chunk frames = [3584,4096) | 20260507 13:52:47.340 | INFO | MemInfo: { | 20260507 13:52:47.340 | INFO | "gpu" : " 13 GB", | 20260507 13:52:47.340 | INFO | "huge" : "", | 20260507 13:52:47.340 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:47.340 | INFO | "malloc" : " 15 GB", | 20260507 13:52:47.340 | INFO | "pinned" : " 28 GB" | 20260507 13:52:47.340 | INFO | } | 20260507 13:52:47.340 | INFO | Perf report for the last 5.14052 seconds: | 20260507 13:52:47.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:47.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260507 13:52:47.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | PRE_HQ : 8.4%, 1.0, 4.3ms, 0 | 20260507 13:52:47.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:47.340 | INFO | Uploaded 36GiB to GPU in 1.52707s (23.5745GiB/s) | 20260507 13:52:47.341 | INFO | Downloaded 24GiB to GPU in 1.03917s (23.0954GiB/s) | 20260507 13:52:47.428 | INFO | Starting a round of Baz IO | 20260507 13:52:50.298 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.87 seconds | 20260507 13:52:50.298 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:52:50.298 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260507 13:52:50.298 | INFO | Baz IO overall: | 20260507 13:52:50.299 | INFO | BytesWritten : 2571416390 | 20260507 13:52:50.299 | INFO | NumEvents : 0 | 20260507 13:52:52.477 | INFO | Analyzing chunk frames = [4096,4608) | 20260507 13:52:52.477 | INFO | MemInfo: { | 20260507 13:52:52.477 | INFO | "gpu" : " 13 GB", | 20260507 13:52:52.478 | INFO | "huge" : "", | 20260507 13:52:52.478 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:52.478 | INFO | "malloc" : " 15 GB", | 20260507 13:52:52.478 | INFO | "pinned" : " 28 GB" | 20260507 13:52:52.478 | INFO | } | 20260507 13:52:52.479 | INFO | Perf report for the last 5.13845 seconds: | 20260507 13:52:52.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:52.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:52.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:52.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:52.479 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 13:52:52.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:52.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:52.479 | INFO | PRE_HQ : 54.6%, 1.2, 35.2ms, 0 | 20260507 13:52:52.479 | INFO | BAZWRITER : 15.4%, 1.0, 7.9ms, 0 | 20260507 13:52:52.479 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260507 13:52:52.479 | INFO | Uploaded 36GiB to GPU in 1.53586s (23.4396GiB/s) | 20260507 13:52:52.479 | INFO | Downloaded 24GiB to GPU in 1.04801s (22.9006GiB/s) | 20260507 13:52:57.606 | INFO | Analyzing chunk frames = [4608,5120) | 20260507 13:52:57.606 | INFO | MemInfo: { | 20260507 13:52:57.606 | INFO | "gpu" : " 13 GB", | 20260507 13:52:57.606 | INFO | "huge" : "", | 20260507 13:52:57.606 | INFO | "huge_shared" : " 29 GB", | 20260507 13:52:57.606 | INFO | "malloc" : " 15 GB", | 20260507 13:52:57.606 | INFO | "pinned" : " 28 GB" | 20260507 13:52:57.606 | INFO | } | 20260507 13:52:57.606 | INFO | Perf report for the last 5.12763 seconds: | 20260507 13:52:57.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:52:57.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.606 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 13:52:57.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.606 | INFO | PRE_HQ : 8.0%, 1.0, 4.1ms, 0 | 20260507 13:52:57.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:52:57.607 | INFO | Uploaded 36GiB to GPU in 1.52804s (23.5596GiB/s) | 20260507 13:52:57.607 | INFO | Downloaded 24GiB to GPU in 1.04174s (23.0383GiB/s) | 20260507 13:53:02.745 | INFO | Analyzing chunk frames = [5120,5632) | 20260507 13:53:02.745 | INFO | MemInfo: { | 20260507 13:53:02.745 | INFO | "gpu" : " 13 GB", | 20260507 13:53:02.745 | INFO | "huge" : "", | 20260507 13:53:02.745 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:02.745 | INFO | "malloc" : " 15 GB", | 20260507 13:53:02.745 | INFO | "pinned" : " 28 GB" | 20260507 13:53:02.745 | INFO | } | 20260507 13:53:02.746 | INFO | Perf report for the last 5.1393 seconds: | 20260507 13:53:02.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:02.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | ANALYSIS : 63.2%, 3.0, 98.3ms, 0 | 20260507 13:53:02.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | PRE_HQ : 8.3%, 1.0, 4.3ms, 0 | 20260507 13:53:02.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:02.746 | INFO | Uploaded 36GiB to GPU in 1.5266s (23.5819GiB/s) | 20260507 13:53:02.746 | INFO | Downloaded 24GiB to GPU in 1.03991s (23.079GiB/s) | 20260507 13:53:07.885 | INFO | Analyzing chunk frames = [5632,6144) | 20260507 13:53:07.885 | INFO | MemInfo: { | 20260507 13:53:07.885 | INFO | "gpu" : " 13 GB", | 20260507 13:53:07.885 | INFO | "huge" : "", | 20260507 13:53:07.885 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:07.885 | INFO | "malloc" : " 15 GB", | 20260507 13:53:07.885 | INFO | "pinned" : " 28 GB" | 20260507 13:53:07.885 | INFO | } | 20260507 13:53:07.885 | INFO | Perf report for the last 5.13946 seconds: | 20260507 13:53:07.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:07.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260507 13:53:07.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | PRE_HQ : 7.6%, 1.0, 3.9ms, 0 | 20260507 13:53:07.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:07.885 | INFO | Uploaded 36GiB to GPU in 1.538s (23.407GiB/s) | 20260507 13:53:07.886 | INFO | Downloaded 24GiB to GPU in 1.05363s (22.7784GiB/s) | 20260507 13:53:13.024 | INFO | Analyzing chunk frames = [6144,6656) | 20260507 13:53:13.024 | INFO | MemInfo: { | 20260507 13:53:13.024 | INFO | "gpu" : " 13 GB", | 20260507 13:53:13.024 | INFO | "huge" : "", | 20260507 13:53:13.024 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:13.024 | INFO | "malloc" : " 15 GB", | 20260507 13:53:13.024 | INFO | "pinned" : " 28 GB" | 20260507 13:53:13.024 | INFO | } | 20260507 13:53:13.024 | INFO | Perf report for the last 5.13925 seconds: | 20260507 13:53:13.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:13.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.024 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 13:53:13.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.024 | INFO | PRE_HQ : 8.0%, 1.0, 4.2ms, 0 | 20260507 13:53:13.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:13.025 | INFO | Uploaded 36GiB to GPU in 1.52656s (23.5824GiB/s) | 20260507 13:53:13.025 | INFO | Downloaded 24GiB to GPU in 1.0405s (23.0659GiB/s) | 20260507 13:53:18.163 | INFO | Analyzing chunk frames = [6656,7168) | 20260507 13:53:18.163 | INFO | MemInfo: { | 20260507 13:53:18.163 | INFO | "gpu" : " 13 GB", | 20260507 13:53:18.163 | INFO | "huge" : "", | 20260507 13:53:18.163 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:18.163 | INFO | "malloc" : " 15 GB", | 20260507 13:53:18.163 | INFO | "pinned" : " 28 GB" | 20260507 13:53:18.164 | INFO | } | 20260507 13:53:18.164 | INFO | Perf report for the last 5.13945 seconds: | 20260507 13:53:18.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:18.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | ANALYSIS : 57.9%, 3.0, 90.0ms, 0 | 20260507 13:53:18.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | PRE_HQ : 7.5%, 1.0, 3.9ms, 0 | 20260507 13:53:18.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:18.164 | INFO | Uploaded 36GiB to GPU in 1.54094s (23.3624GiB/s) | 20260507 13:53:18.164 | INFO | Downloaded 24GiB to GPU in 1.05943s (22.6537GiB/s) | 20260507 13:53:23.303 | INFO | Analyzing chunk frames = [7168,7680) | 20260507 13:53:23.303 | INFO | MemInfo: { | 20260507 13:53:23.303 | INFO | "gpu" : " 13 GB", | 20260507 13:53:23.303 | INFO | "huge" : "", | 20260507 13:53:23.303 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:23.303 | INFO | "malloc" : " 15 GB", | 20260507 13:53:23.303 | INFO | "pinned" : " 28 GB" | 20260507 13:53:23.303 | INFO | } | 20260507 13:53:23.303 | INFO | Perf report for the last 5.13911 seconds: | 20260507 13:53:23.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:23.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260507 13:53:23.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | PRE_HQ : 8.0%, 1.0, 4.2ms, 0 | 20260507 13:53:23.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:23.303 | INFO | Uploaded 36GiB to GPU in 1.52654s (23.5827GiB/s) | 20260507 13:53:23.303 | INFO | Downloaded 24GiB to GPU in 1.04139s (23.0462GiB/s) | 20260507 13:53:28.442 | INFO | Analyzing chunk frames = [7680,8192) | 20260507 13:53:28.442 | INFO | MemInfo: { | 20260507 13:53:28.442 | INFO | "gpu" : " 13 GB", | 20260507 13:53:28.442 | INFO | "huge" : "", | 20260507 13:53:28.442 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:28.442 | INFO | "malloc" : " 15 GB", | 20260507 13:53:28.442 | INFO | "pinned" : " 28 GB" | 20260507 13:53:28.442 | INFO | } | 20260507 13:53:28.442 | INFO | Perf report for the last 5.13913 seconds: | 20260507 13:53:28.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:28.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.442 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260507 13:53:28.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.442 | INFO | PRE_HQ : 8.2%, 1.0, 4.3ms, 0 | 20260507 13:53:28.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:28.443 | INFO | Uploaded 36GiB to GPU in 1.52708s (23.5744GiB/s) | 20260507 13:53:28.443 | INFO | Downloaded 24GiB to GPU in 1.04227s (23.0267GiB/s) | 20260507 13:53:28.443 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 13:53:28.443 | INFO | Total mem usage: 14.6 GiB | 20260507 13:53:28.443 | INFO | All Time High Water Marks sum: 14.6 GiB (Savings: 0.0527%) | 20260507 13:53:28.443 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: 0.0527%) | 20260507 13:53:28.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:53:28.443 | INFO | High Water Breakdown: | 20260507 13:53:28.443 | INFO | 54.6% : 7.96 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 13:53:28.443 | INFO | 45.3% : 6.6 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 13:53:28.443 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 13:53:28.443 | INFO | ------------------------------------------ | 20260507 13:53:28.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:53:28.443 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 13:53:28.443 | INFO | Total mem usage: 26.5 GiB | 20260507 13:53:28.443 | INFO | All Time High Water Marks sum: 26.5 GiB (Savings: 0%) | 20260507 13:53:28.443 | INFO | Recent High Water Marks Sum: 26.5 GiB (Savings: 0%) | 20260507 13:53:28.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:53:28.443 | INFO | High Water Breakdown: | 20260507 13:53:28.443 | INFO | 54.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:53:28.443 | INFO | 35% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:53:28.443 | INFO | 3.54% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:53:28.443 | INFO | 3.01% : 816 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 13:53:28.443 | INFO | 1.9% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:53:28.443 | INFO | 1.41% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:53:28.443 | INFO | 0.987% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:53:28.443 | INFO | 0.0147% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 13:53:28.443 | INFO | 0.0147% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:53:28.443 | INFO | 0.00147% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 13:53:28.443 | INFO | 9.03e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 13:53:28.443 | INFO | ------------------------------------------ | 20260507 13:53:28.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:53:28.443 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 13:53:28.443 | INFO | Total mem usage: 27.8 GiB | 20260507 13:53:28.443 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 13:53:28.443 | INFO | Recent High Water Marks Sum: 27.8 GiB (Savings: 0%) | 20260507 13:53:28.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:53:28.443 | INFO | High Water Breakdown: | 20260507 13:53:28.443 | INFO | 100% : 27.8 GiB ( 100% Of Peak) : WXIPCDataSourceConfig | 20260507 13:53:28.443 | INFO | ------------------------------------------ | 20260507 13:53:28.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:53:28.443 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 13:53:28.443 | INFO | Total mem usage: 12.5 GiB | 20260507 13:53:28.443 | INFO | All Time High Water Marks sum: 12.5 GiB (Savings: 0%) | 20260507 13:53:28.443 | INFO | Recent High Water Marks Sum: 12.5 GiB (Savings: 0%) | 20260507 13:53:28.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:53:28.443 | INFO | High Water Breakdown: | 20260507 13:53:28.443 | INFO | 43.3% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 13:53:28.443 | INFO | 17.7% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 13:53:28.443 | INFO | 11.3% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 13:53:28.443 | INFO | 9.03% : 1.12 GiB ( 100% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 13:53:28.443 | INFO | 4.51% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 13:53:28.443 | INFO | 3.6% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:53:28.443 | INFO | 3.01% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 13:53:28.443 | INFO | 2.33% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:53:28.443 | INFO | 2.19% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 13:53:28.443 | INFO | 1% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 13:53:28.443 | INFO | 1% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 13:53:28.443 | INFO | 0.752% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 13:53:28.443 | INFO | 0.235% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:53:28.443 | INFO | 0.0941% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:53:28.443 | INFO | ------------------------------------------ | 20260507 13:53:28.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:53:28.443 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 13:53:28.443 | INFO | PerfReport: Gathered 1440 samples. | 20260507 13:53:28.443 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:53:28.443 | INFO | PerfReport: Full Execution -- 1.03e+02 / -nan, 3.13e+01 / 0.00e+00 | 20260507 13:53:28.443 | INFO | PerfReport: 15.66%, Upload -- 1.61e+01 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260507 13:53:28.443 | INFO | PerfReport: 10.63%, Download -- 1.09e+01 / 0.00e+00, 2.50e-01 / 0.00e+00 | 20260507 13:53:28.444 | INFO | PerfReport: 33.38%, Compute -- 3.43e+01 / 0.00e+00, 1.58e+01 / 0.00e+00 | 20260507 13:53:28.444 | INFO | PerfReport: 0.50%, LockUpload -- 5.12e-01 / 0.00e+00, 3.68e+00 / 0.00e+00 | 20260507 13:53:28.444 | INFO | PerfReport: 0.00%, LockDownload -- 5.87e-04 / 0.00e+00, 5.21e-04 / 0.00e+00 | 20260507 13:53:28.444 | INFO | PerfReport: 39.83%, LockCompute -- 4.09e+01 / 0.00e+00, 2.07e+01 / 0.00e+00 | 20260507 13:53:28.444 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 2.75e-02 / 0.00e+00 | 20260507 13:53:28.544 | INFO | Starting a round of Baz IO | 20260507 13:53:31.134 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.589 seconds | 20260507 13:53:31.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:53:31.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260507 13:53:31.134 | INFO | Baz IO overall: | 20260507 13:53:31.134 | INFO | BytesWritten : 2571419942 | 20260507 13:53:31.134 | INFO | NumEvents : 0 | 20260507 13:53:33.581 | INFO | Analyzing chunk frames = [8192,8704) | 20260507 13:53:33.581 | INFO | MemInfo: { | 20260507 13:53:33.581 | INFO | "gpu" : " 13 GB", | 20260507 13:53:33.581 | INFO | "huge" : "", | 20260507 13:53:33.581 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:33.581 | INFO | "malloc" : " 15 GB", | 20260507 13:53:33.581 | INFO | "pinned" : " 28 GB" | 20260507 13:53:33.581 | INFO | } | 20260507 13:53:33.582 | INFO | Perf report for the last 5.13951 seconds: | 20260507 13:53:33.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:33.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:33.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:33.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:33.582 | INFO | ANALYSIS : 51.4%, 3.0, 79.9ms, 0 | 20260507 13:53:33.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:33.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:33.582 | INFO | PRE_HQ : 48.1%, 1.1, 28.1ms, 0 | 20260507 13:53:33.582 | INFO | BAZWRITER : 13.4%, 1.0, 6.9ms, 0 | 20260507 13:53:33.582 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260507 13:53:33.582 | INFO | Uploaded 36GiB to GPU in 1.5613s (23.0577GiB/s) | 20260507 13:53:33.582 | INFO | Downloaded 24GiB to GPU in 1.08025s (22.2171GiB/s) | 20260507 13:53:38.721 | INFO | Analyzing chunk frames = [8704,9216) | 20260507 13:53:38.721 | INFO | MemInfo: { | 20260507 13:53:38.721 | INFO | "gpu" : " 13 GB", | 20260507 13:53:38.721 | INFO | "huge" : "", | 20260507 13:53:38.721 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:38.721 | INFO | "malloc" : " 15 GB", | 20260507 13:53:38.721 | INFO | "pinned" : " 28 GB" | 20260507 13:53:38.721 | INFO | } | 20260507 13:53:38.721 | INFO | Perf report for the last 5.13934 seconds: | 20260507 13:53:38.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:38.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.721 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260507 13:53:38.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.721 | INFO | PRE_HQ : 8.2%, 1.0, 4.3ms, 0 | 20260507 13:53:38.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:38.722 | INFO | Uploaded 36GiB to GPU in 1.5275s (23.5679GiB/s) | 20260507 13:53:38.722 | INFO | Downloaded 24GiB to GPU in 1.03971s (23.0834GiB/s) | 20260507 13:53:43.860 | INFO | Analyzing chunk frames = [9216,9728) | 20260507 13:53:43.860 | INFO | MemInfo: { | 20260507 13:53:43.860 | INFO | "gpu" : " 13 GB", | 20260507 13:53:43.860 | INFO | "huge" : "", | 20260507 13:53:43.860 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:43.860 | INFO | "malloc" : " 15 GB", | 20260507 13:53:43.860 | INFO | "pinned" : " 28 GB" | 20260507 13:53:43.860 | INFO | } | 20260507 13:53:43.860 | INFO | Perf report for the last 5.13927 seconds: | 20260507 13:53:43.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:43.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260507 13:53:43.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | PRE_HQ : 8.1%, 1.0, 4.2ms, 0 | 20260507 13:53:43.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:43.860 | INFO | Uploaded 36GiB to GPU in 1.52657s (23.5822GiB/s) | 20260507 13:53:43.860 | INFO | Downloaded 24GiB to GPU in 1.04002s (23.0764GiB/s) | 20260507 13:53:48.999 | INFO | Analyzing chunk frames = [9728,10240) | 20260507 13:53:48.999 | INFO | MemInfo: { | 20260507 13:53:48.999 | INFO | "gpu" : " 13 GB", | 20260507 13:53:48.999 | INFO | "huge" : "", | 20260507 13:53:48.999 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:48.999 | INFO | "malloc" : " 15 GB", | 20260507 13:53:48.999 | INFO | "pinned" : " 28 GB" | 20260507 13:53:48.999 | INFO | } | 20260507 13:53:48.999 | INFO | Perf report for the last 5.13927 seconds: | 20260507 13:53:48.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:48.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:48.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:48.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:48.999 | INFO | ANALYSIS : 63.1%, 3.0, 98.2ms, 0 | 20260507 13:53:48.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:49.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:49.000 | INFO | PRE_HQ : 8.1%, 1.0, 4.2ms, 0 | 20260507 13:53:49.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:49.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:49.000 | INFO | Uploaded 36GiB to GPU in 1.52711s (23.5739GiB/s) | 20260507 13:53:49.000 | INFO | Downloaded 24GiB to GPU in 1.03883s (23.1029GiB/s) | 20260507 13:53:54.139 | INFO | Analyzing chunk frames = [10240,10752) | 20260507 13:53:54.139 | INFO | MemInfo: { | 20260507 13:53:54.139 | INFO | "gpu" : " 13 GB", | 20260507 13:53:54.139 | INFO | "huge" : "", | 20260507 13:53:54.139 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:54.139 | INFO | "malloc" : " 15 GB", | 20260507 13:53:54.139 | INFO | "pinned" : " 28 GB" | 20260507 13:53:54.139 | INFO | } | 20260507 13:53:54.139 | INFO | Perf report for the last 5.1393 seconds: | 20260507 13:53:54.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:54.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | ANALYSIS : 55.1%, 3.0, 85.8ms, 0 | 20260507 13:53:54.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | PRE_HQ : 7.4%, 1.0, 3.8ms, 0 | 20260507 13:53:54.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:54.139 | INFO | Uploaded 36GiB to GPU in 1.52796s (23.5609GiB/s) | 20260507 13:53:54.139 | INFO | Downloaded 24GiB to GPU in 1.04384s (22.9921GiB/s) | 20260507 13:53:59.278 | INFO | Analyzing chunk frames = [10752,11264) | 20260507 13:53:59.278 | INFO | MemInfo: { | 20260507 13:53:59.278 | INFO | "gpu" : " 13 GB", | 20260507 13:53:59.278 | INFO | "huge" : "", | 20260507 13:53:59.278 | INFO | "huge_shared" : " 29 GB", | 20260507 13:53:59.278 | INFO | "malloc" : " 15 GB", | 20260507 13:53:59.278 | INFO | "pinned" : " 28 GB" | 20260507 13:53:59.278 | INFO | } | 20260507 13:53:59.278 | INFO | Perf report for the last 5.13947 seconds: | 20260507 13:53:59.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:53:59.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.278 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 13:53:59.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.278 | INFO | PRE_HQ : 7.9%, 1.0, 4.1ms, 0 | 20260507 13:53:59.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:53:59.279 | INFO | Uploaded 36GiB to GPU in 1.52667s (23.5807GiB/s) | 20260507 13:53:59.279 | INFO | Downloaded 24GiB to GPU in 1.04127s (23.0488GiB/s) | 20260507 13:54:04.417 | INFO | Analyzing chunk frames = [11264,11776) | 20260507 13:54:04.417 | INFO | MemInfo: { | 20260507 13:54:04.417 | INFO | "gpu" : " 13 GB", | 20260507 13:54:04.417 | INFO | "huge" : "", | 20260507 13:54:04.417 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:04.417 | INFO | "malloc" : " 15 GB", | 20260507 13:54:04.418 | INFO | "pinned" : " 28 GB" | 20260507 13:54:04.418 | INFO | } | 20260507 13:54:04.418 | INFO | Perf report for the last 5.13935 seconds: | 20260507 13:54:04.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:04.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260507 13:54:04.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | PRE_HQ : 8.0%, 1.0, 4.2ms, 0 | 20260507 13:54:04.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:04.418 | INFO | Uploaded 36GiB to GPU in 1.52735s (23.5702GiB/s) | 20260507 13:54:04.418 | INFO | Downloaded 24GiB to GPU in 1.04106s (23.0534GiB/s) | 20260507 13:54:09.557 | INFO | Analyzing chunk frames = [11776,12288) | 20260507 13:54:09.557 | INFO | MemInfo: { | 20260507 13:54:09.557 | INFO | "gpu" : " 13 GB", | 20260507 13:54:09.557 | INFO | "huge" : "", | 20260507 13:54:09.557 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:09.557 | INFO | "malloc" : " 15 GB", | 20260507 13:54:09.557 | INFO | "pinned" : " 28 GB" | 20260507 13:54:09.557 | INFO | } | 20260507 13:54:09.557 | INFO | Perf report for the last 5.13921 seconds: | 20260507 13:54:09.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:09.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 13:54:09.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | PRE_HQ : 7.7%, 1.0, 4.0ms, 0 | 20260507 13:54:09.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:09.557 | INFO | Uploaded 36GiB to GPU in 1.52722s (23.5723GiB/s) | 20260507 13:54:09.557 | INFO | Downloaded 24GiB to GPU in 1.04306s (23.0093GiB/s) | 20260507 13:54:09.638 | INFO | Starting a round of Baz IO | 20260507 13:54:11.831 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.193 seconds | 20260507 13:54:11.831 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:54:11.831 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260507 13:54:11.831 | INFO | Baz IO overall: | 20260507 13:54:11.831 | INFO | BytesWritten : 2571423494 | 20260507 13:54:11.831 | INFO | NumEvents : 0 | 20260507 13:54:14.696 | INFO | Analyzing chunk frames = [12288,12800) | 20260507 13:54:14.696 | INFO | MemInfo: { | 20260507 13:54:14.696 | INFO | "gpu" : " 14 GB", | 20260507 13:54:14.696 | INFO | "huge" : "", | 20260507 13:54:14.696 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:14.696 | INFO | "malloc" : " 15 GB", | 20260507 13:54:14.696 | INFO | "pinned" : " 28 GB" | 20260507 13:54:14.696 | INFO | } | 20260507 13:54:14.696 | INFO | Perf report for the last 5.1393 seconds: | 20260507 13:54:14.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:14.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:14.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:14.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:14.696 | INFO | ANALYSIS : 43.7%, 3.0, 68.0ms, 0 | 20260507 13:54:14.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:14.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:14.696 | INFO | PRE_HQ : 39.4%, 1.0, 21.1ms, 0 | 20260507 13:54:14.696 | INFO | BAZWRITER : 11.0%, 1.0, 5.7ms, 0 | 20260507 13:54:14.696 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260507 13:54:14.696 | INFO | Uploaded 36GiB to GPU in 1.58517s (22.7104GiB/s) | 20260507 13:54:14.696 | INFO | Downloaded 24GiB to GPU in 1.11023s (21.6171GiB/s) | 20260507 13:54:19.835 | INFO | Analyzing chunk frames = [12800,13312) | 20260507 13:54:19.835 | INFO | MemInfo: { | 20260507 13:54:19.835 | INFO | "gpu" : " 14 GB", | 20260507 13:54:19.835 | INFO | "huge" : "", | 20260507 13:54:19.835 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:19.835 | INFO | "malloc" : " 15 GB", | 20260507 13:54:19.835 | INFO | "pinned" : " 28 GB" | 20260507 13:54:19.835 | INFO | } | 20260507 13:54:19.836 | INFO | Perf report for the last 5.13937 seconds: | 20260507 13:54:19.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:19.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | ANALYSIS : 44.1%, 3.0, 68.6ms, 0 | 20260507 13:54:19.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | PRE_HQ : 5.7%, 1.0, 3.0ms, 0 | 20260507 13:54:19.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:19.836 | INFO | Uploaded 36GiB to GPU in 1.57552s (22.8496GiB/s) | 20260507 13:54:19.836 | INFO | Downloaded 24GiB to GPU in 1.09669s (21.884GiB/s) | 20260507 13:54:24.975 | INFO | Analyzing chunk frames = [13312,13824) | 20260507 13:54:24.975 | INFO | MemInfo: { | 20260507 13:54:24.975 | INFO | "gpu" : " 14 GB", | 20260507 13:54:24.975 | INFO | "huge" : "", | 20260507 13:54:24.975 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:24.975 | INFO | "malloc" : " 15 GB", | 20260507 13:54:24.975 | INFO | "pinned" : " 28 GB" | 20260507 13:54:24.975 | INFO | } | 20260507 13:54:24.975 | INFO | Perf report for the last 5.13918 seconds: | 20260507 13:54:24.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:24.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | ANALYSIS : 44.4%, 3.0, 69.1ms, 0 | 20260507 13:54:24.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260507 13:54:24.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:24.975 | INFO | Uploaded 36GiB to GPU in 1.57335s (22.8811GiB/s) | 20260507 13:54:24.975 | INFO | Downloaded 24GiB to GPU in 1.09642s (21.8894GiB/s) | 20260507 13:54:30.114 | INFO | Analyzing chunk frames = [13824,14336) | 20260507 13:54:30.114 | INFO | MemInfo: { | 20260507 13:54:30.114 | INFO | "gpu" : " 14 GB", | 20260507 13:54:30.114 | INFO | "huge" : "", | 20260507 13:54:30.114 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:30.114 | INFO | "malloc" : " 15 GB", | 20260507 13:54:30.114 | INFO | "pinned" : " 28 GB" | 20260507 13:54:30.114 | INFO | } | 20260507 13:54:30.114 | INFO | Perf report for the last 5.13956 seconds: | 20260507 13:54:30.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:30.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.114 | INFO | ANALYSIS : 44.5%, 3.0, 69.2ms, 0 | 20260507 13:54:30.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.114 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260507 13:54:30.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:30.115 | INFO | Uploaded 36GiB to GPU in 1.57247s (22.8939GiB/s) | 20260507 13:54:30.115 | INFO | Downloaded 24GiB to GPU in 1.0961s (21.8959GiB/s) | 20260507 13:54:35.254 | INFO | Analyzing chunk frames = [14336,14848) | 20260507 13:54:35.254 | INFO | MemInfo: { | 20260507 13:54:35.254 | INFO | "gpu" : " 16 GB", | 20260507 13:54:35.254 | INFO | "huge" : "", | 20260507 13:54:35.254 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:35.254 | INFO | "malloc" : " 15 GB", | 20260507 13:54:35.254 | INFO | "pinned" : " 28 GB" | 20260507 13:54:35.254 | INFO | } | 20260507 13:54:35.254 | INFO | Perf report for the last 5.13941 seconds: | 20260507 13:54:35.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:35.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 13:54:35.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | PRE_HQ : 10.4%, 1.0, 5.6ms, 0 | 20260507 13:54:35.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:35.254 | INFO | Uploaded 36GiB to GPU in 1.57319s (22.8834GiB/s) | 20260507 13:54:35.254 | INFO | Downloaded 26GiB to GPU in 1.21714s (21.3616GiB/s) | 20260507 13:54:40.393 | INFO | Analyzing chunk frames = [14848,15360) | 20260507 13:54:40.393 | INFO | MemInfo: { | 20260507 13:54:40.393 | INFO | "gpu" : " 16 GB", | 20260507 13:54:40.393 | INFO | "huge" : "", | 20260507 13:54:40.393 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:40.393 | INFO | "malloc" : " 15 GB", | 20260507 13:54:40.393 | INFO | "pinned" : " 28 GB" | 20260507 13:54:40.393 | INFO | } | 20260507 13:54:40.393 | INFO | Perf report for the last 5.13908 seconds: | 20260507 13:54:40.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:40.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 13:54:40.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | PRE_HQ : 13.8%, 1.0, 7.5ms, 0 | 20260507 13:54:40.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:40.393 | INFO | Uploaded 36GiB to GPU in 1.57434s (22.8667GiB/s) | 20260507 13:54:40.393 | INFO | Downloaded 27GiB to GPU in 1.19952s (22.5091GiB/s) | 20260507 13:54:45.532 | INFO | Analyzing chunk frames = [15360,15872) | 20260507 13:54:45.532 | INFO | MemInfo: { | 20260507 13:54:45.532 | INFO | "gpu" : " 16 GB", | 20260507 13:54:45.532 | INFO | "huge" : "", | 20260507 13:54:45.532 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:45.532 | INFO | "malloc" : " 15 GB", | 20260507 13:54:45.532 | INFO | "pinned" : " 28 GB" | 20260507 13:54:45.532 | INFO | } | 20260507 13:54:45.532 | INFO | Perf report for the last 5.13963 seconds: | 20260507 13:54:45.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:45.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.533 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260507 13:54:45.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.533 | INFO | PRE_HQ : 17.9%, 1.0, 9.6ms, 0 | 20260507 13:54:45.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:45.533 | INFO | Uploaded 36GiB to GPU in 1.64585s (21.8731GiB/s) | 20260507 13:54:45.533 | INFO | Downloaded 28GiB to GPU in 1.32333s (21.1587GiB/s) | 20260507 13:54:50.671 | INFO | Analyzing chunk frames = [15872,16384) | 20260507 13:54:50.671 | INFO | MemInfo: { | 20260507 13:54:50.671 | INFO | "gpu" : " 16 GB", | 20260507 13:54:50.671 | INFO | "huge" : "", | 20260507 13:54:50.671 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:50.671 | INFO | "malloc" : " 15 GB", | 20260507 13:54:50.671 | INFO | "pinned" : " 28 GB" | 20260507 13:54:50.671 | INFO | } | 20260507 13:54:50.671 | INFO | Perf report for the last 5.13904 seconds: | 20260507 13:54:50.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:50.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260507 13:54:50.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | PRE_HQ : 20.7%, 1.0, 11.1ms, 0 | 20260507 13:54:50.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:50.672 | INFO | Uploaded 36GiB to GPU in 1.58376s (22.7307GiB/s) | 20260507 13:54:50.672 | INFO | Downloaded 29GiB to GPU in 1.30714s (22.1858GiB/s) | 20260507 13:54:50.672 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 13:54:50.672 | INFO | Total mem usage: 14.6 GiB | 20260507 13:54:50.672 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.166%) | 20260507 13:54:50.672 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: 0.166%) | 20260507 13:54:50.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:54:50.672 | INFO | High Water Breakdown: | 20260507 13:54:50.672 | INFO | 54.8% : 8.04 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 13:54:50.672 | INFO | 45% : 6.6 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 13:54:50.672 | INFO | 0.127% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 13:54:50.672 | INFO | ------------------------------------------ | 20260507 13:54:50.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:54:50.672 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 13:54:50.672 | INFO | Total mem usage: 26.5 GiB | 20260507 13:54:50.672 | INFO | All Time High Water Marks sum: 26.7 GiB (Savings: 0.768%) | 20260507 13:54:50.672 | INFO | Recent High Water Marks Sum: 26.7 GiB (Savings: 0.768%) | 20260507 13:54:50.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:54:50.672 | INFO | High Water Breakdown: | 20260507 13:54:50.672 | INFO | 53.7% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:54:50.672 | INFO | 34.7% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:54:50.672 | INFO | 3.51% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:54:50.672 | INFO | 2.98% : 816 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 13:54:50.672 | INFO | 1.89% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:54:50.672 | INFO | 1.4% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:54:50.672 | INFO | 0.98% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:54:50.672 | INFO | 0.768% : 210 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:54:50.672 | INFO | 0.0146% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 13:54:50.672 | INFO | 0.0146% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:54:50.672 | INFO | 0.00146% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 13:54:50.672 | INFO | 8.96e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 13:54:50.672 | INFO | ------------------------------------------ | 20260507 13:54:50.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:54:50.672 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 13:54:50.672 | INFO | Total mem usage: 27.8 GiB | 20260507 13:54:50.672 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 13:54:50.672 | INFO | Recent High Water Marks Sum: 24.5 GiB (Savings: -13.3%) | 20260507 13:54:50.672 | INFO | Unused Cache: 3.25 GiB (Waste: 11.7%) | 20260507 13:54:50.672 | INFO | High Water Breakdown: | 20260507 13:54:50.672 | INFO | 100% : 24.5 GiB ( 88.3% Of Peak) : WXIPCDataSourceConfig | 20260507 13:54:50.672 | INFO | ------------------------------------------ | 20260507 13:54:50.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:54:50.672 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 13:54:50.672 | INFO | Total mem usage: 14.7 GiB | 20260507 13:54:50.672 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 8.25%) | 20260507 13:54:50.672 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: 8.25%) | 20260507 13:54:50.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:54:50.672 | INFO | High Water Breakdown: | 20260507 13:54:50.672 | INFO | 33.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 13:54:50.672 | INFO | 13.8% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 13:54:50.672 | INFO | 8.8% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 13:54:50.672 | INFO | 7.04% : 1.12 GiB ( 100% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 13:54:50.672 | INFO | 6.45% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 13:54:50.672 | INFO | 5.08% : 832 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:54:50.672 | INFO | 5.04% : 824 MiB ( 100% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 13:54:50.672 | INFO | 3.52% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 13:54:50.672 | INFO | 3.16% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:54:50.672 | INFO | 2.8% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:54:50.672 | INFO | 2.35% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 13:54:50.672 | INFO | 1.81% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:54:50.672 | INFO | 1.71% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 13:54:50.672 | INFO | 1.63% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 13:54:50.672 | INFO | 0.782% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 13:54:50.672 | INFO | 0.782% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 13:54:50.672 | INFO | 0.642% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:54:50.672 | INFO | 0.587% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 13:54:50.672 | INFO | 0.183% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:54:50.672 | INFO | 0.0733% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:54:50.672 | INFO | 0.0244% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:54:50.672 | INFO | 0.00611% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 13:54:50.672 | INFO | ------------------------------------------ | 20260507 13:54:50.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:54:50.672 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 13:54:50.672 | INFO | PerfReport: Gathered 1536 samples. | 20260507 13:54:50.672 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:54:50.672 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 2.22e+01 / 0.00e+00 | 20260507 13:54:50.672 | INFO | PerfReport: 18.08%, Upload -- 1.62e+01 / 0.00e+00, 1.74e+00 / 0.00e+00 | 20260507 13:54:50.673 | INFO | PerfReport: 12.92%, Download -- 1.16e+01 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260507 13:54:50.673 | INFO | PerfReport: 33.33%, Compute -- 2.99e+01 / 0.00e+00, 1.22e+01 / 0.00e+00 | 20260507 13:54:50.673 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 13:54:50.673 | INFO | PerfReport: 0.00%, LockDownload -- 5.18e-04 / 0.00e+00, 2.80e-04 / 0.00e+00 | 20260507 13:54:50.673 | INFO | PerfReport: 35.09%, LockCompute -- 3.15e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260507 13:54:50.673 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 13:54:50.791 | INFO | Starting a round of Baz IO | 20260507 13:54:54.028 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 3.236 seconds | 20260507 13:54:54.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:54:54.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0154575% / 0% / 0% | 20260507 13:54:54.028 | INFO | Baz IO overall: | 20260507 13:54:54.028 | INFO | BytesWritten : 2571427046 | 20260507 13:54:54.028 | INFO | NumEvents : 0 | 20260507 13:54:55.811 | INFO | Analyzing chunk frames = [16384,16896) | 20260507 13:54:55.811 | INFO | MemInfo: { | 20260507 13:54:55.811 | INFO | "gpu" : " 17 GB", | 20260507 13:54:55.811 | INFO | "huge" : "", | 20260507 13:54:55.811 | INFO | "huge_shared" : " 29 GB", | 20260507 13:54:55.811 | INFO | "malloc" : " 15 GB", | 20260507 13:54:55.811 | INFO | "pinned" : " 28 GB" | 20260507 13:54:55.811 | INFO | } | 20260507 13:54:55.811 | INFO | Perf report for the last 5.13926 seconds: | 20260507 13:54:55.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:54:55.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:55.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:55.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:55.811 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260507 13:54:55.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:55.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:54:55.811 | INFO | PRE_HQ : 52.9%, 1.6, 44.4ms, 0 | 20260507 13:54:55.811 | INFO | BAZWRITER : 10.2%, 1.0, 5.3ms, 0 | 20260507 13:54:55.811 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 13:54:55.811 | INFO | Uploaded 36GiB to GPU in 1.73017s (20.8072GiB/s) | 20260507 13:54:55.811 | INFO | Downloaded 33GiB to GPU in 1.60628s (20.5444GiB/s) | 20260507 13:55:00.950 | INFO | Analyzing chunk frames = [16896,17408) | 20260507 13:55:00.950 | INFO | MemInfo: { | 20260507 13:55:00.950 | INFO | "gpu" : " 17 GB", | 20260507 13:55:00.950 | INFO | "huge" : "", | 20260507 13:55:00.950 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:00.950 | INFO | "malloc" : " 15 GB", | 20260507 13:55:00.950 | INFO | "pinned" : " 28 GB" | 20260507 13:55:00.950 | INFO | } | 20260507 13:55:00.950 | INFO | Perf report for the last 5.13947 seconds: | 20260507 13:55:00.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:00.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.950 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260507 13:55:00.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.950 | INFO | PRE_HQ : 33.1%, 1.1, 19.4ms, 0 | 20260507 13:55:00.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:00.951 | INFO | Uploaded 36GiB to GPU in 1.59523s (22.5673GiB/s) | 20260507 13:55:00.951 | INFO | Downloaded 32GiB to GPU in 1.41643s (22.592GiB/s) | 20260507 13:55:06.089 | INFO | Analyzing chunk frames = [17408,17920) | 20260507 13:55:06.089 | INFO | MemInfo: { | 20260507 13:55:06.089 | INFO | "gpu" : " 17 GB", | 20260507 13:55:06.089 | INFO | "huge" : "", | 20260507 13:55:06.089 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:06.090 | INFO | "malloc" : " 15 GB", | 20260507 13:55:06.090 | INFO | "pinned" : " 28 GB" | 20260507 13:55:06.090 | INFO | } | 20260507 13:55:06.090 | INFO | Perf report for the last 5.13932 seconds: | 20260507 13:55:06.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:06.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260507 13:55:06.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | PRE_HQ : 29.5%, 1.0, 15.6ms, 0 | 20260507 13:55:06.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:06.090 | INFO | Uploaded 36GiB to GPU in 1.60294s (22.4587GiB/s) | 20260507 13:55:06.090 | INFO | Downloaded 33GiB to GPU in 1.47468s (22.3777GiB/s) | 20260507 13:55:11.229 | INFO | Analyzing chunk frames = [17920,18432) | 20260507 13:55:11.229 | INFO | MemInfo: { | 20260507 13:55:11.229 | INFO | "gpu" : " 17 GB", | 20260507 13:55:11.229 | INFO | "huge" : "", | 20260507 13:55:11.229 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:11.229 | INFO | "malloc" : " 15 GB", | 20260507 13:55:11.229 | INFO | "pinned" : " 28 GB" | 20260507 13:55:11.229 | INFO | } | 20260507 13:55:11.229 | INFO | Perf report for the last 5.1393 seconds: | 20260507 13:55:11.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:11.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | ANALYSIS : 62.6%, 3.0, 96.9ms, 0 | 20260507 13:55:11.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | PRE_HQ : 31.7%, 1.0, 17.0ms, 0 | 20260507 13:55:11.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:11.229 | INFO | Uploaded 36GiB to GPU in 1.60743s (22.396GiB/s) | 20260507 13:55:11.229 | INFO | Downloaded 34GiB to GPU in 1.52728s (22.2618GiB/s) | 20260507 13:55:16.368 | INFO | Analyzing chunk frames = [18432,18944) | 20260507 13:55:16.368 | INFO | MemInfo: { | 20260507 13:55:16.368 | INFO | "gpu" : " 17 GB", | 20260507 13:55:16.368 | INFO | "huge" : "", | 20260507 13:55:16.368 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:16.368 | INFO | "malloc" : " 15 GB", | 20260507 13:55:16.368 | INFO | "pinned" : " 28 GB" | 20260507 13:55:16.368 | INFO | } | 20260507 13:55:16.368 | INFO | Perf report for the last 5.13955 seconds: | 20260507 13:55:16.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:16.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 13:55:16.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 13:55:16.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:16.369 | INFO | Uploaded 36GiB to GPU in 1.60571s (22.42GiB/s) | 20260507 13:55:16.369 | INFO | Downloaded 34GiB to GPU in 1.52476s (22.2986GiB/s) | 20260507 13:55:21.507 | INFO | Analyzing chunk frames = [18944,19456) | 20260507 13:55:21.507 | INFO | MemInfo: { | 20260507 13:55:21.507 | INFO | "gpu" : " 17 GB", | 20260507 13:55:21.507 | INFO | "huge" : "", | 20260507 13:55:21.507 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:21.507 | INFO | "malloc" : " 15 GB", | 20260507 13:55:21.508 | INFO | "pinned" : " 28 GB" | 20260507 13:55:21.508 | INFO | } | 20260507 13:55:21.508 | INFO | Perf report for the last 5.13909 seconds: | 20260507 13:55:21.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:21.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 13:55:21.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 13:55:21.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:21.508 | INFO | Uploaded 36GiB to GPU in 1.60552s (22.4227GiB/s) | 20260507 13:55:21.508 | INFO | Downloaded 34GiB to GPU in 1.52538s (22.2895GiB/s) | 20260507 13:55:26.647 | INFO | Analyzing chunk frames = [19456,19968) | 20260507 13:55:26.647 | INFO | MemInfo: { | 20260507 13:55:26.647 | INFO | "gpu" : " 17 GB", | 20260507 13:55:26.647 | INFO | "huge" : "", | 20260507 13:55:26.647 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:26.647 | INFO | "malloc" : " 15 GB", | 20260507 13:55:26.647 | INFO | "pinned" : " 28 GB" | 20260507 13:55:26.647 | INFO | } | 20260507 13:55:26.647 | INFO | Perf report for the last 5.13934 seconds: | 20260507 13:55:26.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:26.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 13:55:26.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 13:55:26.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:26.647 | INFO | Uploaded 36GiB to GPU in 1.60689s (22.4035GiB/s) | 20260507 13:55:26.647 | INFO | Downloaded 34GiB to GPU in 1.52645s (22.2738GiB/s) | 20260507 13:55:31.786 | INFO | Analyzing chunk frames = [19968,20480) | 20260507 13:55:31.786 | INFO | MemInfo: { | 20260507 13:55:31.786 | INFO | "gpu" : " 17 GB", | 20260507 13:55:31.786 | INFO | "huge" : "", | 20260507 13:55:31.786 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:31.786 | INFO | "malloc" : " 15 GB", | 20260507 13:55:31.786 | INFO | "pinned" : " 28 GB" | 20260507 13:55:31.786 | INFO | } | 20260507 13:55:31.786 | INFO | Perf report for the last 5.13922 seconds: | 20260507 13:55:31.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:31.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 13:55:31.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 13:55:31.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:31.786 | INFO | Uploaded 36GiB to GPU in 1.60552s (22.4226GiB/s) | 20260507 13:55:31.786 | INFO | Downloaded 34GiB to GPU in 1.5255s (22.2877GiB/s) | 20260507 13:55:31.880 | INFO | Starting a round of Baz IO | 20260507 13:55:35.072 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 3.191 seconds | 20260507 13:55:35.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:55:35.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.73727% / 0% / 0% | 20260507 13:55:35.072 | INFO | Baz IO overall: | 20260507 13:55:35.072 | INFO | BytesWritten : 2571430598 | 20260507 13:55:35.072 | INFO | NumEvents : 0 | 20260507 13:55:36.926 | INFO | Analyzing chunk frames = [20480,20992) | 20260507 13:55:36.926 | INFO | MemInfo: { | 20260507 13:55:36.926 | INFO | "gpu" : " 17 GB", | 20260507 13:55:36.926 | INFO | "huge" : "", | 20260507 13:55:36.926 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:36.926 | INFO | "malloc" : " 15 GB", | 20260507 13:55:36.926 | INFO | "pinned" : " 29 GB" | 20260507 13:55:36.926 | INFO | } | 20260507 13:55:36.926 | INFO | Perf report for the last 5.13947 seconds: | 20260507 13:55:36.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:36.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:36.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:36.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:36.926 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260507 13:55:36.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:36.926 | INFO | RT_METRICS : 31.4%, 1.0, 21.8ms, 0 | 20260507 13:55:36.926 | INFO | PRE_HQ : 55.6%, 1.8, 53.2ms, 0 | 20260507 13:55:36.926 | INFO | BAZWRITER : 10.4%, 1.0, 5.4ms, 0 | 20260507 13:55:36.926 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260507 13:55:36.926 | INFO | Uploaded 36GiB to GPU in 1.70409s (21.1257GiB/s) | 20260507 13:55:36.926 | INFO | Downloaded 38GiB to GPU in 1.77187s (21.4463GiB/s) | 20260507 13:55:42.065 | INFO | Analyzing chunk frames = [20992,21504) | 20260507 13:55:42.065 | INFO | MemInfo: { | 20260507 13:55:42.065 | INFO | "gpu" : " 17 GB", | 20260507 13:55:42.065 | INFO | "huge" : "", | 20260507 13:55:42.065 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:42.065 | INFO | "malloc" : " 15 GB", | 20260507 13:55:42.065 | INFO | "pinned" : " 29 GB" | 20260507 13:55:42.065 | INFO | } | 20260507 13:55:42.065 | INFO | Perf report for the last 5.13951 seconds: | 20260507 13:55:42.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:42.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260507 13:55:42.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | PRE_HQ : 35.7%, 1.0, 19.4ms, 0 | 20260507 13:55:42.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:42.065 | INFO | Uploaded 36GiB to GPU in 1.60233s (22.4673GiB/s) | 20260507 13:55:42.065 | INFO | Downloaded 34GiB to GPU in 1.52183s (22.3415GiB/s) | 20260507 13:55:47.204 | INFO | Analyzing chunk frames = [21504,22016) | 20260507 13:55:47.204 | INFO | MemInfo: { | 20260507 13:55:47.204 | INFO | "gpu" : " 17 GB", | 20260507 13:55:47.204 | INFO | "huge" : "", | 20260507 13:55:47.204 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:47.204 | INFO | "malloc" : " 15 GB", | 20260507 13:55:47.204 | INFO | "pinned" : " 29 GB" | 20260507 13:55:47.204 | INFO | } | 20260507 13:55:47.204 | INFO | Perf report for the last 5.13929 seconds: | 20260507 13:55:47.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:47.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 13:55:47.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 13:55:47.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:47.205 | INFO | Uploaded 36GiB to GPU in 1.60679s (22.405GiB/s) | 20260507 13:55:47.205 | INFO | Downloaded 34GiB to GPU in 1.52654s (22.2725GiB/s) | 20260507 13:55:52.343 | INFO | Analyzing chunk frames = [22016,22528) | 20260507 13:55:52.343 | INFO | MemInfo: { | 20260507 13:55:52.343 | INFO | "gpu" : " 17 GB", | 20260507 13:55:52.343 | INFO | "huge" : "", | 20260507 13:55:52.344 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:52.344 | INFO | "malloc" : " 15 GB", | 20260507 13:55:52.344 | INFO | "pinned" : " 29 GB" | 20260507 13:55:52.344 | INFO | } | 20260507 13:55:52.344 | INFO | Perf report for the last 5.1391 seconds: | 20260507 13:55:52.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:52.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 13:55:52.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 13:55:52.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:52.344 | INFO | Uploaded 36GiB to GPU in 1.60651s (22.4089GiB/s) | 20260507 13:55:52.344 | INFO | Downloaded 34GiB to GPU in 1.52739s (22.2601GiB/s) | 20260507 13:55:57.483 | INFO | Analyzing chunk frames = [22528,23040) | 20260507 13:55:57.483 | INFO | MemInfo: { | 20260507 13:55:57.483 | INFO | "gpu" : " 17 GB", | 20260507 13:55:57.483 | INFO | "huge" : "", | 20260507 13:55:57.483 | INFO | "huge_shared" : " 29 GB", | 20260507 13:55:57.483 | INFO | "malloc" : " 15 GB", | 20260507 13:55:57.483 | INFO | "pinned" : " 29 GB" | 20260507 13:55:57.483 | INFO | } | 20260507 13:55:57.483 | INFO | Perf report for the last 5.13934 seconds: | 20260507 13:55:57.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:55:57.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 13:55:57.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 13:55:57.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:55:57.483 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4315GiB/s) | 20260507 13:55:57.483 | INFO | Downloaded 34GiB to GPU in 1.52405s (22.309GiB/s) | 20260507 13:56:02.622 | INFO | Analyzing chunk frames = [23040,23552) | 20260507 13:56:02.622 | INFO | MemInfo: { | 20260507 13:56:02.622 | INFO | "gpu" : " 17 GB", | 20260507 13:56:02.622 | INFO | "huge" : "", | 20260507 13:56:02.622 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:02.622 | INFO | "malloc" : " 15 GB", | 20260507 13:56:02.622 | INFO | "pinned" : " 29 GB" | 20260507 13:56:02.622 | INFO | } | 20260507 13:56:02.622 | INFO | Perf report for the last 5.13947 seconds: | 20260507 13:56:02.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:02.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.622 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 13:56:02.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.623 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 13:56:02.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:02.623 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.4239GiB/s) | 20260507 13:56:02.623 | INFO | Downloaded 34GiB to GPU in 1.52562s (22.2861GiB/s) | 20260507 13:56:07.762 | INFO | Analyzing chunk frames = [23552,24064) | 20260507 13:56:07.762 | INFO | MemInfo: { | 20260507 13:56:07.762 | INFO | "gpu" : " 17 GB", | 20260507 13:56:07.762 | INFO | "huge" : "", | 20260507 13:56:07.762 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:07.762 | INFO | "malloc" : " 15 GB", | 20260507 13:56:07.762 | INFO | "pinned" : " 29 GB" | 20260507 13:56:07.762 | INFO | } | 20260507 13:56:07.762 | INFO | Perf report for the last 5.13927 seconds: | 20260507 13:56:07.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:07.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 13:56:07.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 13:56:07.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:07.762 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4257GiB/s) | 20260507 13:56:07.762 | INFO | Downloaded 34GiB to GPU in 1.52545s (22.2885GiB/s) | 20260507 13:56:12.901 | INFO | Analyzing chunk frames = [24064,24576) | 20260507 13:56:12.901 | INFO | MemInfo: { | 20260507 13:56:12.901 | INFO | "gpu" : " 17 GB", | 20260507 13:56:12.901 | INFO | "huge" : "", | 20260507 13:56:12.901 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:12.901 | INFO | "malloc" : " 15 GB", | 20260507 13:56:12.901 | INFO | "pinned" : " 29 GB" | 20260507 13:56:12.901 | INFO | } | 20260507 13:56:12.901 | INFO | Perf report for the last 5.13933 seconds: | 20260507 13:56:12.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:12.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 13:56:12.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 13:56:12.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:12.901 | INFO | Uploaded 36GiB to GPU in 1.60551s (22.4228GiB/s) | 20260507 13:56:12.901 | INFO | Downloaded 34GiB to GPU in 1.52532s (22.2904GiB/s) | 20260507 13:56:12.902 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 13:56:12.902 | INFO | Total mem usage: 14.7 GiB | 20260507 13:56:12.902 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260507 13:56:12.902 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: 0.0524%) | 20260507 13:56:12.902 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:56:12.902 | INFO | High Water Breakdown: | 20260507 13:56:12.902 | INFO | 54.8% : 8.04 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 13:56:12.902 | INFO | 45.1% : 6.61 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 13:56:12.902 | INFO | 0.127% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 13:56:12.902 | INFO | ------------------------------------------ | 20260507 13:56:12.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:56:12.902 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 13:56:12.902 | INFO | Total mem usage: 26.5 GiB | 20260507 13:56:12.902 | INFO | All Time High Water Marks sum: 27 GiB (Savings: 1.91%) | 20260507 13:56:12.902 | INFO | Recent High Water Marks Sum: 26.2 GiB (Savings: -1.07%) | 20260507 13:56:12.902 | INFO | Unused Cache: 1.03 GiB (Waste: 3.9%) | 20260507 13:56:12.902 | INFO | High Water Breakdown: | 20260507 13:56:12.902 | INFO | 54.7% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:56:12.902 | INFO | 35.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:56:12.902 | INFO | 3.57% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:56:12.902 | INFO | 1.93% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:56:12.902 | INFO | 1.56% : 418 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:56:12.902 | INFO | 1.43% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:56:12.902 | INFO | 0.998% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:56:12.902 | INFO | 0.408% : 110 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 13:56:12.902 | INFO | 0.0149% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 13:56:12.902 | INFO | 0.0149% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:56:12.902 | INFO | 0.00148% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 13:56:12.902 | INFO | 9.13e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 13:56:12.902 | INFO | 8.52e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 13:56:12.902 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 13:56:12.902 | INFO | ------------------------------------------ | 20260507 13:56:12.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:56:12.902 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 13:56:12.902 | INFO | Total mem usage: 27.8 GiB | 20260507 13:56:12.902 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 13:56:12.902 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -13.8%) | 20260507 13:56:12.902 | INFO | Unused Cache: 3.38 GiB (Waste: 12.2%) | 20260507 13:56:12.902 | INFO | High Water Breakdown: | 20260507 13:56:12.902 | INFO | 100% : 24.4 GiB ( 87.8% Of Peak) : WXIPCDataSourceConfig | 20260507 13:56:12.902 | INFO | ------------------------------------------ | 20260507 13:56:12.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:56:12.902 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 13:56:12.902 | INFO | Total mem usage: 15.7 GiB | 20260507 13:56:12.902 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 13:56:12.902 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: 7.99%) | 20260507 13:56:12.902 | INFO | Unused Cache: 786 MiB (Waste: 4.89%) | 20260507 13:56:12.902 | INFO | High Water Breakdown: | 20260507 13:56:12.902 | INFO | 31.6% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 13:56:12.902 | INFO | 12.9% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 13:56:12.902 | INFO | 8.25% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 13:56:12.902 | INFO | 8.14% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:56:12.902 | INFO | 6.05% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 13:56:12.902 | INFO | 5.92% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 13:56:12.902 | INFO | 4.73% : 824 MiB ( 100% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 13:56:12.902 | INFO | 3.3% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 13:56:12.902 | INFO | 3.3% : 576 MiB ( 50% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 13:56:12.902 | INFO | 2.96% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:56:12.902 | INFO | 2.63% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:56:12.902 | INFO | 2.2% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 13:56:12.902 | INFO | 1.7% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:56:12.902 | INFO | 1.6% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 13:56:12.902 | INFO | 1.53% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 13:56:12.902 | INFO | 0.734% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 13:56:12.902 | INFO | 0.734% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 13:56:12.902 | INFO | 0.602% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:56:12.902 | INFO | 0.55% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 13:56:12.902 | INFO | 0.209% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 13:56:12.902 | INFO | 0.172% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:56:12.902 | INFO | 0.0688% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:56:12.902 | INFO | 0.0229% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:56:12.903 | INFO | 0.00573% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 13:56:12.903 | INFO | 1.31e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 13:56:12.903 | INFO | ------------------------------------------ | 20260507 13:56:12.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:56:12.903 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 13:56:12.903 | INFO | PerfReport: Gathered 288 samples. | 20260507 13:56:12.903 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:56:12.903 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.17e+01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 0.29%, AggregateStats -- 8.59e-02 / 0.00e+00, 3.38e-03 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 3.03%, Histogram -- 9.08e-01 / 0.00e+00, 4.72e-02 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 16.24%, DME -- 3.90e+01 / 0.00e+00, 1.52e+00 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.08e-02 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.20e+00 / 0.00e+00, 1.26e-01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 3.40%, Metrics -- 1.02e+00 / 0.00e+00, 1.35e-01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.11e-01 / 0.00e+00, 6.18e-02 / 0.00e+00 | 20260507 13:56:12.903 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 13:56:12.903 | INFO | PerfReport: Gathered 1536 samples. | 20260507 13:56:12.903 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:56:12.903 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.72e+01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 17.53%, Upload -- 1.69e+01 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 16.64%, Download -- 1.60e+01 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 33.31%, Compute -- 3.20e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 0.52%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 0.02%, LockDownload -- 1.89e-02 / 0.00e+00, 4.15e-01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 31.96%, LockCompute -- 3.07e+01 / 0.00e+00, 2.42e+01 / 0.00e+00 | 20260507 13:56:12.903 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260507 13:56:13.001 | INFO | Starting a round of Baz IO | 20260507 13:56:15.894 | INFO | Populating RTMetrics report | 20260507 13:56:15.894 | INFO | RTMetrics report populated | 20260507 13:56:15.894 | INFO | Opening tmp RTMetrics json file | 20260507 13:56:15.895 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.893 seconds | 20260507 13:56:15.895 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:56:15.895 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.38195% / 0% / 0% | 20260507 13:56:15.895 | INFO | Baz IO overall: | 20260507 13:56:15.895 | INFO | BytesWritten : 2571434150 | 20260507 13:56:15.895 | INFO | NumEvents : 0 | 20260507 13:56:15.896 | INFO | Serializing RTMetrics to json | 20260507 13:56:15.897 | INFO | Serialization done, swapping tmp file now... | 20260507 13:56:15.897 | INFO | RTMetrics done | 20260507 13:56:18.041 | INFO | Analyzing chunk frames = [24576,25088) | 20260507 13:56:18.041 | INFO | MemInfo: { | 20260507 13:56:18.041 | INFO | "gpu" : " 17 GB", | 20260507 13:56:18.041 | INFO | "huge" : "", | 20260507 13:56:18.041 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:18.041 | INFO | "malloc" : " 15 GB", | 20260507 13:56:18.041 | INFO | "pinned" : " 29 GB" | 20260507 13:56:18.041 | INFO | } | 20260507 13:56:18.041 | INFO | Perf report for the last 5.13973 seconds: | 20260507 13:56:18.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:18.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:18.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:18.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:18.041 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260507 13:56:18.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:18.041 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 13:56:18.041 | INFO | PRE_HQ : 54.9%, 1.9, 52.8ms, 0 | 20260507 13:56:18.041 | INFO | BAZWRITER : 11.1%, 1.0, 5.7ms, 0 | 20260507 13:56:18.041 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 13:56:18.041 | INFO | Uploaded 36GiB to GPU in 1.67633s (21.4755GiB/s) | 20260507 13:56:18.041 | INFO | Downloaded 38GiB to GPU in 1.74148s (21.8205GiB/s) | 20260507 13:56:23.180 | INFO | Analyzing chunk frames = [25088,25600) | 20260507 13:56:23.180 | INFO | MemInfo: { | 20260507 13:56:23.180 | INFO | "gpu" : " 17 GB", | 20260507 13:56:23.180 | INFO | "huge" : "", | 20260507 13:56:23.180 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:23.180 | INFO | "malloc" : " 15 GB", | 20260507 13:56:23.180 | INFO | "pinned" : " 29 GB" | 20260507 13:56:23.180 | INFO | } | 20260507 13:56:23.180 | INFO | Perf report for the last 5.13891 seconds: | 20260507 13:56:23.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:23.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 13:56:23.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | PRE_HQ : 34.5%, 1.1, 19.1ms, 0 | 20260507 13:56:23.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:23.180 | INFO | Uploaded 36GiB to GPU in 1.60293s (22.4589GiB/s) | 20260507 13:56:23.180 | INFO | Downloaded 34GiB to GPU in 1.52134s (22.3488GiB/s) | 20260507 13:56:28.319 | INFO | Analyzing chunk frames = [25600,26112) | 20260507 13:56:28.319 | INFO | MemInfo: { | 20260507 13:56:28.319 | INFO | "gpu" : " 17 GB", | 20260507 13:56:28.319 | INFO | "huge" : "", | 20260507 13:56:28.319 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:28.319 | INFO | "malloc" : " 15 GB", | 20260507 13:56:28.319 | INFO | "pinned" : " 29 GB" | 20260507 13:56:28.319 | INFO | } | 20260507 13:56:28.319 | INFO | Perf report for the last 5.13926 seconds: | 20260507 13:56:28.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:28.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260507 13:56:28.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 13:56:28.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:28.319 | INFO | Uploaded 36GiB to GPU in 1.6066s (22.4075GiB/s) | 20260507 13:56:28.319 | INFO | Downloaded 34GiB to GPU in 1.5258s (22.2834GiB/s) | 20260507 13:56:33.458 | INFO | Analyzing chunk frames = [26112,26624) | 20260507 13:56:33.458 | INFO | MemInfo: { | 20260507 13:56:33.458 | INFO | "gpu" : " 17 GB", | 20260507 13:56:33.458 | INFO | "huge" : "", | 20260507 13:56:33.458 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:33.458 | INFO | "malloc" : " 15 GB", | 20260507 13:56:33.458 | INFO | "pinned" : " 29 GB" | 20260507 13:56:33.458 | INFO | } | 20260507 13:56:33.458 | INFO | Perf report for the last 5.13929 seconds: | 20260507 13:56:33.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:33.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.458 | INFO | ANALYSIS : 56.3%, 3.0, 87.2ms, 0 | 20260507 13:56:33.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.458 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 13:56:33.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:33.459 | INFO | Uploaded 36GiB to GPU in 1.60727s (22.3983GiB/s) | 20260507 13:56:33.459 | INFO | Downloaded 34GiB to GPU in 1.5263s (22.276GiB/s) | 20260507 13:56:38.598 | INFO | Analyzing chunk frames = [26624,27136) | 20260507 13:56:38.598 | INFO | MemInfo: { | 20260507 13:56:38.598 | INFO | "gpu" : " 17 GB", | 20260507 13:56:38.598 | INFO | "huge" : "", | 20260507 13:56:38.598 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:38.598 | INFO | "malloc" : " 15 GB", | 20260507 13:56:38.598 | INFO | "pinned" : " 29 GB" | 20260507 13:56:38.598 | INFO | } | 20260507 13:56:38.598 | INFO | Perf report for the last 5.13976 seconds: | 20260507 13:56:38.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:38.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.598 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 13:56:38.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.598 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 13:56:38.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:38.599 | INFO | Uploaded 36GiB to GPU in 1.60438s (22.4386GiB/s) | 20260507 13:56:38.599 | INFO | Downloaded 34GiB to GPU in 1.52353s (22.3167GiB/s) | 20260507 13:56:43.737 | INFO | Analyzing chunk frames = [27136,27648) | 20260507 13:56:43.737 | INFO | MemInfo: { | 20260507 13:56:43.737 | INFO | "gpu" : " 17 GB", | 20260507 13:56:43.737 | INFO | "huge" : "", | 20260507 13:56:43.737 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:43.737 | INFO | "malloc" : " 15 GB", | 20260507 13:56:43.737 | INFO | "pinned" : " 29 GB" | 20260507 13:56:43.737 | INFO | } | 20260507 13:56:43.737 | INFO | Perf report for the last 5.13913 seconds: | 20260507 13:56:43.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:43.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.737 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 13:56:43.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.737 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 13:56:43.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:43.738 | INFO | Uploaded 36GiB to GPU in 1.60499s (22.4301GiB/s) | 20260507 13:56:43.738 | INFO | Downloaded 34GiB to GPU in 1.52424s (22.3062GiB/s) | 20260507 13:56:48.876 | INFO | Analyzing chunk frames = [27648,28160) | 20260507 13:56:48.876 | INFO | MemInfo: { | 20260507 13:56:48.876 | INFO | "gpu" : " 17 GB", | 20260507 13:56:48.876 | INFO | "huge" : "", | 20260507 13:56:48.876 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:48.876 | INFO | "malloc" : " 15 GB", | 20260507 13:56:48.877 | INFO | "pinned" : " 29 GB" | 20260507 13:56:48.877 | INFO | } | 20260507 13:56:48.877 | INFO | Perf report for the last 5.13939 seconds: | 20260507 13:56:48.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:48.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 13:56:48.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 13:56:48.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:48.877 | INFO | Uploaded 36GiB to GPU in 1.60523s (22.4267GiB/s) | 20260507 13:56:48.877 | INFO | Downloaded 34GiB to GPU in 1.52499s (22.2952GiB/s) | 20260507 13:56:54.016 | INFO | Analyzing chunk frames = [28160,28672) | 20260507 13:56:54.016 | INFO | MemInfo: { | 20260507 13:56:54.016 | INFO | "gpu" : " 17 GB", | 20260507 13:56:54.016 | INFO | "huge" : "", | 20260507 13:56:54.016 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:54.016 | INFO | "malloc" : " 15 GB", | 20260507 13:56:54.016 | INFO | "pinned" : " 29 GB" | 20260507 13:56:54.016 | INFO | } | 20260507 13:56:54.016 | INFO | Perf report for the last 5.13921 seconds: | 20260507 13:56:54.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:54.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 13:56:54.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 13:56:54.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:54.016 | INFO | Uploaded 36GiB to GPU in 1.60601s (22.4158GiB/s) | 20260507 13:56:54.016 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.289GiB/s) | 20260507 13:56:54.113 | INFO | Starting a round of Baz IO | 20260507 13:56:57.008 | INFO | Populating RTMetrics report | 20260507 13:56:57.008 | INFO | RTMetrics report populated | 20260507 13:56:57.008 | INFO | Opening tmp RTMetrics json file | 20260507 13:56:57.009 | INFO | Serializing RTMetrics to json | 20260507 13:56:57.009 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.896 seconds | 20260507 13:56:57.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:56:57.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.70272% / 0% / 0% | 20260507 13:56:57.009 | INFO | Baz IO overall: | 20260507 13:56:57.009 | INFO | BytesWritten : 2571437702 | 20260507 13:56:57.009 | INFO | NumEvents : 0 | 20260507 13:56:57.011 | INFO | Serialization done, swapping tmp file now... | 20260507 13:56:57.011 | INFO | RTMetrics done | 20260507 13:56:59.155 | INFO | Analyzing chunk frames = [28672,29184) | 20260507 13:56:59.155 | INFO | MemInfo: { | 20260507 13:56:59.155 | INFO | "gpu" : " 17 GB", | 20260507 13:56:59.155 | INFO | "huge" : "", | 20260507 13:56:59.155 | INFO | "huge_shared" : " 29 GB", | 20260507 13:56:59.155 | INFO | "malloc" : " 15 GB", | 20260507 13:56:59.155 | INFO | "pinned" : " 29 GB" | 20260507 13:56:59.155 | INFO | } | 20260507 13:56:59.155 | INFO | Perf report for the last 5.1395 seconds: | 20260507 13:56:59.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:56:59.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:59.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:59.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:59.155 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 13:56:59.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:56:59.155 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 13:56:59.155 | INFO | PRE_HQ : 54.7%, 1.8, 52.5ms, 0 | 20260507 13:56:59.155 | INFO | BAZWRITER : 10.2%, 1.0, 5.2ms, 0 | 20260507 13:56:59.155 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260507 13:56:59.156 | INFO | Uploaded 36GiB to GPU in 1.6518s (21.7944GiB/s) | 20260507 13:56:59.156 | INFO | Downloaded 38GiB to GPU in 1.71531s (22.1534GiB/s) | 20260507 13:57:04.294 | INFO | Analyzing chunk frames = [29184,29696) | 20260507 13:57:04.294 | INFO | MemInfo: { | 20260507 13:57:04.294 | INFO | "gpu" : " 17 GB", | 20260507 13:57:04.294 | INFO | "huge" : "", | 20260507 13:57:04.294 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:04.294 | INFO | "malloc" : " 15 GB", | 20260507 13:57:04.295 | INFO | "pinned" : " 29 GB" | 20260507 13:57:04.295 | INFO | } | 20260507 13:57:04.295 | INFO | Perf report for the last 5.13922 seconds: | 20260507 13:57:04.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:04.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 13:57:04.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | PRE_HQ : 32.3%, 1.0, 17.0ms, 0 | 20260507 13:57:04.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:04.295 | INFO | Uploaded 36GiB to GPU in 1.60292s (22.459GiB/s) | 20260507 13:57:04.295 | INFO | Downloaded 34GiB to GPU in 1.5213s (22.3493GiB/s) | 20260507 13:57:09.434 | INFO | Analyzing chunk frames = [29696,30208) | 20260507 13:57:09.434 | INFO | MemInfo: { | 20260507 13:57:09.434 | INFO | "gpu" : " 17 GB", | 20260507 13:57:09.434 | INFO | "huge" : "", | 20260507 13:57:09.434 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:09.434 | INFO | "malloc" : " 15 GB", | 20260507 13:57:09.434 | INFO | "pinned" : " 29 GB" | 20260507 13:57:09.434 | INFO | } | 20260507 13:57:09.434 | INFO | Perf report for the last 5.1392 seconds: | 20260507 13:57:09.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:09.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 13:57:09.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 13:57:09.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:09.434 | INFO | Uploaded 36GiB to GPU in 1.60626s (22.4123GiB/s) | 20260507 13:57:09.434 | INFO | Downloaded 34GiB to GPU in 1.5266s (22.2716GiB/s) | 20260507 13:57:14.573 | INFO | Analyzing chunk frames = [30208,30720) | 20260507 13:57:14.573 | INFO | MemInfo: { | 20260507 13:57:14.573 | INFO | "gpu" : " 17 GB", | 20260507 13:57:14.573 | INFO | "huge" : "", | 20260507 13:57:14.573 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:14.573 | INFO | "malloc" : " 15 GB", | 20260507 13:57:14.573 | INFO | "pinned" : " 29 GB" | 20260507 13:57:14.573 | INFO | } | 20260507 13:57:14.573 | INFO | Perf report for the last 5.1395 seconds: | 20260507 13:57:14.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:14.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.573 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 13:57:14.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.573 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 13:57:14.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:14.574 | INFO | Uploaded 36GiB to GPU in 1.60615s (22.4139GiB/s) | 20260507 13:57:14.574 | INFO | Downloaded 34GiB to GPU in 1.52551s (22.2876GiB/s) | 20260507 13:57:19.712 | INFO | Analyzing chunk frames = [30720,31232) | 20260507 13:57:19.712 | INFO | MemInfo: { | 20260507 13:57:19.712 | INFO | "gpu" : " 17 GB", | 20260507 13:57:19.712 | INFO | "huge" : "", | 20260507 13:57:19.712 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:19.712 | INFO | "malloc" : " 15 GB", | 20260507 13:57:19.712 | INFO | "pinned" : " 29 GB" | 20260507 13:57:19.712 | INFO | } | 20260507 13:57:19.713 | INFO | Perf report for the last 5.13922 seconds: | 20260507 13:57:19.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:19.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 13:57:19.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 13:57:19.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:19.713 | INFO | Uploaded 36GiB to GPU in 1.60447s (22.4373GiB/s) | 20260507 13:57:19.713 | INFO | Downloaded 34GiB to GPU in 1.5236s (22.3156GiB/s) | 20260507 13:57:24.852 | INFO | Analyzing chunk frames = [31232,31744) | 20260507 13:57:24.852 | INFO | MemInfo: { | 20260507 13:57:24.852 | INFO | "gpu" : " 17 GB", | 20260507 13:57:24.852 | INFO | "huge" : "", | 20260507 13:57:24.852 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:24.852 | INFO | "malloc" : " 15 GB", | 20260507 13:57:24.852 | INFO | "pinned" : " 29 GB" | 20260507 13:57:24.852 | INFO | } | 20260507 13:57:24.852 | INFO | Perf report for the last 5.13929 seconds: | 20260507 13:57:24.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:24.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 13:57:24.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 13:57:24.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:24.852 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4388GiB/s) | 20260507 13:57:24.852 | INFO | Downloaded 34GiB to GPU in 1.52408s (22.3086GiB/s) | 20260507 13:57:29.991 | INFO | Analyzing chunk frames = [31744,32256) | 20260507 13:57:29.991 | INFO | MemInfo: { | 20260507 13:57:29.991 | INFO | "gpu" : " 17 GB", | 20260507 13:57:29.991 | INFO | "huge" : "", | 20260507 13:57:29.991 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:29.991 | INFO | "malloc" : " 15 GB", | 20260507 13:57:29.991 | INFO | "pinned" : " 29 GB" | 20260507 13:57:29.991 | INFO | } | 20260507 13:57:29.991 | INFO | Perf report for the last 5.13933 seconds: | 20260507 13:57:29.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:29.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 13:57:29.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 13:57:29.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:29.991 | INFO | Uploaded 36GiB to GPU in 1.60447s (22.4374GiB/s) | 20260507 13:57:29.991 | INFO | Downloaded 34GiB to GPU in 1.52425s (22.306GiB/s) | 20260507 13:57:35.130 | INFO | Analyzing chunk frames = [32256,32768) | 20260507 13:57:35.130 | INFO | MemInfo: { | 20260507 13:57:35.130 | INFO | "gpu" : " 17 GB", | 20260507 13:57:35.130 | INFO | "huge" : "", | 20260507 13:57:35.130 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:35.130 | INFO | "malloc" : " 15 GB", | 20260507 13:57:35.131 | INFO | "pinned" : " 29 GB" | 20260507 13:57:35.131 | INFO | } | 20260507 13:57:35.131 | INFO | Perf report for the last 5.13941 seconds: | 20260507 13:57:35.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:35.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 13:57:35.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 13:57:35.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:35.131 | INFO | Uploaded 36GiB to GPU in 1.60515s (22.4278GiB/s) | 20260507 13:57:35.131 | INFO | Downloaded 34GiB to GPU in 1.525s (22.2951GiB/s) | 20260507 13:57:35.131 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 13:57:35.131 | INFO | Total mem usage: 14.7 GiB | 20260507 13:57:35.131 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260507 13:57:35.131 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.685%) | 20260507 13:57:35.131 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:57:35.131 | INFO | High Water Breakdown: | 20260507 13:57:35.131 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 13:57:35.131 | INFO | 45.2% : 6.59 GiB ( 99.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 13:57:35.131 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 13:57:35.131 | INFO | ------------------------------------------ | 20260507 13:57:35.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:57:35.131 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 13:57:35.131 | INFO | Total mem usage: 26.5 GiB | 20260507 13:57:35.131 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 13:57:35.131 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 13:57:35.131 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 13:57:35.131 | INFO | High Water Breakdown: | 20260507 13:57:35.131 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:57:35.131 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:57:35.131 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:57:35.131 | INFO | 1.6% : 418 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:57:35.131 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:57:35.131 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 13:57:35.131 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 13:57:35.132 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 13:57:35.132 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 13:57:35.132 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 13:57:35.132 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 13:57:35.132 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 13:57:35.132 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 13:57:35.132 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 13:57:35.132 | INFO | ------------------------------------------ | 20260507 13:57:35.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:57:35.132 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 13:57:35.132 | INFO | Total mem usage: 27.8 GiB | 20260507 13:57:35.132 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 13:57:35.132 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 13:57:35.132 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 13:57:35.132 | INFO | High Water Breakdown: | 20260507 13:57:35.132 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 13:57:35.132 | INFO | ------------------------------------------ | 20260507 13:57:35.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:57:35.132 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 13:57:35.132 | INFO | Total mem usage: 15.7 GiB | 20260507 13:57:35.132 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 13:57:35.132 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 13:57:35.132 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 13:57:35.132 | INFO | High Water Breakdown: | 20260507 13:57:35.132 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 13:57:35.132 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 13:57:35.132 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 13:57:35.132 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:57:35.132 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 13:57:35.132 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 13:57:35.132 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 13:57:35.132 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 13:57:35.132 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:57:35.132 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:57:35.132 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 13:57:35.132 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:57:35.132 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 13:57:35.132 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 13:57:35.132 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 13:57:35.132 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 13:57:35.132 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:57:35.132 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 13:57:35.132 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 13:57:35.132 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:57:35.132 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:57:35.132 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:57:35.132 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 13:57:35.132 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 13:57:35.132 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 13:57:35.132 | INFO | ------------------------------------------ | 20260507 13:57:35.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:57:35.132 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 13:57:35.132 | INFO | PerfReport: Gathered 1536 samples. | 20260507 13:57:35.132 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:57:35.132 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 0.29%, AggregateStats -- 8.60e-02 / 0.00e+00, 4.04e-03 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 3.02%, Histogram -- 9.08e-01 / 0.00e+00, 5.10e-02 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.51e-02 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / 0.00e+00, 1.31e-01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / 0.00e+00, 1.80e-01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.16e-01 / 0.00e+00, 6.47e-02 / 0.00e+00 | 20260507 13:57:35.132 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 13:57:35.132 | INFO | PerfReport: Gathered 1536 samples. | 20260507 13:57:35.132 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:57:35.132 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 18.62%, Upload -- 1.68e+01 / 0.00e+00, 7.08e-01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 17.89%, Download -- 1.61e+01 / 0.00e+00, 9.63e-01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 33.29%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 0.00%, LockDownload -- 4.40e-04 / 0.00e+00, 1.31e-04 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 29.63%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 13:57:35.132 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260507 13:57:35.230 | INFO | Starting a round of Baz IO | 20260507 13:57:38.125 | INFO | Populating RTMetrics report | 20260507 13:57:38.126 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.895 seconds | 20260507 13:57:38.126 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:57:38.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.14745% / 0% / 0% | 20260507 13:57:38.126 | INFO | Baz IO overall: | 20260507 13:57:38.126 | INFO | BytesWritten : 2571441254 | 20260507 13:57:38.126 | INFO | RTMetrics report populated | 20260507 13:57:38.126 | INFO | NumEvents : 0 | 20260507 13:57:38.126 | INFO | Opening tmp RTMetrics json file | 20260507 13:57:38.127 | INFO | Serializing RTMetrics to json | 20260507 13:57:38.129 | INFO | Serialization done, swapping tmp file now... | 20260507 13:57:38.129 | INFO | RTMetrics done | 20260507 13:57:40.270 | INFO | Analyzing chunk frames = [32768,33280) | 20260507 13:57:40.270 | INFO | MemInfo: { | 20260507 13:57:40.270 | INFO | "gpu" : " 17 GB", | 20260507 13:57:40.270 | INFO | "huge" : "", | 20260507 13:57:40.270 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:40.270 | INFO | "malloc" : " 15 GB", | 20260507 13:57:40.270 | INFO | "pinned" : " 29 GB" | 20260507 13:57:40.270 | INFO | } | 20260507 13:57:40.270 | INFO | Perf report for the last 5.13933 seconds: | 20260507 13:57:40.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:40.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:40.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:40.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:40.270 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260507 13:57:40.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:40.270 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 13:57:40.270 | INFO | PRE_HQ : 54.6%, 1.9, 52.7ms, 0 | 20260507 13:57:40.270 | INFO | BAZWRITER : 10.5%, 1.0, 5.4ms, 0 | 20260507 13:57:40.270 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 13:57:40.270 | INFO | Uploaded 36GiB to GPU in 1.64687s (21.8597GiB/s) | 20260507 13:57:40.270 | INFO | Downloaded 38GiB to GPU in 1.71088s (22.2108GiB/s) | 20260507 13:57:45.409 | INFO | Analyzing chunk frames = [33280,33792) | 20260507 13:57:45.409 | INFO | MemInfo: { | 20260507 13:57:45.409 | INFO | "gpu" : " 17 GB", | 20260507 13:57:45.409 | INFO | "huge" : "", | 20260507 13:57:45.409 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:45.410 | INFO | "malloc" : " 15 GB", | 20260507 13:57:45.410 | INFO | "pinned" : " 29 GB" | 20260507 13:57:45.410 | INFO | } | 20260507 13:57:45.410 | INFO | Perf report for the last 5.13966 seconds: | 20260507 13:57:45.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:45.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 13:57:45.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | PRE_HQ : 32.2%, 1.0, 16.9ms, 0 | 20260507 13:57:45.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:45.410 | INFO | Uploaded 36GiB to GPU in 1.60205s (22.4712GiB/s) | 20260507 13:57:45.410 | INFO | Downloaded 34GiB to GPU in 1.52115s (22.3516GiB/s) | 20260507 13:57:50.548 | INFO | Analyzing chunk frames = [33792,34304) | 20260507 13:57:50.548 | INFO | MemInfo: { | 20260507 13:57:50.548 | INFO | "gpu" : " 17 GB", | 20260507 13:57:50.549 | INFO | "huge" : "", | 20260507 13:57:50.549 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:50.549 | INFO | "malloc" : " 15 GB", | 20260507 13:57:50.549 | INFO | "pinned" : " 29 GB" | 20260507 13:57:50.549 | INFO | } | 20260507 13:57:50.549 | INFO | Perf report for the last 5.13902 seconds: | 20260507 13:57:50.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:50.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 13:57:50.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 13:57:50.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:50.549 | INFO | Uploaded 36GiB to GPU in 1.60692s (22.4031GiB/s) | 20260507 13:57:50.549 | INFO | Downloaded 34GiB to GPU in 1.52722s (22.2627GiB/s) | 20260507 13:57:55.688 | INFO | Analyzing chunk frames = [34304,34816) | 20260507 13:57:55.688 | INFO | MemInfo: { | 20260507 13:57:55.688 | INFO | "gpu" : " 17 GB", | 20260507 13:57:55.688 | INFO | "huge" : "", | 20260507 13:57:55.688 | INFO | "huge_shared" : " 29 GB", | 20260507 13:57:55.688 | INFO | "malloc" : " 15 GB", | 20260507 13:57:55.688 | INFO | "pinned" : " 29 GB" | 20260507 13:57:55.688 | INFO | } | 20260507 13:57:55.688 | INFO | Perf report for the last 5.13934 seconds: | 20260507 13:57:55.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:57:55.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260507 13:57:55.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 13:57:55.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:57:55.688 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4196GiB/s) | 20260507 13:57:55.688 | INFO | Downloaded 34GiB to GPU in 1.52563s (22.2858GiB/s) | 20260507 13:58:00.827 | INFO | Analyzing chunk frames = [34816,35328) | 20260507 13:58:00.827 | INFO | MemInfo: { | 20260507 13:58:00.827 | INFO | "gpu" : " 17 GB", | 20260507 13:58:00.827 | INFO | "huge" : "", | 20260507 13:58:00.827 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:00.827 | INFO | "malloc" : " 15 GB", | 20260507 13:58:00.827 | INFO | "pinned" : " 29 GB" | 20260507 13:58:00.827 | INFO | } | 20260507 13:58:00.827 | INFO | Perf report for the last 5.13936 seconds: | 20260507 13:58:00.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:00.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.827 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 13:58:00.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.827 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 13:58:00.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:00.828 | INFO | Uploaded 36GiB to GPU in 1.60483s (22.4323GiB/s) | 20260507 13:58:00.828 | INFO | Downloaded 34GiB to GPU in 1.52355s (22.3163GiB/s) | 20260507 13:58:05.967 | INFO | Analyzing chunk frames = [35328,35840) | 20260507 13:58:05.967 | INFO | MemInfo: { | 20260507 13:58:05.967 | INFO | "gpu" : " 17 GB", | 20260507 13:58:05.967 | INFO | "huge" : "", | 20260507 13:58:05.967 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:05.967 | INFO | "malloc" : " 15 GB", | 20260507 13:58:05.967 | INFO | "pinned" : " 29 GB" | 20260507 13:58:05.967 | INFO | } | 20260507 13:58:05.967 | INFO | Perf report for the last 5.13936 seconds: | 20260507 13:58:05.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:05.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 13:58:05.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 13:58:05.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:05.967 | INFO | Uploaded 36GiB to GPU in 1.60498s (22.4301GiB/s) | 20260507 13:58:05.967 | INFO | Downloaded 34GiB to GPU in 1.52452s (22.3021GiB/s) | 20260507 13:58:11.106 | INFO | Analyzing chunk frames = [35840,36352) | 20260507 13:58:11.106 | INFO | MemInfo: { | 20260507 13:58:11.106 | INFO | "gpu" : " 17 GB", | 20260507 13:58:11.106 | INFO | "huge" : "", | 20260507 13:58:11.106 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:11.106 | INFO | "malloc" : " 15 GB", | 20260507 13:58:11.106 | INFO | "pinned" : " 29 GB" | 20260507 13:58:11.106 | INFO | } | 20260507 13:58:11.106 | INFO | Perf report for the last 5.13932 seconds: | 20260507 13:58:11.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:11.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 13:58:11.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 13:58:11.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:11.106 | INFO | Uploaded 36GiB to GPU in 1.60529s (22.4259GiB/s) | 20260507 13:58:11.106 | INFO | Downloaded 34GiB to GPU in 1.52461s (22.3008GiB/s) | 20260507 13:58:16.245 | INFO | Analyzing chunk frames = [36352,36864) | 20260507 13:58:16.245 | INFO | MemInfo: { | 20260507 13:58:16.245 | INFO | "gpu" : " 17 GB", | 20260507 13:58:16.245 | INFO | "huge" : "", | 20260507 13:58:16.245 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:16.245 | INFO | "malloc" : " 15 GB", | 20260507 13:58:16.245 | INFO | "pinned" : " 29 GB" | 20260507 13:58:16.245 | INFO | } | 20260507 13:58:16.245 | INFO | Perf report for the last 5.13919 seconds: | 20260507 13:58:16.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:16.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 13:58:16.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 13:58:16.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:16.245 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260507 13:58:16.246 | INFO | Downloaded 34GiB to GPU in 1.5256s (22.2864GiB/s) | 20260507 13:58:16.337 | INFO | Starting a round of Baz IO | 20260507 13:58:19.242 | INFO | Populating RTMetrics report | 20260507 13:58:19.242 | INFO | RTMetrics report populated | 20260507 13:58:19.242 | INFO | Opening tmp RTMetrics json file | 20260507 13:58:19.243 | INFO | Serializing RTMetrics to json | 20260507 13:58:19.244 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.906 seconds | 20260507 13:58:19.244 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:58:19.244 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.30888% / 0% / 0% | 20260507 13:58:19.244 | INFO | Baz IO overall: | 20260507 13:58:19.244 | INFO | BytesWritten : 2571444806 | 20260507 13:58:19.244 | INFO | NumEvents : 0 | 20260507 13:58:19.246 | INFO | Serialization done, swapping tmp file now... | 20260507 13:58:19.247 | INFO | RTMetrics done | 20260507 13:58:21.384 | INFO | Analyzing chunk frames = [36864,37376) | 20260507 13:58:21.384 | INFO | MemInfo: { | 20260507 13:58:21.384 | INFO | "gpu" : " 17 GB", | 20260507 13:58:21.384 | INFO | "huge" : "", | 20260507 13:58:21.384 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:21.385 | INFO | "malloc" : " 15 GB", | 20260507 13:58:21.385 | INFO | "pinned" : " 29 GB" | 20260507 13:58:21.385 | INFO | } | 20260507 13:58:21.385 | INFO | Perf report for the last 5.13938 seconds: | 20260507 13:58:21.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:21.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:21.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:21.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:21.385 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260507 13:58:21.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:21.385 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 13:58:21.385 | INFO | PRE_HQ : 54.6%, 1.8, 52.4ms, 0 | 20260507 13:58:21.385 | INFO | BAZWRITER : 10.6%, 1.0, 5.4ms, 0 | 20260507 13:58:21.385 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 13:58:21.385 | INFO | Uploaded 36GiB to GPU in 1.646s (21.8712GiB/s) | 20260507 13:58:21.385 | INFO | Downloaded 38GiB to GPU in 1.71134s (22.2048GiB/s) | 20260507 13:58:26.524 | INFO | Analyzing chunk frames = [37376,37888) | 20260507 13:58:26.524 | INFO | MemInfo: { | 20260507 13:58:26.524 | INFO | "gpu" : " 17 GB", | 20260507 13:58:26.524 | INFO | "huge" : "", | 20260507 13:58:26.524 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:26.524 | INFO | "malloc" : " 15 GB", | 20260507 13:58:26.524 | INFO | "pinned" : " 29 GB" | 20260507 13:58:26.524 | INFO | } | 20260507 13:58:26.524 | INFO | Perf report for the last 5.1395 seconds: | 20260507 13:58:26.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:26.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 13:58:26.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260507 13:58:26.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:26.524 | INFO | Uploaded 36GiB to GPU in 1.60176s (22.4752GiB/s) | 20260507 13:58:26.524 | INFO | Downloaded 34GiB to GPU in 1.52138s (22.3482GiB/s) | 20260507 13:58:31.663 | INFO | Analyzing chunk frames = [37888,38400) | 20260507 13:58:31.663 | INFO | MemInfo: { | 20260507 13:58:31.663 | INFO | "gpu" : " 17 GB", | 20260507 13:58:31.663 | INFO | "huge" : "", | 20260507 13:58:31.663 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:31.663 | INFO | "malloc" : " 15 GB", | 20260507 13:58:31.663 | INFO | "pinned" : " 29 GB" | 20260507 13:58:31.663 | INFO | } | 20260507 13:58:31.663 | INFO | Perf report for the last 5.13918 seconds: | 20260507 13:58:31.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:31.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.663 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 13:58:31.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.663 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 13:58:31.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:31.664 | INFO | Uploaded 36GiB to GPU in 1.60608s (22.4148GiB/s) | 20260507 13:58:31.664 | INFO | Downloaded 34GiB to GPU in 1.528s (22.2513GiB/s) | 20260507 13:58:36.803 | INFO | Analyzing chunk frames = [38400,38912) | 20260507 13:58:36.803 | INFO | MemInfo: { | 20260507 13:58:36.803 | INFO | "gpu" : " 17 GB", | 20260507 13:58:36.803 | INFO | "huge" : "", | 20260507 13:58:36.803 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:36.803 | INFO | "malloc" : " 15 GB", | 20260507 13:58:36.803 | INFO | "pinned" : " 29 GB" | 20260507 13:58:36.803 | INFO | } | 20260507 13:58:36.803 | INFO | Perf report for the last 5.13928 seconds: | 20260507 13:58:36.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:36.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 13:58:36.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 13:58:36.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:36.803 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4214GiB/s) | 20260507 13:58:36.803 | INFO | Downloaded 34GiB to GPU in 1.52549s (22.2879GiB/s) | 20260507 13:58:41.942 | INFO | Analyzing chunk frames = [38912,39424) | 20260507 13:58:41.942 | INFO | MemInfo: { | 20260507 13:58:41.942 | INFO | "gpu" : " 17 GB", | 20260507 13:58:41.942 | INFO | "huge" : "", | 20260507 13:58:41.942 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:41.942 | INFO | "malloc" : " 15 GB", | 20260507 13:58:41.942 | INFO | "pinned" : " 29 GB" | 20260507 13:58:41.942 | INFO | } | 20260507 13:58:41.942 | INFO | Perf report for the last 5.13942 seconds: | 20260507 13:58:41.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:41.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 13:58:41.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 13:58:41.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:41.942 | INFO | Uploaded 36GiB to GPU in 1.60451s (22.4367GiB/s) | 20260507 13:58:41.943 | INFO | Downloaded 34GiB to GPU in 1.52263s (22.3297GiB/s) | 20260507 13:58:47.081 | INFO | Analyzing chunk frames = [39424,39936) | 20260507 13:58:47.081 | INFO | MemInfo: { | 20260507 13:58:47.081 | INFO | "gpu" : " 17 GB", | 20260507 13:58:47.081 | INFO | "huge" : "", | 20260507 13:58:47.081 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:47.081 | INFO | "malloc" : " 15 GB", | 20260507 13:58:47.081 | INFO | "pinned" : " 29 GB" | 20260507 13:58:47.081 | INFO | } | 20260507 13:58:47.081 | INFO | Perf report for the last 5.1393 seconds: | 20260507 13:58:47.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:47.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.081 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 13:58:47.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.082 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 13:58:47.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:47.082 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4511GiB/s) | 20260507 13:58:47.082 | INFO | Downloaded 34GiB to GPU in 1.52395s (22.3105GiB/s) | 20260507 13:58:52.221 | INFO | Analyzing chunk frames = [39936,40448) | 20260507 13:58:52.221 | INFO | MemInfo: { | 20260507 13:58:52.221 | INFO | "gpu" : " 17 GB", | 20260507 13:58:52.221 | INFO | "huge" : "", | 20260507 13:58:52.221 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:52.221 | INFO | "malloc" : " 15 GB", | 20260507 13:58:52.221 | INFO | "pinned" : " 29 GB" | 20260507 13:58:52.221 | INFO | } | 20260507 13:58:52.221 | INFO | Perf report for the last 5.1394 seconds: | 20260507 13:58:52.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:52.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 13:58:52.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 13:58:52.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:52.221 | INFO | Uploaded 36GiB to GPU in 1.60405s (22.4432GiB/s) | 20260507 13:58:52.221 | INFO | Downloaded 34GiB to GPU in 1.52454s (22.3019GiB/s) | 20260507 13:58:57.360 | INFO | Analyzing chunk frames = [40448,40960) | 20260507 13:58:57.360 | INFO | MemInfo: { | 20260507 13:58:57.360 | INFO | "gpu" : " 17 GB", | 20260507 13:58:57.360 | INFO | "huge" : "", | 20260507 13:58:57.360 | INFO | "huge_shared" : " 29 GB", | 20260507 13:58:57.360 | INFO | "malloc" : " 15 GB", | 20260507 13:58:57.360 | INFO | "pinned" : " 29 GB" | 20260507 13:58:57.360 | INFO | } | 20260507 13:58:57.360 | INFO | Perf report for the last 5.13917 seconds: | 20260507 13:58:57.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:58:57.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 13:58:57.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 13:58:57.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:58:57.360 | INFO | Uploaded 36GiB to GPU in 1.6038s (22.4467GiB/s) | 20260507 13:58:57.360 | INFO | Downloaded 34GiB to GPU in 1.52448s (22.3027GiB/s) | 20260507 13:58:57.360 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 13:58:57.360 | INFO | Total mem usage: 14.7 GiB | 20260507 13:58:57.360 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260507 13:58:57.360 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.685%) | 20260507 13:58:57.360 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 13:58:57.360 | INFO | High Water Breakdown: | 20260507 13:58:57.360 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 13:58:57.360 | INFO | 45.2% : 6.59 GiB ( 99.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 13:58:57.360 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 13:58:57.361 | INFO | ------------------------------------------ | 20260507 13:58:57.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:58:57.361 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 13:58:57.361 | INFO | Total mem usage: 26.5 GiB | 20260507 13:58:57.361 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 13:58:57.361 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 13:58:57.361 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 13:58:57.361 | INFO | High Water Breakdown: | 20260507 13:58:57.361 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:58:57.361 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:58:57.361 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:58:57.361 | INFO | 1.6% : 418 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:58:57.361 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:58:57.361 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 13:58:57.361 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 13:58:57.361 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 13:58:57.361 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 13:58:57.361 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 13:58:57.361 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 13:58:57.361 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 13:58:57.361 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 13:58:57.361 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 13:58:57.361 | INFO | ------------------------------------------ | 20260507 13:58:57.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:58:57.361 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 13:58:57.361 | INFO | Total mem usage: 27.8 GiB | 20260507 13:58:57.361 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 13:58:57.361 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 13:58:57.361 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 13:58:57.361 | INFO | High Water Breakdown: | 20260507 13:58:57.361 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 13:58:57.361 | INFO | ------------------------------------------ | 20260507 13:58:57.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:58:57.361 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 13:58:57.361 | INFO | Total mem usage: 15.7 GiB | 20260507 13:58:57.361 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 13:58:57.361 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 13:58:57.361 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 13:58:57.361 | INFO | High Water Breakdown: | 20260507 13:58:57.361 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 13:58:57.361 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 13:58:57.361 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 13:58:57.361 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 13:58:57.361 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 13:58:57.361 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 13:58:57.361 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 13:58:57.361 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 13:58:57.361 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 13:58:57.361 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 13:58:57.361 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 13:58:57.361 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 13:58:57.361 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 13:58:57.361 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 13:58:57.361 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 13:58:57.361 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 13:58:57.361 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 13:58:57.361 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 13:58:57.361 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 13:58:57.361 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 13:58:57.361 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 13:58:57.361 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 13:58:57.361 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 13:58:57.361 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 13:58:57.361 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 13:58:57.361 | INFO | ------------------------------------------ | 20260507 13:58:57.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 13:58:57.361 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 13:58:57.361 | INFO | PerfReport: Gathered 1536 samples. | 20260507 13:58:57.361 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:58:57.361 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 0.29%, AggregateStats -- 8.60e-02 / 0.00e+00, 4.04e-03 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 3.03%, Histogram -- 9.11e-01 / 0.00e+00, 5.16e-02 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 16.10%, DME -- 3.87e+01 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.45e-02 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.22e+00 / 0.00e+00, 1.33e-01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 3.71%, Metrics -- 1.12e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.21e-01 / 0.00e+00, 6.54e-02 / 0.00e+00 | 20260507 13:58:57.361 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 13:58:57.361 | INFO | PerfReport: Gathered 1536 samples. | 20260507 13:58:57.361 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 13:58:57.361 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 18.55%, Upload -- 1.68e+01 / 0.00e+00, 3.95e-01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 17.84%, Download -- 1.61e+01 / 0.00e+00, 7.15e-01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 33.29%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 9.11e-05 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 29.74%, LockCompute -- 2.69e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260507 13:58:57.361 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260507 13:58:57.456 | INFO | Starting a round of Baz IO | 20260507 13:59:00.360 | INFO | Populating RTMetrics report | 20260507 13:59:00.360 | INFO | RTMetrics report populated | 20260507 13:59:00.360 | INFO | Opening tmp RTMetrics json file | 20260507 13:59:00.361 | INFO | Serializing RTMetrics to json | 20260507 13:59:00.361 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.905 seconds | 20260507 13:59:00.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:59:00.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.42219% / 0% / 0% | 20260507 13:59:00.361 | INFO | Baz IO overall: | 20260507 13:59:00.362 | INFO | BytesWritten : 2571448358 | 20260507 13:59:00.362 | INFO | NumEvents : 0 | 20260507 13:59:00.362 | INFO | Serialization done, swapping tmp file now... | 20260507 13:59:00.363 | INFO | RTMetrics done | 20260507 13:59:02.499 | INFO | Analyzing chunk frames = [40960,41472) | 20260507 13:59:02.499 | INFO | MemInfo: { | 20260507 13:59:02.499 | INFO | "gpu" : " 17 GB", | 20260507 13:59:02.499 | INFO | "huge" : "", | 20260507 13:59:02.499 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:02.499 | INFO | "malloc" : " 15 GB", | 20260507 13:59:02.499 | INFO | "pinned" : " 29 GB" | 20260507 13:59:02.499 | INFO | } | 20260507 13:59:02.499 | INFO | Perf report for the last 5.13942 seconds: | 20260507 13:59:02.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:02.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:02.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:02.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:02.500 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260507 13:59:02.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:02.500 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 13:59:02.500 | INFO | PRE_HQ : 55.1%, 1.9, 53.8ms, 0 | 20260507 13:59:02.500 | INFO | BAZWRITER : 11.1%, 1.0, 5.7ms, 0 | 20260507 13:59:02.500 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 13:59:02.500 | INFO | Uploaded 36GiB to GPU in 1.64395s (21.8985GiB/s) | 20260507 13:59:02.500 | INFO | Downloaded 38GiB to GPU in 1.70952s (22.2285GiB/s) | 20260507 13:59:07.639 | INFO | Analyzing chunk frames = [41472,41984) | 20260507 13:59:07.639 | INFO | MemInfo: { | 20260507 13:59:07.639 | INFO | "gpu" : " 17 GB", | 20260507 13:59:07.639 | INFO | "huge" : "", | 20260507 13:59:07.639 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:07.639 | INFO | "malloc" : " 15 GB", | 20260507 13:59:07.639 | INFO | "pinned" : " 29 GB" | 20260507 13:59:07.639 | INFO | } | 20260507 13:59:07.639 | INFO | Perf report for the last 5.13937 seconds: | 20260507 13:59:07.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:07.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 13:59:07.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | PRE_HQ : 31.5%, 1.0, 16.5ms, 0 | 20260507 13:59:07.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:07.639 | INFO | Uploaded 36GiB to GPU in 1.60214s (22.4699GiB/s) | 20260507 13:59:07.639 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3543GiB/s) | 20260507 13:59:12.778 | INFO | Analyzing chunk frames = [41984,42496) | 20260507 13:59:12.778 | INFO | MemInfo: { | 20260507 13:59:12.778 | INFO | "gpu" : " 17 GB", | 20260507 13:59:12.778 | INFO | "huge" : "", | 20260507 13:59:12.778 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:12.778 | INFO | "malloc" : " 15 GB", | 20260507 13:59:12.778 | INFO | "pinned" : " 29 GB" | 20260507 13:59:12.778 | INFO | } | 20260507 13:59:12.778 | INFO | Perf report for the last 5.13928 seconds: | 20260507 13:59:12.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:12.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260507 13:59:12.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 13:59:12.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:12.778 | INFO | Uploaded 36GiB to GPU in 1.60608s (22.4149GiB/s) | 20260507 13:59:12.778 | INFO | Downloaded 34GiB to GPU in 1.52561s (22.2862GiB/s) | 20260507 13:59:17.917 | INFO | Analyzing chunk frames = [42496,43008) | 20260507 13:59:17.917 | INFO | MemInfo: { | 20260507 13:59:17.917 | INFO | "gpu" : " 17 GB", | 20260507 13:59:17.917 | INFO | "huge" : "", | 20260507 13:59:17.917 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:17.917 | INFO | "malloc" : " 15 GB", | 20260507 13:59:17.917 | INFO | "pinned" : " 29 GB" | 20260507 13:59:17.917 | INFO | } | 20260507 13:59:17.918 | INFO | Perf report for the last 5.13938 seconds: | 20260507 13:59:17.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:17.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260507 13:59:17.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 13:59:17.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:17.918 | INFO | Uploaded 36GiB to GPU in 1.60613s (22.4141GiB/s) | 20260507 13:59:17.918 | INFO | Downloaded 34GiB to GPU in 1.52628s (22.2764GiB/s) | 20260507 13:59:23.057 | INFO | Analyzing chunk frames = [43008,43520) | 20260507 13:59:23.057 | INFO | MemInfo: { | 20260507 13:59:23.057 | INFO | "gpu" : " 17 GB", | 20260507 13:59:23.057 | INFO | "huge" : "", | 20260507 13:59:23.057 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:23.057 | INFO | "malloc" : " 15 GB", | 20260507 13:59:23.057 | INFO | "pinned" : " 29 GB" | 20260507 13:59:23.057 | INFO | } | 20260507 13:59:23.057 | INFO | Perf report for the last 5.13925 seconds: | 20260507 13:59:23.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:23.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 13:59:23.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260507 13:59:23.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:23.057 | INFO | Uploaded 36GiB to GPU in 1.60424s (22.4405GiB/s) | 20260507 13:59:23.057 | INFO | Downloaded 34GiB to GPU in 1.52258s (22.3306GiB/s) | 20260507 13:59:28.196 | INFO | Analyzing chunk frames = [43520,44032) | 20260507 13:59:28.196 | INFO | MemInfo: { | 20260507 13:59:28.196 | INFO | "gpu" : " 17 GB", | 20260507 13:59:28.196 | INFO | "huge" : "", | 20260507 13:59:28.196 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:28.196 | INFO | "malloc" : " 15 GB", | 20260507 13:59:28.196 | INFO | "pinned" : " 29 GB" | 20260507 13:59:28.196 | INFO | } | 20260507 13:59:28.196 | INFO | Perf report for the last 5.13931 seconds: | 20260507 13:59:28.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:28.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 13:59:28.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 13:59:28.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:28.196 | INFO | Uploaded 36GiB to GPU in 1.60441s (22.4381GiB/s) | 20260507 13:59:28.196 | INFO | Downloaded 34GiB to GPU in 1.52314s (22.3223GiB/s) | 20260507 13:59:33.335 | INFO | Analyzing chunk frames = [44032,44544) | 20260507 13:59:33.335 | INFO | MemInfo: { | 20260507 13:59:33.335 | INFO | "gpu" : " 17 GB", | 20260507 13:59:33.335 | INFO | "huge" : "", | 20260507 13:59:33.336 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:33.336 | INFO | "malloc" : " 15 GB", | 20260507 13:59:33.336 | INFO | "pinned" : " 29 GB" | 20260507 13:59:33.336 | INFO | } | 20260507 13:59:33.336 | INFO | Perf report for the last 5.1395 seconds: | 20260507 13:59:33.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:33.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 13:59:33.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 13:59:33.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:33.336 | INFO | Uploaded 36GiB to GPU in 1.6041s (22.4425GiB/s) | 20260507 13:59:33.336 | INFO | Downloaded 34GiB to GPU in 1.52517s (22.2926GiB/s) | 20260507 13:59:38.475 | INFO | Analyzing chunk frames = [44544,45056) | 20260507 13:59:38.475 | INFO | MemInfo: { | 20260507 13:59:38.475 | INFO | "gpu" : " 17 GB", | 20260507 13:59:38.475 | INFO | "huge" : "", | 20260507 13:59:38.475 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:38.475 | INFO | "malloc" : " 15 GB", | 20260507 13:59:38.475 | INFO | "pinned" : " 29 GB" | 20260507 13:59:38.475 | INFO | } | 20260507 13:59:38.475 | INFO | Perf report for the last 5.13911 seconds: | 20260507 13:59:38.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:38.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 13:59:38.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 13:59:38.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:38.475 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260507 13:59:38.475 | INFO | Downloaded 34GiB to GPU in 1.52469s (22.2996GiB/s) | 20260507 13:59:38.569 | INFO | Starting a round of Baz IO | 20260507 13:59:41.478 | INFO | Populating RTMetrics report | 20260507 13:59:41.478 | INFO | RTMetrics report populated | 20260507 13:59:41.478 | INFO | Opening tmp RTMetrics json file | 20260507 13:59:41.479 | INFO | Serializing RTMetrics to json | 20260507 13:59:41.480 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.91 seconds | 20260507 13:59:41.480 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 13:59:41.480 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.53979% / 0% / 0% | 20260507 13:59:41.480 | INFO | Baz IO overall: | 20260507 13:59:41.480 | INFO | BytesWritten : 2571451910 | 20260507 13:59:41.480 | INFO | NumEvents : 0 | 20260507 13:59:41.481 | INFO | Serialization done, swapping tmp file now... | 20260507 13:59:41.481 | INFO | RTMetrics done | 20260507 13:59:43.614 | INFO | Analyzing chunk frames = [45056,45568) | 20260507 13:59:43.614 | INFO | MemInfo: { | 20260507 13:59:43.614 | INFO | "gpu" : " 17 GB", | 20260507 13:59:43.614 | INFO | "huge" : "", | 20260507 13:59:43.614 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:43.614 | INFO | "malloc" : " 15 GB", | 20260507 13:59:43.614 | INFO | "pinned" : " 29 GB" | 20260507 13:59:43.614 | INFO | } | 20260507 13:59:43.614 | INFO | Perf report for the last 5.13939 seconds: | 20260507 13:59:43.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:43.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:43.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:43.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:43.614 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260507 13:59:43.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:43.614 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 13:59:43.614 | INFO | PRE_HQ : 55.0%, 1.9, 53.6ms, 0 | 20260507 13:59:43.614 | INFO | BAZWRITER : 10.5%, 1.0, 5.4ms, 0 | 20260507 13:59:43.614 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 13:59:43.615 | INFO | Uploaded 36GiB to GPU in 1.64907s (21.8305GiB/s) | 20260507 13:59:43.615 | INFO | Downloaded 38GiB to GPU in 1.71163s (22.2011GiB/s) | 20260507 13:59:48.753 | INFO | Analyzing chunk frames = [45568,46080) | 20260507 13:59:48.753 | INFO | MemInfo: { | 20260507 13:59:48.753 | INFO | "gpu" : " 17 GB", | 20260507 13:59:48.753 | INFO | "huge" : "", | 20260507 13:59:48.753 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:48.753 | INFO | "malloc" : " 15 GB", | 20260507 13:59:48.753 | INFO | "pinned" : " 29 GB" | 20260507 13:59:48.753 | INFO | } | 20260507 13:59:48.753 | INFO | Perf report for the last 5.13934 seconds: | 20260507 13:59:48.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:48.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 13:59:48.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | PRE_HQ : 31.0%, 1.0, 16.3ms, 0 | 20260507 13:59:48.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:48.754 | INFO | Uploaded 36GiB to GPU in 1.60178s (22.475GiB/s) | 20260507 13:59:48.754 | INFO | Downloaded 34GiB to GPU in 1.52194s (22.3399GiB/s) | 20260507 13:59:53.893 | INFO | Analyzing chunk frames = [46080,46592) | 20260507 13:59:53.893 | INFO | MemInfo: { | 20260507 13:59:53.893 | INFO | "gpu" : " 17 GB", | 20260507 13:59:53.893 | INFO | "huge" : "", | 20260507 13:59:53.893 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:53.893 | INFO | "malloc" : " 15 GB", | 20260507 13:59:53.893 | INFO | "pinned" : " 29 GB" | 20260507 13:59:53.893 | INFO | } | 20260507 13:59:53.893 | INFO | Perf report for the last 5.13941 seconds: | 20260507 13:59:53.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:53.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 13:59:53.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 13:59:53.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:53.893 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4176GiB/s) | 20260507 13:59:53.893 | INFO | Downloaded 34GiB to GPU in 1.52481s (22.2979GiB/s) | 20260507 13:59:59.032 | INFO | Analyzing chunk frames = [46592,47104) | 20260507 13:59:59.032 | INFO | MemInfo: { | 20260507 13:59:59.032 | INFO | "gpu" : " 17 GB", | 20260507 13:59:59.032 | INFO | "huge" : "", | 20260507 13:59:59.032 | INFO | "huge_shared" : " 29 GB", | 20260507 13:59:59.032 | INFO | "malloc" : " 15 GB", | 20260507 13:59:59.032 | INFO | "pinned" : " 29 GB" | 20260507 13:59:59.032 | INFO | } | 20260507 13:59:59.032 | INFO | Perf report for the last 5.13958 seconds: | 20260507 13:59:59.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 13:59:59.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 13:59:59.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 13:59:59.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 13:59:59.033 | INFO | Uploaded 36GiB to GPU in 1.60548s (22.4232GiB/s) | 20260507 13:59:59.033 | INFO | Downloaded 34GiB to GPU in 1.52453s (22.302GiB/s) | 20260507 14:00:04.171 | INFO | Analyzing chunk frames = [47104,47616) | 20260507 14:00:04.171 | INFO | MemInfo: { | 20260507 14:00:04.171 | INFO | "gpu" : " 17 GB", | 20260507 14:00:04.171 | INFO | "huge" : "", | 20260507 14:00:04.171 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:04.171 | INFO | "malloc" : " 15 GB", | 20260507 14:00:04.171 | INFO | "pinned" : " 29 GB" | 20260507 14:00:04.172 | INFO | } | 20260507 14:00:04.172 | INFO | Perf report for the last 5.13903 seconds: | 20260507 14:00:04.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:04.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:00:04.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 14:00:04.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:04.172 | INFO | Uploaded 36GiB to GPU in 1.60326s (22.4542GiB/s) | 20260507 14:00:04.172 | INFO | Downloaded 34GiB to GPU in 1.52194s (22.34GiB/s) | 20260507 14:00:09.311 | INFO | Analyzing chunk frames = [47616,48128) | 20260507 14:00:09.311 | INFO | MemInfo: { | 20260507 14:00:09.311 | INFO | "gpu" : " 17 GB", | 20260507 14:00:09.311 | INFO | "huge" : "", | 20260507 14:00:09.311 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:09.311 | INFO | "malloc" : " 15 GB", | 20260507 14:00:09.311 | INFO | "pinned" : " 29 GB" | 20260507 14:00:09.311 | INFO | } | 20260507 14:00:09.311 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:00:09.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:09.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.311 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:00:09.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.311 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 14:00:09.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:09.312 | INFO | Uploaded 36GiB to GPU in 1.60394s (22.4448GiB/s) | 20260507 14:00:09.312 | INFO | Downloaded 34GiB to GPU in 1.52279s (22.3275GiB/s) | 20260507 14:00:14.450 | INFO | Analyzing chunk frames = [48128,48640) | 20260507 14:00:14.450 | INFO | MemInfo: { | 20260507 14:00:14.450 | INFO | "gpu" : " 17 GB", | 20260507 14:00:14.450 | INFO | "huge" : "", | 20260507 14:00:14.450 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:14.450 | INFO | "malloc" : " 15 GB", | 20260507 14:00:14.450 | INFO | "pinned" : " 29 GB" | 20260507 14:00:14.450 | INFO | } | 20260507 14:00:14.450 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:00:14.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:14.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.450 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:00:14.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.450 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:00:14.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:14.451 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4546GiB/s) | 20260507 14:00:14.451 | INFO | Downloaded 34GiB to GPU in 1.5234s (22.3185GiB/s) | 20260507 14:00:19.589 | INFO | Analyzing chunk frames = [48640,49152) | 20260507 14:00:19.590 | INFO | MemInfo: { | 20260507 14:00:19.590 | INFO | "gpu" : " 17 GB", | 20260507 14:00:19.590 | INFO | "huge" : "", | 20260507 14:00:19.590 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:19.590 | INFO | "malloc" : " 15 GB", | 20260507 14:00:19.590 | INFO | "pinned" : " 29 GB" | 20260507 14:00:19.590 | INFO | } | 20260507 14:00:19.590 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:00:19.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:19.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:00:19.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:00:19.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:19.590 | INFO | Uploaded 36GiB to GPU in 1.60465s (22.4347GiB/s) | 20260507 14:00:19.590 | INFO | Downloaded 34GiB to GPU in 1.52507s (22.2941GiB/s) | 20260507 14:00:19.590 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:00:19.590 | INFO | Total mem usage: 14.7 GiB | 20260507 14:00:19.590 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260507 14:00:19.590 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.685%) | 20260507 14:00:19.590 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:00:19.590 | INFO | High Water Breakdown: | 20260507 14:00:19.590 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:00:19.590 | INFO | 45.2% : 6.59 GiB ( 99.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:00:19.590 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:00:19.590 | INFO | ------------------------------------------ | 20260507 14:00:19.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:00:19.590 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:00:19.590 | INFO | Total mem usage: 26.5 GiB | 20260507 14:00:19.590 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:00:19.590 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:00:19.590 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 14:00:19.590 | INFO | High Water Breakdown: | 20260507 14:00:19.590 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:00:19.590 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:00:19.590 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:00:19.590 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:00:19.590 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:00:19.590 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:00:19.590 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:00:19.590 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:00:19.590 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:00:19.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:00:19.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:00:19.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:00:19.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:00:19.590 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:00:19.591 | INFO | ------------------------------------------ | 20260507 14:00:19.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:00:19.591 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:00:19.591 | INFO | Total mem usage: 27.8 GiB | 20260507 14:00:19.591 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:00:19.591 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:00:19.591 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:00:19.591 | INFO | High Water Breakdown: | 20260507 14:00:19.591 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:00:19.591 | INFO | ------------------------------------------ | 20260507 14:00:19.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:00:19.591 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:00:19.591 | INFO | Total mem usage: 15.7 GiB | 20260507 14:00:19.591 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:00:19.591 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:00:19.591 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:00:19.591 | INFO | High Water Breakdown: | 20260507 14:00:19.591 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:00:19.591 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:00:19.591 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:00:19.591 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:00:19.591 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:00:19.591 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:00:19.591 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:00:19.591 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:00:19.591 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:00:19.591 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:00:19.591 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:00:19.591 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:00:19.591 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:00:19.591 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:00:19.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:00:19.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:00:19.591 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:00:19.591 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:00:19.591 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:00:19.591 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:00:19.591 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:00:19.591 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:00:19.591 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:00:19.591 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:00:19.591 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:00:19.591 | INFO | ------------------------------------------ | 20260507 14:00:19.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:00:19.591 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:00:19.591 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:00:19.591 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:00:19.591 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.23e+01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / 0.00e+00, 6.16e-03 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 3.03%, Histogram -- 9.13e-01 / 0.00e+00, 5.02e-02 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 16.10%, DME -- 3.88e+01 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.38e-02 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / 0.00e+00, 1.33e-01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 3.73%, Metrics -- 1.12e+00 / 0.00e+00, 1.83e-01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.25e-01 / 0.00e+00, 6.53e-02 / 0.00e+00 | 20260507 14:00:19.591 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:00:19.591 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:00:19.591 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:00:19.591 | INFO | PerfReport: Full Execution -- 9.05e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 18.53%, Upload -- 1.68e+01 / 0.00e+00, 3.93e-01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 17.81%, Download -- 1.61e+01 / 0.00e+00, 7.17e-01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 33.29%, Compute -- 3.01e+01 / 0.00e+00, 1.23e+01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 0.00%, LockDownload -- 4.44e-04 / 0.00e+00, 1.31e-04 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 29.79%, LockCompute -- 2.70e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:00:19.591 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260507 14:00:19.684 | INFO | Starting a round of Baz IO | 20260507 14:00:22.593 | INFO | Populating RTMetrics report | 20260507 14:00:22.594 | INFO | RTMetrics report populated | 20260507 14:00:22.594 | INFO | Opening tmp RTMetrics json file | 20260507 14:00:22.595 | INFO | Serializing RTMetrics to json | 20260507 14:00:22.595 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.91 seconds | 20260507 14:00:22.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260507 14:00:22.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.63729% / 0% / 0% | 20260507 14:00:22.595 | INFO | Baz IO overall: | 20260507 14:00:22.595 | INFO | BytesWritten : 2571455462 | 20260507 14:00:22.595 | INFO | NumEvents : 0 | 20260507 14:00:22.596 | INFO | Serialization done, swapping tmp file now... | 20260507 14:00:22.597 | INFO | RTMetrics done | 20260507 14:00:24.729 | INFO | Analyzing chunk frames = [49152,49664) | 20260507 14:00:24.729 | INFO | MemInfo: { | 20260507 14:00:24.729 | INFO | "gpu" : " 17 GB", | 20260507 14:00:24.729 | INFO | "huge" : "", | 20260507 14:00:24.729 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:24.729 | INFO | "malloc" : " 15 GB", | 20260507 14:00:24.729 | INFO | "pinned" : " 29 GB" | 20260507 14:00:24.729 | INFO | } | 20260507 14:00:24.729 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:00:24.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:24.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:24.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:24.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:24.729 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260507 14:00:24.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:24.729 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 14:00:24.729 | INFO | PRE_HQ : 55.1%, 1.9, 53.8ms, 0 | 20260507 14:00:24.729 | INFO | BAZWRITER : 10.6%, 1.0, 5.4ms, 0 | 20260507 14:00:24.729 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260507 14:00:24.730 | INFO | Uploaded 36GiB to GPU in 1.64605s (21.8706GiB/s) | 20260507 14:00:24.730 | INFO | Downloaded 38GiB to GPU in 1.70813s (22.2465GiB/s) | 20260507 14:00:29.868 | INFO | Analyzing chunk frames = [49664,50176) | 20260507 14:00:29.868 | INFO | MemInfo: { | 20260507 14:00:29.868 | INFO | "gpu" : " 17 GB", | 20260507 14:00:29.868 | INFO | "huge" : "", | 20260507 14:00:29.868 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:29.868 | INFO | "malloc" : " 15 GB", | 20260507 14:00:29.868 | INFO | "pinned" : " 29 GB" | 20260507 14:00:29.868 | INFO | } | 20260507 14:00:29.868 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:00:29.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:29.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:00:29.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260507 14:00:29.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:29.868 | INFO | Uploaded 36GiB to GPU in 1.60191s (22.4732GiB/s) | 20260507 14:00:29.868 | INFO | Downloaded 34GiB to GPU in 1.52117s (22.3512GiB/s) | 20260507 14:00:35.008 | INFO | Analyzing chunk frames = [50176,50688) | 20260507 14:00:35.008 | INFO | MemInfo: { | 20260507 14:00:35.008 | INFO | "gpu" : " 17 GB", | 20260507 14:00:35.008 | INFO | "huge" : "", | 20260507 14:00:35.008 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:35.008 | INFO | "malloc" : " 15 GB", | 20260507 14:00:35.008 | INFO | "pinned" : " 29 GB" | 20260507 14:00:35.008 | INFO | } | 20260507 14:00:35.008 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:00:35.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:35.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:00:35.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 14:00:35.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:35.008 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.4131GiB/s) | 20260507 14:00:35.008 | INFO | Downloaded 34GiB to GPU in 1.52596s (22.2811GiB/s) | 20260507 14:00:40.147 | INFO | Analyzing chunk frames = [50688,51200) | 20260507 14:00:40.147 | INFO | MemInfo: { | 20260507 14:00:40.147 | INFO | "gpu" : " 17 GB", | 20260507 14:00:40.147 | INFO | "huge" : "", | 20260507 14:00:40.147 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:40.147 | INFO | "malloc" : " 15 GB", | 20260507 14:00:40.147 | INFO | "pinned" : " 29 GB" | 20260507 14:00:40.147 | INFO | } | 20260507 14:00:40.147 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:00:40.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:40.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.147 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260507 14:00:40.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.147 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 14:00:40.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:40.148 | INFO | Uploaded 36GiB to GPU in 1.60478s (22.433GiB/s) | 20260507 14:00:40.148 | INFO | Downloaded 34GiB to GPU in 1.52557s (22.2867GiB/s) | 20260507 14:00:45.286 | INFO | Analyzing chunk frames = [51200,51712) | 20260507 14:00:45.286 | INFO | MemInfo: { | 20260507 14:00:45.286 | INFO | "gpu" : " 17 GB", | 20260507 14:00:45.286 | INFO | "huge" : "", | 20260507 14:00:45.286 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:45.286 | INFO | "malloc" : " 15 GB", | 20260507 14:00:45.286 | INFO | "pinned" : " 29 GB" | 20260507 14:00:45.287 | INFO | } | 20260507 14:00:45.287 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:00:45.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:45.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:00:45.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 14:00:45.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:45.287 | INFO | Uploaded 36GiB to GPU in 1.604s (22.4439GiB/s) | 20260507 14:00:45.287 | INFO | Downloaded 34GiB to GPU in 1.52233s (22.3342GiB/s) | 20260507 14:00:50.426 | INFO | Analyzing chunk frames = [51712,52224) | 20260507 14:00:50.426 | INFO | MemInfo: { | 20260507 14:00:50.426 | INFO | "gpu" : " 17 GB", | 20260507 14:00:50.426 | INFO | "huge" : "", | 20260507 14:00:50.426 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:50.426 | INFO | "malloc" : " 15 GB", | 20260507 14:00:50.426 | INFO | "pinned" : " 29 GB" | 20260507 14:00:50.426 | INFO | } | 20260507 14:00:50.426 | INFO | Perf report for the last 5.13908 seconds: | 20260507 14:00:50.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:50.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:00:50.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 14:00:50.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:50.426 | INFO | Uploaded 36GiB to GPU in 1.604s (22.4439GiB/s) | 20260507 14:00:50.426 | INFO | Downloaded 34GiB to GPU in 1.52354s (22.3165GiB/s) | 20260507 14:00:55.565 | INFO | Analyzing chunk frames = [52224,52736) | 20260507 14:00:55.565 | INFO | MemInfo: { | 20260507 14:00:55.565 | INFO | "gpu" : " 17 GB", | 20260507 14:00:55.565 | INFO | "huge" : "", | 20260507 14:00:55.565 | INFO | "huge_shared" : " 29 GB", | 20260507 14:00:55.565 | INFO | "malloc" : " 15 GB", | 20260507 14:00:55.565 | INFO | "pinned" : " 29 GB" | 20260507 14:00:55.565 | INFO | } | 20260507 14:00:55.565 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:00:55.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:00:55.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.565 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:00:55.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.565 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:00:55.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:00:55.566 | INFO | Uploaded 36GiB to GPU in 1.60405s (22.4432GiB/s) | 20260507 14:00:55.566 | INFO | Downloaded 34GiB to GPU in 1.52343s (22.3181GiB/s) | 20260507 14:01:00.704 | INFO | Analyzing chunk frames = [52736,53248) | 20260507 14:01:00.704 | INFO | MemInfo: { | 20260507 14:01:00.704 | INFO | "gpu" : " 17 GB", | 20260507 14:01:00.704 | INFO | "huge" : "", | 20260507 14:01:00.704 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:00.704 | INFO | "malloc" : " 15 GB", | 20260507 14:01:00.704 | INFO | "pinned" : " 29 GB" | 20260507 14:01:00.704 | INFO | } | 20260507 14:01:00.704 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:01:00.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:00.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.704 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:01:00.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.705 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:01:00.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:00.705 | INFO | Uploaded 36GiB to GPU in 1.60459s (22.4357GiB/s) | 20260507 14:01:00.705 | INFO | Downloaded 34GiB to GPU in 1.52413s (22.3079GiB/s) | 20260507 14:01:00.802 | INFO | Starting a round of Baz IO | 20260507 14:01:03.709 | INFO | Populating RTMetrics report | 20260507 14:01:03.709 | INFO | RTMetrics report populated | 20260507 14:01:03.709 | INFO | Opening tmp RTMetrics json file | 20260507 14:01:03.710 | INFO | Serializing RTMetrics to json | 20260507 14:01:03.712 | INFO | Serialization done, swapping tmp file now... | 20260507 14:01:03.713 | INFO | RTMetrics done | 20260507 14:01:03.951 | INFO | Finished a round of Baz IO: Wrote 1.18 GiB in 3.148 seconds | 20260507 14:01:03.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 79.87% / 20.13% / 0.00% | 20260507 14:01:03.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.67685% / 0.081857% / 0% | 20260507 14:01:03.951 | INFO | Baz IO overall: | 20260507 14:01:03.951 | INFO | BytesWritten : 3838940770 | 20260507 14:01:03.951 | INFO | NumEvents : 476761640 | 20260507 14:01:05.844 | INFO | Analyzing chunk frames = [53248,53760) | 20260507 14:01:05.844 | INFO | MemInfo: { | 20260507 14:01:05.844 | INFO | "gpu" : " 17 GB", | 20260507 14:01:05.844 | INFO | "huge" : "", | 20260507 14:01:05.844 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:05.844 | INFO | "malloc" : " 15 GB", | 20260507 14:01:05.844 | INFO | "pinned" : " 29 GB" | 20260507 14:01:05.844 | INFO | } | 20260507 14:01:05.844 | INFO | Perf report for the last 5.13963 seconds: | 20260507 14:01:05.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:05.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:05.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:05.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:05.844 | INFO | ANALYSIS : 57.6%, 2.9, 88.3ms, 0 | 20260507 14:01:05.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:05.844 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 14:01:05.844 | INFO | PRE_HQ : 57.3%, 2.1, 61.0ms, 0 | 20260507 14:01:05.844 | INFO | BAZWRITER : 60.9%, 4.3, 134.5ms, 0 | 20260507 14:01:05.844 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 14:01:05.844 | INFO | Uploaded 36GiB to GPU in 1.64618s (21.8688GiB/s) | 20260507 14:01:05.844 | INFO | Downloaded 38GiB to GPU in 1.71011s (22.2208GiB/s) | 20260507 14:01:10.983 | INFO | Analyzing chunk frames = [53760,54272) | 20260507 14:01:10.983 | INFO | MemInfo: { | 20260507 14:01:10.983 | INFO | "gpu" : " 17 GB", | 20260507 14:01:10.983 | INFO | "huge" : "", | 20260507 14:01:10.983 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:10.983 | INFO | "malloc" : " 15 GB", | 20260507 14:01:10.983 | INFO | "pinned" : " 29 GB" | 20260507 14:01:10.983 | INFO | } | 20260507 14:01:10.983 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:01:10.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:10.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.983 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260507 14:01:10.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.983 | INFO | PRE_HQ : 31.1%, 1.0, 16.3ms, 0 | 20260507 14:01:10.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:10.984 | INFO | Uploaded 36GiB to GPU in 1.60182s (22.4745GiB/s) | 20260507 14:01:10.984 | INFO | Downloaded 34GiB to GPU in 1.52084s (22.3561GiB/s) | 20260507 14:01:16.122 | INFO | Analyzing chunk frames = [54272,54784) | 20260507 14:01:16.122 | INFO | MemInfo: { | 20260507 14:01:16.122 | INFO | "gpu" : " 17 GB", | 20260507 14:01:16.122 | INFO | "huge" : "", | 20260507 14:01:16.122 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:16.123 | INFO | "malloc" : " 15 GB", | 20260507 14:01:16.123 | INFO | "pinned" : " 29 GB" | 20260507 14:01:16.123 | INFO | } | 20260507 14:01:16.123 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:01:16.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:16.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260507 14:01:16.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 14:01:16.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:16.123 | INFO | Uploaded 36GiB to GPU in 1.60553s (22.4225GiB/s) | 20260507 14:01:16.123 | INFO | Downloaded 34GiB to GPU in 1.52492s (22.2963GiB/s) | 20260507 14:01:21.262 | INFO | Analyzing chunk frames = [54784,55296) | 20260507 14:01:21.262 | INFO | MemInfo: { | 20260507 14:01:21.262 | INFO | "gpu" : " 17 GB", | 20260507 14:01:21.262 | INFO | "huge" : "", | 20260507 14:01:21.262 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:21.262 | INFO | "malloc" : " 15 GB", | 20260507 14:01:21.262 | INFO | "pinned" : " 29 GB" | 20260507 14:01:21.262 | INFO | } | 20260507 14:01:21.262 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:01:21.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:21.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260507 14:01:21.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 14:01:21.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:21.262 | INFO | Uploaded 36GiB to GPU in 1.60482s (22.4324GiB/s) | 20260507 14:01:21.262 | INFO | Downloaded 34GiB to GPU in 1.52479s (22.2982GiB/s) | 20260507 14:01:26.401 | INFO | Analyzing chunk frames = [55296,55808) | 20260507 14:01:26.401 | INFO | MemInfo: { | 20260507 14:01:26.401 | INFO | "gpu" : " 17 GB", | 20260507 14:01:26.401 | INFO | "huge" : "", | 20260507 14:01:26.401 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:26.401 | INFO | "malloc" : " 15 GB", | 20260507 14:01:26.401 | INFO | "pinned" : " 29 GB" | 20260507 14:01:26.401 | INFO | } | 20260507 14:01:26.401 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:01:26.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:26.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.401 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:01:26.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.401 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:01:26.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:26.402 | INFO | Uploaded 36GiB to GPU in 1.60403s (22.4434GiB/s) | 20260507 14:01:26.402 | INFO | Downloaded 34GiB to GPU in 1.522s (22.339GiB/s) | 20260507 14:01:31.540 | INFO | Analyzing chunk frames = [55808,56320) | 20260507 14:01:31.540 | INFO | MemInfo: { | 20260507 14:01:31.540 | INFO | "gpu" : " 17 GB", | 20260507 14:01:31.540 | INFO | "huge" : "", | 20260507 14:01:31.540 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:31.540 | INFO | "malloc" : " 15 GB", | 20260507 14:01:31.540 | INFO | "pinned" : " 29 GB" | 20260507 14:01:31.540 | INFO | } | 20260507 14:01:31.540 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:01:31.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:31.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:01:31.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 14:01:31.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:31.541 | INFO | Uploaded 36GiB to GPU in 1.60311s (22.4564GiB/s) | 20260507 14:01:31.541 | INFO | Downloaded 34GiB to GPU in 1.5227s (22.3287GiB/s) | 20260507 14:01:36.680 | INFO | Analyzing chunk frames = [56320,56832) | 20260507 14:01:36.680 | INFO | MemInfo: { | 20260507 14:01:36.680 | INFO | "gpu" : " 17 GB", | 20260507 14:01:36.680 | INFO | "huge" : "", | 20260507 14:01:36.680 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:36.680 | INFO | "malloc" : " 15 GB", | 20260507 14:01:36.680 | INFO | "pinned" : " 29 GB" | 20260507 14:01:36.680 | INFO | } | 20260507 14:01:36.680 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:01:36.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:36.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:01:36.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:01:36.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:36.680 | INFO | Uploaded 36GiB to GPU in 1.60404s (22.4433GiB/s) | 20260507 14:01:36.680 | INFO | Downloaded 34GiB to GPU in 1.52359s (22.3157GiB/s) | 20260507 14:01:41.819 | INFO | Analyzing chunk frames = [56832,57344) | 20260507 14:01:41.819 | INFO | MemInfo: { | 20260507 14:01:41.819 | INFO | "gpu" : " 17 GB", | 20260507 14:01:41.819 | INFO | "huge" : "", | 20260507 14:01:41.819 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:41.819 | INFO | "malloc" : " 15 GB", | 20260507 14:01:41.819 | INFO | "pinned" : " 29 GB" | 20260507 14:01:41.819 | INFO | } | 20260507 14:01:41.819 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:01:41.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:41.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:01:41.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:01:41.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:41.819 | INFO | Uploaded 36GiB to GPU in 1.60406s (22.4431GiB/s) | 20260507 14:01:41.819 | INFO | Downloaded 34GiB to GPU in 1.5239s (22.3112GiB/s) | 20260507 14:01:41.820 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:01:41.820 | INFO | Total mem usage: 15.5 GiB | 20260507 14:01:41.820 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 1.06%) | 20260507 14:01:41.820 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: 1.06%) | 20260507 14:01:41.820 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:01:41.820 | INFO | High Water Breakdown: | 20260507 14:01:41.820 | INFO | 57% : 8.93 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:01:41.820 | INFO | 42.9% : 6.72 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:01:41.820 | INFO | 0.119% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:01:41.820 | INFO | ------------------------------------------ | 20260507 14:01:41.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:01:41.820 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:01:41.820 | INFO | Total mem usage: 26.5 GiB | 20260507 14:01:41.820 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:01:41.820 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:01:41.820 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:01:41.820 | INFO | High Water Breakdown: | 20260507 14:01:41.820 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:01:41.820 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:01:41.820 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:01:41.820 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:01:41.820 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:01:41.820 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:01:41.820 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:01:41.820 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:01:41.820 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:01:41.820 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:01:41.820 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:01:41.820 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:01:41.820 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:01:41.820 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:01:41.820 | INFO | ------------------------------------------ | 20260507 14:01:41.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:01:41.820 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:01:41.820 | INFO | Total mem usage: 27.8 GiB | 20260507 14:01:41.820 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:01:41.820 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:01:41.820 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:01:41.820 | INFO | High Water Breakdown: | 20260507 14:01:41.820 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:01:41.820 | INFO | ------------------------------------------ | 20260507 14:01:41.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:01:41.820 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:01:41.820 | INFO | Total mem usage: 15.7 GiB | 20260507 14:01:41.820 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:01:41.820 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:01:41.820 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:01:41.820 | INFO | High Water Breakdown: | 20260507 14:01:41.820 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:01:41.820 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:01:41.820 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:01:41.820 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:01:41.820 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:01:41.820 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:01:41.820 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:01:41.820 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:01:41.820 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:01:41.820 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:01:41.820 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:01:41.820 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:01:41.820 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:01:41.820 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:01:41.820 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:01:41.820 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:01:41.820 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:01:41.820 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:01:41.820 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:01:41.820 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:01:41.820 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:01:41.820 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:01:41.820 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:01:41.820 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:01:41.820 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:01:41.820 | INFO | ------------------------------------------ | 20260507 14:01:41.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:01:41.820 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:01:41.820 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:01:41.820 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:01:41.820 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 14:01:41.820 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / 0.00e+00, 7.55e-03 / 0.00e+00 | 20260507 14:01:41.820 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 14:01:41.820 | INFO | PerfReport: 3.04%, Histogram -- 9.15e-01 / 0.00e+00, 5.10e-02 / 0.00e+00 | 20260507 14:01:41.820 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 14:01:41.820 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.44e-02 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.24e+00 / 0.00e+00, 1.36e-01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 3.77%, Metrics -- 1.14e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.29e-01 / 0.00e+00, 6.53e-02 / 0.00e+00 | 20260507 14:01:41.821 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:01:41.821 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:01:41.821 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:01:41.821 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 18.54%, Upload -- 1.68e+01 / 0.00e+00, 3.98e-01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 17.82%, Download -- 1.61e+01 / 0.00e+00, 7.18e-01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 33.33%, Compute -- 3.01e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 0.56%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 0.00%, LockDownload -- 4.41e-04 / 0.00e+00, 9.35e-05 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 29.74%, LockCompute -- 2.69e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:01:41.821 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.37e-02 / 0.00e+00 | 20260507 14:01:41.916 | INFO | Starting a round of Baz IO | 20260507 14:01:44.828 | INFO | Populating RTMetrics report | 20260507 14:01:44.828 | INFO | RTMetrics report populated | 20260507 14:01:44.828 | INFO | Opening tmp RTMetrics json file | 20260507 14:01:44.858 | INFO | Serializing RTMetrics to json | 20260507 14:01:44.864 | INFO | Serialization done, swapping tmp file now... | 20260507 14:01:44.869 | INFO | RTMetrics done | 20260507 14:01:45.094 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.177 seconds | 20260507 14:01:45.094 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 67.42% / 12.45% / 20.13% | 20260507 14:01:45.094 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.752% / 0.0510136% / 0% | 20260507 14:01:45.094 | INFO | Baz IO overall: | 20260507 14:01:45.094 | INFO | BytesWritten : 5584605400 | 20260507 14:01:45.094 | INFO | NumEvents : 1248572709 | 20260507 14:01:46.958 | INFO | Analyzing chunk frames = [57344,57856) | 20260507 14:01:46.958 | INFO | MemInfo: { | 20260507 14:01:46.958 | INFO | "gpu" : " 17 GB", | 20260507 14:01:46.959 | INFO | "huge" : "", | 20260507 14:01:46.959 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:46.959 | INFO | "malloc" : " 15 GB", | 20260507 14:01:46.959 | INFO | "pinned" : " 29 GB" | 20260507 14:01:46.959 | INFO | } | 20260507 14:01:46.959 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:01:46.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:46.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:46.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:46.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:46.959 | INFO | ANALYSIS : 57.7%, 2.8, 85.1ms, 0 | 20260507 14:01:46.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:46.959 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 14:01:46.959 | INFO | PRE_HQ : 57.1%, 2.1, 62.8ms, 0 | 20260507 14:01:46.959 | INFO | BAZWRITER : 61.4%, 4.7, 148.0ms, 0 | 20260507 14:01:46.959 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 14:01:46.959 | INFO | Uploaded 36GiB to GPU in 1.65821s (21.7102GiB/s) | 20260507 14:01:46.959 | INFO | Downloaded 38GiB to GPU in 1.72449s (22.0355GiB/s) | 20260507 14:01:52.098 | INFO | Analyzing chunk frames = [57856,58368) | 20260507 14:01:52.098 | INFO | MemInfo: { | 20260507 14:01:52.098 | INFO | "gpu" : " 17 GB", | 20260507 14:01:52.098 | INFO | "huge" : "", | 20260507 14:01:52.098 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:52.098 | INFO | "malloc" : " 15 GB", | 20260507 14:01:52.098 | INFO | "pinned" : " 29 GB" | 20260507 14:01:52.098 | INFO | } | 20260507 14:01:52.098 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:01:52.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:52.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260507 14:01:52.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | PRE_HQ : 30.5%, 1.0, 16.4ms, 0 | 20260507 14:01:52.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:52.098 | INFO | Uploaded 36GiB to GPU in 1.60157s (22.478GiB/s) | 20260507 14:01:52.098 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3576GiB/s) | 20260507 14:01:57.237 | INFO | Analyzing chunk frames = [58368,58880) | 20260507 14:01:57.237 | INFO | MemInfo: { | 20260507 14:01:57.237 | INFO | "gpu" : " 17 GB", | 20260507 14:01:57.237 | INFO | "huge" : "", | 20260507 14:01:57.237 | INFO | "huge_shared" : " 29 GB", | 20260507 14:01:57.237 | INFO | "malloc" : " 15 GB", | 20260507 14:01:57.237 | INFO | "pinned" : " 29 GB" | 20260507 14:01:57.237 | INFO | } | 20260507 14:01:57.237 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:01:57.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:01:57.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.237 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260507 14:01:57.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.237 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:01:57.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:01:57.238 | INFO | Uploaded 36GiB to GPU in 1.60592s (22.417GiB/s) | 20260507 14:01:57.238 | INFO | Downloaded 34GiB to GPU in 1.52498s (22.2953GiB/s) | 20260507 14:02:02.376 | INFO | Analyzing chunk frames = [58880,59392) | 20260507 14:02:02.376 | INFO | MemInfo: { | 20260507 14:02:02.376 | INFO | "gpu" : " 17 GB", | 20260507 14:02:02.376 | INFO | "huge" : "", | 20260507 14:02:02.376 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:02.376 | INFO | "malloc" : " 15 GB", | 20260507 14:02:02.377 | INFO | "pinned" : " 29 GB" | 20260507 14:02:02.377 | INFO | } | 20260507 14:02:02.377 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:02:02.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:02.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260507 14:02:02.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:02:02.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:02.377 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260507 14:02:02.377 | INFO | Downloaded 34GiB to GPU in 1.52426s (22.3059GiB/s) | 20260507 14:02:07.516 | INFO | Analyzing chunk frames = [59392,59904) | 20260507 14:02:07.516 | INFO | MemInfo: { | 20260507 14:02:07.516 | INFO | "gpu" : " 17 GB", | 20260507 14:02:07.516 | INFO | "huge" : "", | 20260507 14:02:07.516 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:07.516 | INFO | "malloc" : " 15 GB", | 20260507 14:02:07.516 | INFO | "pinned" : " 29 GB" | 20260507 14:02:07.516 | INFO | } | 20260507 14:02:07.516 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:02:07.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:07.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:02:07.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 14:02:07.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:07.516 | INFO | Uploaded 36GiB to GPU in 1.60435s (22.4389GiB/s) | 20260507 14:02:07.516 | INFO | Downloaded 34GiB to GPU in 1.5228s (22.3273GiB/s) | 20260507 14:02:12.655 | INFO | Analyzing chunk frames = [59904,60416) | 20260507 14:02:12.655 | INFO | MemInfo: { | 20260507 14:02:12.655 | INFO | "gpu" : " 17 GB", | 20260507 14:02:12.655 | INFO | "huge" : "", | 20260507 14:02:12.655 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:12.655 | INFO | "malloc" : " 15 GB", | 20260507 14:02:12.655 | INFO | "pinned" : " 29 GB" | 20260507 14:02:12.655 | INFO | } | 20260507 14:02:12.655 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:02:12.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:12.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:02:12.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 14:02:12.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:12.655 | INFO | Uploaded 36GiB to GPU in 1.60403s (22.4435GiB/s) | 20260507 14:02:12.656 | INFO | Downloaded 34GiB to GPU in 1.52222s (22.3358GiB/s) | 20260507 14:02:17.794 | INFO | Analyzing chunk frames = [60416,60928) | 20260507 14:02:17.794 | INFO | MemInfo: { | 20260507 14:02:17.794 | INFO | "gpu" : " 17 GB", | 20260507 14:02:17.795 | INFO | "huge" : "", | 20260507 14:02:17.795 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:17.795 | INFO | "malloc" : " 15 GB", | 20260507 14:02:17.795 | INFO | "pinned" : " 29 GB" | 20260507 14:02:17.795 | INFO | } | 20260507 14:02:17.795 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:02:17.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:17.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:02:17.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:02:17.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:17.795 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260507 14:02:17.795 | INFO | Downloaded 34GiB to GPU in 1.52534s (22.2901GiB/s) | 20260507 14:02:22.934 | INFO | Analyzing chunk frames = [60928,61440) | 20260507 14:02:22.934 | INFO | MemInfo: { | 20260507 14:02:22.934 | INFO | "gpu" : " 17 GB", | 20260507 14:02:22.934 | INFO | "huge" : "", | 20260507 14:02:22.934 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:22.934 | INFO | "malloc" : " 15 GB", | 20260507 14:02:22.934 | INFO | "pinned" : " 29 GB" | 20260507 14:02:22.934 | INFO | } | 20260507 14:02:22.934 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:02:22.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:22.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:02:22.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:02:22.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:22.934 | INFO | Uploaded 36GiB to GPU in 1.60359s (22.4497GiB/s) | 20260507 14:02:22.934 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3123GiB/s) | 20260507 14:02:23.032 | INFO | Starting a round of Baz IO | 20260507 14:02:25.940 | INFO | Populating RTMetrics report | 20260507 14:02:25.941 | INFO | RTMetrics report populated | 20260507 14:02:25.941 | INFO | Opening tmp RTMetrics json file | 20260507 14:02:25.948 | INFO | Serializing RTMetrics to json | 20260507 14:02:26.001 | INFO | Serialization done, swapping tmp file now... | 20260507 14:02:26.014 | INFO | RTMetrics done | 20260507 14:02:26.039 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.007 seconds | 20260507 14:02:26.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 66.87% / 0.55% / 32.58% | 20260507 14:02:26.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.78758% / 0.00243187% / 0% | 20260507 14:02:26.039 | INFO | Baz IO overall: | 20260507 14:02:26.039 | INFO | BytesWritten : 7176473546 | 20260507 14:02:26.039 | INFO | NumEvents : 2030928548 | 20260507 14:02:28.073 | INFO | Analyzing chunk frames = [61440,61952) | 20260507 14:02:28.073 | INFO | MemInfo: { | 20260507 14:02:28.073 | INFO | "gpu" : " 17 GB", | 20260507 14:02:28.073 | INFO | "huge" : "", | 20260507 14:02:28.073 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:28.073 | INFO | "malloc" : " 15 GB", | 20260507 14:02:28.073 | INFO | "pinned" : " 29 GB" | 20260507 14:02:28.073 | INFO | } | 20260507 14:02:28.073 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:02:28.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:28.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:28.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:28.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:28.073 | INFO | ANALYSIS : 57.6%, 3.0, 89.7ms, 0 | 20260507 14:02:28.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:28.073 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 14:02:28.073 | INFO | PRE_HQ : 56.9%, 2.1, 61.8ms, 0 | 20260507 14:02:28.073 | INFO | BAZWRITER : 58.1%, 3.7, 108.9ms, 0 | 20260507 14:02:28.073 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 14:02:28.074 | INFO | Uploaded 36GiB to GPU in 1.63611s (22.0034GiB/s) | 20260507 14:02:28.074 | INFO | Downloaded 38GiB to GPU in 1.6988s (22.3688GiB/s) | 20260507 14:02:33.212 | INFO | Analyzing chunk frames = [61952,62464) | 20260507 14:02:33.212 | INFO | MemInfo: { | 20260507 14:02:33.212 | INFO | "gpu" : " 17 GB", | 20260507 14:02:33.213 | INFO | "huge" : "", | 20260507 14:02:33.213 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:33.213 | INFO | "malloc" : " 15 GB", | 20260507 14:02:33.213 | INFO | "pinned" : " 29 GB" | 20260507 14:02:33.213 | INFO | } | 20260507 14:02:33.213 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:02:33.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:33.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260507 14:02:33.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 14:02:33.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:33.213 | INFO | Uploaded 36GiB to GPU in 1.60206s (22.4711GiB/s) | 20260507 14:02:33.213 | INFO | Downloaded 34GiB to GPU in 1.521s (22.3537GiB/s) | 20260507 14:02:38.352 | INFO | Analyzing chunk frames = [62464,62976) | 20260507 14:02:38.352 | INFO | MemInfo: { | 20260507 14:02:38.352 | INFO | "gpu" : " 17 GB", | 20260507 14:02:38.352 | INFO | "huge" : "", | 20260507 14:02:38.352 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:38.352 | INFO | "malloc" : " 15 GB", | 20260507 14:02:38.352 | INFO | "pinned" : " 29 GB" | 20260507 14:02:38.352 | INFO | } | 20260507 14:02:38.352 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:02:38.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:38.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:02:38.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 14:02:38.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:38.352 | INFO | Uploaded 36GiB to GPU in 1.60517s (22.4276GiB/s) | 20260507 14:02:38.352 | INFO | Downloaded 34GiB to GPU in 1.52652s (22.2728GiB/s) | 20260507 14:02:43.491 | INFO | Analyzing chunk frames = [62976,63488) | 20260507 14:02:43.491 | INFO | MemInfo: { | 20260507 14:02:43.491 | INFO | "gpu" : " 17 GB", | 20260507 14:02:43.491 | INFO | "huge" : "", | 20260507 14:02:43.491 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:43.491 | INFO | "malloc" : " 15 GB", | 20260507 14:02:43.491 | INFO | "pinned" : " 29 GB" | 20260507 14:02:43.491 | INFO | } | 20260507 14:02:43.491 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:02:43.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:43.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.491 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260507 14:02:43.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.491 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:02:43.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:43.492 | INFO | Uploaded 36GiB to GPU in 1.60485s (22.432GiB/s) | 20260507 14:02:43.492 | INFO | Downloaded 34GiB to GPU in 1.52478s (22.2983GiB/s) | 20260507 14:02:48.631 | INFO | Analyzing chunk frames = [63488,64000) | 20260507 14:02:48.631 | INFO | MemInfo: { | 20260507 14:02:48.631 | INFO | "gpu" : " 17 GB", | 20260507 14:02:48.631 | INFO | "huge" : "", | 20260507 14:02:48.631 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:48.631 | INFO | "malloc" : " 15 GB", | 20260507 14:02:48.631 | INFO | "pinned" : " 29 GB" | 20260507 14:02:48.631 | INFO | } | 20260507 14:02:48.631 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:02:48.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:48.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:02:48.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 14:02:48.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:48.631 | INFO | Uploaded 36GiB to GPU in 1.60335s (22.453GiB/s) | 20260507 14:02:48.631 | INFO | Downloaded 34GiB to GPU in 1.5235s (22.3171GiB/s) | 20260507 14:02:53.770 | INFO | Analyzing chunk frames = [64000,64512) | 20260507 14:02:53.770 | INFO | MemInfo: { | 20260507 14:02:53.770 | INFO | "gpu" : " 17 GB", | 20260507 14:02:53.770 | INFO | "huge" : "", | 20260507 14:02:53.770 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:53.770 | INFO | "malloc" : " 15 GB", | 20260507 14:02:53.770 | INFO | "pinned" : " 29 GB" | 20260507 14:02:53.770 | INFO | } | 20260507 14:02:53.770 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:02:53.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:53.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:02:53.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:02:53.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:53.770 | INFO | Uploaded 36GiB to GPU in 1.60336s (22.4528GiB/s) | 20260507 14:02:53.770 | INFO | Downloaded 34GiB to GPU in 1.52294s (22.3252GiB/s) | 20260507 14:02:58.909 | INFO | Analyzing chunk frames = [64512,65024) | 20260507 14:02:58.909 | INFO | MemInfo: { | 20260507 14:02:58.909 | INFO | "gpu" : " 17 GB", | 20260507 14:02:58.909 | INFO | "huge" : "", | 20260507 14:02:58.909 | INFO | "huge_shared" : " 29 GB", | 20260507 14:02:58.909 | INFO | "malloc" : " 15 GB", | 20260507 14:02:58.909 | INFO | "pinned" : " 29 GB" | 20260507 14:02:58.909 | INFO | } | 20260507 14:02:58.910 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:02:58.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:02:58.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:02:58.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 14:02:58.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:02:58.910 | INFO | Uploaded 36GiB to GPU in 1.60171s (22.476GiB/s) | 20260507 14:02:58.910 | INFO | Downloaded 34GiB to GPU in 1.52359s (22.3158GiB/s) | 20260507 14:03:04.049 | INFO | Analyzing chunk frames = [65024,65536) | 20260507 14:03:04.049 | INFO | MemInfo: { | 20260507 14:03:04.049 | INFO | "gpu" : " 17 GB", | 20260507 14:03:04.049 | INFO | "huge" : "", | 20260507 14:03:04.049 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:04.049 | INFO | "malloc" : " 15 GB", | 20260507 14:03:04.049 | INFO | "pinned" : " 29 GB" | 20260507 14:03:04.049 | INFO | } | 20260507 14:03:04.049 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:03:04.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:04.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:03:04.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:03:04.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:04.049 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:03:04.049 | INFO | Total mem usage: 15.5 GiB | 20260507 14:03:04.049 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 1.07%) | 20260507 14:03:04.049 | INFO | Uploaded 36GiB to GPU in 1.60373s (22.4476GiB/s) | 20260507 14:03:04.049 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -0.851%) | 20260507 14:03:04.049 | INFO | Downloaded 34GiB to GPU in 1.52443s (22.3034GiB/s) | 20260507 14:03:04.049 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:03:04.049 | INFO | High Water Breakdown: | 20260507 14:03:04.049 | INFO | 57.9% : 8.91 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:03:04.049 | INFO | 41.9% : 6.45 GiB ( 96% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:03:04.049 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:03:04.049 | INFO | ------------------------------------------ | 20260507 14:03:04.049 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:03:04.049 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:03:04.049 | INFO | Total mem usage: 26.5 GiB | 20260507 14:03:04.049 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:03:04.049 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:03:04.049 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:03:04.049 | INFO | High Water Breakdown: | 20260507 14:03:04.050 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:03:04.050 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:03:04.050 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:03:04.050 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:03:04.050 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:03:04.050 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:03:04.050 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:03:04.050 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:03:04.050 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:03:04.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:03:04.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:03:04.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:03:04.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:03:04.050 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:03:04.050 | INFO | ------------------------------------------ | 20260507 14:03:04.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:03:04.050 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:03:04.050 | INFO | Total mem usage: 27.8 GiB | 20260507 14:03:04.050 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:03:04.050 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:03:04.050 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:03:04.050 | INFO | High Water Breakdown: | 20260507 14:03:04.050 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:03:04.050 | INFO | ------------------------------------------ | 20260507 14:03:04.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:03:04.050 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:03:04.050 | INFO | Total mem usage: 15.7 GiB | 20260507 14:03:04.050 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:03:04.050 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:03:04.050 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:03:04.050 | INFO | High Water Breakdown: | 20260507 14:03:04.050 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:03:04.050 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:03:04.050 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:03:04.050 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:03:04.050 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:03:04.050 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:03:04.050 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:03:04.050 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:03:04.050 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:03:04.050 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:03:04.050 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:03:04.050 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:03:04.050 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:03:04.050 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:03:04.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:03:04.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:03:04.050 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:03:04.050 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:03:04.050 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:03:04.050 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:03:04.050 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:03:04.050 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:03:04.050 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:03:04.050 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:03:04.050 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:03:04.050 | INFO | ------------------------------------------ | 20260507 14:03:04.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:03:04.050 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:03:04.050 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:03:04.050 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:03:04.050 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.45e-03 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 3.04%, Histogram -- 9.18e-01 / 0.00e+00, 5.11e-02 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 16.04%, DME -- 3.87e+01 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.38e-02 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.25e+00 / 0.00e+00, 1.36e-01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 3.79%, Metrics -- 1.14e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.32e-01 / 0.00e+00, 6.42e-02 / 0.00e+00 | 20260507 14:03:04.050 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:03:04.050 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:03:04.050 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:03:04.050 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.06e-01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 17.86%, Download -- 1.61e+01 / 0.00e+00, 7.36e-01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 33.41%, Compute -- 3.02e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 0.00%, LockDownload -- 4.45e-04 / 0.00e+00, 1.81e-04 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 29.58%, LockCompute -- 2.67e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:03:04.050 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 14:03:04.146 | INFO | Starting a round of Baz IO | 20260507 14:03:07.058 | INFO | Populating RTMetrics report | 20260507 14:03:07.058 | INFO | RTMetrics report populated | 20260507 14:03:07.058 | INFO | Opening tmp RTMetrics json file | 20260507 14:03:07.112 | INFO | Serializing RTMetrics to json | 20260507 14:03:07.120 | INFO | Serialization done, swapping tmp file now... | 20260507 14:03:07.127 | INFO | RTMetrics done | 20260507 14:03:07.144 | INFO | Finished a round of Baz IO: Wrote 1.5 GiB in 2.998 seconds | 20260507 14:03:07.145 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 66.37% / 0.50% / 33.13% | 20260507 14:03:07.145 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.80091% / 0.00205437% / 0% | 20260507 14:03:07.145 | INFO | Baz IO overall: | 20260507 14:03:07.145 | INFO | BytesWritten : 8787281674 | 20260507 14:03:07.145 | INFO | NumEvents : 2821395675 | 20260507 14:03:09.188 | INFO | Analyzing chunk frames = [65536,66048) | 20260507 14:03:09.188 | INFO | MemInfo: { | 20260507 14:03:09.188 | INFO | "gpu" : " 17 GB", | 20260507 14:03:09.188 | INFO | "huge" : "", | 20260507 14:03:09.188 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:09.188 | INFO | "malloc" : " 15 GB", | 20260507 14:03:09.188 | INFO | "pinned" : " 29 GB" | 20260507 14:03:09.188 | INFO | } | 20260507 14:03:09.188 | INFO | Perf report for the last 5.13891 seconds: | 20260507 14:03:09.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:09.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:09.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:09.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:09.188 | INFO | ANALYSIS : 57.7%, 3.0, 89.6ms, 0 | 20260507 14:03:09.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:09.188 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 14:03:09.188 | INFO | PRE_HQ : 56.7%, 2.1, 61.5ms, 0 | 20260507 14:03:09.188 | INFO | BAZWRITER : 57.9%, 3.8, 113.9ms, 0 | 20260507 14:03:09.188 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:03:09.188 | INFO | Uploaded 36GiB to GPU in 1.63994s (21.952GiB/s) | 20260507 14:03:09.188 | INFO | Downloaded 38GiB to GPU in 1.7031s (22.3123GiB/s) | 20260507 14:03:14.327 | INFO | Analyzing chunk frames = [66048,66560) | 20260507 14:03:14.327 | INFO | MemInfo: { | 20260507 14:03:14.327 | INFO | "gpu" : " 17 GB", | 20260507 14:03:14.327 | INFO | "huge" : "", | 20260507 14:03:14.327 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:14.327 | INFO | "malloc" : " 15 GB", | 20260507 14:03:14.327 | INFO | "pinned" : " 29 GB" | 20260507 14:03:14.327 | INFO | } | 20260507 14:03:14.328 | INFO | Perf report for the last 5.1396 seconds: | 20260507 14:03:14.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:14.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260507 14:03:14.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:03:14.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:14.328 | INFO | Uploaded 36GiB to GPU in 1.60163s (22.4771GiB/s) | 20260507 14:03:14.328 | INFO | Downloaded 34GiB to GPU in 1.52062s (22.3594GiB/s) | 20260507 14:03:19.467 | INFO | Analyzing chunk frames = [66560,67072) | 20260507 14:03:19.467 | INFO | MemInfo: { | 20260507 14:03:19.467 | INFO | "gpu" : " 17 GB", | 20260507 14:03:19.467 | INFO | "huge" : "", | 20260507 14:03:19.467 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:19.467 | INFO | "malloc" : " 15 GB", | 20260507 14:03:19.467 | INFO | "pinned" : " 29 GB" | 20260507 14:03:19.467 | INFO | } | 20260507 14:03:19.467 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:03:19.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:19.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260507 14:03:19.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:03:19.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:19.467 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4255GiB/s) | 20260507 14:03:19.467 | INFO | Downloaded 34GiB to GPU in 1.52469s (22.2996GiB/s) | 20260507 14:03:24.606 | INFO | Analyzing chunk frames = [67072,67584) | 20260507 14:03:24.606 | INFO | MemInfo: { | 20260507 14:03:24.606 | INFO | "gpu" : " 17 GB", | 20260507 14:03:24.606 | INFO | "huge" : "", | 20260507 14:03:24.606 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:24.606 | INFO | "malloc" : " 15 GB", | 20260507 14:03:24.606 | INFO | "pinned" : " 29 GB" | 20260507 14:03:24.606 | INFO | } | 20260507 14:03:24.606 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:03:24.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:24.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:03:24.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:03:24.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:24.606 | INFO | Uploaded 36GiB to GPU in 1.60544s (22.4237GiB/s) | 20260507 14:03:24.606 | INFO | Downloaded 34GiB to GPU in 1.52503s (22.2947GiB/s) | 20260507 14:03:29.745 | INFO | Analyzing chunk frames = [67584,68096) | 20260507 14:03:29.745 | INFO | MemInfo: { | 20260507 14:03:29.745 | INFO | "gpu" : " 17 GB", | 20260507 14:03:29.745 | INFO | "huge" : "", | 20260507 14:03:29.745 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:29.745 | INFO | "malloc" : " 15 GB", | 20260507 14:03:29.745 | INFO | "pinned" : " 29 GB" | 20260507 14:03:29.745 | INFO | } | 20260507 14:03:29.745 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:03:29.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:29.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.745 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:03:29.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.745 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 14:03:29.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:29.746 | INFO | Uploaded 36GiB to GPU in 1.60375s (22.4475GiB/s) | 20260507 14:03:29.746 | INFO | Downloaded 34GiB to GPU in 1.5215s (22.3464GiB/s) | 20260507 14:03:34.885 | INFO | Analyzing chunk frames = [68096,68608) | 20260507 14:03:34.885 | INFO | MemInfo: { | 20260507 14:03:34.885 | INFO | "gpu" : " 17 GB", | 20260507 14:03:34.885 | INFO | "huge" : "", | 20260507 14:03:34.885 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:34.885 | INFO | "malloc" : " 15 GB", | 20260507 14:03:34.885 | INFO | "pinned" : " 29 GB" | 20260507 14:03:34.885 | INFO | } | 20260507 14:03:34.885 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:03:34.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:34.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:03:34.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:03:34.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:34.885 | INFO | Uploaded 36GiB to GPU in 1.60404s (22.4433GiB/s) | 20260507 14:03:34.885 | INFO | Downloaded 34GiB to GPU in 1.5229s (22.3258GiB/s) | 20260507 14:03:40.024 | INFO | Analyzing chunk frames = [68608,69120) | 20260507 14:03:40.024 | INFO | MemInfo: { | 20260507 14:03:40.024 | INFO | "gpu" : " 17 GB", | 20260507 14:03:40.024 | INFO | "huge" : "", | 20260507 14:03:40.024 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:40.024 | INFO | "malloc" : " 15 GB", | 20260507 14:03:40.024 | INFO | "pinned" : " 29 GB" | 20260507 14:03:40.024 | INFO | } | 20260507 14:03:40.024 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:03:40.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:40.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:03:40.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 14:03:40.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:40.024 | INFO | Uploaded 36GiB to GPU in 1.6045s (22.4369GiB/s) | 20260507 14:03:40.024 | INFO | Downloaded 34GiB to GPU in 1.52339s (22.3186GiB/s) | 20260507 14:03:45.163 | INFO | Analyzing chunk frames = [69120,69632) | 20260507 14:03:45.163 | INFO | MemInfo: { | 20260507 14:03:45.163 | INFO | "gpu" : " 17 GB", | 20260507 14:03:45.163 | INFO | "huge" : "", | 20260507 14:03:45.163 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:45.163 | INFO | "malloc" : " 15 GB", | 20260507 14:03:45.163 | INFO | "pinned" : " 29 GB" | 20260507 14:03:45.163 | INFO | } | 20260507 14:03:45.163 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:03:45.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:45.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.163 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:03:45.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.164 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:03:45.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:45.164 | INFO | Uploaded 36GiB to GPU in 1.60497s (22.4303GiB/s) | 20260507 14:03:45.164 | INFO | Downloaded 34GiB to GPU in 1.52364s (22.3151GiB/s) | 20260507 14:03:45.261 | INFO | Starting a round of Baz IO | 20260507 14:03:48.174 | INFO | Populating RTMetrics report | 20260507 14:03:48.174 | INFO | RTMetrics report populated | 20260507 14:03:48.174 | INFO | Opening tmp RTMetrics json file | 20260507 14:03:48.188 | INFO | Serializing RTMetrics to json | 20260507 14:03:48.205 | INFO | Serialization done, swapping tmp file now... | 20260507 14:03:48.223 | INFO | RTMetrics done | 20260507 14:03:48.254 | INFO | Finished a round of Baz IO: Wrote 1.516 GiB in 2.993 seconds | 20260507 14:03:48.254 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.92% / 0.45% / 33.63% | 20260507 14:03:48.254 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.81714% / 0.0019153% / 0% | 20260507 14:03:48.254 | INFO | Baz IO overall: | 20260507 14:03:48.254 | INFO | BytesWritten : 10415523738 | 20260507 14:03:48.254 | INFO | NumEvents : 3619982170 | 20260507 14:03:50.303 | INFO | Analyzing chunk frames = [69632,70144) | 20260507 14:03:50.303 | INFO | MemInfo: { | 20260507 14:03:50.303 | INFO | "gpu" : " 17 GB", | 20260507 14:03:50.303 | INFO | "huge" : "", | 20260507 14:03:50.303 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:50.303 | INFO | "malloc" : " 15 GB", | 20260507 14:03:50.303 | INFO | "pinned" : " 29 GB" | 20260507 14:03:50.303 | INFO | } | 20260507 14:03:50.303 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:03:50.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:50.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:50.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:50.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:50.303 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260507 14:03:50.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:50.303 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 14:03:50.303 | INFO | PRE_HQ : 56.7%, 2.1, 61.6ms, 0 | 20260507 14:03:50.303 | INFO | BAZWRITER : 57.8%, 3.7, 110.9ms, 0 | 20260507 14:03:50.303 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 14:03:50.303 | INFO | Uploaded 36GiB to GPU in 1.64031s (21.947GiB/s) | 20260507 14:03:50.303 | INFO | Downloaded 38GiB to GPU in 1.70364s (22.3052GiB/s) | 20260507 14:03:55.442 | INFO | Analyzing chunk frames = [70144,70656) | 20260507 14:03:55.442 | INFO | MemInfo: { | 20260507 14:03:55.442 | INFO | "gpu" : " 17 GB", | 20260507 14:03:55.442 | INFO | "huge" : "", | 20260507 14:03:55.442 | INFO | "huge_shared" : " 29 GB", | 20260507 14:03:55.442 | INFO | "malloc" : " 15 GB", | 20260507 14:03:55.442 | INFO | "pinned" : " 29 GB" | 20260507 14:03:55.442 | INFO | } | 20260507 14:03:55.442 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:03:55.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:03:55.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260507 14:03:55.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260507 14:03:55.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:03:55.442 | INFO | Uploaded 36GiB to GPU in 1.6018s (22.4748GiB/s) | 20260507 14:03:55.442 | INFO | Downloaded 34GiB to GPU in 1.5208s (22.3566GiB/s) | 20260507 14:04:00.581 | INFO | Analyzing chunk frames = [70656,71168) | 20260507 14:04:00.581 | INFO | MemInfo: { | 20260507 14:04:00.581 | INFO | "gpu" : " 17 GB", | 20260507 14:04:00.581 | INFO | "huge" : "", | 20260507 14:04:00.581 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:00.581 | INFO | "malloc" : " 15 GB", | 20260507 14:04:00.581 | INFO | "pinned" : " 29 GB" | 20260507 14:04:00.581 | INFO | } | 20260507 14:04:00.581 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:04:00.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:00.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.581 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260507 14:04:00.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.582 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:04:00.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:00.582 | INFO | Uploaded 36GiB to GPU in 1.6052s (22.4271GiB/s) | 20260507 14:04:00.582 | INFO | Downloaded 34GiB to GPU in 1.52482s (22.2977GiB/s) | 20260507 14:04:05.721 | INFO | Analyzing chunk frames = [71168,71680) | 20260507 14:04:05.721 | INFO | MemInfo: { | 20260507 14:04:05.721 | INFO | "gpu" : " 17 GB", | 20260507 14:04:05.721 | INFO | "huge" : "", | 20260507 14:04:05.721 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:05.721 | INFO | "malloc" : " 15 GB", | 20260507 14:04:05.721 | INFO | "pinned" : " 29 GB" | 20260507 14:04:05.721 | INFO | } | 20260507 14:04:05.721 | INFO | Perf report for the last 5.1396 seconds: | 20260507 14:04:05.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:05.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 14:04:05.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 14:04:05.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:05.721 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.4279GiB/s) | 20260507 14:04:05.721 | INFO | Downloaded 34GiB to GPU in 1.52427s (22.3057GiB/s) | 20260507 14:04:10.860 | INFO | Analyzing chunk frames = [71680,72192) | 20260507 14:04:10.860 | INFO | MemInfo: { | 20260507 14:04:10.860 | INFO | "gpu" : " 17 GB", | 20260507 14:04:10.860 | INFO | "huge" : "", | 20260507 14:04:10.860 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:10.860 | INFO | "malloc" : " 15 GB", | 20260507 14:04:10.860 | INFO | "pinned" : " 29 GB" | 20260507 14:04:10.860 | INFO | } | 20260507 14:04:10.860 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:04:10.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:10.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.860 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:04:10.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.860 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:04:10.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:10.861 | INFO | Uploaded 36GiB to GPU in 1.60373s (22.4476GiB/s) | 20260507 14:04:10.861 | INFO | Downloaded 34GiB to GPU in 1.5225s (22.3317GiB/s) | 20260507 14:04:15.999 | INFO | Analyzing chunk frames = [72192,72704) | 20260507 14:04:15.999 | INFO | MemInfo: { | 20260507 14:04:15.999 | INFO | "gpu" : " 17 GB", | 20260507 14:04:15.999 | INFO | "huge" : "", | 20260507 14:04:16.000 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:16.000 | INFO | "malloc" : " 15 GB", | 20260507 14:04:16.000 | INFO | "pinned" : " 29 GB" | 20260507 14:04:16.000 | INFO | } | 20260507 14:04:16.000 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:04:16.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:16.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:04:16.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:04:16.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:16.000 | INFO | Uploaded 36GiB to GPU in 1.60393s (22.4448GiB/s) | 20260507 14:04:16.000 | INFO | Downloaded 34GiB to GPU in 1.52349s (22.3172GiB/s) | 20260507 14:04:21.139 | INFO | Analyzing chunk frames = [72704,73216) | 20260507 14:04:21.139 | INFO | MemInfo: { | 20260507 14:04:21.139 | INFO | "gpu" : " 17 GB", | 20260507 14:04:21.139 | INFO | "huge" : "", | 20260507 14:04:21.139 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:21.139 | INFO | "malloc" : " 15 GB", | 20260507 14:04:21.139 | INFO | "pinned" : " 29 GB" | 20260507 14:04:21.139 | INFO | } | 20260507 14:04:21.139 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:04:21.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:21.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:04:21.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 14:04:21.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:21.139 | INFO | Uploaded 36GiB to GPU in 1.60389s (22.4454GiB/s) | 20260507 14:04:21.139 | INFO | Downloaded 34GiB to GPU in 1.52314s (22.3223GiB/s) | 20260507 14:04:26.278 | INFO | Analyzing chunk frames = [73216,73728) | 20260507 14:04:26.278 | INFO | MemInfo: { | 20260507 14:04:26.278 | INFO | "gpu" : " 17 GB", | 20260507 14:04:26.278 | INFO | "huge" : "", | 20260507 14:04:26.278 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:26.278 | INFO | "malloc" : " 15 GB", | 20260507 14:04:26.278 | INFO | "pinned" : " 29 GB" | 20260507 14:04:26.278 | INFO | } | 20260507 14:04:26.278 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:04:26.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:26.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.278 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:04:26.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.278 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:04:26.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:26.279 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:04:26.279 | INFO | Total mem usage: 15.5 GiB | 20260507 14:04:26.279 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 1.07%) | 20260507 14:04:26.279 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.63%) | 20260507 14:04:26.279 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:04:26.279 | INFO | High Water Breakdown: | 20260507 14:04:26.279 | INFO | 57.9% : 8.58 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:04:26.279 | INFO | 42% : 6.22 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:04:26.279 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:04:26.279 | INFO | ------------------------------------------ | 20260507 14:04:26.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:04:26.279 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:04:26.279 | INFO | Total mem usage: 26.5 GiB | 20260507 14:04:26.279 | INFO | Uploaded 36GiB to GPU in 1.60433s (22.4392GiB/s) | 20260507 14:04:26.279 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:04:26.279 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:04:26.279 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:04:26.279 | INFO | High Water Breakdown: | 20260507 14:04:26.279 | INFO | Downloaded 34GiB to GPU in 1.5236s (22.3156GiB/s) | 20260507 14:04:26.279 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:04:26.279 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:04:26.279 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:04:26.279 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:04:26.279 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:04:26.279 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:04:26.279 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:04:26.279 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:04:26.279 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:04:26.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:04:26.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:04:26.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:04:26.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:04:26.279 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:04:26.279 | INFO | ------------------------------------------ | 20260507 14:04:26.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:04:26.279 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:04:26.279 | INFO | Total mem usage: 27.8 GiB | 20260507 14:04:26.279 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:04:26.279 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:04:26.279 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:04:26.279 | INFO | High Water Breakdown: | 20260507 14:04:26.279 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:04:26.279 | INFO | ------------------------------------------ | 20260507 14:04:26.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:04:26.279 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:04:26.279 | INFO | Total mem usage: 15.7 GiB | 20260507 14:04:26.279 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:04:26.279 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:04:26.279 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:04:26.279 | INFO | High Water Breakdown: | 20260507 14:04:26.279 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:04:26.279 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:04:26.279 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:04:26.279 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:04:26.280 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:04:26.280 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:04:26.280 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:04:26.280 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:04:26.280 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:04:26.280 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:04:26.280 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:04:26.280 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:04:26.280 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:04:26.280 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:04:26.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:04:26.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:04:26.280 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:04:26.280 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:04:26.280 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:04:26.280 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:04:26.280 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:04:26.280 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:04:26.280 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:04:26.280 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:04:26.280 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:04:26.280 | INFO | ------------------------------------------ | 20260507 14:04:26.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:04:26.280 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:04:26.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:04:26.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:04:26.280 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / 0.00e+00, 5.45e-03 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 3.05%, Histogram -- 9.20e-01 / 0.00e+00, 5.22e-02 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 16.00%, DME -- 3.86e+01 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.49e-02 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.25e+00 / 0.00e+00, 1.36e-01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 3.81%, Metrics -- 1.15e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.34e-01 / 0.00e+00, 6.33e-02 / 0.00e+00 | 20260507 14:04:26.280 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:04:26.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:04:26.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:04:26.280 | INFO | PerfReport: Full Execution -- 9.06e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 18.50%, Upload -- 1.68e+01 / 0.00e+00, 3.81e-01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 17.78%, Download -- 1.61e+01 / 0.00e+00, 6.95e-01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 33.30%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 0.00%, LockDownload -- 4.43e-04 / 0.00e+00, 1.80e-04 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 29.85%, LockCompute -- 2.70e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260507 14:04:26.280 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 14:04:26.376 | INFO | Starting a round of Baz IO | 20260507 14:04:29.287 | INFO | Populating RTMetrics report | 20260507 14:04:29.287 | INFO | RTMetrics report populated | 20260507 14:04:29.287 | INFO | Opening tmp RTMetrics json file | 20260507 14:04:29.320 | INFO | Serializing RTMetrics to json | 20260507 14:04:29.343 | INFO | Serialization done, swapping tmp file now... | 20260507 14:04:29.344 | INFO | RTMetrics done | 20260507 14:04:29.368 | INFO | Finished a round of Baz IO: Wrote 1.533 GiB in 2.991 seconds | 20260507 14:04:29.368 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.50% / 0.42% / 34.08% | 20260507 14:04:29.368 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.86618% / 0.00192324% / 7.94729e-06% | 20260507 14:04:29.368 | INFO | Baz IO overall: | 20260507 14:04:29.368 | INFO | BytesWritten : 12061360252 | 20260507 14:04:29.368 | INFO | NumEvents : 4426901993 | 20260507 14:04:31.417 | INFO | Analyzing chunk frames = [73728,74240) | 20260507 14:04:31.417 | INFO | MemInfo: { | 20260507 14:04:31.417 | INFO | "gpu" : " 17 GB", | 20260507 14:04:31.417 | INFO | "huge" : "", | 20260507 14:04:31.418 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:31.418 | INFO | "malloc" : " 15 GB", | 20260507 14:04:31.418 | INFO | "pinned" : " 29 GB" | 20260507 14:04:31.418 | INFO | } | 20260507 14:04:31.418 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:04:31.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:31.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:31.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:31.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:31.418 | INFO | ANALYSIS : 57.7%, 2.9, 89.2ms, 0 | 20260507 14:04:31.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:31.418 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 14:04:31.418 | INFO | PRE_HQ : 56.8%, 2.1, 61.9ms, 0 | 20260507 14:04:31.418 | INFO | BAZWRITER : 57.8%, 3.9, 116.4ms, 0 | 20260507 14:04:31.418 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 14:04:31.418 | INFO | Uploaded 36GiB to GPU in 1.64862s (21.8365GiB/s) | 20260507 14:04:31.418 | INFO | Downloaded 38GiB to GPU in 1.71124s (22.2062GiB/s) | 20260507 14:04:36.557 | INFO | Analyzing chunk frames = [74240,74752) | 20260507 14:04:36.557 | INFO | MemInfo: { | 20260507 14:04:36.557 | INFO | "gpu" : " 17 GB", | 20260507 14:04:36.557 | INFO | "huge" : "", | 20260507 14:04:36.557 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:36.557 | INFO | "malloc" : " 15 GB", | 20260507 14:04:36.557 | INFO | "pinned" : " 29 GB" | 20260507 14:04:36.557 | INFO | } | 20260507 14:04:36.557 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:04:36.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:36.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260507 14:04:36.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:04:36.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:36.557 | INFO | Uploaded 36GiB to GPU in 1.60154s (22.4783GiB/s) | 20260507 14:04:36.558 | INFO | Downloaded 34GiB to GPU in 1.52033s (22.3635GiB/s) | 20260507 14:04:41.696 | INFO | Analyzing chunk frames = [74752,75264) | 20260507 14:04:41.696 | INFO | MemInfo: { | 20260507 14:04:41.696 | INFO | "gpu" : " 17 GB", | 20260507 14:04:41.696 | INFO | "huge" : "", | 20260507 14:04:41.696 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:41.696 | INFO | "malloc" : " 15 GB", | 20260507 14:04:41.696 | INFO | "pinned" : " 29 GB" | 20260507 14:04:41.696 | INFO | } | 20260507 14:04:41.696 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:04:41.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:41.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.696 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260507 14:04:41.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.697 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 14:04:41.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:41.697 | INFO | Uploaded 36GiB to GPU in 1.60584s (22.4181GiB/s) | 20260507 14:04:41.697 | INFO | Downloaded 34GiB to GPU in 1.52495s (22.2958GiB/s) | 20260507 14:04:46.835 | INFO | Analyzing chunk frames = [75264,75776) | 20260507 14:04:46.835 | INFO | MemInfo: { | 20260507 14:04:46.836 | INFO | "gpu" : " 17 GB", | 20260507 14:04:46.836 | INFO | "huge" : "", | 20260507 14:04:46.836 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:46.836 | INFO | "malloc" : " 15 GB", | 20260507 14:04:46.836 | INFO | "pinned" : " 29 GB" | 20260507 14:04:46.836 | INFO | } | 20260507 14:04:46.836 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:04:46.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:46.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260507 14:04:46.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 14:04:46.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:46.836 | INFO | Uploaded 36GiB to GPU in 1.60521s (22.4269GiB/s) | 20260507 14:04:46.836 | INFO | Downloaded 34GiB to GPU in 1.52446s (22.3029GiB/s) | 20260507 14:04:51.975 | INFO | Analyzing chunk frames = [75776,76288) | 20260507 14:04:51.975 | INFO | MemInfo: { | 20260507 14:04:51.975 | INFO | "gpu" : " 17 GB", | 20260507 14:04:51.975 | INFO | "huge" : "", | 20260507 14:04:51.975 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:51.975 | INFO | "malloc" : " 15 GB", | 20260507 14:04:51.975 | INFO | "pinned" : " 29 GB" | 20260507 14:04:51.975 | INFO | } | 20260507 14:04:51.975 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:04:51.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:51.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:04:51.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 14:04:51.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:51.975 | INFO | Uploaded 36GiB to GPU in 1.60381s (22.4466GiB/s) | 20260507 14:04:51.975 | INFO | Downloaded 34GiB to GPU in 1.52238s (22.3335GiB/s) | 20260507 14:04:57.114 | INFO | Analyzing chunk frames = [76288,76800) | 20260507 14:04:57.114 | INFO | MemInfo: { | 20260507 14:04:57.114 | INFO | "gpu" : " 17 GB", | 20260507 14:04:57.114 | INFO | "huge" : "", | 20260507 14:04:57.114 | INFO | "huge_shared" : " 29 GB", | 20260507 14:04:57.114 | INFO | "malloc" : " 15 GB", | 20260507 14:04:57.114 | INFO | "pinned" : " 29 GB" | 20260507 14:04:57.114 | INFO | } | 20260507 14:04:57.114 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:04:57.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:04:57.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.115 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:04:57.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.115 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:04:57.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:04:57.115 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4511GiB/s) | 20260507 14:04:57.115 | INFO | Downloaded 34GiB to GPU in 1.5229s (22.3258GiB/s) | 20260507 14:05:02.254 | INFO | Analyzing chunk frames = [76800,77312) | 20260507 14:05:02.254 | INFO | MemInfo: { | 20260507 14:05:02.254 | INFO | "gpu" : " 17 GB", | 20260507 14:05:02.254 | INFO | "huge" : "", | 20260507 14:05:02.254 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:02.254 | INFO | "malloc" : " 15 GB", | 20260507 14:05:02.254 | INFO | "pinned" : " 29 GB" | 20260507 14:05:02.254 | INFO | } | 20260507 14:05:02.254 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:05:02.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:02.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:05:02.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 14:05:02.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:02.254 | INFO | Uploaded 36GiB to GPU in 1.60371s (22.448GiB/s) | 20260507 14:05:02.254 | INFO | Downloaded 34GiB to GPU in 1.52479s (22.2981GiB/s) | 20260507 14:05:07.393 | INFO | Analyzing chunk frames = [77312,77824) | 20260507 14:05:07.393 | INFO | MemInfo: { | 20260507 14:05:07.393 | INFO | "gpu" : " 17 GB", | 20260507 14:05:07.393 | INFO | "huge" : "", | 20260507 14:05:07.393 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:07.393 | INFO | "malloc" : " 15 GB", | 20260507 14:05:07.393 | INFO | "pinned" : " 29 GB" | 20260507 14:05:07.393 | INFO | } | 20260507 14:05:07.393 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:05:07.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:07.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:05:07.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:05:07.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:07.393 | INFO | Uploaded 36GiB to GPU in 1.60496s (22.4304GiB/s) | 20260507 14:05:07.393 | INFO | Downloaded 34GiB to GPU in 1.52405s (22.309GiB/s) | 20260507 14:05:07.491 | INFO | Starting a round of Baz IO | 20260507 14:05:10.401 | INFO | Populating RTMetrics report | 20260507 14:05:10.401 | INFO | RTMetrics report populated | 20260507 14:05:10.401 | INFO | Opening tmp RTMetrics json file | 20260507 14:05:10.402 | INFO | Serializing RTMetrics to json | 20260507 14:05:10.419 | INFO | Serialization done, swapping tmp file now... | 20260507 14:05:10.429 | INFO | RTMetrics done | 20260507 14:05:10.470 | INFO | Finished a round of Baz IO: Wrote 1.545 GiB in 2.979 seconds | 20260507 14:05:10.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.10% / 0.40% / 34.50% | 20260507 14:05:10.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.89242% / 0.00176827% / 0% | 20260507 14:05:10.470 | INFO | Baz IO overall: | 20260507 14:05:10.470 | INFO | BytesWritten : 13720107094 | 20260507 14:05:10.470 | INFO | NumEvents : 5241116981 | 20260507 14:05:12.532 | INFO | Analyzing chunk frames = [77824,78336) | 20260507 14:05:12.532 | INFO | MemInfo: { | 20260507 14:05:12.532 | INFO | "gpu" : " 17 GB", | 20260507 14:05:12.532 | INFO | "huge" : "", | 20260507 14:05:12.532 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:12.532 | INFO | "malloc" : " 15 GB", | 20260507 14:05:12.532 | INFO | "pinned" : " 29 GB" | 20260507 14:05:12.532 | INFO | } | 20260507 14:05:12.532 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:05:12.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:12.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:12.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:12.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:12.533 | INFO | ANALYSIS : 57.6%, 2.9, 88.8ms, 0 | 20260507 14:05:12.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:12.533 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 14:05:12.533 | INFO | PRE_HQ : 56.6%, 2.1, 61.2ms, 0 | 20260507 14:05:12.533 | INFO | BAZWRITER : 57.5%, 4.1, 120.9ms, 0 | 20260507 14:05:12.533 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 14:05:12.533 | INFO | Uploaded 36GiB to GPU in 1.64359s (21.9033GiB/s) | 20260507 14:05:12.533 | INFO | Downloaded 38GiB to GPU in 1.70681s (22.2637GiB/s) | 20260507 14:05:17.672 | INFO | Analyzing chunk frames = [78336,78848) | 20260507 14:05:17.672 | INFO | MemInfo: { | 20260507 14:05:17.672 | INFO | "gpu" : " 17 GB", | 20260507 14:05:17.672 | INFO | "huge" : "", | 20260507 14:05:17.672 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:17.672 | INFO | "malloc" : " 15 GB", | 20260507 14:05:17.672 | INFO | "pinned" : " 29 GB" | 20260507 14:05:17.672 | INFO | } | 20260507 14:05:17.672 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:05:17.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:17.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260507 14:05:17.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | PRE_HQ : 29.6%, 1.0, 15.5ms, 0 | 20260507 14:05:17.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:17.672 | INFO | Uploaded 36GiB to GPU in 1.60222s (22.4688GiB/s) | 20260507 14:05:17.672 | INFO | Downloaded 34GiB to GPU in 1.52087s (22.3556GiB/s) | 20260507 14:05:22.811 | INFO | Analyzing chunk frames = [78848,79360) | 20260507 14:05:22.811 | INFO | MemInfo: { | 20260507 14:05:22.811 | INFO | "gpu" : " 17 GB", | 20260507 14:05:22.811 | INFO | "huge" : "", | 20260507 14:05:22.811 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:22.811 | INFO | "malloc" : " 15 GB", | 20260507 14:05:22.811 | INFO | "pinned" : " 29 GB" | 20260507 14:05:22.811 | INFO | } | 20260507 14:05:22.811 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:05:22.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:22.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:05:22.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:05:22.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:22.812 | INFO | Uploaded 36GiB to GPU in 1.60637s (22.4107GiB/s) | 20260507 14:05:22.812 | INFO | Downloaded 34GiB to GPU in 1.52586s (22.2825GiB/s) | 20260507 14:05:27.950 | INFO | Analyzing chunk frames = [79360,79872) | 20260507 14:05:27.950 | INFO | MemInfo: { | 20260507 14:05:27.950 | INFO | "gpu" : " 17 GB", | 20260507 14:05:27.950 | INFO | "huge" : "", | 20260507 14:05:27.950 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:27.950 | INFO | "malloc" : " 15 GB", | 20260507 14:05:27.950 | INFO | "pinned" : " 29 GB" | 20260507 14:05:27.950 | INFO | } | 20260507 14:05:27.950 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:05:27.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:27.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.950 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260507 14:05:27.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.950 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 14:05:27.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:27.951 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4215GiB/s) | 20260507 14:05:27.951 | INFO | Downloaded 34GiB to GPU in 1.52501s (22.2949GiB/s) | 20260507 14:05:33.090 | INFO | Analyzing chunk frames = [79872,80384) | 20260507 14:05:33.090 | INFO | MemInfo: { | 20260507 14:05:33.090 | INFO | "gpu" : " 17 GB", | 20260507 14:05:33.090 | INFO | "huge" : "", | 20260507 14:05:33.090 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:33.090 | INFO | "malloc" : " 15 GB", | 20260507 14:05:33.090 | INFO | "pinned" : " 29 GB" | 20260507 14:05:33.090 | INFO | } | 20260507 14:05:33.090 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:05:33.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:33.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:05:33.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 14:05:33.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:33.090 | INFO | Uploaded 36GiB to GPU in 1.60497s (22.4303GiB/s) | 20260507 14:05:33.090 | INFO | Downloaded 34GiB to GPU in 1.52417s (22.3072GiB/s) | 20260507 14:05:38.229 | INFO | Analyzing chunk frames = [80384,80896) | 20260507 14:05:38.229 | INFO | MemInfo: { | 20260507 14:05:38.229 | INFO | "gpu" : " 17 GB", | 20260507 14:05:38.229 | INFO | "huge" : "", | 20260507 14:05:38.229 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:38.229 | INFO | "malloc" : " 15 GB", | 20260507 14:05:38.229 | INFO | "pinned" : " 29 GB" | 20260507 14:05:38.229 | INFO | } | 20260507 14:05:38.229 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:05:38.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:38.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:05:38.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 14:05:38.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:38.229 | INFO | Uploaded 36GiB to GPU in 1.60386s (22.4458GiB/s) | 20260507 14:05:38.229 | INFO | Downloaded 34GiB to GPU in 1.52235s (22.3338GiB/s) | 20260507 14:05:43.368 | INFO | Analyzing chunk frames = [80896,81408) | 20260507 14:05:43.368 | INFO | MemInfo: { | 20260507 14:05:43.368 | INFO | "gpu" : " 17 GB", | 20260507 14:05:43.368 | INFO | "huge" : "", | 20260507 14:05:43.368 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:43.368 | INFO | "malloc" : " 15 GB", | 20260507 14:05:43.368 | INFO | "pinned" : " 29 GB" | 20260507 14:05:43.368 | INFO | } | 20260507 14:05:43.368 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:05:43.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:43.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.368 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:05:43.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.368 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 14:05:43.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:43.369 | INFO | Uploaded 36GiB to GPU in 1.60498s (22.4302GiB/s) | 20260507 14:05:43.369 | INFO | Downloaded 34GiB to GPU in 1.52433s (22.3049GiB/s) | 20260507 14:05:48.508 | INFO | Analyzing chunk frames = [81408,81920) | 20260507 14:05:48.508 | INFO | MemInfo: { | 20260507 14:05:48.508 | INFO | "gpu" : " 17 GB", | 20260507 14:05:48.508 | INFO | "huge" : "", | 20260507 14:05:48.508 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:48.508 | INFO | "malloc" : " 15 GB", | 20260507 14:05:48.508 | INFO | "pinned" : " 29 GB" | 20260507 14:05:48.508 | INFO | } | 20260507 14:05:48.508 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:05:48.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:48.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:05:48.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:05:48.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:48.508 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:05:48.508 | INFO | Total mem usage: 15.5 GiB | 20260507 14:05:48.508 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 1.07%) | 20260507 14:05:48.508 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -3.99%) | 20260507 14:05:48.508 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:05:48.508 | INFO | High Water Breakdown: | 20260507 14:05:48.508 | INFO | 58.1% : 8.67 GiB ( 97% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:05:48.508 | INFO | 41.7% : 6.22 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:05:48.508 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:05:48.508 | INFO | Uploaded 36GiB to GPU in 1.60495s (22.4306GiB/s) | 20260507 14:05:48.508 | INFO | ------------------------------------------ | 20260507 14:05:48.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:05:48.508 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:05:48.508 | INFO | Downloaded 34GiB to GPU in 1.52477s (22.2985GiB/s) | 20260507 14:05:48.508 | INFO | Total mem usage: 26.5 GiB | 20260507 14:05:48.508 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:05:48.508 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:05:48.508 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:05:48.509 | INFO | High Water Breakdown: | 20260507 14:05:48.509 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:05:48.509 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:05:48.509 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:05:48.509 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:05:48.509 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:05:48.509 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:05:48.509 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:05:48.509 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:05:48.509 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:05:48.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:05:48.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:05:48.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:05:48.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:05:48.509 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:05:48.509 | INFO | ------------------------------------------ | 20260507 14:05:48.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:05:48.509 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:05:48.509 | INFO | Total mem usage: 27.8 GiB | 20260507 14:05:48.509 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:05:48.509 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:05:48.509 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:05:48.509 | INFO | High Water Breakdown: | 20260507 14:05:48.509 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:05:48.509 | INFO | ------------------------------------------ | 20260507 14:05:48.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:05:48.509 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:05:48.509 | INFO | Total mem usage: 15.7 GiB | 20260507 14:05:48.509 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:05:48.509 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:05:48.509 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:05:48.509 | INFO | High Water Breakdown: | 20260507 14:05:48.509 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:05:48.509 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:05:48.509 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:05:48.509 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:05:48.509 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:05:48.509 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:05:48.509 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:05:48.509 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:05:48.509 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:05:48.509 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:05:48.509 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:05:48.509 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:05:48.509 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:05:48.509 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:05:48.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:05:48.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:05:48.509 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:05:48.509 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:05:48.509 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:05:48.509 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:05:48.509 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:05:48.509 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:05:48.509 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:05:48.509 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:05:48.509 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:05:48.509 | INFO | ------------------------------------------ | 20260507 14:05:48.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:05:48.509 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:05:48.509 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:05:48.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:05:48.509 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.23e-03 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / 0.00e+00, 5.19e-02 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 15.99%, DME -- 3.86e+01 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.35e-02 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.25e+00 / 0.00e+00, 1.38e-01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 3.81%, Metrics -- 1.15e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.33e-01 / 0.00e+00, 6.34e-02 / 0.00e+00 | 20260507 14:05:48.509 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:05:48.509 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:05:48.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:05:48.509 | INFO | PerfReport: Full Execution -- 9.05e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 18.53%, Upload -- 1.68e+01 / 0.00e+00, 3.96e-01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 17.81%, Download -- 1.61e+01 / 0.00e+00, 7.19e-01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 33.33%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 0.00%, LockDownload -- 4.35e-04 / 0.00e+00, 9.07e-05 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 29.75%, LockCompute -- 2.69e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:05:48.509 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260507 14:05:48.605 | INFO | Starting a round of Baz IO | 20260507 14:05:51.514 | INFO | Populating RTMetrics report | 20260507 14:05:51.514 | INFO | RTMetrics report populated | 20260507 14:05:51.514 | INFO | Opening tmp RTMetrics json file | 20260507 14:05:51.573 | INFO | Serializing RTMetrics to json | 20260507 14:05:51.575 | INFO | Serialization done, swapping tmp file now... | 20260507 14:05:51.583 | INFO | RTMetrics done | 20260507 14:05:51.790 | INFO | Finished a round of Baz IO: Wrote 1.546 GiB in 3.184 seconds | 20260507 14:05:51.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.72% / 0.38% / 34.90% | 20260507 14:05:51.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.75542% / 0.00151793% / 0% | 20260507 14:05:51.790 | INFO | Baz IO overall: | 20260507 14:05:51.790 | INFO | BytesWritten : 15379908936 | 20260507 14:05:51.790 | INFO | NumEvents : 6055484429 | 20260507 14:05:53.647 | INFO | Analyzing chunk frames = [81920,82432) | 20260507 14:05:53.647 | INFO | MemInfo: { | 20260507 14:05:53.647 | INFO | "gpu" : " 17 GB", | 20260507 14:05:53.647 | INFO | "huge" : "", | 20260507 14:05:53.647 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:53.647 | INFO | "malloc" : " 15 GB", | 20260507 14:05:53.647 | INFO | "pinned" : " 29 GB" | 20260507 14:05:53.647 | INFO | } | 20260507 14:05:53.647 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:05:53.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:53.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:53.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:53.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:53.647 | INFO | ANALYSIS : 57.6%, 2.9, 88.9ms, 0 | 20260507 14:05:53.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:53.647 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 14:05:53.647 | INFO | PRE_HQ : 56.6%, 2.1, 60.8ms, 0 | 20260507 14:05:53.647 | INFO | BAZWRITER : 61.5%, 4.0, 126.3ms, 0 | 20260507 14:05:53.647 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 14:05:53.648 | INFO | Uploaded 36GiB to GPU in 1.64494s (21.8853GiB/s) | 20260507 14:05:53.648 | INFO | Downloaded 38GiB to GPU in 1.70914s (22.2334GiB/s) | 20260507 14:05:58.787 | INFO | Analyzing chunk frames = [82432,82944) | 20260507 14:05:58.787 | INFO | MemInfo: { | 20260507 14:05:58.787 | INFO | "gpu" : " 17 GB", | 20260507 14:05:58.787 | INFO | "huge" : "", | 20260507 14:05:58.787 | INFO | "huge_shared" : " 29 GB", | 20260507 14:05:58.787 | INFO | "malloc" : " 15 GB", | 20260507 14:05:58.787 | INFO | "pinned" : " 29 GB" | 20260507 14:05:58.787 | INFO | } | 20260507 14:05:58.787 | INFO | Perf report for the last 5.14002 seconds: | 20260507 14:05:58.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:05:58.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260507 14:05:58.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 14:05:58.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:05:58.787 | INFO | Uploaded 36GiB to GPU in 1.6017s (22.4762GiB/s) | 20260507 14:05:58.787 | INFO | Downloaded 34GiB to GPU in 1.52022s (22.3652GiB/s) | 20260507 14:06:03.926 | INFO | Analyzing chunk frames = [82944,83456) | 20260507 14:06:03.926 | INFO | MemInfo: { | 20260507 14:06:03.926 | INFO | "gpu" : " 17 GB", | 20260507 14:06:03.926 | INFO | "huge" : "", | 20260507 14:06:03.926 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:03.926 | INFO | "malloc" : " 15 GB", | 20260507 14:06:03.926 | INFO | "pinned" : " 29 GB" | 20260507 14:06:03.926 | INFO | } | 20260507 14:06:03.926 | INFO | Perf report for the last 5.13878 seconds: | 20260507 14:06:03.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:03.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:06:03.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:06:03.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:03.926 | INFO | Uploaded 36GiB to GPU in 1.60625s (22.4125GiB/s) | 20260507 14:06:03.926 | INFO | Downloaded 34GiB to GPU in 1.52625s (22.2769GiB/s) | 20260507 14:06:09.065 | INFO | Analyzing chunk frames = [83456,83968) | 20260507 14:06:09.065 | INFO | MemInfo: { | 20260507 14:06:09.065 | INFO | "gpu" : " 17 GB", | 20260507 14:06:09.065 | INFO | "huge" : "", | 20260507 14:06:09.065 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:09.065 | INFO | "malloc" : " 15 GB", | 20260507 14:06:09.065 | INFO | "pinned" : " 29 GB" | 20260507 14:06:09.065 | INFO | } | 20260507 14:06:09.065 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:06:09.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:09.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 14:06:09.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 14:06:09.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:09.065 | INFO | Uploaded 36GiB to GPU in 1.60505s (22.4292GiB/s) | 20260507 14:06:09.065 | INFO | Downloaded 34GiB to GPU in 1.52549s (22.2879GiB/s) | 20260507 14:06:14.204 | INFO | Analyzing chunk frames = [83968,84480) | 20260507 14:06:14.205 | INFO | MemInfo: { | 20260507 14:06:14.205 | INFO | "gpu" : " 17 GB", | 20260507 14:06:14.205 | INFO | "huge" : "", | 20260507 14:06:14.205 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:14.205 | INFO | "malloc" : " 15 GB", | 20260507 14:06:14.205 | INFO | "pinned" : " 29 GB" | 20260507 14:06:14.205 | INFO | } | 20260507 14:06:14.205 | INFO | Perf report for the last 5.13961 seconds: | 20260507 14:06:14.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:14.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:06:14.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:06:14.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:14.205 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4512GiB/s) | 20260507 14:06:14.205 | INFO | Downloaded 34GiB to GPU in 1.52211s (22.3374GiB/s) | 20260507 14:06:19.344 | INFO | Analyzing chunk frames = [84480,84992) | 20260507 14:06:19.344 | INFO | MemInfo: { | 20260507 14:06:19.344 | INFO | "gpu" : " 17 GB", | 20260507 14:06:19.344 | INFO | "huge" : "", | 20260507 14:06:19.344 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:19.344 | INFO | "malloc" : " 15 GB", | 20260507 14:06:19.344 | INFO | "pinned" : " 29 GB" | 20260507 14:06:19.344 | INFO | } | 20260507 14:06:19.344 | INFO | Perf report for the last 5.13902 seconds: | 20260507 14:06:19.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:19.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:06:19.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:06:19.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:19.344 | INFO | Uploaded 36GiB to GPU in 1.6039s (22.4453GiB/s) | 20260507 14:06:19.344 | INFO | Downloaded 34GiB to GPU in 1.5232s (22.3214GiB/s) | 20260507 14:06:24.483 | INFO | Analyzing chunk frames = [84992,85504) | 20260507 14:06:24.483 | INFO | MemInfo: { | 20260507 14:06:24.483 | INFO | "gpu" : " 17 GB", | 20260507 14:06:24.483 | INFO | "huge" : "", | 20260507 14:06:24.483 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:24.483 | INFO | "malloc" : " 15 GB", | 20260507 14:06:24.483 | INFO | "pinned" : " 29 GB" | 20260507 14:06:24.483 | INFO | } | 20260507 14:06:24.483 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:06:24.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:24.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 14:06:24.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:06:24.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:24.483 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.435GiB/s) | 20260507 14:06:24.483 | INFO | Downloaded 34GiB to GPU in 1.52367s (22.3145GiB/s) | 20260507 14:06:29.622 | INFO | Analyzing chunk frames = [85504,86016) | 20260507 14:06:29.622 | INFO | MemInfo: { | 20260507 14:06:29.622 | INFO | "gpu" : " 17 GB", | 20260507 14:06:29.622 | INFO | "huge" : "", | 20260507 14:06:29.622 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:29.623 | INFO | "malloc" : " 15 GB", | 20260507 14:06:29.623 | INFO | "pinned" : " 29 GB" | 20260507 14:06:29.623 | INFO | } | 20260507 14:06:29.623 | INFO | Perf report for the last 5.13958 seconds: | 20260507 14:06:29.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:29.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:06:29.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:06:29.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:29.623 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4266GiB/s) | 20260507 14:06:29.623 | INFO | Downloaded 34GiB to GPU in 1.52483s (22.2975GiB/s) | 20260507 14:06:29.720 | INFO | Starting a round of Baz IO | 20260507 14:06:32.630 | INFO | Populating RTMetrics report | 20260507 14:06:32.630 | INFO | RTMetrics report populated | 20260507 14:06:32.630 | INFO | Opening tmp RTMetrics json file | 20260507 14:06:32.636 | INFO | Serializing RTMetrics to json | 20260507 14:06:32.688 | INFO | Serialization done, swapping tmp file now... | 20260507 14:06:32.698 | INFO | RTMetrics done | 20260507 14:06:32.715 | INFO | Finished a round of Baz IO: Wrote 1.553 GiB in 2.994 seconds | 20260507 14:06:32.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.36% / 0.36% / 35.28% | 20260507 14:06:32.715 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.71828% / 0.00163317% / 3.97364e-06% | 20260507 14:06:32.715 | INFO | Baz IO overall: | 20260507 14:06:32.715 | INFO | BytesWritten : 17047898608 | 20260507 14:06:32.715 | INFO | NumEvents : 6875021880 | 20260507 14:06:34.762 | INFO | Analyzing chunk frames = [86016,86528) | 20260507 14:06:34.762 | INFO | MemInfo: { | 20260507 14:06:34.762 | INFO | "gpu" : " 17 GB", | 20260507 14:06:34.762 | INFO | "huge" : "", | 20260507 14:06:34.762 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:34.762 | INFO | "malloc" : " 15 GB", | 20260507 14:06:34.762 | INFO | "pinned" : " 29 GB" | 20260507 14:06:34.762 | INFO | } | 20260507 14:06:34.762 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:06:34.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:34.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:34.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:34.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:34.762 | INFO | ANALYSIS : 57.7%, 2.9, 88.5ms, 0 | 20260507 14:06:34.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:34.762 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 14:06:34.762 | INFO | PRE_HQ : 56.7%, 2.1, 61.5ms, 0 | 20260507 14:06:34.762 | INFO | BAZWRITER : 57.8%, 4.2, 125.2ms, 0 | 20260507 14:06:34.762 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:06:34.762 | INFO | Uploaded 36GiB to GPU in 1.65902s (21.6996GiB/s) | 20260507 14:06:34.762 | INFO | Downloaded 38GiB to GPU in 1.71276s (22.1864GiB/s) | 20260507 14:06:39.901 | INFO | Analyzing chunk frames = [86528,87040) | 20260507 14:06:39.901 | INFO | MemInfo: { | 20260507 14:06:39.901 | INFO | "gpu" : " 17 GB", | 20260507 14:06:39.901 | INFO | "huge" : "", | 20260507 14:06:39.901 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:39.901 | INFO | "malloc" : " 15 GB", | 20260507 14:06:39.901 | INFO | "pinned" : " 29 GB" | 20260507 14:06:39.901 | INFO | } | 20260507 14:06:39.901 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:06:39.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:39.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:06:39.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:06:39.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:39.901 | INFO | Uploaded 36GiB to GPU in 1.6023s (22.4677GiB/s) | 20260507 14:06:39.901 | INFO | Downloaded 34GiB to GPU in 1.5208s (22.3567GiB/s) | 20260507 14:06:45.040 | INFO | Analyzing chunk frames = [87040,87552) | 20260507 14:06:45.040 | INFO | MemInfo: { | 20260507 14:06:45.040 | INFO | "gpu" : " 17 GB", | 20260507 14:06:45.040 | INFO | "huge" : "", | 20260507 14:06:45.040 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:45.040 | INFO | "malloc" : " 15 GB", | 20260507 14:06:45.040 | INFO | "pinned" : " 29 GB" | 20260507 14:06:45.040 | INFO | } | 20260507 14:06:45.041 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:06:45.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:45.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:06:45.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:06:45.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:45.041 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.421GiB/s) | 20260507 14:06:45.041 | INFO | Downloaded 34GiB to GPU in 1.52594s (22.2813GiB/s) | 20260507 14:06:50.180 | INFO | Analyzing chunk frames = [87552,88064) | 20260507 14:06:50.180 | INFO | MemInfo: { | 20260507 14:06:50.180 | INFO | "gpu" : " 17 GB", | 20260507 14:06:50.180 | INFO | "huge" : "", | 20260507 14:06:50.180 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:50.180 | INFO | "malloc" : " 15 GB", | 20260507 14:06:50.180 | INFO | "pinned" : " 29 GB" | 20260507 14:06:50.180 | INFO | } | 20260507 14:06:50.180 | INFO | Perf report for the last 5.13958 seconds: | 20260507 14:06:50.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:50.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.180 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:06:50.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.180 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:06:50.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:50.181 | INFO | Uploaded 36GiB to GPU in 1.60569s (22.4203GiB/s) | 20260507 14:06:50.181 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.2889GiB/s) | 20260507 14:06:55.319 | INFO | Analyzing chunk frames = [88064,88576) | 20260507 14:06:55.319 | INFO | MemInfo: { | 20260507 14:06:55.319 | INFO | "gpu" : " 17 GB", | 20260507 14:06:55.319 | INFO | "huge" : "", | 20260507 14:06:55.319 | INFO | "huge_shared" : " 29 GB", | 20260507 14:06:55.319 | INFO | "malloc" : " 15 GB", | 20260507 14:06:55.319 | INFO | "pinned" : " 29 GB" | 20260507 14:06:55.319 | INFO | } | 20260507 14:06:55.319 | INFO | Perf report for the last 5.1391 seconds: | 20260507 14:06:55.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:06:55.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.319 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:06:55.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.319 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:06:55.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:06:55.320 | INFO | Uploaded 36GiB to GPU in 1.6038s (22.4467GiB/s) | 20260507 14:06:55.320 | INFO | Downloaded 34GiB to GPU in 1.5223s (22.3347GiB/s) | 20260507 14:07:00.458 | INFO | Analyzing chunk frames = [88576,89088) | 20260507 14:07:00.458 | INFO | MemInfo: { | 20260507 14:07:00.458 | INFO | "gpu" : " 17 GB", | 20260507 14:07:00.458 | INFO | "huge" : "", | 20260507 14:07:00.459 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:00.459 | INFO | "malloc" : " 15 GB", | 20260507 14:07:00.459 | INFO | "pinned" : " 29 GB" | 20260507 14:07:00.459 | INFO | } | 20260507 14:07:00.459 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:07:00.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:00.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:07:00.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:07:00.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:00.459 | INFO | Uploaded 36GiB to GPU in 1.60406s (22.443GiB/s) | 20260507 14:07:00.459 | INFO | Downloaded 34GiB to GPU in 1.52328s (22.3203GiB/s) | 20260507 14:07:05.598 | INFO | Analyzing chunk frames = [89088,89600) | 20260507 14:07:05.598 | INFO | MemInfo: { | 20260507 14:07:05.598 | INFO | "gpu" : " 17 GB", | 20260507 14:07:05.598 | INFO | "huge" : "", | 20260507 14:07:05.598 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:05.598 | INFO | "malloc" : " 15 GB", | 20260507 14:07:05.598 | INFO | "pinned" : " 29 GB" | 20260507 14:07:05.598 | INFO | } | 20260507 14:07:05.598 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:07:05.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:05.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:07:05.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:07:05.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:05.598 | INFO | Uploaded 36GiB to GPU in 1.60483s (22.4323GiB/s) | 20260507 14:07:05.598 | INFO | Downloaded 34GiB to GPU in 1.52427s (22.3058GiB/s) | 20260507 14:07:10.737 | INFO | Analyzing chunk frames = [89600,90112) | 20260507 14:07:10.737 | INFO | MemInfo: { | 20260507 14:07:10.737 | INFO | "gpu" : " 17 GB", | 20260507 14:07:10.737 | INFO | "huge" : "", | 20260507 14:07:10.737 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:10.737 | INFO | "malloc" : " 15 GB", | 20260507 14:07:10.737 | INFO | "pinned" : " 29 GB" | 20260507 14:07:10.737 | INFO | } | 20260507 14:07:10.737 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:07:10.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:10.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.737 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 14:07:10.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.737 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:07:10.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:10.738 | INFO | Uploaded 36GiB to GPU in 1.60522s (22.4269GiB/s) | 20260507 14:07:10.738 | INFO | Downloaded 34GiB to GPU in 1.52501s (22.295GiB/s) | 20260507 14:07:10.738 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:07:10.738 | INFO | Total mem usage: 15.5 GiB | 20260507 14:07:10.738 | INFO | All Time High Water Marks sum: 15.8 GiB (Savings: 2.01%) | 20260507 14:07:10.738 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -0.379%) | 20260507 14:07:10.738 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:07:10.738 | INFO | High Water Breakdown: | 20260507 14:07:10.738 | INFO | 58.8% : 9.08 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:07:10.738 | INFO | 41.1% : 6.34 GiB ( 94.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:07:10.738 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:07:10.738 | INFO | ------------------------------------------ | 20260507 14:07:10.738 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:07:10.738 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:07:10.738 | INFO | Total mem usage: 26.5 GiB | 20260507 14:07:10.738 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:07:10.738 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:07:10.738 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:07:10.738 | INFO | High Water Breakdown: | 20260507 14:07:10.738 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:07:10.738 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:07:10.738 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:07:10.738 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:07:10.738 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:07:10.738 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:07:10.738 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:07:10.738 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:07:10.738 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:07:10.738 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:07:10.738 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:07:10.738 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:07:10.738 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:07:10.738 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:07:10.738 | INFO | ------------------------------------------ | 20260507 14:07:10.738 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:07:10.738 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:07:10.738 | INFO | Total mem usage: 27.8 GiB | 20260507 14:07:10.738 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:07:10.738 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:07:10.738 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:07:10.738 | INFO | High Water Breakdown: | 20260507 14:07:10.738 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:07:10.738 | INFO | ------------------------------------------ | 20260507 14:07:10.738 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:07:10.738 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:07:10.738 | INFO | Total mem usage: 15.7 GiB | 20260507 14:07:10.738 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:07:10.738 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:07:10.738 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:07:10.738 | INFO | High Water Breakdown: | 20260507 14:07:10.738 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:07:10.738 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:07:10.738 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:07:10.738 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:07:10.738 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:07:10.738 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:07:10.738 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:07:10.738 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:07:10.738 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:07:10.738 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:07:10.738 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:07:10.738 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:07:10.738 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:07:10.738 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:07:10.739 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:07:10.739 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:07:10.739 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:07:10.739 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:07:10.739 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:07:10.739 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:07:10.739 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:07:10.739 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:07:10.739 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:07:10.739 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:07:10.739 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:07:10.739 | INFO | ------------------------------------------ | 20260507 14:07:10.739 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:07:10.739 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:07:10.739 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:07:10.739 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:07:10.739 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.34e-03 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / 0.00e+00, 5.21e-02 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 15.99%, DME -- 3.85e+01 / 0.00e+00, 3.24e+00 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.41e-02 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.25e+00 / 0.00e+00, 1.36e-01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 3.79%, Metrics -- 1.14e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.29e-01 / 0.00e+00, 6.34e-02 / 0.00e+00 | 20260507 14:07:10.739 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:07:10.739 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:07:10.739 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:07:10.739 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 18.56%, Upload -- 1.68e+01 / 0.00e+00, 4.76e-01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 17.84%, Download -- 1.61e+01 / 0.00e+00, 7.31e-01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 33.35%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 0.00%, LockDownload -- 4.36e-04 / 0.00e+00, 1.27e-04 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 29.68%, LockCompute -- 2.68e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:07:10.739 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 14:07:10.834 | INFO | Starting a round of Baz IO | 20260507 14:07:13.744 | INFO | Populating RTMetrics report | 20260507 14:07:13.744 | INFO | RTMetrics report populated | 20260507 14:07:13.744 | INFO | Opening tmp RTMetrics json file | 20260507 14:07:13.825 | INFO | Serializing RTMetrics to json | 20260507 14:07:13.851 | INFO | Serialization done, swapping tmp file now... | 20260507 14:07:13.853 | INFO | RTMetrics done | 20260507 14:07:14.034 | INFO | Finished a round of Baz IO: Wrote 1.557 GiB in 3.199 seconds | 20260507 14:07:14.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.01% / 0.35% / 35.64% | 20260507 14:07:14.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.58825% / 0.00150601% / 3.97364e-06% | 20260507 14:07:14.034 | INFO | Baz IO overall: | 20260507 14:07:14.034 | INFO | BytesWritten : 18719661327 | 20260507 14:07:14.034 | INFO | NumEvents : 7695325231 | 20260507 14:07:15.876 | INFO | Analyzing chunk frames = [90112,90624) | 20260507 14:07:15.877 | INFO | MemInfo: { | 20260507 14:07:15.877 | INFO | "gpu" : " 17 GB", | 20260507 14:07:15.877 | INFO | "huge" : "", | 20260507 14:07:15.877 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:15.877 | INFO | "malloc" : " 15 GB", | 20260507 14:07:15.877 | INFO | "pinned" : " 29 GB" | 20260507 14:07:15.877 | INFO | } | 20260507 14:07:15.877 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:07:15.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:15.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:15.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:15.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:15.877 | INFO | ANALYSIS : 57.6%, 2.9, 88.6ms, 0 | 20260507 14:07:15.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:15.877 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 14:07:15.877 | INFO | PRE_HQ : 56.7%, 2.1, 60.8ms, 0 | 20260507 14:07:15.877 | INFO | BAZWRITER : 61.8%, 4.1, 131.0ms, 0 | 20260507 14:07:15.877 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 14:07:15.877 | INFO | Uploaded 36GiB to GPU in 1.65205s (21.7911GiB/s) | 20260507 14:07:15.877 | INFO | Downloaded 38GiB to GPU in 1.71584s (22.1466GiB/s) | 20260507 14:07:21.016 | INFO | Analyzing chunk frames = [90624,91136) | 20260507 14:07:21.016 | INFO | MemInfo: { | 20260507 14:07:21.016 | INFO | "gpu" : " 17 GB", | 20260507 14:07:21.016 | INFO | "huge" : "", | 20260507 14:07:21.016 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:21.016 | INFO | "malloc" : " 15 GB", | 20260507 14:07:21.016 | INFO | "pinned" : " 29 GB" | 20260507 14:07:21.016 | INFO | } | 20260507 14:07:21.016 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:07:21.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:21.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.016 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:07:21.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.016 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:07:21.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:21.017 | INFO | Uploaded 36GiB to GPU in 1.60188s (22.4736GiB/s) | 20260507 14:07:21.017 | INFO | Downloaded 34GiB to GPU in 1.52124s (22.3502GiB/s) | 20260507 14:07:26.155 | INFO | Analyzing chunk frames = [91136,91648) | 20260507 14:07:26.155 | INFO | MemInfo: { | 20260507 14:07:26.155 | INFO | "gpu" : " 17 GB", | 20260507 14:07:26.155 | INFO | "huge" : "", | 20260507 14:07:26.155 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:26.155 | INFO | "malloc" : " 15 GB", | 20260507 14:07:26.155 | INFO | "pinned" : " 29 GB" | 20260507 14:07:26.155 | INFO | } | 20260507 14:07:26.155 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:07:26.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:26.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.155 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:07:26.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.156 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 14:07:26.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:26.156 | INFO | Uploaded 36GiB to GPU in 1.60568s (22.4205GiB/s) | 20260507 14:07:26.156 | INFO | Downloaded 34GiB to GPU in 1.5257s (22.2848GiB/s) | 20260507 14:07:31.294 | INFO | Analyzing chunk frames = [91648,92160) | 20260507 14:07:31.294 | INFO | MemInfo: { | 20260507 14:07:31.295 | INFO | "gpu" : " 17 GB", | 20260507 14:07:31.295 | INFO | "huge" : "", | 20260507 14:07:31.295 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:31.295 | INFO | "malloc" : " 15 GB", | 20260507 14:07:31.295 | INFO | "pinned" : " 29 GB" | 20260507 14:07:31.295 | INFO | } | 20260507 14:07:31.295 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:07:31.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:31.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 14:07:31.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:07:31.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:31.295 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4183GiB/s) | 20260507 14:07:31.295 | INFO | Downloaded 34GiB to GPU in 1.52601s (22.2804GiB/s) | 20260507 14:07:36.434 | INFO | Analyzing chunk frames = [92160,92672) | 20260507 14:07:36.434 | INFO | MemInfo: { | 20260507 14:07:36.434 | INFO | "gpu" : " 17 GB", | 20260507 14:07:36.434 | INFO | "huge" : "", | 20260507 14:07:36.434 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:36.434 | INFO | "malloc" : " 15 GB", | 20260507 14:07:36.434 | INFO | "pinned" : " 29 GB" | 20260507 14:07:36.434 | INFO | } | 20260507 14:07:36.434 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:07:36.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:36.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:07:36.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:07:36.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:36.434 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.4639GiB/s) | 20260507 14:07:36.434 | INFO | Downloaded 34GiB to GPU in 1.5231s (22.3229GiB/s) | 20260507 14:07:41.573 | INFO | Analyzing chunk frames = [92672,93184) | 20260507 14:07:41.573 | INFO | MemInfo: { | 20260507 14:07:41.573 | INFO | "gpu" : " 17 GB", | 20260507 14:07:41.573 | INFO | "huge" : "", | 20260507 14:07:41.573 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:41.573 | INFO | "malloc" : " 15 GB", | 20260507 14:07:41.573 | INFO | "pinned" : " 29 GB" | 20260507 14:07:41.573 | INFO | } | 20260507 14:07:41.573 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:07:41.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:41.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.573 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:07:41.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.574 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:07:41.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:41.574 | INFO | Uploaded 36GiB to GPU in 1.60411s (22.4423GiB/s) | 20260507 14:07:41.574 | INFO | Downloaded 34GiB to GPU in 1.52384s (22.3121GiB/s) | 20260507 14:07:46.713 | INFO | Analyzing chunk frames = [93184,93696) | 20260507 14:07:46.713 | INFO | MemInfo: { | 20260507 14:07:46.713 | INFO | "gpu" : " 17 GB", | 20260507 14:07:46.713 | INFO | "huge" : "", | 20260507 14:07:46.713 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:46.713 | INFO | "malloc" : " 15 GB", | 20260507 14:07:46.713 | INFO | "pinned" : " 29 GB" | 20260507 14:07:46.713 | INFO | } | 20260507 14:07:46.713 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:07:46.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:46.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:07:46.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:07:46.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:46.713 | INFO | Uploaded 36GiB to GPU in 1.60432s (22.4395GiB/s) | 20260507 14:07:46.713 | INFO | Downloaded 34GiB to GPU in 1.52429s (22.3055GiB/s) | 20260507 14:07:51.852 | INFO | Analyzing chunk frames = [93696,94208) | 20260507 14:07:51.852 | INFO | MemInfo: { | 20260507 14:07:51.852 | INFO | "gpu" : " 17 GB", | 20260507 14:07:51.852 | INFO | "huge" : "", | 20260507 14:07:51.852 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:51.852 | INFO | "malloc" : " 15 GB", | 20260507 14:07:51.852 | INFO | "pinned" : " 29 GB" | 20260507 14:07:51.852 | INFO | } | 20260507 14:07:51.852 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:07:51.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:51.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:07:51.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:07:51.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:51.852 | INFO | Uploaded 36GiB to GPU in 1.60507s (22.429GiB/s) | 20260507 14:07:51.852 | INFO | Downloaded 34GiB to GPU in 1.52543s (22.2889GiB/s) | 20260507 14:07:51.949 | INFO | Starting a round of Baz IO | 20260507 14:07:54.859 | INFO | Populating RTMetrics report | 20260507 14:07:54.859 | INFO | RTMetrics report populated | 20260507 14:07:54.859 | INFO | Opening tmp RTMetrics json file | 20260507 14:07:54.871 | INFO | Serializing RTMetrics to json | 20260507 14:07:54.931 | INFO | Serialization done, swapping tmp file now... | 20260507 14:07:54.935 | INFO | RTMetrics done | 20260507 14:07:54.960 | INFO | Finished a round of Baz IO: Wrote 1.564 GiB in 3.011 seconds | 20260507 14:07:54.960 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 63.68% / 0.33% / 35.99% | 20260507 14:07:54.960 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.51232% / 0.00143846% / 3.97364e-06% | 20260507 14:07:54.960 | INFO | Baz IO overall: | 20260507 14:07:54.960 | INFO | BytesWritten : 20399211951 | 20260507 14:07:54.960 | INFO | NumEvents : 8519270661 | 20260507 14:07:56.991 | INFO | Analyzing chunk frames = [94208,94720) | 20260507 14:07:56.991 | INFO | MemInfo: { | 20260507 14:07:56.991 | INFO | "gpu" : " 17 GB", | 20260507 14:07:56.991 | INFO | "huge" : "", | 20260507 14:07:56.991 | INFO | "huge_shared" : " 29 GB", | 20260507 14:07:56.991 | INFO | "malloc" : " 15 GB", | 20260507 14:07:56.991 | INFO | "pinned" : " 29 GB" | 20260507 14:07:56.991 | INFO | } | 20260507 14:07:56.991 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:07:56.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:07:56.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:56.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:56.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:56.992 | INFO | ANALYSIS : 57.6%, 2.9, 88.6ms, 0 | 20260507 14:07:56.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:07:56.992 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 14:07:56.992 | INFO | PRE_HQ : 56.6%, 2.1, 60.5ms, 0 | 20260507 14:07:56.992 | INFO | BAZWRITER : 58.1%, 4.3, 128.0ms, 0 | 20260507 14:07:56.992 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 14:07:56.992 | INFO | Uploaded 36GiB to GPU in 1.65211s (21.7904GiB/s) | 20260507 14:07:56.992 | INFO | Downloaded 38GiB to GPU in 1.71572s (22.1482GiB/s) | 20260507 14:08:02.131 | INFO | Analyzing chunk frames = [94720,95232) | 20260507 14:08:02.131 | INFO | MemInfo: { | 20260507 14:08:02.131 | INFO | "gpu" : " 17 GB", | 20260507 14:08:02.131 | INFO | "huge" : "", | 20260507 14:08:02.131 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:02.131 | INFO | "malloc" : " 15 GB", | 20260507 14:08:02.131 | INFO | "pinned" : " 29 GB" | 20260507 14:08:02.131 | INFO | } | 20260507 14:08:02.131 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:08:02.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:02.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:08:02.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 14:08:02.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:02.131 | INFO | Uploaded 36GiB to GPU in 1.60148s (22.4792GiB/s) | 20260507 14:08:02.131 | INFO | Downloaded 34GiB to GPU in 1.52067s (22.3586GiB/s) | 20260507 14:08:07.270 | INFO | Analyzing chunk frames = [95232,95744) | 20260507 14:08:07.270 | INFO | MemInfo: { | 20260507 14:08:07.270 | INFO | "gpu" : " 17 GB", | 20260507 14:08:07.270 | INFO | "huge" : "", | 20260507 14:08:07.270 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:07.270 | INFO | "malloc" : " 15 GB", | 20260507 14:08:07.270 | INFO | "pinned" : " 29 GB" | 20260507 14:08:07.270 | INFO | } | 20260507 14:08:07.270 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:08:07.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:07.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:08:07.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:08:07.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:07.270 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4195GiB/s) | 20260507 14:08:07.270 | INFO | Downloaded 34GiB to GPU in 1.52651s (22.273GiB/s) | 20260507 14:08:12.409 | INFO | Analyzing chunk frames = [95744,96256) | 20260507 14:08:12.409 | INFO | MemInfo: { | 20260507 14:08:12.409 | INFO | "gpu" : " 17 GB", | 20260507 14:08:12.409 | INFO | "huge" : "", | 20260507 14:08:12.409 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:12.409 | INFO | "malloc" : " 15 GB", | 20260507 14:08:12.409 | INFO | "pinned" : " 29 GB" | 20260507 14:08:12.409 | INFO | } | 20260507 14:08:12.409 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:08:12.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:12.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:08:12.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:08:12.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:12.409 | INFO | Uploaded 36GiB to GPU in 1.60451s (22.4368GiB/s) | 20260507 14:08:12.410 | INFO | Downloaded 34GiB to GPU in 1.52501s (22.2949GiB/s) | 20260507 14:08:17.549 | INFO | Analyzing chunk frames = [96256,96768) | 20260507 14:08:17.549 | INFO | MemInfo: { | 20260507 14:08:17.549 | INFO | "gpu" : " 17 GB", | 20260507 14:08:17.549 | INFO | "huge" : "", | 20260507 14:08:17.549 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:17.549 | INFO | "malloc" : " 15 GB", | 20260507 14:08:17.549 | INFO | "pinned" : " 29 GB" | 20260507 14:08:17.549 | INFO | } | 20260507 14:08:17.549 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:08:17.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:17.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:08:17.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:08:17.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:17.549 | INFO | Uploaded 36GiB to GPU in 1.60381s (22.4465GiB/s) | 20260507 14:08:17.549 | INFO | Downloaded 34GiB to GPU in 1.5231s (22.3229GiB/s) | 20260507 14:08:22.688 | INFO | Analyzing chunk frames = [96768,97280) | 20260507 14:08:22.688 | INFO | MemInfo: { | 20260507 14:08:22.688 | INFO | "gpu" : " 17 GB", | 20260507 14:08:22.688 | INFO | "huge" : "", | 20260507 14:08:22.688 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:22.688 | INFO | "malloc" : " 15 GB", | 20260507 14:08:22.688 | INFO | "pinned" : " 29 GB" | 20260507 14:08:22.688 | INFO | } | 20260507 14:08:22.688 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:08:22.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:22.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:08:22.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:08:22.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:22.688 | INFO | Uploaded 36GiB to GPU in 1.60377s (22.4471GiB/s) | 20260507 14:08:22.688 | INFO | Downloaded 34GiB to GPU in 1.52237s (22.3336GiB/s) | 20260507 14:08:27.827 | INFO | Analyzing chunk frames = [97280,97792) | 20260507 14:08:27.827 | INFO | MemInfo: { | 20260507 14:08:27.827 | INFO | "gpu" : " 17 GB", | 20260507 14:08:27.827 | INFO | "huge" : "", | 20260507 14:08:27.827 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:27.827 | INFO | "malloc" : " 15 GB", | 20260507 14:08:27.827 | INFO | "pinned" : " 29 GB" | 20260507 14:08:27.827 | INFO | } | 20260507 14:08:27.827 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:08:27.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:27.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.827 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:08:27.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.828 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:08:27.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:27.828 | INFO | Uploaded 36GiB to GPU in 1.60481s (22.4325GiB/s) | 20260507 14:08:27.828 | INFO | Downloaded 34GiB to GPU in 1.52392s (22.3109GiB/s) | 20260507 14:08:32.967 | INFO | Analyzing chunk frames = [97792,98304) | 20260507 14:08:32.967 | INFO | MemInfo: { | 20260507 14:08:32.967 | INFO | "gpu" : " 17 GB", | 20260507 14:08:32.967 | INFO | "huge" : "", | 20260507 14:08:32.967 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:32.967 | INFO | "malloc" : " 15 GB", | 20260507 14:08:32.967 | INFO | "pinned" : " 29 GB" | 20260507 14:08:32.967 | INFO | } | 20260507 14:08:32.967 | INFO | Perf report for the last 5.13954 seconds: | 20260507 14:08:32.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:32.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.967 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:08:32.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.967 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:08:32.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:32.968 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:08:32.968 | INFO | Total mem usage: 15.5 GiB | 20260507 14:08:32.968 | INFO | All Time High Water Marks sum: 15.8 GiB (Savings: 2.01%) | 20260507 14:08:32.968 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -0.461%) | 20260507 14:08:32.968 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:08:32.968 | INFO | High Water Breakdown: | 20260507 14:08:32.968 | INFO | 58.9% : 9.08 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:08:32.968 | INFO | 41% : 6.33 GiB ( 94.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:08:32.968 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:08:32.968 | INFO | ------------------------------------------ | 20260507 14:08:32.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:08:32.968 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:08:32.968 | INFO | Total mem usage: 26.5 GiB | 20260507 14:08:32.968 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:08:32.968 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:08:32.968 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:08:32.968 | INFO | High Water Breakdown: | 20260507 14:08:32.968 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:08:32.968 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:08:32.968 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:08:32.968 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:08:32.968 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:08:32.968 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:08:32.968 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:08:32.968 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:08:32.968 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:08:32.968 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:08:32.968 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:08:32.968 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:08:32.968 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:08:32.968 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:08:32.968 | INFO | ------------------------------------------ | 20260507 14:08:32.968 | INFO | Uploaded 36GiB to GPU in 1.60493s (22.4308GiB/s) | 20260507 14:08:32.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:08:32.968 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:08:32.968 | INFO | Total mem usage: 27.8 GiB | 20260507 14:08:32.968 | INFO | Downloaded 34GiB to GPU in 1.5244s (22.3039GiB/s) | 20260507 14:08:32.968 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:08:32.968 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:08:32.968 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:08:32.968 | INFO | High Water Breakdown: | 20260507 14:08:32.968 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:08:32.968 | INFO | ------------------------------------------ | 20260507 14:08:32.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:08:32.968 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:08:32.968 | INFO | Total mem usage: 15.7 GiB | 20260507 14:08:32.968 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:08:32.968 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:08:32.968 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:08:32.968 | INFO | High Water Breakdown: | 20260507 14:08:32.968 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:08:32.968 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:08:32.968 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:08:32.968 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:08:32.968 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:08:32.968 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:08:32.968 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:08:32.968 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:08:32.968 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:08:32.968 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:08:32.968 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:08:32.968 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:08:32.968 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:08:32.968 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:08:32.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:08:32.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:08:32.968 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:08:32.968 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:08:32.968 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:08:32.968 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:08:32.968 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:08:32.968 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:08:32.968 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:08:32.968 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:08:32.968 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:08:32.968 | INFO | ------------------------------------------ | 20260507 14:08:32.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:08:32.969 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:08:32.969 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:08:32.969 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:08:32.969 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / 0.00e+00, 4.49e-03 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / 0.00e+00, 5.25e-02 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 16.03%, DME -- 3.86e+01 / 0.00e+00, 3.25e+00 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.40e-02 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.24e+00 / 0.00e+00, 1.37e-01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 3.77%, Metrics -- 1.14e+00 / 0.00e+00, 1.82e-01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.26e-01 / 0.00e+00, 6.38e-02 / 0.00e+00 | 20260507 14:08:32.969 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:08:32.969 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:08:32.969 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:08:32.969 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.21e-01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 17.85%, Download -- 1.61e+01 / 0.00e+00, 7.47e-01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 33.35%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 0.00%, LockDownload -- 4.35e-04 / 0.00e+00, 1.67e-04 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 29.66%, LockCompute -- 2.68e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:08:32.969 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.09e-02 / 0.00e+00 | 20260507 14:08:33.064 | INFO | Starting a round of Baz IO | 20260507 14:08:35.971 | INFO | Populating RTMetrics report | 20260507 14:08:35.972 | INFO | RTMetrics report populated | 20260507 14:08:35.972 | INFO | Opening tmp RTMetrics json file | 20260507 14:08:36.030 | INFO | Serializing RTMetrics to json | 20260507 14:08:36.053 | INFO | Serialization done, swapping tmp file now... | 20260507 14:08:36.059 | INFO | RTMetrics done | 20260507 14:08:36.084 | INFO | Finished a round of Baz IO: Wrote 1.572 GiB in 3.019 seconds | 20260507 14:08:36.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 63.39% / 0.29% / 36.32% | 20260507 14:08:36.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.47273% / 0.00116825% / 1.19209e-05% | 20260507 14:08:36.084 | INFO | Baz IO overall: | 20260507 14:08:36.084 | INFO | BytesWritten : 22087297568 | 20260507 14:08:36.084 | INFO | NumEvents : 9347525232 | 20260507 14:08:38.106 | INFO | Analyzing chunk frames = [98304,98816) | 20260507 14:08:38.106 | INFO | MemInfo: { | 20260507 14:08:38.106 | INFO | "gpu" : " 17 GB", | 20260507 14:08:38.106 | INFO | "huge" : "", | 20260507 14:08:38.106 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:38.106 | INFO | "malloc" : " 15 GB", | 20260507 14:08:38.106 | INFO | "pinned" : " 29 GB" | 20260507 14:08:38.106 | INFO | } | 20260507 14:08:38.106 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:08:38.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:38.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:38.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:38.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:38.106 | INFO | ANALYSIS : 57.6%, 3.0, 89.5ms, 0 | 20260507 14:08:38.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:38.106 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 14:08:38.106 | INFO | PRE_HQ : 56.4%, 2.1, 61.1ms, 0 | 20260507 14:08:38.106 | INFO | BAZWRITER : 58.3%, 3.8, 113.4ms, 0 | 20260507 14:08:38.106 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 14:08:38.107 | INFO | Uploaded 36GiB to GPU in 1.63691s (21.9927GiB/s) | 20260507 14:08:38.107 | INFO | Downloaded 38GiB to GPU in 1.70246s (22.3207GiB/s) | 20260507 14:08:43.246 | INFO | Analyzing chunk frames = [98816,99328) | 20260507 14:08:43.246 | INFO | MemInfo: { | 20260507 14:08:43.246 | INFO | "gpu" : " 17 GB", | 20260507 14:08:43.246 | INFO | "huge" : "", | 20260507 14:08:43.246 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:43.246 | INFO | "malloc" : " 15 GB", | 20260507 14:08:43.246 | INFO | "pinned" : " 29 GB" | 20260507 14:08:43.246 | INFO | } | 20260507 14:08:43.246 | INFO | Perf report for the last 5.13959 seconds: | 20260507 14:08:43.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:43.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:08:43.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:08:43.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:43.246 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4704GiB/s) | 20260507 14:08:43.247 | INFO | Downloaded 34GiB to GPU in 1.52086s (22.3558GiB/s) | 20260507 14:08:48.385 | INFO | Analyzing chunk frames = [99328,99840) | 20260507 14:08:48.385 | INFO | MemInfo: { | 20260507 14:08:48.385 | INFO | "gpu" : " 17 GB", | 20260507 14:08:48.385 | INFO | "huge" : "", | 20260507 14:08:48.385 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:48.385 | INFO | "malloc" : " 15 GB", | 20260507 14:08:48.385 | INFO | "pinned" : " 29 GB" | 20260507 14:08:48.385 | INFO | } | 20260507 14:08:48.385 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:08:48.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:48.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260507 14:08:48.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 14:08:48.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:48.385 | INFO | Uploaded 36GiB to GPU in 1.60643s (22.41GiB/s) | 20260507 14:08:48.386 | INFO | Downloaded 34GiB to GPU in 1.52614s (22.2785GiB/s) | 20260507 14:08:53.524 | INFO | Analyzing chunk frames = [99840,100352) | 20260507 14:08:53.524 | INFO | MemInfo: { | 20260507 14:08:53.524 | INFO | "gpu" : " 17 GB", | 20260507 14:08:53.524 | INFO | "huge" : "", | 20260507 14:08:53.524 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:53.524 | INFO | "malloc" : " 15 GB", | 20260507 14:08:53.524 | INFO | "pinned" : " 29 GB" | 20260507 14:08:53.524 | INFO | } | 20260507 14:08:53.524 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:08:53.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:53.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.524 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:08:53.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.525 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:08:53.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:53.525 | INFO | Uploaded 36GiB to GPU in 1.60532s (22.4254GiB/s) | 20260507 14:08:53.525 | INFO | Downloaded 34GiB to GPU in 1.52505s (22.2944GiB/s) | 20260507 14:08:58.663 | INFO | Analyzing chunk frames = [100352,100864) | 20260507 14:08:58.663 | INFO | MemInfo: { | 20260507 14:08:58.663 | INFO | "gpu" : " 17 GB", | 20260507 14:08:58.663 | INFO | "huge" : "", | 20260507 14:08:58.663 | INFO | "huge_shared" : " 29 GB", | 20260507 14:08:58.663 | INFO | "malloc" : " 15 GB", | 20260507 14:08:58.663 | INFO | "pinned" : " 29 GB" | 20260507 14:08:58.663 | INFO | } | 20260507 14:08:58.664 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:08:58.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:08:58.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:08:58.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:08:58.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:08:58.664 | INFO | Uploaded 36GiB to GPU in 1.60434s (22.4392GiB/s) | 20260507 14:08:58.664 | INFO | Downloaded 34GiB to GPU in 1.52294s (22.3252GiB/s) | 20260507 14:09:03.803 | INFO | Analyzing chunk frames = [100864,101376) | 20260507 14:09:03.803 | INFO | MemInfo: { | 20260507 14:09:03.803 | INFO | "gpu" : " 17 GB", | 20260507 14:09:03.803 | INFO | "huge" : "", | 20260507 14:09:03.803 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:03.803 | INFO | "malloc" : " 15 GB", | 20260507 14:09:03.803 | INFO | "pinned" : " 29 GB" | 20260507 14:09:03.803 | INFO | } | 20260507 14:09:03.803 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:09:03.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:03.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:09:03.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 14:09:03.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:03.803 | INFO | Uploaded 36GiB to GPU in 1.60443s (22.4378GiB/s) | 20260507 14:09:03.803 | INFO | Downloaded 34GiB to GPU in 1.52391s (22.3111GiB/s) | 20260507 14:09:08.942 | INFO | Analyzing chunk frames = [101376,101888) | 20260507 14:09:08.942 | INFO | MemInfo: { | 20260507 14:09:08.942 | INFO | "gpu" : " 17 GB", | 20260507 14:09:08.942 | INFO | "huge" : "", | 20260507 14:09:08.942 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:08.942 | INFO | "malloc" : " 15 GB", | 20260507 14:09:08.942 | INFO | "pinned" : " 29 GB" | 20260507 14:09:08.942 | INFO | } | 20260507 14:09:08.942 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:09:08.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:08.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.942 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:09:08.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.943 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:09:08.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:08.943 | INFO | Uploaded 36GiB to GPU in 1.61354s (22.3112GiB/s) | 20260507 14:09:08.943 | INFO | Downloaded 34GiB to GPU in 1.53323s (22.1755GiB/s) | 20260507 14:09:14.082 | INFO | Analyzing chunk frames = [101888,102400) | 20260507 14:09:14.082 | INFO | MemInfo: { | 20260507 14:09:14.082 | INFO | "gpu" : " 17 GB", | 20260507 14:09:14.082 | INFO | "huge" : "", | 20260507 14:09:14.082 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:14.082 | INFO | "malloc" : " 15 GB", | 20260507 14:09:14.082 | INFO | "pinned" : " 29 GB" | 20260507 14:09:14.082 | INFO | } | 20260507 14:09:14.082 | INFO | Perf report for the last 5.13962 seconds: | 20260507 14:09:14.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:14.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:09:14.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:09:14.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:14.082 | INFO | Uploaded 36GiB to GPU in 1.60497s (22.4303GiB/s) | 20260507 14:09:14.082 | INFO | Downloaded 34GiB to GPU in 1.52455s (22.3017GiB/s) | 20260507 14:09:14.179 | INFO | Starting a round of Baz IO | 20260507 14:09:17.085 | INFO | Populating RTMetrics report | 20260507 14:09:17.085 | INFO | RTMetrics report populated | 20260507 14:09:17.085 | INFO | Opening tmp RTMetrics json file | 20260507 14:09:17.103 | INFO | Serializing RTMetrics to json | 20260507 14:09:17.177 | INFO | Serialization done, swapping tmp file now... | 20260507 14:09:17.182 | INFO | RTMetrics done | 20260507 14:09:17.202 | INFO | Finished a round of Baz IO: Wrote 1.576 GiB in 3.023 seconds | 20260507 14:09:17.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 63.13% / 0.26% / 36.61% | 20260507 14:09:17.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.39385% / 0.00100136% / 3.97364e-06% | 20260507 14:09:17.202 | INFO | Baz IO overall: | 20260507 14:09:17.202 | INFO | BytesWritten : 23780030347 | 20260507 14:09:17.202 | INFO | NumEvents : 10177602990 | 20260507 14:09:19.221 | INFO | Analyzing chunk frames = [102400,102912) | 20260507 14:09:19.221 | INFO | MemInfo: { | 20260507 14:09:19.221 | INFO | "gpu" : " 17 GB", | 20260507 14:09:19.221 | INFO | "huge" : "", | 20260507 14:09:19.221 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:19.221 | INFO | "malloc" : " 15 GB", | 20260507 14:09:19.221 | INFO | "pinned" : " 29 GB" | 20260507 14:09:19.221 | INFO | } | 20260507 14:09:19.221 | INFO | Perf report for the last 5.13877 seconds: | 20260507 14:09:19.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:19.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:19.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:19.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:19.221 | INFO | ANALYSIS : 57.6%, 2.9, 88.4ms, 0 | 20260507 14:09:19.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:19.221 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 14:09:19.221 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260507 14:09:19.221 | INFO | BAZWRITER : 58.3%, 4.2, 125.3ms, 0 | 20260507 14:09:19.221 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 14:09:19.221 | INFO | Uploaded 36GiB to GPU in 1.65009s (21.817GiB/s) | 20260507 14:09:19.221 | INFO | Downloaded 38GiB to GPU in 1.71363s (22.1751GiB/s) | 20260507 14:09:24.360 | INFO | Analyzing chunk frames = [102912,103424) | 20260507 14:09:24.360 | INFO | MemInfo: { | 20260507 14:09:24.360 | INFO | "gpu" : " 17 GB", | 20260507 14:09:24.360 | INFO | "huge" : "", | 20260507 14:09:24.360 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:24.360 | INFO | "malloc" : " 15 GB", | 20260507 14:09:24.360 | INFO | "pinned" : " 29 GB" | 20260507 14:09:24.360 | INFO | } | 20260507 14:09:24.360 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:09:24.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:24.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.360 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:09:24.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.360 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:09:24.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:24.361 | INFO | Uploaded 36GiB to GPU in 1.60175s (22.4755GiB/s) | 20260507 14:09:24.361 | INFO | Downloaded 34GiB to GPU in 1.52077s (22.3571GiB/s) | 20260507 14:09:29.499 | INFO | Analyzing chunk frames = [103424,103936) | 20260507 14:09:29.499 | INFO | MemInfo: { | 20260507 14:09:29.499 | INFO | "gpu" : " 17 GB", | 20260507 14:09:29.499 | INFO | "huge" : "", | 20260507 14:09:29.499 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:29.499 | INFO | "malloc" : " 15 GB", | 20260507 14:09:29.499 | INFO | "pinned" : " 29 GB" | 20260507 14:09:29.499 | INFO | } | 20260507 14:09:29.500 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:09:29.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:29.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:09:29.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:09:29.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:29.500 | INFO | Uploaded 36GiB to GPU in 1.6062s (22.4132GiB/s) | 20260507 14:09:29.500 | INFO | Downloaded 34GiB to GPU in 1.52657s (22.2722GiB/s) | 20260507 14:09:34.639 | INFO | Analyzing chunk frames = [103936,104448) | 20260507 14:09:34.639 | INFO | MemInfo: { | 20260507 14:09:34.639 | INFO | "gpu" : " 17 GB", | 20260507 14:09:34.639 | INFO | "huge" : "", | 20260507 14:09:34.639 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:34.639 | INFO | "malloc" : " 15 GB", | 20260507 14:09:34.639 | INFO | "pinned" : " 29 GB" | 20260507 14:09:34.639 | INFO | } | 20260507 14:09:34.639 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:09:34.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:34.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.639 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:09:34.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.639 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 14:09:34.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:34.640 | INFO | Uploaded 36GiB to GPU in 1.60633s (22.4113GiB/s) | 20260507 14:09:34.640 | INFO | Downloaded 34GiB to GPU in 1.52579s (22.2836GiB/s) | 20260507 14:09:39.778 | INFO | Analyzing chunk frames = [104448,104960) | 20260507 14:09:39.778 | INFO | MemInfo: { | 20260507 14:09:39.778 | INFO | "gpu" : " 17 GB", | 20260507 14:09:39.778 | INFO | "huge" : "", | 20260507 14:09:39.778 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:39.778 | INFO | "malloc" : " 15 GB", | 20260507 14:09:39.778 | INFO | "pinned" : " 29 GB" | 20260507 14:09:39.778 | INFO | } | 20260507 14:09:39.778 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:09:39.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:39.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.778 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:09:39.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.778 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:09:39.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:39.779 | INFO | Uploaded 36GiB to GPU in 1.60318s (22.4554GiB/s) | 20260507 14:09:39.779 | INFO | Downloaded 34GiB to GPU in 1.52409s (22.3085GiB/s) | 20260507 14:09:44.917 | INFO | Analyzing chunk frames = [104960,105472) | 20260507 14:09:44.917 | INFO | MemInfo: { | 20260507 14:09:44.917 | INFO | "gpu" : " 17 GB", | 20260507 14:09:44.918 | INFO | "huge" : "", | 20260507 14:09:44.918 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:44.918 | INFO | "malloc" : " 15 GB", | 20260507 14:09:44.918 | INFO | "pinned" : " 29 GB" | 20260507 14:09:44.918 | INFO | } | 20260507 14:09:44.918 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:09:44.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:44.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:09:44.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:09:44.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:44.918 | INFO | Uploaded 36GiB to GPU in 1.6044s (22.4383GiB/s) | 20260507 14:09:44.918 | INFO | Downloaded 34GiB to GPU in 1.52338s (22.3188GiB/s) | 20260507 14:09:50.057 | INFO | Analyzing chunk frames = [105472,105984) | 20260507 14:09:50.057 | INFO | MemInfo: { | 20260507 14:09:50.057 | INFO | "gpu" : " 17 GB", | 20260507 14:09:50.057 | INFO | "huge" : "", | 20260507 14:09:50.057 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:50.057 | INFO | "malloc" : " 15 GB", | 20260507 14:09:50.057 | INFO | "pinned" : " 29 GB" | 20260507 14:09:50.057 | INFO | } | 20260507 14:09:50.057 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:09:50.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:50.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.057 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:09:50.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.057 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:09:50.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:50.058 | INFO | Uploaded 36GiB to GPU in 1.60447s (22.4374GiB/s) | 20260507 14:09:50.058 | INFO | Downloaded 34GiB to GPU in 1.52389s (22.3113GiB/s) | 20260507 14:09:55.196 | INFO | Analyzing chunk frames = [105984,106496) | 20260507 14:09:55.196 | INFO | MemInfo: { | 20260507 14:09:55.196 | INFO | "gpu" : " 17 GB", | 20260507 14:09:55.196 | INFO | "huge" : "", | 20260507 14:09:55.196 | INFO | "huge_shared" : " 29 GB", | 20260507 14:09:55.196 | INFO | "malloc" : " 15 GB", | 20260507 14:09:55.196 | INFO | "pinned" : " 29 GB" | 20260507 14:09:55.196 | INFO | } | 20260507 14:09:55.196 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:09:55.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:09:55.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 14:09:55.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:09:55.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:09:55.196 | INFO | Uploaded 36GiB to GPU in 1.60581s (22.4185GiB/s) | 20260507 14:09:55.197 | INFO | Downloaded 34GiB to GPU in 1.5252s (22.2921GiB/s) | 20260507 14:09:55.197 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:09:55.197 | INFO | Total mem usage: 15.5 GiB | 20260507 14:09:55.197 | INFO | All Time High Water Marks sum: 15.8 GiB (Savings: 2.01%) | 20260507 14:09:55.197 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.96%) | 20260507 14:09:55.197 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:09:55.197 | INFO | High Water Breakdown: | 20260507 14:09:55.197 | INFO | 58.1% : 8.59 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:09:55.197 | INFO | 41.8% : 6.17 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:09:55.197 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:09:55.197 | INFO | ------------------------------------------ | 20260507 14:09:55.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:09:55.197 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:09:55.197 | INFO | Total mem usage: 26.5 GiB | 20260507 14:09:55.197 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:09:55.197 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:09:55.197 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:09:55.197 | INFO | High Water Breakdown: | 20260507 14:09:55.197 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:09:55.197 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:09:55.197 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:09:55.197 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:09:55.197 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:09:55.197 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:09:55.197 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:09:55.197 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:09:55.197 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:09:55.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:09:55.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:09:55.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:09:55.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:09:55.197 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:09:55.197 | INFO | ------------------------------------------ | 20260507 14:09:55.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:09:55.197 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:09:55.197 | INFO | Total mem usage: 27.8 GiB | 20260507 14:09:55.197 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:09:55.197 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:09:55.197 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:09:55.197 | INFO | High Water Breakdown: | 20260507 14:09:55.197 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:09:55.197 | INFO | ------------------------------------------ | 20260507 14:09:55.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:09:55.197 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:09:55.197 | INFO | Total mem usage: 15.7 GiB | 20260507 14:09:55.197 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:09:55.197 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:09:55.197 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:09:55.197 | INFO | High Water Breakdown: | 20260507 14:09:55.197 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:09:55.197 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:09:55.197 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:09:55.197 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:09:55.197 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:09:55.197 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:09:55.197 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:09:55.197 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:09:55.197 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:09:55.197 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:09:55.197 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:09:55.197 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:09:55.197 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:09:55.197 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:09:55.197 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:09:55.197 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:09:55.197 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:09:55.197 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:09:55.197 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:09:55.197 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:09:55.197 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:09:55.197 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:09:55.197 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:09:55.197 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:09:55.197 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:09:55.197 | INFO | ------------------------------------------ | 20260507 14:09:55.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:09:55.197 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:09:55.197 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:09:55.197 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:09:55.197 | INFO | PerfReport: Full Execution -- 3.01e+01 / 7.54e+01, 1.26e+01 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / 8.13e-02, 4.66e-03 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / 3.25e+00, 1.69e-02 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / 1.05e+00, 5.20e-02 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / 4.99e+01, 3.25e+00 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.72e+01 / 1.72e+01, 5.14e-02 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.24e+00 / 2.29e+00, 1.36e-01 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 3.76%, Metrics -- 1.13e+00 / 1.12e+00, 1.82e-01 / 0.00e+00 | 20260507 14:09:55.197 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.24e-01 / 4.49e-01, 6.30e-02 / 0.00e+00 | 20260507 14:09:55.197 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:09:55.197 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:09:55.197 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:09:55.197 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 18.56%, Upload -- 1.68e+01 / 0.00e+00, 4.63e-01 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 17.84%, Download -- 1.61e+01 / 0.00e+00, 7.52e-01 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 33.33%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 0.00%, LockDownload -- 4.40e-04 / 0.00e+00, 9.44e-05 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 29.70%, LockCompute -- 2.68e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:09:55.198 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 14:09:55.293 | INFO | Starting a round of Baz IO | 20260507 14:09:58.200 | INFO | Populating RTMetrics report | 20260507 14:09:58.201 | INFO | RTMetrics report populated | 20260507 14:09:58.201 | INFO | Opening tmp RTMetrics json file | 20260507 14:09:58.249 | INFO | Serializing RTMetrics to json | 20260507 14:09:58.264 | INFO | Serialization done, swapping tmp file now... | 20260507 14:09:58.265 | INFO | RTMetrics done | 20260507 14:09:58.289 | INFO | Finished a round of Baz IO: Wrote 1.583 GiB in 2.995 seconds | 20260507 14:09:58.289 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.88% / 0.25% / 36.87% | 20260507 14:09:58.289 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.34536% / 0.000886122% / 7.94729e-06% | 20260507 14:09:58.289 | INFO | Baz IO overall: | 20260507 14:09:58.289 | INFO | BytesWritten : 25479736166 | 20260507 14:09:58.289 | INFO | NumEvents : 11011301922 | 20260507 14:10:00.336 | INFO | Analyzing chunk frames = [106496,107008) | 20260507 14:10:00.336 | INFO | MemInfo: { | 20260507 14:10:00.336 | INFO | "gpu" : " 17 GB", | 20260507 14:10:00.336 | INFO | "huge" : "", | 20260507 14:10:00.336 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:00.336 | INFO | "malloc" : " 15 GB", | 20260507 14:10:00.336 | INFO | "pinned" : " 29 GB" | 20260507 14:10:00.336 | INFO | } | 20260507 14:10:00.336 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:10:00.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:00.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:00.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:00.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:00.336 | INFO | ANALYSIS : 57.6%, 2.9, 88.5ms, 0 | 20260507 14:10:00.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:00.336 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 14:10:00.336 | INFO | PRE_HQ : 56.4%, 2.1, 60.3ms, 0 | 20260507 14:10:00.336 | INFO | BAZWRITER : 57.9%, 4.3, 128.0ms, 0 | 20260507 14:10:00.336 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 14:10:00.336 | INFO | Uploaded 36GiB to GPU in 1.65312s (21.777GiB/s) | 20260507 14:10:00.336 | INFO | Downloaded 38GiB to GPU in 1.71769s (22.1227GiB/s) | 20260507 14:10:05.475 | INFO | Analyzing chunk frames = [107008,107520) | 20260507 14:10:05.475 | INFO | MemInfo: { | 20260507 14:10:05.475 | INFO | "gpu" : " 17 GB", | 20260507 14:10:05.475 | INFO | "huge" : "", | 20260507 14:10:05.475 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:05.475 | INFO | "malloc" : " 15 GB", | 20260507 14:10:05.475 | INFO | "pinned" : " 29 GB" | 20260507 14:10:05.475 | INFO | } | 20260507 14:10:05.475 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:10:05.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:05.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:10:05.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | PRE_HQ : 29.1%, 1.0, 15.6ms, 0 | 20260507 14:10:05.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:05.475 | INFO | Uploaded 36GiB to GPU in 1.6022s (22.4691GiB/s) | 20260507 14:10:05.475 | INFO | Downloaded 34GiB to GPU in 1.521s (22.3536GiB/s) | 20260507 14:10:10.614 | INFO | Analyzing chunk frames = [107520,108032) | 20260507 14:10:10.614 | INFO | MemInfo: { | 20260507 14:10:10.614 | INFO | "gpu" : " 17 GB", | 20260507 14:10:10.614 | INFO | "huge" : "", | 20260507 14:10:10.614 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:10.615 | INFO | "malloc" : " 15 GB", | 20260507 14:10:10.615 | INFO | "pinned" : " 29 GB" | 20260507 14:10:10.615 | INFO | } | 20260507 14:10:10.615 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:10:10.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:10.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260507 14:10:10.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:10:10.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:10.615 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4128GiB/s) | 20260507 14:10:10.615 | INFO | Downloaded 34GiB to GPU in 1.52618s (22.2779GiB/s) | 20260507 14:10:15.754 | INFO | Analyzing chunk frames = [108032,108544) | 20260507 14:10:15.754 | INFO | MemInfo: { | 20260507 14:10:15.754 | INFO | "gpu" : " 17 GB", | 20260507 14:10:15.754 | INFO | "huge" : "", | 20260507 14:10:15.754 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:15.754 | INFO | "malloc" : " 15 GB", | 20260507 14:10:15.754 | INFO | "pinned" : " 29 GB" | 20260507 14:10:15.754 | INFO | } | 20260507 14:10:15.754 | INFO | Perf report for the last 5.13912 seconds: | 20260507 14:10:15.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:15.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:10:15.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:10:15.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:15.754 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4178GiB/s) | 20260507 14:10:15.754 | INFO | Downloaded 34GiB to GPU in 1.52566s (22.2854GiB/s) | 20260507 14:10:20.893 | INFO | Analyzing chunk frames = [108544,109056) | 20260507 14:10:20.893 | INFO | MemInfo: { | 20260507 14:10:20.893 | INFO | "gpu" : " 17 GB", | 20260507 14:10:20.893 | INFO | "huge" : "", | 20260507 14:10:20.893 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:20.893 | INFO | "malloc" : " 15 GB", | 20260507 14:10:20.893 | INFO | "pinned" : " 29 GB" | 20260507 14:10:20.893 | INFO | } | 20260507 14:10:20.893 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:10:20.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:20.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.893 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:10:20.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.893 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:10:20.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:20.894 | INFO | Uploaded 36GiB to GPU in 1.6041s (22.4425GiB/s) | 20260507 14:10:20.894 | INFO | Downloaded 34GiB to GPU in 1.52322s (22.3212GiB/s) | 20260507 14:10:26.032 | INFO | Analyzing chunk frames = [109056,109568) | 20260507 14:10:26.032 | INFO | MemInfo: { | 20260507 14:10:26.032 | INFO | "gpu" : " 17 GB", | 20260507 14:10:26.032 | INFO | "huge" : "", | 20260507 14:10:26.032 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:26.032 | INFO | "malloc" : " 15 GB", | 20260507 14:10:26.032 | INFO | "pinned" : " 29 GB" | 20260507 14:10:26.032 | INFO | } | 20260507 14:10:26.032 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:10:26.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:26.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.032 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:10:26.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.032 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:10:26.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:26.033 | INFO | Uploaded 36GiB to GPU in 1.60462s (22.4352GiB/s) | 20260507 14:10:26.033 | INFO | Downloaded 34GiB to GPU in 1.52355s (22.3163GiB/s) | 20260507 14:10:31.172 | INFO | Analyzing chunk frames = [109568,110080) | 20260507 14:10:31.172 | INFO | MemInfo: { | 20260507 14:10:31.172 | INFO | "gpu" : " 17 GB", | 20260507 14:10:31.172 | INFO | "huge" : "", | 20260507 14:10:31.172 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:31.172 | INFO | "malloc" : " 15 GB", | 20260507 14:10:31.172 | INFO | "pinned" : " 29 GB" | 20260507 14:10:31.172 | INFO | } | 20260507 14:10:31.172 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:10:31.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:31.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:10:31.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:10:31.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:31.172 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.4251GiB/s) | 20260507 14:10:31.172 | INFO | Downloaded 34GiB to GPU in 1.5245s (22.3023GiB/s) | 20260507 14:10:36.311 | INFO | Analyzing chunk frames = [110080,110592) | 20260507 14:10:36.311 | INFO | MemInfo: { | 20260507 14:10:36.311 | INFO | "gpu" : " 17 GB", | 20260507 14:10:36.311 | INFO | "huge" : "", | 20260507 14:10:36.311 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:36.311 | INFO | "malloc" : " 15 GB", | 20260507 14:10:36.311 | INFO | "pinned" : " 29 GB" | 20260507 14:10:36.311 | INFO | } | 20260507 14:10:36.311 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:10:36.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:36.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 14:10:36.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:10:36.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:36.311 | INFO | Uploaded 36GiB to GPU in 1.60554s (22.4224GiB/s) | 20260507 14:10:36.311 | INFO | Downloaded 34GiB to GPU in 1.5246s (22.3009GiB/s) | 20260507 14:10:36.408 | INFO | Starting a round of Baz IO | 20260507 14:10:39.316 | INFO | Populating RTMetrics report | 20260507 14:10:39.316 | INFO | RTMetrics report populated | 20260507 14:10:39.316 | INFO | Opening tmp RTMetrics json file | 20260507 14:10:39.342 | INFO | Serializing RTMetrics to json | 20260507 14:10:39.390 | INFO | Serialization done, swapping tmp file now... | 20260507 14:10:39.411 | INFO | RTMetrics done | 20260507 14:10:39.443 | INFO | Finished a round of Baz IO: Wrote 1.587 GiB in 3.034 seconds | 20260507 14:10:39.443 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.65% / 0.24% / 37.12% | 20260507 14:10:39.443 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.26985% / 0.000917912% / 1.98682e-05% | 20260507 14:10:39.443 | INFO | Baz IO overall: | 20260507 14:10:39.443 | INFO | BytesWritten : 27184087953 | 20260507 14:10:39.443 | INFO | NumEvents : 11846300383 | 20260507 14:10:41.450 | INFO | Analyzing chunk frames = [110592,111104) | 20260507 14:10:41.450 | INFO | MemInfo: { | 20260507 14:10:41.450 | INFO | "gpu" : " 17 GB", | 20260507 14:10:41.450 | INFO | "huge" : "", | 20260507 14:10:41.450 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:41.450 | INFO | "malloc" : " 15 GB", | 20260507 14:10:41.450 | INFO | "pinned" : " 29 GB" | 20260507 14:10:41.450 | INFO | } | 20260507 14:10:41.450 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:10:41.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:41.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:41.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:41.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:41.451 | INFO | ANALYSIS : 57.6%, 3.0, 89.4ms, 0 | 20260507 14:10:41.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:41.451 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 14:10:41.451 | INFO | PRE_HQ : 56.4%, 2.1, 60.7ms, 0 | 20260507 14:10:41.451 | INFO | BAZWRITER : 58.6%, 4.0, 121.1ms, 0 | 20260507 14:10:41.451 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260507 14:10:41.451 | INFO | Uploaded 36GiB to GPU in 1.64386s (21.8997GiB/s) | 20260507 14:10:41.451 | INFO | Downloaded 38GiB to GPU in 1.70871s (22.239GiB/s) | 20260507 14:10:46.590 | INFO | Analyzing chunk frames = [111104,111616) | 20260507 14:10:46.590 | INFO | MemInfo: { | 20260507 14:10:46.590 | INFO | "gpu" : " 17 GB", | 20260507 14:10:46.590 | INFO | "huge" : "", | 20260507 14:10:46.590 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:46.590 | INFO | "malloc" : " 15 GB", | 20260507 14:10:46.590 | INFO | "pinned" : " 29 GB" | 20260507 14:10:46.590 | INFO | } | 20260507 14:10:46.590 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:10:46.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:46.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:10:46.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | PRE_HQ : 28.7%, 1.0, 15.1ms, 0 | 20260507 14:10:46.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:46.590 | INFO | Uploaded 36GiB to GPU in 1.60239s (22.4664GiB/s) | 20260507 14:10:46.590 | INFO | Downloaded 34GiB to GPU in 1.52205s (22.3383GiB/s) | 20260507 14:10:51.729 | INFO | Analyzing chunk frames = [111616,112128) | 20260507 14:10:51.729 | INFO | MemInfo: { | 20260507 14:10:51.729 | INFO | "gpu" : " 17 GB", | 20260507 14:10:51.729 | INFO | "huge" : "", | 20260507 14:10:51.729 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:51.729 | INFO | "malloc" : " 15 GB", | 20260507 14:10:51.729 | INFO | "pinned" : " 29 GB" | 20260507 14:10:51.729 | INFO | } | 20260507 14:10:51.729 | INFO | Perf report for the last 5.13967 seconds: | 20260507 14:10:51.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:51.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.729 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:10:51.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.729 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:10:51.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:51.730 | INFO | Uploaded 36GiB to GPU in 1.60611s (22.4144GiB/s) | 20260507 14:10:51.730 | INFO | Downloaded 34GiB to GPU in 1.5256s (22.2863GiB/s) | 20260507 14:10:56.868 | INFO | Analyzing chunk frames = [112128,112640) | 20260507 14:10:56.868 | INFO | MemInfo: { | 20260507 14:10:56.868 | INFO | "gpu" : " 17 GB", | 20260507 14:10:56.868 | INFO | "huge" : "", | 20260507 14:10:56.868 | INFO | "huge_shared" : " 29 GB", | 20260507 14:10:56.868 | INFO | "malloc" : " 15 GB", | 20260507 14:10:56.868 | INFO | "pinned" : " 29 GB" | 20260507 14:10:56.868 | INFO | } | 20260507 14:10:56.868 | INFO | Perf report for the last 5.13914 seconds: | 20260507 14:10:56.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:10:56.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:10:56.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:10:56.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:10:56.869 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4192GiB/s) | 20260507 14:10:56.869 | INFO | Downloaded 34GiB to GPU in 1.5254s (22.2892GiB/s) | 20260507 14:11:02.008 | INFO | Analyzing chunk frames = [112640,113152) | 20260507 14:11:02.008 | INFO | MemInfo: { | 20260507 14:11:02.008 | INFO | "gpu" : " 17 GB", | 20260507 14:11:02.008 | INFO | "huge" : "", | 20260507 14:11:02.008 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:02.008 | INFO | "malloc" : " 15 GB", | 20260507 14:11:02.008 | INFO | "pinned" : " 29 GB" | 20260507 14:11:02.008 | INFO | } | 20260507 14:11:02.008 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:11:02.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:02.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.008 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:11:02.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.008 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:11:02.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:02.009 | INFO | Uploaded 36GiB to GPU in 1.6034s (22.4523GiB/s) | 20260507 14:11:02.009 | INFO | Downloaded 34GiB to GPU in 1.52257s (22.3306GiB/s) | 20260507 14:11:07.147 | INFO | Analyzing chunk frames = [113152,113664) | 20260507 14:11:07.147 | INFO | MemInfo: { | 20260507 14:11:07.147 | INFO | "gpu" : " 17 GB", | 20260507 14:11:07.147 | INFO | "huge" : "", | 20260507 14:11:07.147 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:07.147 | INFO | "malloc" : " 15 GB", | 20260507 14:11:07.147 | INFO | "pinned" : " 29 GB" | 20260507 14:11:07.147 | INFO | } | 20260507 14:11:07.147 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:11:07.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:07.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.147 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:11:07.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.147 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:11:07.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:07.148 | INFO | Uploaded 36GiB to GPU in 1.604s (22.4439GiB/s) | 20260507 14:11:07.148 | INFO | Downloaded 34GiB to GPU in 1.52367s (22.3146GiB/s) | 20260507 14:11:12.286 | INFO | Analyzing chunk frames = [113664,114176) | 20260507 14:11:12.286 | INFO | MemInfo: { | 20260507 14:11:12.286 | INFO | "gpu" : " 17 GB", | 20260507 14:11:12.286 | INFO | "huge" : "", | 20260507 14:11:12.286 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:12.286 | INFO | "malloc" : " 15 GB", | 20260507 14:11:12.286 | INFO | "pinned" : " 29 GB" | 20260507 14:11:12.286 | INFO | } | 20260507 14:11:12.287 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:11:12.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:12.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:11:12.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:11:12.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:12.287 | INFO | Uploaded 36GiB to GPU in 1.60385s (22.446GiB/s) | 20260507 14:11:12.287 | INFO | Downloaded 34GiB to GPU in 1.52475s (22.2988GiB/s) | 20260507 14:11:17.426 | INFO | Analyzing chunk frames = [114176,114688) | 20260507 14:11:17.426 | INFO | MemInfo: { | 20260507 14:11:17.426 | INFO | "gpu" : " 17 GB", | 20260507 14:11:17.426 | INFO | "huge" : "", | 20260507 14:11:17.426 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:17.426 | INFO | "malloc" : " 15 GB", | 20260507 14:11:17.426 | INFO | "pinned" : " 29 GB" | 20260507 14:11:17.426 | INFO | } | 20260507 14:11:17.426 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:11:17.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:17.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:11:17.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 14:11:17.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:17.426 | INFO | Uploaded 36GiB to GPU in 1.60556s (22.4221GiB/s) | 20260507 14:11:17.426 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:11:17.426 | INFO | Total mem usage: 15.5 GiB | 20260507 14:11:17.426 | INFO | Downloaded 34GiB to GPU in 1.52465s (22.3003GiB/s) | 20260507 14:11:17.426 | INFO | All Time High Water Marks sum: 15.8 GiB (Savings: 2.01%) | 20260507 14:11:17.426 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -4.18%) | 20260507 14:11:17.426 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:11:17.426 | INFO | High Water Breakdown: | 20260507 14:11:17.426 | INFO | 58.3% : 8.67 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:11:17.426 | INFO | 41.6% : 6.19 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:11:17.427 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:11:17.427 | INFO | ------------------------------------------ | 20260507 14:11:17.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:11:17.427 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:11:17.427 | INFO | Total mem usage: 26.5 GiB | 20260507 14:11:17.427 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:11:17.427 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:11:17.427 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:11:17.427 | INFO | High Water Breakdown: | 20260507 14:11:17.427 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:11:17.427 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:11:17.427 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:11:17.427 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:11:17.427 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:11:17.427 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:11:17.427 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:11:17.427 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:11:17.427 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:11:17.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:11:17.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:11:17.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:11:17.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:11:17.427 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:11:17.427 | INFO | ------------------------------------------ | 20260507 14:11:17.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:11:17.427 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:11:17.427 | INFO | Total mem usage: 27.8 GiB | 20260507 14:11:17.427 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:11:17.427 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:11:17.427 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:11:17.427 | INFO | High Water Breakdown: | 20260507 14:11:17.427 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:11:17.427 | INFO | ------------------------------------------ | 20260507 14:11:17.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:11:17.427 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:11:17.427 | INFO | Total mem usage: 15.7 GiB | 20260507 14:11:17.427 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:11:17.427 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:11:17.427 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:11:17.427 | INFO | High Water Breakdown: | 20260507 14:11:17.427 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:11:17.427 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:11:17.427 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:11:17.427 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:11:17.427 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:11:17.427 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:11:17.427 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:11:17.427 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:11:17.427 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:11:17.427 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:11:17.427 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:11:17.427 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:11:17.427 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:11:17.427 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:11:17.427 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:11:17.427 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:11:17.427 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:11:17.427 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:11:17.427 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:11:17.427 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:11:17.427 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:11:17.427 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:11:17.427 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:11:17.427 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:11:17.427 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:11:17.427 | INFO | ------------------------------------------ | 20260507 14:11:17.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:11:17.427 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:11:17.427 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:11:17.427 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:11:17.427 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.84e-03 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.92e-02 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.11e-02 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.18e+00 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 3.74%, Metrics -- 1.13e+00 / -nan, 1.81e-01 / 0.00e+00 | 20260507 14:11:17.427 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.22e-01 / -nan, 6.19e-02 / 0.00e+00 | 20260507 14:11:17.427 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:11:17.427 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:11:17.427 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:11:17.427 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.11e-01 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 17.85%, Download -- 1.61e+01 / 0.00e+00, 7.35e-01 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 33.33%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 0.00%, LockDownload -- 4.41e-04 / 0.00e+00, 9.54e-05 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 29.68%, LockCompute -- 2.68e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:11:17.428 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260507 14:11:17.523 | INFO | Starting a round of Baz IO | 20260507 14:11:20.430 | INFO | Populating RTMetrics report | 20260507 14:11:20.431 | INFO | RTMetrics report populated | 20260507 14:11:20.431 | INFO | Opening tmp RTMetrics json file | 20260507 14:11:20.432 | INFO | Serializing RTMetrics to json | 20260507 14:11:20.433 | INFO | Serialization done, swapping tmp file now... | 20260507 14:11:20.434 | INFO | RTMetrics done | 20260507 14:11:20.506 | INFO | Finished a round of Baz IO: Wrote 1.591 GiB in 2.983 seconds | 20260507 14:11:20.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.42% / 0.23% / 37.35% | 20260507 14:11:20.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.19508% / 0.000929832% / 7.94729e-06% | 20260507 14:11:20.506 | INFO | Baz IO overall: | 20260507 14:11:20.506 | INFO | BytesWritten : 28892630887 | 20260507 14:11:20.506 | INFO | NumEvents : 12682974699 | 20260507 14:11:22.565 | INFO | Analyzing chunk frames = [114688,115200) | 20260507 14:11:22.565 | INFO | MemInfo: { | 20260507 14:11:22.565 | INFO | "gpu" : " 17 GB", | 20260507 14:11:22.565 | INFO | "huge" : "", | 20260507 14:11:22.565 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:22.565 | INFO | "malloc" : " 15 GB", | 20260507 14:11:22.565 | INFO | "pinned" : " 29 GB" | 20260507 14:11:22.565 | INFO | } | 20260507 14:11:22.565 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:11:22.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:22.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:22.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:22.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:22.565 | INFO | ANALYSIS : 57.6%, 3.0, 89.4ms, 0 | 20260507 14:11:22.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:22.565 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 14:11:22.565 | INFO | PRE_HQ : 56.4%, 2.1, 60.9ms, 0 | 20260507 14:11:22.565 | INFO | BAZWRITER : 57.6%, 4.2, 123.1ms, 0 | 20260507 14:11:22.565 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:11:22.566 | INFO | Uploaded 36GiB to GPU in 1.64319s (21.9087GiB/s) | 20260507 14:11:22.566 | INFO | Downloaded 38GiB to GPU in 1.70697s (22.2617GiB/s) | 20260507 14:11:27.704 | INFO | Analyzing chunk frames = [115200,115712) | 20260507 14:11:27.704 | INFO | MemInfo: { | 20260507 14:11:27.704 | INFO | "gpu" : " 17 GB", | 20260507 14:11:27.704 | INFO | "huge" : "", | 20260507 14:11:27.704 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:27.704 | INFO | "malloc" : " 15 GB", | 20260507 14:11:27.704 | INFO | "pinned" : " 29 GB" | 20260507 14:11:27.705 | INFO | } | 20260507 14:11:27.705 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:11:27.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:27.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:11:27.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:11:27.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:27.705 | INFO | Uploaded 36GiB to GPU in 1.60207s (22.4709GiB/s) | 20260507 14:11:27.705 | INFO | Downloaded 34GiB to GPU in 1.52092s (22.3549GiB/s) | 20260507 14:11:32.844 | INFO | Analyzing chunk frames = [115712,116224) | 20260507 14:11:32.844 | INFO | MemInfo: { | 20260507 14:11:32.844 | INFO | "gpu" : " 17 GB", | 20260507 14:11:32.844 | INFO | "huge" : "", | 20260507 14:11:32.844 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:32.844 | INFO | "malloc" : " 15 GB", | 20260507 14:11:32.844 | INFO | "pinned" : " 29 GB" | 20260507 14:11:32.844 | INFO | } | 20260507 14:11:32.844 | INFO | Perf report for the last 5.13967 seconds: | 20260507 14:11:32.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:32.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:11:32.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:11:32.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:32.844 | INFO | Uploaded 36GiB to GPU in 1.60723s (22.3987GiB/s) | 20260507 14:11:32.845 | INFO | Downloaded 34GiB to GPU in 1.52695s (22.2667GiB/s) | 20260507 14:11:37.983 | INFO | Analyzing chunk frames = [116224,116736) | 20260507 14:11:37.983 | INFO | MemInfo: { | 20260507 14:11:37.983 | INFO | "gpu" : " 17 GB", | 20260507 14:11:37.983 | INFO | "huge" : "", | 20260507 14:11:37.983 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:37.983 | INFO | "malloc" : " 15 GB", | 20260507 14:11:37.983 | INFO | "pinned" : " 29 GB" | 20260507 14:11:37.983 | INFO | } | 20260507 14:11:37.983 | INFO | Perf report for the last 5.13891 seconds: | 20260507 14:11:37.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:37.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:11:37.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 14:11:37.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:37.983 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4192GiB/s) | 20260507 14:11:37.983 | INFO | Downloaded 34GiB to GPU in 1.52525s (22.2914GiB/s) | 20260507 14:11:43.122 | INFO | Analyzing chunk frames = [116736,117248) | 20260507 14:11:43.122 | INFO | MemInfo: { | 20260507 14:11:43.122 | INFO | "gpu" : " 17 GB", | 20260507 14:11:43.122 | INFO | "huge" : "", | 20260507 14:11:43.122 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:43.122 | INFO | "malloc" : " 15 GB", | 20260507 14:11:43.122 | INFO | "pinned" : " 29 GB" | 20260507 14:11:43.122 | INFO | } | 20260507 14:11:43.123 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:11:43.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:43.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 14:11:43.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 14:11:43.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:43.123 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4347GiB/s) | 20260507 14:11:43.123 | INFO | Downloaded 34GiB to GPU in 1.52353s (22.3165GiB/s) | 20260507 14:11:48.262 | INFO | Analyzing chunk frames = [117248,117760) | 20260507 14:11:48.262 | INFO | MemInfo: { | 20260507 14:11:48.262 | INFO | "gpu" : " 17 GB", | 20260507 14:11:48.262 | INFO | "huge" : "", | 20260507 14:11:48.262 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:48.262 | INFO | "malloc" : " 15 GB", | 20260507 14:11:48.262 | INFO | "pinned" : " 29 GB" | 20260507 14:11:48.262 | INFO | } | 20260507 14:11:48.262 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:11:48.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:48.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:11:48.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 14:11:48.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:48.262 | INFO | Uploaded 36GiB to GPU in 1.60638s (22.4107GiB/s) | 20260507 14:11:48.262 | INFO | Downloaded 34GiB to GPU in 1.5255s (22.2877GiB/s) | 20260507 14:11:53.401 | INFO | Analyzing chunk frames = [117760,118272) | 20260507 14:11:53.401 | INFO | MemInfo: { | 20260507 14:11:53.401 | INFO | "gpu" : " 17 GB", | 20260507 14:11:53.401 | INFO | "huge" : "", | 20260507 14:11:53.401 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:53.401 | INFO | "malloc" : " 15 GB", | 20260507 14:11:53.401 | INFO | "pinned" : " 29 GB" | 20260507 14:11:53.401 | INFO | } | 20260507 14:11:53.401 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:11:53.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:53.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:11:53.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:11:53.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:53.401 | INFO | Uploaded 36GiB to GPU in 1.60533s (22.4254GiB/s) | 20260507 14:11:53.402 | INFO | Downloaded 34GiB to GPU in 1.52489s (22.2967GiB/s) | 20260507 14:11:58.540 | INFO | Analyzing chunk frames = [118272,118784) | 20260507 14:11:58.540 | INFO | MemInfo: { | 20260507 14:11:58.540 | INFO | "gpu" : " 17 GB", | 20260507 14:11:58.541 | INFO | "huge" : "", | 20260507 14:11:58.541 | INFO | "huge_shared" : " 29 GB", | 20260507 14:11:58.541 | INFO | "malloc" : " 15 GB", | 20260507 14:11:58.541 | INFO | "pinned" : " 29 GB" | 20260507 14:11:58.541 | INFO | } | 20260507 14:11:58.541 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:11:58.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:11:58.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:11:58.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:11:58.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:11:58.541 | INFO | Uploaded 36GiB to GPU in 1.60451s (22.4368GiB/s) | 20260507 14:11:58.541 | INFO | Downloaded 34GiB to GPU in 1.52391s (22.3111GiB/s) | 20260507 14:11:58.638 | INFO | Starting a round of Baz IO | 20260507 14:12:01.544 | INFO | Populating RTMetrics report | 20260507 14:12:01.544 | INFO | RTMetrics report populated | 20260507 14:12:01.544 | INFO | Opening tmp RTMetrics json file | 20260507 14:12:01.550 | INFO | Serializing RTMetrics to json | 20260507 14:12:01.590 | INFO | Serialization done, swapping tmp file now... | 20260507 14:12:01.609 | INFO | RTMetrics done | 20260507 14:12:01.635 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 2.997 seconds | 20260507 14:12:01.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.21% / 0.21% / 37.58% | 20260507 14:12:01.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.1472% / 0.000743071% / 7.94729e-06% | 20260507 14:12:01.636 | INFO | Baz IO overall: | 20260507 14:12:01.636 | INFO | BytesWritten : 30606040958 | 20260507 14:12:01.636 | INFO | NumEvents : 13522622978 | 20260507 14:12:03.680 | INFO | Analyzing chunk frames = [118784,119296) | 20260507 14:12:03.680 | INFO | MemInfo: { | 20260507 14:12:03.680 | INFO | "gpu" : " 17 GB", | 20260507 14:12:03.680 | INFO | "huge" : "", | 20260507 14:12:03.680 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:03.680 | INFO | "malloc" : " 15 GB", | 20260507 14:12:03.680 | INFO | "pinned" : " 29 GB" | 20260507 14:12:03.680 | INFO | } | 20260507 14:12:03.680 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:12:03.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:03.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:03.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:03.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:03.680 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260507 14:12:03.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:03.680 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 14:12:03.680 | INFO | PRE_HQ : 56.4%, 2.1, 60.8ms, 0 | 20260507 14:12:03.680 | INFO | BAZWRITER : 57.9%, 3.7, 110.6ms, 0 | 20260507 14:12:03.680 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:12:03.681 | INFO | Uploaded 36GiB to GPU in 1.64039s (21.946GiB/s) | 20260507 14:12:03.681 | INFO | Downloaded 38GiB to GPU in 1.70466s (22.2918GiB/s) | 20260507 14:12:08.819 | INFO | Analyzing chunk frames = [119296,119808) | 20260507 14:12:08.819 | INFO | MemInfo: { | 20260507 14:12:08.819 | INFO | "gpu" : " 17 GB", | 20260507 14:12:08.819 | INFO | "huge" : "", | 20260507 14:12:08.819 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:08.819 | INFO | "malloc" : " 15 GB", | 20260507 14:12:08.819 | INFO | "pinned" : " 29 GB" | 20260507 14:12:08.819 | INFO | } | 20260507 14:12:08.819 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:12:08.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:08.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.819 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:12:08.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.819 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:12:08.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:08.820 | INFO | Uploaded 36GiB to GPU in 1.602s (22.4719GiB/s) | 20260507 14:12:08.820 | INFO | Downloaded 34GiB to GPU in 1.52082s (22.3564GiB/s) | 20260507 14:12:13.959 | INFO | Analyzing chunk frames = [119808,120320) | 20260507 14:12:13.959 | INFO | MemInfo: { | 20260507 14:12:13.959 | INFO | "gpu" : " 17 GB", | 20260507 14:12:13.959 | INFO | "huge" : "", | 20260507 14:12:13.959 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:13.959 | INFO | "malloc" : " 15 GB", | 20260507 14:12:13.959 | INFO | "pinned" : " 29 GB" | 20260507 14:12:13.959 | INFO | } | 20260507 14:12:13.959 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:12:13.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:13.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:12:13.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:12:13.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:13.959 | INFO | Uploaded 36GiB to GPU in 1.60654s (22.4085GiB/s) | 20260507 14:12:13.959 | INFO | Downloaded 34GiB to GPU in 1.52713s (22.264GiB/s) | 20260507 14:12:19.098 | INFO | Analyzing chunk frames = [120320,120832) | 20260507 14:12:19.098 | INFO | MemInfo: { | 20260507 14:12:19.098 | INFO | "gpu" : " 17 GB", | 20260507 14:12:19.098 | INFO | "huge" : "", | 20260507 14:12:19.098 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:19.098 | INFO | "malloc" : " 15 GB", | 20260507 14:12:19.098 | INFO | "pinned" : " 29 GB" | 20260507 14:12:19.098 | INFO | } | 20260507 14:12:19.098 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:12:19.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:19.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:12:19.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:12:19.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:19.098 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4196GiB/s) | 20260507 14:12:19.099 | INFO | Downloaded 34GiB to GPU in 1.52566s (22.2855GiB/s) | 20260507 14:12:24.237 | INFO | Analyzing chunk frames = [120832,121344) | 20260507 14:12:24.237 | INFO | MemInfo: { | 20260507 14:12:24.237 | INFO | "gpu" : " 17 GB", | 20260507 14:12:24.237 | INFO | "huge" : "", | 20260507 14:12:24.237 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:24.237 | INFO | "malloc" : " 15 GB", | 20260507 14:12:24.237 | INFO | "pinned" : " 29 GB" | 20260507 14:12:24.237 | INFO | } | 20260507 14:12:24.237 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:12:24.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:24.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.237 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:12:24.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.237 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:12:24.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:24.238 | INFO | Uploaded 36GiB to GPU in 1.60505s (22.4292GiB/s) | 20260507 14:12:24.238 | INFO | Downloaded 34GiB to GPU in 1.52415s (22.3075GiB/s) | 20260507 14:12:29.377 | INFO | Analyzing chunk frames = [121344,121856) | 20260507 14:12:29.377 | INFO | MemInfo: { | 20260507 14:12:29.377 | INFO | "gpu" : " 17 GB", | 20260507 14:12:29.377 | INFO | "huge" : "", | 20260507 14:12:29.377 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:29.377 | INFO | "malloc" : " 15 GB", | 20260507 14:12:29.377 | INFO | "pinned" : " 29 GB" | 20260507 14:12:29.377 | INFO | } | 20260507 14:12:29.377 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:12:29.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:29.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:12:29.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:12:29.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:29.377 | INFO | Uploaded 36GiB to GPU in 1.6046s (22.4355GiB/s) | 20260507 14:12:29.377 | INFO | Downloaded 34GiB to GPU in 1.52328s (22.3202GiB/s) | 20260507 14:12:34.516 | INFO | Analyzing chunk frames = [121856,122368) | 20260507 14:12:34.516 | INFO | MemInfo: { | 20260507 14:12:34.516 | INFO | "gpu" : " 17 GB", | 20260507 14:12:34.516 | INFO | "huge" : "", | 20260507 14:12:34.516 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:34.516 | INFO | "malloc" : " 15 GB", | 20260507 14:12:34.516 | INFO | "pinned" : " 29 GB" | 20260507 14:12:34.516 | INFO | } | 20260507 14:12:34.516 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:12:34.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:34.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:12:34.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:12:34.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:34.516 | INFO | Uploaded 36GiB to GPU in 1.60521s (22.4269GiB/s) | 20260507 14:12:34.516 | INFO | Downloaded 34GiB to GPU in 1.52553s (22.2874GiB/s) | 20260507 14:12:39.655 | INFO | Analyzing chunk frames = [122368,122880) | 20260507 14:12:39.655 | INFO | MemInfo: { | 20260507 14:12:39.655 | INFO | "gpu" : " 17 GB", | 20260507 14:12:39.655 | INFO | "huge" : "", | 20260507 14:12:39.655 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:39.655 | INFO | "malloc" : " 15 GB", | 20260507 14:12:39.655 | INFO | "pinned" : " 29 GB" | 20260507 14:12:39.655 | INFO | } | 20260507 14:12:39.655 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:12:39.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:39.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:12:39.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:12:39.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:39.655 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4257GiB/s) | 20260507 14:12:39.655 | INFO | Downloaded 34GiB to GPU in 1.52473s (22.299GiB/s) | 20260507 14:12:39.656 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:12:39.656 | INFO | Total mem usage: 15.5 GiB | 20260507 14:12:39.656 | INFO | All Time High Water Marks sum: 15.8 GiB (Savings: 2.01%) | 20260507 14:12:39.656 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.97%) | 20260507 14:12:39.656 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:12:39.656 | INFO | High Water Breakdown: | 20260507 14:12:39.656 | INFO | 58.2% : 8.6 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:12:39.656 | INFO | 41.7% : 6.16 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:12:39.656 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:12:39.656 | INFO | ------------------------------------------ | 20260507 14:12:39.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:12:39.656 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:12:39.656 | INFO | Total mem usage: 26.5 GiB | 20260507 14:12:39.656 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:12:39.656 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:12:39.656 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:12:39.656 | INFO | High Water Breakdown: | 20260507 14:12:39.656 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:12:39.656 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:12:39.656 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:12:39.656 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:12:39.656 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:12:39.656 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:12:39.656 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:12:39.656 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:12:39.656 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:12:39.656 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:12:39.656 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:12:39.656 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:12:39.656 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:12:39.656 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:12:39.656 | INFO | ------------------------------------------ | 20260507 14:12:39.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:12:39.656 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:12:39.656 | INFO | Total mem usage: 27.8 GiB | 20260507 14:12:39.656 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:12:39.656 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:12:39.656 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:12:39.656 | INFO | High Water Breakdown: | 20260507 14:12:39.656 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:12:39.656 | INFO | ------------------------------------------ | 20260507 14:12:39.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:12:39.656 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:12:39.656 | INFO | Total mem usage: 15.7 GiB | 20260507 14:12:39.656 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:12:39.656 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:12:39.656 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:12:39.656 | INFO | High Water Breakdown: | 20260507 14:12:39.656 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:12:39.656 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:12:39.656 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:12:39.656 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:12:39.656 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:12:39.656 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:12:39.656 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:12:39.656 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:12:39.656 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:12:39.656 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:12:39.656 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:12:39.656 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:12:39.656 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:12:39.656 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:12:39.656 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:12:39.656 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:12:39.656 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:12:39.656 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:12:39.656 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:12:39.656 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:12:39.656 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:12:39.656 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:12:39.656 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:12:39.656 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:12:39.656 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:12:39.656 | INFO | ------------------------------------------ | 20260507 14:12:39.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:12:39.656 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:12:39.656 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:12:39.656 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:12:39.656 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 5.59e-03 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.72e+01 / -nan, 4.39e-02 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.23e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 3.73%, Metrics -- 1.12e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.20e-01 / -nan, 6.20e-02 / 0.00e+00 | 20260507 14:12:39.656 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:12:39.656 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:12:39.656 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:12:39.656 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 18.55%, Upload -- 1.68e+01 / 0.00e+00, 3.91e-01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 17.83%, Download -- 1.61e+01 / 0.00e+00, 7.03e-01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 33.30%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:12:39.656 | INFO | PerfReport: 0.00%, LockDownload -- 4.44e-04 / 0.00e+00, 1.89e-04 / 0.00e+00 | 20260507 14:12:39.657 | INFO | PerfReport: 29.75%, LockCompute -- 2.69e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:12:39.657 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260507 14:12:39.752 | INFO | Starting a round of Baz IO | 20260507 14:12:42.657 | INFO | Populating RTMetrics report | 20260507 14:12:42.657 | INFO | RTMetrics report populated | 20260507 14:12:42.657 | INFO | Opening tmp RTMetrics json file | 20260507 14:12:42.659 | INFO | Serializing RTMetrics to json | 20260507 14:12:42.661 | INFO | Serialization done, swapping tmp file now... | 20260507 14:12:42.661 | INFO | RTMetrics done | 20260507 14:12:42.952 | INFO | Finished a round of Baz IO: Wrote 1.601 GiB in 3.2 seconds | 20260507 14:12:42.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.01% / 0.20% / 37.79% | 20260507 14:12:42.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.10976% / 0.000679493% / 2.38419e-05% | 20260507 14:12:42.952 | INFO | Baz IO overall: | 20260507 14:12:42.952 | INFO | BytesWritten : 32324788015 | 20260507 14:12:42.952 | INFO | NumEvents : 14365112120 | 20260507 14:12:44.794 | INFO | Analyzing chunk frames = [122880,123392) | 20260507 14:12:44.794 | INFO | MemInfo: { | 20260507 14:12:44.794 | INFO | "gpu" : " 17 GB", | 20260507 14:12:44.794 | INFO | "huge" : "", | 20260507 14:12:44.795 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:44.795 | INFO | "malloc" : " 15 GB", | 20260507 14:12:44.795 | INFO | "pinned" : " 29 GB" | 20260507 14:12:44.795 | INFO | } | 20260507 14:12:44.795 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:12:44.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:44.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:44.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:44.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:44.795 | INFO | ANALYSIS : 57.5%, 2.9, 88.4ms, 0 | 20260507 14:12:44.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:44.795 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 14:12:44.795 | INFO | PRE_HQ : 56.3%, 2.1, 60.1ms, 0 | 20260507 14:12:44.795 | INFO | BAZWRITER : 61.8%, 4.1, 131.3ms, 0 | 20260507 14:12:44.795 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 14:12:44.795 | INFO | Uploaded 36GiB to GPU in 1.65216s (21.7897GiB/s) | 20260507 14:12:44.795 | INFO | Downloaded 38GiB to GPU in 1.71648s (22.1383GiB/s) | 20260507 14:12:49.934 | INFO | Analyzing chunk frames = [123392,123904) | 20260507 14:12:49.934 | INFO | MemInfo: { | 20260507 14:12:49.934 | INFO | "gpu" : " 17 GB", | 20260507 14:12:49.934 | INFO | "huge" : "", | 20260507 14:12:49.934 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:49.934 | INFO | "malloc" : " 15 GB", | 20260507 14:12:49.934 | INFO | "pinned" : " 29 GB" | 20260507 14:12:49.934 | INFO | } | 20260507 14:12:49.934 | INFO | Perf report for the last 5.13961 seconds: | 20260507 14:12:49.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:49.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:12:49.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | PRE_HQ : 28.8%, 1.0, 15.1ms, 0 | 20260507 14:12:49.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:49.934 | INFO | Uploaded 36GiB to GPU in 1.6019s (22.4733GiB/s) | 20260507 14:12:49.935 | INFO | Downloaded 34GiB to GPU in 1.52173s (22.343GiB/s) | 20260507 14:12:55.073 | INFO | Analyzing chunk frames = [123904,124416) | 20260507 14:12:55.073 | INFO | MemInfo: { | 20260507 14:12:55.073 | INFO | "gpu" : " 17 GB", | 20260507 14:12:55.073 | INFO | "huge" : "", | 20260507 14:12:55.073 | INFO | "huge_shared" : " 29 GB", | 20260507 14:12:55.073 | INFO | "malloc" : " 15 GB", | 20260507 14:12:55.073 | INFO | "pinned" : " 29 GB" | 20260507 14:12:55.073 | INFO | } | 20260507 14:12:55.073 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:12:55.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:12:55.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:12:55.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:12:55.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:12:55.074 | INFO | Uploaded 36GiB to GPU in 1.60639s (22.4105GiB/s) | 20260507 14:12:55.074 | INFO | Downloaded 34GiB to GPU in 1.52678s (22.2691GiB/s) | 20260507 14:13:00.213 | INFO | Analyzing chunk frames = [124416,124928) | 20260507 14:13:00.213 | INFO | MemInfo: { | 20260507 14:13:00.213 | INFO | "gpu" : " 17 GB", | 20260507 14:13:00.213 | INFO | "huge" : "", | 20260507 14:13:00.213 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:00.213 | INFO | "malloc" : " 15 GB", | 20260507 14:13:00.213 | INFO | "pinned" : " 29 GB" | 20260507 14:13:00.213 | INFO | } | 20260507 14:13:00.213 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:13:00.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:00.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:13:00.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:13:00.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:00.213 | INFO | Uploaded 36GiB to GPU in 1.60553s (22.4224GiB/s) | 20260507 14:13:00.213 | INFO | Downloaded 34GiB to GPU in 1.52518s (22.2925GiB/s) | 20260507 14:13:05.352 | INFO | Analyzing chunk frames = [124928,125440) | 20260507 14:13:05.352 | INFO | MemInfo: { | 20260507 14:13:05.352 | INFO | "gpu" : " 17 GB", | 20260507 14:13:05.352 | INFO | "huge" : "", | 20260507 14:13:05.352 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:05.352 | INFO | "malloc" : " 15 GB", | 20260507 14:13:05.352 | INFO | "pinned" : " 29 GB" | 20260507 14:13:05.352 | INFO | } | 20260507 14:13:05.352 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:13:05.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:05.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:13:05.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:13:05.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:05.352 | INFO | Uploaded 36GiB to GPU in 1.60469s (22.4342GiB/s) | 20260507 14:13:05.352 | INFO | Downloaded 34GiB to GPU in 1.52429s (22.3055GiB/s) | 20260507 14:13:10.491 | INFO | Analyzing chunk frames = [125440,125952) | 20260507 14:13:10.491 | INFO | MemInfo: { | 20260507 14:13:10.491 | INFO | "gpu" : " 17 GB", | 20260507 14:13:10.491 | INFO | "huge" : "", | 20260507 14:13:10.491 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:10.491 | INFO | "malloc" : " 15 GB", | 20260507 14:13:10.491 | INFO | "pinned" : " 29 GB" | 20260507 14:13:10.491 | INFO | } | 20260507 14:13:10.491 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:13:10.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:10.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.491 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:13:10.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.491 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:13:10.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:10.492 | INFO | Uploaded 36GiB to GPU in 1.60416s (22.4416GiB/s) | 20260507 14:13:10.492 | INFO | Downloaded 34GiB to GPU in 1.5233s (22.32GiB/s) | 20260507 14:13:15.631 | INFO | Analyzing chunk frames = [125952,126464) | 20260507 14:13:15.631 | INFO | MemInfo: { | 20260507 14:13:15.631 | INFO | "gpu" : " 17 GB", | 20260507 14:13:15.631 | INFO | "huge" : "", | 20260507 14:13:15.631 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:15.631 | INFO | "malloc" : " 15 GB", | 20260507 14:13:15.631 | INFO | "pinned" : " 29 GB" | 20260507 14:13:15.631 | INFO | } | 20260507 14:13:15.631 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:13:15.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:15.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:13:15.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:13:15.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:15.631 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4347GiB/s) | 20260507 14:13:15.631 | INFO | Downloaded 34GiB to GPU in 1.52502s (22.2948GiB/s) | 20260507 14:13:20.770 | INFO | Analyzing chunk frames = [126464,126976) | 20260507 14:13:20.770 | INFO | MemInfo: { | 20260507 14:13:20.770 | INFO | "gpu" : " 17 GB", | 20260507 14:13:20.770 | INFO | "huge" : "", | 20260507 14:13:20.770 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:20.770 | INFO | "malloc" : " 15 GB", | 20260507 14:13:20.770 | INFO | "pinned" : " 29 GB" | 20260507 14:13:20.770 | INFO | } | 20260507 14:13:20.770 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:13:20.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:20.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:13:20.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:13:20.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:20.770 | INFO | Uploaded 36GiB to GPU in 1.60548s (22.4232GiB/s) | 20260507 14:13:20.771 | INFO | Downloaded 34GiB to GPU in 1.52535s (22.29GiB/s) | 20260507 14:13:20.867 | INFO | Starting a round of Baz IO | 20260507 14:13:23.773 | INFO | Populating RTMetrics report | 20260507 14:13:23.773 | INFO | RTMetrics report populated | 20260507 14:13:23.773 | INFO | Opening tmp RTMetrics json file | 20260507 14:13:23.812 | INFO | Serializing RTMetrics to json | 20260507 14:13:23.816 | INFO | Serialization done, swapping tmp file now... | 20260507 14:13:23.822 | INFO | RTMetrics done | 20260507 14:13:23.929 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.061 seconds | 20260507 14:13:23.929 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.82% / 0.19% / 37.99% | 20260507 14:13:23.929 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.06297% / 0.000747045% / 1.19209e-05% | 20260507 14:13:23.929 | INFO | Baz IO overall: | 20260507 14:13:23.929 | INFO | BytesWritten : 34047814315 | 20260507 14:13:23.929 | INFO | NumEvents : 15209348935 | 20260507 14:13:25.909 | INFO | Analyzing chunk frames = [126976,127488) | 20260507 14:13:25.909 | INFO | MemInfo: { | 20260507 14:13:25.909 | INFO | "gpu" : " 17 GB", | 20260507 14:13:25.909 | INFO | "huge" : "", | 20260507 14:13:25.909 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:25.909 | INFO | "malloc" : " 15 GB", | 20260507 14:13:25.909 | INFO | "pinned" : " 29 GB" | 20260507 14:13:25.909 | INFO | } | 20260507 14:13:25.909 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:13:25.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:25.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:25.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:25.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:25.910 | INFO | ANALYSIS : 57.5%, 2.9, 88.8ms, 0 | 20260507 14:13:25.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:25.910 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260507 14:13:25.910 | INFO | PRE_HQ : 56.3%, 2.1, 60.1ms, 0 | 20260507 14:13:25.910 | INFO | BAZWRITER : 59.1%, 4.2, 126.3ms, 0 | 20260507 14:13:25.910 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260507 14:13:25.910 | INFO | Uploaded 36GiB to GPU in 1.65131s (21.8009GiB/s) | 20260507 14:13:25.910 | INFO | Downloaded 38GiB to GPU in 1.71539s (22.1524GiB/s) | 20260507 14:13:31.049 | INFO | Analyzing chunk frames = [127488,128000) | 20260507 14:13:31.049 | INFO | MemInfo: { | 20260507 14:13:31.049 | INFO | "gpu" : " 17 GB", | 20260507 14:13:31.049 | INFO | "huge" : "", | 20260507 14:13:31.049 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:31.049 | INFO | "malloc" : " 15 GB", | 20260507 14:13:31.049 | INFO | "pinned" : " 29 GB" | 20260507 14:13:31.049 | INFO | } | 20260507 14:13:31.049 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:13:31.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:31.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:13:31.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | PRE_HQ : 28.9%, 1.0, 15.2ms, 0 | 20260507 14:13:31.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:31.049 | INFO | Uploaded 36GiB to GPU in 1.60284s (22.4602GiB/s) | 20260507 14:13:31.049 | INFO | Downloaded 34GiB to GPU in 1.52192s (22.3402GiB/s) | 20260507 14:13:36.188 | INFO | Analyzing chunk frames = [128000,128512) | 20260507 14:13:36.188 | INFO | MemInfo: { | 20260507 14:13:36.188 | INFO | "gpu" : " 17 GB", | 20260507 14:13:36.188 | INFO | "huge" : "", | 20260507 14:13:36.188 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:36.188 | INFO | "malloc" : " 15 GB", | 20260507 14:13:36.188 | INFO | "pinned" : " 29 GB" | 20260507 14:13:36.188 | INFO | } | 20260507 14:13:36.188 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:13:36.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:36.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260507 14:13:36.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:13:36.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:36.188 | INFO | Uploaded 36GiB to GPU in 1.60608s (22.4148GiB/s) | 20260507 14:13:36.189 | INFO | Downloaded 34GiB to GPU in 1.52664s (22.2712GiB/s) | 20260507 14:13:41.327 | INFO | Analyzing chunk frames = [128512,129024) | 20260507 14:13:41.327 | INFO | MemInfo: { | 20260507 14:13:41.327 | INFO | "gpu" : " 17 GB", | 20260507 14:13:41.327 | INFO | "huge" : "", | 20260507 14:13:41.327 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:41.327 | INFO | "malloc" : " 15 GB", | 20260507 14:13:41.327 | INFO | "pinned" : " 29 GB" | 20260507 14:13:41.327 | INFO | } | 20260507 14:13:41.327 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:13:41.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:41.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.327 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:13:41.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.328 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:13:41.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:41.328 | INFO | Uploaded 36GiB to GPU in 1.60617s (22.4136GiB/s) | 20260507 14:13:41.328 | INFO | Downloaded 34GiB to GPU in 1.52595s (22.2811GiB/s) | 20260507 14:13:46.467 | INFO | Analyzing chunk frames = [129024,129536) | 20260507 14:13:46.467 | INFO | MemInfo: { | 20260507 14:13:46.467 | INFO | "gpu" : " 17 GB", | 20260507 14:13:46.467 | INFO | "huge" : "", | 20260507 14:13:46.467 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:46.467 | INFO | "malloc" : " 15 GB", | 20260507 14:13:46.467 | INFO | "pinned" : " 29 GB" | 20260507 14:13:46.467 | INFO | } | 20260507 14:13:46.467 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:13:46.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:46.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.467 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:13:46.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.467 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:13:46.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:46.468 | INFO | Uploaded 36GiB to GPU in 1.60582s (22.4185GiB/s) | 20260507 14:13:46.468 | INFO | Downloaded 34GiB to GPU in 1.52476s (22.2986GiB/s) | 20260507 14:13:51.606 | INFO | Analyzing chunk frames = [129536,130048) | 20260507 14:13:51.606 | INFO | MemInfo: { | 20260507 14:13:51.606 | INFO | "gpu" : " 17 GB", | 20260507 14:13:51.606 | INFO | "huge" : "", | 20260507 14:13:51.606 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:51.606 | INFO | "malloc" : " 15 GB", | 20260507 14:13:51.606 | INFO | "pinned" : " 29 GB" | 20260507 14:13:51.606 | INFO | } | 20260507 14:13:51.606 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:13:51.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:51.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.606 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:13:51.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.606 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:13:51.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:51.607 | INFO | Uploaded 36GiB to GPU in 1.60523s (22.4266GiB/s) | 20260507 14:13:51.607 | INFO | Downloaded 34GiB to GPU in 1.52421s (22.3066GiB/s) | 20260507 14:13:56.745 | INFO | Analyzing chunk frames = [130048,130560) | 20260507 14:13:56.745 | INFO | MemInfo: { | 20260507 14:13:56.745 | INFO | "gpu" : " 17 GB", | 20260507 14:13:56.745 | INFO | "huge" : "", | 20260507 14:13:56.745 | INFO | "huge_shared" : " 29 GB", | 20260507 14:13:56.745 | INFO | "malloc" : " 15 GB", | 20260507 14:13:56.745 | INFO | "pinned" : " 29 GB" | 20260507 14:13:56.745 | INFO | } | 20260507 14:13:56.745 | INFO | Perf report for the last 5.13899 seconds: | 20260507 14:13:56.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:13:56.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.745 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:13:56.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.746 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:13:56.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:13:56.746 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4346GiB/s) | 20260507 14:13:56.746 | INFO | Downloaded 34GiB to GPU in 1.5241s (22.3082GiB/s) | 20260507 14:14:01.885 | INFO | Analyzing chunk frames = [130560,131072) | 20260507 14:14:01.885 | INFO | MemInfo: { | 20260507 14:14:01.885 | INFO | "gpu" : " 17 GB", | 20260507 14:14:01.885 | INFO | "huge" : "", | 20260507 14:14:01.885 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:01.885 | INFO | "malloc" : " 15 GB", | 20260507 14:14:01.885 | INFO | "pinned" : " 29 GB" | 20260507 14:14:01.885 | INFO | } | 20260507 14:14:01.885 | INFO | Perf report for the last 5.14008 seconds: | 20260507 14:14:01.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:01.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.885 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:14:01.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.886 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:14:01.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:01.886 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.421GiB/s) | 20260507 14:14:01.886 | INFO | Downloaded 34GiB to GPU in 1.5258s (22.2834GiB/s) | 20260507 14:14:01.886 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:14:01.886 | INFO | Total mem usage: 15.5 GiB | 20260507 14:14:01.886 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:14:01.886 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: 0%) | 20260507 14:14:01.886 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:14:01.886 | INFO | High Water Breakdown: | 20260507 14:14:01.886 | INFO | 59.1% : 9.17 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:14:01.886 | INFO | 40.8% : 6.33 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:14:01.886 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:14:01.886 | INFO | ------------------------------------------ | 20260507 14:14:01.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:14:01.886 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:14:01.886 | INFO | Total mem usage: 26.5 GiB | 20260507 14:14:01.886 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:14:01.886 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:14:01.886 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:14:01.886 | INFO | High Water Breakdown: | 20260507 14:14:01.886 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:14:01.886 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:14:01.886 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:14:01.886 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:14:01.886 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:14:01.886 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:14:01.886 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:14:01.886 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:14:01.886 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:14:01.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:14:01.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:14:01.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:14:01.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:14:01.886 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:14:01.886 | INFO | ------------------------------------------ | 20260507 14:14:01.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:14:01.886 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:14:01.886 | INFO | Total mem usage: 27.8 GiB | 20260507 14:14:01.886 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:14:01.886 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:14:01.886 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:14:01.886 | INFO | High Water Breakdown: | 20260507 14:14:01.886 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:14:01.886 | INFO | ------------------------------------------ | 20260507 14:14:01.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:14:01.886 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:14:01.886 | INFO | Total mem usage: 15.7 GiB | 20260507 14:14:01.886 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:14:01.886 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:14:01.886 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:14:01.886 | INFO | High Water Breakdown: | 20260507 14:14:01.886 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:14:01.886 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:14:01.886 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:14:01.886 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:14:01.886 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:14:01.886 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:14:01.886 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:14:01.886 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:14:01.886 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:14:01.886 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:14:01.886 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:14:01.886 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:14:01.886 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:14:01.886 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:14:01.886 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:14:01.886 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:14:01.886 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:14:01.886 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:14:01.886 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:14:01.886 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:14:01.886 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:14:01.886 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:14:01.886 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:14:01.886 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:14:01.886 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:14:01.886 | INFO | ------------------------------------------ | 20260507 14:14:01.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:14:01.886 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:14:01.886 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:14:01.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:14:01.886 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 5.28e-03 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.12e-02 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.15e+00 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / -nan, 4.42e-02 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.23e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 3.72%, Metrics -- 1.12e+00 / -nan, 1.79e-01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.19e-01 / -nan, 6.18e-02 / 0.00e+00 | 20260507 14:14:01.886 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:14:01.886 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:14:01.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:14:01.886 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.20e-01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 17.87%, Download -- 1.61e+01 / 0.00e+00, 7.44e-01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 33.34%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 0.00%, LockDownload -- 4.40e-04 / 0.00e+00, 1.24e-04 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 29.62%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:14:01.886 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.09e-02 / 0.00e+00 | 20260507 14:14:01.982 | INFO | Starting a round of Baz IO | 20260507 14:14:04.889 | INFO | Populating RTMetrics report | 20260507 14:14:04.889 | INFO | RTMetrics report populated | 20260507 14:14:04.889 | INFO | Opening tmp RTMetrics json file | 20260507 14:14:05.053 | INFO | Serializing RTMetrics to json | 20260507 14:14:05.109 | INFO | Serialization done, swapping tmp file now... | 20260507 14:14:05.111 | INFO | RTMetrics done | 20260507 14:14:05.126 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.143 seconds | 20260507 14:14:05.126 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.64% / 0.18% / 38.18% | 20260507 14:14:05.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.01089% / 0.000639757% / 1.58946e-05% | 20260507 14:14:05.126 | INFO | Baz IO overall: | 20260507 14:14:05.126 | INFO | BytesWritten : 35775142478 | 20260507 14:14:05.126 | INFO | NumEvents : 16055098577 | 20260507 14:14:07.024 | INFO | Analyzing chunk frames = [131072,131584) | 20260507 14:14:07.024 | INFO | MemInfo: { | 20260507 14:14:07.024 | INFO | "gpu" : " 17 GB", | 20260507 14:14:07.024 | INFO | "huge" : "", | 20260507 14:14:07.024 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:07.024 | INFO | "malloc" : " 14 GB", | 20260507 14:14:07.024 | INFO | "pinned" : " 29 GB" | 20260507 14:14:07.024 | INFO | } | 20260507 14:14:07.024 | INFO | Perf report for the last 5.13882 seconds: | 20260507 14:14:07.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:07.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:07.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:07.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:07.024 | INFO | ANALYSIS : 57.6%, 2.9, 88.8ms, 0 | 20260507 14:14:07.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:07.024 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 14:14:07.024 | INFO | PRE_HQ : 56.6%, 2.1, 60.3ms, 0 | 20260507 14:14:07.024 | INFO | BAZWRITER : 60.7%, 4.1, 127.4ms, 0 | 20260507 14:14:07.024 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260507 14:14:07.025 | INFO | Uploaded 36GiB to GPU in 1.64765s (21.8493GiB/s) | 20260507 14:14:07.025 | INFO | Downloaded 38GiB to GPU in 1.71102s (22.209GiB/s) | 20260507 14:14:12.163 | INFO | Analyzing chunk frames = [131584,132096) | 20260507 14:14:12.163 | INFO | MemInfo: { | 20260507 14:14:12.163 | INFO | "gpu" : " 17 GB", | 20260507 14:14:12.163 | INFO | "huge" : "", | 20260507 14:14:12.164 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:12.164 | INFO | "malloc" : " 14 GB", | 20260507 14:14:12.164 | INFO | "pinned" : " 29 GB" | 20260507 14:14:12.164 | INFO | } | 20260507 14:14:12.164 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:14:12.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:12.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:14:12.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:14:12.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:12.164 | INFO | Uploaded 36GiB to GPU in 1.60268s (22.4624GiB/s) | 20260507 14:14:12.164 | INFO | Downloaded 34GiB to GPU in 1.52103s (22.3533GiB/s) | 20260507 14:14:17.306 | INFO | Analyzing chunk frames = [132096,132608) | 20260507 14:14:17.306 | INFO | MemInfo: { | 20260507 14:14:17.306 | INFO | "gpu" : " 17 GB", | 20260507 14:14:17.306 | INFO | "huge" : "", | 20260507 14:14:17.306 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:17.306 | INFO | "malloc" : " 14 GB", | 20260507 14:14:17.306 | INFO | "pinned" : " 29 GB" | 20260507 14:14:17.306 | INFO | } | 20260507 14:14:17.306 | INFO | Perf report for the last 5.14285 seconds: | 20260507 14:14:17.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:17.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.307 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260507 14:14:17.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.307 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:14:17.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:17.307 | INFO | Uploaded 36GiB to GPU in 1.60644s (22.4099GiB/s) | 20260507 14:14:17.307 | INFO | Downloaded 34GiB to GPU in 1.5259s (22.282GiB/s) | 20260507 14:14:22.442 | INFO | Analyzing chunk frames = [132608,133120) | 20260507 14:14:22.442 | INFO | MemInfo: { | 20260507 14:14:22.442 | INFO | "gpu" : " 17 GB", | 20260507 14:14:22.442 | INFO | "huge" : "", | 20260507 14:14:22.446 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:22.446 | INFO | "malloc" : " 14 GB", | 20260507 14:14:22.446 | INFO | "pinned" : " 29 GB" | 20260507 14:14:22.446 | INFO | } | 20260507 14:14:22.446 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:14:22.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:22.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:14:22.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:14:22.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:22.446 | INFO | Uploaded 36GiB to GPU in 1.60589s (22.4175GiB/s) | 20260507 14:14:22.446 | INFO | Downloaded 34GiB to GPU in 1.5258s (22.2834GiB/s) | 20260507 14:14:27.581 | INFO | Analyzing chunk frames = [133120,133632) | 20260507 14:14:27.581 | INFO | MemInfo: { | 20260507 14:14:27.581 | INFO | "gpu" : " 17 GB", | 20260507 14:14:27.581 | INFO | "huge" : "", | 20260507 14:14:27.581 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:27.582 | INFO | "malloc" : " 14 GB", | 20260507 14:14:27.582 | INFO | "pinned" : " 29 GB" | 20260507 14:14:27.582 | INFO | } | 20260507 14:14:27.582 | INFO | Perf report for the last 5.13566 seconds: | 20260507 14:14:27.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:27.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:14:27.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:14:27.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:27.582 | INFO | Uploaded 36GiB to GPU in 1.60451s (22.4368GiB/s) | 20260507 14:14:27.582 | INFO | Downloaded 34GiB to GPU in 1.52377s (22.3131GiB/s) | 20260507 14:14:32.721 | INFO | Analyzing chunk frames = [133632,134144) | 20260507 14:14:32.721 | INFO | MemInfo: { | 20260507 14:14:32.721 | INFO | "gpu" : " 17 GB", | 20260507 14:14:32.721 | INFO | "huge" : "", | 20260507 14:14:32.721 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:32.721 | INFO | "malloc" : " 14 GB", | 20260507 14:14:32.721 | INFO | "pinned" : " 29 GB" | 20260507 14:14:32.721 | INFO | } | 20260507 14:14:32.721 | INFO | Perf report for the last 5.13969 seconds: | 20260507 14:14:32.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:32.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.721 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:14:32.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.721 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:14:32.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:32.722 | INFO | Uploaded 36GiB to GPU in 1.60517s (22.4275GiB/s) | 20260507 14:14:32.722 | INFO | Downloaded 34GiB to GPU in 1.5247s (22.2995GiB/s) | 20260507 14:14:37.864 | INFO | Analyzing chunk frames = [134144,134656) | 20260507 14:14:37.864 | INFO | MemInfo: { | 20260507 14:14:37.864 | INFO | "gpu" : " 17 GB", | 20260507 14:14:37.864 | INFO | "huge" : "", | 20260507 14:14:37.864 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:37.864 | INFO | "malloc" : " 15 GB", | 20260507 14:14:37.864 | INFO | "pinned" : " 29 GB" | 20260507 14:14:37.864 | INFO | } | 20260507 14:14:37.864 | INFO | Perf report for the last 5.14254 seconds: | 20260507 14:14:37.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:37.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260507 14:14:37.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:14:37.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:37.864 | INFO | Uploaded 36GiB to GPU in 1.60506s (22.4291GiB/s) | 20260507 14:14:37.864 | INFO | Downloaded 34GiB to GPU in 1.52475s (22.2987GiB/s) | 20260507 14:14:43.000 | INFO | Analyzing chunk frames = [134656,135168) | 20260507 14:14:43.000 | INFO | MemInfo: { | 20260507 14:14:43.000 | INFO | "gpu" : " 17 GB", | 20260507 14:14:43.000 | INFO | "huge" : "", | 20260507 14:14:43.000 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:43.000 | INFO | "malloc" : " 15 GB", | 20260507 14:14:43.000 | INFO | "pinned" : " 29 GB" | 20260507 14:14:43.000 | INFO | } | 20260507 14:14:43.000 | INFO | Perf report for the last 5.13608 seconds: | 20260507 14:14:43.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:43.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:14:43.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:14:43.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:43.000 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4184GiB/s) | 20260507 14:14:43.000 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2906GiB/s) | 20260507 14:14:43.097 | INFO | Starting a round of Baz IO | 20260507 14:14:46.001 | INFO | Populating RTMetrics report | 20260507 14:14:46.001 | INFO | RTMetrics report populated | 20260507 14:14:46.001 | INFO | Opening tmp RTMetrics json file | 20260507 14:14:46.023 | INFO | Serializing RTMetrics to json | 20260507 14:14:46.097 | INFO | Serialization done, swapping tmp file now... | 20260507 14:14:46.104 | INFO | RTMetrics done | 20260507 14:14:46.117 | INFO | Finished a round of Baz IO: Wrote 1.615 GiB in 3.019 seconds | 20260507 14:14:46.117 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.47% / 0.17% / 38.36% | 20260507 14:14:46.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.99144% / 0.00068744% / 1.98682e-05% | 20260507 14:14:46.117 | INFO | Baz IO overall: | 20260507 14:14:46.117 | INFO | BytesWritten : 37509330089 | 20260507 14:14:46.117 | INFO | NumEvents : 16904713029 | 20260507 14:14:48.139 | INFO | Analyzing chunk frames = [135168,135680) | 20260507 14:14:48.139 | INFO | MemInfo: { | 20260507 14:14:48.139 | INFO | "gpu" : " 17 GB", | 20260507 14:14:48.139 | INFO | "huge" : "", | 20260507 14:14:48.139 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:48.139 | INFO | "malloc" : " 14 GB", | 20260507 14:14:48.139 | INFO | "pinned" : " 29 GB" | 20260507 14:14:48.139 | INFO | } | 20260507 14:14:48.139 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:14:48.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:48.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:48.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:48.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:48.139 | INFO | ANALYSIS : 57.5%, 2.9, 87.7ms, 0 | 20260507 14:14:48.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:48.139 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 14:14:48.139 | INFO | PRE_HQ : 56.5%, 2.1, 60.6ms, 0 | 20260507 14:14:48.139 | INFO | BAZWRITER : 58.3%, 4.4, 133.3ms, 0 | 20260507 14:14:48.139 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 14:14:48.139 | INFO | Uploaded 36GiB to GPU in 1.65721s (21.7232GiB/s) | 20260507 14:14:48.140 | INFO | Downloaded 38GiB to GPU in 1.72205s (22.0668GiB/s) | 20260507 14:14:53.278 | INFO | Analyzing chunk frames = [135680,136192) | 20260507 14:14:53.279 | INFO | MemInfo: { | 20260507 14:14:53.279 | INFO | "gpu" : " 17 GB", | 20260507 14:14:53.279 | INFO | "huge" : "", | 20260507 14:14:53.282 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:53.282 | INFO | "malloc" : " 14 GB", | 20260507 14:14:53.282 | INFO | "pinned" : " 29 GB" | 20260507 14:14:53.282 | INFO | } | 20260507 14:14:53.282 | INFO | Perf report for the last 5.14243 seconds: | 20260507 14:14:53.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:53.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:14:53.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:14:53.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:53.282 | INFO | Uploaded 36GiB to GPU in 1.60234s (22.4671GiB/s) | 20260507 14:14:53.282 | INFO | Downloaded 34GiB to GPU in 1.52152s (22.3461GiB/s) | 20260507 14:14:58.418 | INFO | Analyzing chunk frames = [136192,136704) | 20260507 14:14:58.418 | INFO | MemInfo: { | 20260507 14:14:58.418 | INFO | "gpu" : " 17 GB", | 20260507 14:14:58.418 | INFO | "huge" : "", | 20260507 14:14:58.418 | INFO | "huge_shared" : " 29 GB", | 20260507 14:14:58.418 | INFO | "malloc" : " 14 GB", | 20260507 14:14:58.418 | INFO | "pinned" : " 29 GB" | 20260507 14:14:58.418 | INFO | } | 20260507 14:14:58.419 | INFO | Perf report for the last 5.13688 seconds: | 20260507 14:14:58.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:14:58.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:14:58.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:14:58.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:14:58.419 | INFO | Uploaded 36GiB to GPU in 1.60666s (22.4068GiB/s) | 20260507 14:14:58.419 | INFO | Downloaded 34GiB to GPU in 1.52653s (22.2728GiB/s) | 20260507 14:15:03.557 | INFO | Analyzing chunk frames = [136704,137216) | 20260507 14:15:03.557 | INFO | MemInfo: { | 20260507 14:15:03.557 | INFO | "gpu" : " 17 GB", | 20260507 14:15:03.557 | INFO | "huge" : "", | 20260507 14:15:03.557 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:03.557 | INFO | "malloc" : " 14 GB", | 20260507 14:15:03.557 | INFO | "pinned" : " 29 GB" | 20260507 14:15:03.557 | INFO | } | 20260507 14:15:03.557 | INFO | Perf report for the last 5.13869 seconds: | 20260507 14:15:03.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:03.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:15:03.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:15:03.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:03.557 | INFO | Uploaded 36GiB to GPU in 1.60688s (22.4036GiB/s) | 20260507 14:15:03.558 | INFO | Downloaded 34GiB to GPU in 1.52609s (22.2791GiB/s) | 20260507 14:15:08.697 | INFO | Analyzing chunk frames = [137216,137728) | 20260507 14:15:08.697 | INFO | MemInfo: { | 20260507 14:15:08.697 | INFO | "gpu" : " 17 GB", | 20260507 14:15:08.697 | INFO | "huge" : "", | 20260507 14:15:08.697 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:08.697 | INFO | "malloc" : " 14 GB", | 20260507 14:15:08.697 | INFO | "pinned" : " 29 GB" | 20260507 14:15:08.697 | INFO | } | 20260507 14:15:08.697 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:15:08.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:08.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:15:08.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:15:08.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:08.697 | INFO | Uploaded 36GiB to GPU in 1.6047s (22.4341GiB/s) | 20260507 14:15:08.697 | INFO | Downloaded 34GiB to GPU in 1.52318s (22.3217GiB/s) | 20260507 14:15:13.836 | INFO | Analyzing chunk frames = [137728,138240) | 20260507 14:15:13.836 | INFO | MemInfo: { | 20260507 14:15:13.836 | INFO | "gpu" : " 17 GB", | 20260507 14:15:13.836 | INFO | "huge" : "", | 20260507 14:15:13.836 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:13.836 | INFO | "malloc" : " 14 GB", | 20260507 14:15:13.836 | INFO | "pinned" : " 29 GB" | 20260507 14:15:13.836 | INFO | } | 20260507 14:15:13.836 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:15:13.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:13.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:15:13.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:15:13.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:13.836 | INFO | Uploaded 36GiB to GPU in 1.60448s (22.4371GiB/s) | 20260507 14:15:13.836 | INFO | Downloaded 34GiB to GPU in 1.52359s (22.3157GiB/s) | 20260507 14:15:18.975 | INFO | Analyzing chunk frames = [138240,138752) | 20260507 14:15:18.975 | INFO | MemInfo: { | 20260507 14:15:18.975 | INFO | "gpu" : " 17 GB", | 20260507 14:15:18.975 | INFO | "huge" : "", | 20260507 14:15:18.975 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:18.975 | INFO | "malloc" : " 14 GB", | 20260507 14:15:18.975 | INFO | "pinned" : " 29 GB" | 20260507 14:15:18.975 | INFO | } | 20260507 14:15:18.975 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:15:18.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:18.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.975 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:15:18.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.975 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:15:18.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:18.976 | INFO | Uploaded 36GiB to GPU in 1.60569s (22.4203GiB/s) | 20260507 14:15:18.976 | INFO | Downloaded 34GiB to GPU in 1.52518s (22.2924GiB/s) | 20260507 14:15:24.114 | INFO | Analyzing chunk frames = [138752,139264) | 20260507 14:15:24.114 | INFO | MemInfo: { | 20260507 14:15:24.114 | INFO | "gpu" : " 17 GB", | 20260507 14:15:24.114 | INFO | "huge" : "", | 20260507 14:15:24.115 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:24.115 | INFO | "malloc" : " 15 GB", | 20260507 14:15:24.115 | INFO | "pinned" : " 29 GB" | 20260507 14:15:24.115 | INFO | } | 20260507 14:15:24.115 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:15:24.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:24.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:15:24.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:15:24.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:24.115 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:15:24.115 | INFO | Total mem usage: 15.5 GiB | 20260507 14:15:24.115 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:15:24.115 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -0.715%) | 20260507 14:15:24.115 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:15:24.115 | INFO | High Water Breakdown: | 20260507 14:15:24.115 | INFO | 59% : 9.09 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:15:24.115 | INFO | 40.8% : 6.29 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:15:24.115 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:15:24.115 | INFO | ------------------------------------------ | 20260507 14:15:24.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:15:24.115 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:15:24.115 | INFO | Total mem usage: 26.5 GiB | 20260507 14:15:24.115 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:15:24.115 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:15:24.115 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:15:24.115 | INFO | High Water Breakdown: | 20260507 14:15:24.115 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:15:24.115 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:15:24.115 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:15:24.115 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:15:24.115 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:15:24.115 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:15:24.115 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:15:24.115 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:15:24.115 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:15:24.115 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:15:24.115 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:15:24.115 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:15:24.115 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:15:24.115 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:15:24.115 | INFO | ------------------------------------------ | 20260507 14:15:24.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:15:24.115 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:15:24.115 | INFO | Total mem usage: 27.8 GiB | 20260507 14:15:24.115 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:15:24.115 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:15:24.115 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:15:24.115 | INFO | High Water Breakdown: | 20260507 14:15:24.115 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:15:24.115 | INFO | ------------------------------------------ | 20260507 14:15:24.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:15:24.115 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:15:24.115 | INFO | Total mem usage: 15.7 GiB | 20260507 14:15:24.115 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:15:24.115 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:15:24.115 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:15:24.115 | INFO | High Water Breakdown: | 20260507 14:15:24.115 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:15:24.115 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:15:24.115 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:15:24.115 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:15:24.115 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:15:24.115 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:15:24.115 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:15:24.115 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:15:24.115 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:15:24.115 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:15:24.115 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:15:24.115 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:15:24.115 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:15:24.115 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:15:24.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:15:24.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:15:24.115 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:15:24.115 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:15:24.115 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:15:24.115 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:15:24.115 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:15:24.115 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:15:24.115 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:15:24.115 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:15:24.115 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:15:24.115 | INFO | ------------------------------------------ | 20260507 14:15:24.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:15:24.115 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:15:24.115 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:15:24.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:15:24.115 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.11e-03 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.28e-02 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 16.07%, DME -- 3.87e+01 / -nan, 3.14e+00 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / -nan, 4.48e-02 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.22e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 3.72%, Metrics -- 1.12e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.17e-01 / -nan, 6.18e-02 / 0.00e+00 | 20260507 14:15:24.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:15:24.115 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:15:24.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:15:24.115 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 18.61%, Upload -- 1.68e+01 / 0.00e+00, 4.29e-01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 17.88%, Download -- 1.61e+01 / 0.00e+00, 7.57e-01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 33.36%, Compute -- 3.01e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 0.00%, LockDownload -- 4.37e-04 / 0.00e+00, 1.40e-04 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 29.57%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:15:24.115 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260507 14:15:24.116 | INFO | Uploaded 36GiB to GPU in 1.60502s (22.4296GiB/s) | 20260507 14:15:24.117 | INFO | Downloaded 34GiB to GPU in 1.52443s (22.3035GiB/s) | 20260507 14:15:24.213 | INFO | Starting a round of Baz IO | 20260507 14:15:27.113 | INFO | Populating RTMetrics report | 20260507 14:15:27.114 | INFO | RTMetrics report populated | 20260507 14:15:27.114 | INFO | Opening tmp RTMetrics json file | 20260507 14:15:27.189 | INFO | Serializing RTMetrics to json | 20260507 14:15:27.198 | INFO | Serialization done, swapping tmp file now... | 20260507 14:15:27.199 | INFO | RTMetrics done | 20260507 14:15:27.314 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.1 seconds | 20260507 14:15:27.314 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.30% / 0.17% / 38.53% | 20260507 14:15:27.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.92077% / 0.000615915% / 1.19209e-05% | 20260507 14:15:27.314 | INFO | Baz IO overall: | 20260507 14:15:27.314 | INFO | BytesWritten : 39245286735 | 20260507 14:15:27.314 | INFO | NumEvents : 17754689426 | 20260507 14:15:29.254 | INFO | Analyzing chunk frames = [139264,139776) | 20260507 14:15:29.254 | INFO | MemInfo: { | 20260507 14:15:29.254 | INFO | "gpu" : " 17 GB", | 20260507 14:15:29.254 | INFO | "huge" : "", | 20260507 14:15:29.254 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:29.254 | INFO | "malloc" : " 14 GB", | 20260507 14:15:29.254 | INFO | "pinned" : " 29 GB" | 20260507 14:15:29.254 | INFO | } | 20260507 14:15:29.254 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:15:29.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:29.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:29.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:29.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:29.254 | INFO | ANALYSIS : 57.5%, 2.9, 88.4ms, 0 | 20260507 14:15:29.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:29.254 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 14:15:29.254 | INFO | PRE_HQ : 56.2%, 2.1, 60.3ms, 0 | 20260507 14:15:29.254 | INFO | BAZWRITER : 59.9%, 4.3, 131.9ms, 0 | 20260507 14:15:29.254 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260507 14:15:29.255 | INFO | Uploaded 36GiB to GPU in 1.65382s (21.7678GiB/s) | 20260507 14:15:29.255 | INFO | Downloaded 38GiB to GPU in 1.71914s (22.1041GiB/s) | 20260507 14:15:34.394 | INFO | Analyzing chunk frames = [139776,140288) | 20260507 14:15:34.394 | INFO | MemInfo: { | 20260507 14:15:34.394 | INFO | "gpu" : " 17 GB", | 20260507 14:15:34.394 | INFO | "huge" : "", | 20260507 14:15:34.394 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:34.394 | INFO | "malloc" : " 14 GB", | 20260507 14:15:34.394 | INFO | "pinned" : " 29 GB" | 20260507 14:15:34.394 | INFO | } | 20260507 14:15:34.394 | INFO | Perf report for the last 5.14024 seconds: | 20260507 14:15:34.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:34.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:15:34.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:15:34.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:34.394 | INFO | Uploaded 36GiB to GPU in 1.60217s (22.4695GiB/s) | 20260507 14:15:34.394 | INFO | Downloaded 34GiB to GPU in 1.52088s (22.3555GiB/s) | 20260507 14:15:39.532 | INFO | Analyzing chunk frames = [140288,140800) | 20260507 14:15:39.532 | INFO | MemInfo: { | 20260507 14:15:39.532 | INFO | "gpu" : " 17 GB", | 20260507 14:15:39.532 | INFO | "huge" : "", | 20260507 14:15:39.532 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:39.532 | INFO | "malloc" : " 14 GB", | 20260507 14:15:39.533 | INFO | "pinned" : " 29 GB" | 20260507 14:15:39.533 | INFO | } | 20260507 14:15:39.533 | INFO | Perf report for the last 5.13836 seconds: | 20260507 14:15:39.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:39.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:15:39.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:15:39.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:39.533 | INFO | Uploaded 36GiB to GPU in 1.60671s (22.4061GiB/s) | 20260507 14:15:39.533 | INFO | Downloaded 34GiB to GPU in 1.52657s (22.2722GiB/s) | 20260507 14:15:44.672 | INFO | Analyzing chunk frames = [140800,141312) | 20260507 14:15:44.672 | INFO | MemInfo: { | 20260507 14:15:44.672 | INFO | "gpu" : " 17 GB", | 20260507 14:15:44.672 | INFO | "huge" : "", | 20260507 14:15:44.672 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:44.672 | INFO | "malloc" : " 14 GB", | 20260507 14:15:44.672 | INFO | "pinned" : " 29 GB" | 20260507 14:15:44.672 | INFO | } | 20260507 14:15:44.672 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:15:44.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:44.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:15:44.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:15:44.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:44.672 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4166GiB/s) | 20260507 14:15:44.672 | INFO | Downloaded 34GiB to GPU in 1.52645s (22.2739GiB/s) | 20260507 14:15:49.811 | INFO | Analyzing chunk frames = [141312,141824) | 20260507 14:15:49.811 | INFO | MemInfo: { | 20260507 14:15:49.811 | INFO | "gpu" : " 17 GB", | 20260507 14:15:49.811 | INFO | "huge" : "", | 20260507 14:15:49.811 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:49.811 | INFO | "malloc" : " 14 GB", | 20260507 14:15:49.811 | INFO | "pinned" : " 29 GB" | 20260507 14:15:49.811 | INFO | } | 20260507 14:15:49.811 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:15:49.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:49.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.811 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:15:49.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.811 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:15:49.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:49.812 | INFO | Uploaded 36GiB to GPU in 1.60572s (22.4199GiB/s) | 20260507 14:15:49.812 | INFO | Downloaded 34GiB to GPU in 1.5246s (22.3009GiB/s) | 20260507 14:15:54.950 | INFO | Analyzing chunk frames = [141824,142336) | 20260507 14:15:54.950 | INFO | MemInfo: { | 20260507 14:15:54.950 | INFO | "gpu" : " 17 GB", | 20260507 14:15:54.950 | INFO | "huge" : "", | 20260507 14:15:54.951 | INFO | "huge_shared" : " 29 GB", | 20260507 14:15:54.951 | INFO | "malloc" : " 14 GB", | 20260507 14:15:54.951 | INFO | "pinned" : " 29 GB" | 20260507 14:15:54.951 | INFO | } | 20260507 14:15:54.951 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:15:54.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:15:54.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:15:54.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:15:54.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:15:54.951 | INFO | Uploaded 36GiB to GPU in 1.60504s (22.4293GiB/s) | 20260507 14:15:54.951 | INFO | Downloaded 34GiB to GPU in 1.52353s (22.3166GiB/s) | 20260507 14:16:00.090 | INFO | Analyzing chunk frames = [142336,142848) | 20260507 14:16:00.090 | INFO | MemInfo: { | 20260507 14:16:00.090 | INFO | "gpu" : " 17 GB", | 20260507 14:16:00.090 | INFO | "huge" : "", | 20260507 14:16:00.090 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:00.090 | INFO | "malloc" : " 14 GB", | 20260507 14:16:00.090 | INFO | "pinned" : " 29 GB" | 20260507 14:16:00.090 | INFO | } | 20260507 14:16:00.090 | INFO | Perf report for the last 5.1391 seconds: | 20260507 14:16:00.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:00.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:16:00.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:16:00.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:00.090 | INFO | Uploaded 36GiB to GPU in 1.60592s (22.4171GiB/s) | 20260507 14:16:00.090 | INFO | Downloaded 34GiB to GPU in 1.52482s (22.2977GiB/s) | 20260507 14:16:05.229 | INFO | Analyzing chunk frames = [142848,143360) | 20260507 14:16:05.229 | INFO | MemInfo: { | 20260507 14:16:05.229 | INFO | "gpu" : " 17 GB", | 20260507 14:16:05.229 | INFO | "huge" : "", | 20260507 14:16:05.229 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:05.230 | INFO | "malloc" : " 15 GB", | 20260507 14:16:05.230 | INFO | "pinned" : " 29 GB" | 20260507 14:16:05.230 | INFO | } | 20260507 14:16:05.230 | INFO | Perf report for the last 5.13982 seconds: | 20260507 14:16:05.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:05.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 14:16:05.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:16:05.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:05.230 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260507 14:16:05.230 | INFO | Downloaded 34GiB to GPU in 1.52425s (22.306GiB/s) | 20260507 14:16:05.327 | INFO | Starting a round of Baz IO | 20260507 14:16:08.229 | INFO | Populating RTMetrics report | 20260507 14:16:08.230 | INFO | RTMetrics report populated | 20260507 14:16:08.230 | INFO | Opening tmp RTMetrics json file | 20260507 14:16:08.231 | INFO | Serializing RTMetrics to json | 20260507 14:16:08.232 | INFO | Serialization done, swapping tmp file now... | 20260507 14:16:08.233 | INFO | RTMetrics done | 20260507 14:16:08.330 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.003 seconds | 20260507 14:16:08.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.14% / 0.16% / 38.70% | 20260507 14:16:08.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.87216% / 0.000584126% / 1.98682e-05% | 20260507 14:16:08.330 | INFO | Baz IO overall: | 20260507 14:16:08.330 | INFO | BytesWritten : 40984336887 | 20260507 14:16:08.330 | INFO | NumEvents : 18606281163 | 20260507 14:16:10.368 | INFO | Analyzing chunk frames = [143360,143872) | 20260507 14:16:10.368 | INFO | MemInfo: { | 20260507 14:16:10.368 | INFO | "gpu" : " 17 GB", | 20260507 14:16:10.368 | INFO | "huge" : "", | 20260507 14:16:10.368 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:10.368 | INFO | "malloc" : " 14 GB", | 20260507 14:16:10.368 | INFO | "pinned" : " 29 GB" | 20260507 14:16:10.369 | INFO | } | 20260507 14:16:10.369 | INFO | Perf report for the last 5.13892 seconds: | 20260507 14:16:10.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:10.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:10.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:10.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:10.369 | INFO | ANALYSIS : 57.5%, 2.9, 88.0ms, 0 | 20260507 14:16:10.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:10.369 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 14:16:10.369 | INFO | PRE_HQ : 56.3%, 2.1, 60.2ms, 0 | 20260507 14:16:10.369 | INFO | BAZWRITER : 58.0%, 4.3, 128.6ms, 0 | 20260507 14:16:10.369 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 14:16:10.369 | INFO | Uploaded 36GiB to GPU in 1.65271s (21.7824GiB/s) | 20260507 14:16:10.369 | INFO | Downloaded 38GiB to GPU in 1.71618s (22.1422GiB/s) | 20260507 14:16:15.508 | INFO | Analyzing chunk frames = [143872,144384) | 20260507 14:16:15.508 | INFO | MemInfo: { | 20260507 14:16:15.508 | INFO | "gpu" : " 17 GB", | 20260507 14:16:15.508 | INFO | "huge" : "", | 20260507 14:16:15.508 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:15.508 | INFO | "malloc" : " 14 GB", | 20260507 14:16:15.508 | INFO | "pinned" : " 29 GB" | 20260507 14:16:15.508 | INFO | } | 20260507 14:16:15.508 | INFO | Perf report for the last 5.13969 seconds: | 20260507 14:16:15.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:15.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.508 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:16:15.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.508 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:16:15.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:15.509 | INFO | Uploaded 36GiB to GPU in 1.60201s (22.4718GiB/s) | 20260507 14:16:15.509 | INFO | Downloaded 34GiB to GPU in 1.52067s (22.3585GiB/s) | 20260507 14:16:20.647 | INFO | Analyzing chunk frames = [144384,144896) | 20260507 14:16:20.647 | INFO | MemInfo: { | 20260507 14:16:20.647 | INFO | "gpu" : " 17 GB", | 20260507 14:16:20.647 | INFO | "huge" : "", | 20260507 14:16:20.647 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:20.647 | INFO | "malloc" : " 14 GB", | 20260507 14:16:20.647 | INFO | "pinned" : " 29 GB" | 20260507 14:16:20.647 | INFO | } | 20260507 14:16:20.647 | INFO | Perf report for the last 5.1391 seconds: | 20260507 14:16:20.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:20.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.647 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:16:20.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.648 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:16:20.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:20.648 | INFO | Uploaded 36GiB to GPU in 1.60678s (22.4051GiB/s) | 20260507 14:16:20.648 | INFO | Downloaded 34GiB to GPU in 1.52656s (22.2723GiB/s) | 20260507 14:16:25.787 | INFO | Analyzing chunk frames = [144896,145408) | 20260507 14:16:25.787 | INFO | MemInfo: { | 20260507 14:16:25.787 | INFO | "gpu" : " 17 GB", | 20260507 14:16:25.787 | INFO | "huge" : "", | 20260507 14:16:25.787 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:25.787 | INFO | "malloc" : " 14 GB", | 20260507 14:16:25.787 | INFO | "pinned" : " 29 GB" | 20260507 14:16:25.787 | INFO | } | 20260507 14:16:25.787 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:16:25.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:25.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:16:25.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:16:25.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:25.787 | INFO | Uploaded 36GiB to GPU in 1.606s (22.4159GiB/s) | 20260507 14:16:25.787 | INFO | Downloaded 34GiB to GPU in 1.52533s (22.2903GiB/s) | 20260507 14:16:30.926 | INFO | Analyzing chunk frames = [145408,145920) | 20260507 14:16:30.926 | INFO | MemInfo: { | 20260507 14:16:30.926 | INFO | "gpu" : " 17 GB", | 20260507 14:16:30.926 | INFO | "huge" : "", | 20260507 14:16:30.926 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:30.926 | INFO | "malloc" : " 14 GB", | 20260507 14:16:30.926 | INFO | "pinned" : " 29 GB" | 20260507 14:16:30.926 | INFO | } | 20260507 14:16:30.926 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:16:30.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:30.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:16:30.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:16:30.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:30.926 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4263GiB/s) | 20260507 14:16:30.927 | INFO | Downloaded 34GiB to GPU in 1.52406s (22.3088GiB/s) | 20260507 14:16:36.065 | INFO | Analyzing chunk frames = [145920,146432) | 20260507 14:16:36.065 | INFO | MemInfo: { | 20260507 14:16:36.065 | INFO | "gpu" : " 17 GB", | 20260507 14:16:36.066 | INFO | "huge" : "", | 20260507 14:16:36.066 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:36.066 | INFO | "malloc" : " 14 GB", | 20260507 14:16:36.066 | INFO | "pinned" : " 29 GB" | 20260507 14:16:36.066 | INFO | } | 20260507 14:16:36.066 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:16:36.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:36.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:16:36.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:16:36.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:36.066 | INFO | Uploaded 36GiB to GPU in 1.60462s (22.4352GiB/s) | 20260507 14:16:36.066 | INFO | Downloaded 34GiB to GPU in 1.52337s (22.3189GiB/s) | 20260507 14:16:41.205 | INFO | Analyzing chunk frames = [146432,146944) | 20260507 14:16:41.205 | INFO | MemInfo: { | 20260507 14:16:41.205 | INFO | "gpu" : " 17 GB", | 20260507 14:16:41.205 | INFO | "huge" : "", | 20260507 14:16:41.205 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:41.205 | INFO | "malloc" : " 15 GB", | 20260507 14:16:41.205 | INFO | "pinned" : " 29 GB" | 20260507 14:16:41.205 | INFO | } | 20260507 14:16:41.205 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:16:41.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:41.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:16:41.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:16:41.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:41.205 | INFO | Uploaded 36GiB to GPU in 1.60559s (22.4217GiB/s) | 20260507 14:16:41.205 | INFO | Downloaded 34GiB to GPU in 1.52478s (22.2983GiB/s) | 20260507 14:16:46.344 | INFO | Analyzing chunk frames = [146944,147456) | 20260507 14:16:46.344 | INFO | MemInfo: { | 20260507 14:16:46.344 | INFO | "gpu" : " 17 GB", | 20260507 14:16:46.344 | INFO | "huge" : "", | 20260507 14:16:46.344 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:46.344 | INFO | "malloc" : " 15 GB", | 20260507 14:16:46.344 | INFO | "pinned" : " 29 GB" | 20260507 14:16:46.344 | INFO | } | 20260507 14:16:46.344 | INFO | Perf report for the last 5.13913 seconds: | 20260507 14:16:46.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:46.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:16:46.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:16:46.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:46.348 | INFO | Uploaded 36GiB to GPU in 1.60645s (22.4097GiB/s) | 20260507 14:16:46.348 | INFO | Downloaded 34GiB to GPU in 1.52664s (22.2711GiB/s) | 20260507 14:16:46.348 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:16:46.348 | INFO | Total mem usage: 15.5 GiB | 20260507 14:16:46.348 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:16:46.348 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -2.96%) | 20260507 14:16:46.348 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:16:46.348 | INFO | High Water Breakdown: | 20260507 14:16:46.348 | INFO | 58.6% : 8.84 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:16:46.348 | INFO | 41.2% : 6.21 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:16:46.348 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:16:46.348 | INFO | ------------------------------------------ | 20260507 14:16:46.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:16:46.348 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:16:46.348 | INFO | Total mem usage: 26.5 GiB | 20260507 14:16:46.348 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:16:46.348 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:16:46.348 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:16:46.348 | INFO | High Water Breakdown: | 20260507 14:16:46.349 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:16:46.349 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:16:46.349 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:16:46.349 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:16:46.349 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:16:46.349 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:16:46.349 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:16:46.349 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:16:46.349 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:16:46.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:16:46.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:16:46.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:16:46.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:16:46.349 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:16:46.349 | INFO | ------------------------------------------ | 20260507 14:16:46.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:16:46.349 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:16:46.349 | INFO | Total mem usage: 27.8 GiB | 20260507 14:16:46.349 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:16:46.349 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:16:46.349 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:16:46.349 | INFO | High Water Breakdown: | 20260507 14:16:46.349 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:16:46.349 | INFO | ------------------------------------------ | 20260507 14:16:46.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:16:46.349 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:16:46.349 | INFO | Total mem usage: 15.7 GiB | 20260507 14:16:46.349 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:16:46.349 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:16:46.349 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:16:46.349 | INFO | High Water Breakdown: | 20260507 14:16:46.349 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:16:46.349 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:16:46.349 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:16:46.349 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:16:46.349 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:16:46.349 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:16:46.349 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:16:46.349 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:16:46.349 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:16:46.349 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:16:46.349 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:16:46.349 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:16:46.349 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:16:46.349 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:16:46.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:16:46.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:16:46.349 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:16:46.349 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:16:46.349 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:16:46.349 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:16:46.349 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:16:46.349 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:16:46.349 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:16:46.349 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:16:46.349 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:16:46.349 | INFO | ------------------------------------------ | 20260507 14:16:46.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:16:46.349 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:16:46.349 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:16:46.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:16:46.349 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:16:46.349 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.35e-03 / 0.00e+00 | 20260507 14:16:46.349 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.63e-02 / 0.00e+00 | 20260507 14:16:46.349 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 16.07%, DME -- 3.87e+01 / -nan, 3.07e+00 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.72e+01 / -nan, 4.40e-02 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 3.71%, Metrics -- 1.12e+00 / -nan, 1.79e-01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.16e-01 / -nan, 6.18e-02 / 0.00e+00 | 20260507 14:16:46.350 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:16:46.350 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:16:46.350 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:16:46.350 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 18.61%, Upload -- 1.68e+01 / 0.00e+00, 4.26e-01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 17.89%, Download -- 1.61e+01 / 0.00e+00, 7.55e-01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 33.36%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 0.00%, LockDownload -- 4.38e-04 / 0.00e+00, 1.29e-04 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 29.56%, LockCompute -- 2.67e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:16:46.350 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 5.44e-02 / 0.00e+00 | 20260507 14:16:46.445 | INFO | Starting a round of Baz IO | 20260507 14:16:49.350 | INFO | Populating RTMetrics report | 20260507 14:16:49.351 | INFO | RTMetrics report populated | 20260507 14:16:49.351 | INFO | Opening tmp RTMetrics json file | 20260507 14:16:49.352 | INFO | Serializing RTMetrics to json | 20260507 14:16:49.380 | INFO | Serialization done, swapping tmp file now... | 20260507 14:16:49.438 | INFO | RTMetrics done | 20260507 14:16:49.466 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.021 seconds | 20260507 14:16:49.466 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.98% / 0.16% / 38.86% | 20260507 14:16:49.466 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.82467% / 0.0005126% / 1.19209e-05% | 20260507 14:16:49.466 | INFO | Baz IO overall: | 20260507 14:16:49.466 | INFO | BytesWritten : 42726289096 | 20260507 14:16:49.466 | INFO | NumEvents : 19458959260 | 20260507 14:16:51.483 | INFO | Analyzing chunk frames = [147456,147968) | 20260507 14:16:51.483 | INFO | MemInfo: { | 20260507 14:16:51.483 | INFO | "gpu" : " 17 GB", | 20260507 14:16:51.483 | INFO | "huge" : "", | 20260507 14:16:51.483 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:51.483 | INFO | "malloc" : " 14 GB", | 20260507 14:16:51.487 | INFO | "pinned" : " 29 GB" | 20260507 14:16:51.487 | INFO | } | 20260507 14:16:51.487 | INFO | Perf report for the last 5.14287 seconds: | 20260507 14:16:51.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:51.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:51.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:51.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:51.487 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260507 14:16:51.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:51.487 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 14:16:51.487 | INFO | PRE_HQ : 56.3%, 2.1, 60.5ms, 0 | 20260507 14:16:51.487 | INFO | BAZWRITER : 58.3%, 3.8, 115.1ms, 0 | 20260507 14:16:51.487 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 14:16:51.487 | INFO | Uploaded 36GiB to GPU in 1.64012s (21.9497GiB/s) | 20260507 14:16:51.487 | INFO | Downloaded 38GiB to GPU in 1.7041s (22.2992GiB/s) | 20260507 14:16:56.623 | INFO | Analyzing chunk frames = [147968,148480) | 20260507 14:16:56.623 | INFO | MemInfo: { | 20260507 14:16:56.623 | INFO | "gpu" : " 17 GB", | 20260507 14:16:56.623 | INFO | "huge" : "", | 20260507 14:16:56.623 | INFO | "huge_shared" : " 29 GB", | 20260507 14:16:56.623 | INFO | "malloc" : " 14 GB", | 20260507 14:16:56.623 | INFO | "pinned" : " 29 GB" | 20260507 14:16:56.623 | INFO | } | 20260507 14:16:56.623 | INFO | Perf report for the last 5.13598 seconds: | 20260507 14:16:56.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:16:56.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:16:56.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:16:56.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:16:56.623 | INFO | Uploaded 36GiB to GPU in 1.60366s (22.4487GiB/s) | 20260507 14:16:56.623 | INFO | Downloaded 34GiB to GPU in 1.52186s (22.3411GiB/s) | 20260507 14:17:01.762 | INFO | Analyzing chunk frames = [148480,148992) | 20260507 14:17:01.762 | INFO | MemInfo: { | 20260507 14:17:01.762 | INFO | "gpu" : " 17 GB", | 20260507 14:17:01.762 | INFO | "huge" : "", | 20260507 14:17:01.762 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:01.762 | INFO | "malloc" : " 14 GB", | 20260507 14:17:01.762 | INFO | "pinned" : " 29 GB" | 20260507 14:17:01.762 | INFO | } | 20260507 14:17:01.762 | INFO | Perf report for the last 5.13906 seconds: | 20260507 14:17:01.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:01.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260507 14:17:01.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:17:01.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:01.762 | INFO | Uploaded 36GiB to GPU in 1.60689s (22.4036GiB/s) | 20260507 14:17:01.762 | INFO | Downloaded 34GiB to GPU in 1.52661s (22.2716GiB/s) | 20260507 14:17:06.902 | INFO | Analyzing chunk frames = [148992,149504) | 20260507 14:17:06.902 | INFO | MemInfo: { | 20260507 14:17:06.902 | INFO | "gpu" : " 17 GB", | 20260507 14:17:06.902 | INFO | "huge" : "", | 20260507 14:17:06.902 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:06.902 | INFO | "malloc" : " 14 GB", | 20260507 14:17:06.902 | INFO | "pinned" : " 29 GB" | 20260507 14:17:06.902 | INFO | } | 20260507 14:17:06.902 | INFO | Perf report for the last 5.14048 seconds: | 20260507 14:17:06.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:06.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:17:06.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:17:06.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:06.903 | INFO | Uploaded 36GiB to GPU in 1.6063s (22.4117GiB/s) | 20260507 14:17:06.903 | INFO | Downloaded 34GiB to GPU in 1.52648s (22.2735GiB/s) | 20260507 14:17:12.041 | INFO | Analyzing chunk frames = [149504,150016) | 20260507 14:17:12.041 | INFO | MemInfo: { | 20260507 14:17:12.041 | INFO | "gpu" : " 17 GB", | 20260507 14:17:12.041 | INFO | "huge" : "", | 20260507 14:17:12.041 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:12.041 | INFO | "malloc" : " 14 GB", | 20260507 14:17:12.041 | INFO | "pinned" : " 29 GB" | 20260507 14:17:12.041 | INFO | } | 20260507 14:17:12.041 | INFO | Perf report for the last 5.13817 seconds: | 20260507 14:17:12.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:12.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:17:12.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:17:12.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:12.041 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.4349GiB/s) | 20260507 14:17:12.041 | INFO | Downloaded 34GiB to GPU in 1.52413s (22.3078GiB/s) | 20260507 14:17:17.180 | INFO | Analyzing chunk frames = [150016,150528) | 20260507 14:17:17.180 | INFO | MemInfo: { | 20260507 14:17:17.180 | INFO | "gpu" : " 17 GB", | 20260507 14:17:17.180 | INFO | "huge" : "", | 20260507 14:17:17.180 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:17.180 | INFO | "malloc" : " 14 GB", | 20260507 14:17:17.180 | INFO | "pinned" : " 29 GB" | 20260507 14:17:17.180 | INFO | } | 20260507 14:17:17.180 | INFO | Perf report for the last 5.1396 seconds: | 20260507 14:17:17.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:17.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.180 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:17:17.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.180 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:17:17.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:17.181 | INFO | Uploaded 36GiB to GPU in 1.60511s (22.4283GiB/s) | 20260507 14:17:17.181 | INFO | Downloaded 34GiB to GPU in 1.52401s (22.3096GiB/s) | 20260507 14:17:22.319 | INFO | Analyzing chunk frames = [150528,151040) | 20260507 14:17:22.319 | INFO | MemInfo: { | 20260507 14:17:22.319 | INFO | "gpu" : " 17 GB", | 20260507 14:17:22.319 | INFO | "huge" : "", | 20260507 14:17:22.319 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:22.320 | INFO | "malloc" : " 14 GB", | 20260507 14:17:22.320 | INFO | "pinned" : " 29 GB" | 20260507 14:17:22.320 | INFO | } | 20260507 14:17:22.320 | INFO | Perf report for the last 5.1397 seconds: | 20260507 14:17:22.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:22.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:17:22.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:17:22.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:22.320 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4365GiB/s) | 20260507 14:17:22.320 | INFO | Downloaded 34GiB to GPU in 1.52459s (22.301GiB/s) | 20260507 14:17:27.460 | INFO | Analyzing chunk frames = [151040,151552) | 20260507 14:17:27.460 | INFO | MemInfo: { | 20260507 14:17:27.460 | INFO | "gpu" : " 17 GB", | 20260507 14:17:27.460 | INFO | "huge" : "", | 20260507 14:17:27.460 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:27.460 | INFO | "malloc" : " 15 GB", | 20260507 14:17:27.460 | INFO | "pinned" : " 29 GB" | 20260507 14:17:27.460 | INFO | } | 20260507 14:17:27.460 | INFO | Perf report for the last 5.1402 seconds: | 20260507 14:17:27.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:27.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:17:27.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:17:27.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:27.460 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4178GiB/s) | 20260507 14:17:27.460 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.2889GiB/s) | 20260507 14:17:27.558 | INFO | Starting a round of Baz IO | 20260507 14:17:30.460 | INFO | Populating RTMetrics report | 20260507 14:17:30.460 | INFO | RTMetrics report populated | 20260507 14:17:30.460 | INFO | Opening tmp RTMetrics json file | 20260507 14:17:30.562 | INFO | Serializing RTMetrics to json | 20260507 14:17:30.594 | INFO | Serialization done, swapping tmp file now... | 20260507 14:17:30.597 | INFO | RTMetrics done | 20260507 14:17:30.690 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.132 seconds | 20260507 14:17:30.690 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.83% / 0.15% / 39.02% | 20260507 14:17:30.690 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.75499% / 0.000528495% / 1.58946e-05% | 20260507 14:17:30.690 | INFO | Baz IO overall: | 20260507 14:17:30.690 | INFO | BytesWritten : 44470906515 | 20260507 14:17:30.690 | INFO | NumEvents : 20312261073 | 20260507 14:17:32.598 | INFO | Analyzing chunk frames = [151552,152064) | 20260507 14:17:32.598 | INFO | MemInfo: { | 20260507 14:17:32.598 | INFO | "gpu" : " 17 GB", | 20260507 14:17:32.598 | INFO | "huge" : "", | 20260507 14:17:32.598 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:32.598 | INFO | "malloc" : " 14 GB", | 20260507 14:17:32.598 | INFO | "pinned" : " 29 GB" | 20260507 14:17:32.598 | INFO | } | 20260507 14:17:32.598 | INFO | Perf report for the last 5.13775 seconds: | 20260507 14:17:32.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:32.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:32.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:32.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:32.598 | INFO | ANALYSIS : 57.5%, 2.9, 88.4ms, 0 | 20260507 14:17:32.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:32.598 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 14:17:32.598 | INFO | PRE_HQ : 56.5%, 2.1, 60.3ms, 0 | 20260507 14:17:32.598 | INFO | BAZWRITER : 60.5%, 4.3, 133.6ms, 0 | 20260507 14:17:32.598 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 14:17:32.598 | INFO | Uploaded 36GiB to GPU in 1.65283s (21.7808GiB/s) | 20260507 14:17:32.598 | INFO | Downloaded 38GiB to GPU in 1.71629s (22.1408GiB/s) | 20260507 14:17:37.737 | INFO | Analyzing chunk frames = [152064,152576) | 20260507 14:17:37.737 | INFO | MemInfo: { | 20260507 14:17:37.737 | INFO | "gpu" : " 17 GB", | 20260507 14:17:37.737 | INFO | "huge" : "", | 20260507 14:17:37.737 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:37.737 | INFO | "malloc" : " 14 GB", | 20260507 14:17:37.737 | INFO | "pinned" : " 29 GB" | 20260507 14:17:37.738 | INFO | } | 20260507 14:17:37.738 | INFO | Perf report for the last 5.13962 seconds: | 20260507 14:17:37.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:37.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:17:37.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:17:37.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:37.738 | INFO | Uploaded 36GiB to GPU in 1.60287s (22.4597GiB/s) | 20260507 14:17:37.738 | INFO | Downloaded 34GiB to GPU in 1.52144s (22.3472GiB/s) | 20260507 14:17:42.880 | INFO | Analyzing chunk frames = [152576,153088) | 20260507 14:17:42.880 | INFO | MemInfo: { | 20260507 14:17:42.880 | INFO | "gpu" : " 17 GB", | 20260507 14:17:42.880 | INFO | "huge" : "", | 20260507 14:17:42.880 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:42.880 | INFO | "malloc" : " 14 GB", | 20260507 14:17:42.880 | INFO | "pinned" : " 29 GB" | 20260507 14:17:42.880 | INFO | } | 20260507 14:17:42.880 | INFO | Perf report for the last 5.14254 seconds: | 20260507 14:17:42.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:42.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.880 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:17:42.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.880 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:17:42.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:42.881 | INFO | Uploaded 36GiB to GPU in 1.60671s (22.406GiB/s) | 20260507 14:17:42.881 | INFO | Downloaded 34GiB to GPU in 1.52698s (22.2662GiB/s) | 20260507 14:17:48.016 | INFO | Analyzing chunk frames = [153088,153600) | 20260507 14:17:48.016 | INFO | MemInfo: { | 20260507 14:17:48.016 | INFO | "gpu" : " 17 GB", | 20260507 14:17:48.016 | INFO | "huge" : "", | 20260507 14:17:48.016 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:48.016 | INFO | "malloc" : " 14 GB", | 20260507 14:17:48.016 | INFO | "pinned" : " 29 GB" | 20260507 14:17:48.016 | INFO | } | 20260507 14:17:48.016 | INFO | Perf report for the last 5.13607 seconds: | 20260507 14:17:48.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:48.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:17:48.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:17:48.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:48.016 | INFO | Uploaded 36GiB to GPU in 1.6064s (22.4103GiB/s) | 20260507 14:17:48.017 | INFO | Downloaded 34GiB to GPU in 1.52643s (22.2742GiB/s) | 20260507 14:17:53.156 | INFO | Analyzing chunk frames = [153600,154112) | 20260507 14:17:53.156 | INFO | MemInfo: { | 20260507 14:17:53.156 | INFO | "gpu" : " 17 GB", | 20260507 14:17:53.156 | INFO | "huge" : "", | 20260507 14:17:53.156 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:53.156 | INFO | "malloc" : " 14 GB", | 20260507 14:17:53.156 | INFO | "pinned" : " 29 GB" | 20260507 14:17:53.156 | INFO | } | 20260507 14:17:53.156 | INFO | Perf report for the last 5.13986 seconds: | 20260507 14:17:53.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:53.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:17:53.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:17:53.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:53.156 | INFO | Uploaded 36GiB to GPU in 1.60472s (22.4338GiB/s) | 20260507 14:17:53.156 | INFO | Downloaded 34GiB to GPU in 1.52466s (22.3001GiB/s) | 20260507 14:17:58.295 | INFO | Analyzing chunk frames = [154112,154624) | 20260507 14:17:58.296 | INFO | MemInfo: { | 20260507 14:17:58.296 | INFO | "gpu" : " 17 GB", | 20260507 14:17:58.296 | INFO | "huge" : "", | 20260507 14:17:58.296 | INFO | "huge_shared" : " 29 GB", | 20260507 14:17:58.296 | INFO | "malloc" : " 14 GB", | 20260507 14:17:58.296 | INFO | "pinned" : " 29 GB" | 20260507 14:17:58.296 | INFO | } | 20260507 14:17:58.296 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:17:58.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:17:58.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:17:58.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:17:58.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:17:58.296 | INFO | Uploaded 36GiB to GPU in 1.60512s (22.4282GiB/s) | 20260507 14:17:58.296 | INFO | Downloaded 34GiB to GPU in 1.52387s (22.3116GiB/s) | 20260507 14:18:03.434 | INFO | Analyzing chunk frames = [154624,155136) | 20260507 14:18:03.434 | INFO | MemInfo: { | 20260507 14:18:03.434 | INFO | "gpu" : " 17 GB", | 20260507 14:18:03.434 | INFO | "huge" : "", | 20260507 14:18:03.434 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:03.434 | INFO | "malloc" : " 14 GB", | 20260507 14:18:03.434 | INFO | "pinned" : " 29 GB" | 20260507 14:18:03.434 | INFO | } | 20260507 14:18:03.434 | INFO | Perf report for the last 5.13848 seconds: | 20260507 14:18:03.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:03.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.434 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:18:03.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.434 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:18:03.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:03.435 | INFO | Uploaded 36GiB to GPU in 1.60592s (22.4171GiB/s) | 20260507 14:18:03.435 | INFO | Downloaded 34GiB to GPU in 1.52668s (22.2706GiB/s) | 20260507 14:18:08.573 | INFO | Analyzing chunk frames = [155136,155648) | 20260507 14:18:08.573 | INFO | MemInfo: { | 20260507 14:18:08.573 | INFO | "gpu" : " 17 GB", | 20260507 14:18:08.573 | INFO | "huge" : "", | 20260507 14:18:08.573 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:08.573 | INFO | "malloc" : " 15 GB", | 20260507 14:18:08.573 | INFO | "pinned" : " 29 GB" | 20260507 14:18:08.573 | INFO | } | 20260507 14:18:08.573 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:18:08.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:08.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:18:08.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:18:08.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:08.574 | INFO | Uploaded 36GiB to GPU in 1.60649s (22.4091GiB/s) | 20260507 14:18:08.574 | INFO | Downloaded 34GiB to GPU in 1.52671s (22.2701GiB/s) | 20260507 14:18:08.574 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:18:08.574 | INFO | Total mem usage: 15.5 GiB | 20260507 14:18:08.574 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:18:08.574 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -3.03%) | 20260507 14:18:08.574 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:18:08.574 | INFO | High Water Breakdown: | 20260507 14:18:08.574 | INFO | 58.7% : 8.83 GiB ( 96.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:18:08.574 | INFO | 41.2% : 6.21 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:18:08.574 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:18:08.574 | INFO | ------------------------------------------ | 20260507 14:18:08.574 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:18:08.574 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:18:08.574 | INFO | Total mem usage: 26.5 GiB | 20260507 14:18:08.574 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:18:08.575 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:18:08.575 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:18:08.575 | INFO | High Water Breakdown: | 20260507 14:18:08.575 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:18:08.575 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:18:08.575 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:18:08.575 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:18:08.575 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:18:08.575 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:18:08.575 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:18:08.575 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:18:08.575 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:18:08.575 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:18:08.575 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:18:08.575 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:18:08.575 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:18:08.575 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:18:08.575 | INFO | ------------------------------------------ | 20260507 14:18:08.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:18:08.575 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:18:08.575 | INFO | Total mem usage: 27.8 GiB | 20260507 14:18:08.575 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:18:08.575 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:18:08.575 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:18:08.575 | INFO | High Water Breakdown: | 20260507 14:18:08.575 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:18:08.575 | INFO | ------------------------------------------ | 20260507 14:18:08.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:18:08.575 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:18:08.575 | INFO | Total mem usage: 15.7 GiB | 20260507 14:18:08.575 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:18:08.575 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:18:08.575 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:18:08.575 | INFO | High Water Breakdown: | 20260507 14:18:08.575 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:18:08.575 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:18:08.575 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:18:08.575 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:18:08.576 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:18:08.576 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:18:08.576 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:18:08.576 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:18:08.576 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:18:08.576 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:18:08.576 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:18:08.576 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:18:08.576 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:18:08.576 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:18:08.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:18:08.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:18:08.576 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:18:08.576 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:18:08.576 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:18:08.576 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:18:08.576 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:18:08.576 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:18:08.576 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:18:08.576 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:18:08.576 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:18:08.576 | INFO | ------------------------------------------ | 20260507 14:18:08.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:18:08.576 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:18:08.576 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:18:08.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:18:08.576 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.56e-03 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 2.00e-02 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.14e-02 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 16.08%, DME -- 3.87e+01 / -nan, 3.13e+00 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.72e+01 / -nan, 4.47e-02 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 3.70%, Metrics -- 1.11e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.15e-01 / -nan, 6.07e-02 / 0.00e+00 | 20260507 14:18:08.576 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:18:08.576 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:18:08.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:18:08.576 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.01e-01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 17.87%, Download -- 1.61e+01 / 0.00e+00, 7.22e-01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 33.32%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 0.00%, LockDownload -- 4.40e-04 / 0.00e+00, 1.67e-04 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 29.64%, LockCompute -- 2.68e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 14:18:08.576 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260507 14:18:08.671 | INFO | Starting a round of Baz IO | 20260507 14:18:11.574 | INFO | Populating RTMetrics report | 20260507 14:18:11.574 | INFO | RTMetrics report populated | 20260507 14:18:11.574 | INFO | Opening tmp RTMetrics json file | 20260507 14:18:11.703 | INFO | Serializing RTMetrics to json | 20260507 14:18:11.706 | INFO | Serialization done, swapping tmp file now... | 20260507 14:18:11.709 | INFO | RTMetrics done | 20260507 14:18:11.718 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.047 seconds | 20260507 14:18:11.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.69% / 0.15% / 39.17% | 20260507 14:18:11.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.73953% / 0.000516574% / 1.98682e-05% | 20260507 14:18:11.718 | INFO | Baz IO overall: | 20260507 14:18:11.718 | INFO | BytesWritten : 46221388840 | 20260507 14:18:11.718 | INFO | NumEvents : 21168860359 | 20260507 14:18:13.713 | INFO | Analyzing chunk frames = [155648,156160) | 20260507 14:18:13.713 | INFO | MemInfo: { | 20260507 14:18:13.713 | INFO | "gpu" : " 17 GB", | 20260507 14:18:13.713 | INFO | "huge" : "", | 20260507 14:18:13.713 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:13.713 | INFO | "malloc" : " 14 GB", | 20260507 14:18:13.713 | INFO | "pinned" : " 29 GB" | 20260507 14:18:13.713 | INFO | } | 20260507 14:18:13.713 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:18:13.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:13.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:13.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:13.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:13.713 | INFO | ANALYSIS : 57.5%, 2.9, 88.4ms, 0 | 20260507 14:18:13.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:13.713 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260507 14:18:13.713 | INFO | PRE_HQ : 56.3%, 2.1, 60.3ms, 0 | 20260507 14:18:13.713 | INFO | BAZWRITER : 58.8%, 4.3, 129.3ms, 0 | 20260507 14:18:13.713 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 14:18:13.714 | INFO | Uploaded 36GiB to GPU in 1.64487s (21.8862GiB/s) | 20260507 14:18:13.714 | INFO | Downloaded 38GiB to GPU in 1.7093s (22.2313GiB/s) | 20260507 14:18:18.852 | INFO | Analyzing chunk frames = [156160,156672) | 20260507 14:18:18.852 | INFO | MemInfo: { | 20260507 14:18:18.852 | INFO | "gpu" : " 17 GB", | 20260507 14:18:18.852 | INFO | "huge" : "", | 20260507 14:18:18.852 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:18.852 | INFO | "malloc" : " 14 GB", | 20260507 14:18:18.852 | INFO | "pinned" : " 29 GB" | 20260507 14:18:18.852 | INFO | } | 20260507 14:18:18.852 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:18:18.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:18.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.852 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:18:18.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.852 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:18:18.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:18.853 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4445GiB/s) | 20260507 14:18:18.853 | INFO | Downloaded 34GiB to GPU in 1.52316s (22.322GiB/s) | 20260507 14:18:23.991 | INFO | Analyzing chunk frames = [156672,157184) | 20260507 14:18:23.991 | INFO | MemInfo: { | 20260507 14:18:23.991 | INFO | "gpu" : " 17 GB", | 20260507 14:18:23.991 | INFO | "huge" : "", | 20260507 14:18:23.991 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:23.991 | INFO | "malloc" : " 14 GB", | 20260507 14:18:23.991 | INFO | "pinned" : " 29 GB" | 20260507 14:18:23.992 | INFO | } | 20260507 14:18:23.992 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:18:23.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:23.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260507 14:18:23.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:18:23.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:23.992 | INFO | Uploaded 36GiB to GPU in 1.60702s (22.4017GiB/s) | 20260507 14:18:23.992 | INFO | Downloaded 34GiB to GPU in 1.52778s (22.2545GiB/s) | 20260507 14:18:29.131 | INFO | Analyzing chunk frames = [157184,157696) | 20260507 14:18:29.131 | INFO | MemInfo: { | 20260507 14:18:29.131 | INFO | "gpu" : " 17 GB", | 20260507 14:18:29.131 | INFO | "huge" : "", | 20260507 14:18:29.131 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:29.131 | INFO | "malloc" : " 14 GB", | 20260507 14:18:29.131 | INFO | "pinned" : " 29 GB" | 20260507 14:18:29.131 | INFO | } | 20260507 14:18:29.131 | INFO | Perf report for the last 5.13962 seconds: | 20260507 14:18:29.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:29.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.131 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:18:29.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.131 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:18:29.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:29.132 | INFO | Uploaded 36GiB to GPU in 1.60546s (22.4234GiB/s) | 20260507 14:18:29.132 | INFO | Downloaded 34GiB to GPU in 1.52609s (22.2791GiB/s) | 20260507 14:18:34.270 | INFO | Analyzing chunk frames = [157696,158208) | 20260507 14:18:34.270 | INFO | MemInfo: { | 20260507 14:18:34.270 | INFO | "gpu" : " 17 GB", | 20260507 14:18:34.270 | INFO | "huge" : "", | 20260507 14:18:34.270 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:34.270 | INFO | "malloc" : " 14 GB", | 20260507 14:18:34.270 | INFO | "pinned" : " 29 GB" | 20260507 14:18:34.270 | INFO | } | 20260507 14:18:34.270 | INFO | Perf report for the last 5.13911 seconds: | 20260507 14:18:34.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:34.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.270 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:18:34.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.271 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:18:34.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:34.271 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260507 14:18:34.271 | INFO | Downloaded 34GiB to GPU in 1.52505s (22.2943GiB/s) | 20260507 14:18:39.409 | INFO | Analyzing chunk frames = [158208,158720) | 20260507 14:18:39.409 | INFO | MemInfo: { | 20260507 14:18:39.409 | INFO | "gpu" : " 17 GB", | 20260507 14:18:39.409 | INFO | "huge" : "", | 20260507 14:18:39.409 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:39.409 | INFO | "malloc" : " 14 GB", | 20260507 14:18:39.409 | INFO | "pinned" : " 29 GB" | 20260507 14:18:39.409 | INFO | } | 20260507 14:18:39.410 | INFO | Perf report for the last 5.13912 seconds: | 20260507 14:18:39.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:39.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:18:39.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:18:39.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:39.410 | INFO | Uploaded 36GiB to GPU in 1.6049s (22.4313GiB/s) | 20260507 14:18:39.410 | INFO | Downloaded 34GiB to GPU in 1.52393s (22.3108GiB/s) | 20260507 14:18:44.549 | INFO | Analyzing chunk frames = [158720,159232) | 20260507 14:18:44.549 | INFO | MemInfo: { | 20260507 14:18:44.549 | INFO | "gpu" : " 17 GB", | 20260507 14:18:44.549 | INFO | "huge" : "", | 20260507 14:18:44.549 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:44.549 | INFO | "malloc" : " 14 GB", | 20260507 14:18:44.549 | INFO | "pinned" : " 29 GB" | 20260507 14:18:44.549 | INFO | } | 20260507 14:18:44.549 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:18:44.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:44.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:18:44.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:18:44.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:44.549 | INFO | Uploaded 36GiB to GPU in 1.60556s (22.422GiB/s) | 20260507 14:18:44.549 | INFO | Downloaded 34GiB to GPU in 1.52584s (22.2828GiB/s) | 20260507 14:18:49.688 | INFO | Analyzing chunk frames = [159232,159744) | 20260507 14:18:49.688 | INFO | MemInfo: { | 20260507 14:18:49.688 | INFO | "gpu" : " 17 GB", | 20260507 14:18:49.688 | INFO | "huge" : "", | 20260507 14:18:49.688 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:49.688 | INFO | "malloc" : " 15 GB", | 20260507 14:18:49.688 | INFO | "pinned" : " 29 GB" | 20260507 14:18:49.688 | INFO | } | 20260507 14:18:49.689 | INFO | Perf report for the last 5.13966 seconds: | 20260507 14:18:49.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:49.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:18:49.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:18:49.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:49.689 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260507 14:18:49.689 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2906GiB/s) | 20260507 14:18:49.786 | INFO | Starting a round of Baz IO | 20260507 14:18:52.687 | INFO | Populating RTMetrics report | 20260507 14:18:52.687 | INFO | RTMetrics report populated | 20260507 14:18:52.687 | INFO | Opening tmp RTMetrics json file | 20260507 14:18:52.688 | INFO | Serializing RTMetrics to json | 20260507 14:18:52.699 | INFO | Serialization done, swapping tmp file now... | 20260507 14:18:52.711 | INFO | RTMetrics done | 20260507 14:18:52.766 | INFO | Finished a round of Baz IO: Wrote 1.635 GiB in 2.98 seconds | 20260507 14:18:52.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.55% / 0.14% / 39.31% | 20260507 14:18:52.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.70118% / 0.000516574% / 2.78155e-05% | 20260507 14:18:52.766 | INFO | Baz IO overall: | 20260507 14:18:52.766 | INFO | BytesWritten : 47977057814 | 20260507 14:18:52.766 | INFO | NumEvents : 22027577444 | 20260507 14:18:54.828 | INFO | Analyzing chunk frames = [159744,160256) | 20260507 14:18:54.828 | INFO | MemInfo: { | 20260507 14:18:54.828 | INFO | "gpu" : " 17 GB", | 20260507 14:18:54.828 | INFO | "huge" : "", | 20260507 14:18:54.828 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:54.828 | INFO | "malloc" : " 14 GB", | 20260507 14:18:54.828 | INFO | "pinned" : " 29 GB" | 20260507 14:18:54.828 | INFO | } | 20260507 14:18:54.828 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:18:54.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:54.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:54.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:54.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:54.828 | INFO | ANALYSIS : 57.5%, 3.0, 89.4ms, 0 | 20260507 14:18:54.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:54.828 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 14:18:54.828 | INFO | PRE_HQ : 56.2%, 2.1, 60.6ms, 0 | 20260507 14:18:54.828 | INFO | BAZWRITER : 57.5%, 3.9, 116.6ms, 0 | 20260507 14:18:54.828 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:18:54.828 | INFO | Uploaded 36GiB to GPU in 1.64569s (21.8753GiB/s) | 20260507 14:18:54.828 | INFO | Downloaded 38GiB to GPU in 1.70974s (22.2256GiB/s) | 20260507 14:18:59.967 | INFO | Analyzing chunk frames = [160256,160768) | 20260507 14:18:59.967 | INFO | MemInfo: { | 20260507 14:18:59.967 | INFO | "gpu" : " 17 GB", | 20260507 14:18:59.967 | INFO | "huge" : "", | 20260507 14:18:59.967 | INFO | "huge_shared" : " 29 GB", | 20260507 14:18:59.967 | INFO | "malloc" : " 14 GB", | 20260507 14:18:59.967 | INFO | "pinned" : " 29 GB" | 20260507 14:18:59.967 | INFO | } | 20260507 14:18:59.967 | INFO | Perf report for the last 5.13964 seconds: | 20260507 14:18:59.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:18:59.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.967 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:18:59.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.967 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:18:59.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:18:59.968 | INFO | Uploaded 36GiB to GPU in 1.60254s (22.4643GiB/s) | 20260507 14:18:59.968 | INFO | Downloaded 34GiB to GPU in 1.52146s (22.347GiB/s) | 20260507 14:19:05.106 | INFO | Analyzing chunk frames = [160768,161280) | 20260507 14:19:05.106 | INFO | MemInfo: { | 20260507 14:19:05.106 | INFO | "gpu" : " 17 GB", | 20260507 14:19:05.106 | INFO | "huge" : "", | 20260507 14:19:05.106 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:05.106 | INFO | "malloc" : " 14 GB", | 20260507 14:19:05.106 | INFO | "pinned" : " 29 GB" | 20260507 14:19:05.107 | INFO | } | 20260507 14:19:05.107 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:19:05.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:05.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:19:05.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:19:05.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:05.107 | INFO | Uploaded 36GiB to GPU in 1.60717s (22.3996GiB/s) | 20260507 14:19:05.107 | INFO | Downloaded 34GiB to GPU in 1.52757s (22.2576GiB/s) | 20260507 14:19:10.246 | INFO | Analyzing chunk frames = [161280,161792) | 20260507 14:19:10.246 | INFO | MemInfo: { | 20260507 14:19:10.246 | INFO | "gpu" : " 17 GB", | 20260507 14:19:10.246 | INFO | "huge" : "", | 20260507 14:19:10.246 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:10.246 | INFO | "malloc" : " 14 GB", | 20260507 14:19:10.246 | INFO | "pinned" : " 29 GB" | 20260507 14:19:10.246 | INFO | } | 20260507 14:19:10.246 | INFO | Perf report for the last 5.13909 seconds: | 20260507 14:19:10.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:10.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:19:10.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:19:10.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:10.246 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4128GiB/s) | 20260507 14:19:10.246 | INFO | Downloaded 34GiB to GPU in 1.52606s (22.2796GiB/s) | 20260507 14:19:15.385 | INFO | Analyzing chunk frames = [161792,162304) | 20260507 14:19:15.385 | INFO | MemInfo: { | 20260507 14:19:15.385 | INFO | "gpu" : " 17 GB", | 20260507 14:19:15.385 | INFO | "huge" : "", | 20260507 14:19:15.385 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:15.385 | INFO | "malloc" : " 14 GB", | 20260507 14:19:15.385 | INFO | "pinned" : " 29 GB" | 20260507 14:19:15.385 | INFO | } | 20260507 14:19:15.385 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:19:15.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:15.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.385 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:19:15.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.385 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:19:15.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:15.386 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4266GiB/s) | 20260507 14:19:15.386 | INFO | Downloaded 34GiB to GPU in 1.52443s (22.3034GiB/s) | 20260507 14:19:20.524 | INFO | Analyzing chunk frames = [162304,162816) | 20260507 14:19:20.524 | INFO | MemInfo: { | 20260507 14:19:20.524 | INFO | "gpu" : " 17 GB", | 20260507 14:19:20.524 | INFO | "huge" : "", | 20260507 14:19:20.524 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:20.524 | INFO | "malloc" : " 14 GB", | 20260507 14:19:20.524 | INFO | "pinned" : " 29 GB" | 20260507 14:19:20.524 | INFO | } | 20260507 14:19:20.524 | INFO | Perf report for the last 5.13912 seconds: | 20260507 14:19:20.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:20.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.524 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:19:20.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.524 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:19:20.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:20.525 | INFO | Uploaded 36GiB to GPU in 1.60482s (22.4324GiB/s) | 20260507 14:19:20.525 | INFO | Downloaded 34GiB to GPU in 1.52381s (22.3124GiB/s) | 20260507 14:19:25.664 | INFO | Analyzing chunk frames = [162816,163328) | 20260507 14:19:25.664 | INFO | MemInfo: { | 20260507 14:19:25.664 | INFO | "gpu" : " 17 GB", | 20260507 14:19:25.664 | INFO | "huge" : "", | 20260507 14:19:25.664 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:25.664 | INFO | "malloc" : " 14 GB", | 20260507 14:19:25.664 | INFO | "pinned" : " 29 GB" | 20260507 14:19:25.664 | INFO | } | 20260507 14:19:25.664 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:19:25.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:25.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:19:25.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:19:25.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:25.664 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4128GiB/s) | 20260507 14:19:25.664 | INFO | Downloaded 34GiB to GPU in 1.52575s (22.2841GiB/s) | 20260507 14:19:30.803 | INFO | Analyzing chunk frames = [163328,163840) | 20260507 14:19:30.803 | INFO | MemInfo: { | 20260507 14:19:30.803 | INFO | "gpu" : " 17 GB", | 20260507 14:19:30.803 | INFO | "huge" : "", | 20260507 14:19:30.803 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:30.803 | INFO | "malloc" : " 14 GB", | 20260507 14:19:30.803 | INFO | "pinned" : " 29 GB" | 20260507 14:19:30.803 | INFO | } | 20260507 14:19:30.803 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:19:30.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:30.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:19:30.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:19:30.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:30.803 | INFO | Uploaded 36GiB to GPU in 1.60556s (22.4221GiB/s) | 20260507 14:19:30.804 | INFO | Downloaded 34GiB to GPU in 1.52505s (22.2943GiB/s) | 20260507 14:19:30.804 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:19:30.804 | INFO | Total mem usage: 15.5 GiB | 20260507 14:19:30.804 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:19:30.804 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.99%) | 20260507 14:19:30.804 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:19:30.804 | INFO | High Water Breakdown: | 20260507 14:19:30.804 | INFO | 58.3% : 8.62 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:19:30.804 | INFO | 41.5% : 6.14 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:19:30.804 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:19:30.804 | INFO | ------------------------------------------ | 20260507 14:19:30.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:19:30.804 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:19:30.804 | INFO | Total mem usage: 26.5 GiB | 20260507 14:19:30.804 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:19:30.804 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:19:30.804 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:19:30.804 | INFO | High Water Breakdown: | 20260507 14:19:30.804 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:19:30.804 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:19:30.804 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:19:30.804 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:19:30.804 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:19:30.804 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:19:30.804 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:19:30.804 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:19:30.804 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:19:30.804 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:19:30.804 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:19:30.804 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:19:30.804 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:19:30.804 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:19:30.804 | INFO | ------------------------------------------ | 20260507 14:19:30.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:19:30.804 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:19:30.804 | INFO | Total mem usage: 27.8 GiB | 20260507 14:19:30.804 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:19:30.804 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:19:30.804 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:19:30.804 | INFO | High Water Breakdown: | 20260507 14:19:30.804 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:19:30.804 | INFO | ------------------------------------------ | 20260507 14:19:30.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:19:30.804 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:19:30.804 | INFO | Total mem usage: 15.7 GiB | 20260507 14:19:30.804 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:19:30.804 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:19:30.804 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:19:30.804 | INFO | High Water Breakdown: | 20260507 14:19:30.804 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:19:30.804 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:19:30.804 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:19:30.804 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:19:30.804 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:19:30.804 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:19:30.804 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:19:30.804 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:19:30.804 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:19:30.804 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:19:30.804 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:19:30.804 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:19:30.804 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:19:30.804 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:19:30.804 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:19:30.804 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:19:30.804 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:19:30.804 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:19:30.804 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:19:30.804 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:19:30.804 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:19:30.804 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:19:30.804 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:19:30.804 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:19:30.804 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:19:30.804 | INFO | ------------------------------------------ | 20260507 14:19:30.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:19:30.804 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:19:30.804 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:19:30.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:19:30.804 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 6.31e-03 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 16.08%, DME -- 3.87e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.22e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 3.69%, Metrics -- 1.11e+00 / -nan, 1.79e-01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.14e-01 / -nan, 6.11e-02 / 0.00e+00 | 20260507 14:19:30.804 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:19:30.804 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:19:30.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:19:30.804 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 3.97e-01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 17.88%, Download -- 1.61e+01 / 0.00e+00, 7.16e-01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 33.33%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 1.68e-04 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 29.63%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:19:30.804 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 9.71e-03 / 0.00e+00 | 20260507 14:19:30.900 | INFO | Starting a round of Baz IO | 20260507 14:19:33.799 | INFO | Populating RTMetrics report | 20260507 14:19:33.799 | INFO | RTMetrics report populated | 20260507 14:19:33.799 | INFO | Opening tmp RTMetrics json file | 20260507 14:19:33.930 | INFO | Serializing RTMetrics to json | 20260507 14:19:33.933 | INFO | Serialization done, swapping tmp file now... | 20260507 14:19:33.939 | INFO | RTMetrics done | 20260507 14:19:34.144 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.244 seconds | 20260507 14:19:34.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.41% / 0.14% / 39.45% | 20260507 14:19:34.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.63179% / 0.000480811% / 3.17891e-05% | 20260507 14:19:34.144 | INFO | Baz IO overall: | 20260507 14:19:34.145 | INFO | BytesWritten : 49733729798 | 20260507 14:19:34.145 | INFO | NumEvents : 22885777889 | 20260507 14:19:35.942 | INFO | Analyzing chunk frames = [163840,164352) | 20260507 14:19:35.942 | INFO | MemInfo: { | 20260507 14:19:35.942 | INFO | "gpu" : " 17 GB", | 20260507 14:19:35.942 | INFO | "huge" : "", | 20260507 14:19:35.942 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:35.942 | INFO | "malloc" : " 14 GB", | 20260507 14:19:35.942 | INFO | "pinned" : " 29 GB" | 20260507 14:19:35.942 | INFO | } | 20260507 14:19:35.942 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:19:35.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:35.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:35.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:35.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:35.943 | INFO | ANALYSIS : 57.4%, 2.9, 87.8ms, 0 | 20260507 14:19:35.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:35.943 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 14:19:35.943 | INFO | PRE_HQ : 56.3%, 2.1, 60.1ms, 0 | 20260507 14:19:35.943 | INFO | BAZWRITER : 62.7%, 4.4, 142.8ms, 0 | 20260507 14:19:35.943 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 14:19:35.943 | INFO | Uploaded 36GiB to GPU in 1.65585s (21.7411GiB/s) | 20260507 14:19:35.943 | INFO | Downloaded 38GiB to GPU in 1.7193s (22.102GiB/s) | 20260507 14:19:41.082 | INFO | Analyzing chunk frames = [164352,164864) | 20260507 14:19:41.082 | INFO | MemInfo: { | 20260507 14:19:41.082 | INFO | "gpu" : " 17 GB", | 20260507 14:19:41.082 | INFO | "huge" : "", | 20260507 14:19:41.082 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:41.082 | INFO | "malloc" : " 14 GB", | 20260507 14:19:41.082 | INFO | "pinned" : " 29 GB" | 20260507 14:19:41.082 | INFO | } | 20260507 14:19:41.082 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:19:41.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:41.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:19:41.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:19:41.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:41.082 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4603GiB/s) | 20260507 14:19:41.082 | INFO | Downloaded 34GiB to GPU in 1.52139s (22.348GiB/s) | 20260507 14:19:46.221 | INFO | Analyzing chunk frames = [164864,165376) | 20260507 14:19:46.221 | INFO | MemInfo: { | 20260507 14:19:46.221 | INFO | "gpu" : " 17 GB", | 20260507 14:19:46.221 | INFO | "huge" : "", | 20260507 14:19:46.221 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:46.221 | INFO | "malloc" : " 14 GB", | 20260507 14:19:46.221 | INFO | "pinned" : " 29 GB" | 20260507 14:19:46.221 | INFO | } | 20260507 14:19:46.221 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:19:46.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:46.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.221 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:19:46.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.221 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:19:46.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:46.222 | INFO | Uploaded 36GiB to GPU in 1.60759s (22.3938GiB/s) | 20260507 14:19:46.222 | INFO | Downloaded 34GiB to GPU in 1.52782s (22.254GiB/s) | 20260507 14:19:51.360 | INFO | Analyzing chunk frames = [165376,165888) | 20260507 14:19:51.360 | INFO | MemInfo: { | 20260507 14:19:51.360 | INFO | "gpu" : " 17 GB", | 20260507 14:19:51.360 | INFO | "huge" : "", | 20260507 14:19:51.360 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:51.360 | INFO | "malloc" : " 14 GB", | 20260507 14:19:51.360 | INFO | "pinned" : " 29 GB" | 20260507 14:19:51.360 | INFO | } | 20260507 14:19:51.360 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:19:51.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:51.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.361 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:19:51.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.361 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:19:51.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:51.361 | INFO | Uploaded 36GiB to GPU in 1.60534s (22.4251GiB/s) | 20260507 14:19:51.361 | INFO | Downloaded 34GiB to GPU in 1.52636s (22.2752GiB/s) | 20260507 14:19:56.500 | INFO | Analyzing chunk frames = [165888,166400) | 20260507 14:19:56.500 | INFO | MemInfo: { | 20260507 14:19:56.500 | INFO | "gpu" : " 17 GB", | 20260507 14:19:56.500 | INFO | "huge" : "", | 20260507 14:19:56.500 | INFO | "huge_shared" : " 29 GB", | 20260507 14:19:56.500 | INFO | "malloc" : " 14 GB", | 20260507 14:19:56.500 | INFO | "pinned" : " 29 GB" | 20260507 14:19:56.500 | INFO | } | 20260507 14:19:56.500 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:19:56.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:19:56.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:19:56.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:19:56.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:19:56.500 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.4349GiB/s) | 20260507 14:19:56.500 | INFO | Downloaded 34GiB to GPU in 1.52373s (22.3137GiB/s) | 20260507 14:20:01.639 | INFO | Analyzing chunk frames = [166400,166912) | 20260507 14:20:01.639 | INFO | MemInfo: { | 20260507 14:20:01.639 | INFO | "gpu" : " 17 GB", | 20260507 14:20:01.639 | INFO | "huge" : "", | 20260507 14:20:01.639 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:01.639 | INFO | "malloc" : " 14 GB", | 20260507 14:20:01.639 | INFO | "pinned" : " 29 GB" | 20260507 14:20:01.639 | INFO | } | 20260507 14:20:01.639 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:20:01.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:01.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.639 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:20:01.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.639 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:20:01.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:01.640 | INFO | Uploaded 36GiB to GPU in 1.60556s (22.422GiB/s) | 20260507 14:20:01.640 | INFO | Downloaded 34GiB to GPU in 1.52521s (22.292GiB/s) | 20260507 14:20:06.778 | INFO | Analyzing chunk frames = [166912,167424) | 20260507 14:20:06.778 | INFO | MemInfo: { | 20260507 14:20:06.778 | INFO | "gpu" : " 17 GB", | 20260507 14:20:06.778 | INFO | "huge" : "", | 20260507 14:20:06.778 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:06.778 | INFO | "malloc" : " 14 GB", | 20260507 14:20:06.778 | INFO | "pinned" : " 29 GB" | 20260507 14:20:06.778 | INFO | } | 20260507 14:20:06.779 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:20:06.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:06.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:20:06.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 14:20:06.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:06.779 | INFO | Uploaded 36GiB to GPU in 1.60469s (22.4343GiB/s) | 20260507 14:20:06.779 | INFO | Downloaded 34GiB to GPU in 1.52595s (22.2811GiB/s) | 20260507 14:20:11.918 | INFO | Analyzing chunk frames = [167424,167936) | 20260507 14:20:11.918 | INFO | MemInfo: { | 20260507 14:20:11.918 | INFO | "gpu" : " 17 GB", | 20260507 14:20:11.918 | INFO | "huge" : "", | 20260507 14:20:11.918 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:11.918 | INFO | "malloc" : " 14 GB", | 20260507 14:20:11.918 | INFO | "pinned" : " 29 GB" | 20260507 14:20:11.918 | INFO | } | 20260507 14:20:11.918 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:20:11.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:11.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:20:11.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 14:20:11.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:11.918 | INFO | Uploaded 36GiB to GPU in 1.60634s (22.4112GiB/s) | 20260507 14:20:11.919 | INFO | Downloaded 34GiB to GPU in 1.52576s (22.284GiB/s) | 20260507 14:20:12.015 | INFO | Starting a round of Baz IO | 20260507 14:20:14.917 | INFO | Populating RTMetrics report | 20260507 14:20:14.917 | INFO | RTMetrics report populated | 20260507 14:20:14.917 | INFO | Opening tmp RTMetrics json file | 20260507 14:20:14.967 | INFO | Serializing RTMetrics to json | 20260507 14:20:15.017 | INFO | Serialization done, swapping tmp file now... | 20260507 14:20:15.018 | INFO | RTMetrics done | 20260507 14:20:15.107 | INFO | Finished a round of Baz IO: Wrote 1.638 GiB in 3.092 seconds | 20260507 14:20:15.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.28% / 0.13% / 39.59% | 20260507 14:20:15.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.59238% / 0.000393391% / 2.38419e-05% | 20260507 14:20:15.107 | INFO | Baz IO overall: | 20260507 14:20:15.107 | INFO | BytesWritten : 51492964414 | 20260507 14:20:15.107 | INFO | NumEvents : 23745493990 | 20260507 14:20:17.057 | INFO | Analyzing chunk frames = [167936,168448) | 20260507 14:20:17.057 | INFO | MemInfo: { | 20260507 14:20:17.057 | INFO | "gpu" : " 17 GB", | 20260507 14:20:17.057 | INFO | "huge" : "", | 20260507 14:20:17.057 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:17.057 | INFO | "malloc" : " 14 GB", | 20260507 14:20:17.057 | INFO | "pinned" : " 29 GB" | 20260507 14:20:17.057 | INFO | } | 20260507 14:20:17.057 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:20:17.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:17.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:17.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:17.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:17.057 | INFO | ANALYSIS : 57.5%, 2.9, 88.0ms, 0 | 20260507 14:20:17.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:17.057 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 14:20:17.057 | INFO | PRE_HQ : 56.4%, 2.1, 60.6ms, 0 | 20260507 14:20:17.057 | INFO | BAZWRITER : 59.7%, 4.3, 130.9ms, 0 | 20260507 14:20:17.057 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 14:20:17.058 | INFO | Uploaded 36GiB to GPU in 1.65375s (21.7688GiB/s) | 20260507 14:20:17.058 | INFO | Downloaded 38GiB to GPU in 1.71958s (22.0985GiB/s) | 20260507 14:20:22.196 | INFO | Analyzing chunk frames = [168448,168960) | 20260507 14:20:22.196 | INFO | MemInfo: { | 20260507 14:20:22.196 | INFO | "gpu" : " 17 GB", | 20260507 14:20:22.196 | INFO | "huge" : "", | 20260507 14:20:22.196 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:22.196 | INFO | "malloc" : " 14 GB", | 20260507 14:20:22.197 | INFO | "pinned" : " 29 GB" | 20260507 14:20:22.197 | INFO | } | 20260507 14:20:22.197 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:20:22.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:22.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:20:22.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:20:22.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:22.197 | INFO | Uploaded 36GiB to GPU in 1.60335s (22.453GiB/s) | 20260507 14:20:22.197 | INFO | Downloaded 34GiB to GPU in 1.52196s (22.3396GiB/s) | 20260507 14:20:27.336 | INFO | Analyzing chunk frames = [168960,169472) | 20260507 14:20:27.336 | INFO | MemInfo: { | 20260507 14:20:27.336 | INFO | "gpu" : " 17 GB", | 20260507 14:20:27.336 | INFO | "huge" : "", | 20260507 14:20:27.336 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:27.336 | INFO | "malloc" : " 14 GB", | 20260507 14:20:27.336 | INFO | "pinned" : " 29 GB" | 20260507 14:20:27.336 | INFO | } | 20260507 14:20:27.336 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:20:27.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:27.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260507 14:20:27.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:20:27.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:27.336 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4086GiB/s) | 20260507 14:20:27.336 | INFO | Downloaded 34GiB to GPU in 1.52675s (22.2695GiB/s) | 20260507 14:20:32.475 | INFO | Analyzing chunk frames = [169472,169984) | 20260507 14:20:32.475 | INFO | MemInfo: { | 20260507 14:20:32.475 | INFO | "gpu" : " 17 GB", | 20260507 14:20:32.475 | INFO | "huge" : "", | 20260507 14:20:32.475 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:32.475 | INFO | "malloc" : " 14 GB", | 20260507 14:20:32.475 | INFO | "pinned" : " 29 GB" | 20260507 14:20:32.475 | INFO | } | 20260507 14:20:32.475 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:20:32.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:32.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.475 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 14:20:32.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.475 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:20:32.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:32.476 | INFO | Uploaded 36GiB to GPU in 1.60705s (22.4013GiB/s) | 20260507 14:20:32.476 | INFO | Downloaded 34GiB to GPU in 1.52719s (22.2631GiB/s) | 20260507 14:20:37.615 | INFO | Analyzing chunk frames = [169984,170496) | 20260507 14:20:37.615 | INFO | MemInfo: { | 20260507 14:20:37.615 | INFO | "gpu" : " 17 GB", | 20260507 14:20:37.615 | INFO | "huge" : "", | 20260507 14:20:37.615 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:37.615 | INFO | "malloc" : " 14 GB", | 20260507 14:20:37.615 | INFO | "pinned" : " 29 GB" | 20260507 14:20:37.615 | INFO | } | 20260507 14:20:37.615 | INFO | Perf report for the last 5.13957 seconds: | 20260507 14:20:37.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:37.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:20:37.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:20:37.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:37.615 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260507 14:20:37.615 | INFO | Downloaded 34GiB to GPU in 1.52461s (22.3008GiB/s) | 20260507 14:20:42.754 | INFO | Analyzing chunk frames = [170496,171008) | 20260507 14:20:42.754 | INFO | MemInfo: { | 20260507 14:20:42.754 | INFO | "gpu" : " 17 GB", | 20260507 14:20:42.754 | INFO | "huge" : "", | 20260507 14:20:42.754 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:42.754 | INFO | "malloc" : " 14 GB", | 20260507 14:20:42.754 | INFO | "pinned" : " 29 GB" | 20260507 14:20:42.754 | INFO | } | 20260507 14:20:42.754 | INFO | Perf report for the last 5.13913 seconds: | 20260507 14:20:42.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:42.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.754 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:20:42.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.754 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:20:42.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:42.755 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4255GiB/s) | 20260507 14:20:42.755 | INFO | Downloaded 34GiB to GPU in 1.52387s (22.3116GiB/s) | 20260507 14:20:47.893 | INFO | Analyzing chunk frames = [171008,171520) | 20260507 14:20:47.893 | INFO | MemInfo: { | 20260507 14:20:47.893 | INFO | "gpu" : " 17 GB", | 20260507 14:20:47.893 | INFO | "huge" : "", | 20260507 14:20:47.893 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:47.893 | INFO | "malloc" : " 14 GB", | 20260507 14:20:47.893 | INFO | "pinned" : " 29 GB" | 20260507 14:20:47.893 | INFO | } | 20260507 14:20:47.893 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:20:47.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:47.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.893 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260507 14:20:47.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.893 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:20:47.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:47.894 | INFO | Uploaded 36GiB to GPU in 1.60596s (22.4164GiB/s) | 20260507 14:20:47.894 | INFO | Downloaded 34GiB to GPU in 1.52534s (22.2901GiB/s) | 20260507 14:20:53.032 | INFO | Analyzing chunk frames = [171520,172032) | 20260507 14:20:53.032 | INFO | MemInfo: { | 20260507 14:20:53.032 | INFO | "gpu" : " 17 GB", | 20260507 14:20:53.032 | INFO | "huge" : "", | 20260507 14:20:53.032 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:53.032 | INFO | "malloc" : " 14 GB", | 20260507 14:20:53.033 | INFO | "pinned" : " 29 GB" | 20260507 14:20:53.033 | INFO | } | 20260507 14:20:53.033 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:20:53.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:53.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:20:53.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:20:53.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:53.033 | INFO | Uploaded 36GiB to GPU in 1.60664s (22.4071GiB/s) | 20260507 14:20:53.033 | INFO | Downloaded 34GiB to GPU in 1.52615s (22.2783GiB/s) | 20260507 14:20:53.033 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:20:53.033 | INFO | Total mem usage: 15.5 GiB | 20260507 14:20:53.033 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:20:53.033 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -0.178%) | 20260507 14:20:53.033 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:20:53.033 | INFO | High Water Breakdown: | 20260507 14:20:53.033 | INFO | 59.2% : 9.17 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:20:53.033 | INFO | 40.7% : 6.3 GiB ( 93.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:20:53.033 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:20:53.033 | INFO | ------------------------------------------ | 20260507 14:20:53.033 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:20:53.034 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:20:53.034 | INFO | Total mem usage: 26.5 GiB | 20260507 14:20:53.034 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:20:53.034 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:20:53.034 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:20:53.034 | INFO | High Water Breakdown: | 20260507 14:20:53.034 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:20:53.034 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:20:53.034 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:20:53.034 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:20:53.034 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:20:53.034 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:20:53.034 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:20:53.034 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:20:53.034 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:20:53.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:20:53.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:20:53.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:20:53.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:20:53.034 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:20:53.034 | INFO | ------------------------------------------ | 20260507 14:20:53.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:20:53.034 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:20:53.034 | INFO | Total mem usage: 27.8 GiB | 20260507 14:20:53.034 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:20:53.034 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:20:53.034 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:20:53.034 | INFO | High Water Breakdown: | 20260507 14:20:53.034 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:20:53.034 | INFO | ------------------------------------------ | 20260507 14:20:53.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:20:53.034 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:20:53.034 | INFO | Total mem usage: 15.7 GiB | 20260507 14:20:53.034 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:20:53.034 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:20:53.034 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:20:53.034 | INFO | High Water Breakdown: | 20260507 14:20:53.034 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:20:53.034 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:20:53.034 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:20:53.034 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:20:53.034 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:20:53.034 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:20:53.034 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:20:53.034 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:20:53.034 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:20:53.034 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:20:53.034 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:20:53.034 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:20:53.034 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:20:53.034 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:20:53.034 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:20:53.034 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:20:53.034 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:20:53.034 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:20:53.034 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:20:53.034 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:20:53.034 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:20:53.034 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:20:53.034 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:20:53.034 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:20:53.034 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:20:53.034 | INFO | ------------------------------------------ | 20260507 14:20:53.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:20:53.034 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:20:53.034 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:20:53.034 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:20:53.034 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 6.44e-03 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.95e-02 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 16.08%, DME -- 3.87e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.21e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 3.69%, Metrics -- 1.11e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260507 14:20:53.034 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.13e-01 / -nan, 6.11e-02 / 0.00e+00 | 20260507 14:20:53.034 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:20:53.034 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:20:53.034 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:20:53.035 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 18.64%, Upload -- 1.68e+01 / 0.00e+00, 4.37e-01 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 17.92%, Download -- 1.61e+01 / 0.00e+00, 7.66e-01 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 33.37%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 0.00%, LockDownload -- 4.38e-04 / 0.00e+00, 9.38e-05 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 29.50%, LockCompute -- 2.66e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:20:53.035 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 14:20:53.130 | INFO | Starting a round of Baz IO | 20260507 14:20:56.032 | INFO | Populating RTMetrics report | 20260507 14:20:56.032 | INFO | RTMetrics report populated | 20260507 14:20:56.032 | INFO | Opening tmp RTMetrics json file | 20260507 14:20:56.087 | INFO | Serializing RTMetrics to json | 20260507 14:20:56.103 | INFO | Serialization done, swapping tmp file now... | 20260507 14:20:56.104 | INFO | RTMetrics done | 20260507 14:20:56.127 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 2.997 seconds | 20260507 14:20:56.127 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.15% / 0.13% / 39.72% | 20260507 14:20:56.127 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.54346% / 0.000429153% / 7.94729e-06% | 20260507 14:20:56.127 | INFO | Baz IO overall: | 20260507 14:20:56.127 | INFO | BytesWritten : 53254523985 | 20260507 14:20:56.127 | INFO | NumEvents : 24606118353 | 20260507 14:20:58.172 | INFO | Analyzing chunk frames = [172032,172544) | 20260507 14:20:58.172 | INFO | MemInfo: { | 20260507 14:20:58.172 | INFO | "gpu" : " 17 GB", | 20260507 14:20:58.172 | INFO | "huge" : "", | 20260507 14:20:58.172 | INFO | "huge_shared" : " 29 GB", | 20260507 14:20:58.172 | INFO | "malloc" : " 14 GB", | 20260507 14:20:58.172 | INFO | "pinned" : " 29 GB" | 20260507 14:20:58.172 | INFO | } | 20260507 14:20:58.172 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:20:58.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:20:58.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:58.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:58.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:58.172 | INFO | ANALYSIS : 57.5%, 2.9, 87.7ms, 0 | 20260507 14:20:58.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:20:58.172 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 14:20:58.172 | INFO | PRE_HQ : 56.3%, 2.1, 60.3ms, 0 | 20260507 14:20:58.172 | INFO | BAZWRITER : 57.8%, 4.5, 133.4ms, 0 | 20260507 14:20:58.172 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:20:58.173 | INFO | Uploaded 36GiB to GPU in 1.65824s (21.7098GiB/s) | 20260507 14:20:58.173 | INFO | Downloaded 38GiB to GPU in 1.72303s (22.0541GiB/s) | 20260507 14:21:03.311 | INFO | Analyzing chunk frames = [172544,173056) | 20260507 14:21:03.311 | INFO | MemInfo: { | 20260507 14:21:03.311 | INFO | "gpu" : " 17 GB", | 20260507 14:21:03.311 | INFO | "huge" : "", | 20260507 14:21:03.311 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:03.311 | INFO | "malloc" : " 14 GB", | 20260507 14:21:03.311 | INFO | "pinned" : " 29 GB" | 20260507 14:21:03.311 | INFO | } | 20260507 14:21:03.311 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:21:03.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:03.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.312 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:21:03.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.312 | INFO | PRE_HQ : 28.6%, 1.0, 15.0ms, 0 | 20260507 14:21:03.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:03.312 | INFO | Uploaded 36GiB to GPU in 1.60361s (22.4494GiB/s) | 20260507 14:21:03.312 | INFO | Downloaded 34GiB to GPU in 1.52221s (22.336GiB/s) | 20260507 14:21:08.451 | INFO | Analyzing chunk frames = [173056,173568) | 20260507 14:21:08.451 | INFO | MemInfo: { | 20260507 14:21:08.451 | INFO | "gpu" : " 17 GB", | 20260507 14:21:08.451 | INFO | "huge" : "", | 20260507 14:21:08.451 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:08.451 | INFO | "malloc" : " 14 GB", | 20260507 14:21:08.451 | INFO | "pinned" : " 29 GB" | 20260507 14:21:08.451 | INFO | } | 20260507 14:21:08.451 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:21:08.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:08.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:21:08.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:21:08.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:08.451 | INFO | Uploaded 36GiB to GPU in 1.60766s (22.3928GiB/s) | 20260507 14:21:08.451 | INFO | Downloaded 34GiB to GPU in 1.52708s (22.2647GiB/s) | 20260507 14:21:13.590 | INFO | Analyzing chunk frames = [173568,174080) | 20260507 14:21:13.590 | INFO | MemInfo: { | 20260507 14:21:13.590 | INFO | "gpu" : " 17 GB", | 20260507 14:21:13.591 | INFO | "huge" : "", | 20260507 14:21:13.591 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:13.591 | INFO | "malloc" : " 14 GB", | 20260507 14:21:13.591 | INFO | "pinned" : " 29 GB" | 20260507 14:21:13.591 | INFO | } | 20260507 14:21:13.591 | INFO | Perf report for the last 5.13976 seconds: | 20260507 14:21:13.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:13.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:21:13.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:21:13.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:13.591 | INFO | Uploaded 36GiB to GPU in 1.60699s (22.4021GiB/s) | 20260507 14:21:13.591 | INFO | Downloaded 34GiB to GPU in 1.5269s (22.2673GiB/s) | 20260507 14:21:18.729 | INFO | Analyzing chunk frames = [174080,174592) | 20260507 14:21:18.729 | INFO | MemInfo: { | 20260507 14:21:18.729 | INFO | "gpu" : " 17 GB", | 20260507 14:21:18.729 | INFO | "huge" : "", | 20260507 14:21:18.729 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:18.729 | INFO | "malloc" : " 14 GB", | 20260507 14:21:18.729 | INFO | "pinned" : " 29 GB" | 20260507 14:21:18.729 | INFO | } | 20260507 14:21:18.729 | INFO | Perf report for the last 5.13876 seconds: | 20260507 14:21:18.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:18.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.729 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:21:18.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.730 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:21:18.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:18.730 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.425GiB/s) | 20260507 14:21:18.730 | INFO | Downloaded 34GiB to GPU in 1.52431s (22.3052GiB/s) | 20260507 14:21:23.869 | INFO | Analyzing chunk frames = [174592,175104) | 20260507 14:21:23.869 | INFO | MemInfo: { | 20260507 14:21:23.869 | INFO | "gpu" : " 17 GB", | 20260507 14:21:23.869 | INFO | "huge" : "", | 20260507 14:21:23.869 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:23.869 | INFO | "malloc" : " 14 GB", | 20260507 14:21:23.869 | INFO | "pinned" : " 29 GB" | 20260507 14:21:23.869 | INFO | } | 20260507 14:21:23.869 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:21:23.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:23.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:21:23.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:21:23.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:23.869 | INFO | Uploaded 36GiB to GPU in 1.60562s (22.4213GiB/s) | 20260507 14:21:23.869 | INFO | Downloaded 34GiB to GPU in 1.5241s (22.3082GiB/s) | 20260507 14:21:29.008 | INFO | Analyzing chunk frames = [175104,175616) | 20260507 14:21:29.008 | INFO | MemInfo: { | 20260507 14:21:29.008 | INFO | "gpu" : " 17 GB", | 20260507 14:21:29.008 | INFO | "huge" : "", | 20260507 14:21:29.008 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:29.008 | INFO | "malloc" : " 14 GB", | 20260507 14:21:29.008 | INFO | "pinned" : " 29 GB" | 20260507 14:21:29.008 | INFO | } | 20260507 14:21:29.008 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:21:29.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:29.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.008 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:21:29.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.008 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:21:29.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:29.009 | INFO | Uploaded 36GiB to GPU in 1.60542s (22.424GiB/s) | 20260507 14:21:29.009 | INFO | Downloaded 34GiB to GPU in 1.52483s (22.2975GiB/s) | 20260507 14:21:34.147 | INFO | Analyzing chunk frames = [175616,176128) | 20260507 14:21:34.147 | INFO | MemInfo: { | 20260507 14:21:34.147 | INFO | "gpu" : " 17 GB", | 20260507 14:21:34.147 | INFO | "huge" : "", | 20260507 14:21:34.147 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:34.147 | INFO | "malloc" : " 14 GB", | 20260507 14:21:34.147 | INFO | "pinned" : " 29 GB" | 20260507 14:21:34.147 | INFO | } | 20260507 14:21:34.147 | INFO | Perf report for the last 5.13914 seconds: | 20260507 14:21:34.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:34.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.147 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:21:34.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.147 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:21:34.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:34.148 | INFO | Uploaded 36GiB to GPU in 1.60641s (22.4102GiB/s) | 20260507 14:21:34.148 | INFO | Downloaded 34GiB to GPU in 1.52646s (22.2737GiB/s) | 20260507 14:21:34.244 | INFO | Starting a round of Baz IO | 20260507 14:21:37.146 | INFO | Populating RTMetrics report | 20260507 14:21:37.146 | INFO | RTMetrics report populated | 20260507 14:21:37.146 | INFO | Opening tmp RTMetrics json file | 20260507 14:21:37.203 | INFO | Serializing RTMetrics to json | 20260507 14:21:37.206 | INFO | Serialization done, swapping tmp file now... | 20260507 14:21:37.208 | INFO | RTMetrics done | 20260507 14:21:37.396 | INFO | Finished a round of Baz IO: Wrote 1.642 GiB in 3.152 seconds | 20260507 14:21:37.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.02% / 0.13% / 39.85% | 20260507 14:21:37.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.49948% / 0.00042518% / 1.58946e-05% | 20260507 14:21:37.396 | INFO | Baz IO overall: | 20260507 14:21:37.396 | INFO | BytesWritten : 55017542832 | 20260507 14:21:37.396 | INFO | NumEvents : 25467622089 | 20260507 14:21:39.287 | INFO | Analyzing chunk frames = [176128,176640) | 20260507 14:21:39.287 | INFO | MemInfo: { | 20260507 14:21:39.287 | INFO | "gpu" : " 17 GB", | 20260507 14:21:39.287 | INFO | "huge" : "", | 20260507 14:21:39.287 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:39.287 | INFO | "malloc" : " 14 GB", | 20260507 14:21:39.287 | INFO | "pinned" : " 29 GB" | 20260507 14:21:39.287 | INFO | } | 20260507 14:21:39.287 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:21:39.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:39.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:39.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:39.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:39.287 | INFO | ANALYSIS : 57.5%, 3.0, 89.1ms, 0 | 20260507 14:21:39.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:39.287 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 14:21:39.287 | INFO | PRE_HQ : 56.1%, 2.1, 60.3ms, 0 | 20260507 14:21:39.287 | INFO | BAZWRITER : 60.9%, 4.2, 130.4ms, 0 | 20260507 14:21:39.287 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:21:39.287 | INFO | Uploaded 36GiB to GPU in 1.64392s (21.8989GiB/s) | 20260507 14:21:39.288 | INFO | Downloaded 38GiB to GPU in 1.70836s (22.2436GiB/s) | 20260507 14:21:44.426 | INFO | Analyzing chunk frames = [176640,177152) | 20260507 14:21:44.426 | INFO | MemInfo: { | 20260507 14:21:44.426 | INFO | "gpu" : " 17 GB", | 20260507 14:21:44.426 | INFO | "huge" : "", | 20260507 14:21:44.426 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:44.426 | INFO | "malloc" : " 14 GB", | 20260507 14:21:44.426 | INFO | "pinned" : " 29 GB" | 20260507 14:21:44.426 | INFO | } | 20260507 14:21:44.426 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:21:44.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:44.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.426 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:21:44.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.426 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:21:44.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:44.427 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4557GiB/s) | 20260507 14:21:44.427 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3369GiB/s) | 20260507 14:21:49.565 | INFO | Analyzing chunk frames = [177152,177664) | 20260507 14:21:49.565 | INFO | MemInfo: { | 20260507 14:21:49.565 | INFO | "gpu" : " 17 GB", | 20260507 14:21:49.565 | INFO | "huge" : "", | 20260507 14:21:49.565 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:49.565 | INFO | "malloc" : " 14 GB", | 20260507 14:21:49.565 | INFO | "pinned" : " 29 GB" | 20260507 14:21:49.565 | INFO | } | 20260507 14:21:49.565 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:21:49.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:49.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:21:49.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:21:49.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:49.566 | INFO | Uploaded 36GiB to GPU in 1.60731s (22.3977GiB/s) | 20260507 14:21:49.566 | INFO | Downloaded 34GiB to GPU in 1.52636s (22.2752GiB/s) | 20260507 14:21:54.705 | INFO | Analyzing chunk frames = [177664,178176) | 20260507 14:21:54.705 | INFO | MemInfo: { | 20260507 14:21:54.705 | INFO | "gpu" : " 17 GB", | 20260507 14:21:54.705 | INFO | "huge" : "", | 20260507 14:21:54.705 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:54.705 | INFO | "malloc" : " 14 GB", | 20260507 14:21:54.705 | INFO | "pinned" : " 29 GB" | 20260507 14:21:54.705 | INFO | } | 20260507 14:21:54.705 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:21:54.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:54.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:21:54.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:21:54.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:54.705 | INFO | Uploaded 36GiB to GPU in 1.6069s (22.4034GiB/s) | 20260507 14:21:54.705 | INFO | Downloaded 34GiB to GPU in 1.5268s (22.2688GiB/s) | 20260507 14:21:59.844 | INFO | Analyzing chunk frames = [178176,178688) | 20260507 14:21:59.844 | INFO | MemInfo: { | 20260507 14:21:59.844 | INFO | "gpu" : " 17 GB", | 20260507 14:21:59.844 | INFO | "huge" : "", | 20260507 14:21:59.844 | INFO | "huge_shared" : " 29 GB", | 20260507 14:21:59.844 | INFO | "malloc" : " 14 GB", | 20260507 14:21:59.844 | INFO | "pinned" : " 29 GB" | 20260507 14:21:59.844 | INFO | } | 20260507 14:21:59.844 | INFO | Perf report for the last 5.13914 seconds: | 20260507 14:21:59.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:21:59.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:21:59.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:21:59.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:21:59.844 | INFO | Uploaded 36GiB to GPU in 1.60533s (22.4252GiB/s) | 20260507 14:21:59.844 | INFO | Downloaded 34GiB to GPU in 1.52457s (22.3013GiB/s) | 20260507 14:22:04.983 | INFO | Analyzing chunk frames = [178688,179200) | 20260507 14:22:04.983 | INFO | MemInfo: { | 20260507 14:22:04.983 | INFO | "gpu" : " 17 GB", | 20260507 14:22:04.983 | INFO | "huge" : "", | 20260507 14:22:04.983 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:04.983 | INFO | "malloc" : " 14 GB", | 20260507 14:22:04.983 | INFO | "pinned" : " 29 GB" | 20260507 14:22:04.983 | INFO | } | 20260507 14:22:04.983 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:22:04.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:04.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.983 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:22:04.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.984 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:22:04.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:04.984 | INFO | Uploaded 36GiB to GPU in 1.60517s (22.4276GiB/s) | 20260507 14:22:04.984 | INFO | Downloaded 34GiB to GPU in 1.52366s (22.3147GiB/s) | 20260507 14:22:10.123 | INFO | Analyzing chunk frames = [179200,179712) | 20260507 14:22:10.123 | INFO | MemInfo: { | 20260507 14:22:10.123 | INFO | "gpu" : " 17 GB", | 20260507 14:22:10.123 | INFO | "huge" : "", | 20260507 14:22:10.123 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:10.123 | INFO | "malloc" : " 14 GB", | 20260507 14:22:10.123 | INFO | "pinned" : " 29 GB" | 20260507 14:22:10.123 | INFO | } | 20260507 14:22:10.123 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:22:10.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:10.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.123 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:22:10.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.123 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:22:10.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:10.124 | INFO | Uploaded 36GiB to GPU in 1.60644s (22.4098GiB/s) | 20260507 14:22:10.124 | INFO | Downloaded 34GiB to GPU in 1.52637s (22.2751GiB/s) | 20260507 14:22:15.262 | INFO | Analyzing chunk frames = [179712,180224) | 20260507 14:22:15.262 | INFO | MemInfo: { | 20260507 14:22:15.262 | INFO | "gpu" : " 17 GB", | 20260507 14:22:15.262 | INFO | "huge" : "", | 20260507 14:22:15.262 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:15.262 | INFO | "malloc" : " 14 GB", | 20260507 14:22:15.262 | INFO | "pinned" : " 29 GB" | 20260507 14:22:15.262 | INFO | } | 20260507 14:22:15.262 | INFO | Perf report for the last 5.13914 seconds: | 20260507 14:22:15.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:15.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.262 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:22:15.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.262 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:22:15.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:15.263 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:22:15.263 | INFO | Total mem usage: 15.5 GiB | 20260507 14:22:15.263 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:22:15.263 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -2.35%) | 20260507 14:22:15.263 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:22:15.263 | INFO | High Water Breakdown: | 20260507 14:22:15.263 | INFO | 58.9% : 8.92 GiB ( 97.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:22:15.263 | INFO | 41% : 6.22 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:22:15.263 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:22:15.263 | INFO | ------------------------------------------ | 20260507 14:22:15.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:22:15.263 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:22:15.263 | INFO | Total mem usage: 26.5 GiB | 20260507 14:22:15.263 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:22:15.263 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:22:15.263 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:22:15.263 | INFO | High Water Breakdown: | 20260507 14:22:15.263 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:22:15.263 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:22:15.263 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:22:15.263 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:22:15.263 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:22:15.263 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:22:15.263 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:22:15.263 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:22:15.263 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:22:15.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:22:15.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:22:15.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:22:15.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:22:15.263 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:22:15.263 | INFO | ------------------------------------------ | 20260507 14:22:15.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:22:15.263 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:22:15.263 | INFO | Total mem usage: 27.8 GiB | 20260507 14:22:15.263 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:22:15.263 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:22:15.263 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:22:15.263 | INFO | High Water Breakdown: | 20260507 14:22:15.263 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:22:15.263 | INFO | ------------------------------------------ | 20260507 14:22:15.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:22:15.263 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:22:15.263 | INFO | Total mem usage: 15.7 GiB | 20260507 14:22:15.263 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:22:15.263 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:22:15.263 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:22:15.263 | INFO | High Water Breakdown: | 20260507 14:22:15.263 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:22:15.263 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:22:15.263 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:22:15.263 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:22:15.263 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:22:15.263 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:22:15.263 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:22:15.263 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:22:15.263 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:22:15.263 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:22:15.263 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:22:15.263 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:22:15.263 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:22:15.263 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:22:15.263 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:22:15.263 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:22:15.263 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:22:15.263 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:22:15.263 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:22:15.263 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:22:15.263 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:22:15.263 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:22:15.263 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:22:15.263 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:22:15.263 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:22:15.263 | INFO | ------------------------------------------ | 20260507 14:22:15.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:22:15.263 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:22:15.263 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:22:15.263 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:22:15.263 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 0.29%, AggregateStats -- 8.60e-02 / -nan, 4.01e-03 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.77e-02 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 16.10%, DME -- 3.87e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 3.68%, Metrics -- 1.11e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260507 14:22:15.263 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.12e-01 / -nan, 6.04e-02 / 0.00e+00 | 20260507 14:22:15.263 | INFO | Uploaded 36GiB to GPU in 1.606s (22.4159GiB/s) | 20260507 14:22:15.263 | INFO | Downloaded 34GiB to GPU in 1.52613s (22.2786GiB/s) | 20260507 14:22:15.263 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:22:15.264 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:22:15.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:22:15.264 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 18.62%, Upload -- 1.68e+01 / 0.00e+00, 4.13e-01 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 17.90%, Download -- 1.61e+01 / 0.00e+00, 7.46e-01 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 33.35%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 0.56%, LockUpload -- 5.03e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 0.00%, LockDownload -- 4.41e-04 / 0.00e+00, 9.71e-05 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 29.55%, LockCompute -- 2.66e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:22:15.264 | INFO | PerfReport: 0.02%, Misc -- 1.36e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260507 14:22:15.360 | INFO | Starting a round of Baz IO | 20260507 14:22:18.263 | INFO | Populating RTMetrics report | 20260507 14:22:18.263 | INFO | RTMetrics report populated | 20260507 14:22:18.263 | INFO | Opening tmp RTMetrics json file | 20260507 14:22:18.269 | INFO | Serializing RTMetrics to json | 20260507 14:22:18.271 | INFO | Serialization done, swapping tmp file now... | 20260507 14:22:18.272 | INFO | RTMetrics done | 20260507 14:22:18.356 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 2.996 seconds | 20260507 14:22:18.356 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.90% / 0.12% / 39.98% | 20260507 14:22:18.356 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.44415% / 0.000421206% / 1.58946e-05% | 20260507 14:22:18.356 | INFO | Baz IO overall: | 20260507 14:22:18.356 | INFO | BytesWritten : 56781482109 | 20260507 14:22:18.356 | INFO | NumEvents : 26329635740 | 20260507 14:22:20.402 | INFO | Analyzing chunk frames = [180224,180736) | 20260507 14:22:20.402 | INFO | MemInfo: { | 20260507 14:22:20.402 | INFO | "gpu" : " 17 GB", | 20260507 14:22:20.402 | INFO | "huge" : "", | 20260507 14:22:20.402 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:20.402 | INFO | "malloc" : " 14 GB", | 20260507 14:22:20.402 | INFO | "pinned" : " 29 GB" | 20260507 14:22:20.402 | INFO | } | 20260507 14:22:20.402 | INFO | Perf report for the last 5.13966 seconds: | 20260507 14:22:20.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:20.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:20.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:20.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:20.402 | INFO | ANALYSIS : 57.4%, 2.9, 88.5ms, 0 | 20260507 14:22:20.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:20.402 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 14:22:20.402 | INFO | PRE_HQ : 56.3%, 2.1, 60.4ms, 0 | 20260507 14:22:20.402 | INFO | BAZWRITER : 57.9%, 4.3, 126.5ms, 0 | 20260507 14:22:20.402 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 14:22:20.402 | INFO | Uploaded 36GiB to GPU in 1.6469s (21.8593GiB/s) | 20260507 14:22:20.402 | INFO | Downloaded 38GiB to GPU in 1.7099s (22.2236GiB/s) | 20260507 14:22:25.541 | INFO | Analyzing chunk frames = [180736,181248) | 20260507 14:22:25.541 | INFO | MemInfo: { | 20260507 14:22:25.541 | INFO | "gpu" : " 17 GB", | 20260507 14:22:25.541 | INFO | "huge" : "", | 20260507 14:22:25.541 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:25.541 | INFO | "malloc" : " 14 GB", | 20260507 14:22:25.541 | INFO | "pinned" : " 29 GB" | 20260507 14:22:25.541 | INFO | } | 20260507 14:22:25.541 | INFO | Perf report for the last 5.13907 seconds: | 20260507 14:22:25.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:25.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:22:25.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:22:25.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:25.541 | INFO | Uploaded 36GiB to GPU in 1.60353s (22.4504GiB/s) | 20260507 14:22:25.541 | INFO | Downloaded 34GiB to GPU in 1.52298s (22.3247GiB/s) | 20260507 14:22:30.680 | INFO | Analyzing chunk frames = [181248,181760) | 20260507 14:22:30.680 | INFO | MemInfo: { | 20260507 14:22:30.680 | INFO | "gpu" : " 17 GB", | 20260507 14:22:30.680 | INFO | "huge" : "", | 20260507 14:22:30.680 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:30.680 | INFO | "malloc" : " 14 GB", | 20260507 14:22:30.680 | INFO | "pinned" : " 29 GB" | 20260507 14:22:30.680 | INFO | } | 20260507 14:22:30.680 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:22:30.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:30.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.680 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:22:30.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.680 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:22:30.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:30.681 | INFO | Uploaded 36GiB to GPU in 1.60755s (22.3943GiB/s) | 20260507 14:22:30.681 | INFO | Downloaded 34GiB to GPU in 1.52745s (22.2593GiB/s) | 20260507 14:22:35.819 | INFO | Analyzing chunk frames = [181760,182272) | 20260507 14:22:35.819 | INFO | MemInfo: { | 20260507 14:22:35.819 | INFO | "gpu" : " 17 GB", | 20260507 14:22:35.820 | INFO | "huge" : "", | 20260507 14:22:35.820 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:35.820 | INFO | "malloc" : " 14 GB", | 20260507 14:22:35.820 | INFO | "pinned" : " 29 GB" | 20260507 14:22:35.820 | INFO | } | 20260507 14:22:35.820 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:22:35.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:35.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:22:35.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:22:35.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:35.820 | INFO | Uploaded 36GiB to GPU in 1.60719s (22.3994GiB/s) | 20260507 14:22:35.820 | INFO | Downloaded 34GiB to GPU in 1.52739s (22.2602GiB/s) | 20260507 14:22:40.959 | INFO | Analyzing chunk frames = [182272,182784) | 20260507 14:22:40.959 | INFO | MemInfo: { | 20260507 14:22:40.959 | INFO | "gpu" : " 17 GB", | 20260507 14:22:40.959 | INFO | "huge" : "", | 20260507 14:22:40.959 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:40.959 | INFO | "malloc" : " 14 GB", | 20260507 14:22:40.959 | INFO | "pinned" : " 29 GB" | 20260507 14:22:40.959 | INFO | } | 20260507 14:22:40.959 | INFO | Perf report for the last 5.13912 seconds: | 20260507 14:22:40.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:40.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:22:40.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 14:22:40.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:40.959 | INFO | Uploaded 36GiB to GPU in 1.60647s (22.4094GiB/s) | 20260507 14:22:40.960 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.2889GiB/s) | 20260507 14:22:46.098 | INFO | Analyzing chunk frames = [182784,183296) | 20260507 14:22:46.098 | INFO | MemInfo: { | 20260507 14:22:46.098 | INFO | "gpu" : " 17 GB", | 20260507 14:22:46.098 | INFO | "huge" : "", | 20260507 14:22:46.098 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:46.098 | INFO | "malloc" : " 14 GB", | 20260507 14:22:46.098 | INFO | "pinned" : " 29 GB" | 20260507 14:22:46.098 | INFO | } | 20260507 14:22:46.098 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:22:46.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:46.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.098 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:22:46.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.098 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:22:46.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:46.099 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4215GiB/s) | 20260507 14:22:46.099 | INFO | Downloaded 34GiB to GPU in 1.52409s (22.3085GiB/s) | 20260507 14:22:51.237 | INFO | Analyzing chunk frames = [183296,183808) | 20260507 14:22:51.237 | INFO | MemInfo: { | 20260507 14:22:51.237 | INFO | "gpu" : " 17 GB", | 20260507 14:22:51.237 | INFO | "huge" : "", | 20260507 14:22:51.237 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:51.237 | INFO | "malloc" : " 14 GB", | 20260507 14:22:51.237 | INFO | "pinned" : " 29 GB" | 20260507 14:22:51.237 | INFO | } | 20260507 14:22:51.237 | INFO | Perf report for the last 5.13905 seconds: | 20260507 14:22:51.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:51.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:22:51.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:22:51.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:51.238 | INFO | Uploaded 36GiB to GPU in 1.60527s (22.4261GiB/s) | 20260507 14:22:51.238 | INFO | Downloaded 34GiB to GPU in 1.52599s (22.2806GiB/s) | 20260507 14:22:56.377 | INFO | Analyzing chunk frames = [183808,184320) | 20260507 14:22:56.377 | INFO | MemInfo: { | 20260507 14:22:56.377 | INFO | "gpu" : " 17 GB", | 20260507 14:22:56.377 | INFO | "huge" : "", | 20260507 14:22:56.377 | INFO | "huge_shared" : " 29 GB", | 20260507 14:22:56.377 | INFO | "malloc" : " 14 GB", | 20260507 14:22:56.377 | INFO | "pinned" : " 29 GB" | 20260507 14:22:56.377 | INFO | } | 20260507 14:22:56.377 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:22:56.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:22:56.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.377 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:22:56.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.377 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:22:56.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:22:56.378 | INFO | Uploaded 36GiB to GPU in 1.60605s (22.4153GiB/s) | 20260507 14:22:56.378 | INFO | Downloaded 34GiB to GPU in 1.52532s (22.2904GiB/s) | 20260507 14:22:56.474 | INFO | Starting a round of Baz IO | 20260507 14:22:59.374 | INFO | Populating RTMetrics report | 20260507 14:22:59.374 | INFO | RTMetrics report populated | 20260507 14:22:59.374 | INFO | Opening tmp RTMetrics json file | 20260507 14:22:59.375 | INFO | Serializing RTMetrics to json | 20260507 14:22:59.378 | INFO | Serialization done, swapping tmp file now... | 20260507 14:22:59.390 | INFO | RTMetrics done | 20260507 14:22:59.481 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.007 seconds | 20260507 14:22:59.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.78% / 0.12% / 40.10% | 20260507 14:22:59.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.39462% / 0.000373522% / 1.58946e-05% | 20260507 14:22:59.481 | INFO | Baz IO overall: | 20260507 14:22:59.481 | INFO | BytesWritten : 58547161385 | 20260507 14:22:59.481 | INFO | NumEvents : 27192214683 | 20260507 14:23:01.516 | INFO | Analyzing chunk frames = [184320,184832) | 20260507 14:23:01.516 | INFO | MemInfo: { | 20260507 14:23:01.516 | INFO | "gpu" : " 17 GB", | 20260507 14:23:01.516 | INFO | "huge" : "", | 20260507 14:23:01.516 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:01.516 | INFO | "malloc" : " 14 GB", | 20260507 14:23:01.516 | INFO | "pinned" : " 29 GB" | 20260507 14:23:01.516 | INFO | } | 20260507 14:23:01.516 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:23:01.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:01.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:01.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:01.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:01.516 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260507 14:23:01.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:01.516 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260507 14:23:01.516 | INFO | PRE_HQ : 56.1%, 2.1, 60.6ms, 0 | 20260507 14:23:01.516 | INFO | BAZWRITER : 58.1%, 4.0, 119.0ms, 0 | 20260507 14:23:01.516 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 14:23:01.517 | INFO | Uploaded 36GiB to GPU in 1.64131s (21.9337GiB/s) | 20260507 14:23:01.517 | INFO | Downloaded 38GiB to GPU in 1.70407s (22.2996GiB/s) | 20260507 14:23:06.655 | INFO | Analyzing chunk frames = [184832,185344) | 20260507 14:23:06.655 | INFO | MemInfo: { | 20260507 14:23:06.655 | INFO | "gpu" : " 17 GB", | 20260507 14:23:06.655 | INFO | "huge" : "", | 20260507 14:23:06.655 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:06.655 | INFO | "malloc" : " 14 GB", | 20260507 14:23:06.656 | INFO | "pinned" : " 29 GB" | 20260507 14:23:06.656 | INFO | } | 20260507 14:23:06.656 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:23:06.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:06.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:23:06.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:23:06.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:06.656 | INFO | Uploaded 36GiB to GPU in 1.6029s (22.4593GiB/s) | 20260507 14:23:06.656 | INFO | Downloaded 34GiB to GPU in 1.52137s (22.3482GiB/s) | 20260507 14:23:11.795 | INFO | Analyzing chunk frames = [185344,185856) | 20260507 14:23:11.795 | INFO | MemInfo: { | 20260507 14:23:11.795 | INFO | "gpu" : " 17 GB", | 20260507 14:23:11.795 | INFO | "huge" : "", | 20260507 14:23:11.795 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:11.795 | INFO | "malloc" : " 14 GB", | 20260507 14:23:11.795 | INFO | "pinned" : " 29 GB" | 20260507 14:23:11.795 | INFO | } | 20260507 14:23:11.795 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:23:11.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:11.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:23:11.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:23:11.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:11.795 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260507 14:23:11.795 | INFO | Downloaded 34GiB to GPU in 1.52779s (22.2543GiB/s) | 20260507 14:23:16.934 | INFO | Analyzing chunk frames = [185856,186368) | 20260507 14:23:16.934 | INFO | MemInfo: { | 20260507 14:23:16.934 | INFO | "gpu" : " 17 GB", | 20260507 14:23:16.934 | INFO | "huge" : "", | 20260507 14:23:16.934 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:16.934 | INFO | "malloc" : " 14 GB", | 20260507 14:23:16.934 | INFO | "pinned" : " 29 GB" | 20260507 14:23:16.934 | INFO | } | 20260507 14:23:16.934 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:23:16.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:16.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.934 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:23:16.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.934 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:23:16.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:16.935 | INFO | Uploaded 36GiB to GPU in 1.60719s (22.3993GiB/s) | 20260507 14:23:16.935 | INFO | Downloaded 34GiB to GPU in 1.52738s (22.2604GiB/s) | 20260507 14:23:22.073 | INFO | Analyzing chunk frames = [186368,186880) | 20260507 14:23:22.073 | INFO | MemInfo: { | 20260507 14:23:22.073 | INFO | "gpu" : " 17 GB", | 20260507 14:23:22.073 | INFO | "huge" : "", | 20260507 14:23:22.073 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:22.073 | INFO | "malloc" : " 14 GB", | 20260507 14:23:22.074 | INFO | "pinned" : " 29 GB" | 20260507 14:23:22.074 | INFO | } | 20260507 14:23:22.074 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:23:22.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:22.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:23:22.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:23:22.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:22.074 | INFO | Uploaded 36GiB to GPU in 1.60578s (22.419GiB/s) | 20260507 14:23:22.074 | INFO | Downloaded 34GiB to GPU in 1.52453s (22.302GiB/s) | 20260507 14:23:27.213 | INFO | Analyzing chunk frames = [186880,187392) | 20260507 14:23:27.213 | INFO | MemInfo: { | 20260507 14:23:27.213 | INFO | "gpu" : " 17 GB", | 20260507 14:23:27.213 | INFO | "huge" : "", | 20260507 14:23:27.213 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:27.213 | INFO | "malloc" : " 14 GB", | 20260507 14:23:27.213 | INFO | "pinned" : " 29 GB" | 20260507 14:23:27.213 | INFO | } | 20260507 14:23:27.213 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:23:27.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:27.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:23:27.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:23:27.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:27.213 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4274GiB/s) | 20260507 14:23:27.214 | INFO | Downloaded 34GiB to GPU in 1.52491s (22.2965GiB/s) | 20260507 14:23:32.352 | INFO | Analyzing chunk frames = [187392,187904) | 20260507 14:23:32.352 | INFO | MemInfo: { | 20260507 14:23:32.352 | INFO | "gpu" : " 17 GB", | 20260507 14:23:32.352 | INFO | "huge" : "", | 20260507 14:23:32.352 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:32.352 | INFO | "malloc" : " 14 GB", | 20260507 14:23:32.352 | INFO | "pinned" : " 29 GB" | 20260507 14:23:32.352 | INFO | } | 20260507 14:23:32.352 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:23:32.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:32.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.352 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:23:32.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.352 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:23:32.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:32.353 | INFO | Uploaded 36GiB to GPU in 1.60683s (22.4043GiB/s) | 20260507 14:23:32.353 | INFO | Downloaded 34GiB to GPU in 1.5261s (22.279GiB/s) | 20260507 14:23:37.491 | INFO | Analyzing chunk frames = [187904,188416) | 20260507 14:23:37.491 | INFO | MemInfo: { | 20260507 14:23:37.491 | INFO | "gpu" : " 17 GB", | 20260507 14:23:37.491 | INFO | "huge" : "", | 20260507 14:23:37.491 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:37.492 | INFO | "malloc" : " 14 GB", | 20260507 14:23:37.492 | INFO | "pinned" : " 29 GB" | 20260507 14:23:37.492 | INFO | } | 20260507 14:23:37.492 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:23:37.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:37.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:23:37.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:23:37.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:37.492 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4215GiB/s) | 20260507 14:23:37.492 | INFO | Downloaded 34GiB to GPU in 1.52647s (22.2737GiB/s) | 20260507 14:23:37.492 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:23:37.492 | INFO | Total mem usage: 15.5 GiB | 20260507 14:23:37.492 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:23:37.492 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.6%) | 20260507 14:23:37.492 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:23:37.492 | INFO | High Water Breakdown: | 20260507 14:23:37.492 | INFO | 58.5% : 8.67 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:23:37.492 | INFO | 41.4% : 6.14 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:23:37.492 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:23:37.492 | INFO | ------------------------------------------ | 20260507 14:23:37.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:23:37.492 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:23:37.492 | INFO | Total mem usage: 26.5 GiB | 20260507 14:23:37.492 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:23:37.492 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:23:37.492 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:23:37.492 | INFO | High Water Breakdown: | 20260507 14:23:37.492 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:23:37.492 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:23:37.492 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:23:37.492 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:23:37.492 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:23:37.492 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:23:37.492 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:23:37.492 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:23:37.492 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:23:37.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:23:37.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:23:37.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:23:37.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:23:37.492 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:23:37.492 | INFO | ------------------------------------------ | 20260507 14:23:37.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:23:37.492 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:23:37.492 | INFO | Total mem usage: 27.8 GiB | 20260507 14:23:37.492 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:23:37.492 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:23:37.492 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:23:37.492 | INFO | High Water Breakdown: | 20260507 14:23:37.492 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:23:37.492 | INFO | ------------------------------------------ | 20260507 14:23:37.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:23:37.492 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:23:37.492 | INFO | Total mem usage: 15.7 GiB | 20260507 14:23:37.492 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:23:37.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:23:37.492 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:23:37.492 | INFO | High Water Breakdown: | 20260507 14:23:37.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:23:37.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:23:37.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:23:37.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:23:37.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:23:37.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:23:37.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:23:37.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:23:37.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:23:37.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:23:37.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:23:37.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:23:37.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:23:37.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:23:37.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:23:37.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:23:37.492 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:23:37.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:23:37.492 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:23:37.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:23:37.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:23:37.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:23:37.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:23:37.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:23:37.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:23:37.492 | INFO | ------------------------------------------ | 20260507 14:23:37.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:23:37.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:23:37.493 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:23:37.493 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:23:37.493 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.67e-03 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 16.09%, DME -- 3.87e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.72e+01 / -nan, 4.35e-02 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 3.67%, Metrics -- 1.10e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.11e-01 / -nan, 5.99e-02 / 0.00e+00 | 20260507 14:23:37.493 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:23:37.493 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:23:37.493 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:23:37.493 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 18.61%, Upload -- 1.68e+01 / 0.00e+00, 3.94e-01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 17.89%, Download -- 1.61e+01 / 0.00e+00, 7.07e-01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 33.32%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 0.00%, LockDownload -- 4.33e-04 / 0.00e+00, 9.22e-05 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 29.61%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:23:37.493 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 14:23:37.589 | INFO | Starting a round of Baz IO | 20260507 14:23:40.489 | INFO | Populating RTMetrics report | 20260507 14:23:40.489 | INFO | RTMetrics report populated | 20260507 14:23:40.489 | INFO | Opening tmp RTMetrics json file | 20260507 14:23:40.506 | INFO | Serializing RTMetrics to json | 20260507 14:23:40.510 | INFO | Serialization done, swapping tmp file now... | 20260507 14:23:40.516 | INFO | RTMetrics done | 20260507 14:23:40.693 | INFO | Finished a round of Baz IO: Wrote 1.649 GiB in 3.104 seconds | 20260507 14:23:40.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.66% / 0.12% / 40.22% | 20260507 14:23:40.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.37975% / 0.00042518% / 1.58946e-05% | 20260507 14:23:40.693 | INFO | Baz IO overall: | 20260507 14:23:40.693 | INFO | BytesWritten : 60317957836 | 20260507 14:23:40.693 | INFO | NumEvents : 28057970320 | 20260507 14:23:42.631 | INFO | Analyzing chunk frames = [188416,188928) | 20260507 14:23:42.631 | INFO | MemInfo: { | 20260507 14:23:42.631 | INFO | "gpu" : " 17 GB", | 20260507 14:23:42.631 | INFO | "huge" : "", | 20260507 14:23:42.631 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:42.631 | INFO | "malloc" : " 14 GB", | 20260507 14:23:42.631 | INFO | "pinned" : " 29 GB" | 20260507 14:23:42.631 | INFO | } | 20260507 14:23:42.631 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:23:42.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:42.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:42.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:42.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:42.631 | INFO | ANALYSIS : 57.4%, 2.9, 86.3ms, 0 | 20260507 14:23:42.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:42.631 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 14:23:42.631 | INFO | PRE_HQ : 56.4%, 2.0, 60.0ms, 0 | 20260507 14:23:42.631 | INFO | BAZWRITER : 60.0%, 4.7, 144.2ms, 0 | 20260507 14:23:42.631 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 14:23:42.631 | INFO | Uploaded 36GiB to GPU in 1.66968s (21.561GiB/s) | 20260507 14:23:42.631 | INFO | Downloaded 38GiB to GPU in 1.7368s (21.8793GiB/s) | 20260507 14:23:47.770 | INFO | Analyzing chunk frames = [188928,189440) | 20260507 14:23:47.770 | INFO | MemInfo: { | 20260507 14:23:47.770 | INFO | "gpu" : " 17 GB", | 20260507 14:23:47.770 | INFO | "huge" : "", | 20260507 14:23:47.770 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:47.770 | INFO | "malloc" : " 14 GB", | 20260507 14:23:47.770 | INFO | "pinned" : " 29 GB" | 20260507 14:23:47.770 | INFO | } | 20260507 14:23:47.770 | INFO | Perf report for the last 5.13951 seconds: | 20260507 14:23:47.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:47.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.770 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:23:47.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.771 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260507 14:23:47.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:47.771 | INFO | Uploaded 36GiB to GPU in 1.60313s (22.4561GiB/s) | 20260507 14:23:47.771 | INFO | Downloaded 34GiB to GPU in 1.5233s (22.32GiB/s) | 20260507 14:23:52.909 | INFO | Analyzing chunk frames = [189440,189952) | 20260507 14:23:52.909 | INFO | MemInfo: { | 20260507 14:23:52.909 | INFO | "gpu" : " 17 GB", | 20260507 14:23:52.909 | INFO | "huge" : "", | 20260507 14:23:52.909 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:52.909 | INFO | "malloc" : " 14 GB", | 20260507 14:23:52.909 | INFO | "pinned" : " 29 GB" | 20260507 14:23:52.909 | INFO | } | 20260507 14:23:52.910 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:23:52.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:52.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260507 14:23:52.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:23:52.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:52.910 | INFO | Uploaded 36GiB to GPU in 1.60729s (22.398GiB/s) | 20260507 14:23:52.910 | INFO | Downloaded 34GiB to GPU in 1.52683s (22.2684GiB/s) | 20260507 14:23:58.049 | INFO | Analyzing chunk frames = [189952,190464) | 20260507 14:23:58.049 | INFO | MemInfo: { | 20260507 14:23:58.049 | INFO | "gpu" : " 17 GB", | 20260507 14:23:58.049 | INFO | "huge" : "", | 20260507 14:23:58.049 | INFO | "huge_shared" : " 29 GB", | 20260507 14:23:58.049 | INFO | "malloc" : " 14 GB", | 20260507 14:23:58.049 | INFO | "pinned" : " 29 GB" | 20260507 14:23:58.049 | INFO | } | 20260507 14:23:58.049 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:23:58.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:23:58.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.049 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:23:58.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.049 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:23:58.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:23:58.050 | INFO | Uploaded 36GiB to GPU in 1.60671s (22.406GiB/s) | 20260507 14:23:58.050 | INFO | Downloaded 34GiB to GPU in 1.52684s (22.2682GiB/s) | 20260507 14:24:03.188 | INFO | Analyzing chunk frames = [190464,190976) | 20260507 14:24:03.188 | INFO | MemInfo: { | 20260507 14:24:03.188 | INFO | "gpu" : " 17 GB", | 20260507 14:24:03.188 | INFO | "huge" : "", | 20260507 14:24:03.188 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:03.188 | INFO | "malloc" : " 14 GB", | 20260507 14:24:03.188 | INFO | "pinned" : " 29 GB" | 20260507 14:24:03.188 | INFO | } | 20260507 14:24:03.188 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:24:03.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:03.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.188 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:24:03.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.188 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:24:03.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:03.189 | INFO | Uploaded 36GiB to GPU in 1.60491s (22.4312GiB/s) | 20260507 14:24:03.189 | INFO | Downloaded 34GiB to GPU in 1.52537s (22.2896GiB/s) | 20260507 14:24:08.327 | INFO | Analyzing chunk frames = [190976,191488) | 20260507 14:24:08.327 | INFO | MemInfo: { | 20260507 14:24:08.327 | INFO | "gpu" : " 17 GB", | 20260507 14:24:08.327 | INFO | "huge" : "", | 20260507 14:24:08.327 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:08.327 | INFO | "malloc" : " 14 GB", | 20260507 14:24:08.328 | INFO | "pinned" : " 29 GB" | 20260507 14:24:08.328 | INFO | } | 20260507 14:24:08.328 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:24:08.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:08.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:24:08.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:24:08.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:08.328 | INFO | Uploaded 36GiB to GPU in 1.60495s (22.4306GiB/s) | 20260507 14:24:08.328 | INFO | Downloaded 34GiB to GPU in 1.52535s (22.2899GiB/s) | 20260507 14:24:13.467 | INFO | Analyzing chunk frames = [191488,192000) | 20260507 14:24:13.467 | INFO | MemInfo: { | 20260507 14:24:13.467 | INFO | "gpu" : " 17 GB", | 20260507 14:24:13.467 | INFO | "huge" : "", | 20260507 14:24:13.467 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:13.467 | INFO | "malloc" : " 14 GB", | 20260507 14:24:13.467 | INFO | "pinned" : " 29 GB" | 20260507 14:24:13.467 | INFO | } | 20260507 14:24:13.467 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:24:13.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:13.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.467 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:24:13.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.467 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:24:13.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:13.468 | INFO | Uploaded 36GiB to GPU in 1.60413s (22.4421GiB/s) | 20260507 14:24:13.468 | INFO | Downloaded 34GiB to GPU in 1.52547s (22.2883GiB/s) | 20260507 14:24:18.606 | INFO | Analyzing chunk frames = [192000,192512) | 20260507 14:24:18.606 | INFO | MemInfo: { | 20260507 14:24:18.606 | INFO | "gpu" : " 17 GB", | 20260507 14:24:18.606 | INFO | "huge" : "", | 20260507 14:24:18.606 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:18.606 | INFO | "malloc" : " 14 GB", | 20260507 14:24:18.606 | INFO | "pinned" : " 29 GB" | 20260507 14:24:18.606 | INFO | } | 20260507 14:24:18.606 | INFO | Perf report for the last 5.13908 seconds: | 20260507 14:24:18.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:18.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.606 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:24:18.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.606 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 14:24:18.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:18.607 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4257GiB/s) | 20260507 14:24:18.607 | INFO | Downloaded 34GiB to GPU in 1.52563s (22.2859GiB/s) | 20260507 14:24:18.705 | INFO | Starting a round of Baz IO | 20260507 14:24:21.604 | INFO | Populating RTMetrics report | 20260507 14:24:21.604 | INFO | RTMetrics report populated | 20260507 14:24:21.604 | INFO | Opening tmp RTMetrics json file | 20260507 14:24:21.610 | INFO | Serializing RTMetrics to json | 20260507 14:24:21.613 | INFO | Serialization done, swapping tmp file now... | 20260507 14:24:21.614 | INFO | RTMetrics done | 20260507 14:24:21.747 | INFO | Finished a round of Baz IO: Wrote 1.649 GiB in 3.042 seconds | 20260507 14:24:21.747 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.55% / 0.12% / 40.34% | 20260507 14:24:21.747 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.31774% / 0.000480811% / 7.94729e-06% | 20260507 14:24:21.747 | INFO | Baz IO overall: | 20260507 14:24:21.747 | INFO | BytesWritten : 62088597505 | 20260507 14:24:21.747 | INFO | NumEvents : 28923130268 | 20260507 14:24:23.746 | INFO | Analyzing chunk frames = [192512,193024) | 20260507 14:24:23.746 | INFO | MemInfo: { | 20260507 14:24:23.746 | INFO | "gpu" : " 17 GB", | 20260507 14:24:23.746 | INFO | "huge" : "", | 20260507 14:24:23.746 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:23.746 | INFO | "malloc" : " 14 GB", | 20260507 14:24:23.746 | INFO | "pinned" : " 29 GB" | 20260507 14:24:23.746 | INFO | } | 20260507 14:24:23.746 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:24:23.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:23.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:23.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:23.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:23.746 | INFO | ANALYSIS : 57.4%, 3.0, 89.3ms, 0 | 20260507 14:24:23.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:23.746 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 14:24:23.746 | INFO | PRE_HQ : 56.3%, 2.1, 60.5ms, 0 | 20260507 14:24:23.746 | INFO | BAZWRITER : 58.8%, 4.1, 123.1ms, 0 | 20260507 14:24:23.746 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 14:24:23.746 | INFO | Uploaded 36GiB to GPU in 1.63734s (21.9869GiB/s) | 20260507 14:24:23.746 | INFO | Downloaded 38GiB to GPU in 1.7029s (22.3149GiB/s) | 20260507 14:24:28.885 | INFO | Analyzing chunk frames = [193024,193536) | 20260507 14:24:28.885 | INFO | MemInfo: { | 20260507 14:24:28.885 | INFO | "gpu" : " 17 GB", | 20260507 14:24:28.885 | INFO | "huge" : "", | 20260507 14:24:28.885 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:28.885 | INFO | "malloc" : " 14 GB", | 20260507 14:24:28.885 | INFO | "pinned" : " 29 GB" | 20260507 14:24:28.885 | INFO | } | 20260507 14:24:28.885 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:24:28.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:28.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:24:28.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:24:28.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:28.885 | INFO | Uploaded 36GiB to GPU in 1.60332s (22.4534GiB/s) | 20260507 14:24:28.885 | INFO | Downloaded 34GiB to GPU in 1.52233s (22.3343GiB/s) | 20260507 14:24:34.024 | INFO | Analyzing chunk frames = [193536,194048) | 20260507 14:24:34.024 | INFO | MemInfo: { | 20260507 14:24:34.024 | INFO | "gpu" : " 17 GB", | 20260507 14:24:34.024 | INFO | "huge" : "", | 20260507 14:24:34.024 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:34.024 | INFO | "malloc" : " 14 GB", | 20260507 14:24:34.024 | INFO | "pinned" : " 29 GB" | 20260507 14:24:34.024 | INFO | } | 20260507 14:24:34.024 | INFO | Perf report for the last 5.13915 seconds: | 20260507 14:24:34.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:34.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.024 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:24:34.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.024 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:24:34.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:34.025 | INFO | Uploaded 36GiB to GPU in 1.60768s (22.3925GiB/s) | 20260507 14:24:34.025 | INFO | Downloaded 34GiB to GPU in 1.52756s (22.2577GiB/s) | 20260507 14:24:39.163 | INFO | Analyzing chunk frames = [194048,194560) | 20260507 14:24:39.163 | INFO | MemInfo: { | 20260507 14:24:39.163 | INFO | "gpu" : " 17 GB", | 20260507 14:24:39.163 | INFO | "huge" : "", | 20260507 14:24:39.163 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:39.163 | INFO | "malloc" : " 14 GB", | 20260507 14:24:39.163 | INFO | "pinned" : " 29 GB" | 20260507 14:24:39.164 | INFO | } | 20260507 14:24:39.164 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:24:39.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:39.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:24:39.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:24:39.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:39.164 | INFO | Uploaded 36GiB to GPU in 1.60716s (22.3997GiB/s) | 20260507 14:24:39.164 | INFO | Downloaded 34GiB to GPU in 1.52721s (22.2629GiB/s) | 20260507 14:24:44.303 | INFO | Analyzing chunk frames = [194560,195072) | 20260507 14:24:44.303 | INFO | MemInfo: { | 20260507 14:24:44.303 | INFO | "gpu" : " 17 GB", | 20260507 14:24:44.303 | INFO | "huge" : "", | 20260507 14:24:44.303 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:44.303 | INFO | "malloc" : " 14 GB", | 20260507 14:24:44.303 | INFO | "pinned" : " 29 GB" | 20260507 14:24:44.303 | INFO | } | 20260507 14:24:44.303 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:24:44.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:44.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:24:44.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:24:44.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:44.303 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.4229GiB/s) | 20260507 14:24:44.304 | INFO | Downloaded 34GiB to GPU in 1.52458s (22.3012GiB/s) | 20260507 14:24:49.442 | INFO | Analyzing chunk frames = [195072,195584) | 20260507 14:24:49.442 | INFO | MemInfo: { | 20260507 14:24:49.442 | INFO | "gpu" : " 17 GB", | 20260507 14:24:49.442 | INFO | "huge" : "", | 20260507 14:24:49.442 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:49.442 | INFO | "malloc" : " 14 GB", | 20260507 14:24:49.442 | INFO | "pinned" : " 29 GB" | 20260507 14:24:49.442 | INFO | } | 20260507 14:24:49.442 | INFO | Perf report for the last 5.13914 seconds: | 20260507 14:24:49.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:49.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.442 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:24:49.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.442 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:24:49.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:49.443 | INFO | Uploaded 36GiB to GPU in 1.60618s (22.4134GiB/s) | 20260507 14:24:49.443 | INFO | Downloaded 34GiB to GPU in 1.52431s (22.3052GiB/s) | 20260507 14:24:54.582 | INFO | Analyzing chunk frames = [195584,196096) | 20260507 14:24:54.582 | INFO | MemInfo: { | 20260507 14:24:54.582 | INFO | "gpu" : " 17 GB", | 20260507 14:24:54.582 | INFO | "huge" : "", | 20260507 14:24:54.582 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:54.582 | INFO | "malloc" : " 14 GB", | 20260507 14:24:54.582 | INFO | "pinned" : " 29 GB" | 20260507 14:24:54.582 | INFO | } | 20260507 14:24:54.582 | INFO | Perf report for the last 5.1396 seconds: | 20260507 14:24:54.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:54.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:24:54.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:24:54.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:54.582 | INFO | Uploaded 36GiB to GPU in 1.60586s (22.4178GiB/s) | 20260507 14:24:54.582 | INFO | Downloaded 34GiB to GPU in 1.5248s (22.298GiB/s) | 20260507 14:24:59.721 | INFO | Analyzing chunk frames = [196096,196608) | 20260507 14:24:59.721 | INFO | MemInfo: { | 20260507 14:24:59.721 | INFO | "gpu" : " 17 GB", | 20260507 14:24:59.721 | INFO | "huge" : "", | 20260507 14:24:59.721 | INFO | "huge_shared" : " 29 GB", | 20260507 14:24:59.721 | INFO | "malloc" : " 15 GB", | 20260507 14:24:59.721 | INFO | "pinned" : " 29 GB" | 20260507 14:24:59.721 | INFO | } | 20260507 14:24:59.721 | INFO | Perf report for the last 5.13903 seconds: | 20260507 14:24:59.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:24:59.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.721 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:24:59.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.721 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:24:59.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:24:59.722 | INFO | Uploaded 36GiB to GPU in 1.60682s (22.4045GiB/s) | 20260507 14:24:59.722 | INFO | Downloaded 34GiB to GPU in 1.52618s (22.2779GiB/s) | 20260507 14:24:59.722 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:24:59.722 | INFO | Total mem usage: 15.5 GiB | 20260507 14:24:59.722 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:24:59.722 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -0.858%) | 20260507 14:24:59.722 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:24:59.722 | INFO | High Water Breakdown: | 20260507 14:24:59.722 | INFO | 59.1% : 9.1 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:24:59.722 | INFO | 40.7% : 6.27 GiB ( 93.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:24:59.722 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:24:59.722 | INFO | ------------------------------------------ | 20260507 14:24:59.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:24:59.722 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:24:59.722 | INFO | Total mem usage: 26.5 GiB | 20260507 14:24:59.722 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:24:59.722 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:24:59.722 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:24:59.722 | INFO | High Water Breakdown: | 20260507 14:24:59.722 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:24:59.722 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:24:59.722 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:24:59.722 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:24:59.722 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:24:59.722 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:24:59.722 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:24:59.722 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:24:59.722 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:24:59.722 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:24:59.722 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:24:59.722 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:24:59.722 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:24:59.722 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:24:59.722 | INFO | ------------------------------------------ | 20260507 14:24:59.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:24:59.722 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:24:59.722 | INFO | Total mem usage: 27.8 GiB | 20260507 14:24:59.722 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:24:59.722 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:24:59.722 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:24:59.722 | INFO | High Water Breakdown: | 20260507 14:24:59.722 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:24:59.722 | INFO | ------------------------------------------ | 20260507 14:24:59.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:24:59.722 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:24:59.722 | INFO | Total mem usage: 15.7 GiB | 20260507 14:24:59.722 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:24:59.722 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:24:59.722 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260507 14:24:59.722 | INFO | High Water Breakdown: | 20260507 14:24:59.722 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:24:59.722 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:24:59.722 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:24:59.722 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:24:59.722 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:24:59.722 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:24:59.722 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:24:59.722 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:24:59.722 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:24:59.722 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:24:59.722 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:24:59.722 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:24:59.722 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:24:59.723 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:24:59.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:24:59.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:24:59.723 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:24:59.723 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:24:59.723 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:24:59.723 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:24:59.723 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:24:59.723 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:24:59.723 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:24:59.723 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:24:59.723 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:24:59.723 | INFO | ------------------------------------------ | 20260507 14:24:59.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:24:59.723 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:24:59.723 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:24:59.723 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:24:59.723 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 6.61e-03 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.98e-02 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 3.06%, Histogram -- 9.19e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 16.09%, DME -- 3.87e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.72e+01 / -nan, 4.49e-02 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 3.67%, Metrics -- 1.10e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.10e-01 / -nan, 6.01e-02 / 0.00e+00 | 20260507 14:24:59.723 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:24:59.723 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:24:59.723 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:24:59.723 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 18.65%, Upload -- 1.68e+01 / 0.00e+00, 4.34e-01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 17.94%, Download -- 1.61e+01 / 0.00e+00, 7.62e-01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 33.38%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 9.56e-05 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 29.46%, LockCompute -- 2.65e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:24:59.723 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260507 14:24:59.818 | INFO | Starting a round of Baz IO | 20260507 14:25:02.718 | INFO | Populating RTMetrics report | 20260507 14:25:02.718 | INFO | RTMetrics report populated | 20260507 14:25:02.718 | INFO | Opening tmp RTMetrics json file | 20260507 14:25:02.750 | INFO | Serializing RTMetrics to json | 20260507 14:25:02.765 | INFO | Serialization done, swapping tmp file now... | 20260507 14:25:02.770 | INFO | RTMetrics done | 20260507 14:25:02.826 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.007 seconds | 20260507 14:25:02.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.43% / 0.11% / 40.45% | 20260507 14:25:02.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.29539% / 0.000317891% / 2.38419e-05% | 20260507 14:25:02.826 | INFO | Baz IO overall: | 20260507 14:25:02.826 | INFO | BytesWritten : 63862752854 | 20260507 14:25:02.826 | INFO | NumEvents : 29790112369 | 20260507 14:25:04.860 | INFO | Analyzing chunk frames = [196608,197120) | 20260507 14:25:04.860 | INFO | MemInfo: { | 20260507 14:25:04.860 | INFO | "gpu" : " 17 GB", | 20260507 14:25:04.860 | INFO | "huge" : "", | 20260507 14:25:04.860 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:04.860 | INFO | "malloc" : " 14 GB", | 20260507 14:25:04.860 | INFO | "pinned" : " 29 GB" | 20260507 14:25:04.860 | INFO | } | 20260507 14:25:04.860 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:25:04.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:04.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:04.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:04.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:04.860 | INFO | ANALYSIS : 57.4%, 2.9, 87.8ms, 0 | 20260507 14:25:04.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:04.860 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 14:25:04.860 | INFO | PRE_HQ : 56.2%, 2.1, 60.0ms, 0 | 20260507 14:25:04.861 | INFO | BAZWRITER : 58.0%, 4.5, 134.7ms, 0 | 20260507 14:25:04.861 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:25:04.861 | INFO | Uploaded 36GiB to GPU in 1.6553s (21.7483GiB/s) | 20260507 14:25:04.861 | INFO | Downloaded 38GiB to GPU in 1.72008s (22.092GiB/s) | 20260507 14:25:10.000 | INFO | Analyzing chunk frames = [197120,197632) | 20260507 14:25:10.000 | INFO | MemInfo: { | 20260507 14:25:10.000 | INFO | "gpu" : " 17 GB", | 20260507 14:25:10.000 | INFO | "huge" : "", | 20260507 14:25:10.000 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:10.000 | INFO | "malloc" : " 14 GB", | 20260507 14:25:10.000 | INFO | "pinned" : " 29 GB" | 20260507 14:25:10.000 | INFO | } | 20260507 14:25:10.000 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:25:10.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:10.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:25:10.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:25:10.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:10.000 | INFO | Uploaded 36GiB to GPU in 1.60332s (22.4534GiB/s) | 20260507 14:25:10.001 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3285GiB/s) | 20260507 14:25:15.139 | INFO | Analyzing chunk frames = [197632,198144) | 20260507 14:25:15.139 | INFO | MemInfo: { | 20260507 14:25:15.139 | INFO | "gpu" : " 17 GB", | 20260507 14:25:15.139 | INFO | "huge" : "", | 20260507 14:25:15.139 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:15.139 | INFO | "malloc" : " 14 GB", | 20260507 14:25:15.139 | INFO | "pinned" : " 29 GB" | 20260507 14:25:15.139 | INFO | } | 20260507 14:25:15.139 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:25:15.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:15.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:25:15.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:25:15.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:15.139 | INFO | Uploaded 36GiB to GPU in 1.60723s (22.3988GiB/s) | 20260507 14:25:15.140 | INFO | Downloaded 34GiB to GPU in 1.52687s (22.2678GiB/s) | 20260507 14:25:20.278 | INFO | Analyzing chunk frames = [198144,198656) | 20260507 14:25:20.278 | INFO | MemInfo: { | 20260507 14:25:20.278 | INFO | "gpu" : " 17 GB", | 20260507 14:25:20.278 | INFO | "huge" : "", | 20260507 14:25:20.278 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:20.278 | INFO | "malloc" : " 14 GB", | 20260507 14:25:20.278 | INFO | "pinned" : " 29 GB" | 20260507 14:25:20.278 | INFO | } | 20260507 14:25:20.279 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:25:20.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:20.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:25:20.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:25:20.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:20.279 | INFO | Uploaded 36GiB to GPU in 1.60696s (22.4025GiB/s) | 20260507 14:25:20.279 | INFO | Downloaded 34GiB to GPU in 1.52663s (22.2712GiB/s) | 20260507 14:25:25.418 | INFO | Analyzing chunk frames = [198656,199168) | 20260507 14:25:25.418 | INFO | MemInfo: { | 20260507 14:25:25.418 | INFO | "gpu" : " 17 GB", | 20260507 14:25:25.418 | INFO | "huge" : "", | 20260507 14:25:25.418 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:25.418 | INFO | "malloc" : " 14 GB", | 20260507 14:25:25.418 | INFO | "pinned" : " 29 GB" | 20260507 14:25:25.418 | INFO | } | 20260507 14:25:25.418 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:25:25.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:25.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:25:25.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:25:25.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:25.418 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4177GiB/s) | 20260507 14:25:25.418 | INFO | Downloaded 34GiB to GPU in 1.52411s (22.3081GiB/s) | 20260507 14:25:30.557 | INFO | Analyzing chunk frames = [199168,199680) | 20260507 14:25:30.557 | INFO | MemInfo: { | 20260507 14:25:30.557 | INFO | "gpu" : " 17 GB", | 20260507 14:25:30.557 | INFO | "huge" : "", | 20260507 14:25:30.557 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:30.557 | INFO | "malloc" : " 14 GB", | 20260507 14:25:30.557 | INFO | "pinned" : " 29 GB" | 20260507 14:25:30.557 | INFO | } | 20260507 14:25:30.557 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:25:30.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:30.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:25:30.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:25:30.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:30.558 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.421GiB/s) | 20260507 14:25:30.558 | INFO | Downloaded 34GiB to GPU in 1.52365s (22.3148GiB/s) | 20260507 14:25:35.696 | INFO | Analyzing chunk frames = [199680,200192) | 20260507 14:25:35.696 | INFO | MemInfo: { | 20260507 14:25:35.696 | INFO | "gpu" : " 17 GB", | 20260507 14:25:35.696 | INFO | "huge" : "", | 20260507 14:25:35.696 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:35.696 | INFO | "malloc" : " 14 GB", | 20260507 14:25:35.696 | INFO | "pinned" : " 29 GB" | 20260507 14:25:35.696 | INFO | } | 20260507 14:25:35.696 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:25:35.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:35.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.697 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:25:35.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.697 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:25:35.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:35.697 | INFO | Uploaded 36GiB to GPU in 1.60616s (22.4137GiB/s) | 20260507 14:25:35.697 | INFO | Downloaded 34GiB to GPU in 1.52538s (22.2895GiB/s) | 20260507 14:25:40.836 | INFO | Analyzing chunk frames = [200192,200704) | 20260507 14:25:40.836 | INFO | MemInfo: { | 20260507 14:25:40.836 | INFO | "gpu" : " 17 GB", | 20260507 14:25:40.836 | INFO | "huge" : "", | 20260507 14:25:40.836 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:40.836 | INFO | "malloc" : " 14 GB", | 20260507 14:25:40.836 | INFO | "pinned" : " 29 GB" | 20260507 14:25:40.836 | INFO | } | 20260507 14:25:40.836 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:25:40.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:40.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.836 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:25:40.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.836 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:25:40.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:40.837 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.413GiB/s) | 20260507 14:25:40.837 | INFO | Downloaded 34GiB to GPU in 1.52642s (22.2744GiB/s) | 20260507 14:25:40.933 | INFO | Starting a round of Baz IO | 20260507 14:25:43.830 | INFO | Populating RTMetrics report | 20260507 14:25:43.831 | INFO | RTMetrics report populated | 20260507 14:25:43.831 | INFO | Opening tmp RTMetrics json file | 20260507 14:25:43.831 | INFO | Serializing RTMetrics to json | 20260507 14:25:43.833 | INFO | Serialization done, swapping tmp file now... | 20260507 14:25:43.833 | INFO | RTMetrics done | 20260507 14:25:43.926 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 2.992 seconds | 20260507 14:25:43.927 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.32% / 0.11% / 40.57% | 20260507 14:25:43.927 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.25357% / 0.000333786% / 2.78155e-05% | 20260507 14:25:43.927 | INFO | Baz IO overall: | 20260507 14:25:43.927 | INFO | BytesWritten : 65639291764 | 20260507 14:25:43.927 | INFO | NumEvents : 30657959159 | 20260507 14:25:45.975 | INFO | Analyzing chunk frames = [200704,201216) | 20260507 14:25:45.975 | INFO | MemInfo: { | 20260507 14:25:45.975 | INFO | "gpu" : " 17 GB", | 20260507 14:25:45.975 | INFO | "huge" : "", | 20260507 14:25:45.975 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:45.975 | INFO | "malloc" : " 14 GB", | 20260507 14:25:45.975 | INFO | "pinned" : " 29 GB" | 20260507 14:25:45.975 | INFO | } | 20260507 14:25:45.975 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:25:45.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:45.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:45.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:45.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:45.975 | INFO | ANALYSIS : 57.4%, 3.0, 89.4ms, 0 | 20260507 14:25:45.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:45.975 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 14:25:45.975 | INFO | PRE_HQ : 56.1%, 2.1, 60.3ms, 0 | 20260507 14:25:45.975 | INFO | BAZWRITER : 57.8%, 3.9, 116.0ms, 0 | 20260507 14:25:45.975 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 14:25:45.976 | INFO | Uploaded 36GiB to GPU in 1.64251s (21.9177GiB/s) | 20260507 14:25:45.976 | INFO | Downloaded 38GiB to GPU in 1.70544s (22.2817GiB/s) | 20260507 14:25:51.114 | INFO | Analyzing chunk frames = [201216,201728) | 20260507 14:25:51.114 | INFO | MemInfo: { | 20260507 14:25:51.114 | INFO | "gpu" : " 17 GB", | 20260507 14:25:51.114 | INFO | "huge" : "", | 20260507 14:25:51.114 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:51.114 | INFO | "malloc" : " 14 GB", | 20260507 14:25:51.114 | INFO | "pinned" : " 29 GB" | 20260507 14:25:51.114 | INFO | } | 20260507 14:25:51.115 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:25:51.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:51.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:25:51.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | PRE_HQ : 29.0%, 1.0, 15.2ms, 0 | 20260507 14:25:51.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:51.115 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4522GiB/s) | 20260507 14:25:51.115 | INFO | Downloaded 34GiB to GPU in 1.5219s (22.3405GiB/s) | 20260507 14:25:56.254 | INFO | Analyzing chunk frames = [201728,202240) | 20260507 14:25:56.254 | INFO | MemInfo: { | 20260507 14:25:56.254 | INFO | "gpu" : " 17 GB", | 20260507 14:25:56.254 | INFO | "huge" : "", | 20260507 14:25:56.254 | INFO | "huge_shared" : " 29 GB", | 20260507 14:25:56.254 | INFO | "malloc" : " 14 GB", | 20260507 14:25:56.254 | INFO | "pinned" : " 29 GB" | 20260507 14:25:56.254 | INFO | } | 20260507 14:25:56.254 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:25:56.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:25:56.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.254 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:25:56.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.254 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:25:56.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:25:56.255 | INFO | Uploaded 36GiB to GPU in 1.60744s (22.3959GiB/s) | 20260507 14:25:56.255 | INFO | Downloaded 34GiB to GPU in 1.52771s (22.2555GiB/s) | 20260507 14:26:01.393 | INFO | Analyzing chunk frames = [202240,202752) | 20260507 14:26:01.393 | INFO | MemInfo: { | 20260507 14:26:01.393 | INFO | "gpu" : " 17 GB", | 20260507 14:26:01.393 | INFO | "huge" : "", | 20260507 14:26:01.393 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:01.393 | INFO | "malloc" : " 14 GB", | 20260507 14:26:01.393 | INFO | "pinned" : " 29 GB" | 20260507 14:26:01.393 | INFO | } | 20260507 14:26:01.393 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:26:01.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:01.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.393 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 14:26:01.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.393 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:26:01.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:01.394 | INFO | Uploaded 36GiB to GPU in 1.60669s (22.4063GiB/s) | 20260507 14:26:01.394 | INFO | Downloaded 34GiB to GPU in 1.52772s (22.2555GiB/s) | 20260507 14:26:06.532 | INFO | Analyzing chunk frames = [202752,203264) | 20260507 14:26:06.532 | INFO | MemInfo: { | 20260507 14:26:06.532 | INFO | "gpu" : " 17 GB", | 20260507 14:26:06.532 | INFO | "huge" : "", | 20260507 14:26:06.532 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:06.532 | INFO | "malloc" : " 14 GB", | 20260507 14:26:06.532 | INFO | "pinned" : " 29 GB" | 20260507 14:26:06.532 | INFO | } | 20260507 14:26:06.533 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:26:06.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:06.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:26:06.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:26:06.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:06.533 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4273GiB/s) | 20260507 14:26:06.533 | INFO | Downloaded 34GiB to GPU in 1.52488s (22.2969GiB/s) | 20260507 14:26:11.672 | INFO | Analyzing chunk frames = [203264,203776) | 20260507 14:26:11.672 | INFO | MemInfo: { | 20260507 14:26:11.672 | INFO | "gpu" : " 17 GB", | 20260507 14:26:11.672 | INFO | "huge" : "", | 20260507 14:26:11.672 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:11.672 | INFO | "malloc" : " 14 GB", | 20260507 14:26:11.672 | INFO | "pinned" : " 29 GB" | 20260507 14:26:11.672 | INFO | } | 20260507 14:26:11.672 | INFO | Perf report for the last 5.13954 seconds: | 20260507 14:26:11.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:11.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:26:11.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:26:11.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:11.672 | INFO | Uploaded 36GiB to GPU in 1.605s (22.4299GiB/s) | 20260507 14:26:11.673 | INFO | Downloaded 34GiB to GPU in 1.52393s (22.3108GiB/s) | 20260507 14:26:16.811 | INFO | Analyzing chunk frames = [203776,204288) | 20260507 14:26:16.811 | INFO | MemInfo: { | 20260507 14:26:16.811 | INFO | "gpu" : " 17 GB", | 20260507 14:26:16.811 | INFO | "huge" : "", | 20260507 14:26:16.811 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:16.811 | INFO | "malloc" : " 14 GB", | 20260507 14:26:16.811 | INFO | "pinned" : " 29 GB" | 20260507 14:26:16.811 | INFO | } | 20260507 14:26:16.811 | INFO | Perf report for the last 5.13911 seconds: | 20260507 14:26:16.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:16.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.811 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:26:16.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.811 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:26:16.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:16.812 | INFO | Uploaded 36GiB to GPU in 1.60612s (22.4142GiB/s) | 20260507 14:26:16.812 | INFO | Downloaded 34GiB to GPU in 1.52599s (22.2806GiB/s) | 20260507 14:26:21.950 | INFO | Analyzing chunk frames = [204288,204800) | 20260507 14:26:21.950 | INFO | MemInfo: { | 20260507 14:26:21.950 | INFO | "gpu" : " 17 GB", | 20260507 14:26:21.950 | INFO | "huge" : "", | 20260507 14:26:21.950 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:21.950 | INFO | "malloc" : " 14 GB", | 20260507 14:26:21.951 | INFO | "pinned" : " 29 GB" | 20260507 14:26:21.951 | INFO | } | 20260507 14:26:21.951 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:26:21.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:21.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:26:21.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 14:26:21.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:21.951 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:26:21.951 | INFO | Total mem usage: 15.5 GiB | 20260507 14:26:21.951 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:26:21.951 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -3.78%) | 20260507 14:26:21.951 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:26:21.951 | INFO | High Water Breakdown: | 20260507 14:26:21.951 | INFO | 58.7% : 8.77 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:26:21.951 | INFO | 41.2% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:26:21.951 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:26:21.951 | INFO | ------------------------------------------ | 20260507 14:26:21.951 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:26:21.951 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:26:21.951 | INFO | Total mem usage: 26.5 GiB | 20260507 14:26:21.951 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:26:21.951 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:26:21.951 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:26:21.951 | INFO | High Water Breakdown: | 20260507 14:26:21.951 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:26:21.951 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:26:21.951 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:26:21.951 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:26:21.951 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:26:21.951 | INFO | Uploaded 36GiB to GPU in 1.60541s (22.4242GiB/s) | 20260507 14:26:21.951 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:26:21.951 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:26:21.951 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:26:21.951 | INFO | Downloaded 34GiB to GPU in 1.52591s (22.2818GiB/s) | 20260507 14:26:21.951 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:26:21.951 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:26:21.951 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:26:21.951 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:26:21.951 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:26:21.951 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:26:21.951 | INFO | ------------------------------------------ | 20260507 14:26:21.951 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:26:21.951 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:26:21.951 | INFO | Total mem usage: 27.8 GiB | 20260507 14:26:21.951 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:26:21.951 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:26:21.951 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:26:21.951 | INFO | High Water Breakdown: | 20260507 14:26:21.951 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:26:21.951 | INFO | ------------------------------------------ | 20260507 14:26:21.951 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:26:21.951 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:26:21.951 | INFO | Total mem usage: 15.7 GiB | 20260507 14:26:21.951 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:26:21.951 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:26:21.951 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:26:21.951 | INFO | High Water Breakdown: | 20260507 14:26:21.951 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:26:21.951 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:26:21.951 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:26:21.951 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:26:21.951 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:26:21.951 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:26:21.951 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:26:21.951 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:26:21.951 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:26:21.951 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:26:21.951 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:26:21.951 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:26:21.951 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:26:21.951 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:26:21.951 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:26:21.951 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:26:21.951 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:26:21.951 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:26:21.951 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:26:21.951 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:26:21.951 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:26:21.951 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:26:21.951 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:26:21.951 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:26:21.951 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:26:21.951 | INFO | ------------------------------------------ | 20260507 14:26:21.951 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:26:21.951 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:26:21.951 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:26:21.951 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:26:21.951 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.30e-03 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 10.97%, Baseline -- 3.29e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 3.06%, Histogram -- 9.19e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 16.09%, DME -- 3.87e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.72e+01 / -nan, 4.28e-02 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.21e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.09e-01 / -nan, 6.01e-02 / 0.00e+00 | 20260507 14:26:21.951 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:26:21.951 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:26:21.951 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:26:21.951 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 18.63%, Upload -- 1.68e+01 / 0.00e+00, 4.07e-01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 17.90%, Download -- 1.61e+01 / 0.00e+00, 7.28e-01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 33.33%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 0.00%, LockDownload -- 4.46e-04 / 0.00e+00, 1.97e-04 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 29.56%, LockCompute -- 2.66e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:26:21.951 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260507 14:26:22.047 | INFO | Starting a round of Baz IO | 20260507 14:26:24.946 | INFO | Populating RTMetrics report | 20260507 14:26:24.946 | INFO | RTMetrics report populated | 20260507 14:26:24.946 | INFO | Opening tmp RTMetrics json file | 20260507 14:26:24.947 | INFO | Serializing RTMetrics to json | 20260507 14:26:24.949 | INFO | Serialization done, swapping tmp file now... | 20260507 14:26:24.950 | INFO | RTMetrics done | 20260507 14:26:25.051 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.003 seconds | 20260507 14:26:25.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.21% / 0.11% / 40.68% | 20260507 14:26:25.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.21414% / 0.000361602% / 3.17891e-05% | 20260507 14:26:25.051 | INFO | Baz IO overall: | 20260507 14:26:25.051 | INFO | BytesWritten : 67417461976 | 20260507 14:26:25.051 | INFO | NumEvents : 31526538515 | 20260507 14:26:27.090 | INFO | Analyzing chunk frames = [204800,205312) | 20260507 14:26:27.090 | INFO | MemInfo: { | 20260507 14:26:27.090 | INFO | "gpu" : " 17 GB", | 20260507 14:26:27.090 | INFO | "huge" : "", | 20260507 14:26:27.090 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:27.090 | INFO | "malloc" : " 14 GB", | 20260507 14:26:27.090 | INFO | "pinned" : " 29 GB" | 20260507 14:26:27.090 | INFO | } | 20260507 14:26:27.090 | INFO | Perf report for the last 5.13906 seconds: | 20260507 14:26:27.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:27.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:27.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:27.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:27.090 | INFO | ANALYSIS : 57.4%, 2.9, 88.3ms, 0 | 20260507 14:26:27.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:27.090 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 14:26:27.090 | INFO | PRE_HQ : 56.3%, 2.1, 60.3ms, 0 | 20260507 14:26:27.090 | INFO | BAZWRITER : 58.0%, 4.3, 127.2ms, 0 | 20260507 14:26:27.090 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 14:26:27.090 | INFO | Uploaded 36GiB to GPU in 1.64966s (21.8227GiB/s) | 20260507 14:26:27.090 | INFO | Downloaded 38GiB to GPU in 1.71296s (22.1838GiB/s) | 20260507 14:26:32.229 | INFO | Analyzing chunk frames = [205312,205824) | 20260507 14:26:32.229 | INFO | MemInfo: { | 20260507 14:26:32.229 | INFO | "gpu" : " 17 GB", | 20260507 14:26:32.229 | INFO | "huge" : "", | 20260507 14:26:32.229 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:32.229 | INFO | "malloc" : " 14 GB", | 20260507 14:26:32.229 | INFO | "pinned" : " 29 GB" | 20260507 14:26:32.229 | INFO | } | 20260507 14:26:32.229 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:26:32.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:32.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.229 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:26:32.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.229 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:26:32.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:32.230 | INFO | Uploaded 36GiB to GPU in 1.60383s (22.4462GiB/s) | 20260507 14:26:32.230 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3333GiB/s) | 20260507 14:26:37.368 | INFO | Analyzing chunk frames = [205824,206336) | 20260507 14:26:37.368 | INFO | MemInfo: { | 20260507 14:26:37.368 | INFO | "gpu" : " 17 GB", | 20260507 14:26:37.368 | INFO | "huge" : "", | 20260507 14:26:37.368 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:37.368 | INFO | "malloc" : " 14 GB", | 20260507 14:26:37.369 | INFO | "pinned" : " 29 GB" | 20260507 14:26:37.369 | INFO | } | 20260507 14:26:37.369 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:26:37.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:37.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:26:37.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:26:37.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:37.369 | INFO | Uploaded 36GiB to GPU in 1.60731s (22.3977GiB/s) | 20260507 14:26:37.369 | INFO | Downloaded 34GiB to GPU in 1.52724s (22.2624GiB/s) | 20260507 14:26:42.508 | INFO | Analyzing chunk frames = [206336,206848) | 20260507 14:26:42.508 | INFO | MemInfo: { | 20260507 14:26:42.508 | INFO | "gpu" : " 17 GB", | 20260507 14:26:42.508 | INFO | "huge" : "", | 20260507 14:26:42.508 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:42.508 | INFO | "malloc" : " 14 GB", | 20260507 14:26:42.508 | INFO | "pinned" : " 29 GB" | 20260507 14:26:42.508 | INFO | } | 20260507 14:26:42.508 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:26:42.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:42.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 14:26:42.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:26:42.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:42.508 | INFO | Uploaded 36GiB to GPU in 1.60652s (22.4086GiB/s) | 20260507 14:26:42.509 | INFO | Downloaded 34GiB to GPU in 1.52632s (22.2757GiB/s) | 20260507 14:26:47.647 | INFO | Analyzing chunk frames = [206848,207360) | 20260507 14:26:47.647 | INFO | MemInfo: { | 20260507 14:26:47.647 | INFO | "gpu" : " 17 GB", | 20260507 14:26:47.647 | INFO | "huge" : "", | 20260507 14:26:47.647 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:47.647 | INFO | "malloc" : " 14 GB", | 20260507 14:26:47.647 | INFO | "pinned" : " 29 GB" | 20260507 14:26:47.647 | INFO | } | 20260507 14:26:47.647 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:26:47.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:47.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.647 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 14:26:47.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.647 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:26:47.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:47.648 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.409GiB/s) | 20260507 14:26:47.648 | INFO | Downloaded 34GiB to GPU in 1.52545s (22.2885GiB/s) | 20260507 14:26:52.786 | INFO | Analyzing chunk frames = [207360,207872) | 20260507 14:26:52.786 | INFO | MemInfo: { | 20260507 14:26:52.786 | INFO | "gpu" : " 17 GB", | 20260507 14:26:52.786 | INFO | "huge" : "", | 20260507 14:26:52.786 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:52.786 | INFO | "malloc" : " 14 GB", | 20260507 14:26:52.786 | INFO | "pinned" : " 29 GB" | 20260507 14:26:52.787 | INFO | } | 20260507 14:26:52.787 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:26:52.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:52.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:26:52.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:26:52.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:52.787 | INFO | Uploaded 36GiB to GPU in 1.60523s (22.4267GiB/s) | 20260507 14:26:52.787 | INFO | Downloaded 34GiB to GPU in 1.52379s (22.3128GiB/s) | 20260507 14:26:57.926 | INFO | Analyzing chunk frames = [207872,208384) | 20260507 14:26:57.926 | INFO | MemInfo: { | 20260507 14:26:57.926 | INFO | "gpu" : " 17 GB", | 20260507 14:26:57.926 | INFO | "huge" : "", | 20260507 14:26:57.926 | INFO | "huge_shared" : " 29 GB", | 20260507 14:26:57.926 | INFO | "malloc" : " 14 GB", | 20260507 14:26:57.926 | INFO | "pinned" : " 29 GB" | 20260507 14:26:57.926 | INFO | } | 20260507 14:26:57.926 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:26:57.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:26:57.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.926 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:26:57.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.926 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:26:57.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:26:57.927 | INFO | Uploaded 36GiB to GPU in 1.60615s (22.4138GiB/s) | 20260507 14:26:57.927 | INFO | Downloaded 34GiB to GPU in 1.52634s (22.2756GiB/s) | 20260507 14:27:03.065 | INFO | Analyzing chunk frames = [208384,208896) | 20260507 14:27:03.065 | INFO | MemInfo: { | 20260507 14:27:03.065 | INFO | "gpu" : " 17 GB", | 20260507 14:27:03.065 | INFO | "huge" : "", | 20260507 14:27:03.065 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:03.065 | INFO | "malloc" : " 14 GB", | 20260507 14:27:03.065 | INFO | "pinned" : " 29 GB" | 20260507 14:27:03.065 | INFO | } | 20260507 14:27:03.065 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:27:03.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:03.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.065 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260507 14:27:03.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.066 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:27:03.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:03.066 | INFO | Uploaded 36GiB to GPU in 1.60638s (22.4106GiB/s) | 20260507 14:27:03.066 | INFO | Downloaded 34GiB to GPU in 1.52645s (22.2739GiB/s) | 20260507 14:27:03.162 | INFO | Starting a round of Baz IO | 20260507 14:27:06.063 | INFO | Populating RTMetrics report | 20260507 14:27:06.063 | INFO | RTMetrics report populated | 20260507 14:27:06.063 | INFO | Opening tmp RTMetrics json file | 20260507 14:27:06.089 | INFO | Serializing RTMetrics to json | 20260507 14:27:06.115 | INFO | Serialization done, swapping tmp file now... | 20260507 14:27:06.120 | INFO | RTMetrics done | 20260507 14:27:06.405 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.243 seconds | 20260507 14:27:06.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.10% / 0.11% / 40.79% | 20260507 14:27:06.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.17426% / 0.000325839% / 1.98682e-05% | 20260507 14:27:06.405 | INFO | Baz IO overall: | 20260507 14:27:06.405 | INFO | BytesWritten : 69197718329 | 20260507 14:27:06.405 | INFO | NumEvents : 32396203576 | 20260507 14:27:08.205 | INFO | Analyzing chunk frames = [208896,209408) | 20260507 14:27:08.205 | INFO | MemInfo: { | 20260507 14:27:08.205 | INFO | "gpu" : " 17 GB", | 20260507 14:27:08.205 | INFO | "huge" : "", | 20260507 14:27:08.205 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:08.205 | INFO | "malloc" : " 14 GB", | 20260507 14:27:08.205 | INFO | "pinned" : " 29 GB" | 20260507 14:27:08.205 | INFO | } | 20260507 14:27:08.205 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:27:08.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:08.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:08.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:08.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:08.205 | INFO | ANALYSIS : 57.5%, 2.9, 87.6ms, 0 | 20260507 14:27:08.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:08.205 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 14:27:08.205 | INFO | PRE_HQ : 56.2%, 2.0, 59.7ms, 0 | 20260507 14:27:08.205 | INFO | BAZWRITER : 62.7%, 4.5, 146.0ms, 0 | 20260507 14:27:08.205 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:27:08.205 | INFO | Uploaded 36GiB to GPU in 1.65957s (21.6924GiB/s) | 20260507 14:27:08.205 | INFO | Downloaded 38GiB to GPU in 1.72526s (22.0257GiB/s) | 20260507 14:27:13.344 | INFO | Analyzing chunk frames = [209408,209920) | 20260507 14:27:13.344 | INFO | MemInfo: { | 20260507 14:27:13.344 | INFO | "gpu" : " 17 GB", | 20260507 14:27:13.344 | INFO | "huge" : "", | 20260507 14:27:13.344 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:13.344 | INFO | "malloc" : " 14 GB", | 20260507 14:27:13.344 | INFO | "pinned" : " 29 GB" | 20260507 14:27:13.344 | INFO | } | 20260507 14:27:13.344 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:27:13.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:13.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.344 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:27:13.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.344 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:27:13.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:13.345 | INFO | Uploaded 36GiB to GPU in 1.60338s (22.4525GiB/s) | 20260507 14:27:13.345 | INFO | Downloaded 34GiB to GPU in 1.52257s (22.3306GiB/s) | 20260507 14:27:18.483 | INFO | Analyzing chunk frames = [209920,210432) | 20260507 14:27:18.483 | INFO | MemInfo: { | 20260507 14:27:18.483 | INFO | "gpu" : " 17 GB", | 20260507 14:27:18.483 | INFO | "huge" : "", | 20260507 14:27:18.483 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:18.483 | INFO | "malloc" : " 14 GB", | 20260507 14:27:18.484 | INFO | "pinned" : " 29 GB" | 20260507 14:27:18.484 | INFO | } | 20260507 14:27:18.484 | INFO | Perf report for the last 5.13959 seconds: | 20260507 14:27:18.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:18.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:27:18.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:27:18.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:18.484 | INFO | Uploaded 36GiB to GPU in 1.60784s (22.3903GiB/s) | 20260507 14:27:18.484 | INFO | Downloaded 34GiB to GPU in 1.52741s (22.2599GiB/s) | 20260507 14:27:23.623 | INFO | Analyzing chunk frames = [210432,210944) | 20260507 14:27:23.623 | INFO | MemInfo: { | 20260507 14:27:23.623 | INFO | "gpu" : " 17 GB", | 20260507 14:27:23.623 | INFO | "huge" : "", | 20260507 14:27:23.623 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:23.623 | INFO | "malloc" : " 14 GB", | 20260507 14:27:23.623 | INFO | "pinned" : " 29 GB" | 20260507 14:27:23.623 | INFO | } | 20260507 14:27:23.623 | INFO | Perf report for the last 5.13911 seconds: | 20260507 14:27:23.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:23.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:27:23.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:27:23.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:23.623 | INFO | Uploaded 36GiB to GPU in 1.60736s (22.397GiB/s) | 20260507 14:27:23.623 | INFO | Downloaded 34GiB to GPU in 1.52732s (22.2612GiB/s) | 20260507 14:27:28.762 | INFO | Analyzing chunk frames = [210944,211456) | 20260507 14:27:28.762 | INFO | MemInfo: { | 20260507 14:27:28.762 | INFO | "gpu" : " 17 GB", | 20260507 14:27:28.762 | INFO | "huge" : "", | 20260507 14:27:28.762 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:28.762 | INFO | "malloc" : " 14 GB", | 20260507 14:27:28.762 | INFO | "pinned" : " 29 GB" | 20260507 14:27:28.762 | INFO | } | 20260507 14:27:28.762 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:27:28.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:28.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:27:28.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:27:28.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:28.762 | INFO | Uploaded 36GiB to GPU in 1.60551s (22.4227GiB/s) | 20260507 14:27:28.762 | INFO | Downloaded 34GiB to GPU in 1.52467s (22.2999GiB/s) | 20260507 14:27:33.901 | INFO | Analyzing chunk frames = [211456,211968) | 20260507 14:27:33.901 | INFO | MemInfo: { | 20260507 14:27:33.901 | INFO | "gpu" : " 17 GB", | 20260507 14:27:33.901 | INFO | "huge" : "", | 20260507 14:27:33.901 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:33.901 | INFO | "malloc" : " 14 GB", | 20260507 14:27:33.901 | INFO | "pinned" : " 29 GB" | 20260507 14:27:33.901 | INFO | } | 20260507 14:27:33.901 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:27:33.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:33.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.901 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:27:33.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.902 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:27:33.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:33.902 | INFO | Uploaded 36GiB to GPU in 1.60652s (22.4088GiB/s) | 20260507 14:27:33.902 | INFO | Downloaded 34GiB to GPU in 1.5252s (22.2922GiB/s) | 20260507 14:27:39.041 | INFO | Analyzing chunk frames = [211968,212480) | 20260507 14:27:39.041 | INFO | MemInfo: { | 20260507 14:27:39.041 | INFO | "gpu" : " 17 GB", | 20260507 14:27:39.041 | INFO | "huge" : "", | 20260507 14:27:39.041 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:39.041 | INFO | "malloc" : " 14 GB", | 20260507 14:27:39.041 | INFO | "pinned" : " 29 GB" | 20260507 14:27:39.041 | INFO | } | 20260507 14:27:39.041 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:27:39.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:39.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.041 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:27:39.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.041 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:27:39.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:39.042 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4191GiB/s) | 20260507 14:27:39.042 | INFO | Downloaded 34GiB to GPU in 1.5259s (22.282GiB/s) | 20260507 14:27:44.180 | INFO | Analyzing chunk frames = [212480,212992) | 20260507 14:27:44.180 | INFO | MemInfo: { | 20260507 14:27:44.180 | INFO | "gpu" : " 17 GB", | 20260507 14:27:44.180 | INFO | "huge" : "", | 20260507 14:27:44.180 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:44.180 | INFO | "malloc" : " 14 GB", | 20260507 14:27:44.180 | INFO | "pinned" : " 29 GB" | 20260507 14:27:44.180 | INFO | } | 20260507 14:27:44.180 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:27:44.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:44.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.180 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:27:44.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.180 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:27:44.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:44.181 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:27:44.181 | INFO | Total mem usage: 15.5 GiB | 20260507 14:27:44.181 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:27:44.181 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -0.315%) | 20260507 14:27:44.181 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:27:44.181 | INFO | High Water Breakdown: | 20260507 14:27:44.181 | INFO | 59.3% : 9.17 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:27:44.181 | INFO | 40.6% : 6.28 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:27:44.181 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:27:44.181 | INFO | ------------------------------------------ | 20260507 14:27:44.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:27:44.181 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:27:44.181 | INFO | Total mem usage: 26.5 GiB | 20260507 14:27:44.181 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:27:44.181 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:27:44.181 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:27:44.181 | INFO | High Water Breakdown: | 20260507 14:27:44.181 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:27:44.181 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:27:44.181 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.409GiB/s) | 20260507 14:27:44.181 | INFO | Downloaded 34GiB to GPU in 1.52535s (22.2899GiB/s) | 20260507 14:27:44.181 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:27:44.181 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:27:44.181 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:27:44.181 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:27:44.181 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:27:44.181 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:27:44.181 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:27:44.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:27:44.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:27:44.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:27:44.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:27:44.181 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:27:44.181 | INFO | ------------------------------------------ | 20260507 14:27:44.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:27:44.181 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:27:44.181 | INFO | Total mem usage: 27.8 GiB | 20260507 14:27:44.181 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:27:44.181 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:27:44.181 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:27:44.181 | INFO | High Water Breakdown: | 20260507 14:27:44.181 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:27:44.181 | INFO | ------------------------------------------ | 20260507 14:27:44.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:27:44.181 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:27:44.181 | INFO | Total mem usage: 15.7 GiB | 20260507 14:27:44.181 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:27:44.181 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:27:44.181 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:27:44.181 | INFO | High Water Breakdown: | 20260507 14:27:44.181 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:27:44.181 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:27:44.181 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:27:44.181 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:27:44.181 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:27:44.181 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:27:44.181 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:27:44.181 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:27:44.181 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:27:44.181 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:27:44.181 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:27:44.181 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:27:44.181 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:27:44.181 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:27:44.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:27:44.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:27:44.181 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:27:44.181 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:27:44.181 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:27:44.181 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:27:44.181 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:27:44.181 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:27:44.182 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:27:44.182 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:27:44.182 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:27:44.182 | INFO | ------------------------------------------ | 20260507 14:27:44.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:27:44.182 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:27:44.182 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:27:44.182 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:27:44.182 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.37e-03 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 10.97%, Baseline -- 3.29e+00 / -nan, 3.96e-02 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 3.06%, Histogram -- 9.19e-01 / -nan, 5.09e-02 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 16.10%, DME -- 3.87e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.72e+01 / -nan, 4.38e-02 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.20e+00 / -nan, 1.31e-01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 3.65%, Metrics -- 1.10e+00 / -nan, 1.76e-01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.09e-01 / -nan, 5.99e-02 / 0.00e+00 | 20260507 14:27:44.182 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:27:44.182 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:27:44.182 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:27:44.182 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 18.66%, Upload -- 1.68e+01 / 0.00e+00, 4.33e-01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 17.93%, Download -- 1.61e+01 / 0.00e+00, 7.60e-01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 33.36%, Compute -- 3.00e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 1.39e-04 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 29.47%, LockCompute -- 2.65e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:27:44.182 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.16e-02 / 0.00e+00 | 20260507 14:27:44.278 | INFO | Starting a round of Baz IO | 20260507 14:27:47.175 | INFO | Populating RTMetrics report | 20260507 14:27:47.176 | INFO | RTMetrics report populated | 20260507 14:27:47.176 | INFO | Opening tmp RTMetrics json file | 20260507 14:27:47.259 | INFO | Serializing RTMetrics to json | 20260507 14:27:47.340 | INFO | Serialization done, swapping tmp file now... | 20260507 14:27:47.359 | INFO | RTMetrics done | 20260507 14:27:47.395 | INFO | Finished a round of Baz IO: Wrote 1.661 GiB in 3.116 seconds | 20260507 14:27:47.395 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.99% / 0.11% / 40.90% | 20260507 14:27:47.395 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.15097% / 0.000317891% / 3.97364e-05% | 20260507 14:27:47.395 | INFO | Baz IO overall: | 20260507 14:27:47.395 | INFO | BytesWritten : 70980730594 | 20260507 14:27:47.395 | INFO | NumEvents : 33267586414 | 20260507 14:27:49.319 | INFO | Analyzing chunk frames = [212992,213504) | 20260507 14:27:49.319 | INFO | MemInfo: { | 20260507 14:27:49.319 | INFO | "gpu" : " 17 GB", | 20260507 14:27:49.319 | INFO | "huge" : "", | 20260507 14:27:49.319 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:49.319 | INFO | "malloc" : " 14 GB", | 20260507 14:27:49.319 | INFO | "pinned" : " 29 GB" | 20260507 14:27:49.319 | INFO | } | 20260507 14:27:49.319 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:27:49.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:49.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:49.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:49.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:49.319 | INFO | ANALYSIS : 57.4%, 2.9, 86.1ms, 0 | 20260507 14:27:49.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:49.320 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 14:27:49.320 | INFO | PRE_HQ : 56.4%, 2.1, 60.2ms, 0 | 20260507 14:27:49.320 | INFO | BAZWRITER : 60.2%, 4.7, 145.2ms, 0 | 20260507 14:27:49.320 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 14:27:49.320 | INFO | Uploaded 36GiB to GPU in 1.6627s (21.6515GiB/s) | 20260507 14:27:49.320 | INFO | Downloaded 38GiB to GPU in 1.72973s (21.9687GiB/s) | 20260507 14:27:54.459 | INFO | Analyzing chunk frames = [213504,214016) | 20260507 14:27:54.459 | INFO | MemInfo: { | 20260507 14:27:54.459 | INFO | "gpu" : " 17 GB", | 20260507 14:27:54.459 | INFO | "huge" : "", | 20260507 14:27:54.459 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:54.459 | INFO | "malloc" : " 14 GB", | 20260507 14:27:54.459 | INFO | "pinned" : " 29 GB" | 20260507 14:27:54.459 | INFO | } | 20260507 14:27:54.459 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:27:54.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:54.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:27:54.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | PRE_HQ : 29.8%, 1.0, 16.0ms, 0 | 20260507 14:27:54.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:54.459 | INFO | Uploaded 36GiB to GPU in 1.60394s (22.4447GiB/s) | 20260507 14:27:54.459 | INFO | Downloaded 34GiB to GPU in 1.52254s (22.331GiB/s) | 20260507 14:27:59.598 | INFO | Analyzing chunk frames = [214016,214528) | 20260507 14:27:59.598 | INFO | MemInfo: { | 20260507 14:27:59.598 | INFO | "gpu" : " 17 GB", | 20260507 14:27:59.598 | INFO | "huge" : "", | 20260507 14:27:59.598 | INFO | "huge_shared" : " 29 GB", | 20260507 14:27:59.598 | INFO | "malloc" : " 14 GB", | 20260507 14:27:59.598 | INFO | "pinned" : " 29 GB" | 20260507 14:27:59.598 | INFO | } | 20260507 14:27:59.598 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:27:59.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:27:59.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.598 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:27:59.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.598 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:27:59.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:27:59.599 | INFO | Uploaded 36GiB to GPU in 1.60739s (22.3965GiB/s) | 20260507 14:27:59.599 | INFO | Downloaded 34GiB to GPU in 1.52761s (22.257GiB/s) | 20260507 14:28:04.737 | INFO | Analyzing chunk frames = [214528,215040) | 20260507 14:28:04.737 | INFO | MemInfo: { | 20260507 14:28:04.737 | INFO | "gpu" : " 17 GB", | 20260507 14:28:04.737 | INFO | "huge" : "", | 20260507 14:28:04.737 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:04.737 | INFO | "malloc" : " 14 GB", | 20260507 14:28:04.737 | INFO | "pinned" : " 29 GB" | 20260507 14:28:04.737 | INFO | } | 20260507 14:28:04.738 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:28:04.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:04.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:28:04.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:28:04.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:04.738 | INFO | Uploaded 36GiB to GPU in 1.60739s (22.3966GiB/s) | 20260507 14:28:04.738 | INFO | Downloaded 34GiB to GPU in 1.52693s (22.2669GiB/s) | 20260507 14:28:09.877 | INFO | Analyzing chunk frames = [215040,215552) | 20260507 14:28:09.877 | INFO | MemInfo: { | 20260507 14:28:09.877 | INFO | "gpu" : " 17 GB", | 20260507 14:28:09.877 | INFO | "huge" : "", | 20260507 14:28:09.877 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:09.877 | INFO | "malloc" : " 14 GB", | 20260507 14:28:09.877 | INFO | "pinned" : " 29 GB" | 20260507 14:28:09.877 | INFO | } | 20260507 14:28:09.877 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:28:09.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:09.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:28:09.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 14:28:09.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:09.877 | INFO | Uploaded 36GiB to GPU in 1.60519s (22.4272GiB/s) | 20260507 14:28:09.877 | INFO | Downloaded 34GiB to GPU in 1.52413s (22.3077GiB/s) | 20260507 14:28:15.016 | INFO | Analyzing chunk frames = [215552,216064) | 20260507 14:28:15.016 | INFO | MemInfo: { | 20260507 14:28:15.016 | INFO | "gpu" : " 17 GB", | 20260507 14:28:15.016 | INFO | "huge" : "", | 20260507 14:28:15.016 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:15.016 | INFO | "malloc" : " 14 GB", | 20260507 14:28:15.016 | INFO | "pinned" : " 29 GB" | 20260507 14:28:15.016 | INFO | } | 20260507 14:28:15.016 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:28:15.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:15.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.016 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:28:15.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.016 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:28:15.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:15.017 | INFO | Uploaded 36GiB to GPU in 1.60585s (22.418GiB/s) | 20260507 14:28:15.017 | INFO | Downloaded 34GiB to GPU in 1.52509s (22.2937GiB/s) | 20260507 14:28:20.155 | INFO | Analyzing chunk frames = [216064,216576) | 20260507 14:28:20.155 | INFO | MemInfo: { | 20260507 14:28:20.155 | INFO | "gpu" : " 17 GB", | 20260507 14:28:20.155 | INFO | "huge" : "", | 20260507 14:28:20.155 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:20.155 | INFO | "malloc" : " 14 GB", | 20260507 14:28:20.156 | INFO | "pinned" : " 29 GB" | 20260507 14:28:20.156 | INFO | } | 20260507 14:28:20.156 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:28:20.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:20.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:28:20.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:28:20.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:20.156 | INFO | Uploaded 36GiB to GPU in 1.60578s (22.419GiB/s) | 20260507 14:28:20.156 | INFO | Downloaded 34GiB to GPU in 1.52601s (22.2803GiB/s) | 20260507 14:28:25.295 | INFO | Analyzing chunk frames = [216576,217088) | 20260507 14:28:25.295 | INFO | MemInfo: { | 20260507 14:28:25.295 | INFO | "gpu" : " 17 GB", | 20260507 14:28:25.295 | INFO | "huge" : "", | 20260507 14:28:25.295 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:25.295 | INFO | "malloc" : " 14 GB", | 20260507 14:28:25.295 | INFO | "pinned" : " 29 GB" | 20260507 14:28:25.295 | INFO | } | 20260507 14:28:25.295 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:28:25.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:25.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:28:25.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:28:25.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:25.295 | INFO | Uploaded 36GiB to GPU in 1.60598s (22.4163GiB/s) | 20260507 14:28:25.295 | INFO | Downloaded 34GiB to GPU in 1.5259s (22.2819GiB/s) | 20260507 14:28:25.393 | INFO | Starting a round of Baz IO | 20260507 14:28:28.290 | INFO | Populating RTMetrics report | 20260507 14:28:28.290 | INFO | RTMetrics report populated | 20260507 14:28:28.290 | INFO | Opening tmp RTMetrics json file | 20260507 14:28:28.291 | INFO | Serializing RTMetrics to json | 20260507 14:28:28.293 | INFO | Serialization done, swapping tmp file now... | 20260507 14:28:28.293 | INFO | RTMetrics done | 20260507 14:28:28.372 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 2.979 seconds | 20260507 14:28:28.372 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.88% / 0.10% / 41.01% | 20260507 14:28:28.372 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.10051% / 0.000325839% / 2.78155e-05% | 20260507 14:28:28.372 | INFO | Baz IO overall: | 20260507 14:28:28.372 | INFO | BytesWritten : 72765597895 | 20260507 14:28:28.372 | INFO | NumEvents : 34139090393 | 20260507 14:28:30.435 | INFO | Analyzing chunk frames = [217088,217600) | 20260507 14:28:30.435 | INFO | MemInfo: { | 20260507 14:28:30.435 | INFO | "gpu" : " 17 GB", | 20260507 14:28:30.435 | INFO | "huge" : "", | 20260507 14:28:30.435 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:30.435 | INFO | "malloc" : " 14 GB", | 20260507 14:28:30.435 | INFO | "pinned" : " 29 GB" | 20260507 14:28:30.435 | INFO | } | 20260507 14:28:30.435 | INFO | Perf report for the last 5.14021 seconds: | 20260507 14:28:30.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:30.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:30.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:30.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:30.435 | INFO | ANALYSIS : 57.4%, 2.9, 86.7ms, 0 | 20260507 14:28:30.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:30.436 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 14:28:30.436 | INFO | PRE_HQ : 56.3%, 2.1, 60.0ms, 0 | 20260507 14:28:30.436 | INFO | BAZWRITER : 57.5%, 4.6, 135.9ms, 0 | 20260507 14:28:30.436 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 14:28:30.436 | INFO | Uploaded 36GiB to GPU in 1.66157s (21.6662GiB/s) | 20260507 14:28:30.436 | INFO | Downloaded 38GiB to GPU in 1.72819s (21.9883GiB/s) | 20260507 14:28:35.573 | INFO | Analyzing chunk frames = [217600,218112) | 20260507 14:28:35.573 | INFO | MemInfo: { | 20260507 14:28:35.573 | INFO | "gpu" : " 17 GB", | 20260507 14:28:35.573 | INFO | "huge" : "", | 20260507 14:28:35.573 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:35.573 | INFO | "malloc" : " 14 GB", | 20260507 14:28:35.574 | INFO | "pinned" : " 29 GB" | 20260507 14:28:35.574 | INFO | } | 20260507 14:28:35.574 | INFO | Perf report for the last 5.13824 seconds: | 20260507 14:28:35.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:35.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:28:35.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:28:35.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:35.574 | INFO | Uploaded 36GiB to GPU in 1.60304s (22.4573GiB/s) | 20260507 14:28:35.574 | INFO | Downloaded 34GiB to GPU in 1.52304s (22.3237GiB/s) | 20260507 14:28:40.714 | INFO | Analyzing chunk frames = [218112,218624) | 20260507 14:28:40.714 | INFO | MemInfo: { | 20260507 14:28:40.714 | INFO | "gpu" : " 17 GB", | 20260507 14:28:40.714 | INFO | "huge" : "", | 20260507 14:28:40.714 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:40.714 | INFO | "malloc" : " 14 GB", | 20260507 14:28:40.714 | INFO | "pinned" : " 29 GB" | 20260507 14:28:40.714 | INFO | } | 20260507 14:28:40.714 | INFO | Perf report for the last 5.14043 seconds: | 20260507 14:28:40.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:40.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:28:40.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:28:40.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:40.714 | INFO | Uploaded 36GiB to GPU in 1.60692s (22.403GiB/s) | 20260507 14:28:40.715 | INFO | Downloaded 34GiB to GPU in 1.52729s (22.2617GiB/s) | 20260507 14:28:45.852 | INFO | Analyzing chunk frames = [218624,219136) | 20260507 14:28:45.852 | INFO | MemInfo: { | 20260507 14:28:45.852 | INFO | "gpu" : " 17 GB", | 20260507 14:28:45.852 | INFO | "huge" : "", | 20260507 14:28:45.852 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:45.853 | INFO | "malloc" : " 14 GB", | 20260507 14:28:45.853 | INFO | "pinned" : " 29 GB" | 20260507 14:28:45.853 | INFO | } | 20260507 14:28:45.853 | INFO | Perf report for the last 5.13865 seconds: | 20260507 14:28:45.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:45.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:28:45.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:28:45.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:45.853 | INFO | Uploaded 36GiB to GPU in 1.60698s (22.4023GiB/s) | 20260507 14:28:45.853 | INFO | Downloaded 34GiB to GPU in 1.5277s (22.2557GiB/s) | 20260507 14:28:50.991 | INFO | Analyzing chunk frames = [219136,219648) | 20260507 14:28:50.991 | INFO | MemInfo: { | 20260507 14:28:50.991 | INFO | "gpu" : " 17 GB", | 20260507 14:28:50.991 | INFO | "huge" : "", | 20260507 14:28:50.991 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:50.991 | INFO | "malloc" : " 14 GB", | 20260507 14:28:50.991 | INFO | "pinned" : " 29 GB" | 20260507 14:28:50.991 | INFO | } | 20260507 14:28:50.992 | INFO | Perf report for the last 5.13885 seconds: | 20260507 14:28:50.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:50.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:28:50.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:28:50.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:50.992 | INFO | Uploaded 36GiB to GPU in 1.60571s (22.4199GiB/s) | 20260507 14:28:50.992 | INFO | Downloaded 34GiB to GPU in 1.52491s (22.2964GiB/s) | 20260507 14:28:56.131 | INFO | Analyzing chunk frames = [219648,220160) | 20260507 14:28:56.131 | INFO | MemInfo: { | 20260507 14:28:56.131 | INFO | "gpu" : " 17 GB", | 20260507 14:28:56.131 | INFO | "huge" : "", | 20260507 14:28:56.131 | INFO | "huge_shared" : " 29 GB", | 20260507 14:28:56.131 | INFO | "malloc" : " 14 GB", | 20260507 14:28:56.131 | INFO | "pinned" : " 29 GB" | 20260507 14:28:56.131 | INFO | } | 20260507 14:28:56.131 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:28:56.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:28:56.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:28:56.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:28:56.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:28:56.131 | INFO | Uploaded 36GiB to GPU in 1.60589s (22.4175GiB/s) | 20260507 14:28:56.131 | INFO | Downloaded 34GiB to GPU in 1.52526s (22.2913GiB/s) | 20260507 14:29:01.270 | INFO | Analyzing chunk frames = [220160,220672) | 20260507 14:29:01.270 | INFO | MemInfo: { | 20260507 14:29:01.270 | INFO | "gpu" : " 17 GB", | 20260507 14:29:01.270 | INFO | "huge" : "", | 20260507 14:29:01.270 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:01.270 | INFO | "malloc" : " 14 GB", | 20260507 14:29:01.270 | INFO | "pinned" : " 29 GB" | 20260507 14:29:01.270 | INFO | } | 20260507 14:29:01.270 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:29:01.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:01.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.270 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:29:01.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.270 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:29:01.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:01.271 | INFO | Uploaded 36GiB to GPU in 1.6063s (22.4117GiB/s) | 20260507 14:29:01.271 | INFO | Downloaded 34GiB to GPU in 1.52744s (22.2594GiB/s) | 20260507 14:29:06.409 | INFO | Analyzing chunk frames = [220672,221184) | 20260507 14:29:06.409 | INFO | MemInfo: { | 20260507 14:29:06.409 | INFO | "gpu" : " 17 GB", | 20260507 14:29:06.409 | INFO | "huge" : "", | 20260507 14:29:06.409 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:06.409 | INFO | "malloc" : " 14 GB", | 20260507 14:29:06.409 | INFO | "pinned" : " 29 GB" | 20260507 14:29:06.410 | INFO | } | 20260507 14:29:06.410 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:29:06.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:06.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:29:06.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:29:06.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:06.410 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:29:06.410 | INFO | Total mem usage: 15.5 GiB | 20260507 14:29:06.410 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:29:06.410 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -1.76%) | 20260507 14:29:06.410 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:29:06.410 | INFO | High Water Breakdown: | 20260507 14:29:06.410 | INFO | 59% : 9 GiB ( 98.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:29:06.410 | INFO | 40.8% : 6.22 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:29:06.410 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:29:06.410 | INFO | ------------------------------------------ | 20260507 14:29:06.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:29:06.410 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:29:06.410 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4328GiB/s) | 20260507 14:29:06.410 | INFO | Total mem usage: 26.5 GiB | 20260507 14:29:06.410 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260507 14:29:06.410 | INFO | Downloaded 34GiB to GPU in 1.52611s (22.2789GiB/s) | 20260507 14:29:06.410 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:29:06.410 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:29:06.410 | INFO | High Water Breakdown: | 20260507 14:29:06.410 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:29:06.410 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:29:06.410 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:29:06.410 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:29:06.411 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:29:06.411 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:29:06.411 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:29:06.411 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:29:06.411 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:29:06.411 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:29:06.411 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:29:06.411 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:29:06.411 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:29:06.411 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:29:06.411 | INFO | ------------------------------------------ | 20260507 14:29:06.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:29:06.411 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:29:06.411 | INFO | Total mem usage: 27.8 GiB | 20260507 14:29:06.411 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:29:06.411 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:29:06.411 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:29:06.411 | INFO | High Water Breakdown: | 20260507 14:29:06.411 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:29:06.411 | INFO | ------------------------------------------ | 20260507 14:29:06.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:29:06.411 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:29:06.411 | INFO | Total mem usage: 15.7 GiB | 20260507 14:29:06.411 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:29:06.411 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:29:06.411 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:29:06.411 | INFO | High Water Breakdown: | 20260507 14:29:06.411 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:29:06.411 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:29:06.411 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:29:06.411 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:29:06.411 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:29:06.411 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:29:06.411 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:29:06.411 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:29:06.411 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:29:06.411 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:29:06.411 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:29:06.411 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:29:06.411 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:29:06.411 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:29:06.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:29:06.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:29:06.411 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:29:06.411 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:29:06.411 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:29:06.411 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:29:06.411 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:29:06.411 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:29:06.411 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:29:06.411 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:29:06.411 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:29:06.411 | INFO | ------------------------------------------ | 20260507 14:29:06.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:29:06.411 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:29:06.411 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:29:06.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:29:06.411 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.35e-03 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 10.97%, Baseline -- 3.29e+00 / -nan, 1.92e-02 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 3.06%, Histogram -- 9.19e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 16.09%, DME -- 3.86e+01 / -nan, 3.12e+00 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / -nan, 4.38e-02 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.20e+00 / -nan, 1.31e-01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 3.65%, Metrics -- 1.09e+00 / -nan, 1.76e-01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.07e-01 / -nan, 5.95e-02 / 0.00e+00 | 20260507 14:29:06.411 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:29:06.411 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:29:06.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:29:06.411 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 18.71%, Upload -- 1.68e+01 / 0.00e+00, 4.57e-01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 17.99%, Download -- 1.62e+01 / 0.00e+00, 7.88e-01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 33.43%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 0.00%, LockDownload -- 4.33e-04 / 0.00e+00, 9.88e-05 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 29.29%, LockCompute -- 2.63e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260507 14:29:06.411 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260507 14:29:06.507 | INFO | Starting a round of Baz IO | 20260507 14:29:09.403 | INFO | Populating RTMetrics report | 20260507 14:29:09.404 | INFO | RTMetrics report populated | 20260507 14:29:09.404 | INFO | Opening tmp RTMetrics json file | 20260507 14:29:09.405 | INFO | Serializing RTMetrics to json | 20260507 14:29:09.406 | INFO | Serialization done, swapping tmp file now... | 20260507 14:29:09.407 | INFO | RTMetrics done | 20260507 14:29:09.479 | INFO | Finished a round of Baz IO: Wrote 1.664 GiB in 2.972 seconds | 20260507 14:29:09.479 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.78% / 0.10% / 41.12% | 20260507 14:29:09.479 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.05544% / 0.00029405% / 3.57628e-05% | 20260507 14:29:09.479 | INFO | Baz IO overall: | 20260507 14:29:09.479 | INFO | BytesWritten : 74552276931 | 20260507 14:29:09.479 | INFO | NumEvents : 35011324061 | 20260507 14:29:11.549 | INFO | Analyzing chunk frames = [221184,221696) | 20260507 14:29:11.549 | INFO | MemInfo: { | 20260507 14:29:11.549 | INFO | "gpu" : " 17 GB", | 20260507 14:29:11.549 | INFO | "huge" : "", | 20260507 14:29:11.549 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:11.549 | INFO | "malloc" : " 14 GB", | 20260507 14:29:11.549 | INFO | "pinned" : " 29 GB" | 20260507 14:29:11.549 | INFO | } | 20260507 14:29:11.549 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:29:11.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:11.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:11.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:11.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:11.549 | INFO | ANALYSIS : 57.4%, 2.9, 88.3ms, 0 | 20260507 14:29:11.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:11.549 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 14:29:11.549 | INFO | PRE_HQ : 56.3%, 2.1, 60.0ms, 0 | 20260507 14:29:11.549 | INFO | BAZWRITER : 57.3%, 4.1, 120.2ms, 0 | 20260507 14:29:11.549 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 14:29:11.550 | INFO | Uploaded 36GiB to GPU in 1.64648s (21.8648GiB/s) | 20260507 14:29:11.550 | INFO | Downloaded 38GiB to GPU in 1.70999s (22.2224GiB/s) | 20260507 14:29:16.688 | INFO | Analyzing chunk frames = [221696,222208) | 20260507 14:29:16.688 | INFO | MemInfo: { | 20260507 14:29:16.688 | INFO | "gpu" : " 17 GB", | 20260507 14:29:16.688 | INFO | "huge" : "", | 20260507 14:29:16.688 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:16.688 | INFO | "malloc" : " 14 GB", | 20260507 14:29:16.688 | INFO | "pinned" : " 29 GB" | 20260507 14:29:16.688 | INFO | } | 20260507 14:29:16.688 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:29:16.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:16.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.688 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:29:16.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.688 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:29:16.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:16.689 | INFO | Uploaded 36GiB to GPU in 1.60349s (22.451GiB/s) | 20260507 14:29:16.689 | INFO | Downloaded 34GiB to GPU in 1.52229s (22.3347GiB/s) | 20260507 14:29:21.827 | INFO | Analyzing chunk frames = [222208,222720) | 20260507 14:29:21.827 | INFO | MemInfo: { | 20260507 14:29:21.827 | INFO | "gpu" : " 17 GB", | 20260507 14:29:21.827 | INFO | "huge" : "", | 20260507 14:29:21.828 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:21.828 | INFO | "malloc" : " 14 GB", | 20260507 14:29:21.828 | INFO | "pinned" : " 29 GB" | 20260507 14:29:21.828 | INFO | } | 20260507 14:29:21.828 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:29:21.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:21.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:29:21.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:29:21.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:21.828 | INFO | Uploaded 36GiB to GPU in 1.60768s (22.3925GiB/s) | 20260507 14:29:21.828 | INFO | Downloaded 34GiB to GPU in 1.52764s (22.2565GiB/s) | 20260507 14:29:26.967 | INFO | Analyzing chunk frames = [222720,223232) | 20260507 14:29:26.967 | INFO | MemInfo: { | 20260507 14:29:26.967 | INFO | "gpu" : " 17 GB", | 20260507 14:29:26.967 | INFO | "huge" : "", | 20260507 14:29:26.967 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:26.967 | INFO | "malloc" : " 14 GB", | 20260507 14:29:26.967 | INFO | "pinned" : " 29 GB" | 20260507 14:29:26.967 | INFO | } | 20260507 14:29:26.967 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:29:26.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:26.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.967 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:29:26.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.967 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:29:26.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:26.968 | INFO | Uploaded 36GiB to GPU in 1.60706s (22.4012GiB/s) | 20260507 14:29:26.968 | INFO | Downloaded 34GiB to GPU in 1.52719s (22.2631GiB/s) | 20260507 14:29:32.106 | INFO | Analyzing chunk frames = [223232,223744) | 20260507 14:29:32.106 | INFO | MemInfo: { | 20260507 14:29:32.106 | INFO | "gpu" : " 17 GB", | 20260507 14:29:32.106 | INFO | "huge" : "", | 20260507 14:29:32.106 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:32.106 | INFO | "malloc" : " 14 GB", | 20260507 14:29:32.106 | INFO | "pinned" : " 29 GB" | 20260507 14:29:32.106 | INFO | } | 20260507 14:29:32.107 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:29:32.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:32.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 14:29:32.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:29:32.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:32.107 | INFO | Uploaded 36GiB to GPU in 1.60651s (22.4089GiB/s) | 20260507 14:29:32.107 | INFO | Downloaded 34GiB to GPU in 1.52527s (22.2911GiB/s) | 20260507 14:29:37.245 | INFO | Analyzing chunk frames = [223744,224256) | 20260507 14:29:37.246 | INFO | MemInfo: { | 20260507 14:29:37.246 | INFO | "gpu" : " 17 GB", | 20260507 14:29:37.246 | INFO | "huge" : "", | 20260507 14:29:37.246 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:37.246 | INFO | "malloc" : " 14 GB", | 20260507 14:29:37.246 | INFO | "pinned" : " 29 GB" | 20260507 14:29:37.246 | INFO | } | 20260507 14:29:37.246 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:29:37.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:37.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:29:37.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:29:37.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:37.246 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260507 14:29:37.246 | INFO | Downloaded 34GiB to GPU in 1.52464s (22.3003GiB/s) | 20260507 14:29:42.385 | INFO | Analyzing chunk frames = [224256,224768) | 20260507 14:29:42.385 | INFO | MemInfo: { | 20260507 14:29:42.385 | INFO | "gpu" : " 17 GB", | 20260507 14:29:42.385 | INFO | "huge" : "", | 20260507 14:29:42.385 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:42.385 | INFO | "malloc" : " 14 GB", | 20260507 14:29:42.385 | INFO | "pinned" : " 29 GB" | 20260507 14:29:42.385 | INFO | } | 20260507 14:29:42.385 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:29:42.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:42.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.385 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:29:42.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.385 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:29:42.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:42.386 | INFO | Uploaded 36GiB to GPU in 1.60659s (22.4077GiB/s) | 20260507 14:29:42.386 | INFO | Downloaded 34GiB to GPU in 1.52664s (22.2712GiB/s) | 20260507 14:29:47.524 | INFO | Analyzing chunk frames = [224768,225280) | 20260507 14:29:47.524 | INFO | MemInfo: { | 20260507 14:29:47.524 | INFO | "gpu" : " 17 GB", | 20260507 14:29:47.524 | INFO | "huge" : "", | 20260507 14:29:47.524 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:47.524 | INFO | "malloc" : " 14 GB", | 20260507 14:29:47.524 | INFO | "pinned" : " 29 GB" | 20260507 14:29:47.524 | INFO | } | 20260507 14:29:47.524 | INFO | Perf report for the last 5.13908 seconds: | 20260507 14:29:47.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:47.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.524 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 14:29:47.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.525 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:29:47.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:47.525 | INFO | Uploaded 36GiB to GPU in 1.60652s (22.4086GiB/s) | 20260507 14:29:47.525 | INFO | Downloaded 34GiB to GPU in 1.52631s (22.2759GiB/s) | 20260507 14:29:47.621 | INFO | Starting a round of Baz IO | 20260507 14:29:50.610 | INFO | Populating RTMetrics report | 20260507 14:29:50.611 | INFO | RTMetrics report populated | 20260507 14:29:50.611 | INFO | Opening tmp RTMetrics json file | 20260507 14:29:50.651 | INFO | Serializing RTMetrics to json | 20260507 14:29:50.674 | INFO | Serialization done, swapping tmp file now... | 20260507 14:29:50.679 | INFO | RTMetrics done | 20260507 14:29:50.755 | INFO | Finished a round of Baz IO: Wrote 1.665 GiB in 3.133 seconds | 20260507 14:29:50.755 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.68% / 0.10% / 41.22% | 20260507 14:29:50.755 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.01677% / 0.000305971% / 3.97364e-06% | 20260507 14:29:50.755 | INFO | Baz IO overall: | 20260507 14:29:50.755 | INFO | BytesWritten : 76340196889 | 20260507 14:29:50.755 | INFO | NumEvents : 35884250619 | 20260507 14:29:52.664 | INFO | Analyzing chunk frames = [225280,225792) | 20260507 14:29:52.664 | INFO | MemInfo: { | 20260507 14:29:52.664 | INFO | "gpu" : " 17 GB", | 20260507 14:29:52.664 | INFO | "huge" : "", | 20260507 14:29:52.664 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:52.664 | INFO | "malloc" : " 14 GB", | 20260507 14:29:52.664 | INFO | "pinned" : " 29 GB" | 20260507 14:29:52.664 | INFO | } | 20260507 14:29:52.664 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:29:52.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:52.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:52.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:52.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:52.664 | INFO | ANALYSIS : 59.2%, 2.9, 89.4ms, 0 | 20260507 14:29:52.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:52.664 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260507 14:29:52.664 | INFO | PRE_HQ : 57.5%, 2.2, 65.9ms, 0 | 20260507 14:29:52.664 | INFO | BAZWRITER : 60.5%, 4.3, 134.7ms, 0 | 20260507 14:29:52.664 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 14:29:52.664 | INFO | Uploaded 36GiB to GPU in 1.74272s (20.6573GiB/s) | 20260507 14:29:52.664 | INFO | Downloaded 38GiB to GPU in 1.81641s (20.9204GiB/s) | 20260507 14:29:57.803 | INFO | Analyzing chunk frames = [225792,226304) | 20260507 14:29:57.803 | INFO | MemInfo: { | 20260507 14:29:57.803 | INFO | "gpu" : " 17 GB", | 20260507 14:29:57.803 | INFO | "huge" : "", | 20260507 14:29:57.803 | INFO | "huge_shared" : " 29 GB", | 20260507 14:29:57.803 | INFO | "malloc" : " 14 GB", | 20260507 14:29:57.803 | INFO | "pinned" : " 29 GB" | 20260507 14:29:57.803 | INFO | } | 20260507 14:29:57.803 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:29:57.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:29:57.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260507 14:29:57.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:29:57.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:29:57.803 | INFO | Uploaded 36GiB to GPU in 1.58307s (22.7406GiB/s) | 20260507 14:29:57.803 | INFO | Downloaded 34GiB to GPU in 1.53105s (22.207GiB/s) | 20260507 14:30:02.942 | INFO | Analyzing chunk frames = [226304,226816) | 20260507 14:30:02.942 | INFO | MemInfo: { | 20260507 14:30:02.942 | INFO | "gpu" : " 17 GB", | 20260507 14:30:02.942 | INFO | "huge" : "", | 20260507 14:30:02.942 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:02.942 | INFO | "malloc" : " 14 GB", | 20260507 14:30:02.942 | INFO | "pinned" : " 29 GB" | 20260507 14:30:02.942 | INFO | } | 20260507 14:30:02.942 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:30:02.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:02.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.943 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:30:02.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.943 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:30:02.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:02.943 | INFO | Uploaded 36GiB to GPU in 1.58887s (22.6576GiB/s) | 20260507 14:30:02.943 | INFO | Downloaded 34GiB to GPU in 1.5367s (22.1254GiB/s) | 20260507 14:30:08.081 | INFO | Analyzing chunk frames = [226816,227328) | 20260507 14:30:08.081 | INFO | MemInfo: { | 20260507 14:30:08.082 | INFO | "gpu" : " 17 GB", | 20260507 14:30:08.082 | INFO | "huge" : "", | 20260507 14:30:08.082 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:08.082 | INFO | "malloc" : " 14 GB", | 20260507 14:30:08.082 | INFO | "pinned" : " 29 GB" | 20260507 14:30:08.082 | INFO | } | 20260507 14:30:08.082 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:30:08.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:08.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 14:30:08.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:30:08.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:08.082 | INFO | Uploaded 36GiB to GPU in 1.58657s (22.6904GiB/s) | 20260507 14:30:08.082 | INFO | Downloaded 34GiB to GPU in 1.53558s (22.1414GiB/s) | 20260507 14:30:13.221 | INFO | Analyzing chunk frames = [227328,227840) | 20260507 14:30:13.221 | INFO | MemInfo: { | 20260507 14:30:13.221 | INFO | "gpu" : " 17 GB", | 20260507 14:30:13.221 | INFO | "huge" : "", | 20260507 14:30:13.221 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:13.221 | INFO | "malloc" : " 14 GB", | 20260507 14:30:13.221 | INFO | "pinned" : " 29 GB" | 20260507 14:30:13.221 | INFO | } | 20260507 14:30:13.221 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:30:13.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:13.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.221 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 14:30:13.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.221 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:30:13.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:13.222 | INFO | Uploaded 36GiB to GPU in 1.58627s (22.6947GiB/s) | 20260507 14:30:13.222 | INFO | Downloaded 34GiB to GPU in 1.53409s (22.163GiB/s) | 20260507 14:30:18.361 | INFO | Analyzing chunk frames = [227840,228352) | 20260507 14:30:18.361 | INFO | MemInfo: { | 20260507 14:30:18.361 | INFO | "gpu" : " 17 GB", | 20260507 14:30:18.361 | INFO | "huge" : "", | 20260507 14:30:18.361 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:18.361 | INFO | "malloc" : " 14 GB", | 20260507 14:30:18.361 | INFO | "pinned" : " 29 GB" | 20260507 14:30:18.361 | INFO | } | 20260507 14:30:18.361 | INFO | Perf report for the last 5.14009 seconds: | 20260507 14:30:18.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:18.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.361 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:30:18.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.361 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:30:18.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:18.362 | INFO | Uploaded 36GiB to GPU in 1.58692s (22.6855GiB/s) | 20260507 14:30:18.362 | INFO | Downloaded 34GiB to GPU in 1.5346s (22.1557GiB/s) | 20260507 14:30:23.500 | INFO | Analyzing chunk frames = [228352,228864) | 20260507 14:30:23.501 | INFO | MemInfo: { | 20260507 14:30:23.501 | INFO | "gpu" : " 17 GB", | 20260507 14:30:23.501 | INFO | "huge" : "", | 20260507 14:30:23.501 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:23.501 | INFO | "malloc" : " 14 GB", | 20260507 14:30:23.501 | INFO | "pinned" : " 29 GB" | 20260507 14:30:23.501 | INFO | } | 20260507 14:30:23.501 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:30:23.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:23.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:30:23.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:30:23.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:23.501 | INFO | Uploaded 36GiB to GPU in 1.5865s (22.6914GiB/s) | 20260507 14:30:23.501 | INFO | Downloaded 34GiB to GPU in 1.53535s (22.1448GiB/s) | 20260507 14:30:28.639 | INFO | Analyzing chunk frames = [228864,229376) | 20260507 14:30:28.639 | INFO | MemInfo: { | 20260507 14:30:28.639 | INFO | "gpu" : " 17 GB", | 20260507 14:30:28.639 | INFO | "huge" : "", | 20260507 14:30:28.639 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:28.639 | INFO | "malloc" : " 14 GB", | 20260507 14:30:28.639 | INFO | "pinned" : " 29 GB" | 20260507 14:30:28.639 | INFO | } | 20260507 14:30:28.639 | INFO | Perf report for the last 5.13836 seconds: | 20260507 14:30:28.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:28.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:30:28.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:30:28.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:28.639 | INFO | Uploaded 36GiB to GPU in 1.5862s (22.6957GiB/s) | 20260507 14:30:28.639 | INFO | Downloaded 34GiB to GPU in 1.53555s (22.1419GiB/s) | 20260507 14:30:28.640 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:30:28.640 | INFO | Total mem usage: 15.5 GiB | 20260507 14:30:28.640 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:30:28.640 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.76%) | 20260507 14:30:28.640 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:30:28.640 | INFO | High Water Breakdown: | 20260507 14:30:28.640 | INFO | 58.5% : 8.67 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:30:28.640 | INFO | 41.3% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:30:28.640 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:30:28.640 | INFO | ------------------------------------------ | 20260507 14:30:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:30:28.640 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:30:28.640 | INFO | Total mem usage: 26.5 GiB | 20260507 14:30:28.640 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260507 14:30:28.640 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 14:30:28.640 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:30:28.640 | INFO | High Water Breakdown: | 20260507 14:30:28.640 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:30:28.640 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:30:28.640 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:30:28.640 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:30:28.640 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:30:28.640 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:30:28.640 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:30:28.640 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:30:28.640 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:30:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:30:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:30:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:30:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:30:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:30:28.640 | INFO | ------------------------------------------ | 20260507 14:30:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:30:28.640 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:30:28.640 | INFO | Total mem usage: 27.8 GiB | 20260507 14:30:28.640 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:30:28.640 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:30:28.640 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:30:28.640 | INFO | High Water Breakdown: | 20260507 14:30:28.640 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:30:28.640 | INFO | ------------------------------------------ | 20260507 14:30:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:30:28.640 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:30:28.640 | INFO | Total mem usage: 15.7 GiB | 20260507 14:30:28.640 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260507 14:30:28.640 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:30:28.640 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260507 14:30:28.640 | INFO | High Water Breakdown: | 20260507 14:30:28.640 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:30:28.640 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:30:28.640 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:30:28.640 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:30:28.640 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:30:28.640 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:30:28.640 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:30:28.640 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:30:28.640 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:30:28.640 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:30:28.640 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:30:28.640 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:30:28.640 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:30:28.640 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:30:28.640 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:30:28.640 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:30:28.640 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:30:28.640 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:30:28.640 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:30:28.640 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:30:28.640 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:30:28.640 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:30:28.640 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:30:28.640 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:30:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:30:28.640 | INFO | ------------------------------------------ | 20260507 14:30:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:30:28.640 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:30:28.640 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:30:28.640 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:30:28.640 | INFO | PerfReport: Full Execution -- 3.01e+01 / 1.17e+02, 1.28e+01 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / 9.49e-02, 4.67e-03 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 11.15%, Baseline -- 3.35e+00 / 9.46e+01, 2.31e+00 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 3.06%, Histogram -- 9.20e-01 / 9.10e-01, 5.20e-02 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 16.07%, DME -- 3.87e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / 1.73e+01, 4.41e-02 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.20e+00 / 2.33e+00, 1.31e-01 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 3.63%, Metrics -- 1.09e+00 / 1.47e+00, 1.75e-01 / 0.00e+00 | 20260507 14:30:28.640 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.07e-01 / 4.94e-01, 5.96e-02 / 0.00e+00 | 20260507 14:30:28.641 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:30:28.641 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:30:28.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:30:28.641 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 2.34e+00 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 33.38%, Compute -- 3.01e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 0.00%, LockDownload -- 4.37e-04 / 0.00e+00, 1.20e-04 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 29.42%, LockCompute -- 2.65e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:30:28.641 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260507 14:30:28.736 | INFO | Starting a round of Baz IO | 20260507 14:30:31.676 | INFO | Populating RTMetrics report | 20260507 14:30:31.676 | INFO | RTMetrics report populated | 20260507 14:30:31.676 | INFO | Opening tmp RTMetrics json file | 20260507 14:30:31.810 | INFO | Serializing RTMetrics to json | 20260507 14:30:31.815 | INFO | Serialization done, swapping tmp file now... | 20260507 14:30:31.819 | INFO | RTMetrics done | 20260507 14:30:32.038 | INFO | Finished a round of Baz IO: Wrote 1.668 GiB in 3.301 seconds | 20260507 14:30:32.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.58% / 0.10% / 41.32% | 20260507 14:30:32.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.98912% / 0.000286102% / 1.58946e-05% | 20260507 14:30:32.038 | INFO | Baz IO overall: | 20260507 14:30:32.038 | INFO | BytesWritten : 78130870380 | 20260507 14:30:32.038 | INFO | NumEvents : 36758590699 | 20260507 14:30:33.778 | INFO | Analyzing chunk frames = [229376,229888) | 20260507 14:30:33.778 | INFO | MemInfo: { | 20260507 14:30:33.778 | INFO | "gpu" : " 17 GB", | 20260507 14:30:33.778 | INFO | "huge" : "", | 20260507 14:30:33.778 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:33.778 | INFO | "malloc" : " 14 GB", | 20260507 14:30:33.779 | INFO | "pinned" : " 29 GB" | 20260507 14:30:33.779 | INFO | } | 20260507 14:30:33.779 | INFO | Perf report for the last 5.13983 seconds: | 20260507 14:30:33.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:33.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:33.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:33.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:33.779 | INFO | ANALYSIS : 58.2%, 2.8, 87.0ms, 0 | 20260507 14:30:33.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:33.779 | INFO | RT_METRICS : 50.1%, 1.0, 26.2ms, 0 | 20260507 14:30:33.779 | INFO | PRE_HQ : 57.4%, 2.3, 67.6ms, 0 | 20260507 14:30:33.779 | INFO | BAZWRITER : 63.8%, 4.4, 142.9ms, 0 | 20260507 14:30:33.779 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260507 14:30:33.779 | INFO | Uploaded 36GiB to GPU in 1.67269s (21.5223GiB/s) | 20260507 14:30:33.779 | INFO | Downloaded 38GiB to GPU in 1.76109s (21.5776GiB/s) | 20260507 14:30:38.918 | INFO | Analyzing chunk frames = [229888,230400) | 20260507 14:30:38.918 | INFO | MemInfo: { | 20260507 14:30:38.918 | INFO | "gpu" : " 17 GB", | 20260507 14:30:38.918 | INFO | "huge" : "", | 20260507 14:30:38.918 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:38.918 | INFO | "malloc" : " 14 GB", | 20260507 14:30:38.918 | INFO | "pinned" : " 29 GB" | 20260507 14:30:38.918 | INFO | } | 20260507 14:30:38.918 | INFO | Perf report for the last 5.13909 seconds: | 20260507 14:30:38.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:38.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.918 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:30:38.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.918 | INFO | PRE_HQ : 41.2%, 1.0, 21.4ms, 0 | 20260507 14:30:38.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:38.919 | INFO | Uploaded 36GiB to GPU in 1.58964s (22.6467GiB/s) | 20260507 14:30:38.919 | INFO | Downloaded 34GiB to GPU in 1.53637s (22.1301GiB/s) | 20260507 14:30:44.057 | INFO | Analyzing chunk frames = [230400,230912) | 20260507 14:30:44.057 | INFO | MemInfo: { | 20260507 14:30:44.057 | INFO | "gpu" : " 17 GB", | 20260507 14:30:44.058 | INFO | "huge" : "", | 20260507 14:30:44.058 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:44.058 | INFO | "malloc" : " 14 GB", | 20260507 14:30:44.058 | INFO | "pinned" : " 29 GB" | 20260507 14:30:44.058 | INFO | } | 20260507 14:30:44.058 | INFO | Perf report for the last 5.13951 seconds: | 20260507 14:30:44.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:44.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:30:44.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | PRE_HQ : 41.1%, 1.0, 21.3ms, 0 | 20260507 14:30:44.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:44.058 | INFO | Uploaded 36GiB to GPU in 1.59282s (22.6014GiB/s) | 20260507 14:30:44.058 | INFO | Downloaded 34GiB to GPU in 1.54136s (22.0585GiB/s) | 20260507 14:30:49.196 | INFO | Analyzing chunk frames = [230912,231424) | 20260507 14:30:49.197 | INFO | MemInfo: { | 20260507 14:30:49.197 | INFO | "gpu" : " 17 GB", | 20260507 14:30:49.197 | INFO | "huge" : "", | 20260507 14:30:49.197 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:49.197 | INFO | "malloc" : " 14 GB", | 20260507 14:30:49.197 | INFO | "pinned" : " 29 GB" | 20260507 14:30:49.197 | INFO | } | 20260507 14:30:49.197 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:30:49.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:49.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260507 14:30:49.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | PRE_HQ : 41.6%, 1.0, 21.7ms, 0 | 20260507 14:30:49.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:49.197 | INFO | Uploaded 36GiB to GPU in 1.59264s (22.604GiB/s) | 20260507 14:30:49.197 | INFO | Downloaded 34GiB to GPU in 1.54099s (22.0637GiB/s) | 20260507 14:30:54.336 | INFO | Analyzing chunk frames = [231424,231936) | 20260507 14:30:54.336 | INFO | MemInfo: { | 20260507 14:30:54.336 | INFO | "gpu" : " 17 GB", | 20260507 14:30:54.336 | INFO | "huge" : "", | 20260507 14:30:54.336 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:54.336 | INFO | "malloc" : " 14 GB", | 20260507 14:30:54.336 | INFO | "pinned" : " 29 GB" | 20260507 14:30:54.336 | INFO | } | 20260507 14:30:54.336 | INFO | Perf report for the last 5.13893 seconds: | 20260507 14:30:54.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:54.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:30:54.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | PRE_HQ : 41.0%, 1.0, 21.3ms, 0 | 20260507 14:30:54.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:54.336 | INFO | Uploaded 36GiB to GPU in 1.59215s (22.611GiB/s) | 20260507 14:30:54.336 | INFO | Downloaded 34GiB to GPU in 1.53872s (22.0962GiB/s) | 20260507 14:30:59.477 | INFO | Analyzing chunk frames = [231936,232448) | 20260507 14:30:59.477 | INFO | MemInfo: { | 20260507 14:30:59.477 | INFO | "gpu" : " 17 GB", | 20260507 14:30:59.477 | INFO | "huge" : "", | 20260507 14:30:59.478 | INFO | "huge_shared" : " 29 GB", | 20260507 14:30:59.478 | INFO | "malloc" : " 14 GB", | 20260507 14:30:59.478 | INFO | "pinned" : " 29 GB" | 20260507 14:30:59.478 | INFO | } | 20260507 14:30:59.478 | INFO | Perf report for the last 5.14234 seconds: | 20260507 14:30:59.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:30:59.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:30:59.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:30:59.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:30:59.478 | INFO | Uploaded 36GiB to GPU in 1.58556s (22.7049GiB/s) | 20260507 14:30:59.478 | INFO | Downloaded 34GiB to GPU in 1.53408s (22.1631GiB/s) | 20260507 14:31:04.614 | INFO | Analyzing chunk frames = [232448,232960) | 20260507 14:31:04.614 | INFO | MemInfo: { | 20260507 14:31:04.614 | INFO | "gpu" : " 17 GB", | 20260507 14:31:04.614 | INFO | "huge" : "", | 20260507 14:31:04.614 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:04.614 | INFO | "malloc" : " 14 GB", | 20260507 14:31:04.614 | INFO | "pinned" : " 29 GB" | 20260507 14:31:04.614 | INFO | } | 20260507 14:31:04.614 | INFO | Perf report for the last 5.13619 seconds: | 20260507 14:31:04.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:04.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.614 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:31:04.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.614 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:31:04.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:04.615 | INFO | Uploaded 36GiB to GPU in 1.586s (22.6986GiB/s) | 20260507 14:31:04.615 | INFO | Downloaded 34GiB to GPU in 1.535s (22.1499GiB/s) | 20260507 14:31:09.755 | INFO | Analyzing chunk frames = [232960,233472) | 20260507 14:31:09.755 | INFO | MemInfo: { | 20260507 14:31:09.755 | INFO | "gpu" : " 17 GB", | 20260507 14:31:09.755 | INFO | "huge" : "", | 20260507 14:31:09.755 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:09.756 | INFO | "malloc" : " 14 GB", | 20260507 14:31:09.756 | INFO | "pinned" : " 29 GB" | 20260507 14:31:09.756 | INFO | } | 20260507 14:31:09.756 | INFO | Perf report for the last 5.14122 seconds: | 20260507 14:31:09.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:09.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:31:09.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:31:09.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:09.756 | INFO | Uploaded 36GiB to GPU in 1.58635s (22.6936GiB/s) | 20260507 14:31:09.756 | INFO | Downloaded 34GiB to GPU in 1.53526s (22.1461GiB/s) | 20260507 14:31:09.853 | INFO | Starting a round of Baz IO | 20260507 14:31:12.751 | INFO | Populating RTMetrics report | 20260507 14:31:12.751 | INFO | RTMetrics report populated | 20260507 14:31:12.751 | INFO | Opening tmp RTMetrics json file | 20260507 14:31:12.769 | INFO | Serializing RTMetrics to json | 20260507 14:31:12.865 | INFO | Serialization done, swapping tmp file now... | 20260507 14:31:12.874 | INFO | RTMetrics done | 20260507 14:31:12.893 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.04 seconds | 20260507 14:31:12.893 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.48% / 0.10% / 41.42% | 20260507 14:31:12.893 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.97747% / 0.000258287% / 1.58946e-05% | 20260507 14:31:12.893 | INFO | Baz IO overall: | 20260507 14:31:12.893 | INFO | BytesWritten : 79925616980 | 20260507 14:31:12.894 | INFO | NumEvents : 37635403987 | 20260507 14:31:14.893 | INFO | Analyzing chunk frames = [233472,233984) | 20260507 14:31:14.893 | INFO | MemInfo: { | 20260507 14:31:14.893 | INFO | "gpu" : " 17 GB", | 20260507 14:31:14.893 | INFO | "huge" : "", | 20260507 14:31:14.893 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:14.893 | INFO | "malloc" : " 14 GB", | 20260507 14:31:14.893 | INFO | "pinned" : " 29 GB" | 20260507 14:31:14.893 | INFO | } | 20260507 14:31:14.893 | INFO | Perf report for the last 5.13759 seconds: | 20260507 14:31:14.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:14.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:14.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:14.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:14.893 | INFO | ANALYSIS : 57.4%, 2.9, 87.5ms, 0 | 20260507 14:31:14.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:14.893 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 14:31:14.893 | INFO | PRE_HQ : 56.3%, 2.1, 60.2ms, 0 | 20260507 14:31:14.893 | INFO | BAZWRITER : 58.7%, 4.4, 132.7ms, 0 | 20260507 14:31:14.893 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 14:31:14.894 | INFO | Uploaded 36GiB to GPU in 1.64114s (21.936GiB/s) | 20260507 14:31:14.894 | INFO | Downloaded 38GiB to GPU in 1.72314s (22.0528GiB/s) | 20260507 14:31:20.033 | INFO | Analyzing chunk frames = [233984,234496) | 20260507 14:31:20.033 | INFO | MemInfo: { | 20260507 14:31:20.033 | INFO | "gpu" : " 17 GB", | 20260507 14:31:20.033 | INFO | "huge" : "", | 20260507 14:31:20.033 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:20.033 | INFO | "malloc" : " 14 GB", | 20260507 14:31:20.033 | INFO | "pinned" : " 29 GB" | 20260507 14:31:20.033 | INFO | } | 20260507 14:31:20.033 | INFO | Perf report for the last 5.13953 seconds: | 20260507 14:31:20.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:20.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:31:20.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | PRE_HQ : 28.8%, 1.0, 15.3ms, 0 | 20260507 14:31:20.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:20.033 | INFO | Uploaded 36GiB to GPU in 1.59083s (22.6297GiB/s) | 20260507 14:31:20.033 | INFO | Downloaded 34GiB to GPU in 1.52909s (22.2354GiB/s) | 20260507 14:31:25.172 | INFO | Analyzing chunk frames = [234496,235008) | 20260507 14:31:25.172 | INFO | MemInfo: { | 20260507 14:31:25.172 | INFO | "gpu" : " 17 GB", | 20260507 14:31:25.172 | INFO | "huge" : "", | 20260507 14:31:25.172 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:25.172 | INFO | "malloc" : " 14 GB", | 20260507 14:31:25.172 | INFO | "pinned" : " 29 GB" | 20260507 14:31:25.172 | INFO | } | 20260507 14:31:25.172 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:31:25.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:25.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:31:25.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:31:25.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:25.172 | INFO | Uploaded 36GiB to GPU in 1.58731s (22.6798GiB/s) | 20260507 14:31:25.172 | INFO | Downloaded 34GiB to GPU in 1.53615s (22.1332GiB/s) | 20260507 14:31:30.311 | INFO | Analyzing chunk frames = [235008,235520) | 20260507 14:31:30.311 | INFO | MemInfo: { | 20260507 14:31:30.311 | INFO | "gpu" : " 17 GB", | 20260507 14:31:30.311 | INFO | "huge" : "", | 20260507 14:31:30.311 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:30.311 | INFO | "malloc" : " 14 GB", | 20260507 14:31:30.311 | INFO | "pinned" : " 29 GB" | 20260507 14:31:30.311 | INFO | } | 20260507 14:31:30.311 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:31:30.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:30.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.311 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:31:30.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.311 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 14:31:30.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:30.312 | INFO | Uploaded 36GiB to GPU in 1.58648s (22.6917GiB/s) | 20260507 14:31:30.312 | INFO | Downloaded 34GiB to GPU in 1.53675s (22.1246GiB/s) | 20260507 14:31:35.450 | INFO | Analyzing chunk frames = [235520,236032) | 20260507 14:31:35.450 | INFO | MemInfo: { | 20260507 14:31:35.450 | INFO | "gpu" : " 17 GB", | 20260507 14:31:35.450 | INFO | "huge" : "", | 20260507 14:31:35.450 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:35.451 | INFO | "malloc" : " 14 GB", | 20260507 14:31:35.451 | INFO | "pinned" : " 29 GB" | 20260507 14:31:35.451 | INFO | } | 20260507 14:31:35.451 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:31:35.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:35.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:31:35.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:31:35.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:35.451 | INFO | Uploaded 36GiB to GPU in 1.58518s (22.7104GiB/s) | 20260507 14:31:35.451 | INFO | Downloaded 34GiB to GPU in 1.53402s (22.164GiB/s) | 20260507 14:31:40.590 | INFO | Analyzing chunk frames = [236032,236544) | 20260507 14:31:40.590 | INFO | MemInfo: { | 20260507 14:31:40.590 | INFO | "gpu" : " 17 GB", | 20260507 14:31:40.590 | INFO | "huge" : "", | 20260507 14:31:40.590 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:40.590 | INFO | "malloc" : " 14 GB", | 20260507 14:31:40.590 | INFO | "pinned" : " 29 GB" | 20260507 14:31:40.590 | INFO | } | 20260507 14:31:40.590 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:31:40.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:40.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:31:40.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:31:40.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:40.590 | INFO | Uploaded 36GiB to GPU in 1.58683s (22.6867GiB/s) | 20260507 14:31:40.590 | INFO | Downloaded 34GiB to GPU in 1.53359s (22.1703GiB/s) | 20260507 14:31:45.729 | INFO | Analyzing chunk frames = [236544,237056) | 20260507 14:31:45.729 | INFO | MemInfo: { | 20260507 14:31:45.729 | INFO | "gpu" : " 17 GB", | 20260507 14:31:45.729 | INFO | "huge" : "", | 20260507 14:31:45.729 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:45.729 | INFO | "malloc" : " 14 GB", | 20260507 14:31:45.729 | INFO | "pinned" : " 29 GB" | 20260507 14:31:45.729 | INFO | } | 20260507 14:31:45.729 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:31:45.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:45.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.729 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:31:45.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.729 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:31:45.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:45.730 | INFO | Uploaded 36GiB to GPU in 1.58562s (22.704GiB/s) | 20260507 14:31:45.730 | INFO | Downloaded 34GiB to GPU in 1.53523s (22.1465GiB/s) | 20260507 14:31:50.868 | INFO | Analyzing chunk frames = [237056,237568) | 20260507 14:31:50.868 | INFO | MemInfo: { | 20260507 14:31:50.868 | INFO | "gpu" : " 17 GB", | 20260507 14:31:50.868 | INFO | "huge" : "", | 20260507 14:31:50.868 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:50.868 | INFO | "malloc" : " 14 GB", | 20260507 14:31:50.868 | INFO | "pinned" : " 29 GB" | 20260507 14:31:50.868 | INFO | } | 20260507 14:31:50.868 | INFO | Perf report for the last 5.139 seconds: | 20260507 14:31:50.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:50.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:31:50.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 14:31:50.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:50.869 | INFO | Uploaded 36GiB to GPU in 1.58625s (22.695GiB/s) | 20260507 14:31:50.869 | INFO | Downloaded 34GiB to GPU in 1.53581s (22.1382GiB/s) | 20260507 14:31:50.869 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:31:50.869 | INFO | Total mem usage: 15.5 GiB | 20260507 14:31:50.869 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:31:50.869 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -1.77%) | 20260507 14:31:50.869 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:31:50.869 | INFO | High Water Breakdown: | 20260507 14:31:50.869 | INFO | 59.1% : 9 GiB ( 98.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:31:50.869 | INFO | 40.8% : 6.22 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:31:50.869 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:31:50.869 | INFO | ------------------------------------------ | 20260507 14:31:50.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:31:50.869 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:31:50.869 | INFO | Total mem usage: 26.5 GiB | 20260507 14:31:50.869 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:31:50.869 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 14:31:50.869 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:31:50.869 | INFO | High Water Breakdown: | 20260507 14:31:50.869 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:31:50.869 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:31:50.869 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:31:50.870 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:31:50.870 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:31:50.870 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:31:50.870 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:31:50.870 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:31:50.870 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:31:50.870 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:31:50.870 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:31:50.870 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:31:50.870 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:31:50.870 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:31:50.870 | INFO | ------------------------------------------ | 20260507 14:31:50.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:31:50.870 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:31:50.870 | INFO | Total mem usage: 27.8 GiB | 20260507 14:31:50.870 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:31:50.870 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:31:50.870 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:31:50.870 | INFO | High Water Breakdown: | 20260507 14:31:50.870 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:31:50.870 | INFO | ------------------------------------------ | 20260507 14:31:50.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:31:50.870 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:31:50.870 | INFO | Total mem usage: 15.7 GiB | 20260507 14:31:50.870 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:31:50.870 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 5.01%) | 20260507 14:31:50.870 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:31:50.870 | INFO | High Water Breakdown: | 20260507 14:31:50.870 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:31:50.870 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:31:50.870 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:31:50.870 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:31:50.870 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:31:50.870 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:31:50.870 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:31:50.870 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:31:50.870 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:31:50.870 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:31:50.870 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:31:50.870 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:31:50.870 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:31:50.870 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:31:50.870 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:31:50.870 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:31:50.870 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:31:50.870 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:31:50.870 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:31:50.870 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:31:50.870 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:31:50.870 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:31:50.870 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:31:50.870 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:31:50.870 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:31:50.870 | INFO | ------------------------------------------ | 20260507 14:31:50.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:31:50.870 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:31:50.870 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:31:50.870 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:31:50.870 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 0.30%, AggregateStats -- 8.88e-02 / -nan, 6.76e-03 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 10.99%, Baseline -- 3.30e+00 / -nan, 2.32e-02 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 3.07%, Histogram -- 9.22e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.72e+01 / -nan, 7.45e-02 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / -nan, 3.58e-01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 3.64%, Metrics -- 1.09e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.10e-01 / -nan, 6.04e-02 / 0.00e+00 | 20260507 14:31:50.870 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:31:50.870 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:31:50.870 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:31:50.870 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 18.49%, Upload -- 1.66e+01 / 0.00e+00, 8.14e-01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 18.08%, Download -- 1.63e+01 / 0.00e+00, 1.20e+00 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 33.41%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 0.01%, LockDownload -- 7.81e-03 / 0.00e+00, 2.84e-01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 29.43%, LockCompute -- 2.65e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 14:31:50.870 | INFO | PerfReport: 0.02%, Misc -- 1.61e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260507 14:31:50.965 | INFO | Starting a round of Baz IO | 20260507 14:31:53.861 | INFO | Populating RTMetrics report | 20260507 14:31:53.862 | INFO | RTMetrics report populated | 20260507 14:31:53.862 | INFO | Opening tmp RTMetrics json file | 20260507 14:31:53.935 | INFO | Serializing RTMetrics to json | 20260507 14:31:53.960 | INFO | Serialization done, swapping tmp file now... | 20260507 14:31:53.962 | INFO | RTMetrics done | 20260507 14:31:54.068 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.102 seconds | 20260507 14:31:54.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.39% / 0.10% / 41.52% | 20260507 14:31:54.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.9348% / 0.000290076% / 1.98682e-05% | 20260507 14:31:54.068 | INFO | Baz IO overall: | 20260507 14:31:54.068 | INFO | BytesWritten : 81721605711 | 20260507 14:31:54.068 | INFO | NumEvents : 38512484526 | 20260507 14:31:56.008 | INFO | Analyzing chunk frames = [237568,238080) | 20260507 14:31:56.008 | INFO | MemInfo: { | 20260507 14:31:56.008 | INFO | "gpu" : " 17 GB", | 20260507 14:31:56.008 | INFO | "huge" : "", | 20260507 14:31:56.008 | INFO | "huge_shared" : " 29 GB", | 20260507 14:31:56.008 | INFO | "malloc" : " 14 GB", | 20260507 14:31:56.008 | INFO | "pinned" : " 29 GB" | 20260507 14:31:56.008 | INFO | } | 20260507 14:31:56.008 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:31:56.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:31:56.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:56.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:56.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:56.008 | INFO | ANALYSIS : 57.4%, 2.9, 86.8ms, 0 | 20260507 14:31:56.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:31:56.008 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 14:31:56.008 | INFO | PRE_HQ : 56.2%, 2.0, 59.5ms, 0 | 20260507 14:31:56.008 | INFO | BAZWRITER : 59.9%, 4.6, 141.9ms, 0 | 20260507 14:31:56.008 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260507 14:31:56.009 | INFO | Uploaded 36GiB to GPU in 1.65696s (21.7266GiB/s) | 20260507 14:31:56.009 | INFO | Downloaded 38GiB to GPU in 1.73902s (21.8514GiB/s) | 20260507 14:32:01.147 | INFO | Analyzing chunk frames = [238080,238592) | 20260507 14:32:01.147 | INFO | MemInfo: { | 20260507 14:32:01.147 | INFO | "gpu" : " 17 GB", | 20260507 14:32:01.147 | INFO | "huge" : "", | 20260507 14:32:01.147 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:01.147 | INFO | "malloc" : " 14 GB", | 20260507 14:32:01.147 | INFO | "pinned" : " 29 GB" | 20260507 14:32:01.147 | INFO | } | 20260507 14:32:01.147 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:32:01.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:01.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.147 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:32:01.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.147 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:32:01.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:01.148 | INFO | Uploaded 36GiB to GPU in 1.58424s (22.7238GiB/s) | 20260507 14:32:01.148 | INFO | Downloaded 34GiB to GPU in 1.53197s (22.1937GiB/s) | 20260507 14:32:06.286 | INFO | Analyzing chunk frames = [238592,239104) | 20260507 14:32:06.286 | INFO | MemInfo: { | 20260507 14:32:06.286 | INFO | "gpu" : " 17 GB", | 20260507 14:32:06.286 | INFO | "huge" : "", | 20260507 14:32:06.286 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:06.287 | INFO | "malloc" : " 14 GB", | 20260507 14:32:06.287 | INFO | "pinned" : " 29 GB" | 20260507 14:32:06.287 | INFO | } | 20260507 14:32:06.287 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:32:06.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:06.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 14:32:06.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:32:06.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:06.287 | INFO | Uploaded 36GiB to GPU in 1.58694s (22.6852GiB/s) | 20260507 14:32:06.287 | INFO | Downloaded 34GiB to GPU in 1.536s (22.1354GiB/s) | 20260507 14:32:11.426 | INFO | Analyzing chunk frames = [239104,239616) | 20260507 14:32:11.426 | INFO | MemInfo: { | 20260507 14:32:11.426 | INFO | "gpu" : " 17 GB", | 20260507 14:32:11.426 | INFO | "huge" : "", | 20260507 14:32:11.426 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:11.426 | INFO | "malloc" : " 14 GB", | 20260507 14:32:11.426 | INFO | "pinned" : " 29 GB" | 20260507 14:32:11.426 | INFO | } | 20260507 14:32:11.426 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:32:11.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:11.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:32:11.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:32:11.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:11.426 | INFO | Uploaded 36GiB to GPU in 1.58799s (22.6702GiB/s) | 20260507 14:32:11.426 | INFO | Downloaded 34GiB to GPU in 1.53505s (22.1491GiB/s) | 20260507 14:32:16.565 | INFO | Analyzing chunk frames = [239616,240128) | 20260507 14:32:16.565 | INFO | MemInfo: { | 20260507 14:32:16.565 | INFO | "gpu" : " 17 GB", | 20260507 14:32:16.565 | INFO | "huge" : "", | 20260507 14:32:16.565 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:16.565 | INFO | "malloc" : " 14 GB", | 20260507 14:32:16.565 | INFO | "pinned" : " 29 GB" | 20260507 14:32:16.565 | INFO | } | 20260507 14:32:16.565 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:32:16.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:16.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.565 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:32:16.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.565 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:32:16.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:16.566 | INFO | Uploaded 36GiB to GPU in 1.58598s (22.6989GiB/s) | 20260507 14:32:16.566 | INFO | Downloaded 34GiB to GPU in 1.53396s (22.1648GiB/s) | 20260507 14:32:21.705 | INFO | Analyzing chunk frames = [240128,240640) | 20260507 14:32:21.705 | INFO | MemInfo: { | 20260507 14:32:21.705 | INFO | "gpu" : " 17 GB", | 20260507 14:32:21.705 | INFO | "huge" : "", | 20260507 14:32:21.705 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:21.705 | INFO | "malloc" : " 14 GB", | 20260507 14:32:21.705 | INFO | "pinned" : " 29 GB" | 20260507 14:32:21.705 | INFO | } | 20260507 14:32:21.705 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:32:21.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:21.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.705 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:32:21.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.705 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:32:21.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:21.706 | INFO | Uploaded 36GiB to GPU in 1.58586s (22.7006GiB/s) | 20260507 14:32:21.706 | INFO | Downloaded 34GiB to GPU in 1.533s (22.1787GiB/s) | 20260507 14:32:26.844 | INFO | Analyzing chunk frames = [240640,241152) | 20260507 14:32:26.844 | INFO | MemInfo: { | 20260507 14:32:26.844 | INFO | "gpu" : " 17 GB", | 20260507 14:32:26.844 | INFO | "huge" : "", | 20260507 14:32:26.844 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:26.844 | INFO | "malloc" : " 14 GB", | 20260507 14:32:26.844 | INFO | "pinned" : " 29 GB" | 20260507 14:32:26.844 | INFO | } | 20260507 14:32:26.844 | INFO | Perf report for the last 5.13967 seconds: | 20260507 14:32:26.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:26.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.844 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:32:26.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.844 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:32:26.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:26.845 | INFO | Uploaded 36GiB to GPU in 1.58636s (22.6935GiB/s) | 20260507 14:32:26.845 | INFO | Downloaded 34GiB to GPU in 1.53522s (22.1466GiB/s) | 20260507 14:32:31.983 | INFO | Analyzing chunk frames = [241152,241664) | 20260507 14:32:31.983 | INFO | MemInfo: { | 20260507 14:32:31.983 | INFO | "gpu" : " 17 GB", | 20260507 14:32:31.983 | INFO | "huge" : "", | 20260507 14:32:31.983 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:31.983 | INFO | "malloc" : " 14 GB", | 20260507 14:32:31.983 | INFO | "pinned" : " 29 GB" | 20260507 14:32:31.983 | INFO | } | 20260507 14:32:31.983 | INFO | Perf report for the last 5.13902 seconds: | 20260507 14:32:31.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:31.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.983 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:32:31.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.983 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:32:31.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:31.984 | INFO | Uploaded 36GiB to GPU in 1.58642s (22.6926GiB/s) | 20260507 14:32:31.984 | INFO | Downloaded 34GiB to GPU in 1.53512s (22.1481GiB/s) | 20260507 14:32:32.080 | INFO | Starting a round of Baz IO | 20260507 14:32:34.976 | INFO | Populating RTMetrics report | 20260507 14:32:34.977 | INFO | RTMetrics report populated | 20260507 14:32:34.977 | INFO | Opening tmp RTMetrics json file | 20260507 14:32:34.982 | INFO | Serializing RTMetrics to json | 20260507 14:32:35.000 | INFO | Serialization done, swapping tmp file now... | 20260507 14:32:35.018 | INFO | RTMetrics done | 20260507 14:32:35.072 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 2.991 seconds | 20260507 14:32:35.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.29% / 0.10% / 41.61% | 20260507 14:32:35.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.87424% / 0.000290076% / 3.57628e-05% | 20260507 14:32:35.072 | INFO | Baz IO overall: | 20260507 14:32:35.072 | INFO | BytesWritten : 83517911001 | 20260507 14:32:35.072 | INFO | NumEvents : 39388980767 | 20260507 14:32:37.122 | INFO | Analyzing chunk frames = [241664,242176) | 20260507 14:32:37.122 | INFO | MemInfo: { | 20260507 14:32:37.122 | INFO | "gpu" : " 17 GB", | 20260507 14:32:37.122 | INFO | "huge" : "", | 20260507 14:32:37.123 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:37.123 | INFO | "malloc" : " 14 GB", | 20260507 14:32:37.123 | INFO | "pinned" : " 29 GB" | 20260507 14:32:37.123 | INFO | } | 20260507 14:32:37.123 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:32:37.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:37.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:37.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:37.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:37.123 | INFO | ANALYSIS : 57.3%, 2.9, 87.7ms, 0 | 20260507 14:32:37.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:37.123 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 14:32:37.123 | INFO | PRE_HQ : 56.1%, 2.0, 59.7ms, 0 | 20260507 14:32:37.123 | INFO | BAZWRITER : 57.7%, 4.4, 130.6ms, 0 | 20260507 14:32:37.123 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 14:32:37.123 | INFO | Uploaded 36GiB to GPU in 1.64699s (21.8581GiB/s) | 20260507 14:32:37.123 | INFO | Downloaded 38GiB to GPU in 1.72788s (21.9923GiB/s) | 20260507 14:32:42.262 | INFO | Analyzing chunk frames = [242176,242688) | 20260507 14:32:42.262 | INFO | MemInfo: { | 20260507 14:32:42.262 | INFO | "gpu" : " 17 GB", | 20260507 14:32:42.262 | INFO | "huge" : "", | 20260507 14:32:42.262 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:42.262 | INFO | "malloc" : " 14 GB", | 20260507 14:32:42.262 | INFO | "pinned" : " 29 GB" | 20260507 14:32:42.262 | INFO | } | 20260507 14:32:42.262 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:32:42.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:42.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:32:42.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:32:42.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:42.262 | INFO | Uploaded 36GiB to GPU in 1.58348s (22.7347GiB/s) | 20260507 14:32:42.262 | INFO | Downloaded 34GiB to GPU in 1.53134s (22.2028GiB/s) | 20260507 14:32:47.401 | INFO | Analyzing chunk frames = [242688,243200) | 20260507 14:32:47.401 | INFO | MemInfo: { | 20260507 14:32:47.401 | INFO | "gpu" : " 17 GB", | 20260507 14:32:47.401 | INFO | "huge" : "", | 20260507 14:32:47.401 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:47.401 | INFO | "malloc" : " 14 GB", | 20260507 14:32:47.401 | INFO | "pinned" : " 29 GB" | 20260507 14:32:47.401 | INFO | } | 20260507 14:32:47.401 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:32:47.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:47.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.402 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:32:47.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.402 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:32:47.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:47.402 | INFO | Uploaded 36GiB to GPU in 1.5881s (22.6685GiB/s) | 20260507 14:32:47.402 | INFO | Downloaded 34GiB to GPU in 1.53708s (22.1198GiB/s) | 20260507 14:32:52.540 | INFO | Analyzing chunk frames = [243200,243712) | 20260507 14:32:52.540 | INFO | MemInfo: { | 20260507 14:32:52.541 | INFO | "gpu" : " 17 GB", | 20260507 14:32:52.541 | INFO | "huge" : "", | 20260507 14:32:52.541 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:52.541 | INFO | "malloc" : " 14 GB", | 20260507 14:32:52.541 | INFO | "pinned" : " 29 GB" | 20260507 14:32:52.541 | INFO | } | 20260507 14:32:52.541 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:32:52.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:52.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:32:52.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:32:52.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:52.541 | INFO | Uploaded 36GiB to GPU in 1.58777s (22.6733GiB/s) | 20260507 14:32:52.541 | INFO | Downloaded 34GiB to GPU in 1.53515s (22.1477GiB/s) | 20260507 14:32:57.680 | INFO | Analyzing chunk frames = [243712,244224) | 20260507 14:32:57.680 | INFO | MemInfo: { | 20260507 14:32:57.680 | INFO | "gpu" : " 17 GB", | 20260507 14:32:57.680 | INFO | "huge" : "", | 20260507 14:32:57.680 | INFO | "huge_shared" : " 29 GB", | 20260507 14:32:57.680 | INFO | "malloc" : " 14 GB", | 20260507 14:32:57.680 | INFO | "pinned" : " 29 GB" | 20260507 14:32:57.680 | INFO | } | 20260507 14:32:57.680 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:32:57.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:32:57.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.680 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 14:32:57.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.680 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:32:57.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:32:57.681 | INFO | Uploaded 36GiB to GPU in 1.58622s (22.6955GiB/s) | 20260507 14:32:57.681 | INFO | Downloaded 34GiB to GPU in 1.53379s (22.1674GiB/s) | 20260507 14:33:02.823 | INFO | Analyzing chunk frames = [244224,244736) | 20260507 14:33:02.823 | INFO | MemInfo: { | 20260507 14:33:02.823 | INFO | "gpu" : " 17 GB", | 20260507 14:33:02.823 | INFO | "huge" : "", | 20260507 14:33:02.823 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:02.823 | INFO | "malloc" : " 14 GB", | 20260507 14:33:02.823 | INFO | "pinned" : " 29 GB" | 20260507 14:33:02.823 | INFO | } | 20260507 14:33:02.823 | INFO | Perf report for the last 5.14292 seconds: | 20260507 14:33:02.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:02.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:33:02.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:33:02.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:02.823 | INFO | Uploaded 36GiB to GPU in 1.58695s (22.685GiB/s) | 20260507 14:33:02.824 | INFO | Downloaded 34GiB to GPU in 1.53457s (22.1561GiB/s) | 20260507 14:33:07.959 | INFO | Analyzing chunk frames = [244736,245248) | 20260507 14:33:07.959 | INFO | MemInfo: { | 20260507 14:33:07.959 | INFO | "gpu" : " 17 GB", | 20260507 14:33:07.959 | INFO | "huge" : "", | 20260507 14:33:07.959 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:07.959 | INFO | "malloc" : " 14 GB", | 20260507 14:33:07.959 | INFO | "pinned" : " 29 GB" | 20260507 14:33:07.959 | INFO | } | 20260507 14:33:07.959 | INFO | Perf report for the last 5.13582 seconds: | 20260507 14:33:07.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:07.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:33:07.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:33:07.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:07.959 | INFO | Uploaded 36GiB to GPU in 1.58633s (22.6939GiB/s) | 20260507 14:33:07.960 | INFO | Downloaded 34GiB to GPU in 1.53498s (22.1501GiB/s) | 20260507 14:33:13.098 | INFO | Analyzing chunk frames = [245248,245760) | 20260507 14:33:13.098 | INFO | MemInfo: { | 20260507 14:33:13.098 | INFO | "gpu" : " 17 GB", | 20260507 14:33:13.098 | INFO | "huge" : "", | 20260507 14:33:13.098 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:13.098 | INFO | "malloc" : " 14 GB", | 20260507 14:33:13.098 | INFO | "pinned" : " 29 GB" | 20260507 14:33:13.098 | INFO | } | 20260507 14:33:13.098 | INFO | Perf report for the last 5.13919 seconds: | 20260507 14:33:13.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:13.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.098 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:33:13.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.098 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:33:13.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:13.099 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:33:13.099 | INFO | Total mem usage: 15.5 GiB | 20260507 14:33:13.099 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:33:13.099 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -1.02%) | 20260507 14:33:13.099 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:33:13.099 | INFO | High Water Breakdown: | 20260507 14:33:13.099 | INFO | 59.2% : 9.09 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:33:13.099 | INFO | 40.7% : 6.24 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:33:13.099 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:33:13.099 | INFO | ------------------------------------------ | 20260507 14:33:13.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:33:13.099 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:33:13.099 | INFO | Total mem usage: 26.5 GiB | 20260507 14:33:13.099 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:33:13.099 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:33:13.099 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:33:13.099 | INFO | High Water Breakdown: | 20260507 14:33:13.099 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:33:13.099 | INFO | Uploaded 36GiB to GPU in 1.58657s (22.6904GiB/s) | 20260507 14:33:13.099 | INFO | Downloaded 34GiB to GPU in 1.53489s (22.1514GiB/s) | 20260507 14:33:13.099 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:33:13.099 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:33:13.099 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:33:13.099 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:33:13.099 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:33:13.099 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:33:13.099 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:33:13.099 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:33:13.099 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:33:13.099 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:33:13.099 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:33:13.099 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:33:13.099 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:33:13.099 | INFO | ------------------------------------------ | 20260507 14:33:13.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:33:13.099 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:33:13.099 | INFO | Total mem usage: 27.8 GiB | 20260507 14:33:13.099 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:33:13.099 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:33:13.099 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:33:13.099 | INFO | High Water Breakdown: | 20260507 14:33:13.099 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:33:13.099 | INFO | ------------------------------------------ | 20260507 14:33:13.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:33:13.099 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:33:13.099 | INFO | Total mem usage: 15.7 GiB | 20260507 14:33:13.099 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:33:13.099 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:33:13.099 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:33:13.099 | INFO | High Water Breakdown: | 20260507 14:33:13.099 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:33:13.099 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:33:13.099 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:33:13.099 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:33:13.099 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:33:13.099 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:33:13.099 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:33:13.099 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:33:13.099 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:33:13.099 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:33:13.099 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:33:13.099 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:33:13.099 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:33:13.099 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:33:13.099 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:33:13.099 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:33:13.099 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:33:13.099 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:33:13.099 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:33:13.099 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:33:13.099 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:33:13.099 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:33:13.099 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:33:13.099 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:33:13.099 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:33:13.099 | INFO | ------------------------------------------ | 20260507 14:33:13.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:33:13.099 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:33:13.099 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:33:13.099 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:33:13.099 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.74e-03 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.84e-02 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / -nan, 5.14e-02 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 16.09%, DME -- 3.86e+01 / -nan, 3.12e+00 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.19e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 3.63%, Metrics -- 1.09e+00 / -nan, 1.74e-01 / 0.00e+00 | 20260507 14:33:13.099 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.05e-01 / -nan, 5.97e-02 / 0.00e+00 | 20260507 14:33:13.100 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:33:13.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:33:13.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:33:13.100 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 18.48%, Upload -- 1.66e+01 / 0.00e+00, 4.84e-01 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 18.07%, Download -- 1.62e+01 / 0.00e+00, 7.84e-01 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 33.39%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 0.00%, LockDownload -- 4.37e-04 / 0.00e+00, 8.83e-05 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 29.48%, LockCompute -- 2.65e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:33:13.100 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260507 14:33:13.195 | INFO | Starting a round of Baz IO | 20260507 14:33:16.092 | INFO | Populating RTMetrics report | 20260507 14:33:16.092 | INFO | RTMetrics report populated | 20260507 14:33:16.092 | INFO | Opening tmp RTMetrics json file | 20260507 14:33:16.284 | INFO | Serializing RTMetrics to json | 20260507 14:33:16.305 | INFO | Serialization done, swapping tmp file now... | 20260507 14:33:16.306 | INFO | RTMetrics done | 20260507 14:33:16.418 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.222 seconds | 20260507 14:33:16.418 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.20% / 0.09% / 41.71% | 20260507 14:33:16.418 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.84697% / 0.000258287% / 3.17891e-05% | 20260507 14:33:16.418 | INFO | Baz IO overall: | 20260507 14:33:16.418 | INFO | BytesWritten : 85317011480 | 20260507 14:33:16.418 | INFO | NumEvents : 40266804995 | 20260507 14:33:18.241 | INFO | Analyzing chunk frames = [245760,246272) | 20260507 14:33:18.241 | INFO | MemInfo: { | 20260507 14:33:18.241 | INFO | "gpu" : " 17 GB", | 20260507 14:33:18.241 | INFO | "huge" : "", | 20260507 14:33:18.241 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:18.241 | INFO | "malloc" : " 14 GB", | 20260507 14:33:18.241 | INFO | "pinned" : " 29 GB" | 20260507 14:33:18.241 | INFO | } | 20260507 14:33:18.241 | INFO | Perf report for the last 5.1431 seconds: | 20260507 14:33:18.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:18.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:18.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:18.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:18.241 | INFO | ANALYSIS : 57.3%, 3.0, 88.9ms, 0 | 20260507 14:33:18.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:18.241 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260507 14:33:18.241 | INFO | PRE_HQ : 56.1%, 2.1, 60.6ms, 0 | 20260507 14:33:18.241 | INFO | BAZWRITER : 62.1%, 4.0, 129.1ms, 0 | 20260507 14:33:18.242 | INFO | BAZ_SAMPLING : 3.9%, 1.0, 2.0ms, 0 | 20260507 14:33:18.242 | INFO | Uploaded 36GiB to GPU in 1.63001s (22.0858GiB/s) | 20260507 14:33:18.242 | INFO | Downloaded 38GiB to GPU in 1.71217s (22.1941GiB/s) | 20260507 14:33:23.377 | INFO | Analyzing chunk frames = [246272,246784) | 20260507 14:33:23.377 | INFO | MemInfo: { | 20260507 14:33:23.377 | INFO | "gpu" : " 17 GB", | 20260507 14:33:23.377 | INFO | "huge" : "", | 20260507 14:33:23.377 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:23.377 | INFO | "malloc" : " 14 GB", | 20260507 14:33:23.377 | INFO | "pinned" : " 29 GB" | 20260507 14:33:23.377 | INFO | } | 20260507 14:33:23.377 | INFO | Perf report for the last 5.13561 seconds: | 20260507 14:33:23.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:23.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260507 14:33:23.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:33:23.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:23.377 | INFO | Uploaded 36GiB to GPU in 1.58384s (22.7295GiB/s) | 20260507 14:33:23.377 | INFO | Downloaded 34GiB to GPU in 1.53093s (22.2087GiB/s) | 20260507 14:33:28.516 | INFO | Analyzing chunk frames = [246784,247296) | 20260507 14:33:28.516 | INFO | MemInfo: { | 20260507 14:33:28.516 | INFO | "gpu" : " 17 GB", | 20260507 14:33:28.516 | INFO | "huge" : "", | 20260507 14:33:28.516 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:28.516 | INFO | "malloc" : " 14 GB", | 20260507 14:33:28.516 | INFO | "pinned" : " 29 GB" | 20260507 14:33:28.516 | INFO | } | 20260507 14:33:28.516 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:33:28.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:28.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.516 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:33:28.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.516 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:33:28.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:28.517 | INFO | Uploaded 36GiB to GPU in 1.58689s (22.6859GiB/s) | 20260507 14:33:28.517 | INFO | Downloaded 34GiB to GPU in 1.53656s (22.1273GiB/s) | 20260507 14:33:33.655 | INFO | Analyzing chunk frames = [247296,247808) | 20260507 14:33:33.655 | INFO | MemInfo: { | 20260507 14:33:33.655 | INFO | "gpu" : " 17 GB", | 20260507 14:33:33.655 | INFO | "huge" : "", | 20260507 14:33:33.655 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:33.655 | INFO | "malloc" : " 14 GB", | 20260507 14:33:33.655 | INFO | "pinned" : " 29 GB" | 20260507 14:33:33.655 | INFO | } | 20260507 14:33:33.655 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:33:33.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:33.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.656 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 14:33:33.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.656 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:33:33.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:33.656 | INFO | Uploaded 36GiB to GPU in 1.58685s (22.6864GiB/s) | 20260507 14:33:33.656 | INFO | Downloaded 34GiB to GPU in 1.53591s (22.1367GiB/s) | 20260507 14:33:38.795 | INFO | Analyzing chunk frames = [247808,248320) | 20260507 14:33:38.795 | INFO | MemInfo: { | 20260507 14:33:38.795 | INFO | "gpu" : " 17 GB", | 20260507 14:33:38.795 | INFO | "huge" : "", | 20260507 14:33:38.795 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:38.795 | INFO | "malloc" : " 14 GB", | 20260507 14:33:38.795 | INFO | "pinned" : " 29 GB" | 20260507 14:33:38.795 | INFO | } | 20260507 14:33:38.795 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:33:38.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:38.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 14:33:38.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:33:38.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:38.795 | INFO | Uploaded 36GiB to GPU in 1.5862s (22.6958GiB/s) | 20260507 14:33:38.795 | INFO | Downloaded 34GiB to GPU in 1.53438s (22.1588GiB/s) | 20260507 14:33:43.934 | INFO | Analyzing chunk frames = [248320,248832) | 20260507 14:33:43.934 | INFO | MemInfo: { | 20260507 14:33:43.934 | INFO | "gpu" : " 17 GB", | 20260507 14:33:43.934 | INFO | "huge" : "", | 20260507 14:33:43.934 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:43.934 | INFO | "malloc" : " 14 GB", | 20260507 14:33:43.934 | INFO | "pinned" : " 29 GB" | 20260507 14:33:43.934 | INFO | } | 20260507 14:33:43.934 | INFO | Perf report for the last 5.13969 seconds: | 20260507 14:33:43.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:43.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.934 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:33:43.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.935 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:33:43.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:43.935 | INFO | Uploaded 36GiB to GPU in 1.5867s (22.6885GiB/s) | 20260507 14:33:43.935 | INFO | Downloaded 34GiB to GPU in 1.53292s (22.18GiB/s) | 20260507 14:33:49.073 | INFO | Analyzing chunk frames = [248832,249344) | 20260507 14:33:49.073 | INFO | MemInfo: { | 20260507 14:33:49.073 | INFO | "gpu" : " 17 GB", | 20260507 14:33:49.073 | INFO | "huge" : "", | 20260507 14:33:49.073 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:49.074 | INFO | "malloc" : " 14 GB", | 20260507 14:33:49.074 | INFO | "pinned" : " 29 GB" | 20260507 14:33:49.074 | INFO | } | 20260507 14:33:49.074 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:33:49.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:49.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:33:49.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 14:33:49.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:49.074 | INFO | Uploaded 36GiB to GPU in 1.58653s (22.6911GiB/s) | 20260507 14:33:49.074 | INFO | Downloaded 34GiB to GPU in 1.53528s (22.1458GiB/s) | 20260507 14:33:54.213 | INFO | Analyzing chunk frames = [249344,249856) | 20260507 14:33:54.213 | INFO | MemInfo: { | 20260507 14:33:54.213 | INFO | "gpu" : " 17 GB", | 20260507 14:33:54.213 | INFO | "huge" : "", | 20260507 14:33:54.213 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:54.213 | INFO | "malloc" : " 14 GB", | 20260507 14:33:54.213 | INFO | "pinned" : " 29 GB" | 20260507 14:33:54.213 | INFO | } | 20260507 14:33:54.213 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:33:54.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:54.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:33:54.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:33:54.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:54.213 | INFO | Uploaded 36GiB to GPU in 1.58602s (22.6983GiB/s) | 20260507 14:33:54.213 | INFO | Downloaded 34GiB to GPU in 1.53474s (22.1535GiB/s) | 20260507 14:33:54.310 | INFO | Starting a round of Baz IO | 20260507 14:33:57.211 | INFO | Populating RTMetrics report | 20260507 14:33:57.211 | INFO | RTMetrics report populated | 20260507 14:33:57.211 | INFO | Opening tmp RTMetrics json file | 20260507 14:33:57.213 | INFO | Serializing RTMetrics to json | 20260507 14:33:57.214 | INFO | Serialization done, swapping tmp file now... | 20260507 14:33:57.215 | INFO | RTMetrics done | 20260507 14:33:57.306 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 2.996 seconds | 20260507 14:33:57.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.11% / 0.09% / 41.80% | 20260507 14:33:57.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.82524% / 0.000234445% / 2.78155e-05% | 20260507 14:33:57.307 | INFO | Baz IO overall: | 20260507 14:33:57.307 | INFO | BytesWritten : 87117850988 | 20260507 14:33:57.307 | INFO | NumEvents : 41145934448 | 20260507 14:33:59.352 | INFO | Analyzing chunk frames = [249856,250368) | 20260507 14:33:59.352 | INFO | MemInfo: { | 20260507 14:33:59.352 | INFO | "gpu" : " 17 GB", | 20260507 14:33:59.352 | INFO | "huge" : "", | 20260507 14:33:59.352 | INFO | "huge_shared" : " 29 GB", | 20260507 14:33:59.352 | INFO | "malloc" : " 14 GB", | 20260507 14:33:59.352 | INFO | "pinned" : " 29 GB" | 20260507 14:33:59.352 | INFO | } | 20260507 14:33:59.352 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:33:59.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:33:59.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:59.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:59.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:59.352 | INFO | ANALYSIS : 57.4%, 2.9, 88.5ms, 0 | 20260507 14:33:59.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:33:59.352 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 14:33:59.352 | INFO | PRE_HQ : 56.3%, 2.1, 60.6ms, 0 | 20260507 14:33:59.352 | INFO | BAZWRITER : 57.8%, 4.3, 126.8ms, 0 | 20260507 14:33:59.352 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 14:33:59.353 | INFO | Uploaded 36GiB to GPU in 1.63281s (22.0479GiB/s) | 20260507 14:33:59.353 | INFO | Downloaded 38GiB to GPU in 1.71503s (22.157GiB/s) | 20260507 14:34:04.491 | INFO | Analyzing chunk frames = [250368,250880) | 20260507 14:34:04.491 | INFO | MemInfo: { | 20260507 14:34:04.491 | INFO | "gpu" : " 17 GB", | 20260507 14:34:04.491 | INFO | "huge" : "", | 20260507 14:34:04.492 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:04.492 | INFO | "malloc" : " 14 GB", | 20260507 14:34:04.492 | INFO | "pinned" : " 29 GB" | 20260507 14:34:04.492 | INFO | } | 20260507 14:34:04.492 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:34:04.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:04.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:34:04.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | PRE_HQ : 28.4%, 1.0, 14.9ms, 0 | 20260507 14:34:04.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:04.492 | INFO | Uploaded 36GiB to GPU in 1.58809s (22.6687GiB/s) | 20260507 14:34:04.492 | INFO | Downloaded 34GiB to GPU in 1.52905s (22.236GiB/s) | 20260507 14:34:09.631 | INFO | Analyzing chunk frames = [250880,251392) | 20260507 14:34:09.631 | INFO | MemInfo: { | 20260507 14:34:09.631 | INFO | "gpu" : " 17 GB", | 20260507 14:34:09.631 | INFO | "huge" : "", | 20260507 14:34:09.631 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:09.631 | INFO | "malloc" : " 14 GB", | 20260507 14:34:09.631 | INFO | "pinned" : " 29 GB" | 20260507 14:34:09.631 | INFO | } | 20260507 14:34:09.631 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:34:09.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:09.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:34:09.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:34:09.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:09.631 | INFO | Uploaded 36GiB to GPU in 1.60753s (22.3947GiB/s) | 20260507 14:34:09.631 | INFO | Downloaded 34GiB to GPU in 1.52727s (22.2619GiB/s) | 20260507 14:34:14.770 | INFO | Analyzing chunk frames = [251392,251904) | 20260507 14:34:14.770 | INFO | MemInfo: { | 20260507 14:34:14.770 | INFO | "gpu" : " 17 GB", | 20260507 14:34:14.770 | INFO | "huge" : "", | 20260507 14:34:14.770 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:14.770 | INFO | "malloc" : " 14 GB", | 20260507 14:34:14.770 | INFO | "pinned" : " 29 GB" | 20260507 14:34:14.770 | INFO | } | 20260507 14:34:14.770 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:34:14.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:14.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.770 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:34:14.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.770 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:34:14.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:14.771 | INFO | Uploaded 36GiB to GPU in 1.60753s (22.3946GiB/s) | 20260507 14:34:14.771 | INFO | Downloaded 34GiB to GPU in 1.52774s (22.2551GiB/s) | 20260507 14:34:19.909 | INFO | Analyzing chunk frames = [251904,252416) | 20260507 14:34:19.909 | INFO | MemInfo: { | 20260507 14:34:19.909 | INFO | "gpu" : " 17 GB", | 20260507 14:34:19.909 | INFO | "huge" : "", | 20260507 14:34:19.909 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:19.909 | INFO | "malloc" : " 14 GB", | 20260507 14:34:19.909 | INFO | "pinned" : " 29 GB" | 20260507 14:34:19.910 | INFO | } | 20260507 14:34:19.910 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:34:19.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:19.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:34:19.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:34:19.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:19.910 | INFO | Uploaded 36GiB to GPU in 1.606s (22.4159GiB/s) | 20260507 14:34:19.910 | INFO | Downloaded 34GiB to GPU in 1.52509s (22.2937GiB/s) | 20260507 14:34:25.049 | INFO | Analyzing chunk frames = [252416,252928) | 20260507 14:34:25.049 | INFO | MemInfo: { | 20260507 14:34:25.049 | INFO | "gpu" : " 17 GB", | 20260507 14:34:25.049 | INFO | "huge" : "", | 20260507 14:34:25.049 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:25.049 | INFO | "malloc" : " 14 GB", | 20260507 14:34:25.049 | INFO | "pinned" : " 29 GB" | 20260507 14:34:25.049 | INFO | } | 20260507 14:34:25.049 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:34:25.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:25.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.049 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:34:25.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.049 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:34:25.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:25.050 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4128GiB/s) | 20260507 14:34:25.050 | INFO | Downloaded 34GiB to GPU in 1.52556s (22.2869GiB/s) | 20260507 14:34:30.188 | INFO | Analyzing chunk frames = [252928,253440) | 20260507 14:34:30.188 | INFO | MemInfo: { | 20260507 14:34:30.188 | INFO | "gpu" : " 17 GB", | 20260507 14:34:30.188 | INFO | "huge" : "", | 20260507 14:34:30.188 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:30.188 | INFO | "malloc" : " 14 GB", | 20260507 14:34:30.188 | INFO | "pinned" : " 29 GB" | 20260507 14:34:30.188 | INFO | } | 20260507 14:34:30.188 | INFO | Perf report for the last 5.13914 seconds: | 20260507 14:34:30.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:30.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.188 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:34:30.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.188 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:34:30.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:30.189 | INFO | Uploaded 36GiB to GPU in 1.60609s (22.4147GiB/s) | 20260507 14:34:30.189 | INFO | Downloaded 34GiB to GPU in 1.52586s (22.2826GiB/s) | 20260507 14:34:35.328 | INFO | Analyzing chunk frames = [253440,253952) | 20260507 14:34:35.328 | INFO | MemInfo: { | 20260507 14:34:35.328 | INFO | "gpu" : " 17 GB", | 20260507 14:34:35.328 | INFO | "huge" : "", | 20260507 14:34:35.328 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:35.328 | INFO | "malloc" : " 14 GB", | 20260507 14:34:35.328 | INFO | "pinned" : " 29 GB" | 20260507 14:34:35.328 | INFO | } | 20260507 14:34:35.328 | INFO | Perf report for the last 5.13979 seconds: | 20260507 14:34:35.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:35.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:34:35.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:34:35.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:35.328 | INFO | Uploaded 36GiB to GPU in 1.60686s (22.404GiB/s) | 20260507 14:34:35.328 | INFO | Downloaded 34GiB to GPU in 1.52679s (22.269GiB/s) | 20260507 14:34:35.328 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:34:35.328 | INFO | Total mem usage: 15.5 GiB | 20260507 14:34:35.328 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:34:35.328 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -4.04%) | 20260507 14:34:35.328 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:34:35.329 | INFO | High Water Breakdown: | 20260507 14:34:35.329 | INFO | 58.7% : 8.75 GiB ( 95.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:34:35.329 | INFO | 41.2% : 6.14 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:34:35.329 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:34:35.329 | INFO | ------------------------------------------ | 20260507 14:34:35.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:34:35.329 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:34:35.329 | INFO | Total mem usage: 26.5 GiB | 20260507 14:34:35.329 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:34:35.329 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:34:35.329 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:34:35.329 | INFO | High Water Breakdown: | 20260507 14:34:35.329 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:34:35.329 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:34:35.329 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:34:35.329 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:34:35.329 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:34:35.329 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:34:35.329 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:34:35.329 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:34:35.329 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:34:35.329 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:34:35.329 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:34:35.329 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:34:35.329 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:34:35.329 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:34:35.329 | INFO | ------------------------------------------ | 20260507 14:34:35.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:34:35.329 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:34:35.329 | INFO | Total mem usage: 27.8 GiB | 20260507 14:34:35.329 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:34:35.329 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:34:35.329 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:34:35.329 | INFO | High Water Breakdown: | 20260507 14:34:35.329 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:34:35.329 | INFO | ------------------------------------------ | 20260507 14:34:35.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:34:35.329 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:34:35.329 | INFO | Total mem usage: 15.7 GiB | 20260507 14:34:35.329 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:34:35.329 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:34:35.329 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:34:35.329 | INFO | High Water Breakdown: | 20260507 14:34:35.329 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:34:35.329 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:34:35.329 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:34:35.329 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:34:35.329 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:34:35.329 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:34:35.329 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:34:35.329 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:34:35.329 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:34:35.329 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:34:35.329 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:34:35.329 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:34:35.329 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:34:35.329 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:34:35.329 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:34:35.329 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:34:35.329 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:34:35.329 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:34:35.329 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:34:35.329 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:34:35.329 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:34:35.329 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:34:35.329 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:34:35.329 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:34:35.329 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:34:35.329 | INFO | ------------------------------------------ | 20260507 14:34:35.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:34:35.330 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:34:35.330 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:34:35.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:34:35.330 | INFO | PerfReport: Full Execution -- 3.00e+01 / 6.98e+01, 1.27e+01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / 8.49e-02, 4.19e-03 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 10.99%, Baseline -- 3.30e+00 / 8.02e+00, 1.20e-01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / 1.06e+00, 5.26e-02 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 16.09%, DME -- 3.86e+01 / 3.96e+01, 3.10e+00 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.72e+01 / 1.71e+01, 4.40e-02 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.19e+00 / 2.24e+00, 1.29e-01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 3.63%, Metrics -- 1.09e+00 / 1.30e+00, 1.74e-01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.04e-01 / 4.00e-01, 5.91e-02 / 0.00e+00 | 20260507 14:34:35.330 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:34:35.330 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:34:35.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:34:35.330 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 18.50%, Upload -- 1.67e+01 / 0.00e+00, 4.11e-01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 17.97%, Download -- 1.62e+01 / 0.00e+00, 7.12e-01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 33.33%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 0.00%, LockDownload -- 4.33e-04 / 0.00e+00, 9.55e-05 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 29.62%, LockCompute -- 2.67e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:34:35.330 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260507 14:34:35.425 | INFO | Starting a round of Baz IO | 20260507 14:34:38.321 | INFO | Populating RTMetrics report | 20260507 14:34:38.322 | INFO | RTMetrics report populated | 20260507 14:34:38.322 | INFO | Opening tmp RTMetrics json file | 20260507 14:34:38.426 | INFO | Serializing RTMetrics to json | 20260507 14:34:38.438 | INFO | Serialization done, swapping tmp file now... | 20260507 14:34:38.443 | INFO | RTMetrics done | 20260507 14:34:38.457 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.032 seconds | 20260507 14:34:38.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.02% / 0.09% / 41.89% | 20260507 14:34:38.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.76755% / 0.000290076% / 2.78155e-05% | 20260507 14:34:38.458 | INFO | Baz IO overall: | 20260507 14:34:38.458 | INFO | BytesWritten : 88919088818 | 20260507 14:34:38.458 | INFO | NumEvents : 42025010551 | 20260507 14:34:40.467 | INFO | Analyzing chunk frames = [253952,254464) | 20260507 14:34:40.467 | INFO | MemInfo: { | 20260507 14:34:40.467 | INFO | "gpu" : " 17 GB", | 20260507 14:34:40.467 | INFO | "huge" : "", | 20260507 14:34:40.467 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:40.467 | INFO | "malloc" : " 14 GB", | 20260507 14:34:40.467 | INFO | "pinned" : " 29 GB" | 20260507 14:34:40.467 | INFO | } | 20260507 14:34:40.467 | INFO | Perf report for the last 5.13896 seconds: | 20260507 14:34:40.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:40.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:40.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:40.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:40.467 | INFO | ANALYSIS : 57.4%, 2.9, 86.6ms, 0 | 20260507 14:34:40.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:40.467 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 14:34:40.467 | INFO | PRE_HQ : 56.3%, 2.1, 60.1ms, 0 | 20260507 14:34:40.467 | INFO | BAZWRITER : 58.6%, 4.6, 138.8ms, 0 | 20260507 14:34:40.467 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 14:34:40.467 | INFO | Uploaded 36GiB to GPU in 1.64844s (21.8388GiB/s) | 20260507 14:34:40.467 | INFO | Downloaded 38GiB to GPU in 1.72863s (21.9827GiB/s) | 20260507 14:34:45.606 | INFO | Analyzing chunk frames = [254464,254976) | 20260507 14:34:45.606 | INFO | MemInfo: { | 20260507 14:34:45.606 | INFO | "gpu" : " 17 GB", | 20260507 14:34:45.606 | INFO | "huge" : "", | 20260507 14:34:45.606 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:45.606 | INFO | "malloc" : " 14 GB", | 20260507 14:34:45.606 | INFO | "pinned" : " 29 GB" | 20260507 14:34:45.606 | INFO | } | 20260507 14:34:45.606 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:34:45.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:45.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.606 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:34:45.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.607 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:34:45.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:45.607 | INFO | Uploaded 36GiB to GPU in 1.58349s (22.7346GiB/s) | 20260507 14:34:45.607 | INFO | Downloaded 34GiB to GPU in 1.53213s (22.1913GiB/s) | 20260507 14:34:50.745 | INFO | Analyzing chunk frames = [254976,255488) | 20260507 14:34:50.745 | INFO | MemInfo: { | 20260507 14:34:50.745 | INFO | "gpu" : " 17 GB", | 20260507 14:34:50.745 | INFO | "huge" : "", | 20260507 14:34:50.745 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:50.746 | INFO | "malloc" : " 14 GB", | 20260507 14:34:50.746 | INFO | "pinned" : " 29 GB" | 20260507 14:34:50.746 | INFO | } | 20260507 14:34:50.746 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:34:50.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:50.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:34:50.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:34:50.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:50.746 | INFO | Uploaded 36GiB to GPU in 1.58734s (22.6795GiB/s) | 20260507 14:34:50.746 | INFO | Downloaded 34GiB to GPU in 1.5361s (22.134GiB/s) | 20260507 14:34:55.885 | INFO | Analyzing chunk frames = [255488,256000) | 20260507 14:34:55.885 | INFO | MemInfo: { | 20260507 14:34:55.885 | INFO | "gpu" : " 17 GB", | 20260507 14:34:55.885 | INFO | "huge" : "", | 20260507 14:34:55.885 | INFO | "huge_shared" : " 29 GB", | 20260507 14:34:55.885 | INFO | "malloc" : " 14 GB", | 20260507 14:34:55.885 | INFO | "pinned" : " 29 GB" | 20260507 14:34:55.885 | INFO | } | 20260507 14:34:55.885 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:34:55.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:34:55.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.885 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:34:55.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.885 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:34:55.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:34:55.886 | INFO | Uploaded 36GiB to GPU in 1.58685s (22.6865GiB/s) | 20260507 14:34:55.886 | INFO | Downloaded 34GiB to GPU in 1.53635s (22.1304GiB/s) | 20260507 14:35:01.024 | INFO | Analyzing chunk frames = [256000,256512) | 20260507 14:35:01.024 | INFO | MemInfo: { | 20260507 14:35:01.024 | INFO | "gpu" : " 17 GB", | 20260507 14:35:01.024 | INFO | "huge" : "", | 20260507 14:35:01.024 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:01.024 | INFO | "malloc" : " 14 GB", | 20260507 14:35:01.024 | INFO | "pinned" : " 29 GB" | 20260507 14:35:01.024 | INFO | } | 20260507 14:35:01.024 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:35:01.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:01.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.024 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:35:01.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.025 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:35:01.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:01.025 | INFO | Uploaded 36GiB to GPU in 1.58576s (22.7021GiB/s) | 20260507 14:35:01.025 | INFO | Downloaded 34GiB to GPU in 1.53365s (22.1693GiB/s) | 20260507 14:35:06.163 | INFO | Analyzing chunk frames = [256512,257024) | 20260507 14:35:06.164 | INFO | MemInfo: { | 20260507 14:35:06.164 | INFO | "gpu" : " 17 GB", | 20260507 14:35:06.164 | INFO | "huge" : "", | 20260507 14:35:06.164 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:06.164 | INFO | "malloc" : " 14 GB", | 20260507 14:35:06.164 | INFO | "pinned" : " 29 GB" | 20260507 14:35:06.164 | INFO | } | 20260507 14:35:06.164 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:35:06.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:06.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:35:06.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:35:06.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:06.164 | INFO | Uploaded 36GiB to GPU in 1.5864s (22.6928GiB/s) | 20260507 14:35:06.164 | INFO | Downloaded 34GiB to GPU in 1.53349s (22.1717GiB/s) | 20260507 14:35:11.303 | INFO | Analyzing chunk frames = [257024,257536) | 20260507 14:35:11.303 | INFO | MemInfo: { | 20260507 14:35:11.303 | INFO | "gpu" : " 17 GB", | 20260507 14:35:11.303 | INFO | "huge" : "", | 20260507 14:35:11.303 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:11.303 | INFO | "malloc" : " 14 GB", | 20260507 14:35:11.303 | INFO | "pinned" : " 29 GB" | 20260507 14:35:11.303 | INFO | } | 20260507 14:35:11.303 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:35:11.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:11.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:35:11.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:35:11.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:11.303 | INFO | Uploaded 36GiB to GPU in 1.58597s (22.6991GiB/s) | 20260507 14:35:11.304 | INFO | Downloaded 34GiB to GPU in 1.53502s (22.1495GiB/s) | 20260507 14:35:16.442 | INFO | Analyzing chunk frames = [257536,258048) | 20260507 14:35:16.442 | INFO | MemInfo: { | 20260507 14:35:16.442 | INFO | "gpu" : " 17 GB", | 20260507 14:35:16.442 | INFO | "huge" : "", | 20260507 14:35:16.442 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:16.442 | INFO | "malloc" : " 14 GB", | 20260507 14:35:16.442 | INFO | "pinned" : " 29 GB" | 20260507 14:35:16.442 | INFO | } | 20260507 14:35:16.443 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:35:16.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:16.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:35:16.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:35:16.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:16.443 | INFO | Uploaded 36GiB to GPU in 1.5874s (22.6786GiB/s) | 20260507 14:35:16.443 | INFO | Downloaded 34GiB to GPU in 1.53588s (22.1371GiB/s) | 20260507 14:35:16.539 | INFO | Starting a round of Baz IO | 20260507 14:35:19.436 | INFO | Populating RTMetrics report | 20260507 14:35:19.436 | INFO | RTMetrics report populated | 20260507 14:35:19.436 | INFO | Opening tmp RTMetrics json file | 20260507 14:35:19.440 | INFO | Serializing RTMetrics to json | 20260507 14:35:19.445 | INFO | Serialization done, swapping tmp file now... | 20260507 14:35:19.446 | INFO | RTMetrics done | 20260507 14:35:19.596 | INFO | Finished a round of Baz IO: Wrote 1.682 GiB in 3.056 seconds | 20260507 14:35:19.596 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.93% / 0.09% / 41.98% | 20260507 14:35:19.596 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.77459% / 0.00026226% / 1.98682e-05% | 20260507 14:35:19.596 | INFO | Baz IO overall: | 20260507 14:35:19.596 | INFO | BytesWritten : 90725407321 | 20260507 14:35:19.596 | INFO | NumEvents : 42907240152 | 20260507 14:35:21.581 | INFO | Analyzing chunk frames = [258048,258560) | 20260507 14:35:21.581 | INFO | MemInfo: { | 20260507 14:35:21.581 | INFO | "gpu" : " 17 GB", | 20260507 14:35:21.582 | INFO | "huge" : "", | 20260507 14:35:21.582 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:21.582 | INFO | "malloc" : " 14 GB", | 20260507 14:35:21.582 | INFO | "pinned" : " 29 GB" | 20260507 14:35:21.582 | INFO | } | 20260507 14:35:21.582 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:35:21.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:21.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:21.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:21.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:21.582 | INFO | ANALYSIS : 57.4%, 2.9, 87.4ms, 0 | 20260507 14:35:21.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:21.582 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 14:35:21.582 | INFO | PRE_HQ : 56.1%, 2.1, 59.9ms, 0 | 20260507 14:35:21.582 | INFO | BAZWRITER : 59.0%, 4.5, 137.7ms, 0 | 20260507 14:35:21.582 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 14:35:21.582 | INFO | Uploaded 36GiB to GPU in 1.65022s (21.8153GiB/s) | 20260507 14:35:21.582 | INFO | Downloaded 38GiB to GPU in 1.73231s (21.936GiB/s) | 20260507 14:35:26.721 | INFO | Analyzing chunk frames = [258560,259072) | 20260507 14:35:26.721 | INFO | MemInfo: { | 20260507 14:35:26.721 | INFO | "gpu" : " 17 GB", | 20260507 14:35:26.721 | INFO | "huge" : "", | 20260507 14:35:26.721 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:26.721 | INFO | "malloc" : " 14 GB", | 20260507 14:35:26.721 | INFO | "pinned" : " 29 GB" | 20260507 14:35:26.721 | INFO | } | 20260507 14:35:26.721 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:35:26.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:26.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:35:26.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:35:26.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:26.721 | INFO | Uploaded 36GiB to GPU in 1.58409s (22.7261GiB/s) | 20260507 14:35:26.721 | INFO | Downloaded 34GiB to GPU in 1.53178s (22.1965GiB/s) | 20260507 14:35:31.860 | INFO | Analyzing chunk frames = [259072,259584) | 20260507 14:35:31.860 | INFO | MemInfo: { | 20260507 14:35:31.860 | INFO | "gpu" : " 17 GB", | 20260507 14:35:31.860 | INFO | "huge" : "", | 20260507 14:35:31.860 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:31.860 | INFO | "malloc" : " 14 GB", | 20260507 14:35:31.860 | INFO | "pinned" : " 29 GB" | 20260507 14:35:31.861 | INFO | } | 20260507 14:35:31.861 | INFO | Perf report for the last 5.1395 seconds: | 20260507 14:35:31.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:31.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | ANALYSIS : 56.5%, 3.0, 87.9ms, 0 | 20260507 14:35:31.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:35:31.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:31.861 | INFO | Uploaded 36GiB to GPU in 1.58725s (22.6807GiB/s) | 20260507 14:35:31.861 | INFO | Downloaded 34GiB to GPU in 1.53573s (22.1393GiB/s) | 20260507 14:35:37.000 | INFO | Analyzing chunk frames = [259584,260096) | 20260507 14:35:37.000 | INFO | MemInfo: { | 20260507 14:35:37.000 | INFO | "gpu" : " 17 GB", | 20260507 14:35:37.000 | INFO | "huge" : "", | 20260507 14:35:37.000 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:37.000 | INFO | "malloc" : " 14 GB", | 20260507 14:35:37.000 | INFO | "pinned" : " 29 GB" | 20260507 14:35:37.000 | INFO | } | 20260507 14:35:37.000 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:35:37.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:37.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:35:37.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:35:37.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:37.000 | INFO | Uploaded 36GiB to GPU in 1.58813s (22.6682GiB/s) | 20260507 14:35:37.000 | INFO | Downloaded 34GiB to GPU in 1.53598s (22.1357GiB/s) | 20260507 14:35:42.139 | INFO | Analyzing chunk frames = [260096,260608) | 20260507 14:35:42.139 | INFO | MemInfo: { | 20260507 14:35:42.139 | INFO | "gpu" : " 17 GB", | 20260507 14:35:42.139 | INFO | "huge" : "", | 20260507 14:35:42.139 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:42.139 | INFO | "malloc" : " 14 GB", | 20260507 14:35:42.139 | INFO | "pinned" : " 29 GB" | 20260507 14:35:42.139 | INFO | } | 20260507 14:35:42.139 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:35:42.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:42.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 14:35:42.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:35:42.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:42.140 | INFO | Uploaded 36GiB to GPU in 1.58589s (22.7001GiB/s) | 20260507 14:35:42.140 | INFO | Downloaded 34GiB to GPU in 1.53377s (22.1677GiB/s) | 20260507 14:35:47.279 | INFO | Analyzing chunk frames = [260608,261120) | 20260507 14:35:47.279 | INFO | MemInfo: { | 20260507 14:35:47.279 | INFO | "gpu" : " 17 GB", | 20260507 14:35:47.279 | INFO | "huge" : "", | 20260507 14:35:47.279 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:47.279 | INFO | "malloc" : " 14 GB", | 20260507 14:35:47.279 | INFO | "pinned" : " 29 GB" | 20260507 14:35:47.279 | INFO | } | 20260507 14:35:47.279 | INFO | Perf report for the last 5.13959 seconds: | 20260507 14:35:47.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:47.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:35:47.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 14:35:47.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:47.279 | INFO | Uploaded 36GiB to GPU in 1.58748s (22.6774GiB/s) | 20260507 14:35:47.279 | INFO | Downloaded 34GiB to GPU in 1.5347s (22.1542GiB/s) | 20260507 14:35:52.418 | INFO | Analyzing chunk frames = [261120,261632) | 20260507 14:35:52.418 | INFO | MemInfo: { | 20260507 14:35:52.418 | INFO | "gpu" : " 17 GB", | 20260507 14:35:52.418 | INFO | "huge" : "", | 20260507 14:35:52.418 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:52.418 | INFO | "malloc" : " 14 GB", | 20260507 14:35:52.418 | INFO | "pinned" : " 29 GB" | 20260507 14:35:52.418 | INFO | } | 20260507 14:35:52.418 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:35:52.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:52.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:35:52.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:35:52.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:52.419 | INFO | Uploaded 36GiB to GPU in 1.58622s (22.6955GiB/s) | 20260507 14:35:52.419 | INFO | Downloaded 34GiB to GPU in 1.53594s (22.1362GiB/s) | 20260507 14:35:57.558 | INFO | Analyzing chunk frames = [261632,262144) | 20260507 14:35:57.558 | INFO | MemInfo: { | 20260507 14:35:57.558 | INFO | "gpu" : " 17 GB", | 20260507 14:35:57.558 | INFO | "huge" : "", | 20260507 14:35:57.558 | INFO | "huge_shared" : " 29 GB", | 20260507 14:35:57.558 | INFO | "malloc" : " 14 GB", | 20260507 14:35:57.558 | INFO | "pinned" : " 29 GB" | 20260507 14:35:57.558 | INFO | } | 20260507 14:35:57.558 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:35:57.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:35:57.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:35:57.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:35:57.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:35:57.558 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:35:57.558 | INFO | Total mem usage: 15.5 GiB | 20260507 14:35:57.558 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.48%) | 20260507 14:35:57.558 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -3.14%) | 20260507 14:35:57.558 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:35:57.558 | INFO | High Water Breakdown: | 20260507 14:35:57.558 | INFO | 58.9% : 8.86 GiB ( 96.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:35:57.558 | INFO | 41% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:35:57.558 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:35:57.558 | INFO | ------------------------------------------ | 20260507 14:35:57.558 | INFO | Uploaded 36GiB to GPU in 1.58674s (22.688GiB/s) | 20260507 14:35:57.558 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:35:57.559 | INFO | Downloaded 34GiB to GPU in 1.53608s (22.1342GiB/s) | 20260507 14:35:57.559 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:35:57.559 | INFO | Total mem usage: 26.5 GiB | 20260507 14:35:57.559 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:35:57.559 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:35:57.559 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:35:57.559 | INFO | High Water Breakdown: | 20260507 14:35:57.559 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:35:57.559 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:35:57.559 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:35:57.559 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:35:57.559 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:35:57.559 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:35:57.559 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:35:57.559 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:35:57.559 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:35:57.559 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:35:57.559 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:35:57.559 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:35:57.559 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:35:57.559 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:35:57.559 | INFO | ------------------------------------------ | 20260507 14:35:57.559 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:35:57.559 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:35:57.559 | INFO | Total mem usage: 27.8 GiB | 20260507 14:35:57.559 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:35:57.559 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:35:57.559 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:35:57.559 | INFO | High Water Breakdown: | 20260507 14:35:57.559 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:35:57.559 | INFO | ------------------------------------------ | 20260507 14:35:57.559 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:35:57.559 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:35:57.559 | INFO | Total mem usage: 15.7 GiB | 20260507 14:35:57.559 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:35:57.559 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:35:57.559 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:35:57.559 | INFO | High Water Breakdown: | 20260507 14:35:57.559 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:35:57.559 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:35:57.559 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:35:57.559 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:35:57.559 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:35:57.559 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:35:57.559 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:35:57.559 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:35:57.559 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:35:57.559 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:35:57.559 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:35:57.559 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:35:57.559 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:35:57.559 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:35:57.559 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:35:57.559 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:35:57.559 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:35:57.559 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:35:57.559 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:35:57.559 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:35:57.559 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:35:57.559 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:35:57.559 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:35:57.559 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:35:57.559 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:35:57.559 | INFO | ------------------------------------------ | 20260507 14:35:57.559 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:35:57.559 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:35:57.559 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:35:57.559 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:35:57.559 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.95e-03 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 10.99%, Baseline -- 3.30e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 16.10%, DME -- 3.86e+01 / -nan, 2.96e+00 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.19e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 3.62%, Metrics -- 1.09e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260507 14:35:57.559 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.03e-01 / -nan, 5.93e-02 / 0.00e+00 | 20260507 14:35:57.560 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:35:57.560 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:35:57.560 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:35:57.560 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 18.49%, Upload -- 1.66e+01 / 0.00e+00, 4.73e-01 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 18.08%, Download -- 1.62e+01 / 0.00e+00, 7.71e-01 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 33.40%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 0.00%, LockDownload -- 4.30e-04 / 0.00e+00, 9.23e-05 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 29.44%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:35:57.560 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260507 14:35:57.655 | INFO | Starting a round of Baz IO | 20260507 14:36:00.556 | INFO | Populating RTMetrics report | 20260507 14:36:00.557 | INFO | RTMetrics report populated | 20260507 14:36:00.557 | INFO | Opening tmp RTMetrics json file | 20260507 14:36:00.600 | INFO | Serializing RTMetrics to json | 20260507 14:36:00.611 | INFO | Serialization done, swapping tmp file now... | 20260507 14:36:00.619 | INFO | RTMetrics done | 20260507 14:36:00.903 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.247 seconds | 20260507 14:36:00.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.84% / 0.09% / 42.07% | 20260507 14:36:00.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.69912% / 0.000222524% / 1.19209e-05% | 20260507 14:36:00.903 | INFO | Baz IO overall: | 20260507 14:36:00.903 | INFO | BytesWritten : 92530077973 | 20260507 14:36:00.903 | INFO | NumEvents : 43787698139 | 20260507 14:36:02.697 | INFO | Analyzing chunk frames = [262144,262656) | 20260507 14:36:02.697 | INFO | MemInfo: { | 20260507 14:36:02.697 | INFO | "gpu" : " 17 GB", | 20260507 14:36:02.697 | INFO | "huge" : "", | 20260507 14:36:02.697 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:02.697 | INFO | "malloc" : " 14 GB", | 20260507 14:36:02.697 | INFO | "pinned" : " 29 GB" | 20260507 14:36:02.697 | INFO | } | 20260507 14:36:02.697 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:36:02.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:02.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:02.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:02.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:02.697 | INFO | ANALYSIS : 57.5%, 2.9, 87.3ms, 0 | 20260507 14:36:02.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:02.697 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 14:36:02.697 | INFO | PRE_HQ : 56.2%, 2.1, 60.0ms, 0 | 20260507 14:36:02.697 | INFO | BAZWRITER : 62.7%, 4.6, 148.7ms, 0 | 20260507 14:36:02.697 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 14:36:02.698 | INFO | Uploaded 36GiB to GPU in 1.647s (21.8579GiB/s) | 20260507 14:36:02.698 | INFO | Downloaded 38GiB to GPU in 1.72777s (21.9937GiB/s) | 20260507 14:36:07.836 | INFO | Analyzing chunk frames = [262656,263168) | 20260507 14:36:07.836 | INFO | MemInfo: { | 20260507 14:36:07.836 | INFO | "gpu" : " 17 GB", | 20260507 14:36:07.836 | INFO | "huge" : "", | 20260507 14:36:07.836 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:07.836 | INFO | "malloc" : " 14 GB", | 20260507 14:36:07.836 | INFO | "pinned" : " 29 GB" | 20260507 14:36:07.836 | INFO | } | 20260507 14:36:07.836 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:36:07.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:07.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.837 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:36:07.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.837 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:36:07.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:07.837 | INFO | Uploaded 36GiB to GPU in 1.58333s (22.7369GiB/s) | 20260507 14:36:07.837 | INFO | Downloaded 34GiB to GPU in 1.53107s (22.2067GiB/s) | 20260507 14:36:12.976 | INFO | Analyzing chunk frames = [263168,263680) | 20260507 14:36:12.976 | INFO | MemInfo: { | 20260507 14:36:12.976 | INFO | "gpu" : " 17 GB", | 20260507 14:36:12.976 | INFO | "huge" : "", | 20260507 14:36:12.976 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:12.976 | INFO | "malloc" : " 14 GB", | 20260507 14:36:12.976 | INFO | "pinned" : " 29 GB" | 20260507 14:36:12.976 | INFO | } | 20260507 14:36:12.976 | INFO | Perf report for the last 5.13987 seconds: | 20260507 14:36:12.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:12.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.976 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:36:12.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.976 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:36:12.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:12.977 | INFO | Uploaded 36GiB to GPU in 1.58739s (22.6788GiB/s) | 20260507 14:36:12.977 | INFO | Downloaded 34GiB to GPU in 1.53668s (22.1256GiB/s) | 20260507 14:36:18.115 | INFO | Analyzing chunk frames = [263680,264192) | 20260507 14:36:18.115 | INFO | MemInfo: { | 20260507 14:36:18.115 | INFO | "gpu" : " 17 GB", | 20260507 14:36:18.115 | INFO | "huge" : "", | 20260507 14:36:18.115 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:18.115 | INFO | "malloc" : " 14 GB", | 20260507 14:36:18.115 | INFO | "pinned" : " 29 GB" | 20260507 14:36:18.115 | INFO | } | 20260507 14:36:18.115 | INFO | Perf report for the last 5.1389 seconds: | 20260507 14:36:18.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:18.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.115 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:36:18.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.115 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:36:18.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:18.116 | INFO | Uploaded 36GiB to GPU in 1.58744s (22.678GiB/s) | 20260507 14:36:18.116 | INFO | Downloaded 34GiB to GPU in 1.53642s (22.1293GiB/s) | 20260507 14:36:23.255 | INFO | Analyzing chunk frames = [264192,264704) | 20260507 14:36:23.255 | INFO | MemInfo: { | 20260507 14:36:23.255 | INFO | "gpu" : " 17 GB", | 20260507 14:36:23.255 | INFO | "huge" : "", | 20260507 14:36:23.255 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:23.255 | INFO | "malloc" : " 14 GB", | 20260507 14:36:23.255 | INFO | "pinned" : " 29 GB" | 20260507 14:36:23.255 | INFO | } | 20260507 14:36:23.255 | INFO | Perf report for the last 5.13953 seconds: | 20260507 14:36:23.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:23.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 14:36:23.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:36:23.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:23.255 | INFO | Uploaded 36GiB to GPU in 1.58615s (22.6965GiB/s) | 20260507 14:36:23.255 | INFO | Downloaded 34GiB to GPU in 1.53446s (22.1576GiB/s) | 20260507 14:36:28.394 | INFO | Analyzing chunk frames = [264704,265216) | 20260507 14:36:28.394 | INFO | MemInfo: { | 20260507 14:36:28.394 | INFO | "gpu" : " 17 GB", | 20260507 14:36:28.394 | INFO | "huge" : "", | 20260507 14:36:28.394 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:28.394 | INFO | "malloc" : " 14 GB", | 20260507 14:36:28.394 | INFO | "pinned" : " 29 GB" | 20260507 14:36:28.394 | INFO | } | 20260507 14:36:28.394 | INFO | Perf report for the last 5.13895 seconds: | 20260507 14:36:28.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:28.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:36:28.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:36:28.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:28.394 | INFO | Uploaded 36GiB to GPU in 1.58555s (22.705GiB/s) | 20260507 14:36:28.394 | INFO | Downloaded 34GiB to GPU in 1.53324s (22.1753GiB/s) | 20260507 14:36:33.533 | INFO | Analyzing chunk frames = [265216,265728) | 20260507 14:36:33.533 | INFO | MemInfo: { | 20260507 14:36:33.533 | INFO | "gpu" : " 17 GB", | 20260507 14:36:33.533 | INFO | "huge" : "", | 20260507 14:36:33.533 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:33.533 | INFO | "malloc" : " 14 GB", | 20260507 14:36:33.533 | INFO | "pinned" : " 29 GB" | 20260507 14:36:33.533 | INFO | } | 20260507 14:36:33.533 | INFO | Perf report for the last 5.13954 seconds: | 20260507 14:36:33.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:33.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:36:33.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:36:33.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:33.533 | INFO | Uploaded 36GiB to GPU in 1.58572s (22.7027GiB/s) | 20260507 14:36:33.534 | INFO | Downloaded 34GiB to GPU in 1.53617s (22.133GiB/s) | 20260507 14:36:38.672 | INFO | Analyzing chunk frames = [265728,266240) | 20260507 14:36:38.672 | INFO | MemInfo: { | 20260507 14:36:38.672 | INFO | "gpu" : " 17 GB", | 20260507 14:36:38.672 | INFO | "huge" : "", | 20260507 14:36:38.673 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:38.673 | INFO | "malloc" : " 14 GB", | 20260507 14:36:38.673 | INFO | "pinned" : " 29 GB" | 20260507 14:36:38.673 | INFO | } | 20260507 14:36:38.673 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:36:38.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:38.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:36:38.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 14:36:38.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:38.673 | INFO | Uploaded 36GiB to GPU in 1.5858s (22.7015GiB/s) | 20260507 14:36:38.673 | INFO | Downloaded 34GiB to GPU in 1.53586s (22.1374GiB/s) | 20260507 14:36:38.770 | INFO | Starting a round of Baz IO | 20260507 14:36:41.662 | INFO | Populating RTMetrics report | 20260507 14:36:41.663 | INFO | RTMetrics report populated | 20260507 14:36:41.663 | INFO | Opening tmp RTMetrics json file | 20260507 14:36:41.666 | INFO | Serializing RTMetrics to json | 20260507 14:36:41.709 | INFO | Serialization done, swapping tmp file now... | 20260507 14:36:41.753 | INFO | RTMetrics done | 20260507 14:36:41.789 | INFO | Finished a round of Baz IO: Wrote 1.682 GiB in 3.018 seconds | 20260507 14:36:41.789 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.76% / 0.09% / 42.16% | 20260507 14:36:41.789 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.6674% / 0.000242392% / 3.17891e-05% | 20260507 14:36:41.789 | INFO | Baz IO overall: | 20260507 14:36:41.789 | INFO | BytesWritten : 94336321633 | 20260507 14:36:41.789 | INFO | NumEvents : 44668710825 | 20260507 14:36:43.812 | INFO | Analyzing chunk frames = [266240,266752) | 20260507 14:36:43.812 | INFO | MemInfo: { | 20260507 14:36:43.812 | INFO | "gpu" : " 17 GB", | 20260507 14:36:43.812 | INFO | "huge" : "", | 20260507 14:36:43.812 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:43.812 | INFO | "malloc" : " 14 GB", | 20260507 14:36:43.812 | INFO | "pinned" : " 29 GB" | 20260507 14:36:43.812 | INFO | } | 20260507 14:36:43.812 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:36:43.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:43.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:43.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:43.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:43.812 | INFO | ANALYSIS : 57.3%, 3.0, 89.2ms, 0 | 20260507 14:36:43.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:43.812 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 14:36:43.812 | INFO | PRE_HQ : 56.0%, 2.1, 60.0ms, 0 | 20260507 14:36:43.812 | INFO | BAZWRITER : 58.3%, 4.0, 120.0ms, 0 | 20260507 14:36:43.812 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 14:36:43.812 | INFO | Uploaded 36GiB to GPU in 1.62914s (22.0975GiB/s) | 20260507 14:36:43.812 | INFO | Downloaded 38GiB to GPU in 1.71136s (22.2046GiB/s) | 20260507 14:36:48.951 | INFO | Analyzing chunk frames = [266752,267264) | 20260507 14:36:48.951 | INFO | MemInfo: { | 20260507 14:36:48.951 | INFO | "gpu" : " 17 GB", | 20260507 14:36:48.951 | INFO | "huge" : "", | 20260507 14:36:48.951 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:48.951 | INFO | "malloc" : " 14 GB", | 20260507 14:36:48.951 | INFO | "pinned" : " 29 GB" | 20260507 14:36:48.951 | INFO | } | 20260507 14:36:48.951 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:36:48.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:48.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.952 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:36:48.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.952 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:36:48.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:48.952 | INFO | Uploaded 36GiB to GPU in 1.58358s (22.7334GiB/s) | 20260507 14:36:48.952 | INFO | Downloaded 34GiB to GPU in 1.53211s (22.1916GiB/s) | 20260507 14:36:54.090 | INFO | Analyzing chunk frames = [267264,267776) | 20260507 14:36:54.090 | INFO | MemInfo: { | 20260507 14:36:54.090 | INFO | "gpu" : " 17 GB", | 20260507 14:36:54.090 | INFO | "huge" : "", | 20260507 14:36:54.090 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:54.091 | INFO | "malloc" : " 14 GB", | 20260507 14:36:54.091 | INFO | "pinned" : " 29 GB" | 20260507 14:36:54.091 | INFO | } | 20260507 14:36:54.091 | INFO | Perf report for the last 5.13915 seconds: | 20260507 14:36:54.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:54.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:36:54.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:36:54.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:54.091 | INFO | Uploaded 36GiB to GPU in 1.58682s (22.6869GiB/s) | 20260507 14:36:54.091 | INFO | Downloaded 34GiB to GPU in 1.53651s (22.128GiB/s) | 20260507 14:36:59.230 | INFO | Analyzing chunk frames = [267776,268288) | 20260507 14:36:59.230 | INFO | MemInfo: { | 20260507 14:36:59.230 | INFO | "gpu" : " 17 GB", | 20260507 14:36:59.230 | INFO | "huge" : "", | 20260507 14:36:59.230 | INFO | "huge_shared" : " 29 GB", | 20260507 14:36:59.230 | INFO | "malloc" : " 14 GB", | 20260507 14:36:59.230 | INFO | "pinned" : " 29 GB" | 20260507 14:36:59.230 | INFO | } | 20260507 14:36:59.230 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:36:59.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:36:59.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.230 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:36:59.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.230 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:36:59.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:36:59.231 | INFO | Uploaded 36GiB to GPU in 1.5875s (22.6772GiB/s) | 20260507 14:36:59.231 | INFO | Downloaded 34GiB to GPU in 1.53558s (22.1415GiB/s) | 20260507 14:37:04.369 | INFO | Analyzing chunk frames = [268288,268800) | 20260507 14:37:04.369 | INFO | MemInfo: { | 20260507 14:37:04.369 | INFO | "gpu" : " 17 GB", | 20260507 14:37:04.369 | INFO | "huge" : "", | 20260507 14:37:04.369 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:04.369 | INFO | "malloc" : " 14 GB", | 20260507 14:37:04.369 | INFO | "pinned" : " 29 GB" | 20260507 14:37:04.369 | INFO | } | 20260507 14:37:04.369 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:37:04.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:04.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.370 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 14:37:04.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.370 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:37:04.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:04.370 | INFO | Uploaded 36GiB to GPU in 1.58602s (22.6983GiB/s) | 20260507 14:37:04.370 | INFO | Downloaded 34GiB to GPU in 1.53479s (22.1529GiB/s) | 20260507 14:37:09.508 | INFO | Analyzing chunk frames = [268800,269312) | 20260507 14:37:09.508 | INFO | MemInfo: { | 20260507 14:37:09.508 | INFO | "gpu" : " 17 GB", | 20260507 14:37:09.508 | INFO | "huge" : "", | 20260507 14:37:09.509 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:09.509 | INFO | "malloc" : " 14 GB", | 20260507 14:37:09.509 | INFO | "pinned" : " 29 GB" | 20260507 14:37:09.509 | INFO | } | 20260507 14:37:09.509 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:37:09.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:09.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | ANALYSIS : 56.4%, 3.0, 88.1ms, 0 | 20260507 14:37:09.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:37:09.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:09.509 | INFO | Uploaded 36GiB to GPU in 1.58622s (22.6954GiB/s) | 20260507 14:37:09.509 | INFO | Downloaded 34GiB to GPU in 1.53356s (22.1706GiB/s) | 20260507 14:37:14.648 | INFO | Analyzing chunk frames = [269312,269824) | 20260507 14:37:14.648 | INFO | MemInfo: { | 20260507 14:37:14.648 | INFO | "gpu" : " 17 GB", | 20260507 14:37:14.648 | INFO | "huge" : "", | 20260507 14:37:14.648 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:14.648 | INFO | "malloc" : " 14 GB", | 20260507 14:37:14.648 | INFO | "pinned" : " 29 GB" | 20260507 14:37:14.648 | INFO | } | 20260507 14:37:14.648 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:37:14.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:14.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.648 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:37:14.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.648 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:37:14.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:14.649 | INFO | Uploaded 36GiB to GPU in 1.5869s (22.6857GiB/s) | 20260507 14:37:14.649 | INFO | Downloaded 34GiB to GPU in 1.53533s (22.1451GiB/s) | 20260507 14:37:19.787 | INFO | Analyzing chunk frames = [269824,270336) | 20260507 14:37:19.787 | INFO | MemInfo: { | 20260507 14:37:19.787 | INFO | "gpu" : " 17 GB", | 20260507 14:37:19.787 | INFO | "huge" : "", | 20260507 14:37:19.787 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:19.787 | INFO | "malloc" : " 14 GB", | 20260507 14:37:19.787 | INFO | "pinned" : " 29 GB" | 20260507 14:37:19.787 | INFO | } | 20260507 14:37:19.788 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:37:19.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:19.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:37:19.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 14:37:19.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:19.788 | INFO | Uploaded 36GiB to GPU in 1.58659s (22.6901GiB/s) | 20260507 14:37:19.788 | INFO | Downloaded 34GiB to GPU in 1.53575s (22.139GiB/s) | 20260507 14:37:19.788 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:37:19.788 | INFO | Total mem usage: 15.7 GiB | 20260507 14:37:19.788 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:37:19.788 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: 0.0491%) | 20260507 14:37:19.788 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:37:19.788 | INFO | High Water Breakdown: | 20260507 14:37:19.788 | INFO | 59.6% : 9.34 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:37:19.788 | INFO | 40.3% : 6.31 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:37:19.788 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:37:19.788 | INFO | ------------------------------------------ | 20260507 14:37:19.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:37:19.788 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:37:19.788 | INFO | Total mem usage: 26.5 GiB | 20260507 14:37:19.788 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:37:19.788 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:37:19.788 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:37:19.788 | INFO | High Water Breakdown: | 20260507 14:37:19.788 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:37:19.788 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:37:19.788 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:37:19.788 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:37:19.788 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:37:19.788 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:37:19.788 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:37:19.788 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:37:19.788 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:37:19.788 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:37:19.788 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:37:19.788 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:37:19.788 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:37:19.788 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:37:19.788 | INFO | ------------------------------------------ | 20260507 14:37:19.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:37:19.788 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:37:19.788 | INFO | Total mem usage: 27.8 GiB | 20260507 14:37:19.788 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:37:19.788 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:37:19.788 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:37:19.788 | INFO | High Water Breakdown: | 20260507 14:37:19.788 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:37:19.788 | INFO | ------------------------------------------ | 20260507 14:37:19.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:37:19.788 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:37:19.788 | INFO | Total mem usage: 15.7 GiB | 20260507 14:37:19.788 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:37:19.788 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:37:19.788 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:37:19.788 | INFO | High Water Breakdown: | 20260507 14:37:19.788 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:37:19.788 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:37:19.788 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:37:19.788 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:37:19.788 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:37:19.788 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:37:19.788 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:37:19.788 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:37:19.788 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:37:19.788 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:37:19.788 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:37:19.788 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:37:19.788 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:37:19.788 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:37:19.788 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:37:19.788 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:37:19.788 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:37:19.788 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:37:19.788 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:37:19.788 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:37:19.788 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:37:19.788 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:37:19.788 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:37:19.788 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:37:19.788 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:37:19.788 | INFO | ------------------------------------------ | 20260507 14:37:19.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:37:19.788 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:37:19.788 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:37:19.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:37:19.789 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.30e-03 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 10.98%, Baseline -- 3.29e+00 / -nan, 1.79e-02 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 16.10%, DME -- 3.86e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.72e+01 / -nan, 4.46e-02 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.19e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.02e-01 / -nan, 5.89e-02 / 0.00e+00 | 20260507 14:37:19.789 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:37:19.789 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:37:19.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:37:19.789 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 18.44%, Upload -- 1.66e+01 / 0.00e+00, 4.42e-01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 18.04%, Download -- 1.62e+01 / 0.00e+00, 7.27e-01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 33.34%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 0.57%, LockUpload -- 5.11e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 0.00%, LockDownload -- 4.30e-04 / 0.00e+00, 9.54e-05 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 29.59%, LockCompute -- 2.66e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:37:19.789 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260507 14:37:19.885 | INFO | Starting a round of Baz IO | 20260507 14:37:22.776 | INFO | Populating RTMetrics report | 20260507 14:37:22.777 | INFO | RTMetrics report populated | 20260507 14:37:22.777 | INFO | Opening tmp RTMetrics json file | 20260507 14:37:22.794 | INFO | Serializing RTMetrics to json | 20260507 14:37:22.799 | INFO | Serialization done, swapping tmp file now... | 20260507 14:37:22.801 | INFO | RTMetrics done | 20260507 14:37:23.026 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.141 seconds | 20260507 14:37:23.026 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.67% / 0.09% / 42.24% | 20260507 14:37:23.026 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.64505% / 0.000202656% / 1.19209e-05% | 20260507 14:37:23.026 | INFO | Baz IO overall: | 20260507 14:37:23.026 | INFO | BytesWritten : 96144810256 | 20260507 14:37:23.026 | INFO | NumEvents : 45551105719 | 20260507 14:37:24.927 | INFO | Analyzing chunk frames = [270336,270848) | 20260507 14:37:24.927 | INFO | MemInfo: { | 20260507 14:37:24.927 | INFO | "gpu" : " 17 GB", | 20260507 14:37:24.927 | INFO | "huge" : "", | 20260507 14:37:24.927 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:24.927 | INFO | "malloc" : " 14 GB", | 20260507 14:37:24.927 | INFO | "pinned" : " 29 GB" | 20260507 14:37:24.927 | INFO | } | 20260507 14:37:24.927 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:37:24.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:24.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:24.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:24.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:24.927 | INFO | ANALYSIS : 57.3%, 2.9, 87.5ms, 0 | 20260507 14:37:24.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:24.927 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 14:37:24.927 | INFO | PRE_HQ : 56.1%, 2.0, 59.7ms, 0 | 20260507 14:37:24.927 | INFO | BAZWRITER : 60.6%, 4.5, 140.3ms, 0 | 20260507 14:37:24.927 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 14:37:24.927 | INFO | Uploaded 36GiB to GPU in 1.6457s (21.8751GiB/s) | 20260507 14:37:24.927 | INFO | Downloaded 38GiB to GPU in 1.72716s (22.0015GiB/s) | 20260507 14:37:30.066 | INFO | Analyzing chunk frames = [270848,271360) | 20260507 14:37:30.066 | INFO | MemInfo: { | 20260507 14:37:30.066 | INFO | "gpu" : " 17 GB", | 20260507 14:37:30.066 | INFO | "huge" : "", | 20260507 14:37:30.066 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:30.066 | INFO | "malloc" : " 14 GB", | 20260507 14:37:30.066 | INFO | "pinned" : " 29 GB" | 20260507 14:37:30.066 | INFO | } | 20260507 14:37:30.066 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:37:30.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:30.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 14:37:30.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:37:30.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:30.066 | INFO | Uploaded 36GiB to GPU in 1.5832s (22.7387GiB/s) | 20260507 14:37:30.066 | INFO | Downloaded 34GiB to GPU in 1.53142s (22.2016GiB/s) | 20260507 14:37:35.205 | INFO | Analyzing chunk frames = [271360,271872) | 20260507 14:37:35.205 | INFO | MemInfo: { | 20260507 14:37:35.205 | INFO | "gpu" : " 17 GB", | 20260507 14:37:35.205 | INFO | "huge" : "", | 20260507 14:37:35.205 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:35.205 | INFO | "malloc" : " 14 GB", | 20260507 14:37:35.205 | INFO | "pinned" : " 29 GB" | 20260507 14:37:35.205 | INFO | } | 20260507 14:37:35.205 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:37:35.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:35.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.205 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:37:35.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.205 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:37:35.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:35.206 | INFO | Uploaded 36GiB to GPU in 1.58656s (22.6906GiB/s) | 20260507 14:37:35.206 | INFO | Downloaded 34GiB to GPU in 1.53634s (22.1305GiB/s) | 20260507 14:37:40.345 | INFO | Analyzing chunk frames = [271872,272384) | 20260507 14:37:40.345 | INFO | MemInfo: { | 20260507 14:37:40.345 | INFO | "gpu" : " 17 GB", | 20260507 14:37:40.345 | INFO | "huge" : "", | 20260507 14:37:40.345 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:40.345 | INFO | "malloc" : " 14 GB", | 20260507 14:37:40.345 | INFO | "pinned" : " 29 GB" | 20260507 14:37:40.345 | INFO | } | 20260507 14:37:40.345 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:37:40.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:40.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:37:40.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:37:40.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:40.345 | INFO | Uploaded 36GiB to GPU in 1.58714s (22.6824GiB/s) | 20260507 14:37:40.345 | INFO | Downloaded 34GiB to GPU in 1.53588s (22.1371GiB/s) | 20260507 14:37:45.484 | INFO | Analyzing chunk frames = [272384,272896) | 20260507 14:37:45.484 | INFO | MemInfo: { | 20260507 14:37:45.484 | INFO | "gpu" : " 17 GB", | 20260507 14:37:45.484 | INFO | "huge" : "", | 20260507 14:37:45.484 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:45.484 | INFO | "malloc" : " 14 GB", | 20260507 14:37:45.484 | INFO | "pinned" : " 29 GB" | 20260507 14:37:45.484 | INFO | } | 20260507 14:37:45.484 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:37:45.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:45.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.484 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 14:37:45.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.484 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:37:45.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:45.485 | INFO | Uploaded 36GiB to GPU in 1.5855s (22.7058GiB/s) | 20260507 14:37:45.485 | INFO | Downloaded 34GiB to GPU in 1.53384s (22.1666GiB/s) | 20260507 14:37:50.623 | INFO | Analyzing chunk frames = [272896,273408) | 20260507 14:37:50.623 | INFO | MemInfo: { | 20260507 14:37:50.623 | INFO | "gpu" : " 17 GB", | 20260507 14:37:50.623 | INFO | "huge" : "", | 20260507 14:37:50.624 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:50.624 | INFO | "malloc" : " 14 GB", | 20260507 14:37:50.624 | INFO | "pinned" : " 29 GB" | 20260507 14:37:50.624 | INFO | } | 20260507 14:37:50.624 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:37:50.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:50.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:37:50.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:37:50.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:50.624 | INFO | Uploaded 36GiB to GPU in 1.58653s (22.691GiB/s) | 20260507 14:37:50.624 | INFO | Downloaded 34GiB to GPU in 1.53423s (22.161GiB/s) | 20260507 14:37:55.763 | INFO | Analyzing chunk frames = [273408,273920) | 20260507 14:37:55.763 | INFO | MemInfo: { | 20260507 14:37:55.763 | INFO | "gpu" : " 17 GB", | 20260507 14:37:55.763 | INFO | "huge" : "", | 20260507 14:37:55.763 | INFO | "huge_shared" : " 29 GB", | 20260507 14:37:55.763 | INFO | "malloc" : " 14 GB", | 20260507 14:37:55.763 | INFO | "pinned" : " 29 GB" | 20260507 14:37:55.763 | INFO | } | 20260507 14:37:55.763 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:37:55.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:37:55.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:37:55.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 14:37:55.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:37:55.763 | INFO | Uploaded 36GiB to GPU in 1.58577s (22.7019GiB/s) | 20260507 14:37:55.763 | INFO | Downloaded 34GiB to GPU in 1.53536s (22.1447GiB/s) | 20260507 14:38:00.902 | INFO | Analyzing chunk frames = [273920,274432) | 20260507 14:38:00.902 | INFO | MemInfo: { | 20260507 14:38:00.902 | INFO | "gpu" : " 17 GB", | 20260507 14:38:00.902 | INFO | "huge" : "", | 20260507 14:38:00.902 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:00.902 | INFO | "malloc" : " 14 GB", | 20260507 14:38:00.902 | INFO | "pinned" : " 29 GB" | 20260507 14:38:00.902 | INFO | } | 20260507 14:38:00.902 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:38:00.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:00.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.902 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:38:00.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.902 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:38:00.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:00.903 | INFO | Uploaded 36GiB to GPU in 1.58624s (22.6951GiB/s) | 20260507 14:38:00.903 | INFO | Downloaded 34GiB to GPU in 1.5358s (22.1384GiB/s) | 20260507 14:38:01.000 | INFO | Starting a round of Baz IO | 20260507 14:38:03.895 | INFO | Populating RTMetrics report | 20260507 14:38:03.895 | INFO | RTMetrics report populated | 20260507 14:38:03.895 | INFO | Opening tmp RTMetrics json file | 20260507 14:38:03.902 | INFO | Serializing RTMetrics to json | 20260507 14:38:03.969 | INFO | Serialization done, swapping tmp file now... | 20260507 14:38:03.994 | INFO | RTMetrics done | 20260507 14:38:04.016 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.015 seconds | 20260507 14:38:04.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.59% / 0.08% / 42.33% | 20260507 14:38:04.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.61219% / 0.000254313% / 1.19209e-05% | 20260507 14:38:04.016 | INFO | Baz IO overall: | 20260507 14:38:04.016 | INFO | BytesWritten : 97954970735 | 20260507 14:38:04.016 | INFO | NumEvents : 46434347922 | 20260507 14:38:06.041 | INFO | Analyzing chunk frames = [274432,274944) | 20260507 14:38:06.041 | INFO | MemInfo: { | 20260507 14:38:06.041 | INFO | "gpu" : " 17 GB", | 20260507 14:38:06.041 | INFO | "huge" : "", | 20260507 14:38:06.041 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:06.042 | INFO | "malloc" : " 14 GB", | 20260507 14:38:06.042 | INFO | "pinned" : " 29 GB" | 20260507 14:38:06.042 | INFO | } | 20260507 14:38:06.042 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:38:06.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:06.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:06.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:06.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:06.042 | INFO | ANALYSIS : 57.3%, 2.9, 87.1ms, 0 | 20260507 14:38:06.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:06.042 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 14:38:06.042 | INFO | PRE_HQ : 56.2%, 2.1, 60.2ms, 0 | 20260507 14:38:06.042 | INFO | BAZWRITER : 58.2%, 4.4, 132.3ms, 0 | 20260507 14:38:06.042 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 14:38:06.043 | INFO | Uploaded 36GiB to GPU in 1.64613s (21.8695GiB/s) | 20260507 14:38:06.043 | INFO | Downloaded 38GiB to GPU in 1.72888s (21.9796GiB/s) | 20260507 14:38:11.181 | INFO | Analyzing chunk frames = [274944,275456) | 20260507 14:38:11.181 | INFO | MemInfo: { | 20260507 14:38:11.181 | INFO | "gpu" : " 17 GB", | 20260507 14:38:11.181 | INFO | "huge" : "", | 20260507 14:38:11.181 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:11.181 | INFO | "malloc" : " 14 GB", | 20260507 14:38:11.181 | INFO | "pinned" : " 29 GB" | 20260507 14:38:11.181 | INFO | } | 20260507 14:38:11.181 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:38:11.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:11.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.181 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:38:11.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.181 | INFO | PRE_HQ : 40.7%, 1.0, 21.3ms, 0 | 20260507 14:38:11.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:11.182 | INFO | Uploaded 36GiB to GPU in 1.59067s (22.6319GiB/s) | 20260507 14:38:11.182 | INFO | Downloaded 34GiB to GPU in 1.53664s (22.1261GiB/s) | 20260507 14:38:16.320 | INFO | Analyzing chunk frames = [275456,275968) | 20260507 14:38:16.320 | INFO | MemInfo: { | 20260507 14:38:16.320 | INFO | "gpu" : " 17 GB", | 20260507 14:38:16.321 | INFO | "huge" : "", | 20260507 14:38:16.321 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:16.321 | INFO | "malloc" : " 14 GB", | 20260507 14:38:16.321 | INFO | "pinned" : " 29 GB" | 20260507 14:38:16.321 | INFO | } | 20260507 14:38:16.321 | INFO | Perf report for the last 5.13958 seconds: | 20260507 14:38:16.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:16.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:38:16.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | PRE_HQ : 41.1%, 1.0, 21.4ms, 0 | 20260507 14:38:16.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:16.321 | INFO | Uploaded 36GiB to GPU in 1.59367s (22.5893GiB/s) | 20260507 14:38:16.321 | INFO | Downloaded 34GiB to GPU in 1.54191s (22.0505GiB/s) | 20260507 14:38:21.459 | INFO | Analyzing chunk frames = [275968,276480) | 20260507 14:38:21.459 | INFO | MemInfo: { | 20260507 14:38:21.459 | INFO | "gpu" : " 17 GB", | 20260507 14:38:21.460 | INFO | "huge" : "", | 20260507 14:38:21.460 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:21.460 | INFO | "malloc" : " 14 GB", | 20260507 14:38:21.460 | INFO | "pinned" : " 29 GB" | 20260507 14:38:21.460 | INFO | } | 20260507 14:38:21.460 | INFO | Perf report for the last 5.13907 seconds: | 20260507 14:38:21.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:21.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.460 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:38:21.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.460 | INFO | PRE_HQ : 41.5%, 1.0, 21.6ms, 0 | 20260507 14:38:21.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:21.461 | INFO | Uploaded 36GiB to GPU in 1.59455s (22.5768GiB/s) | 20260507 14:38:21.461 | INFO | Downloaded 34GiB to GPU in 1.54076s (22.0671GiB/s) | 20260507 14:38:26.599 | INFO | Analyzing chunk frames = [276480,276992) | 20260507 14:38:26.599 | INFO | MemInfo: { | 20260507 14:38:26.599 | INFO | "gpu" : " 17 GB", | 20260507 14:38:26.599 | INFO | "huge" : "", | 20260507 14:38:26.599 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:26.599 | INFO | "malloc" : " 14 GB", | 20260507 14:38:26.599 | INFO | "pinned" : " 29 GB" | 20260507 14:38:26.599 | INFO | } | 20260507 14:38:26.599 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:38:26.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:26.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:38:26.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 14:38:26.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:26.599 | INFO | Uploaded 36GiB to GPU in 1.59082s (22.6299GiB/s) | 20260507 14:38:26.600 | INFO | Downloaded 34GiB to GPU in 1.53861s (22.0978GiB/s) | 20260507 14:38:31.738 | INFO | Analyzing chunk frames = [276992,277504) | 20260507 14:38:31.738 | INFO | MemInfo: { | 20260507 14:38:31.738 | INFO | "gpu" : " 17 GB", | 20260507 14:38:31.738 | INFO | "huge" : "", | 20260507 14:38:31.738 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:31.738 | INFO | "malloc" : " 14 GB", | 20260507 14:38:31.738 | INFO | "pinned" : " 29 GB" | 20260507 14:38:31.738 | INFO | } | 20260507 14:38:31.738 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:38:31.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:31.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.738 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:38:31.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.738 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:38:31.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:31.739 | INFO | Uploaded 36GiB to GPU in 1.58567s (22.7033GiB/s) | 20260507 14:38:31.739 | INFO | Downloaded 34GiB to GPU in 1.53546s (22.1431GiB/s) | 20260507 14:38:36.877 | INFO | Analyzing chunk frames = [277504,278016) | 20260507 14:38:36.877 | INFO | MemInfo: { | 20260507 14:38:36.877 | INFO | "gpu" : " 17 GB", | 20260507 14:38:36.877 | INFO | "huge" : "", | 20260507 14:38:36.878 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:36.878 | INFO | "malloc" : " 14 GB", | 20260507 14:38:36.878 | INFO | "pinned" : " 29 GB" | 20260507 14:38:36.878 | INFO | } | 20260507 14:38:36.878 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:38:36.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:36.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:38:36.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:38:36.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:36.878 | INFO | Uploaded 36GiB to GPU in 1.5864s (22.6929GiB/s) | 20260507 14:38:36.878 | INFO | Downloaded 34GiB to GPU in 1.53536s (22.1447GiB/s) | 20260507 14:38:42.017 | INFO | Analyzing chunk frames = [278016,278528) | 20260507 14:38:42.017 | INFO | MemInfo: { | 20260507 14:38:42.017 | INFO | "gpu" : " 17 GB", | 20260507 14:38:42.017 | INFO | "huge" : "", | 20260507 14:38:42.017 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:42.017 | INFO | "malloc" : " 14 GB", | 20260507 14:38:42.017 | INFO | "pinned" : " 29 GB" | 20260507 14:38:42.017 | INFO | } | 20260507 14:38:42.017 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:38:42.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:42.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:38:42.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:38:42.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:42.017 | INFO | Uploaded 36GiB to GPU in 1.58674s (22.6881GiB/s) | 20260507 14:38:42.017 | INFO | Downloaded 34GiB to GPU in 1.53505s (22.1491GiB/s) | 20260507 14:38:42.017 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:38:42.017 | INFO | Total mem usage: 15.7 GiB | 20260507 14:38:42.017 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:38:42.017 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -2.12%) | 20260507 14:38:42.017 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:38:42.017 | INFO | High Water Breakdown: | 20260507 14:38:42.017 | INFO | 59.3% : 9.09 GiB ( 97.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:38:42.017 | INFO | 40.6% : 6.23 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:38:42.017 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:38:42.017 | INFO | ------------------------------------------ | 20260507 14:38:42.017 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:38:42.017 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:38:42.017 | INFO | Total mem usage: 26.5 GiB | 20260507 14:38:42.017 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:38:42.017 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:38:42.017 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:38:42.017 | INFO | High Water Breakdown: | 20260507 14:38:42.017 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:38:42.017 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:38:42.017 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:38:42.017 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:38:42.017 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:38:42.017 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:38:42.017 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:38:42.017 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:38:42.017 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:38:42.018 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:38:42.018 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:38:42.018 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:38:42.018 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:38:42.018 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:38:42.018 | INFO | ------------------------------------------ | 20260507 14:38:42.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:38:42.018 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:38:42.018 | INFO | Total mem usage: 27.8 GiB | 20260507 14:38:42.018 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:38:42.018 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:38:42.018 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:38:42.018 | INFO | High Water Breakdown: | 20260507 14:38:42.018 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:38:42.018 | INFO | ------------------------------------------ | 20260507 14:38:42.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:38:42.018 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:38:42.018 | INFO | Total mem usage: 15.7 GiB | 20260507 14:38:42.018 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:38:42.018 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:38:42.018 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:38:42.018 | INFO | High Water Breakdown: | 20260507 14:38:42.018 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:38:42.018 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:38:42.018 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:38:42.018 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:38:42.018 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:38:42.018 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:38:42.018 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:38:42.018 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:38:42.018 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:38:42.018 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:38:42.018 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:38:42.018 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:38:42.018 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:38:42.018 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:38:42.018 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:38:42.018 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:38:42.018 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:38:42.018 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:38:42.018 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:38:42.018 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:38:42.018 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:38:42.018 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:38:42.018 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:38:42.018 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:38:42.018 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:38:42.018 | INFO | ------------------------------------------ | 20260507 14:38:42.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:38:42.018 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:38:42.018 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:38:42.018 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:38:42.018 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 0.29%, AggregateStats -- 8.85e-02 / -nan, 5.78e-03 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 11.00%, Baseline -- 3.30e+00 / -nan, 2.22e-02 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.28e-02 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 16.09%, DME -- 3.86e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.19e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.74e-01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.05e-01 / -nan, 5.93e-02 / 0.00e+00 | 20260507 14:38:42.018 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:38:42.018 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:38:42.018 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:38:42.018 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 18.48%, Upload -- 1.66e+01 / 0.00e+00, 4.62e-01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 18.08%, Download -- 1.63e+01 / 0.00e+00, 7.55e-01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 33.38%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 0.00%, LockDownload -- 4.79e-04 / 0.00e+00, 2.88e-04 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 29.47%, LockCompute -- 2.65e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:38:42.018 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 14:38:42.114 | INFO | Starting a round of Baz IO | 20260507 14:38:45.008 | INFO | Populating RTMetrics report | 20260507 14:38:45.008 | INFO | RTMetrics report populated | 20260507 14:38:45.008 | INFO | Opening tmp RTMetrics json file | 20260507 14:38:45.050 | INFO | Serializing RTMetrics to json | 20260507 14:38:45.120 | INFO | Serialization done, swapping tmp file now... | 20260507 14:38:45.158 | INFO | RTMetrics done | 20260507 14:38:45.282 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.167 seconds | 20260507 14:38:45.282 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.51% / 0.08% / 42.41% | 20260507 14:38:45.282 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.56606% / 0.000246366% / 1.98682e-05% | 20260507 14:38:45.282 | INFO | Baz IO overall: | 20260507 14:38:45.282 | INFO | BytesWritten : 99765470206 | 20260507 14:38:45.282 | INFO | NumEvents : 47317422198 | 20260507 14:38:47.156 | INFO | Analyzing chunk frames = [278528,279040) | 20260507 14:38:47.156 | INFO | MemInfo: { | 20260507 14:38:47.156 | INFO | "gpu" : " 17 GB", | 20260507 14:38:47.156 | INFO | "huge" : "", | 20260507 14:38:47.156 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:47.156 | INFO | "malloc" : " 14 GB", | 20260507 14:38:47.156 | INFO | "pinned" : " 29 GB" | 20260507 14:38:47.156 | INFO | } | 20260507 14:38:47.157 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:38:47.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:47.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:47.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:47.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:47.157 | INFO | ANALYSIS : 57.3%, 2.9, 87.4ms, 0 | 20260507 14:38:47.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:47.157 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 14:38:47.157 | INFO | PRE_HQ : 56.2%, 2.1, 60.4ms, 0 | 20260507 14:38:47.157 | INFO | BAZWRITER : 61.2%, 4.5, 140.0ms, 0 | 20260507 14:38:47.157 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:38:47.157 | INFO | Uploaded 36GiB to GPU in 1.64483s (21.8868GiB/s) | 20260507 14:38:47.157 | INFO | Downloaded 38GiB to GPU in 1.72869s (21.9819GiB/s) | 20260507 14:38:52.296 | INFO | Analyzing chunk frames = [279040,279552) | 20260507 14:38:52.296 | INFO | MemInfo: { | 20260507 14:38:52.296 | INFO | "gpu" : " 17 GB", | 20260507 14:38:52.296 | INFO | "huge" : "", | 20260507 14:38:52.296 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:52.296 | INFO | "malloc" : " 14 GB", | 20260507 14:38:52.296 | INFO | "pinned" : " 29 GB" | 20260507 14:38:52.296 | INFO | } | 20260507 14:38:52.296 | INFO | Perf report for the last 5.13953 seconds: | 20260507 14:38:52.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:52.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.296 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:38:52.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.296 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 14:38:52.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:52.297 | INFO | Uploaded 36GiB to GPU in 1.61018s (22.3577GiB/s) | 20260507 14:38:52.297 | INFO | Downloaded 34GiB to GPU in 1.52771s (22.2555GiB/s) | 20260507 14:38:57.435 | INFO | Analyzing chunk frames = [279552,280064) | 20260507 14:38:57.435 | INFO | MemInfo: { | 20260507 14:38:57.435 | INFO | "gpu" : " 17 GB", | 20260507 14:38:57.435 | INFO | "huge" : "", | 20260507 14:38:57.435 | INFO | "huge_shared" : " 29 GB", | 20260507 14:38:57.435 | INFO | "malloc" : " 14 GB", | 20260507 14:38:57.435 | INFO | "pinned" : " 29 GB" | 20260507 14:38:57.435 | INFO | } | 20260507 14:38:57.435 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:38:57.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:38:57.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:38:57.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | PRE_HQ : 39.9%, 1.0, 20.7ms, 0 | 20260507 14:38:57.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:38:57.436 | INFO | Uploaded 36GiB to GPU in 1.61329s (22.3146GiB/s) | 20260507 14:38:57.436 | INFO | Downloaded 34GiB to GPU in 1.53197s (22.1937GiB/s) | 20260507 14:39:02.574 | INFO | Analyzing chunk frames = [280064,280576) | 20260507 14:39:02.575 | INFO | MemInfo: { | 20260507 14:39:02.575 | INFO | "gpu" : " 17 GB", | 20260507 14:39:02.575 | INFO | "huge" : "", | 20260507 14:39:02.575 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:02.575 | INFO | "malloc" : " 14 GB", | 20260507 14:39:02.575 | INFO | "pinned" : " 29 GB" | 20260507 14:39:02.575 | INFO | } | 20260507 14:39:02.575 | INFO | Perf report for the last 5.13948 seconds: | 20260507 14:39:02.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:02.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:39:02.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | PRE_HQ : 41.5%, 1.0, 21.5ms, 0 | 20260507 14:39:02.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:02.575 | INFO | Uploaded 36GiB to GPU in 1.61373s (22.3085GiB/s) | 20260507 14:39:02.575 | INFO | Downloaded 34GiB to GPU in 1.53269s (22.1832GiB/s) | 20260507 14:39:07.713 | INFO | Analyzing chunk frames = [280576,281088) | 20260507 14:39:07.713 | INFO | MemInfo: { | 20260507 14:39:07.713 | INFO | "gpu" : " 17 GB", | 20260507 14:39:07.713 | INFO | "huge" : "", | 20260507 14:39:07.713 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:07.713 | INFO | "malloc" : " 14 GB", | 20260507 14:39:07.714 | INFO | "pinned" : " 29 GB" | 20260507 14:39:07.714 | INFO | } | 20260507 14:39:07.714 | INFO | Perf report for the last 5.13866 seconds: | 20260507 14:39:07.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:07.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:39:07.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | PRE_HQ : 41.1%, 1.0, 21.3ms, 0 | 20260507 14:39:07.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:07.714 | INFO | Uploaded 36GiB to GPU in 1.61155s (22.3388GiB/s) | 20260507 14:39:07.714 | INFO | Downloaded 34GiB to GPU in 1.52998s (22.2225GiB/s) | 20260507 14:39:12.853 | INFO | Analyzing chunk frames = [281088,281600) | 20260507 14:39:12.853 | INFO | MemInfo: { | 20260507 14:39:12.853 | INFO | "gpu" : " 17 GB", | 20260507 14:39:12.853 | INFO | "huge" : "", | 20260507 14:39:12.853 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:12.853 | INFO | "malloc" : " 14 GB", | 20260507 14:39:12.853 | INFO | "pinned" : " 29 GB" | 20260507 14:39:12.853 | INFO | } | 20260507 14:39:12.853 | INFO | Perf report for the last 5.13959 seconds: | 20260507 14:39:12.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:12.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.853 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:39:12.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.853 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:39:12.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:12.854 | INFO | Uploaded 36GiB to GPU in 1.60553s (22.4226GiB/s) | 20260507 14:39:12.854 | INFO | Downloaded 34GiB to GPU in 1.52547s (22.2883GiB/s) | 20260507 14:39:17.992 | INFO | Analyzing chunk frames = [281600,282112) | 20260507 14:39:17.992 | INFO | MemInfo: { | 20260507 14:39:17.992 | INFO | "gpu" : " 17 GB", | 20260507 14:39:17.992 | INFO | "huge" : "", | 20260507 14:39:17.992 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:17.992 | INFO | "malloc" : " 14 GB", | 20260507 14:39:17.992 | INFO | "pinned" : " 29 GB" | 20260507 14:39:17.992 | INFO | } | 20260507 14:39:17.992 | INFO | Perf report for the last 5.1391 seconds: | 20260507 14:39:17.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:17.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.992 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:39:17.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.992 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 14:39:17.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:17.993 | INFO | Uploaded 36GiB to GPU in 1.606s (22.416GiB/s) | 20260507 14:39:17.993 | INFO | Downloaded 34GiB to GPU in 1.52727s (22.2619GiB/s) | 20260507 14:39:23.132 | INFO | Analyzing chunk frames = [282112,282624) | 20260507 14:39:23.132 | INFO | MemInfo: { | 20260507 14:39:23.132 | INFO | "gpu" : " 17 GB", | 20260507 14:39:23.132 | INFO | "huge" : "", | 20260507 14:39:23.132 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:23.132 | INFO | "malloc" : " 14 GB", | 20260507 14:39:23.132 | INFO | "pinned" : " 29 GB" | 20260507 14:39:23.132 | INFO | } | 20260507 14:39:23.132 | INFO | Perf report for the last 5.13967 seconds: | 20260507 14:39:23.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:23.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:39:23.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:39:23.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:23.132 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.4238GiB/s) | 20260507 14:39:23.132 | INFO | Downloaded 34GiB to GPU in 1.52648s (22.2735GiB/s) | 20260507 14:39:23.230 | INFO | Starting a round of Baz IO | 20260507 14:39:26.122 | INFO | Populating RTMetrics report | 20260507 14:39:26.122 | INFO | RTMetrics report populated | 20260507 14:39:26.122 | INFO | Opening tmp RTMetrics json file | 20260507 14:39:26.181 | INFO | Serializing RTMetrics to json | 20260507 14:39:26.210 | INFO | Serialization done, swapping tmp file now... | 20260507 14:39:26.213 | INFO | RTMetrics done | 20260507 14:39:26.324 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.093 seconds | 20260507 14:39:26.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.42% / 0.08% / 42.49% | 20260507 14:39:26.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.54391% / 0.000234445% / 2.78155e-05% | 20260507 14:39:26.324 | INFO | Baz IO overall: | 20260507 14:39:26.324 | INFO | BytesWritten : 101577464061 | 20260507 14:39:26.324 | INFO | NumEvents : 48201626553 | 20260507 14:39:28.271 | INFO | Analyzing chunk frames = [282624,283136) | 20260507 14:39:28.271 | INFO | MemInfo: { | 20260507 14:39:28.271 | INFO | "gpu" : " 17 GB", | 20260507 14:39:28.271 | INFO | "huge" : "", | 20260507 14:39:28.271 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:28.271 | INFO | "malloc" : " 14 GB", | 20260507 14:39:28.271 | INFO | "pinned" : " 29 GB" | 20260507 14:39:28.271 | INFO | } | 20260507 14:39:28.271 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:39:28.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:28.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:28.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:28.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:28.271 | INFO | ANALYSIS : 57.3%, 2.9, 88.0ms, 0 | 20260507 14:39:28.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:28.272 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 14:39:28.272 | INFO | PRE_HQ : 56.0%, 2.1, 59.6ms, 0 | 20260507 14:39:28.272 | INFO | BAZWRITER : 59.8%, 4.4, 134.8ms, 0 | 20260507 14:39:28.272 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 14:39:28.272 | INFO | Uploaded 36GiB to GPU in 1.6387s (21.9687GiB/s) | 20260507 14:39:28.272 | INFO | Downloaded 38GiB to GPU in 1.71808s (22.1177GiB/s) | 20260507 14:39:33.410 | INFO | Analyzing chunk frames = [283136,283648) | 20260507 14:39:33.410 | INFO | MemInfo: { | 20260507 14:39:33.410 | INFO | "gpu" : " 17 GB", | 20260507 14:39:33.410 | INFO | "huge" : "", | 20260507 14:39:33.411 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:33.411 | INFO | "malloc" : " 14 GB", | 20260507 14:39:33.411 | INFO | "pinned" : " 29 GB" | 20260507 14:39:33.411 | INFO | } | 20260507 14:39:33.411 | INFO | Perf report for the last 5.13957 seconds: | 20260507 14:39:33.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:33.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.411 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:39:33.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.411 | INFO | PRE_HQ : 41.0%, 1.0, 21.7ms, 0 | 20260507 14:39:33.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:33.412 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4256GiB/s) | 20260507 14:39:33.412 | INFO | Downloaded 34GiB to GPU in 1.53219s (22.1905GiB/s) | 20260507 14:39:38.550 | INFO | Analyzing chunk frames = [283648,284160) | 20260507 14:39:38.550 | INFO | MemInfo: { | 20260507 14:39:38.550 | INFO | "gpu" : " 17 GB", | 20260507 14:39:38.550 | INFO | "huge" : "", | 20260507 14:39:38.550 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:38.550 | INFO | "malloc" : " 14 GB", | 20260507 14:39:38.550 | INFO | "pinned" : " 29 GB" | 20260507 14:39:38.550 | INFO | } | 20260507 14:39:38.550 | INFO | Perf report for the last 5.13913 seconds: | 20260507 14:39:38.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:38.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.550 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:39:38.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.550 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 14:39:38.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:38.551 | INFO | Uploaded 36GiB to GPU in 1.61433s (22.3003GiB/s) | 20260507 14:39:38.551 | INFO | Downloaded 34GiB to GPU in 1.53351s (22.1714GiB/s) | 20260507 14:39:43.689 | INFO | Analyzing chunk frames = [284160,284672) | 20260507 14:39:43.689 | INFO | MemInfo: { | 20260507 14:39:43.689 | INFO | "gpu" : " 17 GB", | 20260507 14:39:43.689 | INFO | "huge" : "", | 20260507 14:39:43.689 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:43.689 | INFO | "malloc" : " 14 GB", | 20260507 14:39:43.689 | INFO | "pinned" : " 29 GB" | 20260507 14:39:43.689 | INFO | } | 20260507 14:39:43.689 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:39:43.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:43.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:39:43.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 14:39:43.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:43.690 | INFO | Uploaded 36GiB to GPU in 1.61339s (22.3132GiB/s) | 20260507 14:39:43.690 | INFO | Downloaded 34GiB to GPU in 1.53176s (22.1966GiB/s) | 20260507 14:39:48.828 | INFO | Analyzing chunk frames = [284672,285184) | 20260507 14:39:48.828 | INFO | MemInfo: { | 20260507 14:39:48.828 | INFO | "gpu" : " 17 GB", | 20260507 14:39:48.828 | INFO | "huge" : "", | 20260507 14:39:48.828 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:48.828 | INFO | "malloc" : " 14 GB", | 20260507 14:39:48.828 | INFO | "pinned" : " 29 GB" | 20260507 14:39:48.828 | INFO | } | 20260507 14:39:48.828 | INFO | Perf report for the last 5.13904 seconds: | 20260507 14:39:48.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:48.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.829 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:39:48.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.829 | INFO | PRE_HQ : 41.8%, 1.0, 21.8ms, 0 | 20260507 14:39:48.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:48.829 | INFO | Uploaded 36GiB to GPU in 1.61021s (22.3573GiB/s) | 20260507 14:39:48.829 | INFO | Downloaded 34GiB to GPU in 1.53048s (22.2152GiB/s) | 20260507 14:39:53.968 | INFO | Analyzing chunk frames = [285184,285696) | 20260507 14:39:53.968 | INFO | MemInfo: { | 20260507 14:39:53.968 | INFO | "gpu" : " 17 GB", | 20260507 14:39:53.968 | INFO | "huge" : "", | 20260507 14:39:53.968 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:53.968 | INFO | "malloc" : " 14 GB", | 20260507 14:39:53.968 | INFO | "pinned" : " 29 GB" | 20260507 14:39:53.968 | INFO | } | 20260507 14:39:53.968 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:39:53.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:53.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:39:53.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:39:53.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:53.968 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4346GiB/s) | 20260507 14:39:53.968 | INFO | Downloaded 34GiB to GPU in 1.52504s (22.2945GiB/s) | 20260507 14:39:59.107 | INFO | Analyzing chunk frames = [285696,286208) | 20260507 14:39:59.107 | INFO | MemInfo: { | 20260507 14:39:59.107 | INFO | "gpu" : " 17 GB", | 20260507 14:39:59.107 | INFO | "huge" : "", | 20260507 14:39:59.107 | INFO | "huge_shared" : " 29 GB", | 20260507 14:39:59.107 | INFO | "malloc" : " 14 GB", | 20260507 14:39:59.107 | INFO | "pinned" : " 29 GB" | 20260507 14:39:59.107 | INFO | } | 20260507 14:39:59.107 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:39:59.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:39:59.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.107 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:39:59.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.107 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:39:59.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:39:59.108 | INFO | Uploaded 36GiB to GPU in 1.6063s (22.4118GiB/s) | 20260507 14:39:59.108 | INFO | Downloaded 34GiB to GPU in 1.52678s (22.2691GiB/s) | 20260507 14:40:04.246 | INFO | Analyzing chunk frames = [286208,286720) | 20260507 14:40:04.246 | INFO | MemInfo: { | 20260507 14:40:04.246 | INFO | "gpu" : " 17 GB", | 20260507 14:40:04.246 | INFO | "huge" : "", | 20260507 14:40:04.246 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:04.246 | INFO | "malloc" : " 14 GB", | 20260507 14:40:04.246 | INFO | "pinned" : " 29 GB" | 20260507 14:40:04.246 | INFO | } | 20260507 14:40:04.246 | INFO | Perf report for the last 5.13909 seconds: | 20260507 14:40:04.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:04.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.246 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:40:04.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.247 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:40:04.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:04.247 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:40:04.247 | INFO | Total mem usage: 15.7 GiB | 20260507 14:40:04.247 | INFO | Uploaded 36GiB to GPU in 1.60598s (22.4163GiB/s) | 20260507 14:40:04.247 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:40:04.247 | INFO | Downloaded 34GiB to GPU in 1.52723s (22.2626GiB/s) | 20260507 14:40:04.247 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -2.78%) | 20260507 14:40:04.247 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:40:04.247 | INFO | High Water Breakdown: | 20260507 14:40:04.247 | INFO | 59.1% : 9 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:40:04.247 | INFO | 40.8% : 6.21 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:40:04.247 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:40:04.247 | INFO | ------------------------------------------ | 20260507 14:40:04.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:40:04.247 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:40:04.247 | INFO | Total mem usage: 26.5 GiB | 20260507 14:40:04.247 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:40:04.247 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:40:04.247 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:40:04.247 | INFO | High Water Breakdown: | 20260507 14:40:04.247 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:40:04.247 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:40:04.247 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:40:04.247 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:40:04.247 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:40:04.247 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:40:04.247 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:40:04.247 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:40:04.247 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:40:04.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:40:04.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:40:04.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:40:04.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:40:04.247 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:40:04.248 | INFO | ------------------------------------------ | 20260507 14:40:04.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:40:04.248 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:40:04.248 | INFO | Total mem usage: 27.8 GiB | 20260507 14:40:04.248 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:40:04.248 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:40:04.248 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:40:04.248 | INFO | High Water Breakdown: | 20260507 14:40:04.248 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:40:04.248 | INFO | ------------------------------------------ | 20260507 14:40:04.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:40:04.248 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:40:04.248 | INFO | Total mem usage: 15.7 GiB | 20260507 14:40:04.248 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:40:04.248 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:40:04.248 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:40:04.248 | INFO | High Water Breakdown: | 20260507 14:40:04.248 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:40:04.248 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:40:04.248 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:40:04.248 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:40:04.248 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:40:04.248 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:40:04.248 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:40:04.248 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:40:04.248 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:40:04.248 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:40:04.248 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:40:04.248 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:40:04.248 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:40:04.248 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:40:04.248 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:40:04.248 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:40:04.248 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:40:04.248 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:40:04.248 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:40:04.248 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:40:04.248 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:40:04.248 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:40:04.248 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:40:04.248 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:40:04.248 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:40:04.248 | INFO | ------------------------------------------ | 20260507 14:40:04.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:40:04.248 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:40:04.248 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:40:04.248 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:40:04.248 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 0.30%, AggregateStats -- 9.01e-02 / -nan, 8.54e-03 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 11.01%, Baseline -- 3.31e+00 / -nan, 2.36e-02 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 3.07%, Histogram -- 9.22e-01 / -nan, 5.44e-02 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 16.08%, DME -- 3.86e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / -nan, 1.08e-01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.20e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.08e-01 / -nan, 5.89e-02 / 0.00e+00 | 20260507 14:40:04.248 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:40:04.248 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:40:04.248 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:40:04.248 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 18.67%, Upload -- 1.68e+01 / 0.00e+00, 4.15e-01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 7.62e-01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 33.37%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 0.57%, LockUpload -- 5.11e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 0.00%, LockDownload -- 5.77e-04 / 0.00e+00, 4.24e-04 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 29.40%, LockCompute -- 2.65e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:40:04.248 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 14:40:04.343 | INFO | Starting a round of Baz IO | 20260507 14:40:07.342 | INFO | Populating RTMetrics report | 20260507 14:40:07.342 | INFO | RTMetrics report populated | 20260507 14:40:07.342 | INFO | Opening tmp RTMetrics json file | 20260507 14:40:07.430 | INFO | Serializing RTMetrics to json | 20260507 14:40:07.485 | INFO | Serialization done, swapping tmp file now... | 20260507 14:40:07.487 | INFO | RTMetrics done | 20260507 14:40:07.638 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.294 seconds | 20260507 14:40:07.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.34% / 0.08% / 42.58% | 20260507 14:40:07.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.51097% / 0.000230471% / 2.78155e-05% | 20260507 14:40:07.638 | INFO | Baz IO overall: | 20260507 14:40:07.638 | INFO | BytesWritten : 103390891369 | 20260507 14:40:07.638 | INFO | NumEvents : 49086553677 | 20260507 14:40:09.386 | INFO | Analyzing chunk frames = [286720,287232) | 20260507 14:40:09.386 | INFO | MemInfo: { | 20260507 14:40:09.386 | INFO | "gpu" : " 17 GB", | 20260507 14:40:09.386 | INFO | "huge" : "", | 20260507 14:40:09.386 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:09.386 | INFO | "malloc" : " 14 GB", | 20260507 14:40:09.386 | INFO | "pinned" : " 29 GB" | 20260507 14:40:09.386 | INFO | } | 20260507 14:40:09.386 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:40:09.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:09.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:09.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:09.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:09.386 | INFO | ANALYSIS : 58.4%, 2.8, 85.3ms, 0 | 20260507 14:40:09.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:09.386 | INFO | RT_METRICS : 49.7%, 1.0, 26.1ms, 0 | 20260507 14:40:09.386 | INFO | PRE_HQ : 57.9%, 2.2, 66.2ms, 0 | 20260507 14:40:09.386 | INFO | BAZWRITER : 63.5%, 4.6, 149.2ms, 0 | 20260507 14:40:09.386 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 14:40:09.387 | INFO | Uploaded 36GiB to GPU in 1.62916s (22.0973GiB/s) | 20260507 14:40:09.387 | INFO | Downloaded 38GiB to GPU in 1.70873s (22.2388GiB/s) | 20260507 14:40:14.525 | INFO | Analyzing chunk frames = [287232,287744) | 20260507 14:40:14.525 | INFO | MemInfo: { | 20260507 14:40:14.525 | INFO | "gpu" : " 17 GB", | 20260507 14:40:14.525 | INFO | "huge" : "", | 20260507 14:40:14.525 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:14.525 | INFO | "malloc" : " 14 GB", | 20260507 14:40:14.525 | INFO | "pinned" : " 29 GB" | 20260507 14:40:14.525 | INFO | } | 20260507 14:40:14.525 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:40:14.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:14.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.525 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:40:14.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.525 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 14:40:14.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:14.526 | INFO | Uploaded 36GiB to GPU in 1.61054s (22.3527GiB/s) | 20260507 14:40:14.526 | INFO | Downloaded 34GiB to GPU in 1.52877s (22.2401GiB/s) | 20260507 14:40:19.665 | INFO | Analyzing chunk frames = [287744,288256) | 20260507 14:40:19.665 | INFO | MemInfo: { | 20260507 14:40:19.665 | INFO | "gpu" : " 17 GB", | 20260507 14:40:19.665 | INFO | "huge" : "", | 20260507 14:40:19.665 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:19.665 | INFO | "malloc" : " 14 GB", | 20260507 14:40:19.665 | INFO | "pinned" : " 29 GB" | 20260507 14:40:19.665 | INFO | } | 20260507 14:40:19.665 | INFO | Perf report for the last 5.13967 seconds: | 20260507 14:40:19.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:19.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.665 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:40:19.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.665 | INFO | PRE_HQ : 41.2%, 1.0, 21.5ms, 0 | 20260507 14:40:19.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:19.666 | INFO | Uploaded 36GiB to GPU in 1.6142s (22.3021GiB/s) | 20260507 14:40:19.666 | INFO | Downloaded 34GiB to GPU in 1.53334s (22.1738GiB/s) | 20260507 14:40:24.804 | INFO | Analyzing chunk frames = [288256,288768) | 20260507 14:40:24.804 | INFO | MemInfo: { | 20260507 14:40:24.804 | INFO | "gpu" : " 17 GB", | 20260507 14:40:24.804 | INFO | "huge" : "", | 20260507 14:40:24.804 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:24.804 | INFO | "malloc" : " 14 GB", | 20260507 14:40:24.804 | INFO | "pinned" : " 29 GB" | 20260507 14:40:24.804 | INFO | } | 20260507 14:40:24.804 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:40:24.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:24.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.804 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:40:24.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.804 | INFO | PRE_HQ : 41.1%, 1.0, 21.3ms, 0 | 20260507 14:40:24.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:24.805 | INFO | Uploaded 36GiB to GPU in 1.61432s (22.3005GiB/s) | 20260507 14:40:24.805 | INFO | Downloaded 34GiB to GPU in 1.53173s (22.1971GiB/s) | 20260507 14:40:29.943 | INFO | Analyzing chunk frames = [288768,289280) | 20260507 14:40:29.943 | INFO | MemInfo: { | 20260507 14:40:29.943 | INFO | "gpu" : " 17 GB", | 20260507 14:40:29.943 | INFO | "huge" : "", | 20260507 14:40:29.943 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:29.943 | INFO | "malloc" : " 14 GB", | 20260507 14:40:29.943 | INFO | "pinned" : " 29 GB" | 20260507 14:40:29.943 | INFO | } | 20260507 14:40:29.943 | INFO | Perf report for the last 5.13911 seconds: | 20260507 14:40:29.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:29.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.943 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:40:29.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.944 | INFO | PRE_HQ : 41.0%, 1.0, 21.3ms, 0 | 20260507 14:40:29.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:29.944 | INFO | Uploaded 36GiB to GPU in 1.61232s (22.3281GiB/s) | 20260507 14:40:29.944 | INFO | Downloaded 34GiB to GPU in 1.53103s (22.2073GiB/s) | 20260507 14:40:35.082 | INFO | Analyzing chunk frames = [289280,289792) | 20260507 14:40:35.082 | INFO | MemInfo: { | 20260507 14:40:35.082 | INFO | "gpu" : " 17 GB", | 20260507 14:40:35.082 | INFO | "huge" : "", | 20260507 14:40:35.082 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:35.082 | INFO | "malloc" : " 14 GB", | 20260507 14:40:35.082 | INFO | "pinned" : " 29 GB" | 20260507 14:40:35.082 | INFO | } | 20260507 14:40:35.082 | INFO | Perf report for the last 5.13901 seconds: | 20260507 14:40:35.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:35.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.082 | INFO | ANALYSIS : 56.5%, 3.0, 88.0ms, 0 | 20260507 14:40:35.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.083 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:40:35.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:35.083 | INFO | Uploaded 36GiB to GPU in 1.60534s (22.4251GiB/s) | 20260507 14:40:35.083 | INFO | Downloaded 34GiB to GPU in 1.52574s (22.2843GiB/s) | 20260507 14:40:40.222 | INFO | Analyzing chunk frames = [289792,290304) | 20260507 14:40:40.222 | INFO | MemInfo: { | 20260507 14:40:40.222 | INFO | "gpu" : " 17 GB", | 20260507 14:40:40.222 | INFO | "huge" : "", | 20260507 14:40:40.222 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:40.222 | INFO | "malloc" : " 14 GB", | 20260507 14:40:40.222 | INFO | "pinned" : " 29 GB" | 20260507 14:40:40.222 | INFO | } | 20260507 14:40:40.222 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:40:40.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:40.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:40:40.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:40:40.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:40.222 | INFO | Uploaded 36GiB to GPU in 1.60677s (22.4051GiB/s) | 20260507 14:40:40.222 | INFO | Downloaded 34GiB to GPU in 1.52711s (22.2643GiB/s) | 20260507 14:40:45.361 | INFO | Analyzing chunk frames = [290304,290816) | 20260507 14:40:45.361 | INFO | MemInfo: { | 20260507 14:40:45.361 | INFO | "gpu" : " 17 GB", | 20260507 14:40:45.361 | INFO | "huge" : "", | 20260507 14:40:45.361 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:45.361 | INFO | "malloc" : " 14 GB", | 20260507 14:40:45.361 | INFO | "pinned" : " 29 GB" | 20260507 14:40:45.361 | INFO | } | 20260507 14:40:45.361 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:40:45.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:45.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.361 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:40:45.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.361 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:40:45.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:45.362 | INFO | Uploaded 36GiB to GPU in 1.6067s (22.4062GiB/s) | 20260507 14:40:45.362 | INFO | Downloaded 34GiB to GPU in 1.52633s (22.2756GiB/s) | 20260507 14:40:45.459 | INFO | Starting a round of Baz IO | 20260507 14:40:48.419 | INFO | Populating RTMetrics report | 20260507 14:40:48.419 | INFO | RTMetrics report populated | 20260507 14:40:48.419 | INFO | Opening tmp RTMetrics json file | 20260507 14:40:48.436 | INFO | Serializing RTMetrics to json | 20260507 14:40:48.490 | INFO | Serialization done, swapping tmp file now... | 20260507 14:40:48.523 | INFO | RTMetrics done | 20260507 14:40:48.675 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.216 seconds | 20260507 14:40:48.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.26% / 0.08% / 42.66% | 20260507 14:40:48.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.45441% / 0.000222524% / 1.19209e-05% | 20260507 14:40:48.676 | INFO | Baz IO overall: | 20260507 14:40:48.676 | INFO | BytesWritten : 105204509637 | 20260507 14:40:48.676 | INFO | NumEvents : 49970558671 | 20260507 14:40:50.500 | INFO | Analyzing chunk frames = [290816,291328) | 20260507 14:40:50.500 | INFO | MemInfo: { | 20260507 14:40:50.500 | INFO | "gpu" : " 17 GB", | 20260507 14:40:50.500 | INFO | "huge" : "", | 20260507 14:40:50.500 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:50.500 | INFO | "malloc" : " 14 GB", | 20260507 14:40:50.500 | INFO | "pinned" : " 29 GB" | 20260507 14:40:50.501 | INFO | } | 20260507 14:40:50.501 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:40:50.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:50.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:50.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:50.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:50.501 | INFO | ANALYSIS : 58.3%, 2.7, 84.1ms, 0 | 20260507 14:40:50.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:50.501 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260507 14:40:50.501 | INFO | PRE_HQ : 57.4%, 2.2, 65.0ms, 0 | 20260507 14:40:50.501 | INFO | BAZWRITER : 62.1%, 4.7, 151.2ms, 0 | 20260507 14:40:50.501 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:40:50.501 | INFO | Uploaded 36GiB to GPU in 1.64665s (21.8626GiB/s) | 20260507 14:40:50.501 | INFO | Downloaded 38GiB to GPU in 1.72191s (22.0685GiB/s) | 20260507 14:40:55.640 | INFO | Analyzing chunk frames = [291328,291840) | 20260507 14:40:55.640 | INFO | MemInfo: { | 20260507 14:40:55.640 | INFO | "gpu" : " 17 GB", | 20260507 14:40:55.640 | INFO | "huge" : "", | 20260507 14:40:55.640 | INFO | "huge_shared" : " 29 GB", | 20260507 14:40:55.640 | INFO | "malloc" : " 14 GB", | 20260507 14:40:55.640 | INFO | "pinned" : " 29 GB" | 20260507 14:40:55.640 | INFO | } | 20260507 14:40:55.640 | INFO | Perf report for the last 5.1396 seconds: | 20260507 14:40:55.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:40:55.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.640 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260507 14:40:55.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.641 | INFO | PRE_HQ : 41.9%, 1.0, 22.7ms, 0 | 20260507 14:40:55.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:40:55.641 | INFO | Uploaded 36GiB to GPU in 1.60964s (22.3653GiB/s) | 20260507 14:40:55.641 | INFO | Downloaded 34GiB to GPU in 1.52706s (22.2651GiB/s) | 20260507 14:41:00.779 | INFO | Analyzing chunk frames = [291840,292352) | 20260507 14:41:00.779 | INFO | MemInfo: { | 20260507 14:41:00.779 | INFO | "gpu" : " 17 GB", | 20260507 14:41:00.779 | INFO | "huge" : "", | 20260507 14:41:00.779 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:00.779 | INFO | "malloc" : " 14 GB", | 20260507 14:41:00.780 | INFO | "pinned" : " 29 GB" | 20260507 14:41:00.780 | INFO | } | 20260507 14:41:00.780 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:41:00.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:00.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260507 14:41:00.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 14:41:00.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:00.780 | INFO | Uploaded 36GiB to GPU in 1.61368s (22.3093GiB/s) | 20260507 14:41:00.780 | INFO | Downloaded 34GiB to GPU in 1.53244s (22.1868GiB/s) | 20260507 14:41:05.919 | INFO | Analyzing chunk frames = [292352,292864) | 20260507 14:41:05.919 | INFO | MemInfo: { | 20260507 14:41:05.919 | INFO | "gpu" : " 17 GB", | 20260507 14:41:05.919 | INFO | "huge" : "", | 20260507 14:41:05.919 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:05.919 | INFO | "malloc" : " 14 GB", | 20260507 14:41:05.919 | INFO | "pinned" : " 29 GB" | 20260507 14:41:05.919 | INFO | } | 20260507 14:41:05.919 | INFO | Perf report for the last 5.13959 seconds: | 20260507 14:41:05.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:05.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.919 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:41:05.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.920 | INFO | PRE_HQ : 41.9%, 1.0, 21.8ms, 0 | 20260507 14:41:05.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:05.920 | INFO | Uploaded 36GiB to GPU in 1.61433s (22.3003GiB/s) | 20260507 14:41:05.920 | INFO | Downloaded 34GiB to GPU in 1.53293s (22.1798GiB/s) | 20260507 14:41:11.058 | INFO | Analyzing chunk frames = [292864,293376) | 20260507 14:41:11.058 | INFO | MemInfo: { | 20260507 14:41:11.058 | INFO | "gpu" : " 17 GB", | 20260507 14:41:11.058 | INFO | "huge" : "", | 20260507 14:41:11.058 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:11.058 | INFO | "malloc" : " 14 GB", | 20260507 14:41:11.058 | INFO | "pinned" : " 29 GB" | 20260507 14:41:11.058 | INFO | } | 20260507 14:41:11.058 | INFO | Perf report for the last 5.13882 seconds: | 20260507 14:41:11.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:11.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:41:11.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 14:41:11.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:11.058 | INFO | Uploaded 36GiB to GPU in 1.60974s (22.3639GiB/s) | 20260507 14:41:11.058 | INFO | Downloaded 34GiB to GPU in 1.52878s (22.2399GiB/s) | 20260507 14:41:16.197 | INFO | Analyzing chunk frames = [293376,293888) | 20260507 14:41:16.197 | INFO | MemInfo: { | 20260507 14:41:16.197 | INFO | "gpu" : " 17 GB", | 20260507 14:41:16.197 | INFO | "huge" : "", | 20260507 14:41:16.197 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:16.197 | INFO | "malloc" : " 14 GB", | 20260507 14:41:16.197 | INFO | "pinned" : " 29 GB" | 20260507 14:41:16.198 | INFO | } | 20260507 14:41:16.198 | INFO | Perf report for the last 5.13951 seconds: | 20260507 14:41:16.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:16.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:41:16.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:41:16.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:16.198 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4177GiB/s) | 20260507 14:41:16.198 | INFO | Downloaded 34GiB to GPU in 1.5263s (22.2762GiB/s) | 20260507 14:41:21.336 | INFO | Analyzing chunk frames = [293888,294400) | 20260507 14:41:21.336 | INFO | MemInfo: { | 20260507 14:41:21.336 | INFO | "gpu" : " 17 GB", | 20260507 14:41:21.336 | INFO | "huge" : "", | 20260507 14:41:21.336 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:21.336 | INFO | "malloc" : " 14 GB", | 20260507 14:41:21.337 | INFO | "pinned" : " 29 GB" | 20260507 14:41:21.337 | INFO | } | 20260507 14:41:21.337 | INFO | Perf report for the last 5.13897 seconds: | 20260507 14:41:21.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:21.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:41:21.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:41:21.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:21.337 | INFO | Uploaded 36GiB to GPU in 1.60476s (22.4333GiB/s) | 20260507 14:41:21.337 | INFO | Downloaded 34GiB to GPU in 1.52645s (22.2738GiB/s) | 20260507 14:41:26.476 | INFO | Analyzing chunk frames = [294400,294912) | 20260507 14:41:26.476 | INFO | MemInfo: { | 20260507 14:41:26.476 | INFO | "gpu" : " 17 GB", | 20260507 14:41:26.476 | INFO | "huge" : "", | 20260507 14:41:26.476 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:26.476 | INFO | "malloc" : " 14 GB", | 20260507 14:41:26.476 | INFO | "pinned" : " 29 GB" | 20260507 14:41:26.476 | INFO | } | 20260507 14:41:26.476 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:41:26.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:26.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:41:26.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:41:26.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:26.476 | INFO | Uploaded 36GiB to GPU in 1.60685s (22.4041GiB/s) | 20260507 14:41:26.477 | INFO | Downloaded 34GiB to GPU in 1.52666s (22.2709GiB/s) | 20260507 14:41:26.477 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:41:26.477 | INFO | Total mem usage: 15.7 GiB | 20260507 14:41:26.477 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:41:26.477 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -2.17%) | 20260507 14:41:26.477 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:41:26.477 | INFO | High Water Breakdown: | 20260507 14:41:26.477 | INFO | 59.3% : 9.08 GiB ( 97.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:41:26.477 | INFO | 40.6% : 6.22 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:41:26.477 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:41:26.477 | INFO | ------------------------------------------ | 20260507 14:41:26.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:41:26.477 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:41:26.477 | INFO | Total mem usage: 26.5 GiB | 20260507 14:41:26.477 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:41:26.477 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 14:41:26.477 | INFO | Unused Cache: 1.57 GiB (Waste: 5.92%) | 20260507 14:41:26.477 | INFO | High Water Breakdown: | 20260507 14:41:26.477 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:41:26.477 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:41:26.477 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:41:26.477 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:41:26.477 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:41:26.477 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:41:26.477 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:41:26.477 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:41:26.477 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:41:26.477 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:41:26.477 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:41:26.477 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:41:26.477 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:41:26.477 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:41:26.477 | INFO | ------------------------------------------ | 20260507 14:41:26.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:41:26.477 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:41:26.477 | INFO | Total mem usage: 27.8 GiB | 20260507 14:41:26.477 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:41:26.477 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:41:26.477 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:41:26.477 | INFO | High Water Breakdown: | 20260507 14:41:26.477 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:41:26.477 | INFO | ------------------------------------------ | 20260507 14:41:26.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:41:26.477 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:41:26.477 | INFO | Total mem usage: 15.7 GiB | 20260507 14:41:26.477 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:41:26.477 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:41:26.477 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:41:26.477 | INFO | High Water Breakdown: | 20260507 14:41:26.477 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:41:26.477 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:41:26.477 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:41:26.477 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:41:26.477 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:41:26.477 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:41:26.477 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:41:26.477 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:41:26.477 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:41:26.477 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:41:26.478 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:41:26.478 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:41:26.478 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:41:26.478 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:41:26.478 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:41:26.478 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:41:26.478 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:41:26.478 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:41:26.478 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:41:26.478 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:41:26.478 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:41:26.478 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:41:26.478 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:41:26.478 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:41:26.478 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:41:26.478 | INFO | ------------------------------------------ | 20260507 14:41:26.478 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:41:26.478 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:41:26.478 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:41:26.478 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:41:26.478 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 0.30%, AggregateStats -- 9.01e-02 / -nan, 9.35e-03 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 11.00%, Baseline -- 3.30e+00 / -nan, 2.53e-02 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 3.07%, Histogram -- 9.22e-01 / -nan, 5.42e-02 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 16.09%, DME -- 3.87e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / -nan, 4.44e-02 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.20e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.07e-01 / -nan, 5.96e-02 / 0.00e+00 | 20260507 14:41:26.478 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:41:26.478 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:41:26.478 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:41:26.478 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 18.75%, Upload -- 1.68e+01 / 0.00e+00, 4.14e-01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 18.04%, Download -- 1.62e+01 / 0.00e+00, 7.47e-01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 33.51%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 0.00%, LockDownload -- 5.51e-04 / 0.00e+00, 3.89e-04 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 29.11%, LockCompute -- 2.61e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260507 14:41:26.478 | INFO | PerfReport: 0.02%, Misc -- 1.66e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260507 14:41:26.574 | INFO | Starting a round of Baz IO | 20260507 14:41:29.528 | INFO | Populating RTMetrics report | 20260507 14:41:29.529 | INFO | RTMetrics report populated | 20260507 14:41:29.529 | INFO | Opening tmp RTMetrics json file | 20260507 14:41:29.540 | INFO | Serializing RTMetrics to json | 20260507 14:41:29.588 | INFO | Serialization done, swapping tmp file now... | 20260507 14:41:29.595 | INFO | RTMetrics done | 20260507 14:41:29.628 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.054 seconds | 20260507 14:41:29.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.18% / 0.08% / 42.74% | 20260507 14:41:29.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.45248% / 0.000290076% / 2.78155e-05% | 20260507 14:41:29.628 | INFO | Baz IO overall: | 20260507 14:41:29.628 | INFO | BytesWritten : 107022072454 | 20260507 14:41:29.628 | INFO | NumEvents : 50856666376 | 20260507 14:41:31.615 | INFO | Analyzing chunk frames = [294912,295424) | 20260507 14:41:31.615 | INFO | MemInfo: { | 20260507 14:41:31.615 | INFO | "gpu" : " 17 GB", | 20260507 14:41:31.615 | INFO | "huge" : "", | 20260507 14:41:31.615 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:31.615 | INFO | "malloc" : " 14 GB", | 20260507 14:41:31.616 | INFO | "pinned" : " 29 GB" | 20260507 14:41:31.616 | INFO | } | 20260507 14:41:31.616 | INFO | Perf report for the last 5.13954 seconds: | 20260507 14:41:31.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:31.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:31.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:31.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:31.616 | INFO | ANALYSIS : 57.8%, 2.9, 86.5ms, 0 | 20260507 14:41:31.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:31.616 | INFO | RT_METRICS : 49.7%, 1.0, 26.1ms, 0 | 20260507 14:41:31.616 | INFO | PRE_HQ : 57.4%, 2.3, 69.1ms, 0 | 20260507 14:41:31.616 | INFO | BAZWRITER : 59.0%, 4.3, 131.5ms, 0 | 20260507 14:41:31.616 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 14:41:31.616 | INFO | Uploaded 36GiB to GPU in 1.63741s (21.986GiB/s) | 20260507 14:41:31.616 | INFO | Downloaded 38GiB to GPU in 1.71362s (22.1752GiB/s) | 20260507 14:41:36.755 | INFO | Analyzing chunk frames = [295424,295936) | 20260507 14:41:36.755 | INFO | MemInfo: { | 20260507 14:41:36.755 | INFO | "gpu" : " 17 GB", | 20260507 14:41:36.755 | INFO | "huge" : "", | 20260507 14:41:36.755 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:36.755 | INFO | "malloc" : " 14 GB", | 20260507 14:41:36.755 | INFO | "pinned" : " 29 GB" | 20260507 14:41:36.755 | INFO | } | 20260507 14:41:36.755 | INFO | Perf report for the last 5.13957 seconds: | 20260507 14:41:36.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:36.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.755 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:41:36.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.755 | INFO | PRE_HQ : 41.0%, 1.0, 21.3ms, 0 | 20260507 14:41:36.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:36.756 | INFO | Uploaded 36GiB to GPU in 1.61049s (22.3535GiB/s) | 20260507 14:41:36.756 | INFO | Downloaded 34GiB to GPU in 1.52883s (22.2392GiB/s) | 20260507 14:41:41.894 | INFO | Analyzing chunk frames = [295936,296448) | 20260507 14:41:41.894 | INFO | MemInfo: { | 20260507 14:41:41.894 | INFO | "gpu" : " 17 GB", | 20260507 14:41:41.894 | INFO | "huge" : "", | 20260507 14:41:41.894 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:41.894 | INFO | "malloc" : " 14 GB", | 20260507 14:41:41.894 | INFO | "pinned" : " 29 GB" | 20260507 14:41:41.894 | INFO | } | 20260507 14:41:41.894 | INFO | Perf report for the last 5.13906 seconds: | 20260507 14:41:41.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:41.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.894 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:41:41.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.895 | INFO | PRE_HQ : 40.5%, 1.0, 21.0ms, 0 | 20260507 14:41:41.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:41.895 | INFO | Uploaded 36GiB to GPU in 1.61424s (22.3016GiB/s) | 20260507 14:41:41.895 | INFO | Downloaded 34GiB to GPU in 1.53392s (22.1654GiB/s) | 20260507 14:41:47.033 | INFO | Analyzing chunk frames = [296448,296960) | 20260507 14:41:47.033 | INFO | MemInfo: { | 20260507 14:41:47.033 | INFO | "gpu" : " 17 GB", | 20260507 14:41:47.033 | INFO | "huge" : "", | 20260507 14:41:47.034 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:47.034 | INFO | "malloc" : " 14 GB", | 20260507 14:41:47.034 | INFO | "pinned" : " 29 GB" | 20260507 14:41:47.034 | INFO | } | 20260507 14:41:47.034 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:41:47.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:47.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:41:47.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | PRE_HQ : 40.9%, 1.0, 21.2ms, 0 | 20260507 14:41:47.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:47.034 | INFO | Uploaded 36GiB to GPU in 1.61354s (22.3112GiB/s) | 20260507 14:41:47.034 | INFO | Downloaded 34GiB to GPU in 1.53241s (22.1873GiB/s) | 20260507 14:41:52.173 | INFO | Analyzing chunk frames = [296960,297472) | 20260507 14:41:52.173 | INFO | MemInfo: { | 20260507 14:41:52.173 | INFO | "gpu" : " 17 GB", | 20260507 14:41:52.173 | INFO | "huge" : "", | 20260507 14:41:52.173 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:52.173 | INFO | "malloc" : " 14 GB", | 20260507 14:41:52.173 | INFO | "pinned" : " 29 GB" | 20260507 14:41:52.173 | INFO | } | 20260507 14:41:52.173 | INFO | Perf report for the last 5.13915 seconds: | 20260507 14:41:52.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:52.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:41:52.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260507 14:41:52.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:52.173 | INFO | Uploaded 36GiB to GPU in 1.60811s (22.3865GiB/s) | 20260507 14:41:52.173 | INFO | Downloaded 34GiB to GPU in 1.52689s (22.2674GiB/s) | 20260507 14:41:57.312 | INFO | Analyzing chunk frames = [297472,297984) | 20260507 14:41:57.312 | INFO | MemInfo: { | 20260507 14:41:57.312 | INFO | "gpu" : " 17 GB", | 20260507 14:41:57.312 | INFO | "huge" : "", | 20260507 14:41:57.312 | INFO | "huge_shared" : " 29 GB", | 20260507 14:41:57.312 | INFO | "malloc" : " 14 GB", | 20260507 14:41:57.312 | INFO | "pinned" : " 29 GB" | 20260507 14:41:57.312 | INFO | } | 20260507 14:41:57.312 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:41:57.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:41:57.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:41:57.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:41:57.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:41:57.312 | INFO | Uploaded 36GiB to GPU in 1.61043s (22.3542GiB/s) | 20260507 14:41:57.312 | INFO | Downloaded 34GiB to GPU in 1.52565s (22.2855GiB/s) | 20260507 14:42:02.451 | INFO | Analyzing chunk frames = [297984,298496) | 20260507 14:42:02.451 | INFO | MemInfo: { | 20260507 14:42:02.451 | INFO | "gpu" : " 17 GB", | 20260507 14:42:02.451 | INFO | "huge" : "", | 20260507 14:42:02.451 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:02.451 | INFO | "malloc" : " 14 GB", | 20260507 14:42:02.451 | INFO | "pinned" : " 29 GB" | 20260507 14:42:02.451 | INFO | } | 20260507 14:42:02.451 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:42:02.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:02.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.451 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:42:02.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.452 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:42:02.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:02.452 | INFO | Uploaded 36GiB to GPU in 1.60707s (22.4011GiB/s) | 20260507 14:42:02.452 | INFO | Downloaded 34GiB to GPU in 1.52672s (22.27GiB/s) | 20260507 14:42:07.590 | INFO | Analyzing chunk frames = [298496,299008) | 20260507 14:42:07.591 | INFO | MemInfo: { | 20260507 14:42:07.591 | INFO | "gpu" : " 17 GB", | 20260507 14:42:07.591 | INFO | "huge" : "", | 20260507 14:42:07.591 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:07.591 | INFO | "malloc" : " 14 GB", | 20260507 14:42:07.591 | INFO | "pinned" : " 29 GB" | 20260507 14:42:07.591 | INFO | } | 20260507 14:42:07.591 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:42:07.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:07.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:42:07.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:42:07.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:07.591 | INFO | Uploaded 36GiB to GPU in 1.60719s (22.3993GiB/s) | 20260507 14:42:07.591 | INFO | Downloaded 34GiB to GPU in 1.52706s (22.265GiB/s) | 20260507 14:42:07.688 | INFO | Starting a round of Baz IO | 20260507 14:42:10.680 | INFO | Populating RTMetrics report | 20260507 14:42:10.681 | INFO | RTMetrics report populated | 20260507 14:42:10.681 | INFO | Opening tmp RTMetrics json file | 20260507 14:42:10.686 | INFO | Serializing RTMetrics to json | 20260507 14:42:10.716 | INFO | Serialization done, swapping tmp file now... | 20260507 14:42:10.719 | INFO | RTMetrics done | 20260507 14:42:10.808 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.12 seconds | 20260507 14:42:10.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.11% / 0.08% / 42.82% | 20260507 14:42:10.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.42008% / 0.000238419% / 2.78155e-05% | 20260507 14:42:10.808 | INFO | Baz IO overall: | 20260507 14:42:10.808 | INFO | BytesWritten : 108840155990 | 20260507 14:42:10.808 | INFO | NumEvents : 51742985540 | 20260507 14:42:12.730 | INFO | Analyzing chunk frames = [299008,299520) | 20260507 14:42:12.730 | INFO | MemInfo: { | 20260507 14:42:12.730 | INFO | "gpu" : " 17 GB", | 20260507 14:42:12.730 | INFO | "huge" : "", | 20260507 14:42:12.730 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:12.730 | INFO | "malloc" : " 14 GB", | 20260507 14:42:12.730 | INFO | "pinned" : " 29 GB" | 20260507 14:42:12.730 | INFO | } | 20260507 14:42:12.730 | INFO | Perf report for the last 5.13972 seconds: | 20260507 14:42:12.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:12.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:12.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:12.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:12.730 | INFO | ANALYSIS : 58.6%, 2.8, 85.2ms, 0 | 20260507 14:42:12.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:12.731 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260507 14:42:12.731 | INFO | PRE_HQ : 58.1%, 2.2, 65.8ms, 0 | 20260507 14:42:12.731 | INFO | BAZWRITER : 60.3%, 4.6, 142.2ms, 0 | 20260507 14:42:12.731 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 14:42:12.731 | INFO | Uploaded 36GiB to GPU in 1.63904s (21.964GiB/s) | 20260507 14:42:12.731 | INFO | Downloaded 38GiB to GPU in 1.71731s (22.1276GiB/s) | 20260507 14:42:17.869 | INFO | Analyzing chunk frames = [299520,300032) | 20260507 14:42:17.869 | INFO | MemInfo: { | 20260507 14:42:17.869 | INFO | "gpu" : " 17 GB", | 20260507 14:42:17.870 | INFO | "huge" : "", | 20260507 14:42:17.870 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:17.870 | INFO | "malloc" : " 14 GB", | 20260507 14:42:17.870 | INFO | "pinned" : " 29 GB" | 20260507 14:42:17.870 | INFO | } | 20260507 14:42:17.870 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:42:17.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:17.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.870 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:42:17.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.870 | INFO | PRE_HQ : 41.7%, 1.0, 22.7ms, 0 | 20260507 14:42:17.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:17.871 | INFO | Uploaded 36GiB to GPU in 1.5988s (22.5169GiB/s) | 20260507 14:42:17.871 | INFO | Downloaded 34GiB to GPU in 1.53412s (22.1625GiB/s) | 20260507 14:42:23.009 | INFO | Analyzing chunk frames = [300032,300544) | 20260507 14:42:23.009 | INFO | MemInfo: { | 20260507 14:42:23.009 | INFO | "gpu" : " 17 GB", | 20260507 14:42:23.009 | INFO | "huge" : "", | 20260507 14:42:23.009 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:23.009 | INFO | "malloc" : " 14 GB", | 20260507 14:42:23.009 | INFO | "pinned" : " 29 GB" | 20260507 14:42:23.009 | INFO | } | 20260507 14:42:23.009 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:42:23.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:23.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.009 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:42:23.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.010 | INFO | PRE_HQ : 40.9%, 1.0, 21.3ms, 0 | 20260507 14:42:23.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:23.010 | INFO | Uploaded 36GiB to GPU in 1.59443s (22.5786GiB/s) | 20260507 14:42:23.010 | INFO | Downloaded 34GiB to GPU in 1.54139s (22.058GiB/s) | 20260507 14:42:28.148 | INFO | Analyzing chunk frames = [300544,301056) | 20260507 14:42:28.148 | INFO | MemInfo: { | 20260507 14:42:28.148 | INFO | "gpu" : " 17 GB", | 20260507 14:42:28.148 | INFO | "huge" : "", | 20260507 14:42:28.148 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:28.148 | INFO | "malloc" : " 14 GB", | 20260507 14:42:28.148 | INFO | "pinned" : " 29 GB" | 20260507 14:42:28.148 | INFO | } | 20260507 14:42:28.148 | INFO | Perf report for the last 5.13892 seconds: | 20260507 14:42:28.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:28.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 14:42:28.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | PRE_HQ : 40.5%, 1.0, 21.1ms, 0 | 20260507 14:42:28.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:28.149 | INFO | Uploaded 36GiB to GPU in 1.59311s (22.5973GiB/s) | 20260507 14:42:28.149 | INFO | Downloaded 34GiB to GPU in 1.54191s (22.0506GiB/s) | 20260507 14:42:33.287 | INFO | Analyzing chunk frames = [301056,301568) | 20260507 14:42:33.287 | INFO | MemInfo: { | 20260507 14:42:33.287 | INFO | "gpu" : " 17 GB", | 20260507 14:42:33.287 | INFO | "huge" : "", | 20260507 14:42:33.287 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:33.287 | INFO | "malloc" : " 14 GB", | 20260507 14:42:33.287 | INFO | "pinned" : " 29 GB" | 20260507 14:42:33.287 | INFO | } | 20260507 14:42:33.288 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:42:33.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:33.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:42:33.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | PRE_HQ : 38.7%, 1.0, 20.2ms, 0 | 20260507 14:42:33.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:33.288 | INFO | Uploaded 36GiB to GPU in 1.59089s (22.6289GiB/s) | 20260507 14:42:33.288 | INFO | Downloaded 34GiB to GPU in 1.53974s (22.0817GiB/s) | 20260507 14:42:38.427 | INFO | Analyzing chunk frames = [301568,302080) | 20260507 14:42:38.427 | INFO | MemInfo: { | 20260507 14:42:38.427 | INFO | "gpu" : " 17 GB", | 20260507 14:42:38.427 | INFO | "huge" : "", | 20260507 14:42:38.427 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:38.427 | INFO | "malloc" : " 14 GB", | 20260507 14:42:38.427 | INFO | "pinned" : " 29 GB" | 20260507 14:42:38.427 | INFO | } | 20260507 14:42:38.427 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:42:38.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:38.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:42:38.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:42:38.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:38.427 | INFO | Uploaded 36GiB to GPU in 1.58669s (22.6887GiB/s) | 20260507 14:42:38.427 | INFO | Downloaded 34GiB to GPU in 1.53436s (22.159GiB/s) | 20260507 14:42:43.566 | INFO | Analyzing chunk frames = [302080,302592) | 20260507 14:42:43.566 | INFO | MemInfo: { | 20260507 14:42:43.566 | INFO | "gpu" : " 17 GB", | 20260507 14:42:43.566 | INFO | "huge" : "", | 20260507 14:42:43.566 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:43.566 | INFO | "malloc" : " 14 GB", | 20260507 14:42:43.566 | INFO | "pinned" : " 29 GB" | 20260507 14:42:43.566 | INFO | } | 20260507 14:42:43.566 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:42:43.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:43.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.566 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:42:43.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.566 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:42:43.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:43.567 | INFO | Uploaded 36GiB to GPU in 1.58586s (22.7006GiB/s) | 20260507 14:42:43.567 | INFO | Downloaded 34GiB to GPU in 1.53579s (22.1385GiB/s) | 20260507 14:42:48.706 | INFO | Analyzing chunk frames = [302592,303104) | 20260507 14:42:48.706 | INFO | MemInfo: { | 20260507 14:42:48.706 | INFO | "gpu" : " 17 GB", | 20260507 14:42:48.706 | INFO | "huge" : "", | 20260507 14:42:48.706 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:48.706 | INFO | "malloc" : " 14 GB", | 20260507 14:42:48.706 | INFO | "pinned" : " 29 GB" | 20260507 14:42:48.706 | INFO | } | 20260507 14:42:48.706 | INFO | Perf report for the last 5.13959 seconds: | 20260507 14:42:48.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:48.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:42:48.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:42:48.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:48.706 | INFO | Uploaded 36GiB to GPU in 1.5871s (22.6829GiB/s) | 20260507 14:42:48.706 | INFO | Downloaded 34GiB to GPU in 1.53564s (22.1407GiB/s) | 20260507 14:42:48.706 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:42:48.706 | INFO | Total mem usage: 15.7 GiB | 20260507 14:42:48.706 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:42:48.706 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -1.43%) | 20260507 14:42:48.706 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:42:48.706 | INFO | High Water Breakdown: | 20260507 14:42:48.706 | INFO | 59.4% : 9.17 GiB ( 98.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:42:48.706 | INFO | 40.5% : 6.25 GiB ( 93% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:42:48.706 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:42:48.706 | INFO | ------------------------------------------ | 20260507 14:42:48.706 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:42:48.706 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:42:48.706 | INFO | Total mem usage: 26.5 GiB | 20260507 14:42:48.706 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:42:48.706 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 14:42:48.706 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:42:48.706 | INFO | High Water Breakdown: | 20260507 14:42:48.706 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:42:48.706 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:42:48.706 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:42:48.706 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:42:48.706 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:42:48.706 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:42:48.706 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:42:48.706 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:42:48.706 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:42:48.706 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:42:48.706 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:42:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:42:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:42:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:42:48.707 | INFO | ------------------------------------------ | 20260507 14:42:48.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:42:48.707 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:42:48.707 | INFO | Total mem usage: 27.8 GiB | 20260507 14:42:48.707 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:42:48.707 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 14:42:48.707 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 14:42:48.707 | INFO | High Water Breakdown: | 20260507 14:42:48.707 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 14:42:48.707 | INFO | ------------------------------------------ | 20260507 14:42:48.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:42:48.707 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:42:48.707 | INFO | Total mem usage: 15.7 GiB | 20260507 14:42:48.707 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:42:48.707 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:42:48.707 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:42:48.707 | INFO | High Water Breakdown: | 20260507 14:42:48.707 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:42:48.707 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:42:48.707 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:42:48.707 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:42:48.707 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:42:48.707 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:42:48.707 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:42:48.707 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:42:48.707 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:42:48.707 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:42:48.707 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:42:48.707 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:42:48.707 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:42:48.707 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:42:48.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:42:48.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:42:48.707 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:42:48.707 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:42:48.707 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:42:48.707 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:42:48.707 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:42:48.707 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:42:48.707 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:42:48.707 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:42:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:42:48.707 | INFO | ------------------------------------------ | 20260507 14:42:48.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:42:48.707 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:42:48.707 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:42:48.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:42:48.707 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 0.30%, AggregateStats -- 9.01e-02 / -nan, 6.29e-03 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 11.01%, Baseline -- 3.31e+00 / -nan, 2.58e-02 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 3.07%, Histogram -- 9.23e-01 / -nan, 5.46e-02 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 16.09%, DME -- 3.87e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / -nan, 4.88e-02 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.19e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.06e-01 / -nan, 5.93e-02 / 0.00e+00 | 20260507 14:42:48.707 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:42:48.707 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:42:48.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:42:48.707 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 18.63%, Upload -- 1.67e+01 / 0.00e+00, 4.38e-01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 18.06%, Download -- 1.62e+01 / 0.00e+00, 7.34e-01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 33.46%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 0.59%, LockUpload -- 5.31e-01 / 0.00e+00, 3.68e+00 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 0.00%, LockDownload -- 5.70e-04 / 0.00e+00, 4.16e-04 / 0.00e+00 | 20260507 14:42:48.707 | INFO | PerfReport: 29.23%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:42:48.708 | INFO | PerfReport: 0.02%, Misc -- 1.63e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260507 14:42:48.803 | INFO | Starting a round of Baz IO | 20260507 14:42:51.844 | INFO | Populating RTMetrics report | 20260507 14:42:51.844 | INFO | RTMetrics report populated | 20260507 14:42:51.844 | INFO | Opening tmp RTMetrics json file | 20260507 14:42:51.848 | INFO | Serializing RTMetrics to json | 20260507 14:42:51.867 | INFO | Serialization done, swapping tmp file now... | 20260507 14:42:51.900 | INFO | RTMetrics done | 20260507 14:42:52.093 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.29 seconds | 20260507 14:42:52.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.03% / 0.08% / 42.89% | 20260507 14:42:52.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.41009% / 0.000198682% / 3.57628e-05% | 20260507 14:42:52.093 | INFO | Baz IO overall: | 20260507 14:42:52.093 | INFO | BytesWritten : 110661637056 | 20260507 14:42:52.093 | INFO | NumEvents : 52631635832 | 20260507 14:42:53.845 | INFO | Analyzing chunk frames = [303104,303616) | 20260507 14:42:53.845 | INFO | MemInfo: { | 20260507 14:42:53.845 | INFO | "gpu" : " 17 GB", | 20260507 14:42:53.845 | INFO | "huge" : "", | 20260507 14:42:53.845 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:53.845 | INFO | "malloc" : " 14 GB", | 20260507 14:42:53.845 | INFO | "pinned" : " 29 GB" | 20260507 14:42:53.845 | INFO | } | 20260507 14:42:53.845 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:42:53.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:53.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:53.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:53.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:53.845 | INFO | ANALYSIS : 59.2%, 2.7, 84.4ms, 0 | 20260507 14:42:53.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:53.845 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260507 14:42:53.846 | INFO | PRE_HQ : 58.7%, 2.2, 67.7ms, 0 | 20260507 14:42:53.846 | INFO | BAZWRITER : 63.6%, 4.8, 156.7ms, 0 | 20260507 14:42:53.846 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 14:42:53.846 | INFO | Uploaded 36GiB to GPU in 1.64201s (21.9243GiB/s) | 20260507 14:42:53.846 | INFO | Downloaded 38GiB to GPU in 1.71795s (22.1193GiB/s) | 20260507 14:42:58.984 | INFO | Analyzing chunk frames = [303616,304128) | 20260507 14:42:58.984 | INFO | MemInfo: { | 20260507 14:42:58.984 | INFO | "gpu" : " 17 GB", | 20260507 14:42:58.984 | INFO | "huge" : "", | 20260507 14:42:58.985 | INFO | "huge_shared" : " 29 GB", | 20260507 14:42:58.985 | INFO | "malloc" : " 14 GB", | 20260507 14:42:58.985 | INFO | "pinned" : " 29 GB" | 20260507 14:42:58.985 | INFO | } | 20260507 14:42:58.985 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:42:58.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:42:58.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260507 14:42:58.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | PRE_HQ : 40.8%, 1.0, 21.6ms, 0 | 20260507 14:42:58.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:42:58.985 | INFO | Uploaded 36GiB to GPU in 1.60699s (22.4021GiB/s) | 20260507 14:42:58.985 | INFO | Downloaded 34GiB to GPU in 1.53033s (22.2174GiB/s) | 20260507 14:43:04.123 | INFO | Analyzing chunk frames = [304128,304640) | 20260507 14:43:04.123 | INFO | MemInfo: { | 20260507 14:43:04.123 | INFO | "gpu" : " 17 GB", | 20260507 14:43:04.123 | INFO | "huge" : "", | 20260507 14:43:04.124 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:04.124 | INFO | "malloc" : " 14 GB", | 20260507 14:43:04.124 | INFO | "pinned" : " 29 GB" | 20260507 14:43:04.124 | INFO | } | 20260507 14:43:04.124 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:43:04.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:04.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.124 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260507 14:43:04.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.124 | INFO | PRE_HQ : 41.0%, 1.0, 21.3ms, 0 | 20260507 14:43:04.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:04.125 | INFO | Uploaded 36GiB to GPU in 1.59465s (22.5755GiB/s) | 20260507 14:43:04.125 | INFO | Downloaded 34GiB to GPU in 1.54253s (22.0417GiB/s) | 20260507 14:43:09.263 | INFO | Analyzing chunk frames = [304640,305152) | 20260507 14:43:09.263 | INFO | MemInfo: { | 20260507 14:43:09.263 | INFO | "gpu" : " 17 GB", | 20260507 14:43:09.263 | INFO | "huge" : "", | 20260507 14:43:09.263 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:09.263 | INFO | "malloc" : " 14 GB", | 20260507 14:43:09.263 | INFO | "pinned" : " 29 GB" | 20260507 14:43:09.263 | INFO | } | 20260507 14:43:09.263 | INFO | Perf report for the last 5.1389 seconds: | 20260507 14:43:09.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:09.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.263 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 14:43:09.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.263 | INFO | PRE_HQ : 41.5%, 1.0, 21.5ms, 0 | 20260507 14:43:09.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:09.264 | INFO | Uploaded 36GiB to GPU in 1.5937s (22.589GiB/s) | 20260507 14:43:09.264 | INFO | Downloaded 34GiB to GPU in 1.541s (22.0636GiB/s) | 20260507 14:43:14.402 | INFO | Analyzing chunk frames = [305152,305664) | 20260507 14:43:14.402 | INFO | MemInfo: { | 20260507 14:43:14.402 | INFO | "gpu" : " 17 GB", | 20260507 14:43:14.402 | INFO | "huge" : "", | 20260507 14:43:14.402 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:14.402 | INFO | "malloc" : " 14 GB", | 20260507 14:43:14.402 | INFO | "pinned" : " 29 GB" | 20260507 14:43:14.402 | INFO | } | 20260507 14:43:14.402 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:43:14.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:14.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.402 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:43:14.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.402 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:43:14.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:14.403 | INFO | Uploaded 36GiB to GPU in 1.5856s (22.7043GiB/s) | 20260507 14:43:14.403 | INFO | Downloaded 34GiB to GPU in 1.53439s (22.1586GiB/s) | 20260507 14:43:19.541 | INFO | Analyzing chunk frames = [305664,306176) | 20260507 14:43:19.541 | INFO | MemInfo: { | 20260507 14:43:19.541 | INFO | "gpu" : " 17 GB", | 20260507 14:43:19.542 | INFO | "huge" : "", | 20260507 14:43:19.542 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:19.542 | INFO | "malloc" : " 14 GB", | 20260507 14:43:19.542 | INFO | "pinned" : " 29 GB" | 20260507 14:43:19.542 | INFO | } | 20260507 14:43:19.542 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:43:19.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:19.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:43:19.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:43:19.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:19.542 | INFO | Uploaded 36GiB to GPU in 1.58622s (22.6955GiB/s) | 20260507 14:43:19.542 | INFO | Downloaded 34GiB to GPU in 1.53448s (22.1573GiB/s) | 20260507 14:43:24.681 | INFO | Analyzing chunk frames = [306176,306688) | 20260507 14:43:24.681 | INFO | MemInfo: { | 20260507 14:43:24.681 | INFO | "gpu" : " 17 GB", | 20260507 14:43:24.681 | INFO | "huge" : "", | 20260507 14:43:24.681 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:24.681 | INFO | "malloc" : " 14 GB", | 20260507 14:43:24.681 | INFO | "pinned" : " 29 GB" | 20260507 14:43:24.681 | INFO | } | 20260507 14:43:24.681 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:43:24.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:24.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:43:24.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 14:43:24.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:24.681 | INFO | Uploaded 36GiB to GPU in 1.58664s (22.6894GiB/s) | 20260507 14:43:24.681 | INFO | Downloaded 34GiB to GPU in 1.53521s (22.1468GiB/s) | 20260507 14:43:29.820 | INFO | Analyzing chunk frames = [306688,307200) | 20260507 14:43:29.820 | INFO | MemInfo: { | 20260507 14:43:29.820 | INFO | "gpu" : " 17 GB", | 20260507 14:43:29.820 | INFO | "huge" : "", | 20260507 14:43:29.820 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:29.820 | INFO | "malloc" : " 14 GB", | 20260507 14:43:29.820 | INFO | "pinned" : " 29 GB" | 20260507 14:43:29.820 | INFO | } | 20260507 14:43:29.820 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:43:29.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:29.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.821 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:43:29.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.821 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:43:29.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:29.821 | INFO | Uploaded 36GiB to GPU in 1.58672s (22.6884GiB/s) | 20260507 14:43:29.821 | INFO | Downloaded 34GiB to GPU in 1.5353s (22.1455GiB/s) | 20260507 14:43:29.918 | INFO | Starting a round of Baz IO | 20260507 14:43:32.934 | INFO | Populating RTMetrics report | 20260507 14:43:32.934 | INFO | RTMetrics report populated | 20260507 14:43:32.934 | INFO | Opening tmp RTMetrics json file | 20260507 14:43:32.949 | INFO | Serializing RTMetrics to json | 20260507 14:43:32.951 | INFO | Serialization done, swapping tmp file now... | 20260507 14:43:32.951 | INFO | RTMetrics done | 20260507 14:43:33.065 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.147 seconds | 20260507 14:43:33.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.95% / 0.08% / 42.97% | 20260507 14:43:33.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.33705% / 0.000198682% / 1.58946e-05% | 20260507 14:43:33.065 | INFO | Baz IO overall: | 20260507 14:43:33.065 | INFO | BytesWritten : 112481852751 | 20260507 14:43:33.065 | INFO | NumEvents : 53518484108 | 20260507 14:43:34.959 | INFO | Analyzing chunk frames = [307200,307712) | 20260507 14:43:34.959 | INFO | MemInfo: { | 20260507 14:43:34.959 | INFO | "gpu" : " 17 GB", | 20260507 14:43:34.960 | INFO | "huge" : "", | 20260507 14:43:34.960 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:34.960 | INFO | "malloc" : " 14 GB", | 20260507 14:43:34.960 | INFO | "pinned" : " 29 GB" | 20260507 14:43:34.960 | INFO | } | 20260507 14:43:34.960 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:43:34.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:34.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:34.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:34.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:34.960 | INFO | ANALYSIS : 58.2%, 2.8, 86.0ms, 0 | 20260507 14:43:34.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:34.960 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260507 14:43:34.960 | INFO | PRE_HQ : 58.0%, 2.3, 68.9ms, 0 | 20260507 14:43:34.960 | INFO | BAZWRITER : 60.8%, 4.5, 139.6ms, 0 | 20260507 14:43:34.960 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 14:43:34.960 | INFO | Uploaded 36GiB to GPU in 1.64361s (21.903GiB/s) | 20260507 14:43:34.960 | INFO | Downloaded 38GiB to GPU in 1.72042s (22.0877GiB/s) | 20260507 14:43:40.099 | INFO | Analyzing chunk frames = [307712,308224) | 20260507 14:43:40.099 | INFO | MemInfo: { | 20260507 14:43:40.099 | INFO | "gpu" : " 17 GB", | 20260507 14:43:40.099 | INFO | "huge" : "", | 20260507 14:43:40.099 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:40.099 | INFO | "malloc" : " 14 GB", | 20260507 14:43:40.099 | INFO | "pinned" : " 29 GB" | 20260507 14:43:40.099 | INFO | } | 20260507 14:43:40.099 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:43:40.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:40.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:43:40.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | PRE_HQ : 28.8%, 1.0, 15.1ms, 0 | 20260507 14:43:40.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:40.099 | INFO | Uploaded 36GiB to GPU in 1.5843s (22.723GiB/s) | 20260507 14:43:40.099 | INFO | Downloaded 34GiB to GPU in 1.5321s (22.1917GiB/s) | 20260507 14:43:45.238 | INFO | Analyzing chunk frames = [308224,308736) | 20260507 14:43:45.238 | INFO | MemInfo: { | 20260507 14:43:45.238 | INFO | "gpu" : " 17 GB", | 20260507 14:43:45.238 | INFO | "huge" : "", | 20260507 14:43:45.238 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:45.238 | INFO | "malloc" : " 14 GB", | 20260507 14:43:45.238 | INFO | "pinned" : " 29 GB" | 20260507 14:43:45.238 | INFO | } | 20260507 14:43:45.238 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:43:45.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:45.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.238 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 14:43:45.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.239 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:43:45.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:45.239 | INFO | Uploaded 36GiB to GPU in 1.5889s (22.6571GiB/s) | 20260507 14:43:45.239 | INFO | Downloaded 34GiB to GPU in 1.53756s (22.1129GiB/s) | 20260507 14:43:50.377 | INFO | Analyzing chunk frames = [308736,309248) | 20260507 14:43:50.377 | INFO | MemInfo: { | 20260507 14:43:50.377 | INFO | "gpu" : " 17 GB", | 20260507 14:43:50.378 | INFO | "huge" : "", | 20260507 14:43:50.378 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:50.378 | INFO | "malloc" : " 14 GB", | 20260507 14:43:50.378 | INFO | "pinned" : " 29 GB" | 20260507 14:43:50.378 | INFO | } | 20260507 14:43:50.378 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:43:50.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:50.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:43:50.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:43:50.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:50.378 | INFO | Uploaded 36GiB to GPU in 1.58698s (22.6846GiB/s) | 20260507 14:43:50.378 | INFO | Downloaded 34GiB to GPU in 1.53688s (22.1227GiB/s) | 20260507 14:43:55.517 | INFO | Analyzing chunk frames = [309248,309760) | 20260507 14:43:55.517 | INFO | MemInfo: { | 20260507 14:43:55.517 | INFO | "gpu" : " 17 GB", | 20260507 14:43:55.517 | INFO | "huge" : "", | 20260507 14:43:55.517 | INFO | "huge_shared" : " 29 GB", | 20260507 14:43:55.517 | INFO | "malloc" : " 14 GB", | 20260507 14:43:55.517 | INFO | "pinned" : " 29 GB" | 20260507 14:43:55.517 | INFO | } | 20260507 14:43:55.517 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:43:55.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:43:55.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:43:55.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:43:55.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:43:55.517 | INFO | Uploaded 36GiB to GPU in 1.58705s (22.6836GiB/s) | 20260507 14:43:55.517 | INFO | Downloaded 34GiB to GPU in 1.53485s (22.1519GiB/s) | 20260507 14:44:00.656 | INFO | Analyzing chunk frames = [309760,310272) | 20260507 14:44:00.656 | INFO | MemInfo: { | 20260507 14:44:00.656 | INFO | "gpu" : " 17 GB", | 20260507 14:44:00.656 | INFO | "huge" : "", | 20260507 14:44:00.656 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:00.656 | INFO | "malloc" : " 14 GB", | 20260507 14:44:00.656 | INFO | "pinned" : " 29 GB" | 20260507 14:44:00.656 | INFO | } | 20260507 14:44:00.656 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:44:00.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:00.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.656 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:44:00.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.656 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:44:00.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:00.657 | INFO | Uploaded 36GiB to GPU in 1.58579s (22.7016GiB/s) | 20260507 14:44:00.657 | INFO | Downloaded 34GiB to GPU in 1.53486s (22.1519GiB/s) | 20260507 14:44:05.796 | INFO | Analyzing chunk frames = [310272,310784) | 20260507 14:44:05.796 | INFO | MemInfo: { | 20260507 14:44:05.796 | INFO | "gpu" : " 17 GB", | 20260507 14:44:05.796 | INFO | "huge" : "", | 20260507 14:44:05.796 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:05.796 | INFO | "malloc" : " 14 GB", | 20260507 14:44:05.796 | INFO | "pinned" : " 29 GB" | 20260507 14:44:05.796 | INFO | } | 20260507 14:44:05.796 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:44:05.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:05.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:44:05.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:44:05.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:05.796 | INFO | Uploaded 36GiB to GPU in 1.5876s (22.6757GiB/s) | 20260507 14:44:05.796 | INFO | Downloaded 34GiB to GPU in 1.53595s (22.1361GiB/s) | 20260507 14:44:10.935 | INFO | Analyzing chunk frames = [310784,311296) | 20260507 14:44:10.935 | INFO | MemInfo: { | 20260507 14:44:10.935 | INFO | "gpu" : " 17 GB", | 20260507 14:44:10.935 | INFO | "huge" : "", | 20260507 14:44:10.935 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:10.935 | INFO | "malloc" : " 14 GB", | 20260507 14:44:10.935 | INFO | "pinned" : " 29 GB" | 20260507 14:44:10.935 | INFO | } | 20260507 14:44:10.935 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:44:10.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:10.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 14:44:10.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:44:10.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:10.935 | INFO | Uploaded 36GiB to GPU in 1.58677s (22.6876GiB/s) | 20260507 14:44:10.935 | INFO | Downloaded 34GiB to GPU in 1.53516s (22.1475GiB/s) | 20260507 14:44:10.936 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:44:10.936 | INFO | Total mem usage: 15.7 GiB | 20260507 14:44:10.936 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:44:10.936 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -1.34%) | 20260507 14:44:10.936 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:44:10.936 | INFO | High Water Breakdown: | 20260507 14:44:10.936 | INFO | 59.4% : 9.18 GiB ( 98.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:44:10.936 | INFO | 40.5% : 6.26 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:44:10.936 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:44:10.936 | INFO | ------------------------------------------ | 20260507 14:44:10.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:44:10.936 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:44:10.936 | INFO | Total mem usage: 26.5 GiB | 20260507 14:44:10.936 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:44:10.936 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 14:44:10.936 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 14:44:10.936 | INFO | High Water Breakdown: | 20260507 14:44:10.936 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:44:10.936 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:44:10.936 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:44:10.936 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:44:10.936 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:44:10.936 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:44:10.936 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:44:10.936 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:44:10.936 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:44:10.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:44:10.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:44:10.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:44:10.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:44:10.936 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:44:10.936 | INFO | ------------------------------------------ | 20260507 14:44:10.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:44:10.936 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:44:10.936 | INFO | Total mem usage: 27.8 GiB | 20260507 14:44:10.936 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:44:10.936 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:44:10.936 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:44:10.936 | INFO | High Water Breakdown: | 20260507 14:44:10.936 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:44:10.936 | INFO | ------------------------------------------ | 20260507 14:44:10.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:44:10.936 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:44:10.936 | INFO | Total mem usage: 15.7 GiB | 20260507 14:44:10.936 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:44:10.936 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:44:10.936 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:44:10.936 | INFO | High Water Breakdown: | 20260507 14:44:10.936 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:44:10.936 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:44:10.936 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:44:10.936 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:44:10.936 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:44:10.936 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:44:10.936 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:44:10.936 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:44:10.936 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:44:10.936 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:44:10.936 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:44:10.937 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:44:10.937 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:44:10.937 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:44:10.937 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:44:10.937 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:44:10.937 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:44:10.937 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:44:10.937 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:44:10.937 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:44:10.937 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:44:10.937 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:44:10.937 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:44:10.937 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:44:10.937 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:44:10.937 | INFO | ------------------------------------------ | 20260507 14:44:10.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:44:10.937 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:44:10.937 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:44:10.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:44:10.937 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 0.30%, AggregateStats -- 8.86e-02 / -nan, 6.36e-03 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 11.00%, Baseline -- 3.30e+00 / -nan, 2.17e-02 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.44e-02 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 16.10%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.72e+01 / -nan, 4.40e-02 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.19e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 3.59%, Metrics -- 1.08e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.02e-01 / -nan, 5.81e-02 / 0.00e+00 | 20260507 14:44:10.937 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:44:10.937 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:44:10.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:44:10.937 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 18.56%, Upload -- 1.66e+01 / 0.00e+00, 4.60e-01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 18.13%, Download -- 1.62e+01 / 0.00e+00, 7.41e-01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 33.48%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 0.60%, LockUpload -- 5.40e-01 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 0.00%, LockDownload -- 5.07e-04 / 0.00e+00, 3.36e-04 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 29.21%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:44:10.937 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 14:44:11.033 | INFO | Starting a round of Baz IO | 20260507 14:44:13.929 | INFO | Populating RTMetrics report | 20260507 14:44:13.929 | INFO | RTMetrics report populated | 20260507 14:44:13.929 | INFO | Opening tmp RTMetrics json file | 20260507 14:44:14.031 | INFO | Serializing RTMetrics to json | 20260507 14:44:14.037 | INFO | Serialization done, swapping tmp file now... | 20260507 14:44:14.044 | INFO | RTMetrics done | 20260507 14:44:14.065 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.032 seconds | 20260507 14:44:14.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.88% / 0.07% / 43.05% | 20260507 14:44:14.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.32415% / 0.000194709% / 2.78155e-05% | 20260507 14:44:14.065 | INFO | Baz IO overall: | 20260507 14:44:14.065 | INFO | BytesWritten : 114304261784 | 20260507 14:44:14.065 | INFO | NumEvents : 54407025085 | 20260507 14:44:16.074 | INFO | Analyzing chunk frames = [311296,311808) | 20260507 14:44:16.074 | INFO | MemInfo: { | 20260507 14:44:16.074 | INFO | "gpu" : " 17 GB", | 20260507 14:44:16.074 | INFO | "huge" : "", | 20260507 14:44:16.074 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:16.074 | INFO | "malloc" : " 14 GB", | 20260507 14:44:16.074 | INFO | "pinned" : " 29 GB" | 20260507 14:44:16.074 | INFO | } | 20260507 14:44:16.074 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:44:16.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:16.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:16.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:16.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:16.074 | INFO | ANALYSIS : 57.3%, 2.9, 87.3ms, 0 | 20260507 14:44:16.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:16.074 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 14:44:16.074 | INFO | PRE_HQ : 56.2%, 2.0, 59.8ms, 0 | 20260507 14:44:16.074 | INFO | BAZWRITER : 58.5%, 4.6, 137.1ms, 0 | 20260507 14:44:16.074 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 14:44:16.075 | INFO | Uploaded 36GiB to GPU in 1.65132s (21.8008GiB/s) | 20260507 14:44:16.075 | INFO | Downloaded 38GiB to GPU in 1.73262s (21.9321GiB/s) | 20260507 14:44:21.214 | INFO | Analyzing chunk frames = [311808,312320) | 20260507 14:44:21.214 | INFO | MemInfo: { | 20260507 14:44:21.214 | INFO | "gpu" : " 17 GB", | 20260507 14:44:21.214 | INFO | "huge" : "", | 20260507 14:44:21.214 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:21.214 | INFO | "malloc" : " 14 GB", | 20260507 14:44:21.214 | INFO | "pinned" : " 29 GB" | 20260507 14:44:21.214 | INFO | } | 20260507 14:44:21.214 | INFO | Perf report for the last 5.1396 seconds: | 20260507 14:44:21.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:21.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:44:21.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:44:21.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:21.214 | INFO | Uploaded 36GiB to GPU in 1.60411s (22.4424GiB/s) | 20260507 14:44:21.214 | INFO | Downloaded 34GiB to GPU in 1.52323s (22.321GiB/s) | 20260507 14:44:26.353 | INFO | Analyzing chunk frames = [312320,312832) | 20260507 14:44:26.353 | INFO | MemInfo: { | 20260507 14:44:26.353 | INFO | "gpu" : " 17 GB", | 20260507 14:44:26.353 | INFO | "huge" : "", | 20260507 14:44:26.353 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:26.353 | INFO | "malloc" : " 14 GB", | 20260507 14:44:26.353 | INFO | "pinned" : " 29 GB" | 20260507 14:44:26.353 | INFO | } | 20260507 14:44:26.353 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:44:26.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:26.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.353 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:44:26.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.353 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:44:26.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:26.354 | INFO | Uploaded 36GiB to GPU in 1.60868s (22.3787GiB/s) | 20260507 14:44:26.354 | INFO | Downloaded 34GiB to GPU in 1.52842s (22.2452GiB/s) | 20260507 14:44:31.492 | INFO | Analyzing chunk frames = [312832,313344) | 20260507 14:44:31.492 | INFO | MemInfo: { | 20260507 14:44:31.492 | INFO | "gpu" : " 17 GB", | 20260507 14:44:31.492 | INFO | "huge" : "", | 20260507 14:44:31.492 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:31.492 | INFO | "malloc" : " 14 GB", | 20260507 14:44:31.492 | INFO | "pinned" : " 29 GB" | 20260507 14:44:31.492 | INFO | } | 20260507 14:44:31.493 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:44:31.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:31.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:44:31.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:44:31.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:31.493 | INFO | Uploaded 36GiB to GPU in 1.60816s (22.3858GiB/s) | 20260507 14:44:31.493 | INFO | Downloaded 34GiB to GPU in 1.52776s (22.2548GiB/s) | 20260507 14:44:36.632 | INFO | Analyzing chunk frames = [313344,313856) | 20260507 14:44:36.632 | INFO | MemInfo: { | 20260507 14:44:36.632 | INFO | "gpu" : " 17 GB", | 20260507 14:44:36.632 | INFO | "huge" : "", | 20260507 14:44:36.632 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:36.632 | INFO | "malloc" : " 14 GB", | 20260507 14:44:36.632 | INFO | "pinned" : " 29 GB" | 20260507 14:44:36.632 | INFO | } | 20260507 14:44:36.632 | INFO | Perf report for the last 5.13951 seconds: | 20260507 14:44:36.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:36.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 14:44:36.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:44:36.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:36.632 | INFO | Uploaded 36GiB to GPU in 1.60617s (22.4136GiB/s) | 20260507 14:44:36.632 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2905GiB/s) | 20260507 14:44:41.771 | INFO | Analyzing chunk frames = [313856,314368) | 20260507 14:44:41.771 | INFO | MemInfo: { | 20260507 14:44:41.771 | INFO | "gpu" : " 17 GB", | 20260507 14:44:41.771 | INFO | "huge" : "", | 20260507 14:44:41.771 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:41.771 | INFO | "malloc" : " 14 GB", | 20260507 14:44:41.771 | INFO | "pinned" : " 29 GB" | 20260507 14:44:41.771 | INFO | } | 20260507 14:44:41.771 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:44:41.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:41.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:44:41.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:44:41.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:41.771 | INFO | Uploaded 36GiB to GPU in 1.60632s (22.4115GiB/s) | 20260507 14:44:41.771 | INFO | Downloaded 34GiB to GPU in 1.52508s (22.294GiB/s) | 20260507 14:44:46.910 | INFO | Analyzing chunk frames = [314368,314880) | 20260507 14:44:46.910 | INFO | MemInfo: { | 20260507 14:44:46.910 | INFO | "gpu" : " 17 GB", | 20260507 14:44:46.911 | INFO | "huge" : "", | 20260507 14:44:46.911 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:46.911 | INFO | "malloc" : " 14 GB", | 20260507 14:44:46.911 | INFO | "pinned" : " 29 GB" | 20260507 14:44:46.911 | INFO | } | 20260507 14:44:46.911 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:44:46.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:46.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:44:46.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:44:46.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:46.911 | INFO | Uploaded 36GiB to GPU in 1.6067s (22.4062GiB/s) | 20260507 14:44:46.911 | INFO | Downloaded 34GiB to GPU in 1.52587s (22.2824GiB/s) | 20260507 14:44:52.050 | INFO | Analyzing chunk frames = [314880,315392) | 20260507 14:44:52.050 | INFO | MemInfo: { | 20260507 14:44:52.050 | INFO | "gpu" : " 17 GB", | 20260507 14:44:52.050 | INFO | "huge" : "", | 20260507 14:44:52.050 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:52.050 | INFO | "malloc" : " 14 GB", | 20260507 14:44:52.050 | INFO | "pinned" : " 29 GB" | 20260507 14:44:52.050 | INFO | } | 20260507 14:44:52.050 | INFO | Perf report for the last 5.13963 seconds: | 20260507 14:44:52.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:52.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.050 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:44:52.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.050 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 14:44:52.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:52.051 | INFO | Uploaded 36GiB to GPU in 1.60627s (22.4122GiB/s) | 20260507 14:44:52.051 | INFO | Downloaded 34GiB to GPU in 1.52745s (22.2593GiB/s) | 20260507 14:44:52.150 | INFO | Starting a round of Baz IO | 20260507 14:44:55.039 | INFO | Populating RTMetrics report | 20260507 14:44:55.040 | INFO | RTMetrics report populated | 20260507 14:44:55.040 | INFO | Opening tmp RTMetrics json file | 20260507 14:44:55.059 | INFO | Serializing RTMetrics to json | 20260507 14:44:55.083 | INFO | Serialization done, swapping tmp file now... | 20260507 14:44:55.098 | INFO | RTMetrics done | 20260507 14:44:55.177 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.026 seconds | 20260507 14:44:55.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.80% / 0.07% / 43.12% | 20260507 14:44:55.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.31252% / 0.000182788% / 2.38419e-05% | 20260507 14:44:55.177 | INFO | Baz IO overall: | 20260507 14:44:55.177 | INFO | BytesWritten : 116128973890 | 20260507 14:44:55.177 | INFO | NumEvents : 55297422529 | 20260507 14:44:57.189 | INFO | Analyzing chunk frames = [315392,315904) | 20260507 14:44:57.189 | INFO | MemInfo: { | 20260507 14:44:57.189 | INFO | "gpu" : " 17 GB", | 20260507 14:44:57.189 | INFO | "huge" : "", | 20260507 14:44:57.189 | INFO | "huge_shared" : " 29 GB", | 20260507 14:44:57.189 | INFO | "malloc" : " 14 GB", | 20260507 14:44:57.189 | INFO | "pinned" : " 29 GB" | 20260507 14:44:57.189 | INFO | } | 20260507 14:44:57.189 | INFO | Perf report for the last 5.13884 seconds: | 20260507 14:44:57.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:44:57.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:57.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:57.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:57.189 | INFO | ANALYSIS : 57.3%, 2.9, 87.1ms, 0 | 20260507 14:44:57.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:44:57.189 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 14:44:57.189 | INFO | PRE_HQ : 56.1%, 2.1, 59.8ms, 0 | 20260507 14:44:57.189 | INFO | BAZWRITER : 58.5%, 4.5, 136.6ms, 0 | 20260507 14:44:57.189 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 14:44:57.189 | INFO | Uploaded 36GiB to GPU in 1.65599s (21.7393GiB/s) | 20260507 14:44:57.189 | INFO | Downloaded 38GiB to GPU in 1.73618s (21.8872GiB/s) | 20260507 14:45:02.328 | INFO | Analyzing chunk frames = [315904,316416) | 20260507 14:45:02.328 | INFO | MemInfo: { | 20260507 14:45:02.328 | INFO | "gpu" : " 17 GB", | 20260507 14:45:02.329 | INFO | "huge" : "", | 20260507 14:45:02.329 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:02.329 | INFO | "malloc" : " 14 GB", | 20260507 14:45:02.329 | INFO | "pinned" : " 29 GB" | 20260507 14:45:02.329 | INFO | } | 20260507 14:45:02.329 | INFO | Perf report for the last 5.13953 seconds: | 20260507 14:45:02.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:02.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:45:02.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:45:02.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:02.329 | INFO | Uploaded 36GiB to GPU in 1.58407s (22.7263GiB/s) | 20260507 14:45:02.329 | INFO | Downloaded 34GiB to GPU in 1.53248s (22.1863GiB/s) | 20260507 14:45:07.468 | INFO | Analyzing chunk frames = [316416,316928) | 20260507 14:45:07.468 | INFO | MemInfo: { | 20260507 14:45:07.468 | INFO | "gpu" : " 17 GB", | 20260507 14:45:07.468 | INFO | "huge" : "", | 20260507 14:45:07.468 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:07.468 | INFO | "malloc" : " 14 GB", | 20260507 14:45:07.468 | INFO | "pinned" : " 29 GB" | 20260507 14:45:07.468 | INFO | } | 20260507 14:45:07.468 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:45:07.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:07.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:45:07.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:45:07.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:07.468 | INFO | Uploaded 36GiB to GPU in 1.58795s (22.6707GiB/s) | 20260507 14:45:07.468 | INFO | Downloaded 34GiB to GPU in 1.53688s (22.1228GiB/s) | 20260507 14:45:12.607 | INFO | Analyzing chunk frames = [316928,317440) | 20260507 14:45:12.607 | INFO | MemInfo: { | 20260507 14:45:12.607 | INFO | "gpu" : " 17 GB", | 20260507 14:45:12.607 | INFO | "huge" : "", | 20260507 14:45:12.607 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:12.607 | INFO | "malloc" : " 14 GB", | 20260507 14:45:12.607 | INFO | "pinned" : " 29 GB" | 20260507 14:45:12.607 | INFO | } | 20260507 14:45:12.607 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:45:12.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:12.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:45:12.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:45:12.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:12.607 | INFO | Uploaded 36GiB to GPU in 1.58738s (22.6789GiB/s) | 20260507 14:45:12.607 | INFO | Downloaded 34GiB to GPU in 1.53611s (22.1338GiB/s) | 20260507 14:45:17.746 | INFO | Analyzing chunk frames = [317440,317952) | 20260507 14:45:17.746 | INFO | MemInfo: { | 20260507 14:45:17.746 | INFO | "gpu" : " 17 GB", | 20260507 14:45:17.747 | INFO | "huge" : "", | 20260507 14:45:17.747 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:17.747 | INFO | "malloc" : " 14 GB", | 20260507 14:45:17.747 | INFO | "pinned" : " 29 GB" | 20260507 14:45:17.747 | INFO | } | 20260507 14:45:17.747 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:45:17.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:17.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:45:17.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:45:17.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:17.747 | INFO | Uploaded 36GiB to GPU in 1.58662s (22.6897GiB/s) | 20260507 14:45:17.747 | INFO | Downloaded 34GiB to GPU in 1.53518s (22.1472GiB/s) | 20260507 14:45:22.886 | INFO | Analyzing chunk frames = [317952,318464) | 20260507 14:45:22.886 | INFO | MemInfo: { | 20260507 14:45:22.886 | INFO | "gpu" : " 17 GB", | 20260507 14:45:22.886 | INFO | "huge" : "", | 20260507 14:45:22.886 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:22.886 | INFO | "malloc" : " 14 GB", | 20260507 14:45:22.886 | INFO | "pinned" : " 29 GB" | 20260507 14:45:22.886 | INFO | } | 20260507 14:45:22.886 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:45:22.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:22.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.886 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:45:22.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.886 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 14:45:22.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:22.887 | INFO | Uploaded 36GiB to GPU in 1.58584s (22.701GiB/s) | 20260507 14:45:22.887 | INFO | Downloaded 34GiB to GPU in 1.53425s (22.1607GiB/s) | 20260507 14:45:28.025 | INFO | Analyzing chunk frames = [318464,318976) | 20260507 14:45:28.025 | INFO | MemInfo: { | 20260507 14:45:28.025 | INFO | "gpu" : " 17 GB", | 20260507 14:45:28.025 | INFO | "huge" : "", | 20260507 14:45:28.025 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:28.025 | INFO | "malloc" : " 14 GB", | 20260507 14:45:28.025 | INFO | "pinned" : " 29 GB" | 20260507 14:45:28.025 | INFO | } | 20260507 14:45:28.025 | INFO | Perf report for the last 5.1391 seconds: | 20260507 14:45:28.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:28.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.025 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:45:28.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.025 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:45:28.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:28.026 | INFO | Uploaded 36GiB to GPU in 1.58673s (22.6882GiB/s) | 20260507 14:45:28.026 | INFO | Downloaded 34GiB to GPU in 1.53522s (22.1467GiB/s) | 20260507 14:45:33.164 | INFO | Analyzing chunk frames = [318976,319488) | 20260507 14:45:33.164 | INFO | MemInfo: { | 20260507 14:45:33.164 | INFO | "gpu" : " 17 GB", | 20260507 14:45:33.165 | INFO | "huge" : "", | 20260507 14:45:33.165 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:33.165 | INFO | "malloc" : " 14 GB", | 20260507 14:45:33.165 | INFO | "pinned" : " 29 GB" | 20260507 14:45:33.165 | INFO | } | 20260507 14:45:33.165 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:45:33.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:33.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:45:33.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:45:33.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:33.165 | INFO | Uploaded 36GiB to GPU in 1.58756s (22.6763GiB/s) | 20260507 14:45:33.165 | INFO | Downloaded 34GiB to GPU in 1.53615s (22.1333GiB/s) | 20260507 14:45:33.165 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:45:33.165 | INFO | Total mem usage: 15.7 GiB | 20260507 14:45:33.165 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:45:33.165 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -4.77%) | 20260507 14:45:33.165 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:45:33.165 | INFO | High Water Breakdown: | 20260507 14:45:33.165 | INFO | 58.9% : 8.8 GiB ( 94.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:45:33.165 | INFO | 41% : 6.13 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:45:33.166 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:45:33.166 | INFO | ------------------------------------------ | 20260507 14:45:33.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:45:33.166 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:45:33.166 | INFO | Total mem usage: 26.5 GiB | 20260507 14:45:33.166 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:45:33.166 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:45:33.166 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:45:33.166 | INFO | High Water Breakdown: | 20260507 14:45:33.166 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:45:33.166 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:45:33.166 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:45:33.166 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:45:33.166 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:45:33.166 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:45:33.166 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:45:33.166 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:45:33.166 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:45:33.166 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:45:33.166 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:45:33.166 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:45:33.166 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:45:33.166 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:45:33.166 | INFO | ------------------------------------------ | 20260507 14:45:33.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:45:33.166 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:45:33.166 | INFO | Total mem usage: 27.8 GiB | 20260507 14:45:33.166 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:45:33.166 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:45:33.166 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:45:33.166 | INFO | High Water Breakdown: | 20260507 14:45:33.166 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:45:33.166 | INFO | ------------------------------------------ | 20260507 14:45:33.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:45:33.166 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:45:33.166 | INFO | Total mem usage: 15.7 GiB | 20260507 14:45:33.166 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:45:33.166 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:45:33.166 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:45:33.166 | INFO | High Water Breakdown: | 20260507 14:45:33.166 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:45:33.166 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:45:33.166 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:45:33.166 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:45:33.166 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:45:33.166 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:45:33.166 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:45:33.166 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:45:33.166 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:45:33.166 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:45:33.166 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:45:33.166 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:45:33.166 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:45:33.166 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:45:33.166 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:45:33.166 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:45:33.166 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:45:33.167 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:45:33.167 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:45:33.167 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:45:33.167 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:45:33.167 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:45:33.167 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:45:33.167 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:45:33.167 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:45:33.167 | INFO | ------------------------------------------ | 20260507 14:45:33.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:45:33.167 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:45:33.167 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:45:33.167 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:45:33.167 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.68e-03 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 10.99%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 16.11%, DME -- 3.86e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.72e+01 / -nan, 4.39e-02 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.18e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 3.59%, Metrics -- 1.07e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 1.33%, PulseSerializing -- 3.98e-01 / -nan, 5.76e-02 / 0.00e+00 | 20260507 14:45:33.167 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:45:33.167 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:45:33.167 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:45:33.167 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 18.62%, Upload -- 1.67e+01 / 0.00e+00, 4.76e-01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 18.06%, Download -- 1.62e+01 / 0.00e+00, 8.05e-01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 33.39%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 0.00%, LockDownload -- 4.24e-04 / 0.00e+00, 9.67e-05 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 29.35%, LockCompute -- 2.63e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:45:33.167 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260507 14:45:33.261 | INFO | Starting a round of Baz IO | 20260507 14:45:36.155 | INFO | Populating RTMetrics report | 20260507 14:45:36.156 | INFO | RTMetrics report populated | 20260507 14:45:36.156 | INFO | Opening tmp RTMetrics json file | 20260507 14:45:36.157 | INFO | Serializing RTMetrics to json | 20260507 14:45:36.159 | INFO | Serialization done, swapping tmp file now... | 20260507 14:45:36.159 | INFO | RTMetrics done | 20260507 14:45:36.253 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 2.991 seconds | 20260507 14:45:36.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.73% / 0.07% / 43.20% | 20260507 14:45:36.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.27686% / 0.000194709% / 2.78155e-05% | 20260507 14:45:36.253 | INFO | Baz IO overall: | 20260507 14:45:36.253 | INFO | BytesWritten : 117956424712 | 20260507 14:45:36.253 | INFO | NumEvents : 56188633273 | 20260507 14:45:38.304 | INFO | Analyzing chunk frames = [319488,320000) | 20260507 14:45:38.304 | INFO | MemInfo: { | 20260507 14:45:38.304 | INFO | "gpu" : " 17 GB", | 20260507 14:45:38.304 | INFO | "huge" : "", | 20260507 14:45:38.304 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:38.304 | INFO | "malloc" : " 14 GB", | 20260507 14:45:38.304 | INFO | "pinned" : " 29 GB" | 20260507 14:45:38.304 | INFO | } | 20260507 14:45:38.304 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:45:38.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:38.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:38.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:38.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:38.304 | INFO | ANALYSIS : 57.3%, 2.9, 88.0ms, 0 | 20260507 14:45:38.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:38.304 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 14:45:38.304 | INFO | PRE_HQ : 56.0%, 2.1, 59.8ms, 0 | 20260507 14:45:38.304 | INFO | BAZWRITER : 57.8%, 4.3, 126.6ms, 0 | 20260507 14:45:38.304 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 14:45:38.304 | INFO | Uploaded 36GiB to GPU in 1.63973s (21.9548GiB/s) | 20260507 14:45:38.304 | INFO | Downloaded 38GiB to GPU in 1.72163s (22.0721GiB/s) | 20260507 14:45:43.443 | INFO | Analyzing chunk frames = [320000,320512) | 20260507 14:45:43.443 | INFO | MemInfo: { | 20260507 14:45:43.443 | INFO | "gpu" : " 17 GB", | 20260507 14:45:43.443 | INFO | "huge" : "", | 20260507 14:45:43.443 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:43.443 | INFO | "malloc" : " 14 GB", | 20260507 14:45:43.443 | INFO | "pinned" : " 29 GB" | 20260507 14:45:43.443 | INFO | } | 20260507 14:45:43.444 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:45:43.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:43.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:45:43.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:45:43.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:43.444 | INFO | Uploaded 36GiB to GPU in 1.58443s (22.7212GiB/s) | 20260507 14:45:43.444 | INFO | Downloaded 34GiB to GPU in 1.53249s (22.1862GiB/s) | 20260507 14:45:48.583 | INFO | Analyzing chunk frames = [320512,321024) | 20260507 14:45:48.583 | INFO | MemInfo: { | 20260507 14:45:48.583 | INFO | "gpu" : " 17 GB", | 20260507 14:45:48.583 | INFO | "huge" : "", | 20260507 14:45:48.583 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:48.583 | INFO | "malloc" : " 14 GB", | 20260507 14:45:48.583 | INFO | "pinned" : " 29 GB" | 20260507 14:45:48.583 | INFO | } | 20260507 14:45:48.583 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:45:48.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:48.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | ANALYSIS : 56.5%, 3.0, 87.9ms, 0 | 20260507 14:45:48.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:45:48.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:48.583 | INFO | Uploaded 36GiB to GPU in 1.58767s (22.6748GiB/s) | 20260507 14:45:48.583 | INFO | Downloaded 34GiB to GPU in 1.53711s (22.1195GiB/s) | 20260507 14:45:53.722 | INFO | Analyzing chunk frames = [321024,321536) | 20260507 14:45:53.722 | INFO | MemInfo: { | 20260507 14:45:53.722 | INFO | "gpu" : " 17 GB", | 20260507 14:45:53.722 | INFO | "huge" : "", | 20260507 14:45:53.722 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:53.722 | INFO | "malloc" : " 14 GB", | 20260507 14:45:53.722 | INFO | "pinned" : " 29 GB" | 20260507 14:45:53.722 | INFO | } | 20260507 14:45:53.722 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:45:53.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:53.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.722 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:45:53.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.722 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:45:53.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:53.723 | INFO | Uploaded 36GiB to GPU in 1.58893s (22.6568GiB/s) | 20260507 14:45:53.723 | INFO | Downloaded 34GiB to GPU in 1.53706s (22.1201GiB/s) | 20260507 14:45:58.861 | INFO | Analyzing chunk frames = [321536,322048) | 20260507 14:45:58.861 | INFO | MemInfo: { | 20260507 14:45:58.861 | INFO | "gpu" : " 17 GB", | 20260507 14:45:58.861 | INFO | "huge" : "", | 20260507 14:45:58.861 | INFO | "huge_shared" : " 29 GB", | 20260507 14:45:58.861 | INFO | "malloc" : " 14 GB", | 20260507 14:45:58.861 | INFO | "pinned" : " 29 GB" | 20260507 14:45:58.861 | INFO | } | 20260507 14:45:58.862 | INFO | Perf report for the last 5.13924 seconds: | 20260507 14:45:58.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:45:58.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 14:45:58.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:45:58.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:45:58.862 | INFO | Uploaded 36GiB to GPU in 1.58649s (22.6915GiB/s) | 20260507 14:45:58.862 | INFO | Downloaded 34GiB to GPU in 1.53457s (22.1561GiB/s) | 20260507 14:46:04.001 | INFO | Analyzing chunk frames = [322048,322560) | 20260507 14:46:04.001 | INFO | MemInfo: { | 20260507 14:46:04.001 | INFO | "gpu" : " 17 GB", | 20260507 14:46:04.001 | INFO | "huge" : "", | 20260507 14:46:04.001 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:04.001 | INFO | "malloc" : " 14 GB", | 20260507 14:46:04.001 | INFO | "pinned" : " 29 GB" | 20260507 14:46:04.001 | INFO | } | 20260507 14:46:04.001 | INFO | Perf report for the last 5.1393 seconds: | 20260507 14:46:04.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:04.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:46:04.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:46:04.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:04.001 | INFO | Uploaded 36GiB to GPU in 1.58722s (22.6812GiB/s) | 20260507 14:46:04.001 | INFO | Downloaded 34GiB to GPU in 1.53425s (22.1607GiB/s) | 20260507 14:46:09.140 | INFO | Analyzing chunk frames = [322560,323072) | 20260507 14:46:09.140 | INFO | MemInfo: { | 20260507 14:46:09.140 | INFO | "gpu" : " 17 GB", | 20260507 14:46:09.140 | INFO | "huge" : "", | 20260507 14:46:09.140 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:09.140 | INFO | "malloc" : " 14 GB", | 20260507 14:46:09.140 | INFO | "pinned" : " 29 GB" | 20260507 14:46:09.140 | INFO | } | 20260507 14:46:09.140 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:46:09.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:09.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:46:09.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:46:09.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:09.140 | INFO | Uploaded 36GiB to GPU in 1.58593s (22.6997GiB/s) | 20260507 14:46:09.140 | INFO | Downloaded 34GiB to GPU in 1.53552s (22.1424GiB/s) | 20260507 14:46:14.279 | INFO | Analyzing chunk frames = [323072,323584) | 20260507 14:46:14.279 | INFO | MemInfo: { | 20260507 14:46:14.279 | INFO | "gpu" : " 17 GB", | 20260507 14:46:14.279 | INFO | "huge" : "", | 20260507 14:46:14.280 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:14.280 | INFO | "malloc" : " 14 GB", | 20260507 14:46:14.280 | INFO | "pinned" : " 29 GB" | 20260507 14:46:14.280 | INFO | } | 20260507 14:46:14.280 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:46:14.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:14.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:46:14.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:46:14.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:14.280 | INFO | Uploaded 36GiB to GPU in 1.58557s (22.7048GiB/s) | 20260507 14:46:14.280 | INFO | Downloaded 34GiB to GPU in 1.5358s (22.1383GiB/s) | 20260507 14:46:14.376 | INFO | Starting a round of Baz IO | 20260507 14:46:17.268 | INFO | Populating RTMetrics report | 20260507 14:46:17.268 | INFO | RTMetrics report populated | 20260507 14:46:17.268 | INFO | Opening tmp RTMetrics json file | 20260507 14:46:17.342 | INFO | Serializing RTMetrics to json | 20260507 14:46:17.402 | INFO | Serialization done, swapping tmp file now... | 20260507 14:46:17.432 | INFO | RTMetrics done | 20260507 14:46:17.544 | INFO | Finished a round of Baz IO: Wrote 1.704 GiB in 3.168 seconds | 20260507 14:46:17.544 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.66% / 0.07% / 43.27% | 20260507 14:46:17.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.26565% / 0.000158946% / 2.38419e-05% | 20260507 14:46:17.544 | INFO | Baz IO overall: | 20260507 14:46:17.544 | INFO | BytesWritten : 119786231837 | 20260507 14:46:17.544 | INFO | NumEvents : 57080783712 | 20260507 14:46:19.419 | INFO | Analyzing chunk frames = [323584,324096) | 20260507 14:46:19.419 | INFO | MemInfo: { | 20260507 14:46:19.419 | INFO | "gpu" : " 17 GB", | 20260507 14:46:19.419 | INFO | "huge" : "", | 20260507 14:46:19.419 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:19.419 | INFO | "malloc" : " 14 GB", | 20260507 14:46:19.419 | INFO | "pinned" : " 29 GB" | 20260507 14:46:19.419 | INFO | } | 20260507 14:46:19.419 | INFO | Perf report for the last 5.13907 seconds: | 20260507 14:46:19.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:19.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:19.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:19.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:19.419 | INFO | ANALYSIS : 57.3%, 2.9, 86.4ms, 0 | 20260507 14:46:19.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:19.419 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 14:46:19.419 | INFO | PRE_HQ : 56.1%, 2.0, 59.6ms, 0 | 20260507 14:46:19.419 | INFO | BAZWRITER : 61.2%, 4.6, 145.3ms, 0 | 20260507 14:46:19.419 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 14:46:19.419 | INFO | Uploaded 36GiB to GPU in 1.6587s (21.7037GiB/s) | 20260507 14:46:19.419 | INFO | Downloaded 38GiB to GPU in 1.74265s (21.8059GiB/s) | 20260507 14:46:24.558 | INFO | Analyzing chunk frames = [324096,324608) | 20260507 14:46:24.558 | INFO | MemInfo: { | 20260507 14:46:24.558 | INFO | "gpu" : " 17 GB", | 20260507 14:46:24.558 | INFO | "huge" : "", | 20260507 14:46:24.558 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:24.558 | INFO | "malloc" : " 14 GB", | 20260507 14:46:24.558 | INFO | "pinned" : " 29 GB" | 20260507 14:46:24.558 | INFO | } | 20260507 14:46:24.558 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:46:24.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:24.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:46:24.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:46:24.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:24.558 | INFO | Uploaded 36GiB to GPU in 1.58406s (22.7264GiB/s) | 20260507 14:46:24.558 | INFO | Downloaded 34GiB to GPU in 1.53139s (22.202GiB/s) | 20260507 14:46:29.697 | INFO | Analyzing chunk frames = [324608,325120) | 20260507 14:46:29.697 | INFO | MemInfo: { | 20260507 14:46:29.697 | INFO | "gpu" : " 17 GB", | 20260507 14:46:29.697 | INFO | "huge" : "", | 20260507 14:46:29.697 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:29.697 | INFO | "malloc" : " 14 GB", | 20260507 14:46:29.697 | INFO | "pinned" : " 29 GB" | 20260507 14:46:29.697 | INFO | } | 20260507 14:46:29.698 | INFO | Perf report for the last 5.13957 seconds: | 20260507 14:46:29.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:29.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:46:29.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:46:29.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:29.698 | INFO | Uploaded 36GiB to GPU in 1.58768s (22.6745GiB/s) | 20260507 14:46:29.698 | INFO | Downloaded 34GiB to GPU in 1.53669s (22.1255GiB/s) | 20260507 14:46:34.837 | INFO | Analyzing chunk frames = [325120,325632) | 20260507 14:46:34.837 | INFO | MemInfo: { | 20260507 14:46:34.837 | INFO | "gpu" : " 17 GB", | 20260507 14:46:34.837 | INFO | "huge" : "", | 20260507 14:46:34.837 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:34.837 | INFO | "malloc" : " 14 GB", | 20260507 14:46:34.837 | INFO | "pinned" : " 29 GB" | 20260507 14:46:34.837 | INFO | } | 20260507 14:46:34.837 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:46:34.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:34.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:46:34.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:46:34.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:34.837 | INFO | Uploaded 36GiB to GPU in 1.588s (22.67GiB/s) | 20260507 14:46:34.837 | INFO | Downloaded 34GiB to GPU in 1.53587s (22.1373GiB/s) | 20260507 14:46:39.976 | INFO | Analyzing chunk frames = [325632,326144) | 20260507 14:46:39.976 | INFO | MemInfo: { | 20260507 14:46:39.976 | INFO | "gpu" : " 17 GB", | 20260507 14:46:39.976 | INFO | "huge" : "", | 20260507 14:46:39.976 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:39.976 | INFO | "malloc" : " 14 GB", | 20260507 14:46:39.976 | INFO | "pinned" : " 29 GB" | 20260507 14:46:39.976 | INFO | } | 20260507 14:46:39.976 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:46:39.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:39.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.976 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:46:39.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.976 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:46:39.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:39.977 | INFO | Uploaded 36GiB to GPU in 1.58686s (22.6863GiB/s) | 20260507 14:46:39.977 | INFO | Downloaded 34GiB to GPU in 1.53495s (22.1505GiB/s) | 20260507 14:46:45.115 | INFO | Analyzing chunk frames = [326144,326656) | 20260507 14:46:45.115 | INFO | MemInfo: { | 20260507 14:46:45.115 | INFO | "gpu" : " 17 GB", | 20260507 14:46:45.115 | INFO | "huge" : "", | 20260507 14:46:45.115 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:45.115 | INFO | "malloc" : " 14 GB", | 20260507 14:46:45.116 | INFO | "pinned" : " 29 GB" | 20260507 14:46:45.116 | INFO | } | 20260507 14:46:45.116 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:46:45.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:45.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:46:45.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:46:45.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:45.116 | INFO | Uploaded 36GiB to GPU in 1.58663s (22.6896GiB/s) | 20260507 14:46:45.116 | INFO | Downloaded 34GiB to GPU in 1.53406s (22.1633GiB/s) | 20260507 14:46:50.255 | INFO | Analyzing chunk frames = [326656,327168) | 20260507 14:46:50.255 | INFO | MemInfo: { | 20260507 14:46:50.255 | INFO | "gpu" : " 17 GB", | 20260507 14:46:50.255 | INFO | "huge" : "", | 20260507 14:46:50.255 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:50.255 | INFO | "malloc" : " 14 GB", | 20260507 14:46:50.255 | INFO | "pinned" : " 29 GB" | 20260507 14:46:50.255 | INFO | } | 20260507 14:46:50.255 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:46:50.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:50.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:46:50.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:46:50.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:50.255 | INFO | Uploaded 36GiB to GPU in 1.58717s (22.6819GiB/s) | 20260507 14:46:50.255 | INFO | Downloaded 34GiB to GPU in 1.53614s (22.1334GiB/s) | 20260507 14:46:55.394 | INFO | Analyzing chunk frames = [327168,327680) | 20260507 14:46:55.394 | INFO | MemInfo: { | 20260507 14:46:55.394 | INFO | "gpu" : " 17 GB", | 20260507 14:46:55.394 | INFO | "huge" : "", | 20260507 14:46:55.394 | INFO | "huge_shared" : " 29 GB", | 20260507 14:46:55.394 | INFO | "malloc" : " 14 GB", | 20260507 14:46:55.394 | INFO | "pinned" : " 29 GB" | 20260507 14:46:55.394 | INFO | } | 20260507 14:46:55.394 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:46:55.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:46:55.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.394 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:46:55.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.394 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:46:55.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:46:55.395 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:46:55.395 | INFO | Total mem usage: 15.7 GiB | 20260507 14:46:55.395 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:46:55.395 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -2.78%) | 20260507 14:46:55.395 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:46:55.395 | INFO | High Water Breakdown: | 20260507 14:46:55.395 | INFO | 59.2% : 9.02 GiB ( 96.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:46:55.395 | INFO | 40.6% : 6.19 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:46:55.395 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:46:55.395 | INFO | Uploaded 36GiB to GPU in 1.58683s (22.6867GiB/s) | 20260507 14:46:55.395 | INFO | ------------------------------------------ | 20260507 14:46:55.395 | INFO | Downloaded 34GiB to GPU in 1.53584s (22.1377GiB/s) | 20260507 14:46:55.395 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:46:55.395 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:46:55.395 | INFO | Total mem usage: 26.5 GiB | 20260507 14:46:55.395 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:46:55.395 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:46:55.395 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:46:55.395 | INFO | High Water Breakdown: | 20260507 14:46:55.395 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:46:55.395 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:46:55.395 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:46:55.395 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:46:55.395 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:46:55.395 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:46:55.395 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:46:55.395 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:46:55.395 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:46:55.395 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:46:55.395 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:46:55.395 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:46:55.395 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:46:55.396 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:46:55.396 | INFO | ------------------------------------------ | 20260507 14:46:55.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:46:55.396 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:46:55.396 | INFO | Total mem usage: 27.8 GiB | 20260507 14:46:55.396 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:46:55.396 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:46:55.396 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:46:55.396 | INFO | High Water Breakdown: | 20260507 14:46:55.396 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:46:55.396 | INFO | ------------------------------------------ | 20260507 14:46:55.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:46:55.396 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:46:55.396 | INFO | Total mem usage: 15.7 GiB | 20260507 14:46:55.396 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:46:55.396 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:46:55.396 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:46:55.396 | INFO | High Water Breakdown: | 20260507 14:46:55.396 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:46:55.396 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:46:55.396 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:46:55.396 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:46:55.396 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:46:55.396 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:46:55.396 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:46:55.396 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:46:55.396 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:46:55.396 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:46:55.396 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:46:55.396 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:46:55.396 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:46:55.396 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:46:55.396 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:46:55.396 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:46:55.396 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:46:55.396 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:46:55.396 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:46:55.396 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:46:55.396 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:46:55.396 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:46:55.396 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:46:55.396 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:46:55.396 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:46:55.396 | INFO | ------------------------------------------ | 20260507 14:46:55.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:46:55.396 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:46:55.396 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:46:55.396 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:46:55.396 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.20e-03 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 11.00%, Baseline -- 3.30e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / -nan, 5.14e-02 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 16.12%, DME -- 3.87e+01 / -nan, 3.12e+00 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.18e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 3.58%, Metrics -- 1.07e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 1.33%, PulseSerializing -- 3.98e-01 / -nan, 5.81e-02 / 0.00e+00 | 20260507 14:46:55.396 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:46:55.396 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:46:55.396 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:46:55.396 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 18.50%, Upload -- 1.66e+01 / 0.00e+00, 4.80e-01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 18.10%, Download -- 1.62e+01 / 0.00e+00, 7.82e-01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 33.39%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 0.00%, LockDownload -- 4.46e-04 / 0.00e+00, 2.12e-04 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 29.43%, LockCompute -- 2.64e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:46:55.396 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260507 14:46:55.491 | INFO | Starting a round of Baz IO | 20260507 14:46:58.382 | INFO | Populating RTMetrics report | 20260507 14:46:58.383 | INFO | RTMetrics report populated | 20260507 14:46:58.383 | INFO | Opening tmp RTMetrics json file | 20260507 14:46:58.386 | INFO | Serializing RTMetrics to json | 20260507 14:46:58.391 | INFO | Serialization done, swapping tmp file now... | 20260507 14:46:58.398 | INFO | RTMetrics done | 20260507 14:46:58.481 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 2.989 seconds | 20260507 14:46:58.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.59% / 0.07% / 43.34% | 20260507 14:46:58.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.20912% / 0.000194709% / 2.78155e-05% | 20260507 14:46:58.481 | INFO | Baz IO overall: | 20260507 14:46:58.481 | INFO | BytesWritten : 121614261680 | 20260507 14:46:58.482 | INFO | NumEvents : 57971451463 | 20260507 14:47:00.533 | INFO | Analyzing chunk frames = [327680,328192) | 20260507 14:47:00.533 | INFO | MemInfo: { | 20260507 14:47:00.533 | INFO | "gpu" : " 17 GB", | 20260507 14:47:00.533 | INFO | "huge" : "", | 20260507 14:47:00.533 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:00.533 | INFO | "malloc" : " 14 GB", | 20260507 14:47:00.533 | INFO | "pinned" : " 29 GB" | 20260507 14:47:00.533 | INFO | } | 20260507 14:47:00.534 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:47:00.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:00.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:00.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:00.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:00.534 | INFO | ANALYSIS : 57.3%, 2.9, 86.8ms, 0 | 20260507 14:47:00.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:00.534 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 14:47:00.534 | INFO | PRE_HQ : 56.0%, 2.1, 59.6ms, 0 | 20260507 14:47:00.534 | INFO | BAZWRITER : 57.7%, 4.6, 135.1ms, 0 | 20260507 14:47:00.534 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:47:00.534 | INFO | Uploaded 36GiB to GPU in 1.65178s (21.7946GiB/s) | 20260507 14:47:00.534 | INFO | Downloaded 38GiB to GPU in 1.73418s (21.9123GiB/s) | 20260507 14:47:05.673 | INFO | Analyzing chunk frames = [328192,328704) | 20260507 14:47:05.673 | INFO | MemInfo: { | 20260507 14:47:05.673 | INFO | "gpu" : " 17 GB", | 20260507 14:47:05.673 | INFO | "huge" : "", | 20260507 14:47:05.673 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:05.673 | INFO | "malloc" : " 14 GB", | 20260507 14:47:05.673 | INFO | "pinned" : " 29 GB" | 20260507 14:47:05.673 | INFO | } | 20260507 14:47:05.673 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:47:05.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:05.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:47:05.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:47:05.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:05.673 | INFO | Uploaded 36GiB to GPU in 1.58429s (22.7231GiB/s) | 20260507 14:47:05.673 | INFO | Downloaded 34GiB to GPU in 1.53154s (22.1999GiB/s) | 20260507 14:47:10.812 | INFO | Analyzing chunk frames = [328704,329216) | 20260507 14:47:10.812 | INFO | MemInfo: { | 20260507 14:47:10.812 | INFO | "gpu" : " 17 GB", | 20260507 14:47:10.812 | INFO | "huge" : "", | 20260507 14:47:10.812 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:10.812 | INFO | "malloc" : " 14 GB", | 20260507 14:47:10.812 | INFO | "pinned" : " 29 GB" | 20260507 14:47:10.812 | INFO | } | 20260507 14:47:10.812 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:47:10.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:10.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.812 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:47:10.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.812 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:47:10.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:10.813 | INFO | Uploaded 36GiB to GPU in 1.58855s (22.6622GiB/s) | 20260507 14:47:10.813 | INFO | Downloaded 34GiB to GPU in 1.53782s (22.1092GiB/s) | 20260507 14:47:15.951 | INFO | Analyzing chunk frames = [329216,329728) | 20260507 14:47:15.951 | INFO | MemInfo: { | 20260507 14:47:15.951 | INFO | "gpu" : " 17 GB", | 20260507 14:47:15.951 | INFO | "huge" : "", | 20260507 14:47:15.951 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:15.951 | INFO | "malloc" : " 14 GB", | 20260507 14:47:15.951 | INFO | "pinned" : " 29 GB" | 20260507 14:47:15.951 | INFO | } | 20260507 14:47:15.952 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:47:15.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:15.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:47:15.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:47:15.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:15.952 | INFO | Uploaded 36GiB to GPU in 1.58823s (22.6668GiB/s) | 20260507 14:47:15.952 | INFO | Downloaded 34GiB to GPU in 1.53567s (22.1402GiB/s) | 20260507 14:47:21.091 | INFO | Analyzing chunk frames = [329728,330240) | 20260507 14:47:21.091 | INFO | MemInfo: { | 20260507 14:47:21.091 | INFO | "gpu" : " 17 GB", | 20260507 14:47:21.091 | INFO | "huge" : "", | 20260507 14:47:21.091 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:21.091 | INFO | "malloc" : " 14 GB", | 20260507 14:47:21.091 | INFO | "pinned" : " 29 GB" | 20260507 14:47:21.091 | INFO | } | 20260507 14:47:21.091 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:47:21.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:21.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:47:21.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:47:21.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:21.091 | INFO | Uploaded 36GiB to GPU in 1.58711s (22.6827GiB/s) | 20260507 14:47:21.091 | INFO | Downloaded 34GiB to GPU in 1.53503s (22.1495GiB/s) | 20260507 14:47:26.230 | INFO | Analyzing chunk frames = [330240,330752) | 20260507 14:47:26.230 | INFO | MemInfo: { | 20260507 14:47:26.230 | INFO | "gpu" : " 17 GB", | 20260507 14:47:26.230 | INFO | "huge" : "", | 20260507 14:47:26.230 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:26.230 | INFO | "malloc" : " 14 GB", | 20260507 14:47:26.230 | INFO | "pinned" : " 29 GB" | 20260507 14:47:26.230 | INFO | } | 20260507 14:47:26.230 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:47:26.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:26.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.230 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:47:26.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.230 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:47:26.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:26.231 | INFO | Uploaded 36GiB to GPU in 1.58713s (22.6824GiB/s) | 20260507 14:47:26.231 | INFO | Downloaded 34GiB to GPU in 1.53401s (22.1642GiB/s) | 20260507 14:47:31.369 | INFO | Analyzing chunk frames = [330752,331264) | 20260507 14:47:31.369 | INFO | MemInfo: { | 20260507 14:47:31.369 | INFO | "gpu" : " 17 GB", | 20260507 14:47:31.369 | INFO | "huge" : "", | 20260507 14:47:31.369 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:31.369 | INFO | "malloc" : " 14 GB", | 20260507 14:47:31.370 | INFO | "pinned" : " 29 GB" | 20260507 14:47:31.370 | INFO | } | 20260507 14:47:31.370 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:47:31.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:31.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:47:31.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:47:31.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:31.370 | INFO | Uploaded 36GiB to GPU in 1.58835s (22.6651GiB/s) | 20260507 14:47:31.370 | INFO | Downloaded 34GiB to GPU in 1.53644s (22.129GiB/s) | 20260507 14:47:36.509 | INFO | Analyzing chunk frames = [331264,331776) | 20260507 14:47:36.509 | INFO | MemInfo: { | 20260507 14:47:36.509 | INFO | "gpu" : " 17 GB", | 20260507 14:47:36.509 | INFO | "huge" : "", | 20260507 14:47:36.509 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:36.509 | INFO | "malloc" : " 14 GB", | 20260507 14:47:36.509 | INFO | "pinned" : " 29 GB" | 20260507 14:47:36.509 | INFO | } | 20260507 14:47:36.509 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:47:36.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:36.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.509 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:47:36.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.509 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 14:47:36.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:36.510 | INFO | Uploaded 36GiB to GPU in 1.58642s (22.6927GiB/s) | 20260507 14:47:36.510 | INFO | Downloaded 34GiB to GPU in 1.53654s (22.1276GiB/s) | 20260507 14:47:36.606 | INFO | Starting a round of Baz IO | 20260507 14:47:39.497 | INFO | Populating RTMetrics report | 20260507 14:47:39.497 | INFO | RTMetrics report populated | 20260507 14:47:39.497 | INFO | Opening tmp RTMetrics json file | 20260507 14:47:39.498 | INFO | Serializing RTMetrics to json | 20260507 14:47:39.500 | INFO | Serialization done, swapping tmp file now... | 20260507 14:47:39.500 | INFO | RTMetrics done | 20260507 14:47:39.600 | INFO | Finished a round of Baz IO: Wrote 1.704 GiB in 2.993 seconds | 20260507 14:47:39.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.52% / 0.07% / 43.41% | 20260507 14:47:39.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.17792% / 0.000190735% / 2.78155e-05% | 20260507 14:47:39.600 | INFO | Baz IO overall: | 20260507 14:47:39.600 | INFO | BytesWritten : 123443574183 | 20260507 14:47:39.600 | INFO | NumEvents : 58862955533 | 20260507 14:47:41.648 | INFO | Analyzing chunk frames = [331776,332288) | 20260507 14:47:41.648 | INFO | MemInfo: { | 20260507 14:47:41.648 | INFO | "gpu" : " 17 GB", | 20260507 14:47:41.648 | INFO | "huge" : "", | 20260507 14:47:41.648 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:41.648 | INFO | "malloc" : " 14 GB", | 20260507 14:47:41.648 | INFO | "pinned" : " 29 GB" | 20260507 14:47:41.648 | INFO | } | 20260507 14:47:41.648 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:47:41.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:41.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:41.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:41.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:41.648 | INFO | ANALYSIS : 57.2%, 3.0, 88.6ms, 0 | 20260507 14:47:41.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:41.648 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 14:47:41.648 | INFO | PRE_HQ : 55.9%, 2.1, 59.7ms, 0 | 20260507 14:47:41.648 | INFO | BAZWRITER : 57.7%, 4.2, 126.0ms, 0 | 20260507 14:47:41.649 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 14:47:41.649 | INFO | Uploaded 36GiB to GPU in 1.6314s (22.067GiB/s) | 20260507 14:47:41.649 | INFO | Downloaded 38GiB to GPU in 1.71313s (22.1816GiB/s) | 20260507 14:47:46.787 | INFO | Analyzing chunk frames = [332288,332800) | 20260507 14:47:46.787 | INFO | MemInfo: { | 20260507 14:47:46.787 | INFO | "gpu" : " 17 GB", | 20260507 14:47:46.787 | INFO | "huge" : "", | 20260507 14:47:46.787 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:46.787 | INFO | "malloc" : " 14 GB", | 20260507 14:47:46.788 | INFO | "pinned" : " 29 GB" | 20260507 14:47:46.788 | INFO | } | 20260507 14:47:46.788 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:47:46.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:46.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:47:46.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:47:46.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:46.788 | INFO | Uploaded 36GiB to GPU in 1.58463s (22.7183GiB/s) | 20260507 14:47:46.788 | INFO | Downloaded 34GiB to GPU in 1.53219s (22.1904GiB/s) | 20260507 14:47:51.927 | INFO | Analyzing chunk frames = [332800,333312) | 20260507 14:47:51.927 | INFO | MemInfo: { | 20260507 14:47:51.927 | INFO | "gpu" : " 17 GB", | 20260507 14:47:51.927 | INFO | "huge" : "", | 20260507 14:47:51.927 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:51.927 | INFO | "malloc" : " 14 GB", | 20260507 14:47:51.927 | INFO | "pinned" : " 29 GB" | 20260507 14:47:51.927 | INFO | } | 20260507 14:47:51.927 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:47:51.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:51.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:47:51.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:47:51.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:51.927 | INFO | Uploaded 36GiB to GPU in 1.58829s (22.6658GiB/s) | 20260507 14:47:51.927 | INFO | Downloaded 34GiB to GPU in 1.53724s (22.1176GiB/s) | 20260507 14:47:57.066 | INFO | Analyzing chunk frames = [333312,333824) | 20260507 14:47:57.066 | INFO | MemInfo: { | 20260507 14:47:57.066 | INFO | "gpu" : " 17 GB", | 20260507 14:47:57.066 | INFO | "huge" : "", | 20260507 14:47:57.066 | INFO | "huge_shared" : " 29 GB", | 20260507 14:47:57.066 | INFO | "malloc" : " 14 GB", | 20260507 14:47:57.066 | INFO | "pinned" : " 29 GB" | 20260507 14:47:57.066 | INFO | } | 20260507 14:47:57.066 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:47:57.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:47:57.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.066 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 14:47:57.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.067 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:47:57.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:47:57.067 | INFO | Uploaded 36GiB to GPU in 1.58825s (22.6664GiB/s) | 20260507 14:47:57.067 | INFO | Downloaded 34GiB to GPU in 1.53561s (22.141GiB/s) | 20260507 14:48:02.206 | INFO | Analyzing chunk frames = [333824,334336) | 20260507 14:48:02.206 | INFO | MemInfo: { | 20260507 14:48:02.206 | INFO | "gpu" : " 17 GB", | 20260507 14:48:02.206 | INFO | "huge" : "", | 20260507 14:48:02.206 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:02.206 | INFO | "malloc" : " 14 GB", | 20260507 14:48:02.206 | INFO | "pinned" : " 29 GB" | 20260507 14:48:02.206 | INFO | } | 20260507 14:48:02.206 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:48:02.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:02.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 14:48:02.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:48:02.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:02.206 | INFO | Uploaded 36GiB to GPU in 1.58547s (22.7061GiB/s) | 20260507 14:48:02.206 | INFO | Downloaded 34GiB to GPU in 1.53574s (22.1391GiB/s) | 20260507 14:48:07.345 | INFO | Analyzing chunk frames = [334336,334848) | 20260507 14:48:07.345 | INFO | MemInfo: { | 20260507 14:48:07.345 | INFO | "gpu" : " 17 GB", | 20260507 14:48:07.345 | INFO | "huge" : "", | 20260507 14:48:07.345 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:07.345 | INFO | "malloc" : " 14 GB", | 20260507 14:48:07.345 | INFO | "pinned" : " 29 GB" | 20260507 14:48:07.345 | INFO | } | 20260507 14:48:07.345 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:48:07.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:07.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.345 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:48:07.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.345 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:48:07.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:07.346 | INFO | Uploaded 36GiB to GPU in 1.5866s (22.69GiB/s) | 20260507 14:48:07.346 | INFO | Downloaded 34GiB to GPU in 1.53515s (22.1477GiB/s) | 20260507 14:48:12.484 | INFO | Analyzing chunk frames = [334848,335360) | 20260507 14:48:12.484 | INFO | MemInfo: { | 20260507 14:48:12.484 | INFO | "gpu" : " 17 GB", | 20260507 14:48:12.484 | INFO | "huge" : "", | 20260507 14:48:12.484 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:12.484 | INFO | "malloc" : " 14 GB", | 20260507 14:48:12.484 | INFO | "pinned" : " 29 GB" | 20260507 14:48:12.484 | INFO | } | 20260507 14:48:12.484 | INFO | Perf report for the last 5.13891 seconds: | 20260507 14:48:12.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:12.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.484 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:48:12.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.484 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:48:12.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:12.485 | INFO | Uploaded 36GiB to GPU in 1.58697s (22.6848GiB/s) | 20260507 14:48:12.485 | INFO | Downloaded 34GiB to GPU in 1.53562s (22.1408GiB/s) | 20260507 14:48:17.624 | INFO | Analyzing chunk frames = [335360,335872) | 20260507 14:48:17.624 | INFO | MemInfo: { | 20260507 14:48:17.624 | INFO | "gpu" : " 17 GB", | 20260507 14:48:17.624 | INFO | "huge" : "", | 20260507 14:48:17.624 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:17.624 | INFO | "malloc" : " 14 GB", | 20260507 14:48:17.624 | INFO | "pinned" : " 29 GB" | 20260507 14:48:17.624 | INFO | } | 20260507 14:48:17.624 | INFO | Perf report for the last 5.13969 seconds: | 20260507 14:48:17.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:17.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:48:17.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:48:17.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:17.624 | INFO | Uploaded 36GiB to GPU in 1.58756s (22.6763GiB/s) | 20260507 14:48:17.624 | INFO | Downloaded 34GiB to GPU in 1.53645s (22.1289GiB/s) | 20260507 14:48:17.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:48:17.624 | INFO | Total mem usage: 15.7 GiB | 20260507 14:48:17.624 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:48:17.624 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -5.07%) | 20260507 14:48:17.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:48:17.624 | INFO | High Water Breakdown: | 20260507 14:48:17.624 | INFO | 58.8% : 8.77 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:48:17.624 | INFO | 41% : 6.11 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:48:17.624 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:48:17.624 | INFO | ------------------------------------------ | 20260507 14:48:17.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:48:17.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:48:17.624 | INFO | Total mem usage: 26.5 GiB | 20260507 14:48:17.624 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:48:17.624 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:48:17.625 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:48:17.625 | INFO | High Water Breakdown: | 20260507 14:48:17.625 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:48:17.625 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:48:17.625 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:48:17.625 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:48:17.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:48:17.625 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:48:17.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:48:17.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:48:17.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:48:17.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:48:17.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:48:17.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:48:17.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:48:17.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:48:17.625 | INFO | ------------------------------------------ | 20260507 14:48:17.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:48:17.625 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:48:17.625 | INFO | Total mem usage: 27.8 GiB | 20260507 14:48:17.625 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:48:17.625 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:48:17.625 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:48:17.625 | INFO | High Water Breakdown: | 20260507 14:48:17.625 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:48:17.625 | INFO | ------------------------------------------ | 20260507 14:48:17.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:48:17.625 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:48:17.625 | INFO | Total mem usage: 15.7 GiB | 20260507 14:48:17.625 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:48:17.625 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:48:17.625 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:48:17.625 | INFO | High Water Breakdown: | 20260507 14:48:17.625 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:48:17.625 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:48:17.625 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:48:17.625 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:48:17.625 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:48:17.625 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:48:17.625 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:48:17.625 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:48:17.625 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:48:17.625 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:48:17.625 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:48:17.625 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:48:17.625 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:48:17.625 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:48:17.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:48:17.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:48:17.625 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:48:17.625 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:48:17.625 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:48:17.625 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:48:17.625 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:48:17.625 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:48:17.625 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:48:17.625 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:48:17.625 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:48:17.625 | INFO | ------------------------------------------ | 20260507 14:48:17.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:48:17.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:48:17.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:48:17.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:48:17.625 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:48:17.625 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.13e-03 / 0.00e+00 | 20260507 14:48:17.625 | INFO | PerfReport: 11.00%, Baseline -- 3.29e+00 / -nan, 1.71e-02 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.18e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 3.58%, Metrics -- 1.07e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 1.32%, PulseSerializing -- 3.97e-01 / -nan, 5.79e-02 / 0.00e+00 | 20260507 14:48:17.626 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:48:17.626 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:48:17.626 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:48:17.626 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 18.49%, Upload -- 1.66e+01 / 0.00e+00, 4.54e-01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 18.08%, Download -- 1.62e+01 / 0.00e+00, 7.43e-01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 33.36%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 0.00%, LockDownload -- 4.42e-04 / 0.00e+00, 1.90e-04 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 29.49%, LockCompute -- 2.65e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:48:17.626 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260507 14:48:17.723 | INFO | Starting a round of Baz IO | 20260507 14:48:20.614 | INFO | Populating RTMetrics report | 20260507 14:48:20.614 | INFO | RTMetrics report populated | 20260507 14:48:20.614 | INFO | Opening tmp RTMetrics json file | 20260507 14:48:20.665 | INFO | Serializing RTMetrics to json | 20260507 14:48:20.713 | INFO | Serialization done, swapping tmp file now... | 20260507 14:48:20.758 | INFO | RTMetrics done | 20260507 14:48:20.812 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.088 seconds | 20260507 14:48:20.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.45% / 0.07% / 43.48% | 20260507 14:48:20.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.16355% / 0.000135104% / 2.78155e-05% | 20260507 14:48:20.812 | INFO | Baz IO overall: | 20260507 14:48:20.812 | INFO | BytesWritten : 125275164381 | 20260507 14:48:20.812 | INFO | NumEvents : 59756204501 | 20260507 14:48:22.763 | INFO | Analyzing chunk frames = [335872,336384) | 20260507 14:48:22.763 | INFO | MemInfo: { | 20260507 14:48:22.763 | INFO | "gpu" : " 17 GB", | 20260507 14:48:22.763 | INFO | "huge" : "", | 20260507 14:48:22.763 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:22.763 | INFO | "malloc" : " 14 GB", | 20260507 14:48:22.763 | INFO | "pinned" : " 29 GB" | 20260507 14:48:22.763 | INFO | } | 20260507 14:48:22.763 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:48:22.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:22.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:22.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:22.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:22.763 | INFO | ANALYSIS : 57.3%, 2.9, 88.2ms, 0 | 20260507 14:48:22.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:22.763 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 14:48:22.763 | INFO | PRE_HQ : 55.9%, 2.1, 59.7ms, 0 | 20260507 14:48:22.763 | INFO | BAZWRITER : 59.7%, 4.4, 133.6ms, 0 | 20260507 14:48:22.763 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260507 14:48:22.763 | INFO | Uploaded 36GiB to GPU in 1.63656s (21.9973GiB/s) | 20260507 14:48:22.763 | INFO | Downloaded 38GiB to GPU in 1.71775s (22.122GiB/s) | 20260507 14:48:27.902 | INFO | Analyzing chunk frames = [336384,336896) | 20260507 14:48:27.902 | INFO | MemInfo: { | 20260507 14:48:27.902 | INFO | "gpu" : " 17 GB", | 20260507 14:48:27.902 | INFO | "huge" : "", | 20260507 14:48:27.902 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:27.902 | INFO | "malloc" : " 14 GB", | 20260507 14:48:27.902 | INFO | "pinned" : " 29 GB" | 20260507 14:48:27.902 | INFO | } | 20260507 14:48:27.903 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:48:27.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:27.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:48:27.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:48:27.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:27.903 | INFO | Uploaded 36GiB to GPU in 1.58474s (22.7167GiB/s) | 20260507 14:48:27.903 | INFO | Downloaded 34GiB to GPU in 1.53291s (22.18GiB/s) | 20260507 14:48:33.042 | INFO | Analyzing chunk frames = [336896,337408) | 20260507 14:48:33.042 | INFO | MemInfo: { | 20260507 14:48:33.042 | INFO | "gpu" : " 17 GB", | 20260507 14:48:33.042 | INFO | "huge" : "", | 20260507 14:48:33.042 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:33.042 | INFO | "malloc" : " 14 GB", | 20260507 14:48:33.042 | INFO | "pinned" : " 29 GB" | 20260507 14:48:33.042 | INFO | } | 20260507 14:48:33.042 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:48:33.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:33.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:48:33.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:48:33.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:33.042 | INFO | Uploaded 36GiB to GPU in 1.58781s (22.6727GiB/s) | 20260507 14:48:33.042 | INFO | Downloaded 34GiB to GPU in 1.53736s (22.1159GiB/s) | 20260507 14:48:38.181 | INFO | Analyzing chunk frames = [337408,337920) | 20260507 14:48:38.181 | INFO | MemInfo: { | 20260507 14:48:38.181 | INFO | "gpu" : " 17 GB", | 20260507 14:48:38.181 | INFO | "huge" : "", | 20260507 14:48:38.181 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:38.181 | INFO | "malloc" : " 14 GB", | 20260507 14:48:38.181 | INFO | "pinned" : " 29 GB" | 20260507 14:48:38.181 | INFO | } | 20260507 14:48:38.181 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:48:38.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:38.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.181 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 14:48:38.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.181 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:48:38.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:38.182 | INFO | Uploaded 36GiB to GPU in 1.58925s (22.6523GiB/s) | 20260507 14:48:38.182 | INFO | Downloaded 34GiB to GPU in 1.53642s (22.1294GiB/s) | 20260507 14:48:43.320 | INFO | Analyzing chunk frames = [337920,338432) | 20260507 14:48:43.320 | INFO | MemInfo: { | 20260507 14:48:43.320 | INFO | "gpu" : " 17 GB", | 20260507 14:48:43.320 | INFO | "huge" : "", | 20260507 14:48:43.320 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:43.320 | INFO | "malloc" : " 14 GB", | 20260507 14:48:43.320 | INFO | "pinned" : " 29 GB" | 20260507 14:48:43.321 | INFO | } | 20260507 14:48:43.321 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:48:43.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:43.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:48:43.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:48:43.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:43.321 | INFO | Uploaded 36GiB to GPU in 1.58678s (22.6874GiB/s) | 20260507 14:48:43.321 | INFO | Downloaded 34GiB to GPU in 1.53477s (22.1531GiB/s) | 20260507 14:48:48.460 | INFO | Analyzing chunk frames = [338432,338944) | 20260507 14:48:48.460 | INFO | MemInfo: { | 20260507 14:48:48.460 | INFO | "gpu" : " 17 GB", | 20260507 14:48:48.460 | INFO | "huge" : "", | 20260507 14:48:48.460 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:48.460 | INFO | "malloc" : " 14 GB", | 20260507 14:48:48.460 | INFO | "pinned" : " 29 GB" | 20260507 14:48:48.460 | INFO | } | 20260507 14:48:48.460 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:48:48.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:48.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 14:48:48.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:48:48.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:48.460 | INFO | Uploaded 36GiB to GPU in 1.58716s (22.682GiB/s) | 20260507 14:48:48.460 | INFO | Downloaded 34GiB to GPU in 1.53497s (22.1503GiB/s) | 20260507 14:48:53.599 | INFO | Analyzing chunk frames = [338944,339456) | 20260507 14:48:53.599 | INFO | MemInfo: { | 20260507 14:48:53.599 | INFO | "gpu" : " 17 GB", | 20260507 14:48:53.599 | INFO | "huge" : "", | 20260507 14:48:53.599 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:53.599 | INFO | "malloc" : " 14 GB", | 20260507 14:48:53.599 | INFO | "pinned" : " 29 GB" | 20260507 14:48:53.599 | INFO | } | 20260507 14:48:53.599 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:48:53.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:53.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.599 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:48:53.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.599 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:48:53.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:53.600 | INFO | Uploaded 36GiB to GPU in 1.58713s (22.6825GiB/s) | 20260507 14:48:53.600 | INFO | Downloaded 34GiB to GPU in 1.53572s (22.1394GiB/s) | 20260507 14:48:58.738 | INFO | Analyzing chunk frames = [339456,339968) | 20260507 14:48:58.738 | INFO | MemInfo: { | 20260507 14:48:58.738 | INFO | "gpu" : " 17 GB", | 20260507 14:48:58.738 | INFO | "huge" : "", | 20260507 14:48:58.739 | INFO | "huge_shared" : " 29 GB", | 20260507 14:48:58.739 | INFO | "malloc" : " 14 GB", | 20260507 14:48:58.739 | INFO | "pinned" : " 29 GB" | 20260507 14:48:58.739 | INFO | } | 20260507 14:48:58.739 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:48:58.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:48:58.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:48:58.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:48:58.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:48:58.739 | INFO | Uploaded 36GiB to GPU in 1.58754s (22.6765GiB/s) | 20260507 14:48:58.739 | INFO | Downloaded 34GiB to GPU in 1.53658s (22.127GiB/s) | 20260507 14:48:58.836 | INFO | Starting a round of Baz IO | 20260507 14:49:01.731 | INFO | Populating RTMetrics report | 20260507 14:49:01.732 | INFO | RTMetrics report populated | 20260507 14:49:01.732 | INFO | Opening tmp RTMetrics json file | 20260507 14:49:01.791 | INFO | Serializing RTMetrics to json | 20260507 14:49:01.850 | INFO | Serialization done, swapping tmp file now... | 20260507 14:49:01.868 | INFO | RTMetrics done | 20260507 14:49:01.902 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.066 seconds | 20260507 14:49:01.902 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.38% / 0.07% / 43.55% | 20260507 14:49:01.902 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.12403% / 0.000210603% / 1.98682e-05% | 20260507 14:49:01.902 | INFO | Baz IO overall: | 20260507 14:49:01.902 | INFO | BytesWritten : 127106850037 | 20260507 14:49:01.902 | INFO | NumEvents : 60649310196 | 20260507 14:49:03.878 | INFO | Analyzing chunk frames = [339968,340480) | 20260507 14:49:03.878 | INFO | MemInfo: { | 20260507 14:49:03.878 | INFO | "gpu" : " 17 GB", | 20260507 14:49:03.878 | INFO | "huge" : "", | 20260507 14:49:03.878 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:03.878 | INFO | "malloc" : " 14 GB", | 20260507 14:49:03.878 | INFO | "pinned" : " 29 GB" | 20260507 14:49:03.878 | INFO | } | 20260507 14:49:03.878 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:49:03.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:03.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:03.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:03.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:03.878 | INFO | ANALYSIS : 57.3%, 2.9, 87.2ms, 0 | 20260507 14:49:03.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:03.878 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260507 14:49:03.878 | INFO | PRE_HQ : 56.1%, 2.1, 59.8ms, 0 | 20260507 14:49:03.878 | INFO | BAZWRITER : 59.2%, 4.6, 141.1ms, 0 | 20260507 14:49:03.878 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 14:49:03.878 | INFO | Uploaded 36GiB to GPU in 1.65505s (21.7515GiB/s) | 20260507 14:49:03.879 | INFO | Downloaded 38GiB to GPU in 1.73748s (21.8708GiB/s) | 20260507 14:49:09.017 | INFO | Analyzing chunk frames = [340480,340992) | 20260507 14:49:09.017 | INFO | MemInfo: { | 20260507 14:49:09.017 | INFO | "gpu" : " 17 GB", | 20260507 14:49:09.017 | INFO | "huge" : "", | 20260507 14:49:09.017 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:09.017 | INFO | "malloc" : " 14 GB", | 20260507 14:49:09.017 | INFO | "pinned" : " 29 GB" | 20260507 14:49:09.017 | INFO | } | 20260507 14:49:09.017 | INFO | Perf report for the last 5.13907 seconds: | 20260507 14:49:09.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:09.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.017 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:49:09.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.017 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:49:09.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:09.018 | INFO | Uploaded 36GiB to GPU in 1.58441s (22.7214GiB/s) | 20260507 14:49:09.018 | INFO | Downloaded 34GiB to GPU in 1.53258s (22.1848GiB/s) | 20260507 14:49:14.157 | INFO | Analyzing chunk frames = [340992,341504) | 20260507 14:49:14.157 | INFO | MemInfo: { | 20260507 14:49:14.157 | INFO | "gpu" : " 17 GB", | 20260507 14:49:14.157 | INFO | "huge" : "", | 20260507 14:49:14.157 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:14.157 | INFO | "malloc" : " 14 GB", | 20260507 14:49:14.157 | INFO | "pinned" : " 29 GB" | 20260507 14:49:14.157 | INFO | } | 20260507 14:49:14.157 | INFO | Perf report for the last 5.13965 seconds: | 20260507 14:49:14.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:14.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:49:14.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:49:14.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:14.157 | INFO | Uploaded 36GiB to GPU in 1.58763s (22.6752GiB/s) | 20260507 14:49:14.157 | INFO | Downloaded 34GiB to GPU in 1.53697s (22.1214GiB/s) | 20260507 14:49:19.296 | INFO | Analyzing chunk frames = [341504,342016) | 20260507 14:49:19.296 | INFO | MemInfo: { | 20260507 14:49:19.296 | INFO | "gpu" : " 17 GB", | 20260507 14:49:19.296 | INFO | "huge" : "", | 20260507 14:49:19.296 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:19.296 | INFO | "malloc" : " 14 GB", | 20260507 14:49:19.296 | INFO | "pinned" : " 29 GB" | 20260507 14:49:19.296 | INFO | } | 20260507 14:49:19.296 | INFO | Perf report for the last 5.13906 seconds: | 20260507 14:49:19.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:19.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:49:19.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:49:19.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:19.296 | INFO | Uploaded 36GiB to GPU in 1.58837s (22.6647GiB/s) | 20260507 14:49:19.296 | INFO | Downloaded 34GiB to GPU in 1.53707s (22.12GiB/s) | 20260507 14:49:24.435 | INFO | Analyzing chunk frames = [342016,342528) | 20260507 14:49:24.435 | INFO | MemInfo: { | 20260507 14:49:24.435 | INFO | "gpu" : " 17 GB", | 20260507 14:49:24.435 | INFO | "huge" : "", | 20260507 14:49:24.435 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:24.435 | INFO | "malloc" : " 14 GB", | 20260507 14:49:24.435 | INFO | "pinned" : " 29 GB" | 20260507 14:49:24.435 | INFO | } | 20260507 14:49:24.435 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:49:24.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:24.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.435 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 14:49:24.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.435 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:49:24.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:24.436 | INFO | Uploaded 36GiB to GPU in 1.58601s (22.6985GiB/s) | 20260507 14:49:24.436 | INFO | Downloaded 34GiB to GPU in 1.53477s (22.1532GiB/s) | 20260507 14:49:29.574 | INFO | Analyzing chunk frames = [342528,343040) | 20260507 14:49:29.574 | INFO | MemInfo: { | 20260507 14:49:29.574 | INFO | "gpu" : " 17 GB", | 20260507 14:49:29.574 | INFO | "huge" : "", | 20260507 14:49:29.574 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:29.574 | INFO | "malloc" : " 14 GB", | 20260507 14:49:29.574 | INFO | "pinned" : " 29 GB" | 20260507 14:49:29.575 | INFO | } | 20260507 14:49:29.575 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:49:29.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:29.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:49:29.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:49:29.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:29.575 | INFO | Uploaded 36GiB to GPU in 1.58738s (22.6789GiB/s) | 20260507 14:49:29.575 | INFO | Downloaded 34GiB to GPU in 1.53424s (22.1608GiB/s) | 20260507 14:49:34.714 | INFO | Analyzing chunk frames = [343040,343552) | 20260507 14:49:34.714 | INFO | MemInfo: { | 20260507 14:49:34.714 | INFO | "gpu" : " 17 GB", | 20260507 14:49:34.714 | INFO | "huge" : "", | 20260507 14:49:34.714 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:34.714 | INFO | "malloc" : " 14 GB", | 20260507 14:49:34.714 | INFO | "pinned" : " 29 GB" | 20260507 14:49:34.714 | INFO | } | 20260507 14:49:34.714 | INFO | Perf report for the last 5.13945 seconds: | 20260507 14:49:34.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:34.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.714 | INFO | ANALYSIS : 56.6%, 3.0, 87.4ms, 0 | 20260507 14:49:34.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.714 | INFO | PRE_HQ : 27.9%, 1.0, 14.3ms, 0 | 20260507 14:49:34.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:34.715 | INFO | Uploaded 36GiB to GPU in 1.58678s (22.6874GiB/s) | 20260507 14:49:34.715 | INFO | Downloaded 34GiB to GPU in 1.53652s (22.1279GiB/s) | 20260507 14:49:39.853 | INFO | Analyzing chunk frames = [343552,344064) | 20260507 14:49:39.853 | INFO | MemInfo: { | 20260507 14:49:39.853 | INFO | "gpu" : " 17 GB", | 20260507 14:49:39.853 | INFO | "huge" : "", | 20260507 14:49:39.853 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:39.853 | INFO | "malloc" : " 14 GB", | 20260507 14:49:39.853 | INFO | "pinned" : " 29 GB" | 20260507 14:49:39.853 | INFO | } | 20260507 14:49:39.853 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:49:39.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:39.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.853 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:49:39.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.853 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:49:39.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:39.854 | INFO | Uploaded 36GiB to GPU in 1.58717s (22.6818GiB/s) | 20260507 14:49:39.854 | INFO | Downloaded 34GiB to GPU in 1.53671s (22.1251GiB/s) | 20260507 14:49:39.854 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:49:39.854 | INFO | Total mem usage: 15.7 GiB | 20260507 14:49:39.854 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.61%) | 20260507 14:49:39.854 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -4.99%) | 20260507 14:49:39.854 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:49:39.854 | INFO | High Water Breakdown: | 20260507 14:49:39.854 | INFO | 58.9% : 8.78 GiB ( 94.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:49:39.854 | INFO | 41% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:49:39.854 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:49:39.854 | INFO | ------------------------------------------ | 20260507 14:49:39.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:49:39.854 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:49:39.854 | INFO | Total mem usage: 26.5 GiB | 20260507 14:49:39.854 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:49:39.854 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:49:39.854 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:49:39.854 | INFO | High Water Breakdown: | 20260507 14:49:39.854 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:49:39.854 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:49:39.854 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:49:39.854 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:49:39.854 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:49:39.854 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:49:39.854 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:49:39.854 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:49:39.854 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:49:39.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:49:39.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:49:39.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:49:39.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:49:39.854 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:49:39.854 | INFO | ------------------------------------------ | 20260507 14:49:39.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:49:39.854 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:49:39.854 | INFO | Total mem usage: 27.8 GiB | 20260507 14:49:39.854 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:49:39.854 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:49:39.854 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:49:39.854 | INFO | High Water Breakdown: | 20260507 14:49:39.854 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:49:39.854 | INFO | ------------------------------------------ | 20260507 14:49:39.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:49:39.854 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:49:39.854 | INFO | Total mem usage: 15.7 GiB | 20260507 14:49:39.854 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:49:39.854 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:49:39.855 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:49:39.855 | INFO | High Water Breakdown: | 20260507 14:49:39.855 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:49:39.855 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:49:39.855 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:49:39.855 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:49:39.855 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:49:39.855 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:49:39.855 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:49:39.855 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:49:39.855 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:49:39.855 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:49:39.855 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:49:39.855 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:49:39.855 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:49:39.855 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:49:39.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:49:39.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:49:39.855 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:49:39.855 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:49:39.855 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:49:39.855 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:49:39.855 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:49:39.855 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:49:39.855 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:49:39.855 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:49:39.855 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:49:39.855 | INFO | ------------------------------------------ | 20260507 14:49:39.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:49:39.855 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:49:39.855 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:49:39.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:49:39.855 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 4.45e-03 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 11.00%, Baseline -- 3.30e+00 / -nan, 2.02e-02 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 2.96e+00 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.72e+01 / -nan, 4.35e-02 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.18e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 3.57%, Metrics -- 1.07e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 1.32%, PulseSerializing -- 3.96e-01 / -nan, 5.81e-02 / 0.00e+00 | 20260507 14:49:39.855 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:49:39.855 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:49:39.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:49:39.855 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 18.50%, Upload -- 1.66e+01 / 0.00e+00, 4.57e-01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 18.09%, Download -- 1.62e+01 / 0.00e+00, 7.53e-01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 33.37%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 0.00%, LockDownload -- 4.35e-04 / 0.00e+00, 9.43e-05 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 29.46%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:49:39.855 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260507 14:49:39.950 | INFO | Starting a round of Baz IO | 20260507 14:49:42.843 | INFO | Populating RTMetrics report | 20260507 14:49:42.843 | INFO | RTMetrics report populated | 20260507 14:49:42.843 | INFO | Opening tmp RTMetrics json file | 20260507 14:49:42.847 | INFO | Serializing RTMetrics to json | 20260507 14:49:42.850 | INFO | Serialization done, swapping tmp file now... | 20260507 14:49:42.861 | INFO | RTMetrics done | 20260507 14:49:42.956 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.006 seconds | 20260507 14:49:42.956 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.31% / 0.07% / 43.62% | 20260507 14:49:42.956 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.09517% / 0.000150998% / 2.38419e-05% | 20260507 14:49:42.957 | INFO | Baz IO overall: | 20260507 14:49:42.957 | INFO | BytesWritten : 128938865780 | 20260507 14:49:42.957 | INFO | NumEvents : 61542505725 | 20260507 14:49:44.992 | INFO | Analyzing chunk frames = [344064,344576) | 20260507 14:49:44.993 | INFO | MemInfo: { | 20260507 14:49:44.993 | INFO | "gpu" : " 17 GB", | 20260507 14:49:44.993 | INFO | "huge" : "", | 20260507 14:49:44.993 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:44.993 | INFO | "malloc" : " 14 GB", | 20260507 14:49:44.993 | INFO | "pinned" : " 29 GB" | 20260507 14:49:44.993 | INFO | } | 20260507 14:49:44.993 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:49:44.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:44.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:44.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:44.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:44.993 | INFO | ANALYSIS : 57.3%, 3.0, 88.7ms, 0 | 20260507 14:49:44.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:44.993 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 14:49:44.993 | INFO | PRE_HQ : 55.9%, 2.1, 59.9ms, 0 | 20260507 14:49:44.993 | INFO | BAZWRITER : 58.1%, 4.2, 126.3ms, 0 | 20260507 14:49:44.993 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 14:49:44.993 | INFO | Uploaded 36GiB to GPU in 1.63424s (22.0287GiB/s) | 20260507 14:49:44.993 | INFO | Downloaded 38GiB to GPU in 1.71437s (22.1656GiB/s) | 20260507 14:49:50.132 | INFO | Analyzing chunk frames = [344576,345088) | 20260507 14:49:50.132 | INFO | MemInfo: { | 20260507 14:49:50.132 | INFO | "gpu" : " 17 GB", | 20260507 14:49:50.132 | INFO | "huge" : "", | 20260507 14:49:50.132 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:50.132 | INFO | "malloc" : " 14 GB", | 20260507 14:49:50.132 | INFO | "pinned" : " 29 GB" | 20260507 14:49:50.132 | INFO | } | 20260507 14:49:50.133 | INFO | Perf report for the last 5.13993 seconds: | 20260507 14:49:50.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:50.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:49:50.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:49:50.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:50.133 | INFO | Uploaded 36GiB to GPU in 1.58412s (22.7255GiB/s) | 20260507 14:49:50.133 | INFO | Downloaded 34GiB to GPU in 1.53217s (22.1907GiB/s) | 20260507 14:49:55.271 | INFO | Analyzing chunk frames = [345088,345600) | 20260507 14:49:55.271 | INFO | MemInfo: { | 20260507 14:49:55.271 | INFO | "gpu" : " 17 GB", | 20260507 14:49:55.271 | INFO | "huge" : "", | 20260507 14:49:55.271 | INFO | "huge_shared" : " 29 GB", | 20260507 14:49:55.271 | INFO | "malloc" : " 14 GB", | 20260507 14:49:55.271 | INFO | "pinned" : " 29 GB" | 20260507 14:49:55.271 | INFO | } | 20260507 14:49:55.272 | INFO | Perf report for the last 5.13901 seconds: | 20260507 14:49:55.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:49:55.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 14:49:55.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:49:55.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:49:55.272 | INFO | Uploaded 36GiB to GPU in 1.58813s (22.6682GiB/s) | 20260507 14:49:55.272 | INFO | Downloaded 34GiB to GPU in 1.53747s (22.1143GiB/s) | 20260507 14:50:00.425 | INFO | Analyzing chunk frames = [345600,346112) | 20260507 14:50:00.425 | INFO | MemInfo: { | 20260507 14:50:00.425 | INFO | "gpu" : " 17 GB", | 20260507 14:50:00.425 | INFO | "huge" : "", | 20260507 14:50:00.425 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:00.425 | INFO | "malloc" : " 14 GB", | 20260507 14:50:00.426 | INFO | "pinned" : " 29 GB" | 20260507 14:50:00.426 | INFO | } | 20260507 14:50:00.426 | INFO | Perf report for the last 5.15413 seconds: | 20260507 14:50:00.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:00.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | ANALYSIS : 56.0%, 3.0, 87.1ms, 0 | 20260507 14:50:00.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 14:50:00.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:00.426 | INFO | Uploaded 36GiB to GPU in 1.58803s (22.6695GiB/s) | 20260507 14:50:00.426 | INFO | Downloaded 34GiB to GPU in 1.53557s (22.1416GiB/s) | 20260507 14:50:05.551 | INFO | Analyzing chunk frames = [346112,346624) | 20260507 14:50:05.551 | INFO | MemInfo: { | 20260507 14:50:05.551 | INFO | "gpu" : " 17 GB", | 20260507 14:50:05.551 | INFO | "huge" : "", | 20260507 14:50:05.551 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:05.551 | INFO | "malloc" : " 14 GB", | 20260507 14:50:05.551 | INFO | "pinned" : " 29 GB" | 20260507 14:50:05.551 | INFO | } | 20260507 14:50:05.551 | INFO | Perf report for the last 5.12537 seconds: | 20260507 14:50:05.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:05.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 14:50:05.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260507 14:50:05.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:05.551 | INFO | Uploaded 36GiB to GPU in 1.58634s (22.6937GiB/s) | 20260507 14:50:05.551 | INFO | Downloaded 34GiB to GPU in 1.53494s (22.1507GiB/s) | 20260507 14:50:10.690 | INFO | Analyzing chunk frames = [346624,347136) | 20260507 14:50:10.690 | INFO | MemInfo: { | 20260507 14:50:10.690 | INFO | "gpu" : " 17 GB", | 20260507 14:50:10.690 | INFO | "huge" : "", | 20260507 14:50:10.690 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:10.690 | INFO | "malloc" : " 14 GB", | 20260507 14:50:10.690 | INFO | "pinned" : " 29 GB" | 20260507 14:50:10.690 | INFO | } | 20260507 14:50:10.690 | INFO | Perf report for the last 5.13889 seconds: | 20260507 14:50:10.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:10.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:50:10.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 14:50:10.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:10.690 | INFO | Uploaded 36GiB to GPU in 1.58757s (22.6761GiB/s) | 20260507 14:50:10.690 | INFO | Downloaded 34GiB to GPU in 1.53746s (22.1144GiB/s) | 20260507 14:50:15.829 | INFO | Analyzing chunk frames = [347136,347648) | 20260507 14:50:15.829 | INFO | MemInfo: { | 20260507 14:50:15.829 | INFO | "gpu" : " 17 GB", | 20260507 14:50:15.829 | INFO | "huge" : "", | 20260507 14:50:15.832 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:15.832 | INFO | "malloc" : " 14 GB", | 20260507 14:50:15.832 | INFO | "pinned" : " 29 GB" | 20260507 14:50:15.832 | INFO | } | 20260507 14:50:15.832 | INFO | Perf report for the last 5.14187 seconds: | 20260507 14:50:15.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:15.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | ANALYSIS : 56.5%, 3.0, 88.3ms, 0 | 20260507 14:50:15.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 14:50:15.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:15.832 | INFO | Uploaded 36GiB to GPU in 1.58577s (22.702GiB/s) | 20260507 14:50:15.832 | INFO | Downloaded 34GiB to GPU in 1.53666s (22.1259GiB/s) | 20260507 14:50:20.968 | INFO | Analyzing chunk frames = [347648,348160) | 20260507 14:50:20.968 | INFO | MemInfo: { | 20260507 14:50:20.968 | INFO | "gpu" : " 17 GB", | 20260507 14:50:20.968 | INFO | "huge" : "", | 20260507 14:50:20.968 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:20.968 | INFO | "malloc" : " 14 GB", | 20260507 14:50:20.968 | INFO | "pinned" : " 29 GB" | 20260507 14:50:20.968 | INFO | } | 20260507 14:50:20.968 | INFO | Perf report for the last 5.13633 seconds: | 20260507 14:50:20.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:20.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.968 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 14:50:20.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.968 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 14:50:20.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:20.969 | INFO | Uploaded 36GiB to GPU in 1.58652s (22.6912GiB/s) | 20260507 14:50:20.969 | INFO | Downloaded 34GiB to GPU in 1.53551s (22.1424GiB/s) | 20260507 14:50:21.065 | INFO | Starting a round of Baz IO | 20260507 14:50:23.955 | INFO | Populating RTMetrics report | 20260507 14:50:23.955 | INFO | RTMetrics report populated | 20260507 14:50:23.955 | INFO | Opening tmp RTMetrics json file | 20260507 14:50:24.000 | INFO | Serializing RTMetrics to json | 20260507 14:50:24.003 | INFO | Serialization done, swapping tmp file now... | 20260507 14:50:24.009 | INFO | RTMetrics done | 20260507 14:50:24.619 | INFO | Finished a round of Baz IO: Wrote 1.708 GiB in 3.553 seconds | 20260507 14:50:24.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.24% / 0.07% / 43.69% | 20260507 14:50:24.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.07341% / 0.000143051% / 3.17891e-05% | 20260507 14:50:24.619 | INFO | Baz IO overall: | 20260507 14:50:24.619 | INFO | BytesWritten : 130772831116 | 20260507 14:50:24.619 | INFO | NumEvents : 62437105897 | 20260507 14:50:26.107 | INFO | Analyzing chunk frames = [348160,348672) | 20260507 14:50:26.107 | INFO | MemInfo: { | 20260507 14:50:26.107 | INFO | "gpu" : " 17 GB", | 20260507 14:50:26.107 | INFO | "huge" : "", | 20260507 14:50:26.107 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:26.107 | INFO | "malloc" : " 14 GB", | 20260507 14:50:26.107 | INFO | "pinned" : " 29 GB" | 20260507 14:50:26.107 | INFO | } | 20260507 14:50:26.107 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:50:26.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:26.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:26.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:26.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:26.107 | INFO | ANALYSIS : 57.2%, 2.9, 86.8ms, 0 | 20260507 14:50:26.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:26.107 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 14:50:26.108 | INFO | PRE_HQ : 56.0%, 2.0, 59.5ms, 0 | 20260507 14:50:26.108 | INFO | BAZWRITER : 68.6%, 4.7, 164.4ms, 0 | 20260507 14:50:26.108 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 14:50:26.108 | INFO | Uploaded 36GiB to GPU in 1.65212s (21.7902GiB/s) | 20260507 14:50:26.108 | INFO | Downloaded 38GiB to GPU in 1.73638s (21.8846GiB/s) | 20260507 14:50:31.257 | INFO | Analyzing chunk frames = [348672,349184) | 20260507 14:50:31.257 | INFO | MemInfo: { | 20260507 14:50:31.257 | INFO | "gpu" : " 17 GB", | 20260507 14:50:31.257 | INFO | "huge" : "", | 20260507 14:50:31.257 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:31.257 | INFO | "malloc" : " 14 GB", | 20260507 14:50:31.257 | INFO | "pinned" : " 29 GB" | 20260507 14:50:31.258 | INFO | } | 20260507 14:50:31.258 | INFO | Perf report for the last 5.15015 seconds: | 20260507 14:50:31.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:31.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | ANALYSIS : 57.0%, 3.0, 89.1ms, 0 | 20260507 14:50:31.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:50:31.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:31.258 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4444GiB/s) | 20260507 14:50:31.258 | INFO | Downloaded 34GiB to GPU in 1.52339s (22.3187GiB/s) | 20260507 14:50:36.386 | INFO | Analyzing chunk frames = [349184,349696) | 20260507 14:50:36.386 | INFO | MemInfo: { | 20260507 14:50:36.386 | INFO | "gpu" : " 17 GB", | 20260507 14:50:36.386 | INFO | "huge" : "", | 20260507 14:50:36.386 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:36.386 | INFO | "malloc" : " 14 GB", | 20260507 14:50:36.386 | INFO | "pinned" : " 29 GB" | 20260507 14:50:36.386 | INFO | } | 20260507 14:50:36.386 | INFO | Perf report for the last 5.12862 seconds: | 20260507 14:50:36.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:36.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.386 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260507 14:50:36.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.386 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:50:36.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:36.387 | INFO | Uploaded 36GiB to GPU in 1.60935s (22.3693GiB/s) | 20260507 14:50:36.387 | INFO | Downloaded 34GiB to GPU in 1.53026s (22.2185GiB/s) | 20260507 14:50:41.525 | INFO | Analyzing chunk frames = [349696,350208) | 20260507 14:50:41.525 | INFO | MemInfo: { | 20260507 14:50:41.525 | INFO | "gpu" : " 17 GB", | 20260507 14:50:41.525 | INFO | "huge" : "", | 20260507 14:50:41.525 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:41.525 | INFO | "malloc" : " 14 GB", | 20260507 14:50:41.525 | INFO | "pinned" : " 29 GB" | 20260507 14:50:41.525 | INFO | } | 20260507 14:50:41.526 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:50:41.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:41.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:50:41.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:50:41.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:41.526 | INFO | Uploaded 36GiB to GPU in 1.60706s (22.4012GiB/s) | 20260507 14:50:41.526 | INFO | Downloaded 34GiB to GPU in 1.52709s (22.2646GiB/s) | 20260507 14:50:46.665 | INFO | Analyzing chunk frames = [350208,350720) | 20260507 14:50:46.665 | INFO | MemInfo: { | 20260507 14:50:46.665 | INFO | "gpu" : " 17 GB", | 20260507 14:50:46.665 | INFO | "huge" : "", | 20260507 14:50:46.665 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:46.665 | INFO | "malloc" : " 14 GB", | 20260507 14:50:46.665 | INFO | "pinned" : " 29 GB" | 20260507 14:50:46.665 | INFO | } | 20260507 14:50:46.665 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:50:46.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:46.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:50:46.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:50:46.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:46.665 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4074GiB/s) | 20260507 14:50:46.665 | INFO | Downloaded 34GiB to GPU in 1.52751s (22.2584GiB/s) | 20260507 14:50:51.804 | INFO | Analyzing chunk frames = [350720,351232) | 20260507 14:50:51.804 | INFO | MemInfo: { | 20260507 14:50:51.804 | INFO | "gpu" : " 17 GB", | 20260507 14:50:51.804 | INFO | "huge" : "", | 20260507 14:50:51.804 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:51.804 | INFO | "malloc" : " 14 GB", | 20260507 14:50:51.804 | INFO | "pinned" : " 29 GB" | 20260507 14:50:51.804 | INFO | } | 20260507 14:50:51.804 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:50:51.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:51.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.804 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:50:51.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.804 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:50:51.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:51.805 | INFO | Uploaded 36GiB to GPU in 1.60666s (22.4067GiB/s) | 20260507 14:50:51.805 | INFO | Downloaded 34GiB to GPU in 1.5256s (22.2864GiB/s) | 20260507 14:50:56.943 | INFO | Analyzing chunk frames = [351232,351744) | 20260507 14:50:56.943 | INFO | MemInfo: { | 20260507 14:50:56.943 | INFO | "gpu" : " 17 GB", | 20260507 14:50:56.943 | INFO | "huge" : "", | 20260507 14:50:56.944 | INFO | "huge_shared" : " 29 GB", | 20260507 14:50:56.944 | INFO | "malloc" : " 14 GB", | 20260507 14:50:56.944 | INFO | "pinned" : " 29 GB" | 20260507 14:50:56.944 | INFO | } | 20260507 14:50:56.944 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:50:56.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:50:56.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:50:56.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:50:56.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:50:56.944 | INFO | Uploaded 36GiB to GPU in 1.60746s (22.3956GiB/s) | 20260507 14:50:56.944 | INFO | Downloaded 34GiB to GPU in 1.52772s (22.2554GiB/s) | 20260507 14:51:02.083 | INFO | Analyzing chunk frames = [351744,352256) | 20260507 14:51:02.083 | INFO | MemInfo: { | 20260507 14:51:02.083 | INFO | "gpu" : " 17 GB", | 20260507 14:51:02.083 | INFO | "huge" : "", | 20260507 14:51:02.083 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:02.083 | INFO | "malloc" : " 14 GB", | 20260507 14:51:02.083 | INFO | "pinned" : " 29 GB" | 20260507 14:51:02.083 | INFO | } | 20260507 14:51:02.083 | INFO | Perf report for the last 5.13972 seconds: | 20260507 14:51:02.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:02.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.083 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:51:02.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.084 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:51:02.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:02.084 | INFO | Uploaded 36GiB to GPU in 1.6072s (22.3992GiB/s) | 20260507 14:51:02.084 | INFO | Downloaded 34GiB to GPU in 1.52787s (22.2532GiB/s) | 20260507 14:51:02.084 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:51:02.084 | INFO | Total mem usage: 16.4 GiB | 20260507 14:51:02.084 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:51:02.084 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: 0.0234%) | 20260507 14:51:02.084 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:51:02.084 | INFO | High Water Breakdown: | 20260507 14:51:02.084 | INFO | 60.5% : 9.93 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:51:02.084 | INFO | 39.4% : 6.46 GiB ( 96.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:51:02.084 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:51:02.084 | INFO | ------------------------------------------ | 20260507 14:51:02.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:51:02.084 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:51:02.084 | INFO | Total mem usage: 26.5 GiB | 20260507 14:51:02.084 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:51:02.084 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:51:02.085 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:51:02.085 | INFO | High Water Breakdown: | 20260507 14:51:02.085 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:51:02.085 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:51:02.085 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:51:02.085 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:51:02.085 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:51:02.085 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:51:02.085 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:51:02.085 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:51:02.085 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:51:02.085 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:51:02.085 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:51:02.085 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:51:02.085 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:51:02.085 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:51:02.085 | INFO | ------------------------------------------ | 20260507 14:51:02.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:51:02.085 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:51:02.085 | INFO | Total mem usage: 27.8 GiB | 20260507 14:51:02.085 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:51:02.085 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:51:02.085 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:51:02.085 | INFO | High Water Breakdown: | 20260507 14:51:02.085 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:51:02.085 | INFO | ------------------------------------------ | 20260507 14:51:02.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:51:02.085 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:51:02.085 | INFO | Total mem usage: 15.7 GiB | 20260507 14:51:02.085 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:51:02.085 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:51:02.085 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:51:02.085 | INFO | High Water Breakdown: | 20260507 14:51:02.085 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:51:02.085 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:51:02.085 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:51:02.085 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:51:02.085 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:51:02.085 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:51:02.085 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:51:02.085 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:51:02.085 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:51:02.085 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:51:02.085 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:51:02.085 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:51:02.085 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:51:02.085 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:51:02.085 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:51:02.085 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:51:02.085 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:51:02.085 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:51:02.085 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:51:02.085 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:51:02.085 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:51:02.085 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:51:02.085 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:51:02.085 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:51:02.085 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:51:02.085 | INFO | ------------------------------------------ | 20260507 14:51:02.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:51:02.085 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:51:02.085 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:51:02.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:51:02.085 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 5.47e-03 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 11.00%, Baseline -- 3.30e+00 / -nan, 2.11e-02 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.72e+01 / -nan, 4.42e-02 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.18e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 3.57%, Metrics -- 1.07e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 14:51:02.085 | INFO | PerfReport: 1.32%, PulseSerializing -- 3.95e-01 / -nan, 5.77e-02 / 0.00e+00 | 20260507 14:51:02.085 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:51:02.085 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:51:02.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:51:02.086 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 18.59%, Upload -- 1.67e+01 / 0.00e+00, 4.52e-01 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 18.04%, Download -- 1.62e+01 / 0.00e+00, 7.59e-01 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 33.36%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 0.00%, LockDownload -- 4.35e-04 / 0.00e+00, 1.53e-04 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 29.42%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:51:02.086 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 2.45e-02 / 0.00e+00 | 20260507 14:51:02.181 | INFO | Starting a round of Baz IO | 20260507 14:51:05.069 | INFO | Populating RTMetrics report | 20260507 14:51:05.069 | INFO | RTMetrics report populated | 20260507 14:51:05.069 | INFO | Opening tmp RTMetrics json file | 20260507 14:51:05.172 | INFO | Serializing RTMetrics to json | 20260507 14:51:05.232 | INFO | Serialization done, swapping tmp file now... | 20260507 14:51:05.249 | INFO | RTMetrics done | 20260507 14:51:05.274 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.093 seconds | 20260507 14:51:05.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.17% / 0.07% / 43.76% | 20260507 14:51:05.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.06368% / 0.000154972% / 1.19209e-05% | 20260507 14:51:05.274 | INFO | Baz IO overall: | 20260507 14:51:05.274 | INFO | BytesWritten : 132609771813 | 20260507 14:51:05.274 | INFO | NumEvents : 63333462928 | 20260507 14:51:07.222 | INFO | Analyzing chunk frames = [352256,352768) | 20260507 14:51:07.222 | INFO | MemInfo: { | 20260507 14:51:07.222 | INFO | "gpu" : " 17 GB", | 20260507 14:51:07.222 | INFO | "huge" : "", | 20260507 14:51:07.222 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:07.222 | INFO | "malloc" : " 14 GB", | 20260507 14:51:07.222 | INFO | "pinned" : " 29 GB" | 20260507 14:51:07.222 | INFO | } | 20260507 14:51:07.222 | INFO | Perf report for the last 5.1388 seconds: | 20260507 14:51:07.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:07.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:07.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:07.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:07.222 | INFO | ANALYSIS : 57.2%, 2.9, 86.0ms, 0 | 20260507 14:51:07.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:07.222 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 14:51:07.222 | INFO | PRE_HQ : 56.1%, 2.0, 59.6ms, 0 | 20260507 14:51:07.222 | INFO | BAZWRITER : 59.8%, 4.7, 143.5ms, 0 | 20260507 14:51:07.222 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 14:51:07.223 | INFO | Uploaded 36GiB to GPU in 1.6593s (21.696GiB/s) | 20260507 14:51:07.223 | INFO | Downloaded 38GiB to GPU in 1.74091s (21.8276GiB/s) | 20260507 14:51:12.361 | INFO | Analyzing chunk frames = [352768,353280) | 20260507 14:51:12.361 | INFO | MemInfo: { | 20260507 14:51:12.361 | INFO | "gpu" : " 17 GB", | 20260507 14:51:12.361 | INFO | "huge" : "", | 20260507 14:51:12.362 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:12.362 | INFO | "malloc" : " 14 GB", | 20260507 14:51:12.362 | INFO | "pinned" : " 29 GB" | 20260507 14:51:12.362 | INFO | } | 20260507 14:51:12.362 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:51:12.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:12.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 14:51:12.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 14:51:12.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:12.362 | INFO | Uploaded 36GiB to GPU in 1.58379s (22.7304GiB/s) | 20260507 14:51:12.362 | INFO | Downloaded 34GiB to GPU in 1.53212s (22.1915GiB/s) | 20260507 14:51:17.501 | INFO | Analyzing chunk frames = [353280,353792) | 20260507 14:51:17.501 | INFO | MemInfo: { | 20260507 14:51:17.501 | INFO | "gpu" : " 17 GB", | 20260507 14:51:17.501 | INFO | "huge" : "", | 20260507 14:51:17.501 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:17.501 | INFO | "malloc" : " 14 GB", | 20260507 14:51:17.501 | INFO | "pinned" : " 29 GB" | 20260507 14:51:17.501 | INFO | } | 20260507 14:51:17.501 | INFO | Perf report for the last 5.13953 seconds: | 20260507 14:51:17.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:17.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.501 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:51:17.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.501 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:51:17.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:17.502 | INFO | Uploaded 36GiB to GPU in 1.58941s (22.6499GiB/s) | 20260507 14:51:17.502 | INFO | Downloaded 34GiB to GPU in 1.53763s (22.112GiB/s) | 20260507 14:51:22.640 | INFO | Analyzing chunk frames = [353792,354304) | 20260507 14:51:22.640 | INFO | MemInfo: { | 20260507 14:51:22.640 | INFO | "gpu" : " 17 GB", | 20260507 14:51:22.640 | INFO | "huge" : "", | 20260507 14:51:22.640 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:22.640 | INFO | "malloc" : " 14 GB", | 20260507 14:51:22.640 | INFO | "pinned" : " 29 GB" | 20260507 14:51:22.640 | INFO | } | 20260507 14:51:22.640 | INFO | Perf report for the last 5.13917 seconds: | 20260507 14:51:22.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:22.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.641 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:51:22.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.641 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 14:51:22.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:22.641 | INFO | Uploaded 36GiB to GPU in 1.58794s (22.6709GiB/s) | 20260507 14:51:22.641 | INFO | Downloaded 34GiB to GPU in 1.53664s (22.1262GiB/s) | 20260507 14:51:27.780 | INFO | Analyzing chunk frames = [354304,354816) | 20260507 14:51:27.780 | INFO | MemInfo: { | 20260507 14:51:27.780 | INFO | "gpu" : " 17 GB", | 20260507 14:51:27.780 | INFO | "huge" : "", | 20260507 14:51:27.780 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:27.780 | INFO | "malloc" : " 14 GB", | 20260507 14:51:27.780 | INFO | "pinned" : " 29 GB" | 20260507 14:51:27.780 | INFO | } | 20260507 14:51:27.780 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:51:27.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:27.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:51:27.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:51:27.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:27.780 | INFO | Uploaded 36GiB to GPU in 1.58457s (22.7191GiB/s) | 20260507 14:51:27.780 | INFO | Downloaded 34GiB to GPU in 1.53462s (22.1553GiB/s) | 20260507 14:51:32.919 | INFO | Analyzing chunk frames = [354816,355328) | 20260507 14:51:32.919 | INFO | MemInfo: { | 20260507 14:51:32.919 | INFO | "gpu" : " 17 GB", | 20260507 14:51:32.919 | INFO | "huge" : "", | 20260507 14:51:32.919 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:32.919 | INFO | "malloc" : " 14 GB", | 20260507 14:51:32.919 | INFO | "pinned" : " 29 GB" | 20260507 14:51:32.919 | INFO | } | 20260507 14:51:32.919 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:51:32.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:32.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.919 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 14:51:32.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.919 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:51:32.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:32.920 | INFO | Uploaded 36GiB to GPU in 1.58699s (22.6845GiB/s) | 20260507 14:51:32.920 | INFO | Downloaded 34GiB to GPU in 1.53556s (22.1418GiB/s) | 20260507 14:51:38.058 | INFO | Analyzing chunk frames = [355328,355840) | 20260507 14:51:38.058 | INFO | MemInfo: { | 20260507 14:51:38.058 | INFO | "gpu" : " 17 GB", | 20260507 14:51:38.058 | INFO | "huge" : "", | 20260507 14:51:38.058 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:38.058 | INFO | "malloc" : " 14 GB", | 20260507 14:51:38.058 | INFO | "pinned" : " 29 GB" | 20260507 14:51:38.058 | INFO | } | 20260507 14:51:38.058 | INFO | Perf report for the last 5.13899 seconds: | 20260507 14:51:38.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:38.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.058 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:51:38.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.058 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:51:38.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:38.059 | INFO | Uploaded 36GiB to GPU in 1.58632s (22.6941GiB/s) | 20260507 14:51:38.059 | INFO | Downloaded 34GiB to GPU in 1.53539s (22.1441GiB/s) | 20260507 14:51:43.197 | INFO | Analyzing chunk frames = [355840,356352) | 20260507 14:51:43.197 | INFO | MemInfo: { | 20260507 14:51:43.197 | INFO | "gpu" : " 17 GB", | 20260507 14:51:43.197 | INFO | "huge" : "", | 20260507 14:51:43.197 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:43.197 | INFO | "malloc" : " 14 GB", | 20260507 14:51:43.197 | INFO | "pinned" : " 29 GB" | 20260507 14:51:43.198 | INFO | } | 20260507 14:51:43.198 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:51:43.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:43.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:51:43.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:51:43.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:43.198 | INFO | Uploaded 36GiB to GPU in 1.58638s (22.6932GiB/s) | 20260507 14:51:43.198 | INFO | Downloaded 34GiB to GPU in 1.53639s (22.1298GiB/s) | 20260507 14:51:43.294 | INFO | Starting a round of Baz IO | 20260507 14:51:46.185 | INFO | Populating RTMetrics report | 20260507 14:51:46.186 | INFO | RTMetrics report populated | 20260507 14:51:46.186 | INFO | Opening tmp RTMetrics json file | 20260507 14:51:46.201 | INFO | Serializing RTMetrics to json | 20260507 14:51:46.205 | INFO | Serialization done, swapping tmp file now... | 20260507 14:51:46.209 | INFO | RTMetrics done | 20260507 14:51:46.475 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.18 seconds | 20260507 14:51:46.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.10% / 0.07% / 43.83% | 20260507 14:51:46.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.99569% / 0.000182788% / 4.76837e-05% | 20260507 14:51:46.475 | INFO | Baz IO overall: | 20260507 14:51:46.475 | INFO | BytesWritten : 134444611667 | 20260507 14:51:46.475 | INFO | NumEvents : 64227612159 | 20260507 14:51:48.337 | INFO | Analyzing chunk frames = [356352,356864) | 20260507 14:51:48.337 | INFO | MemInfo: { | 20260507 14:51:48.337 | INFO | "gpu" : " 17 GB", | 20260507 14:51:48.337 | INFO | "huge" : "", | 20260507 14:51:48.337 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:48.337 | INFO | "malloc" : " 14 GB", | 20260507 14:51:48.337 | INFO | "pinned" : " 29 GB" | 20260507 14:51:48.337 | INFO | } | 20260507 14:51:48.337 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:51:48.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:48.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:48.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:48.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:48.337 | INFO | ANALYSIS : 57.2%, 2.9, 85.6ms, 0 | 20260507 14:51:48.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:48.337 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 14:51:48.337 | INFO | PRE_HQ : 56.2%, 2.1, 59.8ms, 0 | 20260507 14:51:48.337 | INFO | BAZWRITER : 61.4%, 4.8, 151.1ms, 0 | 20260507 14:51:48.337 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 14:51:48.338 | INFO | Uploaded 36GiB to GPU in 1.66304s (21.6472GiB/s) | 20260507 14:51:48.338 | INFO | Downloaded 38GiB to GPU in 1.74605s (21.7634GiB/s) | 20260507 14:51:53.476 | INFO | Analyzing chunk frames = [356864,357376) | 20260507 14:51:53.476 | INFO | MemInfo: { | 20260507 14:51:53.476 | INFO | "gpu" : " 17 GB", | 20260507 14:51:53.476 | INFO | "huge" : "", | 20260507 14:51:53.476 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:53.476 | INFO | "malloc" : " 14 GB", | 20260507 14:51:53.476 | INFO | "pinned" : " 29 GB" | 20260507 14:51:53.476 | INFO | } | 20260507 14:51:53.476 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:51:53.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:53.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.476 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:51:53.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.476 | INFO | PRE_HQ : 28.8%, 1.0, 15.3ms, 0 | 20260507 14:51:53.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:53.477 | INFO | Uploaded 36GiB to GPU in 1.58475s (22.7166GiB/s) | 20260507 14:51:53.477 | INFO | Downloaded 34GiB to GPU in 1.5326s (22.1846GiB/s) | 20260507 14:51:58.615 | INFO | Analyzing chunk frames = [357376,357888) | 20260507 14:51:58.615 | INFO | MemInfo: { | 20260507 14:51:58.615 | INFO | "gpu" : " 17 GB", | 20260507 14:51:58.615 | INFO | "huge" : "", | 20260507 14:51:58.615 | INFO | "huge_shared" : " 29 GB", | 20260507 14:51:58.615 | INFO | "malloc" : " 14 GB", | 20260507 14:51:58.615 | INFO | "pinned" : " 29 GB" | 20260507 14:51:58.615 | INFO | } | 20260507 14:51:58.616 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:51:58.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:51:58.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 14:51:58.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:51:58.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:51:58.616 | INFO | Uploaded 36GiB to GPU in 1.58805s (22.6693GiB/s) | 20260507 14:51:58.616 | INFO | Downloaded 34GiB to GPU in 1.5382s (22.1037GiB/s) | 20260507 14:52:03.755 | INFO | Analyzing chunk frames = [357888,358400) | 20260507 14:52:03.755 | INFO | MemInfo: { | 20260507 14:52:03.755 | INFO | "gpu" : " 17 GB", | 20260507 14:52:03.755 | INFO | "huge" : "", | 20260507 14:52:03.755 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:03.755 | INFO | "malloc" : " 14 GB", | 20260507 14:52:03.755 | INFO | "pinned" : " 29 GB" | 20260507 14:52:03.755 | INFO | } | 20260507 14:52:03.755 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:52:03.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:03.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:52:03.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:52:03.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:03.755 | INFO | Uploaded 36GiB to GPU in 1.5875s (22.6772GiB/s) | 20260507 14:52:03.755 | INFO | Downloaded 34GiB to GPU in 1.53743s (22.1148GiB/s) | 20260507 14:52:08.895 | INFO | Analyzing chunk frames = [358400,358912) | 20260507 14:52:08.895 | INFO | MemInfo: { | 20260507 14:52:08.895 | INFO | "gpu" : " 17 GB", | 20260507 14:52:08.895 | INFO | "huge" : "", | 20260507 14:52:08.895 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:08.895 | INFO | "malloc" : " 14 GB", | 20260507 14:52:08.895 | INFO | "pinned" : " 29 GB" | 20260507 14:52:08.895 | INFO | } | 20260507 14:52:08.895 | INFO | Perf report for the last 5.14024 seconds: | 20260507 14:52:08.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:08.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.895 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:52:08.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.895 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:52:08.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:08.896 | INFO | Uploaded 36GiB to GPU in 1.58643s (22.6925GiB/s) | 20260507 14:52:08.896 | INFO | Downloaded 34GiB to GPU in 1.53519s (22.1471GiB/s) | 20260507 14:52:14.034 | INFO | Analyzing chunk frames = [358912,359424) | 20260507 14:52:14.034 | INFO | MemInfo: { | 20260507 14:52:14.034 | INFO | "gpu" : " 17 GB", | 20260507 14:52:14.034 | INFO | "huge" : "", | 20260507 14:52:14.034 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:14.034 | INFO | "malloc" : " 14 GB", | 20260507 14:52:14.034 | INFO | "pinned" : " 29 GB" | 20260507 14:52:14.034 | INFO | } | 20260507 14:52:14.034 | INFO | Perf report for the last 5.13859 seconds: | 20260507 14:52:14.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:14.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:52:14.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:52:14.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:14.034 | INFO | Uploaded 36GiB to GPU in 1.5867s (22.6886GiB/s) | 20260507 14:52:14.034 | INFO | Downloaded 34GiB to GPU in 1.53424s (22.1608GiB/s) | 20260507 14:52:19.173 | INFO | Analyzing chunk frames = [359424,359936) | 20260507 14:52:19.173 | INFO | MemInfo: { | 20260507 14:52:19.173 | INFO | "gpu" : " 17 GB", | 20260507 14:52:19.173 | INFO | "huge" : "", | 20260507 14:52:19.173 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:19.173 | INFO | "malloc" : " 14 GB", | 20260507 14:52:19.173 | INFO | "pinned" : " 29 GB" | 20260507 14:52:19.173 | INFO | } | 20260507 14:52:19.173 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:52:19.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:19.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.173 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:52:19.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.173 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:52:19.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:19.174 | INFO | Uploaded 36GiB to GPU in 1.58676s (22.6878GiB/s) | 20260507 14:52:19.174 | INFO | Downloaded 34GiB to GPU in 1.5353s (22.1455GiB/s) | 20260507 14:52:24.312 | INFO | Analyzing chunk frames = [359936,360448) | 20260507 14:52:24.312 | INFO | MemInfo: { | 20260507 14:52:24.312 | INFO | "gpu" : " 17 GB", | 20260507 14:52:24.312 | INFO | "huge" : "", | 20260507 14:52:24.312 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:24.312 | INFO | "malloc" : " 14 GB", | 20260507 14:52:24.312 | INFO | "pinned" : " 29 GB" | 20260507 14:52:24.312 | INFO | } | 20260507 14:52:24.312 | INFO | Perf report for the last 5.13907 seconds: | 20260507 14:52:24.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:24.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.312 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:52:24.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.313 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:52:24.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:24.313 | INFO | Uploaded 36GiB to GPU in 1.58749s (22.6773GiB/s) | 20260507 14:52:24.313 | INFO | Downloaded 34GiB to GPU in 1.53652s (22.1279GiB/s) | 20260507 14:52:24.313 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:52:24.313 | INFO | Total mem usage: 16.4 GiB | 20260507 14:52:24.313 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:52:24.313 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -6.39%) | 20260507 14:52:24.313 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:52:24.313 | INFO | High Water Breakdown: | 20260507 14:52:24.313 | INFO | 59.5% : 9.17 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:52:24.313 | INFO | 40.4% : 6.23 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:52:24.313 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:52:24.313 | INFO | ------------------------------------------ | 20260507 14:52:24.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:52:24.313 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:52:24.313 | INFO | Total mem usage: 26.5 GiB | 20260507 14:52:24.313 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:52:24.313 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:52:24.313 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:52:24.313 | INFO | High Water Breakdown: | 20260507 14:52:24.313 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:52:24.313 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:52:24.313 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:52:24.313 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:52:24.313 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:52:24.313 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:52:24.313 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:52:24.313 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:52:24.313 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:52:24.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:52:24.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:52:24.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:52:24.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:52:24.313 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:52:24.313 | INFO | ------------------------------------------ | 20260507 14:52:24.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:52:24.313 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:52:24.313 | INFO | Total mem usage: 27.8 GiB | 20260507 14:52:24.313 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:52:24.313 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:52:24.313 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:52:24.313 | INFO | High Water Breakdown: | 20260507 14:52:24.314 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:52:24.314 | INFO | ------------------------------------------ | 20260507 14:52:24.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:52:24.314 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:52:24.314 | INFO | Total mem usage: 15.7 GiB | 20260507 14:52:24.314 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:52:24.314 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:52:24.314 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:52:24.314 | INFO | High Water Breakdown: | 20260507 14:52:24.314 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:52:24.314 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:52:24.314 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:52:24.314 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:52:24.314 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:52:24.314 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:52:24.314 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:52:24.314 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:52:24.314 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:52:24.314 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:52:24.314 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:52:24.314 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:52:24.314 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:52:24.314 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:52:24.314 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:52:24.314 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:52:24.314 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:52:24.314 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:52:24.314 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:52:24.314 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:52:24.314 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:52:24.314 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:52:24.314 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:52:24.314 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:52:24.314 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:52:24.314 | INFO | ------------------------------------------ | 20260507 14:52:24.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:52:24.314 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:52:24.314 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:52:24.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:52:24.314 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.31e-03 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 11.00%, Baseline -- 3.29e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.72e+01 / -nan, 4.54e-02 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.17e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 3.56%, Metrics -- 1.07e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 1.32%, PulseSerializing -- 3.94e-01 / -nan, 5.74e-02 / 0.00e+00 | 20260507 14:52:24.314 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:52:24.314 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:52:24.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:52:24.314 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 18.57%, Upload -- 1.66e+01 / 0.00e+00, 5.10e-01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 18.17%, Download -- 1.63e+01 / 0.00e+00, 8.24e-01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 33.45%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 0.00%, LockDownload -- 4.31e-04 / 0.00e+00, 9.77e-05 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 29.23%, LockCompute -- 2.62e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 14:52:24.314 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 14:52:24.409 | INFO | Starting a round of Baz IO | 20260507 14:52:27.301 | INFO | Populating RTMetrics report | 20260507 14:52:27.302 | INFO | RTMetrics report populated | 20260507 14:52:27.302 | INFO | Opening tmp RTMetrics json file | 20260507 14:52:27.305 | INFO | Serializing RTMetrics to json | 20260507 14:52:27.333 | INFO | Serialization done, swapping tmp file now... | 20260507 14:52:27.369 | INFO | RTMetrics done | 20260507 14:52:27.394 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 2.984 seconds | 20260507 14:52:27.394 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.04% / 0.07% / 43.90% | 20260507 14:52:27.394 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.97267% / 0.000186761% / 2.78155e-05% | 20260507 14:52:27.394 | INFO | Baz IO overall: | 20260507 14:52:27.394 | INFO | BytesWritten : 136281097524 | 20260507 14:52:27.394 | INFO | NumEvents : 65122481550 | 20260507 14:52:29.452 | INFO | Analyzing chunk frames = [360448,360960) | 20260507 14:52:29.452 | INFO | MemInfo: { | 20260507 14:52:29.452 | INFO | "gpu" : " 17 GB", | 20260507 14:52:29.452 | INFO | "huge" : "", | 20260507 14:52:29.452 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:29.452 | INFO | "malloc" : " 14 GB", | 20260507 14:52:29.452 | INFO | "pinned" : " 29 GB" | 20260507 14:52:29.452 | INFO | } | 20260507 14:52:29.452 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:52:29.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:29.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:29.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:29.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:29.453 | INFO | ANALYSIS : 57.3%, 2.9, 88.0ms, 0 | 20260507 14:52:29.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:29.453 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 14:52:29.453 | INFO | PRE_HQ : 56.0%, 2.1, 59.7ms, 0 | 20260507 14:52:29.453 | INFO | BAZWRITER : 57.7%, 4.5, 131.9ms, 0 | 20260507 14:52:29.453 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 14:52:29.453 | INFO | Uploaded 36GiB to GPU in 1.64042s (21.9456GiB/s) | 20260507 14:52:29.453 | INFO | Downloaded 38GiB to GPU in 1.72212s (22.0659GiB/s) | 20260507 14:52:34.591 | INFO | Analyzing chunk frames = [360960,361472) | 20260507 14:52:34.591 | INFO | MemInfo: { | 20260507 14:52:34.591 | INFO | "gpu" : " 17 GB", | 20260507 14:52:34.591 | INFO | "huge" : "", | 20260507 14:52:34.591 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:34.591 | INFO | "malloc" : " 14 GB", | 20260507 14:52:34.591 | INFO | "pinned" : " 29 GB" | 20260507 14:52:34.591 | INFO | } | 20260507 14:52:34.591 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:52:34.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:34.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:52:34.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:52:34.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:34.591 | INFO | Uploaded 36GiB to GPU in 1.58426s (22.7236GiB/s) | 20260507 14:52:34.591 | INFO | Downloaded 34GiB to GPU in 1.53242s (22.1872GiB/s) | 20260507 14:52:39.730 | INFO | Analyzing chunk frames = [361472,361984) | 20260507 14:52:39.730 | INFO | MemInfo: { | 20260507 14:52:39.730 | INFO | "gpu" : " 17 GB", | 20260507 14:52:39.730 | INFO | "huge" : "", | 20260507 14:52:39.730 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:39.730 | INFO | "malloc" : " 14 GB", | 20260507 14:52:39.730 | INFO | "pinned" : " 29 GB" | 20260507 14:52:39.730 | INFO | } | 20260507 14:52:39.730 | INFO | Perf report for the last 5.13911 seconds: | 20260507 14:52:39.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:39.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 14:52:39.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:52:39.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:39.730 | INFO | Uploaded 36GiB to GPU in 1.58744s (22.6781GiB/s) | 20260507 14:52:39.730 | INFO | Downloaded 34GiB to GPU in 1.53789s (22.1083GiB/s) | 20260507 14:52:44.870 | INFO | Analyzing chunk frames = [361984,362496) | 20260507 14:52:44.870 | INFO | MemInfo: { | 20260507 14:52:44.870 | INFO | "gpu" : " 17 GB", | 20260507 14:52:44.871 | INFO | "huge" : "", | 20260507 14:52:44.871 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:44.871 | INFO | "malloc" : " 14 GB", | 20260507 14:52:44.871 | INFO | "pinned" : " 29 GB" | 20260507 14:52:44.871 | INFO | } | 20260507 14:52:44.871 | INFO | Perf report for the last 5.14077 seconds: | 20260507 14:52:44.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:44.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 14:52:44.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:52:44.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:44.871 | INFO | Uploaded 36GiB to GPU in 1.58843s (22.6638GiB/s) | 20260507 14:52:44.871 | INFO | Downloaded 34GiB to GPU in 1.53607s (22.1344GiB/s) | 20260507 14:52:50.010 | INFO | Analyzing chunk frames = [362496,363008) | 20260507 14:52:50.010 | INFO | MemInfo: { | 20260507 14:52:50.010 | INFO | "gpu" : " 17 GB", | 20260507 14:52:50.010 | INFO | "huge" : "", | 20260507 14:52:50.010 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:50.010 | INFO | "malloc" : " 14 GB", | 20260507 14:52:50.010 | INFO | "pinned" : " 29 GB" | 20260507 14:52:50.010 | INFO | } | 20260507 14:52:50.010 | INFO | Perf report for the last 5.13904 seconds: | 20260507 14:52:50.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:50.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:52:50.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:52:50.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:50.010 | INFO | Uploaded 36GiB to GPU in 1.58636s (22.6935GiB/s) | 20260507 14:52:50.010 | INFO | Downloaded 34GiB to GPU in 1.53542s (22.1437GiB/s) | 20260507 14:52:55.148 | INFO | Analyzing chunk frames = [363008,363520) | 20260507 14:52:55.148 | INFO | MemInfo: { | 20260507 14:52:55.148 | INFO | "gpu" : " 17 GB", | 20260507 14:52:55.148 | INFO | "huge" : "", | 20260507 14:52:55.148 | INFO | "huge_shared" : " 29 GB", | 20260507 14:52:55.148 | INFO | "malloc" : " 14 GB", | 20260507 14:52:55.148 | INFO | "pinned" : " 29 GB" | 20260507 14:52:55.148 | INFO | } | 20260507 14:52:55.148 | INFO | Perf report for the last 5.13827 seconds: | 20260507 14:52:55.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:52:55.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.148 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 14:52:55.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.148 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:52:55.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:52:55.149 | INFO | Uploaded 36GiB to GPU in 1.58675s (22.6878GiB/s) | 20260507 14:52:55.149 | INFO | Downloaded 34GiB to GPU in 1.53467s (22.1546GiB/s) | 20260507 14:53:00.287 | INFO | Analyzing chunk frames = [363520,364032) | 20260507 14:53:00.287 | INFO | MemInfo: { | 20260507 14:53:00.287 | INFO | "gpu" : " 17 GB", | 20260507 14:53:00.288 | INFO | "huge" : "", | 20260507 14:53:00.288 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:00.288 | INFO | "malloc" : " 14 GB", | 20260507 14:53:00.288 | INFO | "pinned" : " 29 GB" | 20260507 14:53:00.288 | INFO | } | 20260507 14:53:00.288 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:53:00.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:00.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:53:00.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:53:00.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:00.288 | INFO | Uploaded 36GiB to GPU in 1.5865s (22.6915GiB/s) | 20260507 14:53:00.288 | INFO | Downloaded 34GiB to GPU in 1.53537s (22.1445GiB/s) | 20260507 14:53:05.427 | INFO | Analyzing chunk frames = [364032,364544) | 20260507 14:53:05.427 | INFO | MemInfo: { | 20260507 14:53:05.427 | INFO | "gpu" : " 17 GB", | 20260507 14:53:05.427 | INFO | "huge" : "", | 20260507 14:53:05.427 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:05.427 | INFO | "malloc" : " 14 GB", | 20260507 14:53:05.427 | INFO | "pinned" : " 29 GB" | 20260507 14:53:05.427 | INFO | } | 20260507 14:53:05.427 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:53:05.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:05.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:53:05.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 14:53:05.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:05.427 | INFO | Uploaded 36GiB to GPU in 1.5872s (22.6815GiB/s) | 20260507 14:53:05.427 | INFO | Downloaded 34GiB to GPU in 1.5369s (22.1225GiB/s) | 20260507 14:53:05.525 | INFO | Starting a round of Baz IO | 20260507 14:53:08.416 | INFO | Populating RTMetrics report | 20260507 14:53:08.416 | INFO | RTMetrics report populated | 20260507 14:53:08.416 | INFO | Opening tmp RTMetrics json file | 20260507 14:53:08.423 | INFO | Serializing RTMetrics to json | 20260507 14:53:08.504 | INFO | Serialization done, swapping tmp file now... | 20260507 14:53:08.505 | INFO | RTMetrics done | 20260507 14:53:08.534 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.008 seconds | 20260507 14:53:08.534 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.97% / 0.07% / 43.96% | 20260507 14:53:08.534 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.94295% / 0.000147025% / 2.38419e-05% | 20260507 14:53:08.534 | INFO | Baz IO overall: | 20260507 14:53:08.534 | INFO | BytesWritten : 138118598492 | 20260507 14:53:08.534 | INFO | NumEvents : 66017707095 | 20260507 14:53:10.566 | INFO | Analyzing chunk frames = [364544,365056) | 20260507 14:53:10.566 | INFO | MemInfo: { | 20260507 14:53:10.566 | INFO | "gpu" : " 17 GB", | 20260507 14:53:10.566 | INFO | "huge" : "", | 20260507 14:53:10.566 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:10.566 | INFO | "malloc" : " 14 GB", | 20260507 14:53:10.566 | INFO | "pinned" : " 29 GB" | 20260507 14:53:10.567 | INFO | } | 20260507 14:53:10.567 | INFO | Perf report for the last 5.13949 seconds: | 20260507 14:53:10.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:10.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:10.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:10.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:10.567 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260507 14:53:10.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:10.567 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 14:53:10.567 | INFO | PRE_HQ : 56.0%, 2.1, 60.0ms, 0 | 20260507 14:53:10.567 | INFO | BAZWRITER : 58.1%, 4.0, 118.7ms, 0 | 20260507 14:53:10.567 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 14:53:10.567 | INFO | Uploaded 36GiB to GPU in 1.63424s (22.0285GiB/s) | 20260507 14:53:10.567 | INFO | Downloaded 38GiB to GPU in 1.71394s (22.1711GiB/s) | 20260507 14:53:15.706 | INFO | Analyzing chunk frames = [365056,365568) | 20260507 14:53:15.706 | INFO | MemInfo: { | 20260507 14:53:15.706 | INFO | "gpu" : " 17 GB", | 20260507 14:53:15.706 | INFO | "huge" : "", | 20260507 14:53:15.706 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:15.706 | INFO | "malloc" : " 14 GB", | 20260507 14:53:15.706 | INFO | "pinned" : " 29 GB" | 20260507 14:53:15.706 | INFO | } | 20260507 14:53:15.706 | INFO | Perf report for the last 5.13971 seconds: | 20260507 14:53:15.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:15.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.706 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:53:15.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.706 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:53:15.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:15.707 | INFO | Uploaded 36GiB to GPU in 1.58415s (22.7251GiB/s) | 20260507 14:53:15.707 | INFO | Downloaded 34GiB to GPU in 1.53189s (22.1948GiB/s) | 20260507 14:53:20.845 | INFO | Analyzing chunk frames = [365568,366080) | 20260507 14:53:20.845 | INFO | MemInfo: { | 20260507 14:53:20.845 | INFO | "gpu" : " 17 GB", | 20260507 14:53:20.845 | INFO | "huge" : "", | 20260507 14:53:20.845 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:20.845 | INFO | "malloc" : " 14 GB", | 20260507 14:53:20.845 | INFO | "pinned" : " 29 GB" | 20260507 14:53:20.845 | INFO | } | 20260507 14:53:20.845 | INFO | Perf report for the last 5.13876 seconds: | 20260507 14:53:20.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:20.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:53:20.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:53:20.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:20.845 | INFO | Uploaded 36GiB to GPU in 1.58786s (22.672GiB/s) | 20260507 14:53:20.845 | INFO | Downloaded 34GiB to GPU in 1.53736s (22.1158GiB/s) | 20260507 14:53:25.984 | INFO | Analyzing chunk frames = [366080,366592) | 20260507 14:53:25.984 | INFO | MemInfo: { | 20260507 14:53:25.984 | INFO | "gpu" : " 17 GB", | 20260507 14:53:25.984 | INFO | "huge" : "", | 20260507 14:53:25.984 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:25.984 | INFO | "malloc" : " 14 GB", | 20260507 14:53:25.984 | INFO | "pinned" : " 29 GB" | 20260507 14:53:25.984 | INFO | } | 20260507 14:53:25.984 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:53:25.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:25.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.984 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:53:25.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.984 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:53:25.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:25.985 | INFO | Uploaded 36GiB to GPU in 1.58815s (22.6678GiB/s) | 20260507 14:53:25.985 | INFO | Downloaded 34GiB to GPU in 1.53645s (22.1289GiB/s) | 20260507 14:53:31.142 | INFO | Analyzing chunk frames = [366592,367104) | 20260507 14:53:31.142 | INFO | MemInfo: { | 20260507 14:53:31.142 | INFO | "gpu" : " 17 GB", | 20260507 14:53:31.142 | INFO | "huge" : "", | 20260507 14:53:31.142 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:31.142 | INFO | "malloc" : " 14 GB", | 20260507 14:53:31.142 | INFO | "pinned" : " 29 GB" | 20260507 14:53:31.142 | INFO | } | 20260507 14:53:31.142 | INFO | Perf report for the last 5.15812 seconds: | 20260507 14:53:31.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:31.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | ANALYSIS : 55.9%, 3.0, 87.4ms, 0 | 20260507 14:53:31.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:53:31.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:31.143 | INFO | Uploaded 36GiB to GPU in 1.58704s (22.6837GiB/s) | 20260507 14:53:31.143 | INFO | Downloaded 34GiB to GPU in 1.53586s (22.1375GiB/s) | 20260507 14:53:36.263 | INFO | Analyzing chunk frames = [367104,367616) | 20260507 14:53:36.263 | INFO | MemInfo: { | 20260507 14:53:36.263 | INFO | "gpu" : " 17 GB", | 20260507 14:53:36.263 | INFO | "huge" : "", | 20260507 14:53:36.263 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:36.263 | INFO | "malloc" : " 14 GB", | 20260507 14:53:36.263 | INFO | "pinned" : " 29 GB" | 20260507 14:53:36.263 | INFO | } | 20260507 14:53:36.263 | INFO | Perf report for the last 5.1206 seconds: | 20260507 14:53:36.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:36.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.263 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 14:53:36.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.263 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 14:53:36.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:36.264 | INFO | Uploaded 36GiB to GPU in 1.58669s (22.6887GiB/s) | 20260507 14:53:36.264 | INFO | Downloaded 34GiB to GPU in 1.53545s (22.1433GiB/s) | 20260507 14:53:41.402 | INFO | Analyzing chunk frames = [367616,368128) | 20260507 14:53:41.402 | INFO | MemInfo: { | 20260507 14:53:41.402 | INFO | "gpu" : " 17 GB", | 20260507 14:53:41.402 | INFO | "huge" : "", | 20260507 14:53:41.402 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:41.402 | INFO | "malloc" : " 14 GB", | 20260507 14:53:41.402 | INFO | "pinned" : " 29 GB" | 20260507 14:53:41.402 | INFO | } | 20260507 14:53:41.403 | INFO | Perf report for the last 5.13938 seconds: | 20260507 14:53:41.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:41.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:53:41.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:53:41.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:41.403 | INFO | Uploaded 36GiB to GPU in 1.58673s (22.6882GiB/s) | 20260507 14:53:41.403 | INFO | Downloaded 34GiB to GPU in 1.53591s (22.1367GiB/s) | 20260507 14:53:46.542 | INFO | Analyzing chunk frames = [368128,368640) | 20260507 14:53:46.542 | INFO | MemInfo: { | 20260507 14:53:46.542 | INFO | "gpu" : " 17 GB", | 20260507 14:53:46.542 | INFO | "huge" : "", | 20260507 14:53:46.542 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:46.542 | INFO | "malloc" : " 14 GB", | 20260507 14:53:46.542 | INFO | "pinned" : " 29 GB" | 20260507 14:53:46.542 | INFO | } | 20260507 14:53:46.542 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:53:46.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:46.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:53:46.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:53:46.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:46.542 | INFO | Uploaded 36GiB to GPU in 1.58788s (22.6717GiB/s) | 20260507 14:53:46.542 | INFO | Downloaded 34GiB to GPU in 1.53683s (22.1235GiB/s) | 20260507 14:53:46.542 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:53:46.542 | INFO | Total mem usage: 16.4 GiB | 20260507 14:53:46.542 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:53:46.542 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 14:53:46.542 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:53:46.542 | INFO | High Water Breakdown: | 20260507 14:53:46.542 | INFO | 58.9% : 8.76 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:53:46.543 | INFO | 41% : 6.1 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:53:46.543 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:53:46.543 | INFO | ------------------------------------------ | 20260507 14:53:46.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:53:46.543 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:53:46.543 | INFO | Total mem usage: 26.5 GiB | 20260507 14:53:46.543 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:53:46.543 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:53:46.543 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:53:46.543 | INFO | High Water Breakdown: | 20260507 14:53:46.543 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:53:46.543 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:53:46.543 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:53:46.543 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:53:46.543 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:53:46.543 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:53:46.543 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:53:46.543 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:53:46.543 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:53:46.543 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:53:46.543 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:53:46.543 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:53:46.543 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:53:46.543 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:53:46.543 | INFO | ------------------------------------------ | 20260507 14:53:46.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:53:46.543 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:53:46.543 | INFO | Total mem usage: 27.8 GiB | 20260507 14:53:46.543 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:53:46.543 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:53:46.543 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:53:46.543 | INFO | High Water Breakdown: | 20260507 14:53:46.543 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:53:46.543 | INFO | ------------------------------------------ | 20260507 14:53:46.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:53:46.543 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:53:46.543 | INFO | Total mem usage: 15.7 GiB | 20260507 14:53:46.543 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:53:46.543 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:53:46.543 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:53:46.543 | INFO | High Water Breakdown: | 20260507 14:53:46.543 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:53:46.543 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:53:46.543 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:53:46.543 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:53:46.543 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:53:46.543 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:53:46.543 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:53:46.543 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:53:46.543 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:53:46.543 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:53:46.543 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:53:46.543 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:53:46.543 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:53:46.543 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:53:46.543 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:53:46.543 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:53:46.543 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:53:46.543 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:53:46.543 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:53:46.543 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:53:46.543 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:53:46.543 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:53:46.543 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:53:46.543 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:53:46.543 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:53:46.543 | INFO | ------------------------------------------ | 20260507 14:53:46.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:53:46.543 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:53:46.543 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:53:46.543 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:53:46.543 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:53:46.543 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.19e-03 / 0.00e+00 | 20260507 14:53:46.543 | INFO | PerfReport: 11.01%, Baseline -- 3.30e+00 / -nan, 2.01e-02 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.72e+01 / -nan, 4.32e-02 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.17e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 3.56%, Metrics -- 1.07e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 1.32%, PulseSerializing -- 3.94e-01 / -nan, 5.72e-02 / 0.00e+00 | 20260507 14:53:46.544 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:53:46.544 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:53:46.544 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:53:46.544 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 18.47%, Upload -- 1.66e+01 / 0.00e+00, 4.31e-01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 18.06%, Download -- 1.62e+01 / 0.00e+00, 7.13e-01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 33.32%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 0.00%, LockDownload -- 4.29e-04 / 0.00e+00, 9.48e-05 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 29.57%, LockCompute -- 2.66e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:53:46.544 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260507 14:53:46.639 | INFO | Starting a round of Baz IO | 20260507 14:53:49.527 | INFO | Populating RTMetrics report | 20260507 14:53:49.528 | INFO | RTMetrics report populated | 20260507 14:53:49.528 | INFO | Opening tmp RTMetrics json file | 20260507 14:53:49.580 | INFO | Serializing RTMetrics to json | 20260507 14:53:49.597 | INFO | Serialization done, swapping tmp file now... | 20260507 14:53:49.617 | INFO | RTMetrics done | 20260507 14:53:49.641 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.001 seconds | 20260507 14:53:49.641 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.91% / 0.06% / 44.03% | 20260507 14:53:49.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.93931% / 0.000166893% / 2.78155e-05% | 20260507 14:53:49.641 | INFO | Baz IO overall: | 20260507 14:53:49.641 | INFO | BytesWritten : 139959557528 | 20260507 14:53:49.641 | INFO | NumEvents : 66915612069 | 20260507 14:53:51.681 | INFO | Analyzing chunk frames = [368640,369152) | 20260507 14:53:51.681 | INFO | MemInfo: { | 20260507 14:53:51.681 | INFO | "gpu" : " 17 GB", | 20260507 14:53:51.681 | INFO | "huge" : "", | 20260507 14:53:51.681 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:51.681 | INFO | "malloc" : " 14 GB", | 20260507 14:53:51.681 | INFO | "pinned" : " 29 GB" | 20260507 14:53:51.681 | INFO | } | 20260507 14:53:51.681 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:53:51.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:51.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:51.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:51.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:51.681 | INFO | ANALYSIS : 57.2%, 2.9, 87.7ms, 0 | 20260507 14:53:51.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:51.681 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 14:53:51.681 | INFO | PRE_HQ : 56.2%, 2.1, 60.2ms, 0 | 20260507 14:53:51.681 | INFO | BAZWRITER : 58.0%, 4.4, 131.1ms, 0 | 20260507 14:53:51.681 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 14:53:51.681 | INFO | Uploaded 36GiB to GPU in 1.64239s (21.9193GiB/s) | 20260507 14:53:51.681 | INFO | Downloaded 38GiB to GPU in 1.72491s (22.0302GiB/s) | 20260507 14:53:56.820 | INFO | Analyzing chunk frames = [369152,369664) | 20260507 14:53:56.820 | INFO | MemInfo: { | 20260507 14:53:56.820 | INFO | "gpu" : " 17 GB", | 20260507 14:53:56.820 | INFO | "huge" : "", | 20260507 14:53:56.820 | INFO | "huge_shared" : " 29 GB", | 20260507 14:53:56.820 | INFO | "malloc" : " 14 GB", | 20260507 14:53:56.820 | INFO | "pinned" : " 29 GB" | 20260507 14:53:56.820 | INFO | } | 20260507 14:53:56.821 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:53:56.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:53:56.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:53:56.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:53:56.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:53:56.821 | INFO | Uploaded 36GiB to GPU in 1.58366s (22.7322GiB/s) | 20260507 14:53:56.822 | INFO | Downloaded 34GiB to GPU in 1.53277s (22.1821GiB/s) | 20260507 14:54:01.960 | INFO | Analyzing chunk frames = [369664,370176) | 20260507 14:54:01.960 | INFO | MemInfo: { | 20260507 14:54:01.960 | INFO | "gpu" : " 17 GB", | 20260507 14:54:01.960 | INFO | "huge" : "", | 20260507 14:54:01.960 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:01.960 | INFO | "malloc" : " 14 GB", | 20260507 14:54:01.960 | INFO | "pinned" : " 29 GB" | 20260507 14:54:01.960 | INFO | } | 20260507 14:54:01.960 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:54:01.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:01.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:54:01.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:54:01.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:01.960 | INFO | Uploaded 36GiB to GPU in 1.58801s (22.6698GiB/s) | 20260507 14:54:01.960 | INFO | Downloaded 34GiB to GPU in 1.53823s (22.1033GiB/s) | 20260507 14:54:07.099 | INFO | Analyzing chunk frames = [370176,370688) | 20260507 14:54:07.099 | INFO | MemInfo: { | 20260507 14:54:07.099 | INFO | "gpu" : " 17 GB", | 20260507 14:54:07.099 | INFO | "huge" : "", | 20260507 14:54:07.099 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:07.099 | INFO | "malloc" : " 14 GB", | 20260507 14:54:07.099 | INFO | "pinned" : " 29 GB" | 20260507 14:54:07.099 | INFO | } | 20260507 14:54:07.099 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:54:07.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:07.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.099 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:54:07.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.099 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:54:07.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:07.100 | INFO | Uploaded 36GiB to GPU in 1.5876s (22.6758GiB/s) | 20260507 14:54:07.100 | INFO | Downloaded 34GiB to GPU in 1.5364s (22.1297GiB/s) | 20260507 14:54:12.238 | INFO | Analyzing chunk frames = [370688,371200) | 20260507 14:54:12.238 | INFO | MemInfo: { | 20260507 14:54:12.238 | INFO | "gpu" : " 17 GB", | 20260507 14:54:12.238 | INFO | "huge" : "", | 20260507 14:54:12.238 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:12.238 | INFO | "malloc" : " 14 GB", | 20260507 14:54:12.238 | INFO | "pinned" : " 29 GB" | 20260507 14:54:12.239 | INFO | } | 20260507 14:54:12.239 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:54:12.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:12.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 14:54:12.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:54:12.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:12.239 | INFO | Uploaded 36GiB to GPU in 1.58617s (22.6962GiB/s) | 20260507 14:54:12.239 | INFO | Downloaded 34GiB to GPU in 1.53516s (22.1475GiB/s) | 20260507 14:54:17.378 | INFO | Analyzing chunk frames = [371200,371712) | 20260507 14:54:17.378 | INFO | MemInfo: { | 20260507 14:54:17.378 | INFO | "gpu" : " 17 GB", | 20260507 14:54:17.378 | INFO | "huge" : "", | 20260507 14:54:17.378 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:17.378 | INFO | "malloc" : " 14 GB", | 20260507 14:54:17.378 | INFO | "pinned" : " 29 GB" | 20260507 14:54:17.378 | INFO | } | 20260507 14:54:17.378 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:54:17.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:17.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:54:17.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:54:17.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:17.378 | INFO | Uploaded 36GiB to GPU in 1.58675s (22.6878GiB/s) | 20260507 14:54:17.378 | INFO | Downloaded 34GiB to GPU in 1.53481s (22.1526GiB/s) | 20260507 14:54:22.517 | INFO | Analyzing chunk frames = [371712,372224) | 20260507 14:54:22.517 | INFO | MemInfo: { | 20260507 14:54:22.517 | INFO | "gpu" : " 17 GB", | 20260507 14:54:22.517 | INFO | "huge" : "", | 20260507 14:54:22.517 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:22.517 | INFO | "malloc" : " 14 GB", | 20260507 14:54:22.517 | INFO | "pinned" : " 29 GB" | 20260507 14:54:22.517 | INFO | } | 20260507 14:54:22.517 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:54:22.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:22.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.517 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:54:22.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.517 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:54:22.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:22.518 | INFO | Uploaded 36GiB to GPU in 1.58631s (22.6942GiB/s) | 20260507 14:54:22.518 | INFO | Downloaded 34GiB to GPU in 1.53652s (22.128GiB/s) | 20260507 14:54:27.656 | INFO | Analyzing chunk frames = [372224,372736) | 20260507 14:54:27.656 | INFO | MemInfo: { | 20260507 14:54:27.656 | INFO | "gpu" : " 17 GB", | 20260507 14:54:27.656 | INFO | "huge" : "", | 20260507 14:54:27.656 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:27.656 | INFO | "malloc" : " 14 GB", | 20260507 14:54:27.656 | INFO | "pinned" : " 29 GB" | 20260507 14:54:27.657 | INFO | } | 20260507 14:54:27.657 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:54:27.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:27.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:54:27.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 14:54:27.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:27.657 | INFO | Uploaded 36GiB to GPU in 1.5866s (22.6901GiB/s) | 20260507 14:54:27.657 | INFO | Downloaded 34GiB to GPU in 1.53615s (22.1333GiB/s) | 20260507 14:54:27.753 | INFO | Starting a round of Baz IO | 20260507 14:54:30.642 | INFO | Populating RTMetrics report | 20260507 14:54:30.643 | INFO | RTMetrics report populated | 20260507 14:54:30.643 | INFO | Opening tmp RTMetrics json file | 20260507 14:54:30.698 | INFO | Serializing RTMetrics to json | 20260507 14:54:30.735 | INFO | Serialization done, swapping tmp file now... | 20260507 14:54:30.739 | INFO | RTMetrics done | 20260507 14:54:30.890 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.136 seconds | 20260507 14:54:30.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.84% / 0.06% / 44.09% | 20260507 14:54:30.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.89251% / 0.000186761% / 1.19209e-05% | 20260507 14:54:30.890 | INFO | Baz IO overall: | 20260507 14:54:30.890 | INFO | BytesWritten : 141799827803 | 20260507 14:54:30.890 | INFO | NumEvents : 67812559663 | 20260507 14:54:32.796 | INFO | Analyzing chunk frames = [372736,373248) | 20260507 14:54:32.796 | INFO | MemInfo: { | 20260507 14:54:32.796 | INFO | "gpu" : " 17 GB", | 20260507 14:54:32.796 | INFO | "huge" : "", | 20260507 14:54:32.796 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:32.796 | INFO | "malloc" : " 14 GB", | 20260507 14:54:32.796 | INFO | "pinned" : " 29 GB" | 20260507 14:54:32.796 | INFO | } | 20260507 14:54:32.796 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:54:32.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:32.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:32.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:32.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:32.796 | INFO | ANALYSIS : 57.2%, 2.9, 85.6ms, 0 | 20260507 14:54:32.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:32.796 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 14:54:32.796 | INFO | PRE_HQ : 56.0%, 2.0, 59.4ms, 0 | 20260507 14:54:32.796 | INFO | BAZWRITER : 60.6%, 4.7, 147.8ms, 0 | 20260507 14:54:32.796 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 14:54:32.796 | INFO | Uploaded 36GiB to GPU in 1.66664s (21.6003GiB/s) | 20260507 14:54:32.796 | INFO | Downloaded 38GiB to GPU in 1.75156s (21.695GiB/s) | 20260507 14:54:37.935 | INFO | Analyzing chunk frames = [373248,373760) | 20260507 14:54:37.935 | INFO | MemInfo: { | 20260507 14:54:37.935 | INFO | "gpu" : " 17 GB", | 20260507 14:54:37.935 | INFO | "huge" : "", | 20260507 14:54:37.935 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:37.935 | INFO | "malloc" : " 14 GB", | 20260507 14:54:37.935 | INFO | "pinned" : " 29 GB" | 20260507 14:54:37.935 | INFO | } | 20260507 14:54:37.935 | INFO | Perf report for the last 5.13944 seconds: | 20260507 14:54:37.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:37.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.935 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:54:37.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.936 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:54:37.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:37.936 | INFO | Uploaded 36GiB to GPU in 1.58461s (22.7186GiB/s) | 20260507 14:54:37.936 | INFO | Downloaded 34GiB to GPU in 1.53259s (22.1846GiB/s) | 20260507 14:54:43.074 | INFO | Analyzing chunk frames = [373760,374272) | 20260507 14:54:43.074 | INFO | MemInfo: { | 20260507 14:54:43.074 | INFO | "gpu" : " 17 GB", | 20260507 14:54:43.074 | INFO | "huge" : "", | 20260507 14:54:43.074 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:43.074 | INFO | "malloc" : " 14 GB", | 20260507 14:54:43.075 | INFO | "pinned" : " 29 GB" | 20260507 14:54:43.075 | INFO | } | 20260507 14:54:43.075 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:54:43.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:43.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:54:43.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:54:43.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:43.075 | INFO | Uploaded 36GiB to GPU in 1.58799s (22.6702GiB/s) | 20260507 14:54:43.075 | INFO | Downloaded 34GiB to GPU in 1.53782s (22.1093GiB/s) | 20260507 14:54:48.214 | INFO | Analyzing chunk frames = [374272,374784) | 20260507 14:54:48.214 | INFO | MemInfo: { | 20260507 14:54:48.214 | INFO | "gpu" : " 17 GB", | 20260507 14:54:48.214 | INFO | "huge" : "", | 20260507 14:54:48.214 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:48.214 | INFO | "malloc" : " 14 GB", | 20260507 14:54:48.214 | INFO | "pinned" : " 29 GB" | 20260507 14:54:48.214 | INFO | } | 20260507 14:54:48.214 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:54:48.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:48.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 14:54:48.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:54:48.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:48.214 | INFO | Uploaded 36GiB to GPU in 1.58816s (22.6677GiB/s) | 20260507 14:54:48.214 | INFO | Downloaded 34GiB to GPU in 1.53693s (22.1221GiB/s) | 20260507 14:54:53.353 | INFO | Analyzing chunk frames = [374784,375296) | 20260507 14:54:53.353 | INFO | MemInfo: { | 20260507 14:54:53.353 | INFO | "gpu" : " 17 GB", | 20260507 14:54:53.353 | INFO | "huge" : "", | 20260507 14:54:53.353 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:53.353 | INFO | "malloc" : " 14 GB", | 20260507 14:54:53.353 | INFO | "pinned" : " 29 GB" | 20260507 14:54:53.353 | INFO | } | 20260507 14:54:53.353 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:54:53.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:53.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.353 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 14:54:53.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.353 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:54:53.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:53.354 | INFO | Uploaded 36GiB to GPU in 1.58639s (22.693GiB/s) | 20260507 14:54:53.354 | INFO | Downloaded 34GiB to GPU in 1.53563s (22.1407GiB/s) | 20260507 14:54:58.492 | INFO | Analyzing chunk frames = [375296,375808) | 20260507 14:54:58.492 | INFO | MemInfo: { | 20260507 14:54:58.492 | INFO | "gpu" : " 17 GB", | 20260507 14:54:58.492 | INFO | "huge" : "", | 20260507 14:54:58.492 | INFO | "huge_shared" : " 29 GB", | 20260507 14:54:58.492 | INFO | "malloc" : " 14 GB", | 20260507 14:54:58.493 | INFO | "pinned" : " 29 GB" | 20260507 14:54:58.493 | INFO | } | 20260507 14:54:58.493 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:54:58.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:54:58.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 14:54:58.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:54:58.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:54:58.493 | INFO | Uploaded 36GiB to GPU in 1.58683s (22.6867GiB/s) | 20260507 14:54:58.493 | INFO | Downloaded 34GiB to GPU in 1.5342s (22.1614GiB/s) | 20260507 14:55:03.632 | INFO | Analyzing chunk frames = [375808,376320) | 20260507 14:55:03.632 | INFO | MemInfo: { | 20260507 14:55:03.632 | INFO | "gpu" : " 17 GB", | 20260507 14:55:03.632 | INFO | "huge" : "", | 20260507 14:55:03.632 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:03.632 | INFO | "malloc" : " 14 GB", | 20260507 14:55:03.632 | INFO | "pinned" : " 29 GB" | 20260507 14:55:03.632 | INFO | } | 20260507 14:55:03.632 | INFO | Perf report for the last 5.13961 seconds: | 20260507 14:55:03.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:03.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:55:03.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:55:03.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:03.632 | INFO | Uploaded 36GiB to GPU in 1.58744s (22.678GiB/s) | 20260507 14:55:03.633 | INFO | Downloaded 34GiB to GPU in 1.53664s (22.1261GiB/s) | 20260507 14:55:08.771 | INFO | Analyzing chunk frames = [376320,376832) | 20260507 14:55:08.771 | INFO | MemInfo: { | 20260507 14:55:08.771 | INFO | "gpu" : " 17 GB", | 20260507 14:55:08.771 | INFO | "huge" : "", | 20260507 14:55:08.771 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:08.771 | INFO | "malloc" : " 14 GB", | 20260507 14:55:08.771 | INFO | "pinned" : " 29 GB" | 20260507 14:55:08.771 | INFO | } | 20260507 14:55:08.771 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:55:08.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:08.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.771 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:55:08.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.772 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:55:08.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:08.772 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:55:08.772 | INFO | Total mem usage: 16.4 GiB | 20260507 14:55:08.772 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:55:08.772 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.25%) | 20260507 14:55:08.772 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:55:08.772 | INFO | High Water Breakdown: | 20260507 14:55:08.772 | INFO | Uploaded 36GiB to GPU in 1.58699s (22.6844GiB/s) | 20260507 14:55:08.772 | INFO | 59.4% : 9.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:55:08.772 | INFO | Downloaded 34GiB to GPU in 1.53614s (22.1334GiB/s) | 20260507 14:55:08.772 | INFO | 40.4% : 6.18 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:55:08.772 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:55:08.772 | INFO | ------------------------------------------ | 20260507 14:55:08.772 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:55:08.772 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:55:08.772 | INFO | Total mem usage: 26.5 GiB | 20260507 14:55:08.772 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:55:08.772 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:55:08.772 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:55:08.772 | INFO | High Water Breakdown: | 20260507 14:55:08.772 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:55:08.772 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:55:08.772 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:55:08.772 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:55:08.772 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:55:08.772 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:55:08.772 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:55:08.772 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:55:08.772 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:55:08.772 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:55:08.772 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:55:08.772 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:55:08.772 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:55:08.772 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:55:08.772 | INFO | ------------------------------------------ | 20260507 14:55:08.772 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:55:08.773 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:55:08.773 | INFO | Total mem usage: 27.8 GiB | 20260507 14:55:08.773 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:55:08.773 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:55:08.773 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:55:08.773 | INFO | High Water Breakdown: | 20260507 14:55:08.773 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:55:08.773 | INFO | ------------------------------------------ | 20260507 14:55:08.773 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:55:08.773 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:55:08.773 | INFO | Total mem usage: 15.7 GiB | 20260507 14:55:08.773 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:55:08.773 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:55:08.773 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 14:55:08.773 | INFO | High Water Breakdown: | 20260507 14:55:08.773 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:55:08.773 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:55:08.773 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:55:08.773 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:55:08.773 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:55:08.773 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:55:08.773 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:55:08.773 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:55:08.773 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:55:08.773 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:55:08.773 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:55:08.773 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:55:08.773 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:55:08.773 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:55:08.773 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:55:08.773 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:55:08.773 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:55:08.773 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:55:08.773 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:55:08.773 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:55:08.773 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:55:08.773 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:55:08.773 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:55:08.773 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:55:08.773 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:55:08.773 | INFO | ------------------------------------------ | 20260507 14:55:08.773 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:55:08.773 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:55:08.773 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:55:08.773 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:55:08.773 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.79e-03 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 11.00%, Baseline -- 3.29e+00 / -nan, 1.79e-02 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.07e+00 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.72e+01 / -nan, 4.32e-02 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.17e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 3.55%, Metrics -- 1.06e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.93e-01 / -nan, 5.78e-02 / 0.00e+00 | 20260507 14:55:08.773 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:55:08.773 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:55:08.773 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:55:08.773 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 18.54%, Upload -- 1.66e+01 / 0.00e+00, 4.90e-01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 18.14%, Download -- 1.63e+01 / 0.00e+00, 7.98e-01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 33.41%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 0.00%, LockDownload -- 4.25e-04 / 0.00e+00, 1.01e-04 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 29.33%, LockCompute -- 2.63e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:55:08.773 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 14:55:08.869 | INFO | Starting a round of Baz IO | 20260507 14:55:11.759 | INFO | Populating RTMetrics report | 20260507 14:55:11.759 | INFO | RTMetrics report populated | 20260507 14:55:11.759 | INFO | Opening tmp RTMetrics json file | 20260507 14:55:11.922 | INFO | Serializing RTMetrics to json | 20260507 14:55:11.924 | INFO | Serialization done, swapping tmp file now... | 20260507 14:55:11.926 | INFO | RTMetrics done | 20260507 14:55:11.937 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.068 seconds | 20260507 14:55:11.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.78% / 0.06% / 44.16% | 20260507 14:55:11.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.86564% / 0.000119209% / 3.97364e-06% | 20260507 14:55:11.937 | INFO | Baz IO overall: | 20260507 14:55:11.937 | INFO | BytesWritten : 143640425481 | 20260507 14:55:11.937 | INFO | NumEvents : 68709385824 | 20260507 14:55:13.910 | INFO | Analyzing chunk frames = [376832,377344) | 20260507 14:55:13.910 | INFO | MemInfo: { | 20260507 14:55:13.910 | INFO | "gpu" : " 17 GB", | 20260507 14:55:13.911 | INFO | "huge" : "", | 20260507 14:55:13.911 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:13.911 | INFO | "malloc" : " 14 GB", | 20260507 14:55:13.911 | INFO | "pinned" : " 29 GB" | 20260507 14:55:13.911 | INFO | } | 20260507 14:55:13.911 | INFO | Perf report for the last 5.13915 seconds: | 20260507 14:55:13.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:13.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:13.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:13.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:13.911 | INFO | ANALYSIS : 57.2%, 2.9, 87.6ms, 0 | 20260507 14:55:13.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:13.911 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 14:55:13.911 | INFO | PRE_HQ : 56.0%, 2.0, 59.6ms, 0 | 20260507 14:55:13.911 | INFO | BAZWRITER : 59.3%, 4.5, 135.9ms, 0 | 20260507 14:55:13.911 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.6ms, 0 | 20260507 14:55:13.911 | INFO | Uploaded 36GiB to GPU in 1.64327s (21.9075GiB/s) | 20260507 14:55:13.911 | INFO | Downloaded 38GiB to GPU in 1.72633s (22.012GiB/s) | 20260507 14:55:19.050 | INFO | Analyzing chunk frames = [377344,377856) | 20260507 14:55:19.050 | INFO | MemInfo: { | 20260507 14:55:19.050 | INFO | "gpu" : " 17 GB", | 20260507 14:55:19.050 | INFO | "huge" : "", | 20260507 14:55:19.050 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:19.050 | INFO | "malloc" : " 14 GB", | 20260507 14:55:19.050 | INFO | "pinned" : " 29 GB" | 20260507 14:55:19.050 | INFO | } | 20260507 14:55:19.050 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:55:19.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:19.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:55:19.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | PRE_HQ : 28.1%, 1.0, 14.8ms, 0 | 20260507 14:55:19.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:19.050 | INFO | Uploaded 36GiB to GPU in 1.59376s (22.5881GiB/s) | 20260507 14:55:19.050 | INFO | Downloaded 34GiB to GPU in 1.52835s (22.2462GiB/s) | 20260507 14:55:24.189 | INFO | Analyzing chunk frames = [377856,378368) | 20260507 14:55:24.189 | INFO | MemInfo: { | 20260507 14:55:24.189 | INFO | "gpu" : " 17 GB", | 20260507 14:55:24.189 | INFO | "huge" : "", | 20260507 14:55:24.189 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:24.189 | INFO | "malloc" : " 14 GB", | 20260507 14:55:24.189 | INFO | "pinned" : " 29 GB" | 20260507 14:55:24.189 | INFO | } | 20260507 14:55:24.189 | INFO | Perf report for the last 5.1392 seconds: | 20260507 14:55:24.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:24.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.189 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 14:55:24.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.189 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:55:24.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:24.190 | INFO | Uploaded 36GiB to GPU in 1.58873s (22.6597GiB/s) | 20260507 14:55:24.190 | INFO | Downloaded 34GiB to GPU in 1.53753s (22.1134GiB/s) | 20260507 14:55:29.328 | INFO | Analyzing chunk frames = [378368,378880) | 20260507 14:55:29.328 | INFO | MemInfo: { | 20260507 14:55:29.328 | INFO | "gpu" : " 17 GB", | 20260507 14:55:29.328 | INFO | "huge" : "", | 20260507 14:55:29.328 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:29.328 | INFO | "malloc" : " 14 GB", | 20260507 14:55:29.329 | INFO | "pinned" : " 29 GB" | 20260507 14:55:29.329 | INFO | } | 20260507 14:55:29.329 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:55:29.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:29.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 14:55:29.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:55:29.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:29.329 | INFO | Uploaded 36GiB to GPU in 1.58777s (22.6733GiB/s) | 20260507 14:55:29.329 | INFO | Downloaded 34GiB to GPU in 1.5376s (22.1123GiB/s) | 20260507 14:55:34.468 | INFO | Analyzing chunk frames = [378880,379392) | 20260507 14:55:34.468 | INFO | MemInfo: { | 20260507 14:55:34.468 | INFO | "gpu" : " 17 GB", | 20260507 14:55:34.468 | INFO | "huge" : "", | 20260507 14:55:34.468 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:34.468 | INFO | "malloc" : " 14 GB", | 20260507 14:55:34.468 | INFO | "pinned" : " 29 GB" | 20260507 14:55:34.468 | INFO | } | 20260507 14:55:34.468 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:55:34.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:34.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.468 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:55:34.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.468 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:55:34.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:34.469 | INFO | Uploaded 36GiB to GPU in 1.58685s (22.6864GiB/s) | 20260507 14:55:34.469 | INFO | Downloaded 34GiB to GPU in 1.53574s (22.1392GiB/s) | 20260507 14:55:39.607 | INFO | Analyzing chunk frames = [379392,379904) | 20260507 14:55:39.607 | INFO | MemInfo: { | 20260507 14:55:39.607 | INFO | "gpu" : " 17 GB", | 20260507 14:55:39.607 | INFO | "huge" : "", | 20260507 14:55:39.607 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:39.607 | INFO | "malloc" : " 14 GB", | 20260507 14:55:39.607 | INFO | "pinned" : " 29 GB" | 20260507 14:55:39.607 | INFO | } | 20260507 14:55:39.607 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:55:39.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:39.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.607 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:55:39.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.607 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:55:39.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:39.608 | INFO | Uploaded 36GiB to GPU in 1.58794s (22.6709GiB/s) | 20260507 14:55:39.608 | INFO | Downloaded 34GiB to GPU in 1.53515s (22.1476GiB/s) | 20260507 14:55:44.746 | INFO | Analyzing chunk frames = [379904,380416) | 20260507 14:55:44.746 | INFO | MemInfo: { | 20260507 14:55:44.746 | INFO | "gpu" : " 17 GB", | 20260507 14:55:44.746 | INFO | "huge" : "", | 20260507 14:55:44.747 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:44.747 | INFO | "malloc" : " 14 GB", | 20260507 14:55:44.747 | INFO | "pinned" : " 29 GB" | 20260507 14:55:44.747 | INFO | } | 20260507 14:55:44.747 | INFO | Perf report for the last 5.13941 seconds: | 20260507 14:55:44.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:44.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:55:44.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 14:55:44.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:44.747 | INFO | Uploaded 36GiB to GPU in 1.5872s (22.6814GiB/s) | 20260507 14:55:44.747 | INFO | Downloaded 34GiB to GPU in 1.53654s (22.1277GiB/s) | 20260507 14:55:49.886 | INFO | Analyzing chunk frames = [380416,380928) | 20260507 14:55:49.886 | INFO | MemInfo: { | 20260507 14:55:49.886 | INFO | "gpu" : " 17 GB", | 20260507 14:55:49.886 | INFO | "huge" : "", | 20260507 14:55:49.886 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:49.886 | INFO | "malloc" : " 14 GB", | 20260507 14:55:49.886 | INFO | "pinned" : " 29 GB" | 20260507 14:55:49.886 | INFO | } | 20260507 14:55:49.886 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:55:49.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:49.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:55:49.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 14:55:49.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:49.886 | INFO | Uploaded 36GiB to GPU in 1.58733s (22.6796GiB/s) | 20260507 14:55:49.887 | INFO | Downloaded 34GiB to GPU in 1.53675s (22.1246GiB/s) | 20260507 14:55:49.983 | INFO | Starting a round of Baz IO | 20260507 14:55:52.872 | INFO | Populating RTMetrics report | 20260507 14:55:52.872 | INFO | RTMetrics report populated | 20260507 14:55:52.872 | INFO | Opening tmp RTMetrics json file | 20260507 14:55:52.889 | INFO | Serializing RTMetrics to json | 20260507 14:55:52.912 | INFO | Serialization done, swapping tmp file now... | 20260507 14:55:52.936 | INFO | RTMetrics done | 20260507 14:55:52.986 | INFO | Finished a round of Baz IO: Wrote 1.713 GiB in 3.002 seconds | 20260507 14:55:52.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.72% / 0.06% / 44.22% | 20260507 14:55:52.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.82091% / 0.000178814% / 2.38419e-05% | 20260507 14:55:52.986 | INFO | Baz IO overall: | 20260507 14:55:52.986 | INFO | BytesWritten : 145479875207 | 20260507 14:55:52.986 | INFO | NumEvents : 69605226625 | 20260507 14:55:55.025 | INFO | Analyzing chunk frames = [380928,381440) | 20260507 14:55:55.025 | INFO | MemInfo: { | 20260507 14:55:55.025 | INFO | "gpu" : " 17 GB", | 20260507 14:55:55.025 | INFO | "huge" : "", | 20260507 14:55:55.025 | INFO | "huge_shared" : " 29 GB", | 20260507 14:55:55.025 | INFO | "malloc" : " 14 GB", | 20260507 14:55:55.025 | INFO | "pinned" : " 29 GB" | 20260507 14:55:55.025 | INFO | } | 20260507 14:55:55.025 | INFO | Perf report for the last 5.13946 seconds: | 20260507 14:55:55.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:55:55.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:55.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:55.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:55.025 | INFO | ANALYSIS : 57.2%, 2.9, 87.3ms, 0 | 20260507 14:55:55.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:55:55.025 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 14:55:55.025 | INFO | PRE_HQ : 56.0%, 2.0, 59.4ms, 0 | 20260507 14:55:55.025 | INFO | BAZWRITER : 58.0%, 4.5, 133.3ms, 0 | 20260507 14:55:55.025 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 14:55:55.026 | INFO | Uploaded 36GiB to GPU in 1.64994s (21.8189GiB/s) | 20260507 14:55:55.026 | INFO | Downloaded 38GiB to GPU in 1.7323s (21.9362GiB/s) | 20260507 14:56:00.164 | INFO | Analyzing chunk frames = [381440,381952) | 20260507 14:56:00.164 | INFO | MemInfo: { | 20260507 14:56:00.164 | INFO | "gpu" : " 17 GB", | 20260507 14:56:00.164 | INFO | "huge" : "", | 20260507 14:56:00.165 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:00.165 | INFO | "malloc" : " 14 GB", | 20260507 14:56:00.165 | INFO | "pinned" : " 29 GB" | 20260507 14:56:00.165 | INFO | } | 20260507 14:56:00.165 | INFO | Perf report for the last 5.13928 seconds: | 20260507 14:56:00.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:00.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:56:00.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:56:00.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:00.165 | INFO | Uploaded 36GiB to GPU in 1.58461s (22.7185GiB/s) | 20260507 14:56:00.165 | INFO | Downloaded 34GiB to GPU in 1.53275s (22.1823GiB/s) | 20260507 14:56:05.304 | INFO | Analyzing chunk frames = [381952,382464) | 20260507 14:56:05.304 | INFO | MemInfo: { | 20260507 14:56:05.304 | INFO | "gpu" : " 17 GB", | 20260507 14:56:05.304 | INFO | "huge" : "", | 20260507 14:56:05.304 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:05.304 | INFO | "malloc" : " 14 GB", | 20260507 14:56:05.304 | INFO | "pinned" : " 29 GB" | 20260507 14:56:05.304 | INFO | } | 20260507 14:56:05.304 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:56:05.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:05.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:56:05.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:56:05.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:05.304 | INFO | Uploaded 36GiB to GPU in 1.58755s (22.6765GiB/s) | 20260507 14:56:05.304 | INFO | Downloaded 34GiB to GPU in 1.53666s (22.1259GiB/s) | 20260507 14:56:10.443 | INFO | Analyzing chunk frames = [382464,382976) | 20260507 14:56:10.443 | INFO | MemInfo: { | 20260507 14:56:10.443 | INFO | "gpu" : " 17 GB", | 20260507 14:56:10.443 | INFO | "huge" : "", | 20260507 14:56:10.443 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:10.443 | INFO | "malloc" : " 14 GB", | 20260507 14:56:10.443 | INFO | "pinned" : " 29 GB" | 20260507 14:56:10.443 | INFO | } | 20260507 14:56:10.444 | INFO | Perf report for the last 5.13956 seconds: | 20260507 14:56:10.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:10.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 14:56:10.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:56:10.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:10.444 | INFO | Uploaded 36GiB to GPU in 1.58851s (22.6627GiB/s) | 20260507 14:56:10.444 | INFO | Downloaded 34GiB to GPU in 1.53681s (22.1237GiB/s) | 20260507 14:56:15.582 | INFO | Analyzing chunk frames = [382976,383488) | 20260507 14:56:15.583 | INFO | MemInfo: { | 20260507 14:56:15.583 | INFO | "gpu" : " 17 GB", | 20260507 14:56:15.583 | INFO | "huge" : "", | 20260507 14:56:15.583 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:15.583 | INFO | "malloc" : " 14 GB", | 20260507 14:56:15.583 | INFO | "pinned" : " 29 GB" | 20260507 14:56:15.583 | INFO | } | 20260507 14:56:15.583 | INFO | Perf report for the last 5.13918 seconds: | 20260507 14:56:15.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:15.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 14:56:15.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:56:15.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:15.583 | INFO | Uploaded 36GiB to GPU in 1.58662s (22.6897GiB/s) | 20260507 14:56:15.583 | INFO | Downloaded 34GiB to GPU in 1.53497s (22.1503GiB/s) | 20260507 14:56:20.722 | INFO | Analyzing chunk frames = [383488,384000) | 20260507 14:56:20.722 | INFO | MemInfo: { | 20260507 14:56:20.722 | INFO | "gpu" : " 17 GB", | 20260507 14:56:20.722 | INFO | "huge" : "", | 20260507 14:56:20.722 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:20.722 | INFO | "malloc" : " 14 GB", | 20260507 14:56:20.722 | INFO | "pinned" : " 29 GB" | 20260507 14:56:20.722 | INFO | } | 20260507 14:56:20.722 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:56:20.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:20.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.722 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:56:20.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.723 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:56:20.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:20.723 | INFO | Uploaded 36GiB to GPU in 1.5871s (22.6828GiB/s) | 20260507 14:56:20.723 | INFO | Downloaded 34GiB to GPU in 1.53496s (22.1504GiB/s) | 20260507 14:56:25.861 | INFO | Analyzing chunk frames = [384000,384512) | 20260507 14:56:25.861 | INFO | MemInfo: { | 20260507 14:56:25.861 | INFO | "gpu" : " 17 GB", | 20260507 14:56:25.861 | INFO | "huge" : "", | 20260507 14:56:25.861 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:25.861 | INFO | "malloc" : " 14 GB", | 20260507 14:56:25.861 | INFO | "pinned" : " 29 GB" | 20260507 14:56:25.861 | INFO | } | 20260507 14:56:25.861 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:56:25.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:25.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.861 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:56:25.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.862 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:56:25.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:25.862 | INFO | Uploaded 36GiB to GPU in 1.58702s (22.684GiB/s) | 20260507 14:56:25.862 | INFO | Downloaded 34GiB to GPU in 1.53565s (22.1405GiB/s) | 20260507 14:56:31.014 | INFO | Analyzing chunk frames = [384512,385024) | 20260507 14:56:31.014 | INFO | MemInfo: { | 20260507 14:56:31.014 | INFO | "gpu" : " 17 GB", | 20260507 14:56:31.014 | INFO | "huge" : "", | 20260507 14:56:31.014 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:31.014 | INFO | "malloc" : " 14 GB", | 20260507 14:56:31.014 | INFO | "pinned" : " 29 GB" | 20260507 14:56:31.014 | INFO | } | 20260507 14:56:31.014 | INFO | Perf report for the last 5.15266 seconds: | 20260507 14:56:31.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:31.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.014 | INFO | ANALYSIS : 56.4%, 3.0, 88.3ms, 0 | 20260507 14:56:31.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.014 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 14:56:31.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:31.015 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:56:31.015 | INFO | Total mem usage: 16.4 GiB | 20260507 14:56:31.015 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:56:31.015 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 14:56:31.015 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:56:31.015 | INFO | High Water Breakdown: | 20260507 14:56:31.015 | INFO | 58.9% : 8.76 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:56:31.015 | INFO | 41% : 6.1 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:56:31.015 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:56:31.015 | INFO | ------------------------------------------ | 20260507 14:56:31.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:56:31.015 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:56:31.015 | INFO | Total mem usage: 26.5 GiB | 20260507 14:56:31.015 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:56:31.015 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:56:31.015 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:56:31.015 | INFO | High Water Breakdown: | 20260507 14:56:31.015 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:56:31.015 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:56:31.015 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:56:31.015 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:56:31.015 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:56:31.015 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:56:31.015 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:56:31.015 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:56:31.015 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:56:31.015 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:56:31.015 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:56:31.015 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:56:31.015 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:56:31.015 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:56:31.015 | INFO | ------------------------------------------ | 20260507 14:56:31.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:56:31.015 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:56:31.015 | INFO | Total mem usage: 27.8 GiB | 20260507 14:56:31.015 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:56:31.015 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:56:31.015 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:56:31.015 | INFO | High Water Breakdown: | 20260507 14:56:31.015 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:56:31.015 | INFO | ------------------------------------------ | 20260507 14:56:31.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:56:31.015 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:56:31.015 | INFO | Total mem usage: 15.7 GiB | 20260507 14:56:31.015 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:56:31.015 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:56:31.015 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:56:31.015 | INFO | High Water Breakdown: | 20260507 14:56:31.015 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:56:31.015 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:56:31.015 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:56:31.015 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:56:31.015 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:56:31.015 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:56:31.015 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:56:31.015 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:56:31.015 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:56:31.015 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:56:31.015 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:56:31.015 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:56:31.015 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:56:31.015 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:56:31.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:56:31.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:56:31.015 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:56:31.015 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:56:31.015 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:56:31.015 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:56:31.015 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:56:31.015 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:56:31.015 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:56:31.015 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:56:31.015 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:56:31.015 | INFO | ------------------------------------------ | 20260507 14:56:31.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:56:31.015 | INFO | Uploaded 36GiB to GPU in 1.58675s (22.6879GiB/s) | 20260507 14:56:31.015 | INFO | Downloaded 34GiB to GPU in 1.53677s (22.1243GiB/s) | 20260507 14:56:31.015 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:56:31.015 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:56:31.016 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:56:31.016 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.71e-03 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 11.01%, Baseline -- 3.29e+00 / -nan, 1.87e-02 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.16e-02 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.17e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 3.54%, Metrics -- 1.06e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.92e-01 / -nan, 5.72e-02 / 0.00e+00 | 20260507 14:56:31.016 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:56:31.016 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:56:31.016 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:56:31.016 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 18.52%, Upload -- 1.66e+01 / 0.00e+00, 4.64e-01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 18.11%, Download -- 1.62e+01 / 0.00e+00, 7.63e-01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 33.37%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 0.00%, LockDownload -- 4.31e-04 / 0.00e+00, 1.20e-04 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 29.41%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:56:31.016 | INFO | PerfReport: 0.02%, Misc -- 1.36e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 14:56:31.111 | INFO | Starting a round of Baz IO | 20260507 14:56:34.007 | INFO | Populating RTMetrics report | 20260507 14:56:34.007 | INFO | RTMetrics report populated | 20260507 14:56:34.007 | INFO | Opening tmp RTMetrics json file | 20260507 14:56:34.014 | INFO | Serializing RTMetrics to json | 20260507 14:56:34.049 | INFO | Serialization done, swapping tmp file now... | 20260507 14:56:34.096 | INFO | RTMetrics done | 20260507 14:56:34.129 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.018 seconds | 20260507 14:56:34.130 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.65% / 0.06% / 44.28% | 20260507 14:56:34.130 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.81169% / 0.000198682% / 3.57628e-05% | 20260507 14:56:34.130 | INFO | Baz IO overall: | 20260507 14:56:34.130 | INFO | BytesWritten : 147322742124 | 20260507 14:56:34.130 | INFO | NumEvents : 70503355191 | 20260507 14:56:36.140 | INFO | Analyzing chunk frames = [385024,385536) | 20260507 14:56:36.140 | INFO | MemInfo: { | 20260507 14:56:36.140 | INFO | "gpu" : " 17 GB", | 20260507 14:56:36.140 | INFO | "huge" : "", | 20260507 14:56:36.140 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:36.140 | INFO | "malloc" : " 14 GB", | 20260507 14:56:36.140 | INFO | "pinned" : " 29 GB" | 20260507 14:56:36.140 | INFO | } | 20260507 14:56:36.140 | INFO | Perf report for the last 5.12591 seconds: | 20260507 14:56:36.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:36.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:36.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:36.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:36.140 | INFO | ANALYSIS : 57.5%, 3.0, 89.2ms, 0 | 20260507 14:56:36.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:36.140 | INFO | RT_METRICS : 44.1%, 1.0, 23.0ms, 0 | 20260507 14:56:36.140 | INFO | PRE_HQ : 56.2%, 2.1, 60.1ms, 0 | 20260507 14:56:36.140 | INFO | BAZWRITER : 58.5%, 4.1, 121.5ms, 0 | 20260507 14:56:36.140 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 14:56:36.140 | INFO | Uploaded 36GiB to GPU in 1.63034s (22.0812GiB/s) | 20260507 14:56:36.140 | INFO | Downloaded 38GiB to GPU in 1.71105s (22.2086GiB/s) | 20260507 14:56:41.279 | INFO | Analyzing chunk frames = [385536,386048) | 20260507 14:56:41.279 | INFO | MemInfo: { | 20260507 14:56:41.279 | INFO | "gpu" : " 17 GB", | 20260507 14:56:41.279 | INFO | "huge" : "", | 20260507 14:56:41.279 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:41.279 | INFO | "malloc" : " 14 GB", | 20260507 14:56:41.279 | INFO | "pinned" : " 29 GB" | 20260507 14:56:41.279 | INFO | } | 20260507 14:56:41.280 | INFO | Perf report for the last 5.13952 seconds: | 20260507 14:56:41.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:41.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:56:41.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:56:41.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:41.280 | INFO | Uploaded 36GiB to GPU in 1.58433s (22.7226GiB/s) | 20260507 14:56:41.280 | INFO | Downloaded 34GiB to GPU in 1.53225s (22.1896GiB/s) | 20260507 14:56:46.419 | INFO | Analyzing chunk frames = [386048,386560) | 20260507 14:56:46.419 | INFO | MemInfo: { | 20260507 14:56:46.419 | INFO | "gpu" : " 17 GB", | 20260507 14:56:46.419 | INFO | "huge" : "", | 20260507 14:56:46.419 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:46.419 | INFO | "malloc" : " 14 GB", | 20260507 14:56:46.419 | INFO | "pinned" : " 29 GB" | 20260507 14:56:46.419 | INFO | } | 20260507 14:56:46.419 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:56:46.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:46.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:56:46.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:56:46.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:46.419 | INFO | Uploaded 36GiB to GPU in 1.58969s (22.6459GiB/s) | 20260507 14:56:46.419 | INFO | Downloaded 34GiB to GPU in 1.53944s (22.0859GiB/s) | 20260507 14:56:51.558 | INFO | Analyzing chunk frames = [386560,387072) | 20260507 14:56:51.558 | INFO | MemInfo: { | 20260507 14:56:51.558 | INFO | "gpu" : " 17 GB", | 20260507 14:56:51.558 | INFO | "huge" : "", | 20260507 14:56:51.558 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:51.558 | INFO | "malloc" : " 14 GB", | 20260507 14:56:51.558 | INFO | "pinned" : " 29 GB" | 20260507 14:56:51.558 | INFO | } | 20260507 14:56:51.558 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:56:51.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:51.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.558 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:56:51.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.558 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:56:51.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:51.559 | INFO | Uploaded 36GiB to GPU in 1.58738s (22.6789GiB/s) | 20260507 14:56:51.559 | INFO | Downloaded 34GiB to GPU in 1.53666s (22.1258GiB/s) | 20260507 14:56:56.697 | INFO | Analyzing chunk frames = [387072,387584) | 20260507 14:56:56.697 | INFO | MemInfo: { | 20260507 14:56:56.697 | INFO | "gpu" : " 17 GB", | 20260507 14:56:56.697 | INFO | "huge" : "", | 20260507 14:56:56.697 | INFO | "huge_shared" : " 29 GB", | 20260507 14:56:56.697 | INFO | "malloc" : " 14 GB", | 20260507 14:56:56.697 | INFO | "pinned" : " 29 GB" | 20260507 14:56:56.697 | INFO | } | 20260507 14:56:56.698 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:56:56.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:56:56.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 14:56:56.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:56:56.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:56:56.698 | INFO | Uploaded 36GiB to GPU in 1.58689s (22.6859GiB/s) | 20260507 14:56:56.698 | INFO | Downloaded 34GiB to GPU in 1.53629s (22.1312GiB/s) | 20260507 14:57:01.837 | INFO | Analyzing chunk frames = [387584,388096) | 20260507 14:57:01.837 | INFO | MemInfo: { | 20260507 14:57:01.837 | INFO | "gpu" : " 17 GB", | 20260507 14:57:01.837 | INFO | "huge" : "", | 20260507 14:57:01.837 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:01.837 | INFO | "malloc" : " 14 GB", | 20260507 14:57:01.837 | INFO | "pinned" : " 29 GB" | 20260507 14:57:01.837 | INFO | } | 20260507 14:57:01.837 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:57:01.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:01.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:57:01.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:57:01.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:01.837 | INFO | Uploaded 36GiB to GPU in 1.58791s (22.6714GiB/s) | 20260507 14:57:01.837 | INFO | Downloaded 34GiB to GPU in 1.53464s (22.155GiB/s) | 20260507 14:57:06.976 | INFO | Analyzing chunk frames = [388096,388608) | 20260507 14:57:06.976 | INFO | MemInfo: { | 20260507 14:57:06.976 | INFO | "gpu" : " 17 GB", | 20260507 14:57:06.976 | INFO | "huge" : "", | 20260507 14:57:06.976 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:06.976 | INFO | "malloc" : " 14 GB", | 20260507 14:57:06.976 | INFO | "pinned" : " 29 GB" | 20260507 14:57:06.976 | INFO | } | 20260507 14:57:06.976 | INFO | Perf report for the last 5.13925 seconds: | 20260507 14:57:06.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:06.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.976 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:57:06.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.976 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:57:06.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:06.977 | INFO | Uploaded 36GiB to GPU in 1.5854s (22.7072GiB/s) | 20260507 14:57:06.977 | INFO | Downloaded 34GiB to GPU in 1.53668s (22.1256GiB/s) | 20260507 14:57:12.115 | INFO | Analyzing chunk frames = [388608,389120) | 20260507 14:57:12.115 | INFO | MemInfo: { | 20260507 14:57:12.115 | INFO | "gpu" : " 17 GB", | 20260507 14:57:12.115 | INFO | "huge" : "", | 20260507 14:57:12.115 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:12.115 | INFO | "malloc" : " 14 GB", | 20260507 14:57:12.115 | INFO | "pinned" : " 29 GB" | 20260507 14:57:12.115 | INFO | } | 20260507 14:57:12.116 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:57:12.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:12.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 14:57:12.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:57:12.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:12.116 | INFO | Uploaded 36GiB to GPU in 1.58725s (22.6807GiB/s) | 20260507 14:57:12.116 | INFO | Downloaded 34GiB to GPU in 1.53674s (22.1247GiB/s) | 20260507 14:57:12.213 | INFO | Starting a round of Baz IO | 20260507 14:57:15.100 | INFO | Populating RTMetrics report | 20260507 14:57:15.101 | INFO | RTMetrics report populated | 20260507 14:57:15.101 | INFO | Opening tmp RTMetrics json file | 20260507 14:57:15.200 | INFO | Serializing RTMetrics to json | 20260507 14:57:15.300 | INFO | Serialization done, swapping tmp file now... | 20260507 14:57:15.326 | INFO | RTMetrics done | 20260507 14:57:15.535 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.322 seconds | 20260507 14:57:15.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.59% / 0.06% / 44.35% | 20260507 14:57:15.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.80856% / 0.000127157% / 2.38419e-05% | 20260507 14:57:15.535 | INFO | Baz IO overall: | 20260507 14:57:15.535 | INFO | BytesWritten : 149168381498 | 20260507 14:57:15.535 | INFO | NumEvents : 71403179080 | 20260507 14:57:17.255 | INFO | Analyzing chunk frames = [389120,389632) | 20260507 14:57:17.255 | INFO | MemInfo: { | 20260507 14:57:17.255 | INFO | "gpu" : " 17 GB", | 20260507 14:57:17.255 | INFO | "huge" : "", | 20260507 14:57:17.255 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:17.255 | INFO | "malloc" : " 14 GB", | 20260507 14:57:17.255 | INFO | "pinned" : " 29 GB" | 20260507 14:57:17.255 | INFO | } | 20260507 14:57:17.255 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:57:17.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:17.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:17.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:17.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:17.255 | INFO | ANALYSIS : 57.2%, 2.9, 87.2ms, 0 | 20260507 14:57:17.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:17.255 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 14:57:17.255 | INFO | PRE_HQ : 56.1%, 2.1, 60.2ms, 0 | 20260507 14:57:17.255 | INFO | BAZWRITER : 64.2%, 4.5, 148.6ms, 0 | 20260507 14:57:17.255 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260507 14:57:17.255 | INFO | Uploaded 36GiB to GPU in 1.64556s (21.877GiB/s) | 20260507 14:57:17.255 | INFO | Downloaded 38GiB to GPU in 1.72772s (21.9943GiB/s) | 20260507 14:57:22.394 | INFO | Analyzing chunk frames = [389632,390144) | 20260507 14:57:22.394 | INFO | MemInfo: { | 20260507 14:57:22.394 | INFO | "gpu" : " 17 GB", | 20260507 14:57:22.394 | INFO | "huge" : "", | 20260507 14:57:22.394 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:22.394 | INFO | "malloc" : " 14 GB", | 20260507 14:57:22.394 | INFO | "pinned" : " 29 GB" | 20260507 14:57:22.394 | INFO | } | 20260507 14:57:22.394 | INFO | Perf report for the last 5.13943 seconds: | 20260507 14:57:22.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:22.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.394 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 14:57:22.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.395 | INFO | PRE_HQ : 28.3%, 1.0, 14.9ms, 0 | 20260507 14:57:22.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:22.395 | INFO | Uploaded 36GiB to GPU in 1.58441s (22.7214GiB/s) | 20260507 14:57:22.395 | INFO | Downloaded 34GiB to GPU in 1.53279s (22.1818GiB/s) | 20260507 14:57:27.533 | INFO | Analyzing chunk frames = [390144,390656) | 20260507 14:57:27.533 | INFO | MemInfo: { | 20260507 14:57:27.533 | INFO | "gpu" : " 17 GB", | 20260507 14:57:27.533 | INFO | "huge" : "", | 20260507 14:57:27.533 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:27.533 | INFO | "malloc" : " 14 GB", | 20260507 14:57:27.533 | INFO | "pinned" : " 29 GB" | 20260507 14:57:27.533 | INFO | } | 20260507 14:57:27.534 | INFO | Perf report for the last 5.13916 seconds: | 20260507 14:57:27.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:27.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 14:57:27.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:57:27.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:27.534 | INFO | Uploaded 36GiB to GPU in 1.58811s (22.6685GiB/s) | 20260507 14:57:27.534 | INFO | Downloaded 34GiB to GPU in 1.53757s (22.1129GiB/s) | 20260507 14:57:32.673 | INFO | Analyzing chunk frames = [390656,391168) | 20260507 14:57:32.673 | INFO | MemInfo: { | 20260507 14:57:32.673 | INFO | "gpu" : " 17 GB", | 20260507 14:57:32.673 | INFO | "huge" : "", | 20260507 14:57:32.673 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:32.673 | INFO | "malloc" : " 14 GB", | 20260507 14:57:32.673 | INFO | "pinned" : " 29 GB" | 20260507 14:57:32.673 | INFO | } | 20260507 14:57:32.673 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:57:32.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:32.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 14:57:32.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 14:57:32.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:32.673 | INFO | Uploaded 36GiB to GPU in 1.58924s (22.6524GiB/s) | 20260507 14:57:32.673 | INFO | Downloaded 34GiB to GPU in 1.53752s (22.1135GiB/s) | 20260507 14:57:37.812 | INFO | Analyzing chunk frames = [391168,391680) | 20260507 14:57:37.812 | INFO | MemInfo: { | 20260507 14:57:37.812 | INFO | "gpu" : " 17 GB", | 20260507 14:57:37.812 | INFO | "huge" : "", | 20260507 14:57:37.812 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:37.812 | INFO | "malloc" : " 14 GB", | 20260507 14:57:37.812 | INFO | "pinned" : " 29 GB" | 20260507 14:57:37.812 | INFO | } | 20260507 14:57:37.812 | INFO | Perf report for the last 5.13923 seconds: | 20260507 14:57:37.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:37.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.812 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 14:57:37.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.812 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:57:37.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:37.813 | INFO | Uploaded 36GiB to GPU in 1.587s (22.6844GiB/s) | 20260507 14:57:37.813 | INFO | Downloaded 34GiB to GPU in 1.53577s (22.1388GiB/s) | 20260507 14:57:42.951 | INFO | Analyzing chunk frames = [391680,392192) | 20260507 14:57:42.951 | INFO | MemInfo: { | 20260507 14:57:42.951 | INFO | "gpu" : " 17 GB", | 20260507 14:57:42.951 | INFO | "huge" : "", | 20260507 14:57:42.951 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:42.951 | INFO | "malloc" : " 14 GB", | 20260507 14:57:42.951 | INFO | "pinned" : " 29 GB" | 20260507 14:57:42.951 | INFO | } | 20260507 14:57:42.952 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:57:42.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:42.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 14:57:42.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:57:42.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:42.952 | INFO | Uploaded 36GiB to GPU in 1.58617s (22.6962GiB/s) | 20260507 14:57:42.952 | INFO | Downloaded 34GiB to GPU in 1.53569s (22.1399GiB/s) | 20260507 14:57:48.091 | INFO | Analyzing chunk frames = [392192,392704) | 20260507 14:57:48.091 | INFO | MemInfo: { | 20260507 14:57:48.091 | INFO | "gpu" : " 17 GB", | 20260507 14:57:48.091 | INFO | "huge" : "", | 20260507 14:57:48.091 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:48.091 | INFO | "malloc" : " 14 GB", | 20260507 14:57:48.091 | INFO | "pinned" : " 29 GB" | 20260507 14:57:48.091 | INFO | } | 20260507 14:57:48.091 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:57:48.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:48.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:57:48.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 14:57:48.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:48.091 | INFO | Uploaded 36GiB to GPU in 1.58755s (22.6765GiB/s) | 20260507 14:57:48.091 | INFO | Downloaded 34GiB to GPU in 1.53679s (22.124GiB/s) | 20260507 14:57:53.230 | INFO | Analyzing chunk frames = [392704,393216) | 20260507 14:57:53.230 | INFO | MemInfo: { | 20260507 14:57:53.230 | INFO | "gpu" : " 17 GB", | 20260507 14:57:53.230 | INFO | "huge" : "", | 20260507 14:57:53.230 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:53.230 | INFO | "malloc" : " 14 GB", | 20260507 14:57:53.230 | INFO | "pinned" : " 29 GB" | 20260507 14:57:53.230 | INFO | } | 20260507 14:57:53.230 | INFO | Perf report for the last 5.13942 seconds: | 20260507 14:57:53.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:53.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.230 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 14:57:53.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.230 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:57:53.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:53.231 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:57:53.231 | INFO | Total mem usage: 16.4 GiB | 20260507 14:57:53.231 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:57:53.231 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.67%) | 20260507 14:57:53.231 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:57:53.231 | INFO | High Water Breakdown: | 20260507 14:57:53.231 | INFO | Uploaded 36GiB to GPU in 1.5874s (22.6786GiB/s) | 20260507 14:57:53.231 | INFO | 59.6% : 9.26 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:57:53.231 | INFO | Downloaded 34GiB to GPU in 1.5361s (22.134GiB/s) | 20260507 14:57:53.231 | INFO | 40.2% : 6.25 GiB ( 93% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:57:53.231 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:57:53.231 | INFO | ------------------------------------------ | 20260507 14:57:53.231 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:57:53.231 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:57:53.231 | INFO | Total mem usage: 26.5 GiB | 20260507 14:57:53.231 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:57:53.231 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:57:53.231 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:57:53.231 | INFO | High Water Breakdown: | 20260507 14:57:53.231 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:57:53.231 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:57:53.231 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:57:53.231 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:57:53.231 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:57:53.231 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:57:53.231 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:57:53.231 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:57:53.231 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:57:53.231 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:57:53.231 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:57:53.231 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:57:53.231 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:57:53.232 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:57:53.232 | INFO | ------------------------------------------ | 20260507 14:57:53.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:57:53.232 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:57:53.232 | INFO | Total mem usage: 27.8 GiB | 20260507 14:57:53.232 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:57:53.232 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:57:53.232 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:57:53.232 | INFO | High Water Breakdown: | 20260507 14:57:53.232 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:57:53.232 | INFO | ------------------------------------------ | 20260507 14:57:53.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:57:53.232 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:57:53.232 | INFO | Total mem usage: 15.7 GiB | 20260507 14:57:53.232 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:57:53.232 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:57:53.232 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:57:53.232 | INFO | High Water Breakdown: | 20260507 14:57:53.232 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:57:53.232 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:57:53.232 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:57:53.232 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:57:53.232 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:57:53.232 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:57:53.232 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:57:53.232 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:57:53.232 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:57:53.232 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:57:53.232 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:57:53.232 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:57:53.232 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:57:53.232 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:57:53.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:57:53.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:57:53.232 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:57:53.232 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:57:53.232 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:57:53.232 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:57:53.232 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:57:53.232 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:57:53.232 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:57:53.232 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:57:53.232 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:57:53.232 | INFO | ------------------------------------------ | 20260507 14:57:53.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:57:53.232 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:57:53.232 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:57:53.232 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:57:53.232 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.66e-03 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 11.02%, Baseline -- 3.30e+00 / -nan, 9.74e-02 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 16.13%, DME -- 3.86e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.72e+01 / -nan, 4.38e-02 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.17e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 3.54%, Metrics -- 1.06e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.92e-01 / -nan, 5.71e-02 / 0.00e+00 | 20260507 14:57:53.232 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:57:53.232 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:57:53.232 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:57:53.232 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 18.48%, Upload -- 1.66e+01 / 0.00e+00, 4.37e-01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 18.09%, Download -- 1.62e+01 / 0.00e+00, 7.21e-01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 33.34%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 14:57:53.232 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:57:53.233 | INFO | PerfReport: 0.00%, LockDownload -- 4.24e-04 / 0.00e+00, 9.43e-05 / 0.00e+00 | 20260507 14:57:53.233 | INFO | PerfReport: 29.51%, LockCompute -- 2.65e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 14:57:53.233 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 14:57:53.327 | INFO | Starting a round of Baz IO | 20260507 14:57:56.213 | INFO | Populating RTMetrics report | 20260507 14:57:56.214 | INFO | RTMetrics report populated | 20260507 14:57:56.214 | INFO | Opening tmp RTMetrics json file | 20260507 14:57:56.216 | INFO | Serializing RTMetrics to json | 20260507 14:57:56.238 | INFO | Serialization done, swapping tmp file now... | 20260507 14:57:56.245 | INFO | RTMetrics done | 20260507 14:57:56.305 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 2.977 seconds | 20260507 14:57:56.305 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.53% / 0.06% / 44.41% | 20260507 14:57:56.305 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.76956% / 0.000135104% / 3.17891e-05% | 20260507 14:57:56.305 | INFO | Baz IO overall: | 20260507 14:57:56.305 | INFO | BytesWritten : 151013538401 | 20260507 14:57:56.305 | INFO | NumEvents : 72302228466 | 20260507 14:57:58.369 | INFO | Analyzing chunk frames = [393216,393728) | 20260507 14:57:58.369 | INFO | MemInfo: { | 20260507 14:57:58.369 | INFO | "gpu" : " 17 GB", | 20260507 14:57:58.369 | INFO | "huge" : "", | 20260507 14:57:58.369 | INFO | "huge_shared" : " 29 GB", | 20260507 14:57:58.369 | INFO | "malloc" : " 14 GB", | 20260507 14:57:58.369 | INFO | "pinned" : " 29 GB" | 20260507 14:57:58.369 | INFO | } | 20260507 14:57:58.370 | INFO | Perf report for the last 5.13922 seconds: | 20260507 14:57:58.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:57:58.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:58.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:58.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:58.370 | INFO | ANALYSIS : 57.2%, 2.9, 87.0ms, 0 | 20260507 14:57:58.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:57:58.370 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 14:57:58.370 | INFO | PRE_HQ : 56.0%, 2.1, 59.9ms, 0 | 20260507 14:57:58.370 | INFO | BAZWRITER : 57.5%, 4.4, 131.0ms, 0 | 20260507 14:57:58.370 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 14:57:58.370 | INFO | Uploaded 36GiB to GPU in 1.64111s (21.9364GiB/s) | 20260507 14:57:58.370 | INFO | Downloaded 38GiB to GPU in 1.72508s (22.028GiB/s) | 20260507 14:58:03.509 | INFO | Analyzing chunk frames = [393728,394240) | 20260507 14:58:03.509 | INFO | MemInfo: { | 20260507 14:58:03.509 | INFO | "gpu" : " 17 GB", | 20260507 14:58:03.509 | INFO | "huge" : "", | 20260507 14:58:03.509 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:03.509 | INFO | "malloc" : " 14 GB", | 20260507 14:58:03.509 | INFO | "pinned" : " 29 GB" | 20260507 14:58:03.509 | INFO | } | 20260507 14:58:03.509 | INFO | Perf report for the last 5.13955 seconds: | 20260507 14:58:03.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:03.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 14:58:03.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:58:03.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:03.509 | INFO | Uploaded 36GiB to GPU in 1.60419s (22.4412GiB/s) | 20260507 14:58:03.510 | INFO | Downloaded 34GiB to GPU in 1.52399s (22.3098GiB/s) | 20260507 14:58:08.648 | INFO | Analyzing chunk frames = [394240,394752) | 20260507 14:58:08.648 | INFO | MemInfo: { | 20260507 14:58:08.648 | INFO | "gpu" : " 17 GB", | 20260507 14:58:08.648 | INFO | "huge" : "", | 20260507 14:58:08.648 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:08.648 | INFO | "malloc" : " 14 GB", | 20260507 14:58:08.648 | INFO | "pinned" : " 29 GB" | 20260507 14:58:08.648 | INFO | } | 20260507 14:58:08.648 | INFO | Perf report for the last 5.13927 seconds: | 20260507 14:58:08.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:08.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.648 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 14:58:08.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.649 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:58:08.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:08.649 | INFO | Uploaded 36GiB to GPU in 1.60824s (22.3847GiB/s) | 20260507 14:58:08.649 | INFO | Downloaded 34GiB to GPU in 1.52831s (22.2468GiB/s) | 20260507 14:58:13.787 | INFO | Analyzing chunk frames = [394752,395264) | 20260507 14:58:13.787 | INFO | MemInfo: { | 20260507 14:58:13.787 | INFO | "gpu" : " 17 GB", | 20260507 14:58:13.788 | INFO | "huge" : "", | 20260507 14:58:13.788 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:13.788 | INFO | "malloc" : " 14 GB", | 20260507 14:58:13.788 | INFO | "pinned" : " 29 GB" | 20260507 14:58:13.788 | INFO | } | 20260507 14:58:13.788 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:58:13.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:13.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 14:58:13.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:58:13.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:13.788 | INFO | Uploaded 36GiB to GPU in 1.60824s (22.3847GiB/s) | 20260507 14:58:13.788 | INFO | Downloaded 34GiB to GPU in 1.52838s (22.2458GiB/s) | 20260507 14:58:18.927 | INFO | Analyzing chunk frames = [395264,395776) | 20260507 14:58:18.927 | INFO | MemInfo: { | 20260507 14:58:18.927 | INFO | "gpu" : " 17 GB", | 20260507 14:58:18.927 | INFO | "huge" : "", | 20260507 14:58:18.927 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:18.927 | INFO | "malloc" : " 14 GB", | 20260507 14:58:18.927 | INFO | "pinned" : " 29 GB" | 20260507 14:58:18.927 | INFO | } | 20260507 14:58:18.927 | INFO | Perf report for the last 5.13929 seconds: | 20260507 14:58:18.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:18.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 14:58:18.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 14:58:18.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:18.927 | INFO | Uploaded 36GiB to GPU in 1.6076s (22.3936GiB/s) | 20260507 14:58:18.927 | INFO | Downloaded 34GiB to GPU in 1.52723s (22.2625GiB/s) | 20260507 14:58:24.066 | INFO | Analyzing chunk frames = [395776,396288) | 20260507 14:58:24.066 | INFO | MemInfo: { | 20260507 14:58:24.066 | INFO | "gpu" : " 17 GB", | 20260507 14:58:24.066 | INFO | "huge" : "", | 20260507 14:58:24.066 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:24.066 | INFO | "malloc" : " 14 GB", | 20260507 14:58:24.066 | INFO | "pinned" : " 29 GB" | 20260507 14:58:24.066 | INFO | } | 20260507 14:58:24.066 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:58:24.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:24.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.066 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 14:58:24.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.067 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:58:24.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:24.067 | INFO | Uploaded 36GiB to GPU in 1.60677s (22.4052GiB/s) | 20260507 14:58:24.067 | INFO | Downloaded 34GiB to GPU in 1.52583s (22.2829GiB/s) | 20260507 14:58:29.206 | INFO | Analyzing chunk frames = [396288,396800) | 20260507 14:58:29.206 | INFO | MemInfo: { | 20260507 14:58:29.206 | INFO | "gpu" : " 17 GB", | 20260507 14:58:29.206 | INFO | "huge" : "", | 20260507 14:58:29.206 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:29.206 | INFO | "malloc" : " 14 GB", | 20260507 14:58:29.206 | INFO | "pinned" : " 29 GB" | 20260507 14:58:29.206 | INFO | } | 20260507 14:58:29.206 | INFO | Perf report for the last 5.13981 seconds: | 20260507 14:58:29.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:29.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:58:29.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:58:29.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:29.206 | INFO | Uploaded 36GiB to GPU in 1.60721s (22.3991GiB/s) | 20260507 14:58:29.206 | INFO | Downloaded 34GiB to GPU in 1.52682s (22.2685GiB/s) | 20260507 14:58:34.345 | INFO | Analyzing chunk frames = [396800,397312) | 20260507 14:58:34.345 | INFO | MemInfo: { | 20260507 14:58:34.345 | INFO | "gpu" : " 17 GB", | 20260507 14:58:34.345 | INFO | "huge" : "", | 20260507 14:58:34.345 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:34.345 | INFO | "malloc" : " 14 GB", | 20260507 14:58:34.345 | INFO | "pinned" : " 29 GB" | 20260507 14:58:34.345 | INFO | } | 20260507 14:58:34.345 | INFO | Perf report for the last 5.13883 seconds: | 20260507 14:58:34.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:34.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.345 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 14:58:34.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.345 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 14:58:34.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:34.346 | INFO | Uploaded 36GiB to GPU in 1.60762s (22.3934GiB/s) | 20260507 14:58:34.346 | INFO | Downloaded 34GiB to GPU in 1.52762s (22.2568GiB/s) | 20260507 14:58:34.442 | INFO | Starting a round of Baz IO | 20260507 14:58:37.334 | INFO | Populating RTMetrics report | 20260507 14:58:37.334 | INFO | RTMetrics report populated | 20260507 14:58:37.334 | INFO | Opening tmp RTMetrics json file | 20260507 14:58:37.335 | INFO | Serializing RTMetrics to json | 20260507 14:58:37.338 | INFO | Serialization done, swapping tmp file now... | 20260507 14:58:37.339 | INFO | RTMetrics done | 20260507 14:58:37.473 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.031 seconds | 20260507 14:58:37.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.47% / 0.06% / 44.47% | 20260507 14:58:37.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.738% / 0.000150998% / 2.38419e-05% | 20260507 14:58:37.473 | INFO | Baz IO overall: | 20260507 14:58:37.474 | INFO | BytesWritten : 152859582290 | 20260507 14:58:37.474 | INFO | NumEvents : 73201363601 | 20260507 14:58:39.484 | INFO | Analyzing chunk frames = [397312,397824) | 20260507 14:58:39.484 | INFO | MemInfo: { | 20260507 14:58:39.484 | INFO | "gpu" : " 17 GB", | 20260507 14:58:39.484 | INFO | "huge" : "", | 20260507 14:58:39.484 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:39.484 | INFO | "malloc" : " 14 GB", | 20260507 14:58:39.484 | INFO | "pinned" : " 29 GB" | 20260507 14:58:39.484 | INFO | } | 20260507 14:58:39.484 | INFO | Perf report for the last 5.13935 seconds: | 20260507 14:58:39.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:39.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:39.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:39.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:39.485 | INFO | ANALYSIS : 57.3%, 2.9, 88.3ms, 0 | 20260507 14:58:39.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:39.485 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 14:58:39.485 | INFO | PRE_HQ : 55.9%, 2.1, 59.8ms, 0 | 20260507 14:58:39.485 | INFO | BAZWRITER : 58.4%, 4.2, 127.4ms, 0 | 20260507 14:58:39.485 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 14:58:39.485 | INFO | Uploaded 36GiB to GPU in 1.64215s (21.9225GiB/s) | 20260507 14:58:39.485 | INFO | Downloaded 38GiB to GPU in 1.72238s (22.0625GiB/s) | 20260507 14:58:44.624 | INFO | Analyzing chunk frames = [397824,398336) | 20260507 14:58:44.624 | INFO | MemInfo: { | 20260507 14:58:44.624 | INFO | "gpu" : " 17 GB", | 20260507 14:58:44.624 | INFO | "huge" : "", | 20260507 14:58:44.624 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:44.624 | INFO | "malloc" : " 14 GB", | 20260507 14:58:44.624 | INFO | "pinned" : " 29 GB" | 20260507 14:58:44.624 | INFO | } | 20260507 14:58:44.624 | INFO | Perf report for the last 5.13932 seconds: | 20260507 14:58:44.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:44.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:58:44.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:58:44.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:44.624 | INFO | Uploaded 36GiB to GPU in 1.58462s (22.7184GiB/s) | 20260507 14:58:44.624 | INFO | Downloaded 34GiB to GPU in 1.5335s (22.1715GiB/s) | 20260507 14:58:49.763 | INFO | Analyzing chunk frames = [398336,398848) | 20260507 14:58:49.763 | INFO | MemInfo: { | 20260507 14:58:49.763 | INFO | "gpu" : " 17 GB", | 20260507 14:58:49.763 | INFO | "huge" : "", | 20260507 14:58:49.763 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:49.763 | INFO | "malloc" : " 14 GB", | 20260507 14:58:49.763 | INFO | "pinned" : " 29 GB" | 20260507 14:58:49.763 | INFO | } | 20260507 14:58:49.763 | INFO | Perf report for the last 5.13934 seconds: | 20260507 14:58:49.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:49.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.763 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 14:58:49.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.763 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:58:49.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:49.764 | INFO | Uploaded 36GiB to GPU in 1.58855s (22.6622GiB/s) | 20260507 14:58:49.764 | INFO | Downloaded 34GiB to GPU in 1.537s (22.121GiB/s) | 20260507 14:58:54.902 | INFO | Analyzing chunk frames = [398848,399360) | 20260507 14:58:54.902 | INFO | MemInfo: { | 20260507 14:58:54.902 | INFO | "gpu" : " 17 GB", | 20260507 14:58:54.902 | INFO | "huge" : "", | 20260507 14:58:54.902 | INFO | "huge_shared" : " 29 GB", | 20260507 14:58:54.902 | INFO | "malloc" : " 14 GB", | 20260507 14:58:54.902 | INFO | "pinned" : " 29 GB" | 20260507 14:58:54.902 | INFO | } | 20260507 14:58:54.902 | INFO | Perf report for the last 5.13933 seconds: | 20260507 14:58:54.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:58:54.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.903 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 14:58:54.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.903 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 14:58:54.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:58:54.903 | INFO | Uploaded 36GiB to GPU in 1.58844s (22.6638GiB/s) | 20260507 14:58:54.903 | INFO | Downloaded 34GiB to GPU in 1.53759s (22.1125GiB/s) | 20260507 14:59:00.042 | INFO | Analyzing chunk frames = [399360,399872) | 20260507 14:59:00.042 | INFO | MemInfo: { | 20260507 14:59:00.042 | INFO | "gpu" : " 17 GB", | 20260507 14:59:00.042 | INFO | "huge" : "", | 20260507 14:59:00.042 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:00.042 | INFO | "malloc" : " 14 GB", | 20260507 14:59:00.042 | INFO | "pinned" : " 29 GB" | 20260507 14:59:00.042 | INFO | } | 20260507 14:59:00.042 | INFO | Perf report for the last 5.1394 seconds: | 20260507 14:59:00.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:00.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 14:59:00.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 14:59:00.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:00.042 | INFO | Uploaded 36GiB to GPU in 1.58707s (22.6834GiB/s) | 20260507 14:59:00.042 | INFO | Downloaded 34GiB to GPU in 1.53533s (22.1451GiB/s) | 20260507 14:59:05.181 | INFO | Analyzing chunk frames = [399872,400384) | 20260507 14:59:05.181 | INFO | MemInfo: { | 20260507 14:59:05.181 | INFO | "gpu" : " 17 GB", | 20260507 14:59:05.181 | INFO | "huge" : "", | 20260507 14:59:05.181 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:05.181 | INFO | "malloc" : " 14 GB", | 20260507 14:59:05.181 | INFO | "pinned" : " 29 GB" | 20260507 14:59:05.181 | INFO | } | 20260507 14:59:05.181 | INFO | Perf report for the last 5.13915 seconds: | 20260507 14:59:05.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:05.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:59:05.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 14:59:05.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:05.181 | INFO | Uploaded 36GiB to GPU in 1.58751s (22.677GiB/s) | 20260507 14:59:05.182 | INFO | Downloaded 34GiB to GPU in 1.53556s (22.1417GiB/s) | 20260507 14:59:10.320 | INFO | Analyzing chunk frames = [400384,400896) | 20260507 14:59:10.320 | INFO | MemInfo: { | 20260507 14:59:10.320 | INFO | "gpu" : " 17 GB", | 20260507 14:59:10.320 | INFO | "huge" : "", | 20260507 14:59:10.320 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:10.320 | INFO | "malloc" : " 14 GB", | 20260507 14:59:10.320 | INFO | "pinned" : " 29 GB" | 20260507 14:59:10.320 | INFO | } | 20260507 14:59:10.320 | INFO | Perf report for the last 5.13939 seconds: | 20260507 14:59:10.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:10.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.321 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 14:59:10.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.321 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 14:59:10.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:10.321 | INFO | Uploaded 36GiB to GPU in 1.58736s (22.6792GiB/s) | 20260507 14:59:10.321 | INFO | Downloaded 34GiB to GPU in 1.53658s (22.127GiB/s) | 20260507 14:59:15.460 | INFO | Analyzing chunk frames = [400896,401408) | 20260507 14:59:15.460 | INFO | MemInfo: { | 20260507 14:59:15.460 | INFO | "gpu" : " 17 GB", | 20260507 14:59:15.460 | INFO | "huge" : "", | 20260507 14:59:15.460 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:15.460 | INFO | "malloc" : " 14 GB", | 20260507 14:59:15.460 | INFO | "pinned" : " 29 GB" | 20260507 14:59:15.460 | INFO | } | 20260507 14:59:15.460 | INFO | Perf report for the last 5.13937 seconds: | 20260507 14:59:15.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:15.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 14:59:15.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 14:59:15.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:15.460 | INFO | Uploaded 36GiB to GPU in 1.58769s (22.6745GiB/s) | 20260507 14:59:15.460 | INFO | Downloaded 34GiB to GPU in 1.53577s (22.1388GiB/s) | 20260507 14:59:15.460 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 14:59:15.460 | INFO | Total mem usage: 16.4 GiB | 20260507 14:59:15.460 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 14:59:15.460 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11%) | 20260507 14:59:15.460 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 14:59:15.460 | INFO | High Water Breakdown: | 20260507 14:59:15.460 | INFO | 58.8% : 8.68 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 14:59:15.460 | INFO | 41.1% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 14:59:15.460 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 14:59:15.460 | INFO | ------------------------------------------ | 20260507 14:59:15.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:59:15.460 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 14:59:15.460 | INFO | Total mem usage: 26.5 GiB | 20260507 14:59:15.460 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 14:59:15.460 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 14:59:15.460 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 14:59:15.460 | INFO | High Water Breakdown: | 20260507 14:59:15.460 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:59:15.460 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:59:15.460 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:59:15.460 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:59:15.460 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:59:15.460 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:59:15.460 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 14:59:15.460 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 14:59:15.460 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:59:15.460 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 14:59:15.460 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 14:59:15.460 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 14:59:15.460 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 14:59:15.460 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 14:59:15.460 | INFO | ------------------------------------------ | 20260507 14:59:15.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:59:15.460 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 14:59:15.460 | INFO | Total mem usage: 27.8 GiB | 20260507 14:59:15.460 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 14:59:15.460 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 14:59:15.460 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 14:59:15.460 | INFO | High Water Breakdown: | 20260507 14:59:15.460 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 14:59:15.460 | INFO | ------------------------------------------ | 20260507 14:59:15.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:59:15.460 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 14:59:15.460 | INFO | Total mem usage: 15.7 GiB | 20260507 14:59:15.460 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 14:59:15.460 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 14:59:15.460 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 14:59:15.460 | INFO | High Water Breakdown: | 20260507 14:59:15.460 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 14:59:15.460 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 14:59:15.460 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 14:59:15.460 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 14:59:15.460 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 14:59:15.460 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 14:59:15.460 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 14:59:15.460 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 14:59:15.460 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 14:59:15.460 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 14:59:15.460 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 14:59:15.460 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 14:59:15.460 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 14:59:15.460 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 14:59:15.460 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 14:59:15.460 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 14:59:15.460 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 14:59:15.460 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 14:59:15.460 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 14:59:15.460 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 14:59:15.460 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 14:59:15.460 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 14:59:15.460 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 14:59:15.460 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 14:59:15.460 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 14:59:15.460 | INFO | ------------------------------------------ | 20260507 14:59:15.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 14:59:15.461 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 14:59:15.461 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:59:15.461 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:59:15.462 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.15e-03 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 11.01%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.72e+01 / -nan, 4.32e-02 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.16e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 3.54%, Metrics -- 1.06e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.91e-01 / -nan, 5.71e-02 / 0.00e+00 | 20260507 14:59:15.462 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 14:59:15.462 | INFO | PerfReport: Gathered 1536 samples. | 20260507 14:59:15.462 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 14:59:15.462 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 18.61%, Upload -- 1.67e+01 / 0.00e+00, 4.48e-01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 18.06%, Download -- 1.62e+01 / 0.00e+00, 7.60e-01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 33.36%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 0.00%, LockDownload -- 4.33e-04 / 0.00e+00, 1.74e-04 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 29.38%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 14:59:15.462 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260507 14:59:15.556 | INFO | Starting a round of Baz IO | 20260507 14:59:18.447 | INFO | Populating RTMetrics report | 20260507 14:59:18.448 | INFO | RTMetrics report populated | 20260507 14:59:18.448 | INFO | Opening tmp RTMetrics json file | 20260507 14:59:18.499 | INFO | Serializing RTMetrics to json | 20260507 14:59:18.583 | INFO | Serialization done, swapping tmp file now... | 20260507 14:59:18.664 | INFO | RTMetrics done | 20260507 14:59:18.721 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.164 seconds | 20260507 14:59:18.721 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.41% / 0.06% / 44.53% | 20260507 14:59:18.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.70338% / 0.000127157% / 4.76837e-05% | 20260507 14:59:18.721 | INFO | Baz IO overall: | 20260507 14:59:18.722 | INFO | BytesWritten : 154705864142 | 20260507 14:59:18.722 | INFO | NumEvents : 74100831641 | 20260507 14:59:20.599 | INFO | Analyzing chunk frames = [401408,401920) | 20260507 14:59:20.599 | INFO | MemInfo: { | 20260507 14:59:20.599 | INFO | "gpu" : " 17 GB", | 20260507 14:59:20.599 | INFO | "huge" : "", | 20260507 14:59:20.599 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:20.599 | INFO | "malloc" : " 14 GB", | 20260507 14:59:20.599 | INFO | "pinned" : " 29 GB" | 20260507 14:59:20.599 | INFO | } | 20260507 14:59:20.599 | INFO | Perf report for the last 5.13921 seconds: | 20260507 14:59:20.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:20.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:20.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:20.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:20.599 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260507 14:59:20.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:20.599 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260507 14:59:20.599 | INFO | PRE_HQ : 56.2%, 2.1, 60.0ms, 0 | 20260507 14:59:20.599 | INFO | BAZWRITER : 61.1%, 4.2, 131.6ms, 0 | 20260507 14:59:20.599 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 14:59:20.600 | INFO | Uploaded 36GiB to GPU in 1.63735s (21.9868GiB/s) | 20260507 14:59:20.600 | INFO | Downloaded 38GiB to GPU in 1.71732s (22.1274GiB/s) | 20260507 14:59:25.738 | INFO | Analyzing chunk frames = [401920,402432) | 20260507 14:59:25.738 | INFO | MemInfo: { | 20260507 14:59:25.738 | INFO | "gpu" : " 17 GB", | 20260507 14:59:25.738 | INFO | "huge" : "", | 20260507 14:59:25.738 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:25.738 | INFO | "malloc" : " 14 GB", | 20260507 14:59:25.738 | INFO | "pinned" : " 29 GB" | 20260507 14:59:25.738 | INFO | } | 20260507 14:59:25.739 | INFO | Perf report for the last 5.13947 seconds: | 20260507 14:59:25.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:25.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 14:59:25.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 14:59:25.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:25.739 | INFO | Uploaded 36GiB to GPU in 1.58436s (22.722GiB/s) | 20260507 14:59:25.739 | INFO | Downloaded 34GiB to GPU in 1.5332s (22.1758GiB/s) | 20260507 14:59:30.888 | INFO | Analyzing chunk frames = [402432,402944) | 20260507 14:59:30.888 | INFO | MemInfo: { | 20260507 14:59:30.888 | INFO | "gpu" : " 17 GB", | 20260507 14:59:30.888 | INFO | "huge" : "", | 20260507 14:59:30.888 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:30.888 | INFO | "malloc" : " 14 GB", | 20260507 14:59:30.888 | INFO | "pinned" : " 29 GB" | 20260507 14:59:30.888 | INFO | } | 20260507 14:59:30.888 | INFO | Perf report for the last 5.1494 seconds: | 20260507 14:59:30.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:30.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 14:59:30.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 14:59:30.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:30.888 | INFO | Uploaded 36GiB to GPU in 1.58846s (22.6634GiB/s) | 20260507 14:59:30.888 | INFO | Downloaded 34GiB to GPU in 1.53762s (22.1121GiB/s) | 20260507 14:59:36.017 | INFO | Analyzing chunk frames = [402944,403456) | 20260507 14:59:36.017 | INFO | MemInfo: { | 20260507 14:59:36.017 | INFO | "gpu" : " 17 GB", | 20260507 14:59:36.017 | INFO | "huge" : "", | 20260507 14:59:36.017 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:36.017 | INFO | "malloc" : " 14 GB", | 20260507 14:59:36.017 | INFO | "pinned" : " 29 GB" | 20260507 14:59:36.018 | INFO | } | 20260507 14:59:36.018 | INFO | Perf report for the last 5.12961 seconds: | 20260507 14:59:36.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:36.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | ANALYSIS : 56.2%, 3.0, 86.9ms, 0 | 20260507 14:59:36.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 14:59:36.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:36.018 | INFO | Uploaded 36GiB to GPU in 1.58815s (22.6679GiB/s) | 20260507 14:59:36.018 | INFO | Downloaded 34GiB to GPU in 1.53773s (22.1105GiB/s) | 20260507 14:59:41.156 | INFO | Analyzing chunk frames = [403456,403968) | 20260507 14:59:41.157 | INFO | MemInfo: { | 20260507 14:59:41.157 | INFO | "gpu" : " 17 GB", | 20260507 14:59:41.157 | INFO | "huge" : "", | 20260507 14:59:41.157 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:41.157 | INFO | "malloc" : " 14 GB", | 20260507 14:59:41.157 | INFO | "pinned" : " 29 GB" | 20260507 14:59:41.157 | INFO | } | 20260507 14:59:41.157 | INFO | Perf report for the last 5.13931 seconds: | 20260507 14:59:41.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:41.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.157 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 14:59:41.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.157 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 14:59:41.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:41.158 | INFO | Uploaded 36GiB to GPU in 1.59338s (22.5935GiB/s) | 20260507 14:59:41.158 | INFO | Downloaded 34GiB to GPU in 1.54003s (22.0775GiB/s) | 20260507 14:59:46.296 | INFO | Analyzing chunk frames = [403968,404480) | 20260507 14:59:46.296 | INFO | MemInfo: { | 20260507 14:59:46.296 | INFO | "gpu" : " 17 GB", | 20260507 14:59:46.296 | INFO | "huge" : "", | 20260507 14:59:46.296 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:46.296 | INFO | "malloc" : " 14 GB", | 20260507 14:59:46.296 | INFO | "pinned" : " 29 GB" | 20260507 14:59:46.296 | INFO | } | 20260507 14:59:46.296 | INFO | Perf report for the last 5.13926 seconds: | 20260507 14:59:46.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:46.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.296 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 14:59:46.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.296 | INFO | PRE_HQ : 41.3%, 1.0, 21.4ms, 0 | 20260507 14:59:46.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:46.297 | INFO | Uploaded 36GiB to GPU in 1.59365s (22.5897GiB/s) | 20260507 14:59:46.297 | INFO | Downloaded 34GiB to GPU in 1.54051s (22.0707GiB/s) | 20260507 14:59:51.435 | INFO | Analyzing chunk frames = [404480,404992) | 20260507 14:59:51.435 | INFO | MemInfo: { | 20260507 14:59:51.435 | INFO | "gpu" : " 17 GB", | 20260507 14:59:51.435 | INFO | "huge" : "", | 20260507 14:59:51.435 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:51.435 | INFO | "malloc" : " 14 GB", | 20260507 14:59:51.435 | INFO | "pinned" : " 29 GB" | 20260507 14:59:51.435 | INFO | } | 20260507 14:59:51.436 | INFO | Perf report for the last 5.13936 seconds: | 20260507 14:59:51.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:51.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:59:51.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | PRE_HQ : 41.2%, 1.0, 21.4ms, 0 | 20260507 14:59:51.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:51.436 | INFO | Uploaded 36GiB to GPU in 1.59336s (22.5937GiB/s) | 20260507 14:59:51.436 | INFO | Downloaded 34GiB to GPU in 1.54161s (22.0548GiB/s) | 20260507 14:59:56.574 | INFO | Analyzing chunk frames = [404992,405504) | 20260507 14:59:56.574 | INFO | MemInfo: { | 20260507 14:59:56.574 | INFO | "gpu" : " 17 GB", | 20260507 14:59:56.574 | INFO | "huge" : "", | 20260507 14:59:56.574 | INFO | "huge_shared" : " 29 GB", | 20260507 14:59:56.574 | INFO | "malloc" : " 14 GB", | 20260507 14:59:56.574 | INFO | "pinned" : " 29 GB" | 20260507 14:59:56.574 | INFO | } | 20260507 14:59:56.575 | INFO | Perf report for the last 5.13897 seconds: | 20260507 14:59:56.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 14:59:56.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 14:59:56.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | PRE_HQ : 40.9%, 1.0, 21.3ms, 0 | 20260507 14:59:56.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 14:59:56.575 | INFO | Uploaded 36GiB to GPU in 1.59321s (22.5959GiB/s) | 20260507 14:59:56.575 | INFO | Downloaded 34GiB to GPU in 1.54153s (22.056GiB/s) | 20260507 14:59:56.672 | INFO | Starting a round of Baz IO | 20260507 14:59:59.586 | INFO | Populating RTMetrics report | 20260507 14:59:59.587 | INFO | RTMetrics report populated | 20260507 14:59:59.587 | INFO | Opening tmp RTMetrics json file | 20260507 14:59:59.630 | INFO | Serializing RTMetrics to json | 20260507 14:59:59.639 | INFO | Serialization done, swapping tmp file now... | 20260507 14:59:59.648 | INFO | RTMetrics done | 20260507 14:59:59.820 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.148 seconds | 20260507 14:59:59.820 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.35% / 0.06% / 44.59% | 20260507 14:59:59.820 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.68632% / 0.000147025% / 2.78155e-05% | 20260507 14:59:59.820 | INFO | Baz IO overall: | 20260507 14:59:59.820 | INFO | BytesWritten : 156553296353 | 20260507 14:59:59.820 | INFO | NumEvents : 75001248283 | 20260507 15:00:01.714 | INFO | Analyzing chunk frames = [405504,406016) | 20260507 15:00:01.714 | INFO | MemInfo: { | 20260507 15:00:01.714 | INFO | "gpu" : " 17 GB", | 20260507 15:00:01.714 | INFO | "huge" : "", | 20260507 15:00:01.714 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:01.714 | INFO | "malloc" : " 14 GB", | 20260507 15:00:01.714 | INFO | "pinned" : " 29 GB" | 20260507 15:00:01.714 | INFO | } | 20260507 15:00:01.714 | INFO | Perf report for the last 5.13975 seconds: | 20260507 15:00:01.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:01.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:01.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:01.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:01.714 | INFO | ANALYSIS : 57.5%, 2.9, 87.5ms, 0 | 20260507 15:00:01.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:01.715 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 15:00:01.715 | INFO | PRE_HQ : 56.7%, 2.1, 61.9ms, 0 | 20260507 15:00:01.715 | INFO | BAZWRITER : 60.8%, 4.3, 134.0ms, 0 | 20260507 15:00:01.715 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 15:00:01.715 | INFO | Uploaded 36GiB to GPU in 1.63792s (21.9791GiB/s) | 20260507 15:00:01.715 | INFO | Downloaded 38GiB to GPU in 1.72072s (22.0837GiB/s) | 20260507 15:00:06.853 | INFO | Analyzing chunk frames = [406016,406528) | 20260507 15:00:06.853 | INFO | MemInfo: { | 20260507 15:00:06.853 | INFO | "gpu" : " 17 GB", | 20260507 15:00:06.853 | INFO | "huge" : "", | 20260507 15:00:06.854 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:06.854 | INFO | "malloc" : " 14 GB", | 20260507 15:00:06.854 | INFO | "pinned" : " 29 GB" | 20260507 15:00:06.854 | INFO | } | 20260507 15:00:06.854 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:00:06.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:06.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 15:00:06.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | PRE_HQ : 40.5%, 1.0, 21.1ms, 0 | 20260507 15:00:06.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:06.854 | INFO | Uploaded 36GiB to GPU in 1.59176s (22.6164GiB/s) | 20260507 15:00:06.854 | INFO | Downloaded 34GiB to GPU in 1.53823s (22.1033GiB/s) | 20260507 15:00:11.993 | INFO | Analyzing chunk frames = [406528,407040) | 20260507 15:00:11.993 | INFO | MemInfo: { | 20260507 15:00:11.993 | INFO | "gpu" : " 17 GB", | 20260507 15:00:11.993 | INFO | "huge" : "", | 20260507 15:00:11.993 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:11.993 | INFO | "malloc" : " 14 GB", | 20260507 15:00:11.993 | INFO | "pinned" : " 29 GB" | 20260507 15:00:11.993 | INFO | } | 20260507 15:00:11.993 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:00:11.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:11.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 15:00:11.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | PRE_HQ : 40.5%, 1.0, 21.0ms, 0 | 20260507 15:00:11.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:11.993 | INFO | Uploaded 36GiB to GPU in 1.59385s (22.5869GiB/s) | 20260507 15:00:11.993 | INFO | Downloaded 34GiB to GPU in 1.54187s (22.0512GiB/s) | 20260507 15:00:17.132 | INFO | Analyzing chunk frames = [407040,407552) | 20260507 15:00:17.132 | INFO | MemInfo: { | 20260507 15:00:17.132 | INFO | "gpu" : " 17 GB", | 20260507 15:00:17.132 | INFO | "huge" : "", | 20260507 15:00:17.132 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:17.132 | INFO | "malloc" : " 14 GB", | 20260507 15:00:17.132 | INFO | "pinned" : " 29 GB" | 20260507 15:00:17.132 | INFO | } | 20260507 15:00:17.132 | INFO | Perf report for the last 5.13886 seconds: | 20260507 15:00:17.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:17.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260507 15:00:17.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 15:00:17.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:17.132 | INFO | Uploaded 36GiB to GPU in 1.59538s (22.5651GiB/s) | 20260507 15:00:17.132 | INFO | Downloaded 34GiB to GPU in 1.54277s (22.0382GiB/s) | 20260507 15:00:22.271 | INFO | Analyzing chunk frames = [407552,408064) | 20260507 15:00:22.271 | INFO | MemInfo: { | 20260507 15:00:22.271 | INFO | "gpu" : " 17 GB", | 20260507 15:00:22.271 | INFO | "huge" : "", | 20260507 15:00:22.271 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:22.271 | INFO | "malloc" : " 14 GB", | 20260507 15:00:22.271 | INFO | "pinned" : " 29 GB" | 20260507 15:00:22.271 | INFO | } | 20260507 15:00:22.271 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:00:22.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:22.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.271 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:00:22.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.271 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:00:22.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:22.272 | INFO | Uploaded 36GiB to GPU in 1.58559s (22.7045GiB/s) | 20260507 15:00:22.272 | INFO | Downloaded 34GiB to GPU in 1.5356s (22.1411GiB/s) | 20260507 15:00:27.410 | INFO | Analyzing chunk frames = [408064,408576) | 20260507 15:00:27.410 | INFO | MemInfo: { | 20260507 15:00:27.410 | INFO | "gpu" : " 17 GB", | 20260507 15:00:27.410 | INFO | "huge" : "", | 20260507 15:00:27.410 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:27.410 | INFO | "malloc" : " 14 GB", | 20260507 15:00:27.410 | INFO | "pinned" : " 29 GB" | 20260507 15:00:27.410 | INFO | } | 20260507 15:00:27.410 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:00:27.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:27.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:00:27.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:00:27.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:27.411 | INFO | Uploaded 36GiB to GPU in 1.58686s (22.6863GiB/s) | 20260507 15:00:27.411 | INFO | Downloaded 34GiB to GPU in 1.53521s (22.1468GiB/s) | 20260507 15:00:32.550 | INFO | Analyzing chunk frames = [408576,409088) | 20260507 15:00:32.550 | INFO | MemInfo: { | 20260507 15:00:32.550 | INFO | "gpu" : " 17 GB", | 20260507 15:00:32.550 | INFO | "huge" : "", | 20260507 15:00:32.550 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:32.550 | INFO | "malloc" : " 14 GB", | 20260507 15:00:32.550 | INFO | "pinned" : " 29 GB" | 20260507 15:00:32.550 | INFO | } | 20260507 15:00:32.550 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:00:32.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:32.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.550 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:00:32.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.550 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:00:32.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:32.551 | INFO | Uploaded 36GiB to GPU in 1.58809s (22.6688GiB/s) | 20260507 15:00:32.551 | INFO | Downloaded 34GiB to GPU in 1.53637s (22.1302GiB/s) | 20260507 15:00:37.689 | INFO | Analyzing chunk frames = [409088,409600) | 20260507 15:00:37.689 | INFO | MemInfo: { | 20260507 15:00:37.689 | INFO | "gpu" : " 17 GB", | 20260507 15:00:37.689 | INFO | "huge" : "", | 20260507 15:00:37.689 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:37.689 | INFO | "malloc" : " 14 GB", | 20260507 15:00:37.689 | INFO | "pinned" : " 29 GB" | 20260507 15:00:37.689 | INFO | } | 20260507 15:00:37.689 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:00:37.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:37.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.689 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:00:37.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.689 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:00:37.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:37.690 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:00:37.690 | INFO | Total mem usage: 16.4 GiB | 20260507 15:00:37.690 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:00:37.690 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.52%) | 20260507 15:00:37.690 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:00:37.690 | INFO | High Water Breakdown: | 20260507 15:00:37.690 | INFO | 59% : 8.84 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:00:37.690 | INFO | 40.9% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:00:37.690 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:00:37.690 | INFO | ------------------------------------------ | 20260507 15:00:37.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:00:37.690 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:00:37.690 | INFO | Total mem usage: 26.5 GiB | 20260507 15:00:37.690 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260507 15:00:37.690 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:00:37.690 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:00:37.690 | INFO | High Water Breakdown: | 20260507 15:00:37.690 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:00:37.690 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:00:37.690 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:00:37.690 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:00:37.690 | INFO | Uploaded 36GiB to GPU in 1.58648s (22.6918GiB/s) | 20260507 15:00:37.690 | INFO | Downloaded 34GiB to GPU in 1.53607s (22.1344GiB/s) | 20260507 15:00:37.690 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:00:37.690 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:00:37.690 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:00:37.690 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:00:37.690 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:00:37.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:00:37.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:00:37.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:00:37.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:00:37.690 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:00:37.690 | INFO | ------------------------------------------ | 20260507 15:00:37.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:00:37.690 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:00:37.690 | INFO | Total mem usage: 27.8 GiB | 20260507 15:00:37.690 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:00:37.690 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:00:37.690 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:00:37.690 | INFO | High Water Breakdown: | 20260507 15:00:37.690 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:00:37.690 | INFO | ------------------------------------------ | 20260507 15:00:37.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:00:37.691 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:00:37.691 | INFO | Total mem usage: 15.7 GiB | 20260507 15:00:37.691 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:00:37.691 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:00:37.691 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:00:37.691 | INFO | High Water Breakdown: | 20260507 15:00:37.691 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:00:37.691 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:00:37.691 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:00:37.691 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:00:37.691 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:00:37.691 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:00:37.691 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:00:37.691 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:00:37.691 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:00:37.691 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:00:37.691 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:00:37.691 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:00:37.691 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:00:37.691 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:00:37.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:00:37.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:00:37.691 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:00:37.691 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:00:37.691 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:00:37.691 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:00:37.691 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:00:37.691 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:00:37.691 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:00:37.691 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:00:37.691 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:00:37.691 | INFO | ------------------------------------------ | 20260507 15:00:37.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:00:37.691 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:00:37.691 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:00:37.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:00:37.691 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 0.30%, AggregateStats -- 9.02e-02 / -nan, 6.43e-03 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 11.04%, Baseline -- 3.31e+00 / -nan, 2.31e-02 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 3.08%, Histogram -- 9.22e-01 / -nan, 5.34e-02 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 16.10%, DME -- 3.86e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.72e+01 / -nan, 4.52e-02 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.17e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 3.54%, Metrics -- 1.06e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 1.32%, PulseSerializing -- 3.96e-01 / -nan, 5.73e-02 / 0.00e+00 | 20260507 15:00:37.691 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:00:37.691 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:00:37.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:00:37.691 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 18.49%, Upload -- 1.66e+01 / 0.00e+00, 4.33e-01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 18.09%, Download -- 1.63e+01 / 0.00e+00, 7.07e-01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 33.34%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 0.00%, LockDownload -- 5.44e-04 / 0.00e+00, 4.13e-04 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 29.49%, LockCompute -- 2.65e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:00:37.691 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 15:00:37.787 | INFO | Starting a round of Baz IO | 20260507 15:00:40.779 | INFO | Populating RTMetrics report | 20260507 15:00:40.779 | INFO | RTMetrics report populated | 20260507 15:00:40.779 | INFO | Opening tmp RTMetrics json file | 20260507 15:00:40.815 | INFO | Serializing RTMetrics to json | 20260507 15:00:40.817 | INFO | Serialization done, swapping tmp file now... | 20260507 15:00:40.817 | INFO | RTMetrics done | 20260507 15:00:40.938 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.15 seconds | 20260507 15:00:40.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.29% / 0.06% / 44.65% | 20260507 15:00:40.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.6618% / 0.000135104% / 1.98682e-05% | 20260507 15:00:40.938 | INFO | Baz IO overall: | 20260507 15:00:40.938 | INFO | BytesWritten : 158400517355 | 20260507 15:00:40.938 | INFO | NumEvents : 75901425280 | 20260507 15:00:42.829 | INFO | Analyzing chunk frames = [409600,410112) | 20260507 15:00:42.829 | INFO | MemInfo: { | 20260507 15:00:42.829 | INFO | "gpu" : " 17 GB", | 20260507 15:00:42.829 | INFO | "huge" : "", | 20260507 15:00:42.829 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:42.829 | INFO | "malloc" : " 14 GB", | 20260507 15:00:42.829 | INFO | "pinned" : " 29 GB" | 20260507 15:00:42.829 | INFO | } | 20260507 15:00:42.829 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:00:42.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:42.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:42.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:42.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:42.829 | INFO | ANALYSIS : 58.6%, 2.8, 86.9ms, 0 | 20260507 15:00:42.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:42.829 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260507 15:00:42.829 | INFO | PRE_HQ : 57.9%, 2.2, 65.4ms, 0 | 20260507 15:00:42.829 | INFO | BAZWRITER : 60.9%, 4.5, 141.7ms, 0 | 20260507 15:00:42.829 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:00:42.830 | INFO | Uploaded 36GiB to GPU in 1.67199s (21.5312GiB/s) | 20260507 15:00:42.830 | INFO | Downloaded 38GiB to GPU in 1.75487s (21.6541GiB/s) | 20260507 15:00:47.968 | INFO | Analyzing chunk frames = [410112,410624) | 20260507 15:00:47.968 | INFO | MemInfo: { | 20260507 15:00:47.968 | INFO | "gpu" : " 17 GB", | 20260507 15:00:47.968 | INFO | "huge" : "", | 20260507 15:00:47.968 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:47.968 | INFO | "malloc" : " 14 GB", | 20260507 15:00:47.968 | INFO | "pinned" : " 29 GB" | 20260507 15:00:47.968 | INFO | } | 20260507 15:00:47.968 | INFO | Perf report for the last 5.13953 seconds: | 20260507 15:00:47.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:47.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.969 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 15:00:47.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.969 | INFO | PRE_HQ : 40.0%, 1.0, 20.8ms, 0 | 20260507 15:00:47.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:47.969 | INFO | Uploaded 36GiB to GPU in 1.58979s (22.6445GiB/s) | 20260507 15:00:47.969 | INFO | Downloaded 34GiB to GPU in 1.53873s (22.0962GiB/s) | 20260507 15:00:53.107 | INFO | Analyzing chunk frames = [410624,411136) | 20260507 15:00:53.107 | INFO | MemInfo: { | 20260507 15:00:53.107 | INFO | "gpu" : " 17 GB", | 20260507 15:00:53.107 | INFO | "huge" : "", | 20260507 15:00:53.107 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:53.107 | INFO | "malloc" : " 14 GB", | 20260507 15:00:53.108 | INFO | "pinned" : " 29 GB" | 20260507 15:00:53.108 | INFO | } | 20260507 15:00:53.108 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:00:53.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:53.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 15:00:53.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 15:00:53.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:53.108 | INFO | Uploaded 36GiB to GPU in 1.59377s (22.588GiB/s) | 20260507 15:00:53.108 | INFO | Downloaded 34GiB to GPU in 1.54378s (22.0238GiB/s) | 20260507 15:00:58.247 | INFO | Analyzing chunk frames = [411136,411648) | 20260507 15:00:58.247 | INFO | MemInfo: { | 20260507 15:00:58.247 | INFO | "gpu" : " 17 GB", | 20260507 15:00:58.247 | INFO | "huge" : "", | 20260507 15:00:58.247 | INFO | "huge_shared" : " 29 GB", | 20260507 15:00:58.247 | INFO | "malloc" : " 14 GB", | 20260507 15:00:58.247 | INFO | "pinned" : " 29 GB" | 20260507 15:00:58.247 | INFO | } | 20260507 15:00:58.247 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:00:58.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:00:58.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.247 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 15:00:58.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.247 | INFO | PRE_HQ : 40.6%, 1.0, 21.1ms, 0 | 20260507 15:00:58.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:00:58.248 | INFO | Uploaded 36GiB to GPU in 1.59323s (22.5956GiB/s) | 20260507 15:00:58.248 | INFO | Downloaded 34GiB to GPU in 1.54196s (22.0499GiB/s) | 20260507 15:01:03.386 | INFO | Analyzing chunk frames = [411648,412160) | 20260507 15:01:03.386 | INFO | MemInfo: { | 20260507 15:01:03.386 | INFO | "gpu" : " 17 GB", | 20260507 15:01:03.386 | INFO | "huge" : "", | 20260507 15:01:03.386 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:03.386 | INFO | "malloc" : " 14 GB", | 20260507 15:01:03.386 | INFO | "pinned" : " 29 GB" | 20260507 15:01:03.386 | INFO | } | 20260507 15:01:03.386 | INFO | Perf report for the last 5.13904 seconds: | 20260507 15:01:03.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:03.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:01:03.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 15:01:03.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:03.386 | INFO | Uploaded 36GiB to GPU in 1.58936s (22.6507GiB/s) | 20260507 15:01:03.386 | INFO | Downloaded 34GiB to GPU in 1.53746s (22.1144GiB/s) | 20260507 15:01:08.525 | INFO | Analyzing chunk frames = [412160,412672) | 20260507 15:01:08.525 | INFO | MemInfo: { | 20260507 15:01:08.525 | INFO | "gpu" : " 17 GB", | 20260507 15:01:08.525 | INFO | "huge" : "", | 20260507 15:01:08.525 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:08.525 | INFO | "malloc" : " 14 GB", | 20260507 15:01:08.525 | INFO | "pinned" : " 29 GB" | 20260507 15:01:08.525 | INFO | } | 20260507 15:01:08.525 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:01:08.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:08.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.525 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:01:08.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.526 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:01:08.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:08.526 | INFO | Uploaded 36GiB to GPU in 1.58745s (22.6778GiB/s) | 20260507 15:01:08.526 | INFO | Downloaded 34GiB to GPU in 1.53442s (22.1582GiB/s) | 20260507 15:01:13.665 | INFO | Analyzing chunk frames = [412672,413184) | 20260507 15:01:13.665 | INFO | MemInfo: { | 20260507 15:01:13.665 | INFO | "gpu" : " 17 GB", | 20260507 15:01:13.665 | INFO | "huge" : "", | 20260507 15:01:13.665 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:13.665 | INFO | "malloc" : " 14 GB", | 20260507 15:01:13.665 | INFO | "pinned" : " 29 GB" | 20260507 15:01:13.665 | INFO | } | 20260507 15:01:13.665 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:01:13.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:13.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:01:13.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:01:13.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:13.665 | INFO | Uploaded 36GiB to GPU in 1.58734s (22.6794GiB/s) | 20260507 15:01:13.665 | INFO | Downloaded 34GiB to GPU in 1.53616s (22.1331GiB/s) | 20260507 15:01:18.804 | INFO | Analyzing chunk frames = [413184,413696) | 20260507 15:01:18.804 | INFO | MemInfo: { | 20260507 15:01:18.804 | INFO | "gpu" : " 17 GB", | 20260507 15:01:18.804 | INFO | "huge" : "", | 20260507 15:01:18.804 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:18.804 | INFO | "malloc" : " 14 GB", | 20260507 15:01:18.804 | INFO | "pinned" : " 29 GB" | 20260507 15:01:18.804 | INFO | } | 20260507 15:01:18.804 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:01:18.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:18.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 15:01:18.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:01:18.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:18.804 | INFO | Uploaded 36GiB to GPU in 1.5866s (22.6901GiB/s) | 20260507 15:01:18.804 | INFO | Downloaded 34GiB to GPU in 1.53716s (22.1187GiB/s) | 20260507 15:01:18.901 | INFO | Starting a round of Baz IO | 20260507 15:01:21.839 | INFO | Populating RTMetrics report | 20260507 15:01:21.839 | INFO | RTMetrics report populated | 20260507 15:01:21.839 | INFO | Opening tmp RTMetrics json file | 20260507 15:01:21.840 | INFO | Serializing RTMetrics to json | 20260507 15:01:21.841 | INFO | Serialization done, swapping tmp file now... | 20260507 15:01:21.841 | INFO | RTMetrics done | 20260507 15:01:21.935 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.033 seconds | 20260507 15:01:21.935 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.23% / 0.06% / 44.71% | 20260507 15:01:21.935 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.64192% / 0.00013113% / 2.78155e-05% | 20260507 15:01:21.935 | INFO | Baz IO overall: | 20260507 15:01:21.935 | INFO | BytesWritten : 160249157912 | 20260507 15:01:21.935 | INFO | NumEvents : 76802804047 | 20260507 15:01:23.943 | INFO | Analyzing chunk frames = [413696,414208) | 20260507 15:01:23.943 | INFO | MemInfo: { | 20260507 15:01:23.943 | INFO | "gpu" : " 17 GB", | 20260507 15:01:23.943 | INFO | "huge" : "", | 20260507 15:01:23.943 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:23.943 | INFO | "malloc" : " 14 GB", | 20260507 15:01:23.943 | INFO | "pinned" : " 29 GB" | 20260507 15:01:23.943 | INFO | } | 20260507 15:01:23.943 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:01:23.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:23.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:23.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:23.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:23.943 | INFO | ANALYSIS : 58.2%, 2.7, 83.8ms, 0 | 20260507 15:01:23.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:23.944 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 15:01:23.944 | INFO | PRE_HQ : 57.3%, 2.2, 65.2ms, 0 | 20260507 15:01:23.944 | INFO | BAZWRITER : 58.6%, 4.7, 142.0ms, 0 | 20260507 15:01:23.944 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 15:01:23.944 | INFO | Uploaded 36GiB to GPU in 1.64665s (21.8625GiB/s) | 20260507 15:01:23.944 | INFO | Downloaded 38GiB to GPU in 1.72898s (21.9783GiB/s) | 20260507 15:01:29.083 | INFO | Analyzing chunk frames = [414208,414720) | 20260507 15:01:29.083 | INFO | MemInfo: { | 20260507 15:01:29.083 | INFO | "gpu" : " 17 GB", | 20260507 15:01:29.083 | INFO | "huge" : "", | 20260507 15:01:29.083 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:29.083 | INFO | "malloc" : " 14 GB", | 20260507 15:01:29.083 | INFO | "pinned" : " 29 GB" | 20260507 15:01:29.083 | INFO | } | 20260507 15:01:29.083 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:01:29.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:29.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.083 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 15:01:29.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.083 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:01:29.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:29.084 | INFO | Uploaded 36GiB to GPU in 1.60419s (22.4413GiB/s) | 20260507 15:01:29.084 | INFO | Downloaded 34GiB to GPU in 1.52448s (22.3026GiB/s) | 20260507 15:01:34.222 | INFO | Analyzing chunk frames = [414720,415232) | 20260507 15:01:34.222 | INFO | MemInfo: { | 20260507 15:01:34.222 | INFO | "gpu" : " 17 GB", | 20260507 15:01:34.222 | INFO | "huge" : "", | 20260507 15:01:34.222 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:34.222 | INFO | "malloc" : " 14 GB", | 20260507 15:01:34.222 | INFO | "pinned" : " 29 GB" | 20260507 15:01:34.222 | INFO | } | 20260507 15:01:34.222 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:01:34.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:34.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.222 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:01:34.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.222 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:01:34.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:34.223 | INFO | Uploaded 36GiB to GPU in 1.60903s (22.3738GiB/s) | 20260507 15:01:34.223 | INFO | Downloaded 34GiB to GPU in 1.52911s (22.2352GiB/s) | 20260507 15:01:39.361 | INFO | Analyzing chunk frames = [415232,415744) | 20260507 15:01:39.361 | INFO | MemInfo: { | 20260507 15:01:39.361 | INFO | "gpu" : " 17 GB", | 20260507 15:01:39.361 | INFO | "huge" : "", | 20260507 15:01:39.361 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:39.361 | INFO | "malloc" : " 14 GB", | 20260507 15:01:39.361 | INFO | "pinned" : " 29 GB" | 20260507 15:01:39.361 | INFO | } | 20260507 15:01:39.361 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:01:39.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:39.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:01:39.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:01:39.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:39.362 | INFO | Uploaded 36GiB to GPU in 1.60817s (22.3857GiB/s) | 20260507 15:01:39.362 | INFO | Downloaded 34GiB to GPU in 1.52834s (22.2464GiB/s) | 20260507 15:01:44.501 | INFO | Analyzing chunk frames = [415744,416256) | 20260507 15:01:44.501 | INFO | MemInfo: { | 20260507 15:01:44.501 | INFO | "gpu" : " 17 GB", | 20260507 15:01:44.501 | INFO | "huge" : "", | 20260507 15:01:44.501 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:44.501 | INFO | "malloc" : " 14 GB", | 20260507 15:01:44.501 | INFO | "pinned" : " 29 GB" | 20260507 15:01:44.501 | INFO | } | 20260507 15:01:44.501 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:01:44.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:44.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:01:44.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:01:44.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:44.501 | INFO | Uploaded 36GiB to GPU in 1.60695s (22.4027GiB/s) | 20260507 15:01:44.501 | INFO | Downloaded 34GiB to GPU in 1.52691s (22.2672GiB/s) | 20260507 15:01:49.640 | INFO | Analyzing chunk frames = [416256,416768) | 20260507 15:01:49.640 | INFO | MemInfo: { | 20260507 15:01:49.640 | INFO | "gpu" : " 17 GB", | 20260507 15:01:49.640 | INFO | "huge" : "", | 20260507 15:01:49.640 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:49.640 | INFO | "malloc" : " 14 GB", | 20260507 15:01:49.640 | INFO | "pinned" : " 29 GB" | 20260507 15:01:49.640 | INFO | } | 20260507 15:01:49.640 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:01:49.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:49.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.640 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:01:49.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.640 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:01:49.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:49.641 | INFO | Uploaded 36GiB to GPU in 1.60703s (22.4016GiB/s) | 20260507 15:01:49.641 | INFO | Downloaded 34GiB to GPU in 1.52607s (22.2794GiB/s) | 20260507 15:01:54.779 | INFO | Analyzing chunk frames = [416768,417280) | 20260507 15:01:54.779 | INFO | MemInfo: { | 20260507 15:01:54.779 | INFO | "gpu" : " 17 GB", | 20260507 15:01:54.779 | INFO | "huge" : "", | 20260507 15:01:54.779 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:54.779 | INFO | "malloc" : " 14 GB", | 20260507 15:01:54.779 | INFO | "pinned" : " 29 GB" | 20260507 15:01:54.779 | INFO | } | 20260507 15:01:54.779 | INFO | Perf report for the last 5.13907 seconds: | 20260507 15:01:54.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:54.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.779 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:01:54.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.779 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:01:54.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:54.780 | INFO | Uploaded 36GiB to GPU in 1.60735s (22.3971GiB/s) | 20260507 15:01:54.780 | INFO | Downloaded 34GiB to GPU in 1.52752s (22.2583GiB/s) | 20260507 15:01:59.919 | INFO | Analyzing chunk frames = [417280,417792) | 20260507 15:01:59.919 | INFO | MemInfo: { | 20260507 15:01:59.919 | INFO | "gpu" : " 17 GB", | 20260507 15:01:59.919 | INFO | "huge" : "", | 20260507 15:01:59.919 | INFO | "huge_shared" : " 29 GB", | 20260507 15:01:59.919 | INFO | "malloc" : " 14 GB", | 20260507 15:01:59.919 | INFO | "pinned" : " 29 GB" | 20260507 15:01:59.919 | INFO | } | 20260507 15:01:59.919 | INFO | Perf report for the last 5.13954 seconds: | 20260507 15:01:59.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:01:59.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:01:59.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:01:59.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:01:59.919 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:01:59.919 | INFO | Total mem usage: 16.4 GiB | 20260507 15:01:59.919 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:01:59.919 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.66%) | 20260507 15:01:59.919 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:01:59.919 | INFO | High Water Breakdown: | 20260507 15:01:59.919 | INFO | 59.2% : 8.93 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:01:59.919 | INFO | 40.7% : 6.14 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:01:59.919 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:01:59.919 | INFO | Uploaded 36GiB to GPU in 1.6076s (22.3936GiB/s) | 20260507 15:01:59.919 | INFO | ------------------------------------------ | 20260507 15:01:59.920 | INFO | Downloaded 34GiB to GPU in 1.52756s (22.2578GiB/s) | 20260507 15:01:59.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:01:59.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:01:59.920 | INFO | Total mem usage: 26.5 GiB | 20260507 15:01:59.920 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:01:59.920 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 15:01:59.920 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 15:01:59.920 | INFO | High Water Breakdown: | 20260507 15:01:59.920 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:01:59.920 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:01:59.920 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:01:59.920 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:01:59.920 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:01:59.920 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:01:59.920 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:01:59.920 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:01:59.920 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:01:59.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:01:59.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:01:59.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:01:59.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:01:59.920 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:01:59.920 | INFO | ------------------------------------------ | 20260507 15:01:59.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:01:59.920 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:01:59.920 | INFO | Total mem usage: 27.8 GiB | 20260507 15:01:59.920 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:01:59.920 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:01:59.920 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:01:59.920 | INFO | High Water Breakdown: | 20260507 15:01:59.920 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:01:59.920 | INFO | ------------------------------------------ | 20260507 15:01:59.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:01:59.920 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:01:59.920 | INFO | Total mem usage: 15.7 GiB | 20260507 15:01:59.920 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:01:59.920 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 5.01%) | 20260507 15:01:59.920 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:01:59.920 | INFO | High Water Breakdown: | 20260507 15:01:59.920 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:01:59.920 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:01:59.920 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:01:59.920 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:01:59.920 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:01:59.920 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:01:59.920 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:01:59.920 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:01:59.920 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:01:59.920 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:01:59.920 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:01:59.920 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:01:59.920 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:01:59.920 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:01:59.920 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:01:59.920 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:01:59.920 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:01:59.920 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:01:59.920 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:01:59.920 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:01:59.920 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:01:59.920 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:01:59.920 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:01:59.920 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:01:59.920 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:01:59.920 | INFO | ------------------------------------------ | 20260507 15:01:59.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:01:59.920 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:01:59.920 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:01:59.920 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:01:59.920 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 0.29%, AggregateStats -- 8.82e-02 / -nan, 7.31e-03 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 11.02%, Baseline -- 3.30e+00 / -nan, 2.25e-02 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.72e+01 / -nan, 4.45e-02 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.17e+00 / -nan, 3.28e-01 / 0.00e+00 | 20260507 15:01:59.920 | INFO | PerfReport: 3.53%, Metrics -- 1.06e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.93e-01 / -nan, 5.72e-02 / 0.00e+00 | 20260507 15:01:59.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:01:59.921 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:01:59.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:01:59.921 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 18.70%, Upload -- 1.67e+01 / 0.00e+00, 8.61e-01 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 18.15%, Download -- 1.62e+01 / 0.00e+00, 1.13e+00 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 33.48%, Compute -- 3.00e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 0.58%, LockUpload -- 5.23e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 0.01%, LockDownload -- 7.82e-03 / 0.00e+00, 2.85e-01 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 29.05%, LockCompute -- 2.60e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:01:59.921 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260507 15:02:00.016 | INFO | Starting a round of Baz IO | 20260507 15:02:02.906 | INFO | Populating RTMetrics report | 20260507 15:02:02.906 | INFO | RTMetrics report populated | 20260507 15:02:02.906 | INFO | Opening tmp RTMetrics json file | 20260507 15:02:02.908 | INFO | Serializing RTMetrics to json | 20260507 15:02:02.909 | INFO | Serialization done, swapping tmp file now... | 20260507 15:02:02.909 | INFO | RTMetrics done | 20260507 15:02:02.994 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 2.977 seconds | 20260507 15:02:02.994 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.17% / 0.06% / 44.77% | 20260507 15:02:02.994 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.60531% / 0.000139078% / 1.19209e-05% | 20260507 15:02:02.994 | INFO | Baz IO overall: | 20260507 15:02:02.994 | INFO | BytesWritten : 162098475749 | 20260507 15:02:02.994 | INFO | NumEvents : 77704050255 | 20260507 15:02:05.058 | INFO | Analyzing chunk frames = [417792,418304) | 20260507 15:02:05.058 | INFO | MemInfo: { | 20260507 15:02:05.058 | INFO | "gpu" : " 17 GB", | 20260507 15:02:05.058 | INFO | "huge" : "", | 20260507 15:02:05.058 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:05.058 | INFO | "malloc" : " 14 GB", | 20260507 15:02:05.058 | INFO | "pinned" : " 29 GB" | 20260507 15:02:05.058 | INFO | } | 20260507 15:02:05.058 | INFO | Perf report for the last 5.13952 seconds: | 20260507 15:02:05.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:05.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:05.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:05.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:05.058 | INFO | ANALYSIS : 57.2%, 2.9, 87.7ms, 0 | 20260507 15:02:05.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:05.059 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 15:02:05.059 | INFO | PRE_HQ : 55.9%, 2.1, 59.6ms, 0 | 20260507 15:02:05.059 | INFO | BAZWRITER : 57.5%, 4.3, 127.3ms, 0 | 20260507 15:02:05.059 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 15:02:05.059 | INFO | Uploaded 36GiB to GPU in 1.64836s (21.8398GiB/s) | 20260507 15:02:05.059 | INFO | Downloaded 38GiB to GPU in 1.73161s (21.9449GiB/s) | 20260507 15:02:10.197 | INFO | Analyzing chunk frames = [418304,418816) | 20260507 15:02:10.197 | INFO | MemInfo: { | 20260507 15:02:10.197 | INFO | "gpu" : " 17 GB", | 20260507 15:02:10.197 | INFO | "huge" : "", | 20260507 15:02:10.198 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:10.198 | INFO | "malloc" : " 14 GB", | 20260507 15:02:10.198 | INFO | "pinned" : " 29 GB" | 20260507 15:02:10.198 | INFO | } | 20260507 15:02:10.198 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:02:10.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:10.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 15:02:10.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | PRE_HQ : 40.7%, 1.0, 21.6ms, 0 | 20260507 15:02:10.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:10.198 | INFO | Uploaded 36GiB to GPU in 1.61083s (22.3487GiB/s) | 20260507 15:02:10.198 | INFO | Downloaded 34GiB to GPU in 1.52944s (22.2303GiB/s) | 20260507 15:02:15.337 | INFO | Analyzing chunk frames = [418816,419328) | 20260507 15:02:15.337 | INFO | MemInfo: { | 20260507 15:02:15.337 | INFO | "gpu" : " 17 GB", | 20260507 15:02:15.337 | INFO | "huge" : "", | 20260507 15:02:15.337 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:15.337 | INFO | "malloc" : " 14 GB", | 20260507 15:02:15.337 | INFO | "pinned" : " 29 GB" | 20260507 15:02:15.337 | INFO | } | 20260507 15:02:15.337 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:02:15.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:15.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.337 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260507 15:02:15.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.337 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:02:15.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:15.338 | INFO | Uploaded 36GiB to GPU in 1.61365s (22.3097GiB/s) | 20260507 15:02:15.338 | INFO | Downloaded 34GiB to GPU in 1.53282s (22.1814GiB/s) | 20260507 15:02:20.476 | INFO | Analyzing chunk frames = [419328,419840) | 20260507 15:02:20.476 | INFO | MemInfo: { | 20260507 15:02:20.476 | INFO | "gpu" : " 17 GB", | 20260507 15:02:20.476 | INFO | "huge" : "", | 20260507 15:02:20.476 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:20.476 | INFO | "malloc" : " 14 GB", | 20260507 15:02:20.476 | INFO | "pinned" : " 29 GB" | 20260507 15:02:20.476 | INFO | } | 20260507 15:02:20.476 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:02:20.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:20.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 15:02:20.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 15:02:20.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:20.476 | INFO | Uploaded 36GiB to GPU in 1.61414s (22.303GiB/s) | 20260507 15:02:20.477 | INFO | Downloaded 34GiB to GPU in 1.5342s (22.1613GiB/s) | 20260507 15:02:25.615 | INFO | Analyzing chunk frames = [419840,420352) | 20260507 15:02:25.615 | INFO | MemInfo: { | 20260507 15:02:25.615 | INFO | "gpu" : " 17 GB", | 20260507 15:02:25.616 | INFO | "huge" : "", | 20260507 15:02:25.616 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:25.616 | INFO | "malloc" : " 14 GB", | 20260507 15:02:25.616 | INFO | "pinned" : " 29 GB" | 20260507 15:02:25.616 | INFO | } | 20260507 15:02:25.616 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:02:25.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:25.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:02:25.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | PRE_HQ : 40.9%, 1.0, 21.3ms, 0 | 20260507 15:02:25.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:25.616 | INFO | Uploaded 36GiB to GPU in 1.6132s (22.3159GiB/s) | 20260507 15:02:25.616 | INFO | Downloaded 34GiB to GPU in 1.53098s (22.208GiB/s) | 20260507 15:02:30.766 | INFO | Analyzing chunk frames = [420352,420864) | 20260507 15:02:30.766 | INFO | MemInfo: { | 20260507 15:02:30.766 | INFO | "gpu" : " 17 GB", | 20260507 15:02:30.766 | INFO | "huge" : "", | 20260507 15:02:30.766 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:30.766 | INFO | "malloc" : " 14 GB", | 20260507 15:02:30.766 | INFO | "pinned" : " 29 GB" | 20260507 15:02:30.766 | INFO | } | 20260507 15:02:30.767 | INFO | Perf report for the last 5.15079 seconds: | 20260507 15:02:30.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:30.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | ANALYSIS : 56.2%, 3.0, 87.9ms, 0 | 20260507 15:02:30.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | PRE_HQ : 34.1%, 1.0, 17.8ms, 0 | 20260507 15:02:30.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:30.767 | INFO | Uploaded 36GiB to GPU in 1.60986s (22.3623GiB/s) | 20260507 15:02:30.767 | INFO | Downloaded 34GiB to GPU in 1.52886s (22.2388GiB/s) | 20260507 15:02:35.894 | INFO | Analyzing chunk frames = [420864,421376) | 20260507 15:02:35.894 | INFO | MemInfo: { | 20260507 15:02:35.894 | INFO | "gpu" : " 17 GB", | 20260507 15:02:35.894 | INFO | "huge" : "", | 20260507 15:02:35.894 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:35.894 | INFO | "malloc" : " 14 GB", | 20260507 15:02:35.894 | INFO | "pinned" : " 29 GB" | 20260507 15:02:35.894 | INFO | } | 20260507 15:02:35.894 | INFO | Perf report for the last 5.12768 seconds: | 20260507 15:02:35.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:35.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.894 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260507 15:02:35.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.894 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:02:35.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:35.895 | INFO | Uploaded 36GiB to GPU in 1.60715s (22.3999GiB/s) | 20260507 15:02:35.895 | INFO | Downloaded 34GiB to GPU in 1.52742s (22.2597GiB/s) | 20260507 15:02:41.033 | INFO | Analyzing chunk frames = [421376,421888) | 20260507 15:02:41.033 | INFO | MemInfo: { | 20260507 15:02:41.033 | INFO | "gpu" : " 17 GB", | 20260507 15:02:41.033 | INFO | "huge" : "", | 20260507 15:02:41.033 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:41.033 | INFO | "malloc" : " 14 GB", | 20260507 15:02:41.033 | INFO | "pinned" : " 29 GB" | 20260507 15:02:41.033 | INFO | } | 20260507 15:02:41.034 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:02:41.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:41.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:02:41.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:02:41.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:41.034 | INFO | Uploaded 36GiB to GPU in 1.60716s (22.3997GiB/s) | 20260507 15:02:41.034 | INFO | Downloaded 34GiB to GPU in 1.52755s (22.2579GiB/s) | 20260507 15:02:41.130 | INFO | Starting a round of Baz IO | 20260507 15:02:44.020 | INFO | Populating RTMetrics report | 20260507 15:02:44.020 | INFO | RTMetrics report populated | 20260507 15:02:44.020 | INFO | Opening tmp RTMetrics json file | 20260507 15:02:44.032 | INFO | Serializing RTMetrics to json | 20260507 15:02:44.073 | INFO | Serialization done, swapping tmp file now... | 20260507 15:02:44.076 | INFO | RTMetrics done | 20260507 15:02:44.292 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.162 seconds | 20260507 15:02:44.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.11% / 0.06% / 44.83% | 20260507 15:02:44.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.60035% / 0.000115236% / 2.78155e-05% | 20260507 15:02:44.293 | INFO | Baz IO overall: | 20260507 15:02:44.293 | INFO | BytesWritten : 163950605274 | 20260507 15:02:44.293 | INFO | NumEvents : 78607077785 | 20260507 15:02:46.173 | INFO | Analyzing chunk frames = [421888,422400) | 20260507 15:02:46.173 | INFO | MemInfo: { | 20260507 15:02:46.173 | INFO | "gpu" : " 17 GB", | 20260507 15:02:46.173 | INFO | "huge" : "", | 20260507 15:02:46.173 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:46.173 | INFO | "malloc" : " 14 GB", | 20260507 15:02:46.173 | INFO | "pinned" : " 29 GB" | 20260507 15:02:46.173 | INFO | } | 20260507 15:02:46.173 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:02:46.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:46.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:46.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:46.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:46.173 | INFO | ANALYSIS : 57.2%, 2.9, 87.4ms, 0 | 20260507 15:02:46.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:46.173 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 15:02:46.173 | INFO | PRE_HQ : 56.0%, 2.0, 59.5ms, 0 | 20260507 15:02:46.173 | INFO | BAZWRITER : 61.1%, 4.6, 143.0ms, 0 | 20260507 15:02:46.173 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:02:46.173 | INFO | Uploaded 36GiB to GPU in 1.64697s (21.8583GiB/s) | 20260507 15:02:46.173 | INFO | Downloaded 38GiB to GPU in 1.73063s (21.9574GiB/s) | 20260507 15:02:51.312 | INFO | Analyzing chunk frames = [422400,422912) | 20260507 15:02:51.312 | INFO | MemInfo: { | 20260507 15:02:51.312 | INFO | "gpu" : " 17 GB", | 20260507 15:02:51.312 | INFO | "huge" : "", | 20260507 15:02:51.312 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:51.312 | INFO | "malloc" : " 14 GB", | 20260507 15:02:51.312 | INFO | "pinned" : " 29 GB" | 20260507 15:02:51.312 | INFO | } | 20260507 15:02:51.312 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:02:51.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:51.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.312 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:02:51.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.312 | INFO | PRE_HQ : 28.3%, 1.0, 14.8ms, 0 | 20260507 15:02:51.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:51.313 | INFO | Uploaded 36GiB to GPU in 1.60462s (22.4353GiB/s) | 20260507 15:02:51.313 | INFO | Downloaded 34GiB to GPU in 1.52406s (22.3089GiB/s) | 20260507 15:02:56.451 | INFO | Analyzing chunk frames = [422912,423424) | 20260507 15:02:56.451 | INFO | MemInfo: { | 20260507 15:02:56.451 | INFO | "gpu" : " 17 GB", | 20260507 15:02:56.451 | INFO | "huge" : "", | 20260507 15:02:56.451 | INFO | "huge_shared" : " 29 GB", | 20260507 15:02:56.451 | INFO | "malloc" : " 14 GB", | 20260507 15:02:56.451 | INFO | "pinned" : " 29 GB" | 20260507 15:02:56.452 | INFO | } | 20260507 15:02:56.452 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:02:56.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:02:56.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:02:56.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:02:56.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:02:56.452 | INFO | Uploaded 36GiB to GPU in 1.60845s (22.3818GiB/s) | 20260507 15:02:56.452 | INFO | Downloaded 34GiB to GPU in 1.52836s (22.2461GiB/s) | 20260507 15:03:01.591 | INFO | Analyzing chunk frames = [423424,423936) | 20260507 15:03:01.591 | INFO | MemInfo: { | 20260507 15:03:01.591 | INFO | "gpu" : " 17 GB", | 20260507 15:03:01.591 | INFO | "huge" : "", | 20260507 15:03:01.591 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:01.591 | INFO | "malloc" : " 14 GB", | 20260507 15:03:01.591 | INFO | "pinned" : " 29 GB" | 20260507 15:03:01.591 | INFO | } | 20260507 15:03:01.591 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:03:01.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:01.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:03:01.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:03:01.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:01.591 | INFO | Uploaded 36GiB to GPU in 1.60918s (22.3716GiB/s) | 20260507 15:03:01.591 | INFO | Downloaded 34GiB to GPU in 1.52873s (22.2406GiB/s) | 20260507 15:03:06.730 | INFO | Analyzing chunk frames = [423936,424448) | 20260507 15:03:06.730 | INFO | MemInfo: { | 20260507 15:03:06.730 | INFO | "gpu" : " 17 GB", | 20260507 15:03:06.730 | INFO | "huge" : "", | 20260507 15:03:06.730 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:06.730 | INFO | "malloc" : " 14 GB", | 20260507 15:03:06.730 | INFO | "pinned" : " 29 GB" | 20260507 15:03:06.730 | INFO | } | 20260507 15:03:06.730 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:03:06.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:06.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:03:06.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:03:06.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:06.730 | INFO | Uploaded 36GiB to GPU in 1.60673s (22.4057GiB/s) | 20260507 15:03:06.730 | INFO | Downloaded 34GiB to GPU in 1.5263s (22.2761GiB/s) | 20260507 15:03:11.869 | INFO | Analyzing chunk frames = [424448,424960) | 20260507 15:03:11.869 | INFO | MemInfo: { | 20260507 15:03:11.869 | INFO | "gpu" : " 17 GB", | 20260507 15:03:11.869 | INFO | "huge" : "", | 20260507 15:03:11.869 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:11.869 | INFO | "malloc" : " 14 GB", | 20260507 15:03:11.869 | INFO | "pinned" : " 29 GB" | 20260507 15:03:11.870 | INFO | } | 20260507 15:03:11.870 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:03:11.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:11.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:03:11.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:03:11.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:11.870 | INFO | Uploaded 36GiB to GPU in 1.60734s (22.3972GiB/s) | 20260507 15:03:11.870 | INFO | Downloaded 34GiB to GPU in 1.52628s (22.2763GiB/s) | 20260507 15:03:17.009 | INFO | Analyzing chunk frames = [424960,425472) | 20260507 15:03:17.009 | INFO | MemInfo: { | 20260507 15:03:17.009 | INFO | "gpu" : " 17 GB", | 20260507 15:03:17.009 | INFO | "huge" : "", | 20260507 15:03:17.009 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:17.009 | INFO | "malloc" : " 14 GB", | 20260507 15:03:17.009 | INFO | "pinned" : " 29 GB" | 20260507 15:03:17.009 | INFO | } | 20260507 15:03:17.009 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:03:17.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:17.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:03:17.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:03:17.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:17.009 | INFO | Uploaded 36GiB to GPU in 1.60765s (22.393GiB/s) | 20260507 15:03:17.009 | INFO | Downloaded 34GiB to GPU in 1.52788s (22.253GiB/s) | 20260507 15:03:22.148 | INFO | Analyzing chunk frames = [425472,425984) | 20260507 15:03:22.148 | INFO | MemInfo: { | 20260507 15:03:22.148 | INFO | "gpu" : " 17 GB", | 20260507 15:03:22.148 | INFO | "huge" : "", | 20260507 15:03:22.148 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:22.148 | INFO | "malloc" : " 14 GB", | 20260507 15:03:22.148 | INFO | "pinned" : " 29 GB" | 20260507 15:03:22.148 | INFO | } | 20260507 15:03:22.148 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:03:22.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:22.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:03:22.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:03:22.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:22.148 | INFO | Uploaded 36GiB to GPU in 1.60822s (22.385GiB/s) | 20260507 15:03:22.149 | INFO | Downloaded 34GiB to GPU in 1.52753s (22.2581GiB/s) | 20260507 15:03:22.149 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:03:22.149 | INFO | Total mem usage: 16.4 GiB | 20260507 15:03:22.149 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:03:22.149 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.57%) | 20260507 15:03:22.149 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:03:22.149 | INFO | High Water Breakdown: | 20260507 15:03:22.149 | INFO | 59.1% : 8.93 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:03:22.149 | INFO | 40.7% : 6.16 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:03:22.149 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:03:22.149 | INFO | ------------------------------------------ | 20260507 15:03:22.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:03:22.149 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:03:22.149 | INFO | Total mem usage: 26.5 GiB | 20260507 15:03:22.149 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:03:22.149 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:03:22.149 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:03:22.149 | INFO | High Water Breakdown: | 20260507 15:03:22.149 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:03:22.149 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:03:22.149 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:03:22.149 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:03:22.149 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:03:22.149 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:03:22.149 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:03:22.149 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:03:22.149 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:03:22.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:03:22.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:03:22.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:03:22.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:03:22.149 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:03:22.149 | INFO | ------------------------------------------ | 20260507 15:03:22.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:03:22.149 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:03:22.149 | INFO | Total mem usage: 27.8 GiB | 20260507 15:03:22.149 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:03:22.149 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:03:22.149 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:03:22.149 | INFO | High Water Breakdown: | 20260507 15:03:22.149 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:03:22.149 | INFO | ------------------------------------------ | 20260507 15:03:22.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:03:22.149 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:03:22.149 | INFO | Total mem usage: 15.7 GiB | 20260507 15:03:22.149 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:03:22.149 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:03:22.149 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:03:22.149 | INFO | High Water Breakdown: | 20260507 15:03:22.149 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:03:22.149 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:03:22.149 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:03:22.149 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:03:22.149 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:03:22.149 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:03:22.149 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:03:22.149 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:03:22.149 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:03:22.149 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:03:22.149 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:03:22.149 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:03:22.149 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:03:22.149 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:03:22.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:03:22.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:03:22.149 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:03:22.149 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:03:22.149 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:03:22.149 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:03:22.149 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:03:22.149 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:03:22.149 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:03:22.149 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:03:22.149 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:03:22.149 | INFO | ------------------------------------------ | 20260507 15:03:22.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:03:22.149 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:03:22.149 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:03:22.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:03:22.149 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 0.30%, AggregateStats -- 8.83e-02 / -nan, 5.76e-03 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 11.02%, Baseline -- 3.30e+00 / -nan, 2.23e-02 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 3.07%, Histogram -- 9.20e-01 / -nan, 5.30e-02 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.01e+00 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.72e+01 / -nan, 4.44e-02 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.17e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 3.53%, Metrics -- 1.06e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 15:03:22.149 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.93e-01 / -nan, 5.73e-02 / 0.00e+00 | 20260507 15:03:22.149 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:03:22.149 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:03:22.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:03:22.150 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 18.73%, Upload -- 1.68e+01 / 0.00e+00, 4.28e-01 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 18.04%, Download -- 1.62e+01 / 0.00e+00, 7.92e-01 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 33.37%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 0.00%, LockDownload -- 5.29e-04 / 0.00e+00, 4.15e-04 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 29.28%, LockCompute -- 2.63e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:03:22.150 | INFO | PerfReport: 0.02%, Misc -- 1.56e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260507 15:03:22.245 | INFO | Starting a round of Baz IO | 20260507 15:03:25.133 | INFO | Populating RTMetrics report | 20260507 15:03:25.133 | INFO | RTMetrics report populated | 20260507 15:03:25.133 | INFO | Opening tmp RTMetrics json file | 20260507 15:03:25.173 | INFO | Serializing RTMetrics to json | 20260507 15:03:25.209 | INFO | Serialization done, swapping tmp file now... | 20260507 15:03:25.215 | INFO | RTMetrics done | 20260507 15:03:25.430 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.185 seconds | 20260507 15:03:25.430 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.06% / 0.06% / 44.89% | 20260507 15:03:25.430 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.55307% / 0.000143051% / 2.78155e-05% | 20260507 15:03:25.430 | INFO | Baz IO overall: | 20260507 15:03:25.430 | INFO | BytesWritten : 165802336508 | 20260507 15:03:25.430 | INFO | NumEvents : 79508732444 | 20260507 15:03:27.287 | INFO | Analyzing chunk frames = [425984,426496) | 20260507 15:03:27.287 | INFO | MemInfo: { | 20260507 15:03:27.287 | INFO | "gpu" : " 17 GB", | 20260507 15:03:27.287 | INFO | "huge" : "", | 20260507 15:03:27.287 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:27.287 | INFO | "malloc" : " 14 GB", | 20260507 15:03:27.287 | INFO | "pinned" : " 29 GB" | 20260507 15:03:27.287 | INFO | } | 20260507 15:03:27.287 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:03:27.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:27.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:27.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:27.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:27.288 | INFO | ANALYSIS : 57.2%, 2.9, 87.2ms, 0 | 20260507 15:03:27.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:27.288 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 15:03:27.288 | INFO | PRE_HQ : 56.0%, 2.0, 59.4ms, 0 | 20260507 15:03:27.288 | INFO | BAZWRITER : 61.5%, 4.5, 142.0ms, 0 | 20260507 15:03:27.288 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 15:03:27.288 | INFO | Uploaded 36GiB to GPU in 1.64951s (21.8246GiB/s) | 20260507 15:03:27.288 | INFO | Downloaded 38GiB to GPU in 1.73394s (21.9154GiB/s) | 20260507 15:03:32.427 | INFO | Analyzing chunk frames = [426496,427008) | 20260507 15:03:32.427 | INFO | MemInfo: { | 20260507 15:03:32.427 | INFO | "gpu" : " 17 GB", | 20260507 15:03:32.427 | INFO | "huge" : "", | 20260507 15:03:32.427 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:32.427 | INFO | "malloc" : " 14 GB", | 20260507 15:03:32.427 | INFO | "pinned" : " 29 GB" | 20260507 15:03:32.427 | INFO | } | 20260507 15:03:32.427 | INFO | Perf report for the last 5.13963 seconds: | 20260507 15:03:32.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:32.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:03:32.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:03:32.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:32.427 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4257GiB/s) | 20260507 15:03:32.427 | INFO | Downloaded 34GiB to GPU in 1.52525s (22.2914GiB/s) | 20260507 15:03:37.566 | INFO | Analyzing chunk frames = [427008,427520) | 20260507 15:03:37.566 | INFO | MemInfo: { | 20260507 15:03:37.566 | INFO | "gpu" : " 17 GB", | 20260507 15:03:37.566 | INFO | "huge" : "", | 20260507 15:03:37.566 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:37.566 | INFO | "malloc" : " 14 GB", | 20260507 15:03:37.566 | INFO | "pinned" : " 29 GB" | 20260507 15:03:37.566 | INFO | } | 20260507 15:03:37.566 | INFO | Perf report for the last 5.13897 seconds: | 20260507 15:03:37.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:37.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260507 15:03:37.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | PRE_HQ : 38.3%, 1.0, 19.9ms, 0 | 20260507 15:03:37.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:37.566 | INFO | Uploaded 36GiB to GPU in 1.61367s (22.3095GiB/s) | 20260507 15:03:37.567 | INFO | Downloaded 34GiB to GPU in 1.53286s (22.1807GiB/s) | 20260507 15:03:42.705 | INFO | Analyzing chunk frames = [427520,428032) | 20260507 15:03:42.705 | INFO | MemInfo: { | 20260507 15:03:42.705 | INFO | "gpu" : " 17 GB", | 20260507 15:03:42.705 | INFO | "huge" : "", | 20260507 15:03:42.705 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:42.705 | INFO | "malloc" : " 14 GB", | 20260507 15:03:42.705 | INFO | "pinned" : " 29 GB" | 20260507 15:03:42.705 | INFO | } | 20260507 15:03:42.705 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:03:42.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:42.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.706 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:03:42.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.706 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:03:42.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:42.706 | INFO | Uploaded 36GiB to GPU in 1.60736s (22.397GiB/s) | 20260507 15:03:42.706 | INFO | Downloaded 34GiB to GPU in 1.52815s (22.2491GiB/s) | 20260507 15:03:47.845 | INFO | Analyzing chunk frames = [428032,428544) | 20260507 15:03:47.845 | INFO | MemInfo: { | 20260507 15:03:47.845 | INFO | "gpu" : " 17 GB", | 20260507 15:03:47.845 | INFO | "huge" : "", | 20260507 15:03:47.845 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:47.845 | INFO | "malloc" : " 14 GB", | 20260507 15:03:47.845 | INFO | "pinned" : " 29 GB" | 20260507 15:03:47.845 | INFO | } | 20260507 15:03:47.845 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:03:47.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:47.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:03:47.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:03:47.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:47.845 | INFO | Uploaded 36GiB to GPU in 1.60696s (22.4025GiB/s) | 20260507 15:03:47.845 | INFO | Downloaded 34GiB to GPU in 1.52699s (22.2661GiB/s) | 20260507 15:03:52.984 | INFO | Analyzing chunk frames = [428544,429056) | 20260507 15:03:52.984 | INFO | MemInfo: { | 20260507 15:03:52.984 | INFO | "gpu" : " 17 GB", | 20260507 15:03:52.984 | INFO | "huge" : "", | 20260507 15:03:52.984 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:52.984 | INFO | "malloc" : " 14 GB", | 20260507 15:03:52.984 | INFO | "pinned" : " 29 GB" | 20260507 15:03:52.984 | INFO | } | 20260507 15:03:52.984 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:03:52.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:52.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:03:52.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:03:52.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:52.984 | INFO | Uploaded 36GiB to GPU in 1.6064s (22.4104GiB/s) | 20260507 15:03:52.984 | INFO | Downloaded 34GiB to GPU in 1.52684s (22.2683GiB/s) | 20260507 15:03:58.123 | INFO | Analyzing chunk frames = [429056,429568) | 20260507 15:03:58.123 | INFO | MemInfo: { | 20260507 15:03:58.123 | INFO | "gpu" : " 17 GB", | 20260507 15:03:58.124 | INFO | "huge" : "", | 20260507 15:03:58.124 | INFO | "huge_shared" : " 29 GB", | 20260507 15:03:58.124 | INFO | "malloc" : " 14 GB", | 20260507 15:03:58.124 | INFO | "pinned" : " 29 GB" | 20260507 15:03:58.124 | INFO | } | 20260507 15:03:58.124 | INFO | Perf report for the last 5.13958 seconds: | 20260507 15:03:58.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:03:58.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:03:58.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 15:03:58.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:03:58.124 | INFO | Uploaded 36GiB to GPU in 1.60762s (22.3933GiB/s) | 20260507 15:03:58.124 | INFO | Downloaded 34GiB to GPU in 1.52767s (22.2561GiB/s) | 20260507 15:04:03.263 | INFO | Analyzing chunk frames = [429568,430080) | 20260507 15:04:03.263 | INFO | MemInfo: { | 20260507 15:04:03.263 | INFO | "gpu" : " 17 GB", | 20260507 15:04:03.263 | INFO | "huge" : "", | 20260507 15:04:03.263 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:03.263 | INFO | "malloc" : " 14 GB", | 20260507 15:04:03.263 | INFO | "pinned" : " 29 GB" | 20260507 15:04:03.263 | INFO | } | 20260507 15:04:03.263 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:04:03.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:03.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:04:03.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:04:03.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:03.263 | INFO | Uploaded 36GiB to GPU in 1.60766s (22.3928GiB/s) | 20260507 15:04:03.263 | INFO | Downloaded 34GiB to GPU in 1.52797s (22.2517GiB/s) | 20260507 15:04:03.361 | INFO | Starting a round of Baz IO | 20260507 15:04:06.303 | INFO | Populating RTMetrics report | 20260507 15:04:06.304 | INFO | RTMetrics report populated | 20260507 15:04:06.304 | INFO | Opening tmp RTMetrics json file | 20260507 15:04:06.351 | INFO | Serializing RTMetrics to json | 20260507 15:04:06.397 | INFO | Serialization done, swapping tmp file now... | 20260507 15:04:06.430 | INFO | RTMetrics done | 20260507 15:04:06.532 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.171 seconds | 20260507 15:04:06.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.00% / 0.06% / 44.94% | 20260507 15:04:06.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.52951% / 0.000127157% / 1.98682e-05% | 20260507 15:04:06.532 | INFO | Baz IO overall: | 20260507 15:04:06.532 | INFO | BytesWritten : 167655601592 | 20260507 15:04:06.532 | INFO | NumEvents : 80411068101 | 20260507 15:04:08.402 | INFO | Analyzing chunk frames = [430080,430592) | 20260507 15:04:08.402 | INFO | MemInfo: { | 20260507 15:04:08.402 | INFO | "gpu" : " 17 GB", | 20260507 15:04:08.402 | INFO | "huge" : "", | 20260507 15:04:08.402 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:08.402 | INFO | "malloc" : " 14 GB", | 20260507 15:04:08.402 | INFO | "pinned" : " 29 GB" | 20260507 15:04:08.402 | INFO | } | 20260507 15:04:08.402 | INFO | Perf report for the last 5.13948 seconds: | 20260507 15:04:08.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:08.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:08.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:08.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:08.402 | INFO | ANALYSIS : 57.4%, 2.9, 87.2ms, 0 | 20260507 15:04:08.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:08.402 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 15:04:08.403 | INFO | PRE_HQ : 56.7%, 2.1, 61.8ms, 0 | 20260507 15:04:08.403 | INFO | BAZWRITER : 61.3%, 4.4, 137.8ms, 0 | 20260507 15:04:08.403 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 15:04:08.403 | INFO | Uploaded 36GiB to GPU in 1.63416s (22.0297GiB/s) | 20260507 15:04:08.403 | INFO | Downloaded 38GiB to GPU in 1.71815s (22.1168GiB/s) | 20260507 15:04:13.542 | INFO | Analyzing chunk frames = [430592,431104) | 20260507 15:04:13.542 | INFO | MemInfo: { | 20260507 15:04:13.542 | INFO | "gpu" : " 17 GB", | 20260507 15:04:13.542 | INFO | "huge" : "", | 20260507 15:04:13.542 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:13.542 | INFO | "malloc" : " 14 GB", | 20260507 15:04:13.542 | INFO | "pinned" : " 29 GB" | 20260507 15:04:13.542 | INFO | } | 20260507 15:04:13.542 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:04:13.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:13.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:04:13.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:04:13.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:13.542 | INFO | Uploaded 36GiB to GPU in 1.60421s (22.4409GiB/s) | 20260507 15:04:13.542 | INFO | Downloaded 34GiB to GPU in 1.52444s (22.3032GiB/s) | 20260507 15:04:18.681 | INFO | Analyzing chunk frames = [431104,431616) | 20260507 15:04:18.681 | INFO | MemInfo: { | 20260507 15:04:18.681 | INFO | "gpu" : " 17 GB", | 20260507 15:04:18.681 | INFO | "huge" : "", | 20260507 15:04:18.681 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:18.681 | INFO | "malloc" : " 14 GB", | 20260507 15:04:18.681 | INFO | "pinned" : " 29 GB" | 20260507 15:04:18.681 | INFO | } | 20260507 15:04:18.681 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:04:18.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:18.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:04:18.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:04:18.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:18.681 | INFO | Uploaded 36GiB to GPU in 1.60855s (22.3804GiB/s) | 20260507 15:04:18.681 | INFO | Downloaded 34GiB to GPU in 1.53036s (22.2169GiB/s) | 20260507 15:04:23.820 | INFO | Analyzing chunk frames = [431616,432128) | 20260507 15:04:23.820 | INFO | MemInfo: { | 20260507 15:04:23.820 | INFO | "gpu" : " 17 GB", | 20260507 15:04:23.820 | INFO | "huge" : "", | 20260507 15:04:23.820 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:23.820 | INFO | "malloc" : " 14 GB", | 20260507 15:04:23.820 | INFO | "pinned" : " 29 GB" | 20260507 15:04:23.820 | INFO | } | 20260507 15:04:23.820 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:04:23.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:23.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.820 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:04:23.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.820 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 15:04:23.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:23.821 | INFO | Uploaded 36GiB to GPU in 1.60829s (22.384GiB/s) | 20260507 15:04:23.821 | INFO | Downloaded 34GiB to GPU in 1.52828s (22.2473GiB/s) | 20260507 15:04:28.960 | INFO | Analyzing chunk frames = [432128,432640) | 20260507 15:04:28.960 | INFO | MemInfo: { | 20260507 15:04:28.960 | INFO | "gpu" : " 17 GB", | 20260507 15:04:28.960 | INFO | "huge" : "", | 20260507 15:04:28.960 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:28.960 | INFO | "malloc" : " 14 GB", | 20260507 15:04:28.960 | INFO | "pinned" : " 29 GB" | 20260507 15:04:28.960 | INFO | } | 20260507 15:04:28.960 | INFO | Perf report for the last 5.13963 seconds: | 20260507 15:04:28.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:28.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:04:28.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:04:28.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:28.960 | INFO | Uploaded 36GiB to GPU in 1.60701s (22.4019GiB/s) | 20260507 15:04:28.960 | INFO | Downloaded 34GiB to GPU in 1.52723s (22.2625GiB/s) | 20260507 15:04:34.099 | INFO | Analyzing chunk frames = [432640,433152) | 20260507 15:04:34.099 | INFO | MemInfo: { | 20260507 15:04:34.099 | INFO | "gpu" : " 17 GB", | 20260507 15:04:34.099 | INFO | "huge" : "", | 20260507 15:04:34.099 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:34.099 | INFO | "malloc" : " 14 GB", | 20260507 15:04:34.099 | INFO | "pinned" : " 29 GB" | 20260507 15:04:34.099 | INFO | } | 20260507 15:04:34.099 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:04:34.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:34.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:04:34.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 15:04:34.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:34.099 | INFO | Uploaded 36GiB to GPU in 1.60648s (22.4092GiB/s) | 20260507 15:04:34.100 | INFO | Downloaded 34GiB to GPU in 1.5258s (22.2834GiB/s) | 20260507 15:04:39.238 | INFO | Analyzing chunk frames = [433152,433664) | 20260507 15:04:39.238 | INFO | MemInfo: { | 20260507 15:04:39.238 | INFO | "gpu" : " 17 GB", | 20260507 15:04:39.238 | INFO | "huge" : "", | 20260507 15:04:39.238 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:39.238 | INFO | "malloc" : " 14 GB", | 20260507 15:04:39.238 | INFO | "pinned" : " 29 GB" | 20260507 15:04:39.238 | INFO | } | 20260507 15:04:39.238 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:04:39.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:39.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.239 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:04:39.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.239 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:04:39.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:39.239 | INFO | Uploaded 36GiB to GPU in 1.60736s (22.397GiB/s) | 20260507 15:04:39.239 | INFO | Downloaded 34GiB to GPU in 1.5275s (22.2586GiB/s) | 20260507 15:04:44.378 | INFO | Analyzing chunk frames = [433664,434176) | 20260507 15:04:44.378 | INFO | MemInfo: { | 20260507 15:04:44.378 | INFO | "gpu" : " 17 GB", | 20260507 15:04:44.378 | INFO | "huge" : "", | 20260507 15:04:44.378 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:44.378 | INFO | "malloc" : " 14 GB", | 20260507 15:04:44.378 | INFO | "pinned" : " 29 GB" | 20260507 15:04:44.378 | INFO | } | 20260507 15:04:44.378 | INFO | Perf report for the last 5.13948 seconds: | 20260507 15:04:44.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:44.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:04:44.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 15:04:44.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:44.378 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:04:44.378 | INFO | Total mem usage: 16.4 GiB | 20260507 15:04:44.379 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:04:44.379 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.98%) | 20260507 15:04:44.379 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:04:44.379 | INFO | High Water Breakdown: | 20260507 15:04:44.379 | INFO | Uploaded 36GiB to GPU in 1.60863s (22.3792GiB/s) | 20260507 15:04:44.379 | INFO | Downloaded 34GiB to GPU in 1.52882s (22.2394GiB/s) | 20260507 15:04:44.379 | INFO | 59.3% : 9.01 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:04:44.379 | INFO | 40.6% : 6.17 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:04:44.379 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:04:44.379 | INFO | ------------------------------------------ | 20260507 15:04:44.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:04:44.379 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:04:44.379 | INFO | Total mem usage: 26.5 GiB | 20260507 15:04:44.379 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:04:44.379 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:04:44.379 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:04:44.379 | INFO | High Water Breakdown: | 20260507 15:04:44.379 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:04:44.379 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:04:44.379 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:04:44.379 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:04:44.379 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:04:44.379 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:04:44.379 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:04:44.379 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:04:44.379 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:04:44.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:04:44.379 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:04:44.379 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:04:44.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:04:44.379 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:04:44.379 | INFO | ------------------------------------------ | 20260507 15:04:44.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:04:44.379 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:04:44.379 | INFO | Total mem usage: 27.8 GiB | 20260507 15:04:44.379 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:04:44.379 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:04:44.379 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:04:44.379 | INFO | High Water Breakdown: | 20260507 15:04:44.379 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:04:44.379 | INFO | ------------------------------------------ | 20260507 15:04:44.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:04:44.379 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:04:44.379 | INFO | Total mem usage: 15.7 GiB | 20260507 15:04:44.379 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:04:44.379 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:04:44.379 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:04:44.379 | INFO | High Water Breakdown: | 20260507 15:04:44.379 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:04:44.379 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:04:44.379 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:04:44.379 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:04:44.379 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:04:44.379 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:04:44.379 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:04:44.379 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:04:44.379 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:04:44.379 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:04:44.379 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:04:44.379 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:04:44.379 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:04:44.379 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:04:44.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:04:44.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:04:44.379 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:04:44.379 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:04:44.379 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:04:44.379 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:04:44.379 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:04:44.379 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:04:44.379 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:04:44.379 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:04:44.379 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:04:44.379 | INFO | ------------------------------------------ | 20260507 15:04:44.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:04:44.380 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:04:44.380 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:04:44.380 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:04:44.380 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 4.70e-03 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 11.02%, Baseline -- 3.30e+00 / -nan, 2.12e-02 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.72e+01 / -nan, 4.38e-02 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.16e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 3.53%, Metrics -- 1.05e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 1.30%, PulseSerializing -- 3.90e-01 / -nan, 5.64e-02 / 0.00e+00 | 20260507 15:04:44.380 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:04:44.380 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:04:44.380 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:04:44.380 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 18.73%, Upload -- 1.68e+01 / 0.00e+00, 4.22e-01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 18.04%, Download -- 1.62e+01 / 0.00e+00, 7.79e-01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 33.38%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 0.57%, LockUpload -- 5.12e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 0.00%, LockDownload -- 4.63e-04 / 0.00e+00, 2.08e-04 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 29.26%, LockCompute -- 2.62e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:04:44.380 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 15:04:44.476 | INFO | Starting a round of Baz IO | 20260507 15:04:47.463 | INFO | Populating RTMetrics report | 20260507 15:04:47.464 | INFO | RTMetrics report populated | 20260507 15:04:47.464 | INFO | Opening tmp RTMetrics json file | 20260507 15:04:47.487 | INFO | Serializing RTMetrics to json | 20260507 15:04:47.491 | INFO | Serialization done, swapping tmp file now... | 20260507 15:04:47.495 | INFO | RTMetrics done | 20260507 15:04:47.704 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.228 seconds | 20260507 15:04:47.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.94% / 0.06% / 45.00% | 20260507 15:04:47.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.52536% / 0.000135104% / 1.58946e-05% | 20260507 15:04:47.704 | INFO | Baz IO overall: | 20260507 15:04:47.704 | INFO | BytesWritten : 169511061966 | 20260507 15:04:47.704 | INFO | NumEvents : 81315100083 | 20260507 15:04:49.517 | INFO | Analyzing chunk frames = [434176,434688) | 20260507 15:04:49.517 | INFO | MemInfo: { | 20260507 15:04:49.517 | INFO | "gpu" : " 17 GB", | 20260507 15:04:49.517 | INFO | "huge" : "", | 20260507 15:04:49.517 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:49.517 | INFO | "malloc" : " 14 GB", | 20260507 15:04:49.517 | INFO | "pinned" : " 29 GB" | 20260507 15:04:49.517 | INFO | } | 20260507 15:04:49.517 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:04:49.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:49.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:49.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:49.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:49.517 | INFO | ANALYSIS : 57.7%, 2.8, 86.0ms, 0 | 20260507 15:04:49.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:49.517 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260507 15:04:49.517 | INFO | PRE_HQ : 57.3%, 2.2, 66.2ms, 0 | 20260507 15:04:49.517 | INFO | BAZWRITER : 62.4%, 4.5, 144.5ms, 0 | 20260507 15:04:49.517 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:04:49.518 | INFO | Uploaded 36GiB to GPU in 1.64825s (21.8413GiB/s) | 20260507 15:04:49.518 | INFO | Downloaded 38GiB to GPU in 1.7315s (21.9463GiB/s) | 20260507 15:04:54.656 | INFO | Analyzing chunk frames = [434688,435200) | 20260507 15:04:54.656 | INFO | MemInfo: { | 20260507 15:04:54.656 | INFO | "gpu" : " 17 GB", | 20260507 15:04:54.656 | INFO | "huge" : "", | 20260507 15:04:54.656 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:54.657 | INFO | "malloc" : " 14 GB", | 20260507 15:04:54.657 | INFO | "pinned" : " 29 GB" | 20260507 15:04:54.657 | INFO | } | 20260507 15:04:54.657 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:04:54.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:54.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:04:54.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:04:54.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:54.657 | INFO | Uploaded 36GiB to GPU in 1.60445s (22.4376GiB/s) | 20260507 15:04:54.657 | INFO | Downloaded 34GiB to GPU in 1.52351s (22.3169GiB/s) | 20260507 15:04:59.796 | INFO | Analyzing chunk frames = [435200,435712) | 20260507 15:04:59.796 | INFO | MemInfo: { | 20260507 15:04:59.796 | INFO | "gpu" : " 17 GB", | 20260507 15:04:59.796 | INFO | "huge" : "", | 20260507 15:04:59.796 | INFO | "huge_shared" : " 29 GB", | 20260507 15:04:59.796 | INFO | "malloc" : " 14 GB", | 20260507 15:04:59.796 | INFO | "pinned" : " 29 GB" | 20260507 15:04:59.796 | INFO | } | 20260507 15:04:59.796 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:04:59.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:04:59.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.796 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:04:59.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.796 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:04:59.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:04:59.797 | INFO | Uploaded 36GiB to GPU in 1.60899s (22.3743GiB/s) | 20260507 15:04:59.797 | INFO | Downloaded 34GiB to GPU in 1.52925s (22.2331GiB/s) | 20260507 15:05:04.935 | INFO | Analyzing chunk frames = [435712,436224) | 20260507 15:05:04.935 | INFO | MemInfo: { | 20260507 15:05:04.935 | INFO | "gpu" : " 17 GB", | 20260507 15:05:04.935 | INFO | "huge" : "", | 20260507 15:05:04.935 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:04.935 | INFO | "malloc" : " 14 GB", | 20260507 15:05:04.935 | INFO | "pinned" : " 29 GB" | 20260507 15:05:04.935 | INFO | } | 20260507 15:05:04.935 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:05:04.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:04.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.935 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:05:04.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.935 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:05:04.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:04.936 | INFO | Uploaded 36GiB to GPU in 1.60873s (22.3779GiB/s) | 20260507 15:05:04.936 | INFO | Downloaded 34GiB to GPU in 1.52852s (22.2437GiB/s) | 20260507 15:05:10.075 | INFO | Analyzing chunk frames = [436224,436736) | 20260507 15:05:10.075 | INFO | MemInfo: { | 20260507 15:05:10.075 | INFO | "gpu" : " 17 GB", | 20260507 15:05:10.075 | INFO | "huge" : "", | 20260507 15:05:10.075 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:10.075 | INFO | "malloc" : " 14 GB", | 20260507 15:05:10.075 | INFO | "pinned" : " 29 GB" | 20260507 15:05:10.075 | INFO | } | 20260507 15:05:10.075 | INFO | Perf report for the last 5.14008 seconds: | 20260507 15:05:10.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:10.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.076 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:05:10.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.076 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 15:05:10.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:10.076 | INFO | Uploaded 36GiB to GPU in 1.60663s (22.4071GiB/s) | 20260507 15:05:10.076 | INFO | Downloaded 34GiB to GPU in 1.52624s (22.2769GiB/s) | 20260507 15:05:15.214 | INFO | Analyzing chunk frames = [436736,437248) | 20260507 15:05:15.214 | INFO | MemInfo: { | 20260507 15:05:15.214 | INFO | "gpu" : " 17 GB", | 20260507 15:05:15.214 | INFO | "huge" : "", | 20260507 15:05:15.214 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:15.214 | INFO | "malloc" : " 14 GB", | 20260507 15:05:15.214 | INFO | "pinned" : " 29 GB" | 20260507 15:05:15.214 | INFO | } | 20260507 15:05:15.214 | INFO | Perf report for the last 5.13884 seconds: | 20260507 15:05:15.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:15.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.214 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:05:15.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.214 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:05:15.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:15.215 | INFO | Uploaded 36GiB to GPU in 1.60711s (22.4004GiB/s) | 20260507 15:05:15.215 | INFO | Downloaded 34GiB to GPU in 1.52617s (22.278GiB/s) | 20260507 15:05:20.353 | INFO | Analyzing chunk frames = [437248,437760) | 20260507 15:05:20.353 | INFO | MemInfo: { | 20260507 15:05:20.353 | INFO | "gpu" : " 17 GB", | 20260507 15:05:20.353 | INFO | "huge" : "", | 20260507 15:05:20.353 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:20.353 | INFO | "malloc" : " 14 GB", | 20260507 15:05:20.353 | INFO | "pinned" : " 29 GB" | 20260507 15:05:20.353 | INFO | } | 20260507 15:05:20.354 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:05:20.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:20.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:05:20.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:05:20.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:20.354 | INFO | Uploaded 36GiB to GPU in 1.60779s (22.3909GiB/s) | 20260507 15:05:20.354 | INFO | Downloaded 34GiB to GPU in 1.52772s (22.2553GiB/s) | 20260507 15:05:25.493 | INFO | Analyzing chunk frames = [437760,438272) | 20260507 15:05:25.493 | INFO | MemInfo: { | 20260507 15:05:25.493 | INFO | "gpu" : " 17 GB", | 20260507 15:05:25.493 | INFO | "huge" : "", | 20260507 15:05:25.493 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:25.493 | INFO | "malloc" : " 14 GB", | 20260507 15:05:25.493 | INFO | "pinned" : " 29 GB" | 20260507 15:05:25.493 | INFO | } | 20260507 15:05:25.493 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:05:25.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:25.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 15:05:25.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 15:05:25.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:25.493 | INFO | Uploaded 36GiB to GPU in 1.60618s (22.4135GiB/s) | 20260507 15:05:25.493 | INFO | Downloaded 34GiB to GPU in 1.52671s (22.2701GiB/s) | 20260507 15:05:25.590 | INFO | Starting a round of Baz IO | 20260507 15:05:28.571 | INFO | Populating RTMetrics report | 20260507 15:05:28.572 | INFO | RTMetrics report populated | 20260507 15:05:28.572 | INFO | Opening tmp RTMetrics json file | 20260507 15:05:28.584 | INFO | Serializing RTMetrics to json | 20260507 15:05:28.586 | INFO | Serialization done, swapping tmp file now... | 20260507 15:05:28.587 | INFO | RTMetrics done | 20260507 15:05:28.864 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.274 seconds | 20260507 15:05:28.864 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.89% / 0.06% / 45.06% | 20260507 15:05:28.865 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.49684% / 0.00013113% / 2.38419e-05% | 20260507 15:05:28.865 | INFO | Baz IO overall: | 20260507 15:05:28.865 | INFO | BytesWritten : 171367576878 | 20260507 15:05:28.865 | INFO | NumEvents : 82219500225 | 20260507 15:05:30.652 | INFO | Analyzing chunk frames = [438272,438784) | 20260507 15:05:30.653 | INFO | MemInfo: { | 20260507 15:05:30.653 | INFO | "gpu" : " 17 GB", | 20260507 15:05:30.653 | INFO | "huge" : "", | 20260507 15:05:30.653 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:30.653 | INFO | "malloc" : " 14 GB", | 20260507 15:05:30.653 | INFO | "pinned" : " 29 GB" | 20260507 15:05:30.653 | INFO | } | 20260507 15:05:30.653 | INFO | Perf report for the last 5.15998 seconds: | 20260507 15:05:30.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:30.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:30.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:30.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:30.653 | INFO | ANALYSIS : 58.0%, 2.8, 84.1ms, 0 | 20260507 15:05:30.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:30.653 | INFO | RT_METRICS : 46.9%, 1.0, 24.7ms, 0 | 20260507 15:05:30.653 | INFO | PRE_HQ : 57.4%, 2.2, 65.9ms, 0 | 20260507 15:05:30.653 | INFO | BAZWRITER : 63.0%, 4.7, 151.9ms, 0 | 20260507 15:05:30.653 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 15:05:30.653 | INFO | Uploaded 36GiB to GPU in 1.6454s (21.8792GiB/s) | 20260507 15:05:30.654 | INFO | Downloaded 38GiB to GPU in 1.7261s (22.015GiB/s) | 20260507 15:05:35.772 | INFO | Analyzing chunk frames = [438784,439296) | 20260507 15:05:35.772 | INFO | MemInfo: { | 20260507 15:05:35.772 | INFO | "gpu" : " 17 GB", | 20260507 15:05:35.772 | INFO | "huge" : "", | 20260507 15:05:35.772 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:35.772 | INFO | "malloc" : " 14 GB", | 20260507 15:05:35.772 | INFO | "pinned" : " 29 GB" | 20260507 15:05:35.772 | INFO | } | 20260507 15:05:35.772 | INFO | Perf report for the last 5.11896 seconds: | 20260507 15:05:35.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:35.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260507 15:05:35.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | PRE_HQ : 39.9%, 1.0, 20.6ms, 0 | 20260507 15:05:35.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:35.772 | INFO | Uploaded 36GiB to GPU in 1.61017s (22.3579GiB/s) | 20260507 15:05:35.772 | INFO | Downloaded 34GiB to GPU in 1.52983s (22.2246GiB/s) | 20260507 15:05:40.911 | INFO | Analyzing chunk frames = [439296,439808) | 20260507 15:05:40.911 | INFO | MemInfo: { | 20260507 15:05:40.911 | INFO | "gpu" : " 17 GB", | 20260507 15:05:40.911 | INFO | "huge" : "", | 20260507 15:05:40.911 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:40.911 | INFO | "malloc" : " 14 GB", | 20260507 15:05:40.911 | INFO | "pinned" : " 29 GB" | 20260507 15:05:40.911 | INFO | } | 20260507 15:05:40.912 | INFO | Perf report for the last 5.13969 seconds: | 20260507 15:05:40.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:40.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | ANALYSIS : 56.5%, 3.0, 87.9ms, 0 | 20260507 15:05:40.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 15:05:40.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:40.912 | INFO | Uploaded 36GiB to GPU in 1.61293s (22.3196GiB/s) | 20260507 15:05:40.912 | INFO | Downloaded 34GiB to GPU in 1.53309s (22.1774GiB/s) | 20260507 15:05:46.050 | INFO | Analyzing chunk frames = [439808,440320) | 20260507 15:05:46.050 | INFO | MemInfo: { | 20260507 15:05:46.050 | INFO | "gpu" : " 17 GB", | 20260507 15:05:46.050 | INFO | "huge" : "", | 20260507 15:05:46.050 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:46.050 | INFO | "malloc" : " 14 GB", | 20260507 15:05:46.050 | INFO | "pinned" : " 29 GB" | 20260507 15:05:46.050 | INFO | } | 20260507 15:05:46.051 | INFO | Perf report for the last 5.13897 seconds: | 20260507 15:05:46.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:46.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260507 15:05:46.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | PRE_HQ : 39.9%, 1.0, 20.7ms, 0 | 20260507 15:05:46.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:46.051 | INFO | Uploaded 36GiB to GPU in 1.61467s (22.2956GiB/s) | 20260507 15:05:46.051 | INFO | Downloaded 34GiB to GPU in 1.53379s (22.1673GiB/s) | 20260507 15:05:51.189 | INFO | Analyzing chunk frames = [440320,440832) | 20260507 15:05:51.189 | INFO | MemInfo: { | 20260507 15:05:51.189 | INFO | "gpu" : " 17 GB", | 20260507 15:05:51.189 | INFO | "huge" : "", | 20260507 15:05:51.189 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:51.189 | INFO | "malloc" : " 14 GB", | 20260507 15:05:51.190 | INFO | "pinned" : " 29 GB" | 20260507 15:05:51.190 | INFO | } | 20260507 15:05:51.190 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:05:51.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:51.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 15:05:51.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | PRE_HQ : 41.5%, 1.0, 21.5ms, 0 | 20260507 15:05:51.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:51.190 | INFO | Uploaded 36GiB to GPU in 1.61327s (22.3149GiB/s) | 20260507 15:05:51.190 | INFO | Downloaded 34GiB to GPU in 1.53235s (22.1881GiB/s) | 20260507 15:05:56.329 | INFO | Analyzing chunk frames = [440832,441344) | 20260507 15:05:56.329 | INFO | MemInfo: { | 20260507 15:05:56.329 | INFO | "gpu" : " 17 GB", | 20260507 15:05:56.329 | INFO | "huge" : "", | 20260507 15:05:56.329 | INFO | "huge_shared" : " 29 GB", | 20260507 15:05:56.329 | INFO | "malloc" : " 14 GB", | 20260507 15:05:56.329 | INFO | "pinned" : " 29 GB" | 20260507 15:05:56.329 | INFO | } | 20260507 15:05:56.329 | INFO | Perf report for the last 5.13904 seconds: | 20260507 15:05:56.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:05:56.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:05:56.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | PRE_HQ : 40.5%, 1.0, 21.0ms, 0 | 20260507 15:05:56.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:05:56.329 | INFO | Uploaded 36GiB to GPU in 1.61307s (22.3177GiB/s) | 20260507 15:05:56.329 | INFO | Downloaded 34GiB to GPU in 1.53237s (22.1879GiB/s) | 20260507 15:06:01.468 | INFO | Analyzing chunk frames = [441344,441856) | 20260507 15:06:01.468 | INFO | MemInfo: { | 20260507 15:06:01.468 | INFO | "gpu" : " 17 GB", | 20260507 15:06:01.468 | INFO | "huge" : "", | 20260507 15:06:01.468 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:01.468 | INFO | "malloc" : " 14 GB", | 20260507 15:06:01.468 | INFO | "pinned" : " 29 GB" | 20260507 15:06:01.468 | INFO | } | 20260507 15:06:01.468 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:06:01.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:01.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:06:01.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:06:01.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:01.468 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4213GiB/s) | 20260507 15:06:01.469 | INFO | Downloaded 34GiB to GPU in 1.52835s (22.2462GiB/s) | 20260507 15:06:06.607 | INFO | Analyzing chunk frames = [441856,442368) | 20260507 15:06:06.607 | INFO | MemInfo: { | 20260507 15:06:06.607 | INFO | "gpu" : " 17 GB", | 20260507 15:06:06.607 | INFO | "huge" : "", | 20260507 15:06:06.607 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:06.607 | INFO | "malloc" : " 14 GB", | 20260507 15:06:06.607 | INFO | "pinned" : " 29 GB" | 20260507 15:06:06.607 | INFO | } | 20260507 15:06:06.607 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:06:06.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:06.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.608 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:06:06.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.608 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:06:06.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:06.608 | INFO | Uploaded 36GiB to GPU in 1.60789s (22.3895GiB/s) | 20260507 15:06:06.608 | INFO | Downloaded 34GiB to GPU in 1.52801s (22.2512GiB/s) | 20260507 15:06:06.608 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:06:06.608 | INFO | Total mem usage: 16.4 GiB | 20260507 15:06:06.608 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:06:06.608 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8%) | 20260507 15:06:06.608 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:06:06.608 | INFO | High Water Breakdown: | 20260507 15:06:06.608 | INFO | 59.3% : 9.01 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:06:06.608 | INFO | 40.6% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:06:06.608 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:06:06.608 | INFO | ------------------------------------------ | 20260507 15:06:06.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:06:06.608 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:06:06.608 | INFO | Total mem usage: 26.5 GiB | 20260507 15:06:06.608 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:06:06.608 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 15:06:06.608 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:06:06.608 | INFO | High Water Breakdown: | 20260507 15:06:06.608 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:06:06.608 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:06:06.608 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:06:06.608 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:06:06.608 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:06:06.608 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:06:06.608 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:06:06.608 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:06:06.608 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:06:06.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:06:06.608 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:06:06.608 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:06:06.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:06:06.608 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:06:06.608 | INFO | ------------------------------------------ | 20260507 15:06:06.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:06:06.608 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:06:06.608 | INFO | Total mem usage: 27.8 GiB | 20260507 15:06:06.608 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:06:06.608 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:06:06.608 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:06:06.608 | INFO | High Water Breakdown: | 20260507 15:06:06.608 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:06:06.608 | INFO | ------------------------------------------ | 20260507 15:06:06.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:06:06.608 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:06:06.608 | INFO | Total mem usage: 15.7 GiB | 20260507 15:06:06.608 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:06:06.608 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:06:06.608 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:06:06.608 | INFO | High Water Breakdown: | 20260507 15:06:06.608 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:06:06.608 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:06:06.608 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:06:06.608 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:06:06.608 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:06:06.608 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:06:06.608 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:06:06.608 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:06:06.608 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:06:06.608 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:06:06.608 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:06:06.608 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:06:06.608 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:06:06.608 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:06:06.608 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:06:06.608 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:06:06.608 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:06:06.608 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:06:06.608 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:06:06.608 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:06:06.608 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:06:06.608 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:06:06.608 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:06:06.608 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:06:06.608 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:06:06.608 | INFO | ------------------------------------------ | 20260507 15:06:06.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:06:06.608 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:06:06.608 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:06:06.608 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:06:06.608 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 0.30%, AggregateStats -- 8.88e-02 / -nan, 6.12e-03 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 11.02%, Baseline -- 3.30e+00 / -nan, 2.13e-02 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.34e-02 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 16.11%, DME -- 3.86e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.72e+01 / -nan, 4.50e-02 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.17e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 3.54%, Metrics -- 1.06e+00 / -nan, 2.38e-01 / 0.00e+00 | 20260507 15:06:06.608 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.93e-01 / -nan, 5.67e-02 / 0.00e+00 | 20260507 15:06:06.608 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:06:06.609 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:06:06.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:06:06.609 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 18.80%, Upload -- 1.68e+01 / 0.00e+00, 4.29e-01 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 18.10%, Download -- 1.62e+01 / 0.00e+00, 7.91e-01 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 33.49%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 0.59%, LockUpload -- 5.28e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 0.00%, LockDownload -- 5.21e-04 / 0.00e+00, 3.68e-04 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 29.00%, LockCompute -- 2.59e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 15:06:06.609 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260507 15:06:06.704 | INFO | Starting a round of Baz IO | 20260507 15:06:09.650 | INFO | Populating RTMetrics report | 20260507 15:06:09.651 | INFO | RTMetrics report populated | 20260507 15:06:09.651 | INFO | Opening tmp RTMetrics json file | 20260507 15:06:09.675 | INFO | Serializing RTMetrics to json | 20260507 15:06:09.677 | INFO | Serialization done, swapping tmp file now... | 20260507 15:06:09.678 | INFO | RTMetrics done | 20260507 15:06:09.881 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.176 seconds | 20260507 15:06:09.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.83% / 0.06% / 45.11% | 20260507 15:06:09.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.46323% / 0.000107288% / 1.98682e-05% | 20260507 15:06:09.882 | INFO | Baz IO overall: | 20260507 15:06:09.882 | INFO | BytesWritten : 173224308484 | 20260507 15:06:09.882 | INFO | NumEvents : 83123452063 | 20260507 15:06:11.747 | INFO | Analyzing chunk frames = [442368,442880) | 20260507 15:06:11.747 | INFO | MemInfo: { | 20260507 15:06:11.747 | INFO | "gpu" : " 17 GB", | 20260507 15:06:11.747 | INFO | "huge" : "", | 20260507 15:06:11.747 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:11.747 | INFO | "malloc" : " 14 GB", | 20260507 15:06:11.747 | INFO | "pinned" : " 29 GB" | 20260507 15:06:11.747 | INFO | } | 20260507 15:06:11.747 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:06:11.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:11.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:11.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:11.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:11.747 | INFO | ANALYSIS : 57.4%, 2.9, 86.9ms, 0 | 20260507 15:06:11.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:11.747 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 15:06:11.747 | INFO | PRE_HQ : 56.6%, 2.1, 60.9ms, 0 | 20260507 15:06:11.747 | INFO | BAZWRITER : 61.4%, 4.5, 142.7ms, 0 | 20260507 15:06:11.747 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 15:06:11.747 | INFO | Uploaded 36GiB to GPU in 1.64519s (21.882GiB/s) | 20260507 15:06:11.747 | INFO | Downloaded 38GiB to GPU in 1.72977s (21.9682GiB/s) | 20260507 15:06:16.886 | INFO | Analyzing chunk frames = [442880,443392) | 20260507 15:06:16.886 | INFO | MemInfo: { | 20260507 15:06:16.886 | INFO | "gpu" : " 17 GB", | 20260507 15:06:16.886 | INFO | "huge" : "", | 20260507 15:06:16.886 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:16.886 | INFO | "malloc" : " 14 GB", | 20260507 15:06:16.886 | INFO | "pinned" : " 29 GB" | 20260507 15:06:16.886 | INFO | } | 20260507 15:06:16.886 | INFO | Perf report for the last 5.13951 seconds: | 20260507 15:06:16.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:16.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.886 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:06:16.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.887 | INFO | PRE_HQ : 27.7%, 1.0, 14.5ms, 0 | 20260507 15:06:16.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:16.887 | INFO | Uploaded 36GiB to GPU in 1.59175s (22.6166GiB/s) | 20260507 15:06:16.887 | INFO | Downloaded 34GiB to GPU in 1.52998s (22.2225GiB/s) | 20260507 15:06:22.025 | INFO | Analyzing chunk frames = [443392,443904) | 20260507 15:06:22.025 | INFO | MemInfo: { | 20260507 15:06:22.025 | INFO | "gpu" : " 17 GB", | 20260507 15:06:22.025 | INFO | "huge" : "", | 20260507 15:06:22.025 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:22.025 | INFO | "malloc" : " 14 GB", | 20260507 15:06:22.025 | INFO | "pinned" : " 29 GB" | 20260507 15:06:22.025 | INFO | } | 20260507 15:06:22.025 | INFO | Perf report for the last 5.13904 seconds: | 20260507 15:06:22.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:22.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:06:22.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:06:22.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:22.026 | INFO | Uploaded 36GiB to GPU in 1.60896s (22.3746GiB/s) | 20260507 15:06:22.026 | INFO | Downloaded 34GiB to GPU in 1.52897s (22.2371GiB/s) | 20260507 15:06:27.165 | INFO | Analyzing chunk frames = [443904,444416) | 20260507 15:06:27.165 | INFO | MemInfo: { | 20260507 15:06:27.165 | INFO | "gpu" : " 17 GB", | 20260507 15:06:27.165 | INFO | "huge" : "", | 20260507 15:06:27.165 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:27.165 | INFO | "malloc" : " 14 GB", | 20260507 15:06:27.165 | INFO | "pinned" : " 29 GB" | 20260507 15:06:27.165 | INFO | } | 20260507 15:06:27.165 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:06:27.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:27.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:06:27.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:06:27.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:27.165 | INFO | Uploaded 36GiB to GPU in 1.60861s (22.3796GiB/s) | 20260507 15:06:27.165 | INFO | Downloaded 34GiB to GPU in 1.52907s (22.2358GiB/s) | 20260507 15:06:32.304 | INFO | Analyzing chunk frames = [444416,444928) | 20260507 15:06:32.304 | INFO | MemInfo: { | 20260507 15:06:32.304 | INFO | "gpu" : " 17 GB", | 20260507 15:06:32.304 | INFO | "huge" : "", | 20260507 15:06:32.304 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:32.304 | INFO | "malloc" : " 14 GB", | 20260507 15:06:32.304 | INFO | "pinned" : " 29 GB" | 20260507 15:06:32.304 | INFO | } | 20260507 15:06:32.304 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:06:32.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:32.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.304 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:06:32.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.304 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 15:06:32.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:32.305 | INFO | Uploaded 36GiB to GPU in 1.60638s (22.4106GiB/s) | 20260507 15:06:32.305 | INFO | Downloaded 34GiB to GPU in 1.52652s (22.2728GiB/s) | 20260507 15:06:37.443 | INFO | Analyzing chunk frames = [444928,445440) | 20260507 15:06:37.443 | INFO | MemInfo: { | 20260507 15:06:37.443 | INFO | "gpu" : " 17 GB", | 20260507 15:06:37.443 | INFO | "huge" : "", | 20260507 15:06:37.443 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:37.444 | INFO | "malloc" : " 14 GB", | 20260507 15:06:37.444 | INFO | "pinned" : " 29 GB" | 20260507 15:06:37.444 | INFO | } | 20260507 15:06:37.444 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:06:37.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:37.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:06:37.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:06:37.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:37.444 | INFO | Uploaded 36GiB to GPU in 1.60771s (22.3921GiB/s) | 20260507 15:06:37.444 | INFO | Downloaded 34GiB to GPU in 1.52676s (22.2694GiB/s) | 20260507 15:06:42.583 | INFO | Analyzing chunk frames = [445440,445952) | 20260507 15:06:42.583 | INFO | MemInfo: { | 20260507 15:06:42.583 | INFO | "gpu" : " 17 GB", | 20260507 15:06:42.583 | INFO | "huge" : "", | 20260507 15:06:42.583 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:42.583 | INFO | "malloc" : " 14 GB", | 20260507 15:06:42.583 | INFO | "pinned" : " 29 GB" | 20260507 15:06:42.583 | INFO | } | 20260507 15:06:42.583 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:06:42.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:42.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:06:42.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:06:42.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:42.583 | INFO | Uploaded 36GiB to GPU in 1.60656s (22.4081GiB/s) | 20260507 15:06:42.584 | INFO | Downloaded 34GiB to GPU in 1.52857s (22.2431GiB/s) | 20260507 15:06:47.722 | INFO | Analyzing chunk frames = [445952,446464) | 20260507 15:06:47.722 | INFO | MemInfo: { | 20260507 15:06:47.722 | INFO | "gpu" : " 17 GB", | 20260507 15:06:47.722 | INFO | "huge" : "", | 20260507 15:06:47.722 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:47.722 | INFO | "malloc" : " 14 GB", | 20260507 15:06:47.722 | INFO | "pinned" : " 29 GB" | 20260507 15:06:47.722 | INFO | } | 20260507 15:06:47.723 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:06:47.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:47.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:06:47.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 15:06:47.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:47.723 | INFO | Uploaded 36GiB to GPU in 1.60942s (22.3684GiB/s) | 20260507 15:06:47.723 | INFO | Downloaded 34GiB to GPU in 1.5299s (22.2237GiB/s) | 20260507 15:06:47.818 | INFO | Starting a round of Baz IO | 20260507 15:06:50.821 | INFO | Populating RTMetrics report | 20260507 15:06:50.822 | INFO | RTMetrics report populated | 20260507 15:06:50.822 | INFO | Opening tmp RTMetrics json file | 20260507 15:06:50.847 | INFO | Serializing RTMetrics to json | 20260507 15:06:50.881 | INFO | Serialization done, swapping tmp file now... | 20260507 15:06:50.884 | INFO | RTMetrics done | 20260507 15:06:51.022 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.203 seconds | 20260507 15:06:51.022 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.78% / 0.05% / 45.17% | 20260507 15:06:51.022 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.44337% / 0.000111262% / 1.58946e-05% | 20260507 15:06:51.022 | INFO | Baz IO overall: | 20260507 15:06:51.022 | INFO | BytesWritten : 175081127307 | 20260507 15:06:51.022 | INFO | NumEvents : 84027464785 | 20260507 15:06:52.861 | INFO | Analyzing chunk frames = [446464,446976) | 20260507 15:06:52.861 | INFO | MemInfo: { | 20260507 15:06:52.861 | INFO | "gpu" : " 17 GB", | 20260507 15:06:52.861 | INFO | "huge" : "", | 20260507 15:06:52.862 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:52.862 | INFO | "malloc" : " 14 GB", | 20260507 15:06:52.862 | INFO | "pinned" : " 29 GB" | 20260507 15:06:52.862 | INFO | } | 20260507 15:06:52.862 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:06:52.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:52.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:52.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:52.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:52.862 | INFO | ANALYSIS : 58.0%, 2.8, 85.6ms, 0 | 20260507 15:06:52.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:52.862 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260507 15:06:52.862 | INFO | PRE_HQ : 57.3%, 2.2, 65.7ms, 0 | 20260507 15:06:52.862 | INFO | BAZWRITER : 61.9%, 4.5, 142.6ms, 0 | 20260507 15:06:52.862 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:06:52.862 | INFO | Uploaded 36GiB to GPU in 1.63777s (21.9812GiB/s) | 20260507 15:06:52.862 | INFO | Downloaded 38GiB to GPU in 1.72084s (22.0823GiB/s) | 20260507 15:06:58.001 | INFO | Analyzing chunk frames = [446976,447488) | 20260507 15:06:58.001 | INFO | MemInfo: { | 20260507 15:06:58.001 | INFO | "gpu" : " 17 GB", | 20260507 15:06:58.001 | INFO | "huge" : "", | 20260507 15:06:58.001 | INFO | "huge_shared" : " 29 GB", | 20260507 15:06:58.001 | INFO | "malloc" : " 14 GB", | 20260507 15:06:58.001 | INFO | "pinned" : " 29 GB" | 20260507 15:06:58.001 | INFO | } | 20260507 15:06:58.001 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:06:58.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:06:58.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.001 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:06:58.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.001 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:06:58.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:06:58.002 | INFO | Uploaded 36GiB to GPU in 1.58476s (22.7163GiB/s) | 20260507 15:06:58.002 | INFO | Downloaded 34GiB to GPU in 1.53382s (22.1669GiB/s) | 20260507 15:07:03.140 | INFO | Analyzing chunk frames = [447488,448000) | 20260507 15:07:03.140 | INFO | MemInfo: { | 20260507 15:07:03.140 | INFO | "gpu" : " 17 GB", | 20260507 15:07:03.140 | INFO | "huge" : "", | 20260507 15:07:03.140 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:03.140 | INFO | "malloc" : " 14 GB", | 20260507 15:07:03.140 | INFO | "pinned" : " 29 GB" | 20260507 15:07:03.140 | INFO | } | 20260507 15:07:03.140 | INFO | Perf report for the last 5.13899 seconds: | 20260507 15:07:03.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:03.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.140 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:07:03.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.141 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:07:03.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:03.141 | INFO | Uploaded 36GiB to GPU in 1.58823s (22.6668GiB/s) | 20260507 15:07:03.141 | INFO | Downloaded 34GiB to GPU in 1.53837s (22.1013GiB/s) | 20260507 15:07:08.279 | INFO | Analyzing chunk frames = [448000,448512) | 20260507 15:07:08.280 | INFO | MemInfo: { | 20260507 15:07:08.280 | INFO | "gpu" : " 17 GB", | 20260507 15:07:08.280 | INFO | "huge" : "", | 20260507 15:07:08.280 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:08.280 | INFO | "malloc" : " 14 GB", | 20260507 15:07:08.280 | INFO | "pinned" : " 29 GB" | 20260507 15:07:08.280 | INFO | } | 20260507 15:07:08.280 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:07:08.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:08.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:07:08.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:07:08.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:08.280 | INFO | Uploaded 36GiB to GPU in 1.58868s (22.6603GiB/s) | 20260507 15:07:08.280 | INFO | Downloaded 34GiB to GPU in 1.53701s (22.1209GiB/s) | 20260507 15:07:13.419 | INFO | Analyzing chunk frames = [448512,449024) | 20260507 15:07:13.419 | INFO | MemInfo: { | 20260507 15:07:13.419 | INFO | "gpu" : " 17 GB", | 20260507 15:07:13.419 | INFO | "huge" : "", | 20260507 15:07:13.419 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:13.419 | INFO | "malloc" : " 14 GB", | 20260507 15:07:13.419 | INFO | "pinned" : " 29 GB" | 20260507 15:07:13.419 | INFO | } | 20260507 15:07:13.419 | INFO | Perf report for the last 5.13956 seconds: | 20260507 15:07:13.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:13.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.419 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:07:13.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.419 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:07:13.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:13.420 | INFO | Uploaded 36GiB to GPU in 1.58741s (22.6784GiB/s) | 20260507 15:07:13.420 | INFO | Downloaded 34GiB to GPU in 1.53656s (22.1273GiB/s) | 20260507 15:07:18.558 | INFO | Analyzing chunk frames = [449024,449536) | 20260507 15:07:18.558 | INFO | MemInfo: { | 20260507 15:07:18.558 | INFO | "gpu" : " 17 GB", | 20260507 15:07:18.558 | INFO | "huge" : "", | 20260507 15:07:18.558 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:18.558 | INFO | "malloc" : " 14 GB", | 20260507 15:07:18.558 | INFO | "pinned" : " 29 GB" | 20260507 15:07:18.558 | INFO | } | 20260507 15:07:18.559 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:07:18.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:18.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:07:18.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 15:07:18.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:18.559 | INFO | Uploaded 36GiB to GPU in 1.58603s (22.6982GiB/s) | 20260507 15:07:18.559 | INFO | Downloaded 34GiB to GPU in 1.53578s (22.1386GiB/s) | 20260507 15:07:23.698 | INFO | Analyzing chunk frames = [449536,450048) | 20260507 15:07:23.698 | INFO | MemInfo: { | 20260507 15:07:23.698 | INFO | "gpu" : " 17 GB", | 20260507 15:07:23.698 | INFO | "huge" : "", | 20260507 15:07:23.698 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:23.698 | INFO | "malloc" : " 14 GB", | 20260507 15:07:23.698 | INFO | "pinned" : " 29 GB" | 20260507 15:07:23.698 | INFO | } | 20260507 15:07:23.698 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:07:23.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:23.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:07:23.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:07:23.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:23.698 | INFO | Uploaded 36GiB to GPU in 1.58646s (22.692GiB/s) | 20260507 15:07:23.698 | INFO | Downloaded 34GiB to GPU in 1.53717s (22.1185GiB/s) | 20260507 15:07:28.837 | INFO | Analyzing chunk frames = [450048,450560) | 20260507 15:07:28.837 | INFO | MemInfo: { | 20260507 15:07:28.837 | INFO | "gpu" : " 17 GB", | 20260507 15:07:28.837 | INFO | "huge" : "", | 20260507 15:07:28.837 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:28.837 | INFO | "malloc" : " 14 GB", | 20260507 15:07:28.837 | INFO | "pinned" : " 29 GB" | 20260507 15:07:28.837 | INFO | } | 20260507 15:07:28.837 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:07:28.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:28.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 15:07:28.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 15:07:28.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:28.837 | INFO | Uploaded 36GiB to GPU in 1.58815s (22.6679GiB/s) | 20260507 15:07:28.837 | INFO | Downloaded 34GiB to GPU in 1.53807s (22.1056GiB/s) | 20260507 15:07:28.837 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:07:28.837 | INFO | Total mem usage: 16.4 GiB | 20260507 15:07:28.837 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:07:28.837 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.59%) | 20260507 15:07:28.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:07:28.838 | INFO | High Water Breakdown: | 20260507 15:07:28.838 | INFO | 59.2% : 8.94 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:07:28.838 | INFO | 40.7% : 6.15 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:07:28.838 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:07:28.838 | INFO | ------------------------------------------ | 20260507 15:07:28.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:07:28.838 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:07:28.838 | INFO | Total mem usage: 26.5 GiB | 20260507 15:07:28.838 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:07:28.838 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 15:07:28.838 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:07:28.838 | INFO | High Water Breakdown: | 20260507 15:07:28.838 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:07:28.838 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:07:28.838 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:07:28.838 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:07:28.838 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:07:28.838 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:07:28.838 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:07:28.838 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:07:28.838 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:07:28.838 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:07:28.838 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:07:28.838 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:07:28.838 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:07:28.838 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:07:28.838 | INFO | ------------------------------------------ | 20260507 15:07:28.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:07:28.838 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:07:28.838 | INFO | Total mem usage: 27.8 GiB | 20260507 15:07:28.838 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:07:28.838 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:07:28.838 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:07:28.838 | INFO | High Water Breakdown: | 20260507 15:07:28.838 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:07:28.838 | INFO | ------------------------------------------ | 20260507 15:07:28.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:07:28.838 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:07:28.838 | INFO | Total mem usage: 15.7 GiB | 20260507 15:07:28.838 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:07:28.838 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:07:28.838 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:07:28.838 | INFO | High Water Breakdown: | 20260507 15:07:28.838 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:07:28.838 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:07:28.838 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:07:28.838 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:07:28.838 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:07:28.838 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:07:28.838 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:07:28.838 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:07:28.838 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:07:28.838 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:07:28.838 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:07:28.838 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:07:28.838 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:07:28.838 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:07:28.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:07:28.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:07:28.838 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:07:28.838 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:07:28.838 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:07:28.838 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:07:28.838 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:07:28.838 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:07:28.838 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:07:28.838 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:07:28.838 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:07:28.838 | INFO | ------------------------------------------ | 20260507 15:07:28.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:07:28.838 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:07:28.838 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:07:28.838 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:07:28.838 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.92e-03 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 11.02%, Baseline -- 3.29e+00 / -nan, 2.05e-02 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 16.14%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.72e+01 / -nan, 4.41e-02 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.16e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 3.52%, Metrics -- 1.05e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260507 15:07:28.838 | INFO | PerfReport: 1.30%, PulseSerializing -- 3.89e-01 / -nan, 5.71e-02 / 0.00e+00 | 20260507 15:07:28.838 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:07:28.838 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:07:28.838 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:07:28.838 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 18.65%, Upload -- 1.67e+01 / 0.00e+00, 4.51e-01 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 18.12%, Download -- 1.62e+01 / 0.00e+00, 7.68e-01 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 33.43%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 0.58%, LockUpload -- 5.21e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 0.00%, LockDownload -- 4.48e-04 / 0.00e+00, 1.90e-04 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 29.20%, LockCompute -- 2.61e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:07:28.839 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 9.48e-03 / 0.00e+00 | 20260507 15:07:28.934 | INFO | Starting a round of Baz IO | 20260507 15:07:31.849 | INFO | Populating RTMetrics report | 20260507 15:07:31.850 | INFO | RTMetrics report populated | 20260507 15:07:31.850 | INFO | Opening tmp RTMetrics json file | 20260507 15:07:31.851 | INFO | Serializing RTMetrics to json | 20260507 15:07:31.853 | INFO | Serialization done, swapping tmp file now... | 20260507 15:07:31.853 | INFO | RTMetrics done | 20260507 15:07:32.162 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.227 seconds | 20260507 15:07:32.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.72% / 0.05% / 45.22% | 20260507 15:07:32.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.43934% / 0.00013113% / 4.76837e-05% | 20260507 15:07:32.162 | INFO | Baz IO overall: | 20260507 15:07:32.162 | INFO | BytesWritten : 176940813126 | 20260507 15:07:32.162 | INFO | NumEvents : 84933369603 | 20260507 15:07:33.977 | INFO | Analyzing chunk frames = [450560,451072) | 20260507 15:07:33.977 | INFO | MemInfo: { | 20260507 15:07:33.977 | INFO | "gpu" : " 17 GB", | 20260507 15:07:33.977 | INFO | "huge" : "", | 20260507 15:07:33.977 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:33.977 | INFO | "malloc" : " 14 GB", | 20260507 15:07:33.977 | INFO | "pinned" : " 29 GB" | 20260507 15:07:33.977 | INFO | } | 20260507 15:07:33.977 | INFO | Perf report for the last 5.1397 seconds: | 20260507 15:07:33.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:33.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:33.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:33.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:33.977 | INFO | ANALYSIS : 57.7%, 2.9, 86.8ms, 0 | 20260507 15:07:33.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:33.977 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 15:07:33.977 | INFO | PRE_HQ : 56.7%, 2.2, 64.0ms, 0 | 20260507 15:07:33.977 | INFO | BAZWRITER : 62.3%, 4.3, 138.7ms, 0 | 20260507 15:07:33.977 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:07:33.977 | INFO | Uploaded 36GiB to GPU in 1.62596s (22.1407GiB/s) | 20260507 15:07:33.977 | INFO | Downloaded 38GiB to GPU in 1.7066s (22.2665GiB/s) | 20260507 15:07:39.116 | INFO | Analyzing chunk frames = [451072,451584) | 20260507 15:07:39.116 | INFO | MemInfo: { | 20260507 15:07:39.116 | INFO | "gpu" : " 17 GB", | 20260507 15:07:39.116 | INFO | "huge" : "", | 20260507 15:07:39.116 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:39.116 | INFO | "malloc" : " 14 GB", | 20260507 15:07:39.116 | INFO | "pinned" : " 29 GB" | 20260507 15:07:39.116 | INFO | } | 20260507 15:07:39.116 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:07:39.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:39.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.116 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260507 15:07:39.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.117 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 15:07:39.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:39.117 | INFO | Uploaded 36GiB to GPU in 1.6101s (22.3588GiB/s) | 20260507 15:07:39.117 | INFO | Downloaded 34GiB to GPU in 1.52904s (22.2362GiB/s) | 20260507 15:07:44.255 | INFO | Analyzing chunk frames = [451584,452096) | 20260507 15:07:44.255 | INFO | MemInfo: { | 20260507 15:07:44.255 | INFO | "gpu" : " 17 GB", | 20260507 15:07:44.255 | INFO | "huge" : "", | 20260507 15:07:44.255 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:44.255 | INFO | "malloc" : " 14 GB", | 20260507 15:07:44.255 | INFO | "pinned" : " 29 GB" | 20260507 15:07:44.255 | INFO | } | 20260507 15:07:44.255 | INFO | Perf report for the last 5.13897 seconds: | 20260507 15:07:44.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:44.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.255 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 15:07:44.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.255 | INFO | PRE_HQ : 39.8%, 1.0, 20.7ms, 0 | 20260507 15:07:44.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:44.256 | INFO | Uploaded 36GiB to GPU in 1.61471s (22.295GiB/s) | 20260507 15:07:44.256 | INFO | Downloaded 34GiB to GPU in 1.53356s (22.1706GiB/s) | 20260507 15:07:49.394 | INFO | Analyzing chunk frames = [452096,452608) | 20260507 15:07:49.394 | INFO | MemInfo: { | 20260507 15:07:49.394 | INFO | "gpu" : " 17 GB", | 20260507 15:07:49.394 | INFO | "huge" : "", | 20260507 15:07:49.394 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:49.394 | INFO | "malloc" : " 14 GB", | 20260507 15:07:49.395 | INFO | "pinned" : " 29 GB" | 20260507 15:07:49.395 | INFO | } | 20260507 15:07:49.395 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:07:49.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:49.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:07:49.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:07:49.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:49.395 | INFO | Uploaded 36GiB to GPU in 1.60779s (22.391GiB/s) | 20260507 15:07:49.395 | INFO | Downloaded 34GiB to GPU in 1.52869s (22.2413GiB/s) | 20260507 15:07:54.534 | INFO | Analyzing chunk frames = [452608,453120) | 20260507 15:07:54.534 | INFO | MemInfo: { | 20260507 15:07:54.534 | INFO | "gpu" : " 17 GB", | 20260507 15:07:54.534 | INFO | "huge" : "", | 20260507 15:07:54.534 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:54.534 | INFO | "malloc" : " 14 GB", | 20260507 15:07:54.534 | INFO | "pinned" : " 29 GB" | 20260507 15:07:54.534 | INFO | } | 20260507 15:07:54.534 | INFO | Perf report for the last 5.13901 seconds: | 20260507 15:07:54.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:54.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:07:54.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:07:54.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:54.534 | INFO | Uploaded 36GiB to GPU in 1.60715s (22.3999GiB/s) | 20260507 15:07:54.534 | INFO | Downloaded 34GiB to GPU in 1.52692s (22.267GiB/s) | 20260507 15:07:59.673 | INFO | Analyzing chunk frames = [453120,453632) | 20260507 15:07:59.673 | INFO | MemInfo: { | 20260507 15:07:59.673 | INFO | "gpu" : " 17 GB", | 20260507 15:07:59.673 | INFO | "huge" : "", | 20260507 15:07:59.673 | INFO | "huge_shared" : " 29 GB", | 20260507 15:07:59.673 | INFO | "malloc" : " 14 GB", | 20260507 15:07:59.673 | INFO | "pinned" : " 29 GB" | 20260507 15:07:59.673 | INFO | } | 20260507 15:07:59.673 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:07:59.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:07:59.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:07:59.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:07:59.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:07:59.673 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.409GiB/s) | 20260507 15:07:59.674 | INFO | Downloaded 34GiB to GPU in 1.52659s (22.2718GiB/s) | 20260507 15:08:04.812 | INFO | Analyzing chunk frames = [453632,454144) | 20260507 15:08:04.812 | INFO | MemInfo: { | 20260507 15:08:04.812 | INFO | "gpu" : " 17 GB", | 20260507 15:08:04.812 | INFO | "huge" : "", | 20260507 15:08:04.812 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:04.812 | INFO | "malloc" : " 14 GB", | 20260507 15:08:04.812 | INFO | "pinned" : " 29 GB" | 20260507 15:08:04.812 | INFO | } | 20260507 15:08:04.812 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:08:04.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:04.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:08:04.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:08:04.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:04.813 | INFO | Uploaded 36GiB to GPU in 1.60742s (22.3962GiB/s) | 20260507 15:08:04.813 | INFO | Downloaded 34GiB to GPU in 1.52786s (22.2534GiB/s) | 20260507 15:08:09.952 | INFO | Analyzing chunk frames = [454144,454656) | 20260507 15:08:09.952 | INFO | MemInfo: { | 20260507 15:08:09.952 | INFO | "gpu" : " 17 GB", | 20260507 15:08:09.952 | INFO | "huge" : "", | 20260507 15:08:09.952 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:09.952 | INFO | "malloc" : " 14 GB", | 20260507 15:08:09.952 | INFO | "pinned" : " 29 GB" | 20260507 15:08:09.952 | INFO | } | 20260507 15:08:09.952 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:08:09.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:09.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:08:09.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:08:09.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:09.952 | INFO | Uploaded 36GiB to GPU in 1.60706s (22.4011GiB/s) | 20260507 15:08:09.952 | INFO | Downloaded 34GiB to GPU in 1.52783s (22.2539GiB/s) | 20260507 15:08:10.049 | INFO | Starting a round of Baz IO | 20260507 15:08:12.951 | INFO | Populating RTMetrics report | 20260507 15:08:12.951 | INFO | RTMetrics report populated | 20260507 15:08:12.951 | INFO | Opening tmp RTMetrics json file | 20260507 15:08:13.000 | INFO | Serializing RTMetrics to json | 20260507 15:08:13.049 | INFO | Serialization done, swapping tmp file now... | 20260507 15:08:13.057 | INFO | RTMetrics done | 20260507 15:08:13.173 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.124 seconds | 20260507 15:08:13.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.67% / 0.05% / 45.28% | 20260507 15:08:13.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.40968% / 0.00013113% / 2.38419e-05% | 20260507 15:08:13.173 | INFO | Baz IO overall: | 20260507 15:08:13.173 | INFO | BytesWritten : 178800638209 | 20260507 15:08:13.174 | INFO | NumEvents : 85839256900 | 20260507 15:08:15.091 | INFO | Analyzing chunk frames = [454656,455168) | 20260507 15:08:15.091 | INFO | MemInfo: { | 20260507 15:08:15.091 | INFO | "gpu" : " 17 GB", | 20260507 15:08:15.091 | INFO | "huge" : "", | 20260507 15:08:15.091 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:15.091 | INFO | "malloc" : " 14 GB", | 20260507 15:08:15.091 | INFO | "pinned" : " 29 GB" | 20260507 15:08:15.091 | INFO | } | 20260507 15:08:15.091 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:08:15.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:15.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:15.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:15.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:15.091 | INFO | ANALYSIS : 57.5%, 2.9, 86.5ms, 0 | 20260507 15:08:15.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:15.091 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260507 15:08:15.092 | INFO | PRE_HQ : 56.4%, 2.1, 61.6ms, 0 | 20260507 15:08:15.092 | INFO | BAZWRITER : 60.3%, 4.5, 138.0ms, 0 | 20260507 15:08:15.092 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 15:08:15.092 | INFO | Uploaded 36GiB to GPU in 1.64323s (21.908GiB/s) | 20260507 15:08:15.092 | INFO | Downloaded 38GiB to GPU in 1.7255s (22.0226GiB/s) | 20260507 15:08:20.230 | INFO | Analyzing chunk frames = [455168,455680) | 20260507 15:08:20.230 | INFO | MemInfo: { | 20260507 15:08:20.230 | INFO | "gpu" : " 17 GB", | 20260507 15:08:20.230 | INFO | "huge" : "", | 20260507 15:08:20.230 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:20.230 | INFO | "malloc" : " 14 GB", | 20260507 15:08:20.230 | INFO | "pinned" : " 29 GB" | 20260507 15:08:20.230 | INFO | } | 20260507 15:08:20.230 | INFO | Perf report for the last 5.13907 seconds: | 20260507 15:08:20.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:20.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:08:20.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:08:20.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:20.231 | INFO | Uploaded 36GiB to GPU in 1.60156s (22.4781GiB/s) | 20260507 15:08:20.231 | INFO | Downloaded 34GiB to GPU in 1.52458s (22.3012GiB/s) | 20260507 15:08:25.370 | INFO | Analyzing chunk frames = [455680,456192) | 20260507 15:08:25.370 | INFO | MemInfo: { | 20260507 15:08:25.370 | INFO | "gpu" : " 17 GB", | 20260507 15:08:25.370 | INFO | "huge" : "", | 20260507 15:08:25.370 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:25.370 | INFO | "malloc" : " 14 GB", | 20260507 15:08:25.370 | INFO | "pinned" : " 29 GB" | 20260507 15:08:25.370 | INFO | } | 20260507 15:08:25.370 | INFO | Perf report for the last 5.13963 seconds: | 20260507 15:08:25.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:25.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.370 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:08:25.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.370 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:08:25.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:25.371 | INFO | Uploaded 36GiB to GPU in 1.60972s (22.3641GiB/s) | 20260507 15:08:25.371 | INFO | Downloaded 34GiB to GPU in 1.52995s (22.223GiB/s) | 20260507 15:08:30.532 | INFO | Analyzing chunk frames = [456192,456704) | 20260507 15:08:30.532 | INFO | MemInfo: { | 20260507 15:08:30.532 | INFO | "gpu" : " 17 GB", | 20260507 15:08:30.532 | INFO | "huge" : "", | 20260507 15:08:30.532 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:30.532 | INFO | "malloc" : " 14 GB", | 20260507 15:08:30.532 | INFO | "pinned" : " 29 GB" | 20260507 15:08:30.532 | INFO | } | 20260507 15:08:30.532 | INFO | Perf report for the last 5.16191 seconds: | 20260507 15:08:30.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:30.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.532 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:08:30.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.532 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:08:30.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:30.533 | INFO | Uploaded 36GiB to GPU in 1.60891s (22.3754GiB/s) | 20260507 15:08:30.533 | INFO | Downloaded 34GiB to GPU in 1.52821s (22.2482GiB/s) | 20260507 15:08:35.649 | INFO | Analyzing chunk frames = [456704,457216) | 20260507 15:08:35.649 | INFO | MemInfo: { | 20260507 15:08:35.649 | INFO | "gpu" : " 17 GB", | 20260507 15:08:35.649 | INFO | "huge" : "", | 20260507 15:08:35.649 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:35.649 | INFO | "malloc" : " 14 GB", | 20260507 15:08:35.649 | INFO | "pinned" : " 29 GB" | 20260507 15:08:35.649 | INFO | } | 20260507 15:08:35.649 | INFO | Perf report for the last 5.11705 seconds: | 20260507 15:08:35.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:35.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.649 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:08:35.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.649 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:08:35.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:35.650 | INFO | Uploaded 36GiB to GPU in 1.60773s (22.3918GiB/s) | 20260507 15:08:35.650 | INFO | Downloaded 34GiB to GPU in 1.5277s (22.2557GiB/s) | 20260507 15:08:40.788 | INFO | Analyzing chunk frames = [457216,457728) | 20260507 15:08:40.788 | INFO | MemInfo: { | 20260507 15:08:40.788 | INFO | "gpu" : " 17 GB", | 20260507 15:08:40.788 | INFO | "huge" : "", | 20260507 15:08:40.788 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:40.788 | INFO | "malloc" : " 14 GB", | 20260507 15:08:40.788 | INFO | "pinned" : " 29 GB" | 20260507 15:08:40.788 | INFO | } | 20260507 15:08:40.788 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:08:40.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:40.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.789 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:08:40.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.789 | INFO | PRE_HQ : 40.7%, 1.0, 21.1ms, 0 | 20260507 15:08:40.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:40.789 | INFO | Uploaded 36GiB to GPU in 1.6142s (22.3021GiB/s) | 20260507 15:08:40.789 | INFO | Downloaded 34GiB to GPU in 1.53131s (22.2031GiB/s) | 20260507 15:08:45.928 | INFO | Analyzing chunk frames = [457728,458240) | 20260507 15:08:45.928 | INFO | MemInfo: { | 20260507 15:08:45.928 | INFO | "gpu" : " 17 GB", | 20260507 15:08:45.928 | INFO | "huge" : "", | 20260507 15:08:45.928 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:45.928 | INFO | "malloc" : " 14 GB", | 20260507 15:08:45.928 | INFO | "pinned" : " 29 GB" | 20260507 15:08:45.928 | INFO | } | 20260507 15:08:45.928 | INFO | Perf report for the last 5.13963 seconds: | 20260507 15:08:45.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:45.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.928 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 15:08:45.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.929 | INFO | PRE_HQ : 40.9%, 1.0, 21.3ms, 0 | 20260507 15:08:45.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:45.929 | INFO | Uploaded 36GiB to GPU in 1.61303s (22.3183GiB/s) | 20260507 15:08:45.929 | INFO | Downloaded 34GiB to GPU in 1.53345s (22.1722GiB/s) | 20260507 15:08:51.066 | INFO | Analyzing chunk frames = [458240,458752) | 20260507 15:08:51.066 | INFO | MemInfo: { | 20260507 15:08:51.066 | INFO | "gpu" : " 17 GB", | 20260507 15:08:51.066 | INFO | "huge" : "", | 20260507 15:08:51.066 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:51.066 | INFO | "malloc" : " 14 GB", | 20260507 15:08:51.067 | INFO | "pinned" : " 29 GB" | 20260507 15:08:51.067 | INFO | } | 20260507 15:08:51.067 | INFO | Perf report for the last 5.1385 seconds: | 20260507 15:08:51.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:51.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:08:51.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | PRE_HQ : 40.3%, 1.0, 20.9ms, 0 | 20260507 15:08:51.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:51.067 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:08:51.067 | INFO | Total mem usage: 16.4 GiB | 20260507 15:08:51.067 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:08:51.067 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8.07%) | 20260507 15:08:51.067 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:08:51.067 | INFO | High Water Breakdown: | 20260507 15:08:51.067 | INFO | 59.3% : 9 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:08:51.067 | INFO | 40.6% : 6.16 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:08:51.067 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:08:51.067 | INFO | ------------------------------------------ | 20260507 15:08:51.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:08:51.067 | INFO | Uploaded 36GiB to GPU in 1.6137s (22.309GiB/s) | 20260507 15:08:51.067 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:08:51.067 | INFO | Total mem usage: 26.5 GiB | 20260507 15:08:51.067 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:08:51.067 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 15:08:51.067 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:08:51.067 | INFO | High Water Breakdown: | 20260507 15:08:51.067 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:08:51.067 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:08:51.067 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:08:51.067 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:08:51.067 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:08:51.067 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:08:51.067 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:08:51.067 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:08:51.067 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:08:51.067 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:08:51.067 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:08:51.067 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:08:51.067 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:08:51.067 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:08:51.068 | INFO | ------------------------------------------ | 20260507 15:08:51.068 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:08:51.068 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:08:51.068 | INFO | Total mem usage: 27.8 GiB | 20260507 15:08:51.068 | INFO | Downloaded 34GiB to GPU in 1.53251s (22.1859GiB/s) | 20260507 15:08:51.068 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:08:51.068 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:08:51.068 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:08:51.068 | INFO | High Water Breakdown: | 20260507 15:08:51.068 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:08:51.068 | INFO | ------------------------------------------ | 20260507 15:08:51.068 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:08:51.068 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:08:51.068 | INFO | Total mem usage: 15.7 GiB | 20260507 15:08:51.068 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:08:51.068 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:08:51.068 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:08:51.068 | INFO | High Water Breakdown: | 20260507 15:08:51.068 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:08:51.068 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:08:51.068 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:08:51.068 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:08:51.068 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:08:51.068 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:08:51.068 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:08:51.068 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:08:51.068 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:08:51.068 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:08:51.068 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:08:51.068 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:08:51.068 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:08:51.068 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:08:51.068 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:08:51.068 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:08:51.068 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:08:51.068 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:08:51.068 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:08:51.068 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:08:51.068 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:08:51.068 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:08:51.068 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:08:51.068 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:08:51.068 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:08:51.068 | INFO | ------------------------------------------ | 20260507 15:08:51.068 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:08:51.068 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:08:51.068 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:08:51.068 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:08:51.068 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 0.30%, AggregateStats -- 8.85e-02 / -nan, 5.57e-03 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 11.03%, Baseline -- 3.30e+00 / -nan, 2.23e-02 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.34e-02 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 16.11%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.72e+01 / -nan, 4.41e-02 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.16e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 3.52%, Metrics -- 1.05e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.92e-01 / -nan, 5.69e-02 / 0.00e+00 | 20260507 15:08:51.068 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:08:51.068 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:08:51.068 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:08:51.068 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 3.98e-01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 18.05%, Download -- 1.62e+01 / 0.00e+00, 7.35e-01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 33.41%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 0.57%, LockUpload -- 5.13e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 0.00%, LockDownload -- 5.35e-04 / 0.00e+00, 5.29e-04 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 29.21%, LockCompute -- 2.62e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 15:08:51.068 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 15:08:51.164 | INFO | Starting a round of Baz IO | 20260507 15:08:54.110 | INFO | Populating RTMetrics report | 20260507 15:08:54.110 | INFO | RTMetrics report populated | 20260507 15:08:54.110 | INFO | Opening tmp RTMetrics json file | 20260507 15:08:54.111 | INFO | Serializing RTMetrics to json | 20260507 15:08:54.119 | INFO | Serialization done, swapping tmp file now... | 20260507 15:08:54.135 | INFO | RTMetrics done | 20260507 15:08:54.201 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.037 seconds | 20260507 15:08:54.201 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.62% / 0.05% / 45.33% | 20260507 15:08:54.201 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.38456% / 9.53674e-05% / 3.57628e-05% | 20260507 15:08:54.201 | INFO | Baz IO overall: | 20260507 15:08:54.201 | INFO | BytesWritten : 180660744090 | 20260507 15:08:54.201 | INFO | NumEvents : 86745345865 | 20260507 15:08:56.206 | INFO | Analyzing chunk frames = [458752,459264) | 20260507 15:08:56.206 | INFO | MemInfo: { | 20260507 15:08:56.206 | INFO | "gpu" : " 17 GB", | 20260507 15:08:56.206 | INFO | "huge" : "", | 20260507 15:08:56.206 | INFO | "huge_shared" : " 29 GB", | 20260507 15:08:56.206 | INFO | "malloc" : " 14 GB", | 20260507 15:08:56.206 | INFO | "pinned" : " 29 GB" | 20260507 15:08:56.206 | INFO | } | 20260507 15:08:56.206 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:08:56.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:08:56.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:56.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:56.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:56.206 | INFO | ANALYSIS : 58.3%, 2.8, 84.9ms, 0 | 20260507 15:08:56.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:08:56.206 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 15:08:56.206 | INFO | PRE_HQ : 57.3%, 2.2, 64.8ms, 0 | 20260507 15:08:56.206 | INFO | BAZWRITER : 58.7%, 4.7, 140.9ms, 0 | 20260507 15:08:56.206 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 15:08:56.206 | INFO | Uploaded 36GiB to GPU in 1.64656s (21.8638GiB/s) | 20260507 15:08:56.207 | INFO | Downloaded 38GiB to GPU in 1.72878s (21.9808GiB/s) | 20260507 15:09:01.345 | INFO | Analyzing chunk frames = [459264,459776) | 20260507 15:09:01.345 | INFO | MemInfo: { | 20260507 15:09:01.345 | INFO | "gpu" : " 17 GB", | 20260507 15:09:01.345 | INFO | "huge" : "", | 20260507 15:09:01.345 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:01.345 | INFO | "malloc" : " 14 GB", | 20260507 15:09:01.346 | INFO | "pinned" : " 29 GB" | 20260507 15:09:01.346 | INFO | } | 20260507 15:09:01.346 | INFO | Perf report for the last 5.13988 seconds: | 20260507 15:09:01.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:01.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:09:01.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | PRE_HQ : 31.0%, 1.0, 16.6ms, 0 | 20260507 15:09:01.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:01.346 | INFO | Uploaded 36GiB to GPU in 1.60626s (22.4124GiB/s) | 20260507 15:09:01.346 | INFO | Downloaded 34GiB to GPU in 1.52618s (22.2779GiB/s) | 20260507 15:09:06.485 | INFO | Analyzing chunk frames = [459776,460288) | 20260507 15:09:06.485 | INFO | MemInfo: { | 20260507 15:09:06.485 | INFO | "gpu" : " 17 GB", | 20260507 15:09:06.485 | INFO | "huge" : "", | 20260507 15:09:06.485 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:06.485 | INFO | "malloc" : " 14 GB", | 20260507 15:09:06.485 | INFO | "pinned" : " 29 GB" | 20260507 15:09:06.485 | INFO | } | 20260507 15:09:06.485 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:09:06.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:06.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.485 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 15:09:06.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.485 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 15:09:06.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:06.486 | INFO | Uploaded 36GiB to GPU in 1.61409s (22.3036GiB/s) | 20260507 15:09:06.486 | INFO | Downloaded 34GiB to GPU in 1.53535s (22.1448GiB/s) | 20260507 15:09:11.624 | INFO | Analyzing chunk frames = [460288,460800) | 20260507 15:09:11.624 | INFO | MemInfo: { | 20260507 15:09:11.624 | INFO | "gpu" : " 17 GB", | 20260507 15:09:11.624 | INFO | "huge" : "", | 20260507 15:09:11.624 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:11.624 | INFO | "malloc" : " 14 GB", | 20260507 15:09:11.624 | INFO | "pinned" : " 29 GB" | 20260507 15:09:11.625 | INFO | } | 20260507 15:09:11.625 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:09:11.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:11.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 15:09:11.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | PRE_HQ : 40.0%, 1.0, 20.8ms, 0 | 20260507 15:09:11.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:11.625 | INFO | Uploaded 36GiB to GPU in 1.61413s (22.3031GiB/s) | 20260507 15:09:11.625 | INFO | Downloaded 34GiB to GPU in 1.53476s (22.1533GiB/s) | 20260507 15:09:16.764 | INFO | Analyzing chunk frames = [460800,461312) | 20260507 15:09:16.764 | INFO | MemInfo: { | 20260507 15:09:16.764 | INFO | "gpu" : " 17 GB", | 20260507 15:09:16.764 | INFO | "huge" : "", | 20260507 15:09:16.764 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:16.764 | INFO | "malloc" : " 14 GB", | 20260507 15:09:16.764 | INFO | "pinned" : " 29 GB" | 20260507 15:09:16.764 | INFO | } | 20260507 15:09:16.764 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:09:16.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:16.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.764 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:09:16.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.765 | INFO | PRE_HQ : 40.5%, 1.0, 21.0ms, 0 | 20260507 15:09:16.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:16.765 | INFO | Uploaded 36GiB to GPU in 1.61338s (22.3135GiB/s) | 20260507 15:09:16.765 | INFO | Downloaded 34GiB to GPU in 1.53226s (22.1895GiB/s) | 20260507 15:09:21.903 | INFO | Analyzing chunk frames = [461312,461824) | 20260507 15:09:21.903 | INFO | MemInfo: { | 20260507 15:09:21.903 | INFO | "gpu" : " 17 GB", | 20260507 15:09:21.903 | INFO | "huge" : "", | 20260507 15:09:21.903 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:21.903 | INFO | "malloc" : " 14 GB", | 20260507 15:09:21.903 | INFO | "pinned" : " 29 GB" | 20260507 15:09:21.903 | INFO | } | 20260507 15:09:21.903 | INFO | Perf report for the last 5.13888 seconds: | 20260507 15:09:21.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:21.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.903 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:09:21.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.904 | INFO | PRE_HQ : 41.0%, 1.0, 21.3ms, 0 | 20260507 15:09:21.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:21.904 | INFO | Uploaded 36GiB to GPU in 1.61387s (22.3066GiB/s) | 20260507 15:09:21.904 | INFO | Downloaded 34GiB to GPU in 1.53232s (22.1885GiB/s) | 20260507 15:09:27.042 | INFO | Analyzing chunk frames = [461824,462336) | 20260507 15:09:27.042 | INFO | MemInfo: { | 20260507 15:09:27.042 | INFO | "gpu" : " 17 GB", | 20260507 15:09:27.042 | INFO | "huge" : "", | 20260507 15:09:27.042 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:27.043 | INFO | "malloc" : " 14 GB", | 20260507 15:09:27.043 | INFO | "pinned" : " 29 GB" | 20260507 15:09:27.043 | INFO | } | 20260507 15:09:27.043 | INFO | Perf report for the last 5.13966 seconds: | 20260507 15:09:27.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:27.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.043 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 15:09:27.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.043 | INFO | PRE_HQ : 40.5%, 1.0, 21.1ms, 0 | 20260507 15:09:27.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:27.044 | INFO | Uploaded 36GiB to GPU in 1.61322s (22.3156GiB/s) | 20260507 15:09:27.044 | INFO | Downloaded 34GiB to GPU in 1.53228s (22.1891GiB/s) | 20260507 15:09:32.181 | INFO | Analyzing chunk frames = [462336,462848) | 20260507 15:09:32.181 | INFO | MemInfo: { | 20260507 15:09:32.181 | INFO | "gpu" : " 17 GB", | 20260507 15:09:32.182 | INFO | "huge" : "", | 20260507 15:09:32.182 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:32.182 | INFO | "malloc" : " 14 GB", | 20260507 15:09:32.182 | INFO | "pinned" : " 29 GB" | 20260507 15:09:32.182 | INFO | } | 20260507 15:09:32.182 | INFO | Perf report for the last 5.13911 seconds: | 20260507 15:09:32.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:32.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.182 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 15:09:32.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.182 | INFO | PRE_HQ : 40.5%, 1.0, 21.0ms, 0 | 20260507 15:09:32.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:32.183 | INFO | Uploaded 36GiB to GPU in 1.61281s (22.3213GiB/s) | 20260507 15:09:32.183 | INFO | Downloaded 34GiB to GPU in 1.53335s (22.1736GiB/s) | 20260507 15:09:32.294 | INFO | Starting a round of Baz IO | 20260507 15:09:35.187 | INFO | Populating RTMetrics report | 20260507 15:09:35.188 | INFO | RTMetrics report populated | 20260507 15:09:35.188 | INFO | Opening tmp RTMetrics json file | 20260507 15:09:35.222 | INFO | Serializing RTMetrics to json | 20260507 15:09:35.264 | INFO | Serialization done, swapping tmp file now... | 20260507 15:09:35.265 | INFO | RTMetrics done | 20260507 15:09:35.312 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.017 seconds | 20260507 15:09:35.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.57% / 0.05% / 45.38% | 20260507 15:09:35.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.36472% / 9.53674e-05% / 2.78155e-05% | 20260507 15:09:35.312 | INFO | Baz IO overall: | 20260507 15:09:35.312 | INFO | BytesWritten : 182522993811 | 20260507 15:09:35.312 | INFO | NumEvents : 87652460843 | 20260507 15:09:37.321 | INFO | Analyzing chunk frames = [462848,463360) | 20260507 15:09:37.321 | INFO | MemInfo: { | 20260507 15:09:37.321 | INFO | "gpu" : " 17 GB", | 20260507 15:09:37.321 | INFO | "huge" : "", | 20260507 15:09:37.321 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:37.321 | INFO | "malloc" : " 14 GB", | 20260507 15:09:37.321 | INFO | "pinned" : " 29 GB" | 20260507 15:09:37.321 | INFO | } | 20260507 15:09:37.321 | INFO | Perf report for the last 5.13902 seconds: | 20260507 15:09:37.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:37.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:37.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:37.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:37.321 | INFO | ANALYSIS : 57.6%, 2.8, 84.7ms, 0 | 20260507 15:09:37.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:37.321 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260507 15:09:37.321 | INFO | PRE_HQ : 56.8%, 2.2, 63.7ms, 0 | 20260507 15:09:37.321 | INFO | BAZWRITER : 58.2%, 4.8, 142.5ms, 0 | 20260507 15:09:37.321 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 15:09:37.321 | INFO | Uploaded 36GiB to GPU in 1.66051s (21.6801GiB/s) | 20260507 15:09:37.321 | INFO | Downloaded 38GiB to GPU in 1.74671s (21.7552GiB/s) | 20260507 15:09:42.460 | INFO | Analyzing chunk frames = [463360,463872) | 20260507 15:09:42.460 | INFO | MemInfo: { | 20260507 15:09:42.460 | INFO | "gpu" : " 17 GB", | 20260507 15:09:42.460 | INFO | "huge" : "", | 20260507 15:09:42.460 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:42.460 | INFO | "malloc" : " 14 GB", | 20260507 15:09:42.460 | INFO | "pinned" : " 29 GB" | 20260507 15:09:42.460 | INFO | } | 20260507 15:09:42.461 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:09:42.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:42.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:09:42.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:09:42.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:42.461 | INFO | Uploaded 36GiB to GPU in 1.58469s (22.7173GiB/s) | 20260507 15:09:42.461 | INFO | Downloaded 34GiB to GPU in 1.53355s (22.1708GiB/s) | 20260507 15:09:47.599 | INFO | Analyzing chunk frames = [463872,464384) | 20260507 15:09:47.599 | INFO | MemInfo: { | 20260507 15:09:47.599 | INFO | "gpu" : " 17 GB", | 20260507 15:09:47.599 | INFO | "huge" : "", | 20260507 15:09:47.599 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:47.599 | INFO | "malloc" : " 14 GB", | 20260507 15:09:47.600 | INFO | "pinned" : " 29 GB" | 20260507 15:09:47.600 | INFO | } | 20260507 15:09:47.600 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:09:47.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:47.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:09:47.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:09:47.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:47.600 | INFO | Uploaded 36GiB to GPU in 1.58807s (22.669GiB/s) | 20260507 15:09:47.600 | INFO | Downloaded 34GiB to GPU in 1.53773s (22.1105GiB/s) | 20260507 15:09:52.739 | INFO | Analyzing chunk frames = [464384,464896) | 20260507 15:09:52.739 | INFO | MemInfo: { | 20260507 15:09:52.739 | INFO | "gpu" : " 17 GB", | 20260507 15:09:52.739 | INFO | "huge" : "", | 20260507 15:09:52.739 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:52.739 | INFO | "malloc" : " 14 GB", | 20260507 15:09:52.739 | INFO | "pinned" : " 29 GB" | 20260507 15:09:52.739 | INFO | } | 20260507 15:09:52.739 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:09:52.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:52.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:09:52.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:09:52.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:52.739 | INFO | Uploaded 36GiB to GPU in 1.58892s (22.6569GiB/s) | 20260507 15:09:52.739 | INFO | Downloaded 34GiB to GPU in 1.53714s (22.119GiB/s) | 20260507 15:09:57.878 | INFO | Analyzing chunk frames = [464896,465408) | 20260507 15:09:57.878 | INFO | MemInfo: { | 20260507 15:09:57.878 | INFO | "gpu" : " 17 GB", | 20260507 15:09:57.878 | INFO | "huge" : "", | 20260507 15:09:57.878 | INFO | "huge_shared" : " 29 GB", | 20260507 15:09:57.878 | INFO | "malloc" : " 14 GB", | 20260507 15:09:57.878 | INFO | "pinned" : " 29 GB" | 20260507 15:09:57.878 | INFO | } | 20260507 15:09:57.878 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:09:57.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:09:57.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.878 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:09:57.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.879 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:09:57.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:09:57.879 | INFO | Uploaded 36GiB to GPU in 1.58725s (22.6808GiB/s) | 20260507 15:09:57.879 | INFO | Downloaded 34GiB to GPU in 1.53541s (22.144GiB/s) | 20260507 15:10:03.017 | INFO | Analyzing chunk frames = [465408,465920) | 20260507 15:10:03.017 | INFO | MemInfo: { | 20260507 15:10:03.017 | INFO | "gpu" : " 17 GB", | 20260507 15:10:03.017 | INFO | "huge" : "", | 20260507 15:10:03.017 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:03.017 | INFO | "malloc" : " 14 GB", | 20260507 15:10:03.017 | INFO | "pinned" : " 29 GB" | 20260507 15:10:03.017 | INFO | } | 20260507 15:10:03.018 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:10:03.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:03.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:10:03.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:10:03.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:03.018 | INFO | Uploaded 36GiB to GPU in 1.58849s (22.663GiB/s) | 20260507 15:10:03.018 | INFO | Downloaded 34GiB to GPU in 1.53652s (22.1279GiB/s) | 20260507 15:10:08.157 | INFO | Analyzing chunk frames = [465920,466432) | 20260507 15:10:08.157 | INFO | MemInfo: { | 20260507 15:10:08.157 | INFO | "gpu" : " 17 GB", | 20260507 15:10:08.157 | INFO | "huge" : "", | 20260507 15:10:08.157 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:08.157 | INFO | "malloc" : " 14 GB", | 20260507 15:10:08.157 | INFO | "pinned" : " 29 GB" | 20260507 15:10:08.157 | INFO | } | 20260507 15:10:08.157 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:10:08.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:08.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:10:08.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 15:10:08.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:08.157 | INFO | Uploaded 36GiB to GPU in 1.58793s (22.671GiB/s) | 20260507 15:10:08.157 | INFO | Downloaded 34GiB to GPU in 1.53668s (22.1256GiB/s) | 20260507 15:10:13.296 | INFO | Analyzing chunk frames = [466432,466944) | 20260507 15:10:13.296 | INFO | MemInfo: { | 20260507 15:10:13.296 | INFO | "gpu" : " 17 GB", | 20260507 15:10:13.296 | INFO | "huge" : "", | 20260507 15:10:13.296 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:13.296 | INFO | "malloc" : " 14 GB", | 20260507 15:10:13.296 | INFO | "pinned" : " 29 GB" | 20260507 15:10:13.296 | INFO | } | 20260507 15:10:13.296 | INFO | Perf report for the last 5.13911 seconds: | 20260507 15:10:13.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:13.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.296 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:10:13.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.296 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 15:10:13.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:13.297 | INFO | Uploaded 36GiB to GPU in 1.59457s (22.5766GiB/s) | 20260507 15:10:13.297 | INFO | Downloaded 34GiB to GPU in 1.54077s (22.067GiB/s) | 20260507 15:10:13.297 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:10:13.297 | INFO | Total mem usage: 16.4 GiB | 20260507 15:10:13.297 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:10:13.297 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.55%) | 20260507 15:10:13.297 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:10:13.297 | INFO | High Water Breakdown: | 20260507 15:10:13.297 | INFO | 59.2% : 8.95 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:10:13.297 | INFO | 40.7% : 6.15 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:10:13.297 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:10:13.297 | INFO | ------------------------------------------ | 20260507 15:10:13.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:10:13.297 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:10:13.297 | INFO | Total mem usage: 26.5 GiB | 20260507 15:10:13.297 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:10:13.297 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:10:13.297 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:10:13.297 | INFO | High Water Breakdown: | 20260507 15:10:13.297 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:10:13.297 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:10:13.297 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:10:13.297 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:10:13.297 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:10:13.297 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:10:13.297 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:10:13.297 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:10:13.297 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:10:13.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:10:13.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:10:13.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:10:13.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:10:13.298 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:10:13.298 | INFO | ------------------------------------------ | 20260507 15:10:13.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:10:13.298 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:10:13.298 | INFO | Total mem usage: 27.8 GiB | 20260507 15:10:13.298 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:10:13.298 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:10:13.298 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:10:13.298 | INFO | High Water Breakdown: | 20260507 15:10:13.298 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:10:13.298 | INFO | ------------------------------------------ | 20260507 15:10:13.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:10:13.298 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:10:13.298 | INFO | Total mem usage: 15.7 GiB | 20260507 15:10:13.298 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:10:13.298 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:10:13.298 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:10:13.298 | INFO | High Water Breakdown: | 20260507 15:10:13.298 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:10:13.298 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:10:13.298 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:10:13.298 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:10:13.298 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:10:13.298 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:10:13.298 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:10:13.298 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:10:13.298 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:10:13.298 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:10:13.298 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:10:13.298 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:10:13.298 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:10:13.298 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:10:13.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:10:13.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:10:13.298 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:10:13.298 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:10:13.298 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:10:13.298 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:10:13.298 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:10:13.298 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:10:13.298 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:10:13.298 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:10:13.298 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:10:13.298 | INFO | ------------------------------------------ | 20260507 15:10:13.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:10:13.298 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:10:13.298 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:10:13.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:10:13.298 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 0.30%, AggregateStats -- 8.96e-02 / -nan, 7.25e-03 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 11.04%, Baseline -- 3.30e+00 / -nan, 2.38e-02 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.34e-02 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 16.11%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.72e+01 / -nan, 4.45e-02 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.16e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 3.52%, Metrics -- 1.05e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:10:13.298 | INFO | PerfReport: 1.31%, PulseSerializing -- 3.92e-01 / -nan, 5.70e-02 / 0.00e+00 | 20260507 15:10:13.298 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:10:13.298 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:10:13.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:10:13.298 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 18.73%, Upload -- 1.67e+01 / 0.00e+00, 5.02e-01 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 18.18%, Download -- 1.63e+01 / 0.00e+00, 8.11e-01 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 33.50%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 0.00%, LockDownload -- 5.47e-04 / 0.00e+00, 3.75e-04 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 29.01%, LockCompute -- 2.59e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:10:13.299 | INFO | PerfReport: 0.02%, Misc -- 1.69e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 15:10:13.396 | INFO | Starting a round of Baz IO | 20260507 15:10:16.289 | INFO | Populating RTMetrics report | 20260507 15:10:16.289 | INFO | RTMetrics report populated | 20260507 15:10:16.289 | INFO | Opening tmp RTMetrics json file | 20260507 15:10:16.311 | INFO | Serializing RTMetrics to json | 20260507 15:10:16.315 | INFO | Serialization done, swapping tmp file now... | 20260507 15:10:16.320 | INFO | RTMetrics done | 20260507 15:10:16.612 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.216 seconds | 20260507 15:10:16.612 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.51% / 0.05% / 45.43% | 20260507 15:10:16.612 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.33616% / 0.000166893% / 2.38419e-05% | 20260507 15:10:16.613 | INFO | Baz IO overall: | 20260507 15:10:16.613 | INFO | BytesWritten : 184385078734 | 20260507 15:10:16.613 | INFO | NumEvents : 88559360420 | 20260507 15:10:18.435 | INFO | Analyzing chunk frames = [466944,467456) | 20260507 15:10:18.435 | INFO | MemInfo: { | 20260507 15:10:18.435 | INFO | "gpu" : " 17 GB", | 20260507 15:10:18.436 | INFO | "huge" : "", | 20260507 15:10:18.436 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:18.436 | INFO | "malloc" : " 14 GB", | 20260507 15:10:18.436 | INFO | "pinned" : " 29 GB" | 20260507 15:10:18.436 | INFO | } | 20260507 15:10:18.436 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:10:18.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:18.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:18.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:18.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:18.436 | INFO | ANALYSIS : 57.3%, 2.8, 84.8ms, 0 | 20260507 15:10:18.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:18.436 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 15:10:18.436 | INFO | PRE_HQ : 56.2%, 2.1, 61.1ms, 0 | 20260507 15:10:18.436 | INFO | BAZWRITER : 62.1%, 4.7, 151.4ms, 0 | 20260507 15:10:18.436 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:10:18.436 | INFO | Uploaded 36GiB to GPU in 1.65236s (21.7871GiB/s) | 20260507 15:10:18.436 | INFO | Downloaded 38GiB to GPU in 1.7355s (21.8957GiB/s) | 20260507 15:10:23.575 | INFO | Analyzing chunk frames = [467456,467968) | 20260507 15:10:23.575 | INFO | MemInfo: { | 20260507 15:10:23.575 | INFO | "gpu" : " 17 GB", | 20260507 15:10:23.575 | INFO | "huge" : "", | 20260507 15:10:23.575 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:23.575 | INFO | "malloc" : " 14 GB", | 20260507 15:10:23.575 | INFO | "pinned" : " 29 GB" | 20260507 15:10:23.575 | INFO | } | 20260507 15:10:23.575 | INFO | Perf report for the last 5.13908 seconds: | 20260507 15:10:23.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:23.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | ANALYSIS : 57.2%, 3.0, 89.1ms, 0 | 20260507 15:10:23.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | PRE_HQ : 39.9%, 1.0, 21.0ms, 0 | 20260507 15:10:23.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:23.575 | INFO | Uploaded 36GiB to GPU in 1.59141s (22.6214GiB/s) | 20260507 15:10:23.576 | INFO | Downloaded 34GiB to GPU in 1.53901s (22.0922GiB/s) | 20260507 15:10:28.714 | INFO | Analyzing chunk frames = [467968,468480) | 20260507 15:10:28.714 | INFO | MemInfo: { | 20260507 15:10:28.714 | INFO | "gpu" : " 17 GB", | 20260507 15:10:28.714 | INFO | "huge" : "", | 20260507 15:10:28.714 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:28.714 | INFO | "malloc" : " 14 GB", | 20260507 15:10:28.714 | INFO | "pinned" : " 29 GB" | 20260507 15:10:28.714 | INFO | } | 20260507 15:10:28.714 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:10:28.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:28.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:10:28.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:10:28.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:28.714 | INFO | Uploaded 36GiB to GPU in 1.58709s (22.683GiB/s) | 20260507 15:10:28.714 | INFO | Downloaded 34GiB to GPU in 1.53784s (22.1089GiB/s) | 20260507 15:10:33.853 | INFO | Analyzing chunk frames = [468480,468992) | 20260507 15:10:33.853 | INFO | MemInfo: { | 20260507 15:10:33.853 | INFO | "gpu" : " 17 GB", | 20260507 15:10:33.853 | INFO | "huge" : "", | 20260507 15:10:33.853 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:33.853 | INFO | "malloc" : " 14 GB", | 20260507 15:10:33.853 | INFO | "pinned" : " 29 GB" | 20260507 15:10:33.853 | INFO | } | 20260507 15:10:33.854 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:10:33.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:33.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:10:33.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 15:10:33.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:33.854 | INFO | Uploaded 36GiB to GPU in 1.58942s (22.6498GiB/s) | 20260507 15:10:33.854 | INFO | Downloaded 34GiB to GPU in 1.53746s (22.1144GiB/s) | 20260507 15:10:38.993 | INFO | Analyzing chunk frames = [468992,469504) | 20260507 15:10:38.993 | INFO | MemInfo: { | 20260507 15:10:38.993 | INFO | "gpu" : " 17 GB", | 20260507 15:10:38.993 | INFO | "huge" : "", | 20260507 15:10:38.993 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:38.993 | INFO | "malloc" : " 14 GB", | 20260507 15:10:38.993 | INFO | "pinned" : " 29 GB" | 20260507 15:10:38.993 | INFO | } | 20260507 15:10:38.993 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:10:38.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:38.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:10:38.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:10:38.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:38.993 | INFO | Uploaded 36GiB to GPU in 1.58716s (22.682GiB/s) | 20260507 15:10:38.993 | INFO | Downloaded 34GiB to GPU in 1.53629s (22.1312GiB/s) | 20260507 15:10:44.132 | INFO | Analyzing chunk frames = [469504,470016) | 20260507 15:10:44.132 | INFO | MemInfo: { | 20260507 15:10:44.132 | INFO | "gpu" : " 17 GB", | 20260507 15:10:44.132 | INFO | "huge" : "", | 20260507 15:10:44.132 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:44.132 | INFO | "malloc" : " 14 GB", | 20260507 15:10:44.132 | INFO | "pinned" : " 29 GB" | 20260507 15:10:44.132 | INFO | } | 20260507 15:10:44.132 | INFO | Perf report for the last 5.13967 seconds: | 20260507 15:10:44.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:44.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.132 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:10:44.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.133 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:10:44.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:44.133 | INFO | Uploaded 36GiB to GPU in 1.5876s (22.6757GiB/s) | 20260507 15:10:44.133 | INFO | Downloaded 34GiB to GPU in 1.53541s (22.144GiB/s) | 20260507 15:10:49.271 | INFO | Analyzing chunk frames = [470016,470528) | 20260507 15:10:49.271 | INFO | MemInfo: { | 20260507 15:10:49.271 | INFO | "gpu" : " 17 GB", | 20260507 15:10:49.271 | INFO | "huge" : "", | 20260507 15:10:49.271 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:49.271 | INFO | "malloc" : " 14 GB", | 20260507 15:10:49.271 | INFO | "pinned" : " 29 GB" | 20260507 15:10:49.271 | INFO | } | 20260507 15:10:49.271 | INFO | Perf report for the last 5.13902 seconds: | 20260507 15:10:49.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:49.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.272 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:10:49.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.272 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:10:49.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:49.272 | INFO | Uploaded 36GiB to GPU in 1.5875s (22.6772GiB/s) | 20260507 15:10:49.272 | INFO | Downloaded 34GiB to GPU in 1.53697s (22.1214GiB/s) | 20260507 15:10:54.411 | INFO | Analyzing chunk frames = [470528,471040) | 20260507 15:10:54.411 | INFO | MemInfo: { | 20260507 15:10:54.411 | INFO | "gpu" : " 17 GB", | 20260507 15:10:54.411 | INFO | "huge" : "", | 20260507 15:10:54.411 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:54.411 | INFO | "malloc" : " 14 GB", | 20260507 15:10:54.411 | INFO | "pinned" : " 29 GB" | 20260507 15:10:54.411 | INFO | } | 20260507 15:10:54.411 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:10:54.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:54.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.411 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:10:54.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.411 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:10:54.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:54.412 | INFO | Uploaded 36GiB to GPU in 1.58724s (22.6808GiB/s) | 20260507 15:10:54.412 | INFO | Downloaded 34GiB to GPU in 1.53693s (22.1221GiB/s) | 20260507 15:10:54.508 | INFO | Starting a round of Baz IO | 20260507 15:10:57.393 | INFO | Populating RTMetrics report | 20260507 15:10:57.393 | INFO | RTMetrics report populated | 20260507 15:10:57.393 | INFO | Opening tmp RTMetrics json file | 20260507 15:10:57.429 | INFO | Serializing RTMetrics to json | 20260507 15:10:57.487 | INFO | Serialization done, swapping tmp file now... | 20260507 15:10:57.496 | INFO | RTMetrics done | 20260507 15:10:57.518 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.009 seconds | 20260507 15:10:57.518 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.46% / 0.05% / 45.49% | 20260507 15:10:57.518 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.32231% / 0.000115236% / 2.38419e-05% | 20260507 15:10:57.518 | INFO | Baz IO overall: | 20260507 15:10:57.518 | INFO | BytesWritten : 186248222712 | 20260507 15:10:57.518 | INFO | NumEvents : 89467102641 | 20260507 15:10:59.550 | INFO | Analyzing chunk frames = [471040,471552) | 20260507 15:10:59.550 | INFO | MemInfo: { | 20260507 15:10:59.550 | INFO | "gpu" : " 17 GB", | 20260507 15:10:59.550 | INFO | "huge" : "", | 20260507 15:10:59.550 | INFO | "huge_shared" : " 29 GB", | 20260507 15:10:59.550 | INFO | "malloc" : " 14 GB", | 20260507 15:10:59.550 | INFO | "pinned" : " 29 GB" | 20260507 15:10:59.550 | INFO | } | 20260507 15:10:59.550 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:10:59.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:10:59.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:59.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:59.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:59.550 | INFO | ANALYSIS : 57.1%, 2.9, 86.7ms, 0 | 20260507 15:10:59.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:10:59.550 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260507 15:10:59.550 | INFO | PRE_HQ : 56.0%, 2.1, 60.8ms, 0 | 20260507 15:10:59.550 | INFO | BAZWRITER : 58.1%, 4.5, 133.0ms, 0 | 20260507 15:10:59.550 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260507 15:10:59.551 | INFO | Uploaded 36GiB to GPU in 1.65068s (21.8092GiB/s) | 20260507 15:10:59.551 | INFO | Downloaded 38GiB to GPU in 1.73218s (21.9377GiB/s) | 20260507 15:11:04.689 | INFO | Analyzing chunk frames = [471552,472064) | 20260507 15:11:04.689 | INFO | MemInfo: { | 20260507 15:11:04.689 | INFO | "gpu" : " 17 GB", | 20260507 15:11:04.689 | INFO | "huge" : "", | 20260507 15:11:04.690 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:04.690 | INFO | "malloc" : " 14 GB", | 20260507 15:11:04.690 | INFO | "pinned" : " 29 GB" | 20260507 15:11:04.690 | INFO | } | 20260507 15:11:04.690 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:11:04.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:04.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:11:04.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:11:04.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:04.690 | INFO | Uploaded 36GiB to GPU in 1.60432s (22.4394GiB/s) | 20260507 15:11:04.690 | INFO | Downloaded 34GiB to GPU in 1.52417s (22.3072GiB/s) | 20260507 15:11:09.829 | INFO | Analyzing chunk frames = [472064,472576) | 20260507 15:11:09.829 | INFO | MemInfo: { | 20260507 15:11:09.829 | INFO | "gpu" : " 17 GB", | 20260507 15:11:09.829 | INFO | "huge" : "", | 20260507 15:11:09.829 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:09.829 | INFO | "malloc" : " 14 GB", | 20260507 15:11:09.829 | INFO | "pinned" : " 29 GB" | 20260507 15:11:09.829 | INFO | } | 20260507 15:11:09.829 | INFO | Perf report for the last 5.13911 seconds: | 20260507 15:11:09.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:09.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:11:09.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:11:09.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:09.829 | INFO | Uploaded 36GiB to GPU in 1.60875s (22.3776GiB/s) | 20260507 15:11:09.829 | INFO | Downloaded 34GiB to GPU in 1.5286s (22.2426GiB/s) | 20260507 15:11:14.968 | INFO | Analyzing chunk frames = [472576,473088) | 20260507 15:11:14.968 | INFO | MemInfo: { | 20260507 15:11:14.968 | INFO | "gpu" : " 17 GB", | 20260507 15:11:14.968 | INFO | "huge" : "", | 20260507 15:11:14.968 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:14.968 | INFO | "malloc" : " 14 GB", | 20260507 15:11:14.968 | INFO | "pinned" : " 29 GB" | 20260507 15:11:14.968 | INFO | } | 20260507 15:11:14.968 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:11:14.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:14.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.968 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:11:14.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.968 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:11:14.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:14.969 | INFO | Uploaded 36GiB to GPU in 1.60875s (22.3777GiB/s) | 20260507 15:11:14.969 | INFO | Downloaded 34GiB to GPU in 1.52828s (22.2472GiB/s) | 20260507 15:11:20.108 | INFO | Analyzing chunk frames = [473088,473600) | 20260507 15:11:20.108 | INFO | MemInfo: { | 20260507 15:11:20.108 | INFO | "gpu" : " 17 GB", | 20260507 15:11:20.108 | INFO | "huge" : "", | 20260507 15:11:20.108 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:20.108 | INFO | "malloc" : " 14 GB", | 20260507 15:11:20.108 | INFO | "pinned" : " 29 GB" | 20260507 15:11:20.108 | INFO | } | 20260507 15:11:20.108 | INFO | Perf report for the last 5.1398 seconds: | 20260507 15:11:20.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:20.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:11:20.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:11:20.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:20.108 | INFO | Uploaded 36GiB to GPU in 1.60755s (22.3943GiB/s) | 20260507 15:11:20.108 | INFO | Downloaded 34GiB to GPU in 1.52655s (22.2724GiB/s) | 20260507 15:11:25.247 | INFO | Analyzing chunk frames = [473600,474112) | 20260507 15:11:25.247 | INFO | MemInfo: { | 20260507 15:11:25.247 | INFO | "gpu" : " 17 GB", | 20260507 15:11:25.247 | INFO | "huge" : "", | 20260507 15:11:25.247 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:25.247 | INFO | "malloc" : " 14 GB", | 20260507 15:11:25.247 | INFO | "pinned" : " 29 GB" | 20260507 15:11:25.247 | INFO | } | 20260507 15:11:25.247 | INFO | Perf report for the last 5.13882 seconds: | 20260507 15:11:25.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:25.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:11:25.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:11:25.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:25.247 | INFO | Uploaded 36GiB to GPU in 1.60713s (22.4001GiB/s) | 20260507 15:11:25.247 | INFO | Downloaded 34GiB to GPU in 1.52567s (22.2853GiB/s) | 20260507 15:11:30.404 | INFO | Analyzing chunk frames = [474112,474624) | 20260507 15:11:30.404 | INFO | MemInfo: { | 20260507 15:11:30.404 | INFO | "gpu" : " 17 GB", | 20260507 15:11:30.404 | INFO | "huge" : "", | 20260507 15:11:30.404 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:30.404 | INFO | "malloc" : " 14 GB", | 20260507 15:11:30.404 | INFO | "pinned" : " 29 GB" | 20260507 15:11:30.404 | INFO | } | 20260507 15:11:30.404 | INFO | Perf report for the last 5.15731 seconds: | 20260507 15:11:30.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:30.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.404 | INFO | ANALYSIS : 56.3%, 3.0, 88.1ms, 0 | 20260507 15:11:30.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.404 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:11:30.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:30.405 | INFO | Uploaded 36GiB to GPU in 1.60803s (22.3877GiB/s) | 20260507 15:11:30.405 | INFO | Downloaded 34GiB to GPU in 1.52749s (22.2587GiB/s) | 20260507 15:11:35.526 | INFO | Analyzing chunk frames = [474624,475136) | 20260507 15:11:35.526 | INFO | MemInfo: { | 20260507 15:11:35.526 | INFO | "gpu" : " 17 GB", | 20260507 15:11:35.526 | INFO | "huge" : "", | 20260507 15:11:35.526 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:35.526 | INFO | "malloc" : " 14 GB", | 20260507 15:11:35.526 | INFO | "pinned" : " 29 GB" | 20260507 15:11:35.526 | INFO | } | 20260507 15:11:35.526 | INFO | Perf report for the last 5.12186 seconds: | 20260507 15:11:35.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:35.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.526 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260507 15:11:35.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.526 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 15:11:35.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:35.527 | INFO | Uploaded 36GiB to GPU in 1.60725s (22.3984GiB/s) | 20260507 15:11:35.527 | INFO | Downloaded 34GiB to GPU in 1.52847s (22.2444GiB/s) | 20260507 15:11:35.527 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:11:35.527 | INFO | Total mem usage: 16.4 GiB | 20260507 15:11:35.527 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:11:35.527 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -6.68%) | 20260507 15:11:35.527 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:11:35.527 | INFO | High Water Breakdown: | 20260507 15:11:35.527 | INFO | 59.6% : 9.17 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:11:35.527 | INFO | 40.2% : 6.19 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:11:35.527 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:11:35.527 | INFO | ------------------------------------------ | 20260507 15:11:35.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:11:35.527 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:11:35.527 | INFO | Total mem usage: 26.5 GiB | 20260507 15:11:35.527 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:11:35.527 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:11:35.527 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:11:35.527 | INFO | High Water Breakdown: | 20260507 15:11:35.527 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:11:35.527 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:11:35.527 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:11:35.527 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:11:35.527 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:11:35.527 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:11:35.527 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:11:35.527 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:11:35.527 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:11:35.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:11:35.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:11:35.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:11:35.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:11:35.527 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:11:35.527 | INFO | ------------------------------------------ | 20260507 15:11:35.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:11:35.527 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:11:35.527 | INFO | Total mem usage: 27.8 GiB | 20260507 15:11:35.527 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:11:35.527 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:11:35.527 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:11:35.527 | INFO | High Water Breakdown: | 20260507 15:11:35.527 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:11:35.527 | INFO | ------------------------------------------ | 20260507 15:11:35.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:11:35.528 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:11:35.528 | INFO | Total mem usage: 15.7 GiB | 20260507 15:11:35.528 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:11:35.528 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:11:35.528 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:11:35.528 | INFO | High Water Breakdown: | 20260507 15:11:35.528 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:11:35.528 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:11:35.528 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:11:35.528 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:11:35.528 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:11:35.528 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:11:35.528 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:11:35.528 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:11:35.528 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:11:35.528 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:11:35.528 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:11:35.528 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:11:35.528 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:11:35.528 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:11:35.528 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:11:35.528 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:11:35.528 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:11:35.528 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:11:35.528 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:11:35.528 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:11:35.528 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:11:35.528 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:11:35.528 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:11:35.528 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:11:35.528 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:11:35.528 | INFO | ------------------------------------------ | 20260507 15:11:35.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:11:35.528 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:11:35.528 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:11:35.528 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:11:35.528 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 5.45e-03 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 11.03%, Baseline -- 3.30e+00 / -nan, 2.15e-02 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.14e-02 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.72e+01 / -nan, 4.40e-02 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.16e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 3.51%, Metrics -- 1.05e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 1.30%, PulseSerializing -- 3.88e-01 / -nan, 5.66e-02 / 0.00e+00 | 20260507 15:11:35.528 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:11:35.528 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:11:35.528 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:11:35.528 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 18.70%, Upload -- 1.67e+01 / 0.00e+00, 4.68e-01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 18.15%, Download -- 1.62e+01 / 0.00e+00, 7.93e-01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 33.45%, Compute -- 2.99e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 0.00%, LockDownload -- 4.55e-04 / 0.00e+00, 1.83e-04 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 29.13%, LockCompute -- 2.60e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:11:35.528 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260507 15:11:35.624 | INFO | Starting a round of Baz IO | 20260507 15:11:38.533 | INFO | Populating RTMetrics report | 20260507 15:11:38.533 | INFO | RTMetrics report populated | 20260507 15:11:38.533 | INFO | Opening tmp RTMetrics json file | 20260507 15:11:38.584 | INFO | Serializing RTMetrics to json | 20260507 15:11:38.590 | INFO | Serialization done, swapping tmp file now... | 20260507 15:11:38.595 | INFO | RTMetrics done | 20260507 15:11:38.666 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.042 seconds | 20260507 15:11:38.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.41% / 0.05% / 45.54% | 20260507 15:11:38.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.30583% / 8.34465e-05% / 2.38419e-05% | 20260507 15:11:38.666 | INFO | Baz IO overall: | 20260507 15:11:38.666 | INFO | BytesWritten : 188112058678 | 20260507 15:11:38.666 | INFO | NumEvents : 90375174543 | 20260507 15:11:40.665 | INFO | Analyzing chunk frames = [475136,475648) | 20260507 15:11:40.665 | INFO | MemInfo: { | 20260507 15:11:40.665 | INFO | "gpu" : " 17 GB", | 20260507 15:11:40.665 | INFO | "huge" : "", | 20260507 15:11:40.665 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:40.665 | INFO | "malloc" : " 14 GB", | 20260507 15:11:40.665 | INFO | "pinned" : " 29 GB" | 20260507 15:11:40.665 | INFO | } | 20260507 15:11:40.665 | INFO | Perf report for the last 5.13868 seconds: | 20260507 15:11:40.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:40.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:40.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:40.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:40.665 | INFO | ANALYSIS : 57.6%, 2.9, 86.3ms, 0 | 20260507 15:11:40.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:40.665 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 15:11:40.665 | INFO | PRE_HQ : 56.7%, 2.1, 63.3ms, 0 | 20260507 15:11:40.665 | INFO | BAZWRITER : 58.7%, 4.5, 135.5ms, 0 | 20260507 15:11:40.665 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:11:40.665 | INFO | Uploaded 36GiB to GPU in 1.63587s (22.0067GiB/s) | 20260507 15:11:40.665 | INFO | Downloaded 38GiB to GPU in 1.71634s (22.1401GiB/s) | 20260507 15:11:45.804 | INFO | Analyzing chunk frames = [475648,476160) | 20260507 15:11:45.804 | INFO | MemInfo: { | 20260507 15:11:45.804 | INFO | "gpu" : " 17 GB", | 20260507 15:11:45.804 | INFO | "huge" : "", | 20260507 15:11:45.804 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:45.804 | INFO | "malloc" : " 14 GB", | 20260507 15:11:45.804 | INFO | "pinned" : " 29 GB" | 20260507 15:11:45.804 | INFO | } | 20260507 15:11:45.804 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:11:45.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:45.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.804 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:11:45.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.804 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:11:45.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:45.805 | INFO | Uploaded 36GiB to GPU in 1.58413s (22.7253GiB/s) | 20260507 15:11:45.805 | INFO | Downloaded 34GiB to GPU in 1.53273s (22.1826GiB/s) | 20260507 15:11:50.943 | INFO | Analyzing chunk frames = [476160,476672) | 20260507 15:11:50.943 | INFO | MemInfo: { | 20260507 15:11:50.943 | INFO | "gpu" : " 17 GB", | 20260507 15:11:50.943 | INFO | "huge" : "", | 20260507 15:11:50.943 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:50.943 | INFO | "malloc" : " 14 GB", | 20260507 15:11:50.943 | INFO | "pinned" : " 29 GB" | 20260507 15:11:50.943 | INFO | } | 20260507 15:11:50.943 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:11:50.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:50.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:11:50.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:11:50.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:50.944 | INFO | Uploaded 36GiB to GPU in 1.58832s (22.6654GiB/s) | 20260507 15:11:50.944 | INFO | Downloaded 34GiB to GPU in 1.53836s (22.1015GiB/s) | 20260507 15:11:56.083 | INFO | Analyzing chunk frames = [476672,477184) | 20260507 15:11:56.083 | INFO | MemInfo: { | 20260507 15:11:56.083 | INFO | "gpu" : " 17 GB", | 20260507 15:11:56.083 | INFO | "huge" : "", | 20260507 15:11:56.083 | INFO | "huge_shared" : " 29 GB", | 20260507 15:11:56.083 | INFO | "malloc" : " 14 GB", | 20260507 15:11:56.083 | INFO | "pinned" : " 29 GB" | 20260507 15:11:56.083 | INFO | } | 20260507 15:11:56.083 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:11:56.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:11:56.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.083 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:11:56.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.083 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:11:56.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:11:56.084 | INFO | Uploaded 36GiB to GPU in 1.58857s (22.6619GiB/s) | 20260507 15:11:56.084 | INFO | Downloaded 34GiB to GPU in 1.53688s (22.1228GiB/s) | 20260507 15:12:01.222 | INFO | Analyzing chunk frames = [477184,477696) | 20260507 15:12:01.222 | INFO | MemInfo: { | 20260507 15:12:01.222 | INFO | "gpu" : " 17 GB", | 20260507 15:12:01.222 | INFO | "huge" : "", | 20260507 15:12:01.222 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:01.222 | INFO | "malloc" : " 14 GB", | 20260507 15:12:01.222 | INFO | "pinned" : " 29 GB" | 20260507 15:12:01.222 | INFO | } | 20260507 15:12:01.223 | INFO | Perf report for the last 5.13973 seconds: | 20260507 15:12:01.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:01.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:12:01.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:12:01.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:01.223 | INFO | Uploaded 36GiB to GPU in 1.58752s (22.6769GiB/s) | 20260507 15:12:01.223 | INFO | Downloaded 34GiB to GPU in 1.53682s (22.1236GiB/s) | 20260507 15:12:06.361 | INFO | Analyzing chunk frames = [477696,478208) | 20260507 15:12:06.361 | INFO | MemInfo: { | 20260507 15:12:06.361 | INFO | "gpu" : " 17 GB", | 20260507 15:12:06.361 | INFO | "huge" : "", | 20260507 15:12:06.361 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:06.361 | INFO | "malloc" : " 14 GB", | 20260507 15:12:06.361 | INFO | "pinned" : " 29 GB" | 20260507 15:12:06.362 | INFO | } | 20260507 15:12:06.362 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:12:06.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:06.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:12:06.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:12:06.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:06.362 | INFO | Uploaded 36GiB to GPU in 1.58693s (22.6853GiB/s) | 20260507 15:12:06.362 | INFO | Downloaded 34GiB to GPU in 1.53778s (22.1098GiB/s) | 20260507 15:12:11.501 | INFO | Analyzing chunk frames = [478208,478720) | 20260507 15:12:11.501 | INFO | MemInfo: { | 20260507 15:12:11.501 | INFO | "gpu" : " 17 GB", | 20260507 15:12:11.501 | INFO | "huge" : "", | 20260507 15:12:11.501 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:11.501 | INFO | "malloc" : " 14 GB", | 20260507 15:12:11.501 | INFO | "pinned" : " 29 GB" | 20260507 15:12:11.501 | INFO | } | 20260507 15:12:11.501 | INFO | Perf report for the last 5.13974 seconds: | 20260507 15:12:11.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:11.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:12:11.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:12:11.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:11.502 | INFO | Uploaded 36GiB to GPU in 1.58691s (22.6856GiB/s) | 20260507 15:12:11.502 | INFO | Downloaded 34GiB to GPU in 1.53649s (22.1283GiB/s) | 20260507 15:12:16.640 | INFO | Analyzing chunk frames = [478720,479232) | 20260507 15:12:16.640 | INFO | MemInfo: { | 20260507 15:12:16.640 | INFO | "gpu" : " 17 GB", | 20260507 15:12:16.640 | INFO | "huge" : "", | 20260507 15:12:16.640 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:16.640 | INFO | "malloc" : " 14 GB", | 20260507 15:12:16.640 | INFO | "pinned" : " 29 GB" | 20260507 15:12:16.640 | INFO | } | 20260507 15:12:16.640 | INFO | Perf report for the last 5.13905 seconds: | 20260507 15:12:16.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:16.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:12:16.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 15:12:16.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:16.641 | INFO | Uploaded 36GiB to GPU in 1.58843s (22.6639GiB/s) | 20260507 15:12:16.641 | INFO | Downloaded 34GiB to GPU in 1.53861s (22.0979GiB/s) | 20260507 15:12:16.737 | INFO | Starting a round of Baz IO | 20260507 15:12:19.629 | INFO | Populating RTMetrics report | 20260507 15:12:19.629 | INFO | RTMetrics report populated | 20260507 15:12:19.629 | INFO | Opening tmp RTMetrics json file | 20260507 15:12:19.630 | INFO | Serializing RTMetrics to json | 20260507 15:12:19.634 | INFO | Serialization done, swapping tmp file now... | 20260507 15:12:19.634 | INFO | RTMetrics done | 20260507 15:12:19.737 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 2.999 seconds | 20260507 15:12:19.737 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.36% / 0.05% / 45.59% | 20260507 15:12:19.737 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.27776% / 0.000139078% / 2.78155e-05% | 20260507 15:12:19.737 | INFO | Baz IO overall: | 20260507 15:12:19.737 | INFO | BytesWritten : 189976563516 | 20260507 15:12:19.737 | INFO | NumEvents : 91283430364 | 20260507 15:12:21.780 | INFO | Analyzing chunk frames = [479232,479744) | 20260507 15:12:21.780 | INFO | MemInfo: { | 20260507 15:12:21.780 | INFO | "gpu" : " 17 GB", | 20260507 15:12:21.780 | INFO | "huge" : "", | 20260507 15:12:21.780 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:21.780 | INFO | "malloc" : " 14 GB", | 20260507 15:12:21.780 | INFO | "pinned" : " 29 GB" | 20260507 15:12:21.780 | INFO | } | 20260507 15:12:21.780 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:12:21.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:21.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:21.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:21.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:21.780 | INFO | ANALYSIS : 57.3%, 2.9, 86.9ms, 0 | 20260507 15:12:21.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:21.780 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 15:12:21.780 | INFO | PRE_HQ : 56.2%, 2.1, 60.7ms, 0 | 20260507 15:12:21.780 | INFO | BAZWRITER : 57.9%, 4.5, 132.5ms, 0 | 20260507 15:12:21.780 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 15:12:21.781 | INFO | Uploaded 36GiB to GPU in 1.64341s (21.9057GiB/s) | 20260507 15:12:21.781 | INFO | Downloaded 38GiB to GPU in 1.73s (21.9653GiB/s) | 20260507 15:12:26.919 | INFO | Analyzing chunk frames = [479744,480256) | 20260507 15:12:26.919 | INFO | MemInfo: { | 20260507 15:12:26.919 | INFO | "gpu" : " 17 GB", | 20260507 15:12:26.919 | INFO | "huge" : "", | 20260507 15:12:26.919 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:26.919 | INFO | "malloc" : " 14 GB", | 20260507 15:12:26.919 | INFO | "pinned" : " 29 GB" | 20260507 15:12:26.919 | INFO | } | 20260507 15:12:26.919 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:12:26.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:26.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.919 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:12:26.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.919 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260507 15:12:26.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:26.920 | INFO | Uploaded 36GiB to GPU in 1.60693s (22.4029GiB/s) | 20260507 15:12:26.920 | INFO | Downloaded 34GiB to GPU in 1.52613s (22.2786GiB/s) | 20260507 15:12:32.058 | INFO | Analyzing chunk frames = [480256,480768) | 20260507 15:12:32.058 | INFO | MemInfo: { | 20260507 15:12:32.058 | INFO | "gpu" : " 17 GB", | 20260507 15:12:32.058 | INFO | "huge" : "", | 20260507 15:12:32.058 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:32.058 | INFO | "malloc" : " 14 GB", | 20260507 15:12:32.058 | INFO | "pinned" : " 29 GB" | 20260507 15:12:32.058 | INFO | } | 20260507 15:12:32.058 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:12:32.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:32.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.058 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:12:32.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.058 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 15:12:32.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:32.059 | INFO | Uploaded 36GiB to GPU in 1.60908s (22.373GiB/s) | 20260507 15:12:32.059 | INFO | Downloaded 34GiB to GPU in 1.52937s (22.2314GiB/s) | 20260507 15:12:37.198 | INFO | Analyzing chunk frames = [480768,481280) | 20260507 15:12:37.198 | INFO | MemInfo: { | 20260507 15:12:37.198 | INFO | "gpu" : " 17 GB", | 20260507 15:12:37.198 | INFO | "huge" : "", | 20260507 15:12:37.198 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:37.198 | INFO | "malloc" : " 14 GB", | 20260507 15:12:37.198 | INFO | "pinned" : " 29 GB" | 20260507 15:12:37.198 | INFO | } | 20260507 15:12:37.198 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:12:37.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:37.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:12:37.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:12:37.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:37.198 | INFO | Uploaded 36GiB to GPU in 1.60872s (22.378GiB/s) | 20260507 15:12:37.198 | INFO | Downloaded 34GiB to GPU in 1.52832s (22.2467GiB/s) | 20260507 15:12:42.337 | INFO | Analyzing chunk frames = [481280,481792) | 20260507 15:12:42.337 | INFO | MemInfo: { | 20260507 15:12:42.337 | INFO | "gpu" : " 17 GB", | 20260507 15:12:42.337 | INFO | "huge" : "", | 20260507 15:12:42.337 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:42.337 | INFO | "malloc" : " 14 GB", | 20260507 15:12:42.337 | INFO | "pinned" : " 29 GB" | 20260507 15:12:42.337 | INFO | } | 20260507 15:12:42.337 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:12:42.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:42.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:12:42.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:12:42.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:42.337 | INFO | Uploaded 36GiB to GPU in 1.61005s (22.3596GiB/s) | 20260507 15:12:42.337 | INFO | Downloaded 34GiB to GPU in 1.52708s (22.2648GiB/s) | 20260507 15:12:47.476 | INFO | Analyzing chunk frames = [481792,482304) | 20260507 15:12:47.476 | INFO | MemInfo: { | 20260507 15:12:47.476 | INFO | "gpu" : " 17 GB", | 20260507 15:12:47.476 | INFO | "huge" : "", | 20260507 15:12:47.476 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:47.476 | INFO | "malloc" : " 14 GB", | 20260507 15:12:47.476 | INFO | "pinned" : " 29 GB" | 20260507 15:12:47.476 | INFO | } | 20260507 15:12:47.476 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:12:47.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:47.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.476 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:12:47.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.476 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:12:47.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:47.477 | INFO | Uploaded 36GiB to GPU in 1.60773s (22.3918GiB/s) | 20260507 15:12:47.477 | INFO | Downloaded 34GiB to GPU in 1.52699s (22.2661GiB/s) | 20260507 15:12:52.616 | INFO | Analyzing chunk frames = [482304,482816) | 20260507 15:12:52.616 | INFO | MemInfo: { | 20260507 15:12:52.616 | INFO | "gpu" : " 17 GB", | 20260507 15:12:52.616 | INFO | "huge" : "", | 20260507 15:12:52.616 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:52.616 | INFO | "malloc" : " 14 GB", | 20260507 15:12:52.616 | INFO | "pinned" : " 29 GB" | 20260507 15:12:52.616 | INFO | } | 20260507 15:12:52.616 | INFO | Perf report for the last 5.13964 seconds: | 20260507 15:12:52.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:52.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.616 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:12:52.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.616 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:12:52.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:52.617 | INFO | Uploaded 36GiB to GPU in 1.60976s (22.3636GiB/s) | 20260507 15:12:52.617 | INFO | Downloaded 34GiB to GPU in 1.52715s (22.2637GiB/s) | 20260507 15:12:57.755 | INFO | Analyzing chunk frames = [482816,483328) | 20260507 15:12:57.755 | INFO | MemInfo: { | 20260507 15:12:57.755 | INFO | "gpu" : " 17 GB", | 20260507 15:12:57.755 | INFO | "huge" : "", | 20260507 15:12:57.755 | INFO | "huge_shared" : " 29 GB", | 20260507 15:12:57.755 | INFO | "malloc" : " 14 GB", | 20260507 15:12:57.755 | INFO | "pinned" : " 29 GB" | 20260507 15:12:57.755 | INFO | } | 20260507 15:12:57.755 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:12:57.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:12:57.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.755 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:12:57.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.755 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 15:12:57.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:12:57.756 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:12:57.756 | INFO | Total mem usage: 16.4 GiB | 20260507 15:12:57.756 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:12:57.756 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.4%) | 20260507 15:12:57.756 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:12:57.756 | INFO | High Water Breakdown: | 20260507 15:12:57.756 | INFO | 59% : 8.76 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:12:57.756 | INFO | 40.9% : 6.08 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:12:57.756 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:12:57.756 | INFO | ------------------------------------------ | 20260507 15:12:57.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:12:57.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:12:57.756 | INFO | Total mem usage: 26.5 GiB | 20260507 15:12:57.756 | INFO | Uploaded 36GiB to GPU in 1.60896s (22.3747GiB/s) | 20260507 15:12:57.756 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:12:57.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 15:12:57.756 | INFO | Downloaded 34GiB to GPU in 1.52927s (22.2328GiB/s) | 20260507 15:12:57.756 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:12:57.756 | INFO | High Water Breakdown: | 20260507 15:12:57.756 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:12:57.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:12:57.756 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:12:57.756 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:12:57.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:12:57.756 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:12:57.756 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:12:57.756 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:12:57.756 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:12:57.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:12:57.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:12:57.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:12:57.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:12:57.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:12:57.756 | INFO | ------------------------------------------ | 20260507 15:12:57.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:12:57.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:12:57.756 | INFO | Total mem usage: 27.8 GiB | 20260507 15:12:57.756 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:12:57.756 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:12:57.756 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:12:57.756 | INFO | High Water Breakdown: | 20260507 15:12:57.756 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:12:57.756 | INFO | ------------------------------------------ | 20260507 15:12:57.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:12:57.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:12:57.756 | INFO | Total mem usage: 15.7 GiB | 20260507 15:12:57.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:12:57.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:12:57.756 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:12:57.756 | INFO | High Water Breakdown: | 20260507 15:12:57.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:12:57.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:12:57.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:12:57.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:12:57.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:12:57.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:12:57.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:12:57.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:12:57.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:12:57.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:12:57.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:12:57.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:12:57.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:12:57.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:12:57.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:12:57.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:12:57.756 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:12:57.756 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:12:57.756 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:12:57.756 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:12:57.756 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:12:57.756 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:12:57.756 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:12:57.756 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:12:57.756 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:12:57.756 | INFO | ------------------------------------------ | 20260507 15:12:57.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:12:57.757 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:12:57.757 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:12:57.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:12:57.757 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 4.75e-03 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 11.02%, Baseline -- 3.29e+00 / -nan, 1.98e-02 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 8.24e-02 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 2.94e+00 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.72e+01 / -nan, 4.38e-02 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.15e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 3.50%, Metrics -- 1.05e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 1.29%, PulseSerializing -- 3.86e-01 / -nan, 5.69e-02 / 0.00e+00 | 20260507 15:12:57.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:12:57.757 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:12:57.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:12:57.757 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 18.66%, Upload -- 1.67e+01 / 0.00e+00, 4.42e-01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 18.11%, Download -- 1.62e+01 / 0.00e+00, 7.54e-01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 33.41%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 0.58%, LockUpload -- 5.16e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 0.00%, LockDownload -- 4.51e-04 / 0.00e+00, 2.00e-04 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 29.22%, LockCompute -- 2.62e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:12:57.757 | INFO | PerfReport: 0.02%, Misc -- 1.36e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 15:12:57.852 | INFO | Starting a round of Baz IO | 20260507 15:13:00.748 | INFO | Populating RTMetrics report | 20260507 15:13:00.749 | INFO | RTMetrics report populated | 20260507 15:13:00.749 | INFO | Opening tmp RTMetrics json file | 20260507 15:13:00.823 | INFO | Serializing RTMetrics to json | 20260507 15:13:00.825 | INFO | Serialization done, swapping tmp file now... | 20260507 15:13:00.826 | INFO | RTMetrics done | 20260507 15:13:00.896 | INFO | Finished a round of Baz IO: Wrote 1.737 GiB in 3.043 seconds | 20260507 15:13:00.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.31% / 0.05% / 45.64% | 20260507 15:13:00.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.25557% / 0.000123183% / 2.38419e-05% | 20260507 15:13:00.896 | INFO | Baz IO overall: | 20260507 15:13:00.896 | INFO | BytesWritten : 191841488587 | 20260507 15:13:00.896 | INFO | NumEvents : 92191772765 | 20260507 15:13:02.894 | INFO | Analyzing chunk frames = [483328,483840) | 20260507 15:13:02.894 | INFO | MemInfo: { | 20260507 15:13:02.894 | INFO | "gpu" : " 17 GB", | 20260507 15:13:02.894 | INFO | "huge" : "", | 20260507 15:13:02.894 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:02.894 | INFO | "malloc" : " 14 GB", | 20260507 15:13:02.894 | INFO | "pinned" : " 29 GB" | 20260507 15:13:02.894 | INFO | } | 20260507 15:13:02.894 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:13:02.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:02.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:02.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:02.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:02.894 | INFO | ANALYSIS : 57.4%, 2.9, 87.4ms, 0 | 20260507 15:13:02.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:02.894 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 15:13:02.894 | INFO | PRE_HQ : 56.3%, 2.1, 61.4ms, 0 | 20260507 15:13:02.894 | INFO | BAZWRITER : 58.8%, 4.4, 131.6ms, 0 | 20260507 15:13:02.894 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 15:13:02.895 | INFO | Uploaded 36GiB to GPU in 1.64304s (21.9106GiB/s) | 20260507 15:13:02.895 | INFO | Downloaded 38GiB to GPU in 1.71929s (22.1021GiB/s) | 20260507 15:13:08.034 | INFO | Analyzing chunk frames = [483840,484352) | 20260507 15:13:08.034 | INFO | MemInfo: { | 20260507 15:13:08.034 | INFO | "gpu" : " 17 GB", | 20260507 15:13:08.034 | INFO | "huge" : "", | 20260507 15:13:08.034 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:08.034 | INFO | "malloc" : " 14 GB", | 20260507 15:13:08.034 | INFO | "pinned" : " 29 GB" | 20260507 15:13:08.034 | INFO | } | 20260507 15:13:08.034 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:13:08.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:08.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:13:08.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:13:08.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:08.034 | INFO | Uploaded 36GiB to GPU in 1.60493s (22.4309GiB/s) | 20260507 15:13:08.034 | INFO | Downloaded 34GiB to GPU in 1.52472s (22.2991GiB/s) | 20260507 15:13:13.173 | INFO | Analyzing chunk frames = [484352,484864) | 20260507 15:13:13.173 | INFO | MemInfo: { | 20260507 15:13:13.173 | INFO | "gpu" : " 17 GB", | 20260507 15:13:13.173 | INFO | "huge" : "", | 20260507 15:13:13.173 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:13.173 | INFO | "malloc" : " 14 GB", | 20260507 15:13:13.173 | INFO | "pinned" : " 29 GB" | 20260507 15:13:13.173 | INFO | } | 20260507 15:13:13.173 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:13:13.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:13.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.173 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:13:13.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.173 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:13:13.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:13.174 | INFO | Uploaded 36GiB to GPU in 1.6088s (22.3769GiB/s) | 20260507 15:13:13.174 | INFO | Downloaded 34GiB to GPU in 1.5288s (22.2397GiB/s) | 20260507 15:13:18.312 | INFO | Analyzing chunk frames = [484864,485376) | 20260507 15:13:18.312 | INFO | MemInfo: { | 20260507 15:13:18.312 | INFO | "gpu" : " 17 GB", | 20260507 15:13:18.312 | INFO | "huge" : "", | 20260507 15:13:18.312 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:18.312 | INFO | "malloc" : " 14 GB", | 20260507 15:13:18.312 | INFO | "pinned" : " 29 GB" | 20260507 15:13:18.312 | INFO | } | 20260507 15:13:18.313 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:13:18.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:18.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:13:18.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:13:18.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:18.313 | INFO | Uploaded 36GiB to GPU in 1.61007s (22.3592GiB/s) | 20260507 15:13:18.313 | INFO | Downloaded 34GiB to GPU in 1.53015s (22.2201GiB/s) | 20260507 15:13:23.452 | INFO | Analyzing chunk frames = [485376,485888) | 20260507 15:13:23.452 | INFO | MemInfo: { | 20260507 15:13:23.452 | INFO | "gpu" : " 17 GB", | 20260507 15:13:23.452 | INFO | "huge" : "", | 20260507 15:13:23.452 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:23.452 | INFO | "malloc" : " 14 GB", | 20260507 15:13:23.452 | INFO | "pinned" : " 29 GB" | 20260507 15:13:23.452 | INFO | } | 20260507 15:13:23.452 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:13:23.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:23.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:13:23.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:13:23.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:23.452 | INFO | Uploaded 36GiB to GPU in 1.60738s (22.3966GiB/s) | 20260507 15:13:23.452 | INFO | Downloaded 34GiB to GPU in 1.5267s (22.2702GiB/s) | 20260507 15:13:28.591 | INFO | Analyzing chunk frames = [485888,486400) | 20260507 15:13:28.591 | INFO | MemInfo: { | 20260507 15:13:28.591 | INFO | "gpu" : " 17 GB", | 20260507 15:13:28.591 | INFO | "huge" : "", | 20260507 15:13:28.591 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:28.591 | INFO | "malloc" : " 14 GB", | 20260507 15:13:28.591 | INFO | "pinned" : " 29 GB" | 20260507 15:13:28.591 | INFO | } | 20260507 15:13:28.591 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:13:28.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:28.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.591 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:13:28.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.591 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:13:28.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:28.592 | INFO | Uploaded 36GiB to GPU in 1.6078s (22.3909GiB/s) | 20260507 15:13:28.592 | INFO | Downloaded 34GiB to GPU in 1.52753s (22.2581GiB/s) | 20260507 15:13:33.730 | INFO | Analyzing chunk frames = [486400,486912) | 20260507 15:13:33.730 | INFO | MemInfo: { | 20260507 15:13:33.730 | INFO | "gpu" : " 17 GB", | 20260507 15:13:33.730 | INFO | "huge" : "", | 20260507 15:13:33.730 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:33.730 | INFO | "malloc" : " 14 GB", | 20260507 15:13:33.730 | INFO | "pinned" : " 29 GB" | 20260507 15:13:33.730 | INFO | } | 20260507 15:13:33.731 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:13:33.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:33.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:13:33.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:13:33.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:33.731 | INFO | Uploaded 36GiB to GPU in 1.60743s (22.3959GiB/s) | 20260507 15:13:33.731 | INFO | Downloaded 34GiB to GPU in 1.52829s (22.2471GiB/s) | 20260507 15:13:38.870 | INFO | Analyzing chunk frames = [486912,487424) | 20260507 15:13:38.870 | INFO | MemInfo: { | 20260507 15:13:38.870 | INFO | "gpu" : " 17 GB", | 20260507 15:13:38.870 | INFO | "huge" : "", | 20260507 15:13:38.870 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:38.870 | INFO | "malloc" : " 14 GB", | 20260507 15:13:38.870 | INFO | "pinned" : " 29 GB" | 20260507 15:13:38.870 | INFO | } | 20260507 15:13:38.870 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:13:38.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:38.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.870 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:13:38.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.870 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260507 15:13:38.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:38.871 | INFO | Uploaded 36GiB to GPU in 1.61063s (22.3515GiB/s) | 20260507 15:13:38.871 | INFO | Downloaded 34GiB to GPU in 1.53215s (22.191GiB/s) | 20260507 15:13:38.967 | INFO | Starting a round of Baz IO | 20260507 15:13:41.874 | INFO | Populating RTMetrics report | 20260507 15:13:41.874 | INFO | RTMetrics report populated | 20260507 15:13:41.875 | INFO | Opening tmp RTMetrics json file | 20260507 15:13:41.883 | INFO | Serializing RTMetrics to json | 20260507 15:13:41.902 | INFO | Serialization done, swapping tmp file now... | 20260507 15:13:41.903 | INFO | RTMetrics done | 20260507 15:13:41.972 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.004 seconds | 20260507 15:13:41.972 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.26% / 0.05% / 45.69% | 20260507 15:13:41.972 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.21724% / 7.94729e-05% / 1.58946e-05% | 20260507 15:13:41.972 | INFO | Baz IO overall: | 20260507 15:13:41.972 | INFO | BytesWritten : 193705489875 | 20260507 15:13:41.972 | INFO | NumEvents : 93099466003 | 20260507 15:13:44.009 | INFO | Analyzing chunk frames = [487424,487936) | 20260507 15:13:44.009 | INFO | MemInfo: { | 20260507 15:13:44.009 | INFO | "gpu" : " 17 GB", | 20260507 15:13:44.009 | INFO | "huge" : "", | 20260507 15:13:44.009 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:44.009 | INFO | "malloc" : " 14 GB", | 20260507 15:13:44.010 | INFO | "pinned" : " 29 GB" | 20260507 15:13:44.010 | INFO | } | 20260507 15:13:44.010 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:13:44.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:44.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:44.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:44.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:44.010 | INFO | ANALYSIS : 57.3%, 2.9, 86.6ms, 0 | 20260507 15:13:44.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:44.010 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 15:13:44.010 | INFO | PRE_HQ : 56.2%, 2.1, 62.1ms, 0 | 20260507 15:13:44.010 | INFO | BAZWRITER : 58.0%, 4.5, 134.0ms, 0 | 20260507 15:13:44.010 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:13:44.010 | INFO | Uploaded 36GiB to GPU in 1.65173s (21.7954GiB/s) | 20260507 15:13:44.010 | INFO | Downloaded 38GiB to GPU in 1.72313s (22.0529GiB/s) | 20260507 15:13:49.148 | INFO | Analyzing chunk frames = [487936,488448) | 20260507 15:13:49.148 | INFO | MemInfo: { | 20260507 15:13:49.148 | INFO | "gpu" : " 17 GB", | 20260507 15:13:49.148 | INFO | "huge" : "", | 20260507 15:13:49.148 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:49.148 | INFO | "malloc" : " 14 GB", | 20260507 15:13:49.148 | INFO | "pinned" : " 29 GB" | 20260507 15:13:49.148 | INFO | } | 20260507 15:13:49.149 | INFO | Perf report for the last 5.13896 seconds: | 20260507 15:13:49.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:49.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:13:49.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:13:49.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:49.149 | INFO | Uploaded 36GiB to GPU in 1.60487s (22.4318GiB/s) | 20260507 15:13:49.149 | INFO | Downloaded 34GiB to GPU in 1.5242s (22.3068GiB/s) | 20260507 15:13:54.288 | INFO | Analyzing chunk frames = [488448,488960) | 20260507 15:13:54.288 | INFO | MemInfo: { | 20260507 15:13:54.288 | INFO | "gpu" : " 17 GB", | 20260507 15:13:54.288 | INFO | "huge" : "", | 20260507 15:13:54.288 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:54.288 | INFO | "malloc" : " 14 GB", | 20260507 15:13:54.288 | INFO | "pinned" : " 29 GB" | 20260507 15:13:54.288 | INFO | } | 20260507 15:13:54.288 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:13:54.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:54.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:13:54.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:13:54.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:54.288 | INFO | Uploaded 36GiB to GPU in 1.60888s (22.3758GiB/s) | 20260507 15:13:54.289 | INFO | Downloaded 34GiB to GPU in 1.52923s (22.2334GiB/s) | 20260507 15:13:59.427 | INFO | Analyzing chunk frames = [488960,489472) | 20260507 15:13:59.427 | INFO | MemInfo: { | 20260507 15:13:59.427 | INFO | "gpu" : " 17 GB", | 20260507 15:13:59.427 | INFO | "huge" : "", | 20260507 15:13:59.427 | INFO | "huge_shared" : " 29 GB", | 20260507 15:13:59.427 | INFO | "malloc" : " 14 GB", | 20260507 15:13:59.427 | INFO | "pinned" : " 29 GB" | 20260507 15:13:59.427 | INFO | } | 20260507 15:13:59.428 | INFO | Perf report for the last 5.13973 seconds: | 20260507 15:13:59.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:13:59.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:13:59.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:13:59.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:13:59.428 | INFO | Uploaded 36GiB to GPU in 1.60919s (22.3715GiB/s) | 20260507 15:13:59.428 | INFO | Downloaded 34GiB to GPU in 1.52878s (22.24GiB/s) | 20260507 15:14:04.566 | INFO | Analyzing chunk frames = [489472,489984) | 20260507 15:14:04.566 | INFO | MemInfo: { | 20260507 15:14:04.566 | INFO | "gpu" : " 17 GB", | 20260507 15:14:04.566 | INFO | "huge" : "", | 20260507 15:14:04.566 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:04.566 | INFO | "malloc" : " 14 GB", | 20260507 15:14:04.566 | INFO | "pinned" : " 29 GB" | 20260507 15:14:04.566 | INFO | } | 20260507 15:14:04.566 | INFO | Perf report for the last 5.13887 seconds: | 20260507 15:14:04.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:04.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.566 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:14:04.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.566 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:14:04.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:04.567 | INFO | Uploaded 36GiB to GPU in 1.60707s (22.401GiB/s) | 20260507 15:14:04.567 | INFO | Downloaded 34GiB to GPU in 1.52626s (22.2767GiB/s) | 20260507 15:14:09.706 | INFO | Analyzing chunk frames = [489984,490496) | 20260507 15:14:09.706 | INFO | MemInfo: { | 20260507 15:14:09.706 | INFO | "gpu" : " 17 GB", | 20260507 15:14:09.706 | INFO | "huge" : "", | 20260507 15:14:09.706 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:09.706 | INFO | "malloc" : " 14 GB", | 20260507 15:14:09.706 | INFO | "pinned" : " 29 GB" | 20260507 15:14:09.706 | INFO | } | 20260507 15:14:09.706 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:14:09.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:09.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:14:09.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:14:09.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:09.706 | INFO | Uploaded 36GiB to GPU in 1.60698s (22.4023GiB/s) | 20260507 15:14:09.706 | INFO | Downloaded 34GiB to GPU in 1.52631s (22.276GiB/s) | 20260507 15:14:14.845 | INFO | Analyzing chunk frames = [490496,491008) | 20260507 15:14:14.845 | INFO | MemInfo: { | 20260507 15:14:14.845 | INFO | "gpu" : " 17 GB", | 20260507 15:14:14.845 | INFO | "huge" : "", | 20260507 15:14:14.845 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:14.845 | INFO | "malloc" : " 14 GB", | 20260507 15:14:14.845 | INFO | "pinned" : " 29 GB" | 20260507 15:14:14.845 | INFO | } | 20260507 15:14:14.845 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:14:14.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:14.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.845 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:14:14.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.845 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:14:14.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:14.846 | INFO | Uploaded 36GiB to GPU in 1.60765s (22.393GiB/s) | 20260507 15:14:14.846 | INFO | Downloaded 34GiB to GPU in 1.5275s (22.2586GiB/s) | 20260507 15:14:19.985 | INFO | Analyzing chunk frames = [491008,491520) | 20260507 15:14:19.985 | INFO | MemInfo: { | 20260507 15:14:19.985 | INFO | "gpu" : " 17 GB", | 20260507 15:14:19.985 | INFO | "huge" : "", | 20260507 15:14:19.985 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:19.985 | INFO | "malloc" : " 14 GB", | 20260507 15:14:19.985 | INFO | "pinned" : " 29 GB" | 20260507 15:14:19.985 | INFO | } | 20260507 15:14:19.985 | INFO | Perf report for the last 5.13991 seconds: | 20260507 15:14:19.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:19.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.985 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:14:19.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.985 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:14:19.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:19.986 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:14:19.986 | INFO | Total mem usage: 16.4 GiB | 20260507 15:14:19.986 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:14:19.986 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.5%) | 20260507 15:14:19.986 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:14:19.986 | INFO | High Water Breakdown: | 20260507 15:14:19.986 | INFO | 58.9% : 8.75 GiB ( 88.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:14:19.986 | INFO | 40.9% : 6.07 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:14:19.986 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:14:19.986 | INFO | ------------------------------------------ | 20260507 15:14:19.986 | INFO | Uploaded 36GiB to GPU in 1.60758s (22.3938GiB/s) | 20260507 15:14:19.986 | INFO | Downloaded 34GiB to GPU in 1.52776s (22.2548GiB/s) | 20260507 15:14:19.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:14:19.986 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:14:19.986 | INFO | Total mem usage: 26.5 GiB | 20260507 15:14:19.986 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:14:19.986 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:14:19.986 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:14:19.986 | INFO | High Water Breakdown: | 20260507 15:14:19.986 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:14:19.986 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:14:19.986 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:14:19.986 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:14:19.986 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:14:19.986 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:14:19.986 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:14:19.986 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:14:19.986 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:14:19.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:14:19.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:14:19.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:14:19.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:14:19.986 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:14:19.986 | INFO | ------------------------------------------ | 20260507 15:14:19.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:14:19.986 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:14:19.986 | INFO | Total mem usage: 27.8 GiB | 20260507 15:14:19.986 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:14:19.986 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:14:19.986 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:14:19.986 | INFO | High Water Breakdown: | 20260507 15:14:19.986 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:14:19.986 | INFO | ------------------------------------------ | 20260507 15:14:19.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:14:19.986 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:14:19.986 | INFO | Total mem usage: 15.7 GiB | 20260507 15:14:19.986 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:14:19.987 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:14:19.987 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:14:19.987 | INFO | High Water Breakdown: | 20260507 15:14:19.987 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:14:19.987 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:14:19.987 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:14:19.987 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:14:19.987 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:14:19.987 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:14:19.987 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:14:19.987 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:14:19.987 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:14:19.987 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:14:19.987 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:14:19.987 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:14:19.987 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:14:19.987 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:14:19.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:14:19.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:14:19.987 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:14:19.987 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:14:19.987 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:14:19.987 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:14:19.987 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:14:19.987 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:14:19.987 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:14:19.987 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:14:19.987 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:14:19.987 | INFO | ------------------------------------------ | 20260507 15:14:19.987 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:14:19.987 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:14:19.987 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:14:19.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:14:19.987 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.76e-03 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 11.02%, Baseline -- 3.29e+00 / -nan, 1.88e-02 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 3.07%, Histogram -- 9.19e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 16.13%, DME -- 3.86e+01 / -nan, 2.98e+00 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.15e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 3.50%, Metrics -- 1.05e+00 / -nan, 1.79e-01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 1.29%, PulseSerializing -- 3.85e-01 / -nan, 5.62e-02 / 0.00e+00 | 20260507 15:14:19.987 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:14:19.987 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:14:19.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:14:19.987 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 18.77%, Upload -- 1.68e+01 / 0.00e+00, 4.19e-01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 18.06%, Download -- 1.62e+01 / 0.00e+00, 7.59e-01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 33.39%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 0.58%, LockUpload -- 5.17e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 0.00%, LockDownload -- 4.38e-04 / 0.00e+00, 2.18e-04 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 29.19%, LockCompute -- 2.61e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:14:19.987 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 15:14:20.082 | INFO | Starting a round of Baz IO | 20260507 15:14:22.967 | INFO | Populating RTMetrics report | 20260507 15:14:22.967 | INFO | RTMetrics report populated | 20260507 15:14:22.967 | INFO | Opening tmp RTMetrics json file | 20260507 15:14:23.104 | INFO | Serializing RTMetrics to json | 20260507 15:14:23.121 | INFO | Serialization done, swapping tmp file now... | 20260507 15:14:23.130 | INFO | RTMetrics done | 20260507 15:14:23.156 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.074 seconds | 20260507 15:14:23.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.21% / 0.05% / 45.74% | 20260507 15:14:23.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.19317% / 9.93411e-05% / 4.76837e-05% | 20260507 15:14:23.156 | INFO | Baz IO overall: | 20260507 15:14:23.156 | INFO | BytesWritten : 195569357412 | 20260507 15:14:23.156 | INFO | NumEvents : 94007077041 | 20260507 15:14:25.124 | INFO | Analyzing chunk frames = [491520,492032) | 20260507 15:14:25.124 | INFO | MemInfo: { | 20260507 15:14:25.124 | INFO | "gpu" : " 17 GB", | 20260507 15:14:25.124 | INFO | "huge" : "", | 20260507 15:14:25.124 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:25.124 | INFO | "malloc" : " 14 GB", | 20260507 15:14:25.124 | INFO | "pinned" : " 29 GB" | 20260507 15:14:25.124 | INFO | } | 20260507 15:14:25.124 | INFO | Perf report for the last 5.13861 seconds: | 20260507 15:14:25.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:25.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:25.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:25.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:25.124 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260507 15:14:25.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:25.124 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 15:14:25.124 | INFO | PRE_HQ : 55.8%, 2.1, 60.7ms, 0 | 20260507 15:14:25.124 | INFO | BAZWRITER : 59.3%, 4.2, 129.1ms, 0 | 20260507 15:14:25.124 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260507 15:14:25.124 | INFO | Uploaded 36GiB to GPU in 1.63833s (21.9736GiB/s) | 20260507 15:14:25.124 | INFO | Downloaded 38GiB to GPU in 1.70625s (22.2711GiB/s) | 20260507 15:14:30.284 | INFO | Analyzing chunk frames = [492032,492544) | 20260507 15:14:30.284 | INFO | MemInfo: { | 20260507 15:14:30.284 | INFO | "gpu" : " 17 GB", | 20260507 15:14:30.284 | INFO | "huge" : "", | 20260507 15:14:30.284 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:30.284 | INFO | "malloc" : " 14 GB", | 20260507 15:14:30.284 | INFO | "pinned" : " 29 GB" | 20260507 15:14:30.284 | INFO | } | 20260507 15:14:30.284 | INFO | Perf report for the last 5.16 seconds: | 20260507 15:14:30.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:30.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | ANALYSIS : 56.7%, 3.0, 88.8ms, 0 | 20260507 15:14:30.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:14:30.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:30.284 | INFO | Uploaded 36GiB to GPU in 1.60435s (22.439GiB/s) | 20260507 15:14:30.285 | INFO | Downloaded 34GiB to GPU in 1.52444s (22.3033GiB/s) | 20260507 15:14:35.403 | INFO | Analyzing chunk frames = [492544,493056) | 20260507 15:14:35.403 | INFO | MemInfo: { | 20260507 15:14:35.403 | INFO | "gpu" : " 17 GB", | 20260507 15:14:35.403 | INFO | "huge" : "", | 20260507 15:14:35.403 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:35.403 | INFO | "malloc" : " 14 GB", | 20260507 15:14:35.403 | INFO | "pinned" : " 29 GB" | 20260507 15:14:35.403 | INFO | } | 20260507 15:14:35.403 | INFO | Perf report for the last 5.11942 seconds: | 20260507 15:14:35.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:35.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.403 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260507 15:14:35.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.403 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260507 15:14:35.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:35.404 | INFO | Uploaded 36GiB to GPU in 1.60697s (22.4024GiB/s) | 20260507 15:14:35.404 | INFO | Downloaded 34GiB to GPU in 1.52916s (22.2344GiB/s) | 20260507 15:14:40.542 | INFO | Analyzing chunk frames = [493056,493568) | 20260507 15:14:40.542 | INFO | MemInfo: { | 20260507 15:14:40.542 | INFO | "gpu" : " 17 GB", | 20260507 15:14:40.542 | INFO | "huge" : "", | 20260507 15:14:40.542 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:40.542 | INFO | "malloc" : " 14 GB", | 20260507 15:14:40.542 | INFO | "pinned" : " 29 GB" | 20260507 15:14:40.542 | INFO | } | 20260507 15:14:40.543 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:14:40.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:40.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 15:14:40.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 15:14:40.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:40.543 | INFO | Uploaded 36GiB to GPU in 1.61363s (22.3099GiB/s) | 20260507 15:14:40.543 | INFO | Downloaded 34GiB to GPU in 1.53238s (22.1877GiB/s) | 20260507 15:14:45.681 | INFO | Analyzing chunk frames = [493568,494080) | 20260507 15:14:45.681 | INFO | MemInfo: { | 20260507 15:14:45.681 | INFO | "gpu" : " 17 GB", | 20260507 15:14:45.682 | INFO | "huge" : "", | 20260507 15:14:45.682 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:45.682 | INFO | "malloc" : " 14 GB", | 20260507 15:14:45.682 | INFO | "pinned" : " 29 GB" | 20260507 15:14:45.682 | INFO | } | 20260507 15:14:45.682 | INFO | Perf report for the last 5.13925 seconds: | 20260507 15:14:45.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:45.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:14:45.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:14:45.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:45.682 | INFO | Uploaded 36GiB to GPU in 1.61236s (22.3275GiB/s) | 20260507 15:14:45.682 | INFO | Downloaded 34GiB to GPU in 1.53193s (22.1942GiB/s) | 20260507 15:14:50.821 | INFO | Analyzing chunk frames = [494080,494592) | 20260507 15:14:50.821 | INFO | MemInfo: { | 20260507 15:14:50.821 | INFO | "gpu" : " 17 GB", | 20260507 15:14:50.821 | INFO | "huge" : "", | 20260507 15:14:50.821 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:50.821 | INFO | "malloc" : " 14 GB", | 20260507 15:14:50.821 | INFO | "pinned" : " 29 GB" | 20260507 15:14:50.821 | INFO | } | 20260507 15:14:50.821 | INFO | Perf report for the last 5.13953 seconds: | 20260507 15:14:50.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:50.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.821 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:14:50.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.822 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 15:14:50.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:50.822 | INFO | Uploaded 36GiB to GPU in 1.61213s (22.3308GiB/s) | 20260507 15:14:50.822 | INFO | Downloaded 34GiB to GPU in 1.53131s (22.2032GiB/s) | 20260507 15:14:55.960 | INFO | Analyzing chunk frames = [494592,495104) | 20260507 15:14:55.960 | INFO | MemInfo: { | 20260507 15:14:55.960 | INFO | "gpu" : " 17 GB", | 20260507 15:14:55.960 | INFO | "huge" : "", | 20260507 15:14:55.960 | INFO | "huge_shared" : " 29 GB", | 20260507 15:14:55.960 | INFO | "malloc" : " 14 GB", | 20260507 15:14:55.960 | INFO | "pinned" : " 29 GB" | 20260507 15:14:55.960 | INFO | } | 20260507 15:14:55.960 | INFO | Perf report for the last 5.13885 seconds: | 20260507 15:14:55.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:14:55.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.960 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:14:55.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.960 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:14:55.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:14:55.961 | INFO | Uploaded 36GiB to GPU in 1.61425s (22.3013GiB/s) | 20260507 15:14:55.961 | INFO | Downloaded 34GiB to GPU in 1.53347s (22.1719GiB/s) | 20260507 15:15:01.099 | INFO | Analyzing chunk frames = [495104,495616) | 20260507 15:15:01.099 | INFO | MemInfo: { | 20260507 15:15:01.099 | INFO | "gpu" : " 17 GB", | 20260507 15:15:01.099 | INFO | "huge" : "", | 20260507 15:15:01.099 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:01.100 | INFO | "malloc" : " 14 GB", | 20260507 15:15:01.100 | INFO | "pinned" : " 29 GB" | 20260507 15:15:01.100 | INFO | } | 20260507 15:15:01.100 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:15:01.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:01.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:15:01.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:15:01.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:01.100 | INFO | Uploaded 36GiB to GPU in 1.60747s (22.3954GiB/s) | 20260507 15:15:01.100 | INFO | Downloaded 34GiB to GPU in 1.5284s (22.2454GiB/s) | 20260507 15:15:01.197 | INFO | Starting a round of Baz IO | 20260507 15:15:04.121 | INFO | Populating RTMetrics report | 20260507 15:15:04.121 | INFO | RTMetrics report populated | 20260507 15:15:04.121 | INFO | Opening tmp RTMetrics json file | 20260507 15:15:04.126 | INFO | Serializing RTMetrics to json | 20260507 15:15:04.154 | INFO | Serialization done, swapping tmp file now... | 20260507 15:15:04.155 | INFO | RTMetrics done | 20260507 15:15:04.357 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.159 seconds | 20260507 15:15:04.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.15% / 0.05% / 45.79% | 20260507 15:15:04.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.1786% / 7.94729e-05% / 3.57628e-05% | 20260507 15:15:04.357 | INFO | Baz IO overall: | 20260507 15:15:04.357 | INFO | BytesWritten : 197435036416 | 20260507 15:15:04.357 | INFO | NumEvents : 94915620807 | 20260507 15:15:06.239 | INFO | Analyzing chunk frames = [495616,496128) | 20260507 15:15:06.239 | INFO | MemInfo: { | 20260507 15:15:06.239 | INFO | "gpu" : " 17 GB", | 20260507 15:15:06.239 | INFO | "huge" : "", | 20260507 15:15:06.239 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:06.239 | INFO | "malloc" : " 14 GB", | 20260507 15:15:06.239 | INFO | "pinned" : " 29 GB" | 20260507 15:15:06.239 | INFO | } | 20260507 15:15:06.239 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:15:06.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:06.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:06.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:06.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:06.239 | INFO | ANALYSIS : 57.9%, 2.8, 85.8ms, 0 | 20260507 15:15:06.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:06.239 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 15:15:06.239 | INFO | PRE_HQ : 56.9%, 2.1, 63.2ms, 0 | 20260507 15:15:06.239 | INFO | BAZWRITER : 61.0%, 4.5, 140.0ms, 0 | 20260507 15:15:06.239 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 15:15:06.239 | INFO | Uploaded 36GiB to GPU in 1.64543s (21.8788GiB/s) | 20260507 15:15:06.240 | INFO | Downloaded 38GiB to GPU in 1.71157s (22.2018GiB/s) | 20260507 15:15:11.378 | INFO | Analyzing chunk frames = [496128,496640) | 20260507 15:15:11.378 | INFO | MemInfo: { | 20260507 15:15:11.378 | INFO | "gpu" : " 17 GB", | 20260507 15:15:11.378 | INFO | "huge" : "", | 20260507 15:15:11.378 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:11.378 | INFO | "malloc" : " 14 GB", | 20260507 15:15:11.378 | INFO | "pinned" : " 29 GB" | 20260507 15:15:11.378 | INFO | } | 20260507 15:15:11.378 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:15:11.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:11.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.378 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:15:11.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.378 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:15:11.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:11.379 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.4359GiB/s) | 20260507 15:15:11.379 | INFO | Downloaded 34GiB to GPU in 1.52355s (22.3164GiB/s) | 20260507 15:15:16.517 | INFO | Analyzing chunk frames = [496640,497152) | 20260507 15:15:16.517 | INFO | MemInfo: { | 20260507 15:15:16.517 | INFO | "gpu" : " 17 GB", | 20260507 15:15:16.517 | INFO | "huge" : "", | 20260507 15:15:16.517 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:16.518 | INFO | "malloc" : " 14 GB", | 20260507 15:15:16.518 | INFO | "pinned" : " 29 GB" | 20260507 15:15:16.518 | INFO | } | 20260507 15:15:16.518 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:15:16.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:16.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:15:16.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:15:16.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:16.518 | INFO | Uploaded 36GiB to GPU in 1.60967s (22.3648GiB/s) | 20260507 15:15:16.518 | INFO | Downloaded 34GiB to GPU in 1.53043s (22.216GiB/s) | 20260507 15:15:21.657 | INFO | Analyzing chunk frames = [497152,497664) | 20260507 15:15:21.657 | INFO | MemInfo: { | 20260507 15:15:21.657 | INFO | "gpu" : " 17 GB", | 20260507 15:15:21.657 | INFO | "huge" : "", | 20260507 15:15:21.657 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:21.657 | INFO | "malloc" : " 14 GB", | 20260507 15:15:21.657 | INFO | "pinned" : " 29 GB" | 20260507 15:15:21.657 | INFO | } | 20260507 15:15:21.657 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:15:21.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:21.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:15:21.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:15:21.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:21.657 | INFO | Uploaded 36GiB to GPU in 1.60868s (22.3786GiB/s) | 20260507 15:15:21.657 | INFO | Downloaded 34GiB to GPU in 1.52881s (22.2395GiB/s) | 20260507 15:15:26.796 | INFO | Analyzing chunk frames = [497664,498176) | 20260507 15:15:26.796 | INFO | MemInfo: { | 20260507 15:15:26.796 | INFO | "gpu" : " 17 GB", | 20260507 15:15:26.796 | INFO | "huge" : "", | 20260507 15:15:26.796 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:26.796 | INFO | "malloc" : " 14 GB", | 20260507 15:15:26.796 | INFO | "pinned" : " 29 GB" | 20260507 15:15:26.796 | INFO | } | 20260507 15:15:26.796 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:15:26.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:26.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.796 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:15:26.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.796 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:15:26.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:26.797 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3922GiB/s) | 20260507 15:15:26.797 | INFO | Downloaded 34GiB to GPU in 1.52681s (22.2687GiB/s) | 20260507 15:15:31.935 | INFO | Analyzing chunk frames = [498176,498688) | 20260507 15:15:31.935 | INFO | MemInfo: { | 20260507 15:15:31.935 | INFO | "gpu" : " 17 GB", | 20260507 15:15:31.935 | INFO | "huge" : "", | 20260507 15:15:31.935 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:31.935 | INFO | "malloc" : " 14 GB", | 20260507 15:15:31.936 | INFO | "pinned" : " 29 GB" | 20260507 15:15:31.936 | INFO | } | 20260507 15:15:31.936 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:15:31.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:31.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:15:31.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:15:31.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:31.936 | INFO | Uploaded 36GiB to GPU in 1.60728s (22.3981GiB/s) | 20260507 15:15:31.936 | INFO | Downloaded 34GiB to GPU in 1.52729s (22.2617GiB/s) | 20260507 15:15:37.075 | INFO | Analyzing chunk frames = [498688,499200) | 20260507 15:15:37.075 | INFO | MemInfo: { | 20260507 15:15:37.075 | INFO | "gpu" : " 17 GB", | 20260507 15:15:37.075 | INFO | "huge" : "", | 20260507 15:15:37.075 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:37.075 | INFO | "malloc" : " 14 GB", | 20260507 15:15:37.075 | INFO | "pinned" : " 29 GB" | 20260507 15:15:37.075 | INFO | } | 20260507 15:15:37.075 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:15:37.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:37.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:15:37.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:15:37.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:37.075 | INFO | Uploaded 36GiB to GPU in 1.60823s (22.3849GiB/s) | 20260507 15:15:37.075 | INFO | Downloaded 34GiB to GPU in 1.52832s (22.2467GiB/s) | 20260507 15:15:42.214 | INFO | Analyzing chunk frames = [499200,499712) | 20260507 15:15:42.214 | INFO | MemInfo: { | 20260507 15:15:42.214 | INFO | "gpu" : " 17 GB", | 20260507 15:15:42.214 | INFO | "huge" : "", | 20260507 15:15:42.214 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:42.214 | INFO | "malloc" : " 14 GB", | 20260507 15:15:42.214 | INFO | "pinned" : " 29 GB" | 20260507 15:15:42.215 | INFO | } | 20260507 15:15:42.215 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:15:42.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:42.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:15:42.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 15:15:42.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:42.215 | INFO | Uploaded 36GiB to GPU in 1.60797s (22.3885GiB/s) | 20260507 15:15:42.215 | INFO | Downloaded 34GiB to GPU in 1.52887s (22.2386GiB/s) | 20260507 15:15:42.215 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:15:42.215 | INFO | Total mem usage: 16.4 GiB | 20260507 15:15:42.215 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:15:42.215 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.91%) | 20260507 15:15:42.215 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:15:42.215 | INFO | High Water Breakdown: | 20260507 15:15:42.215 | INFO | 59.2% : 8.92 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:15:42.215 | INFO | 40.7% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:15:42.215 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:15:42.215 | INFO | ------------------------------------------ | 20260507 15:15:42.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:15:42.215 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:15:42.215 | INFO | Total mem usage: 26.5 GiB | 20260507 15:15:42.215 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:15:42.215 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:15:42.215 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:15:42.215 | INFO | High Water Breakdown: | 20260507 15:15:42.215 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:15:42.215 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:15:42.215 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:15:42.215 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:15:42.215 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:15:42.215 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:15:42.215 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:15:42.215 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:15:42.215 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:15:42.215 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:15:42.215 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:15:42.215 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:15:42.215 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:15:42.215 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:15:42.215 | INFO | ------------------------------------------ | 20260507 15:15:42.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:15:42.215 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:15:42.215 | INFO | Total mem usage: 27.8 GiB | 20260507 15:15:42.215 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:15:42.216 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:15:42.216 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:15:42.216 | INFO | High Water Breakdown: | 20260507 15:15:42.216 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:15:42.216 | INFO | ------------------------------------------ | 20260507 15:15:42.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:15:42.216 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:15:42.216 | INFO | Total mem usage: 15.7 GiB | 20260507 15:15:42.216 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:15:42.216 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:15:42.216 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:15:42.216 | INFO | High Water Breakdown: | 20260507 15:15:42.216 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:15:42.216 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:15:42.216 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:15:42.216 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:15:42.216 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:15:42.216 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:15:42.216 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:15:42.216 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:15:42.216 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:15:42.216 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:15:42.216 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:15:42.216 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:15:42.216 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:15:42.216 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:15:42.216 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:15:42.216 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:15:42.216 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:15:42.216 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:15:42.216 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:15:42.216 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:15:42.216 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:15:42.216 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:15:42.216 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:15:42.216 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:15:42.216 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:15:42.216 | INFO | ------------------------------------------ | 20260507 15:15:42.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:15:42.216 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:15:42.216 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:15:42.216 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:15:42.216 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 0.29%, AggregateStats -- 8.80e-02 / -nan, 5.46e-03 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 11.03%, Baseline -- 3.30e+00 / -nan, 2.17e-02 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 3.08%, Histogram -- 9.20e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.72e+01 / -nan, 6.57e-02 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.16e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 3.50%, Metrics -- 1.05e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 1.30%, PulseSerializing -- 3.88e-01 / -nan, 5.69e-02 / 0.00e+00 | 20260507 15:15:42.216 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:15:42.216 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:15:42.216 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:15:42.216 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 18.75%, Upload -- 1.68e+01 / 0.00e+00, 3.96e-01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 18.03%, Download -- 1.62e+01 / 0.00e+00, 7.08e-01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 33.37%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 0.00%, LockDownload -- 5.08e-04 / 0.00e+00, 3.22e-04 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 29.26%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:15:42.216 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260507 15:15:42.311 | INFO | Starting a round of Baz IO | 20260507 15:15:45.194 | INFO | Populating RTMetrics report | 20260507 15:15:45.194 | INFO | RTMetrics report populated | 20260507 15:15:45.194 | INFO | Opening tmp RTMetrics json file | 20260507 15:15:45.197 | INFO | Serializing RTMetrics to json | 20260507 15:15:45.203 | INFO | Serialization done, swapping tmp file now... | 20260507 15:15:45.203 | INFO | RTMetrics done | 20260507 15:15:45.318 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 3.006 seconds | 20260507 15:15:45.318 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.11% / 0.05% / 45.85% | 20260507 15:15:45.318 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.16711% / 0.000111262% / 3.57628e-05% | 20260507 15:15:45.318 | INFO | Baz IO overall: | 20260507 15:15:45.318 | INFO | BytesWritten : 199302501319 | 20260507 15:15:45.318 | INFO | NumEvents : 95824811728 | 20260507 15:15:47.354 | INFO | Analyzing chunk frames = [499712,500224) | 20260507 15:15:47.354 | INFO | MemInfo: { | 20260507 15:15:47.354 | INFO | "gpu" : " 17 GB", | 20260507 15:15:47.354 | INFO | "huge" : "", | 20260507 15:15:47.354 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:47.354 | INFO | "malloc" : " 14 GB", | 20260507 15:15:47.354 | INFO | "pinned" : " 29 GB" | 20260507 15:15:47.354 | INFO | } | 20260507 15:15:47.354 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:15:47.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:47.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:47.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:47.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:47.354 | INFO | ANALYSIS : 57.1%, 2.9, 85.4ms, 0 | 20260507 15:15:47.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:47.354 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 15:15:47.354 | INFO | PRE_HQ : 56.1%, 2.0, 59.4ms, 0 | 20260507 15:15:47.354 | INFO | BAZWRITER : 58.1%, 4.8, 142.3ms, 0 | 20260507 15:15:47.354 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 15:15:47.354 | INFO | Uploaded 36GiB to GPU in 1.67487s (21.4943GiB/s) | 20260507 15:15:47.354 | INFO | Downloaded 38GiB to GPU in 1.74454s (21.7823GiB/s) | 20260507 15:15:52.493 | INFO | Analyzing chunk frames = [500224,500736) | 20260507 15:15:52.493 | INFO | MemInfo: { | 20260507 15:15:52.493 | INFO | "gpu" : " 17 GB", | 20260507 15:15:52.493 | INFO | "huge" : "", | 20260507 15:15:52.493 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:52.493 | INFO | "malloc" : " 14 GB", | 20260507 15:15:52.493 | INFO | "pinned" : " 29 GB" | 20260507 15:15:52.493 | INFO | } | 20260507 15:15:52.493 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:15:52.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:52.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:15:52.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | PRE_HQ : 31.6%, 1.0, 16.5ms, 0 | 20260507 15:15:52.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:52.493 | INFO | Uploaded 36GiB to GPU in 1.60674s (22.4056GiB/s) | 20260507 15:15:52.493 | INFO | Downloaded 34GiB to GPU in 1.5265s (22.2732GiB/s) | 20260507 15:15:57.632 | INFO | Analyzing chunk frames = [500736,501248) | 20260507 15:15:57.632 | INFO | MemInfo: { | 20260507 15:15:57.632 | INFO | "gpu" : " 17 GB", | 20260507 15:15:57.632 | INFO | "huge" : "", | 20260507 15:15:57.632 | INFO | "huge_shared" : " 29 GB", | 20260507 15:15:57.632 | INFO | "malloc" : " 14 GB", | 20260507 15:15:57.632 | INFO | "pinned" : " 29 GB" | 20260507 15:15:57.632 | INFO | } | 20260507 15:15:57.633 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:15:57.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:15:57.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:15:57.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:15:57.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:15:57.633 | INFO | Uploaded 36GiB to GPU in 1.60939s (22.3687GiB/s) | 20260507 15:15:57.633 | INFO | Downloaded 34GiB to GPU in 1.52942s (22.2306GiB/s) | 20260507 15:16:02.772 | INFO | Analyzing chunk frames = [501248,501760) | 20260507 15:16:02.772 | INFO | MemInfo: { | 20260507 15:16:02.772 | INFO | "gpu" : " 17 GB", | 20260507 15:16:02.772 | INFO | "huge" : "", | 20260507 15:16:02.772 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:02.772 | INFO | "malloc" : " 14 GB", | 20260507 15:16:02.772 | INFO | "pinned" : " 29 GB" | 20260507 15:16:02.772 | INFO | } | 20260507 15:16:02.772 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:16:02.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:02.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:16:02.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 15:16:02.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:02.772 | INFO | Uploaded 36GiB to GPU in 1.60896s (22.3747GiB/s) | 20260507 15:16:02.772 | INFO | Downloaded 34GiB to GPU in 1.52995s (22.2229GiB/s) | 20260507 15:16:07.911 | INFO | Analyzing chunk frames = [501760,502272) | 20260507 15:16:07.911 | INFO | MemInfo: { | 20260507 15:16:07.911 | INFO | "gpu" : " 17 GB", | 20260507 15:16:07.911 | INFO | "huge" : "", | 20260507 15:16:07.911 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:07.911 | INFO | "malloc" : " 14 GB", | 20260507 15:16:07.911 | INFO | "pinned" : " 29 GB" | 20260507 15:16:07.911 | INFO | } | 20260507 15:16:07.911 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:16:07.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:07.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:16:07.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:16:07.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:07.911 | INFO | Uploaded 36GiB to GPU in 1.60647s (22.4094GiB/s) | 20260507 15:16:07.912 | INFO | Downloaded 34GiB to GPU in 1.5268s (22.2688GiB/s) | 20260507 15:16:13.050 | INFO | Analyzing chunk frames = [502272,502784) | 20260507 15:16:13.050 | INFO | MemInfo: { | 20260507 15:16:13.050 | INFO | "gpu" : " 17 GB", | 20260507 15:16:13.050 | INFO | "huge" : "", | 20260507 15:16:13.050 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:13.050 | INFO | "malloc" : " 14 GB", | 20260507 15:16:13.050 | INFO | "pinned" : " 29 GB" | 20260507 15:16:13.050 | INFO | } | 20260507 15:16:13.050 | INFO | Perf report for the last 5.13913 seconds: | 20260507 15:16:13.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:13.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.050 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:16:13.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.050 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:16:13.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:13.051 | INFO | Uploaded 36GiB to GPU in 1.60729s (22.3979GiB/s) | 20260507 15:16:13.051 | INFO | Downloaded 34GiB to GPU in 1.52653s (22.2728GiB/s) | 20260507 15:16:18.190 | INFO | Analyzing chunk frames = [502784,503296) | 20260507 15:16:18.190 | INFO | MemInfo: { | 20260507 15:16:18.190 | INFO | "gpu" : " 17 GB", | 20260507 15:16:18.190 | INFO | "huge" : "", | 20260507 15:16:18.190 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:18.190 | INFO | "malloc" : " 14 GB", | 20260507 15:16:18.190 | INFO | "pinned" : " 29 GB" | 20260507 15:16:18.190 | INFO | } | 20260507 15:16:18.190 | INFO | Perf report for the last 5.13953 seconds: | 20260507 15:16:18.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:18.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:16:18.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:16:18.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:18.190 | INFO | Uploaded 36GiB to GPU in 1.60784s (22.3903GiB/s) | 20260507 15:16:18.191 | INFO | Downloaded 34GiB to GPU in 1.52805s (22.2506GiB/s) | 20260507 15:16:23.329 | INFO | Analyzing chunk frames = [503296,503808) | 20260507 15:16:23.329 | INFO | MemInfo: { | 20260507 15:16:23.329 | INFO | "gpu" : " 17 GB", | 20260507 15:16:23.329 | INFO | "huge" : "", | 20260507 15:16:23.329 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:23.329 | INFO | "malloc" : " 14 GB", | 20260507 15:16:23.329 | INFO | "pinned" : " 29 GB" | 20260507 15:16:23.329 | INFO | } | 20260507 15:16:23.330 | INFO | Perf report for the last 5.13953 seconds: | 20260507 15:16:23.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:23.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:16:23.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:16:23.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:23.330 | INFO | Uploaded 36GiB to GPU in 1.60777s (22.3913GiB/s) | 20260507 15:16:23.330 | INFO | Downloaded 34GiB to GPU in 1.52865s (22.2419GiB/s) | 20260507 15:16:23.443 | INFO | Starting a round of Baz IO | 20260507 15:16:26.311 | INFO | Populating RTMetrics report | 20260507 15:16:26.312 | INFO | RTMetrics report populated | 20260507 15:16:26.312 | INFO | Opening tmp RTMetrics json file | 20260507 15:16:26.313 | INFO | Serializing RTMetrics to json | 20260507 15:16:26.314 | INFO | Serialization done, swapping tmp file now... | 20260507 15:16:26.315 | INFO | RTMetrics done | 20260507 15:16:26.395 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 2.952 seconds | 20260507 15:16:26.395 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.06% / 0.05% / 45.89% | 20260507 15:16:26.395 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.12801% / 9.13938e-05% / 3.17891e-05% | 20260507 15:16:26.395 | INFO | Baz IO overall: | 20260507 15:16:26.395 | INFO | BytesWritten : 201170123504 | 20260507 15:16:26.395 | INFO | NumEvents : 96733420811 | 20260507 15:16:28.468 | INFO | Analyzing chunk frames = [503808,504320) | 20260507 15:16:28.468 | INFO | MemInfo: { | 20260507 15:16:28.468 | INFO | "gpu" : " 17 GB", | 20260507 15:16:28.468 | INFO | "huge" : "", | 20260507 15:16:28.469 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:28.469 | INFO | "malloc" : " 14 GB", | 20260507 15:16:28.469 | INFO | "pinned" : " 29 GB" | 20260507 15:16:28.469 | INFO | } | 20260507 15:16:28.469 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:16:28.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:28.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:28.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:28.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:28.469 | INFO | ANALYSIS : 57.1%, 2.9, 87.5ms, 0 | 20260507 15:16:28.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:28.469 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 15:16:28.469 | INFO | PRE_HQ : 55.9%, 2.1, 61.2ms, 0 | 20260507 15:16:28.469 | INFO | BAZWRITER : 57.1%, 4.2, 124.5ms, 0 | 20260507 15:16:28.469 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:16:28.469 | INFO | Uploaded 36GiB to GPU in 1.65438s (21.7604GiB/s) | 20260507 15:16:28.469 | INFO | Downloaded 38GiB to GPU in 1.72085s (22.0822GiB/s) | 20260507 15:16:33.608 | INFO | Analyzing chunk frames = [504320,504832) | 20260507 15:16:33.608 | INFO | MemInfo: { | 20260507 15:16:33.608 | INFO | "gpu" : " 17 GB", | 20260507 15:16:33.608 | INFO | "huge" : "", | 20260507 15:16:33.608 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:33.608 | INFO | "malloc" : " 14 GB", | 20260507 15:16:33.608 | INFO | "pinned" : " 29 GB" | 20260507 15:16:33.608 | INFO | } | 20260507 15:16:33.608 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:16:33.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:33.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:16:33.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:16:33.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:33.608 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4262GiB/s) | 20260507 15:16:33.608 | INFO | Downloaded 34GiB to GPU in 1.52477s (22.2985GiB/s) | 20260507 15:16:38.747 | INFO | Analyzing chunk frames = [504832,505344) | 20260507 15:16:38.747 | INFO | MemInfo: { | 20260507 15:16:38.747 | INFO | "gpu" : " 17 GB", | 20260507 15:16:38.747 | INFO | "huge" : "", | 20260507 15:16:38.747 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:38.747 | INFO | "malloc" : " 14 GB", | 20260507 15:16:38.747 | INFO | "pinned" : " 29 GB" | 20260507 15:16:38.747 | INFO | } | 20260507 15:16:38.747 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:16:38.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:38.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.747 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:16:38.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.747 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:16:38.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:38.748 | INFO | Uploaded 36GiB to GPU in 1.60899s (22.3742GiB/s) | 20260507 15:16:38.748 | INFO | Downloaded 34GiB to GPU in 1.52927s (22.2328GiB/s) | 20260507 15:16:43.886 | INFO | Analyzing chunk frames = [505344,505856) | 20260507 15:16:43.886 | INFO | MemInfo: { | 20260507 15:16:43.886 | INFO | "gpu" : " 17 GB", | 20260507 15:16:43.886 | INFO | "huge" : "", | 20260507 15:16:43.886 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:43.886 | INFO | "malloc" : " 14 GB", | 20260507 15:16:43.886 | INFO | "pinned" : " 29 GB" | 20260507 15:16:43.887 | INFO | } | 20260507 15:16:43.887 | INFO | Perf report for the last 5.13917 seconds: | 20260507 15:16:43.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:43.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:16:43.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:16:43.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:43.887 | INFO | Uploaded 36GiB to GPU in 1.60904s (22.3736GiB/s) | 20260507 15:16:43.887 | INFO | Downloaded 34GiB to GPU in 1.52924s (22.2332GiB/s) | 20260507 15:16:49.026 | INFO | Analyzing chunk frames = [505856,506368) | 20260507 15:16:49.026 | INFO | MemInfo: { | 20260507 15:16:49.026 | INFO | "gpu" : " 17 GB", | 20260507 15:16:49.026 | INFO | "huge" : "", | 20260507 15:16:49.026 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:49.026 | INFO | "malloc" : " 14 GB", | 20260507 15:16:49.026 | INFO | "pinned" : " 29 GB" | 20260507 15:16:49.026 | INFO | } | 20260507 15:16:49.026 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:16:49.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:49.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:16:49.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:16:49.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:49.026 | INFO | Uploaded 36GiB to GPU in 1.60665s (22.4068GiB/s) | 20260507 15:16:49.026 | INFO | Downloaded 34GiB to GPU in 1.52668s (22.2706GiB/s) | 20260507 15:16:54.165 | INFO | Analyzing chunk frames = [506368,506880) | 20260507 15:16:54.165 | INFO | MemInfo: { | 20260507 15:16:54.165 | INFO | "gpu" : " 17 GB", | 20260507 15:16:54.165 | INFO | "huge" : "", | 20260507 15:16:54.165 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:54.165 | INFO | "malloc" : " 14 GB", | 20260507 15:16:54.165 | INFO | "pinned" : " 29 GB" | 20260507 15:16:54.165 | INFO | } | 20260507 15:16:54.165 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:16:54.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:54.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.165 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:16:54.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.165 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 15:16:54.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:54.166 | INFO | Uploaded 36GiB to GPU in 1.60763s (22.3932GiB/s) | 20260507 15:16:54.166 | INFO | Downloaded 34GiB to GPU in 1.52652s (22.2729GiB/s) | 20260507 15:16:59.304 | INFO | Analyzing chunk frames = [506880,507392) | 20260507 15:16:59.304 | INFO | MemInfo: { | 20260507 15:16:59.304 | INFO | "gpu" : " 17 GB", | 20260507 15:16:59.304 | INFO | "huge" : "", | 20260507 15:16:59.304 | INFO | "huge_shared" : " 29 GB", | 20260507 15:16:59.305 | INFO | "malloc" : " 14 GB", | 20260507 15:16:59.305 | INFO | "pinned" : " 29 GB" | 20260507 15:16:59.305 | INFO | } | 20260507 15:16:59.305 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:16:59.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:16:59.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:16:59.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:16:59.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:16:59.305 | INFO | Uploaded 36GiB to GPU in 1.60778s (22.3912GiB/s) | 20260507 15:16:59.305 | INFO | Downloaded 34GiB to GPU in 1.52806s (22.2505GiB/s) | 20260507 15:17:04.444 | INFO | Analyzing chunk frames = [507392,507904) | 20260507 15:17:04.444 | INFO | MemInfo: { | 20260507 15:17:04.444 | INFO | "gpu" : " 17 GB", | 20260507 15:17:04.444 | INFO | "huge" : "", | 20260507 15:17:04.444 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:04.444 | INFO | "malloc" : " 14 GB", | 20260507 15:17:04.444 | INFO | "pinned" : " 29 GB" | 20260507 15:17:04.444 | INFO | } | 20260507 15:17:04.444 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:17:04.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:04.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.444 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:17:04.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.444 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260507 15:17:04.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:04.445 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:17:04.445 | INFO | Total mem usage: 16.4 GiB | 20260507 15:17:04.445 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:17:04.445 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.91%) | 20260507 15:17:04.445 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:17:04.445 | INFO | High Water Breakdown: | 20260507 15:17:04.445 | INFO | 59.3% : 9.02 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:17:04.445 | INFO | 40.6% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:17:04.445 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:17:04.445 | INFO | Uploaded 36GiB to GPU in 1.60996s (22.3608GiB/s) | 20260507 15:17:04.445 | INFO | ------------------------------------------ | 20260507 15:17:04.445 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:17:04.445 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:17:04.445 | INFO | Total mem usage: 26.5 GiB | 20260507 15:17:04.445 | INFO | Downloaded 34GiB to GPU in 1.53062s (22.2132GiB/s) | 20260507 15:17:04.445 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:17:04.445 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:17:04.445 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:17:04.445 | INFO | High Water Breakdown: | 20260507 15:17:04.445 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:17:04.445 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:17:04.445 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:17:04.445 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:17:04.445 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:17:04.445 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:17:04.445 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:17:04.445 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:17:04.445 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:17:04.445 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:17:04.445 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:17:04.445 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:17:04.445 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:17:04.445 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:17:04.445 | INFO | ------------------------------------------ | 20260507 15:17:04.445 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:17:04.445 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:17:04.445 | INFO | Total mem usage: 27.8 GiB | 20260507 15:17:04.445 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:17:04.445 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:17:04.445 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:17:04.445 | INFO | High Water Breakdown: | 20260507 15:17:04.445 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:17:04.445 | INFO | ------------------------------------------ | 20260507 15:17:04.445 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:17:04.445 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:17:04.446 | INFO | Total mem usage: 15.7 GiB | 20260507 15:17:04.446 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:17:04.446 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:17:04.446 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:17:04.446 | INFO | High Water Breakdown: | 20260507 15:17:04.446 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:17:04.446 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:17:04.446 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:17:04.446 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:17:04.446 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:17:04.446 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:17:04.446 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:17:04.446 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:17:04.446 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:17:04.446 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:17:04.446 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:17:04.446 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:17:04.446 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:17:04.446 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:17:04.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:17:04.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:17:04.446 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:17:04.446 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:17:04.446 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:17:04.446 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:17:04.446 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:17:04.446 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:17:04.446 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:17:04.446 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:17:04.446 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:17:04.446 | INFO | ------------------------------------------ | 20260507 15:17:04.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:17:04.446 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:17:04.446 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:17:04.446 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:17:04.446 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 4.90e-03 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 11.02%, Baseline -- 3.29e+00 / -nan, 1.87e-02 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 3.07%, Histogram -- 9.18e-01 / -nan, 5.14e-02 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 16.13%, DME -- 3.86e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.72e+01 / -nan, 4.40e-02 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.15e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 3.49%, Metrics -- 1.04e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 1.29%, PulseSerializing -- 3.85e-01 / -nan, 5.66e-02 / 0.00e+00 | 20260507 15:17:04.446 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:17:04.446 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:17:04.446 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:17:04.446 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 18.81%, Upload -- 1.68e+01 / 0.00e+00, 4.55e-01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 18.10%, Download -- 1.62e+01 / 0.00e+00, 8.04e-01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 33.41%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 1.69e-04 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 29.10%, LockCompute -- 2.60e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:17:04.446 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 1.37e-02 / 0.00e+00 | 20260507 15:17:04.542 | INFO | Starting a round of Baz IO | 20260507 15:17:07.431 | INFO | Populating RTMetrics report | 20260507 15:17:07.431 | INFO | RTMetrics report populated | 20260507 15:17:07.431 | INFO | Opening tmp RTMetrics json file | 20260507 15:17:07.433 | INFO | Serializing RTMetrics to json | 20260507 15:17:07.434 | INFO | Serialization done, swapping tmp file now... | 20260507 15:17:07.435 | INFO | RTMetrics done | 20260507 15:17:07.568 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.025 seconds | 20260507 15:17:07.568 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.01% / 0.05% / 45.94% | 20260507 15:17:07.568 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.12284% / 0.000139078% / 2.78155e-05% | 20260507 15:17:07.568 | INFO | Baz IO overall: | 20260507 15:17:07.568 | INFO | BytesWritten : 203039539107 | 20260507 15:17:07.568 | INFO | NumEvents : 97643380194 | 20260507 15:17:09.583 | INFO | Analyzing chunk frames = [507904,508416) | 20260507 15:17:09.583 | INFO | MemInfo: { | 20260507 15:17:09.583 | INFO | "gpu" : " 17 GB", | 20260507 15:17:09.583 | INFO | "huge" : "", | 20260507 15:17:09.583 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:09.583 | INFO | "malloc" : " 14 GB", | 20260507 15:17:09.583 | INFO | "pinned" : " 29 GB" | 20260507 15:17:09.583 | INFO | } | 20260507 15:17:09.583 | INFO | Perf report for the last 5.13897 seconds: | 20260507 15:17:09.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:09.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:09.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:09.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:09.583 | INFO | ANALYSIS : 57.2%, 2.9, 86.8ms, 0 | 20260507 15:17:09.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:09.583 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 15:17:09.583 | INFO | PRE_HQ : 56.3%, 2.1, 61.6ms, 0 | 20260507 15:17:09.583 | INFO | BAZWRITER : 58.4%, 4.4, 132.6ms, 0 | 20260507 15:17:09.583 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:17:09.584 | INFO | Uploaded 36GiB to GPU in 1.64758s (21.8502GiB/s) | 20260507 15:17:09.584 | INFO | Downloaded 38GiB to GPU in 1.71671s (22.1354GiB/s) | 20260507 15:17:14.722 | INFO | Analyzing chunk frames = [508416,508928) | 20260507 15:17:14.722 | INFO | MemInfo: { | 20260507 15:17:14.722 | INFO | "gpu" : " 17 GB", | 20260507 15:17:14.722 | INFO | "huge" : "", | 20260507 15:17:14.722 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:14.722 | INFO | "malloc" : " 14 GB", | 20260507 15:17:14.722 | INFO | "pinned" : " 29 GB" | 20260507 15:17:14.723 | INFO | } | 20260507 15:17:14.723 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:17:14.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:14.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:17:14.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:17:14.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:14.723 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4365GiB/s) | 20260507 15:17:14.723 | INFO | Downloaded 34GiB to GPU in 1.52503s (22.2946GiB/s) | 20260507 15:17:19.862 | INFO | Analyzing chunk frames = [508928,509440) | 20260507 15:17:19.862 | INFO | MemInfo: { | 20260507 15:17:19.862 | INFO | "gpu" : " 17 GB", | 20260507 15:17:19.862 | INFO | "huge" : "", | 20260507 15:17:19.862 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:19.862 | INFO | "malloc" : " 14 GB", | 20260507 15:17:19.862 | INFO | "pinned" : " 29 GB" | 20260507 15:17:19.862 | INFO | } | 20260507 15:17:19.862 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:17:19.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:19.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:17:19.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:17:19.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:19.862 | INFO | Uploaded 36GiB to GPU in 1.60854s (22.3806GiB/s) | 20260507 15:17:19.862 | INFO | Downloaded 34GiB to GPU in 1.52925s (22.2331GiB/s) | 20260507 15:17:25.001 | INFO | Analyzing chunk frames = [509440,509952) | 20260507 15:17:25.001 | INFO | MemInfo: { | 20260507 15:17:25.001 | INFO | "gpu" : " 17 GB", | 20260507 15:17:25.001 | INFO | "huge" : "", | 20260507 15:17:25.001 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:25.001 | INFO | "malloc" : " 14 GB", | 20260507 15:17:25.001 | INFO | "pinned" : " 29 GB" | 20260507 15:17:25.001 | INFO | } | 20260507 15:17:25.001 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:17:25.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:25.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.001 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:17:25.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.001 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:17:25.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:25.002 | INFO | Uploaded 36GiB to GPU in 1.60886s (22.3761GiB/s) | 20260507 15:17:25.002 | INFO | Downloaded 34GiB to GPU in 1.52878s (22.24GiB/s) | 20260507 15:17:30.151 | INFO | Analyzing chunk frames = [509952,510464) | 20260507 15:17:30.151 | INFO | MemInfo: { | 20260507 15:17:30.151 | INFO | "gpu" : " 17 GB", | 20260507 15:17:30.151 | INFO | "huge" : "", | 20260507 15:17:30.151 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:30.151 | INFO | "malloc" : " 14 GB", | 20260507 15:17:30.151 | INFO | "pinned" : " 29 GB" | 20260507 15:17:30.151 | INFO | } | 20260507 15:17:30.151 | INFO | Perf report for the last 5.14996 seconds: | 20260507 15:17:30.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:30.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.151 | INFO | ANALYSIS : 55.8%, 3.0, 87.3ms, 0 | 20260507 15:17:30.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.151 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:17:30.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:30.152 | INFO | Uploaded 36GiB to GPU in 1.60724s (22.3986GiB/s) | 20260507 15:17:30.152 | INFO | Downloaded 34GiB to GPU in 1.5272s (22.263GiB/s) | 20260507 15:17:35.281 | INFO | Analyzing chunk frames = [510464,510976) | 20260507 15:17:35.281 | INFO | MemInfo: { | 20260507 15:17:35.281 | INFO | "gpu" : " 17 GB", | 20260507 15:17:35.281 | INFO | "huge" : "", | 20260507 15:17:35.281 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:35.281 | INFO | "malloc" : " 14 GB", | 20260507 15:17:35.281 | INFO | "pinned" : " 29 GB" | 20260507 15:17:35.281 | INFO | } | 20260507 15:17:35.281 | INFO | Perf report for the last 5.12974 seconds: | 20260507 15:17:35.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:35.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 15:17:35.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:17:35.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:35.281 | INFO | Uploaded 36GiB to GPU in 1.60746s (22.3956GiB/s) | 20260507 15:17:35.281 | INFO | Downloaded 34GiB to GPU in 1.52666s (22.2708GiB/s) | 20260507 15:17:40.419 | INFO | Analyzing chunk frames = [510976,511488) | 20260507 15:17:40.419 | INFO | MemInfo: { | 20260507 15:17:40.419 | INFO | "gpu" : " 17 GB", | 20260507 15:17:40.419 | INFO | "huge" : "", | 20260507 15:17:40.419 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:40.419 | INFO | "malloc" : " 14 GB", | 20260507 15:17:40.419 | INFO | "pinned" : " 29 GB" | 20260507 15:17:40.419 | INFO | } | 20260507 15:17:40.419 | INFO | Perf report for the last 5.13809 seconds: | 20260507 15:17:40.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:40.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:17:40.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:17:40.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:40.420 | INFO | Uploaded 36GiB to GPU in 1.60846s (22.3816GiB/s) | 20260507 15:17:40.420 | INFO | Downloaded 34GiB to GPU in 1.5281s (22.2499GiB/s) | 20260507 15:17:45.559 | INFO | Analyzing chunk frames = [511488,512000) | 20260507 15:17:45.559 | INFO | MemInfo: { | 20260507 15:17:45.559 | INFO | "gpu" : " 17 GB", | 20260507 15:17:45.559 | INFO | "huge" : "", | 20260507 15:17:45.559 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:45.559 | INFO | "malloc" : " 14 GB", | 20260507 15:17:45.559 | INFO | "pinned" : " 29 GB" | 20260507 15:17:45.559 | INFO | } | 20260507 15:17:45.559 | INFO | Perf report for the last 5.13969 seconds: | 20260507 15:17:45.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:45.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:17:45.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 15:17:45.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:45.559 | INFO | Uploaded 36GiB to GPU in 1.60732s (22.3975GiB/s) | 20260507 15:17:45.559 | INFO | Downloaded 34GiB to GPU in 1.52787s (22.2532GiB/s) | 20260507 15:17:45.656 | INFO | Starting a round of Baz IO | 20260507 15:17:48.538 | INFO | Populating RTMetrics report | 20260507 15:17:48.538 | INFO | RTMetrics report populated | 20260507 15:17:48.538 | INFO | Opening tmp RTMetrics json file | 20260507 15:17:48.540 | INFO | Serializing RTMetrics to json | 20260507 15:17:48.543 | INFO | Serialization done, swapping tmp file now... | 20260507 15:17:48.544 | INFO | RTMetrics done | 20260507 15:17:48.636 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 2.979 seconds | 20260507 15:17:48.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.96% / 0.05% / 45.99% | 20260507 15:17:48.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.09198% / 0.000103315% / 2.78155e-05% | 20260507 15:17:48.636 | INFO | Baz IO overall: | 20260507 15:17:48.636 | INFO | BytesWritten : 204908506372 | 20260507 15:17:48.636 | INFO | NumEvents : 98553353278 | 20260507 15:17:50.698 | INFO | Analyzing chunk frames = [512000,512512) | 20260507 15:17:50.698 | INFO | MemInfo: { | 20260507 15:17:50.698 | INFO | "gpu" : " 17 GB", | 20260507 15:17:50.698 | INFO | "huge" : "", | 20260507 15:17:50.698 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:50.698 | INFO | "malloc" : " 14 GB", | 20260507 15:17:50.698 | INFO | "pinned" : " 29 GB" | 20260507 15:17:50.698 | INFO | } | 20260507 15:17:50.698 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:17:50.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:50.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:50.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:50.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:50.698 | INFO | ANALYSIS : 57.1%, 2.9, 86.4ms, 0 | 20260507 15:17:50.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:50.698 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 15:17:50.698 | INFO | PRE_HQ : 55.8%, 2.1, 59.9ms, 0 | 20260507 15:17:50.698 | INFO | BAZWRITER : 57.6%, 4.6, 134.8ms, 0 | 20260507 15:17:50.698 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 15:17:50.699 | INFO | Uploaded 36GiB to GPU in 1.66162s (21.6656GiB/s) | 20260507 15:17:50.699 | INFO | Downloaded 38GiB to GPU in 1.73004s (21.9648GiB/s) | 20260507 15:17:55.837 | INFO | Analyzing chunk frames = [512512,513024) | 20260507 15:17:55.837 | INFO | MemInfo: { | 20260507 15:17:55.837 | INFO | "gpu" : " 17 GB", | 20260507 15:17:55.837 | INFO | "huge" : "", | 20260507 15:17:55.837 | INFO | "huge_shared" : " 29 GB", | 20260507 15:17:55.837 | INFO | "malloc" : " 14 GB", | 20260507 15:17:55.837 | INFO | "pinned" : " 29 GB" | 20260507 15:17:55.837 | INFO | } | 20260507 15:17:55.837 | INFO | Perf report for the last 5.13908 seconds: | 20260507 15:17:55.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:17:55.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.837 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:17:55.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.838 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:17:55.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:17:55.838 | INFO | Uploaded 36GiB to GPU in 1.60478s (22.433GiB/s) | 20260507 15:17:55.838 | INFO | Downloaded 34GiB to GPU in 1.52461s (22.3008GiB/s) | 20260507 15:18:00.977 | INFO | Analyzing chunk frames = [513024,513536) | 20260507 15:18:00.977 | INFO | MemInfo: { | 20260507 15:18:00.977 | INFO | "gpu" : " 17 GB", | 20260507 15:18:00.977 | INFO | "huge" : "", | 20260507 15:18:00.977 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:00.977 | INFO | "malloc" : " 14 GB", | 20260507 15:18:00.977 | INFO | "pinned" : " 29 GB" | 20260507 15:18:00.977 | INFO | } | 20260507 15:18:00.977 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:18:00.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:00.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:18:00.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:18:00.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:00.977 | INFO | Uploaded 36GiB to GPU in 1.6094s (22.3686GiB/s) | 20260507 15:18:00.977 | INFO | Downloaded 34GiB to GPU in 1.52954s (22.2288GiB/s) | 20260507 15:18:06.116 | INFO | Analyzing chunk frames = [513536,514048) | 20260507 15:18:06.116 | INFO | MemInfo: { | 20260507 15:18:06.116 | INFO | "gpu" : " 17 GB", | 20260507 15:18:06.116 | INFO | "huge" : "", | 20260507 15:18:06.116 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:06.116 | INFO | "malloc" : " 14 GB", | 20260507 15:18:06.116 | INFO | "pinned" : " 29 GB" | 20260507 15:18:06.116 | INFO | } | 20260507 15:18:06.116 | INFO | Perf report for the last 5.13956 seconds: | 20260507 15:18:06.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:06.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:18:06.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:18:06.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:06.116 | INFO | Uploaded 36GiB to GPU in 1.60884s (22.3764GiB/s) | 20260507 15:18:06.116 | INFO | Downloaded 34GiB to GPU in 1.52852s (22.2437GiB/s) | 20260507 15:18:11.255 | INFO | Analyzing chunk frames = [514048,514560) | 20260507 15:18:11.255 | INFO | MemInfo: { | 20260507 15:18:11.255 | INFO | "gpu" : " 17 GB", | 20260507 15:18:11.255 | INFO | "huge" : "", | 20260507 15:18:11.255 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:11.255 | INFO | "malloc" : " 14 GB", | 20260507 15:18:11.255 | INFO | "pinned" : " 29 GB" | 20260507 15:18:11.255 | INFO | } | 20260507 15:18:11.255 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:18:11.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:11.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.256 | INFO | ANALYSIS : 55.9%, 3.0, 87.3ms, 0 | 20260507 15:18:11.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.256 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:18:11.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:11.256 | INFO | Uploaded 36GiB to GPU in 1.60786s (22.39GiB/s) | 20260507 15:18:11.256 | INFO | Downloaded 34GiB to GPU in 1.52693s (22.2669GiB/s) | 20260507 15:18:16.395 | INFO | Analyzing chunk frames = [514560,515072) | 20260507 15:18:16.395 | INFO | MemInfo: { | 20260507 15:18:16.395 | INFO | "gpu" : " 17 GB", | 20260507 15:18:16.395 | INFO | "huge" : "", | 20260507 15:18:16.395 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:16.395 | INFO | "malloc" : " 14 GB", | 20260507 15:18:16.395 | INFO | "pinned" : " 29 GB" | 20260507 15:18:16.395 | INFO | } | 20260507 15:18:16.395 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:18:16.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:16.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:18:16.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:18:16.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:16.395 | INFO | Uploaded 36GiB to GPU in 1.60854s (22.3806GiB/s) | 20260507 15:18:16.395 | INFO | Downloaded 34GiB to GPU in 1.52766s (22.2563GiB/s) | 20260507 15:18:21.534 | INFO | Analyzing chunk frames = [515072,515584) | 20260507 15:18:21.534 | INFO | MemInfo: { | 20260507 15:18:21.534 | INFO | "gpu" : " 17 GB", | 20260507 15:18:21.534 | INFO | "huge" : "", | 20260507 15:18:21.534 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:21.534 | INFO | "malloc" : " 14 GB", | 20260507 15:18:21.534 | INFO | "pinned" : " 29 GB" | 20260507 15:18:21.534 | INFO | } | 20260507 15:18:21.534 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:18:21.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:21.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:18:21.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:18:21.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:21.534 | INFO | Uploaded 36GiB to GPU in 1.60712s (22.4004GiB/s) | 20260507 15:18:21.534 | INFO | Downloaded 34GiB to GPU in 1.52786s (22.2534GiB/s) | 20260507 15:18:26.673 | INFO | Analyzing chunk frames = [515584,516096) | 20260507 15:18:26.673 | INFO | MemInfo: { | 20260507 15:18:26.673 | INFO | "gpu" : " 17 GB", | 20260507 15:18:26.673 | INFO | "huge" : "", | 20260507 15:18:26.674 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:26.674 | INFO | "malloc" : " 14 GB", | 20260507 15:18:26.674 | INFO | "pinned" : " 29 GB" | 20260507 15:18:26.674 | INFO | } | 20260507 15:18:26.674 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:18:26.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:26.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:18:26.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 15:18:26.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:26.674 | INFO | Uploaded 36GiB to GPU in 1.60844s (22.3819GiB/s) | 20260507 15:18:26.674 | INFO | Downloaded 34GiB to GPU in 1.52918s (22.2342GiB/s) | 20260507 15:18:26.674 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:18:26.674 | INFO | Total mem usage: 16.4 GiB | 20260507 15:18:26.674 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:18:26.674 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.4%) | 20260507 15:18:26.674 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:18:26.674 | INFO | High Water Breakdown: | 20260507 15:18:26.674 | INFO | 59% : 8.76 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:18:26.674 | INFO | 40.9% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:18:26.674 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:18:26.674 | INFO | ------------------------------------------ | 20260507 15:18:26.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:18:26.674 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:18:26.674 | INFO | Total mem usage: 26.5 GiB | 20260507 15:18:26.674 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:18:26.674 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:18:26.674 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:18:26.674 | INFO | High Water Breakdown: | 20260507 15:18:26.674 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:18:26.674 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:18:26.674 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:18:26.674 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:18:26.674 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:18:26.674 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:18:26.674 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:18:26.674 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:18:26.674 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:18:26.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:18:26.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:18:26.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:18:26.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:18:26.674 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:18:26.674 | INFO | ------------------------------------------ | 20260507 15:18:26.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:18:26.674 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:18:26.674 | INFO | Total mem usage: 27.8 GiB | 20260507 15:18:26.674 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:18:26.674 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:18:26.674 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:18:26.674 | INFO | High Water Breakdown: | 20260507 15:18:26.674 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:18:26.674 | INFO | ------------------------------------------ | 20260507 15:18:26.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:18:26.674 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:18:26.674 | INFO | Total mem usage: 15.7 GiB | 20260507 15:18:26.674 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:18:26.674 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:18:26.674 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:18:26.674 | INFO | High Water Breakdown: | 20260507 15:18:26.674 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:18:26.674 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:18:26.674 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:18:26.674 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:18:26.674 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:18:26.674 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:18:26.674 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:18:26.674 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:18:26.674 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:18:26.674 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:18:26.674 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:18:26.674 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:18:26.674 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:18:26.674 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:18:26.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:18:26.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:18:26.674 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:18:26.674 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:18:26.674 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:18:26.674 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:18:26.674 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:18:26.674 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:18:26.674 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:18:26.674 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:18:26.674 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:18:26.674 | INFO | ------------------------------------------ | 20260507 15:18:26.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:18:26.674 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:18:26.674 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:18:26.674 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:18:26.675 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.12e-03 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 11.03%, Baseline -- 3.29e+00 / -nan, 1.94e-02 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 3.07%, Histogram -- 9.18e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.01e+00 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.15e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 3.49%, Metrics -- 1.04e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.83e-01 / -nan, 5.55e-02 / 0.00e+00 | 20260507 15:18:26.675 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:18:26.675 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:18:26.675 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:18:26.675 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 18.80%, Upload -- 1.68e+01 / 0.00e+00, 4.36e-01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 18.08%, Download -- 1.62e+01 / 0.00e+00, 7.73e-01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 33.40%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 0.57%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 0.00%, LockDownload -- 4.31e-04 / 0.00e+00, 1.69e-04 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 29.14%, LockCompute -- 2.61e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:18:26.675 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 15:18:26.771 | INFO | Starting a round of Baz IO | 20260507 15:18:29.652 | INFO | Populating RTMetrics report | 20260507 15:18:29.652 | INFO | RTMetrics report populated | 20260507 15:18:29.652 | INFO | Opening tmp RTMetrics json file | 20260507 15:18:29.655 | INFO | Serializing RTMetrics to json | 20260507 15:18:29.661 | INFO | Serialization done, swapping tmp file now... | 20260507 15:18:29.679 | INFO | RTMetrics done | 20260507 15:18:29.737 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 2.966 seconds | 20260507 15:18:29.737 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.91% / 0.05% / 46.04% | 20260507 15:18:29.737 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.06865% / 0.000111262% / 2.38419e-05% | 20260507 15:18:29.737 | INFO | Baz IO overall: | 20260507 15:18:29.737 | INFO | BytesWritten : 206777720933 | 20260507 15:18:29.737 | INFO | NumEvents : 99463421819 | 20260507 15:18:31.813 | INFO | Analyzing chunk frames = [516096,516608) | 20260507 15:18:31.813 | INFO | MemInfo: { | 20260507 15:18:31.813 | INFO | "gpu" : " 17 GB", | 20260507 15:18:31.813 | INFO | "huge" : "", | 20260507 15:18:31.813 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:31.813 | INFO | "malloc" : " 14 GB", | 20260507 15:18:31.813 | INFO | "pinned" : " 29 GB" | 20260507 15:18:31.813 | INFO | } | 20260507 15:18:31.813 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:18:31.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:31.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:31.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:31.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:31.813 | INFO | ANALYSIS : 57.1%, 2.9, 87.1ms, 0 | 20260507 15:18:31.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:31.813 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 15:18:31.813 | INFO | PRE_HQ : 55.8%, 2.1, 61.0ms, 0 | 20260507 15:18:31.813 | INFO | BAZWRITER : 57.3%, 4.3, 127.4ms, 0 | 20260507 15:18:31.813 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 15:18:31.813 | INFO | Uploaded 36GiB to GPU in 1.65642s (21.7336GiB/s) | 20260507 15:18:31.814 | INFO | Downloaded 38GiB to GPU in 1.7234s (22.0495GiB/s) | 20260507 15:18:36.952 | INFO | Analyzing chunk frames = [516608,517120) | 20260507 15:18:36.952 | INFO | MemInfo: { | 20260507 15:18:36.952 | INFO | "gpu" : " 17 GB", | 20260507 15:18:36.952 | INFO | "huge" : "", | 20260507 15:18:36.952 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:36.952 | INFO | "malloc" : " 14 GB", | 20260507 15:18:36.952 | INFO | "pinned" : " 29 GB" | 20260507 15:18:36.952 | INFO | } | 20260507 15:18:36.952 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:18:36.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:36.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.952 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:18:36.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.953 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 15:18:36.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:36.953 | INFO | Uploaded 36GiB to GPU in 1.60555s (22.4223GiB/s) | 20260507 15:18:36.953 | INFO | Downloaded 34GiB to GPU in 1.52521s (22.292GiB/s) | 20260507 15:18:42.092 | INFO | Analyzing chunk frames = [517120,517632) | 20260507 15:18:42.092 | INFO | MemInfo: { | 20260507 15:18:42.092 | INFO | "gpu" : " 17 GB", | 20260507 15:18:42.092 | INFO | "huge" : "", | 20260507 15:18:42.092 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:42.092 | INFO | "malloc" : " 14 GB", | 20260507 15:18:42.092 | INFO | "pinned" : " 29 GB" | 20260507 15:18:42.092 | INFO | } | 20260507 15:18:42.092 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:18:42.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:42.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:18:42.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:18:42.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:42.092 | INFO | Uploaded 36GiB to GPU in 1.60945s (22.3679GiB/s) | 20260507 15:18:42.092 | INFO | Downloaded 34GiB to GPU in 1.52947s (22.2299GiB/s) | 20260507 15:18:47.231 | INFO | Analyzing chunk frames = [517632,518144) | 20260507 15:18:47.231 | INFO | MemInfo: { | 20260507 15:18:47.231 | INFO | "gpu" : " 17 GB", | 20260507 15:18:47.231 | INFO | "huge" : "", | 20260507 15:18:47.231 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:47.231 | INFO | "malloc" : " 14 GB", | 20260507 15:18:47.231 | INFO | "pinned" : " 29 GB" | 20260507 15:18:47.231 | INFO | } | 20260507 15:18:47.231 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:18:47.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:47.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.231 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:18:47.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.231 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:18:47.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:47.232 | INFO | Uploaded 36GiB to GPU in 1.60947s (22.3676GiB/s) | 20260507 15:18:47.232 | INFO | Downloaded 34GiB to GPU in 1.52882s (22.2393GiB/s) | 20260507 15:18:52.370 | INFO | Analyzing chunk frames = [518144,518656) | 20260507 15:18:52.370 | INFO | MemInfo: { | 20260507 15:18:52.370 | INFO | "gpu" : " 17 GB", | 20260507 15:18:52.370 | INFO | "huge" : "", | 20260507 15:18:52.370 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:52.370 | INFO | "malloc" : " 14 GB", | 20260507 15:18:52.370 | INFO | "pinned" : " 29 GB" | 20260507 15:18:52.370 | INFO | } | 20260507 15:18:52.370 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:18:52.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:52.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.370 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:18:52.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.370 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:18:52.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:52.371 | INFO | Uploaded 36GiB to GPU in 1.60791s (22.3893GiB/s) | 20260507 15:18:52.371 | INFO | Downloaded 34GiB to GPU in 1.52718s (22.2633GiB/s) | 20260507 15:18:57.509 | INFO | Analyzing chunk frames = [518656,519168) | 20260507 15:18:57.509 | INFO | MemInfo: { | 20260507 15:18:57.509 | INFO | "gpu" : " 17 GB", | 20260507 15:18:57.509 | INFO | "huge" : "", | 20260507 15:18:57.509 | INFO | "huge_shared" : " 29 GB", | 20260507 15:18:57.510 | INFO | "malloc" : " 14 GB", | 20260507 15:18:57.510 | INFO | "pinned" : " 29 GB" | 20260507 15:18:57.510 | INFO | } | 20260507 15:18:57.510 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:18:57.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:18:57.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:18:57.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:18:57.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:18:57.510 | INFO | Uploaded 36GiB to GPU in 1.60758s (22.3939GiB/s) | 20260507 15:18:57.510 | INFO | Downloaded 34GiB to GPU in 1.52646s (22.2738GiB/s) | 20260507 15:19:02.649 | INFO | Analyzing chunk frames = [519168,519680) | 20260507 15:19:02.649 | INFO | MemInfo: { | 20260507 15:19:02.649 | INFO | "gpu" : " 17 GB", | 20260507 15:19:02.649 | INFO | "huge" : "", | 20260507 15:19:02.649 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:02.649 | INFO | "malloc" : " 14 GB", | 20260507 15:19:02.649 | INFO | "pinned" : " 29 GB" | 20260507 15:19:02.649 | INFO | } | 20260507 15:19:02.649 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:19:02.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:02.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.649 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:19:02.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.649 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 15:19:02.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:02.650 | INFO | Uploaded 36GiB to GPU in 1.60886s (22.376GiB/s) | 20260507 15:19:02.650 | INFO | Downloaded 34GiB to GPU in 1.52926s (22.233GiB/s) | 20260507 15:19:07.788 | INFO | Analyzing chunk frames = [519680,520192) | 20260507 15:19:07.788 | INFO | MemInfo: { | 20260507 15:19:07.788 | INFO | "gpu" : " 17 GB", | 20260507 15:19:07.788 | INFO | "huge" : "", | 20260507 15:19:07.788 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:07.788 | INFO | "malloc" : " 14 GB", | 20260507 15:19:07.788 | INFO | "pinned" : " 29 GB" | 20260507 15:19:07.788 | INFO | } | 20260507 15:19:07.788 | INFO | Perf report for the last 5.13902 seconds: | 20260507 15:19:07.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:07.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.788 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:19:07.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.788 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260507 15:19:07.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:07.789 | INFO | Uploaded 36GiB to GPU in 1.61016s (22.358GiB/s) | 20260507 15:19:07.789 | INFO | Downloaded 34GiB to GPU in 1.53068s (22.2124GiB/s) | 20260507 15:19:07.885 | INFO | Starting a round of Baz IO | 20260507 15:19:10.767 | INFO | Populating RTMetrics report | 20260507 15:19:10.768 | INFO | RTMetrics report populated | 20260507 15:19:10.768 | INFO | Opening tmp RTMetrics json file | 20260507 15:19:10.852 | INFO | Serializing RTMetrics to json | 20260507 15:19:10.858 | INFO | Serialization done, swapping tmp file now... | 20260507 15:19:10.860 | INFO | RTMetrics done | 20260507 15:19:11.016 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.13 seconds | 20260507 15:19:11.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.86% / 0.05% / 46.09% | 20260507 15:19:11.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.0514% / 9.13938e-05% / 3.17891e-05% | 20260507 15:19:11.016 | INFO | Baz IO overall: | 20260507 15:19:11.016 | INFO | BytesWritten : 208649330901 | 20260507 15:19:11.016 | INFO | NumEvents : 100374657761 | 20260507 15:19:12.928 | INFO | Analyzing chunk frames = [520192,520704) | 20260507 15:19:12.928 | INFO | MemInfo: { | 20260507 15:19:12.928 | INFO | "gpu" : " 17 GB", | 20260507 15:19:12.928 | INFO | "huge" : "", | 20260507 15:19:12.928 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:12.928 | INFO | "malloc" : " 14 GB", | 20260507 15:19:12.928 | INFO | "pinned" : " 29 GB" | 20260507 15:19:12.928 | INFO | } | 20260507 15:19:12.928 | INFO | Perf report for the last 5.13976 seconds: | 20260507 15:19:12.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:12.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:12.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:12.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:12.928 | INFO | ANALYSIS : 57.1%, 2.9, 88.2ms, 0 | 20260507 15:19:12.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:12.928 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 15:19:12.928 | INFO | PRE_HQ : 55.9%, 2.1, 60.5ms, 0 | 20260507 15:19:12.928 | INFO | BAZWRITER : 60.5%, 4.2, 131.6ms, 0 | 20260507 15:19:12.929 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 15:19:12.929 | INFO | Uploaded 36GiB to GPU in 1.6445s (21.8911GiB/s) | 20260507 15:19:12.929 | INFO | Downloaded 38GiB to GPU in 1.71071s (22.2131GiB/s) | 20260507 15:19:18.067 | INFO | Analyzing chunk frames = [520704,521216) | 20260507 15:19:18.067 | INFO | MemInfo: { | 20260507 15:19:18.067 | INFO | "gpu" : " 17 GB", | 20260507 15:19:18.067 | INFO | "huge" : "", | 20260507 15:19:18.067 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:18.067 | INFO | "malloc" : " 14 GB", | 20260507 15:19:18.067 | INFO | "pinned" : " 29 GB" | 20260507 15:19:18.067 | INFO | } | 20260507 15:19:18.067 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:19:18.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:18.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 15:19:18.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | PRE_HQ : 39.4%, 1.0, 20.5ms, 0 | 20260507 15:19:18.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:18.068 | INFO | Uploaded 36GiB to GPU in 1.61096s (22.3469GiB/s) | 20260507 15:19:18.069 | INFO | Downloaded 34GiB to GPU in 1.53059s (22.2136GiB/s) | 20260507 15:19:23.206 | INFO | Analyzing chunk frames = [521216,521728) | 20260507 15:19:23.206 | INFO | MemInfo: { | 20260507 15:19:23.206 | INFO | "gpu" : " 17 GB", | 20260507 15:19:23.206 | INFO | "huge" : "", | 20260507 15:19:23.207 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:23.207 | INFO | "malloc" : " 14 GB", | 20260507 15:19:23.207 | INFO | "pinned" : " 29 GB" | 20260507 15:19:23.207 | INFO | } | 20260507 15:19:23.207 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:19:23.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:23.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 15:19:23.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | PRE_HQ : 39.7%, 1.0, 20.6ms, 0 | 20260507 15:19:23.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:23.207 | INFO | Uploaded 36GiB to GPU in 1.61465s (22.2959GiB/s) | 20260507 15:19:23.208 | INFO | Downloaded 34GiB to GPU in 1.53479s (22.1529GiB/s) | 20260507 15:19:28.346 | INFO | Analyzing chunk frames = [521728,522240) | 20260507 15:19:28.346 | INFO | MemInfo: { | 20260507 15:19:28.346 | INFO | "gpu" : " 17 GB", | 20260507 15:19:28.346 | INFO | "huge" : "", | 20260507 15:19:28.346 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:28.346 | INFO | "malloc" : " 14 GB", | 20260507 15:19:28.346 | INFO | "pinned" : " 29 GB" | 20260507 15:19:28.346 | INFO | } | 20260507 15:19:28.346 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:19:28.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:28.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.346 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 15:19:28.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.346 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:19:28.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:28.347 | INFO | Uploaded 36GiB to GPU in 1.61528s (22.2871GiB/s) | 20260507 15:19:28.347 | INFO | Downloaded 34GiB to GPU in 1.53479s (22.1529GiB/s) | 20260507 15:19:33.485 | INFO | Analyzing chunk frames = [522240,522752) | 20260507 15:19:33.485 | INFO | MemInfo: { | 20260507 15:19:33.485 | INFO | "gpu" : " 17 GB", | 20260507 15:19:33.485 | INFO | "huge" : "", | 20260507 15:19:33.485 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:33.485 | INFO | "malloc" : " 14 GB", | 20260507 15:19:33.485 | INFO | "pinned" : " 29 GB" | 20260507 15:19:33.485 | INFO | } | 20260507 15:19:33.486 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:19:33.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:33.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:19:33.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 15:19:33.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:33.486 | INFO | Uploaded 36GiB to GPU in 1.61117s (22.344GiB/s) | 20260507 15:19:33.486 | INFO | Downloaded 34GiB to GPU in 1.53213s (22.1914GiB/s) | 20260507 15:19:38.624 | INFO | Analyzing chunk frames = [522752,523264) | 20260507 15:19:38.624 | INFO | MemInfo: { | 20260507 15:19:38.624 | INFO | "gpu" : " 17 GB", | 20260507 15:19:38.624 | INFO | "huge" : "", | 20260507 15:19:38.625 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:38.625 | INFO | "malloc" : " 14 GB", | 20260507 15:19:38.625 | INFO | "pinned" : " 29 GB" | 20260507 15:19:38.625 | INFO | } | 20260507 15:19:38.625 | INFO | Perf report for the last 5.13902 seconds: | 20260507 15:19:38.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:38.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:19:38.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 15:19:38.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:38.625 | INFO | Uploaded 36GiB to GPU in 1.61366s (22.3095GiB/s) | 20260507 15:19:38.626 | INFO | Downloaded 34GiB to GPU in 1.53189s (22.1947GiB/s) | 20260507 15:19:43.764 | INFO | Analyzing chunk frames = [523264,523776) | 20260507 15:19:43.764 | INFO | MemInfo: { | 20260507 15:19:43.764 | INFO | "gpu" : " 17 GB", | 20260507 15:19:43.764 | INFO | "huge" : "", | 20260507 15:19:43.764 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:43.764 | INFO | "malloc" : " 14 GB", | 20260507 15:19:43.764 | INFO | "pinned" : " 29 GB" | 20260507 15:19:43.764 | INFO | } | 20260507 15:19:43.764 | INFO | Perf report for the last 5.13952 seconds: | 20260507 15:19:43.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:43.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.764 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 15:19:43.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.764 | INFO | PRE_HQ : 40.3%, 1.0, 21.0ms, 0 | 20260507 15:19:43.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:43.765 | INFO | Uploaded 36GiB to GPU in 1.61257s (22.3246GiB/s) | 20260507 15:19:43.765 | INFO | Downloaded 34GiB to GPU in 1.53336s (22.1735GiB/s) | 20260507 15:19:48.903 | INFO | Analyzing chunk frames = [523776,524288) | 20260507 15:19:48.903 | INFO | MemInfo: { | 20260507 15:19:48.903 | INFO | "gpu" : " 17 GB", | 20260507 15:19:48.903 | INFO | "huge" : "", | 20260507 15:19:48.903 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:48.903 | INFO | "malloc" : " 14 GB", | 20260507 15:19:48.903 | INFO | "pinned" : " 29 GB" | 20260507 15:19:48.903 | INFO | } | 20260507 15:19:48.904 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:19:48.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:48.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 15:19:48.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | PRE_HQ : 41.2%, 1.0, 21.4ms, 0 | 20260507 15:19:48.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:48.904 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:19:48.904 | INFO | Total mem usage: 16.4 GiB | 20260507 15:19:48.904 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:19:48.904 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.5%) | 20260507 15:19:48.904 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:19:48.904 | INFO | High Water Breakdown: | 20260507 15:19:48.904 | INFO | 59% : 8.76 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:19:48.904 | INFO | 40.9% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:19:48.904 | INFO | Uploaded 36GiB to GPU in 1.61494s (22.2919GiB/s) | 20260507 15:19:48.904 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:19:48.904 | INFO | Downloaded 34GiB to GPU in 1.53364s (22.1694GiB/s) | 20260507 15:19:48.904 | INFO | ------------------------------------------ | 20260507 15:19:48.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:19:48.905 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:19:48.905 | INFO | Total mem usage: 26.5 GiB | 20260507 15:19:48.905 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:19:48.905 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:19:48.905 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:19:48.905 | INFO | High Water Breakdown: | 20260507 15:19:48.905 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:19:48.905 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:19:48.905 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:19:48.905 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:19:48.905 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:19:48.905 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:19:48.905 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:19:48.905 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:19:48.905 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:19:48.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:19:48.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:19:48.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:19:48.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:19:48.905 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:19:48.905 | INFO | ------------------------------------------ | 20260507 15:19:48.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:19:48.905 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:19:48.905 | INFO | Total mem usage: 27.8 GiB | 20260507 15:19:48.905 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:19:48.905 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:19:48.905 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:19:48.905 | INFO | High Water Breakdown: | 20260507 15:19:48.905 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:19:48.905 | INFO | ------------------------------------------ | 20260507 15:19:48.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:19:48.905 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:19:48.905 | INFO | Total mem usage: 15.7 GiB | 20260507 15:19:48.905 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:19:48.905 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:19:48.905 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:19:48.905 | INFO | High Water Breakdown: | 20260507 15:19:48.905 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:19:48.905 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:19:48.905 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:19:48.905 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:19:48.905 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:19:48.905 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:19:48.905 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:19:48.905 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:19:48.905 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:19:48.905 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:19:48.905 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:19:48.905 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:19:48.905 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:19:48.905 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:19:48.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:19:48.905 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:19:48.905 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:19:48.905 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:19:48.905 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:19:48.905 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:19:48.905 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:19:48.905 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:19:48.905 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:19:48.905 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:19:48.905 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:19:48.905 | INFO | ------------------------------------------ | 20260507 15:19:48.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:19:48.906 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:19:48.906 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:19:48.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:19:48.906 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 0.30%, AggregateStats -- 8.99e-02 / -nan, 5.94e-03 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 11.05%, Baseline -- 3.30e+00 / -nan, 2.31e-02 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 16.11%, DME -- 3.85e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.16e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 3.50%, Metrics -- 1.05e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 1.30%, PulseSerializing -- 3.89e-01 / -nan, 5.64e-02 / 0.00e+00 | 20260507 15:19:48.906 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:19:48.906 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:19:48.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:19:48.906 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 18.77%, Upload -- 1.68e+01 / 0.00e+00, 4.18e-01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 18.05%, Download -- 1.62e+01 / 0.00e+00, 7.33e-01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 33.35%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 0.00%, LockDownload -- 5.48e-04 / 0.00e+00, 3.77e-04 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 29.25%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:19:48.906 | INFO | PerfReport: 0.02%, Misc -- 1.71e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260507 15:19:49.018 | INFO | Starting a round of Baz IO | 20260507 15:19:51.912 | INFO | Populating RTMetrics report | 20260507 15:19:51.913 | INFO | RTMetrics report populated | 20260507 15:19:51.913 | INFO | Opening tmp RTMetrics json file | 20260507 15:19:51.922 | INFO | Serializing RTMetrics to json | 20260507 15:19:51.925 | INFO | Serialization done, swapping tmp file now... | 20260507 15:19:51.930 | INFO | RTMetrics done | 20260507 15:19:52.051 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.033 seconds | 20260507 15:19:52.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.81% / 0.05% / 46.14% | 20260507 15:19:52.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.03955% / 9.53674e-05% / 2.38419e-05% | 20260507 15:19:52.051 | INFO | Baz IO overall: | 20260507 15:19:52.051 | INFO | BytesWritten : 210521907398 | 20260507 15:19:52.051 | INFO | NumEvents : 101286522689 | 20260507 15:19:54.042 | INFO | Analyzing chunk frames = [524288,524800) | 20260507 15:19:54.043 | INFO | MemInfo: { | 20260507 15:19:54.043 | INFO | "gpu" : " 17 GB", | 20260507 15:19:54.043 | INFO | "huge" : "", | 20260507 15:19:54.043 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:54.043 | INFO | "malloc" : " 14 GB", | 20260507 15:19:54.043 | INFO | "pinned" : " 29 GB" | 20260507 15:19:54.043 | INFO | } | 20260507 15:19:54.043 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:19:54.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:54.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:54.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:54.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:54.043 | INFO | ANALYSIS : 57.6%, 2.9, 86.3ms, 0 | 20260507 15:19:54.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:54.043 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 15:19:54.043 | INFO | PRE_HQ : 56.7%, 2.2, 64.4ms, 0 | 20260507 15:19:54.043 | INFO | BAZWRITER : 58.6%, 4.4, 133.3ms, 0 | 20260507 15:19:54.043 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:19:54.043 | INFO | Uploaded 36GiB to GPU in 1.6453s (21.8805GiB/s) | 20260507 15:19:54.044 | INFO | Downloaded 38GiB to GPU in 1.7253s (22.0251GiB/s) | 20260507 15:19:59.182 | INFO | Analyzing chunk frames = [524800,525312) | 20260507 15:19:59.182 | INFO | MemInfo: { | 20260507 15:19:59.182 | INFO | "gpu" : " 17 GB", | 20260507 15:19:59.182 | INFO | "huge" : "", | 20260507 15:19:59.182 | INFO | "huge_shared" : " 29 GB", | 20260507 15:19:59.182 | INFO | "malloc" : " 14 GB", | 20260507 15:19:59.182 | INFO | "pinned" : " 29 GB" | 20260507 15:19:59.182 | INFO | } | 20260507 15:19:59.182 | INFO | Perf report for the last 5.13925 seconds: | 20260507 15:19:59.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:19:59.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 15:19:59.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | PRE_HQ : 35.4%, 1.0, 18.6ms, 0 | 20260507 15:19:59.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:19:59.182 | INFO | Uploaded 36GiB to GPU in 1.58948s (22.6489GiB/s) | 20260507 15:19:59.183 | INFO | Downloaded 34GiB to GPU in 1.53648s (22.1284GiB/s) | 20260507 15:20:04.321 | INFO | Analyzing chunk frames = [525312,525824) | 20260507 15:20:04.321 | INFO | MemInfo: { | 20260507 15:20:04.321 | INFO | "gpu" : " 17 GB", | 20260507 15:20:04.321 | INFO | "huge" : "", | 20260507 15:20:04.321 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:04.321 | INFO | "malloc" : " 14 GB", | 20260507 15:20:04.321 | INFO | "pinned" : " 29 GB" | 20260507 15:20:04.321 | INFO | } | 20260507 15:20:04.321 | INFO | Perf report for the last 5.13895 seconds: | 20260507 15:20:04.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:04.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.321 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:20:04.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.321 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:20:04.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:04.322 | INFO | Uploaded 36GiB to GPU in 1.58894s (22.6567GiB/s) | 20260507 15:20:04.322 | INFO | Downloaded 34GiB to GPU in 1.53816s (22.1043GiB/s) | 20260507 15:20:09.460 | INFO | Analyzing chunk frames = [525824,526336) | 20260507 15:20:09.460 | INFO | MemInfo: { | 20260507 15:20:09.460 | INFO | "gpu" : " 17 GB", | 20260507 15:20:09.460 | INFO | "huge" : "", | 20260507 15:20:09.460 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:09.460 | INFO | "malloc" : " 14 GB", | 20260507 15:20:09.460 | INFO | "pinned" : " 29 GB" | 20260507 15:20:09.460 | INFO | } | 20260507 15:20:09.461 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:20:09.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:09.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:20:09.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:20:09.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:09.461 | INFO | Uploaded 36GiB to GPU in 1.58797s (22.6705GiB/s) | 20260507 15:20:09.461 | INFO | Downloaded 34GiB to GPU in 1.53766s (22.1115GiB/s) | 20260507 15:20:14.600 | INFO | Analyzing chunk frames = [526336,526848) | 20260507 15:20:14.600 | INFO | MemInfo: { | 20260507 15:20:14.600 | INFO | "gpu" : " 17 GB", | 20260507 15:20:14.600 | INFO | "huge" : "", | 20260507 15:20:14.600 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:14.600 | INFO | "malloc" : " 14 GB", | 20260507 15:20:14.600 | INFO | "pinned" : " 29 GB" | 20260507 15:20:14.600 | INFO | } | 20260507 15:20:14.600 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:20:14.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:14.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.600 | INFO | ANALYSIS : 55.9%, 3.0, 87.3ms, 0 | 20260507 15:20:14.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.600 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 15:20:14.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:14.601 | INFO | Uploaded 36GiB to GPU in 1.5878s (22.6729GiB/s) | 20260507 15:20:14.601 | INFO | Downloaded 34GiB to GPU in 1.53599s (22.1355GiB/s) | 20260507 15:20:19.739 | INFO | Analyzing chunk frames = [526848,527360) | 20260507 15:20:19.739 | INFO | MemInfo: { | 20260507 15:20:19.739 | INFO | "gpu" : " 17 GB", | 20260507 15:20:19.739 | INFO | "huge" : "", | 20260507 15:20:19.739 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:19.739 | INFO | "malloc" : " 14 GB", | 20260507 15:20:19.739 | INFO | "pinned" : " 29 GB" | 20260507 15:20:19.739 | INFO | } | 20260507 15:20:19.739 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:20:19.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:19.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.739 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:20:19.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.739 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:20:19.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:19.740 | INFO | Uploaded 36GiB to GPU in 1.58743s (22.6781GiB/s) | 20260507 15:20:19.740 | INFO | Downloaded 34GiB to GPU in 1.53584s (22.1377GiB/s) | 20260507 15:20:24.878 | INFO | Analyzing chunk frames = [527360,527872) | 20260507 15:20:24.878 | INFO | MemInfo: { | 20260507 15:20:24.878 | INFO | "gpu" : " 17 GB", | 20260507 15:20:24.878 | INFO | "huge" : "", | 20260507 15:20:24.878 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:24.878 | INFO | "malloc" : " 14 GB", | 20260507 15:20:24.878 | INFO | "pinned" : " 29 GB" | 20260507 15:20:24.878 | INFO | } | 20260507 15:20:24.878 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:20:24.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:24.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.878 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:20:24.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.879 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:20:24.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:24.879 | INFO | Uploaded 36GiB to GPU in 1.58762s (22.6754GiB/s) | 20260507 15:20:24.879 | INFO | Downloaded 34GiB to GPU in 1.53665s (22.126GiB/s) | 20260507 15:20:30.018 | INFO | Analyzing chunk frames = [527872,528384) | 20260507 15:20:30.018 | INFO | MemInfo: { | 20260507 15:20:30.018 | INFO | "gpu" : " 17 GB", | 20260507 15:20:30.018 | INFO | "huge" : "", | 20260507 15:20:30.018 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:30.018 | INFO | "malloc" : " 14 GB", | 20260507 15:20:30.018 | INFO | "pinned" : " 29 GB" | 20260507 15:20:30.018 | INFO | } | 20260507 15:20:30.018 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:20:30.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:30.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:20:30.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:20:30.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:30.018 | INFO | Uploaded 36GiB to GPU in 1.58747s (22.6775GiB/s) | 20260507 15:20:30.019 | INFO | Downloaded 34GiB to GPU in 1.53708s (22.1198GiB/s) | 20260507 15:20:30.115 | INFO | Starting a round of Baz IO | 20260507 15:20:33.019 | INFO | Populating RTMetrics report | 20260507 15:20:33.019 | INFO | RTMetrics report populated | 20260507 15:20:33.019 | INFO | Opening tmp RTMetrics json file | 20260507 15:20:33.020 | INFO | Serializing RTMetrics to json | 20260507 15:20:33.023 | INFO | Serialization done, swapping tmp file now... | 20260507 15:20:33.026 | INFO | RTMetrics done | 20260507 15:20:33.238 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.122 seconds | 20260507 15:20:33.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.77% / 0.05% / 46.19% | 20260507 15:20:33.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.02603% / 0.000119209% / 2.38419e-05% | 20260507 15:20:33.238 | INFO | Baz IO overall: | 20260507 15:20:33.238 | INFO | BytesWritten : 212395539703 | 20260507 15:20:33.238 | INFO | NumEvents : 102198766407 | 20260507 15:20:35.157 | INFO | Analyzing chunk frames = [528384,528896) | 20260507 15:20:35.157 | INFO | MemInfo: { | 20260507 15:20:35.157 | INFO | "gpu" : " 17 GB", | 20260507 15:20:35.157 | INFO | "huge" : "", | 20260507 15:20:35.157 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:35.157 | INFO | "malloc" : " 14 GB", | 20260507 15:20:35.157 | INFO | "pinned" : " 29 GB" | 20260507 15:20:35.157 | INFO | } | 20260507 15:20:35.157 | INFO | Perf report for the last 5.13951 seconds: | 20260507 15:20:35.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:35.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:35.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:35.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:35.157 | INFO | ANALYSIS : 57.5%, 2.8, 85.2ms, 0 | 20260507 15:20:35.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:35.157 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 15:20:35.157 | INFO | PRE_HQ : 56.4%, 2.1, 61.5ms, 0 | 20260507 15:20:35.157 | INFO | BAZWRITER : 60.3%, 4.7, 147.0ms, 0 | 20260507 15:20:35.158 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:20:35.158 | INFO | Uploaded 36GiB to GPU in 1.6552s (21.7496GiB/s) | 20260507 15:20:35.158 | INFO | Downloaded 38GiB to GPU in 1.73832s (21.8601GiB/s) | 20260507 15:20:40.296 | INFO | Analyzing chunk frames = [528896,529408) | 20260507 15:20:40.296 | INFO | MemInfo: { | 20260507 15:20:40.296 | INFO | "gpu" : " 17 GB", | 20260507 15:20:40.296 | INFO | "huge" : "", | 20260507 15:20:40.296 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:40.296 | INFO | "malloc" : " 14 GB", | 20260507 15:20:40.297 | INFO | "pinned" : " 29 GB" | 20260507 15:20:40.297 | INFO | } | 20260507 15:20:40.297 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:20:40.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:40.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:20:40.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:20:40.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:40.297 | INFO | Uploaded 36GiB to GPU in 1.58535s (22.7079GiB/s) | 20260507 15:20:40.297 | INFO | Downloaded 34GiB to GPU in 1.53364s (22.1695GiB/s) | 20260507 15:20:45.436 | INFO | Analyzing chunk frames = [529408,529920) | 20260507 15:20:45.436 | INFO | MemInfo: { | 20260507 15:20:45.436 | INFO | "gpu" : " 17 GB", | 20260507 15:20:45.436 | INFO | "huge" : "", | 20260507 15:20:45.436 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:45.436 | INFO | "malloc" : " 14 GB", | 20260507 15:20:45.436 | INFO | "pinned" : " 29 GB" | 20260507 15:20:45.436 | INFO | } | 20260507 15:20:45.436 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:20:45.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:45.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:20:45.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:20:45.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:45.436 | INFO | Uploaded 36GiB to GPU in 1.58855s (22.6621GiB/s) | 20260507 15:20:45.436 | INFO | Downloaded 34GiB to GPU in 1.53771s (22.1108GiB/s) | 20260507 15:20:50.575 | INFO | Analyzing chunk frames = [529920,530432) | 20260507 15:20:50.575 | INFO | MemInfo: { | 20260507 15:20:50.575 | INFO | "gpu" : " 17 GB", | 20260507 15:20:50.575 | INFO | "huge" : "", | 20260507 15:20:50.575 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:50.575 | INFO | "malloc" : " 14 GB", | 20260507 15:20:50.575 | INFO | "pinned" : " 29 GB" | 20260507 15:20:50.575 | INFO | } | 20260507 15:20:50.575 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:20:50.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:50.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.575 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:20:50.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.576 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:20:50.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:50.576 | INFO | Uploaded 36GiB to GPU in 1.5904s (22.6358GiB/s) | 20260507 15:20:50.576 | INFO | Downloaded 34GiB to GPU in 1.53707s (22.12GiB/s) | 20260507 15:20:55.715 | INFO | Analyzing chunk frames = [530432,530944) | 20260507 15:20:55.715 | INFO | MemInfo: { | 20260507 15:20:55.715 | INFO | "gpu" : " 17 GB", | 20260507 15:20:55.715 | INFO | "huge" : "", | 20260507 15:20:55.715 | INFO | "huge_shared" : " 29 GB", | 20260507 15:20:55.715 | INFO | "malloc" : " 14 GB", | 20260507 15:20:55.715 | INFO | "pinned" : " 29 GB" | 20260507 15:20:55.715 | INFO | } | 20260507 15:20:55.715 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:20:55.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:20:55.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | ANALYSIS : 55.9%, 3.0, 87.3ms, 0 | 20260507 15:20:55.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:20:55.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:20:55.715 | INFO | Uploaded 36GiB to GPU in 1.58783s (22.6725GiB/s) | 20260507 15:20:55.716 | INFO | Downloaded 34GiB to GPU in 1.53579s (22.1385GiB/s) | 20260507 15:21:00.854 | INFO | Analyzing chunk frames = [530944,531456) | 20260507 15:21:00.854 | INFO | MemInfo: { | 20260507 15:21:00.854 | INFO | "gpu" : " 17 GB", | 20260507 15:21:00.854 | INFO | "huge" : "", | 20260507 15:21:00.854 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:00.854 | INFO | "malloc" : " 14 GB", | 20260507 15:21:00.854 | INFO | "pinned" : " 29 GB" | 20260507 15:21:00.854 | INFO | } | 20260507 15:21:00.854 | INFO | Perf report for the last 5.13915 seconds: | 20260507 15:21:00.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:00.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:21:00.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:21:00.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:00.854 | INFO | Uploaded 36GiB to GPU in 1.58927s (22.6519GiB/s) | 20260507 15:21:00.854 | INFO | Downloaded 34GiB to GPU in 1.53622s (22.1322GiB/s) | 20260507 15:21:05.993 | INFO | Analyzing chunk frames = [531456,531968) | 20260507 15:21:05.993 | INFO | MemInfo: { | 20260507 15:21:05.993 | INFO | "gpu" : " 17 GB", | 20260507 15:21:05.993 | INFO | "huge" : "", | 20260507 15:21:05.993 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:05.993 | INFO | "malloc" : " 14 GB", | 20260507 15:21:05.993 | INFO | "pinned" : " 29 GB" | 20260507 15:21:05.993 | INFO | } | 20260507 15:21:05.993 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:21:05.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:05.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.993 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:21:05.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.993 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:21:05.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:05.994 | INFO | Uploaded 36GiB to GPU in 1.58804s (22.6694GiB/s) | 20260507 15:21:05.994 | INFO | Downloaded 34GiB to GPU in 1.53762s (22.1121GiB/s) | 20260507 15:21:11.133 | INFO | Analyzing chunk frames = [531968,532480) | 20260507 15:21:11.133 | INFO | MemInfo: { | 20260507 15:21:11.133 | INFO | "gpu" : " 17 GB", | 20260507 15:21:11.133 | INFO | "huge" : "", | 20260507 15:21:11.133 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:11.133 | INFO | "malloc" : " 14 GB", | 20260507 15:21:11.133 | INFO | "pinned" : " 29 GB" | 20260507 15:21:11.133 | INFO | } | 20260507 15:21:11.133 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:21:11.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:11.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:21:11.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260507 15:21:11.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:11.133 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:21:11.133 | INFO | Total mem usage: 16.4 GiB | 20260507 15:21:11.133 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:21:11.133 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.35%) | 20260507 15:21:11.133 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:21:11.133 | INFO | High Water Breakdown: | 20260507 15:21:11.133 | INFO | 59.5% : 9.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:21:11.133 | INFO | 40.4% : 6.17 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:21:11.133 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:21:11.133 | INFO | ------------------------------------------ | 20260507 15:21:11.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:21:11.133 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:21:11.133 | INFO | Total mem usage: 26.5 GiB | 20260507 15:21:11.133 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:21:11.133 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 15:21:11.133 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 15:21:11.133 | INFO | High Water Breakdown: | 20260507 15:21:11.133 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:21:11.133 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:21:11.133 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:21:11.133 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:21:11.133 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:21:11.133 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:21:11.133 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:21:11.133 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:21:11.133 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:21:11.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:21:11.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:21:11.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:21:11.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:21:11.133 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:21:11.133 | INFO | ------------------------------------------ | 20260507 15:21:11.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:21:11.133 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:21:11.133 | INFO | Total mem usage: 27.8 GiB | 20260507 15:21:11.133 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:21:11.133 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:21:11.133 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:21:11.133 | INFO | High Water Breakdown: | 20260507 15:21:11.133 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:21:11.133 | INFO | ------------------------------------------ | 20260507 15:21:11.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:21:11.133 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:21:11.133 | INFO | Total mem usage: 15.7 GiB | 20260507 15:21:11.133 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:21:11.133 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:21:11.133 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:21:11.133 | INFO | High Water Breakdown: | 20260507 15:21:11.133 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:21:11.133 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:21:11.133 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:21:11.133 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:21:11.133 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:21:11.133 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:21:11.133 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:21:11.133 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:21:11.133 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:21:11.133 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:21:11.133 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:21:11.133 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:21:11.133 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:21:11.133 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:21:11.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:21:11.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:21:11.133 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:21:11.133 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:21:11.133 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:21:11.133 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:21:11.133 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:21:11.133 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:21:11.133 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:21:11.133 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:21:11.133 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:21:11.133 | INFO | ------------------------------------------ | 20260507 15:21:11.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:21:11.133 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:21:11.133 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:21:11.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:21:11.133 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 0.29%, AggregateStats -- 8.73e-02 / -nan, 6.72e-03 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 11.04%, Baseline -- 3.30e+00 / -nan, 2.03e-02 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 16.12%, DME -- 3.85e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.72e+01 / -nan, 4.46e-02 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.15e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 3.48%, Metrics -- 1.04e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.84e-01 / -nan, 5.58e-02 / 0.00e+00 | 20260507 15:21:11.133 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:21:11.133 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:21:11.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:21:11.133 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 18.61%, Upload -- 1.66e+01 / 0.00e+00, 4.85e-01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 18.20%, Download -- 1.63e+01 / 0.00e+00, 7.70e-01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 33.44%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 0.59%, LockUpload -- 5.26e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 0.00%, LockDownload -- 4.54e-04 / 0.00e+00, 1.90e-04 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 29.14%, LockCompute -- 2.60e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260507 15:21:11.133 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 15:21:11.133 | INFO | Uploaded 36GiB to GPU in 1.59195s (22.6138GiB/s) | 20260507 15:21:11.133 | INFO | Downloaded 34GiB to GPU in 1.53906s (22.0914GiB/s) | 20260507 15:21:11.231 | INFO | Starting a round of Baz IO | 20260507 15:21:14.119 | INFO | Populating RTMetrics report | 20260507 15:21:14.119 | INFO | RTMetrics report populated | 20260507 15:21:14.119 | INFO | Opening tmp RTMetrics json file | 20260507 15:21:14.120 | INFO | Serializing RTMetrics to json | 20260507 15:21:14.121 | INFO | Serialization done, swapping tmp file now... | 20260507 15:21:14.122 | INFO | RTMetrics done | 20260507 15:21:14.268 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.036 seconds | 20260507 15:21:14.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.72% / 0.05% / 46.23% | 20260507 15:21:14.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.99053% / 0.000127157% / 2.38419e-05% | 20260507 15:21:14.268 | INFO | Baz IO overall: | 20260507 15:21:14.268 | INFO | BytesWritten : 214266762329 | 20260507 15:21:14.268 | INFO | NumEvents : 103109233589 | 20260507 15:21:16.272 | INFO | Analyzing chunk frames = [532480,532992) | 20260507 15:21:16.272 | INFO | MemInfo: { | 20260507 15:21:16.272 | INFO | "gpu" : " 17 GB", | 20260507 15:21:16.272 | INFO | "huge" : "", | 20260507 15:21:16.272 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:16.272 | INFO | "malloc" : " 14 GB", | 20260507 15:21:16.272 | INFO | "pinned" : " 29 GB" | 20260507 15:21:16.272 | INFO | } | 20260507 15:21:16.272 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:21:16.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:16.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:16.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:16.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:16.272 | INFO | ANALYSIS : 57.2%, 2.8, 84.5ms, 0 | 20260507 15:21:16.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:16.272 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 15:21:16.272 | INFO | PRE_HQ : 56.0%, 2.1, 60.5ms, 0 | 20260507 15:21:16.272 | INFO | BAZWRITER : 58.7%, 4.8, 143.9ms, 0 | 20260507 15:21:16.272 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 15:21:16.273 | INFO | Uploaded 36GiB to GPU in 1.65439s (21.7603GiB/s) | 20260507 15:21:16.273 | INFO | Downloaded 38GiB to GPU in 1.73766s (21.8685GiB/s) | 20260507 15:21:21.411 | INFO | Analyzing chunk frames = [532992,533504) | 20260507 15:21:21.411 | INFO | MemInfo: { | 20260507 15:21:21.411 | INFO | "gpu" : " 17 GB", | 20260507 15:21:21.411 | INFO | "huge" : "", | 20260507 15:21:21.411 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:21.411 | INFO | "malloc" : " 14 GB", | 20260507 15:21:21.411 | INFO | "pinned" : " 29 GB" | 20260507 15:21:21.411 | INFO | } | 20260507 15:21:21.411 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:21:21.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:21.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.411 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:21:21.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.412 | INFO | PRE_HQ : 27.8%, 1.0, 14.6ms, 0 | 20260507 15:21:21.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:21.412 | INFO | Uploaded 36GiB to GPU in 1.58522s (22.7098GiB/s) | 20260507 15:21:21.412 | INFO | Downloaded 34GiB to GPU in 1.53308s (22.1776GiB/s) | 20260507 15:21:26.550 | INFO | Analyzing chunk frames = [533504,534016) | 20260507 15:21:26.550 | INFO | MemInfo: { | 20260507 15:21:26.550 | INFO | "gpu" : " 17 GB", | 20260507 15:21:26.550 | INFO | "huge" : "", | 20260507 15:21:26.550 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:26.550 | INFO | "malloc" : " 14 GB", | 20260507 15:21:26.551 | INFO | "pinned" : " 29 GB" | 20260507 15:21:26.551 | INFO | } | 20260507 15:21:26.551 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:21:26.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:26.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:21:26.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:21:26.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:26.551 | INFO | Uploaded 36GiB to GPU in 1.58914s (22.6538GiB/s) | 20260507 15:21:26.551 | INFO | Downloaded 34GiB to GPU in 1.53887s (22.0942GiB/s) | 20260507 15:21:31.690 | INFO | Analyzing chunk frames = [534016,534528) | 20260507 15:21:31.690 | INFO | MemInfo: { | 20260507 15:21:31.690 | INFO | "gpu" : " 17 GB", | 20260507 15:21:31.690 | INFO | "huge" : "", | 20260507 15:21:31.690 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:31.690 | INFO | "malloc" : " 14 GB", | 20260507 15:21:31.690 | INFO | "pinned" : " 29 GB" | 20260507 15:21:31.690 | INFO | } | 20260507 15:21:31.690 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:21:31.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:31.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:21:31.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:21:31.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:31.690 | INFO | Uploaded 36GiB to GPU in 1.5883s (22.6657GiB/s) | 20260507 15:21:31.691 | INFO | Downloaded 34GiB to GPU in 1.53922s (22.0891GiB/s) | 20260507 15:21:36.829 | INFO | Analyzing chunk frames = [534528,535040) | 20260507 15:21:36.829 | INFO | MemInfo: { | 20260507 15:21:36.829 | INFO | "gpu" : " 17 GB", | 20260507 15:21:36.829 | INFO | "huge" : "", | 20260507 15:21:36.829 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:36.829 | INFO | "malloc" : " 14 GB", | 20260507 15:21:36.829 | INFO | "pinned" : " 29 GB" | 20260507 15:21:36.829 | INFO | } | 20260507 15:21:36.829 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:21:36.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:36.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.829 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:21:36.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.830 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:21:36.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:36.830 | INFO | Uploaded 36GiB to GPU in 1.58777s (22.6733GiB/s) | 20260507 15:21:36.830 | INFO | Downloaded 34GiB to GPU in 1.53628s (22.1313GiB/s) | 20260507 15:21:41.969 | INFO | Analyzing chunk frames = [535040,535552) | 20260507 15:21:41.969 | INFO | MemInfo: { | 20260507 15:21:41.969 | INFO | "gpu" : " 17 GB", | 20260507 15:21:41.969 | INFO | "huge" : "", | 20260507 15:21:41.969 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:41.969 | INFO | "malloc" : " 14 GB", | 20260507 15:21:41.969 | INFO | "pinned" : " 29 GB" | 20260507 15:21:41.969 | INFO | } | 20260507 15:21:41.969 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:21:41.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:41.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:21:41.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:21:41.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:41.969 | INFO | Uploaded 36GiB to GPU in 1.58692s (22.6854GiB/s) | 20260507 15:21:41.969 | INFO | Downloaded 34GiB to GPU in 1.53651s (22.128GiB/s) | 20260507 15:21:47.108 | INFO | Analyzing chunk frames = [535552,536064) | 20260507 15:21:47.108 | INFO | MemInfo: { | 20260507 15:21:47.108 | INFO | "gpu" : " 17 GB", | 20260507 15:21:47.108 | INFO | "huge" : "", | 20260507 15:21:47.108 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:47.108 | INFO | "malloc" : " 14 GB", | 20260507 15:21:47.108 | INFO | "pinned" : " 29 GB" | 20260507 15:21:47.108 | INFO | } | 20260507 15:21:47.108 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:21:47.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:47.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.108 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:21:47.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.108 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:21:47.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:47.109 | INFO | Uploaded 36GiB to GPU in 1.5881s (22.6686GiB/s) | 20260507 15:21:47.109 | INFO | Downloaded 34GiB to GPU in 1.53828s (22.1027GiB/s) | 20260507 15:21:52.247 | INFO | Analyzing chunk frames = [536064,536576) | 20260507 15:21:52.247 | INFO | MemInfo: { | 20260507 15:21:52.247 | INFO | "gpu" : " 17 GB", | 20260507 15:21:52.247 | INFO | "huge" : "", | 20260507 15:21:52.247 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:52.247 | INFO | "malloc" : " 14 GB", | 20260507 15:21:52.247 | INFO | "pinned" : " 29 GB" | 20260507 15:21:52.247 | INFO | } | 20260507 15:21:52.248 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:21:52.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:52.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 15:21:52.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 15:21:52.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:52.248 | INFO | Uploaded 36GiB to GPU in 1.59069s (22.6317GiB/s) | 20260507 15:21:52.248 | INFO | Downloaded 34GiB to GPU in 1.54031s (22.0735GiB/s) | 20260507 15:21:52.345 | INFO | Starting a round of Baz IO | 20260507 15:21:55.252 | INFO | Populating RTMetrics report | 20260507 15:21:55.252 | INFO | RTMetrics report populated | 20260507 15:21:55.252 | INFO | Opening tmp RTMetrics json file | 20260507 15:21:55.263 | INFO | Serializing RTMetrics to json | 20260507 15:21:55.273 | INFO | Serialization done, swapping tmp file now... | 20260507 15:21:55.289 | INFO | RTMetrics done | 20260507 15:21:55.357 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.011 seconds | 20260507 15:21:55.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.67% / 0.05% / 46.28% | 20260507 15:21:55.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.96674% / 7.54992e-05% / 2.38419e-05% | 20260507 15:21:55.357 | INFO | Baz IO overall: | 20260507 15:21:55.357 | INFO | BytesWritten : 216137838813 | 20260507 15:21:55.357 | INFO | NumEvents : 104019931988 | 20260507 15:21:57.387 | INFO | Analyzing chunk frames = [536576,537088) | 20260507 15:21:57.387 | INFO | MemInfo: { | 20260507 15:21:57.387 | INFO | "gpu" : " 17 GB", | 20260507 15:21:57.387 | INFO | "huge" : "", | 20260507 15:21:57.387 | INFO | "huge_shared" : " 29 GB", | 20260507 15:21:57.387 | INFO | "malloc" : " 14 GB", | 20260507 15:21:57.387 | INFO | "pinned" : " 29 GB" | 20260507 15:21:57.387 | INFO | } | 20260507 15:21:57.387 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:21:57.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:21:57.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:57.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:57.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:57.387 | INFO | ANALYSIS : 57.6%, 2.8, 85.1ms, 0 | 20260507 15:21:57.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:21:57.387 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 15:21:57.387 | INFO | PRE_HQ : 56.5%, 2.1, 61.3ms, 0 | 20260507 15:21:57.387 | INFO | BAZWRITER : 58.2%, 4.7, 140.7ms, 0 | 20260507 15:21:57.387 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:21:57.388 | INFO | Uploaded 36GiB to GPU in 1.64718s (21.8556GiB/s) | 20260507 15:21:57.388 | INFO | Downloaded 38GiB to GPU in 1.73963s (21.8437GiB/s) | 20260507 15:22:02.526 | INFO | Analyzing chunk frames = [537088,537600) | 20260507 15:22:02.526 | INFO | MemInfo: { | 20260507 15:22:02.526 | INFO | "gpu" : " 17 GB", | 20260507 15:22:02.526 | INFO | "huge" : "", | 20260507 15:22:02.526 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:02.526 | INFO | "malloc" : " 14 GB", | 20260507 15:22:02.526 | INFO | "pinned" : " 29 GB" | 20260507 15:22:02.526 | INFO | } | 20260507 15:22:02.526 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:22:02.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:02.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.526 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:22:02.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.526 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:22:02.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:02.527 | INFO | Uploaded 36GiB to GPU in 1.58593s (22.6996GiB/s) | 20260507 15:22:02.527 | INFO | Downloaded 34GiB to GPU in 1.53351s (22.1713GiB/s) | 20260507 15:22:07.665 | INFO | Analyzing chunk frames = [537600,538112) | 20260507 15:22:07.665 | INFO | MemInfo: { | 20260507 15:22:07.665 | INFO | "gpu" : " 17 GB", | 20260507 15:22:07.665 | INFO | "huge" : "", | 20260507 15:22:07.665 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:07.665 | INFO | "malloc" : " 14 GB", | 20260507 15:22:07.666 | INFO | "pinned" : " 29 GB" | 20260507 15:22:07.666 | INFO | } | 20260507 15:22:07.666 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:22:07.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:07.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:22:07.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:22:07.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:07.666 | INFO | Uploaded 36GiB to GPU in 1.58905s (22.6551GiB/s) | 20260507 15:22:07.666 | INFO | Downloaded 34GiB to GPU in 1.53878s (22.0955GiB/s) | 20260507 15:22:12.805 | INFO | Analyzing chunk frames = [538112,538624) | 20260507 15:22:12.805 | INFO | MemInfo: { | 20260507 15:22:12.805 | INFO | "gpu" : " 17 GB", | 20260507 15:22:12.805 | INFO | "huge" : "", | 20260507 15:22:12.805 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:12.805 | INFO | "malloc" : " 14 GB", | 20260507 15:22:12.805 | INFO | "pinned" : " 29 GB" | 20260507 15:22:12.805 | INFO | } | 20260507 15:22:12.805 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:22:12.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:12.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:22:12.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:22:12.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:12.805 | INFO | Uploaded 36GiB to GPU in 1.58796s (22.6706GiB/s) | 20260507 15:22:12.805 | INFO | Downloaded 34GiB to GPU in 1.53725s (22.1175GiB/s) | 20260507 15:22:17.944 | INFO | Analyzing chunk frames = [538624,539136) | 20260507 15:22:17.944 | INFO | MemInfo: { | 20260507 15:22:17.944 | INFO | "gpu" : " 17 GB", | 20260507 15:22:17.944 | INFO | "huge" : "", | 20260507 15:22:17.944 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:17.944 | INFO | "malloc" : " 14 GB", | 20260507 15:22:17.944 | INFO | "pinned" : " 29 GB" | 20260507 15:22:17.944 | INFO | } | 20260507 15:22:17.944 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:22:17.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:17.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.944 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:22:17.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.944 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:22:17.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:17.945 | INFO | Uploaded 36GiB to GPU in 1.58858s (22.6618GiB/s) | 20260507 15:22:17.945 | INFO | Downloaded 34GiB to GPU in 1.53693s (22.1221GiB/s) | 20260507 15:22:23.083 | INFO | Analyzing chunk frames = [539136,539648) | 20260507 15:22:23.084 | INFO | MemInfo: { | 20260507 15:22:23.084 | INFO | "gpu" : " 17 GB", | 20260507 15:22:23.084 | INFO | "huge" : "", | 20260507 15:22:23.084 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:23.084 | INFO | "malloc" : " 14 GB", | 20260507 15:22:23.084 | INFO | "pinned" : " 29 GB" | 20260507 15:22:23.084 | INFO | } | 20260507 15:22:23.084 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:22:23.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:23.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:22:23.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:22:23.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:23.084 | INFO | Uploaded 36GiB to GPU in 1.58898s (22.656GiB/s) | 20260507 15:22:23.084 | INFO | Downloaded 34GiB to GPU in 1.53523s (22.1466GiB/s) | 20260507 15:22:28.223 | INFO | Analyzing chunk frames = [539648,540160) | 20260507 15:22:28.223 | INFO | MemInfo: { | 20260507 15:22:28.223 | INFO | "gpu" : " 17 GB", | 20260507 15:22:28.223 | INFO | "huge" : "", | 20260507 15:22:28.223 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:28.223 | INFO | "malloc" : " 14 GB", | 20260507 15:22:28.223 | INFO | "pinned" : " 29 GB" | 20260507 15:22:28.223 | INFO | } | 20260507 15:22:28.223 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:22:28.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:28.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:22:28.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:22:28.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:28.223 | INFO | Uploaded 36GiB to GPU in 1.58819s (22.6674GiB/s) | 20260507 15:22:28.223 | INFO | Downloaded 34GiB to GPU in 1.53762s (22.112GiB/s) | 20260507 15:22:33.362 | INFO | Analyzing chunk frames = [540160,540672) | 20260507 15:22:33.362 | INFO | MemInfo: { | 20260507 15:22:33.362 | INFO | "gpu" : " 17 GB", | 20260507 15:22:33.362 | INFO | "huge" : "", | 20260507 15:22:33.362 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:33.362 | INFO | "malloc" : " 14 GB", | 20260507 15:22:33.362 | INFO | "pinned" : " 29 GB" | 20260507 15:22:33.362 | INFO | } | 20260507 15:22:33.362 | INFO | Perf report for the last 5.13906 seconds: | 20260507 15:22:33.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:33.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.362 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 15:22:33.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.362 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260507 15:22:33.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:33.363 | INFO | Uploaded 36GiB to GPU in 1.59321s (22.5959GiB/s) | 20260507 15:22:33.363 | INFO | Downloaded 34GiB to GPU in 1.54s (22.0779GiB/s) | 20260507 15:22:33.363 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:22:33.363 | INFO | Total mem usage: 16.4 GiB | 20260507 15:22:33.363 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:22:33.363 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.52%) | 20260507 15:22:33.363 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:22:33.363 | INFO | High Water Breakdown: | 20260507 15:22:33.363 | INFO | 59.3% : 8.97 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:22:33.363 | INFO | 40.5% : 6.13 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:22:33.363 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:22:33.363 | INFO | ------------------------------------------ | 20260507 15:22:33.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:22:33.363 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:22:33.363 | INFO | Total mem usage: 26.5 GiB | 20260507 15:22:33.363 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:22:33.363 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:22:33.363 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:22:33.363 | INFO | High Water Breakdown: | 20260507 15:22:33.363 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:22:33.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:22:33.363 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:22:33.363 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:22:33.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:22:33.363 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:22:33.363 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:22:33.363 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:22:33.363 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:22:33.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:22:33.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:22:33.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:22:33.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:22:33.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:22:33.363 | INFO | ------------------------------------------ | 20260507 15:22:33.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:22:33.363 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:22:33.363 | INFO | Total mem usage: 27.8 GiB | 20260507 15:22:33.363 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:22:33.363 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:22:33.363 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:22:33.363 | INFO | High Water Breakdown: | 20260507 15:22:33.363 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:22:33.363 | INFO | ------------------------------------------ | 20260507 15:22:33.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:22:33.363 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:22:33.363 | INFO | Total mem usage: 15.7 GiB | 20260507 15:22:33.363 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:22:33.363 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:22:33.363 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:22:33.363 | INFO | High Water Breakdown: | 20260507 15:22:33.363 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:22:33.363 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:22:33.363 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:22:33.363 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:22:33.363 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:22:33.363 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:22:33.363 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:22:33.363 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:22:33.363 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:22:33.363 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:22:33.363 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:22:33.363 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:22:33.363 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:22:33.363 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:22:33.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:22:33.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:22:33.364 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:22:33.364 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:22:33.364 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:22:33.364 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:22:33.364 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:22:33.364 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:22:33.364 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:22:33.364 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:22:33.364 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:22:33.364 | INFO | ------------------------------------------ | 20260507 15:22:33.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:22:33.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:22:33.364 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:22:33.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:22:33.364 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 0.29%, AggregateStats -- 8.72e-02 / -nan, 5.38e-03 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 11.03%, Baseline -- 3.30e+00 / -nan, 2.29e-02 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.29e-02 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 16.14%, DME -- 3.86e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.72e+01 / -nan, 4.80e-02 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.15e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 3.48%, Metrics -- 1.04e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.83e-01 / -nan, 5.56e-02 / 0.00e+00 | 20260507 15:22:33.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:22:33.364 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:22:33.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:22:33.364 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 18.63%, Upload -- 1.66e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 18.24%, Download -- 1.63e+01 / 0.00e+00, 8.11e-01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 33.48%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 0.58%, LockUpload -- 5.17e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 0.00%, LockDownload -- 4.62e-04 / 0.00e+00, 2.04e-04 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 29.04%, LockCompute -- 2.59e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:22:33.364 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 5.65e-02 / 0.00e+00 | 20260507 15:22:33.459 | INFO | Starting a round of Baz IO | 20260507 15:22:36.345 | INFO | Populating RTMetrics report | 20260507 15:22:36.345 | INFO | RTMetrics report populated | 20260507 15:22:36.345 | INFO | Opening tmp RTMetrics json file | 20260507 15:22:36.409 | INFO | Serializing RTMetrics to json | 20260507 15:22:36.412 | INFO | Serialization done, swapping tmp file now... | 20260507 15:22:36.415 | INFO | RTMetrics done | 20260507 15:22:36.673 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.213 seconds | 20260507 15:22:36.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.63% / 0.05% / 46.33% | 20260507 15:22:36.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.96133% / 8.34465e-05% / 2.78155e-05% | 20260507 15:22:36.673 | INFO | Baz IO overall: | 20260507 15:22:36.673 | INFO | BytesWritten : 218011864398 | 20260507 15:22:36.673 | INFO | NumEvents : 104933016790 | 20260507 15:22:38.501 | INFO | Analyzing chunk frames = [540672,541184) | 20260507 15:22:38.501 | INFO | MemInfo: { | 20260507 15:22:38.501 | INFO | "gpu" : " 17 GB", | 20260507 15:22:38.501 | INFO | "huge" : "", | 20260507 15:22:38.501 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:38.501 | INFO | "malloc" : " 14 GB", | 20260507 15:22:38.501 | INFO | "pinned" : " 29 GB" | 20260507 15:22:38.501 | INFO | } | 20260507 15:22:38.502 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:22:38.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:38.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:38.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:38.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:38.502 | INFO | ANALYSIS : 57.2%, 2.8, 84.9ms, 0 | 20260507 15:22:38.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:38.502 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 15:22:38.502 | INFO | PRE_HQ : 56.1%, 2.1, 60.5ms, 0 | 20260507 15:22:38.502 | INFO | BAZWRITER : 62.1%, 4.7, 150.4ms, 0 | 20260507 15:22:38.502 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 15:22:38.502 | INFO | Uploaded 36GiB to GPU in 1.6632s (21.645GiB/s) | 20260507 15:22:38.502 | INFO | Downloaded 38GiB to GPU in 1.74577s (21.7669GiB/s) | 20260507 15:22:43.641 | INFO | Analyzing chunk frames = [541184,541696) | 20260507 15:22:43.641 | INFO | MemInfo: { | 20260507 15:22:43.641 | INFO | "gpu" : " 17 GB", | 20260507 15:22:43.641 | INFO | "huge" : "", | 20260507 15:22:43.641 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:43.641 | INFO | "malloc" : " 14 GB", | 20260507 15:22:43.641 | INFO | "pinned" : " 29 GB" | 20260507 15:22:43.641 | INFO | } | 20260507 15:22:43.641 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:22:43.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:43.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:22:43.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:22:43.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:43.641 | INFO | Uploaded 36GiB to GPU in 1.58522s (22.7098GiB/s) | 20260507 15:22:43.641 | INFO | Downloaded 34GiB to GPU in 1.53318s (22.1762GiB/s) | 20260507 15:22:48.780 | INFO | Analyzing chunk frames = [541696,542208) | 20260507 15:22:48.780 | INFO | MemInfo: { | 20260507 15:22:48.780 | INFO | "gpu" : " 17 GB", | 20260507 15:22:48.780 | INFO | "huge" : "", | 20260507 15:22:48.780 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:48.780 | INFO | "malloc" : " 14 GB", | 20260507 15:22:48.780 | INFO | "pinned" : " 29 GB" | 20260507 15:22:48.780 | INFO | } | 20260507 15:22:48.780 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:22:48.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:48.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.780 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:22:48.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.780 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:22:48.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:48.781 | INFO | Uploaded 36GiB to GPU in 1.58897s (22.6562GiB/s) | 20260507 15:22:48.781 | INFO | Downloaded 34GiB to GPU in 1.53775s (22.1102GiB/s) | 20260507 15:22:53.919 | INFO | Analyzing chunk frames = [542208,542720) | 20260507 15:22:53.919 | INFO | MemInfo: { | 20260507 15:22:53.919 | INFO | "gpu" : " 17 GB", | 20260507 15:22:53.919 | INFO | "huge" : "", | 20260507 15:22:53.919 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:53.919 | INFO | "malloc" : " 14 GB", | 20260507 15:22:53.920 | INFO | "pinned" : " 29 GB" | 20260507 15:22:53.920 | INFO | } | 20260507 15:22:53.920 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:22:53.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:53.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:22:53.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:22:53.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:53.920 | INFO | Uploaded 36GiB to GPU in 1.59727s (22.5385GiB/s) | 20260507 15:22:53.920 | INFO | Downloaded 34GiB to GPU in 1.54328s (22.031GiB/s) | 20260507 15:22:59.059 | INFO | Analyzing chunk frames = [542720,543232) | 20260507 15:22:59.059 | INFO | MemInfo: { | 20260507 15:22:59.059 | INFO | "gpu" : " 17 GB", | 20260507 15:22:59.059 | INFO | "huge" : "", | 20260507 15:22:59.059 | INFO | "huge_shared" : " 29 GB", | 20260507 15:22:59.059 | INFO | "malloc" : " 14 GB", | 20260507 15:22:59.059 | INFO | "pinned" : " 29 GB" | 20260507 15:22:59.059 | INFO | } | 20260507 15:22:59.059 | INFO | Perf report for the last 5.13955 seconds: | 20260507 15:22:59.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:22:59.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.059 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:22:59.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.059 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:22:59.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:22:59.060 | INFO | Uploaded 36GiB to GPU in 1.58786s (22.672GiB/s) | 20260507 15:22:59.060 | INFO | Downloaded 34GiB to GPU in 1.53614s (22.1334GiB/s) | 20260507 15:23:04.198 | INFO | Analyzing chunk frames = [543232,543744) | 20260507 15:23:04.198 | INFO | MemInfo: { | 20260507 15:23:04.198 | INFO | "gpu" : " 17 GB", | 20260507 15:23:04.198 | INFO | "huge" : "", | 20260507 15:23:04.198 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:04.198 | INFO | "malloc" : " 14 GB", | 20260507 15:23:04.198 | INFO | "pinned" : " 29 GB" | 20260507 15:23:04.198 | INFO | } | 20260507 15:23:04.198 | INFO | Perf report for the last 5.13915 seconds: | 20260507 15:23:04.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:04.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.198 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:23:04.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.198 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:23:04.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:04.199 | INFO | Uploaded 36GiB to GPU in 1.58971s (22.6457GiB/s) | 20260507 15:23:04.199 | INFO | Downloaded 34GiB to GPU in 1.53577s (22.1387GiB/s) | 20260507 15:23:09.338 | INFO | Analyzing chunk frames = [543744,544256) | 20260507 15:23:09.338 | INFO | MemInfo: { | 20260507 15:23:09.338 | INFO | "gpu" : " 17 GB", | 20260507 15:23:09.338 | INFO | "huge" : "", | 20260507 15:23:09.338 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:09.338 | INFO | "malloc" : " 14 GB", | 20260507 15:23:09.338 | INFO | "pinned" : " 29 GB" | 20260507 15:23:09.338 | INFO | } | 20260507 15:23:09.338 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:23:09.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:09.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | ANALYSIS : 56.4%, 3.0, 88.1ms, 0 | 20260507 15:23:09.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:23:09.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:09.338 | INFO | Uploaded 36GiB to GPU in 1.58785s (22.6722GiB/s) | 20260507 15:23:09.338 | INFO | Downloaded 34GiB to GPU in 1.53769s (22.1111GiB/s) | 20260507 15:23:14.477 | INFO | Analyzing chunk frames = [544256,544768) | 20260507 15:23:14.477 | INFO | MemInfo: { | 20260507 15:23:14.477 | INFO | "gpu" : " 17 GB", | 20260507 15:23:14.477 | INFO | "huge" : "", | 20260507 15:23:14.477 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:14.477 | INFO | "malloc" : " 14 GB", | 20260507 15:23:14.477 | INFO | "pinned" : " 29 GB" | 20260507 15:23:14.477 | INFO | } | 20260507 15:23:14.477 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:23:14.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:14.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.477 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 15:23:14.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.477 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:23:14.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:14.478 | INFO | Uploaded 36GiB to GPU in 1.59659s (22.5481GiB/s) | 20260507 15:23:14.478 | INFO | Downloaded 34GiB to GPU in 1.54557s (21.9984GiB/s) | 20260507 15:23:14.575 | INFO | Starting a round of Baz IO | 20260507 15:23:17.519 | INFO | Populating RTMetrics report | 20260507 15:23:17.520 | INFO | RTMetrics report populated | 20260507 15:23:17.520 | INFO | Opening tmp RTMetrics json file | 20260507 15:23:17.529 | INFO | Serializing RTMetrics to json | 20260507 15:23:17.532 | INFO | Serialization done, swapping tmp file now... | 20260507 15:23:17.533 | INFO | RTMetrics done | 20260507 15:23:18.059 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.484 seconds | 20260507 15:23:18.059 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.58% / 0.05% / 46.37% | 20260507 15:23:18.059 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.92857% / 5.16574e-05% / 2.78155e-05% | 20260507 15:23:18.059 | INFO | Baz IO overall: | 20260507 15:23:18.059 | INFO | BytesWritten : 219884001059 | 20260507 15:23:18.059 | INFO | NumEvents : 105844970330 | 20260507 15:23:19.616 | INFO | Analyzing chunk frames = [544768,545280) | 20260507 15:23:19.616 | INFO | MemInfo: { | 20260507 15:23:19.616 | INFO | "gpu" : " 17 GB", | 20260507 15:23:19.616 | INFO | "huge" : "", | 20260507 15:23:19.616 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:19.616 | INFO | "malloc" : " 14 GB", | 20260507 15:23:19.616 | INFO | "pinned" : " 29 GB" | 20260507 15:23:19.616 | INFO | } | 20260507 15:23:19.616 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:23:19.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:19.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:19.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:19.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:19.616 | INFO | ANALYSIS : 58.0%, 2.8, 85.1ms, 0 | 20260507 15:23:19.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:19.617 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 15:23:19.617 | INFO | PRE_HQ : 57.2%, 2.2, 64.2ms, 0 | 20260507 15:23:19.617 | INFO | BAZWRITER : 67.4%, 4.7, 164.1ms, 0 | 20260507 15:23:19.617 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:23:19.617 | INFO | Uploaded 36GiB to GPU in 1.6536s (21.7706GiB/s) | 20260507 15:23:19.617 | INFO | Downloaded 38GiB to GPU in 1.73441s (21.9094GiB/s) | 20260507 15:23:24.756 | INFO | Analyzing chunk frames = [545280,545792) | 20260507 15:23:24.756 | INFO | MemInfo: { | 20260507 15:23:24.756 | INFO | "gpu" : " 17 GB", | 20260507 15:23:24.756 | INFO | "huge" : "", | 20260507 15:23:24.756 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:24.756 | INFO | "malloc" : " 14 GB", | 20260507 15:23:24.756 | INFO | "pinned" : " 29 GB" | 20260507 15:23:24.756 | INFO | } | 20260507 15:23:24.756 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:23:24.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:24.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:23:24.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260507 15:23:24.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:24.756 | INFO | Uploaded 36GiB to GPU in 1.60646s (22.4095GiB/s) | 20260507 15:23:24.756 | INFO | Downloaded 34GiB to GPU in 1.5249s (22.2965GiB/s) | 20260507 15:23:29.895 | INFO | Analyzing chunk frames = [545792,546304) | 20260507 15:23:29.895 | INFO | MemInfo: { | 20260507 15:23:29.895 | INFO | "gpu" : " 17 GB", | 20260507 15:23:29.895 | INFO | "huge" : "", | 20260507 15:23:29.895 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:29.895 | INFO | "malloc" : " 14 GB", | 20260507 15:23:29.895 | INFO | "pinned" : " 29 GB" | 20260507 15:23:29.895 | INFO | } | 20260507 15:23:29.895 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:23:29.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:29.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:23:29.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:23:29.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:29.895 | INFO | Uploaded 36GiB to GPU in 1.60904s (22.3735GiB/s) | 20260507 15:23:29.895 | INFO | Downloaded 34GiB to GPU in 1.52921s (22.2337GiB/s) | 20260507 15:23:35.034 | INFO | Analyzing chunk frames = [546304,546816) | 20260507 15:23:35.034 | INFO | MemInfo: { | 20260507 15:23:35.034 | INFO | "gpu" : " 17 GB", | 20260507 15:23:35.034 | INFO | "huge" : "", | 20260507 15:23:35.034 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:35.034 | INFO | "malloc" : " 14 GB", | 20260507 15:23:35.034 | INFO | "pinned" : " 29 GB" | 20260507 15:23:35.034 | INFO | } | 20260507 15:23:35.034 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:23:35.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:35.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.034 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:23:35.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.034 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:23:35.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:35.035 | INFO | Uploaded 36GiB to GPU in 1.60989s (22.3618GiB/s) | 20260507 15:23:35.035 | INFO | Downloaded 34GiB to GPU in 1.52918s (22.2341GiB/s) | 20260507 15:23:40.174 | INFO | Analyzing chunk frames = [546816,547328) | 20260507 15:23:40.174 | INFO | MemInfo: { | 20260507 15:23:40.174 | INFO | "gpu" : " 17 GB", | 20260507 15:23:40.174 | INFO | "huge" : "", | 20260507 15:23:40.174 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:40.174 | INFO | "malloc" : " 14 GB", | 20260507 15:23:40.174 | INFO | "pinned" : " 29 GB" | 20260507 15:23:40.174 | INFO | } | 20260507 15:23:40.174 | INFO | Perf report for the last 5.13948 seconds: | 20260507 15:23:40.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:40.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:23:40.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:23:40.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:40.174 | INFO | Uploaded 36GiB to GPU in 1.60777s (22.3913GiB/s) | 20260507 15:23:40.174 | INFO | Downloaded 34GiB to GPU in 1.52668s (22.2706GiB/s) | 20260507 15:23:45.313 | INFO | Analyzing chunk frames = [547328,547840) | 20260507 15:23:45.313 | INFO | MemInfo: { | 20260507 15:23:45.313 | INFO | "gpu" : " 17 GB", | 20260507 15:23:45.313 | INFO | "huge" : "", | 20260507 15:23:45.313 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:45.313 | INFO | "malloc" : " 14 GB", | 20260507 15:23:45.313 | INFO | "pinned" : " 29 GB" | 20260507 15:23:45.313 | INFO | } | 20260507 15:23:45.313 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:23:45.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:45.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:23:45.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:23:45.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:45.313 | INFO | Uploaded 36GiB to GPU in 1.60814s (22.3862GiB/s) | 20260507 15:23:45.313 | INFO | Downloaded 34GiB to GPU in 1.52668s (22.2705GiB/s) | 20260507 15:23:50.452 | INFO | Analyzing chunk frames = [547840,548352) | 20260507 15:23:50.452 | INFO | MemInfo: { | 20260507 15:23:50.452 | INFO | "gpu" : " 17 GB", | 20260507 15:23:50.452 | INFO | "huge" : "", | 20260507 15:23:50.452 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:50.452 | INFO | "malloc" : " 14 GB", | 20260507 15:23:50.452 | INFO | "pinned" : " 29 GB" | 20260507 15:23:50.452 | INFO | } | 20260507 15:23:50.453 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:23:50.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:50.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:23:50.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:23:50.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:50.453 | INFO | Uploaded 36GiB to GPU in 1.60828s (22.3841GiB/s) | 20260507 15:23:50.453 | INFO | Downloaded 34GiB to GPU in 1.52861s (22.2424GiB/s) | 20260507 15:23:55.592 | INFO | Analyzing chunk frames = [548352,548864) | 20260507 15:23:55.592 | INFO | MemInfo: { | 20260507 15:23:55.592 | INFO | "gpu" : " 17 GB", | 20260507 15:23:55.592 | INFO | "huge" : "", | 20260507 15:23:55.592 | INFO | "huge_shared" : " 29 GB", | 20260507 15:23:55.592 | INFO | "malloc" : " 14 GB", | 20260507 15:23:55.592 | INFO | "pinned" : " 29 GB" | 20260507 15:23:55.592 | INFO | } | 20260507 15:23:55.592 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:23:55.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:23:55.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:23:55.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:23:55.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:23:55.592 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:23:55.592 | INFO | Total mem usage: 16.4 GiB | 20260507 15:23:55.592 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:23:55.592 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -2.89%) | 20260507 15:23:55.592 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:23:55.592 | INFO | High Water Breakdown: | 20260507 15:23:55.592 | INFO | 60.2% : 9.6 GiB ( 96.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:23:55.592 | INFO | Uploaded 36GiB to GPU in 1.60801s (22.3879GiB/s) | 20260507 15:23:55.592 | INFO | 39.6% : 6.32 GiB ( 94% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:23:55.592 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:23:55.592 | INFO | ------------------------------------------ | 20260507 15:23:55.592 | INFO | Downloaded 34GiB to GPU in 1.52832s (22.2466GiB/s) | 20260507 15:23:55.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:23:55.592 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:23:55.592 | INFO | Total mem usage: 26.5 GiB | 20260507 15:23:55.592 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:23:55.592 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:23:55.592 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:23:55.592 | INFO | High Water Breakdown: | 20260507 15:23:55.592 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:23:55.592 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:23:55.592 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:23:55.592 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:23:55.592 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:23:55.592 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:23:55.592 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:23:55.592 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:23:55.592 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:23:55.592 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:23:55.592 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:23:55.592 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:23:55.592 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:23:55.592 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:23:55.592 | INFO | ------------------------------------------ | 20260507 15:23:55.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:23:55.592 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:23:55.592 | INFO | Total mem usage: 27.8 GiB | 20260507 15:23:55.592 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:23:55.592 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:23:55.592 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:23:55.592 | INFO | High Water Breakdown: | 20260507 15:23:55.592 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:23:55.592 | INFO | ------------------------------------------ | 20260507 15:23:55.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:23:55.592 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:23:55.592 | INFO | Total mem usage: 15.7 GiB | 20260507 15:23:55.592 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:23:55.592 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:23:55.592 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:23:55.592 | INFO | High Water Breakdown: | 20260507 15:23:55.592 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:23:55.592 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:23:55.592 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:23:55.592 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:23:55.592 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:23:55.592 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:23:55.592 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:23:55.592 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:23:55.592 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:23:55.592 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:23:55.592 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:23:55.592 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:23:55.592 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:23:55.592 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:23:55.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:23:55.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:23:55.592 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:23:55.592 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:23:55.592 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:23:55.592 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:23:55.592 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:23:55.592 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:23:55.592 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:23:55.592 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:23:55.592 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:23:55.592 | INFO | ------------------------------------------ | 20260507 15:23:55.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:23:55.592 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:23:55.592 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:23:55.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:23:55.592 | INFO | PerfReport: Full Execution -- 2.99e+01 / 7.21e+01, 1.25e+01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / 8.52e-02, 5.31e-03 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 11.03%, Baseline -- 3.29e+00 / 3.27e+00, 1.99e-02 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 3.07%, Histogram -- 9.18e-01 / 1.06e+00, 5.19e-02 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 16.16%, DME -- 3.86e+01 / 4.69e+01, 3.14e+00 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.72e+01 / 1.72e+01, 4.34e-02 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.14e+00 / 2.19e+00, 1.22e-01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 3.47%, Metrics -- 1.04e+00 / 1.02e+00, 1.68e-01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.82e-01 / 4.08e-01, 5.57e-02 / 0.00e+00 | 20260507 15:23:55.592 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:23:55.592 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:23:55.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:23:55.592 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 18.76%, Upload -- 1.67e+01 / 0.00e+00, 5.47e-01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 18.20%, Download -- 1.62e+01 / 0.00e+00, 8.59e-01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 33.48%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 0.60%, LockUpload -- 5.33e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 0.00%, LockDownload -- 4.36e-04 / 0.00e+00, 1.40e-04 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 28.95%, LockCompute -- 2.58e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 15:23:55.592 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260507 15:23:55.689 | INFO | Starting a round of Baz IO | 20260507 15:23:58.578 | INFO | Populating RTMetrics report | 20260507 15:23:58.578 | INFO | RTMetrics report populated | 20260507 15:23:58.578 | INFO | Opening tmp RTMetrics json file | 20260507 15:23:58.656 | INFO | Serializing RTMetrics to json | 20260507 15:23:58.679 | INFO | Serialization done, swapping tmp file now... | 20260507 15:23:58.679 | INFO | RTMetrics done | 20260507 15:23:58.827 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.138 seconds | 20260507 15:23:58.827 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.53% / 0.05% / 46.42% | 20260507 15:23:58.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.91985% / 0.000119209% / 2.38419e-05% | 20260507 15:23:58.827 | INFO | Baz IO overall: | 20260507 15:23:58.827 | INFO | BytesWritten : 221757907030 | 20260507 15:23:58.827 | INFO | NumEvents : 106757792749 | 20260507 15:24:00.731 | INFO | Analyzing chunk frames = [548864,549376) | 20260507 15:24:00.731 | INFO | MemInfo: { | 20260507 15:24:00.731 | INFO | "gpu" : " 17 GB", | 20260507 15:24:00.731 | INFO | "huge" : "", | 20260507 15:24:00.731 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:00.731 | INFO | "malloc" : " 14 GB", | 20260507 15:24:00.731 | INFO | "pinned" : " 29 GB" | 20260507 15:24:00.731 | INFO | } | 20260507 15:24:00.731 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:24:00.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:00.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:00.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:00.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:00.731 | INFO | ANALYSIS : 57.2%, 2.9, 85.8ms, 0 | 20260507 15:24:00.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:00.731 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 15:24:00.731 | INFO | PRE_HQ : 55.9%, 2.1, 60.2ms, 0 | 20260507 15:24:00.731 | INFO | BAZWRITER : 60.6%, 4.5, 141.1ms, 0 | 20260507 15:24:00.731 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 15:24:00.732 | INFO | Uploaded 36GiB to GPU in 1.64761s (21.8498GiB/s) | 20260507 15:24:00.732 | INFO | Downloaded 38GiB to GPU in 1.73126s (21.9494GiB/s) | 20260507 15:24:05.870 | INFO | Analyzing chunk frames = [549376,549888) | 20260507 15:24:05.870 | INFO | MemInfo: { | 20260507 15:24:05.870 | INFO | "gpu" : " 17 GB", | 20260507 15:24:05.870 | INFO | "huge" : "", | 20260507 15:24:05.870 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:05.870 | INFO | "malloc" : " 14 GB", | 20260507 15:24:05.870 | INFO | "pinned" : " 29 GB" | 20260507 15:24:05.870 | INFO | } | 20260507 15:24:05.870 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:24:05.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:05.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:24:05.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | PRE_HQ : 27.6%, 1.0, 14.5ms, 0 | 20260507 15:24:05.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:05.871 | INFO | Uploaded 36GiB to GPU in 1.58374s (22.731GiB/s) | 20260507 15:24:05.871 | INFO | Downloaded 34GiB to GPU in 1.53391s (22.1656GiB/s) | 20260507 15:24:11.010 | INFO | Analyzing chunk frames = [549888,550400) | 20260507 15:24:11.010 | INFO | MemInfo: { | 20260507 15:24:11.010 | INFO | "gpu" : " 17 GB", | 20260507 15:24:11.010 | INFO | "huge" : "", | 20260507 15:24:11.010 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:11.010 | INFO | "malloc" : " 14 GB", | 20260507 15:24:11.010 | INFO | "pinned" : " 29 GB" | 20260507 15:24:11.010 | INFO | } | 20260507 15:24:11.010 | INFO | Perf report for the last 5.13925 seconds: | 20260507 15:24:11.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:11.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.010 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:24:11.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.010 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:24:11.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:11.011 | INFO | Uploaded 36GiB to GPU in 1.58856s (22.6621GiB/s) | 20260507 15:24:11.011 | INFO | Downloaded 34GiB to GPU in 1.53815s (22.1044GiB/s) | 20260507 15:24:16.149 | INFO | Analyzing chunk frames = [550400,550912) | 20260507 15:24:16.149 | INFO | MemInfo: { | 20260507 15:24:16.149 | INFO | "gpu" : " 17 GB", | 20260507 15:24:16.149 | INFO | "huge" : "", | 20260507 15:24:16.149 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:16.149 | INFO | "malloc" : " 14 GB", | 20260507 15:24:16.149 | INFO | "pinned" : " 29 GB" | 20260507 15:24:16.149 | INFO | } | 20260507 15:24:16.149 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:24:16.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:16.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.149 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:24:16.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.149 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:24:16.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:16.150 | INFO | Uploaded 36GiB to GPU in 1.58797s (22.6704GiB/s) | 20260507 15:24:16.150 | INFO | Downloaded 34GiB to GPU in 1.53903s (22.0918GiB/s) | 20260507 15:24:21.288 | INFO | Analyzing chunk frames = [550912,551424) | 20260507 15:24:21.288 | INFO | MemInfo: { | 20260507 15:24:21.288 | INFO | "gpu" : " 17 GB", | 20260507 15:24:21.288 | INFO | "huge" : "", | 20260507 15:24:21.288 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:21.288 | INFO | "malloc" : " 14 GB", | 20260507 15:24:21.288 | INFO | "pinned" : " 29 GB" | 20260507 15:24:21.288 | INFO | } | 20260507 15:24:21.288 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:24:21.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:21.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.288 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:24:21.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.289 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:24:21.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:21.289 | INFO | Uploaded 36GiB to GPU in 1.58757s (22.6762GiB/s) | 20260507 15:24:21.289 | INFO | Downloaded 34GiB to GPU in 1.53539s (22.1442GiB/s) | 20260507 15:24:26.428 | INFO | Analyzing chunk frames = [551424,551936) | 20260507 15:24:26.428 | INFO | MemInfo: { | 20260507 15:24:26.428 | INFO | "gpu" : " 17 GB", | 20260507 15:24:26.428 | INFO | "huge" : "", | 20260507 15:24:26.428 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:26.428 | INFO | "malloc" : " 14 GB", | 20260507 15:24:26.428 | INFO | "pinned" : " 29 GB" | 20260507 15:24:26.428 | INFO | } | 20260507 15:24:26.428 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:24:26.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:26.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:24:26.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:24:26.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:26.428 | INFO | Uploaded 36GiB to GPU in 1.58733s (22.6796GiB/s) | 20260507 15:24:26.428 | INFO | Downloaded 34GiB to GPU in 1.53652s (22.1279GiB/s) | 20260507 15:24:31.567 | INFO | Analyzing chunk frames = [551936,552448) | 20260507 15:24:31.567 | INFO | MemInfo: { | 20260507 15:24:31.567 | INFO | "gpu" : " 17 GB", | 20260507 15:24:31.567 | INFO | "huge" : "", | 20260507 15:24:31.567 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:31.567 | INFO | "malloc" : " 14 GB", | 20260507 15:24:31.567 | INFO | "pinned" : " 29 GB" | 20260507 15:24:31.567 | INFO | } | 20260507 15:24:31.567 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:24:31.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:31.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:24:31.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:24:31.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:31.567 | INFO | Uploaded 36GiB to GPU in 1.588s (22.67GiB/s) | 20260507 15:24:31.568 | INFO | Downloaded 34GiB to GPU in 1.53782s (22.1092GiB/s) | 20260507 15:24:36.706 | INFO | Analyzing chunk frames = [552448,552960) | 20260507 15:24:36.706 | INFO | MemInfo: { | 20260507 15:24:36.706 | INFO | "gpu" : " 17 GB", | 20260507 15:24:36.706 | INFO | "huge" : "", | 20260507 15:24:36.706 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:36.707 | INFO | "malloc" : " 14 GB", | 20260507 15:24:36.707 | INFO | "pinned" : " 29 GB" | 20260507 15:24:36.707 | INFO | } | 20260507 15:24:36.707 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:24:36.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:36.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:24:36.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:24:36.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:36.707 | INFO | Uploaded 36GiB to GPU in 1.58817s (22.6675GiB/s) | 20260507 15:24:36.707 | INFO | Downloaded 34GiB to GPU in 1.53753s (22.1134GiB/s) | 20260507 15:24:36.804 | INFO | Starting a round of Baz IO | 20260507 15:24:39.713 | INFO | Populating RTMetrics report | 20260507 15:24:39.713 | INFO | RTMetrics report populated | 20260507 15:24:39.713 | INFO | Opening tmp RTMetrics json file | 20260507 15:24:39.754 | INFO | Serializing RTMetrics to json | 20260507 15:24:39.759 | INFO | Serialization done, swapping tmp file now... | 20260507 15:24:39.763 | INFO | RTMetrics done | 20260507 15:24:40.082 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.278 seconds | 20260507 15:24:40.082 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.49% / 0.05% / 46.47% | 20260507 15:24:40.082 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.89627% / 8.74201e-05% / 3.57628e-05% | 20260507 15:24:40.082 | INFO | Baz IO overall: | 20260507 15:24:40.082 | INFO | BytesWritten : 223632090156 | 20260507 15:24:40.082 | INFO | NumEvents : 107670702207 | 20260507 15:24:41.846 | INFO | Analyzing chunk frames = [552960,553472) | 20260507 15:24:41.846 | INFO | MemInfo: { | 20260507 15:24:41.846 | INFO | "gpu" : " 17 GB", | 20260507 15:24:41.846 | INFO | "huge" : "", | 20260507 15:24:41.846 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:41.846 | INFO | "malloc" : " 14 GB", | 20260507 15:24:41.846 | INFO | "pinned" : " 29 GB" | 20260507 15:24:41.846 | INFO | } | 20260507 15:24:41.846 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:24:41.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:41.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:41.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:41.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:41.846 | INFO | ANALYSIS : 57.6%, 2.8, 85.1ms, 0 | 20260507 15:24:41.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:41.847 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 15:24:41.847 | INFO | PRE_HQ : 56.5%, 2.1, 61.2ms, 0 | 20260507 15:24:41.847 | INFO | BAZWRITER : 63.4%, 4.7, 154.2ms, 0 | 20260507 15:24:41.847 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:24:41.847 | INFO | Uploaded 36GiB to GPU in 1.65325s (21.7752GiB/s) | 20260507 15:24:41.847 | INFO | Downloaded 38GiB to GPU in 1.73723s (21.8739GiB/s) | 20260507 15:24:46.985 | INFO | Analyzing chunk frames = [553472,553984) | 20260507 15:24:46.985 | INFO | MemInfo: { | 20260507 15:24:46.985 | INFO | "gpu" : " 17 GB", | 20260507 15:24:46.985 | INFO | "huge" : "", | 20260507 15:24:46.985 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:46.986 | INFO | "malloc" : " 14 GB", | 20260507 15:24:46.986 | INFO | "pinned" : " 29 GB" | 20260507 15:24:46.986 | INFO | } | 20260507 15:24:46.986 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:24:46.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:46.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 15:24:46.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | PRE_HQ : 39.7%, 1.0, 20.9ms, 0 | 20260507 15:24:46.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:46.986 | INFO | Uploaded 36GiB to GPU in 1.60734s (22.3972GiB/s) | 20260507 15:24:46.986 | INFO | Downloaded 34GiB to GPU in 1.53116s (22.2054GiB/s) | 20260507 15:24:52.125 | INFO | Analyzing chunk frames = [553984,554496) | 20260507 15:24:52.125 | INFO | MemInfo: { | 20260507 15:24:52.125 | INFO | "gpu" : " 17 GB", | 20260507 15:24:52.125 | INFO | "huge" : "", | 20260507 15:24:52.125 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:52.125 | INFO | "malloc" : " 14 GB", | 20260507 15:24:52.125 | INFO | "pinned" : " 29 GB" | 20260507 15:24:52.125 | INFO | } | 20260507 15:24:52.125 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:24:52.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:52.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.125 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:24:52.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.125 | INFO | PRE_HQ : 39.8%, 1.0, 20.7ms, 0 | 20260507 15:24:52.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:52.126 | INFO | Uploaded 36GiB to GPU in 1.61561s (22.2826GiB/s) | 20260507 15:24:52.126 | INFO | Downloaded 34GiB to GPU in 1.53473s (22.1537GiB/s) | 20260507 15:24:57.264 | INFO | Analyzing chunk frames = [554496,555008) | 20260507 15:24:57.264 | INFO | MemInfo: { | 20260507 15:24:57.264 | INFO | "gpu" : " 17 GB", | 20260507 15:24:57.264 | INFO | "huge" : "", | 20260507 15:24:57.264 | INFO | "huge_shared" : " 29 GB", | 20260507 15:24:57.264 | INFO | "malloc" : " 14 GB", | 20260507 15:24:57.264 | INFO | "pinned" : " 29 GB" | 20260507 15:24:57.264 | INFO | } | 20260507 15:24:57.264 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:24:57.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:24:57.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.264 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 15:24:57.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.265 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 15:24:57.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:24:57.265 | INFO | Uploaded 36GiB to GPU in 1.61501s (22.2908GiB/s) | 20260507 15:24:57.265 | INFO | Downloaded 34GiB to GPU in 1.53417s (22.1619GiB/s) | 20260507 15:25:02.403 | INFO | Analyzing chunk frames = [555008,555520) | 20260507 15:25:02.403 | INFO | MemInfo: { | 20260507 15:25:02.403 | INFO | "gpu" : " 17 GB", | 20260507 15:25:02.403 | INFO | "huge" : "", | 20260507 15:25:02.403 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:02.403 | INFO | "malloc" : " 14 GB", | 20260507 15:25:02.403 | INFO | "pinned" : " 29 GB" | 20260507 15:25:02.403 | INFO | } | 20260507 15:25:02.404 | INFO | Perf report for the last 5.13913 seconds: | 20260507 15:25:02.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:02.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:25:02.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260507 15:25:02.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:02.404 | INFO | Uploaded 36GiB to GPU in 1.60963s (22.3654GiB/s) | 20260507 15:25:02.404 | INFO | Downloaded 34GiB to GPU in 1.52876s (22.2402GiB/s) | 20260507 15:25:07.543 | INFO | Analyzing chunk frames = [555520,556032) | 20260507 15:25:07.543 | INFO | MemInfo: { | 20260507 15:25:07.543 | INFO | "gpu" : " 17 GB", | 20260507 15:25:07.543 | INFO | "huge" : "", | 20260507 15:25:07.543 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:07.543 | INFO | "malloc" : " 14 GB", | 20260507 15:25:07.543 | INFO | "pinned" : " 29 GB" | 20260507 15:25:07.543 | INFO | } | 20260507 15:25:07.543 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:25:07.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:07.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:25:07.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:25:07.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:07.543 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4073GiB/s) | 20260507 15:25:07.543 | INFO | Downloaded 34GiB to GPU in 1.52666s (22.2709GiB/s) | 20260507 15:25:12.682 | INFO | Analyzing chunk frames = [556032,556544) | 20260507 15:25:12.682 | INFO | MemInfo: { | 20260507 15:25:12.682 | INFO | "gpu" : " 17 GB", | 20260507 15:25:12.682 | INFO | "huge" : "", | 20260507 15:25:12.682 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:12.682 | INFO | "malloc" : " 14 GB", | 20260507 15:25:12.682 | INFO | "pinned" : " 29 GB" | 20260507 15:25:12.682 | INFO | } | 20260507 15:25:12.682 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:25:12.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:12.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:25:12.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:25:12.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:12.682 | INFO | Uploaded 36GiB to GPU in 1.60954s (22.3666GiB/s) | 20260507 15:25:12.683 | INFO | Downloaded 34GiB to GPU in 1.52831s (22.2468GiB/s) | 20260507 15:25:17.821 | INFO | Analyzing chunk frames = [556544,557056) | 20260507 15:25:17.821 | INFO | MemInfo: { | 20260507 15:25:17.821 | INFO | "gpu" : " 17 GB", | 20260507 15:25:17.821 | INFO | "huge" : "", | 20260507 15:25:17.821 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:17.821 | INFO | "malloc" : " 14 GB", | 20260507 15:25:17.821 | INFO | "pinned" : " 29 GB" | 20260507 15:25:17.821 | INFO | } | 20260507 15:25:17.822 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:25:17.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:17.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:25:17.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:25:17.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:17.822 | INFO | Uploaded 36GiB to GPU in 1.60811s (22.3866GiB/s) | 20260507 15:25:17.822 | INFO | Downloaded 34GiB to GPU in 1.52922s (22.2335GiB/s) | 20260507 15:25:17.822 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:25:17.822 | INFO | Total mem usage: 16.4 GiB | 20260507 15:25:17.822 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:25:17.822 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.5%) | 20260507 15:25:17.822 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:25:17.822 | INFO | High Water Breakdown: | 20260507 15:25:17.822 | INFO | 59.8% : 9.3 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:25:17.822 | INFO | 40% : 6.23 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:25:17.822 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:25:17.822 | INFO | ------------------------------------------ | 20260507 15:25:17.822 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:25:17.822 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:25:17.822 | INFO | Total mem usage: 26.5 GiB | 20260507 15:25:17.822 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:25:17.822 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:25:17.822 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:25:17.822 | INFO | High Water Breakdown: | 20260507 15:25:17.822 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:25:17.822 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:25:17.822 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:25:17.822 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:25:17.822 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:25:17.822 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:25:17.822 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:25:17.822 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:25:17.822 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:25:17.822 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:25:17.822 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:25:17.822 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:25:17.822 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:25:17.823 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:25:17.823 | INFO | ------------------------------------------ | 20260507 15:25:17.823 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:25:17.823 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:25:17.823 | INFO | Total mem usage: 27.8 GiB | 20260507 15:25:17.823 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:25:17.823 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:25:17.823 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:25:17.823 | INFO | High Water Breakdown: | 20260507 15:25:17.823 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:25:17.823 | INFO | ------------------------------------------ | 20260507 15:25:17.823 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:25:17.823 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:25:17.823 | INFO | Total mem usage: 15.7 GiB | 20260507 15:25:17.823 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:25:17.823 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:25:17.823 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 15:25:17.823 | INFO | High Water Breakdown: | 20260507 15:25:17.823 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:25:17.823 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:25:17.823 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:25:17.823 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:25:17.823 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:25:17.823 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:25:17.823 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:25:17.823 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:25:17.823 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:25:17.823 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:25:17.823 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:25:17.823 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:25:17.823 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:25:17.823 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:25:17.823 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:25:17.823 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:25:17.823 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:25:17.823 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:25:17.823 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:25:17.823 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:25:17.823 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:25:17.823 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:25:17.823 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:25:17.823 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:25:17.823 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:25:17.823 | INFO | ------------------------------------------ | 20260507 15:25:17.823 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:25:17.823 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:25:17.823 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:25:17.823 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:25:17.823 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 0.29%, AggregateStats -- 8.81e-02 / -nan, 7.15e-03 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 11.05%, Baseline -- 3.30e+00 / -nan, 2.28e-02 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 3.08%, Histogram -- 9.20e-01 / -nan, 5.31e-02 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 16.12%, DME -- 3.85e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.72e+01 / -nan, 4.41e-02 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.15e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 3.47%, Metrics -- 1.04e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.84e-01 / -nan, 5.58e-02 / 0.00e+00 | 20260507 15:25:17.823 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:25:17.823 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:25:17.823 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:25:17.823 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 18.72%, Upload -- 1.67e+01 / 0.00e+00, 4.78e-01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 18.18%, Download -- 1.62e+01 / 0.00e+00, 7.97e-01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 33.46%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 0.58%, LockUpload -- 5.17e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 3.32e-04 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 29.05%, LockCompute -- 2.59e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 15:25:17.823 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260507 15:25:17.918 | INFO | Starting a round of Baz IO | 20260507 15:25:20.813 | INFO | Populating RTMetrics report | 20260507 15:25:20.813 | INFO | RTMetrics report populated | 20260507 15:25:20.813 | INFO | Opening tmp RTMetrics json file | 20260507 15:25:20.891 | INFO | Serializing RTMetrics to json | 20260507 15:25:20.921 | INFO | Serialization done, swapping tmp file now... | 20260507 15:25:20.924 | INFO | RTMetrics done | 20260507 15:25:21.086 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.167 seconds | 20260507 15:25:21.086 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.44% / 0.04% / 46.51% | 20260507 15:25:21.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.89214% / 6.35783e-05% / 1.98682e-05% | 20260507 15:25:21.087 | INFO | Baz IO overall: | 20260507 15:25:21.087 | INFO | BytesWritten : 225508176951 | 20260507 15:25:21.087 | INFO | NumEvents : 108584772868 | 20260507 15:25:22.961 | INFO | Analyzing chunk frames = [557056,557568) | 20260507 15:25:22.961 | INFO | MemInfo: { | 20260507 15:25:22.961 | INFO | "gpu" : " 17 GB", | 20260507 15:25:22.961 | INFO | "huge" : "", | 20260507 15:25:22.961 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:22.961 | INFO | "malloc" : " 14 GB", | 20260507 15:25:22.961 | INFO | "pinned" : " 29 GB" | 20260507 15:25:22.961 | INFO | } | 20260507 15:25:22.961 | INFO | Perf report for the last 5.13908 seconds: | 20260507 15:25:22.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:22.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:22.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:22.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:22.961 | INFO | ANALYSIS : 57.3%, 2.8, 85.4ms, 0 | 20260507 15:25:22.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:22.961 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 15:25:22.961 | INFO | PRE_HQ : 56.2%, 2.1, 60.5ms, 0 | 20260507 15:25:22.961 | INFO | BAZWRITER : 61.1%, 4.8, 149.7ms, 0 | 20260507 15:25:22.961 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 15:25:22.961 | INFO | Uploaded 36GiB to GPU in 1.66469s (21.6256GiB/s) | 20260507 15:25:22.961 | INFO | Downloaded 38GiB to GPU in 1.74862s (21.7314GiB/s) | 20260507 15:25:28.100 | INFO | Analyzing chunk frames = [557568,558080) | 20260507 15:25:28.100 | INFO | MemInfo: { | 20260507 15:25:28.100 | INFO | "gpu" : " 17 GB", | 20260507 15:25:28.100 | INFO | "huge" : "", | 20260507 15:25:28.100 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:28.100 | INFO | "malloc" : " 14 GB", | 20260507 15:25:28.100 | INFO | "pinned" : " 29 GB" | 20260507 15:25:28.100 | INFO | } | 20260507 15:25:28.100 | INFO | Perf report for the last 5.13952 seconds: | 20260507 15:25:28.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:28.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:25:28.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:25:28.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:28.100 | INFO | Uploaded 36GiB to GPU in 1.58491s (22.7142GiB/s) | 20260507 15:25:28.101 | INFO | Downloaded 34GiB to GPU in 1.53346s (22.1721GiB/s) | 20260507 15:25:33.239 | INFO | Analyzing chunk frames = [558080,558592) | 20260507 15:25:33.239 | INFO | MemInfo: { | 20260507 15:25:33.239 | INFO | "gpu" : " 17 GB", | 20260507 15:25:33.239 | INFO | "huge" : "", | 20260507 15:25:33.239 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:33.239 | INFO | "malloc" : " 14 GB", | 20260507 15:25:33.239 | INFO | "pinned" : " 29 GB" | 20260507 15:25:33.239 | INFO | } | 20260507 15:25:33.239 | INFO | Perf report for the last 5.13908 seconds: | 20260507 15:25:33.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:33.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.239 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:25:33.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.240 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:25:33.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:33.240 | INFO | Uploaded 36GiB to GPU in 1.58982s (22.644GiB/s) | 20260507 15:25:33.240 | INFO | Downloaded 34GiB to GPU in 1.5398s (22.0808GiB/s) | 20260507 15:25:38.379 | INFO | Analyzing chunk frames = [558592,559104) | 20260507 15:25:38.379 | INFO | MemInfo: { | 20260507 15:25:38.379 | INFO | "gpu" : " 17 GB", | 20260507 15:25:38.379 | INFO | "huge" : "", | 20260507 15:25:38.379 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:38.379 | INFO | "malloc" : " 14 GB", | 20260507 15:25:38.379 | INFO | "pinned" : " 29 GB" | 20260507 15:25:38.379 | INFO | } | 20260507 15:25:38.379 | INFO | Perf report for the last 5.13953 seconds: | 20260507 15:25:38.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:38.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:25:38.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:25:38.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:38.379 | INFO | Uploaded 36GiB to GPU in 1.58751s (22.677GiB/s) | 20260507 15:25:38.380 | INFO | Downloaded 34GiB to GPU in 1.53745s (22.1145GiB/s) | 20260507 15:25:43.518 | INFO | Analyzing chunk frames = [559104,559616) | 20260507 15:25:43.518 | INFO | MemInfo: { | 20260507 15:25:43.518 | INFO | "gpu" : " 17 GB", | 20260507 15:25:43.518 | INFO | "huge" : "", | 20260507 15:25:43.518 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:43.518 | INFO | "malloc" : " 14 GB", | 20260507 15:25:43.518 | INFO | "pinned" : " 29 GB" | 20260507 15:25:43.518 | INFO | } | 20260507 15:25:43.518 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:25:43.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:43.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.518 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:25:43.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.518 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:25:43.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:43.519 | INFO | Uploaded 36GiB to GPU in 1.58706s (22.6835GiB/s) | 20260507 15:25:43.519 | INFO | Downloaded 34GiB to GPU in 1.53693s (22.122GiB/s) | 20260507 15:25:48.657 | INFO | Analyzing chunk frames = [559616,560128) | 20260507 15:25:48.657 | INFO | MemInfo: { | 20260507 15:25:48.657 | INFO | "gpu" : " 17 GB", | 20260507 15:25:48.657 | INFO | "huge" : "", | 20260507 15:25:48.657 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:48.657 | INFO | "malloc" : " 14 GB", | 20260507 15:25:48.657 | INFO | "pinned" : " 29 GB" | 20260507 15:25:48.657 | INFO | } | 20260507 15:25:48.658 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:25:48.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:48.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:25:48.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:25:48.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:48.658 | INFO | Uploaded 36GiB to GPU in 1.58744s (22.678GiB/s) | 20260507 15:25:48.658 | INFO | Downloaded 34GiB to GPU in 1.53577s (22.1387GiB/s) | 20260507 15:25:53.797 | INFO | Analyzing chunk frames = [560128,560640) | 20260507 15:25:53.797 | INFO | MemInfo: { | 20260507 15:25:53.797 | INFO | "gpu" : " 17 GB", | 20260507 15:25:53.797 | INFO | "huge" : "", | 20260507 15:25:53.797 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:53.797 | INFO | "malloc" : " 14 GB", | 20260507 15:25:53.797 | INFO | "pinned" : " 29 GB" | 20260507 15:25:53.797 | INFO | } | 20260507 15:25:53.797 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:25:53.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:53.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.797 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:25:53.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.797 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:25:53.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:53.798 | INFO | Uploaded 36GiB to GPU in 1.58823s (22.6667GiB/s) | 20260507 15:25:53.798 | INFO | Downloaded 34GiB to GPU in 1.53777s (22.11GiB/s) | 20260507 15:25:58.936 | INFO | Analyzing chunk frames = [560640,561152) | 20260507 15:25:58.936 | INFO | MemInfo: { | 20260507 15:25:58.936 | INFO | "gpu" : " 17 GB", | 20260507 15:25:58.936 | INFO | "huge" : "", | 20260507 15:25:58.936 | INFO | "huge_shared" : " 29 GB", | 20260507 15:25:58.936 | INFO | "malloc" : " 14 GB", | 20260507 15:25:58.936 | INFO | "pinned" : " 29 GB" | 20260507 15:25:58.936 | INFO | } | 20260507 15:25:58.936 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:25:58.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:25:58.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.936 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:25:58.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.936 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 15:25:58.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:25:58.937 | INFO | Uploaded 36GiB to GPU in 1.58901s (22.6556GiB/s) | 20260507 15:25:58.937 | INFO | Downloaded 34GiB to GPU in 1.53779s (22.1096GiB/s) | 20260507 15:25:59.034 | INFO | Starting a round of Baz IO | 20260507 15:26:01.953 | INFO | Populating RTMetrics report | 20260507 15:26:01.953 | INFO | RTMetrics report populated | 20260507 15:26:01.953 | INFO | Opening tmp RTMetrics json file | 20260507 15:26:01.954 | INFO | Serializing RTMetrics to json | 20260507 15:26:01.956 | INFO | Serialization done, swapping tmp file now... | 20260507 15:26:01.957 | INFO | RTMetrics done | 20260507 15:26:02.045 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.011 seconds | 20260507 15:26:02.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.40% / 0.05% / 46.56% | 20260507 15:26:02.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.88056% / 0.000115236% / 7.94729e-06% | 20260507 15:26:02.045 | INFO | Baz IO overall: | 20260507 15:26:02.045 | INFO | BytesWritten : 227387156937 | 20260507 15:26:02.046 | INFO | NumEvents : 109500299192 | 20260507 15:26:04.075 | INFO | Analyzing chunk frames = [561152,561664) | 20260507 15:26:04.075 | INFO | MemInfo: { | 20260507 15:26:04.075 | INFO | "gpu" : " 17 GB", | 20260507 15:26:04.075 | INFO | "huge" : "", | 20260507 15:26:04.075 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:04.075 | INFO | "malloc" : " 14 GB", | 20260507 15:26:04.075 | INFO | "pinned" : " 29 GB" | 20260507 15:26:04.075 | INFO | } | 20260507 15:26:04.075 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:26:04.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:04.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:04.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:04.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:04.076 | INFO | ANALYSIS : 57.1%, 2.9, 86.1ms, 0 | 20260507 15:26:04.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:04.076 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 15:26:04.076 | INFO | PRE_HQ : 56.3%, 2.1, 60.2ms, 0 | 20260507 15:26:04.076 | INFO | BAZWRITER : 58.2%, 4.6, 137.2ms, 0 | 20260507 15:26:04.076 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 15:26:04.076 | INFO | Uploaded 36GiB to GPU in 1.65634s (21.7346GiB/s) | 20260507 15:26:04.076 | INFO | Downloaded 38GiB to GPU in 1.73921s (21.849GiB/s) | 20260507 15:26:09.215 | INFO | Analyzing chunk frames = [561664,562176) | 20260507 15:26:09.215 | INFO | MemInfo: { | 20260507 15:26:09.215 | INFO | "gpu" : " 17 GB", | 20260507 15:26:09.215 | INFO | "huge" : "", | 20260507 15:26:09.215 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:09.215 | INFO | "malloc" : " 14 GB", | 20260507 15:26:09.215 | INFO | "pinned" : " 29 GB" | 20260507 15:26:09.215 | INFO | } | 20260507 15:26:09.215 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:26:09.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:09.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:26:09.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:26:09.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:09.215 | INFO | Uploaded 36GiB to GPU in 1.58516s (22.7107GiB/s) | 20260507 15:26:09.215 | INFO | Downloaded 34GiB to GPU in 1.533s (22.1787GiB/s) | 20260507 15:26:14.354 | INFO | Analyzing chunk frames = [562176,562688) | 20260507 15:26:14.354 | INFO | MemInfo: { | 20260507 15:26:14.354 | INFO | "gpu" : " 17 GB", | 20260507 15:26:14.354 | INFO | "huge" : "", | 20260507 15:26:14.354 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:14.354 | INFO | "malloc" : " 14 GB", | 20260507 15:26:14.354 | INFO | "pinned" : " 29 GB" | 20260507 15:26:14.354 | INFO | } | 20260507 15:26:14.354 | INFO | Perf report for the last 5.13908 seconds: | 20260507 15:26:14.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:14.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.354 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:26:14.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.354 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:26:14.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:14.355 | INFO | Uploaded 36GiB to GPU in 1.58879s (22.6587GiB/s) | 20260507 15:26:14.355 | INFO | Downloaded 34GiB to GPU in 1.53901s (22.0922GiB/s) | 20260507 15:26:19.493 | INFO | Analyzing chunk frames = [562688,563200) | 20260507 15:26:19.493 | INFO | MemInfo: { | 20260507 15:26:19.493 | INFO | "gpu" : " 17 GB", | 20260507 15:26:19.493 | INFO | "huge" : "", | 20260507 15:26:19.493 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:19.493 | INFO | "malloc" : " 14 GB", | 20260507 15:26:19.493 | INFO | "pinned" : " 29 GB" | 20260507 15:26:19.493 | INFO | } | 20260507 15:26:19.493 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:26:19.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:19.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:26:19.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:26:19.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:19.494 | INFO | Uploaded 36GiB to GPU in 1.58973s (22.6453GiB/s) | 20260507 15:26:19.494 | INFO | Downloaded 34GiB to GPU in 1.53741s (22.1151GiB/s) | 20260507 15:26:24.633 | INFO | Analyzing chunk frames = [563200,563712) | 20260507 15:26:24.633 | INFO | MemInfo: { | 20260507 15:26:24.633 | INFO | "gpu" : " 17 GB", | 20260507 15:26:24.633 | INFO | "huge" : "", | 20260507 15:26:24.633 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:24.633 | INFO | "malloc" : " 14 GB", | 20260507 15:26:24.633 | INFO | "pinned" : " 29 GB" | 20260507 15:26:24.633 | INFO | } | 20260507 15:26:24.633 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:26:24.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:24.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.633 | INFO | ANALYSIS : 55.9%, 3.0, 87.3ms, 0 | 20260507 15:26:24.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.633 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:26:24.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:24.634 | INFO | Uploaded 36GiB to GPU in 1.58782s (22.6726GiB/s) | 20260507 15:26:24.634 | INFO | Downloaded 34GiB to GPU in 1.53544s (22.1435GiB/s) | 20260507 15:26:29.772 | INFO | Analyzing chunk frames = [563712,564224) | 20260507 15:26:29.772 | INFO | MemInfo: { | 20260507 15:26:29.772 | INFO | "gpu" : " 17 GB", | 20260507 15:26:29.772 | INFO | "huge" : "", | 20260507 15:26:29.772 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:29.772 | INFO | "malloc" : " 14 GB", | 20260507 15:26:29.772 | INFO | "pinned" : " 29 GB" | 20260507 15:26:29.772 | INFO | } | 20260507 15:26:29.772 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:26:29.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:29.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.772 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:26:29.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.772 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:26:29.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:29.773 | INFO | Uploaded 36GiB to GPU in 1.58888s (22.6575GiB/s) | 20260507 15:26:29.773 | INFO | Downloaded 34GiB to GPU in 1.53591s (22.1368GiB/s) | 20260507 15:26:34.911 | INFO | Analyzing chunk frames = [564224,564736) | 20260507 15:26:34.911 | INFO | MemInfo: { | 20260507 15:26:34.911 | INFO | "gpu" : " 17 GB", | 20260507 15:26:34.911 | INFO | "huge" : "", | 20260507 15:26:34.911 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:34.911 | INFO | "malloc" : " 14 GB", | 20260507 15:26:34.911 | INFO | "pinned" : " 29 GB" | 20260507 15:26:34.911 | INFO | } | 20260507 15:26:34.912 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:26:34.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:34.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:26:34.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:26:34.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:34.912 | INFO | Uploaded 36GiB to GPU in 1.58784s (22.6723GiB/s) | 20260507 15:26:34.912 | INFO | Downloaded 34GiB to GPU in 1.53708s (22.1198GiB/s) | 20260507 15:26:40.051 | INFO | Analyzing chunk frames = [564736,565248) | 20260507 15:26:40.051 | INFO | MemInfo: { | 20260507 15:26:40.051 | INFO | "gpu" : " 17 GB", | 20260507 15:26:40.051 | INFO | "huge" : "", | 20260507 15:26:40.051 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:40.051 | INFO | "malloc" : " 14 GB", | 20260507 15:26:40.051 | INFO | "pinned" : " 29 GB" | 20260507 15:26:40.051 | INFO | } | 20260507 15:26:40.051 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:26:40.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:40.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.051 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:26:40.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.051 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:26:40.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:40.052 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:26:40.052 | INFO | Total mem usage: 16.4 GiB | 20260507 15:26:40.052 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:26:40.052 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.19%) | 20260507 15:26:40.052 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:26:40.052 | INFO | High Water Breakdown: | 20260507 15:26:40.052 | INFO | 59.6% : 9.12 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:26:40.052 | INFO | 40.3% : 6.17 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:26:40.052 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:26:40.052 | INFO | ------------------------------------------ | 20260507 15:26:40.052 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:26:40.052 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:26:40.052 | INFO | Total mem usage: 26.5 GiB | 20260507 15:26:40.052 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:26:40.052 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:26:40.052 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:26:40.052 | INFO | High Water Breakdown: | 20260507 15:26:40.052 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:26:40.052 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:26:40.052 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:26:40.052 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:26:40.052 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:26:40.052 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:26:40.052 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:26:40.052 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:26:40.052 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:26:40.052 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:26:40.052 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:26:40.052 | INFO | Uploaded 36GiB to GPU in 1.58801s (22.6699GiB/s) | 20260507 15:26:40.052 | INFO | Downloaded 34GiB to GPU in 1.53753s (22.1133GiB/s) | 20260507 15:26:40.052 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:26:40.052 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:26:40.052 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:26:40.052 | INFO | ------------------------------------------ | 20260507 15:26:40.052 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:26:40.052 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:26:40.052 | INFO | Total mem usage: 27.8 GiB | 20260507 15:26:40.052 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:26:40.052 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:26:40.052 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:26:40.052 | INFO | High Water Breakdown: | 20260507 15:26:40.052 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:26:40.052 | INFO | ------------------------------------------ | 20260507 15:26:40.052 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:26:40.052 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:26:40.052 | INFO | Total mem usage: 15.7 GiB | 20260507 15:26:40.052 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:26:40.052 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 15:26:40.052 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:26:40.052 | INFO | High Water Breakdown: | 20260507 15:26:40.052 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:26:40.052 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:26:40.052 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:26:40.052 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:26:40.052 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:26:40.052 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:26:40.052 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:26:40.052 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:26:40.052 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:26:40.052 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:26:40.052 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:26:40.052 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:26:40.052 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:26:40.052 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:26:40.052 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:26:40.052 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:26:40.052 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:26:40.053 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:26:40.053 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:26:40.053 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:26:40.053 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:26:40.053 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:26:40.053 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:26:40.053 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:26:40.053 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:26:40.053 | INFO | ------------------------------------------ | 20260507 15:26:40.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:26:40.053 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:26:40.053 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:26:40.053 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:26:40.053 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 0.29%, AggregateStats -- 8.70e-02 / -nan, 7.87e-03 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 11.03%, Baseline -- 3.29e+00 / -nan, 1.82e-02 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 16.14%, DME -- 3.86e+01 / -nan, 2.92e+00 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.14e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 3.47%, Metrics -- 1.04e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.81e-01 / -nan, 5.62e-02 / 0.00e+00 | 20260507 15:26:40.053 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:26:40.053 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:26:40.053 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:26:40.053 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 18.63%, Upload -- 1.66e+01 / 0.00e+00, 5.06e-01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 18.23%, Download -- 1.63e+01 / 0.00e+00, 8.12e-01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 33.44%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 0.00%, LockDownload -- 4.45e-04 / 0.00e+00, 2.36e-04 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 29.12%, LockCompute -- 2.60e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260507 15:26:40.053 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 15:26:40.148 | INFO | Starting a round of Baz IO | 20260507 15:26:43.053 | INFO | Populating RTMetrics report | 20260507 15:26:43.053 | INFO | RTMetrics report populated | 20260507 15:26:43.053 | INFO | Opening tmp RTMetrics json file | 20260507 15:26:43.054 | INFO | Serializing RTMetrics to json | 20260507 15:26:43.056 | INFO | Serialization done, swapping tmp file now... | 20260507 15:26:43.056 | INFO | RTMetrics done | 20260507 15:26:43.163 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.014 seconds | 20260507 15:26:43.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.35% / 0.04% / 46.60% | 20260507 15:26:43.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.86422% / 0.000123183% / 2.38419e-05% | 20260507 15:26:43.163 | INFO | Baz IO overall: | 20260507 15:26:43.163 | INFO | BytesWritten : 229266719011 | 20260507 15:26:43.163 | INFO | NumEvents : 110416091308 | 20260507 15:26:45.190 | INFO | Analyzing chunk frames = [565248,565760) | 20260507 15:26:45.190 | INFO | MemInfo: { | 20260507 15:26:45.190 | INFO | "gpu" : " 17 GB", | 20260507 15:26:45.190 | INFO | "huge" : "", | 20260507 15:26:45.190 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:45.190 | INFO | "malloc" : " 14 GB", | 20260507 15:26:45.190 | INFO | "pinned" : " 29 GB" | 20260507 15:26:45.190 | INFO | } | 20260507 15:26:45.190 | INFO | Perf report for the last 5.13897 seconds: | 20260507 15:26:45.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:45.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:45.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:45.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:45.190 | INFO | ANALYSIS : 57.6%, 2.9, 86.8ms, 0 | 20260507 15:26:45.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:45.190 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 15:26:45.191 | INFO | PRE_HQ : 56.3%, 2.1, 60.6ms, 0 | 20260507 15:26:45.191 | INFO | BAZWRITER : 58.3%, 4.6, 136.9ms, 0 | 20260507 15:26:45.191 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 15:26:45.191 | INFO | Uploaded 36GiB to GPU in 1.64617s (21.8689GiB/s) | 20260507 15:26:45.191 | INFO | Downloaded 38GiB to GPU in 1.7274s (21.9983GiB/s) | 20260507 15:26:50.329 | INFO | Analyzing chunk frames = [565760,566272) | 20260507 15:26:50.329 | INFO | MemInfo: { | 20260507 15:26:50.329 | INFO | "gpu" : " 17 GB", | 20260507 15:26:50.330 | INFO | "huge" : "", | 20260507 15:26:50.330 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:50.330 | INFO | "malloc" : " 14 GB", | 20260507 15:26:50.330 | INFO | "pinned" : " 29 GB" | 20260507 15:26:50.330 | INFO | } | 20260507 15:26:50.330 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:26:50.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:50.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:26:50.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:26:50.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:50.330 | INFO | Uploaded 36GiB to GPU in 1.58524s (22.7095GiB/s) | 20260507 15:26:50.330 | INFO | Downloaded 34GiB to GPU in 1.53371s (22.1685GiB/s) | 20260507 15:26:55.469 | INFO | Analyzing chunk frames = [566272,566784) | 20260507 15:26:55.469 | INFO | MemInfo: { | 20260507 15:26:55.469 | INFO | "gpu" : " 17 GB", | 20260507 15:26:55.469 | INFO | "huge" : "", | 20260507 15:26:55.469 | INFO | "huge_shared" : " 29 GB", | 20260507 15:26:55.469 | INFO | "malloc" : " 14 GB", | 20260507 15:26:55.469 | INFO | "pinned" : " 29 GB" | 20260507 15:26:55.469 | INFO | } | 20260507 15:26:55.469 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:26:55.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:26:55.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:26:55.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:26:55.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:26:55.469 | INFO | Uploaded 36GiB to GPU in 1.58766s (22.6749GiB/s) | 20260507 15:26:55.469 | INFO | Downloaded 34GiB to GPU in 1.53793s (22.1077GiB/s) | 20260507 15:27:00.608 | INFO | Analyzing chunk frames = [566784,567296) | 20260507 15:27:00.608 | INFO | MemInfo: { | 20260507 15:27:00.608 | INFO | "gpu" : " 17 GB", | 20260507 15:27:00.608 | INFO | "huge" : "", | 20260507 15:27:00.608 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:00.608 | INFO | "malloc" : " 14 GB", | 20260507 15:27:00.608 | INFO | "pinned" : " 29 GB" | 20260507 15:27:00.608 | INFO | } | 20260507 15:27:00.608 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:27:00.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:00.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:27:00.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:27:00.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:00.609 | INFO | Uploaded 36GiB to GPU in 1.5896s (22.6473GiB/s) | 20260507 15:27:00.609 | INFO | Downloaded 34GiB to GPU in 1.53736s (22.1158GiB/s) | 20260507 15:27:05.748 | INFO | Analyzing chunk frames = [567296,567808) | 20260507 15:27:05.748 | INFO | MemInfo: { | 20260507 15:27:05.748 | INFO | "gpu" : " 17 GB", | 20260507 15:27:05.748 | INFO | "huge" : "", | 20260507 15:27:05.748 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:05.748 | INFO | "malloc" : " 14 GB", | 20260507 15:27:05.748 | INFO | "pinned" : " 29 GB" | 20260507 15:27:05.748 | INFO | } | 20260507 15:27:05.748 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:27:05.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:05.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:27:05.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:27:05.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:05.748 | INFO | Uploaded 36GiB to GPU in 1.58752s (22.6768GiB/s) | 20260507 15:27:05.748 | INFO | Downloaded 34GiB to GPU in 1.53638s (22.1299GiB/s) | 20260507 15:27:10.887 | INFO | Analyzing chunk frames = [567808,568320) | 20260507 15:27:10.887 | INFO | MemInfo: { | 20260507 15:27:10.887 | INFO | "gpu" : " 17 GB", | 20260507 15:27:10.887 | INFO | "huge" : "", | 20260507 15:27:10.887 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:10.887 | INFO | "malloc" : " 14 GB", | 20260507 15:27:10.887 | INFO | "pinned" : " 29 GB" | 20260507 15:27:10.887 | INFO | } | 20260507 15:27:10.887 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:27:10.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:10.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:27:10.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:27:10.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:10.887 | INFO | Uploaded 36GiB to GPU in 1.58835s (22.6651GiB/s) | 20260507 15:27:10.887 | INFO | Downloaded 34GiB to GPU in 1.53585s (22.1376GiB/s) | 20260507 15:27:16.026 | INFO | Analyzing chunk frames = [568320,568832) | 20260507 15:27:16.026 | INFO | MemInfo: { | 20260507 15:27:16.026 | INFO | "gpu" : " 17 GB", | 20260507 15:27:16.026 | INFO | "huge" : "", | 20260507 15:27:16.026 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:16.026 | INFO | "malloc" : " 14 GB", | 20260507 15:27:16.026 | INFO | "pinned" : " 29 GB" | 20260507 15:27:16.026 | INFO | } | 20260507 15:27:16.026 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:27:16.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:16.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.026 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:27:16.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.026 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:27:16.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:16.027 | INFO | Uploaded 36GiB to GPU in 1.58736s (22.6791GiB/s) | 20260507 15:27:16.027 | INFO | Downloaded 34GiB to GPU in 1.53689s (22.1226GiB/s) | 20260507 15:27:21.166 | INFO | Analyzing chunk frames = [568832,569344) | 20260507 15:27:21.166 | INFO | MemInfo: { | 20260507 15:27:21.166 | INFO | "gpu" : " 17 GB", | 20260507 15:27:21.166 | INFO | "huge" : "", | 20260507 15:27:21.166 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:21.166 | INFO | "malloc" : " 14 GB", | 20260507 15:27:21.166 | INFO | "pinned" : " 29 GB" | 20260507 15:27:21.166 | INFO | } | 20260507 15:27:21.166 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:27:21.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:21.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:27:21.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 15:27:21.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:21.166 | INFO | Uploaded 36GiB to GPU in 1.5893s (22.6515GiB/s) | 20260507 15:27:21.166 | INFO | Downloaded 34GiB to GPU in 1.53875s (22.0958GiB/s) | 20260507 15:27:21.262 | INFO | Starting a round of Baz IO | 20260507 15:27:24.148 | INFO | Populating RTMetrics report | 20260507 15:27:24.149 | INFO | RTMetrics report populated | 20260507 15:27:24.149 | INFO | Opening tmp RTMetrics json file | 20260507 15:27:24.149 | INFO | Serializing RTMetrics to json | 20260507 15:27:24.151 | INFO | Serialization done, swapping tmp file now... | 20260507 15:27:24.151 | INFO | RTMetrics done | 20260507 15:27:24.234 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 2.971 seconds | 20260507 15:27:24.234 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.31% / 0.04% / 46.65% | 20260507 15:27:24.234 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.85296% / 8.74201e-05% / 1.19209e-05% | 20260507 15:27:24.235 | INFO | Baz IO overall: | 20260507 15:27:24.235 | INFO | BytesWritten : 231148141180 | 20260507 15:27:24.235 | INFO | NumEvents : 111333267752 | 20260507 15:27:26.305 | INFO | Analyzing chunk frames = [569344,569856) | 20260507 15:27:26.305 | INFO | MemInfo: { | 20260507 15:27:26.305 | INFO | "gpu" : " 17 GB", | 20260507 15:27:26.305 | INFO | "huge" : "", | 20260507 15:27:26.305 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:26.305 | INFO | "malloc" : " 14 GB", | 20260507 15:27:26.305 | INFO | "pinned" : " 29 GB" | 20260507 15:27:26.305 | INFO | } | 20260507 15:27:26.305 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:27:26.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:26.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:26.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:26.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:26.305 | INFO | ANALYSIS : 57.2%, 2.9, 87.6ms, 0 | 20260507 15:27:26.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:26.305 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 15:27:26.305 | INFO | PRE_HQ : 55.8%, 2.1, 61.0ms, 0 | 20260507 15:27:26.305 | INFO | BAZWRITER : 57.4%, 4.4, 128.3ms, 0 | 20260507 15:27:26.305 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 15:27:26.305 | INFO | Uploaded 36GiB to GPU in 1.65011s (21.8167GiB/s) | 20260507 15:27:26.305 | INFO | Downloaded 38GiB to GPU in 1.73629s (21.8857GiB/s) | 20260507 15:27:31.444 | INFO | Analyzing chunk frames = [569856,570368) | 20260507 15:27:31.444 | INFO | MemInfo: { | 20260507 15:27:31.444 | INFO | "gpu" : " 17 GB", | 20260507 15:27:31.444 | INFO | "huge" : "", | 20260507 15:27:31.444 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:31.444 | INFO | "malloc" : " 14 GB", | 20260507 15:27:31.444 | INFO | "pinned" : " 29 GB" | 20260507 15:27:31.444 | INFO | } | 20260507 15:27:31.444 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:27:31.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:31.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.444 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:27:31.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.444 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:27:31.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:31.445 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4166GiB/s) | 20260507 15:27:31.445 | INFO | Downloaded 34GiB to GPU in 1.52474s (22.2989GiB/s) | 20260507 15:27:36.583 | INFO | Analyzing chunk frames = [570368,570880) | 20260507 15:27:36.583 | INFO | MemInfo: { | 20260507 15:27:36.583 | INFO | "gpu" : " 17 GB", | 20260507 15:27:36.583 | INFO | "huge" : "", | 20260507 15:27:36.583 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:36.583 | INFO | "malloc" : " 14 GB", | 20260507 15:27:36.583 | INFO | "pinned" : " 29 GB" | 20260507 15:27:36.583 | INFO | } | 20260507 15:27:36.584 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:27:36.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:36.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:27:36.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:27:36.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:36.584 | INFO | Uploaded 36GiB to GPU in 1.60789s (22.3896GiB/s) | 20260507 15:27:36.584 | INFO | Downloaded 34GiB to GPU in 1.52997s (22.2227GiB/s) | 20260507 15:27:41.723 | INFO | Analyzing chunk frames = [570880,571392) | 20260507 15:27:41.723 | INFO | MemInfo: { | 20260507 15:27:41.723 | INFO | "gpu" : " 17 GB", | 20260507 15:27:41.723 | INFO | "huge" : "", | 20260507 15:27:41.723 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:41.723 | INFO | "malloc" : " 14 GB", | 20260507 15:27:41.723 | INFO | "pinned" : " 29 GB" | 20260507 15:27:41.723 | INFO | } | 20260507 15:27:41.723 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:27:41.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:41.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.723 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:27:41.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.723 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:27:41.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:41.724 | INFO | Uploaded 36GiB to GPU in 1.60931s (22.3698GiB/s) | 20260507 15:27:41.724 | INFO | Downloaded 34GiB to GPU in 1.52912s (22.235GiB/s) | 20260507 15:27:46.862 | INFO | Analyzing chunk frames = [571392,571904) | 20260507 15:27:46.862 | INFO | MemInfo: { | 20260507 15:27:46.862 | INFO | "gpu" : " 17 GB", | 20260507 15:27:46.862 | INFO | "huge" : "", | 20260507 15:27:46.862 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:46.862 | INFO | "malloc" : " 14 GB", | 20260507 15:27:46.862 | INFO | "pinned" : " 29 GB" | 20260507 15:27:46.862 | INFO | } | 20260507 15:27:46.863 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:27:46.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:46.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:27:46.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:27:46.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:46.863 | INFO | Uploaded 36GiB to GPU in 1.60728s (22.3981GiB/s) | 20260507 15:27:46.863 | INFO | Downloaded 34GiB to GPU in 1.52752s (22.2583GiB/s) | 20260507 15:27:52.002 | INFO | Analyzing chunk frames = [571904,572416) | 20260507 15:27:52.002 | INFO | MemInfo: { | 20260507 15:27:52.002 | INFO | "gpu" : " 17 GB", | 20260507 15:27:52.002 | INFO | "huge" : "", | 20260507 15:27:52.002 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:52.002 | INFO | "malloc" : " 14 GB", | 20260507 15:27:52.002 | INFO | "pinned" : " 29 GB" | 20260507 15:27:52.002 | INFO | } | 20260507 15:27:52.002 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:27:52.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:52.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:27:52.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:27:52.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:52.002 | INFO | Uploaded 36GiB to GPU in 1.60778s (22.3911GiB/s) | 20260507 15:27:52.003 | INFO | Downloaded 34GiB to GPU in 1.52693s (22.267GiB/s) | 20260507 15:27:57.141 | INFO | Analyzing chunk frames = [572416,572928) | 20260507 15:27:57.141 | INFO | MemInfo: { | 20260507 15:27:57.141 | INFO | "gpu" : " 17 GB", | 20260507 15:27:57.141 | INFO | "huge" : "", | 20260507 15:27:57.141 | INFO | "huge_shared" : " 29 GB", | 20260507 15:27:57.141 | INFO | "malloc" : " 14 GB", | 20260507 15:27:57.141 | INFO | "pinned" : " 29 GB" | 20260507 15:27:57.141 | INFO | } | 20260507 15:27:57.141 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:27:57.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:27:57.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.141 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:27:57.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.142 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:27:57.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:27:57.142 | INFO | Uploaded 36GiB to GPU in 1.60691s (22.4033GiB/s) | 20260507 15:27:57.142 | INFO | Downloaded 34GiB to GPU in 1.52779s (22.2544GiB/s) | 20260507 15:28:02.280 | INFO | Analyzing chunk frames = [572928,573440) | 20260507 15:28:02.280 | INFO | MemInfo: { | 20260507 15:28:02.280 | INFO | "gpu" : " 17 GB", | 20260507 15:28:02.280 | INFO | "huge" : "", | 20260507 15:28:02.280 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:02.280 | INFO | "malloc" : " 14 GB", | 20260507 15:28:02.280 | INFO | "pinned" : " 29 GB" | 20260507 15:28:02.280 | INFO | } | 20260507 15:28:02.280 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:28:02.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:02.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:28:02.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 15:28:02.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:02.281 | INFO | Uploaded 36GiB to GPU in 1.6085s (22.3811GiB/s) | 20260507 15:28:02.281 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:28:02.281 | INFO | Total mem usage: 16.4 GiB | 20260507 15:28:02.281 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:28:02.281 | INFO | Downloaded 34GiB to GPU in 1.52913s (22.2349GiB/s) | 20260507 15:28:02.281 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.53%) | 20260507 15:28:02.281 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:28:02.281 | INFO | High Water Breakdown: | 20260507 15:28:02.281 | INFO | 59.2% : 8.87 GiB ( 89.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:28:02.281 | INFO | 40.6% : 6.09 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:28:02.281 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:28:02.281 | INFO | ------------------------------------------ | 20260507 15:28:02.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:28:02.281 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:28:02.281 | INFO | Total mem usage: 26.5 GiB | 20260507 15:28:02.281 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:28:02.281 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:28:02.281 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:28:02.281 | INFO | High Water Breakdown: | 20260507 15:28:02.281 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:28:02.281 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:28:02.281 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:28:02.281 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:28:02.281 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:28:02.281 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:28:02.281 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:28:02.281 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:28:02.281 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:28:02.281 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:28:02.281 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:28:02.281 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:28:02.281 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:28:02.281 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:28:02.281 | INFO | ------------------------------------------ | 20260507 15:28:02.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:28:02.281 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:28:02.281 | INFO | Total mem usage: 27.8 GiB | 20260507 15:28:02.281 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:28:02.281 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:28:02.281 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:28:02.281 | INFO | High Water Breakdown: | 20260507 15:28:02.281 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:28:02.281 | INFO | ------------------------------------------ | 20260507 15:28:02.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:28:02.281 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:28:02.281 | INFO | Total mem usage: 15.7 GiB | 20260507 15:28:02.281 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:28:02.281 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:28:02.281 | INFO | Unused Cache: 844 MiB (Waste: 5.25%) | 20260507 15:28:02.281 | INFO | High Water Breakdown: | 20260507 15:28:02.281 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:28:02.281 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:28:02.281 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:28:02.281 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:28:02.281 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:28:02.281 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:28:02.281 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:28:02.281 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:28:02.281 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:28:02.281 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:28:02.281 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:28:02.281 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:28:02.281 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:28:02.281 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:28:02.281 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:28:02.281 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:28:02.281 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:28:02.281 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:28:02.281 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:28:02.281 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:28:02.281 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:28:02.281 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:28:02.281 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:28:02.281 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:28:02.281 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:28:02.281 | INFO | ------------------------------------------ | 20260507 15:28:02.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:28:02.281 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:28:02.281 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:28:02.281 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:28:02.281 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 0.30%, AggregateStats -- 8.84e-02 / -nan, 6.57e-02 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 11.04%, Baseline -- 3.30e+00 / -nan, 1.99e-02 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 3.07%, Histogram -- 9.18e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 2.99e+00 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.72e+01 / -nan, 4.39e-02 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 3.48%, Metrics -- 1.04e+00 / -nan, 2.39e-01 / 0.00e+00 | 20260507 15:28:02.281 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.81e-01 / -nan, 5.59e-02 / 0.00e+00 | 20260507 15:28:02.281 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:28:02.282 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:28:02.282 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:28:02.282 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 18.68%, Upload -- 1.67e+01 / 0.00e+00, 4.58e-01 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 18.13%, Download -- 1.62e+01 / 0.00e+00, 7.89e-01 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 33.38%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 0.00%, LockDownload -- 4.42e-04 / 0.00e+00, 1.50e-04 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 29.23%, LockCompute -- 2.61e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:28:02.282 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 15:28:02.377 | INFO | Starting a round of Baz IO | 20260507 15:28:05.258 | INFO | Populating RTMetrics report | 20260507 15:28:05.259 | INFO | RTMetrics report populated | 20260507 15:28:05.259 | INFO | Opening tmp RTMetrics json file | 20260507 15:28:05.262 | INFO | Serializing RTMetrics to json | 20260507 15:28:05.289 | INFO | Serialization done, swapping tmp file now... | 20260507 15:28:05.322 | INFO | RTMetrics done | 20260507 15:28:05.370 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 2.992 seconds | 20260507 15:28:05.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.26% / 0.04% / 46.69% | 20260507 15:28:05.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.81886% / 7.15256e-05% / 2.38419e-05% | 20260507 15:28:05.370 | INFO | Baz IO overall: | 20260507 15:28:05.370 | INFO | BytesWritten : 233029636485 | 20260507 15:28:05.370 | INFO | NumEvents : 112249566512 | 20260507 15:28:07.420 | INFO | Analyzing chunk frames = [573440,573952) | 20260507 15:28:07.420 | INFO | MemInfo: { | 20260507 15:28:07.420 | INFO | "gpu" : " 17 GB", | 20260507 15:28:07.420 | INFO | "huge" : "", | 20260507 15:28:07.420 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:07.420 | INFO | "malloc" : " 14 GB", | 20260507 15:28:07.420 | INFO | "pinned" : " 29 GB" | 20260507 15:28:07.420 | INFO | } | 20260507 15:28:07.420 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:28:07.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:07.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:07.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:07.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:07.420 | INFO | ANALYSIS : 57.1%, 2.9, 87.7ms, 0 | 20260507 15:28:07.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:07.420 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260507 15:28:07.420 | INFO | PRE_HQ : 55.9%, 2.1, 60.9ms, 0 | 20260507 15:28:07.420 | INFO | BAZWRITER : 57.8%, 4.3, 128.3ms, 0 | 20260507 15:28:07.420 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260507 15:28:07.420 | INFO | Uploaded 36GiB to GPU in 1.64006s (21.9504GiB/s) | 20260507 15:28:07.420 | INFO | Downloaded 38GiB to GPU in 1.724s (22.0418GiB/s) | 20260507 15:28:12.559 | INFO | Analyzing chunk frames = [573952,574464) | 20260507 15:28:12.559 | INFO | MemInfo: { | 20260507 15:28:12.559 | INFO | "gpu" : " 17 GB", | 20260507 15:28:12.559 | INFO | "huge" : "", | 20260507 15:28:12.559 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:12.559 | INFO | "malloc" : " 14 GB", | 20260507 15:28:12.559 | INFO | "pinned" : " 29 GB" | 20260507 15:28:12.559 | INFO | } | 20260507 15:28:12.559 | INFO | Perf report for the last 5.13974 seconds: | 20260507 15:28:12.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:12.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.559 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:28:12.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.559 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:28:12.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:12.560 | INFO | Uploaded 36GiB to GPU in 1.60462s (22.4353GiB/s) | 20260507 15:28:12.560 | INFO | Downloaded 34GiB to GPU in 1.525s (22.2951GiB/s) | 20260507 15:28:17.698 | INFO | Analyzing chunk frames = [574464,574976) | 20260507 15:28:17.698 | INFO | MemInfo: { | 20260507 15:28:17.698 | INFO | "gpu" : " 17 GB", | 20260507 15:28:17.698 | INFO | "huge" : "", | 20260507 15:28:17.698 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:17.698 | INFO | "malloc" : " 14 GB", | 20260507 15:28:17.698 | INFO | "pinned" : " 29 GB" | 20260507 15:28:17.698 | INFO | } | 20260507 15:28:17.698 | INFO | Perf report for the last 5.13892 seconds: | 20260507 15:28:17.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:17.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.699 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:28:17.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.699 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:28:17.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:17.699 | INFO | Uploaded 36GiB to GPU in 1.61011s (22.3587GiB/s) | 20260507 15:28:17.699 | INFO | Downloaded 34GiB to GPU in 1.53071s (22.2119GiB/s) | 20260507 15:28:22.838 | INFO | Analyzing chunk frames = [574976,575488) | 20260507 15:28:22.838 | INFO | MemInfo: { | 20260507 15:28:22.838 | INFO | "gpu" : " 17 GB", | 20260507 15:28:22.838 | INFO | "huge" : "", | 20260507 15:28:22.838 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:22.838 | INFO | "malloc" : " 14 GB", | 20260507 15:28:22.838 | INFO | "pinned" : " 29 GB" | 20260507 15:28:22.838 | INFO | } | 20260507 15:28:22.838 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:28:22.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:22.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:28:22.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:28:22.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:22.838 | INFO | Uploaded 36GiB to GPU in 1.60843s (22.382GiB/s) | 20260507 15:28:22.838 | INFO | Downloaded 34GiB to GPU in 1.52894s (22.2377GiB/s) | 20260507 15:28:27.977 | INFO | Analyzing chunk frames = [575488,576000) | 20260507 15:28:27.977 | INFO | MemInfo: { | 20260507 15:28:27.977 | INFO | "gpu" : " 17 GB", | 20260507 15:28:27.977 | INFO | "huge" : "", | 20260507 15:28:27.977 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:27.977 | INFO | "malloc" : " 14 GB", | 20260507 15:28:27.977 | INFO | "pinned" : " 29 GB" | 20260507 15:28:27.977 | INFO | } | 20260507 15:28:27.977 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:28:27.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:27.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.977 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:28:27.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.977 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:28:27.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:27.978 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3923GiB/s) | 20260507 15:28:27.978 | INFO | Downloaded 34GiB to GPU in 1.52746s (22.2592GiB/s) | 20260507 15:28:33.116 | INFO | Analyzing chunk frames = [576000,576512) | 20260507 15:28:33.116 | INFO | MemInfo: { | 20260507 15:28:33.116 | INFO | "gpu" : " 17 GB", | 20260507 15:28:33.116 | INFO | "huge" : "", | 20260507 15:28:33.116 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:33.116 | INFO | "malloc" : " 14 GB", | 20260507 15:28:33.117 | INFO | "pinned" : " 29 GB" | 20260507 15:28:33.117 | INFO | } | 20260507 15:28:33.117 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:28:33.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:33.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:28:33.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 15:28:33.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:33.117 | INFO | Uploaded 36GiB to GPU in 1.60852s (22.3808GiB/s) | 20260507 15:28:33.117 | INFO | Downloaded 34GiB to GPU in 1.52756s (22.2577GiB/s) | 20260507 15:28:38.256 | INFO | Analyzing chunk frames = [576512,577024) | 20260507 15:28:38.256 | INFO | MemInfo: { | 20260507 15:28:38.256 | INFO | "gpu" : " 17 GB", | 20260507 15:28:38.256 | INFO | "huge" : "", | 20260507 15:28:38.256 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:38.256 | INFO | "malloc" : " 14 GB", | 20260507 15:28:38.256 | INFO | "pinned" : " 29 GB" | 20260507 15:28:38.256 | INFO | } | 20260507 15:28:38.256 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:28:38.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:38.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:28:38.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 15:28:38.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:38.256 | INFO | Uploaded 36GiB to GPU in 1.60771s (22.3922GiB/s) | 20260507 15:28:38.256 | INFO | Downloaded 34GiB to GPU in 1.52933s (22.2319GiB/s) | 20260507 15:28:43.395 | INFO | Analyzing chunk frames = [577024,577536) | 20260507 15:28:43.395 | INFO | MemInfo: { | 20260507 15:28:43.395 | INFO | "gpu" : " 17 GB", | 20260507 15:28:43.395 | INFO | "huge" : "", | 20260507 15:28:43.395 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:43.395 | INFO | "malloc" : " 14 GB", | 20260507 15:28:43.395 | INFO | "pinned" : " 29 GB" | 20260507 15:28:43.395 | INFO | } | 20260507 15:28:43.395 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:28:43.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:43.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.395 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:28:43.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.395 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:28:43.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:43.396 | INFO | Uploaded 36GiB to GPU in 1.60758s (22.3939GiB/s) | 20260507 15:28:43.396 | INFO | Downloaded 34GiB to GPU in 1.52791s (22.2527GiB/s) | 20260507 15:28:43.492 | INFO | Starting a round of Baz IO | 20260507 15:28:46.421 | INFO | Populating RTMetrics report | 20260507 15:28:46.422 | INFO | RTMetrics report populated | 20260507 15:28:46.422 | INFO | Opening tmp RTMetrics json file | 20260507 15:28:46.458 | INFO | Serializing RTMetrics to json | 20260507 15:28:46.491 | INFO | Serialization done, swapping tmp file now... | 20260507 15:28:46.496 | INFO | RTMetrics done | 20260507 15:28:46.730 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.237 seconds | 20260507 15:28:46.730 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.22% / 0.04% / 46.74% | 20260507 15:28:46.730 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.80401% / 7.15256e-05% / 3.17891e-05% | 20260507 15:28:46.730 | INFO | Baz IO overall: | 20260507 15:28:46.730 | INFO | BytesWritten : 234911912340 | 20260507 15:28:46.730 | INFO | NumEvents : 113166170041 | 20260507 15:28:48.534 | INFO | Analyzing chunk frames = [577536,578048) | 20260507 15:28:48.534 | INFO | MemInfo: { | 20260507 15:28:48.534 | INFO | "gpu" : " 17 GB", | 20260507 15:28:48.535 | INFO | "huge" : "", | 20260507 15:28:48.535 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:48.535 | INFO | "malloc" : " 14 GB", | 20260507 15:28:48.535 | INFO | "pinned" : " 29 GB" | 20260507 15:28:48.535 | INFO | } | 20260507 15:28:48.535 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:28:48.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:48.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:48.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:48.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:48.535 | INFO | ANALYSIS : 57.5%, 2.9, 86.1ms, 0 | 20260507 15:28:48.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:48.535 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 15:28:48.535 | INFO | PRE_HQ : 56.5%, 2.1, 61.8ms, 0 | 20260507 15:28:48.535 | INFO | BAZWRITER : 62.5%, 4.6, 148.3ms, 0 | 20260507 15:28:48.535 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 15:28:48.536 | INFO | Uploaded 36GiB to GPU in 1.64178s (21.9274GiB/s) | 20260507 15:28:48.536 | INFO | Downloaded 38GiB to GPU in 1.72602s (22.016GiB/s) | 20260507 15:28:53.674 | INFO | Analyzing chunk frames = [578048,578560) | 20260507 15:28:53.674 | INFO | MemInfo: { | 20260507 15:28:53.674 | INFO | "gpu" : " 17 GB", | 20260507 15:28:53.674 | INFO | "huge" : "", | 20260507 15:28:53.674 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:53.674 | INFO | "malloc" : " 14 GB", | 20260507 15:28:53.674 | INFO | "pinned" : " 29 GB" | 20260507 15:28:53.674 | INFO | } | 20260507 15:28:53.674 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:28:53.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:53.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.674 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260507 15:28:53.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.674 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 15:28:53.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:53.675 | INFO | Uploaded 36GiB to GPU in 1.61095s (22.347GiB/s) | 20260507 15:28:53.675 | INFO | Downloaded 34GiB to GPU in 1.52888s (22.2385GiB/s) | 20260507 15:28:58.813 | INFO | Analyzing chunk frames = [578560,579072) | 20260507 15:28:58.813 | INFO | MemInfo: { | 20260507 15:28:58.813 | INFO | "gpu" : " 17 GB", | 20260507 15:28:58.813 | INFO | "huge" : "", | 20260507 15:28:58.813 | INFO | "huge_shared" : " 29 GB", | 20260507 15:28:58.813 | INFO | "malloc" : " 14 GB", | 20260507 15:28:58.813 | INFO | "pinned" : " 29 GB" | 20260507 15:28:58.813 | INFO | } | 20260507 15:28:58.813 | INFO | Perf report for the last 5.13894 seconds: | 20260507 15:28:58.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:28:58.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.814 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:28:58.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.814 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:28:58.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:28:58.814 | INFO | Uploaded 36GiB to GPU in 1.61548s (22.2845GiB/s) | 20260507 15:28:58.814 | INFO | Downloaded 34GiB to GPU in 1.53524s (22.1463GiB/s) | 20260507 15:29:03.953 | INFO | Analyzing chunk frames = [579072,579584) | 20260507 15:29:03.953 | INFO | MemInfo: { | 20260507 15:29:03.953 | INFO | "gpu" : " 17 GB", | 20260507 15:29:03.953 | INFO | "huge" : "", | 20260507 15:29:03.953 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:03.953 | INFO | "malloc" : " 14 GB", | 20260507 15:29:03.953 | INFO | "pinned" : " 29 GB" | 20260507 15:29:03.953 | INFO | } | 20260507 15:29:03.953 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:29:03.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:03.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 15:29:03.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:29:03.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:03.953 | INFO | Uploaded 36GiB to GPU in 1.61621s (22.2744GiB/s) | 20260507 15:29:03.953 | INFO | Downloaded 34GiB to GPU in 1.53532s (22.1452GiB/s) | 20260507 15:29:09.092 | INFO | Analyzing chunk frames = [579584,580096) | 20260507 15:29:09.092 | INFO | MemInfo: { | 20260507 15:29:09.092 | INFO | "gpu" : " 17 GB", | 20260507 15:29:09.092 | INFO | "huge" : "", | 20260507 15:29:09.092 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:09.092 | INFO | "malloc" : " 14 GB", | 20260507 15:29:09.092 | INFO | "pinned" : " 29 GB" | 20260507 15:29:09.092 | INFO | } | 20260507 15:29:09.092 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:29:09.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:09.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:29:09.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:29:09.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:09.092 | INFO | Uploaded 36GiB to GPU in 1.6072s (22.3992GiB/s) | 20260507 15:29:09.092 | INFO | Downloaded 34GiB to GPU in 1.52805s (22.2506GiB/s) | 20260507 15:29:14.231 | INFO | Analyzing chunk frames = [580096,580608) | 20260507 15:29:14.231 | INFO | MemInfo: { | 20260507 15:29:14.231 | INFO | "gpu" : " 17 GB", | 20260507 15:29:14.231 | INFO | "huge" : "", | 20260507 15:29:14.231 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:14.232 | INFO | "malloc" : " 14 GB", | 20260507 15:29:14.233 | INFO | "pinned" : " 29 GB" | 20260507 15:29:14.233 | INFO | } | 20260507 15:29:14.233 | INFO | Perf report for the last 5.14086 seconds: | 20260507 15:29:14.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:14.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:29:14.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:29:14.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:14.233 | INFO | Uploaded 36GiB to GPU in 1.60734s (22.3973GiB/s) | 20260507 15:29:14.233 | INFO | Downloaded 34GiB to GPU in 1.52673s (22.2699GiB/s) | 20260507 15:29:19.371 | INFO | Analyzing chunk frames = [580608,581120) | 20260507 15:29:19.371 | INFO | MemInfo: { | 20260507 15:29:19.371 | INFO | "gpu" : " 17 GB", | 20260507 15:29:19.371 | INFO | "huge" : "", | 20260507 15:29:19.371 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:19.371 | INFO | "malloc" : " 14 GB", | 20260507 15:29:19.371 | INFO | "pinned" : " 29 GB" | 20260507 15:29:19.371 | INFO | } | 20260507 15:29:19.371 | INFO | Perf report for the last 5.13798 seconds: | 20260507 15:29:19.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:19.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:29:19.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:29:19.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:19.371 | INFO | Uploaded 36GiB to GPU in 1.60676s (22.4054GiB/s) | 20260507 15:29:19.371 | INFO | Downloaded 34GiB to GPU in 1.52892s (22.2379GiB/s) | 20260507 15:29:24.510 | INFO | Analyzing chunk frames = [581120,581632) | 20260507 15:29:24.510 | INFO | MemInfo: { | 20260507 15:29:24.510 | INFO | "gpu" : " 17 GB", | 20260507 15:29:24.510 | INFO | "huge" : "", | 20260507 15:29:24.510 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:24.510 | INFO | "malloc" : " 14 GB", | 20260507 15:29:24.510 | INFO | "pinned" : " 29 GB" | 20260507 15:29:24.510 | INFO | } | 20260507 15:29:24.510 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:29:24.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:24.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.510 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:29:24.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.510 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:29:24.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:24.511 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:29:24.511 | INFO | Total mem usage: 16.4 GiB | 20260507 15:29:24.511 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:29:24.511 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.42%) | 20260507 15:29:24.511 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:29:24.511 | INFO | High Water Breakdown: | 20260507 15:29:24.511 | INFO | 59.5% : 9.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:29:24.511 | INFO | 40.3% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:29:24.511 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:29:24.511 | INFO | ------------------------------------------ | 20260507 15:29:24.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:29:24.511 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:29:24.511 | INFO | Total mem usage: 26.5 GiB | 20260507 15:29:24.511 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:29:24.511 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 15:29:24.511 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:29:24.511 | INFO | High Water Breakdown: | 20260507 15:29:24.511 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:29:24.511 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:29:24.511 | INFO | Uploaded 36GiB to GPU in 1.60711s (22.4005GiB/s) | 20260507 15:29:24.511 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:29:24.511 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:29:24.511 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:29:24.511 | INFO | Downloaded 34GiB to GPU in 1.52825s (22.2476GiB/s) | 20260507 15:29:24.511 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:29:24.511 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:29:24.511 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:29:24.511 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:29:24.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:29:24.511 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:29:24.511 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:29:24.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:29:24.511 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:29:24.511 | INFO | ------------------------------------------ | 20260507 15:29:24.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:29:24.511 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:29:24.511 | INFO | Total mem usage: 27.8 GiB | 20260507 15:29:24.511 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:29:24.511 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:29:24.511 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:29:24.511 | INFO | High Water Breakdown: | 20260507 15:29:24.511 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:29:24.511 | INFO | ------------------------------------------ | 20260507 15:29:24.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:29:24.511 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:29:24.511 | INFO | Total mem usage: 15.7 GiB | 20260507 15:29:24.511 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:29:24.511 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:29:24.511 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:29:24.511 | INFO | High Water Breakdown: | 20260507 15:29:24.511 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:29:24.511 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:29:24.511 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:29:24.511 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:29:24.511 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:29:24.511 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:29:24.511 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:29:24.512 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:29:24.512 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:29:24.512 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:29:24.512 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:29:24.512 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:29:24.512 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:29:24.512 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:29:24.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:29:24.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:29:24.512 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:29:24.512 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:29:24.512 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:29:24.512 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:29:24.512 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:29:24.512 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:29:24.512 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:29:24.512 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:29:24.512 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:29:24.512 | INFO | ------------------------------------------ | 20260507 15:29:24.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:29:24.512 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:29:24.512 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:29:24.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:29:24.512 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 0.29%, AggregateStats -- 8.77e-02 / -nan, 5.15e-03 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 11.04%, Baseline -- 3.30e+00 / -nan, 2.27e-02 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.72e+01 / -nan, 4.50e-02 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.15e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 3.47%, Metrics -- 1.04e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.83e-01 / -nan, 5.61e-02 / 0.00e+00 | 20260507 15:29:24.512 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:29:24.512 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:29:24.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:29:24.512 | INFO | PerfReport: Full Execution -- 8.95e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 18.78%, Upload -- 1.68e+01 / 0.00e+00, 4.11e-01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 18.09%, Download -- 1.62e+01 / 0.00e+00, 7.70e-01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 33.39%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 0.58%, LockUpload -- 5.23e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 2.99e-04 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 29.14%, LockCompute -- 2.61e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:29:24.512 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 15:29:24.607 | INFO | Starting a round of Baz IO | 20260507 15:29:27.499 | INFO | Populating RTMetrics report | 20260507 15:29:27.499 | INFO | RTMetrics report populated | 20260507 15:29:27.499 | INFO | Opening tmp RTMetrics json file | 20260507 15:29:27.548 | INFO | Serializing RTMetrics to json | 20260507 15:29:27.552 | INFO | Serialization done, swapping tmp file now... | 20260507 15:29:27.554 | INFO | RTMetrics done | 20260507 15:29:27.669 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.061 seconds | 20260507 15:29:27.669 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.18% / 0.04% / 46.78% | 20260507 15:29:27.669 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.78374% / 6.35783e-05% / 1.19209e-05% | 20260507 15:29:27.669 | INFO | Baz IO overall: | 20260507 15:29:27.669 | INFO | BytesWritten : 236794404093 | 20260507 15:29:27.669 | INFO | NumEvents : 114083083192 | 20260507 15:29:29.649 | INFO | Analyzing chunk frames = [581632,582144) | 20260507 15:29:29.649 | INFO | MemInfo: { | 20260507 15:29:29.649 | INFO | "gpu" : " 17 GB", | 20260507 15:29:29.649 | INFO | "huge" : "", | 20260507 15:29:29.649 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:29.649 | INFO | "malloc" : " 14 GB", | 20260507 15:29:29.649 | INFO | "pinned" : " 29 GB" | 20260507 15:29:29.649 | INFO | } | 20260507 15:29:29.649 | INFO | Perf report for the last 5.13905 seconds: | 20260507 15:29:29.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:29.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:29.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:29.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:29.649 | INFO | ANALYSIS : 57.3%, 2.8, 84.9ms, 0 | 20260507 15:29:29.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:29.649 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 15:29:29.649 | INFO | PRE_HQ : 56.1%, 2.1, 60.0ms, 0 | 20260507 15:29:29.649 | INFO | BAZWRITER : 59.2%, 4.7, 143.9ms, 0 | 20260507 15:29:29.649 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:29:29.650 | INFO | Uploaded 36GiB to GPU in 1.65732s (21.7219GiB/s) | 20260507 15:29:29.650 | INFO | Downloaded 38GiB to GPU in 1.73932s (21.8476GiB/s) | 20260507 15:29:34.789 | INFO | Analyzing chunk frames = [582144,582656) | 20260507 15:29:34.789 | INFO | MemInfo: { | 20260507 15:29:34.789 | INFO | "gpu" : " 17 GB", | 20260507 15:29:34.789 | INFO | "huge" : "", | 20260507 15:29:34.789 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:34.789 | INFO | "malloc" : " 14 GB", | 20260507 15:29:34.789 | INFO | "pinned" : " 29 GB" | 20260507 15:29:34.789 | INFO | } | 20260507 15:29:34.789 | INFO | Perf report for the last 5.13954 seconds: | 20260507 15:29:34.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:34.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:29:34.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:29:34.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:34.789 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.4279GiB/s) | 20260507 15:29:34.789 | INFO | Downloaded 34GiB to GPU in 1.52512s (22.2933GiB/s) | 20260507 15:29:39.928 | INFO | Analyzing chunk frames = [582656,583168) | 20260507 15:29:39.928 | INFO | MemInfo: { | 20260507 15:29:39.928 | INFO | "gpu" : " 17 GB", | 20260507 15:29:39.928 | INFO | "huge" : "", | 20260507 15:29:39.928 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:39.928 | INFO | "malloc" : " 14 GB", | 20260507 15:29:39.928 | INFO | "pinned" : " 29 GB" | 20260507 15:29:39.928 | INFO | } | 20260507 15:29:39.928 | INFO | Perf report for the last 5.13915 seconds: | 20260507 15:29:39.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:39.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:29:39.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:29:39.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:39.928 | INFO | Uploaded 36GiB to GPU in 1.60978s (22.3634GiB/s) | 20260507 15:29:39.928 | INFO | Downloaded 34GiB to GPU in 1.52968s (22.2269GiB/s) | 20260507 15:29:45.067 | INFO | Analyzing chunk frames = [583168,583680) | 20260507 15:29:45.067 | INFO | MemInfo: { | 20260507 15:29:45.067 | INFO | "gpu" : " 17 GB", | 20260507 15:29:45.067 | INFO | "huge" : "", | 20260507 15:29:45.067 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:45.067 | INFO | "malloc" : " 14 GB", | 20260507 15:29:45.067 | INFO | "pinned" : " 29 GB" | 20260507 15:29:45.067 | INFO | } | 20260507 15:29:45.068 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:29:45.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:45.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:29:45.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:29:45.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:45.068 | INFO | Uploaded 36GiB to GPU in 1.60989s (22.3617GiB/s) | 20260507 15:29:45.068 | INFO | Downloaded 34GiB to GPU in 1.52956s (22.2286GiB/s) | 20260507 15:29:50.207 | INFO | Analyzing chunk frames = [583680,584192) | 20260507 15:29:50.207 | INFO | MemInfo: { | 20260507 15:29:50.207 | INFO | "gpu" : " 17 GB", | 20260507 15:29:50.207 | INFO | "huge" : "", | 20260507 15:29:50.207 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:50.207 | INFO | "malloc" : " 14 GB", | 20260507 15:29:50.207 | INFO | "pinned" : " 29 GB" | 20260507 15:29:50.207 | INFO | } | 20260507 15:29:50.207 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:29:50.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:50.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.207 | INFO | ANALYSIS : 55.9%, 3.0, 87.3ms, 0 | 20260507 15:29:50.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.207 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:29:50.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:50.208 | INFO | Uploaded 36GiB to GPU in 1.60804s (22.3875GiB/s) | 20260507 15:29:50.208 | INFO | Downloaded 34GiB to GPU in 1.52649s (22.2734GiB/s) | 20260507 15:29:55.346 | INFO | Analyzing chunk frames = [584192,584704) | 20260507 15:29:55.346 | INFO | MemInfo: { | 20260507 15:29:55.346 | INFO | "gpu" : " 17 GB", | 20260507 15:29:55.346 | INFO | "huge" : "", | 20260507 15:29:55.346 | INFO | "huge_shared" : " 29 GB", | 20260507 15:29:55.346 | INFO | "malloc" : " 14 GB", | 20260507 15:29:55.346 | INFO | "pinned" : " 29 GB" | 20260507 15:29:55.346 | INFO | } | 20260507 15:29:55.346 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:29:55.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:29:55.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:29:55.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:29:55.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:29:55.346 | INFO | Uploaded 36GiB to GPU in 1.60843s (22.3821GiB/s) | 20260507 15:29:55.346 | INFO | Downloaded 34GiB to GPU in 1.52826s (22.2475GiB/s) | 20260507 15:30:00.485 | INFO | Analyzing chunk frames = [584704,585216) | 20260507 15:30:00.485 | INFO | MemInfo: { | 20260507 15:30:00.485 | INFO | "gpu" : " 17 GB", | 20260507 15:30:00.485 | INFO | "huge" : "", | 20260507 15:30:00.485 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:00.485 | INFO | "malloc" : " 14 GB", | 20260507 15:30:00.485 | INFO | "pinned" : " 29 GB" | 20260507 15:30:00.485 | INFO | } | 20260507 15:30:00.486 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:30:00.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:00.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:30:00.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:30:00.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:00.486 | INFO | Uploaded 36GiB to GPU in 1.60923s (22.371GiB/s) | 20260507 15:30:00.486 | INFO | Downloaded 34GiB to GPU in 1.53116s (22.2054GiB/s) | 20260507 15:30:05.625 | INFO | Analyzing chunk frames = [585216,585728) | 20260507 15:30:05.625 | INFO | MemInfo: { | 20260507 15:30:05.625 | INFO | "gpu" : " 17 GB", | 20260507 15:30:05.625 | INFO | "huge" : "", | 20260507 15:30:05.625 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:05.625 | INFO | "malloc" : " 14 GB", | 20260507 15:30:05.625 | INFO | "pinned" : " 29 GB" | 20260507 15:30:05.625 | INFO | } | 20260507 15:30:05.625 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:30:05.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:05.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:30:05.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 15:30:05.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:05.625 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3923GiB/s) | 20260507 15:30:05.625 | INFO | Downloaded 34GiB to GPU in 1.52869s (22.2413GiB/s) | 20260507 15:30:05.722 | INFO | Starting a round of Baz IO | 20260507 15:30:08.604 | INFO | Populating RTMetrics report | 20260507 15:30:08.604 | INFO | RTMetrics report populated | 20260507 15:30:08.604 | INFO | Opening tmp RTMetrics json file | 20260507 15:30:08.659 | INFO | Serializing RTMetrics to json | 20260507 15:30:08.730 | INFO | Serialization done, swapping tmp file now... | 20260507 15:30:08.737 | INFO | RTMetrics done | 20260507 15:30:08.959 | INFO | Finished a round of Baz IO: Wrote 1.751 GiB in 3.237 seconds | 20260507 15:30:08.959 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.13% / 0.04% / 46.82% | 20260507 15:30:08.959 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.73924% / 9.13938e-05% / 2.38419e-05% | 20260507 15:30:08.959 | INFO | Baz IO overall: | 20260507 15:30:08.959 | INFO | BytesWritten : 238673999937 | 20260507 15:30:08.959 | INFO | NumEvents : 114997981682 | 20260507 15:30:10.764 | INFO | Analyzing chunk frames = [585728,586240) | 20260507 15:30:10.764 | INFO | MemInfo: { | 20260507 15:30:10.764 | INFO | "gpu" : " 17 GB", | 20260507 15:30:10.764 | INFO | "huge" : "", | 20260507 15:30:10.764 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:10.764 | INFO | "malloc" : " 14 GB", | 20260507 15:30:10.764 | INFO | "pinned" : " 29 GB" | 20260507 15:30:10.764 | INFO | } | 20260507 15:30:10.764 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:30:10.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:10.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:10.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:10.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:10.764 | INFO | ANALYSIS : 57.1%, 2.9, 86.6ms, 0 | 20260507 15:30:10.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:10.764 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 15:30:10.764 | INFO | PRE_HQ : 55.8%, 2.1, 59.5ms, 0 | 20260507 15:30:10.764 | INFO | BAZWRITER : 62.5%, 4.6, 149.0ms, 0 | 20260507 15:30:10.764 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 15:30:10.765 | INFO | Uploaded 36GiB to GPU in 1.65289s (21.78GiB/s) | 20260507 15:30:10.765 | INFO | Downloaded 38GiB to GPU in 1.73385s (21.9166GiB/s) | 20260507 15:30:15.903 | INFO | Analyzing chunk frames = [586240,586752) | 20260507 15:30:15.903 | INFO | MemInfo: { | 20260507 15:30:15.903 | INFO | "gpu" : " 17 GB", | 20260507 15:30:15.903 | INFO | "huge" : "", | 20260507 15:30:15.903 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:15.903 | INFO | "malloc" : " 14 GB", | 20260507 15:30:15.903 | INFO | "pinned" : " 29 GB" | 20260507 15:30:15.903 | INFO | } | 20260507 15:30:15.904 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:30:15.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:15.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:30:15.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:30:15.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:15.904 | INFO | Uploaded 36GiB to GPU in 1.58529s (22.7088GiB/s) | 20260507 15:30:15.904 | INFO | Downloaded 34GiB to GPU in 1.53392s (22.1654GiB/s) | 20260507 15:30:21.043 | INFO | Analyzing chunk frames = [586752,587264) | 20260507 15:30:21.043 | INFO | MemInfo: { | 20260507 15:30:21.043 | INFO | "gpu" : " 17 GB", | 20260507 15:30:21.043 | INFO | "huge" : "", | 20260507 15:30:21.043 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:21.043 | INFO | "malloc" : " 14 GB", | 20260507 15:30:21.043 | INFO | "pinned" : " 29 GB" | 20260507 15:30:21.043 | INFO | } | 20260507 15:30:21.043 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:30:21.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:21.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:30:21.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:30:21.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:21.043 | INFO | Uploaded 36GiB to GPU in 1.58936s (22.6506GiB/s) | 20260507 15:30:21.043 | INFO | Downloaded 34GiB to GPU in 1.53895s (22.093GiB/s) | 20260507 15:30:26.182 | INFO | Analyzing chunk frames = [587264,587776) | 20260507 15:30:26.182 | INFO | MemInfo: { | 20260507 15:30:26.182 | INFO | "gpu" : " 17 GB", | 20260507 15:30:26.182 | INFO | "huge" : "", | 20260507 15:30:26.182 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:26.182 | INFO | "malloc" : " 14 GB", | 20260507 15:30:26.182 | INFO | "pinned" : " 29 GB" | 20260507 15:30:26.182 | INFO | } | 20260507 15:30:26.182 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:30:26.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:26.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.182 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:30:26.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.182 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:30:26.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:26.183 | INFO | Uploaded 36GiB to GPU in 1.58877s (22.6591GiB/s) | 20260507 15:30:26.183 | INFO | Downloaded 34GiB to GPU in 1.53821s (22.1036GiB/s) | 20260507 15:30:31.321 | INFO | Analyzing chunk frames = [587776,588288) | 20260507 15:30:31.321 | INFO | MemInfo: { | 20260507 15:30:31.321 | INFO | "gpu" : " 17 GB", | 20260507 15:30:31.321 | INFO | "huge" : "", | 20260507 15:30:31.321 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:31.321 | INFO | "malloc" : " 14 GB", | 20260507 15:30:31.321 | INFO | "pinned" : " 29 GB" | 20260507 15:30:31.322 | INFO | } | 20260507 15:30:31.322 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:30:31.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:31.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:30:31.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:30:31.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:31.322 | INFO | Uploaded 36GiB to GPU in 1.58794s (22.6708GiB/s) | 20260507 15:30:31.322 | INFO | Downloaded 34GiB to GPU in 1.53617s (22.133GiB/s) | 20260507 15:30:36.461 | INFO | Analyzing chunk frames = [588288,588800) | 20260507 15:30:36.461 | INFO | MemInfo: { | 20260507 15:30:36.461 | INFO | "gpu" : " 17 GB", | 20260507 15:30:36.461 | INFO | "huge" : "", | 20260507 15:30:36.461 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:36.461 | INFO | "malloc" : " 14 GB", | 20260507 15:30:36.461 | INFO | "pinned" : " 29 GB" | 20260507 15:30:36.461 | INFO | } | 20260507 15:30:36.461 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:30:36.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:36.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:30:36.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:30:36.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:36.461 | INFO | Uploaded 36GiB to GPU in 1.58869s (22.6602GiB/s) | 20260507 15:30:36.461 | INFO | Downloaded 34GiB to GPU in 1.53647s (22.1286GiB/s) | 20260507 15:30:41.600 | INFO | Analyzing chunk frames = [588800,589312) | 20260507 15:30:41.600 | INFO | MemInfo: { | 20260507 15:30:41.600 | INFO | "gpu" : " 17 GB", | 20260507 15:30:41.600 | INFO | "huge" : "", | 20260507 15:30:41.600 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:41.600 | INFO | "malloc" : " 14 GB", | 20260507 15:30:41.600 | INFO | "pinned" : " 29 GB" | 20260507 15:30:41.600 | INFO | } | 20260507 15:30:41.600 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:30:41.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:41.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.600 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:30:41.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.600 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:30:41.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:41.601 | INFO | Uploaded 36GiB to GPU in 1.58713s (22.6825GiB/s) | 20260507 15:30:41.601 | INFO | Downloaded 34GiB to GPU in 1.53677s (22.1243GiB/s) | 20260507 15:30:46.739 | INFO | Analyzing chunk frames = [589312,589824) | 20260507 15:30:46.739 | INFO | MemInfo: { | 20260507 15:30:46.739 | INFO | "gpu" : " 17 GB", | 20260507 15:30:46.739 | INFO | "huge" : "", | 20260507 15:30:46.739 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:46.739 | INFO | "malloc" : " 14 GB", | 20260507 15:30:46.739 | INFO | "pinned" : " 29 GB" | 20260507 15:30:46.740 | INFO | } | 20260507 15:30:46.740 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:30:46.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:46.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:30:46.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:30:46.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:46.740 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:30:46.740 | INFO | Total mem usage: 16.4 GiB | 20260507 15:30:46.740 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:30:46.740 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.16%) | 20260507 15:30:46.740 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:30:46.740 | INFO | High Water Breakdown: | 20260507 15:30:46.740 | INFO | 59.6% : 9.12 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:30:46.740 | INFO | 40.3% : 6.17 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:30:46.740 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:30:46.740 | INFO | ------------------------------------------ | 20260507 15:30:46.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:30:46.740 | INFO | Uploaded 36GiB to GPU in 1.58987s (22.6434GiB/s) | 20260507 15:30:46.740 | INFO | Downloaded 34GiB to GPU in 1.53926s (22.0886GiB/s) | 20260507 15:30:46.740 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:30:46.740 | INFO | Total mem usage: 26.5 GiB | 20260507 15:30:46.740 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:30:46.741 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:30:46.741 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:30:46.741 | INFO | High Water Breakdown: | 20260507 15:30:46.741 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:30:46.741 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:30:46.741 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:30:46.741 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:30:46.741 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:30:46.741 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:30:46.741 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:30:46.741 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:30:46.741 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:30:46.741 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:30:46.741 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:30:46.741 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:30:46.741 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:30:46.741 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:30:46.741 | INFO | ------------------------------------------ | 20260507 15:30:46.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:30:46.741 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:30:46.741 | INFO | Total mem usage: 27.8 GiB | 20260507 15:30:46.741 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:30:46.741 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:30:46.741 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:30:46.741 | INFO | High Water Breakdown: | 20260507 15:30:46.741 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:30:46.741 | INFO | ------------------------------------------ | 20260507 15:30:46.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:30:46.741 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:30:46.741 | INFO | Total mem usage: 15.7 GiB | 20260507 15:30:46.741 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:30:46.741 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:30:46.741 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:30:46.741 | INFO | High Water Breakdown: | 20260507 15:30:46.741 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:30:46.741 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:30:46.741 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:30:46.741 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:30:46.741 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:30:46.741 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:30:46.741 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:30:46.741 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:30:46.741 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:30:46.741 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:30:46.741 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:30:46.741 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:30:46.741 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:30:46.741 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:30:46.741 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:30:46.741 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:30:46.741 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:30:46.741 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:30:46.741 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:30:46.741 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:30:46.741 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:30:46.741 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:30:46.741 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:30:46.741 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:30:46.741 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:30:46.741 | INFO | ------------------------------------------ | 20260507 15:30:46.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:30:46.741 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:30:46.741 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:30:46.741 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:30:46.741 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 5.22e-03 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 11.03%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.13e-02 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 3.46%, Metrics -- 1.03e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.79e-01 / -nan, 5.58e-02 / 0.00e+00 | 20260507 15:30:46.741 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:30:46.741 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:30:46.741 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:30:46.741 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 18.74%, Upload -- 1.67e+01 / 0.00e+00, 4.85e-01 / 0.00e+00 | 20260507 15:30:46.741 | INFO | PerfReport: 18.19%, Download -- 1.62e+01 / 0.00e+00, 8.06e-01 / 0.00e+00 | 20260507 15:30:46.742 | INFO | PerfReport: 33.44%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:30:46.742 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:30:46.742 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 2.15e-04 / 0.00e+00 | 20260507 15:30:46.742 | INFO | PerfReport: 29.05%, LockCompute -- 2.59e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 15:30:46.742 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260507 15:30:46.837 | INFO | Starting a round of Baz IO | 20260507 15:30:49.750 | INFO | Populating RTMetrics report | 20260507 15:30:49.751 | INFO | RTMetrics report populated | 20260507 15:30:49.751 | INFO | Opening tmp RTMetrics json file | 20260507 15:30:49.840 | INFO | Serializing RTMetrics to json | 20260507 15:30:49.898 | INFO | Serialization done, swapping tmp file now... | 20260507 15:30:49.923 | INFO | RTMetrics done | 20260507 15:30:50.052 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.214 seconds | 20260507 15:30:50.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.09% / 0.04% / 46.87% | 20260507 15:30:50.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.73407% / 0.000115236% / 3.97364e-05% | 20260507 15:30:50.052 | INFO | Baz IO overall: | 20260507 15:30:50.052 | INFO | BytesWritten : 240555541665 | 20260507 15:30:50.052 | INFO | NumEvents : 115914174340 | 20260507 15:30:51.879 | INFO | Analyzing chunk frames = [589824,590336) | 20260507 15:30:51.879 | INFO | MemInfo: { | 20260507 15:30:51.879 | INFO | "gpu" : " 17 GB", | 20260507 15:30:51.879 | INFO | "huge" : "", | 20260507 15:30:51.879 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:51.879 | INFO | "malloc" : " 14 GB", | 20260507 15:30:51.879 | INFO | "pinned" : " 29 GB" | 20260507 15:30:51.879 | INFO | } | 20260507 15:30:51.879 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:30:51.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:51.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:51.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:51.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:51.879 | INFO | ANALYSIS : 57.5%, 2.8, 84.1ms, 0 | 20260507 15:30:51.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:51.879 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260507 15:30:51.879 | INFO | PRE_HQ : 56.5%, 2.1, 62.6ms, 0 | 20260507 15:30:51.879 | INFO | BAZWRITER : 62.2%, 4.8, 152.8ms, 0 | 20260507 15:30:51.879 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 15:30:51.880 | INFO | Uploaded 36GiB to GPU in 1.65286s (21.7804GiB/s) | 20260507 15:30:51.880 | INFO | Downloaded 38GiB to GPU in 1.73551s (21.8956GiB/s) | 20260507 15:30:57.018 | INFO | Analyzing chunk frames = [590336,590848) | 20260507 15:30:57.018 | INFO | MemInfo: { | 20260507 15:30:57.018 | INFO | "gpu" : " 17 GB", | 20260507 15:30:57.018 | INFO | "huge" : "", | 20260507 15:30:57.018 | INFO | "huge_shared" : " 29 GB", | 20260507 15:30:57.018 | INFO | "malloc" : " 14 GB", | 20260507 15:30:57.018 | INFO | "pinned" : " 29 GB" | 20260507 15:30:57.018 | INFO | } | 20260507 15:30:57.018 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:30:57.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:30:57.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.018 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:30:57.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.018 | INFO | PRE_HQ : 34.4%, 1.0, 18.0ms, 0 | 20260507 15:30:57.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:30:57.019 | INFO | Uploaded 36GiB to GPU in 1.59993s (22.501GiB/s) | 20260507 15:30:57.019 | INFO | Downloaded 34GiB to GPU in 1.53123s (22.2043GiB/s) | 20260507 15:31:02.157 | INFO | Analyzing chunk frames = [590848,591360) | 20260507 15:31:02.157 | INFO | MemInfo: { | 20260507 15:31:02.157 | INFO | "gpu" : " 17 GB", | 20260507 15:31:02.157 | INFO | "huge" : "", | 20260507 15:31:02.157 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:02.157 | INFO | "malloc" : " 14 GB", | 20260507 15:31:02.157 | INFO | "pinned" : " 29 GB" | 20260507 15:31:02.158 | INFO | } | 20260507 15:31:02.158 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:31:02.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:02.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:31:02.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:31:02.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:02.158 | INFO | Uploaded 36GiB to GPU in 1.58842s (22.664GiB/s) | 20260507 15:31:02.158 | INFO | Downloaded 34GiB to GPU in 1.53946s (22.0856GiB/s) | 20260507 15:31:07.297 | INFO | Analyzing chunk frames = [591360,591872) | 20260507 15:31:07.297 | INFO | MemInfo: { | 20260507 15:31:07.297 | INFO | "gpu" : " 17 GB", | 20260507 15:31:07.297 | INFO | "huge" : "", | 20260507 15:31:07.297 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:07.297 | INFO | "malloc" : " 14 GB", | 20260507 15:31:07.297 | INFO | "pinned" : " 29 GB" | 20260507 15:31:07.297 | INFO | } | 20260507 15:31:07.297 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:31:07.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:07.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:31:07.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:31:07.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:07.297 | INFO | Uploaded 36GiB to GPU in 1.58819s (22.6673GiB/s) | 20260507 15:31:07.297 | INFO | Downloaded 34GiB to GPU in 1.53883s (22.0947GiB/s) | 20260507 15:31:12.436 | INFO | Analyzing chunk frames = [591872,592384) | 20260507 15:31:12.436 | INFO | MemInfo: { | 20260507 15:31:12.436 | INFO | "gpu" : " 17 GB", | 20260507 15:31:12.436 | INFO | "huge" : "", | 20260507 15:31:12.436 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:12.436 | INFO | "malloc" : " 14 GB", | 20260507 15:31:12.436 | INFO | "pinned" : " 29 GB" | 20260507 15:31:12.436 | INFO | } | 20260507 15:31:12.436 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:31:12.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:12.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.436 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:31:12.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.436 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:31:12.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:12.437 | INFO | Uploaded 36GiB to GPU in 1.58737s (22.679GiB/s) | 20260507 15:31:12.437 | INFO | Downloaded 34GiB to GPU in 1.53573s (22.1393GiB/s) | 20260507 15:31:17.575 | INFO | Analyzing chunk frames = [592384,592896) | 20260507 15:31:17.575 | INFO | MemInfo: { | 20260507 15:31:17.575 | INFO | "gpu" : " 17 GB", | 20260507 15:31:17.575 | INFO | "huge" : "", | 20260507 15:31:17.575 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:17.575 | INFO | "malloc" : " 14 GB", | 20260507 15:31:17.575 | INFO | "pinned" : " 29 GB" | 20260507 15:31:17.575 | INFO | } | 20260507 15:31:17.575 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:31:17.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:17.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:31:17.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:31:17.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:17.576 | INFO | Uploaded 36GiB to GPU in 1.58896s (22.6563GiB/s) | 20260507 15:31:17.576 | INFO | Downloaded 34GiB to GPU in 1.53645s (22.129GiB/s) | 20260507 15:31:22.715 | INFO | Analyzing chunk frames = [592896,593408) | 20260507 15:31:22.715 | INFO | MemInfo: { | 20260507 15:31:22.715 | INFO | "gpu" : " 17 GB", | 20260507 15:31:22.715 | INFO | "huge" : "", | 20260507 15:31:22.715 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:22.715 | INFO | "malloc" : " 14 GB", | 20260507 15:31:22.715 | INFO | "pinned" : " 29 GB" | 20260507 15:31:22.715 | INFO | } | 20260507 15:31:22.715 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:31:22.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:22.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:31:22.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:31:22.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:22.715 | INFO | Uploaded 36GiB to GPU in 1.58724s (22.6809GiB/s) | 20260507 15:31:22.715 | INFO | Downloaded 34GiB to GPU in 1.53714s (22.119GiB/s) | 20260507 15:31:27.854 | INFO | Analyzing chunk frames = [593408,593920) | 20260507 15:31:27.854 | INFO | MemInfo: { | 20260507 15:31:27.854 | INFO | "gpu" : " 17 GB", | 20260507 15:31:27.854 | INFO | "huge" : "", | 20260507 15:31:27.854 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:27.854 | INFO | "malloc" : " 14 GB", | 20260507 15:31:27.854 | INFO | "pinned" : " 29 GB" | 20260507 15:31:27.854 | INFO | } | 20260507 15:31:27.854 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:31:27.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:27.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:31:27.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:31:27.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:27.855 | INFO | Uploaded 36GiB to GPU in 1.58826s (22.6664GiB/s) | 20260507 15:31:27.855 | INFO | Downloaded 34GiB to GPU in 1.53744s (22.1147GiB/s) | 20260507 15:31:27.952 | INFO | Starting a round of Baz IO | 20260507 15:31:30.873 | INFO | Populating RTMetrics report | 20260507 15:31:30.873 | INFO | RTMetrics report populated | 20260507 15:31:30.873 | INFO | Opening tmp RTMetrics json file | 20260507 15:31:30.880 | INFO | Serializing RTMetrics to json | 20260507 15:31:30.902 | INFO | Serialization done, swapping tmp file now... | 20260507 15:31:30.904 | INFO | RTMetrics done | 20260507 15:31:31.011 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.058 seconds | 20260507 15:31:31.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.05% / 0.04% / 46.91% | 20260507 15:31:31.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.71144% / 0.000111262% / 4.37101e-05% | 20260507 15:31:31.011 | INFO | Baz IO overall: | 20260507 15:31:31.011 | INFO | BytesWritten : 242438242959 | 20260507 15:31:31.011 | INFO | NumEvents : 116830386678 | 20260507 15:31:32.994 | INFO | Analyzing chunk frames = [593920,594432) | 20260507 15:31:32.994 | INFO | MemInfo: { | 20260507 15:31:32.994 | INFO | "gpu" : " 17 GB", | 20260507 15:31:32.994 | INFO | "huge" : "", | 20260507 15:31:32.994 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:32.994 | INFO | "malloc" : " 14 GB", | 20260507 15:31:32.994 | INFO | "pinned" : " 29 GB" | 20260507 15:31:32.994 | INFO | } | 20260507 15:31:32.994 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:31:32.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:32.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:32.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:32.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:32.994 | INFO | ANALYSIS : 57.9%, 2.8, 85.0ms, 0 | 20260507 15:31:32.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:32.994 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 15:31:32.994 | INFO | PRE_HQ : 56.9%, 2.1, 63.1ms, 0 | 20260507 15:31:32.994 | INFO | BAZWRITER : 59.1%, 4.6, 138.7ms, 0 | 20260507 15:31:32.994 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 15:31:32.994 | INFO | Uploaded 36GiB to GPU in 1.63998s (21.9514GiB/s) | 20260507 15:31:32.994 | INFO | Downloaded 38GiB to GPU in 1.7152s (22.1549GiB/s) | 20260507 15:31:38.133 | INFO | Analyzing chunk frames = [594432,594944) | 20260507 15:31:38.133 | INFO | MemInfo: { | 20260507 15:31:38.133 | INFO | "gpu" : " 17 GB", | 20260507 15:31:38.133 | INFO | "huge" : "", | 20260507 15:31:38.133 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:38.133 | INFO | "malloc" : " 14 GB", | 20260507 15:31:38.133 | INFO | "pinned" : " 29 GB" | 20260507 15:31:38.133 | INFO | } | 20260507 15:31:38.133 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:31:38.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:38.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.133 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:31:38.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.133 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:31:38.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:38.134 | INFO | Uploaded 36GiB to GPU in 1.60515s (22.4279GiB/s) | 20260507 15:31:38.134 | INFO | Downloaded 34GiB to GPU in 1.5245s (22.3024GiB/s) | 20260507 15:31:43.272 | INFO | Analyzing chunk frames = [594944,595456) | 20260507 15:31:43.272 | INFO | MemInfo: { | 20260507 15:31:43.272 | INFO | "gpu" : " 17 GB", | 20260507 15:31:43.272 | INFO | "huge" : "", | 20260507 15:31:43.272 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:43.272 | INFO | "malloc" : " 14 GB", | 20260507 15:31:43.272 | INFO | "pinned" : " 29 GB" | 20260507 15:31:43.272 | INFO | } | 20260507 15:31:43.272 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:31:43.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:43.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.272 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:31:43.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.273 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:31:43.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:43.273 | INFO | Uploaded 36GiB to GPU in 1.60929s (22.3701GiB/s) | 20260507 15:31:43.273 | INFO | Downloaded 34GiB to GPU in 1.53007s (22.2212GiB/s) | 20260507 15:31:48.412 | INFO | Analyzing chunk frames = [595456,595968) | 20260507 15:31:48.412 | INFO | MemInfo: { | 20260507 15:31:48.412 | INFO | "gpu" : " 17 GB", | 20260507 15:31:48.412 | INFO | "huge" : "", | 20260507 15:31:48.412 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:48.412 | INFO | "malloc" : " 14 GB", | 20260507 15:31:48.412 | INFO | "pinned" : " 29 GB" | 20260507 15:31:48.412 | INFO | } | 20260507 15:31:48.412 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:31:48.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:48.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:31:48.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:31:48.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:48.412 | INFO | Uploaded 36GiB to GPU in 1.60875s (22.3777GiB/s) | 20260507 15:31:48.412 | INFO | Downloaded 34GiB to GPU in 1.52923s (22.2334GiB/s) | 20260507 15:31:53.551 | INFO | Analyzing chunk frames = [595968,596480) | 20260507 15:31:53.551 | INFO | MemInfo: { | 20260507 15:31:53.551 | INFO | "gpu" : " 17 GB", | 20260507 15:31:53.551 | INFO | "huge" : "", | 20260507 15:31:53.551 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:53.551 | INFO | "malloc" : " 14 GB", | 20260507 15:31:53.551 | INFO | "pinned" : " 29 GB" | 20260507 15:31:53.551 | INFO | } | 20260507 15:31:53.551 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:31:53.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:53.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:31:53.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:31:53.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:53.551 | INFO | Uploaded 36GiB to GPU in 1.60812s (22.3864GiB/s) | 20260507 15:31:53.551 | INFO | Downloaded 34GiB to GPU in 1.52748s (22.2589GiB/s) | 20260507 15:31:58.690 | INFO | Analyzing chunk frames = [596480,596992) | 20260507 15:31:58.690 | INFO | MemInfo: { | 20260507 15:31:58.690 | INFO | "gpu" : " 17 GB", | 20260507 15:31:58.690 | INFO | "huge" : "", | 20260507 15:31:58.690 | INFO | "huge_shared" : " 29 GB", | 20260507 15:31:58.690 | INFO | "malloc" : " 14 GB", | 20260507 15:31:58.690 | INFO | "pinned" : " 29 GB" | 20260507 15:31:58.690 | INFO | } | 20260507 15:31:58.690 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:31:58.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:31:58.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.690 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:31:58.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.690 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:31:58.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:31:58.691 | INFO | Uploaded 36GiB to GPU in 1.60726s (22.3983GiB/s) | 20260507 15:31:58.691 | INFO | Downloaded 34GiB to GPU in 1.52669s (22.2704GiB/s) | 20260507 15:32:03.830 | INFO | Analyzing chunk frames = [596992,597504) | 20260507 15:32:03.830 | INFO | MemInfo: { | 20260507 15:32:03.830 | INFO | "gpu" : " 17 GB", | 20260507 15:32:03.830 | INFO | "huge" : "", | 20260507 15:32:03.830 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:03.830 | INFO | "malloc" : " 14 GB", | 20260507 15:32:03.830 | INFO | "pinned" : " 29 GB" | 20260507 15:32:03.830 | INFO | } | 20260507 15:32:03.830 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:32:03.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:03.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:32:03.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:32:03.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:03.830 | INFO | Uploaded 36GiB to GPU in 1.60785s (22.3901GiB/s) | 20260507 15:32:03.830 | INFO | Downloaded 34GiB to GPU in 1.5298s (22.2252GiB/s) | 20260507 15:32:08.969 | INFO | Analyzing chunk frames = [597504,598016) | 20260507 15:32:08.969 | INFO | MemInfo: { | 20260507 15:32:08.969 | INFO | "gpu" : " 17 GB", | 20260507 15:32:08.969 | INFO | "huge" : "", | 20260507 15:32:08.969 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:08.969 | INFO | "malloc" : " 14 GB", | 20260507 15:32:08.969 | INFO | "pinned" : " 29 GB" | 20260507 15:32:08.969 | INFO | } | 20260507 15:32:08.969 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:32:08.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:08.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.969 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:32:08.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.969 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 15:32:08.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:08.970 | INFO | Uploaded 36GiB to GPU in 1.61193s (22.3334GiB/s) | 20260507 15:32:08.970 | INFO | Downloaded 34GiB to GPU in 1.53075s (22.2113GiB/s) | 20260507 15:32:08.970 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:32:08.970 | INFO | Total mem usage: 16.4 GiB | 20260507 15:32:08.970 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:32:08.970 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.16%) | 20260507 15:32:08.970 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:32:08.970 | INFO | High Water Breakdown: | 20260507 15:32:08.970 | INFO | 60.4% : 9.7 GiB ( 97.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:32:08.970 | INFO | 39.5% : 6.34 GiB ( 94.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:32:08.970 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:32:08.970 | INFO | ------------------------------------------ | 20260507 15:32:08.970 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:32:08.970 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:32:08.970 | INFO | Total mem usage: 26.5 GiB | 20260507 15:32:08.970 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:32:08.970 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 15:32:08.970 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:32:08.970 | INFO | High Water Breakdown: | 20260507 15:32:08.970 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:32:08.970 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:32:08.970 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:32:08.970 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:32:08.970 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:32:08.970 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:32:08.970 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:32:08.970 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:32:08.970 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:32:08.970 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:32:08.970 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:32:08.970 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:32:08.970 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:32:08.970 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:32:08.970 | INFO | ------------------------------------------ | 20260507 15:32:08.970 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:32:08.970 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:32:08.970 | INFO | Total mem usage: 27.8 GiB | 20260507 15:32:08.970 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:32:08.970 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:32:08.970 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:32:08.970 | INFO | High Water Breakdown: | 20260507 15:32:08.970 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:32:08.970 | INFO | ------------------------------------------ | 20260507 15:32:08.970 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:32:08.970 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:32:08.970 | INFO | Total mem usage: 15.7 GiB | 20260507 15:32:08.970 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:32:08.970 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:32:08.970 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:32:08.970 | INFO | High Water Breakdown: | 20260507 15:32:08.970 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:32:08.970 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:32:08.970 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:32:08.970 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:32:08.970 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:32:08.970 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:32:08.970 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:32:08.970 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:32:08.970 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:32:08.970 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:32:08.970 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:32:08.970 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:32:08.970 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:32:08.970 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:32:08.970 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:32:08.970 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:32:08.970 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:32:08.970 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:32:08.970 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:32:08.970 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:32:08.970 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:32:08.970 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:32:08.970 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:32:08.970 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:32:08.970 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:32:08.970 | INFO | ------------------------------------------ | 20260507 15:32:08.970 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:32:08.970 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:32:08.971 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:32:08.971 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:32:08.971 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 0.29%, AggregateStats -- 8.72e-02 / -nan, 5.97e-03 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 11.04%, Baseline -- 3.30e+00 / -nan, 2.08e-02 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.72e+01 / -nan, 4.42e-02 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 3.46%, Metrics -- 1.03e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.80e-01 / -nan, 5.61e-02 / 0.00e+00 | 20260507 15:32:08.971 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:32:08.971 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:32:08.971 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:32:08.971 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 18.76%, Upload -- 1.67e+01 / 0.00e+00, 4.59e-01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 18.20%, Download -- 1.62e+01 / 0.00e+00, 7.68e-01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 33.50%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 0.59%, LockUpload -- 5.27e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 0.00%, LockDownload -- 4.66e-04 / 0.00e+00, 2.59e-04 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 28.94%, LockCompute -- 2.58e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260507 15:32:08.971 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260507 15:32:09.066 | INFO | Starting a round of Baz IO | 20260507 15:32:11.979 | INFO | Populating RTMetrics report | 20260507 15:32:11.980 | INFO | RTMetrics report populated | 20260507 15:32:11.980 | INFO | Opening tmp RTMetrics json file | 20260507 15:32:11.981 | INFO | Serializing RTMetrics to json | 20260507 15:32:11.984 | INFO | Serialization done, swapping tmp file now... | 20260507 15:32:11.984 | INFO | RTMetrics done | 20260507 15:32:12.066 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3 seconds | 20260507 15:32:12.066 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.01% / 0.04% / 46.95% | 20260507 15:32:12.066 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.6987% / 8.34465e-05% / 3.97364e-05% | 20260507 15:32:12.066 | INFO | Baz IO overall: | 20260507 15:32:12.066 | INFO | BytesWritten : 244322288764 | 20260507 15:32:12.066 | INFO | NumEvents : 117747401104 | 20260507 15:32:14.109 | INFO | Analyzing chunk frames = [598016,598528) | 20260507 15:32:14.109 | INFO | MemInfo: { | 20260507 15:32:14.109 | INFO | "gpu" : " 17 GB", | 20260507 15:32:14.109 | INFO | "huge" : "", | 20260507 15:32:14.109 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:14.109 | INFO | "malloc" : " 14 GB", | 20260507 15:32:14.109 | INFO | "pinned" : " 29 GB" | 20260507 15:32:14.109 | INFO | } | 20260507 15:32:14.109 | INFO | Perf report for the last 5.13977 seconds: | 20260507 15:32:14.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:14.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:14.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:14.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:14.109 | INFO | ANALYSIS : 57.7%, 2.8, 85.8ms, 0 | 20260507 15:32:14.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:14.109 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 15:32:14.109 | INFO | PRE_HQ : 56.6%, 2.2, 63.3ms, 0 | 20260507 15:32:14.109 | INFO | BAZWRITER : 57.9%, 4.5, 135.2ms, 0 | 20260507 15:32:14.109 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 15:32:14.110 | INFO | Uploaded 36GiB to GPU in 1.64836s (21.8399GiB/s) | 20260507 15:32:14.110 | INFO | Downloaded 38GiB to GPU in 1.71477s (22.1605GiB/s) | 20260507 15:32:19.248 | INFO | Analyzing chunk frames = [598528,599040) | 20260507 15:32:19.248 | INFO | MemInfo: { | 20260507 15:32:19.248 | INFO | "gpu" : " 17 GB", | 20260507 15:32:19.248 | INFO | "huge" : "", | 20260507 15:32:19.248 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:19.248 | INFO | "malloc" : " 14 GB", | 20260507 15:32:19.248 | INFO | "pinned" : " 29 GB" | 20260507 15:32:19.248 | INFO | } | 20260507 15:32:19.248 | INFO | Perf report for the last 5.13877 seconds: | 20260507 15:32:19.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:19.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:32:19.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260507 15:32:19.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:19.248 | INFO | Uploaded 36GiB to GPU in 1.60778s (22.3911GiB/s) | 20260507 15:32:19.248 | INFO | Downloaded 34GiB to GPU in 1.52602s (22.2802GiB/s) | 20260507 15:32:24.387 | INFO | Analyzing chunk frames = [599040,599552) | 20260507 15:32:24.387 | INFO | MemInfo: { | 20260507 15:32:24.387 | INFO | "gpu" : " 17 GB", | 20260507 15:32:24.387 | INFO | "huge" : "", | 20260507 15:32:24.387 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:24.387 | INFO | "malloc" : " 14 GB", | 20260507 15:32:24.387 | INFO | "pinned" : " 29 GB" | 20260507 15:32:24.387 | INFO | } | 20260507 15:32:24.387 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:32:24.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:24.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.387 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:32:24.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.387 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:32:24.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:24.388 | INFO | Uploaded 36GiB to GPU in 1.60932s (22.3697GiB/s) | 20260507 15:32:24.388 | INFO | Downloaded 34GiB to GPU in 1.52923s (22.2334GiB/s) | 20260507 15:32:29.526 | INFO | Analyzing chunk frames = [599552,600064) | 20260507 15:32:29.526 | INFO | MemInfo: { | 20260507 15:32:29.526 | INFO | "gpu" : " 17 GB", | 20260507 15:32:29.526 | INFO | "huge" : "", | 20260507 15:32:29.526 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:29.526 | INFO | "malloc" : " 14 GB", | 20260507 15:32:29.526 | INFO | "pinned" : " 29 GB" | 20260507 15:32:29.526 | INFO | } | 20260507 15:32:29.527 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:32:29.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:29.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:32:29.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:32:29.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:29.527 | INFO | Uploaded 36GiB to GPU in 1.6092s (22.3714GiB/s) | 20260507 15:32:29.527 | INFO | Downloaded 34GiB to GPU in 1.52917s (22.2343GiB/s) | 20260507 15:32:34.666 | INFO | Analyzing chunk frames = [600064,600576) | 20260507 15:32:34.666 | INFO | MemInfo: { | 20260507 15:32:34.666 | INFO | "gpu" : " 17 GB", | 20260507 15:32:34.666 | INFO | "huge" : "", | 20260507 15:32:34.666 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:34.666 | INFO | "malloc" : " 14 GB", | 20260507 15:32:34.666 | INFO | "pinned" : " 29 GB" | 20260507 15:32:34.666 | INFO | } | 20260507 15:32:34.666 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:32:34.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:34.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:32:34.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:32:34.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:34.666 | INFO | Uploaded 36GiB to GPU in 1.60813s (22.3863GiB/s) | 20260507 15:32:34.666 | INFO | Downloaded 34GiB to GPU in 1.52711s (22.2642GiB/s) | 20260507 15:32:39.805 | INFO | Analyzing chunk frames = [600576,601088) | 20260507 15:32:39.805 | INFO | MemInfo: { | 20260507 15:32:39.805 | INFO | "gpu" : " 17 GB", | 20260507 15:32:39.805 | INFO | "huge" : "", | 20260507 15:32:39.805 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:39.805 | INFO | "malloc" : " 14 GB", | 20260507 15:32:39.805 | INFO | "pinned" : " 29 GB" | 20260507 15:32:39.805 | INFO | } | 20260507 15:32:39.805 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:32:39.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:39.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:32:39.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:32:39.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:39.805 | INFO | Uploaded 36GiB to GPU in 1.60764s (22.3931GiB/s) | 20260507 15:32:39.805 | INFO | Downloaded 34GiB to GPU in 1.52741s (22.2598GiB/s) | 20260507 15:32:44.944 | INFO | Analyzing chunk frames = [601088,601600) | 20260507 15:32:44.944 | INFO | MemInfo: { | 20260507 15:32:44.944 | INFO | "gpu" : " 17 GB", | 20260507 15:32:44.944 | INFO | "huge" : "", | 20260507 15:32:44.944 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:44.944 | INFO | "malloc" : " 14 GB", | 20260507 15:32:44.944 | INFO | "pinned" : " 29 GB" | 20260507 15:32:44.944 | INFO | } | 20260507 15:32:44.944 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:32:44.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:44.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.944 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:32:44.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.945 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:32:44.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:44.945 | INFO | Uploaded 36GiB to GPU in 1.60787s (22.3898GiB/s) | 20260507 15:32:44.945 | INFO | Downloaded 34GiB to GPU in 1.52884s (22.239GiB/s) | 20260507 15:32:50.084 | INFO | Analyzing chunk frames = [601600,602112) | 20260507 15:32:50.084 | INFO | MemInfo: { | 20260507 15:32:50.084 | INFO | "gpu" : " 17 GB", | 20260507 15:32:50.084 | INFO | "huge" : "", | 20260507 15:32:50.084 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:50.084 | INFO | "malloc" : " 14 GB", | 20260507 15:32:50.084 | INFO | "pinned" : " 29 GB" | 20260507 15:32:50.084 | INFO | } | 20260507 15:32:50.084 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:32:50.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:50.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260507 15:32:50.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 15:32:50.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:50.084 | INFO | Uploaded 36GiB to GPU in 1.60899s (22.3743GiB/s) | 20260507 15:32:50.085 | INFO | Downloaded 34GiB to GPU in 1.52867s (22.2415GiB/s) | 20260507 15:32:50.199 | INFO | Starting a round of Baz IO | 20260507 15:32:53.080 | INFO | Populating RTMetrics report | 20260507 15:32:53.080 | INFO | RTMetrics report populated | 20260507 15:32:53.080 | INFO | Opening tmp RTMetrics json file | 20260507 15:32:53.104 | INFO | Serializing RTMetrics to json | 20260507 15:32:53.137 | INFO | Serialization done, swapping tmp file now... | 20260507 15:32:53.157 | INFO | RTMetrics done | 20260507 15:32:53.215 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.015 seconds | 20260507 15:32:53.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.96% / 0.04% / 46.99% | 20260507 15:32:53.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.6943% / 8.34465e-05% / 3.17891e-05% | 20260507 15:32:53.215 | INFO | Baz IO overall: | 20260507 15:32:53.215 | INFO | BytesWritten : 246208194216 | 20260507 15:32:53.215 | INFO | NumEvents : 118665865087 | 20260507 15:32:55.223 | INFO | Analyzing chunk frames = [602112,602624) | 20260507 15:32:55.223 | INFO | MemInfo: { | 20260507 15:32:55.223 | INFO | "gpu" : " 17 GB", | 20260507 15:32:55.223 | INFO | "huge" : "", | 20260507 15:32:55.223 | INFO | "huge_shared" : " 29 GB", | 20260507 15:32:55.223 | INFO | "malloc" : " 14 GB", | 20260507 15:32:55.223 | INFO | "pinned" : " 29 GB" | 20260507 15:32:55.223 | INFO | } | 20260507 15:32:55.223 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:32:55.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:32:55.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:55.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:55.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:55.223 | INFO | ANALYSIS : 57.4%, 2.9, 86.9ms, 0 | 20260507 15:32:55.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:32:55.223 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 15:32:55.223 | INFO | PRE_HQ : 56.4%, 2.2, 63.8ms, 0 | 20260507 15:32:55.223 | INFO | BAZWRITER : 58.3%, 4.4, 133.1ms, 0 | 20260507 15:32:55.223 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 15:32:55.223 | INFO | Uploaded 36GiB to GPU in 1.64595s (21.8719GiB/s) | 20260507 15:32:55.223 | INFO | Downloaded 38GiB to GPU in 1.72589s (22.0177GiB/s) | 20260507 15:33:00.362 | INFO | Analyzing chunk frames = [602624,603136) | 20260507 15:33:00.362 | INFO | MemInfo: { | 20260507 15:33:00.362 | INFO | "gpu" : " 17 GB", | 20260507 15:33:00.362 | INFO | "huge" : "", | 20260507 15:33:00.362 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:00.362 | INFO | "malloc" : " 14 GB", | 20260507 15:33:00.362 | INFO | "pinned" : " 29 GB" | 20260507 15:33:00.362 | INFO | } | 20260507 15:33:00.363 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:33:00.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:00.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:33:00.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:33:00.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:00.363 | INFO | Uploaded 36GiB to GPU in 1.58564s (22.7038GiB/s) | 20260507 15:33:00.363 | INFO | Downloaded 34GiB to GPU in 1.53414s (22.1623GiB/s) | 20260507 15:33:05.502 | INFO | Analyzing chunk frames = [603136,603648) | 20260507 15:33:05.502 | INFO | MemInfo: { | 20260507 15:33:05.502 | INFO | "gpu" : " 17 GB", | 20260507 15:33:05.502 | INFO | "huge" : "", | 20260507 15:33:05.502 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:05.502 | INFO | "malloc" : " 14 GB", | 20260507 15:33:05.502 | INFO | "pinned" : " 29 GB" | 20260507 15:33:05.502 | INFO | } | 20260507 15:33:05.502 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:33:05.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:05.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.502 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:33:05.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.502 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:33:05.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:05.503 | INFO | Uploaded 36GiB to GPU in 1.58912s (22.6541GiB/s) | 20260507 15:33:05.503 | INFO | Downloaded 34GiB to GPU in 1.53916s (22.09GiB/s) | 20260507 15:33:10.641 | INFO | Analyzing chunk frames = [603648,604160) | 20260507 15:33:10.641 | INFO | MemInfo: { | 20260507 15:33:10.641 | INFO | "gpu" : " 17 GB", | 20260507 15:33:10.641 | INFO | "huge" : "", | 20260507 15:33:10.641 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:10.641 | INFO | "malloc" : " 14 GB", | 20260507 15:33:10.641 | INFO | "pinned" : " 29 GB" | 20260507 15:33:10.641 | INFO | } | 20260507 15:33:10.641 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:33:10.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:10.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.641 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:33:10.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.641 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:33:10.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:10.642 | INFO | Uploaded 36GiB to GPU in 1.58978s (22.6446GiB/s) | 20260507 15:33:10.642 | INFO | Downloaded 34GiB to GPU in 1.53781s (22.1093GiB/s) | 20260507 15:33:15.780 | INFO | Analyzing chunk frames = [604160,604672) | 20260507 15:33:15.780 | INFO | MemInfo: { | 20260507 15:33:15.780 | INFO | "gpu" : " 17 GB", | 20260507 15:33:15.780 | INFO | "huge" : "", | 20260507 15:33:15.780 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:15.780 | INFO | "malloc" : " 14 GB", | 20260507 15:33:15.780 | INFO | "pinned" : " 29 GB" | 20260507 15:33:15.780 | INFO | } | 20260507 15:33:15.780 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:33:15.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:15.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.781 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:33:15.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.781 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:33:15.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:15.781 | INFO | Uploaded 36GiB to GPU in 1.58837s (22.6647GiB/s) | 20260507 15:33:15.781 | INFO | Downloaded 34GiB to GPU in 1.53647s (22.1286GiB/s) | 20260507 15:33:20.920 | INFO | Analyzing chunk frames = [604672,605184) | 20260507 15:33:20.920 | INFO | MemInfo: { | 20260507 15:33:20.920 | INFO | "gpu" : " 17 GB", | 20260507 15:33:20.920 | INFO | "huge" : "", | 20260507 15:33:20.920 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:20.920 | INFO | "malloc" : " 14 GB", | 20260507 15:33:20.920 | INFO | "pinned" : " 29 GB" | 20260507 15:33:20.920 | INFO | } | 20260507 15:33:20.920 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:33:20.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:20.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:33:20.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:33:20.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:20.920 | INFO | Uploaded 36GiB to GPU in 1.58966s (22.6464GiB/s) | 20260507 15:33:20.920 | INFO | Downloaded 34GiB to GPU in 1.53615s (22.1332GiB/s) | 20260507 15:33:26.059 | INFO | Analyzing chunk frames = [605184,605696) | 20260507 15:33:26.059 | INFO | MemInfo: { | 20260507 15:33:26.059 | INFO | "gpu" : " 17 GB", | 20260507 15:33:26.059 | INFO | "huge" : "", | 20260507 15:33:26.059 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:26.059 | INFO | "malloc" : " 14 GB", | 20260507 15:33:26.059 | INFO | "pinned" : " 29 GB" | 20260507 15:33:26.060 | INFO | } | 20260507 15:33:26.060 | INFO | Perf report for the last 5.13969 seconds: | 20260507 15:33:26.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:26.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.060 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:33:26.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.060 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:33:26.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:26.061 | INFO | Uploaded 36GiB to GPU in 1.58822s (22.6668GiB/s) | 20260507 15:33:26.061 | INFO | Downloaded 34GiB to GPU in 1.53796s (22.1072GiB/s) | 20260507 15:33:31.198 | INFO | Analyzing chunk frames = [605696,606208) | 20260507 15:33:31.198 | INFO | MemInfo: { | 20260507 15:33:31.198 | INFO | "gpu" : " 17 GB", | 20260507 15:33:31.198 | INFO | "huge" : "", | 20260507 15:33:31.198 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:31.198 | INFO | "malloc" : " 14 GB", | 20260507 15:33:31.198 | INFO | "pinned" : " 29 GB" | 20260507 15:33:31.198 | INFO | } | 20260507 15:33:31.199 | INFO | Perf report for the last 5.13891 seconds: | 20260507 15:33:31.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:31.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:33:31.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 15:33:31.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:31.199 | INFO | Uploaded 36GiB to GPU in 1.58932s (22.6511GiB/s) | 20260507 15:33:31.199 | INFO | Downloaded 34GiB to GPU in 1.53844s (22.1003GiB/s) | 20260507 15:33:31.199 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:33:31.199 | INFO | Total mem usage: 16.4 GiB | 20260507 15:33:31.199 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:33:31.199 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.73%) | 20260507 15:33:31.199 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:33:31.199 | INFO | High Water Breakdown: | 20260507 15:33:31.199 | INFO | 59.2% : 8.85 GiB ( 89.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:33:31.199 | INFO | 40.7% : 6.08 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:33:31.199 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:33:31.199 | INFO | ------------------------------------------ | 20260507 15:33:31.199 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:33:31.199 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:33:31.199 | INFO | Total mem usage: 26.5 GiB | 20260507 15:33:31.199 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:33:31.199 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:33:31.199 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:33:31.199 | INFO | High Water Breakdown: | 20260507 15:33:31.199 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:33:31.199 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:33:31.199 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:33:31.199 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:33:31.199 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:33:31.199 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:33:31.199 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:33:31.199 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:33:31.199 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:33:31.199 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:33:31.199 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:33:31.199 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:33:31.199 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:33:31.199 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:33:31.199 | INFO | ------------------------------------------ | 20260507 15:33:31.199 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:33:31.199 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:33:31.199 | INFO | Total mem usage: 27.8 GiB | 20260507 15:33:31.199 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:33:31.199 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:33:31.199 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:33:31.199 | INFO | High Water Breakdown: | 20260507 15:33:31.199 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:33:31.199 | INFO | ------------------------------------------ | 20260507 15:33:31.199 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:33:31.199 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:33:31.199 | INFO | Total mem usage: 15.7 GiB | 20260507 15:33:31.199 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:33:31.199 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:33:31.200 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:33:31.200 | INFO | High Water Breakdown: | 20260507 15:33:31.200 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:33:31.200 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:33:31.200 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:33:31.200 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:33:31.200 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:33:31.200 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:33:31.200 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:33:31.200 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:33:31.200 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:33:31.200 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:33:31.200 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:33:31.200 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:33:31.200 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:33:31.200 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:33:31.200 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:33:31.200 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:33:31.200 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:33:31.200 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:33:31.200 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:33:31.200 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:33:31.200 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:33:31.200 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:33:31.200 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:33:31.200 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:33:31.200 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:33:31.200 | INFO | ------------------------------------------ | 20260507 15:33:31.200 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:33:31.200 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:33:31.200 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:33:31.200 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:33:31.200 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 0.29%, AggregateStats -- 8.69e-02 / -nan, 4.69e-03 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 11.04%, Baseline -- 3.29e+00 / -nan, 1.91e-02 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.72e+01 / -nan, 4.41e-02 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 3.46%, Metrics -- 1.03e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.79e-01 / -nan, 5.62e-02 / 0.00e+00 | 20260507 15:33:31.200 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:33:31.200 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:33:31.200 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:33:31.200 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 18.71%, Upload -- 1.67e+01 / 0.00e+00, 4.48e-01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 18.14%, Download -- 1.62e+01 / 0.00e+00, 7.46e-01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 33.41%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 0.58%, LockUpload -- 5.22e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 0.00%, LockDownload -- 4.58e-04 / 0.00e+00, 2.87e-04 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 29.14%, LockCompute -- 2.60e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:33:31.200 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.57e-02 / 0.00e+00 | 20260507 15:33:31.307 | INFO | Starting a round of Baz IO | 20260507 15:33:34.177 | INFO | Populating RTMetrics report | 20260507 15:33:34.178 | INFO | RTMetrics report populated | 20260507 15:33:34.178 | INFO | Opening tmp RTMetrics json file | 20260507 15:33:34.254 | INFO | Serializing RTMetrics to json | 20260507 15:33:34.265 | INFO | Serialization done, swapping tmp file now... | 20260507 15:33:34.272 | INFO | RTMetrics done | 20260507 15:33:34.296 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 2.988 seconds | 20260507 15:33:34.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.92% / 0.04% / 47.04% | 20260507 15:33:34.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.66725% / 8.34465e-05% / 3.57628e-05% | 20260507 15:33:34.296 | INFO | Baz IO overall: | 20260507 15:33:34.297 | INFO | BytesWritten : 248093421709 | 20260507 15:33:34.297 | INFO | NumEvents : 119583559808 | 20260507 15:33:36.338 | INFO | Analyzing chunk frames = [606208,606720) | 20260507 15:33:36.338 | INFO | MemInfo: { | 20260507 15:33:36.338 | INFO | "gpu" : " 17 GB", | 20260507 15:33:36.338 | INFO | "huge" : "", | 20260507 15:33:36.338 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:36.338 | INFO | "malloc" : " 14 GB", | 20260507 15:33:36.338 | INFO | "pinned" : " 29 GB" | 20260507 15:33:36.338 | INFO | } | 20260507 15:33:36.338 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:33:36.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:36.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:36.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:36.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:36.338 | INFO | ANALYSIS : 57.1%, 2.9, 86.5ms, 0 | 20260507 15:33:36.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:36.338 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 15:33:36.338 | INFO | PRE_HQ : 55.9%, 2.1, 60.1ms, 0 | 20260507 15:33:36.338 | INFO | BAZWRITER : 58.0%, 4.6, 137.5ms, 0 | 20260507 15:33:36.338 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 15:33:36.338 | INFO | Uploaded 36GiB to GPU in 1.65166s (21.7962GiB/s) | 20260507 15:33:36.338 | INFO | Downloaded 38GiB to GPU in 1.7344s (21.9096GiB/s) | 20260507 15:33:41.477 | INFO | Analyzing chunk frames = [606720,607232) | 20260507 15:33:41.477 | INFO | MemInfo: { | 20260507 15:33:41.477 | INFO | "gpu" : " 17 GB", | 20260507 15:33:41.478 | INFO | "huge" : "", | 20260507 15:33:41.478 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:41.478 | INFO | "malloc" : " 14 GB", | 20260507 15:33:41.478 | INFO | "pinned" : " 29 GB" | 20260507 15:33:41.478 | INFO | } | 20260507 15:33:41.478 | INFO | Perf report for the last 5.13979 seconds: | 20260507 15:33:41.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:41.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 15:33:41.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:33:41.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:41.478 | INFO | Uploaded 36GiB to GPU in 1.59234s (22.6082GiB/s) | 20260507 15:33:41.478 | INFO | Downloaded 34GiB to GPU in 1.53968s (22.0825GiB/s) | 20260507 15:33:46.616 | INFO | Analyzing chunk frames = [607232,607744) | 20260507 15:33:46.616 | INFO | MemInfo: { | 20260507 15:33:46.616 | INFO | "gpu" : " 17 GB", | 20260507 15:33:46.616 | INFO | "huge" : "", | 20260507 15:33:46.617 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:46.617 | INFO | "malloc" : " 14 GB", | 20260507 15:33:46.617 | INFO | "pinned" : " 29 GB" | 20260507 15:33:46.617 | INFO | } | 20260507 15:33:46.617 | INFO | Perf report for the last 5.13886 seconds: | 20260507 15:33:46.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:46.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260507 15:33:46.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | PRE_HQ : 40.9%, 1.0, 21.2ms, 0 | 20260507 15:33:46.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:46.617 | INFO | Uploaded 36GiB to GPU in 1.59671s (22.5463GiB/s) | 20260507 15:33:46.617 | INFO | Downloaded 34GiB to GPU in 1.54409s (22.0194GiB/s) | 20260507 15:33:51.756 | INFO | Analyzing chunk frames = [607744,608256) | 20260507 15:33:51.756 | INFO | MemInfo: { | 20260507 15:33:51.756 | INFO | "gpu" : " 17 GB", | 20260507 15:33:51.756 | INFO | "huge" : "", | 20260507 15:33:51.756 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:51.756 | INFO | "malloc" : " 14 GB", | 20260507 15:33:51.756 | INFO | "pinned" : " 29 GB" | 20260507 15:33:51.756 | INFO | } | 20260507 15:33:51.756 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:33:51.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:51.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 15:33:51.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | PRE_HQ : 40.2%, 1.0, 20.8ms, 0 | 20260507 15:33:51.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:51.756 | INFO | Uploaded 36GiB to GPU in 1.59508s (22.5694GiB/s) | 20260507 15:33:51.756 | INFO | Downloaded 34GiB to GPU in 1.54337s (22.0297GiB/s) | 20260507 15:33:56.895 | INFO | Analyzing chunk frames = [608256,608768) | 20260507 15:33:56.895 | INFO | MemInfo: { | 20260507 15:33:56.895 | INFO | "gpu" : " 17 GB", | 20260507 15:33:56.895 | INFO | "huge" : "", | 20260507 15:33:56.896 | INFO | "huge_shared" : " 29 GB", | 20260507 15:33:56.896 | INFO | "malloc" : " 14 GB", | 20260507 15:33:56.896 | INFO | "pinned" : " 29 GB" | 20260507 15:33:56.896 | INFO | } | 20260507 15:33:56.896 | INFO | Perf report for the last 5.13982 seconds: | 20260507 15:33:56.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:33:56.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:33:56.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:33:56.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:33:56.896 | INFO | Uploaded 36GiB to GPU in 1.59446s (22.5782GiB/s) | 20260507 15:33:56.896 | INFO | Downloaded 34GiB to GPU in 1.54603s (21.9918GiB/s) | 20260507 15:34:02.034 | INFO | Analyzing chunk frames = [608768,609280) | 20260507 15:34:02.034 | INFO | MemInfo: { | 20260507 15:34:02.034 | INFO | "gpu" : " 17 GB", | 20260507 15:34:02.034 | INFO | "huge" : "", | 20260507 15:34:02.035 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:02.035 | INFO | "malloc" : " 14 GB", | 20260507 15:34:02.035 | INFO | "pinned" : " 29 GB" | 20260507 15:34:02.035 | INFO | } | 20260507 15:34:02.035 | INFO | Perf report for the last 5.13886 seconds: | 20260507 15:34:02.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:02.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.035 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:34:02.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.035 | INFO | PRE_HQ : 40.6%, 1.0, 21.2ms, 0 | 20260507 15:34:02.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:02.036 | INFO | Uploaded 36GiB to GPU in 1.59469s (22.5749GiB/s) | 20260507 15:34:02.036 | INFO | Downloaded 34GiB to GPU in 1.54224s (22.0459GiB/s) | 20260507 15:34:07.174 | INFO | Analyzing chunk frames = [609280,609792) | 20260507 15:34:07.174 | INFO | MemInfo: { | 20260507 15:34:07.174 | INFO | "gpu" : " 17 GB", | 20260507 15:34:07.174 | INFO | "huge" : "", | 20260507 15:34:07.174 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:07.174 | INFO | "malloc" : " 14 GB", | 20260507 15:34:07.174 | INFO | "pinned" : " 29 GB" | 20260507 15:34:07.174 | INFO | } | 20260507 15:34:07.174 | INFO | Perf report for the last 5.13956 seconds: | 20260507 15:34:07.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:07.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.174 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:34:07.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.175 | INFO | PRE_HQ : 40.6%, 1.0, 21.1ms, 0 | 20260507 15:34:07.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:07.175 | INFO | Uploaded 36GiB to GPU in 1.59405s (22.584GiB/s) | 20260507 15:34:07.175 | INFO | Downloaded 34GiB to GPU in 1.54287s (22.0368GiB/s) | 20260507 15:34:12.313 | INFO | Analyzing chunk frames = [609792,610304) | 20260507 15:34:12.313 | INFO | MemInfo: { | 20260507 15:34:12.313 | INFO | "gpu" : " 17 GB", | 20260507 15:34:12.313 | INFO | "huge" : "", | 20260507 15:34:12.313 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:12.313 | INFO | "malloc" : " 14 GB", | 20260507 15:34:12.313 | INFO | "pinned" : " 29 GB" | 20260507 15:34:12.313 | INFO | } | 20260507 15:34:12.314 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:34:12.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:12.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 15:34:12.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:34:12.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:12.314 | INFO | Uploaded 36GiB to GPU in 1.59419s (22.582GiB/s) | 20260507 15:34:12.314 | INFO | Downloaded 34GiB to GPU in 1.54374s (22.0244GiB/s) | 20260507 15:34:12.427 | INFO | Starting a round of Baz IO | 20260507 15:34:15.430 | INFO | Populating RTMetrics report | 20260507 15:34:15.430 | INFO | RTMetrics report populated | 20260507 15:34:15.430 | INFO | Opening tmp RTMetrics json file | 20260507 15:34:15.431 | INFO | Serializing RTMetrics to json | 20260507 15:34:15.434 | INFO | Serialization done, swapping tmp file now... | 20260507 15:34:15.435 | INFO | RTMetrics done | 20260507 15:34:15.711 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.283 seconds | 20260507 15:34:15.711 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.88% / 0.04% / 47.08% | 20260507 15:34:15.711 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.65902% / 9.13938e-05% / 7.94729e-06% | 20260507 15:34:15.711 | INFO | Baz IO overall: | 20260507 15:34:15.711 | INFO | BytesWritten : 249981457597 | 20260507 15:34:15.711 | INFO | NumEvents : 120502977066 | 20260507 15:34:17.453 | INFO | Analyzing chunk frames = [610304,610816) | 20260507 15:34:17.453 | INFO | MemInfo: { | 20260507 15:34:17.453 | INFO | "gpu" : " 17 GB", | 20260507 15:34:17.453 | INFO | "huge" : "", | 20260507 15:34:17.453 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:17.453 | INFO | "malloc" : " 14 GB", | 20260507 15:34:17.453 | INFO | "pinned" : " 29 GB" | 20260507 15:34:17.453 | INFO | } | 20260507 15:34:17.453 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:34:17.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:17.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:17.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:17.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:17.453 | INFO | ANALYSIS : 58.2%, 2.8, 85.7ms, 0 | 20260507 15:34:17.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:17.453 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260507 15:34:17.453 | INFO | PRE_HQ : 57.5%, 2.3, 68.3ms, 0 | 20260507 15:34:17.453 | INFO | BAZWRITER : 63.4%, 4.5, 147.6ms, 0 | 20260507 15:34:17.453 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260507 15:34:17.453 | INFO | Uploaded 36GiB to GPU in 1.64374s (21.9013GiB/s) | 20260507 15:34:17.453 | INFO | Downloaded 38GiB to GPU in 1.72331s (22.0506GiB/s) | 20260507 15:34:22.592 | INFO | Analyzing chunk frames = [610816,611328) | 20260507 15:34:22.592 | INFO | MemInfo: { | 20260507 15:34:22.592 | INFO | "gpu" : " 17 GB", | 20260507 15:34:22.592 | INFO | "huge" : "", | 20260507 15:34:22.592 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:22.592 | INFO | "malloc" : " 14 GB", | 20260507 15:34:22.592 | INFO | "pinned" : " 29 GB" | 20260507 15:34:22.592 | INFO | } | 20260507 15:34:22.592 | INFO | Perf report for the last 5.13888 seconds: | 20260507 15:34:22.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:22.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:34:22.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:34:22.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:22.592 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.4239GiB/s) | 20260507 15:34:22.592 | INFO | Downloaded 34GiB to GPU in 1.52435s (22.3046GiB/s) | 20260507 15:34:27.731 | INFO | Analyzing chunk frames = [611328,611840) | 20260507 15:34:27.731 | INFO | MemInfo: { | 20260507 15:34:27.731 | INFO | "gpu" : " 17 GB", | 20260507 15:34:27.731 | INFO | "huge" : "", | 20260507 15:34:27.731 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:27.731 | INFO | "malloc" : " 14 GB", | 20260507 15:34:27.731 | INFO | "pinned" : " 29 GB" | 20260507 15:34:27.731 | INFO | } | 20260507 15:34:27.731 | INFO | Perf report for the last 5.13956 seconds: | 20260507 15:34:27.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:27.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:34:27.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:34:27.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:27.731 | INFO | Uploaded 36GiB to GPU in 1.60927s (22.3703GiB/s) | 20260507 15:34:27.732 | INFO | Downloaded 34GiB to GPU in 1.5296s (22.228GiB/s) | 20260507 15:34:32.871 | INFO | Analyzing chunk frames = [611840,612352) | 20260507 15:34:32.871 | INFO | MemInfo: { | 20260507 15:34:32.871 | INFO | "gpu" : " 17 GB", | 20260507 15:34:32.871 | INFO | "huge" : "", | 20260507 15:34:32.871 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:32.871 | INFO | "malloc" : " 14 GB", | 20260507 15:34:32.874 | INFO | "pinned" : " 29 GB" | 20260507 15:34:32.874 | INFO | } | 20260507 15:34:32.874 | INFO | Perf report for the last 5.14294 seconds: | 20260507 15:34:32.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:32.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.874 | INFO | ANALYSIS : 55.9%, 3.0, 86.8ms, 0 | 20260507 15:34:32.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.874 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260507 15:34:32.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:32.875 | INFO | Uploaded 36GiB to GPU in 1.6102s (22.3574GiB/s) | 20260507 15:34:32.875 | INFO | Downloaded 34GiB to GPU in 1.52938s (22.2313GiB/s) | 20260507 15:34:38.010 | INFO | Analyzing chunk frames = [612352,612864) | 20260507 15:34:38.010 | INFO | MemInfo: { | 20260507 15:34:38.010 | INFO | "gpu" : " 17 GB", | 20260507 15:34:38.010 | INFO | "huge" : "", | 20260507 15:34:38.010 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:38.010 | INFO | "malloc" : " 14 GB", | 20260507 15:34:38.011 | INFO | "pinned" : " 29 GB" | 20260507 15:34:38.011 | INFO | } | 20260507 15:34:38.011 | INFO | Perf report for the last 5.13691 seconds: | 20260507 15:34:38.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:38.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.011 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:34:38.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.011 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:34:38.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:38.012 | INFO | Uploaded 36GiB to GPU in 1.60736s (22.3969GiB/s) | 20260507 15:34:38.012 | INFO | Downloaded 34GiB to GPU in 1.52666s (22.2709GiB/s) | 20260507 15:34:43.149 | INFO | Analyzing chunk frames = [612864,613376) | 20260507 15:34:43.149 | INFO | MemInfo: { | 20260507 15:34:43.149 | INFO | "gpu" : " 17 GB", | 20260507 15:34:43.149 | INFO | "huge" : "", | 20260507 15:34:43.149 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:43.149 | INFO | "malloc" : " 14 GB", | 20260507 15:34:43.149 | INFO | "pinned" : " 29 GB" | 20260507 15:34:43.149 | INFO | } | 20260507 15:34:43.149 | INFO | Perf report for the last 5.13813 seconds: | 20260507 15:34:43.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:43.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.149 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:34:43.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.149 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:34:43.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:43.150 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3922GiB/s) | 20260507 15:34:43.150 | INFO | Downloaded 34GiB to GPU in 1.52695s (22.2666GiB/s) | 20260507 15:34:48.289 | INFO | Analyzing chunk frames = [613376,613888) | 20260507 15:34:48.289 | INFO | MemInfo: { | 20260507 15:34:48.289 | INFO | "gpu" : " 17 GB", | 20260507 15:34:48.289 | INFO | "huge" : "", | 20260507 15:34:48.289 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:48.289 | INFO | "malloc" : " 14 GB", | 20260507 15:34:48.289 | INFO | "pinned" : " 29 GB" | 20260507 15:34:48.289 | INFO | } | 20260507 15:34:48.289 | INFO | Perf report for the last 5.13965 seconds: | 20260507 15:34:48.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:48.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:34:48.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:34:48.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:48.289 | INFO | Uploaded 36GiB to GPU in 1.60844s (22.382GiB/s) | 20260507 15:34:48.289 | INFO | Downloaded 34GiB to GPU in 1.52925s (22.2331GiB/s) | 20260507 15:34:53.428 | INFO | Analyzing chunk frames = [613888,614400) | 20260507 15:34:53.428 | INFO | MemInfo: { | 20260507 15:34:53.428 | INFO | "gpu" : " 17 GB", | 20260507 15:34:53.428 | INFO | "huge" : "", | 20260507 15:34:53.428 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:53.428 | INFO | "malloc" : " 14 GB", | 20260507 15:34:53.428 | INFO | "pinned" : " 29 GB" | 20260507 15:34:53.428 | INFO | } | 20260507 15:34:53.428 | INFO | Perf report for the last 5.139 seconds: | 20260507 15:34:53.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:53.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:34:53.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260507 15:34:53.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:53.428 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:34:53.428 | INFO | Total mem usage: 16.4 GiB | 20260507 15:34:53.428 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:34:53.428 | INFO | Uploaded 36GiB to GPU in 1.61118s (22.3438GiB/s) | 20260507 15:34:53.428 | INFO | Downloaded 34GiB to GPU in 1.53068s (22.2123GiB/s) | 20260507 15:34:53.428 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.5%) | 20260507 15:34:53.428 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:34:53.428 | INFO | High Water Breakdown: | 20260507 15:34:53.428 | INFO | 59.6% : 9.09 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:34:53.428 | INFO | 40.3% : 6.15 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:34:53.428 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:34:53.428 | INFO | ------------------------------------------ | 20260507 15:34:53.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:34:53.428 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:34:53.428 | INFO | Total mem usage: 26.5 GiB | 20260507 15:34:53.428 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:34:53.428 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:34:53.428 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:34:53.428 | INFO | High Water Breakdown: | 20260507 15:34:53.429 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:34:53.429 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:34:53.429 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:34:53.429 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:34:53.429 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:34:53.429 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:34:53.429 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:34:53.429 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:34:53.429 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:34:53.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:34:53.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:34:53.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:34:53.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:34:53.429 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:34:53.429 | INFO | ------------------------------------------ | 20260507 15:34:53.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:34:53.429 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:34:53.429 | INFO | Total mem usage: 27.8 GiB | 20260507 15:34:53.429 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:34:53.429 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:34:53.429 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:34:53.429 | INFO | High Water Breakdown: | 20260507 15:34:53.429 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:34:53.429 | INFO | ------------------------------------------ | 20260507 15:34:53.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:34:53.429 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:34:53.429 | INFO | Total mem usage: 15.7 GiB | 20260507 15:34:53.429 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:34:53.429 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:34:53.429 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:34:53.429 | INFO | High Water Breakdown: | 20260507 15:34:53.429 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:34:53.429 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:34:53.429 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:34:53.429 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:34:53.429 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:34:53.429 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:34:53.429 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:34:53.429 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:34:53.429 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:34:53.429 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:34:53.429 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:34:53.429 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:34:53.429 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:34:53.429 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:34:53.429 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:34:53.429 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:34:53.429 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:34:53.429 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:34:53.429 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:34:53.429 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:34:53.429 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:34:53.429 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:34:53.429 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:34:53.429 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:34:53.429 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:34:53.429 | INFO | ------------------------------------------ | 20260507 15:34:53.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:34:53.429 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:34:53.429 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:34:53.429 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:34:53.429 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / -nan, 7.94e-03 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 11.06%, Baseline -- 3.31e+00 / -nan, 2.41e-02 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.40e-02 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 16.13%, DME -- 3.86e+01 / -nan, 2.98e+00 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.72e+01 / -nan, 4.32e-02 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.15e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 3.46%, Metrics -- 1.03e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 1.29%, PulseSerializing -- 3.85e-01 / -nan, 5.71e-02 / 0.00e+00 | 20260507 15:34:53.429 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:34:53.429 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:34:53.429 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:34:53.429 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 18.72%, Upload -- 1.67e+01 / 0.00e+00, 4.46e-01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 18.17%, Download -- 1.62e+01 / 0.00e+00, 7.76e-01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 33.43%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 0.58%, LockUpload -- 5.21e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:34:53.429 | INFO | PerfReport: 0.00%, LockDownload -- 5.66e-04 / 0.00e+00, 4.28e-04 / 0.00e+00 | 20260507 15:34:53.430 | INFO | PerfReport: 29.08%, LockCompute -- 2.60e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 15:34:53.430 | INFO | PerfReport: 0.02%, Misc -- 1.66e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 15:34:53.525 | INFO | Starting a round of Baz IO | 20260507 15:34:56.460 | INFO | Populating RTMetrics report | 20260507 15:34:56.460 | INFO | RTMetrics report populated | 20260507 15:34:56.460 | INFO | Opening tmp RTMetrics json file | 20260507 15:34:56.564 | INFO | Serializing RTMetrics to json | 20260507 15:34:56.568 | INFO | Serialization done, swapping tmp file now... | 20260507 15:34:56.569 | INFO | RTMetrics done | 20260507 15:34:56.790 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.264 seconds | 20260507 15:34:56.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.84% / 0.04% / 47.12% | 20260507 15:34:56.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.62346% / 8.74201e-05% / 2.78155e-05% | 20260507 15:34:56.790 | INFO | Baz IO overall: | 20260507 15:34:56.790 | INFO | BytesWritten : 251868952257 | 20260507 15:34:56.790 | INFO | NumEvents : 121421431191 | 20260507 15:34:58.567 | INFO | Analyzing chunk frames = [614400,614912) | 20260507 15:34:58.567 | INFO | MemInfo: { | 20260507 15:34:58.567 | INFO | "gpu" : " 17 GB", | 20260507 15:34:58.567 | INFO | "huge" : "", | 20260507 15:34:58.567 | INFO | "huge_shared" : " 29 GB", | 20260507 15:34:58.567 | INFO | "malloc" : " 14 GB", | 20260507 15:34:58.567 | INFO | "pinned" : " 29 GB" | 20260507 15:34:58.567 | INFO | } | 20260507 15:34:58.567 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:34:58.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:34:58.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:58.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:58.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:58.567 | INFO | ANALYSIS : 58.0%, 2.8, 85.1ms, 0 | 20260507 15:34:58.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:34:58.567 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 15:34:58.568 | INFO | PRE_HQ : 56.9%, 2.1, 62.4ms, 0 | 20260507 15:34:58.568 | INFO | BAZWRITER : 63.1%, 4.7, 151.5ms, 0 | 20260507 15:34:58.568 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 15:34:58.568 | INFO | Uploaded 36GiB to GPU in 1.65118s (21.8026GiB/s) | 20260507 15:34:58.568 | INFO | Downloaded 38GiB to GPU in 1.73148s (21.9466GiB/s) | 20260507 15:35:03.707 | INFO | Analyzing chunk frames = [614912,615424) | 20260507 15:35:03.707 | INFO | MemInfo: { | 20260507 15:35:03.707 | INFO | "gpu" : " 17 GB", | 20260507 15:35:03.707 | INFO | "huge" : "", | 20260507 15:35:03.707 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:03.707 | INFO | "malloc" : " 14 GB", | 20260507 15:35:03.707 | INFO | "pinned" : " 29 GB" | 20260507 15:35:03.707 | INFO | } | 20260507 15:35:03.707 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:35:03.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:03.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:35:03.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:35:03.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:03.707 | INFO | Uploaded 36GiB to GPU in 1.58551s (22.7057GiB/s) | 20260507 15:35:03.707 | INFO | Downloaded 34GiB to GPU in 1.53456s (22.1562GiB/s) | 20260507 15:35:08.846 | INFO | Analyzing chunk frames = [615424,615936) | 20260507 15:35:08.846 | INFO | MemInfo: { | 20260507 15:35:08.846 | INFO | "gpu" : " 17 GB", | 20260507 15:35:08.846 | INFO | "huge" : "", | 20260507 15:35:08.846 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:08.846 | INFO | "malloc" : " 14 GB", | 20260507 15:35:08.846 | INFO | "pinned" : " 29 GB" | 20260507 15:35:08.846 | INFO | } | 20260507 15:35:08.846 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:35:08.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:08.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:35:08.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:35:08.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:08.846 | INFO | Uploaded 36GiB to GPU in 1.59046s (22.6349GiB/s) | 20260507 15:35:08.846 | INFO | Downloaded 34GiB to GPU in 1.53945s (22.0858GiB/s) | 20260507 15:35:13.989 | INFO | Analyzing chunk frames = [615936,616448) | 20260507 15:35:13.989 | INFO | MemInfo: { | 20260507 15:35:13.989 | INFO | "gpu" : " 17 GB", | 20260507 15:35:13.989 | INFO | "huge" : "", | 20260507 15:35:13.989 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:13.989 | INFO | "malloc" : " 14 GB", | 20260507 15:35:13.989 | INFO | "pinned" : " 29 GB" | 20260507 15:35:13.989 | INFO | } | 20260507 15:35:13.989 | INFO | Perf report for the last 5.14321 seconds: | 20260507 15:35:13.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:13.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.989 | INFO | ANALYSIS : 55.9%, 3.0, 86.8ms, 0 | 20260507 15:35:13.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.989 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:35:13.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:13.990 | INFO | Uploaded 36GiB to GPU in 1.58852s (22.6626GiB/s) | 20260507 15:35:13.990 | INFO | Downloaded 34GiB to GPU in 1.53788s (22.1083GiB/s) | 20260507 15:35:19.125 | INFO | Analyzing chunk frames = [616448,616960) | 20260507 15:35:19.125 | INFO | MemInfo: { | 20260507 15:35:19.125 | INFO | "gpu" : " 17 GB", | 20260507 15:35:19.125 | INFO | "huge" : "", | 20260507 15:35:19.125 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:19.125 | INFO | "malloc" : " 14 GB", | 20260507 15:35:19.125 | INFO | "pinned" : " 29 GB" | 20260507 15:35:19.125 | INFO | } | 20260507 15:35:19.125 | INFO | Perf report for the last 5.13557 seconds: | 20260507 15:35:19.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:19.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.125 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:35:19.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.125 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:35:19.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:19.126 | INFO | Uploaded 36GiB to GPU in 1.58797s (22.6705GiB/s) | 20260507 15:35:19.126 | INFO | Downloaded 34GiB to GPU in 1.53664s (22.1262GiB/s) | 20260507 15:35:24.264 | INFO | Analyzing chunk frames = [616960,617472) | 20260507 15:35:24.264 | INFO | MemInfo: { | 20260507 15:35:24.264 | INFO | "gpu" : " 17 GB", | 20260507 15:35:24.264 | INFO | "huge" : "", | 20260507 15:35:24.264 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:24.264 | INFO | "malloc" : " 14 GB", | 20260507 15:35:24.264 | INFO | "pinned" : " 29 GB" | 20260507 15:35:24.264 | INFO | } | 20260507 15:35:24.264 | INFO | Perf report for the last 5.13915 seconds: | 20260507 15:35:24.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:24.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:35:24.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:35:24.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:24.264 | INFO | Uploaded 36GiB to GPU in 1.58871s (22.6599GiB/s) | 20260507 15:35:24.264 | INFO | Downloaded 34GiB to GPU in 1.53685s (22.1232GiB/s) | 20260507 15:35:29.403 | INFO | Analyzing chunk frames = [617472,617984) | 20260507 15:35:29.403 | INFO | MemInfo: { | 20260507 15:35:29.403 | INFO | "gpu" : " 17 GB", | 20260507 15:35:29.403 | INFO | "huge" : "", | 20260507 15:35:29.407 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:29.407 | INFO | "malloc" : " 14 GB", | 20260507 15:35:29.407 | INFO | "pinned" : " 29 GB" | 20260507 15:35:29.407 | INFO | } | 20260507 15:35:29.408 | INFO | Perf report for the last 5.14343 seconds: | 20260507 15:35:29.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:29.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:35:29.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:35:29.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:29.408 | INFO | Uploaded 36GiB to GPU in 1.58872s (22.6598GiB/s) | 20260507 15:35:29.408 | INFO | Downloaded 34GiB to GPU in 1.53716s (22.1187GiB/s) | 20260507 15:35:34.543 | INFO | Analyzing chunk frames = [617984,618496) | 20260507 15:35:34.543 | INFO | MemInfo: { | 20260507 15:35:34.543 | INFO | "gpu" : " 17 GB", | 20260507 15:35:34.543 | INFO | "huge" : "", | 20260507 15:35:34.543 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:34.543 | INFO | "malloc" : " 14 GB", | 20260507 15:35:34.543 | INFO | "pinned" : " 29 GB" | 20260507 15:35:34.543 | INFO | } | 20260507 15:35:34.543 | INFO | Perf report for the last 5.13573 seconds: | 20260507 15:35:34.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:34.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.543 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 15:35:34.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.544 | INFO | PRE_HQ : 39.5%, 1.0, 20.5ms, 0 | 20260507 15:35:34.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:34.544 | INFO | Uploaded 36GiB to GPU in 1.59489s (22.5721GiB/s) | 20260507 15:35:34.544 | INFO | Downloaded 34GiB to GPU in 1.5427s (22.0393GiB/s) | 20260507 15:35:34.656 | INFO | Starting a round of Baz IO | 20260507 15:35:37.567 | INFO | Populating RTMetrics report | 20260507 15:35:37.567 | INFO | RTMetrics report populated | 20260507 15:35:37.567 | INFO | Opening tmp RTMetrics json file | 20260507 15:35:37.569 | INFO | Serializing RTMetrics to json | 20260507 15:35:37.603 | INFO | Serialization done, swapping tmp file now... | 20260507 15:35:37.628 | INFO | RTMetrics done | 20260507 15:35:37.952 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.296 seconds | 20260507 15:35:37.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.80% / 0.04% / 47.16% | 20260507 15:35:37.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.61316% / 7.15256e-05% / 3.17891e-05% | 20260507 15:35:37.952 | INFO | Baz IO overall: | 20260507 15:35:37.952 | INFO | BytesWritten : 253758518473 | 20260507 15:35:37.952 | INFO | NumEvents : 122341404118 | 20260507 15:35:39.682 | INFO | Analyzing chunk frames = [618496,619008) | 20260507 15:35:39.682 | INFO | MemInfo: { | 20260507 15:35:39.682 | INFO | "gpu" : " 17 GB", | 20260507 15:35:39.682 | INFO | "huge" : "", | 20260507 15:35:39.682 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:39.682 | INFO | "malloc" : " 14 GB", | 20260507 15:35:39.682 | INFO | "pinned" : " 29 GB" | 20260507 15:35:39.682 | INFO | } | 20260507 15:35:39.682 | INFO | Perf report for the last 5.13906 seconds: | 20260507 15:35:39.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:39.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:39.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:39.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:39.682 | INFO | ANALYSIS : 57.9%, 2.8, 84.9ms, 0 | 20260507 15:35:39.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:39.682 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 15:35:39.682 | INFO | PRE_HQ : 57.2%, 2.1, 63.6ms, 0 | 20260507 15:35:39.683 | INFO | BAZWRITER : 63.7%, 4.8, 156.7ms, 0 | 20260507 15:35:39.683 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 15:35:39.683 | INFO | Uploaded 36GiB to GPU in 1.66633s (21.6044GiB/s) | 20260507 15:35:39.683 | INFO | Downloaded 38GiB to GPU in 1.73798s (21.8644GiB/s) | 20260507 15:35:44.825 | INFO | Analyzing chunk frames = [619008,619520) | 20260507 15:35:44.825 | INFO | MemInfo: { | 20260507 15:35:44.825 | INFO | "gpu" : " 17 GB", | 20260507 15:35:44.825 | INFO | "huge" : "", | 20260507 15:35:44.825 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:44.825 | INFO | "malloc" : " 14 GB", | 20260507 15:35:44.825 | INFO | "pinned" : " 29 GB" | 20260507 15:35:44.825 | INFO | } | 20260507 15:35:44.826 | INFO | Perf report for the last 5.14319 seconds: | 20260507 15:35:44.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:44.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 15:35:44.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | PRE_HQ : 39.3%, 1.0, 21.1ms, 0 | 20260507 15:35:44.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:44.826 | INFO | Uploaded 36GiB to GPU in 1.61198s (22.3328GiB/s) | 20260507 15:35:44.826 | INFO | Downloaded 34GiB to GPU in 1.52933s (22.2319GiB/s) | 20260507 15:35:49.961 | INFO | Analyzing chunk frames = [619520,620032) | 20260507 15:35:49.961 | INFO | MemInfo: { | 20260507 15:35:49.961 | INFO | "gpu" : " 17 GB", | 20260507 15:35:49.961 | INFO | "huge" : "", | 20260507 15:35:49.964 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:49.964 | INFO | "malloc" : " 14 GB", | 20260507 15:35:49.964 | INFO | "pinned" : " 29 GB" | 20260507 15:35:49.964 | INFO | } | 20260507 15:35:49.964 | INFO | Perf report for the last 5.13882 seconds: | 20260507 15:35:49.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:49.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.964 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:35:49.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.965 | INFO | PRE_HQ : 40.0%, 1.0, 20.8ms, 0 | 20260507 15:35:49.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:49.965 | INFO | Uploaded 36GiB to GPU in 1.61624s (22.2739GiB/s) | 20260507 15:35:49.965 | INFO | Downloaded 34GiB to GPU in 1.53539s (22.1442GiB/s) | 20260507 15:35:55.101 | INFO | Analyzing chunk frames = [620032,620544) | 20260507 15:35:55.101 | INFO | MemInfo: { | 20260507 15:35:55.101 | INFO | "gpu" : " 17 GB", | 20260507 15:35:55.101 | INFO | "huge" : "", | 20260507 15:35:55.101 | INFO | "huge_shared" : " 29 GB", | 20260507 15:35:55.101 | INFO | "malloc" : " 14 GB", | 20260507 15:35:55.101 | INFO | "pinned" : " 29 GB" | 20260507 15:35:55.101 | INFO | } | 20260507 15:35:55.101 | INFO | Perf report for the last 5.13699 seconds: | 20260507 15:35:55.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:35:55.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.102 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 15:35:55.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.102 | INFO | PRE_HQ : 40.3%, 1.0, 20.9ms, 0 | 20260507 15:35:55.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:35:55.102 | INFO | Uploaded 36GiB to GPU in 1.61549s (22.2842GiB/s) | 20260507 15:35:55.102 | INFO | Downloaded 34GiB to GPU in 1.53512s (22.1481GiB/s) | 20260507 15:36:00.239 | INFO | Analyzing chunk frames = [620544,621056) | 20260507 15:36:00.239 | INFO | MemInfo: { | 20260507 15:36:00.239 | INFO | "gpu" : " 17 GB", | 20260507 15:36:00.239 | INFO | "huge" : "", | 20260507 15:36:00.240 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:00.240 | INFO | "malloc" : " 14 GB", | 20260507 15:36:00.240 | INFO | "pinned" : " 29 GB" | 20260507 15:36:00.240 | INFO | } | 20260507 15:36:00.240 | INFO | Perf report for the last 5.13822 seconds: | 20260507 15:36:00.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:00.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:36:00.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 15:36:00.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:00.240 | INFO | Uploaded 36GiB to GPU in 1.61242s (22.3267GiB/s) | 20260507 15:36:00.240 | INFO | Downloaded 34GiB to GPU in 1.53122s (22.2045GiB/s) | 20260507 15:36:05.379 | INFO | Analyzing chunk frames = [621056,621568) | 20260507 15:36:05.379 | INFO | MemInfo: { | 20260507 15:36:05.379 | INFO | "gpu" : " 17 GB", | 20260507 15:36:05.379 | INFO | "huge" : "", | 20260507 15:36:05.379 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:05.379 | INFO | "malloc" : " 14 GB", | 20260507 15:36:05.379 | INFO | "pinned" : " 29 GB" | 20260507 15:36:05.379 | INFO | } | 20260507 15:36:05.379 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:36:05.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:05.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:36:05.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:36:05.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:05.379 | INFO | Uploaded 36GiB to GPU in 1.60797s (22.3885GiB/s) | 20260507 15:36:05.379 | INFO | Downloaded 34GiB to GPU in 1.52771s (22.2556GiB/s) | 20260507 15:36:10.518 | INFO | Analyzing chunk frames = [621568,622080) | 20260507 15:36:10.518 | INFO | MemInfo: { | 20260507 15:36:10.518 | INFO | "gpu" : " 17 GB", | 20260507 15:36:10.518 | INFO | "huge" : "", | 20260507 15:36:10.518 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:10.518 | INFO | "malloc" : " 14 GB", | 20260507 15:36:10.518 | INFO | "pinned" : " 29 GB" | 20260507 15:36:10.518 | INFO | } | 20260507 15:36:10.518 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:36:10.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:10.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:36:10.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:36:10.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:10.518 | INFO | Uploaded 36GiB to GPU in 1.60821s (22.3851GiB/s) | 20260507 15:36:10.519 | INFO | Downloaded 34GiB to GPU in 1.52858s (22.2428GiB/s) | 20260507 15:36:15.657 | INFO | Analyzing chunk frames = [622080,622592) | 20260507 15:36:15.657 | INFO | MemInfo: { | 20260507 15:36:15.657 | INFO | "gpu" : " 17 GB", | 20260507 15:36:15.657 | INFO | "huge" : "", | 20260507 15:36:15.657 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:15.657 | INFO | "malloc" : " 14 GB", | 20260507 15:36:15.657 | INFO | "pinned" : " 29 GB" | 20260507 15:36:15.658 | INFO | } | 20260507 15:36:15.658 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:36:15.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:15.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:36:15.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:36:15.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:15.658 | INFO | Uploaded 36GiB to GPU in 1.60851s (22.3809GiB/s) | 20260507 15:36:15.658 | INFO | Downloaded 34GiB to GPU in 1.52833s (22.2466GiB/s) | 20260507 15:36:15.658 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:36:15.658 | INFO | Total mem usage: 16.4 GiB | 20260507 15:36:15.658 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:36:15.658 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.24%) | 20260507 15:36:15.658 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:36:15.658 | INFO | High Water Breakdown: | 20260507 15:36:15.658 | INFO | 59.9% : 9.34 GiB ( 94.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:36:15.658 | INFO | 40% : 6.23 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:36:15.658 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:36:15.658 | INFO | ------------------------------------------ | 20260507 15:36:15.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:36:15.658 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:36:15.658 | INFO | Total mem usage: 26.5 GiB | 20260507 15:36:15.658 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:36:15.658 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:36:15.658 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:36:15.658 | INFO | High Water Breakdown: | 20260507 15:36:15.658 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:36:15.658 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:36:15.658 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:36:15.658 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:36:15.658 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:36:15.658 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:36:15.658 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:36:15.658 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:36:15.658 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:36:15.658 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:36:15.658 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:36:15.658 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:36:15.658 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:36:15.658 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:36:15.658 | INFO | ------------------------------------------ | 20260507 15:36:15.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:36:15.658 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:36:15.658 | INFO | Total mem usage: 27.8 GiB | 20260507 15:36:15.658 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:36:15.658 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:36:15.658 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:36:15.658 | INFO | High Water Breakdown: | 20260507 15:36:15.658 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:36:15.658 | INFO | ------------------------------------------ | 20260507 15:36:15.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:36:15.658 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:36:15.658 | INFO | Total mem usage: 15.7 GiB | 20260507 15:36:15.658 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:36:15.658 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:36:15.658 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:36:15.658 | INFO | High Water Breakdown: | 20260507 15:36:15.658 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:36:15.658 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:36:15.658 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:36:15.658 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:36:15.658 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:36:15.658 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:36:15.658 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:36:15.658 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:36:15.658 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:36:15.658 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:36:15.658 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:36:15.658 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:36:15.658 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:36:15.658 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:36:15.658 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:36:15.658 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:36:15.658 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:36:15.658 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:36:15.658 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:36:15.658 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:36:15.658 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:36:15.658 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:36:15.658 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:36:15.658 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:36:15.658 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:36:15.658 | INFO | ------------------------------------------ | 20260507 15:36:15.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:36:15.658 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:36:15.658 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:36:15.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:36:15.658 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 0.30%, AggregateStats -- 8.92e-02 / -nan, 7.62e-03 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 11.06%, Baseline -- 3.30e+00 / -nan, 3.92e-02 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 3.08%, Histogram -- 9.20e-01 / -nan, 5.39e-02 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.72e+01 / -nan, 4.38e-02 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 3.45%, Metrics -- 1.03e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.82e-01 / -nan, 5.69e-02 / 0.00e+00 | 20260507 15:36:15.658 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:36:15.658 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:36:15.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:36:15.658 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 18.77%, Upload -- 1.67e+01 / 0.00e+00, 4.97e-01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 18.21%, Download -- 1.62e+01 / 0.00e+00, 8.08e-01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 33.48%, Compute -- 2.99e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 0.00%, LockDownload -- 5.17e-04 / 0.00e+00, 3.39e-04 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 28.96%, LockCompute -- 2.58e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 15:36:15.658 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260507 15:36:15.757 | INFO | Starting a round of Baz IO | 20260507 15:36:18.674 | INFO | Populating RTMetrics report | 20260507 15:36:18.675 | INFO | RTMetrics report populated | 20260507 15:36:18.675 | INFO | Opening tmp RTMetrics json file | 20260507 15:36:18.694 | INFO | Serializing RTMetrics to json | 20260507 15:36:18.699 | INFO | Serialization done, swapping tmp file now... | 20260507 15:36:18.704 | INFO | RTMetrics done | 20260507 15:36:18.951 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.193 seconds | 20260507 15:36:18.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.76% / 0.04% / 47.20% | 20260507 15:36:18.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.58993% / 0.000111262% / 1.58946e-05% | 20260507 15:36:18.951 | INFO | Baz IO overall: | 20260507 15:36:18.951 | INFO | BytesWritten : 255646209865 | 20260507 15:36:18.951 | INFO | NumEvents : 123259825188 | 20260507 15:36:20.797 | INFO | Analyzing chunk frames = [622592,623104) | 20260507 15:36:20.797 | INFO | MemInfo: { | 20260507 15:36:20.797 | INFO | "gpu" : " 17 GB", | 20260507 15:36:20.797 | INFO | "huge" : "", | 20260507 15:36:20.797 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:20.797 | INFO | "malloc" : " 14 GB", | 20260507 15:36:20.797 | INFO | "pinned" : " 29 GB" | 20260507 15:36:20.797 | INFO | } | 20260507 15:36:20.797 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:36:20.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:20.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:20.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:20.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:20.797 | INFO | ANALYSIS : 57.8%, 2.8, 84.3ms, 0 | 20260507 15:36:20.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:20.797 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 15:36:20.797 | INFO | PRE_HQ : 56.6%, 2.1, 63.0ms, 0 | 20260507 15:36:20.797 | INFO | BAZWRITER : 61.8%, 4.8, 151.4ms, 0 | 20260507 15:36:20.797 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:36:20.797 | INFO | Uploaded 36GiB to GPU in 1.65884s (21.7019GiB/s) | 20260507 15:36:20.797 | INFO | Downloaded 38GiB to GPU in 1.73372s (21.9182GiB/s) | 20260507 15:36:25.936 | INFO | Analyzing chunk frames = [623104,623616) | 20260507 15:36:25.936 | INFO | MemInfo: { | 20260507 15:36:25.936 | INFO | "gpu" : " 17 GB", | 20260507 15:36:25.936 | INFO | "huge" : "", | 20260507 15:36:25.936 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:25.936 | INFO | "malloc" : " 14 GB", | 20260507 15:36:25.937 | INFO | "pinned" : " 29 GB" | 20260507 15:36:25.937 | INFO | } | 20260507 15:36:25.937 | INFO | Perf report for the last 5.14035 seconds: | 20260507 15:36:25.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:25.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:36:25.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:36:25.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:25.937 | INFO | Uploaded 36GiB to GPU in 1.60495s (22.4306GiB/s) | 20260507 15:36:25.937 | INFO | Downloaded 34GiB to GPU in 1.52487s (22.297GiB/s) | 20260507 15:36:31.077 | INFO | Analyzing chunk frames = [623616,624128) | 20260507 15:36:31.077 | INFO | MemInfo: { | 20260507 15:36:31.077 | INFO | "gpu" : " 17 GB", | 20260507 15:36:31.077 | INFO | "huge" : "", | 20260507 15:36:31.077 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:31.077 | INFO | "malloc" : " 14 GB", | 20260507 15:36:31.077 | INFO | "pinned" : " 29 GB" | 20260507 15:36:31.077 | INFO | } | 20260507 15:36:31.077 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:36:31.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:31.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:36:31.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:36:31.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:31.077 | INFO | Uploaded 36GiB to GPU in 1.60981s (22.3629GiB/s) | 20260507 15:36:31.077 | INFO | Downloaded 34GiB to GPU in 1.53015s (22.2201GiB/s) | 20260507 15:36:36.215 | INFO | Analyzing chunk frames = [624128,624640) | 20260507 15:36:36.215 | INFO | MemInfo: { | 20260507 15:36:36.215 | INFO | "gpu" : " 17 GB", | 20260507 15:36:36.215 | INFO | "huge" : "", | 20260507 15:36:36.215 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:36.215 | INFO | "malloc" : " 14 GB", | 20260507 15:36:36.215 | INFO | "pinned" : " 29 GB" | 20260507 15:36:36.215 | INFO | } | 20260507 15:36:36.215 | INFO | Perf report for the last 5.13864 seconds: | 20260507 15:36:36.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:36.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.215 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:36:36.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.215 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:36:36.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:36.216 | INFO | Uploaded 36GiB to GPU in 1.60881s (22.3768GiB/s) | 20260507 15:36:36.216 | INFO | Downloaded 34GiB to GPU in 1.52928s (22.2328GiB/s) | 20260507 15:36:41.354 | INFO | Analyzing chunk frames = [624640,625152) | 20260507 15:36:41.354 | INFO | MemInfo: { | 20260507 15:36:41.354 | INFO | "gpu" : " 17 GB", | 20260507 15:36:41.354 | INFO | "huge" : "", | 20260507 15:36:41.354 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:41.354 | INFO | "malloc" : " 14 GB", | 20260507 15:36:41.354 | INFO | "pinned" : " 29 GB" | 20260507 15:36:41.354 | INFO | } | 20260507 15:36:41.354 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:36:41.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:41.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.354 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:36:41.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.355 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:36:41.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:41.355 | INFO | Uploaded 36GiB to GPU in 1.60806s (22.3872GiB/s) | 20260507 15:36:41.355 | INFO | Downloaded 34GiB to GPU in 1.52774s (22.2551GiB/s) | 20260507 15:36:46.494 | INFO | Analyzing chunk frames = [625152,625664) | 20260507 15:36:46.494 | INFO | MemInfo: { | 20260507 15:36:46.494 | INFO | "gpu" : " 17 GB", | 20260507 15:36:46.494 | INFO | "huge" : "", | 20260507 15:36:46.494 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:46.494 | INFO | "malloc" : " 14 GB", | 20260507 15:36:46.494 | INFO | "pinned" : " 29 GB" | 20260507 15:36:46.494 | INFO | } | 20260507 15:36:46.494 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:36:46.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:46.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:36:46.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:36:46.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:46.494 | INFO | Uploaded 36GiB to GPU in 1.60805s (22.3874GiB/s) | 20260507 15:36:46.494 | INFO | Downloaded 34GiB to GPU in 1.52772s (22.2554GiB/s) | 20260507 15:36:51.633 | INFO | Analyzing chunk frames = [625664,626176) | 20260507 15:36:51.633 | INFO | MemInfo: { | 20260507 15:36:51.633 | INFO | "gpu" : " 17 GB", | 20260507 15:36:51.633 | INFO | "huge" : "", | 20260507 15:36:51.633 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:51.633 | INFO | "malloc" : " 14 GB", | 20260507 15:36:51.633 | INFO | "pinned" : " 29 GB" | 20260507 15:36:51.634 | INFO | } | 20260507 15:36:51.634 | INFO | Perf report for the last 5.14042 seconds: | 20260507 15:36:51.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:51.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.634 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:36:51.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.634 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:36:51.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:51.635 | INFO | Uploaded 36GiB to GPU in 1.60797s (22.3885GiB/s) | 20260507 15:36:51.635 | INFO | Downloaded 34GiB to GPU in 1.529s (22.2368GiB/s) | 20260507 15:36:56.772 | INFO | Analyzing chunk frames = [626176,626688) | 20260507 15:36:56.772 | INFO | MemInfo: { | 20260507 15:36:56.772 | INFO | "gpu" : " 17 GB", | 20260507 15:36:56.772 | INFO | "huge" : "", | 20260507 15:36:56.772 | INFO | "huge_shared" : " 29 GB", | 20260507 15:36:56.772 | INFO | "malloc" : " 14 GB", | 20260507 15:36:56.772 | INFO | "pinned" : " 29 GB" | 20260507 15:36:56.772 | INFO | } | 20260507 15:36:56.772 | INFO | Perf report for the last 5.1382 seconds: | 20260507 15:36:56.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:36:56.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.773 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:36:56.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.773 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 15:36:56.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:36:56.773 | INFO | Uploaded 36GiB to GPU in 1.60915s (22.3721GiB/s) | 20260507 15:36:56.773 | INFO | Downloaded 34GiB to GPU in 1.52964s (22.2274GiB/s) | 20260507 15:36:56.869 | INFO | Starting a round of Baz IO | 20260507 15:36:59.793 | INFO | Populating RTMetrics report | 20260507 15:36:59.793 | INFO | RTMetrics report populated | 20260507 15:36:59.793 | INFO | Opening tmp RTMetrics json file | 20260507 15:36:59.795 | INFO | Serializing RTMetrics to json | 20260507 15:36:59.796 | INFO | Serialization done, swapping tmp file now... | 20260507 15:36:59.797 | INFO | RTMetrics done | 20260507 15:37:00.061 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.191 seconds | 20260507 15:37:00.061 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.72% / 0.04% / 47.24% | 20260507 15:37:00.061 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.57591% / 8.74201e-05% / 3.57628e-05% | 20260507 15:37:00.061 | INFO | Baz IO overall: | 20260507 15:37:00.061 | INFO | BytesWritten : 257535079432 | 20260507 15:37:00.061 | INFO | NumEvents : 124178920925 | 20260507 15:37:01.912 | INFO | Analyzing chunk frames = [626688,627200) | 20260507 15:37:01.912 | INFO | MemInfo: { | 20260507 15:37:01.912 | INFO | "gpu" : " 17 GB", | 20260507 15:37:01.912 | INFO | "huge" : "", | 20260507 15:37:01.912 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:01.912 | INFO | "malloc" : " 14 GB", | 20260507 15:37:01.912 | INFO | "pinned" : " 29 GB" | 20260507 15:37:01.912 | INFO | } | 20260507 15:37:01.912 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:37:01.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:01.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:01.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:01.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:01.912 | INFO | ANALYSIS : 57.9%, 2.8, 85.6ms, 0 | 20260507 15:37:01.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:01.912 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 15:37:01.912 | INFO | PRE_HQ : 56.9%, 2.2, 65.8ms, 0 | 20260507 15:37:01.912 | INFO | BAZWRITER : 61.7%, 4.5, 142.3ms, 0 | 20260507 15:37:01.912 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:37:01.912 | INFO | Uploaded 36GiB to GPU in 1.64077s (21.9409GiB/s) | 20260507 15:37:01.912 | INFO | Downloaded 38GiB to GPU in 1.71914s (22.1041GiB/s) | 20260507 15:37:07.051 | INFO | Analyzing chunk frames = [627200,627712) | 20260507 15:37:07.051 | INFO | MemInfo: { | 20260507 15:37:07.051 | INFO | "gpu" : " 17 GB", | 20260507 15:37:07.051 | INFO | "huge" : "", | 20260507 15:37:07.051 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:07.051 | INFO | "malloc" : " 14 GB", | 20260507 15:37:07.051 | INFO | "pinned" : " 29 GB" | 20260507 15:37:07.051 | INFO | } | 20260507 15:37:07.051 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:37:07.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:07.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.051 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:37:07.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.051 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 15:37:07.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:07.052 | INFO | Uploaded 36GiB to GPU in 1.61147s (22.3398GiB/s) | 20260507 15:37:07.052 | INFO | Downloaded 34GiB to GPU in 1.53038s (22.2167GiB/s) | 20260507 15:37:12.190 | INFO | Analyzing chunk frames = [627712,628224) | 20260507 15:37:12.190 | INFO | MemInfo: { | 20260507 15:37:12.190 | INFO | "gpu" : " 17 GB", | 20260507 15:37:12.190 | INFO | "huge" : "", | 20260507 15:37:12.190 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:12.190 | INFO | "malloc" : " 14 GB", | 20260507 15:37:12.191 | INFO | "pinned" : " 29 GB" | 20260507 15:37:12.191 | INFO | } | 20260507 15:37:12.191 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:37:12.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:12.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 15:37:12.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:37:12.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:12.191 | INFO | Uploaded 36GiB to GPU in 1.61573s (22.2809GiB/s) | 20260507 15:37:12.191 | INFO | Downloaded 34GiB to GPU in 1.53478s (22.153GiB/s) | 20260507 15:37:17.330 | INFO | Analyzing chunk frames = [628224,628736) | 20260507 15:37:17.330 | INFO | MemInfo: { | 20260507 15:37:17.330 | INFO | "gpu" : " 17 GB", | 20260507 15:37:17.330 | INFO | "huge" : "", | 20260507 15:37:17.330 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:17.330 | INFO | "malloc" : " 14 GB", | 20260507 15:37:17.330 | INFO | "pinned" : " 29 GB" | 20260507 15:37:17.330 | INFO | } | 20260507 15:37:17.330 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:37:17.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:17.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.330 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:37:17.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.330 | INFO | PRE_HQ : 39.8%, 1.0, 20.6ms, 0 | 20260507 15:37:17.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:17.331 | INFO | Uploaded 36GiB to GPU in 1.61579s (22.2802GiB/s) | 20260507 15:37:17.331 | INFO | Downloaded 34GiB to GPU in 1.53609s (22.1341GiB/s) | 20260507 15:37:22.469 | INFO | Analyzing chunk frames = [628736,629248) | 20260507 15:37:22.469 | INFO | MemInfo: { | 20260507 15:37:22.469 | INFO | "gpu" : " 17 GB", | 20260507 15:37:22.469 | INFO | "huge" : "", | 20260507 15:37:22.469 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:22.469 | INFO | "malloc" : " 14 GB", | 20260507 15:37:22.469 | INFO | "pinned" : " 29 GB" | 20260507 15:37:22.469 | INFO | } | 20260507 15:37:22.469 | INFO | Perf report for the last 5.13895 seconds: | 20260507 15:37:22.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:22.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:37:22.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:37:22.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:22.469 | INFO | Uploaded 36GiB to GPU in 1.61308s (22.3176GiB/s) | 20260507 15:37:22.469 | INFO | Downloaded 34GiB to GPU in 1.53364s (22.1694GiB/s) | 20260507 15:37:27.608 | INFO | Analyzing chunk frames = [629248,629760) | 20260507 15:37:27.608 | INFO | MemInfo: { | 20260507 15:37:27.608 | INFO | "gpu" : " 17 GB", | 20260507 15:37:27.608 | INFO | "huge" : "", | 20260507 15:37:27.608 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:27.608 | INFO | "malloc" : " 14 GB", | 20260507 15:37:27.608 | INFO | "pinned" : " 29 GB" | 20260507 15:37:27.608 | INFO | } | 20260507 15:37:27.608 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:37:27.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:27.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.608 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:37:27.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.608 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:37:27.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:27.609 | INFO | Uploaded 36GiB to GPU in 1.60812s (22.3864GiB/s) | 20260507 15:37:27.609 | INFO | Downloaded 34GiB to GPU in 1.52708s (22.2648GiB/s) | 20260507 15:37:32.748 | INFO | Analyzing chunk frames = [629760,630272) | 20260507 15:37:32.748 | INFO | MemInfo: { | 20260507 15:37:32.748 | INFO | "gpu" : " 17 GB", | 20260507 15:37:32.748 | INFO | "huge" : "", | 20260507 15:37:32.748 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:32.748 | INFO | "malloc" : " 14 GB", | 20260507 15:37:32.748 | INFO | "pinned" : " 29 GB" | 20260507 15:37:32.748 | INFO | } | 20260507 15:37:32.748 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:37:32.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:32.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.748 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:37:32.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.748 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:37:32.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:32.749 | INFO | Uploaded 36GiB to GPU in 1.60754s (22.3945GiB/s) | 20260507 15:37:32.749 | INFO | Downloaded 34GiB to GPU in 1.52871s (22.2409GiB/s) | 20260507 15:37:37.887 | INFO | Analyzing chunk frames = [630272,630784) | 20260507 15:37:37.887 | INFO | MemInfo: { | 20260507 15:37:37.887 | INFO | "gpu" : " 17 GB", | 20260507 15:37:37.887 | INFO | "huge" : "", | 20260507 15:37:37.887 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:37.887 | INFO | "malloc" : " 14 GB", | 20260507 15:37:37.887 | INFO | "pinned" : " 29 GB" | 20260507 15:37:37.887 | INFO | } | 20260507 15:37:37.887 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:37:37.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:37.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:37:37.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 15:37:37.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:37.887 | INFO | Uploaded 36GiB to GPU in 1.60861s (22.3796GiB/s) | 20260507 15:37:37.887 | INFO | Downloaded 34GiB to GPU in 1.52887s (22.2387GiB/s) | 20260507 15:37:37.888 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:37:37.888 | INFO | Total mem usage: 16.4 GiB | 20260507 15:37:37.888 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:37:37.888 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.91%) | 20260507 15:37:37.888 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:37:37.888 | INFO | High Water Breakdown: | 20260507 15:37:37.888 | INFO | 59.8% : 9.27 GiB ( 93.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:37:37.888 | INFO | 40% : 6.2 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:37:37.888 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:37:37.888 | INFO | ------------------------------------------ | 20260507 15:37:37.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:37:37.888 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:37:37.888 | INFO | Total mem usage: 26.5 GiB | 20260507 15:37:37.888 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:37:37.888 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:37:37.888 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:37:37.888 | INFO | High Water Breakdown: | 20260507 15:37:37.888 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:37:37.888 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:37:37.888 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:37:37.888 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:37:37.888 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:37:37.888 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:37:37.888 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:37:37.888 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:37:37.888 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:37:37.888 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:37:37.888 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:37:37.888 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:37:37.888 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:37:37.888 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:37:37.888 | INFO | ------------------------------------------ | 20260507 15:37:37.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:37:37.888 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:37:37.888 | INFO | Total mem usage: 27.8 GiB | 20260507 15:37:37.888 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:37:37.888 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:37:37.888 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:37:37.888 | INFO | High Water Breakdown: | 20260507 15:37:37.888 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:37:37.888 | INFO | ------------------------------------------ | 20260507 15:37:37.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:37:37.888 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:37:37.888 | INFO | Total mem usage: 15.7 GiB | 20260507 15:37:37.888 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:37:37.888 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:37:37.888 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:37:37.888 | INFO | High Water Breakdown: | 20260507 15:37:37.888 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:37:37.888 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:37:37.888 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:37:37.888 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:37:37.888 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:37:37.888 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:37:37.888 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:37:37.888 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:37:37.888 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:37:37.888 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:37:37.888 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:37:37.888 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:37:37.888 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:37:37.888 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:37:37.888 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:37:37.888 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:37:37.888 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:37:37.888 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:37:37.888 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:37:37.888 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:37:37.888 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:37:37.888 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:37:37.888 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:37:37.888 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:37:37.888 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:37:37.888 | INFO | ------------------------------------------ | 20260507 15:37:37.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:37:37.888 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:37:37.888 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:37:37.888 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:37:37.888 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 15:37:37.888 | INFO | PerfReport: 0.30%, AggregateStats -- 8.84e-02 / -nan, 7.35e-03 / 0.00e+00 | 20260507 15:37:37.888 | INFO | PerfReport: 11.05%, Baseline -- 3.30e+00 / -nan, 2.18e-02 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 3.08%, Histogram -- 9.20e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 16.14%, DME -- 3.86e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 3.45%, Metrics -- 1.03e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.82e-01 / -nan, 5.63e-02 / 0.00e+00 | 20260507 15:37:37.889 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:37:37.889 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:37:37.889 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:37:37.889 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 18.86%, Upload -- 1.68e+01 / 0.00e+00, 4.39e-01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 18.15%, Download -- 1.62e+01 / 0.00e+00, 7.77e-01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 33.48%, Compute -- 2.99e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 0.58%, LockUpload -- 5.21e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 0.00%, LockDownload -- 5.38e-04 / 0.00e+00, 4.13e-04 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 28.90%, LockCompute -- 2.58e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 15:37:37.889 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 15:37:37.984 | INFO | Starting a round of Baz IO | 20260507 15:37:40.891 | INFO | Populating RTMetrics report | 20260507 15:37:40.891 | INFO | RTMetrics report populated | 20260507 15:37:40.891 | INFO | Opening tmp RTMetrics json file | 20260507 15:37:40.893 | INFO | Serializing RTMetrics to json | 20260507 15:37:40.894 | INFO | Serialization done, swapping tmp file now... | 20260507 15:37:40.894 | INFO | RTMetrics done | 20260507 15:37:40.980 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 2.995 seconds | 20260507 15:37:40.980 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.68% / 0.04% / 47.28% | 20260507 15:37:40.980 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.56599% / 5.5631e-05% / 4.37101e-05% | 20260507 15:37:40.980 | INFO | Baz IO overall: | 20260507 15:37:40.980 | INFO | BytesWritten : 259426308299 | 20260507 15:37:40.980 | INFO | NumEvents : 125099665830 | 20260507 15:37:43.026 | INFO | Analyzing chunk frames = [630784,631296) | 20260507 15:37:43.026 | INFO | MemInfo: { | 20260507 15:37:43.026 | INFO | "gpu" : " 17 GB", | 20260507 15:37:43.026 | INFO | "huge" : "", | 20260507 15:37:43.026 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:43.026 | INFO | "malloc" : " 14 GB", | 20260507 15:37:43.026 | INFO | "pinned" : " 29 GB" | 20260507 15:37:43.026 | INFO | } | 20260507 15:37:43.026 | INFO | Perf report for the last 5.13915 seconds: | 20260507 15:37:43.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:43.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:43.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:43.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:43.027 | INFO | ANALYSIS : 57.6%, 2.9, 86.6ms, 0 | 20260507 15:37:43.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:43.027 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 15:37:43.027 | INFO | PRE_HQ : 56.4%, 2.2, 63.2ms, 0 | 20260507 15:37:43.027 | INFO | BAZWRITER : 57.8%, 4.3, 129.1ms, 0 | 20260507 15:37:43.028 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 15:37:43.028 | INFO | Uploaded 36GiB to GPU in 1.64291s (21.9124GiB/s) | 20260507 15:37:43.028 | INFO | Downloaded 38GiB to GPU in 1.72381s (22.0442GiB/s) | 20260507 15:37:48.166 | INFO | Analyzing chunk frames = [631296,631808) | 20260507 15:37:48.166 | INFO | MemInfo: { | 20260507 15:37:48.166 | INFO | "gpu" : " 17 GB", | 20260507 15:37:48.166 | INFO | "huge" : "", | 20260507 15:37:48.166 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:48.166 | INFO | "malloc" : " 14 GB", | 20260507 15:37:48.166 | INFO | "pinned" : " 29 GB" | 20260507 15:37:48.166 | INFO | } | 20260507 15:37:48.166 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:37:48.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:48.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.166 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260507 15:37:48.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.166 | INFO | PRE_HQ : 39.2%, 1.0, 20.4ms, 0 | 20260507 15:37:48.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:48.167 | INFO | Uploaded 36GiB to GPU in 1.59233s (22.6084GiB/s) | 20260507 15:37:48.167 | INFO | Downloaded 34GiB to GPU in 1.5398s (22.0808GiB/s) | 20260507 15:37:53.305 | INFO | Analyzing chunk frames = [631808,632320) | 20260507 15:37:53.305 | INFO | MemInfo: { | 20260507 15:37:53.305 | INFO | "gpu" : " 17 GB", | 20260507 15:37:53.305 | INFO | "huge" : "", | 20260507 15:37:53.305 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:53.305 | INFO | "malloc" : " 14 GB", | 20260507 15:37:53.305 | INFO | "pinned" : " 29 GB" | 20260507 15:37:53.305 | INFO | } | 20260507 15:37:53.305 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:37:53.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:53.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.305 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260507 15:37:53.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.305 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 15:37:53.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:53.306 | INFO | Uploaded 36GiB to GPU in 1.59438s (22.5793GiB/s) | 20260507 15:37:53.306 | INFO | Downloaded 34GiB to GPU in 1.54282s (22.0375GiB/s) | 20260507 15:37:58.444 | INFO | Analyzing chunk frames = [632320,632832) | 20260507 15:37:58.444 | INFO | MemInfo: { | 20260507 15:37:58.444 | INFO | "gpu" : " 17 GB", | 20260507 15:37:58.444 | INFO | "huge" : "", | 20260507 15:37:58.444 | INFO | "huge_shared" : " 29 GB", | 20260507 15:37:58.445 | INFO | "malloc" : " 14 GB", | 20260507 15:37:58.445 | INFO | "pinned" : " 29 GB" | 20260507 15:37:58.445 | INFO | } | 20260507 15:37:58.445 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:37:58.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:37:58.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:37:58.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | PRE_HQ : 39.7%, 1.0, 20.7ms, 0 | 20260507 15:37:58.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:37:58.445 | INFO | Uploaded 36GiB to GPU in 1.5959s (22.5579GiB/s) | 20260507 15:37:58.445 | INFO | Downloaded 34GiB to GPU in 1.5442s (22.0179GiB/s) | 20260507 15:38:03.587 | INFO | Analyzing chunk frames = [632832,633344) | 20260507 15:38:03.587 | INFO | MemInfo: { | 20260507 15:38:03.587 | INFO | "gpu" : " 17 GB", | 20260507 15:38:03.587 | INFO | "huge" : "", | 20260507 15:38:03.587 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:03.587 | INFO | "malloc" : " 14 GB", | 20260507 15:38:03.587 | INFO | "pinned" : " 29 GB" | 20260507 15:38:03.587 | INFO | } | 20260507 15:38:03.588 | INFO | Perf report for the last 5.1429 seconds: | 20260507 15:38:03.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:03.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:38:03.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 15:38:03.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:03.588 | INFO | Uploaded 36GiB to GPU in 1.593s (22.5989GiB/s) | 20260507 15:38:03.588 | INFO | Downloaded 34GiB to GPU in 1.54144s (22.0573GiB/s) | 20260507 15:38:08.723 | INFO | Analyzing chunk frames = [633344,633856) | 20260507 15:38:08.723 | INFO | MemInfo: { | 20260507 15:38:08.723 | INFO | "gpu" : " 17 GB", | 20260507 15:38:08.723 | INFO | "huge" : "", | 20260507 15:38:08.723 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:08.723 | INFO | "malloc" : " 14 GB", | 20260507 15:38:08.723 | INFO | "pinned" : " 29 GB" | 20260507 15:38:08.723 | INFO | } | 20260507 15:38:08.723 | INFO | Perf report for the last 5.13554 seconds: | 20260507 15:38:08.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:08.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.723 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:38:08.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.723 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:38:08.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:08.724 | INFO | Uploaded 36GiB to GPU in 1.58696s (22.6849GiB/s) | 20260507 15:38:08.724 | INFO | Downloaded 34GiB to GPU in 1.53725s (22.1174GiB/s) | 20260507 15:38:13.863 | INFO | Analyzing chunk frames = [633856,634368) | 20260507 15:38:13.863 | INFO | MemInfo: { | 20260507 15:38:13.863 | INFO | "gpu" : " 17 GB", | 20260507 15:38:13.863 | INFO | "huge" : "", | 20260507 15:38:13.863 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:13.863 | INFO | "malloc" : " 14 GB", | 20260507 15:38:13.863 | INFO | "pinned" : " 29 GB" | 20260507 15:38:13.863 | INFO | } | 20260507 15:38:13.863 | INFO | Perf report for the last 5.13973 seconds: | 20260507 15:38:13.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:13.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:38:13.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:38:13.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:13.863 | INFO | Uploaded 36GiB to GPU in 1.58751s (22.677GiB/s) | 20260507 15:38:13.863 | INFO | Downloaded 34GiB to GPU in 1.53805s (22.1059GiB/s) | 20260507 15:38:19.002 | INFO | Analyzing chunk frames = [634368,634880) | 20260507 15:38:19.002 | INFO | MemInfo: { | 20260507 15:38:19.002 | INFO | "gpu" : " 17 GB", | 20260507 15:38:19.002 | INFO | "huge" : "", | 20260507 15:38:19.002 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:19.002 | INFO | "malloc" : " 14 GB", | 20260507 15:38:19.002 | INFO | "pinned" : " 29 GB" | 20260507 15:38:19.002 | INFO | } | 20260507 15:38:19.002 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:38:19.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:19.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:38:19.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 15:38:19.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:19.002 | INFO | Uploaded 36GiB to GPU in 1.58766s (22.6748GiB/s) | 20260507 15:38:19.003 | INFO | Downloaded 34GiB to GPU in 1.53827s (22.1027GiB/s) | 20260507 15:38:19.099 | INFO | Starting a round of Baz IO | 20260507 15:38:22.038 | INFO | Populating RTMetrics report | 20260507 15:38:22.038 | INFO | RTMetrics report populated | 20260507 15:38:22.038 | INFO | Opening tmp RTMetrics json file | 20260507 15:38:22.047 | INFO | Serializing RTMetrics to json | 20260507 15:38:22.074 | INFO | Serialization done, swapping tmp file now... | 20260507 15:38:22.098 | INFO | RTMetrics done | 20260507 15:38:22.125 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.026 seconds | 20260507 15:38:22.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.64% / 0.04% / 47.32% | 20260507 15:38:22.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.54848% / 7.15256e-05% / 4.76837e-05% | 20260507 15:38:22.126 | INFO | Baz IO overall: | 20260507 15:38:22.126 | INFO | BytesWritten : 261317197599 | 20260507 15:38:22.126 | INFO | NumEvents : 126020144105 | 20260507 15:38:24.141 | INFO | Analyzing chunk frames = [634880,635392) | 20260507 15:38:24.141 | INFO | MemInfo: { | 20260507 15:38:24.141 | INFO | "gpu" : " 17 GB", | 20260507 15:38:24.141 | INFO | "huge" : "", | 20260507 15:38:24.141 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:24.141 | INFO | "malloc" : " 14 GB", | 20260507 15:38:24.141 | INFO | "pinned" : " 29 GB" | 20260507 15:38:24.141 | INFO | } | 20260507 15:38:24.142 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:38:24.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:24.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:24.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:24.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:24.142 | INFO | ANALYSIS : 57.3%, 2.8, 85.5ms, 0 | 20260507 15:38:24.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:24.142 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260507 15:38:24.142 | INFO | PRE_HQ : 56.7%, 2.1, 61.7ms, 0 | 20260507 15:38:24.142 | INFO | BAZWRITER : 58.4%, 4.7, 141.5ms, 0 | 20260507 15:38:24.142 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:38:24.142 | INFO | Uploaded 36GiB to GPU in 1.65466s (21.7567GiB/s) | 20260507 15:38:24.142 | INFO | Downloaded 38GiB to GPU in 1.7389s (21.853GiB/s) | 20260507 15:38:29.281 | INFO | Analyzing chunk frames = [635392,635904) | 20260507 15:38:29.281 | INFO | MemInfo: { | 20260507 15:38:29.281 | INFO | "gpu" : " 17 GB", | 20260507 15:38:29.281 | INFO | "huge" : "", | 20260507 15:38:29.281 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:29.281 | INFO | "malloc" : " 14 GB", | 20260507 15:38:29.281 | INFO | "pinned" : " 29 GB" | 20260507 15:38:29.281 | INFO | } | 20260507 15:38:29.281 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:38:29.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:29.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:38:29.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:38:29.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:29.281 | INFO | Uploaded 36GiB to GPU in 1.61025s (22.3567GiB/s) | 20260507 15:38:29.282 | INFO | Downloaded 34GiB to GPU in 1.53071s (22.2119GiB/s) | 20260507 15:38:34.420 | INFO | Analyzing chunk frames = [635904,636416) | 20260507 15:38:34.420 | INFO | MemInfo: { | 20260507 15:38:34.420 | INFO | "gpu" : " 17 GB", | 20260507 15:38:34.420 | INFO | "huge" : "", | 20260507 15:38:34.420 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:34.420 | INFO | "malloc" : " 14 GB", | 20260507 15:38:34.420 | INFO | "pinned" : " 29 GB" | 20260507 15:38:34.420 | INFO | } | 20260507 15:38:34.420 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:38:34.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:34.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.420 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:38:34.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.420 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:38:34.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:34.421 | INFO | Uploaded 36GiB to GPU in 1.61693s (22.2644GiB/s) | 20260507 15:38:34.421 | INFO | Downloaded 34GiB to GPU in 1.53587s (22.1373GiB/s) | 20260507 15:38:39.559 | INFO | Analyzing chunk frames = [636416,636928) | 20260507 15:38:39.559 | INFO | MemInfo: { | 20260507 15:38:39.559 | INFO | "gpu" : " 17 GB", | 20260507 15:38:39.559 | INFO | "huge" : "", | 20260507 15:38:39.559 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:39.559 | INFO | "malloc" : " 14 GB", | 20260507 15:38:39.559 | INFO | "pinned" : " 29 GB" | 20260507 15:38:39.559 | INFO | } | 20260507 15:38:39.559 | INFO | Perf report for the last 5.139 seconds: | 20260507 15:38:39.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:39.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.559 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:38:39.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.559 | INFO | PRE_HQ : 39.9%, 1.0, 20.7ms, 0 | 20260507 15:38:39.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:39.560 | INFO | Uploaded 36GiB to GPU in 1.61514s (22.289GiB/s) | 20260507 15:38:39.560 | INFO | Downloaded 34GiB to GPU in 1.53439s (22.1586GiB/s) | 20260507 15:38:44.698 | INFO | Analyzing chunk frames = [636928,637440) | 20260507 15:38:44.698 | INFO | MemInfo: { | 20260507 15:38:44.698 | INFO | "gpu" : " 17 GB", | 20260507 15:38:44.698 | INFO | "huge" : "", | 20260507 15:38:44.698 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:44.698 | INFO | "malloc" : " 14 GB", | 20260507 15:38:44.699 | INFO | "pinned" : " 29 GB" | 20260507 15:38:44.699 | INFO | } | 20260507 15:38:44.699 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:38:44.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:44.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:38:44.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:38:44.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:44.699 | INFO | Uploaded 36GiB to GPU in 1.60815s (22.386GiB/s) | 20260507 15:38:44.699 | INFO | Downloaded 34GiB to GPU in 1.52834s (22.2463GiB/s) | 20260507 15:38:49.838 | INFO | Analyzing chunk frames = [637440,637952) | 20260507 15:38:49.838 | INFO | MemInfo: { | 20260507 15:38:49.838 | INFO | "gpu" : " 17 GB", | 20260507 15:38:49.838 | INFO | "huge" : "", | 20260507 15:38:49.838 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:49.838 | INFO | "malloc" : " 14 GB", | 20260507 15:38:49.838 | INFO | "pinned" : " 29 GB" | 20260507 15:38:49.838 | INFO | } | 20260507 15:38:49.838 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:38:49.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:49.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:38:49.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:38:49.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:49.838 | INFO | Uploaded 36GiB to GPU in 1.60786s (22.39GiB/s) | 20260507 15:38:49.838 | INFO | Downloaded 34GiB to GPU in 1.52715s (22.2637GiB/s) | 20260507 15:38:54.977 | INFO | Analyzing chunk frames = [637952,638464) | 20260507 15:38:54.977 | INFO | MemInfo: { | 20260507 15:38:54.977 | INFO | "gpu" : " 17 GB", | 20260507 15:38:54.977 | INFO | "huge" : "", | 20260507 15:38:54.977 | INFO | "huge_shared" : " 29 GB", | 20260507 15:38:54.977 | INFO | "malloc" : " 14 GB", | 20260507 15:38:54.977 | INFO | "pinned" : " 29 GB" | 20260507 15:38:54.977 | INFO | } | 20260507 15:38:54.977 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:38:54.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:38:54.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.977 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:38:54.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.977 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:38:54.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:38:54.978 | INFO | Uploaded 36GiB to GPU in 1.6083s (22.3839GiB/s) | 20260507 15:38:54.978 | INFO | Downloaded 34GiB to GPU in 1.52851s (22.2439GiB/s) | 20260507 15:39:00.116 | INFO | Analyzing chunk frames = [638464,638976) | 20260507 15:39:00.116 | INFO | MemInfo: { | 20260507 15:39:00.116 | INFO | "gpu" : " 17 GB", | 20260507 15:39:00.116 | INFO | "huge" : "", | 20260507 15:39:00.116 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:00.117 | INFO | "malloc" : " 14 GB", | 20260507 15:39:00.117 | INFO | "pinned" : " 29 GB" | 20260507 15:39:00.117 | INFO | } | 20260507 15:39:00.117 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:39:00.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:00.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:39:00.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:39:00.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:00.117 | INFO | Uploaded 36GiB to GPU in 1.60855s (22.3804GiB/s) | 20260507 15:39:00.117 | INFO | Downloaded 34GiB to GPU in 1.529s (22.2367GiB/s) | 20260507 15:39:00.117 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:39:00.117 | INFO | Total mem usage: 16.4 GiB | 20260507 15:39:00.117 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:39:00.117 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.34%) | 20260507 15:39:00.117 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:39:00.117 | INFO | High Water Breakdown: | 20260507 15:39:00.117 | INFO | 59.5% : 9.01 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:39:00.117 | INFO | 40.3% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:39:00.117 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:39:00.117 | INFO | ------------------------------------------ | 20260507 15:39:00.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:39:00.117 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:39:00.117 | INFO | Total mem usage: 26.5 GiB | 20260507 15:39:00.117 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:39:00.117 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 15:39:00.117 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 15:39:00.117 | INFO | High Water Breakdown: | 20260507 15:39:00.117 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:39:00.117 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:39:00.117 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:39:00.117 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:39:00.117 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:39:00.117 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:39:00.117 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:39:00.117 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:39:00.117 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:39:00.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:39:00.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:39:00.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:39:00.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:39:00.117 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:39:00.117 | INFO | ------------------------------------------ | 20260507 15:39:00.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:39:00.117 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:39:00.117 | INFO | Total mem usage: 27.8 GiB | 20260507 15:39:00.117 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:39:00.117 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:39:00.117 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:39:00.117 | INFO | High Water Breakdown: | 20260507 15:39:00.117 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:39:00.117 | INFO | ------------------------------------------ | 20260507 15:39:00.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:39:00.118 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:39:00.118 | INFO | Total mem usage: 15.7 GiB | 20260507 15:39:00.118 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:39:00.118 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:39:00.118 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:39:00.118 | INFO | High Water Breakdown: | 20260507 15:39:00.118 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:39:00.118 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:39:00.118 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:39:00.118 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:39:00.118 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:39:00.118 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:39:00.118 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:39:00.118 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:39:00.118 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:39:00.118 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:39:00.118 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:39:00.118 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:39:00.118 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:39:00.118 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:39:00.118 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:39:00.118 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:39:00.118 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:39:00.118 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:39:00.118 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:39:00.118 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:39:00.118 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:39:00.118 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:39:00.118 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:39:00.118 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:39:00.118 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:39:00.118 | INFO | ------------------------------------------ | 20260507 15:39:00.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:39:00.118 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:39:00.118 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:39:00.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:39:00.118 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 0.30%, AggregateStats -- 9.00e-02 / -nan, 7.95e-03 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 11.07%, Baseline -- 3.31e+00 / -nan, 2.49e-02 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.38e-02 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 16.12%, DME -- 3.85e+01 / -nan, 2.99e+00 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.14e+00 / -nan, 1.22e-01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 3.45%, Metrics -- 1.03e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.83e-01 / -nan, 5.64e-02 / 0.00e+00 | 20260507 15:39:00.118 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:39:00.118 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:39:00.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:39:00.118 | INFO | PerfReport: Full Execution -- 8.94e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 18.73%, Upload -- 1.67e+01 / 0.00e+00, 4.61e-01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 18.18%, Download -- 1.63e+01 / 0.00e+00, 7.81e-01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 33.43%, Compute -- 2.99e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 0.59%, LockUpload -- 5.27e-01 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 0.00%, LockDownload -- 5.70e-04 / 0.00e+00, 4.18e-04 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 29.06%, LockCompute -- 2.60e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:39:00.118 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260507 15:39:00.214 | INFO | Starting a round of Baz IO | 20260507 15:39:03.092 | INFO | Populating RTMetrics report | 20260507 15:39:03.092 | INFO | RTMetrics report populated | 20260507 15:39:03.092 | INFO | Opening tmp RTMetrics json file | 20260507 15:39:03.189 | INFO | Serializing RTMetrics to json | 20260507 15:39:03.209 | INFO | Serialization done, swapping tmp file now... | 20260507 15:39:03.210 | INFO | RTMetrics done | 20260507 15:39:03.341 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.127 seconds | 20260507 15:39:03.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.60% / 0.04% / 47.36% | 20260507 15:39:03.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.51926% / 8.34465e-05% / 2.38419e-05% | 20260507 15:39:03.341 | INFO | Baz IO overall: | 20260507 15:39:03.341 | INFO | BytesWritten : 263207009266 | 20260507 15:39:03.341 | INFO | NumEvents : 126939970569 | 20260507 15:39:05.256 | INFO | Analyzing chunk frames = [638976,639488) | 20260507 15:39:05.256 | INFO | MemInfo: { | 20260507 15:39:05.256 | INFO | "gpu" : " 17 GB", | 20260507 15:39:05.256 | INFO | "huge" : "", | 20260507 15:39:05.256 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:05.256 | INFO | "malloc" : " 14 GB", | 20260507 15:39:05.256 | INFO | "pinned" : " 29 GB" | 20260507 15:39:05.256 | INFO | } | 20260507 15:39:05.256 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:39:05.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:05.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:05.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:05.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:05.256 | INFO | ANALYSIS : 57.0%, 2.9, 85.4ms, 0 | 20260507 15:39:05.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:05.256 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 15:39:05.256 | INFO | PRE_HQ : 56.0%, 2.0, 59.0ms, 0 | 20260507 15:39:05.256 | INFO | BAZWRITER : 60.4%, 4.8, 148.1ms, 0 | 20260507 15:39:05.256 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 15:39:05.256 | INFO | Uploaded 36GiB to GPU in 1.66848s (21.5766GiB/s) | 20260507 15:39:05.256 | INFO | Downloaded 38GiB to GPU in 1.7517s (21.6932GiB/s) | 20260507 15:39:10.395 | INFO | Analyzing chunk frames = [639488,640000) | 20260507 15:39:10.395 | INFO | MemInfo: { | 20260507 15:39:10.395 | INFO | "gpu" : " 17 GB", | 20260507 15:39:10.395 | INFO | "huge" : "", | 20260507 15:39:10.395 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:10.395 | INFO | "malloc" : " 14 GB", | 20260507 15:39:10.395 | INFO | "pinned" : " 29 GB" | 20260507 15:39:10.395 | INFO | } | 20260507 15:39:10.395 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:39:10.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:10.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.395 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:39:10.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.395 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:39:10.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:10.396 | INFO | Uploaded 36GiB to GPU in 1.58405s (22.7265GiB/s) | 20260507 15:39:10.396 | INFO | Downloaded 34GiB to GPU in 1.53439s (22.1587GiB/s) | 20260507 15:39:15.535 | INFO | Analyzing chunk frames = [640000,640512) | 20260507 15:39:15.535 | INFO | MemInfo: { | 20260507 15:39:15.535 | INFO | "gpu" : " 17 GB", | 20260507 15:39:15.535 | INFO | "huge" : "", | 20260507 15:39:15.535 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:15.535 | INFO | "malloc" : " 14 GB", | 20260507 15:39:15.535 | INFO | "pinned" : " 29 GB" | 20260507 15:39:15.535 | INFO | } | 20260507 15:39:15.535 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:39:15.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:15.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:39:15.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:39:15.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:15.535 | INFO | Uploaded 36GiB to GPU in 1.5896s (22.6472GiB/s) | 20260507 15:39:15.535 | INFO | Downloaded 34GiB to GPU in 1.53929s (22.088GiB/s) | 20260507 15:39:20.674 | INFO | Analyzing chunk frames = [640512,641024) | 20260507 15:39:20.674 | INFO | MemInfo: { | 20260507 15:39:20.674 | INFO | "gpu" : " 17 GB", | 20260507 15:39:20.674 | INFO | "huge" : "", | 20260507 15:39:20.674 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:20.674 | INFO | "malloc" : " 14 GB", | 20260507 15:39:20.674 | INFO | "pinned" : " 29 GB" | 20260507 15:39:20.674 | INFO | } | 20260507 15:39:20.674 | INFO | Perf report for the last 5.13906 seconds: | 20260507 15:39:20.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:20.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:39:20.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:39:20.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:20.674 | INFO | Uploaded 36GiB to GPU in 1.5887s (22.6601GiB/s) | 20260507 15:39:20.674 | INFO | Downloaded 34GiB to GPU in 1.53868s (22.0969GiB/s) | 20260507 15:39:25.813 | INFO | Analyzing chunk frames = [641024,641536) | 20260507 15:39:25.813 | INFO | MemInfo: { | 20260507 15:39:25.813 | INFO | "gpu" : " 17 GB", | 20260507 15:39:25.813 | INFO | "huge" : "", | 20260507 15:39:25.813 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:25.813 | INFO | "malloc" : " 14 GB", | 20260507 15:39:25.813 | INFO | "pinned" : " 29 GB" | 20260507 15:39:25.813 | INFO | } | 20260507 15:39:25.813 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:39:25.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:25.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.813 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:39:25.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.813 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:39:25.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:25.814 | INFO | Uploaded 36GiB to GPU in 1.58785s (22.6721GiB/s) | 20260507 15:39:25.814 | INFO | Downloaded 34GiB to GPU in 1.53661s (22.1267GiB/s) | 20260507 15:39:30.953 | INFO | Analyzing chunk frames = [641536,642048) | 20260507 15:39:30.953 | INFO | MemInfo: { | 20260507 15:39:30.953 | INFO | "gpu" : " 17 GB", | 20260507 15:39:30.953 | INFO | "huge" : "", | 20260507 15:39:30.953 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:30.953 | INFO | "malloc" : " 14 GB", | 20260507 15:39:30.953 | INFO | "pinned" : " 29 GB" | 20260507 15:39:30.953 | INFO | } | 20260507 15:39:30.953 | INFO | Perf report for the last 5.13952 seconds: | 20260507 15:39:30.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:30.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:39:30.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:39:30.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:30.953 | INFO | Uploaded 36GiB to GPU in 1.58688s (22.686GiB/s) | 20260507 15:39:30.953 | INFO | Downloaded 34GiB to GPU in 1.53686s (22.1231GiB/s) | 20260507 15:39:36.092 | INFO | Analyzing chunk frames = [642048,642560) | 20260507 15:39:36.092 | INFO | MemInfo: { | 20260507 15:39:36.092 | INFO | "gpu" : " 17 GB", | 20260507 15:39:36.092 | INFO | "huge" : "", | 20260507 15:39:36.092 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:36.092 | INFO | "malloc" : " 14 GB", | 20260507 15:39:36.092 | INFO | "pinned" : " 29 GB" | 20260507 15:39:36.092 | INFO | } | 20260507 15:39:36.092 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:39:36.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:36.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:39:36.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:39:36.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:36.092 | INFO | Uploaded 36GiB to GPU in 1.58797s (22.6705GiB/s) | 20260507 15:39:36.092 | INFO | Downloaded 34GiB to GPU in 1.53821s (22.1036GiB/s) | 20260507 15:39:41.231 | INFO | Analyzing chunk frames = [642560,643072) | 20260507 15:39:41.231 | INFO | MemInfo: { | 20260507 15:39:41.231 | INFO | "gpu" : " 17 GB", | 20260507 15:39:41.231 | INFO | "huge" : "", | 20260507 15:39:41.231 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:41.231 | INFO | "malloc" : " 14 GB", | 20260507 15:39:41.231 | INFO | "pinned" : " 29 GB" | 20260507 15:39:41.231 | INFO | } | 20260507 15:39:41.231 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:39:41.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:41.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.231 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:39:41.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.231 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:39:41.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:41.232 | INFO | Uploaded 36GiB to GPU in 1.58822s (22.6669GiB/s) | 20260507 15:39:41.232 | INFO | Downloaded 34GiB to GPU in 1.53783s (22.109GiB/s) | 20260507 15:39:41.328 | INFO | Starting a round of Baz IO | 20260507 15:39:44.205 | INFO | Populating RTMetrics report | 20260507 15:39:44.205 | INFO | RTMetrics report populated | 20260507 15:39:44.205 | INFO | Opening tmp RTMetrics json file | 20260507 15:39:44.209 | INFO | Serializing RTMetrics to json | 20260507 15:39:44.222 | INFO | Serialization done, swapping tmp file now... | 20260507 15:39:44.251 | INFO | RTMetrics done | 20260507 15:39:44.391 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.063 seconds | 20260507 15:39:44.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.56% / 0.04% / 47.40% | 20260507 15:39:44.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.50936% / 7.15256e-05% / 1.58946e-05% | 20260507 15:39:44.391 | INFO | Baz IO overall: | 20260507 15:39:44.391 | INFO | BytesWritten : 265098040767 | 20260507 15:39:44.391 | INFO | NumEvents : 127860354972 | 20260507 15:39:46.371 | INFO | Analyzing chunk frames = [643072,643584) | 20260507 15:39:46.371 | INFO | MemInfo: { | 20260507 15:39:46.371 | INFO | "gpu" : " 17 GB", | 20260507 15:39:46.371 | INFO | "huge" : "", | 20260507 15:39:46.371 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:46.371 | INFO | "malloc" : " 14 GB", | 20260507 15:39:46.371 | INFO | "pinned" : " 29 GB" | 20260507 15:39:46.371 | INFO | } | 20260507 15:39:46.371 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:39:46.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:46.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:46.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:46.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:46.371 | INFO | ANALYSIS : 57.0%, 2.9, 87.0ms, 0 | 20260507 15:39:46.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:46.371 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 15:39:46.371 | INFO | PRE_HQ : 55.8%, 2.0, 59.4ms, 0 | 20260507 15:39:46.371 | INFO | BAZWRITER : 59.1%, 4.5, 137.2ms, 0 | 20260507 15:39:46.371 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:39:46.371 | INFO | Uploaded 36GiB to GPU in 1.64314s (21.9092GiB/s) | 20260507 15:39:46.371 | INFO | Downloaded 38GiB to GPU in 1.72795s (21.9913GiB/s) | 20260507 15:39:51.510 | INFO | Analyzing chunk frames = [643584,644096) | 20260507 15:39:51.510 | INFO | MemInfo: { | 20260507 15:39:51.510 | INFO | "gpu" : " 17 GB", | 20260507 15:39:51.510 | INFO | "huge" : "", | 20260507 15:39:51.510 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:51.510 | INFO | "malloc" : " 14 GB", | 20260507 15:39:51.510 | INFO | "pinned" : " 29 GB" | 20260507 15:39:51.510 | INFO | } | 20260507 15:39:51.510 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:39:51.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:51.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:39:51.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:39:51.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:51.510 | INFO | Uploaded 36GiB to GPU in 1.58573s (22.7024GiB/s) | 20260507 15:39:51.511 | INFO | Downloaded 34GiB to GPU in 1.53409s (22.163GiB/s) | 20260507 15:39:56.649 | INFO | Analyzing chunk frames = [644096,644608) | 20260507 15:39:56.649 | INFO | MemInfo: { | 20260507 15:39:56.649 | INFO | "gpu" : " 17 GB", | 20260507 15:39:56.649 | INFO | "huge" : "", | 20260507 15:39:56.649 | INFO | "huge_shared" : " 29 GB", | 20260507 15:39:56.649 | INFO | "malloc" : " 14 GB", | 20260507 15:39:56.650 | INFO | "pinned" : " 29 GB" | 20260507 15:39:56.650 | INFO | } | 20260507 15:39:56.650 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:39:56.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:39:56.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:39:56.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:39:56.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:39:56.650 | INFO | Uploaded 36GiB to GPU in 1.58893s (22.6568GiB/s) | 20260507 15:39:56.650 | INFO | Downloaded 34GiB to GPU in 1.53895s (22.093GiB/s) | 20260507 15:40:01.788 | INFO | Analyzing chunk frames = [644608,645120) | 20260507 15:40:01.789 | INFO | MemInfo: { | 20260507 15:40:01.789 | INFO | "gpu" : " 17 GB", | 20260507 15:40:01.789 | INFO | "huge" : "", | 20260507 15:40:01.789 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:01.789 | INFO | "malloc" : " 14 GB", | 20260507 15:40:01.789 | INFO | "pinned" : " 29 GB" | 20260507 15:40:01.789 | INFO | } | 20260507 15:40:01.789 | INFO | Perf report for the last 5.13905 seconds: | 20260507 15:40:01.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:01.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:40:01.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:40:01.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:01.789 | INFO | Uploaded 36GiB to GPU in 1.59086s (22.6292GiB/s) | 20260507 15:40:01.789 | INFO | Downloaded 34GiB to GPU in 1.5395s (22.0851GiB/s) | 20260507 15:40:06.928 | INFO | Analyzing chunk frames = [645120,645632) | 20260507 15:40:06.928 | INFO | MemInfo: { | 20260507 15:40:06.928 | INFO | "gpu" : " 17 GB", | 20260507 15:40:06.928 | INFO | "huge" : "", | 20260507 15:40:06.928 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:06.928 | INFO | "malloc" : " 14 GB", | 20260507 15:40:06.928 | INFO | "pinned" : " 29 GB" | 20260507 15:40:06.928 | INFO | } | 20260507 15:40:06.928 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:40:06.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:06.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:40:06.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:40:06.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:06.928 | INFO | Uploaded 36GiB to GPU in 1.5873s (22.68GiB/s) | 20260507 15:40:06.928 | INFO | Downloaded 34GiB to GPU in 1.53687s (22.1229GiB/s) | 20260507 15:40:12.067 | INFO | Analyzing chunk frames = [645632,646144) | 20260507 15:40:12.067 | INFO | MemInfo: { | 20260507 15:40:12.067 | INFO | "gpu" : " 17 GB", | 20260507 15:40:12.067 | INFO | "huge" : "", | 20260507 15:40:12.067 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:12.067 | INFO | "malloc" : " 14 GB", | 20260507 15:40:12.067 | INFO | "pinned" : " 29 GB" | 20260507 15:40:12.067 | INFO | } | 20260507 15:40:12.067 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:40:12.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:12.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.068 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:40:12.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.068 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:40:12.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:12.068 | INFO | Uploaded 36GiB to GPU in 1.58655s (22.6908GiB/s) | 20260507 15:40:12.068 | INFO | Downloaded 34GiB to GPU in 1.53615s (22.1333GiB/s) | 20260507 15:40:17.207 | INFO | Analyzing chunk frames = [646144,646656) | 20260507 15:40:17.207 | INFO | MemInfo: { | 20260507 15:40:17.207 | INFO | "gpu" : " 17 GB", | 20260507 15:40:17.207 | INFO | "huge" : "", | 20260507 15:40:17.207 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:17.207 | INFO | "malloc" : " 14 GB", | 20260507 15:40:17.207 | INFO | "pinned" : " 29 GB" | 20260507 15:40:17.207 | INFO | } | 20260507 15:40:17.207 | INFO | Perf report for the last 5.13948 seconds: | 20260507 15:40:17.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:17.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:40:17.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:40:17.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:17.207 | INFO | Uploaded 36GiB to GPU in 1.58819s (22.6673GiB/s) | 20260507 15:40:17.208 | INFO | Downloaded 34GiB to GPU in 1.5381s (22.1052GiB/s) | 20260507 15:40:22.346 | INFO | Analyzing chunk frames = [646656,647168) | 20260507 15:40:22.346 | INFO | MemInfo: { | 20260507 15:40:22.346 | INFO | "gpu" : " 17 GB", | 20260507 15:40:22.346 | INFO | "huge" : "", | 20260507 15:40:22.346 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:22.346 | INFO | "malloc" : " 14 GB", | 20260507 15:40:22.346 | INFO | "pinned" : " 29 GB" | 20260507 15:40:22.346 | INFO | } | 20260507 15:40:22.346 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:40:22.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:22.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:40:22.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:40:22.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:22.346 | INFO | Uploaded 36GiB to GPU in 1.58787s (22.6718GiB/s) | 20260507 15:40:22.346 | INFO | Downloaded 34GiB to GPU in 1.53867s (22.0971GiB/s) | 20260507 15:40:22.346 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:40:22.346 | INFO | Total mem usage: 16.4 GiB | 20260507 15:40:22.346 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:40:22.346 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.37%) | 20260507 15:40:22.347 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:40:22.347 | INFO | High Water Breakdown: | 20260507 15:40:22.347 | INFO | 59.6% : 9.11 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:40:22.347 | INFO | 40.2% : 6.15 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:40:22.347 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:40:22.347 | INFO | ------------------------------------------ | 20260507 15:40:22.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:40:22.347 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:40:22.347 | INFO | Total mem usage: 26.5 GiB | 20260507 15:40:22.347 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:40:22.347 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:40:22.347 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:40:22.347 | INFO | High Water Breakdown: | 20260507 15:40:22.347 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:40:22.347 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:40:22.347 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:40:22.347 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:40:22.347 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:40:22.347 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:40:22.347 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:40:22.347 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:40:22.347 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:40:22.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:40:22.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:40:22.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:40:22.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:40:22.347 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:40:22.347 | INFO | ------------------------------------------ | 20260507 15:40:22.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:40:22.347 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:40:22.347 | INFO | Total mem usage: 27.8 GiB | 20260507 15:40:22.347 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:40:22.347 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:40:22.347 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:40:22.347 | INFO | High Water Breakdown: | 20260507 15:40:22.347 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:40:22.347 | INFO | ------------------------------------------ | 20260507 15:40:22.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:40:22.347 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:40:22.347 | INFO | Total mem usage: 15.7 GiB | 20260507 15:40:22.347 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:40:22.347 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:40:22.347 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:40:22.347 | INFO | High Water Breakdown: | 20260507 15:40:22.347 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:40:22.347 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:40:22.347 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:40:22.347 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:40:22.347 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:40:22.347 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:40:22.347 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:40:22.347 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:40:22.347 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:40:22.347 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:40:22.347 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:40:22.347 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:40:22.347 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:40:22.347 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:40:22.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:40:22.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:40:22.347 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:40:22.347 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:40:22.347 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:40:22.347 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:40:22.347 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:40:22.347 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:40:22.347 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:40:22.347 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:40:22.347 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:40:22.347 | INFO | ------------------------------------------ | 20260507 15:40:22.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:40:22.347 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:40:22.347 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:40:22.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:40:22.347 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.79e-03 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 11.04%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.13e+00 / -nan, 1.21e-01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 3.44%, Metrics -- 1.03e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.76e-01 / -nan, 5.59e-02 / 0.00e+00 | 20260507 15:40:22.347 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:40:22.347 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:40:22.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:40:22.347 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 18.63%, Upload -- 1.66e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 18.24%, Download -- 1.63e+01 / 0.00e+00, 7.99e-01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 33.41%, Compute -- 2.98e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 0.57%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 0.00%, LockDownload -- 4.39e-04 / 0.00e+00, 9.46e-05 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 29.14%, LockCompute -- 2.60e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:40:22.347 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 15:40:22.445 | INFO | Starting a round of Baz IO | 20260507 15:40:25.322 | INFO | Populating RTMetrics report | 20260507 15:40:25.323 | INFO | RTMetrics report populated | 20260507 15:40:25.323 | INFO | Opening tmp RTMetrics json file | 20260507 15:40:25.324 | INFO | Serializing RTMetrics to json | 20260507 15:40:25.325 | INFO | Serialization done, swapping tmp file now... | 20260507 15:40:25.328 | INFO | RTMetrics done | 20260507 15:40:25.640 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.195 seconds | 20260507 15:40:25.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.52% / 0.04% / 47.44% | 20260507 15:40:25.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.50863% / 9.93411e-05% / 2.78155e-05% | 20260507 15:40:25.641 | INFO | Baz IO overall: | 20260507 15:40:25.641 | INFO | BytesWritten : 266990720537 | 20260507 15:40:25.641 | INFO | NumEvents : 128782301524 | 20260507 15:40:27.485 | INFO | Analyzing chunk frames = [647168,647680) | 20260507 15:40:27.485 | INFO | MemInfo: { | 20260507 15:40:27.485 | INFO | "gpu" : " 17 GB", | 20260507 15:40:27.485 | INFO | "huge" : "", | 20260507 15:40:27.485 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:27.485 | INFO | "malloc" : " 14 GB", | 20260507 15:40:27.486 | INFO | "pinned" : " 29 GB" | 20260507 15:40:27.486 | INFO | } | 20260507 15:40:27.486 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:40:27.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:27.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:27.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:27.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:27.486 | INFO | ANALYSIS : 57.0%, 2.9, 86.7ms, 0 | 20260507 15:40:27.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:27.486 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 15:40:27.486 | INFO | PRE_HQ : 55.8%, 2.0, 59.4ms, 0 | 20260507 15:40:27.486 | INFO | BAZWRITER : 61.7%, 4.6, 145.9ms, 0 | 20260507 15:40:27.486 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 15:40:27.486 | INFO | Uploaded 36GiB to GPU in 1.64192s (21.9256GiB/s) | 20260507 15:40:27.486 | INFO | Downloaded 38GiB to GPU in 1.72704s (22.003GiB/s) | 20260507 15:40:32.625 | INFO | Analyzing chunk frames = [647680,648192) | 20260507 15:40:32.625 | INFO | MemInfo: { | 20260507 15:40:32.625 | INFO | "gpu" : " 17 GB", | 20260507 15:40:32.625 | INFO | "huge" : "", | 20260507 15:40:32.625 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:32.625 | INFO | "malloc" : " 14 GB", | 20260507 15:40:32.625 | INFO | "pinned" : " 29 GB" | 20260507 15:40:32.625 | INFO | } | 20260507 15:40:32.625 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:40:32.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:32.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:40:32.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:40:32.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:32.625 | INFO | Uploaded 36GiB to GPU in 1.58627s (22.6947GiB/s) | 20260507 15:40:32.625 | INFO | Downloaded 34GiB to GPU in 1.53498s (22.1501GiB/s) | 20260507 15:40:37.764 | INFO | Analyzing chunk frames = [648192,648704) | 20260507 15:40:37.764 | INFO | MemInfo: { | 20260507 15:40:37.764 | INFO | "gpu" : " 17 GB", | 20260507 15:40:37.764 | INFO | "huge" : "", | 20260507 15:40:37.764 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:37.764 | INFO | "malloc" : " 14 GB", | 20260507 15:40:37.764 | INFO | "pinned" : " 29 GB" | 20260507 15:40:37.764 | INFO | } | 20260507 15:40:37.764 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:40:37.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:37.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.764 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 15:40:37.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.764 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:40:37.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:37.765 | INFO | Uploaded 36GiB to GPU in 1.589s (22.6557GiB/s) | 20260507 15:40:37.765 | INFO | Downloaded 34GiB to GPU in 1.53861s (22.0979GiB/s) | 20260507 15:40:42.903 | INFO | Analyzing chunk frames = [648704,649216) | 20260507 15:40:42.903 | INFO | MemInfo: { | 20260507 15:40:42.903 | INFO | "gpu" : " 17 GB", | 20260507 15:40:42.903 | INFO | "huge" : "", | 20260507 15:40:42.904 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:42.904 | INFO | "malloc" : " 14 GB", | 20260507 15:40:42.904 | INFO | "pinned" : " 29 GB" | 20260507 15:40:42.904 | INFO | } | 20260507 15:40:42.904 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:40:42.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:42.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:40:42.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:40:42.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:42.904 | INFO | Uploaded 36GiB to GPU in 1.59035s (22.6365GiB/s) | 20260507 15:40:42.904 | INFO | Downloaded 34GiB to GPU in 1.53821s (22.1036GiB/s) | 20260507 15:40:48.043 | INFO | Analyzing chunk frames = [649216,649728) | 20260507 15:40:48.043 | INFO | MemInfo: { | 20260507 15:40:48.043 | INFO | "gpu" : " 17 GB", | 20260507 15:40:48.043 | INFO | "huge" : "", | 20260507 15:40:48.043 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:48.043 | INFO | "malloc" : " 14 GB", | 20260507 15:40:48.043 | INFO | "pinned" : " 29 GB" | 20260507 15:40:48.043 | INFO | } | 20260507 15:40:48.043 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:40:48.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:48.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:40:48.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:40:48.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:48.043 | INFO | Uploaded 36GiB to GPU in 1.5884s (22.6643GiB/s) | 20260507 15:40:48.043 | INFO | Downloaded 34GiB to GPU in 1.53693s (22.122GiB/s) | 20260507 15:40:53.182 | INFO | Analyzing chunk frames = [649728,650240) | 20260507 15:40:53.182 | INFO | MemInfo: { | 20260507 15:40:53.182 | INFO | "gpu" : " 17 GB", | 20260507 15:40:53.182 | INFO | "huge" : "", | 20260507 15:40:53.182 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:53.182 | INFO | "malloc" : " 14 GB", | 20260507 15:40:53.182 | INFO | "pinned" : " 29 GB" | 20260507 15:40:53.182 | INFO | } | 20260507 15:40:53.182 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:40:53.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:53.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:40:53.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:40:53.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:53.182 | INFO | Uploaded 36GiB to GPU in 1.58842s (22.664GiB/s) | 20260507 15:40:53.182 | INFO | Downloaded 34GiB to GPU in 1.53663s (22.1263GiB/s) | 20260507 15:40:58.321 | INFO | Analyzing chunk frames = [650240,650752) | 20260507 15:40:58.321 | INFO | MemInfo: { | 20260507 15:40:58.321 | INFO | "gpu" : " 17 GB", | 20260507 15:40:58.321 | INFO | "huge" : "", | 20260507 15:40:58.321 | INFO | "huge_shared" : " 29 GB", | 20260507 15:40:58.321 | INFO | "malloc" : " 14 GB", | 20260507 15:40:58.321 | INFO | "pinned" : " 29 GB" | 20260507 15:40:58.321 | INFO | } | 20260507 15:40:58.321 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:40:58.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:40:58.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.321 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:40:58.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.322 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:40:58.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:40:58.322 | INFO | Uploaded 36GiB to GPU in 1.58705s (22.6836GiB/s) | 20260507 15:40:58.322 | INFO | Downloaded 34GiB to GPU in 1.53742s (22.115GiB/s) | 20260507 15:41:03.461 | INFO | Analyzing chunk frames = [650752,651264) | 20260507 15:41:03.461 | INFO | MemInfo: { | 20260507 15:41:03.461 | INFO | "gpu" : " 17 GB", | 20260507 15:41:03.461 | INFO | "huge" : "", | 20260507 15:41:03.461 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:03.461 | INFO | "malloc" : " 14 GB", | 20260507 15:41:03.461 | INFO | "pinned" : " 29 GB" | 20260507 15:41:03.461 | INFO | } | 20260507 15:41:03.461 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:41:03.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:03.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.461 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:41:03.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.461 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:41:03.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:03.462 | INFO | Uploaded 36GiB to GPU in 1.58944s (22.6495GiB/s) | 20260507 15:41:03.462 | INFO | Downloaded 34GiB to GPU in 1.53873s (22.0962GiB/s) | 20260507 15:41:03.558 | INFO | Starting a round of Baz IO | 20260507 15:41:06.461 | INFO | Populating RTMetrics report | 20260507 15:41:06.462 | INFO | RTMetrics report populated | 20260507 15:41:06.462 | INFO | Opening tmp RTMetrics json file | 20260507 15:41:06.472 | INFO | Serializing RTMetrics to json | 20260507 15:41:06.542 | INFO | Serialization done, swapping tmp file now... | 20260507 15:41:06.564 | INFO | RTMetrics done | 20260507 15:41:06.845 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.286 seconds | 20260507 15:41:06.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.48% / 0.04% / 47.48% | 20260507 15:41:06.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.47716% / 7.94729e-05% / 3.57628e-05% | 20260507 15:41:06.845 | INFO | Baz IO overall: | 20260507 15:41:06.845 | INFO | BytesWritten : 268882578679 | 20260507 15:41:06.845 | INFO | NumEvents : 129703159290 | 20260507 15:41:08.600 | INFO | Analyzing chunk frames = [651264,651776) | 20260507 15:41:08.600 | INFO | MemInfo: { | 20260507 15:41:08.600 | INFO | "gpu" : " 17 GB", | 20260507 15:41:08.600 | INFO | "huge" : "", | 20260507 15:41:08.600 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:08.600 | INFO | "malloc" : " 14 GB", | 20260507 15:41:08.600 | INFO | "pinned" : " 29 GB" | 20260507 15:41:08.600 | INFO | } | 20260507 15:41:08.600 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:41:08.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:08.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:08.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:08.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:08.601 | INFO | ANALYSIS : 57.0%, 2.9, 85.8ms, 0 | 20260507 15:41:08.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:08.601 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 15:41:08.601 | INFO | PRE_HQ : 55.9%, 2.0, 59.3ms, 0 | 20260507 15:41:08.601 | INFO | BAZWRITER : 63.5%, 4.7, 153.2ms, 0 | 20260507 15:41:08.601 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 15:41:08.601 | INFO | Uploaded 36GiB to GPU in 1.66022s (21.6839GiB/s) | 20260507 15:41:08.601 | INFO | Downloaded 38GiB to GPU in 1.74606s (21.7633GiB/s) | 20260507 15:41:13.740 | INFO | Analyzing chunk frames = [651776,652288) | 20260507 15:41:13.740 | INFO | MemInfo: { | 20260507 15:41:13.740 | INFO | "gpu" : " 17 GB", | 20260507 15:41:13.740 | INFO | "huge" : "", | 20260507 15:41:13.740 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:13.740 | INFO | "malloc" : " 14 GB", | 20260507 15:41:13.740 | INFO | "pinned" : " 29 GB" | 20260507 15:41:13.740 | INFO | } | 20260507 15:41:13.740 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:41:13.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:13.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 15:41:13.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 15:41:13.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:13.740 | INFO | Uploaded 36GiB to GPU in 1.59218s (22.6105GiB/s) | 20260507 15:41:13.740 | INFO | Downloaded 34GiB to GPU in 1.53878s (22.0954GiB/s) | 20260507 15:41:18.879 | INFO | Analyzing chunk frames = [652288,652800) | 20260507 15:41:18.879 | INFO | MemInfo: { | 20260507 15:41:18.879 | INFO | "gpu" : " 17 GB", | 20260507 15:41:18.879 | INFO | "huge" : "", | 20260507 15:41:18.879 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:18.879 | INFO | "malloc" : " 14 GB", | 20260507 15:41:18.879 | INFO | "pinned" : " 29 GB" | 20260507 15:41:18.879 | INFO | } | 20260507 15:41:18.879 | INFO | Perf report for the last 5.13955 seconds: | 20260507 15:41:18.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:18.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.879 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:41:18.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.879 | INFO | PRE_HQ : 40.3%, 1.0, 20.9ms, 0 | 20260507 15:41:18.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:18.880 | INFO | Uploaded 36GiB to GPU in 1.59464s (22.5756GiB/s) | 20260507 15:41:18.880 | INFO | Downloaded 34GiB to GPU in 1.54553s (21.9989GiB/s) | 20260507 15:41:24.018 | INFO | Analyzing chunk frames = [652800,653312) | 20260507 15:41:24.018 | INFO | MemInfo: { | 20260507 15:41:24.018 | INFO | "gpu" : " 17 GB", | 20260507 15:41:24.018 | INFO | "huge" : "", | 20260507 15:41:24.018 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:24.018 | INFO | "malloc" : " 14 GB", | 20260507 15:41:24.018 | INFO | "pinned" : " 29 GB" | 20260507 15:41:24.018 | INFO | } | 20260507 15:41:24.018 | INFO | Perf report for the last 5.13901 seconds: | 20260507 15:41:24.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:24.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.018 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:41:24.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.018 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:41:24.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:24.019 | INFO | Uploaded 36GiB to GPU in 1.59017s (22.6392GiB/s) | 20260507 15:41:24.019 | INFO | Downloaded 34GiB to GPU in 1.53823s (22.1033GiB/s) | 20260507 15:41:29.158 | INFO | Analyzing chunk frames = [653312,653824) | 20260507 15:41:29.158 | INFO | MemInfo: { | 20260507 15:41:29.158 | INFO | "gpu" : " 17 GB", | 20260507 15:41:29.158 | INFO | "huge" : "", | 20260507 15:41:29.158 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:29.158 | INFO | "malloc" : " 14 GB", | 20260507 15:41:29.158 | INFO | "pinned" : " 29 GB" | 20260507 15:41:29.158 | INFO | } | 20260507 15:41:29.158 | INFO | Perf report for the last 5.13983 seconds: | 20260507 15:41:29.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:29.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.158 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:41:29.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.158 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:41:29.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:29.159 | INFO | Uploaded 36GiB to GPU in 1.58777s (22.6733GiB/s) | 20260507 15:41:29.159 | INFO | Downloaded 34GiB to GPU in 1.53617s (22.133GiB/s) | 20260507 15:41:34.297 | INFO | Analyzing chunk frames = [653824,654336) | 20260507 15:41:34.297 | INFO | MemInfo: { | 20260507 15:41:34.297 | INFO | "gpu" : " 17 GB", | 20260507 15:41:34.297 | INFO | "huge" : "", | 20260507 15:41:34.297 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:34.297 | INFO | "malloc" : " 14 GB", | 20260507 15:41:34.297 | INFO | "pinned" : " 29 GB" | 20260507 15:41:34.297 | INFO | } | 20260507 15:41:34.297 | INFO | Perf report for the last 5.13872 seconds: | 20260507 15:41:34.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:34.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:41:34.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:41:34.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:34.297 | INFO | Uploaded 36GiB to GPU in 1.58857s (22.6619GiB/s) | 20260507 15:41:34.297 | INFO | Downloaded 34GiB to GPU in 1.53714s (22.1191GiB/s) | 20260507 15:41:39.436 | INFO | Analyzing chunk frames = [654336,654848) | 20260507 15:41:39.436 | INFO | MemInfo: { | 20260507 15:41:39.436 | INFO | "gpu" : " 17 GB", | 20260507 15:41:39.436 | INFO | "huge" : "", | 20260507 15:41:39.436 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:39.436 | INFO | "malloc" : " 14 GB", | 20260507 15:41:39.436 | INFO | "pinned" : " 29 GB" | 20260507 15:41:39.436 | INFO | } | 20260507 15:41:39.436 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:41:39.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:39.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.436 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:41:39.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.436 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:41:39.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:39.437 | INFO | Uploaded 36GiB to GPU in 1.58909s (22.6544GiB/s) | 20260507 15:41:39.437 | INFO | Downloaded 34GiB to GPU in 1.53758s (22.1127GiB/s) | 20260507 15:41:44.576 | INFO | Analyzing chunk frames = [654848,655360) | 20260507 15:41:44.576 | INFO | MemInfo: { | 20260507 15:41:44.576 | INFO | "gpu" : " 17 GB", | 20260507 15:41:44.576 | INFO | "huge" : "", | 20260507 15:41:44.576 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:44.576 | INFO | "malloc" : " 14 GB", | 20260507 15:41:44.576 | INFO | "pinned" : " 29 GB" | 20260507 15:41:44.576 | INFO | } | 20260507 15:41:44.576 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:41:44.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:44.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:41:44.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 15:41:44.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:44.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:41:44.576 | INFO | Total mem usage: 16.4 GiB | 20260507 15:41:44.576 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:41:44.576 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -6.73%) | 20260507 15:41:44.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:41:44.576 | INFO | High Water Breakdown: | 20260507 15:41:44.576 | INFO | 59.7% : 9.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:41:44.576 | INFO | 40.2% : 6.17 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:41:44.576 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:41:44.576 | INFO | ------------------------------------------ | 20260507 15:41:44.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:41:44.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:41:44.576 | INFO | Total mem usage: 26.5 GiB | 20260507 15:41:44.576 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:41:44.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:41:44.576 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:41:44.576 | INFO | High Water Breakdown: | 20260507 15:41:44.576 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:41:44.576 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:41:44.576 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:41:44.576 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:41:44.576 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:41:44.576 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:41:44.576 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:41:44.576 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:41:44.576 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:41:44.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:41:44.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:41:44.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:41:44.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:41:44.576 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:41:44.576 | INFO | ------------------------------------------ | 20260507 15:41:44.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:41:44.576 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:41:44.576 | INFO | Total mem usage: 27.8 GiB | 20260507 15:41:44.576 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:41:44.576 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:41:44.576 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:41:44.576 | INFO | High Water Breakdown: | 20260507 15:41:44.576 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:41:44.576 | INFO | ------------------------------------------ | 20260507 15:41:44.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:41:44.576 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:41:44.576 | INFO | Total mem usage: 15.7 GiB | 20260507 15:41:44.576 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:41:44.576 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:41:44.576 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:41:44.576 | INFO | High Water Breakdown: | 20260507 15:41:44.576 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:41:44.576 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:41:44.576 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:41:44.576 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:41:44.576 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:41:44.576 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:41:44.576 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:41:44.576 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:41:44.576 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:41:44.576 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:41:44.576 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:41:44.576 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:41:44.576 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:41:44.576 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:41:44.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:41:44.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:41:44.576 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:41:44.576 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:41:44.576 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:41:44.576 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:41:44.576 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:41:44.576 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:41:44.576 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:41:44.576 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:41:44.576 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:41:44.576 | INFO | ------------------------------------------ | 20260507 15:41:44.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:41:44.576 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:41:44.576 | INFO | Uploaded 36GiB to GPU in 1.5891s (22.6544GiB/s) | 20260507 15:41:44.577 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:41:44.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:41:44.577 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | Downloaded 34GiB to GPU in 1.53892s (22.0934GiB/s) | 20260507 15:41:44.577 | INFO | PerfReport: 0.29%, AggregateStats -- 8.77e-02 / -nan, 7.46e-03 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 11.05%, Baseline -- 3.30e+00 / -nan, 2.21e-02 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.14e+00 / -nan, 1.21e-01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 3.44%, Metrics -- 1.03e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.78e-01 / -nan, 5.58e-02 / 0.00e+00 | 20260507 15:41:44.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:41:44.577 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:41:44.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:41:44.577 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 18.63%, Upload -- 1.66e+01 / 0.00e+00, 4.82e-01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 18.23%, Download -- 1.63e+01 / 0.00e+00, 7.87e-01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 33.41%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 0.00%, LockDownload -- 4.76e-04 / 0.00e+00, 2.72e-04 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 29.15%, LockCompute -- 2.60e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:41:44.577 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 15:41:44.674 | INFO | Starting a round of Baz IO | 20260507 15:41:47.552 | INFO | Populating RTMetrics report | 20260507 15:41:47.553 | INFO | RTMetrics report populated | 20260507 15:41:47.553 | INFO | Opening tmp RTMetrics json file | 20260507 15:41:47.561 | INFO | Serializing RTMetrics to json | 20260507 15:41:47.584 | INFO | Serialization done, swapping tmp file now... | 20260507 15:41:47.592 | INFO | RTMetrics done | 20260507 15:41:48.009 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.335 seconds | 20260507 15:41:48.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.44% / 0.04% / 47.52% | 20260507 15:41:48.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.4683% / 9.13938e-05% / 3.97364e-05% | 20260507 15:41:48.009 | INFO | Baz IO overall: | 20260507 15:41:48.009 | INFO | BytesWritten : 270775691701 | 20260507 15:41:48.009 | INFO | NumEvents : 130624839496 | 20260507 15:41:49.715 | INFO | Analyzing chunk frames = [655360,655872) | 20260507 15:41:49.715 | INFO | MemInfo: { | 20260507 15:41:49.715 | INFO | "gpu" : " 17 GB", | 20260507 15:41:49.715 | INFO | "huge" : "", | 20260507 15:41:49.715 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:49.715 | INFO | "malloc" : " 14 GB", | 20260507 15:41:49.715 | INFO | "pinned" : " 29 GB" | 20260507 15:41:49.715 | INFO | } | 20260507 15:41:49.715 | INFO | Perf report for the last 5.13963 seconds: | 20260507 15:41:49.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:49.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:49.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:49.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:49.715 | INFO | ANALYSIS : 57.0%, 2.9, 85.7ms, 0 | 20260507 15:41:49.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:49.715 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 15:41:49.715 | INFO | PRE_HQ : 55.8%, 2.0, 58.6ms, 0 | 20260507 15:41:49.715 | INFO | BAZWRITER : 64.4%, 4.8, 158.8ms, 0 | 20260507 15:41:49.715 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 15:41:49.716 | INFO | Uploaded 36GiB to GPU in 1.66656s (21.6014GiB/s) | 20260507 15:41:49.716 | INFO | Downloaded 38GiB to GPU in 1.75233s (21.6854GiB/s) | 20260507 15:41:54.854 | INFO | Analyzing chunk frames = [655872,656384) | 20260507 15:41:54.854 | INFO | MemInfo: { | 20260507 15:41:54.855 | INFO | "gpu" : " 17 GB", | 20260507 15:41:54.855 | INFO | "huge" : "", | 20260507 15:41:54.855 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:54.855 | INFO | "malloc" : " 14 GB", | 20260507 15:41:54.855 | INFO | "pinned" : " 29 GB" | 20260507 15:41:54.855 | INFO | } | 20260507 15:41:54.855 | INFO | Perf report for the last 5.13973 seconds: | 20260507 15:41:54.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:54.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | ANALYSIS : 57.1%, 3.0, 88.9ms, 0 | 20260507 15:41:54.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | PRE_HQ : 33.1%, 1.0, 17.8ms, 0 | 20260507 15:41:54.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:54.855 | INFO | Uploaded 36GiB to GPU in 1.59708s (22.5412GiB/s) | 20260507 15:41:54.855 | INFO | Downloaded 34GiB to GPU in 1.53285s (22.181GiB/s) | 20260507 15:41:59.994 | INFO | Analyzing chunk frames = [656384,656896) | 20260507 15:41:59.994 | INFO | MemInfo: { | 20260507 15:41:59.994 | INFO | "gpu" : " 17 GB", | 20260507 15:41:59.994 | INFO | "huge" : "", | 20260507 15:41:59.994 | INFO | "huge_shared" : " 29 GB", | 20260507 15:41:59.994 | INFO | "malloc" : " 14 GB", | 20260507 15:41:59.994 | INFO | "pinned" : " 29 GB" | 20260507 15:41:59.994 | INFO | } | 20260507 15:41:59.994 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:41:59.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:41:59.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.994 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:41:59.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.995 | INFO | PRE_HQ : 39.8%, 1.0, 20.7ms, 0 | 20260507 15:41:59.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:41:59.995 | INFO | Uploaded 36GiB to GPU in 1.6154s (22.2855GiB/s) | 20260507 15:41:59.995 | INFO | Downloaded 34GiB to GPU in 1.53463s (22.1551GiB/s) | 20260507 15:42:05.133 | INFO | Analyzing chunk frames = [656896,657408) | 20260507 15:42:05.133 | INFO | MemInfo: { | 20260507 15:42:05.133 | INFO | "gpu" : " 17 GB", | 20260507 15:42:05.133 | INFO | "huge" : "", | 20260507 15:42:05.133 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:05.133 | INFO | "malloc" : " 14 GB", | 20260507 15:42:05.133 | INFO | "pinned" : " 29 GB" | 20260507 15:42:05.133 | INFO | } | 20260507 15:42:05.133 | INFO | Perf report for the last 5.13903 seconds: | 20260507 15:42:05.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:05.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.133 | INFO | ANALYSIS : 56.2%, 3.0, 87.1ms, 0 | 20260507 15:42:05.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.134 | INFO | PRE_HQ : 40.0%, 1.0, 20.7ms, 0 | 20260507 15:42:05.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:05.134 | INFO | Uploaded 36GiB to GPU in 1.61491s (22.2922GiB/s) | 20260507 15:42:05.134 | INFO | Downloaded 34GiB to GPU in 1.53504s (22.1493GiB/s) | 20260507 15:42:10.272 | INFO | Analyzing chunk frames = [657408,657920) | 20260507 15:42:10.272 | INFO | MemInfo: { | 20260507 15:42:10.272 | INFO | "gpu" : " 17 GB", | 20260507 15:42:10.272 | INFO | "huge" : "", | 20260507 15:42:10.272 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:10.272 | INFO | "malloc" : " 14 GB", | 20260507 15:42:10.272 | INFO | "pinned" : " 29 GB" | 20260507 15:42:10.272 | INFO | } | 20260507 15:42:10.272 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:42:10.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:10.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:42:10.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | PRE_HQ : 39.4%, 1.0, 20.4ms, 0 | 20260507 15:42:10.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:10.273 | INFO | Uploaded 36GiB to GPU in 1.6139s (22.3062GiB/s) | 20260507 15:42:10.273 | INFO | Downloaded 34GiB to GPU in 1.53277s (22.1821GiB/s) | 20260507 15:42:15.412 | INFO | Analyzing chunk frames = [657920,658432) | 20260507 15:42:15.412 | INFO | MemInfo: { | 20260507 15:42:15.412 | INFO | "gpu" : " 17 GB", | 20260507 15:42:15.412 | INFO | "huge" : "", | 20260507 15:42:15.412 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:15.412 | INFO | "malloc" : " 14 GB", | 20260507 15:42:15.412 | INFO | "pinned" : " 29 GB" | 20260507 15:42:15.412 | INFO | } | 20260507 15:42:15.412 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:42:15.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:15.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:42:15.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 15:42:15.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:15.412 | INFO | Uploaded 36GiB to GPU in 1.60808s (22.3869GiB/s) | 20260507 15:42:15.413 | INFO | Downloaded 34GiB to GPU in 1.52824s (22.2479GiB/s) | 20260507 15:42:20.551 | INFO | Analyzing chunk frames = [658432,658944) | 20260507 15:42:20.551 | INFO | MemInfo: { | 20260507 15:42:20.551 | INFO | "gpu" : " 17 GB", | 20260507 15:42:20.551 | INFO | "huge" : "", | 20260507 15:42:20.551 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:20.551 | INFO | "malloc" : " 14 GB", | 20260507 15:42:20.551 | INFO | "pinned" : " 29 GB" | 20260507 15:42:20.551 | INFO | } | 20260507 15:42:20.551 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:42:20.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:20.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.551 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:42:20.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.551 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:42:20.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:20.552 | INFO | Uploaded 36GiB to GPU in 1.60865s (22.379GiB/s) | 20260507 15:42:20.552 | INFO | Downloaded 34GiB to GPU in 1.52973s (22.2262GiB/s) | 20260507 15:42:25.690 | INFO | Analyzing chunk frames = [658944,659456) | 20260507 15:42:25.690 | INFO | MemInfo: { | 20260507 15:42:25.690 | INFO | "gpu" : " 17 GB", | 20260507 15:42:25.690 | INFO | "huge" : "", | 20260507 15:42:25.690 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:25.690 | INFO | "malloc" : " 14 GB", | 20260507 15:42:25.690 | INFO | "pinned" : " 29 GB" | 20260507 15:42:25.690 | INFO | } | 20260507 15:42:25.690 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:42:25.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:25.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.691 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:42:25.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.691 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 15:42:25.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:25.691 | INFO | Uploaded 36GiB to GPU in 1.60843s (22.3821GiB/s) | 20260507 15:42:25.691 | INFO | Downloaded 34GiB to GPU in 1.52884s (22.2392GiB/s) | 20260507 15:42:25.788 | INFO | Starting a round of Baz IO | 20260507 15:42:28.714 | INFO | Populating RTMetrics report | 20260507 15:42:28.714 | INFO | RTMetrics report populated | 20260507 15:42:28.714 | INFO | Opening tmp RTMetrics json file | 20260507 15:42:28.733 | INFO | Serializing RTMetrics to json | 20260507 15:42:28.766 | INFO | Serialization done, swapping tmp file now... | 20260507 15:42:28.772 | INFO | RTMetrics done | 20260507 15:42:28.829 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.04 seconds | 20260507 15:42:28.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.40% / 0.04% / 47.56% | 20260507 15:42:28.829 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.4537% / 8.34465e-05% / 2.78155e-05% | 20260507 15:42:28.829 | INFO | Baz IO overall: | 20260507 15:42:28.829 | INFO | BytesWritten : 272670183308 | 20260507 15:42:28.829 | INFO | NumEvents : 131547424901 | 20260507 15:42:30.830 | INFO | Analyzing chunk frames = [659456,659968) | 20260507 15:42:30.830 | INFO | MemInfo: { | 20260507 15:42:30.830 | INFO | "gpu" : " 17 GB", | 20260507 15:42:30.830 | INFO | "huge" : "", | 20260507 15:42:30.830 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:30.830 | INFO | "malloc" : " 14 GB", | 20260507 15:42:30.830 | INFO | "pinned" : " 29 GB" | 20260507 15:42:30.830 | INFO | } | 20260507 15:42:30.830 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:42:30.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:30.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:30.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:30.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:30.830 | INFO | ANALYSIS : 57.5%, 2.8, 85.7ms, 0 | 20260507 15:42:30.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:30.830 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 15:42:30.830 | INFO | PRE_HQ : 56.6%, 2.2, 63.3ms, 0 | 20260507 15:42:30.830 | INFO | BAZWRITER : 58.7%, 4.5, 135.4ms, 0 | 20260507 15:42:30.830 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 15:42:30.830 | INFO | Uploaded 36GiB to GPU in 1.64785s (21.8467GiB/s) | 20260507 15:42:30.830 | INFO | Downloaded 38GiB to GPU in 1.73159s (21.9451GiB/s) | 20260507 15:42:35.969 | INFO | Analyzing chunk frames = [659968,660480) | 20260507 15:42:35.969 | INFO | MemInfo: { | 20260507 15:42:35.969 | INFO | "gpu" : " 17 GB", | 20260507 15:42:35.969 | INFO | "huge" : "", | 20260507 15:42:35.969 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:35.969 | INFO | "malloc" : " 14 GB", | 20260507 15:42:35.969 | INFO | "pinned" : " 29 GB" | 20260507 15:42:35.969 | INFO | } | 20260507 15:42:35.969 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:42:35.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:35.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.969 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260507 15:42:35.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.969 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:42:35.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:35.970 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260507 15:42:35.970 | INFO | Downloaded 34GiB to GPU in 1.52551s (22.2876GiB/s) | 20260507 15:42:41.108 | INFO | Analyzing chunk frames = [660480,660992) | 20260507 15:42:41.108 | INFO | MemInfo: { | 20260507 15:42:41.108 | INFO | "gpu" : " 17 GB", | 20260507 15:42:41.108 | INFO | "huge" : "", | 20260507 15:42:41.108 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:41.108 | INFO | "malloc" : " 14 GB", | 20260507 15:42:41.109 | INFO | "pinned" : " 29 GB" | 20260507 15:42:41.109 | INFO | } | 20260507 15:42:41.109 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:42:41.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:41.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:42:41.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:42:41.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:41.109 | INFO | Uploaded 36GiB to GPU in 1.60993s (22.3612GiB/s) | 20260507 15:42:41.109 | INFO | Downloaded 34GiB to GPU in 1.53048s (22.2153GiB/s) | 20260507 15:42:46.248 | INFO | Analyzing chunk frames = [660992,661504) | 20260507 15:42:46.248 | INFO | MemInfo: { | 20260507 15:42:46.248 | INFO | "gpu" : " 17 GB", | 20260507 15:42:46.248 | INFO | "huge" : "", | 20260507 15:42:46.248 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:46.248 | INFO | "malloc" : " 14 GB", | 20260507 15:42:46.248 | INFO | "pinned" : " 29 GB" | 20260507 15:42:46.248 | INFO | } | 20260507 15:42:46.248 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:42:46.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:46.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:42:46.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:42:46.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:46.248 | INFO | Uploaded 36GiB to GPU in 1.6099s (22.3616GiB/s) | 20260507 15:42:46.249 | INFO | Downloaded 34GiB to GPU in 1.53032s (22.2176GiB/s) | 20260507 15:42:51.387 | INFO | Analyzing chunk frames = [661504,662016) | 20260507 15:42:51.387 | INFO | MemInfo: { | 20260507 15:42:51.387 | INFO | "gpu" : " 17 GB", | 20260507 15:42:51.387 | INFO | "huge" : "", | 20260507 15:42:51.387 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:51.387 | INFO | "malloc" : " 14 GB", | 20260507 15:42:51.387 | INFO | "pinned" : " 29 GB" | 20260507 15:42:51.387 | INFO | } | 20260507 15:42:51.387 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:42:51.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:51.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.387 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:42:51.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.387 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:42:51.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:51.388 | INFO | Uploaded 36GiB to GPU in 1.60843s (22.3821GiB/s) | 20260507 15:42:51.388 | INFO | Downloaded 34GiB to GPU in 1.52731s (22.2614GiB/s) | 20260507 15:42:56.526 | INFO | Analyzing chunk frames = [662016,662528) | 20260507 15:42:56.526 | INFO | MemInfo: { | 20260507 15:42:56.526 | INFO | "gpu" : " 17 GB", | 20260507 15:42:56.526 | INFO | "huge" : "", | 20260507 15:42:56.526 | INFO | "huge_shared" : " 29 GB", | 20260507 15:42:56.526 | INFO | "malloc" : " 14 GB", | 20260507 15:42:56.527 | INFO | "pinned" : " 29 GB" | 20260507 15:42:56.527 | INFO | } | 20260507 15:42:56.527 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:42:56.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:42:56.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:42:56.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:42:56.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:42:56.527 | INFO | Uploaded 36GiB to GPU in 1.60786s (22.39GiB/s) | 20260507 15:42:56.527 | INFO | Downloaded 34GiB to GPU in 1.52761s (22.2569GiB/s) | 20260507 15:43:01.666 | INFO | Analyzing chunk frames = [662528,663040) | 20260507 15:43:01.666 | INFO | MemInfo: { | 20260507 15:43:01.666 | INFO | "gpu" : " 17 GB", | 20260507 15:43:01.666 | INFO | "huge" : "", | 20260507 15:43:01.666 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:01.666 | INFO | "malloc" : " 14 GB", | 20260507 15:43:01.666 | INFO | "pinned" : " 29 GB" | 20260507 15:43:01.666 | INFO | } | 20260507 15:43:01.666 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:43:01.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:01.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:43:01.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 15:43:01.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:01.666 | INFO | Uploaded 36GiB to GPU in 1.60944s (22.368GiB/s) | 20260507 15:43:01.666 | INFO | Downloaded 34GiB to GPU in 1.53045s (22.2157GiB/s) | 20260507 15:43:06.805 | INFO | Analyzing chunk frames = [663040,663552) | 20260507 15:43:06.805 | INFO | MemInfo: { | 20260507 15:43:06.805 | INFO | "gpu" : " 17 GB", | 20260507 15:43:06.805 | INFO | "huge" : "", | 20260507 15:43:06.805 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:06.805 | INFO | "malloc" : " 14 GB", | 20260507 15:43:06.805 | INFO | "pinned" : " 29 GB" | 20260507 15:43:06.805 | INFO | } | 20260507 15:43:06.805 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:43:06.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:06.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.805 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:43:06.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.806 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:43:06.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:06.806 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:43:06.806 | INFO | Total mem usage: 16.4 GiB | 20260507 15:43:06.806 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:43:06.806 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -4.5%) | 20260507 15:43:06.806 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:43:06.806 | INFO | Uploaded 36GiB to GPU in 1.61493s (22.292GiB/s) | 20260507 15:43:06.806 | INFO | High Water Breakdown: | 20260507 15:43:06.806 | INFO | Downloaded 34GiB to GPU in 1.53398s (22.1646GiB/s) | 20260507 15:43:06.806 | INFO | 60.1% : 9.43 GiB ( 95% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:43:06.806 | INFO | 39.8% : 6.25 GiB ( 93% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:43:06.806 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:43:06.806 | INFO | ------------------------------------------ | 20260507 15:43:06.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:43:06.806 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:43:06.806 | INFO | Total mem usage: 26.5 GiB | 20260507 15:43:06.806 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:43:06.806 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:43:06.806 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:43:06.806 | INFO | High Water Breakdown: | 20260507 15:43:06.806 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:43:06.806 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:43:06.806 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:43:06.806 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:43:06.806 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:43:06.806 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:43:06.806 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:43:06.806 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:43:06.806 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:43:06.806 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:43:06.806 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:43:06.806 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:43:06.806 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:43:06.806 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:43:06.806 | INFO | ------------------------------------------ | 20260507 15:43:06.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:43:06.806 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:43:06.806 | INFO | Total mem usage: 27.8 GiB | 20260507 15:43:06.806 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:43:06.806 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:43:06.806 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:43:06.806 | INFO | High Water Breakdown: | 20260507 15:43:06.806 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:43:06.806 | INFO | ------------------------------------------ | 20260507 15:43:06.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:43:06.806 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:43:06.806 | INFO | Total mem usage: 15.7 GiB | 20260507 15:43:06.806 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:43:06.806 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:43:06.806 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:43:06.806 | INFO | High Water Breakdown: | 20260507 15:43:06.806 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:43:06.806 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:43:06.806 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:43:06.806 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:43:06.806 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:43:06.806 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:43:06.806 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:43:06.806 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:43:06.806 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:43:06.807 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:43:06.807 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:43:06.807 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:43:06.807 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:43:06.807 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:43:06.807 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:43:06.807 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:43:06.807 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:43:06.807 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:43:06.807 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:43:06.807 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:43:06.807 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:43:06.807 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:43:06.807 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:43:06.807 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:43:06.807 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:43:06.807 | INFO | ------------------------------------------ | 20260507 15:43:06.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:43:06.807 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:43:06.807 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:43:06.807 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:43:06.807 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 0.30%, AggregateStats -- 8.84e-02 / -nan, 5.57e-03 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 11.06%, Baseline -- 3.30e+00 / -nan, 2.34e-02 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 3.08%, Histogram -- 9.20e-01 / -nan, 5.29e-02 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 2.94e+00 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.14e+00 / -nan, 1.21e-01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 3.46%, Metrics -- 1.03e+00 / -nan, 2.99e-01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.80e-01 / -nan, 5.56e-02 / 0.00e+00 | 20260507 15:43:06.807 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:43:06.807 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:43:06.807 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:43:06.807 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 18.85%, Upload -- 1.68e+01 / 0.00e+00, 4.62e-01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 18.17%, Download -- 1.62e+01 / 0.00e+00, 8.36e-01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 33.45%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 0.57%, LockUpload -- 5.06e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 0.00%, LockDownload -- 5.27e-04 / 0.00e+00, 3.66e-04 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 28.95%, LockCompute -- 2.58e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:43:06.807 | INFO | PerfReport: 0.02%, Misc -- 1.66e-02 / 0.00e+00, 3.93e-02 / 0.00e+00 | 20260507 15:43:06.902 | INFO | Starting a round of Baz IO | 20260507 15:43:09.840 | INFO | Populating RTMetrics report | 20260507 15:43:09.840 | INFO | RTMetrics report populated | 20260507 15:43:09.840 | INFO | Opening tmp RTMetrics json file | 20260507 15:43:09.848 | INFO | Serializing RTMetrics to json | 20260507 15:43:09.870 | INFO | Serialization done, swapping tmp file now... | 20260507 15:43:09.902 | INFO | RTMetrics done | 20260507 15:43:09.957 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.054 seconds | 20260507 15:43:09.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.36% / 0.04% / 47.60% | 20260507 15:43:09.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.42212% / 7.15256e-05% / 3.57628e-05% | 20260507 15:43:09.957 | INFO | Baz IO overall: | 20260507 15:43:09.957 | INFO | BytesWritten : 274562760103 | 20260507 15:43:09.957 | INFO | NumEvents : 132468734011 | 20260507 15:43:11.944 | INFO | Analyzing chunk frames = [663552,664064) | 20260507 15:43:11.945 | INFO | MemInfo: { | 20260507 15:43:11.945 | INFO | "gpu" : " 17 GB", | 20260507 15:43:11.945 | INFO | "huge" : "", | 20260507 15:43:11.945 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:11.945 | INFO | "malloc" : " 14 GB", | 20260507 15:43:11.945 | INFO | "pinned" : " 29 GB" | 20260507 15:43:11.945 | INFO | } | 20260507 15:43:11.945 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:43:11.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:11.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:11.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:11.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:11.945 | INFO | ANALYSIS : 57.5%, 2.8, 84.7ms, 0 | 20260507 15:43:11.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:11.945 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260507 15:43:11.945 | INFO | PRE_HQ : 56.9%, 2.2, 63.8ms, 0 | 20260507 15:43:11.945 | INFO | BAZWRITER : 59.0%, 4.7, 141.5ms, 0 | 20260507 15:43:11.945 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 15:43:11.946 | INFO | Uploaded 36GiB to GPU in 1.65777s (21.7159GiB/s) | 20260507 15:43:11.946 | INFO | Downloaded 38GiB to GPU in 1.73861s (21.8566GiB/s) | 20260507 15:43:17.084 | INFO | Analyzing chunk frames = [664064,664576) | 20260507 15:43:17.084 | INFO | MemInfo: { | 20260507 15:43:17.084 | INFO | "gpu" : " 17 GB", | 20260507 15:43:17.084 | INFO | "huge" : "", | 20260507 15:43:17.084 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:17.084 | INFO | "malloc" : " 14 GB", | 20260507 15:43:17.084 | INFO | "pinned" : " 29 GB" | 20260507 15:43:17.084 | INFO | } | 20260507 15:43:17.084 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:43:17.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:17.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.084 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260507 15:43:17.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.084 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:43:17.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:17.085 | INFO | Uploaded 36GiB to GPU in 1.58591s (22.6999GiB/s) | 20260507 15:43:17.085 | INFO | Downloaded 34GiB to GPU in 1.53483s (22.1523GiB/s) | 20260507 15:43:22.223 | INFO | Analyzing chunk frames = [664576,665088) | 20260507 15:43:22.223 | INFO | MemInfo: { | 20260507 15:43:22.223 | INFO | "gpu" : " 17 GB", | 20260507 15:43:22.223 | INFO | "huge" : "", | 20260507 15:43:22.223 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:22.223 | INFO | "malloc" : " 14 GB", | 20260507 15:43:22.223 | INFO | "pinned" : " 29 GB" | 20260507 15:43:22.223 | INFO | } | 20260507 15:43:22.223 | INFO | Perf report for the last 5.13904 seconds: | 20260507 15:43:22.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:22.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.223 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:43:22.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.223 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:43:22.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:22.224 | INFO | Uploaded 36GiB to GPU in 1.58991s (22.6428GiB/s) | 20260507 15:43:22.224 | INFO | Downloaded 34GiB to GPU in 1.53934s (22.0873GiB/s) | 20260507 15:43:27.363 | INFO | Analyzing chunk frames = [665088,665600) | 20260507 15:43:27.363 | INFO | MemInfo: { | 20260507 15:43:27.363 | INFO | "gpu" : " 17 GB", | 20260507 15:43:27.363 | INFO | "huge" : "", | 20260507 15:43:27.363 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:27.363 | INFO | "malloc" : " 14 GB", | 20260507 15:43:27.363 | INFO | "pinned" : " 29 GB" | 20260507 15:43:27.363 | INFO | } | 20260507 15:43:27.363 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:43:27.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:27.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:43:27.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:43:27.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:27.363 | INFO | Uploaded 36GiB to GPU in 1.58855s (22.6621GiB/s) | 20260507 15:43:27.363 | INFO | Downloaded 34GiB to GPU in 1.53916s (22.09GiB/s) | 20260507 15:43:32.502 | INFO | Analyzing chunk frames = [665600,666112) | 20260507 15:43:32.502 | INFO | MemInfo: { | 20260507 15:43:32.502 | INFO | "gpu" : " 17 GB", | 20260507 15:43:32.502 | INFO | "huge" : "", | 20260507 15:43:32.502 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:32.502 | INFO | "malloc" : " 14 GB", | 20260507 15:43:32.502 | INFO | "pinned" : " 29 GB" | 20260507 15:43:32.502 | INFO | } | 20260507 15:43:32.502 | INFO | Perf report for the last 5.1392 seconds: | 20260507 15:43:32.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:32.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:43:32.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:43:32.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:32.502 | INFO | Uploaded 36GiB to GPU in 1.58812s (22.6684GiB/s) | 20260507 15:43:32.502 | INFO | Downloaded 34GiB to GPU in 1.53715s (22.1188GiB/s) | 20260507 15:43:37.641 | INFO | Analyzing chunk frames = [666112,666624) | 20260507 15:43:37.641 | INFO | MemInfo: { | 20260507 15:43:37.641 | INFO | "gpu" : " 17 GB", | 20260507 15:43:37.641 | INFO | "huge" : "", | 20260507 15:43:37.641 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:37.641 | INFO | "malloc" : " 14 GB", | 20260507 15:43:37.641 | INFO | "pinned" : " 29 GB" | 20260507 15:43:37.641 | INFO | } | 20260507 15:43:37.641 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:43:37.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:37.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.641 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 15:43:37.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.641 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:43:37.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:37.642 | INFO | Uploaded 36GiB to GPU in 1.58762s (22.6755GiB/s) | 20260507 15:43:37.642 | INFO | Downloaded 34GiB to GPU in 1.53751s (22.1137GiB/s) | 20260507 15:43:42.781 | INFO | Analyzing chunk frames = [666624,667136) | 20260507 15:43:42.781 | INFO | MemInfo: { | 20260507 15:43:42.781 | INFO | "gpu" : " 17 GB", | 20260507 15:43:42.781 | INFO | "huge" : "", | 20260507 15:43:42.781 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:42.781 | INFO | "malloc" : " 14 GB", | 20260507 15:43:42.781 | INFO | "pinned" : " 29 GB" | 20260507 15:43:42.781 | INFO | } | 20260507 15:43:42.781 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:43:42.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:42.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:43:42.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:43:42.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:42.781 | INFO | Uploaded 36GiB to GPU in 1.58765s (22.6751GiB/s) | 20260507 15:43:42.781 | INFO | Downloaded 34GiB to GPU in 1.53752s (22.1136GiB/s) | 20260507 15:43:47.920 | INFO | Analyzing chunk frames = [667136,667648) | 20260507 15:43:47.920 | INFO | MemInfo: { | 20260507 15:43:47.920 | INFO | "gpu" : " 17 GB", | 20260507 15:43:47.920 | INFO | "huge" : "", | 20260507 15:43:47.920 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:47.920 | INFO | "malloc" : " 14 GB", | 20260507 15:43:47.920 | INFO | "pinned" : " 29 GB" | 20260507 15:43:47.920 | INFO | } | 20260507 15:43:47.920 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:43:47.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:47.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.920 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:43:47.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.920 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260507 15:43:47.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:47.921 | INFO | Uploaded 36GiB to GPU in 1.59217s (22.6106GiB/s) | 20260507 15:43:47.921 | INFO | Downloaded 34GiB to GPU in 1.54072s (22.0677GiB/s) | 20260507 15:43:48.017 | INFO | Starting a round of Baz IO | 20260507 15:43:50.976 | INFO | Populating RTMetrics report | 20260507 15:43:50.977 | INFO | RTMetrics report populated | 20260507 15:43:50.977 | INFO | Opening tmp RTMetrics json file | 20260507 15:43:50.983 | INFO | Serializing RTMetrics to json | 20260507 15:43:50.988 | INFO | Serialization done, swapping tmp file now... | 20260507 15:43:50.992 | INFO | RTMetrics done | 20260507 15:43:51.288 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.27 seconds | 20260507 15:43:51.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.33% / 0.04% / 47.64% | 20260507 15:43:51.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.41418% / 7.54992e-05% / 2.78155e-05% | 20260507 15:43:51.288 | INFO | Baz IO overall: | 20260507 15:43:51.288 | INFO | BytesWritten : 276456158542 | 20260507 15:43:51.288 | INFO | NumEvents : 133390940142 | 20260507 15:43:53.059 | INFO | Analyzing chunk frames = [667648,668160) | 20260507 15:43:53.059 | INFO | MemInfo: { | 20260507 15:43:53.059 | INFO | "gpu" : " 17 GB", | 20260507 15:43:53.059 | INFO | "huge" : "", | 20260507 15:43:53.059 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:53.059 | INFO | "malloc" : " 14 GB", | 20260507 15:43:53.060 | INFO | "pinned" : " 29 GB" | 20260507 15:43:53.060 | INFO | } | 20260507 15:43:53.060 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:43:53.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:53.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:53.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:53.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:53.060 | INFO | ANALYSIS : 57.9%, 2.8, 83.8ms, 0 | 20260507 15:43:53.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:53.060 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260507 15:43:53.060 | INFO | PRE_HQ : 57.3%, 2.2, 64.0ms, 0 | 20260507 15:43:53.060 | INFO | BAZWRITER : 63.3%, 4.8, 155.3ms, 0 | 20260507 15:43:53.060 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 15:43:53.060 | INFO | Uploaded 36GiB to GPU in 1.64757s (21.8503GiB/s) | 20260507 15:43:53.060 | INFO | Downloaded 38GiB to GPU in 1.73632s (21.8853GiB/s) | 20260507 15:43:58.199 | INFO | Analyzing chunk frames = [668160,668672) | 20260507 15:43:58.199 | INFO | MemInfo: { | 20260507 15:43:58.199 | INFO | "gpu" : " 17 GB", | 20260507 15:43:58.199 | INFO | "huge" : "", | 20260507 15:43:58.199 | INFO | "huge_shared" : " 29 GB", | 20260507 15:43:58.199 | INFO | "malloc" : " 14 GB", | 20260507 15:43:58.199 | INFO | "pinned" : " 29 GB" | 20260507 15:43:58.199 | INFO | } | 20260507 15:43:58.199 | INFO | Perf report for the last 5.13903 seconds: | 20260507 15:43:58.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:43:58.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:43:58.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | PRE_HQ : 27.6%, 1.0, 14.5ms, 0 | 20260507 15:43:58.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:43:58.199 | INFO | Uploaded 36GiB to GPU in 1.60446s (22.4375GiB/s) | 20260507 15:43:58.199 | INFO | Downloaded 34GiB to GPU in 1.52478s (22.2983GiB/s) | 20260507 15:44:03.338 | INFO | Analyzing chunk frames = [668672,669184) | 20260507 15:44:03.338 | INFO | MemInfo: { | 20260507 15:44:03.338 | INFO | "gpu" : " 17 GB", | 20260507 15:44:03.338 | INFO | "huge" : "", | 20260507 15:44:03.338 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:03.338 | INFO | "malloc" : " 14 GB", | 20260507 15:44:03.338 | INFO | "pinned" : " 29 GB" | 20260507 15:44:03.338 | INFO | } | 20260507 15:44:03.338 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:44:03.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:03.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.338 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:44:03.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.338 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:44:03.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:03.339 | INFO | Uploaded 36GiB to GPU in 1.61002s (22.36GiB/s) | 20260507 15:44:03.339 | INFO | Downloaded 34GiB to GPU in 1.53023s (22.2188GiB/s) | 20260507 15:44:08.477 | INFO | Analyzing chunk frames = [669184,669696) | 20260507 15:44:08.477 | INFO | MemInfo: { | 20260507 15:44:08.477 | INFO | "gpu" : " 17 GB", | 20260507 15:44:08.477 | INFO | "huge" : "", | 20260507 15:44:08.477 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:08.477 | INFO | "malloc" : " 14 GB", | 20260507 15:44:08.477 | INFO | "pinned" : " 29 GB" | 20260507 15:44:08.477 | INFO | } | 20260507 15:44:08.477 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:44:08.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:08.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.477 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:44:08.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.477 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:44:08.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:08.478 | INFO | Uploaded 36GiB to GPU in 1.60781s (22.3907GiB/s) | 20260507 15:44:08.478 | INFO | Downloaded 34GiB to GPU in 1.52981s (22.2249GiB/s) | 20260507 15:44:13.616 | INFO | Analyzing chunk frames = [669696,670208) | 20260507 15:44:13.616 | INFO | MemInfo: { | 20260507 15:44:13.616 | INFO | "gpu" : " 17 GB", | 20260507 15:44:13.616 | INFO | "huge" : "", | 20260507 15:44:13.617 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:13.617 | INFO | "malloc" : " 14 GB", | 20260507 15:44:13.617 | INFO | "pinned" : " 29 GB" | 20260507 15:44:13.617 | INFO | } | 20260507 15:44:13.617 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:44:13.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:13.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:44:13.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:44:13.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:13.617 | INFO | Uploaded 36GiB to GPU in 1.6075s (22.395GiB/s) | 20260507 15:44:13.617 | INFO | Downloaded 34GiB to GPU in 1.527s (22.2659GiB/s) | 20260507 15:44:18.756 | INFO | Analyzing chunk frames = [670208,670720) | 20260507 15:44:18.756 | INFO | MemInfo: { | 20260507 15:44:18.756 | INFO | "gpu" : " 17 GB", | 20260507 15:44:18.756 | INFO | "huge" : "", | 20260507 15:44:18.756 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:18.756 | INFO | "malloc" : " 14 GB", | 20260507 15:44:18.756 | INFO | "pinned" : " 29 GB" | 20260507 15:44:18.756 | INFO | } | 20260507 15:44:18.756 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:44:18.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:18.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:44:18.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:44:18.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:18.756 | INFO | Uploaded 36GiB to GPU in 1.60837s (22.3828GiB/s) | 20260507 15:44:18.757 | INFO | Downloaded 34GiB to GPU in 1.52829s (22.247GiB/s) | 20260507 15:44:23.895 | INFO | Analyzing chunk frames = [670720,671232) | 20260507 15:44:23.895 | INFO | MemInfo: { | 20260507 15:44:23.895 | INFO | "gpu" : " 17 GB", | 20260507 15:44:23.895 | INFO | "huge" : "", | 20260507 15:44:23.895 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:23.895 | INFO | "malloc" : " 14 GB", | 20260507 15:44:23.895 | INFO | "pinned" : " 29 GB" | 20260507 15:44:23.895 | INFO | } | 20260507 15:44:23.895 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:44:23.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:23.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.895 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:44:23.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.896 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:44:23.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:23.896 | INFO | Uploaded 36GiB to GPU in 1.60755s (22.3943GiB/s) | 20260507 15:44:23.896 | INFO | Downloaded 34GiB to GPU in 1.52848s (22.2444GiB/s) | 20260507 15:44:29.035 | INFO | Analyzing chunk frames = [671232,671744) | 20260507 15:44:29.035 | INFO | MemInfo: { | 20260507 15:44:29.035 | INFO | "gpu" : " 17 GB", | 20260507 15:44:29.035 | INFO | "huge" : "", | 20260507 15:44:29.035 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:29.035 | INFO | "malloc" : " 14 GB", | 20260507 15:44:29.035 | INFO | "pinned" : " 29 GB" | 20260507 15:44:29.035 | INFO | } | 20260507 15:44:29.035 | INFO | Perf report for the last 5.13976 seconds: | 20260507 15:44:29.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:29.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.035 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:44:29.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.035 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 15:44:29.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:29.036 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:44:29.036 | INFO | Total mem usage: 16.4 GiB | 20260507 15:44:29.036 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:44:29.036 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.11%) | 20260507 15:44:29.036 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:44:29.036 | INFO | High Water Breakdown: | 20260507 15:44:29.036 | INFO | 59.7% : 9.14 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:44:29.036 | INFO | 40.2% : 6.15 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:44:29.036 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:44:29.036 | INFO | ------------------------------------------ | 20260507 15:44:29.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:44:29.036 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:44:29.036 | INFO | Total mem usage: 26.5 GiB | 20260507 15:44:29.036 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:44:29.036 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:44:29.036 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:44:29.036 | INFO | High Water Breakdown: | 20260507 15:44:29.036 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:44:29.036 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:44:29.036 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:44:29.036 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:44:29.036 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:44:29.036 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:44:29.036 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:44:29.036 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:44:29.036 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:44:29.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:44:29.036 | INFO | Uploaded 36GiB to GPU in 1.6079s (22.3894GiB/s) | 20260507 15:44:29.036 | INFO | Downloaded 34GiB to GPU in 1.52913s (22.2348GiB/s) | 20260507 15:44:29.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:44:29.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:44:29.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:44:29.036 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:44:29.036 | INFO | ------------------------------------------ | 20260507 15:44:29.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:44:29.036 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:44:29.036 | INFO | Total mem usage: 27.8 GiB | 20260507 15:44:29.036 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:44:29.036 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:44:29.036 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:44:29.036 | INFO | High Water Breakdown: | 20260507 15:44:29.036 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:44:29.036 | INFO | ------------------------------------------ | 20260507 15:44:29.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:44:29.036 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:44:29.036 | INFO | Total mem usage: 15.7 GiB | 20260507 15:44:29.036 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:44:29.036 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:44:29.036 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:44:29.036 | INFO | High Water Breakdown: | 20260507 15:44:29.036 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:44:29.036 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:44:29.036 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:44:29.036 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:44:29.036 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:44:29.036 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:44:29.036 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:44:29.036 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:44:29.036 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:44:29.036 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:44:29.036 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:44:29.036 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:44:29.036 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:44:29.036 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:44:29.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:44:29.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:44:29.036 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:44:29.036 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:44:29.036 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:44:29.036 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:44:29.036 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:44:29.036 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:44:29.036 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:44:29.036 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:44:29.036 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:44:29.036 | INFO | ------------------------------------------ | 20260507 15:44:29.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:44:29.036 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:44:29.036 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:44:29.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:44:29.036 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 5.23e-03 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 11.04%, Baseline -- 3.29e+00 / -nan, 1.94e-02 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 2.93e+00 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 3.43%, Metrics -- 1.02e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.75e-01 / -nan, 5.49e-02 / 0.00e+00 | 20260507 15:44:29.036 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:44:29.036 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:44:29.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:44:29.036 | INFO | PerfReport: Full Execution -- 8.90e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:44:29.036 | INFO | PerfReport: 18.78%, Upload -- 1.67e+01 / 0.00e+00, 4.79e-01 / 0.00e+00 | 20260507 15:44:29.037 | INFO | PerfReport: 18.24%, Download -- 1.62e+01 / 0.00e+00, 8.15e-01 / 0.00e+00 | 20260507 15:44:29.037 | INFO | PerfReport: 33.51%, Compute -- 2.98e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 15:44:29.037 | INFO | PerfReport: 0.59%, LockUpload -- 5.27e-01 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 15:44:29.037 | INFO | PerfReport: 0.00%, LockDownload -- 4.49e-04 / 0.00e+00, 1.80e-04 / 0.00e+00 | 20260507 15:44:29.037 | INFO | PerfReport: 28.85%, LockCompute -- 2.57e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:44:29.037 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 15:44:29.133 | INFO | Starting a round of Baz IO | 20260507 15:44:32.072 | INFO | Populating RTMetrics report | 20260507 15:44:32.072 | INFO | RTMetrics report populated | 20260507 15:44:32.072 | INFO | Opening tmp RTMetrics json file | 20260507 15:44:32.096 | INFO | Serializing RTMetrics to json | 20260507 15:44:32.101 | INFO | Serialization done, swapping tmp file now... | 20260507 15:44:32.105 | INFO | RTMetrics done | 20260507 15:44:32.299 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.165 seconds | 20260507 15:44:32.299 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.29% / 0.04% / 47.67% | 20260507 15:44:32.299 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.37678% / 6.35783e-05% / 1.98682e-05% | 20260507 15:44:32.299 | INFO | Baz IO overall: | 20260507 15:44:32.299 | INFO | BytesWritten : 278347570880 | 20260507 15:44:32.299 | INFO | NumEvents : 134311454556 | 20260507 15:44:34.174 | INFO | Analyzing chunk frames = [671744,672256) | 20260507 15:44:34.174 | INFO | MemInfo: { | 20260507 15:44:34.174 | INFO | "gpu" : " 17 GB", | 20260507 15:44:34.174 | INFO | "huge" : "", | 20260507 15:44:34.174 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:34.174 | INFO | "malloc" : " 14 GB", | 20260507 15:44:34.174 | INFO | "pinned" : " 29 GB" | 20260507 15:44:34.174 | INFO | } | 20260507 15:44:34.174 | INFO | Perf report for the last 5.13897 seconds: | 20260507 15:44:34.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:34.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:34.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:34.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:34.174 | INFO | ANALYSIS : 58.2%, 2.8, 84.4ms, 0 | 20260507 15:44:34.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:34.174 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 15:44:34.174 | INFO | PRE_HQ : 57.2%, 2.1, 63.4ms, 0 | 20260507 15:44:34.175 | INFO | BAZWRITER : 61.3%, 4.8, 150.2ms, 0 | 20260507 15:44:34.175 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:44:34.175 | INFO | Uploaded 36GiB to GPU in 1.65111s (21.8035GiB/s) | 20260507 15:44:34.175 | INFO | Downloaded 38GiB to GPU in 1.73258s (21.9326GiB/s) | 20260507 15:44:39.313 | INFO | Analyzing chunk frames = [672256,672768) | 20260507 15:44:39.313 | INFO | MemInfo: { | 20260507 15:44:39.313 | INFO | "gpu" : " 17 GB", | 20260507 15:44:39.313 | INFO | "huge" : "", | 20260507 15:44:39.313 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:39.313 | INFO | "malloc" : " 14 GB", | 20260507 15:44:39.313 | INFO | "pinned" : " 29 GB" | 20260507 15:44:39.313 | INFO | } | 20260507 15:44:39.314 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:44:39.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:39.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:44:39.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | PRE_HQ : 38.7%, 1.0, 20.2ms, 0 | 20260507 15:44:39.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:39.314 | INFO | Uploaded 36GiB to GPU in 1.59439s (22.5791GiB/s) | 20260507 15:44:39.314 | INFO | Downloaded 34GiB to GPU in 1.53771s (22.1108GiB/s) | 20260507 15:44:44.453 | INFO | Analyzing chunk frames = [672768,673280) | 20260507 15:44:44.453 | INFO | MemInfo: { | 20260507 15:44:44.453 | INFO | "gpu" : " 17 GB", | 20260507 15:44:44.453 | INFO | "huge" : "", | 20260507 15:44:44.453 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:44.453 | INFO | "malloc" : " 14 GB", | 20260507 15:44:44.453 | INFO | "pinned" : " 29 GB" | 20260507 15:44:44.453 | INFO | } | 20260507 15:44:44.453 | INFO | Perf report for the last 5.13978 seconds: | 20260507 15:44:44.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:44.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.453 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:44:44.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.454 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:44:44.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:44.454 | INFO | Uploaded 36GiB to GPU in 1.61568s (22.2817GiB/s) | 20260507 15:44:44.454 | INFO | Downloaded 34GiB to GPU in 1.53527s (22.1459GiB/s) | 20260507 15:44:49.592 | INFO | Analyzing chunk frames = [673280,673792) | 20260507 15:44:49.592 | INFO | MemInfo: { | 20260507 15:44:49.592 | INFO | "gpu" : " 17 GB", | 20260507 15:44:49.592 | INFO | "huge" : "", | 20260507 15:44:49.592 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:49.592 | INFO | "malloc" : " 14 GB", | 20260507 15:44:49.592 | INFO | "pinned" : " 29 GB" | 20260507 15:44:49.592 | INFO | } | 20260507 15:44:49.593 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:44:49.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:49.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:44:49.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 15:44:49.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:49.593 | INFO | Uploaded 36GiB to GPU in 1.61601s (22.2771GiB/s) | 20260507 15:44:49.593 | INFO | Downloaded 34GiB to GPU in 1.53533s (22.1451GiB/s) | 20260507 15:44:54.732 | INFO | Analyzing chunk frames = [673792,674304) | 20260507 15:44:54.732 | INFO | MemInfo: { | 20260507 15:44:54.732 | INFO | "gpu" : " 17 GB", | 20260507 15:44:54.732 | INFO | "huge" : "", | 20260507 15:44:54.732 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:54.732 | INFO | "malloc" : " 14 GB", | 20260507 15:44:54.732 | INFO | "pinned" : " 29 GB" | 20260507 15:44:54.732 | INFO | } | 20260507 15:44:54.732 | INFO | Perf report for the last 5.13917 seconds: | 20260507 15:44:54.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:54.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 15:44:54.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | PRE_HQ : 39.5%, 1.0, 20.5ms, 0 | 20260507 15:44:54.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:54.732 | INFO | Uploaded 36GiB to GPU in 1.61305s (22.3179GiB/s) | 20260507 15:44:54.732 | INFO | Downloaded 34GiB to GPU in 1.53366s (22.1692GiB/s) | 20260507 15:44:59.871 | INFO | Analyzing chunk frames = [674304,674816) | 20260507 15:44:59.871 | INFO | MemInfo: { | 20260507 15:44:59.871 | INFO | "gpu" : " 17 GB", | 20260507 15:44:59.871 | INFO | "huge" : "", | 20260507 15:44:59.871 | INFO | "huge_shared" : " 29 GB", | 20260507 15:44:59.871 | INFO | "malloc" : " 14 GB", | 20260507 15:44:59.871 | INFO | "pinned" : " 29 GB" | 20260507 15:44:59.871 | INFO | } | 20260507 15:44:59.871 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:44:59.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:44:59.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:44:59.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:44:59.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:44:59.871 | INFO | Uploaded 36GiB to GPU in 1.60784s (22.3903GiB/s) | 20260507 15:44:59.871 | INFO | Downloaded 34GiB to GPU in 1.52737s (22.2605GiB/s) | 20260507 15:45:05.010 | INFO | Analyzing chunk frames = [674816,675328) | 20260507 15:45:05.010 | INFO | MemInfo: { | 20260507 15:45:05.010 | INFO | "gpu" : " 17 GB", | 20260507 15:45:05.010 | INFO | "huge" : "", | 20260507 15:45:05.010 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:05.010 | INFO | "malloc" : " 14 GB", | 20260507 15:45:05.010 | INFO | "pinned" : " 29 GB" | 20260507 15:45:05.010 | INFO | } | 20260507 15:45:05.010 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:45:05.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:05.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:45:05.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:45:05.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:05.010 | INFO | Uploaded 36GiB to GPU in 1.60826s (22.3844GiB/s) | 20260507 15:45:05.010 | INFO | Downloaded 34GiB to GPU in 1.52817s (22.2488GiB/s) | 20260507 15:45:10.149 | INFO | Analyzing chunk frames = [675328,675840) | 20260507 15:45:10.149 | INFO | MemInfo: { | 20260507 15:45:10.149 | INFO | "gpu" : " 17 GB", | 20260507 15:45:10.149 | INFO | "huge" : "", | 20260507 15:45:10.149 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:10.149 | INFO | "malloc" : " 14 GB", | 20260507 15:45:10.149 | INFO | "pinned" : " 29 GB" | 20260507 15:45:10.149 | INFO | } | 20260507 15:45:10.150 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:45:10.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:10.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:45:10.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:45:10.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:10.150 | INFO | Uploaded 36GiB to GPU in 1.60911s (22.3727GiB/s) | 20260507 15:45:10.150 | INFO | Downloaded 34GiB to GPU in 1.52976s (22.2257GiB/s) | 20260507 15:45:10.247 | INFO | Starting a round of Baz IO | 20260507 15:45:13.195 | INFO | Populating RTMetrics report | 20260507 15:45:13.196 | INFO | RTMetrics report populated | 20260507 15:45:13.196 | INFO | Opening tmp RTMetrics json file | 20260507 15:45:13.212 | INFO | Serializing RTMetrics to json | 20260507 15:45:13.224 | INFO | Serialization done, swapping tmp file now... | 20260507 15:45:13.225 | INFO | RTMetrics done | 20260507 15:45:13.332 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.085 seconds | 20260507 15:45:13.332 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.25% / 0.04% / 47.71% | 20260507 15:45:13.332 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.36323% / 0.000107288% / 3.97364e-05% | 20260507 15:45:13.332 | INFO | Baz IO overall: | 20260507 15:45:13.332 | INFO | BytesWritten : 280239717668 | 20260507 15:45:13.332 | INFO | NumEvents : 135232483485 | 20260507 15:45:15.289 | INFO | Analyzing chunk frames = [675840,676352) | 20260507 15:45:15.289 | INFO | MemInfo: { | 20260507 15:45:15.289 | INFO | "gpu" : " 17 GB", | 20260507 15:45:15.289 | INFO | "huge" : "", | 20260507 15:45:15.289 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:15.289 | INFO | "malloc" : " 14 GB", | 20260507 15:45:15.289 | INFO | "pinned" : " 29 GB" | 20260507 15:45:15.289 | INFO | } | 20260507 15:45:15.289 | INFO | Perf report for the last 5.13962 seconds: | 20260507 15:45:15.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:15.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:15.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:15.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:15.289 | INFO | ANALYSIS : 57.6%, 2.8, 84.9ms, 0 | 20260507 15:45:15.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:15.289 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 15:45:15.289 | INFO | PRE_HQ : 57.2%, 2.1, 62.2ms, 0 | 20260507 15:45:15.289 | INFO | BAZWRITER : 59.8%, 4.7, 145.7ms, 0 | 20260507 15:45:15.289 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 15:45:15.290 | INFO | Uploaded 36GiB to GPU in 1.65673s (21.7295GiB/s) | 20260507 15:45:15.290 | INFO | Downloaded 38GiB to GPU in 1.74088s (21.8281GiB/s) | 20260507 15:45:20.428 | INFO | Analyzing chunk frames = [676352,676864) | 20260507 15:45:20.428 | INFO | MemInfo: { | 20260507 15:45:20.428 | INFO | "gpu" : " 17 GB", | 20260507 15:45:20.429 | INFO | "huge" : "", | 20260507 15:45:20.429 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:20.429 | INFO | "malloc" : " 14 GB", | 20260507 15:45:20.429 | INFO | "pinned" : " 29 GB" | 20260507 15:45:20.429 | INFO | } | 20260507 15:45:20.429 | INFO | Perf report for the last 5.13951 seconds: | 20260507 15:45:20.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:20.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:45:20.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | PRE_HQ : 39.4%, 1.0, 20.5ms, 0 | 20260507 15:45:20.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:20.429 | INFO | Uploaded 36GiB to GPU in 1.5914s (22.6216GiB/s) | 20260507 15:45:20.429 | INFO | Downloaded 34GiB to GPU in 1.53913s (22.0903GiB/s) | 20260507 15:45:25.568 | INFO | Analyzing chunk frames = [676864,677376) | 20260507 15:45:25.568 | INFO | MemInfo: { | 20260507 15:45:25.568 | INFO | "gpu" : " 17 GB", | 20260507 15:45:25.568 | INFO | "huge" : "", | 20260507 15:45:25.568 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:25.568 | INFO | "malloc" : " 14 GB", | 20260507 15:45:25.568 | INFO | "pinned" : " 29 GB" | 20260507 15:45:25.568 | INFO | } | 20260507 15:45:25.568 | INFO | Perf report for the last 5.13956 seconds: | 20260507 15:45:25.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:25.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.568 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 15:45:25.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.569 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:45:25.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:25.569 | INFO | Uploaded 36GiB to GPU in 1.59495s (22.5713GiB/s) | 20260507 15:45:25.569 | INFO | Downloaded 34GiB to GPU in 1.54488s (22.0082GiB/s) | 20260507 15:45:30.707 | INFO | Analyzing chunk frames = [677376,677888) | 20260507 15:45:30.707 | INFO | MemInfo: { | 20260507 15:45:30.707 | INFO | "gpu" : " 17 GB", | 20260507 15:45:30.707 | INFO | "huge" : "", | 20260507 15:45:30.707 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:30.707 | INFO | "malloc" : " 14 GB", | 20260507 15:45:30.707 | INFO | "pinned" : " 29 GB" | 20260507 15:45:30.707 | INFO | } | 20260507 15:45:30.707 | INFO | Perf report for the last 5.13901 seconds: | 20260507 15:45:30.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:30.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.707 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:45:30.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.708 | INFO | PRE_HQ : 38.8%, 1.0, 20.2ms, 0 | 20260507 15:45:30.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:30.708 | INFO | Uploaded 36GiB to GPU in 1.59399s (22.5848GiB/s) | 20260507 15:45:30.708 | INFO | Downloaded 34GiB to GPU in 1.54366s (22.0256GiB/s) | 20260507 15:45:35.846 | INFO | Analyzing chunk frames = [677888,678400) | 20260507 15:45:35.846 | INFO | MemInfo: { | 20260507 15:45:35.846 | INFO | "gpu" : " 17 GB", | 20260507 15:45:35.846 | INFO | "huge" : "", | 20260507 15:45:35.846 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:35.846 | INFO | "malloc" : " 14 GB", | 20260507 15:45:35.846 | INFO | "pinned" : " 29 GB" | 20260507 15:45:35.846 | INFO | } | 20260507 15:45:35.846 | INFO | Perf report for the last 5.13907 seconds: | 20260507 15:45:35.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:35.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.846 | INFO | ANALYSIS : 55.9%, 3.0, 87.3ms, 0 | 20260507 15:45:35.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.847 | INFO | PRE_HQ : 35.5%, 1.0, 18.5ms, 0 | 20260507 15:45:35.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:35.847 | INFO | Uploaded 36GiB to GPU in 1.59163s (22.6183GiB/s) | 20260507 15:45:35.847 | INFO | Downloaded 34GiB to GPU in 1.54087s (22.0655GiB/s) | 20260507 15:45:40.985 | INFO | Analyzing chunk frames = [678400,678912) | 20260507 15:45:40.985 | INFO | MemInfo: { | 20260507 15:45:40.985 | INFO | "gpu" : " 17 GB", | 20260507 15:45:40.986 | INFO | "huge" : "", | 20260507 15:45:40.986 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:40.986 | INFO | "malloc" : " 14 GB", | 20260507 15:45:40.986 | INFO | "pinned" : " 29 GB" | 20260507 15:45:40.986 | INFO | } | 20260507 15:45:40.986 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:45:40.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:40.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:45:40.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:45:40.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:40.986 | INFO | Uploaded 36GiB to GPU in 1.58807s (22.669GiB/s) | 20260507 15:45:40.986 | INFO | Downloaded 34GiB to GPU in 1.53639s (22.1299GiB/s) | 20260507 15:45:46.125 | INFO | Analyzing chunk frames = [678912,679424) | 20260507 15:45:46.125 | INFO | MemInfo: { | 20260507 15:45:46.125 | INFO | "gpu" : " 17 GB", | 20260507 15:45:46.125 | INFO | "huge" : "", | 20260507 15:45:46.125 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:46.125 | INFO | "malloc" : " 14 GB", | 20260507 15:45:46.125 | INFO | "pinned" : " 29 GB" | 20260507 15:45:46.125 | INFO | } | 20260507 15:45:46.125 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:45:46.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:46.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.125 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:45:46.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.125 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:45:46.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:46.126 | INFO | Uploaded 36GiB to GPU in 1.58799s (22.6702GiB/s) | 20260507 15:45:46.126 | INFO | Downloaded 34GiB to GPU in 1.53846s (22.1GiB/s) | 20260507 15:45:51.264 | INFO | Analyzing chunk frames = [679424,679936) | 20260507 15:45:51.264 | INFO | MemInfo: { | 20260507 15:45:51.264 | INFO | "gpu" : " 17 GB", | 20260507 15:45:51.264 | INFO | "huge" : "", | 20260507 15:45:51.264 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:51.264 | INFO | "malloc" : " 14 GB", | 20260507 15:45:51.264 | INFO | "pinned" : " 29 GB" | 20260507 15:45:51.264 | INFO | } | 20260507 15:45:51.264 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:45:51.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:51.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.265 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:45:51.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.265 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 15:45:51.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:51.265 | INFO | Uploaded 36GiB to GPU in 1.58844s (22.6637GiB/s) | 20260507 15:45:51.265 | INFO | Downloaded 34GiB to GPU in 1.53833s (22.1018GiB/s) | 20260507 15:45:51.265 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:45:51.265 | INFO | Total mem usage: 16.4 GiB | 20260507 15:45:51.265 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:45:51.265 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8.25%) | 20260507 15:45:51.265 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:45:51.265 | INFO | High Water Breakdown: | 20260507 15:45:51.266 | INFO | 59.5% : 9.02 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:45:51.266 | INFO | 40.4% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:45:51.266 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:45:51.266 | INFO | ------------------------------------------ | 20260507 15:45:51.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:45:51.266 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:45:51.266 | INFO | Total mem usage: 26.5 GiB | 20260507 15:45:51.266 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:45:51.266 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 15:45:51.266 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:45:51.266 | INFO | High Water Breakdown: | 20260507 15:45:51.266 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:45:51.266 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:45:51.266 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:45:51.266 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:45:51.266 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:45:51.266 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:45:51.266 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:45:51.266 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:45:51.266 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:45:51.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:45:51.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:45:51.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:45:51.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:45:51.266 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:45:51.266 | INFO | ------------------------------------------ | 20260507 15:45:51.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:45:51.266 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:45:51.266 | INFO | Total mem usage: 27.8 GiB | 20260507 15:45:51.266 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:45:51.266 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:45:51.266 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:45:51.266 | INFO | High Water Breakdown: | 20260507 15:45:51.266 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:45:51.266 | INFO | ------------------------------------------ | 20260507 15:45:51.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:45:51.266 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:45:51.266 | INFO | Total mem usage: 15.7 GiB | 20260507 15:45:51.266 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:45:51.266 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:45:51.266 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:45:51.266 | INFO | High Water Breakdown: | 20260507 15:45:51.266 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:45:51.266 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:45:51.266 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:45:51.266 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:45:51.266 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:45:51.266 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:45:51.266 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:45:51.266 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:45:51.266 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:45:51.266 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:45:51.266 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:45:51.266 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:45:51.266 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:45:51.266 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:45:51.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:45:51.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:45:51.266 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:45:51.266 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:45:51.266 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:45:51.266 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:45:51.266 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:45:51.266 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:45:51.266 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:45:51.266 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:45:51.266 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:45:51.266 | INFO | ------------------------------------------ | 20260507 15:45:51.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:45:51.266 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:45:51.266 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:45:51.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:45:51.266 | INFO | PerfReport: Full Execution -- 2.99e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 0.30%, AggregateStats -- 9.03e-02 / -nan, 7.71e-03 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 11.07%, Baseline -- 3.31e+00 / -nan, 2.19e-02 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / -nan, 5.31e-02 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.01e+00 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.14e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 3.44%, Metrics -- 1.03e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.81e-01 / -nan, 5.56e-02 / 0.00e+00 | 20260507 15:45:51.267 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:45:51.267 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:45:51.267 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:45:51.267 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 18.77%, Upload -- 1.67e+01 / 0.00e+00, 4.82e-01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 18.24%, Download -- 1.63e+01 / 0.00e+00, 8.00e-01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 33.49%, Compute -- 2.99e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 0.00%, LockDownload -- 5.60e-04 / 0.00e+00, 4.16e-04 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 28.91%, LockCompute -- 2.58e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:45:51.267 | INFO | PerfReport: 0.02%, Misc -- 1.67e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260507 15:45:51.362 | INFO | Starting a round of Baz IO | 20260507 15:45:54.261 | INFO | Populating RTMetrics report | 20260507 15:45:54.261 | INFO | RTMetrics report populated | 20260507 15:45:54.261 | INFO | Opening tmp RTMetrics json file | 20260507 15:45:54.266 | INFO | Serializing RTMetrics to json | 20260507 15:45:54.277 | INFO | Serialization done, swapping tmp file now... | 20260507 15:45:54.292 | INFO | RTMetrics done | 20260507 15:45:54.409 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.047 seconds | 20260507 15:45:54.410 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.21% / 0.04% / 47.75% | 20260507 15:45:54.410 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.34741% / 0.000115236% / 4.37101e-05% | 20260507 15:45:54.410 | INFO | Baz IO overall: | 20260507 15:45:54.410 | INFO | BytesWritten : 282132774312 | 20260507 15:45:54.410 | INFO | NumEvents : 136154021575 | 20260507 15:45:56.404 | INFO | Analyzing chunk frames = [679936,680448) | 20260507 15:45:56.404 | INFO | MemInfo: { | 20260507 15:45:56.404 | INFO | "gpu" : " 17 GB", | 20260507 15:45:56.404 | INFO | "huge" : "", | 20260507 15:45:56.404 | INFO | "huge_shared" : " 29 GB", | 20260507 15:45:56.404 | INFO | "malloc" : " 14 GB", | 20260507 15:45:56.404 | INFO | "pinned" : " 29 GB" | 20260507 15:45:56.404 | INFO | } | 20260507 15:45:56.404 | INFO | Perf report for the last 5.13972 seconds: | 20260507 15:45:56.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:45:56.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:56.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:56.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:56.404 | INFO | ANALYSIS : 57.3%, 2.9, 87.3ms, 0 | 20260507 15:45:56.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:45:56.404 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 15:45:56.405 | INFO | PRE_HQ : 56.4%, 2.1, 61.7ms, 0 | 20260507 15:45:56.405 | INFO | BAZWRITER : 58.8%, 4.4, 132.9ms, 0 | 20260507 15:45:56.405 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:45:56.405 | INFO | Uploaded 36GiB to GPU in 1.63169s (22.0631GiB/s) | 20260507 15:45:56.405 | INFO | Downloaded 38GiB to GPU in 1.71693s (22.1325GiB/s) | 20260507 15:46:01.543 | INFO | Analyzing chunk frames = [680448,680960) | 20260507 15:46:01.543 | INFO | MemInfo: { | 20260507 15:46:01.543 | INFO | "gpu" : " 17 GB", | 20260507 15:46:01.543 | INFO | "huge" : "", | 20260507 15:46:01.543 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:01.543 | INFO | "malloc" : " 14 GB", | 20260507 15:46:01.543 | INFO | "pinned" : " 29 GB" | 20260507 15:46:01.543 | INFO | } | 20260507 15:46:01.543 | INFO | Perf report for the last 5.13895 seconds: | 20260507 15:46:01.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:01.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 15:46:01.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 15:46:01.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:01.543 | INFO | Uploaded 36GiB to GPU in 1.60687s (22.4038GiB/s) | 20260507 15:46:01.543 | INFO | Downloaded 34GiB to GPU in 1.52627s (22.2765GiB/s) | 20260507 15:46:06.682 | INFO | Analyzing chunk frames = [680960,681472) | 20260507 15:46:06.682 | INFO | MemInfo: { | 20260507 15:46:06.682 | INFO | "gpu" : " 17 GB", | 20260507 15:46:06.682 | INFO | "huge" : "", | 20260507 15:46:06.682 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:06.682 | INFO | "malloc" : " 14 GB", | 20260507 15:46:06.682 | INFO | "pinned" : " 29 GB" | 20260507 15:46:06.682 | INFO | } | 20260507 15:46:06.682 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:46:06.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:06.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.682 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:46:06.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.683 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:46:06.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:06.683 | INFO | Uploaded 36GiB to GPU in 1.60949s (22.3673GiB/s) | 20260507 15:46:06.683 | INFO | Downloaded 34GiB to GPU in 1.53076s (22.2112GiB/s) | 20260507 15:46:11.822 | INFO | Analyzing chunk frames = [681472,681984) | 20260507 15:46:11.822 | INFO | MemInfo: { | 20260507 15:46:11.822 | INFO | "gpu" : " 17 GB", | 20260507 15:46:11.822 | INFO | "huge" : "", | 20260507 15:46:11.822 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:11.822 | INFO | "malloc" : " 14 GB", | 20260507 15:46:11.822 | INFO | "pinned" : " 29 GB" | 20260507 15:46:11.822 | INFO | } | 20260507 15:46:11.822 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:46:11.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:11.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:46:11.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:46:11.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:11.822 | INFO | Uploaded 36GiB to GPU in 1.60888s (22.3758GiB/s) | 20260507 15:46:11.822 | INFO | Downloaded 34GiB to GPU in 1.53031s (22.2177GiB/s) | 20260507 15:46:16.961 | INFO | Analyzing chunk frames = [681984,682496) | 20260507 15:46:16.961 | INFO | MemInfo: { | 20260507 15:46:16.961 | INFO | "gpu" : " 17 GB", | 20260507 15:46:16.961 | INFO | "huge" : "", | 20260507 15:46:16.961 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:16.961 | INFO | "malloc" : " 14 GB", | 20260507 15:46:16.961 | INFO | "pinned" : " 29 GB" | 20260507 15:46:16.961 | INFO | } | 20260507 15:46:16.961 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:46:16.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:16.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:46:16.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:46:16.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:16.961 | INFO | Uploaded 36GiB to GPU in 1.60848s (22.3814GiB/s) | 20260507 15:46:16.961 | INFO | Downloaded 34GiB to GPU in 1.52869s (22.2413GiB/s) | 20260507 15:46:22.100 | INFO | Analyzing chunk frames = [682496,683008) | 20260507 15:46:22.100 | INFO | MemInfo: { | 20260507 15:46:22.100 | INFO | "gpu" : " 17 GB", | 20260507 15:46:22.100 | INFO | "huge" : "", | 20260507 15:46:22.100 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:22.100 | INFO | "malloc" : " 14 GB", | 20260507 15:46:22.100 | INFO | "pinned" : " 29 GB" | 20260507 15:46:22.100 | INFO | } | 20260507 15:46:22.100 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:46:22.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:22.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:46:22.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:46:22.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:22.101 | INFO | Uploaded 36GiB to GPU in 1.60757s (22.3941GiB/s) | 20260507 15:46:22.101 | INFO | Downloaded 34GiB to GPU in 1.52762s (22.2568GiB/s) | 20260507 15:46:27.240 | INFO | Analyzing chunk frames = [683008,683520) | 20260507 15:46:27.240 | INFO | MemInfo: { | 20260507 15:46:27.240 | INFO | "gpu" : " 17 GB", | 20260507 15:46:27.240 | INFO | "huge" : "", | 20260507 15:46:27.240 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:27.240 | INFO | "malloc" : " 14 GB", | 20260507 15:46:27.240 | INFO | "pinned" : " 29 GB" | 20260507 15:46:27.240 | INFO | } | 20260507 15:46:27.240 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:46:27.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:27.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.240 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:46:27.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.240 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:46:27.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:27.241 | INFO | Uploaded 36GiB to GPU in 1.60799s (22.3882GiB/s) | 20260507 15:46:27.241 | INFO | Downloaded 34GiB to GPU in 1.52884s (22.2391GiB/s) | 20260507 15:46:32.379 | INFO | Analyzing chunk frames = [683520,684032) | 20260507 15:46:32.379 | INFO | MemInfo: { | 20260507 15:46:32.379 | INFO | "gpu" : " 17 GB", | 20260507 15:46:32.379 | INFO | "huge" : "", | 20260507 15:46:32.379 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:32.379 | INFO | "malloc" : " 14 GB", | 20260507 15:46:32.379 | INFO | "pinned" : " 29 GB" | 20260507 15:46:32.379 | INFO | } | 20260507 15:46:32.379 | INFO | Perf report for the last 5.13925 seconds: | 20260507 15:46:32.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:32.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.379 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:46:32.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.379 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260507 15:46:32.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:32.380 | INFO | Uploaded 36GiB to GPU in 1.60956s (22.3664GiB/s) | 20260507 15:46:32.380 | INFO | Downloaded 34GiB to GPU in 1.53067s (22.2126GiB/s) | 20260507 15:46:32.476 | INFO | Starting a round of Baz IO | 20260507 15:46:35.352 | INFO | Populating RTMetrics report | 20260507 15:46:35.352 | INFO | RTMetrics report populated | 20260507 15:46:35.353 | INFO | Opening tmp RTMetrics json file | 20260507 15:46:35.366 | INFO | Serializing RTMetrics to json | 20260507 15:46:35.406 | INFO | Serialization done, swapping tmp file now... | 20260507 15:46:35.434 | INFO | RTMetrics done | 20260507 15:46:35.587 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.11 seconds | 20260507 15:46:35.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.17% / 0.04% / 47.79% | 20260507 15:46:35.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.33112% / 3.17891e-05% / 2.78155e-05% | 20260507 15:46:35.587 | INFO | Baz IO overall: | 20260507 15:46:35.587 | INFO | BytesWritten : 284025952494 | 20260507 15:46:35.587 | INFO | NumEvents : 137075684155 | 20260507 15:46:37.518 | INFO | Analyzing chunk frames = [684032,684544) | 20260507 15:46:37.518 | INFO | MemInfo: { | 20260507 15:46:37.518 | INFO | "gpu" : " 17 GB", | 20260507 15:46:37.518 | INFO | "huge" : "", | 20260507 15:46:37.518 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:37.518 | INFO | "malloc" : " 14 GB", | 20260507 15:46:37.518 | INFO | "pinned" : " 29 GB" | 20260507 15:46:37.518 | INFO | } | 20260507 15:46:37.518 | INFO | Perf report for the last 5.13924 seconds: | 20260507 15:46:37.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:37.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:37.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:37.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:37.519 | INFO | ANALYSIS : 57.0%, 2.9, 85.7ms, 0 | 20260507 15:46:37.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:37.519 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 15:46:37.519 | INFO | PRE_HQ : 55.8%, 2.0, 59.0ms, 0 | 20260507 15:46:37.519 | INFO | BAZWRITER : 59.9%, 4.7, 144.3ms, 0 | 20260507 15:46:37.519 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 15:46:37.519 | INFO | Uploaded 36GiB to GPU in 1.65514s (21.7504GiB/s) | 20260507 15:46:37.519 | INFO | Downloaded 38GiB to GPU in 1.74205s (21.8134GiB/s) | 20260507 15:46:42.658 | INFO | Analyzing chunk frames = [684544,685056) | 20260507 15:46:42.658 | INFO | MemInfo: { | 20260507 15:46:42.658 | INFO | "gpu" : " 17 GB", | 20260507 15:46:42.658 | INFO | "huge" : "", | 20260507 15:46:42.658 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:42.658 | INFO | "malloc" : " 14 GB", | 20260507 15:46:42.658 | INFO | "pinned" : " 29 GB" | 20260507 15:46:42.658 | INFO | } | 20260507 15:46:42.658 | INFO | Perf report for the last 5.13985 seconds: | 20260507 15:46:42.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:42.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.658 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260507 15:46:42.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.658 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 15:46:42.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:42.659 | INFO | Uploaded 36GiB to GPU in 1.60513s (22.4281GiB/s) | 20260507 15:46:42.659 | INFO | Downloaded 34GiB to GPU in 1.52543s (22.2889GiB/s) | 20260507 15:46:47.797 | INFO | Analyzing chunk frames = [685056,685568) | 20260507 15:46:47.797 | INFO | MemInfo: { | 20260507 15:46:47.797 | INFO | "gpu" : " 17 GB", | 20260507 15:46:47.797 | INFO | "huge" : "", | 20260507 15:46:47.797 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:47.797 | INFO | "malloc" : " 14 GB", | 20260507 15:46:47.797 | INFO | "pinned" : " 29 GB" | 20260507 15:46:47.797 | INFO | } | 20260507 15:46:47.797 | INFO | Perf report for the last 5.13906 seconds: | 20260507 15:46:47.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:47.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.797 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:46:47.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.797 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:46:47.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:47.798 | INFO | Uploaded 36GiB to GPU in 1.60995s (22.3609GiB/s) | 20260507 15:46:47.798 | INFO | Downloaded 34GiB to GPU in 1.53074s (22.2115GiB/s) | 20260507 15:46:52.936 | INFO | Analyzing chunk frames = [685568,686080) | 20260507 15:46:52.936 | INFO | MemInfo: { | 20260507 15:46:52.936 | INFO | "gpu" : " 17 GB", | 20260507 15:46:52.936 | INFO | "huge" : "", | 20260507 15:46:52.936 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:52.936 | INFO | "malloc" : " 14 GB", | 20260507 15:46:52.936 | INFO | "pinned" : " 29 GB" | 20260507 15:46:52.936 | INFO | } | 20260507 15:46:52.937 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:46:52.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:52.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 15:46:52.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:46:52.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:52.937 | INFO | Uploaded 36GiB to GPU in 1.60975s (22.3637GiB/s) | 20260507 15:46:52.937 | INFO | Downloaded 34GiB to GPU in 1.53041s (22.2162GiB/s) | 20260507 15:46:58.076 | INFO | Analyzing chunk frames = [686080,686592) | 20260507 15:46:58.076 | INFO | MemInfo: { | 20260507 15:46:58.076 | INFO | "gpu" : " 17 GB", | 20260507 15:46:58.076 | INFO | "huge" : "", | 20260507 15:46:58.076 | INFO | "huge_shared" : " 29 GB", | 20260507 15:46:58.076 | INFO | "malloc" : " 14 GB", | 20260507 15:46:58.076 | INFO | "pinned" : " 29 GB" | 20260507 15:46:58.076 | INFO | } | 20260507 15:46:58.076 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:46:58.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:46:58.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:46:58.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:46:58.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:46:58.076 | INFO | Uploaded 36GiB to GPU in 1.60752s (22.3948GiB/s) | 20260507 15:46:58.076 | INFO | Downloaded 34GiB to GPU in 1.52778s (22.2545GiB/s) | 20260507 15:47:03.215 | INFO | Analyzing chunk frames = [686592,687104) | 20260507 15:47:03.215 | INFO | MemInfo: { | 20260507 15:47:03.215 | INFO | "gpu" : " 17 GB", | 20260507 15:47:03.215 | INFO | "huge" : "", | 20260507 15:47:03.215 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:03.215 | INFO | "malloc" : " 14 GB", | 20260507 15:47:03.215 | INFO | "pinned" : " 29 GB" | 20260507 15:47:03.215 | INFO | } | 20260507 15:47:03.216 | INFO | Perf report for the last 5.13966 seconds: | 20260507 15:47:03.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:03.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:47:03.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:47:03.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:03.216 | INFO | Uploaded 36GiB to GPU in 1.60831s (22.3837GiB/s) | 20260507 15:47:03.216 | INFO | Downloaded 34GiB to GPU in 1.52746s (22.2592GiB/s) | 20260507 15:47:08.354 | INFO | Analyzing chunk frames = [687104,687616) | 20260507 15:47:08.354 | INFO | MemInfo: { | 20260507 15:47:08.354 | INFO | "gpu" : " 17 GB", | 20260507 15:47:08.354 | INFO | "huge" : "", | 20260507 15:47:08.354 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:08.354 | INFO | "malloc" : " 14 GB", | 20260507 15:47:08.354 | INFO | "pinned" : " 29 GB" | 20260507 15:47:08.354 | INFO | } | 20260507 15:47:08.354 | INFO | Perf report for the last 5.13876 seconds: | 20260507 15:47:08.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:08.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:47:08.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | PRE_HQ : 39.9%, 1.0, 20.7ms, 0 | 20260507 15:47:08.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:08.355 | INFO | Uploaded 36GiB to GPU in 1.61445s (22.2986GiB/s) | 20260507 15:47:08.355 | INFO | Downloaded 34GiB to GPU in 1.53416s (22.1619GiB/s) | 20260507 15:47:13.494 | INFO | Analyzing chunk frames = [687616,688128) | 20260507 15:47:13.494 | INFO | MemInfo: { | 20260507 15:47:13.494 | INFO | "gpu" : " 17 GB", | 20260507 15:47:13.494 | INFO | "huge" : "", | 20260507 15:47:13.494 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:13.494 | INFO | "malloc" : " 14 GB", | 20260507 15:47:13.494 | INFO | "pinned" : " 29 GB" | 20260507 15:47:13.494 | INFO | } | 20260507 15:47:13.494 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:47:13.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:13.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:47:13.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:47:13.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:13.494 | INFO | Uploaded 36GiB to GPU in 1.60806s (22.3872GiB/s) | 20260507 15:47:13.494 | INFO | Downloaded 34GiB to GPU in 1.52832s (22.2466GiB/s) | 20260507 15:47:13.494 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:47:13.494 | INFO | Total mem usage: 16.4 GiB | 20260507 15:47:13.494 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:47:13.494 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.7%) | 20260507 15:47:13.494 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:47:13.494 | INFO | High Water Breakdown: | 20260507 15:47:13.494 | INFO | 59.2% : 8.86 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:47:13.494 | INFO | 40.6% : 6.08 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:47:13.494 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:47:13.494 | INFO | ------------------------------------------ | 20260507 15:47:13.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:47:13.494 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:47:13.494 | INFO | Total mem usage: 26.5 GiB | 20260507 15:47:13.494 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:47:13.494 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:47:13.494 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:47:13.494 | INFO | High Water Breakdown: | 20260507 15:47:13.494 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:47:13.494 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:47:13.494 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:47:13.494 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:47:13.494 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:47:13.494 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:47:13.494 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:47:13.494 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:47:13.494 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:47:13.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:47:13.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:47:13.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:47:13.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:47:13.494 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:47:13.494 | INFO | ------------------------------------------ | 20260507 15:47:13.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:47:13.494 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:47:13.494 | INFO | Total mem usage: 27.8 GiB | 20260507 15:47:13.494 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:47:13.494 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:47:13.494 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:47:13.494 | INFO | High Water Breakdown: | 20260507 15:47:13.494 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:47:13.494 | INFO | ------------------------------------------ | 20260507 15:47:13.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:47:13.494 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:47:13.494 | INFO | Total mem usage: 15.7 GiB | 20260507 15:47:13.494 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:47:13.494 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:47:13.494 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:47:13.494 | INFO | High Water Breakdown: | 20260507 15:47:13.494 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:47:13.494 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:47:13.494 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:47:13.494 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:47:13.494 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:47:13.494 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:47:13.494 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:47:13.494 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:47:13.494 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:47:13.494 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:47:13.494 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:47:13.494 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:47:13.494 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:47:13.494 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:47:13.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:47:13.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:47:13.495 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:47:13.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:47:13.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:47:13.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:47:13.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:47:13.495 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:47:13.495 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:47:13.495 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:47:13.495 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:47:13.495 | INFO | ------------------------------------------ | 20260507 15:47:13.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:47:13.495 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:47:13.495 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:47:13.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:47:13.495 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 0.29%, AggregateStats -- 8.69e-02 / -nan, 5.51e-03 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 11.05%, Baseline -- 3.29e+00 / -nan, 1.91e-02 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.72e+01 / -nan, 4.37e-02 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 3.43%, Metrics -- 1.02e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.75e-01 / -nan, 5.50e-02 / 0.00e+00 | 20260507 15:47:13.495 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:47:13.495 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:47:13.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:47:13.495 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 18.82%, Upload -- 1.68e+01 / 0.00e+00, 4.24e-01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 18.13%, Download -- 1.62e+01 / 0.00e+00, 7.85e-01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 33.40%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 0.57%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 0.00%, LockDownload -- 4.69e-04 / 0.00e+00, 2.50e-04 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 29.07%, LockCompute -- 2.60e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:47:13.495 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 15:47:13.591 | INFO | Starting a round of Baz IO | 20260507 15:47:16.470 | INFO | Populating RTMetrics report | 20260507 15:47:16.471 | INFO | RTMetrics report populated | 20260507 15:47:16.471 | INFO | Opening tmp RTMetrics json file | 20260507 15:47:16.475 | INFO | Serializing RTMetrics to json | 20260507 15:47:16.478 | INFO | Serialization done, swapping tmp file now... | 20260507 15:47:16.485 | INFO | RTMetrics done | 20260507 15:47:16.630 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.038 seconds | 20260507 15:47:16.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.14% / 0.04% / 47.83% | 20260507 15:47:16.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.31938% / 5.96046e-05% / 3.17891e-05% | 20260507 15:47:16.630 | INFO | Baz IO overall: | 20260507 15:47:16.630 | INFO | BytesWritten : 285919953189 | 20260507 15:47:16.630 | INFO | NumEvents : 137997827777 | 20260507 15:47:18.633 | INFO | Analyzing chunk frames = [688128,688640) | 20260507 15:47:18.633 | INFO | MemInfo: { | 20260507 15:47:18.633 | INFO | "gpu" : " 17 GB", | 20260507 15:47:18.633 | INFO | "huge" : "", | 20260507 15:47:18.633 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:18.633 | INFO | "malloc" : " 14 GB", | 20260507 15:47:18.633 | INFO | "pinned" : " 29 GB" | 20260507 15:47:18.633 | INFO | } | 20260507 15:47:18.633 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:47:18.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:18.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:18.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:18.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:18.633 | INFO | ANALYSIS : 57.0%, 2.9, 85.4ms, 0 | 20260507 15:47:18.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:18.633 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 15:47:18.633 | INFO | PRE_HQ : 55.9%, 2.1, 59.6ms, 0 | 20260507 15:47:18.633 | INFO | BAZWRITER : 58.7%, 4.7, 142.8ms, 0 | 20260507 15:47:18.633 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 15:47:18.633 | INFO | Uploaded 36GiB to GPU in 1.6552s (21.7496GiB/s) | 20260507 15:47:18.633 | INFO | Downloaded 38GiB to GPU in 1.742s (21.814GiB/s) | 20260507 15:47:23.772 | INFO | Analyzing chunk frames = [688640,689152) | 20260507 15:47:23.772 | INFO | MemInfo: { | 20260507 15:47:23.772 | INFO | "gpu" : " 17 GB", | 20260507 15:47:23.772 | INFO | "huge" : "", | 20260507 15:47:23.772 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:23.772 | INFO | "malloc" : " 14 GB", | 20260507 15:47:23.772 | INFO | "pinned" : " 29 GB" | 20260507 15:47:23.772 | INFO | } | 20260507 15:47:23.773 | INFO | Perf report for the last 5.13955 seconds: | 20260507 15:47:23.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:23.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 15:47:23.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 15:47:23.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:23.773 | INFO | Uploaded 36GiB to GPU in 1.60581s (22.4186GiB/s) | 20260507 15:47:23.773 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.2889GiB/s) | 20260507 15:47:28.912 | INFO | Analyzing chunk frames = [689152,689664) | 20260507 15:47:28.912 | INFO | MemInfo: { | 20260507 15:47:28.912 | INFO | "gpu" : " 17 GB", | 20260507 15:47:28.912 | INFO | "huge" : "", | 20260507 15:47:28.912 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:28.912 | INFO | "malloc" : " 14 GB", | 20260507 15:47:28.912 | INFO | "pinned" : " 29 GB" | 20260507 15:47:28.912 | INFO | } | 20260507 15:47:28.913 | INFO | Perf report for the last 5.1405 seconds: | 20260507 15:47:28.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:28.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:47:28.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:47:28.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:28.913 | INFO | Uploaded 36GiB to GPU in 1.60974s (22.3639GiB/s) | 20260507 15:47:28.913 | INFO | Downloaded 34GiB to GPU in 1.52953s (22.229GiB/s) | 20260507 15:47:34.051 | INFO | Analyzing chunk frames = [689664,690176) | 20260507 15:47:34.051 | INFO | MemInfo: { | 20260507 15:47:34.051 | INFO | "gpu" : " 17 GB", | 20260507 15:47:34.051 | INFO | "huge" : "", | 20260507 15:47:34.051 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:34.051 | INFO | "malloc" : " 14 GB", | 20260507 15:47:34.051 | INFO | "pinned" : " 29 GB" | 20260507 15:47:34.051 | INFO | } | 20260507 15:47:34.051 | INFO | Perf report for the last 5.13815 seconds: | 20260507 15:47:34.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:34.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.051 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:47:34.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.051 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:47:34.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:34.052 | INFO | Uploaded 36GiB to GPU in 1.60974s (22.3639GiB/s) | 20260507 15:47:34.052 | INFO | Downloaded 34GiB to GPU in 1.53002s (22.222GiB/s) | 20260507 15:47:39.190 | INFO | Analyzing chunk frames = [690176,690688) | 20260507 15:47:39.190 | INFO | MemInfo: { | 20260507 15:47:39.190 | INFO | "gpu" : " 17 GB", | 20260507 15:47:39.190 | INFO | "huge" : "", | 20260507 15:47:39.190 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:39.190 | INFO | "malloc" : " 14 GB", | 20260507 15:47:39.191 | INFO | "pinned" : " 29 GB" | 20260507 15:47:39.191 | INFO | } | 20260507 15:47:39.191 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:47:39.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:39.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:47:39.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 15:47:39.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:39.191 | INFO | Uploaded 36GiB to GPU in 1.60791s (22.3893GiB/s) | 20260507 15:47:39.191 | INFO | Downloaded 34GiB to GPU in 1.52791s (22.2526GiB/s) | 20260507 15:47:44.330 | INFO | Analyzing chunk frames = [690688,691200) | 20260507 15:47:44.330 | INFO | MemInfo: { | 20260507 15:47:44.330 | INFO | "gpu" : " 17 GB", | 20260507 15:47:44.330 | INFO | "huge" : "", | 20260507 15:47:44.330 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:44.330 | INFO | "malloc" : " 14 GB", | 20260507 15:47:44.330 | INFO | "pinned" : " 29 GB" | 20260507 15:47:44.330 | INFO | } | 20260507 15:47:44.330 | INFO | Perf report for the last 5.13954 seconds: | 20260507 15:47:44.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:44.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.330 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:47:44.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.330 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260507 15:47:44.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:44.331 | INFO | Uploaded 36GiB to GPU in 1.61044s (22.3541GiB/s) | 20260507 15:47:44.331 | INFO | Downloaded 34GiB to GPU in 1.53008s (22.2211GiB/s) | 20260507 15:47:49.469 | INFO | Analyzing chunk frames = [691200,691712) | 20260507 15:47:49.469 | INFO | MemInfo: { | 20260507 15:47:49.469 | INFO | "gpu" : " 17 GB", | 20260507 15:47:49.469 | INFO | "huge" : "", | 20260507 15:47:49.469 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:49.469 | INFO | "malloc" : " 14 GB", | 20260507 15:47:49.469 | INFO | "pinned" : " 29 GB" | 20260507 15:47:49.469 | INFO | } | 20260507 15:47:49.469 | INFO | Perf report for the last 5.13908 seconds: | 20260507 15:47:49.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:49.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.469 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:47:49.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.469 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:47:49.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:49.470 | INFO | Uploaded 36GiB to GPU in 1.60853s (22.3807GiB/s) | 20260507 15:47:49.470 | INFO | Downloaded 34GiB to GPU in 1.52887s (22.2387GiB/s) | 20260507 15:47:54.608 | INFO | Analyzing chunk frames = [691712,692224) | 20260507 15:47:54.608 | INFO | MemInfo: { | 20260507 15:47:54.608 | INFO | "gpu" : " 17 GB", | 20260507 15:47:54.608 | INFO | "huge" : "", | 20260507 15:47:54.609 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:54.609 | INFO | "malloc" : " 14 GB", | 20260507 15:47:54.609 | INFO | "pinned" : " 29 GB" | 20260507 15:47:54.609 | INFO | } | 20260507 15:47:54.609 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:47:54.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:54.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:47:54.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:47:54.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:54.609 | INFO | Uploaded 36GiB to GPU in 1.60882s (22.3767GiB/s) | 20260507 15:47:54.609 | INFO | Downloaded 34GiB to GPU in 1.52932s (22.2321GiB/s) | 20260507 15:47:54.706 | INFO | Starting a round of Baz IO | 20260507 15:47:57.582 | INFO | Populating RTMetrics report | 20260507 15:47:57.582 | INFO | RTMetrics report populated | 20260507 15:47:57.582 | INFO | Opening tmp RTMetrics json file | 20260507 15:47:57.583 | INFO | Serializing RTMetrics to json | 20260507 15:47:57.584 | INFO | Serialization done, swapping tmp file now... | 20260507 15:47:57.585 | INFO | RTMetrics done | 20260507 15:47:57.829 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.123 seconds | 20260507 15:47:57.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.10% / 0.04% / 47.86% | 20260507 15:47:57.829 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.30435% / 8.74201e-05% / 3.97364e-05% | 20260507 15:47:57.829 | INFO | Baz IO overall: | 20260507 15:47:57.829 | INFO | BytesWritten : 287814803053 | 20260507 15:47:57.829 | INFO | NumEvents : 138920581152 | 20260507 15:47:59.748 | INFO | Analyzing chunk frames = [692224,692736) | 20260507 15:47:59.748 | INFO | MemInfo: { | 20260507 15:47:59.748 | INFO | "gpu" : " 17 GB", | 20260507 15:47:59.748 | INFO | "huge" : "", | 20260507 15:47:59.748 | INFO | "huge_shared" : " 29 GB", | 20260507 15:47:59.748 | INFO | "malloc" : " 14 GB", | 20260507 15:47:59.748 | INFO | "pinned" : " 29 GB" | 20260507 15:47:59.748 | INFO | } | 20260507 15:47:59.748 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:47:59.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:47:59.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:59.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:59.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:59.748 | INFO | ANALYSIS : 57.0%, 2.9, 86.3ms, 0 | 20260507 15:47:59.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:47:59.748 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 15:47:59.748 | INFO | PRE_HQ : 55.8%, 2.0, 59.0ms, 0 | 20260507 15:47:59.748 | INFO | BAZWRITER : 60.2%, 4.6, 142.1ms, 0 | 20260507 15:47:59.748 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 15:47:59.748 | INFO | Uploaded 36GiB to GPU in 1.65068s (21.8092GiB/s) | 20260507 15:47:59.749 | INFO | Downloaded 38GiB to GPU in 1.73565s (21.8938GiB/s) | 20260507 15:48:04.887 | INFO | Analyzing chunk frames = [692736,693248) | 20260507 15:48:04.887 | INFO | MemInfo: { | 20260507 15:48:04.887 | INFO | "gpu" : " 17 GB", | 20260507 15:48:04.887 | INFO | "huge" : "", | 20260507 15:48:04.887 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:04.888 | INFO | "malloc" : " 14 GB", | 20260507 15:48:04.888 | INFO | "pinned" : " 29 GB" | 20260507 15:48:04.888 | INFO | } | 20260507 15:48:04.888 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:48:04.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:04.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 15:48:04.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | PRE_HQ : 27.5%, 1.0, 14.4ms, 0 | 20260507 15:48:04.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:04.888 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4178GiB/s) | 20260507 15:48:04.888 | INFO | Downloaded 34GiB to GPU in 1.52507s (22.2941GiB/s) | 20260507 15:48:10.026 | INFO | Analyzing chunk frames = [693248,693760) | 20260507 15:48:10.026 | INFO | MemInfo: { | 20260507 15:48:10.026 | INFO | "gpu" : " 17 GB", | 20260507 15:48:10.026 | INFO | "huge" : "", | 20260507 15:48:10.026 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:10.026 | INFO | "malloc" : " 14 GB", | 20260507 15:48:10.027 | INFO | "pinned" : " 29 GB" | 20260507 15:48:10.027 | INFO | } | 20260507 15:48:10.027 | INFO | Perf report for the last 5.13899 seconds: | 20260507 15:48:10.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:10.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:48:10.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:48:10.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:10.027 | INFO | Uploaded 36GiB to GPU in 1.61018s (22.3577GiB/s) | 20260507 15:48:10.027 | INFO | Downloaded 34GiB to GPU in 1.53028s (22.2182GiB/s) | 20260507 15:48:15.166 | INFO | Analyzing chunk frames = [693760,694272) | 20260507 15:48:15.166 | INFO | MemInfo: { | 20260507 15:48:15.166 | INFO | "gpu" : " 17 GB", | 20260507 15:48:15.166 | INFO | "huge" : "", | 20260507 15:48:15.166 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:15.166 | INFO | "malloc" : " 14 GB", | 20260507 15:48:15.166 | INFO | "pinned" : " 29 GB" | 20260507 15:48:15.166 | INFO | } | 20260507 15:48:15.167 | INFO | Perf report for the last 5.13983 seconds: | 20260507 15:48:15.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:15.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | ANALYSIS : 55.9%, 3.0, 86.6ms, 0 | 20260507 15:48:15.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:48:15.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:15.167 | INFO | Uploaded 36GiB to GPU in 1.60984s (22.3624GiB/s) | 20260507 15:48:15.167 | INFO | Downloaded 34GiB to GPU in 1.53015s (22.2201GiB/s) | 20260507 15:48:20.305 | INFO | Analyzing chunk frames = [694272,694784) | 20260507 15:48:20.305 | INFO | MemInfo: { | 20260507 15:48:20.305 | INFO | "gpu" : " 17 GB", | 20260507 15:48:20.305 | INFO | "huge" : "", | 20260507 15:48:20.306 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:20.306 | INFO | "malloc" : " 14 GB", | 20260507 15:48:20.306 | INFO | "pinned" : " 29 GB" | 20260507 15:48:20.306 | INFO | } | 20260507 15:48:20.306 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:48:20.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:20.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:48:20.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 15:48:20.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:20.306 | INFO | Uploaded 36GiB to GPU in 1.61466s (22.2957GiB/s) | 20260507 15:48:20.306 | INFO | Downloaded 34GiB to GPU in 1.53306s (22.1779GiB/s) | 20260507 15:48:25.445 | INFO | Analyzing chunk frames = [694784,695296) | 20260507 15:48:25.445 | INFO | MemInfo: { | 20260507 15:48:25.445 | INFO | "gpu" : " 17 GB", | 20260507 15:48:25.445 | INFO | "huge" : "", | 20260507 15:48:25.445 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:25.445 | INFO | "malloc" : " 14 GB", | 20260507 15:48:25.445 | INFO | "pinned" : " 29 GB" | 20260507 15:48:25.445 | INFO | } | 20260507 15:48:25.445 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:48:25.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:25.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.445 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:48:25.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.445 | INFO | PRE_HQ : 40.3%, 1.0, 20.9ms, 0 | 20260507 15:48:25.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:25.446 | INFO | Uploaded 36GiB to GPU in 1.6132s (22.3159GiB/s) | 20260507 15:48:25.446 | INFO | Downloaded 34GiB to GPU in 1.5336s (22.1701GiB/s) | 20260507 15:48:30.584 | INFO | Analyzing chunk frames = [695296,695808) | 20260507 15:48:30.584 | INFO | MemInfo: { | 20260507 15:48:30.584 | INFO | "gpu" : " 17 GB", | 20260507 15:48:30.584 | INFO | "huge" : "", | 20260507 15:48:30.584 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:30.584 | INFO | "malloc" : " 14 GB", | 20260507 15:48:30.584 | INFO | "pinned" : " 29 GB" | 20260507 15:48:30.585 | INFO | } | 20260507 15:48:30.585 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:48:30.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:30.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:48:30.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | PRE_HQ : 39.9%, 1.0, 20.7ms, 0 | 20260507 15:48:30.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:30.585 | INFO | Uploaded 36GiB to GPU in 1.61456s (22.297GiB/s) | 20260507 15:48:30.585 | INFO | Downloaded 34GiB to GPU in 1.53317s (22.1762GiB/s) | 20260507 15:48:35.723 | INFO | Analyzing chunk frames = [695808,696320) | 20260507 15:48:35.723 | INFO | MemInfo: { | 20260507 15:48:35.723 | INFO | "gpu" : " 17 GB", | 20260507 15:48:35.723 | INFO | "huge" : "", | 20260507 15:48:35.723 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:35.723 | INFO | "malloc" : " 14 GB", | 20260507 15:48:35.723 | INFO | "pinned" : " 29 GB" | 20260507 15:48:35.723 | INFO | } | 20260507 15:48:35.723 | INFO | Perf report for the last 5.13857 seconds: | 20260507 15:48:35.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:35.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.723 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:48:35.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.723 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 15:48:35.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:35.724 | INFO | Uploaded 36GiB to GPU in 1.6141s (22.3034GiB/s) | 20260507 15:48:35.724 | INFO | Downloaded 34GiB to GPU in 1.53459s (22.1558GiB/s) | 20260507 15:48:35.724 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:48:35.724 | INFO | Total mem usage: 16.4 GiB | 20260507 15:48:35.724 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:48:35.724 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.81%) | 20260507 15:48:35.724 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:48:35.724 | INFO | High Water Breakdown: | 20260507 15:48:35.724 | INFO | 59.3% : 8.85 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:48:35.724 | INFO | 40.6% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:48:35.724 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:48:35.724 | INFO | ------------------------------------------ | 20260507 15:48:35.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:48:35.724 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:48:35.724 | INFO | Total mem usage: 26.5 GiB | 20260507 15:48:35.724 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:48:35.724 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 15:48:35.724 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 15:48:35.724 | INFO | High Water Breakdown: | 20260507 15:48:35.724 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:48:35.724 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:48:35.724 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:48:35.724 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:48:35.724 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:48:35.724 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:48:35.724 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:48:35.724 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:48:35.724 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:48:35.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:48:35.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:48:35.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:48:35.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:48:35.724 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:48:35.724 | INFO | ------------------------------------------ | 20260507 15:48:35.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:48:35.724 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:48:35.724 | INFO | Total mem usage: 27.8 GiB | 20260507 15:48:35.724 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:48:35.724 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:48:35.724 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:48:35.724 | INFO | High Water Breakdown: | 20260507 15:48:35.724 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:48:35.724 | INFO | ------------------------------------------ | 20260507 15:48:35.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:48:35.724 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:48:35.724 | INFO | Total mem usage: 15.7 GiB | 20260507 15:48:35.724 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:48:35.724 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:48:35.724 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:48:35.724 | INFO | High Water Breakdown: | 20260507 15:48:35.724 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:48:35.724 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:48:35.724 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:48:35.724 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:48:35.724 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:48:35.724 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:48:35.725 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:48:35.725 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:48:35.725 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:48:35.725 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:48:35.725 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:48:35.725 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:48:35.725 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:48:35.725 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:48:35.725 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:48:35.725 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:48:35.725 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:48:35.725 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:48:35.725 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:48:35.725 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:48:35.725 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:48:35.725 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:48:35.725 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:48:35.725 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:48:35.725 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:48:35.725 | INFO | ------------------------------------------ | 20260507 15:48:35.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:48:35.725 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:48:35.725 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:48:35.725 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:48:35.725 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 0.29%, AggregateStats -- 8.80e-02 / -nan, 5.15e-03 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 11.06%, Baseline -- 3.30e+00 / -nan, 2.18e-02 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.72e+01 / -nan, 4.40e-02 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 3.43%, Metrics -- 1.02e+00 / -nan, 1.62e-01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.78e-01 / -nan, 5.55e-02 / 0.00e+00 | 20260507 15:48:35.725 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:48:35.725 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:48:35.725 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:48:35.725 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 18.86%, Upload -- 1.68e+01 / 0.00e+00, 4.45e-01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 18.17%, Download -- 1.62e+01 / 0.00e+00, 8.18e-01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 33.43%, Compute -- 2.98e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 0.00%, LockDownload -- 5.21e-04 / 0.00e+00, 3.72e-04 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 28.96%, LockCompute -- 2.58e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 15:48:35.725 | INFO | PerfReport: 0.02%, Misc -- 1.43e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 15:48:35.820 | INFO | Starting a round of Baz IO | 20260507 15:48:38.755 | INFO | Populating RTMetrics report | 20260507 15:48:38.755 | INFO | RTMetrics report populated | 20260507 15:48:38.755 | INFO | Opening tmp RTMetrics json file | 20260507 15:48:38.788 | INFO | Serializing RTMetrics to json | 20260507 15:48:38.823 | INFO | Serialization done, swapping tmp file now... | 20260507 15:48:38.826 | INFO | RTMetrics done | 20260507 15:48:38.842 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.022 seconds | 20260507 15:48:38.842 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.06% / 0.04% / 47.90% | 20260507 15:48:38.842 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.28271% / 5.16574e-05% / 3.57628e-05% | 20260507 15:48:38.842 | INFO | Baz IO overall: | 20260507 15:48:38.842 | INFO | BytesWritten : 289709320171 | 20260507 15:48:38.842 | INFO | NumEvents : 139842819320 | 20260507 15:48:40.863 | INFO | Analyzing chunk frames = [696320,696832) | 20260507 15:48:40.863 | INFO | MemInfo: { | 20260507 15:48:40.863 | INFO | "gpu" : " 17 GB", | 20260507 15:48:40.863 | INFO | "huge" : "", | 20260507 15:48:40.863 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:40.863 | INFO | "malloc" : " 14 GB", | 20260507 15:48:40.863 | INFO | "pinned" : " 29 GB" | 20260507 15:48:40.863 | INFO | } | 20260507 15:48:40.863 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:48:40.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:40.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:40.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:40.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:40.863 | INFO | ANALYSIS : 57.8%, 2.8, 84.9ms, 0 | 20260507 15:48:40.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:40.863 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260507 15:48:40.863 | INFO | PRE_HQ : 57.0%, 2.2, 64.9ms, 0 | 20260507 15:48:40.863 | INFO | BAZWRITER : 58.4%, 4.7, 139.6ms, 0 | 20260507 15:48:40.863 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 15:48:40.863 | INFO | Uploaded 36GiB to GPU in 1.64792s (21.8457GiB/s) | 20260507 15:48:40.863 | INFO | Downloaded 38GiB to GPU in 1.73309s (21.9261GiB/s) | 20260507 15:48:46.002 | INFO | Analyzing chunk frames = [696832,697344) | 20260507 15:48:46.002 | INFO | MemInfo: { | 20260507 15:48:46.002 | INFO | "gpu" : " 17 GB", | 20260507 15:48:46.002 | INFO | "huge" : "", | 20260507 15:48:46.002 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:46.002 | INFO | "malloc" : " 14 GB", | 20260507 15:48:46.002 | INFO | "pinned" : " 29 GB" | 20260507 15:48:46.002 | INFO | } | 20260507 15:48:46.002 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:48:46.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:46.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.002 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 15:48:46.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.002 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:48:46.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:46.003 | INFO | Uploaded 36GiB to GPU in 1.58501s (22.7127GiB/s) | 20260507 15:48:46.003 | INFO | Downloaded 34GiB to GPU in 1.5344s (22.1585GiB/s) | 20260507 15:48:51.141 | INFO | Analyzing chunk frames = [697344,697856) | 20260507 15:48:51.141 | INFO | MemInfo: { | 20260507 15:48:51.141 | INFO | "gpu" : " 17 GB", | 20260507 15:48:51.141 | INFO | "huge" : "", | 20260507 15:48:51.141 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:51.141 | INFO | "malloc" : " 14 GB", | 20260507 15:48:51.141 | INFO | "pinned" : " 29 GB" | 20260507 15:48:51.141 | INFO | } | 20260507 15:48:51.142 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:48:51.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:51.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:48:51.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:48:51.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:51.142 | INFO | Uploaded 36GiB to GPU in 1.58971s (22.6457GiB/s) | 20260507 15:48:51.142 | INFO | Downloaded 34GiB to GPU in 1.53868s (22.0968GiB/s) | 20260507 15:48:56.282 | INFO | Analyzing chunk frames = [697856,698368) | 20260507 15:48:56.282 | INFO | MemInfo: { | 20260507 15:48:56.282 | INFO | "gpu" : " 17 GB", | 20260507 15:48:56.282 | INFO | "huge" : "", | 20260507 15:48:56.282 | INFO | "huge_shared" : " 29 GB", | 20260507 15:48:56.282 | INFO | "malloc" : " 14 GB", | 20260507 15:48:56.282 | INFO | "pinned" : " 29 GB" | 20260507 15:48:56.282 | INFO | } | 20260507 15:48:56.282 | INFO | Perf report for the last 5.14057 seconds: | 20260507 15:48:56.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:48:56.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:48:56.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:48:56.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:48:56.282 | INFO | Uploaded 36GiB to GPU in 1.5885s (22.6629GiB/s) | 20260507 15:48:56.283 | INFO | Downloaded 34GiB to GPU in 1.53895s (22.093GiB/s) | 20260507 15:49:01.420 | INFO | Analyzing chunk frames = [698368,698880) | 20260507 15:49:01.420 | INFO | MemInfo: { | 20260507 15:49:01.420 | INFO | "gpu" : " 17 GB", | 20260507 15:49:01.420 | INFO | "huge" : "", | 20260507 15:49:01.420 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:01.420 | INFO | "malloc" : " 14 GB", | 20260507 15:49:01.420 | INFO | "pinned" : " 29 GB" | 20260507 15:49:01.420 | INFO | } | 20260507 15:49:01.420 | INFO | Perf report for the last 5.13831 seconds: | 20260507 15:49:01.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:01.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.420 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:49:01.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.420 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 15:49:01.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:01.421 | INFO | Uploaded 36GiB to GPU in 1.58864s (22.6609GiB/s) | 20260507 15:49:01.421 | INFO | Downloaded 34GiB to GPU in 1.53758s (22.1127GiB/s) | 20260507 15:49:06.559 | INFO | Analyzing chunk frames = [698880,699392) | 20260507 15:49:06.559 | INFO | MemInfo: { | 20260507 15:49:06.559 | INFO | "gpu" : " 17 GB", | 20260507 15:49:06.559 | INFO | "huge" : "", | 20260507 15:49:06.559 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:06.559 | INFO | "malloc" : " 14 GB", | 20260507 15:49:06.559 | INFO | "pinned" : " 29 GB" | 20260507 15:49:06.559 | INFO | } | 20260507 15:49:06.560 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:49:06.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:06.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:49:06.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:49:06.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:06.560 | INFO | Uploaded 36GiB to GPU in 1.58711s (22.6827GiB/s) | 20260507 15:49:06.560 | INFO | Downloaded 34GiB to GPU in 1.53648s (22.1285GiB/s) | 20260507 15:49:11.699 | INFO | Analyzing chunk frames = [699392,699904) | 20260507 15:49:11.699 | INFO | MemInfo: { | 20260507 15:49:11.699 | INFO | "gpu" : " 17 GB", | 20260507 15:49:11.699 | INFO | "huge" : "", | 20260507 15:49:11.699 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:11.699 | INFO | "malloc" : " 14 GB", | 20260507 15:49:11.699 | INFO | "pinned" : " 29 GB" | 20260507 15:49:11.699 | INFO | } | 20260507 15:49:11.699 | INFO | Perf report for the last 5.13919 seconds: | 20260507 15:49:11.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:11.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:49:11.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:49:11.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:11.699 | INFO | Uploaded 36GiB to GPU in 1.58856s (22.6621GiB/s) | 20260507 15:49:11.700 | INFO | Downloaded 34GiB to GPU in 1.53768s (22.1112GiB/s) | 20260507 15:49:16.838 | INFO | Analyzing chunk frames = [699904,700416) | 20260507 15:49:16.838 | INFO | MemInfo: { | 20260507 15:49:16.838 | INFO | "gpu" : " 17 GB", | 20260507 15:49:16.838 | INFO | "huge" : "", | 20260507 15:49:16.838 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:16.838 | INFO | "malloc" : " 14 GB", | 20260507 15:49:16.838 | INFO | "pinned" : " 29 GB" | 20260507 15:49:16.838 | INFO | } | 20260507 15:49:16.838 | INFO | Perf report for the last 5.13963 seconds: | 20260507 15:49:16.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:16.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.838 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:49:16.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.839 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 15:49:16.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:16.839 | INFO | Uploaded 36GiB to GPU in 1.5922s (22.6102GiB/s) | 20260507 15:49:16.839 | INFO | Downloaded 34GiB to GPU in 1.54386s (22.0227GiB/s) | 20260507 15:49:16.952 | INFO | Starting a round of Baz IO | 20260507 15:49:19.971 | INFO | Populating RTMetrics report | 20260507 15:49:19.971 | INFO | RTMetrics report populated | 20260507 15:49:19.971 | INFO | Opening tmp RTMetrics json file | 20260507 15:49:19.976 | INFO | Serializing RTMetrics to json | 20260507 15:49:20.001 | INFO | Serialization done, swapping tmp file now... | 20260507 15:49:20.004 | INFO | RTMetrics done | 20260507 15:49:20.143 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.191 seconds | 20260507 15:49:20.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.02% / 0.04% / 47.94% | 20260507 15:49:20.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.27252% / 8.34465e-05% / 2.78155e-05% | 20260507 15:49:20.144 | INFO | Baz IO overall: | 20260507 15:49:20.144 | INFO | BytesWritten : 291605952693 | 20260507 15:49:20.144 | INFO | NumEvents : 140766308745 | 20260507 15:49:21.978 | INFO | Analyzing chunk frames = [700416,700928) | 20260507 15:49:21.978 | INFO | MemInfo: { | 20260507 15:49:21.978 | INFO | "gpu" : " 17 GB", | 20260507 15:49:21.978 | INFO | "huge" : "", | 20260507 15:49:21.978 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:21.978 | INFO | "malloc" : " 14 GB", | 20260507 15:49:21.979 | INFO | "pinned" : " 29 GB" | 20260507 15:49:21.979 | INFO | } | 20260507 15:49:21.979 | INFO | Perf report for the last 5.14015 seconds: | 20260507 15:49:21.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:21.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:21.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:21.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:21.979 | INFO | ANALYSIS : 59.7%, 2.6, 82.1ms, 0 | 20260507 15:49:21.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:21.979 | INFO | RT_METRICS : 51.7%, 1.0, 27.1ms, 0 | 20260507 15:49:21.979 | INFO | PRE_HQ : 58.9%, 2.4, 73.1ms, 0 | 20260507 15:49:21.979 | INFO | BAZWRITER : 61.6%, 4.8, 152.3ms, 0 | 20260507 15:49:21.979 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:49:21.979 | INFO | Uploaded 36GiB to GPU in 1.63457s (22.0241GiB/s) | 20260507 15:49:21.979 | INFO | Downloaded 38GiB to GPU in 1.71394s (22.1712GiB/s) | 20260507 15:49:27.117 | INFO | Analyzing chunk frames = [700928,701440) | 20260507 15:49:27.117 | INFO | MemInfo: { | 20260507 15:49:27.117 | INFO | "gpu" : " 17 GB", | 20260507 15:49:27.117 | INFO | "huge" : "", | 20260507 15:49:27.117 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:27.117 | INFO | "malloc" : " 14 GB", | 20260507 15:49:27.117 | INFO | "pinned" : " 29 GB" | 20260507 15:49:27.117 | INFO | } | 20260507 15:49:27.117 | INFO | Perf report for the last 5.13836 seconds: | 20260507 15:49:27.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:27.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:49:27.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | PRE_HQ : 27.3%, 1.0, 14.3ms, 0 | 20260507 15:49:27.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:27.117 | INFO | Uploaded 36GiB to GPU in 1.5853s (22.7086GiB/s) | 20260507 15:49:27.117 | INFO | Downloaded 34GiB to GPU in 1.53406s (22.1634GiB/s) | 20260507 15:49:32.256 | INFO | Analyzing chunk frames = [701440,701952) | 20260507 15:49:32.256 | INFO | MemInfo: { | 20260507 15:49:32.256 | INFO | "gpu" : " 17 GB", | 20260507 15:49:32.256 | INFO | "huge" : "", | 20260507 15:49:32.256 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:32.256 | INFO | "malloc" : " 14 GB", | 20260507 15:49:32.256 | INFO | "pinned" : " 29 GB" | 20260507 15:49:32.256 | INFO | } | 20260507 15:49:32.256 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:49:32.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:32.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.256 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:49:32.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.256 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:49:32.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:32.257 | INFO | Uploaded 36GiB to GPU in 1.58939s (22.6502GiB/s) | 20260507 15:49:32.257 | INFO | Downloaded 34GiB to GPU in 1.53937s (22.087GiB/s) | 20260507 15:49:37.399 | INFO | Analyzing chunk frames = [701952,702464) | 20260507 15:49:37.399 | INFO | MemInfo: { | 20260507 15:49:37.399 | INFO | "gpu" : " 17 GB", | 20260507 15:49:37.399 | INFO | "huge" : "", | 20260507 15:49:37.399 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:37.399 | INFO | "malloc" : " 14 GB", | 20260507 15:49:37.399 | INFO | "pinned" : " 29 GB" | 20260507 15:49:37.399 | INFO | } | 20260507 15:49:37.399 | INFO | Perf report for the last 5.14298 seconds: | 20260507 15:49:37.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:37.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.399 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:49:37.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.399 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:49:37.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:37.400 | INFO | Uploaded 36GiB to GPU in 1.58933s (22.651GiB/s) | 20260507 15:49:37.400 | INFO | Downloaded 34GiB to GPU in 1.53674s (22.1247GiB/s) | 20260507 15:49:42.536 | INFO | Analyzing chunk frames = [702464,702976) | 20260507 15:49:42.536 | INFO | MemInfo: { | 20260507 15:49:42.536 | INFO | "gpu" : " 17 GB", | 20260507 15:49:42.536 | INFO | "huge" : "", | 20260507 15:49:42.536 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:42.536 | INFO | "malloc" : " 14 GB", | 20260507 15:49:42.536 | INFO | "pinned" : " 29 GB" | 20260507 15:49:42.536 | INFO | } | 20260507 15:49:42.536 | INFO | Perf report for the last 5.13694 seconds: | 20260507 15:49:42.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:42.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:49:42.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:49:42.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:42.536 | INFO | Uploaded 36GiB to GPU in 1.58816s (22.6678GiB/s) | 20260507 15:49:42.537 | INFO | Downloaded 34GiB to GPU in 1.53725s (22.1174GiB/s) | 20260507 15:49:47.674 | INFO | Analyzing chunk frames = [702976,703488) | 20260507 15:49:47.674 | INFO | MemInfo: { | 20260507 15:49:47.674 | INFO | "gpu" : " 17 GB", | 20260507 15:49:47.674 | INFO | "huge" : "", | 20260507 15:49:47.674 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:47.674 | INFO | "malloc" : " 14 GB", | 20260507 15:49:47.674 | INFO | "pinned" : " 29 GB" | 20260507 15:49:47.674 | INFO | } | 20260507 15:49:47.674 | INFO | Perf report for the last 5.13802 seconds: | 20260507 15:49:47.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:47.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:49:47.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 15:49:47.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:47.674 | INFO | Uploaded 36GiB to GPU in 1.58769s (22.6745GiB/s) | 20260507 15:49:47.674 | INFO | Downloaded 34GiB to GPU in 1.53756s (22.1129GiB/s) | 20260507 15:49:52.814 | INFO | Analyzing chunk frames = [703488,704000) | 20260507 15:49:52.814 | INFO | MemInfo: { | 20260507 15:49:52.814 | INFO | "gpu" : " 17 GB", | 20260507 15:49:52.814 | INFO | "huge" : "", | 20260507 15:49:52.814 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:52.814 | INFO | "malloc" : " 14 GB", | 20260507 15:49:52.814 | INFO | "pinned" : " 29 GB" | 20260507 15:49:52.814 | INFO | } | 20260507 15:49:52.814 | INFO | Perf report for the last 5.13973 seconds: | 20260507 15:49:52.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:52.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.814 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:49:52.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.814 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:49:52.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:52.815 | INFO | Uploaded 36GiB to GPU in 1.58708s (22.6832GiB/s) | 20260507 15:49:52.815 | INFO | Downloaded 34GiB to GPU in 1.53819s (22.104GiB/s) | 20260507 15:49:57.953 | INFO | Analyzing chunk frames = [704000,704512) | 20260507 15:49:57.953 | INFO | MemInfo: { | 20260507 15:49:57.953 | INFO | "gpu" : " 17 GB", | 20260507 15:49:57.953 | INFO | "huge" : "", | 20260507 15:49:57.953 | INFO | "huge_shared" : " 29 GB", | 20260507 15:49:57.953 | INFO | "malloc" : " 14 GB", | 20260507 15:49:57.953 | INFO | "pinned" : " 29 GB" | 20260507 15:49:57.953 | INFO | } | 20260507 15:49:57.953 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:49:57.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:49:57.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.953 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:49:57.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.954 | INFO | PRE_HQ : 39.4%, 1.0, 20.4ms, 0 | 20260507 15:49:57.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:49:57.954 | INFO | Uploaded 36GiB to GPU in 1.59238s (22.6077GiB/s) | 20260507 15:49:57.954 | INFO | Downloaded 34GiB to GPU in 1.54398s (22.021GiB/s) | 20260507 15:49:57.954 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:49:57.954 | INFO | Total mem usage: 16.4 GiB | 20260507 15:49:57.954 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:49:57.954 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.89%) | 20260507 15:49:57.954 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:49:57.954 | INFO | High Water Breakdown: | 20260507 15:49:57.954 | INFO | 59.9% : 9.28 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:49:57.954 | INFO | 40% : 6.19 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:49:57.954 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:49:57.954 | INFO | ------------------------------------------ | 20260507 15:49:57.954 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:49:57.955 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:49:57.955 | INFO | Total mem usage: 26.5 GiB | 20260507 15:49:57.955 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:49:57.955 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 15:49:57.955 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:49:57.955 | INFO | High Water Breakdown: | 20260507 15:49:57.955 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:49:57.955 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:49:57.955 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:49:57.955 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:49:57.955 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:49:57.955 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:49:57.955 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:49:57.955 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:49:57.955 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:49:57.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:49:57.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:49:57.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:49:57.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:49:57.955 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:49:57.955 | INFO | ------------------------------------------ | 20260507 15:49:57.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:49:57.955 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:49:57.955 | INFO | Total mem usage: 27.8 GiB | 20260507 15:49:57.955 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:49:57.955 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:49:57.955 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:49:57.955 | INFO | High Water Breakdown: | 20260507 15:49:57.955 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:49:57.955 | INFO | ------------------------------------------ | 20260507 15:49:57.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:49:57.955 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:49:57.955 | INFO | Total mem usage: 15.7 GiB | 20260507 15:49:57.955 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:49:57.955 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:49:57.955 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:49:57.955 | INFO | High Water Breakdown: | 20260507 15:49:57.955 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:49:57.955 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:49:57.955 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:49:57.955 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:49:57.955 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:49:57.955 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:49:57.955 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:49:57.955 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:49:57.955 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:49:57.955 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:49:57.955 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:49:57.955 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:49:57.955 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:49:57.955 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:49:57.955 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:49:57.955 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:49:57.955 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:49:57.955 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:49:57.955 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:49:57.955 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:49:57.955 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:49:57.955 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:49:57.955 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:49:57.955 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:49:57.955 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:49:57.955 | INFO | ------------------------------------------ | 20260507 15:49:57.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:49:57.956 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:49:57.956 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:49:57.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:49:57.956 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 0.29%, AggregateStats -- 8.72e-02 / -nan, 4.84e-03 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 11.06%, Baseline -- 3.30e+00 / -nan, 2.22e-02 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.32e-02 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 3.42%, Metrics -- 1.02e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.75e-01 / -nan, 5.53e-02 / 0.00e+00 | 20260507 15:49:57.956 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:49:57.956 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:49:57.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:49:57.956 | INFO | PerfReport: Full Execution -- 8.89e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 18.68%, Upload -- 1.66e+01 / 0.00e+00, 4.64e-01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 18.29%, Download -- 1.63e+01 / 0.00e+00, 7.61e-01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 33.54%, Compute -- 2.98e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 0.60%, LockUpload -- 5.33e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 0.00%, LockDownload -- 4.71e-04 / 0.00e+00, 2.51e-04 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 28.87%, LockCompute -- 2.57e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260507 15:49:57.956 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 4.02e-02 / 0.00e+00 | 20260507 15:49:58.065 | INFO | Starting a round of Baz IO | 20260507 15:50:00.973 | INFO | Populating RTMetrics report | 20260507 15:50:00.974 | INFO | RTMetrics report populated | 20260507 15:50:00.974 | INFO | Opening tmp RTMetrics json file | 20260507 15:50:00.975 | INFO | Serializing RTMetrics to json | 20260507 15:50:00.977 | INFO | Serialization done, swapping tmp file now... | 20260507 15:50:00.977 | INFO | RTMetrics done | 20260507 15:50:01.064 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 2.998 seconds | 20260507 15:50:01.064 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.99% / 0.04% / 47.98% | 20260507 15:50:01.064 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.25283% / 7.94729e-05% / 1.98682e-05% | 20260507 15:50:01.064 | INFO | Baz IO overall: | 20260507 15:50:01.064 | INFO | BytesWritten : 293502022074 | 20260507 15:50:01.064 | INFO | NumEvents : 141689033527 | 20260507 15:50:03.092 | INFO | Analyzing chunk frames = [704512,705024) | 20260507 15:50:03.092 | INFO | MemInfo: { | 20260507 15:50:03.092 | INFO | "gpu" : " 17 GB", | 20260507 15:50:03.092 | INFO | "huge" : "", | 20260507 15:50:03.092 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:03.092 | INFO | "malloc" : " 14 GB", | 20260507 15:50:03.092 | INFO | "pinned" : " 29 GB" | 20260507 15:50:03.092 | INFO | } | 20260507 15:50:03.092 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:50:03.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:03.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:03.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:03.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:03.093 | INFO | ANALYSIS : 57.9%, 2.8, 85.3ms, 0 | 20260507 15:50:03.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:03.093 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 15:50:03.093 | INFO | PRE_HQ : 57.0%, 2.2, 65.9ms, 0 | 20260507 15:50:03.093 | INFO | BAZWRITER : 57.9%, 4.5, 134.6ms, 0 | 20260507 15:50:03.093 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:50:03.093 | INFO | Uploaded 36GiB to GPU in 1.63664s (21.9963GiB/s) | 20260507 15:50:03.093 | INFO | Downloaded 38GiB to GPU in 1.72093s (22.0811GiB/s) | 20260507 15:50:08.231 | INFO | Analyzing chunk frames = [705024,705536) | 20260507 15:50:08.231 | INFO | MemInfo: { | 20260507 15:50:08.231 | INFO | "gpu" : " 17 GB", | 20260507 15:50:08.231 | INFO | "huge" : "", | 20260507 15:50:08.231 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:08.231 | INFO | "malloc" : " 14 GB", | 20260507 15:50:08.231 | INFO | "pinned" : " 29 GB" | 20260507 15:50:08.231 | INFO | } | 20260507 15:50:08.231 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:50:08.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:08.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:50:08.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:50:08.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:08.232 | INFO | Uploaded 36GiB to GPU in 1.58516s (22.7106GiB/s) | 20260507 15:50:08.232 | INFO | Downloaded 34GiB to GPU in 1.53403s (22.1638GiB/s) | 20260507 15:50:13.371 | INFO | Analyzing chunk frames = [705536,706048) | 20260507 15:50:13.371 | INFO | MemInfo: { | 20260507 15:50:13.371 | INFO | "gpu" : " 17 GB", | 20260507 15:50:13.371 | INFO | "huge" : "", | 20260507 15:50:13.371 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:13.371 | INFO | "malloc" : " 14 GB", | 20260507 15:50:13.371 | INFO | "pinned" : " 29 GB" | 20260507 15:50:13.371 | INFO | } | 20260507 15:50:13.371 | INFO | Perf report for the last 5.1394 seconds: | 20260507 15:50:13.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:13.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.371 | INFO | ANALYSIS : 56.2%, 3.0, 87.2ms, 0 | 20260507 15:50:13.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.371 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:50:13.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:13.372 | INFO | Uploaded 36GiB to GPU in 1.58972s (22.6455GiB/s) | 20260507 15:50:13.372 | INFO | Downloaded 34GiB to GPU in 1.53967s (22.0827GiB/s) | 20260507 15:50:18.511 | INFO | Analyzing chunk frames = [706048,706560) | 20260507 15:50:18.511 | INFO | MemInfo: { | 20260507 15:50:18.511 | INFO | "gpu" : " 17 GB", | 20260507 15:50:18.511 | INFO | "huge" : "", | 20260507 15:50:18.511 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:18.511 | INFO | "malloc" : " 14 GB", | 20260507 15:50:18.511 | INFO | "pinned" : " 29 GB" | 20260507 15:50:18.511 | INFO | } | 20260507 15:50:18.511 | INFO | Perf report for the last 5.14017 seconds: | 20260507 15:50:18.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:18.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:50:18.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:50:18.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:18.511 | INFO | Uploaded 36GiB to GPU in 1.58915s (22.6537GiB/s) | 20260507 15:50:18.511 | INFO | Downloaded 34GiB to GPU in 1.5384s (22.1009GiB/s) | 20260507 15:50:23.649 | INFO | Analyzing chunk frames = [706560,707072) | 20260507 15:50:23.649 | INFO | MemInfo: { | 20260507 15:50:23.649 | INFO | "gpu" : " 17 GB", | 20260507 15:50:23.649 | INFO | "huge" : "", | 20260507 15:50:23.649 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:23.649 | INFO | "malloc" : " 14 GB", | 20260507 15:50:23.649 | INFO | "pinned" : " 29 GB" | 20260507 15:50:23.649 | INFO | } | 20260507 15:50:23.649 | INFO | Perf report for the last 5.13841 seconds: | 20260507 15:50:23.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:23.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:50:23.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:50:23.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:23.650 | INFO | Uploaded 36GiB to GPU in 1.58883s (22.6582GiB/s) | 20260507 15:50:23.650 | INFO | Downloaded 34GiB to GPU in 1.5372s (22.1181GiB/s) | 20260507 15:50:28.789 | INFO | Analyzing chunk frames = [707072,707584) | 20260507 15:50:28.789 | INFO | MemInfo: { | 20260507 15:50:28.789 | INFO | "gpu" : " 17 GB", | 20260507 15:50:28.789 | INFO | "huge" : "", | 20260507 15:50:28.789 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:28.789 | INFO | "malloc" : " 14 GB", | 20260507 15:50:28.789 | INFO | "pinned" : " 29 GB" | 20260507 15:50:28.789 | INFO | } | 20260507 15:50:28.789 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:50:28.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:28.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:50:28.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:50:28.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:28.789 | INFO | Uploaded 36GiB to GPU in 1.58934s (22.6509GiB/s) | 20260507 15:50:28.789 | INFO | Downloaded 34GiB to GPU in 1.5365s (22.1282GiB/s) | 20260507 15:50:33.928 | INFO | Analyzing chunk frames = [707584,708096) | 20260507 15:50:33.928 | INFO | MemInfo: { | 20260507 15:50:33.928 | INFO | "gpu" : " 17 GB", | 20260507 15:50:33.928 | INFO | "huge" : "", | 20260507 15:50:33.928 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:33.929 | INFO | "malloc" : " 14 GB", | 20260507 15:50:33.929 | INFO | "pinned" : " 29 GB" | 20260507 15:50:33.929 | INFO | } | 20260507 15:50:33.929 | INFO | Perf report for the last 5.13957 seconds: | 20260507 15:50:33.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:33.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:50:33.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 15:50:33.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:33.929 | INFO | Uploaded 36GiB to GPU in 1.58932s (22.6511GiB/s) | 20260507 15:50:33.929 | INFO | Downloaded 34GiB to GPU in 1.53929s (22.0881GiB/s) | 20260507 15:50:39.067 | INFO | Analyzing chunk frames = [708096,708608) | 20260507 15:50:39.067 | INFO | MemInfo: { | 20260507 15:50:39.067 | INFO | "gpu" : " 17 GB", | 20260507 15:50:39.068 | INFO | "huge" : "", | 20260507 15:50:39.068 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:39.068 | INFO | "malloc" : " 14 GB", | 20260507 15:50:39.068 | INFO | "pinned" : " 29 GB" | 20260507 15:50:39.068 | INFO | } | 20260507 15:50:39.068 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:50:39.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:39.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260507 15:50:39.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | PRE_HQ : 40.0%, 1.0, 20.7ms, 0 | 20260507 15:50:39.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:39.068 | INFO | Uploaded 36GiB to GPU in 1.59541s (22.5648GiB/s) | 20260507 15:50:39.069 | INFO | Downloaded 34GiB to GPU in 1.54328s (22.031GiB/s) | 20260507 15:50:39.179 | INFO | Starting a round of Baz IO | 20260507 15:50:42.073 | INFO | Populating RTMetrics report | 20260507 15:50:42.074 | INFO | RTMetrics report populated | 20260507 15:50:42.074 | INFO | Opening tmp RTMetrics json file | 20260507 15:50:42.107 | INFO | Serializing RTMetrics to json | 20260507 15:50:42.137 | INFO | Serialization done, swapping tmp file now... | 20260507 15:50:42.145 | INFO | RTMetrics done | 20260507 15:50:42.209 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.029 seconds | 20260507 15:50:42.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.95% / 0.04% / 48.01% | 20260507 15:50:42.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.24789% / 8.34465e-05% / 2.78155e-05% | 20260507 15:50:42.209 | INFO | Baz IO overall: | 20260507 15:50:42.209 | INFO | BytesWritten : 295399856221 | 20260507 15:50:42.209 | INFO | NumEvents : 142613204111 | 20260507 15:50:44.207 | INFO | Analyzing chunk frames = [708608,709120) | 20260507 15:50:44.207 | INFO | MemInfo: { | 20260507 15:50:44.207 | INFO | "gpu" : " 17 GB", | 20260507 15:50:44.207 | INFO | "huge" : "", | 20260507 15:50:44.207 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:44.207 | INFO | "malloc" : " 14 GB", | 20260507 15:50:44.207 | INFO | "pinned" : " 29 GB" | 20260507 15:50:44.207 | INFO | } | 20260507 15:50:44.207 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:50:44.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:44.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:44.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:44.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:44.207 | INFO | ANALYSIS : 57.6%, 2.8, 85.3ms, 0 | 20260507 15:50:44.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:44.207 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 15:50:44.207 | INFO | PRE_HQ : 56.7%, 2.2, 64.2ms, 0 | 20260507 15:50:44.207 | INFO | BAZWRITER : 58.5%, 4.7, 140.3ms, 0 | 20260507 15:50:44.207 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 15:50:44.208 | INFO | Uploaded 36GiB to GPU in 1.65644s (21.7333GiB/s) | 20260507 15:50:44.208 | INFO | Downloaded 38GiB to GPU in 1.73831s (21.8603GiB/s) | 20260507 15:50:49.346 | INFO | Analyzing chunk frames = [709120,709632) | 20260507 15:50:49.346 | INFO | MemInfo: { | 20260507 15:50:49.346 | INFO | "gpu" : " 17 GB", | 20260507 15:50:49.346 | INFO | "huge" : "", | 20260507 15:50:49.346 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:49.346 | INFO | "malloc" : " 14 GB", | 20260507 15:50:49.346 | INFO | "pinned" : " 29 GB" | 20260507 15:50:49.346 | INFO | } | 20260507 15:50:49.346 | INFO | Perf report for the last 5.13894 seconds: | 20260507 15:50:49.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:49.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.346 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 15:50:49.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.346 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:50:49.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:49.347 | INFO | Uploaded 36GiB to GPU in 1.58569s (22.7031GiB/s) | 20260507 15:50:49.347 | INFO | Downloaded 34GiB to GPU in 1.53451s (22.157GiB/s) | 20260507 15:50:54.485 | INFO | Analyzing chunk frames = [709632,710144) | 20260507 15:50:54.486 | INFO | MemInfo: { | 20260507 15:50:54.486 | INFO | "gpu" : " 17 GB", | 20260507 15:50:54.486 | INFO | "huge" : "", | 20260507 15:50:54.486 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:54.486 | INFO | "malloc" : " 14 GB", | 20260507 15:50:54.486 | INFO | "pinned" : " 29 GB" | 20260507 15:50:54.486 | INFO | } | 20260507 15:50:54.486 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:50:54.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:54.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:50:54.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:50:54.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:54.486 | INFO | Uploaded 36GiB to GPU in 1.58986s (22.6435GiB/s) | 20260507 15:50:54.486 | INFO | Downloaded 34GiB to GPU in 1.53873s (22.0961GiB/s) | 20260507 15:50:59.625 | INFO | Analyzing chunk frames = [710144,710656) | 20260507 15:50:59.625 | INFO | MemInfo: { | 20260507 15:50:59.625 | INFO | "gpu" : " 17 GB", | 20260507 15:50:59.625 | INFO | "huge" : "", | 20260507 15:50:59.625 | INFO | "huge_shared" : " 29 GB", | 20260507 15:50:59.625 | INFO | "malloc" : " 14 GB", | 20260507 15:50:59.625 | INFO | "pinned" : " 29 GB" | 20260507 15:50:59.625 | INFO | } | 20260507 15:50:59.625 | INFO | Perf report for the last 5.13939 seconds: | 20260507 15:50:59.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:50:59.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.625 | INFO | ANALYSIS : 55.9%, 3.0, 86.7ms, 0 | 20260507 15:50:59.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.625 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:50:59.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:50:59.626 | INFO | Uploaded 36GiB to GPU in 1.58922s (22.6526GiB/s) | 20260507 15:50:59.626 | INFO | Downloaded 34GiB to GPU in 1.53999s (22.0781GiB/s) | 20260507 15:51:04.764 | INFO | Analyzing chunk frames = [710656,711168) | 20260507 15:51:04.764 | INFO | MemInfo: { | 20260507 15:51:04.764 | INFO | "gpu" : " 17 GB", | 20260507 15:51:04.764 | INFO | "huge" : "", | 20260507 15:51:04.764 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:04.764 | INFO | "malloc" : " 14 GB", | 20260507 15:51:04.764 | INFO | "pinned" : " 29 GB" | 20260507 15:51:04.764 | INFO | } | 20260507 15:51:04.764 | INFO | Perf report for the last 5.13923 seconds: | 20260507 15:51:04.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:04.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.764 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 15:51:04.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.764 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:51:04.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:04.765 | INFO | Uploaded 36GiB to GPU in 1.587s (22.6843GiB/s) | 20260507 15:51:04.765 | INFO | Downloaded 34GiB to GPU in 1.53642s (22.1294GiB/s) | 20260507 15:51:09.903 | INFO | Analyzing chunk frames = [711168,711680) | 20260507 15:51:09.903 | INFO | MemInfo: { | 20260507 15:51:09.903 | INFO | "gpu" : " 17 GB", | 20260507 15:51:09.903 | INFO | "huge" : "", | 20260507 15:51:09.904 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:09.904 | INFO | "malloc" : " 14 GB", | 20260507 15:51:09.904 | INFO | "pinned" : " 29 GB" | 20260507 15:51:09.904 | INFO | } | 20260507 15:51:09.904 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:51:09.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:09.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:51:09.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:51:09.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:09.904 | INFO | Uploaded 36GiB to GPU in 1.58859s (22.6616GiB/s) | 20260507 15:51:09.904 | INFO | Downloaded 34GiB to GPU in 1.53694s (22.1219GiB/s) | 20260507 15:51:15.043 | INFO | Analyzing chunk frames = [711680,712192) | 20260507 15:51:15.043 | INFO | MemInfo: { | 20260507 15:51:15.043 | INFO | "gpu" : " 17 GB", | 20260507 15:51:15.043 | INFO | "huge" : "", | 20260507 15:51:15.043 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:15.043 | INFO | "malloc" : " 14 GB", | 20260507 15:51:15.043 | INFO | "pinned" : " 29 GB" | 20260507 15:51:15.043 | INFO | } | 20260507 15:51:15.044 | INFO | Perf report for the last 5.13984 seconds: | 20260507 15:51:15.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:15.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:51:15.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 15:51:15.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:15.044 | INFO | Uploaded 36GiB to GPU in 1.58937s (22.6505GiB/s) | 20260507 15:51:15.044 | INFO | Downloaded 34GiB to GPU in 1.53912s (22.0905GiB/s) | 20260507 15:51:20.182 | INFO | Analyzing chunk frames = [712192,712704) | 20260507 15:51:20.182 | INFO | MemInfo: { | 20260507 15:51:20.182 | INFO | "gpu" : " 17 GB", | 20260507 15:51:20.182 | INFO | "huge" : "", | 20260507 15:51:20.183 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:20.183 | INFO | "malloc" : " 14 GB", | 20260507 15:51:20.183 | INFO | "pinned" : " 29 GB" | 20260507 15:51:20.183 | INFO | } | 20260507 15:51:20.183 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:51:20.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:20.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | ANALYSIS : 56.7%, 3.0, 88.3ms, 0 | 20260507 15:51:20.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 15:51:20.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:20.183 | INFO | Uploaded 36GiB to GPU in 1.59493s (22.5715GiB/s) | 20260507 15:51:20.183 | INFO | Downloaded 34GiB to GPU in 1.54895s (21.9503GiB/s) | 20260507 15:51:20.183 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:51:20.183 | INFO | Total mem usage: 16.4 GiB | 20260507 15:51:20.183 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:51:20.183 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.93%) | 20260507 15:51:20.183 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:51:20.183 | INFO | High Water Breakdown: | 20260507 15:51:20.183 | INFO | 59.4% : 8.95 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:51:20.183 | INFO | 40.4% : 6.09 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:51:20.183 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:51:20.183 | INFO | ------------------------------------------ | 20260507 15:51:20.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:51:20.183 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:51:20.184 | INFO | Total mem usage: 26.5 GiB | 20260507 15:51:20.184 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:51:20.184 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 15:51:20.184 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:51:20.184 | INFO | High Water Breakdown: | 20260507 15:51:20.184 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:51:20.184 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:51:20.184 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:51:20.184 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:51:20.184 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:51:20.184 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:51:20.184 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:51:20.184 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:51:20.184 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:51:20.184 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:51:20.184 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:51:20.184 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:51:20.184 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:51:20.184 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:51:20.184 | INFO | ------------------------------------------ | 20260507 15:51:20.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:51:20.184 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:51:20.184 | INFO | Total mem usage: 27.8 GiB | 20260507 15:51:20.184 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:51:20.184 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:51:20.184 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:51:20.184 | INFO | High Water Breakdown: | 20260507 15:51:20.184 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:51:20.184 | INFO | ------------------------------------------ | 20260507 15:51:20.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:51:20.184 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:51:20.184 | INFO | Total mem usage: 15.7 GiB | 20260507 15:51:20.184 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:51:20.184 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:51:20.184 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:51:20.184 | INFO | High Water Breakdown: | 20260507 15:51:20.184 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:51:20.184 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:51:20.184 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:51:20.184 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:51:20.184 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:51:20.184 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:51:20.184 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:51:20.184 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:51:20.184 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:51:20.184 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:51:20.184 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:51:20.184 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:51:20.184 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:51:20.184 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:51:20.184 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:51:20.184 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:51:20.184 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:51:20.184 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:51:20.184 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:51:20.184 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:51:20.184 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:51:20.184 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:51:20.184 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:51:20.184 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:51:20.184 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:51:20.184 | INFO | ------------------------------------------ | 20260507 15:51:20.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:51:20.185 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:51:20.185 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:51:20.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:51:20.185 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 0.29%, AggregateStats -- 8.77e-02 / -nan, 5.02e-03 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 11.06%, Baseline -- 3.30e+00 / -nan, 2.10e-02 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 3.08%, Histogram -- 9.19e-01 / -nan, 5.30e-02 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 2.96e+00 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.72e+01 / -nan, 4.15e-02 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.19e-01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 3.42%, Metrics -- 1.02e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.75e-01 / -nan, 5.52e-02 / 0.00e+00 | 20260507 15:51:20.185 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:51:20.185 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:51:20.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:51:20.185 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 18.66%, Upload -- 1.66e+01 / 0.00e+00, 4.70e-01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 18.26%, Download -- 1.63e+01 / 0.00e+00, 7.68e-01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 33.46%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 0.57%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 0.00%, LockDownload -- 4.69e-04 / 0.00e+00, 2.61e-04 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 29.04%, LockCompute -- 2.59e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:51:20.185 | INFO | PerfReport: 0.02%, Misc -- 1.41e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 15:51:20.296 | INFO | Starting a round of Baz IO | 20260507 15:51:23.270 | INFO | Populating RTMetrics report | 20260507 15:51:23.270 | INFO | RTMetrics report populated | 20260507 15:51:23.270 | INFO | Opening tmp RTMetrics json file | 20260507 15:51:23.309 | INFO | Serializing RTMetrics to json | 20260507 15:51:23.314 | INFO | Serialization done, swapping tmp file now... | 20260507 15:51:23.317 | INFO | RTMetrics done | 20260507 15:51:23.576 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.279 seconds | 20260507 15:51:23.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.91% / 0.04% / 48.05% | 20260507 15:51:23.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.23246% / 6.75519e-05% / 3.97364e-05% | 20260507 15:51:23.576 | INFO | Baz IO overall: | 20260507 15:51:23.576 | INFO | BytesWritten : 297297585520 | 20260507 15:51:23.576 | INFO | NumEvents : 143537245571 | 20260507 15:51:25.322 | INFO | Analyzing chunk frames = [712704,713216) | 20260507 15:51:25.322 | INFO | MemInfo: { | 20260507 15:51:25.322 | INFO | "gpu" : " 17 GB", | 20260507 15:51:25.322 | INFO | "huge" : "", | 20260507 15:51:25.322 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:25.322 | INFO | "malloc" : " 14 GB", | 20260507 15:51:25.322 | INFO | "pinned" : " 29 GB" | 20260507 15:51:25.322 | INFO | } | 20260507 15:51:25.322 | INFO | Perf report for the last 5.13893 seconds: | 20260507 15:51:25.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:25.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:25.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:25.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:25.322 | INFO | ANALYSIS : 58.7%, 2.7, 82.7ms, 0 | 20260507 15:51:25.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:25.322 | INFO | RT_METRICS : 50.3%, 1.0, 26.4ms, 0 | 20260507 15:51:25.322 | INFO | PRE_HQ : 58.2%, 2.3, 69.2ms, 0 | 20260507 15:51:25.322 | INFO | BAZWRITER : 63.4%, 4.8, 155.5ms, 0 | 20260507 15:51:25.322 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 15:51:25.322 | INFO | Uploaded 36GiB to GPU in 1.64446s (21.8916GiB/s) | 20260507 15:51:25.323 | INFO | Downloaded 38GiB to GPU in 1.72971s (21.969GiB/s) | 20260507 15:51:30.461 | INFO | Analyzing chunk frames = [713216,713728) | 20260507 15:51:30.461 | INFO | MemInfo: { | 20260507 15:51:30.461 | INFO | "gpu" : " 17 GB", | 20260507 15:51:30.461 | INFO | "huge" : "", | 20260507 15:51:30.461 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:30.461 | INFO | "malloc" : " 14 GB", | 20260507 15:51:30.461 | INFO | "pinned" : " 29 GB" | 20260507 15:51:30.461 | INFO | } | 20260507 15:51:30.461 | INFO | Perf report for the last 5.1397 seconds: | 20260507 15:51:30.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:30.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.461 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:51:30.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.462 | INFO | PRE_HQ : 38.4%, 1.0, 20.0ms, 0 | 20260507 15:51:30.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:30.462 | INFO | Uploaded 36GiB to GPU in 1.61219s (22.3298GiB/s) | 20260507 15:51:30.462 | INFO | Downloaded 34GiB to GPU in 1.52996s (22.2228GiB/s) | 20260507 15:51:35.600 | INFO | Analyzing chunk frames = [713728,714240) | 20260507 15:51:35.600 | INFO | MemInfo: { | 20260507 15:51:35.600 | INFO | "gpu" : " 17 GB", | 20260507 15:51:35.600 | INFO | "huge" : "", | 20260507 15:51:35.600 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:35.600 | INFO | "malloc" : " 14 GB", | 20260507 15:51:35.601 | INFO | "pinned" : " 29 GB" | 20260507 15:51:35.601 | INFO | } | 20260507 15:51:35.601 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:51:35.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:35.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:51:35.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:51:35.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:35.601 | INFO | Uploaded 36GiB to GPU in 1.61619s (22.2746GiB/s) | 20260507 15:51:35.601 | INFO | Downloaded 34GiB to GPU in 1.53536s (22.1446GiB/s) | 20260507 15:51:40.740 | INFO | Analyzing chunk frames = [714240,714752) | 20260507 15:51:40.740 | INFO | MemInfo: { | 20260507 15:51:40.740 | INFO | "gpu" : " 17 GB", | 20260507 15:51:40.740 | INFO | "huge" : "", | 20260507 15:51:40.740 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:40.740 | INFO | "malloc" : " 14 GB", | 20260507 15:51:40.740 | INFO | "pinned" : " 29 GB" | 20260507 15:51:40.741 | INFO | } | 20260507 15:51:40.741 | INFO | Perf report for the last 5.13991 seconds: | 20260507 15:51:40.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:40.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:51:40.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:51:40.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:40.741 | INFO | Uploaded 36GiB to GPU in 1.61454s (22.2973GiB/s) | 20260507 15:51:40.741 | INFO | Downloaded 34GiB to GPU in 1.53565s (22.1404GiB/s) | 20260507 15:51:45.879 | INFO | Analyzing chunk frames = [714752,715264) | 20260507 15:51:45.879 | INFO | MemInfo: { | 20260507 15:51:45.879 | INFO | "gpu" : " 17 GB", | 20260507 15:51:45.879 | INFO | "huge" : "", | 20260507 15:51:45.879 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:45.879 | INFO | "malloc" : " 14 GB", | 20260507 15:51:45.879 | INFO | "pinned" : " 29 GB" | 20260507 15:51:45.879 | INFO | } | 20260507 15:51:45.879 | INFO | Perf report for the last 5.13877 seconds: | 20260507 15:51:45.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:45.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.879 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:51:45.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.880 | INFO | PRE_HQ : 38.6%, 1.0, 20.0ms, 0 | 20260507 15:51:45.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:45.880 | INFO | Uploaded 36GiB to GPU in 1.61359s (22.3105GiB/s) | 20260507 15:51:45.880 | INFO | Downloaded 34GiB to GPU in 1.53142s (22.2016GiB/s) | 20260507 15:51:51.018 | INFO | Analyzing chunk frames = [715264,715776) | 20260507 15:51:51.018 | INFO | MemInfo: { | 20260507 15:51:51.018 | INFO | "gpu" : " 17 GB", | 20260507 15:51:51.019 | INFO | "huge" : "", | 20260507 15:51:51.019 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:51.019 | INFO | "malloc" : " 14 GB", | 20260507 15:51:51.019 | INFO | "pinned" : " 29 GB" | 20260507 15:51:51.019 | INFO | } | 20260507 15:51:51.019 | INFO | Perf report for the last 5.1398 seconds: | 20260507 15:51:51.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:51.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:51:51.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 15:51:51.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:51.019 | INFO | Uploaded 36GiB to GPU in 1.62106s (22.2076GiB/s) | 20260507 15:51:51.019 | INFO | Downloaded 34GiB to GPU in 1.54018s (22.0753GiB/s) | 20260507 15:51:56.158 | INFO | Analyzing chunk frames = [715776,716288) | 20260507 15:51:56.158 | INFO | MemInfo: { | 20260507 15:51:56.158 | INFO | "gpu" : " 17 GB", | 20260507 15:51:56.158 | INFO | "huge" : "", | 20260507 15:51:56.158 | INFO | "huge_shared" : " 29 GB", | 20260507 15:51:56.158 | INFO | "malloc" : " 14 GB", | 20260507 15:51:56.158 | INFO | "pinned" : " 29 GB" | 20260507 15:51:56.158 | INFO | } | 20260507 15:51:56.158 | INFO | Perf report for the last 5.13879 seconds: | 20260507 15:51:56.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:51:56.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.158 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:51:56.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.158 | INFO | PRE_HQ : 39.7%, 1.0, 20.6ms, 0 | 20260507 15:51:56.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:51:56.159 | INFO | Uploaded 36GiB to GPU in 1.61398s (22.3051GiB/s) | 20260507 15:51:56.159 | INFO | Downloaded 34GiB to GPU in 1.53409s (22.163GiB/s) | 20260507 15:52:01.297 | INFO | Analyzing chunk frames = [716288,716800) | 20260507 15:52:01.297 | INFO | MemInfo: { | 20260507 15:52:01.297 | INFO | "gpu" : " 17 GB", | 20260507 15:52:01.297 | INFO | "huge" : "", | 20260507 15:52:01.297 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:01.297 | INFO | "malloc" : " 14 GB", | 20260507 15:52:01.297 | INFO | "pinned" : " 29 GB" | 20260507 15:52:01.297 | INFO | } | 20260507 15:52:01.297 | INFO | Perf report for the last 5.13895 seconds: | 20260507 15:52:01.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:01.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.297 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:52:01.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.297 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:52:01.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:01.298 | INFO | Uploaded 36GiB to GPU in 1.61381s (22.3074GiB/s) | 20260507 15:52:01.298 | INFO | Downloaded 34GiB to GPU in 1.53446s (22.1577GiB/s) | 20260507 15:52:01.394 | INFO | Starting a round of Baz IO | 20260507 15:52:04.272 | INFO | Populating RTMetrics report | 20260507 15:52:04.272 | INFO | RTMetrics report populated | 20260507 15:52:04.272 | INFO | Opening tmp RTMetrics json file | 20260507 15:52:04.283 | INFO | Serializing RTMetrics to json | 20260507 15:52:04.286 | INFO | Serialization done, swapping tmp file now... | 20260507 15:52:04.295 | INFO | RTMetrics done | 20260507 15:52:04.377 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 2.982 seconds | 20260507 15:52:04.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.88% / 0.04% / 48.09% | 20260507 15:52:04.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.20389% / 8.34465e-05% / 1.98682e-05% | 20260507 15:52:04.377 | INFO | Baz IO overall: | 20260507 15:52:04.377 | INFO | BytesWritten : 299194479520 | 20260507 15:52:04.378 | INFO | NumEvents : 144460226591 | 20260507 15:52:06.437 | INFO | Analyzing chunk frames = [716800,717312) | 20260507 15:52:06.437 | INFO | MemInfo: { | 20260507 15:52:06.437 | INFO | "gpu" : " 17 GB", | 20260507 15:52:06.437 | INFO | "huge" : "", | 20260507 15:52:06.437 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:06.437 | INFO | "malloc" : " 14 GB", | 20260507 15:52:06.437 | INFO | "pinned" : " 29 GB" | 20260507 15:52:06.437 | INFO | } | 20260507 15:52:06.437 | INFO | Perf report for the last 5.13986 seconds: | 20260507 15:52:06.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:06.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:06.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:06.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:06.437 | INFO | ANALYSIS : 57.0%, 2.9, 87.8ms, 0 | 20260507 15:52:06.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:06.437 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 15:52:06.437 | INFO | PRE_HQ : 55.7%, 2.1, 59.4ms, 0 | 20260507 15:52:06.437 | INFO | BAZWRITER : 57.6%, 4.3, 127.7ms, 0 | 20260507 15:52:06.437 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 15:52:06.437 | INFO | Uploaded 36GiB to GPU in 1.64217s (21.9222GiB/s) | 20260507 15:52:06.438 | INFO | Downloaded 38GiB to GPU in 1.72413s (22.0401GiB/s) | 20260507 15:52:11.576 | INFO | Analyzing chunk frames = [717312,717824) | 20260507 15:52:11.576 | INFO | MemInfo: { | 20260507 15:52:11.576 | INFO | "gpu" : " 17 GB", | 20260507 15:52:11.576 | INFO | "huge" : "", | 20260507 15:52:11.576 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:11.576 | INFO | "malloc" : " 14 GB", | 20260507 15:52:11.576 | INFO | "pinned" : " 29 GB" | 20260507 15:52:11.576 | INFO | } | 20260507 15:52:11.576 | INFO | Perf report for the last 5.13906 seconds: | 20260507 15:52:11.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:11.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 15:52:11.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:52:11.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:11.576 | INFO | Uploaded 36GiB to GPU in 1.6059s (22.4173GiB/s) | 20260507 15:52:11.576 | INFO | Downloaded 34GiB to GPU in 1.52598s (22.2808GiB/s) | 20260507 15:52:16.715 | INFO | Analyzing chunk frames = [717824,718336) | 20260507 15:52:16.715 | INFO | MemInfo: { | 20260507 15:52:16.715 | INFO | "gpu" : " 17 GB", | 20260507 15:52:16.715 | INFO | "huge" : "", | 20260507 15:52:16.715 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:16.715 | INFO | "malloc" : " 14 GB", | 20260507 15:52:16.715 | INFO | "pinned" : " 29 GB" | 20260507 15:52:16.715 | INFO | } | 20260507 15:52:16.716 | INFO | Perf report for the last 5.14003 seconds: | 20260507 15:52:16.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:16.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 15:52:16.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:52:16.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:16.716 | INFO | Uploaded 36GiB to GPU in 1.61057s (22.3523GiB/s) | 20260507 15:52:16.716 | INFO | Downloaded 34GiB to GPU in 1.53116s (22.2054GiB/s) | 20260507 15:52:21.854 | INFO | Analyzing chunk frames = [718336,718848) | 20260507 15:52:21.854 | INFO | MemInfo: { | 20260507 15:52:21.854 | INFO | "gpu" : " 17 GB", | 20260507 15:52:21.855 | INFO | "huge" : "", | 20260507 15:52:21.855 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:21.855 | INFO | "malloc" : " 14 GB", | 20260507 15:52:21.855 | INFO | "pinned" : " 29 GB" | 20260507 15:52:21.855 | INFO | } | 20260507 15:52:21.855 | INFO | Perf report for the last 5.13879 seconds: | 20260507 15:52:21.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:21.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 15:52:21.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:52:21.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:21.855 | INFO | Uploaded 36GiB to GPU in 1.60991s (22.3615GiB/s) | 20260507 15:52:21.855 | INFO | Downloaded 34GiB to GPU in 1.53021s (22.2191GiB/s) | 20260507 15:52:26.994 | INFO | Analyzing chunk frames = [718848,719360) | 20260507 15:52:26.994 | INFO | MemInfo: { | 20260507 15:52:26.994 | INFO | "gpu" : " 17 GB", | 20260507 15:52:26.995 | INFO | "huge" : "", | 20260507 15:52:26.995 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:26.995 | INFO | "malloc" : " 14 GB", | 20260507 15:52:26.995 | INFO | "pinned" : " 29 GB" | 20260507 15:52:26.995 | INFO | } | 20260507 15:52:26.995 | INFO | Perf report for the last 5.13986 seconds: | 20260507 15:52:26.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:26.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:52:26.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:52:26.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:26.995 | INFO | Uploaded 36GiB to GPU in 1.60814s (22.3861GiB/s) | 20260507 15:52:26.995 | INFO | Downloaded 34GiB to GPU in 1.5277s (22.2557GiB/s) | 20260507 15:52:32.133 | INFO | Analyzing chunk frames = [719360,719872) | 20260507 15:52:32.133 | INFO | MemInfo: { | 20260507 15:52:32.133 | INFO | "gpu" : " 17 GB", | 20260507 15:52:32.133 | INFO | "huge" : "", | 20260507 15:52:32.133 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:32.133 | INFO | "malloc" : " 14 GB", | 20260507 15:52:32.133 | INFO | "pinned" : " 29 GB" | 20260507 15:52:32.133 | INFO | } | 20260507 15:52:32.133 | INFO | Perf report for the last 5.13877 seconds: | 20260507 15:52:32.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:32.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:52:32.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:52:32.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:32.134 | INFO | Uploaded 36GiB to GPU in 1.60844s (22.3819GiB/s) | 20260507 15:52:32.134 | INFO | Downloaded 34GiB to GPU in 1.52915s (22.2345GiB/s) | 20260507 15:52:37.272 | INFO | Analyzing chunk frames = [719872,720384) | 20260507 15:52:37.272 | INFO | MemInfo: { | 20260507 15:52:37.272 | INFO | "gpu" : " 17 GB", | 20260507 15:52:37.273 | INFO | "huge" : "", | 20260507 15:52:37.273 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:37.273 | INFO | "malloc" : " 14 GB", | 20260507 15:52:37.273 | INFO | "pinned" : " 29 GB" | 20260507 15:52:37.273 | INFO | } | 20260507 15:52:37.273 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:52:37.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:37.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:52:37.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 15:52:37.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:37.273 | INFO | Uploaded 36GiB to GPU in 1.6155s (22.2842GiB/s) | 20260507 15:52:37.273 | INFO | Downloaded 34GiB to GPU in 1.53437s (22.159GiB/s) | 20260507 15:52:42.412 | INFO | Analyzing chunk frames = [720384,720896) | 20260507 15:52:42.412 | INFO | MemInfo: { | 20260507 15:52:42.412 | INFO | "gpu" : " 17 GB", | 20260507 15:52:42.412 | INFO | "huge" : "", | 20260507 15:52:42.412 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:42.412 | INFO | "malloc" : " 14 GB", | 20260507 15:52:42.412 | INFO | "pinned" : " 29 GB" | 20260507 15:52:42.412 | INFO | } | 20260507 15:52:42.412 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:52:42.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:42.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:52:42.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:52:42.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:42.412 | INFO | Uploaded 36GiB to GPU in 1.61385s (22.3069GiB/s) | 20260507 15:52:42.413 | INFO | Downloaded 34GiB to GPU in 1.53385s (22.1665GiB/s) | 20260507 15:52:42.413 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:52:42.413 | INFO | Total mem usage: 16.4 GiB | 20260507 15:52:42.413 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:52:42.413 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.56%) | 20260507 15:52:42.413 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:52:42.413 | INFO | High Water Breakdown: | 20260507 15:52:42.413 | INFO | 59.6% : 9.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:52:42.413 | INFO | 40.3% : 6.14 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:52:42.413 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:52:42.413 | INFO | ------------------------------------------ | 20260507 15:52:42.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:52:42.413 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:52:42.413 | INFO | Total mem usage: 26.5 GiB | 20260507 15:52:42.413 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:52:42.413 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 15:52:42.413 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:52:42.413 | INFO | High Water Breakdown: | 20260507 15:52:42.413 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:52:42.413 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:52:42.413 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:52:42.413 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:52:42.413 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:52:42.413 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:52:42.413 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:52:42.413 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:52:42.413 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:52:42.413 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:52:42.413 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:52:42.413 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:52:42.413 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:52:42.413 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:52:42.413 | INFO | ------------------------------------------ | 20260507 15:52:42.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:52:42.413 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:52:42.413 | INFO | Total mem usage: 27.8 GiB | 20260507 15:52:42.413 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:52:42.413 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:52:42.413 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:52:42.413 | INFO | High Water Breakdown: | 20260507 15:52:42.413 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:52:42.413 | INFO | ------------------------------------------ | 20260507 15:52:42.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:52:42.413 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:52:42.413 | INFO | Total mem usage: 15.7 GiB | 20260507 15:52:42.413 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:52:42.413 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:52:42.413 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:52:42.413 | INFO | High Water Breakdown: | 20260507 15:52:42.413 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:52:42.413 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:52:42.414 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:52:42.414 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:52:42.414 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:52:42.414 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:52:42.414 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:52:42.414 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:52:42.414 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:52:42.414 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:52:42.414 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:52:42.414 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:52:42.414 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:52:42.414 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:52:42.414 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:52:42.414 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:52:42.414 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:52:42.414 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:52:42.414 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:52:42.414 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:52:42.414 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:52:42.414 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:52:42.414 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:52:42.414 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:52:42.414 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:52:42.414 | INFO | ------------------------------------------ | 20260507 15:52:42.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:52:42.414 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:52:42.414 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:52:42.414 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:52:42.414 | INFO | PerfReport: Full Execution -- 2.99e+01 / 7.18e+01, 1.25e+01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 0.30%, AggregateStats -- 9.04e-02 / 9.63e-02, 6.42e-03 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 11.08%, Baseline -- 3.31e+00 / 3.28e+00, 2.23e-02 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / 1.08e+00, 5.38e-02 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 16.14%, DME -- 3.86e+01 / 4.67e+01, 3.01e+00 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.72e+01 / 1.71e+01, 4.28e-02 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.14e+00 / 2.14e+00, 1.20e-01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 3.43%, Metrics -- 1.02e+00 / 9.52e-01, 1.63e-01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 1.28%, PulseSerializing -- 3.81e-01 / 3.87e-01, 5.54e-02 / 0.00e+00 | 20260507 15:52:42.414 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:52:42.414 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:52:42.414 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:52:42.414 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 18.87%, Upload -- 1.68e+01 / 0.00e+00, 4.52e-01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 18.17%, Download -- 1.62e+01 / 0.00e+00, 7.91e-01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 33.46%, Compute -- 2.99e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 0.59%, LockUpload -- 5.22e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 0.00%, LockDownload -- 5.78e-04 / 0.00e+00, 4.19e-04 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 28.89%, LockCompute -- 2.58e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260507 15:52:42.414 | INFO | PerfReport: 0.02%, Misc -- 1.67e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 15:52:42.527 | INFO | Starting a round of Baz IO | 20260507 15:52:45.501 | INFO | Populating RTMetrics report | 20260507 15:52:45.501 | INFO | RTMetrics report populated | 20260507 15:52:45.501 | INFO | Opening tmp RTMetrics json file | 20260507 15:52:45.572 | INFO | Serializing RTMetrics to json | 20260507 15:52:45.577 | INFO | Serialization done, swapping tmp file now... | 20260507 15:52:45.581 | INFO | RTMetrics done | 20260507 15:52:45.795 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.268 seconds | 20260507 15:52:45.796 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.84% / 0.04% / 48.12% | 20260507 15:52:45.796 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.20717% / 5.5631e-05% / 2.38419e-05% | 20260507 15:52:45.796 | INFO | Baz IO overall: | 20260507 15:52:45.796 | INFO | BytesWritten : 301094890675 | 20260507 15:52:45.796 | INFO | NumEvents : 145385575315 | 20260507 15:52:47.551 | INFO | Analyzing chunk frames = [720896,721408) | 20260507 15:52:47.551 | INFO | MemInfo: { | 20260507 15:52:47.551 | INFO | "gpu" : " 17 GB", | 20260507 15:52:47.551 | INFO | "huge" : "", | 20260507 15:52:47.551 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:47.551 | INFO | "malloc" : " 14 GB", | 20260507 15:52:47.551 | INFO | "pinned" : " 29 GB" | 20260507 15:52:47.551 | INFO | } | 20260507 15:52:47.551 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:52:47.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:47.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:47.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:47.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:47.551 | INFO | ANALYSIS : 58.7%, 2.7, 82.9ms, 0 | 20260507 15:52:47.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:47.552 | INFO | RT_METRICS : 50.9%, 1.0, 26.7ms, 0 | 20260507 15:52:47.552 | INFO | PRE_HQ : 58.4%, 2.3, 69.7ms, 0 | 20260507 15:52:47.552 | INFO | BAZWRITER : 63.2%, 4.8, 155.2ms, 0 | 20260507 15:52:47.552 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 15:52:47.552 | INFO | Uploaded 36GiB to GPU in 1.65129s (21.8012GiB/s) | 20260507 15:52:47.552 | INFO | Downloaded 38GiB to GPU in 1.72879s (21.9807GiB/s) | 20260507 15:52:52.690 | INFO | Analyzing chunk frames = [721408,721920) | 20260507 15:52:52.690 | INFO | MemInfo: { | 20260507 15:52:52.690 | INFO | "gpu" : " 17 GB", | 20260507 15:52:52.690 | INFO | "huge" : "", | 20260507 15:52:52.691 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:52.691 | INFO | "malloc" : " 14 GB", | 20260507 15:52:52.691 | INFO | "pinned" : " 29 GB" | 20260507 15:52:52.691 | INFO | } | 20260507 15:52:52.691 | INFO | Perf report for the last 5.13951 seconds: | 20260507 15:52:52.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:52.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.691 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260507 15:52:52.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.691 | INFO | PRE_HQ : 39.0%, 1.0, 20.5ms, 0 | 20260507 15:52:52.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:52.692 | INFO | Uploaded 36GiB to GPU in 1.5919s (22.6144GiB/s) | 20260507 15:52:52.692 | INFO | Downloaded 34GiB to GPU in 1.53969s (22.0823GiB/s) | 20260507 15:52:57.830 | INFO | Analyzing chunk frames = [721920,722432) | 20260507 15:52:57.830 | INFO | MemInfo: { | 20260507 15:52:57.830 | INFO | "gpu" : " 17 GB", | 20260507 15:52:57.830 | INFO | "huge" : "", | 20260507 15:52:57.830 | INFO | "huge_shared" : " 29 GB", | 20260507 15:52:57.830 | INFO | "malloc" : " 14 GB", | 20260507 15:52:57.830 | INFO | "pinned" : " 29 GB" | 20260507 15:52:57.830 | INFO | } | 20260507 15:52:57.830 | INFO | Perf report for the last 5.13916 seconds: | 20260507 15:52:57.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:52:57.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.830 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:52:57.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.830 | INFO | PRE_HQ : 39.6%, 1.0, 20.5ms, 0 | 20260507 15:52:57.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:52:57.831 | INFO | Uploaded 36GiB to GPU in 1.61663s (22.2686GiB/s) | 20260507 15:52:57.831 | INFO | Downloaded 34GiB to GPU in 1.5354s (22.144GiB/s) | 20260507 15:53:02.969 | INFO | Analyzing chunk frames = [722432,722944) | 20260507 15:53:02.969 | INFO | MemInfo: { | 20260507 15:53:02.969 | INFO | "gpu" : " 17 GB", | 20260507 15:53:02.969 | INFO | "huge" : "", | 20260507 15:53:02.969 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:02.969 | INFO | "malloc" : " 14 GB", | 20260507 15:53:02.969 | INFO | "pinned" : " 29 GB" | 20260507 15:53:02.969 | INFO | } | 20260507 15:53:02.969 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:53:02.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:02.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.970 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:53:02.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.970 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 15:53:02.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:02.970 | INFO | Uploaded 36GiB to GPU in 1.6159s (22.2787GiB/s) | 20260507 15:53:02.970 | INFO | Downloaded 34GiB to GPU in 1.5348s (22.1527GiB/s) | 20260507 15:53:08.108 | INFO | Analyzing chunk frames = [722944,723456) | 20260507 15:53:08.109 | INFO | MemInfo: { | 20260507 15:53:08.109 | INFO | "gpu" : " 17 GB", | 20260507 15:53:08.109 | INFO | "huge" : "", | 20260507 15:53:08.109 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:08.109 | INFO | "malloc" : " 14 GB", | 20260507 15:53:08.109 | INFO | "pinned" : " 29 GB" | 20260507 15:53:08.109 | INFO | } | 20260507 15:53:08.109 | INFO | Perf report for the last 5.13936 seconds: | 20260507 15:53:08.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:08.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:53:08.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:53:08.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:08.109 | INFO | Uploaded 36GiB to GPU in 1.61312s (22.317GiB/s) | 20260507 15:53:08.109 | INFO | Downloaded 34GiB to GPU in 1.53236s (22.188GiB/s) | 20260507 15:53:13.248 | INFO | Analyzing chunk frames = [723456,723968) | 20260507 15:53:13.248 | INFO | MemInfo: { | 20260507 15:53:13.248 | INFO | "gpu" : " 17 GB", | 20260507 15:53:13.248 | INFO | "huge" : "", | 20260507 15:53:13.248 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:13.248 | INFO | "malloc" : " 14 GB", | 20260507 15:53:13.248 | INFO | "pinned" : " 29 GB" | 20260507 15:53:13.248 | INFO | } | 20260507 15:53:13.248 | INFO | Perf report for the last 5.1393 seconds: | 20260507 15:53:13.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:13.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:53:13.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:53:13.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:13.248 | INFO | Uploaded 36GiB to GPU in 1.60821s (22.3851GiB/s) | 20260507 15:53:13.248 | INFO | Downloaded 34GiB to GPU in 1.52772s (22.2553GiB/s) | 20260507 15:53:18.387 | INFO | Analyzing chunk frames = [723968,724480) | 20260507 15:53:18.387 | INFO | MemInfo: { | 20260507 15:53:18.387 | INFO | "gpu" : " 17 GB", | 20260507 15:53:18.387 | INFO | "huge" : "", | 20260507 15:53:18.387 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:18.387 | INFO | "malloc" : " 14 GB", | 20260507 15:53:18.387 | INFO | "pinned" : " 29 GB" | 20260507 15:53:18.387 | INFO | } | 20260507 15:53:18.387 | INFO | Perf report for the last 5.13903 seconds: | 20260507 15:53:18.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:18.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.387 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:53:18.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.387 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:53:18.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:18.388 | INFO | Uploaded 36GiB to GPU in 1.60934s (22.3694GiB/s) | 20260507 15:53:18.388 | INFO | Downloaded 34GiB to GPU in 1.53036s (22.217GiB/s) | 20260507 15:53:23.527 | INFO | Analyzing chunk frames = [724480,724992) | 20260507 15:53:23.527 | INFO | MemInfo: { | 20260507 15:53:23.527 | INFO | "gpu" : " 17 GB", | 20260507 15:53:23.527 | INFO | "huge" : "", | 20260507 15:53:23.527 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:23.527 | INFO | "malloc" : " 14 GB", | 20260507 15:53:23.527 | INFO | "pinned" : " 29 GB" | 20260507 15:53:23.527 | INFO | } | 20260507 15:53:23.527 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:53:23.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:23.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:53:23.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:53:23.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:23.527 | INFO | Uploaded 36GiB to GPU in 1.60888s (22.3758GiB/s) | 20260507 15:53:23.527 | INFO | Downloaded 34GiB to GPU in 1.52947s (22.2299GiB/s) | 20260507 15:53:23.624 | INFO | Starting a round of Baz IO | 20260507 15:53:26.502 | INFO | Populating RTMetrics report | 20260507 15:53:26.502 | INFO | RTMetrics report populated | 20260507 15:53:26.502 | INFO | Opening tmp RTMetrics json file | 20260507 15:53:26.509 | INFO | Serializing RTMetrics to json | 20260507 15:53:26.513 | INFO | Serialization done, swapping tmp file now... | 20260507 15:53:26.516 | INFO | RTMetrics done | 20260507 15:53:26.735 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.111 seconds | 20260507 15:53:26.735 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.80% / 0.04% / 48.16% | 20260507 15:53:26.735 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.18215% / 5.5631e-05% / 1.58946e-05% | 20260507 15:53:26.735 | INFO | Baz IO overall: | 20260507 15:53:26.735 | INFO | BytesWritten : 302993894631 | 20260507 15:53:26.735 | INFO | NumEvents : 146309889901 | 20260507 15:53:28.666 | INFO | Analyzing chunk frames = [724992,725504) | 20260507 15:53:28.666 | INFO | MemInfo: { | 20260507 15:53:28.666 | INFO | "gpu" : " 17 GB", | 20260507 15:53:28.666 | INFO | "huge" : "", | 20260507 15:53:28.666 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:28.666 | INFO | "malloc" : " 14 GB", | 20260507 15:53:28.666 | INFO | "pinned" : " 29 GB" | 20260507 15:53:28.666 | INFO | } | 20260507 15:53:28.666 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:53:28.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:28.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:28.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:28.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:28.666 | INFO | ANALYSIS : 56.9%, 2.9, 85.6ms, 0 | 20260507 15:53:28.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:28.666 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 15:53:28.666 | INFO | PRE_HQ : 55.7%, 2.0, 58.9ms, 0 | 20260507 15:53:28.666 | INFO | BAZWRITER : 60.1%, 4.7, 145.4ms, 0 | 20260507 15:53:28.666 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 15:53:28.666 | INFO | Uploaded 36GiB to GPU in 1.66291s (21.6488GiB/s) | 20260507 15:53:28.667 | INFO | Downloaded 38GiB to GPU in 1.74765s (21.7435GiB/s) | 20260507 15:53:33.805 | INFO | Analyzing chunk frames = [725504,726016) | 20260507 15:53:33.805 | INFO | MemInfo: { | 20260507 15:53:33.805 | INFO | "gpu" : " 17 GB", | 20260507 15:53:33.805 | INFO | "huge" : "", | 20260507 15:53:33.805 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:33.805 | INFO | "malloc" : " 14 GB", | 20260507 15:53:33.805 | INFO | "pinned" : " 29 GB" | 20260507 15:53:33.805 | INFO | } | 20260507 15:53:33.805 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:53:33.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:33.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.806 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:53:33.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.806 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:53:33.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:33.806 | INFO | Uploaded 36GiB to GPU in 1.5863s (22.6944GiB/s) | 20260507 15:53:33.806 | INFO | Downloaded 34GiB to GPU in 1.53412s (22.1625GiB/s) | 20260507 15:53:38.945 | INFO | Analyzing chunk frames = [726016,726528) | 20260507 15:53:38.945 | INFO | MemInfo: { | 20260507 15:53:38.945 | INFO | "gpu" : " 17 GB", | 20260507 15:53:38.945 | INFO | "huge" : "", | 20260507 15:53:38.945 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:38.945 | INFO | "malloc" : " 14 GB", | 20260507 15:53:38.945 | INFO | "pinned" : " 29 GB" | 20260507 15:53:38.945 | INFO | } | 20260507 15:53:38.945 | INFO | Perf report for the last 5.13985 seconds: | 20260507 15:53:38.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:38.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.945 | INFO | ANALYSIS : 56.2%, 3.0, 87.4ms, 0 | 20260507 15:53:38.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.946 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 15:53:38.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:38.946 | INFO | Uploaded 36GiB to GPU in 1.5946s (22.5762GiB/s) | 20260507 15:53:38.946 | INFO | Downloaded 34GiB to GPU in 1.54269s (22.0395GiB/s) | 20260507 15:53:44.084 | INFO | Analyzing chunk frames = [726528,727040) | 20260507 15:53:44.084 | INFO | MemInfo: { | 20260507 15:53:44.084 | INFO | "gpu" : " 17 GB", | 20260507 15:53:44.084 | INFO | "huge" : "", | 20260507 15:53:44.085 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:44.085 | INFO | "malloc" : " 14 GB", | 20260507 15:53:44.085 | INFO | "pinned" : " 29 GB" | 20260507 15:53:44.085 | INFO | } | 20260507 15:53:44.085 | INFO | Perf report for the last 5.13946 seconds: | 20260507 15:53:44.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:44.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:53:44.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | PRE_HQ : 39.5%, 1.0, 20.5ms, 0 | 20260507 15:53:44.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:44.085 | INFO | Uploaded 36GiB to GPU in 1.59573s (22.5603GiB/s) | 20260507 15:53:44.086 | INFO | Downloaded 34GiB to GPU in 1.54421s (22.0177GiB/s) | 20260507 15:53:49.223 | INFO | Analyzing chunk frames = [727040,727552) | 20260507 15:53:49.223 | INFO | MemInfo: { | 20260507 15:53:49.223 | INFO | "gpu" : " 17 GB", | 20260507 15:53:49.223 | INFO | "huge" : "", | 20260507 15:53:49.223 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:49.224 | INFO | "malloc" : " 14 GB", | 20260507 15:53:49.224 | INFO | "pinned" : " 29 GB" | 20260507 15:53:49.224 | INFO | } | 20260507 15:53:49.224 | INFO | Perf report for the last 5.13878 seconds: | 20260507 15:53:49.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:49.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:53:49.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 15:53:49.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:49.224 | INFO | Uploaded 36GiB to GPU in 1.59448s (22.5778GiB/s) | 20260507 15:53:49.224 | INFO | Downloaded 34GiB to GPU in 1.54283s (22.0374GiB/s) | 20260507 15:53:54.363 | INFO | Analyzing chunk frames = [727552,728064) | 20260507 15:53:54.363 | INFO | MemInfo: { | 20260507 15:53:54.363 | INFO | "gpu" : " 17 GB", | 20260507 15:53:54.363 | INFO | "huge" : "", | 20260507 15:53:54.363 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:54.363 | INFO | "malloc" : " 14 GB", | 20260507 15:53:54.363 | INFO | "pinned" : " 29 GB" | 20260507 15:53:54.363 | INFO | } | 20260507 15:53:54.363 | INFO | Perf report for the last 5.13952 seconds: | 20260507 15:53:54.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:54.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.363 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:53:54.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.363 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 15:53:54.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:54.364 | INFO | Uploaded 36GiB to GPU in 1.5935s (22.5918GiB/s) | 20260507 15:53:54.364 | INFO | Downloaded 34GiB to GPU in 1.54271s (22.0392GiB/s) | 20260507 15:53:59.502 | INFO | Analyzing chunk frames = [728064,728576) | 20260507 15:53:59.502 | INFO | MemInfo: { | 20260507 15:53:59.502 | INFO | "gpu" : " 17 GB", | 20260507 15:53:59.502 | INFO | "huge" : "", | 20260507 15:53:59.502 | INFO | "huge_shared" : " 29 GB", | 20260507 15:53:59.502 | INFO | "malloc" : " 14 GB", | 20260507 15:53:59.502 | INFO | "pinned" : " 29 GB" | 20260507 15:53:59.502 | INFO | } | 20260507 15:53:59.502 | INFO | Perf report for the last 5.13907 seconds: | 20260507 15:53:59.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:53:59.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.502 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:53:59.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.502 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 15:53:59.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:53:59.503 | INFO | Uploaded 36GiB to GPU in 1.58799s (22.6701GiB/s) | 20260507 15:53:59.503 | INFO | Downloaded 34GiB to GPU in 1.53883s (22.0946GiB/s) | 20260507 15:54:04.641 | INFO | Analyzing chunk frames = [728576,729088) | 20260507 15:54:04.641 | INFO | MemInfo: { | 20260507 15:54:04.641 | INFO | "gpu" : " 17 GB", | 20260507 15:54:04.641 | INFO | "huge" : "", | 20260507 15:54:04.641 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:04.641 | INFO | "malloc" : " 14 GB", | 20260507 15:54:04.641 | INFO | "pinned" : " 29 GB" | 20260507 15:54:04.641 | INFO | } | 20260507 15:54:04.642 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:54:04.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:04.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:54:04.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 15:54:04.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:04.642 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:54:04.642 | INFO | Total mem usage: 16.4 GiB | 20260507 15:54:04.642 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:54:04.642 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.97%) | 20260507 15:54:04.642 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:54:04.642 | INFO | High Water Breakdown: | 20260507 15:54:04.642 | INFO | 59.6% : 9.05 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:54:04.642 | INFO | 40.3% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:54:04.642 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:54:04.642 | INFO | ------------------------------------------ | 20260507 15:54:04.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:54:04.642 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:54:04.642 | INFO | Total mem usage: 26.5 GiB | 20260507 15:54:04.642 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:54:04.642 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:54:04.642 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:54:04.642 | INFO | High Water Breakdown: | 20260507 15:54:04.642 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:54:04.642 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:54:04.642 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:54:04.642 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:54:04.642 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:54:04.642 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:54:04.642 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:54:04.642 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:54:04.642 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:54:04.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:54:04.642 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:54:04.642 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:54:04.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:54:04.642 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:54:04.642 | INFO | ------------------------------------------ | 20260507 15:54:04.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:54:04.642 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:54:04.642 | INFO | Total mem usage: 27.8 GiB | 20260507 15:54:04.642 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:54:04.642 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:54:04.642 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:54:04.642 | INFO | High Water Breakdown: | 20260507 15:54:04.642 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:54:04.642 | INFO | ------------------------------------------ | 20260507 15:54:04.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:54:04.642 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:54:04.642 | INFO | Total mem usage: 15.7 GiB | 20260507 15:54:04.642 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:54:04.642 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:54:04.642 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:54:04.642 | INFO | High Water Breakdown: | 20260507 15:54:04.642 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:54:04.642 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:54:04.642 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:54:04.642 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:54:04.642 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:54:04.642 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:54:04.642 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:54:04.642 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:54:04.642 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:54:04.642 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:54:04.642 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:54:04.642 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:54:04.642 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:54:04.642 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:54:04.642 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:54:04.642 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:54:04.642 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:54:04.642 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:54:04.642 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:54:04.642 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:54:04.642 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:54:04.642 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:54:04.642 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:54:04.642 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:54:04.642 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:54:04.642 | INFO | ------------------------------------------ | 20260507 15:54:04.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:54:04.642 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:54:04.642 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:54:04.642 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:54:04.642 | INFO | PerfReport: Full Execution -- 2.98e+01 / 7.04e+01, 1.26e+01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 0.30%, AggregateStats -- 8.98e-02 / 8.54e-02, 7.47e-03 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 11.07%, Baseline -- 3.30e+00 / 3.28e+00, 2.48e-02 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 3.08%, Histogram -- 9.21e-01 / 1.06e+00, 5.42e-02 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / 4.54e+01, 3.03e+00 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.72e+01 / 1.71e+01, 1.17e-01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.13e+00 / 2.14e+00, 1.19e-01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 3.42%, Metrics -- 1.02e+00 / 9.71e-01, 1.65e-01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.78e-01 / 3.89e-01, 5.54e-02 / 0.00e+00 | 20260507 15:54:04.642 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:54:04.642 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:54:04.642 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:54:04.642 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 18.80%, Upload -- 1.67e+01 / 0.00e+00, 5.01e-01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 18.26%, Download -- 1.63e+01 / 0.00e+00, 8.21e-01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 33.50%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 0.59%, LockUpload -- 5.22e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 0.00%, LockDownload -- 5.46e-04 / 0.00e+00, 3.75e-04 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 28.83%, LockCompute -- 2.57e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 15:54:04.642 | INFO | PerfReport: 0.02%, Misc -- 1.69e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260507 15:54:04.642 | INFO | Uploaded 36GiB to GPU in 1.58786s (22.672GiB/s) | 20260507 15:54:04.642 | INFO | Downloaded 34GiB to GPU in 1.53754s (22.1133GiB/s) | 20260507 15:54:04.739 | INFO | Starting a round of Baz IO | 20260507 15:54:07.662 | INFO | Populating RTMetrics report | 20260507 15:54:07.663 | INFO | RTMetrics report populated | 20260507 15:54:07.663 | INFO | Opening tmp RTMetrics json file | 20260507 15:54:07.706 | INFO | Serializing RTMetrics to json | 20260507 15:54:07.716 | INFO | Serialization done, swapping tmp file now... | 20260507 15:54:07.740 | INFO | RTMetrics done | 20260507 15:54:07.952 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.212 seconds | 20260507 15:54:07.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.77% / 0.04% / 48.20% | 20260507 15:54:07.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.16037% / 5.5631e-05% / 1.98682e-05% | 20260507 15:54:07.952 | INFO | Baz IO overall: | 20260507 15:54:07.952 | INFO | BytesWritten : 304892154786 | 20260507 15:54:07.952 | INFO | NumEvents : 147233726682 | 20260507 15:54:09.781 | INFO | Analyzing chunk frames = [729088,729600) | 20260507 15:54:09.781 | INFO | MemInfo: { | 20260507 15:54:09.781 | INFO | "gpu" : " 17 GB", | 20260507 15:54:09.781 | INFO | "huge" : "", | 20260507 15:54:09.781 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:09.781 | INFO | "malloc" : " 14 GB", | 20260507 15:54:09.781 | INFO | "pinned" : " 29 GB" | 20260507 15:54:09.781 | INFO | } | 20260507 15:54:09.781 | INFO | Perf report for the last 5.13934 seconds: | 20260507 15:54:09.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:09.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:09.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:09.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:09.781 | INFO | ANALYSIS : 57.9%, 2.8, 85.5ms, 0 | 20260507 15:54:09.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:09.781 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 15:54:09.781 | INFO | PRE_HQ : 56.8%, 2.2, 63.9ms, 0 | 20260507 15:54:09.781 | INFO | BAZWRITER : 62.1%, 4.5, 142.3ms, 0 | 20260507 15:54:09.781 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 15:54:09.782 | INFO | Uploaded 36GiB to GPU in 1.62978s (22.0889GiB/s) | 20260507 15:54:09.782 | INFO | Downloaded 38GiB to GPU in 1.7135s (22.1768GiB/s) | 20260507 15:54:14.920 | INFO | Analyzing chunk frames = [729600,730112) | 20260507 15:54:14.920 | INFO | MemInfo: { | 20260507 15:54:14.920 | INFO | "gpu" : " 17 GB", | 20260507 15:54:14.920 | INFO | "huge" : "", | 20260507 15:54:14.920 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:14.920 | INFO | "malloc" : " 14 GB", | 20260507 15:54:14.920 | INFO | "pinned" : " 29 GB" | 20260507 15:54:14.920 | INFO | } | 20260507 15:54:14.920 | INFO | Perf report for the last 5.1396 seconds: | 20260507 15:54:14.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:14.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:54:14.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 15:54:14.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:14.921 | INFO | Uploaded 36GiB to GPU in 1.61242s (22.3267GiB/s) | 20260507 15:54:14.921 | INFO | Downloaded 34GiB to GPU in 1.53059s (22.2137GiB/s) | 20260507 15:54:20.059 | INFO | Analyzing chunk frames = [730112,730624) | 20260507 15:54:20.059 | INFO | MemInfo: { | 20260507 15:54:20.059 | INFO | "gpu" : " 17 GB", | 20260507 15:54:20.059 | INFO | "huge" : "", | 20260507 15:54:20.060 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:20.060 | INFO | "malloc" : " 14 GB", | 20260507 15:54:20.060 | INFO | "pinned" : " 29 GB" | 20260507 15:54:20.060 | INFO | } | 20260507 15:54:20.060 | INFO | Perf report for the last 5.13905 seconds: | 20260507 15:54:20.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:20.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:54:20.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | PRE_HQ : 39.4%, 1.0, 20.4ms, 0 | 20260507 15:54:20.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:20.060 | INFO | Uploaded 36GiB to GPU in 1.6168s (22.2663GiB/s) | 20260507 15:54:20.060 | INFO | Downloaded 34GiB to GPU in 1.53571s (22.1396GiB/s) | 20260507 15:54:25.199 | INFO | Analyzing chunk frames = [730624,731136) | 20260507 15:54:25.199 | INFO | MemInfo: { | 20260507 15:54:25.199 | INFO | "gpu" : " 17 GB", | 20260507 15:54:25.199 | INFO | "huge" : "", | 20260507 15:54:25.199 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:25.199 | INFO | "malloc" : " 14 GB", | 20260507 15:54:25.199 | INFO | "pinned" : " 29 GB" | 20260507 15:54:25.199 | INFO | } | 20260507 15:54:25.199 | INFO | Perf report for the last 5.13966 seconds: | 20260507 15:54:25.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:25.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.199 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:54:25.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.199 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 15:54:25.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:25.200 | INFO | Uploaded 36GiB to GPU in 1.61618s (22.2747GiB/s) | 20260507 15:54:25.200 | INFO | Downloaded 34GiB to GPU in 1.53528s (22.1458GiB/s) | 20260507 15:54:30.338 | INFO | Analyzing chunk frames = [731136,731648) | 20260507 15:54:30.338 | INFO | MemInfo: { | 20260507 15:54:30.338 | INFO | "gpu" : " 17 GB", | 20260507 15:54:30.338 | INFO | "huge" : "", | 20260507 15:54:30.338 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:30.338 | INFO | "malloc" : " 14 GB", | 20260507 15:54:30.338 | INFO | "pinned" : " 29 GB" | 20260507 15:54:30.338 | INFO | } | 20260507 15:54:30.338 | INFO | Perf report for the last 5.13893 seconds: | 20260507 15:54:30.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:30.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 15:54:30.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | PRE_HQ : 38.6%, 1.0, 20.0ms, 0 | 20260507 15:54:30.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:30.338 | INFO | Uploaded 36GiB to GPU in 1.61391s (22.306GiB/s) | 20260507 15:54:30.338 | INFO | Downloaded 34GiB to GPU in 1.53146s (22.2011GiB/s) | 20260507 15:54:35.477 | INFO | Analyzing chunk frames = [731648,732160) | 20260507 15:54:35.477 | INFO | MemInfo: { | 20260507 15:54:35.477 | INFO | "gpu" : " 17 GB", | 20260507 15:54:35.477 | INFO | "huge" : "", | 20260507 15:54:35.477 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:35.478 | INFO | "malloc" : " 14 GB", | 20260507 15:54:35.478 | INFO | "pinned" : " 29 GB" | 20260507 15:54:35.478 | INFO | } | 20260507 15:54:35.478 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:54:35.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:35.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 15:54:35.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:54:35.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:35.478 | INFO | Uploaded 36GiB to GPU in 1.6087s (22.3784GiB/s) | 20260507 15:54:35.478 | INFO | Downloaded 34GiB to GPU in 1.52791s (22.2526GiB/s) | 20260507 15:54:40.617 | INFO | Analyzing chunk frames = [732160,732672) | 20260507 15:54:40.617 | INFO | MemInfo: { | 20260507 15:54:40.617 | INFO | "gpu" : " 17 GB", | 20260507 15:54:40.617 | INFO | "huge" : "", | 20260507 15:54:40.617 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:40.617 | INFO | "malloc" : " 14 GB", | 20260507 15:54:40.617 | INFO | "pinned" : " 29 GB" | 20260507 15:54:40.617 | INFO | } | 20260507 15:54:40.617 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:54:40.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:40.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:54:40.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:54:40.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:40.617 | INFO | Uploaded 36GiB to GPU in 1.61522s (22.2881GiB/s) | 20260507 15:54:40.617 | INFO | Downloaded 34GiB to GPU in 1.52915s (22.2346GiB/s) | 20260507 15:54:45.756 | INFO | Analyzing chunk frames = [732672,733184) | 20260507 15:54:45.756 | INFO | MemInfo: { | 20260507 15:54:45.756 | INFO | "gpu" : " 17 GB", | 20260507 15:54:45.756 | INFO | "huge" : "", | 20260507 15:54:45.756 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:45.756 | INFO | "malloc" : " 14 GB", | 20260507 15:54:45.756 | INFO | "pinned" : " 29 GB" | 20260507 15:54:45.756 | INFO | } | 20260507 15:54:45.756 | INFO | Perf report for the last 5.13914 seconds: | 20260507 15:54:45.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:45.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.756 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:54:45.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.756 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 15:54:45.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:45.757 | INFO | Uploaded 36GiB to GPU in 1.6095s (22.3672GiB/s) | 20260507 15:54:45.757 | INFO | Downloaded 34GiB to GPU in 1.52893s (22.2377GiB/s) | 20260507 15:54:45.853 | INFO | Starting a round of Baz IO | 20260507 15:54:48.791 | INFO | Populating RTMetrics report | 20260507 15:54:48.792 | INFO | RTMetrics report populated | 20260507 15:54:48.792 | INFO | Opening tmp RTMetrics json file | 20260507 15:54:49.026 | INFO | Serializing RTMetrics to json | 20260507 15:54:49.122 | INFO | Serialization done, swapping tmp file now... | 20260507 15:54:49.132 | INFO | RTMetrics done | 20260507 15:54:49.283 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.43 seconds | 20260507 15:54:49.284 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.73% / 0.04% / 48.23% | 20260507 15:54:49.284 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.15483% / 6.75519e-05% / 2.38419e-05% | 20260507 15:54:49.284 | INFO | Baz IO overall: | 20260507 15:54:49.284 | INFO | BytesWritten : 306791953241 | 20260507 15:54:49.284 | INFO | NumEvents : 148158904720 | 20260507 15:54:50.895 | INFO | Analyzing chunk frames = [733184,733696) | 20260507 15:54:50.895 | INFO | MemInfo: { | 20260507 15:54:50.895 | INFO | "gpu" : " 17 GB", | 20260507 15:54:50.895 | INFO | "huge" : "", | 20260507 15:54:50.895 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:50.896 | INFO | "malloc" : " 14 GB", | 20260507 15:54:50.896 | INFO | "pinned" : " 29 GB" | 20260507 15:54:50.896 | INFO | } | 20260507 15:54:50.896 | INFO | Perf report for the last 5.13967 seconds: | 20260507 15:54:50.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:50.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:50.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:50.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:50.896 | INFO | ANALYSIS : 58.2%, 2.8, 85.3ms, 0 | 20260507 15:54:50.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:50.896 | INFO | RT_METRICS : 53.2%, 1.0, 27.9ms, 0 | 20260507 15:54:50.896 | INFO | PRE_HQ : 57.3%, 2.2, 65.6ms, 0 | 20260507 15:54:50.896 | INFO | BAZWRITER : 66.3%, 4.5, 152.6ms, 0 | 20260507 15:54:50.896 | INFO | BAZ_SAMPLING : 6.6%, 1.0, 3.4ms, 0 | 20260507 15:54:50.896 | INFO | Uploaded 36GiB to GPU in 1.63073s (22.076GiB/s) | 20260507 15:54:50.896 | INFO | Downloaded 38GiB to GPU in 1.71434s (22.1659GiB/s) | 20260507 15:54:56.035 | INFO | Analyzing chunk frames = [733696,734208) | 20260507 15:54:56.035 | INFO | MemInfo: { | 20260507 15:54:56.035 | INFO | "gpu" : " 17 GB", | 20260507 15:54:56.035 | INFO | "huge" : "", | 20260507 15:54:56.035 | INFO | "huge_shared" : " 29 GB", | 20260507 15:54:56.035 | INFO | "malloc" : " 14 GB", | 20260507 15:54:56.035 | INFO | "pinned" : " 29 GB" | 20260507 15:54:56.035 | INFO | } | 20260507 15:54:56.035 | INFO | Perf report for the last 5.1395 seconds: | 20260507 15:54:56.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:54:56.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.035 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:54:56.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.036 | INFO | PRE_HQ : 39.1%, 1.0, 20.8ms, 0 | 20260507 15:54:56.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:54:56.036 | INFO | Uploaded 36GiB to GPU in 1.60119s (22.4833GiB/s) | 20260507 15:54:56.036 | INFO | Downloaded 34GiB to GPU in 1.53611s (22.1338GiB/s) | 20260507 15:55:01.174 | INFO | Analyzing chunk frames = [734208,734720) | 20260507 15:55:01.174 | INFO | MemInfo: { | 20260507 15:55:01.174 | INFO | "gpu" : " 17 GB", | 20260507 15:55:01.174 | INFO | "huge" : "", | 20260507 15:55:01.174 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:01.174 | INFO | "malloc" : " 14 GB", | 20260507 15:55:01.175 | INFO | "pinned" : " 29 GB" | 20260507 15:55:01.175 | INFO | } | 20260507 15:55:01.175 | INFO | Perf report for the last 5.13932 seconds: | 20260507 15:55:01.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:01.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:55:01.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 15:55:01.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:01.175 | INFO | Uploaded 36GiB to GPU in 1.59556s (22.5626GiB/s) | 20260507 15:55:01.175 | INFO | Downloaded 34GiB to GPU in 1.54492s (22.0075GiB/s) | 20260507 15:55:06.314 | INFO | Analyzing chunk frames = [734720,735232) | 20260507 15:55:06.314 | INFO | MemInfo: { | 20260507 15:55:06.314 | INFO | "gpu" : " 17 GB", | 20260507 15:55:06.314 | INFO | "huge" : "", | 20260507 15:55:06.314 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:06.314 | INFO | "malloc" : " 14 GB", | 20260507 15:55:06.314 | INFO | "pinned" : " 29 GB" | 20260507 15:55:06.314 | INFO | } | 20260507 15:55:06.314 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:55:06.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:06.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.314 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:55:06.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.314 | INFO | PRE_HQ : 39.7%, 1.0, 20.6ms, 0 | 20260507 15:55:06.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:06.315 | INFO | Uploaded 36GiB to GPU in 1.59556s (22.5626GiB/s) | 20260507 15:55:06.315 | INFO | Downloaded 34GiB to GPU in 1.54477s (22.0097GiB/s) | 20260507 15:55:11.453 | INFO | Analyzing chunk frames = [735232,735744) | 20260507 15:55:11.453 | INFO | MemInfo: { | 20260507 15:55:11.453 | INFO | "gpu" : " 17 GB", | 20260507 15:55:11.453 | INFO | "huge" : "", | 20260507 15:55:11.453 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:11.453 | INFO | "malloc" : " 14 GB", | 20260507 15:55:11.453 | INFO | "pinned" : " 29 GB" | 20260507 15:55:11.453 | INFO | } | 20260507 15:55:11.453 | INFO | Perf report for the last 5.1388 seconds: | 20260507 15:55:11.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:11.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.453 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:55:11.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.453 | INFO | PRE_HQ : 38.8%, 1.0, 20.1ms, 0 | 20260507 15:55:11.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:11.454 | INFO | Uploaded 36GiB to GPU in 1.59301s (22.5987GiB/s) | 20260507 15:55:11.454 | INFO | Downloaded 34GiB to GPU in 1.54217s (22.0469GiB/s) | 20260507 15:55:16.592 | INFO | Analyzing chunk frames = [735744,736256) | 20260507 15:55:16.592 | INFO | MemInfo: { | 20260507 15:55:16.592 | INFO | "gpu" : " 17 GB", | 20260507 15:55:16.592 | INFO | "huge" : "", | 20260507 15:55:16.592 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:16.592 | INFO | "malloc" : " 14 GB", | 20260507 15:55:16.592 | INFO | "pinned" : " 29 GB" | 20260507 15:55:16.592 | INFO | } | 20260507 15:55:16.592 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:55:16.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:16.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.592 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:55:16.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.592 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 15:55:16.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:16.593 | INFO | Uploaded 36GiB to GPU in 1.58963s (22.6468GiB/s) | 20260507 15:55:16.593 | INFO | Downloaded 34GiB to GPU in 1.53768s (22.1113GiB/s) | 20260507 15:55:21.732 | INFO | Analyzing chunk frames = [736256,736768) | 20260507 15:55:21.732 | INFO | MemInfo: { | 20260507 15:55:21.732 | INFO | "gpu" : " 17 GB", | 20260507 15:55:21.732 | INFO | "huge" : "", | 20260507 15:55:21.732 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:21.732 | INFO | "malloc" : " 14 GB", | 20260507 15:55:21.732 | INFO | "pinned" : " 29 GB" | 20260507 15:55:21.732 | INFO | } | 20260507 15:55:21.732 | INFO | Perf report for the last 5.13953 seconds: | 20260507 15:55:21.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:21.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:55:21.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:55:21.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:21.732 | INFO | Uploaded 36GiB to GPU in 1.58549s (22.7059GiB/s) | 20260507 15:55:21.732 | INFO | Downloaded 34GiB to GPU in 1.54403s (22.0203GiB/s) | 20260507 15:55:26.871 | INFO | Analyzing chunk frames = [736768,737280) | 20260507 15:55:26.871 | INFO | MemInfo: { | 20260507 15:55:26.871 | INFO | "gpu" : " 17 GB", | 20260507 15:55:26.871 | INFO | "huge" : "", | 20260507 15:55:26.871 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:26.871 | INFO | "malloc" : " 14 GB", | 20260507 15:55:26.871 | INFO | "pinned" : " 29 GB" | 20260507 15:55:26.871 | INFO | } | 20260507 15:55:26.871 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:55:26.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:26.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:55:26.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 15:55:26.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:26.871 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:55:26.871 | INFO | Total mem usage: 16.4 GiB | 20260507 15:55:26.871 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:55:26.871 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.14%) | 20260507 15:55:26.871 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:55:26.871 | INFO | High Water Breakdown: | 20260507 15:55:26.871 | INFO | 59.4% : 8.92 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:55:26.871 | INFO | 40.5% : 6.09 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:55:26.871 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:55:26.871 | INFO | ------------------------------------------ | 20260507 15:55:26.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:55:26.871 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:55:26.871 | INFO | Total mem usage: 26.5 GiB | 20260507 15:55:26.871 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:55:26.871 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 15:55:26.871 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:55:26.871 | INFO | High Water Breakdown: | 20260507 15:55:26.871 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:55:26.871 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:55:26.871 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:55:26.871 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:55:26.871 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:55:26.871 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:55:26.871 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:55:26.871 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:55:26.871 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:55:26.871 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:55:26.871 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:55:26.871 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:55:26.871 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:55:26.871 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:55:26.871 | INFO | ------------------------------------------ | 20260507 15:55:26.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:55:26.871 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:55:26.871 | INFO | Total mem usage: 27.8 GiB | 20260507 15:55:26.871 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:55:26.871 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:55:26.871 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:55:26.871 | INFO | High Water Breakdown: | 20260507 15:55:26.871 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:55:26.871 | INFO | ------------------------------------------ | 20260507 15:55:26.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:55:26.871 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:55:26.871 | INFO | Total mem usage: 15.7 GiB | 20260507 15:55:26.871 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:55:26.871 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:55:26.871 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:55:26.871 | INFO | High Water Breakdown: | 20260507 15:55:26.871 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:55:26.871 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:55:26.871 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:55:26.871 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:55:26.871 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:55:26.871 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:55:26.871 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:55:26.871 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:55:26.871 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:55:26.871 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:55:26.871 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:55:26.871 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:55:26.871 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:55:26.871 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:55:26.871 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:55:26.871 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:55:26.871 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:55:26.871 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:55:26.871 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:55:26.871 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:55:26.871 | INFO | Uploaded 36GiB to GPU in 1.58803s (22.6696GiB/s) | 20260507 15:55:26.871 | INFO | Downloaded 34GiB to GPU in 1.5389s (22.0938GiB/s) | 20260507 15:55:26.871 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:55:26.872 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:55:26.872 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:55:26.872 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:55:26.872 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:55:26.872 | INFO | ------------------------------------------ | 20260507 15:55:26.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:55:26.872 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:55:26.872 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:55:26.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:55:26.872 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 0.30%, AggregateStats -- 9.02e-02 / -nan, 6.15e-03 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 11.08%, Baseline -- 3.31e+00 / -nan, 7.13e-02 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.42e-02 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 16.12%, DME -- 3.85e+01 / -nan, 3.01e+00 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 3.42%, Metrics -- 1.02e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.79e-01 / -nan, 5.60e-02 / 0.00e+00 | 20260507 15:55:26.872 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:55:26.872 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:55:26.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:55:26.872 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 18.75%, Upload -- 1.67e+01 / 0.00e+00, 4.39e-01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 18.20%, Download -- 1.62e+01 / 0.00e+00, 7.29e-01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 33.45%, Compute -- 2.98e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 0.58%, LockUpload -- 5.16e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 0.00%, LockDownload -- 5.68e-04 / 0.00e+00, 4.38e-04 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 29.00%, LockCompute -- 2.59e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:55:26.872 | INFO | PerfReport: 0.02%, Misc -- 1.70e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260507 15:55:26.979 | INFO | Starting a round of Baz IO | 20260507 15:55:29.891 | INFO | Populating RTMetrics report | 20260507 15:55:29.891 | INFO | RTMetrics report populated | 20260507 15:55:29.891 | INFO | Opening tmp RTMetrics json file | 20260507 15:55:29.947 | INFO | Serializing RTMetrics to json | 20260507 15:55:29.952 | INFO | Serialization done, swapping tmp file now... | 20260507 15:55:29.958 | INFO | RTMetrics done | 20260507 15:55:30.275 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.295 seconds | 20260507 15:55:30.275 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.70% / 0.04% / 48.27% | 20260507 15:55:30.275 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.13473% / 5.5631e-05% / 2.38419e-05% | 20260507 15:55:30.275 | INFO | Baz IO overall: | 20260507 15:55:30.275 | INFO | BytesWritten : 308691108370 | 20260507 15:55:30.275 | INFO | NumEvents : 149083324324 | 20260507 15:55:32.010 | INFO | Analyzing chunk frames = [737280,737792) | 20260507 15:55:32.010 | INFO | MemInfo: { | 20260507 15:55:32.010 | INFO | "gpu" : " 17 GB", | 20260507 15:55:32.010 | INFO | "huge" : "", | 20260507 15:55:32.010 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:32.010 | INFO | "malloc" : " 14 GB", | 20260507 15:55:32.010 | INFO | "pinned" : " 29 GB" | 20260507 15:55:32.011 | INFO | } | 20260507 15:55:32.011 | INFO | Perf report for the last 5.13971 seconds: | 20260507 15:55:32.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:32.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:32.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:32.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:32.011 | INFO | ANALYSIS : 57.9%, 2.8, 86.2ms, 0 | 20260507 15:55:32.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:32.011 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 15:55:32.011 | INFO | PRE_HQ : 56.6%, 2.1, 63.2ms, 0 | 20260507 15:55:32.011 | INFO | BAZWRITER : 64.1%, 4.5, 149.9ms, 0 | 20260507 15:55:32.011 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 15:55:32.011 | INFO | Uploaded 36GiB to GPU in 1.64097s (21.9383GiB/s) | 20260507 15:55:32.011 | INFO | Downloaded 38GiB to GPU in 1.72639s (22.0112GiB/s) | 20260507 15:55:37.150 | INFO | Analyzing chunk frames = [737792,738304) | 20260507 15:55:37.150 | INFO | MemInfo: { | 20260507 15:55:37.150 | INFO | "gpu" : " 17 GB", | 20260507 15:55:37.150 | INFO | "huge" : "", | 20260507 15:55:37.150 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:37.150 | INFO | "malloc" : " 14 GB", | 20260507 15:55:37.150 | INFO | "pinned" : " 29 GB" | 20260507 15:55:37.150 | INFO | } | 20260507 15:55:37.150 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:55:37.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:37.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 15:55:37.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 15:55:37.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:37.150 | INFO | Uploaded 36GiB to GPU in 1.61206s (22.3317GiB/s) | 20260507 15:55:37.151 | INFO | Downloaded 34GiB to GPU in 1.5299s (22.2237GiB/s) | 20260507 15:55:42.289 | INFO | Analyzing chunk frames = [738304,738816) | 20260507 15:55:42.289 | INFO | MemInfo: { | 20260507 15:55:42.289 | INFO | "gpu" : " 17 GB", | 20260507 15:55:42.289 | INFO | "huge" : "", | 20260507 15:55:42.289 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:42.289 | INFO | "malloc" : " 14 GB", | 20260507 15:55:42.289 | INFO | "pinned" : " 29 GB" | 20260507 15:55:42.289 | INFO | } | 20260507 15:55:42.289 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:55:42.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:42.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.289 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:55:42.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.289 | INFO | PRE_HQ : 39.0%, 1.0, 20.2ms, 0 | 20260507 15:55:42.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:42.290 | INFO | Uploaded 36GiB to GPU in 1.61571s (22.2812GiB/s) | 20260507 15:55:42.290 | INFO | Downloaded 34GiB to GPU in 1.53497s (22.1502GiB/s) | 20260507 15:55:47.428 | INFO | Analyzing chunk frames = [738816,739328) | 20260507 15:55:47.428 | INFO | MemInfo: { | 20260507 15:55:47.428 | INFO | "gpu" : " 17 GB", | 20260507 15:55:47.428 | INFO | "huge" : "", | 20260507 15:55:47.428 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:47.428 | INFO | "malloc" : " 14 GB", | 20260507 15:55:47.428 | INFO | "pinned" : " 29 GB" | 20260507 15:55:47.428 | INFO | } | 20260507 15:55:47.428 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:55:47.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:47.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.428 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:55:47.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.428 | INFO | PRE_HQ : 39.5%, 1.0, 20.5ms, 0 | 20260507 15:55:47.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:47.429 | INFO | Uploaded 36GiB to GPU in 1.61593s (22.2783GiB/s) | 20260507 15:55:47.429 | INFO | Downloaded 34GiB to GPU in 1.53702s (22.1207GiB/s) | 20260507 15:55:52.568 | INFO | Analyzing chunk frames = [739328,739840) | 20260507 15:55:52.568 | INFO | MemInfo: { | 20260507 15:55:52.568 | INFO | "gpu" : " 17 GB", | 20260507 15:55:52.568 | INFO | "huge" : "", | 20260507 15:55:52.568 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:52.568 | INFO | "malloc" : " 14 GB", | 20260507 15:55:52.568 | INFO | "pinned" : " 29 GB" | 20260507 15:55:52.568 | INFO | } | 20260507 15:55:52.568 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:55:52.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:52.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.568 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:55:52.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.568 | INFO | PRE_HQ : 38.6%, 1.0, 20.0ms, 0 | 20260507 15:55:52.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:52.569 | INFO | Uploaded 36GiB to GPU in 1.61344s (22.3125GiB/s) | 20260507 15:55:52.569 | INFO | Downloaded 34GiB to GPU in 1.53162s (22.1987GiB/s) | 20260507 15:55:57.707 | INFO | Analyzing chunk frames = [739840,740352) | 20260507 15:55:57.707 | INFO | MemInfo: { | 20260507 15:55:57.707 | INFO | "gpu" : " 17 GB", | 20260507 15:55:57.707 | INFO | "huge" : "", | 20260507 15:55:57.707 | INFO | "huge_shared" : " 29 GB", | 20260507 15:55:57.707 | INFO | "malloc" : " 14 GB", | 20260507 15:55:57.707 | INFO | "pinned" : " 29 GB" | 20260507 15:55:57.707 | INFO | } | 20260507 15:55:57.707 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:55:57.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:55:57.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.707 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:55:57.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.708 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 15:55:57.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:55:57.708 | INFO | Uploaded 36GiB to GPU in 1.61278s (22.3217GiB/s) | 20260507 15:55:57.708 | INFO | Downloaded 34GiB to GPU in 1.53266s (22.1837GiB/s) | 20260507 15:56:02.846 | INFO | Analyzing chunk frames = [740352,740864) | 20260507 15:56:02.846 | INFO | MemInfo: { | 20260507 15:56:02.846 | INFO | "gpu" : " 17 GB", | 20260507 15:56:02.846 | INFO | "huge" : "", | 20260507 15:56:02.847 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:02.847 | INFO | "malloc" : " 14 GB", | 20260507 15:56:02.847 | INFO | "pinned" : " 29 GB" | 20260507 15:56:02.847 | INFO | } | 20260507 15:56:02.847 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:56:02.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:02.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 15:56:02.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | PRE_HQ : 39.4%, 1.0, 20.5ms, 0 | 20260507 15:56:02.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:02.847 | INFO | Uploaded 36GiB to GPU in 1.61504s (22.2905GiB/s) | 20260507 15:56:02.847 | INFO | Downloaded 34GiB to GPU in 1.53531s (22.1453GiB/s) | 20260507 15:56:07.986 | INFO | Analyzing chunk frames = [740864,741376) | 20260507 15:56:07.986 | INFO | MemInfo: { | 20260507 15:56:07.986 | INFO | "gpu" : " 17 GB", | 20260507 15:56:07.986 | INFO | "huge" : "", | 20260507 15:56:07.986 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:07.986 | INFO | "malloc" : " 14 GB", | 20260507 15:56:07.986 | INFO | "pinned" : " 29 GB" | 20260507 15:56:07.986 | INFO | } | 20260507 15:56:07.986 | INFO | Perf report for the last 5.13912 seconds: | 20260507 15:56:07.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:07.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260507 15:56:07.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 15:56:07.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:07.986 | INFO | Uploaded 36GiB to GPU in 1.61428s (22.301GiB/s) | 20260507 15:56:07.986 | INFO | Downloaded 34GiB to GPU in 1.53452s (22.1568GiB/s) | 20260507 15:56:08.083 | INFO | Starting a round of Baz IO | 20260507 15:56:11.017 | INFO | Populating RTMetrics report | 20260507 15:56:11.017 | INFO | RTMetrics report populated | 20260507 15:56:11.017 | INFO | Opening tmp RTMetrics json file | 20260507 15:56:11.020 | INFO | Serializing RTMetrics to json | 20260507 15:56:11.043 | INFO | Serialization done, swapping tmp file now... | 20260507 15:56:11.055 | INFO | RTMetrics done | 20260507 15:56:11.207 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.123 seconds | 20260507 15:56:11.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.66% / 0.03% / 48.30% | 20260507 15:56:11.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.11066% / 4.37101e-05% / 4.76837e-05% | 20260507 15:56:11.207 | INFO | Baz IO overall: | 20260507 15:56:11.207 | INFO | BytesWritten : 310590375078 | 20260507 15:56:11.207 | INFO | NumEvents : 150007797068 | 20260507 15:56:13.125 | INFO | Analyzing chunk frames = [741376,741888) | 20260507 15:56:13.125 | INFO | MemInfo: { | 20260507 15:56:13.125 | INFO | "gpu" : " 17 GB", | 20260507 15:56:13.125 | INFO | "huge" : "", | 20260507 15:56:13.125 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:13.125 | INFO | "malloc" : " 14 GB", | 20260507 15:56:13.125 | INFO | "pinned" : " 29 GB" | 20260507 15:56:13.125 | INFO | } | 20260507 15:56:13.125 | INFO | Perf report for the last 5.13944 seconds: | 20260507 15:56:13.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:13.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:13.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:13.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:13.125 | INFO | ANALYSIS : 58.1%, 2.7, 82.9ms, 0 | 20260507 15:56:13.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:13.125 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260507 15:56:13.125 | INFO | PRE_HQ : 57.2%, 2.2, 64.9ms, 0 | 20260507 15:56:13.125 | INFO | BAZWRITER : 60.3%, 4.8, 147.3ms, 0 | 20260507 15:56:13.125 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 15:56:13.126 | INFO | Uploaded 36GiB to GPU in 1.63793s (21.9789GiB/s) | 20260507 15:56:13.126 | INFO | Downloaded 38GiB to GPU in 1.72299s (22.0547GiB/s) | 20260507 15:56:18.264 | INFO | Analyzing chunk frames = [741888,742400) | 20260507 15:56:18.264 | INFO | MemInfo: { | 20260507 15:56:18.264 | INFO | "gpu" : " 17 GB", | 20260507 15:56:18.264 | INFO | "huge" : "", | 20260507 15:56:18.264 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:18.264 | INFO | "malloc" : " 14 GB", | 20260507 15:56:18.264 | INFO | "pinned" : " 29 GB" | 20260507 15:56:18.264 | INFO | } | 20260507 15:56:18.264 | INFO | Perf report for the last 5.13904 seconds: | 20260507 15:56:18.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:18.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:56:18.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | PRE_HQ : 40.7%, 1.0, 21.9ms, 0 | 20260507 15:56:18.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:18.265 | INFO | Uploaded 36GiB to GPU in 1.59888s (22.5158GiB/s) | 20260507 15:56:18.265 | INFO | Downloaded 34GiB to GPU in 1.53746s (22.1144GiB/s) | 20260507 15:56:23.404 | INFO | Analyzing chunk frames = [742400,742912) | 20260507 15:56:23.404 | INFO | MemInfo: { | 20260507 15:56:23.404 | INFO | "gpu" : " 17 GB", | 20260507 15:56:23.404 | INFO | "huge" : "", | 20260507 15:56:23.404 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:23.404 | INFO | "malloc" : " 14 GB", | 20260507 15:56:23.404 | INFO | "pinned" : " 29 GB" | 20260507 15:56:23.404 | INFO | } | 20260507 15:56:23.404 | INFO | Perf report for the last 5.13958 seconds: | 20260507 15:56:23.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:23.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 15:56:23.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 15:56:23.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:23.404 | INFO | Uploaded 36GiB to GPU in 1.58979s (22.6445GiB/s) | 20260507 15:56:23.404 | INFO | Downloaded 34GiB to GPU in 1.54006s (22.0771GiB/s) | 20260507 15:56:28.543 | INFO | Analyzing chunk frames = [742912,743424) | 20260507 15:56:28.543 | INFO | MemInfo: { | 20260507 15:56:28.543 | INFO | "gpu" : " 17 GB", | 20260507 15:56:28.543 | INFO | "huge" : "", | 20260507 15:56:28.543 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:28.543 | INFO | "malloc" : " 14 GB", | 20260507 15:56:28.543 | INFO | "pinned" : " 29 GB" | 20260507 15:56:28.543 | INFO | } | 20260507 15:56:28.543 | INFO | Perf report for the last 5.13928 seconds: | 20260507 15:56:28.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:28.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | ANALYSIS : 55.9%, 3.0, 86.6ms, 0 | 20260507 15:56:28.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 15:56:28.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:28.543 | INFO | Uploaded 36GiB to GPU in 1.59014s (22.6396GiB/s) | 20260507 15:56:28.544 | INFO | Downloaded 34GiB to GPU in 1.53989s (22.0795GiB/s) | 20260507 15:56:33.682 | INFO | Analyzing chunk frames = [743424,743936) | 20260507 15:56:33.682 | INFO | MemInfo: { | 20260507 15:56:33.682 | INFO | "gpu" : " 17 GB", | 20260507 15:56:33.682 | INFO | "huge" : "", | 20260507 15:56:33.683 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:33.683 | INFO | "malloc" : " 14 GB", | 20260507 15:56:33.683 | INFO | "pinned" : " 29 GB" | 20260507 15:56:33.683 | INFO | } | 20260507 15:56:33.683 | INFO | Perf report for the last 5.13949 seconds: | 20260507 15:56:33.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:33.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 15:56:33.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 15:56:33.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:33.683 | INFO | Uploaded 36GiB to GPU in 1.58825s (22.6665GiB/s) | 20260507 15:56:33.683 | INFO | Downloaded 34GiB to GPU in 1.53747s (22.1142GiB/s) | 20260507 15:56:38.822 | INFO | Analyzing chunk frames = [743936,744448) | 20260507 15:56:38.822 | INFO | MemInfo: { | 20260507 15:56:38.822 | INFO | "gpu" : " 17 GB", | 20260507 15:56:38.822 | INFO | "huge" : "", | 20260507 15:56:38.822 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:38.822 | INFO | "malloc" : " 14 GB", | 20260507 15:56:38.822 | INFO | "pinned" : " 29 GB" | 20260507 15:56:38.822 | INFO | } | 20260507 15:56:38.822 | INFO | Perf report for the last 5.13925 seconds: | 20260507 15:56:38.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:38.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.822 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:56:38.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.822 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 15:56:38.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:38.823 | INFO | Uploaded 36GiB to GPU in 1.58823s (22.6667GiB/s) | 20260507 15:56:38.823 | INFO | Downloaded 34GiB to GPU in 1.53669s (22.1254GiB/s) | 20260507 15:56:43.961 | INFO | Analyzing chunk frames = [744448,744960) | 20260507 15:56:43.961 | INFO | MemInfo: { | 20260507 15:56:43.961 | INFO | "gpu" : " 17 GB", | 20260507 15:56:43.961 | INFO | "huge" : "", | 20260507 15:56:43.961 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:43.961 | INFO | "malloc" : " 14 GB", | 20260507 15:56:43.961 | INFO | "pinned" : " 29 GB" | 20260507 15:56:43.961 | INFO | } | 20260507 15:56:43.961 | INFO | Perf report for the last 5.13943 seconds: | 20260507 15:56:43.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:43.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.961 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:56:43.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.961 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:56:43.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:43.962 | INFO | Uploaded 36GiB to GPU in 1.58914s (22.6537GiB/s) | 20260507 15:56:43.962 | INFO | Downloaded 34GiB to GPU in 1.53855s (22.0987GiB/s) | 20260507 15:56:49.100 | INFO | Analyzing chunk frames = [744960,745472) | 20260507 15:56:49.100 | INFO | MemInfo: { | 20260507 15:56:49.100 | INFO | "gpu" : " 17 GB", | 20260507 15:56:49.100 | INFO | "huge" : "", | 20260507 15:56:49.100 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:49.100 | INFO | "malloc" : " 14 GB", | 20260507 15:56:49.100 | INFO | "pinned" : " 29 GB" | 20260507 15:56:49.100 | INFO | } | 20260507 15:56:49.100 | INFO | Perf report for the last 5.13909 seconds: | 20260507 15:56:49.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:49.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.100 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:56:49.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.101 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:56:49.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:49.101 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:56:49.101 | INFO | Total mem usage: 16.4 GiB | 20260507 15:56:49.101 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:56:49.101 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -6.07%) | 20260507 15:56:49.101 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:56:49.101 | INFO | High Water Breakdown: | 20260507 15:56:49.101 | INFO | 59.9% : 9.26 GiB ( 93.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:56:49.101 | INFO | 40% : 6.19 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:56:49.101 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:56:49.101 | INFO | ------------------------------------------ | 20260507 15:56:49.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:56:49.101 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:56:49.101 | INFO | Total mem usage: 26.5 GiB | 20260507 15:56:49.101 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:56:49.101 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:56:49.101 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:56:49.101 | INFO | High Water Breakdown: | 20260507 15:56:49.101 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:56:49.101 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:56:49.101 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:56:49.101 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:56:49.101 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:56:49.101 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:56:49.101 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:56:49.101 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:56:49.101 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:56:49.101 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:56:49.101 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:56:49.101 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:56:49.101 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:56:49.101 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:56:49.101 | INFO | ------------------------------------------ | 20260507 15:56:49.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:56:49.101 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:56:49.101 | INFO | Total mem usage: 27.8 GiB | 20260507 15:56:49.101 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:56:49.101 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:56:49.101 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:56:49.101 | INFO | High Water Breakdown: | 20260507 15:56:49.101 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:56:49.101 | INFO | ------------------------------------------ | 20260507 15:56:49.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:56:49.101 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:56:49.101 | INFO | Total mem usage: 15.7 GiB | 20260507 15:56:49.101 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:56:49.101 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:56:49.101 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 15:56:49.101 | INFO | High Water Breakdown: | 20260507 15:56:49.101 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:56:49.101 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:56:49.101 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:56:49.101 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:56:49.101 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:56:49.101 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:56:49.101 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:56:49.101 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:56:49.101 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:56:49.101 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:56:49.101 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:56:49.101 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:56:49.101 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:56:49.101 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:56:49.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:56:49.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:56:49.101 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:56:49.101 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:56:49.101 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:56:49.101 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:56:49.101 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:56:49.101 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:56:49.101 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:56:49.101 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:56:49.101 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:56:49.101 | INFO | ------------------------------------------ | 20260507 15:56:49.101 | INFO | Uploaded 36GiB to GPU in 1.589s (22.6557GiB/s) | 20260507 15:56:49.101 | INFO | Downloaded 34GiB to GPU in 1.53835s (22.1015GiB/s) | 20260507 15:56:49.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:56:49.102 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:56:49.102 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:56:49.102 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:56:49.102 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.23e+01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / -nan, 6.80e-03 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 11.08%, Baseline -- 3.31e+00 / -nan, 2.46e-02 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.29e-02 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 3.42%, Metrics -- 1.02e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.78e-01 / -nan, 5.49e-02 / 0.00e+00 | 20260507 15:56:49.102 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:56:49.102 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:56:49.102 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:56:49.102 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 18.78%, Upload -- 1.67e+01 / 0.00e+00, 4.54e-01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 18.23%, Download -- 1.62e+01 / 0.00e+00, 7.52e-01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 33.49%, Compute -- 2.98e+01 / 0.00e+00, 1.23e+01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 0.00%, LockDownload -- 5.93e-04 / 0.00e+00, 4.71e-04 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 28.90%, LockCompute -- 2.58e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:56:49.102 | INFO | PerfReport: 0.02%, Misc -- 1.80e-02 / 0.00e+00, 3.76e-02 / 0.00e+00 | 20260507 15:56:49.198 | INFO | Starting a round of Baz IO | 20260507 15:56:52.222 | INFO | Populating RTMetrics report | 20260507 15:56:52.222 | INFO | RTMetrics report populated | 20260507 15:56:52.222 | INFO | Opening tmp RTMetrics json file | 20260507 15:56:52.224 | INFO | Serializing RTMetrics to json | 20260507 15:56:52.227 | INFO | Serialization done, swapping tmp file now... | 20260507 15:56:52.230 | INFO | RTMetrics done | 20260507 15:56:52.551 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.353 seconds | 20260507 15:56:52.551 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.63% / 0.03% / 48.34% | 20260507 15:56:52.551 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.10288% / 5.5631e-05% / 3.97364e-05% | 20260507 15:56:52.551 | INFO | Baz IO overall: | 20260507 15:56:52.551 | INFO | BytesWritten : 312490979463 | 20260507 15:56:52.551 | INFO | NumEvents : 150932826416 | 20260507 15:56:54.240 | INFO | Analyzing chunk frames = [745472,745984) | 20260507 15:56:54.240 | INFO | MemInfo: { | 20260507 15:56:54.240 | INFO | "gpu" : " 17 GB", | 20260507 15:56:54.240 | INFO | "huge" : "", | 20260507 15:56:54.240 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:54.240 | INFO | "malloc" : " 14 GB", | 20260507 15:56:54.240 | INFO | "pinned" : " 29 GB" | 20260507 15:56:54.240 | INFO | } | 20260507 15:56:54.240 | INFO | Perf report for the last 5.13947 seconds: | 20260507 15:56:54.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:54.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:54.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:54.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:54.240 | INFO | ANALYSIS : 59.0%, 2.7, 83.8ms, 0 | 20260507 15:56:54.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:54.240 | INFO | RT_METRICS : 49.0%, 1.0, 25.7ms, 0 | 20260507 15:56:54.240 | INFO | PRE_HQ : 58.5%, 2.2, 68.1ms, 0 | 20260507 15:56:54.240 | INFO | BAZWRITER : 64.8%, 4.7, 156.2ms, 0 | 20260507 15:56:54.240 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:56:54.240 | INFO | Uploaded 36GiB to GPU in 1.63652s (21.9979GiB/s) | 20260507 15:56:54.240 | INFO | Downloaded 38GiB to GPU in 1.71763s (22.1236GiB/s) | 20260507 15:56:59.379 | INFO | Analyzing chunk frames = [745984,746496) | 20260507 15:56:59.379 | INFO | MemInfo: { | 20260507 15:56:59.379 | INFO | "gpu" : " 17 GB", | 20260507 15:56:59.379 | INFO | "huge" : "", | 20260507 15:56:59.379 | INFO | "huge_shared" : " 29 GB", | 20260507 15:56:59.379 | INFO | "malloc" : " 14 GB", | 20260507 15:56:59.379 | INFO | "pinned" : " 29 GB" | 20260507 15:56:59.379 | INFO | } | 20260507 15:56:59.379 | INFO | Perf report for the last 5.13945 seconds: | 20260507 15:56:59.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:56:59.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.380 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260507 15:56:59.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.380 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:56:59.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:56:59.380 | INFO | Uploaded 36GiB to GPU in 1.60585s (22.4181GiB/s) | 20260507 15:56:59.380 | INFO | Downloaded 34GiB to GPU in 1.52582s (22.2831GiB/s) | 20260507 15:57:04.519 | INFO | Analyzing chunk frames = [746496,747008) | 20260507 15:57:04.519 | INFO | MemInfo: { | 20260507 15:57:04.519 | INFO | "gpu" : " 17 GB", | 20260507 15:57:04.519 | INFO | "huge" : "", | 20260507 15:57:04.519 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:04.519 | INFO | "malloc" : " 14 GB", | 20260507 15:57:04.519 | INFO | "pinned" : " 29 GB" | 20260507 15:57:04.519 | INFO | } | 20260507 15:57:04.519 | INFO | Perf report for the last 5.13961 seconds: | 20260507 15:57:04.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:04.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.519 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 15:57:04.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.519 | INFO | PRE_HQ : 39.7%, 1.0, 20.6ms, 0 | 20260507 15:57:04.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:04.520 | INFO | Uploaded 36GiB to GPU in 1.61566s (22.282GiB/s) | 20260507 15:57:04.520 | INFO | Downloaded 34GiB to GPU in 1.5365s (22.1282GiB/s) | 20260507 15:57:09.658 | INFO | Analyzing chunk frames = [747008,747520) | 20260507 15:57:09.658 | INFO | MemInfo: { | 20260507 15:57:09.658 | INFO | "gpu" : " 17 GB", | 20260507 15:57:09.658 | INFO | "huge" : "", | 20260507 15:57:09.658 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:09.658 | INFO | "malloc" : " 14 GB", | 20260507 15:57:09.658 | INFO | "pinned" : " 29 GB" | 20260507 15:57:09.658 | INFO | } | 20260507 15:57:09.658 | INFO | Perf report for the last 5.13887 seconds: | 20260507 15:57:09.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:09.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:57:09.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | PRE_HQ : 39.5%, 1.0, 20.5ms, 0 | 20260507 15:57:09.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:09.658 | INFO | Uploaded 36GiB to GPU in 1.61647s (22.2707GiB/s) | 20260507 15:57:09.658 | INFO | Downloaded 34GiB to GPU in 1.53435s (22.1592GiB/s) | 20260507 15:57:14.797 | INFO | Analyzing chunk frames = [747520,748032) | 20260507 15:57:14.797 | INFO | MemInfo: { | 20260507 15:57:14.797 | INFO | "gpu" : " 17 GB", | 20260507 15:57:14.797 | INFO | "huge" : "", | 20260507 15:57:14.797 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:14.797 | INFO | "malloc" : " 14 GB", | 20260507 15:57:14.797 | INFO | "pinned" : " 29 GB" | 20260507 15:57:14.797 | INFO | } | 20260507 15:57:14.797 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:57:14.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:14.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.797 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:57:14.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.797 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 15:57:14.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:14.798 | INFO | Uploaded 36GiB to GPU in 1.61528s (22.2871GiB/s) | 20260507 15:57:14.798 | INFO | Downloaded 34GiB to GPU in 1.53336s (22.1735GiB/s) | 20260507 15:57:19.936 | INFO | Analyzing chunk frames = [748032,748544) | 20260507 15:57:19.936 | INFO | MemInfo: { | 20260507 15:57:19.936 | INFO | "gpu" : " 17 GB", | 20260507 15:57:19.936 | INFO | "huge" : "", | 20260507 15:57:19.936 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:19.936 | INFO | "malloc" : " 14 GB", | 20260507 15:57:19.936 | INFO | "pinned" : " 29 GB" | 20260507 15:57:19.936 | INFO | } | 20260507 15:57:19.936 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:57:19.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:19.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.937 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 15:57:19.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.937 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 15:57:19.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:19.937 | INFO | Uploaded 36GiB to GPU in 1.60646s (22.4095GiB/s) | 20260507 15:57:19.937 | INFO | Downloaded 34GiB to GPU in 1.52786s (22.2534GiB/s) | 20260507 15:57:25.076 | INFO | Analyzing chunk frames = [748544,749056) | 20260507 15:57:25.076 | INFO | MemInfo: { | 20260507 15:57:25.076 | INFO | "gpu" : " 17 GB", | 20260507 15:57:25.076 | INFO | "huge" : "", | 20260507 15:57:25.076 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:25.076 | INFO | "malloc" : " 14 GB", | 20260507 15:57:25.076 | INFO | "pinned" : " 29 GB" | 20260507 15:57:25.076 | INFO | } | 20260507 15:57:25.076 | INFO | Perf report for the last 5.13959 seconds: | 20260507 15:57:25.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:25.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.076 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 15:57:25.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.076 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:57:25.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:25.077 | INFO | Uploaded 36GiB to GPU in 1.60912s (22.3725GiB/s) | 20260507 15:57:25.077 | INFO | Downloaded 34GiB to GPU in 1.52909s (22.2354GiB/s) | 20260507 15:57:30.215 | INFO | Analyzing chunk frames = [749056,749568) | 20260507 15:57:30.215 | INFO | MemInfo: { | 20260507 15:57:30.215 | INFO | "gpu" : " 17 GB", | 20260507 15:57:30.215 | INFO | "huge" : "", | 20260507 15:57:30.215 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:30.215 | INFO | "malloc" : " 14 GB", | 20260507 15:57:30.215 | INFO | "pinned" : " 29 GB" | 20260507 15:57:30.215 | INFO | } | 20260507 15:57:30.215 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:57:30.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:30.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.215 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:57:30.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.215 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:57:30.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:30.216 | INFO | Uploaded 36GiB to GPU in 1.60664s (22.407GiB/s) | 20260507 15:57:30.216 | INFO | Downloaded 34GiB to GPU in 1.52856s (22.2432GiB/s) | 20260507 15:57:30.312 | INFO | Starting a round of Baz IO | 20260507 15:57:33.229 | INFO | Populating RTMetrics report | 20260507 15:57:33.230 | INFO | RTMetrics report populated | 20260507 15:57:33.230 | INFO | Opening tmp RTMetrics json file | 20260507 15:57:33.232 | INFO | Serializing RTMetrics to json | 20260507 15:57:33.234 | INFO | Serialization done, swapping tmp file now... | 20260507 15:57:33.235 | INFO | RTMetrics done | 20260507 15:57:33.350 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.037 seconds | 20260507 15:57:33.350 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.59% / 0.03% / 48.37% | 20260507 15:57:33.350 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.09263% / 5.96046e-05% / 4.37101e-05% | 20260507 15:57:33.350 | INFO | Baz IO overall: | 20260507 15:57:33.350 | INFO | BytesWritten : 314392639525 | 20260507 15:57:33.350 | INFO | NumEvents : 151858334870 | 20260507 15:57:35.354 | INFO | Analyzing chunk frames = [749568,750080) | 20260507 15:57:35.354 | INFO | MemInfo: { | 20260507 15:57:35.354 | INFO | "gpu" : " 17 GB", | 20260507 15:57:35.355 | INFO | "huge" : "", | 20260507 15:57:35.355 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:35.355 | INFO | "malloc" : " 14 GB", | 20260507 15:57:35.355 | INFO | "pinned" : " 29 GB" | 20260507 15:57:35.355 | INFO | } | 20260507 15:57:35.355 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:57:35.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:35.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:35.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:35.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:35.355 | INFO | ANALYSIS : 57.1%, 2.9, 86.9ms, 0 | 20260507 15:57:35.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:35.355 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 15:57:35.355 | INFO | PRE_HQ : 56.4%, 2.1, 62.2ms, 0 | 20260507 15:57:35.355 | INFO | BAZWRITER : 58.7%, 4.5, 134.6ms, 0 | 20260507 15:57:35.355 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 15:57:35.355 | INFO | Uploaded 36GiB to GPU in 1.64251s (21.9176GiB/s) | 20260507 15:57:35.355 | INFO | Downloaded 38GiB to GPU in 1.72511s (22.0275GiB/s) | 20260507 15:57:40.494 | INFO | Analyzing chunk frames = [750080,750592) | 20260507 15:57:40.494 | INFO | MemInfo: { | 20260507 15:57:40.494 | INFO | "gpu" : " 17 GB", | 20260507 15:57:40.494 | INFO | "huge" : "", | 20260507 15:57:40.494 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:40.494 | INFO | "malloc" : " 14 GB", | 20260507 15:57:40.494 | INFO | "pinned" : " 29 GB" | 20260507 15:57:40.494 | INFO | } | 20260507 15:57:40.494 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:57:40.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:40.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.494 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:57:40.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.494 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:57:40.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:40.495 | INFO | Uploaded 36GiB to GPU in 1.61184s (22.3348GiB/s) | 20260507 15:57:40.495 | INFO | Downloaded 34GiB to GPU in 1.53054s (22.2143GiB/s) | 20260507 15:57:45.633 | INFO | Analyzing chunk frames = [750592,751104) | 20260507 15:57:45.633 | INFO | MemInfo: { | 20260507 15:57:45.633 | INFO | "gpu" : " 17 GB", | 20260507 15:57:45.633 | INFO | "huge" : "", | 20260507 15:57:45.633 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:45.633 | INFO | "malloc" : " 14 GB", | 20260507 15:57:45.633 | INFO | "pinned" : " 29 GB" | 20260507 15:57:45.633 | INFO | } | 20260507 15:57:45.633 | INFO | Perf report for the last 5.13933 seconds: | 20260507 15:57:45.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:45.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:57:45.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:57:45.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:45.634 | INFO | Uploaded 36GiB to GPU in 1.61646s (22.2709GiB/s) | 20260507 15:57:45.634 | INFO | Downloaded 34GiB to GPU in 1.53506s (22.149GiB/s) | 20260507 15:57:50.773 | INFO | Analyzing chunk frames = [751104,751616) | 20260507 15:57:50.773 | INFO | MemInfo: { | 20260507 15:57:50.773 | INFO | "gpu" : " 17 GB", | 20260507 15:57:50.773 | INFO | "huge" : "", | 20260507 15:57:50.773 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:50.773 | INFO | "malloc" : " 14 GB", | 20260507 15:57:50.773 | INFO | "pinned" : " 29 GB" | 20260507 15:57:50.773 | INFO | } | 20260507 15:57:50.773 | INFO | Perf report for the last 5.13937 seconds: | 20260507 15:57:50.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:50.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.773 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 15:57:50.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.773 | INFO | PRE_HQ : 40.1%, 1.0, 20.8ms, 0 | 20260507 15:57:50.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:50.774 | INFO | Uploaded 36GiB to GPU in 1.61514s (22.289GiB/s) | 20260507 15:57:50.774 | INFO | Downloaded 34GiB to GPU in 1.53558s (22.1414GiB/s) | 20260507 15:57:55.912 | INFO | Analyzing chunk frames = [751616,752128) | 20260507 15:57:55.912 | INFO | MemInfo: { | 20260507 15:57:55.912 | INFO | "gpu" : " 17 GB", | 20260507 15:57:55.912 | INFO | "huge" : "", | 20260507 15:57:55.912 | INFO | "huge_shared" : " 29 GB", | 20260507 15:57:55.912 | INFO | "malloc" : " 14 GB", | 20260507 15:57:55.912 | INFO | "pinned" : " 29 GB" | 20260507 15:57:55.912 | INFO | } | 20260507 15:57:55.912 | INFO | Perf report for the last 5.13898 seconds: | 20260507 15:57:55.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:57:55.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 15:57:55.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 15:57:55.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:57:55.912 | INFO | Uploaded 36GiB to GPU in 1.61217s (22.3302GiB/s) | 20260507 15:57:55.912 | INFO | Downloaded 34GiB to GPU in 1.53052s (22.2147GiB/s) | 20260507 15:58:01.051 | INFO | Analyzing chunk frames = [752128,752640) | 20260507 15:58:01.051 | INFO | MemInfo: { | 20260507 15:58:01.051 | INFO | "gpu" : " 17 GB", | 20260507 15:58:01.051 | INFO | "huge" : "", | 20260507 15:58:01.051 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:01.051 | INFO | "malloc" : " 14 GB", | 20260507 15:58:01.051 | INFO | "pinned" : " 29 GB" | 20260507 15:58:01.051 | INFO | } | 20260507 15:58:01.051 | INFO | Perf report for the last 5.13951 seconds: | 20260507 15:58:01.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:01.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.052 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 15:58:01.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.052 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:58:01.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:01.052 | INFO | Uploaded 36GiB to GPU in 1.60806s (22.3872GiB/s) | 20260507 15:58:01.052 | INFO | Downloaded 34GiB to GPU in 1.52746s (22.2592GiB/s) | 20260507 15:58:06.191 | INFO | Analyzing chunk frames = [752640,753152) | 20260507 15:58:06.191 | INFO | MemInfo: { | 20260507 15:58:06.191 | INFO | "gpu" : " 17 GB", | 20260507 15:58:06.191 | INFO | "huge" : "", | 20260507 15:58:06.191 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:06.191 | INFO | "malloc" : " 14 GB", | 20260507 15:58:06.191 | INFO | "pinned" : " 29 GB" | 20260507 15:58:06.191 | INFO | } | 20260507 15:58:06.191 | INFO | Perf report for the last 5.13927 seconds: | 20260507 15:58:06.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:06.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:58:06.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 15:58:06.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:06.191 | INFO | Uploaded 36GiB to GPU in 1.60807s (22.3871GiB/s) | 20260507 15:58:06.191 | INFO | Downloaded 34GiB to GPU in 1.52894s (22.2376GiB/s) | 20260507 15:58:11.330 | INFO | Analyzing chunk frames = [753152,753664) | 20260507 15:58:11.330 | INFO | MemInfo: { | 20260507 15:58:11.330 | INFO | "gpu" : " 17 GB", | 20260507 15:58:11.330 | INFO | "huge" : "", | 20260507 15:58:11.330 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:11.330 | INFO | "malloc" : " 14 GB", | 20260507 15:58:11.330 | INFO | "pinned" : " 29 GB" | 20260507 15:58:11.330 | INFO | } | 20260507 15:58:11.330 | INFO | Perf report for the last 5.13907 seconds: | 20260507 15:58:11.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:11.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:58:11.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 15:58:11.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:11.330 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:58:11.330 | INFO | Total mem usage: 16.4 GiB | 20260507 15:58:11.330 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:58:11.330 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.6%) | 20260507 15:58:11.330 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:58:11.330 | INFO | High Water Breakdown: | 20260507 15:58:11.330 | INFO | 59.6% : 9.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:58:11.330 | INFO | 40.2% : 6.13 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:58:11.330 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:58:11.330 | INFO | ------------------------------------------ | 20260507 15:58:11.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:58:11.330 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:58:11.330 | INFO | Total mem usage: 26.5 GiB | 20260507 15:58:11.330 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:58:11.330 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:58:11.330 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 15:58:11.330 | INFO | High Water Breakdown: | 20260507 15:58:11.330 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:58:11.330 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:58:11.330 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:58:11.330 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:58:11.330 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:58:11.330 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:58:11.330 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:58:11.330 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:58:11.330 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:58:11.330 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:58:11.330 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:58:11.330 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:58:11.330 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:58:11.330 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:58:11.330 | INFO | ------------------------------------------ | 20260507 15:58:11.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:58:11.330 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:58:11.330 | INFO | Total mem usage: 27.8 GiB | 20260507 15:58:11.330 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:58:11.330 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:58:11.330 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:58:11.330 | INFO | High Water Breakdown: | 20260507 15:58:11.330 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:58:11.330 | INFO | ------------------------------------------ | 20260507 15:58:11.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:58:11.330 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:58:11.330 | INFO | Total mem usage: 15.7 GiB | 20260507 15:58:11.330 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:58:11.330 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:58:11.330 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 15:58:11.330 | INFO | High Water Breakdown: | 20260507 15:58:11.330 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:58:11.330 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:58:11.330 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:58:11.330 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:58:11.330 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:58:11.330 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:58:11.330 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:58:11.330 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:58:11.330 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:58:11.330 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:58:11.330 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:58:11.330 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:58:11.330 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:58:11.330 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:58:11.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:58:11.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:58:11.330 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:58:11.330 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:58:11.330 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:58:11.330 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:58:11.330 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:58:11.330 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:58:11.330 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:58:11.330 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:58:11.331 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:58:11.331 | INFO | Uploaded 36GiB to GPU in 1.60956s (22.3664GiB/s) | 20260507 15:58:11.331 | INFO | Downloaded 34GiB to GPU in 1.52941s (22.2308GiB/s) | 20260507 15:58:11.331 | INFO | ------------------------------------------ | 20260507 15:58:11.331 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:58:11.331 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:58:11.331 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:58:11.331 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:58:11.331 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 0.30%, AggregateStats -- 8.96e-02 / -nan, 5.86e-03 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 11.07%, Baseline -- 3.30e+00 / -nan, 2.30e-02 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 3.08%, Histogram -- 9.20e-01 / -nan, 5.31e-02 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 3.42%, Metrics -- 1.02e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.77e-01 / -nan, 5.50e-02 / 0.00e+00 | 20260507 15:58:11.331 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:58:11.331 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:58:11.331 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:58:11.331 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 18.86%, Upload -- 1.68e+01 / 0.00e+00, 4.14e-01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 18.16%, Download -- 1.62e+01 / 0.00e+00, 7.61e-01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 33.45%, Compute -- 2.98e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 0.61%, LockUpload -- 5.43e-01 / 0.00e+00, 3.68e+00 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 0.00%, LockDownload -- 5.77e-04 / 0.00e+00, 4.75e-04 / 0.00e+00 | 20260507 15:58:11.331 | INFO | PerfReport: 28.90%, LockCompute -- 2.58e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:58:11.332 | INFO | PerfReport: 0.02%, Misc -- 1.69e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 15:58:11.427 | INFO | Starting a round of Baz IO | 20260507 15:58:14.359 | INFO | Populating RTMetrics report | 20260507 15:58:14.360 | INFO | RTMetrics report populated | 20260507 15:58:14.360 | INFO | Opening tmp RTMetrics json file | 20260507 15:58:14.421 | INFO | Serializing RTMetrics to json | 20260507 15:58:14.429 | INFO | Serialization done, swapping tmp file now... | 20260507 15:58:14.433 | INFO | RTMetrics done | 20260507 15:58:14.591 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.164 seconds | 20260507 15:58:14.591 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.56% / 0.03% / 48.41% | 20260507 15:58:14.591 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.07896% / 7.54992e-05% / 3.97364e-05% | 20260507 15:58:14.591 | INFO | Baz IO overall: | 20260507 15:58:14.591 | INFO | BytesWritten : 316294856524 | 20260507 15:58:14.591 | INFO | NumEvents : 152784174580 | 20260507 15:58:16.469 | INFO | Analyzing chunk frames = [753664,754176) | 20260507 15:58:16.469 | INFO | MemInfo: { | 20260507 15:58:16.469 | INFO | "gpu" : " 17 GB", | 20260507 15:58:16.469 | INFO | "huge" : "", | 20260507 15:58:16.469 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:16.469 | INFO | "malloc" : " 14 GB", | 20260507 15:58:16.469 | INFO | "pinned" : " 29 GB" | 20260507 15:58:16.469 | INFO | } | 20260507 15:58:16.470 | INFO | Perf report for the last 5.13971 seconds: | 20260507 15:58:16.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:16.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:16.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:16.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:16.470 | INFO | ANALYSIS : 57.9%, 2.8, 85.8ms, 0 | 20260507 15:58:16.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:16.470 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260507 15:58:16.470 | INFO | PRE_HQ : 57.0%, 2.2, 64.0ms, 0 | 20260507 15:58:16.470 | INFO | BAZWRITER : 61.1%, 4.5, 140.5ms, 0 | 20260507 15:58:16.470 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 15:58:16.470 | INFO | Uploaded 36GiB to GPU in 1.63537s (22.0134GiB/s) | 20260507 15:58:16.470 | INFO | Downloaded 38GiB to GPU in 1.71877s (22.1088GiB/s) | 20260507 15:58:21.609 | INFO | Analyzing chunk frames = [754176,754688) | 20260507 15:58:21.609 | INFO | MemInfo: { | 20260507 15:58:21.609 | INFO | "gpu" : " 17 GB", | 20260507 15:58:21.609 | INFO | "huge" : "", | 20260507 15:58:21.609 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:21.609 | INFO | "malloc" : " 14 GB", | 20260507 15:58:21.609 | INFO | "pinned" : " 29 GB" | 20260507 15:58:21.609 | INFO | } | 20260507 15:58:21.609 | INFO | Perf report for the last 5.1391 seconds: | 20260507 15:58:21.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:21.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:58:21.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | PRE_HQ : 38.5%, 1.0, 20.0ms, 0 | 20260507 15:58:21.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:21.609 | INFO | Uploaded 36GiB to GPU in 1.59189s (22.6147GiB/s) | 20260507 15:58:21.610 | INFO | Downloaded 34GiB to GPU in 1.53942s (22.0862GiB/s) | 20260507 15:58:26.748 | INFO | Analyzing chunk frames = [754688,755200) | 20260507 15:58:26.748 | INFO | MemInfo: { | 20260507 15:58:26.748 | INFO | "gpu" : " 17 GB", | 20260507 15:58:26.748 | INFO | "huge" : "", | 20260507 15:58:26.748 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:26.748 | INFO | "malloc" : " 14 GB", | 20260507 15:58:26.748 | INFO | "pinned" : " 29 GB" | 20260507 15:58:26.748 | INFO | } | 20260507 15:58:26.749 | INFO | Perf report for the last 5.13985 seconds: | 20260507 15:58:26.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:26.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.749 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:58:26.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.749 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 15:58:26.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:26.750 | INFO | Uploaded 36GiB to GPU in 1.59611s (22.5548GiB/s) | 20260507 15:58:26.750 | INFO | Downloaded 34GiB to GPU in 1.54471s (22.0107GiB/s) | 20260507 15:58:31.887 | INFO | Analyzing chunk frames = [755200,755712) | 20260507 15:58:31.887 | INFO | MemInfo: { | 20260507 15:58:31.887 | INFO | "gpu" : " 17 GB", | 20260507 15:58:31.888 | INFO | "huge" : "", | 20260507 15:58:31.888 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:31.888 | INFO | "malloc" : " 14 GB", | 20260507 15:58:31.888 | INFO | "pinned" : " 29 GB" | 20260507 15:58:31.888 | INFO | } | 20260507 15:58:31.888 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:58:31.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:31.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:58:31.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 15:58:31.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:31.888 | INFO | Uploaded 36GiB to GPU in 1.59567s (22.5611GiB/s) | 20260507 15:58:31.889 | INFO | Downloaded 34GiB to GPU in 1.54467s (22.0111GiB/s) | 20260507 15:58:37.027 | INFO | Analyzing chunk frames = [755712,756224) | 20260507 15:58:37.027 | INFO | MemInfo: { | 20260507 15:58:37.027 | INFO | "gpu" : " 17 GB", | 20260507 15:58:37.027 | INFO | "huge" : "", | 20260507 15:58:37.027 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:37.027 | INFO | "malloc" : " 14 GB", | 20260507 15:58:37.027 | INFO | "pinned" : " 29 GB" | 20260507 15:58:37.027 | INFO | } | 20260507 15:58:37.027 | INFO | Perf report for the last 5.13906 seconds: | 20260507 15:58:37.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:37.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:58:37.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 15:58:37.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:37.027 | INFO | Uploaded 36GiB to GPU in 1.59373s (22.5885GiB/s) | 20260507 15:58:37.027 | INFO | Downloaded 34GiB to GPU in 1.54216s (22.047GiB/s) | 20260507 15:58:42.166 | INFO | Analyzing chunk frames = [756224,756736) | 20260507 15:58:42.166 | INFO | MemInfo: { | 20260507 15:58:42.166 | INFO | "gpu" : " 17 GB", | 20260507 15:58:42.166 | INFO | "huge" : "", | 20260507 15:58:42.166 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:42.166 | INFO | "malloc" : " 14 GB", | 20260507 15:58:42.166 | INFO | "pinned" : " 29 GB" | 20260507 15:58:42.166 | INFO | } | 20260507 15:58:42.166 | INFO | Perf report for the last 5.13922 seconds: | 20260507 15:58:42.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:42.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 15:58:42.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 15:58:42.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:42.166 | INFO | Uploaded 36GiB to GPU in 1.58739s (22.6787GiB/s) | 20260507 15:58:42.166 | INFO | Downloaded 34GiB to GPU in 1.53722s (22.1178GiB/s) | 20260507 15:58:47.305 | INFO | Analyzing chunk frames = [756736,757248) | 20260507 15:58:47.305 | INFO | MemInfo: { | 20260507 15:58:47.305 | INFO | "gpu" : " 17 GB", | 20260507 15:58:47.305 | INFO | "huge" : "", | 20260507 15:58:47.305 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:47.305 | INFO | "malloc" : " 14 GB", | 20260507 15:58:47.305 | INFO | "pinned" : " 29 GB" | 20260507 15:58:47.305 | INFO | } | 20260507 15:58:47.305 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:58:47.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:47.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.305 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 15:58:47.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.306 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 15:58:47.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:47.306 | INFO | Uploaded 36GiB to GPU in 1.58896s (22.6563GiB/s) | 20260507 15:58:47.306 | INFO | Downloaded 34GiB to GPU in 1.53857s (22.0985GiB/s) | 20260507 15:58:52.445 | INFO | Analyzing chunk frames = [757248,757760) | 20260507 15:58:52.445 | INFO | MemInfo: { | 20260507 15:58:52.445 | INFO | "gpu" : " 17 GB", | 20260507 15:58:52.445 | INFO | "huge" : "", | 20260507 15:58:52.445 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:52.445 | INFO | "malloc" : " 14 GB", | 20260507 15:58:52.445 | INFO | "pinned" : " 29 GB" | 20260507 15:58:52.445 | INFO | } | 20260507 15:58:52.445 | INFO | Perf report for the last 5.13941 seconds: | 20260507 15:58:52.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:52.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 15:58:52.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 15:58:52.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:52.445 | INFO | Uploaded 36GiB to GPU in 1.58889s (22.6573GiB/s) | 20260507 15:58:52.445 | INFO | Downloaded 34GiB to GPU in 1.53795s (22.1073GiB/s) | 20260507 15:58:52.542 | INFO | Starting a round of Baz IO | 20260507 15:58:55.485 | INFO | Populating RTMetrics report | 20260507 15:58:55.485 | INFO | RTMetrics report populated | 20260507 15:58:55.485 | INFO | Opening tmp RTMetrics json file | 20260507 15:58:55.488 | INFO | Serializing RTMetrics to json | 20260507 15:58:55.495 | INFO | Serialization done, swapping tmp file now... | 20260507 15:58:55.497 | INFO | RTMetrics done | 20260507 15:58:55.842 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.299 seconds | 20260507 15:58:55.842 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.53% / 0.03% / 48.44% | 20260507 15:58:55.842 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.06158% / 5.16574e-05% / 5.16574e-05% | 20260507 15:58:55.842 | INFO | Baz IO overall: | 20260507 15:58:55.842 | INFO | BytesWritten : 318197858518 | 20260507 15:58:55.842 | INFO | NumEvents : 153710007205 | 20260507 15:58:57.584 | INFO | Analyzing chunk frames = [757760,758272) | 20260507 15:58:57.584 | INFO | MemInfo: { | 20260507 15:58:57.584 | INFO | "gpu" : " 17 GB", | 20260507 15:58:57.584 | INFO | "huge" : "", | 20260507 15:58:57.584 | INFO | "huge_shared" : " 29 GB", | 20260507 15:58:57.584 | INFO | "malloc" : " 14 GB", | 20260507 15:58:57.584 | INFO | "pinned" : " 29 GB" | 20260507 15:58:57.584 | INFO | } | 20260507 15:58:57.584 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:58:57.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:58:57.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:57.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:57.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:57.584 | INFO | ANALYSIS : 58.3%, 2.8, 85.1ms, 0 | 20260507 15:58:57.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:58:57.585 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 15:58:57.585 | INFO | PRE_HQ : 57.3%, 2.2, 64.3ms, 0 | 20260507 15:58:57.585 | INFO | BAZWRITER : 63.8%, 4.6, 151.4ms, 0 | 20260507 15:58:57.585 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 15:58:57.585 | INFO | Uploaded 36GiB to GPU in 1.63761s (21.9833GiB/s) | 20260507 15:58:57.585 | INFO | Downloaded 38GiB to GPU in 1.72354s (22.0477GiB/s) | 20260507 15:59:02.723 | INFO | Analyzing chunk frames = [758272,758784) | 20260507 15:59:02.723 | INFO | MemInfo: { | 20260507 15:59:02.723 | INFO | "gpu" : " 17 GB", | 20260507 15:59:02.724 | INFO | "huge" : "", | 20260507 15:59:02.724 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:02.724 | INFO | "malloc" : " 14 GB", | 20260507 15:59:02.724 | INFO | "pinned" : " 29 GB" | 20260507 15:59:02.724 | INFO | } | 20260507 15:59:02.724 | INFO | Perf report for the last 5.13942 seconds: | 20260507 15:59:02.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:02.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 15:59:02.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | PRE_HQ : 39.1%, 1.0, 20.6ms, 0 | 20260507 15:59:02.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:02.724 | INFO | Uploaded 36GiB to GPU in 1.59566s (22.5612GiB/s) | 20260507 15:59:02.725 | INFO | Downloaded 34GiB to GPU in 1.53831s (22.1022GiB/s) | 20260507 15:59:07.863 | INFO | Analyzing chunk frames = [758784,759296) | 20260507 15:59:07.863 | INFO | MemInfo: { | 20260507 15:59:07.863 | INFO | "gpu" : " 17 GB", | 20260507 15:59:07.863 | INFO | "huge" : "", | 20260507 15:59:07.863 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:07.863 | INFO | "malloc" : " 14 GB", | 20260507 15:59:07.863 | INFO | "pinned" : " 29 GB" | 20260507 15:59:07.863 | INFO | } | 20260507 15:59:07.863 | INFO | Perf report for the last 5.13929 seconds: | 20260507 15:59:07.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:07.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.863 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:59:07.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.863 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 15:59:07.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:07.864 | INFO | Uploaded 36GiB to GPU in 1.6159s (22.2786GiB/s) | 20260507 15:59:07.864 | INFO | Downloaded 34GiB to GPU in 1.53542s (22.1437GiB/s) | 20260507 15:59:13.002 | INFO | Analyzing chunk frames = [759296,759808) | 20260507 15:59:13.002 | INFO | MemInfo: { | 20260507 15:59:13.002 | INFO | "gpu" : " 17 GB", | 20260507 15:59:13.002 | INFO | "huge" : "", | 20260507 15:59:13.002 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:13.002 | INFO | "malloc" : " 14 GB", | 20260507 15:59:13.002 | INFO | "pinned" : " 29 GB" | 20260507 15:59:13.002 | INFO | } | 20260507 15:59:13.003 | INFO | Perf report for the last 5.13938 seconds: | 20260507 15:59:13.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:13.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 15:59:13.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | PRE_HQ : 38.6%, 1.0, 20.1ms, 0 | 20260507 15:59:13.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:13.003 | INFO | Uploaded 36GiB to GPU in 1.61626s (22.2736GiB/s) | 20260507 15:59:13.003 | INFO | Downloaded 34GiB to GPU in 1.53499s (22.1499GiB/s) | 20260507 15:59:18.141 | INFO | Analyzing chunk frames = [759808,760320) | 20260507 15:59:18.141 | INFO | MemInfo: { | 20260507 15:59:18.141 | INFO | "gpu" : " 17 GB", | 20260507 15:59:18.141 | INFO | "huge" : "", | 20260507 15:59:18.141 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:18.141 | INFO | "malloc" : " 14 GB", | 20260507 15:59:18.141 | INFO | "pinned" : " 29 GB" | 20260507 15:59:18.141 | INFO | } | 20260507 15:59:18.142 | INFO | Perf report for the last 5.13904 seconds: | 20260507 15:59:18.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:18.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 15:59:18.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260507 15:59:18.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:18.142 | INFO | Uploaded 36GiB to GPU in 1.61117s (22.344GiB/s) | 20260507 15:59:18.142 | INFO | Downloaded 34GiB to GPU in 1.53002s (22.222GiB/s) | 20260507 15:59:23.281 | INFO | Analyzing chunk frames = [760320,760832) | 20260507 15:59:23.281 | INFO | MemInfo: { | 20260507 15:59:23.281 | INFO | "gpu" : " 17 GB", | 20260507 15:59:23.281 | INFO | "huge" : "", | 20260507 15:59:23.281 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:23.281 | INFO | "malloc" : " 14 GB", | 20260507 15:59:23.281 | INFO | "pinned" : " 29 GB" | 20260507 15:59:23.281 | INFO | } | 20260507 15:59:23.281 | INFO | Perf report for the last 5.13926 seconds: | 20260507 15:59:23.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:23.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | ANALYSIS : 56.1%, 3.0, 87.6ms, 0 | 20260507 15:59:23.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 15:59:23.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:23.281 | INFO | Uploaded 36GiB to GPU in 1.60823s (22.3849GiB/s) | 20260507 15:59:23.281 | INFO | Downloaded 34GiB to GPU in 1.52825s (22.2477GiB/s) | 20260507 15:59:28.420 | INFO | Analyzing chunk frames = [760832,761344) | 20260507 15:59:28.420 | INFO | MemInfo: { | 20260507 15:59:28.420 | INFO | "gpu" : " 17 GB", | 20260507 15:59:28.420 | INFO | "huge" : "", | 20260507 15:59:28.420 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:28.420 | INFO | "malloc" : " 14 GB", | 20260507 15:59:28.420 | INFO | "pinned" : " 29 GB" | 20260507 15:59:28.420 | INFO | } | 20260507 15:59:28.420 | INFO | Perf report for the last 5.13935 seconds: | 20260507 15:59:28.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:28.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.420 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 15:59:28.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.420 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 15:59:28.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:28.421 | INFO | Uploaded 36GiB to GPU in 1.60813s (22.3862GiB/s) | 20260507 15:59:28.421 | INFO | Downloaded 34GiB to GPU in 1.52894s (22.2376GiB/s) | 20260507 15:59:33.559 | INFO | Analyzing chunk frames = [761344,761856) | 20260507 15:59:33.559 | INFO | MemInfo: { | 20260507 15:59:33.559 | INFO | "gpu" : " 17 GB", | 20260507 15:59:33.559 | INFO | "huge" : "", | 20260507 15:59:33.559 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:33.559 | INFO | "malloc" : " 14 GB", | 20260507 15:59:33.559 | INFO | "pinned" : " 29 GB" | 20260507 15:59:33.559 | INFO | } | 20260507 15:59:33.559 | INFO | Perf report for the last 5.13918 seconds: | 20260507 15:59:33.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:33.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.559 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 15:59:33.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.559 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 15:59:33.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:33.560 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 15:59:33.560 | INFO | Total mem usage: 16.4 GiB | 20260507 15:59:33.560 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 15:59:33.560 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -6.1%) | 20260507 15:59:33.560 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 15:59:33.560 | INFO | High Water Breakdown: | 20260507 15:59:33.560 | INFO | 59.9% : 9.26 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 15:59:33.560 | INFO | 40% : 6.18 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 15:59:33.560 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 15:59:33.560 | INFO | ------------------------------------------ | 20260507 15:59:33.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:59:33.560 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 15:59:33.560 | INFO | Total mem usage: 26.5 GiB | 20260507 15:59:33.560 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 15:59:33.560 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 15:59:33.560 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 15:59:33.560 | INFO | High Water Breakdown: | 20260507 15:59:33.560 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:59:33.560 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:59:33.560 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:59:33.560 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:59:33.560 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:59:33.560 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:59:33.560 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 15:59:33.560 | INFO | Uploaded 36GiB to GPU in 1.60833s (22.3834GiB/s) | 20260507 15:59:33.560 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 15:59:33.560 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:59:33.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 15:59:33.560 | INFO | Downloaded 34GiB to GPU in 1.52914s (22.2348GiB/s) | 20260507 15:59:33.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 15:59:33.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 15:59:33.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 15:59:33.560 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 15:59:33.560 | INFO | ------------------------------------------ | 20260507 15:59:33.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:59:33.560 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 15:59:33.560 | INFO | Total mem usage: 27.8 GiB | 20260507 15:59:33.560 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 15:59:33.560 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 15:59:33.560 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 15:59:33.560 | INFO | High Water Breakdown: | 20260507 15:59:33.560 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 15:59:33.560 | INFO | ------------------------------------------ | 20260507 15:59:33.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:59:33.560 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 15:59:33.560 | INFO | Total mem usage: 15.7 GiB | 20260507 15:59:33.560 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 15:59:33.560 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 15:59:33.561 | INFO | Unused Cache: 865 MiB (Waste: 5.39%) | 20260507 15:59:33.561 | INFO | High Water Breakdown: | 20260507 15:59:33.561 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 15:59:33.561 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 15:59:33.561 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 15:59:33.561 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 15:59:33.561 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 15:59:33.561 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 15:59:33.561 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 15:59:33.561 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 15:59:33.561 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 15:59:33.561 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 15:59:33.561 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 15:59:33.561 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 15:59:33.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 15:59:33.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 15:59:33.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 15:59:33.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 15:59:33.561 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 15:59:33.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 15:59:33.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 15:59:33.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 15:59:33.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 15:59:33.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 15:59:33.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 15:59:33.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 15:59:33.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 15:59:33.561 | INFO | ------------------------------------------ | 20260507 15:59:33.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 15:59:33.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 15:59:33.561 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:59:33.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:59:33.561 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 0.30%, AggregateStats -- 9.03e-02 / -nan, 7.16e-03 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 11.08%, Baseline -- 3.31e+00 / -nan, 2.43e-02 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.39e-02 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 3.41%, Metrics -- 1.02e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.77e-01 / -nan, 5.58e-02 / 0.00e+00 | 20260507 15:59:33.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 15:59:33.561 | INFO | PerfReport: Gathered 1536 samples. | 20260507 15:59:33.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 15:59:33.561 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 18.74%, Upload -- 1.67e+01 / 0.00e+00, 4.40e-01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 18.21%, Download -- 1.62e+01 / 0.00e+00, 7.40e-01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 33.45%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 0.59%, LockUpload -- 5.26e-01 / 0.00e+00, 3.67e+00 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 0.00%, LockDownload -- 5.61e-04 / 0.00e+00, 3.98e-04 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 28.99%, LockCompute -- 2.59e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 15:59:33.561 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 15:59:33.656 | INFO | Starting a round of Baz IO | 20260507 15:59:36.596 | INFO | Populating RTMetrics report | 20260507 15:59:36.597 | INFO | RTMetrics report populated | 20260507 15:59:36.597 | INFO | Opening tmp RTMetrics json file | 20260507 15:59:36.674 | INFO | Serializing RTMetrics to json | 20260507 15:59:36.702 | INFO | Serialization done, swapping tmp file now... | 20260507 15:59:36.706 | INFO | RTMetrics done | 20260507 15:59:36.869 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.212 seconds | 20260507 15:59:36.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.49% / 0.03% / 48.47% | 20260507 15:59:36.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.04753% / 5.16574e-05% / 2.78155e-05% | 20260507 15:59:36.869 | INFO | Baz IO overall: | 20260507 15:59:36.869 | INFO | BytesWritten : 320102220203 | 20260507 15:59:36.869 | INFO | NumEvents : 154636335238 | 20260507 15:59:38.699 | INFO | Analyzing chunk frames = [761856,762368) | 20260507 15:59:38.699 | INFO | MemInfo: { | 20260507 15:59:38.699 | INFO | "gpu" : " 17 GB", | 20260507 15:59:38.699 | INFO | "huge" : "", | 20260507 15:59:38.699 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:38.699 | INFO | "malloc" : " 14 GB", | 20260507 15:59:38.699 | INFO | "pinned" : " 29 GB" | 20260507 15:59:38.699 | INFO | } | 20260507 15:59:38.699 | INFO | Perf report for the last 5.1398 seconds: | 20260507 15:59:38.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:38.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:38.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:38.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:38.699 | INFO | ANALYSIS : 58.2%, 2.7, 83.5ms, 0 | 20260507 15:59:38.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:38.699 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260507 15:59:38.699 | INFO | PRE_HQ : 57.2%, 2.2, 64.9ms, 0 | 20260507 15:59:38.699 | INFO | BAZWRITER : 62.1%, 4.7, 149.5ms, 0 | 20260507 15:59:38.699 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 15:59:38.700 | INFO | Uploaded 36GiB to GPU in 1.63796s (21.9785GiB/s) | 20260507 15:59:38.700 | INFO | Downloaded 38GiB to GPU in 1.72299s (22.0546GiB/s) | 20260507 15:59:43.838 | INFO | Analyzing chunk frames = [762368,762880) | 20260507 15:59:43.838 | INFO | MemInfo: { | 20260507 15:59:43.838 | INFO | "gpu" : " 17 GB", | 20260507 15:59:43.838 | INFO | "huge" : "", | 20260507 15:59:43.838 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:43.838 | INFO | "malloc" : " 14 GB", | 20260507 15:59:43.838 | INFO | "pinned" : " 29 GB" | 20260507 15:59:43.838 | INFO | } | 20260507 15:59:43.838 | INFO | Perf report for the last 5.13921 seconds: | 20260507 15:59:43.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:43.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.838 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 15:59:43.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.839 | INFO | PRE_HQ : 38.3%, 1.0, 19.9ms, 0 | 20260507 15:59:43.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:43.839 | INFO | Uploaded 36GiB to GPU in 1.61114s (22.3444GiB/s) | 20260507 15:59:43.839 | INFO | Downloaded 34GiB to GPU in 1.52963s (22.2276GiB/s) | 20260507 15:59:48.977 | INFO | Analyzing chunk frames = [762880,763392) | 20260507 15:59:48.977 | INFO | MemInfo: { | 20260507 15:59:48.977 | INFO | "gpu" : " 17 GB", | 20260507 15:59:48.977 | INFO | "huge" : "", | 20260507 15:59:48.978 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:48.978 | INFO | "malloc" : " 14 GB", | 20260507 15:59:48.978 | INFO | "pinned" : " 29 GB" | 20260507 15:59:48.978 | INFO | } | 20260507 15:59:48.978 | INFO | Perf report for the last 5.13931 seconds: | 20260507 15:59:48.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:48.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | ANALYSIS : 56.4%, 3.0, 87.6ms, 0 | 20260507 15:59:48.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 15:59:48.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:48.978 | INFO | Uploaded 36GiB to GPU in 1.61635s (22.2724GiB/s) | 20260507 15:59:48.979 | INFO | Downloaded 34GiB to GPU in 1.53651s (22.128GiB/s) | 20260507 15:59:54.117 | INFO | Analyzing chunk frames = [763392,763904) | 20260507 15:59:54.117 | INFO | MemInfo: { | 20260507 15:59:54.117 | INFO | "gpu" : " 17 GB", | 20260507 15:59:54.117 | INFO | "huge" : "", | 20260507 15:59:54.117 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:54.117 | INFO | "malloc" : " 14 GB", | 20260507 15:59:54.117 | INFO | "pinned" : " 29 GB" | 20260507 15:59:54.117 | INFO | } | 20260507 15:59:54.117 | INFO | Perf report for the last 5.13958 seconds: | 20260507 15:59:54.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:54.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.117 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 15:59:54.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.117 | INFO | PRE_HQ : 39.0%, 1.0, 20.3ms, 0 | 20260507 15:59:54.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:54.118 | INFO | Uploaded 36GiB to GPU in 1.61597s (22.2776GiB/s) | 20260507 15:59:54.118 | INFO | Downloaded 34GiB to GPU in 1.53437s (22.159GiB/s) | 20260507 15:59:59.256 | INFO | Analyzing chunk frames = [763904,764416) | 20260507 15:59:59.256 | INFO | MemInfo: { | 20260507 15:59:59.256 | INFO | "gpu" : " 17 GB", | 20260507 15:59:59.256 | INFO | "huge" : "", | 20260507 15:59:59.256 | INFO | "huge_shared" : " 29 GB", | 20260507 15:59:59.256 | INFO | "malloc" : " 14 GB", | 20260507 15:59:59.256 | INFO | "pinned" : " 29 GB" | 20260507 15:59:59.256 | INFO | } | 20260507 15:59:59.256 | INFO | Perf report for the last 5.13902 seconds: | 20260507 15:59:59.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 15:59:59.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.256 | INFO | ANALYSIS : 55.9%, 3.0, 87.1ms, 0 | 20260507 15:59:59.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.256 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260507 15:59:59.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 15:59:59.257 | INFO | Uploaded 36GiB to GPU in 1.60974s (22.3639GiB/s) | 20260507 15:59:59.257 | INFO | Downloaded 34GiB to GPU in 1.52988s (22.224GiB/s) | 20260507 16:00:04.395 | INFO | Analyzing chunk frames = [764416,764928) | 20260507 16:00:04.395 | INFO | MemInfo: { | 20260507 16:00:04.395 | INFO | "gpu" : " 17 GB", | 20260507 16:00:04.395 | INFO | "huge" : "", | 20260507 16:00:04.395 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:04.395 | INFO | "malloc" : " 14 GB", | 20260507 16:00:04.395 | INFO | "pinned" : " 29 GB" | 20260507 16:00:04.395 | INFO | } | 20260507 16:00:04.396 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:00:04.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:04.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 16:00:04.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:00:04.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:04.396 | INFO | Uploaded 36GiB to GPU in 1.60665s (22.4069GiB/s) | 20260507 16:00:04.396 | INFO | Downloaded 34GiB to GPU in 1.52795s (22.2521GiB/s) | 20260507 16:00:09.535 | INFO | Analyzing chunk frames = [764928,765440) | 20260507 16:00:09.535 | INFO | MemInfo: { | 20260507 16:00:09.535 | INFO | "gpu" : " 17 GB", | 20260507 16:00:09.535 | INFO | "huge" : "", | 20260507 16:00:09.535 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:09.535 | INFO | "malloc" : " 14 GB", | 20260507 16:00:09.535 | INFO | "pinned" : " 29 GB" | 20260507 16:00:09.535 | INFO | } | 20260507 16:00:09.535 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:00:09.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:09.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 16:00:09.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 16:00:09.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:09.535 | INFO | Uploaded 36GiB to GPU in 1.60886s (22.3761GiB/s) | 20260507 16:00:09.535 | INFO | Downloaded 34GiB to GPU in 1.52981s (22.2249GiB/s) | 20260507 16:00:14.674 | INFO | Analyzing chunk frames = [765440,765952) | 20260507 16:00:14.674 | INFO | MemInfo: { | 20260507 16:00:14.674 | INFO | "gpu" : " 17 GB", | 20260507 16:00:14.674 | INFO | "huge" : "", | 20260507 16:00:14.674 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:14.674 | INFO | "malloc" : " 14 GB", | 20260507 16:00:14.674 | INFO | "pinned" : " 29 GB" | 20260507 16:00:14.674 | INFO | } | 20260507 16:00:14.674 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:00:14.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:14.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.674 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:00:14.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.675 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 16:00:14.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:14.675 | INFO | Uploaded 36GiB to GPU in 1.60867s (22.3787GiB/s) | 20260507 16:00:14.675 | INFO | Downloaded 34GiB to GPU in 1.52877s (22.2401GiB/s) | 20260507 16:00:14.771 | INFO | Starting a round of Baz IO | 20260507 16:00:17.718 | INFO | Populating RTMetrics report | 20260507 16:00:17.718 | INFO | RTMetrics report populated | 20260507 16:00:17.718 | INFO | Opening tmp RTMetrics json file | 20260507 16:00:17.765 | INFO | Serializing RTMetrics to json | 20260507 16:00:17.863 | INFO | Serialization done, swapping tmp file now... | 20260507 16:00:17.867 | INFO | RTMetrics done | 20260507 16:00:18.060 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.289 seconds | 20260507 16:00:18.060 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.46% / 0.03% / 48.51% | 20260507 16:00:18.060 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.03483% / 6.75519e-05% / 1.58946e-05% | 20260507 16:00:18.060 | INFO | Baz IO overall: | 20260507 16:00:18.060 | INFO | BytesWritten : 322008634741 | 20260507 16:00:18.060 | INFO | NumEvents : 155563792027 | 20260507 16:00:19.814 | INFO | Analyzing chunk frames = [765952,766464) | 20260507 16:00:19.814 | INFO | MemInfo: { | 20260507 16:00:19.814 | INFO | "gpu" : " 17 GB", | 20260507 16:00:19.814 | INFO | "huge" : "", | 20260507 16:00:19.814 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:19.814 | INFO | "malloc" : " 14 GB", | 20260507 16:00:19.814 | INFO | "pinned" : " 29 GB" | 20260507 16:00:19.814 | INFO | } | 20260507 16:00:19.814 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:00:19.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:19.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:19.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:19.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:19.814 | INFO | ANALYSIS : 58.3%, 2.8, 85.5ms, 0 | 20260507 16:00:19.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:19.814 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260507 16:00:19.814 | INFO | PRE_HQ : 57.5%, 2.2, 64.6ms, 0 | 20260507 16:00:19.814 | INFO | BAZWRITER : 63.5%, 4.5, 148.4ms, 0 | 20260507 16:00:19.814 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 16:00:19.814 | INFO | Uploaded 36GiB to GPU in 1.64177s (21.9275GiB/s) | 20260507 16:00:19.814 | INFO | Downloaded 38GiB to GPU in 1.72001s (22.0929GiB/s) | 20260507 16:00:24.953 | INFO | Analyzing chunk frames = [766464,766976) | 20260507 16:00:24.953 | INFO | MemInfo: { | 20260507 16:00:24.953 | INFO | "gpu" : " 17 GB", | 20260507 16:00:24.953 | INFO | "huge" : "", | 20260507 16:00:24.953 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:24.953 | INFO | "malloc" : " 14 GB", | 20260507 16:00:24.953 | INFO | "pinned" : " 29 GB" | 20260507 16:00:24.953 | INFO | } | 20260507 16:00:24.953 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:00:24.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:24.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.953 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 16:00:24.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.954 | INFO | PRE_HQ : 38.8%, 1.0, 20.2ms, 0 | 20260507 16:00:24.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:24.954 | INFO | Uploaded 36GiB to GPU in 1.59207s (22.612GiB/s) | 20260507 16:00:24.954 | INFO | Downloaded 34GiB to GPU in 1.54009s (22.0766GiB/s) | 20260507 16:00:30.092 | INFO | Analyzing chunk frames = [766976,767488) | 20260507 16:00:30.092 | INFO | MemInfo: { | 20260507 16:00:30.092 | INFO | "gpu" : " 17 GB", | 20260507 16:00:30.092 | INFO | "huge" : "", | 20260507 16:00:30.093 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:30.093 | INFO | "malloc" : " 14 GB", | 20260507 16:00:30.093 | INFO | "pinned" : " 29 GB" | 20260507 16:00:30.093 | INFO | } | 20260507 16:00:30.093 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:00:30.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:30.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.093 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:00:30.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.093 | INFO | PRE_HQ : 38.4%, 1.0, 19.9ms, 0 | 20260507 16:00:30.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:30.094 | INFO | Uploaded 36GiB to GPU in 1.59567s (22.561GiB/s) | 20260507 16:00:30.094 | INFO | Downloaded 34GiB to GPU in 1.54423s (22.0174GiB/s) | 20260507 16:00:35.232 | INFO | Analyzing chunk frames = [767488,768000) | 20260507 16:00:35.232 | INFO | MemInfo: { | 20260507 16:00:35.232 | INFO | "gpu" : " 17 GB", | 20260507 16:00:35.232 | INFO | "huge" : "", | 20260507 16:00:35.232 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:35.232 | INFO | "malloc" : " 14 GB", | 20260507 16:00:35.232 | INFO | "pinned" : " 29 GB" | 20260507 16:00:35.232 | INFO | } | 20260507 16:00:35.232 | INFO | Perf report for the last 5.13892 seconds: | 20260507 16:00:35.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:35.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 16:00:35.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | PRE_HQ : 39.7%, 1.0, 20.7ms, 0 | 20260507 16:00:35.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:35.232 | INFO | Uploaded 36GiB to GPU in 1.59609s (22.5552GiB/s) | 20260507 16:00:35.232 | INFO | Downloaded 34GiB to GPU in 1.54433s (22.016GiB/s) | 20260507 16:00:40.371 | INFO | Analyzing chunk frames = [768000,768512) | 20260507 16:00:40.371 | INFO | MemInfo: { | 20260507 16:00:40.371 | INFO | "gpu" : " 17 GB", | 20260507 16:00:40.371 | INFO | "huge" : "", | 20260507 16:00:40.371 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:40.371 | INFO | "malloc" : " 14 GB", | 20260507 16:00:40.371 | INFO | "pinned" : " 29 GB" | 20260507 16:00:40.371 | INFO | } | 20260507 16:00:40.371 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:00:40.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:40.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.371 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:00:40.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.371 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:00:40.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:40.372 | INFO | Uploaded 36GiB to GPU in 1.58672s (22.6883GiB/s) | 20260507 16:00:40.372 | INFO | Downloaded 34GiB to GPU in 1.53736s (22.1158GiB/s) | 20260507 16:00:45.510 | INFO | Analyzing chunk frames = [768512,769024) | 20260507 16:00:45.510 | INFO | MemInfo: { | 20260507 16:00:45.510 | INFO | "gpu" : " 17 GB", | 20260507 16:00:45.510 | INFO | "huge" : "", | 20260507 16:00:45.510 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:45.510 | INFO | "malloc" : " 14 GB", | 20260507 16:00:45.510 | INFO | "pinned" : " 29 GB" | 20260507 16:00:45.510 | INFO | } | 20260507 16:00:45.510 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:00:45.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:45.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.510 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 16:00:45.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.511 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:00:45.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:45.511 | INFO | Uploaded 36GiB to GPU in 1.58772s (22.674GiB/s) | 20260507 16:00:45.511 | INFO | Downloaded 34GiB to GPU in 1.53654s (22.1277GiB/s) | 20260507 16:00:50.649 | INFO | Analyzing chunk frames = [769024,769536) | 20260507 16:00:50.649 | INFO | MemInfo: { | 20260507 16:00:50.649 | INFO | "gpu" : " 17 GB", | 20260507 16:00:50.649 | INFO | "huge" : "", | 20260507 16:00:50.649 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:50.649 | INFO | "malloc" : " 14 GB", | 20260507 16:00:50.650 | INFO | "pinned" : " 29 GB" | 20260507 16:00:50.650 | INFO | } | 20260507 16:00:50.650 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:00:50.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:50.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:00:50.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:00:50.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:50.650 | INFO | Uploaded 36GiB to GPU in 1.58903s (22.6554GiB/s) | 20260507 16:00:50.650 | INFO | Downloaded 34GiB to GPU in 1.53885s (22.0944GiB/s) | 20260507 16:00:55.789 | INFO | Analyzing chunk frames = [769536,770048) | 20260507 16:00:55.789 | INFO | MemInfo: { | 20260507 16:00:55.789 | INFO | "gpu" : " 17 GB", | 20260507 16:00:55.789 | INFO | "huge" : "", | 20260507 16:00:55.789 | INFO | "huge_shared" : " 29 GB", | 20260507 16:00:55.789 | INFO | "malloc" : " 14 GB", | 20260507 16:00:55.789 | INFO | "pinned" : " 29 GB" | 20260507 16:00:55.789 | INFO | } | 20260507 16:00:55.789 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:00:55.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:00:55.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.789 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:00:55.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.789 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 16:00:55.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:00:55.790 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:00:55.790 | INFO | Total mem usage: 16.4 GiB | 20260507 16:00:55.790 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:00:55.790 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.32%) | 20260507 16:00:55.790 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:00:55.790 | INFO | High Water Breakdown: | 20260507 16:00:55.790 | INFO | 59.5% : 9.02 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:00:55.790 | INFO | 40.3% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:00:55.790 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:00:55.790 | INFO | ------------------------------------------ | 20260507 16:00:55.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:00:55.790 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:00:55.790 | INFO | Total mem usage: 26.5 GiB | 20260507 16:00:55.790 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:00:55.790 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 16:00:55.790 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 16:00:55.790 | INFO | High Water Breakdown: | 20260507 16:00:55.790 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:00:55.790 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:00:55.790 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:00:55.790 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:00:55.790 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:00:55.790 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:00:55.790 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:00:55.790 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:00:55.790 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:00:55.790 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:00:55.790 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:00:55.790 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:00:55.790 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:00:55.790 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:00:55.790 | INFO | ------------------------------------------ | 20260507 16:00:55.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:00:55.790 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:00:55.790 | INFO | Total mem usage: 27.8 GiB | 20260507 16:00:55.790 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:00:55.790 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:00:55.790 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:00:55.790 | INFO | High Water Breakdown: | 20260507 16:00:55.790 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:00:55.790 | INFO | ------------------------------------------ | 20260507 16:00:55.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:00:55.790 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:00:55.790 | INFO | Total mem usage: 15.7 GiB | 20260507 16:00:55.790 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:00:55.790 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:00:55.790 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:00:55.790 | INFO | High Water Breakdown: | 20260507 16:00:55.790 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:00:55.790 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:00:55.790 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:00:55.790 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:00:55.790 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:00:55.790 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:00:55.790 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:00:55.790 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:00:55.790 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:00:55.790 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:00:55.790 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:00:55.790 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:00:55.790 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:00:55.790 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:00:55.790 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:00:55.790 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:00:55.790 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:00:55.790 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:00:55.790 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:00:55.790 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:00:55.790 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:00:55.790 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:00:55.790 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:00:55.790 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:00:55.790 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:00:55.790 | INFO | ------------------------------------------ | 20260507 16:00:55.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:00:55.790 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:00:55.790 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:00:55.790 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:00:55.790 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 0.30%, AggregateStats -- 8.99e-02 / -nan, 7.03e-03 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 11.07%, Baseline -- 3.30e+00 / -nan, 2.42e-02 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.42e-02 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 2.96e+00 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 3.41%, Metrics -- 1.02e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 16:00:55.790 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.77e-01 / -nan, 5.53e-02 / 0.00e+00 | 20260507 16:00:55.790 | INFO | Uploaded 36GiB to GPU in 1.58854s (22.6623GiB/s) | 20260507 16:00:55.790 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:00:55.791 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:00:55.791 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:00:55.791 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 18.78%, Upload -- 1.67e+01 / 0.00e+00, 4.40e-01 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 18.23%, Download -- 1.62e+01 / 0.00e+00, 7.46e-01 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 33.49%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 0.60%, LockUpload -- 5.38e-01 / 0.00e+00, 3.67e+00 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 0.00%, LockDownload -- 5.38e-04 / 0.00e+00, 3.68e-04 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 28.87%, LockCompute -- 2.57e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 16:00:55.791 | INFO | PerfReport: 0.02%, Misc -- 1.61e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 16:00:55.790 | INFO | Downloaded 34GiB to GPU in 1.5382s (22.1037GiB/s) | 20260507 16:00:55.888 | INFO | Starting a round of Baz IO | 20260507 16:00:58.824 | INFO | Populating RTMetrics report | 20260507 16:00:58.824 | INFO | RTMetrics report populated | 20260507 16:00:58.824 | INFO | Opening tmp RTMetrics json file | 20260507 16:00:58.886 | INFO | Serializing RTMetrics to json | 20260507 16:00:58.892 | INFO | Serialization done, swapping tmp file now... | 20260507 16:00:58.893 | INFO | RTMetrics done | 20260507 16:00:58.993 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.104 seconds | 20260507 16:00:58.993 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.43% / 0.03% / 48.54% | 20260507 16:00:58.993 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.02122% / 5.5631e-05% / 3.17891e-05% | 20260507 16:00:58.993 | INFO | Baz IO overall: | 20260507 16:00:58.993 | INFO | BytesWritten : 323914844868 | 20260507 16:00:58.993 | INFO | NumEvents : 156491351943 | 20260507 16:01:00.929 | INFO | Analyzing chunk frames = [770048,770560) | 20260507 16:01:00.929 | INFO | MemInfo: { | 20260507 16:01:00.929 | INFO | "gpu" : " 17 GB", | 20260507 16:01:00.929 | INFO | "huge" : "", | 20260507 16:01:00.929 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:00.929 | INFO | "malloc" : " 14 GB", | 20260507 16:01:00.929 | INFO | "pinned" : " 29 GB" | 20260507 16:01:00.929 | INFO | } | 20260507 16:01:00.929 | INFO | Perf report for the last 5.13969 seconds: | 20260507 16:01:00.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:00.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:00.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:00.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:00.929 | INFO | ANALYSIS : 58.1%, 2.7, 83.5ms, 0 | 20260507 16:01:00.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:00.929 | INFO | RT_METRICS : 47.8%, 1.0, 25.0ms, 0 | 20260507 16:01:00.929 | INFO | PRE_HQ : 57.3%, 2.2, 65.0ms, 0 | 20260507 16:01:00.929 | INFO | BAZWRITER : 60.0%, 4.8, 146.6ms, 0 | 20260507 16:01:00.929 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 16:01:00.929 | INFO | Uploaded 36GiB to GPU in 1.64904s (21.8309GiB/s) | 20260507 16:01:00.929 | INFO | Downloaded 38GiB to GPU in 1.72634s (22.0119GiB/s) | 20260507 16:01:06.068 | INFO | Analyzing chunk frames = [770560,771072) | 20260507 16:01:06.068 | INFO | MemInfo: { | 20260507 16:01:06.068 | INFO | "gpu" : " 17 GB", | 20260507 16:01:06.068 | INFO | "huge" : "", | 20260507 16:01:06.068 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:06.068 | INFO | "malloc" : " 14 GB", | 20260507 16:01:06.068 | INFO | "pinned" : " 29 GB" | 20260507 16:01:06.068 | INFO | } | 20260507 16:01:06.068 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:01:06.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:06.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.069 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 16:01:06.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.069 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:01:06.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:06.069 | INFO | Uploaded 36GiB to GPU in 1.61139s (22.3409GiB/s) | 20260507 16:01:06.069 | INFO | Downloaded 34GiB to GPU in 1.53071s (22.212GiB/s) | 20260507 16:01:11.207 | INFO | Analyzing chunk frames = [771072,771584) | 20260507 16:01:11.207 | INFO | MemInfo: { | 20260507 16:01:11.207 | INFO | "gpu" : " 17 GB", | 20260507 16:01:11.207 | INFO | "huge" : "", | 20260507 16:01:11.207 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:11.207 | INFO | "malloc" : " 14 GB", | 20260507 16:01:11.207 | INFO | "pinned" : " 29 GB" | 20260507 16:01:11.207 | INFO | } | 20260507 16:01:11.208 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:01:11.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:11.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:01:11.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 16:01:11.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:11.208 | INFO | Uploaded 36GiB to GPU in 1.61496s (22.2915GiB/s) | 20260507 16:01:11.208 | INFO | Downloaded 34GiB to GPU in 1.53615s (22.1332GiB/s) | 20260507 16:01:16.347 | INFO | Analyzing chunk frames = [771584,772096) | 20260507 16:01:16.347 | INFO | MemInfo: { | 20260507 16:01:16.347 | INFO | "gpu" : " 17 GB", | 20260507 16:01:16.347 | INFO | "huge" : "", | 20260507 16:01:16.347 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:16.347 | INFO | "malloc" : " 14 GB", | 20260507 16:01:16.347 | INFO | "pinned" : " 29 GB" | 20260507 16:01:16.347 | INFO | } | 20260507 16:01:16.347 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:01:16.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:16.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.347 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 16:01:16.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.347 | INFO | PRE_HQ : 39.0%, 1.0, 20.3ms, 0 | 20260507 16:01:16.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:16.348 | INFO | Uploaded 36GiB to GPU in 1.6154s (22.2855GiB/s) | 20260507 16:01:16.348 | INFO | Downloaded 34GiB to GPU in 1.53595s (22.1362GiB/s) | 20260507 16:01:21.486 | INFO | Analyzing chunk frames = [772096,772608) | 20260507 16:01:21.486 | INFO | MemInfo: { | 20260507 16:01:21.486 | INFO | "gpu" : " 17 GB", | 20260507 16:01:21.486 | INFO | "huge" : "", | 20260507 16:01:21.486 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:21.486 | INFO | "malloc" : " 14 GB", | 20260507 16:01:21.486 | INFO | "pinned" : " 29 GB" | 20260507 16:01:21.486 | INFO | } | 20260507 16:01:21.486 | INFO | Perf report for the last 5.13868 seconds: | 20260507 16:01:21.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:21.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 16:01:21.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 16:01:21.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:21.486 | INFO | Uploaded 36GiB to GPU in 1.61272s (22.3226GiB/s) | 20260507 16:01:21.486 | INFO | Downloaded 34GiB to GPU in 1.53184s (22.1956GiB/s) | 20260507 16:01:26.625 | INFO | Analyzing chunk frames = [772608,773120) | 20260507 16:01:26.625 | INFO | MemInfo: { | 20260507 16:01:26.625 | INFO | "gpu" : " 17 GB", | 20260507 16:01:26.625 | INFO | "huge" : "", | 20260507 16:01:26.625 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:26.625 | INFO | "malloc" : " 14 GB", | 20260507 16:01:26.625 | INFO | "pinned" : " 29 GB" | 20260507 16:01:26.625 | INFO | } | 20260507 16:01:26.625 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:01:26.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:26.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.625 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:01:26.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.625 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 16:01:26.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:26.626 | INFO | Uploaded 36GiB to GPU in 1.60806s (22.3873GiB/s) | 20260507 16:01:26.626 | INFO | Downloaded 34GiB to GPU in 1.5281s (22.2498GiB/s) | 20260507 16:01:31.764 | INFO | Analyzing chunk frames = [773120,773632) | 20260507 16:01:31.764 | INFO | MemInfo: { | 20260507 16:01:31.764 | INFO | "gpu" : " 17 GB", | 20260507 16:01:31.764 | INFO | "huge" : "", | 20260507 16:01:31.764 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:31.764 | INFO | "malloc" : " 14 GB", | 20260507 16:01:31.764 | INFO | "pinned" : " 29 GB" | 20260507 16:01:31.765 | INFO | } | 20260507 16:01:31.765 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:01:31.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:31.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:01:31.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:01:31.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:31.765 | INFO | Uploaded 36GiB to GPU in 1.60774s (22.3917GiB/s) | 20260507 16:01:31.765 | INFO | Downloaded 34GiB to GPU in 1.52968s (22.2269GiB/s) | 20260507 16:01:36.904 | INFO | Analyzing chunk frames = [773632,774144) | 20260507 16:01:36.904 | INFO | MemInfo: { | 20260507 16:01:36.904 | INFO | "gpu" : " 17 GB", | 20260507 16:01:36.904 | INFO | "huge" : "", | 20260507 16:01:36.904 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:36.904 | INFO | "malloc" : " 14 GB", | 20260507 16:01:36.904 | INFO | "pinned" : " 29 GB" | 20260507 16:01:36.904 | INFO | } | 20260507 16:01:36.904 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:01:36.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:36.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:01:36.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:01:36.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:36.904 | INFO | Uploaded 36GiB to GPU in 1.60798s (22.3884GiB/s) | 20260507 16:01:36.904 | INFO | Downloaded 34GiB to GPU in 1.52852s (22.2437GiB/s) | 20260507 16:01:37.004 | INFO | Starting a round of Baz IO | 20260507 16:01:39.946 | INFO | Populating RTMetrics report | 20260507 16:01:39.946 | INFO | RTMetrics report populated | 20260507 16:01:39.946 | INFO | Opening tmp RTMetrics json file | 20260507 16:01:39.982 | INFO | Serializing RTMetrics to json | 20260507 16:01:40.055 | INFO | Serialization done, swapping tmp file now... | 20260507 16:01:40.089 | INFO | RTMetrics done | 20260507 16:01:40.305 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.301 seconds | 20260507 16:01:40.305 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.39% / 0.03% / 48.57% | 20260507 16:01:40.305 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.9976% / 6.75519e-05% / 2.38419e-05% | 20260507 16:01:40.305 | INFO | Baz IO overall: | 20260507 16:01:40.305 | INFO | BytesWritten : 325818674400 | 20260507 16:01:40.305 | INFO | NumEvents : 157417099606 | 20260507 16:01:42.043 | INFO | Analyzing chunk frames = [774144,774656) | 20260507 16:01:42.043 | INFO | MemInfo: { | 20260507 16:01:42.043 | INFO | "gpu" : " 17 GB", | 20260507 16:01:42.043 | INFO | "huge" : "", | 20260507 16:01:42.043 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:42.043 | INFO | "malloc" : " 14 GB", | 20260507 16:01:42.043 | INFO | "pinned" : " 29 GB" | 20260507 16:01:42.043 | INFO | } | 20260507 16:01:42.044 | INFO | Perf report for the last 5.13965 seconds: | 20260507 16:01:42.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:42.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:42.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:42.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:42.044 | INFO | ANALYSIS : 58.0%, 2.8, 85.3ms, 0 | 20260507 16:01:42.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:42.044 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260507 16:01:42.044 | INFO | PRE_HQ : 57.2%, 2.2, 66.4ms, 0 | 20260507 16:01:42.044 | INFO | BAZWRITER : 63.9%, 4.5, 148.7ms, 0 | 20260507 16:01:42.044 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260507 16:01:42.044 | INFO | Uploaded 36GiB to GPU in 1.63807s (21.9771GiB/s) | 20260507 16:01:42.044 | INFO | Downloaded 38GiB to GPU in 1.70868s (22.2394GiB/s) | 20260507 16:01:47.182 | INFO | Analyzing chunk frames = [774656,775168) | 20260507 16:01:47.182 | INFO | MemInfo: { | 20260507 16:01:47.182 | INFO | "gpu" : " 17 GB", | 20260507 16:01:47.183 | INFO | "huge" : "", | 20260507 16:01:47.183 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:47.183 | INFO | "malloc" : " 14 GB", | 20260507 16:01:47.183 | INFO | "pinned" : " 29 GB" | 20260507 16:01:47.183 | INFO | } | 20260507 16:01:47.183 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:01:47.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:47.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 16:01:47.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | PRE_HQ : 38.6%, 1.0, 20.0ms, 0 | 20260507 16:01:47.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:47.183 | INFO | Uploaded 36GiB to GPU in 1.59323s (22.5956GiB/s) | 20260507 16:01:47.183 | INFO | Downloaded 34GiB to GPU in 1.5397s (22.0822GiB/s) | 20260507 16:01:52.322 | INFO | Analyzing chunk frames = [775168,775680) | 20260507 16:01:52.322 | INFO | MemInfo: { | 20260507 16:01:52.322 | INFO | "gpu" : " 17 GB", | 20260507 16:01:52.322 | INFO | "huge" : "", | 20260507 16:01:52.322 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:52.322 | INFO | "malloc" : " 14 GB", | 20260507 16:01:52.322 | INFO | "pinned" : " 29 GB" | 20260507 16:01:52.322 | INFO | } | 20260507 16:01:52.322 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:01:52.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:52.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.322 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260507 16:01:52.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.322 | INFO | PRE_HQ : 39.5%, 1.0, 20.5ms, 0 | 20260507 16:01:52.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:52.323 | INFO | Uploaded 36GiB to GPU in 1.59491s (22.5719GiB/s) | 20260507 16:01:52.323 | INFO | Downloaded 34GiB to GPU in 1.545s (22.0065GiB/s) | 20260507 16:01:57.461 | INFO | Analyzing chunk frames = [775680,776192) | 20260507 16:01:57.461 | INFO | MemInfo: { | 20260507 16:01:57.461 | INFO | "gpu" : " 17 GB", | 20260507 16:01:57.461 | INFO | "huge" : "", | 20260507 16:01:57.461 | INFO | "huge_shared" : " 29 GB", | 20260507 16:01:57.461 | INFO | "malloc" : " 14 GB", | 20260507 16:01:57.462 | INFO | "pinned" : " 29 GB" | 20260507 16:01:57.462 | INFO | } | 20260507 16:01:57.462 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:01:57.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:01:57.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 16:01:57.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | PRE_HQ : 39.4%, 1.0, 20.5ms, 0 | 20260507 16:01:57.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:01:57.462 | INFO | Uploaded 36GiB to GPU in 1.59601s (22.5562GiB/s) | 20260507 16:01:57.463 | INFO | Downloaded 34GiB to GPU in 1.54422s (22.0177GiB/s) | 20260507 16:02:02.600 | INFO | Analyzing chunk frames = [776192,776704) | 20260507 16:02:02.600 | INFO | MemInfo: { | 20260507 16:02:02.600 | INFO | "gpu" : " 17 GB", | 20260507 16:02:02.600 | INFO | "huge" : "", | 20260507 16:02:02.600 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:02.600 | INFO | "malloc" : " 14 GB", | 20260507 16:02:02.600 | INFO | "pinned" : " 29 GB" | 20260507 16:02:02.600 | INFO | } | 20260507 16:02:02.600 | INFO | Perf report for the last 5.13883 seconds: | 20260507 16:02:02.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:02.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:02:02.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | PRE_HQ : 39.0%, 1.0, 20.2ms, 0 | 20260507 16:02:02.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:02.601 | INFO | Uploaded 36GiB to GPU in 1.59354s (22.5912GiB/s) | 20260507 16:02:02.601 | INFO | Downloaded 34GiB to GPU in 1.54243s (22.0432GiB/s) | 20260507 16:02:07.740 | INFO | Analyzing chunk frames = [776704,777216) | 20260507 16:02:07.740 | INFO | MemInfo: { | 20260507 16:02:07.740 | INFO | "gpu" : " 17 GB", | 20260507 16:02:07.740 | INFO | "huge" : "", | 20260507 16:02:07.740 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:07.740 | INFO | "malloc" : " 14 GB", | 20260507 16:02:07.740 | INFO | "pinned" : " 29 GB" | 20260507 16:02:07.740 | INFO | } | 20260507 16:02:07.740 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:02:07.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:07.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:02:07.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:02:07.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:07.740 | INFO | Uploaded 36GiB to GPU in 1.58745s (22.6779GiB/s) | 20260507 16:02:07.740 | INFO | Downloaded 34GiB to GPU in 1.53797s (22.1071GiB/s) | 20260507 16:02:12.879 | INFO | Analyzing chunk frames = [777216,777728) | 20260507 16:02:12.879 | INFO | MemInfo: { | 20260507 16:02:12.879 | INFO | "gpu" : " 17 GB", | 20260507 16:02:12.879 | INFO | "huge" : "", | 20260507 16:02:12.879 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:12.879 | INFO | "malloc" : " 14 GB", | 20260507 16:02:12.879 | INFO | "pinned" : " 29 GB" | 20260507 16:02:12.879 | INFO | } | 20260507 16:02:12.879 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:02:12.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:12.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.879 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:02:12.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.879 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:02:12.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:12.880 | INFO | Uploaded 36GiB to GPU in 1.58789s (22.6717GiB/s) | 20260507 16:02:12.880 | INFO | Downloaded 34GiB to GPU in 1.5377s (22.1109GiB/s) | 20260507 16:02:18.018 | INFO | Analyzing chunk frames = [777728,778240) | 20260507 16:02:18.018 | INFO | MemInfo: { | 20260507 16:02:18.018 | INFO | "gpu" : " 17 GB", | 20260507 16:02:18.018 | INFO | "huge" : "", | 20260507 16:02:18.018 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:18.019 | INFO | "malloc" : " 14 GB", | 20260507 16:02:18.019 | INFO | "pinned" : " 29 GB" | 20260507 16:02:18.019 | INFO | } | 20260507 16:02:18.019 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:02:18.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:18.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:02:18.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 16:02:18.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:18.019 | INFO | Uploaded 36GiB to GPU in 1.58926s (22.6521GiB/s) | 20260507 16:02:18.019 | INFO | Downloaded 34GiB to GPU in 1.53907s (22.0913GiB/s) | 20260507 16:02:18.019 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:02:18.019 | INFO | Total mem usage: 16.4 GiB | 20260507 16:02:18.019 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:02:18.019 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.44%) | 20260507 16:02:18.019 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:02:18.019 | INFO | High Water Breakdown: | 20260507 16:02:18.019 | INFO | 59.5% : 9.01 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:02:18.019 | INFO | 40.3% : 6.1 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:02:18.019 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:02:18.019 | INFO | ------------------------------------------ | 20260507 16:02:18.019 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:02:18.019 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:02:18.019 | INFO | Total mem usage: 26.5 GiB | 20260507 16:02:18.019 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:02:18.019 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 16:02:18.019 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 16:02:18.019 | INFO | High Water Breakdown: | 20260507 16:02:18.019 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:02:18.019 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:02:18.019 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:02:18.019 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:02:18.019 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:02:18.019 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:02:18.020 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:02:18.020 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:02:18.020 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:02:18.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:02:18.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:02:18.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:02:18.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:02:18.020 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:02:18.020 | INFO | ------------------------------------------ | 20260507 16:02:18.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:02:18.020 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:02:18.020 | INFO | Total mem usage: 27.8 GiB | 20260507 16:02:18.020 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:02:18.020 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:02:18.020 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:02:18.020 | INFO | High Water Breakdown: | 20260507 16:02:18.020 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:02:18.020 | INFO | ------------------------------------------ | 20260507 16:02:18.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:02:18.020 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:02:18.020 | INFO | Total mem usage: 15.7 GiB | 20260507 16:02:18.020 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:02:18.020 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:02:18.020 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:02:18.020 | INFO | High Water Breakdown: | 20260507 16:02:18.020 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:02:18.020 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:02:18.020 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:02:18.020 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:02:18.020 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:02:18.020 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:02:18.020 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:02:18.020 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:02:18.020 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:02:18.020 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:02:18.020 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:02:18.020 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:02:18.020 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:02:18.020 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:02:18.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:02:18.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:02:18.020 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:02:18.020 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:02:18.020 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:02:18.020 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:02:18.020 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:02:18.020 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:02:18.020 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:02:18.020 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:02:18.020 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:02:18.020 | INFO | ------------------------------------------ | 20260507 16:02:18.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:02:18.020 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:02:18.020 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:02:18.020 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:02:18.020 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 0.30%, AggregateStats -- 9.04e-02 / -nan, 6.88e-03 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 11.08%, Baseline -- 3.30e+00 / -nan, 2.58e-02 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.72e+01 / -nan, 5.90e-02 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.13e+00 / -nan, 1.19e-01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 3.41%, Metrics -- 1.02e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.77e-01 / -nan, 5.62e-02 / 0.00e+00 | 20260507 16:02:18.020 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:02:18.020 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:02:18.020 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:02:18.020 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 18.79%, Upload -- 1.67e+01 / 0.00e+00, 4.49e-01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 18.23%, Download -- 1.62e+01 / 0.00e+00, 7.35e-01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 33.50%, Compute -- 2.98e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 0.00%, LockDownload -- 5.78e-04 / 0.00e+00, 4.82e-04 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 28.88%, LockCompute -- 2.57e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260507 16:02:18.020 | INFO | PerfReport: 0.02%, Misc -- 1.86e-02 / 0.00e+00, 4.51e-02 / 0.00e+00 | 20260507 16:02:18.115 | INFO | Starting a round of Baz IO | 20260507 16:02:21.041 | INFO | Populating RTMetrics report | 20260507 16:02:21.041 | INFO | RTMetrics report populated | 20260507 16:02:21.041 | INFO | Opening tmp RTMetrics json file | 20260507 16:02:21.087 | INFO | Serializing RTMetrics to json | 20260507 16:02:21.092 | INFO | Serialization done, swapping tmp file now... | 20260507 16:02:21.096 | INFO | RTMetrics done | 20260507 16:02:21.356 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.24 seconds | 20260507 16:02:21.356 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.36% / 0.03% / 48.61% | 20260507 16:02:21.356 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.97183% / 3.97364e-05% / 1.58946e-05% | 20260507 16:02:21.356 | INFO | Baz IO overall: | 20260507 16:02:21.356 | INFO | BytesWritten : 327721692179 | 20260507 16:02:21.356 | INFO | NumEvents : 158342069876 | 20260507 16:02:23.158 | INFO | Analyzing chunk frames = [778240,778752) | 20260507 16:02:23.158 | INFO | MemInfo: { | 20260507 16:02:23.158 | INFO | "gpu" : " 17 GB", | 20260507 16:02:23.158 | INFO | "huge" : "", | 20260507 16:02:23.158 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:23.158 | INFO | "malloc" : " 14 GB", | 20260507 16:02:23.158 | INFO | "pinned" : " 29 GB" | 20260507 16:02:23.158 | INFO | } | 20260507 16:02:23.158 | INFO | Perf report for the last 5.13974 seconds: | 20260507 16:02:23.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:23.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:23.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:23.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:23.158 | INFO | ANALYSIS : 57.9%, 2.8, 85.5ms, 0 | 20260507 16:02:23.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:23.159 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260507 16:02:23.159 | INFO | PRE_HQ : 56.8%, 2.1, 62.9ms, 0 | 20260507 16:02:23.159 | INFO | BAZWRITER : 62.6%, 4.6, 148.7ms, 0 | 20260507 16:02:23.159 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 16:02:23.159 | INFO | Uploaded 36GiB to GPU in 1.63568s (22.0092GiB/s) | 20260507 16:02:23.159 | INFO | Downloaded 38GiB to GPU in 1.71523s (22.1544GiB/s) | 20260507 16:02:28.297 | INFO | Analyzing chunk frames = [778752,779264) | 20260507 16:02:28.297 | INFO | MemInfo: { | 20260507 16:02:28.297 | INFO | "gpu" : " 17 GB", | 20260507 16:02:28.297 | INFO | "huge" : "", | 20260507 16:02:28.297 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:28.298 | INFO | "malloc" : " 14 GB", | 20260507 16:02:28.298 | INFO | "pinned" : " 29 GB" | 20260507 16:02:28.298 | INFO | } | 20260507 16:02:28.298 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:02:28.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:28.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.298 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 16:02:28.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.298 | INFO | PRE_HQ : 37.8%, 1.0, 19.6ms, 0 | 20260507 16:02:28.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:28.299 | INFO | Uploaded 36GiB to GPU in 1.59284s (22.6011GiB/s) | 20260507 16:02:28.299 | INFO | Downloaded 34GiB to GPU in 1.54017s (22.0755GiB/s) | 20260507 16:02:33.437 | INFO | Analyzing chunk frames = [779264,779776) | 20260507 16:02:33.437 | INFO | MemInfo: { | 20260507 16:02:33.437 | INFO | "gpu" : " 17 GB", | 20260507 16:02:33.437 | INFO | "huge" : "", | 20260507 16:02:33.437 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:33.437 | INFO | "malloc" : " 14 GB", | 20260507 16:02:33.437 | INFO | "pinned" : " 29 GB" | 20260507 16:02:33.437 | INFO | } | 20260507 16:02:33.437 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:02:33.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:33.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.437 | INFO | ANALYSIS : 56.4%, 3.0, 87.7ms, 0 | 20260507 16:02:33.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.437 | INFO | PRE_HQ : 39.4%, 1.0, 20.4ms, 0 | 20260507 16:02:33.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:33.438 | INFO | Uploaded 36GiB to GPU in 1.59596s (22.5569GiB/s) | 20260507 16:02:33.438 | INFO | Downloaded 34GiB to GPU in 1.54575s (21.9957GiB/s) | 20260507 16:02:38.576 | INFO | Analyzing chunk frames = [779776,780288) | 20260507 16:02:38.576 | INFO | MemInfo: { | 20260507 16:02:38.576 | INFO | "gpu" : " 17 GB", | 20260507 16:02:38.576 | INFO | "huge" : "", | 20260507 16:02:38.576 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:38.576 | INFO | "malloc" : " 14 GB", | 20260507 16:02:38.576 | INFO | "pinned" : " 29 GB" | 20260507 16:02:38.576 | INFO | } | 20260507 16:02:38.577 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:02:38.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:38.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:02:38.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:02:38.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:38.577 | INFO | Uploaded 36GiB to GPU in 1.59605s (22.5557GiB/s) | 20260507 16:02:38.577 | INFO | Downloaded 34GiB to GPU in 1.54506s (22.0056GiB/s) | 20260507 16:02:43.715 | INFO | Analyzing chunk frames = [780288,780800) | 20260507 16:02:43.715 | INFO | MemInfo: { | 20260507 16:02:43.715 | INFO | "gpu" : " 17 GB", | 20260507 16:02:43.715 | INFO | "huge" : "", | 20260507 16:02:43.715 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:43.715 | INFO | "malloc" : " 14 GB", | 20260507 16:02:43.715 | INFO | "pinned" : " 29 GB" | 20260507 16:02:43.715 | INFO | } | 20260507 16:02:43.716 | INFO | Perf report for the last 5.13901 seconds: | 20260507 16:02:43.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:43.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:02:43.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | PRE_HQ : 38.6%, 1.0, 20.0ms, 0 | 20260507 16:02:43.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:43.716 | INFO | Uploaded 36GiB to GPU in 1.59325s (22.5954GiB/s) | 20260507 16:02:43.716 | INFO | Downloaded 34GiB to GPU in 1.54141s (22.0577GiB/s) | 20260507 16:02:48.854 | INFO | Analyzing chunk frames = [780800,781312) | 20260507 16:02:48.854 | INFO | MemInfo: { | 20260507 16:02:48.854 | INFO | "gpu" : " 17 GB", | 20260507 16:02:48.854 | INFO | "huge" : "", | 20260507 16:02:48.854 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:48.854 | INFO | "malloc" : " 14 GB", | 20260507 16:02:48.855 | INFO | "pinned" : " 29 GB" | 20260507 16:02:48.855 | INFO | } | 20260507 16:02:48.855 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:02:48.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:48.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:02:48.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:02:48.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:48.855 | INFO | Uploaded 36GiB to GPU in 1.58958s (22.6475GiB/s) | 20260507 16:02:48.855 | INFO | Downloaded 34GiB to GPU in 1.53887s (22.0942GiB/s) | 20260507 16:02:53.994 | INFO | Analyzing chunk frames = [781312,781824) | 20260507 16:02:53.994 | INFO | MemInfo: { | 20260507 16:02:53.994 | INFO | "gpu" : " 17 GB", | 20260507 16:02:53.994 | INFO | "huge" : "", | 20260507 16:02:53.994 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:53.994 | INFO | "malloc" : " 14 GB", | 20260507 16:02:53.994 | INFO | "pinned" : " 29 GB" | 20260507 16:02:53.994 | INFO | } | 20260507 16:02:53.994 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:02:53.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:53.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.994 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:02:53.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.994 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:02:53.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:53.995 | INFO | Uploaded 36GiB to GPU in 1.58917s (22.6533GiB/s) | 20260507 16:02:53.995 | INFO | Downloaded 34GiB to GPU in 1.53789s (22.1082GiB/s) | 20260507 16:02:59.133 | INFO | Analyzing chunk frames = [781824,782336) | 20260507 16:02:59.133 | INFO | MemInfo: { | 20260507 16:02:59.133 | INFO | "gpu" : " 17 GB", | 20260507 16:02:59.133 | INFO | "huge" : "", | 20260507 16:02:59.133 | INFO | "huge_shared" : " 29 GB", | 20260507 16:02:59.133 | INFO | "malloc" : " 14 GB", | 20260507 16:02:59.134 | INFO | "pinned" : " 29 GB" | 20260507 16:02:59.134 | INFO | } | 20260507 16:02:59.134 | INFO | Perf report for the last 5.1396 seconds: | 20260507 16:02:59.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:02:59.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:02:59.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:02:59.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:02:59.134 | INFO | Uploaded 36GiB to GPU in 1.58951s (22.6485GiB/s) | 20260507 16:02:59.134 | INFO | Downloaded 34GiB to GPU in 1.53852s (22.0992GiB/s) | 20260507 16:02:59.230 | INFO | Starting a round of Baz IO | 20260507 16:03:02.164 | INFO | Populating RTMetrics report | 20260507 16:03:02.165 | INFO | RTMetrics report populated | 20260507 16:03:02.165 | INFO | Opening tmp RTMetrics json file | 20260507 16:03:02.182 | INFO | Serializing RTMetrics to json | 20260507 16:03:02.189 | INFO | Serialization done, swapping tmp file now... | 20260507 16:03:02.190 | INFO | RTMetrics done | 20260507 16:03:02.680 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.449 seconds | 20260507 16:03:02.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.33% / 0.03% / 48.64% | 20260507 16:03:02.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.96212% / 6.75519e-05% / 2.38419e-05% | 20260507 16:03:02.680 | INFO | Baz IO overall: | 20260507 16:03:02.680 | INFO | BytesWritten : 329626309118 | 20260507 16:03:02.680 | INFO | NumEvents : 159268126000 | 20260507 16:03:04.273 | INFO | Analyzing chunk frames = [782336,782848) | 20260507 16:03:04.273 | INFO | MemInfo: { | 20260507 16:03:04.273 | INFO | "gpu" : " 17 GB", | 20260507 16:03:04.273 | INFO | "huge" : "", | 20260507 16:03:04.273 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:04.273 | INFO | "malloc" : " 14 GB", | 20260507 16:03:04.273 | INFO | "pinned" : " 29 GB" | 20260507 16:03:04.273 | INFO | } | 20260507 16:03:04.273 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:03:04.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:04.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:04.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:04.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:04.273 | INFO | ANALYSIS : 58.1%, 2.8, 84.0ms, 0 | 20260507 16:03:04.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:04.273 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 16:03:04.273 | INFO | PRE_HQ : 57.1%, 2.2, 64.2ms, 0 | 20260507 16:03:04.273 | INFO | BAZWRITER : 66.7%, 4.7, 160.9ms, 0 | 20260507 16:03:04.273 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 16:03:04.274 | INFO | Uploaded 36GiB to GPU in 1.63669s (21.9956GiB/s) | 20260507 16:03:04.274 | INFO | Downloaded 38GiB to GPU in 1.71758s (22.1242GiB/s) | 20260507 16:03:09.412 | INFO | Analyzing chunk frames = [782848,783360) | 20260507 16:03:09.412 | INFO | MemInfo: { | 20260507 16:03:09.412 | INFO | "gpu" : " 17 GB", | 20260507 16:03:09.412 | INFO | "huge" : "", | 20260507 16:03:09.412 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:09.412 | INFO | "malloc" : " 14 GB", | 20260507 16:03:09.412 | INFO | "pinned" : " 29 GB" | 20260507 16:03:09.412 | INFO | } | 20260507 16:03:09.413 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:03:09.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:09.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 16:03:09.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | PRE_HQ : 38.4%, 1.0, 20.0ms, 0 | 20260507 16:03:09.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:09.413 | INFO | Uploaded 36GiB to GPU in 1.59204s (22.6125GiB/s) | 20260507 16:03:09.413 | INFO | Downloaded 34GiB to GPU in 1.53947s (22.0855GiB/s) | 20260507 16:03:14.551 | INFO | Analyzing chunk frames = [783360,783872) | 20260507 16:03:14.551 | INFO | MemInfo: { | 20260507 16:03:14.551 | INFO | "gpu" : " 17 GB", | 20260507 16:03:14.551 | INFO | "huge" : "", | 20260507 16:03:14.552 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:14.552 | INFO | "malloc" : " 14 GB", | 20260507 16:03:14.552 | INFO | "pinned" : " 29 GB" | 20260507 16:03:14.552 | INFO | } | 20260507 16:03:14.552 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:03:14.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:14.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.552 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:03:14.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.552 | INFO | PRE_HQ : 38.8%, 1.0, 20.2ms, 0 | 20260507 16:03:14.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:14.553 | INFO | Uploaded 36GiB to GPU in 1.59592s (22.5575GiB/s) | 20260507 16:03:14.553 | INFO | Downloaded 34GiB to GPU in 1.54507s (22.0055GiB/s) | 20260507 16:03:19.691 | INFO | Analyzing chunk frames = [783872,784384) | 20260507 16:03:19.691 | INFO | MemInfo: { | 20260507 16:03:19.691 | INFO | "gpu" : " 17 GB", | 20260507 16:03:19.691 | INFO | "huge" : "", | 20260507 16:03:19.691 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:19.691 | INFO | "malloc" : " 14 GB", | 20260507 16:03:19.691 | INFO | "pinned" : " 29 GB" | 20260507 16:03:19.691 | INFO | } | 20260507 16:03:19.691 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:03:19.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:19.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.691 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 16:03:19.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.692 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:03:19.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:19.692 | INFO | Uploaded 36GiB to GPU in 1.59564s (22.5614GiB/s) | 20260507 16:03:19.692 | INFO | Downloaded 34GiB to GPU in 1.54477s (22.0098GiB/s) | 20260507 16:03:24.830 | INFO | Analyzing chunk frames = [784384,784896) | 20260507 16:03:24.830 | INFO | MemInfo: { | 20260507 16:03:24.830 | INFO | "gpu" : " 17 GB", | 20260507 16:03:24.830 | INFO | "huge" : "", | 20260507 16:03:24.830 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:24.830 | INFO | "malloc" : " 14 GB", | 20260507 16:03:24.830 | INFO | "pinned" : " 29 GB" | 20260507 16:03:24.830 | INFO | } | 20260507 16:03:24.830 | INFO | Perf report for the last 5.13887 seconds: | 20260507 16:03:24.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:24.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.830 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 16:03:24.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.830 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 16:03:24.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:24.831 | INFO | Uploaded 36GiB to GPU in 1.59496s (22.5711GiB/s) | 20260507 16:03:24.831 | INFO | Downloaded 34GiB to GPU in 1.54213s (22.0474GiB/s) | 20260507 16:03:29.969 | INFO | Analyzing chunk frames = [784896,785408) | 20260507 16:03:29.969 | INFO | MemInfo: { | 20260507 16:03:29.969 | INFO | "gpu" : " 17 GB", | 20260507 16:03:29.969 | INFO | "huge" : "", | 20260507 16:03:29.969 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:29.970 | INFO | "malloc" : " 14 GB", | 20260507 16:03:29.970 | INFO | "pinned" : " 29 GB" | 20260507 16:03:29.970 | INFO | } | 20260507 16:03:29.970 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:03:29.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:29.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:03:29.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:03:29.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:29.970 | INFO | Uploaded 36GiB to GPU in 1.58859s (22.6616GiB/s) | 20260507 16:03:29.970 | INFO | Downloaded 34GiB to GPU in 1.53725s (22.1174GiB/s) | 20260507 16:03:35.109 | INFO | Analyzing chunk frames = [785408,785920) | 20260507 16:03:35.109 | INFO | MemInfo: { | 20260507 16:03:35.109 | INFO | "gpu" : " 17 GB", | 20260507 16:03:35.109 | INFO | "huge" : "", | 20260507 16:03:35.109 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:35.109 | INFO | "malloc" : " 14 GB", | 20260507 16:03:35.109 | INFO | "pinned" : " 29 GB" | 20260507 16:03:35.109 | INFO | } | 20260507 16:03:35.109 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:03:35.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:35.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:03:35.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:03:35.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:35.109 | INFO | Uploaded 36GiB to GPU in 1.58853s (22.6625GiB/s) | 20260507 16:03:35.109 | INFO | Downloaded 34GiB to GPU in 1.5384s (22.1009GiB/s) | 20260507 16:03:40.248 | INFO | Analyzing chunk frames = [785920,786432) | 20260507 16:03:40.248 | INFO | MemInfo: { | 20260507 16:03:40.248 | INFO | "gpu" : " 17 GB", | 20260507 16:03:40.248 | INFO | "huge" : "", | 20260507 16:03:40.248 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:40.248 | INFO | "malloc" : " 14 GB", | 20260507 16:03:40.248 | INFO | "pinned" : " 29 GB" | 20260507 16:03:40.248 | INFO | } | 20260507 16:03:40.248 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:03:40.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:40.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.248 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 16:03:40.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.249 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:03:40.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:40.249 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:03:40.249 | INFO | Total mem usage: 16.4 GiB | 20260507 16:03:40.249 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:03:40.249 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -3.94%) | 20260507 16:03:40.249 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:03:40.249 | INFO | High Water Breakdown: | 20260507 16:03:40.249 | INFO | 60.2% : 9.51 GiB ( 95.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:03:40.249 | INFO | 39.6% : 6.25 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:03:40.249 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:03:40.249 | INFO | ------------------------------------------ | 20260507 16:03:40.249 | INFO | Uploaded 36GiB to GPU in 1.58946s (22.6492GiB/s) | 20260507 16:03:40.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:03:40.249 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:03:40.249 | INFO | Downloaded 34GiB to GPU in 1.5385s (22.0994GiB/s) | 20260507 16:03:40.249 | INFO | Total mem usage: 26.5 GiB | 20260507 16:03:40.249 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:03:40.249 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 16:03:40.249 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 16:03:40.249 | INFO | High Water Breakdown: | 20260507 16:03:40.249 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:03:40.249 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:03:40.249 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:03:40.249 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:03:40.249 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:03:40.249 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:03:40.249 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:03:40.249 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:03:40.249 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:03:40.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:03:40.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:03:40.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:03:40.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:03:40.249 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:03:40.249 | INFO | ------------------------------------------ | 20260507 16:03:40.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:03:40.249 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:03:40.249 | INFO | Total mem usage: 27.8 GiB | 20260507 16:03:40.249 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:03:40.249 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:03:40.249 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:03:40.249 | INFO | High Water Breakdown: | 20260507 16:03:40.249 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:03:40.249 | INFO | ------------------------------------------ | 20260507 16:03:40.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:03:40.250 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:03:40.250 | INFO | Total mem usage: 15.7 GiB | 20260507 16:03:40.250 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:03:40.250 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:03:40.250 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:03:40.250 | INFO | High Water Breakdown: | 20260507 16:03:40.250 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:03:40.250 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:03:40.250 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:03:40.250 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:03:40.250 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:03:40.250 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:03:40.250 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:03:40.250 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:03:40.250 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:03:40.250 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:03:40.250 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:03:40.250 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:03:40.250 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:03:40.250 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:03:40.250 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:03:40.250 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:03:40.250 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:03:40.250 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:03:40.250 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:03:40.250 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:03:40.250 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:03:40.250 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:03:40.250 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:03:40.250 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:03:40.250 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:03:40.250 | INFO | ------------------------------------------ | 20260507 16:03:40.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:03:40.250 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:03:40.250 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:03:40.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:03:40.250 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 0.30%, AggregateStats -- 9.04e-02 / -nan, 6.91e-03 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 11.09%, Baseline -- 3.31e+00 / -nan, 2.39e-02 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.43e-02 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.72e+01 / -nan, 4.22e-02 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.13e+00 / -nan, 1.19e-01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 3.40%, Metrics -- 1.01e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.76e-01 / -nan, 5.56e-02 / 0.00e+00 | 20260507 16:03:40.250 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:03:40.250 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:03:40.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:03:40.250 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 18.68%, Upload -- 1.66e+01 / 0.00e+00, 4.39e-01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 18.27%, Download -- 1.63e+01 / 0.00e+00, 7.07e-01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 33.48%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 0.59%, LockUpload -- 5.29e-01 / 0.00e+00, 3.67e+00 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 0.00%, LockDownload -- 5.85e-04 / 0.00e+00, 4.80e-04 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 28.96%, LockCompute -- 2.58e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 16:03:40.250 | INFO | PerfReport: 0.02%, Misc -- 1.64e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 16:03:40.346 | INFO | Starting a round of Baz IO | 20260507 16:03:43.282 | INFO | Populating RTMetrics report | 20260507 16:03:43.283 | INFO | RTMetrics report populated | 20260507 16:03:43.283 | INFO | Opening tmp RTMetrics json file | 20260507 16:03:43.308 | INFO | Serializing RTMetrics to json | 20260507 16:03:43.311 | INFO | Serialization done, swapping tmp file now... | 20260507 16:03:43.313 | INFO | RTMetrics done | 20260507 16:03:43.466 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.119 seconds | 20260507 16:03:43.466 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.29% / 0.03% / 48.67% | 20260507 16:03:43.466 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.95554% / 3.57628e-05% / 1.58946e-05% | 20260507 16:03:43.467 | INFO | Baz IO overall: | 20260507 16:03:43.467 | INFO | BytesWritten : 331531746725 | 20260507 16:03:43.467 | INFO | NumEvents : 160194754642 | 20260507 16:03:45.387 | INFO | Analyzing chunk frames = [786432,786944) | 20260507 16:03:45.388 | INFO | MemInfo: { | 20260507 16:03:45.388 | INFO | "gpu" : " 17 GB", | 20260507 16:03:45.388 | INFO | "huge" : "", | 20260507 16:03:45.388 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:45.388 | INFO | "malloc" : " 14 GB", | 20260507 16:03:45.388 | INFO | "pinned" : " 29 GB" | 20260507 16:03:45.388 | INFO | } | 20260507 16:03:45.388 | INFO | Perf report for the last 5.13946 seconds: | 20260507 16:03:45.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:45.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:45.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:45.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:45.388 | INFO | ANALYSIS : 58.0%, 2.8, 85.0ms, 0 | 20260507 16:03:45.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:45.388 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 16:03:45.388 | INFO | PRE_HQ : 57.3%, 2.2, 64.7ms, 0 | 20260507 16:03:45.388 | INFO | BAZWRITER : 60.2%, 4.6, 143.1ms, 0 | 20260507 16:03:45.388 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:03:45.388 | INFO | Uploaded 36GiB to GPU in 1.64156s (21.9303GiB/s) | 20260507 16:03:45.388 | INFO | Downloaded 38GiB to GPU in 1.71683s (22.1338GiB/s) | 20260507 16:03:50.527 | INFO | Analyzing chunk frames = [786944,787456) | 20260507 16:03:50.527 | INFO | MemInfo: { | 20260507 16:03:50.527 | INFO | "gpu" : " 17 GB", | 20260507 16:03:50.527 | INFO | "huge" : "", | 20260507 16:03:50.527 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:50.527 | INFO | "malloc" : " 14 GB", | 20260507 16:03:50.527 | INFO | "pinned" : " 29 GB" | 20260507 16:03:50.527 | INFO | } | 20260507 16:03:50.527 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:03:50.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:50.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.527 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 16:03:50.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.528 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 16:03:50.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:50.528 | INFO | Uploaded 36GiB to GPU in 1.61731s (22.2592GiB/s) | 20260507 16:03:50.528 | INFO | Downloaded 34GiB to GPU in 1.53053s (22.2145GiB/s) | 20260507 16:03:55.666 | INFO | Analyzing chunk frames = [787456,787968) | 20260507 16:03:55.666 | INFO | MemInfo: { | 20260507 16:03:55.666 | INFO | "gpu" : " 17 GB", | 20260507 16:03:55.667 | INFO | "huge" : "", | 20260507 16:03:55.667 | INFO | "huge_shared" : " 29 GB", | 20260507 16:03:55.667 | INFO | "malloc" : " 14 GB", | 20260507 16:03:55.667 | INFO | "pinned" : " 29 GB" | 20260507 16:03:55.667 | INFO | } | 20260507 16:03:55.667 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:03:55.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:03:55.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:03:55.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | PRE_HQ : 38.4%, 1.0, 19.9ms, 0 | 20260507 16:03:55.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:03:55.667 | INFO | Uploaded 36GiB to GPU in 1.61643s (22.2712GiB/s) | 20260507 16:03:55.667 | INFO | Downloaded 34GiB to GPU in 1.53628s (22.1313GiB/s) | 20260507 16:04:00.805 | INFO | Analyzing chunk frames = [787968,788480) | 20260507 16:04:00.805 | INFO | MemInfo: { | 20260507 16:04:00.805 | INFO | "gpu" : " 17 GB", | 20260507 16:04:00.806 | INFO | "huge" : "", | 20260507 16:04:00.806 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:00.806 | INFO | "malloc" : " 14 GB", | 20260507 16:04:00.806 | INFO | "pinned" : " 29 GB" | 20260507 16:04:00.806 | INFO | } | 20260507 16:04:00.806 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:04:00.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:00.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | ANALYSIS : 56.1%, 3.0, 87.0ms, 0 | 20260507 16:04:00.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 16:04:00.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:00.806 | INFO | Uploaded 36GiB to GPU in 1.61571s (22.2812GiB/s) | 20260507 16:04:00.806 | INFO | Downloaded 34GiB to GPU in 1.54151s (22.0564GiB/s) | 20260507 16:04:05.945 | INFO | Analyzing chunk frames = [788480,788992) | 20260507 16:04:05.945 | INFO | MemInfo: { | 20260507 16:04:05.945 | INFO | "gpu" : " 17 GB", | 20260507 16:04:05.945 | INFO | "huge" : "", | 20260507 16:04:05.945 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:05.945 | INFO | "malloc" : " 14 GB", | 20260507 16:04:05.945 | INFO | "pinned" : " 29 GB" | 20260507 16:04:05.945 | INFO | } | 20260507 16:04:05.945 | INFO | Perf report for the last 5.139 seconds: | 20260507 16:04:05.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:05.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:04:05.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:04:05.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:05.945 | INFO | Uploaded 36GiB to GPU in 1.60838s (22.3828GiB/s) | 20260507 16:04:05.945 | INFO | Downloaded 34GiB to GPU in 1.52858s (22.2429GiB/s) | 20260507 16:04:11.084 | INFO | Analyzing chunk frames = [788992,789504) | 20260507 16:04:11.084 | INFO | MemInfo: { | 20260507 16:04:11.084 | INFO | "gpu" : " 17 GB", | 20260507 16:04:11.084 | INFO | "huge" : "", | 20260507 16:04:11.084 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:11.084 | INFO | "malloc" : " 14 GB", | 20260507 16:04:11.084 | INFO | "pinned" : " 29 GB" | 20260507 16:04:11.084 | INFO | } | 20260507 16:04:11.084 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:04:11.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:11.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.084 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 16:04:11.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.085 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:04:11.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:11.085 | INFO | Uploaded 36GiB to GPU in 1.60785s (22.3901GiB/s) | 20260507 16:04:11.085 | INFO | Downloaded 34GiB to GPU in 1.52788s (22.2531GiB/s) | 20260507 16:04:16.223 | INFO | Analyzing chunk frames = [789504,790016) | 20260507 16:04:16.223 | INFO | MemInfo: { | 20260507 16:04:16.223 | INFO | "gpu" : " 17 GB", | 20260507 16:04:16.223 | INFO | "huge" : "", | 20260507 16:04:16.223 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:16.224 | INFO | "malloc" : " 14 GB", | 20260507 16:04:16.224 | INFO | "pinned" : " 29 GB" | 20260507 16:04:16.224 | INFO | } | 20260507 16:04:16.224 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:04:16.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:16.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:04:16.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:04:16.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:16.224 | INFO | Uploaded 36GiB to GPU in 1.60932s (22.3697GiB/s) | 20260507 16:04:16.224 | INFO | Downloaded 34GiB to GPU in 1.52969s (22.2267GiB/s) | 20260507 16:04:21.363 | INFO | Analyzing chunk frames = [790016,790528) | 20260507 16:04:21.363 | INFO | MemInfo: { | 20260507 16:04:21.363 | INFO | "gpu" : " 17 GB", | 20260507 16:04:21.363 | INFO | "huge" : "", | 20260507 16:04:21.363 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:21.363 | INFO | "malloc" : " 14 GB", | 20260507 16:04:21.363 | INFO | "pinned" : " 29 GB" | 20260507 16:04:21.363 | INFO | } | 20260507 16:04:21.363 | INFO | Perf report for the last 5.13946 seconds: | 20260507 16:04:21.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:21.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:04:21.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:04:21.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:21.363 | INFO | Uploaded 36GiB to GPU in 1.60877s (22.3774GiB/s) | 20260507 16:04:21.363 | INFO | Downloaded 34GiB to GPU in 1.52885s (22.239GiB/s) | 20260507 16:04:21.460 | INFO | Starting a round of Baz IO | 20260507 16:04:24.378 | INFO | Populating RTMetrics report | 20260507 16:04:24.379 | INFO | RTMetrics report populated | 20260507 16:04:24.379 | INFO | Opening tmp RTMetrics json file | 20260507 16:04:24.405 | INFO | Serializing RTMetrics to json | 20260507 16:04:24.461 | INFO | Serialization done, swapping tmp file now... | 20260507 16:04:24.462 | INFO | RTMetrics done | 20260507 16:04:24.830 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.37 seconds | 20260507 16:04:24.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.26% / 0.03% / 48.71% | 20260507 16:04:24.831 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.96118% / 5.5631e-05% / 1.98682e-05% | 20260507 16:04:24.831 | INFO | Baz IO overall: | 20260507 16:04:24.831 | INFO | BytesWritten : 333441020581 | 20260507 16:04:24.831 | INFO | NumEvents : 161124164620 | 20260507 16:04:26.502 | INFO | Analyzing chunk frames = [790528,791040) | 20260507 16:04:26.502 | INFO | MemInfo: { | 20260507 16:04:26.502 | INFO | "gpu" : " 17 GB", | 20260507 16:04:26.502 | INFO | "huge" : "", | 20260507 16:04:26.502 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:26.502 | INFO | "malloc" : " 14 GB", | 20260507 16:04:26.502 | INFO | "pinned" : " 29 GB" | 20260507 16:04:26.503 | INFO | } | 20260507 16:04:26.503 | INFO | Perf report for the last 5.1395 seconds: | 20260507 16:04:26.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:26.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:26.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:26.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:26.503 | INFO | ANALYSIS : 57.4%, 2.8, 85.5ms, 0 | 20260507 16:04:26.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:26.503 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260507 16:04:26.503 | INFO | PRE_HQ : 56.7%, 2.2, 63.4ms, 0 | 20260507 16:04:26.503 | INFO | BAZWRITER : 65.1%, 4.5, 151.7ms, 0 | 20260507 16:04:26.503 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:04:26.503 | INFO | Uploaded 36GiB to GPU in 1.6505s (21.8116GiB/s) | 20260507 16:04:26.503 | INFO | Downloaded 38GiB to GPU in 1.71615s (22.1426GiB/s) | 20260507 16:04:31.642 | INFO | Analyzing chunk frames = [791040,791552) | 20260507 16:04:31.642 | INFO | MemInfo: { | 20260507 16:04:31.642 | INFO | "gpu" : " 17 GB", | 20260507 16:04:31.642 | INFO | "huge" : "", | 20260507 16:04:31.642 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:31.642 | INFO | "malloc" : " 14 GB", | 20260507 16:04:31.642 | INFO | "pinned" : " 29 GB" | 20260507 16:04:31.642 | INFO | } | 20260507 16:04:31.642 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:04:31.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:31.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.642 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 16:04:31.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.642 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 16:04:31.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:31.643 | INFO | Uploaded 36GiB to GPU in 1.61202s (22.3322GiB/s) | 20260507 16:04:31.643 | INFO | Downloaded 34GiB to GPU in 1.53026s (22.2184GiB/s) | 20260507 16:04:36.781 | INFO | Analyzing chunk frames = [791552,792064) | 20260507 16:04:36.781 | INFO | MemInfo: { | 20260507 16:04:36.781 | INFO | "gpu" : " 17 GB", | 20260507 16:04:36.781 | INFO | "huge" : "", | 20260507 16:04:36.781 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:36.781 | INFO | "malloc" : " 14 GB", | 20260507 16:04:36.781 | INFO | "pinned" : " 29 GB" | 20260507 16:04:36.781 | INFO | } | 20260507 16:04:36.782 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:04:36.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:36.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:04:36.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | PRE_HQ : 38.6%, 1.0, 20.1ms, 0 | 20260507 16:04:36.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:36.782 | INFO | Uploaded 36GiB to GPU in 1.61502s (22.2908GiB/s) | 20260507 16:04:36.782 | INFO | Downloaded 34GiB to GPU in 1.53564s (22.1406GiB/s) | 20260507 16:04:41.920 | INFO | Analyzing chunk frames = [792064,792576) | 20260507 16:04:41.920 | INFO | MemInfo: { | 20260507 16:04:41.920 | INFO | "gpu" : " 17 GB", | 20260507 16:04:41.920 | INFO | "huge" : "", | 20260507 16:04:41.920 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:41.920 | INFO | "malloc" : " 14 GB", | 20260507 16:04:41.920 | INFO | "pinned" : " 29 GB" | 20260507 16:04:41.920 | INFO | } | 20260507 16:04:41.921 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:04:41.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:41.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:04:41.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | PRE_HQ : 39.0%, 1.0, 20.2ms, 0 | 20260507 16:04:41.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:41.921 | INFO | Uploaded 36GiB to GPU in 1.61567s (22.2817GiB/s) | 20260507 16:04:41.921 | INFO | Downloaded 34GiB to GPU in 1.53569s (22.1398GiB/s) | 20260507 16:04:47.060 | INFO | Analyzing chunk frames = [792576,793088) | 20260507 16:04:47.060 | INFO | MemInfo: { | 20260507 16:04:47.060 | INFO | "gpu" : " 17 GB", | 20260507 16:04:47.060 | INFO | "huge" : "", | 20260507 16:04:47.060 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:47.060 | INFO | "malloc" : " 14 GB", | 20260507 16:04:47.060 | INFO | "pinned" : " 29 GB" | 20260507 16:04:47.060 | INFO | } | 20260507 16:04:47.060 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:04:47.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:47.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.060 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:04:47.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.060 | INFO | PRE_HQ : 38.9%, 1.0, 20.2ms, 0 | 20260507 16:04:47.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:47.061 | INFO | Uploaded 36GiB to GPU in 1.61374s (22.3084GiB/s) | 20260507 16:04:47.061 | INFO | Downloaded 34GiB to GPU in 1.53233s (22.1885GiB/s) | 20260507 16:04:52.199 | INFO | Analyzing chunk frames = [793088,793600) | 20260507 16:04:52.199 | INFO | MemInfo: { | 20260507 16:04:52.199 | INFO | "gpu" : " 17 GB", | 20260507 16:04:52.199 | INFO | "huge" : "", | 20260507 16:04:52.199 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:52.199 | INFO | "malloc" : " 14 GB", | 20260507 16:04:52.199 | INFO | "pinned" : " 29 GB" | 20260507 16:04:52.199 | INFO | } | 20260507 16:04:52.199 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:04:52.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:52.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 16:04:52.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:04:52.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:52.200 | INFO | Uploaded 36GiB to GPU in 1.61324s (22.3153GiB/s) | 20260507 16:04:52.200 | INFO | Downloaded 34GiB to GPU in 1.53221s (22.1902GiB/s) | 20260507 16:04:57.338 | INFO | Analyzing chunk frames = [793600,794112) | 20260507 16:04:57.338 | INFO | MemInfo: { | 20260507 16:04:57.338 | INFO | "gpu" : " 17 GB", | 20260507 16:04:57.339 | INFO | "huge" : "", | 20260507 16:04:57.339 | INFO | "huge_shared" : " 29 GB", | 20260507 16:04:57.339 | INFO | "malloc" : " 14 GB", | 20260507 16:04:57.339 | INFO | "pinned" : " 29 GB" | 20260507 16:04:57.339 | INFO | } | 20260507 16:04:57.339 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:04:57.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:04:57.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 16:04:57.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 16:04:57.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:04:57.339 | INFO | Uploaded 36GiB to GPU in 1.61369s (22.3091GiB/s) | 20260507 16:04:57.339 | INFO | Downloaded 34GiB to GPU in 1.53372s (22.1683GiB/s) | 20260507 16:05:02.478 | INFO | Analyzing chunk frames = [794112,794624) | 20260507 16:05:02.478 | INFO | MemInfo: { | 20260507 16:05:02.478 | INFO | "gpu" : " 17 GB", | 20260507 16:05:02.478 | INFO | "huge" : "", | 20260507 16:05:02.478 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:02.478 | INFO | "malloc" : " 14 GB", | 20260507 16:05:02.478 | INFO | "pinned" : " 29 GB" | 20260507 16:05:02.478 | INFO | } | 20260507 16:05:02.478 | INFO | Perf report for the last 5.13913 seconds: | 20260507 16:05:02.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:02.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 16:05:02.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 16:05:02.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:02.478 | INFO | Uploaded 36GiB to GPU in 1.61015s (22.3582GiB/s) | 20260507 16:05:02.478 | INFO | Downloaded 34GiB to GPU in 1.53092s (22.2088GiB/s) | 20260507 16:05:02.479 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:05:02.479 | INFO | Total mem usage: 16.4 GiB | 20260507 16:05:02.479 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:05:02.479 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.65%) | 20260507 16:05:02.479 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:05:02.479 | INFO | High Water Breakdown: | 20260507 16:05:02.479 | INFO | 59.7% : 9.09 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:05:02.479 | INFO | 40.2% : 6.13 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:05:02.479 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:05:02.479 | INFO | ------------------------------------------ | 20260507 16:05:02.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:05:02.479 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:05:02.479 | INFO | Total mem usage: 26.5 GiB | 20260507 16:05:02.479 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:05:02.479 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:05:02.479 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 16:05:02.479 | INFO | High Water Breakdown: | 20260507 16:05:02.479 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:05:02.479 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:05:02.479 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:05:02.479 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:05:02.479 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:05:02.479 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:05:02.479 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:05:02.479 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:05:02.479 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:05:02.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:05:02.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:05:02.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:05:02.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:05:02.479 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:05:02.479 | INFO | ------------------------------------------ | 20260507 16:05:02.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:05:02.479 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:05:02.479 | INFO | Total mem usage: 27.8 GiB | 20260507 16:05:02.479 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:05:02.479 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:05:02.479 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:05:02.479 | INFO | High Water Breakdown: | 20260507 16:05:02.479 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:05:02.479 | INFO | ------------------------------------------ | 20260507 16:05:02.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:05:02.479 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:05:02.479 | INFO | Total mem usage: 15.7 GiB | 20260507 16:05:02.479 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:05:02.479 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:05:02.479 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:05:02.479 | INFO | High Water Breakdown: | 20260507 16:05:02.479 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:05:02.479 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:05:02.479 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:05:02.479 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:05:02.479 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:05:02.479 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:05:02.479 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:05:02.479 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:05:02.479 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:05:02.479 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:05:02.479 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:05:02.479 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:05:02.479 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:05:02.479 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:05:02.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:05:02.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:05:02.479 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:05:02.479 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:05:02.480 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:05:02.480 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:05:02.480 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:05:02.480 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:05:02.480 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:05:02.480 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:05:02.480 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:05:02.480 | INFO | ------------------------------------------ | 20260507 16:05:02.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:05:02.480 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:05:02.480 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:05:02.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:05:02.480 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / -nan, 6.57e-03 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 11.10%, Baseline -- 3.31e+00 / -nan, 2.31e-01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / -nan, 5.47e-02 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 16.13%, DME -- 3.85e+01 / -nan, 2.95e+00 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.72e+01 / -nan, 6.90e-02 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.13e+00 / -nan, 1.19e-01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 3.40%, Metrics -- 1.02e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.77e-01 / -nan, 5.53e-02 / 0.00e+00 | 20260507 16:05:02.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:05:02.480 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:05:02.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:05:02.480 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 18.88%, Upload -- 1.68e+01 / 0.00e+00, 4.46e-01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 18.16%, Download -- 1.62e+01 / 0.00e+00, 7.50e-01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 33.46%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 0.59%, LockUpload -- 5.28e-01 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 0.00%, LockDownload -- 5.84e-04 / 0.00e+00, 4.42e-04 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 28.89%, LockCompute -- 2.58e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 16:05:02.480 | INFO | PerfReport: 0.02%, Misc -- 1.77e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260507 16:05:02.576 | INFO | Starting a round of Baz IO | 20260507 16:05:05.450 | INFO | Populating RTMetrics report | 20260507 16:05:05.450 | INFO | RTMetrics report populated | 20260507 16:05:05.450 | INFO | Opening tmp RTMetrics json file | 20260507 16:05:05.544 | INFO | Serializing RTMetrics to json | 20260507 16:05:05.565 | INFO | Serialization done, swapping tmp file now... | 20260507 16:05:05.570 | INFO | RTMetrics done | 20260507 16:05:05.650 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.073 seconds | 20260507 16:05:05.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.23% / 0.03% / 48.74% | 20260507 16:05:05.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.93959% / 5.96046e-05% / 3.57628e-05% | 20260507 16:05:05.650 | INFO | Baz IO overall: | 20260507 16:05:05.650 | INFO | BytesWritten : 335347780871 | 20260507 16:05:05.650 | INFO | NumEvents : 162052011992 | 20260507 16:05:07.617 | INFO | Analyzing chunk frames = [794624,795136) | 20260507 16:05:07.617 | INFO | MemInfo: { | 20260507 16:05:07.617 | INFO | "gpu" : " 17 GB", | 20260507 16:05:07.617 | INFO | "huge" : "", | 20260507 16:05:07.617 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:07.617 | INFO | "malloc" : " 14 GB", | 20260507 16:05:07.617 | INFO | "pinned" : " 29 GB" | 20260507 16:05:07.617 | INFO | } | 20260507 16:05:07.617 | INFO | Perf report for the last 5.13885 seconds: | 20260507 16:05:07.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:07.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:07.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:07.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:07.617 | INFO | ANALYSIS : 57.0%, 2.9, 85.3ms, 0 | 20260507 16:05:07.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:07.617 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 16:05:07.617 | INFO | PRE_HQ : 55.9%, 2.0, 58.9ms, 0 | 20260507 16:05:07.617 | INFO | BAZWRITER : 59.3%, 4.8, 145.1ms, 0 | 20260507 16:05:07.617 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:05:07.617 | INFO | Uploaded 36GiB to GPU in 1.67313s (21.5166GiB/s) | 20260507 16:05:07.617 | INFO | Downloaded 38GiB to GPU in 1.74404s (21.7885GiB/s) | 20260507 16:05:12.756 | INFO | Analyzing chunk frames = [795136,795648) | 20260507 16:05:12.756 | INFO | MemInfo: { | 20260507 16:05:12.756 | INFO | "gpu" : " 17 GB", | 20260507 16:05:12.756 | INFO | "huge" : "", | 20260507 16:05:12.756 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:12.756 | INFO | "malloc" : " 14 GB", | 20260507 16:05:12.756 | INFO | "pinned" : " 29 GB" | 20260507 16:05:12.756 | INFO | } | 20260507 16:05:12.756 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:05:12.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:12.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.756 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:05:12.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.756 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:05:12.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:12.757 | INFO | Uploaded 36GiB to GPU in 1.60556s (22.4221GiB/s) | 20260507 16:05:12.757 | INFO | Downloaded 34GiB to GPU in 1.52497s (22.2955GiB/s) | 20260507 16:05:17.896 | INFO | Analyzing chunk frames = [795648,796160) | 20260507 16:05:17.896 | INFO | MemInfo: { | 20260507 16:05:17.896 | INFO | "gpu" : " 17 GB", | 20260507 16:05:17.896 | INFO | "huge" : "", | 20260507 16:05:17.896 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:17.896 | INFO | "malloc" : " 14 GB", | 20260507 16:05:17.896 | INFO | "pinned" : " 29 GB" | 20260507 16:05:17.896 | INFO | } | 20260507 16:05:17.896 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:05:17.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:17.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 16:05:17.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:05:17.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:17.896 | INFO | Uploaded 36GiB to GPU in 1.61077s (22.3496GiB/s) | 20260507 16:05:17.896 | INFO | Downloaded 34GiB to GPU in 1.53112s (22.2059GiB/s) | 20260507 16:05:23.035 | INFO | Analyzing chunk frames = [796160,796672) | 20260507 16:05:23.035 | INFO | MemInfo: { | 20260507 16:05:23.035 | INFO | "gpu" : " 17 GB", | 20260507 16:05:23.035 | INFO | "huge" : "", | 20260507 16:05:23.035 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:23.035 | INFO | "malloc" : " 14 GB", | 20260507 16:05:23.035 | INFO | "pinned" : " 29 GB" | 20260507 16:05:23.035 | INFO | } | 20260507 16:05:23.035 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:05:23.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:23.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.035 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:05:23.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.035 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:05:23.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:23.036 | INFO | Uploaded 36GiB to GPU in 1.61011s (22.3587GiB/s) | 20260507 16:05:23.036 | INFO | Downloaded 34GiB to GPU in 1.52956s (22.2286GiB/s) | 20260507 16:05:28.174 | INFO | Analyzing chunk frames = [796672,797184) | 20260507 16:05:28.174 | INFO | MemInfo: { | 20260507 16:05:28.174 | INFO | "gpu" : " 17 GB", | 20260507 16:05:28.174 | INFO | "huge" : "", | 20260507 16:05:28.174 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:28.174 | INFO | "malloc" : " 14 GB", | 20260507 16:05:28.175 | INFO | "pinned" : " 29 GB" | 20260507 16:05:28.175 | INFO | } | 20260507 16:05:28.175 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:05:28.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:28.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:05:28.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:05:28.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:28.175 | INFO | Uploaded 36GiB to GPU in 1.60801s (22.388GiB/s) | 20260507 16:05:28.175 | INFO | Downloaded 34GiB to GPU in 1.52759s (22.2573GiB/s) | 20260507 16:05:33.314 | INFO | Analyzing chunk frames = [797184,797696) | 20260507 16:05:33.314 | INFO | MemInfo: { | 20260507 16:05:33.314 | INFO | "gpu" : " 17 GB", | 20260507 16:05:33.314 | INFO | "huge" : "", | 20260507 16:05:33.314 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:33.314 | INFO | "malloc" : " 14 GB", | 20260507 16:05:33.314 | INFO | "pinned" : " 29 GB" | 20260507 16:05:33.314 | INFO | } | 20260507 16:05:33.314 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:05:33.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:33.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.314 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 16:05:33.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.314 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 16:05:33.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:33.315 | INFO | Uploaded 36GiB to GPU in 1.61047s (22.3537GiB/s) | 20260507 16:05:33.315 | INFO | Downloaded 34GiB to GPU in 1.53019s (22.2195GiB/s) | 20260507 16:05:38.453 | INFO | Analyzing chunk frames = [797696,798208) | 20260507 16:05:38.453 | INFO | MemInfo: { | 20260507 16:05:38.453 | INFO | "gpu" : " 17 GB", | 20260507 16:05:38.453 | INFO | "huge" : "", | 20260507 16:05:38.453 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:38.453 | INFO | "malloc" : " 14 GB", | 20260507 16:05:38.453 | INFO | "pinned" : " 29 GB" | 20260507 16:05:38.454 | INFO | } | 20260507 16:05:38.454 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:05:38.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:38.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 16:05:38.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 16:05:38.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:38.454 | INFO | Uploaded 36GiB to GPU in 1.61441s (22.2991GiB/s) | 20260507 16:05:38.454 | INFO | Downloaded 34GiB to GPU in 1.53429s (22.1601GiB/s) | 20260507 16:05:43.592 | INFO | Analyzing chunk frames = [798208,798720) | 20260507 16:05:43.592 | INFO | MemInfo: { | 20260507 16:05:43.592 | INFO | "gpu" : " 17 GB", | 20260507 16:05:43.592 | INFO | "huge" : "", | 20260507 16:05:43.593 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:43.593 | INFO | "malloc" : " 14 GB", | 20260507 16:05:43.593 | INFO | "pinned" : " 29 GB" | 20260507 16:05:43.593 | INFO | } | 20260507 16:05:43.593 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:05:43.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:43.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.593 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260507 16:05:43.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.593 | INFO | PRE_HQ : 38.2%, 1.0, 19.9ms, 0 | 20260507 16:05:43.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:43.594 | INFO | Uploaded 36GiB to GPU in 1.61416s (22.3026GiB/s) | 20260507 16:05:43.594 | INFO | Downloaded 34GiB to GPU in 1.5346s (22.1556GiB/s) | 20260507 16:05:43.704 | INFO | Starting a round of Baz IO | 20260507 16:05:46.616 | INFO | Populating RTMetrics report | 20260507 16:05:46.616 | INFO | RTMetrics report populated | 20260507 16:05:46.616 | INFO | Opening tmp RTMetrics json file | 20260507 16:05:46.617 | INFO | Serializing RTMetrics to json | 20260507 16:05:46.619 | INFO | Serialization done, swapping tmp file now... | 20260507 16:05:46.619 | INFO | RTMetrics done | 20260507 16:05:46.736 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.031 seconds | 20260507 16:05:46.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.20% / 0.03% / 48.77% | 20260507 16:05:46.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.9354% / 8.34465e-05% / 1.58946e-05% | 20260507 16:05:46.736 | INFO | Baz IO overall: | 20260507 16:05:46.736 | INFO | BytesWritten : 337256342709 | 20260507 16:05:46.736 | INFO | NumEvents : 162981206243 | 20260507 16:05:48.732 | INFO | Analyzing chunk frames = [798720,799232) | 20260507 16:05:48.732 | INFO | MemInfo: { | 20260507 16:05:48.732 | INFO | "gpu" : " 17 GB", | 20260507 16:05:48.732 | INFO | "huge" : "", | 20260507 16:05:48.732 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:48.732 | INFO | "malloc" : " 14 GB", | 20260507 16:05:48.732 | INFO | "pinned" : " 29 GB" | 20260507 16:05:48.732 | INFO | } | 20260507 16:05:48.732 | INFO | Perf report for the last 5.13904 seconds: | 20260507 16:05:48.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:48.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:48.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:48.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:48.732 | INFO | ANALYSIS : 57.9%, 2.8, 83.8ms, 0 | 20260507 16:05:48.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:48.732 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260507 16:05:48.732 | INFO | PRE_HQ : 57.3%, 2.2, 66.7ms, 0 | 20260507 16:05:48.732 | INFO | BAZWRITER : 58.5%, 4.6, 139.7ms, 0 | 20260507 16:05:48.732 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 16:05:48.732 | INFO | Uploaded 36GiB to GPU in 1.6449s (21.8858GiB/s) | 20260507 16:05:48.732 | INFO | Downloaded 38GiB to GPU in 1.72456s (22.0347GiB/s) | 20260507 16:05:53.871 | INFO | Analyzing chunk frames = [799232,799744) | 20260507 16:05:53.871 | INFO | MemInfo: { | 20260507 16:05:53.871 | INFO | "gpu" : " 17 GB", | 20260507 16:05:53.871 | INFO | "huge" : "", | 20260507 16:05:53.871 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:53.871 | INFO | "malloc" : " 14 GB", | 20260507 16:05:53.871 | INFO | "pinned" : " 29 GB" | 20260507 16:05:53.871 | INFO | } | 20260507 16:05:53.871 | INFO | Perf report for the last 5.13914 seconds: | 20260507 16:05:53.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:53.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 16:05:53.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:05:53.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:53.871 | INFO | Uploaded 36GiB to GPU in 1.60648s (22.4092GiB/s) | 20260507 16:05:53.872 | INFO | Downloaded 34GiB to GPU in 1.52619s (22.2778GiB/s) | 20260507 16:05:59.011 | INFO | Analyzing chunk frames = [799744,800256) | 20260507 16:05:59.011 | INFO | MemInfo: { | 20260507 16:05:59.011 | INFO | "gpu" : " 17 GB", | 20260507 16:05:59.011 | INFO | "huge" : "", | 20260507 16:05:59.011 | INFO | "huge_shared" : " 29 GB", | 20260507 16:05:59.011 | INFO | "malloc" : " 14 GB", | 20260507 16:05:59.011 | INFO | "pinned" : " 29 GB" | 20260507 16:05:59.011 | INFO | } | 20260507 16:05:59.011 | INFO | Perf report for the last 5.13976 seconds: | 20260507 16:05:59.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:05:59.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:05:59.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:05:59.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:05:59.011 | INFO | Uploaded 36GiB to GPU in 1.61037s (22.355GiB/s) | 20260507 16:05:59.011 | INFO | Downloaded 34GiB to GPU in 1.5308s (22.2106GiB/s) | 20260507 16:06:04.150 | INFO | Analyzing chunk frames = [800256,800768) | 20260507 16:06:04.150 | INFO | MemInfo: { | 20260507 16:06:04.150 | INFO | "gpu" : " 17 GB", | 20260507 16:06:04.150 | INFO | "huge" : "", | 20260507 16:06:04.150 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:04.150 | INFO | "malloc" : " 14 GB", | 20260507 16:06:04.150 | INFO | "pinned" : " 29 GB" | 20260507 16:06:04.150 | INFO | } | 20260507 16:06:04.150 | INFO | Perf report for the last 5.13917 seconds: | 20260507 16:06:04.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:04.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:06:04.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:06:04.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:04.150 | INFO | Uploaded 36GiB to GPU in 1.61083s (22.3487GiB/s) | 20260507 16:06:04.150 | INFO | Downloaded 34GiB to GPU in 1.53082s (22.2104GiB/s) | 20260507 16:06:09.289 | INFO | Analyzing chunk frames = [800768,801280) | 20260507 16:06:09.289 | INFO | MemInfo: { | 20260507 16:06:09.289 | INFO | "gpu" : " 17 GB", | 20260507 16:06:09.289 | INFO | "huge" : "", | 20260507 16:06:09.289 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:09.289 | INFO | "malloc" : " 14 GB", | 20260507 16:06:09.289 | INFO | "pinned" : " 29 GB" | 20260507 16:06:09.289 | INFO | } | 20260507 16:06:09.289 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:06:09.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:09.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.289 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:06:09.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.290 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:06:09.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:09.290 | INFO | Uploaded 36GiB to GPU in 1.60897s (22.3746GiB/s) | 20260507 16:06:09.290 | INFO | Downloaded 34GiB to GPU in 1.52822s (22.2481GiB/s) | 20260507 16:06:14.428 | INFO | Analyzing chunk frames = [801280,801792) | 20260507 16:06:14.428 | INFO | MemInfo: { | 20260507 16:06:14.428 | INFO | "gpu" : " 17 GB", | 20260507 16:06:14.428 | INFO | "huge" : "", | 20260507 16:06:14.428 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:14.428 | INFO | "malloc" : " 14 GB", | 20260507 16:06:14.429 | INFO | "pinned" : " 29 GB" | 20260507 16:06:14.429 | INFO | } | 20260507 16:06:14.429 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:06:14.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:14.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:06:14.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:06:14.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:14.429 | INFO | Uploaded 36GiB to GPU in 1.60869s (22.3785GiB/s) | 20260507 16:06:14.429 | INFO | Downloaded 34GiB to GPU in 1.52799s (22.2515GiB/s) | 20260507 16:06:19.568 | INFO | Analyzing chunk frames = [801792,802304) | 20260507 16:06:19.568 | INFO | MemInfo: { | 20260507 16:06:19.568 | INFO | "gpu" : " 17 GB", | 20260507 16:06:19.568 | INFO | "huge" : "", | 20260507 16:06:19.568 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:19.568 | INFO | "malloc" : " 14 GB", | 20260507 16:06:19.568 | INFO | "pinned" : " 29 GB" | 20260507 16:06:19.568 | INFO | } | 20260507 16:06:19.568 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:06:19.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:19.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:06:19.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:06:19.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:19.568 | INFO | Uploaded 36GiB to GPU in 1.60915s (22.372GiB/s) | 20260507 16:06:19.568 | INFO | Downloaded 34GiB to GPU in 1.52883s (22.2392GiB/s) | 20260507 16:06:24.707 | INFO | Analyzing chunk frames = [802304,802816) | 20260507 16:06:24.707 | INFO | MemInfo: { | 20260507 16:06:24.707 | INFO | "gpu" : " 17 GB", | 20260507 16:06:24.707 | INFO | "huge" : "", | 20260507 16:06:24.707 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:24.707 | INFO | "malloc" : " 14 GB", | 20260507 16:06:24.707 | INFO | "pinned" : " 29 GB" | 20260507 16:06:24.707 | INFO | } | 20260507 16:06:24.707 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:06:24.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:24.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:06:24.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:06:24.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:24.707 | INFO | Uploaded 36GiB to GPU in 1.60935s (22.3692GiB/s) | 20260507 16:06:24.707 | INFO | Downloaded 34GiB to GPU in 1.52932s (22.2322GiB/s) | 20260507 16:06:24.708 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:06:24.708 | INFO | Total mem usage: 16.4 GiB | 20260507 16:06:24.708 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:06:24.708 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.17%) | 20260507 16:06:24.708 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:06:24.708 | INFO | High Water Breakdown: | 20260507 16:06:24.708 | INFO | 59.4% : 8.93 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:06:24.708 | INFO | 40.4% : 6.07 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:06:24.708 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:06:24.708 | INFO | ------------------------------------------ | 20260507 16:06:24.708 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:06:24.708 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:06:24.708 | INFO | Total mem usage: 26.5 GiB | 20260507 16:06:24.708 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:06:24.708 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:06:24.708 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 16:06:24.708 | INFO | High Water Breakdown: | 20260507 16:06:24.708 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:06:24.708 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:06:24.708 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:06:24.708 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:06:24.708 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:06:24.708 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:06:24.708 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:06:24.708 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:06:24.708 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:06:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:06:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:06:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:06:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:06:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:06:24.708 | INFO | ------------------------------------------ | 20260507 16:06:24.708 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:06:24.708 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:06:24.708 | INFO | Total mem usage: 27.8 GiB | 20260507 16:06:24.708 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:06:24.708 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:06:24.708 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:06:24.708 | INFO | High Water Breakdown: | 20260507 16:06:24.708 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:06:24.708 | INFO | ------------------------------------------ | 20260507 16:06:24.708 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:06:24.708 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:06:24.708 | INFO | Total mem usage: 15.7 GiB | 20260507 16:06:24.708 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:06:24.708 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:06:24.708 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:06:24.708 | INFO | High Water Breakdown: | 20260507 16:06:24.708 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:06:24.708 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:06:24.708 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:06:24.708 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:06:24.709 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:06:24.709 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:06:24.709 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:06:24.709 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:06:24.709 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:06:24.709 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:06:24.709 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:06:24.709 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:06:24.709 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:06:24.709 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:06:24.709 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:06:24.709 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:06:24.709 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:06:24.709 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:06:24.709 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:06:24.709 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:06:24.709 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:06:24.709 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:06:24.709 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:06:24.709 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:06:24.709 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:06:24.709 | INFO | ------------------------------------------ | 20260507 16:06:24.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:06:24.709 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:06:24.709 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:06:24.709 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:06:24.709 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 0.29%, AggregateStats -- 8.74e-02 / -nan, 6.14e-03 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 11.06%, Baseline -- 3.30e+00 / -nan, 1.99e-02 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.12e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 3.39%, Metrics -- 1.01e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.71e-01 / -nan, 5.50e-02 / 0.00e+00 | 20260507 16:06:24.709 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:06:24.709 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:06:24.709 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:06:24.709 | INFO | PerfReport: Full Execution -- 8.90e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 18.92%, Upload -- 1.68e+01 / 0.00e+00, 4.65e-01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 18.21%, Download -- 1.62e+01 / 0.00e+00, 8.16e-01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 33.49%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 0.58%, LockUpload -- 5.15e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 0.00%, LockDownload -- 4.76e-04 / 0.00e+00, 2.74e-04 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 28.78%, LockCompute -- 2.56e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 16:06:24.709 | INFO | PerfReport: 0.02%, Misc -- 1.34e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 16:06:24.804 | INFO | Starting a round of Baz IO | 20260507 16:06:27.698 | INFO | Populating RTMetrics report | 20260507 16:06:27.698 | INFO | RTMetrics report populated | 20260507 16:06:27.698 | INFO | Opening tmp RTMetrics json file | 20260507 16:06:27.774 | INFO | Serializing RTMetrics to json | 20260507 16:06:27.783 | INFO | Serialization done, swapping tmp file now... | 20260507 16:06:27.785 | INFO | RTMetrics done | 20260507 16:06:27.812 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.007 seconds | 20260507 16:06:27.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.17% / 0.03% / 48.80% | 20260507 16:06:27.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.89691% / 5.5631e-05% / 2.38419e-05% | 20260507 16:06:27.812 | INFO | Baz IO overall: | 20260507 16:06:27.812 | INFO | BytesWritten : 339162601840 | 20260507 16:06:27.812 | INFO | NumEvents : 163908296922 | 20260507 16:06:29.846 | INFO | Analyzing chunk frames = [802816,803328) | 20260507 16:06:29.846 | INFO | MemInfo: { | 20260507 16:06:29.846 | INFO | "gpu" : " 17 GB", | 20260507 16:06:29.846 | INFO | "huge" : "", | 20260507 16:06:29.846 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:29.846 | INFO | "malloc" : " 14 GB", | 20260507 16:06:29.846 | INFO | "pinned" : " 29 GB" | 20260507 16:06:29.846 | INFO | } | 20260507 16:06:29.847 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:06:29.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:29.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:29.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:29.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:29.847 | INFO | ANALYSIS : 57.3%, 2.8, 84.3ms, 0 | 20260507 16:06:29.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:29.847 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 16:06:29.847 | INFO | PRE_HQ : 56.4%, 2.1, 61.4ms, 0 | 20260507 16:06:29.847 | INFO | BAZWRITER : 58.1%, 4.8, 141.9ms, 0 | 20260507 16:06:29.847 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 16:06:29.847 | INFO | Uploaded 36GiB to GPU in 1.66044s (21.681GiB/s) | 20260507 16:06:29.847 | INFO | Downloaded 38GiB to GPU in 1.74068s (21.8306GiB/s) | 20260507 16:06:34.986 | INFO | Analyzing chunk frames = [803328,803840) | 20260507 16:06:34.986 | INFO | MemInfo: { | 20260507 16:06:34.986 | INFO | "gpu" : " 17 GB", | 20260507 16:06:34.986 | INFO | "huge" : "", | 20260507 16:06:34.986 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:34.986 | INFO | "malloc" : " 14 GB", | 20260507 16:06:34.986 | INFO | "pinned" : " 29 GB" | 20260507 16:06:34.986 | INFO | } | 20260507 16:06:34.986 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:06:34.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:34.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:06:34.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:06:34.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:34.986 | INFO | Uploaded 36GiB to GPU in 1.58571s (22.7027GiB/s) | 20260507 16:06:34.986 | INFO | Downloaded 34GiB to GPU in 1.53429s (22.1601GiB/s) | 20260507 16:06:40.125 | INFO | Analyzing chunk frames = [803840,804352) | 20260507 16:06:40.125 | INFO | MemInfo: { | 20260507 16:06:40.125 | INFO | "gpu" : " 17 GB", | 20260507 16:06:40.125 | INFO | "huge" : "", | 20260507 16:06:40.125 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:40.125 | INFO | "malloc" : " 14 GB", | 20260507 16:06:40.125 | INFO | "pinned" : " 29 GB" | 20260507 16:06:40.125 | INFO | } | 20260507 16:06:40.125 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:06:40.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:40.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.125 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:06:40.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.125 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:06:40.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:40.126 | INFO | Uploaded 36GiB to GPU in 1.59099s (22.6274GiB/s) | 20260507 16:06:40.126 | INFO | Downloaded 34GiB to GPU in 1.53991s (22.0793GiB/s) | 20260507 16:06:45.264 | INFO | Analyzing chunk frames = [804352,804864) | 20260507 16:06:45.264 | INFO | MemInfo: { | 20260507 16:06:45.264 | INFO | "gpu" : " 17 GB", | 20260507 16:06:45.264 | INFO | "huge" : "", | 20260507 16:06:45.265 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:45.265 | INFO | "malloc" : " 14 GB", | 20260507 16:06:45.265 | INFO | "pinned" : " 29 GB" | 20260507 16:06:45.265 | INFO | } | 20260507 16:06:45.265 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:06:45.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:45.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | ANALYSIS : 55.9%, 3.0, 86.6ms, 0 | 20260507 16:06:45.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:06:45.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:45.265 | INFO | Uploaded 36GiB to GPU in 1.58955s (22.648GiB/s) | 20260507 16:06:45.265 | INFO | Downloaded 34GiB to GPU in 1.54007s (22.077GiB/s) | 20260507 16:06:50.404 | INFO | Analyzing chunk frames = [804864,805376) | 20260507 16:06:50.404 | INFO | MemInfo: { | 20260507 16:06:50.404 | INFO | "gpu" : " 17 GB", | 20260507 16:06:50.404 | INFO | "huge" : "", | 20260507 16:06:50.404 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:50.404 | INFO | "malloc" : " 14 GB", | 20260507 16:06:50.404 | INFO | "pinned" : " 29 GB" | 20260507 16:06:50.404 | INFO | } | 20260507 16:06:50.404 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:06:50.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:50.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:06:50.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:06:50.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:50.404 | INFO | Uploaded 36GiB to GPU in 1.58864s (22.6608GiB/s) | 20260507 16:06:50.404 | INFO | Downloaded 34GiB to GPU in 1.53769s (22.111GiB/s) | 20260507 16:06:55.543 | INFO | Analyzing chunk frames = [805376,805888) | 20260507 16:06:55.543 | INFO | MemInfo: { | 20260507 16:06:55.543 | INFO | "gpu" : " 17 GB", | 20260507 16:06:55.543 | INFO | "huge" : "", | 20260507 16:06:55.543 | INFO | "huge_shared" : " 29 GB", | 20260507 16:06:55.543 | INFO | "malloc" : " 14 GB", | 20260507 16:06:55.543 | INFO | "pinned" : " 29 GB" | 20260507 16:06:55.543 | INFO | } | 20260507 16:06:55.543 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:06:55.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:06:55.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.543 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 16:06:55.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.544 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:06:55.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:06:55.544 | INFO | Uploaded 36GiB to GPU in 1.58837s (22.6647GiB/s) | 20260507 16:06:55.544 | INFO | Downloaded 34GiB to GPU in 1.53771s (22.1109GiB/s) | 20260507 16:07:00.683 | INFO | Analyzing chunk frames = [805888,806400) | 20260507 16:07:00.683 | INFO | MemInfo: { | 20260507 16:07:00.683 | INFO | "gpu" : " 17 GB", | 20260507 16:07:00.683 | INFO | "huge" : "", | 20260507 16:07:00.683 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:00.683 | INFO | "malloc" : " 14 GB", | 20260507 16:07:00.683 | INFO | "pinned" : " 29 GB" | 20260507 16:07:00.683 | INFO | } | 20260507 16:07:00.683 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:07:00.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:00.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:07:00.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:07:00.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:00.683 | INFO | Uploaded 36GiB to GPU in 1.59011s (22.64GiB/s) | 20260507 16:07:00.683 | INFO | Downloaded 34GiB to GPU in 1.53909s (22.0909GiB/s) | 20260507 16:07:05.822 | INFO | Analyzing chunk frames = [806400,806912) | 20260507 16:07:05.822 | INFO | MemInfo: { | 20260507 16:07:05.822 | INFO | "gpu" : " 17 GB", | 20260507 16:07:05.822 | INFO | "huge" : "", | 20260507 16:07:05.822 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:05.822 | INFO | "malloc" : " 14 GB", | 20260507 16:07:05.822 | INFO | "pinned" : " 29 GB" | 20260507 16:07:05.822 | INFO | } | 20260507 16:07:05.822 | INFO | Perf report for the last 5.13898 seconds: | 20260507 16:07:05.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:05.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:07:05.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:07:05.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:05.822 | INFO | Uploaded 36GiB to GPU in 1.58929s (22.6517GiB/s) | 20260507 16:07:05.822 | INFO | Downloaded 34GiB to GPU in 1.53776s (22.1101GiB/s) | 20260507 16:07:05.919 | INFO | Starting a round of Baz IO | 20260507 16:07:08.791 | INFO | Populating RTMetrics report | 20260507 16:07:08.792 | INFO | RTMetrics report populated | 20260507 16:07:08.792 | INFO | Opening tmp RTMetrics json file | 20260507 16:07:08.795 | INFO | Serializing RTMetrics to json | 20260507 16:07:08.800 | INFO | Serialization done, swapping tmp file now... | 20260507 16:07:08.806 | INFO | RTMetrics done | 20260507 16:07:08.892 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 2.973 seconds | 20260507 16:07:08.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.13% / 0.03% / 48.83% | 20260507 16:07:08.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.89269% / 9.53674e-05% / 1.19209e-05% | 20260507 16:07:08.892 | INFO | Baz IO overall: | 20260507 16:07:08.892 | INFO | BytesWritten : 341070008063 | 20260507 16:07:08.892 | INFO | NumEvents : 164836496218 | 20260507 16:07:10.961 | INFO | Analyzing chunk frames = [806912,807424) | 20260507 16:07:10.961 | INFO | MemInfo: { | 20260507 16:07:10.961 | INFO | "gpu" : " 17 GB", | 20260507 16:07:10.961 | INFO | "huge" : "", | 20260507 16:07:10.961 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:10.961 | INFO | "malloc" : " 14 GB", | 20260507 16:07:10.961 | INFO | "pinned" : " 29 GB" | 20260507 16:07:10.961 | INFO | } | 20260507 16:07:10.961 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:07:10.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:10.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:10.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:10.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:10.961 | INFO | ANALYSIS : 56.9%, 2.9, 87.7ms, 0 | 20260507 16:07:10.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:10.961 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 16:07:10.962 | INFO | PRE_HQ : 55.4%, 2.1, 59.1ms, 0 | 20260507 16:07:10.962 | INFO | BAZWRITER : 57.4%, 4.3, 126.7ms, 0 | 20260507 16:07:10.962 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 16:07:10.962 | INFO | Uploaded 36GiB to GPU in 1.64214s (21.9226GiB/s) | 20260507 16:07:10.962 | INFO | Downloaded 38GiB to GPU in 1.72224s (22.0643GiB/s) | 20260507 16:07:16.101 | INFO | Analyzing chunk frames = [807424,807936) | 20260507 16:07:16.101 | INFO | MemInfo: { | 20260507 16:07:16.101 | INFO | "gpu" : " 17 GB", | 20260507 16:07:16.101 | INFO | "huge" : "", | 20260507 16:07:16.101 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:16.101 | INFO | "malloc" : " 14 GB", | 20260507 16:07:16.101 | INFO | "pinned" : " 29 GB" | 20260507 16:07:16.101 | INFO | } | 20260507 16:07:16.101 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:07:16.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:16.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:07:16.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:07:16.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:16.101 | INFO | Uploaded 36GiB to GPU in 1.60646s (22.4095GiB/s) | 20260507 16:07:16.101 | INFO | Downloaded 34GiB to GPU in 1.52574s (22.2843GiB/s) | 20260507 16:07:21.240 | INFO | Analyzing chunk frames = [807936,808448) | 20260507 16:07:21.240 | INFO | MemInfo: { | 20260507 16:07:21.240 | INFO | "gpu" : " 17 GB", | 20260507 16:07:21.240 | INFO | "huge" : "", | 20260507 16:07:21.240 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:21.240 | INFO | "malloc" : " 14 GB", | 20260507 16:07:21.240 | INFO | "pinned" : " 29 GB" | 20260507 16:07:21.240 | INFO | } | 20260507 16:07:21.240 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:07:21.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:21.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.240 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:07:21.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.240 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:07:21.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:21.241 | INFO | Uploaded 36GiB to GPU in 1.61079s (22.3493GiB/s) | 20260507 16:07:21.241 | INFO | Downloaded 34GiB to GPU in 1.53118s (22.2051GiB/s) | 20260507 16:07:26.379 | INFO | Analyzing chunk frames = [808448,808960) | 20260507 16:07:26.379 | INFO | MemInfo: { | 20260507 16:07:26.379 | INFO | "gpu" : " 17 GB", | 20260507 16:07:26.379 | INFO | "huge" : "", | 20260507 16:07:26.379 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:26.379 | INFO | "malloc" : " 14 GB", | 20260507 16:07:26.379 | INFO | "pinned" : " 29 GB" | 20260507 16:07:26.379 | INFO | } | 20260507 16:07:26.380 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:07:26.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:26.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:07:26.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:07:26.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:26.380 | INFO | Uploaded 36GiB to GPU in 1.61018s (22.3577GiB/s) | 20260507 16:07:26.380 | INFO | Downloaded 34GiB to GPU in 1.53084s (22.21GiB/s) | 20260507 16:07:31.519 | INFO | Analyzing chunk frames = [808960,809472) | 20260507 16:07:31.519 | INFO | MemInfo: { | 20260507 16:07:31.519 | INFO | "gpu" : " 17 GB", | 20260507 16:07:31.519 | INFO | "huge" : "", | 20260507 16:07:31.519 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:31.519 | INFO | "malloc" : " 14 GB", | 20260507 16:07:31.519 | INFO | "pinned" : " 29 GB" | 20260507 16:07:31.519 | INFO | } | 20260507 16:07:31.519 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:07:31.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:31.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:07:31.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:07:31.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:31.519 | INFO | Uploaded 36GiB to GPU in 1.60968s (22.3647GiB/s) | 20260507 16:07:31.519 | INFO | Downloaded 34GiB to GPU in 1.52983s (22.2247GiB/s) | 20260507 16:07:36.658 | INFO | Analyzing chunk frames = [809472,809984) | 20260507 16:07:36.658 | INFO | MemInfo: { | 20260507 16:07:36.658 | INFO | "gpu" : " 17 GB", | 20260507 16:07:36.658 | INFO | "huge" : "", | 20260507 16:07:36.658 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:36.658 | INFO | "malloc" : " 14 GB", | 20260507 16:07:36.658 | INFO | "pinned" : " 29 GB" | 20260507 16:07:36.658 | INFO | } | 20260507 16:07:36.658 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:07:36.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:36.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.658 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:07:36.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.659 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:07:36.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:36.659 | INFO | Uploaded 36GiB to GPU in 1.60898s (22.3745GiB/s) | 20260507 16:07:36.659 | INFO | Downloaded 34GiB to GPU in 1.52772s (22.2554GiB/s) | 20260507 16:07:41.797 | INFO | Analyzing chunk frames = [809984,810496) | 20260507 16:07:41.797 | INFO | MemInfo: { | 20260507 16:07:41.797 | INFO | "gpu" : " 17 GB", | 20260507 16:07:41.797 | INFO | "huge" : "", | 20260507 16:07:41.797 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:41.797 | INFO | "malloc" : " 14 GB", | 20260507 16:07:41.798 | INFO | "pinned" : " 29 GB" | 20260507 16:07:41.798 | INFO | } | 20260507 16:07:41.798 | INFO | Perf report for the last 5.13917 seconds: | 20260507 16:07:41.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:41.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:07:41.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:07:41.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:41.798 | INFO | Uploaded 36GiB to GPU in 1.60971s (22.3643GiB/s) | 20260507 16:07:41.798 | INFO | Downloaded 34GiB to GPU in 1.5297s (22.2265GiB/s) | 20260507 16:07:46.937 | INFO | Analyzing chunk frames = [810496,811008) | 20260507 16:07:46.937 | INFO | MemInfo: { | 20260507 16:07:46.937 | INFO | "gpu" : " 17 GB", | 20260507 16:07:46.937 | INFO | "huge" : "", | 20260507 16:07:46.937 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:46.937 | INFO | "malloc" : " 14 GB", | 20260507 16:07:46.937 | INFO | "pinned" : " 29 GB" | 20260507 16:07:46.937 | INFO | } | 20260507 16:07:46.937 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:07:46.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:46.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:07:46.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:07:46.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:46.937 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:07:46.937 | INFO | Total mem usage: 16.4 GiB | 20260507 16:07:46.937 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:07:46.937 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.14%) | 20260507 16:07:46.937 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:07:46.937 | INFO | High Water Breakdown: | 20260507 16:07:46.937 | INFO | 59.5% : 8.94 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:07:46.937 | INFO | 40.4% : 6.07 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:07:46.937 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:07:46.937 | INFO | ------------------------------------------ | 20260507 16:07:46.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:07:46.937 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:07:46.937 | INFO | Total mem usage: 26.5 GiB | 20260507 16:07:46.937 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:07:46.937 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 16:07:46.937 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 16:07:46.937 | INFO | High Water Breakdown: | 20260507 16:07:46.937 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:07:46.937 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:07:46.937 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:07:46.937 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:07:46.937 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:07:46.937 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:07:46.937 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:07:46.937 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:07:46.937 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:07:46.937 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:07:46.937 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:07:46.937 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:07:46.937 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:07:46.937 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:07:46.937 | INFO | ------------------------------------------ | 20260507 16:07:46.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:07:46.937 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:07:46.937 | INFO | Total mem usage: 27.8 GiB | 20260507 16:07:46.937 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:07:46.937 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:07:46.937 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:07:46.937 | INFO | High Water Breakdown: | 20260507 16:07:46.937 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:07:46.937 | INFO | ------------------------------------------ | 20260507 16:07:46.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:07:46.937 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:07:46.937 | INFO | Total mem usage: 15.7 GiB | 20260507 16:07:46.937 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:07:46.937 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:07:46.937 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:07:46.937 | INFO | High Water Breakdown: | 20260507 16:07:46.937 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:07:46.937 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:07:46.937 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:07:46.937 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:07:46.937 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:07:46.937 | INFO | Uploaded 36GiB to GPU in 1.60952s (22.3669GiB/s) | 20260507 16:07:46.937 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:07:46.938 | INFO | Downloaded 34GiB to GPU in 1.52968s (22.2269GiB/s) | 20260507 16:07:46.938 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:07:46.938 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:07:46.938 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:07:46.938 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:07:46.938 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:07:46.938 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:07:46.938 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:07:46.938 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:07:46.938 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:07:46.938 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:07:46.938 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:07:46.938 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:07:46.938 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:07:46.938 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:07:46.939 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:07:46.939 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:07:46.939 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:07:46.939 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:07:46.939 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:07:46.939 | INFO | ------------------------------------------ | 20260507 16:07:46.939 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:07:46.939 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:07:46.939 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:07:46.939 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:07:46.939 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.22e+01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.78e-03 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 11.06%, Baseline -- 3.29e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 3.08%, Histogram -- 9.17e-01 / -nan, 5.03e-02 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.72e+01 / -nan, 4.26e-02 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.12e+00 / -nan, 1.19e-01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 3.39%, Metrics -- 1.01e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.69e-01 / -nan, 5.42e-02 / 0.00e+00 | 20260507 16:07:46.939 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:07:46.939 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:07:46.939 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:07:46.939 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 18.77%, Upload -- 1.67e+01 / 0.00e+00, 4.67e-01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 18.22%, Download -- 1.62e+01 / 0.00e+00, 7.87e-01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 33.42%, Compute -- 2.98e+01 / 0.00e+00, 1.22e+01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 0.59%, LockUpload -- 5.25e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 0.00%, LockDownload -- 4.26e-04 / 0.00e+00, 1.01e-04 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 28.99%, LockCompute -- 2.58e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 16:07:46.939 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 16:07:47.035 | INFO | Starting a round of Baz IO | 20260507 16:07:49.908 | INFO | Populating RTMetrics report | 20260507 16:07:49.909 | INFO | RTMetrics report populated | 20260507 16:07:49.909 | INFO | Opening tmp RTMetrics json file | 20260507 16:07:49.911 | INFO | Serializing RTMetrics to json | 20260507 16:07:49.913 | INFO | Serialization done, swapping tmp file now... | 20260507 16:07:49.913 | INFO | RTMetrics done | 20260507 16:07:50.014 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 2.979 seconds | 20260507 16:07:50.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.10% / 0.03% / 48.87% | 20260507 16:07:50.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.87691% / 7.15256e-05% / 1.58946e-05% | 20260507 16:07:50.014 | INFO | Baz IO overall: | 20260507 16:07:50.014 | INFO | BytesWritten : 342978872717 | 20260507 16:07:50.014 | INFO | NumEvents : 165765634710 | 20260507 16:07:52.076 | INFO | Analyzing chunk frames = [811008,811520) | 20260507 16:07:52.076 | INFO | MemInfo: { | 20260507 16:07:52.076 | INFO | "gpu" : " 17 GB", | 20260507 16:07:52.076 | INFO | "huge" : "", | 20260507 16:07:52.076 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:52.076 | INFO | "malloc" : " 14 GB", | 20260507 16:07:52.076 | INFO | "pinned" : " 29 GB" | 20260507 16:07:52.076 | INFO | } | 20260507 16:07:52.076 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:07:52.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:52.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:52.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:52.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:52.076 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260507 16:07:52.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:52.076 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 16:07:52.076 | INFO | PRE_HQ : 55.4%, 2.1, 58.9ms, 0 | 20260507 16:07:52.076 | INFO | BAZWRITER : 57.5%, 4.3, 126.5ms, 0 | 20260507 16:07:52.076 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:07:52.077 | INFO | Uploaded 36GiB to GPU in 1.63786s (21.9798GiB/s) | 20260507 16:07:52.077 | INFO | Downloaded 38GiB to GPU in 1.71694s (22.1324GiB/s) | 20260507 16:07:57.215 | INFO | Analyzing chunk frames = [811520,812032) | 20260507 16:07:57.215 | INFO | MemInfo: { | 20260507 16:07:57.215 | INFO | "gpu" : " 17 GB", | 20260507 16:07:57.215 | INFO | "huge" : "", | 20260507 16:07:57.215 | INFO | "huge_shared" : " 29 GB", | 20260507 16:07:57.215 | INFO | "malloc" : " 14 GB", | 20260507 16:07:57.215 | INFO | "pinned" : " 29 GB" | 20260507 16:07:57.216 | INFO | } | 20260507 16:07:57.216 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:07:57.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:07:57.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:07:57.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:07:57.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:07:57.216 | INFO | Uploaded 36GiB to GPU in 1.58607s (22.6976GiB/s) | 20260507 16:07:57.216 | INFO | Downloaded 34GiB to GPU in 1.53485s (22.152GiB/s) | 20260507 16:08:02.355 | INFO | Analyzing chunk frames = [812032,812544) | 20260507 16:08:02.355 | INFO | MemInfo: { | 20260507 16:08:02.355 | INFO | "gpu" : " 17 GB", | 20260507 16:08:02.355 | INFO | "huge" : "", | 20260507 16:08:02.355 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:02.355 | INFO | "malloc" : " 14 GB", | 20260507 16:08:02.355 | INFO | "pinned" : " 29 GB" | 20260507 16:08:02.355 | INFO | } | 20260507 16:08:02.355 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:08:02.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:02.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.355 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:08:02.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.355 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:08:02.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:02.356 | INFO | Uploaded 36GiB to GPU in 1.59031s (22.6371GiB/s) | 20260507 16:08:02.356 | INFO | Downloaded 34GiB to GPU in 1.5398s (22.0808GiB/s) | 20260507 16:08:07.494 | INFO | Analyzing chunk frames = [812544,813056) | 20260507 16:08:07.494 | INFO | MemInfo: { | 20260507 16:08:07.494 | INFO | "gpu" : " 17 GB", | 20260507 16:08:07.494 | INFO | "huge" : "", | 20260507 16:08:07.494 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:07.494 | INFO | "malloc" : " 14 GB", | 20260507 16:08:07.494 | INFO | "pinned" : " 29 GB" | 20260507 16:08:07.494 | INFO | } | 20260507 16:08:07.495 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:08:07.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:07.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:08:07.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260507 16:08:07.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:07.495 | INFO | Uploaded 36GiB to GPU in 1.59281s (22.6016GiB/s) | 20260507 16:08:07.495 | INFO | Downloaded 34GiB to GPU in 1.5425s (22.0421GiB/s) | 20260507 16:08:12.634 | INFO | Analyzing chunk frames = [813056,813568) | 20260507 16:08:12.634 | INFO | MemInfo: { | 20260507 16:08:12.634 | INFO | "gpu" : " 17 GB", | 20260507 16:08:12.634 | INFO | "huge" : "", | 20260507 16:08:12.634 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:12.634 | INFO | "malloc" : " 14 GB", | 20260507 16:08:12.634 | INFO | "pinned" : " 29 GB" | 20260507 16:08:12.634 | INFO | } | 20260507 16:08:12.634 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:08:12.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:12.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.634 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 16:08:12.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.634 | INFO | PRE_HQ : 38.8%, 1.0, 20.1ms, 0 | 20260507 16:08:12.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:12.635 | INFO | Uploaded 36GiB to GPU in 1.59394s (22.5855GiB/s) | 20260507 16:08:12.635 | INFO | Downloaded 34GiB to GPU in 1.54121s (22.0606GiB/s) | 20260507 16:08:17.773 | INFO | Analyzing chunk frames = [813568,814080) | 20260507 16:08:17.773 | INFO | MemInfo: { | 20260507 16:08:17.773 | INFO | "gpu" : " 17 GB", | 20260507 16:08:17.773 | INFO | "huge" : "", | 20260507 16:08:17.773 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:17.773 | INFO | "malloc" : " 14 GB", | 20260507 16:08:17.773 | INFO | "pinned" : " 29 GB" | 20260507 16:08:17.774 | INFO | } | 20260507 16:08:17.774 | INFO | Perf report for the last 5.13966 seconds: | 20260507 16:08:17.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:17.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:08:17.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | PRE_HQ : 39.9%, 1.0, 20.8ms, 0 | 20260507 16:08:17.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:17.774 | INFO | Uploaded 36GiB to GPU in 1.59532s (22.566GiB/s) | 20260507 16:08:17.775 | INFO | Downloaded 34GiB to GPU in 1.54344s (22.0287GiB/s) | 20260507 16:08:22.912 | INFO | Analyzing chunk frames = [814080,814592) | 20260507 16:08:22.912 | INFO | MemInfo: { | 20260507 16:08:22.912 | INFO | "gpu" : " 17 GB", | 20260507 16:08:22.912 | INFO | "huge" : "", | 20260507 16:08:22.913 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:22.913 | INFO | "malloc" : " 14 GB", | 20260507 16:08:22.913 | INFO | "pinned" : " 29 GB" | 20260507 16:08:22.913 | INFO | } | 20260507 16:08:22.913 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:08:22.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:22.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.913 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 16:08:22.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.913 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:08:22.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:22.914 | INFO | Uploaded 36GiB to GPU in 1.60273s (22.4617GiB/s) | 20260507 16:08:22.914 | INFO | Downloaded 34GiB to GPU in 1.55188s (21.9089GiB/s) | 20260507 16:08:28.052 | INFO | Analyzing chunk frames = [814592,815104) | 20260507 16:08:28.052 | INFO | MemInfo: { | 20260507 16:08:28.052 | INFO | "gpu" : " 17 GB", | 20260507 16:08:28.052 | INFO | "huge" : "", | 20260507 16:08:28.052 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:28.052 | INFO | "malloc" : " 14 GB", | 20260507 16:08:28.052 | INFO | "pinned" : " 29 GB" | 20260507 16:08:28.052 | INFO | } | 20260507 16:08:28.052 | INFO | Perf report for the last 5.13902 seconds: | 20260507 16:08:28.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:28.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:08:28.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 16:08:28.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:28.052 | INFO | Uploaded 36GiB to GPU in 1.59076s (22.6306GiB/s) | 20260507 16:08:28.052 | INFO | Downloaded 34GiB to GPU in 1.53995s (22.0786GiB/s) | 20260507 16:08:28.149 | INFO | Starting a round of Baz IO | 20260507 16:08:31.120 | INFO | Populating RTMetrics report | 20260507 16:08:31.120 | INFO | RTMetrics report populated | 20260507 16:08:31.120 | INFO | Opening tmp RTMetrics json file | 20260507 16:08:31.122 | INFO | Serializing RTMetrics to json | 20260507 16:08:31.124 | INFO | Serialization done, swapping tmp file now... | 20260507 16:08:31.125 | INFO | RTMetrics done | 20260507 16:08:31.470 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.321 seconds | 20260507 16:08:31.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.07% / 0.03% / 48.90% | 20260507 16:08:31.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.87295% / 5.5631e-05% / 3.17891e-05% | 20260507 16:08:31.470 | INFO | Baz IO overall: | 20260507 16:08:31.470 | INFO | BytesWritten : 344888452679 | 20260507 16:08:31.470 | INFO | NumEvents : 166695118425 | 20260507 16:08:33.191 | INFO | Analyzing chunk frames = [815104,815616) | 20260507 16:08:33.191 | INFO | MemInfo: { | 20260507 16:08:33.191 | INFO | "gpu" : " 17 GB", | 20260507 16:08:33.191 | INFO | "huge" : "", | 20260507 16:08:33.191 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:33.191 | INFO | "malloc" : " 14 GB", | 20260507 16:08:33.191 | INFO | "pinned" : " 29 GB" | 20260507 16:08:33.191 | INFO | } | 20260507 16:08:33.191 | INFO | Perf report for the last 5.1395 seconds: | 20260507 16:08:33.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:33.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:33.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:33.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:33.191 | INFO | ANALYSIS : 58.0%, 2.8, 84.7ms, 0 | 20260507 16:08:33.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:33.192 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260507 16:08:33.192 | INFO | PRE_HQ : 57.4%, 2.2, 65.4ms, 0 | 20260507 16:08:33.192 | INFO | BAZWRITER : 64.2%, 4.6, 150.3ms, 0 | 20260507 16:08:33.192 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 16:08:33.192 | INFO | Uploaded 36GiB to GPU in 1.63183s (22.0611GiB/s) | 20260507 16:08:33.192 | INFO | Downloaded 38GiB to GPU in 1.71095s (22.2099GiB/s) | 20260507 16:08:38.330 | INFO | Analyzing chunk frames = [815616,816128) | 20260507 16:08:38.330 | INFO | MemInfo: { | 20260507 16:08:38.330 | INFO | "gpu" : " 17 GB", | 20260507 16:08:38.330 | INFO | "huge" : "", | 20260507 16:08:38.330 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:38.330 | INFO | "malloc" : " 14 GB", | 20260507 16:08:38.330 | INFO | "pinned" : " 29 GB" | 20260507 16:08:38.331 | INFO | } | 20260507 16:08:38.331 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:08:38.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:38.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 16:08:38.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | PRE_HQ : 38.4%, 1.0, 19.9ms, 0 | 20260507 16:08:38.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:38.331 | INFO | Uploaded 36GiB to GPU in 1.59215s (22.6109GiB/s) | 20260507 16:08:38.331 | INFO | Downloaded 34GiB to GPU in 1.5399s (22.0793GiB/s) | 20260507 16:08:43.470 | INFO | Analyzing chunk frames = [816128,816640) | 20260507 16:08:43.470 | INFO | MemInfo: { | 20260507 16:08:43.470 | INFO | "gpu" : " 17 GB", | 20260507 16:08:43.470 | INFO | "huge" : "", | 20260507 16:08:43.470 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:43.470 | INFO | "malloc" : " 14 GB", | 20260507 16:08:43.470 | INFO | "pinned" : " 29 GB" | 20260507 16:08:43.470 | INFO | } | 20260507 16:08:43.470 | INFO | Perf report for the last 5.13962 seconds: | 20260507 16:08:43.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:43.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.470 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:08:43.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.470 | INFO | PRE_HQ : 38.8%, 1.0, 20.2ms, 0 | 20260507 16:08:43.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:43.471 | INFO | Uploaded 36GiB to GPU in 1.5955s (22.5635GiB/s) | 20260507 16:08:43.471 | INFO | Downloaded 34GiB to GPU in 1.54425s (22.0172GiB/s) | 20260507 16:08:48.609 | INFO | Analyzing chunk frames = [816640,817152) | 20260507 16:08:48.609 | INFO | MemInfo: { | 20260507 16:08:48.609 | INFO | "gpu" : " 17 GB", | 20260507 16:08:48.609 | INFO | "huge" : "", | 20260507 16:08:48.609 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:48.609 | INFO | "malloc" : " 14 GB", | 20260507 16:08:48.609 | INFO | "pinned" : " 29 GB" | 20260507 16:08:48.609 | INFO | } | 20260507 16:08:48.610 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:08:48.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:48.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | ANALYSIS : 56.0%, 3.0, 86.9ms, 0 | 20260507 16:08:48.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | PRE_HQ : 39.8%, 1.0, 20.7ms, 0 | 20260507 16:08:48.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:48.610 | INFO | Uploaded 36GiB to GPU in 1.59716s (22.54GiB/s) | 20260507 16:08:48.610 | INFO | Downloaded 34GiB to GPU in 1.54657s (21.9841GiB/s) | 20260507 16:08:53.748 | INFO | Analyzing chunk frames = [817152,817664) | 20260507 16:08:53.748 | INFO | MemInfo: { | 20260507 16:08:53.748 | INFO | "gpu" : " 17 GB", | 20260507 16:08:53.748 | INFO | "huge" : "", | 20260507 16:08:53.748 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:53.748 | INFO | "malloc" : " 14 GB", | 20260507 16:08:53.748 | INFO | "pinned" : " 29 GB" | 20260507 16:08:53.748 | INFO | } | 20260507 16:08:53.748 | INFO | Perf report for the last 5.13892 seconds: | 20260507 16:08:53.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:53.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.749 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 16:08:53.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.749 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 16:08:53.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:53.749 | INFO | Uploaded 36GiB to GPU in 1.59335s (22.5939GiB/s) | 20260507 16:08:53.749 | INFO | Downloaded 34GiB to GPU in 1.54203s (22.0488GiB/s) | 20260507 16:08:58.888 | INFO | Analyzing chunk frames = [817664,818176) | 20260507 16:08:58.888 | INFO | MemInfo: { | 20260507 16:08:58.888 | INFO | "gpu" : " 17 GB", | 20260507 16:08:58.888 | INFO | "huge" : "", | 20260507 16:08:58.888 | INFO | "huge_shared" : " 29 GB", | 20260507 16:08:58.888 | INFO | "malloc" : " 14 GB", | 20260507 16:08:58.888 | INFO | "pinned" : " 29 GB" | 20260507 16:08:58.888 | INFO | } | 20260507 16:08:58.888 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:08:58.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:08:58.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:08:58.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:08:58.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:08:58.888 | INFO | Uploaded 36GiB to GPU in 1.58728s (22.6803GiB/s) | 20260507 16:08:58.888 | INFO | Downloaded 34GiB to GPU in 1.53859s (22.0982GiB/s) | 20260507 16:09:04.027 | INFO | Analyzing chunk frames = [818176,818688) | 20260507 16:09:04.027 | INFO | MemInfo: { | 20260507 16:09:04.027 | INFO | "gpu" : " 17 GB", | 20260507 16:09:04.027 | INFO | "huge" : "", | 20260507 16:09:04.027 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:04.027 | INFO | "malloc" : " 14 GB", | 20260507 16:09:04.027 | INFO | "pinned" : " 29 GB" | 20260507 16:09:04.027 | INFO | } | 20260507 16:09:04.027 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:09:04.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:04.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.027 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:09:04.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.027 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:09:04.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:04.028 | INFO | Uploaded 36GiB to GPU in 1.58885s (22.6579GiB/s) | 20260507 16:09:04.028 | INFO | Downloaded 34GiB to GPU in 1.53813s (22.1048GiB/s) | 20260507 16:09:09.166 | INFO | Analyzing chunk frames = [818688,819200) | 20260507 16:09:09.166 | INFO | MemInfo: { | 20260507 16:09:09.166 | INFO | "gpu" : " 17 GB", | 20260507 16:09:09.166 | INFO | "huge" : "", | 20260507 16:09:09.166 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:09.167 | INFO | "malloc" : " 14 GB", | 20260507 16:09:09.167 | INFO | "pinned" : " 29 GB" | 20260507 16:09:09.167 | INFO | } | 20260507 16:09:09.167 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:09:09.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:09.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:09:09.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:09:09.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:09.167 | INFO | Uploaded 36GiB to GPU in 1.58923s (22.6525GiB/s) | 20260507 16:09:09.167 | INFO | Downloaded 34GiB to GPU in 1.53848s (22.0997GiB/s) | 20260507 16:09:09.167 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:09:09.167 | INFO | Total mem usage: 16.4 GiB | 20260507 16:09:09.167 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:09:09.167 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.69%) | 20260507 16:09:09.167 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:09:09.167 | INFO | High Water Breakdown: | 20260507 16:09:09.167 | INFO | 59.7% : 9.09 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:09:09.167 | INFO | 40.2% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:09:09.167 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:09:09.167 | INFO | ------------------------------------------ | 20260507 16:09:09.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:09:09.167 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:09:09.167 | INFO | Total mem usage: 26.5 GiB | 20260507 16:09:09.167 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:09:09.167 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:09:09.167 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:09:09.167 | INFO | High Water Breakdown: | 20260507 16:09:09.167 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:09:09.167 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:09:09.167 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:09:09.167 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:09:09.167 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:09:09.167 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:09:09.167 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:09:09.167 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:09:09.167 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:09:09.167 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:09:09.167 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:09:09.167 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:09:09.167 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:09:09.167 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:09:09.167 | INFO | ------------------------------------------ | 20260507 16:09:09.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:09:09.167 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:09:09.167 | INFO | Total mem usage: 27.8 GiB | 20260507 16:09:09.167 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:09:09.167 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:09:09.167 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:09:09.167 | INFO | High Water Breakdown: | 20260507 16:09:09.167 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:09:09.167 | INFO | ------------------------------------------ | 20260507 16:09:09.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:09:09.167 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:09:09.167 | INFO | Total mem usage: 15.7 GiB | 20260507 16:09:09.167 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:09:09.167 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:09:09.167 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:09:09.167 | INFO | High Water Breakdown: | 20260507 16:09:09.167 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:09:09.167 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:09:09.167 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:09:09.167 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:09:09.167 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:09:09.167 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:09:09.167 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:09:09.167 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:09:09.167 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:09:09.167 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:09:09.167 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:09:09.167 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:09:09.167 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:09:09.167 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:09:09.167 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:09:09.167 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:09:09.167 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:09:09.167 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:09:09.167 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:09:09.167 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:09:09.167 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:09:09.167 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:09:09.167 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:09:09.167 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:09:09.167 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:09:09.167 | INFO | ------------------------------------------ | 20260507 16:09:09.167 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:09:09.167 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:09:09.167 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:09:09.167 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:09:09.167 | INFO | PerfReport: Full Execution -- 2.98e+01 / 7.02e+01, 1.28e+01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 0.30%, AggregateStats -- 9.04e-02 / 9.37e-02, 7.48e-03 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 11.09%, Baseline -- 3.31e+00 / 3.33e+00, 2.67e-02 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 3.09%, Histogram -- 9.21e-01 / 1.06e+00, 5.43e-02 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 16.16%, DME -- 3.86e+01 / 4.56e+01, 3.01e+00 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.72e+01 / 1.71e+01, 4.24e-02 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.13e+00 / 1.96e+00, 1.44e-01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 3.40%, Metrics -- 1.01e+00 / 7.77e-01, 1.66e-01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.75e-01 / 3.00e-01, 5.48e-02 / 0.00e+00 | 20260507 16:09:09.167 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:09:09.167 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:09:09.167 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:09:09.167 | INFO | PerfReport: Full Execution -- 8.93e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 18.63%, Upload -- 1.66e+01 / 0.00e+00, 4.71e-01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 18.23%, Download -- 1.63e+01 / 0.00e+00, 7.24e-01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 33.40%, Compute -- 2.98e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 0.58%, LockUpload -- 5.16e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 0.00%, LockDownload -- 5.96e-04 / 0.00e+00, 5.11e-04 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 29.14%, LockCompute -- 2.60e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 16:09:09.167 | INFO | PerfReport: 0.02%, Misc -- 1.80e-02 / 0.00e+00, 3.86e-02 / 0.00e+00 | 20260507 16:09:09.263 | INFO | Starting a round of Baz IO | 20260507 16:09:12.230 | INFO | Populating RTMetrics report | 20260507 16:09:12.230 | INFO | RTMetrics report populated | 20260507 16:09:12.230 | INFO | Opening tmp RTMetrics json file | 20260507 16:09:12.272 | INFO | Serializing RTMetrics to json | 20260507 16:09:12.302 | INFO | Serialization done, swapping tmp file now... | 20260507 16:09:12.318 | INFO | RTMetrics done | 20260507 16:09:12.421 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.157 seconds | 20260507 16:09:12.421 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.04% / 0.03% / 48.93% | 20260507 16:09:12.421 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.83908% / 3.97364e-05% / 3.17891e-05% | 20260507 16:09:12.421 | INFO | Baz IO overall: | 20260507 16:09:12.421 | INFO | BytesWritten : 346798096688 | 20260507 16:09:12.421 | INFO | NumEvents : 167624558281 | 20260507 16:09:14.306 | INFO | Analyzing chunk frames = [819200,819712) | 20260507 16:09:14.306 | INFO | MemInfo: { | 20260507 16:09:14.306 | INFO | "gpu" : " 17 GB", | 20260507 16:09:14.306 | INFO | "huge" : "", | 20260507 16:09:14.306 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:14.306 | INFO | "malloc" : " 14 GB", | 20260507 16:09:14.306 | INFO | "pinned" : " 29 GB" | 20260507 16:09:14.306 | INFO | } | 20260507 16:09:14.306 | INFO | Perf report for the last 5.13962 seconds: | 20260507 16:09:14.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:14.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:14.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:14.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:14.306 | INFO | ANALYSIS : 57.7%, 2.8, 84.5ms, 0 | 20260507 16:09:14.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:14.307 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 16:09:14.307 | INFO | PRE_HQ : 56.8%, 2.1, 63.0ms, 0 | 20260507 16:09:14.307 | INFO | BAZWRITER : 61.0%, 4.7, 148.8ms, 0 | 20260507 16:09:14.307 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:09:14.307 | INFO | Uploaded 36GiB to GPU in 1.65448s (21.7591GiB/s) | 20260507 16:09:14.307 | INFO | Downloaded 38GiB to GPU in 1.73476s (21.905GiB/s) | 20260507 16:09:19.445 | INFO | Analyzing chunk frames = [819712,820224) | 20260507 16:09:19.445 | INFO | MemInfo: { | 20260507 16:09:19.445 | INFO | "gpu" : " 17 GB", | 20260507 16:09:19.445 | INFO | "huge" : "", | 20260507 16:09:19.445 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:19.445 | INFO | "malloc" : " 14 GB", | 20260507 16:09:19.445 | INFO | "pinned" : " 29 GB" | 20260507 16:09:19.445 | INFO | } | 20260507 16:09:19.446 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:09:19.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:19.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 16:09:19.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | PRE_HQ : 39.3%, 1.0, 20.6ms, 0 | 20260507 16:09:19.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:19.446 | INFO | Uploaded 36GiB to GPU in 1.59717s (22.5399GiB/s) | 20260507 16:09:19.446 | INFO | Downloaded 34GiB to GPU in 1.53735s (22.116GiB/s) | 20260507 16:09:24.585 | INFO | Analyzing chunk frames = [820224,820736) | 20260507 16:09:24.585 | INFO | MemInfo: { | 20260507 16:09:24.585 | INFO | "gpu" : " 17 GB", | 20260507 16:09:24.585 | INFO | "huge" : "", | 20260507 16:09:24.585 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:24.585 | INFO | "malloc" : " 14 GB", | 20260507 16:09:24.585 | INFO | "pinned" : " 29 GB" | 20260507 16:09:24.585 | INFO | } | 20260507 16:09:24.585 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:09:24.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:24.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.585 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:09:24.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.585 | INFO | PRE_HQ : 39.0%, 1.0, 20.3ms, 0 | 20260507 16:09:24.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:24.586 | INFO | Uploaded 36GiB to GPU in 1.61701s (22.2633GiB/s) | 20260507 16:09:24.586 | INFO | Downloaded 34GiB to GPU in 1.53705s (22.1203GiB/s) | 20260507 16:09:29.724 | INFO | Analyzing chunk frames = [820736,821248) | 20260507 16:09:29.724 | INFO | MemInfo: { | 20260507 16:09:29.724 | INFO | "gpu" : " 17 GB", | 20260507 16:09:29.724 | INFO | "huge" : "", | 20260507 16:09:29.724 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:29.724 | INFO | "malloc" : " 14 GB", | 20260507 16:09:29.724 | INFO | "pinned" : " 29 GB" | 20260507 16:09:29.724 | INFO | } | 20260507 16:09:29.724 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:09:29.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:29.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.724 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:09:29.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.725 | INFO | PRE_HQ : 39.1%, 1.0, 20.3ms, 0 | 20260507 16:09:29.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:29.725 | INFO | Uploaded 36GiB to GPU in 1.61675s (22.2669GiB/s) | 20260507 16:09:29.725 | INFO | Downloaded 34GiB to GPU in 1.53536s (22.1447GiB/s) | 20260507 16:09:34.863 | INFO | Analyzing chunk frames = [821248,821760) | 20260507 16:09:34.863 | INFO | MemInfo: { | 20260507 16:09:34.863 | INFO | "gpu" : " 17 GB", | 20260507 16:09:34.863 | INFO | "huge" : "", | 20260507 16:09:34.863 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:34.863 | INFO | "malloc" : " 14 GB", | 20260507 16:09:34.863 | INFO | "pinned" : " 29 GB" | 20260507 16:09:34.863 | INFO | } | 20260507 16:09:34.863 | INFO | Perf report for the last 5.13886 seconds: | 20260507 16:09:34.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:34.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 16:09:34.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260507 16:09:34.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:34.863 | INFO | Uploaded 36GiB to GPU in 1.61254s (22.3251GiB/s) | 20260507 16:09:34.864 | INFO | Downloaded 34GiB to GPU in 1.53107s (22.2067GiB/s) | 20260507 16:09:40.002 | INFO | Analyzing chunk frames = [821760,822272) | 20260507 16:09:40.002 | INFO | MemInfo: { | 20260507 16:09:40.002 | INFO | "gpu" : " 17 GB", | 20260507 16:09:40.002 | INFO | "huge" : "", | 20260507 16:09:40.002 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:40.003 | INFO | "malloc" : " 14 GB", | 20260507 16:09:40.003 | INFO | "pinned" : " 29 GB" | 20260507 16:09:40.003 | INFO | } | 20260507 16:09:40.003 | INFO | Perf report for the last 5.13946 seconds: | 20260507 16:09:40.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:40.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:09:40.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:09:40.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:40.003 | INFO | Uploaded 36GiB to GPU in 1.60874s (22.3778GiB/s) | 20260507 16:09:40.003 | INFO | Downloaded 34GiB to GPU in 1.52886s (22.2387GiB/s) | 20260507 16:09:45.142 | INFO | Analyzing chunk frames = [822272,822784) | 20260507 16:09:45.142 | INFO | MemInfo: { | 20260507 16:09:45.142 | INFO | "gpu" : " 17 GB", | 20260507 16:09:45.142 | INFO | "huge" : "", | 20260507 16:09:45.142 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:45.142 | INFO | "malloc" : " 14 GB", | 20260507 16:09:45.142 | INFO | "pinned" : " 29 GB" | 20260507 16:09:45.142 | INFO | } | 20260507 16:09:45.142 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:09:45.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:45.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.142 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:09:45.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.142 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:09:45.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:45.143 | INFO | Uploaded 36GiB to GPU in 1.6082s (22.3853GiB/s) | 20260507 16:09:45.143 | INFO | Downloaded 34GiB to GPU in 1.5288s (22.2397GiB/s) | 20260507 16:09:50.281 | INFO | Analyzing chunk frames = [822784,823296) | 20260507 16:09:50.281 | INFO | MemInfo: { | 20260507 16:09:50.281 | INFO | "gpu" : " 17 GB", | 20260507 16:09:50.281 | INFO | "huge" : "", | 20260507 16:09:50.281 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:50.281 | INFO | "malloc" : " 14 GB", | 20260507 16:09:50.281 | INFO | "pinned" : " 29 GB" | 20260507 16:09:50.281 | INFO | } | 20260507 16:09:50.281 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:09:50.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:50.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.281 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:09:50.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.281 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:09:50.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:50.282 | INFO | Uploaded 36GiB to GPU in 1.60978s (22.3632GiB/s) | 20260507 16:09:50.282 | INFO | Downloaded 34GiB to GPU in 1.52901s (22.2366GiB/s) | 20260507 16:09:50.378 | INFO | Starting a round of Baz IO | 20260507 16:09:53.288 | INFO | Populating RTMetrics report | 20260507 16:09:53.289 | INFO | RTMetrics report populated | 20260507 16:09:53.289 | INFO | Opening tmp RTMetrics json file | 20260507 16:09:53.295 | INFO | Serializing RTMetrics to json | 20260507 16:09:53.301 | INFO | Serialization done, swapping tmp file now... | 20260507 16:09:53.306 | INFO | RTMetrics done | 20260507 16:09:53.596 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.218 seconds | 20260507 16:09:53.596 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.01% / 0.03% / 48.96% | 20260507 16:09:53.596 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.82667% / 9.13938e-05% / 2.78155e-05% | 20260507 16:09:53.596 | INFO | Baz IO overall: | 20260507 16:09:53.596 | INFO | BytesWritten : 348706760063 | 20260507 16:09:53.596 | INFO | NumEvents : 168552887744 | 20260507 16:09:55.420 | INFO | Analyzing chunk frames = [823296,823808) | 20260507 16:09:55.420 | INFO | MemInfo: { | 20260507 16:09:55.420 | INFO | "gpu" : " 17 GB", | 20260507 16:09:55.420 | INFO | "huge" : "", | 20260507 16:09:55.421 | INFO | "huge_shared" : " 29 GB", | 20260507 16:09:55.421 | INFO | "malloc" : " 14 GB", | 20260507 16:09:55.421 | INFO | "pinned" : " 29 GB" | 20260507 16:09:55.421 | INFO | } | 20260507 16:09:55.421 | INFO | Perf report for the last 5.13956 seconds: | 20260507 16:09:55.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:09:55.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:55.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:55.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:55.421 | INFO | ANALYSIS : 57.6%, 2.8, 84.1ms, 0 | 20260507 16:09:55.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:09:55.421 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 16:09:55.421 | INFO | PRE_HQ : 56.6%, 2.2, 63.7ms, 0 | 20260507 16:09:55.421 | INFO | BAZWRITER : 62.2%, 4.7, 149.2ms, 0 | 20260507 16:09:55.421 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 16:09:55.422 | INFO | Uploaded 36GiB to GPU in 1.63933s (21.9602GiB/s) | 20260507 16:09:55.422 | INFO | Downloaded 38GiB to GPU in 1.72027s (22.0895GiB/s) | 20260507 16:10:00.560 | INFO | Analyzing chunk frames = [823808,824320) | 20260507 16:10:00.560 | INFO | MemInfo: { | 20260507 16:10:00.560 | INFO | "gpu" : " 17 GB", | 20260507 16:10:00.560 | INFO | "huge" : "", | 20260507 16:10:00.560 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:00.560 | INFO | "malloc" : " 14 GB", | 20260507 16:10:00.560 | INFO | "pinned" : " 29 GB" | 20260507 16:10:00.560 | INFO | } | 20260507 16:10:00.561 | INFO | Perf report for the last 5.13969 seconds: | 20260507 16:10:00.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:00.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260507 16:10:00.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | PRE_HQ : 38.6%, 1.0, 20.1ms, 0 | 20260507 16:10:00.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:00.561 | INFO | Uploaded 36GiB to GPU in 1.59231s (22.6086GiB/s) | 20260507 16:10:00.561 | INFO | Downloaded 34GiB to GPU in 1.53984s (22.0802GiB/s) | 20260507 16:10:05.699 | INFO | Analyzing chunk frames = [824320,824832) | 20260507 16:10:05.699 | INFO | MemInfo: { | 20260507 16:10:05.699 | INFO | "gpu" : " 17 GB", | 20260507 16:10:05.700 | INFO | "huge" : "", | 20260507 16:10:05.700 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:05.700 | INFO | "malloc" : " 14 GB", | 20260507 16:10:05.700 | INFO | "pinned" : " 29 GB" | 20260507 16:10:05.700 | INFO | } | 20260507 16:10:05.700 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:10:05.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:05.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:10:05.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | PRE_HQ : 39.0%, 1.0, 20.3ms, 0 | 20260507 16:10:05.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:05.700 | INFO | Uploaded 36GiB to GPU in 1.59666s (22.5471GiB/s) | 20260507 16:10:05.700 | INFO | Downloaded 34GiB to GPU in 1.54535s (22.0015GiB/s) | 20260507 16:10:10.838 | INFO | Analyzing chunk frames = [824832,825344) | 20260507 16:10:10.838 | INFO | MemInfo: { | 20260507 16:10:10.838 | INFO | "gpu" : " 17 GB", | 20260507 16:10:10.838 | INFO | "huge" : "", | 20260507 16:10:10.839 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:10.839 | INFO | "malloc" : " 14 GB", | 20260507 16:10:10.839 | INFO | "pinned" : " 29 GB" | 20260507 16:10:10.839 | INFO | } | 20260507 16:10:10.839 | INFO | Perf report for the last 5.13893 seconds: | 20260507 16:10:10.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:10.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 16:10:10.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | PRE_HQ : 40.0%, 1.0, 20.8ms, 0 | 20260507 16:10:10.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:10.839 | INFO | Uploaded 36GiB to GPU in 1.5956s (22.562GiB/s) | 20260507 16:10:10.839 | INFO | Downloaded 34GiB to GPU in 1.54412s (22.0189GiB/s) | 20260507 16:10:15.978 | INFO | Analyzing chunk frames = [825344,825856) | 20260507 16:10:15.978 | INFO | MemInfo: { | 20260507 16:10:15.978 | INFO | "gpu" : " 17 GB", | 20260507 16:10:15.978 | INFO | "huge" : "", | 20260507 16:10:15.978 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:15.978 | INFO | "malloc" : " 14 GB", | 20260507 16:10:15.978 | INFO | "pinned" : " 29 GB" | 20260507 16:10:15.978 | INFO | } | 20260507 16:10:15.978 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:10:15.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:15.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | ANALYSIS : 55.7%, 3.0, 87.0ms, 0 | 20260507 16:10:15.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:10:15.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:15.978 | INFO | Uploaded 36GiB to GPU in 1.58858s (22.6617GiB/s) | 20260507 16:10:15.978 | INFO | Downloaded 34GiB to GPU in 1.5378s (22.1095GiB/s) | 20260507 16:10:21.117 | INFO | Analyzing chunk frames = [825856,826368) | 20260507 16:10:21.117 | INFO | MemInfo: { | 20260507 16:10:21.117 | INFO | "gpu" : " 17 GB", | 20260507 16:10:21.117 | INFO | "huge" : "", | 20260507 16:10:21.117 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:21.117 | INFO | "malloc" : " 14 GB", | 20260507 16:10:21.117 | INFO | "pinned" : " 29 GB" | 20260507 16:10:21.117 | INFO | } | 20260507 16:10:21.117 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:10:21.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:21.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.118 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 16:10:21.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.118 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:10:21.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:21.118 | INFO | Uploaded 36GiB to GPU in 1.5891s (22.6543GiB/s) | 20260507 16:10:21.118 | INFO | Downloaded 34GiB to GPU in 1.53762s (22.1121GiB/s) | 20260507 16:10:26.257 | INFO | Analyzing chunk frames = [826368,826880) | 20260507 16:10:26.257 | INFO | MemInfo: { | 20260507 16:10:26.257 | INFO | "gpu" : " 17 GB", | 20260507 16:10:26.257 | INFO | "huge" : "", | 20260507 16:10:26.257 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:26.257 | INFO | "malloc" : " 14 GB", | 20260507 16:10:26.257 | INFO | "pinned" : " 29 GB" | 20260507 16:10:26.257 | INFO | } | 20260507 16:10:26.257 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:10:26.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:26.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:10:26.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:10:26.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:26.257 | INFO | Uploaded 36GiB to GPU in 1.58933s (22.6511GiB/s) | 20260507 16:10:26.257 | INFO | Downloaded 34GiB to GPU in 1.53821s (22.1036GiB/s) | 20260507 16:10:31.396 | INFO | Analyzing chunk frames = [826880,827392) | 20260507 16:10:31.396 | INFO | MemInfo: { | 20260507 16:10:31.396 | INFO | "gpu" : " 17 GB", | 20260507 16:10:31.396 | INFO | "huge" : "", | 20260507 16:10:31.396 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:31.396 | INFO | "malloc" : " 14 GB", | 20260507 16:10:31.396 | INFO | "pinned" : " 29 GB" | 20260507 16:10:31.396 | INFO | } | 20260507 16:10:31.396 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:10:31.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:31.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:10:31.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 16:10:31.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:31.396 | INFO | Uploaded 36GiB to GPU in 1.58947s (22.6491GiB/s) | 20260507 16:10:31.397 | INFO | Downloaded 34GiB to GPU in 1.53968s (22.0825GiB/s) | 20260507 16:10:31.397 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:10:31.397 | INFO | Total mem usage: 16.4 GiB | 20260507 16:10:31.397 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:10:31.397 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.62%) | 20260507 16:10:31.397 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:10:31.397 | INFO | High Water Breakdown: | 20260507 16:10:31.397 | INFO | 59.7% : 9.1 GiB ( 91.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:10:31.397 | INFO | 40.2% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:10:31.397 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:10:31.397 | INFO | ------------------------------------------ | 20260507 16:10:31.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:10:31.397 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:10:31.397 | INFO | Total mem usage: 26.5 GiB | 20260507 16:10:31.397 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:10:31.397 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 16:10:31.397 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 16:10:31.397 | INFO | High Water Breakdown: | 20260507 16:10:31.397 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:10:31.397 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:10:31.397 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:10:31.397 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:10:31.397 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:10:31.397 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:10:31.397 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:10:31.397 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:10:31.397 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:10:31.397 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:10:31.397 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:10:31.397 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:10:31.397 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:10:31.397 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:10:31.397 | INFO | ------------------------------------------ | 20260507 16:10:31.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:10:31.397 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:10:31.397 | INFO | Total mem usage: 27.8 GiB | 20260507 16:10:31.397 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:10:31.397 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:10:31.397 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:10:31.397 | INFO | High Water Breakdown: | 20260507 16:10:31.397 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:10:31.397 | INFO | ------------------------------------------ | 20260507 16:10:31.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:10:31.397 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:10:31.397 | INFO | Total mem usage: 15.7 GiB | 20260507 16:10:31.397 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:10:31.397 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:10:31.397 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:10:31.397 | INFO | High Water Breakdown: | 20260507 16:10:31.397 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:10:31.397 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:10:31.397 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:10:31.397 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:10:31.397 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:10:31.397 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:10:31.397 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:10:31.397 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:10:31.397 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:10:31.397 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:10:31.397 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:10:31.397 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:10:31.397 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:10:31.397 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:10:31.397 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:10:31.397 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:10:31.397 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:10:31.398 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:10:31.398 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:10:31.398 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:10:31.398 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:10:31.398 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:10:31.398 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:10:31.398 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:10:31.398 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:10:31.398 | INFO | ------------------------------------------ | 20260507 16:10:31.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:10:31.398 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:10:31.398 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:10:31.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:10:31.398 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 0.30%, AggregateStats -- 8.99e-02 / -nan, 6.43e-03 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 11.08%, Baseline -- 3.30e+00 / -nan, 2.24e-02 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 3.09%, Histogram -- 9.20e-01 / -nan, 5.34e-02 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 2.95e+00 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.12e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 3.39%, Metrics -- 1.01e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.74e-01 / -nan, 5.47e-02 / 0.00e+00 | 20260507 16:10:31.398 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:10:31.398 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:10:31.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:10:31.398 | INFO | PerfReport: Full Execution -- 8.90e+01 / -nan, 1.84e+01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 18.80%, Upload -- 1.67e+01 / 0.00e+00, 4.72e-01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 18.26%, Download -- 1.63e+01 / 0.00e+00, 7.77e-01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 33.50%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 0.60%, LockUpload -- 5.31e-01 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 0.00%, LockDownload -- 5.60e-04 / 0.00e+00, 4.17e-04 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 28.82%, LockCompute -- 2.56e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260507 16:10:31.398 | INFO | PerfReport: 0.02%, Misc -- 1.63e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 16:10:31.493 | INFO | Starting a round of Baz IO | 20260507 16:10:34.434 | INFO | Populating RTMetrics report | 20260507 16:10:34.434 | INFO | RTMetrics report populated | 20260507 16:10:34.434 | INFO | Opening tmp RTMetrics json file | 20260507 16:10:34.499 | INFO | Serializing RTMetrics to json | 20260507 16:10:34.503 | INFO | Serialization done, swapping tmp file now... | 20260507 16:10:34.507 | INFO | RTMetrics done | 20260507 16:10:34.623 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.129 seconds | 20260507 16:10:34.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.98% / 0.03% / 48.99% | 20260507 16:10:34.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.80877% / 5.16574e-05% / 2.38419e-05% | 20260507 16:10:34.623 | INFO | Baz IO overall: | 20260507 16:10:34.623 | INFO | BytesWritten : 350615665206 | 20260507 16:10:34.623 | INFO | NumEvents : 169481452772 | 20260507 16:10:36.535 | INFO | Analyzing chunk frames = [827392,827904) | 20260507 16:10:36.535 | INFO | MemInfo: { | 20260507 16:10:36.535 | INFO | "gpu" : " 17 GB", | 20260507 16:10:36.535 | INFO | "huge" : "", | 20260507 16:10:36.535 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:36.535 | INFO | "malloc" : " 14 GB", | 20260507 16:10:36.535 | INFO | "pinned" : " 29 GB" | 20260507 16:10:36.535 | INFO | } | 20260507 16:10:36.536 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:10:36.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:36.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:36.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:36.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:36.536 | INFO | ANALYSIS : 58.2%, 2.7, 83.7ms, 0 | 20260507 16:10:36.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:36.536 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260507 16:10:36.536 | INFO | PRE_HQ : 57.4%, 2.2, 64.8ms, 0 | 20260507 16:10:36.536 | INFO | BAZWRITER : 60.5%, 4.7, 147.1ms, 0 | 20260507 16:10:36.536 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 16:10:36.536 | INFO | Uploaded 36GiB to GPU in 1.65419s (21.7629GiB/s) | 20260507 16:10:36.537 | INFO | Downloaded 38GiB to GPU in 1.73539s (21.8971GiB/s) | 20260507 16:10:41.675 | INFO | Analyzing chunk frames = [827904,828416) | 20260507 16:10:41.675 | INFO | MemInfo: { | 20260507 16:10:41.675 | INFO | "gpu" : " 17 GB", | 20260507 16:10:41.675 | INFO | "huge" : "", | 20260507 16:10:41.675 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:41.675 | INFO | "malloc" : " 14 GB", | 20260507 16:10:41.675 | INFO | "pinned" : " 29 GB" | 20260507 16:10:41.675 | INFO | } | 20260507 16:10:41.675 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:10:41.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:41.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.675 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 16:10:41.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.675 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 16:10:41.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:41.676 | INFO | Uploaded 36GiB to GPU in 1.59635s (22.5515GiB/s) | 20260507 16:10:41.676 | INFO | Downloaded 34GiB to GPU in 1.54021s (22.0749GiB/s) | 20260507 16:10:46.814 | INFO | Analyzing chunk frames = [828416,828928) | 20260507 16:10:46.814 | INFO | MemInfo: { | 20260507 16:10:46.814 | INFO | "gpu" : " 17 GB", | 20260507 16:10:46.814 | INFO | "huge" : "", | 20260507 16:10:46.814 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:46.814 | INFO | "malloc" : " 14 GB", | 20260507 16:10:46.814 | INFO | "pinned" : " 29 GB" | 20260507 16:10:46.814 | INFO | } | 20260507 16:10:46.815 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:10:46.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:46.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:10:46.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | PRE_HQ : 39.2%, 1.0, 20.4ms, 0 | 20260507 16:10:46.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:46.815 | INFO | Uploaded 36GiB to GPU in 1.59669s (22.5466GiB/s) | 20260507 16:10:46.815 | INFO | Downloaded 34GiB to GPU in 1.5457s (21.9966GiB/s) | 20260507 16:10:51.953 | INFO | Analyzing chunk frames = [828928,829440) | 20260507 16:10:51.953 | INFO | MemInfo: { | 20260507 16:10:51.953 | INFO | "gpu" : " 17 GB", | 20260507 16:10:51.954 | INFO | "huge" : "", | 20260507 16:10:51.954 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:51.954 | INFO | "malloc" : " 14 GB", | 20260507 16:10:51.954 | INFO | "pinned" : " 29 GB" | 20260507 16:10:51.954 | INFO | } | 20260507 16:10:51.954 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:10:51.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:51.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 16:10:51.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 16:10:51.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:51.954 | INFO | Uploaded 36GiB to GPU in 1.597s (22.5423GiB/s) | 20260507 16:10:51.954 | INFO | Downloaded 34GiB to GPU in 1.54493s (22.0074GiB/s) | 20260507 16:10:57.093 | INFO | Analyzing chunk frames = [829440,829952) | 20260507 16:10:57.093 | INFO | MemInfo: { | 20260507 16:10:57.093 | INFO | "gpu" : " 17 GB", | 20260507 16:10:57.093 | INFO | "huge" : "", | 20260507 16:10:57.093 | INFO | "huge_shared" : " 29 GB", | 20260507 16:10:57.093 | INFO | "malloc" : " 14 GB", | 20260507 16:10:57.093 | INFO | "pinned" : " 29 GB" | 20260507 16:10:57.093 | INFO | } | 20260507 16:10:57.093 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:10:57.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:10:57.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:10:57.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 16:10:57.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:10:57.093 | INFO | Uploaded 36GiB to GPU in 1.59017s (22.6391GiB/s) | 20260507 16:10:57.094 | INFO | Downloaded 34GiB to GPU in 1.53936s (22.0872GiB/s) | 20260507 16:11:02.232 | INFO | Analyzing chunk frames = [829952,830464) | 20260507 16:11:02.232 | INFO | MemInfo: { | 20260507 16:11:02.232 | INFO | "gpu" : " 17 GB", | 20260507 16:11:02.232 | INFO | "huge" : "", | 20260507 16:11:02.232 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:02.232 | INFO | "malloc" : " 14 GB", | 20260507 16:11:02.232 | INFO | "pinned" : " 29 GB" | 20260507 16:11:02.232 | INFO | } | 20260507 16:11:02.232 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:11:02.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:02.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.232 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:11:02.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.233 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:11:02.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:02.233 | INFO | Uploaded 36GiB to GPU in 1.58923s (22.6525GiB/s) | 20260507 16:11:02.233 | INFO | Downloaded 34GiB to GPU in 1.5378s (22.1095GiB/s) | 20260507 16:11:07.371 | INFO | Analyzing chunk frames = [830464,830976) | 20260507 16:11:07.371 | INFO | MemInfo: { | 20260507 16:11:07.371 | INFO | "gpu" : " 17 GB", | 20260507 16:11:07.371 | INFO | "huge" : "", | 20260507 16:11:07.371 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:07.371 | INFO | "malloc" : " 14 GB", | 20260507 16:11:07.371 | INFO | "pinned" : " 29 GB" | 20260507 16:11:07.371 | INFO | } | 20260507 16:11:07.372 | INFO | Perf report for the last 5.13915 seconds: | 20260507 16:11:07.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:07.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:11:07.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:11:07.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:07.372 | INFO | Uploaded 36GiB to GPU in 1.58879s (22.6587GiB/s) | 20260507 16:11:07.372 | INFO | Downloaded 34GiB to GPU in 1.5388s (22.0952GiB/s) | 20260507 16:11:12.511 | INFO | Analyzing chunk frames = [830976,831488) | 20260507 16:11:12.511 | INFO | MemInfo: { | 20260507 16:11:12.511 | INFO | "gpu" : " 17 GB", | 20260507 16:11:12.511 | INFO | "huge" : "", | 20260507 16:11:12.511 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:12.511 | INFO | "malloc" : " 14 GB", | 20260507 16:11:12.511 | INFO | "pinned" : " 29 GB" | 20260507 16:11:12.511 | INFO | } | 20260507 16:11:12.511 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:11:12.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:12.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:11:12.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:11:12.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:12.511 | INFO | Uploaded 36GiB to GPU in 1.5886s (22.6614GiB/s) | 20260507 16:11:12.511 | INFO | Downloaded 34GiB to GPU in 1.53784s (22.1089GiB/s) | 20260507 16:11:12.608 | INFO | Starting a round of Baz IO | 20260507 16:11:15.511 | INFO | Populating RTMetrics report | 20260507 16:11:15.512 | INFO | RTMetrics report populated | 20260507 16:11:15.512 | INFO | Opening tmp RTMetrics json file | 20260507 16:11:15.529 | INFO | Serializing RTMetrics to json | 20260507 16:11:15.534 | INFO | Serialization done, swapping tmp file now... | 20260507 16:11:15.539 | INFO | RTMetrics done | 20260507 16:11:15.680 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.071 seconds | 20260507 16:11:15.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.95% / 0.03% / 49.02% | 20260507 16:11:15.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.79868% / 5.5631e-05% / 3.57628e-05% | 20260507 16:11:15.680 | INFO | Baz IO overall: | 20260507 16:11:15.680 | INFO | BytesWritten : 352524770657 | 20260507 16:11:15.680 | INFO | NumEvents : 170410199543 | 20260507 16:11:17.650 | INFO | Analyzing chunk frames = [831488,832000) | 20260507 16:11:17.650 | INFO | MemInfo: { | 20260507 16:11:17.650 | INFO | "gpu" : " 17 GB", | 20260507 16:11:17.650 | INFO | "huge" : "", | 20260507 16:11:17.650 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:17.650 | INFO | "malloc" : " 14 GB", | 20260507 16:11:17.651 | INFO | "pinned" : " 29 GB" | 20260507 16:11:17.651 | INFO | } | 20260507 16:11:17.651 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:11:17.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:17.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:17.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:17.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:17.651 | INFO | ANALYSIS : 57.2%, 2.8, 84.7ms, 0 | 20260507 16:11:17.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:17.651 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 16:11:17.651 | INFO | PRE_HQ : 56.7%, 2.2, 63.5ms, 0 | 20260507 16:11:17.651 | INFO | BAZWRITER : 59.3%, 4.6, 141.8ms, 0 | 20260507 16:11:17.651 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 16:11:17.651 | INFO | Uploaded 36GiB to GPU in 1.65222s (21.7889GiB/s) | 20260507 16:11:17.651 | INFO | Downloaded 38GiB to GPU in 1.73341s (21.9221GiB/s) | 20260507 16:11:22.790 | INFO | Analyzing chunk frames = [832000,832512) | 20260507 16:11:22.790 | INFO | MemInfo: { | 20260507 16:11:22.790 | INFO | "gpu" : " 17 GB", | 20260507 16:11:22.790 | INFO | "huge" : "", | 20260507 16:11:22.790 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:22.790 | INFO | "malloc" : " 14 GB", | 20260507 16:11:22.790 | INFO | "pinned" : " 29 GB" | 20260507 16:11:22.790 | INFO | } | 20260507 16:11:22.790 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:11:22.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:22.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.790 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260507 16:11:22.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.790 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 16:11:22.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:22.791 | INFO | Uploaded 36GiB to GPU in 1.61259s (22.3243GiB/s) | 20260507 16:11:22.791 | INFO | Downloaded 34GiB to GPU in 1.53017s (22.2198GiB/s) | 20260507 16:11:27.929 | INFO | Analyzing chunk frames = [832512,833024) | 20260507 16:11:27.929 | INFO | MemInfo: { | 20260507 16:11:27.929 | INFO | "gpu" : " 17 GB", | 20260507 16:11:27.929 | INFO | "huge" : "", | 20260507 16:11:27.929 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:27.929 | INFO | "malloc" : " 14 GB", | 20260507 16:11:27.929 | INFO | "pinned" : " 29 GB" | 20260507 16:11:27.929 | INFO | } | 20260507 16:11:27.929 | INFO | Perf report for the last 5.139 seconds: | 20260507 16:11:27.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:27.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.929 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260507 16:11:27.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.929 | INFO | PRE_HQ : 39.2%, 1.0, 20.4ms, 0 | 20260507 16:11:27.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:27.930 | INFO | Uploaded 36GiB to GPU in 1.61582s (22.2797GiB/s) | 20260507 16:11:27.930 | INFO | Downloaded 34GiB to GPU in 1.53634s (22.1305GiB/s) | 20260507 16:11:33.068 | INFO | Analyzing chunk frames = [833024,833536) | 20260507 16:11:33.068 | INFO | MemInfo: { | 20260507 16:11:33.068 | INFO | "gpu" : " 17 GB", | 20260507 16:11:33.068 | INFO | "huge" : "", | 20260507 16:11:33.068 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:33.068 | INFO | "malloc" : " 14 GB", | 20260507 16:11:33.068 | INFO | "pinned" : " 29 GB" | 20260507 16:11:33.068 | INFO | } | 20260507 16:11:33.068 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:11:33.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:33.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.068 | INFO | ANALYSIS : 56.1%, 3.0, 86.9ms, 0 | 20260507 16:11:33.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.069 | INFO | PRE_HQ : 39.9%, 1.0, 20.7ms, 0 | 20260507 16:11:33.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:33.069 | INFO | Uploaded 36GiB to GPU in 1.61615s (22.2752GiB/s) | 20260507 16:11:33.069 | INFO | Downloaded 34GiB to GPU in 1.53544s (22.1435GiB/s) | 20260507 16:11:38.207 | INFO | Analyzing chunk frames = [833536,834048) | 20260507 16:11:38.207 | INFO | MemInfo: { | 20260507 16:11:38.207 | INFO | "gpu" : " 17 GB", | 20260507 16:11:38.207 | INFO | "huge" : "", | 20260507 16:11:38.207 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:38.207 | INFO | "malloc" : " 14 GB", | 20260507 16:11:38.207 | INFO | "pinned" : " 29 GB" | 20260507 16:11:38.207 | INFO | } | 20260507 16:11:38.208 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:11:38.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:38.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:11:38.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:11:38.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:38.208 | INFO | Uploaded 36GiB to GPU in 1.608s (22.3881GiB/s) | 20260507 16:11:38.208 | INFO | Downloaded 34GiB to GPU in 1.52762s (22.2569GiB/s) | 20260507 16:11:43.347 | INFO | Analyzing chunk frames = [834048,834560) | 20260507 16:11:43.347 | INFO | MemInfo: { | 20260507 16:11:43.347 | INFO | "gpu" : " 17 GB", | 20260507 16:11:43.347 | INFO | "huge" : "", | 20260507 16:11:43.347 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:43.347 | INFO | "malloc" : " 14 GB", | 20260507 16:11:43.347 | INFO | "pinned" : " 29 GB" | 20260507 16:11:43.347 | INFO | } | 20260507 16:11:43.347 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:11:43.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:43.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:11:43.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:11:43.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:43.347 | INFO | Uploaded 36GiB to GPU in 1.60891s (22.3753GiB/s) | 20260507 16:11:43.347 | INFO | Downloaded 34GiB to GPU in 1.52831s (22.2467GiB/s) | 20260507 16:11:48.486 | INFO | Analyzing chunk frames = [834560,835072) | 20260507 16:11:48.486 | INFO | MemInfo: { | 20260507 16:11:48.486 | INFO | "gpu" : " 17 GB", | 20260507 16:11:48.486 | INFO | "huge" : "", | 20260507 16:11:48.486 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:48.486 | INFO | "malloc" : " 14 GB", | 20260507 16:11:48.486 | INFO | "pinned" : " 29 GB" | 20260507 16:11:48.486 | INFO | } | 20260507 16:11:48.486 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:11:48.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:48.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.486 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:11:48.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.486 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:11:48.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:48.487 | INFO | Uploaded 36GiB to GPU in 1.60921s (22.3712GiB/s) | 20260507 16:11:48.487 | INFO | Downloaded 34GiB to GPU in 1.52975s (22.2258GiB/s) | 20260507 16:11:53.625 | INFO | Analyzing chunk frames = [835072,835584) | 20260507 16:11:53.625 | INFO | MemInfo: { | 20260507 16:11:53.625 | INFO | "gpu" : " 17 GB", | 20260507 16:11:53.625 | INFO | "huge" : "", | 20260507 16:11:53.625 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:53.625 | INFO | "malloc" : " 14 GB", | 20260507 16:11:53.625 | INFO | "pinned" : " 29 GB" | 20260507 16:11:53.625 | INFO | } | 20260507 16:11:53.626 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:11:53.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:53.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:11:53.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:11:53.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:53.626 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:11:53.626 | INFO | Total mem usage: 16.4 GiB | 20260507 16:11:53.626 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:11:53.626 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.41%) | 20260507 16:11:53.626 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:11:53.626 | INFO | High Water Breakdown: | 20260507 16:11:53.626 | INFO | 59.6% : 9.01 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:11:53.626 | INFO | 40.3% : 6.1 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:11:53.626 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:11:53.626 | INFO | ------------------------------------------ | 20260507 16:11:53.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:11:53.626 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:11:53.626 | INFO | Total mem usage: 26.5 GiB | 20260507 16:11:53.626 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:11:53.626 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 16:11:53.626 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 16:11:53.626 | INFO | High Water Breakdown: | 20260507 16:11:53.626 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:11:53.626 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:11:53.626 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:11:53.626 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:11:53.626 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:11:53.626 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:11:53.626 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:11:53.626 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:11:53.626 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:11:53.626 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:11:53.626 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:11:53.626 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:11:53.626 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:11:53.626 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:11:53.626 | INFO | ------------------------------------------ | 20260507 16:11:53.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:11:53.626 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:11:53.626 | INFO | Total mem usage: 27.8 GiB | 20260507 16:11:53.626 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:11:53.626 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:11:53.626 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:11:53.626 | INFO | High Water Breakdown: | 20260507 16:11:53.626 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:11:53.626 | INFO | ------------------------------------------ | 20260507 16:11:53.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:11:53.626 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:11:53.626 | INFO | Total mem usage: 15.7 GiB | 20260507 16:11:53.626 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:11:53.626 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:11:53.626 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:11:53.626 | INFO | High Water Breakdown: | 20260507 16:11:53.626 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:11:53.626 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:11:53.626 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:11:53.626 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:11:53.626 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:11:53.626 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:11:53.626 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:11:53.626 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:11:53.626 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:11:53.626 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:11:53.626 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:11:53.626 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:11:53.626 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:11:53.626 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:11:53.626 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:11:53.626 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:11:53.626 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:11:53.626 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:11:53.626 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:11:53.626 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:11:53.626 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:11:53.626 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:11:53.626 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:11:53.626 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:11:53.626 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:11:53.626 | INFO | ------------------------------------------ | 20260507 16:11:53.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:11:53.626 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:11:53.626 | INFO | Uploaded 36GiB to GPU in 1.6095s (22.3672GiB/s) | 20260507 16:11:53.626 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:11:53.626 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:11:53.626 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 16:11:53.626 | INFO | PerfReport: 0.30%, AggregateStats -- 8.97e-02 / -nan, 6.51e-03 / 0.00e+00 | 20260507 16:11:53.626 | INFO | Downloaded 34GiB to GPU in 1.52973s (22.2261GiB/s) | 20260507 16:11:53.626 | INFO | PerfReport: 11.08%, Baseline -- 3.30e+00 / -nan, 2.51e-02 / 0.00e+00 | 20260507 16:11:53.626 | INFO | PerfReport: 3.09%, Histogram -- 9.20e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 16:11:53.626 | INFO | PerfReport: 16.15%, DME -- 3.85e+01 / -nan, 2.96e+00 / 0.00e+00 | 20260507 16:11:53.626 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.72e+01 / -nan, 4.19e-02 / 0.00e+00 | 20260507 16:11:53.626 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.12e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:11:53.626 | INFO | PerfReport: 3.39%, Metrics -- 1.01e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.73e-01 / -nan, 5.55e-02 / 0.00e+00 | 20260507 16:11:53.627 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:11:53.627 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:11:53.627 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:11:53.627 | INFO | PerfReport: Full Execution -- 8.90e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 18.83%, Upload -- 1.68e+01 / 0.00e+00, 4.88e-01 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 18.27%, Download -- 1.63e+01 / 0.00e+00, 7.95e-01 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 33.50%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 0.59%, LockUpload -- 5.24e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 0.00%, LockDownload -- 5.42e-04 / 0.00e+00, 4.02e-04 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 28.79%, LockCompute -- 2.56e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260507 16:11:53.627 | INFO | PerfReport: 0.02%, Misc -- 1.64e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 16:11:53.723 | INFO | Starting a round of Baz IO | 20260507 16:11:56.683 | INFO | Populating RTMetrics report | 20260507 16:11:56.684 | INFO | RTMetrics report populated | 20260507 16:11:56.684 | INFO | Opening tmp RTMetrics json file | 20260507 16:11:56.761 | INFO | Serializing RTMetrics to json | 20260507 16:11:56.770 | INFO | Serialization done, swapping tmp file now... | 20260507 16:11:56.772 | INFO | RTMetrics done | 20260507 16:11:56.832 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.108 seconds | 20260507 16:11:56.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.91% / 0.03% / 49.05% | 20260507 16:11:56.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.80034% / 7.94729e-05% / 2.78155e-05% | 20260507 16:11:56.832 | INFO | Baz IO overall: | 20260507 16:11:56.832 | INFO | BytesWritten : 354435941663 | 20260507 16:11:56.832 | INFO | NumEvents : 171340438705 | 20260507 16:11:58.765 | INFO | Analyzing chunk frames = [835584,836096) | 20260507 16:11:58.765 | INFO | MemInfo: { | 20260507 16:11:58.765 | INFO | "gpu" : " 17 GB", | 20260507 16:11:58.765 | INFO | "huge" : "", | 20260507 16:11:58.765 | INFO | "huge_shared" : " 29 GB", | 20260507 16:11:58.765 | INFO | "malloc" : " 14 GB", | 20260507 16:11:58.765 | INFO | "pinned" : " 29 GB" | 20260507 16:11:58.765 | INFO | } | 20260507 16:11:58.765 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:11:58.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:11:58.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:58.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:58.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:58.765 | INFO | ANALYSIS : 57.8%, 2.9, 86.6ms, 0 | 20260507 16:11:58.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:11:58.765 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260507 16:11:58.765 | INFO | PRE_HQ : 57.1%, 2.2, 65.8ms, 0 | 20260507 16:11:58.765 | INFO | BAZWRITER : 60.1%, 4.5, 137.9ms, 0 | 20260507 16:11:58.765 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 16:11:58.766 | INFO | Uploaded 36GiB to GPU in 1.63856s (21.9705GiB/s) | 20260507 16:11:58.766 | INFO | Downloaded 38GiB to GPU in 1.7175s (22.1252GiB/s) | 20260507 16:12:03.904 | INFO | Analyzing chunk frames = [836096,836608) | 20260507 16:12:03.904 | INFO | MemInfo: { | 20260507 16:12:03.904 | INFO | "gpu" : " 17 GB", | 20260507 16:12:03.904 | INFO | "huge" : "", | 20260507 16:12:03.904 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:03.904 | INFO | "malloc" : " 14 GB", | 20260507 16:12:03.904 | INFO | "pinned" : " 29 GB" | 20260507 16:12:03.904 | INFO | } | 20260507 16:12:03.905 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:12:03.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:03.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 16:12:03.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | PRE_HQ : 38.8%, 1.0, 20.3ms, 0 | 20260507 16:12:03.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:03.905 | INFO | Uploaded 36GiB to GPU in 1.61075s (22.3498GiB/s) | 20260507 16:12:03.905 | INFO | Downloaded 34GiB to GPU in 1.53192s (22.1944GiB/s) | 20260507 16:12:09.043 | INFO | Analyzing chunk frames = [836608,837120) | 20260507 16:12:09.043 | INFO | MemInfo: { | 20260507 16:12:09.043 | INFO | "gpu" : " 17 GB", | 20260507 16:12:09.043 | INFO | "huge" : "", | 20260507 16:12:09.044 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:09.044 | INFO | "malloc" : " 14 GB", | 20260507 16:12:09.044 | INFO | "pinned" : " 29 GB" | 20260507 16:12:09.044 | INFO | } | 20260507 16:12:09.044 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:12:09.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:09.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:12:09.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 16:12:09.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:09.044 | INFO | Uploaded 36GiB to GPU in 1.61608s (22.2761GiB/s) | 20260507 16:12:09.045 | INFO | Downloaded 34GiB to GPU in 1.53582s (22.138GiB/s) | 20260507 16:12:14.183 | INFO | Analyzing chunk frames = [837120,837632) | 20260507 16:12:14.183 | INFO | MemInfo: { | 20260507 16:12:14.183 | INFO | "gpu" : " 17 GB", | 20260507 16:12:14.183 | INFO | "huge" : "", | 20260507 16:12:14.183 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:14.183 | INFO | "malloc" : " 14 GB", | 20260507 16:12:14.183 | INFO | "pinned" : " 29 GB" | 20260507 16:12:14.183 | INFO | } | 20260507 16:12:14.184 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:12:14.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:14.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:12:14.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:12:14.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:14.184 | INFO | Uploaded 36GiB to GPU in 1.61499s (22.2912GiB/s) | 20260507 16:12:14.184 | INFO | Downloaded 34GiB to GPU in 1.53509s (22.1486GiB/s) | 20260507 16:12:19.322 | INFO | Analyzing chunk frames = [837632,838144) | 20260507 16:12:19.322 | INFO | MemInfo: { | 20260507 16:12:19.322 | INFO | "gpu" : " 17 GB", | 20260507 16:12:19.322 | INFO | "huge" : "", | 20260507 16:12:19.322 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:19.322 | INFO | "malloc" : " 14 GB", | 20260507 16:12:19.322 | INFO | "pinned" : " 29 GB" | 20260507 16:12:19.322 | INFO | } | 20260507 16:12:19.322 | INFO | Perf report for the last 5.1387 seconds: | 20260507 16:12:19.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:19.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.322 | INFO | ANALYSIS : 55.8%, 3.0, 87.1ms, 0 | 20260507 16:12:19.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.322 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260507 16:12:19.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:19.323 | INFO | Uploaded 36GiB to GPU in 1.60952s (22.3669GiB/s) | 20260507 16:12:19.323 | INFO | Downloaded 34GiB to GPU in 1.52868s (22.2414GiB/s) | 20260507 16:12:24.461 | INFO | Analyzing chunk frames = [838144,838656) | 20260507 16:12:24.461 | INFO | MemInfo: { | 20260507 16:12:24.461 | INFO | "gpu" : " 17 GB", | 20260507 16:12:24.461 | INFO | "huge" : "", | 20260507 16:12:24.461 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:24.461 | INFO | "malloc" : " 14 GB", | 20260507 16:12:24.462 | INFO | "pinned" : " 29 GB" | 20260507 16:12:24.462 | INFO | } | 20260507 16:12:24.462 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:12:24.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:24.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:12:24.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:12:24.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:24.462 | INFO | Uploaded 36GiB to GPU in 1.60861s (22.3795GiB/s) | 20260507 16:12:24.462 | INFO | Downloaded 34GiB to GPU in 1.52841s (22.2454GiB/s) | 20260507 16:12:29.601 | INFO | Analyzing chunk frames = [838656,839168) | 20260507 16:12:29.601 | INFO | MemInfo: { | 20260507 16:12:29.601 | INFO | "gpu" : " 17 GB", | 20260507 16:12:29.601 | INFO | "huge" : "", | 20260507 16:12:29.601 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:29.601 | INFO | "malloc" : " 14 GB", | 20260507 16:12:29.601 | INFO | "pinned" : " 29 GB" | 20260507 16:12:29.601 | INFO | } | 20260507 16:12:29.601 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:12:29.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:29.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:12:29.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:12:29.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:29.601 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3922GiB/s) | 20260507 16:12:29.602 | INFO | Downloaded 34GiB to GPU in 1.52826s (22.2475GiB/s) | 20260507 16:12:34.740 | INFO | Analyzing chunk frames = [839168,839680) | 20260507 16:12:34.740 | INFO | MemInfo: { | 20260507 16:12:34.740 | INFO | "gpu" : " 17 GB", | 20260507 16:12:34.740 | INFO | "huge" : "", | 20260507 16:12:34.740 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:34.740 | INFO | "malloc" : " 14 GB", | 20260507 16:12:34.740 | INFO | "pinned" : " 29 GB" | 20260507 16:12:34.740 | INFO | } | 20260507 16:12:34.740 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:12:34.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:34.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.740 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:12:34.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.740 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:12:34.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:34.741 | INFO | Uploaded 36GiB to GPU in 1.60903s (22.3737GiB/s) | 20260507 16:12:34.741 | INFO | Downloaded 34GiB to GPU in 1.52972s (22.2263GiB/s) | 20260507 16:12:34.839 | INFO | Starting a round of Baz IO | 20260507 16:12:37.808 | INFO | Populating RTMetrics report | 20260507 16:12:37.809 | INFO | RTMetrics report populated | 20260507 16:12:37.809 | INFO | Opening tmp RTMetrics json file | 20260507 16:12:37.818 | INFO | Serializing RTMetrics to json | 20260507 16:12:37.897 | INFO | Serialization done, swapping tmp file now... | 20260507 16:12:37.902 | INFO | RTMetrics done | 20260507 16:12:38.085 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.245 seconds | 20260507 16:12:38.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.88% / 0.03% / 49.09% | 20260507 16:12:38.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.79468% / 4.76837e-05% / 2.78155e-05% | 20260507 16:12:38.085 | INFO | Baz IO overall: | 20260507 16:12:38.085 | INFO | BytesWritten : 356349524847 | 20260507 16:12:38.085 | INFO | NumEvents : 172271761792 | 20260507 16:12:39.880 | INFO | Analyzing chunk frames = [839680,840192) | 20260507 16:12:39.880 | INFO | MemInfo: { | 20260507 16:12:39.880 | INFO | "gpu" : " 17 GB", | 20260507 16:12:39.880 | INFO | "huge" : "", | 20260507 16:12:39.880 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:39.880 | INFO | "malloc" : " 14 GB", | 20260507 16:12:39.880 | INFO | "pinned" : " 29 GB" | 20260507 16:12:39.880 | INFO | } | 20260507 16:12:39.880 | INFO | Perf report for the last 5.13982 seconds: | 20260507 16:12:39.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:39.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:39.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:39.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:39.880 | INFO | ANALYSIS : 58.1%, 2.8, 84.4ms, 0 | 20260507 16:12:39.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:39.880 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260507 16:12:39.880 | INFO | PRE_HQ : 57.7%, 2.3, 67.6ms, 0 | 20260507 16:12:39.880 | INFO | BAZWRITER : 62.7%, 4.6, 148.6ms, 0 | 20260507 16:12:39.880 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 16:12:39.881 | INFO | Uploaded 36GiB to GPU in 1.6483s (21.8407GiB/s) | 20260507 16:12:39.881 | INFO | Downloaded 38GiB to GPU in 1.71818s (22.1165GiB/s) | 20260507 16:12:45.019 | INFO | Analyzing chunk frames = [840192,840704) | 20260507 16:12:45.019 | INFO | MemInfo: { | 20260507 16:12:45.019 | INFO | "gpu" : " 17 GB", | 20260507 16:12:45.019 | INFO | "huge" : "", | 20260507 16:12:45.019 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:45.019 | INFO | "malloc" : " 14 GB", | 20260507 16:12:45.019 | INFO | "pinned" : " 29 GB" | 20260507 16:12:45.019 | INFO | } | 20260507 16:12:45.019 | INFO | Perf report for the last 5.13916 seconds: | 20260507 16:12:45.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:45.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.019 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 16:12:45.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.020 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 16:12:45.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:45.020 | INFO | Uploaded 36GiB to GPU in 1.61152s (22.3391GiB/s) | 20260507 16:12:45.020 | INFO | Downloaded 34GiB to GPU in 1.52997s (22.2227GiB/s) | 20260507 16:12:50.158 | INFO | Analyzing chunk frames = [840704,841216) | 20260507 16:12:50.158 | INFO | MemInfo: { | 20260507 16:12:50.158 | INFO | "gpu" : " 17 GB", | 20260507 16:12:50.158 | INFO | "huge" : "", | 20260507 16:12:50.158 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:50.159 | INFO | "malloc" : " 14 GB", | 20260507 16:12:50.159 | INFO | "pinned" : " 29 GB" | 20260507 16:12:50.159 | INFO | } | 20260507 16:12:50.159 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:12:50.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:50.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | ANALYSIS : 56.3%, 3.0, 87.6ms, 0 | 20260507 16:12:50.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | PRE_HQ : 39.6%, 1.0, 20.6ms, 0 | 20260507 16:12:50.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:50.159 | INFO | Uploaded 36GiB to GPU in 1.61682s (22.266GiB/s) | 20260507 16:12:50.159 | INFO | Downloaded 34GiB to GPU in 1.53606s (22.1346GiB/s) | 20260507 16:12:55.298 | INFO | Analyzing chunk frames = [841216,841728) | 20260507 16:12:55.298 | INFO | MemInfo: { | 20260507 16:12:55.298 | INFO | "gpu" : " 17 GB", | 20260507 16:12:55.298 | INFO | "huge" : "", | 20260507 16:12:55.298 | INFO | "huge_shared" : " 29 GB", | 20260507 16:12:55.298 | INFO | "malloc" : " 14 GB", | 20260507 16:12:55.298 | INFO | "pinned" : " 29 GB" | 20260507 16:12:55.298 | INFO | } | 20260507 16:12:55.298 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:12:55.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:12:55.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.298 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:12:55.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.298 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 16:12:55.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:12:55.299 | INFO | Uploaded 36GiB to GPU in 1.61698s (22.2637GiB/s) | 20260507 16:12:55.299 | INFO | Downloaded 34GiB to GPU in 1.53628s (22.1314GiB/s) | 20260507 16:13:00.437 | INFO | Analyzing chunk frames = [841728,842240) | 20260507 16:13:00.437 | INFO | MemInfo: { | 20260507 16:13:00.437 | INFO | "gpu" : " 17 GB", | 20260507 16:13:00.437 | INFO | "huge" : "", | 20260507 16:13:00.437 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:00.437 | INFO | "malloc" : " 14 GB", | 20260507 16:13:00.437 | INFO | "pinned" : " 29 GB" | 20260507 16:13:00.437 | INFO | } | 20260507 16:13:00.437 | INFO | Perf report for the last 5.13885 seconds: | 20260507 16:13:00.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:00.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:13:00.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 16:13:00.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:00.437 | INFO | Uploaded 36GiB to GPU in 1.60833s (22.3835GiB/s) | 20260507 16:13:00.437 | INFO | Downloaded 34GiB to GPU in 1.52855s (22.2433GiB/s) | 20260507 16:13:05.576 | INFO | Analyzing chunk frames = [842240,842752) | 20260507 16:13:05.576 | INFO | MemInfo: { | 20260507 16:13:05.576 | INFO | "gpu" : " 17 GB", | 20260507 16:13:05.576 | INFO | "huge" : "", | 20260507 16:13:05.576 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:05.576 | INFO | "malloc" : " 14 GB", | 20260507 16:13:05.576 | INFO | "pinned" : " 29 GB" | 20260507 16:13:05.576 | INFO | } | 20260507 16:13:05.576 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:13:05.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:05.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.576 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:13:05.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.577 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:13:05.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:05.577 | INFO | Uploaded 36GiB to GPU in 1.60837s (22.3829GiB/s) | 20260507 16:13:05.577 | INFO | Downloaded 34GiB to GPU in 1.52838s (22.2458GiB/s) | 20260507 16:13:10.716 | INFO | Analyzing chunk frames = [842752,843264) | 20260507 16:13:10.716 | INFO | MemInfo: { | 20260507 16:13:10.716 | INFO | "gpu" : " 17 GB", | 20260507 16:13:10.716 | INFO | "huge" : "", | 20260507 16:13:10.716 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:10.716 | INFO | "malloc" : " 14 GB", | 20260507 16:13:10.716 | INFO | "pinned" : " 29 GB" | 20260507 16:13:10.716 | INFO | } | 20260507 16:13:10.716 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:13:10.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:10.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:13:10.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:13:10.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:10.716 | INFO | Uploaded 36GiB to GPU in 1.60778s (22.3911GiB/s) | 20260507 16:13:10.716 | INFO | Downloaded 34GiB to GPU in 1.52937s (22.2313GiB/s) | 20260507 16:13:15.855 | INFO | Analyzing chunk frames = [843264,843776) | 20260507 16:13:15.855 | INFO | MemInfo: { | 20260507 16:13:15.855 | INFO | "gpu" : " 17 GB", | 20260507 16:13:15.855 | INFO | "huge" : "", | 20260507 16:13:15.855 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:15.855 | INFO | "malloc" : " 14 GB", | 20260507 16:13:15.855 | INFO | "pinned" : " 29 GB" | 20260507 16:13:15.855 | INFO | } | 20260507 16:13:15.855 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:13:15.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:15.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.855 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:13:15.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.855 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 16:13:15.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:15.856 | INFO | Uploaded 36GiB to GPU in 1.60873s (22.378GiB/s) | 20260507 16:13:15.856 | INFO | Downloaded 34GiB to GPU in 1.52951s (22.2293GiB/s) | 20260507 16:13:15.856 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:13:15.856 | INFO | Total mem usage: 16.4 GiB | 20260507 16:13:15.856 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:13:15.856 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.1%) | 20260507 16:13:15.856 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:13:15.856 | INFO | High Water Breakdown: | 20260507 16:13:15.856 | INFO | 59.3% : 8.84 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:13:15.856 | INFO | 40.6% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:13:15.856 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:13:15.856 | INFO | ------------------------------------------ | 20260507 16:13:15.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:13:15.856 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:13:15.856 | INFO | Total mem usage: 26.5 GiB | 20260507 16:13:15.856 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:13:15.856 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 16:13:15.856 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 16:13:15.856 | INFO | High Water Breakdown: | 20260507 16:13:15.856 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:13:15.856 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:13:15.856 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:13:15.856 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:13:15.856 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:13:15.856 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:13:15.856 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:13:15.856 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:13:15.856 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:13:15.856 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:13:15.856 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:13:15.856 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:13:15.856 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:13:15.856 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:13:15.856 | INFO | ------------------------------------------ | 20260507 16:13:15.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:13:15.856 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:13:15.856 | INFO | Total mem usage: 27.8 GiB | 20260507 16:13:15.856 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:13:15.856 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:13:15.856 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:13:15.856 | INFO | High Water Breakdown: | 20260507 16:13:15.856 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:13:15.856 | INFO | ------------------------------------------ | 20260507 16:13:15.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:13:15.856 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:13:15.856 | INFO | Total mem usage: 15.7 GiB | 20260507 16:13:15.856 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:13:15.856 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:13:15.856 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:13:15.856 | INFO | High Water Breakdown: | 20260507 16:13:15.856 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:13:15.856 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:13:15.856 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:13:15.856 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:13:15.856 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:13:15.856 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:13:15.856 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:13:15.856 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:13:15.856 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:13:15.856 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:13:15.856 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:13:15.856 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:13:15.856 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:13:15.856 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:13:15.856 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:13:15.856 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:13:15.856 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:13:15.856 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:13:15.856 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:13:15.856 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:13:15.856 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:13:15.856 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:13:15.856 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:13:15.856 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:13:15.856 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:13:15.856 | INFO | ------------------------------------------ | 20260507 16:13:15.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:13:15.856 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:13:15.856 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:13:15.856 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:13:15.856 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 0.30%, AggregateStats -- 8.97e-02 / -nan, 1.23e-02 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 11.08%, Baseline -- 3.30e+00 / -nan, 2.58e-02 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 3.09%, Histogram -- 9.20e-01 / -nan, 5.39e-02 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 16.14%, DME -- 3.85e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.12e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 3.39%, Metrics -- 1.01e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.73e-01 / -nan, 5.40e-02 / 0.00e+00 | 20260507 16:13:15.857 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:13:15.857 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:13:15.857 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:13:15.857 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 18.88%, Upload -- 1.68e+01 / 0.00e+00, 4.16e-01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 18.17%, Download -- 1.62e+01 / 0.00e+00, 7.52e-01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 33.45%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 0.61%, LockUpload -- 5.43e-01 / 0.00e+00, 3.67e+00 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 0.00%, LockDownload -- 5.35e-04 / 0.00e+00, 3.68e-04 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 28.88%, LockCompute -- 2.57e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 16:13:15.857 | INFO | PerfReport: 0.02%, Misc -- 1.63e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260507 16:13:15.952 | INFO | Starting a round of Baz IO | 20260507 16:13:18.958 | INFO | Populating RTMetrics report | 20260507 16:13:18.958 | INFO | RTMetrics report populated | 20260507 16:13:18.958 | INFO | Opening tmp RTMetrics json file | 20260507 16:13:18.962 | INFO | Serializing RTMetrics to json | 20260507 16:13:18.964 | INFO | Serialization done, swapping tmp file now... | 20260507 16:13:18.965 | INFO | RTMetrics done | 20260507 16:13:19.219 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.267 seconds | 20260507 16:13:19.219 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.85% / 0.03% / 49.12% | 20260507 16:13:19.219 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.78105% / 5.5631e-05% / 3.17891e-05% | 20260507 16:13:19.219 | INFO | Baz IO overall: | 20260507 16:13:19.219 | INFO | BytesWritten : 358262694678 | 20260507 16:13:19.219 | INFO | NumEvents : 173202547888 | 20260507 16:13:20.994 | INFO | Analyzing chunk frames = [843776,844288) | 20260507 16:13:20.994 | INFO | MemInfo: { | 20260507 16:13:20.994 | INFO | "gpu" : " 17 GB", | 20260507 16:13:20.994 | INFO | "huge" : "", | 20260507 16:13:20.994 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:20.994 | INFO | "malloc" : " 14 GB", | 20260507 16:13:20.994 | INFO | "pinned" : " 29 GB" | 20260507 16:13:20.994 | INFO | } | 20260507 16:13:20.995 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:13:20.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:20.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:20.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:20.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:20.995 | INFO | ANALYSIS : 58.6%, 2.7, 83.7ms, 0 | 20260507 16:13:20.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:20.995 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260507 16:13:20.995 | INFO | PRE_HQ : 58.2%, 2.2, 66.9ms, 0 | 20260507 16:13:20.995 | INFO | BAZWRITER : 63.1%, 4.8, 154.4ms, 0 | 20260507 16:13:20.995 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:13:20.995 | INFO | Uploaded 36GiB to GPU in 1.64591s (21.8724GiB/s) | 20260507 16:13:20.995 | INFO | Downloaded 38GiB to GPU in 1.72328s (22.051GiB/s) | 20260507 16:13:26.134 | INFO | Analyzing chunk frames = [844288,844800) | 20260507 16:13:26.134 | INFO | MemInfo: { | 20260507 16:13:26.134 | INFO | "gpu" : " 17 GB", | 20260507 16:13:26.134 | INFO | "huge" : "", | 20260507 16:13:26.134 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:26.134 | INFO | "malloc" : " 14 GB", | 20260507 16:13:26.134 | INFO | "pinned" : " 29 GB" | 20260507 16:13:26.134 | INFO | } | 20260507 16:13:26.134 | INFO | Perf report for the last 5.13909 seconds: | 20260507 16:13:26.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:26.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260507 16:13:26.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 16:13:26.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:26.134 | INFO | Uploaded 36GiB to GPU in 1.61259s (22.3243GiB/s) | 20260507 16:13:26.134 | INFO | Downloaded 34GiB to GPU in 1.5299s (22.2237GiB/s) | 20260507 16:13:31.273 | INFO | Analyzing chunk frames = [844800,845312) | 20260507 16:13:31.273 | INFO | MemInfo: { | 20260507 16:13:31.273 | INFO | "gpu" : " 17 GB", | 20260507 16:13:31.273 | INFO | "huge" : "", | 20260507 16:13:31.273 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:31.273 | INFO | "malloc" : " 14 GB", | 20260507 16:13:31.273 | INFO | "pinned" : " 29 GB" | 20260507 16:13:31.273 | INFO | } | 20260507 16:13:31.273 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:13:31.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:31.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:13:31.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:13:31.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:31.273 | INFO | Uploaded 36GiB to GPU in 1.61076s (22.3496GiB/s) | 20260507 16:13:31.273 | INFO | Downloaded 34GiB to GPU in 1.53141s (22.2018GiB/s) | 20260507 16:13:36.412 | INFO | Analyzing chunk frames = [845312,845824) | 20260507 16:13:36.412 | INFO | MemInfo: { | 20260507 16:13:36.412 | INFO | "gpu" : " 17 GB", | 20260507 16:13:36.412 | INFO | "huge" : "", | 20260507 16:13:36.412 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:36.412 | INFO | "malloc" : " 14 GB", | 20260507 16:13:36.412 | INFO | "pinned" : " 29 GB" | 20260507 16:13:36.412 | INFO | } | 20260507 16:13:36.412 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:13:36.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:36.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.412 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:13:36.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.413 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:13:36.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:36.413 | INFO | Uploaded 36GiB to GPU in 1.6098s (22.363GiB/s) | 20260507 16:13:36.413 | INFO | Downloaded 34GiB to GPU in 1.52983s (22.2247GiB/s) | 20260507 16:13:41.552 | INFO | Analyzing chunk frames = [845824,846336) | 20260507 16:13:41.552 | INFO | MemInfo: { | 20260507 16:13:41.552 | INFO | "gpu" : " 17 GB", | 20260507 16:13:41.552 | INFO | "huge" : "", | 20260507 16:13:41.552 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:41.552 | INFO | "malloc" : " 14 GB", | 20260507 16:13:41.552 | INFO | "pinned" : " 29 GB" | 20260507 16:13:41.552 | INFO | } | 20260507 16:13:41.552 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:13:41.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:41.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:13:41.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 16:13:41.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:41.552 | INFO | Uploaded 36GiB to GPU in 1.60907s (22.3732GiB/s) | 20260507 16:13:41.552 | INFO | Downloaded 34GiB to GPU in 1.52927s (22.2328GiB/s) | 20260507 16:13:46.691 | INFO | Analyzing chunk frames = [846336,846848) | 20260507 16:13:46.691 | INFO | MemInfo: { | 20260507 16:13:46.691 | INFO | "gpu" : " 17 GB", | 20260507 16:13:46.691 | INFO | "huge" : "", | 20260507 16:13:46.691 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:46.691 | INFO | "malloc" : " 14 GB", | 20260507 16:13:46.691 | INFO | "pinned" : " 29 GB" | 20260507 16:13:46.691 | INFO | } | 20260507 16:13:46.691 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:13:46.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:46.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | ANALYSIS : 56.1%, 3.0, 87.5ms, 0 | 20260507 16:13:46.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:13:46.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:46.691 | INFO | Uploaded 36GiB to GPU in 1.60938s (22.3689GiB/s) | 20260507 16:13:46.691 | INFO | Downloaded 34GiB to GPU in 1.52876s (22.2403GiB/s) | 20260507 16:13:51.830 | INFO | Analyzing chunk frames = [846848,847360) | 20260507 16:13:51.830 | INFO | MemInfo: { | 20260507 16:13:51.830 | INFO | "gpu" : " 17 GB", | 20260507 16:13:51.830 | INFO | "huge" : "", | 20260507 16:13:51.830 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:51.830 | INFO | "malloc" : " 14 GB", | 20260507 16:13:51.830 | INFO | "pinned" : " 29 GB" | 20260507 16:13:51.830 | INFO | } | 20260507 16:13:51.831 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:13:51.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:51.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:13:51.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:13:51.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:51.831 | INFO | Uploaded 36GiB to GPU in 1.60882s (22.3767GiB/s) | 20260507 16:13:51.831 | INFO | Downloaded 34GiB to GPU in 1.52954s (22.229GiB/s) | 20260507 16:13:56.970 | INFO | Analyzing chunk frames = [847360,847872) | 20260507 16:13:56.970 | INFO | MemInfo: { | 20260507 16:13:56.970 | INFO | "gpu" : " 17 GB", | 20260507 16:13:56.970 | INFO | "huge" : "", | 20260507 16:13:56.970 | INFO | "huge_shared" : " 29 GB", | 20260507 16:13:56.970 | INFO | "malloc" : " 14 GB", | 20260507 16:13:56.970 | INFO | "pinned" : " 29 GB" | 20260507 16:13:56.970 | INFO | } | 20260507 16:13:56.970 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:13:56.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:13:56.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260507 16:13:56.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 16:13:56.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:13:56.970 | INFO | Uploaded 36GiB to GPU in 1.60883s (22.3765GiB/s) | 20260507 16:13:56.970 | INFO | Downloaded 34GiB to GPU in 1.52907s (22.2357GiB/s) | 20260507 16:13:57.067 | INFO | Starting a round of Baz IO | 20260507 16:14:00.010 | INFO | Populating RTMetrics report | 20260507 16:14:00.011 | INFO | RTMetrics report populated | 20260507 16:14:00.011 | INFO | Opening tmp RTMetrics json file | 20260507 16:14:00.038 | INFO | Serializing RTMetrics to json | 20260507 16:14:00.087 | INFO | Serialization done, swapping tmp file now... | 20260507 16:14:00.112 | INFO | RTMetrics done | 20260507 16:14:00.219 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.152 seconds | 20260507 16:14:00.219 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.82% / 0.03% / 49.15% | 20260507 16:14:00.219 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.76773% / 2.38419e-05% / 1.98682e-05% | 20260507 16:14:00.220 | INFO | Baz IO overall: | 20260507 16:14:00.220 | INFO | BytesWritten : 360174828199 | 20260507 16:14:00.220 | INFO | NumEvents : 174133018317 | 20260507 16:14:02.109 | INFO | Analyzing chunk frames = [847872,848384) | 20260507 16:14:02.109 | INFO | MemInfo: { | 20260507 16:14:02.109 | INFO | "gpu" : " 17 GB", | 20260507 16:14:02.109 | INFO | "huge" : "", | 20260507 16:14:02.109 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:02.109 | INFO | "malloc" : " 14 GB", | 20260507 16:14:02.109 | INFO | "pinned" : " 29 GB" | 20260507 16:14:02.109 | INFO | } | 20260507 16:14:02.109 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:14:02.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:02.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:02.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:02.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:02.109 | INFO | ANALYSIS : 58.3%, 2.8, 84.1ms, 0 | 20260507 16:14:02.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:02.109 | INFO | RT_METRICS : 48.0%, 1.0, 25.1ms, 0 | 20260507 16:14:02.109 | INFO | PRE_HQ : 57.4%, 2.2, 64.3ms, 0 | 20260507 16:14:02.109 | INFO | BAZWRITER : 60.9%, 4.8, 149.1ms, 0 | 20260507 16:14:02.109 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:14:02.110 | INFO | Uploaded 36GiB to GPU in 1.6541s (21.764GiB/s) | 20260507 16:14:02.110 | INFO | Downloaded 38GiB to GPU in 1.73592s (21.8905GiB/s) | 20260507 16:14:07.248 | INFO | Analyzing chunk frames = [848384,848896) | 20260507 16:14:07.248 | INFO | MemInfo: { | 20260507 16:14:07.248 | INFO | "gpu" : " 17 GB", | 20260507 16:14:07.248 | INFO | "huge" : "", | 20260507 16:14:07.248 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:07.248 | INFO | "malloc" : " 14 GB", | 20260507 16:14:07.249 | INFO | "pinned" : " 29 GB" | 20260507 16:14:07.249 | INFO | } | 20260507 16:14:07.249 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:14:07.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:07.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 16:14:07.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:14:07.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:07.249 | INFO | Uploaded 36GiB to GPU in 1.58635s (22.6935GiB/s) | 20260507 16:14:07.249 | INFO | Downloaded 34GiB to GPU in 1.53435s (22.1592GiB/s) | 20260507 16:14:12.388 | INFO | Analyzing chunk frames = [848896,849408) | 20260507 16:14:12.388 | INFO | MemInfo: { | 20260507 16:14:12.388 | INFO | "gpu" : " 17 GB", | 20260507 16:14:12.388 | INFO | "huge" : "", | 20260507 16:14:12.388 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:12.388 | INFO | "malloc" : " 14 GB", | 20260507 16:14:12.388 | INFO | "pinned" : " 29 GB" | 20260507 16:14:12.388 | INFO | } | 20260507 16:14:12.388 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:14:12.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:12.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:14:12.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:14:12.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:12.388 | INFO | Uploaded 36GiB to GPU in 1.59116s (22.625GiB/s) | 20260507 16:14:12.388 | INFO | Downloaded 34GiB to GPU in 1.53938s (22.0869GiB/s) | 20260507 16:14:17.527 | INFO | Analyzing chunk frames = [849408,849920) | 20260507 16:14:17.527 | INFO | MemInfo: { | 20260507 16:14:17.527 | INFO | "gpu" : " 17 GB", | 20260507 16:14:17.527 | INFO | "huge" : "", | 20260507 16:14:17.527 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:17.531 | INFO | "malloc" : " 14 GB", | 20260507 16:14:17.531 | INFO | "pinned" : " 29 GB" | 20260507 16:14:17.531 | INFO | } | 20260507 16:14:17.531 | INFO | Perf report for the last 5.14286 seconds: | 20260507 16:14:17.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:17.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:14:17.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:14:17.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:17.531 | INFO | Uploaded 36GiB to GPU in 1.5901s (22.6401GiB/s) | 20260507 16:14:17.531 | INFO | Downloaded 34GiB to GPU in 1.54035s (22.073GiB/s) | 20260507 16:14:22.670 | INFO | Analyzing chunk frames = [849920,850432) | 20260507 16:14:22.670 | INFO | MemInfo: { | 20260507 16:14:22.670 | INFO | "gpu" : " 17 GB", | 20260507 16:14:22.670 | INFO | "huge" : "", | 20260507 16:14:22.670 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:22.670 | INFO | "malloc" : " 14 GB", | 20260507 16:14:22.670 | INFO | "pinned" : " 29 GB" | 20260507 16:14:22.670 | INFO | } | 20260507 16:14:22.671 | INFO | Perf report for the last 5.13966 seconds: | 20260507 16:14:22.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:22.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:14:22.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:14:22.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:22.671 | INFO | Uploaded 36GiB to GPU in 1.58885s (22.6579GiB/s) | 20260507 16:14:22.671 | INFO | Downloaded 34GiB to GPU in 1.53834s (22.1017GiB/s) | 20260507 16:14:27.806 | INFO | Analyzing chunk frames = [850432,850944) | 20260507 16:14:27.806 | INFO | MemInfo: { | 20260507 16:14:27.806 | INFO | "gpu" : " 17 GB", | 20260507 16:14:27.806 | INFO | "huge" : "", | 20260507 16:14:27.806 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:27.806 | INFO | "malloc" : " 14 GB", | 20260507 16:14:27.806 | INFO | "pinned" : " 29 GB" | 20260507 16:14:27.806 | INFO | } | 20260507 16:14:27.806 | INFO | Perf report for the last 5.13562 seconds: | 20260507 16:14:27.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:27.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | ANALYSIS : 56.1%, 3.0, 87.4ms, 0 | 20260507 16:14:27.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:14:27.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:27.806 | INFO | Uploaded 36GiB to GPU in 1.58754s (22.6766GiB/s) | 20260507 16:14:27.806 | INFO | Downloaded 34GiB to GPU in 1.53861s (22.0978GiB/s) | 20260507 16:14:32.945 | INFO | Analyzing chunk frames = [850944,851456) | 20260507 16:14:32.946 | INFO | MemInfo: { | 20260507 16:14:32.946 | INFO | "gpu" : " 17 GB", | 20260507 16:14:32.946 | INFO | "huge" : "", | 20260507 16:14:32.946 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:32.947 | INFO | "malloc" : " 14 GB", | 20260507 16:14:32.947 | INFO | "pinned" : " 29 GB" | 20260507 16:14:32.947 | INFO | } | 20260507 16:14:32.947 | INFO | Perf report for the last 5.14062 seconds: | 20260507 16:14:32.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:32.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:14:32.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:14:32.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:32.947 | INFO | Uploaded 36GiB to GPU in 1.58978s (22.6447GiB/s) | 20260507 16:14:32.947 | INFO | Downloaded 34GiB to GPU in 1.53901s (22.0922GiB/s) | 20260507 16:14:38.085 | INFO | Analyzing chunk frames = [851456,851968) | 20260507 16:14:38.085 | INFO | MemInfo: { | 20260507 16:14:38.085 | INFO | "gpu" : " 17 GB", | 20260507 16:14:38.085 | INFO | "huge" : "", | 20260507 16:14:38.085 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:38.085 | INFO | "malloc" : " 14 GB", | 20260507 16:14:38.085 | INFO | "pinned" : " 29 GB" | 20260507 16:14:38.085 | INFO | } | 20260507 16:14:38.085 | INFO | Perf report for the last 5.138 seconds: | 20260507 16:14:38.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:38.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:14:38.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:14:38.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:38.085 | INFO | Uploaded 36GiB to GPU in 1.58882s (22.6584GiB/s) | 20260507 16:14:38.085 | INFO | Downloaded 34GiB to GPU in 1.53814s (22.1046GiB/s) | 20260507 16:14:38.085 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:14:38.085 | INFO | Total mem usage: 16.4 GiB | 20260507 16:14:38.085 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:14:38.085 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.62%) | 20260507 16:14:38.085 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:14:38.085 | INFO | High Water Breakdown: | 20260507 16:14:38.085 | INFO | 59.7% : 9.1 GiB ( 91.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:14:38.085 | INFO | 40.2% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:14:38.085 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:14:38.085 | INFO | ------------------------------------------ | 20260507 16:14:38.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:14:38.085 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:14:38.085 | INFO | Total mem usage: 26.5 GiB | 20260507 16:14:38.085 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:14:38.085 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:14:38.085 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 16:14:38.086 | INFO | High Water Breakdown: | 20260507 16:14:38.086 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:14:38.086 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:14:38.086 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:14:38.086 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:14:38.086 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:14:38.086 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:14:38.086 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:14:38.086 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:14:38.086 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:14:38.086 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:14:38.086 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:14:38.086 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:14:38.086 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:14:38.086 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:14:38.086 | INFO | ------------------------------------------ | 20260507 16:14:38.086 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:14:38.086 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:14:38.086 | INFO | Total mem usage: 27.8 GiB | 20260507 16:14:38.086 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:14:38.086 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 16:14:38.086 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 16:14:38.086 | INFO | High Water Breakdown: | 20260507 16:14:38.086 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 16:14:38.086 | INFO | ------------------------------------------ | 20260507 16:14:38.086 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:14:38.086 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:14:38.086 | INFO | Total mem usage: 15.7 GiB | 20260507 16:14:38.086 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:14:38.086 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:14:38.086 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:14:38.086 | INFO | High Water Breakdown: | 20260507 16:14:38.086 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:14:38.086 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:14:38.086 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:14:38.086 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:14:38.086 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:14:38.086 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:14:38.086 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:14:38.086 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:14:38.086 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:14:38.086 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:14:38.086 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:14:38.086 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:14:38.086 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:14:38.086 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:14:38.086 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:14:38.086 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:14:38.086 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:14:38.086 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:14:38.086 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:14:38.086 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:14:38.086 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:14:38.086 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:14:38.086 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:14:38.086 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:14:38.086 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:14:38.086 | INFO | ------------------------------------------ | 20260507 16:14:38.086 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:14:38.087 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:14:38.087 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:14:38.087 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:14:38.087 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 0.29%, AggregateStats -- 8.71e-02 / -nan, 5.46e-03 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 11.07%, Baseline -- 3.30e+00 / -nan, 2.18e-02 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 2.98e+00 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.11e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 3.38%, Metrics -- 1.01e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.68e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 16:14:38.087 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:14:38.087 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:14:38.087 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:14:38.087 | INFO | PerfReport: Full Execution -- 8.88e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 18.83%, Upload -- 1.67e+01 / 0.00e+00, 4.72e-01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 18.28%, Download -- 1.62e+01 / 0.00e+00, 7.89e-01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 33.52%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 0.62%, LockUpload -- 5.47e-01 / 0.00e+00, 3.70e+00 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 0.00%, LockDownload -- 4.67e-04 / 0.00e+00, 2.76e-04 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 28.74%, LockCompute -- 2.55e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 16:14:38.087 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260507 16:14:38.181 | INFO | Starting a round of Baz IO | 20260507 16:14:41.052 | INFO | Populating RTMetrics report | 20260507 16:14:41.053 | INFO | RTMetrics report populated | 20260507 16:14:41.053 | INFO | Opening tmp RTMetrics json file | 20260507 16:14:41.054 | INFO | Serializing RTMetrics to json | 20260507 16:14:41.057 | INFO | Serialization done, swapping tmp file now... | 20260507 16:14:41.060 | INFO | RTMetrics done | 20260507 16:14:41.319 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.137 seconds | 20260507 16:14:41.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.79% / 0.03% / 49.18% | 20260507 16:14:41.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.73375% / 4.37101e-05% / 1.98682e-05% | 20260507 16:14:41.319 | INFO | Baz IO overall: | 20260507 16:14:41.319 | INFO | BytesWritten : 362084822412 | 20260507 16:14:41.319 | INFO | NumEvents : 175061823168 | 20260507 16:14:43.224 | INFO | Analyzing chunk frames = [851968,852480) | 20260507 16:14:43.224 | INFO | MemInfo: { | 20260507 16:14:43.224 | INFO | "gpu" : " 17 GB", | 20260507 16:14:43.224 | INFO | "huge" : "", | 20260507 16:14:43.224 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:43.224 | INFO | "malloc" : " 14 GB", | 20260507 16:14:43.224 | INFO | "pinned" : " 29 GB" | 20260507 16:14:43.224 | INFO | } | 20260507 16:14:43.224 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:14:43.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:43.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:43.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:43.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:43.224 | INFO | ANALYSIS : 56.9%, 2.9, 85.5ms, 0 | 20260507 16:14:43.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:43.224 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 16:14:43.224 | INFO | PRE_HQ : 55.7%, 2.0, 58.8ms, 0 | 20260507 16:14:43.224 | INFO | BAZWRITER : 60.5%, 4.7, 146.6ms, 0 | 20260507 16:14:43.224 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 16:14:43.224 | INFO | Uploaded 36GiB to GPU in 1.65462s (21.7572GiB/s) | 20260507 16:14:43.224 | INFO | Downloaded 38GiB to GPU in 1.73938s (21.8468GiB/s) | 20260507 16:14:48.363 | INFO | Analyzing chunk frames = [852480,852992) | 20260507 16:14:48.363 | INFO | MemInfo: { | 20260507 16:14:48.363 | INFO | "gpu" : " 17 GB", | 20260507 16:14:48.363 | INFO | "huge" : "", | 20260507 16:14:48.363 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:48.363 | INFO | "malloc" : " 14 GB", | 20260507 16:14:48.363 | INFO | "pinned" : " 29 GB" | 20260507 16:14:48.363 | INFO | } | 20260507 16:14:48.363 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:14:48.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:48.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 16:14:48.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:14:48.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:48.364 | INFO | Uploaded 36GiB to GPU in 1.60625s (22.4124GiB/s) | 20260507 16:14:48.364 | INFO | Downloaded 34GiB to GPU in 1.52598s (22.2808GiB/s) | 20260507 16:14:53.502 | INFO | Analyzing chunk frames = [852992,853504) | 20260507 16:14:53.502 | INFO | MemInfo: { | 20260507 16:14:53.502 | INFO | "gpu" : " 17 GB", | 20260507 16:14:53.502 | INFO | "huge" : "", | 20260507 16:14:53.502 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:53.504 | INFO | "malloc" : " 14 GB", | 20260507 16:14:53.504 | INFO | "pinned" : " 29 GB" | 20260507 16:14:53.504 | INFO | } | 20260507 16:14:53.504 | INFO | Perf report for the last 5.14031 seconds: | 20260507 16:14:53.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:53.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:14:53.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:14:53.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:53.504 | INFO | Uploaded 36GiB to GPU in 1.61011s (22.3587GiB/s) | 20260507 16:14:53.504 | INFO | Downloaded 34GiB to GPU in 1.53111s (22.2061GiB/s) | 20260507 16:14:58.642 | INFO | Analyzing chunk frames = [853504,854016) | 20260507 16:14:58.642 | INFO | MemInfo: { | 20260507 16:14:58.642 | INFO | "gpu" : " 17 GB", | 20260507 16:14:58.642 | INFO | "huge" : "", | 20260507 16:14:58.642 | INFO | "huge_shared" : " 29 GB", | 20260507 16:14:58.645 | INFO | "malloc" : " 14 GB", | 20260507 16:14:58.645 | INFO | "pinned" : " 29 GB" | 20260507 16:14:58.645 | INFO | } | 20260507 16:14:58.645 | INFO | Perf report for the last 5.14157 seconds: | 20260507 16:14:58.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:14:58.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.646 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:14:58.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.646 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:14:58.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:14:58.646 | INFO | Uploaded 36GiB to GPU in 1.61093s (22.3473GiB/s) | 20260507 16:14:58.646 | INFO | Downloaded 34GiB to GPU in 1.53019s (22.2195GiB/s) | 20260507 16:15:03.781 | INFO | Analyzing chunk frames = [854016,854528) | 20260507 16:15:03.781 | INFO | MemInfo: { | 20260507 16:15:03.781 | INFO | "gpu" : " 17 GB", | 20260507 16:15:03.781 | INFO | "huge" : "", | 20260507 16:15:03.781 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:03.781 | INFO | "malloc" : " 14 GB", | 20260507 16:15:03.781 | INFO | "pinned" : " 29 GB" | 20260507 16:15:03.781 | INFO | } | 20260507 16:15:03.781 | INFO | Perf report for the last 5.13601 seconds: | 20260507 16:15:03.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:03.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.781 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:15:03.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.782 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:15:03.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:03.782 | INFO | Uploaded 36GiB to GPU in 1.60872s (22.3781GiB/s) | 20260507 16:15:03.782 | INFO | Downloaded 34GiB to GPU in 1.52819s (22.2486GiB/s) | 20260507 16:15:08.922 | INFO | Analyzing chunk frames = [854528,855040) | 20260507 16:15:08.922 | INFO | MemInfo: { | 20260507 16:15:08.922 | INFO | "gpu" : " 17 GB", | 20260507 16:15:08.922 | INFO | "huge" : "", | 20260507 16:15:08.922 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:08.922 | INFO | "malloc" : " 14 GB", | 20260507 16:15:08.922 | INFO | "pinned" : " 29 GB" | 20260507 16:15:08.922 | INFO | } | 20260507 16:15:08.922 | INFO | Perf report for the last 5.14092 seconds: | 20260507 16:15:08.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:08.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 16:15:08.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:15:08.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:08.922 | INFO | Uploaded 36GiB to GPU in 1.60955s (22.3665GiB/s) | 20260507 16:15:08.923 | INFO | Downloaded 34GiB to GPU in 1.52892s (22.2379GiB/s) | 20260507 16:15:14.060 | INFO | Analyzing chunk frames = [855040,855552) | 20260507 16:15:14.060 | INFO | MemInfo: { | 20260507 16:15:14.060 | INFO | "gpu" : " 17 GB", | 20260507 16:15:14.060 | INFO | "huge" : "", | 20260507 16:15:14.060 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:14.060 | INFO | "malloc" : " 14 GB", | 20260507 16:15:14.060 | INFO | "pinned" : " 29 GB" | 20260507 16:15:14.060 | INFO | } | 20260507 16:15:14.061 | INFO | Perf report for the last 5.13817 seconds: | 20260507 16:15:14.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:14.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260507 16:15:14.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:15:14.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:14.061 | INFO | Uploaded 36GiB to GPU in 1.60901s (22.374GiB/s) | 20260507 16:15:14.061 | INFO | Downloaded 34GiB to GPU in 1.52911s (22.2351GiB/s) | 20260507 16:15:19.199 | INFO | Analyzing chunk frames = [855552,856064) | 20260507 16:15:19.199 | INFO | MemInfo: { | 20260507 16:15:19.199 | INFO | "gpu" : " 17 GB", | 20260507 16:15:19.199 | INFO | "huge" : "", | 20260507 16:15:19.199 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:19.199 | INFO | "malloc" : " 14 GB", | 20260507 16:15:19.199 | INFO | "pinned" : " 29 GB" | 20260507 16:15:19.199 | INFO | } | 20260507 16:15:19.200 | INFO | Perf report for the last 5.13896 seconds: | 20260507 16:15:19.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:19.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:15:19.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:15:19.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:19.200 | INFO | Uploaded 36GiB to GPU in 1.60978s (22.3632GiB/s) | 20260507 16:15:19.200 | INFO | Downloaded 34GiB to GPU in 1.52924s (22.2332GiB/s) | 20260507 16:15:19.296 | INFO | Starting a round of Baz IO | 20260507 16:15:22.168 | INFO | Populating RTMetrics report | 20260507 16:15:22.168 | INFO | RTMetrics report populated | 20260507 16:15:22.168 | INFO | Opening tmp RTMetrics json file | 20260507 16:15:22.205 | INFO | Serializing RTMetrics to json | 20260507 16:15:22.263 | INFO | Serialization done, swapping tmp file now... | 20260507 16:15:22.285 | INFO | RTMetrics done | 20260507 16:15:22.589 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.292 seconds | 20260507 16:15:22.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.76% / 0.03% / 49.21% | 20260507 16:15:22.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.7461% / 4.37101e-05% / 2.78155e-05% | 20260507 16:15:22.589 | INFO | Baz IO overall: | 20260507 16:15:22.590 | INFO | BytesWritten : 363997431997 | 20260507 16:15:22.590 | INFO | NumEvents : 175992988719 | 20260507 16:15:24.338 | INFO | Analyzing chunk frames = [856064,856576) | 20260507 16:15:24.339 | INFO | MemInfo: { | 20260507 16:15:24.339 | INFO | "gpu" : " 17 GB", | 20260507 16:15:24.339 | INFO | "huge" : "", | 20260507 16:15:24.339 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:24.339 | INFO | "malloc" : " 14 GB", | 20260507 16:15:24.339 | INFO | "pinned" : " 29 GB" | 20260507 16:15:24.339 | INFO | } | 20260507 16:15:24.339 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:15:24.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:24.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:24.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:24.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:24.339 | INFO | ANALYSIS : 56.9%, 2.9, 85.6ms, 0 | 20260507 16:15:24.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:24.339 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 16:15:24.339 | INFO | PRE_HQ : 55.5%, 2.0, 58.2ms, 0 | 20260507 16:15:24.339 | INFO | BAZWRITER : 63.7%, 4.8, 156.5ms, 0 | 20260507 16:15:24.339 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 16:15:24.339 | INFO | Uploaded 36GiB to GPU in 1.6698s (21.5594GiB/s) | 20260507 16:15:24.339 | INFO | Downloaded 38GiB to GPU in 1.75315s (21.6752GiB/s) | 20260507 16:15:29.478 | INFO | Analyzing chunk frames = [856576,857088) | 20260507 16:15:29.478 | INFO | MemInfo: { | 20260507 16:15:29.478 | INFO | "gpu" : " 17 GB", | 20260507 16:15:29.478 | INFO | "huge" : "", | 20260507 16:15:29.478 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:29.478 | INFO | "malloc" : " 14 GB", | 20260507 16:15:29.478 | INFO | "pinned" : " 29 GB" | 20260507 16:15:29.478 | INFO | } | 20260507 16:15:29.478 | INFO | Perf report for the last 5.13965 seconds: | 20260507 16:15:29.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:29.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.478 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 16:15:29.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.479 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:15:29.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:29.479 | INFO | Uploaded 36GiB to GPU in 1.58619s (22.6959GiB/s) | 20260507 16:15:29.479 | INFO | Downloaded 34GiB to GPU in 1.53431s (22.1598GiB/s) | 20260507 16:15:34.617 | INFO | Analyzing chunk frames = [857088,857600) | 20260507 16:15:34.617 | INFO | MemInfo: { | 20260507 16:15:34.617 | INFO | "gpu" : " 17 GB", | 20260507 16:15:34.617 | INFO | "huge" : "", | 20260507 16:15:34.618 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:34.618 | INFO | "malloc" : " 14 GB", | 20260507 16:15:34.618 | INFO | "pinned" : " 29 GB" | 20260507 16:15:34.618 | INFO | } | 20260507 16:15:34.618 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:15:34.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:34.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:15:34.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:15:34.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:34.618 | INFO | Uploaded 36GiB to GPU in 1.59082s (22.6298GiB/s) | 20260507 16:15:34.618 | INFO | Downloaded 34GiB to GPU in 1.54001s (22.0778GiB/s) | 20260507 16:15:39.757 | INFO | Analyzing chunk frames = [857600,858112) | 20260507 16:15:39.757 | INFO | MemInfo: { | 20260507 16:15:39.757 | INFO | "gpu" : " 17 GB", | 20260507 16:15:39.757 | INFO | "huge" : "", | 20260507 16:15:39.757 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:39.757 | INFO | "malloc" : " 14 GB", | 20260507 16:15:39.757 | INFO | "pinned" : " 29 GB" | 20260507 16:15:39.757 | INFO | } | 20260507 16:15:39.757 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:15:39.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:39.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:15:39.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:15:39.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:39.757 | INFO | Uploaded 36GiB to GPU in 1.5905s (22.6345GiB/s) | 20260507 16:15:39.757 | INFO | Downloaded 34GiB to GPU in 1.54004s (22.0773GiB/s) | 20260507 16:15:44.896 | INFO | Analyzing chunk frames = [858112,858624) | 20260507 16:15:44.896 | INFO | MemInfo: { | 20260507 16:15:44.896 | INFO | "gpu" : " 17 GB", | 20260507 16:15:44.896 | INFO | "huge" : "", | 20260507 16:15:44.896 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:44.896 | INFO | "malloc" : " 14 GB", | 20260507 16:15:44.896 | INFO | "pinned" : " 29 GB" | 20260507 16:15:44.896 | INFO | } | 20260507 16:15:44.896 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:15:44.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:44.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.896 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:15:44.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.896 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:15:44.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:44.897 | INFO | Uploaded 36GiB to GPU in 1.58879s (22.6587GiB/s) | 20260507 16:15:44.897 | INFO | Downloaded 34GiB to GPU in 1.53752s (22.1135GiB/s) | 20260507 16:15:50.035 | INFO | Analyzing chunk frames = [858624,859136) | 20260507 16:15:50.035 | INFO | MemInfo: { | 20260507 16:15:50.035 | INFO | "gpu" : " 17 GB", | 20260507 16:15:50.035 | INFO | "huge" : "", | 20260507 16:15:50.035 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:50.035 | INFO | "malloc" : " 14 GB", | 20260507 16:15:50.035 | INFO | "pinned" : " 29 GB" | 20260507 16:15:50.035 | INFO | } | 20260507 16:15:50.036 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:15:50.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:50.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 16:15:50.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:15:50.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:50.036 | INFO | Uploaded 36GiB to GPU in 1.59037s (22.6363GiB/s) | 20260507 16:15:50.036 | INFO | Downloaded 34GiB to GPU in 1.5376s (22.1124GiB/s) | 20260507 16:15:55.175 | INFO | Analyzing chunk frames = [859136,859648) | 20260507 16:15:55.175 | INFO | MemInfo: { | 20260507 16:15:55.175 | INFO | "gpu" : " 17 GB", | 20260507 16:15:55.175 | INFO | "huge" : "", | 20260507 16:15:55.175 | INFO | "huge_shared" : " 29 GB", | 20260507 16:15:55.175 | INFO | "malloc" : " 14 GB", | 20260507 16:15:55.175 | INFO | "pinned" : " 29 GB" | 20260507 16:15:55.175 | INFO | } | 20260507 16:15:55.175 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:15:55.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:15:55.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:15:55.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:15:55.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:15:55.175 | INFO | Uploaded 36GiB to GPU in 1.58886s (22.6578GiB/s) | 20260507 16:15:55.175 | INFO | Downloaded 34GiB to GPU in 1.53767s (22.1113GiB/s) | 20260507 16:16:00.314 | INFO | Analyzing chunk frames = [859648,860160) | 20260507 16:16:00.314 | INFO | MemInfo: { | 20260507 16:16:00.314 | INFO | "gpu" : " 17 GB", | 20260507 16:16:00.314 | INFO | "huge" : "", | 20260507 16:16:00.314 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:00.314 | INFO | "malloc" : " 14 GB", | 20260507 16:16:00.314 | INFO | "pinned" : " 29 GB" | 20260507 16:16:00.314 | INFO | } | 20260507 16:16:00.314 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:16:00.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:00.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.314 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:16:00.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.314 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 16:16:00.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:00.315 | INFO | Uploaded 36GiB to GPU in 1.5898s (22.6444GiB/s) | 20260507 16:16:00.315 | INFO | Downloaded 34GiB to GPU in 1.53881s (22.0949GiB/s) | 20260507 16:16:00.315 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:16:00.315 | INFO | Total mem usage: 16.4 GiB | 20260507 16:16:00.315 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:16:00.315 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.34%) | 20260507 16:16:00.315 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:16:00.315 | INFO | High Water Breakdown: | 20260507 16:16:00.315 | INFO | 60.1% : 9.36 GiB ( 94.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:16:00.315 | INFO | 39.8% : 6.2 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:16:00.315 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:16:00.315 | INFO | ------------------------------------------ | 20260507 16:16:00.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:16:00.315 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:16:00.315 | INFO | Total mem usage: 26.5 GiB | 20260507 16:16:00.315 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:16:00.315 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:16:00.315 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:16:00.315 | INFO | High Water Breakdown: | 20260507 16:16:00.315 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:16:00.315 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:16:00.315 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:16:00.315 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:16:00.315 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:16:00.315 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:16:00.315 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:16:00.315 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:16:00.315 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:16:00.315 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:16:00.315 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:16:00.315 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:16:00.315 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:16:00.315 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:16:00.315 | INFO | ------------------------------------------ | 20260507 16:16:00.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:16:00.315 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:16:00.315 | INFO | Total mem usage: 27.8 GiB | 20260507 16:16:00.315 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:16:00.315 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:16:00.315 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:16:00.315 | INFO | High Water Breakdown: | 20260507 16:16:00.315 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:16:00.315 | INFO | ------------------------------------------ | 20260507 16:16:00.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:16:00.315 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:16:00.315 | INFO | Total mem usage: 15.7 GiB | 20260507 16:16:00.315 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:16:00.315 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:16:00.315 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:16:00.315 | INFO | High Water Breakdown: | 20260507 16:16:00.315 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:16:00.315 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:16:00.315 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:16:00.315 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:16:00.315 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:16:00.315 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:16:00.315 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:16:00.315 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:16:00.315 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:16:00.315 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:16:00.315 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:16:00.315 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:16:00.315 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:16:00.315 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:16:00.315 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:16:00.315 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:16:00.315 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:16:00.315 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:16:00.315 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:16:00.315 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:16:00.315 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:16:00.315 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:16:00.315 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:16:00.315 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:16:00.315 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:16:00.315 | INFO | ------------------------------------------ | 20260507 16:16:00.315 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:16:00.315 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:16:00.315 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:16:00.316 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:16:00.316 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 5.29e-03 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 11.06%, Baseline -- 3.29e+00 / -nan, 1.83e-02 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 3.08%, Histogram -- 9.17e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 16.17%, DME -- 3.85e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.11e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 3.38%, Metrics -- 1.00e+00 / -nan, 1.63e-01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.67e-01 / -nan, 5.38e-02 / 0.00e+00 | 20260507 16:16:00.316 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:16:00.316 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:16:00.316 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:16:00.316 | INFO | PerfReport: Full Execution -- 8.90e+01 / -nan, 1.84e+01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 18.81%, Upload -- 1.67e+01 / 0.00e+00, 4.90e-01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 18.26%, Download -- 1.63e+01 / 0.00e+00, 8.40e-01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 33.44%, Compute -- 2.98e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 0.00%, LockDownload -- 4.36e-04 / 0.00e+00, 1.31e-04 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 28.91%, LockCompute -- 2.57e+01 / 0.00e+00, 1.66e+01 / 0.00e+00 | 20260507 16:16:00.316 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 16:16:00.411 | INFO | Starting a round of Baz IO | 20260507 16:16:03.285 | INFO | Populating RTMetrics report | 20260507 16:16:03.286 | INFO | RTMetrics report populated | 20260507 16:16:03.286 | INFO | Opening tmp RTMetrics json file | 20260507 16:16:03.331 | INFO | Serializing RTMetrics to json | 20260507 16:16:03.368 | INFO | Serialization done, swapping tmp file now... | 20260507 16:16:03.392 | INFO | RTMetrics done | 20260507 16:16:03.423 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.012 seconds | 20260507 16:16:03.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.73% / 0.03% / 49.24% | 20260507 16:16:03.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.73469% / 4.76837e-05% / 1.58946e-05% | 20260507 16:16:03.424 | INFO | Baz IO overall: | 20260507 16:16:03.424 | INFO | BytesWritten : 365909385297 | 20260507 16:16:03.424 | INFO | NumEvents : 176924039224 | 20260507 16:16:05.453 | INFO | Analyzing chunk frames = [860160,860672) | 20260507 16:16:05.453 | INFO | MemInfo: { | 20260507 16:16:05.453 | INFO | "gpu" : " 17 GB", | 20260507 16:16:05.453 | INFO | "huge" : "", | 20260507 16:16:05.453 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:05.453 | INFO | "malloc" : " 14 GB", | 20260507 16:16:05.453 | INFO | "pinned" : " 29 GB" | 20260507 16:16:05.453 | INFO | } | 20260507 16:16:05.454 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:16:05.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:05.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:05.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:05.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:05.454 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260507 16:16:05.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:05.454 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 16:16:05.454 | INFO | PRE_HQ : 55.5%, 2.0, 59.0ms, 0 | 20260507 16:16:05.454 | INFO | BAZWRITER : 58.2%, 4.3, 129.9ms, 0 | 20260507 16:16:05.454 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 16:16:05.454 | INFO | Uploaded 36GiB to GPU in 1.63338s (22.0402GiB/s) | 20260507 16:16:05.454 | INFO | Downloaded 38GiB to GPU in 1.71578s (22.1474GiB/s) | 20260507 16:16:10.593 | INFO | Analyzing chunk frames = [860672,861184) | 20260507 16:16:10.593 | INFO | MemInfo: { | 20260507 16:16:10.593 | INFO | "gpu" : " 17 GB", | 20260507 16:16:10.593 | INFO | "huge" : "", | 20260507 16:16:10.593 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:10.593 | INFO | "malloc" : " 14 GB", | 20260507 16:16:10.593 | INFO | "pinned" : " 29 GB" | 20260507 16:16:10.593 | INFO | } | 20260507 16:16:10.593 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:16:10.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:10.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 16:16:10.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:16:10.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:10.593 | INFO | Uploaded 36GiB to GPU in 1.58651s (22.6914GiB/s) | 20260507 16:16:10.593 | INFO | Downloaded 34GiB to GPU in 1.53459s (22.1557GiB/s) | 20260507 16:16:15.732 | INFO | Analyzing chunk frames = [861184,861696) | 20260507 16:16:15.732 | INFO | MemInfo: { | 20260507 16:16:15.732 | INFO | "gpu" : " 17 GB", | 20260507 16:16:15.732 | INFO | "huge" : "", | 20260507 16:16:15.732 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:15.732 | INFO | "malloc" : " 14 GB", | 20260507 16:16:15.732 | INFO | "pinned" : " 29 GB" | 20260507 16:16:15.732 | INFO | } | 20260507 16:16:15.732 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:16:15.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:15.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.732 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:16:15.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.733 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:16:15.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:15.733 | INFO | Uploaded 36GiB to GPU in 1.58988s (22.6432GiB/s) | 20260507 16:16:15.733 | INFO | Downloaded 34GiB to GPU in 1.53978s (22.081GiB/s) | 20260507 16:16:20.871 | INFO | Analyzing chunk frames = [861696,862208) | 20260507 16:16:20.871 | INFO | MemInfo: { | 20260507 16:16:20.871 | INFO | "gpu" : " 17 GB", | 20260507 16:16:20.871 | INFO | "huge" : "", | 20260507 16:16:20.871 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:20.871 | INFO | "malloc" : " 14 GB", | 20260507 16:16:20.872 | INFO | "pinned" : " 29 GB" | 20260507 16:16:20.872 | INFO | } | 20260507 16:16:20.872 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:16:20.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:20.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:16:20.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:16:20.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:20.872 | INFO | Uploaded 36GiB to GPU in 1.59085s (22.6294GiB/s) | 20260507 16:16:20.872 | INFO | Downloaded 34GiB to GPU in 1.53935s (22.0873GiB/s) | 20260507 16:16:26.011 | INFO | Analyzing chunk frames = [862208,862720) | 20260507 16:16:26.011 | INFO | MemInfo: { | 20260507 16:16:26.011 | INFO | "gpu" : " 17 GB", | 20260507 16:16:26.011 | INFO | "huge" : "", | 20260507 16:16:26.011 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:26.011 | INFO | "malloc" : " 14 GB", | 20260507 16:16:26.011 | INFO | "pinned" : " 29 GB" | 20260507 16:16:26.011 | INFO | } | 20260507 16:16:26.011 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:16:26.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:26.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:16:26.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:16:26.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:26.011 | INFO | Uploaded 36GiB to GPU in 1.58842s (22.664GiB/s) | 20260507 16:16:26.011 | INFO | Downloaded 34GiB to GPU in 1.53728s (22.117GiB/s) | 20260507 16:16:31.150 | INFO | Analyzing chunk frames = [862720,863232) | 20260507 16:16:31.150 | INFO | MemInfo: { | 20260507 16:16:31.150 | INFO | "gpu" : " 17 GB", | 20260507 16:16:31.150 | INFO | "huge" : "", | 20260507 16:16:31.150 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:31.150 | INFO | "malloc" : " 14 GB", | 20260507 16:16:31.150 | INFO | "pinned" : " 29 GB" | 20260507 16:16:31.150 | INFO | } | 20260507 16:16:31.150 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:16:31.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:31.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.151 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:16:31.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.151 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:16:31.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:31.151 | INFO | Uploaded 36GiB to GPU in 1.58892s (22.6568GiB/s) | 20260507 16:16:31.151 | INFO | Downloaded 34GiB to GPU in 1.5386s (22.098GiB/s) | 20260507 16:16:36.289 | INFO | Analyzing chunk frames = [863232,863744) | 20260507 16:16:36.289 | INFO | MemInfo: { | 20260507 16:16:36.289 | INFO | "gpu" : " 17 GB", | 20260507 16:16:36.289 | INFO | "huge" : "", | 20260507 16:16:36.290 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:36.290 | INFO | "malloc" : " 14 GB", | 20260507 16:16:36.290 | INFO | "pinned" : " 29 GB" | 20260507 16:16:36.290 | INFO | } | 20260507 16:16:36.290 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:16:36.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:36.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:16:36.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:16:36.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:36.290 | INFO | Uploaded 36GiB to GPU in 1.58884s (22.6581GiB/s) | 20260507 16:16:36.290 | INFO | Downloaded 34GiB to GPU in 1.53879s (22.0953GiB/s) | 20260507 16:16:41.429 | INFO | Analyzing chunk frames = [863744,864256) | 20260507 16:16:41.429 | INFO | MemInfo: { | 20260507 16:16:41.429 | INFO | "gpu" : " 17 GB", | 20260507 16:16:41.429 | INFO | "huge" : "", | 20260507 16:16:41.429 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:41.429 | INFO | "malloc" : " 14 GB", | 20260507 16:16:41.429 | INFO | "pinned" : " 29 GB" | 20260507 16:16:41.429 | INFO | } | 20260507 16:16:41.429 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:16:41.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:41.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:16:41.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:16:41.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:41.429 | INFO | Uploaded 36GiB to GPU in 1.58936s (22.6506GiB/s) | 20260507 16:16:41.429 | INFO | Downloaded 34GiB to GPU in 1.53959s (22.0838GiB/s) | 20260507 16:16:41.526 | INFO | Starting a round of Baz IO | 20260507 16:16:44.399 | INFO | Populating RTMetrics report | 20260507 16:16:44.400 | INFO | RTMetrics report populated | 20260507 16:16:44.400 | INFO | Opening tmp RTMetrics json file | 20260507 16:16:44.433 | INFO | Serializing RTMetrics to json | 20260507 16:16:44.495 | INFO | Serialization done, swapping tmp file now... | 20260507 16:16:44.502 | INFO | RTMetrics done | 20260507 16:16:44.557 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.03 seconds | 20260507 16:16:44.557 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.70% / 0.03% / 49.27% | 20260507 16:16:44.557 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.71995% / 6.75519e-05% / 3.97364e-05% | 20260507 16:16:44.557 | INFO | Baz IO overall: | 20260507 16:16:44.557 | INFO | BytesWritten : 367821688754 | 20260507 16:16:44.557 | INFO | NumEvents : 177854866228 | 20260507 16:16:46.568 | INFO | Analyzing chunk frames = [864256,864768) | 20260507 16:16:46.568 | INFO | MemInfo: { | 20260507 16:16:46.568 | INFO | "gpu" : " 17 GB", | 20260507 16:16:46.568 | INFO | "huge" : "", | 20260507 16:16:46.568 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:46.572 | INFO | "malloc" : " 14 GB", | 20260507 16:16:46.572 | INFO | "pinned" : " 29 GB" | 20260507 16:16:46.572 | INFO | } | 20260507 16:16:46.572 | INFO | Perf report for the last 5.14312 seconds: | 20260507 16:16:46.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:46.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:46.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:46.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:46.572 | INFO | ANALYSIS : 56.9%, 2.9, 85.8ms, 0 | 20260507 16:16:46.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:46.572 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260507 16:16:46.572 | INFO | PRE_HQ : 55.5%, 2.0, 58.8ms, 0 | 20260507 16:16:46.572 | INFO | BAZWRITER : 58.5%, 4.7, 141.8ms, 0 | 20260507 16:16:46.572 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 16:16:46.573 | INFO | Uploaded 36GiB to GPU in 1.66098s (21.674GiB/s) | 20260507 16:16:46.573 | INFO | Downloaded 38GiB to GPU in 1.74582s (21.7663GiB/s) | 20260507 16:16:51.708 | INFO | Analyzing chunk frames = [864768,865280) | 20260507 16:16:51.708 | INFO | MemInfo: { | 20260507 16:16:51.708 | INFO | "gpu" : " 17 GB", | 20260507 16:16:51.708 | INFO | "huge" : "", | 20260507 16:16:51.709 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:51.709 | INFO | "malloc" : " 14 GB", | 20260507 16:16:51.709 | INFO | "pinned" : " 29 GB" | 20260507 16:16:51.709 | INFO | } | 20260507 16:16:51.709 | INFO | Perf report for the last 5.13659 seconds: | 20260507 16:16:51.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:51.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:16:51.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:16:51.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:51.709 | INFO | Uploaded 36GiB to GPU in 1.60637s (22.4108GiB/s) | 20260507 16:16:51.709 | INFO | Downloaded 34GiB to GPU in 1.52636s (22.2752GiB/s) | 20260507 16:16:56.847 | INFO | Analyzing chunk frames = [865280,865792) | 20260507 16:16:56.847 | INFO | MemInfo: { | 20260507 16:16:56.847 | INFO | "gpu" : " 17 GB", | 20260507 16:16:56.847 | INFO | "huge" : "", | 20260507 16:16:56.847 | INFO | "huge_shared" : " 29 GB", | 20260507 16:16:56.847 | INFO | "malloc" : " 14 GB", | 20260507 16:16:56.847 | INFO | "pinned" : " 29 GB" | 20260507 16:16:56.847 | INFO | } | 20260507 16:16:56.847 | INFO | Perf report for the last 5.13825 seconds: | 20260507 16:16:56.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:16:56.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.847 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:16:56.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.847 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:16:56.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:16:56.848 | INFO | Uploaded 36GiB to GPU in 1.61083s (22.3488GiB/s) | 20260507 16:16:56.848 | INFO | Downloaded 34GiB to GPU in 1.53088s (22.2094GiB/s) | 20260507 16:17:01.986 | INFO | Analyzing chunk frames = [865792,866304) | 20260507 16:17:01.986 | INFO | MemInfo: { | 20260507 16:17:01.986 | INFO | "gpu" : " 17 GB", | 20260507 16:17:01.986 | INFO | "huge" : "", | 20260507 16:17:01.986 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:01.986 | INFO | "malloc" : " 14 GB", | 20260507 16:17:01.986 | INFO | "pinned" : " 29 GB" | 20260507 16:17:01.986 | INFO | } | 20260507 16:17:01.986 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:17:01.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:01.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.986 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:17:01.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.986 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:17:01.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:01.987 | INFO | Uploaded 36GiB to GPU in 1.61133s (22.3418GiB/s) | 20260507 16:17:01.987 | INFO | Downloaded 34GiB to GPU in 1.5311s (22.2063GiB/s) | 20260507 16:17:07.125 | INFO | Analyzing chunk frames = [866304,866816) | 20260507 16:17:07.125 | INFO | MemInfo: { | 20260507 16:17:07.125 | INFO | "gpu" : " 17 GB", | 20260507 16:17:07.125 | INFO | "huge" : "", | 20260507 16:17:07.125 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:07.125 | INFO | "malloc" : " 14 GB", | 20260507 16:17:07.126 | INFO | "pinned" : " 29 GB" | 20260507 16:17:07.126 | INFO | } | 20260507 16:17:07.126 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:17:07.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:07.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:17:07.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:17:07.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:07.126 | INFO | Uploaded 36GiB to GPU in 1.60876s (22.3775GiB/s) | 20260507 16:17:07.126 | INFO | Downloaded 34GiB to GPU in 1.52848s (22.2443GiB/s) | 20260507 16:17:12.265 | INFO | Analyzing chunk frames = [866816,867328) | 20260507 16:17:12.265 | INFO | MemInfo: { | 20260507 16:17:12.265 | INFO | "gpu" : " 17 GB", | 20260507 16:17:12.265 | INFO | "huge" : "", | 20260507 16:17:12.265 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:12.265 | INFO | "malloc" : " 14 GB", | 20260507 16:17:12.265 | INFO | "pinned" : " 29 GB" | 20260507 16:17:12.265 | INFO | } | 20260507 16:17:12.265 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:17:12.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:12.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:17:12.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:17:12.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:12.265 | INFO | Uploaded 36GiB to GPU in 1.60863s (22.3792GiB/s) | 20260507 16:17:12.265 | INFO | Downloaded 34GiB to GPU in 1.52827s (22.2474GiB/s) | 20260507 16:17:17.404 | INFO | Analyzing chunk frames = [867328,867840) | 20260507 16:17:17.404 | INFO | MemInfo: { | 20260507 16:17:17.404 | INFO | "gpu" : " 17 GB", | 20260507 16:17:17.404 | INFO | "huge" : "", | 20260507 16:17:17.404 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:17.404 | INFO | "malloc" : " 14 GB", | 20260507 16:17:17.404 | INFO | "pinned" : " 29 GB" | 20260507 16:17:17.404 | INFO | } | 20260507 16:17:17.405 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:17:17.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:17.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:17:17.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:17:17.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:17.405 | INFO | Uploaded 36GiB to GPU in 1.6095s (22.3671GiB/s) | 20260507 16:17:17.405 | INFO | Downloaded 34GiB to GPU in 1.52965s (22.2273GiB/s) | 20260507 16:17:22.544 | INFO | Analyzing chunk frames = [867840,868352) | 20260507 16:17:22.544 | INFO | MemInfo: { | 20260507 16:17:22.544 | INFO | "gpu" : " 17 GB", | 20260507 16:17:22.544 | INFO | "huge" : "", | 20260507 16:17:22.544 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:22.544 | INFO | "malloc" : " 14 GB", | 20260507 16:17:22.544 | INFO | "pinned" : " 29 GB" | 20260507 16:17:22.544 | INFO | } | 20260507 16:17:22.544 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:17:22.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:22.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:17:22.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 16:17:22.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:22.544 | INFO | Uploaded 36GiB to GPU in 1.60915s (22.3721GiB/s) | 20260507 16:17:22.544 | INFO | Downloaded 34GiB to GPU in 1.52952s (22.2292GiB/s) | 20260507 16:17:22.544 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:17:22.544 | INFO | Total mem usage: 16.4 GiB | 20260507 16:17:22.544 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:17:22.544 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.94%) | 20260507 16:17:22.544 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:17:22.544 | INFO | High Water Breakdown: | 20260507 16:17:22.544 | INFO | 59.4% : 8.86 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:17:22.544 | INFO | 40.5% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:17:22.544 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:17:22.545 | INFO | ------------------------------------------ | 20260507 16:17:22.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:17:22.545 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:17:22.545 | INFO | Total mem usage: 26.5 GiB | 20260507 16:17:22.545 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:17:22.545 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:17:22.545 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:17:22.545 | INFO | High Water Breakdown: | 20260507 16:17:22.545 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:17:22.545 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:17:22.545 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:17:22.545 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:17:22.545 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:17:22.545 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:17:22.545 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:17:22.545 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:17:22.545 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:17:22.545 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:17:22.545 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:17:22.545 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:17:22.545 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:17:22.545 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:17:22.545 | INFO | ------------------------------------------ | 20260507 16:17:22.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:17:22.545 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:17:22.545 | INFO | Total mem usage: 27.8 GiB | 20260507 16:17:22.545 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:17:22.545 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:17:22.545 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:17:22.545 | INFO | High Water Breakdown: | 20260507 16:17:22.545 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:17:22.545 | INFO | ------------------------------------------ | 20260507 16:17:22.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:17:22.545 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:17:22.545 | INFO | Total mem usage: 15.7 GiB | 20260507 16:17:22.545 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:17:22.545 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:17:22.545 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:17:22.545 | INFO | High Water Breakdown: | 20260507 16:17:22.545 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:17:22.545 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:17:22.545 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:17:22.545 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:17:22.545 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:17:22.545 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:17:22.545 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:17:22.545 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:17:22.545 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:17:22.545 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:17:22.545 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:17:22.545 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:17:22.545 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:17:22.545 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:17:22.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:17:22.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:17:22.545 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:17:22.545 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:17:22.545 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:17:22.545 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:17:22.545 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:17:22.545 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:17:22.545 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:17:22.545 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:17:22.545 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:17:22.545 | INFO | ------------------------------------------ | 20260507 16:17:22.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:17:22.545 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:17:22.545 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:17:22.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:17:22.545 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260507 16:17:22.545 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.25e-03 / 0.00e+00 | 20260507 16:17:22.545 | INFO | PerfReport: 11.07%, Baseline -- 3.29e+00 / -nan, 1.91e-02 / 0.00e+00 | 20260507 16:17:22.545 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.09e-02 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 16.17%, DME -- 3.85e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.11e+00 / -nan, 1.17e-01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 3.37%, Metrics -- 1.00e+00 / -nan, 1.65e-01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.66e-01 / -nan, 5.41e-02 / 0.00e+00 | 20260507 16:17:22.546 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:17:22.546 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:17:22.546 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:17:22.546 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 18.74%, Upload -- 1.67e+01 / 0.00e+00, 4.59e-01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 18.20%, Download -- 1.62e+01 / 0.00e+00, 7.78e-01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 33.37%, Compute -- 2.98e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 0.57%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 0.00%, LockDownload -- 4.30e-04 / 0.00e+00, 8.88e-05 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 29.11%, LockCompute -- 2.60e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 16:17:22.546 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 8.98e-03 / 0.00e+00 | 20260507 16:17:22.641 | INFO | Starting a round of Baz IO | 20260507 16:17:25.512 | INFO | Populating RTMetrics report | 20260507 16:17:25.512 | INFO | RTMetrics report populated | 20260507 16:17:25.512 | INFO | Opening tmp RTMetrics json file | 20260507 16:17:25.593 | INFO | Serializing RTMetrics to json | 20260507 16:17:25.617 | INFO | Serialization done, swapping tmp file now... | 20260507 16:17:25.642 | INFO | RTMetrics done | 20260507 16:17:25.800 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.159 seconds | 20260507 16:17:25.800 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.67% / 0.03% / 49.30% | 20260507 16:17:25.800 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.69285% / 6.35783e-05% / 2.78155e-05% | 20260507 16:17:25.800 | INFO | Baz IO overall: | 20260507 16:17:25.800 | INFO | BytesWritten : 369733170929 | 20260507 16:17:25.800 | INFO | NumEvents : 178784910524 | 20260507 16:17:27.684 | INFO | Analyzing chunk frames = [868352,868864) | 20260507 16:17:27.684 | INFO | MemInfo: { | 20260507 16:17:27.684 | INFO | "gpu" : " 17 GB", | 20260507 16:17:27.684 | INFO | "huge" : "", | 20260507 16:17:27.684 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:27.684 | INFO | "malloc" : " 14 GB", | 20260507 16:17:27.684 | INFO | "pinned" : " 29 GB" | 20260507 16:17:27.684 | INFO | } | 20260507 16:17:27.684 | INFO | Perf report for the last 5.14015 seconds: | 20260507 16:17:27.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:27.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:27.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:27.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:27.684 | INFO | ANALYSIS : 56.9%, 2.9, 85.5ms, 0 | 20260507 16:17:27.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:27.684 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 16:17:27.684 | INFO | PRE_HQ : 55.6%, 2.0, 58.5ms, 0 | 20260507 16:17:27.684 | INFO | BAZWRITER : 61.1%, 4.7, 148.8ms, 0 | 20260507 16:17:27.684 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:17:27.684 | INFO | Uploaded 36GiB to GPU in 1.66549s (21.6152GiB/s) | 20260507 16:17:27.684 | INFO | Downloaded 38GiB to GPU in 1.7505s (21.7081GiB/s) | 20260507 16:17:32.822 | INFO | Analyzing chunk frames = [868864,869376) | 20260507 16:17:32.822 | INFO | MemInfo: { | 20260507 16:17:32.822 | INFO | "gpu" : " 17 GB", | 20260507 16:17:32.822 | INFO | "huge" : "", | 20260507 16:17:32.822 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:32.822 | INFO | "malloc" : " 14 GB", | 20260507 16:17:32.823 | INFO | "pinned" : " 29 GB" | 20260507 16:17:32.823 | INFO | } | 20260507 16:17:32.823 | INFO | Perf report for the last 5.13852 seconds: | 20260507 16:17:32.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:32.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 16:17:32.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:17:32.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:32.823 | INFO | Uploaded 36GiB to GPU in 1.58667s (22.689GiB/s) | 20260507 16:17:32.823 | INFO | Downloaded 34GiB to GPU in 1.5347s (22.1542GiB/s) | 20260507 16:17:37.962 | INFO | Analyzing chunk frames = [869376,869888) | 20260507 16:17:37.962 | INFO | MemInfo: { | 20260507 16:17:37.962 | INFO | "gpu" : " 17 GB", | 20260507 16:17:37.962 | INFO | "huge" : "", | 20260507 16:17:37.962 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:37.962 | INFO | "malloc" : " 14 GB", | 20260507 16:17:37.962 | INFO | "pinned" : " 29 GB" | 20260507 16:17:37.963 | INFO | } | 20260507 16:17:37.963 | INFO | Perf report for the last 5.14 seconds: | 20260507 16:17:37.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:37.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:17:37.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:17:37.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:37.963 | INFO | Uploaded 36GiB to GPU in 1.59078s (22.6304GiB/s) | 20260507 16:17:37.963 | INFO | Downloaded 34GiB to GPU in 1.5398s (22.0807GiB/s) | 20260507 16:17:43.101 | INFO | Analyzing chunk frames = [869888,870400) | 20260507 16:17:43.101 | INFO | MemInfo: { | 20260507 16:17:43.101 | INFO | "gpu" : " 17 GB", | 20260507 16:17:43.101 | INFO | "huge" : "", | 20260507 16:17:43.101 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:43.101 | INFO | "malloc" : " 14 GB", | 20260507 16:17:43.101 | INFO | "pinned" : " 29 GB" | 20260507 16:17:43.101 | INFO | } | 20260507 16:17:43.101 | INFO | Perf report for the last 5.13867 seconds: | 20260507 16:17:43.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:43.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.101 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:17:43.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.101 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:17:43.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:43.102 | INFO | Uploaded 36GiB to GPU in 1.58947s (22.649GiB/s) | 20260507 16:17:43.102 | INFO | Downloaded 34GiB to GPU in 1.54018s (22.0754GiB/s) | 20260507 16:17:48.240 | INFO | Analyzing chunk frames = [870400,870912) | 20260507 16:17:48.240 | INFO | MemInfo: { | 20260507 16:17:48.240 | INFO | "gpu" : " 17 GB", | 20260507 16:17:48.240 | INFO | "huge" : "", | 20260507 16:17:48.240 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:48.240 | INFO | "malloc" : " 14 GB", | 20260507 16:17:48.240 | INFO | "pinned" : " 29 GB" | 20260507 16:17:48.240 | INFO | } | 20260507 16:17:48.240 | INFO | Perf report for the last 5.13905 seconds: | 20260507 16:17:48.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:48.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:17:48.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:17:48.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:48.241 | INFO | Uploaded 36GiB to GPU in 1.5876s (22.6757GiB/s) | 20260507 16:17:48.241 | INFO | Downloaded 34GiB to GPU in 1.53801s (22.1066GiB/s) | 20260507 16:17:53.380 | INFO | Analyzing chunk frames = [870912,871424) | 20260507 16:17:53.380 | INFO | MemInfo: { | 20260507 16:17:53.380 | INFO | "gpu" : " 17 GB", | 20260507 16:17:53.380 | INFO | "huge" : "", | 20260507 16:17:53.380 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:53.380 | INFO | "malloc" : " 14 GB", | 20260507 16:17:53.380 | INFO | "pinned" : " 29 GB" | 20260507 16:17:53.380 | INFO | } | 20260507 16:17:53.380 | INFO | Perf report for the last 5.13964 seconds: | 20260507 16:17:53.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:53.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:17:53.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:17:53.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:53.380 | INFO | Uploaded 36GiB to GPU in 1.58851s (22.6627GiB/s) | 20260507 16:17:53.380 | INFO | Downloaded 34GiB to GPU in 1.53826s (22.1028GiB/s) | 20260507 16:17:58.519 | INFO | Analyzing chunk frames = [871424,871936) | 20260507 16:17:58.519 | INFO | MemInfo: { | 20260507 16:17:58.519 | INFO | "gpu" : " 17 GB", | 20260507 16:17:58.519 | INFO | "huge" : "", | 20260507 16:17:58.519 | INFO | "huge_shared" : " 29 GB", | 20260507 16:17:58.519 | INFO | "malloc" : " 14 GB", | 20260507 16:17:58.519 | INFO | "pinned" : " 29 GB" | 20260507 16:17:58.519 | INFO | } | 20260507 16:17:58.519 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:17:58.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:17:58.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.519 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 16:17:58.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.519 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:17:58.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:17:58.520 | INFO | Uploaded 36GiB to GPU in 1.5885s (22.6629GiB/s) | 20260507 16:17:58.520 | INFO | Downloaded 34GiB to GPU in 1.53821s (22.1036GiB/s) | 20260507 16:18:03.658 | INFO | Analyzing chunk frames = [871936,872448) | 20260507 16:18:03.658 | INFO | MemInfo: { | 20260507 16:18:03.658 | INFO | "gpu" : " 17 GB", | 20260507 16:18:03.658 | INFO | "huge" : "", | 20260507 16:18:03.658 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:03.658 | INFO | "malloc" : " 14 GB", | 20260507 16:18:03.658 | INFO | "pinned" : " 29 GB" | 20260507 16:18:03.659 | INFO | } | 20260507 16:18:03.659 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:18:03.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:03.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:18:03.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:18:03.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:03.659 | INFO | Uploaded 36GiB to GPU in 1.58887s (22.6576GiB/s) | 20260507 16:18:03.659 | INFO | Downloaded 34GiB to GPU in 1.53884s (22.0946GiB/s) | 20260507 16:18:03.756 | INFO | Starting a round of Baz IO | 20260507 16:18:06.628 | INFO | Populating RTMetrics report | 20260507 16:18:06.628 | INFO | RTMetrics report populated | 20260507 16:18:06.629 | INFO | Opening tmp RTMetrics json file | 20260507 16:18:06.629 | INFO | Serializing RTMetrics to json | 20260507 16:18:06.632 | INFO | Serialization done, swapping tmp file now... | 20260507 16:18:06.632 | INFO | RTMetrics done | 20260507 16:18:06.724 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 2.968 seconds | 20260507 16:18:06.725 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.64% / 0.03% / 49.33% | 20260507 16:18:06.725 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.68409% / 4.76837e-05% / 1.98682e-05% | 20260507 16:18:06.725 | INFO | Baz IO overall: | 20260507 16:18:06.725 | INFO | BytesWritten : 371645561648 | 20260507 16:18:06.725 | INFO | NumEvents : 179715808031 | 20260507 16:18:08.798 | INFO | Analyzing chunk frames = [872448,872960) | 20260507 16:18:08.798 | INFO | MemInfo: { | 20260507 16:18:08.798 | INFO | "gpu" : " 17 GB", | 20260507 16:18:08.798 | INFO | "huge" : "", | 20260507 16:18:08.798 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:08.798 | INFO | "malloc" : " 14 GB", | 20260507 16:18:08.798 | INFO | "pinned" : " 29 GB" | 20260507 16:18:08.798 | INFO | } | 20260507 16:18:08.798 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:18:08.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:08.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:08.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:08.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:08.798 | INFO | ANALYSIS : 56.9%, 3.0, 88.0ms, 0 | 20260507 16:18:08.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:08.798 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 16:18:08.798 | INFO | PRE_HQ : 55.7%, 2.1, 59.5ms, 0 | 20260507 16:18:08.798 | INFO | BAZWRITER : 57.2%, 4.4, 128.1ms, 0 | 20260507 16:18:08.798 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:18:08.798 | INFO | Uploaded 36GiB to GPU in 1.63104s (22.0718GiB/s) | 20260507 16:18:08.798 | INFO | Downloaded 38GiB to GPU in 1.71563s (22.1492GiB/s) | 20260507 16:18:13.937 | INFO | Analyzing chunk frames = [872960,873472) | 20260507 16:18:13.937 | INFO | MemInfo: { | 20260507 16:18:13.937 | INFO | "gpu" : " 17 GB", | 20260507 16:18:13.937 | INFO | "huge" : "", | 20260507 16:18:13.937 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:13.937 | INFO | "malloc" : " 14 GB", | 20260507 16:18:13.937 | INFO | "pinned" : " 29 GB" | 20260507 16:18:13.937 | INFO | } | 20260507 16:18:13.937 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:18:13.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:13.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.937 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 16:18:13.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.937 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:18:13.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:13.938 | INFO | Uploaded 36GiB to GPU in 1.58601s (22.6985GiB/s) | 20260507 16:18:13.938 | INFO | Downloaded 34GiB to GPU in 1.53472s (22.1539GiB/s) | 20260507 16:18:19.076 | INFO | Analyzing chunk frames = [873472,873984) | 20260507 16:18:19.076 | INFO | MemInfo: { | 20260507 16:18:19.076 | INFO | "gpu" : " 17 GB", | 20260507 16:18:19.076 | INFO | "huge" : "", | 20260507 16:18:19.076 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:19.076 | INFO | "malloc" : " 14 GB", | 20260507 16:18:19.076 | INFO | "pinned" : " 29 GB" | 20260507 16:18:19.077 | INFO | } | 20260507 16:18:19.077 | INFO | Perf report for the last 5.13917 seconds: | 20260507 16:18:19.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:19.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | ANALYSIS : 56.0%, 3.0, 87.1ms, 0 | 20260507 16:18:19.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:18:19.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:19.077 | INFO | Uploaded 36GiB to GPU in 1.58984s (22.6438GiB/s) | 20260507 16:18:19.077 | INFO | Downloaded 34GiB to GPU in 1.53987s (22.0798GiB/s) | 20260507 16:18:24.216 | INFO | Analyzing chunk frames = [873984,874496) | 20260507 16:18:24.216 | INFO | MemInfo: { | 20260507 16:18:24.216 | INFO | "gpu" : " 17 GB", | 20260507 16:18:24.216 | INFO | "huge" : "", | 20260507 16:18:24.216 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:24.216 | INFO | "malloc" : " 14 GB", | 20260507 16:18:24.216 | INFO | "pinned" : " 29 GB" | 20260507 16:18:24.216 | INFO | } | 20260507 16:18:24.216 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:18:24.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:24.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.216 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:18:24.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.216 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:18:24.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:24.217 | INFO | Uploaded 36GiB to GPU in 1.59071s (22.6314GiB/s) | 20260507 16:18:24.217 | INFO | Downloaded 34GiB to GPU in 1.53938s (22.0868GiB/s) | 20260507 16:18:29.355 | INFO | Analyzing chunk frames = [874496,875008) | 20260507 16:18:29.355 | INFO | MemInfo: { | 20260507 16:18:29.355 | INFO | "gpu" : " 17 GB", | 20260507 16:18:29.355 | INFO | "huge" : "", | 20260507 16:18:29.355 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:29.355 | INFO | "malloc" : " 14 GB", | 20260507 16:18:29.355 | INFO | "pinned" : " 29 GB" | 20260507 16:18:29.355 | INFO | } | 20260507 16:18:29.355 | INFO | Perf report for the last 5.13907 seconds: | 20260507 16:18:29.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:29.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.355 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:18:29.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.355 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:18:29.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:29.356 | INFO | Uploaded 36GiB to GPU in 1.5887s (22.6601GiB/s) | 20260507 16:18:29.356 | INFO | Downloaded 34GiB to GPU in 1.53801s (22.1065GiB/s) | 20260507 16:18:34.494 | INFO | Analyzing chunk frames = [875008,875520) | 20260507 16:18:34.494 | INFO | MemInfo: { | 20260507 16:18:34.494 | INFO | "gpu" : " 17 GB", | 20260507 16:18:34.494 | INFO | "huge" : "", | 20260507 16:18:34.494 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:34.494 | INFO | "malloc" : " 14 GB", | 20260507 16:18:34.494 | INFO | "pinned" : " 29 GB" | 20260507 16:18:34.495 | INFO | } | 20260507 16:18:34.495 | INFO | Perf report for the last 5.13965 seconds: | 20260507 16:18:34.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:34.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:18:34.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:18:34.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:34.495 | INFO | Uploaded 36GiB to GPU in 1.58957s (22.6476GiB/s) | 20260507 16:18:34.495 | INFO | Downloaded 34GiB to GPU in 1.53728s (22.1169GiB/s) | 20260507 16:18:39.634 | INFO | Analyzing chunk frames = [875520,876032) | 20260507 16:18:39.634 | INFO | MemInfo: { | 20260507 16:18:39.634 | INFO | "gpu" : " 17 GB", | 20260507 16:18:39.634 | INFO | "huge" : "", | 20260507 16:18:39.634 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:39.634 | INFO | "malloc" : " 14 GB", | 20260507 16:18:39.634 | INFO | "pinned" : " 29 GB" | 20260507 16:18:39.634 | INFO | } | 20260507 16:18:39.634 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:18:39.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:39.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.634 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:18:39.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.634 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:18:39.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:39.635 | INFO | Uploaded 36GiB to GPU in 1.58876s (22.6593GiB/s) | 20260507 16:18:39.635 | INFO | Downloaded 34GiB to GPU in 1.53844s (22.1003GiB/s) | 20260507 16:18:44.773 | INFO | Analyzing chunk frames = [876032,876544) | 20260507 16:18:44.773 | INFO | MemInfo: { | 20260507 16:18:44.773 | INFO | "gpu" : " 17 GB", | 20260507 16:18:44.773 | INFO | "huge" : "", | 20260507 16:18:44.773 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:44.773 | INFO | "malloc" : " 14 GB", | 20260507 16:18:44.773 | INFO | "pinned" : " 29 GB" | 20260507 16:18:44.773 | INFO | } | 20260507 16:18:44.773 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:18:44.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:44.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.773 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:18:44.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.773 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:18:44.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:44.774 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:18:44.774 | INFO | Total mem usage: 16.4 GiB | 20260507 16:18:44.774 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:18:44.774 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.25%) | 20260507 16:18:44.774 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:18:44.774 | INFO | High Water Breakdown: | 20260507 16:18:44.774 | INFO | 59.5% : 8.93 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:18:44.774 | INFO | 40.4% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:18:44.774 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:18:44.774 | INFO | ------------------------------------------ | 20260507 16:18:44.774 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:18:44.774 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:18:44.774 | INFO | Total mem usage: 26.5 GiB | 20260507 16:18:44.774 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:18:44.774 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:18:44.774 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:18:44.774 | INFO | High Water Breakdown: | 20260507 16:18:44.774 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:18:44.774 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:18:44.774 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:18:44.774 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:18:44.774 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:18:44.774 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:18:44.774 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:18:44.774 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:18:44.774 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:18:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:18:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:18:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:18:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:18:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:18:44.774 | INFO | ------------------------------------------ | 20260507 16:18:44.774 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:18:44.774 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:18:44.774 | INFO | Total mem usage: 27.8 GiB | 20260507 16:18:44.774 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:18:44.774 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:18:44.774 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:18:44.774 | INFO | High Water Breakdown: | 20260507 16:18:44.774 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:18:44.774 | INFO | ------------------------------------------ | 20260507 16:18:44.774 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:18:44.774 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:18:44.774 | INFO | Total mem usage: 15.7 GiB | 20260507 16:18:44.774 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:18:44.774 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:18:44.774 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:18:44.774 | INFO | High Water Breakdown: | 20260507 16:18:44.774 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:18:44.774 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:18:44.774 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:18:44.774 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:18:44.774 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:18:44.774 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:18:44.774 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:18:44.774 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:18:44.774 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:18:44.774 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:18:44.774 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:18:44.774 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:18:44.774 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:18:44.774 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:18:44.774 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:18:44.774 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:18:44.774 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:18:44.774 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:18:44.774 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:18:44.774 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:18:44.774 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:18:44.774 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:18:44.774 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:18:44.774 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:18:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:18:44.774 | INFO | ------------------------------------------ | 20260507 16:18:44.774 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:18:44.774 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:18:44.774 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:18:44.774 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:18:44.774 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.73e-03 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 11.07%, Baseline -- 3.29e+00 / -nan, 1.85e-02 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 3.09%, Histogram -- 9.18e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.11e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 3.37%, Metrics -- 1.00e+00 / -nan, 1.66e-01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.66e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 16:18:44.774 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:18:44.774 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:18:44.774 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:18:44.774 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 18.65%, Upload -- 1.66e+01 / 0.00e+00, 4.74e-01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 18.26%, Download -- 1.63e+01 / 0.00e+00, 7.67e-01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | PerfReport: 33.38%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:18:44.774 | INFO | Uploaded 36GiB to GPU in 1.58881s (22.6585GiB/s) | 20260507 16:18:44.774 | INFO | Downloaded 34GiB to GPU in 1.53895s (22.093GiB/s) | 20260507 16:18:44.774 | INFO | PerfReport: 0.57%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 16:18:44.775 | INFO | PerfReport: 0.00%, LockDownload -- 4.41e-04 / 0.00e+00, 9.22e-05 / 0.00e+00 | 20260507 16:18:44.775 | INFO | PerfReport: 29.13%, LockCompute -- 2.60e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 16:18:44.775 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 2.82e-02 / 0.00e+00 | 20260507 16:18:44.871 | INFO | Starting a round of Baz IO | 20260507 16:18:47.743 | INFO | Populating RTMetrics report | 20260507 16:18:47.743 | INFO | RTMetrics report populated | 20260507 16:18:47.743 | INFO | Opening tmp RTMetrics json file | 20260507 16:18:47.750 | INFO | Serializing RTMetrics to json | 20260507 16:18:47.754 | INFO | Serialization done, swapping tmp file now... | 20260507 16:18:47.758 | INFO | RTMetrics done | 20260507 16:18:48.003 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.131 seconds | 20260507 16:18:48.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.61% / 0.03% / 49.36% | 20260507 16:18:48.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.65299% / 5.5631e-05% / 3.57628e-05% | 20260507 16:18:48.003 | INFO | Baz IO overall: | 20260507 16:18:48.003 | INFO | BytesWritten : 373557021641 | 20260507 16:18:48.003 | INFO | NumEvents : 180645826813 | 20260507 16:18:49.912 | INFO | Analyzing chunk frames = [876544,877056) | 20260507 16:18:49.912 | INFO | MemInfo: { | 20260507 16:18:49.912 | INFO | "gpu" : " 17 GB", | 20260507 16:18:49.912 | INFO | "huge" : "", | 20260507 16:18:49.912 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:49.912 | INFO | "malloc" : " 14 GB", | 20260507 16:18:49.913 | INFO | "pinned" : " 29 GB" | 20260507 16:18:49.913 | INFO | } | 20260507 16:18:49.913 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:18:49.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:49.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:49.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:49.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:49.913 | INFO | ANALYSIS : 56.9%, 2.9, 85.6ms, 0 | 20260507 16:18:49.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:49.913 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 16:18:49.913 | INFO | PRE_HQ : 55.8%, 2.0, 58.7ms, 0 | 20260507 16:18:49.913 | INFO | BAZWRITER : 60.5%, 4.7, 146.9ms, 0 | 20260507 16:18:49.913 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 16:18:49.913 | INFO | Uploaded 36GiB to GPU in 1.66524s (21.6185GiB/s) | 20260507 16:18:49.913 | INFO | Downloaded 38GiB to GPU in 1.75108s (21.701GiB/s) | 20260507 16:18:55.052 | INFO | Analyzing chunk frames = [877056,877568) | 20260507 16:18:55.052 | INFO | MemInfo: { | 20260507 16:18:55.052 | INFO | "gpu" : " 17 GB", | 20260507 16:18:55.052 | INFO | "huge" : "", | 20260507 16:18:55.052 | INFO | "huge_shared" : " 29 GB", | 20260507 16:18:55.052 | INFO | "malloc" : " 14 GB", | 20260507 16:18:55.052 | INFO | "pinned" : " 29 GB" | 20260507 16:18:55.052 | INFO | } | 20260507 16:18:55.052 | INFO | Perf report for the last 5.13964 seconds: | 20260507 16:18:55.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:18:55.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.052 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 16:18:55.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.052 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:18:55.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:18:55.053 | INFO | Uploaded 36GiB to GPU in 1.58671s (22.6885GiB/s) | 20260507 16:18:55.053 | INFO | Downloaded 34GiB to GPU in 1.53452s (22.1567GiB/s) | 20260507 16:19:00.191 | INFO | Analyzing chunk frames = [877568,878080) | 20260507 16:19:00.191 | INFO | MemInfo: { | 20260507 16:19:00.191 | INFO | "gpu" : " 17 GB", | 20260507 16:19:00.191 | INFO | "huge" : "", | 20260507 16:19:00.191 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:00.191 | INFO | "malloc" : " 14 GB", | 20260507 16:19:00.191 | INFO | "pinned" : " 29 GB" | 20260507 16:19:00.191 | INFO | } | 20260507 16:19:00.191 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:19:00.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:00.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.192 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:19:00.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.192 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:19:00.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:00.192 | INFO | Uploaded 36GiB to GPU in 1.59014s (22.6396GiB/s) | 20260507 16:19:00.192 | INFO | Downloaded 34GiB to GPU in 1.54016s (22.0756GiB/s) | 20260507 16:19:05.330 | INFO | Analyzing chunk frames = [878080,878592) | 20260507 16:19:05.330 | INFO | MemInfo: { | 20260507 16:19:05.330 | INFO | "gpu" : " 17 GB", | 20260507 16:19:05.330 | INFO | "huge" : "", | 20260507 16:19:05.330 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:05.330 | INFO | "malloc" : " 14 GB", | 20260507 16:19:05.330 | INFO | "pinned" : " 29 GB" | 20260507 16:19:05.330 | INFO | } | 20260507 16:19:05.331 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:19:05.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:05.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:19:05.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:19:05.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:05.331 | INFO | Uploaded 36GiB to GPU in 1.5917s (22.6173GiB/s) | 20260507 16:19:05.331 | INFO | Downloaded 34GiB to GPU in 1.54013s (22.0761GiB/s) | 20260507 16:19:10.470 | INFO | Analyzing chunk frames = [878592,879104) | 20260507 16:19:10.470 | INFO | MemInfo: { | 20260507 16:19:10.470 | INFO | "gpu" : " 17 GB", | 20260507 16:19:10.470 | INFO | "huge" : "", | 20260507 16:19:10.470 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:10.470 | INFO | "malloc" : " 14 GB", | 20260507 16:19:10.470 | INFO | "pinned" : " 29 GB" | 20260507 16:19:10.470 | INFO | } | 20260507 16:19:10.470 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:19:10.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:10.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.470 | INFO | ANALYSIS : 55.7%, 3.0, 87.0ms, 0 | 20260507 16:19:10.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.470 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:19:10.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:10.471 | INFO | Uploaded 36GiB to GPU in 1.58901s (22.6556GiB/s) | 20260507 16:19:10.471 | INFO | Downloaded 34GiB to GPU in 1.53807s (22.1057GiB/s) | 20260507 16:19:15.609 | INFO | Analyzing chunk frames = [879104,879616) | 20260507 16:19:15.609 | INFO | MemInfo: { | 20260507 16:19:15.609 | INFO | "gpu" : " 17 GB", | 20260507 16:19:15.609 | INFO | "huge" : "", | 20260507 16:19:15.609 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:15.609 | INFO | "malloc" : " 14 GB", | 20260507 16:19:15.609 | INFO | "pinned" : " 29 GB" | 20260507 16:19:15.609 | INFO | } | 20260507 16:19:15.609 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:19:15.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:15.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:19:15.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:19:15.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:15.610 | INFO | Uploaded 36GiB to GPU in 1.58998s (22.6417GiB/s) | 20260507 16:19:15.610 | INFO | Downloaded 34GiB to GPU in 1.53775s (22.1103GiB/s) | 20260507 16:19:20.749 | INFO | Analyzing chunk frames = [879616,880128) | 20260507 16:19:20.749 | INFO | MemInfo: { | 20260507 16:19:20.749 | INFO | "gpu" : " 17 GB", | 20260507 16:19:20.749 | INFO | "huge" : "", | 20260507 16:19:20.749 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:20.749 | INFO | "malloc" : " 14 GB", | 20260507 16:19:20.749 | INFO | "pinned" : " 29 GB" | 20260507 16:19:20.749 | INFO | } | 20260507 16:19:20.749 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:19:20.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:20.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:19:20.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:19:20.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:20.749 | INFO | Uploaded 36GiB to GPU in 1.58867s (22.6604GiB/s) | 20260507 16:19:20.749 | INFO | Downloaded 34GiB to GPU in 1.53744s (22.1148GiB/s) | 20260507 16:19:25.888 | INFO | Analyzing chunk frames = [880128,880640) | 20260507 16:19:25.888 | INFO | MemInfo: { | 20260507 16:19:25.888 | INFO | "gpu" : " 17 GB", | 20260507 16:19:25.888 | INFO | "huge" : "", | 20260507 16:19:25.888 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:25.888 | INFO | "malloc" : " 14 GB", | 20260507 16:19:25.888 | INFO | "pinned" : " 29 GB" | 20260507 16:19:25.888 | INFO | } | 20260507 16:19:25.888 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:19:25.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:25.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.889 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:19:25.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.889 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:19:25.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:25.889 | INFO | Uploaded 36GiB to GPU in 1.58931s (22.6513GiB/s) | 20260507 16:19:25.889 | INFO | Downloaded 34GiB to GPU in 1.53902s (22.0919GiB/s) | 20260507 16:19:25.985 | INFO | Starting a round of Baz IO | 20260507 16:19:28.862 | INFO | Populating RTMetrics report | 20260507 16:19:28.862 | INFO | RTMetrics report populated | 20260507 16:19:28.862 | INFO | Opening tmp RTMetrics json file | 20260507 16:19:28.908 | INFO | Serializing RTMetrics to json | 20260507 16:19:28.964 | INFO | Serialization done, swapping tmp file now... | 20260507 16:19:28.965 | INFO | RTMetrics done | 20260507 16:19:29.164 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.178 seconds | 20260507 16:19:29.164 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.58% / 0.03% / 49.39% | 20260507 16:19:29.164 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.65379% / 6.75519e-05% / 2.78155e-05% | 20260507 16:19:29.164 | INFO | Baz IO overall: | 20260507 16:19:29.164 | INFO | BytesWritten : 375469831496 | 20260507 16:19:29.164 | INFO | NumEvents : 181577142937 | 20260507 16:19:31.027 | INFO | Analyzing chunk frames = [880640,881152) | 20260507 16:19:31.027 | INFO | MemInfo: { | 20260507 16:19:31.027 | INFO | "gpu" : " 17 GB", | 20260507 16:19:31.027 | INFO | "huge" : "", | 20260507 16:19:31.027 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:31.027 | INFO | "malloc" : " 14 GB", | 20260507 16:19:31.027 | INFO | "pinned" : " 29 GB" | 20260507 16:19:31.027 | INFO | } | 20260507 16:19:31.027 | INFO | Perf report for the last 5.1388 seconds: | 20260507 16:19:31.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:31.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:31.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:31.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:31.027 | INFO | ANALYSIS : 56.9%, 2.9, 87.3ms, 0 | 20260507 16:19:31.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:31.027 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 16:19:31.027 | INFO | PRE_HQ : 55.6%, 2.0, 58.9ms, 0 | 20260507 16:19:31.027 | INFO | BAZWRITER : 61.4%, 4.5, 140.9ms, 0 | 20260507 16:19:31.027 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 16:19:31.028 | INFO | Uploaded 36GiB to GPU in 1.64904s (21.8308GiB/s) | 20260507 16:19:31.028 | INFO | Downloaded 38GiB to GPU in 1.73264s (21.9318GiB/s) | 20260507 16:19:36.167 | INFO | Analyzing chunk frames = [881152,881664) | 20260507 16:19:36.167 | INFO | MemInfo: { | 20260507 16:19:36.167 | INFO | "gpu" : " 17 GB", | 20260507 16:19:36.167 | INFO | "huge" : "", | 20260507 16:19:36.167 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:36.167 | INFO | "malloc" : " 14 GB", | 20260507 16:19:36.167 | INFO | "pinned" : " 29 GB" | 20260507 16:19:36.167 | INFO | } | 20260507 16:19:36.167 | INFO | Perf report for the last 5.13958 seconds: | 20260507 16:19:36.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:36.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 16:19:36.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 16:19:36.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:36.167 | INFO | Uploaded 36GiB to GPU in 1.58688s (22.686GiB/s) | 20260507 16:19:36.167 | INFO | Downloaded 34GiB to GPU in 1.53567s (22.1402GiB/s) | 20260507 16:19:41.306 | INFO | Analyzing chunk frames = [881664,882176) | 20260507 16:19:41.306 | INFO | MemInfo: { | 20260507 16:19:41.306 | INFO | "gpu" : " 17 GB", | 20260507 16:19:41.306 | INFO | "huge" : "", | 20260507 16:19:41.306 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:41.306 | INFO | "malloc" : " 14 GB", | 20260507 16:19:41.306 | INFO | "pinned" : " 29 GB" | 20260507 16:19:41.306 | INFO | } | 20260507 16:19:41.306 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:19:41.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:41.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | ANALYSIS : 56.1%, 3.0, 87.2ms, 0 | 20260507 16:19:41.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:19:41.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:41.306 | INFO | Uploaded 36GiB to GPU in 1.59012s (22.6398GiB/s) | 20260507 16:19:41.307 | INFO | Downloaded 34GiB to GPU in 1.54018s (22.0753GiB/s) | 20260507 16:19:46.445 | INFO | Analyzing chunk frames = [882176,882688) | 20260507 16:19:46.445 | INFO | MemInfo: { | 20260507 16:19:46.445 | INFO | "gpu" : " 17 GB", | 20260507 16:19:46.445 | INFO | "huge" : "", | 20260507 16:19:46.445 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:46.445 | INFO | "malloc" : " 14 GB", | 20260507 16:19:46.445 | INFO | "pinned" : " 29 GB" | 20260507 16:19:46.445 | INFO | } | 20260507 16:19:46.445 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:19:46.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:46.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.446 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:19:46.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.446 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:19:46.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:46.446 | INFO | Uploaded 36GiB to GPU in 1.59202s (22.6127GiB/s) | 20260507 16:19:46.446 | INFO | Downloaded 34GiB to GPU in 1.54004s (22.0773GiB/s) | 20260507 16:19:51.585 | INFO | Analyzing chunk frames = [882688,883200) | 20260507 16:19:51.585 | INFO | MemInfo: { | 20260507 16:19:51.585 | INFO | "gpu" : " 17 GB", | 20260507 16:19:51.585 | INFO | "huge" : "", | 20260507 16:19:51.585 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:51.585 | INFO | "malloc" : " 14 GB", | 20260507 16:19:51.585 | INFO | "pinned" : " 29 GB" | 20260507 16:19:51.585 | INFO | } | 20260507 16:19:51.585 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:19:51.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:51.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:19:51.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:19:51.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:51.585 | INFO | Uploaded 36GiB to GPU in 1.58873s (22.6596GiB/s) | 20260507 16:19:51.585 | INFO | Downloaded 34GiB to GPU in 1.53745s (22.1146GiB/s) | 20260507 16:19:56.724 | INFO | Analyzing chunk frames = [883200,883712) | 20260507 16:19:56.724 | INFO | MemInfo: { | 20260507 16:19:56.724 | INFO | "gpu" : " 17 GB", | 20260507 16:19:56.724 | INFO | "huge" : "", | 20260507 16:19:56.724 | INFO | "huge_shared" : " 29 GB", | 20260507 16:19:56.724 | INFO | "malloc" : " 14 GB", | 20260507 16:19:56.724 | INFO | "pinned" : " 29 GB" | 20260507 16:19:56.724 | INFO | } | 20260507 16:19:56.724 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:19:56.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:19:56.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.724 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 16:19:56.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.724 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:19:56.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:19:56.725 | INFO | Uploaded 36GiB to GPU in 1.58945s (22.6493GiB/s) | 20260507 16:19:56.725 | INFO | Downloaded 34GiB to GPU in 1.53696s (22.1216GiB/s) | 20260507 16:20:01.863 | INFO | Analyzing chunk frames = [883712,884224) | 20260507 16:20:01.863 | INFO | MemInfo: { | 20260507 16:20:01.863 | INFO | "gpu" : " 17 GB", | 20260507 16:20:01.863 | INFO | "huge" : "", | 20260507 16:20:01.863 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:01.863 | INFO | "malloc" : " 14 GB", | 20260507 16:20:01.863 | INFO | "pinned" : " 29 GB" | 20260507 16:20:01.863 | INFO | } | 20260507 16:20:01.863 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:20:01.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:01.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:20:01.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:20:01.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:01.864 | INFO | Uploaded 36GiB to GPU in 1.58967s (22.6462GiB/s) | 20260507 16:20:01.864 | INFO | Downloaded 34GiB to GPU in 1.53958s (22.0839GiB/s) | 20260507 16:20:07.003 | INFO | Analyzing chunk frames = [884224,884736) | 20260507 16:20:07.003 | INFO | MemInfo: { | 20260507 16:20:07.003 | INFO | "gpu" : " 17 GB", | 20260507 16:20:07.003 | INFO | "huge" : "", | 20260507 16:20:07.003 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:07.003 | INFO | "malloc" : " 14 GB", | 20260507 16:20:07.003 | INFO | "pinned" : " 29 GB" | 20260507 16:20:07.003 | INFO | } | 20260507 16:20:07.003 | INFO | Perf report for the last 5.13962 seconds: | 20260507 16:20:07.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:07.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:20:07.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 16:20:07.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:07.003 | INFO | Uploaded 36GiB to GPU in 1.58841s (22.6642GiB/s) | 20260507 16:20:07.003 | INFO | Downloaded 34GiB to GPU in 1.53951s (22.085GiB/s) | 20260507 16:20:07.004 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:20:07.004 | INFO | Total mem usage: 16.4 GiB | 20260507 16:20:07.004 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:20:07.004 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.42%) | 20260507 16:20:07.004 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:20:07.004 | INFO | High Water Breakdown: | 20260507 16:20:07.004 | INFO | 59.6% : 9.02 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:20:07.004 | INFO | 40.3% : 6.09 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:20:07.004 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:20:07.004 | INFO | ------------------------------------------ | 20260507 16:20:07.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:20:07.004 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:20:07.004 | INFO | Total mem usage: 26.5 GiB | 20260507 16:20:07.004 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:20:07.004 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:20:07.004 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:20:07.004 | INFO | High Water Breakdown: | 20260507 16:20:07.004 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:20:07.004 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:20:07.004 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:20:07.004 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:20:07.004 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:20:07.004 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:20:07.004 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:20:07.004 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:20:07.004 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:20:07.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:20:07.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:20:07.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:20:07.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:20:07.004 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:20:07.004 | INFO | ------------------------------------------ | 20260507 16:20:07.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:20:07.004 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:20:07.004 | INFO | Total mem usage: 27.8 GiB | 20260507 16:20:07.004 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:20:07.004 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:20:07.004 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:20:07.004 | INFO | High Water Breakdown: | 20260507 16:20:07.004 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:20:07.004 | INFO | ------------------------------------------ | 20260507 16:20:07.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:20:07.004 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:20:07.004 | INFO | Total mem usage: 15.7 GiB | 20260507 16:20:07.004 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:20:07.004 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:20:07.004 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:20:07.004 | INFO | High Water Breakdown: | 20260507 16:20:07.004 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:20:07.004 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:20:07.004 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:20:07.004 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:20:07.004 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:20:07.004 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:20:07.004 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:20:07.004 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:20:07.004 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:20:07.004 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:20:07.004 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:20:07.004 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:20:07.004 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:20:07.004 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:20:07.004 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:20:07.004 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:20:07.005 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:20:07.005 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:20:07.005 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:20:07.005 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:20:07.005 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:20:07.005 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:20:07.005 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:20:07.005 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:20:07.005 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:20:07.005 | INFO | ------------------------------------------ | 20260507 16:20:07.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:20:07.005 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:20:07.005 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:20:07.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:20:07.005 | INFO | PerfReport: Full Execution -- 2.97e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.82e-03 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 11.07%, Baseline -- 3.29e+00 / -nan, 1.91e-02 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 3.09%, Histogram -- 9.18e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 2.99e+00 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.72e+01 / -nan, 4.20e-02 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.11e+00 / -nan, 1.17e-01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 3.37%, Metrics -- 1.00e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.65e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260507 16:20:07.005 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:20:07.005 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:20:07.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:20:07.005 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 18.68%, Upload -- 1.66e+01 / 0.00e+00, 5.00e-01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 18.28%, Download -- 1.63e+01 / 0.00e+00, 8.08e-01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 33.39%, Compute -- 2.98e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 0.57%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 0.00%, LockDownload -- 4.40e-04 / 0.00e+00, 1.36e-04 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 29.06%, LockCompute -- 2.59e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260507 16:20:07.005 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 4.62e-02 / 0.00e+00 | 20260507 16:20:07.100 | INFO | Starting a round of Baz IO | 20260507 16:20:09.978 | INFO | Populating RTMetrics report | 20260507 16:20:09.979 | INFO | RTMetrics report populated | 20260507 16:20:09.979 | INFO | Opening tmp RTMetrics json file | 20260507 16:20:09.980 | INFO | Serializing RTMetrics to json | 20260507 16:20:09.983 | INFO | Serialization done, swapping tmp file now... | 20260507 16:20:09.984 | INFO | RTMetrics done | 20260507 16:20:10.124 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.024 seconds | 20260507 16:20:10.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.55% / 0.03% / 49.42% | 20260507 16:20:10.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.62778% / 5.16574e-05% / 3.17891e-05% | 20260507 16:20:10.124 | INFO | Baz IO overall: | 20260507 16:20:10.125 | INFO | BytesWritten : 377380597013 | 20260507 16:20:10.125 | INFO | NumEvents : 182507081104 | 20260507 16:20:12.142 | INFO | Analyzing chunk frames = [884736,885248) | 20260507 16:20:12.142 | INFO | MemInfo: { | 20260507 16:20:12.142 | INFO | "gpu" : " 17 GB", | 20260507 16:20:12.142 | INFO | "huge" : "", | 20260507 16:20:12.142 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:12.142 | INFO | "malloc" : " 14 GB", | 20260507 16:20:12.142 | INFO | "pinned" : " 29 GB" | 20260507 16:20:12.142 | INFO | } | 20260507 16:20:12.142 | INFO | Perf report for the last 5.13909 seconds: | 20260507 16:20:12.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:12.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:12.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:12.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:12.142 | INFO | ANALYSIS : 57.0%, 2.9, 87.8ms, 0 | 20260507 16:20:12.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:12.142 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 16:20:12.142 | INFO | PRE_HQ : 55.7%, 2.0, 58.9ms, 0 | 20260507 16:20:12.142 | INFO | BAZWRITER : 58.4%, 4.5, 133.7ms, 0 | 20260507 16:20:12.142 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 16:20:12.143 | INFO | Uploaded 36GiB to GPU in 1.64556s (21.8771GiB/s) | 20260507 16:20:12.143 | INFO | Downloaded 38GiB to GPU in 1.72785s (21.9926GiB/s) | 20260507 16:20:17.281 | INFO | Analyzing chunk frames = [885248,885760) | 20260507 16:20:17.281 | INFO | MemInfo: { | 20260507 16:20:17.281 | INFO | "gpu" : " 17 GB", | 20260507 16:20:17.281 | INFO | "huge" : "", | 20260507 16:20:17.281 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:17.282 | INFO | "malloc" : " 14 GB", | 20260507 16:20:17.282 | INFO | "pinned" : " 29 GB" | 20260507 16:20:17.282 | INFO | } | 20260507 16:20:17.282 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:20:17.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:17.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:20:17.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:20:17.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:17.282 | INFO | Uploaded 36GiB to GPU in 1.58679s (22.6873GiB/s) | 20260507 16:20:17.282 | INFO | Downloaded 34GiB to GPU in 1.5356s (22.1411GiB/s) | 20260507 16:20:22.421 | INFO | Analyzing chunk frames = [885760,886272) | 20260507 16:20:22.421 | INFO | MemInfo: { | 20260507 16:20:22.421 | INFO | "gpu" : " 17 GB", | 20260507 16:20:22.421 | INFO | "huge" : "", | 20260507 16:20:22.421 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:22.421 | INFO | "malloc" : " 14 GB", | 20260507 16:20:22.421 | INFO | "pinned" : " 29 GB" | 20260507 16:20:22.421 | INFO | } | 20260507 16:20:22.421 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:20:22.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:22.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:20:22.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:20:22.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:22.421 | INFO | Uploaded 36GiB to GPU in 1.59047s (22.6348GiB/s) | 20260507 16:20:22.421 | INFO | Downloaded 34GiB to GPU in 1.54014s (22.0759GiB/s) | 20260507 16:20:27.560 | INFO | Analyzing chunk frames = [886272,886784) | 20260507 16:20:27.560 | INFO | MemInfo: { | 20260507 16:20:27.560 | INFO | "gpu" : " 17 GB", | 20260507 16:20:27.560 | INFO | "huge" : "", | 20260507 16:20:27.560 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:27.560 | INFO | "malloc" : " 14 GB", | 20260507 16:20:27.560 | INFO | "pinned" : " 29 GB" | 20260507 16:20:27.560 | INFO | } | 20260507 16:20:27.560 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:20:27.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:27.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.560 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:20:27.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.561 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:20:27.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:27.561 | INFO | Uploaded 36GiB to GPU in 1.58904s (22.6553GiB/s) | 20260507 16:20:27.561 | INFO | Downloaded 34GiB to GPU in 1.53954s (22.0845GiB/s) | 20260507 16:20:32.700 | INFO | Analyzing chunk frames = [886784,887296) | 20260507 16:20:32.700 | INFO | MemInfo: { | 20260507 16:20:32.700 | INFO | "gpu" : " 17 GB", | 20260507 16:20:32.700 | INFO | "huge" : "", | 20260507 16:20:32.700 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:32.700 | INFO | "malloc" : " 14 GB", | 20260507 16:20:32.700 | INFO | "pinned" : " 29 GB" | 20260507 16:20:32.700 | INFO | } | 20260507 16:20:32.700 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:20:32.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:32.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:20:32.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:20:32.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:32.700 | INFO | Uploaded 36GiB to GPU in 1.58846s (22.6635GiB/s) | 20260507 16:20:32.700 | INFO | Downloaded 34GiB to GPU in 1.53851s (22.0993GiB/s) | 20260507 16:20:37.839 | INFO | Analyzing chunk frames = [887296,887808) | 20260507 16:20:37.839 | INFO | MemInfo: { | 20260507 16:20:37.839 | INFO | "gpu" : " 17 GB", | 20260507 16:20:37.839 | INFO | "huge" : "", | 20260507 16:20:37.839 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:37.839 | INFO | "malloc" : " 14 GB", | 20260507 16:20:37.839 | INFO | "pinned" : " 29 GB" | 20260507 16:20:37.839 | INFO | } | 20260507 16:20:37.839 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:20:37.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:37.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.839 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:20:37.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.839 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:20:37.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:37.840 | INFO | Uploaded 36GiB to GPU in 1.58878s (22.6589GiB/s) | 20260507 16:20:37.840 | INFO | Downloaded 34GiB to GPU in 1.53748s (22.1141GiB/s) | 20260507 16:20:42.978 | INFO | Analyzing chunk frames = [887808,888320) | 20260507 16:20:42.978 | INFO | MemInfo: { | 20260507 16:20:42.978 | INFO | "gpu" : " 17 GB", | 20260507 16:20:42.978 | INFO | "huge" : "", | 20260507 16:20:42.978 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:42.978 | INFO | "malloc" : " 14 GB", | 20260507 16:20:42.978 | INFO | "pinned" : " 29 GB" | 20260507 16:20:42.978 | INFO | } | 20260507 16:20:42.978 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:20:42.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:42.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.978 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260507 16:20:42.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.978 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:20:42.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:42.979 | INFO | Uploaded 36GiB to GPU in 1.58834s (22.6652GiB/s) | 20260507 16:20:42.979 | INFO | Downloaded 34GiB to GPU in 1.53832s (22.102GiB/s) | 20260507 16:20:48.117 | INFO | Analyzing chunk frames = [888320,888832) | 20260507 16:20:48.117 | INFO | MemInfo: { | 20260507 16:20:48.117 | INFO | "gpu" : " 17 GB", | 20260507 16:20:48.117 | INFO | "huge" : "", | 20260507 16:20:48.118 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:48.118 | INFO | "malloc" : " 14 GB", | 20260507 16:20:48.118 | INFO | "pinned" : " 29 GB" | 20260507 16:20:48.118 | INFO | } | 20260507 16:20:48.118 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:20:48.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:48.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:20:48.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:20:48.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:48.118 | INFO | Uploaded 36GiB to GPU in 1.5886s (22.6615GiB/s) | 20260507 16:20:48.118 | INFO | Downloaded 34GiB to GPU in 1.53894s (22.0932GiB/s) | 20260507 16:20:48.215 | INFO | Starting a round of Baz IO | 20260507 16:20:51.083 | INFO | Populating RTMetrics report | 20260507 16:20:51.084 | INFO | RTMetrics report populated | 20260507 16:20:51.084 | INFO | Opening tmp RTMetrics json file | 20260507 16:20:51.088 | INFO | Serializing RTMetrics to json | 20260507 16:20:51.125 | INFO | Serialization done, swapping tmp file now... | 20260507 16:20:51.152 | INFO | RTMetrics done | 20260507 16:20:51.375 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.16 seconds | 20260507 16:20:51.375 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.52% / 0.03% / 49.45% | 20260507 16:20:51.375 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.63387% / 7.54992e-05% / 2.38419e-05% | 20260507 16:20:51.375 | INFO | Baz IO overall: | 20260507 16:20:51.375 | INFO | BytesWritten : 379294457764 | 20260507 16:20:51.375 | INFO | NumEvents : 183438949272 | 20260507 16:20:53.257 | INFO | Analyzing chunk frames = [888832,889344) | 20260507 16:20:53.257 | INFO | MemInfo: { | 20260507 16:20:53.257 | INFO | "gpu" : " 17 GB", | 20260507 16:20:53.257 | INFO | "huge" : "", | 20260507 16:20:53.257 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:53.257 | INFO | "malloc" : " 14 GB", | 20260507 16:20:53.257 | INFO | "pinned" : " 29 GB" | 20260507 16:20:53.257 | INFO | } | 20260507 16:20:53.257 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:20:53.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:53.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:53.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:53.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:53.257 | INFO | ANALYSIS : 56.8%, 2.9, 86.1ms, 0 | 20260507 16:20:53.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:53.257 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 16:20:53.257 | INFO | PRE_HQ : 55.5%, 2.0, 58.7ms, 0 | 20260507 16:20:53.257 | INFO | BAZWRITER : 60.9%, 4.7, 146.9ms, 0 | 20260507 16:20:53.257 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 16:20:53.258 | INFO | Uploaded 36GiB to GPU in 1.65769s (21.7169GiB/s) | 20260507 16:20:53.258 | INFO | Downloaded 38GiB to GPU in 1.74243s (21.8086GiB/s) | 20260507 16:20:58.396 | INFO | Analyzing chunk frames = [889344,889856) | 20260507 16:20:58.396 | INFO | MemInfo: { | 20260507 16:20:58.396 | INFO | "gpu" : " 17 GB", | 20260507 16:20:58.396 | INFO | "huge" : "", | 20260507 16:20:58.396 | INFO | "huge_shared" : " 29 GB", | 20260507 16:20:58.397 | INFO | "malloc" : " 14 GB", | 20260507 16:20:58.397 | INFO | "pinned" : " 29 GB" | 20260507 16:20:58.397 | INFO | } | 20260507 16:20:58.397 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:20:58.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:20:58.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260507 16:20:58.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:20:58.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:20:58.397 | INFO | Uploaded 36GiB to GPU in 1.58584s (22.7009GiB/s) | 20260507 16:20:58.397 | INFO | Downloaded 34GiB to GPU in 1.53453s (22.1566GiB/s) | 20260507 16:21:03.536 | INFO | Analyzing chunk frames = [889856,890368) | 20260507 16:21:03.536 | INFO | MemInfo: { | 20260507 16:21:03.536 | INFO | "gpu" : " 17 GB", | 20260507 16:21:03.536 | INFO | "huge" : "", | 20260507 16:21:03.536 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:03.536 | INFO | "malloc" : " 14 GB", | 20260507 16:21:03.536 | INFO | "pinned" : " 29 GB" | 20260507 16:21:03.536 | INFO | } | 20260507 16:21:03.536 | INFO | Perf report for the last 5.13901 seconds: | 20260507 16:21:03.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:03.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:21:03.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:21:03.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:03.536 | INFO | Uploaded 36GiB to GPU in 1.58991s (22.6428GiB/s) | 20260507 16:21:03.536 | INFO | Downloaded 34GiB to GPU in 1.54038s (22.0725GiB/s) | 20260507 16:21:08.675 | INFO | Analyzing chunk frames = [890368,890880) | 20260507 16:21:08.675 | INFO | MemInfo: { | 20260507 16:21:08.675 | INFO | "gpu" : " 17 GB", | 20260507 16:21:08.675 | INFO | "huge" : "", | 20260507 16:21:08.675 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:08.675 | INFO | "malloc" : " 14 GB", | 20260507 16:21:08.675 | INFO | "pinned" : " 29 GB" | 20260507 16:21:08.675 | INFO | } | 20260507 16:21:08.675 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:21:08.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:08.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:21:08.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:21:08.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:08.675 | INFO | Uploaded 36GiB to GPU in 1.59094s (22.6281GiB/s) | 20260507 16:21:08.675 | INFO | Downloaded 34GiB to GPU in 1.53882s (22.0949GiB/s) | 20260507 16:21:13.814 | INFO | Analyzing chunk frames = [890880,891392) | 20260507 16:21:13.814 | INFO | MemInfo: { | 20260507 16:21:13.814 | INFO | "gpu" : " 17 GB", | 20260507 16:21:13.814 | INFO | "huge" : "", | 20260507 16:21:13.814 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:13.814 | INFO | "malloc" : " 14 GB", | 20260507 16:21:13.814 | INFO | "pinned" : " 29 GB" | 20260507 16:21:13.814 | INFO | } | 20260507 16:21:13.814 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:21:13.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:13.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:21:13.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:21:13.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:13.814 | INFO | Uploaded 36GiB to GPU in 1.58902s (22.6555GiB/s) | 20260507 16:21:13.815 | INFO | Downloaded 34GiB to GPU in 1.5381s (22.1051GiB/s) | 20260507 16:21:18.954 | INFO | Analyzing chunk frames = [891392,891904) | 20260507 16:21:18.954 | INFO | MemInfo: { | 20260507 16:21:18.954 | INFO | "gpu" : " 17 GB", | 20260507 16:21:18.954 | INFO | "huge" : "", | 20260507 16:21:18.954 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:18.954 | INFO | "malloc" : " 14 GB", | 20260507 16:21:18.954 | INFO | "pinned" : " 29 GB" | 20260507 16:21:18.954 | INFO | } | 20260507 16:21:18.954 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:21:18.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:18.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:21:18.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:21:18.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:18.954 | INFO | Uploaded 36GiB to GPU in 1.58865s (22.6607GiB/s) | 20260507 16:21:18.954 | INFO | Downloaded 34GiB to GPU in 1.53788s (22.1084GiB/s) | 20260507 16:21:24.093 | INFO | Analyzing chunk frames = [891904,892416) | 20260507 16:21:24.093 | INFO | MemInfo: { | 20260507 16:21:24.093 | INFO | "gpu" : " 17 GB", | 20260507 16:21:24.093 | INFO | "huge" : "", | 20260507 16:21:24.093 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:24.093 | INFO | "malloc" : " 14 GB", | 20260507 16:21:24.093 | INFO | "pinned" : " 29 GB" | 20260507 16:21:24.093 | INFO | } | 20260507 16:21:24.093 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:21:24.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:24.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:21:24.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:21:24.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:24.093 | INFO | Uploaded 36GiB to GPU in 1.58888s (22.6575GiB/s) | 20260507 16:21:24.094 | INFO | Downloaded 34GiB to GPU in 1.53829s (22.1024GiB/s) | 20260507 16:21:29.233 | INFO | Analyzing chunk frames = [892416,892928) | 20260507 16:21:29.233 | INFO | MemInfo: { | 20260507 16:21:29.233 | INFO | "gpu" : " 17 GB", | 20260507 16:21:29.233 | INFO | "huge" : "", | 20260507 16:21:29.233 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:29.233 | INFO | "malloc" : " 14 GB", | 20260507 16:21:29.233 | INFO | "pinned" : " 29 GB" | 20260507 16:21:29.233 | INFO | } | 20260507 16:21:29.233 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:21:29.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:29.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:21:29.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:21:29.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:29.233 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:21:29.233 | INFO | Total mem usage: 16.4 GiB | 20260507 16:21:29.233 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:21:29.233 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -6.94%) | 20260507 16:21:29.233 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:21:29.233 | INFO | High Water Breakdown: | 20260507 16:21:29.233 | INFO | 59.8% : 9.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:21:29.233 | INFO | 40% : 6.14 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:21:29.233 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:21:29.233 | INFO | ------------------------------------------ | 20260507 16:21:29.233 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:21:29.233 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:21:29.233 | INFO | Total mem usage: 26.5 GiB | 20260507 16:21:29.233 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:21:29.233 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:21:29.233 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:21:29.233 | INFO | High Water Breakdown: | 20260507 16:21:29.233 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:21:29.233 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:21:29.233 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:21:29.233 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:21:29.233 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:21:29.233 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:21:29.233 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:21:29.233 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:21:29.233 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:21:29.233 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:21:29.233 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:21:29.233 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:21:29.233 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:21:29.233 | INFO | Uploaded 36GiB to GPU in 1.58855s (22.6621GiB/s) | 20260507 16:21:29.233 | INFO | Downloaded 34GiB to GPU in 1.53802s (22.1064GiB/s) | 20260507 16:21:29.233 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:21:29.233 | INFO | ------------------------------------------ | 20260507 16:21:29.233 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:21:29.233 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:21:29.233 | INFO | Total mem usage: 27.8 GiB | 20260507 16:21:29.233 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:21:29.233 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:21:29.234 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:21:29.234 | INFO | High Water Breakdown: | 20260507 16:21:29.234 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:21:29.234 | INFO | ------------------------------------------ | 20260507 16:21:29.234 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:21:29.234 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:21:29.234 | INFO | Total mem usage: 15.7 GiB | 20260507 16:21:29.234 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:21:29.234 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:21:29.234 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:21:29.234 | INFO | High Water Breakdown: | 20260507 16:21:29.234 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:21:29.234 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:21:29.234 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:21:29.234 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:21:29.234 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:21:29.234 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:21:29.234 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:21:29.234 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:21:29.234 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:21:29.234 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:21:29.234 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:21:29.234 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:21:29.234 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:21:29.234 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:21:29.234 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:21:29.234 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:21:29.234 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:21:29.234 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:21:29.234 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:21:29.234 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:21:29.234 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:21:29.234 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:21:29.234 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:21:29.234 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:21:29.234 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:21:29.234 | INFO | ------------------------------------------ | 20260507 16:21:29.234 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:21:29.234 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:21:29.234 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:21:29.234 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:21:29.234 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 0.30%, AggregateStats -- 9.01e-02 / -nan, 1.40e-01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 11.07%, Baseline -- 3.29e+00 / -nan, 2.03e-02 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 3.08%, Histogram -- 9.18e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.72e+01 / -nan, 4.28e-02 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.11e+00 / -nan, 1.18e-01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 3.37%, Metrics -- 1.00e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.65e-01 / -nan, 5.32e-02 / 0.00e+00 | 20260507 16:21:29.234 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:21:29.234 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:21:29.234 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:21:29.234 | INFO | PerfReport: Full Execution -- 8.92e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 18.65%, Upload -- 1.66e+01 / 0.00e+00, 4.86e-01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 18.26%, Download -- 1.63e+01 / 0.00e+00, 7.88e-01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 33.37%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 0.00%, LockDownload -- 4.34e-04 / 0.00e+00, 9.55e-05 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 29.13%, LockCompute -- 2.60e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 16:21:29.234 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 16:21:29.329 | INFO | Starting a round of Baz IO | 20260507 16:21:32.200 | INFO | Populating RTMetrics report | 20260507 16:21:32.200 | INFO | RTMetrics report populated | 20260507 16:21:32.200 | INFO | Opening tmp RTMetrics json file | 20260507 16:21:32.318 | INFO | Serializing RTMetrics to json | 20260507 16:21:32.328 | INFO | Serialization done, swapping tmp file now... | 20260507 16:21:32.333 | INFO | RTMetrics done | 20260507 16:21:32.480 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.15 seconds | 20260507 16:21:32.480 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.49% / 0.03% / 49.48% | 20260507 16:21:32.480 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.61975% / 5.96046e-05% / 1.58946e-05% | 20260507 16:21:32.480 | INFO | Baz IO overall: | 20260507 16:21:32.480 | INFO | BytesWritten : 381208928578 | 20260507 16:21:32.480 | INFO | NumEvents : 184371255289 | 20260507 16:21:34.372 | INFO | Analyzing chunk frames = [892928,893440) | 20260507 16:21:34.372 | INFO | MemInfo: { | 20260507 16:21:34.372 | INFO | "gpu" : " 17 GB", | 20260507 16:21:34.372 | INFO | "huge" : "", | 20260507 16:21:34.372 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:34.372 | INFO | "malloc" : " 14 GB", | 20260507 16:21:34.372 | INFO | "pinned" : " 29 GB" | 20260507 16:21:34.372 | INFO | } | 20260507 16:21:34.372 | INFO | Perf report for the last 5.13887 seconds: | 20260507 16:21:34.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:34.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:34.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:34.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:34.372 | INFO | ANALYSIS : 56.9%, 2.9, 85.4ms, 0 | 20260507 16:21:34.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:34.372 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 16:21:34.372 | INFO | PRE_HQ : 55.5%, 2.0, 58.3ms, 0 | 20260507 16:21:34.372 | INFO | BAZWRITER : 61.0%, 4.8, 149.4ms, 0 | 20260507 16:21:34.372 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 16:21:34.372 | INFO | Uploaded 36GiB to GPU in 1.67176s (21.5341GiB/s) | 20260507 16:21:34.372 | INFO | Downloaded 38GiB to GPU in 1.75572s (21.6436GiB/s) | 20260507 16:21:39.511 | INFO | Analyzing chunk frames = [893440,893952) | 20260507 16:21:39.511 | INFO | MemInfo: { | 20260507 16:21:39.511 | INFO | "gpu" : " 17 GB", | 20260507 16:21:39.511 | INFO | "huge" : "", | 20260507 16:21:39.511 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:39.511 | INFO | "malloc" : " 14 GB", | 20260507 16:21:39.511 | INFO | "pinned" : " 29 GB" | 20260507 16:21:39.511 | INFO | } | 20260507 16:21:39.511 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:21:39.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:39.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260507 16:21:39.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260507 16:21:39.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:39.511 | INFO | Uploaded 36GiB to GPU in 1.58557s (22.7048GiB/s) | 20260507 16:21:39.511 | INFO | Downloaded 34GiB to GPU in 1.53454s (22.1564GiB/s) | 20260507 16:21:44.650 | INFO | Analyzing chunk frames = [893952,894464) | 20260507 16:21:44.650 | INFO | MemInfo: { | 20260507 16:21:44.650 | INFO | "gpu" : " 17 GB", | 20260507 16:21:44.650 | INFO | "huge" : "", | 20260507 16:21:44.650 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:44.650 | INFO | "malloc" : " 14 GB", | 20260507 16:21:44.650 | INFO | "pinned" : " 29 GB" | 20260507 16:21:44.650 | INFO | } | 20260507 16:21:44.650 | INFO | Perf report for the last 5.13915 seconds: | 20260507 16:21:44.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:44.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.650 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:21:44.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.651 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:21:44.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:44.651 | INFO | Uploaded 36GiB to GPU in 1.59021s (22.6385GiB/s) | 20260507 16:21:44.651 | INFO | Downloaded 34GiB to GPU in 1.54022s (22.0747GiB/s) | 20260507 16:21:49.790 | INFO | Analyzing chunk frames = [894464,894976) | 20260507 16:21:49.790 | INFO | MemInfo: { | 20260507 16:21:49.790 | INFO | "gpu" : " 17 GB", | 20260507 16:21:49.790 | INFO | "huge" : "", | 20260507 16:21:49.790 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:49.790 | INFO | "malloc" : " 14 GB", | 20260507 16:21:49.790 | INFO | "pinned" : " 29 GB" | 20260507 16:21:49.790 | INFO | } | 20260507 16:21:49.790 | INFO | Perf report for the last 5.1395 seconds: | 20260507 16:21:49.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:49.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | ANALYSIS : 55.8%, 3.0, 86.6ms, 0 | 20260507 16:21:49.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:21:49.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:49.790 | INFO | Uploaded 36GiB to GPU in 1.59073s (22.6311GiB/s) | 20260507 16:21:49.790 | INFO | Downloaded 34GiB to GPU in 1.53902s (22.092GiB/s) | 20260507 16:21:54.929 | INFO | Analyzing chunk frames = [894976,895488) | 20260507 16:21:54.929 | INFO | MemInfo: { | 20260507 16:21:54.929 | INFO | "gpu" : " 17 GB", | 20260507 16:21:54.929 | INFO | "huge" : "", | 20260507 16:21:54.929 | INFO | "huge_shared" : " 29 GB", | 20260507 16:21:54.929 | INFO | "malloc" : " 14 GB", | 20260507 16:21:54.929 | INFO | "pinned" : " 29 GB" | 20260507 16:21:54.929 | INFO | } | 20260507 16:21:54.929 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:21:54.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:21:54.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | ANALYSIS : 55.7%, 3.0, 86.9ms, 0 | 20260507 16:21:54.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:21:54.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:21:54.929 | INFO | Uploaded 36GiB to GPU in 1.5891s (22.6544GiB/s) | 20260507 16:21:54.929 | INFO | Downloaded 34GiB to GPU in 1.53796s (22.1073GiB/s) | 20260507 16:22:00.068 | INFO | Analyzing chunk frames = [895488,896000) | 20260507 16:22:00.068 | INFO | MemInfo: { | 20260507 16:22:00.068 | INFO | "gpu" : " 17 GB", | 20260507 16:22:00.068 | INFO | "huge" : "", | 20260507 16:22:00.068 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:00.068 | INFO | "malloc" : " 14 GB", | 20260507 16:22:00.068 | INFO | "pinned" : " 29 GB" | 20260507 16:22:00.068 | INFO | } | 20260507 16:22:00.069 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:22:00.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:00.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | ANALYSIS : 56.0%, 3.0, 87.4ms, 0 | 20260507 16:22:00.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:22:00.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:00.069 | INFO | Uploaded 36GiB to GPU in 1.58957s (22.6477GiB/s) | 20260507 16:22:00.069 | INFO | Downloaded 34GiB to GPU in 1.53711s (22.1194GiB/s) | 20260507 16:22:05.208 | INFO | Analyzing chunk frames = [896000,896512) | 20260507 16:22:05.208 | INFO | MemInfo: { | 20260507 16:22:05.208 | INFO | "gpu" : " 17 GB", | 20260507 16:22:05.208 | INFO | "huge" : "", | 20260507 16:22:05.208 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:05.208 | INFO | "malloc" : " 14 GB", | 20260507 16:22:05.208 | INFO | "pinned" : " 29 GB" | 20260507 16:22:05.208 | INFO | } | 20260507 16:22:05.208 | INFO | Perf report for the last 5.13959 seconds: | 20260507 16:22:05.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:05.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260507 16:22:05.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:22:05.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:05.208 | INFO | Uploaded 36GiB to GPU in 1.58869s (22.6602GiB/s) | 20260507 16:22:05.209 | INFO | Downloaded 34GiB to GPU in 1.5379s (22.108GiB/s) | 20260507 16:22:10.347 | INFO | Analyzing chunk frames = [896512,897024) | 20260507 16:22:10.347 | INFO | MemInfo: { | 20260507 16:22:10.347 | INFO | "gpu" : " 17 GB", | 20260507 16:22:10.347 | INFO | "huge" : "", | 20260507 16:22:10.347 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:10.347 | INFO | "malloc" : " 14 GB", | 20260507 16:22:10.347 | INFO | "pinned" : " 29 GB" | 20260507 16:22:10.347 | INFO | } | 20260507 16:22:10.347 | INFO | Perf report for the last 5.13906 seconds: | 20260507 16:22:10.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:10.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:22:10.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:22:10.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:10.347 | INFO | Uploaded 36GiB to GPU in 1.58927s (22.652GiB/s) | 20260507 16:22:10.347 | INFO | Downloaded 34GiB to GPU in 1.5396s (22.0837GiB/s) | 20260507 16:22:10.444 | INFO | Starting a round of Baz IO | 20260507 16:22:13.317 | INFO | Populating RTMetrics report | 20260507 16:22:13.317 | INFO | RTMetrics report populated | 20260507 16:22:13.317 | INFO | Opening tmp RTMetrics json file | 20260507 16:22:13.336 | INFO | Serializing RTMetrics to json | 20260507 16:22:13.398 | INFO | Serialization done, swapping tmp file now... | 20260507 16:22:13.427 | INFO | RTMetrics done | 20260507 16:22:13.454 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.009 seconds | 20260507 16:22:13.454 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.46% / 0.03% / 49.51% | 20260507 16:22:13.454 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.58982% / 6.35783e-05% / 2.78155e-05% | 20260507 16:22:13.454 | INFO | Baz IO overall: | 20260507 16:22:13.454 | INFO | BytesWritten : 383121136828 | 20260507 16:22:13.454 | INFO | NumEvents : 185301547837 | 20260507 16:22:15.486 | INFO | Analyzing chunk frames = [897024,897536) | 20260507 16:22:15.487 | INFO | MemInfo: { | 20260507 16:22:15.487 | INFO | "gpu" : " 17 GB", | 20260507 16:22:15.487 | INFO | "huge" : "", | 20260507 16:22:15.487 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:15.487 | INFO | "malloc" : " 14 GB", | 20260507 16:22:15.487 | INFO | "pinned" : " 29 GB" | 20260507 16:22:15.487 | INFO | } | 20260507 16:22:15.487 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:22:15.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:15.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:15.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:15.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:15.487 | INFO | ANALYSIS : 56.9%, 2.9, 87.9ms, 0 | 20260507 16:22:15.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:15.487 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 16:22:15.487 | INFO | PRE_HQ : 55.4%, 2.0, 58.7ms, 0 | 20260507 16:22:15.487 | INFO | BAZWRITER : 58.1%, 4.3, 129.4ms, 0 | 20260507 16:22:15.487 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 16:22:15.488 | INFO | Uploaded 36GiB to GPU in 1.63849s (21.9715GiB/s) | 20260507 16:22:15.488 | INFO | Downloaded 38GiB to GPU in 1.72164s (22.072GiB/s) | 20260507 16:22:20.630 | INFO | Analyzing chunk frames = [897536,898048) | 20260507 16:22:20.630 | INFO | MemInfo: { | 20260507 16:22:20.630 | INFO | "gpu" : " 17 GB", | 20260507 16:22:20.630 | INFO | "huge" : "", | 20260507 16:22:20.630 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:20.630 | INFO | "malloc" : " 14 GB", | 20260507 16:22:20.630 | INFO | "pinned" : " 29 GB" | 20260507 16:22:20.630 | INFO | } | 20260507 16:22:20.630 | INFO | Perf report for the last 5.14374 seconds: | 20260507 16:22:20.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:20.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260507 16:22:20.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:22:20.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:20.631 | INFO | Uploaded 36GiB to GPU in 1.58653s (22.6911GiB/s) | 20260507 16:22:20.631 | INFO | Downloaded 34GiB to GPU in 1.53591s (22.1368GiB/s) | 20260507 16:22:25.765 | INFO | Analyzing chunk frames = [898048,898560) | 20260507 16:22:25.765 | INFO | MemInfo: { | 20260507 16:22:25.765 | INFO | "gpu" : " 17 GB", | 20260507 16:22:25.765 | INFO | "huge" : "", | 20260507 16:22:25.765 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:25.765 | INFO | "malloc" : " 14 GB", | 20260507 16:22:25.765 | INFO | "pinned" : " 29 GB" | 20260507 16:22:25.765 | INFO | } | 20260507 16:22:25.765 | INFO | Perf report for the last 5.13488 seconds: | 20260507 16:22:25.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:25.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.765 | INFO | ANALYSIS : 56.1%, 3.0, 87.1ms, 0 | 20260507 16:22:25.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.765 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:22:25.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:25.766 | INFO | Uploaded 36GiB to GPU in 1.58998s (22.6418GiB/s) | 20260507 16:22:25.766 | INFO | Downloaded 34GiB to GPU in 1.5404s (22.0722GiB/s) | 20260507 16:22:30.904 | INFO | Analyzing chunk frames = [898560,899072) | 20260507 16:22:30.904 | INFO | MemInfo: { | 20260507 16:22:30.904 | INFO | "gpu" : " 17 GB", | 20260507 16:22:30.904 | INFO | "huge" : "", | 20260507 16:22:30.904 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:30.904 | INFO | "malloc" : " 14 GB", | 20260507 16:22:30.904 | INFO | "pinned" : " 29 GB" | 20260507 16:22:30.905 | INFO | } | 20260507 16:22:30.905 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:22:30.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:30.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | ANALYSIS : 55.8%, 3.0, 86.5ms, 0 | 20260507 16:22:30.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 16:22:30.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:30.905 | INFO | Uploaded 36GiB to GPU in 1.59196s (22.6136GiB/s) | 20260507 16:22:30.905 | INFO | Downloaded 34GiB to GPU in 1.5428s (22.0378GiB/s) | 20260507 16:22:36.044 | INFO | Analyzing chunk frames = [899072,899584) | 20260507 16:22:36.044 | INFO | MemInfo: { | 20260507 16:22:36.044 | INFO | "gpu" : " 17 GB", | 20260507 16:22:36.044 | INFO | "huge" : "", | 20260507 16:22:36.044 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:36.044 | INFO | "malloc" : " 14 GB", | 20260507 16:22:36.044 | INFO | "pinned" : " 29 GB" | 20260507 16:22:36.044 | INFO | } | 20260507 16:22:36.044 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:22:36.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:36.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | ANALYSIS : 55.8%, 3.0, 87.0ms, 0 | 20260507 16:22:36.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:22:36.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:36.044 | INFO | Uploaded 36GiB to GPU in 1.58767s (22.6747GiB/s) | 20260507 16:22:36.044 | INFO | Downloaded 34GiB to GPU in 1.53789s (22.1082GiB/s) | 20260507 16:22:41.183 | INFO | Analyzing chunk frames = [899584,900096) | 20260507 16:22:41.183 | INFO | MemInfo: { | 20260507 16:22:41.183 | INFO | "gpu" : " 17 GB", | 20260507 16:22:41.183 | INFO | "huge" : "", | 20260507 16:22:41.183 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:41.183 | INFO | "malloc" : " 14 GB", | 20260507 16:22:41.183 | INFO | "pinned" : " 29 GB" | 20260507 16:22:41.183 | INFO | } | 20260507 16:22:41.183 | INFO | Perf report for the last 5.13958 seconds: | 20260507 16:22:41.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:41.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | ANALYSIS : 56.0%, 3.0, 87.3ms, 0 | 20260507 16:22:41.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:22:41.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:41.184 | INFO | Uploaded 36GiB to GPU in 1.58852s (22.6626GiB/s) | 20260507 16:22:41.184 | INFO | Downloaded 34GiB to GPU in 1.53806s (22.1057GiB/s) | 20260507 16:22:46.323 | INFO | Analyzing chunk frames = [900096,900608) | 20260507 16:22:46.323 | INFO | MemInfo: { | 20260507 16:22:46.323 | INFO | "gpu" : " 17 GB", | 20260507 16:22:46.323 | INFO | "huge" : "", | 20260507 16:22:46.323 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:46.323 | INFO | "malloc" : " 14 GB", | 20260507 16:22:46.323 | INFO | "pinned" : " 29 GB" | 20260507 16:22:46.323 | INFO | } | 20260507 16:22:46.323 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:22:46.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:46.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:22:46.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 16:22:46.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:46.323 | INFO | Uploaded 36GiB to GPU in 1.74145s (20.6725GiB/s) | 20260507 16:22:46.323 | INFO | Downloaded 34GiB to GPU in 1.56366s (21.7439GiB/s) | 20260507 16:22:51.462 | INFO | Analyzing chunk frames = [900608,901120) | 20260507 16:22:51.462 | INFO | MemInfo: { | 20260507 16:22:51.462 | INFO | "gpu" : " 17 GB", | 20260507 16:22:51.462 | INFO | "huge" : "", | 20260507 16:22:51.462 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:51.462 | INFO | "malloc" : " 14 GB", | 20260507 16:22:51.462 | INFO | "pinned" : " 29 GB" | 20260507 16:22:51.462 | INFO | } | 20260507 16:22:51.462 | INFO | Perf report for the last 5.13901 seconds: | 20260507 16:22:51.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:51.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260507 16:22:51.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:22:51.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:51.462 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:22:51.462 | INFO | Total mem usage: 16.4 GiB | 20260507 16:22:51.462 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:22:51.462 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.25%) | 20260507 16:22:51.462 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:22:51.462 | INFO | High Water Breakdown: | 20260507 16:22:51.462 | INFO | 59.5% : 8.93 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:22:51.462 | INFO | 40.4% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:22:51.462 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:22:51.462 | INFO | ------------------------------------------ | 20260507 16:22:51.462 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:22:51.462 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:22:51.462 | INFO | Total mem usage: 26.5 GiB | 20260507 16:22:51.462 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 16:22:51.462 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:22:51.462 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:22:51.462 | INFO | High Water Breakdown: | 20260507 16:22:51.462 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:22:51.462 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:22:51.462 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:22:51.462 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:22:51.462 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:22:51.462 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:22:51.462 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:22:51.462 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:22:51.462 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:22:51.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:22:51.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:22:51.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:22:51.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:22:51.463 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:22:51.463 | INFO | ------------------------------------------ | 20260507 16:22:51.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:22:51.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:22:51.463 | INFO | Total mem usage: 27.8 GiB | 20260507 16:22:51.463 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:22:51.463 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:22:51.463 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:22:51.463 | INFO | High Water Breakdown: | 20260507 16:22:51.463 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:22:51.463 | INFO | ------------------------------------------ | 20260507 16:22:51.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:22:51.463 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:22:51.463 | INFO | Total mem usage: 15.7 GiB | 20260507 16:22:51.463 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:22:51.463 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:22:51.463 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:22:51.463 | INFO | High Water Breakdown: | 20260507 16:22:51.463 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:22:51.463 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:22:51.463 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:22:51.463 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:22:51.463 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:22:51.463 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:22:51.463 | INFO | Uploaded 36GiB to GPU in 1.58973s (22.6454GiB/s) | 20260507 16:22:51.463 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:22:51.463 | INFO | Downloaded 34GiB to GPU in 1.54035s (22.0728GiB/s) | 20260507 16:22:51.463 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:22:51.463 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:22:51.463 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:22:51.463 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:22:51.463 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:22:51.463 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:22:51.463 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:22:51.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:22:51.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:22:51.463 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:22:51.463 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:22:51.463 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:22:51.463 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:22:51.463 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:22:51.463 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:22:51.463 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:22:51.463 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:22:51.463 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:22:51.463 | INFO | ------------------------------------------ | 20260507 16:22:51.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:22:51.463 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:22:51.463 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:22:51.463 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:22:51.463 | INFO | PerfReport: Full Execution -- 2.98e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 6.30e-03 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 11.07%, Baseline -- 3.29e+00 / -nan, 1.92e-02 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 3.09%, Histogram -- 9.18e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 16.16%, DME -- 3.85e+01 / -nan, 2.94e+00 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.11e+00 / -nan, 1.17e-01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 3.37%, Metrics -- 1.00e+00 / -nan, 1.64e-01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.65e-01 / -nan, 5.37e-02 / 0.00e+00 | 20260507 16:22:51.463 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:22:51.463 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:22:51.463 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:22:51.463 | INFO | PerfReport: Full Execution -- 8.91e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 18.78%, Upload -- 1.67e+01 / 0.00e+00, 8.18e-01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 18.29%, Download -- 1.63e+01 / 0.00e+00, 7.95e-01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 33.38%, Compute -- 2.98e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 0.57%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 0.00%, LockDownload -- 4.41e-04 / 0.00e+00, 1.53e-04 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 28.96%, LockCompute -- 2.58e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260507 16:22:51.463 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 16:22:51.559 | INFO | Starting a round of Baz IO | 20260507 16:22:54.466 | INFO | Populating RTMetrics report | 20260507 16:22:54.466 | INFO | RTMetrics report populated | 20260507 16:22:54.466 | INFO | Opening tmp RTMetrics json file | 20260507 16:22:54.486 | INFO | Serializing RTMetrics to json | 20260507 16:22:54.492 | INFO | Serialization done, swapping tmp file now... | 20260507 16:22:54.497 | INFO | RTMetrics done | 20260507 16:22:54.767 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.207 seconds | 20260507 16:22:54.767 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.43% / 0.03% / 49.54% | 20260507 16:22:54.767 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.59631% / 2.38419e-05% / 3.97364e-05% | 20260507 16:22:54.767 | INFO | Baz IO overall: | 20260507 16:22:54.767 | INFO | BytesWritten : 385035598851 | 20260507 16:22:54.767 | INFO | NumEvents : 186233076421 | 20260507 16:22:56.601 | INFO | Analyzing chunk frames = [901120,901632) | 20260507 16:22:56.601 | INFO | MemInfo: { | 20260507 16:22:56.601 | INFO | "gpu" : " 17 GB", | 20260507 16:22:56.601 | INFO | "huge" : "", | 20260507 16:22:56.601 | INFO | "huge_shared" : " 29 GB", | 20260507 16:22:56.601 | INFO | "malloc" : " 14 GB", | 20260507 16:22:56.601 | INFO | "pinned" : " 29 GB" | 20260507 16:22:56.601 | INFO | } | 20260507 16:22:56.601 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:22:56.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:22:56.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:56.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:56.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:56.601 | INFO | ANALYSIS : 57.5%, 2.8, 86.0ms, 0 | 20260507 16:22:56.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:22:56.601 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260507 16:22:56.601 | INFO | PRE_HQ : 56.3%, 2.1, 62.8ms, 0 | 20260507 16:22:56.601 | INFO | BAZWRITER : 62.0%, 4.5, 144.9ms, 0 | 20260507 16:22:56.601 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260507 16:22:56.602 | INFO | Uploaded 36GiB to GPU in 1.67183s (21.5333GiB/s) | 20260507 16:22:56.602 | INFO | Downloaded 38GiB to GPU in 1.80416s (21.0625GiB/s) | 20260507 16:23:01.741 | INFO | Analyzing chunk frames = [901632,902144) | 20260507 16:23:01.741 | INFO | MemInfo: { | 20260507 16:23:01.741 | INFO | "gpu" : " 17 GB", | 20260507 16:23:01.741 | INFO | "huge" : "", | 20260507 16:23:01.741 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:01.741 | INFO | "malloc" : " 14 GB", | 20260507 16:23:01.741 | INFO | "pinned" : " 29 GB" | 20260507 16:23:01.741 | INFO | } | 20260507 16:23:01.741 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:23:01.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:01.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260507 16:23:01.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260507 16:23:01.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:01.741 | INFO | Uploaded 36GiB to GPU in 1.63614s (22.0031GiB/s) | 20260507 16:23:01.741 | INFO | Downloaded 34GiB to GPU in 1.63816s (20.7549GiB/s) | 20260507 16:23:06.880 | INFO | Analyzing chunk frames = [902144,902656) | 20260507 16:23:06.880 | INFO | MemInfo: { | 20260507 16:23:06.880 | INFO | "gpu" : " 17 GB", | 20260507 16:23:06.880 | INFO | "huge" : "", | 20260507 16:23:06.880 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:06.880 | INFO | "malloc" : " 14 GB", | 20260507 16:23:06.880 | INFO | "pinned" : " 29 GB" | 20260507 16:23:06.880 | INFO | } | 20260507 16:23:06.880 | INFO | Perf report for the last 5.13897 seconds: | 20260507 16:23:06.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:06.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | ANALYSIS : 56.2%, 3.0, 87.3ms, 0 | 20260507 16:23:06.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 16:23:06.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:06.880 | INFO | Uploaded 36GiB to GPU in 1.6373s (21.9875GiB/s) | 20260507 16:23:06.880 | INFO | Downloaded 34GiB to GPU in 1.6483s (20.6274GiB/s) | 20260507 16:23:12.019 | INFO | Analyzing chunk frames = [902656,903168) | 20260507 16:23:12.019 | INFO | MemInfo: { | 20260507 16:23:12.019 | INFO | "gpu" : " 17 GB", | 20260507 16:23:12.019 | INFO | "huge" : "", | 20260507 16:23:12.019 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:12.019 | INFO | "malloc" : " 14 GB", | 20260507 16:23:12.019 | INFO | "pinned" : " 29 GB" | 20260507 16:23:12.019 | INFO | } | 20260507 16:23:12.019 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:23:12.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:12.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.019 | INFO | ANALYSIS : 55.9%, 3.0, 86.6ms, 0 | 20260507 16:23:12.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.019 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 16:23:12.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:12.020 | INFO | Uploaded 36GiB to GPU in 1.63943s (21.9589GiB/s) | 20260507 16:23:12.020 | INFO | Downloaded 34GiB to GPU in 1.64788s (20.6326GiB/s) | 20260507 16:23:17.158 | INFO | Analyzing chunk frames = [903168,903680) | 20260507 16:23:17.158 | INFO | MemInfo: { | 20260507 16:23:17.158 | INFO | "gpu" : " 17 GB", | 20260507 16:23:17.158 | INFO | "huge" : "", | 20260507 16:23:17.158 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:17.158 | INFO | "malloc" : " 14 GB", | 20260507 16:23:17.158 | INFO | "pinned" : " 29 GB" | 20260507 16:23:17.158 | INFO | } | 20260507 16:23:17.158 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:23:17.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:17.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.159 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 16:23:17.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.159 | INFO | PRE_HQ : 38.9%, 1.0, 20.3ms, 0 | 20260507 16:23:17.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:17.159 | INFO | Uploaded 36GiB to GPU in 1.61625s (22.2738GiB/s) | 20260507 16:23:17.159 | INFO | Downloaded 34GiB to GPU in 1.5427s (22.0392GiB/s) | 20260507 16:23:22.297 | INFO | Analyzing chunk frames = [903680,904192) | 20260507 16:23:22.297 | INFO | MemInfo: { | 20260507 16:23:22.297 | INFO | "gpu" : " 17 GB", | 20260507 16:23:22.298 | INFO | "huge" : "", | 20260507 16:23:22.298 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:22.298 | INFO | "malloc" : " 14 GB", | 20260507 16:23:22.298 | INFO | "pinned" : " 29 GB" | 20260507 16:23:22.298 | INFO | } | 20260507 16:23:22.298 | INFO | Perf report for the last 5.13905 seconds: | 20260507 16:23:22.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:22.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | ANALYSIS : 56.1%, 3.0, 86.6ms, 0 | 20260507 16:23:22.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | PRE_HQ : 32.4%, 1.0, 16.6ms, 0 | 20260507 16:23:22.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:22.298 | INFO | Uploaded 36GiB to GPU in 1.61085s (22.3484GiB/s) | 20260507 16:23:22.298 | INFO | Downloaded 34GiB to GPU in 1.53085s (22.2098GiB/s) | 20260507 16:23:27.437 | INFO | Analyzing chunk frames = [904192,904704) | 20260507 16:23:27.437 | INFO | MemInfo: { | 20260507 16:23:27.437 | INFO | "gpu" : " 17 GB", | 20260507 16:23:27.437 | INFO | "huge" : "", | 20260507 16:23:27.437 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:27.437 | INFO | "malloc" : " 14 GB", | 20260507 16:23:27.437 | INFO | "pinned" : " 29 GB" | 20260507 16:23:27.437 | INFO | } | 20260507 16:23:27.437 | INFO | Perf report for the last 5.13986 seconds: | 20260507 16:23:27.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:27.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.437 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260507 16:23:27.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.438 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260507 16:23:27.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:27.438 | INFO | Uploaded 36GiB to GPU in 1.61143s (22.3404GiB/s) | 20260507 16:23:27.438 | INFO | Downloaded 34GiB to GPU in 1.53194s (22.1941GiB/s) | 20260507 16:23:32.577 | INFO | Analyzing chunk frames = [904704,905216) | 20260507 16:23:32.577 | INFO | MemInfo: { | 20260507 16:23:32.577 | INFO | "gpu" : " 17 GB", | 20260507 16:23:32.577 | INFO | "huge" : "", | 20260507 16:23:32.577 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:32.577 | INFO | "malloc" : " 14 GB", | 20260507 16:23:32.577 | INFO | "pinned" : " 29 GB" | 20260507 16:23:32.577 | INFO | } | 20260507 16:23:32.577 | INFO | Perf report for the last 5.13956 seconds: | 20260507 16:23:32.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:32.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260507 16:23:32.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | PRE_HQ : 36.1%, 1.0, 18.9ms, 0 | 20260507 16:23:32.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:32.577 | INFO | Uploaded 36GiB to GPU in 1.62951s (22.0926GiB/s) | 20260507 16:23:32.577 | INFO | Downloaded 34GiB to GPU in 1.56807s (21.6827GiB/s) | 20260507 16:23:32.689 | INFO | Starting a round of Baz IO | 20260507 16:23:35.874 | INFO | Populating RTMetrics report | 20260507 16:23:35.874 | INFO | RTMetrics report populated | 20260507 16:23:35.874 | INFO | Opening tmp RTMetrics json file | 20260507 16:23:35.905 | INFO | Serializing RTMetrics to json | 20260507 16:23:35.958 | INFO | Serialization done, swapping tmp file now... | 20260507 16:23:35.980 | INFO | RTMetrics done | 20260507 16:23:36.010 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.32 seconds | 20260507 16:23:36.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.40% / 0.03% / 49.57% | 20260507 16:23:36.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.56712% / 5.5631e-05% / 3.57628e-05% | 20260507 16:23:36.010 | INFO | Baz IO overall: | 20260507 16:23:36.010 | INFO | BytesWritten : 386949269103 | 20260507 16:23:36.010 | INFO | NumEvents : 187163835550 | 20260507 16:23:37.716 | INFO | Analyzing chunk frames = [905216,905728) | 20260507 16:23:37.716 | INFO | MemInfo: { | 20260507 16:23:37.716 | INFO | "gpu" : " 17 GB", | 20260507 16:23:37.716 | INFO | "huge" : "", | 20260507 16:23:37.716 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:37.716 | INFO | "malloc" : " 14 GB", | 20260507 16:23:37.716 | INFO | "pinned" : " 29 GB" | 20260507 16:23:37.716 | INFO | } | 20260507 16:23:37.716 | INFO | Perf report for the last 5.1389 seconds: | 20260507 16:23:37.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:37.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:37.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:37.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:37.716 | INFO | ANALYSIS : 63.0%, 2.5, 82.5ms, 0 | 20260507 16:23:37.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:37.716 | INFO | RT_METRICS : 55.8%, 1.0, 29.2ms, 0 | 20260507 16:23:37.716 | INFO | PRE_HQ : 62.6%, 2.7, 86.2ms, 0 | 20260507 16:23:37.716 | INFO | BAZWRITER : 64.1%, 4.7, 154.2ms, 0 | 20260507 16:23:37.716 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:23:37.716 | INFO | Uploaded 36GiB to GPU in 1.6688s (21.5723GiB/s) | 20260507 16:23:37.716 | INFO | Downloaded 38GiB to GPU in 1.84262s (20.6228GiB/s) | 20260507 16:23:42.855 | INFO | Analyzing chunk frames = [905728,906240) | 20260507 16:23:42.855 | INFO | MemInfo: { | 20260507 16:23:42.855 | INFO | "gpu" : " 17 GB", | 20260507 16:23:42.856 | INFO | "huge" : "", | 20260507 16:23:42.856 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:42.856 | INFO | "malloc" : " 14 GB", | 20260507 16:23:42.856 | INFO | "pinned" : " 29 GB" | 20260507 16:23:42.856 | INFO | } | 20260507 16:23:42.856 | INFO | Perf report for the last 5.13993 seconds: | 20260507 16:23:42.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:42.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260507 16:23:42.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | PRE_HQ : 41.4%, 1.0, 22.1ms, 0 | 20260507 16:23:42.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:42.856 | INFO | Uploaded 36GiB to GPU in 1.6222s (22.1921GiB/s) | 20260507 16:23:42.857 | INFO | Downloaded 34GiB to GPU in 1.57538s (21.5821GiB/s) | 20260507 16:23:47.995 | INFO | Analyzing chunk frames = [906240,906752) | 20260507 16:23:47.995 | INFO | MemInfo: { | 20260507 16:23:47.995 | INFO | "gpu" : " 17 GB", | 20260507 16:23:47.995 | INFO | "huge" : "", | 20260507 16:23:47.995 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:47.995 | INFO | "malloc" : " 14 GB", | 20260507 16:23:47.995 | INFO | "pinned" : " 29 GB" | 20260507 16:23:47.995 | INFO | } | 20260507 16:23:47.995 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:23:47.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:47.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | ANALYSIS : 56.3%, 3.0, 87.5ms, 0 | 20260507 16:23:47.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | PRE_HQ : 40.4%, 1.0, 21.6ms, 0 | 20260507 16:23:47.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:47.995 | INFO | Uploaded 36GiB to GPU in 1.61931s (22.2317GiB/s) | 20260507 16:23:47.995 | INFO | Downloaded 34GiB to GPU in 1.58653s (21.4304GiB/s) | 20260507 16:23:53.134 | INFO | Analyzing chunk frames = [906752,907264) | 20260507 16:23:53.134 | INFO | MemInfo: { | 20260507 16:23:53.134 | INFO | "gpu" : " 17 GB", | 20260507 16:23:53.134 | INFO | "huge" : "", | 20260507 16:23:53.134 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:53.134 | INFO | "malloc" : " 14 GB", | 20260507 16:23:53.134 | INFO | "pinned" : " 29 GB" | 20260507 16:23:53.134 | INFO | } | 20260507 16:23:53.134 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:23:53.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:53.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.134 | INFO | ANALYSIS : 56.0%, 3.0, 86.8ms, 0 | 20260507 16:23:53.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.134 | INFO | PRE_HQ : 39.2%, 1.0, 20.3ms, 0 | 20260507 16:23:53.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:53.135 | INFO | Uploaded 36GiB to GPU in 1.61351s (22.3116GiB/s) | 20260507 16:23:53.135 | INFO | Downloaded 34GiB to GPU in 1.54036s (22.0728GiB/s) | 20260507 16:23:58.274 | INFO | Analyzing chunk frames = [907264,907776) | 20260507 16:23:58.274 | INFO | MemInfo: { | 20260507 16:23:58.274 | INFO | "gpu" : " 17 GB", | 20260507 16:23:58.274 | INFO | "huge" : "", | 20260507 16:23:58.274 | INFO | "huge_shared" : " 29 GB", | 20260507 16:23:58.274 | INFO | "malloc" : " 14 GB", | 20260507 16:23:58.274 | INFO | "pinned" : " 29 GB" | 20260507 16:23:58.274 | INFO | } | 20260507 16:23:58.274 | INFO | Perf report for the last 5.13996 seconds: | 20260507 16:23:58.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:23:58.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.274 | INFO | ANALYSIS : 55.9%, 3.0, 87.2ms, 0 | 20260507 16:23:58.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.275 | INFO | PRE_HQ : 38.5%, 1.0, 20.0ms, 0 | 20260507 16:23:58.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:23:58.275 | INFO | Uploaded 36GiB to GPU in 1.61047s (22.3538GiB/s) | 20260507 16:23:58.275 | INFO | Downloaded 34GiB to GPU in 1.53536s (22.1446GiB/s) | 20260507 16:24:03.413 | INFO | Analyzing chunk frames = [907776,908288) | 20260507 16:24:03.413 | INFO | MemInfo: { | 20260507 16:24:03.413 | INFO | "gpu" : " 17 GB", | 20260507 16:24:03.413 | INFO | "huge" : "", | 20260507 16:24:03.414 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:03.414 | INFO | "malloc" : " 14 GB", | 20260507 16:24:03.414 | INFO | "pinned" : " 29 GB" | 20260507 16:24:03.414 | INFO | } | 20260507 16:24:03.414 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:24:03.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:03.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | ANALYSIS : 58.8%, 3.0, 91.6ms, 0 | 20260507 16:24:03.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | PRE_HQ : 36.9%, 1.0, 19.3ms, 0 | 20260507 16:24:03.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:03.414 | INFO | Uploaded 36GiB to GPU in 2.07738s (17.3296GiB/s) | 20260507 16:24:03.414 | INFO | Downloaded 34GiB to GPU in 1.86057s (18.274GiB/s) | 20260507 16:24:08.552 | INFO | Analyzing chunk frames = [908288,908800) | 20260507 16:24:08.552 | INFO | MemInfo: { | 20260507 16:24:08.552 | INFO | "gpu" : " 17 GB", | 20260507 16:24:08.553 | INFO | "huge" : "", | 20260507 16:24:08.553 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:08.553 | INFO | "malloc" : " 14 GB", | 20260507 16:24:08.553 | INFO | "pinned" : " 29 GB" | 20260507 16:24:08.553 | INFO | } | 20260507 16:24:08.553 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:24:08.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:08.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.553 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260507 16:24:08.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.553 | INFO | PRE_HQ : 36.0%, 1.0, 19.1ms, 0 | 20260507 16:24:08.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:08.554 | INFO | Uploaded 36GiB to GPU in 3.05737s (11.7748GiB/s) | 20260507 16:24:08.554 | INFO | Downloaded 34GiB to GPU in 2.42394s (14.0267GiB/s) | 20260507 16:24:13.692 | INFO | Analyzing chunk frames = [908800,909312) | 20260507 16:24:13.692 | INFO | MemInfo: { | 20260507 16:24:13.692 | INFO | "gpu" : " 17 GB", | 20260507 16:24:13.692 | INFO | "huge" : "", | 20260507 16:24:13.692 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:13.692 | INFO | "malloc" : " 14 GB", | 20260507 16:24:13.692 | INFO | "pinned" : " 29 GB" | 20260507 16:24:13.692 | INFO | } | 20260507 16:24:13.692 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:24:13.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:13.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:24:13.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | PRE_HQ : 40.6%, 1.0, 21.2ms, 0 | 20260507 16:24:13.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:13.692 | INFO | Uploaded 36GiB to GPU in 2.67892s (13.4383GiB/s) | 20260507 16:24:13.693 | INFO | Downloaded 34GiB to GPU in 2.20518s (15.4183GiB/s) | 20260507 16:24:13.693 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:24:13.693 | INFO | Total mem usage: 16.4 GiB | 20260507 16:24:13.693 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:24:13.693 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.47%) | 20260507 16:24:13.693 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:24:13.693 | INFO | High Water Breakdown: | 20260507 16:24:13.693 | INFO | 59.6% : 9.01 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:24:13.693 | INFO | 40.3% : 6.09 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:24:13.693 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:24:13.693 | INFO | ------------------------------------------ | 20260507 16:24:13.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:24:13.693 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:24:13.693 | INFO | Total mem usage: 26.5 GiB | 20260507 16:24:13.693 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:24:13.693 | INFO | Recent High Water Marks Sum: 25.8 GiB (Savings: -2.89%) | 20260507 16:24:13.693 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260507 16:24:13.693 | INFO | High Water Breakdown: | 20260507 16:24:13.693 | INFO | 55.7% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:24:13.693 | INFO | 36% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:24:13.693 | INFO | 3.64% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:24:13.693 | INFO | 2.38% : 628 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:24:13.693 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:24:13.693 | INFO | 0.83% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:24:13.693 | INFO | 0.00151% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:24:13.693 | INFO | 9.29e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:24:13.693 | INFO | 8.68e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:24:13.693 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:24:13.693 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:24:13.693 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:24:13.693 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:24:13.693 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:24:13.693 | INFO | ------------------------------------------ | 20260507 16:24:13.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:24:13.693 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:24:13.693 | INFO | Total mem usage: 27.8 GiB | 20260507 16:24:13.693 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:24:13.693 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:24:13.693 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:24:13.693 | INFO | High Water Breakdown: | 20260507 16:24:13.693 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:24:13.693 | INFO | ------------------------------------------ | 20260507 16:24:13.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:24:13.693 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:24:13.693 | INFO | Total mem usage: 15.7 GiB | 20260507 16:24:13.693 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:24:13.693 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:24:13.693 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:24:13.693 | INFO | High Water Breakdown: | 20260507 16:24:13.693 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:24:13.693 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:24:13.693 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:24:13.693 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:24:13.693 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:24:13.694 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:24:13.694 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:24:13.694 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:24:13.694 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:24:13.694 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:24:13.694 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:24:13.694 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:24:13.694 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:24:13.694 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:24:13.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:24:13.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:24:13.694 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:24:13.694 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:24:13.694 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:24:13.694 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:24:13.694 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:24:13.694 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:24:13.694 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:24:13.694 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:24:13.694 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:24:13.694 | INFO | ------------------------------------------ | 20260507 16:24:13.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:24:13.694 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:24:13.694 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:24:13.694 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:24:13.694 | INFO | PerfReport: Full Execution -- 2.99e+01 / 1.44e+02, 1.30e+01 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 0.31%, AggregateStats -- 9.18e-02 / 1.51e-01, 7.69e-03 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 11.27%, Baseline -- 3.37e+00 / 8.26e+01, 2.01e+00 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 3.09%, Histogram -- 9.24e-01 / 1.05e+00, 5.89e-02 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 16.12%, DME -- 3.86e+01 / 3.96e+01, 3.04e+00 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.72e+01 / 1.71e+01, 7.93e-02 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.12e+00 / 2.12e+00, 1.27e-01 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 3.38%, Metrics -- 1.01e+00 / 1.23e+00, 1.70e-01 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.76e-01 / 3.80e-01, 5.66e-02 / 0.00e+00 | 20260507 16:24:13.694 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:24:13.694 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:24:13.694 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:24:13.694 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 2.07e+01 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 20.65%, Upload -- 1.89e+01 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 19.57%, Download -- 1.79e+01 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 32.74%, Compute -- 2.99e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 1.97%, LockUpload -- 1.80e+00 / 0.00e+00, 5.91e+00 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 0.22%, LockDownload -- 2.02e-01 / 0.00e+00, 1.35e+00 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 24.83%, LockCompute -- 2.27e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260507 16:24:13.694 | INFO | PerfReport: 0.02%, Misc -- 1.87e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 16:24:13.803 | INFO | Starting a round of Baz IO | 20260507 16:24:17.241 | INFO | Populating RTMetrics report | 20260507 16:24:17.241 | INFO | RTMetrics report populated | 20260507 16:24:17.241 | INFO | Opening tmp RTMetrics json file | 20260507 16:24:17.245 | INFO | Serializing RTMetrics to json | 20260507 16:24:17.247 | INFO | Serialization done, swapping tmp file now... | 20260507 16:24:17.248 | INFO | RTMetrics done | 20260507 16:24:17.346 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.543 seconds | 20260507 16:24:17.346 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.37% / 0.03% / 49.60% | 20260507 16:24:17.346 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.55857% / 4.37101e-05% / 2.78155e-05% | 20260507 16:24:17.346 | INFO | Baz IO overall: | 20260507 16:24:17.346 | INFO | BytesWritten : 388863174294 | 20260507 16:24:17.346 | INFO | NumEvents : 188094905996 | 20260507 16:24:18.831 | INFO | Analyzing chunk frames = [909312,909824) | 20260507 16:24:18.831 | INFO | MemInfo: { | 20260507 16:24:18.831 | INFO | "gpu" : " 17 GB", | 20260507 16:24:18.831 | INFO | "huge" : "", | 20260507 16:24:18.831 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:18.831 | INFO | "malloc" : " 14 GB", | 20260507 16:24:18.831 | INFO | "pinned" : " 29 GB" | 20260507 16:24:18.831 | INFO | } | 20260507 16:24:18.831 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:24:18.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:18.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:18.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:18.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:18.832 | INFO | ANALYSIS : 68.3%, 2.9, 105.4ms, 0 | 20260507 16:24:18.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:18.832 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 16:24:18.832 | INFO | PRE_HQ : 66.8%, 2.0, 70.5ms, 0 | 20260507 16:24:18.832 | INFO | BAZWRITER : 68.4%, 4.0, 142.4ms, 0 | 20260507 16:24:18.832 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 16:24:18.833 | INFO | Uploaded 36GiB to GPU in 2.7564s (13.0605GiB/s) | 20260507 16:24:18.833 | INFO | Downloaded 38GiB to GPU in 2.4295s (15.6411GiB/s) | 20260507 16:24:23.970 | INFO | Analyzing chunk frames = [909824,910336) | 20260507 16:24:23.970 | INFO | MemInfo: { | 20260507 16:24:23.970 | INFO | "gpu" : " 17 GB", | 20260507 16:24:23.971 | INFO | "huge" : "", | 20260507 16:24:23.971 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:23.971 | INFO | "malloc" : " 14 GB", | 20260507 16:24:23.971 | INFO | "pinned" : " 29 GB" | 20260507 16:24:23.971 | INFO | } | 20260507 16:24:23.971 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:24:23.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:23.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.971 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 16:24:23.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.971 | INFO | PRE_HQ : 39.4%, 1.0, 20.5ms, 0 | 20260507 16:24:23.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:23.972 | INFO | Uploaded 36GiB to GPU in 2.69739s (13.3462GiB/s) | 20260507 16:24:23.972 | INFO | Downloaded 34GiB to GPU in 2.16702s (15.6898GiB/s) | 20260507 16:24:29.110 | INFO | Analyzing chunk frames = [910336,910848) | 20260507 16:24:29.110 | INFO | MemInfo: { | 20260507 16:24:29.110 | INFO | "gpu" : " 17 GB", | 20260507 16:24:29.110 | INFO | "huge" : "", | 20260507 16:24:29.110 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:29.111 | INFO | "malloc" : " 14 GB", | 20260507 16:24:29.111 | INFO | "pinned" : " 29 GB" | 20260507 16:24:29.111 | INFO | } | 20260507 16:24:29.111 | INFO | Perf report for the last 5.13974 seconds: | 20260507 16:24:29.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:29.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260507 16:24:29.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | PRE_HQ : 40.3%, 1.0, 21.0ms, 0 | 20260507 16:24:29.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:29.111 | INFO | Uploaded 36GiB to GPU in 2.68655s (13.4001GiB/s) | 20260507 16:24:29.112 | INFO | Downloaded 34GiB to GPU in 2.22241s (15.2987GiB/s) | 20260507 16:24:34.249 | INFO | Analyzing chunk frames = [910848,911360) | 20260507 16:24:34.249 | INFO | MemInfo: { | 20260507 16:24:34.249 | INFO | "gpu" : " 17 GB", | 20260507 16:24:34.249 | INFO | "huge" : "", | 20260507 16:24:34.249 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:34.249 | INFO | "malloc" : " 14 GB", | 20260507 16:24:34.249 | INFO | "pinned" : " 29 GB" | 20260507 16:24:34.249 | INFO | } | 20260507 16:24:34.250 | INFO | Perf report for the last 5.13883 seconds: | 20260507 16:24:34.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:34.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | ANALYSIS : 65.8%, 3.0, 101.9ms, 0 | 20260507 16:24:34.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | PRE_HQ : 40.8%, 1.0, 21.2ms, 0 | 20260507 16:24:34.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:34.250 | INFO | Uploaded 36GiB to GPU in 2.68303s (13.4177GiB/s) | 20260507 16:24:34.250 | INFO | Downloaded 34GiB to GPU in 2.17982s (15.5976GiB/s) | 20260507 16:24:39.388 | INFO | Analyzing chunk frames = [911360,911872) | 20260507 16:24:39.388 | INFO | MemInfo: { | 20260507 16:24:39.388 | INFO | "gpu" : " 17 GB", | 20260507 16:24:39.389 | INFO | "huge" : "", | 20260507 16:24:39.389 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:39.389 | INFO | "malloc" : " 14 GB", | 20260507 16:24:39.389 | INFO | "pinned" : " 29 GB" | 20260507 16:24:39.389 | INFO | } | 20260507 16:24:39.389 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:24:39.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:39.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260507 16:24:39.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | PRE_HQ : 41.7%, 1.0, 21.8ms, 0 | 20260507 16:24:39.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:39.389 | INFO | Uploaded 36GiB to GPU in 2.70114s (13.3277GiB/s) | 20260507 16:24:39.389 | INFO | Downloaded 34GiB to GPU in 2.17894s (15.6039GiB/s) | 20260507 16:24:44.528 | INFO | Analyzing chunk frames = [911872,912384) | 20260507 16:24:44.528 | INFO | MemInfo: { | 20260507 16:24:44.528 | INFO | "gpu" : " 17 GB", | 20260507 16:24:44.528 | INFO | "huge" : "", | 20260507 16:24:44.528 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:44.528 | INFO | "malloc" : " 14 GB", | 20260507 16:24:44.528 | INFO | "pinned" : " 29 GB" | 20260507 16:24:44.528 | INFO | } | 20260507 16:24:44.529 | INFO | Perf report for the last 5.13966 seconds: | 20260507 16:24:44.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:44.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 16:24:44.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | PRE_HQ : 40.2%, 1.0, 20.9ms, 0 | 20260507 16:24:44.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:44.529 | INFO | Uploaded 36GiB to GPU in 2.70196s (13.3237GiB/s) | 20260507 16:24:44.529 | INFO | Downloaded 34GiB to GPU in 2.19776s (15.4703GiB/s) | 20260507 16:24:49.667 | INFO | Analyzing chunk frames = [912384,912896) | 20260507 16:24:49.667 | INFO | MemInfo: { | 20260507 16:24:49.667 | INFO | "gpu" : " 17 GB", | 20260507 16:24:49.667 | INFO | "huge" : "", | 20260507 16:24:49.667 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:49.667 | INFO | "malloc" : " 14 GB", | 20260507 16:24:49.667 | INFO | "pinned" : " 29 GB" | 20260507 16:24:49.668 | INFO | } | 20260507 16:24:49.668 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:24:49.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:49.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 16:24:49.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | PRE_HQ : 41.4%, 1.0, 21.6ms, 0 | 20260507 16:24:49.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:49.668 | INFO | Uploaded 36GiB to GPU in 2.68257s (13.4199GiB/s) | 20260507 16:24:49.668 | INFO | Downloaded 34GiB to GPU in 2.1768s (15.6192GiB/s) | 20260507 16:24:54.807 | INFO | Analyzing chunk frames = [912896,913408) | 20260507 16:24:54.807 | INFO | MemInfo: { | 20260507 16:24:54.807 | INFO | "gpu" : " 17 GB", | 20260507 16:24:54.807 | INFO | "huge" : "", | 20260507 16:24:54.807 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:54.807 | INFO | "malloc" : " 14 GB", | 20260507 16:24:54.807 | INFO | "pinned" : " 29 GB" | 20260507 16:24:54.807 | INFO | } | 20260507 16:24:54.807 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:24:54.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:54.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.807 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 16:24:54.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.807 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 16:24:54.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:54.808 | INFO | Uploaded 36GiB to GPU in 2.64494s (13.6109GiB/s) | 20260507 16:24:54.808 | INFO | Downloaded 34GiB to GPU in 2.23462s (15.2151GiB/s) | 20260507 16:24:54.922 | INFO | Starting a round of Baz IO | 20260507 16:24:58.350 | INFO | Populating RTMetrics report | 20260507 16:24:58.350 | INFO | RTMetrics report populated | 20260507 16:24:58.350 | INFO | Opening tmp RTMetrics json file | 20260507 16:24:58.353 | INFO | Serializing RTMetrics to json | 20260507 16:24:58.355 | INFO | Serialization done, swapping tmp file now... | 20260507 16:24:58.356 | INFO | RTMetrics done | 20260507 16:24:58.471 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.549 seconds | 20260507 16:24:58.472 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.34% / 0.03% / 49.63% | 20260507 16:24:58.472 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.53089% / 6.75519e-05% / 2.78155e-05% | 20260507 16:24:58.472 | INFO | Baz IO overall: | 20260507 16:24:58.472 | INFO | BytesWritten : 390776444118 | 20260507 16:24:58.472 | INFO | NumEvents : 189025756896 | 20260507 16:24:59.946 | INFO | Analyzing chunk frames = [913408,913920) | 20260507 16:24:59.946 | INFO | MemInfo: { | 20260507 16:24:59.946 | INFO | "gpu" : " 17 GB", | 20260507 16:24:59.946 | INFO | "huge" : "", | 20260507 16:24:59.946 | INFO | "huge_shared" : " 29 GB", | 20260507 16:24:59.946 | INFO | "malloc" : " 14 GB", | 20260507 16:24:59.946 | INFO | "pinned" : " 29 GB" | 20260507 16:24:59.946 | INFO | } | 20260507 16:24:59.946 | INFO | Perf report for the last 5.13907 seconds: | 20260507 16:24:59.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:24:59.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:59.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:59.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:59.946 | INFO | ANALYSIS : 68.0%, 2.9, 104.6ms, 0 | 20260507 16:24:59.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:24:59.946 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260507 16:24:59.946 | INFO | PRE_HQ : 66.5%, 2.0, 70.4ms, 0 | 20260507 16:24:59.946 | INFO | BAZWRITER : 68.7%, 3.9, 139.4ms, 0 | 20260507 16:24:59.947 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 16:24:59.947 | INFO | Uploaded 36GiB to GPU in 2.74758s (13.1024GiB/s) | 20260507 16:24:59.947 | INFO | Downloaded 38GiB to GPU in 2.47299s (15.366GiB/s) | 20260507 16:25:05.085 | INFO | Analyzing chunk frames = [913920,914432) | 20260507 16:25:05.085 | INFO | MemInfo: { | 20260507 16:25:05.085 | INFO | "gpu" : " 17 GB", | 20260507 16:25:05.085 | INFO | "huge" : "", | 20260507 16:25:05.085 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:05.085 | INFO | "malloc" : " 14 GB", | 20260507 16:25:05.085 | INFO | "pinned" : " 29 GB" | 20260507 16:25:05.085 | INFO | } | 20260507 16:25:05.085 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:25:05.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:05.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.086 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 16:25:05.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.086 | INFO | PRE_HQ : 39.5%, 1.0, 20.6ms, 0 | 20260507 16:25:05.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:05.086 | INFO | Uploaded 36GiB to GPU in 2.69374s (13.3643GiB/s) | 20260507 16:25:05.086 | INFO | Downloaded 34GiB to GPU in 2.20809s (15.3979GiB/s) | 20260507 16:25:10.225 | INFO | Analyzing chunk frames = [914432,914944) | 20260507 16:25:10.225 | INFO | MemInfo: { | 20260507 16:25:10.225 | INFO | "gpu" : " 17 GB", | 20260507 16:25:10.225 | INFO | "huge" : "", | 20260507 16:25:10.225 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:10.225 | INFO | "malloc" : " 14 GB", | 20260507 16:25:10.225 | INFO | "pinned" : " 29 GB" | 20260507 16:25:10.225 | INFO | } | 20260507 16:25:10.225 | INFO | Perf report for the last 5.13957 seconds: | 20260507 16:25:10.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:10.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260507 16:25:10.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | PRE_HQ : 39.9%, 1.0, 20.8ms, 0 | 20260507 16:25:10.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:10.225 | INFO | Uploaded 36GiB to GPU in 2.68093s (13.4282GiB/s) | 20260507 16:25:10.225 | INFO | Downloaded 34GiB to GPU in 2.07518s (16.3841GiB/s) | 20260507 16:25:15.364 | INFO | Analyzing chunk frames = [914944,915456) | 20260507 16:25:15.364 | INFO | MemInfo: { | 20260507 16:25:15.364 | INFO | "gpu" : " 17 GB", | 20260507 16:25:15.364 | INFO | "huge" : "", | 20260507 16:25:15.364 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:15.364 | INFO | "malloc" : " 14 GB", | 20260507 16:25:15.364 | INFO | "pinned" : " 29 GB" | 20260507 16:25:15.364 | INFO | } | 20260507 16:25:15.364 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:25:15.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:15.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.365 | INFO | ANALYSIS : 65.4%, 2.9, 101.1ms, 0 | 20260507 16:25:15.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.365 | INFO | PRE_HQ : 40.4%, 1.0, 21.0ms, 0 | 20260507 16:25:15.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:15.365 | INFO | Uploaded 36GiB to GPU in 2.68034s (13.4311GiB/s) | 20260507 16:25:15.365 | INFO | Downloaded 34GiB to GPU in 2.17599s (15.6251GiB/s) | 20260507 16:25:20.503 | INFO | Analyzing chunk frames = [915456,915968) | 20260507 16:25:20.503 | INFO | MemInfo: { | 20260507 16:25:20.503 | INFO | "gpu" : " 17 GB", | 20260507 16:25:20.504 | INFO | "huge" : "", | 20260507 16:25:20.504 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:20.504 | INFO | "malloc" : " 14 GB", | 20260507 16:25:20.504 | INFO | "pinned" : " 29 GB" | 20260507 16:25:20.504 | INFO | } | 20260507 16:25:20.504 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:25:20.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:20.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 16:25:20.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | PRE_HQ : 39.3%, 1.0, 20.5ms, 0 | 20260507 16:25:20.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:20.504 | INFO | Uploaded 36GiB to GPU in 2.70523s (13.3076GiB/s) | 20260507 16:25:20.504 | INFO | Downloaded 34GiB to GPU in 2.1527s (15.7941GiB/s) | 20260507 16:25:25.642 | INFO | Analyzing chunk frames = [915968,916480) | 20260507 16:25:25.642 | INFO | MemInfo: { | 20260507 16:25:25.642 | INFO | "gpu" : " 17 GB", | 20260507 16:25:25.642 | INFO | "huge" : "", | 20260507 16:25:25.642 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:25.642 | INFO | "malloc" : " 14 GB", | 20260507 16:25:25.642 | INFO | "pinned" : " 29 GB" | 20260507 16:25:25.643 | INFO | } | 20260507 16:25:25.643 | INFO | Perf report for the last 5.13892 seconds: | 20260507 16:25:25.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:25.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260507 16:25:25.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | PRE_HQ : 38.2%, 1.0, 19.9ms, 0 | 20260507 16:25:25.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:25.643 | INFO | Uploaded 36GiB to GPU in 2.67699s (13.4479GiB/s) | 20260507 16:25:25.643 | INFO | Downloaded 34GiB to GPU in 2.14192s (15.8736GiB/s) | 20260507 16:25:30.782 | INFO | Analyzing chunk frames = [916480,916992) | 20260507 16:25:30.782 | INFO | MemInfo: { | 20260507 16:25:30.782 | INFO | "gpu" : " 17 GB", | 20260507 16:25:30.782 | INFO | "huge" : "", | 20260507 16:25:30.782 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:30.782 | INFO | "malloc" : " 14 GB", | 20260507 16:25:30.782 | INFO | "pinned" : " 29 GB" | 20260507 16:25:30.782 | INFO | } | 20260507 16:25:30.782 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:25:30.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:30.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 16:25:30.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | PRE_HQ : 38.0%, 1.0, 19.8ms, 0 | 20260507 16:25:30.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:30.783 | INFO | Uploaded 36GiB to GPU in 2.66008s (13.5334GiB/s) | 20260507 16:25:30.783 | INFO | Downloaded 34GiB to GPU in 2.25396s (15.0845GiB/s) | 20260507 16:25:35.921 | INFO | Analyzing chunk frames = [916992,917504) | 20260507 16:25:35.921 | INFO | MemInfo: { | 20260507 16:25:35.921 | INFO | "gpu" : " 17 GB", | 20260507 16:25:35.921 | INFO | "huge" : "", | 20260507 16:25:35.921 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:35.922 | INFO | "malloc" : " 14 GB", | 20260507 16:25:35.922 | INFO | "pinned" : " 29 GB" | 20260507 16:25:35.922 | INFO | } | 20260507 16:25:35.922 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:25:35.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:35.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 16:25:35.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | PRE_HQ : 41.4%, 1.0, 21.6ms, 0 | 20260507 16:25:35.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:35.922 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:25:35.922 | INFO | Total mem usage: 16.4 GiB | 20260507 16:25:35.922 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:25:35.922 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 16:25:35.922 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:25:35.922 | INFO | High Water Breakdown: | 20260507 16:25:35.922 | INFO | 59.3% : 8.79 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:25:35.922 | INFO | 40.6% : 6.01 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:25:35.922 | INFO | Uploaded 36GiB to GPU in 2.69721s (13.3471GiB/s) | 20260507 16:25:35.922 | INFO | Downloaded 34GiB to GPU in 2.13347s (15.9365GiB/s) | 20260507 16:25:35.922 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:25:35.922 | INFO | ------------------------------------------ | 20260507 16:25:35.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:25:35.923 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:25:35.923 | INFO | Total mem usage: 26.5 GiB | 20260507 16:25:35.923 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:25:35.923 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:25:35.923 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:25:35.923 | INFO | High Water Breakdown: | 20260507 16:25:35.923 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:25:35.923 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:25:35.923 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:25:35.923 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:25:35.923 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:25:35.923 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:25:35.923 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:25:35.923 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:25:35.923 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:25:35.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:25:35.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:25:35.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:25:35.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:25:35.923 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:25:35.923 | INFO | ------------------------------------------ | 20260507 16:25:35.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:25:35.923 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:25:35.923 | INFO | Total mem usage: 27.8 GiB | 20260507 16:25:35.923 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:25:35.923 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:25:35.923 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:25:35.923 | INFO | High Water Breakdown: | 20260507 16:25:35.923 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:25:35.923 | INFO | ------------------------------------------ | 20260507 16:25:35.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:25:35.923 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:25:35.923 | INFO | Total mem usage: 15.7 GiB | 20260507 16:25:35.923 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:25:35.923 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.94%) | 20260507 16:25:35.923 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:25:35.923 | INFO | High Water Breakdown: | 20260507 16:25:35.923 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:25:35.923 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:25:35.923 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:25:35.923 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:25:35.923 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:25:35.923 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:25:35.923 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:25:35.923 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:25:35.923 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:25:35.923 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:25:35.923 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:25:35.923 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:25:35.924 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:25:35.924 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:25:35.924 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:25:35.924 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:25:35.924 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:25:35.924 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:25:35.924 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:25:35.924 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:25:35.924 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:25:35.924 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:25:35.924 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:25:35.924 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:25:35.924 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:25:35.924 | INFO | ------------------------------------------ | 20260507 16:25:35.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:25:35.924 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:25:35.924 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:25:35.924 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:25:35.924 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 0.32%, AggregateStats -- 9.59e-02 / -nan, 8.52e-03 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 11.14%, Baseline -- 3.36e+00 / -nan, 1.20e-01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 3.12%, Histogram -- 9.39e-01 / -nan, 8.05e-02 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 16.11%, DME -- 3.88e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.73e+01 / -nan, 1.34e-01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.15e+00 / -nan, 1.59e-01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 3.50%, Metrics -- 1.05e+00 / -nan, 2.32e-01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 1.29%, PulseSerializing -- 3.89e-01 / -nan, 6.59e-02 / 0.00e+00 | 20260507 16:25:35.924 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:25:35.924 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:25:35.924 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:25:35.924 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 26.74%, Upload -- 2.81e+01 / 0.00e+00, 8.80e+00 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 21.97%, Download -- 2.30e+01 / 0.00e+00, 6.52e+00 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 28.73%, Compute -- 3.01e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 7.99%, LockUpload -- 8.39e+00 / 0.00e+00, 9.80e+00 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 1.44%, LockDownload -- 1.52e+00 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 13.10%, LockCompute -- 1.37e+01 / 0.00e+00, 1.65e+01 / 0.00e+00 | 20260507 16:25:35.924 | INFO | PerfReport: 0.02%, Misc -- 1.94e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 16:25:36.035 | INFO | Starting a round of Baz IO | 20260507 16:25:39.481 | INFO | Populating RTMetrics report | 20260507 16:25:39.481 | INFO | RTMetrics report populated | 20260507 16:25:39.481 | INFO | Opening tmp RTMetrics json file | 20260507 16:25:39.484 | INFO | Serializing RTMetrics to json | 20260507 16:25:39.488 | INFO | Serialization done, swapping tmp file now... | 20260507 16:25:39.489 | INFO | RTMetrics done | 20260507 16:25:39.582 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.547 seconds | 20260507 16:25:39.582 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.32% / 0.03% / 49.66% | 20260507 16:25:39.582 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.50626% / 3.57628e-05% / 5.5631e-05% | 20260507 16:25:39.582 | INFO | Baz IO overall: | 20260507 16:25:39.582 | INFO | BytesWritten : 392688361326 | 20260507 16:25:39.582 | INFO | NumEvents : 189955531460 | 20260507 16:25:41.060 | INFO | Analyzing chunk frames = [917504,918016) | 20260507 16:25:41.060 | INFO | MemInfo: { | 20260507 16:25:41.060 | INFO | "gpu" : " 17 GB", | 20260507 16:25:41.060 | INFO | "huge" : "", | 20260507 16:25:41.060 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:41.060 | INFO | "malloc" : " 14 GB", | 20260507 16:25:41.060 | INFO | "pinned" : " 29 GB" | 20260507 16:25:41.060 | INFO | } | 20260507 16:25:41.061 | INFO | Perf report for the last 5.13887 seconds: | 20260507 16:25:41.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:41.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:41.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:41.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:41.061 | INFO | ANALYSIS : 68.4%, 2.9, 104.3ms, 0 | 20260507 16:25:41.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:41.061 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260507 16:25:41.061 | INFO | PRE_HQ : 67.2%, 2.0, 71.0ms, 0 | 20260507 16:25:41.061 | INFO | BAZWRITER : 68.6%, 4.3, 150.0ms, 0 | 20260507 16:25:41.061 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:25:41.061 | INFO | Uploaded 36GiB to GPU in 2.74305s (13.1241GiB/s) | 20260507 16:25:41.061 | INFO | Downloaded 38GiB to GPU in 2.47174s (15.3738GiB/s) | 20260507 16:25:46.200 | INFO | Analyzing chunk frames = [918016,918528) | 20260507 16:25:46.200 | INFO | MemInfo: { | 20260507 16:25:46.200 | INFO | "gpu" : " 17 GB", | 20260507 16:25:46.200 | INFO | "huge" : "", | 20260507 16:25:46.200 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:46.200 | INFO | "malloc" : " 14 GB", | 20260507 16:25:46.200 | INFO | "pinned" : " 29 GB" | 20260507 16:25:46.200 | INFO | } | 20260507 16:25:46.200 | INFO | Perf report for the last 5.13981 seconds: | 20260507 16:25:46.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:46.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.200 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 16:25:46.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.200 | INFO | PRE_HQ : 38.7%, 1.0, 20.2ms, 0 | 20260507 16:25:46.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:46.201 | INFO | Uploaded 36GiB to GPU in 2.69072s (13.3793GiB/s) | 20260507 16:25:46.201 | INFO | Downloaded 34GiB to GPU in 2.1114s (16.1031GiB/s) | 20260507 16:25:51.339 | INFO | Analyzing chunk frames = [918528,919040) | 20260507 16:25:51.339 | INFO | MemInfo: { | 20260507 16:25:51.339 | INFO | "gpu" : " 17 GB", | 20260507 16:25:51.340 | INFO | "huge" : "", | 20260507 16:25:51.340 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:51.340 | INFO | "malloc" : " 14 GB", | 20260507 16:25:51.340 | INFO | "pinned" : " 29 GB" | 20260507 16:25:51.340 | INFO | } | 20260507 16:25:51.340 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:25:51.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:51.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.340 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260507 16:25:51.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.340 | INFO | PRE_HQ : 40.3%, 1.0, 21.0ms, 0 | 20260507 16:25:51.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:51.341 | INFO | Uploaded 36GiB to GPU in 2.68034s (13.4311GiB/s) | 20260507 16:25:51.341 | INFO | Downloaded 34GiB to GPU in 2.15452s (15.7808GiB/s) | 20260507 16:25:56.479 | INFO | Analyzing chunk frames = [919040,919552) | 20260507 16:25:56.479 | INFO | MemInfo: { | 20260507 16:25:56.479 | INFO | "gpu" : " 17 GB", | 20260507 16:25:56.479 | INFO | "huge" : "", | 20260507 16:25:56.479 | INFO | "huge_shared" : " 29 GB", | 20260507 16:25:56.479 | INFO | "malloc" : " 14 GB", | 20260507 16:25:56.479 | INFO | "pinned" : " 29 GB" | 20260507 16:25:56.479 | INFO | } | 20260507 16:25:56.479 | INFO | Perf report for the last 5.13879 seconds: | 20260507 16:25:56.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:25:56.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.479 | INFO | ANALYSIS : 65.4%, 3.0, 101.2ms, 0 | 20260507 16:25:56.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.479 | INFO | PRE_HQ : 40.2%, 1.0, 21.0ms, 0 | 20260507 16:25:56.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:25:56.480 | INFO | Uploaded 36GiB to GPU in 2.69004s (13.3827GiB/s) | 20260507 16:25:56.480 | INFO | Downloaded 34GiB to GPU in 2.1262s (15.991GiB/s) | 20260507 16:26:01.618 | INFO | Analyzing chunk frames = [919552,920064) | 20260507 16:26:01.618 | INFO | MemInfo: { | 20260507 16:26:01.618 | INFO | "gpu" : " 17 GB", | 20260507 16:26:01.618 | INFO | "huge" : "", | 20260507 16:26:01.618 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:01.618 | INFO | "malloc" : " 14 GB", | 20260507 16:26:01.618 | INFO | "pinned" : " 29 GB" | 20260507 16:26:01.618 | INFO | } | 20260507 16:26:01.619 | INFO | Perf report for the last 5.13974 seconds: | 20260507 16:26:01.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:01.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260507 16:26:01.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | PRE_HQ : 41.3%, 1.0, 21.6ms, 0 | 20260507 16:26:01.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:01.619 | INFO | Uploaded 36GiB to GPU in 2.70703s (13.2987GiB/s) | 20260507 16:26:01.619 | INFO | Downloaded 34GiB to GPU in 2.14419s (15.8568GiB/s) | 20260507 16:26:06.757 | INFO | Analyzing chunk frames = [920064,920576) | 20260507 16:26:06.757 | INFO | MemInfo: { | 20260507 16:26:06.757 | INFO | "gpu" : " 17 GB", | 20260507 16:26:06.758 | INFO | "huge" : "", | 20260507 16:26:06.758 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:06.758 | INFO | "malloc" : " 14 GB", | 20260507 16:26:06.758 | INFO | "pinned" : " 29 GB" | 20260507 16:26:06.758 | INFO | } | 20260507 16:26:06.758 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:26:06.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:06.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 16:26:06.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 16:26:06.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:06.758 | INFO | Uploaded 36GiB to GPU in 2.696s (13.3531GiB/s) | 20260507 16:26:06.758 | INFO | Downloaded 34GiB to GPU in 2.26138s (15.035GiB/s) | 20260507 16:26:11.897 | INFO | Analyzing chunk frames = [920576,921088) | 20260507 16:26:11.897 | INFO | MemInfo: { | 20260507 16:26:11.897 | INFO | "gpu" : " 17 GB", | 20260507 16:26:11.897 | INFO | "huge" : "", | 20260507 16:26:11.897 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:11.897 | INFO | "malloc" : " 14 GB", | 20260507 16:26:11.897 | INFO | "pinned" : " 29 GB" | 20260507 16:26:11.897 | INFO | } | 20260507 16:26:11.897 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:26:11.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:11.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.897 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260507 16:26:11.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.897 | INFO | PRE_HQ : 38.6%, 1.0, 20.1ms, 0 | 20260507 16:26:11.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:11.898 | INFO | Uploaded 36GiB to GPU in 2.67779s (13.4439GiB/s) | 20260507 16:26:11.898 | INFO | Downloaded 34GiB to GPU in 2.246s (15.138GiB/s) | 20260507 16:26:17.036 | INFO | Analyzing chunk frames = [921088,921600) | 20260507 16:26:17.036 | INFO | MemInfo: { | 20260507 16:26:17.036 | INFO | "gpu" : " 17 GB", | 20260507 16:26:17.036 | INFO | "huge" : "", | 20260507 16:26:17.036 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:17.036 | INFO | "malloc" : " 14 GB", | 20260507 16:26:17.036 | INFO | "pinned" : " 29 GB" | 20260507 16:26:17.036 | INFO | } | 20260507 16:26:17.036 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:26:17.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:17.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.036 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 16:26:17.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.037 | INFO | PRE_HQ : 40.9%, 1.0, 21.3ms, 0 | 20260507 16:26:17.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:17.037 | INFO | Uploaded 36GiB to GPU in 2.67259s (13.4701GiB/s) | 20260507 16:26:17.037 | INFO | Downloaded 34GiB to GPU in 2.12936s (15.9672GiB/s) | 20260507 16:26:17.149 | INFO | Starting a round of Baz IO | 20260507 16:26:20.625 | INFO | Populating RTMetrics report | 20260507 16:26:20.626 | INFO | RTMetrics report populated | 20260507 16:26:20.626 | INFO | Opening tmp RTMetrics json file | 20260507 16:26:20.630 | INFO | Serializing RTMetrics to json | 20260507 16:26:20.635 | INFO | Serialization done, swapping tmp file now... | 20260507 16:26:20.636 | INFO | RTMetrics done | 20260507 16:26:20.788 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.639 seconds | 20260507 16:26:20.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.29% / 0.03% / 49.68% | 20260507 16:26:20.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.521% / 1.98682e-05% / 3.17891e-05% | 20260507 16:26:20.788 | INFO | Baz IO overall: | 20260507 16:26:20.788 | INFO | BytesWritten : 394603775709 | 20260507 16:26:20.788 | INFO | NumEvents : 190887582976 | 20260507 16:26:22.175 | INFO | Analyzing chunk frames = [921600,922112) | 20260507 16:26:22.175 | INFO | MemInfo: { | 20260507 16:26:22.175 | INFO | "gpu" : " 17 GB", | 20260507 16:26:22.175 | INFO | "huge" : "", | 20260507 16:26:22.175 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:22.175 | INFO | "malloc" : " 14 GB", | 20260507 16:26:22.175 | INFO | "pinned" : " 29 GB" | 20260507 16:26:22.175 | INFO | } | 20260507 16:26:22.176 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:26:22.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:22.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:22.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:22.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:22.176 | INFO | ANALYSIS : 68.4%, 2.7, 98.2ms, 0 | 20260507 16:26:22.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:22.176 | INFO | RT_METRICS : 55.1%, 1.0, 28.9ms, 0 | 20260507 16:26:22.176 | INFO | PRE_HQ : 68.4%, 2.3, 81.6ms, 0 | 20260507 16:26:22.176 | INFO | BAZWRITER : 70.4%, 4.8, 174.2ms, 0 | 20260507 16:26:22.176 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 16:26:22.176 | INFO | Uploaded 36GiB to GPU in 2.45811s (14.6454GiB/s) | 20260507 16:26:22.176 | INFO | Downloaded 38GiB to GPU in 2.37579s (15.9947GiB/s) | 20260507 16:26:27.314 | INFO | Analyzing chunk frames = [922112,922624) | 20260507 16:26:27.314 | INFO | MemInfo: { | 20260507 16:26:27.314 | INFO | "gpu" : " 17 GB", | 20260507 16:26:27.314 | INFO | "huge" : "", | 20260507 16:26:27.314 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:27.315 | INFO | "malloc" : " 14 GB", | 20260507 16:26:27.315 | INFO | "pinned" : " 29 GB" | 20260507 16:26:27.315 | INFO | } | 20260507 16:26:27.315 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:26:27.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:27.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260507 16:26:27.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 16:26:27.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:27.315 | INFO | Uploaded 36GiB to GPU in 2.40973s (14.9394GiB/s) | 20260507 16:26:27.315 | INFO | Downloaded 34GiB to GPU in 2.13422s (15.9309GiB/s) | 20260507 16:26:32.454 | INFO | Analyzing chunk frames = [922624,923136) | 20260507 16:26:32.454 | INFO | MemInfo: { | 20260507 16:26:32.454 | INFO | "gpu" : " 17 GB", | 20260507 16:26:32.454 | INFO | "huge" : "", | 20260507 16:26:32.454 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:32.454 | INFO | "malloc" : " 14 GB", | 20260507 16:26:32.454 | INFO | "pinned" : " 29 GB" | 20260507 16:26:32.454 | INFO | } | 20260507 16:26:32.454 | INFO | Perf report for the last 5.13964 seconds: | 20260507 16:26:32.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:32.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.454 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260507 16:26:32.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.454 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 16:26:32.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:32.455 | INFO | Uploaded 36GiB to GPU in 2.37578s (15.1529GiB/s) | 20260507 16:26:32.455 | INFO | Downloaded 34GiB to GPU in 2.08938s (16.2727GiB/s) | 20260507 16:26:37.593 | INFO | Analyzing chunk frames = [923136,923648) | 20260507 16:26:37.593 | INFO | MemInfo: { | 20260507 16:26:37.593 | INFO | "gpu" : " 17 GB", | 20260507 16:26:37.593 | INFO | "huge" : "", | 20260507 16:26:37.593 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:37.593 | INFO | "malloc" : " 14 GB", | 20260507 16:26:37.593 | INFO | "pinned" : " 29 GB" | 20260507 16:26:37.593 | INFO | } | 20260507 16:26:37.593 | INFO | Perf report for the last 5.13914 seconds: | 20260507 16:26:37.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:37.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.594 | INFO | ANALYSIS : 64.9%, 2.9, 100.4ms, 0 | 20260507 16:26:37.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.594 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:26:37.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:37.594 | INFO | Uploaded 36GiB to GPU in 2.34895s (15.326GiB/s) | 20260507 16:26:37.594 | INFO | Downloaded 34GiB to GPU in 2.14408s (15.8576GiB/s) | 20260507 16:26:42.732 | INFO | Analyzing chunk frames = [923648,924160) | 20260507 16:26:42.732 | INFO | MemInfo: { | 20260507 16:26:42.732 | INFO | "gpu" : " 17 GB", | 20260507 16:26:42.732 | INFO | "huge" : "", | 20260507 16:26:42.732 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:42.733 | INFO | "malloc" : " 14 GB", | 20260507 16:26:42.733 | INFO | "pinned" : " 29 GB" | 20260507 16:26:42.733 | INFO | } | 20260507 16:26:42.733 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:26:42.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:42.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.733 | INFO | ANALYSIS : 65.0%, 3.0, 101.4ms, 0 | 20260507 16:26:42.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.733 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 16:26:42.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:42.734 | INFO | Uploaded 36GiB to GPU in 2.37229s (15.1752GiB/s) | 20260507 16:26:42.734 | INFO | Downloaded 34GiB to GPU in 2.13862s (15.8981GiB/s) | 20260507 16:26:47.872 | INFO | Analyzing chunk frames = [924160,924672) | 20260507 16:26:47.872 | INFO | MemInfo: { | 20260507 16:26:47.872 | INFO | "gpu" : " 17 GB", | 20260507 16:26:47.872 | INFO | "huge" : "", | 20260507 16:26:47.872 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:47.872 | INFO | "malloc" : " 14 GB", | 20260507 16:26:47.872 | INFO | "pinned" : " 29 GB" | 20260507 16:26:47.872 | INFO | } | 20260507 16:26:47.872 | INFO | Perf report for the last 5.13946 seconds: | 20260507 16:26:47.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:47.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260507 16:26:47.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 16:26:47.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:47.872 | INFO | Uploaded 36GiB to GPU in 2.38312s (15.1062GiB/s) | 20260507 16:26:47.873 | INFO | Downloaded 34GiB to GPU in 2.18622s (15.5519GiB/s) | 20260507 16:26:53.011 | INFO | Analyzing chunk frames = [924672,925184) | 20260507 16:26:53.011 | INFO | MemInfo: { | 20260507 16:26:53.011 | INFO | "gpu" : " 17 GB", | 20260507 16:26:53.011 | INFO | "huge" : "", | 20260507 16:26:53.011 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:53.011 | INFO | "malloc" : " 14 GB", | 20260507 16:26:53.012 | INFO | "pinned" : " 29 GB" | 20260507 16:26:53.012 | INFO | } | 20260507 16:26:53.012 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:26:53.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:53.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:26:53.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:26:53.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:53.012 | INFO | Uploaded 36GiB to GPU in 2.37946s (15.1295GiB/s) | 20260507 16:26:53.012 | INFO | Downloaded 34GiB to GPU in 2.32742s (14.6084GiB/s) | 20260507 16:26:58.151 | INFO | Analyzing chunk frames = [925184,925696) | 20260507 16:26:58.151 | INFO | MemInfo: { | 20260507 16:26:58.151 | INFO | "gpu" : " 17 GB", | 20260507 16:26:58.151 | INFO | "huge" : "", | 20260507 16:26:58.151 | INFO | "huge_shared" : " 29 GB", | 20260507 16:26:58.151 | INFO | "malloc" : " 14 GB", | 20260507 16:26:58.151 | INFO | "pinned" : " 29 GB" | 20260507 16:26:58.151 | INFO | } | 20260507 16:26:58.151 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:26:58.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:26:58.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 16:26:58.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:26:58.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:26:58.151 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:26:58.151 | INFO | Total mem usage: 16.4 GiB | 20260507 16:26:58.151 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:26:58.151 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -6.01%) | 20260507 16:26:58.151 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:26:58.151 | INFO | High Water Breakdown: | 20260507 16:26:58.151 | INFO | 60% : 9.29 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:26:58.151 | INFO | 39.8% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:26:58.151 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:26:58.151 | INFO | ------------------------------------------ | 20260507 16:26:58.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:26:58.152 | INFO | Uploaded 36GiB to GPU in 2.40071s (14.9955GiB/s) | 20260507 16:26:58.152 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:26:58.152 | INFO | Total mem usage: 26.5 GiB | 20260507 16:26:58.152 | INFO | Downloaded 34GiB to GPU in 2.40111s (14.1601GiB/s) | 20260507 16:26:58.152 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:26:58.152 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 16:26:58.152 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:26:58.152 | INFO | High Water Breakdown: | 20260507 16:26:58.152 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:26:58.152 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:26:58.152 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:26:58.152 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:26:58.152 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:26:58.152 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:26:58.152 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:26:58.152 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:26:58.152 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:26:58.152 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:26:58.152 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:26:58.152 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:26:58.152 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:26:58.152 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:26:58.152 | INFO | ------------------------------------------ | 20260507 16:26:58.152 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:26:58.152 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:26:58.152 | INFO | Total mem usage: 27.8 GiB | 20260507 16:26:58.152 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:26:58.152 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:26:58.152 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:26:58.152 | INFO | High Water Breakdown: | 20260507 16:26:58.152 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:26:58.152 | INFO | ------------------------------------------ | 20260507 16:26:58.152 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:26:58.152 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:26:58.152 | INFO | Total mem usage: 15.7 GiB | 20260507 16:26:58.152 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:26:58.152 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.94%) | 20260507 16:26:58.152 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:26:58.152 | INFO | High Water Breakdown: | 20260507 16:26:58.152 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:26:58.152 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:26:58.152 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:26:58.152 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:26:58.152 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:26:58.152 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:26:58.152 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:26:58.152 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:26:58.152 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:26:58.152 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:26:58.152 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:26:58.152 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:26:58.152 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:26:58.152 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:26:58.152 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:26:58.152 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:26:58.152 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:26:58.152 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:26:58.152 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:26:58.152 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:26:58.152 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:26:58.152 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:26:58.152 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:26:58.152 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:26:58.152 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:26:58.152 | INFO | ------------------------------------------ | 20260507 16:26:58.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:26:58.153 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:26:58.153 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:26:58.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:26:58.153 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.47e+01, 1.34e+01 / 4.10e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 0.31%, AggregateStats -- 9.58e-02 / 1.02e-01, 1.00e-02 / 6.06e-03 | 20260507 16:26:58.153 | INFO | PerfReport: 11.40%, Baseline -- 3.57e+00 / 4.33e+00, 4.60e-01 / 4.10e-01 | 20260507 16:26:58.153 | INFO | PerfReport: 3.20%, Histogram -- 1.00e+00 / 1.40e+00, 1.60e-01 / 1.42e-01 | 20260507 16:26:58.153 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.45e+01, 4.09e+00 / 3.71e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.79e+01 / 2.00e+01, 1.60e+00 / 2.10e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.68e+00, 3.15e-01 / 3.07e-01 | 20260507 16:26:58.153 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.33e+00, 2.73e-01 / 2.14e-01 | 20260507 16:26:58.153 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.91e-01 / 4.20e-01, 6.47e-02 / 4.74e-02 | 20260507 16:26:58.153 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:26:58.153 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:26:58.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:26:58.153 | INFO | PerfReport: Full Execution -- 1.04e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 25.36%, Upload -- 2.65e+01 / 0.00e+00, 8.87e+00 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 22.10%, Download -- 2.31e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 30.03%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 5.70%, LockUpload -- 5.96e+00 / 0.00e+00, 8.73e+00 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 1.52%, LockDownload -- 1.59e+00 / 0.00e+00, 3.74e+00 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 15.27%, LockCompute -- 1.60e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260507 16:26:58.153 | INFO | PerfReport: 0.02%, Misc -- 1.69e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 16:26:58.248 | INFO | Starting a round of Baz IO | 20260507 16:27:01.810 | INFO | Populating RTMetrics report | 20260507 16:27:01.810 | INFO | RTMetrics report populated | 20260507 16:27:01.810 | INFO | Opening tmp RTMetrics json file | 20260507 16:27:01.837 | INFO | Serializing RTMetrics to json | 20260507 16:27:01.848 | INFO | Serialization done, swapping tmp file now... | 20260507 16:27:01.868 | INFO | RTMetrics done | 20260507 16:27:01.925 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.676 seconds | 20260507 16:27:01.925 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.26% / 0.03% / 49.71% | 20260507 16:27:01.925 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.52946% / 3.57628e-05% / 3.17891e-05% | 20260507 16:27:01.925 | INFO | Baz IO overall: | 20260507 16:27:01.925 | INFO | BytesWritten : 396522993460 | 20260507 16:27:01.925 | INFO | NumEvents : 191822057242 | 20260507 16:27:03.290 | INFO | Analyzing chunk frames = [925696,926208) | 20260507 16:27:03.290 | INFO | MemInfo: { | 20260507 16:27:03.290 | INFO | "gpu" : " 17 GB", | 20260507 16:27:03.290 | INFO | "huge" : "", | 20260507 16:27:03.290 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:03.290 | INFO | "malloc" : " 14 GB", | 20260507 16:27:03.290 | INFO | "pinned" : " 29 GB" | 20260507 16:27:03.290 | INFO | } | 20260507 16:27:03.290 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:27:03.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:03.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:03.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:03.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:03.290 | INFO | ANALYSIS : 70.7%, 3.0, 109.6ms, 0 | 20260507 16:27:03.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:03.291 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 16:27:03.291 | INFO | PRE_HQ : 66.4%, 1.7, 60.2ms, 0 | 20260507 16:27:03.291 | INFO | BAZWRITER : 71.0%, 3.6, 130.7ms, 0 | 20260507 16:27:03.291 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:27:03.291 | INFO | Uploaded 36GiB to GPU in 2.53869s (14.1805GiB/s) | 20260507 16:27:03.291 | INFO | Downloaded 38GiB to GPU in 2.71014s (14.0214GiB/s) | 20260507 16:27:08.429 | INFO | Analyzing chunk frames = [926208,926720) | 20260507 16:27:08.429 | INFO | MemInfo: { | 20260507 16:27:08.429 | INFO | "gpu" : " 17 GB", | 20260507 16:27:08.429 | INFO | "huge" : "", | 20260507 16:27:08.429 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:08.429 | INFO | "malloc" : " 14 GB", | 20260507 16:27:08.429 | INFO | "pinned" : " 29 GB" | 20260507 16:27:08.430 | INFO | } | 20260507 16:27:08.430 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:27:08.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:08.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.430 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 16:27:08.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.430 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:27:08.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:08.431 | INFO | Uploaded 36GiB to GPU in 2.24144s (16.0611GiB/s) | 20260507 16:27:08.431 | INFO | Downloaded 34GiB to GPU in 2.29093s (14.8412GiB/s) | 20260507 16:27:13.569 | INFO | Analyzing chunk frames = [926720,927232) | 20260507 16:27:13.569 | INFO | MemInfo: { | 20260507 16:27:13.569 | INFO | "gpu" : " 17 GB", | 20260507 16:27:13.569 | INFO | "huge" : "", | 20260507 16:27:13.569 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:13.569 | INFO | "malloc" : " 14 GB", | 20260507 16:27:13.569 | INFO | "pinned" : " 29 GB" | 20260507 16:27:13.569 | INFO | } | 20260507 16:27:13.569 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:27:13.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:13.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | ANALYSIS : 65.3%, 3.0, 101.4ms, 0 | 20260507 16:27:13.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:27:13.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:13.569 | INFO | Uploaded 36GiB to GPU in 2.23633s (16.0978GiB/s) | 20260507 16:27:13.569 | INFO | Downloaded 34GiB to GPU in 2.28967s (14.8493GiB/s) | 20260507 16:27:18.708 | INFO | Analyzing chunk frames = [927232,927744) | 20260507 16:27:18.708 | INFO | MemInfo: { | 20260507 16:27:18.708 | INFO | "gpu" : " 17 GB", | 20260507 16:27:18.708 | INFO | "huge" : "", | 20260507 16:27:18.708 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:18.708 | INFO | "malloc" : " 14 GB", | 20260507 16:27:18.708 | INFO | "pinned" : " 29 GB" | 20260507 16:27:18.708 | INFO | } | 20260507 16:27:18.708 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:27:18.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:18.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.708 | INFO | ANALYSIS : 65.3%, 2.9, 101.0ms, 0 | 20260507 16:27:18.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.708 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:27:18.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:18.709 | INFO | Uploaded 36GiB to GPU in 2.28972s (15.7225GiB/s) | 20260507 16:27:18.709 | INFO | Downloaded 34GiB to GPU in 2.27533s (14.9429GiB/s) | 20260507 16:27:23.847 | INFO | Analyzing chunk frames = [927744,928256) | 20260507 16:27:23.847 | INFO | MemInfo: { | 20260507 16:27:23.847 | INFO | "gpu" : " 17 GB", | 20260507 16:27:23.847 | INFO | "huge" : "", | 20260507 16:27:23.847 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:23.847 | INFO | "malloc" : " 14 GB", | 20260507 16:27:23.848 | INFO | "pinned" : " 29 GB" | 20260507 16:27:23.848 | INFO | } | 20260507 16:27:23.848 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:27:23.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:23.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | ANALYSIS : 65.1%, 3.0, 101.7ms, 0 | 20260507 16:27:23.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 16:27:23.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:23.848 | INFO | Uploaded 36GiB to GPU in 2.26295s (15.9084GiB/s) | 20260507 16:27:23.848 | INFO | Downloaded 34GiB to GPU in 2.35461s (14.4398GiB/s) | 20260507 16:27:28.987 | INFO | Analyzing chunk frames = [928256,928768) | 20260507 16:27:28.987 | INFO | MemInfo: { | 20260507 16:27:28.987 | INFO | "gpu" : " 17 GB", | 20260507 16:27:28.987 | INFO | "huge" : "", | 20260507 16:27:28.987 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:28.987 | INFO | "malloc" : " 14 GB", | 20260507 16:27:28.987 | INFO | "pinned" : " 29 GB" | 20260507 16:27:28.987 | INFO | } | 20260507 16:27:28.987 | INFO | Perf report for the last 5.13946 seconds: | 20260507 16:27:28.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:28.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | ANALYSIS : 69.6%, 3.0, 108.5ms, 0 | 20260507 16:27:28.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:27:28.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:28.987 | INFO | Uploaded 36GiB to GPU in 2.60616s (13.8134GiB/s) | 20260507 16:27:28.987 | INFO | Downloaded 34GiB to GPU in 2.51745s (13.5057GiB/s) | 20260507 16:27:34.126 | INFO | Analyzing chunk frames = [928768,929280) | 20260507 16:27:34.126 | INFO | MemInfo: { | 20260507 16:27:34.126 | INFO | "gpu" : " 17 GB", | 20260507 16:27:34.126 | INFO | "huge" : "", | 20260507 16:27:34.126 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:34.126 | INFO | "malloc" : " 14 GB", | 20260507 16:27:34.126 | INFO | "pinned" : " 29 GB" | 20260507 16:27:34.126 | INFO | } | 20260507 16:27:34.126 | INFO | Perf report for the last 5.13905 seconds: | 20260507 16:27:34.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:34.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.126 | INFO | ANALYSIS : 68.9%, 3.0, 107.4ms, 0 | 20260507 16:27:34.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.126 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:27:34.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:34.127 | INFO | Uploaded 36GiB to GPU in 2.42077s (14.8713GiB/s) | 20260507 16:27:34.127 | INFO | Downloaded 34GiB to GPU in 2.50863s (13.5532GiB/s) | 20260507 16:27:39.265 | INFO | Analyzing chunk frames = [929280,929792) | 20260507 16:27:39.266 | INFO | MemInfo: { | 20260507 16:27:39.266 | INFO | "gpu" : " 17 GB", | 20260507 16:27:39.266 | INFO | "huge" : "", | 20260507 16:27:39.266 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:39.266 | INFO | "malloc" : " 14 GB", | 20260507 16:27:39.266 | INFO | "pinned" : " 29 GB" | 20260507 16:27:39.266 | INFO | } | 20260507 16:27:39.266 | INFO | Perf report for the last 5.1395 seconds: | 20260507 16:27:39.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:39.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | ANALYSIS : 69.1%, 3.0, 107.7ms, 0 | 20260507 16:27:39.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:27:39.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:39.266 | INFO | Uploaded 36GiB to GPU in 2.49025s (14.4564GiB/s) | 20260507 16:27:39.266 | INFO | Downloaded 34GiB to GPU in 2.51594s (13.5138GiB/s) | 20260507 16:27:39.362 | INFO | Starting a round of Baz IO | 20260507 16:27:42.884 | INFO | Populating RTMetrics report | 20260507 16:27:42.884 | INFO | RTMetrics report populated | 20260507 16:27:42.884 | INFO | Opening tmp RTMetrics json file | 20260507 16:27:42.898 | INFO | Serializing RTMetrics to json | 20260507 16:27:42.903 | INFO | Serialization done, swapping tmp file now... | 20260507 16:27:42.903 | INFO | RTMetrics done | 20260507 16:27:42.987 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.624 seconds | 20260507 16:27:42.987 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.23% / 0.03% / 49.74% | 20260507 16:27:42.987 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.50594% / 2.78155e-05% / 1.58946e-05% | 20260507 16:27:42.987 | INFO | Baz IO overall: | 20260507 16:27:42.987 | INFO | BytesWritten : 398440095544 | 20260507 16:27:42.987 | INFO | NumEvents : 192755326246 | 20260507 16:27:44.405 | INFO | Analyzing chunk frames = [929792,930304) | 20260507 16:27:44.405 | INFO | MemInfo: { | 20260507 16:27:44.405 | INFO | "gpu" : " 17 GB", | 20260507 16:27:44.405 | INFO | "huge" : "", | 20260507 16:27:44.405 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:44.405 | INFO | "malloc" : " 14 GB", | 20260507 16:27:44.405 | INFO | "pinned" : " 29 GB" | 20260507 16:27:44.405 | INFO | } | 20260507 16:27:44.405 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:27:44.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:44.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:44.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:44.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:44.405 | INFO | ANALYSIS : 69.9%, 3.0, 108.3ms, 0 | 20260507 16:27:44.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:44.405 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 16:27:44.405 | INFO | PRE_HQ : 66.8%, 1.7, 60.3ms, 0 | 20260507 16:27:44.405 | INFO | BAZWRITER : 70.1%, 3.7, 134.3ms, 0 | 20260507 16:27:44.405 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 16:27:44.405 | INFO | Uploaded 36GiB to GPU in 2.59263s (13.8855GiB/s) | 20260507 16:27:44.405 | INFO | Downloaded 38GiB to GPU in 2.72063s (13.9674GiB/s) | 20260507 16:27:49.544 | INFO | Analyzing chunk frames = [930304,930816) | 20260507 16:27:49.544 | INFO | MemInfo: { | 20260507 16:27:49.544 | INFO | "gpu" : " 17 GB", | 20260507 16:27:49.544 | INFO | "huge" : "", | 20260507 16:27:49.544 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:49.544 | INFO | "malloc" : " 14 GB", | 20260507 16:27:49.544 | INFO | "pinned" : " 29 GB" | 20260507 16:27:49.544 | INFO | } | 20260507 16:27:49.544 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:27:49.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:49.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.544 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:27:49.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.544 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:27:49.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:49.545 | INFO | Uploaded 36GiB to GPU in 2.49978s (14.4013GiB/s) | 20260507 16:27:49.545 | INFO | Downloaded 34GiB to GPU in 2.44187s (13.9237GiB/s) | 20260507 16:27:54.683 | INFO | Analyzing chunk frames = [930816,931328) | 20260507 16:27:54.683 | INFO | MemInfo: { | 20260507 16:27:54.683 | INFO | "gpu" : " 17 GB", | 20260507 16:27:54.683 | INFO | "huge" : "", | 20260507 16:27:54.683 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:54.683 | INFO | "malloc" : " 14 GB", | 20260507 16:27:54.683 | INFO | "pinned" : " 29 GB" | 20260507 16:27:54.683 | INFO | } | 20260507 16:27:54.683 | INFO | Perf report for the last 5.13916 seconds: | 20260507 16:27:54.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:54.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260507 16:27:54.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:27:54.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:54.684 | INFO | Uploaded 36GiB to GPU in 2.44784s (14.7069GiB/s) | 20260507 16:27:54.684 | INFO | Downloaded 34GiB to GPU in 2.35377s (14.4449GiB/s) | 20260507 16:27:59.822 | INFO | Analyzing chunk frames = [931328,931840) | 20260507 16:27:59.822 | INFO | MemInfo: { | 20260507 16:27:59.822 | INFO | "gpu" : " 17 GB", | 20260507 16:27:59.822 | INFO | "huge" : "", | 20260507 16:27:59.822 | INFO | "huge_shared" : " 29 GB", | 20260507 16:27:59.822 | INFO | "malloc" : " 14 GB", | 20260507 16:27:59.822 | INFO | "pinned" : " 29 GB" | 20260507 16:27:59.822 | INFO | } | 20260507 16:27:59.822 | INFO | Perf report for the last 5.13906 seconds: | 20260507 16:27:59.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:27:59.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 16:27:59.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:27:59.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:27:59.823 | INFO | Uploaded 36GiB to GPU in 2.45013s (14.6931GiB/s) | 20260507 16:27:59.823 | INFO | Downloaded 34GiB to GPU in 2.35776s (14.4205GiB/s) | 20260507 16:28:04.962 | INFO | Analyzing chunk frames = [931840,932352) | 20260507 16:28:04.962 | INFO | MemInfo: { | 20260507 16:28:04.962 | INFO | "gpu" : " 17 GB", | 20260507 16:28:04.962 | INFO | "huge" : "", | 20260507 16:28:04.962 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:04.962 | INFO | "malloc" : " 14 GB", | 20260507 16:28:04.962 | INFO | "pinned" : " 29 GB" | 20260507 16:28:04.962 | INFO | } | 20260507 16:28:04.962 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:28:04.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:04.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.962 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:28:04.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.962 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 16:28:04.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:04.963 | INFO | Uploaded 36GiB to GPU in 2.39835s (15.0103GiB/s) | 20260507 16:28:04.963 | INFO | Downloaded 34GiB to GPU in 2.46275s (13.8057GiB/s) | 20260507 16:28:10.101 | INFO | Analyzing chunk frames = [932352,932864) | 20260507 16:28:10.101 | INFO | MemInfo: { | 20260507 16:28:10.101 | INFO | "gpu" : " 17 GB", | 20260507 16:28:10.101 | INFO | "huge" : "", | 20260507 16:28:10.101 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:10.101 | INFO | "malloc" : " 14 GB", | 20260507 16:28:10.101 | INFO | "pinned" : " 29 GB" | 20260507 16:28:10.101 | INFO | } | 20260507 16:28:10.101 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:28:10.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:10.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:28:10.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:28:10.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:10.102 | INFO | Uploaded 36GiB to GPU in 2.43463s (14.7866GiB/s) | 20260507 16:28:10.102 | INFO | Downloaded 34GiB to GPU in 2.46918s (13.7698GiB/s) | 20260507 16:28:15.241 | INFO | Analyzing chunk frames = [932864,933376) | 20260507 16:28:15.241 | INFO | MemInfo: { | 20260507 16:28:15.241 | INFO | "gpu" : " 17 GB", | 20260507 16:28:15.241 | INFO | "huge" : "", | 20260507 16:28:15.241 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:15.241 | INFO | "malloc" : " 14 GB", | 20260507 16:28:15.241 | INFO | "pinned" : " 29 GB" | 20260507 16:28:15.241 | INFO | } | 20260507 16:28:15.241 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:28:15.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:15.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:28:15.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 16:28:15.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:15.241 | INFO | Uploaded 36GiB to GPU in 2.47045s (14.5722GiB/s) | 20260507 16:28:15.241 | INFO | Downloaded 34GiB to GPU in 2.43082s (13.987GiB/s) | 20260507 16:28:20.380 | INFO | Analyzing chunk frames = [933376,933888) | 20260507 16:28:20.380 | INFO | MemInfo: { | 20260507 16:28:20.380 | INFO | "gpu" : " 17 GB", | 20260507 16:28:20.380 | INFO | "huge" : "", | 20260507 16:28:20.380 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:20.380 | INFO | "malloc" : " 14 GB", | 20260507 16:28:20.380 | INFO | "pinned" : " 29 GB" | 20260507 16:28:20.380 | INFO | } | 20260507 16:28:20.380 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:28:20.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:20.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:28:20.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:28:20.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:20.380 | INFO | Uploaded 36GiB to GPU in 2.5216s (14.2766GiB/s) | 20260507 16:28:20.380 | INFO | Downloaded 34GiB to GPU in 2.50988s (13.5464GiB/s) | 20260507 16:28:20.380 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:28:20.380 | INFO | Total mem usage: 16.4 GiB | 20260507 16:28:20.380 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:28:20.381 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.2%) | 20260507 16:28:20.381 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:28:20.381 | INFO | High Water Breakdown: | 20260507 16:28:20.381 | INFO | 59.1% : 8.63 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:28:20.381 | INFO | 40.8% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:28:20.381 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:28:20.381 | INFO | ------------------------------------------ | 20260507 16:28:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:28:20.381 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:28:20.381 | INFO | Total mem usage: 26.5 GiB | 20260507 16:28:20.381 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:28:20.381 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:28:20.381 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:28:20.381 | INFO | High Water Breakdown: | 20260507 16:28:20.381 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:28:20.381 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:28:20.381 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:28:20.381 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:28:20.381 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:28:20.381 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:28:20.381 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:28:20.381 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:28:20.381 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:28:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:28:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:28:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:28:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:28:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:28:20.381 | INFO | ------------------------------------------ | 20260507 16:28:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:28:20.381 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:28:20.381 | INFO | Total mem usage: 27.8 GiB | 20260507 16:28:20.381 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:28:20.381 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:28:20.381 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:28:20.381 | INFO | High Water Breakdown: | 20260507 16:28:20.381 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:28:20.381 | INFO | ------------------------------------------ | 20260507 16:28:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:28:20.381 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:28:20.381 | INFO | Total mem usage: 15.7 GiB | 20260507 16:28:20.381 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:28:20.381 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:28:20.381 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:28:20.381 | INFO | High Water Breakdown: | 20260507 16:28:20.381 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:28:20.381 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:28:20.381 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:28:20.381 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:28:20.381 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:28:20.381 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:28:20.381 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:28:20.381 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:28:20.381 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:28:20.381 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:28:20.381 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:28:20.381 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:28:20.381 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:28:20.381 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:28:20.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:28:20.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:28:20.381 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:28:20.381 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:28:20.381 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:28:20.381 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:28:20.381 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:28:20.381 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:28:20.381 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:28:20.381 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:28:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:28:20.381 | INFO | ------------------------------------------ | 20260507 16:28:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:28:20.381 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:28:20.381 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:28:20.381 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:28:20.381 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.74e+01, 1.45e+01 / 4.25e+00 | 20260507 16:28:20.381 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.07e-02 / 4.91e-03 | 20260507 16:28:20.381 | INFO | PerfReport: 11.97%, Baseline -- 4.02e+00 / 4.47e+00, 6.01e-01 / 2.48e-01 | 20260507 16:28:20.381 | INFO | PerfReport: 3.40%, Histogram -- 1.14e+00 / 1.48e+00, 2.11e-01 / 1.12e-01 | 20260507 16:28:20.381 | INFO | PerfReport: 16.04%, DME -- 4.30e+01 / 4.67e+01, 5.55e+00 / 3.98e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 55.61%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.80e+00 / 1.88e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.56e+00 / 2.91e+00, 4.38e-01 / 2.30e-01 | 20260507 16:28:20.382 | INFO | PerfReport: 3.84%, Metrics -- 1.29e+00 / 1.48e+00, 3.18e-01 / 1.92e-01 | 20260507 16:28:20.382 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.11e-01 / 4.36e-01, 6.70e-02 / 4.13e-02 | 20260507 16:28:20.382 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:28:20.382 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:28:20.382 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:28:20.382 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 23.62%, Upload -- 2.53e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 23.81%, Download -- 2.55e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 31.30%, Compute -- 3.36e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 1.76%, LockUpload -- 1.88e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 2.31%, LockDownload -- 2.47e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 17.19%, LockCompute -- 1.84e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:28:20.382 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 16:28:20.477 | INFO | Starting a round of Baz IO | 20260507 16:28:23.988 | INFO | Populating RTMetrics report | 20260507 16:28:23.988 | INFO | RTMetrics report populated | 20260507 16:28:23.988 | INFO | Opening tmp RTMetrics json file | 20260507 16:28:24.013 | INFO | Serializing RTMetrics to json | 20260507 16:28:24.015 | INFO | Serialization done, swapping tmp file now... | 20260507 16:28:24.015 | INFO | RTMetrics done | 20260507 16:28:24.093 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.616 seconds | 20260507 16:28:24.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.20% / 0.03% / 49.77% | 20260507 16:28:24.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.49693% / 3.97364e-05% / 1.98682e-05% | 20260507 16:28:24.093 | INFO | Baz IO overall: | 20260507 16:28:24.093 | INFO | BytesWritten : 400357665789 | 20260507 16:28:24.093 | INFO | NumEvents : 193689188159 | 20260507 16:28:25.519 | INFO | Analyzing chunk frames = [933888,934400) | 20260507 16:28:25.519 | INFO | MemInfo: { | 20260507 16:28:25.519 | INFO | "gpu" : " 17 GB", | 20260507 16:28:25.519 | INFO | "huge" : "", | 20260507 16:28:25.520 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:25.520 | INFO | "malloc" : " 14 GB", | 20260507 16:28:25.520 | INFO | "pinned" : " 29 GB" | 20260507 16:28:25.520 | INFO | } | 20260507 16:28:25.520 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:28:25.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:25.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:25.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:25.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:25.520 | INFO | ANALYSIS : 69.7%, 2.9, 107.5ms, 0 | 20260507 16:28:25.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:25.520 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 16:28:25.520 | INFO | PRE_HQ : 67.1%, 1.7, 60.2ms, 0 | 20260507 16:28:25.520 | INFO | BAZWRITER : 69.9%, 3.9, 140.1ms, 0 | 20260507 16:28:25.520 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 16:28:25.520 | INFO | Uploaded 36GiB to GPU in 2.56772s (14.0202GiB/s) | 20260507 16:28:25.520 | INFO | Downloaded 38GiB to GPU in 2.72125s (13.9642GiB/s) | 20260507 16:28:30.659 | INFO | Analyzing chunk frames = [934400,934912) | 20260507 16:28:30.659 | INFO | MemInfo: { | 20260507 16:28:30.659 | INFO | "gpu" : " 17 GB", | 20260507 16:28:30.659 | INFO | "huge" : "", | 20260507 16:28:30.659 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:30.659 | INFO | "malloc" : " 14 GB", | 20260507 16:28:30.659 | INFO | "pinned" : " 29 GB" | 20260507 16:28:30.659 | INFO | } | 20260507 16:28:30.659 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:28:30.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:30.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:28:30.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:28:30.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:30.659 | INFO | Uploaded 36GiB to GPU in 2.52892s (14.2353GiB/s) | 20260507 16:28:30.659 | INFO | Downloaded 34GiB to GPU in 2.4165s (14.0699GiB/s) | 20260507 16:28:35.798 | INFO | Analyzing chunk frames = [934912,935424) | 20260507 16:28:35.798 | INFO | MemInfo: { | 20260507 16:28:35.798 | INFO | "gpu" : " 17 GB", | 20260507 16:28:35.798 | INFO | "huge" : "", | 20260507 16:28:35.798 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:35.798 | INFO | "malloc" : " 14 GB", | 20260507 16:28:35.798 | INFO | "pinned" : " 29 GB" | 20260507 16:28:35.798 | INFO | } | 20260507 16:28:35.798 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:28:35.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:35.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.798 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 16:28:35.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.798 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:28:35.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:35.799 | INFO | Uploaded 36GiB to GPU in 2.58087s (13.9488GiB/s) | 20260507 16:28:35.799 | INFO | Downloaded 34GiB to GPU in 2.40014s (14.1659GiB/s) | 20260507 16:28:40.937 | INFO | Analyzing chunk frames = [935424,935936) | 20260507 16:28:40.937 | INFO | MemInfo: { | 20260507 16:28:40.937 | INFO | "gpu" : " 17 GB", | 20260507 16:28:40.937 | INFO | "huge" : "", | 20260507 16:28:40.937 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:40.937 | INFO | "malloc" : " 14 GB", | 20260507 16:28:40.937 | INFO | "pinned" : " 29 GB" | 20260507 16:28:40.937 | INFO | } | 20260507 16:28:40.938 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:28:40.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:40.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 16:28:40.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:28:40.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:40.938 | INFO | Uploaded 36GiB to GPU in 2.59919s (13.8505GiB/s) | 20260507 16:28:40.938 | INFO | Downloaded 34GiB to GPU in 2.42682s (14.0101GiB/s) | 20260507 16:28:46.077 | INFO | Analyzing chunk frames = [935936,936448) | 20260507 16:28:46.077 | INFO | MemInfo: { | 20260507 16:28:46.077 | INFO | "gpu" : " 17 GB", | 20260507 16:28:46.077 | INFO | "huge" : "", | 20260507 16:28:46.077 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:46.077 | INFO | "malloc" : " 14 GB", | 20260507 16:28:46.077 | INFO | "pinned" : " 29 GB" | 20260507 16:28:46.077 | INFO | } | 20260507 16:28:46.077 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:28:46.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:46.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 16:28:46.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 16:28:46.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:46.077 | INFO | Uploaded 36GiB to GPU in 2.41538s (14.9045GiB/s) | 20260507 16:28:46.077 | INFO | Downloaded 34GiB to GPU in 2.45934s (13.8249GiB/s) | 20260507 16:28:51.216 | INFO | Analyzing chunk frames = [936448,936960) | 20260507 16:28:51.216 | INFO | MemInfo: { | 20260507 16:28:51.216 | INFO | "gpu" : " 17 GB", | 20260507 16:28:51.216 | INFO | "huge" : "", | 20260507 16:28:51.216 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:51.216 | INFO | "malloc" : " 14 GB", | 20260507 16:28:51.216 | INFO | "pinned" : " 29 GB" | 20260507 16:28:51.216 | INFO | } | 20260507 16:28:51.216 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:28:51.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:51.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 16:28:51.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 16:28:51.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:51.217 | INFO | Uploaded 36GiB to GPU in 2.47327s (14.5556GiB/s) | 20260507 16:28:51.217 | INFO | Downloaded 34GiB to GPU in 2.44866s (13.8852GiB/s) | 20260507 16:28:56.356 | INFO | Analyzing chunk frames = [936960,937472) | 20260507 16:28:56.356 | INFO | MemInfo: { | 20260507 16:28:56.356 | INFO | "gpu" : " 17 GB", | 20260507 16:28:56.356 | INFO | "huge" : "", | 20260507 16:28:56.356 | INFO | "huge_shared" : " 29 GB", | 20260507 16:28:56.356 | INFO | "malloc" : " 14 GB", | 20260507 16:28:56.356 | INFO | "pinned" : " 29 GB" | 20260507 16:28:56.356 | INFO | } | 20260507 16:28:56.356 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:28:56.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:28:56.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.356 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:28:56.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.356 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:28:56.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:28:56.357 | INFO | Uploaded 36GiB to GPU in 2.51319s (14.3244GiB/s) | 20260507 16:28:56.357 | INFO | Downloaded 34GiB to GPU in 2.41155s (14.0988GiB/s) | 20260507 16:29:01.495 | INFO | Analyzing chunk frames = [937472,937984) | 20260507 16:29:01.495 | INFO | MemInfo: { | 20260507 16:29:01.495 | INFO | "gpu" : " 17 GB", | 20260507 16:29:01.495 | INFO | "huge" : "", | 20260507 16:29:01.495 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:01.495 | INFO | "malloc" : " 14 GB", | 20260507 16:29:01.495 | INFO | "pinned" : " 29 GB" | 20260507 16:29:01.495 | INFO | } | 20260507 16:29:01.495 | INFO | Perf report for the last 5.13902 seconds: | 20260507 16:29:01.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:01.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 16:29:01.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:29:01.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:01.495 | INFO | Uploaded 36GiB to GPU in 2.56114s (14.0563GiB/s) | 20260507 16:29:01.495 | INFO | Downloaded 34GiB to GPU in 2.50827s (13.5551GiB/s) | 20260507 16:29:01.592 | INFO | Starting a round of Baz IO | 20260507 16:29:05.128 | INFO | Populating RTMetrics report | 20260507 16:29:05.128 | INFO | RTMetrics report populated | 20260507 16:29:05.128 | INFO | Opening tmp RTMetrics json file | 20260507 16:29:05.153 | INFO | Serializing RTMetrics to json | 20260507 16:29:05.162 | INFO | Serialization done, swapping tmp file now... | 20260507 16:29:05.165 | INFO | RTMetrics done | 20260507 16:29:05.231 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.639 seconds | 20260507 16:29:05.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.17% / 0.03% / 49.80% | 20260507 16:29:05.231 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.47691% / 5.96046e-05% / 3.17891e-05% | 20260507 16:29:05.231 | INFO | Baz IO overall: | 20260507 16:29:05.231 | INFO | BytesWritten : 402274709763 | 20260507 16:29:05.231 | INFO | NumEvents : 194622426250 | 20260507 16:29:06.634 | INFO | Analyzing chunk frames = [937984,938496) | 20260507 16:29:06.634 | INFO | MemInfo: { | 20260507 16:29:06.634 | INFO | "gpu" : " 17 GB", | 20260507 16:29:06.634 | INFO | "huge" : "", | 20260507 16:29:06.634 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:06.634 | INFO | "malloc" : " 14 GB", | 20260507 16:29:06.634 | INFO | "pinned" : " 29 GB" | 20260507 16:29:06.634 | INFO | } | 20260507 16:29:06.635 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:29:06.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:06.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:06.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:06.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:06.635 | INFO | ANALYSIS : 69.9%, 2.9, 108.1ms, 0 | 20260507 16:29:06.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:06.635 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 16:29:06.635 | INFO | PRE_HQ : 66.9%, 1.7, 60.4ms, 0 | 20260507 16:29:06.635 | INFO | BAZWRITER : 70.4%, 3.9, 141.9ms, 0 | 20260507 16:29:06.635 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:29:06.635 | INFO | Uploaded 36GiB to GPU in 2.58654s (13.9182GiB/s) | 20260507 16:29:06.635 | INFO | Downloaded 38GiB to GPU in 2.72511s (13.9444GiB/s) | 20260507 16:29:11.774 | INFO | Analyzing chunk frames = [938496,939008) | 20260507 16:29:11.774 | INFO | MemInfo: { | 20260507 16:29:11.774 | INFO | "gpu" : " 17 GB", | 20260507 16:29:11.774 | INFO | "huge" : "", | 20260507 16:29:11.774 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:11.774 | INFO | "malloc" : " 14 GB", | 20260507 16:29:11.774 | INFO | "pinned" : " 29 GB" | 20260507 16:29:11.774 | INFO | } | 20260507 16:29:11.774 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:29:11.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:11.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:29:11.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:29:11.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:11.774 | INFO | Uploaded 36GiB to GPU in 2.51095s (14.3372GiB/s) | 20260507 16:29:11.774 | INFO | Downloaded 34GiB to GPU in 2.42148s (14.041GiB/s) | 20260507 16:29:16.913 | INFO | Analyzing chunk frames = [939008,939520) | 20260507 16:29:16.913 | INFO | MemInfo: { | 20260507 16:29:16.913 | INFO | "gpu" : " 17 GB", | 20260507 16:29:16.913 | INFO | "huge" : "", | 20260507 16:29:16.913 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:16.913 | INFO | "malloc" : " 14 GB", | 20260507 16:29:16.913 | INFO | "pinned" : " 29 GB" | 20260507 16:29:16.913 | INFO | } | 20260507 16:29:16.913 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:29:16.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:16.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.913 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 16:29:16.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.913 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:29:16.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:16.914 | INFO | Uploaded 36GiB to GPU in 2.57735s (13.9678GiB/s) | 20260507 16:29:16.914 | INFO | Downloaded 34GiB to GPU in 2.41455s (14.0813GiB/s) | 20260507 16:29:22.052 | INFO | Analyzing chunk frames = [939520,940032) | 20260507 16:29:22.052 | INFO | MemInfo: { | 20260507 16:29:22.052 | INFO | "gpu" : " 17 GB", | 20260507 16:29:22.052 | INFO | "huge" : "", | 20260507 16:29:22.052 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:22.052 | INFO | "malloc" : " 14 GB", | 20260507 16:29:22.052 | INFO | "pinned" : " 29 GB" | 20260507 16:29:22.052 | INFO | } | 20260507 16:29:22.052 | INFO | Perf report for the last 5.13871 seconds: | 20260507 16:29:22.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:22.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.052 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260507 16:29:22.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.052 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:29:22.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:22.053 | INFO | Uploaded 36GiB to GPU in 2.49645s (14.4204GiB/s) | 20260507 16:29:22.053 | INFO | Downloaded 34GiB to GPU in 2.35842s (14.4165GiB/s) | 20260507 16:29:27.191 | INFO | Analyzing chunk frames = [940032,940544) | 20260507 16:29:27.192 | INFO | MemInfo: { | 20260507 16:29:27.192 | INFO | "gpu" : " 17 GB", | 20260507 16:29:27.192 | INFO | "huge" : "", | 20260507 16:29:27.192 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:27.192 | INFO | "malloc" : " 14 GB", | 20260507 16:29:27.192 | INFO | "pinned" : " 29 GB" | 20260507 16:29:27.192 | INFO | } | 20260507 16:29:27.192 | INFO | Perf report for the last 5.13971 seconds: | 20260507 16:29:27.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:27.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 16:29:27.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 16:29:27.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:27.192 | INFO | Uploaded 36GiB to GPU in 2.39662s (15.0211GiB/s) | 20260507 16:29:27.192 | INFO | Downloaded 34GiB to GPU in 2.46961s (13.7674GiB/s) | 20260507 16:29:32.331 | INFO | Analyzing chunk frames = [940544,941056) | 20260507 16:29:32.331 | INFO | MemInfo: { | 20260507 16:29:32.331 | INFO | "gpu" : " 17 GB", | 20260507 16:29:32.331 | INFO | "huge" : "", | 20260507 16:29:32.331 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:32.331 | INFO | "malloc" : " 14 GB", | 20260507 16:29:32.331 | INFO | "pinned" : " 29 GB" | 20260507 16:29:32.331 | INFO | } | 20260507 16:29:32.331 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:29:32.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:32.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:29:32.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:29:32.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:32.331 | INFO | Uploaded 36GiB to GPU in 2.40574s (14.9642GiB/s) | 20260507 16:29:32.331 | INFO | Downloaded 34GiB to GPU in 2.43139s (13.9838GiB/s) | 20260507 16:29:37.470 | INFO | Analyzing chunk frames = [941056,941568) | 20260507 16:29:37.470 | INFO | MemInfo: { | 20260507 16:29:37.470 | INFO | "gpu" : " 17 GB", | 20260507 16:29:37.470 | INFO | "huge" : "", | 20260507 16:29:37.470 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:37.470 | INFO | "malloc" : " 14 GB", | 20260507 16:29:37.470 | INFO | "pinned" : " 29 GB" | 20260507 16:29:37.470 | INFO | } | 20260507 16:29:37.470 | INFO | Perf report for the last 5.13914 seconds: | 20260507 16:29:37.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:37.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.470 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:29:37.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.470 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:29:37.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:37.471 | INFO | Uploaded 36GiB to GPU in 2.50004s (14.3998GiB/s) | 20260507 16:29:37.471 | INFO | Downloaded 34GiB to GPU in 2.4076s (14.1219GiB/s) | 20260507 16:29:42.610 | INFO | Analyzing chunk frames = [941568,942080) | 20260507 16:29:42.610 | INFO | MemInfo: { | 20260507 16:29:42.610 | INFO | "gpu" : " 17 GB", | 20260507 16:29:42.610 | INFO | "huge" : "", | 20260507 16:29:42.610 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:42.610 | INFO | "malloc" : " 14 GB", | 20260507 16:29:42.610 | INFO | "pinned" : " 29 GB" | 20260507 16:29:42.610 | INFO | } | 20260507 16:29:42.610 | INFO | Perf report for the last 5.13965 seconds: | 20260507 16:29:42.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:42.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 16:29:42.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:29:42.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:42.610 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:29:42.610 | INFO | Total mem usage: 16.4 GiB | 20260507 16:29:42.610 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:29:42.610 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.47%) | 20260507 16:29:42.610 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:29:42.610 | INFO | High Water Breakdown: | 20260507 16:29:42.610 | INFO | 59.8% : 9.13 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:29:42.610 | INFO | 40% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:29:42.610 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:29:42.610 | INFO | ------------------------------------------ | 20260507 16:29:42.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:29:42.610 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:29:42.610 | INFO | Total mem usage: 26.5 GiB | 20260507 16:29:42.610 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:29:42.611 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:29:42.611 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:29:42.611 | INFO | High Water Breakdown: | 20260507 16:29:42.611 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:29:42.611 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:29:42.611 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:29:42.611 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:29:42.611 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:29:42.611 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:29:42.611 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:29:42.611 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:29:42.611 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:29:42.611 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:29:42.611 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:29:42.611 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:29:42.611 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:29:42.611 | INFO | Uploaded 36GiB to GPU in 2.5584s (14.0713GiB/s) | 20260507 16:29:42.611 | INFO | Downloaded 34GiB to GPU in 2.50765s (13.5585GiB/s) | 20260507 16:29:42.611 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:29:42.611 | INFO | ------------------------------------------ | 20260507 16:29:42.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:29:42.611 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:29:42.611 | INFO | Total mem usage: 27.8 GiB | 20260507 16:29:42.611 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:29:42.611 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:29:42.611 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:29:42.611 | INFO | High Water Breakdown: | 20260507 16:29:42.611 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:29:42.611 | INFO | ------------------------------------------ | 20260507 16:29:42.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:29:42.611 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:29:42.611 | INFO | Total mem usage: 15.7 GiB | 20260507 16:29:42.611 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:29:42.611 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:29:42.611 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:29:42.611 | INFO | High Water Breakdown: | 20260507 16:29:42.611 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:29:42.611 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:29:42.611 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:29:42.611 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:29:42.611 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:29:42.611 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:29:42.611 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:29:42.611 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:29:42.611 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:29:42.611 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:29:42.611 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:29:42.611 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:29:42.611 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:29:42.611 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:29:42.611 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:29:42.611 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:29:42.611 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:29:42.611 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:29:42.611 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:29:42.611 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:29:42.611 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:29:42.611 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:29:42.611 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:29:42.611 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:29:42.611 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:29:42.611 | INFO | ------------------------------------------ | 20260507 16:29:42.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:29:42.611 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:29:42.611 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:29:42.611 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:29:42.611 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.67e+01, 1.46e+01 / 2.90e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 0.29%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.10e-02 / 7.46e-03 | 20260507 16:29:42.611 | INFO | PerfReport: 11.94%, Baseline -- 4.01e+00 / 4.44e+00, 6.06e-01 / 2.84e-01 | 20260507 16:29:42.611 | INFO | PerfReport: 3.40%, Histogram -- 1.14e+00 / 1.50e+00, 2.15e-01 / 1.05e-01 | 20260507 16:29:42.611 | INFO | PerfReport: 15.73%, DME -- 4.22e+01 / 4.49e+01, 4.36e+00 / 2.89e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.93e+00 / 2.09e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.90e+00, 4.38e-01 / 2.51e-01 | 20260507 16:29:42.611 | INFO | PerfReport: 3.83%, Metrics -- 1.29e+00 / 1.44e+00, 3.29e-01 / 2.08e-01 | 20260507 16:29:42.611 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.10e-01 / 4.30e-01, 6.86e-02 / 4.87e-02 | 20260507 16:29:42.611 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:29:42.611 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:29:42.611 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:29:42.611 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 24.31%, Upload -- 2.62e+01 / 0.00e+00, 8.66e+00 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 23.87%, Download -- 2.57e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 31.10%, Compute -- 3.35e+01 / 0.00e+00, 1.46e+01 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 2.20%, LockUpload -- 2.37e+00 / 0.00e+00, 5.24e+00 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 1.99%, LockDownload -- 2.15e+00 / 0.00e+00, 3.52e+00 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 16.51%, LockCompute -- 1.78e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:29:42.611 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.04e-02 / 0.00e+00 | 20260507 16:29:42.707 | INFO | Starting a round of Baz IO | 20260507 16:29:46.234 | INFO | Populating RTMetrics report | 20260507 16:29:46.234 | INFO | RTMetrics report populated | 20260507 16:29:46.234 | INFO | Opening tmp RTMetrics json file | 20260507 16:29:46.300 | INFO | Serializing RTMetrics to json | 20260507 16:29:46.330 | INFO | Serialization done, swapping tmp file now... | 20260507 16:29:46.334 | INFO | RTMetrics done | 20260507 16:29:46.360 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.652 seconds | 20260507 16:29:46.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.14% / 0.03% / 49.83% | 20260507 16:29:46.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.45443% / 6.75519e-05% / 3.97364e-05% | 20260507 16:29:46.360 | INFO | Baz IO overall: | 20260507 16:29:46.360 | INFO | BytesWritten : 404190703953 | 20260507 16:29:46.360 | INFO | NumEvents : 195554578203 | 20260507 16:29:47.749 | INFO | Analyzing chunk frames = [942080,942592) | 20260507 16:29:47.749 | INFO | MemInfo: { | 20260507 16:29:47.749 | INFO | "gpu" : " 17 GB", | 20260507 16:29:47.749 | INFO | "huge" : "", | 20260507 16:29:47.749 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:47.749 | INFO | "malloc" : " 14 GB", | 20260507 16:29:47.749 | INFO | "pinned" : " 29 GB" | 20260507 16:29:47.749 | INFO | } | 20260507 16:29:47.749 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:29:47.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:47.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:47.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:47.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:47.749 | INFO | ANALYSIS : 70.0%, 3.0, 108.4ms, 0 | 20260507 16:29:47.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:47.749 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 16:29:47.749 | INFO | PRE_HQ : 66.9%, 1.7, 59.9ms, 0 | 20260507 16:29:47.749 | INFO | BAZWRITER : 70.5%, 3.7, 135.7ms, 0 | 20260507 16:29:47.749 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:29:47.750 | INFO | Uploaded 36GiB to GPU in 2.59361s (13.8803GiB/s) | 20260507 16:29:47.750 | INFO | Downloaded 38GiB to GPU in 2.71118s (14.016GiB/s) | 20260507 16:29:52.888 | INFO | Analyzing chunk frames = [942592,943104) | 20260507 16:29:52.888 | INFO | MemInfo: { | 20260507 16:29:52.888 | INFO | "gpu" : " 17 GB", | 20260507 16:29:52.888 | INFO | "huge" : "", | 20260507 16:29:52.888 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:52.888 | INFO | "malloc" : " 14 GB", | 20260507 16:29:52.888 | INFO | "pinned" : " 29 GB" | 20260507 16:29:52.888 | INFO | } | 20260507 16:29:52.888 | INFO | Perf report for the last 5.13916 seconds: | 20260507 16:29:52.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:52.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.888 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 16:29:52.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.888 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:29:52.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:52.889 | INFO | Uploaded 36GiB to GPU in 2.46753s (14.5895GiB/s) | 20260507 16:29:52.889 | INFO | Downloaded 34GiB to GPU in 2.42978s (13.993GiB/s) | 20260507 16:29:58.027 | INFO | Analyzing chunk frames = [943104,943616) | 20260507 16:29:58.027 | INFO | MemInfo: { | 20260507 16:29:58.027 | INFO | "gpu" : " 17 GB", | 20260507 16:29:58.027 | INFO | "huge" : "", | 20260507 16:29:58.027 | INFO | "huge_shared" : " 29 GB", | 20260507 16:29:58.028 | INFO | "malloc" : " 14 GB", | 20260507 16:29:58.028 | INFO | "pinned" : " 29 GB" | 20260507 16:29:58.028 | INFO | } | 20260507 16:29:58.028 | INFO | Perf report for the last 5.13916 seconds: | 20260507 16:29:58.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:29:58.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 16:29:58.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:29:58.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:29:58.028 | INFO | Uploaded 36GiB to GPU in 2.55523s (14.0888GiB/s) | 20260507 16:29:58.028 | INFO | Downloaded 34GiB to GPU in 2.42707s (14.0087GiB/s) | 20260507 16:30:03.167 | INFO | Analyzing chunk frames = [943616,944128) | 20260507 16:30:03.167 | INFO | MemInfo: { | 20260507 16:30:03.167 | INFO | "gpu" : " 17 GB", | 20260507 16:30:03.167 | INFO | "huge" : "", | 20260507 16:30:03.167 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:03.167 | INFO | "malloc" : " 14 GB", | 20260507 16:30:03.167 | INFO | "pinned" : " 29 GB" | 20260507 16:30:03.167 | INFO | } | 20260507 16:30:03.167 | INFO | Perf report for the last 5.13968 seconds: | 20260507 16:30:03.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:03.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 16:30:03.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:30:03.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:03.167 | INFO | Uploaded 36GiB to GPU in 2.46508s (14.604GiB/s) | 20260507 16:30:03.168 | INFO | Downloaded 34GiB to GPU in 2.36342s (14.3859GiB/s) | 20260507 16:30:08.307 | INFO | Analyzing chunk frames = [944128,944640) | 20260507 16:30:08.307 | INFO | MemInfo: { | 20260507 16:30:08.307 | INFO | "gpu" : " 17 GB", | 20260507 16:30:08.307 | INFO | "huge" : "", | 20260507 16:30:08.307 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:08.307 | INFO | "malloc" : " 14 GB", | 20260507 16:30:08.307 | INFO | "pinned" : " 29 GB" | 20260507 16:30:08.307 | INFO | } | 20260507 16:30:08.307 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:30:08.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:08.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:30:08.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:30:08.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:08.307 | INFO | Uploaded 36GiB to GPU in 2.37444s (15.1615GiB/s) | 20260507 16:30:08.307 | INFO | Downloaded 34GiB to GPU in 2.47287s (13.7492GiB/s) | 20260507 16:30:13.446 | INFO | Analyzing chunk frames = [944640,945152) | 20260507 16:30:13.446 | INFO | MemInfo: { | 20260507 16:30:13.446 | INFO | "gpu" : " 17 GB", | 20260507 16:30:13.446 | INFO | "huge" : "", | 20260507 16:30:13.446 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:13.446 | INFO | "malloc" : " 14 GB", | 20260507 16:30:13.446 | INFO | "pinned" : " 29 GB" | 20260507 16:30:13.446 | INFO | } | 20260507 16:30:13.446 | INFO | Perf report for the last 5.13897 seconds: | 20260507 16:30:13.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:13.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:30:13.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:30:13.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:13.446 | INFO | Uploaded 36GiB to GPU in 2.4046s (14.9713GiB/s) | 20260507 16:30:13.446 | INFO | Downloaded 34GiB to GPU in 2.45574s (13.8451GiB/s) | 20260507 16:30:18.585 | INFO | Analyzing chunk frames = [945152,945664) | 20260507 16:30:18.585 | INFO | MemInfo: { | 20260507 16:30:18.585 | INFO | "gpu" : " 17 GB", | 20260507 16:30:18.585 | INFO | "huge" : "", | 20260507 16:30:18.585 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:18.585 | INFO | "malloc" : " 14 GB", | 20260507 16:30:18.585 | INFO | "pinned" : " 29 GB" | 20260507 16:30:18.585 | INFO | } | 20260507 16:30:18.585 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:30:18.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:18.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 16:30:18.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:30:18.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:18.585 | INFO | Uploaded 36GiB to GPU in 2.43574s (14.7799GiB/s) | 20260507 16:30:18.585 | INFO | Downloaded 34GiB to GPU in 2.42165s (14.04GiB/s) | 20260507 16:30:23.724 | INFO | Analyzing chunk frames = [945664,946176) | 20260507 16:30:23.724 | INFO | MemInfo: { | 20260507 16:30:23.724 | INFO | "gpu" : " 17 GB", | 20260507 16:30:23.724 | INFO | "huge" : "", | 20260507 16:30:23.724 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:23.724 | INFO | "malloc" : " 14 GB", | 20260507 16:30:23.724 | INFO | "pinned" : " 29 GB" | 20260507 16:30:23.724 | INFO | } | 20260507 16:30:23.724 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:30:23.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:23.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.724 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 16:30:23.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.724 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:30:23.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:23.725 | INFO | Uploaded 36GiB to GPU in 2.51441s (14.3175GiB/s) | 20260507 16:30:23.725 | INFO | Downloaded 34GiB to GPU in 2.52378s (13.4718GiB/s) | 20260507 16:30:23.821 | INFO | Starting a round of Baz IO | 20260507 16:30:27.340 | INFO | Populating RTMetrics report | 20260507 16:30:27.340 | INFO | RTMetrics report populated | 20260507 16:30:27.340 | INFO | Opening tmp RTMetrics json file | 20260507 16:30:27.396 | INFO | Serializing RTMetrics to json | 20260507 16:30:27.399 | INFO | Serialization done, swapping tmp file now... | 20260507 16:30:27.402 | INFO | RTMetrics done | 20260507 16:30:27.518 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.696 seconds | 20260507 16:30:27.518 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.12% / 0.03% / 49.86% | 20260507 16:30:27.518 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.44729% / 5.16574e-05% / 5.5631e-05% | 20260507 16:30:27.518 | INFO | Baz IO overall: | 20260507 16:30:27.518 | INFO | BytesWritten : 406107402799 | 20260507 16:30:27.518 | INFO | NumEvents : 196487045727 | 20260507 16:30:28.863 | INFO | Analyzing chunk frames = [946176,946688) | 20260507 16:30:28.863 | INFO | MemInfo: { | 20260507 16:30:28.863 | INFO | "gpu" : " 17 GB", | 20260507 16:30:28.864 | INFO | "huge" : "", | 20260507 16:30:28.864 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:28.864 | INFO | "malloc" : " 14 GB", | 20260507 16:30:28.864 | INFO | "pinned" : " 29 GB" | 20260507 16:30:28.864 | INFO | } | 20260507 16:30:28.864 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:30:28.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:28.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:28.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:28.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:28.864 | INFO | ANALYSIS : 69.9%, 2.9, 108.0ms, 0 | 20260507 16:30:28.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:28.864 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 16:30:28.864 | INFO | PRE_HQ : 66.6%, 1.7, 59.7ms, 0 | 20260507 16:30:28.864 | INFO | BAZWRITER : 71.5%, 4.4, 162.4ms, 0 | 20260507 16:30:28.864 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:30:28.864 | INFO | Uploaded 36GiB to GPU in 2.55091s (14.1126GiB/s) | 20260507 16:30:28.864 | INFO | Downloaded 38GiB to GPU in 2.72359s (13.9521GiB/s) | 20260507 16:30:34.003 | INFO | Analyzing chunk frames = [946688,947200) | 20260507 16:30:34.003 | INFO | MemInfo: { | 20260507 16:30:34.003 | INFO | "gpu" : " 17 GB", | 20260507 16:30:34.003 | INFO | "huge" : "", | 20260507 16:30:34.003 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:34.003 | INFO | "malloc" : " 14 GB", | 20260507 16:30:34.003 | INFO | "pinned" : " 29 GB" | 20260507 16:30:34.003 | INFO | } | 20260507 16:30:34.003 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:30:34.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:34.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 16:30:34.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:30:34.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:34.003 | INFO | Uploaded 36GiB to GPU in 2.52108s (14.2796GiB/s) | 20260507 16:30:34.003 | INFO | Downloaded 34GiB to GPU in 2.41881s (14.0565GiB/s) | 20260507 16:30:39.142 | INFO | Analyzing chunk frames = [947200,947712) | 20260507 16:30:39.142 | INFO | MemInfo: { | 20260507 16:30:39.142 | INFO | "gpu" : " 17 GB", | 20260507 16:30:39.142 | INFO | "huge" : "", | 20260507 16:30:39.142 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:39.142 | INFO | "malloc" : " 14 GB", | 20260507 16:30:39.142 | INFO | "pinned" : " 29 GB" | 20260507 16:30:39.143 | INFO | } | 20260507 16:30:39.143 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:30:39.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:39.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260507 16:30:39.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:30:39.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:39.143 | INFO | Uploaded 36GiB to GPU in 2.58091s (13.9486GiB/s) | 20260507 16:30:39.143 | INFO | Downloaded 34GiB to GPU in 2.40188s (14.1556GiB/s) | 20260507 16:30:44.282 | INFO | Analyzing chunk frames = [947712,948224) | 20260507 16:30:44.282 | INFO | MemInfo: { | 20260507 16:30:44.282 | INFO | "gpu" : " 17 GB", | 20260507 16:30:44.282 | INFO | "huge" : "", | 20260507 16:30:44.282 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:44.282 | INFO | "malloc" : " 14 GB", | 20260507 16:30:44.282 | INFO | "pinned" : " 29 GB" | 20260507 16:30:44.282 | INFO | } | 20260507 16:30:44.282 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:30:44.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:44.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.282 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:30:44.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.282 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:30:44.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:44.283 | INFO | Uploaded 36GiB to GPU in 2.46835s (14.5846GiB/s) | 20260507 16:30:44.283 | INFO | Downloaded 34GiB to GPU in 2.35132s (14.4599GiB/s) | 20260507 16:30:49.421 | INFO | Analyzing chunk frames = [948224,948736) | 20260507 16:30:49.421 | INFO | MemInfo: { | 20260507 16:30:49.421 | INFO | "gpu" : " 17 GB", | 20260507 16:30:49.421 | INFO | "huge" : "", | 20260507 16:30:49.421 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:49.421 | INFO | "malloc" : " 14 GB", | 20260507 16:30:49.421 | INFO | "pinned" : " 29 GB" | 20260507 16:30:49.421 | INFO | } | 20260507 16:30:49.421 | INFO | Perf report for the last 5.13893 seconds: | 20260507 16:30:49.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:49.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 16:30:49.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 16:30:49.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:49.421 | INFO | Uploaded 36GiB to GPU in 2.40852s (14.9469GiB/s) | 20260507 16:30:49.421 | INFO | Downloaded 34GiB to GPU in 2.46267s (13.8061GiB/s) | 20260507 16:30:54.560 | INFO | Analyzing chunk frames = [948736,949248) | 20260507 16:30:54.560 | INFO | MemInfo: { | 20260507 16:30:54.560 | INFO | "gpu" : " 17 GB", | 20260507 16:30:54.560 | INFO | "huge" : "", | 20260507 16:30:54.560 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:54.560 | INFO | "malloc" : " 14 GB", | 20260507 16:30:54.560 | INFO | "pinned" : " 29 GB" | 20260507 16:30:54.560 | INFO | } | 20260507 16:30:54.560 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:30:54.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:54.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.560 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:30:54.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.561 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:30:54.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:54.561 | INFO | Uploaded 36GiB to GPU in 2.39537s (15.029GiB/s) | 20260507 16:30:54.561 | INFO | Downloaded 34GiB to GPU in 2.41754s (14.0639GiB/s) | 20260507 16:30:59.700 | INFO | Analyzing chunk frames = [949248,949760) | 20260507 16:30:59.700 | INFO | MemInfo: { | 20260507 16:30:59.700 | INFO | "gpu" : " 17 GB", | 20260507 16:30:59.700 | INFO | "huge" : "", | 20260507 16:30:59.700 | INFO | "huge_shared" : " 29 GB", | 20260507 16:30:59.700 | INFO | "malloc" : " 14 GB", | 20260507 16:30:59.700 | INFO | "pinned" : " 29 GB" | 20260507 16:30:59.700 | INFO | } | 20260507 16:30:59.700 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:30:59.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:30:59.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:30:59.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:30:59.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:30:59.700 | INFO | Uploaded 36GiB to GPU in 2.49528s (14.4273GiB/s) | 20260507 16:30:59.700 | INFO | Downloaded 34GiB to GPU in 2.40727s (14.1239GiB/s) | 20260507 16:31:04.839 | INFO | Analyzing chunk frames = [949760,950272) | 20260507 16:31:04.839 | INFO | MemInfo: { | 20260507 16:31:04.839 | INFO | "gpu" : " 17 GB", | 20260507 16:31:04.839 | INFO | "huge" : "", | 20260507 16:31:04.839 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:04.839 | INFO | "malloc" : " 14 GB", | 20260507 16:31:04.839 | INFO | "pinned" : " 29 GB" | 20260507 16:31:04.839 | INFO | } | 20260507 16:31:04.839 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:31:04.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:04.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.839 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 16:31:04.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.839 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:31:04.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:04.840 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:31:04.840 | INFO | Total mem usage: 16.4 GiB | 20260507 16:31:04.840 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:31:04.840 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.07%) | 20260507 16:31:04.840 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:31:04.840 | INFO | High Water Breakdown: | 20260507 16:31:04.840 | INFO | 59.6% : 8.96 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:31:04.840 | INFO | 40.3% : 6.06 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:31:04.840 | INFO | Uploaded 36GiB to GPU in 2.55694s (14.0793GiB/s) | 20260507 16:31:04.840 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:31:04.840 | INFO | ------------------------------------------ | 20260507 16:31:04.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:31:04.840 | INFO | Downloaded 34GiB to GPU in 2.50723s (13.5608GiB/s) | 20260507 16:31:04.840 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:31:04.840 | INFO | Total mem usage: 26.5 GiB | 20260507 16:31:04.840 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:31:04.840 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:31:04.840 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:31:04.840 | INFO | High Water Breakdown: | 20260507 16:31:04.840 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:31:04.840 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:31:04.840 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:31:04.840 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:31:04.840 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:31:04.840 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:31:04.840 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:31:04.840 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:31:04.840 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:31:04.840 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:31:04.840 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:31:04.840 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:31:04.840 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:31:04.840 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:31:04.840 | INFO | ------------------------------------------ | 20260507 16:31:04.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:31:04.840 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:31:04.840 | INFO | Total mem usage: 27.8 GiB | 20260507 16:31:04.840 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:31:04.840 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:31:04.840 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:31:04.840 | INFO | High Water Breakdown: | 20260507 16:31:04.840 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:31:04.840 | INFO | ------------------------------------------ | 20260507 16:31:04.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:31:04.840 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:31:04.840 | INFO | Total mem usage: 15.7 GiB | 20260507 16:31:04.840 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:31:04.840 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:31:04.840 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:31:04.840 | INFO | High Water Breakdown: | 20260507 16:31:04.840 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:31:04.840 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:31:04.840 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:31:04.840 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:31:04.840 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:31:04.840 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:31:04.840 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:31:04.840 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:31:04.840 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:31:04.840 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:31:04.840 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:31:04.840 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:31:04.840 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:31:04.840 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:31:04.840 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:31:04.840 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:31:04.840 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:31:04.840 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:31:04.840 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:31:04.841 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:31:04.841 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:31:04.841 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:31:04.841 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:31:04.841 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:31:04.841 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:31:04.841 | INFO | ------------------------------------------ | 20260507 16:31:04.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:31:04.841 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:31:04.841 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:31:04.841 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:31:04.841 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.67e+01, 1.43e+01 / 2.88e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 0.29%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.13e-02 / 4.54e-03 | 20260507 16:31:04.841 | INFO | PerfReport: 11.98%, Baseline -- 4.03e+00 / 4.47e+00, 6.10e-01 / 2.49e-01 | 20260507 16:31:04.841 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.49e+00, 2.13e-01 / 9.77e-02 | 20260507 16:31:04.841 | INFO | PerfReport: 15.73%, DME -- 4.23e+01 / 4.52e+01, 4.42e+00 / 2.86e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.91e+00 / 2.07e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.52e+00 / 2.96e+00, 4.38e-01 / 2.24e-01 | 20260507 16:31:04.841 | INFO | PerfReport: 3.82%, Metrics -- 1.28e+00 / 1.47e+00, 3.26e-01 / 1.92e-01 | 20260507 16:31:04.841 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.11e-01 / 4.29e-01, 6.85e-02 / 4.48e-02 | 20260507 16:31:04.841 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:31:04.841 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:31:04.841 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:31:04.841 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 24.05%, Upload -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 23.87%, Download -- 2.57e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 31.19%, Compute -- 3.36e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 1.97%, LockUpload -- 2.12e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 2.14%, LockDownload -- 2.31e+00 / 0.00e+00, 3.69e+00 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 16.76%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:31:04.841 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.03e-02 / 0.00e+00 | 20260507 16:31:04.936 | INFO | Starting a round of Baz IO | 20260507 16:31:08.469 | INFO | Populating RTMetrics report | 20260507 16:31:08.469 | INFO | RTMetrics report populated | 20260507 16:31:08.469 | INFO | Opening tmp RTMetrics json file | 20260507 16:31:08.505 | INFO | Serializing RTMetrics to json | 20260507 16:31:08.565 | INFO | Serialization done, swapping tmp file now... | 20260507 16:31:08.570 | INFO | RTMetrics done | 20260507 16:31:08.605 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.668 seconds | 20260507 16:31:08.605 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.09% / 0.03% / 49.88% | 20260507 16:31:08.605 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.43758% / 5.5631e-05% / 3.17891e-05% | 20260507 16:31:08.605 | INFO | Baz IO overall: | 20260507 16:31:08.605 | INFO | BytesWritten : 408025362354 | 20260507 16:31:08.605 | INFO | NumEvents : 197420274911 | 20260507 16:31:09.978 | INFO | Analyzing chunk frames = [950272,950784) | 20260507 16:31:09.978 | INFO | MemInfo: { | 20260507 16:31:09.978 | INFO | "gpu" : " 17 GB", | 20260507 16:31:09.978 | INFO | "huge" : "", | 20260507 16:31:09.978 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:09.978 | INFO | "malloc" : " 14 GB", | 20260507 16:31:09.978 | INFO | "pinned" : " 29 GB" | 20260507 16:31:09.978 | INFO | } | 20260507 16:31:09.978 | INFO | Perf report for the last 5.13897 seconds: | 20260507 16:31:09.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:09.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:09.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:09.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:09.978 | INFO | ANALYSIS : 70.0%, 2.9, 108.1ms, 0 | 20260507 16:31:09.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:09.978 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 16:31:09.978 | INFO | PRE_HQ : 66.9%, 1.7, 59.7ms, 0 | 20260507 16:31:09.978 | INFO | BAZWRITER : 70.9%, 3.7, 133.0ms, 0 | 20260507 16:31:09.978 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 16:31:09.979 | INFO | Uploaded 36GiB to GPU in 2.58804s (13.9101GiB/s) | 20260507 16:31:09.979 | INFO | Downloaded 38GiB to GPU in 2.7292s (13.9235GiB/s) | 20260507 16:31:15.118 | INFO | Analyzing chunk frames = [950784,951296) | 20260507 16:31:15.118 | INFO | MemInfo: { | 20260507 16:31:15.118 | INFO | "gpu" : " 17 GB", | 20260507 16:31:15.118 | INFO | "huge" : "", | 20260507 16:31:15.118 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:15.118 | INFO | "malloc" : " 14 GB", | 20260507 16:31:15.118 | INFO | "pinned" : " 29 GB" | 20260507 16:31:15.118 | INFO | } | 20260507 16:31:15.118 | INFO | Perf report for the last 5.13989 seconds: | 20260507 16:31:15.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:15.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.118 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:31:15.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.118 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:31:15.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:15.119 | INFO | Uploaded 36GiB to GPU in 2.50603s (14.3654GiB/s) | 20260507 16:31:15.119 | INFO | Downloaded 34GiB to GPU in 2.40421s (14.1419GiB/s) | 20260507 16:31:20.257 | INFO | Analyzing chunk frames = [951296,951808) | 20260507 16:31:20.257 | INFO | MemInfo: { | 20260507 16:31:20.257 | INFO | "gpu" : " 17 GB", | 20260507 16:31:20.257 | INFO | "huge" : "", | 20260507 16:31:20.257 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:20.257 | INFO | "malloc" : " 14 GB", | 20260507 16:31:20.257 | INFO | "pinned" : " 29 GB" | 20260507 16:31:20.257 | INFO | } | 20260507 16:31:20.257 | INFO | Perf report for the last 5.13891 seconds: | 20260507 16:31:20.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:20.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260507 16:31:20.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:31:20.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:20.257 | INFO | Uploaded 36GiB to GPU in 2.50874s (14.3498GiB/s) | 20260507 16:31:20.257 | INFO | Downloaded 34GiB to GPU in 2.33745s (14.5458GiB/s) | 20260507 16:31:25.396 | INFO | Analyzing chunk frames = [951808,952320) | 20260507 16:31:25.396 | INFO | MemInfo: { | 20260507 16:31:25.396 | INFO | "gpu" : " 17 GB", | 20260507 16:31:25.396 | INFO | "huge" : "", | 20260507 16:31:25.396 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:25.397 | INFO | "malloc" : " 14 GB", | 20260507 16:31:25.397 | INFO | "pinned" : " 29 GB" | 20260507 16:31:25.397 | INFO | } | 20260507 16:31:25.397 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:31:25.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:25.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260507 16:31:25.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:31:25.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:25.397 | INFO | Uploaded 36GiB to GPU in 2.476s (14.5396GiB/s) | 20260507 16:31:25.397 | INFO | Downloaded 34GiB to GPU in 2.34761s (14.4828GiB/s) | 20260507 16:31:30.536 | INFO | Analyzing chunk frames = [952320,952832) | 20260507 16:31:30.536 | INFO | MemInfo: { | 20260507 16:31:30.536 | INFO | "gpu" : " 17 GB", | 20260507 16:31:30.536 | INFO | "huge" : "", | 20260507 16:31:30.536 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:30.536 | INFO | "malloc" : " 14 GB", | 20260507 16:31:30.536 | INFO | "pinned" : " 29 GB" | 20260507 16:31:30.536 | INFO | } | 20260507 16:31:30.536 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:31:30.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:30.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 16:31:30.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:31:30.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:30.536 | INFO | Uploaded 36GiB to GPU in 2.42607s (14.8388GiB/s) | 20260507 16:31:30.536 | INFO | Downloaded 34GiB to GPU in 2.468s (13.7763GiB/s) | 20260507 16:31:35.675 | INFO | Analyzing chunk frames = [952832,953344) | 20260507 16:31:35.675 | INFO | MemInfo: { | 20260507 16:31:35.675 | INFO | "gpu" : " 17 GB", | 20260507 16:31:35.675 | INFO | "huge" : "", | 20260507 16:31:35.675 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:35.675 | INFO | "malloc" : " 14 GB", | 20260507 16:31:35.675 | INFO | "pinned" : " 29 GB" | 20260507 16:31:35.675 | INFO | } | 20260507 16:31:35.675 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:31:35.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:35.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.675 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:31:35.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.676 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:31:35.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:35.676 | INFO | Uploaded 36GiB to GPU in 2.39878s (15.0076GiB/s) | 20260507 16:31:35.676 | INFO | Downloaded 34GiB to GPU in 2.43233s (13.9784GiB/s) | 20260507 16:31:40.815 | INFO | Analyzing chunk frames = [953344,953856) | 20260507 16:31:40.815 | INFO | MemInfo: { | 20260507 16:31:40.815 | INFO | "gpu" : " 17 GB", | 20260507 16:31:40.815 | INFO | "huge" : "", | 20260507 16:31:40.815 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:40.815 | INFO | "malloc" : " 14 GB", | 20260507 16:31:40.815 | INFO | "pinned" : " 29 GB" | 20260507 16:31:40.815 | INFO | } | 20260507 16:31:40.815 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:31:40.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:40.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:31:40.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:31:40.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:40.815 | INFO | Uploaded 36GiB to GPU in 2.49553s (14.4258GiB/s) | 20260507 16:31:40.815 | INFO | Downloaded 34GiB to GPU in 2.4054s (14.1349GiB/s) | 20260507 16:31:45.954 | INFO | Analyzing chunk frames = [953856,954368) | 20260507 16:31:45.954 | INFO | MemInfo: { | 20260507 16:31:45.954 | INFO | "gpu" : " 17 GB", | 20260507 16:31:45.954 | INFO | "huge" : "", | 20260507 16:31:45.954 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:45.954 | INFO | "malloc" : " 14 GB", | 20260507 16:31:45.954 | INFO | "pinned" : " 29 GB" | 20260507 16:31:45.954 | INFO | } | 20260507 16:31:45.954 | INFO | Perf report for the last 5.13901 seconds: | 20260507 16:31:45.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:45.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:31:45.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:31:45.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:45.954 | INFO | Uploaded 36GiB to GPU in 2.45859s (14.6425GiB/s) | 20260507 16:31:45.954 | INFO | Downloaded 34GiB to GPU in 2.49531s (13.6256GiB/s) | 20260507 16:31:46.062 | INFO | Starting a round of Baz IO | 20260507 16:31:49.573 | INFO | Populating RTMetrics report | 20260507 16:31:49.574 | INFO | RTMetrics report populated | 20260507 16:31:49.574 | INFO | Opening tmp RTMetrics json file | 20260507 16:31:49.589 | INFO | Serializing RTMetrics to json | 20260507 16:31:49.595 | INFO | Serialization done, swapping tmp file now... | 20260507 16:31:49.595 | INFO | RTMetrics done | 20260507 16:31:49.673 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.61 seconds | 20260507 16:31:49.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.06% / 0.03% / 49.91% | 20260507 16:31:49.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.42329% / 3.97364e-05% / 3.17891e-05% | 20260507 16:31:49.673 | INFO | Baz IO overall: | 20260507 16:31:49.673 | INFO | BytesWritten : 409943148692 | 20260507 16:31:49.673 | INFO | NumEvents : 198353350276 | 20260507 16:31:51.093 | INFO | Analyzing chunk frames = [954368,954880) | 20260507 16:31:51.093 | INFO | MemInfo: { | 20260507 16:31:51.093 | INFO | "gpu" : " 17 GB", | 20260507 16:31:51.093 | INFO | "huge" : "", | 20260507 16:31:51.093 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:51.093 | INFO | "malloc" : " 14 GB", | 20260507 16:31:51.093 | INFO | "pinned" : " 29 GB" | 20260507 16:31:51.093 | INFO | } | 20260507 16:31:51.094 | INFO | Perf report for the last 5.13973 seconds: | 20260507 16:31:51.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:51.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:51.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:51.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:51.094 | INFO | ANALYSIS : 70.0%, 3.0, 108.4ms, 0 | 20260507 16:31:51.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:51.094 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 16:31:51.094 | INFO | PRE_HQ : 66.8%, 1.7, 60.1ms, 0 | 20260507 16:31:51.094 | INFO | BAZWRITER : 70.2%, 3.6, 130.9ms, 0 | 20260507 16:31:51.094 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 16:31:51.094 | INFO | Uploaded 36GiB to GPU in 2.56054s (14.0595GiB/s) | 20260507 16:31:51.094 | INFO | Downloaded 38GiB to GPU in 2.72177s (13.9615GiB/s) | 20260507 16:31:56.233 | INFO | Analyzing chunk frames = [954880,955392) | 20260507 16:31:56.233 | INFO | MemInfo: { | 20260507 16:31:56.233 | INFO | "gpu" : " 17 GB", | 20260507 16:31:56.233 | INFO | "huge" : "", | 20260507 16:31:56.233 | INFO | "huge_shared" : " 29 GB", | 20260507 16:31:56.233 | INFO | "malloc" : " 14 GB", | 20260507 16:31:56.233 | INFO | "pinned" : " 29 GB" | 20260507 16:31:56.233 | INFO | } | 20260507 16:31:56.233 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:31:56.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:31:56.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.233 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 16:31:56.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.233 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:31:56.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:31:56.234 | INFO | Uploaded 36GiB to GPU in 2.52897s (14.235GiB/s) | 20260507 16:31:56.234 | INFO | Downloaded 34GiB to GPU in 2.42162s (14.0402GiB/s) | 20260507 16:32:01.372 | INFO | Analyzing chunk frames = [955392,955904) | 20260507 16:32:01.372 | INFO | MemInfo: { | 20260507 16:32:01.372 | INFO | "gpu" : " 17 GB", | 20260507 16:32:01.372 | INFO | "huge" : "", | 20260507 16:32:01.372 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:01.372 | INFO | "malloc" : " 14 GB", | 20260507 16:32:01.372 | INFO | "pinned" : " 29 GB" | 20260507 16:32:01.372 | INFO | } | 20260507 16:32:01.372 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:32:01.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:01.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.372 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:32:01.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.372 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:32:01.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:01.373 | INFO | Uploaded 36GiB to GPU in 2.56893s (14.0136GiB/s) | 20260507 16:32:01.373 | INFO | Downloaded 34GiB to GPU in 2.42403s (14.0262GiB/s) | 20260507 16:32:06.511 | INFO | Analyzing chunk frames = [955904,956416) | 20260507 16:32:06.511 | INFO | MemInfo: { | 20260507 16:32:06.511 | INFO | "gpu" : " 17 GB", | 20260507 16:32:06.511 | INFO | "huge" : "", | 20260507 16:32:06.511 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:06.511 | INFO | "malloc" : " 14 GB", | 20260507 16:32:06.512 | INFO | "pinned" : " 29 GB" | 20260507 16:32:06.512 | INFO | } | 20260507 16:32:06.512 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:32:06.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:06.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 16:32:06.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:32:06.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:06.512 | INFO | Uploaded 36GiB to GPU in 2.50755s (14.3567GiB/s) | 20260507 16:32:06.512 | INFO | Downloaded 34GiB to GPU in 2.34486s (14.4998GiB/s) | 20260507 16:32:11.651 | INFO | Analyzing chunk frames = [956416,956928) | 20260507 16:32:11.651 | INFO | MemInfo: { | 20260507 16:32:11.651 | INFO | "gpu" : " 17 GB", | 20260507 16:32:11.651 | INFO | "huge" : "", | 20260507 16:32:11.651 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:11.651 | INFO | "malloc" : " 14 GB", | 20260507 16:32:11.651 | INFO | "pinned" : " 29 GB" | 20260507 16:32:11.651 | INFO | } | 20260507 16:32:11.651 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:32:11.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:11.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.651 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:32:11.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.651 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:32:11.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:11.652 | INFO | Uploaded 36GiB to GPU in 2.4061s (14.962GiB/s) | 20260507 16:32:11.652 | INFO | Downloaded 34GiB to GPU in 2.45663s (13.8401GiB/s) | 20260507 16:32:16.790 | INFO | Analyzing chunk frames = [956928,957440) | 20260507 16:32:16.790 | INFO | MemInfo: { | 20260507 16:32:16.790 | INFO | "gpu" : " 17 GB", | 20260507 16:32:16.790 | INFO | "huge" : "", | 20260507 16:32:16.790 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:16.790 | INFO | "malloc" : " 14 GB", | 20260507 16:32:16.790 | INFO | "pinned" : " 29 GB" | 20260507 16:32:16.790 | INFO | } | 20260507 16:32:16.790 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:32:16.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:16.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.790 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:32:16.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.790 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:32:16.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:16.791 | INFO | Uploaded 36GiB to GPU in 2.38336s (15.1047GiB/s) | 20260507 16:32:16.791 | INFO | Downloaded 34GiB to GPU in 2.43284s (13.9755GiB/s) | 20260507 16:32:21.929 | INFO | Analyzing chunk frames = [957440,957952) | 20260507 16:32:21.929 | INFO | MemInfo: { | 20260507 16:32:21.929 | INFO | "gpu" : " 17 GB", | 20260507 16:32:21.929 | INFO | "huge" : "", | 20260507 16:32:21.929 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:21.929 | INFO | "malloc" : " 14 GB", | 20260507 16:32:21.929 | INFO | "pinned" : " 29 GB" | 20260507 16:32:21.929 | INFO | } | 20260507 16:32:21.929 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:32:21.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:21.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.930 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:32:21.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.930 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:32:21.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:21.930 | INFO | Uploaded 36GiB to GPU in 2.48098s (14.5104GiB/s) | 20260507 16:32:21.930 | INFO | Downloaded 34GiB to GPU in 2.40735s (14.1234GiB/s) | 20260507 16:32:27.068 | INFO | Analyzing chunk frames = [957952,958464) | 20260507 16:32:27.068 | INFO | MemInfo: { | 20260507 16:32:27.068 | INFO | "gpu" : " 17 GB", | 20260507 16:32:27.068 | INFO | "huge" : "", | 20260507 16:32:27.068 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:27.069 | INFO | "malloc" : " 14 GB", | 20260507 16:32:27.069 | INFO | "pinned" : " 29 GB" | 20260507 16:32:27.069 | INFO | } | 20260507 16:32:27.069 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:32:27.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:27.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 16:32:27.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:32:27.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:27.069 | INFO | Uploaded 36GiB to GPU in 2.4669s (14.5932GiB/s) | 20260507 16:32:27.069 | INFO | Downloaded 34GiB to GPU in 2.48182s (13.6996GiB/s) | 20260507 16:32:27.069 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:32:27.069 | INFO | Total mem usage: 16.4 GiB | 20260507 16:32:27.069 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:32:27.069 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.4%) | 20260507 16:32:27.069 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:32:27.069 | INFO | High Water Breakdown: | 20260507 16:32:27.069 | INFO | 59.2% : 8.72 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:32:27.069 | INFO | 40.6% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:32:27.069 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:32:27.069 | INFO | ------------------------------------------ | 20260507 16:32:27.069 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:32:27.069 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:32:27.069 | INFO | Total mem usage: 26.5 GiB | 20260507 16:32:27.069 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:32:27.069 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:32:27.069 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:32:27.069 | INFO | High Water Breakdown: | 20260507 16:32:27.069 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:32:27.069 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:32:27.069 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:32:27.069 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:32:27.069 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:32:27.069 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:32:27.069 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:32:27.069 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:32:27.069 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:32:27.069 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:32:27.069 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:32:27.069 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:32:27.069 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:32:27.069 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:32:27.069 | INFO | ------------------------------------------ | 20260507 16:32:27.069 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:32:27.069 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:32:27.069 | INFO | Total mem usage: 27.8 GiB | 20260507 16:32:27.069 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:32:27.069 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:32:27.070 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:32:27.070 | INFO | High Water Breakdown: | 20260507 16:32:27.070 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:32:27.070 | INFO | ------------------------------------------ | 20260507 16:32:27.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:32:27.070 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:32:27.070 | INFO | Total mem usage: 15.7 GiB | 20260507 16:32:27.070 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:32:27.070 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:32:27.070 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:32:27.070 | INFO | High Water Breakdown: | 20260507 16:32:27.070 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:32:27.070 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:32:27.070 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:32:27.070 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:32:27.070 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:32:27.070 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:32:27.070 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:32:27.070 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:32:27.070 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:32:27.070 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:32:27.070 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:32:27.070 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:32:27.070 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:32:27.070 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:32:27.070 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:32:27.070 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:32:27.070 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:32:27.070 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:32:27.070 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:32:27.070 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:32:27.070 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:32:27.070 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:32:27.070 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:32:27.070 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:32:27.070 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:32:27.070 | INFO | ------------------------------------------ | 20260507 16:32:27.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:32:27.070 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:32:27.070 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:32:27.070 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:32:27.070 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.40e+01 / 2.80e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 0.29%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.04e-02 / 5.24e-03 | 20260507 16:32:27.070 | INFO | PerfReport: 12.03%, Baseline -- 4.04e+00 / 4.48e+00, 6.18e-01 / 2.70e-01 | 20260507 16:32:27.070 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.50e+00, 2.15e-01 / 1.02e-01 | 20260507 16:32:27.070 | INFO | PerfReport: 15.73%, DME -- 4.23e+01 / 4.52e+01, 4.53e+00 / 2.59e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.88e+01 / 2.03e+01, 1.88e+00 / 2.11e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.92e+00, 4.31e-01 / 2.28e-01 | 20260507 16:32:27.070 | INFO | PerfReport: 3.84%, Metrics -- 1.29e+00 / 1.48e+00, 3.32e-01 / 2.04e-01 | 20260507 16:32:27.070 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.10e-01 / 4.37e-01, 6.81e-02 / 4.56e-02 | 20260507 16:32:27.070 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:32:27.070 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:32:27.070 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:32:27.070 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 24.05%, Upload -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 23.78%, Download -- 2.56e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 31.22%, Compute -- 3.36e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 1.98%, LockUpload -- 2.13e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 2.10%, LockDownload -- 2.26e+00 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 16.86%, LockCompute -- 1.82e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:32:27.070 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260507 16:32:27.166 | INFO | Starting a round of Baz IO | 20260507 16:32:30.673 | INFO | Populating RTMetrics report | 20260507 16:32:30.674 | INFO | RTMetrics report populated | 20260507 16:32:30.674 | INFO | Opening tmp RTMetrics json file | 20260507 16:32:30.696 | INFO | Serializing RTMetrics to json | 20260507 16:32:30.697 | INFO | Serialization done, swapping tmp file now... | 20260507 16:32:30.698 | INFO | RTMetrics done | 20260507 16:32:30.776 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.61 seconds | 20260507 16:32:30.776 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.03% / 0.03% / 49.94% | 20260507 16:32:30.776 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.41238% / 4.76837e-05% / 3.57628e-05% | 20260507 16:32:30.776 | INFO | Baz IO overall: | 20260507 16:32:30.776 | INFO | BytesWritten : 411860807545 | 20260507 16:32:30.776 | INFO | NumEvents : 199286500751 | 20260507 16:32:32.208 | INFO | Analyzing chunk frames = [958464,958976) | 20260507 16:32:32.208 | INFO | MemInfo: { | 20260507 16:32:32.208 | INFO | "gpu" : " 17 GB", | 20260507 16:32:32.208 | INFO | "huge" : "", | 20260507 16:32:32.208 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:32.208 | INFO | "malloc" : " 14 GB", | 20260507 16:32:32.208 | INFO | "pinned" : " 29 GB" | 20260507 16:32:32.208 | INFO | } | 20260507 16:32:32.208 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:32:32.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:32.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:32.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:32.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:32.208 | INFO | ANALYSIS : 69.7%, 2.9, 107.7ms, 0 | 20260507 16:32:32.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:32.208 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 16:32:32.208 | INFO | PRE_HQ : 66.8%, 1.7, 60.2ms, 0 | 20260507 16:32:32.208 | INFO | BAZWRITER : 69.8%, 4.0, 142.9ms, 0 | 20260507 16:32:32.208 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 16:32:32.209 | INFO | Uploaded 36GiB to GPU in 2.5488s (14.1243GiB/s) | 20260507 16:32:32.209 | INFO | Downloaded 38GiB to GPU in 2.69199s (14.116GiB/s) | 20260507 16:32:37.347 | INFO | Analyzing chunk frames = [958976,959488) | 20260507 16:32:37.347 | INFO | MemInfo: { | 20260507 16:32:37.347 | INFO | "gpu" : " 17 GB", | 20260507 16:32:37.347 | INFO | "huge" : "", | 20260507 16:32:37.347 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:37.347 | INFO | "malloc" : " 14 GB", | 20260507 16:32:37.347 | INFO | "pinned" : " 29 GB" | 20260507 16:32:37.347 | INFO | } | 20260507 16:32:37.347 | INFO | Perf report for the last 5.13914 seconds: | 20260507 16:32:37.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:37.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.347 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:32:37.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.347 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:32:37.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:37.348 | INFO | Uploaded 36GiB to GPU in 2.51608s (14.308GiB/s) | 20260507 16:32:37.348 | INFO | Downloaded 34GiB to GPU in 2.41838s (14.059GiB/s) | 20260507 16:32:42.487 | INFO | Analyzing chunk frames = [959488,960000) | 20260507 16:32:42.487 | INFO | MemInfo: { | 20260507 16:32:42.487 | INFO | "gpu" : " 17 GB", | 20260507 16:32:42.487 | INFO | "huge" : "", | 20260507 16:32:42.487 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:42.487 | INFO | "malloc" : " 14 GB", | 20260507 16:32:42.487 | INFO | "pinned" : " 29 GB" | 20260507 16:32:42.487 | INFO | } | 20260507 16:32:42.487 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:32:42.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:42.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 16:32:42.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:32:42.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:42.487 | INFO | Uploaded 36GiB to GPU in 2.48268s (14.5004GiB/s) | 20260507 16:32:42.487 | INFO | Downloaded 34GiB to GPU in 2.33231s (14.5778GiB/s) | 20260507 16:32:47.626 | INFO | Analyzing chunk frames = [960000,960512) | 20260507 16:32:47.626 | INFO | MemInfo: { | 20260507 16:32:47.626 | INFO | "gpu" : " 17 GB", | 20260507 16:32:47.626 | INFO | "huge" : "", | 20260507 16:32:47.626 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:47.626 | INFO | "malloc" : " 14 GB", | 20260507 16:32:47.626 | INFO | "pinned" : " 29 GB" | 20260507 16:32:47.626 | INFO | } | 20260507 16:32:47.626 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:32:47.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:47.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.626 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260507 16:32:47.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.626 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:32:47.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:47.627 | INFO | Uploaded 36GiB to GPU in 2.52034s (14.2838GiB/s) | 20260507 16:32:47.627 | INFO | Downloaded 34GiB to GPU in 2.35441s (14.441GiB/s) | 20260507 16:32:52.765 | INFO | Analyzing chunk frames = [960512,961024) | 20260507 16:32:52.765 | INFO | MemInfo: { | 20260507 16:32:52.765 | INFO | "gpu" : " 17 GB", | 20260507 16:32:52.765 | INFO | "huge" : "", | 20260507 16:32:52.765 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:52.765 | INFO | "malloc" : " 14 GB", | 20260507 16:32:52.765 | INFO | "pinned" : " 29 GB" | 20260507 16:32:52.765 | INFO | } | 20260507 16:32:52.765 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:32:52.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:52.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.765 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260507 16:32:52.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.765 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 16:32:52.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:52.766 | INFO | Uploaded 36GiB to GPU in 2.44509s (14.7234GiB/s) | 20260507 16:32:52.766 | INFO | Downloaded 34GiB to GPU in 2.45887s (13.8275GiB/s) | 20260507 16:32:57.905 | INFO | Analyzing chunk frames = [961024,961536) | 20260507 16:32:57.905 | INFO | MemInfo: { | 20260507 16:32:57.905 | INFO | "gpu" : " 17 GB", | 20260507 16:32:57.905 | INFO | "huge" : "", | 20260507 16:32:57.905 | INFO | "huge_shared" : " 29 GB", | 20260507 16:32:57.905 | INFO | "malloc" : " 14 GB", | 20260507 16:32:57.905 | INFO | "pinned" : " 29 GB" | 20260507 16:32:57.905 | INFO | } | 20260507 16:32:57.905 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:32:57.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:32:57.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:32:57.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:32:57.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:32:57.905 | INFO | Uploaded 36GiB to GPU in 2.40368s (14.977GiB/s) | 20260507 16:32:57.906 | INFO | Downloaded 34GiB to GPU in 2.43434s (13.9668GiB/s) | 20260507 16:33:03.044 | INFO | Analyzing chunk frames = [961536,962048) | 20260507 16:33:03.044 | INFO | MemInfo: { | 20260507 16:33:03.044 | INFO | "gpu" : " 17 GB", | 20260507 16:33:03.044 | INFO | "huge" : "", | 20260507 16:33:03.044 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:03.044 | INFO | "malloc" : " 14 GB", | 20260507 16:33:03.044 | INFO | "pinned" : " 29 GB" | 20260507 16:33:03.044 | INFO | } | 20260507 16:33:03.044 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:33:03.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:03.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:33:03.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:33:03.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:03.044 | INFO | Uploaded 36GiB to GPU in 2.48732s (14.4734GiB/s) | 20260507 16:33:03.045 | INFO | Downloaded 34GiB to GPU in 2.40596s (14.1316GiB/s) | 20260507 16:33:08.183 | INFO | Analyzing chunk frames = [962048,962560) | 20260507 16:33:08.183 | INFO | MemInfo: { | 20260507 16:33:08.183 | INFO | "gpu" : " 17 GB", | 20260507 16:33:08.183 | INFO | "huge" : "", | 20260507 16:33:08.183 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:08.183 | INFO | "malloc" : " 14 GB", | 20260507 16:33:08.183 | INFO | "pinned" : " 29 GB" | 20260507 16:33:08.183 | INFO | } | 20260507 16:33:08.184 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:33:08.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:08.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 16:33:08.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:33:08.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:08.184 | INFO | Uploaded 36GiB to GPU in 2.51814s (14.2963GiB/s) | 20260507 16:33:08.184 | INFO | Downloaded 34GiB to GPU in 2.48953s (13.6572GiB/s) | 20260507 16:33:08.281 | INFO | Starting a round of Baz IO | 20260507 16:33:11.795 | INFO | Populating RTMetrics report | 20260507 16:33:11.796 | INFO | RTMetrics report populated | 20260507 16:33:11.796 | INFO | Opening tmp RTMetrics json file | 20260507 16:33:11.809 | INFO | Serializing RTMetrics to json | 20260507 16:33:11.814 | INFO | Serialization done, swapping tmp file now... | 20260507 16:33:11.814 | INFO | RTMetrics done | 20260507 16:33:11.886 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.605 seconds | 20260507 16:33:11.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.01% / 0.03% / 49.97% | 20260507 16:33:11.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.39292% / 2.38419e-05% / 3.17891e-05% | 20260507 16:33:11.886 | INFO | Baz IO overall: | 20260507 16:33:11.886 | INFO | BytesWritten : 413777282525 | 20260507 16:33:11.886 | INFO | NumEvents : 200218598332 | 20260507 16:33:13.322 | INFO | Analyzing chunk frames = [962560,963072) | 20260507 16:33:13.322 | INFO | MemInfo: { | 20260507 16:33:13.322 | INFO | "gpu" : " 17 GB", | 20260507 16:33:13.322 | INFO | "huge" : "", | 20260507 16:33:13.323 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:13.323 | INFO | "malloc" : " 14 GB", | 20260507 16:33:13.323 | INFO | "pinned" : " 29 GB" | 20260507 16:33:13.323 | INFO | } | 20260507 16:33:13.323 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:33:13.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:13.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:13.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:13.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:13.323 | INFO | ANALYSIS : 69.8%, 2.9, 107.7ms, 0 | 20260507 16:33:13.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:13.323 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 16:33:13.323 | INFO | PRE_HQ : 66.5%, 1.7, 59.9ms, 0 | 20260507 16:33:13.323 | INFO | BAZWRITER : 69.7%, 3.8, 135.3ms, 0 | 20260507 16:33:13.323 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 16:33:13.323 | INFO | Uploaded 36GiB to GPU in 2.55258s (14.1034GiB/s) | 20260507 16:33:13.323 | INFO | Downloaded 38GiB to GPU in 2.71291s (14.0071GiB/s) | 20260507 16:33:18.462 | INFO | Analyzing chunk frames = [963072,963584) | 20260507 16:33:18.462 | INFO | MemInfo: { | 20260507 16:33:18.462 | INFO | "gpu" : " 17 GB", | 20260507 16:33:18.462 | INFO | "huge" : "", | 20260507 16:33:18.462 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:18.462 | INFO | "malloc" : " 14 GB", | 20260507 16:33:18.462 | INFO | "pinned" : " 29 GB" | 20260507 16:33:18.462 | INFO | } | 20260507 16:33:18.462 | INFO | Perf report for the last 5.1397 seconds: | 20260507 16:33:18.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:18.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.462 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 16:33:18.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.462 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:33:18.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:18.463 | INFO | Uploaded 36GiB to GPU in 2.53059s (14.226GiB/s) | 20260507 16:33:18.463 | INFO | Downloaded 34GiB to GPU in 2.41337s (14.0882GiB/s) | 20260507 16:33:23.601 | INFO | Analyzing chunk frames = [963584,964096) | 20260507 16:33:23.601 | INFO | MemInfo: { | 20260507 16:33:23.601 | INFO | "gpu" : " 17 GB", | 20260507 16:33:23.601 | INFO | "huge" : "", | 20260507 16:33:23.602 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:23.602 | INFO | "malloc" : " 14 GB", | 20260507 16:33:23.602 | INFO | "pinned" : " 29 GB" | 20260507 16:33:23.602 | INFO | } | 20260507 16:33:23.602 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:33:23.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:23.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:33:23.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:33:23.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:23.602 | INFO | Uploaded 36GiB to GPU in 2.55146s (14.1096GiB/s) | 20260507 16:33:23.602 | INFO | Downloaded 34GiB to GPU in 2.42562s (14.017GiB/s) | 20260507 16:33:28.740 | INFO | Analyzing chunk frames = [964096,964608) | 20260507 16:33:28.741 | INFO | MemInfo: { | 20260507 16:33:28.741 | INFO | "gpu" : " 17 GB", | 20260507 16:33:28.741 | INFO | "huge" : "", | 20260507 16:33:28.741 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:28.741 | INFO | "malloc" : " 14 GB", | 20260507 16:33:28.741 | INFO | "pinned" : " 29 GB" | 20260507 16:33:28.741 | INFO | } | 20260507 16:33:28.741 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:33:28.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:28.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260507 16:33:28.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:33:28.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:28.741 | INFO | Uploaded 36GiB to GPU in 2.52309s (14.2682GiB/s) | 20260507 16:33:28.741 | INFO | Downloaded 34GiB to GPU in 2.3543s (14.4417GiB/s) | 20260507 16:33:33.880 | INFO | Analyzing chunk frames = [964608,965120) | 20260507 16:33:33.880 | INFO | MemInfo: { | 20260507 16:33:33.880 | INFO | "gpu" : " 17 GB", | 20260507 16:33:33.880 | INFO | "huge" : "", | 20260507 16:33:33.880 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:33.881 | INFO | "malloc" : " 14 GB", | 20260507 16:33:33.881 | INFO | "pinned" : " 29 GB" | 20260507 16:33:33.881 | INFO | } | 20260507 16:33:33.881 | INFO | Perf report for the last 5.13981 seconds: | 20260507 16:33:33.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:33.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:33:33.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 16:33:33.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:33.881 | INFO | Uploaded 36GiB to GPU in 2.39746s (15.0159GiB/s) | 20260507 16:33:33.881 | INFO | Downloaded 34GiB to GPU in 2.46733s (13.7801GiB/s) | 20260507 16:33:39.019 | INFO | Analyzing chunk frames = [965120,965632) | 20260507 16:33:39.019 | INFO | MemInfo: { | 20260507 16:33:39.019 | INFO | "gpu" : " 17 GB", | 20260507 16:33:39.019 | INFO | "huge" : "", | 20260507 16:33:39.019 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:39.019 | INFO | "malloc" : " 14 GB", | 20260507 16:33:39.019 | INFO | "pinned" : " 29 GB" | 20260507 16:33:39.019 | INFO | } | 20260507 16:33:39.020 | INFO | Perf report for the last 5.13892 seconds: | 20260507 16:33:39.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:39.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:33:39.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:33:39.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:39.020 | INFO | Uploaded 36GiB to GPU in 2.39559s (15.0276GiB/s) | 20260507 16:33:39.020 | INFO | Downloaded 34GiB to GPU in 2.43043s (13.9893GiB/s) | 20260507 16:33:44.159 | INFO | Analyzing chunk frames = [965632,966144) | 20260507 16:33:44.159 | INFO | MemInfo: { | 20260507 16:33:44.159 | INFO | "gpu" : " 17 GB", | 20260507 16:33:44.159 | INFO | "huge" : "", | 20260507 16:33:44.159 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:44.159 | INFO | "malloc" : " 14 GB", | 20260507 16:33:44.159 | INFO | "pinned" : " 29 GB" | 20260507 16:33:44.159 | INFO | } | 20260507 16:33:44.159 | INFO | Perf report for the last 5.13971 seconds: | 20260507 16:33:44.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:44.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.159 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 16:33:44.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.159 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:33:44.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:44.160 | INFO | Uploaded 36GiB to GPU in 2.36365s (15.2307GiB/s) | 20260507 16:33:44.160 | INFO | Downloaded 34GiB to GPU in 2.39473s (14.1978GiB/s) | 20260507 16:33:49.298 | INFO | Analyzing chunk frames = [966144,966656) | 20260507 16:33:49.298 | INFO | MemInfo: { | 20260507 16:33:49.298 | INFO | "gpu" : " 17 GB", | 20260507 16:33:49.298 | INFO | "huge" : "", | 20260507 16:33:49.298 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:49.298 | INFO | "malloc" : " 14 GB", | 20260507 16:33:49.298 | INFO | "pinned" : " 29 GB" | 20260507 16:33:49.298 | INFO | } | 20260507 16:33:49.298 | INFO | Perf report for the last 5.13903 seconds: | 20260507 16:33:49.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:49.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:33:49.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:33:49.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:49.299 | INFO | Uploaded 36GiB to GPU in 2.43791s (14.7668GiB/s) | 20260507 16:33:49.299 | INFO | Downloaded 34GiB to GPU in 2.49716s (13.6155GiB/s) | 20260507 16:33:49.299 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:33:49.299 | INFO | Total mem usage: 16.4 GiB | 20260507 16:33:49.299 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:33:49.299 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 16:33:49.299 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:33:49.299 | INFO | High Water Breakdown: | 20260507 16:33:49.299 | INFO | 59.4% : 8.8 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:33:49.299 | INFO | 40.5% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:33:49.299 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:33:49.299 | INFO | ------------------------------------------ | 20260507 16:33:49.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:33:49.300 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:33:49.300 | INFO | Total mem usage: 26.5 GiB | 20260507 16:33:49.300 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:33:49.300 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:33:49.300 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:33:49.300 | INFO | High Water Breakdown: | 20260507 16:33:49.300 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:33:49.300 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:33:49.300 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:33:49.300 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:33:49.300 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:33:49.300 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:33:49.300 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:33:49.300 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:33:49.300 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:33:49.300 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:33:49.300 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:33:49.300 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:33:49.300 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:33:49.300 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:33:49.300 | INFO | ------------------------------------------ | 20260507 16:33:49.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:33:49.300 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:33:49.300 | INFO | Total mem usage: 27.8 GiB | 20260507 16:33:49.300 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:33:49.300 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:33:49.300 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:33:49.300 | INFO | High Water Breakdown: | 20260507 16:33:49.300 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:33:49.300 | INFO | ------------------------------------------ | 20260507 16:33:49.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:33:49.300 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:33:49.300 | INFO | Total mem usage: 15.7 GiB | 20260507 16:33:49.300 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:33:49.300 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:33:49.300 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:33:49.300 | INFO | High Water Breakdown: | 20260507 16:33:49.300 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:33:49.300 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:33:49.300 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:33:49.300 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:33:49.300 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:33:49.300 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:33:49.300 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:33:49.300 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:33:49.300 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:33:49.300 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:33:49.300 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:33:49.300 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:33:49.300 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:33:49.300 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:33:49.300 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:33:49.300 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:33:49.300 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:33:49.300 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:33:49.300 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:33:49.300 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:33:49.300 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:33:49.300 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:33:49.300 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:33:49.300 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:33:49.300 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:33:49.300 | INFO | ------------------------------------------ | 20260507 16:33:49.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:33:49.300 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:33:49.300 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:33:49.300 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:33:49.300 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.68e+01, 1.45e+01 / 2.70e+00 | 20260507 16:33:49.300 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.11e-02 / 6.24e-03 | 20260507 16:33:49.300 | INFO | PerfReport: 12.01%, Baseline -- 4.03e+00 / 4.49e+00, 6.13e-01 / 2.33e-01 | 20260507 16:33:49.300 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.48e+00, 2.14e-01 / 1.14e-01 | 20260507 16:33:49.300 | INFO | PerfReport: 15.79%, DME -- 4.24e+01 / 4.54e+01, 4.39e+00 / 2.65e+00 | 20260507 16:33:49.300 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.86e+00 / 2.21e+00 | 20260507 16:33:49.300 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.54e+00 / 2.92e+00, 4.43e-01 / 2.40e-01 | 20260507 16:33:49.300 | INFO | PerfReport: 3.84%, Metrics -- 1.29e+00 / 1.46e+00, 3.31e-01 / 1.88e-01 | 20260507 16:33:49.300 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.10e-01 / 4.33e-01, 6.98e-02 / 4.42e-02 | 20260507 16:33:49.300 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:33:49.300 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:33:49.300 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:33:49.300 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 24.02%, Upload -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 23.78%, Download -- 2.56e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 31.24%, Compute -- 3.36e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 2.01%, LockUpload -- 2.16e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 2.07%, LockDownload -- 2.23e+00 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 16.87%, LockCompute -- 1.81e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:33:49.301 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 16:33:49.396 | INFO | Starting a round of Baz IO | 20260507 16:33:52.915 | INFO | Populating RTMetrics report | 20260507 16:33:52.915 | INFO | RTMetrics report populated | 20260507 16:33:52.915 | INFO | Opening tmp RTMetrics json file | 20260507 16:33:52.934 | INFO | Serializing RTMetrics to json | 20260507 16:33:52.936 | INFO | Serialization done, swapping tmp file now... | 20260507 16:33:52.939 | INFO | RTMetrics done | 20260507 16:33:53.024 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.627 seconds | 20260507 16:33:53.024 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.98% / 0.03% / 49.99% | 20260507 16:33:53.024 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.38319% / 5.96046e-05% / 3.97364e-05% | 20260507 16:33:53.024 | INFO | Baz IO overall: | 20260507 16:33:53.024 | INFO | BytesWritten : 415694198647 | 20260507 16:33:53.024 | INFO | NumEvents : 201150804853 | 20260507 16:33:54.438 | INFO | Analyzing chunk frames = [966656,967168) | 20260507 16:33:54.438 | INFO | MemInfo: { | 20260507 16:33:54.438 | INFO | "gpu" : " 17 GB", | 20260507 16:33:54.438 | INFO | "huge" : "", | 20260507 16:33:54.438 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:54.438 | INFO | "malloc" : " 14 GB", | 20260507 16:33:54.438 | INFO | "pinned" : " 29 GB" | 20260507 16:33:54.438 | INFO | } | 20260507 16:33:54.438 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:33:54.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:54.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:54.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:54.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:54.438 | INFO | ANALYSIS : 69.8%, 3.0, 108.1ms, 0 | 20260507 16:33:54.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:54.438 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 16:33:54.438 | INFO | PRE_HQ : 66.5%, 1.7, 60.1ms, 0 | 20260507 16:33:54.438 | INFO | BAZWRITER : 70.1%, 3.8, 137.2ms, 0 | 20260507 16:33:54.438 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 16:33:54.438 | INFO | Uploaded 36GiB to GPU in 2.54894s (14.1235GiB/s) | 20260507 16:33:54.438 | INFO | Downloaded 38GiB to GPU in 2.72401s (13.95GiB/s) | 20260507 16:33:59.577 | INFO | Analyzing chunk frames = [967168,967680) | 20260507 16:33:59.577 | INFO | MemInfo: { | 20260507 16:33:59.577 | INFO | "gpu" : " 17 GB", | 20260507 16:33:59.577 | INFO | "huge" : "", | 20260507 16:33:59.577 | INFO | "huge_shared" : " 29 GB", | 20260507 16:33:59.577 | INFO | "malloc" : " 14 GB", | 20260507 16:33:59.577 | INFO | "pinned" : " 29 GB" | 20260507 16:33:59.577 | INFO | } | 20260507 16:33:59.577 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:33:59.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:33:59.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.577 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 16:33:59.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.577 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:33:59.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:33:59.578 | INFO | Uploaded 36GiB to GPU in 2.49757s (14.414GiB/s) | 20260507 16:33:59.578 | INFO | Downloaded 34GiB to GPU in 2.44563s (13.9023GiB/s) | 20260507 16:34:04.716 | INFO | Analyzing chunk frames = [967680,968192) | 20260507 16:34:04.716 | INFO | MemInfo: { | 20260507 16:34:04.716 | INFO | "gpu" : " 17 GB", | 20260507 16:34:04.716 | INFO | "huge" : "", | 20260507 16:34:04.716 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:04.716 | INFO | "malloc" : " 14 GB", | 20260507 16:34:04.716 | INFO | "pinned" : " 29 GB" | 20260507 16:34:04.716 | INFO | } | 20260507 16:34:04.716 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:34:04.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:04.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.717 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260507 16:34:04.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.717 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:34:04.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:04.717 | INFO | Uploaded 36GiB to GPU in 2.47035s (14.5728GiB/s) | 20260507 16:34:04.717 | INFO | Downloaded 34GiB to GPU in 2.34044s (14.5272GiB/s) | 20260507 16:34:09.855 | INFO | Analyzing chunk frames = [968192,968704) | 20260507 16:34:09.855 | INFO | MemInfo: { | 20260507 16:34:09.855 | INFO | "gpu" : " 17 GB", | 20260507 16:34:09.855 | INFO | "huge" : "", | 20260507 16:34:09.856 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:09.856 | INFO | "malloc" : " 14 GB", | 20260507 16:34:09.856 | INFO | "pinned" : " 29 GB" | 20260507 16:34:09.856 | INFO | } | 20260507 16:34:09.856 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:34:09.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:09.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:34:09.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:34:09.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:09.856 | INFO | Uploaded 36GiB to GPU in 2.47158s (14.5656GiB/s) | 20260507 16:34:09.856 | INFO | Downloaded 34GiB to GPU in 2.36252s (14.3914GiB/s) | 20260507 16:34:14.995 | INFO | Analyzing chunk frames = [968704,969216) | 20260507 16:34:14.995 | INFO | MemInfo: { | 20260507 16:34:14.995 | INFO | "gpu" : " 17 GB", | 20260507 16:34:14.995 | INFO | "huge" : "", | 20260507 16:34:14.995 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:14.996 | INFO | "malloc" : " 14 GB", | 20260507 16:34:14.996 | INFO | "pinned" : " 29 GB" | 20260507 16:34:14.996 | INFO | } | 20260507 16:34:14.996 | INFO | Perf report for the last 5.13995 seconds: | 20260507 16:34:14.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:14.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:34:14.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 16:34:14.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:14.996 | INFO | Uploaded 36GiB to GPU in 2.39202s (15.05GiB/s) | 20260507 16:34:14.996 | INFO | Downloaded 34GiB to GPU in 2.45621s (13.8425GiB/s) | 20260507 16:34:20.134 | INFO | Analyzing chunk frames = [969216,969728) | 20260507 16:34:20.134 | INFO | MemInfo: { | 20260507 16:34:20.134 | INFO | "gpu" : " 17 GB", | 20260507 16:34:20.134 | INFO | "huge" : "", | 20260507 16:34:20.134 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:20.134 | INFO | "malloc" : " 14 GB", | 20260507 16:34:20.134 | INFO | "pinned" : " 29 GB" | 20260507 16:34:20.134 | INFO | } | 20260507 16:34:20.135 | INFO | Perf report for the last 5.13887 seconds: | 20260507 16:34:20.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:20.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:34:20.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:34:20.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:20.135 | INFO | Uploaded 36GiB to GPU in 2.3768s (15.1464GiB/s) | 20260507 16:34:20.135 | INFO | Downloaded 34GiB to GPU in 2.45838s (13.8302GiB/s) | 20260507 16:34:25.274 | INFO | Analyzing chunk frames = [969728,970240) | 20260507 16:34:25.274 | INFO | MemInfo: { | 20260507 16:34:25.274 | INFO | "gpu" : " 17 GB", | 20260507 16:34:25.274 | INFO | "huge" : "", | 20260507 16:34:25.274 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:25.274 | INFO | "malloc" : " 14 GB", | 20260507 16:34:25.274 | INFO | "pinned" : " 29 GB" | 20260507 16:34:25.274 | INFO | } | 20260507 16:34:25.274 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:34:25.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:25.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:34:25.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:34:25.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:25.274 | INFO | Uploaded 36GiB to GPU in 2.42037s (14.8738GiB/s) | 20260507 16:34:25.274 | INFO | Downloaded 34GiB to GPU in 2.41745s (14.0644GiB/s) | 20260507 16:34:30.413 | INFO | Analyzing chunk frames = [970240,970752) | 20260507 16:34:30.413 | INFO | MemInfo: { | 20260507 16:34:30.413 | INFO | "gpu" : " 17 GB", | 20260507 16:34:30.413 | INFO | "huge" : "", | 20260507 16:34:30.413 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:30.413 | INFO | "malloc" : " 14 GB", | 20260507 16:34:30.413 | INFO | "pinned" : " 29 GB" | 20260507 16:34:30.413 | INFO | } | 20260507 16:34:30.413 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:34:30.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:30.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.413 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:34:30.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.413 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:34:30.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:30.414 | INFO | Uploaded 36GiB to GPU in 2.4337s (14.7923GiB/s) | 20260507 16:34:30.414 | INFO | Downloaded 34GiB to GPU in 2.51513s (13.5182GiB/s) | 20260507 16:34:30.511 | INFO | Starting a round of Baz IO | 20260507 16:34:34.026 | INFO | Populating RTMetrics report | 20260507 16:34:34.026 | INFO | RTMetrics report populated | 20260507 16:34:34.026 | INFO | Opening tmp RTMetrics json file | 20260507 16:34:34.027 | INFO | Serializing RTMetrics to json | 20260507 16:34:34.029 | INFO | Serialization done, swapping tmp file now... | 20260507 16:34:34.030 | INFO | RTMetrics done | 20260507 16:34:34.133 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.622 seconds | 20260507 16:34:34.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.95% / 0.03% / 50.02% | 20260507 16:34:34.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.38558% / 4.76837e-05% / 2.78155e-05% | 20260507 16:34:34.133 | INFO | Baz IO overall: | 20260507 16:34:34.133 | INFO | BytesWritten : 417614351490 | 20260507 16:34:34.133 | INFO | NumEvents : 202085171009 | 20260507 16:34:35.552 | INFO | Analyzing chunk frames = [970752,971264) | 20260507 16:34:35.552 | INFO | MemInfo: { | 20260507 16:34:35.552 | INFO | "gpu" : " 17 GB", | 20260507 16:34:35.552 | INFO | "huge" : "", | 20260507 16:34:35.552 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:35.552 | INFO | "malloc" : " 14 GB", | 20260507 16:34:35.552 | INFO | "pinned" : " 29 GB" | 20260507 16:34:35.552 | INFO | } | 20260507 16:34:35.552 | INFO | Perf report for the last 5.13899 seconds: | 20260507 16:34:35.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:35.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:35.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:35.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:35.552 | INFO | ANALYSIS : 69.8%, 3.0, 108.4ms, 0 | 20260507 16:34:35.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:35.552 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 16:34:35.552 | INFO | PRE_HQ : 66.6%, 1.7, 60.2ms, 0 | 20260507 16:34:35.552 | INFO | BAZWRITER : 69.9%, 3.8, 134.8ms, 0 | 20260507 16:34:35.552 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260507 16:34:35.553 | INFO | Uploaded 36GiB to GPU in 2.54987s (14.1184GiB/s) | 20260507 16:34:35.553 | INFO | Downloaded 38GiB to GPU in 2.72165s (13.9621GiB/s) | 20260507 16:34:40.692 | INFO | Analyzing chunk frames = [971264,971776) | 20260507 16:34:40.692 | INFO | MemInfo: { | 20260507 16:34:40.692 | INFO | "gpu" : " 17 GB", | 20260507 16:34:40.692 | INFO | "huge" : "", | 20260507 16:34:40.692 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:40.692 | INFO | "malloc" : " 14 GB", | 20260507 16:34:40.692 | INFO | "pinned" : " 29 GB" | 20260507 16:34:40.692 | INFO | } | 20260507 16:34:40.692 | INFO | Perf report for the last 5.1396 seconds: | 20260507 16:34:40.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:40.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 16:34:40.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:34:40.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:40.692 | INFO | Uploaded 36GiB to GPU in 2.5317s (14.2197GiB/s) | 20260507 16:34:40.692 | INFO | Downloaded 34GiB to GPU in 2.4162s (14.0717GiB/s) | 20260507 16:34:45.831 | INFO | Analyzing chunk frames = [971776,972288) | 20260507 16:34:45.831 | INFO | MemInfo: { | 20260507 16:34:45.831 | INFO | "gpu" : " 17 GB", | 20260507 16:34:45.831 | INFO | "huge" : "", | 20260507 16:34:45.831 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:45.831 | INFO | "malloc" : " 14 GB", | 20260507 16:34:45.831 | INFO | "pinned" : " 29 GB" | 20260507 16:34:45.831 | INFO | } | 20260507 16:34:45.831 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:34:45.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:45.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.832 | INFO | ANALYSIS : 68.5%, 3.0, 106.3ms, 0 | 20260507 16:34:45.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.832 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:34:45.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:45.832 | INFO | Uploaded 36GiB to GPU in 2.58221s (13.9415GiB/s) | 20260507 16:34:45.832 | INFO | Downloaded 34GiB to GPU in 2.39427s (14.2006GiB/s) | 20260507 16:34:50.970 | INFO | Analyzing chunk frames = [972288,972800) | 20260507 16:34:50.970 | INFO | MemInfo: { | 20260507 16:34:50.970 | INFO | "gpu" : " 17 GB", | 20260507 16:34:50.970 | INFO | "huge" : "", | 20260507 16:34:50.970 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:50.971 | INFO | "malloc" : " 14 GB", | 20260507 16:34:50.971 | INFO | "pinned" : " 29 GB" | 20260507 16:34:50.971 | INFO | } | 20260507 16:34:50.971 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:34:50.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:50.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 16:34:50.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:34:50.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:50.971 | INFO | Uploaded 36GiB to GPU in 2.4958s (14.4242GiB/s) | 20260507 16:34:50.971 | INFO | Downloaded 34GiB to GPU in 2.3504s (14.4656GiB/s) | 20260507 16:34:56.110 | INFO | Analyzing chunk frames = [972800,973312) | 20260507 16:34:56.110 | INFO | MemInfo: { | 20260507 16:34:56.110 | INFO | "gpu" : " 17 GB", | 20260507 16:34:56.110 | INFO | "huge" : "", | 20260507 16:34:56.110 | INFO | "huge_shared" : " 29 GB", | 20260507 16:34:56.110 | INFO | "malloc" : " 14 GB", | 20260507 16:34:56.110 | INFO | "pinned" : " 29 GB" | 20260507 16:34:56.110 | INFO | } | 20260507 16:34:56.110 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:34:56.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:34:56.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260507 16:34:56.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260507 16:34:56.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:34:56.110 | INFO | Uploaded 36GiB to GPU in 2.41226s (14.9238GiB/s) | 20260507 16:34:56.111 | INFO | Downloaded 34GiB to GPU in 2.45703s (13.8379GiB/s) | 20260507 16:35:01.249 | INFO | Analyzing chunk frames = [973312,973824) | 20260507 16:35:01.249 | INFO | MemInfo: { | 20260507 16:35:01.249 | INFO | "gpu" : " 17 GB", | 20260507 16:35:01.249 | INFO | "huge" : "", | 20260507 16:35:01.249 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:01.249 | INFO | "malloc" : " 14 GB", | 20260507 16:35:01.249 | INFO | "pinned" : " 29 GB" | 20260507 16:35:01.249 | INFO | } | 20260507 16:35:01.249 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:35:01.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:01.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.249 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:35:01.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.250 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 16:35:01.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:01.250 | INFO | Uploaded 36GiB to GPU in 2.38353s (15.1037GiB/s) | 20260507 16:35:01.250 | INFO | Downloaded 34GiB to GPU in 2.45554s (13.8463GiB/s) | 20260507 16:35:06.388 | INFO | Analyzing chunk frames = [973824,974336) | 20260507 16:35:06.388 | INFO | MemInfo: { | 20260507 16:35:06.388 | INFO | "gpu" : " 17 GB", | 20260507 16:35:06.388 | INFO | "huge" : "", | 20260507 16:35:06.388 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:06.388 | INFO | "malloc" : " 14 GB", | 20260507 16:35:06.388 | INFO | "pinned" : " 29 GB" | 20260507 16:35:06.389 | INFO | } | 20260507 16:35:06.389 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:35:06.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:06.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | ANALYSIS : 67.6%, 3.0, 104.3ms, 0 | 20260507 16:35:06.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | PRE_HQ : 39.1%, 1.0, 20.1ms, 0 | 20260507 16:35:06.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:06.389 | INFO | Uploaded 36GiB to GPU in 2.48667s (14.4772GiB/s) | 20260507 16:35:06.389 | INFO | Downloaded 34GiB to GPU in 2.41343s (14.0878GiB/s) | 20260507 16:35:11.528 | INFO | Analyzing chunk frames = [974336,974848) | 20260507 16:35:11.528 | INFO | MemInfo: { | 20260507 16:35:11.528 | INFO | "gpu" : " 17 GB", | 20260507 16:35:11.528 | INFO | "huge" : "", | 20260507 16:35:11.528 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:11.528 | INFO | "malloc" : " 14 GB", | 20260507 16:35:11.528 | INFO | "pinned" : " 29 GB" | 20260507 16:35:11.528 | INFO | } | 20260507 16:35:11.528 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:35:11.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:11.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.528 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 16:35:11.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.528 | INFO | PRE_HQ : 39.0%, 1.0, 20.3ms, 0 | 20260507 16:35:11.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:11.529 | INFO | Uploaded 36GiB to GPU in 2.56374s (14.042GiB/s) | 20260507 16:35:11.529 | INFO | Downloaded 34GiB to GPU in 2.51115s (13.5396GiB/s) | 20260507 16:35:11.529 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:35:11.529 | INFO | Total mem usage: 16.4 GiB | 20260507 16:35:11.529 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:35:11.529 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 16:35:11.529 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:35:11.529 | INFO | High Water Breakdown: | 20260507 16:35:11.529 | INFO | 59.4% : 8.8 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:35:11.529 | INFO | 40.5% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:35:11.529 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:35:11.529 | INFO | ------------------------------------------ | 20260507 16:35:11.529 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:35:11.529 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:35:11.529 | INFO | Total mem usage: 26.5 GiB | 20260507 16:35:11.529 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:35:11.529 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:35:11.529 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:35:11.529 | INFO | High Water Breakdown: | 20260507 16:35:11.529 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:35:11.529 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:35:11.529 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:35:11.529 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:35:11.529 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:35:11.529 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:35:11.529 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:35:11.529 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:35:11.529 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:35:11.529 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:35:11.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:35:11.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:35:11.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:35:11.530 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:35:11.530 | INFO | ------------------------------------------ | 20260507 16:35:11.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:35:11.530 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:35:11.530 | INFO | Total mem usage: 27.8 GiB | 20260507 16:35:11.530 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:35:11.530 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:35:11.530 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:35:11.530 | INFO | High Water Breakdown: | 20260507 16:35:11.530 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:35:11.530 | INFO | ------------------------------------------ | 20260507 16:35:11.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:35:11.530 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:35:11.530 | INFO | Total mem usage: 15.7 GiB | 20260507 16:35:11.530 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:35:11.530 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:35:11.530 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:35:11.530 | INFO | High Water Breakdown: | 20260507 16:35:11.530 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:35:11.530 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:35:11.530 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:35:11.530 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:35:11.530 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:35:11.530 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:35:11.530 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:35:11.530 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:35:11.530 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:35:11.530 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:35:11.530 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:35:11.530 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:35:11.530 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:35:11.530 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:35:11.530 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:35:11.530 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:35:11.530 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:35:11.530 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:35:11.530 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:35:11.530 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:35:11.530 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:35:11.530 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:35:11.530 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:35:11.530 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:35:11.530 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:35:11.530 | INFO | ------------------------------------------ | 20260507 16:35:11.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:35:11.530 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:35:11.530 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:35:11.530 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:35:11.530 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.41e+01 / 2.72e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 0.29%, AggregateStats -- 9.92e-02 / 1.03e-01, 1.16e-02 / 6.04e-03 | 20260507 16:35:11.530 | INFO | PerfReport: 11.99%, Baseline -- 4.03e+00 / 4.46e+00, 6.18e-01 / 2.86e-01 | 20260507 16:35:11.530 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.48e+00, 2.15e-01 / 1.13e-01 | 20260507 16:35:11.530 | INFO | PerfReport: 15.67%, DME -- 4.22e+01 / 4.52e+01, 4.55e+00 / 2.62e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.89e+01 / 2.04e+01, 1.95e+00 / 2.28e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.53e+00 / 2.94e+00, 4.44e-01 / 2.34e-01 | 20260507 16:35:11.530 | INFO | PerfReport: 3.83%, Metrics -- 1.29e+00 / 1.47e+00, 3.32e-01 / 1.81e-01 | 20260507 16:35:11.530 | INFO | PerfReport: 1.23%, PulseSerializing -- 4.13e-01 / 4.38e-01, 6.99e-02 / 4.60e-02 | 20260507 16:35:11.530 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:35:11.530 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:35:11.530 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:35:11.530 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 23.97%, Upload -- 2.58e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 23.87%, Download -- 2.57e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 31.28%, Compute -- 3.37e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 1.87%, LockUpload -- 2.01e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 2.10%, LockDownload -- 2.26e+00 / 0.00e+00, 3.73e+00 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 16.89%, LockCompute -- 1.82e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:35:11.530 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260507 16:35:11.642 | INFO | Starting a round of Baz IO | 20260507 16:35:15.144 | INFO | Populating RTMetrics report | 20260507 16:35:15.145 | INFO | RTMetrics report populated | 20260507 16:35:15.145 | INFO | Opening tmp RTMetrics json file | 20260507 16:35:15.168 | INFO | Serializing RTMetrics to json | 20260507 16:35:15.170 | INFO | Serialization done, swapping tmp file now... | 20260507 16:35:15.171 | INFO | RTMetrics done | 20260507 16:35:15.256 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.614 seconds | 20260507 16:35:15.256 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.93% / 0.03% / 50.05% | 20260507 16:35:15.256 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.35754% / 3.97364e-05% / 4.37101e-05% | 20260507 16:35:15.256 | INFO | Baz IO overall: | 20260507 16:35:15.256 | INFO | BytesWritten : 419533045640 | 20260507 16:35:15.256 | INFO | NumEvents : 203018201294 | 20260507 16:35:16.667 | INFO | Analyzing chunk frames = [974848,975360) | 20260507 16:35:16.667 | INFO | MemInfo: { | 20260507 16:35:16.667 | INFO | "gpu" : " 17 GB", | 20260507 16:35:16.667 | INFO | "huge" : "", | 20260507 16:35:16.667 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:16.667 | INFO | "malloc" : " 14 GB", | 20260507 16:35:16.667 | INFO | "pinned" : " 29 GB" | 20260507 16:35:16.667 | INFO | } | 20260507 16:35:16.668 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:35:16.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:16.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:16.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:16.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:16.668 | INFO | ANALYSIS : 69.9%, 2.9, 106.0ms, 0 | 20260507 16:35:16.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:16.668 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260507 16:35:16.668 | INFO | PRE_HQ : 69.1%, 2.0, 70.0ms, 0 | 20260507 16:35:16.668 | INFO | BAZWRITER : 69.9%, 4.1, 148.6ms, 0 | 20260507 16:35:16.668 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:35:16.668 | INFO | Uploaded 36GiB to GPU in 2.5542s (14.0944GiB/s) | 20260507 16:35:16.668 | INFO | Downloaded 38GiB to GPU in 2.70961s (14.0242GiB/s) | 20260507 16:35:21.806 | INFO | Analyzing chunk frames = [975360,975872) | 20260507 16:35:21.806 | INFO | MemInfo: { | 20260507 16:35:21.806 | INFO | "gpu" : " 17 GB", | 20260507 16:35:21.807 | INFO | "huge" : "", | 20260507 16:35:21.807 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:21.807 | INFO | "malloc" : " 14 GB", | 20260507 16:35:21.807 | INFO | "pinned" : " 29 GB" | 20260507 16:35:21.807 | INFO | } | 20260507 16:35:21.807 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:35:21.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:21.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:35:21.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 16:35:21.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:21.807 | INFO | Uploaded 36GiB to GPU in 2.53701s (14.1899GiB/s) | 20260507 16:35:21.807 | INFO | Downloaded 34GiB to GPU in 2.45188s (13.8669GiB/s) | 20260507 16:35:26.946 | INFO | Analyzing chunk frames = [975872,976384) | 20260507 16:35:26.946 | INFO | MemInfo: { | 20260507 16:35:26.946 | INFO | "gpu" : " 17 GB", | 20260507 16:35:26.946 | INFO | "huge" : "", | 20260507 16:35:26.946 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:26.946 | INFO | "malloc" : " 14 GB", | 20260507 16:35:26.946 | INFO | "pinned" : " 29 GB" | 20260507 16:35:26.946 | INFO | } | 20260507 16:35:26.946 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:35:26.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:26.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.946 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 16:35:26.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.947 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 16:35:26.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:26.947 | INFO | Uploaded 36GiB to GPU in 2.54075s (14.1691GiB/s) | 20260507 16:35:26.947 | INFO | Downloaded 34GiB to GPU in 2.44507s (13.9055GiB/s) | 20260507 16:35:32.085 | INFO | Analyzing chunk frames = [976384,976896) | 20260507 16:35:32.085 | INFO | MemInfo: { | 20260507 16:35:32.085 | INFO | "gpu" : " 17 GB", | 20260507 16:35:32.085 | INFO | "huge" : "", | 20260507 16:35:32.085 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:32.085 | INFO | "malloc" : " 14 GB", | 20260507 16:35:32.085 | INFO | "pinned" : " 29 GB" | 20260507 16:35:32.085 | INFO | } | 20260507 16:35:32.085 | INFO | Perf report for the last 5.13898 seconds: | 20260507 16:35:32.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:32.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 16:35:32.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 16:35:32.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:32.086 | INFO | Uploaded 36GiB to GPU in 2.48012s (14.5154GiB/s) | 20260507 16:35:32.086 | INFO | Downloaded 34GiB to GPU in 2.36322s (14.3872GiB/s) | 20260507 16:35:37.225 | INFO | Analyzing chunk frames = [976896,977408) | 20260507 16:35:37.225 | INFO | MemInfo: { | 20260507 16:35:37.225 | INFO | "gpu" : " 17 GB", | 20260507 16:35:37.225 | INFO | "huge" : "", | 20260507 16:35:37.225 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:37.225 | INFO | "malloc" : " 14 GB", | 20260507 16:35:37.225 | INFO | "pinned" : " 29 GB" | 20260507 16:35:37.225 | INFO | } | 20260507 16:35:37.225 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:35:37.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:37.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 16:35:37.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 16:35:37.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:37.225 | INFO | Uploaded 36GiB to GPU in 2.29811s (15.6651GiB/s) | 20260507 16:35:37.225 | INFO | Downloaded 34GiB to GPU in 2.47556s (13.7343GiB/s) | 20260507 16:35:42.364 | INFO | Analyzing chunk frames = [977408,977920) | 20260507 16:35:42.364 | INFO | MemInfo: { | 20260507 16:35:42.364 | INFO | "gpu" : " 17 GB", | 20260507 16:35:42.364 | INFO | "huge" : "", | 20260507 16:35:42.364 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:42.364 | INFO | "malloc" : " 14 GB", | 20260507 16:35:42.364 | INFO | "pinned" : " 29 GB" | 20260507 16:35:42.364 | INFO | } | 20260507 16:35:42.364 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:35:42.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:42.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.364 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:35:42.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.364 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 16:35:42.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:42.365 | INFO | Uploaded 36GiB to GPU in 2.33522s (15.4161GiB/s) | 20260507 16:35:42.365 | INFO | Downloaded 34GiB to GPU in 2.47999s (13.7097GiB/s) | 20260507 16:35:47.503 | INFO | Analyzing chunk frames = [977920,978432) | 20260507 16:35:47.503 | INFO | MemInfo: { | 20260507 16:35:47.503 | INFO | "gpu" : " 17 GB", | 20260507 16:35:47.503 | INFO | "huge" : "", | 20260507 16:35:47.503 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:47.503 | INFO | "malloc" : " 14 GB", | 20260507 16:35:47.503 | INFO | "pinned" : " 29 GB" | 20260507 16:35:47.504 | INFO | } | 20260507 16:35:47.504 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:35:47.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:47.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 16:35:47.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:35:47.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:47.504 | INFO | Uploaded 36GiB to GPU in 2.3874s (15.0792GiB/s) | 20260507 16:35:47.504 | INFO | Downloaded 34GiB to GPU in 2.40863s (14.1159GiB/s) | 20260507 16:35:52.643 | INFO | Analyzing chunk frames = [978432,978944) | 20260507 16:35:52.643 | INFO | MemInfo: { | 20260507 16:35:52.643 | INFO | "gpu" : " 17 GB", | 20260507 16:35:52.643 | INFO | "huge" : "", | 20260507 16:35:52.643 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:52.643 | INFO | "malloc" : " 14 GB", | 20260507 16:35:52.643 | INFO | "pinned" : " 29 GB" | 20260507 16:35:52.643 | INFO | } | 20260507 16:35:52.643 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:35:52.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:52.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:35:52.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:35:52.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:52.643 | INFO | Uploaded 36GiB to GPU in 2.43125s (14.8072GiB/s) | 20260507 16:35:52.643 | INFO | Downloaded 34GiB to GPU in 2.51469s (13.5206GiB/s) | 20260507 16:35:52.740 | INFO | Starting a round of Baz IO | 20260507 16:35:56.258 | INFO | Populating RTMetrics report | 20260507 16:35:56.258 | INFO | RTMetrics report populated | 20260507 16:35:56.258 | INFO | Opening tmp RTMetrics json file | 20260507 16:35:56.277 | INFO | Serializing RTMetrics to json | 20260507 16:35:56.342 | INFO | Serialization done, swapping tmp file now... | 20260507 16:35:56.379 | INFO | RTMetrics done | 20260507 16:35:56.403 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.662 seconds | 20260507 16:35:56.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.90% / 0.03% / 50.07% | 20260507 16:35:56.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.36292% / 3.57628e-05% / 2.38419e-05% | 20260507 16:35:56.403 | INFO | Baz IO overall: | 20260507 16:35:56.403 | INFO | BytesWritten : 421453709443 | 20260507 16:35:56.403 | INFO | NumEvents : 203952885145 | 20260507 16:35:57.782 | INFO | Analyzing chunk frames = [978944,979456) | 20260507 16:35:57.782 | INFO | MemInfo: { | 20260507 16:35:57.782 | INFO | "gpu" : " 17 GB", | 20260507 16:35:57.782 | INFO | "huge" : "", | 20260507 16:35:57.782 | INFO | "huge_shared" : " 29 GB", | 20260507 16:35:57.782 | INFO | "malloc" : " 14 GB", | 20260507 16:35:57.782 | INFO | "pinned" : " 29 GB" | 20260507 16:35:57.782 | INFO | } | 20260507 16:35:57.782 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:35:57.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:35:57.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:57.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:57.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:57.782 | INFO | ANALYSIS : 69.9%, 3.0, 108.7ms, 0 | 20260507 16:35:57.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:35:57.783 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 16:35:57.783 | INFO | PRE_HQ : 66.8%, 1.7, 59.8ms, 0 | 20260507 16:35:57.783 | INFO | BAZWRITER : 70.7%, 3.6, 131.0ms, 0 | 20260507 16:35:57.783 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 16:35:57.783 | INFO | Uploaded 36GiB to GPU in 2.55511s (14.0894GiB/s) | 20260507 16:35:57.783 | INFO | Downloaded 38GiB to GPU in 2.71494s (13.9966GiB/s) | 20260507 16:36:02.921 | INFO | Analyzing chunk frames = [979456,979968) | 20260507 16:36:02.921 | INFO | MemInfo: { | 20260507 16:36:02.921 | INFO | "gpu" : " 17 GB", | 20260507 16:36:02.922 | INFO | "huge" : "", | 20260507 16:36:02.922 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:02.922 | INFO | "malloc" : " 14 GB", | 20260507 16:36:02.922 | INFO | "pinned" : " 29 GB" | 20260507 16:36:02.922 | INFO | } | 20260507 16:36:02.922 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:36:02.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:02.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.922 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 16:36:02.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.922 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 16:36:02.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:02.923 | INFO | Uploaded 36GiB to GPU in 2.56489s (14.0357GiB/s) | 20260507 16:36:02.923 | INFO | Downloaded 34GiB to GPU in 2.42241s (14.0356GiB/s) | 20260507 16:36:08.061 | INFO | Analyzing chunk frames = [979968,980480) | 20260507 16:36:08.061 | INFO | MemInfo: { | 20260507 16:36:08.061 | INFO | "gpu" : " 17 GB", | 20260507 16:36:08.061 | INFO | "huge" : "", | 20260507 16:36:08.061 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:08.061 | INFO | "malloc" : " 14 GB", | 20260507 16:36:08.061 | INFO | "pinned" : " 29 GB" | 20260507 16:36:08.061 | INFO | } | 20260507 16:36:08.061 | INFO | Perf report for the last 5.13913 seconds: | 20260507 16:36:08.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:08.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.061 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 16:36:08.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.061 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 16:36:08.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:08.062 | INFO | Uploaded 36GiB to GPU in 2.57807s (13.9639GiB/s) | 20260507 16:36:08.062 | INFO | Downloaded 34GiB to GPU in 2.41156s (14.0988GiB/s) | 20260507 16:36:13.200 | INFO | Analyzing chunk frames = [980480,980992) | 20260507 16:36:13.200 | INFO | MemInfo: { | 20260507 16:36:13.200 | INFO | "gpu" : " 17 GB", | 20260507 16:36:13.200 | INFO | "huge" : "", | 20260507 16:36:13.200 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:13.201 | INFO | "malloc" : " 14 GB", | 20260507 16:36:13.201 | INFO | "pinned" : " 29 GB" | 20260507 16:36:13.201 | INFO | } | 20260507 16:36:13.201 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:36:13.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:13.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.201 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 16:36:13.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.201 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 16:36:13.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:13.202 | INFO | Uploaded 36GiB to GPU in 2.50612s (14.3648GiB/s) | 20260507 16:36:13.202 | INFO | Downloaded 34GiB to GPU in 2.33252s (14.5765GiB/s) | 20260507 16:36:18.340 | INFO | Analyzing chunk frames = [980992,981504) | 20260507 16:36:18.340 | INFO | MemInfo: { | 20260507 16:36:18.340 | INFO | "gpu" : " 17 GB", | 20260507 16:36:18.340 | INFO | "huge" : "", | 20260507 16:36:18.340 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:18.340 | INFO | "malloc" : " 14 GB", | 20260507 16:36:18.340 | INFO | "pinned" : " 29 GB" | 20260507 16:36:18.340 | INFO | } | 20260507 16:36:18.340 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:36:18.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:18.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 16:36:18.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | PRE_HQ : 38.2%, 1.0, 19.9ms, 0 | 20260507 16:36:18.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:18.340 | INFO | Uploaded 36GiB to GPU in 2.35149s (15.3094GiB/s) | 20260507 16:36:18.341 | INFO | Downloaded 34GiB to GPU in 2.47698s (13.7264GiB/s) | 20260507 16:36:23.479 | INFO | Analyzing chunk frames = [981504,982016) | 20260507 16:36:23.479 | INFO | MemInfo: { | 20260507 16:36:23.479 | INFO | "gpu" : " 17 GB", | 20260507 16:36:23.479 | INFO | "huge" : "", | 20260507 16:36:23.479 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:23.479 | INFO | "malloc" : " 14 GB", | 20260507 16:36:23.479 | INFO | "pinned" : " 29 GB" | 20260507 16:36:23.479 | INFO | } | 20260507 16:36:23.479 | INFO | Perf report for the last 5.13896 seconds: | 20260507 16:36:23.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:23.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.479 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:36:23.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.479 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 16:36:23.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:23.480 | INFO | Uploaded 36GiB to GPU in 2.35936s (15.2584GiB/s) | 20260507 16:36:23.480 | INFO | Downloaded 34GiB to GPU in 2.47708s (13.7259GiB/s) | 20260507 16:36:28.618 | INFO | Analyzing chunk frames = [982016,982528) | 20260507 16:36:28.618 | INFO | MemInfo: { | 20260507 16:36:28.618 | INFO | "gpu" : " 17 GB", | 20260507 16:36:28.618 | INFO | "huge" : "", | 20260507 16:36:28.618 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:28.618 | INFO | "malloc" : " 14 GB", | 20260507 16:36:28.618 | INFO | "pinned" : " 29 GB" | 20260507 16:36:28.618 | INFO | } | 20260507 16:36:28.618 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:36:28.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:28.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.618 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:36:28.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.618 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:36:28.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:28.619 | INFO | Uploaded 36GiB to GPU in 2.42522s (14.844GiB/s) | 20260507 16:36:28.619 | INFO | Downloaded 34GiB to GPU in 2.40643s (14.1288GiB/s) | 20260507 16:36:33.758 | INFO | Analyzing chunk frames = [982528,983040) | 20260507 16:36:33.758 | INFO | MemInfo: { | 20260507 16:36:33.758 | INFO | "gpu" : " 17 GB", | 20260507 16:36:33.758 | INFO | "huge" : "", | 20260507 16:36:33.758 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:33.758 | INFO | "malloc" : " 14 GB", | 20260507 16:36:33.758 | INFO | "pinned" : " 29 GB" | 20260507 16:36:33.758 | INFO | } | 20260507 16:36:33.758 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:36:33.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:33.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:36:33.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:36:33.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:33.758 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:36:33.758 | INFO | Total mem usage: 16.4 GiB | 20260507 16:36:33.758 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:36:33.758 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.46%) | 20260507 16:36:33.758 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:36:33.758 | INFO | High Water Breakdown: | 20260507 16:36:33.758 | INFO | 59.9% : 9.14 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:36:33.758 | INFO | 40% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:36:33.758 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:36:33.758 | INFO | ------------------------------------------ | 20260507 16:36:33.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:36:33.758 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:36:33.758 | INFO | Total mem usage: 26.5 GiB | 20260507 16:36:33.758 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:36:33.758 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:36:33.758 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 16:36:33.758 | INFO | High Water Breakdown: | 20260507 16:36:33.758 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:36:33.758 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:36:33.758 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:36:33.758 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:36:33.758 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:36:33.758 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:36:33.758 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:36:33.758 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:36:33.758 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:36:33.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:36:33.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:36:33.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:36:33.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:36:33.758 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:36:33.758 | INFO | ------------------------------------------ | 20260507 16:36:33.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:36:33.758 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:36:33.758 | INFO | Total mem usage: 27.8 GiB | 20260507 16:36:33.758 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:36:33.758 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:36:33.758 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:36:33.759 | INFO | High Water Breakdown: | 20260507 16:36:33.759 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:36:33.759 | INFO | ------------------------------------------ | 20260507 16:36:33.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:36:33.759 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:36:33.759 | INFO | Total mem usage: 15.7 GiB | 20260507 16:36:33.759 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:36:33.759 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:36:33.759 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:36:33.759 | INFO | High Water Breakdown: | 20260507 16:36:33.759 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:36:33.759 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:36:33.759 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:36:33.759 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:36:33.759 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:36:33.759 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:36:33.759 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:36:33.759 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:36:33.759 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:36:33.759 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:36:33.759 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:36:33.759 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:36:33.759 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:36:33.759 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:36:33.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:36:33.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:36:33.759 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:36:33.759 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:36:33.759 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:36:33.759 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:36:33.759 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:36:33.759 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:36:33.759 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:36:33.759 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:36:33.759 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:36:33.759 | INFO | ------------------------------------------ | 20260507 16:36:33.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:36:33.759 | INFO | Uploaded 36GiB to GPU in 2.44951s (14.6968GiB/s) | 20260507 16:36:33.759 | INFO | Downloaded 34GiB to GPU in 2.51157s (13.5373GiB/s) | 20260507 16:36:33.759 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:36:33.760 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:36:33.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:36:33.760 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.66e+01, 1.43e+01 / 2.83e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 0.30%, AggregateStats -- 1.02e-01 / 1.07e-01, 1.18e-02 / 7.24e-03 | 20260507 16:36:33.760 | INFO | PerfReport: 12.12%, Baseline -- 4.07e+00 / 4.56e+00, 6.30e-01 / 1.94e-01 | 20260507 16:36:33.760 | INFO | PerfReport: 3.44%, Histogram -- 1.16e+00 / 1.51e+00, 2.17e-01 / 1.07e-01 | 20260507 16:36:33.760 | INFO | PerfReport: 15.71%, DME -- 4.22e+01 / 4.50e+01, 4.32e+00 / 2.79e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 55.66%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.80e+00 / 2.36e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.57e+00 / 2.94e+00, 4.37e-01 / 2.20e-01 | 20260507 16:36:33.760 | INFO | PerfReport: 3.88%, Metrics -- 1.30e+00 / 1.48e+00, 3.27e-01 / 2.04e-01 | 20260507 16:36:33.760 | INFO | PerfReport: 1.24%, PulseSerializing -- 4.18e-01 / 4.42e-01, 6.87e-02 / 4.27e-02 | 20260507 16:36:33.760 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:36:33.760 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:36:33.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:36:33.760 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 23.85%, Upload -- 2.56e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 24.00%, Download -- 2.58e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 31.29%, Compute -- 3.36e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 1.81%, LockUpload -- 1.94e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 2.28%, LockDownload -- 2.45e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 16.75%, LockCompute -- 1.80e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:36:33.760 | INFO | PerfReport: 0.02%, Misc -- 1.81e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260507 16:36:33.857 | INFO | Starting a round of Baz IO | 20260507 16:36:37.423 | INFO | Populating RTMetrics report | 20260507 16:36:37.423 | INFO | RTMetrics report populated | 20260507 16:36:37.423 | INFO | Opening tmp RTMetrics json file | 20260507 16:36:37.424 | INFO | Serializing RTMetrics to json | 20260507 16:36:37.426 | INFO | Serialization done, swapping tmp file now... | 20260507 16:36:37.427 | INFO | RTMetrics done | 20260507 16:36:37.520 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.663 seconds | 20260507 16:36:37.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.87% / 0.03% / 50.10% | 20260507 16:36:37.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.34691% / 2.38419e-05% / 3.57628e-05% | 20260507 16:36:37.520 | INFO | Baz IO overall: | 20260507 16:36:37.520 | INFO | BytesWritten : 423372910576 | 20260507 16:36:37.520 | INFO | NumEvents : 204886440300 | 20260507 16:36:38.897 | INFO | Analyzing chunk frames = [983040,983552) | 20260507 16:36:38.897 | INFO | MemInfo: { | 20260507 16:36:38.897 | INFO | "gpu" : " 17 GB", | 20260507 16:36:38.897 | INFO | "huge" : "", | 20260507 16:36:38.897 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:38.897 | INFO | "malloc" : " 14 GB", | 20260507 16:36:38.897 | INFO | "pinned" : " 29 GB" | 20260507 16:36:38.897 | INFO | } | 20260507 16:36:38.897 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:36:38.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:38.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:38.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:38.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:38.897 | INFO | ANALYSIS : 69.7%, 3.0, 108.4ms, 0 | 20260507 16:36:38.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:38.897 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 16:36:38.897 | INFO | PRE_HQ : 67.8%, 1.9, 65.8ms, 0 | 20260507 16:36:38.897 | INFO | BAZWRITER : 70.7%, 3.7, 136.1ms, 0 | 20260507 16:36:38.897 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 16:36:38.898 | INFO | Uploaded 36GiB to GPU in 2.49723s (14.416GiB/s) | 20260507 16:36:38.898 | INFO | Downloaded 38GiB to GPU in 2.71204s (14.0116GiB/s) | 20260507 16:36:44.036 | INFO | Analyzing chunk frames = [983552,984064) | 20260507 16:36:44.036 | INFO | MemInfo: { | 20260507 16:36:44.036 | INFO | "gpu" : " 17 GB", | 20260507 16:36:44.036 | INFO | "huge" : "", | 20260507 16:36:44.036 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:44.036 | INFO | "malloc" : " 14 GB", | 20260507 16:36:44.036 | INFO | "pinned" : " 29 GB" | 20260507 16:36:44.036 | INFO | } | 20260507 16:36:44.036 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:36:44.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:44.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.037 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:36:44.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.037 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 16:36:44.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:44.037 | INFO | Uploaded 36GiB to GPU in 2.52846s (14.2379GiB/s) | 20260507 16:36:44.037 | INFO | Downloaded 34GiB to GPU in 2.4466s (13.8968GiB/s) | 20260507 16:36:49.176 | INFO | Analyzing chunk frames = [984064,984576) | 20260507 16:36:49.176 | INFO | MemInfo: { | 20260507 16:36:49.176 | INFO | "gpu" : " 17 GB", | 20260507 16:36:49.176 | INFO | "huge" : "", | 20260507 16:36:49.176 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:49.176 | INFO | "malloc" : " 14 GB", | 20260507 16:36:49.176 | INFO | "pinned" : " 29 GB" | 20260507 16:36:49.176 | INFO | } | 20260507 16:36:49.176 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:36:49.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:49.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260507 16:36:49.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:36:49.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:49.176 | INFO | Uploaded 36GiB to GPU in 2.48805s (14.4692GiB/s) | 20260507 16:36:49.176 | INFO | Downloaded 34GiB to GPU in 2.34253s (14.5142GiB/s) | 20260507 16:36:54.315 | INFO | Analyzing chunk frames = [984576,985088) | 20260507 16:36:54.315 | INFO | MemInfo: { | 20260507 16:36:54.315 | INFO | "gpu" : " 17 GB", | 20260507 16:36:54.315 | INFO | "huge" : "", | 20260507 16:36:54.315 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:54.315 | INFO | "malloc" : " 14 GB", | 20260507 16:36:54.315 | INFO | "pinned" : " 29 GB" | 20260507 16:36:54.315 | INFO | } | 20260507 16:36:54.315 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:36:54.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:54.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:36:54.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:36:54.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:54.315 | INFO | Uploaded 36GiB to GPU in 2.48946s (14.461GiB/s) | 20260507 16:36:54.315 | INFO | Downloaded 34GiB to GPU in 2.35411s (14.4428GiB/s) | 20260507 16:36:59.455 | INFO | Analyzing chunk frames = [985088,985600) | 20260507 16:36:59.455 | INFO | MemInfo: { | 20260507 16:36:59.455 | INFO | "gpu" : " 17 GB", | 20260507 16:36:59.455 | INFO | "huge" : "", | 20260507 16:36:59.455 | INFO | "huge_shared" : " 29 GB", | 20260507 16:36:59.455 | INFO | "malloc" : " 14 GB", | 20260507 16:36:59.455 | INFO | "pinned" : " 29 GB" | 20260507 16:36:59.455 | INFO | } | 20260507 16:36:59.455 | INFO | Perf report for the last 5.13973 seconds: | 20260507 16:36:59.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:36:59.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.455 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 16:36:59.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.455 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260507 16:36:59.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:36:59.456 | INFO | Uploaded 36GiB to GPU in 2.38312s (15.1063GiB/s) | 20260507 16:36:59.456 | INFO | Downloaded 34GiB to GPU in 2.45646s (13.841GiB/s) | 20260507 16:37:04.593 | INFO | Analyzing chunk frames = [985600,986112) | 20260507 16:37:04.593 | INFO | MemInfo: { | 20260507 16:37:04.593 | INFO | "gpu" : " 17 GB", | 20260507 16:37:04.593 | INFO | "huge" : "", | 20260507 16:37:04.593 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:04.594 | INFO | "malloc" : " 14 GB", | 20260507 16:37:04.594 | INFO | "pinned" : " 29 GB" | 20260507 16:37:04.594 | INFO | } | 20260507 16:37:04.594 | INFO | Perf report for the last 5.1389 seconds: | 20260507 16:37:04.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:04.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 16:37:04.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 16:37:04.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:04.594 | INFO | Uploaded 36GiB to GPU in 2.411s (14.9316GiB/s) | 20260507 16:37:04.594 | INFO | Downloaded 34GiB to GPU in 2.43639s (13.955GiB/s) | 20260507 16:37:09.733 | INFO | Analyzing chunk frames = [986112,986624) | 20260507 16:37:09.733 | INFO | MemInfo: { | 20260507 16:37:09.733 | INFO | "gpu" : " 17 GB", | 20260507 16:37:09.733 | INFO | "huge" : "", | 20260507 16:37:09.733 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:09.733 | INFO | "malloc" : " 14 GB", | 20260507 16:37:09.733 | INFO | "pinned" : " 29 GB" | 20260507 16:37:09.733 | INFO | } | 20260507 16:37:09.733 | INFO | Perf report for the last 5.13957 seconds: | 20260507 16:37:09.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:09.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.733 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:37:09.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.734 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 16:37:09.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:09.734 | INFO | Uploaded 36GiB to GPU in 2.49394s (14.435GiB/s) | 20260507 16:37:09.734 | INFO | Downloaded 34GiB to GPU in 2.42025s (14.0481GiB/s) | 20260507 16:37:14.872 | INFO | Analyzing chunk frames = [986624,987136) | 20260507 16:37:14.872 | INFO | MemInfo: { | 20260507 16:37:14.872 | INFO | "gpu" : " 17 GB", | 20260507 16:37:14.872 | INFO | "huge" : "", | 20260507 16:37:14.872 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:14.873 | INFO | "malloc" : " 14 GB", | 20260507 16:37:14.873 | INFO | "pinned" : " 29 GB" | 20260507 16:37:14.873 | INFO | } | 20260507 16:37:14.873 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:37:14.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:14.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 16:37:14.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | PRE_HQ : 38.0%, 1.0, 19.8ms, 0 | 20260507 16:37:14.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:14.873 | INFO | Uploaded 36GiB to GPU in 2.56392s (14.041GiB/s) | 20260507 16:37:14.873 | INFO | Downloaded 34GiB to GPU in 2.50422s (13.5771GiB/s) | 20260507 16:37:14.988 | INFO | Starting a round of Baz IO | 20260507 16:37:18.493 | INFO | Populating RTMetrics report | 20260507 16:37:18.494 | INFO | RTMetrics report populated | 20260507 16:37:18.494 | INFO | Opening tmp RTMetrics json file | 20260507 16:37:18.506 | INFO | Serializing RTMetrics to json | 20260507 16:37:18.555 | INFO | Serialization done, swapping tmp file now... | 20260507 16:37:18.584 | INFO | RTMetrics done | 20260507 16:37:18.620 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.632 seconds | 20260507 16:37:18.620 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.85% / 0.03% / 50.13% | 20260507 16:37:18.620 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.34059% / 5.96046e-05% / 3.17891e-05% | 20260507 16:37:18.620 | INFO | Baz IO overall: | 20260507 16:37:18.620 | INFO | BytesWritten : 425294380934 | 20260507 16:37:18.620 | INFO | NumEvents : 205821477040 | 20260507 16:37:20.012 | INFO | Analyzing chunk frames = [987136,987648) | 20260507 16:37:20.012 | INFO | MemInfo: { | 20260507 16:37:20.012 | INFO | "gpu" : " 17 GB", | 20260507 16:37:20.012 | INFO | "huge" : "", | 20260507 16:37:20.012 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:20.012 | INFO | "malloc" : " 14 GB", | 20260507 16:37:20.012 | INFO | "pinned" : " 29 GB" | 20260507 16:37:20.012 | INFO | } | 20260507 16:37:20.012 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:37:20.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:20.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:20.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:20.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:20.012 | INFO | ANALYSIS : 70.0%, 2.9, 105.7ms, 0 | 20260507 16:37:20.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:20.012 | INFO | RT_METRICS : 51.1%, 1.0, 26.8ms, 0 | 20260507 16:37:20.012 | INFO | PRE_HQ : 69.2%, 2.0, 71.1ms, 0 | 20260507 16:37:20.012 | INFO | BAZWRITER : 70.2%, 4.2, 152.6ms, 0 | 20260507 16:37:20.012 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 16:37:20.013 | INFO | Uploaded 36GiB to GPU in 2.57614s (13.9744GiB/s) | 20260507 16:37:20.013 | INFO | Downloaded 38GiB to GPU in 2.69588s (14.0956GiB/s) | 20260507 16:37:25.151 | INFO | Analyzing chunk frames = [987648,988160) | 20260507 16:37:25.151 | INFO | MemInfo: { | 20260507 16:37:25.151 | INFO | "gpu" : " 17 GB", | 20260507 16:37:25.151 | INFO | "huge" : "", | 20260507 16:37:25.151 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:25.151 | INFO | "malloc" : " 14 GB", | 20260507 16:37:25.151 | INFO | "pinned" : " 29 GB" | 20260507 16:37:25.151 | INFO | } | 20260507 16:37:25.151 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:37:25.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:25.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.151 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260507 16:37:25.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.152 | INFO | PRE_HQ : 36.9%, 1.0, 19.1ms, 0 | 20260507 16:37:25.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:25.152 | INFO | Uploaded 36GiB to GPU in 2.52703s (14.246GiB/s) | 20260507 16:37:25.152 | INFO | Downloaded 34GiB to GPU in 2.41276s (14.0917GiB/s) | 20260507 16:37:30.290 | INFO | Analyzing chunk frames = [988160,988672) | 20260507 16:37:30.290 | INFO | MemInfo: { | 20260507 16:37:30.290 | INFO | "gpu" : " 17 GB", | 20260507 16:37:30.290 | INFO | "huge" : "", | 20260507 16:37:30.291 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:30.291 | INFO | "malloc" : " 14 GB", | 20260507 16:37:30.291 | INFO | "pinned" : " 29 GB" | 20260507 16:37:30.291 | INFO | } | 20260507 16:37:30.291 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:37:30.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:30.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260507 16:37:30.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 16:37:30.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:30.291 | INFO | Uploaded 36GiB to GPU in 2.57335s (13.9896GiB/s) | 20260507 16:37:30.291 | INFO | Downloaded 34GiB to GPU in 2.4266s (14.0114GiB/s) | 20260507 16:37:35.430 | INFO | Analyzing chunk frames = [988672,989184) | 20260507 16:37:35.430 | INFO | MemInfo: { | 20260507 16:37:35.430 | INFO | "gpu" : " 17 GB", | 20260507 16:37:35.430 | INFO | "huge" : "", | 20260507 16:37:35.430 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:35.430 | INFO | "malloc" : " 14 GB", | 20260507 16:37:35.430 | INFO | "pinned" : " 29 GB" | 20260507 16:37:35.430 | INFO | } | 20260507 16:37:35.430 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:37:35.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:35.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.430 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 16:37:35.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.430 | INFO | PRE_HQ : 39.0%, 1.0, 20.3ms, 0 | 20260507 16:37:35.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:35.431 | INFO | Uploaded 36GiB to GPU in 2.49951s (14.4028GiB/s) | 20260507 16:37:35.431 | INFO | Downloaded 34GiB to GPU in 2.33754s (14.5452GiB/s) | 20260507 16:37:40.569 | INFO | Analyzing chunk frames = [989184,989696) | 20260507 16:37:40.569 | INFO | MemInfo: { | 20260507 16:37:40.569 | INFO | "gpu" : " 17 GB", | 20260507 16:37:40.569 | INFO | "huge" : "", | 20260507 16:37:40.569 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:40.569 | INFO | "malloc" : " 14 GB", | 20260507 16:37:40.569 | INFO | "pinned" : " 29 GB" | 20260507 16:37:40.569 | INFO | } | 20260507 16:37:40.569 | INFO | Perf report for the last 5.13894 seconds: | 20260507 16:37:40.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:40.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260507 16:37:40.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 16:37:40.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:40.569 | INFO | Uploaded 36GiB to GPU in 2.42094s (14.8703GiB/s) | 20260507 16:37:40.569 | INFO | Downloaded 34GiB to GPU in 2.48729s (13.6695GiB/s) | 20260507 16:37:45.708 | INFO | Analyzing chunk frames = [989696,990208) | 20260507 16:37:45.708 | INFO | MemInfo: { | 20260507 16:37:45.708 | INFO | "gpu" : " 17 GB", | 20260507 16:37:45.708 | INFO | "huge" : "", | 20260507 16:37:45.708 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:45.708 | INFO | "malloc" : " 14 GB", | 20260507 16:37:45.708 | INFO | "pinned" : " 29 GB" | 20260507 16:37:45.709 | INFO | } | 20260507 16:37:45.709 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:37:45.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:45.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 16:37:45.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 16:37:45.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:45.709 | INFO | Uploaded 36GiB to GPU in 2.39809s (15.0119GiB/s) | 20260507 16:37:45.709 | INFO | Downloaded 34GiB to GPU in 2.44536s (13.9039GiB/s) | 20260507 16:37:50.848 | INFO | Analyzing chunk frames = [990208,990720) | 20260507 16:37:50.848 | INFO | MemInfo: { | 20260507 16:37:50.848 | INFO | "gpu" : " 17 GB", | 20260507 16:37:50.848 | INFO | "huge" : "", | 20260507 16:37:50.848 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:50.848 | INFO | "malloc" : " 14 GB", | 20260507 16:37:50.848 | INFO | "pinned" : " 29 GB" | 20260507 16:37:50.848 | INFO | } | 20260507 16:37:50.848 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:37:50.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:50.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.849 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:37:50.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.849 | INFO | PRE_HQ : 39.3%, 1.0, 20.4ms, 0 | 20260507 16:37:50.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:50.849 | INFO | Uploaded 36GiB to GPU in 2.48464s (14.489GiB/s) | 20260507 16:37:50.849 | INFO | Downloaded 34GiB to GPU in 2.41438s (14.0823GiB/s) | 20260507 16:37:55.987 | INFO | Analyzing chunk frames = [990720,991232) | 20260507 16:37:55.987 | INFO | MemInfo: { | 20260507 16:37:55.987 | INFO | "gpu" : " 17 GB", | 20260507 16:37:55.987 | INFO | "huge" : "", | 20260507 16:37:55.987 | INFO | "huge_shared" : " 29 GB", | 20260507 16:37:55.987 | INFO | "malloc" : " 14 GB", | 20260507 16:37:55.987 | INFO | "pinned" : " 29 GB" | 20260507 16:37:55.987 | INFO | } | 20260507 16:37:55.987 | INFO | Perf report for the last 5.13907 seconds: | 20260507 16:37:55.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:37:55.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.988 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:37:55.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.988 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 16:37:55.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:37:55.988 | INFO | Uploaded 36GiB to GPU in 2.4767s (14.5355GiB/s) | 20260507 16:37:55.988 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:37:55.988 | INFO | Total mem usage: 16.4 GiB | 20260507 16:37:55.988 | INFO | Downloaded 34GiB to GPU in 2.40445s (14.1404GiB/s) | 20260507 16:37:55.988 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:37:55.989 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.7%) | 20260507 16:37:55.989 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:37:55.989 | INFO | High Water Breakdown: | 20260507 16:37:55.989 | INFO | 59.4% : 8.8 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:37:55.989 | INFO | 40.5% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:37:55.989 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:37:55.989 | INFO | ------------------------------------------ | 20260507 16:37:55.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:37:55.989 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:37:55.989 | INFO | Total mem usage: 26.5 GiB | 20260507 16:37:55.989 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:37:55.989 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:37:55.989 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 16:37:55.989 | INFO | High Water Breakdown: | 20260507 16:37:55.989 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:37:55.989 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:37:55.989 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:37:55.989 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:37:55.989 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:37:55.989 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:37:55.989 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:37:55.989 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:37:55.989 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:37:55.989 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:37:55.989 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:37:55.989 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:37:55.989 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:37:55.989 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:37:55.989 | INFO | ------------------------------------------ | 20260507 16:37:55.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:37:55.989 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:37:55.989 | INFO | Total mem usage: 27.8 GiB | 20260507 16:37:55.989 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:37:55.989 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:37:55.989 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:37:55.989 | INFO | High Water Breakdown: | 20260507 16:37:55.989 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:37:55.989 | INFO | ------------------------------------------ | 20260507 16:37:55.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:37:55.989 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:37:55.989 | INFO | Total mem usage: 15.7 GiB | 20260507 16:37:55.989 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:37:55.989 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:37:55.989 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:37:55.989 | INFO | High Water Breakdown: | 20260507 16:37:55.989 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:37:55.989 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:37:55.989 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:37:55.989 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:37:55.989 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:37:55.989 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:37:55.989 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:37:55.989 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:37:55.989 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:37:55.989 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:37:55.989 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:37:55.989 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:37:55.989 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:37:55.990 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:37:55.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:37:55.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:37:55.990 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:37:55.990 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:37:55.990 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:37:55.990 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:37:55.990 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:37:55.990 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:37:55.990 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:37:55.990 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:37:55.990 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:37:55.990 | INFO | ------------------------------------------ | 20260507 16:37:55.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:37:55.990 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:37:55.990 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:37:55.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:37:55.990 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.63e+01, 1.45e+01 / 2.88e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 0.31%, AggregateStats -- 1.03e-01 / 1.09e-01, 1.24e-02 / 7.97e-03 | 20260507 16:37:55.990 | INFO | PerfReport: 12.09%, Baseline -- 4.07e+00 / 4.49e+00, 6.28e-01 / 2.87e-01 | 20260507 16:37:55.990 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.49e+00, 2.18e-01 / 1.19e-01 | 20260507 16:37:55.990 | INFO | PerfReport: 15.64%, DME -- 4.22e+01 / 4.49e+01, 4.32e+00 / 2.49e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.92e+00 / 2.04e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.55e+00 / 2.93e+00, 4.38e-01 / 2.46e-01 | 20260507 16:37:55.990 | INFO | PerfReport: 3.86%, Metrics -- 1.30e+00 / 1.48e+00, 3.34e-01 / 2.06e-01 | 20260507 16:37:55.990 | INFO | PerfReport: 1.24%, PulseSerializing -- 4.19e-01 / 4.46e-01, 6.87e-02 / 4.47e-02 | 20260507 16:37:55.990 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:37:55.990 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:37:55.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:37:55.990 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 24.09%, Upload -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 23.78%, Download -- 2.56e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 31.34%, Compute -- 3.37e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 2.00%, LockUpload -- 2.15e+00 / 0.00e+00, 5.12e+00 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 1.97%, LockDownload -- 2.12e+00 / 0.00e+00, 3.49e+00 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 16.81%, LockCompute -- 1.81e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 16:37:55.990 | INFO | PerfReport: 0.02%, Misc -- 1.98e-02 / 0.00e+00, 4.06e-02 / 0.00e+00 | 20260507 16:37:56.099 | INFO | Starting a round of Baz IO | 20260507 16:37:59.602 | INFO | Populating RTMetrics report | 20260507 16:37:59.603 | INFO | RTMetrics report populated | 20260507 16:37:59.603 | INFO | Opening tmp RTMetrics json file | 20260507 16:37:59.710 | INFO | Serializing RTMetrics to json | 20260507 16:37:59.721 | INFO | Serialization done, swapping tmp file now... | 20260507 16:37:59.723 | INFO | RTMetrics done | 20260507 16:37:59.745 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.645 seconds | 20260507 16:37:59.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.82% / 0.03% / 50.15% | 20260507 16:37:59.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.30917% / 5.96046e-05% / 5.5631e-05% | 20260507 16:37:59.745 | INFO | Baz IO overall: | 20260507 16:37:59.745 | INFO | BytesWritten : 427213336393 | 20260507 16:37:59.745 | INFO | NumEvents : 206754757093 | 20260507 16:38:01.126 | INFO | Analyzing chunk frames = [991232,991744) | 20260507 16:38:01.126 | INFO | MemInfo: { | 20260507 16:38:01.126 | INFO | "gpu" : " 17 GB", | 20260507 16:38:01.126 | INFO | "huge" : "", | 20260507 16:38:01.126 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:01.126 | INFO | "malloc" : " 14 GB", | 20260507 16:38:01.126 | INFO | "pinned" : " 29 GB" | 20260507 16:38:01.126 | INFO | } | 20260507 16:38:01.126 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:38:01.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:01.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:01.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:01.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:01.126 | INFO | ANALYSIS : 69.7%, 2.9, 106.3ms, 0 | 20260507 16:38:01.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:01.126 | INFO | RT_METRICS : 48.9%, 1.0, 25.6ms, 0 | 20260507 16:38:01.126 | INFO | PRE_HQ : 68.2%, 1.9, 67.3ms, 0 | 20260507 16:38:01.126 | INFO | BAZWRITER : 70.5%, 4.1, 147.4ms, 0 | 20260507 16:38:01.126 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260507 16:38:01.127 | INFO | Uploaded 36GiB to GPU in 2.56994s (14.0081GiB/s) | 20260507 16:38:01.127 | INFO | Downloaded 38GiB to GPU in 2.70345s (14.0561GiB/s) | 20260507 16:38:06.266 | INFO | Analyzing chunk frames = [991744,992256) | 20260507 16:38:06.266 | INFO | MemInfo: { | 20260507 16:38:06.266 | INFO | "gpu" : " 17 GB", | 20260507 16:38:06.266 | INFO | "huge" : "", | 20260507 16:38:06.266 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:06.266 | INFO | "malloc" : " 14 GB", | 20260507 16:38:06.266 | INFO | "pinned" : " 29 GB" | 20260507 16:38:06.266 | INFO | } | 20260507 16:38:06.266 | INFO | Perf report for the last 5.13981 seconds: | 20260507 16:38:06.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:06.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:38:06.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | PRE_HQ : 38.2%, 1.0, 19.9ms, 0 | 20260507 16:38:06.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:06.266 | INFO | Uploaded 36GiB to GPU in 2.53522s (14.2GiB/s) | 20260507 16:38:06.267 | INFO | Downloaded 34GiB to GPU in 2.44898s (13.8833GiB/s) | 20260507 16:38:11.405 | INFO | Analyzing chunk frames = [992256,992768) | 20260507 16:38:11.405 | INFO | MemInfo: { | 20260507 16:38:11.405 | INFO | "gpu" : " 17 GB", | 20260507 16:38:11.405 | INFO | "huge" : "", | 20260507 16:38:11.405 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:11.405 | INFO | "malloc" : " 14 GB", | 20260507 16:38:11.405 | INFO | "pinned" : " 29 GB" | 20260507 16:38:11.405 | INFO | } | 20260507 16:38:11.405 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:38:11.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:11.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 16:38:11.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | PRE_HQ : 38.3%, 1.0, 19.9ms, 0 | 20260507 16:38:11.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:11.406 | INFO | Uploaded 36GiB to GPU in 2.55366s (14.0974GiB/s) | 20260507 16:38:11.406 | INFO | Downloaded 34GiB to GPU in 2.4331s (13.9739GiB/s) | 20260507 16:38:16.544 | INFO | Analyzing chunk frames = [992768,993280) | 20260507 16:38:16.544 | INFO | MemInfo: { | 20260507 16:38:16.544 | INFO | "gpu" : " 17 GB", | 20260507 16:38:16.544 | INFO | "huge" : "", | 20260507 16:38:16.545 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:16.545 | INFO | "malloc" : " 14 GB", | 20260507 16:38:16.545 | INFO | "pinned" : " 29 GB" | 20260507 16:38:16.545 | INFO | } | 20260507 16:38:16.545 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:38:16.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:16.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.545 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:38:16.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.545 | INFO | PRE_HQ : 38.7%, 1.0, 20.1ms, 0 | 20260507 16:38:16.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:16.546 | INFO | Uploaded 36GiB to GPU in 2.48215s (14.5036GiB/s) | 20260507 16:38:16.546 | INFO | Downloaded 34GiB to GPU in 2.35043s (14.4654GiB/s) | 20260507 16:38:21.684 | INFO | Analyzing chunk frames = [993280,993792) | 20260507 16:38:21.684 | INFO | MemInfo: { | 20260507 16:38:21.684 | INFO | "gpu" : " 17 GB", | 20260507 16:38:21.684 | INFO | "huge" : "", | 20260507 16:38:21.684 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:21.684 | INFO | "malloc" : " 14 GB", | 20260507 16:38:21.684 | INFO | "pinned" : " 29 GB" | 20260507 16:38:21.684 | INFO | } | 20260507 16:38:21.685 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:38:21.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:21.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:38:21.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 16:38:21.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:21.685 | INFO | Uploaded 36GiB to GPU in 2.33607s (15.4105GiB/s) | 20260507 16:38:21.685 | INFO | Downloaded 34GiB to GPU in 2.47635s (13.7299GiB/s) | 20260507 16:38:26.823 | INFO | Analyzing chunk frames = [993792,994304) | 20260507 16:38:26.823 | INFO | MemInfo: { | 20260507 16:38:26.823 | INFO | "gpu" : " 17 GB", | 20260507 16:38:26.823 | INFO | "huge" : "", | 20260507 16:38:26.823 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:26.824 | INFO | "malloc" : " 14 GB", | 20260507 16:38:26.824 | INFO | "pinned" : " 29 GB" | 20260507 16:38:26.824 | INFO | } | 20260507 16:38:26.824 | INFO | Perf report for the last 5.13908 seconds: | 20260507 16:38:26.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:26.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:38:26.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260507 16:38:26.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:26.824 | INFO | Uploaded 36GiB to GPU in 2.31924s (15.5223GiB/s) | 20260507 16:38:26.824 | INFO | Downloaded 34GiB to GPU in 2.48331s (13.6914GiB/s) | 20260507 16:38:31.963 | INFO | Analyzing chunk frames = [994304,994816) | 20260507 16:38:31.963 | INFO | MemInfo: { | 20260507 16:38:31.963 | INFO | "gpu" : " 17 GB", | 20260507 16:38:31.963 | INFO | "huge" : "", | 20260507 16:38:31.963 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:31.963 | INFO | "malloc" : " 14 GB", | 20260507 16:38:31.963 | INFO | "pinned" : " 29 GB" | 20260507 16:38:31.963 | INFO | } | 20260507 16:38:31.963 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:38:31.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:31.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:38:31.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 16:38:31.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:31.963 | INFO | Uploaded 36GiB to GPU in 2.41712s (14.8937GiB/s) | 20260507 16:38:31.963 | INFO | Downloaded 34GiB to GPU in 2.41591s (14.0734GiB/s) | 20260507 16:38:37.102 | INFO | Analyzing chunk frames = [994816,995328) | 20260507 16:38:37.102 | INFO | MemInfo: { | 20260507 16:38:37.102 | INFO | "gpu" : " 17 GB", | 20260507 16:38:37.102 | INFO | "huge" : "", | 20260507 16:38:37.102 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:37.102 | INFO | "malloc" : " 14 GB", | 20260507 16:38:37.102 | INFO | "pinned" : " 29 GB" | 20260507 16:38:37.102 | INFO | } | 20260507 16:38:37.102 | INFO | Perf report for the last 5.13877 seconds: | 20260507 16:38:37.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:37.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 16:38:37.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:38:37.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:37.102 | INFO | Uploaded 36GiB to GPU in 2.50682s (14.3608GiB/s) | 20260507 16:38:37.102 | INFO | Downloaded 34GiB to GPU in 2.52003s (13.4919GiB/s) | 20260507 16:38:37.199 | INFO | Starting a round of Baz IO | 20260507 16:38:40.710 | INFO | Populating RTMetrics report | 20260507 16:38:40.711 | INFO | RTMetrics report populated | 20260507 16:38:40.711 | INFO | Opening tmp RTMetrics json file | 20260507 16:38:40.728 | INFO | Serializing RTMetrics to json | 20260507 16:38:40.747 | INFO | Serialization done, swapping tmp file now... | 20260507 16:38:40.751 | INFO | RTMetrics done | 20260507 16:38:40.846 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.647 seconds | 20260507 16:38:40.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.79% / 0.03% / 50.18% | 20260507 16:38:40.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.30456% / 3.17891e-05% / 2.38419e-05% | 20260507 16:38:40.846 | INFO | Baz IO overall: | 20260507 16:38:40.846 | INFO | BytesWritten : 429133314388 | 20260507 16:38:40.846 | INFO | NumEvents : 207689000788 | 20260507 16:38:42.241 | INFO | Analyzing chunk frames = [995328,995840) | 20260507 16:38:42.241 | INFO | MemInfo: { | 20260507 16:38:42.241 | INFO | "gpu" : " 17 GB", | 20260507 16:38:42.241 | INFO | "huge" : "", | 20260507 16:38:42.241 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:42.241 | INFO | "malloc" : " 14 GB", | 20260507 16:38:42.241 | INFO | "pinned" : " 29 GB" | 20260507 16:38:42.241 | INFO | } | 20260507 16:38:42.242 | INFO | Perf report for the last 5.13975 seconds: | 20260507 16:38:42.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:42.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:42.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:42.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:42.242 | INFO | ANALYSIS : 69.7%, 2.9, 107.4ms, 0 | 20260507 16:38:42.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:42.242 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 16:38:42.242 | INFO | PRE_HQ : 67.0%, 1.8, 61.1ms, 0 | 20260507 16:38:42.242 | INFO | BAZWRITER : 70.3%, 4.1, 146.6ms, 0 | 20260507 16:38:42.242 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 16:38:42.242 | INFO | Uploaded 36GiB to GPU in 2.56455s (14.0375GiB/s) | 20260507 16:38:42.242 | INFO | Downloaded 38GiB to GPU in 2.69871s (14.0808GiB/s) | 20260507 16:38:47.380 | INFO | Analyzing chunk frames = [995840,996352) | 20260507 16:38:47.380 | INFO | MemInfo: { | 20260507 16:38:47.380 | INFO | "gpu" : " 17 GB", | 20260507 16:38:47.381 | INFO | "huge" : "", | 20260507 16:38:47.381 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:47.381 | INFO | "malloc" : " 14 GB", | 20260507 16:38:47.381 | INFO | "pinned" : " 29 GB" | 20260507 16:38:47.381 | INFO | } | 20260507 16:38:47.381 | INFO | Perf report for the last 5.13939 seconds: | 20260507 16:38:47.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:47.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:38:47.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 16:38:47.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:47.381 | INFO | Uploaded 36GiB to GPU in 2.52677s (14.2474GiB/s) | 20260507 16:38:47.382 | INFO | Downloaded 34GiB to GPU in 2.43807s (13.9454GiB/s) | 20260507 16:38:52.520 | INFO | Analyzing chunk frames = [996352,996864) | 20260507 16:38:52.520 | INFO | MemInfo: { | 20260507 16:38:52.520 | INFO | "gpu" : " 17 GB", | 20260507 16:38:52.520 | INFO | "huge" : "", | 20260507 16:38:52.520 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:52.520 | INFO | "malloc" : " 14 GB", | 20260507 16:38:52.520 | INFO | "pinned" : " 29 GB" | 20260507 16:38:52.520 | INFO | } | 20260507 16:38:52.520 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:38:52.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:52.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.520 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:38:52.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.520 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 16:38:52.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:52.521 | INFO | Uploaded 36GiB to GPU in 2.55471s (14.0916GiB/s) | 20260507 16:38:52.521 | INFO | Downloaded 34GiB to GPU in 2.42903s (13.9974GiB/s) | 20260507 16:38:57.659 | INFO | Analyzing chunk frames = [996864,997376) | 20260507 16:38:57.659 | INFO | MemInfo: { | 20260507 16:38:57.659 | INFO | "gpu" : " 17 GB", | 20260507 16:38:57.659 | INFO | "huge" : "", | 20260507 16:38:57.659 | INFO | "huge_shared" : " 29 GB", | 20260507 16:38:57.659 | INFO | "malloc" : " 14 GB", | 20260507 16:38:57.659 | INFO | "pinned" : " 29 GB" | 20260507 16:38:57.659 | INFO | } | 20260507 16:38:57.659 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:38:57.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:38:57.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.659 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:38:57.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.660 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 16:38:57.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:38:57.660 | INFO | Uploaded 36GiB to GPU in 2.48015s (14.5152GiB/s) | 20260507 16:38:57.660 | INFO | Downloaded 34GiB to GPU in 2.35178s (14.4572GiB/s) | 20260507 16:39:02.799 | INFO | Analyzing chunk frames = [997376,997888) | 20260507 16:39:02.799 | INFO | MemInfo: { | 20260507 16:39:02.799 | INFO | "gpu" : " 17 GB", | 20260507 16:39:02.799 | INFO | "huge" : "", | 20260507 16:39:02.799 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:02.799 | INFO | "malloc" : " 14 GB", | 20260507 16:39:02.799 | INFO | "pinned" : " 29 GB" | 20260507 16:39:02.799 | INFO | } | 20260507 16:39:02.800 | INFO | Perf report for the last 5.14011 seconds: | 20260507 16:39:02.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:02.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 16:39:02.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 16:39:02.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:02.800 | INFO | Uploaded 36GiB to GPU in 2.28685s (15.7422GiB/s) | 20260507 16:39:02.800 | INFO | Downloaded 34GiB to GPU in 2.4799s (13.7103GiB/s) | 20260507 16:39:07.938 | INFO | Analyzing chunk frames = [997888,998400) | 20260507 16:39:07.938 | INFO | MemInfo: { | 20260507 16:39:07.938 | INFO | "gpu" : " 17 GB", | 20260507 16:39:07.938 | INFO | "huge" : "", | 20260507 16:39:07.938 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:07.938 | INFO | "malloc" : " 14 GB", | 20260507 16:39:07.938 | INFO | "pinned" : " 29 GB" | 20260507 16:39:07.938 | INFO | } | 20260507 16:39:07.938 | INFO | Perf report for the last 5.13848 seconds: | 20260507 16:39:07.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:07.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:39:07.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260507 16:39:07.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:07.938 | INFO | Uploaded 36GiB to GPU in 2.33497s (15.4178GiB/s) | 20260507 16:39:07.938 | INFO | Downloaded 34GiB to GPU in 2.50475s (13.5742GiB/s) | 20260507 16:39:13.077 | INFO | Analyzing chunk frames = [998400,998912) | 20260507 16:39:13.077 | INFO | MemInfo: { | 20260507 16:39:13.077 | INFO | "gpu" : " 17 GB", | 20260507 16:39:13.077 | INFO | "huge" : "", | 20260507 16:39:13.077 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:13.077 | INFO | "malloc" : " 14 GB", | 20260507 16:39:13.077 | INFO | "pinned" : " 29 GB" | 20260507 16:39:13.077 | INFO | } | 20260507 16:39:13.078 | INFO | Perf report for the last 5.13965 seconds: | 20260507 16:39:13.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:13.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 16:39:13.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:39:13.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:13.078 | INFO | Uploaded 36GiB to GPU in 2.36009s (15.2536GiB/s) | 20260507 16:39:13.078 | INFO | Downloaded 34GiB to GPU in 2.42379s (14.0276GiB/s) | 20260507 16:39:18.217 | INFO | Analyzing chunk frames = [998912,999424) | 20260507 16:39:18.217 | INFO | MemInfo: { | 20260507 16:39:18.217 | INFO | "gpu" : " 17 GB", | 20260507 16:39:18.217 | INFO | "huge" : "", | 20260507 16:39:18.217 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:18.217 | INFO | "malloc" : " 14 GB", | 20260507 16:39:18.217 | INFO | "pinned" : " 29 GB" | 20260507 16:39:18.217 | INFO | } | 20260507 16:39:18.217 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:39:18.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:18.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:39:18.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:39:18.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:18.217 | INFO | Uploaded 36GiB to GPU in 2.41811s (14.8876GiB/s) | 20260507 16:39:18.217 | INFO | Downloaded 34GiB to GPU in 2.51679s (13.5093GiB/s) | 20260507 16:39:18.218 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:39:18.218 | INFO | Total mem usage: 16.4 GiB | 20260507 16:39:18.218 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:39:18.218 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.48%) | 20260507 16:39:18.218 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:39:18.218 | INFO | High Water Breakdown: | 20260507 16:39:18.218 | INFO | 59.6% : 8.93 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:39:18.218 | INFO | 40.3% : 6.04 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:39:18.218 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:39:18.218 | INFO | ------------------------------------------ | 20260507 16:39:18.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:39:18.218 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:39:18.218 | INFO | Total mem usage: 26.5 GiB | 20260507 16:39:18.218 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:39:18.218 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:39:18.218 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:39:18.218 | INFO | High Water Breakdown: | 20260507 16:39:18.218 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:39:18.218 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:39:18.218 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:39:18.218 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:39:18.218 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:39:18.218 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:39:18.218 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:39:18.218 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:39:18.218 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:39:18.218 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:39:18.218 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:39:18.218 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:39:18.218 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:39:18.218 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:39:18.218 | INFO | ------------------------------------------ | 20260507 16:39:18.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:39:18.218 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:39:18.218 | INFO | Total mem usage: 27.8 GiB | 20260507 16:39:18.218 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:39:18.218 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:39:18.218 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:39:18.218 | INFO | High Water Breakdown: | 20260507 16:39:18.218 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:39:18.218 | INFO | ------------------------------------------ | 20260507 16:39:18.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:39:18.218 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:39:18.218 | INFO | Total mem usage: 15.7 GiB | 20260507 16:39:18.218 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:39:18.218 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:39:18.218 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:39:18.218 | INFO | High Water Breakdown: | 20260507 16:39:18.218 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:39:18.218 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:39:18.218 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:39:18.218 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:39:18.218 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:39:18.218 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:39:18.219 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:39:18.219 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:39:18.219 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:39:18.219 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:39:18.219 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:39:18.219 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:39:18.219 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:39:18.219 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:39:18.219 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:39:18.219 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:39:18.219 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:39:18.219 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:39:18.219 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:39:18.219 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:39:18.219 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:39:18.219 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:39:18.219 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:39:18.219 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:39:18.219 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:39:18.219 | INFO | ------------------------------------------ | 20260507 16:39:18.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:39:18.219 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:39:18.219 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:39:18.219 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:39:18.219 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.66e+01, 1.44e+01 / 2.59e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 0.30%, AggregateStats -- 1.02e-01 / 1.05e-01, 1.14e-02 / 7.91e-03 | 20260507 16:39:18.219 | INFO | PerfReport: 12.12%, Baseline -- 4.07e+00 / 4.54e+00, 6.25e-01 / 1.72e-01 | 20260507 16:39:18.219 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.52e+00, 2.19e-01 / 9.75e-02 | 20260507 16:39:18.219 | INFO | PerfReport: 15.69%, DME -- 4.22e+01 / 4.50e+01, 4.28e+00 / 2.65e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 55.76%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.83e+00 / 2.42e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.56e+00 / 2.94e+00, 4.40e-01 / 2.30e-01 | 20260507 16:39:18.219 | INFO | PerfReport: 3.85%, Metrics -- 1.29e+00 / 1.47e+00, 3.28e-01 / 2.31e-01 | 20260507 16:39:18.219 | INFO | PerfReport: 1.24%, PulseSerializing -- 4.16e-01 / 4.39e-01, 6.73e-02 / 4.42e-02 | 20260507 16:39:18.219 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:39:18.219 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:39:18.219 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:39:18.219 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 23.80%, Upload -- 2.56e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 24.06%, Download -- 2.58e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 31.33%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 1.73%, LockUpload -- 1.85e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 2.33%, LockDownload -- 2.50e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 16.75%, LockCompute -- 1.80e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:39:18.219 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260507 16:39:18.314 | INFO | Starting a round of Baz IO | 20260507 16:39:21.824 | INFO | Populating RTMetrics report | 20260507 16:39:21.824 | INFO | RTMetrics report populated | 20260507 16:39:21.824 | INFO | Opening tmp RTMetrics json file | 20260507 16:39:21.842 | INFO | Serializing RTMetrics to json | 20260507 16:39:21.849 | INFO | Serialization done, swapping tmp file now... | 20260507 16:39:21.849 | INFO | RTMetrics done | 20260507 16:39:21.931 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.616 seconds | 20260507 16:39:21.931 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.77% / 0.03% / 50.21% | 20260507 16:39:21.931 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.27694% / 3.57628e-05% / 3.97364e-05% | 20260507 16:39:21.931 | INFO | Baz IO overall: | 20260507 16:39:21.931 | INFO | BytesWritten : 431051029104 | 20260507 16:39:21.931 | INFO | NumEvents : 208621443690 | 20260507 16:39:23.356 | INFO | Analyzing chunk frames = [999424,999936) | 20260507 16:39:23.356 | INFO | MemInfo: { | 20260507 16:39:23.356 | INFO | "gpu" : " 17 GB", | 20260507 16:39:23.356 | INFO | "huge" : "", | 20260507 16:39:23.356 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:23.356 | INFO | "malloc" : " 14 GB", | 20260507 16:39:23.356 | INFO | "pinned" : " 29 GB" | 20260507 16:39:23.356 | INFO | } | 20260507 16:39:23.356 | INFO | Perf report for the last 5.13902 seconds: | 20260507 16:39:23.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:23.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:23.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:23.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:23.356 | INFO | ANALYSIS : 69.7%, 3.0, 107.9ms, 0 | 20260507 16:39:23.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:23.356 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 16:39:23.356 | INFO | PRE_HQ : 66.7%, 1.7, 59.7ms, 0 | 20260507 16:39:23.356 | INFO | BAZWRITER : 69.9%, 3.9, 140.5ms, 0 | 20260507 16:39:23.356 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 16:39:23.357 | INFO | Uploaded 36GiB to GPU in 2.5095s (14.3455GiB/s) | 20260507 16:39:23.357 | INFO | Downloaded 38GiB to GPU in 2.7261s (13.9393GiB/s) | 20260507 16:39:28.495 | INFO | Analyzing chunk frames = [999936,1000448) | 20260507 16:39:28.495 | INFO | MemInfo: { | 20260507 16:39:28.495 | INFO | "gpu" : " 17 GB", | 20260507 16:39:28.495 | INFO | "huge" : "", | 20260507 16:39:28.495 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:28.496 | INFO | "malloc" : " 14 GB", | 20260507 16:39:28.496 | INFO | "pinned" : " 29 GB" | 20260507 16:39:28.496 | INFO | } | 20260507 16:39:28.496 | INFO | Perf report for the last 5.13955 seconds: | 20260507 16:39:28.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:28.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 16:39:28.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:39:28.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:28.496 | INFO | Uploaded 36GiB to GPU in 2.48294s (14.4989GiB/s) | 20260507 16:39:28.496 | INFO | Downloaded 34GiB to GPU in 2.45113s (13.8711GiB/s) | 20260507 16:39:33.635 | INFO | Analyzing chunk frames = [1000448,1000960) | 20260507 16:39:33.635 | INFO | MemInfo: { | 20260507 16:39:33.635 | INFO | "gpu" : " 17 GB", | 20260507 16:39:33.635 | INFO | "huge" : "", | 20260507 16:39:33.635 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:33.635 | INFO | "malloc" : " 14 GB", | 20260507 16:39:33.635 | INFO | "pinned" : " 29 GB" | 20260507 16:39:33.635 | INFO | } | 20260507 16:39:33.635 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:39:33.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:33.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 16:39:33.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:39:33.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:33.635 | INFO | Uploaded 36GiB to GPU in 2.53273s (14.2139GiB/s) | 20260507 16:39:33.635 | INFO | Downloaded 34GiB to GPU in 2.44482s (13.907GiB/s) | 20260507 16:39:38.774 | INFO | Analyzing chunk frames = [1000960,1001472) | 20260507 16:39:38.774 | INFO | MemInfo: { | 20260507 16:39:38.774 | INFO | "gpu" : " 17 GB", | 20260507 16:39:38.774 | INFO | "huge" : "", | 20260507 16:39:38.774 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:38.774 | INFO | "malloc" : " 14 GB", | 20260507 16:39:38.774 | INFO | "pinned" : " 29 GB" | 20260507 16:39:38.774 | INFO | } | 20260507 16:39:38.774 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:39:38.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:38.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.774 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 16:39:38.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.774 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:39:38.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:38.775 | INFO | Uploaded 36GiB to GPU in 2.43072s (14.8104GiB/s) | 20260507 16:39:38.775 | INFO | Downloaded 34GiB to GPU in 2.37111s (14.3393GiB/s) | 20260507 16:39:43.913 | INFO | Analyzing chunk frames = [1001472,1001984) | 20260507 16:39:43.913 | INFO | MemInfo: { | 20260507 16:39:43.913 | INFO | "gpu" : " 17 GB", | 20260507 16:39:43.913 | INFO | "huge" : "", | 20260507 16:39:43.913 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:43.913 | INFO | "malloc" : " 14 GB", | 20260507 16:39:43.913 | INFO | "pinned" : " 29 GB" | 20260507 16:39:43.913 | INFO | } | 20260507 16:39:43.913 | INFO | Perf report for the last 5.13913 seconds: | 20260507 16:39:43.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:43.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.913 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 16:39:43.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.914 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 16:39:43.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:43.914 | INFO | Uploaded 36GiB to GPU in 2.33266s (15.433GiB/s) | 20260507 16:39:43.914 | INFO | Downloaded 34GiB to GPU in 2.4885s (13.6629GiB/s) | 20260507 16:39:49.052 | INFO | Analyzing chunk frames = [1001984,1002496) | 20260507 16:39:49.052 | INFO | MemInfo: { | 20260507 16:39:49.052 | INFO | "gpu" : " 17 GB", | 20260507 16:39:49.052 | INFO | "huge" : "", | 20260507 16:39:49.052 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:49.052 | INFO | "malloc" : " 14 GB", | 20260507 16:39:49.052 | INFO | "pinned" : " 29 GB" | 20260507 16:39:49.052 | INFO | } | 20260507 16:39:49.052 | INFO | Perf report for the last 5.13898 seconds: | 20260507 16:39:49.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:49.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.053 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:39:49.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.053 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260507 16:39:49.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:49.053 | INFO | Uploaded 36GiB to GPU in 2.28497s (15.7551GiB/s) | 20260507 16:39:49.053 | INFO | Downloaded 34GiB to GPU in 2.49588s (13.6224GiB/s) | 20260507 16:39:54.192 | INFO | Analyzing chunk frames = [1002496,1003008) | 20260507 16:39:54.192 | INFO | MemInfo: { | 20260507 16:39:54.192 | INFO | "gpu" : " 17 GB", | 20260507 16:39:54.192 | INFO | "huge" : "", | 20260507 16:39:54.192 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:54.192 | INFO | "malloc" : " 14 GB", | 20260507 16:39:54.192 | INFO | "pinned" : " 29 GB" | 20260507 16:39:54.192 | INFO | } | 20260507 16:39:54.192 | INFO | Perf report for the last 5.13959 seconds: | 20260507 16:39:54.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:54.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:39:54.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:39:54.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:54.192 | INFO | Uploaded 36GiB to GPU in 2.42674s (14.8347GiB/s) | 20260507 16:39:54.192 | INFO | Downloaded 34GiB to GPU in 2.42846s (14.0006GiB/s) | 20260507 16:39:59.331 | INFO | Analyzing chunk frames = [1003008,1003520) | 20260507 16:39:59.331 | INFO | MemInfo: { | 20260507 16:39:59.331 | INFO | "gpu" : " 17 GB", | 20260507 16:39:59.331 | INFO | "huge" : "", | 20260507 16:39:59.332 | INFO | "huge_shared" : " 29 GB", | 20260507 16:39:59.332 | INFO | "malloc" : " 14 GB", | 20260507 16:39:59.332 | INFO | "pinned" : " 29 GB" | 20260507 16:39:59.332 | INFO | } | 20260507 16:39:59.332 | INFO | Perf report for the last 5.13971 seconds: | 20260507 16:39:59.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:39:59.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 16:39:59.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:39:59.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:39:59.332 | INFO | Uploaded 36GiB to GPU in 2.3677s (15.2046GiB/s) | 20260507 16:39:59.332 | INFO | Downloaded 34GiB to GPU in 2.51157s (13.5374GiB/s) | 20260507 16:39:59.429 | INFO | Starting a round of Baz IO | 20260507 16:40:02.948 | INFO | Populating RTMetrics report | 20260507 16:40:02.949 | INFO | RTMetrics report populated | 20260507 16:40:02.949 | INFO | Opening tmp RTMetrics json file | 20260507 16:40:02.963 | INFO | Serializing RTMetrics to json | 20260507 16:40:02.992 | INFO | Serialization done, swapping tmp file now... | 20260507 16:40:03.017 | INFO | RTMetrics done | 20260507 16:40:03.070 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.64 seconds | 20260507 16:40:03.070 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.74% / 0.03% / 50.23% | 20260507 16:40:03.070 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.28264% / 4.76837e-05% / 3.97364e-05% | 20260507 16:40:03.070 | INFO | Baz IO overall: | 20260507 16:40:03.070 | INFO | BytesWritten : 432971540453 | 20260507 16:40:03.070 | INFO | NumEvents : 209556016301 | 20260507 16:40:04.470 | INFO | Analyzing chunk frames = [1003520,1004032) | 20260507 16:40:04.471 | INFO | MemInfo: { | 20260507 16:40:04.471 | INFO | "gpu" : " 17 GB", | 20260507 16:40:04.471 | INFO | "huge" : "", | 20260507 16:40:04.471 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:04.471 | INFO | "malloc" : " 14 GB", | 20260507 16:40:04.471 | INFO | "pinned" : " 29 GB" | 20260507 16:40:04.471 | INFO | } | 20260507 16:40:04.471 | INFO | Perf report for the last 5.13896 seconds: | 20260507 16:40:04.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:04.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:04.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:04.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:04.471 | INFO | ANALYSIS : 69.6%, 3.0, 108.1ms, 0 | 20260507 16:40:04.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:04.471 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 16:40:04.471 | INFO | PRE_HQ : 67.2%, 1.8, 62.4ms, 0 | 20260507 16:40:04.471 | INFO | BAZWRITER : 70.3%, 3.8, 136.9ms, 0 | 20260507 16:40:04.471 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 16:40:04.471 | INFO | Uploaded 36GiB to GPU in 2.49292s (14.4409GiB/s) | 20260507 16:40:04.471 | INFO | Downloaded 38GiB to GPU in 2.70815s (14.0317GiB/s) | 20260507 16:40:09.610 | INFO | Analyzing chunk frames = [1004032,1004544) | 20260507 16:40:09.610 | INFO | MemInfo: { | 20260507 16:40:09.610 | INFO | "gpu" : " 17 GB", | 20260507 16:40:09.610 | INFO | "huge" : "", | 20260507 16:40:09.610 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:09.610 | INFO | "malloc" : " 14 GB", | 20260507 16:40:09.610 | INFO | "pinned" : " 29 GB" | 20260507 16:40:09.610 | INFO | } | 20260507 16:40:09.610 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:40:09.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:09.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.610 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:40:09.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.610 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:40:09.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:09.611 | INFO | Uploaded 36GiB to GPU in 2.49172s (14.4478GiB/s) | 20260507 16:40:09.611 | INFO | Downloaded 34GiB to GPU in 2.43358s (13.9712GiB/s) | 20260507 16:40:14.749 | INFO | Analyzing chunk frames = [1004544,1005056) | 20260507 16:40:14.749 | INFO | MemInfo: { | 20260507 16:40:14.749 | INFO | "gpu" : " 17 GB", | 20260507 16:40:14.749 | INFO | "huge" : "", | 20260507 16:40:14.749 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:14.749 | INFO | "malloc" : " 14 GB", | 20260507 16:40:14.749 | INFO | "pinned" : " 29 GB" | 20260507 16:40:14.749 | INFO | } | 20260507 16:40:14.749 | INFO | Perf report for the last 5.139 seconds: | 20260507 16:40:14.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:14.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.749 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:40:14.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.749 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:40:14.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:14.750 | INFO | Uploaded 36GiB to GPU in 2.54247s (14.1595GiB/s) | 20260507 16:40:14.750 | INFO | Downloaded 34GiB to GPU in 2.41826s (14.0597GiB/s) | 20260507 16:40:19.888 | INFO | Analyzing chunk frames = [1005056,1005568) | 20260507 16:40:19.888 | INFO | MemInfo: { | 20260507 16:40:19.888 | INFO | "gpu" : " 17 GB", | 20260507 16:40:19.888 | INFO | "huge" : "", | 20260507 16:40:19.889 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:19.889 | INFO | "malloc" : " 14 GB", | 20260507 16:40:19.889 | INFO | "pinned" : " 29 GB" | 20260507 16:40:19.889 | INFO | } | 20260507 16:40:19.889 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:40:19.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:19.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:40:19.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:40:19.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:19.889 | INFO | Uploaded 36GiB to GPU in 2.47913s (14.5212GiB/s) | 20260507 16:40:19.889 | INFO | Downloaded 34GiB to GPU in 2.3573s (14.4233GiB/s) | 20260507 16:40:25.028 | INFO | Analyzing chunk frames = [1005568,1006080) | 20260507 16:40:25.028 | INFO | MemInfo: { | 20260507 16:40:25.028 | INFO | "gpu" : " 17 GB", | 20260507 16:40:25.028 | INFO | "huge" : "", | 20260507 16:40:25.028 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:25.028 | INFO | "malloc" : " 14 GB", | 20260507 16:40:25.028 | INFO | "pinned" : " 29 GB" | 20260507 16:40:25.028 | INFO | } | 20260507 16:40:25.028 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:40:25.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:25.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.028 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260507 16:40:25.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.028 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 16:40:25.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:25.029 | INFO | Uploaded 36GiB to GPU in 2.41342s (14.9166GiB/s) | 20260507 16:40:25.029 | INFO | Downloaded 34GiB to GPU in 2.47976s (13.711GiB/s) | 20260507 16:40:30.167 | INFO | Analyzing chunk frames = [1006080,1006592) | 20260507 16:40:30.167 | INFO | MemInfo: { | 20260507 16:40:30.167 | INFO | "gpu" : " 17 GB", | 20260507 16:40:30.167 | INFO | "huge" : "", | 20260507 16:40:30.167 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:30.167 | INFO | "malloc" : " 14 GB", | 20260507 16:40:30.167 | INFO | "pinned" : " 29 GB" | 20260507 16:40:30.167 | INFO | } | 20260507 16:40:30.167 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:40:30.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:30.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.167 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:40:30.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.168 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260507 16:40:30.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:30.168 | INFO | Uploaded 36GiB to GPU in 2.38084s (15.1207GiB/s) | 20260507 16:40:30.168 | INFO | Downloaded 34GiB to GPU in 2.43987s (13.9352GiB/s) | 20260507 16:40:35.307 | INFO | Analyzing chunk frames = [1006592,1007104) | 20260507 16:40:35.307 | INFO | MemInfo: { | 20260507 16:40:35.307 | INFO | "gpu" : " 17 GB", | 20260507 16:40:35.307 | INFO | "huge" : "", | 20260507 16:40:35.307 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:35.307 | INFO | "malloc" : " 14 GB", | 20260507 16:40:35.307 | INFO | "pinned" : " 29 GB" | 20260507 16:40:35.307 | INFO | } | 20260507 16:40:35.307 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:40:35.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:35.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:40:35.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:40:35.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:35.307 | INFO | Uploaded 36GiB to GPU in 2.47844s (14.5252GiB/s) | 20260507 16:40:35.307 | INFO | Downloaded 34GiB to GPU in 2.40244s (14.1523GiB/s) | 20260507 16:40:40.446 | INFO | Analyzing chunk frames = [1007104,1007616) | 20260507 16:40:40.446 | INFO | MemInfo: { | 20260507 16:40:40.446 | INFO | "gpu" : " 17 GB", | 20260507 16:40:40.446 | INFO | "huge" : "", | 20260507 16:40:40.446 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:40.446 | INFO | "malloc" : " 14 GB", | 20260507 16:40:40.446 | INFO | "pinned" : " 29 GB" | 20260507 16:40:40.446 | INFO | } | 20260507 16:40:40.446 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:40:40.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:40.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.446 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 16:40:40.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.446 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:40:40.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:40.447 | INFO | Uploaded 36GiB to GPU in 2.5158s (14.3096GiB/s) | 20260507 16:40:40.447 | INFO | Downloaded 34GiB to GPU in 2.48969s (13.6563GiB/s) | 20260507 16:40:40.447 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:40:40.447 | INFO | Total mem usage: 16.4 GiB | 20260507 16:40:40.447 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:40:40.447 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.4%) | 20260507 16:40:40.447 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:40:40.447 | INFO | High Water Breakdown: | 20260507 16:40:40.447 | INFO | 59.1% : 8.63 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:40:40.447 | INFO | 40.8% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:40:40.447 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:40:40.447 | INFO | ------------------------------------------ | 20260507 16:40:40.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:40:40.447 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:40:40.447 | INFO | Total mem usage: 26.5 GiB | 20260507 16:40:40.447 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:40:40.447 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:40:40.447 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:40:40.447 | INFO | High Water Breakdown: | 20260507 16:40:40.447 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:40:40.447 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:40:40.447 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:40:40.447 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:40:40.447 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:40:40.447 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:40:40.447 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:40:40.447 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:40:40.447 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:40:40.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:40:40.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:40:40.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:40:40.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:40:40.447 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:40:40.447 | INFO | ------------------------------------------ | 20260507 16:40:40.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:40:40.447 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:40:40.447 | INFO | Total mem usage: 27.8 GiB | 20260507 16:40:40.447 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:40:40.447 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:40:40.447 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:40:40.447 | INFO | High Water Breakdown: | 20260507 16:40:40.447 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:40:40.447 | INFO | ------------------------------------------ | 20260507 16:40:40.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:40:40.447 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:40:40.447 | INFO | Total mem usage: 15.7 GiB | 20260507 16:40:40.447 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:40:40.447 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:40:40.447 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:40:40.448 | INFO | High Water Breakdown: | 20260507 16:40:40.448 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:40:40.448 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:40:40.448 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:40:40.448 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:40:40.448 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:40:40.448 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:40:40.448 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:40:40.448 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:40:40.448 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:40:40.448 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:40:40.448 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:40:40.448 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:40:40.448 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:40:40.448 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:40:40.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:40:40.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:40:40.448 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:40:40.448 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:40:40.448 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:40:40.448 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:40:40.448 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:40:40.448 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:40:40.448 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:40:40.448 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:40:40.448 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:40:40.448 | INFO | ------------------------------------------ | 20260507 16:40:40.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:40:40.448 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:40:40.448 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:40:40.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:40:40.448 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.69e+01, 1.45e+01 / 2.70e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.10e-02 / 5.28e-03 | 20260507 16:40:40.448 | INFO | PerfReport: 12.05%, Baseline -- 4.06e+00 / 4.50e+00, 6.27e-01 / 2.15e-01 | 20260507 16:40:40.448 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.49e+00, 2.18e-01 / 1.06e-01 | 20260507 16:40:40.448 | INFO | PerfReport: 15.70%, DME -- 4.23e+01 / 4.52e+01, 4.39e+00 / 2.83e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 56.00%, FrameLabeling -- 1.89e+01 / 2.08e+01, 1.93e+00 / 2.26e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.53e+00 / 2.89e+00, 4.34e-01 / 2.50e-01 | 20260507 16:40:40.448 | INFO | PerfReport: 3.82%, Metrics -- 1.29e+00 / 1.46e+00, 3.27e-01 / 1.91e-01 | 20260507 16:40:40.448 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.09e-01 / 4.33e-01, 6.85e-02 / 4.34e-02 | 20260507 16:40:40.448 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:40:40.448 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:40:40.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:40:40.448 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 23.71%, Upload -- 2.55e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 24.00%, Download -- 2.58e+01 / 0.00e+00, 8.65e+00 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 31.32%, Compute -- 3.37e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 1.68%, LockUpload -- 1.80e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 2.41%, LockDownload -- 2.59e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 16.87%, LockCompute -- 1.81e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:40:40.448 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260507 16:40:40.543 | INFO | Starting a round of Baz IO | 20260507 16:40:44.061 | INFO | Populating RTMetrics report | 20260507 16:40:44.061 | INFO | RTMetrics report populated | 20260507 16:40:44.061 | INFO | Opening tmp RTMetrics json file | 20260507 16:40:44.062 | INFO | Serializing RTMetrics to json | 20260507 16:40:44.081 | INFO | Serialization done, swapping tmp file now... | 20260507 16:40:44.086 | INFO | RTMetrics done | 20260507 16:40:44.171 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.628 seconds | 20260507 16:40:44.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.72% / 0.03% / 50.26% | 20260507 16:40:44.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.25713% / 1.98682e-05% / 3.97364e-05% | 20260507 16:40:44.171 | INFO | Baz IO overall: | 20260507 16:40:44.171 | INFO | BytesWritten : 434890154750 | 20260507 16:40:44.171 | INFO | NumEvents : 210489115370 | 20260507 16:40:45.585 | INFO | Analyzing chunk frames = [1007616,1008128) | 20260507 16:40:45.585 | INFO | MemInfo: { | 20260507 16:40:45.585 | INFO | "gpu" : " 17 GB", | 20260507 16:40:45.585 | INFO | "huge" : "", | 20260507 16:40:45.585 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:45.585 | INFO | "malloc" : " 14 GB", | 20260507 16:40:45.585 | INFO | "pinned" : " 29 GB" | 20260507 16:40:45.585 | INFO | } | 20260507 16:40:45.585 | INFO | Perf report for the last 5.13899 seconds: | 20260507 16:40:45.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:45.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:45.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:45.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:45.585 | INFO | ANALYSIS : 69.9%, 2.9, 106.8ms, 0 | 20260507 16:40:45.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:45.585 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 16:40:45.585 | INFO | PRE_HQ : 67.3%, 1.8, 63.3ms, 0 | 20260507 16:40:45.585 | INFO | BAZWRITER : 70.2%, 4.0, 145.7ms, 0 | 20260507 16:40:45.585 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 16:40:45.586 | INFO | Uploaded 36GiB to GPU in 2.57723s (13.9685GiB/s) | 20260507 16:40:45.586 | INFO | Downloaded 38GiB to GPU in 2.70502s (14.048GiB/s) | 20260507 16:40:50.724 | INFO | Analyzing chunk frames = [1008128,1008640) | 20260507 16:40:50.724 | INFO | MemInfo: { | 20260507 16:40:50.724 | INFO | "gpu" : " 17 GB", | 20260507 16:40:50.724 | INFO | "huge" : "", | 20260507 16:40:50.724 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:50.724 | INFO | "malloc" : " 14 GB", | 20260507 16:40:50.724 | INFO | "pinned" : " 29 GB" | 20260507 16:40:50.724 | INFO | } | 20260507 16:40:50.725 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:40:50.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:50.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:40:50.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:40:50.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:50.725 | INFO | Uploaded 36GiB to GPU in 2.51634s (14.3065GiB/s) | 20260507 16:40:50.725 | INFO | Downloaded 34GiB to GPU in 2.41772s (14.0628GiB/s) | 20260507 16:40:55.864 | INFO | Analyzing chunk frames = [1008640,1009152) | 20260507 16:40:55.864 | INFO | MemInfo: { | 20260507 16:40:55.864 | INFO | "gpu" : " 17 GB", | 20260507 16:40:55.864 | INFO | "huge" : "", | 20260507 16:40:55.864 | INFO | "huge_shared" : " 29 GB", | 20260507 16:40:55.864 | INFO | "malloc" : " 14 GB", | 20260507 16:40:55.864 | INFO | "pinned" : " 29 GB" | 20260507 16:40:55.864 | INFO | } | 20260507 16:40:55.864 | INFO | Perf report for the last 5.13993 seconds: | 20260507 16:40:55.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:40:55.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260507 16:40:55.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 16:40:55.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:40:55.865 | INFO | Uploaded 36GiB to GPU in 2.57534s (13.9787GiB/s) | 20260507 16:40:55.865 | INFO | Downloaded 34GiB to GPU in 2.39656s (14.187GiB/s) | 20260507 16:41:01.003 | INFO | Analyzing chunk frames = [1009152,1009664) | 20260507 16:41:01.003 | INFO | MemInfo: { | 20260507 16:41:01.003 | INFO | "gpu" : " 17 GB", | 20260507 16:41:01.003 | INFO | "huge" : "", | 20260507 16:41:01.003 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:01.003 | INFO | "malloc" : " 14 GB", | 20260507 16:41:01.003 | INFO | "pinned" : " 29 GB" | 20260507 16:41:01.003 | INFO | } | 20260507 16:41:01.004 | INFO | Perf report for the last 5.13899 seconds: | 20260507 16:41:01.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:01.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260507 16:41:01.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:41:01.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:01.004 | INFO | Uploaded 36GiB to GPU in 2.4978s (14.4127GiB/s) | 20260507 16:41:01.004 | INFO | Downloaded 34GiB to GPU in 2.34626s (14.4912GiB/s) | 20260507 16:41:06.143 | INFO | Analyzing chunk frames = [1009664,1010176) | 20260507 16:41:06.143 | INFO | MemInfo: { | 20260507 16:41:06.143 | INFO | "gpu" : " 17 GB", | 20260507 16:41:06.143 | INFO | "huge" : "", | 20260507 16:41:06.143 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:06.143 | INFO | "malloc" : " 14 GB", | 20260507 16:41:06.143 | INFO | "pinned" : " 29 GB" | 20260507 16:41:06.143 | INFO | } | 20260507 16:41:06.143 | INFO | Perf report for the last 5.13979 seconds: | 20260507 16:41:06.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:06.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.143 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260507 16:41:06.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.143 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 16:41:06.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:06.144 | INFO | Uploaded 36GiB to GPU in 2.43336s (14.7944GiB/s) | 20260507 16:41:06.144 | INFO | Downloaded 34GiB to GPU in 2.47123s (13.7583GiB/s) | 20260507 16:41:11.282 | INFO | Analyzing chunk frames = [1010176,1010688) | 20260507 16:41:11.282 | INFO | MemInfo: { | 20260507 16:41:11.282 | INFO | "gpu" : " 17 GB", | 20260507 16:41:11.282 | INFO | "huge" : "", | 20260507 16:41:11.282 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:11.282 | INFO | "malloc" : " 14 GB", | 20260507 16:41:11.282 | INFO | "pinned" : " 29 GB" | 20260507 16:41:11.282 | INFO | } | 20260507 16:41:11.282 | INFO | Perf report for the last 5.13916 seconds: | 20260507 16:41:11.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:11.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.283 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:41:11.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.283 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260507 16:41:11.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:11.283 | INFO | Uploaded 36GiB to GPU in 2.35421s (15.2917GiB/s) | 20260507 16:41:11.283 | INFO | Downloaded 34GiB to GPU in 2.47845s (13.7183GiB/s) | 20260507 16:41:16.422 | INFO | Analyzing chunk frames = [1010688,1011200) | 20260507 16:41:16.422 | INFO | MemInfo: { | 20260507 16:41:16.422 | INFO | "gpu" : " 17 GB", | 20260507 16:41:16.422 | INFO | "huge" : "", | 20260507 16:41:16.422 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:16.422 | INFO | "malloc" : " 14 GB", | 20260507 16:41:16.422 | INFO | "pinned" : " 29 GB" | 20260507 16:41:16.422 | INFO | } | 20260507 16:41:16.422 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:41:16.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:16.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 16:41:16.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 16:41:16.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:16.422 | INFO | Uploaded 36GiB to GPU in 2.40796s (14.9504GiB/s) | 20260507 16:41:16.422 | INFO | Downloaded 34GiB to GPU in 2.40933s (14.1118GiB/s) | 20260507 16:41:21.560 | INFO | Analyzing chunk frames = [1011200,1011712) | 20260507 16:41:21.560 | INFO | MemInfo: { | 20260507 16:41:21.560 | INFO | "gpu" : " 17 GB", | 20260507 16:41:21.560 | INFO | "huge" : "", | 20260507 16:41:21.560 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:21.561 | INFO | "malloc" : " 14 GB", | 20260507 16:41:21.561 | INFO | "pinned" : " 29 GB" | 20260507 16:41:21.561 | INFO | } | 20260507 16:41:21.561 | INFO | Perf report for the last 5.13884 seconds: | 20260507 16:41:21.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:21.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 16:41:21.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:41:21.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:21.561 | INFO | Uploaded 36GiB to GPU in 2.44903s (14.6997GiB/s) | 20260507 16:41:21.561 | INFO | Downloaded 34GiB to GPU in 2.49455s (13.6297GiB/s) | 20260507 16:41:21.658 | INFO | Starting a round of Baz IO | 20260507 16:41:25.173 | INFO | Populating RTMetrics report | 20260507 16:41:25.173 | INFO | RTMetrics report populated | 20260507 16:41:25.173 | INFO | Opening tmp RTMetrics json file | 20260507 16:41:25.189 | INFO | Serializing RTMetrics to json | 20260507 16:41:25.197 | INFO | Serialization done, swapping tmp file now... | 20260507 16:41:25.198 | INFO | RTMetrics done | 20260507 16:41:25.279 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.621 seconds | 20260507 16:41:25.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.69% / 0.03% / 50.28% | 20260507 16:41:25.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.25378% / 2.38419e-05% / 3.17891e-05% | 20260507 16:41:25.279 | INFO | Baz IO overall: | 20260507 16:41:25.279 | INFO | BytesWritten : 436809909082 | 20260507 16:41:25.279 | INFO | NumEvents : 211422881682 | 20260507 16:41:26.700 | INFO | Analyzing chunk frames = [1011712,1012224) | 20260507 16:41:26.700 | INFO | MemInfo: { | 20260507 16:41:26.700 | INFO | "gpu" : " 17 GB", | 20260507 16:41:26.700 | INFO | "huge" : "", | 20260507 16:41:26.700 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:26.700 | INFO | "malloc" : " 14 GB", | 20260507 16:41:26.700 | INFO | "pinned" : " 29 GB" | 20260507 16:41:26.700 | INFO | } | 20260507 16:41:26.700 | INFO | Perf report for the last 5.13972 seconds: | 20260507 16:41:26.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:26.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:26.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:26.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:26.700 | INFO | ANALYSIS : 69.8%, 2.9, 107.7ms, 0 | 20260507 16:41:26.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:26.701 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 16:41:26.701 | INFO | PRE_HQ : 67.3%, 1.7, 60.4ms, 0 | 20260507 16:41:26.701 | INFO | BAZWRITER : 69.9%, 3.8, 138.1ms, 0 | 20260507 16:41:26.701 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 16:41:26.701 | INFO | Uploaded 36GiB to GPU in 2.5566s (14.0812GiB/s) | 20260507 16:41:26.701 | INFO | Downloaded 38GiB to GPU in 2.73206s (13.9089GiB/s) | 20260507 16:41:31.839 | INFO | Analyzing chunk frames = [1012224,1012736) | 20260507 16:41:31.839 | INFO | MemInfo: { | 20260507 16:41:31.839 | INFO | "gpu" : " 17 GB", | 20260507 16:41:31.839 | INFO | "huge" : "", | 20260507 16:41:31.840 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:31.840 | INFO | "malloc" : " 14 GB", | 20260507 16:41:31.840 | INFO | "pinned" : " 29 GB" | 20260507 16:41:31.840 | INFO | } | 20260507 16:41:31.840 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:41:31.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:31.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:41:31.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:41:31.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:31.840 | INFO | Uploaded 36GiB to GPU in 2.46761s (14.589GiB/s) | 20260507 16:41:31.840 | INFO | Downloaded 34GiB to GPU in 2.45655s (13.8406GiB/s) | 20260507 16:41:36.979 | INFO | Analyzing chunk frames = [1012736,1013248) | 20260507 16:41:36.979 | INFO | MemInfo: { | 20260507 16:41:36.979 | INFO | "gpu" : " 17 GB", | 20260507 16:41:36.979 | INFO | "huge" : "", | 20260507 16:41:36.979 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:36.979 | INFO | "malloc" : " 14 GB", | 20260507 16:41:36.979 | INFO | "pinned" : " 29 GB" | 20260507 16:41:36.979 | INFO | } | 20260507 16:41:36.979 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:41:36.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:36.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 16:41:36.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:41:36.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:36.979 | INFO | Uploaded 36GiB to GPU in 2.54245s (14.1596GiB/s) | 20260507 16:41:36.979 | INFO | Downloaded 34GiB to GPU in 2.43073s (13.9875GiB/s) | 20260507 16:41:42.118 | INFO | Analyzing chunk frames = [1013248,1013760) | 20260507 16:41:42.118 | INFO | MemInfo: { | 20260507 16:41:42.118 | INFO | "gpu" : " 17 GB", | 20260507 16:41:42.118 | INFO | "huge" : "", | 20260507 16:41:42.118 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:42.118 | INFO | "malloc" : " 14 GB", | 20260507 16:41:42.118 | INFO | "pinned" : " 29 GB" | 20260507 16:41:42.118 | INFO | } | 20260507 16:41:42.118 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:41:42.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:42.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.118 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 16:41:42.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.119 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:41:42.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:42.119 | INFO | Uploaded 36GiB to GPU in 2.43341s (14.794GiB/s) | 20260507 16:41:42.119 | INFO | Downloaded 34GiB to GPU in 2.36479s (14.3776GiB/s) | 20260507 16:41:47.258 | INFO | Analyzing chunk frames = [1013760,1014272) | 20260507 16:41:47.258 | INFO | MemInfo: { | 20260507 16:41:47.258 | INFO | "gpu" : " 17 GB", | 20260507 16:41:47.258 | INFO | "huge" : "", | 20260507 16:41:47.258 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:47.258 | INFO | "malloc" : " 14 GB", | 20260507 16:41:47.258 | INFO | "pinned" : " 29 GB" | 20260507 16:41:47.258 | INFO | } | 20260507 16:41:47.258 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:41:47.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:47.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 16:41:47.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 16:41:47.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:47.258 | INFO | Uploaded 36GiB to GPU in 2.3665s (15.2123GiB/s) | 20260507 16:41:47.258 | INFO | Downloaded 34GiB to GPU in 2.48428s (13.6861GiB/s) | 20260507 16:41:52.397 | INFO | Analyzing chunk frames = [1014272,1014784) | 20260507 16:41:52.397 | INFO | MemInfo: { | 20260507 16:41:52.397 | INFO | "gpu" : " 17 GB", | 20260507 16:41:52.397 | INFO | "huge" : "", | 20260507 16:41:52.397 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:52.397 | INFO | "malloc" : " 14 GB", | 20260507 16:41:52.397 | INFO | "pinned" : " 29 GB" | 20260507 16:41:52.397 | INFO | } | 20260507 16:41:52.397 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:41:52.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:52.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.397 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:41:52.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.397 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 16:41:52.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:52.398 | INFO | Uploaded 36GiB to GPU in 2.29613s (15.6786GiB/s) | 20260507 16:41:52.398 | INFO | Downloaded 34GiB to GPU in 2.47756s (13.7232GiB/s) | 20260507 16:41:57.536 | INFO | Analyzing chunk frames = [1014784,1015296) | 20260507 16:41:57.536 | INFO | MemInfo: { | 20260507 16:41:57.536 | INFO | "gpu" : " 17 GB", | 20260507 16:41:57.536 | INFO | "huge" : "", | 20260507 16:41:57.536 | INFO | "huge_shared" : " 29 GB", | 20260507 16:41:57.536 | INFO | "malloc" : " 14 GB", | 20260507 16:41:57.536 | INFO | "pinned" : " 29 GB" | 20260507 16:41:57.536 | INFO | } | 20260507 16:41:57.536 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:41:57.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:41:57.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.536 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:41:57.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.536 | INFO | PRE_HQ : 37.8%, 1.0, 19.6ms, 0 | 20260507 16:41:57.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:41:57.537 | INFO | Uploaded 36GiB to GPU in 2.4133s (14.9173GiB/s) | 20260507 16:41:57.537 | INFO | Downloaded 34GiB to GPU in 2.43355s (13.9713GiB/s) | 20260507 16:42:02.675 | INFO | Analyzing chunk frames = [1015296,1015808) | 20260507 16:42:02.675 | INFO | MemInfo: { | 20260507 16:42:02.675 | INFO | "gpu" : " 17 GB", | 20260507 16:42:02.676 | INFO | "huge" : "", | 20260507 16:42:02.676 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:02.676 | INFO | "malloc" : " 14 GB", | 20260507 16:42:02.676 | INFO | "pinned" : " 29 GB" | 20260507 16:42:02.676 | INFO | } | 20260507 16:42:02.676 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:42:02.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:02.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.676 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 16:42:02.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.676 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 16:42:02.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:02.677 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:42:02.677 | INFO | Total mem usage: 16.4 GiB | 20260507 16:42:02.677 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:42:02.677 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.38%) | 20260507 16:42:02.677 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:42:02.677 | INFO | High Water Breakdown: | 20260507 16:42:02.677 | INFO | 59.7% : 9.04 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:42:02.677 | INFO | 40.2% : 6.08 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:42:02.677 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:42:02.677 | INFO | ------------------------------------------ | 20260507 16:42:02.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:42:02.677 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:42:02.677 | INFO | Total mem usage: 26.5 GiB | 20260507 16:42:02.677 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:42:02.677 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:42:02.677 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:42:02.677 | INFO | High Water Breakdown: | 20260507 16:42:02.677 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:42:02.677 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:42:02.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:42:02.677 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:42:02.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:42:02.677 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:42:02.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:42:02.677 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:42:02.677 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:42:02.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:42:02.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:42:02.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:42:02.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:42:02.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:42:02.677 | INFO | ------------------------------------------ | 20260507 16:42:02.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:42:02.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:42:02.677 | INFO | Total mem usage: 27.8 GiB | 20260507 16:42:02.677 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:42:02.677 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:42:02.677 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:42:02.677 | INFO | High Water Breakdown: | 20260507 16:42:02.677 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:42:02.677 | INFO | ------------------------------------------ | 20260507 16:42:02.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:42:02.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:42:02.677 | INFO | Total mem usage: 15.7 GiB | 20260507 16:42:02.677 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:42:02.677 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:42:02.677 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:42:02.677 | INFO | High Water Breakdown: | 20260507 16:42:02.677 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:42:02.677 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:42:02.677 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:42:02.677 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:42:02.677 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:42:02.677 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:42:02.677 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:42:02.677 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:42:02.677 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:42:02.677 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:42:02.677 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:42:02.677 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:42:02.677 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:42:02.677 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:42:02.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:42:02.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:42:02.677 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:42:02.677 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:42:02.677 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:42:02.677 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:42:02.677 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:42:02.677 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:42:02.677 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:42:02.677 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:42:02.677 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:42:02.677 | INFO | ------------------------------------------ | 20260507 16:42:02.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:42:02.677 | INFO | Uploaded 36GiB to GPU in 2.48721s (14.474GiB/s) | 20260507 16:42:02.677 | INFO | Downloaded 34GiB to GPU in 2.50975s (13.5472GiB/s) | 20260507 16:42:02.677 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:42:02.677 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:42:02.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:42:02.678 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.69e+01, 1.44e+01 / 2.90e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 0.29%, AggregateStats -- 9.93e-02 / 1.04e-01, 1.30e-02 / 6.15e-03 | 20260507 16:42:02.678 | INFO | PerfReport: 12.02%, Baseline -- 4.05e+00 / 4.49e+00, 6.21e-01 / 2.55e-01 | 20260507 16:42:02.678 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.07e-01 | 20260507 16:42:02.678 | INFO | PerfReport: 15.69%, DME -- 4.23e+01 / 4.52e+01, 4.51e+00 / 2.88e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.89e+01 / 2.08e+01, 1.94e+00 / 2.25e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.53e+00 / 2.91e+00, 4.36e-01 / 2.37e-01 | 20260507 16:42:02.678 | INFO | PerfReport: 3.83%, Metrics -- 1.29e+00 / 1.47e+00, 3.32e-01 / 1.92e-01 | 20260507 16:42:02.678 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.10e-01 / 4.34e-01, 6.83e-02 / 4.53e-02 | 20260507 16:42:02.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:42:02.678 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:42:02.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:42:02.678 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 23.84%, Upload -- 2.56e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 23.98%, Download -- 2.58e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 31.33%, Compute -- 3.37e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 1.68%, LockUpload -- 1.81e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 2.33%, LockDownload -- 2.51e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 16.82%, LockCompute -- 1.81e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:42:02.678 | INFO | PerfReport: 0.01%, Misc -- 1.57e-02 / 0.00e+00, 3.14e-02 / 0.00e+00 | 20260507 16:42:02.791 | INFO | Starting a round of Baz IO | 20260507 16:42:06.280 | INFO | Populating RTMetrics report | 20260507 16:42:06.280 | INFO | RTMetrics report populated | 20260507 16:42:06.280 | INFO | Opening tmp RTMetrics json file | 20260507 16:42:06.306 | INFO | Serializing RTMetrics to json | 20260507 16:42:06.307 | INFO | Serialization done, swapping tmp file now... | 20260507 16:42:06.308 | INFO | RTMetrics done | 20260507 16:42:06.394 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.603 seconds | 20260507 16:42:06.394 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.66% / 0.03% / 50.31% | 20260507 16:42:06.394 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.25237% / 5.16574e-05% / 6.35783e-05% | 20260507 16:42:06.394 | INFO | Baz IO overall: | 20260507 16:42:06.394 | INFO | BytesWritten : 438731294306 | 20260507 16:42:06.394 | INFO | NumEvents : 212357461804 | 20260507 16:42:07.815 | INFO | Analyzing chunk frames = [1015808,1016320) | 20260507 16:42:07.815 | INFO | MemInfo: { | 20260507 16:42:07.815 | INFO | "gpu" : " 17 GB", | 20260507 16:42:07.815 | INFO | "huge" : "", | 20260507 16:42:07.815 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:07.815 | INFO | "malloc" : " 14 GB", | 20260507 16:42:07.815 | INFO | "pinned" : " 29 GB" | 20260507 16:42:07.815 | INFO | } | 20260507 16:42:07.815 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:42:07.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:07.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:07.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:07.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:07.815 | INFO | ANALYSIS : 69.6%, 2.9, 106.0ms, 0 | 20260507 16:42:07.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:07.815 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 16:42:07.815 | INFO | PRE_HQ : 67.9%, 1.8, 64.7ms, 0 | 20260507 16:42:07.815 | INFO | BAZWRITER : 69.7%, 3.8, 136.6ms, 0 | 20260507 16:42:07.815 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:42:07.816 | INFO | Uploaded 36GiB to GPU in 2.51325s (14.3241GiB/s) | 20260507 16:42:07.816 | INFO | Downloaded 38GiB to GPU in 2.71152s (14.0143GiB/s) | 20260507 16:42:12.954 | INFO | Analyzing chunk frames = [1016320,1016832) | 20260507 16:42:12.954 | INFO | MemInfo: { | 20260507 16:42:12.954 | INFO | "gpu" : " 17 GB", | 20260507 16:42:12.954 | INFO | "huge" : "", | 20260507 16:42:12.954 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:12.954 | INFO | "malloc" : " 14 GB", | 20260507 16:42:12.954 | INFO | "pinned" : " 29 GB" | 20260507 16:42:12.954 | INFO | } | 20260507 16:42:12.954 | INFO | Perf report for the last 5.13888 seconds: | 20260507 16:42:12.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:12.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.954 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:42:12.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.954 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 16:42:12.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:12.955 | INFO | Uploaded 36GiB to GPU in 2.54146s (14.1651GiB/s) | 20260507 16:42:12.955 | INFO | Downloaded 34GiB to GPU in 2.45376s (13.8563GiB/s) | 20260507 16:42:18.093 | INFO | Analyzing chunk frames = [1016832,1017344) | 20260507 16:42:18.093 | INFO | MemInfo: { | 20260507 16:42:18.093 | INFO | "gpu" : " 17 GB", | 20260507 16:42:18.093 | INFO | "huge" : "", | 20260507 16:42:18.093 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:18.093 | INFO | "malloc" : " 14 GB", | 20260507 16:42:18.093 | INFO | "pinned" : " 29 GB" | 20260507 16:42:18.093 | INFO | } | 20260507 16:42:18.094 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:42:18.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:18.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260507 16:42:18.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:42:18.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:18.094 | INFO | Uploaded 36GiB to GPU in 2.48017s (14.5152GiB/s) | 20260507 16:42:18.094 | INFO | Downloaded 34GiB to GPU in 2.35031s (14.4662GiB/s) | 20260507 16:42:23.232 | INFO | Analyzing chunk frames = [1017344,1017856) | 20260507 16:42:23.232 | INFO | MemInfo: { | 20260507 16:42:23.232 | INFO | "gpu" : " 17 GB", | 20260507 16:42:23.232 | INFO | "huge" : "", | 20260507 16:42:23.233 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:23.233 | INFO | "malloc" : " 14 GB", | 20260507 16:42:23.233 | INFO | "pinned" : " 29 GB" | 20260507 16:42:23.233 | INFO | } | 20260507 16:42:23.233 | INFO | Perf report for the last 5.13907 seconds: | 20260507 16:42:23.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:23.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 16:42:23.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:42:23.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:23.233 | INFO | Uploaded 36GiB to GPU in 2.48367s (14.4947GiB/s) | 20260507 16:42:23.233 | INFO | Downloaded 34GiB to GPU in 2.35952s (14.4097GiB/s) | 20260507 16:42:28.372 | INFO | Analyzing chunk frames = [1017856,1018368) | 20260507 16:42:28.372 | INFO | MemInfo: { | 20260507 16:42:28.372 | INFO | "gpu" : " 17 GB", | 20260507 16:42:28.372 | INFO | "huge" : "", | 20260507 16:42:28.372 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:28.372 | INFO | "malloc" : " 14 GB", | 20260507 16:42:28.372 | INFO | "pinned" : " 29 GB" | 20260507 16:42:28.372 | INFO | } | 20260507 16:42:28.373 | INFO | Perf report for the last 5.13991 seconds: | 20260507 16:42:28.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:28.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:42:28.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 16:42:28.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:28.373 | INFO | Uploaded 36GiB to GPU in 2.40964s (14.94GiB/s) | 20260507 16:42:28.373 | INFO | Downloaded 34GiB to GPU in 2.46304s (13.8041GiB/s) | 20260507 16:42:33.511 | INFO | Analyzing chunk frames = [1018368,1018880) | 20260507 16:42:33.511 | INFO | MemInfo: { | 20260507 16:42:33.511 | INFO | "gpu" : " 17 GB", | 20260507 16:42:33.511 | INFO | "huge" : "", | 20260507 16:42:33.511 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:33.511 | INFO | "malloc" : " 14 GB", | 20260507 16:42:33.511 | INFO | "pinned" : " 29 GB" | 20260507 16:42:33.511 | INFO | } | 20260507 16:42:33.511 | INFO | Perf report for the last 5.13878 seconds: | 20260507 16:42:33.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:33.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.511 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:42:33.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.511 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 16:42:33.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:33.512 | INFO | Uploaded 36GiB to GPU in 2.47937s (14.5198GiB/s) | 20260507 16:42:33.512 | INFO | Downloaded 34GiB to GPU in 2.48335s (13.6912GiB/s) | 20260507 16:42:38.650 | INFO | Analyzing chunk frames = [1018880,1019392) | 20260507 16:42:38.650 | INFO | MemInfo: { | 20260507 16:42:38.650 | INFO | "gpu" : " 17 GB", | 20260507 16:42:38.650 | INFO | "huge" : "", | 20260507 16:42:38.650 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:38.651 | INFO | "malloc" : " 14 GB", | 20260507 16:42:38.651 | INFO | "pinned" : " 29 GB" | 20260507 16:42:38.651 | INFO | } | 20260507 16:42:38.651 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:42:38.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:38.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 16:42:38.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 16:42:38.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:38.651 | INFO | Uploaded 36GiB to GPU in 2.44635s (14.7158GiB/s) | 20260507 16:42:38.651 | INFO | Downloaded 34GiB to GPU in 2.39876s (14.174GiB/s) | 20260507 16:42:43.790 | INFO | Analyzing chunk frames = [1019392,1019904) | 20260507 16:42:43.790 | INFO | MemInfo: { | 20260507 16:42:43.790 | INFO | "gpu" : " 17 GB", | 20260507 16:42:43.790 | INFO | "huge" : "", | 20260507 16:42:43.790 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:43.790 | INFO | "malloc" : " 14 GB", | 20260507 16:42:43.790 | INFO | "pinned" : " 29 GB" | 20260507 16:42:43.790 | INFO | } | 20260507 16:42:43.790 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:42:43.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:43.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:42:43.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:42:43.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:43.790 | INFO | Uploaded 36GiB to GPU in 2.4408s (14.7493GiB/s) | 20260507 16:42:43.790 | INFO | Downloaded 34GiB to GPU in 2.49482s (13.6283GiB/s) | 20260507 16:42:43.887 | INFO | Starting a round of Baz IO | 20260507 16:42:47.406 | INFO | Populating RTMetrics report | 20260507 16:42:47.407 | INFO | RTMetrics report populated | 20260507 16:42:47.407 | INFO | Opening tmp RTMetrics json file | 20260507 16:42:47.420 | INFO | Serializing RTMetrics to json | 20260507 16:42:47.437 | INFO | Serialization done, swapping tmp file now... | 20260507 16:42:47.442 | INFO | RTMetrics done | 20260507 16:42:47.533 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.645 seconds | 20260507 16:42:47.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.64% / 0.03% / 50.34% | 20260507 16:42:47.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.24183% / 4.37101e-05% / 4.37101e-05% | 20260507 16:42:47.533 | INFO | Baz IO overall: | 20260507 16:42:47.533 | INFO | BytesWritten : 440652772138 | 20260507 16:42:47.533 | INFO | NumEvents : 213291983461 | 20260507 16:42:48.929 | INFO | Analyzing chunk frames = [1019904,1020416) | 20260507 16:42:48.929 | INFO | MemInfo: { | 20260507 16:42:48.929 | INFO | "gpu" : " 17 GB", | 20260507 16:42:48.929 | INFO | "huge" : "", | 20260507 16:42:48.929 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:48.929 | INFO | "malloc" : " 14 GB", | 20260507 16:42:48.929 | INFO | "pinned" : " 29 GB" | 20260507 16:42:48.929 | INFO | } | 20260507 16:42:48.929 | INFO | Perf report for the last 5.13935 seconds: | 20260507 16:42:48.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:48.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:48.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:48.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:48.930 | INFO | ANALYSIS : 69.9%, 2.9, 108.0ms, 0 | 20260507 16:42:48.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:48.930 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 16:42:48.930 | INFO | PRE_HQ : 67.0%, 1.7, 60.1ms, 0 | 20260507 16:42:48.930 | INFO | BAZWRITER : 70.5%, 3.7, 135.8ms, 0 | 20260507 16:42:48.930 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 16:42:48.930 | INFO | Uploaded 36GiB to GPU in 2.55828s (14.072GiB/s) | 20260507 16:42:48.930 | INFO | Downloaded 38GiB to GPU in 2.73247s (13.9069GiB/s) | 20260507 16:42:54.069 | INFO | Analyzing chunk frames = [1020416,1020928) | 20260507 16:42:54.069 | INFO | MemInfo: { | 20260507 16:42:54.069 | INFO | "gpu" : " 17 GB", | 20260507 16:42:54.069 | INFO | "huge" : "", | 20260507 16:42:54.069 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:54.069 | INFO | "malloc" : " 14 GB", | 20260507 16:42:54.069 | INFO | "pinned" : " 29 GB" | 20260507 16:42:54.069 | INFO | } | 20260507 16:42:54.069 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:42:54.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:54.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:42:54.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:42:54.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:54.069 | INFO | Uploaded 36GiB to GPU in 2.49935s (14.4038GiB/s) | 20260507 16:42:54.069 | INFO | Downloaded 34GiB to GPU in 2.45005s (13.8773GiB/s) | 20260507 16:42:59.208 | INFO | Analyzing chunk frames = [1020928,1021440) | 20260507 16:42:59.208 | INFO | MemInfo: { | 20260507 16:42:59.208 | INFO | "gpu" : " 17 GB", | 20260507 16:42:59.208 | INFO | "huge" : "", | 20260507 16:42:59.208 | INFO | "huge_shared" : " 29 GB", | 20260507 16:42:59.208 | INFO | "malloc" : " 14 GB", | 20260507 16:42:59.208 | INFO | "pinned" : " 29 GB" | 20260507 16:42:59.208 | INFO | } | 20260507 16:42:59.208 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:42:59.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:42:59.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.208 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260507 16:42:59.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.208 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:42:59.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:42:59.209 | INFO | Uploaded 36GiB to GPU in 2.56339s (14.0439GiB/s) | 20260507 16:42:59.209 | INFO | Downloaded 34GiB to GPU in 2.42194s (14.0383GiB/s) | 20260507 16:43:04.347 | INFO | Analyzing chunk frames = [1021440,1021952) | 20260507 16:43:04.347 | INFO | MemInfo: { | 20260507 16:43:04.347 | INFO | "gpu" : " 17 GB", | 20260507 16:43:04.347 | INFO | "huge" : "", | 20260507 16:43:04.347 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:04.347 | INFO | "malloc" : " 14 GB", | 20260507 16:43:04.347 | INFO | "pinned" : " 29 GB" | 20260507 16:43:04.347 | INFO | } | 20260507 16:43:04.347 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:43:04.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:04.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.348 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 16:43:04.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.348 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:43:04.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:04.348 | INFO | Uploaded 36GiB to GPU in 2.46987s (14.5757GiB/s) | 20260507 16:43:04.348 | INFO | Downloaded 34GiB to GPU in 2.36818s (14.357GiB/s) | 20260507 16:43:09.487 | INFO | Analyzing chunk frames = [1021952,1022464) | 20260507 16:43:09.487 | INFO | MemInfo: { | 20260507 16:43:09.487 | INFO | "gpu" : " 17 GB", | 20260507 16:43:09.487 | INFO | "huge" : "", | 20260507 16:43:09.487 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:09.487 | INFO | "malloc" : " 14 GB", | 20260507 16:43:09.487 | INFO | "pinned" : " 29 GB" | 20260507 16:43:09.487 | INFO | } | 20260507 16:43:09.487 | INFO | Perf report for the last 5.13971 seconds: | 20260507 16:43:09.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:09.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.487 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:43:09.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.487 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 16:43:09.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:09.488 | INFO | Uploaded 36GiB to GPU in 2.33698s (15.4045GiB/s) | 20260507 16:43:09.488 | INFO | Downloaded 34GiB to GPU in 2.47821s (13.7196GiB/s) | 20260507 16:43:14.626 | INFO | Analyzing chunk frames = [1022464,1022976) | 20260507 16:43:14.626 | INFO | MemInfo: { | 20260507 16:43:14.626 | INFO | "gpu" : " 17 GB", | 20260507 16:43:14.626 | INFO | "huge" : "", | 20260507 16:43:14.626 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:14.627 | INFO | "malloc" : " 14 GB", | 20260507 16:43:14.627 | INFO | "pinned" : " 29 GB" | 20260507 16:43:14.627 | INFO | } | 20260507 16:43:14.627 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:43:14.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:14.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:43:14.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | PRE_HQ : 34.8%, 1.0, 18.0ms, 0 | 20260507 16:43:14.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:14.627 | INFO | Uploaded 36GiB to GPU in 2.29234s (15.7045GiB/s) | 20260507 16:43:14.627 | INFO | Downloaded 34GiB to GPU in 2.4982s (13.6098GiB/s) | 20260507 16:43:19.765 | INFO | Analyzing chunk frames = [1022976,1023488) | 20260507 16:43:19.765 | INFO | MemInfo: { | 20260507 16:43:19.765 | INFO | "gpu" : " 17 GB", | 20260507 16:43:19.765 | INFO | "huge" : "", | 20260507 16:43:19.765 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:19.765 | INFO | "malloc" : " 14 GB", | 20260507 16:43:19.765 | INFO | "pinned" : " 29 GB" | 20260507 16:43:19.765 | INFO | } | 20260507 16:43:19.766 | INFO | Perf report for the last 5.13893 seconds: | 20260507 16:43:19.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:19.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.766 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:43:19.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.766 | INFO | PRE_HQ : 38.4%, 1.0, 20.0ms, 0 | 20260507 16:43:19.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:19.767 | INFO | Uploaded 36GiB to GPU in 2.41942s (14.8796GiB/s) | 20260507 16:43:19.767 | INFO | Downloaded 34GiB to GPU in 2.45171s (13.8679GiB/s) | 20260507 16:43:24.905 | INFO | Analyzing chunk frames = [1023488,1024000) | 20260507 16:43:24.905 | INFO | MemInfo: { | 20260507 16:43:24.905 | INFO | "gpu" : " 17 GB", | 20260507 16:43:24.905 | INFO | "huge" : "", | 20260507 16:43:24.905 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:24.905 | INFO | "malloc" : " 14 GB", | 20260507 16:43:24.905 | INFO | "pinned" : " 29 GB" | 20260507 16:43:24.905 | INFO | } | 20260507 16:43:24.905 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:43:24.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:24.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.905 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:43:24.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.906 | INFO | PRE_HQ : 38.6%, 1.0, 20.1ms, 0 | 20260507 16:43:24.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:24.906 | INFO | Uploaded 36GiB to GPU in 2.45642s (14.6555GiB/s) | 20260507 16:43:24.906 | INFO | Downloaded 34GiB to GPU in 2.511s (13.5404GiB/s) | 20260507 16:43:24.906 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:43:24.906 | INFO | Total mem usage: 16.4 GiB | 20260507 16:43:24.906 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:43:24.906 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.8%) | 20260507 16:43:24.906 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:43:24.906 | INFO | High Water Breakdown: | 20260507 16:43:24.906 | INFO | 59.3% : 8.79 GiB ( 88.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:43:24.906 | INFO | 40.5% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:43:24.906 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:43:24.906 | INFO | ------------------------------------------ | 20260507 16:43:24.906 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:43:24.906 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:43:24.906 | INFO | Total mem usage: 26.5 GiB | 20260507 16:43:24.906 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:43:24.906 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:43:24.906 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:43:24.906 | INFO | High Water Breakdown: | 20260507 16:43:24.906 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:43:24.906 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:43:24.906 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:43:24.906 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:43:24.906 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:43:24.906 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:43:24.906 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:43:24.906 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:43:24.906 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:43:24.906 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:43:24.906 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:43:24.906 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:43:24.906 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:43:24.906 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:43:24.906 | INFO | ------------------------------------------ | 20260507 16:43:24.906 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:43:24.906 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:43:24.906 | INFO | Total mem usage: 27.8 GiB | 20260507 16:43:24.906 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:43:24.906 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:43:24.906 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:43:24.907 | INFO | High Water Breakdown: | 20260507 16:43:24.907 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:43:24.907 | INFO | ------------------------------------------ | 20260507 16:43:24.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:43:24.907 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:43:24.907 | INFO | Total mem usage: 15.7 GiB | 20260507 16:43:24.907 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:43:24.907 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:43:24.907 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:43:24.907 | INFO | High Water Breakdown: | 20260507 16:43:24.907 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:43:24.907 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:43:24.907 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:43:24.907 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:43:24.907 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:43:24.907 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:43:24.907 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:43:24.907 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:43:24.907 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:43:24.907 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:43:24.907 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:43:24.907 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:43:24.907 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:43:24.907 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:43:24.907 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:43:24.907 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:43:24.907 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:43:24.907 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:43:24.907 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:43:24.907 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:43:24.907 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:43:24.907 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:43:24.907 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:43:24.907 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:43:24.907 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:43:24.907 | INFO | ------------------------------------------ | 20260507 16:43:24.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:43:24.907 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:43:24.907 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:43:24.907 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:43:24.907 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.65e+01, 1.42e+01 / 2.87e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 0.30%, AggregateStats -- 9.97e-02 / 1.04e-01, 1.18e-02 / 6.16e-03 | 20260507 16:43:24.907 | INFO | PerfReport: 12.01%, Baseline -- 4.04e+00 / 4.49e+00, 6.22e-01 / 2.60e-01 | 20260507 16:43:24.907 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.13e-01 | 20260507 16:43:24.907 | INFO | PerfReport: 15.70%, DME -- 4.23e+01 / 4.52e+01, 4.50e+00 / 2.73e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.92e+00 / 2.39e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.54e+00 / 2.92e+00, 4.74e-01 / 2.39e-01 | 20260507 16:43:24.907 | INFO | PerfReport: 3.82%, Metrics -- 1.29e+00 / 1.46e+00, 3.27e-01 / 1.76e-01 | 20260507 16:43:24.907 | INFO | PerfReport: 1.23%, PulseSerializing -- 4.12e-01 / 4.37e-01, 6.98e-02 / 4.74e-02 | 20260507 16:43:24.907 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:43:24.907 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:43:24.907 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:43:24.907 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 23.88%, Upload -- 2.56e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 24.03%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 31.34%, Compute -- 3.37e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 1.74%, LockUpload -- 1.87e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 2.25%, LockDownload -- 2.42e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 16.74%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:43:24.907 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 16:43:25.017 | INFO | Starting a round of Baz IO | 20260507 16:43:28.523 | INFO | Populating RTMetrics report | 20260507 16:43:28.523 | INFO | RTMetrics report populated | 20260507 16:43:28.523 | INFO | Opening tmp RTMetrics json file | 20260507 16:43:28.555 | INFO | Serializing RTMetrics to json | 20260507 16:43:28.624 | INFO | Serialization done, swapping tmp file now... | 20260507 16:43:28.630 | INFO | RTMetrics done | 20260507 16:43:28.657 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.64 seconds | 20260507 16:43:28.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.61% / 0.03% / 50.36% | 20260507 16:43:28.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.23688% / 3.97364e-05% / 3.17891e-05% | 20260507 16:43:28.658 | INFO | Baz IO overall: | 20260507 16:43:28.658 | INFO | BytesWritten : 442575491524 | 20260507 16:43:28.658 | INFO | NumEvents : 214227363978 | 20260507 16:43:30.044 | INFO | Analyzing chunk frames = [1024000,1024512) | 20260507 16:43:30.044 | INFO | MemInfo: { | 20260507 16:43:30.044 | INFO | "gpu" : " 17 GB", | 20260507 16:43:30.044 | INFO | "huge" : "", | 20260507 16:43:30.044 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:30.044 | INFO | "malloc" : " 14 GB", | 20260507 16:43:30.044 | INFO | "pinned" : " 29 GB" | 20260507 16:43:30.044 | INFO | } | 20260507 16:43:30.044 | INFO | Perf report for the last 5.13887 seconds: | 20260507 16:43:30.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:30.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:30.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:30.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:30.044 | INFO | ANALYSIS : 69.8%, 2.9, 106.8ms, 0 | 20260507 16:43:30.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:30.044 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 16:43:30.044 | INFO | PRE_HQ : 67.5%, 1.8, 63.7ms, 0 | 20260507 16:43:30.044 | INFO | BAZWRITER : 70.4%, 3.8, 137.3ms, 0 | 20260507 16:43:30.044 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 16:43:30.044 | INFO | Uploaded 36GiB to GPU in 2.55087s (14.1128GiB/s) | 20260507 16:43:30.045 | INFO | Downloaded 38GiB to GPU in 2.7074s (14.0356GiB/s) | 20260507 16:43:35.183 | INFO | Analyzing chunk frames = [1024512,1025024) | 20260507 16:43:35.183 | INFO | MemInfo: { | 20260507 16:43:35.183 | INFO | "gpu" : " 17 GB", | 20260507 16:43:35.183 | INFO | "huge" : "", | 20260507 16:43:35.183 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:35.183 | INFO | "malloc" : " 14 GB", | 20260507 16:43:35.183 | INFO | "pinned" : " 29 GB" | 20260507 16:43:35.183 | INFO | } | 20260507 16:43:35.184 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:43:35.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:35.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:43:35.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 16:43:35.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:35.184 | INFO | Uploaded 36GiB to GPU in 2.4799s (14.5167GiB/s) | 20260507 16:43:35.184 | INFO | Downloaded 34GiB to GPU in 2.46648s (13.7848GiB/s) | 20260507 16:43:40.323 | INFO | Analyzing chunk frames = [1025024,1025536) | 20260507 16:43:40.323 | INFO | MemInfo: { | 20260507 16:43:40.323 | INFO | "gpu" : " 17 GB", | 20260507 16:43:40.323 | INFO | "huge" : "", | 20260507 16:43:40.323 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:40.323 | INFO | "malloc" : " 14 GB", | 20260507 16:43:40.323 | INFO | "pinned" : " 29 GB" | 20260507 16:43:40.323 | INFO | } | 20260507 16:43:40.323 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:43:40.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:40.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260507 16:43:40.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:43:40.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:40.323 | INFO | Uploaded 36GiB to GPU in 2.4053s (14.9669GiB/s) | 20260507 16:43:40.323 | INFO | Downloaded 34GiB to GPU in 2.35441s (14.441GiB/s) | 20260507 16:43:45.462 | INFO | Analyzing chunk frames = [1025536,1026048) | 20260507 16:43:45.462 | INFO | MemInfo: { | 20260507 16:43:45.462 | INFO | "gpu" : " 17 GB", | 20260507 16:43:45.462 | INFO | "huge" : "", | 20260507 16:43:45.462 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:45.462 | INFO | "malloc" : " 14 GB", | 20260507 16:43:45.462 | INFO | "pinned" : " 29 GB" | 20260507 16:43:45.462 | INFO | } | 20260507 16:43:45.462 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:43:45.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:45.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 16:43:45.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:43:45.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:45.462 | INFO | Uploaded 36GiB to GPU in 2.4418s (14.7432GiB/s) | 20260507 16:43:45.462 | INFO | Downloaded 34GiB to GPU in 2.36062s (14.403GiB/s) | 20260507 16:43:50.602 | INFO | Analyzing chunk frames = [1026048,1026560) | 20260507 16:43:50.602 | INFO | MemInfo: { | 20260507 16:43:50.602 | INFO | "gpu" : " 17 GB", | 20260507 16:43:50.602 | INFO | "huge" : "", | 20260507 16:43:50.602 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:50.602 | INFO | "malloc" : " 14 GB", | 20260507 16:43:50.602 | INFO | "pinned" : " 29 GB" | 20260507 16:43:50.602 | INFO | } | 20260507 16:43:50.602 | INFO | Perf report for the last 5.14003 seconds: | 20260507 16:43:50.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:50.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.602 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:43:50.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.602 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 16:43:50.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:50.603 | INFO | Uploaded 36GiB to GPU in 2.33638s (15.4085GiB/s) | 20260507 16:43:50.603 | INFO | Downloaded 34GiB to GPU in 2.49042s (13.6523GiB/s) | 20260507 16:43:55.741 | INFO | Analyzing chunk frames = [1026560,1027072) | 20260507 16:43:55.741 | INFO | MemInfo: { | 20260507 16:43:55.741 | INFO | "gpu" : " 17 GB", | 20260507 16:43:55.741 | INFO | "huge" : "", | 20260507 16:43:55.741 | INFO | "huge_shared" : " 29 GB", | 20260507 16:43:55.741 | INFO | "malloc" : " 14 GB", | 20260507 16:43:55.741 | INFO | "pinned" : " 29 GB" | 20260507 16:43:55.741 | INFO | } | 20260507 16:43:55.741 | INFO | Perf report for the last 5.13904 seconds: | 20260507 16:43:55.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:43:55.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.741 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:43:55.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.741 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 16:43:55.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:43:55.742 | INFO | Uploaded 36GiB to GPU in 2.32014s (15.5163GiB/s) | 20260507 16:43:55.742 | INFO | Downloaded 34GiB to GPU in 2.51298s (13.5297GiB/s) | 20260507 16:44:00.880 | INFO | Analyzing chunk frames = [1027072,1027584) | 20260507 16:44:00.880 | INFO | MemInfo: { | 20260507 16:44:00.880 | INFO | "gpu" : " 17 GB", | 20260507 16:44:00.880 | INFO | "huge" : "", | 20260507 16:44:00.880 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:00.880 | INFO | "malloc" : " 14 GB", | 20260507 16:44:00.880 | INFO | "pinned" : " 29 GB" | 20260507 16:44:00.880 | INFO | } | 20260507 16:44:00.881 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:44:00.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:00.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 16:44:00.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 16:44:00.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:00.881 | INFO | Uploaded 36GiB to GPU in 2.361s (15.2478GiB/s) | 20260507 16:44:00.881 | INFO | Downloaded 34GiB to GPU in 2.42676s (14.0105GiB/s) | 20260507 16:44:06.019 | INFO | Analyzing chunk frames = [1027584,1028096) | 20260507 16:44:06.019 | INFO | MemInfo: { | 20260507 16:44:06.019 | INFO | "gpu" : " 17 GB", | 20260507 16:44:06.019 | INFO | "huge" : "", | 20260507 16:44:06.019 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:06.020 | INFO | "malloc" : " 14 GB", | 20260507 16:44:06.020 | INFO | "pinned" : " 29 GB" | 20260507 16:44:06.020 | INFO | } | 20260507 16:44:06.020 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:44:06.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:06.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 16:44:06.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | PRE_HQ : 37.8%, 1.0, 19.7ms, 0 | 20260507 16:44:06.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:06.020 | INFO | Uploaded 36GiB to GPU in 2.47079s (14.5702GiB/s) | 20260507 16:44:06.020 | INFO | Downloaded 34GiB to GPU in 2.51287s (13.5303GiB/s) | 20260507 16:44:06.132 | INFO | Starting a round of Baz IO | 20260507 16:44:09.632 | INFO | Populating RTMetrics report | 20260507 16:44:09.632 | INFO | RTMetrics report populated | 20260507 16:44:09.632 | INFO | Opening tmp RTMetrics json file | 20260507 16:44:09.646 | INFO | Serializing RTMetrics to json | 20260507 16:44:09.654 | INFO | Serialization done, swapping tmp file now... | 20260507 16:44:09.654 | INFO | RTMetrics done | 20260507 16:44:09.739 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.607 seconds | 20260507 16:44:09.739 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.58% / 0.03% / 50.39% | 20260507 16:44:09.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.22812% / 3.97364e-05% / 5.16574e-05% | 20260507 16:44:09.739 | INFO | Baz IO overall: | 20260507 16:44:09.739 | INFO | BytesWritten : 444498147396 | 20260507 16:44:09.739 | INFO | NumEvents : 215162467837 | 20260507 16:44:11.159 | INFO | Analyzing chunk frames = [1028096,1028608) | 20260507 16:44:11.159 | INFO | MemInfo: { | 20260507 16:44:11.159 | INFO | "gpu" : " 17 GB", | 20260507 16:44:11.159 | INFO | "huge" : "", | 20260507 16:44:11.159 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:11.159 | INFO | "malloc" : " 14 GB", | 20260507 16:44:11.159 | INFO | "pinned" : " 29 GB" | 20260507 16:44:11.159 | INFO | } | 20260507 16:44:11.159 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:44:11.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:11.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:11.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:11.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:11.159 | INFO | ANALYSIS : 69.8%, 2.9, 107.5ms, 0 | 20260507 16:44:11.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:11.159 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 16:44:11.159 | INFO | PRE_HQ : 66.9%, 1.8, 62.5ms, 0 | 20260507 16:44:11.159 | INFO | BAZWRITER : 69.7%, 4.1, 148.0ms, 0 | 20260507 16:44:11.159 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:44:11.160 | INFO | Uploaded 36GiB to GPU in 2.5413s (14.166GiB/s) | 20260507 16:44:11.160 | INFO | Downloaded 38GiB to GPU in 2.71734s (13.9843GiB/s) | 20260507 16:44:16.298 | INFO | Analyzing chunk frames = [1028608,1029120) | 20260507 16:44:16.298 | INFO | MemInfo: { | 20260507 16:44:16.298 | INFO | "gpu" : " 17 GB", | 20260507 16:44:16.298 | INFO | "huge" : "", | 20260507 16:44:16.298 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:16.298 | INFO | "malloc" : " 14 GB", | 20260507 16:44:16.298 | INFO | "pinned" : " 29 GB" | 20260507 16:44:16.298 | INFO | } | 20260507 16:44:16.298 | INFO | Perf report for the last 5.1393 seconds: | 20260507 16:44:16.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:16.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.298 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 16:44:16.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.299 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 16:44:16.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:16.299 | INFO | Uploaded 36GiB to GPU in 2.50491s (14.3718GiB/s) | 20260507 16:44:16.299 | INFO | Downloaded 34GiB to GPU in 2.44729s (13.8929GiB/s) | 20260507 16:44:21.437 | INFO | Analyzing chunk frames = [1029120,1029632) | 20260507 16:44:21.437 | INFO | MemInfo: { | 20260507 16:44:21.437 | INFO | "gpu" : " 17 GB", | 20260507 16:44:21.437 | INFO | "huge" : "", | 20260507 16:44:21.437 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:21.437 | INFO | "malloc" : " 14 GB", | 20260507 16:44:21.437 | INFO | "pinned" : " 29 GB" | 20260507 16:44:21.437 | INFO | } | 20260507 16:44:21.437 | INFO | Perf report for the last 5.13905 seconds: | 20260507 16:44:21.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:21.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.438 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:44:21.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.438 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:44:21.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:21.438 | INFO | Uploaded 36GiB to GPU in 2.55773s (14.075GiB/s) | 20260507 16:44:21.438 | INFO | Downloaded 34GiB to GPU in 2.42755s (14.0059GiB/s) | 20260507 16:44:26.577 | INFO | Analyzing chunk frames = [1029632,1030144) | 20260507 16:44:26.577 | INFO | MemInfo: { | 20260507 16:44:26.577 | INFO | "gpu" : " 17 GB", | 20260507 16:44:26.577 | INFO | "huge" : "", | 20260507 16:44:26.577 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:26.577 | INFO | "malloc" : " 14 GB", | 20260507 16:44:26.577 | INFO | "pinned" : " 29 GB" | 20260507 16:44:26.577 | INFO | } | 20260507 16:44:26.577 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:44:26.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:26.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 16:44:26.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:44:26.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:26.577 | INFO | Uploaded 36GiB to GPU in 2.46184s (14.6232GiB/s) | 20260507 16:44:26.577 | INFO | Downloaded 34GiB to GPU in 2.35733s (14.4231GiB/s) | 20260507 16:44:31.716 | INFO | Analyzing chunk frames = [1030144,1030656) | 20260507 16:44:31.716 | INFO | MemInfo: { | 20260507 16:44:31.716 | INFO | "gpu" : " 17 GB", | 20260507 16:44:31.717 | INFO | "huge" : "", | 20260507 16:44:31.717 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:31.717 | INFO | "malloc" : " 14 GB", | 20260507 16:44:31.717 | INFO | "pinned" : " 29 GB" | 20260507 16:44:31.717 | INFO | } | 20260507 16:44:31.717 | INFO | Perf report for the last 5.14003 seconds: | 20260507 16:44:31.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:31.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260507 16:44:31.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260507 16:44:31.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:31.717 | INFO | Uploaded 36GiB to GPU in 2.34578s (15.3467GiB/s) | 20260507 16:44:31.717 | INFO | Downloaded 34GiB to GPU in 2.4745s (13.7401GiB/s) | 20260507 16:44:36.855 | INFO | Analyzing chunk frames = [1030656,1031168) | 20260507 16:44:36.855 | INFO | MemInfo: { | 20260507 16:44:36.855 | INFO | "gpu" : " 17 GB", | 20260507 16:44:36.856 | INFO | "huge" : "", | 20260507 16:44:36.856 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:36.856 | INFO | "malloc" : " 14 GB", | 20260507 16:44:36.856 | INFO | "pinned" : " 29 GB" | 20260507 16:44:36.856 | INFO | } | 20260507 16:44:36.856 | INFO | Perf report for the last 5.13891 seconds: | 20260507 16:44:36.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:36.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:44:36.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 16:44:36.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:36.856 | INFO | Uploaded 36GiB to GPU in 2.38067s (15.1218GiB/s) | 20260507 16:44:36.857 | INFO | Downloaded 34GiB to GPU in 2.43598s (13.9574GiB/s) | 20260507 16:44:41.995 | INFO | Analyzing chunk frames = [1031168,1031680) | 20260507 16:44:41.995 | INFO | MemInfo: { | 20260507 16:44:41.995 | INFO | "gpu" : " 17 GB", | 20260507 16:44:41.995 | INFO | "huge" : "", | 20260507 16:44:41.995 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:41.995 | INFO | "malloc" : " 14 GB", | 20260507 16:44:41.995 | INFO | "pinned" : " 29 GB" | 20260507 16:44:41.995 | INFO | } | 20260507 16:44:41.995 | INFO | Perf report for the last 5.13906 seconds: | 20260507 16:44:41.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:41.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:44:41.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 16:44:41.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:41.995 | INFO | Uploaded 36GiB to GPU in 2.49867s (14.4077GiB/s) | 20260507 16:44:41.995 | INFO | Downloaded 34GiB to GPU in 2.41896s (14.0556GiB/s) | 20260507 16:44:47.134 | INFO | Analyzing chunk frames = [1031680,1032192) | 20260507 16:44:47.134 | INFO | MemInfo: { | 20260507 16:44:47.134 | INFO | "gpu" : " 17 GB", | 20260507 16:44:47.134 | INFO | "huge" : "", | 20260507 16:44:47.134 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:47.134 | INFO | "malloc" : " 14 GB", | 20260507 16:44:47.134 | INFO | "pinned" : " 29 GB" | 20260507 16:44:47.134 | INFO | } | 20260507 16:44:47.135 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:44:47.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:47.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 16:44:47.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:44:47.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:47.135 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:44:47.135 | INFO | Total mem usage: 16.4 GiB | 20260507 16:44:47.135 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:44:47.135 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.5%) | 20260507 16:44:47.135 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:44:47.135 | INFO | High Water Breakdown: | 20260507 16:44:47.135 | INFO | 59.2% : 8.71 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:44:47.135 | INFO | 40.7% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:44:47.135 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:44:47.135 | INFO | Uploaded 36GiB to GPU in 2.44975s (14.6954GiB/s) | 20260507 16:44:47.135 | INFO | ------------------------------------------ | 20260507 16:44:47.135 | INFO | Downloaded 34GiB to GPU in 2.49488s (13.6279GiB/s) | 20260507 16:44:47.135 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:44:47.135 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:44:47.135 | INFO | Total mem usage: 26.5 GiB | 20260507 16:44:47.135 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:44:47.135 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:44:47.135 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 16:44:47.135 | INFO | High Water Breakdown: | 20260507 16:44:47.135 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:44:47.135 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:44:47.135 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:44:47.135 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:44:47.135 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:44:47.135 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:44:47.135 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:44:47.135 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:44:47.136 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:44:47.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:44:47.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:44:47.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:44:47.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:44:47.136 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:44:47.136 | INFO | ------------------------------------------ | 20260507 16:44:47.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:44:47.136 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:44:47.136 | INFO | Total mem usage: 27.8 GiB | 20260507 16:44:47.136 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:44:47.136 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:44:47.136 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:44:47.136 | INFO | High Water Breakdown: | 20260507 16:44:47.136 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:44:47.136 | INFO | ------------------------------------------ | 20260507 16:44:47.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:44:47.136 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:44:47.136 | INFO | Total mem usage: 15.7 GiB | 20260507 16:44:47.136 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:44:47.136 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:44:47.136 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:44:47.136 | INFO | High Water Breakdown: | 20260507 16:44:47.136 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:44:47.136 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:44:47.136 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:44:47.136 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:44:47.136 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:44:47.136 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:44:47.136 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:44:47.136 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:44:47.136 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:44:47.136 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:44:47.136 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:44:47.136 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:44:47.136 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:44:47.136 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:44:47.136 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:44:47.136 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:44:47.136 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:44:47.136 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:44:47.136 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:44:47.136 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:44:47.136 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:44:47.136 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:44:47.136 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:44:47.136 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:44:47.136 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:44:47.136 | INFO | ------------------------------------------ | 20260507 16:44:47.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:44:47.136 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:44:47.136 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:44:47.136 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:44:47.136 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.66e+01, 1.45e+01 / 2.72e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.04e-01, 1.25e-02 / 6.70e-03 | 20260507 16:44:47.136 | INFO | PerfReport: 12.06%, Baseline -- 4.06e+00 / 4.53e+00, 6.25e-01 / 1.97e-01 | 20260507 16:44:47.136 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.07e-01 | 20260507 16:44:47.136 | INFO | PerfReport: 15.71%, DME -- 4.23e+01 / 4.52e+01, 4.40e+00 / 2.47e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 55.86%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.88e+00 / 2.12e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.55e+00 / 2.94e+00, 4.40e-01 / 2.35e-01 | 20260507 16:44:47.136 | INFO | PerfReport: 3.84%, Metrics -- 1.29e+00 / 1.47e+00, 3.28e-01 / 1.97e-01 | 20260507 16:44:47.136 | INFO | PerfReport: 1.23%, PulseSerializing -- 4.13e-01 / 4.38e-01, 6.83e-02 / 4.33e-02 | 20260507 16:44:47.136 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:44:47.136 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:44:47.136 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:44:47.136 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 23.73%, Upload -- 2.55e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 24.03%, Download -- 2.58e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 31.37%, Compute -- 3.37e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 1.65%, LockUpload -- 1.78e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 2.42%, LockDownload -- 2.59e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 16.78%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:44:47.136 | INFO | PerfReport: 0.01%, Misc -- 1.57e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 16:44:47.231 | INFO | Starting a round of Baz IO | 20260507 16:44:50.742 | INFO | Populating RTMetrics report | 20260507 16:44:50.742 | INFO | RTMetrics report populated | 20260507 16:44:50.742 | INFO | Opening tmp RTMetrics json file | 20260507 16:44:50.780 | INFO | Serializing RTMetrics to json | 20260507 16:44:50.792 | INFO | Serialization done, swapping tmp file now... | 20260507 16:44:50.806 | INFO | RTMetrics done | 20260507 16:44:50.853 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.621 seconds | 20260507 16:44:50.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.56% / 0.03% / 50.42% | 20260507 16:44:50.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.20461% / 5.16574e-05% / 2.38419e-05% | 20260507 16:44:50.853 | INFO | Baz IO overall: | 20260507 16:44:50.853 | INFO | BytesWritten : 446419089398 | 20260507 16:44:50.853 | INFO | NumEvents : 216096205495 | 20260507 16:44:52.274 | INFO | Analyzing chunk frames = [1032192,1032704) | 20260507 16:44:52.274 | INFO | MemInfo: { | 20260507 16:44:52.274 | INFO | "gpu" : " 17 GB", | 20260507 16:44:52.274 | INFO | "huge" : "", | 20260507 16:44:52.274 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:52.274 | INFO | "malloc" : " 14 GB", | 20260507 16:44:52.274 | INFO | "pinned" : " 29 GB" | 20260507 16:44:52.274 | INFO | } | 20260507 16:44:52.274 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:44:52.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:52.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:52.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:52.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:52.274 | INFO | ANALYSIS : 69.7%, 2.9, 107.8ms, 0 | 20260507 16:44:52.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:52.274 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 16:44:52.274 | INFO | PRE_HQ : 66.7%, 1.8, 60.6ms, 0 | 20260507 16:44:52.274 | INFO | BAZWRITER : 70.0%, 3.9, 139.3ms, 0 | 20260507 16:44:52.274 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:44:52.274 | INFO | Uploaded 36GiB to GPU in 2.55706s (14.0787GiB/s) | 20260507 16:44:52.274 | INFO | Downloaded 38GiB to GPU in 2.70954s (14.0245GiB/s) | 20260507 16:44:57.413 | INFO | Analyzing chunk frames = [1032704,1033216) | 20260507 16:44:57.413 | INFO | MemInfo: { | 20260507 16:44:57.413 | INFO | "gpu" : " 17 GB", | 20260507 16:44:57.413 | INFO | "huge" : "", | 20260507 16:44:57.413 | INFO | "huge_shared" : " 29 GB", | 20260507 16:44:57.413 | INFO | "malloc" : " 14 GB", | 20260507 16:44:57.413 | INFO | "pinned" : " 29 GB" | 20260507 16:44:57.413 | INFO | } | 20260507 16:44:57.413 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:44:57.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:44:57.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:44:57.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 16:44:57.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:44:57.413 | INFO | Uploaded 36GiB to GPU in 2.51561s (14.3106GiB/s) | 20260507 16:44:57.413 | INFO | Downloaded 34GiB to GPU in 2.45394s (13.8553GiB/s) | 20260507 16:45:02.553 | INFO | Analyzing chunk frames = [1033216,1033728) | 20260507 16:45:02.553 | INFO | MemInfo: { | 20260507 16:45:02.553 | INFO | "gpu" : " 17 GB", | 20260507 16:45:02.553 | INFO | "huge" : "", | 20260507 16:45:02.553 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:02.553 | INFO | "malloc" : " 14 GB", | 20260507 16:45:02.553 | INFO | "pinned" : " 29 GB" | 20260507 16:45:02.553 | INFO | } | 20260507 16:45:02.553 | INFO | Perf report for the last 5.13992 seconds: | 20260507 16:45:02.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:02.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 16:45:02.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:45:02.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:02.553 | INFO | Uploaded 36GiB to GPU in 2.51971s (14.2874GiB/s) | 20260507 16:45:02.553 | INFO | Downloaded 34GiB to GPU in 2.45445s (13.8524GiB/s) | 20260507 16:45:07.692 | INFO | Analyzing chunk frames = [1033728,1034240) | 20260507 16:45:07.692 | INFO | MemInfo: { | 20260507 16:45:07.692 | INFO | "gpu" : " 17 GB", | 20260507 16:45:07.692 | INFO | "huge" : "", | 20260507 16:45:07.692 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:07.692 | INFO | "malloc" : " 14 GB", | 20260507 16:45:07.692 | INFO | "pinned" : " 29 GB" | 20260507 16:45:07.692 | INFO | } | 20260507 16:45:07.692 | INFO | Perf report for the last 5.13889 seconds: | 20260507 16:45:07.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:07.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.692 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 16:45:07.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.692 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:45:07.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:07.693 | INFO | Uploaded 36GiB to GPU in 2.46774s (14.5882GiB/s) | 20260507 16:45:07.693 | INFO | Downloaded 34GiB to GPU in 2.36685s (14.3651GiB/s) | 20260507 16:45:12.831 | INFO | Analyzing chunk frames = [1034240,1034752) | 20260507 16:45:12.831 | INFO | MemInfo: { | 20260507 16:45:12.831 | INFO | "gpu" : " 17 GB", | 20260507 16:45:12.832 | INFO | "huge" : "", | 20260507 16:45:12.832 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:12.832 | INFO | "malloc" : " 14 GB", | 20260507 16:45:12.832 | INFO | "pinned" : " 29 GB" | 20260507 16:45:12.832 | INFO | } | 20260507 16:45:12.832 | INFO | Perf report for the last 5.14006 seconds: | 20260507 16:45:12.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:12.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 16:45:12.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 16:45:12.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:12.832 | INFO | Uploaded 36GiB to GPU in 2.38676s (15.0832GiB/s) | 20260507 16:45:12.833 | INFO | Downloaded 34GiB to GPU in 2.47116s (13.7587GiB/s) | 20260507 16:45:17.970 | INFO | Analyzing chunk frames = [1034752,1035264) | 20260507 16:45:17.970 | INFO | MemInfo: { | 20260507 16:45:17.970 | INFO | "gpu" : " 17 GB", | 20260507 16:45:17.970 | INFO | "huge" : "", | 20260507 16:45:17.970 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:17.970 | INFO | "malloc" : " 14 GB", | 20260507 16:45:17.970 | INFO | "pinned" : " 29 GB" | 20260507 16:45:17.970 | INFO | } | 20260507 16:45:17.970 | INFO | Perf report for the last 5.13849 seconds: | 20260507 16:45:17.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:17.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.970 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:45:17.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.970 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 16:45:17.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:17.971 | INFO | Uploaded 36GiB to GPU in 2.38029s (15.1242GiB/s) | 20260507 16:45:17.971 | INFO | Downloaded 34GiB to GPU in 2.50065s (13.5965GiB/s) | 20260507 16:45:23.109 | INFO | Analyzing chunk frames = [1035264,1035776) | 20260507 16:45:23.109 | INFO | MemInfo: { | 20260507 16:45:23.109 | INFO | "gpu" : " 17 GB", | 20260507 16:45:23.110 | INFO | "huge" : "", | 20260507 16:45:23.110 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:23.110 | INFO | "malloc" : " 14 GB", | 20260507 16:45:23.110 | INFO | "pinned" : " 29 GB" | 20260507 16:45:23.110 | INFO | } | 20260507 16:45:23.110 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:45:23.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:23.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 16:45:23.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:45:23.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:23.110 | INFO | Uploaded 36GiB to GPU in 2.41357s (14.9157GiB/s) | 20260507 16:45:23.110 | INFO | Downloaded 34GiB to GPU in 2.41693s (14.0674GiB/s) | 20260507 16:45:28.249 | INFO | Analyzing chunk frames = [1035776,1036288) | 20260507 16:45:28.249 | INFO | MemInfo: { | 20260507 16:45:28.249 | INFO | "gpu" : " 17 GB", | 20260507 16:45:28.249 | INFO | "huge" : "", | 20260507 16:45:28.249 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:28.249 | INFO | "malloc" : " 14 GB", | 20260507 16:45:28.249 | INFO | "pinned" : " 29 GB" | 20260507 16:45:28.249 | INFO | } | 20260507 16:45:28.249 | INFO | Perf report for the last 5.13946 seconds: | 20260507 16:45:28.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:28.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.249 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:45:28.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.249 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:45:28.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:28.250 | INFO | Uploaded 36GiB to GPU in 2.49053s (14.4548GiB/s) | 20260507 16:45:28.250 | INFO | Downloaded 34GiB to GPU in 2.50672s (13.5636GiB/s) | 20260507 16:45:28.346 | INFO | Starting a round of Baz IO | 20260507 16:45:31.864 | INFO | Populating RTMetrics report | 20260507 16:45:31.865 | INFO | RTMetrics report populated | 20260507 16:45:31.865 | INFO | Opening tmp RTMetrics json file | 20260507 16:45:31.871 | INFO | Serializing RTMetrics to json | 20260507 16:45:31.889 | INFO | Serialization done, swapping tmp file now... | 20260507 16:45:31.889 | INFO | RTMetrics done | 20260507 16:45:31.966 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.62 seconds | 20260507 16:45:31.966 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.53% / 0.03% / 50.44% | 20260507 16:45:31.966 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.20339% / 5.16574e-05% / 2.78155e-05% | 20260507 16:45:31.966 | INFO | Baz IO overall: | 20260507 16:45:31.966 | INFO | BytesWritten : 448341711129 | 20260507 16:45:31.966 | INFO | NumEvents : 217031280933 | 20260507 16:45:33.388 | INFO | Analyzing chunk frames = [1036288,1036800) | 20260507 16:45:33.388 | INFO | MemInfo: { | 20260507 16:45:33.388 | INFO | "gpu" : " 17 GB", | 20260507 16:45:33.388 | INFO | "huge" : "", | 20260507 16:45:33.388 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:33.389 | INFO | "malloc" : " 14 GB", | 20260507 16:45:33.389 | INFO | "pinned" : " 29 GB" | 20260507 16:45:33.389 | INFO | } | 20260507 16:45:33.389 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:45:33.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:33.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:33.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:33.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:33.389 | INFO | ANALYSIS : 69.9%, 2.9, 107.7ms, 0 | 20260507 16:45:33.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:33.389 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 16:45:33.389 | INFO | PRE_HQ : 66.9%, 1.8, 60.9ms, 0 | 20260507 16:45:33.389 | INFO | BAZWRITER : 70.0%, 3.9, 138.6ms, 0 | 20260507 16:45:33.389 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 16:45:33.390 | INFO | Uploaded 36GiB to GPU in 2.54381s (14.152GiB/s) | 20260507 16:45:33.390 | INFO | Downloaded 38GiB to GPU in 2.69946s (14.0769GiB/s) | 20260507 16:45:38.528 | INFO | Analyzing chunk frames = [1036800,1037312) | 20260507 16:45:38.528 | INFO | MemInfo: { | 20260507 16:45:38.528 | INFO | "gpu" : " 17 GB", | 20260507 16:45:38.528 | INFO | "huge" : "", | 20260507 16:45:38.528 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:38.528 | INFO | "malloc" : " 14 GB", | 20260507 16:45:38.528 | INFO | "pinned" : " 29 GB" | 20260507 16:45:38.528 | INFO | } | 20260507 16:45:38.528 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:45:38.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:38.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.528 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:45:38.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.529 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 16:45:38.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:38.529 | INFO | Uploaded 36GiB to GPU in 2.54538s (14.1433GiB/s) | 20260507 16:45:38.529 | INFO | Downloaded 34GiB to GPU in 2.41898s (14.0555GiB/s) | 20260507 16:45:43.667 | INFO | Analyzing chunk frames = [1037312,1037824) | 20260507 16:45:43.667 | INFO | MemInfo: { | 20260507 16:45:43.667 | INFO | "gpu" : " 17 GB", | 20260507 16:45:43.667 | INFO | "huge" : "", | 20260507 16:45:43.667 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:43.668 | INFO | "malloc" : " 14 GB", | 20260507 16:45:43.668 | INFO | "pinned" : " 29 GB" | 20260507 16:45:43.668 | INFO | } | 20260507 16:45:43.668 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:45:43.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:43.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260507 16:45:43.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | PRE_HQ : 37.8%, 1.0, 19.6ms, 0 | 20260507 16:45:43.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:43.668 | INFO | Uploaded 36GiB to GPU in 2.56712s (14.0235GiB/s) | 20260507 16:45:43.668 | INFO | Downloaded 34GiB to GPU in 2.41421s (14.0833GiB/s) | 20260507 16:45:48.807 | INFO | Analyzing chunk frames = [1037824,1038336) | 20260507 16:45:48.807 | INFO | MemInfo: { | 20260507 16:45:48.807 | INFO | "gpu" : " 17 GB", | 20260507 16:45:48.807 | INFO | "huge" : "", | 20260507 16:45:48.807 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:48.807 | INFO | "malloc" : " 14 GB", | 20260507 16:45:48.807 | INFO | "pinned" : " 29 GB" | 20260507 16:45:48.807 | INFO | } | 20260507 16:45:48.807 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:45:48.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:48.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 16:45:48.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:45:48.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:48.807 | INFO | Uploaded 36GiB to GPU in 2.49479s (14.43GiB/s) | 20260507 16:45:48.807 | INFO | Downloaded 34GiB to GPU in 2.35962s (14.4091GiB/s) | 20260507 16:45:53.946 | INFO | Analyzing chunk frames = [1038336,1038848) | 20260507 16:45:53.946 | INFO | MemInfo: { | 20260507 16:45:53.947 | INFO | "gpu" : " 17 GB", | 20260507 16:45:53.947 | INFO | "huge" : "", | 20260507 16:45:53.947 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:53.947 | INFO | "malloc" : " 14 GB", | 20260507 16:45:53.947 | INFO | "pinned" : " 29 GB" | 20260507 16:45:53.947 | INFO | } | 20260507 16:45:53.947 | INFO | Perf report for the last 5.14008 seconds: | 20260507 16:45:53.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:53.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:45:53.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 16:45:53.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:53.947 | INFO | Uploaded 36GiB to GPU in 2.34363s (15.3608GiB/s) | 20260507 16:45:53.948 | INFO | Downloaded 34GiB to GPU in 2.46969s (13.7669GiB/s) | 20260507 16:45:59.085 | INFO | Analyzing chunk frames = [1038848,1039360) | 20260507 16:45:59.085 | INFO | MemInfo: { | 20260507 16:45:59.085 | INFO | "gpu" : " 17 GB", | 20260507 16:45:59.085 | INFO | "huge" : "", | 20260507 16:45:59.085 | INFO | "huge_shared" : " 29 GB", | 20260507 16:45:59.085 | INFO | "malloc" : " 14 GB", | 20260507 16:45:59.085 | INFO | "pinned" : " 29 GB" | 20260507 16:45:59.085 | INFO | } | 20260507 16:45:59.086 | INFO | Perf report for the last 5.13858 seconds: | 20260507 16:45:59.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:45:59.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:45:59.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260507 16:45:59.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:45:59.086 | INFO | Uploaded 36GiB to GPU in 2.38612s (15.0873GiB/s) | 20260507 16:45:59.086 | INFO | Downloaded 34GiB to GPU in 2.48741s (13.6689GiB/s) | 20260507 16:46:04.225 | INFO | Analyzing chunk frames = [1039360,1039872) | 20260507 16:46:04.225 | INFO | MemInfo: { | 20260507 16:46:04.225 | INFO | "gpu" : " 17 GB", | 20260507 16:46:04.225 | INFO | "huge" : "", | 20260507 16:46:04.225 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:04.225 | INFO | "malloc" : " 14 GB", | 20260507 16:46:04.225 | INFO | "pinned" : " 29 GB" | 20260507 16:46:04.225 | INFO | } | 20260507 16:46:04.225 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:46:04.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:04.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 16:46:04.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 16:46:04.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:04.225 | INFO | Uploaded 36GiB to GPU in 2.42786s (14.8279GiB/s) | 20260507 16:46:04.225 | INFO | Downloaded 34GiB to GPU in 2.41857s (14.0579GiB/s) | 20260507 16:46:09.364 | INFO | Analyzing chunk frames = [1039872,1040384) | 20260507 16:46:09.364 | INFO | MemInfo: { | 20260507 16:46:09.364 | INFO | "gpu" : " 17 GB", | 20260507 16:46:09.364 | INFO | "huge" : "", | 20260507 16:46:09.364 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:09.364 | INFO | "malloc" : " 14 GB", | 20260507 16:46:09.364 | INFO | "pinned" : " 29 GB" | 20260507 16:46:09.364 | INFO | } | 20260507 16:46:09.364 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:46:09.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:09.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.364 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:46:09.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.364 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:46:09.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:09.365 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:46:09.365 | INFO | Total mem usage: 16.4 GiB | 20260507 16:46:09.365 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:46:09.365 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10%) | 20260507 16:46:09.365 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:46:09.365 | INFO | High Water Breakdown: | 20260507 16:46:09.365 | INFO | 59.5% : 8.87 GiB ( 89.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:46:09.365 | INFO | 40.4% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:46:09.365 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:46:09.365 | INFO | ------------------------------------------ | 20260507 16:46:09.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:46:09.365 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:46:09.365 | INFO | Total mem usage: 26.5 GiB | 20260507 16:46:09.365 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:46:09.365 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 16:46:09.365 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:46:09.365 | INFO | High Water Breakdown: | 20260507 16:46:09.365 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:46:09.365 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:46:09.365 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:46:09.365 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:46:09.365 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:46:09.365 | INFO | Uploaded 36GiB to GPU in 2.49158s (14.4487GiB/s) | 20260507 16:46:09.365 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:46:09.365 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:46:09.365 | INFO | Downloaded 34GiB to GPU in 2.50642s (13.5651GiB/s) | 20260507 16:46:09.365 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:46:09.365 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:46:09.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:46:09.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:46:09.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:46:09.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:46:09.365 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:46:09.365 | INFO | ------------------------------------------ | 20260507 16:46:09.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:46:09.365 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:46:09.365 | INFO | Total mem usage: 27.8 GiB | 20260507 16:46:09.365 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:46:09.365 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:46:09.365 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:46:09.365 | INFO | High Water Breakdown: | 20260507 16:46:09.365 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:46:09.365 | INFO | ------------------------------------------ | 20260507 16:46:09.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:46:09.365 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:46:09.365 | INFO | Total mem usage: 15.7 GiB | 20260507 16:46:09.365 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:46:09.365 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:46:09.365 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:46:09.365 | INFO | High Water Breakdown: | 20260507 16:46:09.365 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:46:09.365 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:46:09.365 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:46:09.365 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:46:09.365 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:46:09.365 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:46:09.365 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:46:09.365 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:46:09.365 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:46:09.365 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:46:09.365 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:46:09.365 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:46:09.365 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:46:09.365 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:46:09.365 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:46:09.365 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:46:09.365 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:46:09.365 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:46:09.365 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:46:09.365 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:46:09.365 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:46:09.365 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:46:09.365 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:46:09.365 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:46:09.365 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:46:09.365 | INFO | ------------------------------------------ | 20260507 16:46:09.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:46:09.365 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:46:09.365 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:46:09.365 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:46:09.365 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.44e+01 / 3.02e+00 | 20260507 16:46:09.365 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.03e-01, 1.30e-02 / 6.38e-03 | 20260507 16:46:09.365 | INFO | PerfReport: 12.06%, Baseline -- 4.06e+00 / 4.50e+00, 6.28e-01 / 2.47e-01 | 20260507 16:46:09.365 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.20e-01 / 1.11e-01 | 20260507 16:46:09.365 | INFO | PerfReport: 15.67%, DME -- 4.22e+01 / 4.50e+01, 4.60e+00 / 2.99e+00 | 20260507 16:46:09.365 | INFO | PerfReport: 55.91%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.88e+00 / 2.23e+00 | 20260507 16:46:09.365 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.54e+00 / 2.90e+00, 4.41e-01 / 2.49e-01 | 20260507 16:46:09.365 | INFO | PerfReport: 3.85%, Metrics -- 1.29e+00 / 1.46e+00, 3.29e-01 / 1.88e-01 | 20260507 16:46:09.365 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.11e-01 / 4.34e-01, 6.86e-02 / 4.20e-02 | 20260507 16:46:09.365 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:46:09.366 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:46:09.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:46:09.366 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 23.93%, Upload -- 2.57e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 24.01%, Download -- 2.58e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 31.28%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 1.76%, LockUpload -- 1.89e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 2.27%, LockDownload -- 2.44e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 16.74%, LockCompute -- 1.80e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 16:46:09.366 | INFO | PerfReport: 0.01%, Misc -- 1.54e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 16:46:09.461 | INFO | Starting a round of Baz IO | 20260507 16:46:12.968 | INFO | Populating RTMetrics report | 20260507 16:46:12.968 | INFO | RTMetrics report populated | 20260507 16:46:12.968 | INFO | Opening tmp RTMetrics json file | 20260507 16:46:12.971 | INFO | Serializing RTMetrics to json | 20260507 16:46:12.994 | INFO | Serialization done, swapping tmp file now... | 20260507 16:46:13.002 | INFO | RTMetrics done | 20260507 16:46:13.097 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.635 seconds | 20260507 16:46:13.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.51% / 0.03% / 50.47% | 20260507 16:46:13.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.19738% / 6.35783e-05% / 2.78155e-05% | 20260507 16:46:13.097 | INFO | Baz IO overall: | 20260507 16:46:13.097 | INFO | BytesWritten : 450266050380 | 20260507 16:46:13.097 | INFO | NumEvents : 217967464282 | 20260507 16:46:14.504 | INFO | Analyzing chunk frames = [1040384,1040896) | 20260507 16:46:14.504 | INFO | MemInfo: { | 20260507 16:46:14.504 | INFO | "gpu" : " 17 GB", | 20260507 16:46:14.504 | INFO | "huge" : "", | 20260507 16:46:14.504 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:14.504 | INFO | "malloc" : " 14 GB", | 20260507 16:46:14.504 | INFO | "pinned" : " 29 GB" | 20260507 16:46:14.504 | INFO | } | 20260507 16:46:14.504 | INFO | Perf report for the last 5.1396 seconds: | 20260507 16:46:14.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:14.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:14.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:14.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:14.504 | INFO | ANALYSIS : 69.7%, 2.9, 107.4ms, 0 | 20260507 16:46:14.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:14.504 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 16:46:14.504 | INFO | PRE_HQ : 66.5%, 1.7, 59.9ms, 0 | 20260507 16:46:14.504 | INFO | BAZWRITER : 70.2%, 3.7, 134.3ms, 0 | 20260507 16:46:14.504 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 16:46:14.504 | INFO | Uploaded 36GiB to GPU in 2.53973s (14.1747GiB/s) | 20260507 16:46:14.504 | INFO | Downloaded 38GiB to GPU in 2.71754s (13.9832GiB/s) | 20260507 16:46:19.643 | INFO | Analyzing chunk frames = [1040896,1041408) | 20260507 16:46:19.643 | INFO | MemInfo: { | 20260507 16:46:19.643 | INFO | "gpu" : " 17 GB", | 20260507 16:46:19.643 | INFO | "huge" : "", | 20260507 16:46:19.643 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:19.643 | INFO | "malloc" : " 14 GB", | 20260507 16:46:19.643 | INFO | "pinned" : " 29 GB" | 20260507 16:46:19.643 | INFO | } | 20260507 16:46:19.643 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:46:19.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:19.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:46:19.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:46:19.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:19.643 | INFO | Uploaded 36GiB to GPU in 2.49004s (14.4576GiB/s) | 20260507 16:46:19.643 | INFO | Downloaded 34GiB to GPU in 2.43829s (13.9442GiB/s) | 20260507 16:46:24.782 | INFO | Analyzing chunk frames = [1041408,1041920) | 20260507 16:46:24.782 | INFO | MemInfo: { | 20260507 16:46:24.782 | INFO | "gpu" : " 17 GB", | 20260507 16:46:24.782 | INFO | "huge" : "", | 20260507 16:46:24.782 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:24.782 | INFO | "malloc" : " 14 GB", | 20260507 16:46:24.782 | INFO | "pinned" : " 29 GB" | 20260507 16:46:24.782 | INFO | } | 20260507 16:46:24.782 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:46:24.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:24.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.782 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 16:46:24.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.783 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:46:24.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:24.783 | INFO | Uploaded 36GiB to GPU in 2.56084s (14.0579GiB/s) | 20260507 16:46:24.783 | INFO | Downloaded 34GiB to GPU in 2.41758s (14.0637GiB/s) | 20260507 16:46:29.921 | INFO | Analyzing chunk frames = [1041920,1042432) | 20260507 16:46:29.921 | INFO | MemInfo: { | 20260507 16:46:29.921 | INFO | "gpu" : " 17 GB", | 20260507 16:46:29.921 | INFO | "huge" : "", | 20260507 16:46:29.921 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:29.921 | INFO | "malloc" : " 14 GB", | 20260507 16:46:29.921 | INFO | "pinned" : " 29 GB" | 20260507 16:46:29.921 | INFO | } | 20260507 16:46:29.921 | INFO | Perf report for the last 5.13902 seconds: | 20260507 16:46:29.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:29.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.921 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 16:46:29.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.922 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:46:29.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:29.922 | INFO | Uploaded 36GiB to GPU in 2.47784s (14.5288GiB/s) | 20260507 16:46:29.922 | INFO | Downloaded 34GiB to GPU in 2.35275s (14.4512GiB/s) | 20260507 16:46:35.061 | INFO | Analyzing chunk frames = [1042432,1042944) | 20260507 16:46:35.061 | INFO | MemInfo: { | 20260507 16:46:35.061 | INFO | "gpu" : " 17 GB", | 20260507 16:46:35.061 | INFO | "huge" : "", | 20260507 16:46:35.061 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:35.061 | INFO | "malloc" : " 14 GB", | 20260507 16:46:35.061 | INFO | "pinned" : " 29 GB" | 20260507 16:46:35.061 | INFO | } | 20260507 16:46:35.061 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:46:35.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:35.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 16:46:35.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 16:46:35.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:35.061 | INFO | Uploaded 36GiB to GPU in 2.38892s (15.0696GiB/s) | 20260507 16:46:35.061 | INFO | Downloaded 34GiB to GPU in 2.48422s (13.6864GiB/s) | 20260507 16:46:40.200 | INFO | Analyzing chunk frames = [1042944,1043456) | 20260507 16:46:40.200 | INFO | MemInfo: { | 20260507 16:46:40.200 | INFO | "gpu" : " 17 GB", | 20260507 16:46:40.200 | INFO | "huge" : "", | 20260507 16:46:40.200 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:40.200 | INFO | "malloc" : " 14 GB", | 20260507 16:46:40.200 | INFO | "pinned" : " 29 GB" | 20260507 16:46:40.200 | INFO | } | 20260507 16:46:40.200 | INFO | Perf report for the last 5.13889 seconds: | 20260507 16:46:40.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:40.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:46:40.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 16:46:40.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:40.200 | INFO | Uploaded 36GiB to GPU in 2.35176s (15.3077GiB/s) | 20260507 16:46:40.200 | INFO | Downloaded 34GiB to GPU in 2.44563s (13.9023GiB/s) | 20260507 16:46:45.339 | INFO | Analyzing chunk frames = [1043456,1043968) | 20260507 16:46:45.339 | INFO | MemInfo: { | 20260507 16:46:45.339 | INFO | "gpu" : " 17 GB", | 20260507 16:46:45.339 | INFO | "huge" : "", | 20260507 16:46:45.339 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:45.339 | INFO | "malloc" : " 14 GB", | 20260507 16:46:45.339 | INFO | "pinned" : " 29 GB" | 20260507 16:46:45.339 | INFO | } | 20260507 16:46:45.339 | INFO | Perf report for the last 5.13948 seconds: | 20260507 16:46:45.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:45.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.339 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:46:45.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.339 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:46:45.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:45.340 | INFO | Uploaded 36GiB to GPU in 2.44432s (14.728GiB/s) | 20260507 16:46:45.340 | INFO | Downloaded 34GiB to GPU in 2.4145s (14.0816GiB/s) | 20260507 16:46:50.479 | INFO | Analyzing chunk frames = [1043968,1044480) | 20260507 16:46:50.479 | INFO | MemInfo: { | 20260507 16:46:50.479 | INFO | "gpu" : " 17 GB", | 20260507 16:46:50.479 | INFO | "huge" : "", | 20260507 16:46:50.479 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:50.479 | INFO | "malloc" : " 14 GB", | 20260507 16:46:50.479 | INFO | "pinned" : " 29 GB" | 20260507 16:46:50.479 | INFO | } | 20260507 16:46:50.479 | INFO | Perf report for the last 5.13966 seconds: | 20260507 16:46:50.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:50.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 16:46:50.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 16:46:50.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:50.479 | INFO | Uploaded 36GiB to GPU in 2.44113s (14.7472GiB/s) | 20260507 16:46:50.479 | INFO | Downloaded 34GiB to GPU in 2.49415s (13.6319GiB/s) | 20260507 16:46:50.576 | INFO | Starting a round of Baz IO | 20260507 16:46:54.082 | INFO | Populating RTMetrics report | 20260507 16:46:54.083 | INFO | RTMetrics report populated | 20260507 16:46:54.083 | INFO | Opening tmp RTMetrics json file | 20260507 16:46:54.086 | INFO | Serializing RTMetrics to json | 20260507 16:46:54.114 | INFO | Serialization done, swapping tmp file now... | 20260507 16:46:54.123 | INFO | RTMetrics done | 20260507 16:46:54.212 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.635 seconds | 20260507 16:46:54.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.48% / 0.03% / 50.49% | 20260507 16:46:54.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.17908% / 3.97364e-05% / 4.37101e-05% | 20260507 16:46:54.212 | INFO | Baz IO overall: | 20260507 16:46:54.212 | INFO | BytesWritten : 452189486442 | 20260507 16:46:54.212 | INFO | NumEvents : 218902942803 | 20260507 16:46:55.618 | INFO | Analyzing chunk frames = [1044480,1044992) | 20260507 16:46:55.618 | INFO | MemInfo: { | 20260507 16:46:55.618 | INFO | "gpu" : " 17 GB", | 20260507 16:46:55.618 | INFO | "huge" : "", | 20260507 16:46:55.618 | INFO | "huge_shared" : " 29 GB", | 20260507 16:46:55.618 | INFO | "malloc" : " 14 GB", | 20260507 16:46:55.618 | INFO | "pinned" : " 29 GB" | 20260507 16:46:55.618 | INFO | } | 20260507 16:46:55.618 | INFO | Perf report for the last 5.13902 seconds: | 20260507 16:46:55.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:46:55.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:55.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:55.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:55.618 | INFO | ANALYSIS : 69.6%, 2.9, 106.2ms, 0 | 20260507 16:46:55.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:46:55.618 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 16:46:55.618 | INFO | PRE_HQ : 67.3%, 1.8, 63.1ms, 0 | 20260507 16:46:55.618 | INFO | BAZWRITER : 70.3%, 3.8, 138.5ms, 0 | 20260507 16:46:55.618 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:46:55.618 | INFO | Uploaded 36GiB to GPU in 2.54832s (14.127GiB/s) | 20260507 16:46:55.618 | INFO | Downloaded 38GiB to GPU in 2.70544s (14.0458GiB/s) | 20260507 16:47:00.757 | INFO | Analyzing chunk frames = [1044992,1045504) | 20260507 16:47:00.757 | INFO | MemInfo: { | 20260507 16:47:00.757 | INFO | "gpu" : " 17 GB", | 20260507 16:47:00.758 | INFO | "huge" : "", | 20260507 16:47:00.758 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:00.758 | INFO | "malloc" : " 14 GB", | 20260507 16:47:00.758 | INFO | "pinned" : " 29 GB" | 20260507 16:47:00.758 | INFO | } | 20260507 16:47:00.758 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:47:00.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:00.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.758 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:47:00.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.758 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:47:00.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:00.759 | INFO | Uploaded 36GiB to GPU in 2.4448s (14.7251GiB/s) | 20260507 16:47:00.759 | INFO | Downloaded 34GiB to GPU in 2.46052s (13.8182GiB/s) | 20260507 16:47:05.896 | INFO | Analyzing chunk frames = [1045504,1046016) | 20260507 16:47:05.896 | INFO | MemInfo: { | 20260507 16:47:05.896 | INFO | "gpu" : " 17 GB", | 20260507 16:47:05.896 | INFO | "huge" : "", | 20260507 16:47:05.897 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:05.897 | INFO | "malloc" : " 14 GB", | 20260507 16:47:05.897 | INFO | "pinned" : " 29 GB" | 20260507 16:47:05.897 | INFO | } | 20260507 16:47:05.897 | INFO | Perf report for the last 5.13894 seconds: | 20260507 16:47:05.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:05.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260507 16:47:05.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 16:47:05.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:05.897 | INFO | Uploaded 36GiB to GPU in 2.51135s (14.3349GiB/s) | 20260507 16:47:05.897 | INFO | Downloaded 34GiB to GPU in 2.44437s (13.9095GiB/s) | 20260507 16:47:11.036 | INFO | Analyzing chunk frames = [1046016,1046528) | 20260507 16:47:11.036 | INFO | MemInfo: { | 20260507 16:47:11.036 | INFO | "gpu" : " 17 GB", | 20260507 16:47:11.036 | INFO | "huge" : "", | 20260507 16:47:11.036 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:11.036 | INFO | "malloc" : " 14 GB", | 20260507 16:47:11.036 | INFO | "pinned" : " 29 GB" | 20260507 16:47:11.036 | INFO | } | 20260507 16:47:11.036 | INFO | Perf report for the last 5.13931 seconds: | 20260507 16:47:11.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:11.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 16:47:11.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 16:47:11.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:11.036 | INFO | Uploaded 36GiB to GPU in 2.41874s (14.8838GiB/s) | 20260507 16:47:11.036 | INFO | Downloaded 34GiB to GPU in 2.37566s (14.3118GiB/s) | 20260507 16:47:16.175 | INFO | Analyzing chunk frames = [1046528,1047040) | 20260507 16:47:16.175 | INFO | MemInfo: { | 20260507 16:47:16.175 | INFO | "gpu" : " 17 GB", | 20260507 16:47:16.175 | INFO | "huge" : "", | 20260507 16:47:16.175 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:16.176 | INFO | "malloc" : " 14 GB", | 20260507 16:47:16.176 | INFO | "pinned" : " 29 GB" | 20260507 16:47:16.176 | INFO | } | 20260507 16:47:16.176 | INFO | Perf report for the last 5.13959 seconds: | 20260507 16:47:16.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:16.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 16:47:16.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 16:47:16.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:16.176 | INFO | Uploaded 36GiB to GPU in 2.32844s (15.461GiB/s) | 20260507 16:47:16.176 | INFO | Downloaded 34GiB to GPU in 2.49487s (13.628GiB/s) | 20260507 16:47:21.314 | INFO | Analyzing chunk frames = [1047040,1047552) | 20260507 16:47:21.314 | INFO | MemInfo: { | 20260507 16:47:21.314 | INFO | "gpu" : " 17 GB", | 20260507 16:47:21.314 | INFO | "huge" : "", | 20260507 16:47:21.315 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:21.315 | INFO | "malloc" : " 14 GB", | 20260507 16:47:21.315 | INFO | "pinned" : " 29 GB" | 20260507 16:47:21.315 | INFO | } | 20260507 16:47:21.315 | INFO | Perf report for the last 5.13908 seconds: | 20260507 16:47:21.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:21.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:47:21.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:47:21.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:21.315 | INFO | Uploaded 36GiB to GPU in 2.34473s (15.3536GiB/s) | 20260507 16:47:21.315 | INFO | Downloaded 34GiB to GPU in 2.4638s (13.7998GiB/s) | 20260507 16:47:26.454 | INFO | Analyzing chunk frames = [1047552,1048064) | 20260507 16:47:26.454 | INFO | MemInfo: { | 20260507 16:47:26.454 | INFO | "gpu" : " 17 GB", | 20260507 16:47:26.454 | INFO | "huge" : "", | 20260507 16:47:26.454 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:26.454 | INFO | "malloc" : " 14 GB", | 20260507 16:47:26.454 | INFO | "pinned" : " 29 GB" | 20260507 16:47:26.454 | INFO | } | 20260507 16:47:26.454 | INFO | Perf report for the last 5.13956 seconds: | 20260507 16:47:26.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:26.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.454 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:47:26.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.454 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:47:26.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:26.455 | INFO | Uploaded 36GiB to GPU in 2.42249s (14.8607GiB/s) | 20260507 16:47:26.455 | INFO | Downloaded 34GiB to GPU in 2.41663s (14.0692GiB/s) | 20260507 16:47:31.593 | INFO | Analyzing chunk frames = [1048064,1048576) | 20260507 16:47:31.593 | INFO | MemInfo: { | 20260507 16:47:31.593 | INFO | "gpu" : " 17 GB", | 20260507 16:47:31.593 | INFO | "huge" : "", | 20260507 16:47:31.594 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:31.594 | INFO | "malloc" : " 14 GB", | 20260507 16:47:31.594 | INFO | "pinned" : " 29 GB" | 20260507 16:47:31.594 | INFO | } | 20260507 16:47:31.594 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:47:31.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:31.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:47:31.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:47:31.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:31.594 | INFO | Uploaded 36GiB to GPU in 2.47262s (14.5595GiB/s) | 20260507 16:47:31.594 | INFO | Downloaded 34GiB to GPU in 2.51066s (13.5422GiB/s) | 20260507 16:47:31.594 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:47:31.594 | INFO | Total mem usage: 16.4 GiB | 20260507 16:47:31.594 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:47:31.594 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.8%) | 20260507 16:47:31.594 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:47:31.594 | INFO | High Water Breakdown: | 20260507 16:47:31.594 | INFO | 59.3% : 8.78 GiB ( 88.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:47:31.594 | INFO | 40.5% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:47:31.594 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:47:31.594 | INFO | ------------------------------------------ | 20260507 16:47:31.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:47:31.595 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:47:31.595 | INFO | Total mem usage: 26.5 GiB | 20260507 16:47:31.595 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:47:31.595 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 16:47:31.595 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 16:47:31.595 | INFO | High Water Breakdown: | 20260507 16:47:31.595 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:47:31.595 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:47:31.595 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:47:31.595 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:47:31.595 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:47:31.595 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:47:31.595 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:47:31.595 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:47:31.595 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:47:31.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:47:31.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:47:31.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:47:31.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:47:31.595 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:47:31.595 | INFO | ------------------------------------------ | 20260507 16:47:31.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:47:31.595 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:47:31.595 | INFO | Total mem usage: 27.8 GiB | 20260507 16:47:31.595 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:47:31.595 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 16:47:31.595 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 16:47:31.595 | INFO | High Water Breakdown: | 20260507 16:47:31.595 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 16:47:31.595 | INFO | ------------------------------------------ | 20260507 16:47:31.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:47:31.595 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:47:31.595 | INFO | Total mem usage: 15.7 GiB | 20260507 16:47:31.595 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:47:31.595 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:47:31.595 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:47:31.595 | INFO | High Water Breakdown: | 20260507 16:47:31.595 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:47:31.595 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:47:31.595 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:47:31.595 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:47:31.595 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:47:31.595 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:47:31.595 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:47:31.595 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:47:31.595 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:47:31.595 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:47:31.595 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:47:31.595 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:47:31.595 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:47:31.595 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:47:31.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:47:31.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:47:31.595 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:47:31.595 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:47:31.595 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:47:31.595 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:47:31.595 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:47:31.595 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:47:31.595 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:47:31.595 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:47:31.595 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:47:31.595 | INFO | ------------------------------------------ | 20260507 16:47:31.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:47:31.596 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:47:31.596 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:47:31.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:47:31.596 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.68e+01, 1.44e+01 / 2.66e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.15e-02 / 6.08e-03 | 20260507 16:47:31.596 | INFO | PerfReport: 12.05%, Baseline -- 4.05e+00 / 4.51e+00, 6.23e-01 / 2.44e-01 | 20260507 16:47:31.596 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.48e+00, 2.18e-01 / 1.17e-01 | 20260507 16:47:31.596 | INFO | PerfReport: 15.71%, DME -- 4.23e+01 / 4.51e+01, 4.39e+00 / 2.77e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.92e+00 / 2.09e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.52e+00 / 2.91e+00, 4.38e-01 / 2.54e-01 | 20260507 16:47:31.596 | INFO | PerfReport: 3.81%, Metrics -- 1.28e+00 / 1.46e+00, 3.29e-01 / 1.93e-01 | 20260507 16:47:31.596 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.08e-01 / 4.32e-01, 6.84e-02 / 4.49e-02 | 20260507 16:47:31.596 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:47:31.596 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:47:31.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:47:31.596 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 23.76%, Upload -- 2.55e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 24.04%, Download -- 2.58e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 31.34%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 1.67%, LockUpload -- 1.79e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 2.40%, LockDownload -- 2.58e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 16.78%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:47:31.596 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260507 16:47:31.691 | INFO | Starting a round of Baz IO | 20260507 16:47:35.203 | INFO | Populating RTMetrics report | 20260507 16:47:35.203 | INFO | RTMetrics report populated | 20260507 16:47:35.203 | INFO | Opening tmp RTMetrics json file | 20260507 16:47:35.233 | INFO | Serializing RTMetrics to json | 20260507 16:47:35.255 | INFO | Serialization done, swapping tmp file now... | 20260507 16:47:35.287 | INFO | RTMetrics done | 20260507 16:47:35.314 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.623 seconds | 20260507 16:47:35.314 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.46% / 0.03% / 50.52% | 20260507 16:47:35.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.18223% / 3.57628e-05% / 1.98682e-05% | 20260507 16:47:35.314 | INFO | Baz IO overall: | 20260507 16:47:35.314 | INFO | BytesWritten : 454115473272 | 20260507 16:47:35.314 | INFO | NumEvents : 219840259843 | 20260507 16:47:36.733 | INFO | Analyzing chunk frames = [1048576,1049088) | 20260507 16:47:36.733 | INFO | MemInfo: { | 20260507 16:47:36.733 | INFO | "gpu" : " 17 GB", | 20260507 16:47:36.733 | INFO | "huge" : "", | 20260507 16:47:36.733 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:36.733 | INFO | "malloc" : " 14 GB", | 20260507 16:47:36.733 | INFO | "pinned" : " 29 GB" | 20260507 16:47:36.733 | INFO | } | 20260507 16:47:36.733 | INFO | Perf report for the last 5.13895 seconds: | 20260507 16:47:36.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:36.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:36.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:36.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:36.733 | INFO | ANALYSIS : 69.7%, 3.0, 107.9ms, 0 | 20260507 16:47:36.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:36.733 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 16:47:36.733 | INFO | PRE_HQ : 67.1%, 1.7, 60.6ms, 0 | 20260507 16:47:36.733 | INFO | BAZWRITER : 70.1%, 3.8, 137.9ms, 0 | 20260507 16:47:36.733 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 16:47:36.733 | INFO | Uploaded 36GiB to GPU in 2.55065s (14.114GiB/s) | 20260507 16:47:36.733 | INFO | Downloaded 38GiB to GPU in 2.71233s (14.0101GiB/s) | 20260507 16:47:41.872 | INFO | Analyzing chunk frames = [1049088,1049600) | 20260507 16:47:41.872 | INFO | MemInfo: { | 20260507 16:47:41.872 | INFO | "gpu" : " 17 GB", | 20260507 16:47:41.872 | INFO | "huge" : "", | 20260507 16:47:41.872 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:41.872 | INFO | "malloc" : " 14 GB", | 20260507 16:47:41.872 | INFO | "pinned" : " 29 GB" | 20260507 16:47:41.872 | INFO | } | 20260507 16:47:41.872 | INFO | Perf report for the last 5.13971 seconds: | 20260507 16:47:41.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:41.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.872 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:47:41.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.873 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 16:47:41.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:41.873 | INFO | Uploaded 36GiB to GPU in 2.5138s (14.321GiB/s) | 20260507 16:47:41.873 | INFO | Downloaded 34GiB to GPU in 2.4238s (14.0276GiB/s) | 20260507 16:47:47.011 | INFO | Analyzing chunk frames = [1049600,1050112) | 20260507 16:47:47.011 | INFO | MemInfo: { | 20260507 16:47:47.011 | INFO | "gpu" : " 17 GB", | 20260507 16:47:47.011 | INFO | "huge" : "", | 20260507 16:47:47.011 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:47.011 | INFO | "malloc" : " 14 GB", | 20260507 16:47:47.012 | INFO | "pinned" : " 29 GB" | 20260507 16:47:47.012 | INFO | } | 20260507 16:47:47.012 | INFO | Perf report for the last 5.13915 seconds: | 20260507 16:47:47.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:47.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260507 16:47:47.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:47:47.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:47.012 | INFO | Uploaded 36GiB to GPU in 2.56843s (14.0163GiB/s) | 20260507 16:47:47.012 | INFO | Downloaded 34GiB to GPU in 2.40996s (14.1081GiB/s) | 20260507 16:47:52.151 | INFO | Analyzing chunk frames = [1050112,1050624) | 20260507 16:47:52.151 | INFO | MemInfo: { | 20260507 16:47:52.151 | INFO | "gpu" : " 17 GB", | 20260507 16:47:52.151 | INFO | "huge" : "", | 20260507 16:47:52.151 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:52.151 | INFO | "malloc" : " 14 GB", | 20260507 16:47:52.151 | INFO | "pinned" : " 29 GB" | 20260507 16:47:52.151 | INFO | } | 20260507 16:47:52.151 | INFO | Perf report for the last 5.13956 seconds: | 20260507 16:47:52.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:52.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.151 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:47:52.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.151 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:47:52.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:52.152 | INFO | Uploaded 36GiB to GPU in 2.48839s (14.4672GiB/s) | 20260507 16:47:52.152 | INFO | Downloaded 34GiB to GPU in 2.35308s (14.4491GiB/s) | 20260507 16:47:57.290 | INFO | Analyzing chunk frames = [1050624,1051136) | 20260507 16:47:57.290 | INFO | MemInfo: { | 20260507 16:47:57.290 | INFO | "gpu" : " 17 GB", | 20260507 16:47:57.290 | INFO | "huge" : "", | 20260507 16:47:57.290 | INFO | "huge_shared" : " 29 GB", | 20260507 16:47:57.291 | INFO | "malloc" : " 14 GB", | 20260507 16:47:57.291 | INFO | "pinned" : " 29 GB" | 20260507 16:47:57.291 | INFO | } | 20260507 16:47:57.291 | INFO | Perf report for the last 5.13958 seconds: | 20260507 16:47:57.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:47:57.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 16:47:57.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 16:47:57.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:47:57.291 | INFO | Uploaded 36GiB to GPU in 2.34943s (15.3229GiB/s) | 20260507 16:47:57.291 | INFO | Downloaded 34GiB to GPU in 2.45717s (13.8371GiB/s) | 20260507 16:48:02.430 | INFO | Analyzing chunk frames = [1051136,1051648) | 20260507 16:48:02.430 | INFO | MemInfo: { | 20260507 16:48:02.430 | INFO | "gpu" : " 17 GB", | 20260507 16:48:02.430 | INFO | "huge" : "", | 20260507 16:48:02.430 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:02.430 | INFO | "malloc" : " 14 GB", | 20260507 16:48:02.430 | INFO | "pinned" : " 29 GB" | 20260507 16:48:02.430 | INFO | } | 20260507 16:48:02.430 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:48:02.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:02.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:48:02.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:48:02.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:02.430 | INFO | Uploaded 36GiB to GPU in 2.36841s (15.2001GiB/s) | 20260507 16:48:02.430 | INFO | Downloaded 34GiB to GPU in 2.45228s (13.8647GiB/s) | 20260507 16:48:07.569 | INFO | Analyzing chunk frames = [1051648,1052160) | 20260507 16:48:07.569 | INFO | MemInfo: { | 20260507 16:48:07.569 | INFO | "gpu" : " 17 GB", | 20260507 16:48:07.569 | INFO | "huge" : "", | 20260507 16:48:07.569 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:07.569 | INFO | "malloc" : " 14 GB", | 20260507 16:48:07.569 | INFO | "pinned" : " 29 GB" | 20260507 16:48:07.569 | INFO | } | 20260507 16:48:07.569 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:48:07.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:07.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.569 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:48:07.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.569 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:48:07.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:07.570 | INFO | Uploaded 36GiB to GPU in 2.45431s (14.6681GiB/s) | 20260507 16:48:07.570 | INFO | Downloaded 34GiB to GPU in 2.41641s (14.0705GiB/s) | 20260507 16:48:12.709 | INFO | Analyzing chunk frames = [1052160,1052672) | 20260507 16:48:12.709 | INFO | MemInfo: { | 20260507 16:48:12.709 | INFO | "gpu" : " 17 GB", | 20260507 16:48:12.709 | INFO | "huge" : "", | 20260507 16:48:12.709 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:12.709 | INFO | "malloc" : " 14 GB", | 20260507 16:48:12.709 | INFO | "pinned" : " 29 GB" | 20260507 16:48:12.709 | INFO | } | 20260507 16:48:12.709 | INFO | Perf report for the last 5.1396 seconds: | 20260507 16:48:12.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:12.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.709 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:48:12.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.709 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:48:12.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:12.710 | INFO | Uploaded 36GiB to GPU in 2.49485s (14.4297GiB/s) | 20260507 16:48:12.710 | INFO | Downloaded 34GiB to GPU in 2.50037s (13.598GiB/s) | 20260507 16:48:12.806 | INFO | Starting a round of Baz IO | 20260507 16:48:16.342 | INFO | Populating RTMetrics report | 20260507 16:48:16.343 | INFO | RTMetrics report populated | 20260507 16:48:16.343 | INFO | Opening tmp RTMetrics json file | 20260507 16:48:16.344 | INFO | Serializing RTMetrics to json | 20260507 16:48:16.374 | INFO | Serialization done, swapping tmp file now... | 20260507 16:48:16.380 | INFO | RTMetrics done | 20260507 16:48:16.462 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.655 seconds | 20260507 16:48:16.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.43% / 0.03% / 50.54% | 20260507 16:48:16.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.15895% / 2.38419e-05% / 3.97364e-05% | 20260507 16:48:16.462 | INFO | Baz IO overall: | 20260507 16:48:16.462 | INFO | BytesWritten : 456039453216 | 20260507 16:48:16.462 | INFO | NumEvents : 220775938818 | 20260507 16:48:17.848 | INFO | Analyzing chunk frames = [1052672,1053184) | 20260507 16:48:17.848 | INFO | MemInfo: { | 20260507 16:48:17.848 | INFO | "gpu" : " 17 GB", | 20260507 16:48:17.848 | INFO | "huge" : "", | 20260507 16:48:17.848 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:17.848 | INFO | "malloc" : " 14 GB", | 20260507 16:48:17.848 | INFO | "pinned" : " 29 GB" | 20260507 16:48:17.848 | INFO | } | 20260507 16:48:17.848 | INFO | Perf report for the last 5.1389 seconds: | 20260507 16:48:17.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:17.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:17.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:17.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:17.848 | INFO | ANALYSIS : 69.4%, 3.0, 108.2ms, 0 | 20260507 16:48:17.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:17.848 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 16:48:17.848 | INFO | PRE_HQ : 67.3%, 1.8, 63.8ms, 0 | 20260507 16:48:17.848 | INFO | BAZWRITER : 70.5%, 3.8, 137.6ms, 0 | 20260507 16:48:17.848 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 16:48:17.848 | INFO | Uploaded 36GiB to GPU in 2.45249s (14.679GiB/s) | 20260507 16:48:17.848 | INFO | Downloaded 38GiB to GPU in 2.72348s (13.9527GiB/s) | 20260507 16:48:22.987 | INFO | Analyzing chunk frames = [1053184,1053696) | 20260507 16:48:22.987 | INFO | MemInfo: { | 20260507 16:48:22.987 | INFO | "gpu" : " 17 GB", | 20260507 16:48:22.987 | INFO | "huge" : "", | 20260507 16:48:22.988 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:22.988 | INFO | "malloc" : " 14 GB", | 20260507 16:48:22.988 | INFO | "pinned" : " 29 GB" | 20260507 16:48:22.988 | INFO | } | 20260507 16:48:22.988 | INFO | Perf report for the last 5.13985 seconds: | 20260507 16:48:22.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:22.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:48:22.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 16:48:22.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:22.988 | INFO | Uploaded 36GiB to GPU in 2.53451s (14.2039GiB/s) | 20260507 16:48:22.988 | INFO | Downloaded 34GiB to GPU in 2.45204s (13.866GiB/s) | 20260507 16:48:28.127 | INFO | Analyzing chunk frames = [1053696,1054208) | 20260507 16:48:28.127 | INFO | MemInfo: { | 20260507 16:48:28.127 | INFO | "gpu" : " 17 GB", | 20260507 16:48:28.127 | INFO | "huge" : "", | 20260507 16:48:28.127 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:28.127 | INFO | "malloc" : " 14 GB", | 20260507 16:48:28.127 | INFO | "pinned" : " 29 GB" | 20260507 16:48:28.127 | INFO | } | 20260507 16:48:28.127 | INFO | Perf report for the last 5.13959 seconds: | 20260507 16:48:28.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:28.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.127 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:48:28.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.128 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:48:28.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:28.128 | INFO | Uploaded 36GiB to GPU in 2.56455s (14.0376GiB/s) | 20260507 16:48:28.128 | INFO | Downloaded 34GiB to GPU in 2.42005s (14.0493GiB/s) | 20260507 16:48:33.266 | INFO | Analyzing chunk frames = [1054208,1054720) | 20260507 16:48:33.266 | INFO | MemInfo: { | 20260507 16:48:33.266 | INFO | "gpu" : " 17 GB", | 20260507 16:48:33.266 | INFO | "huge" : "", | 20260507 16:48:33.266 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:33.266 | INFO | "malloc" : " 14 GB", | 20260507 16:48:33.267 | INFO | "pinned" : " 29 GB" | 20260507 16:48:33.267 | INFO | } | 20260507 16:48:33.267 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:48:33.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:33.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 16:48:33.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:48:33.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:33.267 | INFO | Uploaded 36GiB to GPU in 2.47221s (14.5619GiB/s) | 20260507 16:48:33.267 | INFO | Downloaded 34GiB to GPU in 2.34889s (14.4749GiB/s) | 20260507 16:48:38.406 | INFO | Analyzing chunk frames = [1054720,1055232) | 20260507 16:48:38.406 | INFO | MemInfo: { | 20260507 16:48:38.406 | INFO | "gpu" : " 17 GB", | 20260507 16:48:38.406 | INFO | "huge" : "", | 20260507 16:48:38.406 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:38.406 | INFO | "malloc" : " 14 GB", | 20260507 16:48:38.406 | INFO | "pinned" : " 29 GB" | 20260507 16:48:38.406 | INFO | } | 20260507 16:48:38.407 | INFO | Perf report for the last 5.13976 seconds: | 20260507 16:48:38.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:38.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260507 16:48:38.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 16:48:38.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:38.407 | INFO | Uploaded 36GiB to GPU in 2.37589s (15.1522GiB/s) | 20260507 16:48:38.407 | INFO | Downloaded 34GiB to GPU in 2.47188s (13.7547GiB/s) | 20260507 16:48:43.545 | INFO | Analyzing chunk frames = [1055232,1055744) | 20260507 16:48:43.545 | INFO | MemInfo: { | 20260507 16:48:43.545 | INFO | "gpu" : " 17 GB", | 20260507 16:48:43.545 | INFO | "huge" : "", | 20260507 16:48:43.545 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:43.545 | INFO | "malloc" : " 14 GB", | 20260507 16:48:43.545 | INFO | "pinned" : " 29 GB" | 20260507 16:48:43.545 | INFO | } | 20260507 16:48:43.545 | INFO | Perf report for the last 5.13873 seconds: | 20260507 16:48:43.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:43.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:48:43.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260507 16:48:43.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:43.545 | INFO | Uploaded 36GiB to GPU in 2.40206s (14.9871GiB/s) | 20260507 16:48:43.546 | INFO | Downloaded 34GiB to GPU in 2.49947s (13.6029GiB/s) | 20260507 16:48:48.684 | INFO | Analyzing chunk frames = [1055744,1056256) | 20260507 16:48:48.684 | INFO | MemInfo: { | 20260507 16:48:48.684 | INFO | "gpu" : " 17 GB", | 20260507 16:48:48.684 | INFO | "huge" : "", | 20260507 16:48:48.684 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:48.685 | INFO | "malloc" : " 14 GB", | 20260507 16:48:48.685 | INFO | "pinned" : " 29 GB" | 20260507 16:48:48.685 | INFO | } | 20260507 16:48:48.685 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:48:48.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:48.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:48:48.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:48:48.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:48.685 | INFO | Uploaded 36GiB to GPU in 2.42494s (14.8457GiB/s) | 20260507 16:48:48.685 | INFO | Downloaded 34GiB to GPU in 2.41344s (14.0878GiB/s) | 20260507 16:48:53.824 | INFO | Analyzing chunk frames = [1056256,1056768) | 20260507 16:48:53.824 | INFO | MemInfo: { | 20260507 16:48:53.824 | INFO | "gpu" : " 17 GB", | 20260507 16:48:53.824 | INFO | "huge" : "", | 20260507 16:48:53.824 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:53.824 | INFO | "malloc" : " 14 GB", | 20260507 16:48:53.824 | INFO | "pinned" : " 29 GB" | 20260507 16:48:53.824 | INFO | } | 20260507 16:48:53.824 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:48:53.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:53.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 16:48:53.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:48:53.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:53.824 | INFO | Uploaded 36GiB to GPU in 2.49517s (14.4279GiB/s) | 20260507 16:48:53.824 | INFO | Downloaded 34GiB to GPU in 2.50414s (13.5775GiB/s) | 20260507 16:48:53.824 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:48:53.825 | INFO | Total mem usage: 16.4 GiB | 20260507 16:48:53.825 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:48:53.825 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.5%) | 20260507 16:48:53.825 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:48:53.825 | INFO | High Water Breakdown: | 20260507 16:48:53.825 | INFO | 59.1% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:48:53.825 | INFO | 40.8% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:48:53.825 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:48:53.825 | INFO | ------------------------------------------ | 20260507 16:48:53.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:48:53.825 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:48:53.825 | INFO | Total mem usage: 26.5 GiB | 20260507 16:48:53.825 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:48:53.825 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:48:53.825 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:48:53.825 | INFO | High Water Breakdown: | 20260507 16:48:53.825 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:48:53.825 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:48:53.825 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:48:53.825 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:48:53.825 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:48:53.825 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:48:53.825 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:48:53.825 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:48:53.825 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:48:53.825 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:48:53.825 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:48:53.825 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:48:53.825 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:48:53.825 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:48:53.825 | INFO | ------------------------------------------ | 20260507 16:48:53.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:48:53.825 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:48:53.825 | INFO | Total mem usage: 27.8 GiB | 20260507 16:48:53.825 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:48:53.825 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:48:53.825 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:48:53.825 | INFO | High Water Breakdown: | 20260507 16:48:53.825 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:48:53.825 | INFO | ------------------------------------------ | 20260507 16:48:53.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:48:53.825 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:48:53.825 | INFO | Total mem usage: 15.7 GiB | 20260507 16:48:53.825 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:48:53.825 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:48:53.825 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:48:53.825 | INFO | High Water Breakdown: | 20260507 16:48:53.825 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:48:53.825 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:48:53.825 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:48:53.825 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:48:53.825 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:48:53.825 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:48:53.825 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:48:53.825 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:48:53.825 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:48:53.825 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:48:53.825 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:48:53.825 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:48:53.825 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:48:53.825 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:48:53.825 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:48:53.825 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:48:53.825 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:48:53.825 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:48:53.825 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:48:53.825 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:48:53.825 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:48:53.826 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:48:53.826 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:48:53.826 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:48:53.826 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:48:53.826 | INFO | ------------------------------------------ | 20260507 16:48:53.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:48:53.826 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:48:53.826 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:48:53.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:48:53.826 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.70e+01, 1.45e+01 / 2.90e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 0.29%, AggregateStats -- 9.84e-02 / 1.02e-01, 1.17e-02 / 6.83e-03 | 20260507 16:48:53.826 | INFO | PerfReport: 12.03%, Baseline -- 4.05e+00 / 4.51e+00, 6.15e-01 / 2.21e-01 | 20260507 16:48:53.826 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.51e+00, 2.18e-01 / 1.11e-01 | 20260507 16:48:53.826 | INFO | PerfReport: 15.73%, DME -- 4.24e+01 / 4.52e+01, 4.42e+00 / 2.92e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 55.94%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.91e+00 / 2.26e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.53e+00 / 2.93e+00, 4.36e-01 / 2.15e-01 | 20260507 16:48:53.826 | INFO | PerfReport: 3.84%, Metrics -- 1.29e+00 / 1.47e+00, 3.34e-01 / 2.08e-01 | 20260507 16:48:53.826 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.07e-01 / 4.34e-01, 6.75e-02 / 4.32e-02 | 20260507 16:48:53.826 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:48:53.826 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:48:53.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:48:53.826 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 23.90%, Upload -- 2.57e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 23.93%, Download -- 2.58e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 31.27%, Compute -- 3.37e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 1.76%, LockUpload -- 1.90e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 2.31%, LockDownload -- 2.49e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 16.81%, LockCompute -- 1.81e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:48:53.826 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 16:48:53.921 | INFO | Starting a round of Baz IO | 20260507 16:48:57.440 | INFO | Populating RTMetrics report | 20260507 16:48:57.440 | INFO | RTMetrics report populated | 20260507 16:48:57.440 | INFO | Opening tmp RTMetrics json file | 20260507 16:48:57.441 | INFO | Serializing RTMetrics to json | 20260507 16:48:57.443 | INFO | Serialization done, swapping tmp file now... | 20260507 16:48:57.444 | INFO | RTMetrics done | 20260507 16:48:57.639 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.718 seconds | 20260507 16:48:57.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.41% / 0.02% / 50.57% | 20260507 16:48:57.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.13495% / 4.76837e-05% / 3.17891e-05% | 20260507 16:48:57.639 | INFO | Baz IO overall: | 20260507 16:48:57.639 | INFO | BytesWritten : 457961819502 | 20260507 16:48:57.639 | INFO | NumEvents : 221710524417 | 20260507 16:48:58.963 | INFO | Analyzing chunk frames = [1056768,1057280) | 20260507 16:48:58.963 | INFO | MemInfo: { | 20260507 16:48:58.963 | INFO | "gpu" : " 17 GB", | 20260507 16:48:58.963 | INFO | "huge" : "", | 20260507 16:48:58.963 | INFO | "huge_shared" : " 29 GB", | 20260507 16:48:58.963 | INFO | "malloc" : " 14 GB", | 20260507 16:48:58.963 | INFO | "pinned" : " 29 GB" | 20260507 16:48:58.964 | INFO | } | 20260507 16:48:58.964 | INFO | Perf report for the last 5.13976 seconds: | 20260507 16:48:58.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:48:58.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:58.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:58.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:58.964 | INFO | ANALYSIS : 69.8%, 3.0, 108.1ms, 0 | 20260507 16:48:58.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:48:58.964 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 16:48:58.964 | INFO | PRE_HQ : 66.9%, 1.7, 60.6ms, 0 | 20260507 16:48:58.964 | INFO | BAZWRITER : 71.9%, 4.1, 151.7ms, 0 | 20260507 16:48:58.964 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 16:48:58.964 | INFO | Uploaded 36GiB to GPU in 2.50715s (14.3589GiB/s) | 20260507 16:48:58.964 | INFO | Downloaded 38GiB to GPU in 2.73654s (13.8862GiB/s) | 20260507 16:49:04.102 | INFO | Analyzing chunk frames = [1057280,1057792) | 20260507 16:49:04.102 | INFO | MemInfo: { | 20260507 16:49:04.102 | INFO | "gpu" : " 17 GB", | 20260507 16:49:04.103 | INFO | "huge" : "", | 20260507 16:49:04.103 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:04.103 | INFO | "malloc" : " 14 GB", | 20260507 16:49:04.103 | INFO | "pinned" : " 29 GB" | 20260507 16:49:04.103 | INFO | } | 20260507 16:49:04.103 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:49:04.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:04.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.103 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 16:49:04.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.103 | INFO | PRE_HQ : 37.1%, 1.0, 19.2ms, 0 | 20260507 16:49:04.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:04.104 | INFO | Uploaded 36GiB to GPU in 2.51975s (14.2871GiB/s) | 20260507 16:49:04.104 | INFO | Downloaded 34GiB to GPU in 2.43488s (13.9637GiB/s) | 20260507 16:49:09.241 | INFO | Analyzing chunk frames = [1057792,1058304) | 20260507 16:49:09.241 | INFO | MemInfo: { | 20260507 16:49:09.241 | INFO | "gpu" : " 17 GB", | 20260507 16:49:09.241 | INFO | "huge" : "", | 20260507 16:49:09.241 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:09.242 | INFO | "malloc" : " 14 GB", | 20260507 16:49:09.242 | INFO | "pinned" : " 29 GB" | 20260507 16:49:09.242 | INFO | } | 20260507 16:49:09.242 | INFO | Perf report for the last 5.13871 seconds: | 20260507 16:49:09.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:09.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 16:49:09.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260507 16:49:09.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:09.242 | INFO | Uploaded 36GiB to GPU in 2.54034s (14.1714GiB/s) | 20260507 16:49:09.242 | INFO | Downloaded 34GiB to GPU in 2.42933s (13.9956GiB/s) | 20260507 16:49:14.381 | INFO | Analyzing chunk frames = [1058304,1058816) | 20260507 16:49:14.381 | INFO | MemInfo: { | 20260507 16:49:14.381 | INFO | "gpu" : " 17 GB", | 20260507 16:49:14.381 | INFO | "huge" : "", | 20260507 16:49:14.381 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:14.381 | INFO | "malloc" : " 14 GB", | 20260507 16:49:14.381 | INFO | "pinned" : " 29 GB" | 20260507 16:49:14.381 | INFO | } | 20260507 16:49:14.381 | INFO | Perf report for the last 5.13976 seconds: | 20260507 16:49:14.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:14.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.381 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:49:14.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.382 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:49:14.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:14.382 | INFO | Uploaded 36GiB to GPU in 2.46477s (14.6058GiB/s) | 20260507 16:49:14.382 | INFO | Downloaded 34GiB to GPU in 2.35633s (14.4292GiB/s) | 20260507 16:49:19.521 | INFO | Analyzing chunk frames = [1058816,1059328) | 20260507 16:49:19.521 | INFO | MemInfo: { | 20260507 16:49:19.521 | INFO | "gpu" : " 17 GB", | 20260507 16:49:19.521 | INFO | "huge" : "", | 20260507 16:49:19.521 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:19.521 | INFO | "malloc" : " 14 GB", | 20260507 16:49:19.521 | INFO | "pinned" : " 29 GB" | 20260507 16:49:19.521 | INFO | } | 20260507 16:49:19.521 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:49:19.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:19.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260507 16:49:19.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 16:49:19.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:19.521 | INFO | Uploaded 36GiB to GPU in 2.43365s (14.7926GiB/s) | 20260507 16:49:19.521 | INFO | Downloaded 34GiB to GPU in 2.47743s (13.7239GiB/s) | 20260507 16:49:24.660 | INFO | Analyzing chunk frames = [1059328,1059840) | 20260507 16:49:24.660 | INFO | MemInfo: { | 20260507 16:49:24.660 | INFO | "gpu" : " 17 GB", | 20260507 16:49:24.660 | INFO | "huge" : "", | 20260507 16:49:24.660 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:24.660 | INFO | "malloc" : " 14 GB", | 20260507 16:49:24.660 | INFO | "pinned" : " 29 GB" | 20260507 16:49:24.660 | INFO | } | 20260507 16:49:24.660 | INFO | Perf report for the last 5.13901 seconds: | 20260507 16:49:24.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:24.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:49:24.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:49:24.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:24.660 | INFO | Uploaded 36GiB to GPU in 2.3613s (15.2458GiB/s) | 20260507 16:49:24.660 | INFO | Downloaded 34GiB to GPU in 2.4539s (13.8555GiB/s) | 20260507 16:49:29.799 | INFO | Analyzing chunk frames = [1059840,1060352) | 20260507 16:49:29.799 | INFO | MemInfo: { | 20260507 16:49:29.799 | INFO | "gpu" : " 17 GB", | 20260507 16:49:29.799 | INFO | "huge" : "", | 20260507 16:49:29.799 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:29.799 | INFO | "malloc" : " 14 GB", | 20260507 16:49:29.799 | INFO | "pinned" : " 29 GB" | 20260507 16:49:29.799 | INFO | } | 20260507 16:49:29.799 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:49:29.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:29.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 16:49:29.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:49:29.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:29.799 | INFO | Uploaded 36GiB to GPU in 2.42717s (14.8321GiB/s) | 20260507 16:49:29.799 | INFO | Downloaded 34GiB to GPU in 2.41649s (14.07GiB/s) | 20260507 16:49:34.938 | INFO | Analyzing chunk frames = [1060352,1060864) | 20260507 16:49:34.938 | INFO | MemInfo: { | 20260507 16:49:34.938 | INFO | "gpu" : " 17 GB", | 20260507 16:49:34.938 | INFO | "huge" : "", | 20260507 16:49:34.938 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:34.938 | INFO | "malloc" : " 14 GB", | 20260507 16:49:34.938 | INFO | "pinned" : " 29 GB" | 20260507 16:49:34.938 | INFO | } | 20260507 16:49:34.938 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:49:34.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:34.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.939 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 16:49:34.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.939 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:49:34.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:34.939 | INFO | Uploaded 36GiB to GPU in 2.51785s (14.2979GiB/s) | 20260507 16:49:34.939 | INFO | Downloaded 34GiB to GPU in 2.50181s (13.5902GiB/s) | 20260507 16:49:35.036 | INFO | Starting a round of Baz IO | 20260507 16:49:38.545 | INFO | Populating RTMetrics report | 20260507 16:49:38.545 | INFO | RTMetrics report populated | 20260507 16:49:38.545 | INFO | Opening tmp RTMetrics json file | 20260507 16:49:38.546 | INFO | Serializing RTMetrics to json | 20260507 16:49:38.580 | INFO | Serialization done, swapping tmp file now... | 20260507 16:49:38.620 | INFO | RTMetrics done | 20260507 16:49:38.680 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.644 seconds | 20260507 16:49:38.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.38% / 0.02% / 50.59% | 20260507 16:49:38.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.11965% / 4.76837e-05% / 4.76837e-05% | 20260507 16:49:38.680 | INFO | Baz IO overall: | 20260507 16:49:38.680 | INFO | BytesWritten : 459883946714 | 20260507 16:49:38.680 | INFO | NumEvents : 222645253469 | 20260507 16:49:40.078 | INFO | Analyzing chunk frames = [1060864,1061376) | 20260507 16:49:40.078 | INFO | MemInfo: { | 20260507 16:49:40.078 | INFO | "gpu" : " 17 GB", | 20260507 16:49:40.078 | INFO | "huge" : "", | 20260507 16:49:40.078 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:40.078 | INFO | "malloc" : " 14 GB", | 20260507 16:49:40.078 | INFO | "pinned" : " 29 GB" | 20260507 16:49:40.078 | INFO | } | 20260507 16:49:40.078 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:49:40.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:40.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:40.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:40.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:40.078 | INFO | ANALYSIS : 69.7%, 2.9, 107.7ms, 0 | 20260507 16:49:40.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:40.078 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 16:49:40.078 | INFO | PRE_HQ : 66.4%, 1.7, 60.4ms, 0 | 20260507 16:49:40.078 | INFO | BAZWRITER : 70.5%, 3.8, 138.6ms, 0 | 20260507 16:49:40.078 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 16:49:40.079 | INFO | Uploaded 36GiB to GPU in 2.50073s (14.3958GiB/s) | 20260507 16:49:40.079 | INFO | Downloaded 38GiB to GPU in 2.71946s (13.9734GiB/s) | 20260507 16:49:45.217 | INFO | Analyzing chunk frames = [1061376,1061888) | 20260507 16:49:45.217 | INFO | MemInfo: { | 20260507 16:49:45.217 | INFO | "gpu" : " 17 GB", | 20260507 16:49:45.217 | INFO | "huge" : "", | 20260507 16:49:45.217 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:45.217 | INFO | "malloc" : " 14 GB", | 20260507 16:49:45.217 | INFO | "pinned" : " 29 GB" | 20260507 16:49:45.217 | INFO | } | 20260507 16:49:45.217 | INFO | Perf report for the last 5.13917 seconds: | 20260507 16:49:45.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:45.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 16:49:45.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:49:45.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:45.217 | INFO | Uploaded 36GiB to GPU in 2.50022s (14.3988GiB/s) | 20260507 16:49:45.217 | INFO | Downloaded 34GiB to GPU in 2.43935s (13.9381GiB/s) | 20260507 16:49:50.356 | INFO | Analyzing chunk frames = [1061888,1062400) | 20260507 16:49:50.356 | INFO | MemInfo: { | 20260507 16:49:50.356 | INFO | "gpu" : " 17 GB", | 20260507 16:49:50.356 | INFO | "huge" : "", | 20260507 16:49:50.356 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:50.356 | INFO | "malloc" : " 14 GB", | 20260507 16:49:50.357 | INFO | "pinned" : " 29 GB" | 20260507 16:49:50.357 | INFO | } | 20260507 16:49:50.357 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:49:50.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:50.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 16:49:50.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:49:50.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:50.357 | INFO | Uploaded 36GiB to GPU in 2.54656s (14.1367GiB/s) | 20260507 16:49:50.357 | INFO | Downloaded 34GiB to GPU in 2.43057s (13.9885GiB/s) | 20260507 16:49:55.496 | INFO | Analyzing chunk frames = [1062400,1062912) | 20260507 16:49:55.496 | INFO | MemInfo: { | 20260507 16:49:55.496 | INFO | "gpu" : " 17 GB", | 20260507 16:49:55.496 | INFO | "huge" : "", | 20260507 16:49:55.496 | INFO | "huge_shared" : " 29 GB", | 20260507 16:49:55.496 | INFO | "malloc" : " 14 GB", | 20260507 16:49:55.496 | INFO | "pinned" : " 29 GB" | 20260507 16:49:55.496 | INFO | } | 20260507 16:49:55.496 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:49:55.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:49:55.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.496 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 16:49:55.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.496 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:49:55.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:49:55.497 | INFO | Uploaded 36GiB to GPU in 2.48425s (14.4913GiB/s) | 20260507 16:49:55.497 | INFO | Downloaded 34GiB to GPU in 2.35314s (14.4488GiB/s) | 20260507 16:50:00.635 | INFO | Analyzing chunk frames = [1062912,1063424) | 20260507 16:50:00.635 | INFO | MemInfo: { | 20260507 16:50:00.635 | INFO | "gpu" : " 17 GB", | 20260507 16:50:00.635 | INFO | "huge" : "", | 20260507 16:50:00.635 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:00.635 | INFO | "malloc" : " 14 GB", | 20260507 16:50:00.636 | INFO | "pinned" : " 29 GB" | 20260507 16:50:00.636 | INFO | } | 20260507 16:50:00.636 | INFO | Perf report for the last 5.13969 seconds: | 20260507 16:50:00.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:00.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 16:50:00.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 16:50:00.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:00.636 | INFO | Uploaded 36GiB to GPU in 2.37051s (15.1866GiB/s) | 20260507 16:50:00.636 | INFO | Downloaded 34GiB to GPU in 2.46927s (13.7692GiB/s) | 20260507 16:50:05.774 | INFO | Analyzing chunk frames = [1063424,1063936) | 20260507 16:50:05.774 | INFO | MemInfo: { | 20260507 16:50:05.774 | INFO | "gpu" : " 17 GB", | 20260507 16:50:05.775 | INFO | "huge" : "", | 20260507 16:50:05.775 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:05.775 | INFO | "malloc" : " 14 GB", | 20260507 16:50:05.775 | INFO | "pinned" : " 29 GB" | 20260507 16:50:05.775 | INFO | } | 20260507 16:50:05.775 | INFO | Perf report for the last 5.13906 seconds: | 20260507 16:50:05.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:05.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:50:05.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260507 16:50:05.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:05.775 | INFO | Uploaded 36GiB to GPU in 2.36838s (15.2003GiB/s) | 20260507 16:50:05.775 | INFO | Downloaded 34GiB to GPU in 2.46569s (13.7892GiB/s) | 20260507 16:50:10.914 | INFO | Analyzing chunk frames = [1063936,1064448) | 20260507 16:50:10.914 | INFO | MemInfo: { | 20260507 16:50:10.914 | INFO | "gpu" : " 17 GB", | 20260507 16:50:10.914 | INFO | "huge" : "", | 20260507 16:50:10.914 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:10.914 | INFO | "malloc" : " 14 GB", | 20260507 16:50:10.914 | INFO | "pinned" : " 29 GB" | 20260507 16:50:10.914 | INFO | } | 20260507 16:50:10.914 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:50:10.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:10.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:50:10.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:50:10.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:10.914 | INFO | Uploaded 36GiB to GPU in 2.43417s (14.7895GiB/s) | 20260507 16:50:10.914 | INFO | Downloaded 34GiB to GPU in 2.40555s (14.134GiB/s) | 20260507 16:50:16.053 | INFO | Analyzing chunk frames = [1064448,1064960) | 20260507 16:50:16.053 | INFO | MemInfo: { | 20260507 16:50:16.053 | INFO | "gpu" : " 17 GB", | 20260507 16:50:16.053 | INFO | "huge" : "", | 20260507 16:50:16.053 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:16.053 | INFO | "malloc" : " 14 GB", | 20260507 16:50:16.053 | INFO | "pinned" : " 29 GB" | 20260507 16:50:16.053 | INFO | } | 20260507 16:50:16.053 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:50:16.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:16.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.053 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:50:16.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.054 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:50:16.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:16.054 | INFO | Uploaded 36GiB to GPU in 2.43268s (14.7985GiB/s) | 20260507 16:50:16.054 | INFO | Downloaded 34GiB to GPU in 2.49881s (13.6065GiB/s) | 20260507 16:50:16.054 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:50:16.054 | INFO | Total mem usage: 16.4 GiB | 20260507 16:50:16.054 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:50:16.054 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.4%) | 20260507 16:50:16.054 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:50:16.054 | INFO | High Water Breakdown: | 20260507 16:50:16.054 | INFO | 59.5% : 8.93 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:50:16.054 | INFO | 40.3% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:50:16.054 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:50:16.054 | INFO | ------------------------------------------ | 20260507 16:50:16.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:50:16.054 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:50:16.054 | INFO | Total mem usage: 26.5 GiB | 20260507 16:50:16.054 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:50:16.054 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:50:16.054 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:50:16.054 | INFO | High Water Breakdown: | 20260507 16:50:16.054 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:50:16.054 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:50:16.054 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:50:16.054 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:50:16.054 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:50:16.054 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:50:16.054 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:50:16.054 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:50:16.054 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:50:16.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:50:16.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:50:16.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:50:16.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:50:16.054 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:50:16.054 | INFO | ------------------------------------------ | 20260507 16:50:16.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:50:16.054 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:50:16.054 | INFO | Total mem usage: 27.8 GiB | 20260507 16:50:16.054 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:50:16.054 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:50:16.054 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:50:16.054 | INFO | High Water Breakdown: | 20260507 16:50:16.054 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:50:16.054 | INFO | ------------------------------------------ | 20260507 16:50:16.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:50:16.054 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:50:16.054 | INFO | Total mem usage: 15.7 GiB | 20260507 16:50:16.054 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:50:16.054 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:50:16.054 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:50:16.054 | INFO | High Water Breakdown: | 20260507 16:50:16.054 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:50:16.054 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:50:16.054 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:50:16.054 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:50:16.054 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:50:16.054 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:50:16.054 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:50:16.054 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:50:16.054 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:50:16.054 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:50:16.054 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:50:16.054 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:50:16.054 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:50:16.054 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:50:16.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:50:16.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:50:16.054 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:50:16.054 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:50:16.054 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:50:16.054 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:50:16.054 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:50:16.054 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:50:16.054 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:50:16.054 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:50:16.054 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:50:16.054 | INFO | ------------------------------------------ | 20260507 16:50:16.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:50:16.054 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:50:16.054 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:50:16.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:50:16.054 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.44e+01 / 2.90e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 0.30%, AggregateStats -- 1.01e-01 / 1.02e-01, 9.38e-02 / 5.54e-03 | 20260507 16:50:16.054 | INFO | PerfReport: 12.04%, Baseline -- 4.05e+00 / 4.51e+00, 6.23e-01 / 2.32e-01 | 20260507 16:50:16.054 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 1.15e-01 | 20260507 16:50:16.054 | INFO | PerfReport: 15.70%, DME -- 4.22e+01 / 4.50e+01, 4.48e+00 / 2.91e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.88e+00 / 2.27e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.54e+00 / 2.91e+00, 4.36e-01 / 2.51e-01 | 20260507 16:50:16.054 | INFO | PerfReport: 3.82%, Metrics -- 1.28e+00 / 1.45e+00, 3.27e-01 / 2.01e-01 | 20260507 16:50:16.054 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.07e-01 / 4.33e-01, 6.77e-02 / 4.54e-02 | 20260507 16:50:16.054 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:50:16.054 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:50:16.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:50:16.054 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 23.85%, Upload -- 2.57e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 23.96%, Download -- 2.58e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 31.23%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 1.78%, LockUpload -- 1.91e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 2.36%, LockDownload -- 2.54e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 16.81%, LockCompute -- 1.81e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:50:16.054 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 16:50:16.151 | INFO | Starting a round of Baz IO | 20260507 16:50:19.676 | INFO | Populating RTMetrics report | 20260507 16:50:19.676 | INFO | RTMetrics report populated | 20260507 16:50:19.676 | INFO | Opening tmp RTMetrics json file | 20260507 16:50:19.686 | INFO | Serializing RTMetrics to json | 20260507 16:50:19.696 | INFO | Serialization done, swapping tmp file now... | 20260507 16:50:19.697 | INFO | RTMetrics done | 20260507 16:50:19.769 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.617 seconds | 20260507 16:50:19.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.36% / 0.02% / 50.62% | 20260507 16:50:19.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.11084% / 3.97364e-05% / 4.37101e-05% | 20260507 16:50:19.769 | INFO | Baz IO overall: | 20260507 16:50:19.769 | INFO | BytesWritten : 461806201535 | 20260507 16:50:19.769 | INFO | NumEvents : 223579967485 | 20260507 16:50:21.192 | INFO | Analyzing chunk frames = [1064960,1065472) | 20260507 16:50:21.192 | INFO | MemInfo: { | 20260507 16:50:21.192 | INFO | "gpu" : " 17 GB", | 20260507 16:50:21.192 | INFO | "huge" : "", | 20260507 16:50:21.192 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:21.192 | INFO | "malloc" : " 14 GB", | 20260507 16:50:21.192 | INFO | "pinned" : " 29 GB" | 20260507 16:50:21.192 | INFO | } | 20260507 16:50:21.193 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:50:21.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:21.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:21.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:21.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:21.193 | INFO | ANALYSIS : 70.0%, 2.9, 108.1ms, 0 | 20260507 16:50:21.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:21.193 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 16:50:21.193 | INFO | PRE_HQ : 66.7%, 1.7, 60.3ms, 0 | 20260507 16:50:21.193 | INFO | BAZWRITER : 69.8%, 3.9, 139.9ms, 0 | 20260507 16:50:21.193 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 16:50:21.193 | INFO | Uploaded 36GiB to GPU in 2.57233s (13.9951GiB/s) | 20260507 16:50:21.193 | INFO | Downloaded 38GiB to GPU in 2.71666s (13.9878GiB/s) | 20260507 16:50:26.332 | INFO | Analyzing chunk frames = [1065472,1065984) | 20260507 16:50:26.332 | INFO | MemInfo: { | 20260507 16:50:26.332 | INFO | "gpu" : " 17 GB", | 20260507 16:50:26.332 | INFO | "huge" : "", | 20260507 16:50:26.332 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:26.332 | INFO | "malloc" : " 14 GB", | 20260507 16:50:26.332 | INFO | "pinned" : " 29 GB" | 20260507 16:50:26.332 | INFO | } | 20260507 16:50:26.332 | INFO | Perf report for the last 5.13978 seconds: | 20260507 16:50:26.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:26.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.332 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:50:26.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.332 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:50:26.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:26.333 | INFO | Uploaded 36GiB to GPU in 2.53121s (14.2224GiB/s) | 20260507 16:50:26.333 | INFO | Downloaded 34GiB to GPU in 2.42853s (14.0002GiB/s) | 20260507 16:50:31.471 | INFO | Analyzing chunk frames = [1065984,1066496) | 20260507 16:50:31.471 | INFO | MemInfo: { | 20260507 16:50:31.471 | INFO | "gpu" : " 17 GB", | 20260507 16:50:31.471 | INFO | "huge" : "", | 20260507 16:50:31.471 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:31.472 | INFO | "malloc" : " 14 GB", | 20260507 16:50:31.472 | INFO | "pinned" : " 29 GB" | 20260507 16:50:31.472 | INFO | } | 20260507 16:50:31.472 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:50:31.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:31.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 16:50:31.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:50:31.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:31.472 | INFO | Uploaded 36GiB to GPU in 2.5758s (13.9763GiB/s) | 20260507 16:50:31.472 | INFO | Downloaded 34GiB to GPU in 2.40865s (14.1158GiB/s) | 20260507 16:50:36.611 | INFO | Analyzing chunk frames = [1066496,1067008) | 20260507 16:50:36.611 | INFO | MemInfo: { | 20260507 16:50:36.611 | INFO | "gpu" : " 17 GB", | 20260507 16:50:36.611 | INFO | "huge" : "", | 20260507 16:50:36.611 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:36.611 | INFO | "malloc" : " 14 GB", | 20260507 16:50:36.611 | INFO | "pinned" : " 29 GB" | 20260507 16:50:36.611 | INFO | } | 20260507 16:50:36.611 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:50:36.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:36.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.611 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:50:36.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.611 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:50:36.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:36.612 | INFO | Uploaded 36GiB to GPU in 2.49412s (14.434GiB/s) | 20260507 16:50:36.612 | INFO | Downloaded 34GiB to GPU in 2.35841s (14.4165GiB/s) | 20260507 16:50:41.750 | INFO | Analyzing chunk frames = [1067008,1067520) | 20260507 16:50:41.750 | INFO | MemInfo: { | 20260507 16:50:41.750 | INFO | "gpu" : " 17 GB", | 20260507 16:50:41.750 | INFO | "huge" : "", | 20260507 16:50:41.750 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:41.750 | INFO | "malloc" : " 14 GB", | 20260507 16:50:41.750 | INFO | "pinned" : " 29 GB" | 20260507 16:50:41.750 | INFO | } | 20260507 16:50:41.751 | INFO | Perf report for the last 5.13943 seconds: | 20260507 16:50:41.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:41.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:50:41.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260507 16:50:41.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:41.751 | INFO | Uploaded 36GiB to GPU in 2.35008s (15.3186GiB/s) | 20260507 16:50:41.751 | INFO | Downloaded 34GiB to GPU in 2.47124s (13.7583GiB/s) | 20260507 16:50:46.889 | INFO | Analyzing chunk frames = [1067520,1068032) | 20260507 16:50:46.889 | INFO | MemInfo: { | 20260507 16:50:46.889 | INFO | "gpu" : " 17 GB", | 20260507 16:50:46.889 | INFO | "huge" : "", | 20260507 16:50:46.889 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:46.889 | INFO | "malloc" : " 14 GB", | 20260507 16:50:46.889 | INFO | "pinned" : " 29 GB" | 20260507 16:50:46.889 | INFO | } | 20260507 16:50:46.890 | INFO | Perf report for the last 5.13897 seconds: | 20260507 16:50:46.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:46.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:50:46.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:50:46.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:46.890 | INFO | Uploaded 36GiB to GPU in 2.39397s (15.0378GiB/s) | 20260507 16:50:46.890 | INFO | Downloaded 34GiB to GPU in 2.44657s (13.897GiB/s) | 20260507 16:50:52.028 | INFO | Analyzing chunk frames = [1068032,1068544) | 20260507 16:50:52.028 | INFO | MemInfo: { | 20260507 16:50:52.028 | INFO | "gpu" : " 17 GB", | 20260507 16:50:52.028 | INFO | "huge" : "", | 20260507 16:50:52.028 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:52.029 | INFO | "malloc" : " 14 GB", | 20260507 16:50:52.029 | INFO | "pinned" : " 29 GB" | 20260507 16:50:52.029 | INFO | } | 20260507 16:50:52.029 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:50:52.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:52.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:50:52.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:50:52.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:52.029 | INFO | Uploaded 36GiB to GPU in 2.46072s (14.6299GiB/s) | 20260507 16:50:52.029 | INFO | Downloaded 34GiB to GPU in 2.40889s (14.1144GiB/s) | 20260507 16:50:57.168 | INFO | Analyzing chunk frames = [1068544,1069056) | 20260507 16:50:57.168 | INFO | MemInfo: { | 20260507 16:50:57.168 | INFO | "gpu" : " 17 GB", | 20260507 16:50:57.168 | INFO | "huge" : "", | 20260507 16:50:57.169 | INFO | "huge_shared" : " 29 GB", | 20260507 16:50:57.169 | INFO | "malloc" : " 14 GB", | 20260507 16:50:57.169 | INFO | "pinned" : " 29 GB" | 20260507 16:50:57.169 | INFO | } | 20260507 16:50:57.169 | INFO | Perf report for the last 5.14013 seconds: | 20260507 16:50:57.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:50:57.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 16:50:57.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:50:57.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:50:57.169 | INFO | Uploaded 36GiB to GPU in 2.5317s (14.2197GiB/s) | 20260507 16:50:57.169 | INFO | Downloaded 34GiB to GPU in 2.49505s (13.627GiB/s) | 20260507 16:50:57.267 | INFO | Starting a round of Baz IO | 20260507 16:51:00.777 | INFO | Populating RTMetrics report | 20260507 16:51:00.778 | INFO | RTMetrics report populated | 20260507 16:51:00.778 | INFO | Opening tmp RTMetrics json file | 20260507 16:51:00.785 | INFO | Serializing RTMetrics to json | 20260507 16:51:00.800 | INFO | Serialization done, swapping tmp file now... | 20260507 16:51:00.801 | INFO | RTMetrics done | 20260507 16:51:00.893 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.625 seconds | 20260507 16:51:00.893 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.33% / 0.02% / 50.64% | 20260507 16:51:00.893 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09119% / 3.97364e-05% / 4.76837e-05% | 20260507 16:51:00.893 | INFO | Baz IO overall: | 20260507 16:51:00.893 | INFO | BytesWritten : 463727518987 | 20260507 16:51:00.893 | INFO | NumEvents : 224514303434 | 20260507 16:51:02.308 | INFO | Analyzing chunk frames = [1069056,1069568) | 20260507 16:51:02.308 | INFO | MemInfo: { | 20260507 16:51:02.308 | INFO | "gpu" : " 17 GB", | 20260507 16:51:02.308 | INFO | "huge" : "", | 20260507 16:51:02.308 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:02.308 | INFO | "malloc" : " 14 GB", | 20260507 16:51:02.308 | INFO | "pinned" : " 29 GB" | 20260507 16:51:02.308 | INFO | } | 20260507 16:51:02.308 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:51:02.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:02.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:02.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:02.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:02.308 | INFO | ANALYSIS : 69.8%, 2.9, 107.6ms, 0 | 20260507 16:51:02.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:02.308 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 16:51:02.308 | INFO | PRE_HQ : 66.7%, 1.7, 60.0ms, 0 | 20260507 16:51:02.308 | INFO | BAZWRITER : 70.1%, 3.8, 137.6ms, 0 | 20260507 16:51:02.308 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 16:51:02.308 | INFO | Uploaded 36GiB to GPU in 2.54913s (14.1225GiB/s) | 20260507 16:51:02.308 | INFO | Downloaded 38GiB to GPU in 2.72161s (13.9623GiB/s) | 20260507 16:51:07.447 | INFO | Analyzing chunk frames = [1069568,1070080) | 20260507 16:51:07.447 | INFO | MemInfo: { | 20260507 16:51:07.447 | INFO | "gpu" : " 17 GB", | 20260507 16:51:07.447 | INFO | "huge" : "", | 20260507 16:51:07.447 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:07.447 | INFO | "malloc" : " 14 GB", | 20260507 16:51:07.447 | INFO | "pinned" : " 29 GB" | 20260507 16:51:07.447 | INFO | } | 20260507 16:51:07.447 | INFO | Perf report for the last 5.13911 seconds: | 20260507 16:51:07.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:07.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:51:07.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 16:51:07.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:07.447 | INFO | Uploaded 36GiB to GPU in 2.51929s (14.2897GiB/s) | 20260507 16:51:07.448 | INFO | Downloaded 34GiB to GPU in 2.4265s (14.012GiB/s) | 20260507 16:51:12.586 | INFO | Analyzing chunk frames = [1070080,1070592) | 20260507 16:51:12.586 | INFO | MemInfo: { | 20260507 16:51:12.586 | INFO | "gpu" : " 17 GB", | 20260507 16:51:12.586 | INFO | "huge" : "", | 20260507 16:51:12.586 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:12.586 | INFO | "malloc" : " 14 GB", | 20260507 16:51:12.586 | INFO | "pinned" : " 29 GB" | 20260507 16:51:12.586 | INFO | } | 20260507 16:51:12.586 | INFO | Perf report for the last 5.13907 seconds: | 20260507 16:51:12.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:12.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.586 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260507 16:51:12.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.586 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:51:12.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:12.587 | INFO | Uploaded 36GiB to GPU in 2.58134s (13.9462GiB/s) | 20260507 16:51:12.587 | INFO | Downloaded 34GiB to GPU in 2.39882s (14.1736GiB/s) | 20260507 16:51:17.725 | INFO | Analyzing chunk frames = [1070592,1071104) | 20260507 16:51:17.725 | INFO | MemInfo: { | 20260507 16:51:17.725 | INFO | "gpu" : " 17 GB", | 20260507 16:51:17.725 | INFO | "huge" : "", | 20260507 16:51:17.725 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:17.725 | INFO | "malloc" : " 14 GB", | 20260507 16:51:17.725 | INFO | "pinned" : " 29 GB" | 20260507 16:51:17.725 | INFO | } | 20260507 16:51:17.726 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:51:17.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:17.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260507 16:51:17.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:51:17.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:17.726 | INFO | Uploaded 36GiB to GPU in 2.49391s (14.4352GiB/s) | 20260507 16:51:17.726 | INFO | Downloaded 34GiB to GPU in 2.35134s (14.4599GiB/s) | 20260507 16:51:22.865 | INFO | Analyzing chunk frames = [1071104,1071616) | 20260507 16:51:22.865 | INFO | MemInfo: { | 20260507 16:51:22.865 | INFO | "gpu" : " 17 GB", | 20260507 16:51:22.865 | INFO | "huge" : "", | 20260507 16:51:22.865 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:22.865 | INFO | "malloc" : " 14 GB", | 20260507 16:51:22.865 | INFO | "pinned" : " 29 GB" | 20260507 16:51:22.866 | INFO | } | 20260507 16:51:22.866 | INFO | Perf report for the last 5.14011 seconds: | 20260507 16:51:22.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:22.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:51:22.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 16:51:22.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:22.866 | INFO | Uploaded 36GiB to GPU in 2.36227s (15.2396GiB/s) | 20260507 16:51:22.866 | INFO | Downloaded 34GiB to GPU in 2.45659s (13.8403GiB/s) | 20260507 16:51:28.004 | INFO | Analyzing chunk frames = [1071616,1072128) | 20260507 16:51:28.004 | INFO | MemInfo: { | 20260507 16:51:28.004 | INFO | "gpu" : " 17 GB", | 20260507 16:51:28.004 | INFO | "huge" : "", | 20260507 16:51:28.004 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:28.004 | INFO | "malloc" : " 14 GB", | 20260507 16:51:28.004 | INFO | "pinned" : " 29 GB" | 20260507 16:51:28.004 | INFO | } | 20260507 16:51:28.004 | INFO | Perf report for the last 5.1385 seconds: | 20260507 16:51:28.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:28.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.004 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:51:28.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.004 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 16:51:28.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:28.005 | INFO | Uploaded 36GiB to GPU in 2.36574s (15.2172GiB/s) | 20260507 16:51:28.005 | INFO | Downloaded 34GiB to GPU in 2.45662s (13.8402GiB/s) | 20260507 16:51:33.143 | INFO | Analyzing chunk frames = [1072128,1072640) | 20260507 16:51:33.143 | INFO | MemInfo: { | 20260507 16:51:33.143 | INFO | "gpu" : " 17 GB", | 20260507 16:51:33.143 | INFO | "huge" : "", | 20260507 16:51:33.143 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:33.143 | INFO | "malloc" : " 14 GB", | 20260507 16:51:33.143 | INFO | "pinned" : " 29 GB" | 20260507 16:51:33.144 | INFO | } | 20260507 16:51:33.144 | INFO | Perf report for the last 5.13954 seconds: | 20260507 16:51:33.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:33.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:51:33.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:51:33.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:33.144 | INFO | Uploaded 36GiB to GPU in 2.43283s (14.7976GiB/s) | 20260507 16:51:33.144 | INFO | Downloaded 34GiB to GPU in 2.42504s (14.0204GiB/s) | 20260507 16:51:38.283 | INFO | Analyzing chunk frames = [1072640,1073152) | 20260507 16:51:38.283 | INFO | MemInfo: { | 20260507 16:51:38.283 | INFO | "gpu" : " 17 GB", | 20260507 16:51:38.283 | INFO | "huge" : "", | 20260507 16:51:38.283 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:38.283 | INFO | "malloc" : " 14 GB", | 20260507 16:51:38.283 | INFO | "pinned" : " 29 GB" | 20260507 16:51:38.283 | INFO | } | 20260507 16:51:38.283 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:51:38.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:38.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 16:51:38.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:51:38.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:38.283 | INFO | Uploaded 36GiB to GPU in 2.45789s (14.6467GiB/s) | 20260507 16:51:38.283 | INFO | Downloaded 34GiB to GPU in 2.51521s (13.5178GiB/s) | 20260507 16:51:38.284 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:51:38.284 | INFO | Total mem usage: 16.4 GiB | 20260507 16:51:38.284 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:51:38.284 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.5%) | 20260507 16:51:38.284 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:51:38.284 | INFO | High Water Breakdown: | 20260507 16:51:38.284 | INFO | 59.4% : 8.82 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:51:38.284 | INFO | 40.4% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:51:38.284 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:51:38.284 | INFO | ------------------------------------------ | 20260507 16:51:38.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:51:38.284 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:51:38.284 | INFO | Total mem usage: 26.5 GiB | 20260507 16:51:38.284 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:51:38.284 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:51:38.284 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:51:38.284 | INFO | High Water Breakdown: | 20260507 16:51:38.284 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:51:38.284 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:51:38.284 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:51:38.284 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:51:38.284 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:51:38.284 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:51:38.284 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:51:38.284 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:51:38.284 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:51:38.284 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:51:38.284 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:51:38.284 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:51:38.284 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:51:38.284 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:51:38.284 | INFO | ------------------------------------------ | 20260507 16:51:38.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:51:38.284 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:51:38.284 | INFO | Total mem usage: 27.8 GiB | 20260507 16:51:38.284 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:51:38.284 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:51:38.284 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:51:38.284 | INFO | High Water Breakdown: | 20260507 16:51:38.284 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:51:38.284 | INFO | ------------------------------------------ | 20260507 16:51:38.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:51:38.284 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:51:38.284 | INFO | Total mem usage: 15.7 GiB | 20260507 16:51:38.284 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:51:38.284 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:51:38.284 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:51:38.284 | INFO | High Water Breakdown: | 20260507 16:51:38.284 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:51:38.284 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:51:38.284 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:51:38.284 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:51:38.284 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:51:38.284 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:51:38.284 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:51:38.284 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:51:38.284 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:51:38.284 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:51:38.284 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:51:38.284 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:51:38.284 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:51:38.284 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:51:38.284 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:51:38.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:51:38.285 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:51:38.285 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:51:38.285 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:51:38.285 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:51:38.285 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:51:38.285 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:51:38.285 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:51:38.285 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:51:38.285 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:51:38.285 | INFO | ------------------------------------------ | 20260507 16:51:38.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:51:38.285 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:51:38.285 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:51:38.285 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:51:38.285 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.68e+01, 1.45e+01 / 2.81e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 0.29%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.10e-02 / 4.33e-03 | 20260507 16:51:38.285 | INFO | PerfReport: 12.03%, Baseline -- 4.04e+00 / 4.50e+00, 6.24e-01 / 2.42e-01 | 20260507 16:51:38.285 | INFO | PerfReport: 3.41%, Histogram -- 1.15e+00 / 1.50e+00, 2.18e-01 / 1.01e-01 | 20260507 16:51:38.285 | INFO | PerfReport: 15.72%, DME -- 4.23e+01 / 4.51e+01, 4.26e+00 / 2.59e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.92e+00 / 2.23e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.52e+00 / 2.92e+00, 4.41e-01 / 2.27e-01 | 20260507 16:51:38.285 | INFO | PerfReport: 3.82%, Metrics -- 1.28e+00 / 1.44e+00, 3.34e-01 / 1.80e-01 | 20260507 16:51:38.285 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.07e-01 / 4.34e-01, 6.98e-02 / 4.23e-02 | 20260507 16:51:38.285 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:51:38.285 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:51:38.285 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:51:38.285 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 24.00%, Upload -- 2.58e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 23.89%, Download -- 2.57e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 31.25%, Compute -- 3.36e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 1.96%, LockUpload -- 2.11e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 2.17%, LockDownload -- 2.34e+00 / 0.00e+00, 3.78e+00 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 16.72%, LockCompute -- 1.80e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:51:38.285 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 16:51:38.380 | INFO | Starting a round of Baz IO | 20260507 16:51:41.896 | INFO | Populating RTMetrics report | 20260507 16:51:41.896 | INFO | RTMetrics report populated | 20260507 16:51:41.896 | INFO | Opening tmp RTMetrics json file | 20260507 16:51:41.900 | INFO | Serializing RTMetrics to json | 20260507 16:51:41.913 | INFO | Serialization done, swapping tmp file now... | 20260507 16:51:41.929 | INFO | RTMetrics done | 20260507 16:51:42.026 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.646 seconds | 20260507 16:51:42.026 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.31% / 0.02% / 50.67% | 20260507 16:51:42.026 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.08571% / 3.97364e-05% / 1.98682e-05% | 20260507 16:51:42.026 | INFO | Baz IO overall: | 20260507 16:51:42.026 | INFO | BytesWritten : 465649613687 | 20260507 16:51:42.026 | INFO | NumEvents : 225448955731 | 20260507 16:51:43.422 | INFO | Analyzing chunk frames = [1073152,1073664) | 20260507 16:51:43.422 | INFO | MemInfo: { | 20260507 16:51:43.422 | INFO | "gpu" : " 17 GB", | 20260507 16:51:43.422 | INFO | "huge" : "", | 20260507 16:51:43.422 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:43.422 | INFO | "malloc" : " 14 GB", | 20260507 16:51:43.422 | INFO | "pinned" : " 29 GB" | 20260507 16:51:43.422 | INFO | } | 20260507 16:51:43.422 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:51:43.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:43.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:43.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:43.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:43.422 | INFO | ANALYSIS : 69.8%, 3.0, 108.2ms, 0 | 20260507 16:51:43.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:43.422 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 16:51:43.422 | INFO | PRE_HQ : 66.5%, 1.7, 59.7ms, 0 | 20260507 16:51:43.422 | INFO | BAZWRITER : 70.5%, 3.8, 135.9ms, 0 | 20260507 16:51:43.422 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:51:43.423 | INFO | Uploaded 36GiB to GPU in 2.55233s (14.1048GiB/s) | 20260507 16:51:43.423 | INFO | Downloaded 38GiB to GPU in 2.72898s (13.9246GiB/s) | 20260507 16:51:48.561 | INFO | Analyzing chunk frames = [1073664,1074176) | 20260507 16:51:48.561 | INFO | MemInfo: { | 20260507 16:51:48.561 | INFO | "gpu" : " 17 GB", | 20260507 16:51:48.561 | INFO | "huge" : "", | 20260507 16:51:48.561 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:48.562 | INFO | "malloc" : " 14 GB", | 20260507 16:51:48.562 | INFO | "pinned" : " 29 GB" | 20260507 16:51:48.562 | INFO | } | 20260507 16:51:48.562 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:51:48.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:48.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:51:48.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:51:48.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:48.562 | INFO | Uploaded 36GiB to GPU in 2.49944s (14.4032GiB/s) | 20260507 16:51:48.562 | INFO | Downloaded 34GiB to GPU in 2.43941s (13.9378GiB/s) | 20260507 16:51:53.701 | INFO | Analyzing chunk frames = [1074176,1074688) | 20260507 16:51:53.701 | INFO | MemInfo: { | 20260507 16:51:53.701 | INFO | "gpu" : " 17 GB", | 20260507 16:51:53.701 | INFO | "huge" : "", | 20260507 16:51:53.701 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:53.701 | INFO | "malloc" : " 14 GB", | 20260507 16:51:53.701 | INFO | "pinned" : " 29 GB" | 20260507 16:51:53.701 | INFO | } | 20260507 16:51:53.701 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:51:53.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:53.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.701 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:51:53.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.701 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:51:53.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:53.702 | INFO | Uploaded 36GiB to GPU in 2.5471s (14.1337GiB/s) | 20260507 16:51:53.702 | INFO | Downloaded 34GiB to GPU in 2.42695s (14.0093GiB/s) | 20260507 16:51:58.840 | INFO | Analyzing chunk frames = [1074688,1075200) | 20260507 16:51:58.840 | INFO | MemInfo: { | 20260507 16:51:58.840 | INFO | "gpu" : " 17 GB", | 20260507 16:51:58.840 | INFO | "huge" : "", | 20260507 16:51:58.840 | INFO | "huge_shared" : " 29 GB", | 20260507 16:51:58.840 | INFO | "malloc" : " 14 GB", | 20260507 16:51:58.840 | INFO | "pinned" : " 29 GB" | 20260507 16:51:58.840 | INFO | } | 20260507 16:51:58.840 | INFO | Perf report for the last 5.13915 seconds: | 20260507 16:51:58.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:51:58.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.840 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 16:51:58.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.841 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:51:58.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:51:58.841 | INFO | Uploaded 36GiB to GPU in 2.48296s (14.4988GiB/s) | 20260507 16:51:58.841 | INFO | Downloaded 34GiB to GPU in 2.34963s (14.4704GiB/s) | 20260507 16:52:03.980 | INFO | Analyzing chunk frames = [1075200,1075712) | 20260507 16:52:03.980 | INFO | MemInfo: { | 20260507 16:52:03.980 | INFO | "gpu" : " 17 GB", | 20260507 16:52:03.980 | INFO | "huge" : "", | 20260507 16:52:03.980 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:03.981 | INFO | "malloc" : " 14 GB", | 20260507 16:52:03.981 | INFO | "pinned" : " 29 GB" | 20260507 16:52:03.981 | INFO | } | 20260507 16:52:03.981 | INFO | Perf report for the last 5.14014 seconds: | 20260507 16:52:03.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:03.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:52:03.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:52:03.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:03.981 | INFO | Uploaded 36GiB to GPU in 2.39382s (15.0387GiB/s) | 20260507 16:52:03.981 | INFO | Downloaded 34GiB to GPU in 2.45914s (13.826GiB/s) | 20260507 16:52:09.119 | INFO | Analyzing chunk frames = [1075712,1076224) | 20260507 16:52:09.119 | INFO | MemInfo: { | 20260507 16:52:09.119 | INFO | "gpu" : " 17 GB", | 20260507 16:52:09.119 | INFO | "huge" : "", | 20260507 16:52:09.119 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:09.119 | INFO | "malloc" : " 14 GB", | 20260507 16:52:09.119 | INFO | "pinned" : " 29 GB" | 20260507 16:52:09.119 | INFO | } | 20260507 16:52:09.119 | INFO | Perf report for the last 5.13854 seconds: | 20260507 16:52:09.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:09.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.119 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:52:09.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.119 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 16:52:09.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:09.120 | INFO | Uploaded 36GiB to GPU in 2.36694s (15.2095GiB/s) | 20260507 16:52:09.120 | INFO | Downloaded 34GiB to GPU in 2.44382s (13.9127GiB/s) | 20260507 16:52:14.259 | INFO | Analyzing chunk frames = [1076224,1076736) | 20260507 16:52:14.259 | INFO | MemInfo: { | 20260507 16:52:14.259 | INFO | "gpu" : " 17 GB", | 20260507 16:52:14.259 | INFO | "huge" : "", | 20260507 16:52:14.259 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:14.259 | INFO | "malloc" : " 14 GB", | 20260507 16:52:14.259 | INFO | "pinned" : " 29 GB" | 20260507 16:52:14.259 | INFO | } | 20260507 16:52:14.259 | INFO | Perf report for the last 5.1398 seconds: | 20260507 16:52:14.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:14.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.259 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:52:14.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.259 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:52:14.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:14.260 | INFO | Uploaded 36GiB to GPU in 2.46243s (14.6197GiB/s) | 20260507 16:52:14.260 | INFO | Downloaded 34GiB to GPU in 2.40718s (14.1244GiB/s) | 20260507 16:52:19.398 | INFO | Analyzing chunk frames = [1076736,1077248) | 20260507 16:52:19.398 | INFO | MemInfo: { | 20260507 16:52:19.398 | INFO | "gpu" : " 17 GB", | 20260507 16:52:19.398 | INFO | "huge" : "", | 20260507 16:52:19.398 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:19.398 | INFO | "malloc" : " 14 GB", | 20260507 16:52:19.398 | INFO | "pinned" : " 29 GB" | 20260507 16:52:19.398 | INFO | } | 20260507 16:52:19.398 | INFO | Perf report for the last 5.13885 seconds: | 20260507 16:52:19.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:19.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260507 16:52:19.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:52:19.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:19.398 | INFO | Uploaded 36GiB to GPU in 2.53017s (14.2283GiB/s) | 20260507 16:52:19.398 | INFO | Downloaded 34GiB to GPU in 2.49634s (13.62GiB/s) | 20260507 16:52:19.495 | INFO | Starting a round of Baz IO | 20260507 16:52:23.007 | INFO | Populating RTMetrics report | 20260507 16:52:23.007 | INFO | RTMetrics report populated | 20260507 16:52:23.007 | INFO | Opening tmp RTMetrics json file | 20260507 16:52:23.012 | INFO | Serializing RTMetrics to json | 20260507 16:52:23.040 | INFO | Serialization done, swapping tmp file now... | 20260507 16:52:23.056 | INFO | RTMetrics done | 20260507 16:52:23.136 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.641 seconds | 20260507 16:52:23.136 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.28% / 0.02% / 50.69% | 20260507 16:52:23.136 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.08622% / 6.35783e-05% / 3.57628e-05% | 20260507 16:52:23.136 | INFO | Baz IO overall: | 20260507 16:52:23.137 | INFO | BytesWritten : 467574626550 | 20260507 16:52:23.137 | INFO | NumEvents : 226385240147 | 20260507 16:52:24.537 | INFO | Analyzing chunk frames = [1077248,1077760) | 20260507 16:52:24.537 | INFO | MemInfo: { | 20260507 16:52:24.537 | INFO | "gpu" : " 17 GB", | 20260507 16:52:24.537 | INFO | "huge" : "", | 20260507 16:52:24.537 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:24.537 | INFO | "malloc" : " 14 GB", | 20260507 16:52:24.537 | INFO | "pinned" : " 29 GB" | 20260507 16:52:24.537 | INFO | } | 20260507 16:52:24.537 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:52:24.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:24.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:24.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:24.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:24.537 | INFO | ANALYSIS : 69.8%, 3.0, 107.9ms, 0 | 20260507 16:52:24.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:24.537 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 16:52:24.537 | INFO | PRE_HQ : 66.5%, 1.7, 59.8ms, 0 | 20260507 16:52:24.537 | INFO | BAZWRITER : 70.3%, 3.7, 134.5ms, 0 | 20260507 16:52:24.537 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:52:24.537 | INFO | Uploaded 36GiB to GPU in 2.54481s (14.1464GiB/s) | 20260507 16:52:24.537 | INFO | Downloaded 38GiB to GPU in 2.71659s (13.9881GiB/s) | 20260507 16:52:29.676 | INFO | Analyzing chunk frames = [1077760,1078272) | 20260507 16:52:29.676 | INFO | MemInfo: { | 20260507 16:52:29.676 | INFO | "gpu" : " 17 GB", | 20260507 16:52:29.676 | INFO | "huge" : "", | 20260507 16:52:29.677 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:29.677 | INFO | "malloc" : " 14 GB", | 20260507 16:52:29.677 | INFO | "pinned" : " 29 GB" | 20260507 16:52:29.677 | INFO | } | 20260507 16:52:29.677 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:52:29.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:29.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 16:52:29.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:52:29.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:29.677 | INFO | Uploaded 36GiB to GPU in 2.49155s (14.4489GiB/s) | 20260507 16:52:29.677 | INFO | Downloaded 34GiB to GPU in 2.42347s (14.0295GiB/s) | 20260507 16:52:34.816 | INFO | Analyzing chunk frames = [1078272,1078784) | 20260507 16:52:34.816 | INFO | MemInfo: { | 20260507 16:52:34.816 | INFO | "gpu" : " 17 GB", | 20260507 16:52:34.816 | INFO | "huge" : "", | 20260507 16:52:34.816 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:34.816 | INFO | "malloc" : " 14 GB", | 20260507 16:52:34.816 | INFO | "pinned" : " 29 GB" | 20260507 16:52:34.816 | INFO | } | 20260507 16:52:34.816 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:52:34.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:34.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.816 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 16:52:34.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.816 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:52:34.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:34.817 | INFO | Uploaded 36GiB to GPU in 2.55494s (14.0904GiB/s) | 20260507 16:52:34.817 | INFO | Downloaded 34GiB to GPU in 2.41752s (14.064GiB/s) | 20260507 16:52:39.955 | INFO | Analyzing chunk frames = [1078784,1079296) | 20260507 16:52:39.955 | INFO | MemInfo: { | 20260507 16:52:39.955 | INFO | "gpu" : " 17 GB", | 20260507 16:52:39.955 | INFO | "huge" : "", | 20260507 16:52:39.955 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:39.955 | INFO | "malloc" : " 14 GB", | 20260507 16:52:39.955 | INFO | "pinned" : " 29 GB" | 20260507 16:52:39.955 | INFO | } | 20260507 16:52:39.955 | INFO | Perf report for the last 5.13925 seconds: | 20260507 16:52:39.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:39.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.955 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:52:39.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.955 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:52:39.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:39.956 | INFO | Uploaded 36GiB to GPU in 2.46887s (14.5816GiB/s) | 20260507 16:52:39.956 | INFO | Downloaded 34GiB to GPU in 2.35673s (14.4268GiB/s) | 20260507 16:52:45.095 | INFO | Analyzing chunk frames = [1079296,1079808) | 20260507 16:52:45.095 | INFO | MemInfo: { | 20260507 16:52:45.095 | INFO | "gpu" : " 17 GB", | 20260507 16:52:45.095 | INFO | "huge" : "", | 20260507 16:52:45.095 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:45.095 | INFO | "malloc" : " 14 GB", | 20260507 16:52:45.095 | INFO | "pinned" : " 29 GB" | 20260507 16:52:45.095 | INFO | } | 20260507 16:52:45.095 | INFO | Perf report for the last 5.13945 seconds: | 20260507 16:52:45.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:45.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 16:52:45.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 16:52:45.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:45.095 | INFO | Uploaded 36GiB to GPU in 2.31979s (15.5186GiB/s) | 20260507 16:52:45.095 | INFO | Downloaded 34GiB to GPU in 2.49351s (13.6354GiB/s) | 20260507 16:52:50.234 | INFO | Analyzing chunk frames = [1079808,1080320) | 20260507 16:52:50.234 | INFO | MemInfo: { | 20260507 16:52:50.234 | INFO | "gpu" : " 17 GB", | 20260507 16:52:50.234 | INFO | "huge" : "", | 20260507 16:52:50.234 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:50.234 | INFO | "malloc" : " 14 GB", | 20260507 16:52:50.234 | INFO | "pinned" : " 29 GB" | 20260507 16:52:50.234 | INFO | } | 20260507 16:52:50.234 | INFO | Perf report for the last 5.13944 seconds: | 20260507 16:52:50.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:50.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.234 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:52:50.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.234 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 16:52:50.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:50.235 | INFO | Uploaded 36GiB to GPU in 2.34975s (15.3208GiB/s) | 20260507 16:52:50.235 | INFO | Downloaded 34GiB to GPU in 2.45726s (13.8366GiB/s) | 20260507 16:52:55.373 | INFO | Analyzing chunk frames = [1080320,1080832) | 20260507 16:52:55.373 | INFO | MemInfo: { | 20260507 16:52:55.373 | INFO | "gpu" : " 17 GB", | 20260507 16:52:55.373 | INFO | "huge" : "", | 20260507 16:52:55.373 | INFO | "huge_shared" : " 29 GB", | 20260507 16:52:55.373 | INFO | "malloc" : " 14 GB", | 20260507 16:52:55.373 | INFO | "pinned" : " 29 GB" | 20260507 16:52:55.374 | INFO | } | 20260507 16:52:55.374 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:52:55.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:52:55.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 16:52:55.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:52:55.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:52:55.374 | INFO | Uploaded 36GiB to GPU in 2.45033s (14.6919GiB/s) | 20260507 16:52:55.374 | INFO | Downloaded 34GiB to GPU in 2.41855s (14.058GiB/s) | 20260507 16:53:00.513 | INFO | Analyzing chunk frames = [1080832,1081344) | 20260507 16:53:00.513 | INFO | MemInfo: { | 20260507 16:53:00.513 | INFO | "gpu" : " 17 GB", | 20260507 16:53:00.513 | INFO | "huge" : "", | 20260507 16:53:00.513 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:00.513 | INFO | "malloc" : " 14 GB", | 20260507 16:53:00.513 | INFO | "pinned" : " 29 GB" | 20260507 16:53:00.513 | INFO | } | 20260507 16:53:00.513 | INFO | Perf report for the last 5.13918 seconds: | 20260507 16:53:00.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:00.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:53:00.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:53:00.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:00.513 | INFO | Uploaded 36GiB to GPU in 2.48513s (14.4861GiB/s) | 20260507 16:53:00.513 | INFO | Downloaded 34GiB to GPU in 2.5065s (13.5647GiB/s) | 20260507 16:53:00.513 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:53:00.513 | INFO | Total mem usage: 16.4 GiB | 20260507 16:53:00.513 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:53:00.513 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 16:53:00.513 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:53:00.513 | INFO | High Water Breakdown: | 20260507 16:53:00.513 | INFO | 59.1% : 8.61 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:53:00.513 | INFO | 40.8% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:53:00.513 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:53:00.513 | INFO | ------------------------------------------ | 20260507 16:53:00.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:53:00.514 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:53:00.514 | INFO | Total mem usage: 26.5 GiB | 20260507 16:53:00.514 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:53:00.514 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:53:00.514 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:53:00.514 | INFO | High Water Breakdown: | 20260507 16:53:00.514 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:53:00.514 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:53:00.514 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:53:00.514 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:53:00.514 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:53:00.514 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:53:00.514 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:53:00.514 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:53:00.514 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:53:00.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:53:00.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:53:00.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:53:00.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:53:00.514 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:53:00.514 | INFO | ------------------------------------------ | 20260507 16:53:00.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:53:00.514 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:53:00.514 | INFO | Total mem usage: 27.8 GiB | 20260507 16:53:00.514 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:53:00.514 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:53:00.514 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:53:00.514 | INFO | High Water Breakdown: | 20260507 16:53:00.514 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:53:00.514 | INFO | ------------------------------------------ | 20260507 16:53:00.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:53:00.514 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:53:00.514 | INFO | Total mem usage: 15.7 GiB | 20260507 16:53:00.514 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:53:00.514 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:53:00.514 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:53:00.514 | INFO | High Water Breakdown: | 20260507 16:53:00.514 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:53:00.514 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:53:00.514 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:53:00.514 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:53:00.514 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:53:00.514 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:53:00.514 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:53:00.514 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:53:00.514 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:53:00.514 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:53:00.514 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:53:00.514 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:53:00.514 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:53:00.514 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:53:00.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:53:00.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:53:00.514 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:53:00.514 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:53:00.514 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:53:00.514 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:53:00.514 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:53:00.514 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:53:00.514 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:53:00.514 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:53:00.514 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:53:00.514 | INFO | ------------------------------------------ | 20260507 16:53:00.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:53:00.515 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:53:00.515 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:53:00.515 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:53:00.515 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.67e+01, 1.44e+01 / 2.91e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 0.29%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.15e-02 / 6.09e-03 | 20260507 16:53:00.515 | INFO | PerfReport: 12.06%, Baseline -- 4.06e+00 / 4.50e+00, 6.28e-01 / 2.60e-01 | 20260507 16:53:00.515 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.48e+00, 2.17e-01 / 1.13e-01 | 20260507 16:53:00.515 | INFO | PerfReport: 15.70%, DME -- 4.23e+01 / 4.51e+01, 4.53e+00 / 2.92e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.90e+00 / 2.30e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.54e+00 / 2.93e+00, 4.39e-01 / 2.50e-01 | 20260507 16:53:00.515 | INFO | PerfReport: 3.80%, Metrics -- 1.28e+00 / 1.45e+00, 3.26e-01 / 1.82e-01 | 20260507 16:53:00.515 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.06e-01 / 4.31e-01, 6.83e-02 / 4.11e-02 | 20260507 16:53:00.515 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:53:00.515 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:53:00.515 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:53:00.515 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 23.90%, Upload -- 2.57e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 23.93%, Download -- 2.57e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 31.27%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 1.83%, LockUpload -- 1.96e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 2.23%, LockDownload -- 2.39e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 16.84%, LockCompute -- 1.81e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:53:00.515 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 16:53:00.613 | INFO | Starting a round of Baz IO | 20260507 16:53:04.125 | INFO | Populating RTMetrics report | 20260507 16:53:04.125 | INFO | RTMetrics report populated | 20260507 16:53:04.125 | INFO | Opening tmp RTMetrics json file | 20260507 16:53:04.151 | INFO | Serializing RTMetrics to json | 20260507 16:53:04.153 | INFO | Serialization done, swapping tmp file now... | 20260507 16:53:04.153 | INFO | RTMetrics done | 20260507 16:53:04.251 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.638 seconds | 20260507 16:53:04.252 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.26% / 0.02% / 50.72% | 20260507 16:53:04.252 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0641% / 3.57628e-05% / 1.98682e-05% | 20260507 16:53:04.252 | INFO | Baz IO overall: | 20260507 16:53:04.252 | INFO | BytesWritten : 469497765394 | 20260507 16:53:04.252 | INFO | NumEvents : 227319776957 | 20260507 16:53:05.652 | INFO | Analyzing chunk frames = [1081344,1081856) | 20260507 16:53:05.652 | INFO | MemInfo: { | 20260507 16:53:05.652 | INFO | "gpu" : " 17 GB", | 20260507 16:53:05.652 | INFO | "huge" : "", | 20260507 16:53:05.652 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:05.652 | INFO | "malloc" : " 14 GB", | 20260507 16:53:05.652 | INFO | "pinned" : " 29 GB" | 20260507 16:53:05.652 | INFO | } | 20260507 16:53:05.652 | INFO | Perf report for the last 5.13908 seconds: | 20260507 16:53:05.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:05.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:05.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:05.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:05.652 | INFO | ANALYSIS : 69.8%, 3.0, 108.1ms, 0 | 20260507 16:53:05.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:05.652 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 16:53:05.652 | INFO | PRE_HQ : 66.5%, 1.7, 60.0ms, 0 | 20260507 16:53:05.652 | INFO | BAZWRITER : 70.5%, 4.0, 143.7ms, 0 | 20260507 16:53:05.652 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 16:53:05.652 | INFO | Uploaded 36GiB to GPU in 2.52868s (14.2367GiB/s) | 20260507 16:53:05.652 | INFO | Downloaded 38GiB to GPU in 2.71683s (13.9869GiB/s) | 20260507 16:53:10.791 | INFO | Analyzing chunk frames = [1081856,1082368) | 20260507 16:53:10.791 | INFO | MemInfo: { | 20260507 16:53:10.791 | INFO | "gpu" : " 17 GB", | 20260507 16:53:10.791 | INFO | "huge" : "", | 20260507 16:53:10.791 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:10.791 | INFO | "malloc" : " 14 GB", | 20260507 16:53:10.792 | INFO | "pinned" : " 29 GB" | 20260507 16:53:10.792 | INFO | } | 20260507 16:53:10.792 | INFO | Perf report for the last 5.13961 seconds: | 20260507 16:53:10.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:10.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:53:10.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:53:10.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:10.792 | INFO | Uploaded 36GiB to GPU in 2.51734s (14.3008GiB/s) | 20260507 16:53:10.792 | INFO | Downloaded 34GiB to GPU in 2.43964s (13.9365GiB/s) | 20260507 16:53:15.931 | INFO | Analyzing chunk frames = [1082368,1082880) | 20260507 16:53:15.931 | INFO | MemInfo: { | 20260507 16:53:15.931 | INFO | "gpu" : " 17 GB", | 20260507 16:53:15.931 | INFO | "huge" : "", | 20260507 16:53:15.931 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:15.931 | INFO | "malloc" : " 14 GB", | 20260507 16:53:15.931 | INFO | "pinned" : " 29 GB" | 20260507 16:53:15.931 | INFO | } | 20260507 16:53:15.931 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:53:15.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:15.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 16:53:15.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 16:53:15.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:15.931 | INFO | Uploaded 36GiB to GPU in 2.5639s (14.0411GiB/s) | 20260507 16:53:15.931 | INFO | Downloaded 34GiB to GPU in 2.42803s (14.0031GiB/s) | 20260507 16:53:21.070 | INFO | Analyzing chunk frames = [1082880,1083392) | 20260507 16:53:21.070 | INFO | MemInfo: { | 20260507 16:53:21.070 | INFO | "gpu" : " 17 GB", | 20260507 16:53:21.070 | INFO | "huge" : "", | 20260507 16:53:21.070 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:21.070 | INFO | "malloc" : " 14 GB", | 20260507 16:53:21.070 | INFO | "pinned" : " 29 GB" | 20260507 16:53:21.070 | INFO | } | 20260507 16:53:21.070 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:53:21.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:21.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.070 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 16:53:21.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.070 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:53:21.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:21.071 | INFO | Uploaded 36GiB to GPU in 2.46088s (14.6289GiB/s) | 20260507 16:53:21.071 | INFO | Downloaded 34GiB to GPU in 2.3608s (14.4019GiB/s) | 20260507 16:53:26.209 | INFO | Analyzing chunk frames = [1083392,1083904) | 20260507 16:53:26.209 | INFO | MemInfo: { | 20260507 16:53:26.209 | INFO | "gpu" : " 17 GB", | 20260507 16:53:26.209 | INFO | "huge" : "", | 20260507 16:53:26.209 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:26.210 | INFO | "malloc" : " 14 GB", | 20260507 16:53:26.210 | INFO | "pinned" : " 29 GB" | 20260507 16:53:26.210 | INFO | } | 20260507 16:53:26.210 | INFO | Perf report for the last 5.13919 seconds: | 20260507 16:53:26.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:26.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 16:53:26.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 16:53:26.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:26.210 | INFO | Uploaded 36GiB to GPU in 2.31201s (15.5708GiB/s) | 20260507 16:53:26.210 | INFO | Downloaded 34GiB to GPU in 2.47294s (13.7488GiB/s) | 20260507 16:53:31.349 | INFO | Analyzing chunk frames = [1083904,1084416) | 20260507 16:53:31.349 | INFO | MemInfo: { | 20260507 16:53:31.349 | INFO | "gpu" : " 17 GB", | 20260507 16:53:31.349 | INFO | "huge" : "", | 20260507 16:53:31.349 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:31.349 | INFO | "malloc" : " 14 GB", | 20260507 16:53:31.349 | INFO | "pinned" : " 29 GB" | 20260507 16:53:31.349 | INFO | } | 20260507 16:53:31.349 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:53:31.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:31.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:53:31.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 16:53:31.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:31.349 | INFO | Uploaded 36GiB to GPU in 2.29903s (15.6588GiB/s) | 20260507 16:53:31.349 | INFO | Downloaded 34GiB to GPU in 2.47878s (13.7164GiB/s) | 20260507 16:53:36.488 | INFO | Analyzing chunk frames = [1084416,1084928) | 20260507 16:53:36.488 | INFO | MemInfo: { | 20260507 16:53:36.488 | INFO | "gpu" : " 17 GB", | 20260507 16:53:36.488 | INFO | "huge" : "", | 20260507 16:53:36.488 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:36.488 | INFO | "malloc" : " 14 GB", | 20260507 16:53:36.488 | INFO | "pinned" : " 29 GB" | 20260507 16:53:36.488 | INFO | } | 20260507 16:53:36.488 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:53:36.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:36.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.488 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 16:53:36.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.488 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:53:36.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:36.489 | INFO | Uploaded 36GiB to GPU in 2.4058s (14.9639GiB/s) | 20260507 16:53:36.489 | INFO | Downloaded 34GiB to GPU in 2.42878s (13.9988GiB/s) | 20260507 16:53:41.627 | INFO | Analyzing chunk frames = [1084928,1085440) | 20260507 16:53:41.627 | INFO | MemInfo: { | 20260507 16:53:41.627 | INFO | "gpu" : " 17 GB", | 20260507 16:53:41.627 | INFO | "huge" : "", | 20260507 16:53:41.627 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:41.627 | INFO | "malloc" : " 14 GB", | 20260507 16:53:41.627 | INFO | "pinned" : " 29 GB" | 20260507 16:53:41.627 | INFO | } | 20260507 16:53:41.627 | INFO | Perf report for the last 5.13897 seconds: | 20260507 16:53:41.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:41.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.627 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 16:53:41.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.627 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:53:41.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:41.628 | INFO | Uploaded 36GiB to GPU in 2.47533s (14.5435GiB/s) | 20260507 16:53:41.628 | INFO | Downloaded 34GiB to GPU in 2.53085s (13.4342GiB/s) | 20260507 16:53:41.724 | INFO | Starting a round of Baz IO | 20260507 16:53:45.274 | INFO | Populating RTMetrics report | 20260507 16:53:45.275 | INFO | RTMetrics report populated | 20260507 16:53:45.275 | INFO | Opening tmp RTMetrics json file | 20260507 16:53:45.276 | INFO | Serializing RTMetrics to json | 20260507 16:53:45.277 | INFO | Serialization done, swapping tmp file now... | 20260507 16:53:45.278 | INFO | RTMetrics done | 20260507 16:53:45.396 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.671 seconds | 20260507 16:53:45.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.23% / 0.02% / 50.74% | 20260507 16:53:45.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0452% / 7.15256e-05% / 1.58946e-05% | 20260507 16:53:45.396 | INFO | Baz IO overall: | 20260507 16:53:45.396 | INFO | BytesWritten : 471420204834 | 20260507 16:53:45.396 | INFO | NumEvents : 228254172074 | 20260507 16:53:46.766 | INFO | Analyzing chunk frames = [1085440,1085952) | 20260507 16:53:46.766 | INFO | MemInfo: { | 20260507 16:53:46.766 | INFO | "gpu" : " 17 GB", | 20260507 16:53:46.766 | INFO | "huge" : "", | 20260507 16:53:46.766 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:46.766 | INFO | "malloc" : " 14 GB", | 20260507 16:53:46.766 | INFO | "pinned" : " 29 GB" | 20260507 16:53:46.766 | INFO | } | 20260507 16:53:46.766 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:53:46.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:46.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:46.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:46.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:46.767 | INFO | ANALYSIS : 69.8%, 2.9, 107.8ms, 0 | 20260507 16:53:46.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:46.767 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 16:53:46.767 | INFO | PRE_HQ : 67.3%, 1.8, 63.3ms, 0 | 20260507 16:53:46.767 | INFO | BAZWRITER : 70.9%, 3.9, 141.1ms, 0 | 20260507 16:53:46.767 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:53:46.767 | INFO | Uploaded 36GiB to GPU in 2.54895s (14.1235GiB/s) | 20260507 16:53:46.767 | INFO | Downloaded 38GiB to GPU in 2.74197s (13.8586GiB/s) | 20260507 16:53:51.905 | INFO | Analyzing chunk frames = [1085952,1086464) | 20260507 16:53:51.905 | INFO | MemInfo: { | 20260507 16:53:51.905 | INFO | "gpu" : " 17 GB", | 20260507 16:53:51.905 | INFO | "huge" : "", | 20260507 16:53:51.906 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:51.906 | INFO | "malloc" : " 14 GB", | 20260507 16:53:51.906 | INFO | "pinned" : " 29 GB" | 20260507 16:53:51.906 | INFO | } | 20260507 16:53:51.906 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:53:51.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:51.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:53:51.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:53:51.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:51.906 | INFO | Uploaded 36GiB to GPU in 2.49598s (14.4232GiB/s) | 20260507 16:53:51.906 | INFO | Downloaded 34GiB to GPU in 2.44164s (13.925GiB/s) | 20260507 16:53:57.045 | INFO | Analyzing chunk frames = [1086464,1086976) | 20260507 16:53:57.045 | INFO | MemInfo: { | 20260507 16:53:57.045 | INFO | "gpu" : " 17 GB", | 20260507 16:53:57.045 | INFO | "huge" : "", | 20260507 16:53:57.045 | INFO | "huge_shared" : " 29 GB", | 20260507 16:53:57.045 | INFO | "malloc" : " 14 GB", | 20260507 16:53:57.045 | INFO | "pinned" : " 29 GB" | 20260507 16:53:57.045 | INFO | } | 20260507 16:53:57.045 | INFO | Perf report for the last 5.13903 seconds: | 20260507 16:53:57.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:53:57.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 16:53:57.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:53:57.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:53:57.045 | INFO | Uploaded 36GiB to GPU in 2.54534s (14.1435GiB/s) | 20260507 16:53:57.045 | INFO | Downloaded 34GiB to GPU in 2.42655s (14.0116GiB/s) | 20260507 16:54:02.184 | INFO | Analyzing chunk frames = [1086976,1087488) | 20260507 16:54:02.184 | INFO | MemInfo: { | 20260507 16:54:02.184 | INFO | "gpu" : " 17 GB", | 20260507 16:54:02.184 | INFO | "huge" : "", | 20260507 16:54:02.184 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:02.184 | INFO | "malloc" : " 14 GB", | 20260507 16:54:02.184 | INFO | "pinned" : " 29 GB" | 20260507 16:54:02.184 | INFO | } | 20260507 16:54:02.184 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:54:02.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:02.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 16:54:02.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:54:02.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:02.184 | INFO | Uploaded 36GiB to GPU in 2.48599s (14.4811GiB/s) | 20260507 16:54:02.184 | INFO | Downloaded 34GiB to GPU in 2.35635s (14.4291GiB/s) | 20260507 16:54:07.324 | INFO | Analyzing chunk frames = [1087488,1088000) | 20260507 16:54:07.324 | INFO | MemInfo: { | 20260507 16:54:07.324 | INFO | "gpu" : " 17 GB", | 20260507 16:54:07.324 | INFO | "huge" : "", | 20260507 16:54:07.324 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:07.324 | INFO | "malloc" : " 14 GB", | 20260507 16:54:07.324 | INFO | "pinned" : " 29 GB" | 20260507 16:54:07.324 | INFO | } | 20260507 16:54:07.324 | INFO | Perf report for the last 5.13982 seconds: | 20260507 16:54:07.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:07.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.324 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:54:07.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.324 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 16:54:07.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:07.325 | INFO | Uploaded 36GiB to GPU in 2.34785s (15.3332GiB/s) | 20260507 16:54:07.325 | INFO | Downloaded 34GiB to GPU in 2.47921s (13.714GiB/s) | 20260507 16:54:12.463 | INFO | Analyzing chunk frames = [1088000,1088512) | 20260507 16:54:12.463 | INFO | MemInfo: { | 20260507 16:54:12.463 | INFO | "gpu" : " 17 GB", | 20260507 16:54:12.463 | INFO | "huge" : "", | 20260507 16:54:12.463 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:12.463 | INFO | "malloc" : " 14 GB", | 20260507 16:54:12.463 | INFO | "pinned" : " 29 GB" | 20260507 16:54:12.463 | INFO | } | 20260507 16:54:12.463 | INFO | Perf report for the last 5.1389 seconds: | 20260507 16:54:12.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:12.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:54:12.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 16:54:12.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:12.463 | INFO | Uploaded 36GiB to GPU in 2.37713s (15.1443GiB/s) | 20260507 16:54:12.463 | INFO | Downloaded 34GiB to GPU in 2.44828s (13.8873GiB/s) | 20260507 16:54:17.602 | INFO | Analyzing chunk frames = [1088512,1089024) | 20260507 16:54:17.602 | INFO | MemInfo: { | 20260507 16:54:17.602 | INFO | "gpu" : " 17 GB", | 20260507 16:54:17.602 | INFO | "huge" : "", | 20260507 16:54:17.602 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:17.602 | INFO | "malloc" : " 14 GB", | 20260507 16:54:17.602 | INFO | "pinned" : " 29 GB" | 20260507 16:54:17.602 | INFO | } | 20260507 16:54:17.603 | INFO | Perf report for the last 5.1396 seconds: | 20260507 16:54:17.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:17.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:54:17.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:54:17.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:17.603 | INFO | Uploaded 36GiB to GPU in 2.46904s (14.5806GiB/s) | 20260507 16:54:17.603 | INFO | Downloaded 34GiB to GPU in 2.40917s (14.1127GiB/s) | 20260507 16:54:22.741 | INFO | Analyzing chunk frames = [1089024,1089536) | 20260507 16:54:22.741 | INFO | MemInfo: { | 20260507 16:54:22.741 | INFO | "gpu" : " 17 GB", | 20260507 16:54:22.741 | INFO | "huge" : "", | 20260507 16:54:22.741 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:22.741 | INFO | "malloc" : " 14 GB", | 20260507 16:54:22.742 | INFO | "pinned" : " 29 GB" | 20260507 16:54:22.742 | INFO | } | 20260507 16:54:22.742 | INFO | Perf report for the last 5.13915 seconds: | 20260507 16:54:22.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:22.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 16:54:22.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:54:22.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:22.742 | INFO | Uploaded 36GiB to GPU in 2.54584s (14.1407GiB/s) | 20260507 16:54:22.742 | INFO | Downloaded 34GiB to GPU in 2.50683s (13.563GiB/s) | 20260507 16:54:22.742 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:54:22.742 | INFO | Total mem usage: 16.4 GiB | 20260507 16:54:22.742 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:54:22.742 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.9%) | 20260507 16:54:22.742 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:54:22.742 | INFO | High Water Breakdown: | 20260507 16:54:22.742 | INFO | 59.4% : 8.78 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:54:22.742 | INFO | 40.5% : 5.99 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:54:22.742 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:54:22.742 | INFO | ------------------------------------------ | 20260507 16:54:22.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:54:22.743 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:54:22.743 | INFO | Total mem usage: 26.5 GiB | 20260507 16:54:22.743 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:54:22.743 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:54:22.743 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:54:22.743 | INFO | High Water Breakdown: | 20260507 16:54:22.743 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:54:22.743 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:54:22.743 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:54:22.743 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:54:22.743 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:54:22.743 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:54:22.743 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:54:22.743 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:54:22.743 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:54:22.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:54:22.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:54:22.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:54:22.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:54:22.743 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:54:22.743 | INFO | ------------------------------------------ | 20260507 16:54:22.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:54:22.743 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:54:22.743 | INFO | Total mem usage: 27.8 GiB | 20260507 16:54:22.743 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:54:22.743 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:54:22.743 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:54:22.743 | INFO | High Water Breakdown: | 20260507 16:54:22.743 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:54:22.743 | INFO | ------------------------------------------ | 20260507 16:54:22.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:54:22.743 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:54:22.743 | INFO | Total mem usage: 15.7 GiB | 20260507 16:54:22.743 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:54:22.743 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:54:22.743 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:54:22.743 | INFO | High Water Breakdown: | 20260507 16:54:22.743 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:54:22.743 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:54:22.743 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:54:22.743 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:54:22.743 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:54:22.743 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:54:22.743 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:54:22.743 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:54:22.743 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:54:22.743 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:54:22.743 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:54:22.743 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:54:22.743 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:54:22.743 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:54:22.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:54:22.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:54:22.743 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:54:22.743 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:54:22.743 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:54:22.743 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:54:22.743 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:54:22.743 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:54:22.743 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:54:22.743 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:54:22.743 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:54:22.743 | INFO | ------------------------------------------ | 20260507 16:54:22.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:54:22.743 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:54:22.743 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:54:22.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:54:22.743 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.68e+01, 1.44e+01 / 3.04e+00 | 20260507 16:54:22.743 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.01e-01, 1.15e-02 / 4.76e-03 | 20260507 16:54:22.743 | INFO | PerfReport: 12.03%, Baseline -- 4.04e+00 / 4.51e+00, 6.24e-01 / 2.21e-01 | 20260507 16:54:22.743 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.17e-01 / 1.06e-01 | 20260507 16:54:22.744 | INFO | PerfReport: 15.75%, DME -- 4.23e+01 / 4.52e+01, 4.47e+00 / 2.84e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 56.00%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.87e+00 / 2.35e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.92e+00, 4.37e-01 / 2.28e-01 | 20260507 16:54:22.744 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.45e+00, 3.22e-01 / 2.03e-01 | 20260507 16:54:22.744 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.07e-01 / 4.32e-01, 6.89e-02 / 4.52e-02 | 20260507 16:54:22.744 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:54:22.744 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:54:22.744 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:54:22.744 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 23.85%, Upload -- 2.56e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 31.26%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 1.78%, LockUpload -- 1.91e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 2.33%, LockDownload -- 2.51e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 16.74%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:54:22.744 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260507 16:54:22.839 | INFO | Starting a round of Baz IO | 20260507 16:54:26.353 | INFO | Populating RTMetrics report | 20260507 16:54:26.353 | INFO | RTMetrics report populated | 20260507 16:54:26.353 | INFO | Opening tmp RTMetrics json file | 20260507 16:54:26.373 | INFO | Serializing RTMetrics to json | 20260507 16:54:26.375 | INFO | Serialization done, swapping tmp file now... | 20260507 16:54:26.375 | INFO | RTMetrics done | 20260507 16:54:26.456 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.616 seconds | 20260507 16:54:26.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.21% / 0.02% / 50.77% | 20260507 16:54:26.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0491% / 3.17891e-05% / 3.97364e-05% | 20260507 16:54:26.456 | INFO | Baz IO overall: | 20260507 16:54:26.456 | INFO | BytesWritten : 473343589195 | 20260507 16:54:26.456 | INFO | NumEvents : 229189207514 | 20260507 16:54:27.881 | INFO | Analyzing chunk frames = [1089536,1090048) | 20260507 16:54:27.881 | INFO | MemInfo: { | 20260507 16:54:27.881 | INFO | "gpu" : " 17 GB", | 20260507 16:54:27.881 | INFO | "huge" : "", | 20260507 16:54:27.881 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:27.881 | INFO | "malloc" : " 14 GB", | 20260507 16:54:27.881 | INFO | "pinned" : " 29 GB" | 20260507 16:54:27.881 | INFO | } | 20260507 16:54:27.881 | INFO | Perf report for the last 5.13904 seconds: | 20260507 16:54:27.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:27.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:27.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:27.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:27.881 | INFO | ANALYSIS : 69.9%, 2.9, 107.7ms, 0 | 20260507 16:54:27.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:27.881 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 16:54:27.881 | INFO | PRE_HQ : 66.6%, 1.7, 60.0ms, 0 | 20260507 16:54:27.881 | INFO | BAZWRITER : 69.8%, 3.8, 137.5ms, 0 | 20260507 16:54:27.881 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 16:54:27.881 | INFO | Uploaded 36GiB to GPU in 2.59104s (13.894GiB/s) | 20260507 16:54:27.881 | INFO | Downloaded 38GiB to GPU in 2.70625s (14.0415GiB/s) | 20260507 16:54:33.020 | INFO | Analyzing chunk frames = [1090048,1090560) | 20260507 16:54:33.020 | INFO | MemInfo: { | 20260507 16:54:33.020 | INFO | "gpu" : " 17 GB", | 20260507 16:54:33.020 | INFO | "huge" : "", | 20260507 16:54:33.020 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:33.020 | INFO | "malloc" : " 14 GB", | 20260507 16:54:33.020 | INFO | "pinned" : " 29 GB" | 20260507 16:54:33.020 | INFO | } | 20260507 16:54:33.020 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:54:33.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:33.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.020 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 16:54:33.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.020 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:54:33.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:33.021 | INFO | Uploaded 36GiB to GPU in 2.53455s (14.2037GiB/s) | 20260507 16:54:33.021 | INFO | Downloaded 34GiB to GPU in 2.42727s (14.0075GiB/s) | 20260507 16:54:38.159 | INFO | Analyzing chunk frames = [1090560,1091072) | 20260507 16:54:38.159 | INFO | MemInfo: { | 20260507 16:54:38.159 | INFO | "gpu" : " 17 GB", | 20260507 16:54:38.159 | INFO | "huge" : "", | 20260507 16:54:38.159 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:38.159 | INFO | "malloc" : " 14 GB", | 20260507 16:54:38.159 | INFO | "pinned" : " 29 GB" | 20260507 16:54:38.159 | INFO | } | 20260507 16:54:38.159 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:54:38.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:38.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 16:54:38.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 16:54:38.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:38.160 | INFO | Uploaded 36GiB to GPU in 2.48779s (14.4707GiB/s) | 20260507 16:54:38.160 | INFO | Downloaded 34GiB to GPU in 2.33138s (14.5836GiB/s) | 20260507 16:54:43.299 | INFO | Analyzing chunk frames = [1091072,1091584) | 20260507 16:54:43.299 | INFO | MemInfo: { | 20260507 16:54:43.299 | INFO | "gpu" : " 17 GB", | 20260507 16:54:43.299 | INFO | "huge" : "", | 20260507 16:54:43.299 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:43.299 | INFO | "malloc" : " 14 GB", | 20260507 16:54:43.299 | INFO | "pinned" : " 29 GB" | 20260507 16:54:43.299 | INFO | } | 20260507 16:54:43.299 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:54:43.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:43.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:54:43.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:54:43.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:43.299 | INFO | Uploaded 36GiB to GPU in 2.48109s (14.5097GiB/s) | 20260507 16:54:43.299 | INFO | Downloaded 34GiB to GPU in 2.34557s (14.4954GiB/s) | 20260507 16:54:48.438 | INFO | Analyzing chunk frames = [1091584,1092096) | 20260507 16:54:48.438 | INFO | MemInfo: { | 20260507 16:54:48.438 | INFO | "gpu" : " 17 GB", | 20260507 16:54:48.438 | INFO | "huge" : "", | 20260507 16:54:48.438 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:48.438 | INFO | "malloc" : " 14 GB", | 20260507 16:54:48.439 | INFO | "pinned" : " 29 GB" | 20260507 16:54:48.439 | INFO | } | 20260507 16:54:48.439 | INFO | Perf report for the last 5.13979 seconds: | 20260507 16:54:48.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:48.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:54:48.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 16:54:48.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:48.439 | INFO | Uploaded 36GiB to GPU in 2.39614s (15.0242GiB/s) | 20260507 16:54:48.439 | INFO | Downloaded 34GiB to GPU in 2.45054s (13.8745GiB/s) | 20260507 16:54:53.578 | INFO | Analyzing chunk frames = [1092096,1092608) | 20260507 16:54:53.578 | INFO | MemInfo: { | 20260507 16:54:53.578 | INFO | "gpu" : " 17 GB", | 20260507 16:54:53.578 | INFO | "huge" : "", | 20260507 16:54:53.578 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:53.578 | INFO | "malloc" : " 14 GB", | 20260507 16:54:53.578 | INFO | "pinned" : " 29 GB" | 20260507 16:54:53.578 | INFO | } | 20260507 16:54:53.578 | INFO | Perf report for the last 5.1392 seconds: | 20260507 16:54:53.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:53.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:54:53.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 16:54:53.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:53.578 | INFO | Uploaded 36GiB to GPU in 2.35101s (15.3126GiB/s) | 20260507 16:54:53.578 | INFO | Downloaded 34GiB to GPU in 2.48468s (13.6838GiB/s) | 20260507 16:54:58.717 | INFO | Analyzing chunk frames = [1092608,1093120) | 20260507 16:54:58.717 | INFO | MemInfo: { | 20260507 16:54:58.717 | INFO | "gpu" : " 17 GB", | 20260507 16:54:58.717 | INFO | "huge" : "", | 20260507 16:54:58.717 | INFO | "huge_shared" : " 29 GB", | 20260507 16:54:58.717 | INFO | "malloc" : " 14 GB", | 20260507 16:54:58.717 | INFO | "pinned" : " 29 GB" | 20260507 16:54:58.717 | INFO | } | 20260507 16:54:58.717 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:54:58.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:54:58.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 16:54:58.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:54:58.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:54:58.717 | INFO | Uploaded 36GiB to GPU in 2.41693s (14.8949GiB/s) | 20260507 16:54:58.717 | INFO | Downloaded 34GiB to GPU in 2.4183s (14.0595GiB/s) | 20260507 16:55:03.856 | INFO | Analyzing chunk frames = [1093120,1093632) | 20260507 16:55:03.856 | INFO | MemInfo: { | 20260507 16:55:03.856 | INFO | "gpu" : " 17 GB", | 20260507 16:55:03.856 | INFO | "huge" : "", | 20260507 16:55:03.856 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:03.856 | INFO | "malloc" : " 14 GB", | 20260507 16:55:03.856 | INFO | "pinned" : " 29 GB" | 20260507 16:55:03.856 | INFO | } | 20260507 16:55:03.856 | INFO | Perf report for the last 5.13908 seconds: | 20260507 16:55:03.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:03.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.856 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:55:03.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.856 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:55:03.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:03.857 | INFO | Uploaded 36GiB to GPU in 2.49628s (14.4215GiB/s) | 20260507 16:55:03.857 | INFO | Downloaded 34GiB to GPU in 2.50123s (13.5933GiB/s) | 20260507 16:55:03.953 | INFO | Starting a round of Baz IO | 20260507 16:55:07.460 | INFO | Populating RTMetrics report | 20260507 16:55:07.460 | INFO | RTMetrics report populated | 20260507 16:55:07.460 | INFO | Opening tmp RTMetrics json file | 20260507 16:55:07.471 | INFO | Serializing RTMetrics to json | 20260507 16:55:07.485 | INFO | Serialization done, swapping tmp file now... | 20260507 16:55:07.486 | INFO | RTMetrics done | 20260507 16:55:07.573 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.619 seconds | 20260507 16:55:07.573 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.19% / 0.02% / 50.79% | 20260507 16:55:07.573 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.03156% / 4.76837e-05% / 2.38419e-05% | 20260507 16:55:07.573 | INFO | Baz IO overall: | 20260507 16:55:07.573 | INFO | BytesWritten : 475266879161 | 20260507 16:55:07.573 | INFO | NumEvents : 230124045384 | 20260507 16:55:08.996 | INFO | Analyzing chunk frames = [1093632,1094144) | 20260507 16:55:08.996 | INFO | MemInfo: { | 20260507 16:55:08.996 | INFO | "gpu" : " 17 GB", | 20260507 16:55:08.996 | INFO | "huge" : "", | 20260507 16:55:08.996 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:08.996 | INFO | "malloc" : " 14 GB", | 20260507 16:55:08.996 | INFO | "pinned" : " 29 GB" | 20260507 16:55:08.996 | INFO | } | 20260507 16:55:08.996 | INFO | Perf report for the last 5.13969 seconds: | 20260507 16:55:08.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:08.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:08.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:08.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:08.996 | INFO | ANALYSIS : 69.6%, 2.9, 107.0ms, 0 | 20260507 16:55:08.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:08.996 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 16:55:08.996 | INFO | PRE_HQ : 67.3%, 1.8, 62.9ms, 0 | 20260507 16:55:08.996 | INFO | BAZWRITER : 70.0%, 4.3, 155.5ms, 0 | 20260507 16:55:08.996 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 16:55:08.996 | INFO | Uploaded 36GiB to GPU in 2.54042s (14.1709GiB/s) | 20260507 16:55:08.996 | INFO | Downloaded 38GiB to GPU in 2.71613s (13.9905GiB/s) | 20260507 16:55:14.135 | INFO | Analyzing chunk frames = [1094144,1094656) | 20260507 16:55:14.135 | INFO | MemInfo: { | 20260507 16:55:14.135 | INFO | "gpu" : " 17 GB", | 20260507 16:55:14.135 | INFO | "huge" : "", | 20260507 16:55:14.135 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:14.135 | INFO | "malloc" : " 14 GB", | 20260507 16:55:14.135 | INFO | "pinned" : " 29 GB" | 20260507 16:55:14.135 | INFO | } | 20260507 16:55:14.135 | INFO | Perf report for the last 5.13915 seconds: | 20260507 16:55:14.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:14.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.135 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 16:55:14.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.135 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 16:55:14.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:14.136 | INFO | Uploaded 36GiB to GPU in 2.49504s (14.4286GiB/s) | 20260507 16:55:14.136 | INFO | Downloaded 34GiB to GPU in 2.4387s (13.9418GiB/s) | 20260507 16:55:19.274 | INFO | Analyzing chunk frames = [1094656,1095168) | 20260507 16:55:19.274 | INFO | MemInfo: { | 20260507 16:55:19.274 | INFO | "gpu" : " 17 GB", | 20260507 16:55:19.274 | INFO | "huge" : "", | 20260507 16:55:19.274 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:19.274 | INFO | "malloc" : " 14 GB", | 20260507 16:55:19.274 | INFO | "pinned" : " 29 GB" | 20260507 16:55:19.274 | INFO | } | 20260507 16:55:19.274 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:55:19.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:19.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.274 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 16:55:19.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.274 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:55:19.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:19.275 | INFO | Uploaded 36GiB to GPU in 2.55874s (14.0694GiB/s) | 20260507 16:55:19.275 | INFO | Downloaded 34GiB to GPU in 2.42999s (13.9918GiB/s) | 20260507 16:55:24.414 | INFO | Analyzing chunk frames = [1095168,1095680) | 20260507 16:55:24.414 | INFO | MemInfo: { | 20260507 16:55:24.414 | INFO | "gpu" : " 17 GB", | 20260507 16:55:24.414 | INFO | "huge" : "", | 20260507 16:55:24.414 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:24.414 | INFO | "malloc" : " 14 GB", | 20260507 16:55:24.414 | INFO | "pinned" : " 29 GB" | 20260507 16:55:24.414 | INFO | } | 20260507 16:55:24.414 | INFO | Perf report for the last 5.1396 seconds: | 20260507 16:55:24.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:24.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.414 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 16:55:24.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.414 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:55:24.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:24.415 | INFO | Uploaded 36GiB to GPU in 2.46395s (14.6107GiB/s) | 20260507 16:55:24.415 | INFO | Downloaded 34GiB to GPU in 2.353s (14.4497GiB/s) | 20260507 16:55:29.554 | INFO | Analyzing chunk frames = [1095680,1096192) | 20260507 16:55:29.554 | INFO | MemInfo: { | 20260507 16:55:29.554 | INFO | "gpu" : " 17 GB", | 20260507 16:55:29.554 | INFO | "huge" : "", | 20260507 16:55:29.554 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:29.554 | INFO | "malloc" : " 14 GB", | 20260507 16:55:29.554 | INFO | "pinned" : " 29 GB" | 20260507 16:55:29.554 | INFO | } | 20260507 16:55:29.554 | INFO | Perf report for the last 5.14006 seconds: | 20260507 16:55:29.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:29.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.554 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 16:55:29.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.554 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 16:55:29.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:29.555 | INFO | Uploaded 36GiB to GPU in 2.39292s (15.0444GiB/s) | 20260507 16:55:29.555 | INFO | Downloaded 34GiB to GPU in 2.45138s (13.8697GiB/s) | 20260507 16:55:34.692 | INFO | Analyzing chunk frames = [1096192,1096704) | 20260507 16:55:34.692 | INFO | MemInfo: { | 20260507 16:55:34.692 | INFO | "gpu" : " 17 GB", | 20260507 16:55:34.692 | INFO | "huge" : "", | 20260507 16:55:34.692 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:34.692 | INFO | "malloc" : " 14 GB", | 20260507 16:55:34.692 | INFO | "pinned" : " 29 GB" | 20260507 16:55:34.692 | INFO | } | 20260507 16:55:34.692 | INFO | Perf report for the last 5.13824 seconds: | 20260507 16:55:34.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:34.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.692 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:55:34.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.692 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 16:55:34.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:34.693 | INFO | Uploaded 36GiB to GPU in 2.42384s (14.8525GiB/s) | 20260507 16:55:34.693 | INFO | Downloaded 34GiB to GPU in 2.47148s (13.7569GiB/s) | 20260507 16:55:39.832 | INFO | Analyzing chunk frames = [1096704,1097216) | 20260507 16:55:39.832 | INFO | MemInfo: { | 20260507 16:55:39.832 | INFO | "gpu" : " 17 GB", | 20260507 16:55:39.832 | INFO | "huge" : "", | 20260507 16:55:39.832 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:39.832 | INFO | "malloc" : " 14 GB", | 20260507 16:55:39.832 | INFO | "pinned" : " 29 GB" | 20260507 16:55:39.832 | INFO | } | 20260507 16:55:39.832 | INFO | Perf report for the last 5.13949 seconds: | 20260507 16:55:39.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:39.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 16:55:39.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:55:39.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:39.832 | INFO | Uploaded 36GiB to GPU in 2.45406s (14.6696GiB/s) | 20260507 16:55:39.832 | INFO | Downloaded 34GiB to GPU in 2.41982s (14.0506GiB/s) | 20260507 16:55:44.971 | INFO | Analyzing chunk frames = [1097216,1097728) | 20260507 16:55:44.971 | INFO | MemInfo: { | 20260507 16:55:44.971 | INFO | "gpu" : " 17 GB", | 20260507 16:55:44.971 | INFO | "huge" : "", | 20260507 16:55:44.971 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:44.971 | INFO | "malloc" : " 14 GB", | 20260507 16:55:44.971 | INFO | "pinned" : " 29 GB" | 20260507 16:55:44.971 | INFO | } | 20260507 16:55:44.971 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:55:44.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:44.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.971 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:55:44.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.971 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:55:44.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:44.972 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:55:44.972 | INFO | Total mem usage: 16.4 GiB | 20260507 16:55:44.972 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:55:44.972 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.1%) | 20260507 16:55:44.972 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:55:44.972 | INFO | High Water Breakdown: | 20260507 16:55:44.972 | INFO | 59.5% : 8.86 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:55:44.972 | INFO | 40.4% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:55:44.972 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:55:44.972 | INFO | ------------------------------------------ | 20260507 16:55:44.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:55:44.972 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:55:44.972 | INFO | Total mem usage: 26.5 GiB | 20260507 16:55:44.972 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:55:44.972 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:55:44.972 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:55:44.972 | INFO | High Water Breakdown: | 20260507 16:55:44.972 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:55:44.972 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:55:44.972 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:55:44.972 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:55:44.972 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:55:44.972 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:55:44.972 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:55:44.972 | INFO | Uploaded 36GiB to GPU in 2.48859s (14.466GiB/s) | 20260507 16:55:44.972 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:55:44.972 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:55:44.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:55:44.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:55:44.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:55:44.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:55:44.972 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:55:44.972 | INFO | ------------------------------------------ | 20260507 16:55:44.972 | INFO | Downloaded 34GiB to GPU in 2.50256s (13.5861GiB/s) | 20260507 16:55:44.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:55:44.972 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:55:44.972 | INFO | Total mem usage: 27.8 GiB | 20260507 16:55:44.972 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:55:44.972 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:55:44.972 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:55:44.972 | INFO | High Water Breakdown: | 20260507 16:55:44.972 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:55:44.972 | INFO | ------------------------------------------ | 20260507 16:55:44.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:55:44.972 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:55:44.972 | INFO | Total mem usage: 15.7 GiB | 20260507 16:55:44.972 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:55:44.972 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:55:44.972 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 16:55:44.972 | INFO | High Water Breakdown: | 20260507 16:55:44.972 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:55:44.972 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:55:44.972 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:55:44.972 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:55:44.972 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:55:44.972 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:55:44.972 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:55:44.972 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:55:44.972 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:55:44.972 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:55:44.972 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:55:44.972 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:55:44.972 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:55:44.973 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:55:44.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:55:44.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:55:44.973 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:55:44.973 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:55:44.973 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:55:44.973 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:55:44.973 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:55:44.973 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:55:44.973 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:55:44.973 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:55:44.973 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:55:44.973 | INFO | ------------------------------------------ | 20260507 16:55:44.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:55:44.973 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:55:44.973 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:55:44.973 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:55:44.973 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.63e+01, 1.44e+01 / 2.81e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.12e-02 / 6.69e-03 | 20260507 16:55:44.973 | INFO | PerfReport: 12.04%, Baseline -- 4.04e+00 / 4.51e+00, 6.20e-01 / 2.41e-01 | 20260507 16:55:44.973 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.01e-01 | 20260507 16:55:44.973 | INFO | PerfReport: 15.72%, DME -- 4.22e+01 / 4.52e+01, 4.60e+00 / 2.72e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.88e+01 / 2.03e+01, 1.85e+00 / 2.10e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.53e+00 / 2.92e+00, 4.36e-01 / 2.44e-01 | 20260507 16:55:44.973 | INFO | PerfReport: 3.81%, Metrics -- 1.28e+00 / 1.45e+00, 3.27e-01 / 1.99e-01 | 20260507 16:55:44.973 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.05e-01 / 4.31e-01, 6.85e-02 / 4.61e-02 | 20260507 16:55:44.973 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:55:44.973 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:55:44.973 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:55:44.973 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 23.99%, Upload -- 2.58e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 23.92%, Download -- 2.57e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 31.25%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 1.82%, LockUpload -- 1.95e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 2.20%, LockDownload -- 2.36e+00 / 0.00e+00, 3.73e+00 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 16.81%, LockCompute -- 1.81e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 16:55:44.973 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260507 16:55:45.068 | INFO | Starting a round of Baz IO | 20260507 16:55:48.592 | INFO | Populating RTMetrics report | 20260507 16:55:48.592 | INFO | RTMetrics report populated | 20260507 16:55:48.592 | INFO | Opening tmp RTMetrics json file | 20260507 16:55:48.622 | INFO | Serializing RTMetrics to json | 20260507 16:55:48.650 | INFO | Serialization done, swapping tmp file now... | 20260507 16:55:48.666 | INFO | RTMetrics done | 20260507 16:55:48.692 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.623 seconds | 20260507 16:55:48.692 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.16% / 0.02% / 50.81% | 20260507 16:55:48.692 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0325% / 5.96046e-05% / 2.78155e-05% | 20260507 16:55:48.692 | INFO | Baz IO overall: | 20260507 16:55:48.692 | INFO | BytesWritten : 477192150206 | 20260507 16:55:48.692 | INFO | NumEvents : 231060340585 | 20260507 16:55:50.110 | INFO | Analyzing chunk frames = [1097728,1098240) | 20260507 16:55:50.110 | INFO | MemInfo: { | 20260507 16:55:50.110 | INFO | "gpu" : " 17 GB", | 20260507 16:55:50.110 | INFO | "huge" : "", | 20260507 16:55:50.110 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:50.110 | INFO | "malloc" : " 14 GB", | 20260507 16:55:50.110 | INFO | "pinned" : " 29 GB" | 20260507 16:55:50.110 | INFO | } | 20260507 16:55:50.110 | INFO | Perf report for the last 5.1389 seconds: | 20260507 16:55:50.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:50.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:50.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:50.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:50.110 | INFO | ANALYSIS : 69.7%, 2.9, 107.2ms, 0 | 20260507 16:55:50.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:50.110 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 16:55:50.110 | INFO | PRE_HQ : 67.3%, 1.8, 63.1ms, 0 | 20260507 16:55:50.110 | INFO | BAZWRITER : 70.1%, 3.9, 140.2ms, 0 | 20260507 16:55:50.110 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 16:55:50.110 | INFO | Uploaded 36GiB to GPU in 2.55208s (14.1062GiB/s) | 20260507 16:55:50.110 | INFO | Downloaded 38GiB to GPU in 2.69238s (14.1139GiB/s) | 20260507 16:55:55.249 | INFO | Analyzing chunk frames = [1098240,1098752) | 20260507 16:55:55.249 | INFO | MemInfo: { | 20260507 16:55:55.249 | INFO | "gpu" : " 17 GB", | 20260507 16:55:55.249 | INFO | "huge" : "", | 20260507 16:55:55.249 | INFO | "huge_shared" : " 29 GB", | 20260507 16:55:55.249 | INFO | "malloc" : " 14 GB", | 20260507 16:55:55.249 | INFO | "pinned" : " 29 GB" | 20260507 16:55:55.249 | INFO | } | 20260507 16:55:55.249 | INFO | Perf report for the last 5.13932 seconds: | 20260507 16:55:55.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:55:55.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.250 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:55:55.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.250 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 16:55:55.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:55:55.250 | INFO | Uploaded 36GiB to GPU in 2.53292s (14.2129GiB/s) | 20260507 16:55:55.250 | INFO | Downloaded 34GiB to GPU in 2.42573s (14.0164GiB/s) | 20260507 16:56:00.388 | INFO | Analyzing chunk frames = [1098752,1099264) | 20260507 16:56:00.388 | INFO | MemInfo: { | 20260507 16:56:00.388 | INFO | "gpu" : " 17 GB", | 20260507 16:56:00.388 | INFO | "huge" : "", | 20260507 16:56:00.388 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:00.388 | INFO | "malloc" : " 14 GB", | 20260507 16:56:00.388 | INFO | "pinned" : " 29 GB" | 20260507 16:56:00.388 | INFO | } | 20260507 16:56:00.389 | INFO | Perf report for the last 5.1391 seconds: | 20260507 16:56:00.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:00.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 16:56:00.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:56:00.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:00.389 | INFO | Uploaded 36GiB to GPU in 2.54687s (14.135GiB/s) | 20260507 16:56:00.389 | INFO | Downloaded 34GiB to GPU in 2.41698s (14.0671GiB/s) | 20260507 16:56:05.528 | INFO | Analyzing chunk frames = [1099264,1099776) | 20260507 16:56:05.528 | INFO | MemInfo: { | 20260507 16:56:05.528 | INFO | "gpu" : " 17 GB", | 20260507 16:56:05.528 | INFO | "huge" : "", | 20260507 16:56:05.528 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:05.528 | INFO | "malloc" : " 14 GB", | 20260507 16:56:05.528 | INFO | "pinned" : " 29 GB" | 20260507 16:56:05.528 | INFO | } | 20260507 16:56:05.528 | INFO | Perf report for the last 5.13937 seconds: | 20260507 16:56:05.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:05.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260507 16:56:05.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 16:56:05.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:05.528 | INFO | Uploaded 36GiB to GPU in 2.5045s (14.3741GiB/s) | 20260507 16:56:05.528 | INFO | Downloaded 34GiB to GPU in 2.34391s (14.5057GiB/s) | 20260507 16:56:10.667 | INFO | Analyzing chunk frames = [1099776,1100288) | 20260507 16:56:10.667 | INFO | MemInfo: { | 20260507 16:56:10.667 | INFO | "gpu" : " 17 GB", | 20260507 16:56:10.667 | INFO | "huge" : "", | 20260507 16:56:10.667 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:10.667 | INFO | "malloc" : " 14 GB", | 20260507 16:56:10.667 | INFO | "pinned" : " 29 GB" | 20260507 16:56:10.667 | INFO | } | 20260507 16:56:10.667 | INFO | Perf report for the last 5.13929 seconds: | 20260507 16:56:10.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:10.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 16:56:10.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:56:10.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:10.668 | INFO | Uploaded 36GiB to GPU in 2.37243s (15.1743GiB/s) | 20260507 16:56:10.668 | INFO | Downloaded 34GiB to GPU in 2.46582s (13.7885GiB/s) | 20260507 16:56:15.806 | INFO | Analyzing chunk frames = [1100288,1100800) | 20260507 16:56:15.806 | INFO | MemInfo: { | 20260507 16:56:15.806 | INFO | "gpu" : " 17 GB", | 20260507 16:56:15.806 | INFO | "huge" : "", | 20260507 16:56:15.806 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:15.807 | INFO | "malloc" : " 14 GB", | 20260507 16:56:15.807 | INFO | "pinned" : " 29 GB" | 20260507 16:56:15.807 | INFO | } | 20260507 16:56:15.807 | INFO | Perf report for the last 5.13934 seconds: | 20260507 16:56:15.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:15.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:56:15.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 16:56:15.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:15.807 | INFO | Uploaded 36GiB to GPU in 2.3864s (15.0855GiB/s) | 20260507 16:56:15.807 | INFO | Downloaded 34GiB to GPU in 2.44416s (13.9107GiB/s) | 20260507 16:56:20.946 | INFO | Analyzing chunk frames = [1100800,1101312) | 20260507 16:56:20.946 | INFO | MemInfo: { | 20260507 16:56:20.946 | INFO | "gpu" : " 17 GB", | 20260507 16:56:20.946 | INFO | "huge" : "", | 20260507 16:56:20.946 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:20.946 | INFO | "malloc" : " 14 GB", | 20260507 16:56:20.946 | INFO | "pinned" : " 29 GB" | 20260507 16:56:20.946 | INFO | } | 20260507 16:56:20.946 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:56:20.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:20.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 16:56:20.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:56:20.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:20.946 | INFO | Uploaded 36GiB to GPU in 2.46987s (14.5757GiB/s) | 20260507 16:56:20.946 | INFO | Downloaded 34GiB to GPU in 2.40555s (14.134GiB/s) | 20260507 16:56:26.085 | INFO | Analyzing chunk frames = [1101312,1101824) | 20260507 16:56:26.085 | INFO | MemInfo: { | 20260507 16:56:26.085 | INFO | "gpu" : " 17 GB", | 20260507 16:56:26.085 | INFO | "huge" : "", | 20260507 16:56:26.085 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:26.085 | INFO | "malloc" : " 14 GB", | 20260507 16:56:26.085 | INFO | "pinned" : " 29 GB" | 20260507 16:56:26.085 | INFO | } | 20260507 16:56:26.086 | INFO | Perf report for the last 5.13956 seconds: | 20260507 16:56:26.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:26.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 16:56:26.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:56:26.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:26.086 | INFO | Uploaded 36GiB to GPU in 2.54125s (14.1663GiB/s) | 20260507 16:56:26.086 | INFO | Downloaded 34GiB to GPU in 2.49296s (13.6384GiB/s) | 20260507 16:56:26.183 | INFO | Starting a round of Baz IO | 20260507 16:56:29.708 | INFO | Populating RTMetrics report | 20260507 16:56:29.708 | INFO | RTMetrics report populated | 20260507 16:56:29.708 | INFO | Opening tmp RTMetrics json file | 20260507 16:56:29.709 | INFO | Serializing RTMetrics to json | 20260507 16:56:29.710 | INFO | Serialization done, swapping tmp file now... | 20260507 16:56:29.711 | INFO | RTMetrics done | 20260507 16:56:29.817 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.633 seconds | 20260507 16:56:29.817 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.14% / 0.02% / 50.84% | 20260507 16:56:29.817 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0158% / 2.78155e-05% / 2.78155e-05% | 20260507 16:56:29.817 | INFO | Baz IO overall: | 20260507 16:56:29.817 | INFO | BytesWritten : 479115564706 | 20260507 16:56:29.817 | INFO | NumEvents : 231994942218 | 20260507 16:56:31.224 | INFO | Analyzing chunk frames = [1101824,1102336) | 20260507 16:56:31.224 | INFO | MemInfo: { | 20260507 16:56:31.224 | INFO | "gpu" : " 17 GB", | 20260507 16:56:31.224 | INFO | "huge" : "", | 20260507 16:56:31.224 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:31.224 | INFO | "malloc" : " 14 GB", | 20260507 16:56:31.224 | INFO | "pinned" : " 29 GB" | 20260507 16:56:31.224 | INFO | } | 20260507 16:56:31.224 | INFO | Perf report for the last 5.13889 seconds: | 20260507 16:56:31.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:31.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:31.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:31.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:31.225 | INFO | ANALYSIS : 69.8%, 3.0, 108.7ms, 0 | 20260507 16:56:31.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:31.225 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260507 16:56:31.225 | INFO | PRE_HQ : 66.8%, 1.8, 60.8ms, 0 | 20260507 16:56:31.225 | INFO | BAZWRITER : 70.3%, 4.0, 143.2ms, 0 | 20260507 16:56:31.225 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 16:56:31.225 | INFO | Uploaded 36GiB to GPU in 2.56181s (14.0526GiB/s) | 20260507 16:56:31.225 | INFO | Downloaded 38GiB to GPU in 2.71892s (13.9761GiB/s) | 20260507 16:56:36.364 | INFO | Analyzing chunk frames = [1102336,1102848) | 20260507 16:56:36.364 | INFO | MemInfo: { | 20260507 16:56:36.364 | INFO | "gpu" : " 17 GB", | 20260507 16:56:36.364 | INFO | "huge" : "", | 20260507 16:56:36.364 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:36.364 | INFO | "malloc" : " 14 GB", | 20260507 16:56:36.364 | INFO | "pinned" : " 29 GB" | 20260507 16:56:36.364 | INFO | } | 20260507 16:56:36.364 | INFO | Perf report for the last 5.13936 seconds: | 20260507 16:56:36.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:36.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:56:36.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 16:56:36.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:36.364 | INFO | Uploaded 36GiB to GPU in 2.53904s (14.1786GiB/s) | 20260507 16:56:36.364 | INFO | Downloaded 34GiB to GPU in 2.42618s (14.0138GiB/s) | 20260507 16:56:41.503 | INFO | Analyzing chunk frames = [1102848,1103360) | 20260507 16:56:41.503 | INFO | MemInfo: { | 20260507 16:56:41.503 | INFO | "gpu" : " 17 GB", | 20260507 16:56:41.503 | INFO | "huge" : "", | 20260507 16:56:41.503 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:41.503 | INFO | "malloc" : " 14 GB", | 20260507 16:56:41.503 | INFO | "pinned" : " 29 GB" | 20260507 16:56:41.503 | INFO | } | 20260507 16:56:41.503 | INFO | Perf report for the last 5.13942 seconds: | 20260507 16:56:41.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:41.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.503 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 16:56:41.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.503 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:56:41.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:41.504 | INFO | Uploaded 36GiB to GPU in 2.48817s (14.4685GiB/s) | 20260507 16:56:41.504 | INFO | Downloaded 34GiB to GPU in 2.33525s (14.5595GiB/s) | 20260507 16:56:46.642 | INFO | Analyzing chunk frames = [1103360,1103872) | 20260507 16:56:46.642 | INFO | MemInfo: { | 20260507 16:56:46.642 | INFO | "gpu" : " 17 GB", | 20260507 16:56:46.642 | INFO | "huge" : "", | 20260507 16:56:46.642 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:46.642 | INFO | "malloc" : " 14 GB", | 20260507 16:56:46.642 | INFO | "pinned" : " 29 GB" | 20260507 16:56:46.642 | INFO | } | 20260507 16:56:46.643 | INFO | Perf report for the last 5.13923 seconds: | 20260507 16:56:46.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:46.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 16:56:46.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:56:46.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:46.643 | INFO | Uploaded 36GiB to GPU in 2.49919s (14.4047GiB/s) | 20260507 16:56:46.643 | INFO | Downloaded 34GiB to GPU in 2.35128s (14.4602GiB/s) | 20260507 16:56:51.782 | INFO | Analyzing chunk frames = [1103872,1104384) | 20260507 16:56:51.782 | INFO | MemInfo: { | 20260507 16:56:51.782 | INFO | "gpu" : " 17 GB", | 20260507 16:56:51.782 | INFO | "huge" : "", | 20260507 16:56:51.782 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:51.782 | INFO | "malloc" : " 14 GB", | 20260507 16:56:51.782 | INFO | "pinned" : " 29 GB" | 20260507 16:56:51.782 | INFO | } | 20260507 16:56:51.782 | INFO | Perf report for the last 5.13975 seconds: | 20260507 16:56:51.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:51.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.782 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 16:56:51.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.782 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 16:56:51.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:51.783 | INFO | Uploaded 36GiB to GPU in 2.3778s (15.1401GiB/s) | 20260507 16:56:51.783 | INFO | Downloaded 34GiB to GPU in 2.47218s (13.753GiB/s) | 20260507 16:56:56.921 | INFO | Analyzing chunk frames = [1104384,1104896) | 20260507 16:56:56.921 | INFO | MemInfo: { | 20260507 16:56:56.921 | INFO | "gpu" : " 17 GB", | 20260507 16:56:56.921 | INFO | "huge" : "", | 20260507 16:56:56.921 | INFO | "huge_shared" : " 29 GB", | 20260507 16:56:56.921 | INFO | "malloc" : " 14 GB", | 20260507 16:56:56.921 | INFO | "pinned" : " 29 GB" | 20260507 16:56:56.921 | INFO | } | 20260507 16:56:56.921 | INFO | Perf report for the last 5.13905 seconds: | 20260507 16:56:56.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:56:56.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.921 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:56:56.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.922 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 16:56:56.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:56:56.922 | INFO | Uploaded 36GiB to GPU in 2.37917s (15.1313GiB/s) | 20260507 16:56:56.922 | INFO | Downloaded 34GiB to GPU in 2.4351s (13.9625GiB/s) | 20260507 16:57:02.060 | INFO | Analyzing chunk frames = [1104896,1105408) | 20260507 16:57:02.060 | INFO | MemInfo: { | 20260507 16:57:02.060 | INFO | "gpu" : " 17 GB", | 20260507 16:57:02.060 | INFO | "huge" : "", | 20260507 16:57:02.060 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:02.060 | INFO | "malloc" : " 14 GB", | 20260507 16:57:02.060 | INFO | "pinned" : " 29 GB" | 20260507 16:57:02.061 | INFO | } | 20260507 16:57:02.061 | INFO | Perf report for the last 5.13916 seconds: | 20260507 16:57:02.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:02.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 16:57:02.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:57:02.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:02.061 | INFO | Uploaded 36GiB to GPU in 2.48707s (14.4748GiB/s) | 20260507 16:57:02.061 | INFO | Downloaded 34GiB to GPU in 2.40822s (14.1183GiB/s) | 20260507 16:57:07.200 | INFO | Analyzing chunk frames = [1105408,1105920) | 20260507 16:57:07.200 | INFO | MemInfo: { | 20260507 16:57:07.200 | INFO | "gpu" : " 17 GB", | 20260507 16:57:07.200 | INFO | "huge" : "", | 20260507 16:57:07.200 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:07.200 | INFO | "malloc" : " 14 GB", | 20260507 16:57:07.200 | INFO | "pinned" : " 29 GB" | 20260507 16:57:07.200 | INFO | } | 20260507 16:57:07.200 | INFO | Perf report for the last 5.13947 seconds: | 20260507 16:57:07.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:07.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 16:57:07.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:57:07.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:07.200 | INFO | Uploaded 36GiB to GPU in 2.45026s (14.6923GiB/s) | 20260507 16:57:07.200 | INFO | Downloaded 34GiB to GPU in 2.47971s (13.7113GiB/s) | 20260507 16:57:07.201 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:57:07.201 | INFO | Total mem usage: 16.4 GiB | 20260507 16:57:07.201 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:57:07.201 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.32%) | 20260507 16:57:07.201 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:57:07.201 | INFO | High Water Breakdown: | 20260507 16:57:07.201 | INFO | 59.6% : 8.94 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:57:07.201 | INFO | 40.3% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:57:07.201 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:57:07.201 | INFO | ------------------------------------------ | 20260507 16:57:07.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:57:07.201 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:57:07.201 | INFO | Total mem usage: 26.5 GiB | 20260507 16:57:07.201 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:57:07.201 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 16:57:07.201 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 16:57:07.201 | INFO | High Water Breakdown: | 20260507 16:57:07.201 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:57:07.201 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:57:07.201 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:57:07.201 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:57:07.201 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:57:07.201 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:57:07.201 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:57:07.201 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:57:07.201 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:57:07.201 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:57:07.201 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:57:07.201 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:57:07.201 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:57:07.201 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:57:07.201 | INFO | ------------------------------------------ | 20260507 16:57:07.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:57:07.201 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:57:07.201 | INFO | Total mem usage: 27.8 GiB | 20260507 16:57:07.201 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:57:07.201 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:57:07.201 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:57:07.201 | INFO | High Water Breakdown: | 20260507 16:57:07.201 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:57:07.201 | INFO | ------------------------------------------ | 20260507 16:57:07.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:57:07.201 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:57:07.201 | INFO | Total mem usage: 15.7 GiB | 20260507 16:57:07.201 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:57:07.201 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:57:07.201 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:57:07.201 | INFO | High Water Breakdown: | 20260507 16:57:07.201 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:57:07.201 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:57:07.201 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:57:07.201 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:57:07.201 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:57:07.201 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:57:07.201 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:57:07.201 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:57:07.201 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:57:07.201 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:57:07.201 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:57:07.201 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:57:07.201 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:57:07.201 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:57:07.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:57:07.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:57:07.201 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:57:07.201 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:57:07.201 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:57:07.202 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:57:07.202 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:57:07.202 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:57:07.202 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:57:07.202 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:57:07.202 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:57:07.202 | INFO | ------------------------------------------ | 20260507 16:57:07.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:57:07.202 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:57:07.202 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:57:07.202 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:57:07.202 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.67e+01, 1.43e+01 / 2.90e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.11e-02 / 1.08e-02 | 20260507 16:57:07.202 | INFO | PerfReport: 12.08%, Baseline -- 4.06e+00 / 4.50e+00, 6.23e-01 / 2.13e-01 | 20260507 16:57:07.202 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.14e-01 / 9.89e-02 | 20260507 16:57:07.202 | INFO | PerfReport: 15.73%, DME -- 4.23e+01 / 4.54e+01, 4.68e+00 / 3.17e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 55.93%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.88e+00 / 2.21e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.53e+00 / 2.92e+00, 4.36e-01 / 2.35e-01 | 20260507 16:57:07.202 | INFO | PerfReport: 3.80%, Metrics -- 1.28e+00 / 1.47e+00, 3.26e-01 / 1.93e-01 | 20260507 16:57:07.202 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.06e-01 / 4.33e-01, 6.80e-02 / 4.12e-02 | 20260507 16:57:07.202 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:57:07.202 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:57:07.202 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:57:07.202 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 24.03%, Upload -- 2.58e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 23.80%, Download -- 2.56e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 31.27%, Compute -- 3.36e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 1.97%, LockUpload -- 2.12e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 2.05%, LockDownload -- 2.20e+00 / 0.00e+00, 3.57e+00 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 16.86%, LockCompute -- 1.81e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:57:07.202 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260507 16:57:07.296 | INFO | Starting a round of Baz IO | 20260507 16:57:10.801 | INFO | Populating RTMetrics report | 20260507 16:57:10.801 | INFO | RTMetrics report populated | 20260507 16:57:10.801 | INFO | Opening tmp RTMetrics json file | 20260507 16:57:10.824 | INFO | Serializing RTMetrics to json | 20260507 16:57:10.826 | INFO | Serialization done, swapping tmp file now... | 20260507 16:57:10.827 | INFO | RTMetrics done | 20260507 16:57:10.910 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.613 seconds | 20260507 16:57:10.910 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.11% / 0.02% / 50.86% | 20260507 16:57:10.910 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.01572% / 3.57628e-05% / 1.19209e-05% | 20260507 16:57:10.910 | INFO | Baz IO overall: | 20260507 16:57:10.910 | INFO | BytesWritten : 481041213356 | 20260507 16:57:10.910 | INFO | NumEvents : 232930683531 | 20260507 16:57:12.339 | INFO | Analyzing chunk frames = [1105920,1106432) | 20260507 16:57:12.339 | INFO | MemInfo: { | 20260507 16:57:12.339 | INFO | "gpu" : " 17 GB", | 20260507 16:57:12.339 | INFO | "huge" : "", | 20260507 16:57:12.339 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:12.339 | INFO | "malloc" : " 14 GB", | 20260507 16:57:12.339 | INFO | "pinned" : " 29 GB" | 20260507 16:57:12.339 | INFO | } | 20260507 16:57:12.339 | INFO | Perf report for the last 5.13907 seconds: | 20260507 16:57:12.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:12.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:12.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:12.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:12.339 | INFO | ANALYSIS : 69.6%, 3.0, 108.1ms, 0 | 20260507 16:57:12.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:12.339 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 16:57:12.339 | INFO | PRE_HQ : 66.4%, 1.7, 60.0ms, 0 | 20260507 16:57:12.339 | INFO | BAZWRITER : 69.9%, 4.0, 143.3ms, 0 | 20260507 16:57:12.339 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 16:57:12.340 | INFO | Uploaded 36GiB to GPU in 2.5483s (14.1271GiB/s) | 20260507 16:57:12.340 | INFO | Downloaded 38GiB to GPU in 2.69846s (14.0821GiB/s) | 20260507 16:57:17.479 | INFO | Analyzing chunk frames = [1106432,1106944) | 20260507 16:57:17.479 | INFO | MemInfo: { | 20260507 16:57:17.479 | INFO | "gpu" : " 17 GB", | 20260507 16:57:17.479 | INFO | "huge" : "", | 20260507 16:57:17.479 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:17.479 | INFO | "malloc" : " 14 GB", | 20260507 16:57:17.479 | INFO | "pinned" : " 29 GB" | 20260507 16:57:17.479 | INFO | } | 20260507 16:57:17.479 | INFO | Perf report for the last 5.1398 seconds: | 20260507 16:57:17.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:17.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.479 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 16:57:17.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.479 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 16:57:17.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:17.480 | INFO | Uploaded 36GiB to GPU in 2.51851s (14.2942GiB/s) | 20260507 16:57:17.480 | INFO | Downloaded 34GiB to GPU in 2.42308s (14.0318GiB/s) | 20260507 16:57:22.618 | INFO | Analyzing chunk frames = [1106944,1107456) | 20260507 16:57:22.618 | INFO | MemInfo: { | 20260507 16:57:22.618 | INFO | "gpu" : " 17 GB", | 20260507 16:57:22.618 | INFO | "huge" : "", | 20260507 16:57:22.618 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:22.618 | INFO | "malloc" : " 14 GB", | 20260507 16:57:22.618 | INFO | "pinned" : " 29 GB" | 20260507 16:57:22.618 | INFO | } | 20260507 16:57:22.618 | INFO | Perf report for the last 5.13938 seconds: | 20260507 16:57:22.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:22.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.618 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260507 16:57:22.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.618 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 16:57:22.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:22.619 | INFO | Uploaded 36GiB to GPU in 2.54729s (14.1327GiB/s) | 20260507 16:57:22.619 | INFO | Downloaded 34GiB to GPU in 2.42897s (13.9977GiB/s) | 20260507 16:57:27.757 | INFO | Analyzing chunk frames = [1107456,1107968) | 20260507 16:57:27.757 | INFO | MemInfo: { | 20260507 16:57:27.757 | INFO | "gpu" : " 17 GB", | 20260507 16:57:27.757 | INFO | "huge" : "", | 20260507 16:57:27.757 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:27.757 | INFO | "malloc" : " 14 GB", | 20260507 16:57:27.757 | INFO | "pinned" : " 29 GB" | 20260507 16:57:27.757 | INFO | } | 20260507 16:57:27.757 | INFO | Perf report for the last 5.13908 seconds: | 20260507 16:57:27.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:27.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.757 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:57:27.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.758 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:57:27.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:27.758 | INFO | Uploaded 36GiB to GPU in 2.4725s (14.5602GiB/s) | 20260507 16:57:27.758 | INFO | Downloaded 34GiB to GPU in 2.34154s (14.5203GiB/s) | 20260507 16:57:32.897 | INFO | Analyzing chunk frames = [1107968,1108480) | 20260507 16:57:32.897 | INFO | MemInfo: { | 20260507 16:57:32.897 | INFO | "gpu" : " 17 GB", | 20260507 16:57:32.897 | INFO | "huge" : "", | 20260507 16:57:32.897 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:32.897 | INFO | "malloc" : " 14 GB", | 20260507 16:57:32.897 | INFO | "pinned" : " 29 GB" | 20260507 16:57:32.897 | INFO | } | 20260507 16:57:32.897 | INFO | Perf report for the last 5.13953 seconds: | 20260507 16:57:32.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:32.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.897 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 16:57:32.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.897 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 16:57:32.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:32.898 | INFO | Uploaded 36GiB to GPU in 2.35012s (15.3184GiB/s) | 20260507 16:57:32.898 | INFO | Downloaded 34GiB to GPU in 2.46397s (13.7989GiB/s) | 20260507 16:57:38.036 | INFO | Analyzing chunk frames = [1108480,1108992) | 20260507 16:57:38.036 | INFO | MemInfo: { | 20260507 16:57:38.036 | INFO | "gpu" : " 17 GB", | 20260507 16:57:38.036 | INFO | "huge" : "", | 20260507 16:57:38.036 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:38.036 | INFO | "malloc" : " 14 GB", | 20260507 16:57:38.036 | INFO | "pinned" : " 29 GB" | 20260507 16:57:38.036 | INFO | } | 20260507 16:57:38.036 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:57:38.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:38.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.036 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:57:38.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.036 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 16:57:38.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:38.037 | INFO | Uploaded 36GiB to GPU in 2.38105s (15.1194GiB/s) | 20260507 16:57:38.037 | INFO | Downloaded 34GiB to GPU in 2.43731s (13.9498GiB/s) | 20260507 16:57:43.175 | INFO | Analyzing chunk frames = [1108992,1109504) | 20260507 16:57:43.176 | INFO | MemInfo: { | 20260507 16:57:43.176 | INFO | "gpu" : " 17 GB", | 20260507 16:57:43.176 | INFO | "huge" : "", | 20260507 16:57:43.176 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:43.176 | INFO | "malloc" : " 14 GB", | 20260507 16:57:43.176 | INFO | "pinned" : " 29 GB" | 20260507 16:57:43.176 | INFO | } | 20260507 16:57:43.176 | INFO | Perf report for the last 5.13952 seconds: | 20260507 16:57:43.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:43.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 16:57:43.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:57:43.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:43.176 | INFO | Uploaded 36GiB to GPU in 2.47604s (14.5394GiB/s) | 20260507 16:57:43.176 | INFO | Downloaded 34GiB to GPU in 2.40553s (14.1341GiB/s) | 20260507 16:57:48.315 | INFO | Analyzing chunk frames = [1109504,1110016) | 20260507 16:57:48.315 | INFO | MemInfo: { | 20260507 16:57:48.315 | INFO | "gpu" : " 17 GB", | 20260507 16:57:48.315 | INFO | "huge" : "", | 20260507 16:57:48.315 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:48.315 | INFO | "malloc" : " 14 GB", | 20260507 16:57:48.315 | INFO | "pinned" : " 29 GB" | 20260507 16:57:48.315 | INFO | } | 20260507 16:57:48.315 | INFO | Perf report for the last 5.13905 seconds: | 20260507 16:57:48.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:48.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 16:57:48.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:57:48.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:48.315 | INFO | Uploaded 36GiB to GPU in 2.51s (14.3426GiB/s) | 20260507 16:57:48.315 | INFO | Downloaded 34GiB to GPU in 2.48957s (13.657GiB/s) | 20260507 16:57:48.412 | INFO | Starting a round of Baz IO | 20260507 16:57:51.937 | INFO | Populating RTMetrics report | 20260507 16:57:51.938 | INFO | RTMetrics report populated | 20260507 16:57:51.938 | INFO | Opening tmp RTMetrics json file | 20260507 16:57:51.961 | INFO | Serializing RTMetrics to json | 20260507 16:57:51.970 | INFO | Serialization done, swapping tmp file now... | 20260507 16:57:51.970 | INFO | RTMetrics done | 20260507 16:57:52.074 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.662 seconds | 20260507 16:57:52.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.09% / 0.02% / 50.89% | 20260507 16:57:52.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.00972% / 2.78155e-05% / 1.98682e-05% | 20260507 16:57:52.075 | INFO | Baz IO overall: | 20260507 16:57:52.075 | INFO | BytesWritten : 482967612999 | 20260507 16:57:52.075 | INFO | NumEvents : 233867121502 | 20260507 16:57:53.454 | INFO | Analyzing chunk frames = [1110016,1110528) | 20260507 16:57:53.454 | INFO | MemInfo: { | 20260507 16:57:53.454 | INFO | "gpu" : " 17 GB", | 20260507 16:57:53.454 | INFO | "huge" : "", | 20260507 16:57:53.454 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:53.454 | INFO | "malloc" : " 14 GB", | 20260507 16:57:53.454 | INFO | "pinned" : " 29 GB" | 20260507 16:57:53.454 | INFO | } | 20260507 16:57:53.454 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:57:53.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:53.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:53.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:53.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:53.454 | INFO | ANALYSIS : 69.8%, 3.0, 108.1ms, 0 | 20260507 16:57:53.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:53.454 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 16:57:53.454 | INFO | PRE_HQ : 66.5%, 1.8, 60.9ms, 0 | 20260507 16:57:53.454 | INFO | BAZWRITER : 70.8%, 3.7, 133.1ms, 0 | 20260507 16:57:53.454 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260507 16:57:53.455 | INFO | Uploaded 36GiB to GPU in 2.56911s (14.0126GiB/s) | 20260507 16:57:53.455 | INFO | Downloaded 38GiB to GPU in 2.70588s (14.0435GiB/s) | 20260507 16:57:58.593 | INFO | Analyzing chunk frames = [1110528,1111040) | 20260507 16:57:58.593 | INFO | MemInfo: { | 20260507 16:57:58.593 | INFO | "gpu" : " 17 GB", | 20260507 16:57:58.593 | INFO | "huge" : "", | 20260507 16:57:58.593 | INFO | "huge_shared" : " 29 GB", | 20260507 16:57:58.593 | INFO | "malloc" : " 14 GB", | 20260507 16:57:58.594 | INFO | "pinned" : " 29 GB" | 20260507 16:57:58.594 | INFO | } | 20260507 16:57:58.594 | INFO | Perf report for the last 5.13926 seconds: | 20260507 16:57:58.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:57:58.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 16:57:58.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 16:57:58.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:57:58.594 | INFO | Uploaded 36GiB to GPU in 2.52787s (14.2412GiB/s) | 20260507 16:57:58.594 | INFO | Downloaded 34GiB to GPU in 2.4257s (14.0166GiB/s) | 20260507 16:58:03.733 | INFO | Analyzing chunk frames = [1111040,1111552) | 20260507 16:58:03.733 | INFO | MemInfo: { | 20260507 16:58:03.733 | INFO | "gpu" : " 17 GB", | 20260507 16:58:03.733 | INFO | "huge" : "", | 20260507 16:58:03.733 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:03.733 | INFO | "malloc" : " 14 GB", | 20260507 16:58:03.733 | INFO | "pinned" : " 29 GB" | 20260507 16:58:03.733 | INFO | } | 20260507 16:58:03.733 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:58:03.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:03.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.733 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 16:58:03.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.733 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:58:03.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:03.734 | INFO | Uploaded 36GiB to GPU in 2.47518s (14.5444GiB/s) | 20260507 16:58:03.734 | INFO | Downloaded 34GiB to GPU in 2.33829s (14.5405GiB/s) | 20260507 16:58:08.872 | INFO | Analyzing chunk frames = [1111552,1112064) | 20260507 16:58:08.872 | INFO | MemInfo: { | 20260507 16:58:08.872 | INFO | "gpu" : " 17 GB", | 20260507 16:58:08.872 | INFO | "huge" : "", | 20260507 16:58:08.872 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:08.872 | INFO | "malloc" : " 14 GB", | 20260507 16:58:08.872 | INFO | "pinned" : " 29 GB" | 20260507 16:58:08.872 | INFO | } | 20260507 16:58:08.873 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:58:08.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:08.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260507 16:58:08.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:58:08.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:08.873 | INFO | Uploaded 36GiB to GPU in 2.48892s (14.4641GiB/s) | 20260507 16:58:08.873 | INFO | Downloaded 34GiB to GPU in 2.35139s (14.4595GiB/s) | 20260507 16:58:14.012 | INFO | Analyzing chunk frames = [1112064,1112576) | 20260507 16:58:14.012 | INFO | MemInfo: { | 20260507 16:58:14.012 | INFO | "gpu" : " 17 GB", | 20260507 16:58:14.012 | INFO | "huge" : "", | 20260507 16:58:14.012 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:14.012 | INFO | "malloc" : " 14 GB", | 20260507 16:58:14.012 | INFO | "pinned" : " 29 GB" | 20260507 16:58:14.012 | INFO | } | 20260507 16:58:14.012 | INFO | Perf report for the last 5.13971 seconds: | 20260507 16:58:14.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:14.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.012 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 16:58:14.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.012 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 16:58:14.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:14.013 | INFO | Uploaded 36GiB to GPU in 2.36788s (15.2035GiB/s) | 20260507 16:58:14.013 | INFO | Downloaded 34GiB to GPU in 2.45453s (13.852GiB/s) | 20260507 16:58:19.151 | INFO | Analyzing chunk frames = [1112576,1113088) | 20260507 16:58:19.151 | INFO | MemInfo: { | 20260507 16:58:19.151 | INFO | "gpu" : " 17 GB", | 20260507 16:58:19.151 | INFO | "huge" : "", | 20260507 16:58:19.151 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:19.151 | INFO | "malloc" : " 14 GB", | 20260507 16:58:19.151 | INFO | "pinned" : " 29 GB" | 20260507 16:58:19.151 | INFO | } | 20260507 16:58:19.151 | INFO | Perf report for the last 5.13901 seconds: | 20260507 16:58:19.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:19.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.151 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:58:19.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.151 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260507 16:58:19.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:19.152 | INFO | Uploaded 36GiB to GPU in 2.34321s (15.3636GiB/s) | 20260507 16:58:19.152 | INFO | Downloaded 34GiB to GPU in 2.47951s (13.7124GiB/s) | 20260507 16:58:24.290 | INFO | Analyzing chunk frames = [1113088,1113600) | 20260507 16:58:24.290 | INFO | MemInfo: { | 20260507 16:58:24.290 | INFO | "gpu" : " 17 GB", | 20260507 16:58:24.290 | INFO | "huge" : "", | 20260507 16:58:24.290 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:24.290 | INFO | "malloc" : " 14 GB", | 20260507 16:58:24.290 | INFO | "pinned" : " 29 GB" | 20260507 16:58:24.290 | INFO | } | 20260507 16:58:24.290 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:58:24.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:24.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.290 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 16:58:24.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.290 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 16:58:24.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:24.291 | INFO | Uploaded 36GiB to GPU in 2.44906s (14.6995GiB/s) | 20260507 16:58:24.291 | INFO | Downloaded 34GiB to GPU in 2.41505s (14.0784GiB/s) | 20260507 16:58:29.430 | INFO | Analyzing chunk frames = [1113600,1114112) | 20260507 16:58:29.430 | INFO | MemInfo: { | 20260507 16:58:29.430 | INFO | "gpu" : " 17 GB", | 20260507 16:58:29.430 | INFO | "huge" : "", | 20260507 16:58:29.430 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:29.430 | INFO | "malloc" : " 14 GB", | 20260507 16:58:29.430 | INFO | "pinned" : " 29 GB" | 20260507 16:58:29.430 | INFO | } | 20260507 16:58:29.430 | INFO | Perf report for the last 5.13967 seconds: | 20260507 16:58:29.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:29.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 16:58:29.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:58:29.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:29.430 | INFO | Uploaded 36GiB to GPU in 2.49848s (14.4088GiB/s) | 20260507 16:58:29.430 | INFO | Downloaded 34GiB to GPU in 2.5016s (13.5913GiB/s) | 20260507 16:58:29.431 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:58:29.431 | INFO | Total mem usage: 16.4 GiB | 20260507 16:58:29.431 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:58:29.431 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.5%) | 20260507 16:58:29.431 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:58:29.431 | INFO | High Water Breakdown: | 20260507 16:58:29.431 | INFO | 59.1% : 8.62 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:58:29.431 | INFO | 40.7% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:58:29.431 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:58:29.431 | INFO | ------------------------------------------ | 20260507 16:58:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:58:29.431 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:58:29.431 | INFO | Total mem usage: 26.5 GiB | 20260507 16:58:29.431 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:58:29.431 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:58:29.431 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:58:29.431 | INFO | High Water Breakdown: | 20260507 16:58:29.431 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:58:29.431 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:58:29.431 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:58:29.431 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:58:29.431 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:58:29.431 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:58:29.431 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:58:29.431 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:58:29.431 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:58:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:58:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:58:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:58:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:58:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:58:29.431 | INFO | ------------------------------------------ | 20260507 16:58:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:58:29.431 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:58:29.431 | INFO | Total mem usage: 27.8 GiB | 20260507 16:58:29.431 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:58:29.431 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:58:29.431 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:58:29.431 | INFO | High Water Breakdown: | 20260507 16:58:29.431 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:58:29.431 | INFO | ------------------------------------------ | 20260507 16:58:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:58:29.431 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:58:29.431 | INFO | Total mem usage: 15.7 GiB | 20260507 16:58:29.431 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:58:29.431 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:58:29.431 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:58:29.431 | INFO | High Water Breakdown: | 20260507 16:58:29.431 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:58:29.431 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:58:29.431 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:58:29.431 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:58:29.431 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:58:29.431 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:58:29.431 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:58:29.431 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:58:29.431 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:58:29.431 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:58:29.431 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:58:29.431 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:58:29.431 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:58:29.431 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:58:29.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:58:29.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:58:29.431 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:58:29.431 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:58:29.431 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:58:29.431 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:58:29.431 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:58:29.431 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:58:29.431 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:58:29.431 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:58:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:58:29.431 | INFO | ------------------------------------------ | 20260507 16:58:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:58:29.431 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:58:29.431 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:58:29.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:58:29.431 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.64e+01, 1.41e+01 / 2.66e+00 | 20260507 16:58:29.431 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.13e-02 / 7.28e-03 | 20260507 16:58:29.431 | INFO | PerfReport: 12.06%, Baseline -- 4.05e+00 / 4.48e+00, 6.23e-01 / 2.40e-01 | 20260507 16:58:29.431 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 9.31e-02 | 20260507 16:58:29.431 | INFO | PerfReport: 15.74%, DME -- 4.23e+01 / 4.51e+01, 4.45e+00 / 2.69e+00 | 20260507 16:58:29.431 | INFO | PerfReport: 55.91%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.85e+00 / 1.84e+00 | 20260507 16:58:29.431 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.53e+00 / 2.91e+00, 4.36e-01 / 2.18e-01 | 20260507 16:58:29.431 | INFO | PerfReport: 3.82%, Metrics -- 1.28e+00 / 1.46e+00, 3.29e-01 / 1.95e-01 | 20260507 16:58:29.431 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.05e-01 / 4.30e-01, 6.85e-02 / 4.15e-02 | 20260507 16:58:29.432 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:58:29.432 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:58:29.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:58:29.432 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 23.94%, Upload -- 2.57e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 23.84%, Download -- 2.56e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 31.21%, Compute -- 3.35e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 1.92%, LockUpload -- 2.07e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 2.23%, LockDownload -- 2.40e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 16.84%, LockCompute -- 1.81e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 16:58:29.432 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260507 16:58:29.527 | INFO | Starting a round of Baz IO | 20260507 16:58:33.040 | INFO | Populating RTMetrics report | 20260507 16:58:33.040 | INFO | RTMetrics report populated | 20260507 16:58:33.040 | INFO | Opening tmp RTMetrics json file | 20260507 16:58:33.065 | INFO | Serializing RTMetrics to json | 20260507 16:58:33.067 | INFO | Serialization done, swapping tmp file now... | 20260507 16:58:33.067 | INFO | RTMetrics done | 20260507 16:58:33.147 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.619 seconds | 20260507 16:58:33.147 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.07% / 0.02% / 50.91% | 20260507 16:58:33.147 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0044% / 5.16574e-05% / 2.38419e-05% | 20260507 16:58:33.147 | INFO | Baz IO overall: | 20260507 16:58:33.147 | INFO | BytesWritten : 484894710280 | 20260507 16:58:33.147 | INFO | NumEvents : 234804121494 | 20260507 16:58:34.569 | INFO | Analyzing chunk frames = [1114112,1114624) | 20260507 16:58:34.569 | INFO | MemInfo: { | 20260507 16:58:34.569 | INFO | "gpu" : " 17 GB", | 20260507 16:58:34.569 | INFO | "huge" : "", | 20260507 16:58:34.569 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:34.569 | INFO | "malloc" : " 14 GB", | 20260507 16:58:34.569 | INFO | "pinned" : " 29 GB" | 20260507 16:58:34.569 | INFO | } | 20260507 16:58:34.569 | INFO | Perf report for the last 5.13927 seconds: | 20260507 16:58:34.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:34.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:34.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:34.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:34.569 | INFO | ANALYSIS : 69.8%, 2.9, 107.6ms, 0 | 20260507 16:58:34.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:34.570 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 16:58:34.570 | INFO | PRE_HQ : 66.6%, 1.8, 60.6ms, 0 | 20260507 16:58:34.570 | INFO | BAZWRITER : 70.0%, 4.0, 144.4ms, 0 | 20260507 16:58:34.570 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 16:58:34.570 | INFO | Uploaded 36GiB to GPU in 2.53915s (14.178GiB/s) | 20260507 16:58:34.570 | INFO | Downloaded 38GiB to GPU in 2.72407s (13.9497GiB/s) | 20260507 16:58:39.708 | INFO | Analyzing chunk frames = [1114624,1115136) | 20260507 16:58:39.708 | INFO | MemInfo: { | 20260507 16:58:39.708 | INFO | "gpu" : " 17 GB", | 20260507 16:58:39.708 | INFO | "huge" : "", | 20260507 16:58:39.708 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:39.709 | INFO | "malloc" : " 14 GB", | 20260507 16:58:39.709 | INFO | "pinned" : " 29 GB" | 20260507 16:58:39.709 | INFO | } | 20260507 16:58:39.709 | INFO | Perf report for the last 5.13924 seconds: | 20260507 16:58:39.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:39.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 16:58:39.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:58:39.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:39.709 | INFO | Uploaded 36GiB to GPU in 2.50102s (14.3941GiB/s) | 20260507 16:58:39.709 | INFO | Downloaded 34GiB to GPU in 2.43626s (13.9558GiB/s) | 20260507 16:58:44.847 | INFO | Analyzing chunk frames = [1115136,1115648) | 20260507 16:58:44.847 | INFO | MemInfo: { | 20260507 16:58:44.847 | INFO | "gpu" : " 17 GB", | 20260507 16:58:44.847 | INFO | "huge" : "", | 20260507 16:58:44.848 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:44.848 | INFO | "malloc" : " 14 GB", | 20260507 16:58:44.848 | INFO | "pinned" : " 29 GB" | 20260507 16:58:44.848 | INFO | } | 20260507 16:58:44.848 | INFO | Perf report for the last 5.13912 seconds: | 20260507 16:58:44.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:44.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260507 16:58:44.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 16:58:44.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:44.848 | INFO | Uploaded 36GiB to GPU in 2.56396s (14.0408GiB/s) | 20260507 16:58:44.848 | INFO | Downloaded 34GiB to GPU in 2.42194s (14.0383GiB/s) | 20260507 16:58:49.987 | INFO | Analyzing chunk frames = [1115648,1116160) | 20260507 16:58:49.987 | INFO | MemInfo: { | 20260507 16:58:49.987 | INFO | "gpu" : " 17 GB", | 20260507 16:58:49.987 | INFO | "huge" : "", | 20260507 16:58:49.987 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:49.987 | INFO | "malloc" : " 14 GB", | 20260507 16:58:49.987 | INFO | "pinned" : " 29 GB" | 20260507 16:58:49.987 | INFO | } | 20260507 16:58:49.987 | INFO | Perf report for the last 5.13941 seconds: | 20260507 16:58:49.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:49.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:58:49.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:58:49.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:49.987 | INFO | Uploaded 36GiB to GPU in 2.48838s (14.4672GiB/s) | 20260507 16:58:49.988 | INFO | Downloaded 34GiB to GPU in 2.35551s (14.4342GiB/s) | 20260507 16:58:55.127 | INFO | Analyzing chunk frames = [1116160,1116672) | 20260507 16:58:55.127 | INFO | MemInfo: { | 20260507 16:58:55.127 | INFO | "gpu" : " 17 GB", | 20260507 16:58:55.127 | INFO | "huge" : "", | 20260507 16:58:55.127 | INFO | "huge_shared" : " 29 GB", | 20260507 16:58:55.127 | INFO | "malloc" : " 14 GB", | 20260507 16:58:55.127 | INFO | "pinned" : " 29 GB" | 20260507 16:58:55.127 | INFO | } | 20260507 16:58:55.127 | INFO | Perf report for the last 5.13963 seconds: | 20260507 16:58:55.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:58:55.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:58:55.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 16:58:55.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:58:55.127 | INFO | Uploaded 36GiB to GPU in 2.3796s (15.1286GiB/s) | 20260507 16:58:55.127 | INFO | Downloaded 34GiB to GPU in 2.48678s (13.6723GiB/s) | 20260507 16:59:00.265 | INFO | Analyzing chunk frames = [1116672,1117184) | 20260507 16:59:00.265 | INFO | MemInfo: { | 20260507 16:59:00.265 | INFO | "gpu" : " 17 GB", | 20260507 16:59:00.265 | INFO | "huge" : "", | 20260507 16:59:00.265 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:00.265 | INFO | "malloc" : " 14 GB", | 20260507 16:59:00.265 | INFO | "pinned" : " 29 GB" | 20260507 16:59:00.265 | INFO | } | 20260507 16:59:00.266 | INFO | Perf report for the last 5.1387 seconds: | 20260507 16:59:00.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:00.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 16:59:00.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 16:59:00.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:00.266 | INFO | Uploaded 36GiB to GPU in 2.42853s (14.8238GiB/s) | 20260507 16:59:00.266 | INFO | Downloaded 34GiB to GPU in 2.45964s (13.8231GiB/s) | 20260507 16:59:05.405 | INFO | Analyzing chunk frames = [1117184,1117696) | 20260507 16:59:05.405 | INFO | MemInfo: { | 20260507 16:59:05.405 | INFO | "gpu" : " 17 GB", | 20260507 16:59:05.405 | INFO | "huge" : "", | 20260507 16:59:05.405 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:05.405 | INFO | "malloc" : " 14 GB", | 20260507 16:59:05.405 | INFO | "pinned" : " 29 GB" | 20260507 16:59:05.405 | INFO | } | 20260507 16:59:05.405 | INFO | Perf report for the last 5.1395 seconds: | 20260507 16:59:05.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:05.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 16:59:05.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 16:59:05.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:05.405 | INFO | Uploaded 36GiB to GPU in 2.49828s (14.4099GiB/s) | 20260507 16:59:05.405 | INFO | Downloaded 34GiB to GPU in 2.40721s (14.1242GiB/s) | 20260507 16:59:10.544 | INFO | Analyzing chunk frames = [1117696,1118208) | 20260507 16:59:10.544 | INFO | MemInfo: { | 20260507 16:59:10.544 | INFO | "gpu" : " 17 GB", | 20260507 16:59:10.544 | INFO | "huge" : "", | 20260507 16:59:10.544 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:10.544 | INFO | "malloc" : " 14 GB", | 20260507 16:59:10.544 | INFO | "pinned" : " 29 GB" | 20260507 16:59:10.544 | INFO | } | 20260507 16:59:10.545 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:59:10.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:10.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:59:10.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 16:59:10.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:10.545 | INFO | Uploaded 36GiB to GPU in 2.47182s (14.5642GiB/s) | 20260507 16:59:10.545 | INFO | Downloaded 34GiB to GPU in 2.4985s (13.6082GiB/s) | 20260507 16:59:10.642 | INFO | Starting a round of Baz IO | 20260507 16:59:14.158 | INFO | Populating RTMetrics report | 20260507 16:59:14.158 | INFO | RTMetrics report populated | 20260507 16:59:14.158 | INFO | Opening tmp RTMetrics json file | 20260507 16:59:14.166 | INFO | Serializing RTMetrics to json | 20260507 16:59:14.181 | INFO | Serialization done, swapping tmp file now... | 20260507 16:59:14.182 | INFO | RTMetrics done | 20260507 16:59:14.261 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.619 seconds | 20260507 16:59:14.261 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.04% / 0.02% / 50.93% | 20260507 16:59:14.261 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.99088% / 3.17891e-05% / 3.97364e-05% | 20260507 16:59:14.261 | INFO | Baz IO overall: | 20260507 16:59:14.261 | INFO | BytesWritten : 486821437186 | 20260507 16:59:14.261 | INFO | NumEvents : 235740871323 | 20260507 16:59:15.684 | INFO | Analyzing chunk frames = [1118208,1118720) | 20260507 16:59:15.684 | INFO | MemInfo: { | 20260507 16:59:15.684 | INFO | "gpu" : " 17 GB", | 20260507 16:59:15.684 | INFO | "huge" : "", | 20260507 16:59:15.684 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:15.684 | INFO | "malloc" : " 14 GB", | 20260507 16:59:15.684 | INFO | "pinned" : " 29 GB" | 20260507 16:59:15.684 | INFO | } | 20260507 16:59:15.684 | INFO | Perf report for the last 5.13933 seconds: | 20260507 16:59:15.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:15.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:15.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:15.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:15.684 | INFO | ANALYSIS : 69.8%, 2.9, 107.8ms, 0 | 20260507 16:59:15.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:15.684 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 16:59:15.684 | INFO | PRE_HQ : 66.6%, 1.8, 60.9ms, 0 | 20260507 16:59:15.684 | INFO | BAZWRITER : 70.0%, 3.9, 141.6ms, 0 | 20260507 16:59:15.684 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 16:59:15.684 | INFO | Uploaded 36GiB to GPU in 2.59225s (13.8875GiB/s) | 20260507 16:59:15.685 | INFO | Downloaded 38GiB to GPU in 2.72567s (13.9415GiB/s) | 20260507 16:59:20.823 | INFO | Analyzing chunk frames = [1118720,1119232) | 20260507 16:59:20.823 | INFO | MemInfo: { | 20260507 16:59:20.823 | INFO | "gpu" : " 17 GB", | 20260507 16:59:20.823 | INFO | "huge" : "", | 20260507 16:59:20.823 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:20.823 | INFO | "malloc" : " 14 GB", | 20260507 16:59:20.823 | INFO | "pinned" : " 29 GB" | 20260507 16:59:20.823 | INFO | } | 20260507 16:59:20.823 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:59:20.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:20.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.823 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 16:59:20.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.823 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 16:59:20.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:20.824 | INFO | Uploaded 36GiB to GPU in 2.5307s (14.2253GiB/s) | 20260507 16:59:20.824 | INFO | Downloaded 34GiB to GPU in 2.42725s (14.0076GiB/s) | 20260507 16:59:25.962 | INFO | Analyzing chunk frames = [1119232,1119744) | 20260507 16:59:25.962 | INFO | MemInfo: { | 20260507 16:59:25.962 | INFO | "gpu" : " 17 GB", | 20260507 16:59:25.962 | INFO | "huge" : "", | 20260507 16:59:25.962 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:25.962 | INFO | "malloc" : " 14 GB", | 20260507 16:59:25.962 | INFO | "pinned" : " 29 GB" | 20260507 16:59:25.962 | INFO | } | 20260507 16:59:25.962 | INFO | Perf report for the last 5.13928 seconds: | 20260507 16:59:25.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:25.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.962 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260507 16:59:25.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.963 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 16:59:25.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:25.963 | INFO | Uploaded 36GiB to GPU in 2.48936s (14.4616GiB/s) | 20260507 16:59:25.963 | INFO | Downloaded 34GiB to GPU in 2.33069s (14.5879GiB/s) | 20260507 16:59:31.102 | INFO | Analyzing chunk frames = [1119744,1120256) | 20260507 16:59:31.102 | INFO | MemInfo: { | 20260507 16:59:31.102 | INFO | "gpu" : " 17 GB", | 20260507 16:59:31.102 | INFO | "huge" : "", | 20260507 16:59:31.102 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:31.102 | INFO | "malloc" : " 14 GB", | 20260507 16:59:31.102 | INFO | "pinned" : " 29 GB" | 20260507 16:59:31.102 | INFO | } | 20260507 16:59:31.102 | INFO | Perf report for the last 5.1394 seconds: | 20260507 16:59:31.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:31.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 16:59:31.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 16:59:31.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:31.102 | INFO | Uploaded 36GiB to GPU in 2.4975s (14.4144GiB/s) | 20260507 16:59:31.102 | INFO | Downloaded 34GiB to GPU in 2.34975s (14.4696GiB/s) | 20260507 16:59:36.242 | INFO | Analyzing chunk frames = [1120256,1120768) | 20260507 16:59:36.242 | INFO | MemInfo: { | 20260507 16:59:36.242 | INFO | "gpu" : " 17 GB", | 20260507 16:59:36.242 | INFO | "huge" : "", | 20260507 16:59:36.242 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:36.242 | INFO | "malloc" : " 14 GB", | 20260507 16:59:36.242 | INFO | "pinned" : " 29 GB" | 20260507 16:59:36.242 | INFO | } | 20260507 16:59:36.242 | INFO | Perf report for the last 5.14015 seconds: | 20260507 16:59:36.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:36.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 16:59:36.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 16:59:36.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:36.242 | INFO | Uploaded 36GiB to GPU in 2.38354s (15.1036GiB/s) | 20260507 16:59:36.242 | INFO | Downloaded 34GiB to GPU in 2.48532s (13.6803GiB/s) | 20260507 16:59:41.381 | INFO | Analyzing chunk frames = [1120768,1121280) | 20260507 16:59:41.381 | INFO | MemInfo: { | 20260507 16:59:41.381 | INFO | "gpu" : " 17 GB", | 20260507 16:59:41.381 | INFO | "huge" : "", | 20260507 16:59:41.381 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:41.381 | INFO | "malloc" : " 14 GB", | 20260507 16:59:41.381 | INFO | "pinned" : " 29 GB" | 20260507 16:59:41.381 | INFO | } | 20260507 16:59:41.381 | INFO | Perf report for the last 5.13881 seconds: | 20260507 16:59:41.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:41.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 16:59:41.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260507 16:59:41.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:41.381 | INFO | Uploaded 36GiB to GPU in 2.40856s (14.9467GiB/s) | 20260507 16:59:41.381 | INFO | Downloaded 34GiB to GPU in 2.46677s (13.7832GiB/s) | 20260507 16:59:46.520 | INFO | Analyzing chunk frames = [1121280,1121792) | 20260507 16:59:46.520 | INFO | MemInfo: { | 20260507 16:59:46.520 | INFO | "gpu" : " 17 GB", | 20260507 16:59:46.520 | INFO | "huge" : "", | 20260507 16:59:46.520 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:46.520 | INFO | "malloc" : " 14 GB", | 20260507 16:59:46.520 | INFO | "pinned" : " 29 GB" | 20260507 16:59:46.520 | INFO | } | 20260507 16:59:46.520 | INFO | Perf report for the last 5.13951 seconds: | 20260507 16:59:46.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:46.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.520 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 16:59:46.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.520 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 16:59:46.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:46.521 | INFO | Uploaded 36GiB to GPU in 2.50663s (14.3619GiB/s) | 20260507 16:59:46.521 | INFO | Downloaded 34GiB to GPU in 2.41283s (14.0913GiB/s) | 20260507 16:59:51.659 | INFO | Analyzing chunk frames = [1121792,1122304) | 20260507 16:59:51.659 | INFO | MemInfo: { | 20260507 16:59:51.659 | INFO | "gpu" : " 17 GB", | 20260507 16:59:51.659 | INFO | "huge" : "", | 20260507 16:59:51.659 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:51.659 | INFO | "malloc" : " 14 GB", | 20260507 16:59:51.659 | INFO | "pinned" : " 29 GB" | 20260507 16:59:51.660 | INFO | } | 20260507 16:59:51.660 | INFO | Perf report for the last 5.13922 seconds: | 20260507 16:59:51.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:51.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 16:59:51.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 16:59:51.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:51.660 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 16:59:51.660 | INFO | Total mem usage: 16.4 GiB | 20260507 16:59:51.660 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 16:59:51.660 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.2%) | 20260507 16:59:51.660 | INFO | Uploaded 36GiB to GPU in 2.55621s (14.0834GiB/s) | 20260507 16:59:51.660 | INFO | Downloaded 34GiB to GPU in 2.50792s (13.557GiB/s) | 20260507 16:59:51.660 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 16:59:51.660 | INFO | High Water Breakdown: | 20260507 16:59:51.660 | INFO | 59.6% : 8.96 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 16:59:51.660 | INFO | 40.2% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 16:59:51.660 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 16:59:51.660 | INFO | ------------------------------------------ | 20260507 16:59:51.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:59:51.660 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 16:59:51.660 | INFO | Total mem usage: 26.5 GiB | 20260507 16:59:51.660 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 16:59:51.660 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 16:59:51.660 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 16:59:51.661 | INFO | High Water Breakdown: | 20260507 16:59:51.661 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:59:51.661 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:59:51.661 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:59:51.661 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:59:51.661 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:59:51.661 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:59:51.661 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 16:59:51.661 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 16:59:51.661 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:59:51.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 16:59:51.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 16:59:51.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 16:59:51.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 16:59:51.661 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 16:59:51.661 | INFO | ------------------------------------------ | 20260507 16:59:51.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:59:51.661 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 16:59:51.661 | INFO | Total mem usage: 27.8 GiB | 20260507 16:59:51.661 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 16:59:51.661 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 16:59:51.661 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 16:59:51.661 | INFO | High Water Breakdown: | 20260507 16:59:51.661 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 16:59:51.661 | INFO | ------------------------------------------ | 20260507 16:59:51.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:59:51.661 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 16:59:51.661 | INFO | Total mem usage: 15.7 GiB | 20260507 16:59:51.661 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 16:59:51.661 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 16:59:51.661 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 16:59:51.661 | INFO | High Water Breakdown: | 20260507 16:59:51.661 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 16:59:51.661 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 16:59:51.661 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 16:59:51.661 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 16:59:51.661 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 16:59:51.661 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 16:59:51.661 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 16:59:51.661 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 16:59:51.661 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 16:59:51.661 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 16:59:51.661 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 16:59:51.661 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 16:59:51.661 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 16:59:51.661 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 16:59:51.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 16:59:51.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 16:59:51.661 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 16:59:51.661 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 16:59:51.661 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 16:59:51.661 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 16:59:51.661 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 16:59:51.661 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 16:59:51.661 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 16:59:51.661 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 16:59:51.661 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 16:59:51.661 | INFO | ------------------------------------------ | 20260507 16:59:51.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 16:59:51.661 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 16:59:51.661 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:59:51.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:59:51.661 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.43e+01 / 2.87e+00 | 20260507 16:59:51.661 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.11e-02 / 7.90e-03 | 20260507 16:59:51.661 | INFO | PerfReport: 12.02%, Baseline -- 4.03e+00 / 4.51e+00, 6.17e-01 / 2.27e-01 | 20260507 16:59:51.661 | INFO | PerfReport: 3.41%, Histogram -- 1.14e+00 / 1.50e+00, 2.17e-01 / 1.06e-01 | 20260507 16:59:51.661 | INFO | PerfReport: 15.74%, DME -- 4.22e+01 / 4.51e+01, 4.30e+00 / 2.56e+00 | 20260507 16:59:51.661 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.88e+00 / 2.00e+00 | 20260507 16:59:51.661 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.53e+00 / 2.91e+00, 4.37e-01 / 2.28e-01 | 20260507 16:59:51.662 | INFO | PerfReport: 3.81%, Metrics -- 1.28e+00 / 1.46e+00, 3.27e-01 / 1.87e-01 | 20260507 16:59:51.662 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.05e-01 / 4.30e-01, 6.74e-02 / 4.38e-02 | 20260507 16:59:51.662 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 16:59:51.662 | INFO | PerfReport: Gathered 1536 samples. | 20260507 16:59:51.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 16:59:51.662 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 24.12%, Upload -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 23.92%, Download -- 2.57e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 31.20%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 1.96%, LockUpload -- 2.10e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 2.12%, LockDownload -- 2.28e+00 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 16.68%, LockCompute -- 1.79e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 16:59:51.662 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 16:59:51.756 | INFO | Starting a round of Baz IO | 20260507 16:59:55.257 | INFO | Populating RTMetrics report | 20260507 16:59:55.258 | INFO | RTMetrics report populated | 20260507 16:59:55.258 | INFO | Opening tmp RTMetrics json file | 20260507 16:59:55.277 | INFO | Serializing RTMetrics to json | 20260507 16:59:55.278 | INFO | Serialization done, swapping tmp file now... | 20260507 16:59:55.279 | INFO | RTMetrics done | 20260507 16:59:55.359 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.603 seconds | 20260507 16:59:55.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.02% / 0.02% / 50.96% | 20260507 16:59:55.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.98519% / 3.17891e-05% / 1.58946e-05% | 20260507 16:59:55.360 | INFO | Baz IO overall: | 20260507 16:59:55.360 | INFO | BytesWritten : 488748971999 | 20260507 16:59:55.360 | INFO | NumEvents : 236678049688 | 20260507 16:59:56.798 | INFO | Analyzing chunk frames = [1122304,1122816) | 20260507 16:59:56.798 | INFO | MemInfo: { | 20260507 16:59:56.798 | INFO | "gpu" : " 17 GB", | 20260507 16:59:56.798 | INFO | "huge" : "", | 20260507 16:59:56.798 | INFO | "huge_shared" : " 29 GB", | 20260507 16:59:56.798 | INFO | "malloc" : " 14 GB", | 20260507 16:59:56.798 | INFO | "pinned" : " 29 GB" | 20260507 16:59:56.799 | INFO | } | 20260507 16:59:56.799 | INFO | Perf report for the last 5.13921 seconds: | 20260507 16:59:56.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 16:59:56.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:56.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:56.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:56.799 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 16:59:56.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 16:59:56.799 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 16:59:56.799 | INFO | PRE_HQ : 66.9%, 1.8, 61.0ms, 0 | 20260507 16:59:56.799 | INFO | BAZWRITER : 69.5%, 3.7, 132.5ms, 0 | 20260507 16:59:56.799 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 16:59:56.800 | INFO | Uploaded 36GiB to GPU in 2.5432s (14.1554GiB/s) | 20260507 16:59:56.800 | INFO | Downloaded 38GiB to GPU in 2.71381s (14.0024GiB/s) | 20260507 17:00:01.938 | INFO | Analyzing chunk frames = [1122816,1123328) | 20260507 17:00:01.938 | INFO | MemInfo: { | 20260507 17:00:01.938 | INFO | "gpu" : " 17 GB", | 20260507 17:00:01.938 | INFO | "huge" : "", | 20260507 17:00:01.938 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:01.938 | INFO | "malloc" : " 14 GB", | 20260507 17:00:01.938 | INFO | "pinned" : " 29 GB" | 20260507 17:00:01.938 | INFO | } | 20260507 17:00:01.938 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:00:01.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:01.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.938 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 17:00:01.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.938 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:00:01.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:01.939 | INFO | Uploaded 36GiB to GPU in 2.5271s (14.2456GiB/s) | 20260507 17:00:01.939 | INFO | Downloaded 34GiB to GPU in 2.41981s (14.0507GiB/s) | 20260507 17:00:07.077 | INFO | Analyzing chunk frames = [1123328,1123840) | 20260507 17:00:07.077 | INFO | MemInfo: { | 20260507 17:00:07.077 | INFO | "gpu" : " 17 GB", | 20260507 17:00:07.077 | INFO | "huge" : "", | 20260507 17:00:07.078 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:07.078 | INFO | "malloc" : " 14 GB", | 20260507 17:00:07.078 | INFO | "pinned" : " 29 GB" | 20260507 17:00:07.078 | INFO | } | 20260507 17:00:07.078 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:00:07.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:07.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 17:00:07.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:00:07.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:07.078 | INFO | Uploaded 36GiB to GPU in 2.50134s (14.3923GiB/s) | 20260507 17:00:07.078 | INFO | Downloaded 34GiB to GPU in 2.33406s (14.5669GiB/s) | 20260507 17:00:12.217 | INFO | Analyzing chunk frames = [1123840,1124352) | 20260507 17:00:12.217 | INFO | MemInfo: { | 20260507 17:00:12.217 | INFO | "gpu" : " 17 GB", | 20260507 17:00:12.217 | INFO | "huge" : "", | 20260507 17:00:12.217 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:12.217 | INFO | "malloc" : " 14 GB", | 20260507 17:00:12.217 | INFO | "pinned" : " 29 GB" | 20260507 17:00:12.217 | INFO | } | 20260507 17:00:12.217 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:00:12.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:12.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 17:00:12.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:00:12.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:12.217 | INFO | Uploaded 36GiB to GPU in 2.50205s (14.3882GiB/s) | 20260507 17:00:12.217 | INFO | Downloaded 34GiB to GPU in 2.34353s (14.508GiB/s) | 20260507 17:00:17.356 | INFO | Analyzing chunk frames = [1124352,1124864) | 20260507 17:00:17.356 | INFO | MemInfo: { | 20260507 17:00:17.356 | INFO | "gpu" : " 17 GB", | 20260507 17:00:17.356 | INFO | "huge" : "", | 20260507 17:00:17.356 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:17.356 | INFO | "malloc" : " 14 GB", | 20260507 17:00:17.356 | INFO | "pinned" : " 29 GB" | 20260507 17:00:17.356 | INFO | } | 20260507 17:00:17.356 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:00:17.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:17.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.356 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:00:17.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.356 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:00:17.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:17.357 | INFO | Uploaded 36GiB to GPU in 2.40386s (14.9759GiB/s) | 20260507 17:00:17.357 | INFO | Downloaded 34GiB to GPU in 2.46052s (13.8182GiB/s) | 20260507 17:00:22.495 | INFO | Analyzing chunk frames = [1124864,1125376) | 20260507 17:00:22.495 | INFO | MemInfo: { | 20260507 17:00:22.495 | INFO | "gpu" : " 17 GB", | 20260507 17:00:22.496 | INFO | "huge" : "", | 20260507 17:00:22.496 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:22.496 | INFO | "malloc" : " 14 GB", | 20260507 17:00:22.496 | INFO | "pinned" : " 29 GB" | 20260507 17:00:22.496 | INFO | } | 20260507 17:00:22.496 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:00:22.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:22.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:00:22.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:00:22.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:22.496 | INFO | Uploaded 36GiB to GPU in 2.35142s (15.3099GiB/s) | 20260507 17:00:22.496 | INFO | Downloaded 34GiB to GPU in 2.44961s (13.8797GiB/s) | 20260507 17:00:27.634 | INFO | Analyzing chunk frames = [1125376,1125888) | 20260507 17:00:27.634 | INFO | MemInfo: { | 20260507 17:00:27.634 | INFO | "gpu" : " 17 GB", | 20260507 17:00:27.635 | INFO | "huge" : "", | 20260507 17:00:27.635 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:27.635 | INFO | "malloc" : " 14 GB", | 20260507 17:00:27.635 | INFO | "pinned" : " 29 GB" | 20260507 17:00:27.635 | INFO | } | 20260507 17:00:27.635 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:00:27.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:27.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:00:27.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:00:27.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:27.635 | INFO | Uploaded 36GiB to GPU in 2.45801s (14.646GiB/s) | 20260507 17:00:27.635 | INFO | Downloaded 34GiB to GPU in 2.42062s (14.046GiB/s) | 20260507 17:00:32.774 | INFO | Analyzing chunk frames = [1125888,1126400) | 20260507 17:00:32.774 | INFO | MemInfo: { | 20260507 17:00:32.774 | INFO | "gpu" : " 17 GB", | 20260507 17:00:32.774 | INFO | "huge" : "", | 20260507 17:00:32.774 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:32.774 | INFO | "malloc" : " 14 GB", | 20260507 17:00:32.774 | INFO | "pinned" : " 29 GB" | 20260507 17:00:32.774 | INFO | } | 20260507 17:00:32.774 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:00:32.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:32.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.774 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 17:00:32.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.774 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:00:32.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:32.775 | INFO | Uploaded 36GiB to GPU in 2.49403s (14.4345GiB/s) | 20260507 17:00:32.775 | INFO | Downloaded 34GiB to GPU in 2.51938s (13.4954GiB/s) | 20260507 17:00:32.871 | INFO | Starting a round of Baz IO | 20260507 17:00:36.390 | INFO | Populating RTMetrics report | 20260507 17:00:36.390 | INFO | RTMetrics report populated | 20260507 17:00:36.390 | INFO | Opening tmp RTMetrics json file | 20260507 17:00:36.405 | INFO | Serializing RTMetrics to json | 20260507 17:00:36.413 | INFO | Serialization done, swapping tmp file now... | 20260507 17:00:36.414 | INFO | RTMetrics done | 20260507 17:00:36.490 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.618 seconds | 20260507 17:00:36.490 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.00% / 0.02% / 50.98% | 20260507 17:00:36.490 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.96552% / 3.17891e-05% / 3.57628e-05% | 20260507 17:00:36.490 | INFO | Baz IO overall: | 20260507 17:00:36.490 | INFO | BytesWritten : 490674864020 | 20260507 17:00:36.490 | INFO | NumEvents : 237613828398 | 20260507 17:00:37.913 | INFO | Analyzing chunk frames = [1126400,1126912) | 20260507 17:00:37.913 | INFO | MemInfo: { | 20260507 17:00:37.913 | INFO | "gpu" : " 17 GB", | 20260507 17:00:37.913 | INFO | "huge" : "", | 20260507 17:00:37.913 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:37.913 | INFO | "malloc" : " 14 GB", | 20260507 17:00:37.913 | INFO | "pinned" : " 29 GB" | 20260507 17:00:37.913 | INFO | } | 20260507 17:00:37.913 | INFO | Perf report for the last 5.13924 seconds: | 20260507 17:00:37.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:37.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:37.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:37.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:37.914 | INFO | ANALYSIS : 69.7%, 3.0, 108.0ms, 0 | 20260507 17:00:37.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:37.914 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 17:00:37.914 | INFO | PRE_HQ : 66.5%, 1.8, 60.7ms, 0 | 20260507 17:00:37.914 | INFO | BAZWRITER : 70.1%, 3.8, 137.4ms, 0 | 20260507 17:00:37.914 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:00:37.914 | INFO | Uploaded 36GiB to GPU in 2.52513s (14.2567GiB/s) | 20260507 17:00:37.914 | INFO | Downloaded 38GiB to GPU in 2.69321s (14.1095GiB/s) | 20260507 17:00:43.053 | INFO | Analyzing chunk frames = [1126912,1127424) | 20260507 17:00:43.053 | INFO | MemInfo: { | 20260507 17:00:43.053 | INFO | "gpu" : " 17 GB", | 20260507 17:00:43.053 | INFO | "huge" : "", | 20260507 17:00:43.053 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:43.053 | INFO | "malloc" : " 14 GB", | 20260507 17:00:43.053 | INFO | "pinned" : " 29 GB" | 20260507 17:00:43.053 | INFO | } | 20260507 17:00:43.053 | INFO | Perf report for the last 5.13993 seconds: | 20260507 17:00:43.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:43.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.053 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:00:43.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.053 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:00:43.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:43.054 | INFO | Uploaded 36GiB to GPU in 2.50971s (14.3443GiB/s) | 20260507 17:00:43.054 | INFO | Downloaded 34GiB to GPU in 2.44209s (13.9225GiB/s) | 20260507 17:00:48.192 | INFO | Analyzing chunk frames = [1127424,1127936) | 20260507 17:00:48.192 | INFO | MemInfo: { | 20260507 17:00:48.192 | INFO | "gpu" : " 17 GB", | 20260507 17:00:48.192 | INFO | "huge" : "", | 20260507 17:00:48.192 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:48.192 | INFO | "malloc" : " 14 GB", | 20260507 17:00:48.193 | INFO | "pinned" : " 29 GB" | 20260507 17:00:48.193 | INFO | } | 20260507 17:00:48.193 | INFO | Perf report for the last 5.13925 seconds: | 20260507 17:00:48.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:48.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260507 17:00:48.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:00:48.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:48.193 | INFO | Uploaded 36GiB to GPU in 2.56371s (14.0421GiB/s) | 20260507 17:00:48.193 | INFO | Downloaded 34GiB to GPU in 2.42353s (14.0291GiB/s) | 20260507 17:00:53.331 | INFO | Analyzing chunk frames = [1127936,1128448) | 20260507 17:00:53.332 | INFO | MemInfo: { | 20260507 17:00:53.332 | INFO | "gpu" : " 17 GB", | 20260507 17:00:53.332 | INFO | "huge" : "", | 20260507 17:00:53.332 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:53.332 | INFO | "malloc" : " 14 GB", | 20260507 17:00:53.332 | INFO | "pinned" : " 29 GB" | 20260507 17:00:53.332 | INFO | } | 20260507 17:00:53.332 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:00:53.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:53.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:00:53.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:00:53.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:53.332 | INFO | Uploaded 36GiB to GPU in 2.4742s (14.5502GiB/s) | 20260507 17:00:53.332 | INFO | Downloaded 34GiB to GPU in 2.35471s (14.4391GiB/s) | 20260507 17:00:58.471 | INFO | Analyzing chunk frames = [1128448,1128960) | 20260507 17:00:58.471 | INFO | MemInfo: { | 20260507 17:00:58.471 | INFO | "gpu" : " 17 GB", | 20260507 17:00:58.471 | INFO | "huge" : "", | 20260507 17:00:58.471 | INFO | "huge_shared" : " 29 GB", | 20260507 17:00:58.471 | INFO | "malloc" : " 14 GB", | 20260507 17:00:58.471 | INFO | "pinned" : " 29 GB" | 20260507 17:00:58.471 | INFO | } | 20260507 17:00:58.471 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:00:58.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:00:58.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.471 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:00:58.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.471 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:00:58.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:00:58.472 | INFO | Uploaded 36GiB to GPU in 2.36507s (15.2215GiB/s) | 20260507 17:00:58.472 | INFO | Downloaded 34GiB to GPU in 2.46558s (13.7899GiB/s) | 20260507 17:01:03.610 | INFO | Analyzing chunk frames = [1128960,1129472) | 20260507 17:01:03.610 | INFO | MemInfo: { | 20260507 17:01:03.610 | INFO | "gpu" : " 17 GB", | 20260507 17:01:03.610 | INFO | "huge" : "", | 20260507 17:01:03.610 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:03.610 | INFO | "malloc" : " 14 GB", | 20260507 17:01:03.610 | INFO | "pinned" : " 29 GB" | 20260507 17:01:03.610 | INFO | } | 20260507 17:01:03.610 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:01:03.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:03.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.610 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:01:03.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.610 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 17:01:03.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:03.611 | INFO | Uploaded 36GiB to GPU in 2.38622s (15.0866GiB/s) | 20260507 17:01:03.611 | INFO | Downloaded 34GiB to GPU in 2.46142s (13.8132GiB/s) | 20260507 17:01:08.749 | INFO | Analyzing chunk frames = [1129472,1129984) | 20260507 17:01:08.749 | INFO | MemInfo: { | 20260507 17:01:08.749 | INFO | "gpu" : " 17 GB", | 20260507 17:01:08.749 | INFO | "huge" : "", | 20260507 17:01:08.749 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:08.750 | INFO | "malloc" : " 14 GB", | 20260507 17:01:08.750 | INFO | "pinned" : " 29 GB" | 20260507 17:01:08.750 | INFO | } | 20260507 17:01:08.750 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:01:08.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:08.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:01:08.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:01:08.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:08.750 | INFO | Uploaded 36GiB to GPU in 2.47966s (14.5181GiB/s) | 20260507 17:01:08.750 | INFO | Downloaded 34GiB to GPU in 2.40831s (14.1178GiB/s) | 20260507 17:01:13.889 | INFO | Analyzing chunk frames = [1129984,1130496) | 20260507 17:01:13.889 | INFO | MemInfo: { | 20260507 17:01:13.889 | INFO | "gpu" : " 17 GB", | 20260507 17:01:13.889 | INFO | "huge" : "", | 20260507 17:01:13.889 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:13.889 | INFO | "malloc" : " 14 GB", | 20260507 17:01:13.889 | INFO | "pinned" : " 29 GB" | 20260507 17:01:13.889 | INFO | } | 20260507 17:01:13.889 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:01:13.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:13.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 17:01:13.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:01:13.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:13.889 | INFO | Uploaded 36GiB to GPU in 2.54896s (14.1234GiB/s) | 20260507 17:01:13.889 | INFO | Downloaded 34GiB to GPU in 2.50745s (13.5596GiB/s) | 20260507 17:01:13.889 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:01:13.889 | INFO | Total mem usage: 16.4 GiB | 20260507 17:01:13.889 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:01:13.889 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.9%) | 20260507 17:01:13.889 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:01:13.889 | INFO | High Water Breakdown: | 20260507 17:01:13.889 | INFO | 59.4% : 8.78 GiB ( 88.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:01:13.889 | INFO | 40.5% : 5.99 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:01:13.889 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:01:13.889 | INFO | ------------------------------------------ | 20260507 17:01:13.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:01:13.889 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:01:13.889 | INFO | Total mem usage: 26.5 GiB | 20260507 17:01:13.889 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:01:13.889 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:01:13.889 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:01:13.889 | INFO | High Water Breakdown: | 20260507 17:01:13.889 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:01:13.889 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:01:13.889 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:01:13.889 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:01:13.889 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:01:13.889 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:01:13.889 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:01:13.889 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:01:13.889 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:01:13.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:01:13.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:01:13.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:01:13.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:01:13.889 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:01:13.889 | INFO | ------------------------------------------ | 20260507 17:01:13.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:01:13.889 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:01:13.889 | INFO | Total mem usage: 27.8 GiB | 20260507 17:01:13.889 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:01:13.889 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:01:13.889 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:01:13.889 | INFO | High Water Breakdown: | 20260507 17:01:13.889 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:01:13.889 | INFO | ------------------------------------------ | 20260507 17:01:13.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:01:13.889 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:01:13.889 | INFO | Total mem usage: 15.7 GiB | 20260507 17:01:13.889 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:01:13.890 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:01:13.890 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:01:13.890 | INFO | High Water Breakdown: | 20260507 17:01:13.890 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:01:13.890 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:01:13.890 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:01:13.890 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:01:13.890 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:01:13.890 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:01:13.890 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:01:13.890 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:01:13.890 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:01:13.890 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:01:13.890 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:01:13.890 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:01:13.890 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:01:13.890 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:01:13.890 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:01:13.890 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:01:13.890 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:01:13.890 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:01:13.890 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:01:13.890 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:01:13.890 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:01:13.890 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:01:13.890 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:01:13.890 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:01:13.890 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:01:13.890 | INFO | ------------------------------------------ | 20260507 17:01:13.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:01:13.890 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:01:13.890 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:01:13.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:01:13.890 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.44e+01 / 2.74e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.13e-02 / 7.61e-03 | 20260507 17:01:13.890 | INFO | PerfReport: 12.05%, Baseline -- 4.04e+00 / 4.52e+00, 6.21e-01 / 2.17e-01 | 20260507 17:01:13.890 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 1.01e-01 | 20260507 17:01:13.890 | INFO | PerfReport: 15.73%, DME -- 4.22e+01 / 4.51e+01, 4.46e+00 / 2.58e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 55.92%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.85e+00 / 2.04e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.53e+00 / 2.92e+00, 4.37e-01 / 2.27e-01 | 20260507 17:01:13.890 | INFO | PerfReport: 3.81%, Metrics -- 1.28e+00 / 1.45e+00, 3.27e-01 / 1.95e-01 | 20260507 17:01:13.890 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.04e-01 / 4.27e-01, 6.77e-02 / 4.43e-02 | 20260507 17:01:13.890 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:01:13.890 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:01:13.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:01:13.890 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 24.01%, Upload -- 2.58e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 23.88%, Download -- 2.57e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 31.18%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 1.92%, LockUpload -- 2.07e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 2.23%, LockDownload -- 2.40e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 16.77%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:01:13.890 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260507 17:01:13.986 | INFO | Starting a round of Baz IO | 20260507 17:01:17.502 | INFO | Populating RTMetrics report | 20260507 17:01:17.502 | INFO | RTMetrics report populated | 20260507 17:01:17.502 | INFO | Opening tmp RTMetrics json file | 20260507 17:01:17.521 | INFO | Serializing RTMetrics to json | 20260507 17:01:17.522 | INFO | Serialization done, swapping tmp file now... | 20260507 17:01:17.523 | INFO | RTMetrics done | 20260507 17:01:17.619 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.633 seconds | 20260507 17:01:17.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.97% / 0.02% / 51.00% | 20260507 17:01:17.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.97476% / 3.17891e-05% / 3.17891e-05% | 20260507 17:01:17.619 | INFO | Baz IO overall: | 20260507 17:01:17.619 | INFO | BytesWritten : 492603732874 | 20260507 17:01:17.619 | INFO | NumEvents : 238552042966 | 20260507 17:01:19.028 | INFO | Analyzing chunk frames = [1130496,1131008) | 20260507 17:01:19.028 | INFO | MemInfo: { | 20260507 17:01:19.028 | INFO | "gpu" : " 17 GB", | 20260507 17:01:19.028 | INFO | "huge" : "", | 20260507 17:01:19.028 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:19.028 | INFO | "malloc" : " 14 GB", | 20260507 17:01:19.028 | INFO | "pinned" : " 29 GB" | 20260507 17:01:19.028 | INFO | } | 20260507 17:01:19.028 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:01:19.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:19.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:19.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:19.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:19.028 | INFO | ANALYSIS : 69.7%, 2.9, 107.5ms, 0 | 20260507 17:01:19.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:19.028 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260507 17:01:19.028 | INFO | PRE_HQ : 67.3%, 1.8, 62.7ms, 0 | 20260507 17:01:19.028 | INFO | BAZWRITER : 70.0%, 4.0, 144.5ms, 0 | 20260507 17:01:19.028 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:01:19.029 | INFO | Uploaded 36GiB to GPU in 2.55246s (14.104GiB/s) | 20260507 17:01:19.029 | INFO | Downloaded 38GiB to GPU in 2.69767s (14.0862GiB/s) | 20260507 17:01:24.167 | INFO | Analyzing chunk frames = [1131008,1131520) | 20260507 17:01:24.167 | INFO | MemInfo: { | 20260507 17:01:24.167 | INFO | "gpu" : " 17 GB", | 20260507 17:01:24.167 | INFO | "huge" : "", | 20260507 17:01:24.167 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:24.167 | INFO | "malloc" : " 14 GB", | 20260507 17:01:24.167 | INFO | "pinned" : " 29 GB" | 20260507 17:01:24.168 | INFO | } | 20260507 17:01:24.168 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:01:24.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:24.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 17:01:24.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:01:24.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:24.168 | INFO | Uploaded 36GiB to GPU in 2.52061s (14.2823GiB/s) | 20260507 17:01:24.168 | INFO | Downloaded 34GiB to GPU in 2.42195s (14.0383GiB/s) | 20260507 17:01:29.307 | INFO | Analyzing chunk frames = [1131520,1132032) | 20260507 17:01:29.307 | INFO | MemInfo: { | 20260507 17:01:29.307 | INFO | "gpu" : " 17 GB", | 20260507 17:01:29.307 | INFO | "huge" : "", | 20260507 17:01:29.307 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:29.307 | INFO | "malloc" : " 14 GB", | 20260507 17:01:29.307 | INFO | "pinned" : " 29 GB" | 20260507 17:01:29.307 | INFO | } | 20260507 17:01:29.307 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:01:29.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:29.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 17:01:29.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:01:29.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:29.307 | INFO | Uploaded 36GiB to GPU in 2.48478s (14.4882GiB/s) | 20260507 17:01:29.308 | INFO | Downloaded 34GiB to GPU in 2.33793s (14.5428GiB/s) | 20260507 17:01:34.446 | INFO | Analyzing chunk frames = [1132032,1132544) | 20260507 17:01:34.446 | INFO | MemInfo: { | 20260507 17:01:34.446 | INFO | "gpu" : " 17 GB", | 20260507 17:01:34.446 | INFO | "huge" : "", | 20260507 17:01:34.446 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:34.446 | INFO | "malloc" : " 14 GB", | 20260507 17:01:34.446 | INFO | "pinned" : " 29 GB" | 20260507 17:01:34.446 | INFO | } | 20260507 17:01:34.446 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:01:34.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:34.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.447 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 17:01:34.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.447 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:01:34.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:34.447 | INFO | Uploaded 36GiB to GPU in 2.51077s (14.3382GiB/s) | 20260507 17:01:34.447 | INFO | Downloaded 34GiB to GPU in 2.35055s (14.4647GiB/s) | 20260507 17:01:39.586 | INFO | Analyzing chunk frames = [1132544,1133056) | 20260507 17:01:39.586 | INFO | MemInfo: { | 20260507 17:01:39.586 | INFO | "gpu" : " 17 GB", | 20260507 17:01:39.586 | INFO | "huge" : "", | 20260507 17:01:39.586 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:39.586 | INFO | "malloc" : " 14 GB", | 20260507 17:01:39.586 | INFO | "pinned" : " 29 GB" | 20260507 17:01:39.586 | INFO | } | 20260507 17:01:39.586 | INFO | Perf report for the last 5.14003 seconds: | 20260507 17:01:39.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:39.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.586 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 17:01:39.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.587 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:01:39.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:39.587 | INFO | Uploaded 36GiB to GPU in 2.38077s (15.1212GiB/s) | 20260507 17:01:39.587 | INFO | Downloaded 34GiB to GPU in 2.45702s (13.8379GiB/s) | 20260507 17:01:44.725 | INFO | Analyzing chunk frames = [1133056,1133568) | 20260507 17:01:44.725 | INFO | MemInfo: { | 20260507 17:01:44.725 | INFO | "gpu" : " 17 GB", | 20260507 17:01:44.725 | INFO | "huge" : "", | 20260507 17:01:44.725 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:44.725 | INFO | "malloc" : " 14 GB", | 20260507 17:01:44.725 | INFO | "pinned" : " 29 GB" | 20260507 17:01:44.725 | INFO | } | 20260507 17:01:44.725 | INFO | Perf report for the last 5.13856 seconds: | 20260507 17:01:44.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:44.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:01:44.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:01:44.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:44.725 | INFO | Uploaded 36GiB to GPU in 2.34363s (15.3608GiB/s) | 20260507 17:01:44.725 | INFO | Downloaded 34GiB to GPU in 2.45258s (13.863GiB/s) | 20260507 17:01:49.864 | INFO | Analyzing chunk frames = [1133568,1134080) | 20260507 17:01:49.864 | INFO | MemInfo: { | 20260507 17:01:49.864 | INFO | "gpu" : " 17 GB", | 20260507 17:01:49.864 | INFO | "huge" : "", | 20260507 17:01:49.864 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:49.864 | INFO | "malloc" : " 14 GB", | 20260507 17:01:49.864 | INFO | "pinned" : " 29 GB" | 20260507 17:01:49.864 | INFO | } | 20260507 17:01:49.864 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:01:49.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:49.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.864 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:01:49.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.865 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:01:49.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:49.865 | INFO | Uploaded 36GiB to GPU in 2.43113s (14.8079GiB/s) | 20260507 17:01:49.865 | INFO | Downloaded 34GiB to GPU in 2.41684s (14.068GiB/s) | 20260507 17:01:55.004 | INFO | Analyzing chunk frames = [1134080,1134592) | 20260507 17:01:55.004 | INFO | MemInfo: { | 20260507 17:01:55.004 | INFO | "gpu" : " 17 GB", | 20260507 17:01:55.004 | INFO | "huge" : "", | 20260507 17:01:55.004 | INFO | "huge_shared" : " 29 GB", | 20260507 17:01:55.004 | INFO | "malloc" : " 14 GB", | 20260507 17:01:55.004 | INFO | "pinned" : " 29 GB" | 20260507 17:01:55.004 | INFO | } | 20260507 17:01:55.004 | INFO | Perf report for the last 5.13968 seconds: | 20260507 17:01:55.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:01:55.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:01:55.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:01:55.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:01:55.004 | INFO | Uploaded 36GiB to GPU in 2.48347s (14.4958GiB/s) | 20260507 17:01:55.005 | INFO | Downloaded 34GiB to GPU in 2.50831s (13.5549GiB/s) | 20260507 17:01:55.101 | INFO | Starting a round of Baz IO | 20260507 17:01:58.615 | INFO | Populating RTMetrics report | 20260507 17:01:58.616 | INFO | RTMetrics report populated | 20260507 17:01:58.616 | INFO | Opening tmp RTMetrics json file | 20260507 17:01:58.627 | INFO | Serializing RTMetrics to json | 20260507 17:01:58.694 | INFO | Serialization done, swapping tmp file now... | 20260507 17:01:58.720 | INFO | RTMetrics done | 20260507 17:01:58.755 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.654 seconds | 20260507 17:01:58.755 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.95% / 0.02% / 51.03% | 20260507 17:01:58.755 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.94955% / 5.16574e-05% / 1.58946e-05% | 20260507 17:01:58.755 | INFO | Baz IO overall: | 20260507 17:01:58.755 | INFO | BytesWritten : 494530259540 | 20260507 17:01:58.755 | INFO | NumEvents : 239488711815 | 20260507 17:02:00.143 | INFO | Analyzing chunk frames = [1134592,1135104) | 20260507 17:02:00.143 | INFO | MemInfo: { | 20260507 17:02:00.143 | INFO | "gpu" : " 17 GB", | 20260507 17:02:00.143 | INFO | "huge" : "", | 20260507 17:02:00.143 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:00.143 | INFO | "malloc" : " 14 GB", | 20260507 17:02:00.143 | INFO | "pinned" : " 29 GB" | 20260507 17:02:00.143 | INFO | } | 20260507 17:02:00.143 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:02:00.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:00.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:00.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:00.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:00.143 | INFO | ANALYSIS : 69.5%, 2.9, 105.8ms, 0 | 20260507 17:02:00.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:00.143 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 17:02:00.143 | INFO | PRE_HQ : 68.2%, 1.8, 63.3ms, 0 | 20260507 17:02:00.143 | INFO | BAZWRITER : 70.7%, 3.7, 135.7ms, 0 | 20260507 17:02:00.143 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 17:02:00.144 | INFO | Uploaded 36GiB to GPU in 2.54843s (14.1263GiB/s) | 20260507 17:02:00.144 | INFO | Downloaded 38GiB to GPU in 2.7148s (13.9974GiB/s) | 20260507 17:02:05.283 | INFO | Analyzing chunk frames = [1135104,1135616) | 20260507 17:02:05.283 | INFO | MemInfo: { | 20260507 17:02:05.283 | INFO | "gpu" : " 17 GB", | 20260507 17:02:05.283 | INFO | "huge" : "", | 20260507 17:02:05.283 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:05.283 | INFO | "malloc" : " 14 GB", | 20260507 17:02:05.283 | INFO | "pinned" : " 29 GB" | 20260507 17:02:05.283 | INFO | } | 20260507 17:02:05.283 | INFO | Perf report for the last 5.13969 seconds: | 20260507 17:02:05.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:05.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:02:05.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:02:05.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:05.283 | INFO | Uploaded 36GiB to GPU in 2.4296s (14.8172GiB/s) | 20260507 17:02:05.283 | INFO | Downloaded 34GiB to GPU in 2.45927s (13.8253GiB/s) | 20260507 17:02:10.422 | INFO | Analyzing chunk frames = [1135616,1136128) | 20260507 17:02:10.422 | INFO | MemInfo: { | 20260507 17:02:10.422 | INFO | "gpu" : " 17 GB", | 20260507 17:02:10.422 | INFO | "huge" : "", | 20260507 17:02:10.422 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:10.422 | INFO | "malloc" : " 14 GB", | 20260507 17:02:10.422 | INFO | "pinned" : " 29 GB" | 20260507 17:02:10.422 | INFO | } | 20260507 17:02:10.422 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:02:10.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:10.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.422 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:02:10.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.422 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 17:02:10.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:10.423 | INFO | Uploaded 36GiB to GPU in 2.50707s (14.3594GiB/s) | 20260507 17:02:10.423 | INFO | Downloaded 34GiB to GPU in 2.45007s (13.8771GiB/s) | 20260507 17:02:15.561 | INFO | Analyzing chunk frames = [1136128,1136640) | 20260507 17:02:15.561 | INFO | MemInfo: { | 20260507 17:02:15.561 | INFO | "gpu" : " 17 GB", | 20260507 17:02:15.561 | INFO | "huge" : "", | 20260507 17:02:15.561 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:15.561 | INFO | "malloc" : " 14 GB", | 20260507 17:02:15.561 | INFO | "pinned" : " 29 GB" | 20260507 17:02:15.561 | INFO | } | 20260507 17:02:15.561 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:02:15.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:15.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.561 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 17:02:15.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.561 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:02:15.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:15.562 | INFO | Uploaded 36GiB to GPU in 2.44435s (14.7278GiB/s) | 20260507 17:02:15.562 | INFO | Downloaded 34GiB to GPU in 2.35601s (14.4312GiB/s) | 20260507 17:02:20.700 | INFO | Analyzing chunk frames = [1136640,1137152) | 20260507 17:02:20.700 | INFO | MemInfo: { | 20260507 17:02:20.700 | INFO | "gpu" : " 17 GB", | 20260507 17:02:20.700 | INFO | "huge" : "", | 20260507 17:02:20.700 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:20.700 | INFO | "malloc" : " 14 GB", | 20260507 17:02:20.701 | INFO | "pinned" : " 29 GB" | 20260507 17:02:20.701 | INFO | } | 20260507 17:02:20.701 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:02:20.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:20.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:02:20.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:02:20.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:20.701 | INFO | Uploaded 36GiB to GPU in 2.36167s (15.2435GiB/s) | 20260507 17:02:20.701 | INFO | Downloaded 34GiB to GPU in 2.48174s (13.7001GiB/s) | 20260507 17:02:25.840 | INFO | Analyzing chunk frames = [1137152,1137664) | 20260507 17:02:25.840 | INFO | MemInfo: { | 20260507 17:02:25.840 | INFO | "gpu" : " 17 GB", | 20260507 17:02:25.840 | INFO | "huge" : "", | 20260507 17:02:25.840 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:25.840 | INFO | "malloc" : " 14 GB", | 20260507 17:02:25.840 | INFO | "pinned" : " 29 GB" | 20260507 17:02:25.840 | INFO | } | 20260507 17:02:25.840 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:02:25.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:25.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.840 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:02:25.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.840 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:02:25.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:25.841 | INFO | Uploaded 36GiB to GPU in 2.38584s (15.089GiB/s) | 20260507 17:02:25.841 | INFO | Downloaded 34GiB to GPU in 2.49004s (13.6544GiB/s) | 20260507 17:02:30.979 | INFO | Analyzing chunk frames = [1137664,1138176) | 20260507 17:02:30.980 | INFO | MemInfo: { | 20260507 17:02:30.980 | INFO | "gpu" : " 17 GB", | 20260507 17:02:30.980 | INFO | "huge" : "", | 20260507 17:02:30.980 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:30.980 | INFO | "malloc" : " 14 GB", | 20260507 17:02:30.980 | INFO | "pinned" : " 29 GB" | 20260507 17:02:30.980 | INFO | } | 20260507 17:02:30.980 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:02:30.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:30.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:02:30.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:02:30.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:30.980 | INFO | Uploaded 36GiB to GPU in 2.44732s (14.71GiB/s) | 20260507 17:02:30.980 | INFO | Downloaded 34GiB to GPU in 2.42015s (14.0487GiB/s) | 20260507 17:02:36.119 | INFO | Analyzing chunk frames = [1138176,1138688) | 20260507 17:02:36.119 | INFO | MemInfo: { | 20260507 17:02:36.119 | INFO | "gpu" : " 17 GB", | 20260507 17:02:36.119 | INFO | "huge" : "", | 20260507 17:02:36.119 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:36.119 | INFO | "malloc" : " 14 GB", | 20260507 17:02:36.119 | INFO | "pinned" : " 29 GB" | 20260507 17:02:36.119 | INFO | } | 20260507 17:02:36.119 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:02:36.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:36.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:02:36.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:02:36.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:36.119 | INFO | Uploaded 36GiB to GPU in 2.51855s (14.294GiB/s) | 20260507 17:02:36.119 | INFO | Downloaded 34GiB to GPU in 2.52257s (13.4783GiB/s) | 20260507 17:02:36.119 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:02:36.119 | INFO | Total mem usage: 16.4 GiB | 20260507 17:02:36.119 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:02:36.119 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10%) | 20260507 17:02:36.119 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:02:36.119 | INFO | High Water Breakdown: | 20260507 17:02:36.119 | INFO | 59.5% : 8.87 GiB ( 89.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:02:36.119 | INFO | 40.4% : 6.02 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:02:36.119 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:02:36.119 | INFO | ------------------------------------------ | 20260507 17:02:36.119 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:02:36.119 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:02:36.119 | INFO | Total mem usage: 26.5 GiB | 20260507 17:02:36.119 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:02:36.120 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:02:36.120 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:02:36.120 | INFO | High Water Breakdown: | 20260507 17:02:36.120 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:02:36.120 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:02:36.120 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:02:36.120 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:02:36.120 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:02:36.120 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:02:36.120 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:02:36.120 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:02:36.120 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:02:36.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:02:36.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:02:36.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:02:36.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:02:36.120 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:02:36.120 | INFO | ------------------------------------------ | 20260507 17:02:36.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:02:36.120 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:02:36.120 | INFO | Total mem usage: 27.8 GiB | 20260507 17:02:36.120 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:02:36.120 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:02:36.120 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:02:36.120 | INFO | High Water Breakdown: | 20260507 17:02:36.120 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:02:36.120 | INFO | ------------------------------------------ | 20260507 17:02:36.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:02:36.120 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:02:36.120 | INFO | Total mem usage: 15.7 GiB | 20260507 17:02:36.120 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:02:36.120 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:02:36.120 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:02:36.120 | INFO | High Water Breakdown: | 20260507 17:02:36.120 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:02:36.120 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:02:36.120 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:02:36.120 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:02:36.120 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:02:36.120 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:02:36.120 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:02:36.120 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:02:36.120 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:02:36.120 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:02:36.120 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:02:36.120 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:02:36.120 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:02:36.120 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:02:36.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:02:36.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:02:36.120 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:02:36.120 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:02:36.120 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:02:36.120 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:02:36.120 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:02:36.120 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:02:36.120 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:02:36.120 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:02:36.120 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:02:36.120 | INFO | ------------------------------------------ | 20260507 17:02:36.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:02:36.120 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:02:36.120 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:02:36.120 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:02:36.120 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.64e+01, 1.43e+01 / 2.68e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.01e-01, 1.07e-02 / 5.88e-03 | 20260507 17:02:36.120 | INFO | PerfReport: 12.06%, Baseline -- 4.05e+00 / 4.50e+00, 6.08e-01 / 2.39e-01 | 20260507 17:02:36.120 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.11e-01 / 1.19e-01 | 20260507 17:02:36.120 | INFO | PerfReport: 15.69%, DME -- 4.21e+01 / 4.51e+01, 4.64e+00 / 2.54e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.86e+00 / 2.03e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.53e+00 / 2.89e+00, 4.36e-01 / 2.53e-01 | 20260507 17:02:36.120 | INFO | PerfReport: 3.82%, Metrics -- 1.28e+00 / 1.44e+00, 3.28e-01 / 1.84e-01 | 20260507 17:02:36.120 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.25e-01, 6.72e-02 / 4.09e-02 | 20260507 17:02:36.120 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:02:36.120 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:02:36.120 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:02:36.120 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 23.90%, Upload -- 2.56e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 24.01%, Download -- 2.57e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 31.30%, Compute -- 3.36e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 1.77%, LockUpload -- 1.90e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 2.32%, LockDownload -- 2.48e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 16.69%, LockCompute -- 1.79e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 17:02:36.120 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.88e-02 / 0.00e+00 | 20260507 17:02:36.216 | INFO | Starting a round of Baz IO | 20260507 17:02:39.726 | INFO | Populating RTMetrics report | 20260507 17:02:39.726 | INFO | RTMetrics report populated | 20260507 17:02:39.726 | INFO | Opening tmp RTMetrics json file | 20260507 17:02:39.809 | INFO | Serializing RTMetrics to json | 20260507 17:02:39.839 | INFO | Serialization done, swapping tmp file now... | 20260507 17:02:39.840 | INFO | RTMetrics done | 20260507 17:02:39.860 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.644 seconds | 20260507 17:02:39.860 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.93% / 0.02% / 51.05% | 20260507 17:02:39.860 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.953% / 6.35783e-05% / 1.19209e-05% | 20260507 17:02:39.860 | INFO | Baz IO overall: | 20260507 17:02:39.860 | INFO | BytesWritten : 496458929980 | 20260507 17:02:39.860 | INFO | NumEvents : 240426945516 | 20260507 17:02:41.258 | INFO | Analyzing chunk frames = [1138688,1139200) | 20260507 17:02:41.258 | INFO | MemInfo: { | 20260507 17:02:41.258 | INFO | "gpu" : " 17 GB", | 20260507 17:02:41.258 | INFO | "huge" : "", | 20260507 17:02:41.258 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:41.258 | INFO | "malloc" : " 14 GB", | 20260507 17:02:41.258 | INFO | "pinned" : " 29 GB" | 20260507 17:02:41.258 | INFO | } | 20260507 17:02:41.258 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:02:41.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:41.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:41.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:41.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:41.259 | INFO | ANALYSIS : 69.7%, 3.0, 108.4ms, 0 | 20260507 17:02:41.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:41.259 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 17:02:41.259 | INFO | PRE_HQ : 66.6%, 1.7, 59.5ms, 0 | 20260507 17:02:41.259 | INFO | BAZWRITER : 70.5%, 3.9, 139.9ms, 0 | 20260507 17:02:41.259 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 17:02:41.259 | INFO | Uploaded 36GiB to GPU in 2.52513s (14.2567GiB/s) | 20260507 17:02:41.259 | INFO | Downloaded 38GiB to GPU in 2.72342s (13.953GiB/s) | 20260507 17:02:46.398 | INFO | Analyzing chunk frames = [1139200,1139712) | 20260507 17:02:46.398 | INFO | MemInfo: { | 20260507 17:02:46.398 | INFO | "gpu" : " 17 GB", | 20260507 17:02:46.398 | INFO | "huge" : "", | 20260507 17:02:46.398 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:46.398 | INFO | "malloc" : " 14 GB", | 20260507 17:02:46.398 | INFO | "pinned" : " 29 GB" | 20260507 17:02:46.398 | INFO | } | 20260507 17:02:46.398 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:02:46.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:46.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 17:02:46.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:02:46.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:46.398 | INFO | Uploaded 36GiB to GPU in 2.5357s (14.1973GiB/s) | 20260507 17:02:46.398 | INFO | Downloaded 34GiB to GPU in 2.424s (14.0264GiB/s) | 20260507 17:02:51.537 | INFO | Analyzing chunk frames = [1139712,1140224) | 20260507 17:02:51.537 | INFO | MemInfo: { | 20260507 17:02:51.537 | INFO | "gpu" : " 17 GB", | 20260507 17:02:51.537 | INFO | "huge" : "", | 20260507 17:02:51.537 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:51.537 | INFO | "malloc" : " 14 GB", | 20260507 17:02:51.537 | INFO | "pinned" : " 29 GB" | 20260507 17:02:51.537 | INFO | } | 20260507 17:02:51.537 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:02:51.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:51.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 17:02:51.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:02:51.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:51.537 | INFO | Uploaded 36GiB to GPU in 2.48806s (14.4691GiB/s) | 20260507 17:02:51.537 | INFO | Downloaded 34GiB to GPU in 2.33172s (14.5815GiB/s) | 20260507 17:02:56.676 | INFO | Analyzing chunk frames = [1140224,1140736) | 20260507 17:02:56.676 | INFO | MemInfo: { | 20260507 17:02:56.676 | INFO | "gpu" : " 17 GB", | 20260507 17:02:56.676 | INFO | "huge" : "", | 20260507 17:02:56.676 | INFO | "huge_shared" : " 29 GB", | 20260507 17:02:56.676 | INFO | "malloc" : " 14 GB", | 20260507 17:02:56.676 | INFO | "pinned" : " 29 GB" | 20260507 17:02:56.676 | INFO | } | 20260507 17:02:56.676 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:02:56.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:02:56.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.676 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260507 17:02:56.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.676 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:02:56.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:02:56.677 | INFO | Uploaded 36GiB to GPU in 2.50025s (14.3985GiB/s) | 20260507 17:02:56.677 | INFO | Downloaded 34GiB to GPU in 2.34642s (14.4902GiB/s) | 20260507 17:03:01.816 | INFO | Analyzing chunk frames = [1140736,1141248) | 20260507 17:03:01.816 | INFO | MemInfo: { | 20260507 17:03:01.816 | INFO | "gpu" : " 17 GB", | 20260507 17:03:01.816 | INFO | "huge" : "", | 20260507 17:03:01.816 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:01.816 | INFO | "malloc" : " 14 GB", | 20260507 17:03:01.816 | INFO | "pinned" : " 29 GB" | 20260507 17:03:01.816 | INFO | } | 20260507 17:03:01.816 | INFO | Perf report for the last 5.13975 seconds: | 20260507 17:03:01.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:01.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:03:01.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 17:03:01.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:01.816 | INFO | Uploaded 36GiB to GPU in 2.37809s (15.1382GiB/s) | 20260507 17:03:01.817 | INFO | Downloaded 34GiB to GPU in 2.47123s (13.7583GiB/s) | 20260507 17:03:06.955 | INFO | Analyzing chunk frames = [1141248,1141760) | 20260507 17:03:06.955 | INFO | MemInfo: { | 20260507 17:03:06.955 | INFO | "gpu" : " 17 GB", | 20260507 17:03:06.955 | INFO | "huge" : "", | 20260507 17:03:06.955 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:06.955 | INFO | "malloc" : " 14 GB", | 20260507 17:03:06.955 | INFO | "pinned" : " 29 GB" | 20260507 17:03:06.955 | INFO | } | 20260507 17:03:06.955 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:03:06.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:06.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.955 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:03:06.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.955 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:03:06.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:06.956 | INFO | Uploaded 36GiB to GPU in 2.37285s (15.1716GiB/s) | 20260507 17:03:06.956 | INFO | Downloaded 34GiB to GPU in 2.46049s (13.8184GiB/s) | 20260507 17:03:12.094 | INFO | Analyzing chunk frames = [1141760,1142272) | 20260507 17:03:12.094 | INFO | MemInfo: { | 20260507 17:03:12.094 | INFO | "gpu" : " 17 GB", | 20260507 17:03:12.094 | INFO | "huge" : "", | 20260507 17:03:12.094 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:12.094 | INFO | "malloc" : " 14 GB", | 20260507 17:03:12.094 | INFO | "pinned" : " 29 GB" | 20260507 17:03:12.094 | INFO | } | 20260507 17:03:12.094 | INFO | Perf report for the last 5.13909 seconds: | 20260507 17:03:12.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:12.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.094 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:03:12.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.094 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:03:12.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:12.095 | INFO | Uploaded 36GiB to GPU in 2.40417s (14.974GiB/s) | 20260507 17:03:12.095 | INFO | Downloaded 34GiB to GPU in 2.41439s (14.0822GiB/s) | 20260507 17:03:17.233 | INFO | Analyzing chunk frames = [1142272,1142784) | 20260507 17:03:17.233 | INFO | MemInfo: { | 20260507 17:03:17.233 | INFO | "gpu" : " 17 GB", | 20260507 17:03:17.233 | INFO | "huge" : "", | 20260507 17:03:17.233 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:17.234 | INFO | "malloc" : " 14 GB", | 20260507 17:03:17.234 | INFO | "pinned" : " 29 GB" | 20260507 17:03:17.234 | INFO | } | 20260507 17:03:17.234 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:03:17.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:17.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 17:03:17.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:03:17.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:17.234 | INFO | Uploaded 36GiB to GPU in 2.48909s (14.4631GiB/s) | 20260507 17:03:17.234 | INFO | Downloaded 34GiB to GPU in 2.50159s (13.5914GiB/s) | 20260507 17:03:17.330 | INFO | Starting a round of Baz IO | 20260507 17:03:20.839 | INFO | Populating RTMetrics report | 20260507 17:03:20.839 | INFO | RTMetrics report populated | 20260507 17:03:20.839 | INFO | Opening tmp RTMetrics json file | 20260507 17:03:20.851 | INFO | Serializing RTMetrics to json | 20260507 17:03:20.915 | INFO | Serialization done, swapping tmp file now... | 20260507 17:03:20.954 | INFO | RTMetrics done | 20260507 17:03:20.977 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.646 seconds | 20260507 17:03:20.977 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.90% / 0.02% / 51.07% | 20260507 17:03:20.977 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.93726% / 3.57628e-05% / 2.38419e-05% | 20260507 17:03:20.977 | INFO | Baz IO overall: | 20260507 17:03:20.977 | INFO | BytesWritten : 498387442178 | 20260507 17:03:20.977 | INFO | NumEvents : 241364604366 | 20260507 17:03:22.373 | INFO | Analyzing chunk frames = [1142784,1143296) | 20260507 17:03:22.373 | INFO | MemInfo: { | 20260507 17:03:22.373 | INFO | "gpu" : " 17 GB", | 20260507 17:03:22.373 | INFO | "huge" : "", | 20260507 17:03:22.373 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:22.373 | INFO | "malloc" : " 14 GB", | 20260507 17:03:22.373 | INFO | "pinned" : " 29 GB" | 20260507 17:03:22.373 | INFO | } | 20260507 17:03:22.373 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:03:22.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:22.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:22.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:22.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:22.373 | INFO | ANALYSIS : 69.7%, 2.9, 107.7ms, 0 | 20260507 17:03:22.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:22.373 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 17:03:22.373 | INFO | PRE_HQ : 66.8%, 1.7, 59.8ms, 0 | 20260507 17:03:22.373 | INFO | BAZWRITER : 70.4%, 3.9, 142.1ms, 0 | 20260507 17:03:22.373 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 17:03:22.374 | INFO | Uploaded 36GiB to GPU in 2.52943s (14.2325GiB/s) | 20260507 17:03:22.374 | INFO | Downloaded 38GiB to GPU in 2.73071s (13.9158GiB/s) | 20260507 17:03:27.512 | INFO | Analyzing chunk frames = [1143296,1143808) | 20260507 17:03:27.512 | INFO | MemInfo: { | 20260507 17:03:27.512 | INFO | "gpu" : " 17 GB", | 20260507 17:03:27.512 | INFO | "huge" : "", | 20260507 17:03:27.512 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:27.512 | INFO | "malloc" : " 14 GB", | 20260507 17:03:27.512 | INFO | "pinned" : " 29 GB" | 20260507 17:03:27.512 | INFO | } | 20260507 17:03:27.512 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:03:27.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:27.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.512 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:03:27.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.513 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:03:27.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:27.513 | INFO | Uploaded 36GiB to GPU in 2.4902s (14.4567GiB/s) | 20260507 17:03:27.513 | INFO | Downloaded 34GiB to GPU in 2.42229s (14.0363GiB/s) | 20260507 17:03:32.652 | INFO | Analyzing chunk frames = [1143808,1144320) | 20260507 17:03:32.652 | INFO | MemInfo: { | 20260507 17:03:32.652 | INFO | "gpu" : " 17 GB", | 20260507 17:03:32.652 | INFO | "huge" : "", | 20260507 17:03:32.652 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:32.652 | INFO | "malloc" : " 14 GB", | 20260507 17:03:32.652 | INFO | "pinned" : " 29 GB" | 20260507 17:03:32.652 | INFO | } | 20260507 17:03:32.652 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:03:32.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:32.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:03:32.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:03:32.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:32.652 | INFO | Uploaded 36GiB to GPU in 2.56431s (14.0389GiB/s) | 20260507 17:03:32.652 | INFO | Downloaded 34GiB to GPU in 2.42281s (14.0333GiB/s) | 20260507 17:03:37.791 | INFO | Analyzing chunk frames = [1144320,1144832) | 20260507 17:03:37.791 | INFO | MemInfo: { | 20260507 17:03:37.791 | INFO | "gpu" : " 17 GB", | 20260507 17:03:37.791 | INFO | "huge" : "", | 20260507 17:03:37.791 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:37.791 | INFO | "malloc" : " 14 GB", | 20260507 17:03:37.791 | INFO | "pinned" : " 29 GB" | 20260507 17:03:37.791 | INFO | } | 20260507 17:03:37.791 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:03:37.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:37.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.791 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:03:37.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.791 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:03:37.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:37.792 | INFO | Uploaded 36GiB to GPU in 2.47473s (14.5471GiB/s) | 20260507 17:03:37.792 | INFO | Downloaded 34GiB to GPU in 2.34715s (14.4857GiB/s) | 20260507 17:03:42.931 | INFO | Analyzing chunk frames = [1144832,1145344) | 20260507 17:03:42.931 | INFO | MemInfo: { | 20260507 17:03:42.931 | INFO | "gpu" : " 17 GB", | 20260507 17:03:42.931 | INFO | "huge" : "", | 20260507 17:03:42.931 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:42.931 | INFO | "malloc" : " 14 GB", | 20260507 17:03:42.931 | INFO | "pinned" : " 29 GB" | 20260507 17:03:42.931 | INFO | } | 20260507 17:03:42.931 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:03:42.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:42.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 17:03:42.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 17:03:42.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:42.931 | INFO | Uploaded 36GiB to GPU in 2.41534s (14.9048GiB/s) | 20260507 17:03:42.931 | INFO | Downloaded 34GiB to GPU in 2.47621s (13.7307GiB/s) | 20260507 17:03:48.070 | INFO | Analyzing chunk frames = [1145344,1145856) | 20260507 17:03:48.070 | INFO | MemInfo: { | 20260507 17:03:48.070 | INFO | "gpu" : " 17 GB", | 20260507 17:03:48.070 | INFO | "huge" : "", | 20260507 17:03:48.070 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:48.070 | INFO | "malloc" : " 14 GB", | 20260507 17:03:48.070 | INFO | "pinned" : " 29 GB" | 20260507 17:03:48.070 | INFO | } | 20260507 17:03:48.070 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:03:48.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:48.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.070 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:03:48.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.070 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 17:03:48.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:48.071 | INFO | Uploaded 36GiB to GPU in 2.41526s (14.9052GiB/s) | 20260507 17:03:48.071 | INFO | Downloaded 34GiB to GPU in 2.44064s (13.9308GiB/s) | 20260507 17:03:53.209 | INFO | Analyzing chunk frames = [1145856,1146368) | 20260507 17:03:53.209 | INFO | MemInfo: { | 20260507 17:03:53.209 | INFO | "gpu" : " 17 GB", | 20260507 17:03:53.209 | INFO | "huge" : "", | 20260507 17:03:53.209 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:53.209 | INFO | "malloc" : " 14 GB", | 20260507 17:03:53.209 | INFO | "pinned" : " 29 GB" | 20260507 17:03:53.209 | INFO | } | 20260507 17:03:53.210 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:03:53.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:53.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:03:53.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:03:53.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:53.210 | INFO | Uploaded 36GiB to GPU in 2.44499s (14.724GiB/s) | 20260507 17:03:53.210 | INFO | Downloaded 34GiB to GPU in 2.40783s (14.1206GiB/s) | 20260507 17:03:58.349 | INFO | Analyzing chunk frames = [1146368,1146880) | 20260507 17:03:58.349 | INFO | MemInfo: { | 20260507 17:03:58.349 | INFO | "gpu" : " 17 GB", | 20260507 17:03:58.349 | INFO | "huge" : "", | 20260507 17:03:58.349 | INFO | "huge_shared" : " 29 GB", | 20260507 17:03:58.349 | INFO | "malloc" : " 14 GB", | 20260507 17:03:58.349 | INFO | "pinned" : " 29 GB" | 20260507 17:03:58.349 | INFO | } | 20260507 17:03:58.349 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:03:58.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:03:58.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:03:58.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:03:58.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:03:58.349 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:03:58.349 | INFO | Total mem usage: 16.4 GiB | 20260507 17:03:58.349 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:03:58.349 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.7%) | 20260507 17:03:58.349 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:03:58.349 | INFO | High Water Breakdown: | 20260507 17:03:58.349 | INFO | 59.3% : 8.7 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:03:58.349 | INFO | 40.6% : 5.96 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:03:58.349 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:03:58.349 | INFO | ------------------------------------------ | 20260507 17:03:58.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:03:58.349 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:03:58.349 | INFO | Total mem usage: 26.5 GiB | 20260507 17:03:58.349 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:03:58.349 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:03:58.349 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:03:58.349 | INFO | High Water Breakdown: | 20260507 17:03:58.349 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:03:58.349 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:03:58.349 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:03:58.349 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:03:58.349 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:03:58.349 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:03:58.349 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:03:58.349 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:03:58.349 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:03:58.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:03:58.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:03:58.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:03:58.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:03:58.349 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:03:58.349 | INFO | ------------------------------------------ | 20260507 17:03:58.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:03:58.349 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:03:58.349 | INFO | Total mem usage: 27.8 GiB | 20260507 17:03:58.349 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:03:58.349 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:03:58.349 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:03:58.349 | INFO | High Water Breakdown: | 20260507 17:03:58.349 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:03:58.349 | INFO | ------------------------------------------ | 20260507 17:03:58.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:03:58.349 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:03:58.349 | INFO | Total mem usage: 15.7 GiB | 20260507 17:03:58.349 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:03:58.349 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:03:58.349 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 17:03:58.349 | INFO | High Water Breakdown: | 20260507 17:03:58.349 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:03:58.349 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:03:58.349 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:03:58.349 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:03:58.349 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:03:58.349 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:03:58.349 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:03:58.349 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:03:58.349 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:03:58.349 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:03:58.349 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:03:58.349 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:03:58.349 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:03:58.349 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:03:58.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:03:58.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:03:58.349 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:03:58.349 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:03:58.349 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:03:58.349 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:03:58.349 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:03:58.349 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:03:58.349 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:03:58.349 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:03:58.349 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:03:58.349 | INFO | ------------------------------------------ | 20260507 17:03:58.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:03:58.349 | INFO | Uploaded 36GiB to GPU in 2.50785s (14.3549GiB/s) | 20260507 17:03:58.350 | INFO | Downloaded 34GiB to GPU in 2.48538s (13.68GiB/s) | 20260507 17:03:58.349 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:03:58.350 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:03:58.350 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:03:58.350 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.45e+01 / 2.92e+00 | 20260507 17:03:58.350 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.09e-02 / 6.72e-03 | 20260507 17:03:58.350 | INFO | PerfReport: 12.05%, Baseline -- 4.05e+00 / 4.49e+00, 6.18e-01 / 2.42e-01 | 20260507 17:03:58.350 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 9.88e-02 | 20260507 17:03:58.350 | INFO | PerfReport: 15.72%, DME -- 4.22e+01 / 4.52e+01, 4.55e+00 / 2.81e+00 | 20260507 17:03:58.350 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.90e+00 / 1.98e+00 | 20260507 17:03:58.350 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.52e+00 / 2.93e+00, 4.35e-01 / 2.15e-01 | 20260507 17:03:58.350 | INFO | PerfReport: 3.80%, Metrics -- 1.28e+00 / 1.46e+00, 3.27e-01 / 1.84e-01 | 20260507 17:03:58.350 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.31e-01, 6.84e-02 / 4.03e-02 | 20260507 17:03:58.350 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:03:58.350 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:03:58.350 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:03:58.350 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 17:03:58.350 | INFO | PerfReport: 23.95%, Upload -- 2.57e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 17:03:58.350 | INFO | PerfReport: 23.87%, Download -- 2.57e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 17:03:58.350 | INFO | PerfReport: 31.25%, Compute -- 3.36e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:03:58.351 | INFO | PerfReport: 1.83%, LockUpload -- 1.97e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 17:03:58.351 | INFO | PerfReport: 2.25%, LockDownload -- 2.42e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260507 17:03:58.351 | INFO | PerfReport: 16.84%, LockCompute -- 1.81e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:03:58.351 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 3.49e-02 / 0.00e+00 | 20260507 17:03:58.446 | INFO | Starting a round of Baz IO | 20260507 17:04:01.975 | INFO | Populating RTMetrics report | 20260507 17:04:01.975 | INFO | RTMetrics report populated | 20260507 17:04:01.975 | INFO | Opening tmp RTMetrics json file | 20260507 17:04:01.995 | INFO | Serializing RTMetrics to json | 20260507 17:04:01.999 | INFO | Serialization done, swapping tmp file now... | 20260507 17:04:02.000 | INFO | RTMetrics done | 20260507 17:04:02.078 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.631 seconds | 20260507 17:04:02.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.88% / 0.02% / 51.10% | 20260507 17:04:02.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.92702% / 3.57628e-05% / 3.97364e-05% | 20260507 17:04:02.078 | INFO | Baz IO overall: | 20260507 17:04:02.078 | INFO | BytesWritten : 500316647653 | 20260507 17:04:02.078 | INFO | NumEvents : 242302969193 | 20260507 17:04:03.488 | INFO | Analyzing chunk frames = [1146880,1147392) | 20260507 17:04:03.488 | INFO | MemInfo: { | 20260507 17:04:03.488 | INFO | "gpu" : " 17 GB", | 20260507 17:04:03.488 | INFO | "huge" : "", | 20260507 17:04:03.488 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:03.488 | INFO | "malloc" : " 14 GB", | 20260507 17:04:03.488 | INFO | "pinned" : " 29 GB" | 20260507 17:04:03.488 | INFO | } | 20260507 17:04:03.488 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:04:03.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:03.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:03.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:03.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:03.488 | INFO | ANALYSIS : 69.9%, 2.9, 108.1ms, 0 | 20260507 17:04:03.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:03.488 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 17:04:03.488 | INFO | PRE_HQ : 66.4%, 1.7, 59.8ms, 0 | 20260507 17:04:03.488 | INFO | BAZWRITER : 70.2%, 3.8, 138.7ms, 0 | 20260507 17:04:03.488 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:04:03.488 | INFO | Uploaded 36GiB to GPU in 2.58458s (13.9288GiB/s) | 20260507 17:04:03.489 | INFO | Downloaded 38GiB to GPU in 2.71004s (14.022GiB/s) | 20260507 17:04:08.627 | INFO | Analyzing chunk frames = [1147392,1147904) | 20260507 17:04:08.627 | INFO | MemInfo: { | 20260507 17:04:08.627 | INFO | "gpu" : " 17 GB", | 20260507 17:04:08.627 | INFO | "huge" : "", | 20260507 17:04:08.627 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:08.627 | INFO | "malloc" : " 14 GB", | 20260507 17:04:08.627 | INFO | "pinned" : " 29 GB" | 20260507 17:04:08.627 | INFO | } | 20260507 17:04:08.627 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:04:08.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:08.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.628 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:04:08.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.628 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:04:08.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:08.628 | INFO | Uploaded 36GiB to GPU in 2.52113s (14.2793GiB/s) | 20260507 17:04:08.628 | INFO | Downloaded 34GiB to GPU in 2.42371s (14.0281GiB/s) | 20260507 17:04:13.767 | INFO | Analyzing chunk frames = [1147904,1148416) | 20260507 17:04:13.767 | INFO | MemInfo: { | 20260507 17:04:13.767 | INFO | "gpu" : " 17 GB", | 20260507 17:04:13.767 | INFO | "huge" : "", | 20260507 17:04:13.767 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:13.767 | INFO | "malloc" : " 14 GB", | 20260507 17:04:13.767 | INFO | "pinned" : " 29 GB" | 20260507 17:04:13.767 | INFO | } | 20260507 17:04:13.767 | INFO | Perf report for the last 5.13957 seconds: | 20260507 17:04:13.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:13.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.767 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260507 17:04:13.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.767 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:04:13.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:13.768 | INFO | Uploaded 36GiB to GPU in 2.4623s (14.6205GiB/s) | 20260507 17:04:13.768 | INFO | Downloaded 34GiB to GPU in 2.32414s (14.6291GiB/s) | 20260507 17:04:18.906 | INFO | Analyzing chunk frames = [1148416,1148928) | 20260507 17:04:18.906 | INFO | MemInfo: { | 20260507 17:04:18.906 | INFO | "gpu" : " 17 GB", | 20260507 17:04:18.906 | INFO | "huge" : "", | 20260507 17:04:18.906 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:18.906 | INFO | "malloc" : " 14 GB", | 20260507 17:04:18.907 | INFO | "pinned" : " 29 GB" | 20260507 17:04:18.907 | INFO | } | 20260507 17:04:18.907 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:04:18.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:18.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 17:04:18.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:04:18.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:18.907 | INFO | Uploaded 36GiB to GPU in 2.48474s (14.4884GiB/s) | 20260507 17:04:18.907 | INFO | Downloaded 34GiB to GPU in 2.34946s (14.4714GiB/s) | 20260507 17:04:24.046 | INFO | Analyzing chunk frames = [1148928,1149440) | 20260507 17:04:24.046 | INFO | MemInfo: { | 20260507 17:04:24.046 | INFO | "gpu" : " 17 GB", | 20260507 17:04:24.046 | INFO | "huge" : "", | 20260507 17:04:24.046 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:24.046 | INFO | "malloc" : " 14 GB", | 20260507 17:04:24.046 | INFO | "pinned" : " 29 GB" | 20260507 17:04:24.046 | INFO | } | 20260507 17:04:24.046 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:04:24.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:24.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 17:04:24.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 17:04:24.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:24.046 | INFO | Uploaded 36GiB to GPU in 2.37985s (15.127GiB/s) | 20260507 17:04:24.046 | INFO | Downloaded 34GiB to GPU in 2.45228s (13.8646GiB/s) | 20260507 17:04:29.185 | INFO | Analyzing chunk frames = [1149440,1149952) | 20260507 17:04:29.185 | INFO | MemInfo: { | 20260507 17:04:29.185 | INFO | "gpu" : " 17 GB", | 20260507 17:04:29.185 | INFO | "huge" : "", | 20260507 17:04:29.185 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:29.185 | INFO | "malloc" : " 14 GB", | 20260507 17:04:29.185 | INFO | "pinned" : " 29 GB" | 20260507 17:04:29.185 | INFO | } | 20260507 17:04:29.185 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:04:29.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:29.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.185 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:04:29.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.185 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:04:29.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:29.186 | INFO | Uploaded 36GiB to GPU in 2.35501s (15.2866GiB/s) | 20260507 17:04:29.186 | INFO | Downloaded 34GiB to GPU in 2.44638s (13.8981GiB/s) | 20260507 17:04:34.324 | INFO | Analyzing chunk frames = [1149952,1150464) | 20260507 17:04:34.324 | INFO | MemInfo: { | 20260507 17:04:34.324 | INFO | "gpu" : " 17 GB", | 20260507 17:04:34.324 | INFO | "huge" : "", | 20260507 17:04:34.324 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:34.324 | INFO | "malloc" : " 14 GB", | 20260507 17:04:34.325 | INFO | "pinned" : " 29 GB" | 20260507 17:04:34.325 | INFO | } | 20260507 17:04:34.325 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:04:34.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:34.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:04:34.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 17:04:34.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:34.325 | INFO | Uploaded 36GiB to GPU in 2.46651s (14.5955GiB/s) | 20260507 17:04:34.325 | INFO | Downloaded 34GiB to GPU in 2.4241s (14.0258GiB/s) | 20260507 17:04:39.464 | INFO | Analyzing chunk frames = [1150464,1150976) | 20260507 17:04:39.464 | INFO | MemInfo: { | 20260507 17:04:39.464 | INFO | "gpu" : " 17 GB", | 20260507 17:04:39.464 | INFO | "huge" : "", | 20260507 17:04:39.464 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:39.464 | INFO | "malloc" : " 14 GB", | 20260507 17:04:39.464 | INFO | "pinned" : " 29 GB" | 20260507 17:04:39.464 | INFO | } | 20260507 17:04:39.464 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:04:39.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:39.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.464 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:04:39.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.464 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:04:39.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:39.465 | INFO | Uploaded 36GiB to GPU in 2.47137s (14.5668GiB/s) | 20260507 17:04:39.465 | INFO | Downloaded 34GiB to GPU in 2.4961s (13.6212GiB/s) | 20260507 17:04:39.561 | INFO | Starting a round of Baz IO | 20260507 17:04:43.108 | INFO | Populating RTMetrics report | 20260507 17:04:43.108 | INFO | RTMetrics report populated | 20260507 17:04:43.108 | INFO | Opening tmp RTMetrics json file | 20260507 17:04:43.112 | INFO | Serializing RTMetrics to json | 20260507 17:04:43.114 | INFO | Serialization done, swapping tmp file now... | 20260507 17:04:43.115 | INFO | RTMetrics done | 20260507 17:04:43.221 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.66 seconds | 20260507 17:04:43.221 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.86% / 0.02% / 51.12% | 20260507 17:04:43.221 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.90829% / 2.78155e-05% / 3.57628e-05% | 20260507 17:04:43.221 | INFO | Baz IO overall: | 20260507 17:04:43.221 | INFO | BytesWritten : 502243570544 | 20260507 17:04:43.221 | INFO | NumEvents : 243239696348 | 20260507 17:04:44.604 | INFO | Analyzing chunk frames = [1150976,1151488) | 20260507 17:04:44.604 | INFO | MemInfo: { | 20260507 17:04:44.604 | INFO | "gpu" : " 17 GB", | 20260507 17:04:44.604 | INFO | "huge" : "", | 20260507 17:04:44.604 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:44.604 | INFO | "malloc" : " 14 GB", | 20260507 17:04:44.604 | INFO | "pinned" : " 29 GB" | 20260507 17:04:44.604 | INFO | } | 20260507 17:04:44.604 | INFO | Perf report for the last 5.13967 seconds: | 20260507 17:04:44.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:44.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:44.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:44.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:44.604 | INFO | ANALYSIS : 69.8%, 3.0, 108.2ms, 0 | 20260507 17:04:44.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:44.604 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260507 17:04:44.604 | INFO | PRE_HQ : 67.0%, 1.8, 60.9ms, 0 | 20260507 17:04:44.604 | INFO | BAZWRITER : 70.5%, 3.7, 135.0ms, 0 | 20260507 17:04:44.604 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 17:04:44.604 | INFO | Uploaded 36GiB to GPU in 2.56373s (14.0421GiB/s) | 20260507 17:04:44.604 | INFO | Downloaded 38GiB to GPU in 2.73013s (13.9188GiB/s) | 20260507 17:04:49.743 | INFO | Analyzing chunk frames = [1151488,1152000) | 20260507 17:04:49.743 | INFO | MemInfo: { | 20260507 17:04:49.743 | INFO | "gpu" : " 17 GB", | 20260507 17:04:49.743 | INFO | "huge" : "", | 20260507 17:04:49.743 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:49.743 | INFO | "malloc" : " 14 GB", | 20260507 17:04:49.743 | INFO | "pinned" : " 29 GB" | 20260507 17:04:49.743 | INFO | } | 20260507 17:04:49.743 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:04:49.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:49.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.743 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:04:49.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.743 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:04:49.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:49.744 | INFO | Uploaded 36GiB to GPU in 2.50033s (14.3981GiB/s) | 20260507 17:04:49.744 | INFO | Downloaded 34GiB to GPU in 2.43949s (13.9373GiB/s) | 20260507 17:04:54.882 | INFO | Analyzing chunk frames = [1152000,1152512) | 20260507 17:04:54.882 | INFO | MemInfo: { | 20260507 17:04:54.882 | INFO | "gpu" : " 17 GB", | 20260507 17:04:54.882 | INFO | "huge" : "", | 20260507 17:04:54.882 | INFO | "huge_shared" : " 29 GB", | 20260507 17:04:54.882 | INFO | "malloc" : " 14 GB", | 20260507 17:04:54.882 | INFO | "pinned" : " 29 GB" | 20260507 17:04:54.882 | INFO | } | 20260507 17:04:54.882 | INFO | Perf report for the last 5.13891 seconds: | 20260507 17:04:54.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:04:54.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:04:54.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:04:54.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:04:54.882 | INFO | Uploaded 36GiB to GPU in 2.56984s (14.0087GiB/s) | 20260507 17:04:54.882 | INFO | Downloaded 34GiB to GPU in 2.42737s (14.0069GiB/s) | 20260507 17:05:00.021 | INFO | Analyzing chunk frames = [1152512,1153024) | 20260507 17:05:00.021 | INFO | MemInfo: { | 20260507 17:05:00.021 | INFO | "gpu" : " 17 GB", | 20260507 17:05:00.021 | INFO | "huge" : "", | 20260507 17:05:00.021 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:00.021 | INFO | "malloc" : " 14 GB", | 20260507 17:05:00.022 | INFO | "pinned" : " 29 GB" | 20260507 17:05:00.022 | INFO | } | 20260507 17:05:00.022 | INFO | Perf report for the last 5.13964 seconds: | 20260507 17:05:00.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:00.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:05:00.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:05:00.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:00.022 | INFO | Uploaded 36GiB to GPU in 2.47178s (14.5644GiB/s) | 20260507 17:05:00.022 | INFO | Downloaded 34GiB to GPU in 2.35017s (14.4671GiB/s) | 20260507 17:05:05.161 | INFO | Analyzing chunk frames = [1153024,1153536) | 20260507 17:05:05.161 | INFO | MemInfo: { | 20260507 17:05:05.161 | INFO | "gpu" : " 17 GB", | 20260507 17:05:05.161 | INFO | "huge" : "", | 20260507 17:05:05.161 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:05.161 | INFO | "malloc" : " 14 GB", | 20260507 17:05:05.161 | INFO | "pinned" : " 29 GB" | 20260507 17:05:05.161 | INFO | } | 20260507 17:05:05.161 | INFO | Perf report for the last 5.13937 seconds: | 20260507 17:05:05.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:05.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 17:05:05.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:05:05.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:05.161 | INFO | Uploaded 36GiB to GPU in 2.3797s (15.1279GiB/s) | 20260507 17:05:05.161 | INFO | Downloaded 34GiB to GPU in 2.45449s (13.8522GiB/s) | 20260507 17:05:10.300 | INFO | Analyzing chunk frames = [1153536,1154048) | 20260507 17:05:10.300 | INFO | MemInfo: { | 20260507 17:05:10.300 | INFO | "gpu" : " 17 GB", | 20260507 17:05:10.300 | INFO | "huge" : "", | 20260507 17:05:10.300 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:10.300 | INFO | "malloc" : " 14 GB", | 20260507 17:05:10.300 | INFO | "pinned" : " 29 GB" | 20260507 17:05:10.300 | INFO | } | 20260507 17:05:10.300 | INFO | Perf report for the last 5.13886 seconds: | 20260507 17:05:10.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:10.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:05:10.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:05:10.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:10.300 | INFO | Uploaded 36GiB to GPU in 2.36645s (15.2126GiB/s) | 20260507 17:05:10.301 | INFO | Downloaded 34GiB to GPU in 2.43533s (13.9612GiB/s) | 20260507 17:05:15.439 | INFO | Analyzing chunk frames = [1154048,1154560) | 20260507 17:05:15.439 | INFO | MemInfo: { | 20260507 17:05:15.439 | INFO | "gpu" : " 17 GB", | 20260507 17:05:15.439 | INFO | "huge" : "", | 20260507 17:05:15.439 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:15.439 | INFO | "malloc" : " 14 GB", | 20260507 17:05:15.439 | INFO | "pinned" : " 29 GB" | 20260507 17:05:15.439 | INFO | } | 20260507 17:05:15.440 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:05:15.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:15.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:05:15.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:05:15.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:15.440 | INFO | Uploaded 36GiB to GPU in 2.48424s (14.4913GiB/s) | 20260507 17:05:15.440 | INFO | Downloaded 34GiB to GPU in 2.40531s (14.1354GiB/s) | 20260507 17:05:20.579 | INFO | Analyzing chunk frames = [1154560,1155072) | 20260507 17:05:20.579 | INFO | MemInfo: { | 20260507 17:05:20.579 | INFO | "gpu" : " 17 GB", | 20260507 17:05:20.579 | INFO | "huge" : "", | 20260507 17:05:20.579 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:20.579 | INFO | "malloc" : " 14 GB", | 20260507 17:05:20.579 | INFO | "pinned" : " 29 GB" | 20260507 17:05:20.579 | INFO | } | 20260507 17:05:20.579 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:05:20.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:20.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 17:05:20.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 17:05:20.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:20.579 | INFO | Uploaded 36GiB to GPU in 2.45197s (14.6821GiB/s) | 20260507 17:05:20.579 | INFO | Downloaded 34GiB to GPU in 2.49578s (13.623GiB/s) | 20260507 17:05:20.580 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:05:20.580 | INFO | Total mem usage: 16.4 GiB | 20260507 17:05:20.580 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:05:20.580 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.2%) | 20260507 17:05:20.580 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:05:20.580 | INFO | High Water Breakdown: | 20260507 17:05:20.580 | INFO | 59.7% : 8.96 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:05:20.580 | INFO | 40.2% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:05:20.580 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:05:20.580 | INFO | ------------------------------------------ | 20260507 17:05:20.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:05:20.580 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:05:20.580 | INFO | Total mem usage: 26.5 GiB | 20260507 17:05:20.580 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:05:20.580 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:05:20.580 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:05:20.580 | INFO | High Water Breakdown: | 20260507 17:05:20.580 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:05:20.580 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:05:20.580 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:05:20.580 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:05:20.580 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:05:20.580 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:05:20.580 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:05:20.580 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:05:20.580 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:05:20.580 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:05:20.580 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:05:20.580 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:05:20.580 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:05:20.580 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:05:20.580 | INFO | ------------------------------------------ | 20260507 17:05:20.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:05:20.580 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:05:20.580 | INFO | Total mem usage: 27.8 GiB | 20260507 17:05:20.580 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:05:20.580 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:05:20.580 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:05:20.580 | INFO | High Water Breakdown: | 20260507 17:05:20.580 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:05:20.580 | INFO | ------------------------------------------ | 20260507 17:05:20.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:05:20.580 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:05:20.580 | INFO | Total mem usage: 15.7 GiB | 20260507 17:05:20.580 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:05:20.580 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:05:20.580 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:05:20.580 | INFO | High Water Breakdown: | 20260507 17:05:20.580 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:05:20.580 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:05:20.580 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:05:20.580 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:05:20.580 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:05:20.580 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:05:20.580 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:05:20.580 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:05:20.580 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:05:20.580 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:05:20.580 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:05:20.580 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:05:20.580 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:05:20.580 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:05:20.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:05:20.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:05:20.580 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:05:20.580 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:05:20.580 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:05:20.580 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:05:20.580 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:05:20.580 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:05:20.580 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:05:20.580 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:05:20.580 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:05:20.580 | INFO | ------------------------------------------ | 20260507 17:05:20.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:05:20.580 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:05:20.580 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:05:20.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:05:20.580 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.40e+01 / 2.80e+00 | 20260507 17:05:20.580 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.11e-02 / 4.95e-03 | 20260507 17:05:20.581 | INFO | PerfReport: 12.05%, Baseline -- 4.05e+00 / 4.49e+00, 6.24e-01 / 2.55e-01 | 20260507 17:05:20.581 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 8.92e-02 | 20260507 17:05:20.581 | INFO | PerfReport: 15.71%, DME -- 4.22e+01 / 4.52e+01, 4.32e+00 / 2.57e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.90e+00 / 2.19e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.53e+00 / 2.89e+00, 4.36e-01 / 2.45e-01 | 20260507 17:05:20.581 | INFO | PerfReport: 3.81%, Metrics -- 1.28e+00 / 1.45e+00, 3.27e-01 / 1.79e-01 | 20260507 17:05:20.581 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.30e-01, 6.87e-02 / 4.73e-02 | 20260507 17:05:20.581 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:05:20.581 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:05:20.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:05:20.581 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 23.94%, Upload -- 2.57e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 23.85%, Download -- 2.56e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 31.25%, Compute -- 3.36e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 1.86%, LockUpload -- 2.00e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 16.84%, LockCompute -- 1.81e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 17:05:20.581 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.94e-02 / 0.00e+00 | 20260507 17:05:20.676 | INFO | Starting a round of Baz IO | 20260507 17:05:24.200 | INFO | Populating RTMetrics report | 20260507 17:05:24.200 | INFO | RTMetrics report populated | 20260507 17:05:24.200 | INFO | Opening tmp RTMetrics json file | 20260507 17:05:24.220 | INFO | Serializing RTMetrics to json | 20260507 17:05:24.222 | INFO | Serialization done, swapping tmp file now... | 20260507 17:05:24.222 | INFO | RTMetrics done | 20260507 17:05:24.306 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.63 seconds | 20260507 17:05:24.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.84% / 0.02% / 51.14% | 20260507 17:05:24.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.91227% / 3.97364e-05% / 1.98682e-05% | 20260507 17:05:24.306 | INFO | Baz IO overall: | 20260507 17:05:24.306 | INFO | BytesWritten : 504173484353 | 20260507 17:05:24.306 | INFO | NumEvents : 244178656415 | 20260507 17:05:25.718 | INFO | Analyzing chunk frames = [1155072,1155584) | 20260507 17:05:25.718 | INFO | MemInfo: { | 20260507 17:05:25.718 | INFO | "gpu" : " 17 GB", | 20260507 17:05:25.718 | INFO | "huge" : "", | 20260507 17:05:25.718 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:25.718 | INFO | "malloc" : " 14 GB", | 20260507 17:05:25.718 | INFO | "pinned" : " 29 GB" | 20260507 17:05:25.718 | INFO | } | 20260507 17:05:25.718 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:05:25.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:25.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:25.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:25.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:25.718 | INFO | ANALYSIS : 70.0%, 2.9, 107.9ms, 0 | 20260507 17:05:25.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:25.718 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 17:05:25.718 | INFO | PRE_HQ : 66.8%, 1.7, 59.5ms, 0 | 20260507 17:05:25.718 | INFO | BAZWRITER : 70.2%, 4.1, 147.6ms, 0 | 20260507 17:05:25.718 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 17:05:25.719 | INFO | Uploaded 36GiB to GPU in 2.59622s (13.8663GiB/s) | 20260507 17:05:25.719 | INFO | Downloaded 38GiB to GPU in 2.73753s (13.8811GiB/s) | 20260507 17:05:30.857 | INFO | Analyzing chunk frames = [1155584,1156096) | 20260507 17:05:30.857 | INFO | MemInfo: { | 20260507 17:05:30.857 | INFO | "gpu" : " 17 GB", | 20260507 17:05:30.857 | INFO | "huge" : "", | 20260507 17:05:30.857 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:30.857 | INFO | "malloc" : " 14 GB", | 20260507 17:05:30.857 | INFO | "pinned" : " 29 GB" | 20260507 17:05:30.857 | INFO | } | 20260507 17:05:30.858 | INFO | Perf report for the last 5.13972 seconds: | 20260507 17:05:30.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:30.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:05:30.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:05:30.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:30.858 | INFO | Uploaded 36GiB to GPU in 2.53905s (14.1785GiB/s) | 20260507 17:05:30.858 | INFO | Downloaded 34GiB to GPU in 2.42822s (14.0021GiB/s) | 20260507 17:05:35.996 | INFO | Analyzing chunk frames = [1156096,1156608) | 20260507 17:05:35.996 | INFO | MemInfo: { | 20260507 17:05:35.996 | INFO | "gpu" : " 17 GB", | 20260507 17:05:35.996 | INFO | "huge" : "", | 20260507 17:05:35.996 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:35.997 | INFO | "malloc" : " 14 GB", | 20260507 17:05:35.997 | INFO | "pinned" : " 29 GB" | 20260507 17:05:35.997 | INFO | } | 20260507 17:05:35.997 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:05:35.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:35.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | ANALYSIS : 68.4%, 3.0, 106.0ms, 0 | 20260507 17:05:35.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:05:35.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:35.997 | INFO | Uploaded 36GiB to GPU in 2.5758s (13.9762GiB/s) | 20260507 17:05:35.997 | INFO | Downloaded 34GiB to GPU in 2.40647s (14.1286GiB/s) | 20260507 17:05:41.136 | INFO | Analyzing chunk frames = [1156608,1157120) | 20260507 17:05:41.136 | INFO | MemInfo: { | 20260507 17:05:41.136 | INFO | "gpu" : " 17 GB", | 20260507 17:05:41.136 | INFO | "huge" : "", | 20260507 17:05:41.136 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:41.136 | INFO | "malloc" : " 14 GB", | 20260507 17:05:41.136 | INFO | "pinned" : " 29 GB" | 20260507 17:05:41.136 | INFO | } | 20260507 17:05:41.136 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:05:41.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:41.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.136 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 17:05:41.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.136 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:05:41.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:41.137 | INFO | Uploaded 36GiB to GPU in 2.5048s (14.3724GiB/s) | 20260507 17:05:41.137 | INFO | Downloaded 34GiB to GPU in 2.3451s (14.4983GiB/s) | 20260507 17:05:46.276 | INFO | Analyzing chunk frames = [1157120,1157632) | 20260507 17:05:46.276 | INFO | MemInfo: { | 20260507 17:05:46.276 | INFO | "gpu" : " 17 GB", | 20260507 17:05:46.276 | INFO | "huge" : "", | 20260507 17:05:46.276 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:46.276 | INFO | "malloc" : " 14 GB", | 20260507 17:05:46.276 | INFO | "pinned" : " 29 GB" | 20260507 17:05:46.276 | INFO | } | 20260507 17:05:46.276 | INFO | Perf report for the last 5.13986 seconds: | 20260507 17:05:46.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:46.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.276 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260507 17:05:46.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.276 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:05:46.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:46.277 | INFO | Uploaded 36GiB to GPU in 2.43342s (14.794GiB/s) | 20260507 17:05:46.277 | INFO | Downloaded 34GiB to GPU in 2.46394s (13.799GiB/s) | 20260507 17:05:51.415 | INFO | Analyzing chunk frames = [1157632,1158144) | 20260507 17:05:51.415 | INFO | MemInfo: { | 20260507 17:05:51.415 | INFO | "gpu" : " 17 GB", | 20260507 17:05:51.415 | INFO | "huge" : "", | 20260507 17:05:51.415 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:51.415 | INFO | "malloc" : " 14 GB", | 20260507 17:05:51.415 | INFO | "pinned" : " 29 GB" | 20260507 17:05:51.415 | INFO | } | 20260507 17:05:51.415 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:05:51.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:51.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 17:05:51.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:05:51.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:51.415 | INFO | Uploaded 36GiB to GPU in 2.33478s (15.419GiB/s) | 20260507 17:05:51.415 | INFO | Downloaded 34GiB to GPU in 2.47508s (13.7369GiB/s) | 20260507 17:05:56.554 | INFO | Analyzing chunk frames = [1158144,1158656) | 20260507 17:05:56.554 | INFO | MemInfo: { | 20260507 17:05:56.554 | INFO | "gpu" : " 17 GB", | 20260507 17:05:56.554 | INFO | "huge" : "", | 20260507 17:05:56.554 | INFO | "huge_shared" : " 29 GB", | 20260507 17:05:56.554 | INFO | "malloc" : " 14 GB", | 20260507 17:05:56.554 | INFO | "pinned" : " 29 GB" | 20260507 17:05:56.554 | INFO | } | 20260507 17:05:56.554 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:05:56.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:05:56.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:05:56.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:05:56.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:05:56.554 | INFO | Uploaded 36GiB to GPU in 2.43585s (14.7792GiB/s) | 20260507 17:05:56.554 | INFO | Downloaded 34GiB to GPU in 2.41753s (14.0639GiB/s) | 20260507 17:06:01.693 | INFO | Analyzing chunk frames = [1158656,1159168) | 20260507 17:06:01.693 | INFO | MemInfo: { | 20260507 17:06:01.693 | INFO | "gpu" : " 17 GB", | 20260507 17:06:01.693 | INFO | "huge" : "", | 20260507 17:06:01.693 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:01.693 | INFO | "malloc" : " 14 GB", | 20260507 17:06:01.693 | INFO | "pinned" : " 29 GB" | 20260507 17:06:01.694 | INFO | } | 20260507 17:06:01.694 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:06:01.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:01.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:06:01.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 17:06:01.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:01.694 | INFO | Uploaded 36GiB to GPU in 2.53347s (14.2098GiB/s) | 20260507 17:06:01.694 | INFO | Downloaded 34GiB to GPU in 2.51795s (13.503GiB/s) | 20260507 17:06:01.791 | INFO | Starting a round of Baz IO | 20260507 17:06:05.308 | INFO | Populating RTMetrics report | 20260507 17:06:05.308 | INFO | RTMetrics report populated | 20260507 17:06:05.308 | INFO | Opening tmp RTMetrics json file | 20260507 17:06:05.309 | INFO | Serializing RTMetrics to json | 20260507 17:06:05.311 | INFO | Serialization done, swapping tmp file now... | 20260507 17:06:05.312 | INFO | RTMetrics done | 20260507 17:06:05.407 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.616 seconds | 20260507 17:06:05.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.81% / 0.02% / 51.16% | 20260507 17:06:05.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.88021% / 1.98682e-05% / 3.57628e-05% | 20260507 17:06:05.408 | INFO | Baz IO overall: | 20260507 17:06:05.408 | INFO | BytesWritten : 506100345833 | 20260507 17:06:05.408 | INFO | NumEvents : 245114555553 | 20260507 17:06:06.833 | INFO | Analyzing chunk frames = [1159168,1159680) | 20260507 17:06:06.833 | INFO | MemInfo: { | 20260507 17:06:06.833 | INFO | "gpu" : " 17 GB", | 20260507 17:06:06.833 | INFO | "huge" : "", | 20260507 17:06:06.833 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:06.833 | INFO | "malloc" : " 14 GB", | 20260507 17:06:06.833 | INFO | "pinned" : " 29 GB" | 20260507 17:06:06.833 | INFO | } | 20260507 17:06:06.833 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:06:06.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:06.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:06.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:06.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:06.833 | INFO | ANALYSIS : 69.9%, 3.0, 108.2ms, 0 | 20260507 17:06:06.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:06.833 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 17:06:06.833 | INFO | PRE_HQ : 66.8%, 1.7, 60.1ms, 0 | 20260507 17:06:06.833 | INFO | BAZWRITER : 69.9%, 3.7, 131.9ms, 0 | 20260507 17:06:06.833 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260507 17:06:06.833 | INFO | Uploaded 36GiB to GPU in 2.56325s (14.0447GiB/s) | 20260507 17:06:06.833 | INFO | Downloaded 38GiB to GPU in 2.7071s (14.0372GiB/s) | 20260507 17:06:11.972 | INFO | Analyzing chunk frames = [1159680,1160192) | 20260507 17:06:11.972 | INFO | MemInfo: { | 20260507 17:06:11.972 | INFO | "gpu" : " 17 GB", | 20260507 17:06:11.972 | INFO | "huge" : "", | 20260507 17:06:11.972 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:11.972 | INFO | "malloc" : " 14 GB", | 20260507 17:06:11.972 | INFO | "pinned" : " 29 GB" | 20260507 17:06:11.972 | INFO | } | 20260507 17:06:11.972 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:06:11.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:11.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.973 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260507 17:06:11.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.973 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:06:11.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:11.973 | INFO | Uploaded 36GiB to GPU in 2.48635s (14.4791GiB/s) | 20260507 17:06:11.973 | INFO | Downloaded 34GiB to GPU in 2.42171s (14.0397GiB/s) | 20260507 17:06:17.111 | INFO | Analyzing chunk frames = [1160192,1160704) | 20260507 17:06:17.111 | INFO | MemInfo: { | 20260507 17:06:17.111 | INFO | "gpu" : " 17 GB", | 20260507 17:06:17.111 | INFO | "huge" : "", | 20260507 17:06:17.111 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:17.112 | INFO | "malloc" : " 14 GB", | 20260507 17:06:17.112 | INFO | "pinned" : " 29 GB" | 20260507 17:06:17.112 | INFO | } | 20260507 17:06:17.112 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:06:17.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:17.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 17:06:17.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:06:17.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:17.112 | INFO | Uploaded 36GiB to GPU in 2.56432s (14.0388GiB/s) | 20260507 17:06:17.112 | INFO | Downloaded 34GiB to GPU in 2.43567s (13.9592GiB/s) | 20260507 17:06:22.251 | INFO | Analyzing chunk frames = [1160704,1161216) | 20260507 17:06:22.251 | INFO | MemInfo: { | 20260507 17:06:22.251 | INFO | "gpu" : " 17 GB", | 20260507 17:06:22.251 | INFO | "huge" : "", | 20260507 17:06:22.251 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:22.251 | INFO | "malloc" : " 14 GB", | 20260507 17:06:22.251 | INFO | "pinned" : " 29 GB" | 20260507 17:06:22.251 | INFO | } | 20260507 17:06:22.251 | INFO | Perf report for the last 5.13936 seconds: | 20260507 17:06:22.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:22.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:06:22.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:06:22.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:22.251 | INFO | Uploaded 36GiB to GPU in 2.45834s (14.644GiB/s) | 20260507 17:06:22.251 | INFO | Downloaded 34GiB to GPU in 2.3539s (14.4441GiB/s) | 20260507 17:06:27.390 | INFO | Analyzing chunk frames = [1161216,1161728) | 20260507 17:06:27.390 | INFO | MemInfo: { | 20260507 17:06:27.390 | INFO | "gpu" : " 17 GB", | 20260507 17:06:27.390 | INFO | "huge" : "", | 20260507 17:06:27.390 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:27.390 | INFO | "malloc" : " 14 GB", | 20260507 17:06:27.390 | INFO | "pinned" : " 29 GB" | 20260507 17:06:27.390 | INFO | } | 20260507 17:06:27.390 | INFO | Perf report for the last 5.13939 seconds: | 20260507 17:06:27.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:27.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.391 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:06:27.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.391 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:06:27.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:27.391 | INFO | Uploaded 36GiB to GPU in 2.30815s (15.5969GiB/s) | 20260507 17:06:27.391 | INFO | Downloaded 34GiB to GPU in 2.48869s (13.6618GiB/s) | 20260507 17:06:32.529 | INFO | Analyzing chunk frames = [1161728,1162240) | 20260507 17:06:32.529 | INFO | MemInfo: { | 20260507 17:06:32.529 | INFO | "gpu" : " 17 GB", | 20260507 17:06:32.529 | INFO | "huge" : "", | 20260507 17:06:32.529 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:32.529 | INFO | "malloc" : " 14 GB", | 20260507 17:06:32.529 | INFO | "pinned" : " 29 GB" | 20260507 17:06:32.529 | INFO | } | 20260507 17:06:32.529 | INFO | Perf report for the last 5.13883 seconds: | 20260507 17:06:32.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:32.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.529 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:06:32.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.529 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260507 17:06:32.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:32.530 | INFO | Uploaded 36GiB to GPU in 2.35013s (15.3183GiB/s) | 20260507 17:06:32.530 | INFO | Downloaded 34GiB to GPU in 2.46167s (13.8118GiB/s) | 20260507 17:06:37.668 | INFO | Analyzing chunk frames = [1162240,1162752) | 20260507 17:06:37.668 | INFO | MemInfo: { | 20260507 17:06:37.668 | INFO | "gpu" : " 17 GB", | 20260507 17:06:37.668 | INFO | "huge" : "", | 20260507 17:06:37.668 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:37.668 | INFO | "malloc" : " 14 GB", | 20260507 17:06:37.668 | INFO | "pinned" : " 29 GB" | 20260507 17:06:37.669 | INFO | } | 20260507 17:06:37.669 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:06:37.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:37.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:06:37.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:06:37.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:37.669 | INFO | Uploaded 36GiB to GPU in 2.44597s (14.7181GiB/s) | 20260507 17:06:37.669 | INFO | Downloaded 34GiB to GPU in 2.41874s (14.0569GiB/s) | 20260507 17:06:42.807 | INFO | Analyzing chunk frames = [1162752,1163264) | 20260507 17:06:42.807 | INFO | MemInfo: { | 20260507 17:06:42.807 | INFO | "gpu" : " 17 GB", | 20260507 17:06:42.807 | INFO | "huge" : "", | 20260507 17:06:42.807 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:42.808 | INFO | "malloc" : " 14 GB", | 20260507 17:06:42.808 | INFO | "pinned" : " 29 GB" | 20260507 17:06:42.808 | INFO | } | 20260507 17:06:42.808 | INFO | Perf report for the last 5.13913 seconds: | 20260507 17:06:42.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:42.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:06:42.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:06:42.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:42.808 | INFO | Uploaded 36GiB to GPU in 2.47773s (14.5295GiB/s) | 20260507 17:06:42.808 | INFO | Downloaded 34GiB to GPU in 2.50603s (13.5673GiB/s) | 20260507 17:06:42.808 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:06:42.808 | INFO | Total mem usage: 16.4 GiB | 20260507 17:06:42.808 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:06:42.808 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.96%) | 20260507 17:06:42.808 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:06:42.808 | INFO | High Water Breakdown: | 20260507 17:06:42.808 | INFO | 59.6% : 8.89 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:06:42.808 | INFO | 40.3% : 6.01 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:06:42.808 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:06:42.808 | INFO | ------------------------------------------ | 20260507 17:06:42.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:06:42.808 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:06:42.808 | INFO | Total mem usage: 26.5 GiB | 20260507 17:06:42.808 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:06:42.808 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:06:42.808 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:06:42.808 | INFO | High Water Breakdown: | 20260507 17:06:42.808 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:06:42.808 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:06:42.808 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:06:42.808 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:06:42.808 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:06:42.808 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:06:42.808 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:06:42.808 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:06:42.808 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:06:42.808 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:06:42.808 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:06:42.808 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:06:42.808 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:06:42.808 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:06:42.808 | INFO | ------------------------------------------ | 20260507 17:06:42.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:06:42.808 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:06:42.808 | INFO | Total mem usage: 27.8 GiB | 20260507 17:06:42.808 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:06:42.808 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:06:42.808 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:06:42.808 | INFO | High Water Breakdown: | 20260507 17:06:42.808 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:06:42.808 | INFO | ------------------------------------------ | 20260507 17:06:42.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:06:42.808 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:06:42.808 | INFO | Total mem usage: 15.7 GiB | 20260507 17:06:42.808 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:06:42.808 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:06:42.808 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:06:42.808 | INFO | High Water Breakdown: | 20260507 17:06:42.808 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:06:42.808 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:06:42.808 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:06:42.808 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:06:42.808 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:06:42.808 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:06:42.808 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:06:42.808 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:06:42.808 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:06:42.808 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:06:42.808 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:06:42.808 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:06:42.808 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:06:42.808 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:06:42.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:06:42.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:06:42.808 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:06:42.808 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:06:42.808 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:06:42.808 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:06:42.808 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:06:42.808 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:06:42.808 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:06:42.808 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:06:42.808 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:06:42.808 | INFO | ------------------------------------------ | 20260507 17:06:42.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:06:42.808 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:06:42.808 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:06:42.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:06:42.808 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.68e+01, 1.45e+01 / 2.75e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.11e-02 / 4.92e-03 | 20260507 17:06:42.808 | INFO | PerfReport: 12.05%, Baseline -- 4.05e+00 / 4.52e+00, 6.24e-01 / 1.91e-01 | 20260507 17:06:42.808 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.18e-01 / 8.18e-02 | 20260507 17:06:42.808 | INFO | PerfReport: 15.74%, DME -- 4.23e+01 / 4.52e+01, 4.47e+00 / 2.66e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.91e+00 / 1.98e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.90e+00, 4.37e-01 / 2.45e-01 | 20260507 17:06:42.808 | INFO | PerfReport: 3.79%, Metrics -- 1.27e+00 / 1.44e+00, 3.24e-01 / 1.88e-01 | 20260507 17:06:42.808 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.29e-01, 6.92e-02 / 4.26e-02 | 20260507 17:06:42.808 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:06:42.808 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:06:42.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:06:42.808 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 23.95%, Upload -- 2.58e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 23.94%, Download -- 2.58e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 31.21%, Compute -- 3.36e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 1.87%, LockUpload -- 2.01e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 2.25%, LockDownload -- 2.42e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 16.76%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:06:42.808 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260507 17:06:42.905 | INFO | Starting a round of Baz IO | 20260507 17:06:46.421 | INFO | Populating RTMetrics report | 20260507 17:06:46.422 | INFO | RTMetrics report populated | 20260507 17:06:46.422 | INFO | Opening tmp RTMetrics json file | 20260507 17:06:46.444 | INFO | Serializing RTMetrics to json | 20260507 17:06:46.445 | INFO | Serialization done, swapping tmp file now... | 20260507 17:06:46.446 | INFO | RTMetrics done | 20260507 17:06:46.535 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.63 seconds | 20260507 17:06:46.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.79% / 0.02% / 51.19% | 20260507 17:06:46.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.87878% / 3.97364e-05% / 4.37101e-05% | 20260507 17:06:46.535 | INFO | Baz IO overall: | 20260507 17:06:46.535 | INFO | BytesWritten : 508030775428 | 20260507 17:06:46.535 | INFO | NumEvents : 246053155659 | 20260507 17:06:47.947 | INFO | Analyzing chunk frames = [1163264,1163776) | 20260507 17:06:47.947 | INFO | MemInfo: { | 20260507 17:06:47.947 | INFO | "gpu" : " 17 GB", | 20260507 17:06:47.947 | INFO | "huge" : "", | 20260507 17:06:47.947 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:47.947 | INFO | "malloc" : " 14 GB", | 20260507 17:06:47.947 | INFO | "pinned" : " 29 GB" | 20260507 17:06:47.947 | INFO | } | 20260507 17:06:47.947 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:06:47.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:47.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:47.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:47.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:47.947 | INFO | ANALYSIS : 69.9%, 3.0, 108.6ms, 0 | 20260507 17:06:47.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:47.947 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 17:06:47.947 | INFO | PRE_HQ : 66.9%, 1.7, 60.0ms, 0 | 20260507 17:06:47.947 | INFO | BAZWRITER : 70.2%, 3.8, 136.3ms, 0 | 20260507 17:06:47.947 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 17:06:47.948 | INFO | Uploaded 36GiB to GPU in 2.55628s (14.083GiB/s) | 20260507 17:06:47.948 | INFO | Downloaded 38GiB to GPU in 2.7187s (13.9773GiB/s) | 20260507 17:06:53.086 | INFO | Analyzing chunk frames = [1163776,1164288) | 20260507 17:06:53.086 | INFO | MemInfo: { | 20260507 17:06:53.086 | INFO | "gpu" : " 17 GB", | 20260507 17:06:53.086 | INFO | "huge" : "", | 20260507 17:06:53.086 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:53.086 | INFO | "malloc" : " 14 GB", | 20260507 17:06:53.087 | INFO | "pinned" : " 29 GB" | 20260507 17:06:53.087 | INFO | } | 20260507 17:06:53.087 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:06:53.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:53.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:06:53.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:06:53.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:53.087 | INFO | Uploaded 36GiB to GPU in 2.53346s (14.2098GiB/s) | 20260507 17:06:53.087 | INFO | Downloaded 34GiB to GPU in 2.41782s (14.0622GiB/s) | 20260507 17:06:58.226 | INFO | Analyzing chunk frames = [1164288,1164800) | 20260507 17:06:58.226 | INFO | MemInfo: { | 20260507 17:06:58.226 | INFO | "gpu" : " 17 GB", | 20260507 17:06:58.226 | INFO | "huge" : "", | 20260507 17:06:58.226 | INFO | "huge_shared" : " 29 GB", | 20260507 17:06:58.226 | INFO | "malloc" : " 14 GB", | 20260507 17:06:58.226 | INFO | "pinned" : " 29 GB" | 20260507 17:06:58.226 | INFO | } | 20260507 17:06:58.226 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:06:58.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:06:58.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260507 17:06:58.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:06:58.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:06:58.226 | INFO | Uploaded 36GiB to GPU in 2.47111s (14.5684GiB/s) | 20260507 17:06:58.226 | INFO | Downloaded 34GiB to GPU in 2.33946s (14.5333GiB/s) | 20260507 17:07:03.365 | INFO | Analyzing chunk frames = [1164800,1165312) | 20260507 17:07:03.365 | INFO | MemInfo: { | 20260507 17:07:03.365 | INFO | "gpu" : " 17 GB", | 20260507 17:07:03.365 | INFO | "huge" : "", | 20260507 17:07:03.365 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:03.365 | INFO | "malloc" : " 14 GB", | 20260507 17:07:03.365 | INFO | "pinned" : " 29 GB" | 20260507 17:07:03.365 | INFO | } | 20260507 17:07:03.365 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:07:03.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:03.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.366 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260507 17:07:03.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.366 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:07:03.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:03.366 | INFO | Uploaded 36GiB to GPU in 2.50401s (14.377GiB/s) | 20260507 17:07:03.366 | INFO | Downloaded 34GiB to GPU in 2.345s (14.4989GiB/s) | 20260507 17:07:08.505 | INFO | Analyzing chunk frames = [1165312,1165824) | 20260507 17:07:08.505 | INFO | MemInfo: { | 20260507 17:07:08.505 | INFO | "gpu" : " 17 GB", | 20260507 17:07:08.505 | INFO | "huge" : "", | 20260507 17:07:08.505 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:08.505 | INFO | "malloc" : " 14 GB", | 20260507 17:07:08.505 | INFO | "pinned" : " 29 GB" | 20260507 17:07:08.505 | INFO | } | 20260507 17:07:08.505 | INFO | Perf report for the last 5.13959 seconds: | 20260507 17:07:08.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:08.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.505 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260507 17:07:08.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.505 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 17:07:08.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:08.506 | INFO | Uploaded 36GiB to GPU in 2.38854s (15.072GiB/s) | 20260507 17:07:08.506 | INFO | Downloaded 34GiB to GPU in 2.43562s (13.9595GiB/s) | 20260507 17:07:13.644 | INFO | Analyzing chunk frames = [1165824,1166336) | 20260507 17:07:13.644 | INFO | MemInfo: { | 20260507 17:07:13.644 | INFO | "gpu" : " 17 GB", | 20260507 17:07:13.644 | INFO | "huge" : "", | 20260507 17:07:13.644 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:13.644 | INFO | "malloc" : " 14 GB", | 20260507 17:07:13.644 | INFO | "pinned" : " 29 GB" | 20260507 17:07:13.644 | INFO | } | 20260507 17:07:13.644 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:07:13.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:13.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 17:07:13.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:07:13.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:13.644 | INFO | Uploaded 36GiB to GPU in 2.33684s (15.4054GiB/s) | 20260507 17:07:13.644 | INFO | Downloaded 34GiB to GPU in 2.46059s (13.8178GiB/s) | 20260507 17:07:18.783 | INFO | Analyzing chunk frames = [1166336,1166848) | 20260507 17:07:18.783 | INFO | MemInfo: { | 20260507 17:07:18.783 | INFO | "gpu" : " 17 GB", | 20260507 17:07:18.783 | INFO | "huge" : "", | 20260507 17:07:18.783 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:18.783 | INFO | "malloc" : " 14 GB", | 20260507 17:07:18.783 | INFO | "pinned" : " 29 GB" | 20260507 17:07:18.783 | INFO | } | 20260507 17:07:18.783 | INFO | Perf report for the last 5.13937 seconds: | 20260507 17:07:18.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:18.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.784 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:07:18.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.784 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:07:18.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:18.784 | INFO | Uploaded 36GiB to GPU in 2.44636s (14.7158GiB/s) | 20260507 17:07:18.784 | INFO | Downloaded 34GiB to GPU in 2.41321s (14.0891GiB/s) | 20260507 17:07:23.922 | INFO | Analyzing chunk frames = [1166848,1167360) | 20260507 17:07:23.922 | INFO | MemInfo: { | 20260507 17:07:23.922 | INFO | "gpu" : " 17 GB", | 20260507 17:07:23.922 | INFO | "huge" : "", | 20260507 17:07:23.922 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:23.922 | INFO | "malloc" : " 14 GB", | 20260507 17:07:23.922 | INFO | "pinned" : " 29 GB" | 20260507 17:07:23.923 | INFO | } | 20260507 17:07:23.923 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:07:23.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:23.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:07:23.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:07:23.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:23.923 | INFO | Uploaded 36GiB to GPU in 2.51381s (14.3209GiB/s) | 20260507 17:07:23.923 | INFO | Downloaded 34GiB to GPU in 2.49992s (13.6004GiB/s) | 20260507 17:07:24.019 | INFO | Starting a round of Baz IO | 20260507 17:07:27.537 | INFO | Populating RTMetrics report | 20260507 17:07:27.537 | INFO | RTMetrics report populated | 20260507 17:07:27.537 | INFO | Opening tmp RTMetrics json file | 20260507 17:07:27.547 | INFO | Serializing RTMetrics to json | 20260507 17:07:27.564 | INFO | Serialization done, swapping tmp file now... | 20260507 17:07:27.565 | INFO | RTMetrics done | 20260507 17:07:27.640 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.62 seconds | 20260507 17:07:27.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.77% / 0.02% / 51.21% | 20260507 17:07:27.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.87872% / 5.96046e-05% / 3.97364e-05% | 20260507 17:07:27.640 | INFO | Baz IO overall: | 20260507 17:07:27.640 | INFO | BytesWritten : 509962031972 | 20260507 17:07:27.640 | INFO | NumEvents : 246992522065 | 20260507 17:07:29.062 | INFO | Analyzing chunk frames = [1167360,1167872) | 20260507 17:07:29.062 | INFO | MemInfo: { | 20260507 17:07:29.062 | INFO | "gpu" : " 17 GB", | 20260507 17:07:29.062 | INFO | "huge" : "", | 20260507 17:07:29.062 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:29.062 | INFO | "malloc" : " 14 GB", | 20260507 17:07:29.062 | INFO | "pinned" : " 29 GB" | 20260507 17:07:29.062 | INFO | } | 20260507 17:07:29.062 | INFO | Perf report for the last 5.13957 seconds: | 20260507 17:07:29.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:29.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:29.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:29.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:29.062 | INFO | ANALYSIS : 69.6%, 3.0, 107.7ms, 0 | 20260507 17:07:29.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:29.062 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 17:07:29.062 | INFO | PRE_HQ : 66.7%, 1.7, 59.9ms, 0 | 20260507 17:07:29.062 | INFO | BAZWRITER : 70.0%, 4.2, 153.0ms, 0 | 20260507 17:07:29.062 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 17:07:29.063 | INFO | Uploaded 36GiB to GPU in 2.49317s (14.4395GiB/s) | 20260507 17:07:29.063 | INFO | Downloaded 38GiB to GPU in 2.71796s (13.9811GiB/s) | 20260507 17:07:34.201 | INFO | Analyzing chunk frames = [1167872,1168384) | 20260507 17:07:34.201 | INFO | MemInfo: { | 20260507 17:07:34.201 | INFO | "gpu" : " 17 GB", | 20260507 17:07:34.201 | INFO | "huge" : "", | 20260507 17:07:34.201 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:34.201 | INFO | "malloc" : " 14 GB", | 20260507 17:07:34.201 | INFO | "pinned" : " 29 GB" | 20260507 17:07:34.201 | INFO | } | 20260507 17:07:34.201 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:07:34.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:34.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:07:34.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:07:34.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:34.201 | INFO | Uploaded 36GiB to GPU in 2.51596s (14.3087GiB/s) | 20260507 17:07:34.202 | INFO | Downloaded 34GiB to GPU in 2.4362s (13.9562GiB/s) | 20260507 17:07:39.340 | INFO | Analyzing chunk frames = [1168384,1168896) | 20260507 17:07:39.340 | INFO | MemInfo: { | 20260507 17:07:39.340 | INFO | "gpu" : " 17 GB", | 20260507 17:07:39.340 | INFO | "huge" : "", | 20260507 17:07:39.340 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:39.340 | INFO | "malloc" : " 14 GB", | 20260507 17:07:39.340 | INFO | "pinned" : " 29 GB" | 20260507 17:07:39.340 | INFO | } | 20260507 17:07:39.340 | INFO | Perf report for the last 5.1391 seconds: | 20260507 17:07:39.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:39.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:07:39.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:07:39.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:39.341 | INFO | Uploaded 36GiB to GPU in 2.53273s (14.2139GiB/s) | 20260507 17:07:39.341 | INFO | Downloaded 34GiB to GPU in 2.44182s (13.924GiB/s) | 20260507 17:07:44.480 | INFO | Analyzing chunk frames = [1168896,1169408) | 20260507 17:07:44.480 | INFO | MemInfo: { | 20260507 17:07:44.480 | INFO | "gpu" : " 17 GB", | 20260507 17:07:44.480 | INFO | "huge" : "", | 20260507 17:07:44.480 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:44.480 | INFO | "malloc" : " 14 GB", | 20260507 17:07:44.480 | INFO | "pinned" : " 29 GB" | 20260507 17:07:44.480 | INFO | } | 20260507 17:07:44.480 | INFO | Perf report for the last 5.14004 seconds: | 20260507 17:07:44.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:44.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.481 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:07:44.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.481 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:07:44.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:44.481 | INFO | Uploaded 36GiB to GPU in 2.47711s (14.5331GiB/s) | 20260507 17:07:44.481 | INFO | Downloaded 34GiB to GPU in 2.35177s (14.4572GiB/s) | 20260507 17:07:49.619 | INFO | Analyzing chunk frames = [1169408,1169920) | 20260507 17:07:49.619 | INFO | MemInfo: { | 20260507 17:07:49.619 | INFO | "gpu" : " 17 GB", | 20260507 17:07:49.619 | INFO | "huge" : "", | 20260507 17:07:49.619 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:49.619 | INFO | "malloc" : " 14 GB", | 20260507 17:07:49.619 | INFO | "pinned" : " 29 GB" | 20260507 17:07:49.619 | INFO | } | 20260507 17:07:49.619 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:07:49.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:49.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.620 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 17:07:49.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.620 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 17:07:49.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:49.620 | INFO | Uploaded 36GiB to GPU in 2.35524s (15.2851GiB/s) | 20260507 17:07:49.620 | INFO | Downloaded 34GiB to GPU in 2.45828s (13.8308GiB/s) | 20260507 17:07:54.758 | INFO | Analyzing chunk frames = [1169920,1170432) | 20260507 17:07:54.758 | INFO | MemInfo: { | 20260507 17:07:54.758 | INFO | "gpu" : " 17 GB", | 20260507 17:07:54.758 | INFO | "huge" : "", | 20260507 17:07:54.758 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:54.758 | INFO | "malloc" : " 14 GB", | 20260507 17:07:54.758 | INFO | "pinned" : " 29 GB" | 20260507 17:07:54.758 | INFO | } | 20260507 17:07:54.758 | INFO | Perf report for the last 5.13893 seconds: | 20260507 17:07:54.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:54.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.758 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:07:54.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.759 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:07:54.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:54.759 | INFO | Uploaded 36GiB to GPU in 2.35158s (15.3088GiB/s) | 20260507 17:07:54.759 | INFO | Downloaded 34GiB to GPU in 2.4414s (13.9264GiB/s) | 20260507 17:07:59.898 | INFO | Analyzing chunk frames = [1170432,1170944) | 20260507 17:07:59.898 | INFO | MemInfo: { | 20260507 17:07:59.898 | INFO | "gpu" : " 17 GB", | 20260507 17:07:59.898 | INFO | "huge" : "", | 20260507 17:07:59.898 | INFO | "huge_shared" : " 29 GB", | 20260507 17:07:59.898 | INFO | "malloc" : " 14 GB", | 20260507 17:07:59.898 | INFO | "pinned" : " 29 GB" | 20260507 17:07:59.898 | INFO | } | 20260507 17:07:59.898 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:07:59.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:07:59.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.898 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:07:59.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.898 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:07:59.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:07:59.899 | INFO | Uploaded 36GiB to GPU in 2.4352s (14.7832GiB/s) | 20260507 17:07:59.899 | INFO | Downloaded 34GiB to GPU in 2.41068s (14.1039GiB/s) | 20260507 17:08:05.037 | INFO | Analyzing chunk frames = [1170944,1171456) | 20260507 17:08:05.037 | INFO | MemInfo: { | 20260507 17:08:05.037 | INFO | "gpu" : " 17 GB", | 20260507 17:08:05.037 | INFO | "huge" : "", | 20260507 17:08:05.037 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:05.037 | INFO | "malloc" : " 14 GB", | 20260507 17:08:05.038 | INFO | "pinned" : " 29 GB" | 20260507 17:08:05.038 | INFO | } | 20260507 17:08:05.038 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:08:05.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:05.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:08:05.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:08:05.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:05.038 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:08:05.038 | INFO | Total mem usage: 16.4 GiB | 20260507 17:08:05.038 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:08:05.038 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.14%) | 20260507 17:08:05.038 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:08:05.038 | INFO | High Water Breakdown: | 20260507 17:08:05.038 | INFO | 59.7% : 8.97 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:08:05.038 | INFO | 40.2% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:08:05.038 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:08:05.038 | INFO | ------------------------------------------ | 20260507 17:08:05.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:08:05.038 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:08:05.038 | INFO | Total mem usage: 26.5 GiB | 20260507 17:08:05.038 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:08:05.038 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:08:05.038 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:08:05.038 | INFO | High Water Breakdown: | 20260507 17:08:05.038 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:08:05.038 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:08:05.038 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:08:05.038 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:08:05.038 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:08:05.038 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:08:05.038 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:08:05.038 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:08:05.038 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:08:05.038 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:08:05.038 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:08:05.038 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:08:05.038 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:08:05.038 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:08:05.038 | INFO | ------------------------------------------ | 20260507 17:08:05.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:08:05.038 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:08:05.038 | INFO | Total mem usage: 27.8 GiB | 20260507 17:08:05.038 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:08:05.038 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:08:05.038 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:08:05.038 | INFO | Uploaded 36GiB to GPU in 2.53438s (14.2047GiB/s) | 20260507 17:08:05.039 | INFO | Downloaded 34GiB to GPU in 2.5173s (13.5065GiB/s) | 20260507 17:08:05.038 | INFO | High Water Breakdown: | 20260507 17:08:05.039 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:08:05.039 | INFO | ------------------------------------------ | 20260507 17:08:05.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:08:05.039 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:08:05.039 | INFO | Total mem usage: 15.7 GiB | 20260507 17:08:05.039 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:08:05.039 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:08:05.039 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:08:05.039 | INFO | High Water Breakdown: | 20260507 17:08:05.039 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:08:05.039 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:08:05.039 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:08:05.039 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:08:05.039 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:08:05.039 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:08:05.039 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:08:05.039 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:08:05.039 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:08:05.039 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:08:05.039 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:08:05.039 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:08:05.039 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:08:05.039 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:08:05.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:08:05.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:08:05.039 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:08:05.039 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:08:05.039 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:08:05.039 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:08:05.039 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:08:05.039 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:08:05.039 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:08:05.039 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:08:05.039 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:08:05.039 | INFO | ------------------------------------------ | 20260507 17:08:05.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:08:05.039 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:08:05.039 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:08:05.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:08:05.039 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.68e+01, 1.43e+01 / 2.97e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.11e-02 / 5.68e-03 | 20260507 17:08:05.039 | INFO | PerfReport: 12.06%, Baseline -- 4.05e+00 / 4.51e+00, 6.35e-01 / 2.51e-01 | 20260507 17:08:05.039 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.50e+00, 2.19e-01 / 1.05e-01 | 20260507 17:08:05.039 | INFO | PerfReport: 15.73%, DME -- 4.23e+01 / 4.54e+01, 4.55e+00 / 2.72e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.91e+00 / 2.13e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.54e+00 / 2.90e+00, 4.44e-01 / 2.61e-01 | 20260507 17:08:05.039 | INFO | PerfReport: 3.80%, Metrics -- 1.28e+00 / 1.46e+00, 3.25e-01 / 1.97e-01 | 20260507 17:08:05.039 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.29e-01, 6.85e-02 / 4.19e-02 | 20260507 17:08:05.039 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:08:05.039 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:08:05.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:08:05.039 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 23.91%, Upload -- 2.57e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 23.89%, Download -- 2.57e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 31.31%, Compute -- 3.36e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 1.82%, LockUpload -- 1.96e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 2.22%, LockDownload -- 2.39e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 16.83%, LockCompute -- 1.81e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:08:05.039 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260507 17:08:05.135 | INFO | Starting a round of Baz IO | 20260507 17:08:08.645 | INFO | Populating RTMetrics report | 20260507 17:08:08.645 | INFO | RTMetrics report populated | 20260507 17:08:08.645 | INFO | Opening tmp RTMetrics json file | 20260507 17:08:08.668 | INFO | Serializing RTMetrics to json | 20260507 17:08:08.670 | INFO | Serialization done, swapping tmp file now... | 20260507 17:08:08.670 | INFO | RTMetrics done | 20260507 17:08:08.750 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.615 seconds | 20260507 17:08:08.750 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.75% / 0.02% / 51.23% | 20260507 17:08:08.750 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.85687% / 4.76837e-05% / 2.78155e-05% | 20260507 17:08:08.750 | INFO | Baz IO overall: | 20260507 17:08:08.750 | INFO | BytesWritten : 511890814363 | 20260507 17:08:08.750 | INFO | NumEvents : 247930147691 | 20260507 17:08:10.176 | INFO | Analyzing chunk frames = [1171456,1171968) | 20260507 17:08:10.177 | INFO | MemInfo: { | 20260507 17:08:10.177 | INFO | "gpu" : " 17 GB", | 20260507 17:08:10.177 | INFO | "huge" : "", | 20260507 17:08:10.177 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:10.177 | INFO | "malloc" : " 14 GB", | 20260507 17:08:10.177 | INFO | "pinned" : " 29 GB" | 20260507 17:08:10.177 | INFO | } | 20260507 17:08:10.177 | INFO | Perf report for the last 5.13889 seconds: | 20260507 17:08:10.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:10.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:10.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:10.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:10.177 | INFO | ANALYSIS : 69.7%, 2.9, 107.7ms, 0 | 20260507 17:08:10.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:10.177 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 17:08:10.177 | INFO | PRE_HQ : 66.5%, 1.7, 59.8ms, 0 | 20260507 17:08:10.177 | INFO | BAZWRITER : 69.9%, 4.1, 146.3ms, 0 | 20260507 17:08:10.177 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 17:08:10.177 | INFO | Uploaded 36GiB to GPU in 2.53896s (14.179GiB/s) | 20260507 17:08:10.177 | INFO | Downloaded 38GiB to GPU in 2.71341s (14.0045GiB/s) | 20260507 17:08:15.316 | INFO | Analyzing chunk frames = [1171968,1172480) | 20260507 17:08:15.316 | INFO | MemInfo: { | 20260507 17:08:15.316 | INFO | "gpu" : " 17 GB", | 20260507 17:08:15.316 | INFO | "huge" : "", | 20260507 17:08:15.316 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:15.316 | INFO | "malloc" : " 14 GB", | 20260507 17:08:15.316 | INFO | "pinned" : " 29 GB" | 20260507 17:08:15.316 | INFO | } | 20260507 17:08:15.316 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:08:15.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:15.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.316 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260507 17:08:15.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.316 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:08:15.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:15.317 | INFO | Uploaded 36GiB to GPU in 2.46622s (14.5972GiB/s) | 20260507 17:08:15.317 | INFO | Downloaded 34GiB to GPU in 2.45477s (13.8506GiB/s) | 20260507 17:08:20.455 | INFO | Analyzing chunk frames = [1172480,1172992) | 20260507 17:08:20.455 | INFO | MemInfo: { | 20260507 17:08:20.455 | INFO | "gpu" : " 17 GB", | 20260507 17:08:20.455 | INFO | "huge" : "", | 20260507 17:08:20.455 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:20.455 | INFO | "malloc" : " 14 GB", | 20260507 17:08:20.455 | INFO | "pinned" : " 29 GB" | 20260507 17:08:20.455 | INFO | } | 20260507 17:08:20.455 | INFO | Perf report for the last 5.13907 seconds: | 20260507 17:08:20.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:20.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.455 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:08:20.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.455 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:08:20.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:20.456 | INFO | Uploaded 36GiB to GPU in 2.51889s (14.292GiB/s) | 20260507 17:08:20.456 | INFO | Downloaded 34GiB to GPU in 2.43912s (13.9395GiB/s) | 20260507 17:08:25.594 | INFO | Analyzing chunk frames = [1172992,1173504) | 20260507 17:08:25.594 | INFO | MemInfo: { | 20260507 17:08:25.594 | INFO | "gpu" : " 17 GB", | 20260507 17:08:25.594 | INFO | "huge" : "", | 20260507 17:08:25.594 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:25.594 | INFO | "malloc" : " 14 GB", | 20260507 17:08:25.594 | INFO | "pinned" : " 29 GB" | 20260507 17:08:25.594 | INFO | } | 20260507 17:08:25.594 | INFO | Perf report for the last 5.13913 seconds: | 20260507 17:08:25.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:25.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.595 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:08:25.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.595 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:08:25.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:25.595 | INFO | Uploaded 36GiB to GPU in 2.44747s (14.7091GiB/s) | 20260507 17:08:25.595 | INFO | Downloaded 34GiB to GPU in 2.36126s (14.3991GiB/s) | 20260507 17:08:30.734 | INFO | Analyzing chunk frames = [1173504,1174016) | 20260507 17:08:30.734 | INFO | MemInfo: { | 20260507 17:08:30.734 | INFO | "gpu" : " 17 GB", | 20260507 17:08:30.734 | INFO | "huge" : "", | 20260507 17:08:30.734 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:30.734 | INFO | "malloc" : " 14 GB", | 20260507 17:08:30.734 | INFO | "pinned" : " 29 GB" | 20260507 17:08:30.734 | INFO | } | 20260507 17:08:30.734 | INFO | Perf report for the last 5.13968 seconds: | 20260507 17:08:30.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:30.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.734 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:08:30.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.734 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 17:08:30.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:30.735 | INFO | Uploaded 36GiB to GPU in 2.35516s (15.2856GiB/s) | 20260507 17:08:30.735 | INFO | Downloaded 34GiB to GPU in 2.49249s (13.641GiB/s) | 20260507 17:08:35.873 | INFO | Analyzing chunk frames = [1174016,1174528) | 20260507 17:08:35.873 | INFO | MemInfo: { | 20260507 17:08:35.873 | INFO | "gpu" : " 17 GB", | 20260507 17:08:35.873 | INFO | "huge" : "", | 20260507 17:08:35.873 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:35.873 | INFO | "malloc" : " 14 GB", | 20260507 17:08:35.873 | INFO | "pinned" : " 29 GB" | 20260507 17:08:35.873 | INFO | } | 20260507 17:08:35.873 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:08:35.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:35.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.873 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:08:35.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.873 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:08:35.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:35.874 | INFO | Uploaded 36GiB to GPU in 2.32759s (15.4667GiB/s) | 20260507 17:08:35.874 | INFO | Downloaded 34GiB to GPU in 2.46687s (13.7827GiB/s) | 20260507 17:08:41.012 | INFO | Analyzing chunk frames = [1174528,1175040) | 20260507 17:08:41.012 | INFO | MemInfo: { | 20260507 17:08:41.012 | INFO | "gpu" : " 17 GB", | 20260507 17:08:41.012 | INFO | "huge" : "", | 20260507 17:08:41.012 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:41.013 | INFO | "malloc" : " 14 GB", | 20260507 17:08:41.013 | INFO | "pinned" : " 29 GB" | 20260507 17:08:41.013 | INFO | } | 20260507 17:08:41.013 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:08:41.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:41.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:08:41.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:08:41.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:41.013 | INFO | Uploaded 36GiB to GPU in 2.46051s (14.6311GiB/s) | 20260507 17:08:41.013 | INFO | Downloaded 34GiB to GPU in 2.42303s (14.032GiB/s) | 20260507 17:08:46.152 | INFO | Analyzing chunk frames = [1175040,1175552) | 20260507 17:08:46.152 | INFO | MemInfo: { | 20260507 17:08:46.152 | INFO | "gpu" : " 17 GB", | 20260507 17:08:46.152 | INFO | "huge" : "", | 20260507 17:08:46.152 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:46.152 | INFO | "malloc" : " 14 GB", | 20260507 17:08:46.152 | INFO | "pinned" : " 29 GB" | 20260507 17:08:46.152 | INFO | } | 20260507 17:08:46.152 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:08:46.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:46.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:08:46.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:08:46.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:46.152 | INFO | Uploaded 36GiB to GPU in 2.47178s (14.5644GiB/s) | 20260507 17:08:46.152 | INFO | Downloaded 34GiB to GPU in 2.50695s (13.5623GiB/s) | 20260507 17:08:46.249 | INFO | Starting a round of Baz IO | 20260507 17:08:49.765 | INFO | Populating RTMetrics report | 20260507 17:08:49.766 | INFO | RTMetrics report populated | 20260507 17:08:49.766 | INFO | Opening tmp RTMetrics json file | 20260507 17:08:49.770 | INFO | Serializing RTMetrics to json | 20260507 17:08:49.785 | INFO | Serialization done, swapping tmp file now... | 20260507 17:08:49.785 | INFO | RTMetrics done | 20260507 17:08:49.855 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.605 seconds | 20260507 17:08:49.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.72% / 0.02% / 51.25% | 20260507 17:08:49.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.85143% / 3.97364e-05% / 3.97364e-05% | 20260507 17:08:49.855 | INFO | Baz IO overall: | 20260507 17:08:49.855 | INFO | BytesWritten : 513819915417 | 20260507 17:08:49.855 | INFO | NumEvents : 248867991885 | 20260507 17:08:51.291 | INFO | Analyzing chunk frames = [1175552,1176064) | 20260507 17:08:51.291 | INFO | MemInfo: { | 20260507 17:08:51.291 | INFO | "gpu" : " 17 GB", | 20260507 17:08:51.291 | INFO | "huge" : "", | 20260507 17:08:51.291 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:51.291 | INFO | "malloc" : " 14 GB", | 20260507 17:08:51.291 | INFO | "pinned" : " 29 GB" | 20260507 17:08:51.291 | INFO | } | 20260507 17:08:51.291 | INFO | Perf report for the last 5.13902 seconds: | 20260507 17:08:51.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:51.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:51.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:51.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:51.291 | INFO | ANALYSIS : 69.9%, 2.9, 107.8ms, 0 | 20260507 17:08:51.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:51.291 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 17:08:51.291 | INFO | PRE_HQ : 66.8%, 1.7, 59.4ms, 0 | 20260507 17:08:51.291 | INFO | BAZWRITER : 69.8%, 3.8, 138.0ms, 0 | 20260507 17:08:51.291 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 17:08:51.291 | INFO | Uploaded 36GiB to GPU in 2.5499s (14.1182GiB/s) | 20260507 17:08:51.291 | INFO | Downloaded 38GiB to GPU in 2.72082s (13.9664GiB/s) | 20260507 17:08:56.430 | INFO | Analyzing chunk frames = [1176064,1176576) | 20260507 17:08:56.430 | INFO | MemInfo: { | 20260507 17:08:56.430 | INFO | "gpu" : " 17 GB", | 20260507 17:08:56.430 | INFO | "huge" : "", | 20260507 17:08:56.430 | INFO | "huge_shared" : " 29 GB", | 20260507 17:08:56.430 | INFO | "malloc" : " 14 GB", | 20260507 17:08:56.430 | INFO | "pinned" : " 29 GB" | 20260507 17:08:56.430 | INFO | } | 20260507 17:08:56.430 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:08:56.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:08:56.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.431 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:08:56.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.431 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:08:56.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:08:56.431 | INFO | Uploaded 36GiB to GPU in 2.52474s (14.2589GiB/s) | 20260507 17:08:56.431 | INFO | Downloaded 34GiB to GPU in 2.4231s (14.0316GiB/s) | 20260507 17:09:01.570 | INFO | Analyzing chunk frames = [1176576,1177088) | 20260507 17:09:01.570 | INFO | MemInfo: { | 20260507 17:09:01.570 | INFO | "gpu" : " 17 GB", | 20260507 17:09:01.570 | INFO | "huge" : "", | 20260507 17:09:01.570 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:01.570 | INFO | "malloc" : " 14 GB", | 20260507 17:09:01.570 | INFO | "pinned" : " 29 GB" | 20260507 17:09:01.570 | INFO | } | 20260507 17:09:01.570 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:09:01.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:01.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.570 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 17:09:01.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.570 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:09:01.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:01.571 | INFO | Uploaded 36GiB to GPU in 2.48024s (14.5147GiB/s) | 20260507 17:09:01.571 | INFO | Downloaded 34GiB to GPU in 2.32952s (14.5953GiB/s) | 20260507 17:09:06.709 | INFO | Analyzing chunk frames = [1177088,1177600) | 20260507 17:09:06.709 | INFO | MemInfo: { | 20260507 17:09:06.709 | INFO | "gpu" : " 17 GB", | 20260507 17:09:06.709 | INFO | "huge" : "", | 20260507 17:09:06.709 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:06.709 | INFO | "malloc" : " 14 GB", | 20260507 17:09:06.709 | INFO | "pinned" : " 29 GB" | 20260507 17:09:06.709 | INFO | } | 20260507 17:09:06.709 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:09:06.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:06.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.709 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260507 17:09:06.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.709 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:09:06.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:06.710 | INFO | Uploaded 36GiB to GPU in 2.5012s (14.3931GiB/s) | 20260507 17:09:06.710 | INFO | Downloaded 34GiB to GPU in 2.33769s (14.5443GiB/s) | 20260507 17:09:11.848 | INFO | Analyzing chunk frames = [1177600,1178112) | 20260507 17:09:11.848 | INFO | MemInfo: { | 20260507 17:09:11.848 | INFO | "gpu" : " 17 GB", | 20260507 17:09:11.848 | INFO | "huge" : "", | 20260507 17:09:11.848 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:11.849 | INFO | "malloc" : " 14 GB", | 20260507 17:09:11.849 | INFO | "pinned" : " 29 GB" | 20260507 17:09:11.849 | INFO | } | 20260507 17:09:11.849 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:09:11.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:11.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:09:11.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:09:11.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:11.849 | INFO | Uploaded 36GiB to GPU in 2.36722s (15.2077GiB/s) | 20260507 17:09:11.849 | INFO | Downloaded 34GiB to GPU in 2.46059s (13.8178GiB/s) | 20260507 17:09:16.988 | INFO | Analyzing chunk frames = [1178112,1178624) | 20260507 17:09:16.988 | INFO | MemInfo: { | 20260507 17:09:16.988 | INFO | "gpu" : " 17 GB", | 20260507 17:09:16.988 | INFO | "huge" : "", | 20260507 17:09:16.988 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:16.988 | INFO | "malloc" : " 14 GB", | 20260507 17:09:16.988 | INFO | "pinned" : " 29 GB" | 20260507 17:09:16.988 | INFO | } | 20260507 17:09:16.988 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:09:16.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:16.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.988 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:09:16.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.988 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 17:09:16.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:16.989 | INFO | Uploaded 36GiB to GPU in 2.35845s (15.2643GiB/s) | 20260507 17:09:16.989 | INFO | Downloaded 34GiB to GPU in 2.47122s (13.7584GiB/s) | 20260507 17:09:22.127 | INFO | Analyzing chunk frames = [1178624,1179136) | 20260507 17:09:22.127 | INFO | MemInfo: { | 20260507 17:09:22.127 | INFO | "gpu" : " 17 GB", | 20260507 17:09:22.127 | INFO | "huge" : "", | 20260507 17:09:22.127 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:22.127 | INFO | "malloc" : " 14 GB", | 20260507 17:09:22.127 | INFO | "pinned" : " 29 GB" | 20260507 17:09:22.127 | INFO | } | 20260507 17:09:22.128 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:09:22.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:22.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:09:22.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:09:22.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:22.128 | INFO | Uploaded 36GiB to GPU in 2.4108s (14.9328GiB/s) | 20260507 17:09:22.128 | INFO | Downloaded 34GiB to GPU in 2.41598s (14.0729GiB/s) | 20260507 17:09:27.266 | INFO | Analyzing chunk frames = [1179136,1179648) | 20260507 17:09:27.266 | INFO | MemInfo: { | 20260507 17:09:27.266 | INFO | "gpu" : " 17 GB", | 20260507 17:09:27.266 | INFO | "huge" : "", | 20260507 17:09:27.267 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:27.267 | INFO | "malloc" : " 14 GB", | 20260507 17:09:27.267 | INFO | "pinned" : " 29 GB" | 20260507 17:09:27.267 | INFO | } | 20260507 17:09:27.267 | INFO | Perf report for the last 5.1391 seconds: | 20260507 17:09:27.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:27.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:09:27.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:09:27.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:27.267 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:09:27.267 | INFO | Total mem usage: 16.4 GiB | 20260507 17:09:27.267 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:09:27.267 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.08%) | 20260507 17:09:27.267 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:09:27.267 | INFO | High Water Breakdown: | 20260507 17:09:27.267 | INFO | 59.7% : 8.98 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:09:27.267 | INFO | 40.2% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:09:27.267 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:09:27.267 | INFO | ------------------------------------------ | 20260507 17:09:27.267 | INFO | Uploaded 36GiB to GPU in 2.49897s (14.4059GiB/s) | 20260507 17:09:27.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:09:27.268 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:09:27.268 | INFO | Total mem usage: 26.5 GiB | 20260507 17:09:27.268 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:09:27.268 | INFO | Downloaded 34GiB to GPU in 2.50147s (13.592GiB/s) | 20260507 17:09:27.268 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:09:27.268 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:09:27.268 | INFO | High Water Breakdown: | 20260507 17:09:27.268 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:09:27.268 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:09:27.268 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:09:27.268 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:09:27.268 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:09:27.268 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:09:27.268 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:09:27.268 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:09:27.268 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:09:27.268 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:09:27.268 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:09:27.268 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:09:27.268 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:09:27.268 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:09:27.268 | INFO | ------------------------------------------ | 20260507 17:09:27.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:09:27.268 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:09:27.268 | INFO | Total mem usage: 27.8 GiB | 20260507 17:09:27.268 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:09:27.268 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:09:27.268 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:09:27.268 | INFO | High Water Breakdown: | 20260507 17:09:27.268 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:09:27.268 | INFO | ------------------------------------------ | 20260507 17:09:27.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:09:27.268 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:09:27.268 | INFO | Total mem usage: 15.7 GiB | 20260507 17:09:27.268 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:09:27.268 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:09:27.268 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:09:27.268 | INFO | High Water Breakdown: | 20260507 17:09:27.268 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:09:27.268 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:09:27.268 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:09:27.268 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:09:27.268 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:09:27.268 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:09:27.268 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:09:27.268 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:09:27.268 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:09:27.268 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:09:27.268 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:09:27.268 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:09:27.268 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:09:27.268 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:09:27.268 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:09:27.268 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:09:27.268 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:09:27.268 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:09:27.268 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:09:27.268 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:09:27.268 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:09:27.268 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:09:27.268 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:09:27.268 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:09:27.268 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:09:27.268 | INFO | ------------------------------------------ | 20260507 17:09:27.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:09:27.268 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:09:27.268 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:09:27.268 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:09:27.268 | INFO | PerfReport: Full Execution -- 3.37e+01 / 7.67e+01, 1.44e+01 / 2.71e+00 | 20260507 17:09:27.268 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.07e-02 / 5.68e-03 | 20260507 17:09:27.268 | INFO | PerfReport: 12.09%, Baseline -- 4.07e+00 / 4.54e+00, 6.26e-01 / 1.60e-01 | 20260507 17:09:27.268 | INFO | PerfReport: 3.43%, Histogram -- 1.16e+00 / 1.51e+00, 2.19e-01 / 9.27e-02 | 20260507 17:09:27.269 | INFO | PerfReport: 15.71%, DME -- 4.23e+01 / 4.53e+01, 4.36e+00 / 2.71e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.90e+00 / 2.26e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.53e+00 / 2.89e+00, 4.37e-01 / 2.44e-01 | 20260507 17:09:27.269 | INFO | PerfReport: 3.80%, Metrics -- 1.28e+00 / 1.45e+00, 3.26e-01 / 1.90e-01 | 20260507 17:09:27.269 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.29e-01, 6.85e-02 / 4.41e-02 | 20260507 17:09:27.269 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:09:27.269 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:09:27.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:09:27.269 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 23.81%, Upload -- 2.56e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 23.96%, Download -- 2.57e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 31.34%, Compute -- 3.37e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 1.70%, LockUpload -- 1.83e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 2.32%, LockDownload -- 2.49e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 16.86%, LockCompute -- 1.81e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:09:27.269 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.79e-02 / 0.00e+00 | 20260507 17:09:27.364 | INFO | Starting a round of Baz IO | 20260507 17:09:30.874 | INFO | Populating RTMetrics report | 20260507 17:09:30.875 | INFO | RTMetrics report populated | 20260507 17:09:30.875 | INFO | Opening tmp RTMetrics json file | 20260507 17:09:30.938 | INFO | Serializing RTMetrics to json | 20260507 17:09:30.976 | INFO | Serialization done, swapping tmp file now... | 20260507 17:09:30.984 | INFO | RTMetrics done | 20260507 17:09:31.012 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.647 seconds | 20260507 17:09:31.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.70% / 0.02% / 51.28% | 20260507 17:09:31.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.83452% / 4.37101e-05% / 2.78155e-05% | 20260507 17:09:31.012 | INFO | Baz IO overall: | 20260507 17:09:31.012 | INFO | BytesWritten : 515747938924 | 20260507 17:09:31.012 | INFO | NumEvents : 249804973723 | 20260507 17:09:32.406 | INFO | Analyzing chunk frames = [1179648,1180160) | 20260507 17:09:32.406 | INFO | MemInfo: { | 20260507 17:09:32.406 | INFO | "gpu" : " 17 GB", | 20260507 17:09:32.406 | INFO | "huge" : "", | 20260507 17:09:32.406 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:32.406 | INFO | "malloc" : " 14 GB", | 20260507 17:09:32.406 | INFO | "pinned" : " 29 GB" | 20260507 17:09:32.406 | INFO | } | 20260507 17:09:32.406 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:09:32.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:32.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:32.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:32.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:32.407 | INFO | ANALYSIS : 69.7%, 3.0, 108.4ms, 0 | 20260507 17:09:32.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:32.407 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260507 17:09:32.407 | INFO | PRE_HQ : 66.8%, 1.8, 60.8ms, 0 | 20260507 17:09:32.407 | INFO | BAZWRITER : 70.5%, 3.6, 129.6ms, 0 | 20260507 17:09:32.407 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 17:09:32.407 | INFO | Uploaded 36GiB to GPU in 2.53236s (14.216GiB/s) | 20260507 17:09:32.407 | INFO | Downloaded 38GiB to GPU in 2.73147s (13.9119GiB/s) | 20260507 17:09:37.545 | INFO | Analyzing chunk frames = [1180160,1180672) | 20260507 17:09:37.545 | INFO | MemInfo: { | 20260507 17:09:37.545 | INFO | "gpu" : " 17 GB", | 20260507 17:09:37.545 | INFO | "huge" : "", | 20260507 17:09:37.545 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:37.545 | INFO | "malloc" : " 14 GB", | 20260507 17:09:37.545 | INFO | "pinned" : " 29 GB" | 20260507 17:09:37.545 | INFO | } | 20260507 17:09:37.545 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:09:37.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:37.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.545 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:09:37.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.545 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:09:37.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:37.546 | INFO | Uploaded 36GiB to GPU in 2.4561s (14.6574GiB/s) | 20260507 17:09:37.546 | INFO | Downloaded 34GiB to GPU in 2.44973s (13.8791GiB/s) | 20260507 17:09:42.684 | INFO | Analyzing chunk frames = [1180672,1181184) | 20260507 17:09:42.684 | INFO | MemInfo: { | 20260507 17:09:42.684 | INFO | "gpu" : " 17 GB", | 20260507 17:09:42.684 | INFO | "huge" : "", | 20260507 17:09:42.684 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:42.684 | INFO | "malloc" : " 14 GB", | 20260507 17:09:42.684 | INFO | "pinned" : " 29 GB" | 20260507 17:09:42.684 | INFO | } | 20260507 17:09:42.684 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:09:42.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:42.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.685 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260507 17:09:42.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.685 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:09:42.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:42.685 | INFO | Uploaded 36GiB to GPU in 2.52144s (14.2775GiB/s) | 20260507 17:09:42.685 | INFO | Downloaded 34GiB to GPU in 2.43742s (13.9492GiB/s) | 20260507 17:09:47.824 | INFO | Analyzing chunk frames = [1181184,1181696) | 20260507 17:09:47.824 | INFO | MemInfo: { | 20260507 17:09:47.824 | INFO | "gpu" : " 17 GB", | 20260507 17:09:47.824 | INFO | "huge" : "", | 20260507 17:09:47.824 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:47.824 | INFO | "malloc" : " 14 GB", | 20260507 17:09:47.824 | INFO | "pinned" : " 29 GB" | 20260507 17:09:47.824 | INFO | } | 20260507 17:09:47.824 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:09:47.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:47.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:09:47.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:09:47.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:47.824 | INFO | Uploaded 36GiB to GPU in 2.44019s (14.753GiB/s) | 20260507 17:09:47.824 | INFO | Downloaded 34GiB to GPU in 2.36289s (14.3891GiB/s) | 20260507 17:09:52.964 | INFO | Analyzing chunk frames = [1181696,1182208) | 20260507 17:09:52.964 | INFO | MemInfo: { | 20260507 17:09:52.964 | INFO | "gpu" : " 17 GB", | 20260507 17:09:52.964 | INFO | "huge" : "", | 20260507 17:09:52.964 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:52.964 | INFO | "malloc" : " 14 GB", | 20260507 17:09:52.964 | INFO | "pinned" : " 29 GB" | 20260507 17:09:52.964 | INFO | } | 20260507 17:09:52.964 | INFO | Perf report for the last 5.13989 seconds: | 20260507 17:09:52.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:52.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.964 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 17:09:52.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.964 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 17:09:52.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:52.965 | INFO | Uploaded 36GiB to GPU in 2.29752s (15.669GiB/s) | 20260507 17:09:52.965 | INFO | Downloaded 34GiB to GPU in 2.47648s (13.7292GiB/s) | 20260507 17:09:58.103 | INFO | Analyzing chunk frames = [1182208,1182720) | 20260507 17:09:58.103 | INFO | MemInfo: { | 20260507 17:09:58.103 | INFO | "gpu" : " 17 GB", | 20260507 17:09:58.103 | INFO | "huge" : "", | 20260507 17:09:58.103 | INFO | "huge_shared" : " 29 GB", | 20260507 17:09:58.103 | INFO | "malloc" : " 14 GB", | 20260507 17:09:58.103 | INFO | "pinned" : " 29 GB" | 20260507 17:09:58.103 | INFO | } | 20260507 17:09:58.103 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:09:58.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:09:58.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:09:58.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 17:09:58.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:09:58.103 | INFO | Uploaded 36GiB to GPU in 2.36331s (15.2329GiB/s) | 20260507 17:09:58.103 | INFO | Downloaded 34GiB to GPU in 2.51178s (13.5362GiB/s) | 20260507 17:10:03.241 | INFO | Analyzing chunk frames = [1182720,1183232) | 20260507 17:10:03.241 | INFO | MemInfo: { | 20260507 17:10:03.241 | INFO | "gpu" : " 17 GB", | 20260507 17:10:03.241 | INFO | "huge" : "", | 20260507 17:10:03.241 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:03.241 | INFO | "malloc" : " 14 GB", | 20260507 17:10:03.241 | INFO | "pinned" : " 29 GB" | 20260507 17:10:03.241 | INFO | } | 20260507 17:10:03.242 | INFO | Perf report for the last 5.13887 seconds: | 20260507 17:10:03.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:03.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:10:03.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:10:03.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:03.242 | INFO | Uploaded 36GiB to GPU in 2.42545s (14.8426GiB/s) | 20260507 17:10:03.242 | INFO | Downloaded 34GiB to GPU in 2.427s (14.0091GiB/s) | 20260507 17:10:08.381 | INFO | Analyzing chunk frames = [1183232,1183744) | 20260507 17:10:08.381 | INFO | MemInfo: { | 20260507 17:10:08.381 | INFO | "gpu" : " 17 GB", | 20260507 17:10:08.381 | INFO | "huge" : "", | 20260507 17:10:08.381 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:08.381 | INFO | "malloc" : " 14 GB", | 20260507 17:10:08.381 | INFO | "pinned" : " 29 GB" | 20260507 17:10:08.381 | INFO | } | 20260507 17:10:08.381 | INFO | Perf report for the last 5.13956 seconds: | 20260507 17:10:08.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:08.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.381 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 17:10:08.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.381 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:10:08.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:08.382 | INFO | Uploaded 36GiB to GPU in 2.44967s (14.6959GiB/s) | 20260507 17:10:08.382 | INFO | Downloaded 34GiB to GPU in 2.51185s (13.5358GiB/s) | 20260507 17:10:08.481 | INFO | Starting a round of Baz IO | 20260507 17:10:12.020 | INFO | Populating RTMetrics report | 20260507 17:10:12.021 | INFO | RTMetrics report populated | 20260507 17:10:12.021 | INFO | Opening tmp RTMetrics json file | 20260507 17:10:12.022 | INFO | Serializing RTMetrics to json | 20260507 17:10:12.023 | INFO | Serialization done, swapping tmp file now... | 20260507 17:10:12.024 | INFO | RTMetrics done | 20260507 17:10:12.161 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.679 seconds | 20260507 17:10:12.161 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.68% / 0.02% / 51.30% | 20260507 17:10:12.161 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.8225% / 3.97364e-05% / 1.19209e-05% | 20260507 17:10:12.161 | INFO | Baz IO overall: | 20260507 17:10:12.161 | INFO | BytesWritten : 517676682436 | 20260507 17:10:12.161 | INFO | NumEvents : 250742231979 | 20260507 17:10:13.520 | INFO | Analyzing chunk frames = [1183744,1184256) | 20260507 17:10:13.520 | INFO | MemInfo: { | 20260507 17:10:13.520 | INFO | "gpu" : " 17 GB", | 20260507 17:10:13.520 | INFO | "huge" : "", | 20260507 17:10:13.520 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:13.520 | INFO | "malloc" : " 14 GB", | 20260507 17:10:13.520 | INFO | "pinned" : " 29 GB" | 20260507 17:10:13.520 | INFO | } | 20260507 17:10:13.520 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:10:13.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:13.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:13.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:13.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:13.520 | INFO | ANALYSIS : 69.8%, 3.0, 108.2ms, 0 | 20260507 17:10:13.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:13.520 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 17:10:13.520 | INFO | PRE_HQ : 67.7%, 1.7, 61.4ms, 0 | 20260507 17:10:13.521 | INFO | BAZWRITER : 71.1%, 3.7, 135.1ms, 0 | 20260507 17:10:13.521 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 17:10:13.521 | INFO | Uploaded 36GiB to GPU in 2.51731s (14.301GiB/s) | 20260507 17:10:13.521 | INFO | Downloaded 38GiB to GPU in 2.72239s (13.9583GiB/s) | 20260507 17:10:18.659 | INFO | Analyzing chunk frames = [1184256,1184768) | 20260507 17:10:18.659 | INFO | MemInfo: { | 20260507 17:10:18.659 | INFO | "gpu" : " 17 GB", | 20260507 17:10:18.659 | INFO | "huge" : "", | 20260507 17:10:18.659 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:18.659 | INFO | "malloc" : " 14 GB", | 20260507 17:10:18.659 | INFO | "pinned" : " 29 GB" | 20260507 17:10:18.660 | INFO | } | 20260507 17:10:18.660 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:10:18.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:18.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:10:18.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:10:18.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:18.660 | INFO | Uploaded 36GiB to GPU in 2.47006s (14.5745GiB/s) | 20260507 17:10:18.660 | INFO | Downloaded 34GiB to GPU in 2.45535s (13.8473GiB/s) | 20260507 17:10:23.799 | INFO | Analyzing chunk frames = [1184768,1185280) | 20260507 17:10:23.799 | INFO | MemInfo: { | 20260507 17:10:23.799 | INFO | "gpu" : " 17 GB", | 20260507 17:10:23.799 | INFO | "huge" : "", | 20260507 17:10:23.799 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:23.799 | INFO | "malloc" : " 14 GB", | 20260507 17:10:23.799 | INFO | "pinned" : " 29 GB" | 20260507 17:10:23.799 | INFO | } | 20260507 17:10:23.799 | INFO | Perf report for the last 5.13973 seconds: | 20260507 17:10:23.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:23.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.799 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:10:23.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.800 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:10:23.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:23.800 | INFO | Uploaded 36GiB to GPU in 2.52229s (14.2728GiB/s) | 20260507 17:10:23.800 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.975GiB/s) | 20260507 17:10:28.938 | INFO | Analyzing chunk frames = [1185280,1185792) | 20260507 17:10:28.938 | INFO | MemInfo: { | 20260507 17:10:28.938 | INFO | "gpu" : " 17 GB", | 20260507 17:10:28.938 | INFO | "huge" : "", | 20260507 17:10:28.938 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:28.938 | INFO | "malloc" : " 14 GB", | 20260507 17:10:28.938 | INFO | "pinned" : " 29 GB" | 20260507 17:10:28.938 | INFO | } | 20260507 17:10:28.938 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:10:28.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:28.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.938 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260507 17:10:28.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.939 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:10:28.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:28.939 | INFO | Uploaded 36GiB to GPU in 2.45991s (14.6347GiB/s) | 20260507 17:10:28.939 | INFO | Downloaded 34GiB to GPU in 2.36608s (14.3698GiB/s) | 20260507 17:10:34.078 | INFO | Analyzing chunk frames = [1185792,1186304) | 20260507 17:10:34.078 | INFO | MemInfo: { | 20260507 17:10:34.078 | INFO | "gpu" : " 17 GB", | 20260507 17:10:34.078 | INFO | "huge" : "", | 20260507 17:10:34.078 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:34.078 | INFO | "malloc" : " 14 GB", | 20260507 17:10:34.079 | INFO | "pinned" : " 29 GB" | 20260507 17:10:34.079 | INFO | } | 20260507 17:10:34.079 | INFO | Perf report for the last 5.14025 seconds: | 20260507 17:10:34.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:34.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 17:10:34.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 17:10:34.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:34.079 | INFO | Uploaded 36GiB to GPU in 2.33954s (15.3876GiB/s) | 20260507 17:10:34.079 | INFO | Downloaded 34GiB to GPU in 2.48163s (13.7007GiB/s) | 20260507 17:10:39.217 | INFO | Analyzing chunk frames = [1186304,1186816) | 20260507 17:10:39.217 | INFO | MemInfo: { | 20260507 17:10:39.217 | INFO | "gpu" : " 17 GB", | 20260507 17:10:39.217 | INFO | "huge" : "", | 20260507 17:10:39.217 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:39.217 | INFO | "malloc" : " 14 GB", | 20260507 17:10:39.217 | INFO | "pinned" : " 29 GB" | 20260507 17:10:39.217 | INFO | } | 20260507 17:10:39.217 | INFO | Perf report for the last 5.13814 seconds: | 20260507 17:10:39.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:39.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:10:39.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:10:39.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:39.217 | INFO | Uploaded 36GiB to GPU in 2.35133s (15.3105GiB/s) | 20260507 17:10:39.217 | INFO | Downloaded 34GiB to GPU in 2.46227s (13.8084GiB/s) | 20260507 17:10:44.356 | INFO | Analyzing chunk frames = [1186816,1187328) | 20260507 17:10:44.356 | INFO | MemInfo: { | 20260507 17:10:44.356 | INFO | "gpu" : " 17 GB", | 20260507 17:10:44.356 | INFO | "huge" : "", | 20260507 17:10:44.356 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:44.357 | INFO | "malloc" : " 14 GB", | 20260507 17:10:44.357 | INFO | "pinned" : " 29 GB" | 20260507 17:10:44.357 | INFO | } | 20260507 17:10:44.357 | INFO | Perf report for the last 5.1398 seconds: | 20260507 17:10:44.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:44.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 17:10:44.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:10:44.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:44.357 | INFO | Uploaded 36GiB to GPU in 2.39835s (15.0103GiB/s) | 20260507 17:10:44.357 | INFO | Downloaded 34GiB to GPU in 2.41379s (14.0857GiB/s) | 20260507 17:10:49.496 | INFO | Analyzing chunk frames = [1187328,1187840) | 20260507 17:10:49.496 | INFO | MemInfo: { | 20260507 17:10:49.496 | INFO | "gpu" : " 17 GB", | 20260507 17:10:49.496 | INFO | "huge" : "", | 20260507 17:10:49.496 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:49.496 | INFO | "malloc" : " 14 GB", | 20260507 17:10:49.496 | INFO | "pinned" : " 29 GB" | 20260507 17:10:49.496 | INFO | } | 20260507 17:10:49.496 | INFO | Perf report for the last 5.13933 seconds: | 20260507 17:10:49.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:49.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 17:10:49.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:10:49.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:49.496 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:10:49.496 | INFO | Total mem usage: 16.4 GiB | 20260507 17:10:49.496 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:10:49.496 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.4%) | 20260507 17:10:49.496 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:10:49.496 | INFO | High Water Breakdown: | 20260507 17:10:49.496 | INFO | 59.2% : 8.64 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:10:49.496 | INFO | 40.7% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:10:49.496 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:10:49.496 | INFO | ------------------------------------------ | 20260507 17:10:49.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:10:49.496 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:10:49.496 | INFO | Total mem usage: 26.5 GiB | 20260507 17:10:49.496 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:10:49.496 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:10:49.496 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:10:49.496 | INFO | High Water Breakdown: | 20260507 17:10:49.496 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:10:49.496 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:10:49.496 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:10:49.496 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:10:49.496 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:10:49.496 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:10:49.496 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:10:49.497 | INFO | Uploaded 36GiB to GPU in 2.49978s (14.4012GiB/s) | 20260507 17:10:49.496 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:10:49.497 | INFO | Downloaded 34GiB to GPU in 2.50795s (13.5569GiB/s) | 20260507 17:10:49.497 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:10:49.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:10:49.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:10:49.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:10:49.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:10:49.497 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:10:49.497 | INFO | ------------------------------------------ | 20260507 17:10:49.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:10:49.497 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:10:49.497 | INFO | Total mem usage: 27.8 GiB | 20260507 17:10:49.497 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:10:49.497 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 17:10:49.497 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 17:10:49.497 | INFO | High Water Breakdown: | 20260507 17:10:49.497 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 17:10:49.497 | INFO | ------------------------------------------ | 20260507 17:10:49.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:10:49.497 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:10:49.497 | INFO | Total mem usage: 15.7 GiB | 20260507 17:10:49.497 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:10:49.497 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:10:49.497 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 17:10:49.497 | INFO | High Water Breakdown: | 20260507 17:10:49.497 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:10:49.497 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:10:49.497 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:10:49.497 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:10:49.497 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:10:49.497 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:10:49.497 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:10:49.497 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:10:49.497 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:10:49.497 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:10:49.497 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:10:49.497 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:10:49.497 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:10:49.497 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:10:49.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:10:49.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:10:49.497 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:10:49.497 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:10:49.497 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:10:49.497 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:10:49.497 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:10:49.497 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:10:49.497 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:10:49.497 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:10:49.497 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:10:49.497 | INFO | ------------------------------------------ | 20260507 17:10:49.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:10:49.498 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:10:49.498 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:10:49.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:10:49.498 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.69e+01, 1.46e+01 / 2.99e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 0.29%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.09e-02 / 5.05e-03 | 20260507 17:10:49.498 | INFO | PerfReport: 12.07%, Baseline -- 4.06e+00 / 4.53e+00, 6.29e-01 / 2.10e-01 | 20260507 17:10:49.498 | INFO | PerfReport: 3.44%, Histogram -- 1.16e+00 / 1.51e+00, 2.18e-01 / 9.85e-02 | 20260507 17:10:49.498 | INFO | PerfReport: 15.74%, DME -- 4.23e+01 / 4.53e+01, 4.28e+00 / 2.77e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 55.92%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.88e+00 / 2.15e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.54e+00 / 2.90e+00, 4.40e-01 / 2.66e-01 | 20260507 17:10:49.498 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.45e+00, 3.24e-01 / 2.05e-01 | 20260507 17:10:49.498 | INFO | PerfReport: 1.19%, PulseSerializing -- 4.01e-01 / 4.25e-01, 6.67e-02 / 4.29e-02 | 20260507 17:10:49.498 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:10:49.498 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:10:49.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:10:49.498 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 23.67%, Upload -- 2.54e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 24.09%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 31.31%, Compute -- 3.36e+01 / 0.00e+00, 1.46e+01 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 1.54%, LockUpload -- 1.66e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 2.56%, LockDownload -- 2.75e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 16.81%, LockCompute -- 1.81e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:10:49.498 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 17:10:49.593 | INFO | Starting a round of Baz IO | 20260507 17:10:53.103 | INFO | Populating RTMetrics report | 20260507 17:10:53.103 | INFO | RTMetrics report populated | 20260507 17:10:53.103 | INFO | Opening tmp RTMetrics json file | 20260507 17:10:53.174 | INFO | Serializing RTMetrics to json | 20260507 17:10:53.202 | INFO | Serialization done, swapping tmp file now... | 20260507 17:10:53.206 | INFO | RTMetrics done | 20260507 17:10:53.228 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.634 seconds | 20260507 17:10:53.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.66% / 0.02% / 51.32% | 20260507 17:10:53.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.80876% / 2.38419e-05% / 2.38419e-05% | 20260507 17:10:53.228 | INFO | Baz IO overall: | 20260507 17:10:53.228 | INFO | BytesWritten : 519604938200 | 20260507 17:10:53.228 | INFO | NumEvents : 251679190593 | 20260507 17:10:54.635 | INFO | Analyzing chunk frames = [1187840,1188352) | 20260507 17:10:54.635 | INFO | MemInfo: { | 20260507 17:10:54.635 | INFO | "gpu" : " 17 GB", | 20260507 17:10:54.635 | INFO | "huge" : "", | 20260507 17:10:54.635 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:54.635 | INFO | "malloc" : " 14 GB", | 20260507 17:10:54.635 | INFO | "pinned" : " 29 GB" | 20260507 17:10:54.635 | INFO | } | 20260507 17:10:54.635 | INFO | Perf report for the last 5.13898 seconds: | 20260507 17:10:54.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:54.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:54.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:54.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:54.635 | INFO | ANALYSIS : 69.7%, 2.9, 107.5ms, 0 | 20260507 17:10:54.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:54.635 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 17:10:54.635 | INFO | PRE_HQ : 66.6%, 1.7, 60.2ms, 0 | 20260507 17:10:54.635 | INFO | BAZWRITER : 70.3%, 4.0, 144.8ms, 0 | 20260507 17:10:54.635 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 17:10:54.635 | INFO | Uploaded 36GiB to GPU in 2.56062s (14.0591GiB/s) | 20260507 17:10:54.635 | INFO | Downloaded 38GiB to GPU in 2.71047s (14.0197GiB/s) | 20260507 17:10:59.774 | INFO | Analyzing chunk frames = [1188352,1188864) | 20260507 17:10:59.774 | INFO | MemInfo: { | 20260507 17:10:59.774 | INFO | "gpu" : " 17 GB", | 20260507 17:10:59.774 | INFO | "huge" : "", | 20260507 17:10:59.774 | INFO | "huge_shared" : " 29 GB", | 20260507 17:10:59.775 | INFO | "malloc" : " 14 GB", | 20260507 17:10:59.775 | INFO | "pinned" : " 29 GB" | 20260507 17:10:59.775 | INFO | } | 20260507 17:10:59.775 | INFO | Perf report for the last 5.13957 seconds: | 20260507 17:10:59.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:10:59.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:10:59.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:10:59.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:10:59.775 | INFO | Uploaded 36GiB to GPU in 2.48903s (14.4635GiB/s) | 20260507 17:10:59.775 | INFO | Downloaded 34GiB to GPU in 2.44862s (13.8854GiB/s) | 20260507 17:11:04.914 | INFO | Analyzing chunk frames = [1188864,1189376) | 20260507 17:11:04.914 | INFO | MemInfo: { | 20260507 17:11:04.914 | INFO | "gpu" : " 17 GB", | 20260507 17:11:04.914 | INFO | "huge" : "", | 20260507 17:11:04.914 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:04.914 | INFO | "malloc" : " 14 GB", | 20260507 17:11:04.914 | INFO | "pinned" : " 29 GB" | 20260507 17:11:04.914 | INFO | } | 20260507 17:11:04.914 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:11:04.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:04.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260507 17:11:04.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:11:04.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:04.914 | INFO | Uploaded 36GiB to GPU in 2.55413s (14.0948GiB/s) | 20260507 17:11:04.914 | INFO | Downloaded 34GiB to GPU in 2.41894s (14.0557GiB/s) | 20260507 17:11:10.053 | INFO | Analyzing chunk frames = [1189376,1189888) | 20260507 17:11:10.053 | INFO | MemInfo: { | 20260507 17:11:10.053 | INFO | "gpu" : " 17 GB", | 20260507 17:11:10.053 | INFO | "huge" : "", | 20260507 17:11:10.053 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:10.053 | INFO | "malloc" : " 14 GB", | 20260507 17:11:10.053 | INFO | "pinned" : " 29 GB" | 20260507 17:11:10.053 | INFO | } | 20260507 17:11:10.053 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:11:10.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:10.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:11:10.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:11:10.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:10.053 | INFO | Uploaded 36GiB to GPU in 2.46497s (14.6047GiB/s) | 20260507 17:11:10.053 | INFO | Downloaded 34GiB to GPU in 2.36173s (14.3962GiB/s) | 20260507 17:11:15.192 | INFO | Analyzing chunk frames = [1189888,1190400) | 20260507 17:11:15.192 | INFO | MemInfo: { | 20260507 17:11:15.192 | INFO | "gpu" : " 17 GB", | 20260507 17:11:15.192 | INFO | "huge" : "", | 20260507 17:11:15.192 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:15.193 | INFO | "malloc" : " 14 GB", | 20260507 17:11:15.193 | INFO | "pinned" : " 29 GB" | 20260507 17:11:15.193 | INFO | } | 20260507 17:11:15.193 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:11:15.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:15.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:11:15.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:11:15.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:15.193 | INFO | Uploaded 36GiB to GPU in 2.38066s (15.1218GiB/s) | 20260507 17:11:15.193 | INFO | Downloaded 34GiB to GPU in 2.45386s (13.8557GiB/s) | 20260507 17:11:20.332 | INFO | Analyzing chunk frames = [1190400,1190912) | 20260507 17:11:20.332 | INFO | MemInfo: { | 20260507 17:11:20.332 | INFO | "gpu" : " 17 GB", | 20260507 17:11:20.332 | INFO | "huge" : "", | 20260507 17:11:20.332 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:20.332 | INFO | "malloc" : " 14 GB", | 20260507 17:11:20.332 | INFO | "pinned" : " 29 GB" | 20260507 17:11:20.332 | INFO | } | 20260507 17:11:20.332 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:11:20.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:20.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:11:20.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260507 17:11:20.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:20.332 | INFO | Uploaded 36GiB to GPU in 2.33628s (15.4091GiB/s) | 20260507 17:11:20.332 | INFO | Downloaded 34GiB to GPU in 2.51312s (13.529GiB/s) | 20260507 17:11:25.471 | INFO | Analyzing chunk frames = [1190912,1191424) | 20260507 17:11:25.471 | INFO | MemInfo: { | 20260507 17:11:25.471 | INFO | "gpu" : " 17 GB", | 20260507 17:11:25.471 | INFO | "huge" : "", | 20260507 17:11:25.471 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:25.471 | INFO | "malloc" : " 14 GB", | 20260507 17:11:25.471 | INFO | "pinned" : " 29 GB" | 20260507 17:11:25.471 | INFO | } | 20260507 17:11:25.471 | INFO | Perf report for the last 5.13937 seconds: | 20260507 17:11:25.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:25.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.472 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:11:25.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.472 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:11:25.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:25.472 | INFO | Uploaded 36GiB to GPU in 2.45955s (14.6368GiB/s) | 20260507 17:11:25.472 | INFO | Downloaded 34GiB to GPU in 2.43187s (13.981GiB/s) | 20260507 17:11:30.610 | INFO | Analyzing chunk frames = [1191424,1191936) | 20260507 17:11:30.611 | INFO | MemInfo: { | 20260507 17:11:30.611 | INFO | "gpu" : " 17 GB", | 20260507 17:11:30.611 | INFO | "huge" : "", | 20260507 17:11:30.611 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:30.611 | INFO | "malloc" : " 14 GB", | 20260507 17:11:30.611 | INFO | "pinned" : " 29 GB" | 20260507 17:11:30.611 | INFO | } | 20260507 17:11:30.611 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:11:30.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:30.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 17:11:30.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:11:30.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:30.611 | INFO | Uploaded 36GiB to GPU in 2.50911s (14.3477GiB/s) | 20260507 17:11:30.611 | INFO | Downloaded 34GiB to GPU in 2.5127s (13.5312GiB/s) | 20260507 17:11:30.709 | INFO | Starting a round of Baz IO | 20260507 17:11:34.216 | INFO | Populating RTMetrics report | 20260507 17:11:34.217 | INFO | RTMetrics report populated | 20260507 17:11:34.217 | INFO | Opening tmp RTMetrics json file | 20260507 17:11:34.225 | INFO | Serializing RTMetrics to json | 20260507 17:11:34.241 | INFO | Serialization done, swapping tmp file now... | 20260507 17:11:34.241 | INFO | RTMetrics done | 20260507 17:11:34.317 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.608 seconds | 20260507 17:11:34.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.63% / 0.02% / 51.34% | 20260507 17:11:34.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.80416% / 4.37101e-05% / 3.17891e-05% | 20260507 17:11:34.317 | INFO | Baz IO overall: | 20260507 17:11:34.317 | INFO | BytesWritten : 521533526764 | 20260507 17:11:34.317 | INFO | NumEvents : 252616314337 | 20260507 17:11:35.750 | INFO | Analyzing chunk frames = [1191936,1192448) | 20260507 17:11:35.750 | INFO | MemInfo: { | 20260507 17:11:35.750 | INFO | "gpu" : " 17 GB", | 20260507 17:11:35.750 | INFO | "huge" : "", | 20260507 17:11:35.750 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:35.750 | INFO | "malloc" : " 14 GB", | 20260507 17:11:35.750 | INFO | "pinned" : " 29 GB" | 20260507 17:11:35.750 | INFO | } | 20260507 17:11:35.750 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:11:35.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:35.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:35.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:35.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:35.750 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:11:35.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:35.750 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 17:11:35.750 | INFO | PRE_HQ : 67.0%, 1.7, 60.7ms, 0 | 20260507 17:11:35.750 | INFO | BAZWRITER : 69.6%, 3.6, 127.1ms, 0 | 20260507 17:11:35.750 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 17:11:35.751 | INFO | Uploaded 36GiB to GPU in 2.54899s (14.1233GiB/s) | 20260507 17:11:35.751 | INFO | Downloaded 38GiB to GPU in 2.71435s (13.9997GiB/s) | 20260507 17:11:40.889 | INFO | Analyzing chunk frames = [1192448,1192960) | 20260507 17:11:40.889 | INFO | MemInfo: { | 20260507 17:11:40.889 | INFO | "gpu" : " 17 GB", | 20260507 17:11:40.889 | INFO | "huge" : "", | 20260507 17:11:40.889 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:40.890 | INFO | "malloc" : " 14 GB", | 20260507 17:11:40.890 | INFO | "pinned" : " 29 GB" | 20260507 17:11:40.890 | INFO | } | 20260507 17:11:40.890 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:11:40.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:40.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:11:40.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:11:40.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:40.890 | INFO | Uploaded 36GiB to GPU in 2.5058s (14.3667GiB/s) | 20260507 17:11:40.890 | INFO | Downloaded 34GiB to GPU in 2.44832s (13.887GiB/s) | 20260507 17:11:46.029 | INFO | Analyzing chunk frames = [1192960,1193472) | 20260507 17:11:46.029 | INFO | MemInfo: { | 20260507 17:11:46.029 | INFO | "gpu" : " 17 GB", | 20260507 17:11:46.029 | INFO | "huge" : "", | 20260507 17:11:46.029 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:46.029 | INFO | "malloc" : " 14 GB", | 20260507 17:11:46.029 | INFO | "pinned" : " 29 GB" | 20260507 17:11:46.029 | INFO | } | 20260507 17:11:46.029 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:11:46.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:46.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 17:11:46.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:11:46.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:46.029 | INFO | Uploaded 36GiB to GPU in 2.55148s (14.1095GiB/s) | 20260507 17:11:46.029 | INFO | Downloaded 34GiB to GPU in 2.43526s (13.9615GiB/s) | 20260507 17:11:51.168 | INFO | Analyzing chunk frames = [1193472,1193984) | 20260507 17:11:51.168 | INFO | MemInfo: { | 20260507 17:11:51.168 | INFO | "gpu" : " 17 GB", | 20260507 17:11:51.168 | INFO | "huge" : "", | 20260507 17:11:51.168 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:51.168 | INFO | "malloc" : " 14 GB", | 20260507 17:11:51.168 | INFO | "pinned" : " 29 GB" | 20260507 17:11:51.168 | INFO | } | 20260507 17:11:51.168 | INFO | Perf report for the last 5.1391 seconds: | 20260507 17:11:51.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:51.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:11:51.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:11:51.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:51.168 | INFO | Uploaded 36GiB to GPU in 2.47073s (14.5706GiB/s) | 20260507 17:11:51.168 | INFO | Downloaded 34GiB to GPU in 2.35916s (14.4119GiB/s) | 20260507 17:11:56.307 | INFO | Analyzing chunk frames = [1193984,1194496) | 20260507 17:11:56.307 | INFO | MemInfo: { | 20260507 17:11:56.307 | INFO | "gpu" : " 17 GB", | 20260507 17:11:56.307 | INFO | "huge" : "", | 20260507 17:11:56.307 | INFO | "huge_shared" : " 29 GB", | 20260507 17:11:56.308 | INFO | "malloc" : " 14 GB", | 20260507 17:11:56.308 | INFO | "pinned" : " 29 GB" | 20260507 17:11:56.308 | INFO | } | 20260507 17:11:56.308 | INFO | Perf report for the last 5.13969 seconds: | 20260507 17:11:56.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:11:56.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:11:56.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 17:11:56.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:11:56.308 | INFO | Uploaded 36GiB to GPU in 2.32143s (15.5077GiB/s) | 20260507 17:11:56.308 | INFO | Downloaded 34GiB to GPU in 2.48227s (13.6971GiB/s) | 20260507 17:12:01.447 | INFO | Analyzing chunk frames = [1194496,1195008) | 20260507 17:12:01.447 | INFO | MemInfo: { | 20260507 17:12:01.447 | INFO | "gpu" : " 17 GB", | 20260507 17:12:01.447 | INFO | "huge" : "", | 20260507 17:12:01.447 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:01.447 | INFO | "malloc" : " 14 GB", | 20260507 17:12:01.447 | INFO | "pinned" : " 29 GB" | 20260507 17:12:01.447 | INFO | } | 20260507 17:12:01.447 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:12:01.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:01.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.447 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 17:12:01.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.447 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260507 17:12:01.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:01.448 | INFO | Uploaded 36GiB to GPU in 2.27709s (15.8096GiB/s) | 20260507 17:12:01.448 | INFO | Downloaded 34GiB to GPU in 2.50065s (13.5964GiB/s) | 20260507 17:12:06.586 | INFO | Analyzing chunk frames = [1195008,1195520) | 20260507 17:12:06.586 | INFO | MemInfo: { | 20260507 17:12:06.586 | INFO | "gpu" : " 17 GB", | 20260507 17:12:06.586 | INFO | "huge" : "", | 20260507 17:12:06.586 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:06.586 | INFO | "malloc" : " 14 GB", | 20260507 17:12:06.586 | INFO | "pinned" : " 29 GB" | 20260507 17:12:06.586 | INFO | } | 20260507 17:12:06.586 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:12:06.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:06.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.586 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:12:06.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.587 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:12:06.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:06.587 | INFO | Uploaded 36GiB to GPU in 2.4363s (14.7765GiB/s) | 20260507 17:12:06.587 | INFO | Downloaded 34GiB to GPU in 2.43714s (13.9508GiB/s) | 20260507 17:12:11.725 | INFO | Analyzing chunk frames = [1195520,1196032) | 20260507 17:12:11.725 | INFO | MemInfo: { | 20260507 17:12:11.725 | INFO | "gpu" : " 17 GB", | 20260507 17:12:11.725 | INFO | "huge" : "", | 20260507 17:12:11.725 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:11.725 | INFO | "malloc" : " 14 GB", | 20260507 17:12:11.725 | INFO | "pinned" : " 29 GB" | 20260507 17:12:11.725 | INFO | } | 20260507 17:12:11.725 | INFO | Perf report for the last 5.13898 seconds: | 20260507 17:12:11.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:11.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.725 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 17:12:11.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.726 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:12:11.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:11.726 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:12:11.726 | INFO | Total mem usage: 16.4 GiB | 20260507 17:12:11.726 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:12:11.726 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.7%) | 20260507 17:12:11.726 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:12:11.726 | INFO | High Water Breakdown: | 20260507 17:12:11.726 | INFO | Uploaded 36GiB to GPU in 2.45711s (14.6513GiB/s) | 20260507 17:12:11.726 | INFO | 59.5% : 8.81 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:12:11.726 | INFO | Downloaded 34GiB to GPU in 2.51348s (13.527GiB/s) | 20260507 17:12:11.726 | INFO | 40.4% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:12:11.726 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:12:11.726 | INFO | ------------------------------------------ | 20260507 17:12:11.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:12:11.726 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:12:11.726 | INFO | Total mem usage: 26.5 GiB | 20260507 17:12:11.726 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:12:11.726 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:12:11.726 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:12:11.726 | INFO | High Water Breakdown: | 20260507 17:12:11.726 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:12:11.726 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:12:11.726 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:12:11.726 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:12:11.726 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:12:11.726 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:12:11.726 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:12:11.726 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:12:11.726 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:12:11.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:12:11.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:12:11.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:12:11.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:12:11.726 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:12:11.726 | INFO | ------------------------------------------ | 20260507 17:12:11.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:12:11.726 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:12:11.727 | INFO | Total mem usage: 27.8 GiB | 20260507 17:12:11.727 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:12:11.727 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:12:11.727 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:12:11.727 | INFO | High Water Breakdown: | 20260507 17:12:11.727 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:12:11.727 | INFO | ------------------------------------------ | 20260507 17:12:11.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:12:11.727 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:12:11.727 | INFO | Total mem usage: 15.7 GiB | 20260507 17:12:11.727 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:12:11.727 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:12:11.727 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:12:11.727 | INFO | High Water Breakdown: | 20260507 17:12:11.727 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:12:11.727 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:12:11.727 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:12:11.727 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:12:11.727 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:12:11.727 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:12:11.727 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:12:11.727 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:12:11.727 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:12:11.727 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:12:11.727 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:12:11.727 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:12:11.727 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:12:11.727 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:12:11.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:12:11.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:12:11.727 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:12:11.727 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:12:11.727 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:12:11.727 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:12:11.727 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:12:11.727 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:12:11.727 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:12:11.727 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:12:11.727 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:12:11.727 | INFO | ------------------------------------------ | 20260507 17:12:11.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:12:11.727 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:12:11.727 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:12:11.727 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:12:11.727 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.67e+01, 1.44e+01 / 2.91e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.01e-01, 1.08e-02 / 4.62e-03 | 20260507 17:12:11.727 | INFO | PerfReport: 12.05%, Baseline -- 4.05e+00 / 4.51e+00, 6.25e-01 / 2.59e-01 | 20260507 17:12:11.727 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 1.11e-01 | 20260507 17:12:11.727 | INFO | PerfReport: 15.71%, DME -- 4.22e+01 / 4.51e+01, 4.52e+00 / 2.75e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.90e+00 / 2.17e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.52e+00 / 2.90e+00, 4.41e-01 / 2.35e-01 | 20260507 17:12:11.727 | INFO | PerfReport: 3.79%, Metrics -- 1.27e+00 / 1.45e+00, 3.24e-01 / 1.88e-01 | 20260507 17:12:11.727 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.26e-01, 6.80e-02 / 4.42e-02 | 20260507 17:12:11.727 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:12:11.727 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:12:11.727 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:12:11.727 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 23.82%, Upload -- 2.56e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 24.07%, Download -- 2.59e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 31.26%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 1.70%, LockUpload -- 1.83e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 2.39%, LockDownload -- 2.57e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 16.75%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:12:11.727 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260507 17:12:11.823 | INFO | Starting a round of Baz IO | 20260507 17:12:15.343 | INFO | Populating RTMetrics report | 20260507 17:12:15.344 | INFO | RTMetrics report populated | 20260507 17:12:15.344 | INFO | Opening tmp RTMetrics json file | 20260507 17:12:15.347 | INFO | Serializing RTMetrics to json | 20260507 17:12:15.366 | INFO | Serialization done, swapping tmp file now... | 20260507 17:12:15.368 | INFO | RTMetrics done | 20260507 17:12:15.444 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.621 seconds | 20260507 17:12:15.444 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.61% / 0.02% / 51.37% | 20260507 17:12:15.444 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.80485% / 2.38419e-05% / 3.97364e-05% | 20260507 17:12:15.444 | INFO | Baz IO overall: | 20260507 17:12:15.444 | INFO | BytesWritten : 523462918907 | 20260507 17:12:15.444 | INFO | NumEvents : 253554201563 | 20260507 17:12:16.865 | INFO | Analyzing chunk frames = [1196032,1196544) | 20260507 17:12:16.865 | INFO | MemInfo: { | 20260507 17:12:16.865 | INFO | "gpu" : " 17 GB", | 20260507 17:12:16.865 | INFO | "huge" : "", | 20260507 17:12:16.865 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:16.865 | INFO | "malloc" : " 14 GB", | 20260507 17:12:16.865 | INFO | "pinned" : " 29 GB" | 20260507 17:12:16.865 | INFO | } | 20260507 17:12:16.865 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:12:16.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:16.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:16.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:16.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:16.865 | INFO | ANALYSIS : 69.5%, 3.0, 108.4ms, 0 | 20260507 17:12:16.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:16.865 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 17:12:16.865 | INFO | PRE_HQ : 66.9%, 1.8, 61.0ms, 0 | 20260507 17:12:16.865 | INFO | BAZWRITER : 70.0%, 3.8, 135.6ms, 0 | 20260507 17:12:16.865 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:12:16.866 | INFO | Uploaded 36GiB to GPU in 2.49845s (14.4089GiB/s) | 20260507 17:12:16.866 | INFO | Downloaded 38GiB to GPU in 2.70731s (14.0361GiB/s) | 20260507 17:12:22.004 | INFO | Analyzing chunk frames = [1196544,1197056) | 20260507 17:12:22.004 | INFO | MemInfo: { | 20260507 17:12:22.004 | INFO | "gpu" : " 17 GB", | 20260507 17:12:22.004 | INFO | "huge" : "", | 20260507 17:12:22.004 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:22.004 | INFO | "malloc" : " 14 GB", | 20260507 17:12:22.004 | INFO | "pinned" : " 29 GB" | 20260507 17:12:22.004 | INFO | } | 20260507 17:12:22.004 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:12:22.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:22.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.004 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:12:22.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.004 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:12:22.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:22.005 | INFO | Uploaded 36GiB to GPU in 2.47309s (14.5567GiB/s) | 20260507 17:12:22.005 | INFO | Downloaded 34GiB to GPU in 2.45938s (13.8246GiB/s) | 20260507 17:12:27.143 | INFO | Analyzing chunk frames = [1197056,1197568) | 20260507 17:12:27.143 | INFO | MemInfo: { | 20260507 17:12:27.143 | INFO | "gpu" : " 17 GB", | 20260507 17:12:27.143 | INFO | "huge" : "", | 20260507 17:12:27.143 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:27.143 | INFO | "malloc" : " 14 GB", | 20260507 17:12:27.143 | INFO | "pinned" : " 29 GB" | 20260507 17:12:27.143 | INFO | } | 20260507 17:12:27.144 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:12:27.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:27.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:12:27.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:12:27.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:27.144 | INFO | Uploaded 36GiB to GPU in 2.54282s (14.1575GiB/s) | 20260507 17:12:27.144 | INFO | Downloaded 34GiB to GPU in 2.44142s (13.9263GiB/s) | 20260507 17:12:32.283 | INFO | Analyzing chunk frames = [1197568,1198080) | 20260507 17:12:32.283 | INFO | MemInfo: { | 20260507 17:12:32.283 | INFO | "gpu" : " 17 GB", | 20260507 17:12:32.283 | INFO | "huge" : "", | 20260507 17:12:32.283 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:32.283 | INFO | "malloc" : " 14 GB", | 20260507 17:12:32.283 | INFO | "pinned" : " 29 GB" | 20260507 17:12:32.283 | INFO | } | 20260507 17:12:32.283 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:12:32.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:32.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:12:32.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:12:32.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:32.283 | INFO | Uploaded 36GiB to GPU in 2.44889s (14.7006GiB/s) | 20260507 17:12:32.283 | INFO | Downloaded 34GiB to GPU in 2.37212s (14.3332GiB/s) | 20260507 17:12:37.422 | INFO | Analyzing chunk frames = [1198080,1198592) | 20260507 17:12:37.422 | INFO | MemInfo: { | 20260507 17:12:37.422 | INFO | "gpu" : " 17 GB", | 20260507 17:12:37.422 | INFO | "huge" : "", | 20260507 17:12:37.422 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:37.422 | INFO | "malloc" : " 14 GB", | 20260507 17:12:37.422 | INFO | "pinned" : " 29 GB" | 20260507 17:12:37.422 | INFO | } | 20260507 17:12:37.422 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:12:37.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:37.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.422 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 17:12:37.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.423 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 17:12:37.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:37.423 | INFO | Uploaded 36GiB to GPU in 2.29471s (15.6882GiB/s) | 20260507 17:12:37.423 | INFO | Downloaded 34GiB to GPU in 2.47507s (13.737GiB/s) | 20260507 17:12:42.562 | INFO | Analyzing chunk frames = [1198592,1199104) | 20260507 17:12:42.562 | INFO | MemInfo: { | 20260507 17:12:42.562 | INFO | "gpu" : " 17 GB", | 20260507 17:12:42.562 | INFO | "huge" : "", | 20260507 17:12:42.562 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:42.562 | INFO | "malloc" : " 14 GB", | 20260507 17:12:42.562 | INFO | "pinned" : " 29 GB" | 20260507 17:12:42.562 | INFO | } | 20260507 17:12:42.562 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:12:42.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:42.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:12:42.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 17:12:42.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:42.562 | INFO | Uploaded 36GiB to GPU in 2.44109s (14.7475GiB/s) | 20260507 17:12:42.563 | INFO | Downloaded 34GiB to GPU in 2.51061s (13.5425GiB/s) | 20260507 17:12:47.701 | INFO | Analyzing chunk frames = [1199104,1199616) | 20260507 17:12:47.701 | INFO | MemInfo: { | 20260507 17:12:47.701 | INFO | "gpu" : " 17 GB", | 20260507 17:12:47.701 | INFO | "huge" : "", | 20260507 17:12:47.701 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:47.701 | INFO | "malloc" : " 14 GB", | 20260507 17:12:47.701 | INFO | "pinned" : " 29 GB" | 20260507 17:12:47.701 | INFO | } | 20260507 17:12:47.701 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:12:47.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:47.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:12:47.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:12:47.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:47.701 | INFO | Uploaded 36GiB to GPU in 2.46355s (14.6131GiB/s) | 20260507 17:12:47.701 | INFO | Downloaded 34GiB to GPU in 2.42742s (14.0067GiB/s) | 20260507 17:12:52.840 | INFO | Analyzing chunk frames = [1199616,1200128) | 20260507 17:12:52.840 | INFO | MemInfo: { | 20260507 17:12:52.840 | INFO | "gpu" : " 17 GB", | 20260507 17:12:52.840 | INFO | "huge" : "", | 20260507 17:12:52.840 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:52.840 | INFO | "malloc" : " 14 GB", | 20260507 17:12:52.840 | INFO | "pinned" : " 29 GB" | 20260507 17:12:52.840 | INFO | } | 20260507 17:12:52.840 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:12:52.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:52.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 17:12:52.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:12:52.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:52.840 | INFO | Uploaded 36GiB to GPU in 2.52246s (14.2718GiB/s) | 20260507 17:12:52.841 | INFO | Downloaded 34GiB to GPU in 2.52415s (13.4699GiB/s) | 20260507 17:12:52.937 | INFO | Starting a round of Baz IO | 20260507 17:12:56.452 | INFO | Populating RTMetrics report | 20260507 17:12:56.452 | INFO | RTMetrics report populated | 20260507 17:12:56.452 | INFO | Opening tmp RTMetrics json file | 20260507 17:12:56.456 | INFO | Serializing RTMetrics to json | 20260507 17:12:56.469 | INFO | Serialization done, swapping tmp file now... | 20260507 17:12:56.470 | INFO | RTMetrics done | 20260507 17:12:56.530 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.592 seconds | 20260507 17:12:56.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.59% / 0.02% / 51.39% | 20260507 17:12:56.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.78846% / 4.76837e-05% / 3.17891e-05% | 20260507 17:12:56.530 | INFO | Baz IO overall: | 20260507 17:12:56.530 | INFO | BytesWritten : 525391371300 | 20260507 17:12:56.530 | INFO | NumEvents : 254491206795 | 20260507 17:12:57.979 | INFO | Analyzing chunk frames = [1200128,1200640) | 20260507 17:12:57.979 | INFO | MemInfo: { | 20260507 17:12:57.979 | INFO | "gpu" : " 17 GB", | 20260507 17:12:57.979 | INFO | "huge" : "", | 20260507 17:12:57.979 | INFO | "huge_shared" : " 29 GB", | 20260507 17:12:57.979 | INFO | "malloc" : " 14 GB", | 20260507 17:12:57.979 | INFO | "pinned" : " 29 GB" | 20260507 17:12:57.979 | INFO | } | 20260507 17:12:57.980 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:12:57.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:12:57.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:57.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:57.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:57.980 | INFO | ANALYSIS : 69.8%, 3.0, 108.8ms, 0 | 20260507 17:12:57.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:12:57.980 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 17:12:57.980 | INFO | PRE_HQ : 67.7%, 1.8, 63.1ms, 0 | 20260507 17:12:57.980 | INFO | BAZWRITER : 68.9%, 2.9, 104.4ms, 0 | 20260507 17:12:57.980 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:12:57.980 | INFO | Uploaded 36GiB to GPU in 2.57145s (13.9999GiB/s) | 20260507 17:12:57.980 | INFO | Downloaded 38GiB to GPU in 2.72517s (13.9441GiB/s) | 20260507 17:13:03.119 | INFO | Analyzing chunk frames = [1200640,1201152) | 20260507 17:13:03.119 | INFO | MemInfo: { | 20260507 17:13:03.119 | INFO | "gpu" : " 17 GB", | 20260507 17:13:03.119 | INFO | "huge" : "", | 20260507 17:13:03.119 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:03.119 | INFO | "malloc" : " 14 GB", | 20260507 17:13:03.119 | INFO | "pinned" : " 29 GB" | 20260507 17:13:03.119 | INFO | } | 20260507 17:13:03.119 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:13:03.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:03.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.119 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:13:03.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.119 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:13:03.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:03.120 | INFO | Uploaded 36GiB to GPU in 2.5351s (14.2006GiB/s) | 20260507 17:13:03.120 | INFO | Downloaded 34GiB to GPU in 2.42819s (14.0022GiB/s) | 20260507 17:13:08.258 | INFO | Analyzing chunk frames = [1201152,1201664) | 20260507 17:13:08.258 | INFO | MemInfo: { | 20260507 17:13:08.258 | INFO | "gpu" : " 17 GB", | 20260507 17:13:08.258 | INFO | "huge" : "", | 20260507 17:13:08.258 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:08.258 | INFO | "malloc" : " 14 GB", | 20260507 17:13:08.258 | INFO | "pinned" : " 29 GB" | 20260507 17:13:08.258 | INFO | } | 20260507 17:13:08.258 | INFO | Perf report for the last 5.13925 seconds: | 20260507 17:13:08.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:08.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.258 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260507 17:13:08.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.259 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:13:08.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:08.259 | INFO | Uploaded 36GiB to GPU in 2.47898s (14.5221GiB/s) | 20260507 17:13:08.259 | INFO | Downloaded 34GiB to GPU in 2.33955s (14.5327GiB/s) | 20260507 17:13:13.398 | INFO | Analyzing chunk frames = [1201664,1202176) | 20260507 17:13:13.398 | INFO | MemInfo: { | 20260507 17:13:13.398 | INFO | "gpu" : " 17 GB", | 20260507 17:13:13.398 | INFO | "huge" : "", | 20260507 17:13:13.398 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:13.398 | INFO | "malloc" : " 14 GB", | 20260507 17:13:13.398 | INFO | "pinned" : " 29 GB" | 20260507 17:13:13.398 | INFO | } | 20260507 17:13:13.398 | INFO | Perf report for the last 5.13984 seconds: | 20260507 17:13:13.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:13.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.398 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 17:13:13.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.398 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:13:13.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:13.399 | INFO | Uploaded 36GiB to GPU in 2.49451s (14.4317GiB/s) | 20260507 17:13:13.399 | INFO | Downloaded 34GiB to GPU in 2.35019s (14.4669GiB/s) | 20260507 17:13:18.538 | INFO | Analyzing chunk frames = [1202176,1202688) | 20260507 17:13:18.538 | INFO | MemInfo: { | 20260507 17:13:18.538 | INFO | "gpu" : " 17 GB", | 20260507 17:13:18.538 | INFO | "huge" : "", | 20260507 17:13:18.538 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:18.538 | INFO | "malloc" : " 14 GB", | 20260507 17:13:18.538 | INFO | "pinned" : " 29 GB" | 20260507 17:13:18.538 | INFO | } | 20260507 17:13:18.538 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:13:18.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:18.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:13:18.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 17:13:18.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:18.538 | INFO | Uploaded 36GiB to GPU in 2.37098s (15.1836GiB/s) | 20260507 17:13:18.538 | INFO | Downloaded 34GiB to GPU in 2.4769s (13.7268GiB/s) | 20260507 17:13:23.676 | INFO | Analyzing chunk frames = [1202688,1203200) | 20260507 17:13:23.676 | INFO | MemInfo: { | 20260507 17:13:23.676 | INFO | "gpu" : " 17 GB", | 20260507 17:13:23.676 | INFO | "huge" : "", | 20260507 17:13:23.676 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:23.676 | INFO | "malloc" : " 14 GB", | 20260507 17:13:23.677 | INFO | "pinned" : " 29 GB" | 20260507 17:13:23.677 | INFO | } | 20260507 17:13:23.677 | INFO | Perf report for the last 5.13889 seconds: | 20260507 17:13:23.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:23.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:13:23.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:13:23.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:23.677 | INFO | Uploaded 36GiB to GPU in 2.37491s (15.1585GiB/s) | 20260507 17:13:23.677 | INFO | Downloaded 34GiB to GPU in 2.48525s (13.6807GiB/s) | 20260507 17:13:28.816 | INFO | Analyzing chunk frames = [1203200,1203712) | 20260507 17:13:28.816 | INFO | MemInfo: { | 20260507 17:13:28.816 | INFO | "gpu" : " 17 GB", | 20260507 17:13:28.816 | INFO | "huge" : "", | 20260507 17:13:28.816 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:28.816 | INFO | "malloc" : " 14 GB", | 20260507 17:13:28.816 | INFO | "pinned" : " 29 GB" | 20260507 17:13:28.816 | INFO | } | 20260507 17:13:28.816 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:13:28.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:28.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.816 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:13:28.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.816 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:13:28.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:28.817 | INFO | Uploaded 36GiB to GPU in 2.44266s (14.7381GiB/s) | 20260507 17:13:28.817 | INFO | Downloaded 34GiB to GPU in 2.41855s (14.058GiB/s) | 20260507 17:13:33.955 | INFO | Analyzing chunk frames = [1203712,1204224) | 20260507 17:13:33.955 | INFO | MemInfo: { | 20260507 17:13:33.955 | INFO | "gpu" : " 17 GB", | 20260507 17:13:33.955 | INFO | "huge" : "", | 20260507 17:13:33.955 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:33.955 | INFO | "malloc" : " 14 GB", | 20260507 17:13:33.955 | INFO | "pinned" : " 29 GB" | 20260507 17:13:33.955 | INFO | } | 20260507 17:13:33.955 | INFO | Perf report for the last 5.13913 seconds: | 20260507 17:13:33.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:33.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.955 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:13:33.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.955 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:13:33.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:33.956 | INFO | Uploaded 36GiB to GPU in 2.49446s (14.432GiB/s) | 20260507 17:13:33.956 | INFO | Downloaded 34GiB to GPU in 2.50453s (13.5754GiB/s) | 20260507 17:13:33.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:13:33.956 | INFO | Total mem usage: 16.4 GiB | 20260507 17:13:33.956 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:13:33.956 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 17:13:33.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:13:33.956 | INFO | High Water Breakdown: | 20260507 17:13:33.956 | INFO | 59.1% : 8.61 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:13:33.956 | INFO | 40.7% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:13:33.956 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:13:33.956 | INFO | ------------------------------------------ | 20260507 17:13:33.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:13:33.956 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:13:33.956 | INFO | Total mem usage: 26.5 GiB | 20260507 17:13:33.956 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:13:33.956 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:13:33.956 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:13:33.956 | INFO | High Water Breakdown: | 20260507 17:13:33.956 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:13:33.956 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:13:33.956 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:13:33.956 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:13:33.956 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:13:33.956 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:13:33.956 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:13:33.956 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:13:33.956 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:13:33.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:13:33.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:13:33.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:13:33.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:13:33.956 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:13:33.956 | INFO | ------------------------------------------ | 20260507 17:13:33.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:13:33.956 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:13:33.956 | INFO | Total mem usage: 27.8 GiB | 20260507 17:13:33.956 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:13:33.956 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:13:33.956 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:13:33.956 | INFO | High Water Breakdown: | 20260507 17:13:33.956 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:13:33.956 | INFO | ------------------------------------------ | 20260507 17:13:33.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:13:33.956 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:13:33.956 | INFO | Total mem usage: 15.7 GiB | 20260507 17:13:33.956 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:13:33.956 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:13:33.956 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:13:33.956 | INFO | High Water Breakdown: | 20260507 17:13:33.956 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:13:33.956 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:13:33.956 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:13:33.956 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:13:33.956 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:13:33.956 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:13:33.956 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:13:33.956 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:13:33.956 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:13:33.956 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:13:33.956 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:13:33.956 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:13:33.956 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:13:33.956 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:13:33.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:13:33.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:13:33.956 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:13:33.956 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:13:33.956 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:13:33.956 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:13:33.956 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:13:33.956 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:13:33.956 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:13:33.956 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:13:33.956 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:13:33.956 | INFO | ------------------------------------------ | 20260507 17:13:33.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:13:33.956 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:13:33.956 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:13:33.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:13:33.957 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.44e+01 / 2.92e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.10e-02 / 4.74e-03 | 20260507 17:13:33.957 | INFO | PerfReport: 12.05%, Baseline -- 4.04e+00 / 4.51e+00, 6.20e-01 / 2.25e-01 | 20260507 17:13:33.957 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.06e-01 | 20260507 17:13:33.957 | INFO | PerfReport: 15.74%, DME -- 4.22e+01 / 4.52e+01, 4.42e+00 / 2.52e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.88e+00 / 1.94e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.52e+00 / 2.90e+00, 4.36e-01 / 2.45e-01 | 20260507 17:13:33.957 | INFO | PerfReport: 3.80%, Metrics -- 1.27e+00 / 1.45e+00, 3.26e-01 / 1.96e-01 | 20260507 17:13:33.957 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.01e-01 / 4.25e-01, 6.69e-02 / 4.07e-02 | 20260507 17:13:33.957 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:13:33.957 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:13:33.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:13:33.957 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 23.90%, Upload -- 2.57e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.62e+00 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 31.21%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 1.69%, LockUpload -- 1.82e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 2.36%, LockDownload -- 2.53e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 16.80%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:13:33.957 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.54e-02 / 0.00e+00 | 20260507 17:13:34.053 | INFO | Starting a round of Baz IO | 20260507 17:13:37.559 | INFO | Populating RTMetrics report | 20260507 17:13:37.559 | INFO | RTMetrics report populated | 20260507 17:13:37.559 | INFO | Opening tmp RTMetrics json file | 20260507 17:13:37.576 | INFO | Serializing RTMetrics to json | 20260507 17:13:37.577 | INFO | Serialization done, swapping tmp file now... | 20260507 17:13:37.577 | INFO | RTMetrics done | 20260507 17:13:37.639 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.586 seconds | 20260507 17:13:37.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.57% / 0.02% / 51.41% | 20260507 17:13:37.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.79103% / 2.38419e-05% / 3.17891e-05% | 20260507 17:13:37.639 | INFO | Baz IO overall: | 20260507 17:13:37.639 | INFO | BytesWritten : 527322161520 | 20260507 17:13:37.639 | INFO | NumEvents : 255430199980 | 20260507 17:13:39.095 | INFO | Analyzing chunk frames = [1204224,1204736) | 20260507 17:13:39.095 | INFO | MemInfo: { | 20260507 17:13:39.095 | INFO | "gpu" : " 17 GB", | 20260507 17:13:39.095 | INFO | "huge" : "", | 20260507 17:13:39.095 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:39.095 | INFO | "malloc" : " 14 GB", | 20260507 17:13:39.095 | INFO | "pinned" : " 29 GB" | 20260507 17:13:39.095 | INFO | } | 20260507 17:13:39.095 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:13:39.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:39.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:39.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:39.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:39.095 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:13:39.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:39.095 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 17:13:39.095 | INFO | PRE_HQ : 67.0%, 1.7, 60.6ms, 0 | 20260507 17:13:39.095 | INFO | BAZWRITER : 68.9%, 2.9, 102.9ms, 0 | 20260507 17:13:39.095 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:13:39.096 | INFO | Uploaded 36GiB to GPU in 2.53631s (14.1938GiB/s) | 20260507 17:13:39.096 | INFO | Downloaded 38GiB to GPU in 2.71849s (13.9783GiB/s) | 20260507 17:13:44.234 | INFO | Analyzing chunk frames = [1204736,1205248) | 20260507 17:13:44.234 | INFO | MemInfo: { | 20260507 17:13:44.234 | INFO | "gpu" : " 17 GB", | 20260507 17:13:44.234 | INFO | "huge" : "", | 20260507 17:13:44.234 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:44.234 | INFO | "malloc" : " 14 GB", | 20260507 17:13:44.234 | INFO | "pinned" : " 29 GB" | 20260507 17:13:44.234 | INFO | } | 20260507 17:13:44.234 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:13:44.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:44.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.234 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:13:44.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.235 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:13:44.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:44.235 | INFO | Uploaded 36GiB to GPU in 2.49698s (14.4174GiB/s) | 20260507 17:13:44.235 | INFO | Downloaded 34GiB to GPU in 2.44463s (13.908GiB/s) | 20260507 17:13:49.373 | INFO | Analyzing chunk frames = [1205248,1205760) | 20260507 17:13:49.373 | INFO | MemInfo: { | 20260507 17:13:49.373 | INFO | "gpu" : " 17 GB", | 20260507 17:13:49.373 | INFO | "huge" : "", | 20260507 17:13:49.373 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:49.373 | INFO | "malloc" : " 14 GB", | 20260507 17:13:49.374 | INFO | "pinned" : " 29 GB" | 20260507 17:13:49.374 | INFO | } | 20260507 17:13:49.374 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:13:49.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:49.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:13:49.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:13:49.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:49.374 | INFO | Uploaded 36GiB to GPU in 2.53752s (14.1871GiB/s) | 20260507 17:13:49.374 | INFO | Downloaded 34GiB to GPU in 2.4271s (14.0085GiB/s) | 20260507 17:13:54.513 | INFO | Analyzing chunk frames = [1205760,1206272) | 20260507 17:13:54.513 | INFO | MemInfo: { | 20260507 17:13:54.513 | INFO | "gpu" : " 17 GB", | 20260507 17:13:54.513 | INFO | "huge" : "", | 20260507 17:13:54.513 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:54.513 | INFO | "malloc" : " 14 GB", | 20260507 17:13:54.513 | INFO | "pinned" : " 29 GB" | 20260507 17:13:54.513 | INFO | } | 20260507 17:13:54.513 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:13:54.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:54.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260507 17:13:54.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:13:54.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:54.513 | INFO | Uploaded 36GiB to GPU in 2.48053s (14.5131GiB/s) | 20260507 17:13:54.513 | INFO | Downloaded 34GiB to GPU in 2.35111s (14.4612GiB/s) | 20260507 17:13:59.652 | INFO | Analyzing chunk frames = [1206272,1206784) | 20260507 17:13:59.652 | INFO | MemInfo: { | 20260507 17:13:59.652 | INFO | "gpu" : " 17 GB", | 20260507 17:13:59.652 | INFO | "huge" : "", | 20260507 17:13:59.652 | INFO | "huge_shared" : " 29 GB", | 20260507 17:13:59.652 | INFO | "malloc" : " 14 GB", | 20260507 17:13:59.652 | INFO | "pinned" : " 29 GB" | 20260507 17:13:59.652 | INFO | } | 20260507 17:13:59.653 | INFO | Perf report for the last 5.13995 seconds: | 20260507 17:13:59.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:13:59.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.653 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:13:59.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.653 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 17:13:59.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:13:59.654 | INFO | Uploaded 36GiB to GPU in 2.35308s (15.2991GiB/s) | 20260507 17:13:59.654 | INFO | Downloaded 34GiB to GPU in 2.47505s (13.7371GiB/s) | 20260507 17:14:04.791 | INFO | Analyzing chunk frames = [1206784,1207296) | 20260507 17:14:04.791 | INFO | MemInfo: { | 20260507 17:14:04.791 | INFO | "gpu" : " 17 GB", | 20260507 17:14:04.791 | INFO | "huge" : "", | 20260507 17:14:04.791 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:04.791 | INFO | "malloc" : " 14 GB", | 20260507 17:14:04.791 | INFO | "pinned" : " 29 GB" | 20260507 17:14:04.792 | INFO | } | 20260507 17:14:04.792 | INFO | Perf report for the last 5.13861 seconds: | 20260507 17:14:04.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:04.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:14:04.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:14:04.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:04.792 | INFO | Uploaded 36GiB to GPU in 2.33916s (15.3901GiB/s) | 20260507 17:14:04.792 | INFO | Downloaded 34GiB to GPU in 2.47632s (13.73GiB/s) | 20260507 17:14:09.931 | INFO | Analyzing chunk frames = [1207296,1207808) | 20260507 17:14:09.931 | INFO | MemInfo: { | 20260507 17:14:09.931 | INFO | "gpu" : " 17 GB", | 20260507 17:14:09.931 | INFO | "huge" : "", | 20260507 17:14:09.931 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:09.931 | INFO | "malloc" : " 14 GB", | 20260507 17:14:09.931 | INFO | "pinned" : " 29 GB" | 20260507 17:14:09.931 | INFO | } | 20260507 17:14:09.931 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:14:09.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:09.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.931 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:14:09.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.931 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:14:09.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:09.932 | INFO | Uploaded 36GiB to GPU in 2.4761s (14.539GiB/s) | 20260507 17:14:09.932 | INFO | Downloaded 34GiB to GPU in 2.42671s (14.0108GiB/s) | 20260507 17:14:15.070 | INFO | Analyzing chunk frames = [1207808,1208320) | 20260507 17:14:15.070 | INFO | MemInfo: { | 20260507 17:14:15.070 | INFO | "gpu" : " 17 GB", | 20260507 17:14:15.070 | INFO | "huge" : "", | 20260507 17:14:15.070 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:15.070 | INFO | "malloc" : " 14 GB", | 20260507 17:14:15.070 | INFO | "pinned" : " 29 GB" | 20260507 17:14:15.070 | INFO | } | 20260507 17:14:15.070 | INFO | Perf report for the last 5.1391 seconds: | 20260507 17:14:15.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:15.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.070 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:14:15.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.070 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:14:15.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:15.071 | INFO | Uploaded 36GiB to GPU in 2.4857s (14.4828GiB/s) | 20260507 17:14:15.071 | INFO | Downloaded 34GiB to GPU in 2.50385s (13.5791GiB/s) | 20260507 17:14:15.168 | INFO | Starting a round of Baz IO | 20260507 17:14:18.677 | INFO | Populating RTMetrics report | 20260507 17:14:18.677 | INFO | RTMetrics report populated | 20260507 17:14:18.677 | INFO | Opening tmp RTMetrics json file | 20260507 17:14:18.681 | INFO | Serializing RTMetrics to json | 20260507 17:14:18.695 | INFO | Serialization done, swapping tmp file now... | 20260507 17:14:18.695 | INFO | RTMetrics done | 20260507 17:14:18.756 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.588 seconds | 20260507 17:14:18.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.55% / 0.02% / 51.43% | 20260507 17:14:18.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.78135% / 5.96046e-05% / 5.96046e-05% | 20260507 17:14:18.756 | INFO | Baz IO overall: | 20260507 17:14:18.756 | INFO | BytesWritten : 529252532291 | 20260507 17:14:18.756 | INFO | NumEvents : 256368956193 | 20260507 17:14:20.209 | INFO | Analyzing chunk frames = [1208320,1208832) | 20260507 17:14:20.209 | INFO | MemInfo: { | 20260507 17:14:20.209 | INFO | "gpu" : " 17 GB", | 20260507 17:14:20.209 | INFO | "huge" : "", | 20260507 17:14:20.210 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:20.210 | INFO | "malloc" : " 14 GB", | 20260507 17:14:20.210 | INFO | "pinned" : " 29 GB" | 20260507 17:14:20.210 | INFO | } | 20260507 17:14:20.210 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:14:20.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:20.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:20.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:20.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:20.210 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:14:20.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:20.210 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 17:14:20.210 | INFO | PRE_HQ : 67.0%, 1.8, 62.1ms, 0 | 20260507 17:14:20.210 | INFO | BAZWRITER : 69.3%, 2.9, 103.6ms, 0 | 20260507 17:14:20.210 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:14:20.211 | INFO | Uploaded 36GiB to GPU in 2.54771s (14.1304GiB/s) | 20260507 17:14:20.211 | INFO | Downloaded 38GiB to GPU in 2.72553s (13.9422GiB/s) | 20260507 17:14:25.349 | INFO | Analyzing chunk frames = [1208832,1209344) | 20260507 17:14:25.349 | INFO | MemInfo: { | 20260507 17:14:25.349 | INFO | "gpu" : " 17 GB", | 20260507 17:14:25.349 | INFO | "huge" : "", | 20260507 17:14:25.349 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:25.349 | INFO | "malloc" : " 14 GB", | 20260507 17:14:25.349 | INFO | "pinned" : " 29 GB" | 20260507 17:14:25.349 | INFO | } | 20260507 17:14:25.349 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:14:25.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:25.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:14:25.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:14:25.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:25.349 | INFO | Uploaded 36GiB to GPU in 2.48666s (14.4773GiB/s) | 20260507 17:14:25.349 | INFO | Downloaded 34GiB to GPU in 2.44173s (13.9245GiB/s) | 20260507 17:14:30.488 | INFO | Analyzing chunk frames = [1209344,1209856) | 20260507 17:14:30.488 | INFO | MemInfo: { | 20260507 17:14:30.488 | INFO | "gpu" : " 17 GB", | 20260507 17:14:30.488 | INFO | "huge" : "", | 20260507 17:14:30.488 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:30.488 | INFO | "malloc" : " 14 GB", | 20260507 17:14:30.488 | INFO | "pinned" : " 29 GB" | 20260507 17:14:30.489 | INFO | } | 20260507 17:14:30.489 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:14:30.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:30.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:14:30.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:14:30.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:30.489 | INFO | Uploaded 36GiB to GPU in 2.54716s (14.1334GiB/s) | 20260507 17:14:30.489 | INFO | Downloaded 34GiB to GPU in 2.42665s (14.0111GiB/s) | 20260507 17:14:35.628 | INFO | Analyzing chunk frames = [1209856,1210368) | 20260507 17:14:35.628 | INFO | MemInfo: { | 20260507 17:14:35.628 | INFO | "gpu" : " 17 GB", | 20260507 17:14:35.628 | INFO | "huge" : "", | 20260507 17:14:35.628 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:35.628 | INFO | "malloc" : " 14 GB", | 20260507 17:14:35.628 | INFO | "pinned" : " 29 GB" | 20260507 17:14:35.628 | INFO | } | 20260507 17:14:35.628 | INFO | Perf report for the last 5.13967 seconds: | 20260507 17:14:35.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:35.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.628 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:14:35.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.628 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:14:35.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:35.629 | INFO | Uploaded 36GiB to GPU in 2.47371s (14.5531GiB/s) | 20260507 17:14:35.629 | INFO | Downloaded 34GiB to GPU in 2.35673s (14.4268GiB/s) | 20260507 17:14:40.768 | INFO | Analyzing chunk frames = [1210368,1210880) | 20260507 17:14:40.768 | INFO | MemInfo: { | 20260507 17:14:40.768 | INFO | "gpu" : " 17 GB", | 20260507 17:14:40.768 | INFO | "huge" : "", | 20260507 17:14:40.768 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:40.768 | INFO | "malloc" : " 14 GB", | 20260507 17:14:40.768 | INFO | "pinned" : " 29 GB" | 20260507 17:14:40.768 | INFO | } | 20260507 17:14:40.768 | INFO | Perf report for the last 5.13982 seconds: | 20260507 17:14:40.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:40.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.768 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:14:40.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.768 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:14:40.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:40.769 | INFO | Uploaded 36GiB to GPU in 2.35082s (15.3138GiB/s) | 20260507 17:14:40.769 | INFO | Downloaded 34GiB to GPU in 2.48399s (13.6876GiB/s) | 20260507 17:14:45.906 | INFO | Analyzing chunk frames = [1210880,1211392) | 20260507 17:14:45.906 | INFO | MemInfo: { | 20260507 17:14:45.906 | INFO | "gpu" : " 17 GB", | 20260507 17:14:45.906 | INFO | "huge" : "", | 20260507 17:14:45.906 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:45.906 | INFO | "malloc" : " 14 GB", | 20260507 17:14:45.906 | INFO | "pinned" : " 29 GB" | 20260507 17:14:45.906 | INFO | } | 20260507 17:14:45.906 | INFO | Perf report for the last 5.1381 seconds: | 20260507 17:14:45.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:45.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.906 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:14:45.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.906 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 17:14:45.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:45.907 | INFO | Uploaded 36GiB to GPU in 2.38887s (15.0699GiB/s) | 20260507 17:14:45.907 | INFO | Downloaded 34GiB to GPU in 2.4912s (13.6481GiB/s) | 20260507 17:14:51.046 | INFO | Analyzing chunk frames = [1211392,1211904) | 20260507 17:14:51.046 | INFO | MemInfo: { | 20260507 17:14:51.046 | INFO | "gpu" : " 17 GB", | 20260507 17:14:51.046 | INFO | "huge" : "", | 20260507 17:14:51.046 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:51.046 | INFO | "malloc" : " 14 GB", | 20260507 17:14:51.046 | INFO | "pinned" : " 29 GB" | 20260507 17:14:51.046 | INFO | } | 20260507 17:14:51.046 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:14:51.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:51.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 17:14:51.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:14:51.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:51.046 | INFO | Uploaded 36GiB to GPU in 2.42201s (14.8637GiB/s) | 20260507 17:14:51.046 | INFO | Downloaded 34GiB to GPU in 2.41477s (14.08GiB/s) | 20260507 17:14:56.185 | INFO | Analyzing chunk frames = [1211904,1212416) | 20260507 17:14:56.185 | INFO | MemInfo: { | 20260507 17:14:56.185 | INFO | "gpu" : " 17 GB", | 20260507 17:14:56.185 | INFO | "huge" : "", | 20260507 17:14:56.185 | INFO | "huge_shared" : " 29 GB", | 20260507 17:14:56.185 | INFO | "malloc" : " 14 GB", | 20260507 17:14:56.185 | INFO | "pinned" : " 29 GB" | 20260507 17:14:56.185 | INFO | } | 20260507 17:14:56.185 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:14:56.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:14:56.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.185 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:14:56.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.185 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:14:56.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:14:56.186 | INFO | Uploaded 36GiB to GPU in 2.52763s (14.2426GiB/s) | 20260507 17:14:56.186 | INFO | Downloaded 34GiB to GPU in 2.51984s (13.4929GiB/s) | 20260507 17:14:56.186 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:14:56.186 | INFO | Total mem usage: 16.4 GiB | 20260507 17:14:56.186 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:14:56.186 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.2%) | 20260507 17:14:56.186 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:14:56.186 | INFO | High Water Breakdown: | 20260507 17:14:56.186 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:14:56.186 | INFO | 40.8% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:14:56.186 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:14:56.186 | INFO | ------------------------------------------ | 20260507 17:14:56.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:14:56.186 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:14:56.186 | INFO | Total mem usage: 26.5 GiB | 20260507 17:14:56.186 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:14:56.186 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:14:56.186 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:14:56.186 | INFO | High Water Breakdown: | 20260507 17:14:56.186 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:14:56.186 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:14:56.186 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:14:56.186 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:14:56.186 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:14:56.186 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:14:56.186 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:14:56.186 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:14:56.186 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:14:56.186 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:14:56.186 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:14:56.186 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:14:56.186 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:14:56.186 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:14:56.186 | INFO | ------------------------------------------ | 20260507 17:14:56.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:14:56.186 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:14:56.186 | INFO | Total mem usage: 27.8 GiB | 20260507 17:14:56.186 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:14:56.186 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:14:56.186 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:14:56.186 | INFO | High Water Breakdown: | 20260507 17:14:56.186 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:14:56.186 | INFO | ------------------------------------------ | 20260507 17:14:56.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:14:56.186 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:14:56.186 | INFO | Total mem usage: 15.7 GiB | 20260507 17:14:56.186 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:14:56.186 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:14:56.186 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:14:56.186 | INFO | High Water Breakdown: | 20260507 17:14:56.186 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:14:56.186 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:14:56.186 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:14:56.186 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:14:56.186 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:14:56.186 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:14:56.186 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:14:56.186 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:14:56.186 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:14:56.186 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:14:56.186 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:14:56.186 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:14:56.186 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:14:56.186 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:14:56.186 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:14:56.186 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:14:56.186 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:14:56.186 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:14:56.186 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:14:56.186 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:14:56.186 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:14:56.186 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:14:56.186 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:14:56.186 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:14:56.186 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:14:56.186 | INFO | ------------------------------------------ | 20260507 17:14:56.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:14:56.186 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:14:56.186 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:14:56.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:14:56.186 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.67e+01, 1.45e+01 / 2.83e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 0.29%, AggregateStats -- 9.77e-02 / 1.01e-01, 1.07e-02 / 5.72e-03 | 20260507 17:14:56.186 | INFO | PerfReport: 12.04%, Baseline -- 4.04e+00 / 4.51e+00, 6.17e-01 / 1.99e-01 | 20260507 17:14:56.186 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 9.84e-02 | 20260507 17:14:56.186 | INFO | PerfReport: 15.76%, DME -- 4.23e+01 / 4.51e+01, 4.28e+00 / 2.66e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.87e+00 / 2.24e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.53e+00 / 2.91e+00, 4.38e-01 / 2.30e-01 | 20260507 17:14:56.186 | INFO | PerfReport: 3.79%, Metrics -- 1.27e+00 / 1.44e+00, 3.23e-01 / 1.86e-01 | 20260507 17:14:56.186 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.01e-01 / 4.24e-01, 6.73e-02 / 4.17e-02 | 20260507 17:14:56.186 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:14:56.186 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:14:56.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:14:56.186 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 23.89%, Upload -- 2.57e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 24.03%, Download -- 2.58e+01 / 0.00e+00, 8.64e+00 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 31.20%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 1.69%, LockUpload -- 1.82e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 2.41%, LockDownload -- 2.59e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 16.77%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:14:56.186 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260507 17:14:56.282 | INFO | Starting a round of Baz IO | 20260507 17:14:59.807 | INFO | Populating RTMetrics report | 20260507 17:14:59.808 | INFO | RTMetrics report populated | 20260507 17:14:59.808 | INFO | Opening tmp RTMetrics json file | 20260507 17:14:59.820 | INFO | Serializing RTMetrics to json | 20260507 17:14:59.822 | INFO | Serialization done, swapping tmp file now... | 20260507 17:14:59.824 | INFO | RTMetrics done | 20260507 17:14:59.891 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.608 seconds | 20260507 17:14:59.891 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.53% / 0.02% / 51.45% | 20260507 17:14:59.891 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.75772% / 4.37101e-05% / 3.17891e-05% | 20260507 17:14:59.891 | INFO | Baz IO overall: | 20260507 17:14:59.891 | INFO | BytesWritten : 531181208651 | 20260507 17:14:59.891 | INFO | NumEvents : 257306232858 | 20260507 17:15:01.324 | INFO | Analyzing chunk frames = [1212416,1212928) | 20260507 17:15:01.324 | INFO | MemInfo: { | 20260507 17:15:01.324 | INFO | "gpu" : " 17 GB", | 20260507 17:15:01.324 | INFO | "huge" : "", | 20260507 17:15:01.324 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:01.324 | INFO | "malloc" : " 14 GB", | 20260507 17:15:01.324 | INFO | "pinned" : " 29 GB" | 20260507 17:15:01.324 | INFO | } | 20260507 17:15:01.324 | INFO | Perf report for the last 5.13911 seconds: | 20260507 17:15:01.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:01.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:01.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:01.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:01.324 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:15:01.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:01.324 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 17:15:01.325 | INFO | PRE_HQ : 67.0%, 1.7, 60.4ms, 0 | 20260507 17:15:01.325 | INFO | BAZWRITER : 69.3%, 2.9, 103.4ms, 0 | 20260507 17:15:01.325 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:15:01.325 | INFO | Uploaded 36GiB to GPU in 2.53558s (14.1979GiB/s) | 20260507 17:15:01.325 | INFO | Downloaded 38GiB to GPU in 2.72147s (13.963GiB/s) | 20260507 17:15:06.464 | INFO | Analyzing chunk frames = [1212928,1213440) | 20260507 17:15:06.464 | INFO | MemInfo: { | 20260507 17:15:06.464 | INFO | "gpu" : " 17 GB", | 20260507 17:15:06.464 | INFO | "huge" : "", | 20260507 17:15:06.464 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:06.464 | INFO | "malloc" : " 14 GB", | 20260507 17:15:06.464 | INFO | "pinned" : " 29 GB" | 20260507 17:15:06.464 | INFO | } | 20260507 17:15:06.464 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:15:06.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:06.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:15:06.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:15:06.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:06.464 | INFO | Uploaded 36GiB to GPU in 2.43583s (14.7794GiB/s) | 20260507 17:15:06.464 | INFO | Downloaded 34GiB to GPU in 2.43929s (13.9385GiB/s) | 20260507 17:15:11.603 | INFO | Analyzing chunk frames = [1213440,1213952) | 20260507 17:15:11.603 | INFO | MemInfo: { | 20260507 17:15:11.603 | INFO | "gpu" : " 17 GB", | 20260507 17:15:11.603 | INFO | "huge" : "", | 20260507 17:15:11.603 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:11.603 | INFO | "malloc" : " 14 GB", | 20260507 17:15:11.603 | INFO | "pinned" : " 29 GB" | 20260507 17:15:11.603 | INFO | } | 20260507 17:15:11.603 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:15:11.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:11.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.603 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:15:11.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.603 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:15:11.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:11.604 | INFO | Uploaded 36GiB to GPU in 2.54408s (14.1505GiB/s) | 20260507 17:15:11.604 | INFO | Downloaded 34GiB to GPU in 2.43129s (13.9843GiB/s) | 20260507 17:15:16.742 | INFO | Analyzing chunk frames = [1213952,1214464) | 20260507 17:15:16.742 | INFO | MemInfo: { | 20260507 17:15:16.742 | INFO | "gpu" : " 17 GB", | 20260507 17:15:16.742 | INFO | "huge" : "", | 20260507 17:15:16.742 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:16.742 | INFO | "malloc" : " 14 GB", | 20260507 17:15:16.742 | INFO | "pinned" : " 29 GB" | 20260507 17:15:16.743 | INFO | } | 20260507 17:15:16.743 | INFO | Perf report for the last 5.13946 seconds: | 20260507 17:15:16.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:16.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:15:16.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:15:16.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:16.743 | INFO | Uploaded 36GiB to GPU in 2.47703s (14.5335GiB/s) | 20260507 17:15:16.743 | INFO | Downloaded 34GiB to GPU in 2.35557s (14.4339GiB/s) | 20260507 17:15:21.882 | INFO | Analyzing chunk frames = [1214464,1214976) | 20260507 17:15:21.882 | INFO | MemInfo: { | 20260507 17:15:21.882 | INFO | "gpu" : " 17 GB", | 20260507 17:15:21.882 | INFO | "huge" : "", | 20260507 17:15:21.882 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:21.882 | INFO | "malloc" : " 14 GB", | 20260507 17:15:21.882 | INFO | "pinned" : " 29 GB" | 20260507 17:15:21.882 | INFO | } | 20260507 17:15:21.882 | INFO | Perf report for the last 5.13972 seconds: | 20260507 17:15:21.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:21.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.882 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:15:21.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.883 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 17:15:21.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:21.883 | INFO | Uploaded 36GiB to GPU in 2.34772s (15.334GiB/s) | 20260507 17:15:21.883 | INFO | Downloaded 34GiB to GPU in 2.47519s (13.7363GiB/s) | 20260507 17:15:27.021 | INFO | Analyzing chunk frames = [1214976,1215488) | 20260507 17:15:27.021 | INFO | MemInfo: { | 20260507 17:15:27.021 | INFO | "gpu" : " 17 GB", | 20260507 17:15:27.021 | INFO | "huge" : "", | 20260507 17:15:27.021 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:27.021 | INFO | "malloc" : " 14 GB", | 20260507 17:15:27.021 | INFO | "pinned" : " 29 GB" | 20260507 17:15:27.021 | INFO | } | 20260507 17:15:27.021 | INFO | Perf report for the last 5.13899 seconds: | 20260507 17:15:27.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:27.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.021 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:15:27.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.022 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:15:27.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:27.022 | INFO | Uploaded 36GiB to GPU in 2.34078s (15.3795GiB/s) | 20260507 17:15:27.022 | INFO | Downloaded 34GiB to GPU in 2.45454s (13.8519GiB/s) | 20260507 17:15:32.160 | INFO | Analyzing chunk frames = [1215488,1216000) | 20260507 17:15:32.160 | INFO | MemInfo: { | 20260507 17:15:32.160 | INFO | "gpu" : " 17 GB", | 20260507 17:15:32.160 | INFO | "huge" : "", | 20260507 17:15:32.160 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:32.160 | INFO | "malloc" : " 14 GB", | 20260507 17:15:32.160 | INFO | "pinned" : " 29 GB" | 20260507 17:15:32.160 | INFO | } | 20260507 17:15:32.161 | INFO | Perf report for the last 5.13911 seconds: | 20260507 17:15:32.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:32.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:15:32.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:15:32.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:32.161 | INFO | Uploaded 36GiB to GPU in 2.45194s (14.6822GiB/s) | 20260507 17:15:32.161 | INFO | Downloaded 34GiB to GPU in 2.42771s (14.0049GiB/s) | 20260507 17:15:37.300 | INFO | Analyzing chunk frames = [1216000,1216512) | 20260507 17:15:37.300 | INFO | MemInfo: { | 20260507 17:15:37.300 | INFO | "gpu" : " 17 GB", | 20260507 17:15:37.300 | INFO | "huge" : "", | 20260507 17:15:37.300 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:37.300 | INFO | "malloc" : " 14 GB", | 20260507 17:15:37.300 | INFO | "pinned" : " 29 GB" | 20260507 17:15:37.300 | INFO | } | 20260507 17:15:37.300 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:15:37.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:37.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:15:37.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:15:37.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:37.300 | INFO | Uploaded 36GiB to GPU in 2.48263s (14.5008GiB/s) | 20260507 17:15:37.300 | INFO | Downloaded 34GiB to GPU in 2.50922s (13.55GiB/s) | 20260507 17:15:37.397 | INFO | Starting a round of Baz IO | 20260507 17:15:40.904 | INFO | Populating RTMetrics report | 20260507 17:15:40.905 | INFO | RTMetrics report populated | 20260507 17:15:40.905 | INFO | Opening tmp RTMetrics json file | 20260507 17:15:40.911 | INFO | Serializing RTMetrics to json | 20260507 17:15:40.920 | INFO | Serialization done, swapping tmp file now... | 20260507 17:15:40.921 | INFO | RTMetrics done | 20260507 17:15:40.984 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.586 seconds | 20260507 17:15:40.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.51% / 0.02% / 51.47% | 20260507 17:15:40.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.74883% / 1.58946e-05% / 3.17891e-05% | 20260507 17:15:40.984 | INFO | Baz IO overall: | 20260507 17:15:40.984 | INFO | BytesWritten : 533110550467 | 20260507 17:15:40.984 | INFO | NumEvents : 258243882632 | 20260507 17:15:42.439 | INFO | Analyzing chunk frames = [1216512,1217024) | 20260507 17:15:42.439 | INFO | MemInfo: { | 20260507 17:15:42.439 | INFO | "gpu" : " 17 GB", | 20260507 17:15:42.439 | INFO | "huge" : "", | 20260507 17:15:42.439 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:42.439 | INFO | "malloc" : " 14 GB", | 20260507 17:15:42.439 | INFO | "pinned" : " 29 GB" | 20260507 17:15:42.439 | INFO | } | 20260507 17:15:42.439 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:15:42.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:42.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:42.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:42.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:42.439 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:15:42.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:42.439 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 17:15:42.439 | INFO | PRE_HQ : 67.5%, 1.8, 64.1ms, 0 | 20260507 17:15:42.439 | INFO | BAZWRITER : 68.9%, 3.0, 106.1ms, 0 | 20260507 17:15:42.439 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:15:42.440 | INFO | Uploaded 36GiB to GPU in 2.56138s (14.0549GiB/s) | 20260507 17:15:42.440 | INFO | Downloaded 38GiB to GPU in 2.73123s (13.9132GiB/s) | 20260507 17:15:47.579 | INFO | Analyzing chunk frames = [1217024,1217536) | 20260507 17:15:47.579 | INFO | MemInfo: { | 20260507 17:15:47.579 | INFO | "gpu" : " 17 GB", | 20260507 17:15:47.579 | INFO | "huge" : "", | 20260507 17:15:47.579 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:47.579 | INFO | "malloc" : " 14 GB", | 20260507 17:15:47.579 | INFO | "pinned" : " 29 GB" | 20260507 17:15:47.579 | INFO | } | 20260507 17:15:47.579 | INFO | Perf report for the last 5.13976 seconds: | 20260507 17:15:47.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:47.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.579 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:15:47.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.579 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 17:15:47.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:47.580 | INFO | Uploaded 36GiB to GPU in 2.53274s (14.2139GiB/s) | 20260507 17:15:47.580 | INFO | Downloaded 34GiB to GPU in 2.45351s (13.8577GiB/s) | 20260507 17:15:52.718 | INFO | Analyzing chunk frames = [1217536,1218048) | 20260507 17:15:52.718 | INFO | MemInfo: { | 20260507 17:15:52.718 | INFO | "gpu" : " 17 GB", | 20260507 17:15:52.718 | INFO | "huge" : "", | 20260507 17:15:52.718 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:52.718 | INFO | "malloc" : " 14 GB", | 20260507 17:15:52.718 | INFO | "pinned" : " 29 GB" | 20260507 17:15:52.718 | INFO | } | 20260507 17:15:52.719 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:15:52.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:52.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 17:15:52.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | PRE_HQ : 37.9%, 1.0, 19.6ms, 0 | 20260507 17:15:52.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:52.719 | INFO | Uploaded 36GiB to GPU in 2.53343s (14.21GiB/s) | 20260507 17:15:52.719 | INFO | Downloaded 34GiB to GPU in 2.44306s (13.9169GiB/s) | 20260507 17:15:57.857 | INFO | Analyzing chunk frames = [1218048,1218560) | 20260507 17:15:57.857 | INFO | MemInfo: { | 20260507 17:15:57.857 | INFO | "gpu" : " 17 GB", | 20260507 17:15:57.857 | INFO | "huge" : "", | 20260507 17:15:57.857 | INFO | "huge_shared" : " 29 GB", | 20260507 17:15:57.857 | INFO | "malloc" : " 14 GB", | 20260507 17:15:57.857 | INFO | "pinned" : " 29 GB" | 20260507 17:15:57.857 | INFO | } | 20260507 17:15:57.857 | INFO | Perf report for the last 5.13874 seconds: | 20260507 17:15:57.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:15:57.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.857 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:15:57.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.857 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260507 17:15:57.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:15:57.858 | INFO | Uploaded 36GiB to GPU in 2.48891s (14.4642GiB/s) | 20260507 17:15:57.858 | INFO | Downloaded 34GiB to GPU in 2.35955s (14.4095GiB/s) | 20260507 17:16:02.997 | INFO | Analyzing chunk frames = [1218560,1219072) | 20260507 17:16:02.997 | INFO | MemInfo: { | 20260507 17:16:02.997 | INFO | "gpu" : " 17 GB", | 20260507 17:16:02.997 | INFO | "huge" : "", | 20260507 17:16:02.997 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:02.997 | INFO | "malloc" : " 14 GB", | 20260507 17:16:02.997 | INFO | "pinned" : " 29 GB" | 20260507 17:16:02.997 | INFO | } | 20260507 17:16:02.997 | INFO | Perf report for the last 5.13978 seconds: | 20260507 17:16:02.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:02.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:16:02.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 17:16:02.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:02.997 | INFO | Uploaded 36GiB to GPU in 2.37285s (15.1716GiB/s) | 20260507 17:16:02.997 | INFO | Downloaded 34GiB to GPU in 2.46565s (13.7895GiB/s) | 20260507 17:16:08.136 | INFO | Analyzing chunk frames = [1219072,1219584) | 20260507 17:16:08.136 | INFO | MemInfo: { | 20260507 17:16:08.136 | INFO | "gpu" : " 17 GB", | 20260507 17:16:08.136 | INFO | "huge" : "", | 20260507 17:16:08.136 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:08.136 | INFO | "malloc" : " 14 GB", | 20260507 17:16:08.136 | INFO | "pinned" : " 29 GB" | 20260507 17:16:08.136 | INFO | } | 20260507 17:16:08.136 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:16:08.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:08.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.136 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:16:08.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.137 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260507 17:16:08.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:08.137 | INFO | Uploaded 36GiB to GPU in 2.40861s (14.9463GiB/s) | 20260507 17:16:08.137 | INFO | Downloaded 34GiB to GPU in 2.49551s (13.6245GiB/s) | 20260507 17:16:13.275 | INFO | Analyzing chunk frames = [1219584,1220096) | 20260507 17:16:13.275 | INFO | MemInfo: { | 20260507 17:16:13.275 | INFO | "gpu" : " 17 GB", | 20260507 17:16:13.275 | INFO | "huge" : "", | 20260507 17:16:13.275 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:13.275 | INFO | "malloc" : " 14 GB", | 20260507 17:16:13.275 | INFO | "pinned" : " 29 GB" | 20260507 17:16:13.275 | INFO | } | 20260507 17:16:13.275 | INFO | Perf report for the last 5.13885 seconds: | 20260507 17:16:13.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:13.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.275 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:16:13.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.275 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:16:13.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:13.276 | INFO | Uploaded 36GiB to GPU in 2.46299s (14.6164GiB/s) | 20260507 17:16:13.276 | INFO | Downloaded 34GiB to GPU in 2.42855s (14.0001GiB/s) | 20260507 17:16:18.415 | INFO | Analyzing chunk frames = [1220096,1220608) | 20260507 17:16:18.415 | INFO | MemInfo: { | 20260507 17:16:18.415 | INFO | "gpu" : " 17 GB", | 20260507 17:16:18.415 | INFO | "huge" : "", | 20260507 17:16:18.415 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:18.415 | INFO | "malloc" : " 14 GB", | 20260507 17:16:18.415 | INFO | "pinned" : " 29 GB" | 20260507 17:16:18.415 | INFO | } | 20260507 17:16:18.415 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:16:18.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:18.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:16:18.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:16:18.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:18.415 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:16:18.415 | INFO | Total mem usage: 16.4 GiB | 20260507 17:16:18.415 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:16:18.415 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.2%) | 20260507 17:16:18.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:16:18.415 | INFO | High Water Breakdown: | 20260507 17:16:18.415 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:16:18.415 | INFO | 40.8% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:16:18.415 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:16:18.415 | INFO | ------------------------------------------ | 20260507 17:16:18.415 | INFO | Uploaded 36GiB to GPU in 2.45021s (14.6926GiB/s) | 20260507 17:16:18.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:16:18.416 | INFO | Downloaded 34GiB to GPU in 2.5053s (13.5712GiB/s) | 20260507 17:16:18.416 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:16:18.416 | INFO | Total mem usage: 26.5 GiB | 20260507 17:16:18.416 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:16:18.416 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:16:18.416 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:16:18.416 | INFO | High Water Breakdown: | 20260507 17:16:18.416 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:16:18.416 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:16:18.416 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:16:18.416 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:16:18.416 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:16:18.416 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:16:18.416 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:16:18.416 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:16:18.416 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:16:18.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:16:18.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:16:18.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:16:18.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:16:18.416 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:16:18.416 | INFO | ------------------------------------------ | 20260507 17:16:18.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:16:18.416 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:16:18.416 | INFO | Total mem usage: 27.8 GiB | 20260507 17:16:18.416 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:16:18.416 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:16:18.416 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:16:18.416 | INFO | High Water Breakdown: | 20260507 17:16:18.416 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:16:18.416 | INFO | ------------------------------------------ | 20260507 17:16:18.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:16:18.416 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:16:18.416 | INFO | Total mem usage: 15.7 GiB | 20260507 17:16:18.416 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:16:18.416 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:16:18.416 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:16:18.416 | INFO | High Water Breakdown: | 20260507 17:16:18.416 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:16:18.416 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:16:18.416 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:16:18.416 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:16:18.416 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:16:18.416 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:16:18.416 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:16:18.416 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:16:18.416 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:16:18.416 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:16:18.416 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:16:18.416 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:16:18.416 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:16:18.416 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:16:18.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:16:18.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:16:18.416 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:16:18.416 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:16:18.416 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:16:18.416 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:16:18.416 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:16:18.416 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:16:18.416 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:16:18.416 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:16:18.416 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:16:18.416 | INFO | ------------------------------------------ | 20260507 17:16:18.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:16:18.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:16:18.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:16:18.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:16:18.416 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.45e+01 / 2.90e+00 | 20260507 17:16:18.416 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.03e-01, 1.20e-02 / 5.80e-03 | 20260507 17:16:18.416 | INFO | PerfReport: 12.08%, Baseline -- 4.05e+00 / 4.52e+00, 6.20e-01 / 2.39e-01 | 20260507 17:16:18.416 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.50e+00, 2.18e-01 / 1.06e-01 | 20260507 17:16:18.416 | INFO | PerfReport: 15.73%, DME -- 4.22e+01 / 4.50e+01, 4.40e+00 / 2.73e+00 | 20260507 17:16:18.416 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.83e+00 / 2.12e+00 | 20260507 17:16:18.416 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.54e+00 / 2.89e+00, 4.38e-01 / 2.34e-01 | 20260507 17:16:18.416 | INFO | PerfReport: 3.80%, Metrics -- 1.27e+00 / 1.46e+00, 3.23e-01 / 2.10e-01 | 20260507 17:16:18.417 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.31e-01, 6.71e-02 / 4.40e-02 | 20260507 17:16:18.417 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:16:18.417 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:16:18.417 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:16:18.417 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 23.88%, Upload -- 2.57e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 24.04%, Download -- 2.58e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 31.19%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 1.67%, LockUpload -- 1.80e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 2.45%, LockDownload -- 2.64e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 16.76%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:16:18.417 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 17:16:18.513 | INFO | Starting a round of Baz IO | 20260507 17:16:22.022 | INFO | Populating RTMetrics report | 20260507 17:16:22.022 | INFO | RTMetrics report populated | 20260507 17:16:22.022 | INFO | Opening tmp RTMetrics json file | 20260507 17:16:22.023 | INFO | Serializing RTMetrics to json | 20260507 17:16:22.024 | INFO | Serialization done, swapping tmp file now... | 20260507 17:16:22.025 | INFO | RTMetrics done | 20260507 17:16:22.104 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.591 seconds | 20260507 17:16:22.104 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.48% / 0.02% / 51.49% | 20260507 17:16:22.104 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.75779% / 3.17891e-05% / 3.17891e-05% | 20260507 17:16:22.104 | INFO | Baz IO overall: | 20260507 17:16:22.104 | INFO | BytesWritten : 535043982628 | 20260507 17:16:22.104 | INFO | NumEvents : 259184576176 | 20260507 17:16:23.554 | INFO | Analyzing chunk frames = [1220608,1221120) | 20260507 17:16:23.554 | INFO | MemInfo: { | 20260507 17:16:23.554 | INFO | "gpu" : " 17 GB", | 20260507 17:16:23.554 | INFO | "huge" : "", | 20260507 17:16:23.554 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:23.554 | INFO | "malloc" : " 14 GB", | 20260507 17:16:23.554 | INFO | "pinned" : " 29 GB" | 20260507 17:16:23.554 | INFO | } | 20260507 17:16:23.554 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:16:23.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:23.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:23.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:23.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:23.554 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:16:23.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:23.554 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 17:16:23.554 | INFO | PRE_HQ : 67.5%, 1.8, 62.4ms, 0 | 20260507 17:16:23.554 | INFO | BAZWRITER : 69.4%, 2.9, 103.7ms, 0 | 20260507 17:16:23.554 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 17:16:23.554 | INFO | Uploaded 36GiB to GPU in 2.52719s (14.2451GiB/s) | 20260507 17:16:23.554 | INFO | Downloaded 38GiB to GPU in 2.71865s (13.9775GiB/s) | 20260507 17:16:28.693 | INFO | Analyzing chunk frames = [1221120,1221632) | 20260507 17:16:28.693 | INFO | MemInfo: { | 20260507 17:16:28.693 | INFO | "gpu" : " 17 GB", | 20260507 17:16:28.693 | INFO | "huge" : "", | 20260507 17:16:28.693 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:28.693 | INFO | "malloc" : " 14 GB", | 20260507 17:16:28.693 | INFO | "pinned" : " 29 GB" | 20260507 17:16:28.693 | INFO | } | 20260507 17:16:28.693 | INFO | Perf report for the last 5.13936 seconds: | 20260507 17:16:28.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:28.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.693 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:16:28.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.693 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:16:28.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:28.694 | INFO | Uploaded 36GiB to GPU in 2.50357s (14.3795GiB/s) | 20260507 17:16:28.694 | INFO | Downloaded 34GiB to GPU in 2.43911s (13.9395GiB/s) | 20260507 17:16:33.833 | INFO | Analyzing chunk frames = [1221632,1222144) | 20260507 17:16:33.833 | INFO | MemInfo: { | 20260507 17:16:33.833 | INFO | "gpu" : " 17 GB", | 20260507 17:16:33.833 | INFO | "huge" : "", | 20260507 17:16:33.833 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:33.833 | INFO | "malloc" : " 14 GB", | 20260507 17:16:33.833 | INFO | "pinned" : " 29 GB" | 20260507 17:16:33.833 | INFO | } | 20260507 17:16:33.833 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:16:33.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:33.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:16:33.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:16:33.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:33.833 | INFO | Uploaded 36GiB to GPU in 2.55037s (14.1156GiB/s) | 20260507 17:16:33.833 | INFO | Downloaded 34GiB to GPU in 2.43751s (13.9487GiB/s) | 20260507 17:16:38.972 | INFO | Analyzing chunk frames = [1222144,1222656) | 20260507 17:16:38.972 | INFO | MemInfo: { | 20260507 17:16:38.972 | INFO | "gpu" : " 17 GB", | 20260507 17:16:38.972 | INFO | "huge" : "", | 20260507 17:16:38.972 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:38.972 | INFO | "malloc" : " 14 GB", | 20260507 17:16:38.972 | INFO | "pinned" : " 29 GB" | 20260507 17:16:38.972 | INFO | } | 20260507 17:16:38.972 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:16:38.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:38.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.972 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260507 17:16:38.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.972 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:16:38.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:38.973 | INFO | Uploaded 36GiB to GPU in 2.47468s (14.5473GiB/s) | 20260507 17:16:38.973 | INFO | Downloaded 34GiB to GPU in 2.35322s (14.4483GiB/s) | 20260507 17:16:44.112 | INFO | Analyzing chunk frames = [1222656,1223168) | 20260507 17:16:44.112 | INFO | MemInfo: { | 20260507 17:16:44.112 | INFO | "gpu" : " 17 GB", | 20260507 17:16:44.112 | INFO | "huge" : "", | 20260507 17:16:44.112 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:44.112 | INFO | "malloc" : " 14 GB", | 20260507 17:16:44.112 | INFO | "pinned" : " 29 GB" | 20260507 17:16:44.112 | INFO | } | 20260507 17:16:44.112 | INFO | Perf report for the last 5.13997 seconds: | 20260507 17:16:44.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:44.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 17:16:44.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 17:16:44.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:44.112 | INFO | Uploaded 36GiB to GPU in 2.38556s (15.0908GiB/s) | 20260507 17:16:44.112 | INFO | Downloaded 34GiB to GPU in 2.46708s (13.7815GiB/s) | 20260507 17:16:49.251 | INFO | Analyzing chunk frames = [1223168,1223680) | 20260507 17:16:49.251 | INFO | MemInfo: { | 20260507 17:16:49.251 | INFO | "gpu" : " 17 GB", | 20260507 17:16:49.251 | INFO | "huge" : "", | 20260507 17:16:49.251 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:49.251 | INFO | "malloc" : " 14 GB", | 20260507 17:16:49.251 | INFO | "pinned" : " 29 GB" | 20260507 17:16:49.251 | INFO | } | 20260507 17:16:49.251 | INFO | Perf report for the last 5.1392 seconds: | 20260507 17:16:49.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:49.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.251 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:16:49.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.252 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 17:16:49.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:49.252 | INFO | Uploaded 36GiB to GPU in 2.37541s (15.1552GiB/s) | 20260507 17:16:49.252 | INFO | Downloaded 34GiB to GPU in 2.45716s (13.8371GiB/s) | 20260507 17:16:54.390 | INFO | Analyzing chunk frames = [1223680,1224192) | 20260507 17:16:54.390 | INFO | MemInfo: { | 20260507 17:16:54.390 | INFO | "gpu" : " 17 GB", | 20260507 17:16:54.390 | INFO | "huge" : "", | 20260507 17:16:54.390 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:54.390 | INFO | "malloc" : " 14 GB", | 20260507 17:16:54.391 | INFO | "pinned" : " 29 GB" | 20260507 17:16:54.391 | INFO | } | 20260507 17:16:54.391 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:16:54.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:54.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:16:54.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:16:54.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:54.391 | INFO | Uploaded 36GiB to GPU in 2.48126s (14.5087GiB/s) | 20260507 17:16:54.391 | INFO | Downloaded 34GiB to GPU in 2.40959s (14.1103GiB/s) | 20260507 17:16:59.530 | INFO | Analyzing chunk frames = [1224192,1224704) | 20260507 17:16:59.530 | INFO | MemInfo: { | 20260507 17:16:59.530 | INFO | "gpu" : " 17 GB", | 20260507 17:16:59.530 | INFO | "huge" : "", | 20260507 17:16:59.530 | INFO | "huge_shared" : " 29 GB", | 20260507 17:16:59.530 | INFO | "malloc" : " 14 GB", | 20260507 17:16:59.530 | INFO | "pinned" : " 29 GB" | 20260507 17:16:59.530 | INFO | } | 20260507 17:16:59.530 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:16:59.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:16:59.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.530 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 17:16:59.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.530 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:16:59.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:16:59.531 | INFO | Uploaded 36GiB to GPU in 2.53796s (14.1846GiB/s) | 20260507 17:16:59.531 | INFO | Downloaded 34GiB to GPU in 2.49594s (13.6221GiB/s) | 20260507 17:16:59.627 | INFO | Starting a round of Baz IO | 20260507 17:17:03.155 | INFO | Populating RTMetrics report | 20260507 17:17:03.155 | INFO | RTMetrics report populated | 20260507 17:17:03.155 | INFO | Opening tmp RTMetrics json file | 20260507 17:17:03.156 | INFO | Serializing RTMetrics to json | 20260507 17:17:03.157 | INFO | Serialization done, swapping tmp file now... | 20260507 17:17:03.157 | INFO | RTMetrics done | 20260507 17:17:03.237 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.61 seconds | 20260507 17:17:03.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.46% / 0.02% / 51.52% | 20260507 17:17:03.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.73142% / 4.37101e-05% / 3.17891e-05% | 20260507 17:17:03.237 | INFO | Baz IO overall: | 20260507 17:17:03.237 | INFO | BytesWritten : 536974426001 | 20260507 17:17:03.237 | INFO | NumEvents : 260122899281 | 20260507 17:17:04.669 | INFO | Analyzing chunk frames = [1224704,1225216) | 20260507 17:17:04.669 | INFO | MemInfo: { | 20260507 17:17:04.669 | INFO | "gpu" : " 17 GB", | 20260507 17:17:04.669 | INFO | "huge" : "", | 20260507 17:17:04.669 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:04.669 | INFO | "malloc" : " 14 GB", | 20260507 17:17:04.669 | INFO | "pinned" : " 29 GB" | 20260507 17:17:04.669 | INFO | } | 20260507 17:17:04.669 | INFO | Perf report for the last 5.13872 seconds: | 20260507 17:17:04.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:04.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:04.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:04.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:04.669 | INFO | ANALYSIS : 69.9%, 3.0, 108.9ms, 0 | 20260507 17:17:04.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:04.669 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 17:17:04.669 | INFO | PRE_HQ : 67.8%, 1.8, 64.6ms, 0 | 20260507 17:17:04.669 | INFO | BAZWRITER : 69.5%, 3.0, 107.2ms, 0 | 20260507 17:17:04.669 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:17:04.670 | INFO | Uploaded 36GiB to GPU in 2.58912s (13.9044GiB/s) | 20260507 17:17:04.670 | INFO | Downloaded 38GiB to GPU in 2.73149s (13.9118GiB/s) | 20260507 17:17:09.808 | INFO | Analyzing chunk frames = [1225216,1225728) | 20260507 17:17:09.808 | INFO | MemInfo: { | 20260507 17:17:09.808 | INFO | "gpu" : " 17 GB", | 20260507 17:17:09.808 | INFO | "huge" : "", | 20260507 17:17:09.809 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:09.809 | INFO | "malloc" : " 14 GB", | 20260507 17:17:09.809 | INFO | "pinned" : " 29 GB" | 20260507 17:17:09.809 | INFO | } | 20260507 17:17:09.809 | INFO | Perf report for the last 5.13974 seconds: | 20260507 17:17:09.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:09.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:17:09.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 17:17:09.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:09.809 | INFO | Uploaded 36GiB to GPU in 2.56852s (14.0159GiB/s) | 20260507 17:17:09.809 | INFO | Downloaded 34GiB to GPU in 2.43631s (13.9555GiB/s) | 20260507 17:17:14.947 | INFO | Analyzing chunk frames = [1225728,1226240) | 20260507 17:17:14.947 | INFO | MemInfo: { | 20260507 17:17:14.947 | INFO | "gpu" : " 17 GB", | 20260507 17:17:14.948 | INFO | "huge" : "", | 20260507 17:17:14.948 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:14.948 | INFO | "malloc" : " 14 GB", | 20260507 17:17:14.948 | INFO | "pinned" : " 29 GB" | 20260507 17:17:14.948 | INFO | } | 20260507 17:17:14.948 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:17:14.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:14.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260507 17:17:14.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260507 17:17:14.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:14.948 | INFO | Uploaded 36GiB to GPU in 2.58505s (13.9263GiB/s) | 20260507 17:17:14.948 | INFO | Downloaded 34GiB to GPU in 2.3934s (14.2057GiB/s) | 20260507 17:17:20.087 | INFO | Analyzing chunk frames = [1226240,1226752) | 20260507 17:17:20.087 | INFO | MemInfo: { | 20260507 17:17:20.087 | INFO | "gpu" : " 17 GB", | 20260507 17:17:20.087 | INFO | "huge" : "", | 20260507 17:17:20.087 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:20.087 | INFO | "malloc" : " 14 GB", | 20260507 17:17:20.087 | INFO | "pinned" : " 29 GB" | 20260507 17:17:20.087 | INFO | } | 20260507 17:17:20.087 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:17:20.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:20.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 17:17:20.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:17:20.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:20.087 | INFO | Uploaded 36GiB to GPU in 2.50241s (14.3861GiB/s) | 20260507 17:17:20.087 | INFO | Downloaded 34GiB to GPU in 2.34968s (14.47GiB/s) | 20260507 17:17:25.226 | INFO | Analyzing chunk frames = [1226752,1227264) | 20260507 17:17:25.226 | INFO | MemInfo: { | 20260507 17:17:25.226 | INFO | "gpu" : " 17 GB", | 20260507 17:17:25.226 | INFO | "huge" : "", | 20260507 17:17:25.227 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:25.227 | INFO | "malloc" : " 14 GB", | 20260507 17:17:25.227 | INFO | "pinned" : " 29 GB" | 20260507 17:17:25.227 | INFO | } | 20260507 17:17:25.227 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:17:25.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:25.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:17:25.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 17:17:25.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:25.227 | INFO | Uploaded 36GiB to GPU in 2.39078s (15.0579GiB/s) | 20260507 17:17:25.227 | INFO | Downloaded 34GiB to GPU in 2.46331s (13.8026GiB/s) | 20260507 17:17:30.365 | INFO | Analyzing chunk frames = [1227264,1227776) | 20260507 17:17:30.365 | INFO | MemInfo: { | 20260507 17:17:30.365 | INFO | "gpu" : " 17 GB", | 20260507 17:17:30.365 | INFO | "huge" : "", | 20260507 17:17:30.365 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:30.365 | INFO | "malloc" : " 14 GB", | 20260507 17:17:30.365 | INFO | "pinned" : " 29 GB" | 20260507 17:17:30.365 | INFO | } | 20260507 17:17:30.365 | INFO | Perf report for the last 5.1387 seconds: | 20260507 17:17:30.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:30.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.365 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:17:30.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.365 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260507 17:17:30.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:30.366 | INFO | Uploaded 36GiB to GPU in 2.38828s (15.0736GiB/s) | 20260507 17:17:30.366 | INFO | Downloaded 34GiB to GPU in 2.50611s (13.5669GiB/s) | 20260507 17:17:35.504 | INFO | Analyzing chunk frames = [1227776,1228288) | 20260507 17:17:35.504 | INFO | MemInfo: { | 20260507 17:17:35.504 | INFO | "gpu" : " 17 GB", | 20260507 17:17:35.504 | INFO | "huge" : "", | 20260507 17:17:35.504 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:35.504 | INFO | "malloc" : " 14 GB", | 20260507 17:17:35.505 | INFO | "pinned" : " 29 GB" | 20260507 17:17:35.505 | INFO | } | 20260507 17:17:35.505 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:17:35.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:35.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:17:35.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:17:35.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:35.505 | INFO | Uploaded 36GiB to GPU in 2.44572s (14.7196GiB/s) | 20260507 17:17:35.505 | INFO | Downloaded 34GiB to GPU in 2.4188s (14.0566GiB/s) | 20260507 17:17:40.644 | INFO | Analyzing chunk frames = [1228288,1228800) | 20260507 17:17:40.644 | INFO | MemInfo: { | 20260507 17:17:40.644 | INFO | "gpu" : " 17 GB", | 20260507 17:17:40.644 | INFO | "huge" : "", | 20260507 17:17:40.644 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:40.644 | INFO | "malloc" : " 14 GB", | 20260507 17:17:40.644 | INFO | "pinned" : " 29 GB" | 20260507 17:17:40.644 | INFO | } | 20260507 17:17:40.644 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:17:40.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:40.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:17:40.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:17:40.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:40.644 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:17:40.644 | INFO | Total mem usage: 16.4 GiB | 20260507 17:17:40.644 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:17:40.644 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 17:17:40.644 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:17:40.644 | INFO | High Water Breakdown: | 20260507 17:17:40.644 | INFO | 59.1% : 8.52 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:17:40.644 | INFO | 40.8% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:17:40.644 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:17:40.644 | INFO | ------------------------------------------ | 20260507 17:17:40.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:17:40.644 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:17:40.644 | INFO | Total mem usage: 26.5 GiB | 20260507 17:17:40.644 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:17:40.644 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:17:40.644 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:17:40.644 | INFO | High Water Breakdown: | 20260507 17:17:40.644 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:17:40.644 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:17:40.644 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:17:40.644 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:17:40.644 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:17:40.644 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:17:40.644 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:17:40.644 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:17:40.644 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:17:40.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:17:40.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:17:40.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:17:40.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:17:40.644 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:17:40.644 | INFO | ------------------------------------------ | 20260507 17:17:40.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:17:40.644 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:17:40.644 | INFO | Total mem usage: 27.8 GiB | 20260507 17:17:40.644 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:17:40.644 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:17:40.644 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:17:40.644 | INFO | High Water Breakdown: | 20260507 17:17:40.644 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:17:40.644 | INFO | ------------------------------------------ | 20260507 17:17:40.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:17:40.644 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:17:40.644 | INFO | Total mem usage: 15.7 GiB | 20260507 17:17:40.644 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:17:40.644 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:17:40.644 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:17:40.644 | INFO | High Water Breakdown: | 20260507 17:17:40.644 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:17:40.644 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:17:40.644 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:17:40.644 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:17:40.644 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:17:40.644 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:17:40.644 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:17:40.644 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:17:40.644 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:17:40.644 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:17:40.644 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:17:40.644 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:17:40.644 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:17:40.644 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:17:40.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:17:40.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:17:40.645 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:17:40.645 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:17:40.645 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:17:40.645 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:17:40.645 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:17:40.645 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:17:40.645 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:17:40.645 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:17:40.645 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:17:40.645 | INFO | ------------------------------------------ | 20260507 17:17:40.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:17:40.645 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:17:40.645 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:17:40.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:17:40.645 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.43e+01 / 2.94e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 0.29%, AggregateStats -- 9.84e-02 / 1.02e-01, 1.09e-02 / 6.27e-03 | 20260507 17:17:40.645 | INFO | PerfReport: 12.03%, Baseline -- 4.03e+00 / 4.47e+00, 6.16e-01 / 2.65e-01 | 20260507 17:17:40.645 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 1.06e-01 | 20260507 17:17:40.645 | INFO | PerfReport: 15.70%, DME -- 4.21e+01 / 4.50e+01, 4.40e+00 / 2.99e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.92e+00 / 2.07e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.89e+00, 4.41e-01 / 2.49e-01 | 20260507 17:17:40.645 | INFO | PerfReport: 3.77%, Metrics -- 1.26e+00 / 1.43e+00, 3.20e-01 / 1.97e-01 | 20260507 17:17:40.645 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.25e-01, 6.78e-02 / 4.25e-02 | 20260507 17:17:40.645 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:17:40.645 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:17:40.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:17:40.645 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 24.08%, Upload -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 23.93%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 31.15%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 1.89%, LockUpload -- 2.03e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 2.23%, LockDownload -- 2.40e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 16.71%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:17:40.645 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260507 17:17:40.645 | INFO | Uploaded 36GiB to GPU in 2.51645s (14.3059GiB/s) | 20260507 17:17:40.645 | INFO | Downloaded 34GiB to GPU in 2.50107s (13.5942GiB/s) | 20260507 17:17:40.741 | INFO | Starting a round of Baz IO | 20260507 17:17:44.275 | INFO | Populating RTMetrics report | 20260507 17:17:44.276 | INFO | RTMetrics report populated | 20260507 17:17:44.276 | INFO | Opening tmp RTMetrics json file | 20260507 17:17:44.276 | INFO | Serializing RTMetrics to json | 20260507 17:17:44.277 | INFO | Serialization done, swapping tmp file now... | 20260507 17:17:44.278 | INFO | RTMetrics done | 20260507 17:17:44.361 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.619 seconds | 20260507 17:17:44.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.44% / 0.02% / 51.54% | 20260507 17:17:44.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.71528% / 3.17891e-05% / 4.76837e-05% | 20260507 17:17:44.361 | INFO | Baz IO overall: | 20260507 17:17:44.361 | INFO | BytesWritten : 538903584919 | 20260507 17:17:44.361 | INFO | NumEvents : 261060193079 | 20260507 17:17:45.783 | INFO | Analyzing chunk frames = [1228800,1229312) | 20260507 17:17:45.783 | INFO | MemInfo: { | 20260507 17:17:45.783 | INFO | "gpu" : " 17 GB", | 20260507 17:17:45.783 | INFO | "huge" : "", | 20260507 17:17:45.783 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:45.783 | INFO | "malloc" : " 14 GB", | 20260507 17:17:45.783 | INFO | "pinned" : " 29 GB" | 20260507 17:17:45.783 | INFO | } | 20260507 17:17:45.784 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:17:45.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:45.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:45.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:45.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:45.784 | INFO | ANALYSIS : 69.9%, 3.0, 108.9ms, 0 | 20260507 17:17:45.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:45.784 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 17:17:45.784 | INFO | PRE_HQ : 67.6%, 1.9, 65.2ms, 0 | 20260507 17:17:45.784 | INFO | BAZWRITER : 69.8%, 3.0, 106.7ms, 0 | 20260507 17:17:45.784 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:17:45.784 | INFO | Uploaded 36GiB to GPU in 2.53633s (14.1937GiB/s) | 20260507 17:17:45.784 | INFO | Downloaded 38GiB to GPU in 2.72838s (13.9277GiB/s) | 20260507 17:17:50.923 | INFO | Analyzing chunk frames = [1229312,1229824) | 20260507 17:17:50.923 | INFO | MemInfo: { | 20260507 17:17:50.923 | INFO | "gpu" : " 17 GB", | 20260507 17:17:50.923 | INFO | "huge" : "", | 20260507 17:17:50.923 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:50.923 | INFO | "malloc" : " 14 GB", | 20260507 17:17:50.923 | INFO | "pinned" : " 29 GB" | 20260507 17:17:50.923 | INFO | } | 20260507 17:17:50.923 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:17:50.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:50.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.923 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:17:50.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.924 | INFO | PRE_HQ : 35.9%, 1.0, 18.7ms, 0 | 20260507 17:17:50.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:50.924 | INFO | Uploaded 36GiB to GPU in 2.53542s (14.1988GiB/s) | 20260507 17:17:50.924 | INFO | Downloaded 34GiB to GPU in 2.45439s (13.8527GiB/s) | 20260507 17:17:56.062 | INFO | Analyzing chunk frames = [1229824,1230336) | 20260507 17:17:56.062 | INFO | MemInfo: { | 20260507 17:17:56.062 | INFO | "gpu" : " 17 GB", | 20260507 17:17:56.062 | INFO | "huge" : "", | 20260507 17:17:56.062 | INFO | "huge_shared" : " 29 GB", | 20260507 17:17:56.062 | INFO | "malloc" : " 14 GB", | 20260507 17:17:56.062 | INFO | "pinned" : " 29 GB" | 20260507 17:17:56.062 | INFO | } | 20260507 17:17:56.062 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:17:56.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:17:56.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:17:56.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 17:17:56.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:17:56.062 | INFO | Uploaded 36GiB to GPU in 2.54392s (14.1514GiB/s) | 20260507 17:17:56.062 | INFO | Downloaded 34GiB to GPU in 2.44438s (13.9095GiB/s) | 20260507 17:18:01.201 | INFO | Analyzing chunk frames = [1230336,1230848) | 20260507 17:18:01.201 | INFO | MemInfo: { | 20260507 17:18:01.201 | INFO | "gpu" : " 17 GB", | 20260507 17:18:01.201 | INFO | "huge" : "", | 20260507 17:18:01.201 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:01.201 | INFO | "malloc" : " 14 GB", | 20260507 17:18:01.201 | INFO | "pinned" : " 29 GB" | 20260507 17:18:01.201 | INFO | } | 20260507 17:18:01.201 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:18:01.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:01.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.201 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:18:01.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.202 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:18:01.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:01.202 | INFO | Uploaded 36GiB to GPU in 2.48041s (14.5137GiB/s) | 20260507 17:18:01.202 | INFO | Downloaded 34GiB to GPU in 2.35402s (14.4434GiB/s) | 20260507 17:18:06.341 | INFO | Analyzing chunk frames = [1230848,1231360) | 20260507 17:18:06.341 | INFO | MemInfo: { | 20260507 17:18:06.341 | INFO | "gpu" : " 17 GB", | 20260507 17:18:06.341 | INFO | "huge" : "", | 20260507 17:18:06.341 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:06.341 | INFO | "malloc" : " 14 GB", | 20260507 17:18:06.341 | INFO | "pinned" : " 29 GB" | 20260507 17:18:06.341 | INFO | } | 20260507 17:18:06.341 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:18:06.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:06.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:18:06.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:18:06.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:06.341 | INFO | Uploaded 36GiB to GPU in 2.37785s (15.1397GiB/s) | 20260507 17:18:06.341 | INFO | Downloaded 34GiB to GPU in 2.45871s (13.8284GiB/s) | 20260507 17:18:11.480 | INFO | Analyzing chunk frames = [1231360,1231872) | 20260507 17:18:11.480 | INFO | MemInfo: { | 20260507 17:18:11.480 | INFO | "gpu" : " 17 GB", | 20260507 17:18:11.480 | INFO | "huge" : "", | 20260507 17:18:11.480 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:11.480 | INFO | "malloc" : " 14 GB", | 20260507 17:18:11.480 | INFO | "pinned" : " 29 GB" | 20260507 17:18:11.480 | INFO | } | 20260507 17:18:11.480 | INFO | Perf report for the last 5.13937 seconds: | 20260507 17:18:11.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:11.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.480 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:18:11.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.480 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 17:18:11.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:11.481 | INFO | Uploaded 36GiB to GPU in 2.37641s (15.1489GiB/s) | 20260507 17:18:11.481 | INFO | Downloaded 34GiB to GPU in 2.45446s (13.8523GiB/s) | 20260507 17:18:16.619 | INFO | Analyzing chunk frames = [1231872,1232384) | 20260507 17:18:16.619 | INFO | MemInfo: { | 20260507 17:18:16.619 | INFO | "gpu" : " 17 GB", | 20260507 17:18:16.619 | INFO | "huge" : "", | 20260507 17:18:16.619 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:16.619 | INFO | "malloc" : " 14 GB", | 20260507 17:18:16.619 | INFO | "pinned" : " 29 GB" | 20260507 17:18:16.619 | INFO | } | 20260507 17:18:16.620 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:18:16.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:16.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:18:16.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:18:16.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:16.620 | INFO | Uploaded 36GiB to GPU in 2.48296s (14.4988GiB/s) | 20260507 17:18:16.620 | INFO | Downloaded 34GiB to GPU in 2.41794s (14.0616GiB/s) | 20260507 17:18:21.759 | INFO | Analyzing chunk frames = [1232384,1232896) | 20260507 17:18:21.759 | INFO | MemInfo: { | 20260507 17:18:21.759 | INFO | "gpu" : " 17 GB", | 20260507 17:18:21.759 | INFO | "huge" : "", | 20260507 17:18:21.759 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:21.759 | INFO | "malloc" : " 14 GB", | 20260507 17:18:21.759 | INFO | "pinned" : " 29 GB" | 20260507 17:18:21.759 | INFO | } | 20260507 17:18:21.759 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:18:21.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:21.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.759 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 17:18:21.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.759 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:18:21.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:21.760 | INFO | Uploaded 36GiB to GPU in 2.54393s (14.1513GiB/s) | 20260507 17:18:21.760 | INFO | Downloaded 34GiB to GPU in 2.49616s (13.6209GiB/s) | 20260507 17:18:21.857 | INFO | Starting a round of Baz IO | 20260507 17:18:25.368 | INFO | Populating RTMetrics report | 20260507 17:18:25.368 | INFO | RTMetrics report populated | 20260507 17:18:25.368 | INFO | Opening tmp RTMetrics json file | 20260507 17:18:25.370 | INFO | Serializing RTMetrics to json | 20260507 17:18:25.386 | INFO | Serialization done, swapping tmp file now... | 20260507 17:18:25.386 | INFO | RTMetrics done | 20260507 17:18:25.448 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.591 seconds | 20260507 17:18:25.448 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.42% / 0.02% / 51.56% | 20260507 17:18:25.448 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.70195% / 4.76837e-05% / 2.78155e-05% | 20260507 17:18:25.448 | INFO | Baz IO overall: | 20260507 17:18:25.448 | INFO | BytesWritten : 540833652502 | 20260507 17:18:25.448 | INFO | NumEvents : 261998343278 | 20260507 17:18:26.898 | INFO | Analyzing chunk frames = [1232896,1233408) | 20260507 17:18:26.898 | INFO | MemInfo: { | 20260507 17:18:26.898 | INFO | "gpu" : " 17 GB", | 20260507 17:18:26.898 | INFO | "huge" : "", | 20260507 17:18:26.898 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:26.898 | INFO | "malloc" : " 14 GB", | 20260507 17:18:26.898 | INFO | "pinned" : " 29 GB" | 20260507 17:18:26.898 | INFO | } | 20260507 17:18:26.898 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:18:26.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:26.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:26.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:26.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:26.898 | INFO | ANALYSIS : 69.8%, 3.0, 108.7ms, 0 | 20260507 17:18:26.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:26.898 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 17:18:26.898 | INFO | PRE_HQ : 67.9%, 1.8, 64.6ms, 0 | 20260507 17:18:26.898 | INFO | BAZWRITER : 69.1%, 3.1, 108.7ms, 0 | 20260507 17:18:26.898 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:18:26.898 | INFO | Uploaded 36GiB to GPU in 2.58237s (13.9407GiB/s) | 20260507 17:18:26.898 | INFO | Downloaded 38GiB to GPU in 2.71965s (13.9724GiB/s) | 20260507 17:18:32.038 | INFO | Analyzing chunk frames = [1233408,1233920) | 20260507 17:18:32.038 | INFO | MemInfo: { | 20260507 17:18:32.038 | INFO | "gpu" : " 17 GB", | 20260507 17:18:32.038 | INFO | "huge" : "", | 20260507 17:18:32.038 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:32.038 | INFO | "malloc" : " 14 GB", | 20260507 17:18:32.038 | INFO | "pinned" : " 29 GB" | 20260507 17:18:32.038 | INFO | } | 20260507 17:18:32.038 | INFO | Perf report for the last 5.13973 seconds: | 20260507 17:18:32.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:32.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:18:32.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:18:32.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:32.038 | INFO | Uploaded 36GiB to GPU in 2.53051s (14.2264GiB/s) | 20260507 17:18:32.038 | INFO | Downloaded 34GiB to GPU in 2.42983s (13.9927GiB/s) | 20260507 17:18:37.177 | INFO | Analyzing chunk frames = [1233920,1234432) | 20260507 17:18:37.177 | INFO | MemInfo: { | 20260507 17:18:37.177 | INFO | "gpu" : " 17 GB", | 20260507 17:18:37.177 | INFO | "huge" : "", | 20260507 17:18:37.177 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:37.177 | INFO | "malloc" : " 14 GB", | 20260507 17:18:37.177 | INFO | "pinned" : " 29 GB" | 20260507 17:18:37.177 | INFO | } | 20260507 17:18:37.177 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:18:37.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:37.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.177 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:18:37.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.177 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 17:18:37.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:37.178 | INFO | Uploaded 36GiB to GPU in 2.55899s (14.0681GiB/s) | 20260507 17:18:37.178 | INFO | Downloaded 34GiB to GPU in 2.43096s (13.9862GiB/s) | 20260507 17:18:42.316 | INFO | Analyzing chunk frames = [1234432,1234944) | 20260507 17:18:42.316 | INFO | MemInfo: { | 20260507 17:18:42.316 | INFO | "gpu" : " 17 GB", | 20260507 17:18:42.316 | INFO | "huge" : "", | 20260507 17:18:42.316 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:42.316 | INFO | "malloc" : " 14 GB", | 20260507 17:18:42.316 | INFO | "pinned" : " 29 GB" | 20260507 17:18:42.316 | INFO | } | 20260507 17:18:42.316 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:18:42.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:42.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.316 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:18:42.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.316 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:18:42.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:42.317 | INFO | Uploaded 36GiB to GPU in 2.49714s (14.4165GiB/s) | 20260507 17:18:42.317 | INFO | Downloaded 34GiB to GPU in 2.3517s (14.4576GiB/s) | 20260507 17:18:47.455 | INFO | Analyzing chunk frames = [1234944,1235456) | 20260507 17:18:47.456 | INFO | MemInfo: { | 20260507 17:18:47.456 | INFO | "gpu" : " 17 GB", | 20260507 17:18:47.456 | INFO | "huge" : "", | 20260507 17:18:47.456 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:47.456 | INFO | "malloc" : " 14 GB", | 20260507 17:18:47.456 | INFO | "pinned" : " 29 GB" | 20260507 17:18:47.456 | INFO | } | 20260507 17:18:47.456 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:18:47.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:47.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260507 17:18:47.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 17:18:47.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:47.456 | INFO | Uploaded 36GiB to GPU in 2.41969s (14.878GiB/s) | 20260507 17:18:47.456 | INFO | Downloaded 34GiB to GPU in 2.45435s (13.853GiB/s) | 20260507 17:18:52.595 | INFO | Analyzing chunk frames = [1235456,1235968) | 20260507 17:18:52.595 | INFO | MemInfo: { | 20260507 17:18:52.595 | INFO | "gpu" : " 17 GB", | 20260507 17:18:52.595 | INFO | "huge" : "", | 20260507 17:18:52.595 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:52.595 | INFO | "malloc" : " 14 GB", | 20260507 17:18:52.595 | INFO | "pinned" : " 29 GB" | 20260507 17:18:52.595 | INFO | } | 20260507 17:18:52.595 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:18:52.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:52.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.595 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:18:52.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.595 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260507 17:18:52.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:52.596 | INFO | Uploaded 36GiB to GPU in 2.42129s (14.8681GiB/s) | 20260507 17:18:52.596 | INFO | Downloaded 34GiB to GPU in 2.47754s (13.7233GiB/s) | 20260507 17:18:57.740 | INFO | Analyzing chunk frames = [1235968,1236480) | 20260507 17:18:57.740 | INFO | MemInfo: { | 20260507 17:18:57.740 | INFO | "gpu" : " 17 GB", | 20260507 17:18:57.740 | INFO | "huge" : "", | 20260507 17:18:57.740 | INFO | "huge_shared" : " 29 GB", | 20260507 17:18:57.740 | INFO | "malloc" : " 14 GB", | 20260507 17:18:57.740 | INFO | "pinned" : " 29 GB" | 20260507 17:18:57.740 | INFO | } | 20260507 17:18:57.740 | INFO | Perf report for the last 5.14492 seconds: | 20260507 17:18:57.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:18:57.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.740 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:18:57.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.740 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260507 17:18:57.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:18:57.741 | INFO | Uploaded 36GiB to GPU in 2.47517s (14.5445GiB/s) | 20260507 17:18:57.741 | INFO | Downloaded 34GiB to GPU in 2.41476s (14.0801GiB/s) | 20260507 17:19:02.874 | INFO | Analyzing chunk frames = [1236480,1236992) | 20260507 17:19:02.874 | INFO | MemInfo: { | 20260507 17:19:02.874 | INFO | "gpu" : " 17 GB", | 20260507 17:19:02.874 | INFO | "huge" : "", | 20260507 17:19:02.874 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:02.874 | INFO | "malloc" : " 14 GB", | 20260507 17:19:02.874 | INFO | "pinned" : " 29 GB" | 20260507 17:19:02.874 | INFO | } | 20260507 17:19:02.874 | INFO | Perf report for the last 5.13392 seconds: | 20260507 17:19:02.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:02.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260507 17:19:02.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:19:02.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:02.874 | INFO | Uploaded 36GiB to GPU in 2.51887s (14.2921GiB/s) | 20260507 17:19:02.874 | INFO | Downloaded 34GiB to GPU in 2.50068s (13.5963GiB/s) | 20260507 17:19:02.874 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:19:02.874 | INFO | Total mem usage: 16.4 GiB | 20260507 17:19:02.874 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:19:02.874 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.3%) | 20260507 17:19:02.874 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:19:02.874 | INFO | High Water Breakdown: | 20260507 17:19:02.874 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:19:02.874 | INFO | 40.8% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:19:02.874 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:19:02.874 | INFO | ------------------------------------------ | 20260507 17:19:02.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:19:02.874 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:19:02.874 | INFO | Total mem usage: 26.5 GiB | 20260507 17:19:02.874 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:19:02.874 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:19:02.874 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:19:02.874 | INFO | High Water Breakdown: | 20260507 17:19:02.874 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:19:02.874 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:19:02.874 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:19:02.874 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:19:02.874 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:19:02.874 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:19:02.875 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:19:02.875 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:19:02.875 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:19:02.875 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:19:02.875 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:19:02.875 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:19:02.875 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:19:02.875 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:19:02.875 | INFO | ------------------------------------------ | 20260507 17:19:02.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:19:02.875 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:19:02.875 | INFO | Total mem usage: 27.8 GiB | 20260507 17:19:02.875 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:19:02.875 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:19:02.875 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:19:02.875 | INFO | High Water Breakdown: | 20260507 17:19:02.875 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:19:02.875 | INFO | ------------------------------------------ | 20260507 17:19:02.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:19:02.875 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:19:02.875 | INFO | Total mem usage: 15.7 GiB | 20260507 17:19:02.875 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:19:02.875 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:19:02.875 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:19:02.875 | INFO | High Water Breakdown: | 20260507 17:19:02.875 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:19:02.875 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:19:02.875 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:19:02.875 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:19:02.875 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:19:02.875 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:19:02.875 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:19:02.875 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:19:02.875 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:19:02.875 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:19:02.875 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:19:02.875 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:19:02.875 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:19:02.875 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:19:02.875 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:19:02.875 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:19:02.875 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:19:02.875 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:19:02.875 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:19:02.875 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:19:02.875 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:19:02.875 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:19:02.875 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:19:02.875 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:19:02.875 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:19:02.875 | INFO | ------------------------------------------ | 20260507 17:19:02.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:19:02.875 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:19:02.875 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:19:02.875 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:19:02.875 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.44e+01 / 2.92e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 0.30%, AggregateStats -- 9.93e-02 / 1.03e-01, 1.13e-02 / 6.49e-03 | 20260507 17:19:02.875 | INFO | PerfReport: 12.07%, Baseline -- 4.04e+00 / 4.48e+00, 6.18e-01 / 2.63e-01 | 20260507 17:19:02.875 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.18e-01 | 20260507 17:19:02.875 | INFO | PerfReport: 15.70%, DME -- 4.21e+01 / 4.50e+01, 4.74e+00 / 3.03e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.88e+00 / 2.37e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.51e+00 / 2.88e+00, 4.39e-01 / 2.60e-01 | 20260507 17:19:02.875 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.44e+00, 3.20e-01 / 1.91e-01 | 20260507 17:19:02.875 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.21e-01, 6.71e-02 / 4.13e-02 | 20260507 17:19:02.875 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:19:02.875 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:19:02.875 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:19:02.875 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 24.11%, Upload -- 2.60e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 23.93%, Download -- 2.58e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 31.12%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 1.95%, LockUpload -- 2.10e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 2.19%, LockDownload -- 2.36e+00 / 0.00e+00, 3.73e+00 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 16.68%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:19:02.875 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 17:19:02.972 | INFO | Starting a round of Baz IO | 20260507 17:19:06.483 | INFO | Populating RTMetrics report | 20260507 17:19:06.484 | INFO | RTMetrics report populated | 20260507 17:19:06.484 | INFO | Opening tmp RTMetrics json file | 20260507 17:19:06.485 | INFO | Serializing RTMetrics to json | 20260507 17:19:06.486 | INFO | Serialization done, swapping tmp file now... | 20260507 17:19:06.486 | INFO | RTMetrics done | 20260507 17:19:06.575 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.602 seconds | 20260507 17:19:06.575 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.40% / 0.02% / 51.58% | 20260507 17:19:06.575 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.69899% / 4.37101e-05% / 2.38419e-05% | 20260507 17:19:06.575 | INFO | Baz IO overall: | 20260507 17:19:06.575 | INFO | BytesWritten : 542764586213 | 20260507 17:19:06.575 | INFO | NumEvents : 262937215022 | 20260507 17:19:08.013 | INFO | Analyzing chunk frames = [1236992,1237504) | 20260507 17:19:08.013 | INFO | MemInfo: { | 20260507 17:19:08.013 | INFO | "gpu" : " 17 GB", | 20260507 17:19:08.013 | INFO | "huge" : "", | 20260507 17:19:08.013 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:08.013 | INFO | "malloc" : " 14 GB", | 20260507 17:19:08.013 | INFO | "pinned" : " 29 GB" | 20260507 17:19:08.013 | INFO | } | 20260507 17:19:08.013 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:19:08.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:08.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:08.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:08.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:08.013 | INFO | ANALYSIS : 69.8%, 3.0, 108.0ms, 0 | 20260507 17:19:08.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:08.013 | INFO | RT_METRICS : 48.0%, 1.0, 25.1ms, 0 | 20260507 17:19:08.013 | INFO | PRE_HQ : 67.6%, 2.0, 69.1ms, 0 | 20260507 17:19:08.013 | INFO | BAZWRITER : 69.6%, 3.2, 113.0ms, 0 | 20260507 17:19:08.013 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:19:08.014 | INFO | Uploaded 36GiB to GPU in 2.56499s (14.0351GiB/s) | 20260507 17:19:08.014 | INFO | Downloaded 38GiB to GPU in 2.71771s (13.9824GiB/s) | 20260507 17:19:13.152 | INFO | Analyzing chunk frames = [1237504,1238016) | 20260507 17:19:13.152 | INFO | MemInfo: { | 20260507 17:19:13.152 | INFO | "gpu" : " 17 GB", | 20260507 17:19:13.153 | INFO | "huge" : "", | 20260507 17:19:13.153 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:13.153 | INFO | "malloc" : " 14 GB", | 20260507 17:19:13.153 | INFO | "pinned" : " 29 GB" | 20260507 17:19:13.153 | INFO | } | 20260507 17:19:13.153 | INFO | Perf report for the last 5.13974 seconds: | 20260507 17:19:13.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:13.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:19:13.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260507 17:19:13.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:13.153 | INFO | Uploaded 36GiB to GPU in 2.52708s (14.2457GiB/s) | 20260507 17:19:13.153 | INFO | Downloaded 34GiB to GPU in 2.43647s (13.9546GiB/s) | 20260507 17:19:18.291 | INFO | Analyzing chunk frames = [1238016,1238528) | 20260507 17:19:18.291 | INFO | MemInfo: { | 20260507 17:19:18.291 | INFO | "gpu" : " 17 GB", | 20260507 17:19:18.292 | INFO | "huge" : "", | 20260507 17:19:18.292 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:18.292 | INFO | "malloc" : " 14 GB", | 20260507 17:19:18.292 | INFO | "pinned" : " 29 GB" | 20260507 17:19:18.292 | INFO | } | 20260507 17:19:18.292 | INFO | Perf report for the last 5.13894 seconds: | 20260507 17:19:18.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:18.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260507 17:19:18.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:19:18.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:18.292 | INFO | Uploaded 36GiB to GPU in 2.48952s (14.4606GiB/s) | 20260507 17:19:18.292 | INFO | Downloaded 34GiB to GPU in 2.34117s (14.5227GiB/s) | 20260507 17:19:23.434 | INFO | Analyzing chunk frames = [1238528,1239040) | 20260507 17:19:23.434 | INFO | MemInfo: { | 20260507 17:19:23.434 | INFO | "gpu" : " 17 GB", | 20260507 17:19:23.434 | INFO | "huge" : "", | 20260507 17:19:23.434 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:23.434 | INFO | "malloc" : " 14 GB", | 20260507 17:19:23.434 | INFO | "pinned" : " 29 GB" | 20260507 17:19:23.434 | INFO | } | 20260507 17:19:23.434 | INFO | Perf report for the last 5.14242 seconds: | 20260507 17:19:23.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:23.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.434 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:19:23.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.434 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:19:23.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:23.435 | INFO | Uploaded 36GiB to GPU in 2.50147s (14.3915GiB/s) | 20260507 17:19:23.435 | INFO | Downloaded 34GiB to GPU in 2.35512s (14.4367GiB/s) | 20260507 17:19:28.573 | INFO | Analyzing chunk frames = [1239040,1239552) | 20260507 17:19:28.573 | INFO | MemInfo: { | 20260507 17:19:28.573 | INFO | "gpu" : " 17 GB", | 20260507 17:19:28.574 | INFO | "huge" : "", | 20260507 17:19:28.574 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:28.574 | INFO | "malloc" : " 14 GB", | 20260507 17:19:28.574 | INFO | "pinned" : " 29 GB" | 20260507 17:19:28.574 | INFO | } | 20260507 17:19:28.574 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:19:28.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:28.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 17:19:28.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:19:28.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:28.574 | INFO | Uploaded 36GiB to GPU in 2.40831s (14.9482GiB/s) | 20260507 17:19:28.574 | INFO | Downloaded 34GiB to GPU in 2.44572s (13.9019GiB/s) | 20260507 17:19:33.709 | INFO | Analyzing chunk frames = [1239552,1240064) | 20260507 17:19:33.709 | INFO | MemInfo: { | 20260507 17:19:33.709 | INFO | "gpu" : " 17 GB", | 20260507 17:19:33.710 | INFO | "huge" : "", | 20260507 17:19:33.710 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:33.710 | INFO | "malloc" : " 14 GB", | 20260507 17:19:33.710 | INFO | "pinned" : " 29 GB" | 20260507 17:19:33.710 | INFO | } | 20260507 17:19:33.710 | INFO | Perf report for the last 5.13602 seconds: | 20260507 17:19:33.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:33.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:19:33.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260507 17:19:33.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:33.710 | INFO | Uploaded 36GiB to GPU in 2.4165s (14.8976GiB/s) | 20260507 17:19:33.710 | INFO | Downloaded 34GiB to GPU in 2.48946s (13.6576GiB/s) | 20260507 17:19:38.849 | INFO | Analyzing chunk frames = [1240064,1240576) | 20260507 17:19:38.849 | INFO | MemInfo: { | 20260507 17:19:38.849 | INFO | "gpu" : " 17 GB", | 20260507 17:19:38.849 | INFO | "huge" : "", | 20260507 17:19:38.849 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:38.849 | INFO | "malloc" : " 14 GB", | 20260507 17:19:38.849 | INFO | "pinned" : " 29 GB" | 20260507 17:19:38.849 | INFO | } | 20260507 17:19:38.849 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:19:38.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:38.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.849 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:19:38.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.849 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:19:38.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:38.850 | INFO | Uploaded 36GiB to GPU in 2.52318s (14.2677GiB/s) | 20260507 17:19:38.850 | INFO | Downloaded 34GiB to GPU in 2.43252s (13.9773GiB/s) | 20260507 17:19:43.988 | INFO | Analyzing chunk frames = [1240576,1241088) | 20260507 17:19:43.988 | INFO | MemInfo: { | 20260507 17:19:43.988 | INFO | "gpu" : " 17 GB", | 20260507 17:19:43.988 | INFO | "huge" : "", | 20260507 17:19:43.988 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:43.988 | INFO | "malloc" : " 14 GB", | 20260507 17:19:43.988 | INFO | "pinned" : " 29 GB" | 20260507 17:19:43.988 | INFO | } | 20260507 17:19:43.988 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:19:43.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:43.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 17:19:43.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 17:19:43.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:43.989 | INFO | Uploaded 36GiB to GPU in 2.48775s (14.4709GiB/s) | 20260507 17:19:43.989 | INFO | Downloaded 34GiB to GPU in 2.40968s (14.1098GiB/s) | 20260507 17:19:44.104 | INFO | Starting a round of Baz IO | 20260507 17:19:47.594 | INFO | Populating RTMetrics report | 20260507 17:19:47.595 | INFO | RTMetrics report populated | 20260507 17:19:47.595 | INFO | Opening tmp RTMetrics json file | 20260507 17:19:47.595 | INFO | Serializing RTMetrics to json | 20260507 17:19:47.605 | INFO | Serialization done, swapping tmp file now... | 20260507 17:19:47.614 | INFO | RTMetrics done | 20260507 17:19:47.672 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.568 seconds | 20260507 17:19:47.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.38% / 0.02% / 51.60% | 20260507 17:19:47.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.68659% / 3.97364e-05% / 5.96046e-05% | 20260507 17:19:47.672 | INFO | Baz IO overall: | 20260507 17:19:47.672 | INFO | BytesWritten : 544694305038 | 20260507 17:19:47.672 | INFO | NumEvents : 263875115085 | 20260507 17:19:49.127 | INFO | Analyzing chunk frames = [1241088,1241600) | 20260507 17:19:49.127 | INFO | MemInfo: { | 20260507 17:19:49.127 | INFO | "gpu" : " 17 GB", | 20260507 17:19:49.127 | INFO | "huge" : "", | 20260507 17:19:49.127 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:49.127 | INFO | "malloc" : " 14 GB", | 20260507 17:19:49.127 | INFO | "pinned" : " 29 GB" | 20260507 17:19:49.127 | INFO | } | 20260507 17:19:49.127 | INFO | Perf report for the last 5.13888 seconds: | 20260507 17:19:49.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:49.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:49.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:49.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:49.127 | INFO | ANALYSIS : 69.4%, 2.9, 106.8ms, 0 | 20260507 17:19:49.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:49.127 | INFO | RT_METRICS : 50.3%, 1.0, 26.4ms, 0 | 20260507 17:19:49.127 | INFO | PRE_HQ : 68.0%, 2.0, 72.1ms, 0 | 20260507 17:19:49.127 | INFO | BAZWRITER : 68.7%, 3.2, 114.6ms, 0 | 20260507 17:19:49.127 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:19:49.128 | INFO | Uploaded 36GiB to GPU in 2.5541s (14.095GiB/s) | 20260507 17:19:49.128 | INFO | Downloaded 38GiB to GPU in 2.70153s (14.0661GiB/s) | 20260507 17:19:54.266 | INFO | Analyzing chunk frames = [1241600,1242112) | 20260507 17:19:54.266 | INFO | MemInfo: { | 20260507 17:19:54.266 | INFO | "gpu" : " 17 GB", | 20260507 17:19:54.266 | INFO | "huge" : "", | 20260507 17:19:54.266 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:54.266 | INFO | "malloc" : " 14 GB", | 20260507 17:19:54.266 | INFO | "pinned" : " 29 GB" | 20260507 17:19:54.266 | INFO | } | 20260507 17:19:54.266 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:19:54.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:54.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.266 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:19:54.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.266 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:19:54.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:54.267 | INFO | Uploaded 36GiB to GPU in 2.53587s (14.1963GiB/s) | 20260507 17:19:54.267 | INFO | Downloaded 34GiB to GPU in 2.43305s (13.9743GiB/s) | 20260507 17:19:59.406 | INFO | Analyzing chunk frames = [1242112,1242624) | 20260507 17:19:59.406 | INFO | MemInfo: { | 20260507 17:19:59.406 | INFO | "gpu" : " 17 GB", | 20260507 17:19:59.406 | INFO | "huge" : "", | 20260507 17:19:59.406 | INFO | "huge_shared" : " 29 GB", | 20260507 17:19:59.406 | INFO | "malloc" : " 14 GB", | 20260507 17:19:59.406 | INFO | "pinned" : " 29 GB" | 20260507 17:19:59.406 | INFO | } | 20260507 17:19:59.406 | INFO | Perf report for the last 5.13987 seconds: | 20260507 17:19:59.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:19:59.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.406 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260507 17:19:59.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.406 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:19:59.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:19:59.407 | INFO | Uploaded 36GiB to GPU in 2.57692s (13.9702GiB/s) | 20260507 17:19:59.407 | INFO | Downloaded 34GiB to GPU in 2.40576s (14.1327GiB/s) | 20260507 17:20:04.545 | INFO | Analyzing chunk frames = [1242624,1243136) | 20260507 17:20:04.545 | INFO | MemInfo: { | 20260507 17:20:04.545 | INFO | "gpu" : " 17 GB", | 20260507 17:20:04.545 | INFO | "huge" : "", | 20260507 17:20:04.545 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:04.545 | INFO | "malloc" : " 14 GB", | 20260507 17:20:04.545 | INFO | "pinned" : " 29 GB" | 20260507 17:20:04.545 | INFO | } | 20260507 17:20:04.545 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:20:04.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:04.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.546 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:20:04.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.546 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:20:04.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:04.546 | INFO | Uploaded 36GiB to GPU in 2.49038s (14.4556GiB/s) | 20260507 17:20:04.546 | INFO | Downloaded 34GiB to GPU in 2.35032s (14.4661GiB/s) | 20260507 17:20:09.685 | INFO | Analyzing chunk frames = [1243136,1243648) | 20260507 17:20:09.685 | INFO | MemInfo: { | 20260507 17:20:09.685 | INFO | "gpu" : " 17 GB", | 20260507 17:20:09.685 | INFO | "huge" : "", | 20260507 17:20:09.685 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:09.685 | INFO | "malloc" : " 14 GB", | 20260507 17:20:09.685 | INFO | "pinned" : " 29 GB" | 20260507 17:20:09.685 | INFO | } | 20260507 17:20:09.685 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:20:09.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:09.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260507 17:20:09.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 17:20:09.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:09.685 | INFO | Uploaded 36GiB to GPU in 2.43985s (14.755GiB/s) | 20260507 17:20:09.685 | INFO | Downloaded 34GiB to GPU in 2.46505s (13.7928GiB/s) | 20260507 17:20:14.824 | INFO | Analyzing chunk frames = [1243648,1244160) | 20260507 17:20:14.824 | INFO | MemInfo: { | 20260507 17:20:14.824 | INFO | "gpu" : " 17 GB", | 20260507 17:20:14.824 | INFO | "huge" : "", | 20260507 17:20:14.824 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:14.824 | INFO | "malloc" : " 14 GB", | 20260507 17:20:14.824 | INFO | "pinned" : " 29 GB" | 20260507 17:20:14.824 | INFO | } | 20260507 17:20:14.824 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:20:14.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:14.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:20:14.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 17:20:14.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:14.824 | INFO | Uploaded 36GiB to GPU in 2.41349s (14.9162GiB/s) | 20260507 17:20:14.824 | INFO | Downloaded 34GiB to GPU in 2.50674s (13.5634GiB/s) | 20260507 17:20:19.963 | INFO | Analyzing chunk frames = [1244160,1244672) | 20260507 17:20:19.963 | INFO | MemInfo: { | 20260507 17:20:19.963 | INFO | "gpu" : " 17 GB", | 20260507 17:20:19.963 | INFO | "huge" : "", | 20260507 17:20:19.963 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:19.963 | INFO | "malloc" : " 14 GB", | 20260507 17:20:19.963 | INFO | "pinned" : " 29 GB" | 20260507 17:20:19.963 | INFO | } | 20260507 17:20:19.963 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:20:19.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:19.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.963 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:20:19.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.963 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:20:19.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:19.964 | INFO | Uploaded 36GiB to GPU in 2.46419s (14.6093GiB/s) | 20260507 17:20:19.964 | INFO | Downloaded 34GiB to GPU in 2.42728s (14.0075GiB/s) | 20260507 17:20:25.102 | INFO | Analyzing chunk frames = [1244672,1245184) | 20260507 17:20:25.102 | INFO | MemInfo: { | 20260507 17:20:25.102 | INFO | "gpu" : " 17 GB", | 20260507 17:20:25.102 | INFO | "huge" : "", | 20260507 17:20:25.103 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:25.103 | INFO | "malloc" : " 14 GB", | 20260507 17:20:25.103 | INFO | "pinned" : " 29 GB" | 20260507 17:20:25.103 | INFO | } | 20260507 17:20:25.103 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:20:25.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:25.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 17:20:25.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:20:25.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:25.103 | INFO | Uploaded 36GiB to GPU in 2.49325s (14.439GiB/s) | 20260507 17:20:25.103 | INFO | Downloaded 34GiB to GPU in 2.50526s (13.5714GiB/s) | 20260507 17:20:25.103 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:20:25.103 | INFO | Total mem usage: 16.4 GiB | 20260507 17:20:25.103 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:20:25.103 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.2%) | 20260507 17:20:25.103 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:20:25.103 | INFO | High Water Breakdown: | 20260507 17:20:25.103 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:20:25.103 | INFO | 40.8% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:20:25.103 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:20:25.103 | INFO | ------------------------------------------ | 20260507 17:20:25.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:20:25.104 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:20:25.104 | INFO | Total mem usage: 26.5 GiB | 20260507 17:20:25.104 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:20:25.104 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:20:25.104 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:20:25.104 | INFO | High Water Breakdown: | 20260507 17:20:25.104 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:20:25.104 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:20:25.104 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:20:25.104 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:20:25.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:20:25.104 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:20:25.104 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:20:25.104 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:20:25.104 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:20:25.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:20:25.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:20:25.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:20:25.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:20:25.104 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:20:25.104 | INFO | ------------------------------------------ | 20260507 17:20:25.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:20:25.104 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:20:25.104 | INFO | Total mem usage: 27.8 GiB | 20260507 17:20:25.104 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:20:25.104 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:20:25.104 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:20:25.104 | INFO | High Water Breakdown: | 20260507 17:20:25.104 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:20:25.104 | INFO | ------------------------------------------ | 20260507 17:20:25.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:20:25.104 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:20:25.104 | INFO | Total mem usage: 15.7 GiB | 20260507 17:20:25.104 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:20:25.104 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:20:25.104 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:20:25.104 | INFO | High Water Breakdown: | 20260507 17:20:25.104 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:20:25.104 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:20:25.104 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:20:25.104 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:20:25.104 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:20:25.104 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:20:25.104 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:20:25.104 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:20:25.104 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:20:25.104 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:20:25.104 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:20:25.104 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:20:25.104 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:20:25.104 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:20:25.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:20:25.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:20:25.104 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:20:25.104 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:20:25.104 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:20:25.104 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:20:25.104 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:20:25.104 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:20:25.104 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:20:25.104 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:20:25.104 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:20:25.104 | INFO | ------------------------------------------ | 20260507 17:20:25.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:20:25.104 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:20:25.104 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:20:25.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:20:25.104 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.60e+01, 1.42e+01 / 2.52e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 0.30%, AggregateStats -- 9.93e-02 / 1.03e-01, 1.12e-02 / 6.19e-03 | 20260507 17:20:25.104 | INFO | PerfReport: 12.04%, Baseline -- 4.04e+00 / 4.49e+00, 6.14e-01 / 3.01e-01 | 20260507 17:20:25.104 | INFO | PerfReport: 3.41%, Histogram -- 1.14e+00 / 1.48e+00, 2.12e-01 / 1.17e-01 | 20260507 17:20:25.104 | INFO | PerfReport: 15.60%, DME -- 4.18e+01 / 4.46e+01, 4.36e+00 / 2.37e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.95e+00 / 2.17e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.90e+00, 4.38e-01 / 2.10e-01 | 20260507 17:20:25.104 | INFO | PerfReport: 3.80%, Metrics -- 1.27e+00 / 1.44e+00, 3.31e-01 / 1.94e-01 | 20260507 17:20:25.104 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.03e-01 / 4.29e-01, 6.69e-02 / 3.99e-02 | 20260507 17:20:25.104 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:20:25.104 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:20:25.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:20:25.104 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 24.19%, Upload -- 2.60e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 23.91%, Download -- 2.57e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 31.23%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 1.92%, LockUpload -- 2.06e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 2.12%, LockDownload -- 2.27e+00 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 17:20:25.104 | INFO | PerfReport: 16.63%, LockCompute -- 1.79e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 17:20:25.105 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260507 17:20:25.199 | INFO | Starting a round of Baz IO | 20260507 17:20:28.700 | INFO | Populating RTMetrics report | 20260507 17:20:28.700 | INFO | RTMetrics report populated | 20260507 17:20:28.700 | INFO | Opening tmp RTMetrics json file | 20260507 17:20:28.717 | INFO | Serializing RTMetrics to json | 20260507 17:20:28.718 | INFO | Serialization done, swapping tmp file now... | 20260507 17:20:28.718 | INFO | RTMetrics done | 20260507 17:20:28.778 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.578 seconds | 20260507 17:20:28.778 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.36% / 0.02% / 51.62% | 20260507 17:20:28.778 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.68478% / 3.97364e-05% / 3.57628e-05% | 20260507 17:20:28.778 | INFO | Baz IO overall: | 20260507 17:20:28.778 | INFO | BytesWritten : 546626363481 | 20260507 17:20:28.778 | INFO | NumEvents : 264814716314 | 20260507 17:20:30.242 | INFO | Analyzing chunk frames = [1245184,1245696) | 20260507 17:20:30.242 | INFO | MemInfo: { | 20260507 17:20:30.242 | INFO | "gpu" : " 17 GB", | 20260507 17:20:30.242 | INFO | "huge" : "", | 20260507 17:20:30.242 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:30.242 | INFO | "malloc" : " 14 GB", | 20260507 17:20:30.242 | INFO | "pinned" : " 29 GB" | 20260507 17:20:30.242 | INFO | } | 20260507 17:20:30.242 | INFO | Perf report for the last 5.13905 seconds: | 20260507 17:20:30.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:30.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:30.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:30.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:30.242 | INFO | ANALYSIS : 69.5%, 3.0, 108.3ms, 0 | 20260507 17:20:30.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:30.242 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260507 17:20:30.242 | INFO | PRE_HQ : 67.7%, 1.8, 64.4ms, 0 | 20260507 17:20:30.242 | INFO | BAZWRITER : 69.0%, 3.0, 106.4ms, 0 | 20260507 17:20:30.242 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:20:30.243 | INFO | Uploaded 36GiB to GPU in 2.53552s (14.1982GiB/s) | 20260507 17:20:30.243 | INFO | Downloaded 38GiB to GPU in 2.74016s (13.8678GiB/s) | 20260507 17:20:35.381 | INFO | Analyzing chunk frames = [1245696,1246208) | 20260507 17:20:35.381 | INFO | MemInfo: { | 20260507 17:20:35.381 | INFO | "gpu" : " 17 GB", | 20260507 17:20:35.381 | INFO | "huge" : "", | 20260507 17:20:35.381 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:35.381 | INFO | "malloc" : " 14 GB", | 20260507 17:20:35.381 | INFO | "pinned" : " 29 GB" | 20260507 17:20:35.381 | INFO | } | 20260507 17:20:35.381 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:20:35.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:35.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.381 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:20:35.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.381 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:20:35.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:35.382 | INFO | Uploaded 36GiB to GPU in 2.45362s (14.6722GiB/s) | 20260507 17:20:35.382 | INFO | Downloaded 34GiB to GPU in 2.45814s (13.8316GiB/s) | 20260507 17:20:40.520 | INFO | Analyzing chunk frames = [1246208,1246720) | 20260507 17:20:40.520 | INFO | MemInfo: { | 20260507 17:20:40.520 | INFO | "gpu" : " 17 GB", | 20260507 17:20:40.520 | INFO | "huge" : "", | 20260507 17:20:40.520 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:40.520 | INFO | "malloc" : " 14 GB", | 20260507 17:20:40.520 | INFO | "pinned" : " 29 GB" | 20260507 17:20:40.520 | INFO | } | 20260507 17:20:40.520 | INFO | Perf report for the last 5.13905 seconds: | 20260507 17:20:40.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:40.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.520 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260507 17:20:40.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.520 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:20:40.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:40.521 | INFO | Uploaded 36GiB to GPU in 2.42725s (14.8316GiB/s) | 20260507 17:20:40.521 | INFO | Downloaded 34GiB to GPU in 2.35749s (14.4221GiB/s) | 20260507 17:20:45.661 | INFO | Analyzing chunk frames = [1246720,1247232) | 20260507 17:20:45.661 | INFO | MemInfo: { | 20260507 17:20:45.661 | INFO | "gpu" : " 17 GB", | 20260507 17:20:45.661 | INFO | "huge" : "", | 20260507 17:20:45.661 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:45.661 | INFO | "malloc" : " 14 GB", | 20260507 17:20:45.661 | INFO | "pinned" : " 29 GB" | 20260507 17:20:45.661 | INFO | } | 20260507 17:20:45.661 | INFO | Perf report for the last 5.14046 seconds: | 20260507 17:20:45.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:45.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 17:20:45.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:20:45.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:45.661 | INFO | Uploaded 36GiB to GPU in 2.43724s (14.7708GiB/s) | 20260507 17:20:45.661 | INFO | Downloaded 34GiB to GPU in 2.36158s (14.3972GiB/s) | 20260507 17:20:50.800 | INFO | Analyzing chunk frames = [1247232,1247744) | 20260507 17:20:50.800 | INFO | MemInfo: { | 20260507 17:20:50.800 | INFO | "gpu" : " 17 GB", | 20260507 17:20:50.800 | INFO | "huge" : "", | 20260507 17:20:50.800 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:50.800 | INFO | "malloc" : " 14 GB", | 20260507 17:20:50.800 | INFO | "pinned" : " 29 GB" | 20260507 17:20:50.800 | INFO | } | 20260507 17:20:50.800 | INFO | Perf report for the last 5.13898 seconds: | 20260507 17:20:50.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:50.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:20:50.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 17:20:50.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:50.800 | INFO | Uploaded 36GiB to GPU in 2.3105s (15.581GiB/s) | 20260507 17:20:50.801 | INFO | Downloaded 34GiB to GPU in 2.48279s (13.6943GiB/s) | 20260507 17:20:55.938 | INFO | Analyzing chunk frames = [1247744,1248256) | 20260507 17:20:55.938 | INFO | MemInfo: { | 20260507 17:20:55.938 | INFO | "gpu" : " 17 GB", | 20260507 17:20:55.938 | INFO | "huge" : "", | 20260507 17:20:55.938 | INFO | "huge_shared" : " 29 GB", | 20260507 17:20:55.938 | INFO | "malloc" : " 14 GB", | 20260507 17:20:55.938 | INFO | "pinned" : " 29 GB" | 20260507 17:20:55.938 | INFO | } | 20260507 17:20:55.938 | INFO | Perf report for the last 5.13843 seconds: | 20260507 17:20:55.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:20:55.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.938 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:20:55.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.938 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:20:55.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:20:55.939 | INFO | Uploaded 36GiB to GPU in 2.34463s (15.3542GiB/s) | 20260507 17:20:55.939 | INFO | Downloaded 34GiB to GPU in 2.5154s (13.5167GiB/s) | 20260507 17:21:01.078 | INFO | Analyzing chunk frames = [1248256,1248768) | 20260507 17:21:01.078 | INFO | MemInfo: { | 20260507 17:21:01.078 | INFO | "gpu" : " 17 GB", | 20260507 17:21:01.078 | INFO | "huge" : "", | 20260507 17:21:01.078 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:01.078 | INFO | "malloc" : " 14 GB", | 20260507 17:21:01.078 | INFO | "pinned" : " 29 GB" | 20260507 17:21:01.078 | INFO | } | 20260507 17:21:01.078 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:21:01.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:01.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 17:21:01.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:21:01.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:01.078 | INFO | Uploaded 36GiB to GPU in 2.38816s (15.0744GiB/s) | 20260507 17:21:01.078 | INFO | Downloaded 34GiB to GPU in 2.41356s (14.0871GiB/s) | 20260507 17:21:06.217 | INFO | Analyzing chunk frames = [1248768,1249280) | 20260507 17:21:06.217 | INFO | MemInfo: { | 20260507 17:21:06.217 | INFO | "gpu" : " 17 GB", | 20260507 17:21:06.217 | INFO | "huge" : "", | 20260507 17:21:06.217 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:06.217 | INFO | "malloc" : " 14 GB", | 20260507 17:21:06.217 | INFO | "pinned" : " 29 GB" | 20260507 17:21:06.217 | INFO | } | 20260507 17:21:06.217 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:21:06.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:06.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.217 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:21:06.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.217 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:21:06.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:06.218 | INFO | Uploaded 36GiB to GPU in 2.49691s (14.4178GiB/s) | 20260507 17:21:06.218 | INFO | Downloaded 34GiB to GPU in 2.51844s (13.5004GiB/s) | 20260507 17:21:06.315 | INFO | Starting a round of Baz IO | 20260507 17:21:09.818 | INFO | Populating RTMetrics report | 20260507 17:21:09.818 | INFO | RTMetrics report populated | 20260507 17:21:09.818 | INFO | Opening tmp RTMetrics json file | 20260507 17:21:09.820 | INFO | Serializing RTMetrics to json | 20260507 17:21:09.827 | INFO | Serialization done, swapping tmp file now... | 20260507 17:21:09.838 | INFO | RTMetrics done | 20260507 17:21:09.905 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.59 seconds | 20260507 17:21:09.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.33% / 0.02% / 51.64% | 20260507 17:21:09.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.66913% / 2.38419e-05% / 1.98682e-05% | 20260507 17:21:09.905 | INFO | Baz IO overall: | 20260507 17:21:09.905 | INFO | BytesWritten : 548556183139 | 20260507 17:21:09.906 | INFO | NumEvents : 265752663764 | 20260507 17:21:11.356 | INFO | Analyzing chunk frames = [1249280,1249792) | 20260507 17:21:11.356 | INFO | MemInfo: { | 20260507 17:21:11.356 | INFO | "gpu" : " 17 GB", | 20260507 17:21:11.356 | INFO | "huge" : "", | 20260507 17:21:11.356 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:11.356 | INFO | "malloc" : " 14 GB", | 20260507 17:21:11.356 | INFO | "pinned" : " 29 GB" | 20260507 17:21:11.356 | INFO | } | 20260507 17:21:11.356 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:21:11.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:11.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:11.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:11.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:11.357 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 17:21:11.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:11.357 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 17:21:11.357 | INFO | PRE_HQ : 67.3%, 1.8, 64.0ms, 0 | 20260507 17:21:11.357 | INFO | BAZWRITER : 69.0%, 3.0, 107.3ms, 0 | 20260507 17:21:11.357 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 17:21:11.357 | INFO | Uploaded 36GiB to GPU in 2.50993s (14.343GiB/s) | 20260507 17:21:11.357 | INFO | Downloaded 38GiB to GPU in 2.72467s (13.9466GiB/s) | 20260507 17:21:16.496 | INFO | Analyzing chunk frames = [1249792,1250304) | 20260507 17:21:16.496 | INFO | MemInfo: { | 20260507 17:21:16.496 | INFO | "gpu" : " 17 GB", | 20260507 17:21:16.496 | INFO | "huge" : "", | 20260507 17:21:16.496 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:16.496 | INFO | "malloc" : " 14 GB", | 20260507 17:21:16.496 | INFO | "pinned" : " 29 GB" | 20260507 17:21:16.496 | INFO | } | 20260507 17:21:16.496 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:21:16.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:16.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:21:16.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:21:16.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:16.496 | INFO | Uploaded 36GiB to GPU in 2.48579s (14.4823GiB/s) | 20260507 17:21:16.496 | INFO | Downloaded 34GiB to GPU in 2.45806s (13.8321GiB/s) | 20260507 17:21:21.636 | INFO | Analyzing chunk frames = [1250304,1250816) | 20260507 17:21:21.636 | INFO | MemInfo: { | 20260507 17:21:21.636 | INFO | "gpu" : " 17 GB", | 20260507 17:21:21.636 | INFO | "huge" : "", | 20260507 17:21:21.636 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:21.636 | INFO | "malloc" : " 14 GB", | 20260507 17:21:21.636 | INFO | "pinned" : " 29 GB" | 20260507 17:21:21.636 | INFO | } | 20260507 17:21:21.636 | INFO | Perf report for the last 5.13978 seconds: | 20260507 17:21:21.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:21.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260507 17:21:21.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:21:21.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:21.636 | INFO | Uploaded 36GiB to GPU in 2.43815s (14.7653GiB/s) | 20260507 17:21:21.636 | INFO | Downloaded 34GiB to GPU in 2.36511s (14.3757GiB/s) | 20260507 17:21:26.775 | INFO | Analyzing chunk frames = [1250816,1251328) | 20260507 17:21:26.775 | INFO | MemInfo: { | 20260507 17:21:26.775 | INFO | "gpu" : " 17 GB", | 20260507 17:21:26.775 | INFO | "huge" : "", | 20260507 17:21:26.775 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:26.775 | INFO | "malloc" : " 14 GB", | 20260507 17:21:26.775 | INFO | "pinned" : " 29 GB" | 20260507 17:21:26.775 | INFO | } | 20260507 17:21:26.775 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:21:26.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:26.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 17:21:26.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:21:26.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:26.775 | INFO | Uploaded 36GiB to GPU in 2.45786s (14.6469GiB/s) | 20260507 17:21:26.775 | INFO | Downloaded 34GiB to GPU in 2.36629s (14.3685GiB/s) | 20260507 17:21:31.913 | INFO | Analyzing chunk frames = [1251328,1251840) | 20260507 17:21:31.913 | INFO | MemInfo: { | 20260507 17:21:31.913 | INFO | "gpu" : " 17 GB", | 20260507 17:21:31.913 | INFO | "huge" : "", | 20260507 17:21:31.914 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:31.914 | INFO | "malloc" : " 14 GB", | 20260507 17:21:31.914 | INFO | "pinned" : " 29 GB" | 20260507 17:21:31.914 | INFO | } | 20260507 17:21:31.914 | INFO | Perf report for the last 5.13871 seconds: | 20260507 17:21:31.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:31.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:21:31.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 17:21:31.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:31.914 | INFO | Uploaded 36GiB to GPU in 2.32632s (15.4751GiB/s) | 20260507 17:21:31.914 | INFO | Downloaded 34GiB to GPU in 2.4845s (13.6849GiB/s) | 20260507 17:21:37.052 | INFO | Analyzing chunk frames = [1251840,1252352) | 20260507 17:21:37.052 | INFO | MemInfo: { | 20260507 17:21:37.052 | INFO | "gpu" : " 17 GB", | 20260507 17:21:37.052 | INFO | "huge" : "", | 20260507 17:21:37.052 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:37.052 | INFO | "malloc" : " 14 GB", | 20260507 17:21:37.052 | INFO | "pinned" : " 29 GB" | 20260507 17:21:37.052 | INFO | } | 20260507 17:21:37.052 | INFO | Perf report for the last 5.13873 seconds: | 20260507 17:21:37.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:37.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.052 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:21:37.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.053 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:21:37.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:37.053 | INFO | Uploaded 36GiB to GPU in 2.36572s (15.2174GiB/s) | 20260507 17:21:37.053 | INFO | Downloaded 34GiB to GPU in 2.46604s (13.7873GiB/s) | 20260507 17:21:42.192 | INFO | Analyzing chunk frames = [1252352,1252864) | 20260507 17:21:42.192 | INFO | MemInfo: { | 20260507 17:21:42.192 | INFO | "gpu" : " 17 GB", | 20260507 17:21:42.192 | INFO | "huge" : "", | 20260507 17:21:42.192 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:42.192 | INFO | "malloc" : " 14 GB", | 20260507 17:21:42.192 | INFO | "pinned" : " 29 GB" | 20260507 17:21:42.192 | INFO | } | 20260507 17:21:42.192 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:21:42.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:42.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.192 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:21:42.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.192 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:21:42.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:42.193 | INFO | Uploaded 36GiB to GPU in 2.48472s (14.4885GiB/s) | 20260507 17:21:42.193 | INFO | Downloaded 34GiB to GPU in 2.44034s (13.9325GiB/s) | 20260507 17:21:47.331 | INFO | Analyzing chunk frames = [1252864,1253376) | 20260507 17:21:47.331 | INFO | MemInfo: { | 20260507 17:21:47.331 | INFO | "gpu" : " 17 GB", | 20260507 17:21:47.331 | INFO | "huge" : "", | 20260507 17:21:47.331 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:47.331 | INFO | "malloc" : " 14 GB", | 20260507 17:21:47.331 | INFO | "pinned" : " 29 GB" | 20260507 17:21:47.331 | INFO | } | 20260507 17:21:47.331 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:21:47.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:47.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.332 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:21:47.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.332 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:21:47.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:47.332 | INFO | Uploaded 36GiB to GPU in 2.51416s (14.3189GiB/s) | 20260507 17:21:47.332 | INFO | Downloaded 34GiB to GPU in 2.50648s (13.5649GiB/s) | 20260507 17:21:47.332 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:21:47.332 | INFO | Total mem usage: 16.4 GiB | 20260507 17:21:47.332 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:21:47.332 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -14.1%) | 20260507 17:21:47.332 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:21:47.332 | INFO | High Water Breakdown: | 20260507 17:21:47.332 | INFO | 59% : 8.48 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:21:47.332 | INFO | 40.9% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:21:47.332 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:21:47.332 | INFO | ------------------------------------------ | 20260507 17:21:47.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:21:47.332 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:21:47.332 | INFO | Total mem usage: 26.5 GiB | 20260507 17:21:47.332 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:21:47.332 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:21:47.332 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:21:47.332 | INFO | High Water Breakdown: | 20260507 17:21:47.332 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:21:47.332 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:21:47.332 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:21:47.332 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:21:47.332 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:21:47.332 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:21:47.332 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:21:47.332 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:21:47.332 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:21:47.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:21:47.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:21:47.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:21:47.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:21:47.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:21:47.332 | INFO | ------------------------------------------ | 20260507 17:21:47.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:21:47.332 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:21:47.332 | INFO | Total mem usage: 27.8 GiB | 20260507 17:21:47.332 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:21:47.332 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:21:47.333 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:21:47.333 | INFO | High Water Breakdown: | 20260507 17:21:47.333 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:21:47.333 | INFO | ------------------------------------------ | 20260507 17:21:47.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:21:47.333 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:21:47.333 | INFO | Total mem usage: 15.7 GiB | 20260507 17:21:47.333 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:21:47.333 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:21:47.333 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:21:47.333 | INFO | High Water Breakdown: | 20260507 17:21:47.333 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:21:47.333 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:21:47.333 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:21:47.333 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:21:47.333 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:21:47.333 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:21:47.333 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:21:47.333 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:21:47.333 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:21:47.333 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:21:47.333 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:21:47.333 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:21:47.333 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:21:47.333 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:21:47.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:21:47.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:21:47.333 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:21:47.333 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:21:47.333 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:21:47.333 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:21:47.333 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:21:47.333 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:21:47.333 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:21:47.333 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:21:47.333 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:21:47.333 | INFO | ------------------------------------------ | 20260507 17:21:47.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:21:47.333 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:21:47.333 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:21:47.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:21:47.333 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.43e+01 / 2.70e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.11e-02 / 7.61e-03 | 20260507 17:21:47.333 | INFO | PerfReport: 12.06%, Baseline -- 4.04e+00 / 4.50e+00, 6.17e-01 / 2.31e-01 | 20260507 17:21:47.333 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.47e+00, 2.15e-01 / 1.24e-01 | 20260507 17:21:47.333 | INFO | PerfReport: 15.77%, DME -- 4.23e+01 / 4.53e+01, 4.36e+00 / 2.50e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 2.20e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.53e+00 / 2.91e+00, 4.36e-01 / 2.11e-01 | 20260507 17:21:47.333 | INFO | PerfReport: 3.79%, Metrics -- 1.27e+00 / 1.46e+00, 3.21e-01 / 1.91e-01 | 20260507 17:21:47.333 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.27e-01, 6.76e-02 / 3.96e-02 | 20260507 17:21:47.333 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:21:47.333 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:21:47.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:21:47.333 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 23.67%, Upload -- 2.54e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 24.09%, Download -- 2.58e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 31.26%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 1.54%, LockUpload -- 1.65e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 2.61%, LockDownload -- 2.79e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 16.81%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:21:47.333 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260507 17:21:47.429 | INFO | Starting a round of Baz IO | 20260507 17:21:50.959 | INFO | Populating RTMetrics report | 20260507 17:21:50.960 | INFO | RTMetrics report populated | 20260507 17:21:50.960 | INFO | Opening tmp RTMetrics json file | 20260507 17:21:50.961 | INFO | Serializing RTMetrics to json | 20260507 17:21:50.962 | INFO | Serialization done, swapping tmp file now... | 20260507 17:21:50.962 | INFO | RTMetrics done | 20260507 17:21:51.051 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.622 seconds | 20260507 17:21:51.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.31% / 0.02% / 51.67% | 20260507 17:21:51.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.66971% / 1.58946e-05% / 3.57628e-05% | 20260507 17:21:51.051 | INFO | Baz IO overall: | 20260507 17:21:51.051 | INFO | BytesWritten : 550487755261 | 20260507 17:21:51.052 | INFO | NumEvents : 266691948268 | 20260507 17:21:52.470 | INFO | Analyzing chunk frames = [1253376,1253888) | 20260507 17:21:52.470 | INFO | MemInfo: { | 20260507 17:21:52.470 | INFO | "gpu" : " 17 GB", | 20260507 17:21:52.470 | INFO | "huge" : "", | 20260507 17:21:52.470 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:52.470 | INFO | "malloc" : " 14 GB", | 20260507 17:21:52.470 | INFO | "pinned" : " 29 GB" | 20260507 17:21:52.470 | INFO | } | 20260507 17:21:52.470 | INFO | Perf report for the last 5.13897 seconds: | 20260507 17:21:52.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:52.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:52.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:52.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:52.470 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:21:52.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:52.470 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 17:21:52.470 | INFO | PRE_HQ : 68.0%, 1.8, 64.5ms, 0 | 20260507 17:21:52.470 | INFO | BAZWRITER : 69.7%, 3.0, 107.5ms, 0 | 20260507 17:21:52.470 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:21:52.471 | INFO | Uploaded 36GiB to GPU in 2.54927s (14.1217GiB/s) | 20260507 17:21:52.471 | INFO | Downloaded 38GiB to GPU in 2.72983s (13.9203GiB/s) | 20260507 17:21:57.609 | INFO | Analyzing chunk frames = [1253888,1254400) | 20260507 17:21:57.609 | INFO | MemInfo: { | 20260507 17:21:57.610 | INFO | "gpu" : " 17 GB", | 20260507 17:21:57.610 | INFO | "huge" : "", | 20260507 17:21:57.610 | INFO | "huge_shared" : " 29 GB", | 20260507 17:21:57.610 | INFO | "malloc" : " 14 GB", | 20260507 17:21:57.610 | INFO | "pinned" : " 29 GB" | 20260507 17:21:57.610 | INFO | } | 20260507 17:21:57.610 | INFO | Perf report for the last 5.13959 seconds: | 20260507 17:21:57.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:21:57.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:21:57.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:21:57.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:21:57.610 | INFO | Uploaded 36GiB to GPU in 2.49416s (14.4337GiB/s) | 20260507 17:21:57.610 | INFO | Downloaded 34GiB to GPU in 2.45009s (13.877GiB/s) | 20260507 17:22:02.749 | INFO | Analyzing chunk frames = [1254400,1254912) | 20260507 17:22:02.749 | INFO | MemInfo: { | 20260507 17:22:02.749 | INFO | "gpu" : " 17 GB", | 20260507 17:22:02.749 | INFO | "huge" : "", | 20260507 17:22:02.749 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:02.749 | INFO | "malloc" : " 14 GB", | 20260507 17:22:02.749 | INFO | "pinned" : " 29 GB" | 20260507 17:22:02.749 | INFO | } | 20260507 17:22:02.749 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:22:02.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:02.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260507 17:22:02.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:22:02.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:02.749 | INFO | Uploaded 36GiB to GPU in 2.54864s (14.1252GiB/s) | 20260507 17:22:02.750 | INFO | Downloaded 34GiB to GPU in 2.43499s (13.9631GiB/s) | 20260507 17:22:07.888 | INFO | Analyzing chunk frames = [1254912,1255424) | 20260507 17:22:07.888 | INFO | MemInfo: { | 20260507 17:22:07.888 | INFO | "gpu" : " 17 GB", | 20260507 17:22:07.888 | INFO | "huge" : "", | 20260507 17:22:07.888 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:07.888 | INFO | "malloc" : " 14 GB", | 20260507 17:22:07.888 | INFO | "pinned" : " 29 GB" | 20260507 17:22:07.888 | INFO | } | 20260507 17:22:07.888 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:22:07.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:07.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.888 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:22:07.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.888 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:22:07.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:07.889 | INFO | Uploaded 36GiB to GPU in 2.47364s (14.5534GiB/s) | 20260507 17:22:07.889 | INFO | Downloaded 34GiB to GPU in 2.35997s (14.407GiB/s) | 20260507 17:22:13.028 | INFO | Analyzing chunk frames = [1255424,1255936) | 20260507 17:22:13.028 | INFO | MemInfo: { | 20260507 17:22:13.028 | INFO | "gpu" : " 17 GB", | 20260507 17:22:13.028 | INFO | "huge" : "", | 20260507 17:22:13.028 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:13.028 | INFO | "malloc" : " 14 GB", | 20260507 17:22:13.028 | INFO | "pinned" : " 29 GB" | 20260507 17:22:13.028 | INFO | } | 20260507 17:22:13.028 | INFO | Perf report for the last 5.13967 seconds: | 20260507 17:22:13.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:13.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260507 17:22:13.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 17:22:13.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:13.028 | INFO | Uploaded 36GiB to GPU in 2.29836s (15.6634GiB/s) | 20260507 17:22:13.028 | INFO | Downloaded 34GiB to GPU in 2.45779s (13.8335GiB/s) | 20260507 17:22:18.167 | INFO | Analyzing chunk frames = [1255936,1256448) | 20260507 17:22:18.167 | INFO | MemInfo: { | 20260507 17:22:18.167 | INFO | "gpu" : " 17 GB", | 20260507 17:22:18.167 | INFO | "huge" : "", | 20260507 17:22:18.167 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:18.167 | INFO | "malloc" : " 14 GB", | 20260507 17:22:18.167 | INFO | "pinned" : " 29 GB" | 20260507 17:22:18.167 | INFO | } | 20260507 17:22:18.167 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:22:18.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:18.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:22:18.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260507 17:22:18.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:18.167 | INFO | Uploaded 36GiB to GPU in 2.31881s (15.5252GiB/s) | 20260507 17:22:18.167 | INFO | Downloaded 34GiB to GPU in 2.50983s (13.5467GiB/s) | 20260507 17:22:23.309 | INFO | Analyzing chunk frames = [1256448,1256960) | 20260507 17:22:23.309 | INFO | MemInfo: { | 20260507 17:22:23.309 | INFO | "gpu" : " 17 GB", | 20260507 17:22:23.309 | INFO | "huge" : "", | 20260507 17:22:23.309 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:23.309 | INFO | "malloc" : " 14 GB", | 20260507 17:22:23.310 | INFO | "pinned" : " 29 GB" | 20260507 17:22:23.310 | INFO | } | 20260507 17:22:23.310 | INFO | Perf report for the last 5.14275 seconds: | 20260507 17:22:23.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:23.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 17:22:23.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:22:23.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:23.310 | INFO | Uploaded 36GiB to GPU in 2.39427s (15.0359GiB/s) | 20260507 17:22:23.310 | INFO | Downloaded 34GiB to GPU in 2.43668s (13.9534GiB/s) | 20260507 17:22:28.453 | INFO | Analyzing chunk frames = [1256960,1257472) | 20260507 17:22:28.453 | INFO | MemInfo: { | 20260507 17:22:28.453 | INFO | "gpu" : " 17 GB", | 20260507 17:22:28.453 | INFO | "huge" : "", | 20260507 17:22:28.453 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:28.453 | INFO | "malloc" : " 14 GB", | 20260507 17:22:28.453 | INFO | "pinned" : " 29 GB" | 20260507 17:22:28.453 | INFO | } | 20260507 17:22:28.453 | INFO | Perf report for the last 5.14322 seconds: | 20260507 17:22:28.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:28.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 17:22:28.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:22:28.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:28.453 | INFO | Uploaded 36GiB to GPU in 2.44548s (14.7211GiB/s) | 20260507 17:22:28.453 | INFO | Downloaded 34GiB to GPU in 2.51606s (13.5132GiB/s) | 20260507 17:22:28.550 | INFO | Starting a round of Baz IO | 20260507 17:22:32.094 | INFO | Populating RTMetrics report | 20260507 17:22:32.094 | INFO | RTMetrics report populated | 20260507 17:22:32.094 | INFO | Opening tmp RTMetrics json file | 20260507 17:22:32.095 | INFO | Serializing RTMetrics to json | 20260507 17:22:32.096 | INFO | Serialization done, swapping tmp file now... | 20260507 17:22:32.097 | INFO | RTMetrics done | 20260507 17:22:32.177 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.626 seconds | 20260507 17:22:32.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.29% / 0.02% / 51.69% | 20260507 17:22:32.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.66225% / 4.37101e-05% / 3.57628e-05% | 20260507 17:22:32.177 | INFO | Baz IO overall: | 20260507 17:22:32.177 | INFO | BytesWritten : 552419031558 | 20260507 17:22:32.177 | INFO | NumEvents : 267630859397 | 20260507 17:22:33.585 | INFO | Analyzing chunk frames = [1257472,1257984) | 20260507 17:22:33.585 | INFO | MemInfo: { | 20260507 17:22:33.585 | INFO | "gpu" : " 17 GB", | 20260507 17:22:33.585 | INFO | "huge" : "", | 20260507 17:22:33.585 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:33.588 | INFO | "malloc" : " 14 GB", | 20260507 17:22:33.588 | INFO | "pinned" : " 29 GB" | 20260507 17:22:33.588 | INFO | } | 20260507 17:22:33.588 | INFO | Perf report for the last 5.13545 seconds: | 20260507 17:22:33.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:33.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:33.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:33.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:33.588 | INFO | ANALYSIS : 69.9%, 3.0, 108.7ms, 0 | 20260507 17:22:33.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:33.588 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 17:22:33.588 | INFO | PRE_HQ : 67.8%, 1.8, 64.3ms, 0 | 20260507 17:22:33.588 | INFO | BAZWRITER : 69.9%, 3.0, 106.9ms, 0 | 20260507 17:22:33.589 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:22:33.589 | INFO | Uploaded 36GiB to GPU in 2.54556s (14.1423GiB/s) | 20260507 17:22:33.589 | INFO | Downloaded 38GiB to GPU in 2.74139s (13.8616GiB/s) | 20260507 17:22:38.728 | INFO | Analyzing chunk frames = [1257984,1258496) | 20260507 17:22:38.728 | INFO | MemInfo: { | 20260507 17:22:38.728 | INFO | "gpu" : " 17 GB", | 20260507 17:22:38.728 | INFO | "huge" : "", | 20260507 17:22:38.728 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:38.728 | INFO | "malloc" : " 14 GB", | 20260507 17:22:38.728 | INFO | "pinned" : " 29 GB" | 20260507 17:22:38.728 | INFO | } | 20260507 17:22:38.728 | INFO | Perf report for the last 5.13979 seconds: | 20260507 17:22:38.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:38.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.728 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:22:38.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.728 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 17:22:38.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:38.729 | INFO | Uploaded 36GiB to GPU in 2.52697s (14.2463GiB/s) | 20260507 17:22:38.729 | INFO | Downloaded 34GiB to GPU in 2.45551s (13.8464GiB/s) | 20260507 17:22:43.867 | INFO | Analyzing chunk frames = [1258496,1259008) | 20260507 17:22:43.867 | INFO | MemInfo: { | 20260507 17:22:43.867 | INFO | "gpu" : " 17 GB", | 20260507 17:22:43.867 | INFO | "huge" : "", | 20260507 17:22:43.867 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:43.867 | INFO | "malloc" : " 14 GB", | 20260507 17:22:43.867 | INFO | "pinned" : " 29 GB" | 20260507 17:22:43.867 | INFO | } | 20260507 17:22:43.867 | INFO | Perf report for the last 5.13891 seconds: | 20260507 17:22:43.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:43.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:22:43.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 17:22:43.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:43.867 | INFO | Uploaded 36GiB to GPU in 2.53331s (14.2106GiB/s) | 20260507 17:22:43.867 | INFO | Downloaded 34GiB to GPU in 2.44897s (13.8834GiB/s) | 20260507 17:22:49.003 | INFO | Analyzing chunk frames = [1259008,1259520) | 20260507 17:22:49.003 | INFO | MemInfo: { | 20260507 17:22:49.003 | INFO | "gpu" : " 17 GB", | 20260507 17:22:49.003 | INFO | "huge" : "", | 20260507 17:22:49.003 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:49.003 | INFO | "malloc" : " 14 GB", | 20260507 17:22:49.003 | INFO | "pinned" : " 29 GB" | 20260507 17:22:49.003 | INFO | } | 20260507 17:22:49.003 | INFO | Perf report for the last 5.13605 seconds: | 20260507 17:22:49.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:49.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.003 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:22:49.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.003 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 17:22:49.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:49.004 | INFO | Uploaded 36GiB to GPU in 2.49773s (14.4131GiB/s) | 20260507 17:22:49.004 | INFO | Downloaded 34GiB to GPU in 2.36461s (14.3787GiB/s) | 20260507 17:22:54.143 | INFO | Analyzing chunk frames = [1259520,1260032) | 20260507 17:22:54.143 | INFO | MemInfo: { | 20260507 17:22:54.143 | INFO | "gpu" : " 17 GB", | 20260507 17:22:54.143 | INFO | "huge" : "", | 20260507 17:22:54.143 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:54.143 | INFO | "malloc" : " 14 GB", | 20260507 17:22:54.143 | INFO | "pinned" : " 29 GB" | 20260507 17:22:54.143 | INFO | } | 20260507 17:22:54.143 | INFO | Perf report for the last 5.14021 seconds: | 20260507 17:22:54.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:54.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.143 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:22:54.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.144 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:22:54.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:54.144 | INFO | Uploaded 36GiB to GPU in 2.35773s (15.269GiB/s) | 20260507 17:22:54.144 | INFO | Downloaded 34GiB to GPU in 2.4835s (13.6903GiB/s) | 20260507 17:22:59.285 | INFO | Analyzing chunk frames = [1260032,1260544) | 20260507 17:22:59.285 | INFO | MemInfo: { | 20260507 17:22:59.285 | INFO | "gpu" : " 17 GB", | 20260507 17:22:59.285 | INFO | "huge" : "", | 20260507 17:22:59.285 | INFO | "huge_shared" : " 29 GB", | 20260507 17:22:59.285 | INFO | "malloc" : " 14 GB", | 20260507 17:22:59.285 | INFO | "pinned" : " 29 GB" | 20260507 17:22:59.285 | INFO | } | 20260507 17:22:59.285 | INFO | Perf report for the last 5.14209 seconds: | 20260507 17:22:59.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:22:59.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:22:59.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:22:59.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:22:59.286 | INFO | Uploaded 36GiB to GPU in 2.40121s (14.9924GiB/s) | 20260507 17:22:59.286 | INFO | Downloaded 34GiB to GPU in 2.47454s (13.7399GiB/s) | 20260507 17:23:04.421 | INFO | Analyzing chunk frames = [1260544,1261056) | 20260507 17:23:04.421 | INFO | MemInfo: { | 20260507 17:23:04.421 | INFO | "gpu" : " 17 GB", | 20260507 17:23:04.421 | INFO | "huge" : "", | 20260507 17:23:04.421 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:04.421 | INFO | "malloc" : " 14 GB", | 20260507 17:23:04.421 | INFO | "pinned" : " 29 GB" | 20260507 17:23:04.421 | INFO | } | 20260507 17:23:04.421 | INFO | Perf report for the last 5.13522 seconds: | 20260507 17:23:04.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:04.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:23:04.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:23:04.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:04.421 | INFO | Uploaded 36GiB to GPU in 2.50622s (14.3643GiB/s) | 20260507 17:23:04.421 | INFO | Downloaded 34GiB to GPU in 2.42276s (14.0336GiB/s) | 20260507 17:23:09.560 | INFO | Analyzing chunk frames = [1261056,1261568) | 20260507 17:23:09.560 | INFO | MemInfo: { | 20260507 17:23:09.560 | INFO | "gpu" : " 17 GB", | 20260507 17:23:09.560 | INFO | "huge" : "", | 20260507 17:23:09.560 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:09.561 | INFO | "malloc" : " 14 GB", | 20260507 17:23:09.561 | INFO | "pinned" : " 29 GB" | 20260507 17:23:09.561 | INFO | } | 20260507 17:23:09.561 | INFO | Perf report for the last 5.13984 seconds: | 20260507 17:23:09.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:09.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 17:23:09.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:23:09.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:09.561 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:23:09.561 | INFO | Total mem usage: 16.4 GiB | 20260507 17:23:09.561 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:23:09.561 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.3%) | 20260507 17:23:09.561 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:23:09.561 | INFO | High Water Breakdown: | 20260507 17:23:09.561 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:23:09.561 | INFO | 40.8% : 5.9 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:23:09.561 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:23:09.561 | INFO | ------------------------------------------ | 20260507 17:23:09.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:23:09.561 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:23:09.561 | INFO | Total mem usage: 26.5 GiB | 20260507 17:23:09.561 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:23:09.561 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:23:09.561 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:23:09.561 | INFO | High Water Breakdown: | 20260507 17:23:09.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:23:09.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:23:09.561 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:23:09.561 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:23:09.561 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:23:09.561 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:23:09.561 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:23:09.561 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:23:09.561 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:23:09.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:23:09.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:23:09.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:23:09.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:23:09.561 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:23:09.561 | INFO | ------------------------------------------ | 20260507 17:23:09.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:23:09.561 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:23:09.561 | INFO | Total mem usage: 27.8 GiB | 20260507 17:23:09.561 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:23:09.561 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:23:09.561 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:23:09.561 | INFO | High Water Breakdown: | 20260507 17:23:09.561 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:23:09.561 | INFO | ------------------------------------------ | 20260507 17:23:09.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:23:09.561 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:23:09.561 | INFO | Total mem usage: 15.7 GiB | 20260507 17:23:09.561 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:23:09.561 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:23:09.561 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:23:09.561 | INFO | High Water Breakdown: | 20260507 17:23:09.561 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:23:09.561 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:23:09.561 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:23:09.561 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:23:09.561 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:23:09.561 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:23:09.561 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:23:09.561 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:23:09.561 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:23:09.561 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:23:09.561 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:23:09.561 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:23:09.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:23:09.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:23:09.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:23:09.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:23:09.561 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:23:09.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:23:09.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:23:09.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:23:09.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:23:09.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:23:09.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:23:09.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:23:09.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:23:09.561 | INFO | ------------------------------------------ | 20260507 17:23:09.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:23:09.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:23:09.561 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:23:09.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:23:09.561 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.43e+01 / 2.71e+00 | 20260507 17:23:09.561 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.04e-01, 1.22e-02 / 6.58e-03 | 20260507 17:23:09.561 | INFO | PerfReport: 12.07%, Baseline -- 4.04e+00 / 4.51e+00, 6.13e-01 / 1.98e-01 | 20260507 17:23:09.561 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.13e-01 / 1.18e-01 | 20260507 17:23:09.561 | INFO | PerfReport: 15.74%, DME -- 4.22e+01 / 4.49e+01, 4.18e+00 / 2.66e+00 | 20260507 17:23:09.561 | INFO | PerfReport: 55.91%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.82e+00 / 2.01e+00 | 20260507 17:23:09.561 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.53e+00 / 2.88e+00, 4.36e-01 / 2.51e-01 | 20260507 17:23:09.561 | INFO | PerfReport: 3.78%, Metrics -- 1.26e+00 / 1.44e+00, 3.12e-01 / 1.79e-01 | 20260507 17:23:09.561 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.03e-01 / 4.27e-01, 6.72e-02 / 3.94e-02 | 20260507 17:23:09.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:23:09.562 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:23:09.561 | INFO | Uploaded 36GiB to GPU in 2.53944s (14.1764GiB/s) | 20260507 17:23:09.562 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:23:09.562 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260507 17:23:09.562 | INFO | PerfReport: 23.88%, Upload -- 2.57e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 17:23:09.562 | INFO | PerfReport: 24.10%, Download -- 2.59e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 17:23:09.562 | INFO | PerfReport: 31.14%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:23:09.562 | INFO | Downloaded 34GiB to GPU in 2.50878s (13.5524GiB/s) | 20260507 17:23:09.562 | INFO | PerfReport: 1.72%, LockUpload -- 1.85e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 17:23:09.562 | INFO | PerfReport: 2.46%, LockDownload -- 2.64e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260507 17:23:09.562 | INFO | PerfReport: 16.68%, LockCompute -- 1.79e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 17:23:09.562 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 17:23:09.659 | INFO | Starting a round of Baz IO | 20260507 17:23:13.196 | INFO | Populating RTMetrics report | 20260507 17:23:13.197 | INFO | RTMetrics report populated | 20260507 17:23:13.197 | INFO | Opening tmp RTMetrics json file | 20260507 17:23:13.197 | INFO | Serializing RTMetrics to json | 20260507 17:23:13.200 | INFO | Serialization done, swapping tmp file now... | 20260507 17:23:13.201 | INFO | RTMetrics done | 20260507 17:23:13.291 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.632 seconds | 20260507 17:23:13.291 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.27% / 0.02% / 51.71% | 20260507 17:23:13.291 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.6673% / 3.17891e-05% / 3.57628e-05% | 20260507 17:23:13.291 | INFO | Baz IO overall: | 20260507 17:23:13.291 | INFO | BytesWritten : 554358462348 | 20260507 17:23:13.291 | INFO | NumEvents : 268572775458 | 20260507 17:23:14.700 | INFO | Analyzing chunk frames = [1261568,1262080) | 20260507 17:23:14.700 | INFO | MemInfo: { | 20260507 17:23:14.700 | INFO | "gpu" : " 17 GB", | 20260507 17:23:14.700 | INFO | "huge" : "", | 20260507 17:23:14.700 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:14.700 | INFO | "malloc" : " 14 GB", | 20260507 17:23:14.700 | INFO | "pinned" : " 29 GB" | 20260507 17:23:14.700 | INFO | } | 20260507 17:23:14.700 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:23:14.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:14.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:14.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:14.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:14.703 | INFO | ANALYSIS : 69.5%, 3.0, 108.5ms, 0 | 20260507 17:23:14.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:14.703 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 17:23:14.703 | INFO | PRE_HQ : 67.8%, 1.9, 65.2ms, 0 | 20260507 17:23:14.703 | INFO | BAZWRITER : 69.8%, 3.0, 108.5ms, 0 | 20260507 17:23:14.703 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:23:14.703 | INFO | Uploaded 36GiB to GPU in 2.51542s (14.3117GiB/s) | 20260507 17:23:14.703 | INFO | Downloaded 38GiB to GPU in 2.70467s (14.0498GiB/s) | 20260507 17:23:19.839 | INFO | Analyzing chunk frames = [1262080,1262592) | 20260507 17:23:19.839 | INFO | MemInfo: { | 20260507 17:23:19.839 | INFO | "gpu" : " 17 GB", | 20260507 17:23:19.839 | INFO | "huge" : "", | 20260507 17:23:19.839 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:19.839 | INFO | "malloc" : " 14 GB", | 20260507 17:23:19.839 | INFO | "pinned" : " 29 GB" | 20260507 17:23:19.839 | INFO | } | 20260507 17:23:19.839 | INFO | Perf report for the last 5.13916 seconds: | 20260507 17:23:19.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:19.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.839 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:23:19.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.839 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:23:19.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:19.840 | INFO | Uploaded 36GiB to GPU in 2.51209s (14.3307GiB/s) | 20260507 17:23:19.840 | INFO | Downloaded 34GiB to GPU in 2.44367s (13.9135GiB/s) | 20260507 17:23:24.979 | INFO | Analyzing chunk frames = [1262592,1263104) | 20260507 17:23:24.979 | INFO | MemInfo: { | 20260507 17:23:24.979 | INFO | "gpu" : " 17 GB", | 20260507 17:23:24.979 | INFO | "huge" : "", | 20260507 17:23:24.979 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:24.979 | INFO | "malloc" : " 14 GB", | 20260507 17:23:24.979 | INFO | "pinned" : " 29 GB" | 20260507 17:23:24.979 | INFO | } | 20260507 17:23:24.979 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:23:24.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:24.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:23:24.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:23:24.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:24.979 | INFO | Uploaded 36GiB to GPU in 2.55299s (14.1011GiB/s) | 20260507 17:23:24.980 | INFO | Downloaded 34GiB to GPU in 2.43142s (13.9836GiB/s) | 20260507 17:23:30.118 | INFO | Analyzing chunk frames = [1263104,1263616) | 20260507 17:23:30.118 | INFO | MemInfo: { | 20260507 17:23:30.118 | INFO | "gpu" : " 17 GB", | 20260507 17:23:30.118 | INFO | "huge" : "", | 20260507 17:23:30.118 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:30.118 | INFO | "malloc" : " 14 GB", | 20260507 17:23:30.118 | INFO | "pinned" : " 29 GB" | 20260507 17:23:30.118 | INFO | } | 20260507 17:23:30.118 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:23:30.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:30.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:23:30.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:23:30.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:30.118 | INFO | Uploaded 36GiB to GPU in 2.48159s (14.5068GiB/s) | 20260507 17:23:30.118 | INFO | Downloaded 34GiB to GPU in 2.35594s (14.4316GiB/s) | 20260507 17:23:35.257 | INFO | Analyzing chunk frames = [1263616,1264128) | 20260507 17:23:35.257 | INFO | MemInfo: { | 20260507 17:23:35.257 | INFO | "gpu" : " 17 GB", | 20260507 17:23:35.257 | INFO | "huge" : "", | 20260507 17:23:35.257 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:35.257 | INFO | "malloc" : " 14 GB", | 20260507 17:23:35.257 | INFO | "pinned" : " 29 GB" | 20260507 17:23:35.257 | INFO | } | 20260507 17:23:35.257 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:23:35.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:35.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.257 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:23:35.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.257 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 17:23:35.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:35.258 | INFO | Uploaded 36GiB to GPU in 2.3412s (15.3767GiB/s) | 20260507 17:23:35.258 | INFO | Downloaded 34GiB to GPU in 2.46754s (13.7789GiB/s) | 20260507 17:23:40.396 | INFO | Analyzing chunk frames = [1264128,1264640) | 20260507 17:23:40.396 | INFO | MemInfo: { | 20260507 17:23:40.396 | INFO | "gpu" : " 17 GB", | 20260507 17:23:40.396 | INFO | "huge" : "", | 20260507 17:23:40.396 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:40.396 | INFO | "malloc" : " 14 GB", | 20260507 17:23:40.396 | INFO | "pinned" : " 29 GB" | 20260507 17:23:40.396 | INFO | } | 20260507 17:23:40.396 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:23:40.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:40.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.396 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:23:40.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.397 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 17:23:40.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:40.397 | INFO | Uploaded 36GiB to GPU in 2.39897s (15.0064GiB/s) | 20260507 17:23:40.397 | INFO | Downloaded 34GiB to GPU in 2.51749s (13.5055GiB/s) | 20260507 17:23:45.536 | INFO | Analyzing chunk frames = [1264640,1265152) | 20260507 17:23:45.536 | INFO | MemInfo: { | 20260507 17:23:45.536 | INFO | "gpu" : " 17 GB", | 20260507 17:23:45.536 | INFO | "huge" : "", | 20260507 17:23:45.536 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:45.536 | INFO | "malloc" : " 14 GB", | 20260507 17:23:45.536 | INFO | "pinned" : " 29 GB" | 20260507 17:23:45.536 | INFO | } | 20260507 17:23:45.536 | INFO | Perf report for the last 5.13956 seconds: | 20260507 17:23:45.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:45.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.536 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 17:23:45.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.536 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:23:45.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:45.537 | INFO | Uploaded 36GiB to GPU in 2.38903s (15.0689GiB/s) | 20260507 17:23:45.537 | INFO | Downloaded 34GiB to GPU in 2.42983s (13.9928GiB/s) | 20260507 17:23:50.675 | INFO | Analyzing chunk frames = [1265152,1265664) | 20260507 17:23:50.675 | INFO | MemInfo: { | 20260507 17:23:50.675 | INFO | "gpu" : " 17 GB", | 20260507 17:23:50.675 | INFO | "huge" : "", | 20260507 17:23:50.675 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:50.675 | INFO | "malloc" : " 14 GB", | 20260507 17:23:50.675 | INFO | "pinned" : " 29 GB" | 20260507 17:23:50.675 | INFO | } | 20260507 17:23:50.675 | INFO | Perf report for the last 5.13889 seconds: | 20260507 17:23:50.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:50.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:23:50.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:23:50.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:50.675 | INFO | Uploaded 36GiB to GPU in 2.45299s (14.676GiB/s) | 20260507 17:23:50.675 | INFO | Downloaded 34GiB to GPU in 2.5133s (13.528GiB/s) | 20260507 17:23:50.772 | INFO | Starting a round of Baz IO | 20260507 17:23:54.297 | INFO | Populating RTMetrics report | 20260507 17:23:54.297 | INFO | RTMetrics report populated | 20260507 17:23:54.297 | INFO | Opening tmp RTMetrics json file | 20260507 17:23:54.298 | INFO | Serializing RTMetrics to json | 20260507 17:23:54.299 | INFO | Serialization done, swapping tmp file now... | 20260507 17:23:54.299 | INFO | RTMetrics done | 20260507 17:23:54.382 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.61 seconds | 20260507 17:23:54.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.25% / 0.02% / 51.73% | 20260507 17:23:54.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.64797% / 2.78155e-05% / 3.57628e-05% | 20260507 17:23:54.382 | INFO | Baz IO overall: | 20260507 17:23:54.382 | INFO | BytesWritten : 556295021151 | 20260507 17:23:54.382 | INFO | NumEvents : 269512647673 | 20260507 17:23:55.814 | INFO | Analyzing chunk frames = [1265664,1266176) | 20260507 17:23:55.814 | INFO | MemInfo: { | 20260507 17:23:55.814 | INFO | "gpu" : " 17 GB", | 20260507 17:23:55.814 | INFO | "huge" : "", | 20260507 17:23:55.814 | INFO | "huge_shared" : " 29 GB", | 20260507 17:23:55.814 | INFO | "malloc" : " 14 GB", | 20260507 17:23:55.814 | INFO | "pinned" : " 29 GB" | 20260507 17:23:55.814 | INFO | } | 20260507 17:23:55.814 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:23:55.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:23:55.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:55.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:55.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:55.814 | INFO | ANALYSIS : 69.8%, 3.0, 108.7ms, 0 | 20260507 17:23:55.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:23:55.814 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260507 17:23:55.814 | INFO | PRE_HQ : 67.5%, 1.8, 64.6ms, 0 | 20260507 17:23:55.814 | INFO | BAZWRITER : 69.8%, 3.0, 107.6ms, 0 | 20260507 17:23:55.814 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:23:55.815 | INFO | Uploaded 36GiB to GPU in 2.5183s (14.2954GiB/s) | 20260507 17:23:55.815 | INFO | Downloaded 38GiB to GPU in 2.73571s (13.8904GiB/s) | 20260507 17:24:00.953 | INFO | Analyzing chunk frames = [1266176,1266688) | 20260507 17:24:00.953 | INFO | MemInfo: { | 20260507 17:24:00.953 | INFO | "gpu" : " 17 GB", | 20260507 17:24:00.953 | INFO | "huge" : "", | 20260507 17:24:00.953 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:00.953 | INFO | "malloc" : " 14 GB", | 20260507 17:24:00.954 | INFO | "pinned" : " 29 GB" | 20260507 17:24:00.954 | INFO | } | 20260507 17:24:00.954 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:24:00.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:00.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:24:00.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:24:00.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:00.954 | INFO | Uploaded 36GiB to GPU in 2.45542s (14.6614GiB/s) | 20260507 17:24:00.954 | INFO | Downloaded 34GiB to GPU in 2.46517s (13.7921GiB/s) | 20260507 17:24:06.093 | INFO | Analyzing chunk frames = [1266688,1267200) | 20260507 17:24:06.093 | INFO | MemInfo: { | 20260507 17:24:06.093 | INFO | "gpu" : " 17 GB", | 20260507 17:24:06.093 | INFO | "huge" : "", | 20260507 17:24:06.093 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:06.093 | INFO | "malloc" : " 14 GB", | 20260507 17:24:06.093 | INFO | "pinned" : " 29 GB" | 20260507 17:24:06.093 | INFO | } | 20260507 17:24:06.093 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:24:06.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:06.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:24:06.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:24:06.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:06.093 | INFO | Uploaded 36GiB to GPU in 2.52765s (14.2425GiB/s) | 20260507 17:24:06.093 | INFO | Downloaded 34GiB to GPU in 2.44789s (13.8895GiB/s) | 20260507 17:24:11.232 | INFO | Analyzing chunk frames = [1267200,1267712) | 20260507 17:24:11.232 | INFO | MemInfo: { | 20260507 17:24:11.232 | INFO | "gpu" : " 17 GB", | 20260507 17:24:11.232 | INFO | "huge" : "", | 20260507 17:24:11.232 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:11.232 | INFO | "malloc" : " 14 GB", | 20260507 17:24:11.232 | INFO | "pinned" : " 29 GB" | 20260507 17:24:11.232 | INFO | } | 20260507 17:24:11.233 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:24:11.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:11.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 17:24:11.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:24:11.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:11.233 | INFO | Uploaded 36GiB to GPU in 2.45279s (14.6772GiB/s) | 20260507 17:24:11.233 | INFO | Downloaded 34GiB to GPU in 2.35185s (14.4567GiB/s) | 20260507 17:24:16.372 | INFO | Analyzing chunk frames = [1267712,1268224) | 20260507 17:24:16.372 | INFO | MemInfo: { | 20260507 17:24:16.372 | INFO | "gpu" : " 17 GB", | 20260507 17:24:16.372 | INFO | "huge" : "", | 20260507 17:24:16.372 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:16.372 | INFO | "malloc" : " 14 GB", | 20260507 17:24:16.372 | INFO | "pinned" : " 29 GB" | 20260507 17:24:16.372 | INFO | } | 20260507 17:24:16.372 | INFO | Perf report for the last 5.13985 seconds: | 20260507 17:24:16.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:16.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.372 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:24:16.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.373 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 17:24:16.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:16.373 | INFO | Uploaded 36GiB to GPU in 2.30208s (15.638GiB/s) | 20260507 17:24:16.373 | INFO | Downloaded 34GiB to GPU in 2.47681s (13.7273GiB/s) | 20260507 17:24:21.511 | INFO | Analyzing chunk frames = [1268224,1268736) | 20260507 17:24:21.511 | INFO | MemInfo: { | 20260507 17:24:21.511 | INFO | "gpu" : " 17 GB", | 20260507 17:24:21.511 | INFO | "huge" : "", | 20260507 17:24:21.511 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:21.511 | INFO | "malloc" : " 14 GB", | 20260507 17:24:21.511 | INFO | "pinned" : " 29 GB" | 20260507 17:24:21.511 | INFO | } | 20260507 17:24:21.511 | INFO | Perf report for the last 5.13893 seconds: | 20260507 17:24:21.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:21.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.511 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:24:21.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.511 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260507 17:24:21.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:21.512 | INFO | Uploaded 36GiB to GPU in 2.39754s (15.0154GiB/s) | 20260507 17:24:21.512 | INFO | Downloaded 34GiB to GPU in 2.52401s (13.4706GiB/s) | 20260507 17:24:26.650 | INFO | Analyzing chunk frames = [1268736,1269248) | 20260507 17:24:26.650 | INFO | MemInfo: { | 20260507 17:24:26.650 | INFO | "gpu" : " 17 GB", | 20260507 17:24:26.650 | INFO | "huge" : "", | 20260507 17:24:26.650 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:26.650 | INFO | "malloc" : " 14 GB", | 20260507 17:24:26.650 | INFO | "pinned" : " 29 GB" | 20260507 17:24:26.650 | INFO | } | 20260507 17:24:26.650 | INFO | Perf report for the last 5.13899 seconds: | 20260507 17:24:26.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:26.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.650 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:24:26.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.650 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:24:26.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:26.651 | INFO | Uploaded 36GiB to GPU in 2.43607s (14.7779GiB/s) | 20260507 17:24:26.651 | INFO | Downloaded 34GiB to GPU in 2.43858s (13.9426GiB/s) | 20260507 17:24:31.790 | INFO | Analyzing chunk frames = [1269248,1269760) | 20260507 17:24:31.790 | INFO | MemInfo: { | 20260507 17:24:31.790 | INFO | "gpu" : " 17 GB", | 20260507 17:24:31.790 | INFO | "huge" : "", | 20260507 17:24:31.790 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:31.790 | INFO | "malloc" : " 14 GB", | 20260507 17:24:31.790 | INFO | "pinned" : " 29 GB" | 20260507 17:24:31.790 | INFO | } | 20260507 17:24:31.790 | INFO | Perf report for the last 5.13967 seconds: | 20260507 17:24:31.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:31.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:24:31.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:24:31.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:31.790 | INFO | Uploaded 36GiB to GPU in 2.46612s (14.5978GiB/s) | 20260507 17:24:31.790 | INFO | Downloaded 34GiB to GPU in 2.51955s (13.4945GiB/s) | 20260507 17:24:31.791 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:24:31.791 | INFO | Total mem usage: 16.4 GiB | 20260507 17:24:31.791 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:24:31.791 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.3%) | 20260507 17:24:31.791 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:24:31.791 | INFO | High Water Breakdown: | 20260507 17:24:31.791 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:24:31.791 | INFO | 40.8% : 5.9 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:24:31.791 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:24:31.791 | INFO | ------------------------------------------ | 20260507 17:24:31.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:24:31.791 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:24:31.791 | INFO | Total mem usage: 26.5 GiB | 20260507 17:24:31.791 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:24:31.791 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:24:31.791 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:24:31.791 | INFO | High Water Breakdown: | 20260507 17:24:31.791 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:24:31.791 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:24:31.791 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:24:31.791 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:24:31.791 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:24:31.791 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:24:31.791 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:24:31.791 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:24:31.791 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:24:31.791 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:24:31.791 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:24:31.791 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:24:31.791 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:24:31.791 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:24:31.791 | INFO | ------------------------------------------ | 20260507 17:24:31.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:24:31.791 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:24:31.791 | INFO | Total mem usage: 27.8 GiB | 20260507 17:24:31.791 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:24:31.791 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:24:31.791 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:24:31.791 | INFO | High Water Breakdown: | 20260507 17:24:31.791 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:24:31.791 | INFO | ------------------------------------------ | 20260507 17:24:31.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:24:31.791 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:24:31.791 | INFO | Total mem usage: 15.7 GiB | 20260507 17:24:31.791 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:24:31.791 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:24:31.791 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:24:31.791 | INFO | High Water Breakdown: | 20260507 17:24:31.791 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:24:31.791 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:24:31.791 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:24:31.791 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:24:31.791 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:24:31.791 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:24:31.791 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:24:31.791 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:24:31.791 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:24:31.792 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:24:31.792 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:24:31.792 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:24:31.792 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:24:31.792 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:24:31.792 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:24:31.792 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:24:31.792 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:24:31.792 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:24:31.792 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:24:31.792 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:24:31.792 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:24:31.792 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:24:31.792 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:24:31.792 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:24:31.792 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:24:31.792 | INFO | ------------------------------------------ | 20260507 17:24:31.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:24:31.792 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:24:31.792 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:24:31.792 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:24:31.792 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.69e+01, 1.42e+01 / 2.79e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.10e-02 / 5.53e-03 | 20260507 17:24:31.792 | INFO | PerfReport: 12.09%, Baseline -- 4.06e+00 / 4.53e+00, 6.17e-01 / 2.25e-01 | 20260507 17:24:31.792 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.50e+00, 2.14e-01 / 1.14e-01 | 20260507 17:24:31.792 | INFO | PerfReport: 15.75%, DME -- 4.23e+01 / 4.52e+01, 4.34e+00 / 2.77e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.87e+00 / 2.31e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.87e+00, 4.37e-01 / 2.45e-01 | 20260507 17:24:31.792 | INFO | PerfReport: 3.79%, Metrics -- 1.27e+00 / 1.45e+00, 3.22e-01 / 1.92e-01 | 20260507 17:24:31.792 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.01e-01 / 4.24e-01, 6.73e-02 / 4.30e-02 | 20260507 17:24:31.792 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:24:31.792 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:24:31.792 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:24:31.792 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 23.74%, Upload -- 2.55e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 24.12%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 31.22%, Compute -- 3.36e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 1.60%, LockUpload -- 1.72e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 2.57%, LockDownload -- 2.76e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 16.74%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:24:31.792 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 4.00e-02 / 0.00e+00 | 20260507 17:24:31.890 | INFO | Starting a round of Baz IO | 20260507 17:24:35.404 | INFO | Populating RTMetrics report | 20260507 17:24:35.404 | INFO | RTMetrics report populated | 20260507 17:24:35.404 | INFO | Opening tmp RTMetrics json file | 20260507 17:24:35.405 | INFO | Serializing RTMetrics to json | 20260507 17:24:35.406 | INFO | Serialization done, swapping tmp file now... | 20260507 17:24:35.406 | INFO | RTMetrics done | 20260507 17:24:35.486 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.596 seconds | 20260507 17:24:35.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.23% / 0.02% / 51.75% | 20260507 17:24:35.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.64789% / 3.57628e-05% / 4.76837e-05% | 20260507 17:24:35.486 | INFO | Baz IO overall: | 20260507 17:24:35.487 | INFO | BytesWritten : 558230023793 | 20260507 17:24:35.487 | INFO | NumEvents : 270452876616 | 20260507 17:24:36.929 | INFO | Analyzing chunk frames = [1269760,1270272) | 20260507 17:24:36.929 | INFO | MemInfo: { | 20260507 17:24:36.929 | INFO | "gpu" : " 17 GB", | 20260507 17:24:36.929 | INFO | "huge" : "", | 20260507 17:24:36.930 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:36.930 | INFO | "malloc" : " 14 GB", | 20260507 17:24:36.930 | INFO | "pinned" : " 29 GB" | 20260507 17:24:36.930 | INFO | } | 20260507 17:24:36.930 | INFO | Perf report for the last 5.13968 seconds: | 20260507 17:24:36.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:36.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:36.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:36.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:36.930 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:24:36.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:36.930 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 17:24:36.930 | INFO | PRE_HQ : 67.6%, 1.8, 64.8ms, 0 | 20260507 17:24:36.930 | INFO | BAZWRITER : 69.4%, 3.0, 106.8ms, 0 | 20260507 17:24:36.930 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:24:36.930 | INFO | Uploaded 36GiB to GPU in 2.52756s (14.243GiB/s) | 20260507 17:24:36.930 | INFO | Downloaded 38GiB to GPU in 2.73301s (13.9041GiB/s) | 20260507 17:24:42.068 | INFO | Analyzing chunk frames = [1270272,1270784) | 20260507 17:24:42.068 | INFO | MemInfo: { | 20260507 17:24:42.068 | INFO | "gpu" : " 17 GB", | 20260507 17:24:42.068 | INFO | "huge" : "", | 20260507 17:24:42.068 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:42.069 | INFO | "malloc" : " 14 GB", | 20260507 17:24:42.069 | INFO | "pinned" : " 29 GB" | 20260507 17:24:42.069 | INFO | } | 20260507 17:24:42.069 | INFO | Perf report for the last 5.13892 seconds: | 20260507 17:24:42.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:42.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:24:42.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:24:42.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:42.069 | INFO | Uploaded 36GiB to GPU in 2.50392s (14.3774GiB/s) | 20260507 17:24:42.069 | INFO | Downloaded 34GiB to GPU in 2.43392s (13.9692GiB/s) | 20260507 17:24:47.207 | INFO | Analyzing chunk frames = [1270784,1271296) | 20260507 17:24:47.207 | INFO | MemInfo: { | 20260507 17:24:47.207 | INFO | "gpu" : " 17 GB", | 20260507 17:24:47.207 | INFO | "huge" : "", | 20260507 17:24:47.207 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:47.207 | INFO | "malloc" : " 14 GB", | 20260507 17:24:47.208 | INFO | "pinned" : " 29 GB" | 20260507 17:24:47.208 | INFO | } | 20260507 17:24:47.208 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:24:47.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:47.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:24:47.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:24:47.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:47.208 | INFO | Uploaded 36GiB to GPU in 2.55826s (14.0721GiB/s) | 20260507 17:24:47.208 | INFO | Downloaded 34GiB to GPU in 2.43076s (13.9874GiB/s) | 20260507 17:24:52.347 | INFO | Analyzing chunk frames = [1271296,1271808) | 20260507 17:24:52.347 | INFO | MemInfo: { | 20260507 17:24:52.347 | INFO | "gpu" : " 17 GB", | 20260507 17:24:52.347 | INFO | "huge" : "", | 20260507 17:24:52.347 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:52.347 | INFO | "malloc" : " 14 GB", | 20260507 17:24:52.347 | INFO | "pinned" : " 29 GB" | 20260507 17:24:52.347 | INFO | } | 20260507 17:24:52.347 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:24:52.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:52.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.347 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:24:52.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.347 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:24:52.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:52.348 | INFO | Uploaded 36GiB to GPU in 2.46009s (14.6336GiB/s) | 20260507 17:24:52.348 | INFO | Downloaded 34GiB to GPU in 2.35495s (14.4377GiB/s) | 20260507 17:24:57.486 | INFO | Analyzing chunk frames = [1271808,1272320) | 20260507 17:24:57.486 | INFO | MemInfo: { | 20260507 17:24:57.486 | INFO | "gpu" : " 17 GB", | 20260507 17:24:57.486 | INFO | "huge" : "", | 20260507 17:24:57.486 | INFO | "huge_shared" : " 29 GB", | 20260507 17:24:57.486 | INFO | "malloc" : " 14 GB", | 20260507 17:24:57.486 | INFO | "pinned" : " 29 GB" | 20260507 17:24:57.486 | INFO | } | 20260507 17:24:57.487 | INFO | Perf report for the last 5.13933 seconds: | 20260507 17:24:57.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:24:57.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:24:57.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:24:57.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:24:57.487 | INFO | Uploaded 36GiB to GPU in 2.34701s (15.3387GiB/s) | 20260507 17:24:57.487 | INFO | Downloaded 34GiB to GPU in 2.47532s (13.7356GiB/s) | 20260507 17:25:02.625 | INFO | Analyzing chunk frames = [1272320,1272832) | 20260507 17:25:02.625 | INFO | MemInfo: { | 20260507 17:25:02.625 | INFO | "gpu" : " 17 GB", | 20260507 17:25:02.625 | INFO | "huge" : "", | 20260507 17:25:02.625 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:02.625 | INFO | "malloc" : " 14 GB", | 20260507 17:25:02.626 | INFO | "pinned" : " 29 GB" | 20260507 17:25:02.626 | INFO | } | 20260507 17:25:02.626 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:25:02.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:02.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:25:02.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:25:02.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:02.626 | INFO | Uploaded 36GiB to GPU in 2.32762s (15.4664GiB/s) | 20260507 17:25:02.626 | INFO | Downloaded 34GiB to GPU in 2.47975s (13.7111GiB/s) | 20260507 17:25:07.765 | INFO | Analyzing chunk frames = [1272832,1273344) | 20260507 17:25:07.765 | INFO | MemInfo: { | 20260507 17:25:07.765 | INFO | "gpu" : " 17 GB", | 20260507 17:25:07.765 | INFO | "huge" : "", | 20260507 17:25:07.765 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:07.765 | INFO | "malloc" : " 14 GB", | 20260507 17:25:07.765 | INFO | "pinned" : " 29 GB" | 20260507 17:25:07.765 | INFO | } | 20260507 17:25:07.765 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:25:07.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:07.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:25:07.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:25:07.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:07.765 | INFO | Uploaded 36GiB to GPU in 2.45995s (14.6344GiB/s) | 20260507 17:25:07.765 | INFO | Downloaded 34GiB to GPU in 2.42799s (14.0034GiB/s) | 20260507 17:25:12.905 | INFO | Analyzing chunk frames = [1273344,1273856) | 20260507 17:25:12.905 | INFO | MemInfo: { | 20260507 17:25:12.905 | INFO | "gpu" : " 17 GB", | 20260507 17:25:12.905 | INFO | "huge" : "", | 20260507 17:25:12.905 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:12.905 | INFO | "malloc" : " 14 GB", | 20260507 17:25:12.905 | INFO | "pinned" : " 29 GB" | 20260507 17:25:12.905 | INFO | } | 20260507 17:25:12.905 | INFO | Perf report for the last 5.13984 seconds: | 20260507 17:25:12.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:12.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:25:12.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:25:12.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:12.905 | INFO | Uploaded 36GiB to GPU in 2.50735s (14.3578GiB/s) | 20260507 17:25:12.905 | INFO | Downloaded 34GiB to GPU in 2.50395s (13.5785GiB/s) | 20260507 17:25:13.002 | INFO | Starting a round of Baz IO | 20260507 17:25:16.537 | INFO | Populating RTMetrics report | 20260507 17:25:16.538 | INFO | RTMetrics report populated | 20260507 17:25:16.538 | INFO | Opening tmp RTMetrics json file | 20260507 17:25:16.539 | INFO | Serializing RTMetrics to json | 20260507 17:25:16.540 | INFO | Serialization done, swapping tmp file now... | 20260507 17:25:16.540 | INFO | RTMetrics done | 20260507 17:25:16.626 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.623 seconds | 20260507 17:25:16.626 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.21% / 0.02% / 51.77% | 20260507 17:25:16.626 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.6386% / 3.97364e-05% / 1.58946e-05% | 20260507 17:25:16.626 | INFO | Baz IO overall: | 20260507 17:25:16.626 | INFO | BytesWritten : 560163715286 | 20260507 17:25:16.626 | INFO | NumEvents : 271392494063 | 20260507 17:25:18.044 | INFO | Analyzing chunk frames = [1273856,1274368) | 20260507 17:25:18.044 | INFO | MemInfo: { | 20260507 17:25:18.044 | INFO | "gpu" : " 17 GB", | 20260507 17:25:18.044 | INFO | "huge" : "", | 20260507 17:25:18.044 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:18.044 | INFO | "malloc" : " 14 GB", | 20260507 17:25:18.044 | INFO | "pinned" : " 29 GB" | 20260507 17:25:18.044 | INFO | } | 20260507 17:25:18.044 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:25:18.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:18.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:18.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:18.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:18.044 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 17:25:18.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:18.044 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260507 17:25:18.044 | INFO | PRE_HQ : 68.1%, 1.8, 65.0ms, 0 | 20260507 17:25:18.044 | INFO | BAZWRITER : 69.6%, 3.1, 109.6ms, 0 | 20260507 17:25:18.044 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:25:18.044 | INFO | Uploaded 36GiB to GPU in 2.53233s (14.2162GiB/s) | 20260507 17:25:18.044 | INFO | Downloaded 38GiB to GPU in 2.7313s (13.9128GiB/s) | 20260507 17:25:23.185 | INFO | Analyzing chunk frames = [1274368,1274880) | 20260507 17:25:23.185 | INFO | MemInfo: { | 20260507 17:25:23.185 | INFO | "gpu" : " 17 GB", | 20260507 17:25:23.185 | INFO | "huge" : "", | 20260507 17:25:23.185 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:23.185 | INFO | "malloc" : " 14 GB", | 20260507 17:25:23.185 | INFO | "pinned" : " 29 GB" | 20260507 17:25:23.185 | INFO | } | 20260507 17:25:23.185 | INFO | Perf report for the last 5.14108 seconds: | 20260507 17:25:23.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:23.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.185 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:25:23.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.185 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:25:23.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:23.186 | INFO | Uploaded 36GiB to GPU in 2.47649s (14.5367GiB/s) | 20260507 17:25:23.186 | INFO | Downloaded 34GiB to GPU in 2.4602s (13.82GiB/s) | 20260507 17:25:28.329 | INFO | Analyzing chunk frames = [1274880,1275392) | 20260507 17:25:28.329 | INFO | MemInfo: { | 20260507 17:25:28.329 | INFO | "gpu" : " 17 GB", | 20260507 17:25:28.329 | INFO | "huge" : "", | 20260507 17:25:28.329 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:28.329 | INFO | "malloc" : " 14 GB", | 20260507 17:25:28.329 | INFO | "pinned" : " 29 GB" | 20260507 17:25:28.330 | INFO | } | 20260507 17:25:28.330 | INFO | Perf report for the last 5.14446 seconds: | 20260507 17:25:28.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:28.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 17:25:28.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:25:28.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:28.330 | INFO | Uploaded 36GiB to GPU in 2.40924s (14.9425GiB/s) | 20260507 17:25:28.330 | INFO | Downloaded 34GiB to GPU in 2.352s (14.4558GiB/s) | 20260507 17:25:33.462 | INFO | Analyzing chunk frames = [1275392,1275904) | 20260507 17:25:33.462 | INFO | MemInfo: { | 20260507 17:25:33.462 | INFO | "gpu" : " 17 GB", | 20260507 17:25:33.462 | INFO | "huge" : "", | 20260507 17:25:33.462 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:33.462 | INFO | "malloc" : " 14 GB", | 20260507 17:25:33.462 | INFO | "pinned" : " 29 GB" | 20260507 17:25:33.462 | INFO | } | 20260507 17:25:33.462 | INFO | Perf report for the last 5.13245 seconds: | 20260507 17:25:33.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:33.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260507 17:25:33.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260507 17:25:33.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:33.462 | INFO | Uploaded 36GiB to GPU in 2.44633s (14.7159GiB/s) | 20260507 17:25:33.463 | INFO | Downloaded 34GiB to GPU in 2.37938s (14.2894GiB/s) | 20260507 17:25:38.602 | INFO | Analyzing chunk frames = [1275904,1276416) | 20260507 17:25:38.602 | INFO | MemInfo: { | 20260507 17:25:38.602 | INFO | "gpu" : " 17 GB", | 20260507 17:25:38.602 | INFO | "huge" : "", | 20260507 17:25:38.602 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:38.602 | INFO | "malloc" : " 14 GB", | 20260507 17:25:38.602 | INFO | "pinned" : " 29 GB" | 20260507 17:25:38.602 | INFO | } | 20260507 17:25:38.602 | INFO | Perf report for the last 5.13982 seconds: | 20260507 17:25:38.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:38.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:25:38.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | PRE_HQ : 39.8%, 1.0, 20.7ms, 0 | 20260507 17:25:38.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:38.602 | INFO | Uploaded 36GiB to GPU in 2.28899s (15.7275GiB/s) | 20260507 17:25:38.602 | INFO | Downloaded 34GiB to GPU in 2.49951s (13.6027GiB/s) | 20260507 17:25:43.740 | INFO | Analyzing chunk frames = [1276416,1276928) | 20260507 17:25:43.740 | INFO | MemInfo: { | 20260507 17:25:43.740 | INFO | "gpu" : " 17 GB", | 20260507 17:25:43.740 | INFO | "huge" : "", | 20260507 17:25:43.740 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:43.740 | INFO | "malloc" : " 14 GB", | 20260507 17:25:43.741 | INFO | "pinned" : " 29 GB" | 20260507 17:25:43.741 | INFO | } | 20260507 17:25:43.741 | INFO | Perf report for the last 5.13877 seconds: | 20260507 17:25:43.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:43.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:25:43.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 17:25:43.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:43.741 | INFO | Uploaded 36GiB to GPU in 2.46285s (14.6172GiB/s) | 20260507 17:25:43.741 | INFO | Downloaded 34GiB to GPU in 2.5122s (13.534GiB/s) | 20260507 17:25:48.880 | INFO | Analyzing chunk frames = [1276928,1277440) | 20260507 17:25:48.880 | INFO | MemInfo: { | 20260507 17:25:48.880 | INFO | "gpu" : " 17 GB", | 20260507 17:25:48.880 | INFO | "huge" : "", | 20260507 17:25:48.880 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:48.880 | INFO | "malloc" : " 14 GB", | 20260507 17:25:48.880 | INFO | "pinned" : " 29 GB" | 20260507 17:25:48.880 | INFO | } | 20260507 17:25:48.880 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:25:48.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:48.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.880 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:25:48.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.880 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:25:48.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:48.881 | INFO | Uploaded 36GiB to GPU in 2.47061s (14.5713GiB/s) | 20260507 17:25:48.881 | INFO | Downloaded 34GiB to GPU in 2.43378s (13.9701GiB/s) | 20260507 17:25:54.019 | INFO | Analyzing chunk frames = [1277440,1277952) | 20260507 17:25:54.019 | INFO | MemInfo: { | 20260507 17:25:54.019 | INFO | "gpu" : " 17 GB", | 20260507 17:25:54.019 | INFO | "huge" : "", | 20260507 17:25:54.019 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:54.019 | INFO | "malloc" : " 14 GB", | 20260507 17:25:54.020 | INFO | "pinned" : " 29 GB" | 20260507 17:25:54.020 | INFO | } | 20260507 17:25:54.020 | INFO | Perf report for the last 5.13933 seconds: | 20260507 17:25:54.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:54.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:25:54.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:25:54.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:54.020 | INFO | Uploaded 36GiB to GPU in 2.50608s (14.3651GiB/s) | 20260507 17:25:54.020 | INFO | Downloaded 34GiB to GPU in 2.50705s (13.5618GiB/s) | 20260507 17:25:54.020 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:25:54.020 | INFO | Total mem usage: 16.4 GiB | 20260507 17:25:54.020 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:25:54.020 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.3%) | 20260507 17:25:54.020 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:25:54.020 | INFO | High Water Breakdown: | 20260507 17:25:54.020 | INFO | 59.1% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:25:54.020 | INFO | 40.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:25:54.020 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:25:54.020 | INFO | ------------------------------------------ | 20260507 17:25:54.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:25:54.020 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:25:54.020 | INFO | Total mem usage: 26.5 GiB | 20260507 17:25:54.020 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:25:54.020 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:25:54.020 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:25:54.020 | INFO | High Water Breakdown: | 20260507 17:25:54.020 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:25:54.020 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:25:54.020 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:25:54.020 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:25:54.020 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:25:54.020 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:25:54.020 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:25:54.020 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:25:54.020 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:25:54.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:25:54.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:25:54.020 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:25:54.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:25:54.020 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:25:54.020 | INFO | ------------------------------------------ | 20260507 17:25:54.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:25:54.020 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:25:54.020 | INFO | Total mem usage: 27.8 GiB | 20260507 17:25:54.020 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:25:54.020 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:25:54.020 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:25:54.020 | INFO | High Water Breakdown: | 20260507 17:25:54.020 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:25:54.020 | INFO | ------------------------------------------ | 20260507 17:25:54.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:25:54.020 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:25:54.020 | INFO | Total mem usage: 15.7 GiB | 20260507 17:25:54.020 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:25:54.020 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:25:54.020 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:25:54.020 | INFO | High Water Breakdown: | 20260507 17:25:54.020 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:25:54.020 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:25:54.020 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:25:54.020 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:25:54.020 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:25:54.020 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:25:54.020 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:25:54.020 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:25:54.020 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:25:54.020 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:25:54.020 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:25:54.020 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:25:54.020 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:25:54.020 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:25:54.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:25:54.020 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:25:54.020 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:25:54.020 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:25:54.020 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:25:54.020 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:25:54.020 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:25:54.020 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:25:54.020 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:25:54.020 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:25:54.021 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:25:54.021 | INFO | ------------------------------------------ | 20260507 17:25:54.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:25:54.021 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:25:54.021 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:25:54.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:25:54.021 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.41e+01 / 2.52e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.03e-01, 1.13e-02 / 5.49e-03 | 20260507 17:25:54.021 | INFO | PerfReport: 12.07%, Baseline -- 4.05e+00 / 4.54e+00, 6.21e-01 / 1.70e-01 | 20260507 17:25:54.021 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 8.99e-02 | 20260507 17:25:54.021 | INFO | PerfReport: 15.74%, DME -- 4.22e+01 / 4.53e+01, 4.32e+00 / 2.69e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 55.93%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.85e+00 / 2.26e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.53e+00 / 2.89e+00, 4.37e-01 / 2.42e-01 | 20260507 17:25:54.021 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.45e+00, 3.25e-01 / 1.83e-01 | 20260507 17:25:54.021 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.25e-01, 6.71e-02 / 4.10e-02 | 20260507 17:25:54.021 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:25:54.021 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:25:54.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:25:54.021 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 23.82%, Upload -- 2.56e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 24.08%, Download -- 2.59e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 31.23%, Compute -- 3.35e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 1.63%, LockUpload -- 1.75e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 2.44%, LockDownload -- 2.62e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 16.79%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:25:54.021 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260507 17:25:54.117 | INFO | Starting a round of Baz IO | 20260507 17:25:57.640 | INFO | Populating RTMetrics report | 20260507 17:25:57.640 | INFO | RTMetrics report populated | 20260507 17:25:57.640 | INFO | Opening tmp RTMetrics json file | 20260507 17:25:57.641 | INFO | Serializing RTMetrics to json | 20260507 17:25:57.642 | INFO | Serialization done, swapping tmp file now... | 20260507 17:25:57.642 | INFO | RTMetrics done | 20260507 17:25:57.727 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.61 seconds | 20260507 17:25:57.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.19% / 0.02% / 51.79% | 20260507 17:25:57.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.62767% / 5.5631e-05% / 3.57628e-05% | 20260507 17:25:57.727 | INFO | Baz IO overall: | 20260507 17:25:57.727 | INFO | BytesWritten : 562096987994 | 20260507 17:25:57.727 | INFO | NumEvents : 272332042465 | 20260507 17:25:59.159 | INFO | Analyzing chunk frames = [1277952,1278464) | 20260507 17:25:59.159 | INFO | MemInfo: { | 20260507 17:25:59.159 | INFO | "gpu" : " 17 GB", | 20260507 17:25:59.159 | INFO | "huge" : "", | 20260507 17:25:59.159 | INFO | "huge_shared" : " 29 GB", | 20260507 17:25:59.159 | INFO | "malloc" : " 14 GB", | 20260507 17:25:59.159 | INFO | "pinned" : " 29 GB" | 20260507 17:25:59.159 | INFO | } | 20260507 17:25:59.159 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:25:59.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:25:59.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:59.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:59.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:59.159 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:25:59.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:25:59.160 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 17:25:59.160 | INFO | PRE_HQ : 67.8%, 1.8, 64.9ms, 0 | 20260507 17:25:59.160 | INFO | BAZWRITER : 69.6%, 3.0, 108.8ms, 0 | 20260507 17:25:59.160 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:25:59.160 | INFO | Uploaded 36GiB to GPU in 2.54324s (14.1552GiB/s) | 20260507 17:25:59.160 | INFO | Downloaded 38GiB to GPU in 2.71625s (13.9899GiB/s) | 20260507 17:26:04.298 | INFO | Analyzing chunk frames = [1278464,1278976) | 20260507 17:26:04.298 | INFO | MemInfo: { | 20260507 17:26:04.298 | INFO | "gpu" : " 17 GB", | 20260507 17:26:04.298 | INFO | "huge" : "", | 20260507 17:26:04.298 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:04.298 | INFO | "malloc" : " 14 GB", | 20260507 17:26:04.298 | INFO | "pinned" : " 29 GB" | 20260507 17:26:04.298 | INFO | } | 20260507 17:26:04.298 | INFO | Perf report for the last 5.1391 seconds: | 20260507 17:26:04.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:04.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.299 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:26:04.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.299 | INFO | PRE_HQ : 34.8%, 1.0, 18.0ms, 0 | 20260507 17:26:04.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:04.299 | INFO | Uploaded 36GiB to GPU in 2.53363s (14.2089GiB/s) | 20260507 17:26:04.299 | INFO | Downloaded 34GiB to GPU in 2.46068s (13.8173GiB/s) | 20260507 17:26:09.438 | INFO | Analyzing chunk frames = [1278976,1279488) | 20260507 17:26:09.438 | INFO | MemInfo: { | 20260507 17:26:09.438 | INFO | "gpu" : " 17 GB", | 20260507 17:26:09.438 | INFO | "huge" : "", | 20260507 17:26:09.438 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:09.438 | INFO | "malloc" : " 14 GB", | 20260507 17:26:09.438 | INFO | "pinned" : " 29 GB" | 20260507 17:26:09.438 | INFO | } | 20260507 17:26:09.438 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:26:09.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:09.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:26:09.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:26:09.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:09.438 | INFO | Uploaded 36GiB to GPU in 2.53964s (14.1752GiB/s) | 20260507 17:26:09.438 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9296GiB/s) | 20260507 17:26:14.577 | INFO | Analyzing chunk frames = [1279488,1280000) | 20260507 17:26:14.578 | INFO | MemInfo: { | 20260507 17:26:14.578 | INFO | "gpu" : " 17 GB", | 20260507 17:26:14.578 | INFO | "huge" : "", | 20260507 17:26:14.578 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:14.578 | INFO | "malloc" : " 14 GB", | 20260507 17:26:14.578 | INFO | "pinned" : " 29 GB" | 20260507 17:26:14.578 | INFO | } | 20260507 17:26:14.578 | INFO | Perf report for the last 5.1399 seconds: | 20260507 17:26:14.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:14.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:26:14.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:26:14.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:14.578 | INFO | Uploaded 36GiB to GPU in 2.46358s (14.6129GiB/s) | 20260507 17:26:14.578 | INFO | Downloaded 34GiB to GPU in 2.35429s (14.4417GiB/s) | 20260507 17:26:19.716 | INFO | Analyzing chunk frames = [1280000,1280512) | 20260507 17:26:19.716 | INFO | MemInfo: { | 20260507 17:26:19.716 | INFO | "gpu" : " 17 GB", | 20260507 17:26:19.716 | INFO | "huge" : "", | 20260507 17:26:19.716 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:19.716 | INFO | "malloc" : " 14 GB", | 20260507 17:26:19.716 | INFO | "pinned" : " 29 GB" | 20260507 17:26:19.716 | INFO | } | 20260507 17:26:19.716 | INFO | Perf report for the last 5.13855 seconds: | 20260507 17:26:19.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:19.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.716 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 17:26:19.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.716 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:26:19.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:19.717 | INFO | Uploaded 36GiB to GPU in 2.30006s (15.6517GiB/s) | 20260507 17:26:19.717 | INFO | Downloaded 34GiB to GPU in 2.48419s (13.6866GiB/s) | 20260507 17:26:24.856 | INFO | Analyzing chunk frames = [1280512,1281024) | 20260507 17:26:24.856 | INFO | MemInfo: { | 20260507 17:26:24.856 | INFO | "gpu" : " 17 GB", | 20260507 17:26:24.856 | INFO | "huge" : "", | 20260507 17:26:24.856 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:24.856 | INFO | "malloc" : " 14 GB", | 20260507 17:26:24.856 | INFO | "pinned" : " 29 GB" | 20260507 17:26:24.856 | INFO | } | 20260507 17:26:24.856 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:26:24.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:24.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 17:26:24.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 17:26:24.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:24.856 | INFO | Uploaded 36GiB to GPU in 2.25977s (15.9308GiB/s) | 20260507 17:26:24.857 | INFO | Downloaded 34GiB to GPU in 2.49075s (13.6505GiB/s) | 20260507 17:26:29.995 | INFO | Analyzing chunk frames = [1281024,1281536) | 20260507 17:26:29.995 | INFO | MemInfo: { | 20260507 17:26:29.995 | INFO | "gpu" : " 17 GB", | 20260507 17:26:29.995 | INFO | "huge" : "", | 20260507 17:26:29.995 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:29.995 | INFO | "malloc" : " 14 GB", | 20260507 17:26:29.995 | INFO | "pinned" : " 29 GB" | 20260507 17:26:29.995 | INFO | } | 20260507 17:26:29.995 | INFO | Perf report for the last 5.13937 seconds: | 20260507 17:26:29.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:29.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.995 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:26:29.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.995 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:26:29.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:29.996 | INFO | Uploaded 36GiB to GPU in 2.44064s (14.7502GiB/s) | 20260507 17:26:29.996 | INFO | Downloaded 34GiB to GPU in 2.45167s (13.8681GiB/s) | 20260507 17:26:35.134 | INFO | Analyzing chunk frames = [1281536,1282048) | 20260507 17:26:35.134 | INFO | MemInfo: { | 20260507 17:26:35.134 | INFO | "gpu" : " 17 GB", | 20260507 17:26:35.134 | INFO | "huge" : "", | 20260507 17:26:35.134 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:35.134 | INFO | "malloc" : " 14 GB", | 20260507 17:26:35.134 | INFO | "pinned" : " 29 GB" | 20260507 17:26:35.134 | INFO | } | 20260507 17:26:35.134 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:26:35.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:35.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:26:35.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:26:35.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:35.135 | INFO | Uploaded 36GiB to GPU in 2.48775s (14.4709GiB/s) | 20260507 17:26:35.135 | INFO | Downloaded 34GiB to GPU in 2.52059s (13.4889GiB/s) | 20260507 17:26:35.232 | INFO | Starting a round of Baz IO | 20260507 17:26:38.738 | INFO | Populating RTMetrics report | 20260507 17:26:38.738 | INFO | RTMetrics report populated | 20260507 17:26:38.738 | INFO | Opening tmp RTMetrics json file | 20260507 17:26:38.754 | INFO | Serializing RTMetrics to json | 20260507 17:26:38.756 | INFO | Serialization done, swapping tmp file now... | 20260507 17:26:38.757 | INFO | RTMetrics done | 20260507 17:26:38.812 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.58 seconds | 20260507 17:26:38.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.17% / 0.02% / 51.81% | 20260507 17:26:38.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.62236% / 6.75519e-05% / 2.38419e-05% | 20260507 17:26:38.812 | INFO | Baz IO overall: | 20260507 17:26:38.813 | INFO | BytesWritten : 564030709596 | 20260507 17:26:38.813 | INFO | NumEvents : 273272109787 | 20260507 17:26:40.277 | INFO | Analyzing chunk frames = [1282048,1282560) | 20260507 17:26:40.277 | INFO | MemInfo: { | 20260507 17:26:40.277 | INFO | "gpu" : " 17 GB", | 20260507 17:26:40.277 | INFO | "huge" : "", | 20260507 17:26:40.277 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:40.277 | INFO | "malloc" : " 14 GB", | 20260507 17:26:40.277 | INFO | "pinned" : " 29 GB" | 20260507 17:26:40.277 | INFO | } | 20260507 17:26:40.277 | INFO | Perf report for the last 5.14233 seconds: | 20260507 17:26:40.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:40.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:40.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:40.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:40.277 | INFO | ANALYSIS : 69.3%, 3.0, 108.3ms, 0 | 20260507 17:26:40.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:40.277 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 17:26:40.277 | INFO | PRE_HQ : 67.4%, 1.8, 64.6ms, 0 | 20260507 17:26:40.277 | INFO | BAZWRITER : 68.7%, 3.0, 104.9ms, 0 | 20260507 17:26:40.277 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:26:40.277 | INFO | Uploaded 36GiB to GPU in 2.45354s (14.6727GiB/s) | 20260507 17:26:40.277 | INFO | Downloaded 38GiB to GPU in 2.73107s (13.914GiB/s) | 20260507 17:26:45.413 | INFO | Analyzing chunk frames = [1282560,1283072) | 20260507 17:26:45.413 | INFO | MemInfo: { | 20260507 17:26:45.413 | INFO | "gpu" : " 17 GB", | 20260507 17:26:45.413 | INFO | "huge" : "", | 20260507 17:26:45.413 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:45.413 | INFO | "malloc" : " 14 GB", | 20260507 17:26:45.413 | INFO | "pinned" : " 29 GB" | 20260507 17:26:45.413 | INFO | } | 20260507 17:26:45.413 | INFO | Perf report for the last 5.13632 seconds: | 20260507 17:26:45.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:45.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:26:45.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:26:45.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:45.413 | INFO | Uploaded 36GiB to GPU in 2.52744s (14.2437GiB/s) | 20260507 17:26:45.414 | INFO | Downloaded 34GiB to GPU in 2.45145s (13.8693GiB/s) | 20260507 17:26:50.552 | INFO | Analyzing chunk frames = [1283072,1283584) | 20260507 17:26:50.552 | INFO | MemInfo: { | 20260507 17:26:50.552 | INFO | "gpu" : " 17 GB", | 20260507 17:26:50.552 | INFO | "huge" : "", | 20260507 17:26:50.552 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:50.552 | INFO | "malloc" : " 14 GB", | 20260507 17:26:50.553 | INFO | "pinned" : " 29 GB" | 20260507 17:26:50.553 | INFO | } | 20260507 17:26:50.553 | INFO | Perf report for the last 5.13945 seconds: | 20260507 17:26:50.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:50.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:26:50.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:26:50.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:50.553 | INFO | Uploaded 36GiB to GPU in 2.55991s (14.063GiB/s) | 20260507 17:26:50.553 | INFO | Downloaded 34GiB to GPU in 2.43728s (13.95GiB/s) | 20260507 17:26:55.692 | INFO | Analyzing chunk frames = [1283584,1284096) | 20260507 17:26:55.692 | INFO | MemInfo: { | 20260507 17:26:55.692 | INFO | "gpu" : " 17 GB", | 20260507 17:26:55.692 | INFO | "huge" : "", | 20260507 17:26:55.692 | INFO | "huge_shared" : " 29 GB", | 20260507 17:26:55.692 | INFO | "malloc" : " 14 GB", | 20260507 17:26:55.695 | INFO | "pinned" : " 29 GB" | 20260507 17:26:55.695 | INFO | } | 20260507 17:26:55.695 | INFO | Perf report for the last 5.1424 seconds: | 20260507 17:26:55.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:26:55.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:26:55.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:26:55.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:26:55.695 | INFO | Uploaded 36GiB to GPU in 2.47246s (14.5604GiB/s) | 20260507 17:26:55.695 | INFO | Downloaded 34GiB to GPU in 2.3572s (14.4239GiB/s) | 20260507 17:27:00.832 | INFO | Analyzing chunk frames = [1284096,1284608) | 20260507 17:27:00.832 | INFO | MemInfo: { | 20260507 17:27:00.832 | INFO | "gpu" : " 17 GB", | 20260507 17:27:00.832 | INFO | "huge" : "", | 20260507 17:27:00.832 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:00.832 | INFO | "malloc" : " 14 GB", | 20260507 17:27:00.832 | INFO | "pinned" : " 29 GB" | 20260507 17:27:00.832 | INFO | } | 20260507 17:27:00.833 | INFO | Perf report for the last 5.13744 seconds: | 20260507 17:27:00.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:00.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 17:27:00.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 17:27:00.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:00.833 | INFO | Uploaded 36GiB to GPU in 2.34428s (15.3565GiB/s) | 20260507 17:27:00.833 | INFO | Downloaded 34GiB to GPU in 2.48166s (13.7005GiB/s) | 20260507 17:27:05.970 | INFO | Analyzing chunk frames = [1284608,1285120) | 20260507 17:27:05.970 | INFO | MemInfo: { | 20260507 17:27:05.970 | INFO | "gpu" : " 17 GB", | 20260507 17:27:05.970 | INFO | "huge" : "", | 20260507 17:27:05.970 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:05.970 | INFO | "malloc" : " 14 GB", | 20260507 17:27:05.971 | INFO | "pinned" : " 29 GB" | 20260507 17:27:05.971 | INFO | } | 20260507 17:27:05.971 | INFO | Perf report for the last 5.13819 seconds: | 20260507 17:27:05.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:05.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:27:05.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:27:05.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:05.971 | INFO | Uploaded 36GiB to GPU in 2.40904s (14.9437GiB/s) | 20260507 17:27:05.971 | INFO | Downloaded 34GiB to GPU in 2.47614s (13.731GiB/s) | 20260507 17:27:11.110 | INFO | Analyzing chunk frames = [1285120,1285632) | 20260507 17:27:11.110 | INFO | MemInfo: { | 20260507 17:27:11.110 | INFO | "gpu" : " 17 GB", | 20260507 17:27:11.110 | INFO | "huge" : "", | 20260507 17:27:11.110 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:11.110 | INFO | "malloc" : " 14 GB", | 20260507 17:27:11.110 | INFO | "pinned" : " 29 GB" | 20260507 17:27:11.110 | INFO | } | 20260507 17:27:11.110 | INFO | Perf report for the last 5.13962 seconds: | 20260507 17:27:11.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:11.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.110 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:27:11.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.110 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:27:11.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:11.111 | INFO | Uploaded 36GiB to GPU in 2.46689s (14.5933GiB/s) | 20260507 17:27:11.111 | INFO | Downloaded 34GiB to GPU in 2.4133s (14.0886GiB/s) | 20260507 17:27:16.250 | INFO | Analyzing chunk frames = [1285632,1286144) | 20260507 17:27:16.250 | INFO | MemInfo: { | 20260507 17:27:16.250 | INFO | "gpu" : " 17 GB", | 20260507 17:27:16.250 | INFO | "huge" : "", | 20260507 17:27:16.250 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:16.250 | INFO | "malloc" : " 14 GB", | 20260507 17:27:16.250 | INFO | "pinned" : " 29 GB" | 20260507 17:27:16.250 | INFO | } | 20260507 17:27:16.250 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:27:16.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:16.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 17:27:16.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:27:16.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:16.250 | INFO | Uploaded 36GiB to GPU in 2.53737s (14.1879GiB/s) | 20260507 17:27:16.250 | INFO | Downloaded 34GiB to GPU in 2.49961s (13.6021GiB/s) | 20260507 17:27:16.251 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:27:16.251 | INFO | Total mem usage: 16.4 GiB | 20260507 17:27:16.251 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:27:16.251 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:27:16.251 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:27:16.251 | INFO | High Water Breakdown: | 20260507 17:27:16.251 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:27:16.251 | INFO | 40.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:27:16.251 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:27:16.251 | INFO | ------------------------------------------ | 20260507 17:27:16.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:27:16.251 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:27:16.251 | INFO | Total mem usage: 26.5 GiB | 20260507 17:27:16.251 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:27:16.251 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:27:16.251 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:27:16.251 | INFO | High Water Breakdown: | 20260507 17:27:16.251 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:27:16.251 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:27:16.251 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:27:16.251 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:27:16.251 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:27:16.251 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:27:16.251 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:27:16.251 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:27:16.251 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:27:16.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:27:16.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:27:16.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:27:16.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:27:16.251 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:27:16.251 | INFO | ------------------------------------------ | 20260507 17:27:16.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:27:16.251 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:27:16.251 | INFO | Total mem usage: 27.8 GiB | 20260507 17:27:16.251 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:27:16.251 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:27:16.251 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:27:16.251 | INFO | High Water Breakdown: | 20260507 17:27:16.251 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:27:16.251 | INFO | ------------------------------------------ | 20260507 17:27:16.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:27:16.251 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:27:16.252 | INFO | Total mem usage: 15.7 GiB | 20260507 17:27:16.252 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:27:16.252 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:27:16.252 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:27:16.252 | INFO | High Water Breakdown: | 20260507 17:27:16.252 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:27:16.252 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:27:16.252 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:27:16.252 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:27:16.252 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:27:16.252 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:27:16.252 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:27:16.252 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:27:16.252 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:27:16.252 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:27:16.252 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:27:16.252 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:27:16.252 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:27:16.252 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:27:16.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:27:16.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:27:16.252 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:27:16.252 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:27:16.252 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:27:16.252 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:27:16.252 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:27:16.252 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:27:16.252 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:27:16.252 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:27:16.252 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:27:16.252 | INFO | ------------------------------------------ | 20260507 17:27:16.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:27:16.252 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:27:16.252 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:27:16.252 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:27:16.252 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.68e+01, 1.46e+01 / 2.61e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 0.29%, AggregateStats -- 9.86e-02 / 1.02e-01, 1.19e-02 / 5.78e-03 | 20260507 17:27:16.252 | INFO | PerfReport: 12.09%, Baseline -- 4.05e+00 / 4.52e+00, 6.20e-01 / 2.07e-01 | 20260507 17:27:16.252 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.18e-01 / 9.59e-02 | 20260507 17:27:16.252 | INFO | PerfReport: 15.75%, DME -- 4.23e+01 / 4.51e+01, 4.38e+00 / 2.68e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.85e+00 / 1.98e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.52e+00 / 2.86e+00, 4.36e-01 / 2.48e-01 | 20260507 17:27:16.252 | INFO | PerfReport: 3.77%, Metrics -- 1.26e+00 / 1.43e+00, 3.21e-01 / 2.11e-01 | 20260507 17:27:16.252 | INFO | PerfReport: 1.19%, PulseSerializing -- 4.00e-01 / 4.26e-01, 6.67e-02 / 4.19e-02 | 20260507 17:27:16.252 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:27:16.252 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:27:16.252 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:27:16.252 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 23.83%, Upload -- 2.56e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 24.09%, Download -- 2.59e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 31.21%, Compute -- 3.35e+01 / 0.00e+00, 1.46e+01 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 1.72%, LockUpload -- 1.84e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 2.43%, LockDownload -- 2.62e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 16.70%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:27:16.252 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 17:27:16.347 | INFO | Starting a round of Baz IO | 20260507 17:27:19.857 | INFO | Populating RTMetrics report | 20260507 17:27:19.857 | INFO | RTMetrics report populated | 20260507 17:27:19.857 | INFO | Opening tmp RTMetrics json file | 20260507 17:27:19.858 | INFO | Serializing RTMetrics to json | 20260507 17:27:19.869 | INFO | Serialization done, swapping tmp file now... | 20260507 17:27:19.876 | INFO | RTMetrics done | 20260507 17:27:19.938 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.591 seconds | 20260507 17:27:19.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.15% / 0.02% / 51.83% | 20260507 17:27:19.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.60618% / 2.38419e-05% / 3.57628e-05% | 20260507 17:27:19.938 | INFO | Baz IO overall: | 20260507 17:27:19.938 | INFO | BytesWritten : 565963265858 | 20260507 17:27:19.938 | INFO | NumEvents : 274211209906 | 20260507 17:27:21.390 | INFO | Analyzing chunk frames = [1286144,1286656) | 20260507 17:27:21.390 | INFO | MemInfo: { | 20260507 17:27:21.390 | INFO | "gpu" : " 17 GB", | 20260507 17:27:21.390 | INFO | "huge" : "", | 20260507 17:27:21.390 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:21.390 | INFO | "malloc" : " 14 GB", | 20260507 17:27:21.390 | INFO | "pinned" : " 29 GB" | 20260507 17:27:21.390 | INFO | } | 20260507 17:27:21.390 | INFO | Perf report for the last 5.13976 seconds: | 20260507 17:27:21.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:21.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:21.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:21.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:21.390 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:27:21.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:21.390 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 17:27:21.390 | INFO | PRE_HQ : 67.1%, 1.8, 61.1ms, 0 | 20260507 17:27:21.390 | INFO | BAZWRITER : 68.9%, 2.9, 102.0ms, 0 | 20260507 17:27:21.390 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:27:21.390 | INFO | Uploaded 36GiB to GPU in 2.56862s (14.0153GiB/s) | 20260507 17:27:21.390 | INFO | Downloaded 38GiB to GPU in 2.72695s (13.935GiB/s) | 20260507 17:27:26.528 | INFO | Analyzing chunk frames = [1286656,1287168) | 20260507 17:27:26.528 | INFO | MemInfo: { | 20260507 17:27:26.528 | INFO | "gpu" : " 17 GB", | 20260507 17:27:26.528 | INFO | "huge" : "", | 20260507 17:27:26.528 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:26.528 | INFO | "malloc" : " 14 GB", | 20260507 17:27:26.528 | INFO | "pinned" : " 29 GB" | 20260507 17:27:26.528 | INFO | } | 20260507 17:27:26.528 | INFO | Perf report for the last 5.13873 seconds: | 20260507 17:27:26.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:26.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.529 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:27:26.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.529 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:27:26.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:26.529 | INFO | Uploaded 36GiB to GPU in 2.53231s (14.2163GiB/s) | 20260507 17:27:26.529 | INFO | Downloaded 34GiB to GPU in 2.43268s (13.9764GiB/s) | 20260507 17:27:31.668 | INFO | Analyzing chunk frames = [1287168,1287680) | 20260507 17:27:31.668 | INFO | MemInfo: { | 20260507 17:27:31.668 | INFO | "gpu" : " 17 GB", | 20260507 17:27:31.668 | INFO | "huge" : "", | 20260507 17:27:31.668 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:31.668 | INFO | "malloc" : " 14 GB", | 20260507 17:27:31.668 | INFO | "pinned" : " 29 GB" | 20260507 17:27:31.668 | INFO | } | 20260507 17:27:31.668 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:27:31.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:31.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260507 17:27:31.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:27:31.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:31.668 | INFO | Uploaded 36GiB to GPU in 2.5025s (14.3856GiB/s) | 20260507 17:27:31.668 | INFO | Downloaded 34GiB to GPU in 2.33913s (14.5353GiB/s) | 20260507 17:27:36.807 | INFO | Analyzing chunk frames = [1287680,1288192) | 20260507 17:27:36.807 | INFO | MemInfo: { | 20260507 17:27:36.807 | INFO | "gpu" : " 17 GB", | 20260507 17:27:36.807 | INFO | "huge" : "", | 20260507 17:27:36.807 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:36.808 | INFO | "malloc" : " 14 GB", | 20260507 17:27:36.808 | INFO | "pinned" : " 29 GB" | 20260507 17:27:36.808 | INFO | } | 20260507 17:27:36.808 | INFO | Perf report for the last 5.13983 seconds: | 20260507 17:27:36.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:36.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:27:36.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:27:36.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:36.808 | INFO | Uploaded 36GiB to GPU in 2.49162s (14.4484GiB/s) | 20260507 17:27:36.808 | INFO | Downloaded 34GiB to GPU in 2.34891s (14.4748GiB/s) | 20260507 17:27:41.947 | INFO | Analyzing chunk frames = [1288192,1288704) | 20260507 17:27:41.947 | INFO | MemInfo: { | 20260507 17:27:41.947 | INFO | "gpu" : " 17 GB", | 20260507 17:27:41.947 | INFO | "huge" : "", | 20260507 17:27:41.947 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:41.947 | INFO | "malloc" : " 14 GB", | 20260507 17:27:41.947 | INFO | "pinned" : " 29 GB" | 20260507 17:27:41.947 | INFO | } | 20260507 17:27:41.947 | INFO | Perf report for the last 5.13964 seconds: | 20260507 17:27:41.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:41.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.948 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:27:41.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.948 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 17:27:41.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:41.948 | INFO | Uploaded 36GiB to GPU in 2.3762s (15.1503GiB/s) | 20260507 17:27:41.948 | INFO | Downloaded 34GiB to GPU in 2.44648s (13.8975GiB/s) | 20260507 17:27:47.086 | INFO | Analyzing chunk frames = [1288704,1289216) | 20260507 17:27:47.086 | INFO | MemInfo: { | 20260507 17:27:47.086 | INFO | "gpu" : " 17 GB", | 20260507 17:27:47.086 | INFO | "huge" : "", | 20260507 17:27:47.086 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:47.086 | INFO | "malloc" : " 14 GB", | 20260507 17:27:47.086 | INFO | "pinned" : " 29 GB" | 20260507 17:27:47.086 | INFO | } | 20260507 17:27:47.086 | INFO | Perf report for the last 5.13854 seconds: | 20260507 17:27:47.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:47.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:27:47.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 17:27:47.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:47.086 | INFO | Uploaded 36GiB to GPU in 2.43241s (14.8002GiB/s) | 20260507 17:27:47.086 | INFO | Downloaded 34GiB to GPU in 2.4818s (13.6997GiB/s) | 20260507 17:27:52.225 | INFO | Analyzing chunk frames = [1289216,1289728) | 20260507 17:27:52.225 | INFO | MemInfo: { | 20260507 17:27:52.225 | INFO | "gpu" : " 17 GB", | 20260507 17:27:52.225 | INFO | "huge" : "", | 20260507 17:27:52.225 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:52.225 | INFO | "malloc" : " 14 GB", | 20260507 17:27:52.225 | INFO | "pinned" : " 29 GB" | 20260507 17:27:52.226 | INFO | } | 20260507 17:27:52.226 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:27:52.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:52.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:27:52.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:27:52.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:52.226 | INFO | Uploaded 36GiB to GPU in 2.44296s (14.7362GiB/s) | 20260507 17:27:52.226 | INFO | Downloaded 34GiB to GPU in 2.4075s (14.1225GiB/s) | 20260507 17:27:57.365 | INFO | Analyzing chunk frames = [1289728,1290240) | 20260507 17:27:57.365 | INFO | MemInfo: { | 20260507 17:27:57.365 | INFO | "gpu" : " 17 GB", | 20260507 17:27:57.365 | INFO | "huge" : "", | 20260507 17:27:57.365 | INFO | "huge_shared" : " 29 GB", | 20260507 17:27:57.365 | INFO | "malloc" : " 14 GB", | 20260507 17:27:57.365 | INFO | "pinned" : " 29 GB" | 20260507 17:27:57.365 | INFO | } | 20260507 17:27:57.365 | INFO | Perf report for the last 5.13987 seconds: | 20260507 17:27:57.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:27:57.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.365 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 17:27:57.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.366 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:27:57.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:27:57.366 | INFO | Uploaded 36GiB to GPU in 2.53957s (14.1756GiB/s) | 20260507 17:27:57.366 | INFO | Downloaded 34GiB to GPU in 2.49479s (13.6284GiB/s) | 20260507 17:27:57.462 | INFO | Starting a round of Baz IO | 20260507 17:28:00.973 | INFO | Populating RTMetrics report | 20260507 17:28:00.973 | INFO | RTMetrics report populated | 20260507 17:28:00.973 | INFO | Opening tmp RTMetrics json file | 20260507 17:28:00.979 | INFO | Serializing RTMetrics to json | 20260507 17:28:00.991 | INFO | Serialization done, swapping tmp file now... | 20260507 17:28:00.992 | INFO | RTMetrics done | 20260507 17:28:01.051 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.588 seconds | 20260507 17:28:01.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.13% / 0.02% / 51.85% | 20260507 17:28:01.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.60328% / 3.97364e-05% / 5.16574e-05% | 20260507 17:28:01.051 | INFO | Baz IO overall: | 20260507 17:28:01.051 | INFO | BytesWritten : 567896118195 | 20260507 17:28:01.051 | INFO | NumEvents : 275150505716 | 20260507 17:28:02.504 | INFO | Analyzing chunk frames = [1290240,1290752) | 20260507 17:28:02.504 | INFO | MemInfo: { | 20260507 17:28:02.504 | INFO | "gpu" : " 17 GB", | 20260507 17:28:02.504 | INFO | "huge" : "", | 20260507 17:28:02.504 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:02.504 | INFO | "malloc" : " 14 GB", | 20260507 17:28:02.504 | INFO | "pinned" : " 29 GB" | 20260507 17:28:02.504 | INFO | } | 20260507 17:28:02.504 | INFO | Perf report for the last 5.1387 seconds: | 20260507 17:28:02.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:02.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:02.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:02.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:02.504 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:28:02.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:02.504 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 17:28:02.504 | INFO | PRE_HQ : 67.4%, 1.7, 60.8ms, 0 | 20260507 17:28:02.504 | INFO | BAZWRITER : 68.9%, 2.9, 102.3ms, 0 | 20260507 17:28:02.504 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:28:02.504 | INFO | Uploaded 36GiB to GPU in 2.55403s (14.0954GiB/s) | 20260507 17:28:02.505 | INFO | Downloaded 38GiB to GPU in 2.72249s (13.9578GiB/s) | 20260507 17:28:07.644 | INFO | Analyzing chunk frames = [1290752,1291264) | 20260507 17:28:07.644 | INFO | MemInfo: { | 20260507 17:28:07.644 | INFO | "gpu" : " 17 GB", | 20260507 17:28:07.644 | INFO | "huge" : "", | 20260507 17:28:07.644 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:07.644 | INFO | "malloc" : " 14 GB", | 20260507 17:28:07.644 | INFO | "pinned" : " 29 GB" | 20260507 17:28:07.644 | INFO | } | 20260507 17:28:07.644 | INFO | Perf report for the last 5.13985 seconds: | 20260507 17:28:07.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:07.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:28:07.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260507 17:28:07.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:07.644 | INFO | Uploaded 36GiB to GPU in 2.53826s (14.1829GiB/s) | 20260507 17:28:07.644 | INFO | Downloaded 34GiB to GPU in 2.46159s (13.8122GiB/s) | 20260507 17:28:12.783 | INFO | Analyzing chunk frames = [1291264,1291776) | 20260507 17:28:12.783 | INFO | MemInfo: { | 20260507 17:28:12.783 | INFO | "gpu" : " 17 GB", | 20260507 17:28:12.783 | INFO | "huge" : "", | 20260507 17:28:12.783 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:12.783 | INFO | "malloc" : " 14 GB", | 20260507 17:28:12.783 | INFO | "pinned" : " 29 GB" | 20260507 17:28:12.783 | INFO | } | 20260507 17:28:12.783 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:28:12.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:12.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:28:12.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:28:12.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:12.783 | INFO | Uploaded 36GiB to GPU in 2.53431s (14.2051GiB/s) | 20260507 17:28:12.783 | INFO | Downloaded 34GiB to GPU in 2.43453s (13.9657GiB/s) | 20260507 17:28:17.923 | INFO | Analyzing chunk frames = [1291776,1292288) | 20260507 17:28:17.923 | INFO | MemInfo: { | 20260507 17:28:17.923 | INFO | "gpu" : " 17 GB", | 20260507 17:28:17.923 | INFO | "huge" : "", | 20260507 17:28:17.923 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:17.923 | INFO | "malloc" : " 14 GB", | 20260507 17:28:17.925 | INFO | "pinned" : " 29 GB" | 20260507 17:28:17.925 | INFO | } | 20260507 17:28:17.925 | INFO | Perf report for the last 5.14238 seconds: | 20260507 17:28:17.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:17.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.925 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:28:17.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.925 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:28:17.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:17.926 | INFO | Uploaded 36GiB to GPU in 2.46758s (14.5892GiB/s) | 20260507 17:28:17.926 | INFO | Downloaded 34GiB to GPU in 2.35716s (14.4241GiB/s) | 20260507 17:28:23.062 | INFO | Analyzing chunk frames = [1292288,1292800) | 20260507 17:28:23.062 | INFO | MemInfo: { | 20260507 17:28:23.062 | INFO | "gpu" : " 17 GB", | 20260507 17:28:23.062 | INFO | "huge" : "", | 20260507 17:28:23.062 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:23.062 | INFO | "malloc" : " 14 GB", | 20260507 17:28:23.062 | INFO | "pinned" : " 29 GB" | 20260507 17:28:23.062 | INFO | } | 20260507 17:28:23.062 | INFO | Perf report for the last 5.13698 seconds: | 20260507 17:28:23.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:23.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.062 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:28:23.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.062 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 17:28:23.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:23.063 | INFO | Uploaded 36GiB to GPU in 2.29852s (15.6622GiB/s) | 20260507 17:28:23.063 | INFO | Downloaded 34GiB to GPU in 2.4829s (13.6937GiB/s) | 20260507 17:28:28.204 | INFO | Analyzing chunk frames = [1292800,1293312) | 20260507 17:28:28.204 | INFO | MemInfo: { | 20260507 17:28:28.204 | INFO | "gpu" : " 17 GB", | 20260507 17:28:28.204 | INFO | "huge" : "", | 20260507 17:28:28.204 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:28.204 | INFO | "malloc" : " 14 GB", | 20260507 17:28:28.204 | INFO | "pinned" : " 29 GB" | 20260507 17:28:28.204 | INFO | } | 20260507 17:28:28.205 | INFO | Perf report for the last 5.14208 seconds: | 20260507 17:28:28.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:28.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:28:28.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | PRE_HQ : 36.1%, 1.0, 18.8ms, 0 | 20260507 17:28:28.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:28.205 | INFO | Uploaded 36GiB to GPU in 2.36113s (15.247GiB/s) | 20260507 17:28:28.205 | INFO | Downloaded 34GiB to GPU in 2.46621s (13.7863GiB/s) | 20260507 17:28:33.340 | INFO | Analyzing chunk frames = [1293312,1293824) | 20260507 17:28:33.340 | INFO | MemInfo: { | 20260507 17:28:33.340 | INFO | "gpu" : " 17 GB", | 20260507 17:28:33.340 | INFO | "huge" : "", | 20260507 17:28:33.340 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:33.340 | INFO | "malloc" : " 14 GB", | 20260507 17:28:33.341 | INFO | "pinned" : " 29 GB" | 20260507 17:28:33.341 | INFO | } | 20260507 17:28:33.341 | INFO | Perf report for the last 5.13628 seconds: | 20260507 17:28:33.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:33.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:28:33.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 17:28:33.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:33.341 | INFO | Uploaded 36GiB to GPU in 2.47456s (14.5481GiB/s) | 20260507 17:28:33.341 | INFO | Downloaded 34GiB to GPU in 2.45678s (13.8393GiB/s) | 20260507 17:28:38.480 | INFO | Analyzing chunk frames = [1293824,1294336) | 20260507 17:28:38.480 | INFO | MemInfo: { | 20260507 17:28:38.480 | INFO | "gpu" : " 17 GB", | 20260507 17:28:38.480 | INFO | "huge" : "", | 20260507 17:28:38.480 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:38.480 | INFO | "malloc" : " 14 GB", | 20260507 17:28:38.480 | INFO | "pinned" : " 29 GB" | 20260507 17:28:38.480 | INFO | } | 20260507 17:28:38.480 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:28:38.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:38.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.480 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 17:28:38.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.480 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 17:28:38.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:38.481 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:28:38.481 | INFO | Total mem usage: 16.4 GiB | 20260507 17:28:38.481 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:28:38.481 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.8%) | 20260507 17:28:38.481 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:28:38.481 | INFO | High Water Breakdown: | 20260507 17:28:38.481 | INFO | 59.1% : 8.51 GiB ( 85.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:28:38.481 | INFO | 40.8% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:28:38.481 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:28:38.481 | INFO | ------------------------------------------ | 20260507 17:28:38.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:28:38.481 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:28:38.481 | INFO | Total mem usage: 26.5 GiB | 20260507 17:28:38.481 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:28:38.481 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:28:38.481 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:28:38.481 | INFO | High Water Breakdown: | 20260507 17:28:38.481 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:28:38.481 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:28:38.481 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:28:38.481 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:28:38.481 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:28:38.481 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:28:38.481 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:28:38.481 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:28:38.481 | INFO | Uploaded 36GiB to GPU in 2.48167s (14.5064GiB/s) | 20260507 17:28:38.481 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:28:38.482 | INFO | Downloaded 34GiB to GPU in 2.42989s (13.9924GiB/s) | 20260507 17:28:38.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:28:38.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:28:38.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:28:38.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:28:38.482 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:28:38.482 | INFO | ------------------------------------------ | 20260507 17:28:38.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:28:38.482 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:28:38.482 | INFO | Total mem usage: 27.8 GiB | 20260507 17:28:38.482 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:28:38.482 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:28:38.482 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:28:38.482 | INFO | High Water Breakdown: | 20260507 17:28:38.482 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:28:38.482 | INFO | ------------------------------------------ | 20260507 17:28:38.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:28:38.482 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:28:38.482 | INFO | Total mem usage: 15.7 GiB | 20260507 17:28:38.482 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:28:38.482 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:28:38.482 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:28:38.482 | INFO | High Water Breakdown: | 20260507 17:28:38.482 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:28:38.482 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:28:38.482 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:28:38.482 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:28:38.482 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:28:38.482 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:28:38.482 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:28:38.482 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:28:38.482 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:28:38.482 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:28:38.482 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:28:38.482 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:28:38.482 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:28:38.482 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:28:38.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:28:38.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:28:38.482 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:28:38.482 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:28:38.482 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:28:38.482 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:28:38.482 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:28:38.482 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:28:38.482 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:28:38.482 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:28:38.482 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:28:38.482 | INFO | ------------------------------------------ | 20260507 17:28:38.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:28:38.482 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:28:38.482 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:28:38.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:28:38.482 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.64e+01, 1.44e+01 / 2.56e+00 | 20260507 17:28:38.482 | INFO | PerfReport: 0.30%, AggregateStats -- 9.93e-02 / 1.04e-01, 1.13e-02 / 5.33e-03 | 20260507 17:28:38.482 | INFO | PerfReport: 12.07%, Baseline -- 4.05e+00 / 4.50e+00, 6.18e-01 / 2.41e-01 | 20260507 17:28:38.482 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.18e-01 / 1.17e-01 | 20260507 17:28:38.482 | INFO | PerfReport: 15.70%, DME -- 4.21e+01 / 4.50e+01, 4.34e+00 / 2.67e+00 | 20260507 17:28:38.482 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.88e+00 / 2.11e+00 | 20260507 17:28:38.482 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.89e+00, 4.35e-01 / 2.38e-01 | 20260507 17:28:38.482 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.44e+00, 3.20e-01 / 1.73e-01 | 20260507 17:28:38.482 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.01e-01 / 4.27e-01, 6.58e-02 / 4.30e-02 | 20260507 17:28:38.483 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:28:38.483 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:28:38.483 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:28:38.483 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 23.99%, Upload -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 23.92%, Download -- 2.57e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 31.18%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 1.83%, LockUpload -- 1.97e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.78e+00 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 16.83%, LockCompute -- 1.81e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:28:38.483 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 17:28:38.594 | INFO | Starting a round of Baz IO | 20260507 17:28:42.116 | INFO | Populating RTMetrics report | 20260507 17:28:42.116 | INFO | RTMetrics report populated | 20260507 17:28:42.116 | INFO | Opening tmp RTMetrics json file | 20260507 17:28:42.117 | INFO | Serializing RTMetrics to json | 20260507 17:28:42.118 | INFO | Serialization done, swapping tmp file now... | 20260507 17:28:42.118 | INFO | RTMetrics done | 20260507 17:28:42.214 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.62 seconds | 20260507 17:28:42.214 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.11% / 0.02% / 51.87% | 20260507 17:28:42.214 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.59514% / 2.78155e-05% / 3.97364e-05% | 20260507 17:28:42.214 | INFO | Baz IO overall: | 20260507 17:28:42.214 | INFO | BytesWritten : 569829377950 | 20260507 17:28:42.214 | INFO | NumEvents : 276090142677 | 20260507 17:28:43.619 | INFO | Analyzing chunk frames = [1294336,1294848) | 20260507 17:28:43.619 | INFO | MemInfo: { | 20260507 17:28:43.619 | INFO | "gpu" : " 17 GB", | 20260507 17:28:43.619 | INFO | "huge" : "", | 20260507 17:28:43.619 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:43.619 | INFO | "malloc" : " 14 GB", | 20260507 17:28:43.619 | INFO | "pinned" : " 29 GB" | 20260507 17:28:43.619 | INFO | } | 20260507 17:28:43.619 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:28:43.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:43.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:43.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:43.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:43.619 | INFO | ANALYSIS : 69.9%, 2.9, 107.7ms, 0 | 20260507 17:28:43.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:43.619 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260507 17:28:43.619 | INFO | PRE_HQ : 68.2%, 2.0, 69.7ms, 0 | 20260507 17:28:43.619 | INFO | BAZWRITER : 69.6%, 3.2, 113.5ms, 0 | 20260507 17:28:43.620 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:28:43.620 | INFO | Uploaded 36GiB to GPU in 2.58129s (13.9465GiB/s) | 20260507 17:28:43.620 | INFO | Downloaded 38GiB to GPU in 2.72787s (13.9303GiB/s) | 20260507 17:28:48.758 | INFO | Analyzing chunk frames = [1294848,1295360) | 20260507 17:28:48.758 | INFO | MemInfo: { | 20260507 17:28:48.758 | INFO | "gpu" : " 17 GB", | 20260507 17:28:48.758 | INFO | "huge" : "", | 20260507 17:28:48.758 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:48.758 | INFO | "malloc" : " 14 GB", | 20260507 17:28:48.759 | INFO | "pinned" : " 29 GB" | 20260507 17:28:48.759 | INFO | } | 20260507 17:28:48.759 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:28:48.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:48.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:28:48.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 17:28:48.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:48.759 | INFO | Uploaded 36GiB to GPU in 2.5563s (14.0829GiB/s) | 20260507 17:28:48.759 | INFO | Downloaded 34GiB to GPU in 2.43796s (13.9461GiB/s) | 20260507 17:28:53.898 | INFO | Analyzing chunk frames = [1295360,1295872) | 20260507 17:28:53.898 | INFO | MemInfo: { | 20260507 17:28:53.898 | INFO | "gpu" : " 17 GB", | 20260507 17:28:53.898 | INFO | "huge" : "", | 20260507 17:28:53.898 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:53.898 | INFO | "malloc" : " 14 GB", | 20260507 17:28:53.898 | INFO | "pinned" : " 29 GB" | 20260507 17:28:53.898 | INFO | } | 20260507 17:28:53.898 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:28:53.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:53.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.898 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:28:53.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.899 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 17:28:53.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:53.899 | INFO | Uploaded 36GiB to GPU in 2.56052s (14.0596GiB/s) | 20260507 17:28:53.899 | INFO | Downloaded 34GiB to GPU in 2.43597s (13.9575GiB/s) | 20260507 17:28:59.037 | INFO | Analyzing chunk frames = [1295872,1296384) | 20260507 17:28:59.037 | INFO | MemInfo: { | 20260507 17:28:59.037 | INFO | "gpu" : " 17 GB", | 20260507 17:28:59.037 | INFO | "huge" : "", | 20260507 17:28:59.037 | INFO | "huge_shared" : " 29 GB", | 20260507 17:28:59.037 | INFO | "malloc" : " 14 GB", | 20260507 17:28:59.037 | INFO | "pinned" : " 29 GB" | 20260507 17:28:59.037 | INFO | } | 20260507 17:28:59.038 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:28:59.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:28:59.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:28:59.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 17:28:59.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:28:59.038 | INFO | Uploaded 36GiB to GPU in 2.5048s (14.3724GiB/s) | 20260507 17:28:59.038 | INFO | Downloaded 34GiB to GPU in 2.35454s (14.4402GiB/s) | 20260507 17:29:04.176 | INFO | Analyzing chunk frames = [1296384,1296896) | 20260507 17:29:04.176 | INFO | MemInfo: { | 20260507 17:29:04.176 | INFO | "gpu" : " 17 GB", | 20260507 17:29:04.176 | INFO | "huge" : "", | 20260507 17:29:04.176 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:04.176 | INFO | "malloc" : " 14 GB", | 20260507 17:29:04.177 | INFO | "pinned" : " 29 GB" | 20260507 17:29:04.177 | INFO | } | 20260507 17:29:04.177 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:29:04.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:04.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:29:04.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | PRE_HQ : 38.6%, 1.0, 20.0ms, 0 | 20260507 17:29:04.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:04.177 | INFO | Uploaded 36GiB to GPU in 2.36756s (15.2056GiB/s) | 20260507 17:29:04.178 | INFO | Downloaded 34GiB to GPU in 2.47973s (13.7112GiB/s) | 20260507 17:29:09.316 | INFO | Analyzing chunk frames = [1296896,1297408) | 20260507 17:29:09.316 | INFO | MemInfo: { | 20260507 17:29:09.316 | INFO | "gpu" : " 17 GB", | 20260507 17:29:09.316 | INFO | "huge" : "", | 20260507 17:29:09.316 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:09.316 | INFO | "malloc" : " 14 GB", | 20260507 17:29:09.316 | INFO | "pinned" : " 29 GB" | 20260507 17:29:09.316 | INFO | } | 20260507 17:29:09.316 | INFO | Perf report for the last 5.13946 seconds: | 20260507 17:29:09.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:09.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.316 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:29:09.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.316 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:29:09.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:09.317 | INFO | Uploaded 36GiB to GPU in 2.35735s (15.2714GiB/s) | 20260507 17:29:09.317 | INFO | Downloaded 34GiB to GPU in 2.47381s (13.744GiB/s) | 20260507 17:29:14.456 | INFO | Analyzing chunk frames = [1297408,1297920) | 20260507 17:29:14.456 | INFO | MemInfo: { | 20260507 17:29:14.456 | INFO | "gpu" : " 17 GB", | 20260507 17:29:14.456 | INFO | "huge" : "", | 20260507 17:29:14.456 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:14.456 | INFO | "malloc" : " 14 GB", | 20260507 17:29:14.456 | INFO | "pinned" : " 29 GB" | 20260507 17:29:14.456 | INFO | } | 20260507 17:29:14.456 | INFO | Perf report for the last 5.14008 seconds: | 20260507 17:29:14.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:14.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.456 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:29:14.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.456 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260507 17:29:14.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:14.457 | INFO | Uploaded 36GiB to GPU in 2.4981s (14.4109GiB/s) | 20260507 17:29:14.457 | INFO | Downloaded 34GiB to GPU in 2.43862s (13.9423GiB/s) | 20260507 17:29:19.594 | INFO | Analyzing chunk frames = [1297920,1298432) | 20260507 17:29:19.595 | INFO | MemInfo: { | 20260507 17:29:19.595 | INFO | "gpu" : " 17 GB", | 20260507 17:29:19.595 | INFO | "huge" : "", | 20260507 17:29:19.595 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:19.595 | INFO | "malloc" : " 14 GB", | 20260507 17:29:19.595 | INFO | "pinned" : " 29 GB" | 20260507 17:29:19.595 | INFO | } | 20260507 17:29:19.595 | INFO | Perf report for the last 5.13885 seconds: | 20260507 17:29:19.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:19.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.595 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 17:29:19.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.595 | INFO | PRE_HQ : 38.3%, 1.0, 19.9ms, 0 | 20260507 17:29:19.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:19.596 | INFO | Uploaded 36GiB to GPU in 2.49268s (14.4423GiB/s) | 20260507 17:29:19.596 | INFO | Downloaded 34GiB to GPU in 2.42046s (14.0469GiB/s) | 20260507 17:29:19.709 | INFO | Starting a round of Baz IO | 20260507 17:29:23.186 | INFO | Populating RTMetrics report | 20260507 17:29:23.186 | INFO | RTMetrics report populated | 20260507 17:29:23.186 | INFO | Opening tmp RTMetrics json file | 20260507 17:29:23.194 | INFO | Serializing RTMetrics to json | 20260507 17:29:23.204 | INFO | Serialization done, swapping tmp file now... | 20260507 17:29:23.204 | INFO | RTMetrics done | 20260507 17:29:23.269 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.559 seconds | 20260507 17:29:23.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.09% / 0.02% / 51.89% | 20260507 17:29:23.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.59283% / 3.17891e-05% / 1.19209e-05% | 20260507 17:29:23.269 | INFO | Baz IO overall: | 20260507 17:29:23.269 | INFO | BytesWritten : 571763497490 | 20260507 17:29:23.269 | INFO | NumEvents : 277030381250 | 20260507 17:29:24.734 | INFO | Analyzing chunk frames = [1298432,1298944) | 20260507 17:29:24.734 | INFO | MemInfo: { | 20260507 17:29:24.734 | INFO | "gpu" : " 17 GB", | 20260507 17:29:24.734 | INFO | "huge" : "", | 20260507 17:29:24.734 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:24.734 | INFO | "malloc" : " 14 GB", | 20260507 17:29:24.734 | INFO | "pinned" : " 29 GB" | 20260507 17:29:24.734 | INFO | } | 20260507 17:29:24.734 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:29:24.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:24.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:24.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:24.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:24.734 | INFO | ANALYSIS : 69.4%, 2.9, 107.0ms, 0 | 20260507 17:29:24.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:24.734 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 17:29:24.734 | INFO | PRE_HQ : 67.5%, 1.9, 66.6ms, 0 | 20260507 17:29:24.734 | INFO | BAZWRITER : 68.8%, 3.1, 109.6ms, 0 | 20260507 17:29:24.734 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:29:24.735 | INFO | Uploaded 36GiB to GPU in 2.54056s (14.1701GiB/s) | 20260507 17:29:24.735 | INFO | Downloaded 38GiB to GPU in 2.71145s (14.0146GiB/s) | 20260507 17:29:29.873 | INFO | Analyzing chunk frames = [1298944,1299456) | 20260507 17:29:29.873 | INFO | MemInfo: { | 20260507 17:29:29.873 | INFO | "gpu" : " 17 GB", | 20260507 17:29:29.873 | INFO | "huge" : "", | 20260507 17:29:29.874 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:29.874 | INFO | "malloc" : " 14 GB", | 20260507 17:29:29.874 | INFO | "pinned" : " 29 GB" | 20260507 17:29:29.874 | INFO | } | 20260507 17:29:29.874 | INFO | Perf report for the last 5.13994 seconds: | 20260507 17:29:29.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:29.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:29:29.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 17:29:29.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:29.874 | INFO | Uploaded 36GiB to GPU in 2.49179s (14.4474GiB/s) | 20260507 17:29:29.875 | INFO | Downloaded 34GiB to GPU in 2.46226s (13.8084GiB/s) | 20260507 17:29:35.013 | INFO | Analyzing chunk frames = [1299456,1299968) | 20260507 17:29:35.013 | INFO | MemInfo: { | 20260507 17:29:35.013 | INFO | "gpu" : " 17 GB", | 20260507 17:29:35.013 | INFO | "huge" : "", | 20260507 17:29:35.013 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:35.013 | INFO | "malloc" : " 14 GB", | 20260507 17:29:35.013 | INFO | "pinned" : " 29 GB" | 20260507 17:29:35.013 | INFO | } | 20260507 17:29:35.013 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:29:35.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:35.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.013 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:29:35.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.013 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 17:29:35.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:35.014 | INFO | Uploaded 36GiB to GPU in 2.52579s (14.253GiB/s) | 20260507 17:29:35.014 | INFO | Downloaded 34GiB to GPU in 2.45964s (13.8232GiB/s) | 20260507 17:29:40.152 | INFO | Analyzing chunk frames = [1299968,1300480) | 20260507 17:29:40.152 | INFO | MemInfo: { | 20260507 17:29:40.152 | INFO | "gpu" : " 17 GB", | 20260507 17:29:40.152 | INFO | "huge" : "", | 20260507 17:29:40.152 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:40.152 | INFO | "malloc" : " 14 GB", | 20260507 17:29:40.152 | INFO | "pinned" : " 29 GB" | 20260507 17:29:40.152 | INFO | } | 20260507 17:29:40.152 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:29:40.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:40.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.152 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 17:29:40.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.153 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 17:29:40.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:40.153 | INFO | Uploaded 36GiB to GPU in 2.44051s (14.751GiB/s) | 20260507 17:29:40.153 | INFO | Downloaded 34GiB to GPU in 2.38558s (14.2523GiB/s) | 20260507 17:29:45.291 | INFO | Analyzing chunk frames = [1300480,1300992) | 20260507 17:29:45.291 | INFO | MemInfo: { | 20260507 17:29:45.291 | INFO | "gpu" : " 17 GB", | 20260507 17:29:45.291 | INFO | "huge" : "", | 20260507 17:29:45.292 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:45.292 | INFO | "malloc" : " 14 GB", | 20260507 17:29:45.292 | INFO | "pinned" : " 29 GB" | 20260507 17:29:45.292 | INFO | } | 20260507 17:29:45.292 | INFO | Perf report for the last 5.13924 seconds: | 20260507 17:29:45.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:45.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.292 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 17:29:45.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.292 | INFO | PRE_HQ : 34.8%, 1.0, 18.0ms, 0 | 20260507 17:29:45.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:45.293 | INFO | Uploaded 36GiB to GPU in 2.31278s (15.5657GiB/s) | 20260507 17:29:45.293 | INFO | Downloaded 34GiB to GPU in 2.50308s (13.5833GiB/s) | 20260507 17:29:50.431 | INFO | Analyzing chunk frames = [1300992,1301504) | 20260507 17:29:50.431 | INFO | MemInfo: { | 20260507 17:29:50.431 | INFO | "gpu" : " 17 GB", | 20260507 17:29:50.431 | INFO | "huge" : "", | 20260507 17:29:50.431 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:50.431 | INFO | "malloc" : " 14 GB", | 20260507 17:29:50.431 | INFO | "pinned" : " 29 GB" | 20260507 17:29:50.431 | INFO | } | 20260507 17:29:50.431 | INFO | Perf report for the last 5.13903 seconds: | 20260507 17:29:50.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:50.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:29:50.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 17:29:50.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:50.431 | INFO | Uploaded 36GiB to GPU in 2.43684s (14.7733GiB/s) | 20260507 17:29:50.432 | INFO | Downloaded 34GiB to GPU in 2.50934s (13.5494GiB/s) | 20260507 17:29:55.570 | INFO | Analyzing chunk frames = [1301504,1302016) | 20260507 17:29:55.570 | INFO | MemInfo: { | 20260507 17:29:55.570 | INFO | "gpu" : " 17 GB", | 20260507 17:29:55.570 | INFO | "huge" : "", | 20260507 17:29:55.570 | INFO | "huge_shared" : " 29 GB", | 20260507 17:29:55.570 | INFO | "malloc" : " 14 GB", | 20260507 17:29:55.570 | INFO | "pinned" : " 29 GB" | 20260507 17:29:55.570 | INFO | } | 20260507 17:29:55.570 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:29:55.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:29:55.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.571 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:29:55.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.571 | INFO | PRE_HQ : 38.0%, 1.0, 19.8ms, 0 | 20260507 17:29:55.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:29:55.571 | INFO | Uploaded 36GiB to GPU in 2.49455s (14.4315GiB/s) | 20260507 17:29:55.571 | INFO | Downloaded 34GiB to GPU in 2.44144s (13.9262GiB/s) | 20260507 17:30:00.709 | INFO | Analyzing chunk frames = [1302016,1302528) | 20260507 17:30:00.709 | INFO | MemInfo: { | 20260507 17:30:00.709 | INFO | "gpu" : " 17 GB", | 20260507 17:30:00.709 | INFO | "huge" : "", | 20260507 17:30:00.709 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:00.710 | INFO | "malloc" : " 14 GB", | 20260507 17:30:00.710 | INFO | "pinned" : " 29 GB" | 20260507 17:30:00.710 | INFO | } | 20260507 17:30:00.710 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:30:00.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:00.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 17:30:00.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 17:30:00.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:00.710 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:30:00.710 | INFO | Uploaded 36GiB to GPU in 2.49167s (14.4481GiB/s) | 20260507 17:30:00.710 | INFO | Total mem usage: 16.4 GiB | 20260507 17:30:00.711 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:30:00.711 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 17:30:00.711 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:30:00.711 | INFO | High Water Breakdown: | 20260507 17:30:00.711 | INFO | 59.2% : 8.63 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:30:00.711 | INFO | 40.7% : 5.92 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:30:00.711 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:30:00.711 | INFO | ------------------------------------------ | 20260507 17:30:00.711 | INFO | Downloaded 34GiB to GPU in 2.42452s (14.0234GiB/s) | 20260507 17:30:00.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:30:00.711 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:30:00.711 | INFO | Total mem usage: 26.5 GiB | 20260507 17:30:00.711 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:30:00.711 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 17:30:00.711 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 17:30:00.711 | INFO | High Water Breakdown: | 20260507 17:30:00.711 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:30:00.711 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:30:00.711 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:30:00.711 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:30:00.711 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:30:00.711 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:30:00.711 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:30:00.711 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:30:00.711 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:30:00.711 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:30:00.711 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:30:00.711 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:30:00.711 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:30:00.711 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:30:00.711 | INFO | ------------------------------------------ | 20260507 17:30:00.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:30:00.711 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:30:00.711 | INFO | Total mem usage: 27.8 GiB | 20260507 17:30:00.711 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:30:00.711 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:30:00.711 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:30:00.711 | INFO | High Water Breakdown: | 20260507 17:30:00.711 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:30:00.711 | INFO | ------------------------------------------ | 20260507 17:30:00.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:30:00.711 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:30:00.711 | INFO | Total mem usage: 15.7 GiB | 20260507 17:30:00.711 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:30:00.711 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:30:00.711 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:30:00.711 | INFO | High Water Breakdown: | 20260507 17:30:00.711 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:30:00.711 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:30:00.711 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:30:00.711 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:30:00.711 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:30:00.711 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:30:00.711 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:30:00.711 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:30:00.711 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:30:00.712 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:30:00.712 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:30:00.712 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:30:00.712 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:30:00.712 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:30:00.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:30:00.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:30:00.712 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:30:00.712 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:30:00.712 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:30:00.712 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:30:00.712 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:30:00.712 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:30:00.712 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:30:00.712 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:30:00.712 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:30:00.712 | INFO | ------------------------------------------ | 20260507 17:30:00.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:30:00.712 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:30:00.712 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:30:00.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:30:00.712 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.62e+01, 1.43e+01 / 2.48e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 0.31%, AggregateStats -- 1.04e-01 / 1.07e-01, 1.25e-02 / 6.06e-03 | 20260507 17:30:00.712 | INFO | PerfReport: 12.12%, Baseline -- 4.06e+00 / 4.51e+00, 6.14e-01 / 2.34e-01 | 20260507 17:30:00.712 | INFO | PerfReport: 3.45%, Histogram -- 1.16e+00 / 1.50e+00, 2.16e-01 / 1.36e-01 | 20260507 17:30:00.712 | INFO | PerfReport: 15.62%, DME -- 4.19e+01 / 4.45e+01, 4.10e+00 / 2.37e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 55.91%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.86e+00 / 2.30e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.54e+00 / 2.87e+00, 4.37e-01 / 2.35e-01 | 20260507 17:30:00.712 | INFO | PerfReport: 3.81%, Metrics -- 1.28e+00 / 1.44e+00, 3.20e-01 / 1.91e-01 | 20260507 17:30:00.712 | INFO | PerfReport: 1.22%, PulseSerializing -- 4.09e-01 / 4.33e-01, 6.53e-02 / 3.92e-02 | 20260507 17:30:00.712 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:30:00.712 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:30:00.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:30:00.712 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 24.04%, Upload -- 2.58e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 24.05%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 31.22%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 1.74%, LockUpload -- 1.87e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 2.34%, LockDownload -- 2.51e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 16.59%, LockCompute -- 1.78e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:30:00.712 | INFO | PerfReport: 0.02%, Misc -- 1.99e-02 / 0.00e+00, 4.11e-02 / 0.00e+00 | 20260507 17:30:00.824 | INFO | Starting a round of Baz IO | 20260507 17:30:04.310 | INFO | Populating RTMetrics report | 20260507 17:30:04.310 | INFO | RTMetrics report populated | 20260507 17:30:04.310 | INFO | Opening tmp RTMetrics json file | 20260507 17:30:04.326 | INFO | Serializing RTMetrics to json | 20260507 17:30:04.327 | INFO | Serialization done, swapping tmp file now... | 20260507 17:30:04.327 | INFO | RTMetrics done | 20260507 17:30:04.393 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.568 seconds | 20260507 17:30:04.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.07% / 0.02% / 51.91% | 20260507 17:30:04.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.58154% / 2.78155e-05% / 3.17891e-05% | 20260507 17:30:04.393 | INFO | Baz IO overall: | 20260507 17:30:04.393 | INFO | BytesWritten : 573697622315 | 20260507 17:30:04.393 | INFO | NumEvents : 277970591334 | 20260507 17:30:05.849 | INFO | Analyzing chunk frames = [1302528,1303040) | 20260507 17:30:05.849 | INFO | MemInfo: { | 20260507 17:30:05.849 | INFO | "gpu" : " 17 GB", | 20260507 17:30:05.849 | INFO | "huge" : "", | 20260507 17:30:05.849 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:05.849 | INFO | "malloc" : " 14 GB", | 20260507 17:30:05.849 | INFO | "pinned" : " 29 GB" | 20260507 17:30:05.849 | INFO | } | 20260507 17:30:05.849 | INFO | Perf report for the last 5.13885 seconds: | 20260507 17:30:05.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:05.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:05.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:05.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:05.849 | INFO | ANALYSIS : 69.6%, 3.0, 108.2ms, 0 | 20260507 17:30:05.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:05.849 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260507 17:30:05.849 | INFO | PRE_HQ : 67.9%, 1.9, 66.4ms, 0 | 20260507 17:30:05.849 | INFO | BAZWRITER : 68.1%, 3.0, 106.6ms, 0 | 20260507 17:30:05.849 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:30:05.850 | INFO | Uploaded 36GiB to GPU in 2.54085s (14.1685GiB/s) | 20260507 17:30:05.850 | INFO | Downloaded 38GiB to GPU in 2.73241s (13.9072GiB/s) | 20260507 17:30:10.988 | INFO | Analyzing chunk frames = [1303040,1303552) | 20260507 17:30:10.988 | INFO | MemInfo: { | 20260507 17:30:10.988 | INFO | "gpu" : " 17 GB", | 20260507 17:30:10.988 | INFO | "huge" : "", | 20260507 17:30:10.988 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:10.988 | INFO | "malloc" : " 14 GB", | 20260507 17:30:10.988 | INFO | "pinned" : " 29 GB" | 20260507 17:30:10.988 | INFO | } | 20260507 17:30:10.988 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:30:10.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:10.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.988 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:30:10.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.988 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:30:10.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:10.989 | INFO | Uploaded 36GiB to GPU in 2.53223s (14.2167GiB/s) | 20260507 17:30:10.989 | INFO | Downloaded 34GiB to GPU in 2.45856s (13.8292GiB/s) | 20260507 17:30:16.127 | INFO | Analyzing chunk frames = [1303552,1304064) | 20260507 17:30:16.127 | INFO | MemInfo: { | 20260507 17:30:16.127 | INFO | "gpu" : " 17 GB", | 20260507 17:30:16.127 | INFO | "huge" : "", | 20260507 17:30:16.127 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:16.127 | INFO | "malloc" : " 14 GB", | 20260507 17:30:16.127 | INFO | "pinned" : " 29 GB" | 20260507 17:30:16.127 | INFO | } | 20260507 17:30:16.128 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:30:16.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:16.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260507 17:30:16.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:30:16.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:16.128 | INFO | Uploaded 36GiB to GPU in 2.55213s (14.1059GiB/s) | 20260507 17:30:16.128 | INFO | Downloaded 34GiB to GPU in 2.42587s (14.0156GiB/s) | 20260507 17:30:21.267 | INFO | Analyzing chunk frames = [1304064,1304576) | 20260507 17:30:21.267 | INFO | MemInfo: { | 20260507 17:30:21.267 | INFO | "gpu" : " 17 GB", | 20260507 17:30:21.267 | INFO | "huge" : "", | 20260507 17:30:21.267 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:21.267 | INFO | "malloc" : " 14 GB", | 20260507 17:30:21.267 | INFO | "pinned" : " 29 GB" | 20260507 17:30:21.267 | INFO | } | 20260507 17:30:21.267 | INFO | Perf report for the last 5.13976 seconds: | 20260507 17:30:21.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:21.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.267 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:30:21.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.267 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:30:21.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:21.268 | INFO | Uploaded 36GiB to GPU in 2.47409s (14.5508GiB/s) | 20260507 17:30:21.268 | INFO | Downloaded 34GiB to GPU in 2.35061s (14.4643GiB/s) | 20260507 17:30:26.407 | INFO | Analyzing chunk frames = [1304576,1305088) | 20260507 17:30:26.407 | INFO | MemInfo: { | 20260507 17:30:26.407 | INFO | "gpu" : " 17 GB", | 20260507 17:30:26.407 | INFO | "huge" : "", | 20260507 17:30:26.407 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:26.407 | INFO | "malloc" : " 14 GB", | 20260507 17:30:26.407 | INFO | "pinned" : " 29 GB" | 20260507 17:30:26.407 | INFO | } | 20260507 17:30:26.407 | INFO | Perf report for the last 5.14002 seconds: | 20260507 17:30:26.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:26.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.407 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 17:30:26.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.408 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 17:30:26.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:26.408 | INFO | Uploaded 36GiB to GPU in 2.32753s (15.467GiB/s) | 20260507 17:30:26.408 | INFO | Downloaded 34GiB to GPU in 2.47138s (13.7575GiB/s) | 20260507 17:30:31.545 | INFO | Analyzing chunk frames = [1305088,1305600) | 20260507 17:30:31.545 | INFO | MemInfo: { | 20260507 17:30:31.545 | INFO | "gpu" : " 17 GB", | 20260507 17:30:31.545 | INFO | "huge" : "", | 20260507 17:30:31.545 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:31.545 | INFO | "malloc" : " 14 GB", | 20260507 17:30:31.546 | INFO | "pinned" : " 29 GB" | 20260507 17:30:31.546 | INFO | } | 20260507 17:30:31.546 | INFO | Perf report for the last 5.13826 seconds: | 20260507 17:30:31.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:31.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:30:31.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 17:30:31.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:31.546 | INFO | Uploaded 36GiB to GPU in 2.40458s (14.9714GiB/s) | 20260507 17:30:31.546 | INFO | Downloaded 34GiB to GPU in 2.52867s (13.4458GiB/s) | 20260507 17:30:36.685 | INFO | Analyzing chunk frames = [1305600,1306112) | 20260507 17:30:36.685 | INFO | MemInfo: { | 20260507 17:30:36.685 | INFO | "gpu" : " 17 GB", | 20260507 17:30:36.685 | INFO | "huge" : "", | 20260507 17:30:36.685 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:36.685 | INFO | "malloc" : " 14 GB", | 20260507 17:30:36.685 | INFO | "pinned" : " 29 GB" | 20260507 17:30:36.685 | INFO | } | 20260507 17:30:36.685 | INFO | Perf report for the last 5.13979 seconds: | 20260507 17:30:36.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:36.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:30:36.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | PRE_HQ : 38.3%, 1.0, 19.9ms, 0 | 20260507 17:30:36.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:36.686 | INFO | Uploaded 36GiB to GPU in 2.4556s (14.6604GiB/s) | 20260507 17:30:36.686 | INFO | Downloaded 34GiB to GPU in 2.45485s (13.8501GiB/s) | 20260507 17:30:41.824 | INFO | Analyzing chunk frames = [1306112,1306624) | 20260507 17:30:41.824 | INFO | MemInfo: { | 20260507 17:30:41.824 | INFO | "gpu" : " 17 GB", | 20260507 17:30:41.824 | INFO | "huge" : "", | 20260507 17:30:41.824 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:41.824 | INFO | "malloc" : " 14 GB", | 20260507 17:30:41.825 | INFO | "pinned" : " 29 GB" | 20260507 17:30:41.825 | INFO | } | 20260507 17:30:41.825 | INFO | Perf report for the last 5.13925 seconds: | 20260507 17:30:41.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:41.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:30:41.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 17:30:41.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:41.825 | INFO | Uploaded 36GiB to GPU in 2.42535s (14.8432GiB/s) | 20260507 17:30:41.825 | INFO | Downloaded 34GiB to GPU in 2.43887s (13.9409GiB/s) | 20260507 17:30:41.935 | INFO | Starting a round of Baz IO | 20260507 17:30:45.447 | INFO | Populating RTMetrics report | 20260507 17:30:45.447 | INFO | RTMetrics report populated | 20260507 17:30:45.447 | INFO | Opening tmp RTMetrics json file | 20260507 17:30:45.462 | INFO | Serializing RTMetrics to json | 20260507 17:30:45.463 | INFO | Serialization done, swapping tmp file now... | 20260507 17:30:45.464 | INFO | RTMetrics done | 20260507 17:30:45.530 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.594 seconds | 20260507 17:30:45.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.05% / 0.02% / 51.93% | 20260507 17:30:45.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.56341% / 3.57628e-05% / 3.97364e-05% | 20260507 17:30:45.530 | INFO | Baz IO overall: | 20260507 17:30:45.530 | INFO | BytesWritten : 575630359323 | 20260507 17:30:45.530 | INFO | NumEvents : 278909757661 | 20260507 17:30:46.964 | INFO | Analyzing chunk frames = [1306624,1307136) | 20260507 17:30:46.964 | INFO | MemInfo: { | 20260507 17:30:46.964 | INFO | "gpu" : " 17 GB", | 20260507 17:30:46.964 | INFO | "huge" : "", | 20260507 17:30:46.964 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:46.964 | INFO | "malloc" : " 14 GB", | 20260507 17:30:46.964 | INFO | "pinned" : " 29 GB" | 20260507 17:30:46.964 | INFO | } | 20260507 17:30:46.964 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:30:46.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:46.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:46.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:46.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:46.964 | INFO | ANALYSIS : 69.5%, 3.0, 108.5ms, 0 | 20260507 17:30:46.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:46.964 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 17:30:46.964 | INFO | PRE_HQ : 67.8%, 1.8, 64.8ms, 0 | 20260507 17:30:46.964 | INFO | BAZWRITER : 68.8%, 3.2, 114.8ms, 0 | 20260507 17:30:46.964 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:30:46.964 | INFO | Uploaded 36GiB to GPU in 2.49018s (14.4568GiB/s) | 20260507 17:30:46.964 | INFO | Downloaded 38GiB to GPU in 2.73408s (13.8986GiB/s) | 20260507 17:30:52.103 | INFO | Analyzing chunk frames = [1307136,1307648) | 20260507 17:30:52.103 | INFO | MemInfo: { | 20260507 17:30:52.103 | INFO | "gpu" : " 17 GB", | 20260507 17:30:52.103 | INFO | "huge" : "", | 20260507 17:30:52.103 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:52.103 | INFO | "malloc" : " 14 GB", | 20260507 17:30:52.103 | INFO | "pinned" : " 29 GB" | 20260507 17:30:52.103 | INFO | } | 20260507 17:30:52.103 | INFO | Perf report for the last 5.139 seconds: | 20260507 17:30:52.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:52.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.103 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:30:52.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.103 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:30:52.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:52.104 | INFO | Uploaded 36GiB to GPU in 2.50807s (14.3537GiB/s) | 20260507 17:30:52.104 | INFO | Downloaded 34GiB to GPU in 2.44203s (13.9228GiB/s) | 20260507 17:30:57.242 | INFO | Analyzing chunk frames = [1307648,1308160) | 20260507 17:30:57.242 | INFO | MemInfo: { | 20260507 17:30:57.242 | INFO | "gpu" : " 17 GB", | 20260507 17:30:57.242 | INFO | "huge" : "", | 20260507 17:30:57.242 | INFO | "huge_shared" : " 29 GB", | 20260507 17:30:57.242 | INFO | "malloc" : " 14 GB", | 20260507 17:30:57.242 | INFO | "pinned" : " 29 GB" | 20260507 17:30:57.242 | INFO | } | 20260507 17:30:57.242 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:30:57.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:30:57.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:30:57.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:30:57.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:30:57.243 | INFO | Uploaded 36GiB to GPU in 2.55541s (14.0878GiB/s) | 20260507 17:30:57.243 | INFO | Downloaded 34GiB to GPU in 2.42017s (14.0486GiB/s) | 20260507 17:31:02.381 | INFO | Analyzing chunk frames = [1308160,1308672) | 20260507 17:31:02.381 | INFO | MemInfo: { | 20260507 17:31:02.381 | INFO | "gpu" : " 17 GB", | 20260507 17:31:02.381 | INFO | "huge" : "", | 20260507 17:31:02.381 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:02.381 | INFO | "malloc" : " 14 GB", | 20260507 17:31:02.382 | INFO | "pinned" : " 29 GB" | 20260507 17:31:02.382 | INFO | } | 20260507 17:31:02.382 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:31:02.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:02.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:31:02.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 17:31:02.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:02.382 | INFO | Uploaded 36GiB to GPU in 2.47508s (14.545GiB/s) | 20260507 17:31:02.382 | INFO | Downloaded 34GiB to GPU in 2.34757s (14.4831GiB/s) | 20260507 17:31:07.521 | INFO | Analyzing chunk frames = [1308672,1309184) | 20260507 17:31:07.521 | INFO | MemInfo: { | 20260507 17:31:07.521 | INFO | "gpu" : " 17 GB", | 20260507 17:31:07.521 | INFO | "huge" : "", | 20260507 17:31:07.521 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:07.521 | INFO | "malloc" : " 14 GB", | 20260507 17:31:07.521 | INFO | "pinned" : " 29 GB" | 20260507 17:31:07.521 | INFO | } | 20260507 17:31:07.521 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:31:07.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:07.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.521 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 17:31:07.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.521 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:31:07.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:07.522 | INFO | Uploaded 36GiB to GPU in 2.31355s (15.5605GiB/s) | 20260507 17:31:07.522 | INFO | Downloaded 34GiB to GPU in 2.454s (13.855GiB/s) | 20260507 17:31:12.661 | INFO | Analyzing chunk frames = [1309184,1309696) | 20260507 17:31:12.661 | INFO | MemInfo: { | 20260507 17:31:12.661 | INFO | "gpu" : " 17 GB", | 20260507 17:31:12.661 | INFO | "huge" : "", | 20260507 17:31:12.661 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:12.661 | INFO | "malloc" : " 14 GB", | 20260507 17:31:12.661 | INFO | "pinned" : " 29 GB" | 20260507 17:31:12.661 | INFO | } | 20260507 17:31:12.661 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:31:12.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:12.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.661 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:31:12.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.661 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:31:12.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:12.662 | INFO | Uploaded 36GiB to GPU in 2.40968s (14.9397GiB/s) | 20260507 17:31:12.662 | INFO | Downloaded 34GiB to GPU in 2.4938s (13.6338GiB/s) | 20260507 17:31:17.800 | INFO | Analyzing chunk frames = [1309696,1310208) | 20260507 17:31:17.800 | INFO | MemInfo: { | 20260507 17:31:17.800 | INFO | "gpu" : " 17 GB", | 20260507 17:31:17.800 | INFO | "huge" : "", | 20260507 17:31:17.800 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:17.800 | INFO | "malloc" : " 14 GB", | 20260507 17:31:17.800 | INFO | "pinned" : " 29 GB" | 20260507 17:31:17.800 | INFO | } | 20260507 17:31:17.800 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:31:17.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:17.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.800 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:31:17.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.800 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:31:17.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:17.801 | INFO | Uploaded 36GiB to GPU in 2.47755s (14.5305GiB/s) | 20260507 17:31:17.801 | INFO | Downloaded 34GiB to GPU in 2.43524s (13.9617GiB/s) | 20260507 17:31:22.940 | INFO | Analyzing chunk frames = [1310208,1310720) | 20260507 17:31:22.940 | INFO | MemInfo: { | 20260507 17:31:22.940 | INFO | "gpu" : " 17 GB", | 20260507 17:31:22.940 | INFO | "huge" : "", | 20260507 17:31:22.940 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:22.940 | INFO | "malloc" : " 14 GB", | 20260507 17:31:22.940 | INFO | "pinned" : " 29 GB" | 20260507 17:31:22.940 | INFO | } | 20260507 17:31:22.940 | INFO | Perf report for the last 5.14002 seconds: | 20260507 17:31:22.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:22.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.940 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:31:22.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.940 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260507 17:31:22.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:22.941 | INFO | Uploaded 36GiB to GPU in 2.46411s (14.6097GiB/s) | 20260507 17:31:22.941 | INFO | Downloaded 34GiB to GPU in 2.42061s (14.046GiB/s) | 20260507 17:31:22.941 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:31:22.941 | INFO | Total mem usage: 16.4 GiB | 20260507 17:31:22.941 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:31:22.941 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11.2%) | 20260507 17:31:22.941 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:31:22.941 | INFO | High Water Breakdown: | 20260507 17:31:22.941 | INFO | 59.4% : 8.76 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:31:22.941 | INFO | 40.5% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:31:22.941 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:31:22.941 | INFO | ------------------------------------------ | 20260507 17:31:22.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:31:22.941 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:31:22.941 | INFO | Total mem usage: 26.5 GiB | 20260507 17:31:22.941 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:31:22.941 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 17:31:22.941 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 17:31:22.941 | INFO | High Water Breakdown: | 20260507 17:31:22.941 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:31:22.941 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:31:22.941 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:31:22.941 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:31:22.941 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:31:22.941 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:31:22.941 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:31:22.941 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:31:22.941 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:31:22.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:31:22.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:31:22.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:31:22.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:31:22.941 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:31:22.941 | INFO | ------------------------------------------ | 20260507 17:31:22.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:31:22.941 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:31:22.941 | INFO | Total mem usage: 27.8 GiB | 20260507 17:31:22.941 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:31:22.941 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:31:22.941 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:31:22.941 | INFO | High Water Breakdown: | 20260507 17:31:22.941 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:31:22.941 | INFO | ------------------------------------------ | 20260507 17:31:22.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:31:22.941 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:31:22.941 | INFO | Total mem usage: 15.7 GiB | 20260507 17:31:22.941 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:31:22.941 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:31:22.941 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:31:22.941 | INFO | High Water Breakdown: | 20260507 17:31:22.941 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:31:22.941 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:31:22.941 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:31:22.941 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:31:22.941 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:31:22.941 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:31:22.941 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:31:22.941 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:31:22.941 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:31:22.941 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:31:22.941 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:31:22.941 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:31:22.941 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:31:22.941 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:31:22.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:31:22.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:31:22.941 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:31:22.942 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:31:22.942 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:31:22.942 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:31:22.942 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:31:22.942 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:31:22.942 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:31:22.942 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:31:22.942 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:31:22.942 | INFO | ------------------------------------------ | 20260507 17:31:22.942 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:31:22.942 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:31:22.942 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:31:22.942 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:31:22.942 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.44e+01 / 2.85e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 0.30%, AggregateStats -- 9.99e-02 / 1.04e-01, 1.11e-02 / 6.31e-03 | 20260507 17:31:22.942 | INFO | PerfReport: 12.06%, Baseline -- 4.04e+00 / 4.52e+00, 6.17e-01 / 2.41e-01 | 20260507 17:31:22.942 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.24e-01 | 20260507 17:31:22.942 | INFO | PerfReport: 15.66%, DME -- 4.20e+01 / 4.47e+01, 4.04e+00 / 2.47e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.91e+00 / 2.18e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.89e+00, 4.35e-01 / 2.40e-01 | 20260507 17:31:22.942 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.44e+00, 3.24e-01 / 1.97e-01 | 20260507 17:31:22.942 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.02e-01 / 4.24e-01, 6.60e-02 / 3.97e-02 | 20260507 17:31:22.942 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:31:22.942 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:31:22.942 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:31:22.942 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 23.88%, Upload -- 2.57e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 24.00%, Download -- 2.58e+01 / 0.00e+00, 8.64e+00 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 31.22%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 1.67%, LockUpload -- 1.79e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 2.48%, LockDownload -- 2.66e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 16.73%, LockCompute -- 1.80e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:31:22.942 | INFO | PerfReport: 0.02%, Misc -- 1.65e-02 / 0.00e+00, 2.97e-02 / 0.00e+00 | 20260507 17:31:23.040 | INFO | Starting a round of Baz IO | 20260507 17:31:26.541 | INFO | Populating RTMetrics report | 20260507 17:31:26.541 | INFO | RTMetrics report populated | 20260507 17:31:26.541 | INFO | Opening tmp RTMetrics json file | 20260507 17:31:26.558 | INFO | Serializing RTMetrics to json | 20260507 17:31:26.559 | INFO | Serialization done, swapping tmp file now... | 20260507 17:31:26.559 | INFO | RTMetrics done | 20260507 17:31:26.623 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.583 seconds | 20260507 17:31:26.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.02% / 0.02% / 51.95% | 20260507 17:31:26.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.55862% / 4.76837e-05% / 4.76837e-05% | 20260507 17:31:26.623 | INFO | Baz IO overall: | 20260507 17:31:26.623 | INFO | BytesWritten : 577564140335 | 20260507 17:31:26.623 | INFO | NumEvents : 279849780342 | 20260507 17:31:28.082 | INFO | Analyzing chunk frames = [1310720,1311232) | 20260507 17:31:28.083 | INFO | MemInfo: { | 20260507 17:31:28.083 | INFO | "gpu" : " 17 GB", | 20260507 17:31:28.083 | INFO | "huge" : "", | 20260507 17:31:28.083 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:28.083 | INFO | "malloc" : " 14 GB", | 20260507 17:31:28.083 | INFO | "pinned" : " 29 GB" | 20260507 17:31:28.083 | INFO | } | 20260507 17:31:28.083 | INFO | Perf report for the last 5.14222 seconds: | 20260507 17:31:28.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:28.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:28.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:28.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:28.083 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 17:31:28.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:28.083 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 17:31:28.083 | INFO | PRE_HQ : 66.5%, 1.7, 59.9ms, 0 | 20260507 17:31:28.083 | INFO | BAZWRITER : 68.7%, 2.9, 101.2ms, 0 | 20260507 17:31:28.083 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:31:28.083 | INFO | Uploaded 36GiB to GPU in 2.53133s (14.2218GiB/s) | 20260507 17:31:28.083 | INFO | Downloaded 38GiB to GPU in 2.72095s (13.9657GiB/s) | 20260507 17:31:33.218 | INFO | Analyzing chunk frames = [1311232,1311744) | 20260507 17:31:33.218 | INFO | MemInfo: { | 20260507 17:31:33.218 | INFO | "gpu" : " 17 GB", | 20260507 17:31:33.218 | INFO | "huge" : "", | 20260507 17:31:33.218 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:33.218 | INFO | "malloc" : " 14 GB", | 20260507 17:31:33.218 | INFO | "pinned" : " 29 GB" | 20260507 17:31:33.218 | INFO | } | 20260507 17:31:33.218 | INFO | Perf report for the last 5.13518 seconds: | 20260507 17:31:33.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:33.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:31:33.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:31:33.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:33.218 | INFO | Uploaded 36GiB to GPU in 2.47096s (14.5692GiB/s) | 20260507 17:31:33.219 | INFO | Downloaded 34GiB to GPU in 2.42972s (13.9934GiB/s) | 20260507 17:31:38.357 | INFO | Analyzing chunk frames = [1311744,1312256) | 20260507 17:31:38.357 | INFO | MemInfo: { | 20260507 17:31:38.357 | INFO | "gpu" : " 17 GB", | 20260507 17:31:38.357 | INFO | "huge" : "", | 20260507 17:31:38.357 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:38.357 | INFO | "malloc" : " 14 GB", | 20260507 17:31:38.357 | INFO | "pinned" : " 29 GB" | 20260507 17:31:38.357 | INFO | } | 20260507 17:31:38.357 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:31:38.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:38.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.357 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:31:38.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.358 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:31:38.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:38.358 | INFO | Uploaded 36GiB to GPU in 2.5601s (14.062GiB/s) | 20260507 17:31:38.358 | INFO | Downloaded 34GiB to GPU in 2.43039s (13.9895GiB/s) | 20260507 17:31:43.496 | INFO | Analyzing chunk frames = [1312256,1312768) | 20260507 17:31:43.496 | INFO | MemInfo: { | 20260507 17:31:43.496 | INFO | "gpu" : " 17 GB", | 20260507 17:31:43.496 | INFO | "huge" : "", | 20260507 17:31:43.496 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:43.496 | INFO | "malloc" : " 14 GB", | 20260507 17:31:43.496 | INFO | "pinned" : " 29 GB" | 20260507 17:31:43.496 | INFO | } | 20260507 17:31:43.496 | INFO | Perf report for the last 5.13906 seconds: | 20260507 17:31:43.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:43.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:31:43.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:31:43.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:43.497 | INFO | Uploaded 36GiB to GPU in 2.47101s (14.5689GiB/s) | 20260507 17:31:43.497 | INFO | Downloaded 34GiB to GPU in 2.36045s (14.404GiB/s) | 20260507 17:31:48.636 | INFO | Analyzing chunk frames = [1312768,1313280) | 20260507 17:31:48.636 | INFO | MemInfo: { | 20260507 17:31:48.636 | INFO | "gpu" : " 17 GB", | 20260507 17:31:48.636 | INFO | "huge" : "", | 20260507 17:31:48.637 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:48.637 | INFO | "malloc" : " 14 GB", | 20260507 17:31:48.637 | INFO | "pinned" : " 29 GB" | 20260507 17:31:48.637 | INFO | } | 20260507 17:31:48.637 | INFO | Perf report for the last 5.14016 seconds: | 20260507 17:31:48.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:48.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:31:48.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 17:31:48.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:48.637 | INFO | Uploaded 36GiB to GPU in 2.36767s (15.2048GiB/s) | 20260507 17:31:48.637 | INFO | Downloaded 34GiB to GPU in 2.47708s (13.7258GiB/s) | 20260507 17:31:53.775 | INFO | Analyzing chunk frames = [1313280,1313792) | 20260507 17:31:53.775 | INFO | MemInfo: { | 20260507 17:31:53.775 | INFO | "gpu" : " 17 GB", | 20260507 17:31:53.775 | INFO | "huge" : "", | 20260507 17:31:53.776 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:53.776 | INFO | "malloc" : " 14 GB", | 20260507 17:31:53.776 | INFO | "pinned" : " 29 GB" | 20260507 17:31:53.776 | INFO | } | 20260507 17:31:53.776 | INFO | Perf report for the last 5.13899 seconds: | 20260507 17:31:53.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:53.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:31:53.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:31:53.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:53.776 | INFO | Uploaded 36GiB to GPU in 2.41132s (14.9296GiB/s) | 20260507 17:31:53.776 | INFO | Downloaded 34GiB to GPU in 2.48318s (13.6921GiB/s) | 20260507 17:31:58.915 | INFO | Analyzing chunk frames = [1313792,1314304) | 20260507 17:31:58.915 | INFO | MemInfo: { | 20260507 17:31:58.915 | INFO | "gpu" : " 17 GB", | 20260507 17:31:58.915 | INFO | "huge" : "", | 20260507 17:31:58.915 | INFO | "huge_shared" : " 29 GB", | 20260507 17:31:58.915 | INFO | "malloc" : " 14 GB", | 20260507 17:31:58.915 | INFO | "pinned" : " 29 GB" | 20260507 17:31:58.915 | INFO | } | 20260507 17:31:58.915 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:31:58.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:31:58.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.915 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:31:58.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.915 | INFO | PRE_HQ : 35.9%, 1.0, 18.7ms, 0 | 20260507 17:31:58.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:31:58.917 | INFO | Uploaded 36GiB to GPU in 2.48765s (14.4715GiB/s) | 20260507 17:31:58.917 | INFO | Downloaded 34GiB to GPU in 2.43179s (13.9815GiB/s) | 20260507 17:32:04.054 | INFO | Analyzing chunk frames = [1314304,1314816) | 20260507 17:32:04.054 | INFO | MemInfo: { | 20260507 17:32:04.054 | INFO | "gpu" : " 17 GB", | 20260507 17:32:04.054 | INFO | "huge" : "", | 20260507 17:32:04.054 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:04.054 | INFO | "malloc" : " 14 GB", | 20260507 17:32:04.054 | INFO | "pinned" : " 29 GB" | 20260507 17:32:04.054 | INFO | } | 20260507 17:32:04.054 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:32:04.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:04.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.054 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:32:04.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.054 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:32:04.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:04.055 | INFO | Uploaded 36GiB to GPU in 2.51646s (14.3058GiB/s) | 20260507 17:32:04.055 | INFO | Downloaded 34GiB to GPU in 2.5047s (13.5745GiB/s) | 20260507 17:32:04.151 | INFO | Starting a round of Baz IO | 20260507 17:32:07.662 | INFO | Populating RTMetrics report | 20260507 17:32:07.662 | INFO | RTMetrics report populated | 20260507 17:32:07.662 | INFO | Opening tmp RTMetrics json file | 20260507 17:32:07.669 | INFO | Serializing RTMetrics to json | 20260507 17:32:07.682 | INFO | Serialization done, swapping tmp file now... | 20260507 17:32:07.682 | INFO | RTMetrics done | 20260507 17:32:07.740 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.589 seconds | 20260507 17:32:07.740 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.00% / 0.02% / 51.98% | 20260507 17:32:07.740 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.54565% / 5.5631e-05% / 3.17891e-05% | 20260507 17:32:07.740 | INFO | Baz IO overall: | 20260507 17:32:07.740 | INFO | BytesWritten : 579496913839 | 20260507 17:32:07.740 | INFO | NumEvents : 280789087658 | 20260507 17:32:09.193 | INFO | Analyzing chunk frames = [1314816,1315328) | 20260507 17:32:09.193 | INFO | MemInfo: { | 20260507 17:32:09.193 | INFO | "gpu" : " 17 GB", | 20260507 17:32:09.193 | INFO | "huge" : "", | 20260507 17:32:09.193 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:09.193 | INFO | "malloc" : " 14 GB", | 20260507 17:32:09.193 | INFO | "pinned" : " 29 GB" | 20260507 17:32:09.193 | INFO | } | 20260507 17:32:09.193 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:32:09.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:09.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:09.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:09.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:09.193 | INFO | ANALYSIS : 69.4%, 3.0, 108.5ms, 0 | 20260507 17:32:09.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:09.193 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 17:32:09.193 | INFO | PRE_HQ : 66.8%, 1.7, 59.9ms, 0 | 20260507 17:32:09.193 | INFO | BAZWRITER : 69.0%, 2.9, 101.6ms, 0 | 20260507 17:32:09.193 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:32:09.194 | INFO | Uploaded 36GiB to GPU in 2.4909s (14.4526GiB/s) | 20260507 17:32:09.194 | INFO | Downloaded 38GiB to GPU in 2.72295s (13.9555GiB/s) | 20260507 17:32:14.333 | INFO | Analyzing chunk frames = [1315328,1315840) | 20260507 17:32:14.333 | INFO | MemInfo: { | 20260507 17:32:14.333 | INFO | "gpu" : " 17 GB", | 20260507 17:32:14.333 | INFO | "huge" : "", | 20260507 17:32:14.333 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:14.333 | INFO | "malloc" : " 14 GB", | 20260507 17:32:14.333 | INFO | "pinned" : " 29 GB" | 20260507 17:32:14.333 | INFO | } | 20260507 17:32:14.333 | INFO | Perf report for the last 5.13974 seconds: | 20260507 17:32:14.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:14.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.333 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 17:32:14.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.333 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:32:14.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:14.334 | INFO | Uploaded 36GiB to GPU in 2.44722s (14.7106GiB/s) | 20260507 17:32:14.334 | INFO | Downloaded 34GiB to GPU in 2.43091s (13.9865GiB/s) | 20260507 17:32:19.472 | INFO | Analyzing chunk frames = [1315840,1316352) | 20260507 17:32:19.472 | INFO | MemInfo: { | 20260507 17:32:19.472 | INFO | "gpu" : " 17 GB", | 20260507 17:32:19.472 | INFO | "huge" : "", | 20260507 17:32:19.472 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:19.472 | INFO | "malloc" : " 14 GB", | 20260507 17:32:19.472 | INFO | "pinned" : " 29 GB" | 20260507 17:32:19.472 | INFO | } | 20260507 17:32:19.472 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:32:19.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:19.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.472 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 17:32:19.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.472 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:32:19.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:19.473 | INFO | Uploaded 36GiB to GPU in 2.50375s (14.3785GiB/s) | 20260507 17:32:19.473 | INFO | Downloaded 34GiB to GPU in 2.4459s (13.9008GiB/s) | 20260507 17:32:24.611 | INFO | Analyzing chunk frames = [1316352,1316864) | 20260507 17:32:24.611 | INFO | MemInfo: { | 20260507 17:32:24.611 | INFO | "gpu" : " 17 GB", | 20260507 17:32:24.611 | INFO | "huge" : "", | 20260507 17:32:24.611 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:24.611 | INFO | "malloc" : " 14 GB", | 20260507 17:32:24.612 | INFO | "pinned" : " 29 GB" | 20260507 17:32:24.612 | INFO | } | 20260507 17:32:24.612 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:32:24.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:24.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 17:32:24.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:32:24.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:24.612 | INFO | Uploaded 36GiB to GPU in 2.44463s (14.7262GiB/s) | 20260507 17:32:24.612 | INFO | Downloaded 34GiB to GPU in 2.36241s (14.3921GiB/s) | 20260507 17:32:29.751 | INFO | Analyzing chunk frames = [1316864,1317376) | 20260507 17:32:29.751 | INFO | MemInfo: { | 20260507 17:32:29.751 | INFO | "gpu" : " 17 GB", | 20260507 17:32:29.751 | INFO | "huge" : "", | 20260507 17:32:29.751 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:29.751 | INFO | "malloc" : " 14 GB", | 20260507 17:32:29.751 | INFO | "pinned" : " 29 GB" | 20260507 17:32:29.751 | INFO | } | 20260507 17:32:29.751 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:32:29.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:29.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.751 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:32:29.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.751 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 17:32:29.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:29.752 | INFO | Uploaded 36GiB to GPU in 2.31847s (15.5275GiB/s) | 20260507 17:32:29.752 | INFO | Downloaded 34GiB to GPU in 2.47389s (13.7435GiB/s) | 20260507 17:32:34.890 | INFO | Analyzing chunk frames = [1317376,1317888) | 20260507 17:32:34.890 | INFO | MemInfo: { | 20260507 17:32:34.890 | INFO | "gpu" : " 17 GB", | 20260507 17:32:34.890 | INFO | "huge" : "", | 20260507 17:32:34.890 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:34.890 | INFO | "malloc" : " 14 GB", | 20260507 17:32:34.890 | INFO | "pinned" : " 29 GB" | 20260507 17:32:34.890 | INFO | } | 20260507 17:32:34.891 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:32:34.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:34.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:32:34.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 17:32:34.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:34.891 | INFO | Uploaded 36GiB to GPU in 2.31847s (15.5275GiB/s) | 20260507 17:32:34.891 | INFO | Downloaded 34GiB to GPU in 2.47829s (13.7191GiB/s) | 20260507 17:32:40.029 | INFO | Analyzing chunk frames = [1317888,1318400) | 20260507 17:32:40.029 | INFO | MemInfo: { | 20260507 17:32:40.029 | INFO | "gpu" : " 17 GB", | 20260507 17:32:40.029 | INFO | "huge" : "", | 20260507 17:32:40.029 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:40.029 | INFO | "malloc" : " 14 GB", | 20260507 17:32:40.029 | INFO | "pinned" : " 29 GB" | 20260507 17:32:40.030 | INFO | } | 20260507 17:32:40.030 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:32:40.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:40.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 17:32:40.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:32:40.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:40.030 | INFO | Uploaded 36GiB to GPU in 2.40807s (14.9497GiB/s) | 20260507 17:32:40.030 | INFO | Downloaded 34GiB to GPU in 2.4375s (13.9487GiB/s) | 20260507 17:32:45.169 | INFO | Analyzing chunk frames = [1318400,1318912) | 20260507 17:32:45.169 | INFO | MemInfo: { | 20260507 17:32:45.169 | INFO | "gpu" : " 17 GB", | 20260507 17:32:45.169 | INFO | "huge" : "", | 20260507 17:32:45.169 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:45.169 | INFO | "malloc" : " 14 GB", | 20260507 17:32:45.169 | INFO | "pinned" : " 29 GB" | 20260507 17:32:45.169 | INFO | } | 20260507 17:32:45.170 | INFO | Perf report for the last 5.13995 seconds: | 20260507 17:32:45.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:45.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:32:45.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:32:45.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:45.170 | INFO | Uploaded 36GiB to GPU in 2.47677s (14.5351GiB/s) | 20260507 17:32:45.170 | INFO | Downloaded 34GiB to GPU in 2.51424s (13.5229GiB/s) | 20260507 17:32:45.170 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:32:45.170 | INFO | Total mem usage: 16.4 GiB | 20260507 17:32:45.170 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:32:45.170 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.9%) | 20260507 17:32:45.170 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:32:45.170 | INFO | High Water Breakdown: | 20260507 17:32:45.170 | INFO | 59.1% : 8.51 GiB ( 85.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:32:45.170 | INFO | 40.8% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:32:45.170 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:32:45.170 | INFO | ------------------------------------------ | 20260507 17:32:45.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:32:45.170 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:32:45.170 | INFO | Total mem usage: 26.5 GiB | 20260507 17:32:45.170 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:32:45.170 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:32:45.170 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:32:45.170 | INFO | High Water Breakdown: | 20260507 17:32:45.170 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:32:45.170 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:32:45.170 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:32:45.170 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:32:45.170 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:32:45.170 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:32:45.170 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:32:45.170 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:32:45.170 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:32:45.170 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:32:45.170 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:32:45.170 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:32:45.170 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:32:45.170 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:32:45.171 | INFO | ------------------------------------------ | 20260507 17:32:45.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:32:45.171 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:32:45.171 | INFO | Total mem usage: 27.8 GiB | 20260507 17:32:45.171 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:32:45.171 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:32:45.171 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:32:45.171 | INFO | High Water Breakdown: | 20260507 17:32:45.171 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:32:45.171 | INFO | ------------------------------------------ | 20260507 17:32:45.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:32:45.171 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:32:45.171 | INFO | Total mem usage: 15.7 GiB | 20260507 17:32:45.171 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:32:45.171 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:32:45.171 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:32:45.171 | INFO | High Water Breakdown: | 20260507 17:32:45.171 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:32:45.171 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:32:45.171 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:32:45.171 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:32:45.171 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:32:45.171 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:32:45.171 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:32:45.171 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:32:45.171 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:32:45.171 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:32:45.171 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:32:45.171 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:32:45.171 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:32:45.171 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:32:45.171 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:32:45.171 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:32:45.171 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:32:45.171 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:32:45.171 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:32:45.171 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:32:45.171 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:32:45.171 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:32:45.171 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:32:45.171 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:32:45.171 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:32:45.171 | INFO | ------------------------------------------ | 20260507 17:32:45.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:32:45.171 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:32:45.171 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:32:45.171 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:32:45.171 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.68e+01, 1.43e+01 / 2.79e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.12e-02 / 5.82e-03 | 20260507 17:32:45.171 | INFO | PerfReport: 12.06%, Baseline -- 4.04e+00 / 4.52e+00, 6.22e-01 / 2.02e-01 | 20260507 17:32:45.171 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 9.52e-02 | 20260507 17:32:45.171 | INFO | PerfReport: 15.76%, DME -- 4.23e+01 / 4.52e+01, 4.54e+00 / 2.82e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.85e+00 / 2.10e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.36e-01 / 2.48e-01 | 20260507 17:32:45.171 | INFO | PerfReport: 3.77%, Metrics -- 1.26e+00 / 1.43e+00, 3.22e-01 / 2.06e-01 | 20260507 17:32:45.171 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.99e-01 / 4.17e-01, 6.53e-02 / 4.26e-02 | 20260507 17:32:45.171 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:32:45.171 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:32:45.171 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:32:45.171 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 23.78%, Upload -- 2.55e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 24.07%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 31.21%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 1.61%, LockUpload -- 1.73e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 2.53%, LockDownload -- 2.72e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 16.78%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:32:45.171 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 17:32:45.268 | INFO | Starting a round of Baz IO | 20260507 17:32:48.789 | INFO | Populating RTMetrics report | 20260507 17:32:48.790 | INFO | RTMetrics report populated | 20260507 17:32:48.790 | INFO | Opening tmp RTMetrics json file | 20260507 17:32:48.804 | INFO | Serializing RTMetrics to json | 20260507 17:32:48.805 | INFO | Serialization done, swapping tmp file now... | 20260507 17:32:48.806 | INFO | RTMetrics done | 20260507 17:32:48.866 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.597 seconds | 20260507 17:32:48.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.98% / 0.02% / 52.00% | 20260507 17:32:48.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.53545% / 4.76837e-05% / 2.38419e-05% | 20260507 17:32:48.866 | INFO | Baz IO overall: | 20260507 17:32:48.866 | INFO | BytesWritten : 581430250331 | 20260507 17:32:48.866 | INFO | NumEvents : 281728816108 | 20260507 17:32:50.309 | INFO | Analyzing chunk frames = [1318912,1319424) | 20260507 17:32:50.309 | INFO | MemInfo: { | 20260507 17:32:50.309 | INFO | "gpu" : " 17 GB", | 20260507 17:32:50.309 | INFO | "huge" : "", | 20260507 17:32:50.309 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:50.309 | INFO | "malloc" : " 14 GB", | 20260507 17:32:50.309 | INFO | "pinned" : " 29 GB" | 20260507 17:32:50.309 | INFO | } | 20260507 17:32:50.309 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:32:50.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:50.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:50.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:50.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:50.309 | INFO | ANALYSIS : 69.5%, 3.0, 108.6ms, 0 | 20260507 17:32:50.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:50.309 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 17:32:50.309 | INFO | PRE_HQ : 66.4%, 1.7, 59.7ms, 0 | 20260507 17:32:50.309 | INFO | BAZWRITER : 69.2%, 2.8, 101.2ms, 0 | 20260507 17:32:50.309 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:32:50.309 | INFO | Uploaded 36GiB to GPU in 2.4986s (14.4081GiB/s) | 20260507 17:32:50.309 | INFO | Downloaded 38GiB to GPU in 2.72385s (13.9508GiB/s) | 20260507 17:32:55.448 | INFO | Analyzing chunk frames = [1319424,1319936) | 20260507 17:32:55.448 | INFO | MemInfo: { | 20260507 17:32:55.448 | INFO | "gpu" : " 17 GB", | 20260507 17:32:55.448 | INFO | "huge" : "", | 20260507 17:32:55.448 | INFO | "huge_shared" : " 29 GB", | 20260507 17:32:55.448 | INFO | "malloc" : " 14 GB", | 20260507 17:32:55.448 | INFO | "pinned" : " 29 GB" | 20260507 17:32:55.448 | INFO | } | 20260507 17:32:55.448 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:32:55.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:32:55.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:32:55.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:32:55.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:32:55.448 | INFO | Uploaded 36GiB to GPU in 2.45518s (14.6629GiB/s) | 20260507 17:32:55.448 | INFO | Downloaded 34GiB to GPU in 2.46223s (13.8086GiB/s) | 20260507 17:33:00.587 | INFO | Analyzing chunk frames = [1319936,1320448) | 20260507 17:33:00.587 | INFO | MemInfo: { | 20260507 17:33:00.587 | INFO | "gpu" : " 17 GB", | 20260507 17:33:00.587 | INFO | "huge" : "", | 20260507 17:33:00.587 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:00.587 | INFO | "malloc" : " 14 GB", | 20260507 17:33:00.587 | INFO | "pinned" : " 29 GB" | 20260507 17:33:00.587 | INFO | } | 20260507 17:33:00.588 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:33:00.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:00.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 17:33:00.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:33:00.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:00.588 | INFO | Uploaded 36GiB to GPU in 2.50101s (14.3942GiB/s) | 20260507 17:33:00.588 | INFO | Downloaded 34GiB to GPU in 2.44811s (13.8882GiB/s) | 20260507 17:33:05.726 | INFO | Analyzing chunk frames = [1320448,1320960) | 20260507 17:33:05.726 | INFO | MemInfo: { | 20260507 17:33:05.726 | INFO | "gpu" : " 17 GB", | 20260507 17:33:05.726 | INFO | "huge" : "", | 20260507 17:33:05.726 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:05.726 | INFO | "malloc" : " 14 GB", | 20260507 17:33:05.726 | INFO | "pinned" : " 29 GB" | 20260507 17:33:05.726 | INFO | } | 20260507 17:33:05.727 | INFO | Perf report for the last 5.13882 seconds: | 20260507 17:33:05.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:05.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 17:33:05.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:33:05.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:05.727 | INFO | Uploaded 36GiB to GPU in 2.44856s (14.7025GiB/s) | 20260507 17:33:05.727 | INFO | Downloaded 34GiB to GPU in 2.36036s (14.4046GiB/s) | 20260507 17:33:10.866 | INFO | Analyzing chunk frames = [1320960,1321472) | 20260507 17:33:10.866 | INFO | MemInfo: { | 20260507 17:33:10.866 | INFO | "gpu" : " 17 GB", | 20260507 17:33:10.866 | INFO | "huge" : "", | 20260507 17:33:10.866 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:10.866 | INFO | "malloc" : " 14 GB", | 20260507 17:33:10.866 | INFO | "pinned" : " 29 GB" | 20260507 17:33:10.866 | INFO | } | 20260507 17:33:10.866 | INFO | Perf report for the last 5.13991 seconds: | 20260507 17:33:10.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:10.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.867 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:33:10.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.867 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260507 17:33:10.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:10.867 | INFO | Uploaded 36GiB to GPU in 2.3268s (15.4719GiB/s) | 20260507 17:33:10.867 | INFO | Downloaded 34GiB to GPU in 2.48614s (13.6758GiB/s) | 20260507 17:33:16.005 | INFO | Analyzing chunk frames = [1321472,1321984) | 20260507 17:33:16.005 | INFO | MemInfo: { | 20260507 17:33:16.005 | INFO | "gpu" : " 17 GB", | 20260507 17:33:16.005 | INFO | "huge" : "", | 20260507 17:33:16.005 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:16.005 | INFO | "malloc" : " 14 GB", | 20260507 17:33:16.006 | INFO | "pinned" : " 29 GB" | 20260507 17:33:16.006 | INFO | } | 20260507 17:33:16.006 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:33:16.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:16.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:33:16.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:33:16.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:16.006 | INFO | Uploaded 36GiB to GPU in 2.39166s (15.0523GiB/s) | 20260507 17:33:16.006 | INFO | Downloaded 34GiB to GPU in 2.51085s (13.5412GiB/s) | 20260507 17:33:21.145 | INFO | Analyzing chunk frames = [1321984,1322496) | 20260507 17:33:21.145 | INFO | MemInfo: { | 20260507 17:33:21.145 | INFO | "gpu" : " 17 GB", | 20260507 17:33:21.145 | INFO | "huge" : "", | 20260507 17:33:21.145 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:21.145 | INFO | "malloc" : " 14 GB", | 20260507 17:33:21.145 | INFO | "pinned" : " 29 GB" | 20260507 17:33:21.145 | INFO | } | 20260507 17:33:21.145 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:33:21.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:21.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:33:21.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:33:21.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:21.145 | INFO | Uploaded 36GiB to GPU in 2.44072s (14.7498GiB/s) | 20260507 17:33:21.145 | INFO | Downloaded 34GiB to GPU in 2.43993s (13.9348GiB/s) | 20260507 17:33:26.284 | INFO | Analyzing chunk frames = [1322496,1323008) | 20260507 17:33:26.284 | INFO | MemInfo: { | 20260507 17:33:26.284 | INFO | "gpu" : " 17 GB", | 20260507 17:33:26.284 | INFO | "huge" : "", | 20260507 17:33:26.284 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:26.284 | INFO | "malloc" : " 14 GB", | 20260507 17:33:26.284 | INFO | "pinned" : " 29 GB" | 20260507 17:33:26.284 | INFO | } | 20260507 17:33:26.284 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:33:26.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:26.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.284 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:33:26.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.284 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:33:26.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:26.285 | INFO | Uploaded 36GiB to GPU in 2.47414s (14.5505GiB/s) | 20260507 17:33:26.285 | INFO | Downloaded 34GiB to GPU in 2.49854s (13.6079GiB/s) | 20260507 17:33:26.381 | INFO | Starting a round of Baz IO | 20260507 17:33:29.887 | INFO | Populating RTMetrics report | 20260507 17:33:29.887 | INFO | RTMetrics report populated | 20260507 17:33:29.887 | INFO | Opening tmp RTMetrics json file | 20260507 17:33:29.903 | INFO | Serializing RTMetrics to json | 20260507 17:33:29.905 | INFO | Serialization done, swapping tmp file now... | 20260507 17:33:29.905 | INFO | RTMetrics done | 20260507 17:33:29.965 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.583 seconds | 20260507 17:33:29.965 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.96% / 0.02% / 52.02% | 20260507 17:33:29.965 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.52507% / 2.78155e-05% / 3.57628e-05% | 20260507 17:33:29.965 | INFO | Baz IO overall: | 20260507 17:33:29.965 | INFO | BytesWritten : 583363065213 | 20260507 17:33:29.965 | INFO | NumEvents : 282668217272 | 20260507 17:33:31.423 | INFO | Analyzing chunk frames = [1323008,1323520) | 20260507 17:33:31.423 | INFO | MemInfo: { | 20260507 17:33:31.423 | INFO | "gpu" : " 17 GB", | 20260507 17:33:31.423 | INFO | "huge" : "", | 20260507 17:33:31.423 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:31.423 | INFO | "malloc" : " 14 GB", | 20260507 17:33:31.423 | INFO | "pinned" : " 29 GB" | 20260507 17:33:31.423 | INFO | } | 20260507 17:33:31.423 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:33:31.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:31.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:31.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:31.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:31.423 | INFO | ANALYSIS : 69.4%, 3.0, 108.3ms, 0 | 20260507 17:33:31.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:31.423 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 17:33:31.423 | INFO | PRE_HQ : 66.7%, 1.7, 59.8ms, 0 | 20260507 17:33:31.424 | INFO | BAZWRITER : 68.8%, 2.8, 100.0ms, 0 | 20260507 17:33:31.424 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:33:31.424 | INFO | Uploaded 36GiB to GPU in 2.47864s (14.5241GiB/s) | 20260507 17:33:31.424 | INFO | Downloaded 38GiB to GPU in 2.72871s (13.926GiB/s) | 20260507 17:33:36.562 | INFO | Analyzing chunk frames = [1323520,1324032) | 20260507 17:33:36.562 | INFO | MemInfo: { | 20260507 17:33:36.562 | INFO | "gpu" : " 17 GB", | 20260507 17:33:36.562 | INFO | "huge" : "", | 20260507 17:33:36.562 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:36.562 | INFO | "malloc" : " 14 GB", | 20260507 17:33:36.562 | INFO | "pinned" : " 29 GB" | 20260507 17:33:36.562 | INFO | } | 20260507 17:33:36.562 | INFO | Perf report for the last 5.13894 seconds: | 20260507 17:33:36.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:36.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:33:36.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:33:36.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:36.563 | INFO | Uploaded 36GiB to GPU in 2.51478s (14.3153GiB/s) | 20260507 17:33:36.563 | INFO | Downloaded 34GiB to GPU in 2.44228s (13.9214GiB/s) | 20260507 17:33:41.702 | INFO | Analyzing chunk frames = [1324032,1324544) | 20260507 17:33:41.702 | INFO | MemInfo: { | 20260507 17:33:41.702 | INFO | "gpu" : " 17 GB", | 20260507 17:33:41.702 | INFO | "huge" : "", | 20260507 17:33:41.702 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:41.702 | INFO | "malloc" : " 14 GB", | 20260507 17:33:41.702 | INFO | "pinned" : " 29 GB" | 20260507 17:33:41.702 | INFO | } | 20260507 17:33:41.702 | INFO | Perf report for the last 5.13986 seconds: | 20260507 17:33:41.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:41.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.702 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:33:41.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.702 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:33:41.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:41.703 | INFO | Uploaded 36GiB to GPU in 2.56274s (14.0475GiB/s) | 20260507 17:33:41.703 | INFO | Downloaded 34GiB to GPU in 2.42335s (14.0302GiB/s) | 20260507 17:33:46.841 | INFO | Analyzing chunk frames = [1324544,1325056) | 20260507 17:33:46.841 | INFO | MemInfo: { | 20260507 17:33:46.841 | INFO | "gpu" : " 17 GB", | 20260507 17:33:46.841 | INFO | "huge" : "", | 20260507 17:33:46.841 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:46.841 | INFO | "malloc" : " 14 GB", | 20260507 17:33:46.841 | INFO | "pinned" : " 29 GB" | 20260507 17:33:46.841 | INFO | } | 20260507 17:33:46.841 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:33:46.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:46.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.841 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:33:46.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.841 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:33:46.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:46.842 | INFO | Uploaded 36GiB to GPU in 2.48838s (14.4672GiB/s) | 20260507 17:33:46.842 | INFO | Downloaded 34GiB to GPU in 2.35183s (14.4568GiB/s) | 20260507 17:33:51.981 | INFO | Analyzing chunk frames = [1325056,1325568) | 20260507 17:33:51.981 | INFO | MemInfo: { | 20260507 17:33:51.981 | INFO | "gpu" : " 17 GB", | 20260507 17:33:51.981 | INFO | "huge" : "", | 20260507 17:33:51.981 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:51.981 | INFO | "malloc" : " 14 GB", | 20260507 17:33:51.981 | INFO | "pinned" : " 29 GB" | 20260507 17:33:51.981 | INFO | } | 20260507 17:33:51.982 | INFO | Perf report for the last 5.14016 seconds: | 20260507 17:33:51.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:51.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:33:51.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:33:51.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:51.982 | INFO | Uploaded 36GiB to GPU in 2.36591s (15.2161GiB/s) | 20260507 17:33:51.982 | INFO | Downloaded 34GiB to GPU in 2.45854s (13.8294GiB/s) | 20260507 17:33:57.120 | INFO | Analyzing chunk frames = [1325568,1326080) | 20260507 17:33:57.120 | INFO | MemInfo: { | 20260507 17:33:57.120 | INFO | "gpu" : " 17 GB", | 20260507 17:33:57.120 | INFO | "huge" : "", | 20260507 17:33:57.120 | INFO | "huge_shared" : " 29 GB", | 20260507 17:33:57.120 | INFO | "malloc" : " 14 GB", | 20260507 17:33:57.120 | INFO | "pinned" : " 29 GB" | 20260507 17:33:57.120 | INFO | } | 20260507 17:33:57.120 | INFO | Perf report for the last 5.13894 seconds: | 20260507 17:33:57.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:33:57.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.121 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:33:57.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.121 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:33:57.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:33:57.121 | INFO | Uploaded 36GiB to GPU in 2.44121s (14.7468GiB/s) | 20260507 17:33:57.121 | INFO | Downloaded 34GiB to GPU in 2.481s (13.7042GiB/s) | 20260507 17:34:02.259 | INFO | Analyzing chunk frames = [1326080,1326592) | 20260507 17:34:02.259 | INFO | MemInfo: { | 20260507 17:34:02.259 | INFO | "gpu" : " 17 GB", | 20260507 17:34:02.259 | INFO | "huge" : "", | 20260507 17:34:02.259 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:02.259 | INFO | "malloc" : " 14 GB", | 20260507 17:34:02.259 | INFO | "pinned" : " 29 GB" | 20260507 17:34:02.259 | INFO | } | 20260507 17:34:02.259 | INFO | Perf report for the last 5.13891 seconds: | 20260507 17:34:02.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:02.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:34:02.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:34:02.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:02.260 | INFO | Uploaded 36GiB to GPU in 2.46411s (14.6097GiB/s) | 20260507 17:34:02.260 | INFO | Downloaded 34GiB to GPU in 2.41382s (14.0855GiB/s) | 20260507 17:34:07.399 | INFO | Analyzing chunk frames = [1326592,1327104) | 20260507 17:34:07.399 | INFO | MemInfo: { | 20260507 17:34:07.399 | INFO | "gpu" : " 17 GB", | 20260507 17:34:07.399 | INFO | "huge" : "", | 20260507 17:34:07.399 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:07.399 | INFO | "malloc" : " 14 GB", | 20260507 17:34:07.399 | INFO | "pinned" : " 29 GB" | 20260507 17:34:07.399 | INFO | } | 20260507 17:34:07.399 | INFO | Perf report for the last 5.13964 seconds: | 20260507 17:34:07.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:07.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 17:34:07.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:34:07.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:07.399 | INFO | Uploaded 36GiB to GPU in 2.55674s (14.0804GiB/s) | 20260507 17:34:07.399 | INFO | Downloaded 34GiB to GPU in 2.48916s (13.6592GiB/s) | 20260507 17:34:07.400 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:34:07.400 | INFO | Total mem usage: 16.4 GiB | 20260507 17:34:07.400 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:34:07.400 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.5%) | 20260507 17:34:07.400 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:34:07.400 | INFO | High Water Breakdown: | 20260507 17:34:07.400 | INFO | 59.1% : 8.54 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:34:07.400 | INFO | 40.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:34:07.400 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:34:07.400 | INFO | ------------------------------------------ | 20260507 17:34:07.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:34:07.400 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:34:07.400 | INFO | Total mem usage: 26.5 GiB | 20260507 17:34:07.400 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:34:07.400 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:34:07.400 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:34:07.400 | INFO | High Water Breakdown: | 20260507 17:34:07.400 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:34:07.400 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:34:07.400 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:34:07.400 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:34:07.400 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:34:07.400 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:34:07.400 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:34:07.400 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:34:07.400 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:34:07.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:34:07.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:34:07.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:34:07.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:34:07.400 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:34:07.400 | INFO | ------------------------------------------ | 20260507 17:34:07.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:34:07.400 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:34:07.400 | INFO | Total mem usage: 27.8 GiB | 20260507 17:34:07.400 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:34:07.400 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:34:07.400 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:34:07.400 | INFO | High Water Breakdown: | 20260507 17:34:07.400 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:34:07.400 | INFO | ------------------------------------------ | 20260507 17:34:07.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:34:07.400 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:34:07.400 | INFO | Total mem usage: 15.7 GiB | 20260507 17:34:07.400 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:34:07.400 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:34:07.400 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:34:07.400 | INFO | High Water Breakdown: | 20260507 17:34:07.400 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:34:07.400 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:34:07.400 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:34:07.400 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:34:07.400 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:34:07.400 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:34:07.400 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:34:07.400 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:34:07.400 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:34:07.400 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:34:07.400 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:34:07.400 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:34:07.400 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:34:07.400 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:34:07.400 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:34:07.400 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:34:07.400 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:34:07.400 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:34:07.400 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:34:07.400 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:34:07.400 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:34:07.400 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:34:07.400 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:34:07.401 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:34:07.401 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:34:07.401 | INFO | ------------------------------------------ | 20260507 17:34:07.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:34:07.401 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:34:07.401 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:34:07.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:34:07.401 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.69e+01, 1.45e+01 / 2.60e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.11e-02 / 4.24e-03 | 20260507 17:34:07.401 | INFO | PerfReport: 12.06%, Baseline -- 4.05e+00 / 4.51e+00, 6.21e-01 / 1.99e-01 | 20260507 17:34:07.401 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.52e+00, 2.19e-01 / 9.04e-02 | 20260507 17:34:07.401 | INFO | PerfReport: 15.73%, DME -- 4.22e+01 / 4.52e+01, 4.20e+00 / 2.25e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.90e+00 / 2.20e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.33e-01 / 2.76e-01 | 20260507 17:34:07.401 | INFO | PerfReport: 3.77%, Metrics -- 1.27e+00 / 1.44e+00, 3.21e-01 / 1.91e-01 | 20260507 17:34:07.401 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.99e-01 / 4.28e-01, 6.61e-02 / 3.77e-02 | 20260507 17:34:07.401 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:34:07.401 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:34:07.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:34:07.401 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 23.86%, Upload -- 2.57e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 24.05%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 31.20%, Compute -- 3.36e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 1.78%, LockUpload -- 1.91e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 2.39%, LockDownload -- 2.57e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 16.71%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:34:07.401 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260507 17:34:07.495 | INFO | Starting a round of Baz IO | 20260507 17:34:11.007 | INFO | Populating RTMetrics report | 20260507 17:34:11.007 | INFO | RTMetrics report populated | 20260507 17:34:11.007 | INFO | Opening tmp RTMetrics json file | 20260507 17:34:11.023 | INFO | Serializing RTMetrics to json | 20260507 17:34:11.025 | INFO | Serialization done, swapping tmp file now... | 20260507 17:34:11.025 | INFO | RTMetrics done | 20260507 17:34:11.085 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.589 seconds | 20260507 17:34:11.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.94% / 0.02% / 52.04% | 20260507 17:34:11.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.51525% / 2.38419e-05% / 1.19209e-05% | 20260507 17:34:11.085 | INFO | Baz IO overall: | 20260507 17:34:11.085 | INFO | BytesWritten : 585295051516 | 20260507 17:34:11.085 | INFO | NumEvents : 283607011462 | 20260507 17:34:12.538 | INFO | Analyzing chunk frames = [1327104,1327616) | 20260507 17:34:12.538 | INFO | MemInfo: { | 20260507 17:34:12.538 | INFO | "gpu" : " 17 GB", | 20260507 17:34:12.538 | INFO | "huge" : "", | 20260507 17:34:12.538 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:12.538 | INFO | "malloc" : " 14 GB", | 20260507 17:34:12.538 | INFO | "pinned" : " 29 GB" | 20260507 17:34:12.538 | INFO | } | 20260507 17:34:12.538 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:34:12.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:12.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:12.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:12.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:12.538 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:34:12.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:12.539 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 17:34:12.539 | INFO | PRE_HQ : 66.6%, 1.7, 59.3ms, 0 | 20260507 17:34:12.539 | INFO | BAZWRITER : 69.0%, 2.8, 99.8ms, 0 | 20260507 17:34:12.539 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 17:34:12.539 | INFO | Uploaded 36GiB to GPU in 2.54452s (14.1481GiB/s) | 20260507 17:34:12.539 | INFO | Downloaded 38GiB to GPU in 2.72163s (13.9622GiB/s) | 20260507 17:34:17.677 | INFO | Analyzing chunk frames = [1327616,1328128) | 20260507 17:34:17.677 | INFO | MemInfo: { | 20260507 17:34:17.677 | INFO | "gpu" : " 17 GB", | 20260507 17:34:17.677 | INFO | "huge" : "", | 20260507 17:34:17.677 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:17.677 | INFO | "malloc" : " 14 GB", | 20260507 17:34:17.677 | INFO | "pinned" : " 29 GB" | 20260507 17:34:17.677 | INFO | } | 20260507 17:34:17.678 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:34:17.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:17.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 17:34:17.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:34:17.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:17.678 | INFO | Uploaded 36GiB to GPU in 2.52991s (14.2298GiB/s) | 20260507 17:34:17.678 | INFO | Downloaded 34GiB to GPU in 2.42381s (14.0275GiB/s) | 20260507 17:34:22.817 | INFO | Analyzing chunk frames = [1328128,1328640) | 20260507 17:34:22.817 | INFO | MemInfo: { | 20260507 17:34:22.817 | INFO | "gpu" : " 17 GB", | 20260507 17:34:22.817 | INFO | "huge" : "", | 20260507 17:34:22.817 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:22.817 | INFO | "malloc" : " 14 GB", | 20260507 17:34:22.817 | INFO | "pinned" : " 29 GB" | 20260507 17:34:22.817 | INFO | } | 20260507 17:34:22.817 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:34:22.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:22.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.817 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260507 17:34:22.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.817 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:34:22.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:22.818 | INFO | Uploaded 36GiB to GPU in 2.47642s (14.5371GiB/s) | 20260507 17:34:22.818 | INFO | Downloaded 34GiB to GPU in 2.33274s (14.5752GiB/s) | 20260507 17:34:27.961 | INFO | Analyzing chunk frames = [1328640,1329152) | 20260507 17:34:27.961 | INFO | MemInfo: { | 20260507 17:34:27.961 | INFO | "gpu" : " 17 GB", | 20260507 17:34:27.961 | INFO | "huge" : "", | 20260507 17:34:27.961 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:27.961 | INFO | "malloc" : " 14 GB", | 20260507 17:34:27.962 | INFO | "pinned" : " 29 GB" | 20260507 17:34:27.962 | INFO | } | 20260507 17:34:27.962 | INFO | Perf report for the last 5.14463 seconds: | 20260507 17:34:27.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:27.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260507 17:34:27.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:34:27.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:27.962 | INFO | Uploaded 36GiB to GPU in 2.50698s (14.3599GiB/s) | 20260507 17:34:27.962 | INFO | Downloaded 34GiB to GPU in 2.34468s (14.5009GiB/s) | 20260507 17:34:33.096 | INFO | Analyzing chunk frames = [1329152,1329664) | 20260507 17:34:33.096 | INFO | MemInfo: { | 20260507 17:34:33.096 | INFO | "gpu" : " 17 GB", | 20260507 17:34:33.096 | INFO | "huge" : "", | 20260507 17:34:33.096 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:33.096 | INFO | "malloc" : " 14 GB", | 20260507 17:34:33.096 | INFO | "pinned" : " 29 GB" | 20260507 17:34:33.096 | INFO | } | 20260507 17:34:33.096 | INFO | Perf report for the last 5.13446 seconds: | 20260507 17:34:33.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:33.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 17:34:33.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 17:34:33.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:33.096 | INFO | Uploaded 36GiB to GPU in 2.42423s (14.8501GiB/s) | 20260507 17:34:33.097 | INFO | Downloaded 34GiB to GPU in 2.44832s (13.8871GiB/s) | 20260507 17:34:38.235 | INFO | Analyzing chunk frames = [1329664,1330176) | 20260507 17:34:38.235 | INFO | MemInfo: { | 20260507 17:34:38.235 | INFO | "gpu" : " 17 GB", | 20260507 17:34:38.235 | INFO | "huge" : "", | 20260507 17:34:38.235 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:38.235 | INFO | "malloc" : " 14 GB", | 20260507 17:34:38.235 | INFO | "pinned" : " 29 GB" | 20260507 17:34:38.235 | INFO | } | 20260507 17:34:38.235 | INFO | Perf report for the last 5.13893 seconds: | 20260507 17:34:38.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:38.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:34:38.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:34:38.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:38.235 | INFO | Uploaded 36GiB to GPU in 2.43821s (14.7649GiB/s) | 20260507 17:34:38.235 | INFO | Downloaded 34GiB to GPU in 2.47156s (13.7565GiB/s) | 20260507 17:34:43.374 | INFO | Analyzing chunk frames = [1330176,1330688) | 20260507 17:34:43.374 | INFO | MemInfo: { | 20260507 17:34:43.374 | INFO | "gpu" : " 17 GB", | 20260507 17:34:43.374 | INFO | "huge" : "", | 20260507 17:34:43.374 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:43.374 | INFO | "malloc" : " 14 GB", | 20260507 17:34:43.374 | INFO | "pinned" : " 29 GB" | 20260507 17:34:43.374 | INFO | } | 20260507 17:34:43.374 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:34:43.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:43.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.374 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:34:43.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.374 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:34:43.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:43.375 | INFO | Uploaded 36GiB to GPU in 2.49307s (14.44GiB/s) | 20260507 17:34:43.375 | INFO | Downloaded 34GiB to GPU in 2.41172s (14.0978GiB/s) | 20260507 17:34:48.514 | INFO | Analyzing chunk frames = [1330688,1331200) | 20260507 17:34:48.514 | INFO | MemInfo: { | 20260507 17:34:48.514 | INFO | "gpu" : " 17 GB", | 20260507 17:34:48.514 | INFO | "huge" : "", | 20260507 17:34:48.514 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:48.514 | INFO | "malloc" : " 14 GB", | 20260507 17:34:48.514 | INFO | "pinned" : " 29 GB" | 20260507 17:34:48.514 | INFO | } | 20260507 17:34:48.514 | INFO | Perf report for the last 5.13964 seconds: | 20260507 17:34:48.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:48.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 17:34:48.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:34:48.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:48.514 | INFO | Uploaded 36GiB to GPU in 2.56283s (14.047GiB/s) | 20260507 17:34:48.514 | INFO | Downloaded 34GiB to GPU in 2.50592s (13.5679GiB/s) | 20260507 17:34:48.611 | INFO | Starting a round of Baz IO | 20260507 17:34:52.119 | INFO | Populating RTMetrics report | 20260507 17:34:52.119 | INFO | RTMetrics report populated | 20260507 17:34:52.119 | INFO | Opening tmp RTMetrics json file | 20260507 17:34:52.132 | INFO | Serializing RTMetrics to json | 20260507 17:34:52.136 | INFO | Serialization done, swapping tmp file now... | 20260507 17:34:52.137 | INFO | RTMetrics done | 20260507 17:34:52.202 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.591 seconds | 20260507 17:34:52.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.92% / 0.02% / 52.06% | 20260507 17:34:52.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.50189% / 2.38419e-05% / 4.76837e-05% | 20260507 17:34:52.202 | INFO | Baz IO overall: | 20260507 17:34:52.202 | INFO | BytesWritten : 587225865684 | 20260507 17:34:52.202 | INFO | NumEvents : 284545081596 | 20260507 17:34:53.653 | INFO | Analyzing chunk frames = [1331200,1331712) | 20260507 17:34:53.653 | INFO | MemInfo: { | 20260507 17:34:53.653 | INFO | "gpu" : " 17 GB", | 20260507 17:34:53.653 | INFO | "huge" : "", | 20260507 17:34:53.653 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:53.653 | INFO | "malloc" : " 14 GB", | 20260507 17:34:53.653 | INFO | "pinned" : " 29 GB" | 20260507 17:34:53.653 | INFO | } | 20260507 17:34:53.653 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:34:53.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:53.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:53.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:53.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:53.653 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:34:53.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:53.653 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 17:34:53.653 | INFO | PRE_HQ : 66.7%, 1.7, 59.5ms, 0 | 20260507 17:34:53.653 | INFO | BAZWRITER : 69.1%, 2.9, 103.9ms, 0 | 20260507 17:34:53.653 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:34:53.654 | INFO | Uploaded 36GiB to GPU in 2.57048s (14.0052GiB/s) | 20260507 17:34:53.654 | INFO | Downloaded 38GiB to GPU in 2.72198s (13.9604GiB/s) | 20260507 17:34:58.792 | INFO | Analyzing chunk frames = [1331712,1332224) | 20260507 17:34:58.792 | INFO | MemInfo: { | 20260507 17:34:58.792 | INFO | "gpu" : " 17 GB", | 20260507 17:34:58.792 | INFO | "huge" : "", | 20260507 17:34:58.792 | INFO | "huge_shared" : " 29 GB", | 20260507 17:34:58.792 | INFO | "malloc" : " 14 GB", | 20260507 17:34:58.792 | INFO | "pinned" : " 29 GB" | 20260507 17:34:58.793 | INFO | } | 20260507 17:34:58.793 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:34:58.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:34:58.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:34:58.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 17:34:58.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:34:58.793 | INFO | Uploaded 36GiB to GPU in 2.53115s (14.2228GiB/s) | 20260507 17:34:58.793 | INFO | Downloaded 34GiB to GPU in 2.45806s (13.8321GiB/s) | 20260507 17:35:03.932 | INFO | Analyzing chunk frames = [1332224,1332736) | 20260507 17:35:03.932 | INFO | MemInfo: { | 20260507 17:35:03.932 | INFO | "gpu" : " 17 GB", | 20260507 17:35:03.932 | INFO | "huge" : "", | 20260507 17:35:03.932 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:03.932 | INFO | "malloc" : " 14 GB", | 20260507 17:35:03.932 | INFO | "pinned" : " 29 GB" | 20260507 17:35:03.932 | INFO | } | 20260507 17:35:03.932 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:35:03.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:03.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.932 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:35:03.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.932 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 17:35:03.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:03.933 | INFO | Uploaded 36GiB to GPU in 2.54034s (14.1713GiB/s) | 20260507 17:35:03.933 | INFO | Downloaded 34GiB to GPU in 2.4523s (13.8645GiB/s) | 20260507 17:35:09.071 | INFO | Analyzing chunk frames = [1332736,1333248) | 20260507 17:35:09.071 | INFO | MemInfo: { | 20260507 17:35:09.071 | INFO | "gpu" : " 17 GB", | 20260507 17:35:09.071 | INFO | "huge" : "", | 20260507 17:35:09.071 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:09.071 | INFO | "malloc" : " 14 GB", | 20260507 17:35:09.071 | INFO | "pinned" : " 29 GB" | 20260507 17:35:09.071 | INFO | } | 20260507 17:35:09.071 | INFO | Perf report for the last 5.1391 seconds: | 20260507 17:35:09.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:09.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.071 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:35:09.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.072 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 17:35:09.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:09.072 | INFO | Uploaded 36GiB to GPU in 2.48011s (14.5155GiB/s) | 20260507 17:35:09.072 | INFO | Downloaded 34GiB to GPU in 2.37889s (14.2924GiB/s) | 20260507 17:35:14.211 | INFO | Analyzing chunk frames = [1333248,1333760) | 20260507 17:35:14.211 | INFO | MemInfo: { | 20260507 17:35:14.211 | INFO | "gpu" : " 17 GB", | 20260507 17:35:14.211 | INFO | "huge" : "", | 20260507 17:35:14.211 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:14.211 | INFO | "malloc" : " 14 GB", | 20260507 17:35:14.211 | INFO | "pinned" : " 29 GB" | 20260507 17:35:14.211 | INFO | } | 20260507 17:35:14.211 | INFO | Perf report for the last 5.14013 seconds: | 20260507 17:35:14.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:14.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.211 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:35:14.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.212 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 17:35:14.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:14.212 | INFO | Uploaded 36GiB to GPU in 2.31118s (15.5764GiB/s) | 20260507 17:35:14.212 | INFO | Downloaded 34GiB to GPU in 2.49165s (13.6455GiB/s) | 20260507 17:35:19.350 | INFO | Analyzing chunk frames = [1333760,1334272) | 20260507 17:35:19.350 | INFO | MemInfo: { | 20260507 17:35:19.350 | INFO | "gpu" : " 17 GB", | 20260507 17:35:19.350 | INFO | "huge" : "", | 20260507 17:35:19.350 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:19.350 | INFO | "malloc" : " 14 GB", | 20260507 17:35:19.350 | INFO | "pinned" : " 29 GB" | 20260507 17:35:19.350 | INFO | } | 20260507 17:35:19.350 | INFO | Perf report for the last 5.13851 seconds: | 20260507 17:35:19.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:19.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 17:35:19.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:35:19.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:19.350 | INFO | Uploaded 36GiB to GPU in 2.33569s (15.413GiB/s) | 20260507 17:35:19.351 | INFO | Downloaded 34GiB to GPU in 2.5199s (13.4926GiB/s) | 20260507 17:35:24.489 | INFO | Analyzing chunk frames = [1334272,1334784) | 20260507 17:35:24.489 | INFO | MemInfo: { | 20260507 17:35:24.489 | INFO | "gpu" : " 17 GB", | 20260507 17:35:24.489 | INFO | "huge" : "", | 20260507 17:35:24.489 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:24.489 | INFO | "malloc" : " 14 GB", | 20260507 17:35:24.489 | INFO | "pinned" : " 29 GB" | 20260507 17:35:24.489 | INFO | } | 20260507 17:35:24.489 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:35:24.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:24.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.489 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:35:24.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.490 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:35:24.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:24.490 | INFO | Uploaded 36GiB to GPU in 2.43834s (14.7642GiB/s) | 20260507 17:35:24.490 | INFO | Downloaded 34GiB to GPU in 2.44046s (13.9318GiB/s) | 20260507 17:35:29.628 | INFO | Analyzing chunk frames = [1334784,1335296) | 20260507 17:35:29.629 | INFO | MemInfo: { | 20260507 17:35:29.629 | INFO | "gpu" : " 17 GB", | 20260507 17:35:29.629 | INFO | "huge" : "", | 20260507 17:35:29.629 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:29.629 | INFO | "malloc" : " 14 GB", | 20260507 17:35:29.629 | INFO | "pinned" : " 29 GB" | 20260507 17:35:29.629 | INFO | } | 20260507 17:35:29.629 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:35:29.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:29.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:35:29.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:35:29.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:29.629 | INFO | Uploaded 36GiB to GPU in 2.4862s (14.4799GiB/s) | 20260507 17:35:29.629 | INFO | Downloaded 34GiB to GPU in 2.52924s (13.4428GiB/s) | 20260507 17:35:29.629 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:35:29.629 | INFO | Total mem usage: 16.4 GiB | 20260507 17:35:29.629 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:35:29.629 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -14%) | 20260507 17:35:29.629 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:35:29.629 | INFO | High Water Breakdown: | 20260507 17:35:29.629 | INFO | 59% : 8.5 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:35:29.629 | INFO | 40.8% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:35:29.629 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:35:29.629 | INFO | ------------------------------------------ | 20260507 17:35:29.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:35:29.629 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:35:29.629 | INFO | Total mem usage: 26.5 GiB | 20260507 17:35:29.629 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:35:29.629 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:35:29.629 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:35:29.629 | INFO | High Water Breakdown: | 20260507 17:35:29.629 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:35:29.629 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:35:29.629 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:35:29.629 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:35:29.629 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:35:29.629 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:35:29.629 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:35:29.629 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:35:29.629 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:35:29.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:35:29.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:35:29.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:35:29.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:35:29.629 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:35:29.629 | INFO | ------------------------------------------ | 20260507 17:35:29.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:35:29.629 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:35:29.629 | INFO | Total mem usage: 27.8 GiB | 20260507 17:35:29.629 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:35:29.629 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:35:29.629 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:35:29.629 | INFO | High Water Breakdown: | 20260507 17:35:29.629 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:35:29.629 | INFO | ------------------------------------------ | 20260507 17:35:29.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:35:29.629 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:35:29.629 | INFO | Total mem usage: 15.7 GiB | 20260507 17:35:29.629 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:35:29.629 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:35:29.629 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 17:35:29.629 | INFO | High Water Breakdown: | 20260507 17:35:29.629 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:35:29.629 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:35:29.629 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:35:29.629 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:35:29.629 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:35:29.629 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:35:29.629 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:35:29.629 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:35:29.629 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:35:29.629 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:35:29.629 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:35:29.629 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:35:29.629 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:35:29.629 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:35:29.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:35:29.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:35:29.629 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:35:29.629 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:35:29.629 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:35:29.629 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:35:29.629 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:35:29.629 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:35:29.629 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:35:29.629 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:35:29.629 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:35:29.629 | INFO | ------------------------------------------ | 20260507 17:35:29.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:35:29.629 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:35:29.630 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:35:29.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:35:29.630 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.62e+01, 1.43e+01 / 2.66e+00 | 20260507 17:35:29.630 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.02e-01, 1.18e-02 / 6.67e-03 | 20260507 17:35:29.630 | INFO | PerfReport: 12.04%, Baseline -- 4.02e+00 / 4.50e+00, 6.15e-01 / 2.23e-01 | 20260507 17:35:29.630 | INFO | PerfReport: 3.42%, Histogram -- 1.14e+00 / 1.49e+00, 2.13e-01 / 1.11e-01 | 20260507 17:35:29.630 | INFO | PerfReport: 15.76%, DME -- 4.21e+01 / 4.50e+01, 4.17e+00 / 2.74e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.83e+00 / 2.31e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.52e+00 / 2.90e+00, 4.37e-01 / 2.32e-01 | 20260507 17:35:29.631 | INFO | PerfReport: 3.77%, Metrics -- 1.26e+00 / 1.44e+00, 3.16e-01 / 1.82e-01 | 20260507 17:35:29.631 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.99e-01 / 4.22e-01, 6.50e-02 / 4.14e-02 | 20260507 17:35:29.631 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:35:29.631 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:35:29.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:35:29.631 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 24.03%, Upload -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 31.09%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 1.86%, LockUpload -- 1.99e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 2.29%, LockDownload -- 2.46e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 16.69%, LockCompute -- 1.79e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 17:35:29.631 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 17:35:29.726 | INFO | Starting a round of Baz IO | 20260507 17:35:33.259 | INFO | Populating RTMetrics report | 20260507 17:35:33.259 | INFO | RTMetrics report populated | 20260507 17:35:33.259 | INFO | Opening tmp RTMetrics json file | 20260507 17:35:33.260 | INFO | Serializing RTMetrics to json | 20260507 17:35:33.261 | INFO | Serialization done, swapping tmp file now... | 20260507 17:35:33.262 | INFO | RTMetrics done | 20260507 17:35:33.341 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.615 seconds | 20260507 17:35:33.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.91% / 0.02% / 52.08% | 20260507 17:35:33.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.49741% / 2.38419e-05% / 3.97364e-05% | 20260507 17:35:33.341 | INFO | Baz IO overall: | 20260507 17:35:33.341 | INFO | BytesWritten : 589157924260 | 20260507 17:35:33.341 | INFO | NumEvents : 285484145125 | 20260507 17:35:34.768 | INFO | Analyzing chunk frames = [1335296,1335808) | 20260507 17:35:34.768 | INFO | MemInfo: { | 20260507 17:35:34.768 | INFO | "gpu" : " 17 GB", | 20260507 17:35:34.768 | INFO | "huge" : "", | 20260507 17:35:34.768 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:34.768 | INFO | "malloc" : " 14 GB", | 20260507 17:35:34.768 | INFO | "pinned" : " 29 GB" | 20260507 17:35:34.768 | INFO | } | 20260507 17:35:34.768 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:35:34.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:34.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:34.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:34.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:34.768 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:35:34.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:34.768 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 17:35:34.768 | INFO | PRE_HQ : 67.4%, 1.8, 64.5ms, 0 | 20260507 17:35:34.768 | INFO | BAZWRITER : 69.4%, 3.0, 107.2ms, 0 | 20260507 17:35:34.768 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:35:34.768 | INFO | Uploaded 36GiB to GPU in 2.5504s (14.1154GiB/s) | 20260507 17:35:34.768 | INFO | Downloaded 38GiB to GPU in 2.72168s (13.9619GiB/s) | 20260507 17:35:39.907 | INFO | Analyzing chunk frames = [1335808,1336320) | 20260507 17:35:39.907 | INFO | MemInfo: { | 20260507 17:35:39.907 | INFO | "gpu" : " 17 GB", | 20260507 17:35:39.907 | INFO | "huge" : "", | 20260507 17:35:39.907 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:39.907 | INFO | "malloc" : " 14 GB", | 20260507 17:35:39.907 | INFO | "pinned" : " 29 GB" | 20260507 17:35:39.907 | INFO | } | 20260507 17:35:39.907 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:35:39.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:39.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.907 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:35:39.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.908 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 17:35:39.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:39.908 | INFO | Uploaded 36GiB to GPU in 2.53482s (14.2022GiB/s) | 20260507 17:35:39.908 | INFO | Downloaded 34GiB to GPU in 2.4476s (13.8912GiB/s) | 20260507 17:35:45.047 | INFO | Analyzing chunk frames = [1336320,1336832) | 20260507 17:35:45.047 | INFO | MemInfo: { | 20260507 17:35:45.047 | INFO | "gpu" : " 17 GB", | 20260507 17:35:45.047 | INFO | "huge" : "", | 20260507 17:35:45.047 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:45.047 | INFO | "malloc" : " 14 GB", | 20260507 17:35:45.047 | INFO | "pinned" : " 29 GB" | 20260507 17:35:45.047 | INFO | } | 20260507 17:35:45.047 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:35:45.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:45.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:35:45.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:35:45.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:45.047 | INFO | Uploaded 36GiB to GPU in 2.5603s (14.0609GiB/s) | 20260507 17:35:45.047 | INFO | Downloaded 34GiB to GPU in 2.42814s (14.0025GiB/s) | 20260507 17:35:50.186 | INFO | Analyzing chunk frames = [1336832,1337344) | 20260507 17:35:50.186 | INFO | MemInfo: { | 20260507 17:35:50.186 | INFO | "gpu" : " 17 GB", | 20260507 17:35:50.186 | INFO | "huge" : "", | 20260507 17:35:50.186 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:50.186 | INFO | "malloc" : " 14 GB", | 20260507 17:35:50.186 | INFO | "pinned" : " 29 GB" | 20260507 17:35:50.186 | INFO | } | 20260507 17:35:50.186 | INFO | Perf report for the last 5.13946 seconds: | 20260507 17:35:50.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:50.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:35:50.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:35:50.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:50.186 | INFO | Uploaded 36GiB to GPU in 2.47433s (14.5494GiB/s) | 20260507 17:35:50.187 | INFO | Downloaded 34GiB to GPU in 2.35271s (14.4514GiB/s) | 20260507 17:35:55.326 | INFO | Analyzing chunk frames = [1337344,1337856) | 20260507 17:35:55.326 | INFO | MemInfo: { | 20260507 17:35:55.326 | INFO | "gpu" : " 17 GB", | 20260507 17:35:55.326 | INFO | "huge" : "", | 20260507 17:35:55.326 | INFO | "huge_shared" : " 29 GB", | 20260507 17:35:55.326 | INFO | "malloc" : " 14 GB", | 20260507 17:35:55.326 | INFO | "pinned" : " 29 GB" | 20260507 17:35:55.326 | INFO | } | 20260507 17:35:55.326 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:35:55.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:35:55.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 17:35:55.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:35:55.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:35:55.326 | INFO | Uploaded 36GiB to GPU in 2.34969s (15.3212GiB/s) | 20260507 17:35:55.326 | INFO | Downloaded 34GiB to GPU in 2.46843s (13.7739GiB/s) | 20260507 17:36:00.465 | INFO | Analyzing chunk frames = [1337856,1338368) | 20260507 17:36:00.465 | INFO | MemInfo: { | 20260507 17:36:00.465 | INFO | "gpu" : " 17 GB", | 20260507 17:36:00.465 | INFO | "huge" : "", | 20260507 17:36:00.465 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:00.465 | INFO | "malloc" : " 14 GB", | 20260507 17:36:00.465 | INFO | "pinned" : " 29 GB" | 20260507 17:36:00.465 | INFO | } | 20260507 17:36:00.465 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:36:00.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:00.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:36:00.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 17:36:00.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:00.465 | INFO | Uploaded 36GiB to GPU in 2.41544s (14.9041GiB/s) | 20260507 17:36:00.465 | INFO | Downloaded 34GiB to GPU in 2.49091s (13.6496GiB/s) | 20260507 17:36:05.604 | INFO | Analyzing chunk frames = [1338368,1338880) | 20260507 17:36:05.604 | INFO | MemInfo: { | 20260507 17:36:05.604 | INFO | "gpu" : " 17 GB", | 20260507 17:36:05.604 | INFO | "huge" : "", | 20260507 17:36:05.604 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:05.604 | INFO | "malloc" : " 14 GB", | 20260507 17:36:05.604 | INFO | "pinned" : " 29 GB" | 20260507 17:36:05.604 | INFO | } | 20260507 17:36:05.604 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:36:05.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:05.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:36:05.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | PRE_HQ : 38.0%, 1.0, 19.8ms, 0 | 20260507 17:36:05.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:05.604 | INFO | Uploaded 36GiB to GPU in 2.50359s (14.3793GiB/s) | 20260507 17:36:05.605 | INFO | Downloaded 34GiB to GPU in 2.44073s (13.9303GiB/s) | 20260507 17:36:10.743 | INFO | Analyzing chunk frames = [1338880,1339392) | 20260507 17:36:10.743 | INFO | MemInfo: { | 20260507 17:36:10.743 | INFO | "gpu" : " 17 GB", | 20260507 17:36:10.743 | INFO | "huge" : "", | 20260507 17:36:10.743 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:10.743 | INFO | "malloc" : " 14 GB", | 20260507 17:36:10.744 | INFO | "pinned" : " 29 GB" | 20260507 17:36:10.744 | INFO | } | 20260507 17:36:10.744 | INFO | Perf report for the last 5.13959 seconds: | 20260507 17:36:10.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:10.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.744 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:36:10.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.744 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 17:36:10.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:10.745 | INFO | Uploaded 36GiB to GPU in 2.45776s (14.6475GiB/s) | 20260507 17:36:10.745 | INFO | Downloaded 34GiB to GPU in 2.41969s (14.0514GiB/s) | 20260507 17:36:10.855 | INFO | Starting a round of Baz IO | 20260507 17:36:14.353 | INFO | Populating RTMetrics report | 20260507 17:36:14.353 | INFO | RTMetrics report populated | 20260507 17:36:14.353 | INFO | Opening tmp RTMetrics json file | 20260507 17:36:14.362 | INFO | Serializing RTMetrics to json | 20260507 17:36:14.372 | INFO | Serialization done, swapping tmp file now... | 20260507 17:36:14.373 | INFO | RTMetrics done | 20260507 17:36:14.435 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.579 seconds | 20260507 17:36:14.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.89% / 0.02% / 52.09% | 20260507 17:36:14.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.49082% / 1.58946e-05% / 2.38419e-05% | 20260507 17:36:14.435 | INFO | Baz IO overall: | 20260507 17:36:14.435 | INFO | BytesWritten : 591090438192 | 20260507 17:36:14.435 | INFO | NumEvents : 286423372491 | 20260507 17:36:15.883 | INFO | Analyzing chunk frames = [1339392,1339904) | 20260507 17:36:15.883 | INFO | MemInfo: { | 20260507 17:36:15.883 | INFO | "gpu" : " 17 GB", | 20260507 17:36:15.883 | INFO | "huge" : "", | 20260507 17:36:15.883 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:15.883 | INFO | "malloc" : " 14 GB", | 20260507 17:36:15.883 | INFO | "pinned" : " 29 GB" | 20260507 17:36:15.883 | INFO | } | 20260507 17:36:15.883 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:36:15.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:15.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:15.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:15.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:15.883 | INFO | ANALYSIS : 69.5%, 3.0, 108.3ms, 0 | 20260507 17:36:15.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:15.883 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 17:36:15.883 | INFO | PRE_HQ : 67.4%, 1.8, 63.9ms, 0 | 20260507 17:36:15.883 | INFO | BAZWRITER : 68.7%, 3.1, 108.2ms, 0 | 20260507 17:36:15.883 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:36:15.883 | INFO | Uploaded 36GiB to GPU in 2.48639s (14.4788GiB/s) | 20260507 17:36:15.883 | INFO | Downloaded 38GiB to GPU in 2.7195s (13.9731GiB/s) | 20260507 17:36:21.022 | INFO | Analyzing chunk frames = [1339904,1340416) | 20260507 17:36:21.022 | INFO | MemInfo: { | 20260507 17:36:21.022 | INFO | "gpu" : " 17 GB", | 20260507 17:36:21.022 | INFO | "huge" : "", | 20260507 17:36:21.022 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:21.022 | INFO | "malloc" : " 14 GB", | 20260507 17:36:21.022 | INFO | "pinned" : " 29 GB" | 20260507 17:36:21.022 | INFO | } | 20260507 17:36:21.023 | INFO | Perf report for the last 5.13973 seconds: | 20260507 17:36:21.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:21.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:36:21.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:36:21.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:21.023 | INFO | Uploaded 36GiB to GPU in 2.53567s (14.1974GiB/s) | 20260507 17:36:21.023 | INFO | Downloaded 34GiB to GPU in 2.45555s (13.8462GiB/s) | 20260507 17:36:26.161 | INFO | Analyzing chunk frames = [1340416,1340928) | 20260507 17:36:26.161 | INFO | MemInfo: { | 20260507 17:36:26.161 | INFO | "gpu" : " 17 GB", | 20260507 17:36:26.161 | INFO | "huge" : "", | 20260507 17:36:26.161 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:26.161 | INFO | "malloc" : " 14 GB", | 20260507 17:36:26.162 | INFO | "pinned" : " 29 GB" | 20260507 17:36:26.162 | INFO | } | 20260507 17:36:26.162 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:36:26.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:26.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 17:36:26.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:36:26.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:26.162 | INFO | Uploaded 36GiB to GPU in 2.52085s (14.2809GiB/s) | 20260507 17:36:26.162 | INFO | Downloaded 34GiB to GPU in 2.44505s (13.9057GiB/s) | 20260507 17:36:31.301 | INFO | Analyzing chunk frames = [1340928,1341440) | 20260507 17:36:31.301 | INFO | MemInfo: { | 20260507 17:36:31.301 | INFO | "gpu" : " 17 GB", | 20260507 17:36:31.301 | INFO | "huge" : "", | 20260507 17:36:31.301 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:31.301 | INFO | "malloc" : " 14 GB", | 20260507 17:36:31.301 | INFO | "pinned" : " 29 GB" | 20260507 17:36:31.301 | INFO | } | 20260507 17:36:31.301 | INFO | Perf report for the last 5.13905 seconds: | 20260507 17:36:31.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:31.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:36:31.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 17:36:31.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:31.301 | INFO | Uploaded 36GiB to GPU in 2.46714s (14.5918GiB/s) | 20260507 17:36:31.301 | INFO | Downloaded 34GiB to GPU in 2.36444s (14.3797GiB/s) | 20260507 17:36:36.441 | INFO | Analyzing chunk frames = [1341440,1341952) | 20260507 17:36:36.441 | INFO | MemInfo: { | 20260507 17:36:36.441 | INFO | "gpu" : " 17 GB", | 20260507 17:36:36.441 | INFO | "huge" : "", | 20260507 17:36:36.441 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:36.441 | INFO | "malloc" : " 14 GB", | 20260507 17:36:36.441 | INFO | "pinned" : " 29 GB" | 20260507 17:36:36.441 | INFO | } | 20260507 17:36:36.441 | INFO | Perf report for the last 5.14008 seconds: | 20260507 17:36:36.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:36.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.441 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:36:36.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.441 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 17:36:36.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:36.442 | INFO | Uploaded 36GiB to GPU in 2.35139s (15.3101GiB/s) | 20260507 17:36:36.442 | INFO | Downloaded 34GiB to GPU in 2.48413s (13.6869GiB/s) | 20260507 17:36:41.579 | INFO | Analyzing chunk frames = [1341952,1342464) | 20260507 17:36:41.579 | INFO | MemInfo: { | 20260507 17:36:41.579 | INFO | "gpu" : " 17 GB", | 20260507 17:36:41.579 | INFO | "huge" : "", | 20260507 17:36:41.579 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:41.579 | INFO | "malloc" : " 14 GB", | 20260507 17:36:41.580 | INFO | "pinned" : " 29 GB" | 20260507 17:36:41.580 | INFO | } | 20260507 17:36:41.580 | INFO | Perf report for the last 5.13873 seconds: | 20260507 17:36:41.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:41.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:36:41.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 17:36:41.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:41.580 | INFO | Uploaded 36GiB to GPU in 2.41213s (14.9245GiB/s) | 20260507 17:36:41.581 | INFO | Downloaded 34GiB to GPU in 2.49593s (13.6222GiB/s) | 20260507 17:36:46.719 | INFO | Analyzing chunk frames = [1342464,1342976) | 20260507 17:36:46.719 | INFO | MemInfo: { | 20260507 17:36:46.719 | INFO | "gpu" : " 17 GB", | 20260507 17:36:46.719 | INFO | "huge" : "", | 20260507 17:36:46.719 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:46.719 | INFO | "malloc" : " 14 GB", | 20260507 17:36:46.719 | INFO | "pinned" : " 29 GB" | 20260507 17:36:46.719 | INFO | } | 20260507 17:36:46.719 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:36:46.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:46.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:36:46.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:36:46.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:46.719 | INFO | Uploaded 36GiB to GPU in 2.52581s (14.2529GiB/s) | 20260507 17:36:46.719 | INFO | Downloaded 34GiB to GPU in 2.43923s (13.9388GiB/s) | 20260507 17:36:51.858 | INFO | Analyzing chunk frames = [1342976,1343488) | 20260507 17:36:51.858 | INFO | MemInfo: { | 20260507 17:36:51.858 | INFO | "gpu" : " 17 GB", | 20260507 17:36:51.858 | INFO | "huge" : "", | 20260507 17:36:51.858 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:51.858 | INFO | "malloc" : " 14 GB", | 20260507 17:36:51.858 | INFO | "pinned" : " 29 GB" | 20260507 17:36:51.858 | INFO | } | 20260507 17:36:51.858 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:36:51.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:51.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.858 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 17:36:51.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.858 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:36:51.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:51.859 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:36:51.859 | INFO | Total mem usage: 16.4 GiB | 20260507 17:36:51.859 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:36:51.859 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.8%) | 20260507 17:36:51.859 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:36:51.859 | INFO | High Water Breakdown: | 20260507 17:36:51.859 | INFO | 59.3% : 8.61 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:36:51.859 | INFO | 40.6% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:36:51.859 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:36:51.859 | INFO | ------------------------------------------ | 20260507 17:36:51.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:36:51.859 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:36:51.859 | INFO | Total mem usage: 26.5 GiB | 20260507 17:36:51.859 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:36:51.859 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:36:51.859 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:36:51.859 | INFO | High Water Breakdown: | 20260507 17:36:51.859 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:36:51.859 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:36:51.859 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:36:51.859 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:36:51.859 | INFO | Uploaded 36GiB to GPU in 2.54135s (14.1657GiB/s) | 20260507 17:36:51.859 | INFO | Downloaded 34GiB to GPU in 2.49882s (13.6064GiB/s) | 20260507 17:36:51.859 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:36:51.859 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:36:51.859 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:36:51.859 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:36:51.859 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:36:51.859 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:36:51.859 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:36:51.859 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:36:51.859 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:36:51.859 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:36:51.859 | INFO | ------------------------------------------ | 20260507 17:36:51.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:36:51.859 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:36:51.859 | INFO | Total mem usage: 27.8 GiB | 20260507 17:36:51.859 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:36:51.859 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:36:51.859 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:36:51.859 | INFO | High Water Breakdown: | 20260507 17:36:51.859 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:36:51.859 | INFO | ------------------------------------------ | 20260507 17:36:51.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:36:51.859 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:36:51.859 | INFO | Total mem usage: 15.7 GiB | 20260507 17:36:51.859 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:36:51.859 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:36:51.859 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:36:51.859 | INFO | High Water Breakdown: | 20260507 17:36:51.859 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:36:51.859 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:36:51.859 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:36:51.859 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:36:51.859 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:36:51.859 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:36:51.859 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:36:51.859 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:36:51.859 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:36:51.859 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:36:51.859 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:36:51.859 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:36:51.859 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:36:51.859 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:36:51.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:36:51.860 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:36:51.860 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:36:51.860 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:36:51.860 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:36:51.860 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:36:51.860 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:36:51.860 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:36:51.860 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:36:51.860 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:36:51.860 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:36:51.860 | INFO | ------------------------------------------ | 20260507 17:36:51.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:36:51.860 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:36:51.860 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:36:51.860 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:36:51.860 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.44e+01 / 2.54e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 0.30%, AggregateStats -- 1.02e-01 / 1.08e-01, 1.19e-02 / 9.22e-03 | 20260507 17:36:51.860 | INFO | PerfReport: 12.11%, Baseline -- 4.06e+00 / 4.52e+00, 6.23e-01 / 2.27e-01 | 20260507 17:36:51.860 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 1.07e-01 | 20260507 17:36:51.860 | INFO | PerfReport: 15.68%, DME -- 4.21e+01 / 4.48e+01, 4.28e+00 / 2.54e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 55.93%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.87e+00 / 2.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.53e+00 / 2.85e+00, 4.41e-01 / 2.51e-01 | 20260507 17:36:51.860 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.44e+00, 3.19e-01 / 1.91e-01 | 20260507 17:36:51.860 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.04e-01 / 4.28e-01, 6.45e-02 / 3.84e-02 | 20260507 17:36:51.860 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:36:51.860 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:36:51.860 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:36:51.860 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 24.03%, Upload -- 2.58e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.65e+00 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 31.19%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 1.73%, LockUpload -- 1.86e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 2.31%, LockDownload -- 2.48e+00 / 0.00e+00, 3.85e+00 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 16.71%, LockCompute -- 1.80e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:36:51.860 | INFO | PerfReport: 0.02%, Misc -- 1.88e-02 / 0.00e+00, 2.95e-02 / 0.00e+00 | 20260507 17:36:51.956 | INFO | Starting a round of Baz IO | 20260507 17:36:55.466 | INFO | Populating RTMetrics report | 20260507 17:36:55.466 | INFO | RTMetrics report populated | 20260507 17:36:55.466 | INFO | Opening tmp RTMetrics json file | 20260507 17:36:55.467 | INFO | Serializing RTMetrics to json | 20260507 17:36:55.480 | INFO | Serialization done, swapping tmp file now... | 20260507 17:36:55.485 | INFO | RTMetrics done | 20260507 17:36:55.548 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.592 seconds | 20260507 17:36:55.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.87% / 0.02% / 52.11% | 20260507 17:36:55.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.48609% / 1.98682e-05% / 3.17891e-05% | 20260507 17:36:55.548 | INFO | Baz IO overall: | 20260507 17:36:55.548 | INFO | BytesWritten : 593023777388 | 20260507 17:36:55.548 | INFO | NumEvents : 287363325418 | 20260507 17:36:56.997 | INFO | Analyzing chunk frames = [1343488,1344000) | 20260507 17:36:56.997 | INFO | MemInfo: { | 20260507 17:36:56.997 | INFO | "gpu" : " 17 GB", | 20260507 17:36:56.997 | INFO | "huge" : "", | 20260507 17:36:56.997 | INFO | "huge_shared" : " 29 GB", | 20260507 17:36:56.997 | INFO | "malloc" : " 14 GB", | 20260507 17:36:56.997 | INFO | "pinned" : " 29 GB" | 20260507 17:36:56.997 | INFO | } | 20260507 17:36:56.997 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:36:56.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:36:56.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:56.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:56.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:56.997 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:36:56.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:36:56.997 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 17:36:56.997 | INFO | PRE_HQ : 67.5%, 1.8, 61.4ms, 0 | 20260507 17:36:56.997 | INFO | BAZWRITER : 69.1%, 2.9, 103.4ms, 0 | 20260507 17:36:56.997 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 17:36:56.998 | INFO | Uploaded 36GiB to GPU in 2.58058s (13.9503GiB/s) | 20260507 17:36:56.998 | INFO | Downloaded 38GiB to GPU in 2.71254s (14.009GiB/s) | 20260507 17:37:02.137 | INFO | Analyzing chunk frames = [1344000,1344512) | 20260507 17:37:02.137 | INFO | MemInfo: { | 20260507 17:37:02.137 | INFO | "gpu" : " 17 GB", | 20260507 17:37:02.137 | INFO | "huge" : "", | 20260507 17:37:02.137 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:02.137 | INFO | "malloc" : " 14 GB", | 20260507 17:37:02.137 | INFO | "pinned" : " 29 GB" | 20260507 17:37:02.137 | INFO | } | 20260507 17:37:02.137 | INFO | Perf report for the last 5.13981 seconds: | 20260507 17:37:02.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:02.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.137 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 17:37:02.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.137 | INFO | PRE_HQ : 36.9%, 1.0, 19.1ms, 0 | 20260507 17:37:02.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:02.138 | INFO | Uploaded 36GiB to GPU in 2.55931s (14.0663GiB/s) | 20260507 17:37:02.138 | INFO | Downloaded 34GiB to GPU in 2.44164s (13.9251GiB/s) | 20260507 17:37:07.276 | INFO | Analyzing chunk frames = [1344512,1345024) | 20260507 17:37:07.276 | INFO | MemInfo: { | 20260507 17:37:07.276 | INFO | "gpu" : " 17 GB", | 20260507 17:37:07.276 | INFO | "huge" : "", | 20260507 17:37:07.276 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:07.276 | INFO | "malloc" : " 14 GB", | 20260507 17:37:07.276 | INFO | "pinned" : " 29 GB" | 20260507 17:37:07.276 | INFO | } | 20260507 17:37:07.276 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:37:07.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:07.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.276 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:37:07.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.277 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:37:07.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:07.277 | INFO | Uploaded 36GiB to GPU in 2.5554s (14.0878GiB/s) | 20260507 17:37:07.277 | INFO | Downloaded 34GiB to GPU in 2.4362s (13.9562GiB/s) | 20260507 17:37:12.416 | INFO | Analyzing chunk frames = [1345024,1345536) | 20260507 17:37:12.416 | INFO | MemInfo: { | 20260507 17:37:12.416 | INFO | "gpu" : " 17 GB", | 20260507 17:37:12.416 | INFO | "huge" : "", | 20260507 17:37:12.416 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:12.416 | INFO | "malloc" : " 14 GB", | 20260507 17:37:12.416 | INFO | "pinned" : " 29 GB" | 20260507 17:37:12.416 | INFO | } | 20260507 17:37:12.416 | INFO | Perf report for the last 5.13976 seconds: | 20260507 17:37:12.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:12.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.416 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 17:37:12.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.416 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 17:37:12.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:12.417 | INFO | Uploaded 36GiB to GPU in 2.507s (14.3598GiB/s) | 20260507 17:37:12.417 | INFO | Downloaded 34GiB to GPU in 2.3652s (14.3751GiB/s) | 20260507 17:37:17.555 | INFO | Analyzing chunk frames = [1345536,1346048) | 20260507 17:37:17.555 | INFO | MemInfo: { | 20260507 17:37:17.555 | INFO | "gpu" : " 17 GB", | 20260507 17:37:17.555 | INFO | "huge" : "", | 20260507 17:37:17.555 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:17.555 | INFO | "malloc" : " 14 GB", | 20260507 17:37:17.555 | INFO | "pinned" : " 29 GB" | 20260507 17:37:17.555 | INFO | } | 20260507 17:37:17.555 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:37:17.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:17.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.555 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 17:37:17.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.555 | INFO | PRE_HQ : 35.5%, 1.0, 18.4ms, 0 | 20260507 17:37:17.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:17.556 | INFO | Uploaded 36GiB to GPU in 2.35055s (15.3156GiB/s) | 20260507 17:37:17.556 | INFO | Downloaded 34GiB to GPU in 2.48386s (13.6884GiB/s) | 20260507 17:37:22.694 | INFO | Analyzing chunk frames = [1346048,1346560) | 20260507 17:37:22.694 | INFO | MemInfo: { | 20260507 17:37:22.694 | INFO | "gpu" : " 17 GB", | 20260507 17:37:22.694 | INFO | "huge" : "", | 20260507 17:37:22.694 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:22.694 | INFO | "malloc" : " 14 GB", | 20260507 17:37:22.694 | INFO | "pinned" : " 29 GB" | 20260507 17:37:22.694 | INFO | } | 20260507 17:37:22.695 | INFO | Perf report for the last 5.13938 seconds: | 20260507 17:37:22.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:22.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:37:22.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260507 17:37:22.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:22.695 | INFO | Uploaded 36GiB to GPU in 2.4189s (14.8828GiB/s) | 20260507 17:37:22.695 | INFO | Downloaded 34GiB to GPU in 2.50526s (13.5715GiB/s) | 20260507 17:37:27.840 | INFO | Analyzing chunk frames = [1346560,1347072) | 20260507 17:37:27.840 | INFO | MemInfo: { | 20260507 17:37:27.840 | INFO | "gpu" : " 17 GB", | 20260507 17:37:27.840 | INFO | "huge" : "", | 20260507 17:37:27.840 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:27.840 | INFO | "malloc" : " 14 GB", | 20260507 17:37:27.840 | INFO | "pinned" : " 29 GB" | 20260507 17:37:27.840 | INFO | } | 20260507 17:37:27.840 | INFO | Perf report for the last 5.14543 seconds: | 20260507 17:37:27.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:27.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.840 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:37:27.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.840 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260507 17:37:27.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:27.841 | INFO | Uploaded 36GiB to GPU in 2.44357s (14.7325GiB/s) | 20260507 17:37:27.841 | INFO | Downloaded 34GiB to GPU in 2.43126s (13.9845GiB/s) | 20260507 17:37:32.972 | INFO | Analyzing chunk frames = [1347072,1347584) | 20260507 17:37:32.973 | INFO | MemInfo: { | 20260507 17:37:32.973 | INFO | "gpu" : " 17 GB", | 20260507 17:37:32.973 | INFO | "huge" : "", | 20260507 17:37:32.973 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:32.973 | INFO | "malloc" : " 14 GB", | 20260507 17:37:32.973 | INFO | "pinned" : " 29 GB" | 20260507 17:37:32.973 | INFO | } | 20260507 17:37:32.973 | INFO | Perf report for the last 5.13297 seconds: | 20260507 17:37:32.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:32.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260507 17:37:32.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 17:37:32.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:32.973 | INFO | Uploaded 36GiB to GPU in 2.48618s (14.48GiB/s) | 20260507 17:37:32.973 | INFO | Downloaded 34GiB to GPU in 2.51078s (13.5416GiB/s) | 20260507 17:37:33.075 | INFO | Starting a round of Baz IO | 20260507 17:37:36.593 | INFO | Populating RTMetrics report | 20260507 17:37:36.594 | INFO | RTMetrics report populated | 20260507 17:37:36.594 | INFO | Opening tmp RTMetrics json file | 20260507 17:37:36.607 | INFO | Serializing RTMetrics to json | 20260507 17:37:36.609 | INFO | Serialization done, swapping tmp file now... | 20260507 17:37:36.609 | INFO | RTMetrics done | 20260507 17:37:36.673 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.598 seconds | 20260507 17:37:36.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.85% / 0.02% / 52.13% | 20260507 17:37:36.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.47079% / 3.17891e-05% / 5.16574e-05% | 20260507 17:37:36.673 | INFO | Baz IO overall: | 20260507 17:37:36.673 | INFO | BytesWritten : 594956396253 | 20260507 17:37:36.673 | INFO | NumEvents : 288302452480 | 20260507 17:37:38.112 | INFO | Analyzing chunk frames = [1347584,1348096) | 20260507 17:37:38.112 | INFO | MemInfo: { | 20260507 17:37:38.112 | INFO | "gpu" : " 17 GB", | 20260507 17:37:38.112 | INFO | "huge" : "", | 20260507 17:37:38.112 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:38.112 | INFO | "malloc" : " 14 GB", | 20260507 17:37:38.112 | INFO | "pinned" : " 29 GB" | 20260507 17:37:38.112 | INFO | } | 20260507 17:37:38.112 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:37:38.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:38.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:38.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:38.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:38.112 | INFO | ANALYSIS : 69.8%, 3.0, 108.8ms, 0 | 20260507 17:37:38.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:38.112 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260507 17:37:38.112 | INFO | PRE_HQ : 67.6%, 1.9, 66.7ms, 0 | 20260507 17:37:38.112 | INFO | BAZWRITER : 69.5%, 3.1, 110.7ms, 0 | 20260507 17:37:38.112 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:37:38.112 | INFO | Uploaded 36GiB to GPU in 2.52271s (14.2704GiB/s) | 20260507 17:37:38.113 | INFO | Downloaded 38GiB to GPU in 2.73498s (13.8941GiB/s) | 20260507 17:37:43.251 | INFO | Analyzing chunk frames = [1348096,1348608) | 20260507 17:37:43.252 | INFO | MemInfo: { | 20260507 17:37:43.252 | INFO | "gpu" : " 17 GB", | 20260507 17:37:43.252 | INFO | "huge" : "", | 20260507 17:37:43.252 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:43.252 | INFO | "malloc" : " 14 GB", | 20260507 17:37:43.252 | INFO | "pinned" : " 29 GB" | 20260507 17:37:43.252 | INFO | } | 20260507 17:37:43.252 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:37:43.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:43.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.252 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:37:43.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.252 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 17:37:43.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:43.253 | INFO | Uploaded 36GiB to GPU in 2.54739s (14.1321GiB/s) | 20260507 17:37:43.253 | INFO | Downloaded 34GiB to GPU in 2.45416s (13.854GiB/s) | 20260507 17:37:48.390 | INFO | Analyzing chunk frames = [1348608,1349120) | 20260507 17:37:48.390 | INFO | MemInfo: { | 20260507 17:37:48.390 | INFO | "gpu" : " 17 GB", | 20260507 17:37:48.390 | INFO | "huge" : "", | 20260507 17:37:48.390 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:48.390 | INFO | "malloc" : " 14 GB", | 20260507 17:37:48.390 | INFO | "pinned" : " 29 GB" | 20260507 17:37:48.391 | INFO | } | 20260507 17:37:48.391 | INFO | Perf report for the last 5.13861 seconds: | 20260507 17:37:48.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:48.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 17:37:48.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:37:48.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:48.391 | INFO | Uploaded 36GiB to GPU in 2.54015s (14.1724GiB/s) | 20260507 17:37:48.392 | INFO | Downloaded 34GiB to GPU in 2.45047s (13.8749GiB/s) | 20260507 17:37:53.529 | INFO | Analyzing chunk frames = [1349120,1349632) | 20260507 17:37:53.530 | INFO | MemInfo: { | 20260507 17:37:53.530 | INFO | "gpu" : " 17 GB", | 20260507 17:37:53.530 | INFO | "huge" : "", | 20260507 17:37:53.530 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:53.530 | INFO | "malloc" : " 14 GB", | 20260507 17:37:53.530 | INFO | "pinned" : " 29 GB" | 20260507 17:37:53.530 | INFO | } | 20260507 17:37:53.530 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:37:53.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:53.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.530 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:37:53.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.530 | INFO | PRE_HQ : 36.7%, 1.0, 19.0ms, 0 | 20260507 17:37:53.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:53.531 | INFO | Uploaded 36GiB to GPU in 2.49631s (14.4213GiB/s) | 20260507 17:37:53.531 | INFO | Downloaded 34GiB to GPU in 2.36918s (14.351GiB/s) | 20260507 17:37:58.669 | INFO | Analyzing chunk frames = [1349632,1350144) | 20260507 17:37:58.669 | INFO | MemInfo: { | 20260507 17:37:58.669 | INFO | "gpu" : " 17 GB", | 20260507 17:37:58.669 | INFO | "huge" : "", | 20260507 17:37:58.669 | INFO | "huge_shared" : " 29 GB", | 20260507 17:37:58.669 | INFO | "malloc" : " 14 GB", | 20260507 17:37:58.669 | INFO | "pinned" : " 29 GB" | 20260507 17:37:58.669 | INFO | } | 20260507 17:37:58.669 | INFO | Perf report for the last 5.13911 seconds: | 20260507 17:37:58.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:37:58.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.669 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 17:37:58.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.669 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 17:37:58.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:37:58.670 | INFO | Uploaded 36GiB to GPU in 2.36295s (15.2352GiB/s) | 20260507 17:37:58.670 | INFO | Downloaded 34GiB to GPU in 2.47967s (13.7115GiB/s) | 20260507 17:38:03.808 | INFO | Analyzing chunk frames = [1350144,1350656) | 20260507 17:38:03.808 | INFO | MemInfo: { | 20260507 17:38:03.808 | INFO | "gpu" : " 17 GB", | 20260507 17:38:03.808 | INFO | "huge" : "", | 20260507 17:38:03.808 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:03.809 | INFO | "malloc" : " 14 GB", | 20260507 17:38:03.809 | INFO | "pinned" : " 29 GB" | 20260507 17:38:03.809 | INFO | } | 20260507 17:38:03.809 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:38:03.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:03.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 17:38:03.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260507 17:38:03.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:03.809 | INFO | Uploaded 36GiB to GPU in 2.48285s (14.4995GiB/s) | 20260507 17:38:03.809 | INFO | Downloaded 34GiB to GPU in 2.51398s (13.5244GiB/s) | 20260507 17:38:08.948 | INFO | Analyzing chunk frames = [1350656,1351168) | 20260507 17:38:08.948 | INFO | MemInfo: { | 20260507 17:38:08.948 | INFO | "gpu" : " 17 GB", | 20260507 17:38:08.948 | INFO | "huge" : "", | 20260507 17:38:08.948 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:08.948 | INFO | "malloc" : " 14 GB", | 20260507 17:38:08.948 | INFO | "pinned" : " 29 GB" | 20260507 17:38:08.948 | INFO | } | 20260507 17:38:08.948 | INFO | Perf report for the last 5.13939 seconds: | 20260507 17:38:08.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:08.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:38:08.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:38:08.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:08.948 | INFO | Uploaded 36GiB to GPU in 2.45402s (14.6698GiB/s) | 20260507 17:38:08.948 | INFO | Downloaded 34GiB to GPU in 2.42948s (13.9948GiB/s) | 20260507 17:38:14.087 | INFO | Analyzing chunk frames = [1351168,1351680) | 20260507 17:38:14.087 | INFO | MemInfo: { | 20260507 17:38:14.087 | INFO | "gpu" : " 17 GB", | 20260507 17:38:14.087 | INFO | "huge" : "", | 20260507 17:38:14.087 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:14.087 | INFO | "malloc" : " 14 GB", | 20260507 17:38:14.087 | INFO | "pinned" : " 29 GB" | 20260507 17:38:14.087 | INFO | } | 20260507 17:38:14.087 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:38:14.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:14.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.087 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 17:38:14.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.087 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:38:14.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:14.088 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:38:14.088 | INFO | Total mem usage: 16.4 GiB | 20260507 17:38:14.088 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:38:14.088 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.7%) | 20260507 17:38:14.088 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:38:14.088 | INFO | High Water Breakdown: | 20260507 17:38:14.088 | INFO | 59.2% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:38:14.088 | INFO | 40.7% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:38:14.088 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:38:14.088 | INFO | ------------------------------------------ | 20260507 17:38:14.088 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:38:14.088 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:38:14.088 | INFO | Total mem usage: 26.5 GiB | 20260507 17:38:14.088 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:38:14.088 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:38:14.088 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:38:14.088 | INFO | High Water Breakdown: | 20260507 17:38:14.088 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:38:14.088 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:38:14.088 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:38:14.088 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:38:14.088 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:38:14.088 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:38:14.088 | INFO | Uploaded 36GiB to GPU in 2.52576s (14.2531GiB/s) | 20260507 17:38:14.088 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:38:14.088 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:38:14.088 | INFO | Downloaded 34GiB to GPU in 2.50395s (13.5786GiB/s) | 20260507 17:38:14.088 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:38:14.088 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:38:14.088 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:38:14.088 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:38:14.088 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:38:14.088 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:38:14.088 | INFO | ------------------------------------------ | 20260507 17:38:14.088 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:38:14.088 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:38:14.088 | INFO | Total mem usage: 27.8 GiB | 20260507 17:38:14.088 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:38:14.088 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:38:14.088 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:38:14.088 | INFO | High Water Breakdown: | 20260507 17:38:14.088 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:38:14.088 | INFO | ------------------------------------------ | 20260507 17:38:14.088 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:38:14.088 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:38:14.088 | INFO | Total mem usage: 15.7 GiB | 20260507 17:38:14.088 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:38:14.088 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:38:14.088 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:38:14.089 | INFO | High Water Breakdown: | 20260507 17:38:14.089 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:38:14.089 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:38:14.089 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:38:14.089 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:38:14.089 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:38:14.089 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:38:14.089 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:38:14.090 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:38:14.090 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:38:14.090 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:38:14.090 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:38:14.090 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:38:14.090 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:38:14.090 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:38:14.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:38:14.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:38:14.090 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:38:14.090 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:38:14.090 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:38:14.090 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:38:14.090 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:38:14.090 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:38:14.090 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:38:14.090 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:38:14.090 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:38:14.090 | INFO | ------------------------------------------ | 20260507 17:38:14.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:38:14.090 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:38:14.090 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:38:14.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:38:14.090 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.45e+01 / 2.49e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 0.31%, AggregateStats -- 1.03e-01 / 1.06e-01, 1.28e-02 / 6.57e-03 | 20260507 17:38:14.090 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.53e+00, 6.22e-01 / 1.96e-01 | 20260507 17:38:14.090 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 1.03e-01 | 20260507 17:38:14.090 | INFO | PerfReport: 15.71%, DME -- 4.21e+01 / 4.50e+01, 4.28e+00 / 2.63e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 55.85%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.82e+00 / 2.16e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.55e+00 / 2.86e+00, 4.40e-01 / 2.33e-01 | 20260507 17:38:14.090 | INFO | PerfReport: 3.78%, Metrics -- 1.27e+00 / 1.43e+00, 3.13e-01 / 1.86e-01 | 20260507 17:38:14.090 | INFO | PerfReport: 1.21%, PulseSerializing -- 4.05e-01 / 4.30e-01, 6.51e-02 / 4.00e-02 | 20260507 17:38:14.090 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:38:14.090 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:38:14.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:38:14.090 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 24.09%, Upload -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 24.08%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 31.10%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 1.76%, LockUpload -- 1.89e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 2.37%, LockDownload -- 2.55e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 16.59%, LockCompute -- 1.79e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:38:14.090 | INFO | PerfReport: 0.02%, Misc -- 1.90e-02 / 0.00e+00, 2.81e-02 / 0.00e+00 | 20260507 17:38:14.184 | INFO | Starting a round of Baz IO | 20260507 17:38:17.696 | INFO | Populating RTMetrics report | 20260507 17:38:17.696 | INFO | RTMetrics report populated | 20260507 17:38:17.696 | INFO | Opening tmp RTMetrics json file | 20260507 17:38:17.710 | INFO | Serializing RTMetrics to json | 20260507 17:38:17.711 | INFO | Serialization done, swapping tmp file now... | 20260507 17:38:17.712 | INFO | RTMetrics done | 20260507 17:38:17.773 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.588 seconds | 20260507 17:38:17.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.83% / 0.02% / 52.15% | 20260507 17:38:17.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.46927% / 3.57628e-05% / 2.38419e-05% | 20260507 17:38:17.773 | INFO | Baz IO overall: | 20260507 17:38:17.774 | INFO | BytesWritten : 596891083782 | 20260507 17:38:17.774 | INFO | NumEvents : 289243362495 | 20260507 17:38:19.226 | INFO | Analyzing chunk frames = [1351680,1352192) | 20260507 17:38:19.226 | INFO | MemInfo: { | 20260507 17:38:19.226 | INFO | "gpu" : " 17 GB", | 20260507 17:38:19.226 | INFO | "huge" : "", | 20260507 17:38:19.226 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:19.226 | INFO | "malloc" : " 14 GB", | 20260507 17:38:19.226 | INFO | "pinned" : " 29 GB" | 20260507 17:38:19.226 | INFO | } | 20260507 17:38:19.226 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:38:19.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:19.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:19.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:19.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:19.226 | INFO | ANALYSIS : 69.6%, 3.0, 108.6ms, 0 | 20260507 17:38:19.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:19.226 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 17:38:19.226 | INFO | PRE_HQ : 66.8%, 1.8, 61.3ms, 0 | 20260507 17:38:19.227 | INFO | BAZWRITER : 69.0%, 2.9, 104.2ms, 0 | 20260507 17:38:19.227 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:38:19.227 | INFO | Uploaded 36GiB to GPU in 2.50619s (14.3644GiB/s) | 20260507 17:38:19.227 | INFO | Downloaded 38GiB to GPU in 2.73043s (13.9172GiB/s) | 20260507 17:38:24.365 | INFO | Analyzing chunk frames = [1352192,1352704) | 20260507 17:38:24.365 | INFO | MemInfo: { | 20260507 17:38:24.365 | INFO | "gpu" : " 17 GB", | 20260507 17:38:24.365 | INFO | "huge" : "", | 20260507 17:38:24.365 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:24.365 | INFO | "malloc" : " 14 GB", | 20260507 17:38:24.365 | INFO | "pinned" : " 29 GB" | 20260507 17:38:24.366 | INFO | } | 20260507 17:38:24.366 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:38:24.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:24.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:38:24.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 17:38:24.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:24.366 | INFO | Uploaded 36GiB to GPU in 2.50879s (14.3496GiB/s) | 20260507 17:38:24.366 | INFO | Downloaded 34GiB to GPU in 2.44157s (13.9255GiB/s) | 20260507 17:38:29.505 | INFO | Analyzing chunk frames = [1352704,1353216) | 20260507 17:38:29.505 | INFO | MemInfo: { | 20260507 17:38:29.505 | INFO | "gpu" : " 17 GB", | 20260507 17:38:29.505 | INFO | "huge" : "", | 20260507 17:38:29.505 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:29.505 | INFO | "malloc" : " 14 GB", | 20260507 17:38:29.505 | INFO | "pinned" : " 29 GB" | 20260507 17:38:29.505 | INFO | } | 20260507 17:38:29.505 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:38:29.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:29.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:38:29.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:38:29.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:29.505 | INFO | Uploaded 36GiB to GPU in 2.55911s (14.0674GiB/s) | 20260507 17:38:29.505 | INFO | Downloaded 34GiB to GPU in 2.43321s (13.9733GiB/s) | 20260507 17:38:34.644 | INFO | Analyzing chunk frames = [1353216,1353728) | 20260507 17:38:34.644 | INFO | MemInfo: { | 20260507 17:38:34.644 | INFO | "gpu" : " 17 GB", | 20260507 17:38:34.644 | INFO | "huge" : "", | 20260507 17:38:34.644 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:34.644 | INFO | "malloc" : " 14 GB", | 20260507 17:38:34.644 | INFO | "pinned" : " 29 GB" | 20260507 17:38:34.644 | INFO | } | 20260507 17:38:34.644 | INFO | Perf report for the last 5.13902 seconds: | 20260507 17:38:34.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:34.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:38:34.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:38:34.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:34.644 | INFO | Uploaded 36GiB to GPU in 2.49405s (14.4344GiB/s) | 20260507 17:38:34.644 | INFO | Downloaded 34GiB to GPU in 2.35057s (14.4646GiB/s) | 20260507 17:38:39.784 | INFO | Analyzing chunk frames = [1353728,1354240) | 20260507 17:38:39.784 | INFO | MemInfo: { | 20260507 17:38:39.784 | INFO | "gpu" : " 17 GB", | 20260507 17:38:39.784 | INFO | "huge" : "", | 20260507 17:38:39.784 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:39.784 | INFO | "malloc" : " 14 GB", | 20260507 17:38:39.784 | INFO | "pinned" : " 29 GB" | 20260507 17:38:39.784 | INFO | } | 20260507 17:38:39.784 | INFO | Perf report for the last 5.14009 seconds: | 20260507 17:38:39.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:39.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.784 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:38:39.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.784 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 17:38:39.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:39.785 | INFO | Uploaded 36GiB to GPU in 2.35004s (15.3189GiB/s) | 20260507 17:38:39.785 | INFO | Downloaded 34GiB to GPU in 2.45764s (13.8344GiB/s) | 20260507 17:38:44.923 | INFO | Analyzing chunk frames = [1354240,1354752) | 20260507 17:38:44.923 | INFO | MemInfo: { | 20260507 17:38:44.923 | INFO | "gpu" : " 17 GB", | 20260507 17:38:44.923 | INFO | "huge" : "", | 20260507 17:38:44.923 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:44.923 | INFO | "malloc" : " 14 GB", | 20260507 17:38:44.923 | INFO | "pinned" : " 29 GB" | 20260507 17:38:44.923 | INFO | } | 20260507 17:38:44.923 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:38:44.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:44.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.923 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:38:44.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.923 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 17:38:44.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:44.924 | INFO | Uploaded 36GiB to GPU in 2.46457s (14.607GiB/s) | 20260507 17:38:44.924 | INFO | Downloaded 34GiB to GPU in 2.51291s (13.5301GiB/s) | 20260507 17:38:50.062 | INFO | Analyzing chunk frames = [1354752,1355264) | 20260507 17:38:50.062 | INFO | MemInfo: { | 20260507 17:38:50.062 | INFO | "gpu" : " 17 GB", | 20260507 17:38:50.062 | INFO | "huge" : "", | 20260507 17:38:50.062 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:50.063 | INFO | "malloc" : " 14 GB", | 20260507 17:38:50.063 | INFO | "pinned" : " 29 GB" | 20260507 17:38:50.063 | INFO | } | 20260507 17:38:50.063 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:38:50.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:50.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:38:50.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | PRE_HQ : 38.5%, 1.0, 20.0ms, 0 | 20260507 17:38:50.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:50.063 | INFO | Uploaded 36GiB to GPU in 2.49946s (14.4031GiB/s) | 20260507 17:38:50.063 | INFO | Downloaded 34GiB to GPU in 2.43462s (13.9652GiB/s) | 20260507 17:38:55.202 | INFO | Analyzing chunk frames = [1355264,1355776) | 20260507 17:38:55.202 | INFO | MemInfo: { | 20260507 17:38:55.202 | INFO | "gpu" : " 17 GB", | 20260507 17:38:55.202 | INFO | "huge" : "", | 20260507 17:38:55.202 | INFO | "huge_shared" : " 29 GB", | 20260507 17:38:55.202 | INFO | "malloc" : " 14 GB", | 20260507 17:38:55.202 | INFO | "pinned" : " 29 GB" | 20260507 17:38:55.202 | INFO | } | 20260507 17:38:55.202 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:38:55.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:38:55.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.202 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:38:55.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.202 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 17:38:55.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:38:55.203 | INFO | Uploaded 36GiB to GPU in 2.46556s (14.6011GiB/s) | 20260507 17:38:55.203 | INFO | Downloaded 34GiB to GPU in 2.4178s (14.0624GiB/s) | 20260507 17:38:55.315 | INFO | Starting a round of Baz IO | 20260507 17:38:58.812 | INFO | Populating RTMetrics report | 20260507 17:38:58.812 | INFO | RTMetrics report populated | 20260507 17:38:58.812 | INFO | Opening tmp RTMetrics json file | 20260507 17:38:58.829 | INFO | Serializing RTMetrics to json | 20260507 17:38:58.831 | INFO | Serialization done, swapping tmp file now... | 20260507 17:38:58.831 | INFO | RTMetrics done | 20260507 17:38:58.894 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.578 seconds | 20260507 17:38:58.894 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.81% / 0.02% / 52.17% | 20260507 17:38:58.894 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.44828% / 1.98682e-05% / 2.38419e-05% | 20260507 17:38:58.894 | INFO | Baz IO overall: | 20260507 17:38:58.894 | INFO | BytesWritten : 598823245526 | 20260507 17:38:58.894 | INFO | NumEvents : 290182179628 | 20260507 17:39:00.342 | INFO | Analyzing chunk frames = [1355776,1356288) | 20260507 17:39:00.342 | INFO | MemInfo: { | 20260507 17:39:00.342 | INFO | "gpu" : " 17 GB", | 20260507 17:39:00.342 | INFO | "huge" : "", | 20260507 17:39:00.342 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:00.342 | INFO | "malloc" : " 14 GB", | 20260507 17:39:00.342 | INFO | "pinned" : " 29 GB" | 20260507 17:39:00.342 | INFO | } | 20260507 17:39:00.342 | INFO | Perf report for the last 5.13971 seconds: | 20260507 17:39:00.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:00.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:00.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:00.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:00.342 | INFO | ANALYSIS : 69.5%, 3.0, 108.6ms, 0 | 20260507 17:39:00.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:00.342 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 17:39:00.342 | INFO | PRE_HQ : 67.1%, 1.8, 62.6ms, 0 | 20260507 17:39:00.342 | INFO | BAZWRITER : 68.6%, 3.0, 104.4ms, 0 | 20260507 17:39:00.342 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:39:00.343 | INFO | Uploaded 36GiB to GPU in 2.51056s (14.3394GiB/s) | 20260507 17:39:00.343 | INFO | Downloaded 38GiB to GPU in 2.71827s (13.9795GiB/s) | 20260507 17:39:05.480 | INFO | Analyzing chunk frames = [1356288,1356800) | 20260507 17:39:05.480 | INFO | MemInfo: { | 20260507 17:39:05.480 | INFO | "gpu" : " 17 GB", | 20260507 17:39:05.480 | INFO | "huge" : "", | 20260507 17:39:05.480 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:05.480 | INFO | "malloc" : " 14 GB", | 20260507 17:39:05.480 | INFO | "pinned" : " 29 GB" | 20260507 17:39:05.480 | INFO | } | 20260507 17:39:05.480 | INFO | Perf report for the last 5.13821 seconds: | 20260507 17:39:05.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:05.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.480 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:39:05.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.481 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:39:05.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:05.481 | INFO | Uploaded 36GiB to GPU in 2.48103s (14.5101GiB/s) | 20260507 17:39:05.481 | INFO | Downloaded 34GiB to GPU in 2.47267s (13.7503GiB/s) | 20260507 17:39:10.620 | INFO | Analyzing chunk frames = [1356800,1357312) | 20260507 17:39:10.620 | INFO | MemInfo: { | 20260507 17:39:10.620 | INFO | "gpu" : " 17 GB", | 20260507 17:39:10.620 | INFO | "huge" : "", | 20260507 17:39:10.620 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:10.620 | INFO | "malloc" : " 14 GB", | 20260507 17:39:10.620 | INFO | "pinned" : " 29 GB" | 20260507 17:39:10.620 | INFO | } | 20260507 17:39:10.620 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:39:10.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:10.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 17:39:10.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 17:39:10.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:10.620 | INFO | Uploaded 36GiB to GPU in 2.48283s (14.4996GiB/s) | 20260507 17:39:10.621 | INFO | Downloaded 34GiB to GPU in 2.46183s (13.8109GiB/s) | 20260507 17:39:15.759 | INFO | Analyzing chunk frames = [1357312,1357824) | 20260507 17:39:15.759 | INFO | MemInfo: { | 20260507 17:39:15.759 | INFO | "gpu" : " 17 GB", | 20260507 17:39:15.759 | INFO | "huge" : "", | 20260507 17:39:15.759 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:15.759 | INFO | "malloc" : " 14 GB", | 20260507 17:39:15.759 | INFO | "pinned" : " 29 GB" | 20260507 17:39:15.760 | INFO | } | 20260507 17:39:15.760 | INFO | Perf report for the last 5.13994 seconds: | 20260507 17:39:15.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:15.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 17:39:15.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:39:15.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:15.760 | INFO | Uploaded 36GiB to GPU in 2.45886s (14.6409GiB/s) | 20260507 17:39:15.760 | INFO | Downloaded 34GiB to GPU in 2.37285s (14.3287GiB/s) | 20260507 17:39:20.899 | INFO | Analyzing chunk frames = [1357824,1358336) | 20260507 17:39:20.899 | INFO | MemInfo: { | 20260507 17:39:20.899 | INFO | "gpu" : " 17 GB", | 20260507 17:39:20.899 | INFO | "huge" : "", | 20260507 17:39:20.899 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:20.899 | INFO | "malloc" : " 14 GB", | 20260507 17:39:20.899 | INFO | "pinned" : " 29 GB" | 20260507 17:39:20.899 | INFO | } | 20260507 17:39:20.899 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:39:20.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:20.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:39:20.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 17:39:20.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:20.899 | INFO | Uploaded 36GiB to GPU in 2.31893s (15.5244GiB/s) | 20260507 17:39:20.900 | INFO | Downloaded 34GiB to GPU in 2.50563s (13.5694GiB/s) | 20260507 17:39:26.038 | INFO | Analyzing chunk frames = [1358336,1358848) | 20260507 17:39:26.039 | INFO | MemInfo: { | 20260507 17:39:26.039 | INFO | "gpu" : " 17 GB", | 20260507 17:39:26.039 | INFO | "huge" : "", | 20260507 17:39:26.039 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:26.039 | INFO | "malloc" : " 14 GB", | 20260507 17:39:26.039 | INFO | "pinned" : " 29 GB" | 20260507 17:39:26.039 | INFO | } | 20260507 17:39:26.039 | INFO | Perf report for the last 5.14003 seconds: | 20260507 17:39:26.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:26.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:39:26.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 17:39:26.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:26.039 | INFO | Uploaded 36GiB to GPU in 2.39831s (15.0106GiB/s) | 20260507 17:39:26.039 | INFO | Downloaded 34GiB to GPU in 2.50612s (13.5668GiB/s) | 20260507 17:39:31.177 | INFO | Analyzing chunk frames = [1358848,1359360) | 20260507 17:39:31.177 | INFO | MemInfo: { | 20260507 17:39:31.177 | INFO | "gpu" : " 17 GB", | 20260507 17:39:31.177 | INFO | "huge" : "", | 20260507 17:39:31.177 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:31.177 | INFO | "malloc" : " 14 GB", | 20260507 17:39:31.178 | INFO | "pinned" : " 29 GB" | 20260507 17:39:31.178 | INFO | } | 20260507 17:39:31.178 | INFO | Perf report for the last 5.13866 seconds: | 20260507 17:39:31.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:31.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:39:31.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:39:31.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:31.178 | INFO | Uploaded 36GiB to GPU in 2.47953s (14.5189GiB/s) | 20260507 17:39:31.178 | INFO | Downloaded 34GiB to GPU in 2.44283s (13.9183GiB/s) | 20260507 17:39:36.316 | INFO | Analyzing chunk frames = [1359360,1359872) | 20260507 17:39:36.316 | INFO | MemInfo: { | 20260507 17:39:36.316 | INFO | "gpu" : " 17 GB", | 20260507 17:39:36.316 | INFO | "huge" : "", | 20260507 17:39:36.316 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:36.316 | INFO | "malloc" : " 14 GB", | 20260507 17:39:36.316 | INFO | "pinned" : " 29 GB" | 20260507 17:39:36.317 | INFO | } | 20260507 17:39:36.317 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:39:36.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:36.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.317 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 17:39:36.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.317 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 17:39:36.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:36.318 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:39:36.318 | INFO | Total mem usage: 16.4 GiB | 20260507 17:39:36.318 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:39:36.318 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:39:36.318 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:39:36.318 | INFO | High Water Breakdown: | 20260507 17:39:36.318 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:39:36.318 | INFO | 40.8% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:39:36.318 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:39:36.318 | INFO | ------------------------------------------ | 20260507 17:39:36.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:39:36.318 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:39:36.318 | INFO | Total mem usage: 26.5 GiB | 20260507 17:39:36.318 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:39:36.318 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:39:36.318 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:39:36.318 | INFO | High Water Breakdown: | 20260507 17:39:36.318 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:39:36.318 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:39:36.318 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:39:36.318 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:39:36.318 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:39:36.318 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:39:36.318 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:39:36.318 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:39:36.318 | INFO | Uploaded 36GiB to GPU in 2.4838s (14.4939GiB/s) | 20260507 17:39:36.318 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:39:36.318 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:39:36.318 | INFO | Downloaded 34GiB to GPU in 2.42771s (14.005GiB/s) | 20260507 17:39:36.318 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:39:36.318 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:39:36.318 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:39:36.318 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:39:36.318 | INFO | ------------------------------------------ | 20260507 17:39:36.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:39:36.318 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:39:36.318 | INFO | Total mem usage: 27.8 GiB | 20260507 17:39:36.318 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:39:36.318 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:39:36.318 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:39:36.318 | INFO | High Water Breakdown: | 20260507 17:39:36.318 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:39:36.318 | INFO | ------------------------------------------ | 20260507 17:39:36.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:39:36.318 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:39:36.319 | INFO | Total mem usage: 15.7 GiB | 20260507 17:39:36.319 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:39:36.319 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:39:36.319 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:39:36.319 | INFO | High Water Breakdown: | 20260507 17:39:36.319 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:39:36.319 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:39:36.319 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:39:36.319 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:39:36.319 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:39:36.319 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:39:36.319 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:39:36.319 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:39:36.319 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:39:36.319 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:39:36.319 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:39:36.319 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:39:36.319 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:39:36.319 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:39:36.319 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:39:36.319 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:39:36.319 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:39:36.319 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:39:36.319 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:39:36.319 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:39:36.319 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:39:36.319 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:39:36.319 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:39:36.319 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:39:36.319 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:39:36.319 | INFO | ------------------------------------------ | 20260507 17:39:36.319 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:39:36.319 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:39:36.319 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:39:36.319 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:39:36.319 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.65e+01, 1.44e+01 / 2.59e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 0.30%, AggregateStats -- 1.02e-01 / 1.07e-01, 1.17e-02 / 5.96e-03 | 20260507 17:39:36.319 | INFO | PerfReport: 12.10%, Baseline -- 4.07e+00 / 4.51e+00, 6.18e-01 / 2.53e-01 | 20260507 17:39:36.319 | INFO | PerfReport: 3.44%, Histogram -- 1.16e+00 / 1.51e+00, 2.18e-01 / 1.24e-01 | 20260507 17:39:36.319 | INFO | PerfReport: 15.64%, DME -- 4.20e+01 / 4.47e+01, 4.16e+00 / 2.46e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.92e+00 / 1.97e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.53e+00 / 2.86e+00, 4.35e-01 / 2.85e-01 | 20260507 17:39:36.319 | INFO | PerfReport: 3.76%, Metrics -- 1.26e+00 / 1.41e+00, 3.17e-01 / 1.70e-01 | 20260507 17:39:36.319 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.04e-01 / 4.27e-01, 6.58e-02 / 4.02e-02 | 20260507 17:39:36.319 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:39:36.319 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:39:36.319 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:39:36.319 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 23.90%, Upload -- 2.57e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 24.03%, Download -- 2.58e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 31.25%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 1.67%, LockUpload -- 1.80e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 2.37%, LockDownload -- 2.55e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 16.76%, LockCompute -- 1.80e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:39:36.319 | INFO | PerfReport: 0.02%, Misc -- 1.93e-02 / 0.00e+00, 3.04e-02 / 0.00e+00 | 20260507 17:39:36.434 | INFO | Starting a round of Baz IO | 20260507 17:39:39.921 | INFO | Populating RTMetrics report | 20260507 17:39:39.921 | INFO | RTMetrics report populated | 20260507 17:39:39.921 | INFO | Opening tmp RTMetrics json file | 20260507 17:39:39.937 | INFO | Serializing RTMetrics to json | 20260507 17:39:39.938 | INFO | Serialization done, swapping tmp file now... | 20260507 17:39:39.939 | INFO | RTMetrics done | 20260507 17:39:40.001 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.566 seconds | 20260507 17:39:40.001 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.79% / 0.02% / 52.19% | 20260507 17:39:40.001 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.43876% / 4.37101e-05% / 3.97364e-05% | 20260507 17:39:40.001 | INFO | Baz IO overall: | 20260507 17:39:40.001 | INFO | BytesWritten : 600755021789 | 20260507 17:39:40.001 | INFO | NumEvents : 291120625910 | 20260507 17:39:41.456 | INFO | Analyzing chunk frames = [1359872,1360384) | 20260507 17:39:41.456 | INFO | MemInfo: { | 20260507 17:39:41.456 | INFO | "gpu" : " 17 GB", | 20260507 17:39:41.456 | INFO | "huge" : "", | 20260507 17:39:41.456 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:41.456 | INFO | "malloc" : " 14 GB", | 20260507 17:39:41.456 | INFO | "pinned" : " 29 GB" | 20260507 17:39:41.456 | INFO | } | 20260507 17:39:41.456 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:39:41.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:41.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:41.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:41.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:41.456 | INFO | ANALYSIS : 69.7%, 3.0, 108.3ms, 0 | 20260507 17:39:41.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:41.457 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 17:39:41.457 | INFO | PRE_HQ : 66.8%, 1.8, 63.0ms, 0 | 20260507 17:39:41.457 | INFO | BAZWRITER : 68.4%, 3.0, 103.9ms, 0 | 20260507 17:39:41.457 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:39:41.457 | INFO | Uploaded 36GiB to GPU in 2.56472s (14.0366GiB/s) | 20260507 17:39:41.457 | INFO | Downloaded 38GiB to GPU in 2.71664s (13.9879GiB/s) | 20260507 17:39:46.595 | INFO | Analyzing chunk frames = [1360384,1360896) | 20260507 17:39:46.595 | INFO | MemInfo: { | 20260507 17:39:46.595 | INFO | "gpu" : " 17 GB", | 20260507 17:39:46.595 | INFO | "huge" : "", | 20260507 17:39:46.595 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:46.595 | INFO | "malloc" : " 14 GB", | 20260507 17:39:46.595 | INFO | "pinned" : " 29 GB" | 20260507 17:39:46.595 | INFO | } | 20260507 17:39:46.596 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:39:46.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:46.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 17:39:46.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 17:39:46.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:46.596 | INFO | Uploaded 36GiB to GPU in 2.55166s (14.1085GiB/s) | 20260507 17:39:46.596 | INFO | Downloaded 34GiB to GPU in 2.44335s (13.9153GiB/s) | 20260507 17:39:51.734 | INFO | Analyzing chunk frames = [1360896,1361408) | 20260507 17:39:51.734 | INFO | MemInfo: { | 20260507 17:39:51.734 | INFO | "gpu" : " 17 GB", | 20260507 17:39:51.734 | INFO | "huge" : "", | 20260507 17:39:51.734 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:51.734 | INFO | "malloc" : " 14 GB", | 20260507 17:39:51.734 | INFO | "pinned" : " 29 GB" | 20260507 17:39:51.735 | INFO | } | 20260507 17:39:51.735 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:39:51.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:51.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:39:51.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:39:51.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:51.735 | INFO | Uploaded 36GiB to GPU in 2.56745s (14.0217GiB/s) | 20260507 17:39:51.736 | INFO | Downloaded 34GiB to GPU in 2.43801s (13.9458GiB/s) | 20260507 17:39:56.874 | INFO | Analyzing chunk frames = [1361408,1361920) | 20260507 17:39:56.874 | INFO | MemInfo: { | 20260507 17:39:56.874 | INFO | "gpu" : " 17 GB", | 20260507 17:39:56.874 | INFO | "huge" : "", | 20260507 17:39:56.874 | INFO | "huge_shared" : " 29 GB", | 20260507 17:39:56.874 | INFO | "malloc" : " 14 GB", | 20260507 17:39:56.874 | INFO | "pinned" : " 29 GB" | 20260507 17:39:56.874 | INFO | } | 20260507 17:39:56.874 | INFO | Perf report for the last 5.13924 seconds: | 20260507 17:39:56.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:39:56.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.874 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:39:56.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.874 | INFO | PRE_HQ : 37.8%, 1.0, 19.6ms, 0 | 20260507 17:39:56.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:39:56.875 | INFO | Uploaded 36GiB to GPU in 2.49754s (14.4142GiB/s) | 20260507 17:39:56.875 | INFO | Downloaded 34GiB to GPU in 2.35862s (14.4152GiB/s) | 20260507 17:40:02.013 | INFO | Analyzing chunk frames = [1361920,1362432) | 20260507 17:40:02.013 | INFO | MemInfo: { | 20260507 17:40:02.013 | INFO | "gpu" : " 17 GB", | 20260507 17:40:02.013 | INFO | "huge" : "", | 20260507 17:40:02.013 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:02.013 | INFO | "malloc" : " 14 GB", | 20260507 17:40:02.013 | INFO | "pinned" : " 29 GB" | 20260507 17:40:02.014 | INFO | } | 20260507 17:40:02.014 | INFO | Perf report for the last 5.13964 seconds: | 20260507 17:40:02.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:02.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:40:02.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 17:40:02.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:02.014 | INFO | Uploaded 36GiB to GPU in 2.3523s (15.3042GiB/s) | 20260507 17:40:02.014 | INFO | Downloaded 34GiB to GPU in 2.49157s (13.646GiB/s) | 20260507 17:40:07.152 | INFO | Analyzing chunk frames = [1362432,1362944) | 20260507 17:40:07.153 | INFO | MemInfo: { | 20260507 17:40:07.153 | INFO | "gpu" : " 17 GB", | 20260507 17:40:07.153 | INFO | "huge" : "", | 20260507 17:40:07.153 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:07.153 | INFO | "malloc" : " 14 GB", | 20260507 17:40:07.153 | INFO | "pinned" : " 29 GB" | 20260507 17:40:07.153 | INFO | } | 20260507 17:40:07.153 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:40:07.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:07.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:40:07.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 17:40:07.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:07.153 | INFO | Uploaded 36GiB to GPU in 2.43971s (14.7559GiB/s) | 20260507 17:40:07.153 | INFO | Downloaded 34GiB to GPU in 2.48578s (13.6778GiB/s) | 20260507 17:40:12.292 | INFO | Analyzing chunk frames = [1362944,1363456) | 20260507 17:40:12.292 | INFO | MemInfo: { | 20260507 17:40:12.292 | INFO | "gpu" : " 17 GB", | 20260507 17:40:12.292 | INFO | "huge" : "", | 20260507 17:40:12.292 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:12.292 | INFO | "malloc" : " 14 GB", | 20260507 17:40:12.292 | INFO | "pinned" : " 29 GB" | 20260507 17:40:12.292 | INFO | } | 20260507 17:40:12.292 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:40:12.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:12.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.292 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:40:12.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.292 | INFO | PRE_HQ : 37.5%, 1.0, 19.6ms, 0 | 20260507 17:40:12.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:12.293 | INFO | Uploaded 36GiB to GPU in 2.52601s (14.2517GiB/s) | 20260507 17:40:12.293 | INFO | Downloaded 34GiB to GPU in 2.43572s (13.9589GiB/s) | 20260507 17:40:17.431 | INFO | Analyzing chunk frames = [1363456,1363968) | 20260507 17:40:17.431 | INFO | MemInfo: { | 20260507 17:40:17.431 | INFO | "gpu" : " 17 GB", | 20260507 17:40:17.431 | INFO | "huge" : "", | 20260507 17:40:17.431 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:17.431 | INFO | "malloc" : " 14 GB", | 20260507 17:40:17.431 | INFO | "pinned" : " 29 GB" | 20260507 17:40:17.431 | INFO | } | 20260507 17:40:17.432 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:40:17.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:17.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 17:40:17.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 17:40:17.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:17.432 | INFO | Uploaded 36GiB to GPU in 2.49411s (14.434GiB/s) | 20260507 17:40:17.432 | INFO | Downloaded 34GiB to GPU in 2.41485s (14.0796GiB/s) | 20260507 17:40:17.543 | INFO | Starting a round of Baz IO | 20260507 17:40:21.056 | INFO | Populating RTMetrics report | 20260507 17:40:21.057 | INFO | RTMetrics report populated | 20260507 17:40:21.057 | INFO | Opening tmp RTMetrics json file | 20260507 17:40:21.058 | INFO | Serializing RTMetrics to json | 20260507 17:40:21.059 | INFO | Serialization done, swapping tmp file now... | 20260507 17:40:21.059 | INFO | RTMetrics done | 20260507 17:40:21.143 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.6 seconds | 20260507 17:40:21.143 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.77% / 0.02% / 52.21% | 20260507 17:40:21.143 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.42857% / 5.16574e-05% / 3.57628e-05% | 20260507 17:40:21.143 | INFO | Baz IO overall: | 20260507 17:40:21.144 | INFO | BytesWritten : 602686032523 | 20260507 17:40:21.144 | INFO | NumEvents : 292058563389 | 20260507 17:40:22.573 | INFO | Analyzing chunk frames = [1363968,1364480) | 20260507 17:40:22.573 | INFO | MemInfo: { | 20260507 17:40:22.573 | INFO | "gpu" : " 17 GB", | 20260507 17:40:22.573 | INFO | "huge" : "", | 20260507 17:40:22.573 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:22.573 | INFO | "malloc" : " 14 GB", | 20260507 17:40:22.574 | INFO | "pinned" : " 29 GB" | 20260507 17:40:22.574 | INFO | } | 20260507 17:40:22.574 | INFO | Perf report for the last 5.14209 seconds: | 20260507 17:40:22.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:22.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:22.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:22.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:22.574 | INFO | ANALYSIS : 69.2%, 2.9, 106.3ms, 0 | 20260507 17:40:22.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:22.574 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260507 17:40:22.574 | INFO | PRE_HQ : 67.5%, 1.9, 67.9ms, 0 | 20260507 17:40:22.574 | INFO | BAZWRITER : 69.1%, 3.1, 108.7ms, 0 | 20260507 17:40:22.574 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:40:22.574 | INFO | Uploaded 36GiB to GPU in 2.56448s (14.0379GiB/s) | 20260507 17:40:22.574 | INFO | Downloaded 38GiB to GPU in 2.68s (14.1791GiB/s) | 20260507 17:40:27.714 | INFO | Analyzing chunk frames = [1364480,1364992) | 20260507 17:40:27.714 | INFO | MemInfo: { | 20260507 17:40:27.714 | INFO | "gpu" : " 17 GB", | 20260507 17:40:27.714 | INFO | "huge" : "", | 20260507 17:40:27.714 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:27.714 | INFO | "malloc" : " 14 GB", | 20260507 17:40:27.714 | INFO | "pinned" : " 29 GB" | 20260507 17:40:27.715 | INFO | } | 20260507 17:40:27.715 | INFO | Perf report for the last 5.14097 seconds: | 20260507 17:40:27.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:27.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 17:40:27.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 17:40:27.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:27.715 | INFO | Uploaded 36GiB to GPU in 2.53925s (14.1774GiB/s) | 20260507 17:40:27.715 | INFO | Downloaded 34GiB to GPU in 2.42206s (14.0376GiB/s) | 20260507 17:40:32.849 | INFO | Analyzing chunk frames = [1364992,1365504) | 20260507 17:40:32.849 | INFO | MemInfo: { | 20260507 17:40:32.849 | INFO | "gpu" : " 17 GB", | 20260507 17:40:32.849 | INFO | "huge" : "", | 20260507 17:40:32.849 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:32.849 | INFO | "malloc" : " 14 GB", | 20260507 17:40:32.849 | INFO | "pinned" : " 29 GB" | 20260507 17:40:32.849 | INFO | } | 20260507 17:40:32.849 | INFO | Perf report for the last 5.13481 seconds: | 20260507 17:40:32.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:32.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.850 | INFO | ANALYSIS : 68.5%, 3.0, 106.1ms, 0 | 20260507 17:40:32.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.850 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260507 17:40:32.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:32.850 | INFO | Uploaded 36GiB to GPU in 2.59076s (13.8956GiB/s) | 20260507 17:40:32.850 | INFO | Downloaded 34GiB to GPU in 2.40262s (14.1512GiB/s) | 20260507 17:40:37.988 | INFO | Analyzing chunk frames = [1365504,1366016) | 20260507 17:40:37.988 | INFO | MemInfo: { | 20260507 17:40:37.988 | INFO | "gpu" : " 17 GB", | 20260507 17:40:37.988 | INFO | "huge" : "", | 20260507 17:40:37.988 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:37.988 | INFO | "malloc" : " 14 GB", | 20260507 17:40:37.988 | INFO | "pinned" : " 29 GB" | 20260507 17:40:37.988 | INFO | } | 20260507 17:40:37.988 | INFO | Perf report for the last 5.13888 seconds: | 20260507 17:40:37.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:37.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.988 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:40:37.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.988 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:40:37.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:37.989 | INFO | Uploaded 36GiB to GPU in 2.50525s (14.3698GiB/s) | 20260507 17:40:37.989 | INFO | Downloaded 34GiB to GPU in 2.34952s (14.471GiB/s) | 20260507 17:40:43.128 | INFO | Analyzing chunk frames = [1366016,1366528) | 20260507 17:40:43.128 | INFO | MemInfo: { | 20260507 17:40:43.128 | INFO | "gpu" : " 17 GB", | 20260507 17:40:43.128 | INFO | "huge" : "", | 20260507 17:40:43.128 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:43.128 | INFO | "malloc" : " 14 GB", | 20260507 17:40:43.128 | INFO | "pinned" : " 29 GB" | 20260507 17:40:43.128 | INFO | } | 20260507 17:40:43.128 | INFO | Perf report for the last 5.13956 seconds: | 20260507 17:40:43.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:43.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:40:43.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:40:43.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:43.128 | INFO | Uploaded 36GiB to GPU in 2.35418s (15.2919GiB/s) | 20260507 17:40:43.129 | INFO | Downloaded 34GiB to GPU in 2.44804s (13.8887GiB/s) | 20260507 17:40:48.267 | INFO | Analyzing chunk frames = [1366528,1367040) | 20260507 17:40:48.267 | INFO | MemInfo: { | 20260507 17:40:48.267 | INFO | "gpu" : " 17 GB", | 20260507 17:40:48.267 | INFO | "huge" : "", | 20260507 17:40:48.267 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:48.267 | INFO | "malloc" : " 14 GB", | 20260507 17:40:48.267 | INFO | "pinned" : " 29 GB" | 20260507 17:40:48.267 | INFO | } | 20260507 17:40:48.267 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:40:48.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:48.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:40:48.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:40:48.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:48.268 | INFO | Uploaded 36GiB to GPU in 2.41765s (14.8905GiB/s) | 20260507 17:40:48.268 | INFO | Downloaded 34GiB to GPU in 2.4987s (13.6071GiB/s) | 20260507 17:40:53.406 | INFO | Analyzing chunk frames = [1367040,1367552) | 20260507 17:40:53.406 | INFO | MemInfo: { | 20260507 17:40:53.406 | INFO | "gpu" : " 17 GB", | 20260507 17:40:53.406 | INFO | "huge" : "", | 20260507 17:40:53.406 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:53.406 | INFO | "malloc" : " 14 GB", | 20260507 17:40:53.406 | INFO | "pinned" : " 29 GB" | 20260507 17:40:53.407 | INFO | } | 20260507 17:40:53.407 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:40:53.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:53.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:40:53.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 17:40:53.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:53.407 | INFO | Uploaded 36GiB to GPU in 2.45565s (14.6601GiB/s) | 20260507 17:40:53.407 | INFO | Downloaded 34GiB to GPU in 2.4215s (14.0409GiB/s) | 20260507 17:40:58.546 | INFO | Analyzing chunk frames = [1367552,1368064) | 20260507 17:40:58.546 | INFO | MemInfo: { | 20260507 17:40:58.546 | INFO | "gpu" : " 17 GB", | 20260507 17:40:58.546 | INFO | "huge" : "", | 20260507 17:40:58.546 | INFO | "huge_shared" : " 29 GB", | 20260507 17:40:58.546 | INFO | "malloc" : " 14 GB", | 20260507 17:40:58.546 | INFO | "pinned" : " 29 GB" | 20260507 17:40:58.546 | INFO | } | 20260507 17:40:58.546 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:40:58.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:40:58.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:40:58.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:40:58.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:40:58.546 | INFO | Uploaded 36GiB to GPU in 2.52618s (14.2508GiB/s) | 20260507 17:40:58.546 | INFO | Downloaded 34GiB to GPU in 2.50166s (13.591GiB/s) | 20260507 17:40:58.547 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:40:58.547 | INFO | Total mem usage: 16.4 GiB | 20260507 17:40:58.547 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:40:58.547 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13%) | 20260507 17:40:58.547 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:40:58.547 | INFO | High Water Breakdown: | 20260507 17:40:58.547 | INFO | 59.2% : 8.6 GiB ( 86.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:40:58.547 | INFO | 40.6% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:40:58.547 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:40:58.547 | INFO | ------------------------------------------ | 20260507 17:40:58.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:40:58.547 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:40:58.547 | INFO | Total mem usage: 26.5 GiB | 20260507 17:40:58.547 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:40:58.547 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.58%) | 20260507 17:40:58.547 | INFO | Unused Cache: 1.74 GiB (Waste: 6.56%) | 20260507 17:40:58.547 | INFO | High Water Breakdown: | 20260507 17:40:58.547 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:40:58.547 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:40:58.547 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:40:58.547 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:40:58.547 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:40:58.547 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:40:58.547 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:40:58.547 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:40:58.547 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:40:58.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:40:58.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:40:58.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:40:58.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:40:58.547 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:40:58.547 | INFO | ------------------------------------------ | 20260507 17:40:58.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:40:58.547 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:40:58.547 | INFO | Total mem usage: 27.8 GiB | 20260507 17:40:58.547 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:40:58.547 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 17:40:58.547 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 17:40:58.547 | INFO | High Water Breakdown: | 20260507 17:40:58.547 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 17:40:58.547 | INFO | ------------------------------------------ | 20260507 17:40:58.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:40:58.547 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:40:58.547 | INFO | Total mem usage: 15.7 GiB | 20260507 17:40:58.547 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:40:58.547 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:40:58.547 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:40:58.547 | INFO | High Water Breakdown: | 20260507 17:40:58.547 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:40:58.547 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:40:58.547 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:40:58.547 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:40:58.547 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:40:58.547 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:40:58.547 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:40:58.547 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:40:58.547 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:40:58.547 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:40:58.547 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:40:58.547 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:40:58.547 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:40:58.547 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:40:58.547 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:40:58.547 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:40:58.547 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:40:58.547 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:40:58.547 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:40:58.547 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:40:58.547 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:40:58.547 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:40:58.547 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:40:58.547 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:40:58.547 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:40:58.547 | INFO | ------------------------------------------ | 20260507 17:40:58.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:40:58.547 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:40:58.547 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:40:58.547 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:40:58.547 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.64e+01, 1.44e+01 / 2.62e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 0.30%, AggregateStats -- 1.02e-01 / 1.06e-01, 1.18e-02 / 6.95e-03 | 20260507 17:40:58.548 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.52e+00, 6.17e-01 / 2.10e-01 | 20260507 17:40:58.548 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 1.13e-01 | 20260507 17:40:58.548 | INFO | PerfReport: 15.71%, DME -- 4.21e+01 / 4.49e+01, 4.03e+00 / 2.48e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.87e+00 / 2.29e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.52e+00 / 2.92e+00, 4.39e-01 / 2.01e-01 | 20260507 17:40:58.548 | INFO | PerfReport: 3.76%, Metrics -- 1.26e+00 / 1.43e+00, 3.20e-01 / 1.88e-01 | 20260507 17:40:58.548 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.03e-01 / 4.25e-01, 6.55e-02 / 3.87e-02 | 20260507 17:40:58.548 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:40:58.548 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:40:58.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:40:58.548 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 24.20%, Upload -- 2.60e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 23.93%, Download -- 2.57e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 31.19%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 1.99%, LockUpload -- 2.14e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 2.11%, LockDownload -- 2.27e+00 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 16.57%, LockCompute -- 1.78e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:40:58.548 | INFO | PerfReport: 0.02%, Misc -- 1.71e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260507 17:40:58.643 | INFO | Starting a round of Baz IO | 20260507 17:41:02.151 | INFO | Populating RTMetrics report | 20260507 17:41:02.152 | INFO | RTMetrics report populated | 20260507 17:41:02.152 | INFO | Opening tmp RTMetrics json file | 20260507 17:41:02.165 | INFO | Serializing RTMetrics to json | 20260507 17:41:02.166 | INFO | Serialization done, swapping tmp file now... | 20260507 17:41:02.167 | INFO | RTMetrics done | 20260507 17:41:02.233 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.59 seconds | 20260507 17:41:02.233 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.75% / 0.02% / 52.23% | 20260507 17:41:02.233 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.42938% / 4.37101e-05% / 3.97364e-05% | 20260507 17:41:02.233 | INFO | Baz IO overall: | 20260507 17:41:02.233 | INFO | BytesWritten : 604619604446 | 20260507 17:41:02.233 | INFO | NumEvents : 292998345376 | 20260507 17:41:03.685 | INFO | Analyzing chunk frames = [1368064,1368576) | 20260507 17:41:03.685 | INFO | MemInfo: { | 20260507 17:41:03.685 | INFO | "gpu" : " 17 GB", | 20260507 17:41:03.685 | INFO | "huge" : "", | 20260507 17:41:03.685 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:03.685 | INFO | "malloc" : " 14 GB", | 20260507 17:41:03.685 | INFO | "pinned" : " 29 GB" | 20260507 17:41:03.685 | INFO | } | 20260507 17:41:03.685 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:41:03.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:03.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:03.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:03.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:03.685 | INFO | ANALYSIS : 69.5%, 3.0, 108.5ms, 0 | 20260507 17:41:03.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:03.685 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 17:41:03.685 | INFO | PRE_HQ : 66.7%, 1.7, 59.5ms, 0 | 20260507 17:41:03.685 | INFO | BAZWRITER : 69.0%, 2.9, 101.5ms, 0 | 20260507 17:41:03.685 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:41:03.686 | INFO | Uploaded 36GiB to GPU in 2.52479s (14.2586GiB/s) | 20260507 17:41:03.686 | INFO | Downloaded 38GiB to GPU in 2.73092s (13.9147GiB/s) | 20260507 17:41:08.824 | INFO | Analyzing chunk frames = [1368576,1369088) | 20260507 17:41:08.824 | INFO | MemInfo: { | 20260507 17:41:08.824 | INFO | "gpu" : " 17 GB", | 20260507 17:41:08.824 | INFO | "huge" : "", | 20260507 17:41:08.824 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:08.824 | INFO | "malloc" : " 14 GB", | 20260507 17:41:08.825 | INFO | "pinned" : " 29 GB" | 20260507 17:41:08.825 | INFO | } | 20260507 17:41:08.825 | INFO | Perf report for the last 5.13944 seconds: | 20260507 17:41:08.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:08.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:41:08.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:41:08.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:08.825 | INFO | Uploaded 36GiB to GPU in 2.51889s (14.292GiB/s) | 20260507 17:41:08.825 | INFO | Downloaded 34GiB to GPU in 2.43766s (13.9478GiB/s) | 20260507 17:41:13.964 | INFO | Analyzing chunk frames = [1369088,1369600) | 20260507 17:41:13.964 | INFO | MemInfo: { | 20260507 17:41:13.964 | INFO | "gpu" : " 17 GB", | 20260507 17:41:13.964 | INFO | "huge" : "", | 20260507 17:41:13.964 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:13.964 | INFO | "malloc" : " 14 GB", | 20260507 17:41:13.964 | INFO | "pinned" : " 29 GB" | 20260507 17:41:13.964 | INFO | } | 20260507 17:41:13.964 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:41:13.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:13.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:41:13.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:41:13.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:13.964 | INFO | Uploaded 36GiB to GPU in 2.56174s (14.0529GiB/s) | 20260507 17:41:13.965 | INFO | Downloaded 34GiB to GPU in 2.42812s (14.0026GiB/s) | 20260507 17:41:19.103 | INFO | Analyzing chunk frames = [1369600,1370112) | 20260507 17:41:19.103 | INFO | MemInfo: { | 20260507 17:41:19.103 | INFO | "gpu" : " 17 GB", | 20260507 17:41:19.103 | INFO | "huge" : "", | 20260507 17:41:19.103 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:19.103 | INFO | "malloc" : " 14 GB", | 20260507 17:41:19.103 | INFO | "pinned" : " 29 GB" | 20260507 17:41:19.103 | INFO | } | 20260507 17:41:19.103 | INFO | Perf report for the last 5.13878 seconds: | 20260507 17:41:19.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:19.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.103 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:41:19.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.103 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:41:19.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:19.104 | INFO | Uploaded 36GiB to GPU in 2.49144s (14.4495GiB/s) | 20260507 17:41:19.104 | INFO | Downloaded 34GiB to GPU in 2.34582s (14.4939GiB/s) | 20260507 17:41:24.242 | INFO | Analyzing chunk frames = [1370112,1370624) | 20260507 17:41:24.243 | INFO | MemInfo: { | 20260507 17:41:24.243 | INFO | "gpu" : " 17 GB", | 20260507 17:41:24.243 | INFO | "huge" : "", | 20260507 17:41:24.243 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:24.243 | INFO | "malloc" : " 14 GB", | 20260507 17:41:24.243 | INFO | "pinned" : " 29 GB" | 20260507 17:41:24.243 | INFO | } | 20260507 17:41:24.243 | INFO | Perf report for the last 5.14011 seconds: | 20260507 17:41:24.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:24.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.243 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 17:41:24.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.243 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 17:41:24.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:24.244 | INFO | Uploaded 36GiB to GPU in 2.38139s (15.1172GiB/s) | 20260507 17:41:24.244 | INFO | Downloaded 34GiB to GPU in 2.44915s (13.8824GiB/s) | 20260507 17:41:29.382 | INFO | Analyzing chunk frames = [1370624,1371136) | 20260507 17:41:29.382 | INFO | MemInfo: { | 20260507 17:41:29.382 | INFO | "gpu" : " 17 GB", | 20260507 17:41:29.382 | INFO | "huge" : "", | 20260507 17:41:29.382 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:29.382 | INFO | "malloc" : " 14 GB", | 20260507 17:41:29.382 | INFO | "pinned" : " 29 GB" | 20260507 17:41:29.382 | INFO | } | 20260507 17:41:29.382 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:41:29.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:29.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.382 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:41:29.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.383 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:41:29.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:29.383 | INFO | Uploaded 36GiB to GPU in 2.44121s (14.7468GiB/s) | 20260507 17:41:29.383 | INFO | Downloaded 34GiB to GPU in 2.47769s (13.7224GiB/s) | 20260507 17:41:34.521 | INFO | Analyzing chunk frames = [1371136,1371648) | 20260507 17:41:34.521 | INFO | MemInfo: { | 20260507 17:41:34.521 | INFO | "gpu" : " 17 GB", | 20260507 17:41:34.521 | INFO | "huge" : "", | 20260507 17:41:34.521 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:34.521 | INFO | "malloc" : " 14 GB", | 20260507 17:41:34.521 | INFO | "pinned" : " 29 GB" | 20260507 17:41:34.522 | INFO | } | 20260507 17:41:34.522 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:41:34.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:34.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 17:41:34.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:41:34.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:34.522 | INFO | Uploaded 36GiB to GPU in 2.50797s (14.3542GiB/s) | 20260507 17:41:34.522 | INFO | Downloaded 34GiB to GPU in 2.41557s (14.0753GiB/s) | 20260507 17:41:39.661 | INFO | Analyzing chunk frames = [1371648,1372160) | 20260507 17:41:39.661 | INFO | MemInfo: { | 20260507 17:41:39.661 | INFO | "gpu" : " 17 GB", | 20260507 17:41:39.661 | INFO | "huge" : "", | 20260507 17:41:39.661 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:39.661 | INFO | "malloc" : " 14 GB", | 20260507 17:41:39.661 | INFO | "pinned" : " 29 GB" | 20260507 17:41:39.661 | INFO | } | 20260507 17:41:39.661 | INFO | Perf report for the last 5.13892 seconds: | 20260507 17:41:39.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:39.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 17:41:39.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:41:39.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:39.661 | INFO | Uploaded 36GiB to GPU in 2.5593s (14.0663GiB/s) | 20260507 17:41:39.661 | INFO | Downloaded 34GiB to GPU in 2.49336s (13.6362GiB/s) | 20260507 17:41:39.758 | INFO | Starting a round of Baz IO | 20260507 17:41:43.283 | INFO | Populating RTMetrics report | 20260507 17:41:43.283 | INFO | RTMetrics report populated | 20260507 17:41:43.283 | INFO | Opening tmp RTMetrics json file | 20260507 17:41:43.284 | INFO | Serializing RTMetrics to json | 20260507 17:41:43.285 | INFO | Serialization done, swapping tmp file now... | 20260507 17:41:43.285 | INFO | RTMetrics done | 20260507 17:41:43.382 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.624 seconds | 20260507 17:41:43.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.73% / 0.02% / 52.25% | 20260507 17:41:43.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.41786% / 3.97364e-05% / 2.78155e-05% | 20260507 17:41:43.383 | INFO | Baz IO overall: | 20260507 17:41:43.383 | INFO | BytesWritten : 606551985220 | 20260507 17:41:43.383 | INFO | NumEvents : 293937130034 | 20260507 17:41:44.800 | INFO | Analyzing chunk frames = [1372160,1372672) | 20260507 17:41:44.800 | INFO | MemInfo: { | 20260507 17:41:44.800 | INFO | "gpu" : " 17 GB", | 20260507 17:41:44.800 | INFO | "huge" : "", | 20260507 17:41:44.800 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:44.800 | INFO | "malloc" : " 14 GB", | 20260507 17:41:44.800 | INFO | "pinned" : " 29 GB" | 20260507 17:41:44.800 | INFO | } | 20260507 17:41:44.800 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:41:44.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:44.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:44.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:44.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:44.800 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 17:41:44.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:44.800 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 17:41:44.800 | INFO | PRE_HQ : 67.6%, 1.8, 62.5ms, 0 | 20260507 17:41:44.800 | INFO | BAZWRITER : 69.7%, 2.9, 105.4ms, 0 | 20260507 17:41:44.800 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:41:44.801 | INFO | Uploaded 36GiB to GPU in 2.57415s (13.9852GiB/s) | 20260507 17:41:44.801 | INFO | Downloaded 38GiB to GPU in 2.72378s (13.9512GiB/s) | 20260507 17:41:49.939 | INFO | Analyzing chunk frames = [1372672,1373184) | 20260507 17:41:49.939 | INFO | MemInfo: { | 20260507 17:41:49.939 | INFO | "gpu" : " 17 GB", | 20260507 17:41:49.939 | INFO | "huge" : "", | 20260507 17:41:49.939 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:49.939 | INFO | "malloc" : " 14 GB", | 20260507 17:41:49.939 | INFO | "pinned" : " 29 GB" | 20260507 17:41:49.939 | INFO | } | 20260507 17:41:49.939 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:41:49.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:49.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 17:41:49.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:41:49.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:49.940 | INFO | Uploaded 36GiB to GPU in 2.55317s (14.1001GiB/s) | 20260507 17:41:49.940 | INFO | Downloaded 34GiB to GPU in 2.44555s (13.9028GiB/s) | 20260507 17:41:55.079 | INFO | Analyzing chunk frames = [1373184,1373696) | 20260507 17:41:55.079 | INFO | MemInfo: { | 20260507 17:41:55.079 | INFO | "gpu" : " 17 GB", | 20260507 17:41:55.079 | INFO | "huge" : "", | 20260507 17:41:55.079 | INFO | "huge_shared" : " 29 GB", | 20260507 17:41:55.079 | INFO | "malloc" : " 14 GB", | 20260507 17:41:55.079 | INFO | "pinned" : " 29 GB" | 20260507 17:41:55.079 | INFO | } | 20260507 17:41:55.079 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:41:55.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:41:55.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:41:55.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:41:55.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:41:55.079 | INFO | Uploaded 36GiB to GPU in 2.56703s (14.024GiB/s) | 20260507 17:41:55.079 | INFO | Downloaded 34GiB to GPU in 2.41835s (14.0592GiB/s) | 20260507 17:42:00.218 | INFO | Analyzing chunk frames = [1373696,1374208) | 20260507 17:42:00.218 | INFO | MemInfo: { | 20260507 17:42:00.218 | INFO | "gpu" : " 17 GB", | 20260507 17:42:00.218 | INFO | "huge" : "", | 20260507 17:42:00.218 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:00.218 | INFO | "malloc" : " 14 GB", | 20260507 17:42:00.218 | INFO | "pinned" : " 29 GB" | 20260507 17:42:00.218 | INFO | } | 20260507 17:42:00.218 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:42:00.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:00.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:42:00.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:42:00.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:00.218 | INFO | Uploaded 36GiB to GPU in 2.4943s (14.4329GiB/s) | 20260507 17:42:00.218 | INFO | Downloaded 34GiB to GPU in 2.34015s (14.529GiB/s) | 20260507 17:42:05.358 | INFO | Analyzing chunk frames = [1374208,1374720) | 20260507 17:42:05.358 | INFO | MemInfo: { | 20260507 17:42:05.358 | INFO | "gpu" : " 17 GB", | 20260507 17:42:05.358 | INFO | "huge" : "", | 20260507 17:42:05.358 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:05.358 | INFO | "malloc" : " 14 GB", | 20260507 17:42:05.358 | INFO | "pinned" : " 29 GB" | 20260507 17:42:05.358 | INFO | } | 20260507 17:42:05.358 | INFO | Perf report for the last 5.13985 seconds: | 20260507 17:42:05.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:05.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:42:05.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:42:05.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:05.358 | INFO | Uploaded 36GiB to GPU in 2.36072s (15.2496GiB/s) | 20260507 17:42:05.358 | INFO | Downloaded 34GiB to GPU in 2.46798s (13.7765GiB/s) | 20260507 17:42:10.497 | INFO | Analyzing chunk frames = [1374720,1375232) | 20260507 17:42:10.497 | INFO | MemInfo: { | 20260507 17:42:10.497 | INFO | "gpu" : " 17 GB", | 20260507 17:42:10.497 | INFO | "huge" : "", | 20260507 17:42:10.497 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:10.497 | INFO | "malloc" : " 14 GB", | 20260507 17:42:10.497 | INFO | "pinned" : " 29 GB" | 20260507 17:42:10.497 | INFO | } | 20260507 17:42:10.497 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:42:10.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:10.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:42:10.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:42:10.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:10.497 | INFO | Uploaded 36GiB to GPU in 2.43842s (14.7636GiB/s) | 20260507 17:42:10.497 | INFO | Downloaded 34GiB to GPU in 2.48271s (13.6947GiB/s) | 20260507 17:42:15.636 | INFO | Analyzing chunk frames = [1375232,1375744) | 20260507 17:42:15.636 | INFO | MemInfo: { | 20260507 17:42:15.636 | INFO | "gpu" : " 17 GB", | 20260507 17:42:15.636 | INFO | "huge" : "", | 20260507 17:42:15.636 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:15.636 | INFO | "malloc" : " 14 GB", | 20260507 17:42:15.636 | INFO | "pinned" : " 29 GB" | 20260507 17:42:15.636 | INFO | } | 20260507 17:42:15.636 | INFO | Perf report for the last 5.13879 seconds: | 20260507 17:42:15.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:15.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:42:15.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:42:15.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:15.636 | INFO | Uploaded 36GiB to GPU in 2.5259s (14.2524GiB/s) | 20260507 17:42:15.636 | INFO | Downloaded 34GiB to GPU in 2.43866s (13.9421GiB/s) | 20260507 17:42:20.776 | INFO | Analyzing chunk frames = [1375744,1376256) | 20260507 17:42:20.776 | INFO | MemInfo: { | 20260507 17:42:20.776 | INFO | "gpu" : " 17 GB", | 20260507 17:42:20.776 | INFO | "huge" : "", | 20260507 17:42:20.776 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:20.776 | INFO | "malloc" : " 14 GB", | 20260507 17:42:20.776 | INFO | "pinned" : " 29 GB" | 20260507 17:42:20.776 | INFO | } | 20260507 17:42:20.776 | INFO | Perf report for the last 5.14029 seconds: | 20260507 17:42:20.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:20.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.776 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 17:42:20.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.777 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 17:42:20.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:20.777 | INFO | Uploaded 36GiB to GPU in 2.53507s (14.2008GiB/s) | 20260507 17:42:20.777 | INFO | Downloaded 34GiB to GPU in 2.49914s (13.6047GiB/s) | 20260507 17:42:20.777 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:42:20.777 | INFO | Total mem usage: 16.4 GiB | 20260507 17:42:20.777 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:42:20.777 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.5%) | 20260507 17:42:20.777 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:42:20.777 | INFO | High Water Breakdown: | 20260507 17:42:20.777 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:42:20.777 | INFO | 40.8% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:42:20.777 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:42:20.777 | INFO | ------------------------------------------ | 20260507 17:42:20.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:42:20.777 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:42:20.777 | INFO | Total mem usage: 26.5 GiB | 20260507 17:42:20.777 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:42:20.777 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:42:20.777 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:42:20.777 | INFO | High Water Breakdown: | 20260507 17:42:20.777 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:42:20.777 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:42:20.777 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:42:20.777 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:42:20.777 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:42:20.777 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:42:20.777 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:42:20.777 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:42:20.777 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:42:20.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:42:20.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:42:20.778 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:42:20.778 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:42:20.778 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:42:20.778 | INFO | ------------------------------------------ | 20260507 17:42:20.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:42:20.778 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:42:20.778 | INFO | Total mem usage: 27.8 GiB | 20260507 17:42:20.778 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:42:20.778 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:42:20.778 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:42:20.778 | INFO | High Water Breakdown: | 20260507 17:42:20.778 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:42:20.778 | INFO | ------------------------------------------ | 20260507 17:42:20.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:42:20.778 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:42:20.778 | INFO | Total mem usage: 15.7 GiB | 20260507 17:42:20.778 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:42:20.778 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:42:20.778 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:42:20.778 | INFO | High Water Breakdown: | 20260507 17:42:20.778 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:42:20.778 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:42:20.778 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:42:20.778 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:42:20.778 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:42:20.778 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:42:20.778 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:42:20.778 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:42:20.778 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:42:20.778 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:42:20.778 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:42:20.778 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:42:20.778 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:42:20.778 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:42:20.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:42:20.778 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:42:20.778 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:42:20.778 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:42:20.778 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:42:20.778 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:42:20.778 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:42:20.779 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:42:20.779 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:42:20.779 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:42:20.779 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:42:20.779 | INFO | ------------------------------------------ | 20260507 17:42:20.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:42:20.779 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:42:20.779 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:42:20.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:42:20.779 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.44e+01 / 2.66e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.14e-02 / 6.37e-03 | 20260507 17:42:20.779 | INFO | PerfReport: 12.06%, Baseline -- 4.03e+00 / 4.50e+00, 6.11e-01 / 2.28e-01 | 20260507 17:42:20.779 | INFO | PerfReport: 3.42%, Histogram -- 1.14e+00 / 1.49e+00, 2.13e-01 / 1.04e-01 | 20260507 17:42:20.779 | INFO | PerfReport: 15.73%, DME -- 4.21e+01 / 4.49e+01, 4.43e+00 / 2.69e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.86e+00 / 2.34e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.34e-01 / 2.33e-01 | 20260507 17:42:20.779 | INFO | PerfReport: 3.76%, Metrics -- 1.26e+00 / 1.43e+00, 3.15e-01 / 2.08e-01 | 20260507 17:42:20.779 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.15e-01, 6.50e-02 / 4.43e-02 | 20260507 17:42:20.779 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:42:20.779 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:42:20.779 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:42:20.779 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 24.21%, Upload -- 2.61e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 23.94%, Download -- 2.58e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 31.06%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 2.03%, LockUpload -- 2.19e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 2.14%, LockDownload -- 2.30e+00 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 16.60%, LockCompute -- 1.79e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:42:20.779 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260507 17:42:20.888 | INFO | Starting a round of Baz IO | 20260507 17:42:24.376 | INFO | Populating RTMetrics report | 20260507 17:42:24.376 | INFO | RTMetrics report populated | 20260507 17:42:24.376 | INFO | Opening tmp RTMetrics json file | 20260507 17:42:24.380 | INFO | Serializing RTMetrics to json | 20260507 17:42:24.395 | INFO | Serialization done, swapping tmp file now... | 20260507 17:42:24.395 | INFO | RTMetrics done | 20260507 17:42:24.456 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.567 seconds | 20260507 17:42:24.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.71% / 0.02% / 52.27% | 20260507 17:42:24.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.40798% / 3.57628e-05% / 1.58946e-05% | 20260507 17:42:24.456 | INFO | Baz IO overall: | 20260507 17:42:24.456 | INFO | BytesWritten : 608484301365 | 20260507 17:42:24.456 | INFO | NumEvents : 294876046469 | 20260507 17:42:25.915 | INFO | Analyzing chunk frames = [1376256,1376768) | 20260507 17:42:25.915 | INFO | MemInfo: { | 20260507 17:42:25.915 | INFO | "gpu" : " 17 GB", | 20260507 17:42:25.915 | INFO | "huge" : "", | 20260507 17:42:25.915 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:25.915 | INFO | "malloc" : " 14 GB", | 20260507 17:42:25.915 | INFO | "pinned" : " 29 GB" | 20260507 17:42:25.915 | INFO | } | 20260507 17:42:25.915 | INFO | Perf report for the last 5.13882 seconds: | 20260507 17:42:25.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:25.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:25.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:25.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:25.915 | INFO | ANALYSIS : 69.6%, 2.9, 107.0ms, 0 | 20260507 17:42:25.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:25.915 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 17:42:25.915 | INFO | PRE_HQ : 67.4%, 1.9, 65.8ms, 0 | 20260507 17:42:25.915 | INFO | BAZWRITER : 68.6%, 3.1, 109.6ms, 0 | 20260507 17:42:25.915 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:42:25.916 | INFO | Uploaded 36GiB to GPU in 2.59275s (13.8849GiB/s) | 20260507 17:42:25.916 | INFO | Downloaded 38GiB to GPU in 2.70896s (14.0275GiB/s) | 20260507 17:42:31.054 | INFO | Analyzing chunk frames = [1376768,1377280) | 20260507 17:42:31.054 | INFO | MemInfo: { | 20260507 17:42:31.054 | INFO | "gpu" : " 17 GB", | 20260507 17:42:31.054 | INFO | "huge" : "", | 20260507 17:42:31.054 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:31.054 | INFO | "malloc" : " 14 GB", | 20260507 17:42:31.054 | INFO | "pinned" : " 29 GB" | 20260507 17:42:31.054 | INFO | } | 20260507 17:42:31.054 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:42:31.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:31.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.054 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:42:31.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.055 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:42:31.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:31.055 | INFO | Uploaded 36GiB to GPU in 2.52481s (14.2585GiB/s) | 20260507 17:42:31.055 | INFO | Downloaded 34GiB to GPU in 2.42888s (13.9982GiB/s) | 20260507 17:42:36.193 | INFO | Analyzing chunk frames = [1377280,1377792) | 20260507 17:42:36.193 | INFO | MemInfo: { | 20260507 17:42:36.193 | INFO | "gpu" : " 17 GB", | 20260507 17:42:36.193 | INFO | "huge" : "", | 20260507 17:42:36.193 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:36.193 | INFO | "malloc" : " 14 GB", | 20260507 17:42:36.193 | INFO | "pinned" : " 29 GB" | 20260507 17:42:36.193 | INFO | } | 20260507 17:42:36.194 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:42:36.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:36.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:42:36.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:42:36.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:36.194 | INFO | Uploaded 36GiB to GPU in 2.57855s (13.9613GiB/s) | 20260507 17:42:36.194 | INFO | Downloaded 34GiB to GPU in 2.4124s (14.0939GiB/s) | 20260507 17:42:41.333 | INFO | Analyzing chunk frames = [1377792,1378304) | 20260507 17:42:41.333 | INFO | MemInfo: { | 20260507 17:42:41.333 | INFO | "gpu" : " 17 GB", | 20260507 17:42:41.333 | INFO | "huge" : "", | 20260507 17:42:41.333 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:41.333 | INFO | "malloc" : " 14 GB", | 20260507 17:42:41.333 | INFO | "pinned" : " 29 GB" | 20260507 17:42:41.333 | INFO | } | 20260507 17:42:41.333 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:42:41.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:41.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:42:41.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:42:41.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:41.333 | INFO | Uploaded 36GiB to GPU in 2.50681s (14.3609GiB/s) | 20260507 17:42:41.333 | INFO | Downloaded 34GiB to GPU in 2.34717s (14.4856GiB/s) | 20260507 17:42:46.472 | INFO | Analyzing chunk frames = [1378304,1378816) | 20260507 17:42:46.472 | INFO | MemInfo: { | 20260507 17:42:46.472 | INFO | "gpu" : " 17 GB", | 20260507 17:42:46.472 | INFO | "huge" : "", | 20260507 17:42:46.472 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:46.472 | INFO | "malloc" : " 14 GB", | 20260507 17:42:46.472 | INFO | "pinned" : " 29 GB" | 20260507 17:42:46.473 | INFO | } | 20260507 17:42:46.473 | INFO | Perf report for the last 5.13957 seconds: | 20260507 17:42:46.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:46.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 17:42:46.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:42:46.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:46.473 | INFO | Uploaded 36GiB to GPU in 2.36061s (15.2503GiB/s) | 20260507 17:42:46.473 | INFO | Downloaded 34GiB to GPU in 2.47438s (13.7408GiB/s) | 20260507 17:42:51.611 | INFO | Analyzing chunk frames = [1378816,1379328) | 20260507 17:42:51.611 | INFO | MemInfo: { | 20260507 17:42:51.611 | INFO | "gpu" : " 17 GB", | 20260507 17:42:51.611 | INFO | "huge" : "", | 20260507 17:42:51.611 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:51.611 | INFO | "malloc" : " 14 GB", | 20260507 17:42:51.611 | INFO | "pinned" : " 29 GB" | 20260507 17:42:51.612 | INFO | } | 20260507 17:42:51.612 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:42:51.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:51.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:42:51.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:42:51.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:51.612 | INFO | Uploaded 36GiB to GPU in 2.35703s (15.2735GiB/s) | 20260507 17:42:51.612 | INFO | Downloaded 34GiB to GPU in 2.48504s (13.6819GiB/s) | 20260507 17:42:56.751 | INFO | Analyzing chunk frames = [1379328,1379840) | 20260507 17:42:56.751 | INFO | MemInfo: { | 20260507 17:42:56.751 | INFO | "gpu" : " 17 GB", | 20260507 17:42:56.751 | INFO | "huge" : "", | 20260507 17:42:56.751 | INFO | "huge_shared" : " 29 GB", | 20260507 17:42:56.751 | INFO | "malloc" : " 14 GB", | 20260507 17:42:56.751 | INFO | "pinned" : " 29 GB" | 20260507 17:42:56.751 | INFO | } | 20260507 17:42:56.751 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:42:56.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:42:56.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:42:56.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:42:56.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:42:56.751 | INFO | Uploaded 36GiB to GPU in 2.47352s (14.5542GiB/s) | 20260507 17:42:56.751 | INFO | Downloaded 34GiB to GPU in 2.43527s (13.9615GiB/s) | 20260507 17:43:01.890 | INFO | Analyzing chunk frames = [1379840,1380352) | 20260507 17:43:01.890 | INFO | MemInfo: { | 20260507 17:43:01.890 | INFO | "gpu" : " 17 GB", | 20260507 17:43:01.890 | INFO | "huge" : "", | 20260507 17:43:01.890 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:01.890 | INFO | "malloc" : " 14 GB", | 20260507 17:43:01.890 | INFO | "pinned" : " 29 GB" | 20260507 17:43:01.891 | INFO | } | 20260507 17:43:01.891 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:43:01.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:01.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:43:01.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:43:01.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:01.891 | INFO | Uploaded 36GiB to GPU in 2.51878s (14.2926GiB/s) | 20260507 17:43:01.891 | INFO | Downloaded 34GiB to GPU in 2.50641s (13.5652GiB/s) | 20260507 17:43:01.988 | INFO | Starting a round of Baz IO | 20260507 17:43:05.502 | INFO | Populating RTMetrics report | 20260507 17:43:05.502 | INFO | RTMetrics report populated | 20260507 17:43:05.502 | INFO | Opening tmp RTMetrics json file | 20260507 17:43:05.519 | INFO | Serializing RTMetrics to json | 20260507 17:43:05.521 | INFO | Serialization done, swapping tmp file now... | 20260507 17:43:05.521 | INFO | RTMetrics done | 20260507 17:43:05.582 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.594 seconds | 20260507 17:43:05.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.69% / 0.02% / 52.29% | 20260507 17:43:05.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.40053% / 2.78155e-05% / 3.17891e-05% | 20260507 17:43:05.583 | INFO | Baz IO overall: | 20260507 17:43:05.583 | INFO | BytesWritten : 610417854241 | 20260507 17:43:05.583 | INFO | NumEvents : 295815758543 | 20260507 17:43:07.029 | INFO | Analyzing chunk frames = [1380352,1380864) | 20260507 17:43:07.029 | INFO | MemInfo: { | 20260507 17:43:07.029 | INFO | "gpu" : " 17 GB", | 20260507 17:43:07.029 | INFO | "huge" : "", | 20260507 17:43:07.029 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:07.029 | INFO | "malloc" : " 14 GB", | 20260507 17:43:07.029 | INFO | "pinned" : " 29 GB" | 20260507 17:43:07.029 | INFO | } | 20260507 17:43:07.030 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:43:07.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:07.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:07.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:07.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:07.030 | INFO | ANALYSIS : 69.5%, 3.0, 108.6ms, 0 | 20260507 17:43:07.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:07.030 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 17:43:07.030 | INFO | PRE_HQ : 66.7%, 1.7, 59.6ms, 0 | 20260507 17:43:07.030 | INFO | BAZWRITER : 68.9%, 2.9, 101.7ms, 0 | 20260507 17:43:07.030 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:43:07.030 | INFO | Uploaded 36GiB to GPU in 2.51678s (14.304GiB/s) | 20260507 17:43:07.030 | INFO | Downloaded 38GiB to GPU in 2.72086s (13.9662GiB/s) | 20260507 17:43:12.169 | INFO | Analyzing chunk frames = [1380864,1381376) | 20260507 17:43:12.169 | INFO | MemInfo: { | 20260507 17:43:12.169 | INFO | "gpu" : " 17 GB", | 20260507 17:43:12.169 | INFO | "huge" : "", | 20260507 17:43:12.169 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:12.169 | INFO | "malloc" : " 14 GB", | 20260507 17:43:12.169 | INFO | "pinned" : " 29 GB" | 20260507 17:43:12.169 | INFO | } | 20260507 17:43:12.169 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:43:12.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:12.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:43:12.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 17:43:12.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:12.169 | INFO | Uploaded 36GiB to GPU in 2.49876s (14.4072GiB/s) | 20260507 17:43:12.170 | INFO | Downloaded 34GiB to GPU in 2.44007s (13.934GiB/s) | 20260507 17:43:17.308 | INFO | Analyzing chunk frames = [1381376,1381888) | 20260507 17:43:17.308 | INFO | MemInfo: { | 20260507 17:43:17.308 | INFO | "gpu" : " 17 GB", | 20260507 17:43:17.308 | INFO | "huge" : "", | 20260507 17:43:17.308 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:17.308 | INFO | "malloc" : " 14 GB", | 20260507 17:43:17.308 | INFO | "pinned" : " 29 GB" | 20260507 17:43:17.308 | INFO | } | 20260507 17:43:17.308 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:43:17.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:17.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.308 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:43:17.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.309 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:43:17.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:17.309 | INFO | Uploaded 36GiB to GPU in 2.55714s (14.0783GiB/s) | 20260507 17:43:17.309 | INFO | Downloaded 34GiB to GPU in 2.41932s (14.0535GiB/s) | 20260507 17:43:22.447 | INFO | Analyzing chunk frames = [1381888,1382400) | 20260507 17:43:22.447 | INFO | MemInfo: { | 20260507 17:43:22.447 | INFO | "gpu" : " 17 GB", | 20260507 17:43:22.448 | INFO | "huge" : "", | 20260507 17:43:22.448 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:22.448 | INFO | "malloc" : " 14 GB", | 20260507 17:43:22.448 | INFO | "pinned" : " 29 GB" | 20260507 17:43:22.448 | INFO | } | 20260507 17:43:22.448 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:43:22.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:22.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:43:22.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:43:22.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:22.448 | INFO | Uploaded 36GiB to GPU in 2.49s (14.4578GiB/s) | 20260507 17:43:22.448 | INFO | Downloaded 34GiB to GPU in 2.34769s (14.4823GiB/s) | 20260507 17:43:27.593 | INFO | Analyzing chunk frames = [1382400,1382912) | 20260507 17:43:27.593 | INFO | MemInfo: { | 20260507 17:43:27.593 | INFO | "gpu" : " 17 GB", | 20260507 17:43:27.593 | INFO | "huge" : "", | 20260507 17:43:27.593 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:27.593 | INFO | "malloc" : " 14 GB", | 20260507 17:43:27.593 | INFO | "pinned" : " 29 GB" | 20260507 17:43:27.593 | INFO | } | 20260507 17:43:27.593 | INFO | Perf report for the last 5.14557 seconds: | 20260507 17:43:27.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:27.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.593 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 17:43:27.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.593 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 17:43:27.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:27.594 | INFO | Uploaded 36GiB to GPU in 2.38761s (15.0778GiB/s) | 20260507 17:43:27.594 | INFO | Downloaded 34GiB to GPU in 2.47377s (13.7442GiB/s) | 20260507 17:43:32.726 | INFO | Analyzing chunk frames = [1382912,1383424) | 20260507 17:43:32.726 | INFO | MemInfo: { | 20260507 17:43:32.726 | INFO | "gpu" : " 17 GB", | 20260507 17:43:32.726 | INFO | "huge" : "", | 20260507 17:43:32.726 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:32.726 | INFO | "malloc" : " 14 GB", | 20260507 17:43:32.726 | INFO | "pinned" : " 29 GB" | 20260507 17:43:32.726 | INFO | } | 20260507 17:43:32.726 | INFO | Perf report for the last 5.13316 seconds: | 20260507 17:43:32.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:32.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.727 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:43:32.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.727 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260507 17:43:32.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:32.727 | INFO | Uploaded 36GiB to GPU in 2.44288s (14.7367GiB/s) | 20260507 17:43:32.727 | INFO | Downloaded 34GiB to GPU in 2.47055s (13.7621GiB/s) | 20260507 17:43:37.865 | INFO | Analyzing chunk frames = [1383424,1383936) | 20260507 17:43:37.865 | INFO | MemInfo: { | 20260507 17:43:37.865 | INFO | "gpu" : " 17 GB", | 20260507 17:43:37.865 | INFO | "huge" : "", | 20260507 17:43:37.865 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:37.865 | INFO | "malloc" : " 14 GB", | 20260507 17:43:37.865 | INFO | "pinned" : " 29 GB" | 20260507 17:43:37.865 | INFO | } | 20260507 17:43:37.865 | INFO | Perf report for the last 5.13897 seconds: | 20260507 17:43:37.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:37.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.866 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:43:37.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.866 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 17:43:37.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:37.866 | INFO | Uploaded 36GiB to GPU in 2.50585s (14.3664GiB/s) | 20260507 17:43:37.866 | INFO | Downloaded 34GiB to GPU in 2.42298s (14.0323GiB/s) | 20260507 17:43:43.005 | INFO | Analyzing chunk frames = [1383936,1384448) | 20260507 17:43:43.005 | INFO | MemInfo: { | 20260507 17:43:43.005 | INFO | "gpu" : " 17 GB", | 20260507 17:43:43.005 | INFO | "huge" : "", | 20260507 17:43:43.005 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:43.005 | INFO | "malloc" : " 14 GB", | 20260507 17:43:43.005 | INFO | "pinned" : " 29 GB" | 20260507 17:43:43.005 | INFO | } | 20260507 17:43:43.005 | INFO | Perf report for the last 5.13972 seconds: | 20260507 17:43:43.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:43.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 17:43:43.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:43:43.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:43.005 | INFO | Uploaded 36GiB to GPU in 2.54272s (14.1581GiB/s) | 20260507 17:43:43.006 | INFO | Downloaded 34GiB to GPU in 2.49698s (13.6165GiB/s) | 20260507 17:43:43.006 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:43:43.006 | INFO | Total mem usage: 16.4 GiB | 20260507 17:43:43.006 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:43:43.006 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13%) | 20260507 17:43:43.006 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:43:43.006 | INFO | High Water Breakdown: | 20260507 17:43:43.006 | INFO | 59.2% : 8.59 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:43:43.006 | INFO | 40.7% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:43:43.006 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:43:43.006 | INFO | ------------------------------------------ | 20260507 17:43:43.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:43:43.006 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:43:43.006 | INFO | Total mem usage: 26.5 GiB | 20260507 17:43:43.006 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:43:43.006 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:43:43.006 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:43:43.006 | INFO | High Water Breakdown: | 20260507 17:43:43.006 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:43:43.006 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:43:43.006 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:43:43.006 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:43:43.006 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:43:43.006 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:43:43.006 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:43:43.006 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:43:43.006 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:43:43.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:43:43.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:43:43.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:43:43.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:43:43.006 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:43:43.006 | INFO | ------------------------------------------ | 20260507 17:43:43.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:43:43.006 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:43:43.006 | INFO | Total mem usage: 27.8 GiB | 20260507 17:43:43.006 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:43:43.006 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:43:43.006 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:43:43.006 | INFO | High Water Breakdown: | 20260507 17:43:43.006 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:43:43.006 | INFO | ------------------------------------------ | 20260507 17:43:43.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:43:43.006 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:43:43.006 | INFO | Total mem usage: 15.7 GiB | 20260507 17:43:43.006 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:43:43.006 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:43:43.006 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:43:43.006 | INFO | High Water Breakdown: | 20260507 17:43:43.006 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:43:43.006 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:43:43.006 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:43:43.006 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:43:43.006 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:43:43.006 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:43:43.006 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:43:43.006 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:43:43.006 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:43:43.006 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:43:43.006 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:43:43.006 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:43:43.006 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:43:43.006 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:43:43.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:43:43.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:43:43.006 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:43:43.006 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:43:43.006 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:43:43.006 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:43:43.006 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:43:43.006 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:43:43.006 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:43:43.006 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:43:43.006 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:43:43.006 | INFO | ------------------------------------------ | 20260507 17:43:43.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:43:43.006 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:43:43.006 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:43:43.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:43:43.006 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.67e+01, 1.43e+01 / 2.73e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.07e-02 / 5.30e-03 | 20260507 17:43:43.006 | INFO | PerfReport: 12.08%, Baseline -- 4.04e+00 / 4.52e+00, 6.09e-01 / 1.94e-01 | 20260507 17:43:43.006 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.04e-01 | 20260507 17:43:43.006 | INFO | PerfReport: 15.73%, DME -- 4.21e+01 / 4.51e+01, 4.32e+00 / 2.54e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.85e+00 / 2.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.32e-01 / 2.43e-01 | 20260507 17:43:43.006 | INFO | PerfReport: 3.74%, Metrics -- 1.25e+00 / 1.42e+00, 3.16e-01 / 1.77e-01 | 20260507 17:43:43.006 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.18e-01, 6.45e-02 / 3.74e-02 | 20260507 17:43:43.006 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:43:43.006 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:43:43.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:43:43.006 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 24.13%, Upload -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 23.97%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 31.14%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 1.95%, LockUpload -- 2.09e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 2.22%, LockDownload -- 2.39e+00 / 0.00e+00, 3.78e+00 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 16.58%, LockCompute -- 1.78e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:43:43.006 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260507 17:43:43.102 | INFO | Starting a round of Baz IO | 20260507 17:43:46.607 | INFO | Populating RTMetrics report | 20260507 17:43:46.608 | INFO | RTMetrics report populated | 20260507 17:43:46.608 | INFO | Opening tmp RTMetrics json file | 20260507 17:43:46.609 | INFO | Serializing RTMetrics to json | 20260507 17:43:46.610 | INFO | Serialization done, swapping tmp file now... | 20260507 17:43:46.610 | INFO | RTMetrics done | 20260507 17:43:46.689 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.587 seconds | 20260507 17:43:46.690 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.67% / 0.02% / 52.31% | 20260507 17:43:46.690 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.3875% / 5.16574e-05% / 3.57628e-05% | 20260507 17:43:46.690 | INFO | Baz IO overall: | 20260507 17:43:46.690 | INFO | BytesWritten : 612349744356 | 20260507 17:43:46.690 | INFO | NumEvents : 296754270670 | 20260507 17:43:48.144 | INFO | Analyzing chunk frames = [1384448,1384960) | 20260507 17:43:48.144 | INFO | MemInfo: { | 20260507 17:43:48.144 | INFO | "gpu" : " 17 GB", | 20260507 17:43:48.144 | INFO | "huge" : "", | 20260507 17:43:48.144 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:48.144 | INFO | "malloc" : " 14 GB", | 20260507 17:43:48.144 | INFO | "pinned" : " 29 GB" | 20260507 17:43:48.144 | INFO | } | 20260507 17:43:48.144 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:43:48.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:48.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:48.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:48.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:48.145 | INFO | ANALYSIS : 69.6%, 3.0, 108.5ms, 0 | 20260507 17:43:48.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:48.145 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 17:43:48.145 | INFO | PRE_HQ : 66.9%, 1.7, 59.8ms, 0 | 20260507 17:43:48.145 | INFO | BAZWRITER : 69.0%, 2.9, 102.8ms, 0 | 20260507 17:43:48.145 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:43:48.145 | INFO | Uploaded 36GiB to GPU in 2.56984s (14.0086GiB/s) | 20260507 17:43:48.145 | INFO | Downloaded 38GiB to GPU in 2.71214s (14.0111GiB/s) | 20260507 17:43:53.284 | INFO | Analyzing chunk frames = [1384960,1385472) | 20260507 17:43:53.284 | INFO | MemInfo: { | 20260507 17:43:53.284 | INFO | "gpu" : " 17 GB", | 20260507 17:43:53.284 | INFO | "huge" : "", | 20260507 17:43:53.284 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:53.284 | INFO | "malloc" : " 14 GB", | 20260507 17:43:53.284 | INFO | "pinned" : " 29 GB" | 20260507 17:43:53.284 | INFO | } | 20260507 17:43:53.284 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:43:53.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:53.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:43:53.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 17:43:53.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:53.284 | INFO | Uploaded 36GiB to GPU in 2.55758s (14.0758GiB/s) | 20260507 17:43:53.284 | INFO | Downloaded 34GiB to GPU in 2.44357s (13.9141GiB/s) | 20260507 17:43:58.423 | INFO | Analyzing chunk frames = [1385472,1385984) | 20260507 17:43:58.423 | INFO | MemInfo: { | 20260507 17:43:58.423 | INFO | "gpu" : " 17 GB", | 20260507 17:43:58.423 | INFO | "huge" : "", | 20260507 17:43:58.423 | INFO | "huge_shared" : " 29 GB", | 20260507 17:43:58.423 | INFO | "malloc" : " 14 GB", | 20260507 17:43:58.423 | INFO | "pinned" : " 29 GB" | 20260507 17:43:58.423 | INFO | } | 20260507 17:43:58.423 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:43:58.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:43:58.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:43:58.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:43:58.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:43:58.423 | INFO | Uploaded 36GiB to GPU in 2.57186s (13.9977GiB/s) | 20260507 17:43:58.424 | INFO | Downloaded 34GiB to GPU in 2.41212s (14.0955GiB/s) | 20260507 17:44:03.562 | INFO | Analyzing chunk frames = [1385984,1386496) | 20260507 17:44:03.562 | INFO | MemInfo: { | 20260507 17:44:03.562 | INFO | "gpu" : " 17 GB", | 20260507 17:44:03.562 | INFO | "huge" : "", | 20260507 17:44:03.562 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:03.562 | INFO | "malloc" : " 14 GB", | 20260507 17:44:03.562 | INFO | "pinned" : " 29 GB" | 20260507 17:44:03.562 | INFO | } | 20260507 17:44:03.562 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:44:03.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:03.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.562 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:44:03.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.562 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:44:03.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:03.563 | INFO | Uploaded 36GiB to GPU in 2.49934s (14.4038GiB/s) | 20260507 17:44:03.563 | INFO | Downloaded 34GiB to GPU in 2.34063s (14.526GiB/s) | 20260507 17:44:08.702 | INFO | Analyzing chunk frames = [1386496,1387008) | 20260507 17:44:08.702 | INFO | MemInfo: { | 20260507 17:44:08.702 | INFO | "gpu" : " 17 GB", | 20260507 17:44:08.702 | INFO | "huge" : "", | 20260507 17:44:08.702 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:08.702 | INFO | "malloc" : " 14 GB", | 20260507 17:44:08.702 | INFO | "pinned" : " 29 GB" | 20260507 17:44:08.702 | INFO | } | 20260507 17:44:08.702 | INFO | Perf report for the last 5.14 seconds: | 20260507 17:44:08.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:08.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.703 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 17:44:08.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.703 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:44:08.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:08.703 | INFO | Uploaded 36GiB to GPU in 2.42599s (14.8393GiB/s) | 20260507 17:44:08.703 | INFO | Downloaded 34GiB to GPU in 2.47806s (13.7204GiB/s) | 20260507 17:44:13.841 | INFO | Analyzing chunk frames = [1387008,1387520) | 20260507 17:44:13.841 | INFO | MemInfo: { | 20260507 17:44:13.841 | INFO | "gpu" : " 17 GB", | 20260507 17:44:13.841 | INFO | "huge" : "", | 20260507 17:44:13.841 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:13.841 | INFO | "malloc" : " 14 GB", | 20260507 17:44:13.841 | INFO | "pinned" : " 29 GB" | 20260507 17:44:13.841 | INFO | } | 20260507 17:44:13.841 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:44:13.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:13.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:44:13.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 17:44:13.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:13.842 | INFO | Uploaded 36GiB to GPU in 2.38627s (15.0863GiB/s) | 20260507 17:44:13.842 | INFO | Downloaded 34GiB to GPU in 2.49268s (13.6399GiB/s) | 20260507 17:44:18.980 | INFO | Analyzing chunk frames = [1387520,1388032) | 20260507 17:44:18.980 | INFO | MemInfo: { | 20260507 17:44:18.981 | INFO | "gpu" : " 17 GB", | 20260507 17:44:18.981 | INFO | "huge" : "", | 20260507 17:44:18.981 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:18.981 | INFO | "malloc" : " 14 GB", | 20260507 17:44:18.981 | INFO | "pinned" : " 29 GB" | 20260507 17:44:18.981 | INFO | } | 20260507 17:44:18.981 | INFO | Perf report for the last 5.13923 seconds: | 20260507 17:44:18.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:18.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:44:18.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:44:18.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:18.981 | INFO | Uploaded 36GiB to GPU in 2.4336s (14.7929GiB/s) | 20260507 17:44:18.981 | INFO | Downloaded 34GiB to GPU in 2.44738s (13.8924GiB/s) | 20260507 17:44:24.119 | INFO | Analyzing chunk frames = [1388032,1388544) | 20260507 17:44:24.120 | INFO | MemInfo: { | 20260507 17:44:24.120 | INFO | "gpu" : " 17 GB", | 20260507 17:44:24.120 | INFO | "huge" : "", | 20260507 17:44:24.120 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:24.120 | INFO | "malloc" : " 14 GB", | 20260507 17:44:24.120 | INFO | "pinned" : " 29 GB" | 20260507 17:44:24.120 | INFO | } | 20260507 17:44:24.120 | INFO | Perf report for the last 5.13906 seconds: | 20260507 17:44:24.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:24.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:44:24.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:44:24.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:24.120 | INFO | Uploaded 36GiB to GPU in 2.51791s (14.2976GiB/s) | 20260507 17:44:24.120 | INFO | Downloaded 34GiB to GPU in 2.5071s (13.5615GiB/s) | 20260507 17:44:24.217 | INFO | Starting a round of Baz IO | 20260507 17:44:27.729 | INFO | Populating RTMetrics report | 20260507 17:44:27.729 | INFO | RTMetrics report populated | 20260507 17:44:27.729 | INFO | Opening tmp RTMetrics json file | 20260507 17:44:27.745 | INFO | Serializing RTMetrics to json | 20260507 17:44:27.746 | INFO | Serialization done, swapping tmp file now... | 20260507 17:44:27.747 | INFO | RTMetrics done | 20260507 17:44:27.808 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.591 seconds | 20260507 17:44:27.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.66% / 0.02% / 52.33% | 20260507 17:44:27.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.37375% / 4.76837e-05% / 3.17891e-05% | 20260507 17:44:27.809 | INFO | Baz IO overall: | 20260507 17:44:27.809 | INFO | BytesWritten : 614280216337 | 20260507 17:44:27.809 | INFO | NumEvents : 297691702492 | 20260507 17:44:29.259 | INFO | Analyzing chunk frames = [1388544,1389056) | 20260507 17:44:29.259 | INFO | MemInfo: { | 20260507 17:44:29.259 | INFO | "gpu" : " 17 GB", | 20260507 17:44:29.259 | INFO | "huge" : "", | 20260507 17:44:29.259 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:29.259 | INFO | "malloc" : " 14 GB", | 20260507 17:44:29.259 | INFO | "pinned" : " 29 GB" | 20260507 17:44:29.259 | INFO | } | 20260507 17:44:29.259 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:44:29.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:29.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:29.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:29.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:29.260 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:44:29.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:29.260 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 17:44:29.260 | INFO | PRE_HQ : 66.7%, 1.7, 59.6ms, 0 | 20260507 17:44:29.260 | INFO | BAZWRITER : 68.9%, 2.8, 100.7ms, 0 | 20260507 17:44:29.260 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:44:29.260 | INFO | Uploaded 36GiB to GPU in 2.54366s (14.1528GiB/s) | 20260507 17:44:29.260 | INFO | Downloaded 38GiB to GPU in 2.7298s (13.9204GiB/s) | 20260507 17:44:34.398 | INFO | Analyzing chunk frames = [1389056,1389568) | 20260507 17:44:34.398 | INFO | MemInfo: { | 20260507 17:44:34.398 | INFO | "gpu" : " 17 GB", | 20260507 17:44:34.398 | INFO | "huge" : "", | 20260507 17:44:34.398 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:34.398 | INFO | "malloc" : " 14 GB", | 20260507 17:44:34.399 | INFO | "pinned" : " 29 GB" | 20260507 17:44:34.399 | INFO | } | 20260507 17:44:34.399 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:44:34.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:34.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:44:34.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:44:34.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:34.399 | INFO | Uploaded 36GiB to GPU in 2.52581s (14.2529GiB/s) | 20260507 17:44:34.399 | INFO | Downloaded 34GiB to GPU in 2.44932s (13.8814GiB/s) | 20260507 17:44:39.538 | INFO | Analyzing chunk frames = [1389568,1390080) | 20260507 17:44:39.538 | INFO | MemInfo: { | 20260507 17:44:39.538 | INFO | "gpu" : " 17 GB", | 20260507 17:44:39.538 | INFO | "huge" : "", | 20260507 17:44:39.538 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:39.538 | INFO | "malloc" : " 14 GB", | 20260507 17:44:39.538 | INFO | "pinned" : " 29 GB" | 20260507 17:44:39.538 | INFO | } | 20260507 17:44:39.538 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:44:39.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:39.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.538 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:44:39.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.538 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 17:44:39.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:39.539 | INFO | Uploaded 36GiB to GPU in 2.55091s (14.1126GiB/s) | 20260507 17:44:39.539 | INFO | Downloaded 34GiB to GPU in 2.43186s (13.9811GiB/s) | 20260507 17:44:44.677 | INFO | Analyzing chunk frames = [1390080,1390592) | 20260507 17:44:44.677 | INFO | MemInfo: { | 20260507 17:44:44.677 | INFO | "gpu" : " 17 GB", | 20260507 17:44:44.677 | INFO | "huge" : "", | 20260507 17:44:44.677 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:44.677 | INFO | "malloc" : " 14 GB", | 20260507 17:44:44.677 | INFO | "pinned" : " 29 GB" | 20260507 17:44:44.677 | INFO | } | 20260507 17:44:44.677 | INFO | Perf report for the last 5.13898 seconds: | 20260507 17:44:44.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:44.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.677 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:44:44.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.677 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:44:44.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:44.678 | INFO | Uploaded 36GiB to GPU in 2.48485s (14.4878GiB/s) | 20260507 17:44:44.678 | INFO | Downloaded 34GiB to GPU in 2.35432s (14.4415GiB/s) | 20260507 17:44:49.817 | INFO | Analyzing chunk frames = [1390592,1391104) | 20260507 17:44:49.817 | INFO | MemInfo: { | 20260507 17:44:49.817 | INFO | "gpu" : " 17 GB", | 20260507 17:44:49.817 | INFO | "huge" : "", | 20260507 17:44:49.817 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:49.817 | INFO | "malloc" : " 14 GB", | 20260507 17:44:49.817 | INFO | "pinned" : " 29 GB" | 20260507 17:44:49.817 | INFO | } | 20260507 17:44:49.817 | INFO | Perf report for the last 5.13977 seconds: | 20260507 17:44:49.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:49.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 17:44:49.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:44:49.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:49.817 | INFO | Uploaded 36GiB to GPU in 2.36591s (15.2161GiB/s) | 20260507 17:44:49.817 | INFO | Downloaded 34GiB to GPU in 2.46632s (13.7857GiB/s) | 20260507 17:44:54.956 | INFO | Analyzing chunk frames = [1391104,1391616) | 20260507 17:44:54.956 | INFO | MemInfo: { | 20260507 17:44:54.956 | INFO | "gpu" : " 17 GB", | 20260507 17:44:54.956 | INFO | "huge" : "", | 20260507 17:44:54.956 | INFO | "huge_shared" : " 29 GB", | 20260507 17:44:54.956 | INFO | "malloc" : " 14 GB", | 20260507 17:44:54.956 | INFO | "pinned" : " 29 GB" | 20260507 17:44:54.956 | INFO | } | 20260507 17:44:54.956 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:44:54.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:44:54.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:44:54.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:44:54.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:44:54.956 | INFO | Uploaded 36GiB to GPU in 2.44209s (14.7414GiB/s) | 20260507 17:44:54.957 | INFO | Downloaded 34GiB to GPU in 2.48851s (13.6628GiB/s) | 20260507 17:45:00.095 | INFO | Analyzing chunk frames = [1391616,1392128) | 20260507 17:45:00.095 | INFO | MemInfo: { | 20260507 17:45:00.095 | INFO | "gpu" : " 17 GB", | 20260507 17:45:00.095 | INFO | "huge" : "", | 20260507 17:45:00.095 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:00.095 | INFO | "malloc" : " 14 GB", | 20260507 17:45:00.095 | INFO | "pinned" : " 29 GB" | 20260507 17:45:00.095 | INFO | } | 20260507 17:45:00.095 | INFO | Perf report for the last 5.13921 seconds: | 20260507 17:45:00.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:00.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.095 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:45:00.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.096 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:45:00.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:00.096 | INFO | Uploaded 36GiB to GPU in 2.50181s (14.3896GiB/s) | 20260507 17:45:00.096 | INFO | Downloaded 34GiB to GPU in 2.42309s (14.0317GiB/s) | 20260507 17:45:05.235 | INFO | Analyzing chunk frames = [1392128,1392640) | 20260507 17:45:05.235 | INFO | MemInfo: { | 20260507 17:45:05.235 | INFO | "gpu" : " 17 GB", | 20260507 17:45:05.235 | INFO | "huge" : "", | 20260507 17:45:05.235 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:05.235 | INFO | "malloc" : " 14 GB", | 20260507 17:45:05.235 | INFO | "pinned" : " 29 GB" | 20260507 17:45:05.235 | INFO | } | 20260507 17:45:05.235 | INFO | Perf report for the last 5.13956 seconds: | 20260507 17:45:05.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:05.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 17:45:05.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:45:05.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:05.235 | INFO | Uploaded 36GiB to GPU in 2.55414s (14.0947GiB/s) | 20260507 17:45:05.235 | INFO | Downloaded 34GiB to GPU in 2.49698s (13.6165GiB/s) | 20260507 17:45:05.235 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:45:05.235 | INFO | Total mem usage: 16.4 GiB | 20260507 17:45:05.235 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:45:05.235 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.5%) | 20260507 17:45:05.235 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:45:05.235 | INFO | High Water Breakdown: | 20260507 17:45:05.235 | INFO | 59.1% : 8.54 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:45:05.235 | INFO | 40.8% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:45:05.235 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:45:05.235 | INFO | ------------------------------------------ | 20260507 17:45:05.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:45:05.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:45:05.236 | INFO | Total mem usage: 26.5 GiB | 20260507 17:45:05.236 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:45:05.236 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:45:05.236 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:45:05.236 | INFO | High Water Breakdown: | 20260507 17:45:05.236 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:45:05.236 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:45:05.236 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:45:05.236 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:45:05.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:45:05.236 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:45:05.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:45:05.236 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:45:05.236 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:45:05.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:45:05.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:45:05.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:45:05.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:45:05.236 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:45:05.236 | INFO | ------------------------------------------ | 20260507 17:45:05.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:45:05.236 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:45:05.236 | INFO | Total mem usage: 27.8 GiB | 20260507 17:45:05.236 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:45:05.236 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:45:05.236 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:45:05.236 | INFO | High Water Breakdown: | 20260507 17:45:05.236 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:45:05.236 | INFO | ------------------------------------------ | 20260507 17:45:05.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:45:05.236 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:45:05.236 | INFO | Total mem usage: 15.7 GiB | 20260507 17:45:05.236 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:45:05.236 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:45:05.236 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:45:05.236 | INFO | High Water Breakdown: | 20260507 17:45:05.236 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:45:05.236 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:45:05.236 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:45:05.236 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:45:05.236 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:45:05.236 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:45:05.236 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:45:05.236 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:45:05.236 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:45:05.236 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:45:05.236 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:45:05.236 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:45:05.236 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:45:05.236 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:45:05.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:45:05.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:45:05.236 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:45:05.236 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:45:05.236 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:45:05.236 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:45:05.236 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:45:05.236 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:45:05.236 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:45:05.236 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:45:05.236 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:45:05.236 | INFO | ------------------------------------------ | 20260507 17:45:05.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:45:05.236 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:45:05.236 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:45:05.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:45:05.236 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.68e+01, 1.42e+01 / 2.50e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.13e-02 / 4.81e-03 | 20260507 17:45:05.236 | INFO | PerfReport: 12.05%, Baseline -- 4.03e+00 / 4.52e+00, 6.20e-01 / 1.97e-01 | 20260507 17:45:05.236 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 8.32e-02 | 20260507 17:45:05.236 | INFO | PerfReport: 15.76%, DME -- 4.22e+01 / 4.52e+01, 4.27e+00 / 2.60e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.86e+00 / 2.06e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.87e+00, 4.37e-01 / 2.23e-01 | 20260507 17:45:05.236 | INFO | PerfReport: 3.75%, Metrics -- 1.26e+00 / 1.43e+00, 3.20e-01 / 1.76e-01 | 20260507 17:45:05.236 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.18e-01, 6.59e-02 / 3.88e-02 | 20260507 17:45:05.236 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:45:05.236 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:45:05.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:45:05.236 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 24.15%, Upload -- 2.60e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 23.99%, Download -- 2.58e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 31.08%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 1.93%, LockUpload -- 2.07e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 2.25%, LockDownload -- 2.42e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 16.60%, LockCompute -- 1.79e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:45:05.236 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 17:45:05.332 | INFO | Starting a round of Baz IO | 20260507 17:45:08.841 | INFO | Populating RTMetrics report | 20260507 17:45:08.842 | INFO | RTMetrics report populated | 20260507 17:45:08.842 | INFO | Opening tmp RTMetrics json file | 20260507 17:45:08.851 | INFO | Serializing RTMetrics to json | 20260507 17:45:08.860 | INFO | Serialization done, swapping tmp file now... | 20260507 17:45:08.860 | INFO | RTMetrics done | 20260507 17:45:08.920 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.588 seconds | 20260507 17:45:08.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.64% / 0.02% / 52.34% | 20260507 17:45:08.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.37431% / 3.57628e-05% / 2.78155e-05% | 20260507 17:45:08.921 | INFO | Baz IO overall: | 20260507 17:45:08.921 | INFO | BytesWritten : 616212648316 | 20260507 17:45:08.921 | INFO | NumEvents : 298630610383 | 20260507 17:45:10.374 | INFO | Analyzing chunk frames = [1392640,1393152) | 20260507 17:45:10.374 | INFO | MemInfo: { | 20260507 17:45:10.374 | INFO | "gpu" : " 17 GB", | 20260507 17:45:10.374 | INFO | "huge" : "", | 20260507 17:45:10.374 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:10.374 | INFO | "malloc" : " 14 GB", | 20260507 17:45:10.374 | INFO | "pinned" : " 29 GB" | 20260507 17:45:10.374 | INFO | } | 20260507 17:45:10.374 | INFO | Perf report for the last 5.13875 seconds: | 20260507 17:45:10.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:10.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:10.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:10.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:10.374 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:45:10.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:10.374 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 17:45:10.374 | INFO | PRE_HQ : 66.7%, 1.7, 59.6ms, 0 | 20260507 17:45:10.374 | INFO | BAZWRITER : 68.9%, 2.8, 100.2ms, 0 | 20260507 17:45:10.374 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:45:10.374 | INFO | Uploaded 36GiB to GPU in 2.58502s (13.9264GiB/s) | 20260507 17:45:10.375 | INFO | Downloaded 38GiB to GPU in 2.71608s (13.9907GiB/s) | 20260507 17:45:15.513 | INFO | Analyzing chunk frames = [1393152,1393664) | 20260507 17:45:15.513 | INFO | MemInfo: { | 20260507 17:45:15.513 | INFO | "gpu" : " 17 GB", | 20260507 17:45:15.513 | INFO | "huge" : "", | 20260507 17:45:15.513 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:15.513 | INFO | "malloc" : " 14 GB", | 20260507 17:45:15.513 | INFO | "pinned" : " 29 GB" | 20260507 17:45:15.513 | INFO | } | 20260507 17:45:15.513 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:45:15.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:15.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 17:45:15.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:45:15.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:15.514 | INFO | Uploaded 36GiB to GPU in 2.53959s (14.1755GiB/s) | 20260507 17:45:15.514 | INFO | Downloaded 34GiB to GPU in 2.44355s (13.9142GiB/s) | 20260507 17:45:20.653 | INFO | Analyzing chunk frames = [1393664,1394176) | 20260507 17:45:20.653 | INFO | MemInfo: { | 20260507 17:45:20.653 | INFO | "gpu" : " 17 GB", | 20260507 17:45:20.653 | INFO | "huge" : "", | 20260507 17:45:20.653 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:20.653 | INFO | "malloc" : " 14 GB", | 20260507 17:45:20.653 | INFO | "pinned" : " 29 GB" | 20260507 17:45:20.653 | INFO | } | 20260507 17:45:20.653 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:45:20.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:20.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:45:20.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:45:20.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:20.653 | INFO | Uploaded 36GiB to GPU in 2.57135s (14.0004GiB/s) | 20260507 17:45:20.653 | INFO | Downloaded 34GiB to GPU in 2.4141s (14.0839GiB/s) | 20260507 17:45:25.792 | INFO | Analyzing chunk frames = [1394176,1394688) | 20260507 17:45:25.792 | INFO | MemInfo: { | 20260507 17:45:25.792 | INFO | "gpu" : " 17 GB", | 20260507 17:45:25.792 | INFO | "huge" : "", | 20260507 17:45:25.792 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:25.792 | INFO | "malloc" : " 14 GB", | 20260507 17:45:25.792 | INFO | "pinned" : " 29 GB" | 20260507 17:45:25.792 | INFO | } | 20260507 17:45:25.792 | INFO | Perf report for the last 5.13955 seconds: | 20260507 17:45:25.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:25.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.792 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:45:25.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.792 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:45:25.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:25.793 | INFO | Uploaded 36GiB to GPU in 2.50215s (14.3876GiB/s) | 20260507 17:45:25.793 | INFO | Downloaded 34GiB to GPU in 2.33831s (14.5404GiB/s) | 20260507 17:45:30.931 | INFO | Analyzing chunk frames = [1394688,1395200) | 20260507 17:45:30.931 | INFO | MemInfo: { | 20260507 17:45:30.931 | INFO | "gpu" : " 17 GB", | 20260507 17:45:30.931 | INFO | "huge" : "", | 20260507 17:45:30.931 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:30.931 | INFO | "malloc" : " 14 GB", | 20260507 17:45:30.931 | INFO | "pinned" : " 29 GB" | 20260507 17:45:30.931 | INFO | } | 20260507 17:45:30.931 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:45:30.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:30.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.932 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:45:30.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.932 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:45:30.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:30.932 | INFO | Uploaded 36GiB to GPU in 2.38619s (15.0868GiB/s) | 20260507 17:45:30.932 | INFO | Downloaded 34GiB to GPU in 2.45532s (13.8475GiB/s) | 20260507 17:45:36.070 | INFO | Analyzing chunk frames = [1395200,1395712) | 20260507 17:45:36.070 | INFO | MemInfo: { | 20260507 17:45:36.070 | INFO | "gpu" : " 17 GB", | 20260507 17:45:36.070 | INFO | "huge" : "", | 20260507 17:45:36.070 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:36.070 | INFO | "malloc" : " 14 GB", | 20260507 17:45:36.071 | INFO | "pinned" : " 29 GB" | 20260507 17:45:36.071 | INFO | } | 20260507 17:45:36.071 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:45:36.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:36.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:45:36.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:45:36.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:36.071 | INFO | Uploaded 36GiB to GPU in 2.38734s (15.0796GiB/s) | 20260507 17:45:36.071 | INFO | Downloaded 34GiB to GPU in 2.49789s (13.6115GiB/s) | 20260507 17:45:41.210 | INFO | Analyzing chunk frames = [1395712,1396224) | 20260507 17:45:41.210 | INFO | MemInfo: { | 20260507 17:45:41.210 | INFO | "gpu" : " 17 GB", | 20260507 17:45:41.210 | INFO | "huge" : "", | 20260507 17:45:41.210 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:41.210 | INFO | "malloc" : " 14 GB", | 20260507 17:45:41.210 | INFO | "pinned" : " 29 GB" | 20260507 17:45:41.210 | INFO | } | 20260507 17:45:41.210 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:45:41.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:41.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.210 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:45:41.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.210 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:45:41.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:41.211 | INFO | Uploaded 36GiB to GPU in 2.46234s (14.6202GiB/s) | 20260507 17:45:41.211 | INFO | Downloaded 34GiB to GPU in 2.42706s (14.0087GiB/s) | 20260507 17:45:46.349 | INFO | Analyzing chunk frames = [1396224,1396736) | 20260507 17:45:46.349 | INFO | MemInfo: { | 20260507 17:45:46.349 | INFO | "gpu" : " 17 GB", | 20260507 17:45:46.349 | INFO | "huge" : "", | 20260507 17:45:46.350 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:46.350 | INFO | "malloc" : " 14 GB", | 20260507 17:45:46.350 | INFO | "pinned" : " 29 GB" | 20260507 17:45:46.350 | INFO | } | 20260507 17:45:46.350 | INFO | Perf report for the last 5.13971 seconds: | 20260507 17:45:46.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:46.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 17:45:46.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 17:45:46.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:46.350 | INFO | Uploaded 36GiB to GPU in 2.5018s (14.3897GiB/s) | 20260507 17:45:46.350 | INFO | Downloaded 34GiB to GPU in 2.50594s (13.5678GiB/s) | 20260507 17:45:46.461 | INFO | Starting a round of Baz IO | 20260507 17:45:49.954 | INFO | Populating RTMetrics report | 20260507 17:45:49.955 | INFO | RTMetrics report populated | 20260507 17:45:49.955 | INFO | Opening tmp RTMetrics json file | 20260507 17:45:49.969 | INFO | Serializing RTMetrics to json | 20260507 17:45:49.970 | INFO | Serialization done, swapping tmp file now... | 20260507 17:45:49.971 | INFO | RTMetrics done | 20260507 17:45:50.028 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.566 seconds | 20260507 17:45:50.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.62% / 0.02% / 52.36% | 20260507 17:45:50.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.37013% / 3.17891e-05% / 4.76837e-05% | 20260507 17:45:50.028 | INFO | Baz IO overall: | 20260507 17:45:50.028 | INFO | BytesWritten : 618146185642 | 20260507 17:45:50.028 | INFO | NumEvents : 299570543869 | 20260507 17:45:51.488 | INFO | Analyzing chunk frames = [1396736,1397248) | 20260507 17:45:51.489 | INFO | MemInfo: { | 20260507 17:45:51.489 | INFO | "gpu" : " 17 GB", | 20260507 17:45:51.489 | INFO | "huge" : "", | 20260507 17:45:51.489 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:51.489 | INFO | "malloc" : " 14 GB", | 20260507 17:45:51.489 | INFO | "pinned" : " 29 GB" | 20260507 17:45:51.489 | INFO | } | 20260507 17:45:51.489 | INFO | Perf report for the last 5.13902 seconds: | 20260507 17:45:51.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:51.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:51.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:51.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:51.489 | INFO | ANALYSIS : 69.2%, 2.9, 106.9ms, 0 | 20260507 17:45:51.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:51.489 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 17:45:51.489 | INFO | PRE_HQ : 67.0%, 1.9, 65.8ms, 0 | 20260507 17:45:51.489 | INFO | BAZWRITER : 68.8%, 3.1, 108.8ms, 0 | 20260507 17:45:51.489 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:45:51.489 | INFO | Uploaded 36GiB to GPU in 2.49103s (14.4519GiB/s) | 20260507 17:45:51.489 | INFO | Downloaded 38GiB to GPU in 2.70728s (14.0362GiB/s) | 20260507 17:45:56.628 | INFO | Analyzing chunk frames = [1397248,1397760) | 20260507 17:45:56.628 | INFO | MemInfo: { | 20260507 17:45:56.628 | INFO | "gpu" : " 17 GB", | 20260507 17:45:56.628 | INFO | "huge" : "", | 20260507 17:45:56.628 | INFO | "huge_shared" : " 29 GB", | 20260507 17:45:56.628 | INFO | "malloc" : " 14 GB", | 20260507 17:45:56.628 | INFO | "pinned" : " 29 GB" | 20260507 17:45:56.628 | INFO | } | 20260507 17:45:56.628 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:45:56.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:45:56.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 17:45:56.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 17:45:56.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:45:56.628 | INFO | Uploaded 36GiB to GPU in 2.44295s (14.7363GiB/s) | 20260507 17:45:56.629 | INFO | Downloaded 34GiB to GPU in 2.46303s (13.8042GiB/s) | 20260507 17:46:01.767 | INFO | Analyzing chunk frames = [1397760,1398272) | 20260507 17:46:01.767 | INFO | MemInfo: { | 20260507 17:46:01.767 | INFO | "gpu" : " 17 GB", | 20260507 17:46:01.767 | INFO | "huge" : "", | 20260507 17:46:01.767 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:01.767 | INFO | "malloc" : " 14 GB", | 20260507 17:46:01.767 | INFO | "pinned" : " 29 GB" | 20260507 17:46:01.767 | INFO | } | 20260507 17:46:01.767 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:46:01.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:01.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.767 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 17:46:01.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.768 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:46:01.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:01.768 | INFO | Uploaded 36GiB to GPU in 2.49969s (14.4018GiB/s) | 20260507 17:46:01.768 | INFO | Downloaded 34GiB to GPU in 2.43808s (13.9454GiB/s) | 20260507 17:46:06.906 | INFO | Analyzing chunk frames = [1398272,1398784) | 20260507 17:46:06.906 | INFO | MemInfo: { | 20260507 17:46:06.906 | INFO | "gpu" : " 17 GB", | 20260507 17:46:06.906 | INFO | "huge" : "", | 20260507 17:46:06.906 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:06.906 | INFO | "malloc" : " 14 GB", | 20260507 17:46:06.906 | INFO | "pinned" : " 29 GB" | 20260507 17:46:06.907 | INFO | } | 20260507 17:46:06.907 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:46:06.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:06.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 17:46:06.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:46:06.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:06.907 | INFO | Uploaded 36GiB to GPU in 2.45117s (14.6869GiB/s) | 20260507 17:46:06.907 | INFO | Downloaded 34GiB to GPU in 2.37475s (14.3173GiB/s) | 20260507 17:46:12.046 | INFO | Analyzing chunk frames = [1398784,1399296) | 20260507 17:46:12.046 | INFO | MemInfo: { | 20260507 17:46:12.046 | INFO | "gpu" : " 17 GB", | 20260507 17:46:12.046 | INFO | "huge" : "", | 20260507 17:46:12.046 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:12.046 | INFO | "malloc" : " 14 GB", | 20260507 17:46:12.046 | INFO | "pinned" : " 29 GB" | 20260507 17:46:12.046 | INFO | } | 20260507 17:46:12.046 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:46:12.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:12.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.046 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:46:12.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.046 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:46:12.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:12.047 | INFO | Uploaded 36GiB to GPU in 2.30377s (15.6266GiB/s) | 20260507 17:46:12.047 | INFO | Downloaded 34GiB to GPU in 2.48839s (13.6635GiB/s) | 20260507 17:46:17.185 | INFO | Analyzing chunk frames = [1399296,1399808) | 20260507 17:46:17.185 | INFO | MemInfo: { | 20260507 17:46:17.185 | INFO | "gpu" : " 17 GB", | 20260507 17:46:17.185 | INFO | "huge" : "", | 20260507 17:46:17.185 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:17.185 | INFO | "malloc" : " 14 GB", | 20260507 17:46:17.185 | INFO | "pinned" : " 29 GB" | 20260507 17:46:17.186 | INFO | } | 20260507 17:46:17.186 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:46:17.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:17.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260507 17:46:17.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:46:17.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:17.186 | INFO | Uploaded 36GiB to GPU in 2.27762s (15.806GiB/s) | 20260507 17:46:17.186 | INFO | Downloaded 34GiB to GPU in 2.51212s (13.5344GiB/s) | 20260507 17:46:22.324 | INFO | Analyzing chunk frames = [1399808,1400320) | 20260507 17:46:22.325 | INFO | MemInfo: { | 20260507 17:46:22.325 | INFO | "gpu" : " 17 GB", | 20260507 17:46:22.325 | INFO | "huge" : "", | 20260507 17:46:22.325 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:22.325 | INFO | "malloc" : " 14 GB", | 20260507 17:46:22.325 | INFO | "pinned" : " 29 GB" | 20260507 17:46:22.325 | INFO | } | 20260507 17:46:22.325 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:46:22.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:22.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.325 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:46:22.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.325 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 17:46:22.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:22.326 | INFO | Uploaded 36GiB to GPU in 2.42461s (14.8477GiB/s) | 20260507 17:46:22.326 | INFO | Downloaded 34GiB to GPU in 2.44496s (13.9062GiB/s) | 20260507 17:46:27.468 | INFO | Analyzing chunk frames = [1400320,1400832) | 20260507 17:46:27.468 | INFO | MemInfo: { | 20260507 17:46:27.468 | INFO | "gpu" : " 17 GB", | 20260507 17:46:27.468 | INFO | "huge" : "", | 20260507 17:46:27.468 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:27.468 | INFO | "malloc" : " 14 GB", | 20260507 17:46:27.468 | INFO | "pinned" : " 29 GB" | 20260507 17:46:27.468 | INFO | } | 20260507 17:46:27.468 | INFO | Perf report for the last 5.1431 seconds: | 20260507 17:46:27.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:27.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.468 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:46:27.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.468 | INFO | PRE_HQ : 38.1%, 1.0, 19.9ms, 0 | 20260507 17:46:27.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:27.469 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:46:27.469 | INFO | Total mem usage: 16.4 GiB | 20260507 17:46:27.469 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:46:27.469 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 17:46:27.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:46:27.469 | INFO | High Water Breakdown: | 20260507 17:46:27.469 | INFO | 59.2% : 8.63 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:46:27.469 | INFO | 40.6% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:46:27.469 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:46:27.469 | INFO | ------------------------------------------ | 20260507 17:46:27.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:46:27.469 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:46:27.469 | INFO | Total mem usage: 26.5 GiB | 20260507 17:46:27.469 | INFO | Uploaded 36GiB to GPU in 2.45845s (14.6434GiB/s) | 20260507 17:46:27.469 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:46:27.469 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:46:27.469 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:46:27.469 | INFO | Downloaded 34GiB to GPU in 2.51784s (13.5036GiB/s) | 20260507 17:46:27.469 | INFO | High Water Breakdown: | 20260507 17:46:27.469 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:46:27.469 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:46:27.469 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:46:27.469 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:46:27.469 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:46:27.469 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:46:27.469 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:46:27.469 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:46:27.469 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:46:27.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:46:27.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:46:27.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:46:27.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:46:27.469 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:46:27.469 | INFO | ------------------------------------------ | 20260507 17:46:27.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:46:27.469 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:46:27.469 | INFO | Total mem usage: 27.8 GiB | 20260507 17:46:27.469 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:46:27.469 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:46:27.469 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:46:27.469 | INFO | High Water Breakdown: | 20260507 17:46:27.469 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:46:27.469 | INFO | ------------------------------------------ | 20260507 17:46:27.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:46:27.469 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:46:27.469 | INFO | Total mem usage: 15.7 GiB | 20260507 17:46:27.469 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:46:27.469 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:46:27.470 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:46:27.470 | INFO | High Water Breakdown: | 20260507 17:46:27.470 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:46:27.470 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:46:27.470 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:46:27.470 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:46:27.470 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:46:27.470 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:46:27.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:46:27.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:46:27.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:46:27.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:46:27.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:46:27.470 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:46:27.470 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:46:27.470 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:46:27.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:46:27.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:46:27.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:46:27.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:46:27.470 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:46:27.470 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:46:27.470 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:46:27.470 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:46:27.470 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:46:27.470 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:46:27.470 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:46:27.470 | INFO | ------------------------------------------ | 20260507 17:46:27.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:46:27.470 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:46:27.470 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:46:27.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:46:27.470 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.64e+01, 1.40e+01 / 2.53e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 0.29%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.10e-02 / 5.19e-03 | 20260507 17:46:27.470 | INFO | PerfReport: 12.07%, Baseline -- 4.04e+00 / 4.50e+00, 6.18e-01 / 2.13e-01 | 20260507 17:46:27.470 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 8.77e-02 | 20260507 17:46:27.470 | INFO | PerfReport: 15.72%, DME -- 4.21e+01 / 4.48e+01, 4.08e+00 / 2.72e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.85e+00 / 2.45e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.52e+00 / 2.86e+00, 4.37e-01 / 2.52e-01 | 20260507 17:46:27.470 | INFO | PerfReport: 3.77%, Metrics -- 1.26e+00 / 1.42e+00, 3.20e-01 / 1.84e-01 | 20260507 17:46:27.470 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.97e-01 / 4.18e-01, 6.56e-02 / 4.49e-02 | 20260507 17:46:27.470 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:46:27.470 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:46:27.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:46:27.470 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 23.83%, Upload -- 2.56e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 24.11%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 31.17%, Compute -- 3.35e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 1.71%, LockUpload -- 1.83e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 2.53%, LockDownload -- 2.72e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 16.64%, LockCompute -- 1.79e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 17:46:27.470 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 17:46:27.583 | INFO | Starting a round of Baz IO | 20260507 17:46:31.078 | INFO | Populating RTMetrics report | 20260507 17:46:31.078 | INFO | RTMetrics report populated | 20260507 17:46:31.078 | INFO | Opening tmp RTMetrics json file | 20260507 17:46:31.079 | INFO | Serializing RTMetrics to json | 20260507 17:46:31.080 | INFO | Serialization done, swapping tmp file now... | 20260507 17:46:31.080 | INFO | RTMetrics done | 20260507 17:46:31.160 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.577 seconds | 20260507 17:46:31.160 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.60% / 0.02% / 52.38% | 20260507 17:46:31.160 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.362% / 2.78155e-05% / 3.57628e-05% | 20260507 17:46:31.160 | INFO | Baz IO overall: | 20260507 17:46:31.160 | INFO | BytesWritten : 620079455884 | 20260507 17:46:31.160 | INFO | NumEvents : 300510148226 | 20260507 17:46:32.603 | INFO | Analyzing chunk frames = [1400832,1401344) | 20260507 17:46:32.603 | INFO | MemInfo: { | 20260507 17:46:32.603 | INFO | "gpu" : " 17 GB", | 20260507 17:46:32.603 | INFO | "huge" : "", | 20260507 17:46:32.603 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:32.603 | INFO | "malloc" : " 14 GB", | 20260507 17:46:32.603 | INFO | "pinned" : " 29 GB" | 20260507 17:46:32.603 | INFO | } | 20260507 17:46:32.603 | INFO | Perf report for the last 5.13526 seconds: | 20260507 17:46:32.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:32.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:32.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:32.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:32.604 | INFO | ANALYSIS : 69.8%, 2.9, 107.9ms, 0 | 20260507 17:46:32.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:32.604 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 17:46:32.604 | INFO | PRE_HQ : 67.4%, 1.9, 65.6ms, 0 | 20260507 17:46:32.604 | INFO | BAZWRITER : 68.5%, 3.0, 106.5ms, 0 | 20260507 17:46:32.604 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:46:32.604 | INFO | Uploaded 36GiB to GPU in 2.48865s (14.4657GiB/s) | 20260507 17:46:32.604 | INFO | Downloaded 38GiB to GPU in 2.72211s (13.9597GiB/s) | 20260507 17:46:37.742 | INFO | Analyzing chunk frames = [1401344,1401856) | 20260507 17:46:37.742 | INFO | MemInfo: { | 20260507 17:46:37.742 | INFO | "gpu" : " 17 GB", | 20260507 17:46:37.742 | INFO | "huge" : "", | 20260507 17:46:37.742 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:37.743 | INFO | "malloc" : " 14 GB", | 20260507 17:46:37.743 | INFO | "pinned" : " 29 GB" | 20260507 17:46:37.743 | INFO | } | 20260507 17:46:37.743 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:46:37.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:37.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260507 17:46:37.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 17:46:37.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:37.743 | INFO | Uploaded 36GiB to GPU in 2.43872s (14.7618GiB/s) | 20260507 17:46:37.743 | INFO | Downloaded 34GiB to GPU in 2.44835s (13.8869GiB/s) | 20260507 17:46:42.882 | INFO | Analyzing chunk frames = [1401856,1402368) | 20260507 17:46:42.882 | INFO | MemInfo: { | 20260507 17:46:42.882 | INFO | "gpu" : " 17 GB", | 20260507 17:46:42.882 | INFO | "huge" : "", | 20260507 17:46:42.882 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:42.882 | INFO | "malloc" : " 14 GB", | 20260507 17:46:42.882 | INFO | "pinned" : " 29 GB" | 20260507 17:46:42.882 | INFO | } | 20260507 17:46:42.882 | INFO | Perf report for the last 5.1395 seconds: | 20260507 17:46:42.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:42.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.882 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 17:46:42.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.882 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:46:42.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:42.883 | INFO | Uploaded 36GiB to GPU in 2.52031s (14.2839GiB/s) | 20260507 17:46:42.883 | INFO | Downloaded 34GiB to GPU in 2.44893s (13.8836GiB/s) | 20260507 17:46:48.021 | INFO | Analyzing chunk frames = [1402368,1402880) | 20260507 17:46:48.021 | INFO | MemInfo: { | 20260507 17:46:48.021 | INFO | "gpu" : " 17 GB", | 20260507 17:46:48.021 | INFO | "huge" : "", | 20260507 17:46:48.022 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:48.022 | INFO | "malloc" : " 14 GB", | 20260507 17:46:48.022 | INFO | "pinned" : " 29 GB" | 20260507 17:46:48.022 | INFO | } | 20260507 17:46:48.022 | INFO | Perf report for the last 5.13945 seconds: | 20260507 17:46:48.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:48.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.022 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 17:46:48.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.022 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:46:48.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:48.023 | INFO | Uploaded 36GiB to GPU in 2.43517s (14.7834GiB/s) | 20260507 17:46:48.023 | INFO | Downloaded 34GiB to GPU in 2.36634s (14.3682GiB/s) | 20260507 17:46:53.161 | INFO | Analyzing chunk frames = [1402880,1403392) | 20260507 17:46:53.161 | INFO | MemInfo: { | 20260507 17:46:53.161 | INFO | "gpu" : " 17 GB", | 20260507 17:46:53.161 | INFO | "huge" : "", | 20260507 17:46:53.161 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:53.161 | INFO | "malloc" : " 14 GB", | 20260507 17:46:53.161 | INFO | "pinned" : " 29 GB" | 20260507 17:46:53.161 | INFO | } | 20260507 17:46:53.161 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:46:53.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:53.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.161 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 17:46:53.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.161 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:46:53.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:53.162 | INFO | Uploaded 36GiB to GPU in 2.30675s (15.6064GiB/s) | 20260507 17:46:53.162 | INFO | Downloaded 34GiB to GPU in 2.47825s (13.7194GiB/s) | 20260507 17:46:58.300 | INFO | Analyzing chunk frames = [1403392,1403904) | 20260507 17:46:58.300 | INFO | MemInfo: { | 20260507 17:46:58.300 | INFO | "gpu" : " 17 GB", | 20260507 17:46:58.300 | INFO | "huge" : "", | 20260507 17:46:58.300 | INFO | "huge_shared" : " 29 GB", | 20260507 17:46:58.300 | INFO | "malloc" : " 14 GB", | 20260507 17:46:58.300 | INFO | "pinned" : " 29 GB" | 20260507 17:46:58.300 | INFO | } | 20260507 17:46:58.301 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:46:58.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:46:58.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:46:58.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260507 17:46:58.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:46:58.301 | INFO | Uploaded 36GiB to GPU in 2.32009s (15.5167GiB/s) | 20260507 17:46:58.301 | INFO | Downloaded 34GiB to GPU in 2.51336s (13.5277GiB/s) | 20260507 17:47:03.439 | INFO | Analyzing chunk frames = [1403904,1404416) | 20260507 17:47:03.439 | INFO | MemInfo: { | 20260507 17:47:03.439 | INFO | "gpu" : " 17 GB", | 20260507 17:47:03.439 | INFO | "huge" : "", | 20260507 17:47:03.439 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:03.439 | INFO | "malloc" : " 14 GB", | 20260507 17:47:03.439 | INFO | "pinned" : " 29 GB" | 20260507 17:47:03.439 | INFO | } | 20260507 17:47:03.440 | INFO | Perf report for the last 5.13906 seconds: | 20260507 17:47:03.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:03.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:47:03.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 17:47:03.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:03.440 | INFO | Uploaded 36GiB to GPU in 2.46182s (14.6234GiB/s) | 20260507 17:47:03.440 | INFO | Downloaded 34GiB to GPU in 2.45188s (13.8669GiB/s) | 20260507 17:47:08.578 | INFO | Analyzing chunk frames = [1404416,1404928) | 20260507 17:47:08.578 | INFO | MemInfo: { | 20260507 17:47:08.578 | INFO | "gpu" : " 17 GB", | 20260507 17:47:08.578 | INFO | "huge" : "", | 20260507 17:47:08.578 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:08.578 | INFO | "malloc" : " 14 GB", | 20260507 17:47:08.579 | INFO | "pinned" : " 29 GB" | 20260507 17:47:08.579 | INFO | } | 20260507 17:47:08.579 | INFO | Perf report for the last 5.13909 seconds: | 20260507 17:47:08.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:08.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:47:08.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:47:08.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:08.579 | INFO | Uploaded 36GiB to GPU in 2.43974s (14.7557GiB/s) | 20260507 17:47:08.579 | INFO | Downloaded 34GiB to GPU in 2.43763s (13.948GiB/s) | 20260507 17:47:08.689 | INFO | Starting a round of Baz IO | 20260507 17:47:12.196 | INFO | Populating RTMetrics report | 20260507 17:47:12.197 | INFO | RTMetrics report populated | 20260507 17:47:12.197 | INFO | Opening tmp RTMetrics json file | 20260507 17:47:12.213 | INFO | Serializing RTMetrics to json | 20260507 17:47:12.214 | INFO | Serialization done, swapping tmp file now... | 20260507 17:47:12.214 | INFO | RTMetrics done | 20260507 17:47:12.270 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.58 seconds | 20260507 17:47:12.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.58% / 0.02% / 52.40% | 20260507 17:47:12.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.35256% / 2.38419e-05% / 3.57628e-05% | 20260507 17:47:12.270 | INFO | Baz IO overall: | 20260507 17:47:12.270 | INFO | BytesWritten : 622012358050 | 20260507 17:47:12.270 | INFO | NumEvents : 301449090144 | 20260507 17:47:13.718 | INFO | Analyzing chunk frames = [1404928,1405440) | 20260507 17:47:13.718 | INFO | MemInfo: { | 20260507 17:47:13.718 | INFO | "gpu" : " 17 GB", | 20260507 17:47:13.718 | INFO | "huge" : "", | 20260507 17:47:13.718 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:13.718 | INFO | "malloc" : " 14 GB", | 20260507 17:47:13.718 | INFO | "pinned" : " 29 GB" | 20260507 17:47:13.718 | INFO | } | 20260507 17:47:13.719 | INFO | Perf report for the last 5.1398 seconds: | 20260507 17:47:13.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:13.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:13.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:13.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:13.719 | INFO | ANALYSIS : 69.4%, 3.0, 107.7ms, 0 | 20260507 17:47:13.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:13.719 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260507 17:47:13.719 | INFO | PRE_HQ : 66.8%, 1.9, 67.5ms, 0 | 20260507 17:47:13.719 | INFO | BAZWRITER : 68.9%, 3.1, 109.3ms, 0 | 20260507 17:47:13.719 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 17:47:13.720 | INFO | Uploaded 36GiB to GPU in 2.49841s (14.4092GiB/s) | 20260507 17:47:13.720 | INFO | Downloaded 38GiB to GPU in 2.72245s (13.958GiB/s) | 20260507 17:47:18.857 | INFO | Analyzing chunk frames = [1405440,1405952) | 20260507 17:47:18.857 | INFO | MemInfo: { | 20260507 17:47:18.857 | INFO | "gpu" : " 17 GB", | 20260507 17:47:18.857 | INFO | "huge" : "", | 20260507 17:47:18.857 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:18.857 | INFO | "malloc" : " 14 GB", | 20260507 17:47:18.858 | INFO | "pinned" : " 29 GB" | 20260507 17:47:18.858 | INFO | } | 20260507 17:47:18.858 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:47:18.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:18.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:47:18.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 17:47:18.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:18.858 | INFO | Uploaded 36GiB to GPU in 2.4771s (14.5331GiB/s) | 20260507 17:47:18.858 | INFO | Downloaded 34GiB to GPU in 2.47674s (13.7277GiB/s) | 20260507 17:47:23.997 | INFO | Analyzing chunk frames = [1405952,1406464) | 20260507 17:47:23.997 | INFO | MemInfo: { | 20260507 17:47:23.997 | INFO | "gpu" : " 17 GB", | 20260507 17:47:23.997 | INFO | "huge" : "", | 20260507 17:47:23.997 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:23.997 | INFO | "malloc" : " 14 GB", | 20260507 17:47:23.997 | INFO | "pinned" : " 29 GB" | 20260507 17:47:23.997 | INFO | } | 20260507 17:47:23.997 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:47:23.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:23.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.997 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 17:47:23.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.998 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 17:47:23.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:23.998 | INFO | Uploaded 36GiB to GPU in 2.51961s (14.2879GiB/s) | 20260507 17:47:23.998 | INFO | Downloaded 34GiB to GPU in 2.47208s (13.7536GiB/s) | 20260507 17:47:29.136 | INFO | Analyzing chunk frames = [1406464,1406976) | 20260507 17:47:29.136 | INFO | MemInfo: { | 20260507 17:47:29.136 | INFO | "gpu" : " 17 GB", | 20260507 17:47:29.136 | INFO | "huge" : "", | 20260507 17:47:29.136 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:29.136 | INFO | "malloc" : " 14 GB", | 20260507 17:47:29.137 | INFO | "pinned" : " 29 GB" | 20260507 17:47:29.137 | INFO | } | 20260507 17:47:29.137 | INFO | Perf report for the last 5.13933 seconds: | 20260507 17:47:29.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:29.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 17:47:29.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | PRE_HQ : 36.7%, 1.0, 19.0ms, 0 | 20260507 17:47:29.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:29.137 | INFO | Uploaded 36GiB to GPU in 2.42629s (14.8375GiB/s) | 20260507 17:47:29.137 | INFO | Downloaded 34GiB to GPU in 2.3848s (14.2569GiB/s) | 20260507 17:47:34.276 | INFO | Analyzing chunk frames = [1406976,1407488) | 20260507 17:47:34.276 | INFO | MemInfo: { | 20260507 17:47:34.276 | INFO | "gpu" : " 17 GB", | 20260507 17:47:34.276 | INFO | "huge" : "", | 20260507 17:47:34.276 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:34.276 | INFO | "malloc" : " 14 GB", | 20260507 17:47:34.276 | INFO | "pinned" : " 29 GB" | 20260507 17:47:34.276 | INFO | } | 20260507 17:47:34.276 | INFO | Perf report for the last 5.13917 seconds: | 20260507 17:47:34.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:34.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 17:47:34.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 17:47:34.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:34.276 | INFO | Uploaded 36GiB to GPU in 2.32551s (15.4805GiB/s) | 20260507 17:47:34.276 | INFO | Downloaded 34GiB to GPU in 2.49606s (13.6214GiB/s) | 20260507 17:47:39.415 | INFO | Analyzing chunk frames = [1407488,1408000) | 20260507 17:47:39.415 | INFO | MemInfo: { | 20260507 17:47:39.415 | INFO | "gpu" : " 17 GB", | 20260507 17:47:39.415 | INFO | "huge" : "", | 20260507 17:47:39.415 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:39.415 | INFO | "malloc" : " 14 GB", | 20260507 17:47:39.415 | INFO | "pinned" : " 29 GB" | 20260507 17:47:39.415 | INFO | } | 20260507 17:47:39.415 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:47:39.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:39.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:47:39.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 17:47:39.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:39.415 | INFO | Uploaded 36GiB to GPU in 2.45563s (14.6602GiB/s) | 20260507 17:47:39.415 | INFO | Downloaded 34GiB to GPU in 2.51877s (13.4986GiB/s) | 20260507 17:47:44.555 | INFO | Analyzing chunk frames = [1408000,1408512) | 20260507 17:47:44.555 | INFO | MemInfo: { | 20260507 17:47:44.555 | INFO | "gpu" : " 17 GB", | 20260507 17:47:44.555 | INFO | "huge" : "", | 20260507 17:47:44.555 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:44.555 | INFO | "malloc" : " 14 GB", | 20260507 17:47:44.555 | INFO | "pinned" : " 29 GB" | 20260507 17:47:44.555 | INFO | } | 20260507 17:47:44.555 | INFO | Perf report for the last 5.13992 seconds: | 20260507 17:47:44.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:44.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:47:44.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 17:47:44.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:44.555 | INFO | Uploaded 36GiB to GPU in 2.48913s (14.4629GiB/s) | 20260507 17:47:44.555 | INFO | Downloaded 34GiB to GPU in 2.44442s (13.9093GiB/s) | 20260507 17:47:49.693 | INFO | Analyzing chunk frames = [1408512,1409024) | 20260507 17:47:49.693 | INFO | MemInfo: { | 20260507 17:47:49.693 | INFO | "gpu" : " 17 GB", | 20260507 17:47:49.693 | INFO | "huge" : "", | 20260507 17:47:49.693 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:49.693 | INFO | "malloc" : " 14 GB", | 20260507 17:47:49.694 | INFO | "pinned" : " 29 GB" | 20260507 17:47:49.694 | INFO | } | 20260507 17:47:49.694 | INFO | Perf report for the last 5.13858 seconds: | 20260507 17:47:49.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:49.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 17:47:49.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | PRE_HQ : 38.4%, 1.0, 20.0ms, 0 | 20260507 17:47:49.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:49.694 | INFO | Uploaded 36GiB to GPU in 2.48148s (14.5075GiB/s) | 20260507 17:47:49.694 | INFO | Downloaded 34GiB to GPU in 2.43154s (13.9829GiB/s) | 20260507 17:47:49.695 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:47:49.695 | INFO | Total mem usage: 16.4 GiB | 20260507 17:47:49.695 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:47:49.695 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 17:47:49.695 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:47:49.695 | INFO | High Water Breakdown: | 20260507 17:47:49.695 | INFO | 59.2% : 8.63 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:47:49.695 | INFO | 40.6% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:47:49.695 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:47:49.695 | INFO | ------------------------------------------ | 20260507 17:47:49.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:47:49.695 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:47:49.695 | INFO | Total mem usage: 26.5 GiB | 20260507 17:47:49.695 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:47:49.695 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:47:49.695 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:47:49.695 | INFO | High Water Breakdown: | 20260507 17:47:49.695 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:47:49.695 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:47:49.695 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:47:49.695 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:47:49.695 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:47:49.695 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:47:49.695 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:47:49.695 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:47:49.695 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:47:49.695 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:47:49.695 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:47:49.695 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:47:49.695 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:47:49.695 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:47:49.695 | INFO | ------------------------------------------ | 20260507 17:47:49.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:47:49.695 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:47:49.695 | INFO | Total mem usage: 27.8 GiB | 20260507 17:47:49.695 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:47:49.695 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:47:49.695 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:47:49.695 | INFO | High Water Breakdown: | 20260507 17:47:49.695 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:47:49.695 | INFO | ------------------------------------------ | 20260507 17:47:49.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:47:49.695 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:47:49.696 | INFO | Total mem usage: 15.7 GiB | 20260507 17:47:49.696 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:47:49.696 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:47:49.696 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:47:49.696 | INFO | High Water Breakdown: | 20260507 17:47:49.696 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:47:49.696 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:47:49.696 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:47:49.696 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:47:49.696 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:47:49.696 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:47:49.696 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:47:49.696 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:47:49.696 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:47:49.696 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:47:49.696 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:47:49.696 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:47:49.696 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:47:49.696 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:47:49.696 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:47:49.696 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:47:49.696 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:47:49.696 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:47:49.696 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:47:49.696 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:47:49.696 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:47:49.696 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:47:49.696 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:47:49.696 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:47:49.696 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:47:49.696 | INFO | ------------------------------------------ | 20260507 17:47:49.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:47:49.696 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:47:49.696 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:47:49.696 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:47:49.696 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.63e+01, 1.44e+01 / 2.71e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 0.31%, AggregateStats -- 1.03e-01 / 1.07e-01, 1.29e-02 / 6.47e-03 | 20260507 17:47:49.696 | INFO | PerfReport: 12.12%, Baseline -- 4.07e+00 / 4.50e+00, 6.19e-01 / 2.70e-01 | 20260507 17:47:49.696 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 1.05e-01 | 20260507 17:47:49.696 | INFO | PerfReport: 15.64%, DME -- 4.20e+01 / 4.47e+01, 4.11e+00 / 2.41e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.90e+00 / 2.23e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.83e+00, 4.36e-01 / 2.68e-01 | 20260507 17:47:49.696 | INFO | PerfReport: 3.75%, Metrics -- 1.26e+00 / 1.42e+00, 3.15e-01 / 1.79e-01 | 20260507 17:47:49.696 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.03e-01 / 4.26e-01, 6.54e-02 / 4.26e-02 | 20260507 17:47:49.696 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:47:49.696 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:47:49.696 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:47:49.696 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 23.73%, Upload -- 2.54e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 24.17%, Download -- 2.59e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 31.29%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 1.51%, LockUpload -- 1.62e+00 / 0.00e+00, 4.65e+00 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 2.57%, LockDownload -- 2.76e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 16.70%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:47:49.696 | INFO | PerfReport: 0.02%, Misc -- 1.77e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 17:47:49.807 | INFO | Starting a round of Baz IO | 20260507 17:47:53.302 | INFO | Populating RTMetrics report | 20260507 17:47:53.303 | INFO | RTMetrics report populated | 20260507 17:47:53.303 | INFO | Opening tmp RTMetrics json file | 20260507 17:47:53.315 | INFO | Serializing RTMetrics to json | 20260507 17:47:53.321 | INFO | Serialization done, swapping tmp file now... | 20260507 17:47:53.321 | INFO | RTMetrics done | 20260507 17:47:53.381 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.573 seconds | 20260507 17:47:53.381 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.56% / 0.02% / 52.42% | 20260507 17:47:53.381 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.35165% / 1.19209e-05% / 3.97364e-05% | 20260507 17:47:53.381 | INFO | Baz IO overall: | 20260507 17:47:53.381 | INFO | BytesWritten : 623947346468 | 20260507 17:47:53.381 | INFO | NumEvents : 302389471425 | 20260507 17:47:54.832 | INFO | Analyzing chunk frames = [1409024,1409536) | 20260507 17:47:54.832 | INFO | MemInfo: { | 20260507 17:47:54.832 | INFO | "gpu" : " 17 GB", | 20260507 17:47:54.832 | INFO | "huge" : "", | 20260507 17:47:54.832 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:54.832 | INFO | "malloc" : " 14 GB", | 20260507 17:47:54.832 | INFO | "pinned" : " 29 GB" | 20260507 17:47:54.832 | INFO | } | 20260507 17:47:54.832 | INFO | Perf report for the last 5.13878 seconds: | 20260507 17:47:54.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:54.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:54.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:54.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:54.833 | INFO | ANALYSIS : 69.8%, 3.0, 108.7ms, 0 | 20260507 17:47:54.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:54.833 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 17:47:54.833 | INFO | PRE_HQ : 67.0%, 1.8, 62.5ms, 0 | 20260507 17:47:54.833 | INFO | BAZWRITER : 68.5%, 2.9, 102.8ms, 0 | 20260507 17:47:54.833 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 17:47:54.833 | INFO | Uploaded 36GiB to GPU in 2.56369s (14.0423GiB/s) | 20260507 17:47:54.833 | INFO | Downloaded 38GiB to GPU in 2.72403s (13.9499GiB/s) | 20260507 17:47:59.972 | INFO | Analyzing chunk frames = [1409536,1410048) | 20260507 17:47:59.972 | INFO | MemInfo: { | 20260507 17:47:59.972 | INFO | "gpu" : " 17 GB", | 20260507 17:47:59.972 | INFO | "huge" : "", | 20260507 17:47:59.972 | INFO | "huge_shared" : " 29 GB", | 20260507 17:47:59.972 | INFO | "malloc" : " 14 GB", | 20260507 17:47:59.972 | INFO | "pinned" : " 29 GB" | 20260507 17:47:59.972 | INFO | } | 20260507 17:47:59.972 | INFO | Perf report for the last 5.13981 seconds: | 20260507 17:47:59.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:47:59.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.972 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:47:59.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.972 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 17:47:59.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:47:59.973 | INFO | Uploaded 36GiB to GPU in 2.5354s (14.1989GiB/s) | 20260507 17:47:59.973 | INFO | Downloaded 34GiB to GPU in 2.4243s (14.0247GiB/s) | 20260507 17:48:05.111 | INFO | Analyzing chunk frames = [1410048,1410560) | 20260507 17:48:05.111 | INFO | MemInfo: { | 20260507 17:48:05.111 | INFO | "gpu" : " 17 GB", | 20260507 17:48:05.111 | INFO | "huge" : "", | 20260507 17:48:05.111 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:05.111 | INFO | "malloc" : " 14 GB", | 20260507 17:48:05.111 | INFO | "pinned" : " 29 GB" | 20260507 17:48:05.111 | INFO | } | 20260507 17:48:05.111 | INFO | Perf report for the last 5.13907 seconds: | 20260507 17:48:05.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:05.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.111 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 17:48:05.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.111 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:48:05.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:05.112 | INFO | Uploaded 36GiB to GPU in 2.58133s (13.9463GiB/s) | 20260507 17:48:05.112 | INFO | Downloaded 34GiB to GPU in 2.40157s (14.1574GiB/s) | 20260507 17:48:10.250 | INFO | Analyzing chunk frames = [1410560,1411072) | 20260507 17:48:10.251 | INFO | MemInfo: { | 20260507 17:48:10.251 | INFO | "gpu" : " 17 GB", | 20260507 17:48:10.251 | INFO | "huge" : "", | 20260507 17:48:10.251 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:10.251 | INFO | "malloc" : " 14 GB", | 20260507 17:48:10.251 | INFO | "pinned" : " 29 GB" | 20260507 17:48:10.251 | INFO | } | 20260507 17:48:10.251 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:48:10.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:10.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:48:10.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:48:10.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:10.251 | INFO | Uploaded 36GiB to GPU in 2.49987s (14.4007GiB/s) | 20260507 17:48:10.251 | INFO | Downloaded 34GiB to GPU in 2.34198s (14.5176GiB/s) | 20260507 17:48:15.390 | INFO | Analyzing chunk frames = [1411072,1411584) | 20260507 17:48:15.390 | INFO | MemInfo: { | 20260507 17:48:15.390 | INFO | "gpu" : " 17 GB", | 20260507 17:48:15.390 | INFO | "huge" : "", | 20260507 17:48:15.390 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:15.390 | INFO | "malloc" : " 14 GB", | 20260507 17:48:15.390 | INFO | "pinned" : " 29 GB" | 20260507 17:48:15.390 | INFO | } | 20260507 17:48:15.390 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:48:15.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:15.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:48:15.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:48:15.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:15.390 | INFO | Uploaded 36GiB to GPU in 2.37444s (15.1615GiB/s) | 20260507 17:48:15.390 | INFO | Downloaded 34GiB to GPU in 2.45296s (13.8608GiB/s) | 20260507 17:48:20.529 | INFO | Analyzing chunk frames = [1411584,1412096) | 20260507 17:48:20.529 | INFO | MemInfo: { | 20260507 17:48:20.529 | INFO | "gpu" : " 17 GB", | 20260507 17:48:20.529 | INFO | "huge" : "", | 20260507 17:48:20.529 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:20.529 | INFO | "malloc" : " 14 GB", | 20260507 17:48:20.529 | INFO | "pinned" : " 29 GB" | 20260507 17:48:20.529 | INFO | } | 20260507 17:48:20.529 | INFO | Perf report for the last 5.13929 seconds: | 20260507 17:48:20.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:20.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.529 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:48:20.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.529 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:48:20.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:20.530 | INFO | Uploaded 36GiB to GPU in 2.38483s (15.0954GiB/s) | 20260507 17:48:20.530 | INFO | Downloaded 34GiB to GPU in 2.49148s (13.6465GiB/s) | 20260507 17:48:25.669 | INFO | Analyzing chunk frames = [1412096,1412608) | 20260507 17:48:25.669 | INFO | MemInfo: { | 20260507 17:48:25.669 | INFO | "gpu" : " 17 GB", | 20260507 17:48:25.669 | INFO | "huge" : "", | 20260507 17:48:25.669 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:25.669 | INFO | "malloc" : " 14 GB", | 20260507 17:48:25.669 | INFO | "pinned" : " 29 GB" | 20260507 17:48:25.669 | INFO | } | 20260507 17:48:25.669 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:48:25.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:25.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 17:48:25.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:48:25.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:25.669 | INFO | Uploaded 36GiB to GPU in 2.44831s (14.704GiB/s) | 20260507 17:48:25.669 | INFO | Downloaded 34GiB to GPU in 2.42921s (13.9963GiB/s) | 20260507 17:48:30.808 | INFO | Analyzing chunk frames = [1412608,1413120) | 20260507 17:48:30.808 | INFO | MemInfo: { | 20260507 17:48:30.808 | INFO | "gpu" : " 17 GB", | 20260507 17:48:30.808 | INFO | "huge" : "", | 20260507 17:48:30.808 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:30.808 | INFO | "malloc" : " 14 GB", | 20260507 17:48:30.808 | INFO | "pinned" : " 29 GB" | 20260507 17:48:30.808 | INFO | } | 20260507 17:48:30.808 | INFO | Perf report for the last 5.13936 seconds: | 20260507 17:48:30.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:30.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.808 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:48:30.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.809 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 17:48:30.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:30.809 | INFO | Uploaded 36GiB to GPU in 2.49657s (14.4198GiB/s) | 20260507 17:48:30.809 | INFO | Downloaded 34GiB to GPU in 2.42453s (14.0234GiB/s) | 20260507 17:48:30.922 | INFO | Starting a round of Baz IO | 20260507 17:48:34.426 | INFO | Populating RTMetrics report | 20260507 17:48:34.426 | INFO | RTMetrics report populated | 20260507 17:48:34.426 | INFO | Opening tmp RTMetrics json file | 20260507 17:48:34.427 | INFO | Serializing RTMetrics to json | 20260507 17:48:34.428 | INFO | Serialization done, swapping tmp file now... | 20260507 17:48:34.429 | INFO | RTMetrics done | 20260507 17:48:34.505 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.582 seconds | 20260507 17:48:34.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.54% / 0.02% / 52.44% | 20260507 17:48:34.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.34336% / 3.97364e-05% / 3.17891e-05% | 20260507 17:48:34.505 | INFO | Baz IO overall: | 20260507 17:48:34.505 | INFO | BytesWritten : 625880907795 | 20260507 17:48:34.505 | INFO | NumEvents : 303328712775 | 20260507 17:48:35.947 | INFO | Analyzing chunk frames = [1413120,1413632) | 20260507 17:48:35.947 | INFO | MemInfo: { | 20260507 17:48:35.947 | INFO | "gpu" : " 17 GB", | 20260507 17:48:35.947 | INFO | "huge" : "", | 20260507 17:48:35.947 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:35.947 | INFO | "malloc" : " 14 GB", | 20260507 17:48:35.947 | INFO | "pinned" : " 29 GB" | 20260507 17:48:35.947 | INFO | } | 20260507 17:48:35.947 | INFO | Perf report for the last 5.13907 seconds: | 20260507 17:48:35.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:35.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:35.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:35.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:35.947 | INFO | ANALYSIS : 69.4%, 2.9, 106.7ms, 0 | 20260507 17:48:35.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:35.947 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260507 17:48:35.947 | INFO | PRE_HQ : 67.9%, 2.0, 71.6ms, 0 | 20260507 17:48:35.947 | INFO | BAZWRITER : 69.2%, 3.2, 113.7ms, 0 | 20260507 17:48:35.947 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 17:48:35.948 | INFO | Uploaded 36GiB to GPU in 2.53223s (14.2167GiB/s) | 20260507 17:48:35.948 | INFO | Downloaded 38GiB to GPU in 2.69187s (14.1166GiB/s) | 20260507 17:48:41.087 | INFO | Analyzing chunk frames = [1413632,1414144) | 20260507 17:48:41.087 | INFO | MemInfo: { | 20260507 17:48:41.087 | INFO | "gpu" : " 17 GB", | 20260507 17:48:41.087 | INFO | "huge" : "", | 20260507 17:48:41.087 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:41.087 | INFO | "malloc" : " 14 GB", | 20260507 17:48:41.087 | INFO | "pinned" : " 29 GB" | 20260507 17:48:41.087 | INFO | } | 20260507 17:48:41.087 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:48:41.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:41.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.087 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260507 17:48:41.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.087 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:48:41.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:41.088 | INFO | Uploaded 36GiB to GPU in 2.51684s (14.3036GiB/s) | 20260507 17:48:41.088 | INFO | Downloaded 34GiB to GPU in 2.4507s (13.8736GiB/s) | 20260507 17:48:46.226 | INFO | Analyzing chunk frames = [1414144,1414656) | 20260507 17:48:46.226 | INFO | MemInfo: { | 20260507 17:48:46.226 | INFO | "gpu" : " 17 GB", | 20260507 17:48:46.226 | INFO | "huge" : "", | 20260507 17:48:46.226 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:46.226 | INFO | "malloc" : " 14 GB", | 20260507 17:48:46.226 | INFO | "pinned" : " 29 GB" | 20260507 17:48:46.226 | INFO | } | 20260507 17:48:46.226 | INFO | Perf report for the last 5.13924 seconds: | 20260507 17:48:46.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:46.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.226 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 17:48:46.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.226 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:48:46.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:46.227 | INFO | Uploaded 36GiB to GPU in 2.49073s (14.4536GiB/s) | 20260507 17:48:46.227 | INFO | Downloaded 34GiB to GPU in 2.35033s (14.466GiB/s) | 20260507 17:48:51.365 | INFO | Analyzing chunk frames = [1414656,1415168) | 20260507 17:48:51.365 | INFO | MemInfo: { | 20260507 17:48:51.365 | INFO | "gpu" : " 17 GB", | 20260507 17:48:51.365 | INFO | "huge" : "", | 20260507 17:48:51.365 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:51.365 | INFO | "malloc" : " 14 GB", | 20260507 17:48:51.365 | INFO | "pinned" : " 29 GB" | 20260507 17:48:51.365 | INFO | } | 20260507 17:48:51.365 | INFO | Perf report for the last 5.13924 seconds: | 20260507 17:48:51.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:51.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.365 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:48:51.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.365 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:48:51.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:51.366 | INFO | Uploaded 36GiB to GPU in 2.4867s (14.477GiB/s) | 20260507 17:48:51.366 | INFO | Downloaded 34GiB to GPU in 2.35065s (14.4641GiB/s) | 20260507 17:48:56.505 | INFO | Analyzing chunk frames = [1415168,1415680) | 20260507 17:48:56.505 | INFO | MemInfo: { | 20260507 17:48:56.505 | INFO | "gpu" : " 17 GB", | 20260507 17:48:56.505 | INFO | "huge" : "", | 20260507 17:48:56.505 | INFO | "huge_shared" : " 29 GB", | 20260507 17:48:56.505 | INFO | "malloc" : " 14 GB", | 20260507 17:48:56.505 | INFO | "pinned" : " 29 GB" | 20260507 17:48:56.505 | INFO | } | 20260507 17:48:56.505 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:48:56.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:48:56.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.505 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:48:56.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.505 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:48:56.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:48:56.506 | INFO | Uploaded 36GiB to GPU in 2.34379s (15.3598GiB/s) | 20260507 17:48:56.506 | INFO | Downloaded 34GiB to GPU in 2.46392s (13.7992GiB/s) | 20260507 17:49:01.644 | INFO | Analyzing chunk frames = [1415680,1416192) | 20260507 17:49:01.644 | INFO | MemInfo: { | 20260507 17:49:01.644 | INFO | "gpu" : " 17 GB", | 20260507 17:49:01.644 | INFO | "huge" : "", | 20260507 17:49:01.644 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:01.644 | INFO | "malloc" : " 14 GB", | 20260507 17:49:01.644 | INFO | "pinned" : " 29 GB" | 20260507 17:49:01.644 | INFO | } | 20260507 17:49:01.644 | INFO | Perf report for the last 5.13892 seconds: | 20260507 17:49:01.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:01.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:49:01.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:49:01.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:01.644 | INFO | Uploaded 36GiB to GPU in 2.42945s (14.8182GiB/s) | 20260507 17:49:01.645 | INFO | Downloaded 34GiB to GPU in 2.48172s (13.7001GiB/s) | 20260507 17:49:06.783 | INFO | Analyzing chunk frames = [1416192,1416704) | 20260507 17:49:06.783 | INFO | MemInfo: { | 20260507 17:49:06.783 | INFO | "gpu" : " 17 GB", | 20260507 17:49:06.783 | INFO | "huge" : "", | 20260507 17:49:06.783 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:06.783 | INFO | "malloc" : " 14 GB", | 20260507 17:49:06.783 | INFO | "pinned" : " 29 GB" | 20260507 17:49:06.783 | INFO | } | 20260507 17:49:06.783 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:49:06.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:06.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.783 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:49:06.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.783 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:49:06.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:06.784 | INFO | Uploaded 36GiB to GPU in 2.49566s (14.425GiB/s) | 20260507 17:49:06.784 | INFO | Downloaded 34GiB to GPU in 2.41541s (14.0763GiB/s) | 20260507 17:49:11.923 | INFO | Analyzing chunk frames = [1416704,1417216) | 20260507 17:49:11.923 | INFO | MemInfo: { | 20260507 17:49:11.923 | INFO | "gpu" : " 17 GB", | 20260507 17:49:11.923 | INFO | "huge" : "", | 20260507 17:49:11.923 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:11.923 | INFO | "malloc" : " 14 GB", | 20260507 17:49:11.923 | INFO | "pinned" : " 29 GB" | 20260507 17:49:11.923 | INFO | } | 20260507 17:49:11.923 | INFO | Perf report for the last 5.13966 seconds: | 20260507 17:49:11.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:11.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 17:49:11.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260507 17:49:11.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:11.923 | INFO | Uploaded 36GiB to GPU in 2.55657s (14.0814GiB/s) | 20260507 17:49:11.924 | INFO | Downloaded 34GiB to GPU in 2.4921s (13.6431GiB/s) | 20260507 17:49:11.924 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:49:11.924 | INFO | Total mem usage: 16.4 GiB | 20260507 17:49:11.924 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:49:11.924 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13%) | 20260507 17:49:11.924 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:49:11.924 | INFO | High Water Breakdown: | 20260507 17:49:11.924 | INFO | 59.2% : 8.59 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:49:11.924 | INFO | 40.6% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:49:11.924 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:49:11.924 | INFO | ------------------------------------------ | 20260507 17:49:11.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:49:11.924 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:49:11.924 | INFO | Total mem usage: 26.5 GiB | 20260507 17:49:11.924 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:49:11.924 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:49:11.924 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:49:11.924 | INFO | High Water Breakdown: | 20260507 17:49:11.924 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:49:11.924 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:49:11.924 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:49:11.924 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:49:11.924 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:49:11.924 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:49:11.924 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:49:11.924 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:49:11.924 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:49:11.924 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:49:11.924 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:49:11.924 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:49:11.924 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:49:11.924 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:49:11.924 | INFO | ------------------------------------------ | 20260507 17:49:11.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:49:11.924 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:49:11.924 | INFO | Total mem usage: 27.8 GiB | 20260507 17:49:11.924 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:49:11.924 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:49:11.924 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:49:11.924 | INFO | High Water Breakdown: | 20260507 17:49:11.924 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:49:11.924 | INFO | ------------------------------------------ | 20260507 17:49:11.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:49:11.924 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:49:11.924 | INFO | Total mem usage: 15.7 GiB | 20260507 17:49:11.924 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:49:11.924 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:49:11.924 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:49:11.924 | INFO | High Water Breakdown: | 20260507 17:49:11.924 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:49:11.924 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:49:11.924 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:49:11.924 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:49:11.924 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:49:11.925 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:49:11.925 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:49:11.925 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:49:11.925 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:49:11.925 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:49:11.925 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:49:11.925 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:49:11.925 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:49:11.925 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:49:11.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:49:11.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:49:11.925 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:49:11.925 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:49:11.925 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:49:11.925 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:49:11.925 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:49:11.925 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:49:11.925 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:49:11.925 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:49:11.925 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:49:11.925 | INFO | ------------------------------------------ | 20260507 17:49:11.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:49:11.925 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:49:11.925 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:49:11.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:49:11.925 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.45e+01 / 2.50e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 0.29%, AggregateStats -- 9.84e-02 / 1.03e-01, 1.13e-02 / 7.02e-03 | 20260507 17:49:11.925 | INFO | PerfReport: 12.05%, Baseline -- 4.04e+00 / 4.50e+00, 6.16e-01 / 2.43e-01 | 20260507 17:49:11.925 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.06e-01 | 20260507 17:49:11.925 | INFO | PerfReport: 15.71%, DME -- 4.21e+01 / 4.50e+01, 4.16e+00 / 2.27e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.90e+00 / 1.99e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.37e-01 / 2.61e-01 | 20260507 17:49:11.925 | INFO | PerfReport: 3.76%, Metrics -- 1.26e+00 / 1.43e+00, 4.26e-01 / 1.59e-01 | 20260507 17:49:11.925 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.97e-01 / 4.23e-01, 6.60e-02 / 3.84e-02 | 20260507 17:49:11.925 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:49:11.925 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:49:11.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:49:11.925 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 24.09%, Upload -- 2.59e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 23.88%, Download -- 2.56e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 31.20%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 1.95%, LockUpload -- 2.09e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 2.21%, LockDownload -- 2.37e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 16.67%, LockCompute -- 1.79e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:49:11.925 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260507 17:49:12.033 | INFO | Starting a round of Baz IO | 20260507 17:49:15.533 | INFO | Populating RTMetrics report | 20260507 17:49:15.533 | INFO | RTMetrics report populated | 20260507 17:49:15.533 | INFO | Opening tmp RTMetrics json file | 20260507 17:49:15.549 | INFO | Serializing RTMetrics to json | 20260507 17:49:15.551 | INFO | Serialization done, swapping tmp file now... | 20260507 17:49:15.551 | INFO | RTMetrics done | 20260507 17:49:15.613 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.58 seconds | 20260507 17:49:15.614 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.53% / 0.02% / 52.46% | 20260507 17:49:15.614 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.33623% / 2.78155e-05% / 3.97364e-05% | 20260507 17:49:15.614 | INFO | Baz IO overall: | 20260507 17:49:15.614 | INFO | BytesWritten : 627815710743 | 20260507 17:49:15.614 | INFO | NumEvents : 304268746073 | 20260507 17:49:17.062 | INFO | Analyzing chunk frames = [1417216,1417728) | 20260507 17:49:17.062 | INFO | MemInfo: { | 20260507 17:49:17.062 | INFO | "gpu" : " 17 GB", | 20260507 17:49:17.062 | INFO | "huge" : "", | 20260507 17:49:17.062 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:17.062 | INFO | "malloc" : " 14 GB", | 20260507 17:49:17.062 | INFO | "pinned" : " 29 GB" | 20260507 17:49:17.062 | INFO | } | 20260507 17:49:17.062 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:49:17.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:17.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:17.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:17.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:17.062 | INFO | ANALYSIS : 69.8%, 3.0, 108.4ms, 0 | 20260507 17:49:17.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:17.062 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 17:49:17.062 | INFO | PRE_HQ : 67.3%, 1.8, 63.9ms, 0 | 20260507 17:49:17.062 | INFO | BAZWRITER : 68.7%, 3.0, 105.8ms, 0 | 20260507 17:49:17.062 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:49:17.063 | INFO | Uploaded 36GiB to GPU in 2.56263s (14.0481GiB/s) | 20260507 17:49:17.063 | INFO | Downloaded 38GiB to GPU in 2.71599s (13.9912GiB/s) | 20260507 17:49:22.201 | INFO | Analyzing chunk frames = [1417728,1418240) | 20260507 17:49:22.201 | INFO | MemInfo: { | 20260507 17:49:22.201 | INFO | "gpu" : " 17 GB", | 20260507 17:49:22.201 | INFO | "huge" : "", | 20260507 17:49:22.201 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:22.201 | INFO | "malloc" : " 14 GB", | 20260507 17:49:22.202 | INFO | "pinned" : " 29 GB" | 20260507 17:49:22.202 | INFO | } | 20260507 17:49:22.202 | INFO | Perf report for the last 5.13947 seconds: | 20260507 17:49:22.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:22.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:49:22.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:49:22.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:22.202 | INFO | Uploaded 36GiB to GPU in 2.53849s (14.1817GiB/s) | 20260507 17:49:22.202 | INFO | Downloaded 34GiB to GPU in 2.42388s (14.0271GiB/s) | 20260507 17:49:27.341 | INFO | Analyzing chunk frames = [1418240,1418752) | 20260507 17:49:27.341 | INFO | MemInfo: { | 20260507 17:49:27.341 | INFO | "gpu" : " 17 GB", | 20260507 17:49:27.341 | INFO | "huge" : "", | 20260507 17:49:27.341 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:27.341 | INFO | "malloc" : " 14 GB", | 20260507 17:49:27.341 | INFO | "pinned" : " 29 GB" | 20260507 17:49:27.341 | INFO | } | 20260507 17:49:27.341 | INFO | Perf report for the last 5.13964 seconds: | 20260507 17:49:27.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:27.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.341 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:49:27.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.341 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:49:27.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:27.342 | INFO | Uploaded 36GiB to GPU in 2.57056s (14.0047GiB/s) | 20260507 17:49:27.342 | INFO | Downloaded 34GiB to GPU in 2.4132s (14.0892GiB/s) | 20260507 17:49:32.480 | INFO | Analyzing chunk frames = [1418752,1419264) | 20260507 17:49:32.480 | INFO | MemInfo: { | 20260507 17:49:32.480 | INFO | "gpu" : " 17 GB", | 20260507 17:49:32.480 | INFO | "huge" : "", | 20260507 17:49:32.481 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:32.481 | INFO | "malloc" : " 14 GB", | 20260507 17:49:32.481 | INFO | "pinned" : " 29 GB" | 20260507 17:49:32.481 | INFO | } | 20260507 17:49:32.481 | INFO | Perf report for the last 5.13937 seconds: | 20260507 17:49:32.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:32.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:49:32.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:49:32.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:32.481 | INFO | Uploaded 36GiB to GPU in 2.50595s (14.3658GiB/s) | 20260507 17:49:32.481 | INFO | Downloaded 34GiB to GPU in 2.34893s (14.4747GiB/s) | 20260507 17:49:37.620 | INFO | Analyzing chunk frames = [1419264,1419776) | 20260507 17:49:37.620 | INFO | MemInfo: { | 20260507 17:49:37.620 | INFO | "gpu" : " 17 GB", | 20260507 17:49:37.620 | INFO | "huge" : "", | 20260507 17:49:37.620 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:37.620 | INFO | "malloc" : " 14 GB", | 20260507 17:49:37.620 | INFO | "pinned" : " 29 GB" | 20260507 17:49:37.620 | INFO | } | 20260507 17:49:37.620 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:49:37.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:37.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:49:37.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 17:49:37.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:37.620 | INFO | Uploaded 36GiB to GPU in 2.36473s (15.2237GiB/s) | 20260507 17:49:37.620 | INFO | Downloaded 34GiB to GPU in 2.46938s (13.7687GiB/s) | 20260507 17:49:42.759 | INFO | Analyzing chunk frames = [1419776,1420288) | 20260507 17:49:42.759 | INFO | MemInfo: { | 20260507 17:49:42.759 | INFO | "gpu" : " 17 GB", | 20260507 17:49:42.759 | INFO | "huge" : "", | 20260507 17:49:42.759 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:42.759 | INFO | "malloc" : " 14 GB", | 20260507 17:49:42.759 | INFO | "pinned" : " 29 GB" | 20260507 17:49:42.759 | INFO | } | 20260507 17:49:42.759 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:49:42.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:42.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:49:42.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:49:42.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:42.759 | INFO | Uploaded 36GiB to GPU in 2.4226s (14.86GiB/s) | 20260507 17:49:42.759 | INFO | Downloaded 34GiB to GPU in 2.4777s (13.7224GiB/s) | 20260507 17:49:47.898 | INFO | Analyzing chunk frames = [1420288,1420800) | 20260507 17:49:47.898 | INFO | MemInfo: { | 20260507 17:49:47.898 | INFO | "gpu" : " 17 GB", | 20260507 17:49:47.898 | INFO | "huge" : "", | 20260507 17:49:47.898 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:47.898 | INFO | "malloc" : " 14 GB", | 20260507 17:49:47.898 | INFO | "pinned" : " 29 GB" | 20260507 17:49:47.898 | INFO | } | 20260507 17:49:47.899 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:49:47.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:47.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:49:47.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 17:49:47.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:47.899 | INFO | Uploaded 36GiB to GPU in 2.46998s (14.575GiB/s) | 20260507 17:49:47.899 | INFO | Downloaded 34GiB to GPU in 2.4296s (13.9941GiB/s) | 20260507 17:49:53.037 | INFO | Analyzing chunk frames = [1420800,1421312) | 20260507 17:49:53.037 | INFO | MemInfo: { | 20260507 17:49:53.037 | INFO | "gpu" : " 17 GB", | 20260507 17:49:53.037 | INFO | "huge" : "", | 20260507 17:49:53.038 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:53.038 | INFO | "malloc" : " 14 GB", | 20260507 17:49:53.038 | INFO | "pinned" : " 29 GB" | 20260507 17:49:53.038 | INFO | } | 20260507 17:49:53.038 | INFO | Perf report for the last 5.13925 seconds: | 20260507 17:49:53.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:53.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.038 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:49:53.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.038 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:49:53.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:53.039 | INFO | Uploaded 36GiB to GPU in 2.47879s (14.5232GiB/s) | 20260507 17:49:53.039 | INFO | Downloaded 34GiB to GPU in 2.4207s (14.0455GiB/s) | 20260507 17:49:53.149 | INFO | Starting a round of Baz IO | 20260507 17:49:56.642 | INFO | Populating RTMetrics report | 20260507 17:49:56.642 | INFO | RTMetrics report populated | 20260507 17:49:56.642 | INFO | Opening tmp RTMetrics json file | 20260507 17:49:56.659 | INFO | Serializing RTMetrics to json | 20260507 17:49:56.661 | INFO | Serialization done, swapping tmp file now... | 20260507 17:49:56.661 | INFO | RTMetrics done | 20260507 17:49:56.715 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.565 seconds | 20260507 17:49:56.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.51% / 0.02% / 52.47% | 20260507 17:49:56.715 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.33534% / 3.17891e-05% / 2.38419e-05% | 20260507 17:49:56.715 | INFO | Baz IO overall: | 20260507 17:49:56.715 | INFO | BytesWritten : 629752355037 | 20260507 17:49:56.715 | INFO | NumEvents : 305210339864 | 20260507 17:49:58.177 | INFO | Analyzing chunk frames = [1421312,1421824) | 20260507 17:49:58.177 | INFO | MemInfo: { | 20260507 17:49:58.177 | INFO | "gpu" : " 17 GB", | 20260507 17:49:58.177 | INFO | "huge" : "", | 20260507 17:49:58.177 | INFO | "huge_shared" : " 29 GB", | 20260507 17:49:58.177 | INFO | "malloc" : " 14 GB", | 20260507 17:49:58.177 | INFO | "pinned" : " 29 GB" | 20260507 17:49:58.177 | INFO | } | 20260507 17:49:58.177 | INFO | Perf report for the last 5.13894 seconds: | 20260507 17:49:58.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:49:58.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:58.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:58.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:58.177 | INFO | ANALYSIS : 69.2%, 3.0, 107.1ms, 0 | 20260507 17:49:58.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:49:58.177 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 17:49:58.177 | INFO | PRE_HQ : 67.1%, 1.9, 65.1ms, 0 | 20260507 17:49:58.177 | INFO | BAZWRITER : 68.7%, 3.0, 107.2ms, 0 | 20260507 17:49:58.177 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 17:49:58.177 | INFO | Uploaded 36GiB to GPU in 2.49952s (14.4028GiB/s) | 20260507 17:49:58.177 | INFO | Downloaded 38GiB to GPU in 2.69169s (14.1175GiB/s) | 20260507 17:50:03.316 | INFO | Analyzing chunk frames = [1421824,1422336) | 20260507 17:50:03.316 | INFO | MemInfo: { | 20260507 17:50:03.316 | INFO | "gpu" : " 17 GB", | 20260507 17:50:03.316 | INFO | "huge" : "", | 20260507 17:50:03.316 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:03.316 | INFO | "malloc" : " 14 GB", | 20260507 17:50:03.316 | INFO | "pinned" : " 29 GB" | 20260507 17:50:03.316 | INFO | } | 20260507 17:50:03.316 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:50:03.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:03.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.316 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:50:03.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.316 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:50:03.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:03.317 | INFO | Uploaded 36GiB to GPU in 2.51695s (14.303GiB/s) | 20260507 17:50:03.317 | INFO | Downloaded 34GiB to GPU in 2.44139s (13.9265GiB/s) | 20260507 17:50:08.456 | INFO | Analyzing chunk frames = [1422336,1422848) | 20260507 17:50:08.456 | INFO | MemInfo: { | 20260507 17:50:08.456 | INFO | "gpu" : " 17 GB", | 20260507 17:50:08.456 | INFO | "huge" : "", | 20260507 17:50:08.456 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:08.456 | INFO | "malloc" : " 14 GB", | 20260507 17:50:08.456 | INFO | "pinned" : " 29 GB" | 20260507 17:50:08.456 | INFO | } | 20260507 17:50:08.456 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:50:08.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:08.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:50:08.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:50:08.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:08.456 | INFO | Uploaded 36GiB to GPU in 2.55719s (14.078GiB/s) | 20260507 17:50:08.456 | INFO | Downloaded 34GiB to GPU in 2.42417s (14.0254GiB/s) | 20260507 17:50:13.594 | INFO | Analyzing chunk frames = [1422848,1423360) | 20260507 17:50:13.594 | INFO | MemInfo: { | 20260507 17:50:13.594 | INFO | "gpu" : " 17 GB", | 20260507 17:50:13.594 | INFO | "huge" : "", | 20260507 17:50:13.595 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:13.595 | INFO | "malloc" : " 14 GB", | 20260507 17:50:13.595 | INFO | "pinned" : " 29 GB" | 20260507 17:50:13.595 | INFO | } | 20260507 17:50:13.595 | INFO | Perf report for the last 5.13902 seconds: | 20260507 17:50:13.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:13.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:50:13.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 17:50:13.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:13.595 | INFO | Uploaded 36GiB to GPU in 2.49406s (14.4343GiB/s) | 20260507 17:50:13.595 | INFO | Downloaded 34GiB to GPU in 2.35097s (14.4621GiB/s) | 20260507 17:50:18.734 | INFO | Analyzing chunk frames = [1423360,1423872) | 20260507 17:50:18.734 | INFO | MemInfo: { | 20260507 17:50:18.734 | INFO | "gpu" : " 17 GB", | 20260507 17:50:18.734 | INFO | "huge" : "", | 20260507 17:50:18.734 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:18.734 | INFO | "malloc" : " 14 GB", | 20260507 17:50:18.734 | INFO | "pinned" : " 29 GB" | 20260507 17:50:18.734 | INFO | } | 20260507 17:50:18.734 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:50:18.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:18.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.735 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 17:50:18.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.735 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 17:50:18.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:18.735 | INFO | Uploaded 36GiB to GPU in 2.33603s (15.4108GiB/s) | 20260507 17:50:18.735 | INFO | Downloaded 34GiB to GPU in 2.46831s (13.7746GiB/s) | 20260507 17:50:23.873 | INFO | Analyzing chunk frames = [1423872,1424384) | 20260507 17:50:23.873 | INFO | MemInfo: { | 20260507 17:50:23.873 | INFO | "gpu" : " 17 GB", | 20260507 17:50:23.873 | INFO | "huge" : "", | 20260507 17:50:23.873 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:23.873 | INFO | "malloc" : " 14 GB", | 20260507 17:50:23.873 | INFO | "pinned" : " 29 GB" | 20260507 17:50:23.874 | INFO | } | 20260507 17:50:23.874 | INFO | Perf report for the last 5.1392 seconds: | 20260507 17:50:23.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:23.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:50:23.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:50:23.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:23.874 | INFO | Uploaded 36GiB to GPU in 2.38505s (15.094GiB/s) | 20260507 17:50:23.874 | INFO | Downloaded 34GiB to GPU in 2.44182s (13.924GiB/s) | 20260507 17:50:29.013 | INFO | Analyzing chunk frames = [1424384,1424896) | 20260507 17:50:29.013 | INFO | MemInfo: { | 20260507 17:50:29.013 | INFO | "gpu" : " 17 GB", | 20260507 17:50:29.013 | INFO | "huge" : "", | 20260507 17:50:29.013 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:29.013 | INFO | "malloc" : " 14 GB", | 20260507 17:50:29.013 | INFO | "pinned" : " 29 GB" | 20260507 17:50:29.013 | INFO | } | 20260507 17:50:29.013 | INFO | Perf report for the last 5.13958 seconds: | 20260507 17:50:29.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:29.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:50:29.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 17:50:29.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:29.014 | INFO | Uploaded 36GiB to GPU in 2.50469s (14.373GiB/s) | 20260507 17:50:29.014 | INFO | Downloaded 34GiB to GPU in 2.42637s (14.0127GiB/s) | 20260507 17:50:34.152 | INFO | Analyzing chunk frames = [1424896,1425408) | 20260507 17:50:34.152 | INFO | MemInfo: { | 20260507 17:50:34.152 | INFO | "gpu" : " 17 GB", | 20260507 17:50:34.152 | INFO | "huge" : "", | 20260507 17:50:34.152 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:34.152 | INFO | "malloc" : " 14 GB", | 20260507 17:50:34.152 | INFO | "pinned" : " 29 GB" | 20260507 17:50:34.152 | INFO | } | 20260507 17:50:34.153 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:50:34.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:34.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 17:50:34.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 17:50:34.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:34.153 | INFO | Uploaded 36GiB to GPU in 2.51343s (14.323GiB/s) | 20260507 17:50:34.153 | INFO | Downloaded 34GiB to GPU in 2.41088s (14.1027GiB/s) | 20260507 17:50:34.153 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:50:34.153 | INFO | Total mem usage: 16.4 GiB | 20260507 17:50:34.153 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:50:34.153 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:50:34.153 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:50:34.153 | INFO | High Water Breakdown: | 20260507 17:50:34.154 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:50:34.154 | INFO | 40.7% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:50:34.154 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:50:34.154 | INFO | ------------------------------------------ | 20260507 17:50:34.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:50:34.154 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:50:34.154 | INFO | Total mem usage: 26.5 GiB | 20260507 17:50:34.154 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:50:34.154 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:50:34.154 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:50:34.154 | INFO | High Water Breakdown: | 20260507 17:50:34.154 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:50:34.154 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:50:34.154 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:50:34.154 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:50:34.154 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:50:34.154 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:50:34.154 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:50:34.154 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:50:34.154 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:50:34.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:50:34.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:50:34.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:50:34.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:50:34.154 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:50:34.154 | INFO | ------------------------------------------ | 20260507 17:50:34.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:50:34.154 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:50:34.154 | INFO | Total mem usage: 27.8 GiB | 20260507 17:50:34.154 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:50:34.154 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:50:34.154 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:50:34.154 | INFO | High Water Breakdown: | 20260507 17:50:34.154 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:50:34.154 | INFO | ------------------------------------------ | 20260507 17:50:34.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:50:34.154 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:50:34.154 | INFO | Total mem usage: 15.7 GiB | 20260507 17:50:34.154 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:50:34.154 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:50:34.154 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 17:50:34.154 | INFO | High Water Breakdown: | 20260507 17:50:34.154 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:50:34.154 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:50:34.154 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:50:34.154 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:50:34.154 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:50:34.154 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:50:34.154 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:50:34.154 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:50:34.154 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:50:34.154 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:50:34.154 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:50:34.154 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:50:34.154 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:50:34.154 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:50:34.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:50:34.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:50:34.154 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:50:34.154 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:50:34.154 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:50:34.154 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:50:34.155 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:50:34.155 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:50:34.155 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:50:34.155 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:50:34.155 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:50:34.155 | INFO | ------------------------------------------ | 20260507 17:50:34.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:50:34.155 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:50:34.155 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:50:34.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:50:34.155 | INFO | PerfReport: Full Execution -- 3.36e+01 / 7.64e+01, 1.44e+01 / 2.58e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 0.30%, AggregateStats -- 9.93e-02 / 1.04e-01, 1.10e-02 / 6.50e-03 | 20260507 17:50:34.155 | INFO | PerfReport: 12.09%, Baseline -- 4.06e+00 / 4.50e+00, 6.16e-01 / 2.40e-01 | 20260507 17:50:34.155 | INFO | PerfReport: 3.45%, Histogram -- 1.16e+00 / 1.50e+00, 2.15e-01 / 1.20e-01 | 20260507 17:50:34.155 | INFO | PerfReport: 15.64%, DME -- 4.20e+01 / 4.47e+01, 4.07e+00 / 2.45e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.92e+00 / 2.08e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.28e-01 / 2.41e-01 | 20260507 17:50:34.155 | INFO | PerfReport: 3.76%, Metrics -- 1.26e+00 / 1.44e+00, 3.23e-01 / 1.88e-01 | 20260507 17:50:34.155 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.97e-01 / 4.22e-01, 6.56e-02 / 3.55e-02 | 20260507 17:50:34.155 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:50:34.155 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:50:34.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:50:34.155 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 24.07%, Upload -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 23.85%, Download -- 2.56e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 31.24%, Compute -- 3.36e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 1.97%, LockUpload -- 2.11e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 2.19%, LockDownload -- 2.35e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 16.67%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:50:34.155 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260507 17:50:34.266 | INFO | Starting a round of Baz IO | 20260507 17:50:37.749 | INFO | Populating RTMetrics report | 20260507 17:50:37.750 | INFO | RTMetrics report populated | 20260507 17:50:37.750 | INFO | Opening tmp RTMetrics json file | 20260507 17:50:37.750 | INFO | Serializing RTMetrics to json | 20260507 17:50:37.752 | INFO | Serialization done, swapping tmp file now... | 20260507 17:50:37.752 | INFO | RTMetrics done | 20260507 17:50:37.828 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.561 seconds | 20260507 17:50:37.828 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.49% / 0.02% / 52.49% | 20260507 17:50:37.828 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.32059% / 2.38419e-05% / 4.37101e-05% | 20260507 17:50:37.828 | INFO | Baz IO overall: | 20260507 17:50:37.828 | INFO | BytesWritten : 631687465587 | 20260507 17:50:37.828 | INFO | NumEvents : 306150688995 | 20260507 17:50:39.292 | INFO | Analyzing chunk frames = [1425408,1425920) | 20260507 17:50:39.292 | INFO | MemInfo: { | 20260507 17:50:39.292 | INFO | "gpu" : " 17 GB", | 20260507 17:50:39.292 | INFO | "huge" : "", | 20260507 17:50:39.292 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:39.292 | INFO | "malloc" : " 14 GB", | 20260507 17:50:39.292 | INFO | "pinned" : " 29 GB" | 20260507 17:50:39.292 | INFO | } | 20260507 17:50:39.292 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:50:39.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:39.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:39.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:39.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:39.292 | INFO | ANALYSIS : 69.5%, 2.9, 107.2ms, 0 | 20260507 17:50:39.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:39.292 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 17:50:39.292 | INFO | PRE_HQ : 67.1%, 1.9, 64.4ms, 0 | 20260507 17:50:39.292 | INFO | BAZWRITER : 68.4%, 3.0, 107.0ms, 0 | 20260507 17:50:39.292 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:50:39.292 | INFO | Uploaded 36GiB to GPU in 2.54422s (14.1497GiB/s) | 20260507 17:50:39.292 | INFO | Downloaded 38GiB to GPU in 2.69003s (14.1262GiB/s) | 20260507 17:50:44.431 | INFO | Analyzing chunk frames = [1425920,1426432) | 20260507 17:50:44.431 | INFO | MemInfo: { | 20260507 17:50:44.431 | INFO | "gpu" : " 17 GB", | 20260507 17:50:44.431 | INFO | "huge" : "", | 20260507 17:50:44.431 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:44.431 | INFO | "malloc" : " 14 GB", | 20260507 17:50:44.431 | INFO | "pinned" : " 29 GB" | 20260507 17:50:44.431 | INFO | } | 20260507 17:50:44.431 | INFO | Perf report for the last 5.13935 seconds: | 20260507 17:50:44.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:44.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 17:50:44.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 17:50:44.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:44.431 | INFO | Uploaded 36GiB to GPU in 2.51147s (14.3342GiB/s) | 20260507 17:50:44.432 | INFO | Downloaded 34GiB to GPU in 2.44208s (13.9226GiB/s) | 20260507 17:50:49.570 | INFO | Analyzing chunk frames = [1426432,1426944) | 20260507 17:50:49.570 | INFO | MemInfo: { | 20260507 17:50:49.570 | INFO | "gpu" : " 17 GB", | 20260507 17:50:49.570 | INFO | "huge" : "", | 20260507 17:50:49.570 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:49.570 | INFO | "malloc" : " 14 GB", | 20260507 17:50:49.570 | INFO | "pinned" : " 29 GB" | 20260507 17:50:49.570 | INFO | } | 20260507 17:50:49.570 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:50:49.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:49.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.570 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 17:50:49.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.571 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 17:50:49.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:49.571 | INFO | Uploaded 36GiB to GPU in 2.54116s (14.1668GiB/s) | 20260507 17:50:49.571 | INFO | Downloaded 34GiB to GPU in 2.44621s (13.899GiB/s) | 20260507 17:50:54.709 | INFO | Analyzing chunk frames = [1426944,1427456) | 20260507 17:50:54.709 | INFO | MemInfo: { | 20260507 17:50:54.709 | INFO | "gpu" : " 17 GB", | 20260507 17:50:54.709 | INFO | "huge" : "", | 20260507 17:50:54.709 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:54.709 | INFO | "malloc" : " 14 GB", | 20260507 17:50:54.709 | INFO | "pinned" : " 29 GB" | 20260507 17:50:54.709 | INFO | } | 20260507 17:50:54.710 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:50:54.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:54.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 17:50:54.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:50:54.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:54.710 | INFO | Uploaded 36GiB to GPU in 2.48668s (14.4771GiB/s) | 20260507 17:50:54.710 | INFO | Downloaded 34GiB to GPU in 2.3473s (14.4847GiB/s) | 20260507 17:50:59.849 | INFO | Analyzing chunk frames = [1427456,1427968) | 20260507 17:50:59.849 | INFO | MemInfo: { | 20260507 17:50:59.849 | INFO | "gpu" : " 17 GB", | 20260507 17:50:59.849 | INFO | "huge" : "", | 20260507 17:50:59.849 | INFO | "huge_shared" : " 29 GB", | 20260507 17:50:59.849 | INFO | "malloc" : " 14 GB", | 20260507 17:50:59.849 | INFO | "pinned" : " 29 GB" | 20260507 17:50:59.849 | INFO | } | 20260507 17:50:59.849 | INFO | Perf report for the last 5.13973 seconds: | 20260507 17:50:59.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:50:59.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.849 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:50:59.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.849 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:50:59.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:50:59.850 | INFO | Uploaded 36GiB to GPU in 2.29185s (15.7078GiB/s) | 20260507 17:50:59.850 | INFO | Downloaded 34GiB to GPU in 2.47244s (13.7516GiB/s) | 20260507 17:51:04.988 | INFO | Analyzing chunk frames = [1427968,1428480) | 20260507 17:51:04.988 | INFO | MemInfo: { | 20260507 17:51:04.988 | INFO | "gpu" : " 17 GB", | 20260507 17:51:04.988 | INFO | "huge" : "", | 20260507 17:51:04.988 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:04.988 | INFO | "malloc" : " 14 GB", | 20260507 17:51:04.988 | INFO | "pinned" : " 29 GB" | 20260507 17:51:04.988 | INFO | } | 20260507 17:51:04.989 | INFO | Perf report for the last 5.13925 seconds: | 20260507 17:51:04.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:04.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:51:04.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 17:51:04.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:04.989 | INFO | Uploaded 36GiB to GPU in 2.43047s (14.8119GiB/s) | 20260507 17:51:04.989 | INFO | Downloaded 34GiB to GPU in 2.51074s (13.5418GiB/s) | 20260507 17:51:10.127 | INFO | Analyzing chunk frames = [1428480,1428992) | 20260507 17:51:10.127 | INFO | MemInfo: { | 20260507 17:51:10.127 | INFO | "gpu" : " 17 GB", | 20260507 17:51:10.127 | INFO | "huge" : "", | 20260507 17:51:10.127 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:10.128 | INFO | "malloc" : " 14 GB", | 20260507 17:51:10.128 | INFO | "pinned" : " 29 GB" | 20260507 17:51:10.128 | INFO | } | 20260507 17:51:10.128 | INFO | Perf report for the last 5.13939 seconds: | 20260507 17:51:10.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:10.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:51:10.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 17:51:10.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:10.128 | INFO | Uploaded 36GiB to GPU in 2.44746s (14.7091GiB/s) | 20260507 17:51:10.128 | INFO | Downloaded 34GiB to GPU in 2.42562s (14.0171GiB/s) | 20260507 17:51:15.267 | INFO | Analyzing chunk frames = [1428992,1429504) | 20260507 17:51:15.267 | INFO | MemInfo: { | 20260507 17:51:15.267 | INFO | "gpu" : " 17 GB", | 20260507 17:51:15.267 | INFO | "huge" : "", | 20260507 17:51:15.267 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:15.267 | INFO | "malloc" : " 14 GB", | 20260507 17:51:15.267 | INFO | "pinned" : " 29 GB" | 20260507 17:51:15.267 | INFO | } | 20260507 17:51:15.267 | INFO | Perf report for the last 5.13948 seconds: | 20260507 17:51:15.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:15.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 17:51:15.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 17:51:15.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:15.268 | INFO | Uploaded 36GiB to GPU in 2.49923s (14.4044GiB/s) | 20260507 17:51:15.268 | INFO | Downloaded 34GiB to GPU in 2.43849s (13.9431GiB/s) | 20260507 17:51:15.381 | INFO | Starting a round of Baz IO | 20260507 17:51:18.894 | INFO | Populating RTMetrics report | 20260507 17:51:18.894 | INFO | RTMetrics report populated | 20260507 17:51:18.894 | INFO | Opening tmp RTMetrics json file | 20260507 17:51:18.895 | INFO | Serializing RTMetrics to json | 20260507 17:51:18.896 | INFO | Serialization done, swapping tmp file now... | 20260507 17:51:18.896 | INFO | RTMetrics done | 20260507 17:51:18.972 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.591 seconds | 20260507 17:51:18.972 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.47% / 0.02% / 52.51% | 20260507 17:51:18.972 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.30858% / 3.17891e-05% / 3.17891e-05% | 20260507 17:51:18.972 | INFO | Baz IO overall: | 20260507 17:51:18.972 | INFO | BytesWritten : 633621859981 | 20260507 17:51:18.972 | INFO | NumEvents : 307090392547 | 20260507 17:51:20.406 | INFO | Analyzing chunk frames = [1429504,1430016) | 20260507 17:51:20.406 | INFO | MemInfo: { | 20260507 17:51:20.406 | INFO | "gpu" : " 17 GB", | 20260507 17:51:20.406 | INFO | "huge" : "", | 20260507 17:51:20.406 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:20.406 | INFO | "malloc" : " 14 GB", | 20260507 17:51:20.406 | INFO | "pinned" : " 29 GB" | 20260507 17:51:20.406 | INFO | } | 20260507 17:51:20.406 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:51:20.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:20.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:20.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:20.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:20.406 | INFO | ANALYSIS : 69.4%, 2.9, 106.6ms, 0 | 20260507 17:51:20.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:20.406 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260507 17:51:20.406 | INFO | PRE_HQ : 67.6%, 2.0, 69.3ms, 0 | 20260507 17:51:20.406 | INFO | BAZWRITER : 69.2%, 3.1, 111.4ms, 0 | 20260507 17:51:20.407 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 17:51:20.407 | INFO | Uploaded 36GiB to GPU in 2.49593s (14.4235GiB/s) | 20260507 17:51:20.407 | INFO | Downloaded 38GiB to GPU in 2.70332s (14.0568GiB/s) | 20260507 17:51:25.546 | INFO | Analyzing chunk frames = [1430016,1430528) | 20260507 17:51:25.546 | INFO | MemInfo: { | 20260507 17:51:25.546 | INFO | "gpu" : " 17 GB", | 20260507 17:51:25.546 | INFO | "huge" : "", | 20260507 17:51:25.546 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:25.546 | INFO | "malloc" : " 14 GB", | 20260507 17:51:25.546 | INFO | "pinned" : " 29 GB" | 20260507 17:51:25.546 | INFO | } | 20260507 17:51:25.546 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:51:25.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:25.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:51:25.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:51:25.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:25.546 | INFO | Uploaded 36GiB to GPU in 2.50466s (14.3732GiB/s) | 20260507 17:51:25.547 | INFO | Downloaded 34GiB to GPU in 2.45716s (13.8371GiB/s) | 20260507 17:51:30.685 | INFO | Analyzing chunk frames = [1430528,1431040) | 20260507 17:51:30.685 | INFO | MemInfo: { | 20260507 17:51:30.685 | INFO | "gpu" : " 17 GB", | 20260507 17:51:30.685 | INFO | "huge" : "", | 20260507 17:51:30.685 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:30.685 | INFO | "malloc" : " 14 GB", | 20260507 17:51:30.685 | INFO | "pinned" : " 29 GB" | 20260507 17:51:30.685 | INFO | } | 20260507 17:51:30.685 | INFO | Perf report for the last 5.1392 seconds: | 20260507 17:51:30.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:30.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.685 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 17:51:30.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.685 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:51:30.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:30.686 | INFO | Uploaded 36GiB to GPU in 2.53118s (14.2226GiB/s) | 20260507 17:51:30.686 | INFO | Downloaded 34GiB to GPU in 2.4511s (13.8713GiB/s) | 20260507 17:51:35.824 | INFO | Analyzing chunk frames = [1431040,1431552) | 20260507 17:51:35.824 | INFO | MemInfo: { | 20260507 17:51:35.824 | INFO | "gpu" : " 17 GB", | 20260507 17:51:35.824 | INFO | "huge" : "", | 20260507 17:51:35.824 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:35.824 | INFO | "malloc" : " 14 GB", | 20260507 17:51:35.824 | INFO | "pinned" : " 29 GB" | 20260507 17:51:35.824 | INFO | } | 20260507 17:51:35.824 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:51:35.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:35.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.825 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 17:51:35.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.825 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:51:35.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:35.825 | INFO | Uploaded 36GiB to GPU in 2.45564s (14.6601GiB/s) | 20260507 17:51:35.825 | INFO | Downloaded 34GiB to GPU in 2.35253s (14.4525GiB/s) | 20260507 17:51:40.964 | INFO | Analyzing chunk frames = [1431552,1432064) | 20260507 17:51:40.964 | INFO | MemInfo: { | 20260507 17:51:40.964 | INFO | "gpu" : " 17 GB", | 20260507 17:51:40.964 | INFO | "huge" : "", | 20260507 17:51:40.964 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:40.964 | INFO | "malloc" : " 14 GB", | 20260507 17:51:40.964 | INFO | "pinned" : " 29 GB" | 20260507 17:51:40.964 | INFO | } | 20260507 17:51:40.964 | INFO | Perf report for the last 5.14 seconds: | 20260507 17:51:40.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:40.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.965 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 17:51:40.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.965 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:51:40.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:40.965 | INFO | Uploaded 36GiB to GPU in 2.31771s (15.5326GiB/s) | 20260507 17:51:40.965 | INFO | Downloaded 34GiB to GPU in 2.47787s (13.7215GiB/s) | 20260507 17:51:46.103 | INFO | Analyzing chunk frames = [1432064,1432576) | 20260507 17:51:46.103 | INFO | MemInfo: { | 20260507 17:51:46.103 | INFO | "gpu" : " 17 GB", | 20260507 17:51:46.103 | INFO | "huge" : "", | 20260507 17:51:46.103 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:46.103 | INFO | "malloc" : " 14 GB", | 20260507 17:51:46.103 | INFO | "pinned" : " 29 GB" | 20260507 17:51:46.103 | INFO | } | 20260507 17:51:46.104 | INFO | Perf report for the last 5.13906 seconds: | 20260507 17:51:46.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:46.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:51:46.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 17:51:46.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:46.104 | INFO | Uploaded 36GiB to GPU in 2.41734s (14.8924GiB/s) | 20260507 17:51:46.104 | INFO | Downloaded 34GiB to GPU in 2.51086s (13.5412GiB/s) | 20260507 17:51:51.242 | INFO | Analyzing chunk frames = [1432576,1433088) | 20260507 17:51:51.242 | INFO | MemInfo: { | 20260507 17:51:51.242 | INFO | "gpu" : " 17 GB", | 20260507 17:51:51.242 | INFO | "huge" : "", | 20260507 17:51:51.242 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:51.243 | INFO | "malloc" : " 14 GB", | 20260507 17:51:51.243 | INFO | "pinned" : " 29 GB" | 20260507 17:51:51.243 | INFO | } | 20260507 17:51:51.243 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:51:51.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:51.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:51:51.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 17:51:51.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:51.243 | INFO | Uploaded 36GiB to GPU in 2.45874s (14.6417GiB/s) | 20260507 17:51:51.243 | INFO | Downloaded 34GiB to GPU in 2.45148s (13.8692GiB/s) | 20260507 17:51:56.381 | INFO | Analyzing chunk frames = [1433088,1433600) | 20260507 17:51:56.382 | INFO | MemInfo: { | 20260507 17:51:56.382 | INFO | "gpu" : " 17 GB", | 20260507 17:51:56.382 | INFO | "huge" : "", | 20260507 17:51:56.382 | INFO | "huge_shared" : " 29 GB", | 20260507 17:51:56.382 | INFO | "malloc" : " 14 GB", | 20260507 17:51:56.382 | INFO | "pinned" : " 29 GB" | 20260507 17:51:56.382 | INFO | } | 20260507 17:51:56.382 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:51:56.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:51:56.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:51:56.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 17:51:56.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:51:56.382 | INFO | Uploaded 36GiB to GPU in 2.42552s (14.8422GiB/s) | 20260507 17:51:56.383 | INFO | Downloaded 34GiB to GPU in 2.44014s (13.9336GiB/s) | 20260507 17:51:56.383 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:51:56.383 | INFO | Total mem usage: 16.4 GiB | 20260507 17:51:56.383 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:51:56.383 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.1%) | 20260507 17:51:56.383 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:51:56.383 | INFO | High Water Breakdown: | 20260507 17:51:56.383 | INFO | 59.2% : 8.59 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:51:56.383 | INFO | 40.7% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:51:56.383 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:51:56.383 | INFO | ------------------------------------------ | 20260507 17:51:56.383 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:51:56.383 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:51:56.383 | INFO | Total mem usage: 26.5 GiB | 20260507 17:51:56.383 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:51:56.383 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:51:56.383 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:51:56.383 | INFO | High Water Breakdown: | 20260507 17:51:56.383 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:51:56.383 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:51:56.383 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:51:56.383 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:51:56.383 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:51:56.383 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:51:56.383 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:51:56.383 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:51:56.383 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:51:56.383 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:51:56.383 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:51:56.383 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:51:56.383 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:51:56.383 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:51:56.383 | INFO | ------------------------------------------ | 20260507 17:51:56.383 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:51:56.383 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:51:56.383 | INFO | Total mem usage: 27.8 GiB | 20260507 17:51:56.383 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:51:56.383 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:51:56.383 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:51:56.384 | INFO | High Water Breakdown: | 20260507 17:51:56.384 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:51:56.384 | INFO | ------------------------------------------ | 20260507 17:51:56.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:51:56.384 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:51:56.384 | INFO | Total mem usage: 15.7 GiB | 20260507 17:51:56.384 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:51:56.384 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:51:56.384 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:51:56.384 | INFO | High Water Breakdown: | 20260507 17:51:56.384 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:51:56.384 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:51:56.384 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:51:56.384 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:51:56.384 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:51:56.384 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:51:56.384 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:51:56.384 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:51:56.384 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:51:56.384 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:51:56.384 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:51:56.384 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:51:56.384 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:51:56.384 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:51:56.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:51:56.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:51:56.384 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:51:56.384 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:51:56.384 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:51:56.384 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:51:56.384 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:51:56.384 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:51:56.384 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:51:56.384 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:51:56.384 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:51:56.384 | INFO | ------------------------------------------ | 20260507 17:51:56.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:51:56.384 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:51:56.384 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:51:56.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:51:56.384 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.44e+01 / 2.20e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.05e-01, 1.13e-02 / 6.32e-03 | 20260507 17:51:56.384 | INFO | PerfReport: 12.12%, Baseline -- 4.06e+00 / 4.51e+00, 6.20e-01 / 2.25e-01 | 20260507 17:51:56.384 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.19e-01 / 1.20e-01 | 20260507 17:51:56.384 | INFO | PerfReport: 15.64%, DME -- 4.19e+01 / 4.48e+01, 4.10e+00 / 1.90e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.90e+00 / 2.08e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.52e+00 / 2.85e+00, 4.38e-01 / 2.60e-01 | 20260507 17:51:56.384 | INFO | PerfReport: 3.75%, Metrics -- 1.26e+00 / 1.44e+00, 3.19e-01 / 1.87e-01 | 20260507 17:51:56.384 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.99e-01 / 4.27e-01, 6.57e-02 / 3.84e-02 | 20260507 17:51:56.384 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:51:56.384 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:51:56.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:51:56.384 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 23.90%, Upload -- 2.56e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 24.06%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 31.28%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 1.80%, LockUpload -- 1.93e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 2.37%, LockDownload -- 2.54e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 16.58%, LockCompute -- 1.78e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 17:51:56.384 | INFO | PerfReport: 0.01%, Misc -- 1.58e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 17:51:56.493 | INFO | Starting a round of Baz IO | 20260507 17:51:59.981 | INFO | Populating RTMetrics report | 20260507 17:51:59.981 | INFO | RTMetrics report populated | 20260507 17:51:59.981 | INFO | Opening tmp RTMetrics json file | 20260507 17:51:59.983 | INFO | Serializing RTMetrics to json | 20260507 17:51:59.999 | INFO | Serialization done, swapping tmp file now... | 20260507 17:52:00.000 | INFO | RTMetrics done | 20260507 17:52:00.062 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.568 seconds | 20260507 17:52:00.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.45% / 0.02% / 52.53% | 20260507 17:52:00.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29591% / 3.57628e-05% / 4.76837e-05% | 20260507 17:52:00.062 | INFO | Baz IO overall: | 20260507 17:52:00.062 | INFO | BytesWritten : 635556099578 | 20260507 17:52:00.062 | INFO | NumEvents : 308030021291 | 20260507 17:52:01.521 | INFO | Analyzing chunk frames = [1433600,1434112) | 20260507 17:52:01.521 | INFO | MemInfo: { | 20260507 17:52:01.521 | INFO | "gpu" : " 17 GB", | 20260507 17:52:01.521 | INFO | "huge" : "", | 20260507 17:52:01.522 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:01.522 | INFO | "malloc" : " 14 GB", | 20260507 17:52:01.522 | INFO | "pinned" : " 29 GB" | 20260507 17:52:01.522 | INFO | } | 20260507 17:52:01.522 | INFO | Perf report for the last 5.13982 seconds: | 20260507 17:52:01.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:01.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:01.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:01.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:01.522 | INFO | ANALYSIS : 69.6%, 3.0, 108.2ms, 0 | 20260507 17:52:01.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:01.522 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 17:52:01.522 | INFO | PRE_HQ : 67.6%, 1.8, 64.7ms, 0 | 20260507 17:52:01.522 | INFO | BAZWRITER : 68.5%, 3.0, 107.0ms, 0 | 20260507 17:52:01.522 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:52:01.522 | INFO | Uploaded 36GiB to GPU in 2.53449s (14.204GiB/s) | 20260507 17:52:01.522 | INFO | Downloaded 38GiB to GPU in 2.74182s (13.8594GiB/s) | 20260507 17:52:06.660 | INFO | Analyzing chunk frames = [1434112,1434624) | 20260507 17:52:06.660 | INFO | MemInfo: { | 20260507 17:52:06.660 | INFO | "gpu" : " 17 GB", | 20260507 17:52:06.661 | INFO | "huge" : "", | 20260507 17:52:06.661 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:06.661 | INFO | "malloc" : " 14 GB", | 20260507 17:52:06.661 | INFO | "pinned" : " 29 GB" | 20260507 17:52:06.661 | INFO | } | 20260507 17:52:06.661 | INFO | Perf report for the last 5.13896 seconds: | 20260507 17:52:06.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:06.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:52:06.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | PRE_HQ : 37.1%, 1.0, 19.2ms, 0 | 20260507 17:52:06.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:06.661 | INFO | Uploaded 36GiB to GPU in 2.46224s (14.6208GiB/s) | 20260507 17:52:06.661 | INFO | Downloaded 34GiB to GPU in 2.4739s (13.7435GiB/s) | 20260507 17:52:11.800 | INFO | Analyzing chunk frames = [1434624,1435136) | 20260507 17:52:11.800 | INFO | MemInfo: { | 20260507 17:52:11.800 | INFO | "gpu" : " 17 GB", | 20260507 17:52:11.800 | INFO | "huge" : "", | 20260507 17:52:11.800 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:11.800 | INFO | "malloc" : " 14 GB", | 20260507 17:52:11.800 | INFO | "pinned" : " 29 GB" | 20260507 17:52:11.800 | INFO | } | 20260507 17:52:11.800 | INFO | Perf report for the last 5.13926 seconds: | 20260507 17:52:11.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:11.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.800 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 17:52:11.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.800 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 17:52:11.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:11.801 | INFO | Uploaded 36GiB to GPU in 2.46898s (14.5809GiB/s) | 20260507 17:52:11.801 | INFO | Downloaded 34GiB to GPU in 2.46457s (13.7955GiB/s) | 20260507 17:52:16.939 | INFO | Analyzing chunk frames = [1435136,1435648) | 20260507 17:52:16.939 | INFO | MemInfo: { | 20260507 17:52:16.939 | INFO | "gpu" : " 17 GB", | 20260507 17:52:16.939 | INFO | "huge" : "", | 20260507 17:52:16.939 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:16.939 | INFO | "malloc" : " 14 GB", | 20260507 17:52:16.939 | INFO | "pinned" : " 29 GB" | 20260507 17:52:16.939 | INFO | } | 20260507 17:52:16.939 | INFO | Perf report for the last 5.13898 seconds: | 20260507 17:52:16.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:16.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 17:52:16.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 17:52:16.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:16.939 | INFO | Uploaded 36GiB to GPU in 2.43358s (14.793GiB/s) | 20260507 17:52:16.939 | INFO | Downloaded 34GiB to GPU in 2.3882s (14.2367GiB/s) | 20260507 17:52:22.078 | INFO | Analyzing chunk frames = [1435648,1436160) | 20260507 17:52:22.078 | INFO | MemInfo: { | 20260507 17:52:22.078 | INFO | "gpu" : " 17 GB", | 20260507 17:52:22.079 | INFO | "huge" : "", | 20260507 17:52:22.079 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:22.079 | INFO | "malloc" : " 14 GB", | 20260507 17:52:22.079 | INFO | "pinned" : " 29 GB" | 20260507 17:52:22.079 | INFO | } | 20260507 17:52:22.079 | INFO | Perf report for the last 5.13967 seconds: | 20260507 17:52:22.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:22.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:52:22.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 17:52:22.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:22.079 | INFO | Uploaded 36GiB to GPU in 2.28681s (15.7424GiB/s) | 20260507 17:52:22.079 | INFO | Downloaded 34GiB to GPU in 2.49644s (13.6194GiB/s) | 20260507 17:52:27.218 | INFO | Analyzing chunk frames = [1436160,1436672) | 20260507 17:52:27.218 | INFO | MemInfo: { | 20260507 17:52:27.218 | INFO | "gpu" : " 17 GB", | 20260507 17:52:27.218 | INFO | "huge" : "", | 20260507 17:52:27.218 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:27.218 | INFO | "malloc" : " 14 GB", | 20260507 17:52:27.218 | INFO | "pinned" : " 29 GB" | 20260507 17:52:27.218 | INFO | } | 20260507 17:52:27.218 | INFO | Perf report for the last 5.13957 seconds: | 20260507 17:52:27.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:27.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.218 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:52:27.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.219 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260507 17:52:27.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:27.219 | INFO | Uploaded 36GiB to GPU in 2.43507s (14.7839GiB/s) | 20260507 17:52:27.219 | INFO | Downloaded 34GiB to GPU in 2.52325s (13.4747GiB/s) | 20260507 17:52:32.357 | INFO | Analyzing chunk frames = [1436672,1437184) | 20260507 17:52:32.357 | INFO | MemInfo: { | 20260507 17:52:32.357 | INFO | "gpu" : " 17 GB", | 20260507 17:52:32.358 | INFO | "huge" : "", | 20260507 17:52:32.358 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:32.358 | INFO | "malloc" : " 14 GB", | 20260507 17:52:32.358 | INFO | "pinned" : " 29 GB" | 20260507 17:52:32.358 | INFO | } | 20260507 17:52:32.358 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:52:32.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:32.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:52:32.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 17:52:32.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:32.358 | INFO | Uploaded 36GiB to GPU in 2.50141s (14.3919GiB/s) | 20260507 17:52:32.358 | INFO | Downloaded 34GiB to GPU in 2.44315s (13.9164GiB/s) | 20260507 17:52:37.496 | INFO | Analyzing chunk frames = [1437184,1437696) | 20260507 17:52:37.496 | INFO | MemInfo: { | 20260507 17:52:37.496 | INFO | "gpu" : " 17 GB", | 20260507 17:52:37.496 | INFO | "huge" : "", | 20260507 17:52:37.497 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:37.497 | INFO | "malloc" : " 14 GB", | 20260507 17:52:37.497 | INFO | "pinned" : " 29 GB" | 20260507 17:52:37.497 | INFO | } | 20260507 17:52:37.497 | INFO | Perf report for the last 5.13904 seconds: | 20260507 17:52:37.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:37.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.497 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 17:52:37.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.497 | INFO | PRE_HQ : 38.4%, 1.0, 20.0ms, 0 | 20260507 17:52:37.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:37.498 | INFO | Uploaded 36GiB to GPU in 2.49369s (14.4364GiB/s) | 20260507 17:52:37.498 | INFO | Downloaded 34GiB to GPU in 2.43078s (13.9873GiB/s) | 20260507 17:52:37.611 | INFO | Starting a round of Baz IO | 20260507 17:52:41.121 | INFO | Populating RTMetrics report | 20260507 17:52:41.121 | INFO | RTMetrics report populated | 20260507 17:52:41.121 | INFO | Opening tmp RTMetrics json file | 20260507 17:52:41.124 | INFO | Serializing RTMetrics to json | 20260507 17:52:41.139 | INFO | Serialization done, swapping tmp file now... | 20260507 17:52:41.140 | INFO | RTMetrics done | 20260507 17:52:41.216 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.604 seconds | 20260507 17:52:41.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.43% / 0.02% / 52.55% | 20260507 17:52:41.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29073% / 1.58946e-05% / 4.37101e-05% | 20260507 17:52:41.216 | INFO | Baz IO overall: | 20260507 17:52:41.216 | INFO | BytesWritten : 637490641953 | 20260507 17:52:41.216 | INFO | NumEvents : 308969821140 | 20260507 17:52:42.636 | INFO | Analyzing chunk frames = [1437696,1438208) | 20260507 17:52:42.636 | INFO | MemInfo: { | 20260507 17:52:42.636 | INFO | "gpu" : " 17 GB", | 20260507 17:52:42.636 | INFO | "huge" : "", | 20260507 17:52:42.636 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:42.636 | INFO | "malloc" : " 14 GB", | 20260507 17:52:42.636 | INFO | "pinned" : " 29 GB" | 20260507 17:52:42.636 | INFO | } | 20260507 17:52:42.636 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:52:42.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:42.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:42.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:42.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:42.636 | INFO | ANALYSIS : 69.8%, 3.0, 108.5ms, 0 | 20260507 17:52:42.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:42.636 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260507 17:52:42.636 | INFO | PRE_HQ : 68.1%, 1.9, 67.2ms, 0 | 20260507 17:52:42.636 | INFO | BAZWRITER : 69.5%, 3.1, 110.4ms, 0 | 20260507 17:52:42.636 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:52:42.636 | INFO | Uploaded 36GiB to GPU in 2.58439s (13.9298GiB/s) | 20260507 17:52:42.637 | INFO | Downloaded 38GiB to GPU in 2.72437s (13.9482GiB/s) | 20260507 17:52:47.775 | INFO | Analyzing chunk frames = [1438208,1438720) | 20260507 17:52:47.775 | INFO | MemInfo: { | 20260507 17:52:47.775 | INFO | "gpu" : " 17 GB", | 20260507 17:52:47.775 | INFO | "huge" : "", | 20260507 17:52:47.775 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:47.775 | INFO | "malloc" : " 14 GB", | 20260507 17:52:47.775 | INFO | "pinned" : " 29 GB" | 20260507 17:52:47.775 | INFO | } | 20260507 17:52:47.775 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:52:47.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:47.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.775 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 17:52:47.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.775 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:52:47.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:47.776 | INFO | Uploaded 36GiB to GPU in 2.51879s (14.2926GiB/s) | 20260507 17:52:47.776 | INFO | Downloaded 34GiB to GPU in 2.42067s (14.0457GiB/s) | 20260507 17:52:52.914 | INFO | Analyzing chunk frames = [1438720,1439232) | 20260507 17:52:52.915 | INFO | MemInfo: { | 20260507 17:52:52.915 | INFO | "gpu" : " 17 GB", | 20260507 17:52:52.915 | INFO | "huge" : "", | 20260507 17:52:52.915 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:52.915 | INFO | "malloc" : " 14 GB", | 20260507 17:52:52.915 | INFO | "pinned" : " 29 GB" | 20260507 17:52:52.915 | INFO | } | 20260507 17:52:52.915 | INFO | Perf report for the last 5.13944 seconds: | 20260507 17:52:52.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:52.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:52:52.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:52:52.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:52.915 | INFO | Uploaded 36GiB to GPU in 2.57605s (13.9749GiB/s) | 20260507 17:52:52.915 | INFO | Downloaded 34GiB to GPU in 2.41494s (14.079GiB/s) | 20260507 17:52:58.054 | INFO | Analyzing chunk frames = [1439232,1439744) | 20260507 17:52:58.054 | INFO | MemInfo: { | 20260507 17:52:58.054 | INFO | "gpu" : " 17 GB", | 20260507 17:52:58.054 | INFO | "huge" : "", | 20260507 17:52:58.054 | INFO | "huge_shared" : " 29 GB", | 20260507 17:52:58.054 | INFO | "malloc" : " 14 GB", | 20260507 17:52:58.054 | INFO | "pinned" : " 29 GB" | 20260507 17:52:58.054 | INFO | } | 20260507 17:52:58.054 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:52:58.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:52:58.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 17:52:58.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:52:58.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:52:58.054 | INFO | Uploaded 36GiB to GPU in 2.50007s (14.3996GiB/s) | 20260507 17:52:58.055 | INFO | Downloaded 34GiB to GPU in 2.34809s (14.4799GiB/s) | 20260507 17:53:03.193 | INFO | Analyzing chunk frames = [1439744,1440256) | 20260507 17:53:03.193 | INFO | MemInfo: { | 20260507 17:53:03.193 | INFO | "gpu" : " 17 GB", | 20260507 17:53:03.193 | INFO | "huge" : "", | 20260507 17:53:03.193 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:03.193 | INFO | "malloc" : " 14 GB", | 20260507 17:53:03.193 | INFO | "pinned" : " 29 GB" | 20260507 17:53:03.193 | INFO | } | 20260507 17:53:03.194 | INFO | Perf report for the last 5.13962 seconds: | 20260507 17:53:03.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:03.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 17:53:03.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 17:53:03.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:03.194 | INFO | Uploaded 36GiB to GPU in 2.34676s (15.3403GiB/s) | 20260507 17:53:03.194 | INFO | Downloaded 34GiB to GPU in 2.46511s (13.7925GiB/s) | 20260507 17:53:08.332 | INFO | Analyzing chunk frames = [1440256,1440768) | 20260507 17:53:08.332 | INFO | MemInfo: { | 20260507 17:53:08.332 | INFO | "gpu" : " 17 GB", | 20260507 17:53:08.332 | INFO | "huge" : "", | 20260507 17:53:08.332 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:08.332 | INFO | "malloc" : " 14 GB", | 20260507 17:53:08.332 | INFO | "pinned" : " 29 GB" | 20260507 17:53:08.332 | INFO | } | 20260507 17:53:08.333 | INFO | Perf report for the last 5.13891 seconds: | 20260507 17:53:08.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:08.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:53:08.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260507 17:53:08.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:08.333 | INFO | Uploaded 36GiB to GPU in 2.38171s (15.1152GiB/s) | 20260507 17:53:08.333 | INFO | Downloaded 34GiB to GPU in 2.50118s (13.5936GiB/s) | 20260507 17:53:13.472 | INFO | Analyzing chunk frames = [1440768,1441280) | 20260507 17:53:13.472 | INFO | MemInfo: { | 20260507 17:53:13.472 | INFO | "gpu" : " 17 GB", | 20260507 17:53:13.472 | INFO | "huge" : "", | 20260507 17:53:13.472 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:13.472 | INFO | "malloc" : " 14 GB", | 20260507 17:53:13.472 | INFO | "pinned" : " 29 GB" | 20260507 17:53:13.472 | INFO | } | 20260507 17:53:13.472 | INFO | Perf report for the last 5.13959 seconds: | 20260507 17:53:13.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:13.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.472 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:53:13.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.472 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:53:13.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:13.473 | INFO | Uploaded 36GiB to GPU in 2.47028s (14.5732GiB/s) | 20260507 17:53:13.473 | INFO | Downloaded 34GiB to GPU in 2.42798s (14.0034GiB/s) | 20260507 17:53:18.611 | INFO | Analyzing chunk frames = [1441280,1441792) | 20260507 17:53:18.611 | INFO | MemInfo: { | 20260507 17:53:18.611 | INFO | "gpu" : " 17 GB", | 20260507 17:53:18.611 | INFO | "huge" : "", | 20260507 17:53:18.611 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:18.611 | INFO | "malloc" : " 14 GB", | 20260507 17:53:18.611 | INFO | "pinned" : " 29 GB" | 20260507 17:53:18.611 | INFO | } | 20260507 17:53:18.611 | INFO | Perf report for the last 5.1393 seconds: | 20260507 17:53:18.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:18.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.611 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 17:53:18.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.612 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:53:18.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:18.612 | INFO | Uploaded 36GiB to GPU in 2.51934s (14.2895GiB/s) | 20260507 17:53:18.612 | INFO | Downloaded 34GiB to GPU in 2.50549s (13.5702GiB/s) | 20260507 17:53:18.612 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:53:18.612 | INFO | Total mem usage: 16.4 GiB | 20260507 17:53:18.612 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:53:18.612 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:53:18.612 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:53:18.612 | INFO | High Water Breakdown: | 20260507 17:53:18.612 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:53:18.612 | INFO | 40.7% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:53:18.612 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:53:18.612 | INFO | ------------------------------------------ | 20260507 17:53:18.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:53:18.612 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:53:18.612 | INFO | Total mem usage: 26.5 GiB | 20260507 17:53:18.612 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:53:18.612 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:53:18.612 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:53:18.612 | INFO | High Water Breakdown: | 20260507 17:53:18.612 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:53:18.612 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:53:18.612 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:53:18.612 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:53:18.612 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:53:18.612 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:53:18.612 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:53:18.612 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:53:18.612 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:53:18.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:53:18.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:53:18.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:53:18.613 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:53:18.613 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:53:18.613 | INFO | ------------------------------------------ | 20260507 17:53:18.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:53:18.613 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:53:18.613 | INFO | Total mem usage: 27.8 GiB | 20260507 17:53:18.613 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:53:18.613 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 17:53:18.613 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 17:53:18.613 | INFO | High Water Breakdown: | 20260507 17:53:18.613 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 17:53:18.613 | INFO | ------------------------------------------ | 20260507 17:53:18.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:53:18.613 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:53:18.613 | INFO | Total mem usage: 15.7 GiB | 20260507 17:53:18.613 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:53:18.613 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:53:18.613 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:53:18.613 | INFO | High Water Breakdown: | 20260507 17:53:18.613 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:53:18.613 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:53:18.613 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:53:18.613 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:53:18.613 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:53:18.613 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:53:18.613 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:53:18.613 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:53:18.613 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:53:18.613 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:53:18.613 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:53:18.613 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:53:18.613 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:53:18.613 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:53:18.613 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:53:18.613 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:53:18.613 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:53:18.613 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:53:18.613 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:53:18.613 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:53:18.613 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:53:18.613 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:53:18.613 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:53:18.613 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:53:18.613 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:53:18.613 | INFO | ------------------------------------------ | 20260507 17:53:18.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:53:18.613 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:53:18.613 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:53:18.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:53:18.613 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.45e+01 / 2.67e+00 | 20260507 17:53:18.613 | INFO | PerfReport: 0.30%, AggregateStats -- 1.02e-01 / 1.06e-01, 1.27e-02 / 7.27e-03 | 20260507 17:53:18.613 | INFO | PerfReport: 12.09%, Baseline -- 4.05e+00 / 4.51e+00, 6.49e-01 / 2.43e-01 | 20260507 17:53:18.613 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.18e-01 / 1.02e-01 | 20260507 17:53:18.613 | INFO | PerfReport: 15.70%, DME -- 4.21e+01 / 4.50e+01, 4.16e+00 / 2.64e+00 | 20260507 17:53:18.613 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.87e+00 / 2.20e+00 | 20260507 17:53:18.613 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.52e+00 / 2.85e+00, 4.39e-01 / 2.56e-01 | 20260507 17:53:18.613 | INFO | PerfReport: 3.76%, Metrics -- 1.26e+00 / 1.43e+00, 3.19e-01 / 2.01e-01 | 20260507 17:53:18.613 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.01e-01 / 4.25e-01, 6.60e-02 / 4.36e-02 | 20260507 17:53:18.614 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:53:18.614 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:53:18.614 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:53:18.614 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 23.93%, Upload -- 2.57e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 24.09%, Download -- 2.59e+01 / 0.00e+00, 8.62e+00 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 31.20%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 1.68%, LockUpload -- 1.81e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 2.40%, LockDownload -- 2.58e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 16.68%, LockCompute -- 1.79e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 17:53:18.614 | INFO | PerfReport: 0.02%, Misc -- 1.67e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 17:53:18.708 | INFO | Starting a round of Baz IO | 20260507 17:53:22.222 | INFO | Populating RTMetrics report | 20260507 17:53:22.222 | INFO | RTMetrics report populated | 20260507 17:53:22.222 | INFO | Opening tmp RTMetrics json file | 20260507 17:53:22.223 | INFO | Serializing RTMetrics to json | 20260507 17:53:22.234 | INFO | Serialization done, swapping tmp file now... | 20260507 17:53:22.241 | INFO | RTMetrics done | 20260507 17:53:22.305 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.596 seconds | 20260507 17:53:22.305 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.42% / 0.02% / 52.57% | 20260507 17:53:22.305 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29214% / 3.57628e-05% / 2.78155e-05% | 20260507 17:53:22.305 | INFO | Baz IO overall: | 20260507 17:53:22.305 | INFO | BytesWritten : 639427640348 | 20260507 17:53:22.305 | INFO | NumEvents : 309911312576 | 20260507 17:53:23.751 | INFO | Analyzing chunk frames = [1441792,1442304) | 20260507 17:53:23.751 | INFO | MemInfo: { | 20260507 17:53:23.751 | INFO | "gpu" : " 17 GB", | 20260507 17:53:23.751 | INFO | "huge" : "", | 20260507 17:53:23.751 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:23.751 | INFO | "malloc" : " 14 GB", | 20260507 17:53:23.751 | INFO | "pinned" : " 29 GB" | 20260507 17:53:23.751 | INFO | } | 20260507 17:53:23.751 | INFO | Perf report for the last 5.13922 seconds: | 20260507 17:53:23.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:23.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:23.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:23.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:23.751 | INFO | ANALYSIS : 69.5%, 3.0, 108.5ms, 0 | 20260507 17:53:23.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:23.751 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 17:53:23.751 | INFO | PRE_HQ : 66.4%, 1.7, 60.0ms, 0 | 20260507 17:53:23.751 | INFO | BAZWRITER : 69.1%, 2.9, 102.5ms, 0 | 20260507 17:53:23.751 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:53:23.751 | INFO | Uploaded 36GiB to GPU in 2.47746s (14.531GiB/s) | 20260507 17:53:23.751 | INFO | Downloaded 38GiB to GPU in 2.72085s (13.9662GiB/s) | 20260507 17:53:28.890 | INFO | Analyzing chunk frames = [1442304,1442816) | 20260507 17:53:28.890 | INFO | MemInfo: { | 20260507 17:53:28.890 | INFO | "gpu" : " 17 GB", | 20260507 17:53:28.890 | INFO | "huge" : "", | 20260507 17:53:28.890 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:28.890 | INFO | "malloc" : " 14 GB", | 20260507 17:53:28.890 | INFO | "pinned" : " 29 GB" | 20260507 17:53:28.890 | INFO | } | 20260507 17:53:28.890 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:53:28.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:28.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:53:28.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 17:53:28.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:28.890 | INFO | Uploaded 36GiB to GPU in 2.50322s (14.3815GiB/s) | 20260507 17:53:28.891 | INFO | Downloaded 34GiB to GPU in 2.4691s (13.7702GiB/s) | 20260507 17:53:34.029 | INFO | Analyzing chunk frames = [1442816,1443328) | 20260507 17:53:34.029 | INFO | MemInfo: { | 20260507 17:53:34.029 | INFO | "gpu" : " 17 GB", | 20260507 17:53:34.029 | INFO | "huge" : "", | 20260507 17:53:34.029 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:34.029 | INFO | "malloc" : " 14 GB", | 20260507 17:53:34.029 | INFO | "pinned" : " 29 GB" | 20260507 17:53:34.029 | INFO | } | 20260507 17:53:34.029 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:53:34.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:34.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.030 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 17:53:34.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.030 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 17:53:34.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:34.030 | INFO | Uploaded 36GiB to GPU in 2.50346s (14.3801GiB/s) | 20260507 17:53:34.030 | INFO | Downloaded 34GiB to GPU in 2.46046s (13.8186GiB/s) | 20260507 17:53:39.168 | INFO | Analyzing chunk frames = [1443328,1443840) | 20260507 17:53:39.168 | INFO | MemInfo: { | 20260507 17:53:39.168 | INFO | "gpu" : " 17 GB", | 20260507 17:53:39.168 | INFO | "huge" : "", | 20260507 17:53:39.169 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:39.169 | INFO | "malloc" : " 14 GB", | 20260507 17:53:39.169 | INFO | "pinned" : " 29 GB" | 20260507 17:53:39.169 | INFO | } | 20260507 17:53:39.169 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:53:39.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:39.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 17:53:39.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 17:53:39.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:39.169 | INFO | Uploaded 36GiB to GPU in 2.43933s (14.7581GiB/s) | 20260507 17:53:39.170 | INFO | Downloaded 34GiB to GPU in 2.38417s (14.2607GiB/s) | 20260507 17:53:44.308 | INFO | Analyzing chunk frames = [1443840,1444352) | 20260507 17:53:44.308 | INFO | MemInfo: { | 20260507 17:53:44.308 | INFO | "gpu" : " 17 GB", | 20260507 17:53:44.308 | INFO | "huge" : "", | 20260507 17:53:44.308 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:44.309 | INFO | "malloc" : " 14 GB", | 20260507 17:53:44.309 | INFO | "pinned" : " 29 GB" | 20260507 17:53:44.309 | INFO | } | 20260507 17:53:44.309 | INFO | Perf report for the last 5.13988 seconds: | 20260507 17:53:44.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:44.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 17:53:44.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 17:53:44.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:44.309 | INFO | Uploaded 36GiB to GPU in 2.31492s (15.5513GiB/s) | 20260507 17:53:44.310 | INFO | Downloaded 34GiB to GPU in 2.48577s (13.6778GiB/s) | 20260507 17:53:49.447 | INFO | Analyzing chunk frames = [1444352,1444864) | 20260507 17:53:49.447 | INFO | MemInfo: { | 20260507 17:53:49.447 | INFO | "gpu" : " 17 GB", | 20260507 17:53:49.447 | INFO | "huge" : "", | 20260507 17:53:49.447 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:49.447 | INFO | "malloc" : " 14 GB", | 20260507 17:53:49.447 | INFO | "pinned" : " 29 GB" | 20260507 17:53:49.447 | INFO | } | 20260507 17:53:49.447 | INFO | Perf report for the last 5.13873 seconds: | 20260507 17:53:49.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:49.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.447 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 17:53:49.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.448 | INFO | PRE_HQ : 35.2%, 1.0, 18.2ms, 0 | 20260507 17:53:49.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:49.448 | INFO | Uploaded 36GiB to GPU in 2.42994s (14.8152GiB/s) | 20260507 17:53:49.448 | INFO | Downloaded 34GiB to GPU in 2.5182s (13.5017GiB/s) | 20260507 17:53:54.587 | INFO | Analyzing chunk frames = [1444864,1445376) | 20260507 17:53:54.587 | INFO | MemInfo: { | 20260507 17:53:54.587 | INFO | "gpu" : " 17 GB", | 20260507 17:53:54.587 | INFO | "huge" : "", | 20260507 17:53:54.587 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:54.587 | INFO | "malloc" : " 14 GB", | 20260507 17:53:54.587 | INFO | "pinned" : " 29 GB" | 20260507 17:53:54.587 | INFO | } | 20260507 17:53:54.587 | INFO | Perf report for the last 5.13963 seconds: | 20260507 17:53:54.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:54.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:53:54.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 17:53:54.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:54.587 | INFO | Uploaded 36GiB to GPU in 2.50455s (14.3738GiB/s) | 20260507 17:53:54.587 | INFO | Downloaded 34GiB to GPU in 2.43889s (13.9408GiB/s) | 20260507 17:53:59.726 | INFO | Analyzing chunk frames = [1445376,1445888) | 20260507 17:53:59.726 | INFO | MemInfo: { | 20260507 17:53:59.726 | INFO | "gpu" : " 17 GB", | 20260507 17:53:59.726 | INFO | "huge" : "", | 20260507 17:53:59.726 | INFO | "huge_shared" : " 29 GB", | 20260507 17:53:59.726 | INFO | "malloc" : " 14 GB", | 20260507 17:53:59.726 | INFO | "pinned" : " 29 GB" | 20260507 17:53:59.726 | INFO | } | 20260507 17:53:59.726 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:53:59.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:53:59.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.726 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 17:53:59.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.726 | INFO | PRE_HQ : 38.3%, 1.0, 19.9ms, 0 | 20260507 17:53:59.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:53:59.727 | INFO | Uploaded 36GiB to GPU in 2.4903s (14.4561GiB/s) | 20260507 17:53:59.727 | INFO | Downloaded 34GiB to GPU in 2.42485s (14.0215GiB/s) | 20260507 17:53:59.839 | INFO | Starting a round of Baz IO | 20260507 17:54:03.340 | INFO | Populating RTMetrics report | 20260507 17:54:03.340 | INFO | RTMetrics report populated | 20260507 17:54:03.340 | INFO | Opening tmp RTMetrics json file | 20260507 17:54:03.351 | INFO | Serializing RTMetrics to json | 20260507 17:54:03.358 | INFO | Serialization done, swapping tmp file now... | 20260507 17:54:03.359 | INFO | RTMetrics done | 20260507 17:54:03.421 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.581 seconds | 20260507 17:54:03.421 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.40% / 0.02% / 52.58% | 20260507 17:54:03.421 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.2811% / 2.78155e-05% / 2.38419e-05% | 20260507 17:54:03.421 | INFO | Baz IO overall: | 20260507 17:54:03.421 | INFO | BytesWritten : 641363577553 | 20260507 17:54:03.421 | INFO | NumEvents : 310852078830 | 20260507 17:54:04.865 | INFO | Analyzing chunk frames = [1445888,1446400) | 20260507 17:54:04.865 | INFO | MemInfo: { | 20260507 17:54:04.865 | INFO | "gpu" : " 17 GB", | 20260507 17:54:04.865 | INFO | "huge" : "", | 20260507 17:54:04.865 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:04.865 | INFO | "malloc" : " 14 GB", | 20260507 17:54:04.865 | INFO | "pinned" : " 29 GB" | 20260507 17:54:04.865 | INFO | } | 20260507 17:54:04.865 | INFO | Perf report for the last 5.13895 seconds: | 20260507 17:54:04.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:04.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:04.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:04.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:04.865 | INFO | ANALYSIS : 69.7%, 3.0, 108.6ms, 0 | 20260507 17:54:04.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:04.865 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 17:54:04.865 | INFO | PRE_HQ : 67.4%, 1.8, 62.8ms, 0 | 20260507 17:54:04.866 | INFO | BAZWRITER : 68.7%, 3.0, 104.6ms, 0 | 20260507 17:54:04.866 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:54:04.866 | INFO | Uploaded 36GiB to GPU in 2.55752s (14.0761GiB/s) | 20260507 17:54:04.866 | INFO | Downloaded 38GiB to GPU in 2.73087s (13.915GiB/s) | 20260507 17:54:10.004 | INFO | Analyzing chunk frames = [1446400,1446912) | 20260507 17:54:10.005 | INFO | MemInfo: { | 20260507 17:54:10.005 | INFO | "gpu" : " 17 GB", | 20260507 17:54:10.005 | INFO | "huge" : "", | 20260507 17:54:10.005 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:10.005 | INFO | "malloc" : " 14 GB", | 20260507 17:54:10.005 | INFO | "pinned" : " 29 GB" | 20260507 17:54:10.005 | INFO | } | 20260507 17:54:10.005 | INFO | Perf report for the last 5.13963 seconds: | 20260507 17:54:10.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:10.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.005 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 17:54:10.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.005 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260507 17:54:10.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:10.006 | INFO | Uploaded 36GiB to GPU in 2.54163s (14.1641GiB/s) | 20260507 17:54:10.006 | INFO | Downloaded 34GiB to GPU in 2.42591s (14.0154GiB/s) | 20260507 17:54:15.144 | INFO | Analyzing chunk frames = [1446912,1447424) | 20260507 17:54:15.144 | INFO | MemInfo: { | 20260507 17:54:15.144 | INFO | "gpu" : " 17 GB", | 20260507 17:54:15.144 | INFO | "huge" : "", | 20260507 17:54:15.144 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:15.144 | INFO | "malloc" : " 14 GB", | 20260507 17:54:15.144 | INFO | "pinned" : " 29 GB" | 20260507 17:54:15.144 | INFO | } | 20260507 17:54:15.144 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:54:15.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:15.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.144 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:54:15.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.145 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 17:54:15.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:15.145 | INFO | Uploaded 36GiB to GPU in 2.56711s (14.0235GiB/s) | 20260507 17:54:15.145 | INFO | Downloaded 34GiB to GPU in 2.42291s (14.0327GiB/s) | 20260507 17:54:20.283 | INFO | Analyzing chunk frames = [1447424,1447936) | 20260507 17:54:20.283 | INFO | MemInfo: { | 20260507 17:54:20.283 | INFO | "gpu" : " 17 GB", | 20260507 17:54:20.283 | INFO | "huge" : "", | 20260507 17:54:20.283 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:20.283 | INFO | "malloc" : " 14 GB", | 20260507 17:54:20.283 | INFO | "pinned" : " 29 GB" | 20260507 17:54:20.284 | INFO | } | 20260507 17:54:20.284 | INFO | Perf report for the last 5.13928 seconds: | 20260507 17:54:20.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:20.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.284 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:54:20.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.284 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 17:54:20.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:20.285 | INFO | Uploaded 36GiB to GPU in 2.50266s (14.3847GiB/s) | 20260507 17:54:20.285 | INFO | Downloaded 34GiB to GPU in 2.35825s (14.4175GiB/s) | 20260507 17:54:25.423 | INFO | Analyzing chunk frames = [1447936,1448448) | 20260507 17:54:25.423 | INFO | MemInfo: { | 20260507 17:54:25.423 | INFO | "gpu" : " 17 GB", | 20260507 17:54:25.423 | INFO | "huge" : "", | 20260507 17:54:25.423 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:25.423 | INFO | "malloc" : " 14 GB", | 20260507 17:54:25.423 | INFO | "pinned" : " 29 GB" | 20260507 17:54:25.423 | INFO | } | 20260507 17:54:25.423 | INFO | Perf report for the last 5.1398 seconds: | 20260507 17:54:25.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:25.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.423 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260507 17:54:25.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.424 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 17:54:25.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:25.424 | INFO | Uploaded 36GiB to GPU in 2.4091s (14.9433GiB/s) | 20260507 17:54:25.424 | INFO | Downloaded 34GiB to GPU in 2.48441s (13.6854GiB/s) | 20260507 17:54:30.562 | INFO | Analyzing chunk frames = [1448448,1448960) | 20260507 17:54:30.562 | INFO | MemInfo: { | 20260507 17:54:30.562 | INFO | "gpu" : " 17 GB", | 20260507 17:54:30.562 | INFO | "huge" : "", | 20260507 17:54:30.562 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:30.562 | INFO | "malloc" : " 14 GB", | 20260507 17:54:30.562 | INFO | "pinned" : " 29 GB" | 20260507 17:54:30.562 | INFO | } | 20260507 17:54:30.562 | INFO | Perf report for the last 5.13857 seconds: | 20260507 17:54:30.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:30.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 17:54:30.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:54:30.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:30.562 | INFO | Uploaded 36GiB to GPU in 2.38405s (15.1003GiB/s) | 20260507 17:54:30.562 | INFO | Downloaded 34GiB to GPU in 2.49082s (13.6501GiB/s) | 20260507 17:54:35.701 | INFO | Analyzing chunk frames = [1448960,1449472) | 20260507 17:54:35.701 | INFO | MemInfo: { | 20260507 17:54:35.701 | INFO | "gpu" : " 17 GB", | 20260507 17:54:35.701 | INFO | "huge" : "", | 20260507 17:54:35.701 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:35.701 | INFO | "malloc" : " 14 GB", | 20260507 17:54:35.701 | INFO | "pinned" : " 29 GB" | 20260507 17:54:35.701 | INFO | } | 20260507 17:54:35.701 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:54:35.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:35.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.701 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:54:35.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.701 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:54:35.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:35.702 | INFO | Uploaded 36GiB to GPU in 2.47224s (14.5617GiB/s) | 20260507 17:54:35.702 | INFO | Downloaded 34GiB to GPU in 2.42705s (14.0088GiB/s) | 20260507 17:54:40.841 | INFO | Analyzing chunk frames = [1449472,1449984) | 20260507 17:54:40.841 | INFO | MemInfo: { | 20260507 17:54:40.841 | INFO | "gpu" : " 17 GB", | 20260507 17:54:40.841 | INFO | "huge" : "", | 20260507 17:54:40.841 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:40.841 | INFO | "malloc" : " 14 GB", | 20260507 17:54:40.841 | INFO | "pinned" : " 29 GB" | 20260507 17:54:40.841 | INFO | } | 20260507 17:54:40.841 | INFO | Perf report for the last 5.13988 seconds: | 20260507 17:54:40.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:40.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 17:54:40.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:54:40.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:40.841 | INFO | Uploaded 36GiB to GPU in 2.53332s (14.2106GiB/s) | 20260507 17:54:40.841 | INFO | Downloaded 34GiB to GPU in 2.51714s (13.5074GiB/s) | 20260507 17:54:40.842 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:54:40.842 | INFO | Total mem usage: 16.4 GiB | 20260507 17:54:40.842 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:54:40.842 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:54:40.842 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:54:40.842 | INFO | High Water Breakdown: | 20260507 17:54:40.842 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:54:40.842 | INFO | 40.7% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:54:40.842 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:54:40.842 | INFO | ------------------------------------------ | 20260507 17:54:40.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:54:40.842 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:54:40.842 | INFO | Total mem usage: 26.5 GiB | 20260507 17:54:40.842 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:54:40.842 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:54:40.842 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:54:40.842 | INFO | High Water Breakdown: | 20260507 17:54:40.842 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:54:40.842 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:54:40.842 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:54:40.842 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:54:40.842 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:54:40.842 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:54:40.842 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:54:40.842 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:54:40.842 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:54:40.842 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:54:40.842 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:54:40.842 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:54:40.842 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:54:40.842 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:54:40.842 | INFO | ------------------------------------------ | 20260507 17:54:40.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:54:40.842 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:54:40.842 | INFO | Total mem usage: 27.8 GiB | 20260507 17:54:40.842 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:54:40.842 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:54:40.842 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:54:40.842 | INFO | High Water Breakdown: | 20260507 17:54:40.842 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:54:40.842 | INFO | ------------------------------------------ | 20260507 17:54:40.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:54:40.842 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:54:40.842 | INFO | Total mem usage: 15.7 GiB | 20260507 17:54:40.842 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:54:40.842 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:54:40.842 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:54:40.842 | INFO | High Water Breakdown: | 20260507 17:54:40.842 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:54:40.842 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:54:40.842 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:54:40.842 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:54:40.842 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:54:40.842 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:54:40.842 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:54:40.842 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:54:40.842 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:54:40.842 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:54:40.842 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:54:40.842 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:54:40.842 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:54:40.842 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:54:40.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:54:40.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:54:40.842 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:54:40.842 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:54:40.843 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:54:40.843 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:54:40.843 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:54:40.843 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:54:40.843 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:54:40.843 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:54:40.843 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:54:40.843 | INFO | ------------------------------------------ | 20260507 17:54:40.843 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:54:40.843 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:54:40.843 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:54:40.843 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:54:40.843 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.43e+01 / 2.54e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 0.31%, AggregateStats -- 1.03e-01 / 1.07e-01, 1.20e-02 / 9.67e-03 | 20260507 17:54:40.843 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.50e+00, 6.16e-01 / 2.44e-01 | 20260507 17:54:40.843 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.27e-01 | 20260507 17:54:40.843 | INFO | PerfReport: 15.71%, DME -- 4.21e+01 / 4.50e+01, 4.12e+00 / 2.39e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.86e+00 / 2.22e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.53e+00 / 2.85e+00, 4.36e-01 / 2.60e-01 | 20260507 17:54:40.843 | INFO | PerfReport: 3.75%, Metrics -- 1.25e+00 / 1.43e+00, 3.15e-01 / 2.00e-01 | 20260507 17:54:40.843 | INFO | PerfReport: 1.20%, PulseSerializing -- 4.01e-01 / 4.28e-01, 6.51e-02 / 3.88e-02 | 20260507 17:54:40.843 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:54:40.843 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:54:40.843 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:54:40.843 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 23.99%, Upload -- 2.58e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 24.07%, Download -- 2.59e+01 / 0.00e+00, 8.62e+00 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 31.13%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 1.72%, LockUpload -- 1.85e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 2.39%, LockDownload -- 2.57e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 16.68%, LockCompute -- 1.79e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:54:40.843 | INFO | PerfReport: 0.02%, Misc -- 1.74e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 17:54:40.938 | INFO | Starting a round of Baz IO | 20260507 17:54:44.441 | INFO | Populating RTMetrics report | 20260507 17:54:44.442 | INFO | RTMetrics report populated | 20260507 17:54:44.442 | INFO | Opening tmp RTMetrics json file | 20260507 17:54:44.459 | INFO | Serializing RTMetrics to json | 20260507 17:54:44.460 | INFO | Serialization done, swapping tmp file now... | 20260507 17:54:44.460 | INFO | RTMetrics done | 20260507 17:54:44.520 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.582 seconds | 20260507 17:54:44.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.38% / 0.02% / 52.60% | 20260507 17:54:44.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.26299% / 1.98682e-05% / 3.97364e-05% | 20260507 17:54:44.520 | INFO | Baz IO overall: | 20260507 17:54:44.520 | INFO | BytesWritten : 643297087978 | 20260507 17:54:44.520 | INFO | NumEvents : 311791112494 | 20260507 17:54:45.980 | INFO | Analyzing chunk frames = [1449984,1450496) | 20260507 17:54:45.980 | INFO | MemInfo: { | 20260507 17:54:45.980 | INFO | "gpu" : " 17 GB", | 20260507 17:54:45.980 | INFO | "huge" : "", | 20260507 17:54:45.980 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:45.980 | INFO | "malloc" : " 14 GB", | 20260507 17:54:45.980 | INFO | "pinned" : " 29 GB" | 20260507 17:54:45.980 | INFO | } | 20260507 17:54:45.980 | INFO | Perf report for the last 5.13867 seconds: | 20260507 17:54:45.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:45.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:45.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:45.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:45.980 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 17:54:45.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:45.980 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260507 17:54:45.980 | INFO | PRE_HQ : 67.6%, 1.8, 63.2ms, 0 | 20260507 17:54:45.980 | INFO | BAZWRITER : 68.8%, 3.0, 106.2ms, 0 | 20260507 17:54:45.980 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:54:45.980 | INFO | Uploaded 36GiB to GPU in 2.52299s (14.2688GiB/s) | 20260507 17:54:45.980 | INFO | Downloaded 38GiB to GPU in 2.74014s (13.8679GiB/s) | 20260507 17:54:51.119 | INFO | Analyzing chunk frames = [1450496,1451008) | 20260507 17:54:51.119 | INFO | MemInfo: { | 20260507 17:54:51.119 | INFO | "gpu" : " 17 GB", | 20260507 17:54:51.119 | INFO | "huge" : "", | 20260507 17:54:51.119 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:51.119 | INFO | "malloc" : " 14 GB", | 20260507 17:54:51.119 | INFO | "pinned" : " 29 GB" | 20260507 17:54:51.119 | INFO | } | 20260507 17:54:51.119 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:54:51.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:51.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.119 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260507 17:54:51.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.119 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 17:54:51.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:51.120 | INFO | Uploaded 36GiB to GPU in 2.47892s (14.5225GiB/s) | 20260507 17:54:51.120 | INFO | Downloaded 34GiB to GPU in 2.45223s (13.8649GiB/s) | 20260507 17:54:56.258 | INFO | Analyzing chunk frames = [1451008,1451520) | 20260507 17:54:56.258 | INFO | MemInfo: { | 20260507 17:54:56.258 | INFO | "gpu" : " 17 GB", | 20260507 17:54:56.258 | INFO | "huge" : "", | 20260507 17:54:56.258 | INFO | "huge_shared" : " 29 GB", | 20260507 17:54:56.258 | INFO | "malloc" : " 14 GB", | 20260507 17:54:56.258 | INFO | "pinned" : " 29 GB" | 20260507 17:54:56.258 | INFO | } | 20260507 17:54:56.259 | INFO | Perf report for the last 5.13936 seconds: | 20260507 17:54:56.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:54:56.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260507 17:54:56.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:54:56.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:54:56.259 | INFO | Uploaded 36GiB to GPU in 2.41156s (14.9281GiB/s) | 20260507 17:54:56.259 | INFO | Downloaded 34GiB to GPU in 2.35522s (14.436GiB/s) | 20260507 17:55:01.398 | INFO | Analyzing chunk frames = [1451520,1452032) | 20260507 17:55:01.398 | INFO | MemInfo: { | 20260507 17:55:01.398 | INFO | "gpu" : " 17 GB", | 20260507 17:55:01.398 | INFO | "huge" : "", | 20260507 17:55:01.398 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:01.398 | INFO | "malloc" : " 14 GB", | 20260507 17:55:01.398 | INFO | "pinned" : " 29 GB" | 20260507 17:55:01.398 | INFO | } | 20260507 17:55:01.398 | INFO | Perf report for the last 5.13914 seconds: | 20260507 17:55:01.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:01.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.398 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 17:55:01.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.398 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:55:01.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:01.399 | INFO | Uploaded 36GiB to GPU in 2.43013s (14.814GiB/s) | 20260507 17:55:01.399 | INFO | Downloaded 34GiB to GPU in 2.35554s (14.434GiB/s) | 20260507 17:55:06.537 | INFO | Analyzing chunk frames = [1452032,1452544) | 20260507 17:55:06.537 | INFO | MemInfo: { | 20260507 17:55:06.538 | INFO | "gpu" : " 17 GB", | 20260507 17:55:06.538 | INFO | "huge" : "", | 20260507 17:55:06.538 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:06.538 | INFO | "malloc" : " 14 GB", | 20260507 17:55:06.538 | INFO | "pinned" : " 29 GB" | 20260507 17:55:06.538 | INFO | } | 20260507 17:55:06.538 | INFO | Perf report for the last 5.13969 seconds: | 20260507 17:55:06.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:06.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 17:55:06.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:55:06.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:06.538 | INFO | Uploaded 36GiB to GPU in 2.34173s (15.3732GiB/s) | 20260507 17:55:06.538 | INFO | Downloaded 34GiB to GPU in 2.48174s (13.7001GiB/s) | 20260507 17:55:11.676 | INFO | Analyzing chunk frames = [1452544,1453056) | 20260507 17:55:11.676 | INFO | MemInfo: { | 20260507 17:55:11.676 | INFO | "gpu" : " 17 GB", | 20260507 17:55:11.676 | INFO | "huge" : "", | 20260507 17:55:11.677 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:11.677 | INFO | "malloc" : " 14 GB", | 20260507 17:55:11.677 | INFO | "pinned" : " 29 GB" | 20260507 17:55:11.677 | INFO | } | 20260507 17:55:11.677 | INFO | Perf report for the last 5.13908 seconds: | 20260507 17:55:11.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:11.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:55:11.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:55:11.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:11.677 | INFO | Uploaded 36GiB to GPU in 2.32038s (15.5147GiB/s) | 20260507 17:55:11.678 | INFO | Downloaded 34GiB to GPU in 2.50897s (13.5514GiB/s) | 20260507 17:55:16.816 | INFO | Analyzing chunk frames = [1453056,1453568) | 20260507 17:55:16.816 | INFO | MemInfo: { | 20260507 17:55:16.816 | INFO | "gpu" : " 17 GB", | 20260507 17:55:16.816 | INFO | "huge" : "", | 20260507 17:55:16.816 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:16.816 | INFO | "malloc" : " 14 GB", | 20260507 17:55:16.816 | INFO | "pinned" : " 29 GB" | 20260507 17:55:16.816 | INFO | } | 20260507 17:55:16.816 | INFO | Perf report for the last 5.13943 seconds: | 20260507 17:55:16.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:16.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 17:55:16.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:55:16.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:16.816 | INFO | Uploaded 36GiB to GPU in 2.42654s (14.836GiB/s) | 20260507 17:55:16.816 | INFO | Downloaded 34GiB to GPU in 2.44194s (13.9234GiB/s) | 20260507 17:55:21.955 | INFO | Analyzing chunk frames = [1453568,1454080) | 20260507 17:55:21.955 | INFO | MemInfo: { | 20260507 17:55:21.955 | INFO | "gpu" : " 17 GB", | 20260507 17:55:21.955 | INFO | "huge" : "", | 20260507 17:55:21.955 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:21.955 | INFO | "malloc" : " 14 GB", | 20260507 17:55:21.955 | INFO | "pinned" : " 29 GB" | 20260507 17:55:21.955 | INFO | } | 20260507 17:55:21.955 | INFO | Perf report for the last 5.13913 seconds: | 20260507 17:55:21.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:21.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.955 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:55:21.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.956 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 17:55:21.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:21.956 | INFO | Uploaded 36GiB to GPU in 2.4578s (14.6473GiB/s) | 20260507 17:55:21.956 | INFO | Downloaded 34GiB to GPU in 2.52443s (13.4684GiB/s) | 20260507 17:55:22.053 | INFO | Starting a round of Baz IO | 20260507 17:55:25.560 | INFO | Populating RTMetrics report | 20260507 17:55:25.561 | INFO | RTMetrics report populated | 20260507 17:55:25.561 | INFO | Opening tmp RTMetrics json file | 20260507 17:55:25.575 | INFO | Serializing RTMetrics to json | 20260507 17:55:25.578 | INFO | Serialization done, swapping tmp file now... | 20260507 17:55:25.578 | INFO | RTMetrics done | 20260507 17:55:25.639 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.586 seconds | 20260507 17:55:25.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.36% / 0.02% / 52.62% | 20260507 17:55:25.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.26153% / 1.98682e-05% / 3.97364e-05% | 20260507 17:55:25.639 | INFO | Baz IO overall: | 20260507 17:55:25.639 | INFO | BytesWritten : 645230918724 | 20260507 17:55:25.639 | INFO | NumEvents : 312730697672 | 20260507 17:55:27.095 | INFO | Analyzing chunk frames = [1454080,1454592) | 20260507 17:55:27.095 | INFO | MemInfo: { | 20260507 17:55:27.095 | INFO | "gpu" : " 17 GB", | 20260507 17:55:27.095 | INFO | "huge" : "", | 20260507 17:55:27.095 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:27.095 | INFO | "malloc" : " 14 GB", | 20260507 17:55:27.095 | INFO | "pinned" : " 29 GB" | 20260507 17:55:27.095 | INFO | } | 20260507 17:55:27.095 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:55:27.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:27.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:27.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:27.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:27.095 | INFO | ANALYSIS : 69.4%, 3.0, 108.4ms, 0 | 20260507 17:55:27.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:27.095 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 17:55:27.095 | INFO | PRE_HQ : 66.8%, 1.8, 61.1ms, 0 | 20260507 17:55:27.095 | INFO | BAZWRITER : 68.8%, 2.9, 102.9ms, 0 | 20260507 17:55:27.095 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:55:27.095 | INFO | Uploaded 36GiB to GPU in 2.50514s (14.3705GiB/s) | 20260507 17:55:27.095 | INFO | Downloaded 38GiB to GPU in 2.71232s (14.0101GiB/s) | 20260507 17:55:32.234 | INFO | Analyzing chunk frames = [1454592,1455104) | 20260507 17:55:32.234 | INFO | MemInfo: { | 20260507 17:55:32.234 | INFO | "gpu" : " 17 GB", | 20260507 17:55:32.234 | INFO | "huge" : "", | 20260507 17:55:32.234 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:32.234 | INFO | "malloc" : " 14 GB", | 20260507 17:55:32.234 | INFO | "pinned" : " 29 GB" | 20260507 17:55:32.234 | INFO | } | 20260507 17:55:32.234 | INFO | Perf report for the last 5.1396 seconds: | 20260507 17:55:32.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:32.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.234 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:55:32.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.234 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 17:55:32.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:32.235 | INFO | Uploaded 36GiB to GPU in 2.50836s (14.352GiB/s) | 20260507 17:55:32.235 | INFO | Downloaded 34GiB to GPU in 2.44157s (13.9254GiB/s) | 20260507 17:55:37.373 | INFO | Analyzing chunk frames = [1455104,1455616) | 20260507 17:55:37.373 | INFO | MemInfo: { | 20260507 17:55:37.373 | INFO | "gpu" : " 17 GB", | 20260507 17:55:37.373 | INFO | "huge" : "", | 20260507 17:55:37.373 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:37.374 | INFO | "malloc" : " 14 GB", | 20260507 17:55:37.374 | INFO | "pinned" : " 29 GB" | 20260507 17:55:37.374 | INFO | } | 20260507 17:55:37.374 | INFO | Perf report for the last 5.13918 seconds: | 20260507 17:55:37.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:37.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 17:55:37.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:55:37.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:37.374 | INFO | Uploaded 36GiB to GPU in 2.56825s (14.0173GiB/s) | 20260507 17:55:37.374 | INFO | Downloaded 34GiB to GPU in 2.42554s (14.0175GiB/s) | 20260507 17:55:42.513 | INFO | Analyzing chunk frames = [1455616,1456128) | 20260507 17:55:42.513 | INFO | MemInfo: { | 20260507 17:55:42.513 | INFO | "gpu" : " 17 GB", | 20260507 17:55:42.513 | INFO | "huge" : "", | 20260507 17:55:42.513 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:42.513 | INFO | "malloc" : " 14 GB", | 20260507 17:55:42.513 | INFO | "pinned" : " 29 GB" | 20260507 17:55:42.513 | INFO | } | 20260507 17:55:42.513 | INFO | Perf report for the last 5.13957 seconds: | 20260507 17:55:42.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:42.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 17:55:42.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:55:42.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:42.513 | INFO | Uploaded 36GiB to GPU in 2.48977s (14.4591GiB/s) | 20260507 17:55:42.513 | INFO | Downloaded 34GiB to GPU in 2.34888s (14.475GiB/s) | 20260507 17:55:47.653 | INFO | Analyzing chunk frames = [1456128,1456640) | 20260507 17:55:47.653 | INFO | MemInfo: { | 20260507 17:55:47.653 | INFO | "gpu" : " 17 GB", | 20260507 17:55:47.653 | INFO | "huge" : "", | 20260507 17:55:47.653 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:47.653 | INFO | "malloc" : " 14 GB", | 20260507 17:55:47.653 | INFO | "pinned" : " 29 GB" | 20260507 17:55:47.653 | INFO | } | 20260507 17:55:47.653 | INFO | Perf report for the last 5.14 seconds: | 20260507 17:55:47.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:47.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.653 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 17:55:47.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.653 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 17:55:47.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:47.654 | INFO | Uploaded 36GiB to GPU in 2.38473s (15.096GiB/s) | 20260507 17:55:47.654 | INFO | Downloaded 34GiB to GPU in 2.46583s (13.7885GiB/s) | 20260507 17:55:52.791 | INFO | Analyzing chunk frames = [1456640,1457152) | 20260507 17:55:52.791 | INFO | MemInfo: { | 20260507 17:55:52.791 | INFO | "gpu" : " 17 GB", | 20260507 17:55:52.791 | INFO | "huge" : "", | 20260507 17:55:52.791 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:52.791 | INFO | "malloc" : " 14 GB", | 20260507 17:55:52.791 | INFO | "pinned" : " 29 GB" | 20260507 17:55:52.791 | INFO | } | 20260507 17:55:52.791 | INFO | Perf report for the last 5.13827 seconds: | 20260507 17:55:52.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:52.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:55:52.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:55:52.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:52.792 | INFO | Uploaded 36GiB to GPU in 2.4323s (14.8008GiB/s) | 20260507 17:55:52.792 | INFO | Downloaded 34GiB to GPU in 2.47853s (13.7178GiB/s) | 20260507 17:55:57.931 | INFO | Analyzing chunk frames = [1457152,1457664) | 20260507 17:55:57.931 | INFO | MemInfo: { | 20260507 17:55:57.931 | INFO | "gpu" : " 17 GB", | 20260507 17:55:57.931 | INFO | "huge" : "", | 20260507 17:55:57.931 | INFO | "huge_shared" : " 29 GB", | 20260507 17:55:57.931 | INFO | "malloc" : " 14 GB", | 20260507 17:55:57.931 | INFO | "pinned" : " 29 GB" | 20260507 17:55:57.931 | INFO | } | 20260507 17:55:57.931 | INFO | Perf report for the last 5.13936 seconds: | 20260507 17:55:57.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:55:57.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:55:57.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 17:55:57.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:55:57.931 | INFO | Uploaded 36GiB to GPU in 2.51774s (14.2986GiB/s) | 20260507 17:55:57.931 | INFO | Downloaded 34GiB to GPU in 2.4203s (14.0479GiB/s) | 20260507 17:56:03.070 | INFO | Analyzing chunk frames = [1457664,1458176) | 20260507 17:56:03.070 | INFO | MemInfo: { | 20260507 17:56:03.070 | INFO | "gpu" : " 17 GB", | 20260507 17:56:03.070 | INFO | "huge" : "", | 20260507 17:56:03.070 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:03.070 | INFO | "malloc" : " 14 GB", | 20260507 17:56:03.070 | INFO | "pinned" : " 29 GB" | 20260507 17:56:03.070 | INFO | } | 20260507 17:56:03.070 | INFO | Perf report for the last 5.13949 seconds: | 20260507 17:56:03.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:03.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.070 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 17:56:03.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.071 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 17:56:03.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:03.071 | INFO | Uploaded 36GiB to GPU in 2.48764s (14.4716GiB/s) | 20260507 17:56:03.071 | INFO | Downloaded 34GiB to GPU in 2.40822s (14.1183GiB/s) | 20260507 17:56:03.071 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:56:03.071 | INFO | Total mem usage: 16.4 GiB | 20260507 17:56:03.071 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:56:03.071 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.5%) | 20260507 17:56:03.071 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:56:03.071 | INFO | High Water Breakdown: | 20260507 17:56:03.071 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:56:03.071 | INFO | 40.7% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:56:03.071 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:56:03.071 | INFO | ------------------------------------------ | 20260507 17:56:03.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:56:03.071 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:56:03.071 | INFO | Total mem usage: 26.5 GiB | 20260507 17:56:03.071 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:56:03.071 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:56:03.071 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:56:03.071 | INFO | High Water Breakdown: | 20260507 17:56:03.071 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:56:03.071 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:56:03.071 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:56:03.071 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:56:03.071 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:56:03.071 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:56:03.071 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:56:03.071 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:56:03.071 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:56:03.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:56:03.072 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:56:03.072 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:56:03.072 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:56:03.072 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:56:03.072 | INFO | ------------------------------------------ | 20260507 17:56:03.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:56:03.072 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:56:03.072 | INFO | Total mem usage: 27.8 GiB | 20260507 17:56:03.072 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:56:03.072 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:56:03.072 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:56:03.072 | INFO | High Water Breakdown: | 20260507 17:56:03.072 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:56:03.072 | INFO | ------------------------------------------ | 20260507 17:56:03.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:56:03.072 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:56:03.072 | INFO | Total mem usage: 15.7 GiB | 20260507 17:56:03.072 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:56:03.072 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:56:03.072 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:56:03.072 | INFO | High Water Breakdown: | 20260507 17:56:03.072 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:56:03.072 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:56:03.072 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:56:03.072 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:56:03.072 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:56:03.072 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:56:03.072 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:56:03.072 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:56:03.072 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:56:03.072 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:56:03.072 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:56:03.072 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:56:03.072 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:56:03.072 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:56:03.072 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:56:03.072 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:56:03.072 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:56:03.072 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:56:03.072 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:56:03.072 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:56:03.072 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:56:03.072 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:56:03.072 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:56:03.072 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:56:03.072 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:56:03.072 | INFO | ------------------------------------------ | 20260507 17:56:03.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:56:03.072 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:56:03.072 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:56:03.072 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:56:03.072 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.43e+01 / 2.60e+00 | 20260507 17:56:03.072 | INFO | PerfReport: 0.29%, AggregateStats -- 9.82e-02 / 1.04e-01, 1.10e-02 / 6.59e-03 | 20260507 17:56:03.072 | INFO | PerfReport: 12.06%, Baseline -- 4.04e+00 / 4.50e+00, 6.17e-01 / 2.35e-01 | 20260507 17:56:03.072 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 9.78e-02 | 20260507 17:56:03.072 | INFO | PerfReport: 15.72%, DME -- 4.21e+01 / 4.50e+01, 4.28e+00 / 2.56e+00 | 20260507 17:56:03.072 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.89e+00 / 1.95e+00 | 20260507 17:56:03.072 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.31e-01 / 2.43e-01 | 20260507 17:56:03.072 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.42e+00, 3.16e-01 / 1.77e-01 | 20260507 17:56:03.072 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.20e-01, 6.52e-02 / 3.97e-02 | 20260507 17:56:03.072 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:56:03.072 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:56:03.072 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:56:03.073 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 23.84%, Upload -- 2.56e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 24.01%, Download -- 2.58e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 31.21%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 1.66%, LockUpload -- 1.78e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 2.45%, LockDownload -- 2.63e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 16.80%, LockCompute -- 1.80e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 17:56:03.073 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260507 17:56:03.182 | INFO | Starting a round of Baz IO | 20260507 17:56:06.670 | INFO | Populating RTMetrics report | 20260507 17:56:06.670 | INFO | RTMetrics report populated | 20260507 17:56:06.670 | INFO | Opening tmp RTMetrics json file | 20260507 17:56:06.684 | INFO | Serializing RTMetrics to json | 20260507 17:56:06.687 | INFO | Serialization done, swapping tmp file now... | 20260507 17:56:06.688 | INFO | RTMetrics done | 20260507 17:56:06.756 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.573 seconds | 20260507 17:56:06.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.35% / 0.02% / 52.64% | 20260507 17:56:06.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.25203% / 1.98682e-05% / 5.16574e-05% | 20260507 17:56:06.756 | INFO | Baz IO overall: | 20260507 17:56:06.756 | INFO | BytesWritten : 647163423506 | 20260507 17:56:06.756 | INFO | NumEvents : 313669260524 | 20260507 17:56:08.209 | INFO | Analyzing chunk frames = [1458176,1458688) | 20260507 17:56:08.209 | INFO | MemInfo: { | 20260507 17:56:08.209 | INFO | "gpu" : " 17 GB", | 20260507 17:56:08.209 | INFO | "huge" : "", | 20260507 17:56:08.209 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:08.209 | INFO | "malloc" : " 14 GB", | 20260507 17:56:08.209 | INFO | "pinned" : " 29 GB" | 20260507 17:56:08.209 | INFO | } | 20260507 17:56:08.209 | INFO | Perf report for the last 5.13898 seconds: | 20260507 17:56:08.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:08.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:08.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:08.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:08.210 | INFO | ANALYSIS : 69.6%, 2.9, 106.7ms, 0 | 20260507 17:56:08.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:08.210 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 17:56:08.210 | INFO | PRE_HQ : 67.0%, 1.9, 65.4ms, 0 | 20260507 17:56:08.210 | INFO | BAZWRITER : 68.6%, 3.1, 109.0ms, 0 | 20260507 17:56:08.210 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:56:08.210 | INFO | Uploaded 36GiB to GPU in 2.55863s (14.07GiB/s) | 20260507 17:56:08.210 | INFO | Downloaded 38GiB to GPU in 2.6984s (14.0824GiB/s) | 20260507 17:56:13.349 | INFO | Analyzing chunk frames = [1458688,1459200) | 20260507 17:56:13.349 | INFO | MemInfo: { | 20260507 17:56:13.349 | INFO | "gpu" : " 17 GB", | 20260507 17:56:13.349 | INFO | "huge" : "", | 20260507 17:56:13.349 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:13.349 | INFO | "malloc" : " 14 GB", | 20260507 17:56:13.349 | INFO | "pinned" : " 29 GB" | 20260507 17:56:13.349 | INFO | } | 20260507 17:56:13.349 | INFO | Perf report for the last 5.13959 seconds: | 20260507 17:56:13.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:13.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.349 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:56:13.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.349 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 17:56:13.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:13.350 | INFO | Uploaded 36GiB to GPU in 2.52615s (14.2509GiB/s) | 20260507 17:56:13.350 | INFO | Downloaded 34GiB to GPU in 2.43279s (13.9757GiB/s) | 20260507 17:56:18.488 | INFO | Analyzing chunk frames = [1459200,1459712) | 20260507 17:56:18.488 | INFO | MemInfo: { | 20260507 17:56:18.488 | INFO | "gpu" : " 17 GB", | 20260507 17:56:18.488 | INFO | "huge" : "", | 20260507 17:56:18.488 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:18.488 | INFO | "malloc" : " 14 GB", | 20260507 17:56:18.488 | INFO | "pinned" : " 29 GB" | 20260507 17:56:18.488 | INFO | } | 20260507 17:56:18.488 | INFO | Perf report for the last 5.13927 seconds: | 20260507 17:56:18.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:18.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.488 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 17:56:18.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.489 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:56:18.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:18.489 | INFO | Uploaded 36GiB to GPU in 2.56645s (14.0272GiB/s) | 20260507 17:56:18.489 | INFO | Downloaded 34GiB to GPU in 2.41796s (14.0615GiB/s) | 20260507 17:56:23.628 | INFO | Analyzing chunk frames = [1459712,1460224) | 20260507 17:56:23.628 | INFO | MemInfo: { | 20260507 17:56:23.628 | INFO | "gpu" : " 17 GB", | 20260507 17:56:23.628 | INFO | "huge" : "", | 20260507 17:56:23.628 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:23.628 | INFO | "malloc" : " 14 GB", | 20260507 17:56:23.628 | INFO | "pinned" : " 29 GB" | 20260507 17:56:23.628 | INFO | } | 20260507 17:56:23.628 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:56:23.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:23.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 17:56:23.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:56:23.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:23.628 | INFO | Uploaded 36GiB to GPU in 2.49249s (14.4434GiB/s) | 20260507 17:56:23.628 | INFO | Downloaded 34GiB to GPU in 2.35555s (14.434GiB/s) | 20260507 17:56:28.767 | INFO | Analyzing chunk frames = [1460224,1460736) | 20260507 17:56:28.767 | INFO | MemInfo: { | 20260507 17:56:28.767 | INFO | "gpu" : " 17 GB", | 20260507 17:56:28.767 | INFO | "huge" : "", | 20260507 17:56:28.767 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:28.767 | INFO | "malloc" : " 14 GB", | 20260507 17:56:28.767 | INFO | "pinned" : " 29 GB" | 20260507 17:56:28.767 | INFO | } | 20260507 17:56:28.767 | INFO | Perf report for the last 5.13924 seconds: | 20260507 17:56:28.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:28.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 17:56:28.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:56:28.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:28.767 | INFO | Uploaded 36GiB to GPU in 2.36256s (15.2377GiB/s) | 20260507 17:56:28.767 | INFO | Downloaded 34GiB to GPU in 2.45377s (13.8562GiB/s) | 20260507 17:56:33.906 | INFO | Analyzing chunk frames = [1460736,1461248) | 20260507 17:56:33.906 | INFO | MemInfo: { | 20260507 17:56:33.906 | INFO | "gpu" : " 17 GB", | 20260507 17:56:33.906 | INFO | "huge" : "", | 20260507 17:56:33.906 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:33.906 | INFO | "malloc" : " 14 GB", | 20260507 17:56:33.906 | INFO | "pinned" : " 29 GB" | 20260507 17:56:33.907 | INFO | } | 20260507 17:56:33.907 | INFO | Perf report for the last 5.13978 seconds: | 20260507 17:56:33.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:33.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:56:33.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 17:56:33.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:33.907 | INFO | Uploaded 36GiB to GPU in 2.35945s (15.2578GiB/s) | 20260507 17:56:33.907 | INFO | Downloaded 34GiB to GPU in 2.47206s (13.7537GiB/s) | 20260507 17:56:39.045 | INFO | Analyzing chunk frames = [1461248,1461760) | 20260507 17:56:39.045 | INFO | MemInfo: { | 20260507 17:56:39.045 | INFO | "gpu" : " 17 GB", | 20260507 17:56:39.045 | INFO | "huge" : "", | 20260507 17:56:39.045 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:39.046 | INFO | "malloc" : " 14 GB", | 20260507 17:56:39.046 | INFO | "pinned" : " 29 GB" | 20260507 17:56:39.046 | INFO | } | 20260507 17:56:39.046 | INFO | Perf report for the last 5.13901 seconds: | 20260507 17:56:39.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:39.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.046 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 17:56:39.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.046 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 17:56:39.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:39.047 | INFO | Uploaded 36GiB to GPU in 2.47013s (14.5741GiB/s) | 20260507 17:56:39.047 | INFO | Downloaded 34GiB to GPU in 2.43576s (13.9587GiB/s) | 20260507 17:56:44.185 | INFO | Analyzing chunk frames = [1461760,1462272) | 20260507 17:56:44.185 | INFO | MemInfo: { | 20260507 17:56:44.185 | INFO | "gpu" : " 17 GB", | 20260507 17:56:44.185 | INFO | "huge" : "", | 20260507 17:56:44.185 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:44.185 | INFO | "malloc" : " 14 GB", | 20260507 17:56:44.185 | INFO | "pinned" : " 29 GB" | 20260507 17:56:44.185 | INFO | } | 20260507 17:56:44.185 | INFO | Perf report for the last 5.13961 seconds: | 20260507 17:56:44.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:44.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.186 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 17:56:44.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.186 | INFO | PRE_HQ : 37.6%, 1.0, 19.6ms, 0 | 20260507 17:56:44.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:44.186 | INFO | Uploaded 36GiB to GPU in 2.47593s (14.54GiB/s) | 20260507 17:56:44.186 | INFO | Downloaded 34GiB to GPU in 2.42936s (13.9955GiB/s) | 20260507 17:56:44.301 | INFO | Starting a round of Baz IO | 20260507 17:56:47.787 | INFO | Populating RTMetrics report | 20260507 17:56:47.787 | INFO | RTMetrics report populated | 20260507 17:56:47.787 | INFO | Opening tmp RTMetrics json file | 20260507 17:56:47.793 | INFO | Serializing RTMetrics to json | 20260507 17:56:47.806 | INFO | Serialization done, swapping tmp file now... | 20260507 17:56:47.807 | INFO | RTMetrics done | 20260507 17:56:47.865 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.564 seconds | 20260507 17:56:47.865 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.33% / 0.02% / 52.65% | 20260507 17:56:47.865 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.24434% / 2.38419e-05% / 3.57628e-05% | 20260507 17:56:47.865 | INFO | Baz IO overall: | 20260507 17:56:47.865 | INFO | BytesWritten : 649096923808 | 20260507 17:56:47.865 | INFO | NumEvents : 314608715167 | 20260507 17:56:49.325 | INFO | Analyzing chunk frames = [1462272,1462784) | 20260507 17:56:49.325 | INFO | MemInfo: { | 20260507 17:56:49.325 | INFO | "gpu" : " 17 GB", | 20260507 17:56:49.325 | INFO | "huge" : "", | 20260507 17:56:49.325 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:49.325 | INFO | "malloc" : " 14 GB", | 20260507 17:56:49.325 | INFO | "pinned" : " 29 GB" | 20260507 17:56:49.325 | INFO | } | 20260507 17:56:49.325 | INFO | Perf report for the last 5.13939 seconds: | 20260507 17:56:49.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:49.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:49.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:49.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:49.325 | INFO | ANALYSIS : 69.3%, 2.9, 106.9ms, 0 | 20260507 17:56:49.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:49.325 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 17:56:49.325 | INFO | PRE_HQ : 66.8%, 1.9, 65.0ms, 0 | 20260507 17:56:49.325 | INFO | BAZWRITER : 68.5%, 3.0, 105.9ms, 0 | 20260507 17:56:49.325 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 17:56:49.325 | INFO | Uploaded 36GiB to GPU in 2.52631s (14.25GiB/s) | 20260507 17:56:49.325 | INFO | Downloaded 38GiB to GPU in 2.68698s (14.1423GiB/s) | 20260507 17:56:54.464 | INFO | Analyzing chunk frames = [1462784,1463296) | 20260507 17:56:54.464 | INFO | MemInfo: { | 20260507 17:56:54.464 | INFO | "gpu" : " 17 GB", | 20260507 17:56:54.464 | INFO | "huge" : "", | 20260507 17:56:54.464 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:54.464 | INFO | "malloc" : " 14 GB", | 20260507 17:56:54.464 | INFO | "pinned" : " 29 GB" | 20260507 17:56:54.464 | INFO | } | 20260507 17:56:54.464 | INFO | Perf report for the last 5.13925 seconds: | 20260507 17:56:54.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:54.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 17:56:54.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 17:56:54.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:54.464 | INFO | Uploaded 36GiB to GPU in 2.5082s (14.3529GiB/s) | 20260507 17:56:54.465 | INFO | Downloaded 34GiB to GPU in 2.44345s (13.9147GiB/s) | 20260507 17:56:59.603 | INFO | Analyzing chunk frames = [1463296,1463808) | 20260507 17:56:59.603 | INFO | MemInfo: { | 20260507 17:56:59.603 | INFO | "gpu" : " 17 GB", | 20260507 17:56:59.603 | INFO | "huge" : "", | 20260507 17:56:59.603 | INFO | "huge_shared" : " 29 GB", | 20260507 17:56:59.603 | INFO | "malloc" : " 14 GB", | 20260507 17:56:59.603 | INFO | "pinned" : " 29 GB" | 20260507 17:56:59.603 | INFO | } | 20260507 17:56:59.603 | INFO | Perf report for the last 5.13931 seconds: | 20260507 17:56:59.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:56:59.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.603 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 17:56:59.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.604 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:56:59.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:56:59.604 | INFO | Uploaded 36GiB to GPU in 2.55801s (14.0734GiB/s) | 20260507 17:56:59.604 | INFO | Downloaded 34GiB to GPU in 2.43091s (13.9865GiB/s) | 20260507 17:57:04.742 | INFO | Analyzing chunk frames = [1463808,1464320) | 20260507 17:57:04.742 | INFO | MemInfo: { | 20260507 17:57:04.742 | INFO | "gpu" : " 17 GB", | 20260507 17:57:04.742 | INFO | "huge" : "", | 20260507 17:57:04.742 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:04.742 | INFO | "malloc" : " 14 GB", | 20260507 17:57:04.742 | INFO | "pinned" : " 29 GB" | 20260507 17:57:04.742 | INFO | } | 20260507 17:57:04.742 | INFO | Perf report for the last 5.13892 seconds: | 20260507 17:57:04.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:04.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.742 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 17:57:04.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.742 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:57:04.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:04.743 | INFO | Uploaded 36GiB to GPU in 2.47803s (14.5277GiB/s) | 20260507 17:57:04.743 | INFO | Downloaded 34GiB to GPU in 2.34366s (14.5072GiB/s) | 20260507 17:57:09.882 | INFO | Analyzing chunk frames = [1464320,1464832) | 20260507 17:57:09.882 | INFO | MemInfo: { | 20260507 17:57:09.882 | INFO | "gpu" : " 17 GB", | 20260507 17:57:09.882 | INFO | "huge" : "", | 20260507 17:57:09.882 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:09.882 | INFO | "malloc" : " 14 GB", | 20260507 17:57:09.882 | INFO | "pinned" : " 29 GB" | 20260507 17:57:09.882 | INFO | } | 20260507 17:57:09.882 | INFO | Perf report for the last 5.1394 seconds: | 20260507 17:57:09.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:09.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 17:57:09.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:57:09.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:09.882 | INFO | Uploaded 36GiB to GPU in 2.3669s (15.2098GiB/s) | 20260507 17:57:09.882 | INFO | Downloaded 34GiB to GPU in 2.46833s (13.7745GiB/s) | 20260507 17:57:15.021 | INFO | Analyzing chunk frames = [1464832,1465344) | 20260507 17:57:15.021 | INFO | MemInfo: { | 20260507 17:57:15.021 | INFO | "gpu" : " 17 GB", | 20260507 17:57:15.021 | INFO | "huge" : "", | 20260507 17:57:15.021 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:15.021 | INFO | "malloc" : " 14 GB", | 20260507 17:57:15.021 | INFO | "pinned" : " 29 GB" | 20260507 17:57:15.021 | INFO | } | 20260507 17:57:15.021 | INFO | Perf report for the last 5.13969 seconds: | 20260507 17:57:15.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:15.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.022 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:57:15.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.022 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 17:57:15.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:15.022 | INFO | Uploaded 36GiB to GPU in 2.39438s (15.0352GiB/s) | 20260507 17:57:15.022 | INFO | Downloaded 34GiB to GPU in 2.452s (13.8662GiB/s) | 20260507 17:57:20.161 | INFO | Analyzing chunk frames = [1465344,1465856) | 20260507 17:57:20.161 | INFO | MemInfo: { | 20260507 17:57:20.161 | INFO | "gpu" : " 17 GB", | 20260507 17:57:20.161 | INFO | "huge" : "", | 20260507 17:57:20.161 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:20.161 | INFO | "malloc" : " 14 GB", | 20260507 17:57:20.161 | INFO | "pinned" : " 29 GB" | 20260507 17:57:20.161 | INFO | } | 20260507 17:57:20.161 | INFO | Perf report for the last 5.13953 seconds: | 20260507 17:57:20.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:20.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:57:20.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 17:57:20.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:20.161 | INFO | Uploaded 36GiB to GPU in 2.50346s (14.3801GiB/s) | 20260507 17:57:20.162 | INFO | Downloaded 34GiB to GPU in 2.41892s (14.0559GiB/s) | 20260507 17:57:25.300 | INFO | Analyzing chunk frames = [1465856,1466368) | 20260507 17:57:25.300 | INFO | MemInfo: { | 20260507 17:57:25.300 | INFO | "gpu" : " 17 GB", | 20260507 17:57:25.300 | INFO | "huge" : "", | 20260507 17:57:25.300 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:25.300 | INFO | "malloc" : " 14 GB", | 20260507 17:57:25.300 | INFO | "pinned" : " 29 GB" | 20260507 17:57:25.300 | INFO | } | 20260507 17:57:25.300 | INFO | Perf report for the last 5.13886 seconds: | 20260507 17:57:25.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:25.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 17:57:25.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:57:25.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:25.300 | INFO | Uploaded 36GiB to GPU in 2.54363s (14.153GiB/s) | 20260507 17:57:25.300 | INFO | Downloaded 34GiB to GPU in 2.4914s (13.6469GiB/s) | 20260507 17:57:25.300 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:57:25.300 | INFO | Total mem usage: 16.4 GiB | 20260507 17:57:25.300 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:57:25.300 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:57:25.300 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:57:25.300 | INFO | High Water Breakdown: | 20260507 17:57:25.300 | INFO | 59.1% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:57:25.301 | INFO | 40.7% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:57:25.301 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:57:25.301 | INFO | ------------------------------------------ | 20260507 17:57:25.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:57:25.301 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:57:25.301 | INFO | Total mem usage: 26.5 GiB | 20260507 17:57:25.301 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:57:25.301 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 17:57:25.301 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 17:57:25.301 | INFO | High Water Breakdown: | 20260507 17:57:25.301 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:57:25.301 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:57:25.301 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:57:25.301 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:57:25.301 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:57:25.301 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:57:25.301 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:57:25.301 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:57:25.301 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:57:25.301 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:57:25.301 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:57:25.301 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:57:25.301 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:57:25.301 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:57:25.301 | INFO | ------------------------------------------ | 20260507 17:57:25.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:57:25.301 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:57:25.301 | INFO | Total mem usage: 27.8 GiB | 20260507 17:57:25.301 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:57:25.301 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:57:25.301 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:57:25.301 | INFO | High Water Breakdown: | 20260507 17:57:25.301 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:57:25.301 | INFO | ------------------------------------------ | 20260507 17:57:25.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:57:25.301 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:57:25.301 | INFO | Total mem usage: 15.7 GiB | 20260507 17:57:25.301 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:57:25.301 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:57:25.301 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:57:25.301 | INFO | High Water Breakdown: | 20260507 17:57:25.301 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:57:25.301 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:57:25.301 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:57:25.301 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:57:25.301 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:57:25.301 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:57:25.301 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:57:25.301 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:57:25.301 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:57:25.301 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:57:25.301 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:57:25.301 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:57:25.301 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:57:25.301 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:57:25.301 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:57:25.301 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:57:25.301 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:57:25.301 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:57:25.301 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:57:25.301 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:57:25.301 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:57:25.301 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:57:25.301 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:57:25.301 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:57:25.301 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:57:25.301 | INFO | ------------------------------------------ | 20260507 17:57:25.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:57:25.301 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:57:25.301 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:57:25.301 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:57:25.301 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.64e+01, 1.42e+01 / 2.39e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 0.29%, AggregateStats -- 9.84e-02 / 1.03e-01, 1.06e-02 / 5.09e-03 | 20260507 17:57:25.301 | INFO | PerfReport: 12.09%, Baseline -- 4.05e+00 / 4.50e+00, 6.16e-01 / 2.34e-01 | 20260507 17:57:25.301 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.14e-01 / 1.04e-01 | 20260507 17:57:25.301 | INFO | PerfReport: 15.69%, DME -- 4.20e+01 / 4.49e+01, 4.06e+00 / 2.01e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.88e+00 / 1.68e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.85e+00, 4.32e-01 / 2.09e-01 | 20260507 17:57:25.301 | INFO | PerfReport: 3.74%, Metrics -- 1.25e+00 / 1.41e+00, 3.16e-01 / 1.89e-01 | 20260507 17:57:25.301 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.16e-01, 6.51e-02 / 3.79e-02 | 20260507 17:57:25.301 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:57:25.301 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:57:25.301 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:57:25.301 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 24.09%, Upload -- 2.58e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 23.93%, Download -- 2.57e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 31.20%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 1.93%, LockUpload -- 2.07e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 17:57:25.301 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 17:57:25.302 | INFO | PerfReport: 16.59%, LockCompute -- 1.78e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 17:57:25.302 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260507 17:57:25.397 | INFO | Starting a round of Baz IO | 20260507 17:57:28.912 | INFO | Populating RTMetrics report | 20260507 17:57:28.913 | INFO | RTMetrics report populated | 20260507 17:57:28.913 | INFO | Opening tmp RTMetrics json file | 20260507 17:57:28.913 | INFO | Serializing RTMetrics to json | 20260507 17:57:28.914 | INFO | Serialization done, swapping tmp file now... | 20260507 17:57:28.915 | INFO | RTMetrics done | 20260507 17:57:29.004 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.606 seconds | 20260507 17:57:29.004 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.31% / 0.02% / 52.67% | 20260507 17:57:29.004 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.23435% / 3.97364e-05% / 2.38419e-05% | 20260507 17:57:29.004 | INFO | Baz IO overall: | 20260507 17:57:29.004 | INFO | BytesWritten : 651030256922 | 20260507 17:57:29.004 | INFO | NumEvents : 315548019117 | 20260507 17:57:30.439 | INFO | Analyzing chunk frames = [1466368,1466880) | 20260507 17:57:30.439 | INFO | MemInfo: { | 20260507 17:57:30.439 | INFO | "gpu" : " 17 GB", | 20260507 17:57:30.439 | INFO | "huge" : "", | 20260507 17:57:30.439 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:30.439 | INFO | "malloc" : " 14 GB", | 20260507 17:57:30.439 | INFO | "pinned" : " 29 GB" | 20260507 17:57:30.439 | INFO | } | 20260507 17:57:30.439 | INFO | Perf report for the last 5.13912 seconds: | 20260507 17:57:30.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:30.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:30.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:30.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:30.439 | INFO | ANALYSIS : 69.8%, 3.0, 108.6ms, 0 | 20260507 17:57:30.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:30.439 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 17:57:30.439 | INFO | PRE_HQ : 66.6%, 1.7, 59.4ms, 0 | 20260507 17:57:30.439 | INFO | BAZWRITER : 69.3%, 2.9, 101.8ms, 0 | 20260507 17:57:30.439 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:57:30.439 | INFO | Uploaded 36GiB to GPU in 2.57551s (13.9778GiB/s) | 20260507 17:57:30.439 | INFO | Downloaded 38GiB to GPU in 2.72053s (13.9679GiB/s) | 20260507 17:57:35.578 | INFO | Analyzing chunk frames = [1466880,1467392) | 20260507 17:57:35.578 | INFO | MemInfo: { | 20260507 17:57:35.578 | INFO | "gpu" : " 17 GB", | 20260507 17:57:35.578 | INFO | "huge" : "", | 20260507 17:57:35.578 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:35.578 | INFO | "malloc" : " 14 GB", | 20260507 17:57:35.579 | INFO | "pinned" : " 29 GB" | 20260507 17:57:35.579 | INFO | } | 20260507 17:57:35.579 | INFO | Perf report for the last 5.13956 seconds: | 20260507 17:57:35.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:35.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 17:57:35.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 17:57:35.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:35.579 | INFO | Uploaded 36GiB to GPU in 2.52988s (14.2299GiB/s) | 20260507 17:57:35.579 | INFO | Downloaded 34GiB to GPU in 2.43224s (13.9789GiB/s) | 20260507 17:57:40.718 | INFO | Analyzing chunk frames = [1467392,1467904) | 20260507 17:57:40.718 | INFO | MemInfo: { | 20260507 17:57:40.718 | INFO | "gpu" : " 17 GB", | 20260507 17:57:40.718 | INFO | "huge" : "", | 20260507 17:57:40.718 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:40.718 | INFO | "malloc" : " 14 GB", | 20260507 17:57:40.718 | INFO | "pinned" : " 29 GB" | 20260507 17:57:40.718 | INFO | } | 20260507 17:57:40.718 | INFO | Perf report for the last 5.13956 seconds: | 20260507 17:57:40.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:40.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.718 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 17:57:40.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.718 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:57:40.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:40.719 | INFO | Uploaded 36GiB to GPU in 2.57388s (13.9867GiB/s) | 20260507 17:57:40.719 | INFO | Downloaded 34GiB to GPU in 2.40769s (14.1214GiB/s) | 20260507 17:57:45.857 | INFO | Analyzing chunk frames = [1467904,1468416) | 20260507 17:57:45.857 | INFO | MemInfo: { | 20260507 17:57:45.857 | INFO | "gpu" : " 17 GB", | 20260507 17:57:45.857 | INFO | "huge" : "", | 20260507 17:57:45.857 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:45.857 | INFO | "malloc" : " 14 GB", | 20260507 17:57:45.857 | INFO | "pinned" : " 29 GB" | 20260507 17:57:45.857 | INFO | } | 20260507 17:57:45.857 | INFO | Perf report for the last 5.1392 seconds: | 20260507 17:57:45.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:45.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.857 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 17:57:45.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.857 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:57:45.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:45.858 | INFO | Uploaded 36GiB to GPU in 2.49639s (14.4208GiB/s) | 20260507 17:57:45.858 | INFO | Downloaded 34GiB to GPU in 2.34777s (14.4818GiB/s) | 20260507 17:57:50.997 | INFO | Analyzing chunk frames = [1468416,1468928) | 20260507 17:57:50.997 | INFO | MemInfo: { | 20260507 17:57:50.997 | INFO | "gpu" : " 17 GB", | 20260507 17:57:50.997 | INFO | "huge" : "", | 20260507 17:57:50.997 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:50.997 | INFO | "malloc" : " 14 GB", | 20260507 17:57:50.997 | INFO | "pinned" : " 29 GB" | 20260507 17:57:50.997 | INFO | } | 20260507 17:57:50.997 | INFO | Perf report for the last 5.13979 seconds: | 20260507 17:57:50.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:50.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.997 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 17:57:50.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.997 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:57:50.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:50.998 | INFO | Uploaded 36GiB to GPU in 2.35286s (15.3005GiB/s) | 20260507 17:57:50.998 | INFO | Downloaded 34GiB to GPU in 2.46348s (13.8016GiB/s) | 20260507 17:57:56.136 | INFO | Analyzing chunk frames = [1468928,1469440) | 20260507 17:57:56.136 | INFO | MemInfo: { | 20260507 17:57:56.136 | INFO | "gpu" : " 17 GB", | 20260507 17:57:56.136 | INFO | "huge" : "", | 20260507 17:57:56.136 | INFO | "huge_shared" : " 29 GB", | 20260507 17:57:56.136 | INFO | "malloc" : " 14 GB", | 20260507 17:57:56.136 | INFO | "pinned" : " 29 GB" | 20260507 17:57:56.136 | INFO | } | 20260507 17:57:56.136 | INFO | Perf report for the last 5.13871 seconds: | 20260507 17:57:56.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:57:56.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:57:56.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:57:56.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:57:56.136 | INFO | Uploaded 36GiB to GPU in 2.4337s (14.7923GiB/s) | 20260507 17:57:56.136 | INFO | Downloaded 34GiB to GPU in 2.49961s (13.6021GiB/s) | 20260507 17:58:01.276 | INFO | Analyzing chunk frames = [1469440,1469952) | 20260507 17:58:01.276 | INFO | MemInfo: { | 20260507 17:58:01.276 | INFO | "gpu" : " 17 GB", | 20260507 17:58:01.276 | INFO | "huge" : "", | 20260507 17:58:01.276 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:01.276 | INFO | "malloc" : " 14 GB", | 20260507 17:58:01.276 | INFO | "pinned" : " 29 GB" | 20260507 17:58:01.276 | INFO | } | 20260507 17:58:01.276 | INFO | Perf report for the last 5.13989 seconds: | 20260507 17:58:01.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:01.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 17:58:01.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 17:58:01.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:01.276 | INFO | Uploaded 36GiB to GPU in 2.46893s (14.5812GiB/s) | 20260507 17:58:01.276 | INFO | Downloaded 34GiB to GPU in 2.43044s (13.9893GiB/s) | 20260507 17:58:06.414 | INFO | Analyzing chunk frames = [1469952,1470464) | 20260507 17:58:06.414 | INFO | MemInfo: { | 20260507 17:58:06.414 | INFO | "gpu" : " 17 GB", | 20260507 17:58:06.414 | INFO | "huge" : "", | 20260507 17:58:06.414 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:06.414 | INFO | "malloc" : " 14 GB", | 20260507 17:58:06.414 | INFO | "pinned" : " 29 GB" | 20260507 17:58:06.415 | INFO | } | 20260507 17:58:06.415 | INFO | Perf report for the last 5.13876 seconds: | 20260507 17:58:06.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:06.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 17:58:06.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:58:06.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:06.415 | INFO | Uploaded 36GiB to GPU in 2.53239s (14.2158GiB/s) | 20260507 17:58:06.415 | INFO | Downloaded 34GiB to GPU in 2.50663s (13.564GiB/s) | 20260507 17:58:06.512 | INFO | Starting a round of Baz IO | 20260507 17:58:10.027 | INFO | Populating RTMetrics report | 20260507 17:58:10.027 | INFO | RTMetrics report populated | 20260507 17:58:10.027 | INFO | Opening tmp RTMetrics json file | 20260507 17:58:10.043 | INFO | Serializing RTMetrics to json | 20260507 17:58:10.044 | INFO | Serialization done, swapping tmp file now... | 20260507 17:58:10.045 | INFO | RTMetrics done | 20260507 17:58:10.104 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.591 seconds | 20260507 17:58:10.104 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.29% / 0.02% / 52.69% | 20260507 17:58:10.104 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.22369% / 2.38419e-05% / 1.58946e-05% | 20260507 17:58:10.104 | INFO | Baz IO overall: | 20260507 17:58:10.104 | INFO | BytesWritten : 652962460552 | 20260507 17:58:10.104 | INFO | NumEvents : 316486340027 | 20260507 17:58:11.554 | INFO | Analyzing chunk frames = [1470464,1470976) | 20260507 17:58:11.554 | INFO | MemInfo: { | 20260507 17:58:11.554 | INFO | "gpu" : " 17 GB", | 20260507 17:58:11.554 | INFO | "huge" : "", | 20260507 17:58:11.554 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:11.554 | INFO | "malloc" : " 14 GB", | 20260507 17:58:11.554 | INFO | "pinned" : " 29 GB" | 20260507 17:58:11.554 | INFO | } | 20260507 17:58:11.554 | INFO | Perf report for the last 5.13936 seconds: | 20260507 17:58:11.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:11.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:11.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:11.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:11.554 | INFO | ANALYSIS : 69.5%, 3.0, 108.6ms, 0 | 20260507 17:58:11.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:11.554 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 17:58:11.554 | INFO | PRE_HQ : 66.3%, 1.7, 59.0ms, 0 | 20260507 17:58:11.554 | INFO | BAZWRITER : 69.0%, 2.9, 101.6ms, 0 | 20260507 17:58:11.554 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 17:58:11.554 | INFO | Uploaded 36GiB to GPU in 2.51301s (14.3254GiB/s) | 20260507 17:58:11.554 | INFO | Downloaded 38GiB to GPU in 2.72608s (13.9395GiB/s) | 20260507 17:58:16.693 | INFO | Analyzing chunk frames = [1470976,1471488) | 20260507 17:58:16.693 | INFO | MemInfo: { | 20260507 17:58:16.693 | INFO | "gpu" : " 17 GB", | 20260507 17:58:16.693 | INFO | "huge" : "", | 20260507 17:58:16.693 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:16.693 | INFO | "malloc" : " 14 GB", | 20260507 17:58:16.693 | INFO | "pinned" : " 29 GB" | 20260507 17:58:16.693 | INFO | } | 20260507 17:58:16.694 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:58:16.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:16.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 17:58:16.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 17:58:16.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:16.694 | INFO | Uploaded 36GiB to GPU in 2.42675s (14.8347GiB/s) | 20260507 17:58:16.694 | INFO | Downloaded 34GiB to GPU in 2.46218s (13.8089GiB/s) | 20260507 17:58:21.832 | INFO | Analyzing chunk frames = [1471488,1472000) | 20260507 17:58:21.832 | INFO | MemInfo: { | 20260507 17:58:21.833 | INFO | "gpu" : " 17 GB", | 20260507 17:58:21.833 | INFO | "huge" : "", | 20260507 17:58:21.833 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:21.833 | INFO | "malloc" : " 14 GB", | 20260507 17:58:21.833 | INFO | "pinned" : " 29 GB" | 20260507 17:58:21.833 | INFO | } | 20260507 17:58:21.833 | INFO | Perf report for the last 5.13933 seconds: | 20260507 17:58:21.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:21.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 17:58:21.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:58:21.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:21.833 | INFO | Uploaded 36GiB to GPU in 2.52254s (14.2713GiB/s) | 20260507 17:58:21.833 | INFO | Downloaded 34GiB to GPU in 2.45331s (13.8589GiB/s) | 20260507 17:58:26.972 | INFO | Analyzing chunk frames = [1472000,1472512) | 20260507 17:58:26.972 | INFO | MemInfo: { | 20260507 17:58:26.972 | INFO | "gpu" : " 17 GB", | 20260507 17:58:26.972 | INFO | "huge" : "", | 20260507 17:58:26.972 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:26.972 | INFO | "malloc" : " 14 GB", | 20260507 17:58:26.972 | INFO | "pinned" : " 29 GB" | 20260507 17:58:26.973 | INFO | } | 20260507 17:58:26.973 | INFO | Perf report for the last 5.13975 seconds: | 20260507 17:58:26.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:26.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 17:58:26.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 17:58:26.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:26.973 | INFO | Uploaded 36GiB to GPU in 2.43986s (14.7549GiB/s) | 20260507 17:58:26.973 | INFO | Downloaded 34GiB to GPU in 2.3737s (14.3236GiB/s) | 20260507 17:58:32.112 | INFO | Analyzing chunk frames = [1472512,1473024) | 20260507 17:58:32.112 | INFO | MemInfo: { | 20260507 17:58:32.112 | INFO | "gpu" : " 17 GB", | 20260507 17:58:32.112 | INFO | "huge" : "", | 20260507 17:58:32.112 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:32.112 | INFO | "malloc" : " 14 GB", | 20260507 17:58:32.112 | INFO | "pinned" : " 29 GB" | 20260507 17:58:32.112 | INFO | } | 20260507 17:58:32.112 | INFO | Perf report for the last 5.13971 seconds: | 20260507 17:58:32.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:32.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.112 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:58:32.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.112 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:58:32.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:32.113 | INFO | Uploaded 36GiB to GPU in 2.31319s (15.5629GiB/s) | 20260507 17:58:32.113 | INFO | Downloaded 34GiB to GPU in 2.48782s (13.6666GiB/s) | 20260507 17:58:37.251 | INFO | Analyzing chunk frames = [1473024,1473536) | 20260507 17:58:37.251 | INFO | MemInfo: { | 20260507 17:58:37.251 | INFO | "gpu" : " 17 GB", | 20260507 17:58:37.251 | INFO | "huge" : "", | 20260507 17:58:37.251 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:37.251 | INFO | "malloc" : " 14 GB", | 20260507 17:58:37.251 | INFO | "pinned" : " 29 GB" | 20260507 17:58:37.251 | INFO | } | 20260507 17:58:37.251 | INFO | Perf report for the last 5.13855 seconds: | 20260507 17:58:37.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:37.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 17:58:37.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:58:37.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:37.251 | INFO | Uploaded 36GiB to GPU in 2.41584s (14.9016GiB/s) | 20260507 17:58:37.251 | INFO | Downloaded 34GiB to GPU in 2.49293s (13.6386GiB/s) | 20260507 17:58:42.390 | INFO | Analyzing chunk frames = [1473536,1474048) | 20260507 17:58:42.390 | INFO | MemInfo: { | 20260507 17:58:42.390 | INFO | "gpu" : " 17 GB", | 20260507 17:58:42.390 | INFO | "huge" : "", | 20260507 17:58:42.390 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:42.390 | INFO | "malloc" : " 14 GB", | 20260507 17:58:42.390 | INFO | "pinned" : " 29 GB" | 20260507 17:58:42.390 | INFO | } | 20260507 17:58:42.390 | INFO | Perf report for the last 5.13951 seconds: | 20260507 17:58:42.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:42.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.391 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 17:58:42.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.391 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 17:58:42.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:42.391 | INFO | Uploaded 36GiB to GPU in 2.4606s (14.6306GiB/s) | 20260507 17:58:42.391 | INFO | Downloaded 34GiB to GPU in 2.43513s (13.9623GiB/s) | 20260507 17:58:47.529 | INFO | Analyzing chunk frames = [1474048,1474560) | 20260507 17:58:47.529 | INFO | MemInfo: { | 20260507 17:58:47.529 | INFO | "gpu" : " 17 GB", | 20260507 17:58:47.529 | INFO | "huge" : "", | 20260507 17:58:47.529 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:47.529 | INFO | "malloc" : " 14 GB", | 20260507 17:58:47.529 | INFO | "pinned" : " 29 GB" | 20260507 17:58:47.529 | INFO | } | 20260507 17:58:47.530 | INFO | Perf report for the last 5.13909 seconds: | 20260507 17:58:47.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:47.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 17:58:47.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:58:47.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:47.530 | INFO | Uploaded 36GiB to GPU in 2.51124s (14.3355GiB/s) | 20260507 17:58:47.530 | INFO | Downloaded 34GiB to GPU in 2.50953s (13.5484GiB/s) | 20260507 17:58:47.530 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 17:58:47.530 | INFO | Total mem usage: 16.4 GiB | 20260507 17:58:47.530 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 17:58:47.530 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 17:58:47.530 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 17:58:47.530 | INFO | High Water Breakdown: | 20260507 17:58:47.530 | INFO | 59.2% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 17:58:47.530 | INFO | 40.7% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 17:58:47.530 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 17:58:47.530 | INFO | ------------------------------------------ | 20260507 17:58:47.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:58:47.530 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 17:58:47.530 | INFO | Total mem usage: 26.5 GiB | 20260507 17:58:47.530 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 17:58:47.530 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 17:58:47.530 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 17:58:47.530 | INFO | High Water Breakdown: | 20260507 17:58:47.530 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:58:47.530 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:58:47.530 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:58:47.530 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:58:47.530 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:58:47.530 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:58:47.530 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 17:58:47.530 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 17:58:47.530 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:58:47.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 17:58:47.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 17:58:47.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 17:58:47.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 17:58:47.530 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 17:58:47.530 | INFO | ------------------------------------------ | 20260507 17:58:47.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:58:47.530 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 17:58:47.530 | INFO | Total mem usage: 27.8 GiB | 20260507 17:58:47.530 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 17:58:47.530 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 17:58:47.530 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 17:58:47.530 | INFO | High Water Breakdown: | 20260507 17:58:47.530 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 17:58:47.530 | INFO | ------------------------------------------ | 20260507 17:58:47.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:58:47.530 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 17:58:47.530 | INFO | Total mem usage: 15.7 GiB | 20260507 17:58:47.530 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 17:58:47.530 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 17:58:47.530 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 17:58:47.530 | INFO | High Water Breakdown: | 20260507 17:58:47.530 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 17:58:47.530 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 17:58:47.531 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 17:58:47.531 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 17:58:47.531 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 17:58:47.531 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 17:58:47.531 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 17:58:47.531 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 17:58:47.531 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 17:58:47.531 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 17:58:47.531 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 17:58:47.531 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 17:58:47.531 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 17:58:47.531 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 17:58:47.531 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 17:58:47.531 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 17:58:47.531 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 17:58:47.531 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 17:58:47.531 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 17:58:47.531 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 17:58:47.531 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 17:58:47.531 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 17:58:47.531 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 17:58:47.531 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 17:58:47.531 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 17:58:47.531 | INFO | ------------------------------------------ | 20260507 17:58:47.531 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 17:58:47.531 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 17:58:47.531 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:58:47.531 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:58:47.531 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.69e+01, 1.45e+01 / 2.32e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 0.29%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.09e-02 / 5.01e-03 | 20260507 17:58:47.531 | INFO | PerfReport: 12.07%, Baseline -- 4.03e+00 / 4.53e+00, 6.17e-01 / 1.77e-01 | 20260507 17:58:47.531 | INFO | PerfReport: 3.42%, Histogram -- 1.14e+00 / 1.52e+00, 2.17e-01 / 7.81e-02 | 20260507 17:58:47.531 | INFO | PerfReport: 15.77%, DME -- 4.22e+01 / 4.53e+01, 4.23e+00 / 2.32e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.87e+00 / 2.03e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.49e+00 / 2.83e+00, 4.35e-01 / 2.53e-01 | 20260507 17:58:47.531 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.43e+00, 3.19e-01 / 1.77e-01 | 20260507 17:58:47.531 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.21e-01, 6.58e-02 / 3.95e-02 | 20260507 17:58:47.531 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 17:58:47.531 | INFO | PerfReport: Gathered 1536 samples. | 20260507 17:58:47.531 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 17:58:47.531 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 23.95%, Upload -- 2.58e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 24.06%, Download -- 2.59e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 31.08%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 1.83%, LockUpload -- 1.96e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 2.45%, LockDownload -- 2.64e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 16.61%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 17:58:47.531 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 17:58:47.626 | INFO | Starting a round of Baz IO | 20260507 17:58:51.129 | INFO | Populating RTMetrics report | 20260507 17:58:51.129 | INFO | RTMetrics report populated | 20260507 17:58:51.129 | INFO | Opening tmp RTMetrics json file | 20260507 17:58:51.130 | INFO | Serializing RTMetrics to json | 20260507 17:58:51.131 | INFO | Serialization done, swapping tmp file now... | 20260507 17:58:51.135 | INFO | RTMetrics done | 20260507 17:58:51.209 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.582 seconds | 20260507 17:58:51.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.28% / 0.02% / 52.71% | 20260507 17:58:51.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.21942% / 1.98682e-05% / 3.57628e-05% | 20260507 17:58:51.209 | INFO | Baz IO overall: | 20260507 17:58:51.209 | INFO | BytesWritten : 654895442628 | 20260507 17:58:51.209 | INFO | NumEvents : 317425322503 | 20260507 17:58:52.669 | INFO | Analyzing chunk frames = [1474560,1475072) | 20260507 17:58:52.669 | INFO | MemInfo: { | 20260507 17:58:52.669 | INFO | "gpu" : " 17 GB", | 20260507 17:58:52.669 | INFO | "huge" : "", | 20260507 17:58:52.669 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:52.669 | INFO | "malloc" : " 14 GB", | 20260507 17:58:52.669 | INFO | "pinned" : " 29 GB" | 20260507 17:58:52.669 | INFO | } | 20260507 17:58:52.669 | INFO | Perf report for the last 5.13954 seconds: | 20260507 17:58:52.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:52.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:52.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:52.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:52.669 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 17:58:52.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:52.669 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 17:58:52.669 | INFO | PRE_HQ : 66.5%, 1.7, 59.3ms, 0 | 20260507 17:58:52.669 | INFO | BAZWRITER : 68.7%, 2.9, 102.6ms, 0 | 20260507 17:58:52.669 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:58:52.669 | INFO | Uploaded 36GiB to GPU in 2.55488s (14.0907GiB/s) | 20260507 17:58:52.670 | INFO | Downloaded 38GiB to GPU in 2.7241s (13.9496GiB/s) | 20260507 17:58:57.808 | INFO | Analyzing chunk frames = [1475072,1475584) | 20260507 17:58:57.808 | INFO | MemInfo: { | 20260507 17:58:57.808 | INFO | "gpu" : " 17 GB", | 20260507 17:58:57.808 | INFO | "huge" : "", | 20260507 17:58:57.808 | INFO | "huge_shared" : " 29 GB", | 20260507 17:58:57.809 | INFO | "malloc" : " 14 GB", | 20260507 17:58:57.809 | INFO | "pinned" : " 29 GB" | 20260507 17:58:57.809 | INFO | } | 20260507 17:58:57.809 | INFO | Perf report for the last 5.13959 seconds: | 20260507 17:58:57.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:58:57.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 17:58:57.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 17:58:57.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:58:57.809 | INFO | Uploaded 36GiB to GPU in 2.5023s (14.3868GiB/s) | 20260507 17:58:57.809 | INFO | Downloaded 34GiB to GPU in 2.45356s (13.8574GiB/s) | 20260507 17:59:02.947 | INFO | Analyzing chunk frames = [1475584,1476096) | 20260507 17:59:02.947 | INFO | MemInfo: { | 20260507 17:59:02.947 | INFO | "gpu" : " 17 GB", | 20260507 17:59:02.947 | INFO | "huge" : "", | 20260507 17:59:02.947 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:02.947 | INFO | "malloc" : " 14 GB", | 20260507 17:59:02.947 | INFO | "pinned" : " 29 GB" | 20260507 17:59:02.948 | INFO | } | 20260507 17:59:02.948 | INFO | Perf report for the last 5.1389 seconds: | 20260507 17:59:02.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:02.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 17:59:02.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:59:02.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:02.948 | INFO | Uploaded 36GiB to GPU in 2.54221s (14.1609GiB/s) | 20260507 17:59:02.948 | INFO | Downloaded 34GiB to GPU in 2.44569s (13.902GiB/s) | 20260507 17:59:08.087 | INFO | Analyzing chunk frames = [1476096,1476608) | 20260507 17:59:08.087 | INFO | MemInfo: { | 20260507 17:59:08.087 | INFO | "gpu" : " 17 GB", | 20260507 17:59:08.087 | INFO | "huge" : "", | 20260507 17:59:08.087 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:08.087 | INFO | "malloc" : " 14 GB", | 20260507 17:59:08.087 | INFO | "pinned" : " 29 GB" | 20260507 17:59:08.087 | INFO | } | 20260507 17:59:08.087 | INFO | Perf report for the last 5.13967 seconds: | 20260507 17:59:08.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:08.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.087 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 17:59:08.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.087 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 17:59:08.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:08.088 | INFO | Uploaded 36GiB to GPU in 2.47557s (14.5421GiB/s) | 20260507 17:59:08.088 | INFO | Downloaded 34GiB to GPU in 2.3512s (14.4607GiB/s) | 20260507 17:59:13.226 | INFO | Analyzing chunk frames = [1476608,1477120) | 20260507 17:59:13.226 | INFO | MemInfo: { | 20260507 17:59:13.226 | INFO | "gpu" : " 17 GB", | 20260507 17:59:13.226 | INFO | "huge" : "", | 20260507 17:59:13.226 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:13.226 | INFO | "malloc" : " 14 GB", | 20260507 17:59:13.226 | INFO | "pinned" : " 29 GB" | 20260507 17:59:13.226 | INFO | } | 20260507 17:59:13.226 | INFO | Perf report for the last 5.13919 seconds: | 20260507 17:59:13.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:13.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.226 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 17:59:13.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.227 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 17:59:13.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:13.227 | INFO | Uploaded 36GiB to GPU in 2.32111s (15.5099GiB/s) | 20260507 17:59:13.227 | INFO | Downloaded 34GiB to GPU in 2.47643s (13.7295GiB/s) | 20260507 17:59:18.366 | INFO | Analyzing chunk frames = [1477120,1477632) | 20260507 17:59:18.366 | INFO | MemInfo: { | 20260507 17:59:18.366 | INFO | "gpu" : " 17 GB", | 20260507 17:59:18.366 | INFO | "huge" : "", | 20260507 17:59:18.366 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:18.366 | INFO | "malloc" : " 14 GB", | 20260507 17:59:18.366 | INFO | "pinned" : " 29 GB" | 20260507 17:59:18.366 | INFO | } | 20260507 17:59:18.366 | INFO | Perf report for the last 5.13932 seconds: | 20260507 17:59:18.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:18.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:59:18.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 17:59:18.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:18.366 | INFO | Uploaded 36GiB to GPU in 2.41987s (14.8769GiB/s) | 20260507 17:59:18.366 | INFO | Downloaded 34GiB to GPU in 2.5041s (13.5777GiB/s) | 20260507 17:59:23.505 | INFO | Analyzing chunk frames = [1477632,1478144) | 20260507 17:59:23.505 | INFO | MemInfo: { | 20260507 17:59:23.505 | INFO | "gpu" : " 17 GB", | 20260507 17:59:23.505 | INFO | "huge" : "", | 20260507 17:59:23.505 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:23.505 | INFO | "malloc" : " 14 GB", | 20260507 17:59:23.505 | INFO | "pinned" : " 29 GB" | 20260507 17:59:23.505 | INFO | } | 20260507 17:59:23.505 | INFO | Perf report for the last 5.13934 seconds: | 20260507 17:59:23.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:23.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.505 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 17:59:23.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.505 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 17:59:23.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:23.506 | INFO | Uploaded 36GiB to GPU in 2.47927s (14.5204GiB/s) | 20260507 17:59:23.506 | INFO | Downloaded 34GiB to GPU in 2.43755s (13.9484GiB/s) | 20260507 17:59:28.644 | INFO | Analyzing chunk frames = [1478144,1478656) | 20260507 17:59:28.644 | INFO | MemInfo: { | 20260507 17:59:28.644 | INFO | "gpu" : " 17 GB", | 20260507 17:59:28.644 | INFO | "huge" : "", | 20260507 17:59:28.644 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:28.644 | INFO | "malloc" : " 14 GB", | 20260507 17:59:28.645 | INFO | "pinned" : " 29 GB" | 20260507 17:59:28.645 | INFO | } | 20260507 17:59:28.645 | INFO | Perf report for the last 5.13942 seconds: | 20260507 17:59:28.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:28.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 17:59:28.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:59:28.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:28.645 | INFO | Uploaded 36GiB to GPU in 2.51923s (14.2901GiB/s) | 20260507 17:59:28.645 | INFO | Downloaded 34GiB to GPU in 2.50908s (13.5508GiB/s) | 20260507 17:59:28.742 | INFO | Starting a round of Baz IO | 20260507 17:59:32.242 | INFO | Populating RTMetrics report | 20260507 17:59:32.242 | INFO | RTMetrics report populated | 20260507 17:59:32.242 | INFO | Opening tmp RTMetrics json file | 20260507 17:59:32.258 | INFO | Serializing RTMetrics to json | 20260507 17:59:32.260 | INFO | Serialization done, swapping tmp file now... | 20260507 17:59:32.260 | INFO | RTMetrics done | 20260507 17:59:32.324 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.581 seconds | 20260507 17:59:32.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.26% / 0.02% / 52.72% | 20260507 17:59:32.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.21776% / 1.58946e-05% / 4.76837e-05% | 20260507 17:59:32.324 | INFO | Baz IO overall: | 20260507 17:59:32.324 | INFO | BytesWritten : 656830032626 | 20260507 17:59:32.324 | INFO | NumEvents : 318365453924 | 20260507 17:59:33.784 | INFO | Analyzing chunk frames = [1478656,1479168) | 20260507 17:59:33.784 | INFO | MemInfo: { | 20260507 17:59:33.784 | INFO | "gpu" : " 17 GB", | 20260507 17:59:33.784 | INFO | "huge" : "", | 20260507 17:59:33.784 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:33.784 | INFO | "malloc" : " 14 GB", | 20260507 17:59:33.784 | INFO | "pinned" : " 29 GB" | 20260507 17:59:33.784 | INFO | } | 20260507 17:59:33.784 | INFO | Perf report for the last 5.13915 seconds: | 20260507 17:59:33.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:33.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:33.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:33.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:33.784 | INFO | ANALYSIS : 69.5%, 3.0, 108.3ms, 0 | 20260507 17:59:33.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:33.784 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 17:59:33.784 | INFO | PRE_HQ : 66.6%, 1.7, 59.2ms, 0 | 20260507 17:59:33.784 | INFO | BAZWRITER : 68.8%, 2.9, 102.3ms, 0 | 20260507 17:59:33.784 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 17:59:33.784 | INFO | Uploaded 36GiB to GPU in 2.56542s (14.0328GiB/s) | 20260507 17:59:33.784 | INFO | Downloaded 38GiB to GPU in 2.72488s (13.9456GiB/s) | 20260507 17:59:38.923 | INFO | Analyzing chunk frames = [1479168,1479680) | 20260507 17:59:38.923 | INFO | MemInfo: { | 20260507 17:59:38.923 | INFO | "gpu" : " 17 GB", | 20260507 17:59:38.923 | INFO | "huge" : "", | 20260507 17:59:38.923 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:38.923 | INFO | "malloc" : " 14 GB", | 20260507 17:59:38.923 | INFO | "pinned" : " 29 GB" | 20260507 17:59:38.923 | INFO | } | 20260507 17:59:38.923 | INFO | Perf report for the last 5.13965 seconds: | 20260507 17:59:38.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:38.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.923 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 17:59:38.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.924 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 17:59:38.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:38.924 | INFO | Uploaded 36GiB to GPU in 2.5122s (14.3301GiB/s) | 20260507 17:59:38.924 | INFO | Downloaded 34GiB to GPU in 2.44244s (13.9205GiB/s) | 20260507 17:59:44.063 | INFO | Analyzing chunk frames = [1479680,1480192) | 20260507 17:59:44.063 | INFO | MemInfo: { | 20260507 17:59:44.063 | INFO | "gpu" : " 17 GB", | 20260507 17:59:44.063 | INFO | "huge" : "", | 20260507 17:59:44.063 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:44.063 | INFO | "malloc" : " 14 GB", | 20260507 17:59:44.063 | INFO | "pinned" : " 29 GB" | 20260507 17:59:44.063 | INFO | } | 20260507 17:59:44.063 | INFO | Perf report for the last 5.13982 seconds: | 20260507 17:59:44.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:44.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 17:59:44.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 17:59:44.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:44.063 | INFO | Uploaded 36GiB to GPU in 2.54562s (14.1419GiB/s) | 20260507 17:59:44.064 | INFO | Downloaded 34GiB to GPU in 2.42518s (14.0196GiB/s) | 20260507 17:59:49.202 | INFO | Analyzing chunk frames = [1480192,1480704) | 20260507 17:59:49.202 | INFO | MemInfo: { | 20260507 17:59:49.202 | INFO | "gpu" : " 17 GB", | 20260507 17:59:49.202 | INFO | "huge" : "", | 20260507 17:59:49.202 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:49.202 | INFO | "malloc" : " 14 GB", | 20260507 17:59:49.202 | INFO | "pinned" : " 29 GB" | 20260507 17:59:49.202 | INFO | } | 20260507 17:59:49.202 | INFO | Perf report for the last 5.13878 seconds: | 20260507 17:59:49.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:49.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | ANALYSIS : 66.8%, 3.0, 103.6ms, 0 | 20260507 17:59:49.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 17:59:49.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:49.202 | INFO | Uploaded 36GiB to GPU in 2.48387s (14.4935GiB/s) | 20260507 17:59:49.202 | INFO | Downloaded 34GiB to GPU in 2.36028s (14.405GiB/s) | 20260507 17:59:54.341 | INFO | Analyzing chunk frames = [1480704,1481216) | 20260507 17:59:54.341 | INFO | MemInfo: { | 20260507 17:59:54.341 | INFO | "gpu" : " 17 GB", | 20260507 17:59:54.341 | INFO | "huge" : "", | 20260507 17:59:54.341 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:54.341 | INFO | "malloc" : " 14 GB", | 20260507 17:59:54.341 | INFO | "pinned" : " 29 GB" | 20260507 17:59:54.341 | INFO | } | 20260507 17:59:54.341 | INFO | Perf report for the last 5.13941 seconds: | 20260507 17:59:54.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:54.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.341 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 17:59:54.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.341 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 17:59:54.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:54.342 | INFO | Uploaded 36GiB to GPU in 2.34347s (15.3619GiB/s) | 20260507 17:59:54.342 | INFO | Downloaded 34GiB to GPU in 2.48051s (13.7069GiB/s) | 20260507 17:59:59.481 | INFO | Analyzing chunk frames = [1481216,1481728) | 20260507 17:59:59.481 | INFO | MemInfo: { | 20260507 17:59:59.481 | INFO | "gpu" : " 17 GB", | 20260507 17:59:59.481 | INFO | "huge" : "", | 20260507 17:59:59.481 | INFO | "huge_shared" : " 29 GB", | 20260507 17:59:59.481 | INFO | "malloc" : " 14 GB", | 20260507 17:59:59.481 | INFO | "pinned" : " 29 GB" | 20260507 17:59:59.481 | INFO | } | 20260507 17:59:59.481 | INFO | Perf report for the last 5.13952 seconds: | 20260507 17:59:59.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 17:59:59.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.481 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 17:59:59.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.481 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 17:59:59.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 17:59:59.482 | INFO | Uploaded 36GiB to GPU in 2.2812s (15.7812GiB/s) | 20260507 17:59:59.482 | INFO | Downloaded 34GiB to GPU in 2.49913s (13.6047GiB/s) | 20260507 18:00:04.620 | INFO | Analyzing chunk frames = [1481728,1482240) | 20260507 18:00:04.620 | INFO | MemInfo: { | 20260507 18:00:04.620 | INFO | "gpu" : " 17 GB", | 20260507 18:00:04.620 | INFO | "huge" : "", | 20260507 18:00:04.620 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:04.620 | INFO | "malloc" : " 14 GB", | 20260507 18:00:04.620 | INFO | "pinned" : " 29 GB" | 20260507 18:00:04.620 | INFO | } | 20260507 18:00:04.620 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:00:04.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:04.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.620 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 18:00:04.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.621 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 18:00:04.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:04.621 | INFO | Uploaded 36GiB to GPU in 2.4444s (14.7276GiB/s) | 20260507 18:00:04.621 | INFO | Downloaded 34GiB to GPU in 2.44573s (13.9018GiB/s) | 20260507 18:00:09.759 | INFO | Analyzing chunk frames = [1482240,1482752) | 20260507 18:00:09.759 | INFO | MemInfo: { | 20260507 18:00:09.759 | INFO | "gpu" : " 17 GB", | 20260507 18:00:09.759 | INFO | "huge" : "", | 20260507 18:00:09.759 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:09.759 | INFO | "malloc" : " 14 GB", | 20260507 18:00:09.760 | INFO | "pinned" : " 29 GB" | 20260507 18:00:09.760 | INFO | } | 20260507 18:00:09.760 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:00:09.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:09.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:00:09.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | PRE_HQ : 38.8%, 1.0, 20.1ms, 0 | 20260507 18:00:09.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:09.760 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:00:09.760 | INFO | Total mem usage: 16.4 GiB | 20260507 18:00:09.760 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:00:09.760 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 18:00:09.760 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:00:09.760 | INFO | High Water Breakdown: | 20260507 18:00:09.760 | INFO | 59.2% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:00:09.760 | INFO | 40.7% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:00:09.760 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:00:09.760 | INFO | ------------------------------------------ | 20260507 18:00:09.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:00:09.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:00:09.760 | INFO | Total mem usage: 26.5 GiB | 20260507 18:00:09.760 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:00:09.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:00:09.760 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:00:09.760 | INFO | High Water Breakdown: | 20260507 18:00:09.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:00:09.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:00:09.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:00:09.760 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:00:09.760 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:00:09.760 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:00:09.760 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:00:09.761 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:00:09.760 | INFO | Uploaded 36GiB to GPU in 2.44879s (14.7012GiB/s) | 20260507 18:00:09.761 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:00:09.761 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:00:09.761 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:00:09.761 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:00:09.761 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:00:09.761 | INFO | Downloaded 34GiB to GPU in 2.448s (13.8889GiB/s) | 20260507 18:00:09.761 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:00:09.761 | INFO | ------------------------------------------ | 20260507 18:00:09.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:00:09.761 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:00:09.761 | INFO | Total mem usage: 27.8 GiB | 20260507 18:00:09.761 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:00:09.761 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:00:09.761 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:00:09.761 | INFO | High Water Breakdown: | 20260507 18:00:09.761 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:00:09.761 | INFO | ------------------------------------------ | 20260507 18:00:09.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:00:09.761 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:00:09.761 | INFO | Total mem usage: 15.7 GiB | 20260507 18:00:09.761 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:00:09.761 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:00:09.761 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:00:09.761 | INFO | High Water Breakdown: | 20260507 18:00:09.761 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:00:09.761 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:00:09.761 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:00:09.761 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:00:09.761 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:00:09.761 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:00:09.761 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:00:09.761 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:00:09.761 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:00:09.761 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:00:09.761 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:00:09.761 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:00:09.761 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:00:09.761 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:00:09.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:00:09.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:00:09.761 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:00:09.761 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:00:09.761 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:00:09.761 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:00:09.761 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:00:09.761 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:00:09.761 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:00:09.761 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:00:09.761 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:00:09.761 | INFO | ------------------------------------------ | 20260507 18:00:09.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:00:09.761 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:00:09.761 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:00:09.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:00:09.761 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.44e+01 / 2.54e+00 | 20260507 18:00:09.761 | INFO | PerfReport: 0.29%, AggregateStats -- 9.85e-02 / 1.03e-01, 1.11e-02 / 7.46e-03 | 20260507 18:00:09.762 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.53e+00, 6.21e-01 / 2.31e-01 | 20260507 18:00:09.762 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.04e-01 | 20260507 18:00:09.762 | INFO | PerfReport: 15.69%, DME -- 4.20e+01 / 4.50e+01, 4.40e+00 / 2.40e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.88e+00 / 1.99e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.35e-01 / 2.44e-01 | 20260507 18:00:09.762 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.42e+00, 3.16e-01 / 1.65e-01 | 20260507 18:00:09.762 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.20e-01, 6.52e-02 / 4.10e-02 | 20260507 18:00:09.762 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:00:09.762 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:00:09.762 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:00:09.762 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 23.91%, Upload -- 2.57e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 24.09%, Download -- 2.59e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 31.15%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 1.76%, LockUpload -- 1.89e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 2.43%, LockDownload -- 2.61e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 16.64%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:00:09.762 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 18:00:09.874 | INFO | Starting a round of Baz IO | 20260507 18:00:13.354 | INFO | Populating RTMetrics report | 20260507 18:00:13.355 | INFO | RTMetrics report populated | 20260507 18:00:13.355 | INFO | Opening tmp RTMetrics json file | 20260507 18:00:13.371 | INFO | Serializing RTMetrics to json | 20260507 18:00:13.374 | INFO | Serialization done, swapping tmp file now... | 20260507 18:00:13.374 | INFO | RTMetrics done | 20260507 18:00:13.433 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.558 seconds | 20260507 18:00:13.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.24% / 0.02% / 52.74% | 20260507 18:00:13.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.20888% / 1.58946e-05% / 2.38419e-05% | 20260507 18:00:13.433 | INFO | Baz IO overall: | 20260507 18:00:13.433 | INFO | BytesWritten : 658764145242 | 20260507 18:00:13.433 | INFO | NumEvents : 319305179157 | 20260507 18:00:14.899 | INFO | Analyzing chunk frames = [1482752,1483264) | 20260507 18:00:14.899 | INFO | MemInfo: { | 20260507 18:00:14.899 | INFO | "gpu" : " 17 GB", | 20260507 18:00:14.899 | INFO | "huge" : "", | 20260507 18:00:14.899 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:14.899 | INFO | "malloc" : " 14 GB", | 20260507 18:00:14.899 | INFO | "pinned" : " 29 GB" | 20260507 18:00:14.899 | INFO | } | 20260507 18:00:14.899 | INFO | Perf report for the last 5.13902 seconds: | 20260507 18:00:14.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:14.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:14.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:14.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:14.899 | INFO | ANALYSIS : 69.2%, 2.9, 107.0ms, 0 | 20260507 18:00:14.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:14.899 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 18:00:14.899 | INFO | PRE_HQ : 66.7%, 1.8, 63.9ms, 0 | 20260507 18:00:14.899 | INFO | BAZWRITER : 68.2%, 3.1, 107.1ms, 0 | 20260507 18:00:14.899 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:00:14.899 | INFO | Uploaded 36GiB to GPU in 2.53197s (14.2182GiB/s) | 20260507 18:00:14.899 | INFO | Downloaded 38GiB to GPU in 2.69744s (14.0874GiB/s) | 20260507 18:00:20.038 | INFO | Analyzing chunk frames = [1483264,1483776) | 20260507 18:00:20.038 | INFO | MemInfo: { | 20260507 18:00:20.038 | INFO | "gpu" : " 17 GB", | 20260507 18:00:20.038 | INFO | "huge" : "", | 20260507 18:00:20.038 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:20.038 | INFO | "malloc" : " 14 GB", | 20260507 18:00:20.038 | INFO | "pinned" : " 29 GB" | 20260507 18:00:20.038 | INFO | } | 20260507 18:00:20.038 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:00:20.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:20.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.038 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:00:20.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.038 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:00:20.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:20.039 | INFO | Uploaded 36GiB to GPU in 2.51306s (14.3252GiB/s) | 20260507 18:00:20.039 | INFO | Downloaded 34GiB to GPU in 2.44323s (13.916GiB/s) | 20260507 18:00:25.177 | INFO | Analyzing chunk frames = [1483776,1484288) | 20260507 18:00:25.177 | INFO | MemInfo: { | 20260507 18:00:25.177 | INFO | "gpu" : " 17 GB", | 20260507 18:00:25.177 | INFO | "huge" : "", | 20260507 18:00:25.177 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:25.177 | INFO | "malloc" : " 14 GB", | 20260507 18:00:25.177 | INFO | "pinned" : " 29 GB" | 20260507 18:00:25.177 | INFO | } | 20260507 18:00:25.177 | INFO | Perf report for the last 5.13926 seconds: | 20260507 18:00:25.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:25.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.177 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 18:00:25.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.178 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:00:25.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:25.178 | INFO | Uploaded 36GiB to GPU in 2.54191s (14.1626GiB/s) | 20260507 18:00:25.178 | INFO | Downloaded 34GiB to GPU in 2.4252s (14.0195GiB/s) | 20260507 18:00:30.317 | INFO | Analyzing chunk frames = [1484288,1484800) | 20260507 18:00:30.317 | INFO | MemInfo: { | 20260507 18:00:30.317 | INFO | "gpu" : " 17 GB", | 20260507 18:00:30.317 | INFO | "huge" : "", | 20260507 18:00:30.317 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:30.317 | INFO | "malloc" : " 14 GB", | 20260507 18:00:30.317 | INFO | "pinned" : " 29 GB" | 20260507 18:00:30.317 | INFO | } | 20260507 18:00:30.317 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:00:30.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:30.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.317 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:00:30.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.317 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:00:30.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:30.318 | INFO | Uploaded 36GiB to GPU in 2.48942s (14.4612GiB/s) | 20260507 18:00:30.318 | INFO | Downloaded 34GiB to GPU in 2.35318s (14.4485GiB/s) | 20260507 18:00:35.457 | INFO | Analyzing chunk frames = [1484800,1485312) | 20260507 18:00:35.457 | INFO | MemInfo: { | 20260507 18:00:35.457 | INFO | "gpu" : " 17 GB", | 20260507 18:00:35.457 | INFO | "huge" : "", | 20260507 18:00:35.457 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:35.457 | INFO | "malloc" : " 14 GB", | 20260507 18:00:35.457 | INFO | "pinned" : " 29 GB" | 20260507 18:00:35.457 | INFO | } | 20260507 18:00:35.457 | INFO | Perf report for the last 5.13983 seconds: | 20260507 18:00:35.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:35.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 18:00:35.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:00:35.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:35.457 | INFO | Uploaded 36GiB to GPU in 2.37111s (15.1828GiB/s) | 20260507 18:00:35.457 | INFO | Downloaded 34GiB to GPU in 2.47574s (13.7333GiB/s) | 20260507 18:00:40.595 | INFO | Analyzing chunk frames = [1485312,1485824) | 20260507 18:00:40.595 | INFO | MemInfo: { | 20260507 18:00:40.595 | INFO | "gpu" : " 17 GB", | 20260507 18:00:40.595 | INFO | "huge" : "", | 20260507 18:00:40.595 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:40.596 | INFO | "malloc" : " 14 GB", | 20260507 18:00:40.596 | INFO | "pinned" : " 29 GB" | 20260507 18:00:40.596 | INFO | } | 20260507 18:00:40.596 | INFO | Perf report for the last 5.13878 seconds: | 20260507 18:00:40.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:40.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 18:00:40.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:00:40.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:40.596 | INFO | Uploaded 36GiB to GPU in 2.41268s (14.9211GiB/s) | 20260507 18:00:40.596 | INFO | Downloaded 34GiB to GPU in 2.47242s (13.7517GiB/s) | 20260507 18:00:45.734 | INFO | Analyzing chunk frames = [1485824,1486336) | 20260507 18:00:45.734 | INFO | MemInfo: { | 20260507 18:00:45.735 | INFO | "gpu" : " 17 GB", | 20260507 18:00:45.735 | INFO | "huge" : "", | 20260507 18:00:45.735 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:45.735 | INFO | "malloc" : " 14 GB", | 20260507 18:00:45.735 | INFO | "pinned" : " 29 GB" | 20260507 18:00:45.735 | INFO | } | 20260507 18:00:45.735 | INFO | Perf report for the last 5.1392 seconds: | 20260507 18:00:45.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:45.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:00:45.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 18:00:45.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:45.735 | INFO | Uploaded 36GiB to GPU in 2.49529s (14.4272GiB/s) | 20260507 18:00:45.735 | INFO | Downloaded 34GiB to GPU in 2.41686s (14.0678GiB/s) | 20260507 18:00:50.874 | INFO | Analyzing chunk frames = [1486336,1486848) | 20260507 18:00:50.874 | INFO | MemInfo: { | 20260507 18:00:50.874 | INFO | "gpu" : " 17 GB", | 20260507 18:00:50.874 | INFO | "huge" : "", | 20260507 18:00:50.874 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:50.875 | INFO | "malloc" : " 14 GB", | 20260507 18:00:50.875 | INFO | "pinned" : " 29 GB" | 20260507 18:00:50.875 | INFO | } | 20260507 18:00:50.875 | INFO | Perf report for the last 5.13978 seconds: | 20260507 18:00:50.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:50.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 18:00:50.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:00:50.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:50.875 | INFO | Uploaded 36GiB to GPU in 2.54543s (14.143GiB/s) | 20260507 18:00:50.875 | INFO | Downloaded 34GiB to GPU in 2.49538s (13.6252GiB/s) | 20260507 18:00:50.971 | INFO | Starting a round of Baz IO | 20260507 18:00:54.473 | INFO | Populating RTMetrics report | 20260507 18:00:54.474 | INFO | RTMetrics report populated | 20260507 18:00:54.474 | INFO | Opening tmp RTMetrics json file | 20260507 18:00:54.475 | INFO | Serializing RTMetrics to json | 20260507 18:00:54.476 | INFO | Serialization done, swapping tmp file now... | 20260507 18:00:54.479 | INFO | RTMetrics done | 20260507 18:00:54.553 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.582 seconds | 20260507 18:00:54.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.22% / 0.02% / 52.76% | 20260507 18:00:54.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.2171% / 3.57628e-05% / 5.16574e-05% | 20260507 18:00:54.554 | INFO | Baz IO overall: | 20260507 18:00:54.554 | INFO | BytesWritten : 660702096240 | 20260507 18:00:54.554 | INFO | NumEvents : 320247833572 | 20260507 18:00:56.013 | INFO | Analyzing chunk frames = [1486848,1487360) | 20260507 18:00:56.013 | INFO | MemInfo: { | 20260507 18:00:56.013 | INFO | "gpu" : " 17 GB", | 20260507 18:00:56.013 | INFO | "huge" : "", | 20260507 18:00:56.013 | INFO | "huge_shared" : " 29 GB", | 20260507 18:00:56.013 | INFO | "malloc" : " 14 GB", | 20260507 18:00:56.013 | INFO | "pinned" : " 29 GB" | 20260507 18:00:56.013 | INFO | } | 20260507 18:00:56.014 | INFO | Perf report for the last 5.13892 seconds: | 20260507 18:00:56.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:00:56.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:56.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:56.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:56.014 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 18:00:56.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:00:56.014 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 18:00:56.014 | INFO | PRE_HQ : 66.7%, 1.7, 59.4ms, 0 | 20260507 18:00:56.014 | INFO | BAZWRITER : 68.8%, 2.9, 103.2ms, 0 | 20260507 18:00:56.014 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:00:56.014 | INFO | Uploaded 36GiB to GPU in 2.56433s (14.0388GiB/s) | 20260507 18:00:56.014 | INFO | Downloaded 38GiB to GPU in 2.71066s (14.0187GiB/s) | 20260507 18:01:01.153 | INFO | Analyzing chunk frames = [1487360,1487872) | 20260507 18:01:01.153 | INFO | MemInfo: { | 20260507 18:01:01.153 | INFO | "gpu" : " 17 GB", | 20260507 18:01:01.153 | INFO | "huge" : "", | 20260507 18:01:01.153 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:01.153 | INFO | "malloc" : " 14 GB", | 20260507 18:01:01.153 | INFO | "pinned" : " 29 GB" | 20260507 18:01:01.153 | INFO | } | 20260507 18:01:01.153 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:01:01.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:01.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.153 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:01:01.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.153 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:01:01.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:01.154 | INFO | Uploaded 36GiB to GPU in 2.52746s (14.2435GiB/s) | 20260507 18:01:01.154 | INFO | Downloaded 34GiB to GPU in 2.42872s (13.9991GiB/s) | 20260507 18:01:06.292 | INFO | Analyzing chunk frames = [1487872,1488384) | 20260507 18:01:06.292 | INFO | MemInfo: { | 20260507 18:01:06.292 | INFO | "gpu" : " 17 GB", | 20260507 18:01:06.292 | INFO | "huge" : "", | 20260507 18:01:06.292 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:06.292 | INFO | "malloc" : " 14 GB", | 20260507 18:01:06.292 | INFO | "pinned" : " 29 GB" | 20260507 18:01:06.292 | INFO | } | 20260507 18:01:06.292 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:01:06.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:06.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 18:01:06.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:01:06.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:06.293 | INFO | Uploaded 36GiB to GPU in 2.5783s (13.9627GiB/s) | 20260507 18:01:06.293 | INFO | Downloaded 34GiB to GPU in 2.41264s (14.0924GiB/s) | 20260507 18:01:11.432 | INFO | Analyzing chunk frames = [1488384,1488896) | 20260507 18:01:11.432 | INFO | MemInfo: { | 20260507 18:01:11.432 | INFO | "gpu" : " 17 GB", | 20260507 18:01:11.432 | INFO | "huge" : "", | 20260507 18:01:11.432 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:11.432 | INFO | "malloc" : " 14 GB", | 20260507 18:01:11.432 | INFO | "pinned" : " 29 GB" | 20260507 18:01:11.432 | INFO | } | 20260507 18:01:11.432 | INFO | Perf report for the last 5.13966 seconds: | 20260507 18:01:11.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:11.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.432 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260507 18:01:11.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.432 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:01:11.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:11.433 | INFO | Uploaded 36GiB to GPU in 2.51643s (14.306GiB/s) | 20260507 18:01:11.433 | INFO | Downloaded 34GiB to GPU in 2.3524s (14.4533GiB/s) | 20260507 18:01:16.572 | INFO | Analyzing chunk frames = [1488896,1489408) | 20260507 18:01:16.572 | INFO | MemInfo: { | 20260507 18:01:16.572 | INFO | "gpu" : " 17 GB", | 20260507 18:01:16.572 | INFO | "huge" : "", | 20260507 18:01:16.572 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:16.572 | INFO | "malloc" : " 14 GB", | 20260507 18:01:16.572 | INFO | "pinned" : " 29 GB" | 20260507 18:01:16.572 | INFO | } | 20260507 18:01:16.572 | INFO | Perf report for the last 5.1397 seconds: | 20260507 18:01:16.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:16.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:01:16.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 18:01:16.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:16.572 | INFO | Uploaded 36GiB to GPU in 2.36344s (15.2321GiB/s) | 20260507 18:01:16.572 | INFO | Downloaded 34GiB to GPU in 2.47223s (13.7528GiB/s) | 20260507 18:01:21.710 | INFO | Analyzing chunk frames = [1489408,1489920) | 20260507 18:01:21.711 | INFO | MemInfo: { | 20260507 18:01:21.711 | INFO | "gpu" : " 17 GB", | 20260507 18:01:21.711 | INFO | "huge" : "", | 20260507 18:01:21.711 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:21.711 | INFO | "malloc" : " 14 GB", | 20260507 18:01:21.711 | INFO | "pinned" : " 29 GB" | 20260507 18:01:21.711 | INFO | } | 20260507 18:01:21.711 | INFO | Perf report for the last 5.13894 seconds: | 20260507 18:01:21.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:21.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:01:21.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 18:01:21.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:21.711 | INFO | Uploaded 36GiB to GPU in 2.40894s (14.9443GiB/s) | 20260507 18:01:21.711 | INFO | Downloaded 34GiB to GPU in 2.49526s (13.6258GiB/s) | 20260507 18:01:26.850 | INFO | Analyzing chunk frames = [1489920,1490432) | 20260507 18:01:26.850 | INFO | MemInfo: { | 20260507 18:01:26.850 | INFO | "gpu" : " 17 GB", | 20260507 18:01:26.850 | INFO | "huge" : "", | 20260507 18:01:26.850 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:26.850 | INFO | "malloc" : " 14 GB", | 20260507 18:01:26.850 | INFO | "pinned" : " 29 GB" | 20260507 18:01:26.850 | INFO | } | 20260507 18:01:26.850 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:01:26.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:26.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.850 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 18:01:26.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.850 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:01:26.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:26.851 | INFO | Uploaded 36GiB to GPU in 2.47213s (14.5623GiB/s) | 20260507 18:01:26.851 | INFO | Downloaded 34GiB to GPU in 2.42806s (14.003GiB/s) | 20260507 18:01:31.989 | INFO | Analyzing chunk frames = [1490432,1490944) | 20260507 18:01:31.989 | INFO | MemInfo: { | 20260507 18:01:31.989 | INFO | "gpu" : " 17 GB", | 20260507 18:01:31.989 | INFO | "huge" : "", | 20260507 18:01:31.989 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:31.989 | INFO | "malloc" : " 14 GB", | 20260507 18:01:31.989 | INFO | "pinned" : " 29 GB" | 20260507 18:01:31.989 | INFO | } | 20260507 18:01:31.990 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:01:31.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:31.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 18:01:31.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 18:01:31.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:31.990 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:01:31.990 | INFO | Total mem usage: 16.4 GiB | 20260507 18:01:31.990 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:01:31.990 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 18:01:31.990 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:01:31.990 | INFO | High Water Breakdown: | 20260507 18:01:31.990 | INFO | 59.3% : 8.64 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:01:31.990 | INFO | 40.6% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:01:31.990 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:01:31.990 | INFO | ------------------------------------------ | 20260507 18:01:31.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:01:31.990 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:01:31.990 | INFO | Total mem usage: 26.5 GiB | 20260507 18:01:31.990 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:01:31.990 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:01:31.990 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:01:31.990 | INFO | High Water Breakdown: | 20260507 18:01:31.990 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:01:31.990 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:01:31.990 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:01:31.990 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:01:31.990 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:01:31.990 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:01:31.990 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:01:31.990 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:01:31.990 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:01:31.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:01:31.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:01:31.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:01:31.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:01:31.990 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:01:31.990 | INFO | ------------------------------------------ | 20260507 18:01:31.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:01:31.990 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:01:31.990 | INFO | Total mem usage: 27.8 GiB | 20260507 18:01:31.990 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:01:31.990 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:01:31.990 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:01:31.990 | INFO | High Water Breakdown: | 20260507 18:01:31.990 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:01:31.990 | INFO | ------------------------------------------ | 20260507 18:01:31.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:01:31.990 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:01:31.990 | INFO | Total mem usage: 15.7 GiB | 20260507 18:01:31.990 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:01:31.990 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:01:31.990 | INFO | Uploaded 36GiB to GPU in 2.53746s (14.1874GiB/s) | 20260507 18:01:31.990 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:01:31.991 | INFO | Downloaded 34GiB to GPU in 2.50599s (13.5675GiB/s) | 20260507 18:01:31.991 | INFO | High Water Breakdown: | 20260507 18:01:31.991 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:01:31.991 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:01:31.991 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:01:31.991 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:01:31.991 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:01:31.991 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:01:31.991 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:01:31.991 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:01:31.991 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:01:31.991 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:01:31.991 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:01:31.991 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:01:31.991 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:01:31.991 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:01:31.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:01:31.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:01:31.991 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:01:31.991 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:01:31.991 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:01:31.991 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:01:31.991 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:01:31.991 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:01:31.991 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:01:31.991 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:01:31.991 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:01:31.991 | INFO | ------------------------------------------ | 20260507 18:01:31.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:01:31.991 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:01:31.991 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:01:31.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:01:31.991 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.67e+01, 1.45e+01 / 2.49e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.11e-02 / 5.56e-03 | 20260507 18:01:31.991 | INFO | PerfReport: 12.07%, Baseline -- 4.04e+00 / 4.50e+00, 6.15e-01 / 2.14e-01 | 20260507 18:01:31.991 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.49e+00, 2.14e-01 / 1.12e-01 | 20260507 18:01:31.991 | INFO | PerfReport: 15.75%, DME -- 4.22e+01 / 4.51e+01, 4.08e+00 / 2.32e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.86e+00 / 2.04e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.49e+00 / 2.85e+00, 4.37e-01 / 2.46e-01 | 20260507 18:01:31.991 | INFO | PerfReport: 3.75%, Metrics -- 1.26e+00 / 1.43e+00, 3.23e-01 / 1.96e-01 | 20260507 18:01:31.991 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.94e-01 / 4.16e-01, 6.56e-02 / 4.27e-02 | 20260507 18:01:31.991 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:01:31.991 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:01:31.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:01:31.991 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 24.14%, Upload -- 2.60e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 23.97%, Download -- 2.58e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 31.13%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 1.97%, LockUpload -- 2.12e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 2.19%, LockDownload -- 2.35e+00 / 0.00e+00, 3.74e+00 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 16.59%, LockCompute -- 1.78e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:01:31.991 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260507 18:01:32.087 | INFO | Starting a round of Baz IO | 20260507 18:01:35.591 | INFO | Populating RTMetrics report | 20260507 18:01:35.591 | INFO | RTMetrics report populated | 20260507 18:01:35.591 | INFO | Opening tmp RTMetrics json file | 20260507 18:01:35.607 | INFO | Serializing RTMetrics to json | 20260507 18:01:35.609 | INFO | Serialization done, swapping tmp file now... | 20260507 18:01:35.609 | INFO | RTMetrics done | 20260507 18:01:35.670 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.582 seconds | 20260507 18:01:35.670 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.21% / 0.02% / 52.78% | 20260507 18:01:35.670 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.19972% / 3.17891e-05% / 3.57628e-05% | 20260507 18:01:35.670 | INFO | Baz IO overall: | 20260507 18:01:35.670 | INFO | BytesWritten : 662637335282 | 20260507 18:01:35.670 | INFO | NumEvents : 321188428258 | 20260507 18:01:37.128 | INFO | Analyzing chunk frames = [1490944,1491456) | 20260507 18:01:37.128 | INFO | MemInfo: { | 20260507 18:01:37.128 | INFO | "gpu" : " 17 GB", | 20260507 18:01:37.128 | INFO | "huge" : "", | 20260507 18:01:37.128 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:37.128 | INFO | "malloc" : " 14 GB", | 20260507 18:01:37.128 | INFO | "pinned" : " 29 GB" | 20260507 18:01:37.128 | INFO | } | 20260507 18:01:37.128 | INFO | Perf report for the last 5.13889 seconds: | 20260507 18:01:37.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:37.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:37.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:37.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:37.129 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 18:01:37.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:37.129 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:01:37.129 | INFO | PRE_HQ : 66.9%, 1.7, 59.3ms, 0 | 20260507 18:01:37.129 | INFO | BAZWRITER : 68.9%, 2.9, 101.7ms, 0 | 20260507 18:01:37.129 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:01:37.129 | INFO | Uploaded 36GiB to GPU in 2.5411s (14.1671GiB/s) | 20260507 18:01:37.129 | INFO | Downloaded 38GiB to GPU in 2.73362s (13.901GiB/s) | 20260507 18:01:42.267 | INFO | Analyzing chunk frames = [1491456,1491968) | 20260507 18:01:42.267 | INFO | MemInfo: { | 20260507 18:01:42.267 | INFO | "gpu" : " 17 GB", | 20260507 18:01:42.267 | INFO | "huge" : "", | 20260507 18:01:42.267 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:42.267 | INFO | "malloc" : " 14 GB", | 20260507 18:01:42.267 | INFO | "pinned" : " 29 GB" | 20260507 18:01:42.267 | INFO | } | 20260507 18:01:42.268 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:01:42.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:42.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:01:42.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:01:42.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:42.268 | INFO | Uploaded 36GiB to GPU in 2.53042s (14.2269GiB/s) | 20260507 18:01:42.268 | INFO | Downloaded 34GiB to GPU in 2.45183s (13.8672GiB/s) | 20260507 18:01:47.407 | INFO | Analyzing chunk frames = [1491968,1492480) | 20260507 18:01:47.407 | INFO | MemInfo: { | 20260507 18:01:47.407 | INFO | "gpu" : " 17 GB", | 20260507 18:01:47.407 | INFO | "huge" : "", | 20260507 18:01:47.407 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:47.407 | INFO | "malloc" : " 14 GB", | 20260507 18:01:47.407 | INFO | "pinned" : " 29 GB" | 20260507 18:01:47.407 | INFO | } | 20260507 18:01:47.407 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:01:47.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:47.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.407 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:01:47.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.407 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:01:47.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:47.408 | INFO | Uploaded 36GiB to GPU in 2.53179s (14.2192GiB/s) | 20260507 18:01:47.408 | INFO | Downloaded 34GiB to GPU in 2.44508s (13.9055GiB/s) | 20260507 18:01:52.546 | INFO | Analyzing chunk frames = [1492480,1492992) | 20260507 18:01:52.546 | INFO | MemInfo: { | 20260507 18:01:52.546 | INFO | "gpu" : " 17 GB", | 20260507 18:01:52.546 | INFO | "huge" : "", | 20260507 18:01:52.546 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:52.546 | INFO | "malloc" : " 14 GB", | 20260507 18:01:52.546 | INFO | "pinned" : " 29 GB" | 20260507 18:01:52.546 | INFO | } | 20260507 18:01:52.547 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:01:52.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:52.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:01:52.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:01:52.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:52.547 | INFO | Uploaded 36GiB to GPU in 2.48394s (14.4931GiB/s) | 20260507 18:01:52.547 | INFO | Downloaded 34GiB to GPU in 2.3511s (14.4613GiB/s) | 20260507 18:01:57.686 | INFO | Analyzing chunk frames = [1492992,1493504) | 20260507 18:01:57.686 | INFO | MemInfo: { | 20260507 18:01:57.686 | INFO | "gpu" : " 17 GB", | 20260507 18:01:57.686 | INFO | "huge" : "", | 20260507 18:01:57.686 | INFO | "huge_shared" : " 29 GB", | 20260507 18:01:57.686 | INFO | "malloc" : " 14 GB", | 20260507 18:01:57.686 | INFO | "pinned" : " 29 GB" | 20260507 18:01:57.686 | INFO | } | 20260507 18:01:57.686 | INFO | Perf report for the last 5.13982 seconds: | 20260507 18:01:57.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:01:57.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.686 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:01:57.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.687 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 18:01:57.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:01:57.687 | INFO | Uploaded 36GiB to GPU in 2.3201s (15.5166GiB/s) | 20260507 18:01:57.687 | INFO | Downloaded 34GiB to GPU in 2.4892s (13.659GiB/s) | 20260507 18:02:02.825 | INFO | Analyzing chunk frames = [1493504,1494016) | 20260507 18:02:02.825 | INFO | MemInfo: { | 20260507 18:02:02.825 | INFO | "gpu" : " 17 GB", | 20260507 18:02:02.825 | INFO | "huge" : "", | 20260507 18:02:02.825 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:02.825 | INFO | "malloc" : " 14 GB", | 20260507 18:02:02.825 | INFO | "pinned" : " 29 GB" | 20260507 18:02:02.825 | INFO | } | 20260507 18:02:02.825 | INFO | Perf report for the last 5.13875 seconds: | 20260507 18:02:02.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:02.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.825 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:02:02.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.825 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 18:02:02.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:02.826 | INFO | Uploaded 36GiB to GPU in 2.43088s (14.8095GiB/s) | 20260507 18:02:02.826 | INFO | Downloaded 34GiB to GPU in 2.48865s (13.662GiB/s) | 20260507 18:02:07.964 | INFO | Analyzing chunk frames = [1494016,1494528) | 20260507 18:02:07.964 | INFO | MemInfo: { | 20260507 18:02:07.964 | INFO | "gpu" : " 17 GB", | 20260507 18:02:07.964 | INFO | "huge" : "", | 20260507 18:02:07.964 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:07.964 | INFO | "malloc" : " 14 GB", | 20260507 18:02:07.964 | INFO | "pinned" : " 29 GB" | 20260507 18:02:07.965 | INFO | } | 20260507 18:02:07.965 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:02:07.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:07.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:02:07.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:02:07.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:07.965 | INFO | Uploaded 36GiB to GPU in 2.49192s (14.4467GiB/s) | 20260507 18:02:07.965 | INFO | Downloaded 34GiB to GPU in 2.42265s (14.0342GiB/s) | 20260507 18:02:13.104 | INFO | Analyzing chunk frames = [1494528,1495040) | 20260507 18:02:13.104 | INFO | MemInfo: { | 20260507 18:02:13.104 | INFO | "gpu" : " 17 GB", | 20260507 18:02:13.104 | INFO | "huge" : "", | 20260507 18:02:13.104 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:13.104 | INFO | "malloc" : " 14 GB", | 20260507 18:02:13.104 | INFO | "pinned" : " 29 GB" | 20260507 18:02:13.104 | INFO | } | 20260507 18:02:13.104 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:02:13.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:13.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.104 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 18:02:13.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.104 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:02:13.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:13.105 | INFO | Uploaded 36GiB to GPU in 2.54544s (14.1429GiB/s) | 20260507 18:02:13.105 | INFO | Downloaded 34GiB to GPU in 2.49691s (13.6168GiB/s) | 20260507 18:02:13.201 | INFO | Starting a round of Baz IO | 20260507 18:02:16.709 | INFO | Populating RTMetrics report | 20260507 18:02:16.709 | INFO | RTMetrics report populated | 20260507 18:02:16.709 | INFO | Opening tmp RTMetrics json file | 20260507 18:02:16.724 | INFO | Serializing RTMetrics to json | 20260507 18:02:16.725 | INFO | Serialization done, swapping tmp file now... | 20260507 18:02:16.726 | INFO | RTMetrics done | 20260507 18:02:16.788 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.586 seconds | 20260507 18:02:16.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.19% / 0.02% / 52.79% | 20260507 18:02:16.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.19425% / 2.38419e-05% / 3.57628e-05% | 20260507 18:02:16.788 | INFO | Baz IO overall: | 20260507 18:02:16.788 | INFO | BytesWritten : 664572725598 | 20260507 18:02:16.788 | INFO | NumEvents : 322129026897 | 20260507 18:02:18.243 | INFO | Analyzing chunk frames = [1495040,1495552) | 20260507 18:02:18.243 | INFO | MemInfo: { | 20260507 18:02:18.243 | INFO | "gpu" : " 17 GB", | 20260507 18:02:18.243 | INFO | "huge" : "", | 20260507 18:02:18.243 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:18.243 | INFO | "malloc" : " 14 GB", | 20260507 18:02:18.243 | INFO | "pinned" : " 29 GB" | 20260507 18:02:18.243 | INFO | } | 20260507 18:02:18.243 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:02:18.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:18.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:18.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:18.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:18.243 | INFO | ANALYSIS : 69.7%, 3.0, 108.5ms, 0 | 20260507 18:02:18.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:18.243 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:02:18.243 | INFO | PRE_HQ : 66.9%, 1.7, 59.3ms, 0 | 20260507 18:02:18.243 | INFO | BAZWRITER : 68.9%, 2.8, 100.6ms, 0 | 20260507 18:02:18.244 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:02:18.244 | INFO | Uploaded 36GiB to GPU in 2.56921s (14.0121GiB/s) | 20260507 18:02:18.244 | INFO | Downloaded 38GiB to GPU in 2.717s (13.986GiB/s) | 20260507 18:02:23.382 | INFO | Analyzing chunk frames = [1495552,1496064) | 20260507 18:02:23.382 | INFO | MemInfo: { | 20260507 18:02:23.382 | INFO | "gpu" : " 17 GB", | 20260507 18:02:23.382 | INFO | "huge" : "", | 20260507 18:02:23.382 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:23.382 | INFO | "malloc" : " 14 GB", | 20260507 18:02:23.382 | INFO | "pinned" : " 29 GB" | 20260507 18:02:23.382 | INFO | } | 20260507 18:02:23.382 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:02:23.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:23.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.382 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:02:23.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.383 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:02:23.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:23.383 | INFO | Uploaded 36GiB to GPU in 2.54504s (14.1452GiB/s) | 20260507 18:02:23.383 | INFO | Downloaded 34GiB to GPU in 2.44429s (13.9099GiB/s) | 20260507 18:02:28.522 | INFO | Analyzing chunk frames = [1496064,1496576) | 20260507 18:02:28.522 | INFO | MemInfo: { | 20260507 18:02:28.522 | INFO | "gpu" : " 17 GB", | 20260507 18:02:28.522 | INFO | "huge" : "", | 20260507 18:02:28.522 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:28.522 | INFO | "malloc" : " 14 GB", | 20260507 18:02:28.522 | INFO | "pinned" : " 29 GB" | 20260507 18:02:28.522 | INFO | } | 20260507 18:02:28.522 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:02:28.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:28.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.522 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 18:02:28.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.522 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:02:28.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:28.523 | INFO | Uploaded 36GiB to GPU in 2.57956s (13.9559GiB/s) | 20260507 18:02:28.523 | INFO | Downloaded 34GiB to GPU in 2.41332s (14.0885GiB/s) | 20260507 18:02:33.661 | INFO | Analyzing chunk frames = [1496576,1497088) | 20260507 18:02:33.661 | INFO | MemInfo: { | 20260507 18:02:33.661 | INFO | "gpu" : " 17 GB", | 20260507 18:02:33.661 | INFO | "huge" : "", | 20260507 18:02:33.661 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:33.661 | INFO | "malloc" : " 14 GB", | 20260507 18:02:33.661 | INFO | "pinned" : " 29 GB" | 20260507 18:02:33.661 | INFO | } | 20260507 18:02:33.661 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:02:33.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:33.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.661 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:02:33.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.661 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:02:33.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:33.662 | INFO | Uploaded 36GiB to GPU in 2.49615s (14.4222GiB/s) | 20260507 18:02:33.662 | INFO | Downloaded 34GiB to GPU in 2.34369s (14.507GiB/s) | 20260507 18:02:38.801 | INFO | Analyzing chunk frames = [1497088,1497600) | 20260507 18:02:38.801 | INFO | MemInfo: { | 20260507 18:02:38.801 | INFO | "gpu" : " 17 GB", | 20260507 18:02:38.801 | INFO | "huge" : "", | 20260507 18:02:38.801 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:38.801 | INFO | "malloc" : " 14 GB", | 20260507 18:02:38.801 | INFO | "pinned" : " 29 GB" | 20260507 18:02:38.801 | INFO | } | 20260507 18:02:38.801 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:02:38.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:38.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:02:38.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:02:38.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:38.801 | INFO | Uploaded 36GiB to GPU in 2.32385s (15.4915GiB/s) | 20260507 18:02:38.802 | INFO | Downloaded 34GiB to GPU in 2.46893s (13.7712GiB/s) | 20260507 18:02:43.940 | INFO | Analyzing chunk frames = [1497600,1498112) | 20260507 18:02:43.940 | INFO | MemInfo: { | 20260507 18:02:43.940 | INFO | "gpu" : " 17 GB", | 20260507 18:02:43.940 | INFO | "huge" : "", | 20260507 18:02:43.940 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:43.940 | INFO | "malloc" : " 14 GB", | 20260507 18:02:43.940 | INFO | "pinned" : " 29 GB" | 20260507 18:02:43.940 | INFO | } | 20260507 18:02:43.940 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:02:43.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:43.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:02:43.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:02:43.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:43.940 | INFO | Uploaded 36GiB to GPU in 2.4427s (14.7378GiB/s) | 20260507 18:02:43.940 | INFO | Downloaded 34GiB to GPU in 2.48321s (13.6919GiB/s) | 20260507 18:02:49.079 | INFO | Analyzing chunk frames = [1498112,1498624) | 20260507 18:02:49.079 | INFO | MemInfo: { | 20260507 18:02:49.079 | INFO | "gpu" : " 17 GB", | 20260507 18:02:49.079 | INFO | "huge" : "", | 20260507 18:02:49.079 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:49.079 | INFO | "malloc" : " 14 GB", | 20260507 18:02:49.079 | INFO | "pinned" : " 29 GB" | 20260507 18:02:49.079 | INFO | } | 20260507 18:02:49.080 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:02:49.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:49.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:02:49.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:02:49.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:49.080 | INFO | Uploaded 36GiB to GPU in 2.50952s (14.3454GiB/s) | 20260507 18:02:49.080 | INFO | Downloaded 34GiB to GPU in 2.42037s (14.0475GiB/s) | 20260507 18:02:54.219 | INFO | Analyzing chunk frames = [1498624,1499136) | 20260507 18:02:54.219 | INFO | MemInfo: { | 20260507 18:02:54.219 | INFO | "gpu" : " 17 GB", | 20260507 18:02:54.219 | INFO | "huge" : "", | 20260507 18:02:54.219 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:54.219 | INFO | "malloc" : " 14 GB", | 20260507 18:02:54.219 | INFO | "pinned" : " 29 GB" | 20260507 18:02:54.219 | INFO | } | 20260507 18:02:54.219 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:02:54.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:54.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:02:54.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 18:02:54.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:54.219 | INFO | Uploaded 36GiB to GPU in 2.50997s (14.3428GiB/s) | 20260507 18:02:54.219 | INFO | Downloaded 34GiB to GPU in 2.41532s (14.0768GiB/s) | 20260507 18:02:54.219 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:02:54.219 | INFO | Total mem usage: 16.4 GiB | 20260507 18:02:54.219 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:02:54.219 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 18:02:54.219 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:02:54.219 | INFO | High Water Breakdown: | 20260507 18:02:54.219 | INFO | 59.2% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:02:54.219 | INFO | 40.7% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:02:54.219 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:02:54.219 | INFO | ------------------------------------------ | 20260507 18:02:54.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:02:54.220 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:02:54.220 | INFO | Total mem usage: 26.5 GiB | 20260507 18:02:54.220 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:02:54.220 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:02:54.220 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:02:54.220 | INFO | High Water Breakdown: | 20260507 18:02:54.220 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:02:54.220 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:02:54.220 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:02:54.220 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:02:54.220 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:02:54.220 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:02:54.220 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:02:54.220 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:02:54.220 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:02:54.220 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:02:54.220 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:02:54.220 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:02:54.220 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:02:54.220 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:02:54.220 | INFO | ------------------------------------------ | 20260507 18:02:54.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:02:54.220 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:02:54.220 | INFO | Total mem usage: 27.8 GiB | 20260507 18:02:54.220 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:02:54.220 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:02:54.220 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:02:54.220 | INFO | High Water Breakdown: | 20260507 18:02:54.220 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:02:54.220 | INFO | ------------------------------------------ | 20260507 18:02:54.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:02:54.220 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:02:54.220 | INFO | Total mem usage: 15.7 GiB | 20260507 18:02:54.220 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:02:54.220 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:02:54.220 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:02:54.220 | INFO | High Water Breakdown: | 20260507 18:02:54.220 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:02:54.220 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:02:54.220 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:02:54.220 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:02:54.220 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:02:54.220 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:02:54.220 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:02:54.220 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:02:54.220 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:02:54.220 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:02:54.220 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:02:54.220 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:02:54.220 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:02:54.220 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:02:54.220 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:02:54.220 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:02:54.220 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:02:54.220 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:02:54.220 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:02:54.220 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:02:54.220 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:02:54.220 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:02:54.220 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:02:54.220 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:02:54.220 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:02:54.220 | INFO | ------------------------------------------ | 20260507 18:02:54.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:02:54.220 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:02:54.220 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:02:54.220 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:02:54.220 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.67e+01, 1.45e+01 / 2.37e+00 | 20260507 18:02:54.220 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.20e-02 / 4.52e-03 | 20260507 18:02:54.220 | INFO | PerfReport: 12.07%, Baseline -- 4.03e+00 / 4.51e+00, 6.13e-01 / 2.07e-01 | 20260507 18:02:54.220 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.15e-01 / 1.14e-01 | 20260507 18:02:54.220 | INFO | PerfReport: 15.76%, DME -- 4.21e+01 / 4.52e+01, 4.07e+00 / 2.25e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.83e+00 / 1.77e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.90e+00, 4.35e-01 / 1.97e-01 | 20260507 18:02:54.221 | INFO | PerfReport: 3.74%, Metrics -- 1.25e+00 / 1.43e+00, 3.16e-01 / 1.39e-01 | 20260507 18:02:54.221 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.93e-01 / 4.18e-01, 6.37e-02 / 3.25e-02 | 20260507 18:02:54.221 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:02:54.221 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:02:54.221 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:02:54.221 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 24.13%, Upload -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 23.97%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 31.09%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 2.00%, LockUpload -- 2.15e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 2.20%, LockDownload -- 2.37e+00 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 16.60%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:02:54.221 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.12e-02 / 0.00e+00 | 20260507 18:02:54.316 | INFO | Starting a round of Baz IO | 20260507 18:02:57.836 | INFO | Populating RTMetrics report | 20260507 18:02:57.836 | INFO | RTMetrics report populated | 20260507 18:02:57.836 | INFO | Opening tmp RTMetrics json file | 20260507 18:02:57.850 | INFO | Serializing RTMetrics to json | 20260507 18:02:57.851 | INFO | Serialization done, swapping tmp file now... | 20260507 18:02:57.852 | INFO | RTMetrics done | 20260507 18:02:57.916 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.6 seconds | 20260507 18:02:57.916 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.17% / 0.02% / 52.81% | 20260507 18:02:57.916 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.17423% / 3.57628e-05% / 2.78155e-05% | 20260507 18:02:57.916 | INFO | Baz IO overall: | 20260507 18:02:57.916 | INFO | BytesWritten : 666506413484 | 20260507 18:02:57.916 | INFO | NumEvents : 323068129279 | 20260507 18:02:59.358 | INFO | Analyzing chunk frames = [1499136,1499648) | 20260507 18:02:59.358 | INFO | MemInfo: { | 20260507 18:02:59.358 | INFO | "gpu" : " 17 GB", | 20260507 18:02:59.358 | INFO | "huge" : "", | 20260507 18:02:59.358 | INFO | "huge_shared" : " 29 GB", | 20260507 18:02:59.358 | INFO | "malloc" : " 14 GB", | 20260507 18:02:59.358 | INFO | "pinned" : " 29 GB" | 20260507 18:02:59.358 | INFO | } | 20260507 18:02:59.359 | INFO | Perf report for the last 5.13972 seconds: | 20260507 18:02:59.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:02:59.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:59.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:59.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:59.359 | INFO | ANALYSIS : 69.8%, 3.0, 108.8ms, 0 | 20260507 18:02:59.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:02:59.359 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:02:59.359 | INFO | PRE_HQ : 66.4%, 1.7, 58.9ms, 0 | 20260507 18:02:59.359 | INFO | BAZWRITER : 69.2%, 2.8, 100.2ms, 0 | 20260507 18:02:59.359 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:02:59.359 | INFO | Uploaded 36GiB to GPU in 2.59931s (13.8498GiB/s) | 20260507 18:02:59.359 | INFO | Downloaded 38GiB to GPU in 2.7199s (13.9711GiB/s) | 20260507 18:03:04.497 | INFO | Analyzing chunk frames = [1499648,1500160) | 20260507 18:03:04.497 | INFO | MemInfo: { | 20260507 18:03:04.497 | INFO | "gpu" : " 17 GB", | 20260507 18:03:04.497 | INFO | "huge" : "", | 20260507 18:03:04.497 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:04.497 | INFO | "malloc" : " 14 GB", | 20260507 18:03:04.497 | INFO | "pinned" : " 29 GB" | 20260507 18:03:04.497 | INFO | } | 20260507 18:03:04.497 | INFO | Perf report for the last 5.13875 seconds: | 20260507 18:03:04.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:04.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.497 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:03:04.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.497 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 18:03:04.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:04.498 | INFO | Uploaded 36GiB to GPU in 2.53799s (14.1845GiB/s) | 20260507 18:03:04.498 | INFO | Downloaded 34GiB to GPU in 2.44499s (13.906GiB/s) | 20260507 18:03:09.637 | INFO | Analyzing chunk frames = [1500160,1500672) | 20260507 18:03:09.637 | INFO | MemInfo: { | 20260507 18:03:09.637 | INFO | "gpu" : " 17 GB", | 20260507 18:03:09.637 | INFO | "huge" : "", | 20260507 18:03:09.637 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:09.637 | INFO | "malloc" : " 14 GB", | 20260507 18:03:09.637 | INFO | "pinned" : " 29 GB" | 20260507 18:03:09.637 | INFO | } | 20260507 18:03:09.637 | INFO | Perf report for the last 5.13963 seconds: | 20260507 18:03:09.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:09.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.637 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260507 18:03:09.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.637 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:03:09.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:09.638 | INFO | Uploaded 36GiB to GPU in 2.56481s (14.0361GiB/s) | 20260507 18:03:09.638 | INFO | Downloaded 34GiB to GPU in 2.41592s (14.0733GiB/s) | 20260507 18:03:14.776 | INFO | Analyzing chunk frames = [1500672,1501184) | 20260507 18:03:14.776 | INFO | MemInfo: { | 20260507 18:03:14.776 | INFO | "gpu" : " 17 GB", | 20260507 18:03:14.776 | INFO | "huge" : "", | 20260507 18:03:14.776 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:14.776 | INFO | "malloc" : " 14 GB", | 20260507 18:03:14.776 | INFO | "pinned" : " 29 GB" | 20260507 18:03:14.776 | INFO | } | 20260507 18:03:14.776 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:03:14.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:14.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:03:14.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:03:14.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:14.776 | INFO | Uploaded 36GiB to GPU in 2.5035s (14.3799GiB/s) | 20260507 18:03:14.776 | INFO | Downloaded 34GiB to GPU in 2.34591s (14.4933GiB/s) | 20260507 18:03:19.915 | INFO | Analyzing chunk frames = [1501184,1501696) | 20260507 18:03:19.915 | INFO | MemInfo: { | 20260507 18:03:19.915 | INFO | "gpu" : " 17 GB", | 20260507 18:03:19.915 | INFO | "huge" : "", | 20260507 18:03:19.915 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:19.916 | INFO | "malloc" : " 14 GB", | 20260507 18:03:19.916 | INFO | "pinned" : " 29 GB" | 20260507 18:03:19.916 | INFO | } | 20260507 18:03:19.916 | INFO | Perf report for the last 5.13974 seconds: | 20260507 18:03:19.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:19.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:03:19.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 18:03:19.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:19.916 | INFO | Uploaded 36GiB to GPU in 2.33597s (15.4112GiB/s) | 20260507 18:03:19.916 | INFO | Downloaded 34GiB to GPU in 2.46871s (13.7724GiB/s) | 20260507 18:03:25.055 | INFO | Analyzing chunk frames = [1501696,1502208) | 20260507 18:03:25.055 | INFO | MemInfo: { | 20260507 18:03:25.055 | INFO | "gpu" : " 17 GB", | 20260507 18:03:25.055 | INFO | "huge" : "", | 20260507 18:03:25.055 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:25.055 | INFO | "malloc" : " 14 GB", | 20260507 18:03:25.055 | INFO | "pinned" : " 29 GB" | 20260507 18:03:25.055 | INFO | } | 20260507 18:03:25.055 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:03:25.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:25.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.055 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:03:25.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.055 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:03:25.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:25.056 | INFO | Uploaded 36GiB to GPU in 2.4837s (14.4945GiB/s) | 20260507 18:03:25.056 | INFO | Downloaded 34GiB to GPU in 2.4941s (13.6322GiB/s) | 20260507 18:03:30.194 | INFO | Analyzing chunk frames = [1502208,1502720) | 20260507 18:03:30.194 | INFO | MemInfo: { | 20260507 18:03:30.194 | INFO | "gpu" : " 17 GB", | 20260507 18:03:30.194 | INFO | "huge" : "", | 20260507 18:03:30.194 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:30.194 | INFO | "malloc" : " 14 GB", | 20260507 18:03:30.194 | INFO | "pinned" : " 29 GB" | 20260507 18:03:30.194 | INFO | } | 20260507 18:03:30.194 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:03:30.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:30.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.195 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:03:30.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.195 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:03:30.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:30.195 | INFO | Uploaded 36GiB to GPU in 2.50724s (14.3584GiB/s) | 20260507 18:03:30.195 | INFO | Downloaded 34GiB to GPU in 2.4301s (13.9912GiB/s) | 20260507 18:03:35.333 | INFO | Analyzing chunk frames = [1502720,1503232) | 20260507 18:03:35.333 | INFO | MemInfo: { | 20260507 18:03:35.333 | INFO | "gpu" : " 17 GB", | 20260507 18:03:35.333 | INFO | "huge" : "", | 20260507 18:03:35.333 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:35.333 | INFO | "malloc" : " 14 GB", | 20260507 18:03:35.333 | INFO | "pinned" : " 29 GB" | 20260507 18:03:35.334 | INFO | } | 20260507 18:03:35.334 | INFO | Perf report for the last 5.1391 seconds: | 20260507 18:03:35.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:35.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 18:03:35.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 18:03:35.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:35.334 | INFO | Uploaded 36GiB to GPU in 2.55629s (14.0829GiB/s) | 20260507 18:03:35.334 | INFO | Downloaded 34GiB to GPU in 2.49723s (13.6151GiB/s) | 20260507 18:03:35.448 | INFO | Starting a round of Baz IO | 20260507 18:03:38.970 | INFO | Populating RTMetrics report | 20260507 18:03:38.970 | INFO | RTMetrics report populated | 20260507 18:03:38.970 | INFO | Opening tmp RTMetrics json file | 20260507 18:03:38.971 | INFO | Serializing RTMetrics to json | 20260507 18:03:38.972 | INFO | Serialization done, swapping tmp file now... | 20260507 18:03:38.972 | INFO | RTMetrics done | 20260507 18:03:39.051 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.602 seconds | 20260507 18:03:39.051 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.15% / 0.02% / 52.83% | 20260507 18:03:39.051 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.1838% / 3.97364e-05% / 4.37101e-05% | 20260507 18:03:39.051 | INFO | Baz IO overall: | 20260507 18:03:39.051 | INFO | BytesWritten : 668443754382 | 20260507 18:03:39.051 | INFO | NumEvents : 324009834705 | 20260507 18:03:40.472 | INFO | Analyzing chunk frames = [1503232,1503744) | 20260507 18:03:40.472 | INFO | MemInfo: { | 20260507 18:03:40.472 | INFO | "gpu" : " 17 GB", | 20260507 18:03:40.472 | INFO | "huge" : "", | 20260507 18:03:40.472 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:40.472 | INFO | "malloc" : " 14 GB", | 20260507 18:03:40.472 | INFO | "pinned" : " 29 GB" | 20260507 18:03:40.472 | INFO | } | 20260507 18:03:40.473 | INFO | Perf report for the last 5.1389 seconds: | 20260507 18:03:40.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:40.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:40.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:40.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:40.473 | INFO | ANALYSIS : 69.7%, 2.9, 107.7ms, 0 | 20260507 18:03:40.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:40.473 | INFO | RT_METRICS : 49.4%, 1.0, 25.9ms, 0 | 20260507 18:03:40.473 | INFO | PRE_HQ : 68.8%, 2.0, 71.1ms, 0 | 20260507 18:03:40.473 | INFO | BAZWRITER : 69.4%, 3.2, 113.1ms, 0 | 20260507 18:03:40.473 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:03:40.473 | INFO | Uploaded 36GiB to GPU in 2.53635s (14.1936GiB/s) | 20260507 18:03:40.473 | INFO | Downloaded 38GiB to GPU in 2.6967s (14.0913GiB/s) | 20260507 18:03:45.612 | INFO | Analyzing chunk frames = [1503744,1504256) | 20260507 18:03:45.612 | INFO | MemInfo: { | 20260507 18:03:45.612 | INFO | "gpu" : " 17 GB", | 20260507 18:03:45.612 | INFO | "huge" : "", | 20260507 18:03:45.612 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:45.612 | INFO | "malloc" : " 14 GB", | 20260507 18:03:45.612 | INFO | "pinned" : " 29 GB" | 20260507 18:03:45.612 | INFO | } | 20260507 18:03:45.612 | INFO | Perf report for the last 5.13969 seconds: | 20260507 18:03:45.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:45.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.612 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:03:45.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.612 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:03:45.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:45.613 | INFO | Uploaded 36GiB to GPU in 2.53252s (14.2151GiB/s) | 20260507 18:03:45.613 | INFO | Downloaded 34GiB to GPU in 2.43235s (13.9783GiB/s) | 20260507 18:03:50.751 | INFO | Analyzing chunk frames = [1504256,1504768) | 20260507 18:03:50.751 | INFO | MemInfo: { | 20260507 18:03:50.751 | INFO | "gpu" : " 17 GB", | 20260507 18:03:50.751 | INFO | "huge" : "", | 20260507 18:03:50.751 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:50.751 | INFO | "malloc" : " 14 GB", | 20260507 18:03:50.751 | INFO | "pinned" : " 29 GB" | 20260507 18:03:50.751 | INFO | } | 20260507 18:03:50.751 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:03:50.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:50.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.752 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:03:50.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.752 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:03:50.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:50.752 | INFO | Uploaded 36GiB to GPU in 2.56997s (14.008GiB/s) | 20260507 18:03:50.752 | INFO | Downloaded 34GiB to GPU in 2.42222s (14.0367GiB/s) | 20260507 18:03:55.891 | INFO | Analyzing chunk frames = [1504768,1505280) | 20260507 18:03:55.891 | INFO | MemInfo: { | 20260507 18:03:55.891 | INFO | "gpu" : " 17 GB", | 20260507 18:03:55.891 | INFO | "huge" : "", | 20260507 18:03:55.891 | INFO | "huge_shared" : " 29 GB", | 20260507 18:03:55.891 | INFO | "malloc" : " 14 GB", | 20260507 18:03:55.891 | INFO | "pinned" : " 29 GB" | 20260507 18:03:55.891 | INFO | } | 20260507 18:03:55.891 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:03:55.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:03:55.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:03:55.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:03:55.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:03:55.891 | INFO | Uploaded 36GiB to GPU in 2.49592s (14.4235GiB/s) | 20260507 18:03:55.891 | INFO | Downloaded 34GiB to GPU in 2.336s (14.5548GiB/s) | 20260507 18:04:01.030 | INFO | Analyzing chunk frames = [1505280,1505792) | 20260507 18:04:01.030 | INFO | MemInfo: { | 20260507 18:04:01.030 | INFO | "gpu" : " 17 GB", | 20260507 18:04:01.030 | INFO | "huge" : "", | 20260507 18:04:01.030 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:01.030 | INFO | "malloc" : " 14 GB", | 20260507 18:04:01.030 | INFO | "pinned" : " 29 GB" | 20260507 18:04:01.030 | INFO | } | 20260507 18:04:01.030 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:04:01.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:01.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.030 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:04:01.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.030 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:04:01.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:01.031 | INFO | Uploaded 36GiB to GPU in 2.36118s (15.2466GiB/s) | 20260507 18:04:01.031 | INFO | Downloaded 34GiB to GPU in 2.47614s (13.7311GiB/s) | 20260507 18:04:06.169 | INFO | Analyzing chunk frames = [1505792,1506304) | 20260507 18:04:06.169 | INFO | MemInfo: { | 20260507 18:04:06.169 | INFO | "gpu" : " 17 GB", | 20260507 18:04:06.169 | INFO | "huge" : "", | 20260507 18:04:06.169 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:06.169 | INFO | "malloc" : " 14 GB", | 20260507 18:04:06.169 | INFO | "pinned" : " 29 GB" | 20260507 18:04:06.169 | INFO | } | 20260507 18:04:06.169 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:04:06.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:06.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.169 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:04:06.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.170 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:04:06.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:06.170 | INFO | Uploaded 36GiB to GPU in 2.37305s (15.1703GiB/s) | 20260507 18:04:06.170 | INFO | Downloaded 34GiB to GPU in 2.49268s (13.64GiB/s) | 20260507 18:04:11.309 | INFO | Analyzing chunk frames = [1506304,1506816) | 20260507 18:04:11.309 | INFO | MemInfo: { | 20260507 18:04:11.309 | INFO | "gpu" : " 17 GB", | 20260507 18:04:11.309 | INFO | "huge" : "", | 20260507 18:04:11.309 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:11.309 | INFO | "malloc" : " 14 GB", | 20260507 18:04:11.309 | INFO | "pinned" : " 29 GB" | 20260507 18:04:11.309 | INFO | } | 20260507 18:04:11.309 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:04:11.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:11.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.309 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:04:11.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.309 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:04:11.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:11.310 | INFO | Uploaded 36GiB to GPU in 2.46352s (14.6132GiB/s) | 20260507 18:04:11.310 | INFO | Downloaded 34GiB to GPU in 2.43688s (13.9523GiB/s) | 20260507 18:04:16.448 | INFO | Analyzing chunk frames = [1506816,1507328) | 20260507 18:04:16.448 | INFO | MemInfo: { | 20260507 18:04:16.448 | INFO | "gpu" : " 17 GB", | 20260507 18:04:16.448 | INFO | "huge" : "", | 20260507 18:04:16.448 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:16.448 | INFO | "malloc" : " 14 GB", | 20260507 18:04:16.448 | INFO | "pinned" : " 29 GB" | 20260507 18:04:16.448 | INFO | } | 20260507 18:04:16.448 | INFO | Perf report for the last 5.13936 seconds: | 20260507 18:04:16.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:16.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.448 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 18:04:16.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.448 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260507 18:04:16.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:16.449 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:04:16.449 | INFO | Total mem usage: 16.4 GiB | 20260507 18:04:16.449 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:04:16.449 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 18:04:16.449 | INFO | Uploaded 36GiB to GPU in 2.51144s (14.3344GiB/s) | 20260507 18:04:16.449 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:04:16.449 | INFO | High Water Breakdown: | 20260507 18:04:16.449 | INFO | 59.3% : 8.64 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:04:16.449 | INFO | 40.6% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:04:16.449 | INFO | Downloaded 34GiB to GPU in 2.50566s (13.5693GiB/s) | 20260507 18:04:16.449 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:04:16.449 | INFO | ------------------------------------------ | 20260507 18:04:16.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:04:16.449 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:04:16.449 | INFO | Total mem usage: 26.5 GiB | 20260507 18:04:16.449 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:04:16.449 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:04:16.449 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:04:16.449 | INFO | High Water Breakdown: | 20260507 18:04:16.449 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:04:16.449 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:04:16.449 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:04:16.449 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:04:16.449 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:04:16.449 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:04:16.449 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:04:16.449 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:04:16.449 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:04:16.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:04:16.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:04:16.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:04:16.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:04:16.449 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:04:16.449 | INFO | ------------------------------------------ | 20260507 18:04:16.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:04:16.449 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:04:16.449 | INFO | Total mem usage: 27.8 GiB | 20260507 18:04:16.449 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:04:16.449 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:04:16.449 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:04:16.449 | INFO | High Water Breakdown: | 20260507 18:04:16.449 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:04:16.449 | INFO | ------------------------------------------ | 20260507 18:04:16.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:04:16.449 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:04:16.449 | INFO | Total mem usage: 15.7 GiB | 20260507 18:04:16.449 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:04:16.449 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:04:16.449 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:04:16.449 | INFO | High Water Breakdown: | 20260507 18:04:16.449 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:04:16.449 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:04:16.449 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:04:16.449 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:04:16.449 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:04:16.449 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:04:16.449 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:04:16.449 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:04:16.449 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:04:16.449 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:04:16.449 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:04:16.449 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:04:16.449 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:04:16.449 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:04:16.449 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:04:16.449 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:04:16.450 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:04:16.450 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:04:16.450 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:04:16.450 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:04:16.450 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:04:16.450 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:04:16.450 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:04:16.450 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:04:16.450 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:04:16.450 | INFO | ------------------------------------------ | 20260507 18:04:16.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:04:16.450 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:04:16.450 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:04:16.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:04:16.450 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.43e+01 / 2.49e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 0.29%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.08e-02 / 5.52e-03 | 20260507 18:04:16.450 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.52e+00, 6.16e-01 / 2.01e-01 | 20260507 18:04:16.450 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.13e-01 / 1.12e-01 | 20260507 18:04:16.450 | INFO | PerfReport: 15.74%, DME -- 4.21e+01 / 4.49e+01, 4.10e+00 / 2.29e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.84e+00 / 2.04e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.31e-01 / 2.40e-01 | 20260507 18:04:16.450 | INFO | PerfReport: 3.74%, Metrics -- 1.25e+00 / 1.43e+00, 3.17e-01 / 1.93e-01 | 20260507 18:04:16.450 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.94e-01 / 4.17e-01, 6.38e-02 / 4.13e-02 | 20260507 18:04:16.450 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:04:16.450 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:04:16.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:04:16.450 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 24.17%, Upload -- 2.60e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 23.97%, Download -- 2.58e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 31.08%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 2.02%, LockUpload -- 2.18e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 2.19%, LockDownload -- 2.36e+00 / 0.00e+00, 3.72e+00 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 16.55%, LockCompute -- 1.78e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:04:16.450 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260507 18:04:16.557 | INFO | Starting a round of Baz IO | 20260507 18:04:20.052 | INFO | Populating RTMetrics report | 20260507 18:04:20.052 | INFO | RTMetrics report populated | 20260507 18:04:20.052 | INFO | Opening tmp RTMetrics json file | 20260507 18:04:20.066 | INFO | Serializing RTMetrics to json | 20260507 18:04:20.071 | INFO | Serialization done, swapping tmp file now... | 20260507 18:04:20.071 | INFO | RTMetrics done | 20260507 18:04:20.131 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.574 seconds | 20260507 18:04:20.131 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.14% / 0.02% / 52.85% | 20260507 18:04:20.131 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.16428% / 7.94729e-06% / 2.78155e-05% | 20260507 18:04:20.131 | INFO | Baz IO overall: | 20260507 18:04:20.131 | INFO | BytesWritten : 670378147245 | 20260507 18:04:20.131 | INFO | NumEvents : 324949248145 | 20260507 18:04:21.588 | INFO | Analyzing chunk frames = [1507328,1507840) | 20260507 18:04:21.588 | INFO | MemInfo: { | 20260507 18:04:21.588 | INFO | "gpu" : " 17 GB", | 20260507 18:04:21.588 | INFO | "huge" : "", | 20260507 18:04:21.588 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:21.588 | INFO | "malloc" : " 14 GB", | 20260507 18:04:21.588 | INFO | "pinned" : " 29 GB" | 20260507 18:04:21.588 | INFO | } | 20260507 18:04:21.588 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:04:21.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:21.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:21.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:21.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:21.588 | INFO | ANALYSIS : 69.4%, 3.0, 108.4ms, 0 | 20260507 18:04:21.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:21.588 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 18:04:21.588 | INFO | PRE_HQ : 66.5%, 1.7, 59.8ms, 0 | 20260507 18:04:21.588 | INFO | BAZWRITER : 68.8%, 2.9, 100.8ms, 0 | 20260507 18:04:21.588 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:04:21.588 | INFO | Uploaded 36GiB to GPU in 2.48716s (14.4743GiB/s) | 20260507 18:04:21.588 | INFO | Downloaded 38GiB to GPU in 2.71243s (14.0096GiB/s) | 20260507 18:04:26.727 | INFO | Analyzing chunk frames = [1507840,1508352) | 20260507 18:04:26.727 | INFO | MemInfo: { | 20260507 18:04:26.727 | INFO | "gpu" : " 17 GB", | 20260507 18:04:26.727 | INFO | "huge" : "", | 20260507 18:04:26.727 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:26.727 | INFO | "malloc" : " 14 GB", | 20260507 18:04:26.727 | INFO | "pinned" : " 29 GB" | 20260507 18:04:26.727 | INFO | } | 20260507 18:04:26.727 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:04:26.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:26.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:04:26.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:04:26.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:26.727 | INFO | Uploaded 36GiB to GPU in 2.52423s (14.2618GiB/s) | 20260507 18:04:26.727 | INFO | Downloaded 34GiB to GPU in 2.45443s (13.8525GiB/s) | 20260507 18:04:31.866 | INFO | Analyzing chunk frames = [1508352,1508864) | 20260507 18:04:31.866 | INFO | MemInfo: { | 20260507 18:04:31.866 | INFO | "gpu" : " 17 GB", | 20260507 18:04:31.866 | INFO | "huge" : "", | 20260507 18:04:31.866 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:31.866 | INFO | "malloc" : " 14 GB", | 20260507 18:04:31.866 | INFO | "pinned" : " 29 GB" | 20260507 18:04:31.866 | INFO | } | 20260507 18:04:31.866 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:04:31.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:31.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.866 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 18:04:31.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.867 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:04:31.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:31.867 | INFO | Uploaded 36GiB to GPU in 2.52023s (14.2844GiB/s) | 20260507 18:04:31.867 | INFO | Downloaded 34GiB to GPU in 2.44585s (13.9011GiB/s) | 20260507 18:04:37.005 | INFO | Analyzing chunk frames = [1508864,1509376) | 20260507 18:04:37.005 | INFO | MemInfo: { | 20260507 18:04:37.005 | INFO | "gpu" : " 17 GB", | 20260507 18:04:37.005 | INFO | "huge" : "", | 20260507 18:04:37.005 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:37.006 | INFO | "malloc" : " 14 GB", | 20260507 18:04:37.006 | INFO | "pinned" : " 29 GB" | 20260507 18:04:37.006 | INFO | } | 20260507 18:04:37.006 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:04:37.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:37.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:04:37.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:04:37.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:37.006 | INFO | Uploaded 36GiB to GPU in 2.48538s (14.4847GiB/s) | 20260507 18:04:37.006 | INFO | Downloaded 34GiB to GPU in 2.34338s (14.509GiB/s) | 20260507 18:04:42.145 | INFO | Analyzing chunk frames = [1509376,1509888) | 20260507 18:04:42.145 | INFO | MemInfo: { | 20260507 18:04:42.145 | INFO | "gpu" : " 17 GB", | 20260507 18:04:42.145 | INFO | "huge" : "", | 20260507 18:04:42.145 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:42.145 | INFO | "malloc" : " 14 GB", | 20260507 18:04:42.145 | INFO | "pinned" : " 29 GB" | 20260507 18:04:42.145 | INFO | } | 20260507 18:04:42.145 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:04:42.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:42.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:04:42.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:04:42.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:42.145 | INFO | Uploaded 36GiB to GPU in 2.34181s (15.3727GiB/s) | 20260507 18:04:42.146 | INFO | Downloaded 34GiB to GPU in 2.47472s (13.7389GiB/s) | 20260507 18:04:47.284 | INFO | Analyzing chunk frames = [1509888,1510400) | 20260507 18:04:47.284 | INFO | MemInfo: { | 20260507 18:04:47.284 | INFO | "gpu" : " 17 GB", | 20260507 18:04:47.284 | INFO | "huge" : "", | 20260507 18:04:47.284 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:47.284 | INFO | "malloc" : " 14 GB", | 20260507 18:04:47.284 | INFO | "pinned" : " 29 GB" | 20260507 18:04:47.284 | INFO | } | 20260507 18:04:47.284 | INFO | Perf report for the last 5.139 seconds: | 20260507 18:04:47.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:47.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.284 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:04:47.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.284 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:04:47.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:47.285 | INFO | Uploaded 36GiB to GPU in 2.43496s (14.7847GiB/s) | 20260507 18:04:47.285 | INFO | Downloaded 34GiB to GPU in 2.48887s (13.6608GiB/s) | 20260507 18:04:52.424 | INFO | Analyzing chunk frames = [1510400,1510912) | 20260507 18:04:52.424 | INFO | MemInfo: { | 20260507 18:04:52.424 | INFO | "gpu" : " 17 GB", | 20260507 18:04:52.424 | INFO | "huge" : "", | 20260507 18:04:52.424 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:52.424 | INFO | "malloc" : " 14 GB", | 20260507 18:04:52.424 | INFO | "pinned" : " 29 GB" | 20260507 18:04:52.424 | INFO | } | 20260507 18:04:52.424 | INFO | Perf report for the last 5.13973 seconds: | 20260507 18:04:52.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:52.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.424 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:04:52.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.424 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260507 18:04:52.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:52.425 | INFO | Uploaded 36GiB to GPU in 2.51903s (14.2912GiB/s) | 20260507 18:04:52.425 | INFO | Downloaded 34GiB to GPU in 2.42941s (13.9951GiB/s) | 20260507 18:04:57.563 | INFO | Analyzing chunk frames = [1510912,1511424) | 20260507 18:04:57.563 | INFO | MemInfo: { | 20260507 18:04:57.563 | INFO | "gpu" : " 17 GB", | 20260507 18:04:57.563 | INFO | "huge" : "", | 20260507 18:04:57.563 | INFO | "huge_shared" : " 29 GB", | 20260507 18:04:57.563 | INFO | "malloc" : " 14 GB", | 20260507 18:04:57.563 | INFO | "pinned" : " 29 GB" | 20260507 18:04:57.563 | INFO | } | 20260507 18:04:57.563 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:04:57.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:04:57.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:04:57.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 18:04:57.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:04:57.564 | INFO | Uploaded 36GiB to GPU in 2.53014s (14.2285GiB/s) | 20260507 18:04:57.564 | INFO | Downloaded 34GiB to GPU in 2.42088s (14.0445GiB/s) | 20260507 18:04:57.691 | INFO | Starting a round of Baz IO | 20260507 18:05:01.157 | INFO | Populating RTMetrics report | 20260507 18:05:01.158 | INFO | RTMetrics report populated | 20260507 18:05:01.158 | INFO | Opening tmp RTMetrics json file | 20260507 18:05:01.174 | INFO | Serializing RTMetrics to json | 20260507 18:05:01.176 | INFO | Serialization done, swapping tmp file now... | 20260507 18:05:01.177 | INFO | RTMetrics done | 20260507 18:05:01.236 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.545 seconds | 20260507 18:05:01.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.12% / 0.02% / 52.86% | 20260507 18:05:01.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.16636% / 2.78155e-05% / 4.37101e-05% | 20260507 18:05:01.237 | INFO | Baz IO overall: | 20260507 18:05:01.237 | INFO | BytesWritten : 672315205382 | 20260507 18:05:01.237 | INFO | NumEvents : 325890542149 | 20260507 18:05:02.702 | INFO | Analyzing chunk frames = [1511424,1511936) | 20260507 18:05:02.702 | INFO | MemInfo: { | 20260507 18:05:02.702 | INFO | "gpu" : " 17 GB", | 20260507 18:05:02.702 | INFO | "huge" : "", | 20260507 18:05:02.702 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:02.702 | INFO | "malloc" : " 14 GB", | 20260507 18:05:02.702 | INFO | "pinned" : " 29 GB" | 20260507 18:05:02.702 | INFO | } | 20260507 18:05:02.702 | INFO | Perf report for the last 5.13887 seconds: | 20260507 18:05:02.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:02.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:02.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:02.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:02.702 | INFO | ANALYSIS : 69.2%, 3.0, 107.2ms, 0 | 20260507 18:05:02.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:02.702 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 18:05:02.702 | INFO | PRE_HQ : 66.5%, 1.9, 64.5ms, 0 | 20260507 18:05:02.702 | INFO | BAZWRITER : 68.6%, 3.0, 105.9ms, 0 | 20260507 18:05:02.702 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:05:02.703 | INFO | Uploaded 36GiB to GPU in 2.51219s (14.3302GiB/s) | 20260507 18:05:02.703 | INFO | Downloaded 38GiB to GPU in 2.68652s (14.1447GiB/s) | 20260507 18:05:07.841 | INFO | Analyzing chunk frames = [1511936,1512448) | 20260507 18:05:07.841 | INFO | MemInfo: { | 20260507 18:05:07.841 | INFO | "gpu" : " 17 GB", | 20260507 18:05:07.841 | INFO | "huge" : "", | 20260507 18:05:07.841 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:07.841 | INFO | "malloc" : " 14 GB", | 20260507 18:05:07.842 | INFO | "pinned" : " 29 GB" | 20260507 18:05:07.842 | INFO | } | 20260507 18:05:07.842 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:05:07.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:07.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:05:07.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | PRE_HQ : 26.5%, 1.0, 13.9ms, 0 | 20260507 18:05:07.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:07.842 | INFO | Uploaded 36GiB to GPU in 2.51736s (14.3007GiB/s) | 20260507 18:05:07.842 | INFO | Downloaded 34GiB to GPU in 2.41595s (14.0731GiB/s) | 20260507 18:05:12.981 | INFO | Analyzing chunk frames = [1512448,1512960) | 20260507 18:05:12.981 | INFO | MemInfo: { | 20260507 18:05:12.981 | INFO | "gpu" : " 17 GB", | 20260507 18:05:12.981 | INFO | "huge" : "", | 20260507 18:05:12.981 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:12.981 | INFO | "malloc" : " 14 GB", | 20260507 18:05:12.981 | INFO | "pinned" : " 29 GB" | 20260507 18:05:12.981 | INFO | } | 20260507 18:05:12.981 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:05:12.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:12.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260507 18:05:12.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:05:12.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:12.981 | INFO | Uploaded 36GiB to GPU in 2.57989s (13.9541GiB/s) | 20260507 18:05:12.982 | INFO | Downloaded 34GiB to GPU in 2.4084s (14.1172GiB/s) | 20260507 18:05:18.120 | INFO | Analyzing chunk frames = [1512960,1513472) | 20260507 18:05:18.120 | INFO | MemInfo: { | 20260507 18:05:18.120 | INFO | "gpu" : " 17 GB", | 20260507 18:05:18.120 | INFO | "huge" : "", | 20260507 18:05:18.120 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:18.120 | INFO | "malloc" : " 14 GB", | 20260507 18:05:18.120 | INFO | "pinned" : " 29 GB" | 20260507 18:05:18.121 | INFO | } | 20260507 18:05:18.121 | INFO | Perf report for the last 5.13998 seconds: | 20260507 18:05:18.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:18.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 18:05:18.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:05:18.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:18.121 | INFO | Uploaded 36GiB to GPU in 2.50839s (14.3518GiB/s) | 20260507 18:05:18.121 | INFO | Downloaded 34GiB to GPU in 2.34298s (14.5115GiB/s) | 20260507 18:05:23.260 | INFO | Analyzing chunk frames = [1513472,1513984) | 20260507 18:05:23.260 | INFO | MemInfo: { | 20260507 18:05:23.260 | INFO | "gpu" : " 17 GB", | 20260507 18:05:23.260 | INFO | "huge" : "", | 20260507 18:05:23.260 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:23.260 | INFO | "malloc" : " 14 GB", | 20260507 18:05:23.260 | INFO | "pinned" : " 29 GB" | 20260507 18:05:23.260 | INFO | } | 20260507 18:05:23.260 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:05:23.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:23.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.260 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:05:23.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.260 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:05:23.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:23.261 | INFO | Uploaded 36GiB to GPU in 2.34264s (15.3673GiB/s) | 20260507 18:05:23.261 | INFO | Downloaded 34GiB to GPU in 2.46303s (13.8041GiB/s) | 20260507 18:05:28.399 | INFO | Analyzing chunk frames = [1513984,1514496) | 20260507 18:05:28.399 | INFO | MemInfo: { | 20260507 18:05:28.399 | INFO | "gpu" : " 17 GB", | 20260507 18:05:28.399 | INFO | "huge" : "", | 20260507 18:05:28.399 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:28.399 | INFO | "malloc" : " 14 GB", | 20260507 18:05:28.399 | INFO | "pinned" : " 29 GB" | 20260507 18:05:28.399 | INFO | } | 20260507 18:05:28.399 | INFO | Perf report for the last 5.13892 seconds: | 20260507 18:05:28.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:28.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:05:28.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:05:28.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:28.399 | INFO | Uploaded 36GiB to GPU in 2.42601s (14.8392GiB/s) | 20260507 18:05:28.400 | INFO | Downloaded 34GiB to GPU in 2.50733s (13.5603GiB/s) | 20260507 18:05:33.538 | INFO | Analyzing chunk frames = [1514496,1515008) | 20260507 18:05:33.538 | INFO | MemInfo: { | 20260507 18:05:33.538 | INFO | "gpu" : " 17 GB", | 20260507 18:05:33.538 | INFO | "huge" : "", | 20260507 18:05:33.538 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:33.538 | INFO | "malloc" : " 14 GB", | 20260507 18:05:33.538 | INFO | "pinned" : " 29 GB" | 20260507 18:05:33.538 | INFO | } | 20260507 18:05:33.538 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:05:33.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:33.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:05:33.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 18:05:33.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:33.539 | INFO | Uploaded 36GiB to GPU in 2.48078s (14.5115GiB/s) | 20260507 18:05:33.539 | INFO | Downloaded 34GiB to GPU in 2.43033s (13.9899GiB/s) | 20260507 18:05:38.678 | INFO | Analyzing chunk frames = [1515008,1515520) | 20260507 18:05:38.678 | INFO | MemInfo: { | 20260507 18:05:38.678 | INFO | "gpu" : " 17 GB", | 20260507 18:05:38.678 | INFO | "huge" : "", | 20260507 18:05:38.678 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:38.678 | INFO | "malloc" : " 14 GB", | 20260507 18:05:38.678 | INFO | "pinned" : " 29 GB" | 20260507 18:05:38.678 | INFO | } | 20260507 18:05:38.678 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:05:38.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:38.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 18:05:38.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:05:38.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:38.678 | INFO | Uploaded 36GiB to GPU in 2.53763s (14.1864GiB/s) | 20260507 18:05:38.678 | INFO | Downloaded 34GiB to GPU in 2.50538s (13.5708GiB/s) | 20260507 18:05:38.678 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:05:38.678 | INFO | Total mem usage: 16.4 GiB | 20260507 18:05:38.678 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:05:38.678 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 18:05:38.678 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:05:38.678 | INFO | High Water Breakdown: | 20260507 18:05:38.678 | INFO | 59.3% : 8.64 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:05:38.678 | INFO | 40.6% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:05:38.678 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:05:38.678 | INFO | ------------------------------------------ | 20260507 18:05:38.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:05:38.678 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:05:38.678 | INFO | Total mem usage: 26.5 GiB | 20260507 18:05:38.678 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:05:38.679 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:05:38.679 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:05:38.679 | INFO | High Water Breakdown: | 20260507 18:05:38.679 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:05:38.679 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:05:38.679 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:05:38.679 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:05:38.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:05:38.679 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:05:38.679 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:05:38.679 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:05:38.679 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:05:38.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:05:38.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:05:38.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:05:38.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:05:38.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:05:38.679 | INFO | ------------------------------------------ | 20260507 18:05:38.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:05:38.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:05:38.679 | INFO | Total mem usage: 27.8 GiB | 20260507 18:05:38.679 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:05:38.679 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:05:38.679 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:05:38.679 | INFO | High Water Breakdown: | 20260507 18:05:38.679 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:05:38.679 | INFO | ------------------------------------------ | 20260507 18:05:38.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:05:38.679 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:05:38.679 | INFO | Total mem usage: 15.7 GiB | 20260507 18:05:38.679 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:05:38.679 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:05:38.679 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 18:05:38.679 | INFO | High Water Breakdown: | 20260507 18:05:38.679 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:05:38.679 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:05:38.679 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:05:38.679 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:05:38.679 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:05:38.679 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:05:38.679 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:05:38.679 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:05:38.679 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:05:38.679 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:05:38.679 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:05:38.679 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:05:38.679 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:05:38.679 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:05:38.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:05:38.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:05:38.679 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:05:38.679 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:05:38.679 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:05:38.679 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:05:38.679 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:05:38.679 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:05:38.679 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:05:38.679 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:05:38.679 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:05:38.679 | INFO | ------------------------------------------ | 20260507 18:05:38.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:05:38.679 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:05:38.679 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:05:38.679 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:05:38.679 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.42e+01 / 2.36e+00 | 20260507 18:05:38.679 | INFO | PerfReport: 0.29%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.14e-02 / 7.70e-03 | 20260507 18:05:38.679 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.52e+00, 6.19e-01 / 2.06e-01 | 20260507 18:05:38.679 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.15e-01 / 1.10e-01 | 20260507 18:05:38.679 | INFO | PerfReport: 15.72%, DME -- 4.21e+01 / 4.50e+01, 4.29e+00 / 2.34e+00 | 20260507 18:05:38.679 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.87e+00 / 1.87e+00 | 20260507 18:05:38.679 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.50e+00 / 2.86e+00, 4.33e-01 / 2.25e-01 | 20260507 18:05:38.679 | INFO | PerfReport: 3.75%, Metrics -- 1.25e+00 / 1.45e+00, 3.20e-01 / 1.64e-01 | 20260507 18:05:38.679 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.94e-01 / 4.23e-01, 6.39e-02 / 3.22e-02 | 20260507 18:05:38.679 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:05:38.679 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:05:38.679 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:05:38.679 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 18:05:38.679 | INFO | PerfReport: 24.09%, Upload -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 18:05:38.680 | INFO | PerfReport: 23.96%, Download -- 2.57e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 18:05:38.680 | INFO | PerfReport: 31.17%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:05:38.680 | INFO | PerfReport: 1.94%, LockUpload -- 2.09e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 18:05:38.680 | INFO | PerfReport: 2.22%, LockDownload -- 2.39e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 18:05:38.680 | INFO | PerfReport: 16.60%, LockCompute -- 1.78e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:05:38.680 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 18:05:38.775 | INFO | Starting a round of Baz IO | 20260507 18:05:42.286 | INFO | Populating RTMetrics report | 20260507 18:05:42.286 | INFO | RTMetrics report populated | 20260507 18:05:42.286 | INFO | Opening tmp RTMetrics json file | 20260507 18:05:42.287 | INFO | Serializing RTMetrics to json | 20260507 18:05:42.288 | INFO | Serialization done, swapping tmp file now... | 20260507 18:05:42.288 | INFO | RTMetrics done | 20260507 18:05:42.361 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.586 seconds | 20260507 18:05:42.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.10% / 0.02% / 52.88% | 20260507 18:05:42.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.15293% / 3.97364e-05% / 1.19209e-05% | 20260507 18:05:42.361 | INFO | Baz IO overall: | 20260507 18:05:42.361 | INFO | BytesWritten : 674250259015 | 20260507 18:05:42.361 | INFO | NumEvents : 326830407778 | 20260507 18:05:43.817 | INFO | Analyzing chunk frames = [1515520,1516032) | 20260507 18:05:43.817 | INFO | MemInfo: { | 20260507 18:05:43.817 | INFO | "gpu" : " 17 GB", | 20260507 18:05:43.817 | INFO | "huge" : "", | 20260507 18:05:43.817 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:43.817 | INFO | "malloc" : " 14 GB", | 20260507 18:05:43.817 | INFO | "pinned" : " 29 GB" | 20260507 18:05:43.817 | INFO | } | 20260507 18:05:43.817 | INFO | Perf report for the last 5.1388 seconds: | 20260507 18:05:43.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:43.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:43.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:43.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:43.817 | INFO | ANALYSIS : 69.4%, 3.0, 108.3ms, 0 | 20260507 18:05:43.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:43.817 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 18:05:43.817 | INFO | PRE_HQ : 66.4%, 1.7, 59.0ms, 0 | 20260507 18:05:43.817 | INFO | BAZWRITER : 68.9%, 2.9, 102.6ms, 0 | 20260507 18:05:43.817 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:05:43.817 | INFO | Uploaded 36GiB to GPU in 2.44715s (14.711GiB/s) | 20260507 18:05:43.817 | INFO | Downloaded 38GiB to GPU in 2.72568s (13.9415GiB/s) | 20260507 18:05:48.956 | INFO | Analyzing chunk frames = [1516032,1516544) | 20260507 18:05:48.956 | INFO | MemInfo: { | 20260507 18:05:48.956 | INFO | "gpu" : " 17 GB", | 20260507 18:05:48.956 | INFO | "huge" : "", | 20260507 18:05:48.956 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:48.956 | INFO | "malloc" : " 14 GB", | 20260507 18:05:48.960 | INFO | "pinned" : " 29 GB" | 20260507 18:05:48.960 | INFO | } | 20260507 18:05:48.960 | INFO | Perf report for the last 5.14336 seconds: | 20260507 18:05:48.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:48.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:05:48.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:05:48.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:48.960 | INFO | Uploaded 36GiB to GPU in 2.52308s (14.2683GiB/s) | 20260507 18:05:48.961 | INFO | Downloaded 34GiB to GPU in 2.45149s (13.8691GiB/s) | 20260507 18:05:54.096 | INFO | Analyzing chunk frames = [1516544,1517056) | 20260507 18:05:54.096 | INFO | MemInfo: { | 20260507 18:05:54.096 | INFO | "gpu" : " 17 GB", | 20260507 18:05:54.096 | INFO | "huge" : "", | 20260507 18:05:54.096 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:54.096 | INFO | "malloc" : " 14 GB", | 20260507 18:05:54.096 | INFO | "pinned" : " 29 GB" | 20260507 18:05:54.096 | INFO | } | 20260507 18:05:54.099 | INFO | Perf report for the last 5.13906 seconds: | 20260507 18:05:54.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:54.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 18:05:54.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:05:54.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:54.099 | INFO | Uploaded 36GiB to GPU in 2.52637s (14.2497GiB/s) | 20260507 18:05:54.100 | INFO | Downloaded 34GiB to GPU in 2.4386s (13.9424GiB/s) | 20260507 18:05:59.235 | INFO | Analyzing chunk frames = [1517056,1517568) | 20260507 18:05:59.235 | INFO | MemInfo: { | 20260507 18:05:59.235 | INFO | "gpu" : " 17 GB", | 20260507 18:05:59.235 | INFO | "huge" : "", | 20260507 18:05:59.235 | INFO | "huge_shared" : " 29 GB", | 20260507 18:05:59.235 | INFO | "malloc" : " 14 GB", | 20260507 18:05:59.235 | INFO | "pinned" : " 29 GB" | 20260507 18:05:59.235 | INFO | } | 20260507 18:05:59.235 | INFO | Perf report for the last 5.1361 seconds: | 20260507 18:05:59.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:05:59.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.235 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:05:59.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.235 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:05:59.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:05:59.236 | INFO | Uploaded 36GiB to GPU in 2.49421s (14.4334GiB/s) | 20260507 18:05:59.236 | INFO | Downloaded 34GiB to GPU in 2.3544s (14.441GiB/s) | 20260507 18:06:04.375 | INFO | Analyzing chunk frames = [1517568,1518080) | 20260507 18:06:04.375 | INFO | MemInfo: { | 20260507 18:06:04.375 | INFO | "gpu" : " 17 GB", | 20260507 18:06:04.375 | INFO | "huge" : "", | 20260507 18:06:04.375 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:04.375 | INFO | "malloc" : " 14 GB", | 20260507 18:06:04.375 | INFO | "pinned" : " 29 GB" | 20260507 18:06:04.375 | INFO | } | 20260507 18:06:04.375 | INFO | Perf report for the last 5.13994 seconds: | 20260507 18:06:04.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:04.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.375 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:06:04.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.375 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 18:06:04.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:04.376 | INFO | Uploaded 36GiB to GPU in 2.35097s (15.3128GiB/s) | 20260507 18:06:04.376 | INFO | Downloaded 34GiB to GPU in 2.4549s (13.8499GiB/s) | 20260507 18:06:09.514 | INFO | Analyzing chunk frames = [1518080,1518592) | 20260507 18:06:09.514 | INFO | MemInfo: { | 20260507 18:06:09.514 | INFO | "gpu" : " 17 GB", | 20260507 18:06:09.514 | INFO | "huge" : "", | 20260507 18:06:09.514 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:09.514 | INFO | "malloc" : " 14 GB", | 20260507 18:06:09.514 | INFO | "pinned" : " 29 GB" | 20260507 18:06:09.514 | INFO | } | 20260507 18:06:09.514 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:06:09.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:09.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.514 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:06:09.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.514 | INFO | PRE_HQ : 35.1%, 1.0, 18.3ms, 0 | 20260507 18:06:09.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:09.515 | INFO | Uploaded 36GiB to GPU in 2.45732s (14.6501GiB/s) | 20260507 18:06:09.515 | INFO | Downloaded 34GiB to GPU in 2.48842s (13.6633GiB/s) | 20260507 18:06:14.653 | INFO | Analyzing chunk frames = [1518592,1519104) | 20260507 18:06:14.653 | INFO | MemInfo: { | 20260507 18:06:14.653 | INFO | "gpu" : " 17 GB", | 20260507 18:06:14.653 | INFO | "huge" : "", | 20260507 18:06:14.653 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:14.653 | INFO | "malloc" : " 14 GB", | 20260507 18:06:14.653 | INFO | "pinned" : " 29 GB" | 20260507 18:06:14.653 | INFO | } | 20260507 18:06:14.653 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:06:14.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:14.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:06:14.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 18:06:14.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:14.654 | INFO | Uploaded 36GiB to GPU in 2.52991s (14.2297GiB/s) | 20260507 18:06:14.654 | INFO | Downloaded 34GiB to GPU in 2.42598s (14.015GiB/s) | 20260507 18:06:19.792 | INFO | Analyzing chunk frames = [1519104,1519616) | 20260507 18:06:19.792 | INFO | MemInfo: { | 20260507 18:06:19.792 | INFO | "gpu" : " 17 GB", | 20260507 18:06:19.792 | INFO | "huge" : "", | 20260507 18:06:19.792 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:19.792 | INFO | "malloc" : " 14 GB", | 20260507 18:06:19.792 | INFO | "pinned" : " 29 GB" | 20260507 18:06:19.792 | INFO | } | 20260507 18:06:19.793 | INFO | Perf report for the last 5.13904 seconds: | 20260507 18:06:19.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:19.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:06:19.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 18:06:19.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:19.793 | INFO | Uploaded 36GiB to GPU in 2.53146s (14.221GiB/s) | 20260507 18:06:19.793 | INFO | Downloaded 34GiB to GPU in 2.4142s (14.0833GiB/s) | 20260507 18:06:19.905 | INFO | Starting a round of Baz IO | 20260507 18:06:23.401 | INFO | Populating RTMetrics report | 20260507 18:06:23.402 | INFO | RTMetrics report populated | 20260507 18:06:23.402 | INFO | Opening tmp RTMetrics json file | 20260507 18:06:23.419 | INFO | Serializing RTMetrics to json | 20260507 18:06:23.420 | INFO | Serialization done, swapping tmp file now... | 20260507 18:06:23.421 | INFO | RTMetrics done | 20260507 18:06:23.479 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.573 seconds | 20260507 18:06:23.479 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.09% / 0.02% / 52.90% | 20260507 18:06:23.479 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.14116% / 7.94729e-06% / 3.57628e-05% | 20260507 18:06:23.479 | INFO | Baz IO overall: | 20260507 18:06:23.479 | INFO | BytesWritten : 676184493915 | 20260507 18:06:23.479 | INFO | NumEvents : 327769844401 | 20260507 18:06:24.932 | INFO | Analyzing chunk frames = [1519616,1520128) | 20260507 18:06:24.932 | INFO | MemInfo: { | 20260507 18:06:24.932 | INFO | "gpu" : " 17 GB", | 20260507 18:06:24.932 | INFO | "huge" : "", | 20260507 18:06:24.932 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:24.932 | INFO | "malloc" : " 14 GB", | 20260507 18:06:24.932 | INFO | "pinned" : " 29 GB" | 20260507 18:06:24.932 | INFO | } | 20260507 18:06:24.932 | INFO | Perf report for the last 5.13965 seconds: | 20260507 18:06:24.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:24.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:24.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:24.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:24.932 | INFO | ANALYSIS : 69.5%, 2.9, 107.4ms, 0 | 20260507 18:06:24.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:24.932 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 18:06:24.932 | INFO | PRE_HQ : 67.2%, 1.9, 65.3ms, 0 | 20260507 18:06:24.932 | INFO | BAZWRITER : 68.8%, 3.0, 105.2ms, 0 | 20260507 18:06:24.932 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:06:24.933 | INFO | Uploaded 36GiB to GPU in 2.54546s (14.1428GiB/s) | 20260507 18:06:24.933 | INFO | Downloaded 38GiB to GPU in 2.70743s (14.0355GiB/s) | 20260507 18:06:30.071 | INFO | Analyzing chunk frames = [1520128,1520640) | 20260507 18:06:30.071 | INFO | MemInfo: { | 20260507 18:06:30.071 | INFO | "gpu" : " 17 GB", | 20260507 18:06:30.071 | INFO | "huge" : "", | 20260507 18:06:30.071 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:30.072 | INFO | "malloc" : " 14 GB", | 20260507 18:06:30.072 | INFO | "pinned" : " 29 GB" | 20260507 18:06:30.072 | INFO | } | 20260507 18:06:30.072 | INFO | Perf report for the last 5.13948 seconds: | 20260507 18:06:30.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:30.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 18:06:30.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:06:30.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:30.072 | INFO | Uploaded 36GiB to GPU in 2.55374s (14.097GiB/s) | 20260507 18:06:30.072 | INFO | Downloaded 34GiB to GPU in 2.43566s (13.9592GiB/s) | 20260507 18:06:35.210 | INFO | Analyzing chunk frames = [1520640,1521152) | 20260507 18:06:35.210 | INFO | MemInfo: { | 20260507 18:06:35.210 | INFO | "gpu" : " 17 GB", | 20260507 18:06:35.210 | INFO | "huge" : "", | 20260507 18:06:35.210 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:35.210 | INFO | "malloc" : " 14 GB", | 20260507 18:06:35.211 | INFO | "pinned" : " 29 GB" | 20260507 18:06:35.211 | INFO | } | 20260507 18:06:35.211 | INFO | Perf report for the last 5.13893 seconds: | 20260507 18:06:35.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:35.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:06:35.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:06:35.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:35.211 | INFO | Uploaded 36GiB to GPU in 2.57816s (13.9635GiB/s) | 20260507 18:06:35.211 | INFO | Downloaded 34GiB to GPU in 2.42401s (14.0263GiB/s) | 20260507 18:06:40.350 | INFO | Analyzing chunk frames = [1521152,1521664) | 20260507 18:06:40.350 | INFO | MemInfo: { | 20260507 18:06:40.350 | INFO | "gpu" : " 17 GB", | 20260507 18:06:40.350 | INFO | "huge" : "", | 20260507 18:06:40.350 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:40.350 | INFO | "malloc" : " 14 GB", | 20260507 18:06:40.350 | INFO | "pinned" : " 29 GB" | 20260507 18:06:40.350 | INFO | } | 20260507 18:06:40.350 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:06:40.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:40.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260507 18:06:40.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:06:40.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:40.350 | INFO | Uploaded 36GiB to GPU in 2.50885s (14.3492GiB/s) | 20260507 18:06:40.350 | INFO | Downloaded 34GiB to GPU in 2.34355s (14.5079GiB/s) | 20260507 18:06:45.490 | INFO | Analyzing chunk frames = [1521664,1522176) | 20260507 18:06:45.490 | INFO | MemInfo: { | 20260507 18:06:45.490 | INFO | "gpu" : " 17 GB", | 20260507 18:06:45.490 | INFO | "huge" : "", | 20260507 18:06:45.490 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:45.490 | INFO | "malloc" : " 14 GB", | 20260507 18:06:45.491 | INFO | "pinned" : " 29 GB" | 20260507 18:06:45.491 | INFO | } | 20260507 18:06:45.491 | INFO | Perf report for the last 5.1407 seconds: | 20260507 18:06:45.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:45.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:06:45.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:06:45.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:45.491 | INFO | Uploaded 36GiB to GPU in 2.36661s (15.2116GiB/s) | 20260507 18:06:45.491 | INFO | Downloaded 34GiB to GPU in 2.47218s (13.753GiB/s) | 20260507 18:06:50.628 | INFO | Analyzing chunk frames = [1522176,1522688) | 20260507 18:06:50.628 | INFO | MemInfo: { | 20260507 18:06:50.628 | INFO | "gpu" : " 17 GB", | 20260507 18:06:50.628 | INFO | "huge" : "", | 20260507 18:06:50.628 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:50.628 | INFO | "malloc" : " 14 GB", | 20260507 18:06:50.629 | INFO | "pinned" : " 29 GB" | 20260507 18:06:50.629 | INFO | } | 20260507 18:06:50.629 | INFO | Perf report for the last 5.13787 seconds: | 20260507 18:06:50.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:50.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:06:50.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:06:50.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:50.629 | INFO | Uploaded 36GiB to GPU in 2.44296s (14.7362GiB/s) | 20260507 18:06:50.629 | INFO | Downloaded 34GiB to GPU in 2.48468s (13.6839GiB/s) | 20260507 18:06:55.768 | INFO | Analyzing chunk frames = [1522688,1523200) | 20260507 18:06:55.768 | INFO | MemInfo: { | 20260507 18:06:55.768 | INFO | "gpu" : " 17 GB", | 20260507 18:06:55.768 | INFO | "huge" : "", | 20260507 18:06:55.768 | INFO | "huge_shared" : " 29 GB", | 20260507 18:06:55.768 | INFO | "malloc" : " 14 GB", | 20260507 18:06:55.768 | INFO | "pinned" : " 29 GB" | 20260507 18:06:55.768 | INFO | } | 20260507 18:06:55.768 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:06:55.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:06:55.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.768 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:06:55.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.769 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 18:06:55.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:06:55.769 | INFO | Uploaded 36GiB to GPU in 2.5186s (14.2937GiB/s) | 20260507 18:06:55.769 | INFO | Downloaded 34GiB to GPU in 2.43016s (13.9908GiB/s) | 20260507 18:07:00.907 | INFO | Analyzing chunk frames = [1523200,1523712) | 20260507 18:07:00.907 | INFO | MemInfo: { | 20260507 18:07:00.907 | INFO | "gpu" : " 17 GB", | 20260507 18:07:00.907 | INFO | "huge" : "", | 20260507 18:07:00.907 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:00.907 | INFO | "malloc" : " 14 GB", | 20260507 18:07:00.907 | INFO | "pinned" : " 29 GB" | 20260507 18:07:00.907 | INFO | } | 20260507 18:07:00.907 | INFO | Perf report for the last 5.13888 seconds: | 20260507 18:07:00.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:00.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.907 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:07:00.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.908 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 18:07:00.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:00.908 | INFO | Uploaded 36GiB to GPU in 2.52147s (14.2774GiB/s) | 20260507 18:07:00.908 | INFO | Downloaded 34GiB to GPU in 2.41416s (14.0836GiB/s) | 20260507 18:07:00.908 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:07:00.908 | INFO | Total mem usage: 16.4 GiB | 20260507 18:07:00.908 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:07:00.908 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 18:07:00.908 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:07:00.908 | INFO | High Water Breakdown: | 20260507 18:07:00.908 | INFO | 59.3% : 8.64 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:07:00.908 | INFO | 40.6% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:07:00.908 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:07:00.908 | INFO | ------------------------------------------ | 20260507 18:07:00.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:07:00.908 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:07:00.908 | INFO | Total mem usage: 26.5 GiB | 20260507 18:07:00.908 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:07:00.908 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:07:00.908 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:07:00.908 | INFO | High Water Breakdown: | 20260507 18:07:00.908 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:07:00.908 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:07:00.908 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:07:00.908 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:07:00.908 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:07:00.908 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:07:00.908 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:07:00.908 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:07:00.908 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:07:00.908 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:07:00.908 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:07:00.908 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:07:00.908 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:07:00.908 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:07:00.908 | INFO | ------------------------------------------ | 20260507 18:07:00.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:07:00.908 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:07:00.908 | INFO | Total mem usage: 27.8 GiB | 20260507 18:07:00.908 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:07:00.908 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:07:00.908 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:07:00.908 | INFO | High Water Breakdown: | 20260507 18:07:00.908 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:07:00.908 | INFO | ------------------------------------------ | 20260507 18:07:00.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:07:00.908 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:07:00.908 | INFO | Total mem usage: 15.7 GiB | 20260507 18:07:00.909 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:07:00.909 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:07:00.909 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:07:00.909 | INFO | High Water Breakdown: | 20260507 18:07:00.909 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:07:00.909 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:07:00.909 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:07:00.909 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:07:00.909 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:07:00.909 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:07:00.909 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:07:00.909 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:07:00.909 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:07:00.909 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:07:00.909 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:07:00.909 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:07:00.909 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:07:00.909 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:07:00.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:07:00.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:07:00.909 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:07:00.909 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:07:00.909 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:07:00.909 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:07:00.909 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:07:00.909 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:07:00.909 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:07:00.909 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:07:00.909 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:07:00.909 | INFO | ------------------------------------------ | 20260507 18:07:00.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:07:00.909 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:07:00.909 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:07:00.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:07:00.909 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.42e+01 / 2.37e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 0.30%, AggregateStats -- 9.97e-02 / 1.05e-01, 1.16e-02 / 8.26e-03 | 20260507 18:07:00.909 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.50e+00, 6.12e-01 / 2.56e-01 | 20260507 18:07:00.909 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.15e-01 / 1.09e-01 | 20260507 18:07:00.909 | INFO | PerfReport: 15.66%, DME -- 4.19e+01 / 4.48e+01, 3.98e+00 / 2.29e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.92e+00 / 2.01e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.49e+00 / 2.86e+00, 4.31e-01 / 2.38e-01 | 20260507 18:07:00.909 | INFO | PerfReport: 3.74%, Metrics -- 1.25e+00 / 1.45e+00, 3.17e-01 / 1.82e-01 | 20260507 18:07:00.909 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.95e-01 / 4.22e-01, 6.39e-02 / 3.55e-02 | 20260507 18:07:00.909 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:07:00.909 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:07:00.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:07:00.909 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 24.16%, Upload -- 2.60e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 23.90%, Download -- 2.57e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 31.14%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 2.00%, LockUpload -- 2.15e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 2.15%, LockDownload -- 2.31e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 16.63%, LockCompute -- 1.79e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:07:00.909 | INFO | PerfReport: 0.02%, Misc -- 1.65e-02 / 0.00e+00, 3.02e-02 / 0.00e+00 | 20260507 18:07:01.004 | INFO | Starting a round of Baz IO | 20260507 18:07:04.508 | INFO | Populating RTMetrics report | 20260507 18:07:04.509 | INFO | RTMetrics report populated | 20260507 18:07:04.509 | INFO | Opening tmp RTMetrics json file | 20260507 18:07:04.509 | INFO | Serializing RTMetrics to json | 20260507 18:07:04.511 | INFO | Serialization done, swapping tmp file now... | 20260507 18:07:04.512 | INFO | RTMetrics done | 20260507 18:07:04.586 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.581 seconds | 20260507 18:07:04.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.07% / 0.02% / 52.91% | 20260507 18:07:04.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.14169% / 1.19209e-05% / 3.97364e-05% | 20260507 18:07:04.586 | INFO | Baz IO overall: | 20260507 18:07:04.586 | INFO | BytesWritten : 678120198937 | 20260507 18:07:04.586 | INFO | NumEvents : 328710431270 | 20260507 18:07:06.046 | INFO | Analyzing chunk frames = [1523712,1524224) | 20260507 18:07:06.046 | INFO | MemInfo: { | 20260507 18:07:06.046 | INFO | "gpu" : " 17 GB", | 20260507 18:07:06.046 | INFO | "huge" : "", | 20260507 18:07:06.046 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:06.046 | INFO | "malloc" : " 14 GB", | 20260507 18:07:06.046 | INFO | "pinned" : " 29 GB" | 20260507 18:07:06.046 | INFO | } | 20260507 18:07:06.046 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:07:06.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:06.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:06.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:06.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:06.047 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 18:07:06.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:06.047 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 18:07:06.047 | INFO | PRE_HQ : 66.8%, 1.7, 59.5ms, 0 | 20260507 18:07:06.047 | INFO | BAZWRITER : 68.8%, 2.8, 99.4ms, 0 | 20260507 18:07:06.047 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:07:06.047 | INFO | Uploaded 36GiB to GPU in 2.5805s (13.9508GiB/s) | 20260507 18:07:06.047 | INFO | Downloaded 38GiB to GPU in 2.72712s (13.9341GiB/s) | 20260507 18:07:11.186 | INFO | Analyzing chunk frames = [1524224,1524736) | 20260507 18:07:11.186 | INFO | MemInfo: { | 20260507 18:07:11.186 | INFO | "gpu" : " 17 GB", | 20260507 18:07:11.186 | INFO | "huge" : "", | 20260507 18:07:11.186 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:11.186 | INFO | "malloc" : " 14 GB", | 20260507 18:07:11.186 | INFO | "pinned" : " 29 GB" | 20260507 18:07:11.186 | INFO | } | 20260507 18:07:11.186 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:07:11.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:11.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260507 18:07:11.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:07:11.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:11.186 | INFO | Uploaded 36GiB to GPU in 2.54141s (14.1653GiB/s) | 20260507 18:07:11.186 | INFO | Downloaded 34GiB to GPU in 2.44234s (13.9211GiB/s) | 20260507 18:07:16.325 | INFO | Analyzing chunk frames = [1524736,1525248) | 20260507 18:07:16.325 | INFO | MemInfo: { | 20260507 18:07:16.325 | INFO | "gpu" : " 17 GB", | 20260507 18:07:16.325 | INFO | "huge" : "", | 20260507 18:07:16.325 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:16.325 | INFO | "malloc" : " 14 GB", | 20260507 18:07:16.325 | INFO | "pinned" : " 29 GB" | 20260507 18:07:16.325 | INFO | } | 20260507 18:07:16.325 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:07:16.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:16.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.325 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:07:16.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.325 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:07:16.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:16.326 | INFO | Uploaded 36GiB to GPU in 2.56723s (14.0229GiB/s) | 20260507 18:07:16.326 | INFO | Downloaded 34GiB to GPU in 2.42253s (14.0349GiB/s) | 20260507 18:07:21.464 | INFO | Analyzing chunk frames = [1525248,1525760) | 20260507 18:07:21.465 | INFO | MemInfo: { | 20260507 18:07:21.465 | INFO | "gpu" : " 17 GB", | 20260507 18:07:21.465 | INFO | "huge" : "", | 20260507 18:07:21.465 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:21.465 | INFO | "malloc" : " 14 GB", | 20260507 18:07:21.465 | INFO | "pinned" : " 29 GB" | 20260507 18:07:21.465 | INFO | } | 20260507 18:07:21.465 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:07:21.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:21.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:07:21.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:07:21.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:21.465 | INFO | Uploaded 36GiB to GPU in 2.49786s (14.4123GiB/s) | 20260507 18:07:21.465 | INFO | Downloaded 34GiB to GPU in 2.34173s (14.5192GiB/s) | 20260507 18:07:26.604 | INFO | Analyzing chunk frames = [1525760,1526272) | 20260507 18:07:26.604 | INFO | MemInfo: { | 20260507 18:07:26.604 | INFO | "gpu" : " 17 GB", | 20260507 18:07:26.604 | INFO | "huge" : "", | 20260507 18:07:26.604 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:26.604 | INFO | "malloc" : " 14 GB", | 20260507 18:07:26.605 | INFO | "pinned" : " 29 GB" | 20260507 18:07:26.605 | INFO | } | 20260507 18:07:26.605 | INFO | Perf report for the last 5.13984 seconds: | 20260507 18:07:26.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:26.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:07:26.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:07:26.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:26.605 | INFO | Uploaded 36GiB to GPU in 2.31822s (15.5291GiB/s) | 20260507 18:07:26.605 | INFO | Downloaded 34GiB to GPU in 2.4772s (13.7252GiB/s) | 20260507 18:07:31.743 | INFO | Analyzing chunk frames = [1526272,1526784) | 20260507 18:07:31.743 | INFO | MemInfo: { | 20260507 18:07:31.743 | INFO | "gpu" : " 17 GB", | 20260507 18:07:31.743 | INFO | "huge" : "", | 20260507 18:07:31.743 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:31.743 | INFO | "malloc" : " 14 GB", | 20260507 18:07:31.744 | INFO | "pinned" : " 29 GB" | 20260507 18:07:31.744 | INFO | } | 20260507 18:07:31.744 | INFO | Perf report for the last 5.13902 seconds: | 20260507 18:07:31.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:31.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:07:31.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 18:07:31.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:31.744 | INFO | Uploaded 36GiB to GPU in 2.35516s (15.2856GiB/s) | 20260507 18:07:31.744 | INFO | Downloaded 34GiB to GPU in 2.48839s (13.6634GiB/s) | 20260507 18:07:36.882 | INFO | Analyzing chunk frames = [1526784,1527296) | 20260507 18:07:36.882 | INFO | MemInfo: { | 20260507 18:07:36.882 | INFO | "gpu" : " 17 GB", | 20260507 18:07:36.882 | INFO | "huge" : "", | 20260507 18:07:36.882 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:36.882 | INFO | "malloc" : " 14 GB", | 20260507 18:07:36.883 | INFO | "pinned" : " 29 GB" | 20260507 18:07:36.883 | INFO | } | 20260507 18:07:36.883 | INFO | Perf report for the last 5.13906 seconds: | 20260507 18:07:36.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:36.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:07:36.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260507 18:07:36.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:36.883 | INFO | Uploaded 36GiB to GPU in 2.49343s (14.4379GiB/s) | 20260507 18:07:36.883 | INFO | Downloaded 34GiB to GPU in 2.43933s (13.9383GiB/s) | 20260507 18:07:42.022 | INFO | Analyzing chunk frames = [1527296,1527808) | 20260507 18:07:42.022 | INFO | MemInfo: { | 20260507 18:07:42.022 | INFO | "gpu" : " 17 GB", | 20260507 18:07:42.022 | INFO | "huge" : "", | 20260507 18:07:42.022 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:42.022 | INFO | "malloc" : " 14 GB", | 20260507 18:07:42.022 | INFO | "pinned" : " 29 GB" | 20260507 18:07:42.022 | INFO | } | 20260507 18:07:42.022 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:07:42.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:42.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 18:07:42.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:07:42.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:42.022 | INFO | Uploaded 36GiB to GPU in 2.48495s (14.4872GiB/s) | 20260507 18:07:42.022 | INFO | Downloaded 34GiB to GPU in 2.5031s (13.5832GiB/s) | 20260507 18:07:42.119 | INFO | Starting a round of Baz IO | 20260507 18:07:45.626 | INFO | Populating RTMetrics report | 20260507 18:07:45.627 | INFO | RTMetrics report populated | 20260507 18:07:45.627 | INFO | Opening tmp RTMetrics json file | 20260507 18:07:45.643 | INFO | Serializing RTMetrics to json | 20260507 18:07:45.644 | INFO | Serialization done, swapping tmp file now... | 20260507 18:07:45.645 | INFO | RTMetrics done | 20260507 18:07:45.706 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.586 seconds | 20260507 18:07:45.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.05% / 0.02% / 52.93% | 20260507 18:07:45.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.13319% / 7.94729e-06% / 2.78155e-05% | 20260507 18:07:45.706 | INFO | Baz IO overall: | 20260507 18:07:45.706 | INFO | BytesWritten : 680055880638 | 20260507 18:07:45.706 | INFO | NumEvents : 329650826551 | 20260507 18:07:47.161 | INFO | Analyzing chunk frames = [1527808,1528320) | 20260507 18:07:47.161 | INFO | MemInfo: { | 20260507 18:07:47.161 | INFO | "gpu" : " 17 GB", | 20260507 18:07:47.161 | INFO | "huge" : "", | 20260507 18:07:47.161 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:47.161 | INFO | "malloc" : " 14 GB", | 20260507 18:07:47.161 | INFO | "pinned" : " 29 GB" | 20260507 18:07:47.161 | INFO | } | 20260507 18:07:47.161 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:07:47.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:47.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:47.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:47.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:47.161 | INFO | ANALYSIS : 69.6%, 3.0, 108.5ms, 0 | 20260507 18:07:47.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:47.162 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:07:47.162 | INFO | PRE_HQ : 66.9%, 1.7, 59.4ms, 0 | 20260507 18:07:47.162 | INFO | BAZWRITER : 68.8%, 2.8, 100.3ms, 0 | 20260507 18:07:47.162 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:07:47.162 | INFO | Uploaded 36GiB to GPU in 2.5588s (14.0691GiB/s) | 20260507 18:07:47.162 | INFO | Downloaded 38GiB to GPU in 2.72735s (13.933GiB/s) | 20260507 18:07:52.300 | INFO | Analyzing chunk frames = [1528320,1528832) | 20260507 18:07:52.300 | INFO | MemInfo: { | 20260507 18:07:52.300 | INFO | "gpu" : " 17 GB", | 20260507 18:07:52.300 | INFO | "huge" : "", | 20260507 18:07:52.300 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:52.301 | INFO | "malloc" : " 14 GB", | 20260507 18:07:52.301 | INFO | "pinned" : " 29 GB" | 20260507 18:07:52.301 | INFO | } | 20260507 18:07:52.301 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:07:52.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:52.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 18:07:52.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:07:52.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:52.301 | INFO | Uploaded 36GiB to GPU in 2.44861s (14.7022GiB/s) | 20260507 18:07:52.301 | INFO | Downloaded 34GiB to GPU in 2.47027s (13.7637GiB/s) | 20260507 18:07:57.440 | INFO | Analyzing chunk frames = [1528832,1529344) | 20260507 18:07:57.440 | INFO | MemInfo: { | 20260507 18:07:57.440 | INFO | "gpu" : " 17 GB", | 20260507 18:07:57.440 | INFO | "huge" : "", | 20260507 18:07:57.440 | INFO | "huge_shared" : " 29 GB", | 20260507 18:07:57.440 | INFO | "malloc" : " 14 GB", | 20260507 18:07:57.440 | INFO | "pinned" : " 29 GB" | 20260507 18:07:57.440 | INFO | } | 20260507 18:07:57.440 | INFO | Perf report for the last 5.13956 seconds: | 20260507 18:07:57.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:07:57.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.440 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 18:07:57.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.441 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:07:57.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:07:57.441 | INFO | Uploaded 36GiB to GPU in 2.49683s (14.4183GiB/s) | 20260507 18:07:57.441 | INFO | Downloaded 34GiB to GPU in 2.45066s (13.8738GiB/s) | 20260507 18:08:02.580 | INFO | Analyzing chunk frames = [1529344,1529856) | 20260507 18:08:02.580 | INFO | MemInfo: { | 20260507 18:08:02.580 | INFO | "gpu" : " 17 GB", | 20260507 18:08:02.580 | INFO | "huge" : "", | 20260507 18:08:02.580 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:02.580 | INFO | "malloc" : " 14 GB", | 20260507 18:08:02.580 | INFO | "pinned" : " 29 GB" | 20260507 18:08:02.580 | INFO | } | 20260507 18:08:02.580 | INFO | Perf report for the last 5.1396 seconds: | 20260507 18:08:02.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:02.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:08:02.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:08:02.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:02.580 | INFO | Uploaded 36GiB to GPU in 2.42267s (14.8596GiB/s) | 20260507 18:08:02.580 | INFO | Downloaded 34GiB to GPU in 2.36384s (14.3834GiB/s) | 20260507 18:08:07.719 | INFO | Analyzing chunk frames = [1529856,1530368) | 20260507 18:08:07.719 | INFO | MemInfo: { | 20260507 18:08:07.719 | INFO | "gpu" : " 17 GB", | 20260507 18:08:07.719 | INFO | "huge" : "", | 20260507 18:08:07.719 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:07.719 | INFO | "malloc" : " 14 GB", | 20260507 18:08:07.719 | INFO | "pinned" : " 29 GB" | 20260507 18:08:07.720 | INFO | } | 20260507 18:08:07.720 | INFO | Perf report for the last 5.13973 seconds: | 20260507 18:08:07.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:07.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:08:07.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 18:08:07.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:07.720 | INFO | Uploaded 36GiB to GPU in 2.28674s (15.7429GiB/s) | 20260507 18:08:07.720 | INFO | Downloaded 34GiB to GPU in 2.49677s (13.6176GiB/s) | 20260507 18:08:12.858 | INFO | Analyzing chunk frames = [1530368,1530880) | 20260507 18:08:12.858 | INFO | MemInfo: { | 20260507 18:08:12.858 | INFO | "gpu" : " 17 GB", | 20260507 18:08:12.858 | INFO | "huge" : "", | 20260507 18:08:12.859 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:12.859 | INFO | "malloc" : " 14 GB", | 20260507 18:08:12.859 | INFO | "pinned" : " 29 GB" | 20260507 18:08:12.859 | INFO | } | 20260507 18:08:12.859 | INFO | Perf report for the last 5.13886 seconds: | 20260507 18:08:12.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:12.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 18:08:12.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:08:12.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:12.859 | INFO | Uploaded 36GiB to GPU in 2.35652s (15.2768GiB/s) | 20260507 18:08:12.859 | INFO | Downloaded 34GiB to GPU in 2.52382s (13.4716GiB/s) | 20260507 18:08:17.998 | INFO | Analyzing chunk frames = [1530880,1531392) | 20260507 18:08:17.998 | INFO | MemInfo: { | 20260507 18:08:17.998 | INFO | "gpu" : " 17 GB", | 20260507 18:08:17.998 | INFO | "huge" : "", | 20260507 18:08:17.998 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:17.998 | INFO | "malloc" : " 14 GB", | 20260507 18:08:17.998 | INFO | "pinned" : " 29 GB" | 20260507 18:08:17.998 | INFO | } | 20260507 18:08:17.998 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:08:17.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:17.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 18:08:17.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 18:08:17.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:17.998 | INFO | Uploaded 36GiB to GPU in 2.43753s (14.7691GiB/s) | 20260507 18:08:17.998 | INFO | Downloaded 34GiB to GPU in 2.45015s (13.8767GiB/s) | 20260507 18:08:23.138 | INFO | Analyzing chunk frames = [1531392,1531904) | 20260507 18:08:23.138 | INFO | MemInfo: { | 20260507 18:08:23.138 | INFO | "gpu" : " 17 GB", | 20260507 18:08:23.138 | INFO | "huge" : "", | 20260507 18:08:23.138 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:23.138 | INFO | "malloc" : " 14 GB", | 20260507 18:08:23.138 | INFO | "pinned" : " 29 GB" | 20260507 18:08:23.138 | INFO | } | 20260507 18:08:23.138 | INFO | Perf report for the last 5.13998 seconds: | 20260507 18:08:23.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:23.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:08:23.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 18:08:23.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:23.138 | INFO | Uploaded 36GiB to GPU in 2.43021s (14.8136GiB/s) | 20260507 18:08:23.138 | INFO | Downloaded 34GiB to GPU in 2.44099s (13.9288GiB/s) | 20260507 18:08:23.138 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:08:23.138 | INFO | Total mem usage: 16.4 GiB | 20260507 18:08:23.138 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:08:23.138 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 18:08:23.138 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:08:23.138 | INFO | High Water Breakdown: | 20260507 18:08:23.138 | INFO | 59.2% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:08:23.138 | INFO | 40.7% : 5.88 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:08:23.138 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:08:23.138 | INFO | ------------------------------------------ | 20260507 18:08:23.138 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:08:23.138 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:08:23.138 | INFO | Total mem usage: 26.5 GiB | 20260507 18:08:23.139 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:08:23.139 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:08:23.139 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:08:23.139 | INFO | High Water Breakdown: | 20260507 18:08:23.139 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:08:23.139 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:08:23.139 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:08:23.139 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:08:23.139 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:08:23.139 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:08:23.139 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:08:23.139 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:08:23.139 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:08:23.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:08:23.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:08:23.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:08:23.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:08:23.139 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:08:23.139 | INFO | ------------------------------------------ | 20260507 18:08:23.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:08:23.139 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:08:23.139 | INFO | Total mem usage: 27.8 GiB | 20260507 18:08:23.139 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:08:23.139 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:08:23.139 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:08:23.139 | INFO | High Water Breakdown: | 20260507 18:08:23.139 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:08:23.139 | INFO | ------------------------------------------ | 20260507 18:08:23.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:08:23.139 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:08:23.139 | INFO | Total mem usage: 15.7 GiB | 20260507 18:08:23.139 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:08:23.139 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:08:23.139 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:08:23.139 | INFO | High Water Breakdown: | 20260507 18:08:23.139 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:08:23.139 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:08:23.139 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:08:23.139 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:08:23.139 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:08:23.139 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:08:23.139 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:08:23.139 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:08:23.139 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:08:23.139 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:08:23.139 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:08:23.139 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:08:23.139 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:08:23.139 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:08:23.139 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:08:23.139 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:08:23.139 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:08:23.139 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:08:23.139 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:08:23.139 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:08:23.139 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:08:23.139 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:08:23.139 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:08:23.139 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:08:23.139 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:08:23.139 | INFO | ------------------------------------------ | 20260507 18:08:23.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:08:23.140 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:08:23.140 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:08:23.140 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:08:23.140 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.43e+01 / 2.18e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.10e-02 / 6.26e-03 | 20260507 18:08:23.140 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.51e+00, 6.20e-01 / 1.94e-01 | 20260507 18:08:23.140 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.17e-01 | 20260507 18:08:23.140 | INFO | PerfReport: 15.71%, DME -- 4.20e+01 / 4.50e+01, 4.07e+00 / 2.24e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.86e+00 / 2.35e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.49e+00 / 2.82e+00, 4.33e-01 / 2.60e-01 | 20260507 18:08:23.140 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.44e+00, 3.15e-01 / 1.66e-01 | 20260507 18:08:23.140 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.18e-01, 6.35e-02 / 3.48e-02 | 20260507 18:08:23.140 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:08:23.140 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:08:23.140 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:08:23.140 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 23.82%, Upload -- 2.56e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 24.12%, Download -- 2.59e+01 / 0.00e+00, 8.64e+00 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 31.14%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 1.68%, LockUpload -- 1.80e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 2.55%, LockDownload -- 2.74e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 16.67%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:08:23.140 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260507 18:08:23.235 | INFO | Starting a round of Baz IO | 20260507 18:08:26.735 | INFO | Populating RTMetrics report | 20260507 18:08:26.735 | INFO | RTMetrics report populated | 20260507 18:08:26.735 | INFO | Opening tmp RTMetrics json file | 20260507 18:08:26.750 | INFO | Serializing RTMetrics to json | 20260507 18:08:26.753 | INFO | Serialization done, swapping tmp file now... | 20260507 18:08:26.753 | INFO | RTMetrics done | 20260507 18:08:26.815 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.58 seconds | 20260507 18:08:26.815 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.04% / 0.02% / 52.95% | 20260507 18:08:26.815 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.12078% / 2.38419e-05% / 1.98682e-05% | 20260507 18:08:26.815 | INFO | Baz IO overall: | 20260507 18:08:26.815 | INFO | BytesWritten : 681990058462 | 20260507 18:08:26.815 | INFO | NumEvents : 330590063664 | 20260507 18:08:28.276 | INFO | Analyzing chunk frames = [1531904,1532416) | 20260507 18:08:28.276 | INFO | MemInfo: { | 20260507 18:08:28.276 | INFO | "gpu" : " 17 GB", | 20260507 18:08:28.276 | INFO | "huge" : "", | 20260507 18:08:28.276 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:28.276 | INFO | "malloc" : " 14 GB", | 20260507 18:08:28.276 | INFO | "pinned" : " 29 GB" | 20260507 18:08:28.276 | INFO | } | 20260507 18:08:28.276 | INFO | Perf report for the last 5.13842 seconds: | 20260507 18:08:28.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:28.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:28.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:28.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:28.277 | INFO | ANALYSIS : 69.3%, 3.0, 108.1ms, 0 | 20260507 18:08:28.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:28.277 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:08:28.277 | INFO | PRE_HQ : 66.7%, 1.7, 59.5ms, 0 | 20260507 18:08:28.277 | INFO | BAZWRITER : 68.9%, 2.9, 103.0ms, 0 | 20260507 18:08:28.277 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:08:28.277 | INFO | Uploaded 36GiB to GPU in 2.47753s (14.5306GiB/s) | 20260507 18:08:28.277 | INFO | Downloaded 38GiB to GPU in 2.72656s (13.937GiB/s) | 20260507 18:08:33.416 | INFO | Analyzing chunk frames = [1532416,1532928) | 20260507 18:08:33.416 | INFO | MemInfo: { | 20260507 18:08:33.416 | INFO | "gpu" : " 17 GB", | 20260507 18:08:33.416 | INFO | "huge" : "", | 20260507 18:08:33.416 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:33.416 | INFO | "malloc" : " 14 GB", | 20260507 18:08:33.416 | INFO | "pinned" : " 29 GB" | 20260507 18:08:33.416 | INFO | } | 20260507 18:08:33.416 | INFO | Perf report for the last 5.1395 seconds: | 20260507 18:08:33.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:33.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:08:33.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:08:33.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:33.416 | INFO | Uploaded 36GiB to GPU in 2.54215s (14.1612GiB/s) | 20260507 18:08:33.416 | INFO | Downloaded 34GiB to GPU in 2.4515s (13.8691GiB/s) | 20260507 18:08:38.554 | INFO | Analyzing chunk frames = [1532928,1533440) | 20260507 18:08:38.554 | INFO | MemInfo: { | 20260507 18:08:38.555 | INFO | "gpu" : " 17 GB", | 20260507 18:08:38.555 | INFO | "huge" : "", | 20260507 18:08:38.555 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:38.555 | INFO | "malloc" : " 14 GB", | 20260507 18:08:38.555 | INFO | "pinned" : " 29 GB" | 20260507 18:08:38.555 | INFO | } | 20260507 18:08:38.555 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:08:38.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:38.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 18:08:38.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:08:38.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:38.555 | INFO | Uploaded 36GiB to GPU in 2.55876s (14.0693GiB/s) | 20260507 18:08:38.555 | INFO | Downloaded 34GiB to GPU in 2.42021s (14.0484GiB/s) | 20260507 18:08:43.694 | INFO | Analyzing chunk frames = [1533440,1533952) | 20260507 18:08:43.694 | INFO | MemInfo: { | 20260507 18:08:43.694 | INFO | "gpu" : " 17 GB", | 20260507 18:08:43.694 | INFO | "huge" : "", | 20260507 18:08:43.694 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:43.694 | INFO | "malloc" : " 14 GB", | 20260507 18:08:43.694 | INFO | "pinned" : " 29 GB" | 20260507 18:08:43.694 | INFO | } | 20260507 18:08:43.695 | INFO | Perf report for the last 5.13974 seconds: | 20260507 18:08:43.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:43.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:08:43.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:08:43.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:43.695 | INFO | Uploaded 36GiB to GPU in 2.49823s (14.4102GiB/s) | 20260507 18:08:43.695 | INFO | Downloaded 34GiB to GPU in 2.34835s (14.4783GiB/s) | 20260507 18:08:48.834 | INFO | Analyzing chunk frames = [1533952,1534464) | 20260507 18:08:48.834 | INFO | MemInfo: { | 20260507 18:08:48.834 | INFO | "gpu" : " 17 GB", | 20260507 18:08:48.834 | INFO | "huge" : "", | 20260507 18:08:48.834 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:48.834 | INFO | "malloc" : " 14 GB", | 20260507 18:08:48.834 | INFO | "pinned" : " 29 GB" | 20260507 18:08:48.834 | INFO | } | 20260507 18:08:48.834 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:08:48.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:48.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:08:48.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:08:48.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:48.834 | INFO | Uploaded 36GiB to GPU in 2.37637s (15.1492GiB/s) | 20260507 18:08:48.834 | INFO | Downloaded 34GiB to GPU in 2.4728s (13.7496GiB/s) | 20260507 18:08:53.973 | INFO | Analyzing chunk frames = [1534464,1534976) | 20260507 18:08:53.973 | INFO | MemInfo: { | 20260507 18:08:53.973 | INFO | "gpu" : " 17 GB", | 20260507 18:08:53.973 | INFO | "huge" : "", | 20260507 18:08:53.973 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:53.973 | INFO | "malloc" : " 14 GB", | 20260507 18:08:53.973 | INFO | "pinned" : " 29 GB" | 20260507 18:08:53.973 | INFO | } | 20260507 18:08:53.973 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:08:53.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:53.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.973 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:08:53.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.973 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 18:08:53.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:53.974 | INFO | Uploaded 36GiB to GPU in 2.42892s (14.8214GiB/s) | 20260507 18:08:53.974 | INFO | Downloaded 34GiB to GPU in 2.48952s (13.6572GiB/s) | 20260507 18:08:59.113 | INFO | Analyzing chunk frames = [1534976,1535488) | 20260507 18:08:59.113 | INFO | MemInfo: { | 20260507 18:08:59.113 | INFO | "gpu" : " 17 GB", | 20260507 18:08:59.113 | INFO | "huge" : "", | 20260507 18:08:59.113 | INFO | "huge_shared" : " 29 GB", | 20260507 18:08:59.113 | INFO | "malloc" : " 14 GB", | 20260507 18:08:59.113 | INFO | "pinned" : " 29 GB" | 20260507 18:08:59.113 | INFO | } | 20260507 18:08:59.113 | INFO | Perf report for the last 5.13972 seconds: | 20260507 18:08:59.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:08:59.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 18:08:59.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:08:59.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:08:59.113 | INFO | Uploaded 36GiB to GPU in 2.53008s (14.2288GiB/s) | 20260507 18:08:59.113 | INFO | Downloaded 34GiB to GPU in 2.42576s (14.0162GiB/s) | 20260507 18:09:04.252 | INFO | Analyzing chunk frames = [1535488,1536000) | 20260507 18:09:04.252 | INFO | MemInfo: { | 20260507 18:09:04.252 | INFO | "gpu" : " 17 GB", | 20260507 18:09:04.252 | INFO | "huge" : "", | 20260507 18:09:04.252 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:04.252 | INFO | "malloc" : " 14 GB", | 20260507 18:09:04.252 | INFO | "pinned" : " 29 GB" | 20260507 18:09:04.252 | INFO | } | 20260507 18:09:04.252 | INFO | Perf report for the last 5.13886 seconds: | 20260507 18:09:04.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:04.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 18:09:04.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:09:04.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:04.252 | INFO | Uploaded 36GiB to GPU in 2.55124s (14.1108GiB/s) | 20260507 18:09:04.252 | INFO | Downloaded 34GiB to GPU in 2.48863s (13.6621GiB/s) | 20260507 18:09:04.363 | INFO | Starting a round of Baz IO | 20260507 18:09:07.854 | INFO | Populating RTMetrics report | 20260507 18:09:07.854 | INFO | RTMetrics report populated | 20260507 18:09:07.854 | INFO | Opening tmp RTMetrics json file | 20260507 18:09:07.873 | INFO | Serializing RTMetrics to json | 20260507 18:09:07.874 | INFO | Serialization done, swapping tmp file now... | 20260507 18:09:07.875 | INFO | RTMetrics done | 20260507 18:09:07.937 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.574 seconds | 20260507 18:09:07.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.02% / 0.02% / 52.96% | 20260507 18:09:07.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.11787% / 3.57628e-05% / 2.78155e-05% | 20260507 18:09:07.938 | INFO | Baz IO overall: | 20260507 18:09:07.938 | INFO | BytesWritten : 683925758369 | 20260507 18:09:07.938 | INFO | NumEvents : 331530438115 | 20260507 18:09:09.391 | INFO | Analyzing chunk frames = [1536000,1536512) | 20260507 18:09:09.391 | INFO | MemInfo: { | 20260507 18:09:09.391 | INFO | "gpu" : " 17 GB", | 20260507 18:09:09.391 | INFO | "huge" : "", | 20260507 18:09:09.391 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:09.391 | INFO | "malloc" : " 14 GB", | 20260507 18:09:09.391 | INFO | "pinned" : " 29 GB" | 20260507 18:09:09.391 | INFO | } | 20260507 18:09:09.391 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:09:09.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:09.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:09.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:09.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:09.391 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 18:09:09.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:09.391 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 18:09:09.391 | INFO | PRE_HQ : 66.7%, 1.7, 59.4ms, 0 | 20260507 18:09:09.391 | INFO | BAZWRITER : 69.1%, 2.9, 103.8ms, 0 | 20260507 18:09:09.391 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 18:09:09.391 | INFO | Uploaded 36GiB to GPU in 2.56937s (14.0112GiB/s) | 20260507 18:09:09.391 | INFO | Downloaded 38GiB to GPU in 2.71522s (13.9952GiB/s) | 20260507 18:09:14.531 | INFO | Analyzing chunk frames = [1536512,1537024) | 20260507 18:09:14.531 | INFO | MemInfo: { | 20260507 18:09:14.531 | INFO | "gpu" : " 17 GB", | 20260507 18:09:14.531 | INFO | "huge" : "", | 20260507 18:09:14.531 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:14.531 | INFO | "malloc" : " 14 GB", | 20260507 18:09:14.531 | INFO | "pinned" : " 29 GB" | 20260507 18:09:14.531 | INFO | } | 20260507 18:09:14.531 | INFO | Perf report for the last 5.13984 seconds: | 20260507 18:09:14.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:14.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.531 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:09:14.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.531 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:09:14.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:14.532 | INFO | Uploaded 36GiB to GPU in 2.55096s (14.1124GiB/s) | 20260507 18:09:14.532 | INFO | Downloaded 34GiB to GPU in 2.43622s (13.956GiB/s) | 20260507 18:09:19.669 | INFO | Analyzing chunk frames = [1537024,1537536) | 20260507 18:09:19.669 | INFO | MemInfo: { | 20260507 18:09:19.669 | INFO | "gpu" : " 17 GB", | 20260507 18:09:19.669 | INFO | "huge" : "", | 20260507 18:09:19.669 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:19.670 | INFO | "malloc" : " 14 GB", | 20260507 18:09:19.670 | INFO | "pinned" : " 29 GB" | 20260507 18:09:19.670 | INFO | } | 20260507 18:09:19.670 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:09:19.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:19.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.670 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:09:19.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.670 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:09:19.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:19.671 | INFO | Uploaded 36GiB to GPU in 2.56675s (14.0255GiB/s) | 20260507 18:09:19.671 | INFO | Downloaded 34GiB to GPU in 2.42873s (13.9991GiB/s) | 20260507 18:09:24.809 | INFO | Analyzing chunk frames = [1537536,1538048) | 20260507 18:09:24.809 | INFO | MemInfo: { | 20260507 18:09:24.809 | INFO | "gpu" : " 17 GB", | 20260507 18:09:24.809 | INFO | "huge" : "", | 20260507 18:09:24.809 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:24.809 | INFO | "malloc" : " 14 GB", | 20260507 18:09:24.809 | INFO | "pinned" : " 29 GB" | 20260507 18:09:24.809 | INFO | } | 20260507 18:09:24.809 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:09:24.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:24.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.809 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:09:24.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.809 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:09:24.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:24.810 | INFO | Uploaded 36GiB to GPU in 2.50872s (14.3499GiB/s) | 20260507 18:09:24.810 | INFO | Downloaded 34GiB to GPU in 2.35384s (14.4445GiB/s) | 20260507 18:09:29.949 | INFO | Analyzing chunk frames = [1538048,1538560) | 20260507 18:09:29.949 | INFO | MemInfo: { | 20260507 18:09:29.949 | INFO | "gpu" : " 17 GB", | 20260507 18:09:29.949 | INFO | "huge" : "", | 20260507 18:09:29.949 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:29.949 | INFO | "malloc" : " 14 GB", | 20260507 18:09:29.949 | INFO | "pinned" : " 29 GB" | 20260507 18:09:29.949 | INFO | } | 20260507 18:09:29.949 | INFO | Perf report for the last 5.13975 seconds: | 20260507 18:09:29.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:29.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.949 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 18:09:29.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.949 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:09:29.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:29.950 | INFO | Uploaded 36GiB to GPU in 2.37192s (15.1776GiB/s) | 20260507 18:09:29.950 | INFO | Downloaded 34GiB to GPU in 2.45337s (13.8585GiB/s) | 20260507 18:09:35.088 | INFO | Analyzing chunk frames = [1538560,1539072) | 20260507 18:09:35.088 | INFO | MemInfo: { | 20260507 18:09:35.088 | INFO | "gpu" : " 17 GB", | 20260507 18:09:35.088 | INFO | "huge" : "", | 20260507 18:09:35.088 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:35.088 | INFO | "malloc" : " 14 GB", | 20260507 18:09:35.088 | INFO | "pinned" : " 29 GB" | 20260507 18:09:35.088 | INFO | } | 20260507 18:09:35.088 | INFO | Perf report for the last 5.13875 seconds: | 20260507 18:09:35.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:35.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:09:35.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:09:35.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:35.088 | INFO | Uploaded 36GiB to GPU in 2.44124s (14.7466GiB/s) | 20260507 18:09:35.088 | INFO | Downloaded 34GiB to GPU in 2.51211s (13.5344GiB/s) | 20260507 18:09:40.227 | INFO | Analyzing chunk frames = [1539072,1539584) | 20260507 18:09:40.227 | INFO | MemInfo: { | 20260507 18:09:40.227 | INFO | "gpu" : " 17 GB", | 20260507 18:09:40.227 | INFO | "huge" : "", | 20260507 18:09:40.227 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:40.227 | INFO | "malloc" : " 14 GB", | 20260507 18:09:40.227 | INFO | "pinned" : " 29 GB" | 20260507 18:09:40.227 | INFO | } | 20260507 18:09:40.227 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:09:40.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:40.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.227 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:09:40.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.227 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:09:40.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:40.228 | INFO | Uploaded 36GiB to GPU in 2.48478s (14.4882GiB/s) | 20260507 18:09:40.228 | INFO | Downloaded 34GiB to GPU in 2.43108s (13.9856GiB/s) | 20260507 18:09:45.366 | INFO | Analyzing chunk frames = [1539584,1540096) | 20260507 18:09:45.366 | INFO | MemInfo: { | 20260507 18:09:45.366 | INFO | "gpu" : " 17 GB", | 20260507 18:09:45.366 | INFO | "huge" : "", | 20260507 18:09:45.366 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:45.366 | INFO | "malloc" : " 14 GB", | 20260507 18:09:45.366 | INFO | "pinned" : " 29 GB" | 20260507 18:09:45.366 | INFO | } | 20260507 18:09:45.366 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:09:45.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:45.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.366 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:09:45.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.367 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:09:45.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:45.367 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:09:45.367 | INFO | Total mem usage: 16.4 GiB | 20260507 18:09:45.367 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:09:45.367 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.4%) | 20260507 18:09:45.367 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:09:45.367 | INFO | High Water Breakdown: | 20260507 18:09:45.367 | INFO | 59.2% : 8.56 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:09:45.367 | INFO | 40.7% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:09:45.367 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:09:45.367 | INFO | ------------------------------------------ | 20260507 18:09:45.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:09:45.367 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:09:45.367 | INFO | Total mem usage: 26.5 GiB | 20260507 18:09:45.367 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:09:45.367 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:09:45.367 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:09:45.367 | INFO | High Water Breakdown: | 20260507 18:09:45.367 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:09:45.367 | INFO | Uploaded 36GiB to GPU in 2.45967s (14.6361GiB/s) | 20260507 18:09:45.367 | INFO | Downloaded 34GiB to GPU in 2.4198s (14.0507GiB/s) | 20260507 18:09:45.367 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:09:45.367 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:09:45.367 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:09:45.367 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:09:45.367 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:09:45.367 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:09:45.367 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:09:45.367 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:09:45.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:09:45.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:09:45.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:09:45.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:09:45.367 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:09:45.367 | INFO | ------------------------------------------ | 20260507 18:09:45.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:09:45.367 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:09:45.367 | INFO | Total mem usage: 27.8 GiB | 20260507 18:09:45.367 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:09:45.367 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:09:45.367 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:09:45.367 | INFO | High Water Breakdown: | 20260507 18:09:45.367 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:09:45.368 | INFO | ------------------------------------------ | 20260507 18:09:45.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:09:45.368 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:09:45.368 | INFO | Total mem usage: 15.7 GiB | 20260507 18:09:45.368 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:09:45.368 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:09:45.368 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:09:45.368 | INFO | High Water Breakdown: | 20260507 18:09:45.368 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:09:45.368 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:09:45.368 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:09:45.368 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:09:45.368 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:09:45.368 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:09:45.368 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:09:45.368 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:09:45.368 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:09:45.368 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:09:45.368 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:09:45.368 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:09:45.368 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:09:45.368 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:09:45.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:09:45.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:09:45.368 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:09:45.368 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:09:45.368 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:09:45.368 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:09:45.368 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:09:45.368 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:09:45.368 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:09:45.368 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:09:45.368 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:09:45.368 | INFO | ------------------------------------------ | 20260507 18:09:45.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:09:45.368 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:09:45.368 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:09:45.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:09:45.368 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.67e+01, 1.44e+01 / 2.04e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.22e-02 / 6.65e-03 | 20260507 18:09:45.368 | INFO | PerfReport: 12.07%, Baseline -- 4.04e+00 / 4.50e+00, 6.18e-01 / 2.25e-01 | 20260507 18:09:45.368 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 1.01e-01 | 20260507 18:09:45.368 | INFO | PerfReport: 15.70%, DME -- 4.20e+01 / 4.51e+01, 4.19e+00 / 2.25e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.87e+00 / 1.93e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.32e-01 / 2.39e-01 | 20260507 18:09:45.368 | INFO | PerfReport: 3.74%, Metrics -- 1.25e+00 / 1.43e+00, 3.18e-01 / 1.39e-01 | 20260507 18:09:45.368 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.23e-01, 6.37e-02 / 3.18e-02 | 20260507 18:09:45.368 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:09:45.368 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:09:45.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:09:45.368 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 24.17%, Upload -- 2.60e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 23.96%, Download -- 2.58e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 31.09%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 1.98%, LockUpload -- 2.13e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 2.20%, LockDownload -- 2.37e+00 / 0.00e+00, 3.78e+00 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 16.59%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:09:45.368 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260507 18:09:45.464 | INFO | Starting a round of Baz IO | 20260507 18:09:48.961 | INFO | Populating RTMetrics report | 20260507 18:09:48.961 | INFO | RTMetrics report populated | 20260507 18:09:48.961 | INFO | Opening tmp RTMetrics json file | 20260507 18:09:48.977 | INFO | Serializing RTMetrics to json | 20260507 18:09:48.978 | INFO | Serialization done, swapping tmp file now... | 20260507 18:09:48.979 | INFO | RTMetrics done | 20260507 18:09:49.042 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.578 seconds | 20260507 18:09:49.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.00% / 0.02% / 52.98% | 20260507 18:09:49.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.10204% / 3.57628e-05% / 3.97364e-05% | 20260507 18:09:49.042 | INFO | Baz IO overall: | 20260507 18:09:49.042 | INFO | BytesWritten : 685859171638 | 20260507 18:09:49.042 | INFO | NumEvents : 332468835459 | 20260507 18:09:50.505 | INFO | Analyzing chunk frames = [1540096,1540608) | 20260507 18:09:50.505 | INFO | MemInfo: { | 20260507 18:09:50.505 | INFO | "gpu" : " 17 GB", | 20260507 18:09:50.505 | INFO | "huge" : "", | 20260507 18:09:50.505 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:50.505 | INFO | "malloc" : " 14 GB", | 20260507 18:09:50.505 | INFO | "pinned" : " 29 GB" | 20260507 18:09:50.505 | INFO | } | 20260507 18:09:50.505 | INFO | Perf report for the last 5.13905 seconds: | 20260507 18:09:50.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:50.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:50.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:50.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:50.506 | INFO | ANALYSIS : 69.5%, 3.0, 108.2ms, 0 | 20260507 18:09:50.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:50.506 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:09:50.506 | INFO | PRE_HQ : 66.7%, 1.7, 59.1ms, 0 | 20260507 18:09:50.506 | INFO | BAZWRITER : 68.8%, 2.9, 101.5ms, 0 | 20260507 18:09:50.506 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:09:50.506 | INFO | Uploaded 36GiB to GPU in 2.52231s (14.2727GiB/s) | 20260507 18:09:50.506 | INFO | Downloaded 38GiB to GPU in 2.70751s (14.035GiB/s) | 20260507 18:09:55.645 | INFO | Analyzing chunk frames = [1540608,1541120) | 20260507 18:09:55.645 | INFO | MemInfo: { | 20260507 18:09:55.645 | INFO | "gpu" : " 17 GB", | 20260507 18:09:55.645 | INFO | "huge" : "", | 20260507 18:09:55.645 | INFO | "huge_shared" : " 29 GB", | 20260507 18:09:55.645 | INFO | "malloc" : " 14 GB", | 20260507 18:09:55.645 | INFO | "pinned" : " 29 GB" | 20260507 18:09:55.645 | INFO | } | 20260507 18:09:55.645 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:09:55.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:09:55.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.645 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:09:55.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.645 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:09:55.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:09:55.646 | INFO | Uploaded 36GiB to GPU in 2.54601s (14.1398GiB/s) | 20260507 18:09:55.646 | INFO | Downloaded 34GiB to GPU in 2.44918s (13.8822GiB/s) | 20260507 18:10:00.784 | INFO | Analyzing chunk frames = [1541120,1541632) | 20260507 18:10:00.784 | INFO | MemInfo: { | 20260507 18:10:00.784 | INFO | "gpu" : " 17 GB", | 20260507 18:10:00.784 | INFO | "huge" : "", | 20260507 18:10:00.784 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:00.784 | INFO | "malloc" : " 14 GB", | 20260507 18:10:00.784 | INFO | "pinned" : " 29 GB" | 20260507 18:10:00.784 | INFO | } | 20260507 18:10:00.784 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:10:00.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:00.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.784 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:10:00.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.784 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:10:00.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:00.785 | INFO | Uploaded 36GiB to GPU in 2.56459s (14.0373GiB/s) | 20260507 18:10:00.785 | INFO | Downloaded 34GiB to GPU in 2.42369s (14.0282GiB/s) | 20260507 18:10:05.924 | INFO | Analyzing chunk frames = [1541632,1542144) | 20260507 18:10:05.924 | INFO | MemInfo: { | 20260507 18:10:05.924 | INFO | "gpu" : " 17 GB", | 20260507 18:10:05.924 | INFO | "huge" : "", | 20260507 18:10:05.924 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:05.924 | INFO | "malloc" : " 14 GB", | 20260507 18:10:05.924 | INFO | "pinned" : " 29 GB" | 20260507 18:10:05.924 | INFO | } | 20260507 18:10:05.924 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:10:05.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:05.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:10:05.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:10:05.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:05.924 | INFO | Uploaded 36GiB to GPU in 2.49035s (14.4558GiB/s) | 20260507 18:10:05.924 | INFO | Downloaded 34GiB to GPU in 2.34588s (14.4935GiB/s) | 20260507 18:10:11.064 | INFO | Analyzing chunk frames = [1542144,1542656) | 20260507 18:10:11.064 | INFO | MemInfo: { | 20260507 18:10:11.064 | INFO | "gpu" : " 17 GB", | 20260507 18:10:11.064 | INFO | "huge" : "", | 20260507 18:10:11.064 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:11.064 | INFO | "malloc" : " 14 GB", | 20260507 18:10:11.064 | INFO | "pinned" : " 29 GB" | 20260507 18:10:11.064 | INFO | } | 20260507 18:10:11.064 | INFO | Perf report for the last 5.13998 seconds: | 20260507 18:10:11.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:11.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 18:10:11.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:10:11.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:11.064 | INFO | Uploaded 36GiB to GPU in 2.35598s (15.2803GiB/s) | 20260507 18:10:11.064 | INFO | Downloaded 34GiB to GPU in 2.47037s (13.7631GiB/s) | 20260507 18:10:16.203 | INFO | Analyzing chunk frames = [1542656,1543168) | 20260507 18:10:16.203 | INFO | MemInfo: { | 20260507 18:10:16.203 | INFO | "gpu" : " 17 GB", | 20260507 18:10:16.203 | INFO | "huge" : "", | 20260507 18:10:16.203 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:16.203 | INFO | "malloc" : " 14 GB", | 20260507 18:10:16.203 | INFO | "pinned" : " 29 GB" | 20260507 18:10:16.203 | INFO | } | 20260507 18:10:16.203 | INFO | Perf report for the last 5.13887 seconds: | 20260507 18:10:16.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:16.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:10:16.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:10:16.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:16.203 | INFO | Uploaded 36GiB to GPU in 2.47068s (14.5709GiB/s) | 20260507 18:10:16.203 | INFO | Downloaded 34GiB to GPU in 2.48859s (13.6623GiB/s) | 20260507 18:10:21.342 | INFO | Analyzing chunk frames = [1543168,1543680) | 20260507 18:10:21.342 | INFO | MemInfo: { | 20260507 18:10:21.342 | INFO | "gpu" : " 17 GB", | 20260507 18:10:21.342 | INFO | "huge" : "", | 20260507 18:10:21.342 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:21.342 | INFO | "malloc" : " 14 GB", | 20260507 18:10:21.342 | INFO | "pinned" : " 29 GB" | 20260507 18:10:21.342 | INFO | } | 20260507 18:10:21.342 | INFO | Perf report for the last 5.13918 seconds: | 20260507 18:10:21.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:21.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 18:10:21.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260507 18:10:21.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:21.342 | INFO | Uploaded 36GiB to GPU in 2.51727s (14.3012GiB/s) | 20260507 18:10:21.342 | INFO | Downloaded 34GiB to GPU in 2.42556s (14.0174GiB/s) | 20260507 18:10:26.481 | INFO | Analyzing chunk frames = [1543680,1544192) | 20260507 18:10:26.481 | INFO | MemInfo: { | 20260507 18:10:26.481 | INFO | "gpu" : " 17 GB", | 20260507 18:10:26.481 | INFO | "huge" : "", | 20260507 18:10:26.481 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:26.481 | INFO | "malloc" : " 14 GB", | 20260507 18:10:26.481 | INFO | "pinned" : " 29 GB" | 20260507 18:10:26.481 | INFO | } | 20260507 18:10:26.481 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:10:26.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:26.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.481 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 18:10:26.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.481 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:10:26.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:26.482 | INFO | Uploaded 36GiB to GPU in 2.5504s (14.1154GiB/s) | 20260507 18:10:26.482 | INFO | Downloaded 34GiB to GPU in 2.49492s (13.6277GiB/s) | 20260507 18:10:26.578 | INFO | Starting a round of Baz IO | 20260507 18:10:30.097 | INFO | Populating RTMetrics report | 20260507 18:10:30.097 | INFO | RTMetrics report populated | 20260507 18:10:30.097 | INFO | Opening tmp RTMetrics json file | 20260507 18:10:30.112 | INFO | Serializing RTMetrics to json | 20260507 18:10:30.113 | INFO | Serialization done, swapping tmp file now... | 20260507 18:10:30.113 | INFO | RTMetrics done | 20260507 18:10:30.174 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.595 seconds | 20260507 18:10:30.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.99% / 0.02% / 53.00% | 20260507 18:10:30.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.09482% / 1.19209e-05% / 3.57628e-05% | 20260507 18:10:30.174 | INFO | Baz IO overall: | 20260507 18:10:30.174 | INFO | BytesWritten : 687793213821 | 20260507 18:10:30.174 | INFO | NumEvents : 333407388151 | 20260507 18:10:31.620 | INFO | Analyzing chunk frames = [1544192,1544704) | 20260507 18:10:31.620 | INFO | MemInfo: { | 20260507 18:10:31.620 | INFO | "gpu" : " 17 GB", | 20260507 18:10:31.620 | INFO | "huge" : "", | 20260507 18:10:31.620 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:31.620 | INFO | "malloc" : " 14 GB", | 20260507 18:10:31.620 | INFO | "pinned" : " 29 GB" | 20260507 18:10:31.620 | INFO | } | 20260507 18:10:31.620 | INFO | Perf report for the last 5.1392 seconds: | 20260507 18:10:31.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:31.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:31.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:31.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:31.621 | INFO | ANALYSIS : 69.5%, 3.0, 108.6ms, 0 | 20260507 18:10:31.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:31.621 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260507 18:10:31.621 | INFO | PRE_HQ : 67.5%, 1.8, 63.2ms, 0 | 20260507 18:10:31.621 | INFO | BAZWRITER : 69.1%, 3.0, 108.1ms, 0 | 20260507 18:10:31.621 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:10:31.621 | INFO | Uploaded 36GiB to GPU in 2.54275s (14.1579GiB/s) | 20260507 18:10:31.621 | INFO | Downloaded 38GiB to GPU in 2.72891s (13.925GiB/s) | 20260507 18:10:36.759 | INFO | Analyzing chunk frames = [1544704,1545216) | 20260507 18:10:36.759 | INFO | MemInfo: { | 20260507 18:10:36.759 | INFO | "gpu" : " 17 GB", | 20260507 18:10:36.760 | INFO | "huge" : "", | 20260507 18:10:36.760 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:36.760 | INFO | "malloc" : " 14 GB", | 20260507 18:10:36.760 | INFO | "pinned" : " 29 GB" | 20260507 18:10:36.760 | INFO | } | 20260507 18:10:36.760 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:10:36.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:36.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:10:36.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:10:36.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:36.760 | INFO | Uploaded 36GiB to GPU in 2.47571s (14.5413GiB/s) | 20260507 18:10:36.760 | INFO | Downloaded 34GiB to GPU in 2.43337s (13.9724GiB/s) | 20260507 18:10:41.899 | INFO | Analyzing chunk frames = [1545216,1545728) | 20260507 18:10:41.899 | INFO | MemInfo: { | 20260507 18:10:41.899 | INFO | "gpu" : " 17 GB", | 20260507 18:10:41.899 | INFO | "huge" : "", | 20260507 18:10:41.899 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:41.899 | INFO | "malloc" : " 14 GB", | 20260507 18:10:41.899 | INFO | "pinned" : " 29 GB" | 20260507 18:10:41.899 | INFO | } | 20260507 18:10:41.899 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:10:41.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:41.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:10:41.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:10:41.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:41.899 | INFO | Uploaded 36GiB to GPU in 2.5489s (14.1237GiB/s) | 20260507 18:10:41.899 | INFO | Downloaded 34GiB to GPU in 2.43011s (13.9912GiB/s) | 20260507 18:10:47.039 | INFO | Analyzing chunk frames = [1545728,1546240) | 20260507 18:10:47.039 | INFO | MemInfo: { | 20260507 18:10:47.039 | INFO | "gpu" : " 17 GB", | 20260507 18:10:47.039 | INFO | "huge" : "", | 20260507 18:10:47.039 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:47.039 | INFO | "malloc" : " 14 GB", | 20260507 18:10:47.039 | INFO | "pinned" : " 29 GB" | 20260507 18:10:47.039 | INFO | } | 20260507 18:10:47.039 | INFO | Perf report for the last 5.13979 seconds: | 20260507 18:10:47.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:47.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:10:47.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:10:47.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:47.039 | INFO | Uploaded 36GiB to GPU in 2.45295s (14.6762GiB/s) | 20260507 18:10:47.039 | INFO | Downloaded 34GiB to GPU in 2.37016s (14.345GiB/s) | 20260507 18:10:52.178 | INFO | Analyzing chunk frames = [1546240,1546752) | 20260507 18:10:52.178 | INFO | MemInfo: { | 20260507 18:10:52.178 | INFO | "gpu" : " 17 GB", | 20260507 18:10:52.178 | INFO | "huge" : "", | 20260507 18:10:52.178 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:52.178 | INFO | "malloc" : " 14 GB", | 20260507 18:10:52.178 | INFO | "pinned" : " 29 GB" | 20260507 18:10:52.178 | INFO | } | 20260507 18:10:52.178 | INFO | Perf report for the last 5.13902 seconds: | 20260507 18:10:52.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:52.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:10:52.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 18:10:52.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:52.178 | INFO | Uploaded 36GiB to GPU in 2.27383s (15.8323GiB/s) | 20260507 18:10:52.178 | INFO | Downloaded 34GiB to GPU in 2.49047s (13.652GiB/s) | 20260507 18:10:57.317 | INFO | Analyzing chunk frames = [1546752,1547264) | 20260507 18:10:57.317 | INFO | MemInfo: { | 20260507 18:10:57.317 | INFO | "gpu" : " 17 GB", | 20260507 18:10:57.317 | INFO | "huge" : "", | 20260507 18:10:57.317 | INFO | "huge_shared" : " 29 GB", | 20260507 18:10:57.317 | INFO | "malloc" : " 14 GB", | 20260507 18:10:57.317 | INFO | "pinned" : " 29 GB" | 20260507 18:10:57.317 | INFO | } | 20260507 18:10:57.317 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:10:57.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:10:57.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.317 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 18:10:57.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.317 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:10:57.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:10:57.318 | INFO | Uploaded 36GiB to GPU in 2.32032s (15.5151GiB/s) | 20260507 18:10:57.318 | INFO | Downloaded 34GiB to GPU in 2.51086s (13.5412GiB/s) | 20260507 18:11:02.456 | INFO | Analyzing chunk frames = [1547264,1547776) | 20260507 18:11:02.456 | INFO | MemInfo: { | 20260507 18:11:02.456 | INFO | "gpu" : " 17 GB", | 20260507 18:11:02.456 | INFO | "huge" : "", | 20260507 18:11:02.456 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:02.456 | INFO | "malloc" : " 14 GB", | 20260507 18:11:02.456 | INFO | "pinned" : " 29 GB" | 20260507 18:11:02.456 | INFO | } | 20260507 18:11:02.456 | INFO | Perf report for the last 5.13911 seconds: | 20260507 18:11:02.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:02.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:11:02.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:11:02.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:02.457 | INFO | Uploaded 36GiB to GPU in 2.45244s (14.6792GiB/s) | 20260507 18:11:02.457 | INFO | Downloaded 34GiB to GPU in 2.45682s (13.839GiB/s) | 20260507 18:11:07.596 | INFO | Analyzing chunk frames = [1547776,1548288) | 20260507 18:11:07.596 | INFO | MemInfo: { | 20260507 18:11:07.596 | INFO | "gpu" : " 17 GB", | 20260507 18:11:07.596 | INFO | "huge" : "", | 20260507 18:11:07.596 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:07.596 | INFO | "malloc" : " 14 GB", | 20260507 18:11:07.596 | INFO | "pinned" : " 29 GB" | 20260507 18:11:07.596 | INFO | } | 20260507 18:11:07.596 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:11:07.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:07.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:11:07.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 18:11:07.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:07.596 | INFO | Uploaded 36GiB to GPU in 2.43397s (14.7907GiB/s) | 20260507 18:11:07.596 | INFO | Downloaded 34GiB to GPU in 2.43919s (13.9391GiB/s) | 20260507 18:11:07.597 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:11:07.597 | INFO | Total mem usage: 16.4 GiB | 20260507 18:11:07.597 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:11:07.597 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.6%) | 20260507 18:11:07.597 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:11:07.597 | INFO | High Water Breakdown: | 20260507 18:11:07.597 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:11:07.597 | INFO | 40.6% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:11:07.597 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:11:07.597 | INFO | ------------------------------------------ | 20260507 18:11:07.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:11:07.597 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:11:07.597 | INFO | Total mem usage: 26.5 GiB | 20260507 18:11:07.597 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:11:07.597 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:11:07.597 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:11:07.597 | INFO | High Water Breakdown: | 20260507 18:11:07.597 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:11:07.597 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:11:07.597 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:11:07.597 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:11:07.597 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:11:07.597 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:11:07.597 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:11:07.597 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:11:07.597 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:11:07.597 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:11:07.597 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:11:07.597 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:11:07.597 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:11:07.597 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:11:07.597 | INFO | ------------------------------------------ | 20260507 18:11:07.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:11:07.597 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:11:07.597 | INFO | Total mem usage: 27.8 GiB | 20260507 18:11:07.597 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:11:07.597 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:11:07.597 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:11:07.597 | INFO | High Water Breakdown: | 20260507 18:11:07.597 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:11:07.597 | INFO | ------------------------------------------ | 20260507 18:11:07.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:11:07.597 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:11:07.597 | INFO | Total mem usage: 15.7 GiB | 20260507 18:11:07.597 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:11:07.597 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:11:07.597 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:11:07.597 | INFO | High Water Breakdown: | 20260507 18:11:07.597 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:11:07.597 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:11:07.598 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:11:07.598 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:11:07.598 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:11:07.598 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:11:07.598 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:11:07.598 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:11:07.598 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:11:07.598 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:11:07.598 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:11:07.598 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:11:07.598 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:11:07.598 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:11:07.598 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:11:07.598 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:11:07.598 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:11:07.598 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:11:07.598 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:11:07.598 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:11:07.598 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:11:07.598 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:11:07.598 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:11:07.598 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:11:07.598 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:11:07.598 | INFO | ------------------------------------------ | 20260507 18:11:07.598 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:11:07.598 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:11:07.598 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:11:07.598 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:11:07.598 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.66e+01, 1.45e+01 / 2.54e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.04e-01, 1.19e-02 / 5.20e-03 | 20260507 18:11:07.598 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.49e+00, 6.22e-01 / 2.28e-01 | 20260507 18:11:07.598 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 9.30e-02 | 20260507 18:11:07.598 | INFO | PerfReport: 15.74%, DME -- 4.21e+01 / 4.52e+01, 4.24e+00 / 2.34e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.86e+00 / 2.20e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.35e-01 / 2.33e-01 | 20260507 18:11:07.598 | INFO | PerfReport: 3.72%, Metrics -- 1.24e+00 / 1.43e+00, 3.12e-01 / 1.83e-01 | 20260507 18:11:07.598 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.93e-01 / 4.23e-01, 6.41e-02 / 4.28e-02 | 20260507 18:11:07.598 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:11:07.598 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:11:07.598 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:11:07.598 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 23.95%, Upload -- 2.57e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 24.04%, Download -- 2.58e+01 / 0.00e+00, 8.61e+00 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 31.16%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 1.79%, LockUpload -- 1.92e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 2.36%, LockDownload -- 2.54e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 16.68%, LockCompute -- 1.79e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:11:07.598 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 18:11:07.711 | INFO | Starting a round of Baz IO | 20260507 18:11:11.201 | INFO | Populating RTMetrics report | 20260507 18:11:11.201 | INFO | RTMetrics report populated | 20260507 18:11:11.201 | INFO | Opening tmp RTMetrics json file | 20260507 18:11:11.207 | INFO | Serializing RTMetrics to json | 20260507 18:11:11.221 | INFO | Serialization done, swapping tmp file now... | 20260507 18:11:11.221 | INFO | RTMetrics done | 20260507 18:11:11.281 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.57 seconds | 20260507 18:11:11.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.97% / 0.02% / 53.01% | 20260507 18:11:11.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.0894% / 3.57628e-05% / 4.76837e-05% | 20260507 18:11:11.281 | INFO | Baz IO overall: | 20260507 18:11:11.281 | INFO | BytesWritten : 689727283116 | 20260507 18:11:11.281 | INFO | NumEvents : 334346145970 | 20260507 18:11:12.735 | INFO | Analyzing chunk frames = [1548288,1548800) | 20260507 18:11:12.735 | INFO | MemInfo: { | 20260507 18:11:12.735 | INFO | "gpu" : " 17 GB", | 20260507 18:11:12.735 | INFO | "huge" : "", | 20260507 18:11:12.735 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:12.735 | INFO | "malloc" : " 14 GB", | 20260507 18:11:12.735 | INFO | "pinned" : " 29 GB" | 20260507 18:11:12.735 | INFO | } | 20260507 18:11:12.735 | INFO | Perf report for the last 5.13915 seconds: | 20260507 18:11:12.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:12.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:12.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:12.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:12.735 | INFO | ANALYSIS : 69.4%, 3.0, 108.4ms, 0 | 20260507 18:11:12.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:12.735 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 18:11:12.735 | INFO | PRE_HQ : 66.6%, 1.8, 60.6ms, 0 | 20260507 18:11:12.735 | INFO | BAZWRITER : 68.7%, 2.9, 101.9ms, 0 | 20260507 18:11:12.735 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:11:12.736 | INFO | Uploaded 36GiB to GPU in 2.48019s (14.515GiB/s) | 20260507 18:11:12.736 | INFO | Downloaded 38GiB to GPU in 2.73426s (13.8977GiB/s) | 20260507 18:11:17.875 | INFO | Analyzing chunk frames = [1548800,1549312) | 20260507 18:11:17.875 | INFO | MemInfo: { | 20260507 18:11:17.875 | INFO | "gpu" : " 17 GB", | 20260507 18:11:17.875 | INFO | "huge" : "", | 20260507 18:11:17.875 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:17.875 | INFO | "malloc" : " 14 GB", | 20260507 18:11:17.875 | INFO | "pinned" : " 29 GB" | 20260507 18:11:17.875 | INFO | } | 20260507 18:11:17.875 | INFO | Perf report for the last 5.13996 seconds: | 20260507 18:11:17.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:17.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:11:17.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:11:17.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:17.875 | INFO | Uploaded 36GiB to GPU in 2.52664s (14.2482GiB/s) | 20260507 18:11:17.875 | INFO | Downloaded 34GiB to GPU in 2.45494s (13.8496GiB/s) | 20260507 18:11:23.014 | INFO | Analyzing chunk frames = [1549312,1549824) | 20260507 18:11:23.014 | INFO | MemInfo: { | 20260507 18:11:23.014 | INFO | "gpu" : " 17 GB", | 20260507 18:11:23.014 | INFO | "huge" : "", | 20260507 18:11:23.014 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:23.014 | INFO | "malloc" : " 14 GB", | 20260507 18:11:23.014 | INFO | "pinned" : " 29 GB" | 20260507 18:11:23.014 | INFO | } | 20260507 18:11:23.014 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:11:23.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:23.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.014 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:11:23.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.014 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:11:23.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:23.015 | INFO | Uploaded 36GiB to GPU in 2.54394s (14.1513GiB/s) | 20260507 18:11:23.015 | INFO | Downloaded 34GiB to GPU in 2.43606s (13.957GiB/s) | 20260507 18:11:28.153 | INFO | Analyzing chunk frames = [1549824,1550336) | 20260507 18:11:28.153 | INFO | MemInfo: { | 20260507 18:11:28.153 | INFO | "gpu" : " 17 GB", | 20260507 18:11:28.153 | INFO | "huge" : "", | 20260507 18:11:28.153 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:28.153 | INFO | "malloc" : " 14 GB", | 20260507 18:11:28.153 | INFO | "pinned" : " 29 GB" | 20260507 18:11:28.153 | INFO | } | 20260507 18:11:28.153 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:11:28.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:28.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.154 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:11:28.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.154 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:11:28.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:28.154 | INFO | Uploaded 36GiB to GPU in 2.48598s (14.4812GiB/s) | 20260507 18:11:28.154 | INFO | Downloaded 34GiB to GPU in 2.3454s (14.4965GiB/s) | 20260507 18:11:33.293 | INFO | Analyzing chunk frames = [1550336,1550848) | 20260507 18:11:33.293 | INFO | MemInfo: { | 20260507 18:11:33.293 | INFO | "gpu" : " 17 GB", | 20260507 18:11:33.293 | INFO | "huge" : "", | 20260507 18:11:33.293 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:33.293 | INFO | "malloc" : " 14 GB", | 20260507 18:11:33.293 | INFO | "pinned" : " 29 GB" | 20260507 18:11:33.293 | INFO | } | 20260507 18:11:33.293 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:11:33.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:33.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:11:33.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 18:11:33.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:33.293 | INFO | Uploaded 36GiB to GPU in 2.33744s (15.4015GiB/s) | 20260507 18:11:33.293 | INFO | Downloaded 34GiB to GPU in 2.47846s (13.7182GiB/s) | 20260507 18:11:38.432 | INFO | Analyzing chunk frames = [1550848,1551360) | 20260507 18:11:38.432 | INFO | MemInfo: { | 20260507 18:11:38.432 | INFO | "gpu" : " 17 GB", | 20260507 18:11:38.432 | INFO | "huge" : "", | 20260507 18:11:38.432 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:38.432 | INFO | "malloc" : " 14 GB", | 20260507 18:11:38.432 | INFO | "pinned" : " 29 GB" | 20260507 18:11:38.432 | INFO | } | 20260507 18:11:38.432 | INFO | Perf report for the last 5.13918 seconds: | 20260507 18:11:38.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:38.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.432 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:11:38.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.432 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 18:11:38.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:38.433 | INFO | Uploaded 36GiB to GPU in 2.49046s (14.4552GiB/s) | 20260507 18:11:38.433 | INFO | Downloaded 34GiB to GPU in 2.49144s (13.6467GiB/s) | 20260507 18:11:43.571 | INFO | Analyzing chunk frames = [1551360,1551872) | 20260507 18:11:43.571 | INFO | MemInfo: { | 20260507 18:11:43.571 | INFO | "gpu" : " 17 GB", | 20260507 18:11:43.572 | INFO | "huge" : "", | 20260507 18:11:43.572 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:43.572 | INFO | "malloc" : " 14 GB", | 20260507 18:11:43.572 | INFO | "pinned" : " 29 GB" | 20260507 18:11:43.572 | INFO | } | 20260507 18:11:43.572 | INFO | Perf report for the last 5.13961 seconds: | 20260507 18:11:43.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:43.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:11:43.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 18:11:43.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:43.572 | INFO | Uploaded 36GiB to GPU in 2.50635s (14.3635GiB/s) | 20260507 18:11:43.572 | INFO | Downloaded 34GiB to GPU in 2.42746s (14.0064GiB/s) | 20260507 18:11:48.711 | INFO | Analyzing chunk frames = [1551872,1552384) | 20260507 18:11:48.711 | INFO | MemInfo: { | 20260507 18:11:48.711 | INFO | "gpu" : " 17 GB", | 20260507 18:11:48.711 | INFO | "huge" : "", | 20260507 18:11:48.711 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:48.711 | INFO | "malloc" : " 14 GB", | 20260507 18:11:48.711 | INFO | "pinned" : " 29 GB" | 20260507 18:11:48.711 | INFO | } | 20260507 18:11:48.711 | INFO | Perf report for the last 5.13936 seconds: | 20260507 18:11:48.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:48.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.711 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 18:11:48.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.711 | INFO | PRE_HQ : 38.7%, 1.0, 20.2ms, 0 | 20260507 18:11:48.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:48.712 | INFO | Uploaded 36GiB to GPU in 2.52808s (14.2401GiB/s) | 20260507 18:11:48.712 | INFO | Downloaded 34GiB to GPU in 2.42797s (14.0034GiB/s) | 20260507 18:11:48.824 | INFO | Starting a round of Baz IO | 20260507 18:11:52.310 | INFO | Populating RTMetrics report | 20260507 18:11:52.310 | INFO | RTMetrics report populated | 20260507 18:11:52.310 | INFO | Opening tmp RTMetrics json file | 20260507 18:11:52.320 | INFO | Serializing RTMetrics to json | 20260507 18:11:52.328 | INFO | Serialization done, swapping tmp file now... | 20260507 18:11:52.328 | INFO | RTMetrics done | 20260507 18:11:52.389 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.565 seconds | 20260507 18:11:52.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.95% / 0.02% / 53.03% | 20260507 18:11:52.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.0795% / 2.38419e-05% / 1.98682e-05% | 20260507 18:11:52.390 | INFO | Baz IO overall: | 20260507 18:11:52.390 | INFO | BytesWritten : 691660029587 | 20260507 18:11:52.390 | INFO | NumEvents : 335283931912 | 20260507 18:11:53.850 | INFO | Analyzing chunk frames = [1552384,1552896) | 20260507 18:11:53.850 | INFO | MemInfo: { | 20260507 18:11:53.850 | INFO | "gpu" : " 17 GB", | 20260507 18:11:53.850 | INFO | "huge" : "", | 20260507 18:11:53.850 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:53.850 | INFO | "malloc" : " 14 GB", | 20260507 18:11:53.850 | INFO | "pinned" : " 29 GB" | 20260507 18:11:53.850 | INFO | } | 20260507 18:11:53.850 | INFO | Perf report for the last 5.13896 seconds: | 20260507 18:11:53.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:53.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:53.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:53.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:53.850 | INFO | ANALYSIS : 69.6%, 2.9, 107.0ms, 0 | 20260507 18:11:53.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:53.850 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260507 18:11:53.850 | INFO | PRE_HQ : 67.6%, 1.9, 66.1ms, 0 | 20260507 18:11:53.850 | INFO | BAZWRITER : 68.8%, 3.1, 108.1ms, 0 | 20260507 18:11:53.850 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:11:53.850 | INFO | Uploaded 36GiB to GPU in 2.55481s (14.0911GiB/s) | 20260507 18:11:53.850 | INFO | Downloaded 38GiB to GPU in 2.71006s (14.0218GiB/s) | 20260507 18:11:58.989 | INFO | Analyzing chunk frames = [1552896,1553408) | 20260507 18:11:58.989 | INFO | MemInfo: { | 20260507 18:11:58.989 | INFO | "gpu" : " 17 GB", | 20260507 18:11:58.989 | INFO | "huge" : "", | 20260507 18:11:58.989 | INFO | "huge_shared" : " 29 GB", | 20260507 18:11:58.989 | INFO | "malloc" : " 14 GB", | 20260507 18:11:58.989 | INFO | "pinned" : " 29 GB" | 20260507 18:11:58.989 | INFO | } | 20260507 18:11:58.990 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:11:58.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:11:58.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:11:58.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:11:58.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:11:58.990 | INFO | Uploaded 36GiB to GPU in 2.55382s (14.0965GiB/s) | 20260507 18:11:58.990 | INFO | Downloaded 34GiB to GPU in 2.43874s (13.9416GiB/s) | 20260507 18:12:04.129 | INFO | Analyzing chunk frames = [1553408,1553920) | 20260507 18:12:04.129 | INFO | MemInfo: { | 20260507 18:12:04.129 | INFO | "gpu" : " 17 GB", | 20260507 18:12:04.129 | INFO | "huge" : "", | 20260507 18:12:04.129 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:04.129 | INFO | "malloc" : " 14 GB", | 20260507 18:12:04.129 | INFO | "pinned" : " 29 GB" | 20260507 18:12:04.129 | INFO | } | 20260507 18:12:04.129 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:12:04.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:04.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.129 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:12:04.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.129 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:12:04.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:04.130 | INFO | Uploaded 36GiB to GPU in 2.57848s (13.9617GiB/s) | 20260507 18:12:04.130 | INFO | Downloaded 34GiB to GPU in 2.41654s (14.0697GiB/s) | 20260507 18:12:09.268 | INFO | Analyzing chunk frames = [1553920,1554432) | 20260507 18:12:09.268 | INFO | MemInfo: { | 20260507 18:12:09.268 | INFO | "gpu" : " 17 GB", | 20260507 18:12:09.268 | INFO | "huge" : "", | 20260507 18:12:09.268 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:09.268 | INFO | "malloc" : " 14 GB", | 20260507 18:12:09.268 | INFO | "pinned" : " 29 GB" | 20260507 18:12:09.268 | INFO | } | 20260507 18:12:09.268 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:12:09.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:09.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.268 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:12:09.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.268 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:12:09.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:09.269 | INFO | Uploaded 36GiB to GPU in 2.50519s (14.3702GiB/s) | 20260507 18:12:09.269 | INFO | Downloaded 34GiB to GPU in 2.34963s (14.4704GiB/s) | 20260507 18:12:14.408 | INFO | Analyzing chunk frames = [1554432,1554944) | 20260507 18:12:14.408 | INFO | MemInfo: { | 20260507 18:12:14.408 | INFO | "gpu" : " 17 GB", | 20260507 18:12:14.408 | INFO | "huge" : "", | 20260507 18:12:14.408 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:14.408 | INFO | "malloc" : " 14 GB", | 20260507 18:12:14.408 | INFO | "pinned" : " 29 GB" | 20260507 18:12:14.408 | INFO | } | 20260507 18:12:14.408 | INFO | Perf report for the last 5.14009 seconds: | 20260507 18:12:14.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:14.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.408 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:12:14.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.408 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:12:14.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:14.409 | INFO | Uploaded 36GiB to GPU in 2.35939s (15.2582GiB/s) | 20260507 18:12:14.409 | INFO | Downloaded 34GiB to GPU in 2.46087s (13.8163GiB/s) | 20260507 18:12:19.546 | INFO | Analyzing chunk frames = [1554944,1555456) | 20260507 18:12:19.546 | INFO | MemInfo: { | 20260507 18:12:19.546 | INFO | "gpu" : " 17 GB", | 20260507 18:12:19.546 | INFO | "huge" : "", | 20260507 18:12:19.546 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:19.547 | INFO | "malloc" : " 14 GB", | 20260507 18:12:19.547 | INFO | "pinned" : " 29 GB" | 20260507 18:12:19.547 | INFO | } | 20260507 18:12:19.547 | INFO | Perf report for the last 5.13845 seconds: | 20260507 18:12:19.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:19.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:12:19.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:12:19.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:19.547 | INFO | Uploaded 36GiB to GPU in 2.46971s (14.5766GiB/s) | 20260507 18:12:19.547 | INFO | Downloaded 34GiB to GPU in 2.50924s (13.5499GiB/s) | 20260507 18:12:24.686 | INFO | Analyzing chunk frames = [1555456,1555968) | 20260507 18:12:24.686 | INFO | MemInfo: { | 20260507 18:12:24.686 | INFO | "gpu" : " 17 GB", | 20260507 18:12:24.686 | INFO | "huge" : "", | 20260507 18:12:24.686 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:24.686 | INFO | "malloc" : " 14 GB", | 20260507 18:12:24.686 | INFO | "pinned" : " 29 GB" | 20260507 18:12:24.686 | INFO | } | 20260507 18:12:24.686 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:12:24.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:24.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.686 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:12:24.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.687 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 18:12:24.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:24.687 | INFO | Uploaded 36GiB to GPU in 2.46859s (14.5832GiB/s) | 20260507 18:12:24.687 | INFO | Downloaded 34GiB to GPU in 2.42846s (14.0006GiB/s) | 20260507 18:12:29.825 | INFO | Analyzing chunk frames = [1555968,1556480) | 20260507 18:12:29.825 | INFO | MemInfo: { | 20260507 18:12:29.825 | INFO | "gpu" : " 17 GB", | 20260507 18:12:29.825 | INFO | "huge" : "", | 20260507 18:12:29.825 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:29.825 | INFO | "malloc" : " 14 GB", | 20260507 18:12:29.825 | INFO | "pinned" : " 29 GB" | 20260507 18:12:29.825 | INFO | } | 20260507 18:12:29.825 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:12:29.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:29.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:12:29.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 18:12:29.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:29.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:12:29.826 | INFO | Total mem usage: 16.4 GiB | 20260507 18:12:29.826 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:12:29.826 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.5%) | 20260507 18:12:29.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:12:29.826 | INFO | High Water Breakdown: | 20260507 18:12:29.826 | INFO | 59.3% : 8.65 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:12:29.826 | INFO | 40.5% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:12:29.826 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:12:29.826 | INFO | ------------------------------------------ | 20260507 18:12:29.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:12:29.826 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:12:29.826 | INFO | Total mem usage: 26.5 GiB | 20260507 18:12:29.826 | INFO | Uploaded 36GiB to GPU in 2.48325s (14.4971GiB/s) | 20260507 18:12:29.826 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:12:29.826 | INFO | Downloaded 34GiB to GPU in 2.42196s (14.0382GiB/s) | 20260507 18:12:29.826 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:12:29.826 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 18:12:29.826 | INFO | High Water Breakdown: | 20260507 18:12:29.826 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:12:29.826 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:12:29.826 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:12:29.826 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:12:29.827 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:12:29.827 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:12:29.827 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:12:29.827 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:12:29.827 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:12:29.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:12:29.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:12:29.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:12:29.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:12:29.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:12:29.827 | INFO | ------------------------------------------ | 20260507 18:12:29.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:12:29.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:12:29.827 | INFO | Total mem usage: 27.8 GiB | 20260507 18:12:29.827 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:12:29.827 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:12:29.827 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:12:29.827 | INFO | High Water Breakdown: | 20260507 18:12:29.827 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:12:29.827 | INFO | ------------------------------------------ | 20260507 18:12:29.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:12:29.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:12:29.827 | INFO | Total mem usage: 15.7 GiB | 20260507 18:12:29.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:12:29.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:12:29.827 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:12:29.827 | INFO | High Water Breakdown: | 20260507 18:12:29.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:12:29.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:12:29.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:12:29.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:12:29.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:12:29.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:12:29.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:12:29.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:12:29.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:12:29.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:12:29.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:12:29.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:12:29.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:12:29.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:12:29.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:12:29.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:12:29.827 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:12:29.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:12:29.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:12:29.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:12:29.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:12:29.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:12:29.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:12:29.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:12:29.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:12:29.827 | INFO | ------------------------------------------ | 20260507 18:12:29.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:12:29.827 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:12:29.827 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:12:29.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:12:29.827 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.63e+01, 1.45e+01 / 2.41e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.03e-01, 1.16e-02 / 5.67e-03 | 20260507 18:12:29.827 | INFO | PerfReport: 12.11%, Baseline -- 4.05e+00 / 4.51e+00, 6.18e-01 / 2.39e-01 | 20260507 18:12:29.827 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.17e-01 / 1.29e-01 | 20260507 18:12:29.827 | INFO | PerfReport: 15.67%, DME -- 4.19e+01 / 4.49e+01, 4.10e+00 / 1.96e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.89e+00 / 2.22e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.37e-01 / 2.43e-01 | 20260507 18:12:29.827 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.43e+00, 3.14e-01 / 1.84e-01 | 20260507 18:12:29.827 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.93e-01 / 4.20e-01, 6.35e-02 / 3.57e-02 | 20260507 18:12:29.827 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:12:29.827 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:12:29.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:12:29.827 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 24.17%, Upload -- 2.60e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 23.96%, Download -- 2.57e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 31.14%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 1.95%, LockUpload -- 2.10e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 2.25%, LockDownload -- 2.41e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 16.51%, LockCompute -- 1.77e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:12:29.827 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 18:12:29.958 | INFO | Starting a round of Baz IO | 20260507 18:12:33.447 | INFO | Populating RTMetrics report | 20260507 18:12:33.448 | INFO | RTMetrics report populated | 20260507 18:12:33.448 | INFO | Opening tmp RTMetrics json file | 20260507 18:12:33.449 | INFO | Serializing RTMetrics to json | 20260507 18:12:33.450 | INFO | Serialization done, swapping tmp file now... | 20260507 18:12:33.450 | INFO | RTMetrics done | 20260507 18:12:33.535 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.576 seconds | 20260507 18:12:33.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.94% / 0.02% / 53.05% | 20260507 18:12:33.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.07079% / 4.37101e-05% / 2.38419e-05% | 20260507 18:12:33.535 | INFO | Baz IO overall: | 20260507 18:12:33.535 | INFO | BytesWritten : 693593959740 | 20260507 18:12:33.535 | INFO | NumEvents : 336222645179 | 20260507 18:12:34.965 | INFO | Analyzing chunk frames = [1556480,1556992) | 20260507 18:12:34.965 | INFO | MemInfo: { | 20260507 18:12:34.965 | INFO | "gpu" : " 17 GB", | 20260507 18:12:34.965 | INFO | "huge" : "", | 20260507 18:12:34.965 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:34.965 | INFO | "malloc" : " 14 GB", | 20260507 18:12:34.965 | INFO | "pinned" : " 29 GB" | 20260507 18:12:34.965 | INFO | } | 20260507 18:12:34.965 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:12:34.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:34.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:34.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:34.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:34.965 | INFO | ANALYSIS : 69.2%, 2.9, 106.8ms, 0 | 20260507 18:12:34.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:34.965 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 18:12:34.965 | INFO | PRE_HQ : 67.1%, 1.9, 65.3ms, 0 | 20260507 18:12:34.965 | INFO | BAZWRITER : 69.4%, 3.0, 107.8ms, 0 | 20260507 18:12:34.965 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 18:12:34.965 | INFO | Uploaded 36GiB to GPU in 2.50478s (14.3725GiB/s) | 20260507 18:12:34.965 | INFO | Downloaded 38GiB to GPU in 2.71242s (14.0096GiB/s) | 20260507 18:12:40.105 | INFO | Analyzing chunk frames = [1556992,1557504) | 20260507 18:12:40.105 | INFO | MemInfo: { | 20260507 18:12:40.105 | INFO | "gpu" : " 17 GB", | 20260507 18:12:40.105 | INFO | "huge" : "", | 20260507 18:12:40.105 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:40.105 | INFO | "malloc" : " 14 GB", | 20260507 18:12:40.105 | INFO | "pinned" : " 29 GB" | 20260507 18:12:40.105 | INFO | } | 20260507 18:12:40.105 | INFO | Perf report for the last 5.13996 seconds: | 20260507 18:12:40.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:40.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 18:12:40.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:12:40.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:40.105 | INFO | Uploaded 36GiB to GPU in 2.47836s (14.5257GiB/s) | 20260507 18:12:40.105 | INFO | Downloaded 34GiB to GPU in 2.4609s (13.8161GiB/s) | 20260507 18:12:45.243 | INFO | Analyzing chunk frames = [1557504,1558016) | 20260507 18:12:45.243 | INFO | MemInfo: { | 20260507 18:12:45.243 | INFO | "gpu" : " 17 GB", | 20260507 18:12:45.243 | INFO | "huge" : "", | 20260507 18:12:45.243 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:45.244 | INFO | "malloc" : " 14 GB", | 20260507 18:12:45.244 | INFO | "pinned" : " 29 GB" | 20260507 18:12:45.244 | INFO | } | 20260507 18:12:45.244 | INFO | Perf report for the last 5.13891 seconds: | 20260507 18:12:45.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:45.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:12:45.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:12:45.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:45.244 | INFO | Uploaded 36GiB to GPU in 2.49516s (14.4279GiB/s) | 20260507 18:12:45.244 | INFO | Downloaded 34GiB to GPU in 2.45708s (13.8375GiB/s) | 20260507 18:12:50.383 | INFO | Analyzing chunk frames = [1558016,1558528) | 20260507 18:12:50.383 | INFO | MemInfo: { | 20260507 18:12:50.383 | INFO | "gpu" : " 17 GB", | 20260507 18:12:50.383 | INFO | "huge" : "", | 20260507 18:12:50.383 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:50.383 | INFO | "malloc" : " 14 GB", | 20260507 18:12:50.383 | INFO | "pinned" : " 29 GB" | 20260507 18:12:50.383 | INFO | } | 20260507 18:12:50.383 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:12:50.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:50.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.383 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:12:50.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.383 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:12:50.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:50.384 | INFO | Uploaded 36GiB to GPU in 2.44551s (14.7209GiB/s) | 20260507 18:12:50.384 | INFO | Downloaded 34GiB to GPU in 2.36399s (14.3824GiB/s) | 20260507 18:12:55.523 | INFO | Analyzing chunk frames = [1558528,1559040) | 20260507 18:12:55.523 | INFO | MemInfo: { | 20260507 18:12:55.523 | INFO | "gpu" : " 17 GB", | 20260507 18:12:55.523 | INFO | "huge" : "", | 20260507 18:12:55.523 | INFO | "huge_shared" : " 29 GB", | 20260507 18:12:55.523 | INFO | "malloc" : " 14 GB", | 20260507 18:12:55.523 | INFO | "pinned" : " 29 GB" | 20260507 18:12:55.523 | INFO | } | 20260507 18:12:55.523 | INFO | Perf report for the last 5.13975 seconds: | 20260507 18:12:55.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:12:55.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:12:55.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 18:12:55.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:12:55.523 | INFO | Uploaded 36GiB to GPU in 2.3247s (15.4858GiB/s) | 20260507 18:12:55.523 | INFO | Downloaded 34GiB to GPU in 2.48542s (13.6798GiB/s) | 20260507 18:13:00.661 | INFO | Analyzing chunk frames = [1559040,1559552) | 20260507 18:13:00.661 | INFO | MemInfo: { | 20260507 18:13:00.661 | INFO | "gpu" : " 17 GB", | 20260507 18:13:00.661 | INFO | "huge" : "", | 20260507 18:13:00.661 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:00.661 | INFO | "malloc" : " 14 GB", | 20260507 18:13:00.662 | INFO | "pinned" : " 29 GB" | 20260507 18:13:00.662 | INFO | } | 20260507 18:13:00.662 | INFO | Perf report for the last 5.13908 seconds: | 20260507 18:13:00.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:00.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:13:00.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 18:13:00.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:00.662 | INFO | Uploaded 36GiB to GPU in 2.42893s (14.8213GiB/s) | 20260507 18:13:00.662 | INFO | Downloaded 34GiB to GPU in 2.50669s (13.5637GiB/s) | 20260507 18:13:05.801 | INFO | Analyzing chunk frames = [1559552,1560064) | 20260507 18:13:05.801 | INFO | MemInfo: { | 20260507 18:13:05.801 | INFO | "gpu" : " 17 GB", | 20260507 18:13:05.801 | INFO | "huge" : "", | 20260507 18:13:05.801 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:05.801 | INFO | "malloc" : " 14 GB", | 20260507 18:13:05.801 | INFO | "pinned" : " 29 GB" | 20260507 18:13:05.801 | INFO | } | 20260507 18:13:05.801 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:13:05.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:05.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.801 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:13:05.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.802 | INFO | PRE_HQ : 38.2%, 1.0, 19.9ms, 0 | 20260507 18:13:05.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:05.802 | INFO | Uploaded 36GiB to GPU in 2.49816s (14.4106GiB/s) | 20260507 18:13:05.802 | INFO | Downloaded 34GiB to GPU in 2.43945s (13.9376GiB/s) | 20260507 18:13:10.940 | INFO | Analyzing chunk frames = [1560064,1560576) | 20260507 18:13:10.940 | INFO | MemInfo: { | 20260507 18:13:10.940 | INFO | "gpu" : " 17 GB", | 20260507 18:13:10.940 | INFO | "huge" : "", | 20260507 18:13:10.940 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:10.940 | INFO | "malloc" : " 14 GB", | 20260507 18:13:10.940 | INFO | "pinned" : " 29 GB" | 20260507 18:13:10.940 | INFO | } | 20260507 18:13:10.940 | INFO | Perf report for the last 5.13905 seconds: | 20260507 18:13:10.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:10.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:13:10.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 18:13:10.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:10.941 | INFO | Uploaded 36GiB to GPU in 2.48641s (14.4787GiB/s) | 20260507 18:13:10.941 | INFO | Downloaded 34GiB to GPU in 2.43015s (13.9909GiB/s) | 20260507 18:13:11.051 | INFO | Starting a round of Baz IO | 20260507 18:13:14.564 | INFO | Populating RTMetrics report | 20260507 18:13:14.564 | INFO | RTMetrics report populated | 20260507 18:13:14.564 | INFO | Opening tmp RTMetrics json file | 20260507 18:13:14.578 | INFO | Serializing RTMetrics to json | 20260507 18:13:14.580 | INFO | Serialization done, swapping tmp file now... | 20260507 18:13:14.580 | INFO | RTMetrics done | 20260507 18:13:14.664 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.612 seconds | 20260507 18:13:14.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.92% / 0.02% / 53.06% | 20260507 18:13:14.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.06528% / 1.98682e-05% / 3.97364e-05% | 20260507 18:13:14.664 | INFO | Baz IO overall: | 20260507 18:13:14.664 | INFO | BytesWritten : 695527639717 | 20260507 18:13:14.665 | INFO | NumEvents : 337161022223 | 20260507 18:13:16.080 | INFO | Analyzing chunk frames = [1560576,1561088) | 20260507 18:13:16.080 | INFO | MemInfo: { | 20260507 18:13:16.080 | INFO | "gpu" : " 17 GB", | 20260507 18:13:16.080 | INFO | "huge" : "", | 20260507 18:13:16.080 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:16.080 | INFO | "malloc" : " 14 GB", | 20260507 18:13:16.080 | INFO | "pinned" : " 29 GB" | 20260507 18:13:16.080 | INFO | } | 20260507 18:13:16.080 | INFO | Perf report for the last 5.13956 seconds: | 20260507 18:13:16.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:16.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:16.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:16.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:16.080 | INFO | ANALYSIS : 69.6%, 2.9, 107.4ms, 0 | 20260507 18:13:16.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:16.080 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 18:13:16.080 | INFO | PRE_HQ : 67.2%, 1.9, 65.2ms, 0 | 20260507 18:13:16.080 | INFO | BAZWRITER : 69.4%, 3.0, 108.1ms, 0 | 20260507 18:13:16.080 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:13:16.081 | INFO | Uploaded 36GiB to GPU in 2.57047s (14.0052GiB/s) | 20260507 18:13:16.081 | INFO | Downloaded 38GiB to GPU in 2.69479s (14.1013GiB/s) | 20260507 18:13:21.219 | INFO | Analyzing chunk frames = [1561088,1561600) | 20260507 18:13:21.219 | INFO | MemInfo: { | 20260507 18:13:21.219 | INFO | "gpu" : " 17 GB", | 20260507 18:13:21.219 | INFO | "huge" : "", | 20260507 18:13:21.219 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:21.219 | INFO | "malloc" : " 14 GB", | 20260507 18:13:21.219 | INFO | "pinned" : " 29 GB" | 20260507 18:13:21.219 | INFO | } | 20260507 18:13:21.219 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:13:21.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:21.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.220 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:13:21.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.220 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:13:21.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:21.220 | INFO | Uploaded 36GiB to GPU in 2.52294s (14.2691GiB/s) | 20260507 18:13:21.220 | INFO | Downloaded 34GiB to GPU in 2.45528s (13.8477GiB/s) | 20260507 18:13:26.358 | INFO | Analyzing chunk frames = [1561600,1562112) | 20260507 18:13:26.358 | INFO | MemInfo: { | 20260507 18:13:26.358 | INFO | "gpu" : " 17 GB", | 20260507 18:13:26.358 | INFO | "huge" : "", | 20260507 18:13:26.358 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:26.359 | INFO | "malloc" : " 14 GB", | 20260507 18:13:26.359 | INFO | "pinned" : " 29 GB" | 20260507 18:13:26.359 | INFO | } | 20260507 18:13:26.359 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:13:26.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:26.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:13:26.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:13:26.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:26.359 | INFO | Uploaded 36GiB to GPU in 2.55361s (14.0977GiB/s) | 20260507 18:13:26.359 | INFO | Downloaded 34GiB to GPU in 2.45372s (13.8565GiB/s) | 20260507 18:13:31.498 | INFO | Analyzing chunk frames = [1562112,1562624) | 20260507 18:13:31.498 | INFO | MemInfo: { | 20260507 18:13:31.498 | INFO | "gpu" : " 17 GB", | 20260507 18:13:31.498 | INFO | "huge" : "", | 20260507 18:13:31.498 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:31.498 | INFO | "malloc" : " 14 GB", | 20260507 18:13:31.498 | INFO | "pinned" : " 29 GB" | 20260507 18:13:31.498 | INFO | } | 20260507 18:13:31.498 | INFO | Perf report for the last 5.1395 seconds: | 20260507 18:13:31.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:31.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.498 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:13:31.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.498 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:13:31.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:31.499 | INFO | Uploaded 36GiB to GPU in 2.47653s (14.5365GiB/s) | 20260507 18:13:31.499 | INFO | Downloaded 34GiB to GPU in 2.36223s (14.3932GiB/s) | 20260507 18:13:36.638 | INFO | Analyzing chunk frames = [1562624,1563136) | 20260507 18:13:36.638 | INFO | MemInfo: { | 20260507 18:13:36.638 | INFO | "gpu" : " 17 GB", | 20260507 18:13:36.638 | INFO | "huge" : "", | 20260507 18:13:36.638 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:36.638 | INFO | "malloc" : " 14 GB", | 20260507 18:13:36.638 | INFO | "pinned" : " 29 GB" | 20260507 18:13:36.638 | INFO | } | 20260507 18:13:36.638 | INFO | Perf report for the last 5.13973 seconds: | 20260507 18:13:36.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:36.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.638 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:13:36.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.638 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 18:13:36.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:36.639 | INFO | Uploaded 36GiB to GPU in 2.30437s (15.6225GiB/s) | 20260507 18:13:36.639 | INFO | Downloaded 34GiB to GPU in 2.49458s (13.6295GiB/s) | 20260507 18:13:41.776 | INFO | Analyzing chunk frames = [1563136,1563648) | 20260507 18:13:41.776 | INFO | MemInfo: { | 20260507 18:13:41.776 | INFO | "gpu" : " 17 GB", | 20260507 18:13:41.776 | INFO | "huge" : "", | 20260507 18:13:41.776 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:41.777 | INFO | "malloc" : " 14 GB", | 20260507 18:13:41.777 | INFO | "pinned" : " 29 GB" | 20260507 18:13:41.777 | INFO | } | 20260507 18:13:41.777 | INFO | Perf report for the last 5.13859 seconds: | 20260507 18:13:41.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:41.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:13:41.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 18:13:41.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:41.777 | INFO | Uploaded 36GiB to GPU in 2.41043s (14.9351GiB/s) | 20260507 18:13:41.777 | INFO | Downloaded 34GiB to GPU in 2.49019s (13.6536GiB/s) | 20260507 18:13:46.915 | INFO | Analyzing chunk frames = [1563648,1564160) | 20260507 18:13:46.915 | INFO | MemInfo: { | 20260507 18:13:46.915 | INFO | "gpu" : " 17 GB", | 20260507 18:13:46.916 | INFO | "huge" : "", | 20260507 18:13:46.916 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:46.916 | INFO | "malloc" : " 14 GB", | 20260507 18:13:46.916 | INFO | "pinned" : " 29 GB" | 20260507 18:13:46.916 | INFO | } | 20260507 18:13:46.916 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:13:46.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:46.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.916 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:13:46.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.916 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 18:13:46.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:46.917 | INFO | Uploaded 36GiB to GPU in 2.50781s (14.3551GiB/s) | 20260507 18:13:46.917 | INFO | Downloaded 34GiB to GPU in 2.44853s (13.8859GiB/s) | 20260507 18:13:52.055 | INFO | Analyzing chunk frames = [1564160,1564672) | 20260507 18:13:52.055 | INFO | MemInfo: { | 20260507 18:13:52.055 | INFO | "gpu" : " 17 GB", | 20260507 18:13:52.055 | INFO | "huge" : "", | 20260507 18:13:52.055 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:52.055 | INFO | "malloc" : " 14 GB", | 20260507 18:13:52.055 | INFO | "pinned" : " 29 GB" | 20260507 18:13:52.055 | INFO | } | 20260507 18:13:52.055 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:13:52.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:52.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.056 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:13:52.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.056 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 18:13:52.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:52.056 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:13:52.056 | INFO | Total mem usage: 16.4 GiB | 20260507 18:13:52.056 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:13:52.056 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.2%) | 20260507 18:13:52.056 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:13:52.056 | INFO | High Water Breakdown: | 20260507 18:13:52.056 | INFO | 59.2% : 8.58 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:13:52.056 | INFO | 40.6% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:13:52.056 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:13:52.056 | INFO | ------------------------------------------ | 20260507 18:13:52.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:13:52.056 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:13:52.056 | INFO | Total mem usage: 26.5 GiB | 20260507 18:13:52.056 | INFO | Uploaded 36GiB to GPU in 2.49345s (14.4378GiB/s) | 20260507 18:13:52.056 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:13:52.056 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:13:52.056 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:13:52.056 | INFO | High Water Breakdown: | 20260507 18:13:52.056 | INFO | Downloaded 34GiB to GPU in 2.42915s (13.9967GiB/s) | 20260507 18:13:52.056 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:13:52.056 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:13:52.056 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:13:52.056 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:13:52.056 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:13:52.056 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:13:52.056 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:13:52.056 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:13:52.056 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:13:52.056 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:13:52.056 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:13:52.056 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:13:52.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:13:52.057 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:13:52.057 | INFO | ------------------------------------------ | 20260507 18:13:52.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:13:52.057 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:13:52.057 | INFO | Total mem usage: 27.8 GiB | 20260507 18:13:52.057 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:13:52.057 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:13:52.057 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:13:52.057 | INFO | High Water Breakdown: | 20260507 18:13:52.057 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:13:52.057 | INFO | ------------------------------------------ | 20260507 18:13:52.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:13:52.057 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:13:52.057 | INFO | Total mem usage: 15.7 GiB | 20260507 18:13:52.057 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:13:52.057 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:13:52.057 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:13:52.057 | INFO | High Water Breakdown: | 20260507 18:13:52.057 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:13:52.057 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:13:52.057 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:13:52.057 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:13:52.057 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:13:52.057 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:13:52.057 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:13:52.057 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:13:52.057 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:13:52.057 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:13:52.057 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:13:52.057 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:13:52.057 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:13:52.057 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:13:52.057 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:13:52.057 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:13:52.057 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:13:52.057 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:13:52.057 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:13:52.057 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:13:52.057 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:13:52.057 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:13:52.057 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:13:52.057 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:13:52.057 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:13:52.057 | INFO | ------------------------------------------ | 20260507 18:13:52.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:13:52.058 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:13:52.058 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:13:52.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:13:52.058 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.43e+01 / 2.22e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 0.30%, AggregateStats -- 1.01e-01 / 1.05e-01, 1.27e-02 / 5.77e-03 | 20260507 18:13:52.058 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.52e+00, 6.14e-01 / 2.17e-01 | 20260507 18:13:52.058 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 1.21e-01 | 20260507 18:13:52.058 | INFO | PerfReport: 15.65%, DME -- 4.19e+01 / 4.48e+01, 4.05e+00 / 1.90e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.91e+00 / 2.18e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.37e-01 / 2.65e-01 | 20260507 18:13:52.058 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.45e+00, 3.15e-01 / 1.73e-01 | 20260507 18:13:52.058 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.96e-01 / 4.28e-01, 6.43e-02 / 3.29e-02 | 20260507 18:13:52.058 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:13:52.058 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:13:52.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:13:52.058 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 23.99%, Upload -- 2.57e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 24.10%, Download -- 2.58e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 31.23%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 1.79%, LockUpload -- 1.92e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 2.32%, LockDownload -- 2.48e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 16.56%, LockCompute -- 1.78e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:13:52.058 | INFO | PerfReport: 0.02%, Misc -- 1.65e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260507 18:13:52.168 | INFO | Starting a round of Baz IO | 20260507 18:13:55.669 | INFO | Populating RTMetrics report | 20260507 18:13:55.669 | INFO | RTMetrics report populated | 20260507 18:13:55.669 | INFO | Opening tmp RTMetrics json file | 20260507 18:13:55.685 | INFO | Serializing RTMetrics to json | 20260507 18:13:55.687 | INFO | Serialization done, swapping tmp file now... | 20260507 18:13:55.687 | INFO | RTMetrics done | 20260507 18:13:55.749 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.581 seconds | 20260507 18:13:55.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.90% / 0.02% / 53.08% | 20260507 18:13:55.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.06682% / 1.98682e-05% / 5.96046e-05% | 20260507 18:13:55.749 | INFO | Baz IO overall: | 20260507 18:13:55.749 | INFO | BytesWritten : 697462758522 | 20260507 18:13:55.749 | INFO | NumEvents : 338100520269 | 20260507 18:13:57.194 | INFO | Analyzing chunk frames = [1564672,1565184) | 20260507 18:13:57.194 | INFO | MemInfo: { | 20260507 18:13:57.194 | INFO | "gpu" : " 17 GB", | 20260507 18:13:57.194 | INFO | "huge" : "", | 20260507 18:13:57.194 | INFO | "huge_shared" : " 29 GB", | 20260507 18:13:57.194 | INFO | "malloc" : " 14 GB", | 20260507 18:13:57.194 | INFO | "pinned" : " 29 GB" | 20260507 18:13:57.194 | INFO | } | 20260507 18:13:57.194 | INFO | Perf report for the last 5.13875 seconds: | 20260507 18:13:57.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:13:57.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:57.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:57.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:57.194 | INFO | ANALYSIS : 69.5%, 3.0, 107.6ms, 0 | 20260507 18:13:57.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:13:57.194 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 18:13:57.194 | INFO | PRE_HQ : 66.9%, 1.8, 64.1ms, 0 | 20260507 18:13:57.194 | INFO | BAZWRITER : 68.9%, 3.1, 109.1ms, 0 | 20260507 18:13:57.194 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:13:57.195 | INFO | Uploaded 36GiB to GPU in 2.5439s (14.1515GiB/s) | 20260507 18:13:57.195 | INFO | Downloaded 38GiB to GPU in 2.72322s (13.9541GiB/s) | 20260507 18:14:02.334 | INFO | Analyzing chunk frames = [1565184,1565696) | 20260507 18:14:02.334 | INFO | MemInfo: { | 20260507 18:14:02.334 | INFO | "gpu" : " 17 GB", | 20260507 18:14:02.334 | INFO | "huge" : "", | 20260507 18:14:02.334 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:02.334 | INFO | "malloc" : " 14 GB", | 20260507 18:14:02.334 | INFO | "pinned" : " 29 GB" | 20260507 18:14:02.334 | INFO | } | 20260507 18:14:02.334 | INFO | Perf report for the last 5.14003 seconds: | 20260507 18:14:02.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:02.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.334 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:14:02.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.334 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:14:02.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:02.335 | INFO | Uploaded 36GiB to GPU in 2.50032s (14.3981GiB/s) | 20260507 18:14:02.335 | INFO | Downloaded 34GiB to GPU in 2.45775s (13.8338GiB/s) | 20260507 18:14:07.473 | INFO | Analyzing chunk frames = [1565696,1566208) | 20260507 18:14:07.473 | INFO | MemInfo: { | 20260507 18:14:07.473 | INFO | "gpu" : " 17 GB", | 20260507 18:14:07.473 | INFO | "huge" : "", | 20260507 18:14:07.473 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:07.473 | INFO | "malloc" : " 14 GB", | 20260507 18:14:07.473 | INFO | "pinned" : " 29 GB" | 20260507 18:14:07.473 | INFO | } | 20260507 18:14:07.473 | INFO | Perf report for the last 5.13895 seconds: | 20260507 18:14:07.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:07.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.473 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:14:07.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.473 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:14:07.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:07.474 | INFO | Uploaded 36GiB to GPU in 2.53581s (14.1967GiB/s) | 20260507 18:14:07.474 | INFO | Downloaded 34GiB to GPU in 2.45072s (13.8735GiB/s) | 20260507 18:14:12.612 | INFO | Analyzing chunk frames = [1566208,1566720) | 20260507 18:14:12.612 | INFO | MemInfo: { | 20260507 18:14:12.612 | INFO | "gpu" : " 17 GB", | 20260507 18:14:12.612 | INFO | "huge" : "", | 20260507 18:14:12.612 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:12.612 | INFO | "malloc" : " 14 GB", | 20260507 18:14:12.613 | INFO | "pinned" : " 29 GB" | 20260507 18:14:12.613 | INFO | } | 20260507 18:14:12.613 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:14:12.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:12.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260507 18:14:12.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:14:12.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:12.613 | INFO | Uploaded 36GiB to GPU in 2.47969s (14.5179GiB/s) | 20260507 18:14:12.613 | INFO | Downloaded 34GiB to GPU in 2.35172s (14.4575GiB/s) | 20260507 18:14:17.752 | INFO | Analyzing chunk frames = [1566720,1567232) | 20260507 18:14:17.752 | INFO | MemInfo: { | 20260507 18:14:17.752 | INFO | "gpu" : " 17 GB", | 20260507 18:14:17.752 | INFO | "huge" : "", | 20260507 18:14:17.752 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:17.752 | INFO | "malloc" : " 14 GB", | 20260507 18:14:17.752 | INFO | "pinned" : " 29 GB" | 20260507 18:14:17.752 | INFO | } | 20260507 18:14:17.752 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:14:17.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:17.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:14:17.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:14:17.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:17.752 | INFO | Uploaded 36GiB to GPU in 2.34985s (15.3201GiB/s) | 20260507 18:14:17.752 | INFO | Downloaded 34GiB to GPU in 2.48464s (13.6841GiB/s) | 20260507 18:14:22.891 | INFO | Analyzing chunk frames = [1567232,1567744) | 20260507 18:14:22.891 | INFO | MemInfo: { | 20260507 18:14:22.891 | INFO | "gpu" : " 17 GB", | 20260507 18:14:22.891 | INFO | "huge" : "", | 20260507 18:14:22.891 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:22.891 | INFO | "malloc" : " 14 GB", | 20260507 18:14:22.891 | INFO | "pinned" : " 29 GB" | 20260507 18:14:22.891 | INFO | } | 20260507 18:14:22.891 | INFO | Perf report for the last 5.13883 seconds: | 20260507 18:14:22.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:22.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.891 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:14:22.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.891 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:14:22.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:22.892 | INFO | Uploaded 36GiB to GPU in 2.3994s (15.0038GiB/s) | 20260507 18:14:22.892 | INFO | Downloaded 34GiB to GPU in 2.52662s (13.4567GiB/s) | 20260507 18:14:28.030 | INFO | Analyzing chunk frames = [1567744,1568256) | 20260507 18:14:28.030 | INFO | MemInfo: { | 20260507 18:14:28.030 | INFO | "gpu" : " 17 GB", | 20260507 18:14:28.030 | INFO | "huge" : "", | 20260507 18:14:28.030 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:28.030 | INFO | "malloc" : " 14 GB", | 20260507 18:14:28.030 | INFO | "pinned" : " 29 GB" | 20260507 18:14:28.030 | INFO | } | 20260507 18:14:28.031 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:14:28.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:28.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:14:28.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:14:28.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:28.031 | INFO | Uploaded 36GiB to GPU in 2.46383s (14.6114GiB/s) | 20260507 18:14:28.031 | INFO | Downloaded 34GiB to GPU in 2.45444s (13.8525GiB/s) | 20260507 18:14:33.170 | INFO | Analyzing chunk frames = [1568256,1568768) | 20260507 18:14:33.170 | INFO | MemInfo: { | 20260507 18:14:33.170 | INFO | "gpu" : " 17 GB", | 20260507 18:14:33.170 | INFO | "huge" : "", | 20260507 18:14:33.170 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:33.170 | INFO | "malloc" : " 14 GB", | 20260507 18:14:33.170 | INFO | "pinned" : " 29 GB" | 20260507 18:14:33.170 | INFO | } | 20260507 18:14:33.170 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:14:33.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:33.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:14:33.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 18:14:33.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:33.170 | INFO | Uploaded 36GiB to GPU in 2.43427s (14.7888GiB/s) | 20260507 18:14:33.170 | INFO | Downloaded 34GiB to GPU in 2.43611s (13.9567GiB/s) | 20260507 18:14:33.268 | INFO | Starting a round of Baz IO | 20260507 18:14:36.769 | INFO | Populating RTMetrics report | 20260507 18:14:36.769 | INFO | RTMetrics report populated | 20260507 18:14:36.769 | INFO | Opening tmp RTMetrics json file | 20260507 18:14:36.785 | INFO | Serializing RTMetrics to json | 20260507 18:14:36.787 | INFO | Serialization done, swapping tmp file now... | 20260507 18:14:36.787 | INFO | RTMetrics done | 20260507 18:14:36.848 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.58 seconds | 20260507 18:14:36.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.89% / 0.02% / 53.10% | 20260507 18:14:36.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.05811% / 3.57628e-05% / 5.16574e-05% | 20260507 18:14:36.848 | INFO | Baz IO overall: | 20260507 18:14:36.848 | INFO | BytesWritten : 699398212925 | 20260507 18:14:36.848 | INFO | NumEvents : 339040267319 | 20260507 18:14:38.309 | INFO | Analyzing chunk frames = [1568768,1569280) | 20260507 18:14:38.309 | INFO | MemInfo: { | 20260507 18:14:38.309 | INFO | "gpu" : " 17 GB", | 20260507 18:14:38.309 | INFO | "huge" : "", | 20260507 18:14:38.309 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:38.309 | INFO | "malloc" : " 14 GB", | 20260507 18:14:38.309 | INFO | "pinned" : " 29 GB" | 20260507 18:14:38.310 | INFO | } | 20260507 18:14:38.310 | INFO | Perf report for the last 5.13974 seconds: | 20260507 18:14:38.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:38.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:38.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:38.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:38.310 | INFO | ANALYSIS : 69.3%, 3.0, 108.2ms, 0 | 20260507 18:14:38.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:38.310 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:14:38.310 | INFO | PRE_HQ : 66.5%, 1.7, 59.3ms, 0 | 20260507 18:14:38.310 | INFO | BAZWRITER : 68.9%, 2.9, 102.8ms, 0 | 20260507 18:14:38.310 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:14:38.310 | INFO | Uploaded 36GiB to GPU in 2.48813s (14.4687GiB/s) | 20260507 18:14:38.310 | INFO | Downloaded 38GiB to GPU in 2.72244s (13.9581GiB/s) | 20260507 18:14:43.448 | INFO | Analyzing chunk frames = [1569280,1569792) | 20260507 18:14:43.448 | INFO | MemInfo: { | 20260507 18:14:43.448 | INFO | "gpu" : " 17 GB", | 20260507 18:14:43.448 | INFO | "huge" : "", | 20260507 18:14:43.448 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:43.448 | INFO | "malloc" : " 14 GB", | 20260507 18:14:43.448 | INFO | "pinned" : " 29 GB" | 20260507 18:14:43.448 | INFO | } | 20260507 18:14:43.448 | INFO | Perf report for the last 5.1389 seconds: | 20260507 18:14:43.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:43.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.449 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:14:43.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.449 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:14:43.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:43.449 | INFO | Uploaded 36GiB to GPU in 2.53377s (14.2081GiB/s) | 20260507 18:14:43.449 | INFO | Downloaded 34GiB to GPU in 2.45138s (13.8698GiB/s) | 20260507 18:14:48.587 | INFO | Analyzing chunk frames = [1569792,1570304) | 20260507 18:14:48.587 | INFO | MemInfo: { | 20260507 18:14:48.587 | INFO | "gpu" : " 17 GB", | 20260507 18:14:48.587 | INFO | "huge" : "", | 20260507 18:14:48.587 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:48.588 | INFO | "malloc" : " 14 GB", | 20260507 18:14:48.588 | INFO | "pinned" : " 29 GB" | 20260507 18:14:48.588 | INFO | } | 20260507 18:14:48.588 | INFO | Perf report for the last 5.13926 seconds: | 20260507 18:14:48.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:48.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:14:48.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:14:48.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:48.588 | INFO | Uploaded 36GiB to GPU in 2.552s (14.1066GiB/s) | 20260507 18:14:48.588 | INFO | Downloaded 34GiB to GPU in 2.44317s (13.9163GiB/s) | 20260507 18:14:53.727 | INFO | Analyzing chunk frames = [1570304,1570816) | 20260507 18:14:53.727 | INFO | MemInfo: { | 20260507 18:14:53.727 | INFO | "gpu" : " 17 GB", | 20260507 18:14:53.727 | INFO | "huge" : "", | 20260507 18:14:53.727 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:53.727 | INFO | "malloc" : " 14 GB", | 20260507 18:14:53.727 | INFO | "pinned" : " 29 GB" | 20260507 18:14:53.727 | INFO | } | 20260507 18:14:53.727 | INFO | Perf report for the last 5.13972 seconds: | 20260507 18:14:53.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:53.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.728 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:14:53.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.728 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:14:53.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:53.728 | INFO | Uploaded 36GiB to GPU in 2.48338s (14.4964GiB/s) | 20260507 18:14:53.728 | INFO | Downloaded 34GiB to GPU in 2.34609s (14.4922GiB/s) | 20260507 18:14:58.867 | INFO | Analyzing chunk frames = [1570816,1571328) | 20260507 18:14:58.867 | INFO | MemInfo: { | 20260507 18:14:58.867 | INFO | "gpu" : " 17 GB", | 20260507 18:14:58.867 | INFO | "huge" : "", | 20260507 18:14:58.867 | INFO | "huge_shared" : " 29 GB", | 20260507 18:14:58.867 | INFO | "malloc" : " 14 GB", | 20260507 18:14:58.867 | INFO | "pinned" : " 29 GB" | 20260507 18:14:58.867 | INFO | } | 20260507 18:14:58.867 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:14:58.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:14:58.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:14:58.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:14:58.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:14:58.867 | INFO | Uploaded 36GiB to GPU in 2.35677s (15.2751GiB/s) | 20260507 18:14:58.867 | INFO | Downloaded 34GiB to GPU in 2.47824s (13.7194GiB/s) | 20260507 18:15:04.006 | INFO | Analyzing chunk frames = [1571328,1571840) | 20260507 18:15:04.006 | INFO | MemInfo: { | 20260507 18:15:04.006 | INFO | "gpu" : " 17 GB", | 20260507 18:15:04.006 | INFO | "huge" : "", | 20260507 18:15:04.006 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:04.006 | INFO | "malloc" : " 14 GB", | 20260507 18:15:04.006 | INFO | "pinned" : " 29 GB" | 20260507 18:15:04.006 | INFO | } | 20260507 18:15:04.006 | INFO | Perf report for the last 5.13906 seconds: | 20260507 18:15:04.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:04.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.006 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:15:04.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.006 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:15:04.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:04.007 | INFO | Uploaded 36GiB to GPU in 2.42258s (14.8602GiB/s) | 20260507 18:15:04.007 | INFO | Downloaded 34GiB to GPU in 2.48009s (13.7092GiB/s) | 20260507 18:15:09.145 | INFO | Analyzing chunk frames = [1571840,1572352) | 20260507 18:15:09.145 | INFO | MemInfo: { | 20260507 18:15:09.145 | INFO | "gpu" : " 17 GB", | 20260507 18:15:09.145 | INFO | "huge" : "", | 20260507 18:15:09.145 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:09.145 | INFO | "malloc" : " 14 GB", | 20260507 18:15:09.145 | INFO | "pinned" : " 29 GB" | 20260507 18:15:09.145 | INFO | } | 20260507 18:15:09.145 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:15:09.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:09.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.145 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:15:09.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.146 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:15:09.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:09.146 | INFO | Uploaded 36GiB to GPU in 2.50629s (14.3639GiB/s) | 20260507 18:15:09.146 | INFO | Downloaded 34GiB to GPU in 2.42251s (14.035GiB/s) | 20260507 18:15:14.285 | INFO | Analyzing chunk frames = [1572352,1572864) | 20260507 18:15:14.285 | INFO | MemInfo: { | 20260507 18:15:14.285 | INFO | "gpu" : " 17 GB", | 20260507 18:15:14.285 | INFO | "huge" : "", | 20260507 18:15:14.285 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:14.285 | INFO | "malloc" : " 14 GB", | 20260507 18:15:14.285 | INFO | "pinned" : " 29 GB" | 20260507 18:15:14.285 | INFO | } | 20260507 18:15:14.285 | INFO | Perf report for the last 5.13976 seconds: | 20260507 18:15:14.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:14.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.285 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:15:14.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.285 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260507 18:15:14.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:14.286 | INFO | Uploaded 36GiB to GPU in 2.48728s (14.4736GiB/s) | 20260507 18:15:14.286 | INFO | Downloaded 34GiB to GPU in 2.41109s (14.1015GiB/s) | 20260507 18:15:14.286 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:15:14.286 | INFO | Total mem usage: 16.4 GiB | 20260507 18:15:14.286 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:15:14.286 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.3%) | 20260507 18:15:14.286 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:15:14.286 | INFO | High Water Breakdown: | 20260507 18:15:14.286 | INFO | 59.2% : 8.57 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:15:14.286 | INFO | 40.7% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:15:14.286 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:15:14.286 | INFO | ------------------------------------------ | 20260507 18:15:14.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:15:14.286 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:15:14.286 | INFO | Total mem usage: 26.5 GiB | 20260507 18:15:14.286 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:15:14.286 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:15:14.286 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:15:14.286 | INFO | High Water Breakdown: | 20260507 18:15:14.286 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:15:14.286 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:15:14.286 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:15:14.286 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:15:14.286 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:15:14.286 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:15:14.286 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:15:14.286 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:15:14.286 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:15:14.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:15:14.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:15:14.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:15:14.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:15:14.286 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:15:14.286 | INFO | ------------------------------------------ | 20260507 18:15:14.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:15:14.286 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:15:14.286 | INFO | Total mem usage: 27.8 GiB | 20260507 18:15:14.286 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:15:14.286 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:15:14.286 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:15:14.286 | INFO | High Water Breakdown: | 20260507 18:15:14.286 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:15:14.286 | INFO | ------------------------------------------ | 20260507 18:15:14.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:15:14.288 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:15:14.288 | INFO | Total mem usage: 15.7 GiB | 20260507 18:15:14.288 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:15:14.288 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:15:14.288 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 18:15:14.288 | INFO | High Water Breakdown: | 20260507 18:15:14.288 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:15:14.288 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:15:14.288 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:15:14.288 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:15:14.288 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:15:14.288 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:15:14.288 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:15:14.288 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:15:14.288 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:15:14.288 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:15:14.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:15:14.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:15:14.288 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:15:14.288 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:15:14.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:15:14.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:15:14.288 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:15:14.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:15:14.288 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:15:14.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:15:14.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:15:14.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:15:14.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:15:14.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:15:14.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:15:14.288 | INFO | ------------------------------------------ | 20260507 18:15:14.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:15:14.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:15:14.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:15:14.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:15:14.288 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.42e+01 / 2.46e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 0.29%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.22e-02 / 1.09e-02 | 20260507 18:15:14.288 | INFO | PerfReport: 12.11%, Baseline -- 4.05e+00 / 4.50e+00, 6.21e-01 / 2.54e-01 | 20260507 18:15:14.288 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.48e+00, 2.14e-01 / 1.34e-01 | 20260507 18:15:14.288 | INFO | PerfReport: 15.67%, DME -- 4.19e+01 / 4.49e+01, 4.19e+00 / 2.18e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.90e+00 / 2.23e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.28e-01 / 2.60e-01 | 20260507 18:15:14.288 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.45e+00, 3.18e-01 / 1.80e-01 | 20260507 18:15:14.288 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.91e-01 / 4.19e-01, 6.34e-02 / 3.01e-02 | 20260507 18:15:14.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:15:14.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:15:14.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:15:14.288 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 23.99%, Upload -- 2.57e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 24.05%, Download -- 2.58e+01 / 0.00e+00, 8.62e+00 / 0.00e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 31.18%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 1.77%, LockUpload -- 1.90e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 18:15:14.288 | INFO | PerfReport: 2.38%, LockDownload -- 2.56e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 18:15:14.289 | INFO | PerfReport: 16.61%, LockCompute -- 1.78e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:15:14.289 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 18:15:14.395 | INFO | Starting a round of Baz IO | 20260507 18:15:17.883 | INFO | Populating RTMetrics report | 20260507 18:15:17.884 | INFO | RTMetrics report populated | 20260507 18:15:17.884 | INFO | Opening tmp RTMetrics json file | 20260507 18:15:17.893 | INFO | Serializing RTMetrics to json | 20260507 18:15:17.901 | INFO | Serialization done, swapping tmp file now... | 20260507 18:15:17.902 | INFO | RTMetrics done | 20260507 18:15:17.961 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.565 seconds | 20260507 18:15:17.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.87% / 0.02% / 53.11% | 20260507 18:15:17.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.04412% / 3.97364e-05% / 4.76837e-05% | 20260507 18:15:17.961 | INFO | Baz IO overall: | 20260507 18:15:17.961 | INFO | BytesWritten : 701331849377 | 20260507 18:15:17.961 | INFO | NumEvents : 339978675696 | 20260507 18:15:19.424 | INFO | Analyzing chunk frames = [1572864,1573376) | 20260507 18:15:19.424 | INFO | MemInfo: { | 20260507 18:15:19.424 | INFO | "gpu" : " 17 GB", | 20260507 18:15:19.424 | INFO | "huge" : "", | 20260507 18:15:19.424 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:19.424 | INFO | "malloc" : " 14 GB", | 20260507 18:15:19.424 | INFO | "pinned" : " 29 GB" | 20260507 18:15:19.424 | INFO | } | 20260507 18:15:19.424 | INFO | Perf report for the last 5.1388 seconds: | 20260507 18:15:19.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:19.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:19.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:19.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:19.424 | INFO | ANALYSIS : 69.3%, 2.9, 106.3ms, 0 | 20260507 18:15:19.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:19.424 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 18:15:19.424 | INFO | PRE_HQ : 67.2%, 1.9, 65.8ms, 0 | 20260507 18:15:19.424 | INFO | BAZWRITER : 68.8%, 3.1, 108.6ms, 0 | 20260507 18:15:19.424 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:15:19.424 | INFO | Uploaded 36GiB to GPU in 2.47643s (14.5371GiB/s) | 20260507 18:15:19.424 | INFO | Downloaded 38GiB to GPU in 2.6948s (14.1013GiB/s) | 20260507 18:15:24.563 | INFO | Analyzing chunk frames = [1573376,1573888) | 20260507 18:15:24.563 | INFO | MemInfo: { | 20260507 18:15:24.563 | INFO | "gpu" : " 17 GB", | 20260507 18:15:24.563 | INFO | "huge" : "", | 20260507 18:15:24.563 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:24.563 | INFO | "malloc" : " 14 GB", | 20260507 18:15:24.564 | INFO | "pinned" : " 29 GB" | 20260507 18:15:24.564 | INFO | } | 20260507 18:15:24.564 | INFO | Perf report for the last 5.1396 seconds: | 20260507 18:15:24.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:24.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:15:24.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:15:24.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:24.564 | INFO | Uploaded 36GiB to GPU in 2.52164s (14.2764GiB/s) | 20260507 18:15:24.564 | INFO | Downloaded 34GiB to GPU in 2.45152s (13.8689GiB/s) | 20260507 18:15:29.703 | INFO | Analyzing chunk frames = [1573888,1574400) | 20260507 18:15:29.703 | INFO | MemInfo: { | 20260507 18:15:29.703 | INFO | "gpu" : " 17 GB", | 20260507 18:15:29.703 | INFO | "huge" : "", | 20260507 18:15:29.703 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:29.703 | INFO | "malloc" : " 14 GB", | 20260507 18:15:29.703 | INFO | "pinned" : " 29 GB" | 20260507 18:15:29.703 | INFO | } | 20260507 18:15:29.703 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:15:29.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:29.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.703 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:15:29.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.703 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:15:29.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:29.704 | INFO | Uploaded 36GiB to GPU in 2.52046s (14.2831GiB/s) | 20260507 18:15:29.704 | INFO | Downloaded 34GiB to GPU in 2.45199s (13.8663GiB/s) | 20260507 18:15:34.842 | INFO | Analyzing chunk frames = [1574400,1574912) | 20260507 18:15:34.842 | INFO | MemInfo: { | 20260507 18:15:34.842 | INFO | "gpu" : " 17 GB", | 20260507 18:15:34.842 | INFO | "huge" : "", | 20260507 18:15:34.842 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:34.842 | INFO | "malloc" : " 14 GB", | 20260507 18:15:34.842 | INFO | "pinned" : " 29 GB" | 20260507 18:15:34.842 | INFO | } | 20260507 18:15:34.842 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:15:34.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:34.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.842 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:15:34.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.842 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:15:34.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:34.843 | INFO | Uploaded 36GiB to GPU in 2.46619s (14.5974GiB/s) | 20260507 18:15:34.843 | INFO | Downloaded 34GiB to GPU in 2.36464s (14.3785GiB/s) | 20260507 18:15:39.981 | INFO | Analyzing chunk frames = [1574912,1575424) | 20260507 18:15:39.981 | INFO | MemInfo: { | 20260507 18:15:39.981 | INFO | "gpu" : " 17 GB", | 20260507 18:15:39.981 | INFO | "huge" : "", | 20260507 18:15:39.981 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:39.981 | INFO | "malloc" : " 14 GB", | 20260507 18:15:39.981 | INFO | "pinned" : " 29 GB" | 20260507 18:15:39.981 | INFO | } | 20260507 18:15:39.981 | INFO | Perf report for the last 5.13901 seconds: | 20260507 18:15:39.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:39.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:15:39.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:15:39.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:39.982 | INFO | Uploaded 36GiB to GPU in 2.30209s (15.638GiB/s) | 20260507 18:15:39.982 | INFO | Downloaded 34GiB to GPU in 2.49249s (13.641GiB/s) | 20260507 18:15:45.121 | INFO | Analyzing chunk frames = [1575424,1575936) | 20260507 18:15:45.121 | INFO | MemInfo: { | 20260507 18:15:45.121 | INFO | "gpu" : " 17 GB", | 20260507 18:15:45.121 | INFO | "huge" : "", | 20260507 18:15:45.121 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:45.121 | INFO | "malloc" : " 14 GB", | 20260507 18:15:45.121 | INFO | "pinned" : " 29 GB" | 20260507 18:15:45.121 | INFO | } | 20260507 18:15:45.121 | INFO | Perf report for the last 5.13982 seconds: | 20260507 18:15:45.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:45.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:15:45.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:15:45.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:45.121 | INFO | Uploaded 36GiB to GPU in 2.39985s (15.0009GiB/s) | 20260507 18:15:45.121 | INFO | Downloaded 34GiB to GPU in 2.51299s (13.5297GiB/s) | 20260507 18:15:50.260 | INFO | Analyzing chunk frames = [1575936,1576448) | 20260507 18:15:50.260 | INFO | MemInfo: { | 20260507 18:15:50.260 | INFO | "gpu" : " 17 GB", | 20260507 18:15:50.260 | INFO | "huge" : "", | 20260507 18:15:50.260 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:50.260 | INFO | "malloc" : " 14 GB", | 20260507 18:15:50.260 | INFO | "pinned" : " 29 GB" | 20260507 18:15:50.260 | INFO | } | 20260507 18:15:50.260 | INFO | Perf report for the last 5.13904 seconds: | 20260507 18:15:50.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:50.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.261 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:15:50.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.261 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:15:50.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:50.261 | INFO | Uploaded 36GiB to GPU in 2.47498s (14.5456GiB/s) | 20260507 18:15:50.261 | INFO | Downloaded 34GiB to GPU in 2.44354s (13.9143GiB/s) | 20260507 18:15:55.399 | INFO | Analyzing chunk frames = [1576448,1576960) | 20260507 18:15:55.399 | INFO | MemInfo: { | 20260507 18:15:55.399 | INFO | "gpu" : " 17 GB", | 20260507 18:15:55.399 | INFO | "huge" : "", | 20260507 18:15:55.399 | INFO | "huge_shared" : " 29 GB", | 20260507 18:15:55.399 | INFO | "malloc" : " 14 GB", | 20260507 18:15:55.400 | INFO | "pinned" : " 29 GB" | 20260507 18:15:55.400 | INFO | } | 20260507 18:15:55.400 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:15:55.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:15:55.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.400 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:15:55.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.400 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 18:15:55.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:15:55.401 | INFO | Uploaded 36GiB to GPU in 2.46402s (14.6103GiB/s) | 20260507 18:15:55.401 | INFO | Downloaded 34GiB to GPU in 2.43312s (13.9738GiB/s) | 20260507 18:15:55.512 | INFO | Starting a round of Baz IO | 20260507 18:15:59.001 | INFO | Populating RTMetrics report | 20260507 18:15:59.002 | INFO | RTMetrics report populated | 20260507 18:15:59.002 | INFO | Opening tmp RTMetrics json file | 20260507 18:15:59.012 | INFO | Serializing RTMetrics to json | 20260507 18:15:59.019 | INFO | Serialization done, swapping tmp file now... | 20260507 18:15:59.020 | INFO | RTMetrics done | 20260507 18:15:59.088 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.575 seconds | 20260507 18:15:59.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.86% / 0.02% / 53.13% | 20260507 18:15:59.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.04193% / 1.58946e-05% / 1.98682e-05% | 20260507 18:15:59.088 | INFO | Baz IO overall: | 20260507 18:15:59.088 | INFO | BytesWritten : 703266034711 | 20260507 18:15:59.088 | INFO | NumEvents : 340917427876 | 20260507 18:16:00.539 | INFO | Analyzing chunk frames = [1576960,1577472) | 20260507 18:16:00.539 | INFO | MemInfo: { | 20260507 18:16:00.539 | INFO | "gpu" : " 17 GB", | 20260507 18:16:00.539 | INFO | "huge" : "", | 20260507 18:16:00.539 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:00.539 | INFO | "malloc" : " 14 GB", | 20260507 18:16:00.539 | INFO | "pinned" : " 29 GB" | 20260507 18:16:00.539 | INFO | } | 20260507 18:16:00.539 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:16:00.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:00.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:00.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:00.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:00.539 | INFO | ANALYSIS : 69.6%, 2.9, 107.4ms, 0 | 20260507 18:16:00.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:00.539 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 18:16:00.539 | INFO | PRE_HQ : 67.2%, 1.9, 64.7ms, 0 | 20260507 18:16:00.539 | INFO | BAZWRITER : 68.8%, 3.1, 108.0ms, 0 | 20260507 18:16:00.539 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:16:00.539 | INFO | Uploaded 36GiB to GPU in 2.58685s (13.9165GiB/s) | 20260507 18:16:00.539 | INFO | Downloaded 38GiB to GPU in 2.70325s (14.0572GiB/s) | 20260507 18:16:05.678 | INFO | Analyzing chunk frames = [1577472,1577984) | 20260507 18:16:05.678 | INFO | MemInfo: { | 20260507 18:16:05.678 | INFO | "gpu" : " 17 GB", | 20260507 18:16:05.678 | INFO | "huge" : "", | 20260507 18:16:05.678 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:05.678 | INFO | "malloc" : " 14 GB", | 20260507 18:16:05.678 | INFO | "pinned" : " 29 GB" | 20260507 18:16:05.678 | INFO | } | 20260507 18:16:05.678 | INFO | Perf report for the last 5.13911 seconds: | 20260507 18:16:05.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:05.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:16:05.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:16:05.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:05.678 | INFO | Uploaded 36GiB to GPU in 2.56079s (14.0582GiB/s) | 20260507 18:16:05.678 | INFO | Downloaded 34GiB to GPU in 2.44189s (13.9236GiB/s) | 20260507 18:16:10.817 | INFO | Analyzing chunk frames = [1577984,1578496) | 20260507 18:16:10.817 | INFO | MemInfo: { | 20260507 18:16:10.817 | INFO | "gpu" : " 17 GB", | 20260507 18:16:10.817 | INFO | "huge" : "", | 20260507 18:16:10.817 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:10.817 | INFO | "malloc" : " 14 GB", | 20260507 18:16:10.817 | INFO | "pinned" : " 29 GB" | 20260507 18:16:10.817 | INFO | } | 20260507 18:16:10.817 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:16:10.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:10.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260507 18:16:10.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:16:10.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:10.818 | INFO | Uploaded 36GiB to GPU in 2.49126s (14.4505GiB/s) | 20260507 18:16:10.818 | INFO | Downloaded 34GiB to GPU in 2.33557s (14.5575GiB/s) | 20260507 18:16:15.957 | INFO | Analyzing chunk frames = [1578496,1579008) | 20260507 18:16:15.957 | INFO | MemInfo: { | 20260507 18:16:15.957 | INFO | "gpu" : " 17 GB", | 20260507 18:16:15.957 | INFO | "huge" : "", | 20260507 18:16:15.957 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:15.957 | INFO | "malloc" : " 14 GB", | 20260507 18:16:15.957 | INFO | "pinned" : " 29 GB" | 20260507 18:16:15.957 | INFO | } | 20260507 18:16:15.957 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:16:15.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:15.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:16:15.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:16:15.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:15.957 | INFO | Uploaded 36GiB to GPU in 2.50688s (14.3605GiB/s) | 20260507 18:16:15.957 | INFO | Downloaded 34GiB to GPU in 2.346s (14.4927GiB/s) | 20260507 18:16:21.097 | INFO | Analyzing chunk frames = [1579008,1579520) | 20260507 18:16:21.097 | INFO | MemInfo: { | 20260507 18:16:21.097 | INFO | "gpu" : " 17 GB", | 20260507 18:16:21.097 | INFO | "huge" : "", | 20260507 18:16:21.097 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:21.097 | INFO | "malloc" : " 14 GB", | 20260507 18:16:21.097 | INFO | "pinned" : " 29 GB" | 20260507 18:16:21.097 | INFO | } | 20260507 18:16:21.097 | INFO | Perf report for the last 5.14013 seconds: | 20260507 18:16:21.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:21.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.097 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:16:21.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.097 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:16:21.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:21.098 | INFO | Uploaded 36GiB to GPU in 2.34315s (15.364GiB/s) | 20260507 18:16:21.098 | INFO | Downloaded 34GiB to GPU in 2.46678s (13.7831GiB/s) | 20260507 18:16:26.235 | INFO | Analyzing chunk frames = [1579520,1580032) | 20260507 18:16:26.235 | INFO | MemInfo: { | 20260507 18:16:26.235 | INFO | "gpu" : " 17 GB", | 20260507 18:16:26.235 | INFO | "huge" : "", | 20260507 18:16:26.235 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:26.235 | INFO | "malloc" : " 14 GB", | 20260507 18:16:26.235 | INFO | "pinned" : " 29 GB" | 20260507 18:16:26.235 | INFO | } | 20260507 18:16:26.235 | INFO | Perf report for the last 5.1383 seconds: | 20260507 18:16:26.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:26.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.235 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:16:26.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.235 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:16:26.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:26.236 | INFO | Uploaded 36GiB to GPU in 2.43697s (14.7725GiB/s) | 20260507 18:16:26.236 | INFO | Downloaded 34GiB to GPU in 2.49014s (13.6539GiB/s) | 20260507 18:16:31.374 | INFO | Analyzing chunk frames = [1580032,1580544) | 20260507 18:16:31.375 | INFO | MemInfo: { | 20260507 18:16:31.375 | INFO | "gpu" : " 17 GB", | 20260507 18:16:31.375 | INFO | "huge" : "", | 20260507 18:16:31.375 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:31.375 | INFO | "malloc" : " 14 GB", | 20260507 18:16:31.375 | INFO | "pinned" : " 29 GB" | 20260507 18:16:31.375 | INFO | } | 20260507 18:16:31.375 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:16:31.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:31.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.375 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:16:31.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.375 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 18:16:31.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:31.376 | INFO | Uploaded 36GiB to GPU in 2.51894s (14.2917GiB/s) | 20260507 18:16:31.376 | INFO | Downloaded 34GiB to GPU in 2.42265s (14.0342GiB/s) | 20260507 18:16:36.514 | INFO | Analyzing chunk frames = [1580544,1581056) | 20260507 18:16:36.514 | INFO | MemInfo: { | 20260507 18:16:36.514 | INFO | "gpu" : " 17 GB", | 20260507 18:16:36.514 | INFO | "huge" : "", | 20260507 18:16:36.514 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:36.514 | INFO | "malloc" : " 14 GB", | 20260507 18:16:36.514 | INFO | "pinned" : " 29 GB" | 20260507 18:16:36.514 | INFO | } | 20260507 18:16:36.515 | INFO | Perf report for the last 5.13948 seconds: | 20260507 18:16:36.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:36.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:16:36.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 18:16:36.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:36.515 | INFO | Uploaded 36GiB to GPU in 2.52907s (14.2345GiB/s) | 20260507 18:16:36.515 | INFO | Downloaded 34GiB to GPU in 2.41625s (14.0714GiB/s) | 20260507 18:16:36.515 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:16:36.515 | INFO | Total mem usage: 16.4 GiB | 20260507 18:16:36.515 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:16:36.515 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 18:16:36.515 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:16:36.515 | INFO | High Water Breakdown: | 20260507 18:16:36.515 | INFO | 59.3% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:16:36.515 | INFO | 40.5% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:16:36.515 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:16:36.515 | INFO | ------------------------------------------ | 20260507 18:16:36.515 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:16:36.515 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:16:36.515 | INFO | Total mem usage: 26.5 GiB | 20260507 18:16:36.515 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:16:36.515 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:16:36.515 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 18:16:36.515 | INFO | High Water Breakdown: | 20260507 18:16:36.515 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:16:36.515 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:16:36.515 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:16:36.515 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:16:36.515 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:16:36.515 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:16:36.515 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:16:36.515 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:16:36.515 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:16:36.515 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:16:36.515 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:16:36.515 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:16:36.515 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:16:36.516 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:16:36.516 | INFO | ------------------------------------------ | 20260507 18:16:36.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:16:36.516 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:16:36.516 | INFO | Total mem usage: 27.8 GiB | 20260507 18:16:36.516 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:16:36.516 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:16:36.516 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:16:36.516 | INFO | High Water Breakdown: | 20260507 18:16:36.516 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:16:36.516 | INFO | ------------------------------------------ | 20260507 18:16:36.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:16:36.516 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:16:36.516 | INFO | Total mem usage: 15.7 GiB | 20260507 18:16:36.516 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:16:36.516 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:16:36.516 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:16:36.516 | INFO | High Water Breakdown: | 20260507 18:16:36.516 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:16:36.516 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:16:36.516 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:16:36.516 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:16:36.516 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:16:36.516 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:16:36.516 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:16:36.516 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:16:36.516 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:16:36.516 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:16:36.516 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:16:36.516 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:16:36.516 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:16:36.516 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:16:36.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:16:36.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:16:36.516 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:16:36.516 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:16:36.516 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:16:36.516 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:16:36.516 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:16:36.516 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:16:36.516 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:16:36.516 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:16:36.516 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:16:36.516 | INFO | ------------------------------------------ | 20260507 18:16:36.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:16:36.516 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:16:36.516 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:16:36.516 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:16:36.516 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.60e+01, 1.43e+01 / 2.21e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.03e-01, 1.10e-02 / 5.76e-03 | 20260507 18:16:36.516 | INFO | PerfReport: 12.10%, Baseline -- 4.04e+00 / 4.49e+00, 6.18e-01 / 2.44e-01 | 20260507 18:16:36.516 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 1.22e-01 | 20260507 18:16:36.516 | INFO | PerfReport: 15.69%, DME -- 4.20e+01 / 4.48e+01, 3.99e+00 / 2.18e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.89e+00 / 2.14e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.36e-01 / 2.54e-01 | 20260507 18:16:36.516 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.43e+00, 3.17e-01 / 1.70e-01 | 20260507 18:16:36.516 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.93e-01 / 4.21e-01, 6.38e-02 / 3.71e-02 | 20260507 18:16:36.516 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:16:36.516 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:16:36.516 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:16:36.516 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 24.07%, Upload -- 2.58e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 23.99%, Download -- 2.57e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 31.22%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 1.86%, LockUpload -- 2.00e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 16.60%, LockCompute -- 1.78e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 18:16:36.516 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260507 18:16:36.628 | INFO | Starting a round of Baz IO | 20260507 18:16:40.143 | INFO | Populating RTMetrics report | 20260507 18:16:40.143 | INFO | RTMetrics report populated | 20260507 18:16:40.143 | INFO | Opening tmp RTMetrics json file | 20260507 18:16:40.144 | INFO | Serializing RTMetrics to json | 20260507 18:16:40.146 | INFO | Serialization done, swapping tmp file now... | 20260507 18:16:40.146 | INFO | RTMetrics done | 20260507 18:16:40.238 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.609 seconds | 20260507 18:16:40.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.84% / 0.02% / 53.14% | 20260507 18:16:40.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.03269% / 2.78155e-05% / 2.38419e-05% | 20260507 18:16:40.238 | INFO | Baz IO overall: | 20260507 18:16:40.238 | INFO | BytesWritten : 705199810034 | 20260507 18:16:40.238 | INFO | NumEvents : 341855957414 | 20260507 18:16:41.653 | INFO | Analyzing chunk frames = [1581056,1581568) | 20260507 18:16:41.653 | INFO | MemInfo: { | 20260507 18:16:41.653 | INFO | "gpu" : " 17 GB", | 20260507 18:16:41.653 | INFO | "huge" : "", | 20260507 18:16:41.653 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:41.653 | INFO | "malloc" : " 14 GB", | 20260507 18:16:41.653 | INFO | "pinned" : " 29 GB" | 20260507 18:16:41.653 | INFO | } | 20260507 18:16:41.653 | INFO | Perf report for the last 5.13872 seconds: | 20260507 18:16:41.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:41.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:41.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:41.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:41.654 | INFO | ANALYSIS : 69.6%, 2.9, 107.1ms, 0 | 20260507 18:16:41.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:41.654 | INFO | RT_METRICS : 49.7%, 1.0, 26.0ms, 0 | 20260507 18:16:41.654 | INFO | PRE_HQ : 68.0%, 2.0, 70.1ms, 0 | 20260507 18:16:41.654 | INFO | BAZWRITER : 69.8%, 3.2, 113.8ms, 0 | 20260507 18:16:41.654 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 18:16:41.654 | INFO | Uploaded 36GiB to GPU in 2.58845s (13.9079GiB/s) | 20260507 18:16:41.654 | INFO | Downloaded 38GiB to GPU in 2.69599s (14.095GiB/s) | 20260507 18:16:46.792 | INFO | Analyzing chunk frames = [1581568,1582080) | 20260507 18:16:46.792 | INFO | MemInfo: { | 20260507 18:16:46.792 | INFO | "gpu" : " 17 GB", | 20260507 18:16:46.792 | INFO | "huge" : "", | 20260507 18:16:46.793 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:46.793 | INFO | "malloc" : " 14 GB", | 20260507 18:16:46.793 | INFO | "pinned" : " 29 GB" | 20260507 18:16:46.793 | INFO | } | 20260507 18:16:46.793 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:16:46.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:46.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:16:46.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:16:46.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:46.793 | INFO | Uploaded 36GiB to GPU in 2.49678s (14.4186GiB/s) | 20260507 18:16:46.793 | INFO | Downloaded 34GiB to GPU in 2.4514s (13.8696GiB/s) | 20260507 18:16:51.932 | INFO | Analyzing chunk frames = [1582080,1582592) | 20260507 18:16:51.932 | INFO | MemInfo: { | 20260507 18:16:51.932 | INFO | "gpu" : " 17 GB", | 20260507 18:16:51.932 | INFO | "huge" : "", | 20260507 18:16:51.932 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:51.932 | INFO | "malloc" : " 14 GB", | 20260507 18:16:51.932 | INFO | "pinned" : " 29 GB" | 20260507 18:16:51.932 | INFO | } | 20260507 18:16:51.932 | INFO | Perf report for the last 5.13971 seconds: | 20260507 18:16:51.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:51.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.932 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:16:51.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.932 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:16:51.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:51.933 | INFO | Uploaded 36GiB to GPU in 2.55853s (14.0706GiB/s) | 20260507 18:16:51.933 | INFO | Downloaded 34GiB to GPU in 2.4242s (14.0252GiB/s) | 20260507 18:16:57.071 | INFO | Analyzing chunk frames = [1582592,1583104) | 20260507 18:16:57.071 | INFO | MemInfo: { | 20260507 18:16:57.071 | INFO | "gpu" : " 17 GB", | 20260507 18:16:57.071 | INFO | "huge" : "", | 20260507 18:16:57.071 | INFO | "huge_shared" : " 29 GB", | 20260507 18:16:57.071 | INFO | "malloc" : " 14 GB", | 20260507 18:16:57.071 | INFO | "pinned" : " 29 GB" | 20260507 18:16:57.071 | INFO | } | 20260507 18:16:57.072 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:16:57.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:16:57.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:16:57.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:16:57.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:16:57.072 | INFO | Uploaded 36GiB to GPU in 2.47224s (14.5617GiB/s) | 20260507 18:16:57.072 | INFO | Downloaded 34GiB to GPU in 2.35432s (14.4416GiB/s) | 20260507 18:17:02.211 | INFO | Analyzing chunk frames = [1583104,1583616) | 20260507 18:17:02.211 | INFO | MemInfo: { | 20260507 18:17:02.211 | INFO | "gpu" : " 17 GB", | 20260507 18:17:02.211 | INFO | "huge" : "", | 20260507 18:17:02.211 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:02.211 | INFO | "malloc" : " 14 GB", | 20260507 18:17:02.211 | INFO | "pinned" : " 29 GB" | 20260507 18:17:02.211 | INFO | } | 20260507 18:17:02.211 | INFO | Perf report for the last 5.13986 seconds: | 20260507 18:17:02.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:02.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.211 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 18:17:02.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.212 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:17:02.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:02.212 | INFO | Uploaded 36GiB to GPU in 2.31686s (15.5383GiB/s) | 20260507 18:17:02.212 | INFO | Downloaded 34GiB to GPU in 2.4877s (13.6672GiB/s) | 20260507 18:17:07.350 | INFO | Analyzing chunk frames = [1583616,1584128) | 20260507 18:17:07.350 | INFO | MemInfo: { | 20260507 18:17:07.350 | INFO | "gpu" : " 17 GB", | 20260507 18:17:07.350 | INFO | "huge" : "", | 20260507 18:17:07.350 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:07.350 | INFO | "malloc" : " 14 GB", | 20260507 18:17:07.350 | INFO | "pinned" : " 29 GB" | 20260507 18:17:07.350 | INFO | } | 20260507 18:17:07.350 | INFO | Perf report for the last 5.13868 seconds: | 20260507 18:17:07.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:07.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.350 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 18:17:07.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.350 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:17:07.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:07.351 | INFO | Uploaded 36GiB to GPU in 2.36835s (15.2005GiB/s) | 20260507 18:17:07.351 | INFO | Downloaded 34GiB to GPU in 2.53035s (13.4369GiB/s) | 20260507 18:17:12.489 | INFO | Analyzing chunk frames = [1584128,1584640) | 20260507 18:17:12.489 | INFO | MemInfo: { | 20260507 18:17:12.489 | INFO | "gpu" : " 17 GB", | 20260507 18:17:12.489 | INFO | "huge" : "", | 20260507 18:17:12.489 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:12.490 | INFO | "malloc" : " 14 GB", | 20260507 18:17:12.490 | INFO | "pinned" : " 29 GB" | 20260507 18:17:12.490 | INFO | } | 20260507 18:17:12.490 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:17:12.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:12.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 18:17:12.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:17:12.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:12.490 | INFO | Uploaded 36GiB to GPU in 2.42747s (14.8303GiB/s) | 20260507 18:17:12.490 | INFO | Downloaded 34GiB to GPU in 2.44925s (13.8818GiB/s) | 20260507 18:17:17.629 | INFO | Analyzing chunk frames = [1584640,1585152) | 20260507 18:17:17.629 | INFO | MemInfo: { | 20260507 18:17:17.629 | INFO | "gpu" : " 17 GB", | 20260507 18:17:17.629 | INFO | "huge" : "", | 20260507 18:17:17.629 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:17.629 | INFO | "malloc" : " 14 GB", | 20260507 18:17:17.629 | INFO | "pinned" : " 29 GB" | 20260507 18:17:17.629 | INFO | } | 20260507 18:17:17.629 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:17:17.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:17.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.629 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 18:17:17.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.629 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260507 18:17:17.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:17.630 | INFO | Uploaded 36GiB to GPU in 2.49355s (14.4372GiB/s) | 20260507 18:17:17.630 | INFO | Downloaded 34GiB to GPU in 2.52006s (13.4918GiB/s) | 20260507 18:17:17.726 | INFO | Starting a round of Baz IO | 20260507 18:17:21.236 | INFO | Populating RTMetrics report | 20260507 18:17:21.236 | INFO | RTMetrics report populated | 20260507 18:17:21.236 | INFO | Opening tmp RTMetrics json file | 20260507 18:17:21.237 | INFO | Serializing RTMetrics to json | 20260507 18:17:21.244 | INFO | Serialization done, swapping tmp file now... | 20260507 18:17:21.257 | INFO | RTMetrics done | 20260507 18:17:21.317 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.591 seconds | 20260507 18:17:21.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.82% / 0.02% / 53.16% | 20260507 18:17:21.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.03705% / 2.38419e-05% / 1.58946e-05% | 20260507 18:17:21.317 | INFO | Baz IO overall: | 20260507 18:17:21.317 | INFO | BytesWritten : 707135436624 | 20260507 18:17:21.317 | INFO | NumEvents : 342796099503 | 20260507 18:17:22.768 | INFO | Analyzing chunk frames = [1585152,1585664) | 20260507 18:17:22.768 | INFO | MemInfo: { | 20260507 18:17:22.768 | INFO | "gpu" : " 17 GB", | 20260507 18:17:22.768 | INFO | "huge" : "", | 20260507 18:17:22.768 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:22.768 | INFO | "malloc" : " 14 GB", | 20260507 18:17:22.769 | INFO | "pinned" : " 29 GB" | 20260507 18:17:22.769 | INFO | } | 20260507 18:17:22.769 | INFO | Perf report for the last 5.13948 seconds: | 20260507 18:17:22.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:22.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:22.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:22.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:22.769 | INFO | ANALYSIS : 69.4%, 3.0, 108.5ms, 0 | 20260507 18:17:22.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:22.769 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:17:22.769 | INFO | PRE_HQ : 66.6%, 1.7, 59.3ms, 0 | 20260507 18:17:22.769 | INFO | BAZWRITER : 69.0%, 2.9, 103.9ms, 0 | 20260507 18:17:22.769 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:17:22.769 | INFO | Uploaded 36GiB to GPU in 2.51008s (14.3422GiB/s) | 20260507 18:17:22.769 | INFO | Downloaded 38GiB to GPU in 2.72842s (13.9275GiB/s) | 20260507 18:17:27.907 | INFO | Analyzing chunk frames = [1585664,1586176) | 20260507 18:17:27.907 | INFO | MemInfo: { | 20260507 18:17:27.907 | INFO | "gpu" : " 17 GB", | 20260507 18:17:27.907 | INFO | "huge" : "", | 20260507 18:17:27.907 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:27.907 | INFO | "malloc" : " 14 GB", | 20260507 18:17:27.907 | INFO | "pinned" : " 29 GB" | 20260507 18:17:27.907 | INFO | } | 20260507 18:17:27.908 | INFO | Perf report for the last 5.13892 seconds: | 20260507 18:17:27.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:27.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:17:27.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:17:27.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:27.908 | INFO | Uploaded 36GiB to GPU in 2.52043s (14.2833GiB/s) | 20260507 18:17:27.908 | INFO | Downloaded 34GiB to GPU in 2.45137s (13.8698GiB/s) | 20260507 18:17:33.047 | INFO | Analyzing chunk frames = [1586176,1586688) | 20260507 18:17:33.047 | INFO | MemInfo: { | 20260507 18:17:33.047 | INFO | "gpu" : " 17 GB", | 20260507 18:17:33.047 | INFO | "huge" : "", | 20260507 18:17:33.047 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:33.047 | INFO | "malloc" : " 14 GB", | 20260507 18:17:33.047 | INFO | "pinned" : " 29 GB" | 20260507 18:17:33.047 | INFO | } | 20260507 18:17:33.047 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:17:33.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:33.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.047 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 18:17:33.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.047 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:17:33.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:33.048 | INFO | Uploaded 36GiB to GPU in 2.52926s (14.2334GiB/s) | 20260507 18:17:33.048 | INFO | Downloaded 34GiB to GPU in 2.42577s (14.0162GiB/s) | 20260507 18:17:38.186 | INFO | Analyzing chunk frames = [1586688,1587200) | 20260507 18:17:38.187 | INFO | MemInfo: { | 20260507 18:17:38.187 | INFO | "gpu" : " 17 GB", | 20260507 18:17:38.187 | INFO | "huge" : "", | 20260507 18:17:38.187 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:38.187 | INFO | "malloc" : " 14 GB", | 20260507 18:17:38.187 | INFO | "pinned" : " 29 GB" | 20260507 18:17:38.187 | INFO | } | 20260507 18:17:38.187 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:17:38.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:38.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:17:38.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:17:38.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:38.187 | INFO | Uploaded 36GiB to GPU in 2.4916s (14.4485GiB/s) | 20260507 18:17:38.187 | INFO | Downloaded 34GiB to GPU in 2.34718s (14.4855GiB/s) | 20260507 18:17:43.326 | INFO | Analyzing chunk frames = [1587200,1587712) | 20260507 18:17:43.326 | INFO | MemInfo: { | 20260507 18:17:43.326 | INFO | "gpu" : " 17 GB", | 20260507 18:17:43.326 | INFO | "huge" : "", | 20260507 18:17:43.326 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:43.326 | INFO | "malloc" : " 14 GB", | 20260507 18:17:43.326 | INFO | "pinned" : " 29 GB" | 20260507 18:17:43.326 | INFO | } | 20260507 18:17:43.326 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:17:43.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:43.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:17:43.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:17:43.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:43.326 | INFO | Uploaded 36GiB to GPU in 2.33027s (15.4488GiB/s) | 20260507 18:17:43.326 | INFO | Downloaded 34GiB to GPU in 2.48852s (13.6627GiB/s) | 20260507 18:17:48.465 | INFO | Analyzing chunk frames = [1587712,1588224) | 20260507 18:17:48.465 | INFO | MemInfo: { | 20260507 18:17:48.465 | INFO | "gpu" : " 17 GB", | 20260507 18:17:48.465 | INFO | "huge" : "", | 20260507 18:17:48.465 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:48.465 | INFO | "malloc" : " 14 GB", | 20260507 18:17:48.465 | INFO | "pinned" : " 29 GB" | 20260507 18:17:48.465 | INFO | } | 20260507 18:17:48.465 | INFO | Perf report for the last 5.13962 seconds: | 20260507 18:17:48.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:48.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.465 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:17:48.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.466 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:17:48.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:48.466 | INFO | Uploaded 36GiB to GPU in 2.45458s (14.6665GiB/s) | 20260507 18:17:48.466 | INFO | Downloaded 34GiB to GPU in 2.48886s (13.6609GiB/s) | 20260507 18:17:53.604 | INFO | Analyzing chunk frames = [1588224,1588736) | 20260507 18:17:53.604 | INFO | MemInfo: { | 20260507 18:17:53.604 | INFO | "gpu" : " 17 GB", | 20260507 18:17:53.604 | INFO | "huge" : "", | 20260507 18:17:53.604 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:53.604 | INFO | "malloc" : " 14 GB", | 20260507 18:17:53.604 | INFO | "pinned" : " 29 GB" | 20260507 18:17:53.604 | INFO | } | 20260507 18:17:53.605 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:17:53.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:53.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:17:53.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 18:17:53.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:53.605 | INFO | Uploaded 36GiB to GPU in 2.53023s (14.228GiB/s) | 20260507 18:17:53.605 | INFO | Downloaded 34GiB to GPU in 2.42786s (14.0041GiB/s) | 20260507 18:17:58.744 | INFO | Analyzing chunk frames = [1588736,1589248) | 20260507 18:17:58.744 | INFO | MemInfo: { | 20260507 18:17:58.744 | INFO | "gpu" : " 17 GB", | 20260507 18:17:58.744 | INFO | "huge" : "", | 20260507 18:17:58.744 | INFO | "huge_shared" : " 29 GB", | 20260507 18:17:58.744 | INFO | "malloc" : " 14 GB", | 20260507 18:17:58.744 | INFO | "pinned" : " 29 GB" | 20260507 18:17:58.744 | INFO | } | 20260507 18:17:58.744 | INFO | Perf report for the last 5.13986 seconds: | 20260507 18:17:58.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:17:58.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.745 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 18:17:58.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.745 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260507 18:17:58.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:17:58.745 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:17:58.745 | INFO | Total mem usage: 16.4 GiB | 20260507 18:17:58.745 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:17:58.745 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.8%) | 20260507 18:17:58.745 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:17:58.745 | INFO | High Water Breakdown: | 20260507 18:17:58.745 | INFO | 59.4% : 8.64 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:17:58.745 | INFO | 40.5% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:17:58.745 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:17:58.745 | INFO | ------------------------------------------ | 20260507 18:17:58.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:17:58.745 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:17:58.745 | INFO | Total mem usage: 26.5 GiB | 20260507 18:17:58.745 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:17:58.745 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:17:58.745 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:17:58.745 | INFO | High Water Breakdown: | 20260507 18:17:58.745 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:17:58.745 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:17:58.745 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:17:58.745 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:17:58.745 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:17:58.745 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:17:58.745 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:17:58.745 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:17:58.745 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:17:58.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:17:58.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:17:58.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:17:58.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:17:58.745 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:17:58.745 | INFO | ------------------------------------------ | 20260507 18:17:58.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:17:58.745 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:17:58.745 | INFO | Total mem usage: 27.8 GiB | 20260507 18:17:58.745 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:17:58.745 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:17:58.745 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:17:58.745 | INFO | High Water Breakdown: | 20260507 18:17:58.745 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:17:58.745 | INFO | ------------------------------------------ | 20260507 18:17:58.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:17:58.745 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:17:58.745 | INFO | Total mem usage: 15.7 GiB | 20260507 18:17:58.745 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:17:58.745 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:17:58.745 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:17:58.745 | INFO | High Water Breakdown: | 20260507 18:17:58.745 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:17:58.745 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:17:58.745 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:17:58.745 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:17:58.745 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:17:58.745 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:17:58.745 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:17:58.745 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:17:58.745 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:17:58.745 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:17:58.745 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:17:58.745 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:17:58.745 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:17:58.745 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:17:58.745 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:17:58.745 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:17:58.745 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:17:58.745 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:17:58.745 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:17:58.745 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:17:58.745 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:17:58.745 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:17:58.745 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:17:58.745 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:17:58.745 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:17:58.745 | INFO | ------------------------------------------ | 20260507 18:17:58.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:17:58.745 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:17:58.745 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:17:58.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:17:58.745 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.42e+01 / 2.30e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.03e-01, 1.20e-02 / 4.95e-03 | 20260507 18:17:58.745 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.52e+00, 6.15e-01 / 1.90e-01 | 20260507 18:17:58.745 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 1.18e-01 | 20260507 18:17:58.745 | INFO | PerfReport: 15.72%, DME -- 4.20e+01 / 4.50e+01, 3.93e+00 / 2.23e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.87e+00 / 2.03e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.37e-01 / 2.49e-01 | 20260507 18:17:58.745 | INFO | PerfReport: 3.71%, Metrics -- 1.24e+00 / 1.40e+00, 3.11e-01 / 1.58e-01 | 20260507 18:17:58.745 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.19e-01, 6.36e-02 / 3.42e-02 | 20260507 18:17:58.745 | INFO | Uploaded 36GiB to GPU in 2.48338s (14.4964GiB/s) | 20260507 18:17:58.745 | INFO | Downloaded 34GiB to GPU in 2.41084s (14.103GiB/s) | 20260507 18:17:58.745 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:17:58.745 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:17:58.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:17:58.745 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 24.01%, Upload -- 2.58e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 24.08%, Download -- 2.58e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 31.16%, Compute -- 3.34e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 1.79%, LockUpload -- 1.92e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 2.39%, LockDownload -- 2.57e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 16.54%, LockCompute -- 1.77e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 18:17:58.745 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260507 18:17:58.843 | INFO | Starting a round of Baz IO | 20260507 18:18:02.353 | INFO | Populating RTMetrics report | 20260507 18:18:02.353 | INFO | RTMetrics report populated | 20260507 18:18:02.353 | INFO | Opening tmp RTMetrics json file | 20260507 18:18:02.365 | INFO | Serializing RTMetrics to json | 20260507 18:18:02.372 | INFO | Serialization done, swapping tmp file now... | 20260507 18:18:02.373 | INFO | RTMetrics done | 20260507 18:18:02.433 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.59 seconds | 20260507 18:18:02.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.81% / 0.02% / 53.18% | 20260507 18:18:02.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.02269% / 3.57628e-05% / 3.57628e-05% | 20260507 18:18:02.433 | INFO | Baz IO overall: | 20260507 18:18:02.433 | INFO | BytesWritten : 709069367962 | 20260507 18:18:02.433 | INFO | NumEvents : 343735022842 | 20260507 18:18:03.883 | INFO | Analyzing chunk frames = [1589248,1589760) | 20260507 18:18:03.883 | INFO | MemInfo: { | 20260507 18:18:03.883 | INFO | "gpu" : " 17 GB", | 20260507 18:18:03.883 | INFO | "huge" : "", | 20260507 18:18:03.883 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:03.883 | INFO | "malloc" : " 14 GB", | 20260507 18:18:03.883 | INFO | "pinned" : " 29 GB" | 20260507 18:18:03.883 | INFO | } | 20260507 18:18:03.883 | INFO | Perf report for the last 5.13887 seconds: | 20260507 18:18:03.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:03.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:03.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:03.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:03.884 | INFO | ANALYSIS : 69.5%, 3.0, 108.5ms, 0 | 20260507 18:18:03.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:03.884 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 18:18:03.884 | INFO | PRE_HQ : 66.6%, 1.7, 59.6ms, 0 | 20260507 18:18:03.884 | INFO | BAZWRITER : 69.0%, 2.9, 102.4ms, 0 | 20260507 18:18:03.884 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:18:03.884 | INFO | Uploaded 36GiB to GPU in 2.53841s (14.1821GiB/s) | 20260507 18:18:03.884 | INFO | Downloaded 38GiB to GPU in 2.72569s (13.9414GiB/s) | 20260507 18:18:09.023 | INFO | Analyzing chunk frames = [1589760,1590272) | 20260507 18:18:09.023 | INFO | MemInfo: { | 20260507 18:18:09.023 | INFO | "gpu" : " 17 GB", | 20260507 18:18:09.023 | INFO | "huge" : "", | 20260507 18:18:09.023 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:09.023 | INFO | "malloc" : " 14 GB", | 20260507 18:18:09.023 | INFO | "pinned" : " 29 GB" | 20260507 18:18:09.023 | INFO | } | 20260507 18:18:09.023 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:18:09.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:09.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:18:09.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:18:09.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:09.023 | INFO | Uploaded 36GiB to GPU in 2.51851s (14.2942GiB/s) | 20260507 18:18:09.023 | INFO | Downloaded 34GiB to GPU in 2.46176s (13.8112GiB/s) | 20260507 18:18:14.162 | INFO | Analyzing chunk frames = [1590272,1590784) | 20260507 18:18:14.162 | INFO | MemInfo: { | 20260507 18:18:14.162 | INFO | "gpu" : " 17 GB", | 20260507 18:18:14.162 | INFO | "huge" : "", | 20260507 18:18:14.162 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:14.162 | INFO | "malloc" : " 14 GB", | 20260507 18:18:14.162 | INFO | "pinned" : " 29 GB" | 20260507 18:18:14.162 | INFO | } | 20260507 18:18:14.162 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:18:14.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:14.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:18:14.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:18:14.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:14.162 | INFO | Uploaded 36GiB to GPU in 2.53821s (14.1832GiB/s) | 20260507 18:18:14.162 | INFO | Downloaded 34GiB to GPU in 2.44788s (13.8896GiB/s) | 20260507 18:18:19.301 | INFO | Analyzing chunk frames = [1590784,1591296) | 20260507 18:18:19.301 | INFO | MemInfo: { | 20260507 18:18:19.301 | INFO | "gpu" : " 17 GB", | 20260507 18:18:19.301 | INFO | "huge" : "", | 20260507 18:18:19.301 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:19.301 | INFO | "malloc" : " 14 GB", | 20260507 18:18:19.301 | INFO | "pinned" : " 29 GB" | 20260507 18:18:19.301 | INFO | } | 20260507 18:18:19.301 | INFO | Perf report for the last 5.1392 seconds: | 20260507 18:18:19.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:19.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.301 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:18:19.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.301 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:18:19.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:19.302 | INFO | Uploaded 36GiB to GPU in 2.48142s (14.5078GiB/s) | 20260507 18:18:19.302 | INFO | Downloaded 34GiB to GPU in 2.35358s (14.4461GiB/s) | 20260507 18:18:24.441 | INFO | Analyzing chunk frames = [1591296,1591808) | 20260507 18:18:24.441 | INFO | MemInfo: { | 20260507 18:18:24.441 | INFO | "gpu" : " 17 GB", | 20260507 18:18:24.441 | INFO | "huge" : "", | 20260507 18:18:24.441 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:24.441 | INFO | "malloc" : " 14 GB", | 20260507 18:18:24.441 | INFO | "pinned" : " 29 GB" | 20260507 18:18:24.441 | INFO | } | 20260507 18:18:24.441 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:18:24.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:24.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:18:24.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:18:24.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:24.441 | INFO | Uploaded 36GiB to GPU in 2.30835s (15.5956GiB/s) | 20260507 18:18:24.441 | INFO | Downloaded 34GiB to GPU in 2.49012s (13.6539GiB/s) | 20260507 18:18:29.580 | INFO | Analyzing chunk frames = [1591808,1592320) | 20260507 18:18:29.580 | INFO | MemInfo: { | 20260507 18:18:29.580 | INFO | "gpu" : " 17 GB", | 20260507 18:18:29.580 | INFO | "huge" : "", | 20260507 18:18:29.580 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:29.580 | INFO | "malloc" : " 14 GB", | 20260507 18:18:29.580 | INFO | "pinned" : " 29 GB" | 20260507 18:18:29.580 | INFO | } | 20260507 18:18:29.580 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:18:29.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:29.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.581 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:18:29.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.581 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:18:29.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:29.581 | INFO | Uploaded 36GiB to GPU in 2.43175s (14.8041GiB/s) | 20260507 18:18:29.581 | INFO | Downloaded 34GiB to GPU in 2.5133s (13.528GiB/s) | 20260507 18:18:34.719 | INFO | Analyzing chunk frames = [1592320,1592832) | 20260507 18:18:34.719 | INFO | MemInfo: { | 20260507 18:18:34.719 | INFO | "gpu" : " 17 GB", | 20260507 18:18:34.719 | INFO | "huge" : "", | 20260507 18:18:34.719 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:34.719 | INFO | "malloc" : " 14 GB", | 20260507 18:18:34.719 | INFO | "pinned" : " 29 GB" | 20260507 18:18:34.719 | INFO | } | 20260507 18:18:34.719 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:18:34.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:34.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.719 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:18:34.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.720 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 18:18:34.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:34.720 | INFO | Uploaded 36GiB to GPU in 2.51095s (14.3372GiB/s) | 20260507 18:18:34.720 | INFO | Downloaded 34GiB to GPU in 2.4462s (13.8991GiB/s) | 20260507 18:18:39.858 | INFO | Analyzing chunk frames = [1592832,1593344) | 20260507 18:18:39.858 | INFO | MemInfo: { | 20260507 18:18:39.858 | INFO | "gpu" : " 17 GB", | 20260507 18:18:39.858 | INFO | "huge" : "", | 20260507 18:18:39.858 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:39.858 | INFO | "malloc" : " 14 GB", | 20260507 18:18:39.858 | INFO | "pinned" : " 29 GB" | 20260507 18:18:39.859 | INFO | } | 20260507 18:18:39.859 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:18:39.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:39.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:18:39.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 18:18:39.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:39.859 | INFO | Uploaded 36GiB to GPU in 2.48881s (14.4647GiB/s) | 20260507 18:18:39.859 | INFO | Downloaded 34GiB to GPU in 2.42609s (14.0143GiB/s) | 20260507 18:18:39.971 | INFO | Starting a round of Baz IO | 20260507 18:18:43.470 | INFO | Populating RTMetrics report | 20260507 18:18:43.471 | INFO | RTMetrics report populated | 20260507 18:18:43.471 | INFO | Opening tmp RTMetrics json file | 20260507 18:18:43.493 | INFO | Serializing RTMetrics to json | 20260507 18:18:43.494 | INFO | Serialization done, swapping tmp file now... | 20260507 18:18:43.495 | INFO | RTMetrics done | 20260507 18:18:43.551 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.58 seconds | 20260507 18:18:43.552 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.79% / 0.02% / 53.19% | 20260507 18:18:43.552 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.02056% / 3.17891e-05% / 4.76837e-05% | 20260507 18:18:43.552 | INFO | Baz IO overall: | 20260507 18:18:43.552 | INFO | BytesWritten : 711003910559 | 20260507 18:18:43.552 | INFO | NumEvents : 344674394764 | 20260507 18:18:44.998 | INFO | Analyzing chunk frames = [1593344,1593856) | 20260507 18:18:44.998 | INFO | MemInfo: { | 20260507 18:18:44.998 | INFO | "gpu" : " 17 GB", | 20260507 18:18:44.998 | INFO | "huge" : "", | 20260507 18:18:44.998 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:44.998 | INFO | "malloc" : " 14 GB", | 20260507 18:18:44.998 | INFO | "pinned" : " 29 GB" | 20260507 18:18:44.998 | INFO | } | 20260507 18:18:44.998 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:18:44.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:44.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:44.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:44.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:44.998 | INFO | ANALYSIS : 69.8%, 2.9, 106.0ms, 0 | 20260507 18:18:44.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:44.998 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 18:18:44.998 | INFO | PRE_HQ : 68.2%, 1.8, 65.2ms, 0 | 20260507 18:18:44.998 | INFO | BAZWRITER : 69.3%, 3.6, 129.1ms, 0 | 20260507 18:18:44.998 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:18:44.999 | INFO | Uploaded 36GiB to GPU in 2.6134s (13.7751GiB/s) | 20260507 18:18:44.999 | INFO | Downloaded 38GiB to GPU in 2.70866s (14.0291GiB/s) | 20260507 18:18:50.137 | INFO | Analyzing chunk frames = [1593856,1594368) | 20260507 18:18:50.137 | INFO | MemInfo: { | 20260507 18:18:50.137 | INFO | "gpu" : " 17 GB", | 20260507 18:18:50.137 | INFO | "huge" : "", | 20260507 18:18:50.137 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:50.137 | INFO | "malloc" : " 14 GB", | 20260507 18:18:50.137 | INFO | "pinned" : " 29 GB" | 20260507 18:18:50.137 | INFO | } | 20260507 18:18:50.137 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:18:50.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:50.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:18:50.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:18:50.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:50.137 | INFO | Uploaded 36GiB to GPU in 2.55968s (14.0643GiB/s) | 20260507 18:18:50.137 | INFO | Downloaded 34GiB to GPU in 2.44601s (13.9002GiB/s) | 20260507 18:18:55.276 | INFO | Analyzing chunk frames = [1594368,1594880) | 20260507 18:18:55.276 | INFO | MemInfo: { | 20260507 18:18:55.276 | INFO | "gpu" : " 17 GB", | 20260507 18:18:55.276 | INFO | "huge" : "", | 20260507 18:18:55.276 | INFO | "huge_shared" : " 29 GB", | 20260507 18:18:55.276 | INFO | "malloc" : " 14 GB", | 20260507 18:18:55.276 | INFO | "pinned" : " 29 GB" | 20260507 18:18:55.276 | INFO | } | 20260507 18:18:55.276 | INFO | Perf report for the last 5.13942 seconds: | 20260507 18:18:55.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:18:55.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.276 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:18:55.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.277 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:18:55.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:18:55.277 | INFO | Uploaded 36GiB to GPU in 2.57491s (13.9811GiB/s) | 20260507 18:18:55.277 | INFO | Downloaded 34GiB to GPU in 2.4162s (14.0717GiB/s) | 20260507 18:19:00.416 | INFO | Analyzing chunk frames = [1594880,1595392) | 20260507 18:19:00.416 | INFO | MemInfo: { | 20260507 18:19:00.416 | INFO | "gpu" : " 17 GB", | 20260507 18:19:00.416 | INFO | "huge" : "", | 20260507 18:19:00.416 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:00.416 | INFO | "malloc" : " 14 GB", | 20260507 18:19:00.416 | INFO | "pinned" : " 29 GB" | 20260507 18:19:00.416 | INFO | } | 20260507 18:19:00.416 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:19:00.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:00.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:19:00.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:19:00.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:00.416 | INFO | Uploaded 36GiB to GPU in 2.50149s (14.3914GiB/s) | 20260507 18:19:00.416 | INFO | Downloaded 34GiB to GPU in 2.34336s (14.5091GiB/s) | 20260507 18:19:05.556 | INFO | Analyzing chunk frames = [1595392,1595904) | 20260507 18:19:05.556 | INFO | MemInfo: { | 20260507 18:19:05.556 | INFO | "gpu" : " 17 GB", | 20260507 18:19:05.556 | INFO | "huge" : "", | 20260507 18:19:05.556 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:05.556 | INFO | "malloc" : " 14 GB", | 20260507 18:19:05.556 | INFO | "pinned" : " 29 GB" | 20260507 18:19:05.556 | INFO | } | 20260507 18:19:05.556 | INFO | Perf report for the last 5.14017 seconds: | 20260507 18:19:05.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:05.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 18:19:05.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:19:05.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:05.556 | INFO | Uploaded 36GiB to GPU in 2.36957s (15.1926GiB/s) | 20260507 18:19:05.556 | INFO | Downloaded 34GiB to GPU in 2.46158s (13.8122GiB/s) | 20260507 18:19:10.694 | INFO | Analyzing chunk frames = [1595904,1596416) | 20260507 18:19:10.694 | INFO | MemInfo: { | 20260507 18:19:10.694 | INFO | "gpu" : " 17 GB", | 20260507 18:19:10.694 | INFO | "huge" : "", | 20260507 18:19:10.694 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:10.694 | INFO | "malloc" : " 14 GB", | 20260507 18:19:10.694 | INFO | "pinned" : " 29 GB" | 20260507 18:19:10.694 | INFO | } | 20260507 18:19:10.694 | INFO | Perf report for the last 5.13846 seconds: | 20260507 18:19:10.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:10.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.695 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:19:10.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.695 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:19:10.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:10.695 | INFO | Uploaded 36GiB to GPU in 2.45789s (14.6467GiB/s) | 20260507 18:19:10.695 | INFO | Downloaded 34GiB to GPU in 2.51194s (13.5354GiB/s) | 20260507 18:19:15.834 | INFO | Analyzing chunk frames = [1596416,1596928) | 20260507 18:19:15.834 | INFO | MemInfo: { | 20260507 18:19:15.834 | INFO | "gpu" : " 17 GB", | 20260507 18:19:15.834 | INFO | "huge" : "", | 20260507 18:19:15.834 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:15.834 | INFO | "malloc" : " 14 GB", | 20260507 18:19:15.834 | INFO | "pinned" : " 29 GB" | 20260507 18:19:15.834 | INFO | } | 20260507 18:19:15.834 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:19:15.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:15.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.834 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:19:15.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.834 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 18:19:15.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:15.835 | INFO | Uploaded 36GiB to GPU in 2.46406s (14.6101GiB/s) | 20260507 18:19:15.835 | INFO | Downloaded 34GiB to GPU in 2.43513s (13.9623GiB/s) | 20260507 18:19:20.973 | INFO | Analyzing chunk frames = [1596928,1597440) | 20260507 18:19:20.973 | INFO | MemInfo: { | 20260507 18:19:20.973 | INFO | "gpu" : " 17 GB", | 20260507 18:19:20.973 | INFO | "huge" : "", | 20260507 18:19:20.973 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:20.973 | INFO | "malloc" : " 14 GB", | 20260507 18:19:20.974 | INFO | "pinned" : " 29 GB" | 20260507 18:19:20.974 | INFO | } | 20260507 18:19:20.974 | INFO | Perf report for the last 5.13959 seconds: | 20260507 18:19:20.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:20.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:19:20.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 18:19:20.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:20.974 | INFO | Uploaded 36GiB to GPU in 2.49849s (14.4087GiB/s) | 20260507 18:19:20.974 | INFO | Downloaded 34GiB to GPU in 2.4245s (14.0235GiB/s) | 20260507 18:19:20.974 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:19:20.974 | INFO | Total mem usage: 16.4 GiB | 20260507 18:19:20.974 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:19:20.974 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.8%) | 20260507 18:19:20.974 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:19:20.974 | INFO | High Water Breakdown: | 20260507 18:19:20.974 | INFO | 59.5% : 8.73 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:19:20.974 | INFO | 40.4% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:19:20.974 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:19:20.974 | INFO | ------------------------------------------ | 20260507 18:19:20.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:19:20.974 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:19:20.974 | INFO | Total mem usage: 26.5 GiB | 20260507 18:19:20.974 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:19:20.974 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:19:20.974 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:19:20.974 | INFO | High Water Breakdown: | 20260507 18:19:20.975 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:19:20.975 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:19:20.975 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:19:20.975 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:19:20.975 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:19:20.975 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:19:20.975 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:19:20.975 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:19:20.975 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:19:20.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:19:20.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:19:20.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:19:20.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:19:20.975 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:19:20.975 | INFO | ------------------------------------------ | 20260507 18:19:20.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:19:20.975 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:19:20.975 | INFO | Total mem usage: 27.8 GiB | 20260507 18:19:20.975 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:19:20.975 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:19:20.975 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:19:20.975 | INFO | High Water Breakdown: | 20260507 18:19:20.975 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:19:20.975 | INFO | ------------------------------------------ | 20260507 18:19:20.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:19:20.975 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:19:20.975 | INFO | Total mem usage: 15.7 GiB | 20260507 18:19:20.975 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:19:20.975 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:19:20.975 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:19:20.975 | INFO | High Water Breakdown: | 20260507 18:19:20.975 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:19:20.975 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:19:20.975 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:19:20.975 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:19:20.975 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:19:20.975 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:19:20.975 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:19:20.975 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:19:20.975 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:19:20.975 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:19:20.975 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:19:20.975 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:19:20.975 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:19:20.975 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:19:20.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:19:20.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:19:20.975 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:19:20.975 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:19:20.975 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:19:20.975 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:19:20.975 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:19:20.975 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:19:20.975 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:19:20.975 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:19:20.975 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:19:20.975 | INFO | ------------------------------------------ | 20260507 18:19:20.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:19:20.975 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:19:20.975 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:19:20.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:19:20.975 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.63e+01, 1.42e+01 / 2.00e+00 | 20260507 18:19:20.975 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.05e-01, 1.14e-02 / 6.64e-03 | 20260507 18:19:20.975 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.50e+00, 6.15e-01 / 2.70e-01 | 20260507 18:19:20.976 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.15e-01 / 1.38e-01 | 20260507 18:19:20.976 | INFO | PerfReport: 15.66%, DME -- 4.19e+01 / 4.48e+01, 4.07e+00 / 1.89e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.91e+00 / 2.30e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.50e+00 / 2.80e+00, 4.33e-01 / 2.66e-01 | 20260507 18:19:20.976 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.43e+00, 3.15e-01 / 1.62e-01 | 20260507 18:19:20.976 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.20e-01, 6.32e-02 / 3.22e-02 | 20260507 18:19:20.976 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:19:20.976 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:19:20.976 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:19:20.976 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 24.18%, Upload -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 24.03%, Download -- 2.58e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 31.15%, Compute -- 3.34e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 1.87%, LockUpload -- 2.00e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 2.27%, LockDownload -- 2.43e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 16.49%, LockCompute -- 1.77e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:19:20.976 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 18:19:21.087 | INFO | Starting a round of Baz IO | 20260507 18:19:24.574 | INFO | Populating RTMetrics report | 20260507 18:19:24.575 | INFO | RTMetrics report populated | 20260507 18:19:24.575 | INFO | Opening tmp RTMetrics json file | 20260507 18:19:24.599 | INFO | Serializing RTMetrics to json | 20260507 18:19:24.600 | INFO | Serialization done, swapping tmp file now... | 20260507 18:19:24.601 | INFO | RTMetrics done | 20260507 18:19:24.651 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.564 seconds | 20260507 18:19:24.651 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.78% / 0.02% / 53.21% | 20260507 18:19:24.651 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.00086% / 1.58946e-05% / 2.38419e-05% | 20260507 18:19:24.651 | INFO | Baz IO overall: | 20260507 18:19:24.651 | INFO | BytesWritten : 712935856466 | 20260507 18:19:24.651 | INFO | NumEvents : 345611765504 | 20260507 18:19:26.112 | INFO | Analyzing chunk frames = [1597440,1597952) | 20260507 18:19:26.112 | INFO | MemInfo: { | 20260507 18:19:26.112 | INFO | "gpu" : " 17 GB", | 20260507 18:19:26.112 | INFO | "huge" : "", | 20260507 18:19:26.112 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:26.112 | INFO | "malloc" : " 14 GB", | 20260507 18:19:26.112 | INFO | "pinned" : " 29 GB" | 20260507 18:19:26.112 | INFO | } | 20260507 18:19:26.112 | INFO | Perf report for the last 5.13852 seconds: | 20260507 18:19:26.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:26.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:26.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:26.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:26.112 | INFO | ANALYSIS : 69.3%, 2.9, 104.8ms, 0 | 20260507 18:19:26.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:26.112 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260507 18:19:26.112 | INFO | PRE_HQ : 68.2%, 1.9, 66.0ms, 0 | 20260507 18:19:26.112 | INFO | BAZWRITER : 68.9%, 3.9, 139.5ms, 0 | 20260507 18:19:26.112 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:19:26.112 | INFO | Uploaded 36GiB to GPU in 2.55846s (14.0709GiB/s) | 20260507 18:19:26.113 | INFO | Downloaded 38GiB to GPU in 2.71388s (14.0021GiB/s) | 20260507 18:19:31.252 | INFO | Analyzing chunk frames = [1597952,1598464) | 20260507 18:19:31.252 | INFO | MemInfo: { | 20260507 18:19:31.252 | INFO | "gpu" : " 17 GB", | 20260507 18:19:31.252 | INFO | "huge" : "", | 20260507 18:19:31.252 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:31.252 | INFO | "malloc" : " 14 GB", | 20260507 18:19:31.252 | INFO | "pinned" : " 29 GB" | 20260507 18:19:31.252 | INFO | } | 20260507 18:19:31.252 | INFO | Perf report for the last 5.1399 seconds: | 20260507 18:19:31.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:31.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:19:31.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:19:31.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:31.252 | INFO | Uploaded 36GiB to GPU in 2.4586s (14.6425GiB/s) | 20260507 18:19:31.252 | INFO | Downloaded 34GiB to GPU in 2.46621s (13.7864GiB/s) | 20260507 18:19:36.391 | INFO | Analyzing chunk frames = [1598464,1598976) | 20260507 18:19:36.391 | INFO | MemInfo: { | 20260507 18:19:36.391 | INFO | "gpu" : " 17 GB", | 20260507 18:19:36.391 | INFO | "huge" : "", | 20260507 18:19:36.391 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:36.391 | INFO | "malloc" : " 14 GB", | 20260507 18:19:36.391 | INFO | "pinned" : " 29 GB" | 20260507 18:19:36.391 | INFO | } | 20260507 18:19:36.391 | INFO | Perf report for the last 5.13922 seconds: | 20260507 18:19:36.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:36.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.391 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 18:19:36.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.391 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:19:36.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:36.392 | INFO | Uploaded 36GiB to GPU in 2.49246s (14.4436GiB/s) | 20260507 18:19:36.392 | INFO | Downloaded 34GiB to GPU in 2.44927s (13.8817GiB/s) | 20260507 18:19:41.531 | INFO | Analyzing chunk frames = [1598976,1599488) | 20260507 18:19:41.531 | INFO | MemInfo: { | 20260507 18:19:41.531 | INFO | "gpu" : " 17 GB", | 20260507 18:19:41.531 | INFO | "huge" : "", | 20260507 18:19:41.531 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:41.531 | INFO | "malloc" : " 14 GB", | 20260507 18:19:41.531 | INFO | "pinned" : " 29 GB" | 20260507 18:19:41.531 | INFO | } | 20260507 18:19:41.531 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:19:41.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:41.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 18:19:41.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:19:41.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:41.531 | INFO | Uploaded 36GiB to GPU in 2.43937s (14.7579GiB/s) | 20260507 18:19:41.532 | INFO | Downloaded 34GiB to GPU in 2.36422s (14.3811GiB/s) | 20260507 18:19:46.670 | INFO | Analyzing chunk frames = [1599488,1600000) | 20260507 18:19:46.670 | INFO | MemInfo: { | 20260507 18:19:46.670 | INFO | "gpu" : " 17 GB", | 20260507 18:19:46.670 | INFO | "huge" : "", | 20260507 18:19:46.670 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:46.670 | INFO | "malloc" : " 14 GB", | 20260507 18:19:46.670 | INFO | "pinned" : " 29 GB" | 20260507 18:19:46.670 | INFO | } | 20260507 18:19:46.670 | INFO | Perf report for the last 5.13964 seconds: | 20260507 18:19:46.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:46.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:19:46.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:19:46.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:46.671 | INFO | Uploaded 36GiB to GPU in 2.31545s (15.5477GiB/s) | 20260507 18:19:46.671 | INFO | Downloaded 34GiB to GPU in 2.49581s (13.6228GiB/s) | 20260507 18:19:51.809 | INFO | Analyzing chunk frames = [1600000,1600512) | 20260507 18:19:51.809 | INFO | MemInfo: { | 20260507 18:19:51.809 | INFO | "gpu" : " 17 GB", | 20260507 18:19:51.809 | INFO | "huge" : "", | 20260507 18:19:51.809 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:51.809 | INFO | "malloc" : " 14 GB", | 20260507 18:19:51.809 | INFO | "pinned" : " 29 GB" | 20260507 18:19:51.809 | INFO | } | 20260507 18:19:51.809 | INFO | Perf report for the last 5.13877 seconds: | 20260507 18:19:51.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:51.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.809 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:19:51.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.809 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:19:51.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:51.810 | INFO | Uploaded 36GiB to GPU in 2.43267s (14.7985GiB/s) | 20260507 18:19:51.810 | INFO | Downloaded 34GiB to GPU in 2.51436s (13.5223GiB/s) | 20260507 18:19:56.949 | INFO | Analyzing chunk frames = [1600512,1601024) | 20260507 18:19:56.949 | INFO | MemInfo: { | 20260507 18:19:56.949 | INFO | "gpu" : " 17 GB", | 20260507 18:19:56.949 | INFO | "huge" : "", | 20260507 18:19:56.949 | INFO | "huge_shared" : " 29 GB", | 20260507 18:19:56.949 | INFO | "malloc" : " 14 GB", | 20260507 18:19:56.949 | INFO | "pinned" : " 29 GB" | 20260507 18:19:56.949 | INFO | } | 20260507 18:19:56.949 | INFO | Perf report for the last 5.13979 seconds: | 20260507 18:19:56.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:19:56.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:19:56.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:19:56.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:19:56.949 | INFO | Uploaded 36GiB to GPU in 2.49734s (14.4153GiB/s) | 20260507 18:19:56.949 | INFO | Downloaded 34GiB to GPU in 2.4389s (13.9407GiB/s) | 20260507 18:20:02.088 | INFO | Analyzing chunk frames = [1601024,1601536) | 20260507 18:20:02.088 | INFO | MemInfo: { | 20260507 18:20:02.088 | INFO | "gpu" : " 17 GB", | 20260507 18:20:02.088 | INFO | "huge" : "", | 20260507 18:20:02.088 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:02.088 | INFO | "malloc" : " 14 GB", | 20260507 18:20:02.088 | INFO | "pinned" : " 29 GB" | 20260507 18:20:02.088 | INFO | } | 20260507 18:20:02.088 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:20:02.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:02.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.088 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:20:02.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.089 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:20:02.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:02.089 | INFO | Uploaded 36GiB to GPU in 2.48834s (14.4675GiB/s) | 20260507 18:20:02.089 | INFO | Downloaded 34GiB to GPU in 2.42957s (13.9942GiB/s) | 20260507 18:20:02.186 | INFO | Starting a round of Baz IO | 20260507 18:20:05.695 | INFO | Populating RTMetrics report | 20260507 18:20:05.696 | INFO | RTMetrics report populated | 20260507 18:20:05.696 | INFO | Opening tmp RTMetrics json file | 20260507 18:20:05.724 | INFO | Serializing RTMetrics to json | 20260507 18:20:05.727 | INFO | Serialization done, swapping tmp file now... | 20260507 18:20:05.727 | INFO | RTMetrics done | 20260507 18:20:05.811 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.624 seconds | 20260507 18:20:05.811 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.76% / 0.02% / 53.22% | 20260507 18:20:05.811 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.99586% / 1.58946e-05% / 3.97364e-05% | 20260507 18:20:05.811 | INFO | Baz IO overall: | 20260507 18:20:05.811 | INFO | BytesWritten : 714868616978 | 20260507 18:20:05.811 | INFO | NumEvents : 346549573392 | 20260507 18:20:07.227 | INFO | Analyzing chunk frames = [1601536,1602048) | 20260507 18:20:07.227 | INFO | MemInfo: { | 20260507 18:20:07.227 | INFO | "gpu" : " 17 GB", | 20260507 18:20:07.227 | INFO | "huge" : "", | 20260507 18:20:07.227 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:07.227 | INFO | "malloc" : " 14 GB", | 20260507 18:20:07.228 | INFO | "pinned" : " 29 GB" | 20260507 18:20:07.228 | INFO | } | 20260507 18:20:07.228 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:20:07.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:07.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:07.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:07.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:07.228 | INFO | ANALYSIS : 69.7%, 2.9, 107.8ms, 0 | 20260507 18:20:07.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:07.228 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 18:20:07.228 | INFO | PRE_HQ : 66.1%, 1.7, 58.8ms, 0 | 20260507 18:20:07.228 | INFO | BAZWRITER : 69.7%, 3.8, 135.9ms, 0 | 20260507 18:20:07.228 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 18:20:07.228 | INFO | Uploaded 36GiB to GPU in 2.54909s (14.1227GiB/s) | 20260507 18:20:07.228 | INFO | Downloaded 38GiB to GPU in 2.72262s (13.9572GiB/s) | 20260507 18:20:12.367 | INFO | Analyzing chunk frames = [1602048,1602560) | 20260507 18:20:12.367 | INFO | MemInfo: { | 20260507 18:20:12.367 | INFO | "gpu" : " 17 GB", | 20260507 18:20:12.367 | INFO | "huge" : "", | 20260507 18:20:12.367 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:12.367 | INFO | "malloc" : " 14 GB", | 20260507 18:20:12.367 | INFO | "pinned" : " 29 GB" | 20260507 18:20:12.367 | INFO | } | 20260507 18:20:12.367 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:20:12.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:12.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:20:12.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:20:12.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:12.367 | INFO | Uploaded 36GiB to GPU in 2.50525s (14.3698GiB/s) | 20260507 18:20:12.367 | INFO | Downloaded 34GiB to GPU in 2.45591s (13.8441GiB/s) | 20260507 18:20:17.506 | INFO | Analyzing chunk frames = [1602560,1603072) | 20260507 18:20:17.506 | INFO | MemInfo: { | 20260507 18:20:17.506 | INFO | "gpu" : " 17 GB", | 20260507 18:20:17.506 | INFO | "huge" : "", | 20260507 18:20:17.506 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:17.506 | INFO | "malloc" : " 14 GB", | 20260507 18:20:17.506 | INFO | "pinned" : " 29 GB" | 20260507 18:20:17.506 | INFO | } | 20260507 18:20:17.506 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:20:17.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:17.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.507 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:20:17.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.507 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:20:17.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:17.507 | INFO | Uploaded 36GiB to GPU in 2.53748s (14.1873GiB/s) | 20260507 18:20:17.507 | INFO | Downloaded 34GiB to GPU in 2.44328s (13.9157GiB/s) | 20260507 18:20:22.646 | INFO | Analyzing chunk frames = [1603072,1603584) | 20260507 18:20:22.646 | INFO | MemInfo: { | 20260507 18:20:22.646 | INFO | "gpu" : " 17 GB", | 20260507 18:20:22.646 | INFO | "huge" : "", | 20260507 18:20:22.646 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:22.646 | INFO | "malloc" : " 14 GB", | 20260507 18:20:22.646 | INFO | "pinned" : " 29 GB" | 20260507 18:20:22.646 | INFO | } | 20260507 18:20:22.646 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:20:22.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:22.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:20:22.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:20:22.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:22.646 | INFO | Uploaded 36GiB to GPU in 2.46697s (14.5928GiB/s) | 20260507 18:20:22.647 | INFO | Downloaded 34GiB to GPU in 2.37543s (14.3132GiB/s) | 20260507 18:20:27.785 | INFO | Analyzing chunk frames = [1603584,1604096) | 20260507 18:20:27.785 | INFO | MemInfo: { | 20260507 18:20:27.785 | INFO | "gpu" : " 17 GB", | 20260507 18:20:27.785 | INFO | "huge" : "", | 20260507 18:20:27.785 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:27.785 | INFO | "malloc" : " 14 GB", | 20260507 18:20:27.785 | INFO | "pinned" : " 29 GB" | 20260507 18:20:27.785 | INFO | } | 20260507 18:20:27.785 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:20:27.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:27.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.785 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:20:27.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.785 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 18:20:27.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:27.786 | INFO | Uploaded 36GiB to GPU in 2.32058s (15.5134GiB/s) | 20260507 18:20:27.786 | INFO | Downloaded 34GiB to GPU in 2.49175s (13.645GiB/s) | 20260507 18:20:32.924 | INFO | Analyzing chunk frames = [1604096,1604608) | 20260507 18:20:32.924 | INFO | MemInfo: { | 20260507 18:20:32.924 | INFO | "gpu" : " 17 GB", | 20260507 18:20:32.924 | INFO | "huge" : "", | 20260507 18:20:32.924 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:32.924 | INFO | "malloc" : " 14 GB", | 20260507 18:20:32.924 | INFO | "pinned" : " 29 GB" | 20260507 18:20:32.924 | INFO | } | 20260507 18:20:32.924 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:20:32.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:32.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.924 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:20:32.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.925 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260507 18:20:32.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:32.925 | INFO | Uploaded 36GiB to GPU in 2.35975s (15.2559GiB/s) | 20260507 18:20:32.925 | INFO | Downloaded 34GiB to GPU in 2.52214s (13.4806GiB/s) | 20260507 18:20:38.063 | INFO | Analyzing chunk frames = [1604608,1605120) | 20260507 18:20:38.063 | INFO | MemInfo: { | 20260507 18:20:38.063 | INFO | "gpu" : " 17 GB", | 20260507 18:20:38.063 | INFO | "huge" : "", | 20260507 18:20:38.063 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:38.063 | INFO | "malloc" : " 14 GB", | 20260507 18:20:38.064 | INFO | "pinned" : " 29 GB" | 20260507 18:20:38.064 | INFO | } | 20260507 18:20:38.064 | INFO | Perf report for the last 5.13948 seconds: | 20260507 18:20:38.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:38.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.064 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:20:38.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.064 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 18:20:38.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:38.065 | INFO | Uploaded 36GiB to GPU in 2.45378s (14.6712GiB/s) | 20260507 18:20:38.065 | INFO | Downloaded 34GiB to GPU in 2.45801s (13.8323GiB/s) | 20260507 18:20:43.202 | INFO | Analyzing chunk frames = [1605120,1605632) | 20260507 18:20:43.203 | INFO | MemInfo: { | 20260507 18:20:43.203 | INFO | "gpu" : " 17 GB", | 20260507 18:20:43.203 | INFO | "huge" : "", | 20260507 18:20:43.203 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:43.203 | INFO | "malloc" : " 14 GB", | 20260507 18:20:43.203 | INFO | "pinned" : " 29 GB" | 20260507 18:20:43.203 | INFO | } | 20260507 18:20:43.203 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:20:43.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:43.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:20:43.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | PRE_HQ : 35.9%, 1.0, 18.7ms, 0 | 20260507 18:20:43.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:43.203 | INFO | Uploaded 36GiB to GPU in 2.43945s (14.7575GiB/s) | 20260507 18:20:43.204 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:20:43.204 | INFO | Total mem usage: 16.4 GiB | 20260507 18:20:43.204 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:20:43.204 | INFO | Downloaded 34GiB to GPU in 2.4439s (13.9122GiB/s) | 20260507 18:20:43.204 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 18:20:43.204 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:20:43.204 | INFO | High Water Breakdown: | 20260507 18:20:43.204 | INFO | 59.7% : 8.85 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:20:43.204 | INFO | 40.2% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:20:43.204 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:20:43.204 | INFO | ------------------------------------------ | 20260507 18:20:43.204 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:20:43.204 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:20:43.204 | INFO | Total mem usage: 26.5 GiB | 20260507 18:20:43.204 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:20:43.204 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:20:43.204 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 18:20:43.204 | INFO | High Water Breakdown: | 20260507 18:20:43.204 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:20:43.204 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:20:43.204 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:20:43.204 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:20:43.204 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:20:43.204 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:20:43.204 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:20:43.204 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:20:43.204 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:20:43.204 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:20:43.204 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:20:43.204 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:20:43.204 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:20:43.204 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:20:43.204 | INFO | ------------------------------------------ | 20260507 18:20:43.204 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:20:43.204 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:20:43.204 | INFO | Total mem usage: 27.8 GiB | 20260507 18:20:43.204 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:20:43.204 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:20:43.204 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:20:43.204 | INFO | High Water Breakdown: | 20260507 18:20:43.204 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:20:43.204 | INFO | ------------------------------------------ | 20260507 18:20:43.204 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:20:43.204 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:20:43.205 | INFO | Total mem usage: 15.7 GiB | 20260507 18:20:43.205 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:20:43.205 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:20:43.205 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:20:43.205 | INFO | High Water Breakdown: | 20260507 18:20:43.205 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:20:43.205 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:20:43.205 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:20:43.205 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:20:43.205 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:20:43.205 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:20:43.205 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:20:43.205 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:20:43.205 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:20:43.205 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:20:43.205 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:20:43.205 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:20:43.205 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:20:43.205 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:20:43.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:20:43.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:20:43.205 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:20:43.205 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:20:43.205 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:20:43.205 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:20:43.205 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:20:43.205 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:20:43.205 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:20:43.205 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:20:43.205 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:20:43.205 | INFO | ------------------------------------------ | 20260507 18:20:43.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:20:43.205 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:20:43.205 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:20:43.205 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:20:43.205 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.62e+01, 1.43e+01 / 2.20e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 0.30%, AggregateStats -- 9.96e-02 / 1.04e-01, 1.25e-02 / 5.64e-03 | 20260507 18:20:43.205 | INFO | PerfReport: 12.10%, Baseline -- 4.04e+00 / 4.49e+00, 6.19e-01 / 2.58e-01 | 20260507 18:20:43.205 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.49e+00, 2.17e-01 / 1.16e-01 | 20260507 18:20:43.205 | INFO | PerfReport: 15.63%, DME -- 4.18e+01 / 4.46e+01, 3.85e+00 / 2.14e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.89e+00 / 2.15e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.51e+00 / 2.81e+00, 4.36e-01 / 2.56e-01 | 20260507 18:20:43.205 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.44e+00, 3.19e-01 / 1.80e-01 | 20260507 18:20:43.205 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.24e-01, 6.38e-02 / 4.16e-02 | 20260507 18:20:43.205 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:20:43.205 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:20:43.205 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:20:43.205 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 23.92%, Upload -- 2.56e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 31.24%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 1.62%, LockUpload -- 1.73e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 2.51%, LockDownload -- 2.68e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 16.49%, LockCompute -- 1.76e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:20:43.205 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 18:20:43.315 | INFO | Starting a round of Baz IO | 20260507 18:20:46.815 | INFO | Populating RTMetrics report | 20260507 18:20:46.815 | INFO | RTMetrics report populated | 20260507 18:20:46.815 | INFO | Opening tmp RTMetrics json file | 20260507 18:20:46.852 | INFO | Serializing RTMetrics to json | 20260507 18:20:46.865 | INFO | Serialization done, swapping tmp file now... | 20260507 18:20:46.875 | INFO | RTMetrics done | 20260507 18:20:46.972 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.656 seconds | 20260507 18:20:46.972 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.74% / 0.02% / 53.24% | 20260507 18:20:46.972 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.0071% / 2.78155e-05% / 1.98682e-05% | 20260507 18:20:46.972 | INFO | Baz IO overall: | 20260507 18:20:46.972 | INFO | BytesWritten : 716805267245 | 20260507 18:20:46.972 | INFO | NumEvents : 347490361165 | 20260507 18:20:48.342 | INFO | Analyzing chunk frames = [1605632,1606144) | 20260507 18:20:48.342 | INFO | MemInfo: { | 20260507 18:20:48.342 | INFO | "gpu" : " 17 GB", | 20260507 18:20:48.342 | INFO | "huge" : "", | 20260507 18:20:48.342 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:48.342 | INFO | "malloc" : " 14 GB", | 20260507 18:20:48.342 | INFO | "pinned" : " 29 GB" | 20260507 18:20:48.342 | INFO | } | 20260507 18:20:48.342 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:20:48.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:48.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:48.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:48.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:48.342 | INFO | ANALYSIS : 69.5%, 2.9, 107.5ms, 0 | 20260507 18:20:48.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:48.342 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 18:20:48.342 | INFO | PRE_HQ : 66.7%, 1.8, 62.8ms, 0 | 20260507 18:20:48.342 | INFO | BAZWRITER : 70.9%, 4.1, 147.6ms, 0 | 20260507 18:20:48.342 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 18:20:48.342 | INFO | Uploaded 36GiB to GPU in 2.50978s (14.3439GiB/s) | 20260507 18:20:48.342 | INFO | Downloaded 38GiB to GPU in 2.74469s (13.8449GiB/s) | 20260507 18:20:53.482 | INFO | Analyzing chunk frames = [1606144,1606656) | 20260507 18:20:53.482 | INFO | MemInfo: { | 20260507 18:20:53.482 | INFO | "gpu" : " 17 GB", | 20260507 18:20:53.482 | INFO | "huge" : "", | 20260507 18:20:53.482 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:53.482 | INFO | "malloc" : " 14 GB", | 20260507 18:20:53.482 | INFO | "pinned" : " 29 GB" | 20260507 18:20:53.482 | INFO | } | 20260507 18:20:53.482 | INFO | Perf report for the last 5.14002 seconds: | 20260507 18:20:53.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:53.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:20:53.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:20:53.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:53.482 | INFO | Uploaded 36GiB to GPU in 2.46542s (14.602GiB/s) | 20260507 18:20:53.482 | INFO | Downloaded 34GiB to GPU in 2.46578s (13.7888GiB/s) | 20260507 18:20:58.621 | INFO | Analyzing chunk frames = [1606656,1607168) | 20260507 18:20:58.621 | INFO | MemInfo: { | 20260507 18:20:58.621 | INFO | "gpu" : " 17 GB", | 20260507 18:20:58.621 | INFO | "huge" : "", | 20260507 18:20:58.621 | INFO | "huge_shared" : " 29 GB", | 20260507 18:20:58.621 | INFO | "malloc" : " 14 GB", | 20260507 18:20:58.621 | INFO | "pinned" : " 29 GB" | 20260507 18:20:58.621 | INFO | } | 20260507 18:20:58.621 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:20:58.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:20:58.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 18:20:58.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:20:58.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:20:58.621 | INFO | Uploaded 36GiB to GPU in 2.50998s (14.3428GiB/s) | 20260507 18:20:58.621 | INFO | Downloaded 34GiB to GPU in 2.46018s (13.8201GiB/s) | 20260507 18:21:03.760 | INFO | Analyzing chunk frames = [1607168,1607680) | 20260507 18:21:03.760 | INFO | MemInfo: { | 20260507 18:21:03.760 | INFO | "gpu" : " 17 GB", | 20260507 18:21:03.760 | INFO | "huge" : "", | 20260507 18:21:03.760 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:03.760 | INFO | "malloc" : " 14 GB", | 20260507 18:21:03.760 | INFO | "pinned" : " 29 GB" | 20260507 18:21:03.760 | INFO | } | 20260507 18:21:03.760 | INFO | Perf report for the last 5.13904 seconds: | 20260507 18:21:03.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:03.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.760 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 18:21:03.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.760 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:21:03.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:03.761 | INFO | Uploaded 36GiB to GPU in 2.44371s (14.7317GiB/s) | 20260507 18:21:03.761 | INFO | Downloaded 34GiB to GPU in 2.3655s (14.3733GiB/s) | 20260507 18:21:08.899 | INFO | Analyzing chunk frames = [1607680,1608192) | 20260507 18:21:08.899 | INFO | MemInfo: { | 20260507 18:21:08.899 | INFO | "gpu" : " 17 GB", | 20260507 18:21:08.899 | INFO | "huge" : "", | 20260507 18:21:08.899 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:08.899 | INFO | "malloc" : " 14 GB", | 20260507 18:21:08.899 | INFO | "pinned" : " 29 GB" | 20260507 18:21:08.899 | INFO | } | 20260507 18:21:08.899 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:21:08.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:08.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.900 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:21:08.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.900 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:21:08.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:08.900 | INFO | Uploaded 36GiB to GPU in 2.3019s (15.6392GiB/s) | 20260507 18:21:08.900 | INFO | Downloaded 34GiB to GPU in 2.50179s (13.5903GiB/s) | 20260507 18:21:14.039 | INFO | Analyzing chunk frames = [1608192,1608704) | 20260507 18:21:14.039 | INFO | MemInfo: { | 20260507 18:21:14.039 | INFO | "gpu" : " 17 GB", | 20260507 18:21:14.039 | INFO | "huge" : "", | 20260507 18:21:14.039 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:14.039 | INFO | "malloc" : " 14 GB", | 20260507 18:21:14.039 | INFO | "pinned" : " 29 GB" | 20260507 18:21:14.039 | INFO | } | 20260507 18:21:14.039 | INFO | Perf report for the last 5.13971 seconds: | 20260507 18:21:14.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:14.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.039 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:21:14.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.039 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:21:14.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:14.040 | INFO | Uploaded 36GiB to GPU in 2.48174s (14.5059GiB/s) | 20260507 18:21:14.040 | INFO | Downloaded 34GiB to GPU in 2.51748s (13.5056GiB/s) | 20260507 18:21:19.178 | INFO | Analyzing chunk frames = [1608704,1609216) | 20260507 18:21:19.178 | INFO | MemInfo: { | 20260507 18:21:19.178 | INFO | "gpu" : " 17 GB", | 20260507 18:21:19.178 | INFO | "huge" : "", | 20260507 18:21:19.178 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:19.178 | INFO | "malloc" : " 14 GB", | 20260507 18:21:19.178 | INFO | "pinned" : " 29 GB" | 20260507 18:21:19.178 | INFO | } | 20260507 18:21:19.178 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:21:19.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:19.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.178 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:21:19.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.179 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 18:21:19.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:19.179 | INFO | Uploaded 36GiB to GPU in 2.48734s (14.4733GiB/s) | 20260507 18:21:19.179 | INFO | Downloaded 34GiB to GPU in 2.44348s (13.9146GiB/s) | 20260507 18:21:24.317 | INFO | Analyzing chunk frames = [1609216,1609728) | 20260507 18:21:24.318 | INFO | MemInfo: { | 20260507 18:21:24.318 | INFO | "gpu" : " 17 GB", | 20260507 18:21:24.318 | INFO | "huge" : "", | 20260507 18:21:24.318 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:24.318 | INFO | "malloc" : " 14 GB", | 20260507 18:21:24.318 | INFO | "pinned" : " 29 GB" | 20260507 18:21:24.318 | INFO | } | 20260507 18:21:24.318 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:21:24.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:24.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:21:24.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | PRE_HQ : 37.2%, 1.0, 19.4ms, 0 | 20260507 18:21:24.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:24.318 | INFO | Uploaded 36GiB to GPU in 2.50306s (14.3824GiB/s) | 20260507 18:21:24.318 | INFO | Downloaded 34GiB to GPU in 2.44406s (13.9113GiB/s) | 20260507 18:21:24.430 | INFO | Starting a round of Baz IO | 20260507 18:21:27.925 | INFO | Populating RTMetrics report | 20260507 18:21:27.925 | INFO | RTMetrics report populated | 20260507 18:21:27.925 | INFO | Opening tmp RTMetrics json file | 20260507 18:21:28.154 | INFO | Serializing RTMetrics to json | 20260507 18:21:28.177 | INFO | Serialization done, swapping tmp file now... | 20260507 18:21:28.179 | INFO | RTMetrics done | 20260507 18:21:28.493 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 4.063 seconds | 20260507 18:21:28.493 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.73% / 0.02% / 53.26% | 20260507 18:21:28.493 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.99571% / 1.98682e-05% / 4.37101e-05% | 20260507 18:21:28.493 | INFO | Baz IO overall: | 20260507 18:21:28.493 | INFO | BytesWritten : 718740949628 | 20260507 18:21:28.493 | INFO | NumEvents : 348430023225 | 20260507 18:21:29.457 | INFO | Analyzing chunk frames = [1609728,1610240) | 20260507 18:21:29.457 | INFO | MemInfo: { | 20260507 18:21:29.457 | INFO | "gpu" : " 17 GB", | 20260507 18:21:29.457 | INFO | "huge" : "", | 20260507 18:21:29.457 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:29.457 | INFO | "malloc" : " 14 GB", | 20260507 18:21:29.457 | INFO | "pinned" : " 29 GB" | 20260507 18:21:29.457 | INFO | } | 20260507 18:21:29.457 | INFO | Perf report for the last 5.13957 seconds: | 20260507 18:21:29.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:29.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:29.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:29.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:29.457 | INFO | ANALYSIS : 69.7%, 2.9, 105.1ms, 0 | 20260507 18:21:29.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:29.458 | INFO | RT_METRICS : 51.3%, 1.0, 26.9ms, 0 | 20260507 18:21:29.458 | INFO | PRE_HQ : 68.5%, 1.8, 65.5ms, 0 | 20260507 18:21:29.458 | INFO | BAZWRITER : 78.6%, 4.8, 193.9ms, 0 | 20260507 18:21:29.458 | INFO | BAZ_SAMPLING : 4.6%, 1.0, 2.4ms, 0 | 20260507 18:21:29.458 | INFO | Uploaded 36GiB to GPU in 2.53032s (14.2275GiB/s) | 20260507 18:21:29.458 | INFO | Downloaded 38GiB to GPU in 2.68898s (14.1317GiB/s) | 20260507 18:21:34.596 | INFO | Analyzing chunk frames = [1610240,1610752) | 20260507 18:21:34.596 | INFO | MemInfo: { | 20260507 18:21:34.596 | INFO | "gpu" : " 17 GB", | 20260507 18:21:34.596 | INFO | "huge" : "", | 20260507 18:21:34.596 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:34.596 | INFO | "malloc" : " 14 GB", | 20260507 18:21:34.596 | INFO | "pinned" : " 29 GB" | 20260507 18:21:34.596 | INFO | } | 20260507 18:21:34.596 | INFO | Perf report for the last 5.13882 seconds: | 20260507 18:21:34.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:34.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:21:34.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:21:34.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:34.596 | INFO | Uploaded 36GiB to GPU in 2.56368s (14.0423GiB/s) | 20260507 18:21:34.597 | INFO | Downloaded 34GiB to GPU in 2.43884s (13.941GiB/s) | 20260507 18:21:39.735 | INFO | Analyzing chunk frames = [1610752,1611264) | 20260507 18:21:39.735 | INFO | MemInfo: { | 20260507 18:21:39.735 | INFO | "gpu" : " 17 GB", | 20260507 18:21:39.735 | INFO | "huge" : "", | 20260507 18:21:39.735 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:39.736 | INFO | "malloc" : " 14 GB", | 20260507 18:21:39.736 | INFO | "pinned" : " 29 GB" | 20260507 18:21:39.736 | INFO | } | 20260507 18:21:39.736 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:21:39.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:39.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:21:39.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:21:39.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:39.736 | INFO | Uploaded 36GiB to GPU in 2.57321s (13.9903GiB/s) | 20260507 18:21:39.736 | INFO | Downloaded 34GiB to GPU in 2.42517s (14.0196GiB/s) | 20260507 18:21:44.874 | INFO | Analyzing chunk frames = [1611264,1611776) | 20260507 18:21:44.874 | INFO | MemInfo: { | 20260507 18:21:44.874 | INFO | "gpu" : " 17 GB", | 20260507 18:21:44.874 | INFO | "huge" : "", | 20260507 18:21:44.874 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:44.875 | INFO | "malloc" : " 14 GB", | 20260507 18:21:44.875 | INFO | "pinned" : " 29 GB" | 20260507 18:21:44.875 | INFO | } | 20260507 18:21:44.875 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:21:44.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:44.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:21:44.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:21:44.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:44.875 | INFO | Uploaded 36GiB to GPU in 2.50238s (14.3863GiB/s) | 20260507 18:21:44.875 | INFO | Downloaded 34GiB to GPU in 2.34665s (14.4887GiB/s) | 20260507 18:21:50.014 | INFO | Analyzing chunk frames = [1611776,1612288) | 20260507 18:21:50.014 | INFO | MemInfo: { | 20260507 18:21:50.014 | INFO | "gpu" : " 17 GB", | 20260507 18:21:50.014 | INFO | "huge" : "", | 20260507 18:21:50.014 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:50.014 | INFO | "malloc" : " 14 GB", | 20260507 18:21:50.014 | INFO | "pinned" : " 29 GB" | 20260507 18:21:50.015 | INFO | } | 20260507 18:21:50.015 | INFO | Perf report for the last 5.13985 seconds: | 20260507 18:21:50.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:50.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:21:50.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 18:21:50.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:50.015 | INFO | Uploaded 36GiB to GPU in 2.36999s (15.1899GiB/s) | 20260507 18:21:50.015 | INFO | Downloaded 34GiB to GPU in 2.47632s (13.7301GiB/s) | 20260507 18:21:55.154 | INFO | Analyzing chunk frames = [1612288,1612800) | 20260507 18:21:55.154 | INFO | MemInfo: { | 20260507 18:21:55.154 | INFO | "gpu" : " 17 GB", | 20260507 18:21:55.154 | INFO | "huge" : "", | 20260507 18:21:55.154 | INFO | "huge_shared" : " 29 GB", | 20260507 18:21:55.154 | INFO | "malloc" : " 14 GB", | 20260507 18:21:55.154 | INFO | "pinned" : " 29 GB" | 20260507 18:21:55.154 | INFO | } | 20260507 18:21:55.154 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:21:55.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:21:55.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:21:55.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:21:55.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:21:55.154 | INFO | Uploaded 36GiB to GPU in 2.43653s (14.7751GiB/s) | 20260507 18:21:55.154 | INFO | Downloaded 34GiB to GPU in 2.50156s (13.5915GiB/s) | 20260507 18:22:00.293 | INFO | Analyzing chunk frames = [1612800,1613312) | 20260507 18:22:00.293 | INFO | MemInfo: { | 20260507 18:22:00.293 | INFO | "gpu" : " 17 GB", | 20260507 18:22:00.293 | INFO | "huge" : "", | 20260507 18:22:00.293 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:00.293 | INFO | "malloc" : " 14 GB", | 20260507 18:22:00.293 | INFO | "pinned" : " 29 GB" | 20260507 18:22:00.293 | INFO | } | 20260507 18:22:00.293 | INFO | Perf report for the last 5.13896 seconds: | 20260507 18:22:00.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:00.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 18:22:00.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:22:00.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:00.293 | INFO | Uploaded 36GiB to GPU in 2.46351s (14.6133GiB/s) | 20260507 18:22:00.293 | INFO | Downloaded 34GiB to GPU in 2.43248s (13.9775GiB/s) | 20260507 18:22:05.432 | INFO | Analyzing chunk frames = [1613312,1613824) | 20260507 18:22:05.432 | INFO | MemInfo: { | 20260507 18:22:05.432 | INFO | "gpu" : " 17 GB", | 20260507 18:22:05.432 | INFO | "huge" : "", | 20260507 18:22:05.432 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:05.432 | INFO | "malloc" : " 14 GB", | 20260507 18:22:05.432 | INFO | "pinned" : " 29 GB" | 20260507 18:22:05.432 | INFO | } | 20260507 18:22:05.432 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:22:05.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:05.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.432 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 18:22:05.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.433 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:22:05.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:05.433 | INFO | Uploaded 36GiB to GPU in 2.53115s (14.2228GiB/s) | 20260507 18:22:05.433 | INFO | Downloaded 34GiB to GPU in 2.50329s (13.5821GiB/s) | 20260507 18:22:05.433 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:22:05.433 | INFO | Total mem usage: 16.4 GiB | 20260507 18:22:05.433 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:22:05.433 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -1.35%) | 20260507 18:22:05.433 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:22:05.433 | INFO | High Water Breakdown: | 20260507 18:22:05.433 | INFO | 61% : 9.88 GiB ( 99.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:22:05.433 | INFO | 38.8% : 6.29 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:22:05.433 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:22:05.433 | INFO | ------------------------------------------ | 20260507 18:22:05.433 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:22:05.433 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:22:05.433 | INFO | Total mem usage: 26.5 GiB | 20260507 18:22:05.433 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:22:05.433 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:22:05.433 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:22:05.433 | INFO | High Water Breakdown: | 20260507 18:22:05.433 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:22:05.433 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:22:05.433 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:22:05.433 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:22:05.433 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:22:05.433 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:22:05.433 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:22:05.433 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:22:05.433 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:22:05.433 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:22:05.433 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:22:05.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:22:05.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:22:05.434 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:22:05.434 | INFO | ------------------------------------------ | 20260507 18:22:05.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:22:05.434 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:22:05.434 | INFO | Total mem usage: 27.8 GiB | 20260507 18:22:05.434 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:22:05.434 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:22:05.434 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:22:05.434 | INFO | High Water Breakdown: | 20260507 18:22:05.434 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:22:05.434 | INFO | ------------------------------------------ | 20260507 18:22:05.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:22:05.434 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:22:05.434 | INFO | Total mem usage: 15.7 GiB | 20260507 18:22:05.434 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:22:05.434 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:22:05.434 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:22:05.434 | INFO | High Water Breakdown: | 20260507 18:22:05.434 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:22:05.434 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:22:05.434 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:22:05.434 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:22:05.434 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:22:05.434 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:22:05.434 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:22:05.434 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:22:05.434 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:22:05.434 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:22:05.434 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:22:05.434 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:22:05.434 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:22:05.434 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:22:05.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:22:05.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:22:05.434 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:22:05.434 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:22:05.434 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:22:05.434 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:22:05.434 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:22:05.434 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:22:05.434 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:22:05.434 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:22:05.434 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:22:05.434 | INFO | ------------------------------------------ | 20260507 18:22:05.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:22:05.434 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:22:05.434 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:22:05.434 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:22:05.434 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.68e+01, 1.44e+01 / 2.23e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 0.29%, AggregateStats -- 9.86e-02 / 1.04e-01, 1.08e-02 / 5.62e-03 | 20260507 18:22:05.434 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.50e+00, 6.18e-01 / 2.20e-01 | 20260507 18:22:05.434 | INFO | PerfReport: 3.42%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 1.07e-01 | 20260507 18:22:05.434 | INFO | PerfReport: 15.70%, DME -- 4.20e+01 / 4.52e+01, 4.01e+00 / 2.03e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.91e+00 / 2.29e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.36e-01 / 2.48e-01 | 20260507 18:22:05.434 | INFO | PerfReport: 3.71%, Metrics -- 1.24e+00 / 1.42e+00, 3.14e-01 / 1.65e-01 | 20260507 18:22:05.434 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.91e-01 / 4.22e-01, 6.32e-02 / 4.05e-02 | 20260507 18:22:05.434 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:22:05.434 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:22:05.434 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:22:05.434 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 24.10%, Upload -- 2.58e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 24.15%, Download -- 2.59e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 31.23%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 1.83%, LockUpload -- 1.96e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 2.29%, LockDownload -- 2.45e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 16.39%, LockCompute -- 1.76e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:22:05.434 | INFO | PerfReport: 0.01%, Misc -- 1.56e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260507 18:22:05.529 | INFO | Starting a round of Baz IO | 20260507 18:22:09.037 | INFO | Populating RTMetrics report | 20260507 18:22:09.037 | INFO | RTMetrics report populated | 20260507 18:22:09.037 | INFO | Opening tmp RTMetrics json file | 20260507 18:22:09.038 | INFO | Serializing RTMetrics to json | 20260507 18:22:09.040 | INFO | Serialization done, swapping tmp file now... | 20260507 18:22:09.040 | INFO | RTMetrics done | 20260507 18:22:09.154 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.624 seconds | 20260507 18:22:09.154 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.71% / 0.02% / 53.27% | 20260507 18:22:09.154 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.98369% / 3.57628e-05% / 1.98682e-05% | 20260507 18:22:09.154 | INFO | Baz IO overall: | 20260507 18:22:09.154 | INFO | BytesWritten : 720674390537 | 20260507 18:22:09.154 | INFO | NumEvents : 349368177302 | 20260507 18:22:10.572 | INFO | Analyzing chunk frames = [1613824,1614336) | 20260507 18:22:10.572 | INFO | MemInfo: { | 20260507 18:22:10.572 | INFO | "gpu" : " 17 GB", | 20260507 18:22:10.572 | INFO | "huge" : "", | 20260507 18:22:10.572 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:10.572 | INFO | "malloc" : " 14 GB", | 20260507 18:22:10.572 | INFO | "pinned" : " 29 GB" | 20260507 18:22:10.572 | INFO | } | 20260507 18:22:10.572 | INFO | Perf report for the last 5.13918 seconds: | 20260507 18:22:10.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:10.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:10.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:10.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:10.572 | INFO | ANALYSIS : 69.4%, 3.0, 107.4ms, 0 | 20260507 18:22:10.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:10.572 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 18:22:10.572 | INFO | PRE_HQ : 66.0%, 1.7, 58.7ms, 0 | 20260507 18:22:10.572 | INFO | BAZWRITER : 70.2%, 4.6, 165.8ms, 0 | 20260507 18:22:10.572 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 18:22:10.572 | INFO | Uploaded 36GiB to GPU in 2.51298s (14.3256GiB/s) | 20260507 18:22:10.572 | INFO | Downloaded 38GiB to GPU in 2.71526s (13.995GiB/s) | 20260507 18:22:15.711 | INFO | Analyzing chunk frames = [1614336,1614848) | 20260507 18:22:15.711 | INFO | MemInfo: { | 20260507 18:22:15.711 | INFO | "gpu" : " 17 GB", | 20260507 18:22:15.711 | INFO | "huge" : "", | 20260507 18:22:15.711 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:15.711 | INFO | "malloc" : " 14 GB", | 20260507 18:22:15.711 | INFO | "pinned" : " 29 GB" | 20260507 18:22:15.711 | INFO | } | 20260507 18:22:15.711 | INFO | Perf report for the last 5.13959 seconds: | 20260507 18:22:15.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:15.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.711 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 18:22:15.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.711 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:22:15.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:15.712 | INFO | Uploaded 36GiB to GPU in 2.44981s (14.695GiB/s) | 20260507 18:22:15.712 | INFO | Downloaded 34GiB to GPU in 2.45698s (13.8381GiB/s) | 20260507 18:22:20.850 | INFO | Analyzing chunk frames = [1614848,1615360) | 20260507 18:22:20.850 | INFO | MemInfo: { | 20260507 18:22:20.850 | INFO | "gpu" : " 17 GB", | 20260507 18:22:20.850 | INFO | "huge" : "", | 20260507 18:22:20.850 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:20.850 | INFO | "malloc" : " 14 GB", | 20260507 18:22:20.850 | INFO | "pinned" : " 29 GB" | 20260507 18:22:20.850 | INFO | } | 20260507 18:22:20.850 | INFO | Perf report for the last 5.13913 seconds: | 20260507 18:22:20.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:20.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.850 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 18:22:20.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.851 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:22:20.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:20.851 | INFO | Uploaded 36GiB to GPU in 2.49722s (14.4161GiB/s) | 20260507 18:22:20.851 | INFO | Downloaded 34GiB to GPU in 2.44959s (13.8799GiB/s) | 20260507 18:22:25.990 | INFO | Analyzing chunk frames = [1615360,1615872) | 20260507 18:22:25.990 | INFO | MemInfo: { | 20260507 18:22:25.990 | INFO | "gpu" : " 17 GB", | 20260507 18:22:25.990 | INFO | "huge" : "", | 20260507 18:22:25.990 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:25.990 | INFO | "malloc" : " 14 GB", | 20260507 18:22:25.990 | INFO | "pinned" : " 29 GB" | 20260507 18:22:25.990 | INFO | } | 20260507 18:22:25.990 | INFO | Perf report for the last 5.1398 seconds: | 20260507 18:22:25.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:25.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.990 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:22:25.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.990 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:22:25.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:25.991 | INFO | Uploaded 36GiB to GPU in 2.44181s (14.7432GiB/s) | 20260507 18:22:25.991 | INFO | Downloaded 34GiB to GPU in 2.37037s (14.3437GiB/s) | 20260507 18:22:31.129 | INFO | Analyzing chunk frames = [1615872,1616384) | 20260507 18:22:31.129 | INFO | MemInfo: { | 20260507 18:22:31.129 | INFO | "gpu" : " 17 GB", | 20260507 18:22:31.129 | INFO | "huge" : "", | 20260507 18:22:31.129 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:31.129 | INFO | "malloc" : " 14 GB", | 20260507 18:22:31.129 | INFO | "pinned" : " 29 GB" | 20260507 18:22:31.130 | INFO | } | 20260507 18:22:31.130 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:22:31.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:31.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 18:22:31.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:22:31.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:31.130 | INFO | Uploaded 36GiB to GPU in 2.29608s (15.6789GiB/s) | 20260507 18:22:31.130 | INFO | Downloaded 34GiB to GPU in 2.4919s (13.6442GiB/s) | 20260507 18:22:36.269 | INFO | Analyzing chunk frames = [1616384,1616896) | 20260507 18:22:36.269 | INFO | MemInfo: { | 20260507 18:22:36.269 | INFO | "gpu" : " 17 GB", | 20260507 18:22:36.269 | INFO | "huge" : "", | 20260507 18:22:36.269 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:36.269 | INFO | "malloc" : " 14 GB", | 20260507 18:22:36.269 | INFO | "pinned" : " 29 GB" | 20260507 18:22:36.269 | INFO | } | 20260507 18:22:36.269 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:22:36.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:36.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.270 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:22:36.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.270 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260507 18:22:36.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:36.270 | INFO | Uploaded 36GiB to GPU in 2.42375s (14.853GiB/s) | 20260507 18:22:36.270 | INFO | Downloaded 34GiB to GPU in 2.52614s (13.4593GiB/s) | 20260507 18:22:41.408 | INFO | Analyzing chunk frames = [1616896,1617408) | 20260507 18:22:41.408 | INFO | MemInfo: { | 20260507 18:22:41.408 | INFO | "gpu" : " 17 GB", | 20260507 18:22:41.408 | INFO | "huge" : "", | 20260507 18:22:41.408 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:41.408 | INFO | "malloc" : " 14 GB", | 20260507 18:22:41.408 | INFO | "pinned" : " 29 GB" | 20260507 18:22:41.408 | INFO | } | 20260507 18:22:41.408 | INFO | Perf report for the last 5.13864 seconds: | 20260507 18:22:41.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:41.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 18:22:41.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 18:22:41.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:41.408 | INFO | Uploaded 36GiB to GPU in 2.43805s (14.7659GiB/s) | 20260507 18:22:41.408 | INFO | Downloaded 34GiB to GPU in 2.45062s (13.874GiB/s) | 20260507 18:22:46.547 | INFO | Analyzing chunk frames = [1617408,1617920) | 20260507 18:22:46.547 | INFO | MemInfo: { | 20260507 18:22:46.547 | INFO | "gpu" : " 17 GB", | 20260507 18:22:46.547 | INFO | "huge" : "", | 20260507 18:22:46.547 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:46.547 | INFO | "malloc" : " 14 GB", | 20260507 18:22:46.547 | INFO | "pinned" : " 29 GB" | 20260507 18:22:46.547 | INFO | } | 20260507 18:22:46.548 | INFO | Perf report for the last 5.13974 seconds: | 20260507 18:22:46.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:46.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:22:46.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:22:46.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:46.548 | INFO | Uploaded 36GiB to GPU in 2.42709s (14.8326GiB/s) | 20260507 18:22:46.548 | INFO | Downloaded 34GiB to GPU in 2.4314s (13.9837GiB/s) | 20260507 18:22:46.645 | INFO | Starting a round of Baz IO | 20260507 18:22:50.141 | INFO | Populating RTMetrics report | 20260507 18:22:50.142 | INFO | RTMetrics report populated | 20260507 18:22:50.142 | INFO | Opening tmp RTMetrics json file | 20260507 18:22:50.142 | INFO | Serializing RTMetrics to json | 20260507 18:22:50.145 | INFO | Serialization done, swapping tmp file now... | 20260507 18:22:50.154 | INFO | RTMetrics done | 20260507 18:22:50.229 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.583 seconds | 20260507 18:22:50.229 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.70% / 0.02% / 53.29% | 20260507 18:22:50.229 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.98715% / 2.38419e-05% / 1.19209e-05% | 20260507 18:22:50.229 | INFO | Baz IO overall: | 20260507 18:22:50.229 | INFO | BytesWritten : 722613073758 | 20260507 18:22:50.229 | INFO | NumEvents : 350309254842 | 20260507 18:22:51.687 | INFO | Analyzing chunk frames = [1617920,1618432) | 20260507 18:22:51.687 | INFO | MemInfo: { | 20260507 18:22:51.687 | INFO | "gpu" : " 17 GB", | 20260507 18:22:51.687 | INFO | "huge" : "", | 20260507 18:22:51.687 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:51.687 | INFO | "malloc" : " 14 GB", | 20260507 18:22:51.687 | INFO | "pinned" : " 29 GB" | 20260507 18:22:51.687 | INFO | } | 20260507 18:22:51.687 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:22:51.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:51.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:51.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:51.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:51.687 | INFO | ANALYSIS : 69.2%, 2.9, 106.9ms, 0 | 20260507 18:22:51.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:51.687 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 18:22:51.687 | INFO | PRE_HQ : 66.2%, 1.7, 59.4ms, 0 | 20260507 18:22:51.687 | INFO | BAZWRITER : 69.2%, 4.0, 142.5ms, 0 | 20260507 18:22:51.687 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:22:51.687 | INFO | Uploaded 36GiB to GPU in 2.51858s (14.2938GiB/s) | 20260507 18:22:51.687 | INFO | Downloaded 38GiB to GPU in 2.71313s (14.006GiB/s) | 20260507 18:22:56.826 | INFO | Analyzing chunk frames = [1618432,1618944) | 20260507 18:22:56.826 | INFO | MemInfo: { | 20260507 18:22:56.826 | INFO | "gpu" : " 17 GB", | 20260507 18:22:56.826 | INFO | "huge" : "", | 20260507 18:22:56.826 | INFO | "huge_shared" : " 29 GB", | 20260507 18:22:56.826 | INFO | "malloc" : " 14 GB", | 20260507 18:22:56.826 | INFO | "pinned" : " 29 GB" | 20260507 18:22:56.826 | INFO | } | 20260507 18:22:56.826 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:22:56.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:22:56.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:22:56.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:22:56.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:22:56.826 | INFO | Uploaded 36GiB to GPU in 2.53004s (14.229GiB/s) | 20260507 18:22:56.826 | INFO | Downloaded 34GiB to GPU in 2.45031s (13.8758GiB/s) | 20260507 18:23:01.965 | INFO | Analyzing chunk frames = [1618944,1619456) | 20260507 18:23:01.965 | INFO | MemInfo: { | 20260507 18:23:01.965 | INFO | "gpu" : " 17 GB", | 20260507 18:23:01.965 | INFO | "huge" : "", | 20260507 18:23:01.965 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:01.965 | INFO | "malloc" : " 14 GB", | 20260507 18:23:01.965 | INFO | "pinned" : " 29 GB" | 20260507 18:23:01.965 | INFO | } | 20260507 18:23:01.965 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:23:01.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:01.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.965 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:23:01.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.965 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:23:01.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:01.966 | INFO | Uploaded 36GiB to GPU in 2.54918s (14.1222GiB/s) | 20260507 18:23:01.966 | INFO | Downloaded 34GiB to GPU in 2.42628s (14.0132GiB/s) | 20260507 18:23:07.104 | INFO | Analyzing chunk frames = [1619456,1619968) | 20260507 18:23:07.104 | INFO | MemInfo: { | 20260507 18:23:07.104 | INFO | "gpu" : " 17 GB", | 20260507 18:23:07.104 | INFO | "huge" : "", | 20260507 18:23:07.104 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:07.104 | INFO | "malloc" : " 14 GB", | 20260507 18:23:07.104 | INFO | "pinned" : " 29 GB" | 20260507 18:23:07.104 | INFO | } | 20260507 18:23:07.104 | INFO | Perf report for the last 5.1391 seconds: | 20260507 18:23:07.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:07.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.105 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:23:07.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.105 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:23:07.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:07.105 | INFO | Uploaded 36GiB to GPU in 2.49175s (14.4477GiB/s) | 20260507 18:23:07.105 | INFO | Downloaded 34GiB to GPU in 2.35719s (14.4239GiB/s) | 20260507 18:23:12.244 | INFO | Analyzing chunk frames = [1619968,1620480) | 20260507 18:23:12.244 | INFO | MemInfo: { | 20260507 18:23:12.244 | INFO | "gpu" : " 17 GB", | 20260507 18:23:12.244 | INFO | "huge" : "", | 20260507 18:23:12.244 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:12.244 | INFO | "malloc" : " 14 GB", | 20260507 18:23:12.244 | INFO | "pinned" : " 29 GB" | 20260507 18:23:12.244 | INFO | } | 20260507 18:23:12.244 | INFO | Perf report for the last 5.13989 seconds: | 20260507 18:23:12.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:12.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.244 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:23:12.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.244 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 18:23:12.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:12.245 | INFO | Uploaded 36GiB to GPU in 2.38994s (15.0632GiB/s) | 20260507 18:23:12.245 | INFO | Downloaded 34GiB to GPU in 2.48146s (13.7016GiB/s) | 20260507 18:23:17.383 | INFO | Analyzing chunk frames = [1620480,1620992) | 20260507 18:23:17.383 | INFO | MemInfo: { | 20260507 18:23:17.383 | INFO | "gpu" : " 17 GB", | 20260507 18:23:17.383 | INFO | "huge" : "", | 20260507 18:23:17.383 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:17.383 | INFO | "malloc" : " 14 GB", | 20260507 18:23:17.383 | INFO | "pinned" : " 29 GB" | 20260507 18:23:17.383 | INFO | } | 20260507 18:23:17.383 | INFO | Perf report for the last 5.1387 seconds: | 20260507 18:23:17.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:17.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:23:17.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 18:23:17.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:17.383 | INFO | Uploaded 36GiB to GPU in 2.44691s (14.7125GiB/s) | 20260507 18:23:17.384 | INFO | Downloaded 34GiB to GPU in 2.51228s (13.5335GiB/s) | 20260507 18:23:22.522 | INFO | Analyzing chunk frames = [1620992,1621504) | 20260507 18:23:22.522 | INFO | MemInfo: { | 20260507 18:23:22.522 | INFO | "gpu" : " 17 GB", | 20260507 18:23:22.522 | INFO | "huge" : "", | 20260507 18:23:22.522 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:22.522 | INFO | "malloc" : " 14 GB", | 20260507 18:23:22.522 | INFO | "pinned" : " 29 GB" | 20260507 18:23:22.522 | INFO | } | 20260507 18:23:22.522 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:23:22.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:22.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.522 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:23:22.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.523 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:23:22.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:22.523 | INFO | Uploaded 36GiB to GPU in 2.48258s (14.5011GiB/s) | 20260507 18:23:22.523 | INFO | Downloaded 34GiB to GPU in 2.4321s (13.9797GiB/s) | 20260507 18:23:27.662 | INFO | Analyzing chunk frames = [1621504,1622016) | 20260507 18:23:27.662 | INFO | MemInfo: { | 20260507 18:23:27.662 | INFO | "gpu" : " 17 GB", | 20260507 18:23:27.662 | INFO | "huge" : "", | 20260507 18:23:27.662 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:27.662 | INFO | "malloc" : " 14 GB", | 20260507 18:23:27.662 | INFO | "pinned" : " 29 GB" | 20260507 18:23:27.662 | INFO | } | 20260507 18:23:27.662 | INFO | Perf report for the last 5.13925 seconds: | 20260507 18:23:27.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:27.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:23:27.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:23:27.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:27.662 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:23:27.662 | INFO | Uploaded 36GiB to GPU in 2.48528s (14.4853GiB/s) | 20260507 18:23:27.662 | INFO | Total mem usage: 16.4 GiB | 20260507 18:23:27.662 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:23:27.662 | INFO | Downloaded 34GiB to GPU in 2.42613s (14.0141GiB/s) | 20260507 18:23:27.662 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.48%) | 20260507 18:23:27.662 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:23:27.662 | INFO | High Water Breakdown: | 20260507 18:23:27.662 | INFO | 60.4% : 9.4 GiB ( 94.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:23:27.662 | INFO | 39.4% : 6.13 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:23:27.662 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:23:27.662 | INFO | ------------------------------------------ | 20260507 18:23:27.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:23:27.663 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:23:27.663 | INFO | Total mem usage: 26.5 GiB | 20260507 18:23:27.663 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:23:27.663 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:23:27.663 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:23:27.663 | INFO | High Water Breakdown: | 20260507 18:23:27.663 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:23:27.663 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:23:27.663 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:23:27.663 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:23:27.663 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:23:27.663 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:23:27.663 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:23:27.663 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:23:27.663 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:23:27.663 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:23:27.663 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:23:27.663 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:23:27.663 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:23:27.663 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:23:27.663 | INFO | ------------------------------------------ | 20260507 18:23:27.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:23:27.663 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:23:27.663 | INFO | Total mem usage: 27.8 GiB | 20260507 18:23:27.663 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:23:27.663 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:23:27.663 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:23:27.663 | INFO | High Water Breakdown: | 20260507 18:23:27.663 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:23:27.663 | INFO | ------------------------------------------ | 20260507 18:23:27.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:23:27.663 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:23:27.663 | INFO | Total mem usage: 15.7 GiB | 20260507 18:23:27.663 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:23:27.663 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:23:27.663 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:23:27.663 | INFO | High Water Breakdown: | 20260507 18:23:27.663 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:23:27.663 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:23:27.663 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:23:27.663 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:23:27.663 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:23:27.663 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:23:27.663 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:23:27.663 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:23:27.663 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:23:27.663 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:23:27.663 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:23:27.663 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:23:27.663 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:23:27.663 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:23:27.663 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:23:27.663 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:23:27.663 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:23:27.663 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:23:27.663 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:23:27.663 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:23:27.663 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:23:27.663 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:23:27.663 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:23:27.663 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:23:27.663 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:23:27.663 | INFO | ------------------------------------------ | 20260507 18:23:27.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:23:27.664 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:23:27.664 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:23:27.664 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:23:27.664 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.61e+01, 1.44e+01 / 2.32e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 0.29%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.25e-02 / 4.75e-03 | 20260507 18:23:27.664 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.46e+00, 6.22e-01 / 2.80e-01 | 20260507 18:23:27.664 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.18e-01 / 1.06e-01 | 20260507 18:23:27.664 | INFO | PerfReport: 15.65%, DME -- 4.19e+01 / 4.46e+01, 3.87e+00 / 2.18e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.92e+00 / 2.35e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.82e+00, 4.34e-01 / 2.66e-01 | 20260507 18:23:27.664 | INFO | PerfReport: 3.71%, Metrics -- 1.24e+00 / 1.42e+00, 3.13e-01 / 1.72e-01 | 20260507 18:23:27.664 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.19e-01, 6.37e-02 / 3.19e-02 | 20260507 18:23:27.664 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:23:27.664 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:23:27.664 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:23:27.664 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 23.94%, Upload -- 2.56e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 24.13%, Download -- 2.58e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 31.24%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 1.62%, LockUpload -- 1.74e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 2.50%, LockDownload -- 2.68e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 16.56%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:23:27.664 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 18:23:27.758 | INFO | Starting a round of Baz IO | 20260507 18:23:31.259 | INFO | Populating RTMetrics report | 20260507 18:23:31.259 | INFO | RTMetrics report populated | 20260507 18:23:31.259 | INFO | Opening tmp RTMetrics json file | 20260507 18:23:31.299 | INFO | Serializing RTMetrics to json | 20260507 18:23:31.338 | INFO | Serialization done, swapping tmp file now... | 20260507 18:23:31.339 | INFO | RTMetrics done | 20260507 18:23:31.361 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.603 seconds | 20260507 18:23:31.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.68% / 0.02% / 53.30% | 20260507 18:23:31.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.99411% / 2.78155e-05% / 3.17891e-05% | 20260507 18:23:31.361 | INFO | Baz IO overall: | 20260507 18:23:31.362 | INFO | BytesWritten : 724555509466 | 20260507 18:23:31.362 | INFO | NumEvents : 351251913466 | 20260507 18:23:32.801 | INFO | Analyzing chunk frames = [1622016,1622528) | 20260507 18:23:32.801 | INFO | MemInfo: { | 20260507 18:23:32.801 | INFO | "gpu" : " 17 GB", | 20260507 18:23:32.801 | INFO | "huge" : "", | 20260507 18:23:32.801 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:32.801 | INFO | "malloc" : " 14 GB", | 20260507 18:23:32.801 | INFO | "pinned" : " 29 GB" | 20260507 18:23:32.801 | INFO | } | 20260507 18:23:32.801 | INFO | Perf report for the last 5.13957 seconds: | 20260507 18:23:32.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:32.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:32.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:32.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:32.801 | INFO | ANALYSIS : 69.5%, 3.0, 107.6ms, 0 | 20260507 18:23:32.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:32.801 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 18:23:32.801 | INFO | PRE_HQ : 66.4%, 1.7, 59.2ms, 0 | 20260507 18:23:32.801 | INFO | BAZWRITER : 69.6%, 3.6, 127.7ms, 0 | 20260507 18:23:32.801 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 18:23:32.802 | INFO | Uploaded 36GiB to GPU in 2.53212s (14.2173GiB/s) | 20260507 18:23:32.802 | INFO | Downloaded 38GiB to GPU in 2.72138s (13.9635GiB/s) | 20260507 18:23:37.940 | INFO | Analyzing chunk frames = [1622528,1623040) | 20260507 18:23:37.940 | INFO | MemInfo: { | 20260507 18:23:37.940 | INFO | "gpu" : " 17 GB", | 20260507 18:23:37.940 | INFO | "huge" : "", | 20260507 18:23:37.940 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:37.940 | INFO | "malloc" : " 14 GB", | 20260507 18:23:37.940 | INFO | "pinned" : " 29 GB" | 20260507 18:23:37.940 | INFO | } | 20260507 18:23:37.940 | INFO | Perf report for the last 5.13914 seconds: | 20260507 18:23:37.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:37.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.940 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:23:37.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.940 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:23:37.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:37.941 | INFO | Uploaded 36GiB to GPU in 2.46266s (14.6184GiB/s) | 20260507 18:23:37.941 | INFO | Downloaded 34GiB to GPU in 2.47183s (13.755GiB/s) | 20260507 18:23:43.080 | INFO | Analyzing chunk frames = [1623040,1623552) | 20260507 18:23:43.080 | INFO | MemInfo: { | 20260507 18:23:43.080 | INFO | "gpu" : " 17 GB", | 20260507 18:23:43.080 | INFO | "huge" : "", | 20260507 18:23:43.080 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:43.080 | INFO | "malloc" : " 14 GB", | 20260507 18:23:43.080 | INFO | "pinned" : " 29 GB" | 20260507 18:23:43.080 | INFO | } | 20260507 18:23:43.080 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:23:43.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:43.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.080 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:23:43.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.080 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:23:43.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:43.081 | INFO | Uploaded 36GiB to GPU in 2.52117s (14.2791GiB/s) | 20260507 18:23:43.081 | INFO | Downloaded 34GiB to GPU in 2.438s (13.9458GiB/s) | 20260507 18:23:48.219 | INFO | Analyzing chunk frames = [1623552,1624064) | 20260507 18:23:48.219 | INFO | MemInfo: { | 20260507 18:23:48.219 | INFO | "gpu" : " 17 GB", | 20260507 18:23:48.219 | INFO | "huge" : "", | 20260507 18:23:48.219 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:48.219 | INFO | "malloc" : " 14 GB", | 20260507 18:23:48.219 | INFO | "pinned" : " 29 GB" | 20260507 18:23:48.219 | INFO | } | 20260507 18:23:48.219 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:23:48.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:48.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.219 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:23:48.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.219 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:23:48.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:48.220 | INFO | Uploaded 36GiB to GPU in 2.45231s (14.6801GiB/s) | 20260507 18:23:48.220 | INFO | Downloaded 34GiB to GPU in 2.37734s (14.3017GiB/s) | 20260507 18:23:53.358 | INFO | Analyzing chunk frames = [1624064,1624576) | 20260507 18:23:53.358 | INFO | MemInfo: { | 20260507 18:23:53.358 | INFO | "gpu" : " 17 GB", | 20260507 18:23:53.358 | INFO | "huge" : "", | 20260507 18:23:53.358 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:53.358 | INFO | "malloc" : " 14 GB", | 20260507 18:23:53.358 | INFO | "pinned" : " 29 GB" | 20260507 18:23:53.358 | INFO | } | 20260507 18:23:53.358 | INFO | Perf report for the last 5.13925 seconds: | 20260507 18:23:53.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:53.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.359 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:23:53.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.359 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:23:53.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:53.359 | INFO | Uploaded 36GiB to GPU in 2.30711s (15.6039GiB/s) | 20260507 18:23:53.359 | INFO | Downloaded 34GiB to GPU in 2.50611s (13.5668GiB/s) | 20260507 18:23:58.497 | INFO | Analyzing chunk frames = [1624576,1625088) | 20260507 18:23:58.497 | INFO | MemInfo: { | 20260507 18:23:58.497 | INFO | "gpu" : " 17 GB", | 20260507 18:23:58.497 | INFO | "huge" : "", | 20260507 18:23:58.497 | INFO | "huge_shared" : " 29 GB", | 20260507 18:23:58.497 | INFO | "malloc" : " 14 GB", | 20260507 18:23:58.497 | INFO | "pinned" : " 29 GB" | 20260507 18:23:58.498 | INFO | } | 20260507 18:23:58.498 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:23:58.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:23:58.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:23:58.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:23:58.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:23:58.498 | INFO | Uploaded 36GiB to GPU in 2.38027s (15.1243GiB/s) | 20260507 18:23:58.498 | INFO | Downloaded 34GiB to GPU in 2.50718s (13.5611GiB/s) | 20260507 18:24:03.637 | INFO | Analyzing chunk frames = [1625088,1625600) | 20260507 18:24:03.637 | INFO | MemInfo: { | 20260507 18:24:03.637 | INFO | "gpu" : " 17 GB", | 20260507 18:24:03.637 | INFO | "huge" : "", | 20260507 18:24:03.637 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:03.637 | INFO | "malloc" : " 14 GB", | 20260507 18:24:03.637 | INFO | "pinned" : " 29 GB" | 20260507 18:24:03.637 | INFO | } | 20260507 18:24:03.637 | INFO | Perf report for the last 5.13938 seconds: | 20260507 18:24:03.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:03.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.637 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:24:03.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.637 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:24:03.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:03.638 | INFO | Uploaded 36GiB to GPU in 2.50364s (14.3791GiB/s) | 20260507 18:24:03.638 | INFO | Downloaded 34GiB to GPU in 2.44004s (13.9342GiB/s) | 20260507 18:24:08.776 | INFO | Analyzing chunk frames = [1625600,1626112) | 20260507 18:24:08.776 | INFO | MemInfo: { | 20260507 18:24:08.776 | INFO | "gpu" : " 17 GB", | 20260507 18:24:08.776 | INFO | "huge" : "", | 20260507 18:24:08.776 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:08.776 | INFO | "malloc" : " 14 GB", | 20260507 18:24:08.776 | INFO | "pinned" : " 29 GB" | 20260507 18:24:08.777 | INFO | } | 20260507 18:24:08.777 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:24:08.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:08.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:24:08.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:24:08.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:08.777 | INFO | Uploaded 36GiB to GPU in 2.47424s (14.5499GiB/s) | 20260507 18:24:08.777 | INFO | Downloaded 34GiB to GPU in 2.42198s (14.0381GiB/s) | 20260507 18:24:08.874 | INFO | Starting a round of Baz IO | 20260507 18:24:12.377 | INFO | Populating RTMetrics report | 20260507 18:24:12.377 | INFO | RTMetrics report populated | 20260507 18:24:12.377 | INFO | Opening tmp RTMetrics json file | 20260507 18:24:12.378 | INFO | Serializing RTMetrics to json | 20260507 18:24:12.379 | INFO | Serialization done, swapping tmp file now... | 20260507 18:24:12.380 | INFO | RTMetrics done | 20260507 18:24:12.508 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.633 seconds | 20260507 18:24:12.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.67% / 0.02% / 53.32% | 20260507 18:24:12.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.99425% / 1.19209e-05% / 4.76837e-05% | 20260507 18:24:12.508 | INFO | Baz IO overall: | 20260507 18:24:12.508 | INFO | BytesWritten : 726497922064 | 20260507 18:24:12.508 | INFO | NumEvents : 352195105896 | 20260507 18:24:13.916 | INFO | Analyzing chunk frames = [1626112,1626624) | 20260507 18:24:13.916 | INFO | MemInfo: { | 20260507 18:24:13.916 | INFO | "gpu" : " 17 GB", | 20260507 18:24:13.916 | INFO | "huge" : "", | 20260507 18:24:13.916 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:13.916 | INFO | "malloc" : " 14 GB", | 20260507 18:24:13.916 | INFO | "pinned" : " 29 GB" | 20260507 18:24:13.916 | INFO | } | 20260507 18:24:13.916 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:24:13.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:13.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:13.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:13.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:13.916 | INFO | ANALYSIS : 69.3%, 3.0, 107.3ms, 0 | 20260507 18:24:13.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:13.916 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 18:24:13.916 | INFO | PRE_HQ : 65.9%, 1.7, 59.1ms, 0 | 20260507 18:24:13.916 | INFO | BAZWRITER : 70.3%, 4.0, 144.4ms, 0 | 20260507 18:24:13.916 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 18:24:13.916 | INFO | Uploaded 36GiB to GPU in 2.50872s (14.35GiB/s) | 20260507 18:24:13.916 | INFO | Downloaded 38GiB to GPU in 2.69655s (14.0921GiB/s) | 20260507 18:24:19.055 | INFO | Analyzing chunk frames = [1626624,1627136) | 20260507 18:24:19.055 | INFO | MemInfo: { | 20260507 18:24:19.055 | INFO | "gpu" : " 17 GB", | 20260507 18:24:19.055 | INFO | "huge" : "", | 20260507 18:24:19.055 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:19.055 | INFO | "malloc" : " 14 GB", | 20260507 18:24:19.055 | INFO | "pinned" : " 29 GB" | 20260507 18:24:19.055 | INFO | } | 20260507 18:24:19.055 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:24:19.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:19.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.055 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:24:19.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.056 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 18:24:19.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:19.056 | INFO | Uploaded 36GiB to GPU in 2.52866s (14.2368GiB/s) | 20260507 18:24:19.056 | INFO | Downloaded 34GiB to GPU in 2.45778s (13.8336GiB/s) | 20260507 18:24:24.194 | INFO | Analyzing chunk frames = [1627136,1627648) | 20260507 18:24:24.194 | INFO | MemInfo: { | 20260507 18:24:24.194 | INFO | "gpu" : " 17 GB", | 20260507 18:24:24.194 | INFO | "huge" : "", | 20260507 18:24:24.194 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:24.194 | INFO | "malloc" : " 14 GB", | 20260507 18:24:24.194 | INFO | "pinned" : " 29 GB" | 20260507 18:24:24.194 | INFO | } | 20260507 18:24:24.195 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:24:24.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:24.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:24:24.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:24:24.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:24.195 | INFO | Uploaded 36GiB to GPU in 2.54691s (14.1348GiB/s) | 20260507 18:24:24.195 | INFO | Downloaded 34GiB to GPU in 2.4413s (13.927GiB/s) | 20260507 18:24:29.334 | INFO | Analyzing chunk frames = [1627648,1628160) | 20260507 18:24:29.334 | INFO | MemInfo: { | 20260507 18:24:29.334 | INFO | "gpu" : " 17 GB", | 20260507 18:24:29.334 | INFO | "huge" : "", | 20260507 18:24:29.334 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:29.334 | INFO | "malloc" : " 14 GB", | 20260507 18:24:29.334 | INFO | "pinned" : " 29 GB" | 20260507 18:24:29.334 | INFO | } | 20260507 18:24:29.334 | INFO | Perf report for the last 5.13917 seconds: | 20260507 18:24:29.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:29.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.334 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:24:29.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.334 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:24:29.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:29.335 | INFO | Uploaded 36GiB to GPU in 2.4819s (14.505GiB/s) | 20260507 18:24:29.335 | INFO | Downloaded 34GiB to GPU in 2.35613s (14.4305GiB/s) | 20260507 18:24:34.473 | INFO | Analyzing chunk frames = [1628160,1628672) | 20260507 18:24:34.473 | INFO | MemInfo: { | 20260507 18:24:34.473 | INFO | "gpu" : " 17 GB", | 20260507 18:24:34.473 | INFO | "huge" : "", | 20260507 18:24:34.473 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:34.473 | INFO | "malloc" : " 14 GB", | 20260507 18:24:34.473 | INFO | "pinned" : " 29 GB" | 20260507 18:24:34.473 | INFO | } | 20260507 18:24:34.474 | INFO | Perf report for the last 5.13975 seconds: | 20260507 18:24:34.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:34.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:24:34.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:24:34.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:34.474 | INFO | Uploaded 36GiB to GPU in 2.31694s (15.5377GiB/s) | 20260507 18:24:34.474 | INFO | Downloaded 34GiB to GPU in 2.49008s (13.6542GiB/s) | 20260507 18:24:39.613 | INFO | Analyzing chunk frames = [1628672,1629184) | 20260507 18:24:39.613 | INFO | MemInfo: { | 20260507 18:24:39.613 | INFO | "gpu" : " 17 GB", | 20260507 18:24:39.613 | INFO | "huge" : "", | 20260507 18:24:39.613 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:39.613 | INFO | "malloc" : " 14 GB", | 20260507 18:24:39.613 | INFO | "pinned" : " 29 GB" | 20260507 18:24:39.613 | INFO | } | 20260507 18:24:39.613 | INFO | Perf report for the last 5.13908 seconds: | 20260507 18:24:39.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:39.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:24:39.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:24:39.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:39.613 | INFO | Uploaded 36GiB to GPU in 2.39792s (15.013GiB/s) | 20260507 18:24:39.613 | INFO | Downloaded 34GiB to GPU in 2.52543s (13.4631GiB/s) | 20260507 18:24:44.752 | INFO | Analyzing chunk frames = [1629184,1629696) | 20260507 18:24:44.752 | INFO | MemInfo: { | 20260507 18:24:44.752 | INFO | "gpu" : " 17 GB", | 20260507 18:24:44.752 | INFO | "huge" : "", | 20260507 18:24:44.752 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:44.752 | INFO | "malloc" : " 14 GB", | 20260507 18:24:44.752 | INFO | "pinned" : " 29 GB" | 20260507 18:24:44.752 | INFO | } | 20260507 18:24:44.752 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:24:44.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:44.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 18:24:44.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:24:44.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:44.752 | INFO | Uploaded 36GiB to GPU in 2.43495s (14.7847GiB/s) | 20260507 18:24:44.752 | INFO | Downloaded 34GiB to GPU in 2.44722s (13.8933GiB/s) | 20260507 18:24:49.891 | INFO | Analyzing chunk frames = [1629696,1630208) | 20260507 18:24:49.891 | INFO | MemInfo: { | 20260507 18:24:49.891 | INFO | "gpu" : " 17 GB", | 20260507 18:24:49.891 | INFO | "huge" : "", | 20260507 18:24:49.891 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:49.891 | INFO | "malloc" : " 14 GB", | 20260507 18:24:49.891 | INFO | "pinned" : " 29 GB" | 20260507 18:24:49.891 | INFO | } | 20260507 18:24:49.891 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:24:49.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:49.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.891 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 18:24:49.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.891 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:24:49.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:49.892 | INFO | Uploaded 36GiB to GPU in 2.51897s (14.2915GiB/s) | 20260507 18:24:49.892 | INFO | Downloaded 34GiB to GPU in 2.51378s (13.5254GiB/s) | 20260507 18:24:49.892 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:24:49.892 | INFO | Total mem usage: 16.4 GiB | 20260507 18:24:49.892 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:24:49.892 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.57%) | 20260507 18:24:49.892 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:24:49.892 | INFO | High Water Breakdown: | 20260507 18:24:49.892 | INFO | 60% : 9.06 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:24:49.892 | INFO | 39.9% : 6.03 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:24:49.892 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:24:49.892 | INFO | ------------------------------------------ | 20260507 18:24:49.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:24:49.892 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:24:49.892 | INFO | Total mem usage: 26.5 GiB | 20260507 18:24:49.892 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:24:49.892 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:24:49.892 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:24:49.892 | INFO | High Water Breakdown: | 20260507 18:24:49.892 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:24:49.892 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:24:49.892 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:24:49.892 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:24:49.892 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:24:49.892 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:24:49.892 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:24:49.892 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:24:49.892 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:24:49.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:24:49.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:24:49.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:24:49.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:24:49.892 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:24:49.892 | INFO | ------------------------------------------ | 20260507 18:24:49.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:24:49.892 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:24:49.892 | INFO | Total mem usage: 27.8 GiB | 20260507 18:24:49.892 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:24:49.892 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:24:49.892 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:24:49.892 | INFO | High Water Breakdown: | 20260507 18:24:49.892 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:24:49.892 | INFO | ------------------------------------------ | 20260507 18:24:49.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:24:49.892 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:24:49.892 | INFO | Total mem usage: 15.7 GiB | 20260507 18:24:49.892 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:24:49.892 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:24:49.892 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 18:24:49.892 | INFO | High Water Breakdown: | 20260507 18:24:49.892 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:24:49.892 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:24:49.892 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:24:49.892 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:24:49.893 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:24:49.893 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:24:49.893 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:24:49.893 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:24:49.893 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:24:49.893 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:24:49.893 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:24:49.893 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:24:49.893 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:24:49.893 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:24:49.893 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:24:49.893 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:24:49.893 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:24:49.893 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:24:49.893 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:24:49.893 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:24:49.893 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:24:49.893 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:24:49.893 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:24:49.893 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:24:49.893 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:24:49.893 | INFO | ------------------------------------------ | 20260507 18:24:49.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:24:49.893 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:24:49.893 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:24:49.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:24:49.893 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.63e+01, 1.43e+01 / 2.17e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.17e-02 / 5.38e-03 | 20260507 18:24:49.893 | INFO | PerfReport: 12.07%, Baseline -- 4.03e+00 / 4.49e+00, 6.15e-01 / 2.40e-01 | 20260507 18:24:49.893 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.15e-01 / 1.24e-01 | 20260507 18:24:49.893 | INFO | PerfReport: 15.69%, DME -- 4.19e+01 / 4.48e+01, 4.24e+00 / 2.28e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.87e+00 / 2.07e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.34e-01 / 2.32e-01 | 20260507 18:24:49.893 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.44e+00, 3.17e-01 / 1.84e-01 | 20260507 18:24:49.893 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.90e-01 / 4.16e-01, 6.31e-02 / 2.96e-02 | 20260507 18:24:49.893 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:24:49.893 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:24:49.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:24:49.893 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 23.91%, Upload -- 2.56e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 24.18%, Download -- 2.59e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 31.16%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 1.67%, LockUpload -- 1.79e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 2.55%, LockDownload -- 2.73e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 16.51%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:24:49.893 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260507 18:24:50.005 | INFO | Starting a round of Baz IO | 20260507 18:24:53.518 | INFO | Populating RTMetrics report | 20260507 18:24:53.518 | INFO | RTMetrics report populated | 20260507 18:24:53.518 | INFO | Opening tmp RTMetrics json file | 20260507 18:24:53.519 | INFO | Serializing RTMetrics to json | 20260507 18:24:53.520 | INFO | Serialization done, swapping tmp file now... | 20260507 18:24:53.520 | INFO | RTMetrics done | 20260507 18:24:53.592 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.587 seconds | 20260507 18:24:53.592 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.65% / 0.02% / 53.33% | 20260507 18:24:53.592 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.97995% / 1.98682e-05% / 5.96046e-05% | 20260507 18:24:53.592 | INFO | Baz IO overall: | 20260507 18:24:53.592 | INFO | BytesWritten : 728437315983 | 20260507 18:24:53.592 | INFO | NumEvents : 353136722740 | 20260507 18:24:55.031 | INFO | Analyzing chunk frames = [1630208,1630720) | 20260507 18:24:55.031 | INFO | MemInfo: { | 20260507 18:24:55.031 | INFO | "gpu" : " 17 GB", | 20260507 18:24:55.031 | INFO | "huge" : "", | 20260507 18:24:55.031 | INFO | "huge_shared" : " 29 GB", | 20260507 18:24:55.031 | INFO | "malloc" : " 14 GB", | 20260507 18:24:55.031 | INFO | "pinned" : " 29 GB" | 20260507 18:24:55.031 | INFO | } | 20260507 18:24:55.031 | INFO | Perf report for the last 5.13963 seconds: | 20260507 18:24:55.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:24:55.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:55.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:55.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:55.031 | INFO | ANALYSIS : 69.9%, 2.9, 105.7ms, 0 | 20260507 18:24:55.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:24:55.031 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 18:24:55.031 | INFO | PRE_HQ : 68.6%, 1.8, 64.8ms, 0 | 20260507 18:24:55.031 | INFO | BAZWRITER : 69.4%, 4.0, 141.2ms, 0 | 20260507 18:24:55.031 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:24:55.032 | INFO | Uploaded 36GiB to GPU in 2.52253s (14.2714GiB/s) | 20260507 18:24:55.032 | INFO | Downloaded 38GiB to GPU in 2.70683s (14.0386GiB/s) | 20260507 18:25:00.170 | INFO | Analyzing chunk frames = [1630720,1631232) | 20260507 18:25:00.170 | INFO | MemInfo: { | 20260507 18:25:00.170 | INFO | "gpu" : " 17 GB", | 20260507 18:25:00.170 | INFO | "huge" : "", | 20260507 18:25:00.170 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:00.170 | INFO | "malloc" : " 14 GB", | 20260507 18:25:00.170 | INFO | "pinned" : " 29 GB" | 20260507 18:25:00.170 | INFO | } | 20260507 18:25:00.170 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:25:00.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:00.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.170 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:25:00.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.170 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:25:00.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:00.171 | INFO | Uploaded 36GiB to GPU in 2.51305s (14.3252GiB/s) | 20260507 18:25:00.171 | INFO | Downloaded 34GiB to GPU in 2.4508s (13.873GiB/s) | 20260507 18:25:05.309 | INFO | Analyzing chunk frames = [1631232,1631744) | 20260507 18:25:05.309 | INFO | MemInfo: { | 20260507 18:25:05.309 | INFO | "gpu" : " 17 GB", | 20260507 18:25:05.309 | INFO | "huge" : "", | 20260507 18:25:05.309 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:05.309 | INFO | "malloc" : " 14 GB", | 20260507 18:25:05.309 | INFO | "pinned" : " 29 GB" | 20260507 18:25:05.309 | INFO | } | 20260507 18:25:05.309 | INFO | Perf report for the last 5.13904 seconds: | 20260507 18:25:05.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:05.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.309 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 18:25:05.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.309 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:25:05.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:05.310 | INFO | Uploaded 36GiB to GPU in 2.51781s (14.2981GiB/s) | 20260507 18:25:05.310 | INFO | Downloaded 34GiB to GPU in 2.44435s (13.9096GiB/s) | 20260507 18:25:10.449 | INFO | Analyzing chunk frames = [1631744,1632256) | 20260507 18:25:10.449 | INFO | MemInfo: { | 20260507 18:25:10.449 | INFO | "gpu" : " 17 GB", | 20260507 18:25:10.449 | INFO | "huge" : "", | 20260507 18:25:10.449 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:10.449 | INFO | "malloc" : " 14 GB", | 20260507 18:25:10.449 | INFO | "pinned" : " 29 GB" | 20260507 18:25:10.449 | INFO | } | 20260507 18:25:10.449 | INFO | Perf report for the last 5.1395 seconds: | 20260507 18:25:10.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:10.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:25:10.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:25:10.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:10.449 | INFO | Uploaded 36GiB to GPU in 2.47924s (14.5206GiB/s) | 20260507 18:25:10.449 | INFO | Downloaded 34GiB to GPU in 2.35022s (14.4667GiB/s) | 20260507 18:25:15.589 | INFO | Analyzing chunk frames = [1632256,1632768) | 20260507 18:25:15.589 | INFO | MemInfo: { | 20260507 18:25:15.589 | INFO | "gpu" : " 17 GB", | 20260507 18:25:15.589 | INFO | "huge" : "", | 20260507 18:25:15.589 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:15.589 | INFO | "malloc" : " 14 GB", | 20260507 18:25:15.589 | INFO | "pinned" : " 29 GB" | 20260507 18:25:15.589 | INFO | } | 20260507 18:25:15.589 | INFO | Perf report for the last 5.14035 seconds: | 20260507 18:25:15.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:15.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:25:15.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:25:15.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:15.589 | INFO | Uploaded 36GiB to GPU in 2.34935s (15.3234GiB/s) | 20260507 18:25:15.590 | INFO | Downloaded 34GiB to GPU in 2.48888s (13.6607GiB/s) | 20260507 18:25:20.728 | INFO | Analyzing chunk frames = [1632768,1633280) | 20260507 18:25:20.728 | INFO | MemInfo: { | 20260507 18:25:20.728 | INFO | "gpu" : " 17 GB", | 20260507 18:25:20.728 | INFO | "huge" : "", | 20260507 18:25:20.728 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:20.728 | INFO | "malloc" : " 14 GB", | 20260507 18:25:20.728 | INFO | "pinned" : " 29 GB" | 20260507 18:25:20.728 | INFO | } | 20260507 18:25:20.728 | INFO | Perf report for the last 5.13869 seconds: | 20260507 18:25:20.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:20.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:25:20.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:25:20.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:20.728 | INFO | Uploaded 36GiB to GPU in 2.50941s (14.346GiB/s) | 20260507 18:25:20.728 | INFO | Downloaded 34GiB to GPU in 2.50212s (13.5885GiB/s) | 20260507 18:25:25.866 | INFO | Analyzing chunk frames = [1633280,1633792) | 20260507 18:25:25.867 | INFO | MemInfo: { | 20260507 18:25:25.867 | INFO | "gpu" : " 17 GB", | 20260507 18:25:25.867 | INFO | "huge" : "", | 20260507 18:25:25.867 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:25.867 | INFO | "malloc" : " 14 GB", | 20260507 18:25:25.867 | INFO | "pinned" : " 29 GB" | 20260507 18:25:25.867 | INFO | } | 20260507 18:25:25.867 | INFO | Perf report for the last 5.13883 seconds: | 20260507 18:25:25.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:25.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:25:25.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 18:25:25.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:25.867 | INFO | Uploaded 36GiB to GPU in 2.51839s (14.2948GiB/s) | 20260507 18:25:25.867 | INFO | Downloaded 34GiB to GPU in 2.42842s (14.0009GiB/s) | 20260507 18:25:31.006 | INFO | Analyzing chunk frames = [1633792,1634304) | 20260507 18:25:31.006 | INFO | MemInfo: { | 20260507 18:25:31.006 | INFO | "gpu" : " 17 GB", | 20260507 18:25:31.006 | INFO | "huge" : "", | 20260507 18:25:31.006 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:31.006 | INFO | "malloc" : " 14 GB", | 20260507 18:25:31.006 | INFO | "pinned" : " 29 GB" | 20260507 18:25:31.006 | INFO | } | 20260507 18:25:31.006 | INFO | Perf report for the last 5.13908 seconds: | 20260507 18:25:31.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:31.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:25:31.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:25:31.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:31.006 | INFO | Uploaded 36GiB to GPU in 2.50441s (14.3747GiB/s) | 20260507 18:25:31.006 | INFO | Downloaded 34GiB to GPU in 2.41631s (14.071GiB/s) | 20260507 18:25:31.103 | INFO | Starting a round of Baz IO | 20260507 18:25:34.596 | INFO | Populating RTMetrics report | 20260507 18:25:34.597 | INFO | RTMetrics report populated | 20260507 18:25:34.597 | INFO | Opening tmp RTMetrics json file | 20260507 18:25:34.619 | INFO | Serializing RTMetrics to json | 20260507 18:25:34.621 | INFO | Serialization done, swapping tmp file now... | 20260507 18:25:34.624 | INFO | RTMetrics done | 20260507 18:25:34.720 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.617 seconds | 20260507 18:25:34.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.64% / 0.02% / 53.35% | 20260507 18:25:34.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.98243% / 2.78155e-05% / 1.98682e-05% | 20260507 18:25:34.721 | INFO | Baz IO overall: | 20260507 18:25:34.721 | INFO | BytesWritten : 730378131399 | 20260507 18:25:34.721 | INFO | NumEvents : 354079491747 | 20260507 18:25:36.145 | INFO | Analyzing chunk frames = [1634304,1634816) | 20260507 18:25:36.145 | INFO | MemInfo: { | 20260507 18:25:36.145 | INFO | "gpu" : " 17 GB", | 20260507 18:25:36.145 | INFO | "huge" : "", | 20260507 18:25:36.145 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:36.145 | INFO | "malloc" : " 14 GB", | 20260507 18:25:36.145 | INFO | "pinned" : " 29 GB" | 20260507 18:25:36.145 | INFO | } | 20260507 18:25:36.145 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:25:36.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:36.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:36.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:36.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:36.145 | INFO | ANALYSIS : 69.4%, 3.0, 107.5ms, 0 | 20260507 18:25:36.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:36.145 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 18:25:36.145 | INFO | PRE_HQ : 66.4%, 1.7, 58.9ms, 0 | 20260507 18:25:36.146 | INFO | BAZWRITER : 69.9%, 3.4, 123.7ms, 0 | 20260507 18:25:36.146 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 18:25:36.146 | INFO | Uploaded 36GiB to GPU in 2.56117s (14.0561GiB/s) | 20260507 18:25:36.146 | INFO | Downloaded 38GiB to GPU in 2.70606s (14.0425GiB/s) | 20260507 18:25:41.284 | INFO | Analyzing chunk frames = [1634816,1635328) | 20260507 18:25:41.284 | INFO | MemInfo: { | 20260507 18:25:41.284 | INFO | "gpu" : " 17 GB", | 20260507 18:25:41.284 | INFO | "huge" : "", | 20260507 18:25:41.284 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:41.284 | INFO | "malloc" : " 14 GB", | 20260507 18:25:41.284 | INFO | "pinned" : " 29 GB" | 20260507 18:25:41.285 | INFO | } | 20260507 18:25:41.285 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:25:41.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:41.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 18:25:41.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:25:41.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:41.285 | INFO | Uploaded 36GiB to GPU in 2.56403s (14.0404GiB/s) | 20260507 18:25:41.285 | INFO | Downloaded 34GiB to GPU in 2.44302s (13.9172GiB/s) | 20260507 18:25:46.424 | INFO | Analyzing chunk frames = [1635328,1635840) | 20260507 18:25:46.424 | INFO | MemInfo: { | 20260507 18:25:46.424 | INFO | "gpu" : " 17 GB", | 20260507 18:25:46.424 | INFO | "huge" : "", | 20260507 18:25:46.424 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:46.424 | INFO | "malloc" : " 14 GB", | 20260507 18:25:46.424 | INFO | "pinned" : " 29 GB" | 20260507 18:25:46.424 | INFO | } | 20260507 18:25:46.424 | INFO | Perf report for the last 5.13936 seconds: | 20260507 18:25:46.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:46.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.424 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:25:46.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.424 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:25:46.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:46.425 | INFO | Uploaded 36GiB to GPU in 2.57415s (13.9852GiB/s) | 20260507 18:25:46.425 | INFO | Downloaded 34GiB to GPU in 2.43387s (13.9695GiB/s) | 20260507 18:25:51.563 | INFO | Analyzing chunk frames = [1635840,1636352) | 20260507 18:25:51.563 | INFO | MemInfo: { | 20260507 18:25:51.563 | INFO | "gpu" : " 17 GB", | 20260507 18:25:51.563 | INFO | "huge" : "", | 20260507 18:25:51.563 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:51.563 | INFO | "malloc" : " 14 GB", | 20260507 18:25:51.563 | INFO | "pinned" : " 29 GB" | 20260507 18:25:51.563 | INFO | } | 20260507 18:25:51.563 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:25:51.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:51.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.563 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:25:51.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.563 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:25:51.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:51.564 | INFO | Uploaded 36GiB to GPU in 2.50811s (14.3534GiB/s) | 20260507 18:25:51.564 | INFO | Downloaded 34GiB to GPU in 2.35288s (14.4504GiB/s) | 20260507 18:25:56.703 | INFO | Analyzing chunk frames = [1636352,1636864) | 20260507 18:25:56.703 | INFO | MemInfo: { | 20260507 18:25:56.703 | INFO | "gpu" : " 17 GB", | 20260507 18:25:56.703 | INFO | "huge" : "", | 20260507 18:25:56.703 | INFO | "huge_shared" : " 29 GB", | 20260507 18:25:56.703 | INFO | "malloc" : " 14 GB", | 20260507 18:25:56.703 | INFO | "pinned" : " 29 GB" | 20260507 18:25:56.703 | INFO | } | 20260507 18:25:56.703 | INFO | Perf report for the last 5.13988 seconds: | 20260507 18:25:56.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:25:56.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 18:25:56.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:25:56.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:25:56.703 | INFO | Uploaded 36GiB to GPU in 2.35694s (15.2741GiB/s) | 20260507 18:25:56.703 | INFO | Downloaded 34GiB to GPU in 2.46844s (13.7739GiB/s) | 20260507 18:26:01.842 | INFO | Analyzing chunk frames = [1636864,1637376) | 20260507 18:26:01.842 | INFO | MemInfo: { | 20260507 18:26:01.842 | INFO | "gpu" : " 17 GB", | 20260507 18:26:01.842 | INFO | "huge" : "", | 20260507 18:26:01.842 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:01.842 | INFO | "malloc" : " 14 GB", | 20260507 18:26:01.842 | INFO | "pinned" : " 29 GB" | 20260507 18:26:01.842 | INFO | } | 20260507 18:26:01.842 | INFO | Perf report for the last 5.13918 seconds: | 20260507 18:26:01.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:01.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.842 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:26:01.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.843 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260507 18:26:01.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:01.843 | INFO | Uploaded 36GiB to GPU in 2.48033s (14.5142GiB/s) | 20260507 18:26:01.843 | INFO | Downloaded 34GiB to GPU in 2.48848s (13.663GiB/s) | 20260507 18:26:06.981 | INFO | Analyzing chunk frames = [1637376,1637888) | 20260507 18:26:06.981 | INFO | MemInfo: { | 20260507 18:26:06.981 | INFO | "gpu" : " 17 GB", | 20260507 18:26:06.981 | INFO | "huge" : "", | 20260507 18:26:06.981 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:06.981 | INFO | "malloc" : " 14 GB", | 20260507 18:26:06.981 | INFO | "pinned" : " 29 GB" | 20260507 18:26:06.981 | INFO | } | 20260507 18:26:06.981 | INFO | Perf report for the last 5.13911 seconds: | 20260507 18:26:06.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:06.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:26:06.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | PRE_HQ : 38.5%, 1.0, 20.1ms, 0 | 20260507 18:26:06.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:06.982 | INFO | Uploaded 36GiB to GPU in 2.53158s (14.2204GiB/s) | 20260507 18:26:06.982 | INFO | Downloaded 34GiB to GPU in 2.4324s (13.978GiB/s) | 20260507 18:26:12.120 | INFO | Analyzing chunk frames = [1637888,1638400) | 20260507 18:26:12.120 | INFO | MemInfo: { | 20260507 18:26:12.120 | INFO | "gpu" : " 17 GB", | 20260507 18:26:12.120 | INFO | "huge" : "", | 20260507 18:26:12.120 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:12.121 | INFO | "malloc" : " 14 GB", | 20260507 18:26:12.121 | INFO | "pinned" : " 29 GB" | 20260507 18:26:12.121 | INFO | } | 20260507 18:26:12.121 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:26:12.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:12.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 18:26:12.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 18:26:12.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:12.121 | INFO | Uploaded 36GiB to GPU in 2.51748s (14.3GiB/s) | 20260507 18:26:12.121 | INFO | Downloaded 34GiB to GPU in 2.42169s (14.0398GiB/s) | 20260507 18:26:12.122 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:26:12.122 | INFO | Total mem usage: 16.4 GiB | 20260507 18:26:12.122 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:26:12.122 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.9%) | 20260507 18:26:12.122 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:26:12.122 | INFO | High Water Breakdown: | 20260507 18:26:12.122 | INFO | 59.6% : 8.82 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:26:12.122 | INFO | 40.2% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:26:12.122 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:26:12.122 | INFO | ------------------------------------------ | 20260507 18:26:12.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:26:12.122 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:26:12.122 | INFO | Total mem usage: 26.5 GiB | 20260507 18:26:12.122 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:26:12.122 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 18:26:12.122 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:26:12.122 | INFO | High Water Breakdown: | 20260507 18:26:12.122 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:26:12.122 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:26:12.122 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:26:12.122 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:26:12.122 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:26:12.122 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:26:12.122 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:26:12.122 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:26:12.122 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:26:12.122 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:26:12.122 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:26:12.122 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:26:12.122 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:26:12.122 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:26:12.122 | INFO | ------------------------------------------ | 20260507 18:26:12.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:26:12.122 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:26:12.122 | INFO | Total mem usage: 27.8 GiB | 20260507 18:26:12.122 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:26:12.122 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:26:12.122 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:26:12.122 | INFO | High Water Breakdown: | 20260507 18:26:12.122 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:26:12.122 | INFO | ------------------------------------------ | 20260507 18:26:12.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:26:12.122 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:26:12.122 | INFO | Total mem usage: 15.7 GiB | 20260507 18:26:12.122 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:26:12.122 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:26:12.123 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:26:12.123 | INFO | High Water Breakdown: | 20260507 18:26:12.123 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:26:12.123 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:26:12.123 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:26:12.123 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:26:12.123 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:26:12.123 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:26:12.123 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:26:12.123 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:26:12.123 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:26:12.123 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:26:12.123 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:26:12.123 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:26:12.123 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:26:12.123 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:26:12.123 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:26:12.123 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:26:12.123 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:26:12.123 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:26:12.123 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:26:12.123 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:26:12.123 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:26:12.123 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:26:12.123 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:26:12.123 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:26:12.123 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:26:12.123 | INFO | ------------------------------------------ | 20260507 18:26:12.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:26:12.123 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:26:12.123 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:26:12.123 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:26:12.123 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.42e+01 / 2.24e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.04e-01, 1.12e-02 / 5.63e-03 | 20260507 18:26:12.123 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.49e+00, 6.09e-01 / 2.37e-01 | 20260507 18:26:12.123 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.15e-01 | 20260507 18:26:12.123 | INFO | PerfReport: 15.68%, DME -- 4.20e+01 / 4.50e+01, 4.16e+00 / 2.03e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.89e+00 / 2.27e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.39e-01 / 2.51e-01 | 20260507 18:26:12.123 | INFO | PerfReport: 3.73%, Metrics -- 1.25e+00 / 1.43e+00, 3.18e-01 / 1.44e-01 | 20260507 18:26:12.123 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.94e-01 / 4.21e-01, 6.39e-02 / 3.37e-02 | 20260507 18:26:12.123 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:26:12.123 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:26:12.123 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:26:12.123 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 24.28%, Upload -- 2.60e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 24.00%, Download -- 2.57e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 31.19%, Compute -- 3.35e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 2.01%, LockUpload -- 2.16e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 2.06%, LockDownload -- 2.21e+00 / 0.00e+00, 3.55e+00 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 16.45%, LockCompute -- 1.76e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:26:12.123 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 18:26:12.232 | INFO | Starting a round of Baz IO | 20260507 18:26:15.735 | INFO | Populating RTMetrics report | 20260507 18:26:15.735 | INFO | RTMetrics report populated | 20260507 18:26:15.735 | INFO | Opening tmp RTMetrics json file | 20260507 18:26:15.747 | INFO | Serializing RTMetrics to json | 20260507 18:26:15.754 | INFO | Serialization done, swapping tmp file now... | 20260507 18:26:15.756 | INFO | RTMetrics done | 20260507 18:26:15.813 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.581 seconds | 20260507 18:26:15.814 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.62% / 0.02% / 53.36% | 20260507 18:26:15.814 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.96838% / 2.38419e-05% / 5.16574e-05% | 20260507 18:26:15.814 | INFO | Baz IO overall: | 20260507 18:26:15.814 | INFO | BytesWritten : 732316635446 | 20260507 18:26:15.814 | INFO | NumEvents : 355020735040 | 20260507 18:26:17.260 | INFO | Analyzing chunk frames = [1638400,1638912) | 20260507 18:26:17.260 | INFO | MemInfo: { | 20260507 18:26:17.260 | INFO | "gpu" : " 17 GB", | 20260507 18:26:17.260 | INFO | "huge" : "", | 20260507 18:26:17.260 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:17.260 | INFO | "malloc" : " 14 GB", | 20260507 18:26:17.260 | INFO | "pinned" : " 29 GB" | 20260507 18:26:17.260 | INFO | } | 20260507 18:26:17.260 | INFO | Perf report for the last 5.13906 seconds: | 20260507 18:26:17.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:17.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:17.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:17.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:17.260 | INFO | ANALYSIS : 69.6%, 2.9, 106.0ms, 0 | 20260507 18:26:17.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:17.260 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 18:26:17.260 | INFO | PRE_HQ : 68.2%, 1.8, 64.6ms, 0 | 20260507 18:26:17.260 | INFO | BAZWRITER : 69.2%, 3.6, 129.0ms, 0 | 20260507 18:26:17.260 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:26:17.261 | INFO | Uploaded 36GiB to GPU in 2.52361s (14.2653GiB/s) | 20260507 18:26:17.261 | INFO | Downloaded 38GiB to GPU in 2.68718s (14.1412GiB/s) | 20260507 18:26:22.399 | INFO | Analyzing chunk frames = [1638912,1639424) | 20260507 18:26:22.399 | INFO | MemInfo: { | 20260507 18:26:22.399 | INFO | "gpu" : " 17 GB", | 20260507 18:26:22.399 | INFO | "huge" : "", | 20260507 18:26:22.399 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:22.399 | INFO | "malloc" : " 14 GB", | 20260507 18:26:22.399 | INFO | "pinned" : " 29 GB" | 20260507 18:26:22.399 | INFO | } | 20260507 18:26:22.400 | INFO | Perf report for the last 5.13978 seconds: | 20260507 18:26:22.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:22.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260507 18:26:22.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:26:22.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:22.400 | INFO | Uploaded 36GiB to GPU in 2.49352s (14.4374GiB/s) | 20260507 18:26:22.400 | INFO | Downloaded 34GiB to GPU in 2.42828s (14.0017GiB/s) | 20260507 18:26:27.539 | INFO | Analyzing chunk frames = [1639424,1639936) | 20260507 18:26:27.539 | INFO | MemInfo: { | 20260507 18:26:27.539 | INFO | "gpu" : " 17 GB", | 20260507 18:26:27.539 | INFO | "huge" : "", | 20260507 18:26:27.539 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:27.539 | INFO | "malloc" : " 14 GB", | 20260507 18:26:27.539 | INFO | "pinned" : " 29 GB" | 20260507 18:26:27.539 | INFO | } | 20260507 18:26:27.539 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:26:27.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:27.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | ANALYSIS : 68.1%, 3.0, 105.5ms, 0 | 20260507 18:26:27.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:26:27.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:27.539 | INFO | Uploaded 36GiB to GPU in 2.55505s (14.0898GiB/s) | 20260507 18:26:27.540 | INFO | Downloaded 34GiB to GPU in 2.45389s (13.8555GiB/s) | 20260507 18:26:32.678 | INFO | Analyzing chunk frames = [1639936,1640448) | 20260507 18:26:32.678 | INFO | MemInfo: { | 20260507 18:26:32.678 | INFO | "gpu" : " 17 GB", | 20260507 18:26:32.678 | INFO | "huge" : "", | 20260507 18:26:32.678 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:32.678 | INFO | "malloc" : " 14 GB", | 20260507 18:26:32.678 | INFO | "pinned" : " 29 GB" | 20260507 18:26:32.678 | INFO | } | 20260507 18:26:32.678 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:26:32.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:32.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:26:32.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:26:32.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:32.678 | INFO | Uploaded 36GiB to GPU in 2.48398s (14.4929GiB/s) | 20260507 18:26:32.678 | INFO | Downloaded 34GiB to GPU in 2.37578s (14.3111GiB/s) | 20260507 18:26:37.818 | INFO | Analyzing chunk frames = [1640448,1640960) | 20260507 18:26:37.818 | INFO | MemInfo: { | 20260507 18:26:37.818 | INFO | "gpu" : " 17 GB", | 20260507 18:26:37.818 | INFO | "huge" : "", | 20260507 18:26:37.818 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:37.818 | INFO | "malloc" : " 14 GB", | 20260507 18:26:37.818 | INFO | "pinned" : " 29 GB" | 20260507 18:26:37.818 | INFO | } | 20260507 18:26:37.818 | INFO | Perf report for the last 5.14012 seconds: | 20260507 18:26:37.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:37.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.818 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 18:26:37.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.818 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:26:37.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:37.819 | INFO | Uploaded 36GiB to GPU in 2.31578s (15.5455GiB/s) | 20260507 18:26:37.819 | INFO | Downloaded 34GiB to GPU in 2.47638s (13.7297GiB/s) | 20260507 18:26:42.957 | INFO | Analyzing chunk frames = [1640960,1641472) | 20260507 18:26:42.957 | INFO | MemInfo: { | 20260507 18:26:42.957 | INFO | "gpu" : " 17 GB", | 20260507 18:26:42.957 | INFO | "huge" : "", | 20260507 18:26:42.957 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:42.957 | INFO | "malloc" : " 14 GB", | 20260507 18:26:42.957 | INFO | "pinned" : " 29 GB" | 20260507 18:26:42.957 | INFO | } | 20260507 18:26:42.957 | INFO | Perf report for the last 5.13858 seconds: | 20260507 18:26:42.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:42.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:26:42.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:26:42.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:42.957 | INFO | Uploaded 36GiB to GPU in 2.44231s (14.7401GiB/s) | 20260507 18:26:42.957 | INFO | Downloaded 34GiB to GPU in 2.51662s (13.5102GiB/s) | 20260507 18:26:48.096 | INFO | Analyzing chunk frames = [1641472,1641984) | 20260507 18:26:48.096 | INFO | MemInfo: { | 20260507 18:26:48.096 | INFO | "gpu" : " 17 GB", | 20260507 18:26:48.096 | INFO | "huge" : "", | 20260507 18:26:48.096 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:48.096 | INFO | "malloc" : " 14 GB", | 20260507 18:26:48.096 | INFO | "pinned" : " 29 GB" | 20260507 18:26:48.096 | INFO | } | 20260507 18:26:48.096 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:26:48.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:48.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.096 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:26:48.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.096 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 18:26:48.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:48.097 | INFO | Uploaded 36GiB to GPU in 2.48098s (14.5104GiB/s) | 20260507 18:26:48.097 | INFO | Downloaded 34GiB to GPU in 2.4414s (13.9265GiB/s) | 20260507 18:26:53.235 | INFO | Analyzing chunk frames = [1641984,1642496) | 20260507 18:26:53.235 | INFO | MemInfo: { | 20260507 18:26:53.235 | INFO | "gpu" : " 17 GB", | 20260507 18:26:53.235 | INFO | "huge" : "", | 20260507 18:26:53.235 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:53.235 | INFO | "malloc" : " 14 GB", | 20260507 18:26:53.235 | INFO | "pinned" : " 29 GB" | 20260507 18:26:53.236 | INFO | } | 20260507 18:26:53.236 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:26:53.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:53.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:26:53.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | PRE_HQ : 38.0%, 1.0, 19.7ms, 0 | 20260507 18:26:53.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:53.236 | INFO | Uploaded 36GiB to GPU in 2.47402s (14.5512GiB/s) | 20260507 18:26:53.236 | INFO | Downloaded 34GiB to GPU in 2.42922s (13.9962GiB/s) | 20260507 18:26:53.349 | INFO | Starting a round of Baz IO | 20260507 18:26:56.842 | INFO | Populating RTMetrics report | 20260507 18:26:56.843 | INFO | RTMetrics report populated | 20260507 18:26:56.843 | INFO | Opening tmp RTMetrics json file | 20260507 18:26:56.845 | INFO | Serializing RTMetrics to json | 20260507 18:26:56.869 | INFO | Serialization done, swapping tmp file now... | 20260507 18:26:56.870 | INFO | RTMetrics done | 20260507 18:26:56.948 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.599 seconds | 20260507 18:26:56.948 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.60% / 0.02% / 53.38% | 20260507 18:26:56.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.96322% / 3.17891e-05% / 3.57628e-05% | 20260507 18:26:56.949 | INFO | Baz IO overall: | 20260507 18:26:56.949 | INFO | BytesWritten : 734256902505 | 20260507 18:26:56.949 | INFO | NumEvents : 355962925337 | 20260507 18:26:58.375 | INFO | Analyzing chunk frames = [1642496,1643008) | 20260507 18:26:58.375 | INFO | MemInfo: { | 20260507 18:26:58.375 | INFO | "gpu" : " 17 GB", | 20260507 18:26:58.375 | INFO | "huge" : "", | 20260507 18:26:58.375 | INFO | "huge_shared" : " 29 GB", | 20260507 18:26:58.375 | INFO | "malloc" : " 14 GB", | 20260507 18:26:58.375 | INFO | "pinned" : " 29 GB" | 20260507 18:26:58.375 | INFO | } | 20260507 18:26:58.375 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:26:58.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:26:58.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:58.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:58.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:58.375 | INFO | ANALYSIS : 69.4%, 2.9, 106.0ms, 0 | 20260507 18:26:58.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:26:58.375 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 18:26:58.375 | INFO | PRE_HQ : 67.7%, 1.8, 64.9ms, 0 | 20260507 18:26:58.375 | INFO | BAZWRITER : 69.6%, 3.5, 125.3ms, 0 | 20260507 18:26:58.375 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 18:26:58.375 | INFO | Uploaded 36GiB to GPU in 2.54805s (14.1285GiB/s) | 20260507 18:26:58.376 | INFO | Downloaded 38GiB to GPU in 2.69601s (14.0949GiB/s) | 20260507 18:27:03.514 | INFO | Analyzing chunk frames = [1643008,1643520) | 20260507 18:27:03.514 | INFO | MemInfo: { | 20260507 18:27:03.514 | INFO | "gpu" : " 17 GB", | 20260507 18:27:03.514 | INFO | "huge" : "", | 20260507 18:27:03.514 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:03.514 | INFO | "malloc" : " 14 GB", | 20260507 18:27:03.514 | INFO | "pinned" : " 29 GB" | 20260507 18:27:03.515 | INFO | } | 20260507 18:27:03.515 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:27:03.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:03.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:27:03.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:27:03.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:03.515 | INFO | Uploaded 36GiB to GPU in 2.5323s (14.2163GiB/s) | 20260507 18:27:03.515 | INFO | Downloaded 34GiB to GPU in 2.46887s (13.7715GiB/s) | 20260507 18:27:08.654 | INFO | Analyzing chunk frames = [1643520,1644032) | 20260507 18:27:08.654 | INFO | MemInfo: { | 20260507 18:27:08.654 | INFO | "gpu" : " 17 GB", | 20260507 18:27:08.654 | INFO | "huge" : "", | 20260507 18:27:08.654 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:08.654 | INFO | "malloc" : " 14 GB", | 20260507 18:27:08.654 | INFO | "pinned" : " 29 GB" | 20260507 18:27:08.654 | INFO | } | 20260507 18:27:08.654 | INFO | Perf report for the last 5.13967 seconds: | 20260507 18:27:08.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:08.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.654 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:27:08.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.654 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:27:08.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:08.655 | INFO | Uploaded 36GiB to GPU in 2.54327s (14.155GiB/s) | 20260507 18:27:08.655 | INFO | Downloaded 34GiB to GPU in 2.45s (13.8776GiB/s) | 20260507 18:27:13.793 | INFO | Analyzing chunk frames = [1644032,1644544) | 20260507 18:27:13.793 | INFO | MemInfo: { | 20260507 18:27:13.793 | INFO | "gpu" : " 17 GB", | 20260507 18:27:13.793 | INFO | "huge" : "", | 20260507 18:27:13.793 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:13.793 | INFO | "malloc" : " 14 GB", | 20260507 18:27:13.793 | INFO | "pinned" : " 29 GB" | 20260507 18:27:13.793 | INFO | } | 20260507 18:27:13.793 | INFO | Perf report for the last 5.13901 seconds: | 20260507 18:27:13.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:13.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.793 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:27:13.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.794 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:27:13.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:13.794 | INFO | Uploaded 36GiB to GPU in 2.46435s (14.6083GiB/s) | 20260507 18:27:13.794 | INFO | Downloaded 34GiB to GPU in 2.35829s (14.4172GiB/s) | 20260507 18:27:18.933 | INFO | Analyzing chunk frames = [1644544,1645056) | 20260507 18:27:18.933 | INFO | MemInfo: { | 20260507 18:27:18.933 | INFO | "gpu" : " 17 GB", | 20260507 18:27:18.933 | INFO | "huge" : "", | 20260507 18:27:18.933 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:18.933 | INFO | "malloc" : " 14 GB", | 20260507 18:27:18.933 | INFO | "pinned" : " 29 GB" | 20260507 18:27:18.933 | INFO | } | 20260507 18:27:18.933 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:27:18.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:18.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:27:18.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:27:18.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:18.933 | INFO | Uploaded 36GiB to GPU in 2.31905s (15.5236GiB/s) | 20260507 18:27:18.933 | INFO | Downloaded 34GiB to GPU in 2.48445s (13.6851GiB/s) | 20260507 18:27:24.071 | INFO | Analyzing chunk frames = [1645056,1645568) | 20260507 18:27:24.071 | INFO | MemInfo: { | 20260507 18:27:24.071 | INFO | "gpu" : " 17 GB", | 20260507 18:27:24.071 | INFO | "huge" : "", | 20260507 18:27:24.071 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:24.071 | INFO | "malloc" : " 14 GB", | 20260507 18:27:24.071 | INFO | "pinned" : " 29 GB" | 20260507 18:27:24.072 | INFO | } | 20260507 18:27:24.072 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:27:24.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:24.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:27:24.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:27:24.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:24.072 | INFO | Uploaded 36GiB to GPU in 2.39061s (15.0589GiB/s) | 20260507 18:27:24.072 | INFO | Downloaded 34GiB to GPU in 2.4936s (13.6349GiB/s) | 20260507 18:27:29.211 | INFO | Analyzing chunk frames = [1645568,1646080) | 20260507 18:27:29.211 | INFO | MemInfo: { | 20260507 18:27:29.211 | INFO | "gpu" : " 17 GB", | 20260507 18:27:29.211 | INFO | "huge" : "", | 20260507 18:27:29.211 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:29.211 | INFO | "malloc" : " 14 GB", | 20260507 18:27:29.211 | INFO | "pinned" : " 29 GB" | 20260507 18:27:29.211 | INFO | } | 20260507 18:27:29.212 | INFO | Perf report for the last 5.13973 seconds: | 20260507 18:27:29.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:29.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:27:29.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 18:27:29.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:29.212 | INFO | Uploaded 36GiB to GPU in 2.50437s (14.3749GiB/s) | 20260507 18:27:29.212 | INFO | Downloaded 34GiB to GPU in 2.44217s (13.922GiB/s) | 20260507 18:27:34.350 | INFO | Analyzing chunk frames = [1646080,1646592) | 20260507 18:27:34.350 | INFO | MemInfo: { | 20260507 18:27:34.350 | INFO | "gpu" : " 17 GB", | 20260507 18:27:34.350 | INFO | "huge" : "", | 20260507 18:27:34.350 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:34.350 | INFO | "malloc" : " 14 GB", | 20260507 18:27:34.350 | INFO | "pinned" : " 29 GB" | 20260507 18:27:34.350 | INFO | } | 20260507 18:27:34.350 | INFO | Perf report for the last 5.13894 seconds: | 20260507 18:27:34.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:34.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.351 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:27:34.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.351 | INFO | PRE_HQ : 38.5%, 1.0, 20.0ms, 0 | 20260507 18:27:34.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:34.351 | INFO | Uploaded 36GiB to GPU in 2.49636s (14.421GiB/s) | 20260507 18:27:34.351 | INFO | Downloaded 34GiB to GPU in 2.42799s (14.0034GiB/s) | 20260507 18:27:34.351 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:27:34.351 | INFO | Total mem usage: 16.4 GiB | 20260507 18:27:34.351 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:27:34.351 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.4%) | 20260507 18:27:34.351 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:27:34.351 | INFO | High Water Breakdown: | 20260507 18:27:34.351 | INFO | 59.6% : 8.77 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:27:34.351 | INFO | 40.3% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:27:34.351 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:27:34.351 | INFO | ------------------------------------------ | 20260507 18:27:34.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:27:34.351 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:27:34.351 | INFO | Total mem usage: 26.5 GiB | 20260507 18:27:34.351 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:27:34.351 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:27:34.351 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:27:34.351 | INFO | High Water Breakdown: | 20260507 18:27:34.351 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:27:34.351 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:27:34.351 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:27:34.351 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:27:34.351 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:27:34.351 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:27:34.351 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:27:34.351 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:27:34.351 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:27:34.351 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:27:34.351 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:27:34.351 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:27:34.351 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:27:34.351 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:27:34.351 | INFO | ------------------------------------------ | 20260507 18:27:34.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:27:34.351 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:27:34.351 | INFO | Total mem usage: 27.8 GiB | 20260507 18:27:34.351 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:27:34.351 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:27:34.351 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:27:34.351 | INFO | High Water Breakdown: | 20260507 18:27:34.351 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:27:34.351 | INFO | ------------------------------------------ | 20260507 18:27:34.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:27:34.351 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:27:34.351 | INFO | Total mem usage: 15.7 GiB | 20260507 18:27:34.351 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:27:34.351 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:27:34.351 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:27:34.351 | INFO | High Water Breakdown: | 20260507 18:27:34.351 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:27:34.351 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:27:34.351 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:27:34.351 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:27:34.351 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:27:34.351 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:27:34.351 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:27:34.351 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:27:34.351 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:27:34.351 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:27:34.351 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:27:34.351 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:27:34.351 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:27:34.351 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:27:34.351 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:27:34.351 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:27:34.351 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:27:34.351 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:27:34.351 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:27:34.351 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:27:34.351 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:27:34.351 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:27:34.351 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:27:34.351 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:27:34.351 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:27:34.351 | INFO | ------------------------------------------ | 20260507 18:27:34.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:27:34.351 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:27:34.351 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:27:34.351 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:27:34.351 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.64e+01, 1.45e+01 / 2.07e+00 | 20260507 18:27:34.351 | INFO | PerfReport: 0.30%, AggregateStats -- 9.95e-02 / 1.04e-01, 1.12e-02 / 7.63e-03 | 20260507 18:27:34.351 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.52e+00, 6.19e-01 / 2.42e-01 | 20260507 18:27:34.351 | INFO | PerfReport: 3.45%, Histogram -- 1.15e+00 / 1.51e+00, 2.20e-01 / 1.22e-01 | 20260507 18:27:34.351 | INFO | PerfReport: 15.67%, DME -- 4.19e+01 / 4.49e+01, 4.11e+00 / 1.84e+00 | 20260507 18:27:34.351 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.88e+00 / 2.08e+00 | 20260507 18:27:34.351 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.33e-01 / 2.59e-01 | 20260507 18:27:34.351 | INFO | PerfReport: 3.72%, Metrics -- 1.25e+00 / 1.44e+00, 3.21e-01 / 1.65e-01 | 20260507 18:27:34.351 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.23e-01, 6.39e-02 / 3.41e-02 | 20260507 18:27:34.352 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:27:34.352 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:27:34.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:27:34.352 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 24.06%, Upload -- 2.58e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 24.10%, Download -- 2.58e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 31.25%, Compute -- 3.35e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 1.85%, LockUpload -- 1.98e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 2.23%, LockDownload -- 2.39e+00 / 0.00e+00, 3.77e+00 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 16.51%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:27:34.352 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260507 18:27:34.463 | INFO | Starting a round of Baz IO | 20260507 18:27:37.957 | INFO | Populating RTMetrics report | 20260507 18:27:37.957 | INFO | RTMetrics report populated | 20260507 18:27:37.957 | INFO | Opening tmp RTMetrics json file | 20260507 18:27:37.958 | INFO | Serializing RTMetrics to json | 20260507 18:27:37.972 | INFO | Serialization done, swapping tmp file now... | 20260507 18:27:37.982 | INFO | RTMetrics done | 20260507 18:27:38.063 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.6 seconds | 20260507 18:27:38.063 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.59% / 0.01% / 53.40% | 20260507 18:27:38.063 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.94096% / 1.98682e-05% / 2.38419e-05% | 20260507 18:27:38.063 | INFO | Baz IO overall: | 20260507 18:27:38.063 | INFO | BytesWritten : 736193543086 | 20260507 18:27:38.063 | INFO | NumEvents : 356902252703 | 20260507 18:27:39.490 | INFO | Analyzing chunk frames = [1646592,1647104) | 20260507 18:27:39.490 | INFO | MemInfo: { | 20260507 18:27:39.490 | INFO | "gpu" : " 17 GB", | 20260507 18:27:39.490 | INFO | "huge" : "", | 20260507 18:27:39.490 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:39.490 | INFO | "malloc" : " 14 GB", | 20260507 18:27:39.490 | INFO | "pinned" : " 29 GB" | 20260507 18:27:39.490 | INFO | } | 20260507 18:27:39.490 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:27:39.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:39.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:39.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:39.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:39.490 | INFO | ANALYSIS : 69.7%, 2.9, 105.9ms, 0 | 20260507 18:27:39.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:39.490 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 18:27:39.490 | INFO | PRE_HQ : 67.9%, 1.8, 64.1ms, 0 | 20260507 18:27:39.490 | INFO | BAZWRITER : 69.4%, 3.6, 127.5ms, 0 | 20260507 18:27:39.490 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 18:27:39.490 | INFO | Uploaded 36GiB to GPU in 2.62681s (13.7048GiB/s) | 20260507 18:27:39.490 | INFO | Downloaded 38GiB to GPU in 2.7051s (14.0475GiB/s) | 20260507 18:27:44.629 | INFO | Analyzing chunk frames = [1647104,1647616) | 20260507 18:27:44.629 | INFO | MemInfo: { | 20260507 18:27:44.629 | INFO | "gpu" : " 17 GB", | 20260507 18:27:44.629 | INFO | "huge" : "", | 20260507 18:27:44.629 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:44.629 | INFO | "malloc" : " 14 GB", | 20260507 18:27:44.629 | INFO | "pinned" : " 29 GB" | 20260507 18:27:44.629 | INFO | } | 20260507 18:27:44.629 | INFO | Perf report for the last 5.13978 seconds: | 20260507 18:27:44.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:44.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.630 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:27:44.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.630 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:27:44.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:44.630 | INFO | Uploaded 36GiB to GPU in 2.55095s (14.1124GiB/s) | 20260507 18:27:44.630 | INFO | Downloaded 34GiB to GPU in 2.44364s (13.9137GiB/s) | 20260507 18:27:49.768 | INFO | Analyzing chunk frames = [1647616,1648128) | 20260507 18:27:49.768 | INFO | MemInfo: { | 20260507 18:27:49.768 | INFO | "gpu" : " 17 GB", | 20260507 18:27:49.768 | INFO | "huge" : "", | 20260507 18:27:49.768 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:49.768 | INFO | "malloc" : " 14 GB", | 20260507 18:27:49.768 | INFO | "pinned" : " 29 GB" | 20260507 18:27:49.768 | INFO | } | 20260507 18:27:49.768 | INFO | Perf report for the last 5.1387 seconds: | 20260507 18:27:49.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:49.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.768 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:27:49.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.768 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:27:49.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:49.769 | INFO | Uploaded 36GiB to GPU in 2.54573s (14.1413GiB/s) | 20260507 18:27:49.769 | INFO | Downloaded 34GiB to GPU in 2.43458s (13.9655GiB/s) | 20260507 18:27:54.907 | INFO | Analyzing chunk frames = [1648128,1648640) | 20260507 18:27:54.907 | INFO | MemInfo: { | 20260507 18:27:54.907 | INFO | "gpu" : " 17 GB", | 20260507 18:27:54.907 | INFO | "huge" : "", | 20260507 18:27:54.907 | INFO | "huge_shared" : " 29 GB", | 20260507 18:27:54.907 | INFO | "malloc" : " 14 GB", | 20260507 18:27:54.908 | INFO | "pinned" : " 29 GB" | 20260507 18:27:54.908 | INFO | } | 20260507 18:27:54.908 | INFO | Perf report for the last 5.13942 seconds: | 20260507 18:27:54.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:27:54.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:27:54.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:27:54.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:27:54.908 | INFO | Uploaded 36GiB to GPU in 2.50158s (14.3909GiB/s) | 20260507 18:27:54.908 | INFO | Downloaded 34GiB to GPU in 2.34045s (14.5271GiB/s) | 20260507 18:28:00.047 | INFO | Analyzing chunk frames = [1648640,1649152) | 20260507 18:28:00.047 | INFO | MemInfo: { | 20260507 18:28:00.047 | INFO | "gpu" : " 17 GB", | 20260507 18:28:00.047 | INFO | "huge" : "", | 20260507 18:28:00.047 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:00.047 | INFO | "malloc" : " 14 GB", | 20260507 18:28:00.047 | INFO | "pinned" : " 29 GB" | 20260507 18:28:00.047 | INFO | } | 20260507 18:28:00.047 | INFO | Perf report for the last 5.13961 seconds: | 20260507 18:28:00.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:00.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:28:00.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:28:00.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:00.047 | INFO | Uploaded 36GiB to GPU in 2.34047s (15.3815GiB/s) | 20260507 18:28:00.047 | INFO | Downloaded 34GiB to GPU in 2.48071s (13.7058GiB/s) | 20260507 18:28:05.186 | INFO | Analyzing chunk frames = [1649152,1649664) | 20260507 18:28:05.186 | INFO | MemInfo: { | 20260507 18:28:05.186 | INFO | "gpu" : " 17 GB", | 20260507 18:28:05.186 | INFO | "huge" : "", | 20260507 18:28:05.186 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:05.186 | INFO | "malloc" : " 14 GB", | 20260507 18:28:05.186 | INFO | "pinned" : " 29 GB" | 20260507 18:28:05.186 | INFO | } | 20260507 18:28:05.186 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:28:05.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:05.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.186 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:28:05.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.187 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:28:05.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:05.187 | INFO | Uploaded 36GiB to GPU in 2.44203s (14.7419GiB/s) | 20260507 18:28:05.187 | INFO | Downloaded 34GiB to GPU in 2.48938s (13.658GiB/s) | 20260507 18:28:10.326 | INFO | Analyzing chunk frames = [1649664,1650176) | 20260507 18:28:10.326 | INFO | MemInfo: { | 20260507 18:28:10.326 | INFO | "gpu" : " 17 GB", | 20260507 18:28:10.326 | INFO | "huge" : "", | 20260507 18:28:10.326 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:10.326 | INFO | "malloc" : " 14 GB", | 20260507 18:28:10.326 | INFO | "pinned" : " 29 GB" | 20260507 18:28:10.326 | INFO | } | 20260507 18:28:10.326 | INFO | Perf report for the last 5.13969 seconds: | 20260507 18:28:10.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:10.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:28:10.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 18:28:10.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:10.326 | INFO | Uploaded 36GiB to GPU in 2.51237s (14.3291GiB/s) | 20260507 18:28:10.326 | INFO | Downloaded 34GiB to GPU in 2.42586s (14.0157GiB/s) | 20260507 18:28:15.465 | INFO | Analyzing chunk frames = [1650176,1650688) | 20260507 18:28:15.465 | INFO | MemInfo: { | 20260507 18:28:15.465 | INFO | "gpu" : " 17 GB", | 20260507 18:28:15.465 | INFO | "huge" : "", | 20260507 18:28:15.465 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:15.465 | INFO | "malloc" : " 14 GB", | 20260507 18:28:15.465 | INFO | "pinned" : " 29 GB" | 20260507 18:28:15.465 | INFO | } | 20260507 18:28:15.465 | INFO | Perf report for the last 5.13893 seconds: | 20260507 18:28:15.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:15.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.465 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:28:15.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.465 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:28:15.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:15.466 | INFO | Uploaded 36GiB to GPU in 2.49286s (14.4412GiB/s) | 20260507 18:28:15.466 | INFO | Downloaded 34GiB to GPU in 2.41267s (14.0923GiB/s) | 20260507 18:28:15.562 | INFO | Starting a round of Baz IO | 20260507 18:28:19.085 | INFO | Populating RTMetrics report | 20260507 18:28:19.086 | INFO | RTMetrics report populated | 20260507 18:28:19.086 | INFO | Opening tmp RTMetrics json file | 20260507 18:28:19.107 | INFO | Serializing RTMetrics to json | 20260507 18:28:19.109 | INFO | Serialization done, swapping tmp file now... | 20260507 18:28:19.110 | INFO | RTMetrics done | 20260507 18:28:19.175 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.612 seconds | 20260507 18:28:19.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.57% / 0.02% / 53.41% | 20260507 18:28:19.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.94775% / 1.58946e-05% / 2.78155e-05% | 20260507 18:28:19.175 | INFO | Baz IO overall: | 20260507 18:28:19.175 | INFO | BytesWritten : 738133444127 | 20260507 18:28:19.175 | INFO | NumEvents : 357844176720 | 20260507 18:28:20.604 | INFO | Analyzing chunk frames = [1650688,1651200) | 20260507 18:28:20.604 | INFO | MemInfo: { | 20260507 18:28:20.604 | INFO | "gpu" : " 17 GB", | 20260507 18:28:20.604 | INFO | "huge" : "", | 20260507 18:28:20.604 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:20.604 | INFO | "malloc" : " 14 GB", | 20260507 18:28:20.604 | INFO | "pinned" : " 29 GB" | 20260507 18:28:20.604 | INFO | } | 20260507 18:28:20.604 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:28:20.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:20.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:20.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:20.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:20.604 | INFO | ANALYSIS : 69.7%, 3.0, 108.8ms, 0 | 20260507 18:28:20.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:20.604 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:28:20.604 | INFO | PRE_HQ : 66.4%, 1.7, 59.0ms, 0 | 20260507 18:28:20.604 | INFO | BAZWRITER : 69.8%, 3.7, 132.6ms, 0 | 20260507 18:28:20.604 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:28:20.604 | INFO | Uploaded 36GiB to GPU in 2.55539s (14.0878GiB/s) | 20260507 18:28:20.604 | INFO | Downloaded 38GiB to GPU in 2.72137s (13.9636GiB/s) | 20260507 18:28:25.744 | INFO | Analyzing chunk frames = [1651200,1651712) | 20260507 18:28:25.744 | INFO | MemInfo: { | 20260507 18:28:25.744 | INFO | "gpu" : " 17 GB", | 20260507 18:28:25.744 | INFO | "huge" : "", | 20260507 18:28:25.744 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:25.744 | INFO | "malloc" : " 14 GB", | 20260507 18:28:25.744 | INFO | "pinned" : " 29 GB" | 20260507 18:28:25.744 | INFO | } | 20260507 18:28:25.744 | INFO | Perf report for the last 5.13966 seconds: | 20260507 18:28:25.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:25.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:28:25.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:28:25.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:25.744 | INFO | Uploaded 36GiB to GPU in 2.55811s (14.0729GiB/s) | 20260507 18:28:25.744 | INFO | Downloaded 34GiB to GPU in 2.44408s (13.9112GiB/s) | 20260507 18:28:30.883 | INFO | Analyzing chunk frames = [1651712,1652224) | 20260507 18:28:30.883 | INFO | MemInfo: { | 20260507 18:28:30.883 | INFO | "gpu" : " 17 GB", | 20260507 18:28:30.883 | INFO | "huge" : "", | 20260507 18:28:30.883 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:30.883 | INFO | "malloc" : " 14 GB", | 20260507 18:28:30.883 | INFO | "pinned" : " 29 GB" | 20260507 18:28:30.883 | INFO | } | 20260507 18:28:30.883 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:28:30.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:30.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:28:30.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:28:30.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:30.883 | INFO | Uploaded 36GiB to GPU in 2.56095s (14.0573GiB/s) | 20260507 18:28:30.883 | INFO | Downloaded 34GiB to GPU in 2.42834s (14.0013GiB/s) | 20260507 18:28:36.022 | INFO | Analyzing chunk frames = [1652224,1652736) | 20260507 18:28:36.022 | INFO | MemInfo: { | 20260507 18:28:36.022 | INFO | "gpu" : " 17 GB", | 20260507 18:28:36.022 | INFO | "huge" : "", | 20260507 18:28:36.022 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:36.022 | INFO | "malloc" : " 14 GB", | 20260507 18:28:36.022 | INFO | "pinned" : " 29 GB" | 20260507 18:28:36.022 | INFO | } | 20260507 18:28:36.022 | INFO | Perf report for the last 5.13925 seconds: | 20260507 18:28:36.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:36.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.022 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:28:36.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.022 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:28:36.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:36.023 | INFO | Uploaded 36GiB to GPU in 2.50748s (14.357GiB/s) | 20260507 18:28:36.023 | INFO | Downloaded 34GiB to GPU in 2.34796s (14.4807GiB/s) | 20260507 18:28:41.161 | INFO | Analyzing chunk frames = [1652736,1653248) | 20260507 18:28:41.161 | INFO | MemInfo: { | 20260507 18:28:41.161 | INFO | "gpu" : " 17 GB", | 20260507 18:28:41.161 | INFO | "huge" : "", | 20260507 18:28:41.162 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:41.162 | INFO | "malloc" : " 14 GB", | 20260507 18:28:41.162 | INFO | "pinned" : " 29 GB" | 20260507 18:28:41.162 | INFO | } | 20260507 18:28:41.162 | INFO | Perf report for the last 5.13936 seconds: | 20260507 18:28:41.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:41.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 18:28:41.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:28:41.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:41.162 | INFO | Uploaded 36GiB to GPU in 2.36514s (15.2211GiB/s) | 20260507 18:28:41.162 | INFO | Downloaded 34GiB to GPU in 2.47346s (13.7459GiB/s) | 20260507 18:28:46.301 | INFO | Analyzing chunk frames = [1653248,1653760) | 20260507 18:28:46.301 | INFO | MemInfo: { | 20260507 18:28:46.301 | INFO | "gpu" : " 17 GB", | 20260507 18:28:46.301 | INFO | "huge" : "", | 20260507 18:28:46.301 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:46.301 | INFO | "malloc" : " 14 GB", | 20260507 18:28:46.301 | INFO | "pinned" : " 29 GB" | 20260507 18:28:46.301 | INFO | } | 20260507 18:28:46.301 | INFO | Perf report for the last 5.1396 seconds: | 20260507 18:28:46.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:46.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.301 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:28:46.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.301 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:28:46.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:46.302 | INFO | Uploaded 36GiB to GPU in 2.43445s (14.7877GiB/s) | 20260507 18:28:46.302 | INFO | Downloaded 34GiB to GPU in 2.4905s (13.6519GiB/s) | 20260507 18:28:51.440 | INFO | Analyzing chunk frames = [1653760,1654272) | 20260507 18:28:51.440 | INFO | MemInfo: { | 20260507 18:28:51.440 | INFO | "gpu" : " 17 GB", | 20260507 18:28:51.440 | INFO | "huge" : "", | 20260507 18:28:51.443 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:51.443 | INFO | "malloc" : " 14 GB", | 20260507 18:28:51.443 | INFO | "pinned" : " 29 GB" | 20260507 18:28:51.443 | INFO | } | 20260507 18:28:51.443 | INFO | Perf report for the last 5.14188 seconds: | 20260507 18:28:51.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:51.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.443 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 18:28:51.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.443 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:28:51.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:51.444 | INFO | Uploaded 36GiB to GPU in 2.50484s (14.3722GiB/s) | 20260507 18:28:51.444 | INFO | Downloaded 34GiB to GPU in 2.43056s (13.9885GiB/s) | 20260507 18:28:56.580 | INFO | Analyzing chunk frames = [1654272,1654784) | 20260507 18:28:56.580 | INFO | MemInfo: { | 20260507 18:28:56.580 | INFO | "gpu" : " 17 GB", | 20260507 18:28:56.580 | INFO | "huge" : "", | 20260507 18:28:56.580 | INFO | "huge_shared" : " 29 GB", | 20260507 18:28:56.580 | INFO | "malloc" : " 14 GB", | 20260507 18:28:56.580 | INFO | "pinned" : " 29 GB" | 20260507 18:28:56.580 | INFO | } | 20260507 18:28:56.580 | INFO | Perf report for the last 5.13658 seconds: | 20260507 18:28:56.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:28:56.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:28:56.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:28:56.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:28:56.580 | INFO | Uploaded 36GiB to GPU in 2.51172s (14.3328GiB/s) | 20260507 18:28:56.580 | INFO | Downloaded 34GiB to GPU in 2.4114s (14.0997GiB/s) | 20260507 18:28:56.580 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:28:56.580 | INFO | Total mem usage: 16.4 GiB | 20260507 18:28:56.580 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:28:56.580 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.1%) | 20260507 18:28:56.580 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:28:56.580 | INFO | High Water Breakdown: | 20260507 18:28:56.580 | INFO | 59.7% : 8.9 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:28:56.580 | INFO | 40.1% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:28:56.580 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:28:56.580 | INFO | ------------------------------------------ | 20260507 18:28:56.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:28:56.580 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:28:56.580 | INFO | Total mem usage: 26.5 GiB | 20260507 18:28:56.580 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:28:56.580 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 18:28:56.580 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 18:28:56.580 | INFO | High Water Breakdown: | 20260507 18:28:56.580 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:28:56.580 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:28:56.580 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:28:56.580 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:28:56.580 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:28:56.580 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:28:56.580 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:28:56.581 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:28:56.581 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:28:56.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:28:56.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:28:56.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:28:56.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:28:56.581 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:28:56.581 | INFO | ------------------------------------------ | 20260507 18:28:56.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:28:56.581 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:28:56.581 | INFO | Total mem usage: 27.8 GiB | 20260507 18:28:56.581 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:28:56.581 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:28:56.581 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:28:56.581 | INFO | High Water Breakdown: | 20260507 18:28:56.581 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:28:56.581 | INFO | ------------------------------------------ | 20260507 18:28:56.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:28:56.581 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:28:56.581 | INFO | Total mem usage: 15.7 GiB | 20260507 18:28:56.581 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:28:56.581 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:28:56.581 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:28:56.581 | INFO | High Water Breakdown: | 20260507 18:28:56.581 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:28:56.581 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:28:56.581 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:28:56.581 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:28:56.581 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:28:56.581 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:28:56.581 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:28:56.581 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:28:56.581 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:28:56.581 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:28:56.581 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:28:56.581 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:28:56.581 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:28:56.581 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:28:56.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:28:56.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:28:56.581 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:28:56.581 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:28:56.581 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:28:56.581 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:28:56.581 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:28:56.581 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:28:56.581 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:28:56.581 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:28:56.581 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:28:56.581 | INFO | ------------------------------------------ | 20260507 18:28:56.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:28:56.581 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:28:56.581 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:28:56.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:28:56.581 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.65e+01, 1.44e+01 / 2.09e+00 | 20260507 18:28:56.581 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.11e-02 / 6.12e-03 | 20260507 18:28:56.581 | INFO | PerfReport: 12.09%, Baseline -- 4.05e+00 / 4.52e+00, 6.20e-01 / 2.12e-01 | 20260507 18:28:56.581 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.17e-01 / 1.01e-01 | 20260507 18:28:56.581 | INFO | PerfReport: 15.69%, DME -- 4.20e+01 / 4.50e+01, 4.14e+00 / 2.23e+00 | 20260507 18:28:56.581 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.91e+00 / 2.12e+00 | 20260507 18:28:56.581 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.49e+00 / 2.84e+00, 4.37e-01 / 2.61e-01 | 20260507 18:28:56.581 | INFO | PerfReport: 3.72%, Metrics -- 1.24e+00 / 1.43e+00, 3.14e-01 / 1.71e-01 | 20260507 18:28:56.581 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.90e-01 / 4.19e-01, 6.31e-02 / 3.79e-02 | 20260507 18:28:56.581 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:28:56.581 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:28:56.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:28:56.582 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 24.26%, Upload -- 2.60e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 23.94%, Download -- 2.57e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 31.16%, Compute -- 3.35e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 2.03%, LockUpload -- 2.18e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 2.06%, LockDownload -- 2.21e+00 / 0.00e+00, 3.57e+00 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 16.55%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:28:56.582 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260507 18:28:56.677 | INFO | Starting a round of Baz IO | 20260507 18:29:00.199 | INFO | Populating RTMetrics report | 20260507 18:29:00.199 | INFO | RTMetrics report populated | 20260507 18:29:00.199 | INFO | Opening tmp RTMetrics json file | 20260507 18:29:00.251 | INFO | Serializing RTMetrics to json | 20260507 18:29:00.270 | INFO | Serialization done, swapping tmp file now... | 20260507 18:29:00.270 | INFO | RTMetrics done | 20260507 18:29:00.347 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.669 seconds | 20260507 18:29:00.347 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.56% / 0.02% / 53.43% | 20260507 18:29:00.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.93545% / 1.58946e-05% / 5.96046e-05% | 20260507 18:29:00.347 | INFO | Baz IO overall: | 20260507 18:29:00.347 | INFO | BytesWritten : 740071475375 | 20260507 18:29:00.347 | INFO | NumEvents : 358784863029 | 20260507 18:29:01.719 | INFO | Analyzing chunk frames = [1654784,1655296) | 20260507 18:29:01.719 | INFO | MemInfo: { | 20260507 18:29:01.719 | INFO | "gpu" : " 17 GB", | 20260507 18:29:01.719 | INFO | "huge" : "", | 20260507 18:29:01.719 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:01.719 | INFO | "malloc" : " 14 GB", | 20260507 18:29:01.719 | INFO | "pinned" : " 29 GB" | 20260507 18:29:01.719 | INFO | } | 20260507 18:29:01.719 | INFO | Perf report for the last 5.13961 seconds: | 20260507 18:29:01.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:01.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:01.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:01.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:01.720 | INFO | ANALYSIS : 69.8%, 3.0, 108.6ms, 0 | 20260507 18:29:01.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:01.720 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 18:29:01.720 | INFO | PRE_HQ : 66.7%, 1.7, 59.5ms, 0 | 20260507 18:29:01.720 | INFO | BAZWRITER : 70.8%, 3.8, 139.2ms, 0 | 20260507 18:29:01.720 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 18:29:01.720 | INFO | Uploaded 36GiB to GPU in 2.58184s (13.9435GiB/s) | 20260507 18:29:01.720 | INFO | Downloaded 38GiB to GPU in 2.72802s (13.9295GiB/s) | 20260507 18:29:06.859 | INFO | Analyzing chunk frames = [1655296,1655808) | 20260507 18:29:06.859 | INFO | MemInfo: { | 20260507 18:29:06.859 | INFO | "gpu" : " 17 GB", | 20260507 18:29:06.859 | INFO | "huge" : "", | 20260507 18:29:06.859 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:06.859 | INFO | "malloc" : " 14 GB", | 20260507 18:29:06.859 | INFO | "pinned" : " 29 GB" | 20260507 18:29:06.859 | INFO | } | 20260507 18:29:06.859 | INFO | Perf report for the last 5.13961 seconds: | 20260507 18:29:06.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:06.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:29:06.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:29:06.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:06.859 | INFO | Uploaded 36GiB to GPU in 2.56108s (14.0566GiB/s) | 20260507 18:29:06.859 | INFO | Downloaded 34GiB to GPU in 2.43981s (13.9355GiB/s) | 20260507 18:29:12.001 | INFO | Analyzing chunk frames = [1655808,1656320) | 20260507 18:29:12.001 | INFO | MemInfo: { | 20260507 18:29:12.001 | INFO | "gpu" : " 17 GB", | 20260507 18:29:12.001 | INFO | "huge" : "", | 20260507 18:29:12.001 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:12.001 | INFO | "malloc" : " 14 GB", | 20260507 18:29:12.001 | INFO | "pinned" : " 29 GB" | 20260507 18:29:12.001 | INFO | } | 20260507 18:29:12.001 | INFO | Perf report for the last 5.14174 seconds: | 20260507 18:29:12.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:12.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:29:12.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:29:12.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:12.001 | INFO | Uploaded 36GiB to GPU in 2.57495s (13.9808GiB/s) | 20260507 18:29:12.001 | INFO | Downloaded 34GiB to GPU in 2.42426s (14.0249GiB/s) | 20260507 18:29:17.137 | INFO | Analyzing chunk frames = [1656320,1656832) | 20260507 18:29:17.137 | INFO | MemInfo: { | 20260507 18:29:17.137 | INFO | "gpu" : " 17 GB", | 20260507 18:29:17.137 | INFO | "huge" : "", | 20260507 18:29:17.137 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:17.137 | INFO | "malloc" : " 14 GB", | 20260507 18:29:17.137 | INFO | "pinned" : " 29 GB" | 20260507 18:29:17.137 | INFO | } | 20260507 18:29:17.137 | INFO | Perf report for the last 5.13651 seconds: | 20260507 18:29:17.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:17.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.137 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:29:17.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.137 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 18:29:17.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:17.138 | INFO | Uploaded 36GiB to GPU in 2.51597s (14.3086GiB/s) | 20260507 18:29:17.138 | INFO | Downloaded 34GiB to GPU in 2.35898s (14.413GiB/s) | 20260507 18:29:22.277 | INFO | Analyzing chunk frames = [1656832,1657344) | 20260507 18:29:22.277 | INFO | MemInfo: { | 20260507 18:29:22.277 | INFO | "gpu" : " 17 GB", | 20260507 18:29:22.277 | INFO | "huge" : "", | 20260507 18:29:22.277 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:22.277 | INFO | "malloc" : " 14 GB", | 20260507 18:29:22.277 | INFO | "pinned" : " 29 GB" | 20260507 18:29:22.277 | INFO | } | 20260507 18:29:22.277 | INFO | Perf report for the last 5.13978 seconds: | 20260507 18:29:22.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:22.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:29:22.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:29:22.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:22.277 | INFO | Uploaded 36GiB to GPU in 2.37497s (15.1581GiB/s) | 20260507 18:29:22.277 | INFO | Downloaded 34GiB to GPU in 2.46283s (13.8053GiB/s) | 20260507 18:29:27.416 | INFO | Analyzing chunk frames = [1657344,1657856) | 20260507 18:29:27.416 | INFO | MemInfo: { | 20260507 18:29:27.416 | INFO | "gpu" : " 17 GB", | 20260507 18:29:27.416 | INFO | "huge" : "", | 20260507 18:29:27.416 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:27.416 | INFO | "malloc" : " 14 GB", | 20260507 18:29:27.416 | INFO | "pinned" : " 29 GB" | 20260507 18:29:27.416 | INFO | } | 20260507 18:29:27.416 | INFO | Perf report for the last 5.13896 seconds: | 20260507 18:29:27.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:27.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.416 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:29:27.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.416 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:29:27.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:27.417 | INFO | Uploaded 36GiB to GPU in 2.4315s (14.8057GiB/s) | 20260507 18:29:27.417 | INFO | Downloaded 34GiB to GPU in 2.48064s (13.7061GiB/s) | 20260507 18:29:32.555 | INFO | Analyzing chunk frames = [1657856,1658368) | 20260507 18:29:32.555 | INFO | MemInfo: { | 20260507 18:29:32.555 | INFO | "gpu" : " 17 GB", | 20260507 18:29:32.555 | INFO | "huge" : "", | 20260507 18:29:32.555 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:32.555 | INFO | "malloc" : " 14 GB", | 20260507 18:29:32.555 | INFO | "pinned" : " 29 GB" | 20260507 18:29:32.555 | INFO | } | 20260507 18:29:32.556 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:29:32.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:32.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:29:32.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:29:32.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:32.556 | INFO | Uploaded 36GiB to GPU in 2.52926s (14.2334GiB/s) | 20260507 18:29:32.556 | INFO | Downloaded 34GiB to GPU in 2.42987s (13.9925GiB/s) | 20260507 18:29:37.695 | INFO | Analyzing chunk frames = [1658368,1658880) | 20260507 18:29:37.695 | INFO | MemInfo: { | 20260507 18:29:37.695 | INFO | "gpu" : " 17 GB", | 20260507 18:29:37.695 | INFO | "huge" : "", | 20260507 18:29:37.695 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:37.695 | INFO | "malloc" : " 14 GB", | 20260507 18:29:37.695 | INFO | "pinned" : " 29 GB" | 20260507 18:29:37.695 | INFO | } | 20260507 18:29:37.695 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:29:37.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:37.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:29:37.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:29:37.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:37.695 | INFO | Uploaded 36GiB to GPU in 2.51428s (14.3182GiB/s) | 20260507 18:29:37.695 | INFO | Downloaded 34GiB to GPU in 2.40927s (14.1121GiB/s) | 20260507 18:29:37.792 | INFO | Starting a round of Baz IO | 20260507 18:29:41.303 | INFO | Populating RTMetrics report | 20260507 18:29:41.303 | INFO | RTMetrics report populated | 20260507 18:29:41.303 | INFO | Opening tmp RTMetrics json file | 20260507 18:29:41.318 | INFO | Serializing RTMetrics to json | 20260507 18:29:41.320 | INFO | Serialization done, swapping tmp file now... | 20260507 18:29:41.321 | INFO | RTMetrics done | 20260507 18:29:41.391 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.598 seconds | 20260507 18:29:41.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.54% / 0.02% / 53.44% | 20260507 18:29:41.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.93031% / 1.98682e-05% / 5.16574e-05% | 20260507 18:29:41.391 | INFO | Baz IO overall: | 20260507 18:29:41.391 | INFO | BytesWritten : 742009209296 | 20260507 18:29:41.391 | INFO | NumEvents : 359725585924 | 20260507 18:29:42.834 | INFO | Analyzing chunk frames = [1658880,1659392) | 20260507 18:29:42.834 | INFO | MemInfo: { | 20260507 18:29:42.834 | INFO | "gpu" : " 17 GB", | 20260507 18:29:42.834 | INFO | "huge" : "", | 20260507 18:29:42.834 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:42.834 | INFO | "malloc" : " 14 GB", | 20260507 18:29:42.834 | INFO | "pinned" : " 29 GB" | 20260507 18:29:42.834 | INFO | } | 20260507 18:29:42.835 | INFO | Perf report for the last 5.13967 seconds: | 20260507 18:29:42.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:42.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:42.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:42.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:42.835 | INFO | ANALYSIS : 69.7%, 3.0, 108.2ms, 0 | 20260507 18:29:42.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:42.835 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:29:42.835 | INFO | PRE_HQ : 66.3%, 1.7, 59.3ms, 0 | 20260507 18:29:42.835 | INFO | BAZWRITER : 69.4%, 3.6, 129.1ms, 0 | 20260507 18:29:42.835 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:29:42.835 | INFO | Uploaded 36GiB to GPU in 2.57355s (13.9884GiB/s) | 20260507 18:29:42.835 | INFO | Downloaded 38GiB to GPU in 2.7315s (13.9118GiB/s) | 20260507 18:29:47.973 | INFO | Analyzing chunk frames = [1659392,1659904) | 20260507 18:29:47.973 | INFO | MemInfo: { | 20260507 18:29:47.973 | INFO | "gpu" : " 17 GB", | 20260507 18:29:47.973 | INFO | "huge" : "", | 20260507 18:29:47.973 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:47.974 | INFO | "malloc" : " 14 GB", | 20260507 18:29:47.974 | INFO | "pinned" : " 29 GB" | 20260507 18:29:47.974 | INFO | } | 20260507 18:29:47.974 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:29:47.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:47.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260507 18:29:47.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:29:47.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:47.974 | INFO | Uploaded 36GiB to GPU in 2.53902s (14.1787GiB/s) | 20260507 18:29:47.974 | INFO | Downloaded 34GiB to GPU in 2.44608s (13.8998GiB/s) | 20260507 18:29:53.113 | INFO | Analyzing chunk frames = [1659904,1660416) | 20260507 18:29:53.113 | INFO | MemInfo: { | 20260507 18:29:53.113 | INFO | "gpu" : " 17 GB", | 20260507 18:29:53.113 | INFO | "huge" : "", | 20260507 18:29:53.113 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:53.113 | INFO | "malloc" : " 14 GB", | 20260507 18:29:53.113 | INFO | "pinned" : " 29 GB" | 20260507 18:29:53.113 | INFO | } | 20260507 18:29:53.113 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:29:53.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:53.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260507 18:29:53.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:29:53.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:53.113 | INFO | Uploaded 36GiB to GPU in 2.55257s (14.1035GiB/s) | 20260507 18:29:53.113 | INFO | Downloaded 34GiB to GPU in 2.42542s (14.0182GiB/s) | 20260507 18:29:58.252 | INFO | Analyzing chunk frames = [1660416,1660928) | 20260507 18:29:58.252 | INFO | MemInfo: { | 20260507 18:29:58.252 | INFO | "gpu" : " 17 GB", | 20260507 18:29:58.252 | INFO | "huge" : "", | 20260507 18:29:58.252 | INFO | "huge_shared" : " 29 GB", | 20260507 18:29:58.252 | INFO | "malloc" : " 14 GB", | 20260507 18:29:58.252 | INFO | "pinned" : " 29 GB" | 20260507 18:29:58.252 | INFO | } | 20260507 18:29:58.252 | INFO | Perf report for the last 5.13915 seconds: | 20260507 18:29:58.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:29:58.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:29:58.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:29:58.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:29:58.252 | INFO | Uploaded 36GiB to GPU in 2.50055s (14.3969GiB/s) | 20260507 18:29:58.252 | INFO | Downloaded 34GiB to GPU in 2.34843s (14.4777GiB/s) | 20260507 18:30:03.392 | INFO | Analyzing chunk frames = [1660928,1661440) | 20260507 18:30:03.392 | INFO | MemInfo: { | 20260507 18:30:03.392 | INFO | "gpu" : " 17 GB", | 20260507 18:30:03.392 | INFO | "huge" : "", | 20260507 18:30:03.392 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:03.392 | INFO | "malloc" : " 14 GB", | 20260507 18:30:03.392 | INFO | "pinned" : " 29 GB" | 20260507 18:30:03.392 | INFO | } | 20260507 18:30:03.392 | INFO | Perf report for the last 5.14013 seconds: | 20260507 18:30:03.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:03.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.392 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:30:03.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.392 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:30:03.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:03.393 | INFO | Uploaded 36GiB to GPU in 2.36511s (15.2213GiB/s) | 20260507 18:30:03.393 | INFO | Downloaded 34GiB to GPU in 2.48118s (13.7032GiB/s) | 20260507 18:30:08.531 | INFO | Analyzing chunk frames = [1661440,1661952) | 20260507 18:30:08.531 | INFO | MemInfo: { | 20260507 18:30:08.531 | INFO | "gpu" : " 17 GB", | 20260507 18:30:08.531 | INFO | "huge" : "", | 20260507 18:30:08.531 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:08.531 | INFO | "malloc" : " 14 GB", | 20260507 18:30:08.531 | INFO | "pinned" : " 29 GB" | 20260507 18:30:08.531 | INFO | } | 20260507 18:30:08.531 | INFO | Perf report for the last 5.13876 seconds: | 20260507 18:30:08.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:08.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:30:08.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:30:08.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:08.531 | INFO | Uploaded 36GiB to GPU in 2.42516s (14.8444GiB/s) | 20260507 18:30:08.531 | INFO | Downloaded 34GiB to GPU in 2.49046s (13.6521GiB/s) | 20260507 18:30:13.670 | INFO | Analyzing chunk frames = [1661952,1662464) | 20260507 18:30:13.670 | INFO | MemInfo: { | 20260507 18:30:13.670 | INFO | "gpu" : " 17 GB", | 20260507 18:30:13.670 | INFO | "huge" : "", | 20260507 18:30:13.670 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:13.670 | INFO | "malloc" : " 14 GB", | 20260507 18:30:13.670 | INFO | "pinned" : " 29 GB" | 20260507 18:30:13.670 | INFO | } | 20260507 18:30:13.670 | INFO | Perf report for the last 5.1392 seconds: | 20260507 18:30:13.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:13.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.670 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:30:13.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.670 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:30:13.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:13.671 | INFO | Uploaded 36GiB to GPU in 2.50078s (14.3955GiB/s) | 20260507 18:30:13.671 | INFO | Downloaded 34GiB to GPU in 2.42873s (13.9991GiB/s) | 20260507 18:30:18.809 | INFO | Analyzing chunk frames = [1662464,1662976) | 20260507 18:30:18.809 | INFO | MemInfo: { | 20260507 18:30:18.809 | INFO | "gpu" : " 17 GB", | 20260507 18:30:18.809 | INFO | "huge" : "", | 20260507 18:30:18.809 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:18.809 | INFO | "malloc" : " 14 GB", | 20260507 18:30:18.809 | INFO | "pinned" : " 29 GB" | 20260507 18:30:18.809 | INFO | } | 20260507 18:30:18.809 | INFO | Perf report for the last 5.13906 seconds: | 20260507 18:30:18.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:18.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.809 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 18:30:18.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.809 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:30:18.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:18.810 | INFO | Uploaded 36GiB to GPU in 2.49733s (14.4154GiB/s) | 20260507 18:30:18.810 | INFO | Downloaded 34GiB to GPU in 2.4132s (14.0892GiB/s) | 20260507 18:30:18.810 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:30:18.810 | INFO | Total mem usage: 16.4 GiB | 20260507 18:30:18.810 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:30:18.810 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.5%) | 20260507 18:30:18.810 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:30:18.810 | INFO | High Water Breakdown: | 20260507 18:30:18.810 | INFO | 59.6% : 8.85 GiB ( 89.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:30:18.810 | INFO | 40.3% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:30:18.810 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:30:18.810 | INFO | ------------------------------------------ | 20260507 18:30:18.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:30:18.810 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:30:18.810 | INFO | Total mem usage: 26.5 GiB | 20260507 18:30:18.810 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:30:18.810 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:30:18.810 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:30:18.810 | INFO | High Water Breakdown: | 20260507 18:30:18.810 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:30:18.810 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:30:18.810 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:30:18.810 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:30:18.810 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:30:18.810 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:30:18.810 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:30:18.810 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:30:18.810 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:30:18.810 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:30:18.810 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:30:18.810 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:30:18.810 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:30:18.810 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:30:18.810 | INFO | ------------------------------------------ | 20260507 18:30:18.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:30:18.810 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:30:18.810 | INFO | Total mem usage: 27.8 GiB | 20260507 18:30:18.810 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:30:18.810 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:30:18.810 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:30:18.810 | INFO | High Water Breakdown: | 20260507 18:30:18.810 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:30:18.810 | INFO | ------------------------------------------ | 20260507 18:30:18.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:30:18.810 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:30:18.810 | INFO | Total mem usage: 15.7 GiB | 20260507 18:30:18.810 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:30:18.810 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:30:18.810 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:30:18.810 | INFO | High Water Breakdown: | 20260507 18:30:18.810 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:30:18.810 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:30:18.810 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:30:18.810 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:30:18.810 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:30:18.810 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:30:18.810 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:30:18.810 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:30:18.810 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:30:18.810 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:30:18.810 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:30:18.810 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:30:18.810 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:30:18.810 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:30:18.810 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:30:18.810 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:30:18.810 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:30:18.810 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:30:18.810 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:30:18.810 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:30:18.810 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:30:18.810 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:30:18.810 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:30:18.811 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:30:18.811 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:30:18.811 | INFO | ------------------------------------------ | 20260507 18:30:18.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:30:18.811 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:30:18.811 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:30:18.811 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:30:18.811 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.63e+01, 1.41e+01 / 2.35e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 0.29%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.12e-02 / 6.99e-03 | 20260507 18:30:18.811 | INFO | PerfReport: 12.07%, Baseline -- 4.03e+00 / 4.48e+00, 6.11e-01 / 2.53e-01 | 20260507 18:30:18.811 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.48e+00, 2.13e-01 / 1.12e-01 | 20260507 18:30:18.811 | INFO | PerfReport: 15.69%, DME -- 4.19e+01 / 4.49e+01, 3.97e+00 / 2.35e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.91e+00 / 2.40e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.29e-01 / 2.48e-01 | 20260507 18:30:18.811 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.42e+00, 3.14e-01 / 1.69e-01 | 20260507 18:30:18.811 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.14e-01, 6.31e-02 / 3.51e-02 | 20260507 18:30:18.811 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:30:18.811 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:30:18.811 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:30:18.811 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 24.24%, Upload -- 2.61e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 23.92%, Download -- 2.57e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 31.07%, Compute -- 3.34e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 2.02%, LockUpload -- 2.18e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 2.13%, LockDownload -- 2.29e+00 / 0.00e+00, 3.68e+00 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 16.61%, LockCompute -- 1.79e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:30:18.811 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260507 18:30:18.906 | INFO | Starting a round of Baz IO | 20260507 18:30:22.415 | INFO | Populating RTMetrics report | 20260507 18:30:22.415 | INFO | RTMetrics report populated | 20260507 18:30:22.415 | INFO | Opening tmp RTMetrics json file | 20260507 18:30:22.435 | INFO | Serializing RTMetrics to json | 20260507 18:30:22.438 | INFO | Serialization done, swapping tmp file now... | 20260507 18:30:22.438 | INFO | RTMetrics done | 20260507 18:30:22.528 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.621 seconds | 20260507 18:30:22.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.53% / 0.01% / 53.46% | 20260507 18:30:22.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.91677% / 2.78155e-05% / 5.16574e-05% | 20260507 18:30:22.528 | INFO | Baz IO overall: | 20260507 18:30:22.528 | INFO | BytesWritten : 743945618568 | 20260507 18:30:22.528 | INFO | NumEvents : 360665282601 | 20260507 18:30:23.948 | INFO | Analyzing chunk frames = [1662976,1663488) | 20260507 18:30:23.948 | INFO | MemInfo: { | 20260507 18:30:23.948 | INFO | "gpu" : " 17 GB", | 20260507 18:30:23.948 | INFO | "huge" : "", | 20260507 18:30:23.949 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:23.949 | INFO | "malloc" : " 14 GB", | 20260507 18:30:23.949 | INFO | "pinned" : " 29 GB" | 20260507 18:30:23.949 | INFO | } | 20260507 18:30:23.949 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:30:23.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:23.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:23.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:23.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:23.949 | INFO | ANALYSIS : 69.4%, 2.9, 106.6ms, 0 | 20260507 18:30:23.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:23.949 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 18:30:23.949 | INFO | PRE_HQ : 67.0%, 1.7, 60.7ms, 0 | 20260507 18:30:23.949 | INFO | BAZWRITER : 69.6%, 3.7, 133.3ms, 0 | 20260507 18:30:23.949 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:30:23.949 | INFO | Uploaded 36GiB to GPU in 2.53025s (14.2278GiB/s) | 20260507 18:30:23.949 | INFO | Downloaded 38GiB to GPU in 2.69438s (14.1034GiB/s) | 20260507 18:30:29.088 | INFO | Analyzing chunk frames = [1663488,1664000) | 20260507 18:30:29.088 | INFO | MemInfo: { | 20260507 18:30:29.088 | INFO | "gpu" : " 17 GB", | 20260507 18:30:29.088 | INFO | "huge" : "", | 20260507 18:30:29.088 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:29.088 | INFO | "malloc" : " 14 GB", | 20260507 18:30:29.088 | INFO | "pinned" : " 29 GB" | 20260507 18:30:29.088 | INFO | } | 20260507 18:30:29.088 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:30:29.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:29.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.088 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 18:30:29.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.088 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:30:29.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:29.089 | INFO | Uploaded 36GiB to GPU in 2.55688s (14.0797GiB/s) | 20260507 18:30:29.089 | INFO | Downloaded 34GiB to GPU in 2.43779s (13.947GiB/s) | 20260507 18:30:34.227 | INFO | Analyzing chunk frames = [1664000,1664512) | 20260507 18:30:34.227 | INFO | MemInfo: { | 20260507 18:30:34.227 | INFO | "gpu" : " 17 GB", | 20260507 18:30:34.227 | INFO | "huge" : "", | 20260507 18:30:34.227 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:34.227 | INFO | "malloc" : " 14 GB", | 20260507 18:30:34.227 | INFO | "pinned" : " 29 GB" | 20260507 18:30:34.227 | INFO | } | 20260507 18:30:34.228 | INFO | Perf report for the last 5.13922 seconds: | 20260507 18:30:34.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:34.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:30:34.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:30:34.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:34.228 | INFO | Uploaded 36GiB to GPU in 2.57286s (13.9922GiB/s) | 20260507 18:30:34.228 | INFO | Downloaded 34GiB to GPU in 2.4222s (14.0368GiB/s) | 20260507 18:30:39.367 | INFO | Analyzing chunk frames = [1664512,1665024) | 20260507 18:30:39.367 | INFO | MemInfo: { | 20260507 18:30:39.367 | INFO | "gpu" : " 17 GB", | 20260507 18:30:39.367 | INFO | "huge" : "", | 20260507 18:30:39.367 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:39.367 | INFO | "malloc" : " 14 GB", | 20260507 18:30:39.367 | INFO | "pinned" : " 29 GB" | 20260507 18:30:39.367 | INFO | } | 20260507 18:30:39.367 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:30:39.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:39.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:30:39.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:30:39.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:39.367 | INFO | Uploaded 36GiB to GPU in 2.5075s (14.3569GiB/s) | 20260507 18:30:39.367 | INFO | Downloaded 34GiB to GPU in 2.35156s (14.4585GiB/s) | 20260507 18:30:44.508 | INFO | Analyzing chunk frames = [1665024,1665536) | 20260507 18:30:44.508 | INFO | MemInfo: { | 20260507 18:30:44.508 | INFO | "gpu" : " 17 GB", | 20260507 18:30:44.508 | INFO | "huge" : "", | 20260507 18:30:44.508 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:44.508 | INFO | "malloc" : " 14 GB", | 20260507 18:30:44.508 | INFO | "pinned" : " 29 GB" | 20260507 18:30:44.508 | INFO | } | 20260507 18:30:44.508 | INFO | Perf report for the last 5.14145 seconds: | 20260507 18:30:44.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:44.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 18:30:44.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:30:44.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:44.508 | INFO | Uploaded 36GiB to GPU in 2.38214s (15.1125GiB/s) | 20260507 18:30:44.509 | INFO | Downloaded 34GiB to GPU in 2.46362s (13.8008GiB/s) | 20260507 18:30:49.645 | INFO | Analyzing chunk frames = [1665536,1666048) | 20260507 18:30:49.645 | INFO | MemInfo: { | 20260507 18:30:49.645 | INFO | "gpu" : " 17 GB", | 20260507 18:30:49.645 | INFO | "huge" : "", | 20260507 18:30:49.645 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:49.645 | INFO | "malloc" : " 14 GB", | 20260507 18:30:49.645 | INFO | "pinned" : " 29 GB" | 20260507 18:30:49.645 | INFO | } | 20260507 18:30:49.645 | INFO | Perf report for the last 5.13701 seconds: | 20260507 18:30:49.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:49.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.645 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:30:49.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.645 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:30:49.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:49.646 | INFO | Uploaded 36GiB to GPU in 2.4366s (14.7747GiB/s) | 20260507 18:30:49.646 | INFO | Downloaded 34GiB to GPU in 2.4757s (13.7335GiB/s) | 20260507 18:30:54.785 | INFO | Analyzing chunk frames = [1666048,1666560) | 20260507 18:30:54.785 | INFO | MemInfo: { | 20260507 18:30:54.785 | INFO | "gpu" : " 17 GB", | 20260507 18:30:54.785 | INFO | "huge" : "", | 20260507 18:30:54.785 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:54.785 | INFO | "malloc" : " 14 GB", | 20260507 18:30:54.785 | INFO | "pinned" : " 29 GB" | 20260507 18:30:54.785 | INFO | } | 20260507 18:30:54.785 | INFO | Perf report for the last 5.13962 seconds: | 20260507 18:30:54.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:54.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:30:54.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 18:30:54.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:54.785 | INFO | Uploaded 36GiB to GPU in 2.51925s (14.29GiB/s) | 20260507 18:30:54.785 | INFO | Downloaded 34GiB to GPU in 2.42453s (14.0233GiB/s) | 20260507 18:30:59.924 | INFO | Analyzing chunk frames = [1666560,1667072) | 20260507 18:30:59.924 | INFO | MemInfo: { | 20260507 18:30:59.924 | INFO | "gpu" : " 17 GB", | 20260507 18:30:59.924 | INFO | "huge" : "", | 20260507 18:30:59.924 | INFO | "huge_shared" : " 29 GB", | 20260507 18:30:59.924 | INFO | "malloc" : " 14 GB", | 20260507 18:30:59.924 | INFO | "pinned" : " 29 GB" | 20260507 18:30:59.924 | INFO | } | 20260507 18:30:59.924 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:30:59.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:30:59.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.924 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 18:30:59.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.924 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:30:59.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:30:59.925 | INFO | Uploaded 36GiB to GPU in 2.55807s (14.0731GiB/s) | 20260507 18:30:59.925 | INFO | Downloaded 34GiB to GPU in 2.49401s (13.6327GiB/s) | 20260507 18:31:00.022 | INFO | Starting a round of Baz IO | 20260507 18:31:03.528 | INFO | Populating RTMetrics report | 20260507 18:31:03.529 | INFO | RTMetrics report populated | 20260507 18:31:03.529 | INFO | Opening tmp RTMetrics json file | 20260507 18:31:03.546 | INFO | Serializing RTMetrics to json | 20260507 18:31:03.548 | INFO | Serialization done, swapping tmp file now... | 20260507 18:31:03.548 | INFO | RTMetrics done | 20260507 18:31:03.625 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.603 seconds | 20260507 18:31:03.625 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.51% / 0.01% / 53.47% | 20260507 18:31:03.625 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.91202% / 1.98682e-05% / 5.5631e-05% | 20260507 18:31:03.625 | INFO | Baz IO overall: | 20260507 18:31:03.625 | INFO | BytesWritten : 745881905949 | 20260507 18:31:03.625 | INFO | NumEvents : 361604803290 | 20260507 18:31:05.063 | INFO | Analyzing chunk frames = [1667072,1667584) | 20260507 18:31:05.063 | INFO | MemInfo: { | 20260507 18:31:05.063 | INFO | "gpu" : " 17 GB", | 20260507 18:31:05.063 | INFO | "huge" : "", | 20260507 18:31:05.063 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:05.063 | INFO | "malloc" : " 14 GB", | 20260507 18:31:05.063 | INFO | "pinned" : " 29 GB" | 20260507 18:31:05.063 | INFO | } | 20260507 18:31:05.063 | INFO | Perf report for the last 5.13903 seconds: | 20260507 18:31:05.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:05.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:05.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:05.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:05.063 | INFO | ANALYSIS : 69.7%, 3.0, 108.0ms, 0 | 20260507 18:31:05.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:05.063 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:31:05.063 | INFO | PRE_HQ : 66.6%, 1.7, 59.3ms, 0 | 20260507 18:31:05.063 | INFO | BAZWRITER : 69.6%, 3.6, 128.2ms, 0 | 20260507 18:31:05.063 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:31:05.063 | INFO | Uploaded 36GiB to GPU in 2.57819s (13.9633GiB/s) | 20260507 18:31:05.063 | INFO | Downloaded 38GiB to GPU in 2.72345s (13.9529GiB/s) | 20260507 18:31:10.202 | INFO | Analyzing chunk frames = [1667584,1668096) | 20260507 18:31:10.202 | INFO | MemInfo: { | 20260507 18:31:10.202 | INFO | "gpu" : " 17 GB", | 20260507 18:31:10.202 | INFO | "huge" : "", | 20260507 18:31:10.202 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:10.202 | INFO | "malloc" : " 14 GB", | 20260507 18:31:10.202 | INFO | "pinned" : " 29 GB" | 20260507 18:31:10.202 | INFO | } | 20260507 18:31:10.202 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:31:10.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:10.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:31:10.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:31:10.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:10.203 | INFO | Uploaded 36GiB to GPU in 2.54525s (14.144GiB/s) | 20260507 18:31:10.203 | INFO | Downloaded 34GiB to GPU in 2.43231s (13.9785GiB/s) | 20260507 18:31:15.342 | INFO | Analyzing chunk frames = [1668096,1668608) | 20260507 18:31:15.342 | INFO | MemInfo: { | 20260507 18:31:15.342 | INFO | "gpu" : " 17 GB", | 20260507 18:31:15.342 | INFO | "huge" : "", | 20260507 18:31:15.342 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:15.342 | INFO | "malloc" : " 14 GB", | 20260507 18:31:15.342 | INFO | "pinned" : " 29 GB" | 20260507 18:31:15.342 | INFO | } | 20260507 18:31:15.342 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:31:15.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:15.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:31:15.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:31:15.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:15.342 | INFO | Uploaded 36GiB to GPU in 2.57019s (14.0067GiB/s) | 20260507 18:31:15.342 | INFO | Downloaded 34GiB to GPU in 2.42545s (14.018GiB/s) | 20260507 18:31:20.481 | INFO | Analyzing chunk frames = [1668608,1669120) | 20260507 18:31:20.481 | INFO | MemInfo: { | 20260507 18:31:20.481 | INFO | "gpu" : " 17 GB", | 20260507 18:31:20.481 | INFO | "huge" : "", | 20260507 18:31:20.481 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:20.481 | INFO | "malloc" : " 14 GB", | 20260507 18:31:20.481 | INFO | "pinned" : " 29 GB" | 20260507 18:31:20.481 | INFO | } | 20260507 18:31:20.482 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:31:20.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:20.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:31:20.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:31:20.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:20.482 | INFO | Uploaded 36GiB to GPU in 2.50729s (14.3582GiB/s) | 20260507 18:31:20.482 | INFO | Downloaded 34GiB to GPU in 2.34261s (14.5137GiB/s) | 20260507 18:31:25.620 | INFO | Analyzing chunk frames = [1669120,1669632) | 20260507 18:31:25.621 | INFO | MemInfo: { | 20260507 18:31:25.621 | INFO | "gpu" : " 17 GB", | 20260507 18:31:25.621 | INFO | "huge" : "", | 20260507 18:31:25.621 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:25.621 | INFO | "malloc" : " 14 GB", | 20260507 18:31:25.621 | INFO | "pinned" : " 29 GB" | 20260507 18:31:25.621 | INFO | } | 20260507 18:31:25.621 | INFO | Perf report for the last 5.13918 seconds: | 20260507 18:31:25.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:25.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:31:25.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:31:25.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:25.621 | INFO | Uploaded 36GiB to GPU in 2.35302s (15.2995GiB/s) | 20260507 18:31:25.621 | INFO | Downloaded 34GiB to GPU in 2.47746s (13.7237GiB/s) | 20260507 18:31:30.760 | INFO | Analyzing chunk frames = [1669632,1670144) | 20260507 18:31:30.760 | INFO | MemInfo: { | 20260507 18:31:30.760 | INFO | "gpu" : " 17 GB", | 20260507 18:31:30.760 | INFO | "huge" : "", | 20260507 18:31:30.760 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:30.760 | INFO | "malloc" : " 14 GB", | 20260507 18:31:30.760 | INFO | "pinned" : " 29 GB" | 20260507 18:31:30.760 | INFO | } | 20260507 18:31:30.760 | INFO | Perf report for the last 5.13964 seconds: | 20260507 18:31:30.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:30.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.760 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:31:30.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.760 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260507 18:31:30.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:30.761 | INFO | Uploaded 36GiB to GPU in 2.44059s (14.7505GiB/s) | 20260507 18:31:30.761 | INFO | Downloaded 34GiB to GPU in 2.49671s (13.6179GiB/s) | 20260507 18:31:35.900 | INFO | Analyzing chunk frames = [1670144,1670656) | 20260507 18:31:35.900 | INFO | MemInfo: { | 20260507 18:31:35.900 | INFO | "gpu" : " 17 GB", | 20260507 18:31:35.900 | INFO | "huge" : "", | 20260507 18:31:35.900 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:35.900 | INFO | "malloc" : " 14 GB", | 20260507 18:31:35.900 | INFO | "pinned" : " 29 GB" | 20260507 18:31:35.900 | INFO | } | 20260507 18:31:35.900 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:31:35.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:35.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:31:35.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:31:35.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:35.900 | INFO | Uploaded 36GiB to GPU in 2.51076s (14.3383GiB/s) | 20260507 18:31:35.900 | INFO | Downloaded 34GiB to GPU in 2.42193s (14.0384GiB/s) | 20260507 18:31:41.039 | INFO | Analyzing chunk frames = [1670656,1671168) | 20260507 18:31:41.039 | INFO | MemInfo: { | 20260507 18:31:41.039 | INFO | "gpu" : " 17 GB", | 20260507 18:31:41.039 | INFO | "huge" : "", | 20260507 18:31:41.039 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:41.039 | INFO | "malloc" : " 14 GB", | 20260507 18:31:41.039 | INFO | "pinned" : " 29 GB" | 20260507 18:31:41.039 | INFO | } | 20260507 18:31:41.039 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:31:41.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:41.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.040 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 18:31:41.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.040 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:31:41.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:41.040 | INFO | Uploaded 36GiB to GPU in 2.5708s (14.0034GiB/s) | 20260507 18:31:41.040 | INFO | Downloaded 34GiB to GPU in 2.50395s (13.5785GiB/s) | 20260507 18:31:41.040 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:31:41.040 | INFO | Total mem usage: 16.4 GiB | 20260507 18:31:41.040 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:31:41.040 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.9%) | 20260507 18:31:41.040 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:31:41.040 | INFO | High Water Breakdown: | 20260507 18:31:41.040 | INFO | 59.6% : 8.81 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:31:41.040 | INFO | 40.3% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:31:41.040 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:31:41.040 | INFO | ------------------------------------------ | 20260507 18:31:41.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:31:41.040 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:31:41.040 | INFO | Total mem usage: 26.5 GiB | 20260507 18:31:41.040 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:31:41.040 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:31:41.040 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:31:41.040 | INFO | High Water Breakdown: | 20260507 18:31:41.040 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:31:41.040 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:31:41.040 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:31:41.040 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:31:41.041 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:31:41.041 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:31:41.041 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:31:41.041 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:31:41.041 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:31:41.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:31:41.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:31:41.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:31:41.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:31:41.041 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:31:41.041 | INFO | ------------------------------------------ | 20260507 18:31:41.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:31:41.041 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:31:41.041 | INFO | Total mem usage: 27.8 GiB | 20260507 18:31:41.041 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:31:41.041 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:31:41.041 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:31:41.041 | INFO | High Water Breakdown: | 20260507 18:31:41.041 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:31:41.041 | INFO | ------------------------------------------ | 20260507 18:31:41.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:31:41.041 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:31:41.041 | INFO | Total mem usage: 15.7 GiB | 20260507 18:31:41.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:31:41.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:31:41.041 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:31:41.041 | INFO | High Water Breakdown: | 20260507 18:31:41.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:31:41.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:31:41.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:31:41.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:31:41.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:31:41.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:31:41.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:31:41.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:31:41.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:31:41.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:31:41.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:31:41.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:31:41.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:31:41.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:31:41.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:31:41.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:31:41.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:31:41.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:31:41.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:31:41.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:31:41.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:31:41.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:31:41.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:31:41.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:31:41.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:31:41.041 | INFO | ------------------------------------------ | 20260507 18:31:41.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:31:41.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:31:41.041 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:31:41.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:31:41.041 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.67e+01, 1.44e+01 / 2.19e+00 | 20260507 18:31:41.041 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.15e-02 / 5.41e-03 | 20260507 18:31:41.041 | INFO | PerfReport: 12.07%, Baseline -- 4.02e+00 / 4.51e+00, 6.15e-01 / 1.90e-01 | 20260507 18:31:41.041 | INFO | PerfReport: 3.43%, Histogram -- 1.14e+00 / 1.49e+00, 2.14e-01 / 1.05e-01 | 20260507 18:31:41.041 | INFO | PerfReport: 15.76%, DME -- 4.20e+01 / 4.52e+01, 4.16e+00 / 2.07e+00 | 20260507 18:31:41.041 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.84e+00 / 1.95e+00 | 20260507 18:31:41.041 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.49e+00 / 2.85e+00, 4.37e-01 / 2.38e-01 | 20260507 18:31:41.041 | INFO | PerfReport: 3.72%, Metrics -- 1.24e+00 / 1.42e+00, 3.17e-01 / 1.62e-01 | 20260507 18:31:41.041 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.91e-01 / 4.18e-01, 6.47e-02 / 3.47e-02 | 20260507 18:31:41.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:31:41.042 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:31:41.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:31:41.042 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 24.31%, Upload -- 2.61e+01 / 0.00e+00, 8.63e+00 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 23.98%, Download -- 2.58e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 31.03%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 2.22%, LockUpload -- 2.38e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 2.00%, LockDownload -- 2.15e+00 / 0.00e+00, 3.51e+00 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 16.44%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:31:41.042 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 18:31:41.137 | INFO | Starting a round of Baz IO | 20260507 18:31:44.636 | INFO | Populating RTMetrics report | 20260507 18:31:44.636 | INFO | RTMetrics report populated | 20260507 18:31:44.636 | INFO | Opening tmp RTMetrics json file | 20260507 18:31:44.638 | INFO | Serializing RTMetrics to json | 20260507 18:31:44.662 | INFO | Serialization done, swapping tmp file now... | 20260507 18:31:44.668 | INFO | RTMetrics done | 20260507 18:31:44.774 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.637 seconds | 20260507 18:31:44.774 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.50% / 0.02% / 53.49% | 20260507 18:31:44.774 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.90637% / 1.19209e-05% / 4.76837e-05% | 20260507 18:31:44.774 | INFO | Baz IO overall: | 20260507 18:31:44.774 | INFO | BytesWritten : 747818839755 | 20260507 18:31:44.774 | INFO | NumEvents : 362544637166 | 20260507 18:31:46.179 | INFO | Analyzing chunk frames = [1671168,1671680) | 20260507 18:31:46.179 | INFO | MemInfo: { | 20260507 18:31:46.179 | INFO | "gpu" : " 17 GB", | 20260507 18:31:46.179 | INFO | "huge" : "", | 20260507 18:31:46.179 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:46.179 | INFO | "malloc" : " 14 GB", | 20260507 18:31:46.179 | INFO | "pinned" : " 29 GB" | 20260507 18:31:46.179 | INFO | } | 20260507 18:31:46.179 | INFO | Perf report for the last 5.13938 seconds: | 20260507 18:31:46.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:46.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:46.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:46.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:46.179 | INFO | ANALYSIS : 69.5%, 2.9, 107.4ms, 0 | 20260507 18:31:46.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:46.179 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 18:31:46.179 | INFO | PRE_HQ : 66.6%, 1.7, 59.1ms, 0 | 20260507 18:31:46.179 | INFO | BAZWRITER : 70.4%, 3.7, 132.5ms, 0 | 20260507 18:31:46.179 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 18:31:46.179 | INFO | Uploaded 36GiB to GPU in 2.56603s (14.0294GiB/s) | 20260507 18:31:46.179 | INFO | Downloaded 38GiB to GPU in 2.71257s (14.0089GiB/s) | 20260507 18:31:51.317 | INFO | Analyzing chunk frames = [1671680,1672192) | 20260507 18:31:51.317 | INFO | MemInfo: { | 20260507 18:31:51.317 | INFO | "gpu" : " 17 GB", | 20260507 18:31:51.317 | INFO | "huge" : "", | 20260507 18:31:51.317 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:51.317 | INFO | "malloc" : " 14 GB", | 20260507 18:31:51.317 | INFO | "pinned" : " 29 GB" | 20260507 18:31:51.318 | INFO | } | 20260507 18:31:51.318 | INFO | Perf report for the last 5.13871 seconds: | 20260507 18:31:51.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:51.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:31:51.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:31:51.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:51.318 | INFO | Uploaded 36GiB to GPU in 2.5303s (14.2276GiB/s) | 20260507 18:31:51.318 | INFO | Downloaded 34GiB to GPU in 2.42969s (13.9935GiB/s) | 20260507 18:31:56.457 | INFO | Analyzing chunk frames = [1672192,1672704) | 20260507 18:31:56.457 | INFO | MemInfo: { | 20260507 18:31:56.457 | INFO | "gpu" : " 17 GB", | 20260507 18:31:56.457 | INFO | "huge" : "", | 20260507 18:31:56.457 | INFO | "huge_shared" : " 29 GB", | 20260507 18:31:56.457 | INFO | "malloc" : " 14 GB", | 20260507 18:31:56.457 | INFO | "pinned" : " 29 GB" | 20260507 18:31:56.457 | INFO | } | 20260507 18:31:56.457 | INFO | Perf report for the last 5.13973 seconds: | 20260507 18:31:56.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:31:56.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.457 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260507 18:31:56.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.457 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:31:56.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:31:56.458 | INFO | Uploaded 36GiB to GPU in 2.57756s (13.9667GiB/s) | 20260507 18:31:56.458 | INFO | Downloaded 34GiB to GPU in 2.41247s (14.0934GiB/s) | 20260507 18:32:01.596 | INFO | Analyzing chunk frames = [1672704,1673216) | 20260507 18:32:01.596 | INFO | MemInfo: { | 20260507 18:32:01.596 | INFO | "gpu" : " 17 GB", | 20260507 18:32:01.596 | INFO | "huge" : "", | 20260507 18:32:01.596 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:01.596 | INFO | "malloc" : " 14 GB", | 20260507 18:32:01.596 | INFO | "pinned" : " 29 GB" | 20260507 18:32:01.597 | INFO | } | 20260507 18:32:01.597 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:32:01.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:01.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | ANALYSIS : 66.8%, 3.0, 103.6ms, 0 | 20260507 18:32:01.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:32:01.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:01.597 | INFO | Uploaded 36GiB to GPU in 2.5008s (14.3954GiB/s) | 20260507 18:32:01.597 | INFO | Downloaded 34GiB to GPU in 2.34346s (14.5085GiB/s) | 20260507 18:32:06.736 | INFO | Analyzing chunk frames = [1673216,1673728) | 20260507 18:32:06.736 | INFO | MemInfo: { | 20260507 18:32:06.736 | INFO | "gpu" : " 17 GB", | 20260507 18:32:06.736 | INFO | "huge" : "", | 20260507 18:32:06.736 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:06.736 | INFO | "malloc" : " 14 GB", | 20260507 18:32:06.737 | INFO | "pinned" : " 29 GB" | 20260507 18:32:06.737 | INFO | } | 20260507 18:32:06.737 | INFO | Perf report for the last 5.14009 seconds: | 20260507 18:32:06.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:06.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 18:32:06.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 18:32:06.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:06.737 | INFO | Uploaded 36GiB to GPU in 2.37842s (15.1361GiB/s) | 20260507 18:32:06.737 | INFO | Downloaded 34GiB to GPU in 2.47804s (13.7205GiB/s) | 20260507 18:32:11.875 | INFO | Analyzing chunk frames = [1673728,1674240) | 20260507 18:32:11.875 | INFO | MemInfo: { | 20260507 18:32:11.875 | INFO | "gpu" : " 17 GB", | 20260507 18:32:11.875 | INFO | "huge" : "", | 20260507 18:32:11.875 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:11.875 | INFO | "malloc" : " 14 GB", | 20260507 18:32:11.875 | INFO | "pinned" : " 29 GB" | 20260507 18:32:11.875 | INFO | } | 20260507 18:32:11.875 | INFO | Perf report for the last 5.13869 seconds: | 20260507 18:32:11.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:11.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.875 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:32:11.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.875 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:32:11.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:11.876 | INFO | Uploaded 36GiB to GPU in 2.43963s (14.7564GiB/s) | 20260507 18:32:11.876 | INFO | Downloaded 34GiB to GPU in 2.48313s (13.6924GiB/s) | 20260507 18:32:17.014 | INFO | Analyzing chunk frames = [1674240,1674752) | 20260507 18:32:17.014 | INFO | MemInfo: { | 20260507 18:32:17.014 | INFO | "gpu" : " 17 GB", | 20260507 18:32:17.014 | INFO | "huge" : "", | 20260507 18:32:17.014 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:17.014 | INFO | "malloc" : " 14 GB", | 20260507 18:32:17.014 | INFO | "pinned" : " 29 GB" | 20260507 18:32:17.014 | INFO | } | 20260507 18:32:17.014 | INFO | Perf report for the last 5.13904 seconds: | 20260507 18:32:17.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:17.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.014 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:32:17.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.015 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 18:32:17.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:17.015 | INFO | Uploaded 36GiB to GPU in 2.52466s (14.2593GiB/s) | 20260507 18:32:17.015 | INFO | Downloaded 34GiB to GPU in 2.42484s (14.0215GiB/s) | 20260507 18:32:22.154 | INFO | Analyzing chunk frames = [1674752,1675264) | 20260507 18:32:22.154 | INFO | MemInfo: { | 20260507 18:32:22.154 | INFO | "gpu" : " 17 GB", | 20260507 18:32:22.154 | INFO | "huge" : "", | 20260507 18:32:22.154 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:22.154 | INFO | "malloc" : " 14 GB", | 20260507 18:32:22.154 | INFO | "pinned" : " 29 GB" | 20260507 18:32:22.154 | INFO | } | 20260507 18:32:22.154 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:32:22.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:22.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 18:32:22.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 18:32:22.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:22.154 | INFO | Uploaded 36GiB to GPU in 2.55391s (14.096GiB/s) | 20260507 18:32:22.154 | INFO | Downloaded 34GiB to GPU in 2.48716s (13.6702GiB/s) | 20260507 18:32:22.251 | INFO | Starting a round of Baz IO | 20260507 18:32:25.762 | INFO | Populating RTMetrics report | 20260507 18:32:25.762 | INFO | RTMetrics report populated | 20260507 18:32:25.762 | INFO | Opening tmp RTMetrics json file | 20260507 18:32:25.901 | INFO | Serializing RTMetrics to json | 20260507 18:32:25.910 | INFO | Serialization done, swapping tmp file now... | 20260507 18:32:25.910 | INFO | RTMetrics done | 20260507 18:32:25.921 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.669 seconds | 20260507 18:32:25.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.48% / 0.01% / 53.50% | 20260507 18:32:25.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.90089% / 1.19209e-05% / 5.96046e-05% | 20260507 18:32:25.921 | INFO | Baz IO overall: | 20260507 18:32:25.921 | INFO | BytesWritten : 749756424575 | 20260507 18:32:25.921 | INFO | NumEvents : 363485184324 | 20260507 18:32:27.293 | INFO | Analyzing chunk frames = [1675264,1675776) | 20260507 18:32:27.293 | INFO | MemInfo: { | 20260507 18:32:27.293 | INFO | "gpu" : " 17 GB", | 20260507 18:32:27.293 | INFO | "huge" : "", | 20260507 18:32:27.293 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:27.293 | INFO | "malloc" : " 14 GB", | 20260507 18:32:27.293 | INFO | "pinned" : " 29 GB" | 20260507 18:32:27.293 | INFO | } | 20260507 18:32:27.293 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:32:27.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:27.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:27.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:27.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:27.293 | INFO | ANALYSIS : 69.6%, 3.0, 107.7ms, 0 | 20260507 18:32:27.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:27.293 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 18:32:27.293 | INFO | PRE_HQ : 66.3%, 1.7, 59.1ms, 0 | 20260507 18:32:27.293 | INFO | BAZWRITER : 70.9%, 4.0, 146.7ms, 0 | 20260507 18:32:27.293 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 18:32:27.293 | INFO | Uploaded 36GiB to GPU in 2.54392s (14.1514GiB/s) | 20260507 18:32:27.293 | INFO | Downloaded 38GiB to GPU in 2.71843s (13.9787GiB/s) | 20260507 18:32:32.432 | INFO | Analyzing chunk frames = [1675776,1676288) | 20260507 18:32:32.432 | INFO | MemInfo: { | 20260507 18:32:32.432 | INFO | "gpu" : " 17 GB", | 20260507 18:32:32.432 | INFO | "huge" : "", | 20260507 18:32:32.432 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:32.432 | INFO | "malloc" : " 14 GB", | 20260507 18:32:32.432 | INFO | "pinned" : " 29 GB" | 20260507 18:32:32.432 | INFO | } | 20260507 18:32:32.433 | INFO | Perf report for the last 5.13961 seconds: | 20260507 18:32:32.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:32.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:32:32.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:32:32.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:32.433 | INFO | Uploaded 36GiB to GPU in 2.55625s (14.0831GiB/s) | 20260507 18:32:32.433 | INFO | Downloaded 34GiB to GPU in 2.4457s (13.9019GiB/s) | 20260507 18:32:37.572 | INFO | Analyzing chunk frames = [1676288,1676800) | 20260507 18:32:37.572 | INFO | MemInfo: { | 20260507 18:32:37.572 | INFO | "gpu" : " 17 GB", | 20260507 18:32:37.572 | INFO | "huge" : "", | 20260507 18:32:37.572 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:37.572 | INFO | "malloc" : " 14 GB", | 20260507 18:32:37.572 | INFO | "pinned" : " 29 GB" | 20260507 18:32:37.572 | INFO | } | 20260507 18:32:37.572 | INFO | Perf report for the last 5.13922 seconds: | 20260507 18:32:37.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:37.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:32:37.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:32:37.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:37.572 | INFO | Uploaded 36GiB to GPU in 2.56172s (14.0531GiB/s) | 20260507 18:32:37.572 | INFO | Downloaded 34GiB to GPU in 2.43491s (13.9635GiB/s) | 20260507 18:32:42.711 | INFO | Analyzing chunk frames = [1676800,1677312) | 20260507 18:32:42.711 | INFO | MemInfo: { | 20260507 18:32:42.711 | INFO | "gpu" : " 17 GB", | 20260507 18:32:42.711 | INFO | "huge" : "", | 20260507 18:32:42.711 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:42.711 | INFO | "malloc" : " 14 GB", | 20260507 18:32:42.711 | INFO | "pinned" : " 29 GB" | 20260507 18:32:42.711 | INFO | } | 20260507 18:32:42.711 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:32:42.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:42.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260507 18:32:42.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:32:42.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:42.711 | INFO | Uploaded 36GiB to GPU in 2.50353s (14.3797GiB/s) | 20260507 18:32:42.711 | INFO | Downloaded 34GiB to GPU in 2.34967s (14.4701GiB/s) | 20260507 18:32:47.851 | INFO | Analyzing chunk frames = [1677312,1677824) | 20260507 18:32:47.851 | INFO | MemInfo: { | 20260507 18:32:47.851 | INFO | "gpu" : " 17 GB", | 20260507 18:32:47.851 | INFO | "huge" : "", | 20260507 18:32:47.851 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:47.851 | INFO | "malloc" : " 14 GB", | 20260507 18:32:47.851 | INFO | "pinned" : " 29 GB" | 20260507 18:32:47.851 | INFO | } | 20260507 18:32:47.851 | INFO | Perf report for the last 5.14005 seconds: | 20260507 18:32:47.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:47.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:32:47.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:32:47.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:47.851 | INFO | Uploaded 36GiB to GPU in 2.3516s (15.3087GiB/s) | 20260507 18:32:47.851 | INFO | Downloaded 34GiB to GPU in 2.46777s (13.7776GiB/s) | 20260507 18:32:52.990 | INFO | Analyzing chunk frames = [1677824,1678336) | 20260507 18:32:52.990 | INFO | MemInfo: { | 20260507 18:32:52.990 | INFO | "gpu" : " 17 GB", | 20260507 18:32:52.990 | INFO | "huge" : "", | 20260507 18:32:52.990 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:52.990 | INFO | "malloc" : " 14 GB", | 20260507 18:32:52.990 | INFO | "pinned" : " 29 GB" | 20260507 18:32:52.990 | INFO | } | 20260507 18:32:52.990 | INFO | Perf report for the last 5.13861 seconds: | 20260507 18:32:52.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:52.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:32:52.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:32:52.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:52.990 | INFO | Uploaded 36GiB to GPU in 2.42267s (14.8596GiB/s) | 20260507 18:32:52.990 | INFO | Downloaded 34GiB to GPU in 2.49464s (13.6292GiB/s) | 20260507 18:32:58.129 | INFO | Analyzing chunk frames = [1678336,1678848) | 20260507 18:32:58.129 | INFO | MemInfo: { | 20260507 18:32:58.129 | INFO | "gpu" : " 17 GB", | 20260507 18:32:58.129 | INFO | "huge" : "", | 20260507 18:32:58.129 | INFO | "huge_shared" : " 29 GB", | 20260507 18:32:58.129 | INFO | "malloc" : " 14 GB", | 20260507 18:32:58.129 | INFO | "pinned" : " 29 GB" | 20260507 18:32:58.129 | INFO | } | 20260507 18:32:58.129 | INFO | Perf report for the last 5.13966 seconds: | 20260507 18:32:58.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:32:58.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.129 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:32:58.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.129 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:32:58.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:32:58.130 | INFO | Uploaded 36GiB to GPU in 2.51915s (14.2906GiB/s) | 20260507 18:32:58.130 | INFO | Downloaded 34GiB to GPU in 2.42535s (14.0186GiB/s) | 20260507 18:33:03.268 | INFO | Analyzing chunk frames = [1678848,1679360) | 20260507 18:33:03.268 | INFO | MemInfo: { | 20260507 18:33:03.268 | INFO | "gpu" : " 17 GB", | 20260507 18:33:03.268 | INFO | "huge" : "", | 20260507 18:33:03.268 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:03.268 | INFO | "malloc" : " 14 GB", | 20260507 18:33:03.268 | INFO | "pinned" : " 29 GB" | 20260507 18:33:03.268 | INFO | } | 20260507 18:33:03.268 | INFO | Perf report for the last 5.13902 seconds: | 20260507 18:33:03.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:03.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.268 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:33:03.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.268 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:33:03.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:03.269 | INFO | Uploaded 36GiB to GPU in 2.50542s (14.3688GiB/s) | 20260507 18:33:03.269 | INFO | Downloaded 34GiB to GPU in 2.41363s (14.0866GiB/s) | 20260507 18:33:03.269 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:33:03.269 | INFO | Total mem usage: 16.4 GiB | 20260507 18:33:03.269 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:33:03.269 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.67%) | 20260507 18:33:03.269 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:33:03.269 | INFO | High Water Breakdown: | 20260507 18:33:03.269 | INFO | 59.9% : 9.05 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:33:03.269 | INFO | 39.9% : 6.03 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:33:03.269 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:33:03.269 | INFO | ------------------------------------------ | 20260507 18:33:03.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:33:03.269 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:33:03.269 | INFO | Total mem usage: 26.5 GiB | 20260507 18:33:03.269 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:33:03.269 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:33:03.269 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:33:03.269 | INFO | High Water Breakdown: | 20260507 18:33:03.269 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:33:03.269 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:33:03.269 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:33:03.269 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:33:03.269 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:33:03.269 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:33:03.269 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:33:03.269 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:33:03.269 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:33:03.269 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:33:03.269 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:33:03.269 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:33:03.269 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:33:03.269 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:33:03.269 | INFO | ------------------------------------------ | 20260507 18:33:03.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:33:03.269 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:33:03.269 | INFO | Total mem usage: 27.8 GiB | 20260507 18:33:03.269 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:33:03.269 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:33:03.269 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:33:03.269 | INFO | High Water Breakdown: | 20260507 18:33:03.269 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:33:03.269 | INFO | ------------------------------------------ | 20260507 18:33:03.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:33:03.269 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:33:03.269 | INFO | Total mem usage: 15.7 GiB | 20260507 18:33:03.269 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:33:03.269 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:33:03.269 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:33:03.269 | INFO | High Water Breakdown: | 20260507 18:33:03.269 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:33:03.269 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:33:03.269 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:33:03.269 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:33:03.269 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:33:03.269 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:33:03.269 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:33:03.269 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:33:03.269 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:33:03.269 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:33:03.269 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:33:03.269 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:33:03.269 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:33:03.269 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:33:03.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:33:03.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:33:03.269 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:33:03.269 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:33:03.269 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:33:03.269 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:33:03.269 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:33:03.269 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:33:03.269 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:33:03.269 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:33:03.269 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:33:03.269 | INFO | ------------------------------------------ | 20260507 18:33:03.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:33:03.269 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:33:03.269 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:33:03.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:33:03.269 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.67e+01, 1.44e+01 / 2.02e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.13e-02 / 5.83e-03 | 20260507 18:33:03.269 | INFO | PerfReport: 12.07%, Baseline -- 4.03e+00 / 4.48e+00, 6.14e-01 / 2.67e-01 | 20260507 18:33:03.269 | INFO | PerfReport: 3.43%, Histogram -- 1.14e+00 / 1.49e+00, 2.15e-01 / 1.18e-01 | 20260507 18:33:03.269 | INFO | PerfReport: 15.74%, DME -- 4.20e+01 / 4.52e+01, 4.28e+00 / 2.07e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.87e+00 / 2.11e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.49e+00 / 2.85e+00, 4.36e-01 / 2.34e-01 | 20260507 18:33:03.269 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.42e+00, 3.16e-01 / 1.54e-01 | 20260507 18:33:03.269 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.15e-01, 6.31e-02 / 3.06e-02 | 20260507 18:33:03.269 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:33:03.269 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:33:03.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:33:03.269 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 24.25%, Upload -- 2.61e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 23.94%, Download -- 2.57e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 31.06%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 2.15%, LockUpload -- 2.31e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 2.05%, LockDownload -- 2.20e+00 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 16.54%, LockCompute -- 1.78e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:33:03.269 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.08e-02 / 0.00e+00 | 20260507 18:33:03.365 | INFO | Starting a round of Baz IO | 20260507 18:33:06.870 | INFO | Populating RTMetrics report | 20260507 18:33:06.871 | INFO | RTMetrics report populated | 20260507 18:33:06.871 | INFO | Opening tmp RTMetrics json file | 20260507 18:33:06.916 | INFO | Serializing RTMetrics to json | 20260507 18:33:06.975 | INFO | Serialization done, swapping tmp file now... | 20260507 18:33:06.979 | INFO | RTMetrics done | 20260507 18:33:07.018 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.652 seconds | 20260507 18:33:07.018 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.47% / 0.01% / 53.52% | 20260507 18:33:07.018 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.89288% / 3.97364e-05% / 3.57628e-05% | 20260507 18:33:07.018 | INFO | Baz IO overall: | 20260507 18:33:07.018 | INFO | BytesWritten : 751694486550 | 20260507 18:33:07.018 | INFO | NumEvents : 364426245710 | 20260507 18:33:08.408 | INFO | Analyzing chunk frames = [1679360,1679872) | 20260507 18:33:08.408 | INFO | MemInfo: { | 20260507 18:33:08.408 | INFO | "gpu" : " 17 GB", | 20260507 18:33:08.408 | INFO | "huge" : "", | 20260507 18:33:08.408 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:08.408 | INFO | "malloc" : " 14 GB", | 20260507 18:33:08.408 | INFO | "pinned" : " 29 GB" | 20260507 18:33:08.408 | INFO | } | 20260507 18:33:08.408 | INFO | Perf report for the last 5.13959 seconds: | 20260507 18:33:08.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:08.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:08.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:08.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:08.408 | INFO | ANALYSIS : 69.3%, 3.0, 107.5ms, 0 | 20260507 18:33:08.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:08.408 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 18:33:08.408 | INFO | PRE_HQ : 66.0%, 1.7, 58.8ms, 0 | 20260507 18:33:08.408 | INFO | BAZWRITER : 70.5%, 3.8, 138.8ms, 0 | 20260507 18:33:08.408 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 18:33:08.408 | INFO | Uploaded 36GiB to GPU in 2.53764s (14.1864GiB/s) | 20260507 18:33:08.408 | INFO | Downloaded 38GiB to GPU in 2.69119s (14.1201GiB/s) | 20260507 18:33:13.547 | INFO | Analyzing chunk frames = [1679872,1680384) | 20260507 18:33:13.547 | INFO | MemInfo: { | 20260507 18:33:13.547 | INFO | "gpu" : " 17 GB", | 20260507 18:33:13.547 | INFO | "huge" : "", | 20260507 18:33:13.547 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:13.547 | INFO | "malloc" : " 14 GB", | 20260507 18:33:13.547 | INFO | "pinned" : " 29 GB" | 20260507 18:33:13.547 | INFO | } | 20260507 18:33:13.547 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:33:13.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:13.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.548 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 18:33:13.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.548 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260507 18:33:13.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:13.548 | INFO | Uploaded 36GiB to GPU in 2.55813s (14.0728GiB/s) | 20260507 18:33:13.548 | INFO | Downloaded 34GiB to GPU in 2.43932s (13.9383GiB/s) | 20260507 18:33:18.686 | INFO | Analyzing chunk frames = [1680384,1680896) | 20260507 18:33:18.686 | INFO | MemInfo: { | 20260507 18:33:18.686 | INFO | "gpu" : " 17 GB", | 20260507 18:33:18.686 | INFO | "huge" : "", | 20260507 18:33:18.686 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:18.686 | INFO | "malloc" : " 14 GB", | 20260507 18:33:18.686 | INFO | "pinned" : " 29 GB" | 20260507 18:33:18.686 | INFO | } | 20260507 18:33:18.686 | INFO | Perf report for the last 5.13899 seconds: | 20260507 18:33:18.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:18.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.686 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260507 18:33:18.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.687 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:33:18.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:18.687 | INFO | Uploaded 36GiB to GPU in 2.57325s (13.9901GiB/s) | 20260507 18:33:18.687 | INFO | Downloaded 34GiB to GPU in 2.41494s (14.079GiB/s) | 20260507 18:33:23.826 | INFO | Analyzing chunk frames = [1680896,1681408) | 20260507 18:33:23.826 | INFO | MemInfo: { | 20260507 18:33:23.826 | INFO | "gpu" : " 17 GB", | 20260507 18:33:23.826 | INFO | "huge" : "", | 20260507 18:33:23.826 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:23.826 | INFO | "malloc" : " 14 GB", | 20260507 18:33:23.826 | INFO | "pinned" : " 29 GB" | 20260507 18:33:23.826 | INFO | } | 20260507 18:33:23.826 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:33:23.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:23.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:33:23.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:33:23.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:23.826 | INFO | Uploaded 36GiB to GPU in 2.4998s (14.4012GiB/s) | 20260507 18:33:23.826 | INFO | Downloaded 34GiB to GPU in 2.34942s (14.4717GiB/s) | 20260507 18:33:28.966 | INFO | Analyzing chunk frames = [1681408,1681920) | 20260507 18:33:28.966 | INFO | MemInfo: { | 20260507 18:33:28.966 | INFO | "gpu" : " 17 GB", | 20260507 18:33:28.966 | INFO | "huge" : "", | 20260507 18:33:28.966 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:28.966 | INFO | "malloc" : " 14 GB", | 20260507 18:33:28.966 | INFO | "pinned" : " 29 GB" | 20260507 18:33:28.966 | INFO | } | 20260507 18:33:28.966 | INFO | Perf report for the last 5.13984 seconds: | 20260507 18:33:28.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:28.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:33:28.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:33:28.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:28.966 | INFO | Uploaded 36GiB to GPU in 2.37417s (15.1632GiB/s) | 20260507 18:33:28.966 | INFO | Downloaded 34GiB to GPU in 2.46924s (13.7694GiB/s) | 20260507 18:33:34.105 | INFO | Analyzing chunk frames = [1681920,1682432) | 20260507 18:33:34.105 | INFO | MemInfo: { | 20260507 18:33:34.105 | INFO | "gpu" : " 17 GB", | 20260507 18:33:34.105 | INFO | "huge" : "", | 20260507 18:33:34.105 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:34.105 | INFO | "malloc" : " 14 GB", | 20260507 18:33:34.105 | INFO | "pinned" : " 29 GB" | 20260507 18:33:34.105 | INFO | } | 20260507 18:33:34.105 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:33:34.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:34.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:33:34.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:33:34.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:34.105 | INFO | Uploaded 36GiB to GPU in 2.44918s (14.6988GiB/s) | 20260507 18:33:34.105 | INFO | Downloaded 34GiB to GPU in 2.49679s (13.6175GiB/s) | 20260507 18:33:39.244 | INFO | Analyzing chunk frames = [1682432,1682944) | 20260507 18:33:39.244 | INFO | MemInfo: { | 20260507 18:33:39.244 | INFO | "gpu" : " 17 GB", | 20260507 18:33:39.244 | INFO | "huge" : "", | 20260507 18:33:39.244 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:39.244 | INFO | "malloc" : " 14 GB", | 20260507 18:33:39.244 | INFO | "pinned" : " 29 GB" | 20260507 18:33:39.244 | INFO | } | 20260507 18:33:39.244 | INFO | Perf report for the last 5.13938 seconds: | 20260507 18:33:39.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:39.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:33:39.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 18:33:39.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:39.244 | INFO | Uploaded 36GiB to GPU in 2.51187s (14.332GiB/s) | 20260507 18:33:39.245 | INFO | Downloaded 34GiB to GPU in 2.42907s (13.9972GiB/s) | 20260507 18:33:44.383 | INFO | Analyzing chunk frames = [1682944,1683456) | 20260507 18:33:44.383 | INFO | MemInfo: { | 20260507 18:33:44.383 | INFO | "gpu" : " 17 GB", | 20260507 18:33:44.383 | INFO | "huge" : "", | 20260507 18:33:44.384 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:44.384 | INFO | "malloc" : " 14 GB", | 20260507 18:33:44.384 | INFO | "pinned" : " 29 GB" | 20260507 18:33:44.384 | INFO | } | 20260507 18:33:44.384 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:33:44.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:44.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:33:44.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:33:44.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:44.384 | INFO | Uploaded 36GiB to GPU in 2.50459s (14.3736GiB/s) | 20260507 18:33:44.384 | INFO | Downloaded 34GiB to GPU in 2.41152s (14.099GiB/s) | 20260507 18:33:44.497 | INFO | Starting a round of Baz IO | 20260507 18:33:47.995 | INFO | Populating RTMetrics report | 20260507 18:33:47.995 | INFO | RTMetrics report populated | 20260507 18:33:47.995 | INFO | Opening tmp RTMetrics json file | 20260507 18:33:48.010 | INFO | Serializing RTMetrics to json | 20260507 18:33:48.014 | INFO | Serialization done, swapping tmp file now... | 20260507 18:33:48.014 | INFO | RTMetrics done | 20260507 18:33:48.091 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.594 seconds | 20260507 18:33:48.091 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.45% / 0.01% / 53.53% | 20260507 18:33:48.091 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.88559% / 3.17891e-05% / 6.75519e-05% | 20260507 18:33:48.091 | INFO | Baz IO overall: | 20260507 18:33:48.091 | INFO | BytesWritten : 753630992327 | 20260507 18:33:48.091 | INFO | NumEvents : 365366100412 | 20260507 18:33:49.523 | INFO | Analyzing chunk frames = [1683456,1683968) | 20260507 18:33:49.523 | INFO | MemInfo: { | 20260507 18:33:49.523 | INFO | "gpu" : " 17 GB", | 20260507 18:33:49.523 | INFO | "huge" : "", | 20260507 18:33:49.523 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:49.523 | INFO | "malloc" : " 14 GB", | 20260507 18:33:49.523 | INFO | "pinned" : " 29 GB" | 20260507 18:33:49.523 | INFO | } | 20260507 18:33:49.523 | INFO | Perf report for the last 5.13915 seconds: | 20260507 18:33:49.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:49.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:49.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:49.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:49.523 | INFO | ANALYSIS : 69.5%, 2.9, 106.5ms, 0 | 20260507 18:33:49.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:49.523 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 18:33:49.523 | INFO | PRE_HQ : 67.3%, 1.8, 62.6ms, 0 | 20260507 18:33:49.523 | INFO | BAZWRITER : 69.6%, 4.0, 143.4ms, 0 | 20260507 18:33:49.523 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:33:49.523 | INFO | Uploaded 36GiB to GPU in 2.53845s (14.1819GiB/s) | 20260507 18:33:49.523 | INFO | Downloaded 38GiB to GPU in 2.69466s (14.102GiB/s) | 20260507 18:33:54.662 | INFO | Analyzing chunk frames = [1683968,1684480) | 20260507 18:33:54.662 | INFO | MemInfo: { | 20260507 18:33:54.662 | INFO | "gpu" : " 17 GB", | 20260507 18:33:54.662 | INFO | "huge" : "", | 20260507 18:33:54.662 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:54.662 | INFO | "malloc" : " 14 GB", | 20260507 18:33:54.662 | INFO | "pinned" : " 29 GB" | 20260507 18:33:54.663 | INFO | } | 20260507 18:33:54.663 | INFO | Perf report for the last 5.13975 seconds: | 20260507 18:33:54.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:54.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:33:54.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:33:54.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:54.663 | INFO | Uploaded 36GiB to GPU in 2.55376s (14.0969GiB/s) | 20260507 18:33:54.663 | INFO | Downloaded 34GiB to GPU in 2.44063s (13.9308GiB/s) | 20260507 18:33:59.801 | INFO | Analyzing chunk frames = [1684480,1684992) | 20260507 18:33:59.801 | INFO | MemInfo: { | 20260507 18:33:59.801 | INFO | "gpu" : " 17 GB", | 20260507 18:33:59.801 | INFO | "huge" : "", | 20260507 18:33:59.801 | INFO | "huge_shared" : " 29 GB", | 20260507 18:33:59.801 | INFO | "malloc" : " 14 GB", | 20260507 18:33:59.801 | INFO | "pinned" : " 29 GB" | 20260507 18:33:59.801 | INFO | } | 20260507 18:33:59.801 | INFO | Perf report for the last 5.13888 seconds: | 20260507 18:33:59.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:33:59.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.801 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260507 18:33:59.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.801 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:33:59.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:33:59.802 | INFO | Uploaded 36GiB to GPU in 2.57478s (13.9818GiB/s) | 20260507 18:33:59.802 | INFO | Downloaded 34GiB to GPU in 2.40207s (14.1545GiB/s) | 20260507 18:34:04.941 | INFO | Analyzing chunk frames = [1684992,1685504) | 20260507 18:34:04.941 | INFO | MemInfo: { | 20260507 18:34:04.941 | INFO | "gpu" : " 17 GB", | 20260507 18:34:04.941 | INFO | "huge" : "", | 20260507 18:34:04.941 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:04.941 | INFO | "malloc" : " 14 GB", | 20260507 18:34:04.941 | INFO | "pinned" : " 29 GB" | 20260507 18:34:04.941 | INFO | } | 20260507 18:34:04.941 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:34:04.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:04.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.941 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:34:04.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.941 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:34:04.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:04.942 | INFO | Uploaded 36GiB to GPU in 2.50734s (14.3579GiB/s) | 20260507 18:34:04.942 | INFO | Downloaded 34GiB to GPU in 2.35187s (14.4566GiB/s) | 20260507 18:34:10.080 | INFO | Analyzing chunk frames = [1685504,1686016) | 20260507 18:34:10.080 | INFO | MemInfo: { | 20260507 18:34:10.080 | INFO | "gpu" : " 17 GB", | 20260507 18:34:10.081 | INFO | "huge" : "", | 20260507 18:34:10.081 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:10.081 | INFO | "malloc" : " 14 GB", | 20260507 18:34:10.081 | INFO | "pinned" : " 29 GB" | 20260507 18:34:10.081 | INFO | } | 20260507 18:34:10.081 | INFO | Perf report for the last 5.13963 seconds: | 20260507 18:34:10.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:10.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 18:34:10.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:34:10.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:10.081 | INFO | Uploaded 36GiB to GPU in 2.38021s (15.1247GiB/s) | 20260507 18:34:10.081 | INFO | Downloaded 34GiB to GPU in 2.47069s (13.7613GiB/s) | 20260507 18:34:15.220 | INFO | Analyzing chunk frames = [1686016,1686528) | 20260507 18:34:15.220 | INFO | MemInfo: { | 20260507 18:34:15.220 | INFO | "gpu" : " 17 GB", | 20260507 18:34:15.220 | INFO | "huge" : "", | 20260507 18:34:15.220 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:15.220 | INFO | "malloc" : " 14 GB", | 20260507 18:34:15.220 | INFO | "pinned" : " 29 GB" | 20260507 18:34:15.220 | INFO | } | 20260507 18:34:15.220 | INFO | Perf report for the last 5.13899 seconds: | 20260507 18:34:15.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:15.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:34:15.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:34:15.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:15.220 | INFO | Uploaded 36GiB to GPU in 2.46475s (14.606GiB/s) | 20260507 18:34:15.220 | INFO | Downloaded 34GiB to GPU in 2.48488s (13.6828GiB/s) | 20260507 18:34:20.359 | INFO | Analyzing chunk frames = [1686528,1687040) | 20260507 18:34:20.359 | INFO | MemInfo: { | 20260507 18:34:20.359 | INFO | "gpu" : " 17 GB", | 20260507 18:34:20.359 | INFO | "huge" : "", | 20260507 18:34:20.359 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:20.359 | INFO | "malloc" : " 14 GB", | 20260507 18:34:20.359 | INFO | "pinned" : " 29 GB" | 20260507 18:34:20.359 | INFO | } | 20260507 18:34:20.359 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:34:20.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:20.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:34:20.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:34:20.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:20.359 | INFO | Uploaded 36GiB to GPU in 2.49575s (14.4245GiB/s) | 20260507 18:34:20.359 | INFO | Downloaded 34GiB to GPU in 2.42584s (14.0157GiB/s) | 20260507 18:34:25.498 | INFO | Analyzing chunk frames = [1687040,1687552) | 20260507 18:34:25.498 | INFO | MemInfo: { | 20260507 18:34:25.498 | INFO | "gpu" : " 17 GB", | 20260507 18:34:25.498 | INFO | "huge" : "", | 20260507 18:34:25.498 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:25.498 | INFO | "malloc" : " 14 GB", | 20260507 18:34:25.498 | INFO | "pinned" : " 29 GB" | 20260507 18:34:25.498 | INFO | } | 20260507 18:34:25.498 | INFO | Perf report for the last 5.13889 seconds: | 20260507 18:34:25.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:25.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:34:25.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:34:25.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:25.498 | INFO | Uploaded 36GiB to GPU in 2.52596s (14.252GiB/s) | 20260507 18:34:25.498 | INFO | Downloaded 34GiB to GPU in 2.40725s (14.124GiB/s) | 20260507 18:34:25.498 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:34:25.498 | INFO | Total mem usage: 16.4 GiB | 20260507 18:34:25.498 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:34:25.498 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 18:34:25.498 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:34:25.498 | INFO | High Water Breakdown: | 20260507 18:34:25.498 | INFO | 59.7% : 8.88 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:34:25.498 | INFO | 40.2% : 5.98 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:34:25.498 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:34:25.498 | INFO | ------------------------------------------ | 20260507 18:34:25.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:34:25.499 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:34:25.499 | INFO | Total mem usage: 26.5 GiB | 20260507 18:34:25.499 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:34:25.499 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:34:25.499 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:34:25.499 | INFO | High Water Breakdown: | 20260507 18:34:25.499 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:34:25.499 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:34:25.499 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:34:25.499 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:34:25.499 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:34:25.499 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:34:25.499 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:34:25.499 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:34:25.499 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:34:25.499 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:34:25.499 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:34:25.499 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:34:25.499 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:34:25.499 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:34:25.499 | INFO | ------------------------------------------ | 20260507 18:34:25.499 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:34:25.499 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:34:25.499 | INFO | Total mem usage: 27.8 GiB | 20260507 18:34:25.499 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:34:25.499 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:34:25.499 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:34:25.499 | INFO | High Water Breakdown: | 20260507 18:34:25.499 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:34:25.499 | INFO | ------------------------------------------ | 20260507 18:34:25.499 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:34:25.499 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:34:25.499 | INFO | Total mem usage: 15.7 GiB | 20260507 18:34:25.499 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:34:25.499 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:34:25.499 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:34:25.499 | INFO | High Water Breakdown: | 20260507 18:34:25.499 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:34:25.499 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:34:25.499 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:34:25.499 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:34:25.499 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:34:25.499 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:34:25.499 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:34:25.499 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:34:25.499 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:34:25.499 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:34:25.499 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:34:25.499 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:34:25.499 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:34:25.499 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:34:25.499 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:34:25.499 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:34:25.499 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:34:25.499 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:34:25.499 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:34:25.499 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:34:25.499 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:34:25.499 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:34:25.499 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:34:25.499 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:34:25.499 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:34:25.499 | INFO | ------------------------------------------ | 20260507 18:34:25.499 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:34:25.499 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:34:25.499 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:34:25.499 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:34:25.499 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.44e+01 / 2.40e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 0.29%, AggregateStats -- 9.85e-02 / 1.05e-01, 1.22e-02 / 6.65e-03 | 20260507 18:34:25.499 | INFO | PerfReport: 12.07%, Baseline -- 4.04e+00 / 4.50e+00, 6.13e-01 / 2.71e-01 | 20260507 18:34:25.499 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 1.28e-01 | 20260507 18:34:25.499 | INFO | PerfReport: 15.67%, DME -- 4.19e+01 / 4.47e+01, 4.03e+00 / 2.09e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.93e+00 / 2.09e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.48e+00 / 2.84e+00, 4.30e-01 / 2.36e-01 | 20260507 18:34:25.499 | INFO | PerfReport: 3.71%, Metrics -- 1.24e+00 / 1.43e+00, 3.22e-01 / 1.91e-01 | 20260507 18:34:25.499 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.90e-01 / 4.20e-01, 6.41e-02 / 3.61e-02 | 20260507 18:34:25.499 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:34:25.499 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:34:25.499 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:34:25.499 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 24.28%, Upload -- 2.61e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 23.87%, Download -- 2.56e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 31.15%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 2.12%, LockUpload -- 2.28e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 2.02%, LockDownload -- 2.17e+00 / 0.00e+00, 3.58e+00 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 16.55%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:34:25.499 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 18:34:25.595 | INFO | Starting a round of Baz IO | 20260507 18:34:29.113 | INFO | Populating RTMetrics report | 20260507 18:34:29.113 | INFO | RTMetrics report populated | 20260507 18:34:29.113 | INFO | Opening tmp RTMetrics json file | 20260507 18:34:29.114 | INFO | Serializing RTMetrics to json | 20260507 18:34:29.115 | INFO | Serialization done, swapping tmp file now... | 20260507 18:34:29.115 | INFO | RTMetrics done | 20260507 18:34:29.219 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.624 seconds | 20260507 18:34:29.219 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.44% / 0.02% / 53.55% | 20260507 18:34:29.219 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.88009% / 1.58946e-05% / 4.37101e-05% | 20260507 18:34:29.219 | INFO | Baz IO overall: | 20260507 18:34:29.219 | INFO | BytesWritten : 755567951167 | 20260507 18:34:29.221 | INFO | NumEvents : 366306098101 | 20260507 18:34:30.637 | INFO | Analyzing chunk frames = [1687552,1688064) | 20260507 18:34:30.637 | INFO | MemInfo: { | 20260507 18:34:30.637 | INFO | "gpu" : " 17 GB", | 20260507 18:34:30.637 | INFO | "huge" : "", | 20260507 18:34:30.637 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:30.637 | INFO | "malloc" : " 14 GB", | 20260507 18:34:30.637 | INFO | "pinned" : " 29 GB" | 20260507 18:34:30.637 | INFO | } | 20260507 18:34:30.638 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:34:30.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:30.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:30.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:30.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:30.638 | INFO | ANALYSIS : 69.4%, 3.0, 107.6ms, 0 | 20260507 18:34:30.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:30.638 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:34:30.638 | INFO | PRE_HQ : 66.4%, 1.7, 59.5ms, 0 | 20260507 18:34:30.638 | INFO | BAZWRITER : 70.3%, 3.7, 134.4ms, 0 | 20260507 18:34:30.638 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:34:30.638 | INFO | Uploaded 36GiB to GPU in 2.51488s (14.3148GiB/s) | 20260507 18:34:30.638 | INFO | Downloaded 38GiB to GPU in 2.70332s (14.0568GiB/s) | 20260507 18:34:35.777 | INFO | Analyzing chunk frames = [1688064,1688576) | 20260507 18:34:35.777 | INFO | MemInfo: { | 20260507 18:34:35.777 | INFO | "gpu" : " 17 GB", | 20260507 18:34:35.777 | INFO | "huge" : "", | 20260507 18:34:35.777 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:35.777 | INFO | "malloc" : " 14 GB", | 20260507 18:34:35.777 | INFO | "pinned" : " 29 GB" | 20260507 18:34:35.777 | INFO | } | 20260507 18:34:35.777 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:34:35.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:35.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:34:35.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:34:35.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:35.777 | INFO | Uploaded 36GiB to GPU in 2.54495s (14.1456GiB/s) | 20260507 18:34:35.777 | INFO | Downloaded 34GiB to GPU in 2.45056s (13.8744GiB/s) | 20260507 18:34:40.916 | INFO | Analyzing chunk frames = [1688576,1689088) | 20260507 18:34:40.916 | INFO | MemInfo: { | 20260507 18:34:40.916 | INFO | "gpu" : " 17 GB", | 20260507 18:34:40.916 | INFO | "huge" : "", | 20260507 18:34:40.916 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:40.916 | INFO | "malloc" : " 14 GB", | 20260507 18:34:40.916 | INFO | "pinned" : " 29 GB" | 20260507 18:34:40.916 | INFO | } | 20260507 18:34:40.916 | INFO | Perf report for the last 5.13881 seconds: | 20260507 18:34:40.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:40.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:34:40.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:34:40.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:40.916 | INFO | Uploaded 36GiB to GPU in 2.54455s (14.1479GiB/s) | 20260507 18:34:40.916 | INFO | Downloaded 34GiB to GPU in 2.44799s (13.889GiB/s) | 20260507 18:34:46.055 | INFO | Analyzing chunk frames = [1689088,1689600) | 20260507 18:34:46.055 | INFO | MemInfo: { | 20260507 18:34:46.055 | INFO | "gpu" : " 17 GB", | 20260507 18:34:46.055 | INFO | "huge" : "", | 20260507 18:34:46.055 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:46.055 | INFO | "malloc" : " 14 GB", | 20260507 18:34:46.055 | INFO | "pinned" : " 29 GB" | 20260507 18:34:46.055 | INFO | } | 20260507 18:34:46.055 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:34:46.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:46.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.055 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:34:46.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.055 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:34:46.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:46.056 | INFO | Uploaded 36GiB to GPU in 2.48375s (14.4942GiB/s) | 20260507 18:34:46.056 | INFO | Downloaded 34GiB to GPU in 2.35634s (14.4291GiB/s) | 20260507 18:34:51.195 | INFO | Analyzing chunk frames = [1689600,1690112) | 20260507 18:34:51.195 | INFO | MemInfo: { | 20260507 18:34:51.195 | INFO | "gpu" : " 17 GB", | 20260507 18:34:51.195 | INFO | "huge" : "", | 20260507 18:34:51.195 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:51.195 | INFO | "malloc" : " 14 GB", | 20260507 18:34:51.195 | INFO | "pinned" : " 29 GB" | 20260507 18:34:51.195 | INFO | } | 20260507 18:34:51.195 | INFO | Perf report for the last 5.13989 seconds: | 20260507 18:34:51.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:51.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:34:51.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:34:51.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:51.195 | INFO | Uploaded 36GiB to GPU in 2.3315s (15.4407GiB/s) | 20260507 18:34:51.195 | INFO | Downloaded 34GiB to GPU in 2.48633s (13.6748GiB/s) | 20260507 18:34:56.334 | INFO | Analyzing chunk frames = [1690112,1690624) | 20260507 18:34:56.334 | INFO | MemInfo: { | 20260507 18:34:56.334 | INFO | "gpu" : " 17 GB", | 20260507 18:34:56.334 | INFO | "huge" : "", | 20260507 18:34:56.334 | INFO | "huge_shared" : " 29 GB", | 20260507 18:34:56.334 | INFO | "malloc" : " 14 GB", | 20260507 18:34:56.334 | INFO | "pinned" : " 29 GB" | 20260507 18:34:56.334 | INFO | } | 20260507 18:34:56.334 | INFO | Perf report for the last 5.1389 seconds: | 20260507 18:34:56.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:34:56.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:34:56.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 18:34:56.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:34:56.334 | INFO | Uploaded 36GiB to GPU in 2.42534s (14.8433GiB/s) | 20260507 18:34:56.334 | INFO | Downloaded 34GiB to GPU in 2.49501s (13.6272GiB/s) | 20260507 18:35:01.473 | INFO | Analyzing chunk frames = [1690624,1691136) | 20260507 18:35:01.473 | INFO | MemInfo: { | 20260507 18:35:01.473 | INFO | "gpu" : " 17 GB", | 20260507 18:35:01.473 | INFO | "huge" : "", | 20260507 18:35:01.473 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:01.473 | INFO | "malloc" : " 14 GB", | 20260507 18:35:01.473 | INFO | "pinned" : " 29 GB" | 20260507 18:35:01.473 | INFO | } | 20260507 18:35:01.473 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:35:01.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:01.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.473 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:35:01.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.473 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:35:01.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:01.474 | INFO | Uploaded 36GiB to GPU in 2.48322s (14.4973GiB/s) | 20260507 18:35:01.474 | INFO | Downloaded 34GiB to GPU in 2.44127s (13.9272GiB/s) | 20260507 18:35:06.613 | INFO | Analyzing chunk frames = [1691136,1691648) | 20260507 18:35:06.613 | INFO | MemInfo: { | 20260507 18:35:06.613 | INFO | "gpu" : " 17 GB", | 20260507 18:35:06.613 | INFO | "huge" : "", | 20260507 18:35:06.613 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:06.613 | INFO | "malloc" : " 14 GB", | 20260507 18:35:06.613 | INFO | "pinned" : " 29 GB" | 20260507 18:35:06.613 | INFO | } | 20260507 18:35:06.613 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:35:06.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:06.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 18:35:06.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:35:06.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:06.613 | INFO | Uploaded 36GiB to GPU in 2.52567s (14.2536GiB/s) | 20260507 18:35:06.613 | INFO | Downloaded 34GiB to GPU in 2.5062s (13.5664GiB/s) | 20260507 18:35:06.709 | INFO | Starting a round of Baz IO | 20260507 18:35:10.209 | INFO | Populating RTMetrics report | 20260507 18:35:10.209 | INFO | RTMetrics report populated | 20260507 18:35:10.209 | INFO | Opening tmp RTMetrics json file | 20260507 18:35:10.216 | INFO | Serializing RTMetrics to json | 20260507 18:35:10.231 | INFO | Serialization done, swapping tmp file now... | 20260507 18:35:10.232 | INFO | RTMetrics done | 20260507 18:35:10.341 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.631 seconds | 20260507 18:35:10.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.42% / 0.02% / 53.56% | 20260507 18:35:10.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.86865% / 3.97364e-05% / 3.97364e-05% | 20260507 18:35:10.341 | INFO | Baz IO overall: | 20260507 18:35:10.341 | INFO | BytesWritten : 757504181392 | 20260507 18:35:10.341 | INFO | NumEvents : 367245403436 | 20260507 18:35:11.752 | INFO | Analyzing chunk frames = [1691648,1692160) | 20260507 18:35:11.752 | INFO | MemInfo: { | 20260507 18:35:11.752 | INFO | "gpu" : " 17 GB", | 20260507 18:35:11.752 | INFO | "huge" : "", | 20260507 18:35:11.752 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:11.752 | INFO | "malloc" : " 14 GB", | 20260507 18:35:11.752 | INFO | "pinned" : " 29 GB" | 20260507 18:35:11.752 | INFO | } | 20260507 18:35:11.752 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:35:11.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:11.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:11.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:11.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:11.752 | INFO | ANALYSIS : 69.5%, 3.0, 107.8ms, 0 | 20260507 18:35:11.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:11.752 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:35:11.752 | INFO | PRE_HQ : 66.1%, 1.7, 58.8ms, 0 | 20260507 18:35:11.752 | INFO | BAZWRITER : 70.2%, 3.7, 133.8ms, 0 | 20260507 18:35:11.752 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 18:35:11.752 | INFO | Uploaded 36GiB to GPU in 2.54404s (14.1507GiB/s) | 20260507 18:35:11.752 | INFO | Downloaded 38GiB to GPU in 2.72099s (13.9655GiB/s) | 20260507 18:35:16.891 | INFO | Analyzing chunk frames = [1692160,1692672) | 20260507 18:35:16.891 | INFO | MemInfo: { | 20260507 18:35:16.891 | INFO | "gpu" : " 17 GB", | 20260507 18:35:16.891 | INFO | "huge" : "", | 20260507 18:35:16.891 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:16.891 | INFO | "malloc" : " 14 GB", | 20260507 18:35:16.891 | INFO | "pinned" : " 29 GB" | 20260507 18:35:16.891 | INFO | } | 20260507 18:35:16.891 | INFO | Perf report for the last 5.1391 seconds: | 20260507 18:35:16.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:16.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.891 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:35:16.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.891 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:35:16.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:16.892 | INFO | Uploaded 36GiB to GPU in 2.5182s (14.2959GiB/s) | 20260507 18:35:16.892 | INFO | Downloaded 34GiB to GPU in 2.46025s (13.8197GiB/s) | 20260507 18:35:22.030 | INFO | Analyzing chunk frames = [1692672,1693184) | 20260507 18:35:22.030 | INFO | MemInfo: { | 20260507 18:35:22.030 | INFO | "gpu" : " 17 GB", | 20260507 18:35:22.030 | INFO | "huge" : "", | 20260507 18:35:22.031 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:22.031 | INFO | "malloc" : " 14 GB", | 20260507 18:35:22.031 | INFO | "pinned" : " 29 GB" | 20260507 18:35:22.031 | INFO | } | 20260507 18:35:22.031 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:35:22.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:22.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:35:22.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:35:22.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:22.031 | INFO | Uploaded 36GiB to GPU in 2.53329s (14.2107GiB/s) | 20260507 18:35:22.031 | INFO | Downloaded 34GiB to GPU in 2.45391s (13.8554GiB/s) | 20260507 18:35:27.170 | INFO | Analyzing chunk frames = [1693184,1693696) | 20260507 18:35:27.170 | INFO | MemInfo: { | 20260507 18:35:27.170 | INFO | "gpu" : " 17 GB", | 20260507 18:35:27.170 | INFO | "huge" : "", | 20260507 18:35:27.170 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:27.170 | INFO | "malloc" : " 14 GB", | 20260507 18:35:27.170 | INFO | "pinned" : " 29 GB" | 20260507 18:35:27.170 | INFO | } | 20260507 18:35:27.170 | INFO | Perf report for the last 5.13938 seconds: | 20260507 18:35:27.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:27.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:35:27.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:35:27.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:27.170 | INFO | Uploaded 36GiB to GPU in 2.47285s (14.5581GiB/s) | 20260507 18:35:27.170 | INFO | Downloaded 34GiB to GPU in 2.34857s (14.4769GiB/s) | 20260507 18:35:32.310 | INFO | Analyzing chunk frames = [1693696,1694208) | 20260507 18:35:32.310 | INFO | MemInfo: { | 20260507 18:35:32.310 | INFO | "gpu" : " 17 GB", | 20260507 18:35:32.310 | INFO | "huge" : "", | 20260507 18:35:32.310 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:32.310 | INFO | "malloc" : " 14 GB", | 20260507 18:35:32.310 | INFO | "pinned" : " 29 GB" | 20260507 18:35:32.310 | INFO | } | 20260507 18:35:32.310 | INFO | Perf report for the last 5.14032 seconds: | 20260507 18:35:32.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:32.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.310 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:35:32.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.310 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260507 18:35:32.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:32.311 | INFO | Uploaded 36GiB to GPU in 2.33464s (15.42GiB/s) | 20260507 18:35:32.311 | INFO | Downloaded 34GiB to GPU in 2.49009s (13.6541GiB/s) | 20260507 18:35:37.449 | INFO | Analyzing chunk frames = [1694208,1694720) | 20260507 18:35:37.449 | INFO | MemInfo: { | 20260507 18:35:37.449 | INFO | "gpu" : " 17 GB", | 20260507 18:35:37.449 | INFO | "huge" : "", | 20260507 18:35:37.449 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:37.449 | INFO | "malloc" : " 14 GB", | 20260507 18:35:37.449 | INFO | "pinned" : " 29 GB" | 20260507 18:35:37.449 | INFO | } | 20260507 18:35:37.449 | INFO | Perf report for the last 5.13858 seconds: | 20260507 18:35:37.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:37.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:35:37.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:35:37.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:37.449 | INFO | Uploaded 36GiB to GPU in 2.425s (14.8454GiB/s) | 20260507 18:35:37.449 | INFO | Downloaded 34GiB to GPU in 2.51172s (13.5365GiB/s) | 20260507 18:35:42.588 | INFO | Analyzing chunk frames = [1694720,1695232) | 20260507 18:35:42.588 | INFO | MemInfo: { | 20260507 18:35:42.588 | INFO | "gpu" : " 17 GB", | 20260507 18:35:42.588 | INFO | "huge" : "", | 20260507 18:35:42.588 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:42.588 | INFO | "malloc" : " 14 GB", | 20260507 18:35:42.588 | INFO | "pinned" : " 29 GB" | 20260507 18:35:42.588 | INFO | } | 20260507 18:35:42.588 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:35:42.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:42.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.588 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:35:42.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.588 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:35:42.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:42.589 | INFO | Uploaded 36GiB to GPU in 2.49104s (14.4518GiB/s) | 20260507 18:35:42.589 | INFO | Downloaded 34GiB to GPU in 2.44046s (13.9318GiB/s) | 20260507 18:35:47.727 | INFO | Analyzing chunk frames = [1695232,1695744) | 20260507 18:35:47.727 | INFO | MemInfo: { | 20260507 18:35:47.727 | INFO | "gpu" : " 17 GB", | 20260507 18:35:47.727 | INFO | "huge" : "", | 20260507 18:35:47.727 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:47.727 | INFO | "malloc" : " 14 GB", | 20260507 18:35:47.727 | INFO | "pinned" : " 29 GB" | 20260507 18:35:47.727 | INFO | } | 20260507 18:35:47.727 | INFO | Perf report for the last 5.13915 seconds: | 20260507 18:35:47.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:47.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 18:35:47.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:35:47.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:47.728 | INFO | Uploaded 36GiB to GPU in 2.52022s (14.2845GiB/s) | 20260507 18:35:47.728 | INFO | Downloaded 34GiB to GPU in 2.50921s (13.5501GiB/s) | 20260507 18:35:47.728 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:35:47.728 | INFO | Total mem usage: 16.4 GiB | 20260507 18:35:47.728 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:35:47.728 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.44%) | 20260507 18:35:47.728 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:35:47.728 | INFO | High Water Breakdown: | 20260507 18:35:47.728 | INFO | 59.8% : 8.97 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:35:47.728 | INFO | 40% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:35:47.728 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:35:47.728 | INFO | ------------------------------------------ | 20260507 18:35:47.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:35:47.728 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:35:47.728 | INFO | Total mem usage: 26.5 GiB | 20260507 18:35:47.728 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:35:47.728 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:35:47.728 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:35:47.728 | INFO | High Water Breakdown: | 20260507 18:35:47.728 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:35:47.728 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:35:47.728 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:35:47.728 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:35:47.728 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:35:47.728 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:35:47.728 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:35:47.728 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:35:47.728 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:35:47.728 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:35:47.728 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:35:47.728 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:35:47.728 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:35:47.728 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:35:47.728 | INFO | ------------------------------------------ | 20260507 18:35:47.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:35:47.728 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:35:47.728 | INFO | Total mem usage: 27.8 GiB | 20260507 18:35:47.728 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:35:47.728 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:35:47.728 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:35:47.728 | INFO | High Water Breakdown: | 20260507 18:35:47.728 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:35:47.728 | INFO | ------------------------------------------ | 20260507 18:35:47.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:35:47.729 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:35:47.729 | INFO | Total mem usage: 15.7 GiB | 20260507 18:35:47.729 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:35:47.729 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:35:47.729 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:35:47.729 | INFO | High Water Breakdown: | 20260507 18:35:47.729 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:35:47.729 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:35:47.729 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:35:47.729 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:35:47.729 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:35:47.729 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:35:47.729 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:35:47.729 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:35:47.729 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:35:47.729 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:35:47.729 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:35:47.729 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:35:47.729 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:35:47.729 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:35:47.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:35:47.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:35:47.729 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:35:47.729 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:35:47.729 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:35:47.729 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:35:47.729 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:35:47.729 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:35:47.729 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:35:47.729 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:35:47.729 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:35:47.729 | INFO | ------------------------------------------ | 20260507 18:35:47.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:35:47.729 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:35:47.729 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:35:47.729 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:35:47.729 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.68e+01, 1.45e+01 / 2.13e+00 | 20260507 18:35:47.729 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.11e-02 / 6.83e-03 | 20260507 18:35:47.729 | INFO | PerfReport: 12.07%, Baseline -- 4.03e+00 / 4.51e+00, 6.17e-01 / 1.98e-01 | 20260507 18:35:47.729 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.17e-01 / 1.02e-01 | 20260507 18:35:47.729 | INFO | PerfReport: 15.75%, DME -- 4.21e+01 / 4.52e+01, 4.17e+00 / 2.02e+00 | 20260507 18:35:47.729 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.87e+00 / 2.07e+00 | 20260507 18:35:47.729 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.33e-01 / 2.28e-01 | 20260507 18:35:47.729 | INFO | PerfReport: 3.71%, Metrics -- 1.24e+00 / 1.43e+00, 3.16e-01 / 1.84e-01 | 20260507 18:35:47.729 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.89e-01 / 4.14e-01, 6.28e-02 / 3.28e-02 | 20260507 18:35:47.729 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:35:47.729 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:35:47.729 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:35:47.729 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:35:47.729 | INFO | PerfReport: 24.06%, Upload -- 2.58e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 18:35:47.730 | INFO | PerfReport: 24.14%, Download -- 2.59e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 18:35:47.730 | INFO | PerfReport: 31.09%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:35:47.730 | INFO | PerfReport: 1.88%, LockUpload -- 2.02e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 18:35:47.730 | INFO | PerfReport: 2.30%, LockDownload -- 2.47e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 18:35:47.730 | INFO | PerfReport: 16.52%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:35:47.730 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 3.68e-02 / 0.00e+00 | 20260507 18:35:47.825 | INFO | Starting a round of Baz IO | 20260507 18:35:51.325 | INFO | Populating RTMetrics report | 20260507 18:35:51.325 | INFO | RTMetrics report populated | 20260507 18:35:51.325 | INFO | Opening tmp RTMetrics json file | 20260507 18:35:51.431 | INFO | Serializing RTMetrics to json | 20260507 18:35:51.448 | INFO | Serialization done, swapping tmp file now... | 20260507 18:35:51.448 | INFO | RTMetrics done | 20260507 18:35:51.546 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.72 seconds | 20260507 18:35:51.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.41% / 0.01% / 53.58% | 20260507 18:35:51.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.86018% / 3.97364e-05% / 5.16574e-05% | 20260507 18:35:51.546 | INFO | Baz IO overall: | 20260507 18:35:51.546 | INFO | BytesWritten : 759440785125 | 20260507 18:35:51.546 | INFO | NumEvents : 368184813296 | 20260507 18:35:52.867 | INFO | Analyzing chunk frames = [1695744,1696256) | 20260507 18:35:52.867 | INFO | MemInfo: { | 20260507 18:35:52.867 | INFO | "gpu" : " 17 GB", | 20260507 18:35:52.867 | INFO | "huge" : "", | 20260507 18:35:52.867 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:52.867 | INFO | "malloc" : " 14 GB", | 20260507 18:35:52.867 | INFO | "pinned" : " 29 GB" | 20260507 18:35:52.867 | INFO | } | 20260507 18:35:52.867 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:35:52.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:52.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:52.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:52.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:52.867 | INFO | ANALYSIS : 69.2%, 3.0, 107.8ms, 0 | 20260507 18:35:52.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:52.867 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 18:35:52.867 | INFO | PRE_HQ : 65.9%, 1.7, 58.9ms, 0 | 20260507 18:35:52.867 | INFO | BAZWRITER : 72.0%, 3.8, 142.1ms, 0 | 20260507 18:35:52.867 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 18:35:52.867 | INFO | Uploaded 36GiB to GPU in 2.53235s (14.216GiB/s) | 20260507 18:35:52.867 | INFO | Downloaded 38GiB to GPU in 2.69016s (14.1255GiB/s) | 20260507 18:35:58.006 | INFO | Analyzing chunk frames = [1696256,1696768) | 20260507 18:35:58.006 | INFO | MemInfo: { | 20260507 18:35:58.006 | INFO | "gpu" : " 17 GB", | 20260507 18:35:58.006 | INFO | "huge" : "", | 20260507 18:35:58.006 | INFO | "huge_shared" : " 29 GB", | 20260507 18:35:58.006 | INFO | "malloc" : " 14 GB", | 20260507 18:35:58.006 | INFO | "pinned" : " 29 GB" | 20260507 18:35:58.006 | INFO | } | 20260507 18:35:58.006 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:35:58.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:35:58.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.006 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:35:58.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.006 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:35:58.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:35:58.007 | INFO | Uploaded 36GiB to GPU in 2.53127s (14.2221GiB/s) | 20260507 18:35:58.007 | INFO | Downloaded 34GiB to GPU in 2.45974s (13.8226GiB/s) | 20260507 18:36:03.145 | INFO | Analyzing chunk frames = [1696768,1697280) | 20260507 18:36:03.145 | INFO | MemInfo: { | 20260507 18:36:03.145 | INFO | "gpu" : " 17 GB", | 20260507 18:36:03.146 | INFO | "huge" : "", | 20260507 18:36:03.146 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:03.146 | INFO | "malloc" : " 14 GB", | 20260507 18:36:03.146 | INFO | "pinned" : " 29 GB" | 20260507 18:36:03.146 | INFO | } | 20260507 18:36:03.146 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:36:03.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:03.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | ANALYSIS : 68.1%, 3.0, 105.5ms, 0 | 20260507 18:36:03.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 18:36:03.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:03.146 | INFO | Uploaded 36GiB to GPU in 2.54269s (14.1583GiB/s) | 20260507 18:36:03.146 | INFO | Downloaded 34GiB to GPU in 2.45129s (13.8702GiB/s) | 20260507 18:36:08.285 | INFO | Analyzing chunk frames = [1697280,1697792) | 20260507 18:36:08.285 | INFO | MemInfo: { | 20260507 18:36:08.285 | INFO | "gpu" : " 17 GB", | 20260507 18:36:08.285 | INFO | "huge" : "", | 20260507 18:36:08.285 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:08.285 | INFO | "malloc" : " 14 GB", | 20260507 18:36:08.285 | INFO | "pinned" : " 29 GB" | 20260507 18:36:08.285 | INFO | } | 20260507 18:36:08.285 | INFO | Perf report for the last 5.1396 seconds: | 20260507 18:36:08.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:08.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:36:08.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 18:36:08.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:08.286 | INFO | Uploaded 36GiB to GPU in 2.48761s (14.4717GiB/s) | 20260507 18:36:08.286 | INFO | Downloaded 34GiB to GPU in 2.37359s (14.3243GiB/s) | 20260507 18:36:13.424 | INFO | Analyzing chunk frames = [1697792,1698304) | 20260507 18:36:13.424 | INFO | MemInfo: { | 20260507 18:36:13.424 | INFO | "gpu" : " 17 GB", | 20260507 18:36:13.424 | INFO | "huge" : "", | 20260507 18:36:13.424 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:13.424 | INFO | "malloc" : " 14 GB", | 20260507 18:36:13.424 | INFO | "pinned" : " 29 GB" | 20260507 18:36:13.424 | INFO | } | 20260507 18:36:13.424 | INFO | Perf report for the last 5.13895 seconds: | 20260507 18:36:13.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:13.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.424 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 18:36:13.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.424 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 18:36:13.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:13.425 | INFO | Uploaded 36GiB to GPU in 2.32552s (15.4804GiB/s) | 20260507 18:36:13.425 | INFO | Downloaded 34GiB to GPU in 2.49691s (13.6169GiB/s) | 20260507 18:36:18.564 | INFO | Analyzing chunk frames = [1698304,1698816) | 20260507 18:36:18.564 | INFO | MemInfo: { | 20260507 18:36:18.564 | INFO | "gpu" : " 17 GB", | 20260507 18:36:18.564 | INFO | "huge" : "", | 20260507 18:36:18.564 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:18.564 | INFO | "malloc" : " 14 GB", | 20260507 18:36:18.564 | INFO | "pinned" : " 29 GB" | 20260507 18:36:18.564 | INFO | } | 20260507 18:36:18.564 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:36:18.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:18.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:36:18.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:36:18.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:18.564 | INFO | Uploaded 36GiB to GPU in 2.45381s (14.6711GiB/s) | 20260507 18:36:18.564 | INFO | Downloaded 34GiB to GPU in 2.51634s (13.5117GiB/s) | 20260507 18:36:23.703 | INFO | Analyzing chunk frames = [1698816,1699328) | 20260507 18:36:23.703 | INFO | MemInfo: { | 20260507 18:36:23.703 | INFO | "gpu" : " 17 GB", | 20260507 18:36:23.703 | INFO | "huge" : "", | 20260507 18:36:23.703 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:23.703 | INFO | "malloc" : " 14 GB", | 20260507 18:36:23.703 | INFO | "pinned" : " 29 GB" | 20260507 18:36:23.703 | INFO | } | 20260507 18:36:23.703 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:36:23.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:23.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:36:23.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:36:23.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:23.703 | INFO | Uploaded 36GiB to GPU in 2.47359s (14.5537GiB/s) | 20260507 18:36:23.704 | INFO | Downloaded 34GiB to GPU in 2.43389s (13.9694GiB/s) | 20260507 18:36:28.844 | INFO | Analyzing chunk frames = [1699328,1699840) | 20260507 18:36:28.844 | INFO | MemInfo: { | 20260507 18:36:28.844 | INFO | "gpu" : " 17 GB", | 20260507 18:36:28.844 | INFO | "huge" : "", | 20260507 18:36:28.844 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:28.844 | INFO | "malloc" : " 14 GB", | 20260507 18:36:28.844 | INFO | "pinned" : " 29 GB" | 20260507 18:36:28.845 | INFO | } | 20260507 18:36:28.845 | INFO | Perf report for the last 5.14129 seconds: | 20260507 18:36:28.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:28.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:36:28.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:36:28.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:28.845 | INFO | Uploaded 36GiB to GPU in 2.46981s (14.576GiB/s) | 20260507 18:36:28.845 | INFO | Downloaded 34GiB to GPU in 2.42159s (14.0403GiB/s) | 20260507 18:36:28.942 | INFO | Starting a round of Baz IO | 20260507 18:36:32.442 | INFO | Populating RTMetrics report | 20260507 18:36:32.442 | INFO | RTMetrics report populated | 20260507 18:36:32.442 | INFO | Opening tmp RTMetrics json file | 20260507 18:36:32.459 | INFO | Serializing RTMetrics to json | 20260507 18:36:32.461 | INFO | Serialization done, swapping tmp file now... | 20260507 18:36:32.465 | INFO | RTMetrics done | 20260507 18:36:32.563 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.62 seconds | 20260507 18:36:32.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.39% / 0.01% / 53.59% | 20260507 18:36:32.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.86007% / 3.57628e-05% / 2.38419e-05% | 20260507 18:36:32.563 | INFO | Baz IO overall: | 20260507 18:36:32.563 | INFO | BytesWritten : 761377413304 | 20260507 18:36:32.563 | INFO | NumEvents : 369124426740 | 20260507 18:36:33.982 | INFO | Analyzing chunk frames = [1699840,1700352) | 20260507 18:36:33.982 | INFO | MemInfo: { | 20260507 18:36:33.982 | INFO | "gpu" : " 17 GB", | 20260507 18:36:33.982 | INFO | "huge" : "", | 20260507 18:36:33.982 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:33.982 | INFO | "malloc" : " 14 GB", | 20260507 18:36:33.983 | INFO | "pinned" : " 29 GB" | 20260507 18:36:33.983 | INFO | } | 20260507 18:36:33.983 | INFO | Perf report for the last 5.13801 seconds: | 20260507 18:36:33.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:33.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:33.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:33.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:33.983 | INFO | ANALYSIS : 69.3%, 3.0, 107.8ms, 0 | 20260507 18:36:33.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:33.983 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:36:33.983 | INFO | PRE_HQ : 66.1%, 1.7, 58.8ms, 0 | 20260507 18:36:33.983 | INFO | BAZWRITER : 69.6%, 3.4, 120.0ms, 0 | 20260507 18:36:33.983 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:36:33.983 | INFO | Uploaded 36GiB to GPU in 2.54596s (14.1401GiB/s) | 20260507 18:36:33.983 | INFO | Downloaded 38GiB to GPU in 2.7068s (14.0387GiB/s) | 20260507 18:36:39.121 | INFO | Analyzing chunk frames = [1700352,1700864) | 20260507 18:36:39.121 | INFO | MemInfo: { | 20260507 18:36:39.121 | INFO | "gpu" : " 17 GB", | 20260507 18:36:39.121 | INFO | "huge" : "", | 20260507 18:36:39.121 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:39.121 | INFO | "malloc" : " 14 GB", | 20260507 18:36:39.121 | INFO | "pinned" : " 29 GB" | 20260507 18:36:39.121 | INFO | } | 20260507 18:36:39.121 | INFO | Perf report for the last 5.13884 seconds: | 20260507 18:36:39.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:39.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260507 18:36:39.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:36:39.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:39.122 | INFO | Uploaded 36GiB to GPU in 2.52878s (14.2361GiB/s) | 20260507 18:36:39.122 | INFO | Downloaded 34GiB to GPU in 2.46888s (13.7714GiB/s) | 20260507 18:36:44.261 | INFO | Analyzing chunk frames = [1700864,1701376) | 20260507 18:36:44.261 | INFO | MemInfo: { | 20260507 18:36:44.261 | INFO | "gpu" : " 17 GB", | 20260507 18:36:44.261 | INFO | "huge" : "", | 20260507 18:36:44.261 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:44.261 | INFO | "malloc" : " 14 GB", | 20260507 18:36:44.261 | INFO | "pinned" : " 29 GB" | 20260507 18:36:44.261 | INFO | } | 20260507 18:36:44.261 | INFO | Perf report for the last 5.13971 seconds: | 20260507 18:36:44.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:44.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.261 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:36:44.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.261 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:36:44.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:44.262 | INFO | Uploaded 36GiB to GPU in 2.53535s (14.1992GiB/s) | 20260507 18:36:44.262 | INFO | Downloaded 34GiB to GPU in 2.4502s (13.8764GiB/s) | 20260507 18:36:49.400 | INFO | Analyzing chunk frames = [1701376,1701888) | 20260507 18:36:49.400 | INFO | MemInfo: { | 20260507 18:36:49.400 | INFO | "gpu" : " 17 GB", | 20260507 18:36:49.400 | INFO | "huge" : "", | 20260507 18:36:49.400 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:49.400 | INFO | "malloc" : " 14 GB", | 20260507 18:36:49.400 | INFO | "pinned" : " 29 GB" | 20260507 18:36:49.400 | INFO | } | 20260507 18:36:49.400 | INFO | Perf report for the last 5.13887 seconds: | 20260507 18:36:49.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:49.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:36:49.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:36:49.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:49.400 | INFO | Uploaded 36GiB to GPU in 2.49722s (14.416GiB/s) | 20260507 18:36:49.400 | INFO | Downloaded 34GiB to GPU in 2.38456s (14.2584GiB/s) | 20260507 18:36:54.540 | INFO | Analyzing chunk frames = [1701888,1702400) | 20260507 18:36:54.540 | INFO | MemInfo: { | 20260507 18:36:54.540 | INFO | "gpu" : " 17 GB", | 20260507 18:36:54.540 | INFO | "huge" : "", | 20260507 18:36:54.540 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:54.540 | INFO | "malloc" : " 14 GB", | 20260507 18:36:54.540 | INFO | "pinned" : " 29 GB" | 20260507 18:36:54.540 | INFO | } | 20260507 18:36:54.540 | INFO | Perf report for the last 5.13998 seconds: | 20260507 18:36:54.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:54.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.540 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:36:54.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.540 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:36:54.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:54.541 | INFO | Uploaded 36GiB to GPU in 2.30139s (15.6427GiB/s) | 20260507 18:36:54.541 | INFO | Downloaded 34GiB to GPU in 2.4991s (13.6049GiB/s) | 20260507 18:36:59.679 | INFO | Analyzing chunk frames = [1702400,1702912) | 20260507 18:36:59.679 | INFO | MemInfo: { | 20260507 18:36:59.679 | INFO | "gpu" : " 17 GB", | 20260507 18:36:59.679 | INFO | "huge" : "", | 20260507 18:36:59.679 | INFO | "huge_shared" : " 29 GB", | 20260507 18:36:59.679 | INFO | "malloc" : " 14 GB", | 20260507 18:36:59.679 | INFO | "pinned" : " 29 GB" | 20260507 18:36:59.679 | INFO | } | 20260507 18:36:59.679 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:36:59.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:36:59.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.679 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 18:36:59.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.679 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 18:36:59.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:36:59.680 | INFO | Uploaded 36GiB to GPU in 2.52083s (14.281GiB/s) | 20260507 18:36:59.680 | INFO | Downloaded 34GiB to GPU in 2.40165s (14.1569GiB/s) | 20260507 18:37:04.818 | INFO | Analyzing chunk frames = [1702912,1703424) | 20260507 18:37:04.818 | INFO | MemInfo: { | 20260507 18:37:04.818 | INFO | "gpu" : " 17 GB", | 20260507 18:37:04.818 | INFO | "huge" : "", | 20260507 18:37:04.818 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:04.818 | INFO | "malloc" : " 14 GB", | 20260507 18:37:04.818 | INFO | "pinned" : " 29 GB" | 20260507 18:37:04.818 | INFO | } | 20260507 18:37:04.818 | INFO | Perf report for the last 5.13878 seconds: | 20260507 18:37:04.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:04.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:37:04.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 18:37:04.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:04.818 | INFO | Uploaded 36GiB to GPU in 2.52031s (14.2839GiB/s) | 20260507 18:37:04.819 | INFO | Downloaded 34GiB to GPU in 2.44338s (13.9152GiB/s) | 20260507 18:37:09.957 | INFO | Analyzing chunk frames = [1703424,1703936) | 20260507 18:37:09.957 | INFO | MemInfo: { | 20260507 18:37:09.957 | INFO | "gpu" : " 17 GB", | 20260507 18:37:09.957 | INFO | "huge" : "", | 20260507 18:37:09.958 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:09.958 | INFO | "malloc" : " 14 GB", | 20260507 18:37:09.958 | INFO | "pinned" : " 29 GB" | 20260507 18:37:09.958 | INFO | } | 20260507 18:37:09.958 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:37:09.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:09.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:37:09.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 18:37:09.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:09.958 | INFO | Uploaded 36GiB to GPU in 2.50741s (14.3575GiB/s) | 20260507 18:37:09.959 | INFO | Downloaded 34GiB to GPU in 2.42676s (14.0105GiB/s) | 20260507 18:37:09.959 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:37:09.959 | INFO | Total mem usage: 16.4 GiB | 20260507 18:37:09.959 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:37:09.959 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.7%) | 20260507 18:37:09.959 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:37:09.959 | INFO | High Water Breakdown: | 20260507 18:37:09.959 | INFO | 59.7% : 8.93 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:37:09.959 | INFO | 40.1% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:37:09.959 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:37:09.959 | INFO | ------------------------------------------ | 20260507 18:37:09.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:37:09.959 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:37:09.959 | INFO | Total mem usage: 26.5 GiB | 20260507 18:37:09.959 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:37:09.959 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:37:09.959 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:37:09.959 | INFO | High Water Breakdown: | 20260507 18:37:09.959 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:37:09.959 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:37:09.959 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:37:09.959 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:37:09.959 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:37:09.959 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:37:09.959 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:37:09.959 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:37:09.959 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:37:09.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:37:09.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:37:09.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:37:09.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:37:09.959 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:37:09.959 | INFO | ------------------------------------------ | 20260507 18:37:09.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:37:09.959 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:37:09.959 | INFO | Total mem usage: 27.8 GiB | 20260507 18:37:09.959 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:37:09.959 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:37:09.959 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:37:09.959 | INFO | High Water Breakdown: | 20260507 18:37:09.959 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:37:09.959 | INFO | ------------------------------------------ | 20260507 18:37:09.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:37:09.959 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:37:09.959 | INFO | Total mem usage: 15.7 GiB | 20260507 18:37:09.959 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:37:09.960 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:37:09.960 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:37:09.960 | INFO | High Water Breakdown: | 20260507 18:37:09.960 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:37:09.960 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:37:09.960 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:37:09.960 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:37:09.960 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:37:09.960 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:37:09.960 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:37:09.960 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:37:09.960 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:37:09.960 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:37:09.960 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:37:09.960 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:37:09.960 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:37:09.960 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:37:09.960 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:37:09.960 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:37:09.960 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:37:09.960 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:37:09.960 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:37:09.960 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:37:09.960 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:37:09.960 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:37:09.960 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:37:09.960 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:37:09.960 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:37:09.960 | INFO | ------------------------------------------ | 20260507 18:37:09.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:37:09.960 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:37:09.960 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:37:09.960 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:37:09.960 | INFO | PerfReport: Full Execution -- 3.35e+01 / 7.63e+01, 1.43e+01 / 2.13e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.05e-01, 1.13e-02 / 5.91e-03 | 20260507 18:37:09.960 | INFO | PerfReport: 12.12%, Baseline -- 4.06e+00 / 4.52e+00, 6.21e-01 / 2.29e-01 | 20260507 18:37:09.960 | INFO | PerfReport: 3.45%, Histogram -- 1.15e+00 / 1.51e+00, 2.19e-01 / 1.16e-01 | 20260507 18:37:09.960 | INFO | PerfReport: 15.59%, DME -- 4.18e+01 / 4.46e+01, 3.87e+00 / 1.86e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.88e+01 / 2.09e+01, 1.95e+00 / 2.15e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.36e-01 / 2.56e-01 | 20260507 18:37:09.960 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.43e+00, 3.15e-01 / 1.88e-01 | 20260507 18:37:09.960 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.92e-01 / 4.23e-01, 6.37e-02 / 4.28e-02 | 20260507 18:37:09.960 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:37:09.960 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:37:09.960 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:37:09.960 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 24.13%, Upload -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 24.04%, Download -- 2.58e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 31.20%, Compute -- 3.35e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 1.83%, LockUpload -- 1.96e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 2.23%, LockDownload -- 2.39e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 16.56%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:37:09.960 | INFO | PerfReport: 0.02%, Misc -- 1.69e-02 / 0.00e+00, 2.90e-02 / 0.00e+00 | 20260507 18:37:10.071 | INFO | Starting a round of Baz IO | 20260507 18:37:13.581 | INFO | Populating RTMetrics report | 20260507 18:37:13.581 | INFO | RTMetrics report populated | 20260507 18:37:13.581 | INFO | Opening tmp RTMetrics json file | 20260507 18:37:13.582 | INFO | Serializing RTMetrics to json | 20260507 18:37:13.598 | INFO | Serialization done, swapping tmp file now... | 20260507 18:37:13.602 | INFO | RTMetrics done | 20260507 18:37:13.676 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.605 seconds | 20260507 18:37:13.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.38% / 0.01% / 53.61% | 20260507 18:37:13.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.85652% / 1.58946e-05% / 4.37101e-05% | 20260507 18:37:13.676 | INFO | Baz IO overall: | 20260507 18:37:13.676 | INFO | BytesWritten : 763315437443 | 20260507 18:37:13.676 | INFO | NumEvents : 370064824608 | 20260507 18:37:15.097 | INFO | Analyzing chunk frames = [1703936,1704448) | 20260507 18:37:15.097 | INFO | MemInfo: { | 20260507 18:37:15.097 | INFO | "gpu" : " 17 GB", | 20260507 18:37:15.097 | INFO | "huge" : "", | 20260507 18:37:15.097 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:15.097 | INFO | "malloc" : " 14 GB", | 20260507 18:37:15.097 | INFO | "pinned" : " 29 GB" | 20260507 18:37:15.097 | INFO | } | 20260507 18:37:15.097 | INFO | Perf report for the last 5.1389 seconds: | 20260507 18:37:15.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:15.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:15.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:15.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:15.097 | INFO | ANALYSIS : 70.0%, 2.9, 106.2ms, 0 | 20260507 18:37:15.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:15.097 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 18:37:15.097 | INFO | PRE_HQ : 68.6%, 1.8, 65.4ms, 0 | 20260507 18:37:15.097 | INFO | BAZWRITER : 69.7%, 3.7, 131.0ms, 0 | 20260507 18:37:15.097 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 18:37:15.097 | INFO | Uploaded 36GiB to GPU in 2.62884s (13.6943GiB/s) | 20260507 18:37:15.097 | INFO | Downloaded 38GiB to GPU in 2.73376s (13.9002GiB/s) | 20260507 18:37:20.236 | INFO | Analyzing chunk frames = [1704448,1704960) | 20260507 18:37:20.236 | INFO | MemInfo: { | 20260507 18:37:20.236 | INFO | "gpu" : " 17 GB", | 20260507 18:37:20.236 | INFO | "huge" : "", | 20260507 18:37:20.236 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:20.236 | INFO | "malloc" : " 14 GB", | 20260507 18:37:20.236 | INFO | "pinned" : " 29 GB" | 20260507 18:37:20.236 | INFO | } | 20260507 18:37:20.236 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:37:20.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:20.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:37:20.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:37:20.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:20.236 | INFO | Uploaded 36GiB to GPU in 2.54823s (14.1275GiB/s) | 20260507 18:37:20.236 | INFO | Downloaded 34GiB to GPU in 2.44256s (13.9198GiB/s) | 20260507 18:37:25.375 | INFO | Analyzing chunk frames = [1704960,1705472) | 20260507 18:37:25.375 | INFO | MemInfo: { | 20260507 18:37:25.375 | INFO | "gpu" : " 17 GB", | 20260507 18:37:25.375 | INFO | "huge" : "", | 20260507 18:37:25.376 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:25.376 | INFO | "malloc" : " 14 GB", | 20260507 18:37:25.376 | INFO | "pinned" : " 29 GB" | 20260507 18:37:25.376 | INFO | } | 20260507 18:37:25.376 | INFO | Perf report for the last 5.13967 seconds: | 20260507 18:37:25.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:25.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260507 18:37:25.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:37:25.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:25.376 | INFO | Uploaded 36GiB to GPU in 2.57051s (14.005GiB/s) | 20260507 18:37:25.376 | INFO | Downloaded 34GiB to GPU in 2.41736s (14.0649GiB/s) | 20260507 18:37:30.514 | INFO | Analyzing chunk frames = [1705472,1705984) | 20260507 18:37:30.514 | INFO | MemInfo: { | 20260507 18:37:30.514 | INFO | "gpu" : " 17 GB", | 20260507 18:37:30.514 | INFO | "huge" : "", | 20260507 18:37:30.515 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:30.515 | INFO | "malloc" : " 14 GB", | 20260507 18:37:30.515 | INFO | "pinned" : " 29 GB" | 20260507 18:37:30.515 | INFO | } | 20260507 18:37:30.515 | INFO | Perf report for the last 5.13901 seconds: | 20260507 18:37:30.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:30.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:37:30.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:37:30.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:30.515 | INFO | Uploaded 36GiB to GPU in 2.50637s (14.3634GiB/s) | 20260507 18:37:30.515 | INFO | Downloaded 34GiB to GPU in 2.35636s (14.429GiB/s) | 20260507 18:37:35.654 | INFO | Analyzing chunk frames = [1705984,1706496) | 20260507 18:37:35.654 | INFO | MemInfo: { | 20260507 18:37:35.654 | INFO | "gpu" : " 17 GB", | 20260507 18:37:35.654 | INFO | "huge" : "", | 20260507 18:37:35.654 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:35.654 | INFO | "malloc" : " 14 GB", | 20260507 18:37:35.654 | INFO | "pinned" : " 29 GB" | 20260507 18:37:35.654 | INFO | } | 20260507 18:37:35.654 | INFO | Perf report for the last 5.13973 seconds: | 20260507 18:37:35.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:35.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.654 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:37:35.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.654 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:37:35.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:35.655 | INFO | Uploaded 36GiB to GPU in 2.34347s (15.3618GiB/s) | 20260507 18:37:35.655 | INFO | Downloaded 34GiB to GPU in 2.47745s (13.7238GiB/s) | 20260507 18:37:40.793 | INFO | Analyzing chunk frames = [1706496,1707008) | 20260507 18:37:40.793 | INFO | MemInfo: { | 20260507 18:37:40.793 | INFO | "gpu" : " 17 GB", | 20260507 18:37:40.793 | INFO | "huge" : "", | 20260507 18:37:40.793 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:40.793 | INFO | "malloc" : " 14 GB", | 20260507 18:37:40.793 | INFO | "pinned" : " 29 GB" | 20260507 18:37:40.793 | INFO | } | 20260507 18:37:40.793 | INFO | Perf report for the last 5.13903 seconds: | 20260507 18:37:40.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:40.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.794 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:37:40.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.794 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:37:40.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:40.794 | INFO | Uploaded 36GiB to GPU in 2.4089s (14.9446GiB/s) | 20260507 18:37:40.794 | INFO | Downloaded 34GiB to GPU in 2.48266s (13.695GiB/s) | 20260507 18:37:45.933 | INFO | Analyzing chunk frames = [1707008,1707520) | 20260507 18:37:45.933 | INFO | MemInfo: { | 20260507 18:37:45.933 | INFO | "gpu" : " 17 GB", | 20260507 18:37:45.933 | INFO | "huge" : "", | 20260507 18:37:45.933 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:45.933 | INFO | "malloc" : " 14 GB", | 20260507 18:37:45.933 | INFO | "pinned" : " 29 GB" | 20260507 18:37:45.933 | INFO | } | 20260507 18:37:45.933 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:37:45.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:45.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.933 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:37:45.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.933 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 18:37:45.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:45.934 | INFO | Uploaded 36GiB to GPU in 2.49332s (14.4386GiB/s) | 20260507 18:37:45.934 | INFO | Downloaded 34GiB to GPU in 2.42356s (14.0289GiB/s) | 20260507 18:37:51.072 | INFO | Analyzing chunk frames = [1707520,1708032) | 20260507 18:37:51.072 | INFO | MemInfo: { | 20260507 18:37:51.072 | INFO | "gpu" : " 17 GB", | 20260507 18:37:51.072 | INFO | "huge" : "", | 20260507 18:37:51.072 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:51.072 | INFO | "malloc" : " 14 GB", | 20260507 18:37:51.072 | INFO | "pinned" : " 29 GB" | 20260507 18:37:51.072 | INFO | } | 20260507 18:37:51.072 | INFO | Perf report for the last 5.13926 seconds: | 20260507 18:37:51.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:51.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.072 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:37:51.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.072 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 18:37:51.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:51.073 | INFO | Uploaded 36GiB to GPU in 2.52346s (14.2661GiB/s) | 20260507 18:37:51.073 | INFO | Downloaded 34GiB to GPU in 2.41404s (14.0843GiB/s) | 20260507 18:37:51.180 | INFO | Starting a round of Baz IO | 20260507 18:37:54.683 | INFO | Populating RTMetrics report | 20260507 18:37:54.683 | INFO | RTMetrics report populated | 20260507 18:37:54.683 | INFO | Opening tmp RTMetrics json file | 20260507 18:37:54.730 | INFO | Serializing RTMetrics to json | 20260507 18:37:54.776 | INFO | Serialization done, swapping tmp file now... | 20260507 18:37:54.777 | INFO | RTMetrics done | 20260507 18:37:54.791 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.61 seconds | 20260507 18:37:54.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.37% / 0.01% / 53.62% | 20260507 18:37:54.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.84543% / 3.17891e-05% / 4.37101e-05% | 20260507 18:37:54.791 | INFO | Baz IO overall: | 20260507 18:37:54.791 | INFO | BytesWritten : 765251318670 | 20260507 18:37:54.791 | INFO | NumEvents : 371003948509 | 20260507 18:37:56.211 | INFO | Analyzing chunk frames = [1708032,1708544) | 20260507 18:37:56.211 | INFO | MemInfo: { | 20260507 18:37:56.211 | INFO | "gpu" : " 17 GB", | 20260507 18:37:56.211 | INFO | "huge" : "", | 20260507 18:37:56.211 | INFO | "huge_shared" : " 29 GB", | 20260507 18:37:56.211 | INFO | "malloc" : " 14 GB", | 20260507 18:37:56.211 | INFO | "pinned" : " 29 GB" | 20260507 18:37:56.211 | INFO | } | 20260507 18:37:56.211 | INFO | Perf report for the last 5.13903 seconds: | 20260507 18:37:56.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:37:56.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:56.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:56.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:56.211 | INFO | ANALYSIS : 69.8%, 3.0, 108.3ms, 0 | 20260507 18:37:56.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:37:56.211 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 18:37:56.211 | INFO | PRE_HQ : 66.6%, 1.7, 58.9ms, 0 | 20260507 18:37:56.211 | INFO | BAZWRITER : 70.3%, 3.9, 139.8ms, 0 | 20260507 18:37:56.211 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 18:37:56.212 | INFO | Uploaded 36GiB to GPU in 2.59682s (13.8631GiB/s) | 20260507 18:37:56.212 | INFO | Downloaded 38GiB to GPU in 2.70592s (14.0433GiB/s) | 20260507 18:38:01.350 | INFO | Analyzing chunk frames = [1708544,1709056) | 20260507 18:38:01.350 | INFO | MemInfo: { | 20260507 18:38:01.350 | INFO | "gpu" : " 17 GB", | 20260507 18:38:01.350 | INFO | "huge" : "", | 20260507 18:38:01.350 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:01.351 | INFO | "malloc" : " 14 GB", | 20260507 18:38:01.351 | INFO | "pinned" : " 29 GB" | 20260507 18:38:01.351 | INFO | } | 20260507 18:38:01.351 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:38:01.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:01.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 18:38:01.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:38:01.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:01.351 | INFO | Uploaded 36GiB to GPU in 2.5236s (14.2653GiB/s) | 20260507 18:38:01.351 | INFO | Downloaded 34GiB to GPU in 2.44961s (13.8798GiB/s) | 20260507 18:38:06.490 | INFO | Analyzing chunk frames = [1709056,1709568) | 20260507 18:38:06.490 | INFO | MemInfo: { | 20260507 18:38:06.490 | INFO | "gpu" : " 17 GB", | 20260507 18:38:06.490 | INFO | "huge" : "", | 20260507 18:38:06.490 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:06.490 | INFO | "malloc" : " 14 GB", | 20260507 18:38:06.490 | INFO | "pinned" : " 29 GB" | 20260507 18:38:06.490 | INFO | } | 20260507 18:38:06.490 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:38:06.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:06.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:38:06.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:38:06.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:06.490 | INFO | Uploaded 36GiB to GPU in 2.56454s (14.0376GiB/s) | 20260507 18:38:06.490 | INFO | Downloaded 34GiB to GPU in 2.42761s (14.0055GiB/s) | 20260507 18:38:11.629 | INFO | Analyzing chunk frames = [1709568,1710080) | 20260507 18:38:11.629 | INFO | MemInfo: { | 20260507 18:38:11.629 | INFO | "gpu" : " 17 GB", | 20260507 18:38:11.629 | INFO | "huge" : "", | 20260507 18:38:11.629 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:11.629 | INFO | "malloc" : " 14 GB", | 20260507 18:38:11.629 | INFO | "pinned" : " 29 GB" | 20260507 18:38:11.629 | INFO | } | 20260507 18:38:11.629 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:38:11.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:11.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.629 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:38:11.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.630 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:38:11.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:11.630 | INFO | Uploaded 36GiB to GPU in 2.49017s (14.4568GiB/s) | 20260507 18:38:11.630 | INFO | Downloaded 34GiB to GPU in 2.34842s (14.4778GiB/s) | 20260507 18:38:16.769 | INFO | Analyzing chunk frames = [1710080,1710592) | 20260507 18:38:16.769 | INFO | MemInfo: { | 20260507 18:38:16.769 | INFO | "gpu" : " 17 GB", | 20260507 18:38:16.769 | INFO | "huge" : "", | 20260507 18:38:16.769 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:16.769 | INFO | "malloc" : " 14 GB", | 20260507 18:38:16.769 | INFO | "pinned" : " 29 GB" | 20260507 18:38:16.769 | INFO | } | 20260507 18:38:16.769 | INFO | Perf report for the last 5.13999 seconds: | 20260507 18:38:16.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:16.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.769 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:38:16.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.770 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:38:16.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:16.770 | INFO | Uploaded 36GiB to GPU in 2.34653s (15.3418GiB/s) | 20260507 18:38:16.770 | INFO | Downloaded 34GiB to GPU in 2.48067s (13.706GiB/s) | 20260507 18:38:21.908 | INFO | Analyzing chunk frames = [1710592,1711104) | 20260507 18:38:21.908 | INFO | MemInfo: { | 20260507 18:38:21.908 | INFO | "gpu" : " 17 GB", | 20260507 18:38:21.908 | INFO | "huge" : "", | 20260507 18:38:21.908 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:21.908 | INFO | "malloc" : " 14 GB", | 20260507 18:38:21.908 | INFO | "pinned" : " 29 GB" | 20260507 18:38:21.908 | INFO | } | 20260507 18:38:21.909 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:38:21.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:21.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:38:21.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:38:21.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:21.909 | INFO | Uploaded 36GiB to GPU in 2.4277s (14.8289GiB/s) | 20260507 18:38:21.909 | INFO | Downloaded 34GiB to GPU in 2.49763s (13.6129GiB/s) | 20260507 18:38:27.047 | INFO | Analyzing chunk frames = [1711104,1711616) | 20260507 18:38:27.047 | INFO | MemInfo: { | 20260507 18:38:27.047 | INFO | "gpu" : " 17 GB", | 20260507 18:38:27.047 | INFO | "huge" : "", | 20260507 18:38:27.047 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:27.047 | INFO | "malloc" : " 14 GB", | 20260507 18:38:27.047 | INFO | "pinned" : " 29 GB" | 20260507 18:38:27.047 | INFO | } | 20260507 18:38:27.047 | INFO | Perf report for the last 5.13891 seconds: | 20260507 18:38:27.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:27.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.047 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:38:27.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.048 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:38:27.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:27.048 | INFO | Uploaded 36GiB to GPU in 2.52875s (14.2363GiB/s) | 20260507 18:38:27.048 | INFO | Downloaded 34GiB to GPU in 2.43179s (13.9815GiB/s) | 20260507 18:38:32.187 | INFO | Analyzing chunk frames = [1711616,1712128) | 20260507 18:38:32.187 | INFO | MemInfo: { | 20260507 18:38:32.187 | INFO | "gpu" : " 17 GB", | 20260507 18:38:32.187 | INFO | "huge" : "", | 20260507 18:38:32.187 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:32.187 | INFO | "malloc" : " 14 GB", | 20260507 18:38:32.187 | INFO | "pinned" : " 29 GB" | 20260507 18:38:32.187 | INFO | } | 20260507 18:38:32.187 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:38:32.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:32.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:38:32.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:38:32.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:32.187 | INFO | Uploaded 36GiB to GPU in 2.50653s (14.3625GiB/s) | 20260507 18:38:32.187 | INFO | Downloaded 34GiB to GPU in 2.41105s (14.1017GiB/s) | 20260507 18:38:32.187 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:38:32.187 | INFO | Total mem usage: 16.4 GiB | 20260507 18:38:32.187 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:38:32.187 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11.1%) | 20260507 18:38:32.187 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:38:32.187 | INFO | High Water Breakdown: | 20260507 18:38:32.187 | INFO | 59.6% : 8.8 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:38:32.188 | INFO | 40.3% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:38:32.188 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:38:32.188 | INFO | ------------------------------------------ | 20260507 18:38:32.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:38:32.188 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:38:32.188 | INFO | Total mem usage: 26.5 GiB | 20260507 18:38:32.188 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:38:32.188 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:38:32.188 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:38:32.188 | INFO | High Water Breakdown: | 20260507 18:38:32.188 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:38:32.188 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:38:32.188 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:38:32.188 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:38:32.188 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:38:32.188 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:38:32.188 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:38:32.188 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:38:32.188 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:38:32.188 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:38:32.188 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:38:32.188 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:38:32.188 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:38:32.188 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:38:32.188 | INFO | ------------------------------------------ | 20260507 18:38:32.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:38:32.188 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:38:32.188 | INFO | Total mem usage: 27.8 GiB | 20260507 18:38:32.188 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:38:32.188 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:38:32.188 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:38:32.188 | INFO | High Water Breakdown: | 20260507 18:38:32.188 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:38:32.188 | INFO | ------------------------------------------ | 20260507 18:38:32.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:38:32.188 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:38:32.188 | INFO | Total mem usage: 15.7 GiB | 20260507 18:38:32.188 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:38:32.188 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:38:32.188 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:38:32.188 | INFO | High Water Breakdown: | 20260507 18:38:32.188 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:38:32.188 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:38:32.188 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:38:32.188 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:38:32.188 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:38:32.188 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:38:32.188 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:38:32.188 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:38:32.188 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:38:32.188 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:38:32.188 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:38:32.188 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:38:32.188 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:38:32.188 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:38:32.188 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:38:32.188 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:38:32.188 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:38:32.188 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:38:32.188 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:38:32.188 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:38:32.188 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:38:32.188 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:38:32.188 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:38:32.188 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:38:32.188 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:38:32.188 | INFO | ------------------------------------------ | 20260507 18:38:32.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:38:32.188 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:38:32.189 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:38:32.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:38:32.189 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.43e+01 / 2.15e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 0.29%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.09e-02 / 6.30e-03 | 20260507 18:38:32.189 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.51e+00, 6.14e-01 / 2.24e-01 | 20260507 18:38:32.189 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.49e+00, 2.16e-01 / 1.24e-01 | 20260507 18:38:32.189 | INFO | PerfReport: 15.70%, DME -- 4.20e+01 / 4.50e+01, 4.02e+00 / 1.89e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.88e+00 / 1.83e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.34e-01 / 2.52e-01 | 20260507 18:38:32.189 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.43e+00, 3.16e-01 / 1.90e-01 | 20260507 18:38:32.189 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.17e-01, 6.28e-02 / 3.15e-02 | 20260507 18:38:32.189 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:38:32.189 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:38:32.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:38:32.189 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 24.27%, Upload -- 2.60e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 23.96%, Download -- 2.57e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 31.13%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 2.02%, LockUpload -- 2.17e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 2.09%, LockDownload -- 2.24e+00 / 0.00e+00, 3.55e+00 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 16.51%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:38:32.189 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 18:38:32.285 | INFO | Starting a round of Baz IO | 20260507 18:38:35.779 | INFO | Populating RTMetrics report | 20260507 18:38:35.779 | INFO | RTMetrics report populated | 20260507 18:38:35.779 | INFO | Opening tmp RTMetrics json file | 20260507 18:38:35.783 | INFO | Serializing RTMetrics to json | 20260507 18:38:35.798 | INFO | Serialization done, swapping tmp file now... | 20260507 18:38:35.798 | INFO | RTMetrics done | 20260507 18:38:35.867 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.582 seconds | 20260507 18:38:35.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.35% / 0.01% / 53.63% | 20260507 18:38:35.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.84204% / 1.19209e-05% / 4.76837e-05% | 20260507 18:38:35.867 | INFO | Baz IO overall: | 20260507 18:38:35.867 | INFO | BytesWritten : 767187918528 | 20260507 18:38:35.867 | INFO | NumEvents : 371943886763 | 20260507 18:38:37.326 | INFO | Analyzing chunk frames = [1712128,1712640) | 20260507 18:38:37.326 | INFO | MemInfo: { | 20260507 18:38:37.326 | INFO | "gpu" : " 17 GB", | 20260507 18:38:37.326 | INFO | "huge" : "", | 20260507 18:38:37.326 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:37.326 | INFO | "malloc" : " 14 GB", | 20260507 18:38:37.326 | INFO | "pinned" : " 29 GB" | 20260507 18:38:37.326 | INFO | } | 20260507 18:38:37.326 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:38:37.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:37.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:37.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:37.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:37.326 | INFO | ANALYSIS : 69.4%, 2.9, 107.3ms, 0 | 20260507 18:38:37.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:37.326 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:38:37.326 | INFO | PRE_HQ : 66.3%, 1.7, 59.0ms, 0 | 20260507 18:38:37.326 | INFO | BAZWRITER : 69.3%, 3.6, 128.2ms, 0 | 20260507 18:38:37.326 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:38:37.327 | INFO | Uploaded 36GiB to GPU in 2.56815s (14.0179GiB/s) | 20260507 18:38:37.327 | INFO | Downloaded 38GiB to GPU in 2.70627s (14.0415GiB/s) | 20260507 18:38:42.465 | INFO | Analyzing chunk frames = [1712640,1713152) | 20260507 18:38:42.465 | INFO | MemInfo: { | 20260507 18:38:42.465 | INFO | "gpu" : " 17 GB", | 20260507 18:38:42.465 | INFO | "huge" : "", | 20260507 18:38:42.465 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:42.465 | INFO | "malloc" : " 14 GB", | 20260507 18:38:42.465 | INFO | "pinned" : " 29 GB" | 20260507 18:38:42.466 | INFO | } | 20260507 18:38:42.466 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:38:42.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:42.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260507 18:38:42.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:38:42.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:42.466 | INFO | Uploaded 36GiB to GPU in 2.5579s (14.074GiB/s) | 20260507 18:38:42.466 | INFO | Downloaded 34GiB to GPU in 2.44234s (13.9211GiB/s) | 20260507 18:38:47.605 | INFO | Analyzing chunk frames = [1713152,1713664) | 20260507 18:38:47.605 | INFO | MemInfo: { | 20260507 18:38:47.605 | INFO | "gpu" : " 17 GB", | 20260507 18:38:47.605 | INFO | "huge" : "", | 20260507 18:38:47.605 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:47.605 | INFO | "malloc" : " 14 GB", | 20260507 18:38:47.605 | INFO | "pinned" : " 29 GB" | 20260507 18:38:47.605 | INFO | } | 20260507 18:38:47.605 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:38:47.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:47.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.605 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260507 18:38:47.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.605 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:38:47.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:47.606 | INFO | Uploaded 36GiB to GPU in 2.57144s (13.9999GiB/s) | 20260507 18:38:47.606 | INFO | Downloaded 34GiB to GPU in 2.43091s (13.9865GiB/s) | 20260507 18:38:52.744 | INFO | Analyzing chunk frames = [1713664,1714176) | 20260507 18:38:52.744 | INFO | MemInfo: { | 20260507 18:38:52.744 | INFO | "gpu" : " 17 GB", | 20260507 18:38:52.744 | INFO | "huge" : "", | 20260507 18:38:52.744 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:52.744 | INFO | "malloc" : " 14 GB", | 20260507 18:38:52.744 | INFO | "pinned" : " 29 GB" | 20260507 18:38:52.744 | INFO | } | 20260507 18:38:52.744 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:38:52.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:52.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.744 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:38:52.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.745 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:38:52.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:52.745 | INFO | Uploaded 36GiB to GPU in 2.50847s (14.3514GiB/s) | 20260507 18:38:52.745 | INFO | Downloaded 34GiB to GPU in 2.36125s (14.3991GiB/s) | 20260507 18:38:57.884 | INFO | Analyzing chunk frames = [1714176,1714688) | 20260507 18:38:57.884 | INFO | MemInfo: { | 20260507 18:38:57.884 | INFO | "gpu" : " 17 GB", | 20260507 18:38:57.884 | INFO | "huge" : "", | 20260507 18:38:57.884 | INFO | "huge_shared" : " 29 GB", | 20260507 18:38:57.884 | INFO | "malloc" : " 14 GB", | 20260507 18:38:57.884 | INFO | "pinned" : " 29 GB" | 20260507 18:38:57.884 | INFO | } | 20260507 18:38:57.884 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:38:57.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:38:57.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:38:57.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 18:38:57.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:38:57.884 | INFO | Uploaded 36GiB to GPU in 2.372s (15.1771GiB/s) | 20260507 18:38:57.884 | INFO | Downloaded 34GiB to GPU in 2.47133s (13.7578GiB/s) | 20260507 18:39:03.023 | INFO | Analyzing chunk frames = [1714688,1715200) | 20260507 18:39:03.023 | INFO | MemInfo: { | 20260507 18:39:03.023 | INFO | "gpu" : " 17 GB", | 20260507 18:39:03.023 | INFO | "huge" : "", | 20260507 18:39:03.023 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:03.023 | INFO | "malloc" : " 14 GB", | 20260507 18:39:03.023 | INFO | "pinned" : " 29 GB" | 20260507 18:39:03.023 | INFO | } | 20260507 18:39:03.023 | INFO | Perf report for the last 5.13914 seconds: | 20260507 18:39:03.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:03.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.023 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:39:03.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.023 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:39:03.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:03.024 | INFO | Uploaded 36GiB to GPU in 2.41865s (14.8843GiB/s) | 20260507 18:39:03.024 | INFO | Downloaded 34GiB to GPU in 2.49257s (13.6405GiB/s) | 20260507 18:39:08.162 | INFO | Analyzing chunk frames = [1715200,1715712) | 20260507 18:39:08.162 | INFO | MemInfo: { | 20260507 18:39:08.162 | INFO | "gpu" : " 17 GB", | 20260507 18:39:08.162 | INFO | "huge" : "", | 20260507 18:39:08.162 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:08.162 | INFO | "malloc" : " 14 GB", | 20260507 18:39:08.162 | INFO | "pinned" : " 29 GB" | 20260507 18:39:08.162 | INFO | } | 20260507 18:39:08.163 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:39:08.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:08.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:39:08.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:39:08.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:08.163 | INFO | Uploaded 36GiB to GPU in 2.52198s (14.2745GiB/s) | 20260507 18:39:08.163 | INFO | Downloaded 34GiB to GPU in 2.43051s (13.9888GiB/s) | 20260507 18:39:13.302 | INFO | Analyzing chunk frames = [1715712,1716224) | 20260507 18:39:13.302 | INFO | MemInfo: { | 20260507 18:39:13.302 | INFO | "gpu" : " 17 GB", | 20260507 18:39:13.302 | INFO | "huge" : "", | 20260507 18:39:13.302 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:13.302 | INFO | "malloc" : " 14 GB", | 20260507 18:39:13.302 | INFO | "pinned" : " 29 GB" | 20260507 18:39:13.302 | INFO | } | 20260507 18:39:13.302 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:39:13.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:13.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.302 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:39:13.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.302 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:39:13.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:13.303 | INFO | Uploaded 36GiB to GPU in 2.48664s (14.4773GiB/s) | 20260507 18:39:13.303 | INFO | Downloaded 34GiB to GPU in 2.41453s (14.0814GiB/s) | 20260507 18:39:13.399 | INFO | Starting a round of Baz IO | 20260507 18:39:16.919 | INFO | Populating RTMetrics report | 20260507 18:39:16.919 | INFO | RTMetrics report populated | 20260507 18:39:16.919 | INFO | Opening tmp RTMetrics json file | 20260507 18:39:16.936 | INFO | Serializing RTMetrics to json | 20260507 18:39:16.938 | INFO | Serialization done, swapping tmp file now... | 20260507 18:39:16.938 | INFO | RTMetrics done | 20260507 18:39:17.023 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.623 seconds | 20260507 18:39:17.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.34% / 0.01% / 53.65% | 20260507 18:39:17.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.83303% / 2.78155e-05% / 5.5631e-05% | 20260507 18:39:17.023 | INFO | Baz IO overall: | 20260507 18:39:17.023 | INFO | BytesWritten : 769123485455 | 20260507 18:39:17.023 | INFO | NumEvents : 372883109359 | 20260507 18:39:18.441 | INFO | Analyzing chunk frames = [1716224,1716736) | 20260507 18:39:18.441 | INFO | MemInfo: { | 20260507 18:39:18.441 | INFO | "gpu" : " 17 GB", | 20260507 18:39:18.441 | INFO | "huge" : "", | 20260507 18:39:18.441 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:18.441 | INFO | "malloc" : " 14 GB", | 20260507 18:39:18.441 | INFO | "pinned" : " 29 GB" | 20260507 18:39:18.441 | INFO | } | 20260507 18:39:18.441 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:39:18.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:18.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:18.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:18.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:18.441 | INFO | ANALYSIS : 69.6%, 3.0, 108.8ms, 0 | 20260507 18:39:18.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:18.441 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:39:18.442 | INFO | PRE_HQ : 66.5%, 1.7, 59.3ms, 0 | 20260507 18:39:18.442 | INFO | BAZWRITER : 69.9%, 3.4, 122.1ms, 0 | 20260507 18:39:18.442 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:39:18.442 | INFO | Uploaded 36GiB to GPU in 2.53374s (14.2082GiB/s) | 20260507 18:39:18.442 | INFO | Downloaded 38GiB to GPU in 2.72331s (13.9536GiB/s) | 20260507 18:39:23.580 | INFO | Analyzing chunk frames = [1716736,1717248) | 20260507 18:39:23.580 | INFO | MemInfo: { | 20260507 18:39:23.580 | INFO | "gpu" : " 17 GB", | 20260507 18:39:23.580 | INFO | "huge" : "", | 20260507 18:39:23.580 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:23.580 | INFO | "malloc" : " 14 GB", | 20260507 18:39:23.580 | INFO | "pinned" : " 29 GB" | 20260507 18:39:23.580 | INFO | } | 20260507 18:39:23.581 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:39:23.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:23.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:39:23.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:39:23.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:23.581 | INFO | Uploaded 36GiB to GPU in 2.52513s (14.2567GiB/s) | 20260507 18:39:23.581 | INFO | Downloaded 34GiB to GPU in 2.46497s (13.7933GiB/s) | 20260507 18:39:28.720 | INFO | Analyzing chunk frames = [1717248,1717760) | 20260507 18:39:28.720 | INFO | MemInfo: { | 20260507 18:39:28.720 | INFO | "gpu" : " 17 GB", | 20260507 18:39:28.720 | INFO | "huge" : "", | 20260507 18:39:28.720 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:28.720 | INFO | "malloc" : " 14 GB", | 20260507 18:39:28.720 | INFO | "pinned" : " 29 GB" | 20260507 18:39:28.720 | INFO | } | 20260507 18:39:28.720 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:39:28.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:28.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:39:28.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:39:28.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:28.720 | INFO | Uploaded 36GiB to GPU in 2.54537s (14.1433GiB/s) | 20260507 18:39:28.720 | INFO | Downloaded 34GiB to GPU in 2.45171s (13.8679GiB/s) | 20260507 18:39:33.859 | INFO | Analyzing chunk frames = [1717760,1718272) | 20260507 18:39:33.859 | INFO | MemInfo: { | 20260507 18:39:33.859 | INFO | "gpu" : " 17 GB", | 20260507 18:39:33.859 | INFO | "huge" : "", | 20260507 18:39:33.859 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:33.859 | INFO | "malloc" : " 14 GB", | 20260507 18:39:33.859 | INFO | "pinned" : " 29 GB" | 20260507 18:39:33.859 | INFO | } | 20260507 18:39:33.859 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:39:33.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:33.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.859 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:39:33.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.860 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:39:33.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:33.860 | INFO | Uploaded 36GiB to GPU in 2.47575s (14.5411GiB/s) | 20260507 18:39:33.860 | INFO | Downloaded 34GiB to GPU in 2.36337s (14.3862GiB/s) | 20260507 18:39:38.998 | INFO | Analyzing chunk frames = [1718272,1718784) | 20260507 18:39:38.998 | INFO | MemInfo: { | 20260507 18:39:38.998 | INFO | "gpu" : " 17 GB", | 20260507 18:39:38.998 | INFO | "huge" : "", | 20260507 18:39:38.998 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:38.998 | INFO | "malloc" : " 14 GB", | 20260507 18:39:38.998 | INFO | "pinned" : " 29 GB" | 20260507 18:39:38.998 | INFO | } | 20260507 18:39:38.998 | INFO | Perf report for the last 5.1389 seconds: | 20260507 18:39:38.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:38.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.998 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:39:38.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.998 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:39:38.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:38.999 | INFO | Uploaded 36GiB to GPU in 2.32242s (15.5011GiB/s) | 20260507 18:39:38.999 | INFO | Downloaded 34GiB to GPU in 2.49223s (13.6424GiB/s) | 20260507 18:39:44.138 | INFO | Analyzing chunk frames = [1718784,1719296) | 20260507 18:39:44.138 | INFO | MemInfo: { | 20260507 18:39:44.138 | INFO | "gpu" : " 17 GB", | 20260507 18:39:44.138 | INFO | "huge" : "", | 20260507 18:39:44.138 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:44.138 | INFO | "malloc" : " 14 GB", | 20260507 18:39:44.138 | INFO | "pinned" : " 29 GB" | 20260507 18:39:44.138 | INFO | } | 20260507 18:39:44.138 | INFO | Perf report for the last 5.13981 seconds: | 20260507 18:39:44.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:44.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.138 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:39:44.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.138 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:39:44.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:44.139 | INFO | Uploaded 36GiB to GPU in 2.47864s (14.5241GiB/s) | 20260507 18:39:44.139 | INFO | Downloaded 34GiB to GPU in 2.51988s (13.4927GiB/s) | 20260507 18:39:49.277 | INFO | Analyzing chunk frames = [1719296,1719808) | 20260507 18:39:49.277 | INFO | MemInfo: { | 20260507 18:39:49.277 | INFO | "gpu" : " 17 GB", | 20260507 18:39:49.277 | INFO | "huge" : "", | 20260507 18:39:49.277 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:49.277 | INFO | "malloc" : " 14 GB", | 20260507 18:39:49.277 | INFO | "pinned" : " 29 GB" | 20260507 18:39:49.277 | INFO | } | 20260507 18:39:49.277 | INFO | Perf report for the last 5.13889 seconds: | 20260507 18:39:49.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:49.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.277 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:39:49.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.277 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:39:49.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:49.278 | INFO | Uploaded 36GiB to GPU in 2.4874s (14.4729GiB/s) | 20260507 18:39:49.278 | INFO | Downloaded 34GiB to GPU in 2.44403s (13.9115GiB/s) | 20260507 18:39:54.416 | INFO | Analyzing chunk frames = [1719808,1720320) | 20260507 18:39:54.416 | INFO | MemInfo: { | 20260507 18:39:54.416 | INFO | "gpu" : " 17 GB", | 20260507 18:39:54.416 | INFO | "huge" : "", | 20260507 18:39:54.416 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:54.416 | INFO | "malloc" : " 14 GB", | 20260507 18:39:54.416 | INFO | "pinned" : " 29 GB" | 20260507 18:39:54.416 | INFO | } | 20260507 18:39:54.416 | INFO | Perf report for the last 5.13936 seconds: | 20260507 18:39:54.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:54.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.416 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:39:54.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.416 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:39:54.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:54.417 | INFO | Uploaded 36GiB to GPU in 2.48114s (14.5094GiB/s) | 20260507 18:39:54.417 | INFO | Downloaded 34GiB to GPU in 2.4259s (14.0154GiB/s) | 20260507 18:39:54.417 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:39:54.417 | INFO | Total mem usage: 16.4 GiB | 20260507 18:39:54.417 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:39:54.417 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11.1%) | 20260507 18:39:54.417 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:39:54.417 | INFO | High Water Breakdown: | 20260507 18:39:54.417 | INFO | 59.6% : 8.79 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:39:54.417 | INFO | 40.3% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:39:54.417 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:39:54.417 | INFO | ------------------------------------------ | 20260507 18:39:54.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:39:54.417 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:39:54.417 | INFO | Total mem usage: 26.5 GiB | 20260507 18:39:54.417 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:39:54.417 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:39:54.417 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:39:54.417 | INFO | High Water Breakdown: | 20260507 18:39:54.417 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:39:54.417 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:39:54.417 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:39:54.417 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:39:54.417 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:39:54.417 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:39:54.417 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:39:54.417 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:39:54.417 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:39:54.417 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:39:54.417 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:39:54.417 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:39:54.417 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:39:54.417 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:39:54.417 | INFO | ------------------------------------------ | 20260507 18:39:54.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:39:54.417 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:39:54.417 | INFO | Total mem usage: 27.8 GiB | 20260507 18:39:54.417 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:39:54.418 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:39:54.418 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:39:54.418 | INFO | High Water Breakdown: | 20260507 18:39:54.418 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:39:54.418 | INFO | ------------------------------------------ | 20260507 18:39:54.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:39:54.418 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:39:54.418 | INFO | Total mem usage: 15.7 GiB | 20260507 18:39:54.418 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:39:54.418 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:39:54.418 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:39:54.418 | INFO | High Water Breakdown: | 20260507 18:39:54.418 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:39:54.418 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:39:54.418 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:39:54.418 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:39:54.418 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:39:54.418 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:39:54.418 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:39:54.418 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:39:54.418 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:39:54.418 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:39:54.418 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:39:54.418 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:39:54.418 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:39:54.418 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:39:54.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:39:54.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:39:54.418 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:39:54.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:39:54.418 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:39:54.418 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:39:54.418 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:39:54.418 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:39:54.418 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:39:54.418 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:39:54.418 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:39:54.418 | INFO | ------------------------------------------ | 20260507 18:39:54.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:39:54.418 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:39:54.418 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:39:54.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:39:54.418 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.67e+01, 1.45e+01 / 2.15e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 0.29%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.11e-02 / 6.31e-03 | 20260507 18:39:54.418 | INFO | PerfReport: 12.08%, Baseline -- 4.04e+00 / 4.53e+00, 6.17e-01 / 1.79e-01 | 20260507 18:39:54.418 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.15e-01 / 1.11e-01 | 20260507 18:39:54.418 | INFO | PerfReport: 15.69%, DME -- 4.20e+01 / 4.51e+01, 4.05e+00 / 1.82e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.92e+00 / 2.34e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.49e+00 / 2.83e+00, 4.35e-01 / 2.55e-01 | 20260507 18:39:54.418 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.41e+00, 3.11e-01 / 1.79e-01 | 20260507 18:39:54.418 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.15e-01, 6.30e-02 / 3.44e-02 | 20260507 18:39:54.418 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:39:54.418 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:39:54.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:39:54.418 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 24.15%, Upload -- 2.59e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 31.12%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 1.90%, LockUpload -- 2.04e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 2.19%, LockDownload -- 2.35e+00 / 0.00e+00, 3.72e+00 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 16.61%, LockCompute -- 1.78e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 18:39:54.418 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260507 18:39:54.513 | INFO | Starting a round of Baz IO | 20260507 18:39:58.024 | INFO | Populating RTMetrics report | 20260507 18:39:58.025 | INFO | RTMetrics report populated | 20260507 18:39:58.025 | INFO | Opening tmp RTMetrics json file | 20260507 18:39:58.045 | INFO | Serializing RTMetrics to json | 20260507 18:39:58.049 | INFO | Serialization done, swapping tmp file now... | 20260507 18:39:58.051 | INFO | RTMetrics done | 20260507 18:39:58.154 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.64 seconds | 20260507 18:39:58.154 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.32% / 0.01% / 53.66% | 20260507 18:39:58.154 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.82625% / 2.38419e-05% / 5.96046e-05% | 20260507 18:39:58.154 | INFO | Baz IO overall: | 20260507 18:39:58.154 | INFO | BytesWritten : 771059897077 | 20260507 18:39:58.154 | INFO | NumEvents : 373822996275 | 20260507 18:39:59.556 | INFO | Analyzing chunk frames = [1720320,1720832) | 20260507 18:39:59.556 | INFO | MemInfo: { | 20260507 18:39:59.556 | INFO | "gpu" : " 17 GB", | 20260507 18:39:59.556 | INFO | "huge" : "", | 20260507 18:39:59.556 | INFO | "huge_shared" : " 29 GB", | 20260507 18:39:59.556 | INFO | "malloc" : " 14 GB", | 20260507 18:39:59.556 | INFO | "pinned" : " 29 GB" | 20260507 18:39:59.556 | INFO | } | 20260507 18:39:59.556 | INFO | Perf report for the last 5.13967 seconds: | 20260507 18:39:59.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:39:59.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:59.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:59.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:59.556 | INFO | ANALYSIS : 69.6%, 3.0, 107.9ms, 0 | 20260507 18:39:59.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:39:59.556 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:39:59.556 | INFO | PRE_HQ : 66.0%, 1.7, 58.6ms, 0 | 20260507 18:39:59.556 | INFO | BAZWRITER : 70.4%, 3.9, 142.0ms, 0 | 20260507 18:39:59.556 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:39:59.557 | INFO | Uploaded 36GiB to GPU in 2.52928s (14.2333GiB/s) | 20260507 18:39:59.557 | INFO | Downloaded 38GiB to GPU in 2.71258s (14.0088GiB/s) | 20260507 18:40:04.695 | INFO | Analyzing chunk frames = [1720832,1721344) | 20260507 18:40:04.695 | INFO | MemInfo: { | 20260507 18:40:04.695 | INFO | "gpu" : " 17 GB", | 20260507 18:40:04.695 | INFO | "huge" : "", | 20260507 18:40:04.695 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:04.695 | INFO | "malloc" : " 14 GB", | 20260507 18:40:04.695 | INFO | "pinned" : " 29 GB" | 20260507 18:40:04.695 | INFO | } | 20260507 18:40:04.695 | INFO | Perf report for the last 5.13915 seconds: | 20260507 18:40:04.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:04.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.695 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 18:40:04.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.695 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 18:40:04.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:04.696 | INFO | Uploaded 36GiB to GPU in 2.50506s (14.3709GiB/s) | 20260507 18:40:04.696 | INFO | Downloaded 34GiB to GPU in 2.45403s (13.8548GiB/s) | 20260507 18:40:09.834 | INFO | Analyzing chunk frames = [1721344,1721856) | 20260507 18:40:09.834 | INFO | MemInfo: { | 20260507 18:40:09.834 | INFO | "gpu" : " 17 GB", | 20260507 18:40:09.834 | INFO | "huge" : "", | 20260507 18:40:09.834 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:09.834 | INFO | "malloc" : " 14 GB", | 20260507 18:40:09.834 | INFO | "pinned" : " 29 GB" | 20260507 18:40:09.834 | INFO | } | 20260507 18:40:09.834 | INFO | Perf report for the last 5.13911 seconds: | 20260507 18:40:09.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:09.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.834 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:40:09.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.834 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:40:09.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:09.835 | INFO | Uploaded 36GiB to GPU in 2.56116s (14.0561GiB/s) | 20260507 18:40:09.835 | INFO | Downloaded 34GiB to GPU in 2.42174s (14.0395GiB/s) | 20260507 18:40:14.973 | INFO | Analyzing chunk frames = [1721856,1722368) | 20260507 18:40:14.973 | INFO | MemInfo: { | 20260507 18:40:14.973 | INFO | "gpu" : " 17 GB", | 20260507 18:40:14.973 | INFO | "huge" : "", | 20260507 18:40:14.973 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:14.973 | INFO | "malloc" : " 14 GB", | 20260507 18:40:14.973 | INFO | "pinned" : " 29 GB" | 20260507 18:40:14.973 | INFO | } | 20260507 18:40:14.974 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:40:14.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:14.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:40:14.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:40:14.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:14.974 | INFO | Uploaded 36GiB to GPU in 2.50148s (14.3915GiB/s) | 20260507 18:40:14.974 | INFO | Downloaded 34GiB to GPU in 2.35518s (14.4362GiB/s) | 20260507 18:40:20.113 | INFO | Analyzing chunk frames = [1722368,1722880) | 20260507 18:40:20.113 | INFO | MemInfo: { | 20260507 18:40:20.113 | INFO | "gpu" : " 17 GB", | 20260507 18:40:20.113 | INFO | "huge" : "", | 20260507 18:40:20.113 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:20.113 | INFO | "malloc" : " 14 GB", | 20260507 18:40:20.113 | INFO | "pinned" : " 29 GB" | 20260507 18:40:20.113 | INFO | } | 20260507 18:40:20.113 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:40:20.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:20.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:40:20.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:40:20.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:20.113 | INFO | Uploaded 36GiB to GPU in 2.3487s (15.3276GiB/s) | 20260507 18:40:20.113 | INFO | Downloaded 34GiB to GPU in 2.4798s (13.7108GiB/s) | 20260507 18:40:25.252 | INFO | Analyzing chunk frames = [1722880,1723392) | 20260507 18:40:25.252 | INFO | MemInfo: { | 20260507 18:40:25.252 | INFO | "gpu" : " 17 GB", | 20260507 18:40:25.252 | INFO | "huge" : "", | 20260507 18:40:25.252 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:25.252 | INFO | "malloc" : " 14 GB", | 20260507 18:40:25.252 | INFO | "pinned" : " 29 GB" | 20260507 18:40:25.252 | INFO | } | 20260507 18:40:25.252 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:40:25.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:25.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.252 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:40:25.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.253 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:40:25.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:25.253 | INFO | Uploaded 36GiB to GPU in 2.42716s (14.8321GiB/s) | 20260507 18:40:25.253 | INFO | Downloaded 34GiB to GPU in 2.49822s (13.6097GiB/s) | 20260507 18:40:30.392 | INFO | Analyzing chunk frames = [1723392,1723904) | 20260507 18:40:30.392 | INFO | MemInfo: { | 20260507 18:40:30.392 | INFO | "gpu" : " 17 GB", | 20260507 18:40:30.392 | INFO | "huge" : "", | 20260507 18:40:30.392 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:30.392 | INFO | "malloc" : " 14 GB", | 20260507 18:40:30.392 | INFO | "pinned" : " 29 GB" | 20260507 18:40:30.392 | INFO | } | 20260507 18:40:30.392 | INFO | Perf report for the last 5.13923 seconds: | 20260507 18:40:30.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:30.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:40:30.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:40:30.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:30.392 | INFO | Uploaded 36GiB to GPU in 2.5218s (14.2755GiB/s) | 20260507 18:40:30.392 | INFO | Downloaded 34GiB to GPU in 2.42668s (14.0109GiB/s) | 20260507 18:40:35.531 | INFO | Analyzing chunk frames = [1723904,1724416) | 20260507 18:40:35.531 | INFO | MemInfo: { | 20260507 18:40:35.531 | INFO | "gpu" : " 17 GB", | 20260507 18:40:35.531 | INFO | "huge" : "", | 20260507 18:40:35.531 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:35.531 | INFO | "malloc" : " 14 GB", | 20260507 18:40:35.531 | INFO | "pinned" : " 29 GB" | 20260507 18:40:35.531 | INFO | } | 20260507 18:40:35.531 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:40:35.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:35.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.531 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:40:35.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.531 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:40:35.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:35.532 | INFO | Uploaded 36GiB to GPU in 2.50447s (14.3743GiB/s) | 20260507 18:40:35.532 | INFO | Downloaded 34GiB to GPU in 2.4134s (14.088GiB/s) | 20260507 18:40:35.629 | INFO | Starting a round of Baz IO | 20260507 18:40:39.148 | INFO | Populating RTMetrics report | 20260507 18:40:39.148 | INFO | RTMetrics report populated | 20260507 18:40:39.148 | INFO | Opening tmp RTMetrics json file | 20260507 18:40:39.194 | INFO | Serializing RTMetrics to json | 20260507 18:40:39.222 | INFO | Serialization done, swapping tmp file now... | 20260507 18:40:39.237 | INFO | RTMetrics done | 20260507 18:40:39.250 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.62 seconds | 20260507 18:40:39.250 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.31% / 0.01% / 53.68% | 20260507 18:40:39.250 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.82389% / 3.17891e-05% / 1.98682e-05% | 20260507 18:40:39.250 | INFO | Baz IO overall: | 20260507 18:40:39.250 | INFO | BytesWritten : 772996845229 | 20260507 18:40:39.250 | INFO | NumEvents : 374763515648 | 20260507 18:40:40.670 | INFO | Analyzing chunk frames = [1724416,1724928) | 20260507 18:40:40.670 | INFO | MemInfo: { | 20260507 18:40:40.670 | INFO | "gpu" : " 17 GB", | 20260507 18:40:40.670 | INFO | "huge" : "", | 20260507 18:40:40.670 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:40.670 | INFO | "malloc" : " 14 GB", | 20260507 18:40:40.670 | INFO | "pinned" : " 29 GB" | 20260507 18:40:40.670 | INFO | } | 20260507 18:40:40.670 | INFO | Perf report for the last 5.13922 seconds: | 20260507 18:40:40.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:40.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:40.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:40.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:40.670 | INFO | ANALYSIS : 69.4%, 2.9, 106.7ms, 0 | 20260507 18:40:40.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:40.671 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 18:40:40.671 | INFO | PRE_HQ : 67.0%, 1.8, 62.4ms, 0 | 20260507 18:40:40.671 | INFO | BAZWRITER : 70.0%, 3.7, 134.1ms, 0 | 20260507 18:40:40.671 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 18:40:40.671 | INFO | Uploaded 36GiB to GPU in 2.53658s (14.1923GiB/s) | 20260507 18:40:40.671 | INFO | Downloaded 38GiB to GPU in 2.68208s (14.1681GiB/s) | 20260507 18:40:45.810 | INFO | Analyzing chunk frames = [1724928,1725440) | 20260507 18:40:45.810 | INFO | MemInfo: { | 20260507 18:40:45.810 | INFO | "gpu" : " 17 GB", | 20260507 18:40:45.810 | INFO | "huge" : "", | 20260507 18:40:45.810 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:45.810 | INFO | "malloc" : " 14 GB", | 20260507 18:40:45.810 | INFO | "pinned" : " 29 GB" | 20260507 18:40:45.810 | INFO | } | 20260507 18:40:45.810 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:40:45.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:45.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:40:45.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:40:45.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:45.810 | INFO | Uploaded 36GiB to GPU in 2.56589s (14.0302GiB/s) | 20260507 18:40:45.811 | INFO | Downloaded 34GiB to GPU in 2.44343s (13.9149GiB/s) | 20260507 18:40:50.949 | INFO | Analyzing chunk frames = [1725440,1725952) | 20260507 18:40:50.949 | INFO | MemInfo: { | 20260507 18:40:50.949 | INFO | "gpu" : " 17 GB", | 20260507 18:40:50.949 | INFO | "huge" : "", | 20260507 18:40:50.949 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:50.949 | INFO | "malloc" : " 14 GB", | 20260507 18:40:50.949 | INFO | "pinned" : " 29 GB" | 20260507 18:40:50.949 | INFO | } | 20260507 18:40:50.949 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:40:50.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:50.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.949 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:40:50.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.949 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:40:50.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:50.950 | INFO | Uploaded 36GiB to GPU in 2.54634s (14.1379GiB/s) | 20260507 18:40:50.950 | INFO | Downloaded 34GiB to GPU in 2.43379s (13.97GiB/s) | 20260507 18:40:56.088 | INFO | Analyzing chunk frames = [1725952,1726464) | 20260507 18:40:56.088 | INFO | MemInfo: { | 20260507 18:40:56.088 | INFO | "gpu" : " 17 GB", | 20260507 18:40:56.088 | INFO | "huge" : "", | 20260507 18:40:56.088 | INFO | "huge_shared" : " 29 GB", | 20260507 18:40:56.088 | INFO | "malloc" : " 14 GB", | 20260507 18:40:56.088 | INFO | "pinned" : " 29 GB" | 20260507 18:40:56.088 | INFO | } | 20260507 18:40:56.088 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:40:56.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:40:56.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.088 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:40:56.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.088 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:40:56.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:40:56.089 | INFO | Uploaded 36GiB to GPU in 2.49899s (14.4058GiB/s) | 20260507 18:40:56.089 | INFO | Downloaded 34GiB to GPU in 2.35727s (14.4235GiB/s) | 20260507 18:41:01.228 | INFO | Analyzing chunk frames = [1726464,1726976) | 20260507 18:41:01.228 | INFO | MemInfo: { | 20260507 18:41:01.228 | INFO | "gpu" : " 17 GB", | 20260507 18:41:01.228 | INFO | "huge" : "", | 20260507 18:41:01.228 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:01.228 | INFO | "malloc" : " 14 GB", | 20260507 18:41:01.228 | INFO | "pinned" : " 29 GB" | 20260507 18:41:01.228 | INFO | } | 20260507 18:41:01.229 | INFO | Perf report for the last 5.14025 seconds: | 20260507 18:41:01.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:01.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 18:41:01.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:41:01.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:01.229 | INFO | Uploaded 36GiB to GPU in 2.38913s (15.0682GiB/s) | 20260507 18:41:01.229 | INFO | Downloaded 34GiB to GPU in 2.48035s (13.7077GiB/s) | 20260507 18:41:06.367 | INFO | Analyzing chunk frames = [1726976,1727488) | 20260507 18:41:06.367 | INFO | MemInfo: { | 20260507 18:41:06.367 | INFO | "gpu" : " 17 GB", | 20260507 18:41:06.367 | INFO | "huge" : "", | 20260507 18:41:06.367 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:06.367 | INFO | "malloc" : " 14 GB", | 20260507 18:41:06.367 | INFO | "pinned" : " 29 GB" | 20260507 18:41:06.367 | INFO | } | 20260507 18:41:06.367 | INFO | Perf report for the last 5.13889 seconds: | 20260507 18:41:06.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:06.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.367 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:41:06.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.368 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:41:06.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:06.368 | INFO | Uploaded 36GiB to GPU in 2.43666s (14.7743GiB/s) | 20260507 18:41:06.368 | INFO | Downloaded 34GiB to GPU in 2.48326s (13.6917GiB/s) | 20260507 18:41:11.506 | INFO | Analyzing chunk frames = [1727488,1728000) | 20260507 18:41:11.506 | INFO | MemInfo: { | 20260507 18:41:11.506 | INFO | "gpu" : " 17 GB", | 20260507 18:41:11.506 | INFO | "huge" : "", | 20260507 18:41:11.507 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:11.507 | INFO | "malloc" : " 14 GB", | 20260507 18:41:11.507 | INFO | "pinned" : " 29 GB" | 20260507 18:41:11.507 | INFO | } | 20260507 18:41:11.507 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:41:11.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:11.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:41:11.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:41:11.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:11.507 | INFO | Uploaded 36GiB to GPU in 2.49524s (14.4275GiB/s) | 20260507 18:41:11.507 | INFO | Downloaded 34GiB to GPU in 2.42593s (14.0152GiB/s) | 20260507 18:41:16.646 | INFO | Analyzing chunk frames = [1728000,1728512) | 20260507 18:41:16.646 | INFO | MemInfo: { | 20260507 18:41:16.646 | INFO | "gpu" : " 17 GB", | 20260507 18:41:16.646 | INFO | "huge" : "", | 20260507 18:41:16.646 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:16.646 | INFO | "malloc" : " 14 GB", | 20260507 18:41:16.646 | INFO | "pinned" : " 29 GB" | 20260507 18:41:16.646 | INFO | } | 20260507 18:41:16.646 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:41:16.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:16.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.646 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260507 18:41:16.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.646 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 18:41:16.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:16.647 | INFO | Uploaded 36GiB to GPU in 2.54506s (14.145GiB/s) | 20260507 18:41:16.647 | INFO | Downloaded 34GiB to GPU in 2.40361s (14.1454GiB/s) | 20260507 18:41:16.647 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:41:16.647 | INFO | Total mem usage: 16.4 GiB | 20260507 18:41:16.647 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:41:16.647 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.69%) | 20260507 18:41:16.647 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:41:16.647 | INFO | High Water Breakdown: | 20260507 18:41:16.647 | INFO | 60% : 9.05 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:41:16.647 | INFO | 39.9% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:41:16.647 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:41:16.647 | INFO | ------------------------------------------ | 20260507 18:41:16.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:41:16.647 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:41:16.647 | INFO | Total mem usage: 26.5 GiB | 20260507 18:41:16.647 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:41:16.647 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 18:41:16.647 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 18:41:16.647 | INFO | High Water Breakdown: | 20260507 18:41:16.647 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:41:16.647 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:41:16.647 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:41:16.647 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:41:16.647 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:41:16.647 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:41:16.647 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:41:16.647 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:41:16.647 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:41:16.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:41:16.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:41:16.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:41:16.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:41:16.648 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:41:16.648 | INFO | ------------------------------------------ | 20260507 18:41:16.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:41:16.648 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:41:16.648 | INFO | Total mem usage: 27.8 GiB | 20260507 18:41:16.648 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:41:16.648 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:41:16.648 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:41:16.648 | INFO | High Water Breakdown: | 20260507 18:41:16.648 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:41:16.648 | INFO | ------------------------------------------ | 20260507 18:41:16.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:41:16.648 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:41:16.648 | INFO | Total mem usage: 15.7 GiB | 20260507 18:41:16.648 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:41:16.648 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:41:16.648 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:41:16.648 | INFO | High Water Breakdown: | 20260507 18:41:16.648 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:41:16.648 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:41:16.648 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:41:16.648 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:41:16.648 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:41:16.648 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:41:16.648 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:41:16.648 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:41:16.648 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:41:16.648 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:41:16.648 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:41:16.648 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:41:16.648 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:41:16.648 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:41:16.648 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:41:16.648 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:41:16.648 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:41:16.648 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:41:16.648 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:41:16.648 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:41:16.648 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:41:16.648 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:41:16.648 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:41:16.648 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:41:16.648 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:41:16.648 | INFO | ------------------------------------------ | 20260507 18:41:16.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:41:16.648 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:41:16.648 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:41:16.648 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:41:16.648 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.45e+01 / 2.18e+00 | 20260507 18:41:16.648 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.05e-01, 1.17e-02 / 4.52e-03 | 20260507 18:41:16.648 | INFO | PerfReport: 12.08%, Baseline -- 4.04e+00 / 4.50e+00, 6.15e-01 / 2.55e-01 | 20260507 18:41:16.648 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.15e-01 / 1.13e-01 | 20260507 18:41:16.648 | INFO | PerfReport: 15.75%, DME -- 4.21e+01 / 4.52e+01, 4.18e+00 / 2.14e+00 | 20260507 18:41:16.648 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.88e+00 / 2.16e+00 | 20260507 18:41:16.648 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.86e+00, 4.35e-01 / 2.36e-01 | 20260507 18:41:16.648 | INFO | PerfReport: 3.72%, Metrics -- 1.24e+00 / 1.44e+00, 3.20e-01 / 1.58e-01 | 20260507 18:41:16.648 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.89e-01 / 4.24e-01, 6.32e-02 / 3.43e-02 | 20260507 18:41:16.649 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:41:16.649 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:41:16.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:41:16.649 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 24.19%, Upload -- 2.60e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 23.92%, Download -- 2.57e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 31.12%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 2.10%, LockUpload -- 2.25e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 2.10%, LockDownload -- 2.25e+00 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 16.56%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:41:16.649 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260507 18:41:16.768 | INFO | Starting a round of Baz IO | 20260507 18:41:20.269 | INFO | Populating RTMetrics report | 20260507 18:41:20.269 | INFO | RTMetrics report populated | 20260507 18:41:20.269 | INFO | Opening tmp RTMetrics json file | 20260507 18:41:20.270 | INFO | Serializing RTMetrics to json | 20260507 18:41:20.285 | INFO | Serialization done, swapping tmp file now... | 20260507 18:41:20.290 | INFO | RTMetrics done | 20260507 18:41:20.367 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.598 seconds | 20260507 18:41:20.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.29% / 0.01% / 53.69% | 20260507 18:41:20.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.81654% / 1.19209e-05% / 3.97364e-05% | 20260507 18:41:20.367 | INFO | Baz IO overall: | 20260507 18:41:20.367 | INFO | BytesWritten : 774933308014 | 20260507 18:41:20.367 | INFO | NumEvents : 375703557345 | 20260507 18:41:21.785 | INFO | Analyzing chunk frames = [1728512,1729024) | 20260507 18:41:21.785 | INFO | MemInfo: { | 20260507 18:41:21.785 | INFO | "gpu" : " 17 GB", | 20260507 18:41:21.785 | INFO | "huge" : "", | 20260507 18:41:21.785 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:21.785 | INFO | "malloc" : " 14 GB", | 20260507 18:41:21.785 | INFO | "pinned" : " 29 GB" | 20260507 18:41:21.785 | INFO | } | 20260507 18:41:21.785 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:41:21.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:21.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:21.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:21.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:21.785 | INFO | ANALYSIS : 69.7%, 2.9, 105.4ms, 0 | 20260507 18:41:21.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:21.785 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 18:41:21.786 | INFO | PRE_HQ : 68.5%, 1.8, 65.3ms, 0 | 20260507 18:41:21.786 | INFO | BAZWRITER : 69.9%, 3.5, 126.9ms, 0 | 20260507 18:41:21.786 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:41:21.786 | INFO | Uploaded 36GiB to GPU in 2.59784s (13.8577GiB/s) | 20260507 18:41:21.786 | INFO | Downloaded 38GiB to GPU in 2.70805s (14.0323GiB/s) | 20260507 18:41:26.925 | INFO | Analyzing chunk frames = [1729024,1729536) | 20260507 18:41:26.925 | INFO | MemInfo: { | 20260507 18:41:26.925 | INFO | "gpu" : " 17 GB", | 20260507 18:41:26.925 | INFO | "huge" : "", | 20260507 18:41:26.925 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:26.925 | INFO | "malloc" : " 14 GB", | 20260507 18:41:26.925 | INFO | "pinned" : " 29 GB" | 20260507 18:41:26.925 | INFO | } | 20260507 18:41:26.925 | INFO | Perf report for the last 5.13971 seconds: | 20260507 18:41:26.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:26.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260507 18:41:26.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 18:41:26.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:26.925 | INFO | Uploaded 36GiB to GPU in 2.54747s (14.1317GiB/s) | 20260507 18:41:26.925 | INFO | Downloaded 34GiB to GPU in 2.44718s (13.8935GiB/s) | 20260507 18:41:32.064 | INFO | Analyzing chunk frames = [1729536,1730048) | 20260507 18:41:32.064 | INFO | MemInfo: { | 20260507 18:41:32.064 | INFO | "gpu" : " 17 GB", | 20260507 18:41:32.064 | INFO | "huge" : "", | 20260507 18:41:32.064 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:32.064 | INFO | "malloc" : " 14 GB", | 20260507 18:41:32.064 | INFO | "pinned" : " 29 GB" | 20260507 18:41:32.064 | INFO | } | 20260507 18:41:32.064 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:41:32.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:32.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.064 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:41:32.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.064 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:41:32.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:32.065 | INFO | Uploaded 36GiB to GPU in 2.56788s (14.0194GiB/s) | 20260507 18:41:32.065 | INFO | Downloaded 34GiB to GPU in 2.43185s (13.9811GiB/s) | 20260507 18:41:37.203 | INFO | Analyzing chunk frames = [1730048,1730560) | 20260507 18:41:37.203 | INFO | MemInfo: { | 20260507 18:41:37.203 | INFO | "gpu" : " 17 GB", | 20260507 18:41:37.203 | INFO | "huge" : "", | 20260507 18:41:37.203 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:37.203 | INFO | "malloc" : " 14 GB", | 20260507 18:41:37.203 | INFO | "pinned" : " 29 GB" | 20260507 18:41:37.203 | INFO | } | 20260507 18:41:37.203 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:41:37.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:37.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:41:37.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:41:37.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:37.204 | INFO | Uploaded 36GiB to GPU in 2.50792s (14.3545GiB/s) | 20260507 18:41:37.204 | INFO | Downloaded 34GiB to GPU in 2.35416s (14.4425GiB/s) | 20260507 18:41:42.343 | INFO | Analyzing chunk frames = [1730560,1731072) | 20260507 18:41:42.343 | INFO | MemInfo: { | 20260507 18:41:42.343 | INFO | "gpu" : " 17 GB", | 20260507 18:41:42.343 | INFO | "huge" : "", | 20260507 18:41:42.343 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:42.343 | INFO | "malloc" : " 14 GB", | 20260507 18:41:42.343 | INFO | "pinned" : " 29 GB" | 20260507 18:41:42.343 | INFO | } | 20260507 18:41:42.343 | INFO | Perf report for the last 5.1397 seconds: | 20260507 18:41:42.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:42.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.343 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 18:41:42.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.343 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 18:41:42.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:42.344 | INFO | Uploaded 36GiB to GPU in 2.36414s (15.2275GiB/s) | 20260507 18:41:42.344 | INFO | Downloaded 34GiB to GPU in 2.4724s (13.7518GiB/s) | 20260507 18:41:47.482 | INFO | Analyzing chunk frames = [1731072,1731584) | 20260507 18:41:47.482 | INFO | MemInfo: { | 20260507 18:41:47.482 | INFO | "gpu" : " 17 GB", | 20260507 18:41:47.482 | INFO | "huge" : "", | 20260507 18:41:47.482 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:47.482 | INFO | "malloc" : " 14 GB", | 20260507 18:41:47.482 | INFO | "pinned" : " 29 GB" | 20260507 18:41:47.482 | INFO | } | 20260507 18:41:47.482 | INFO | Perf report for the last 5.13886 seconds: | 20260507 18:41:47.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:47.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:41:47.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:41:47.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:47.482 | INFO | Uploaded 36GiB to GPU in 2.44914s (14.699GiB/s) | 20260507 18:41:47.482 | INFO | Downloaded 34GiB to GPU in 2.50489s (13.5734GiB/s) | 20260507 18:41:52.621 | INFO | Analyzing chunk frames = [1731584,1732096) | 20260507 18:41:52.621 | INFO | MemInfo: { | 20260507 18:41:52.621 | INFO | "gpu" : " 17 GB", | 20260507 18:41:52.622 | INFO | "huge" : "", | 20260507 18:41:52.622 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:52.622 | INFO | "malloc" : " 14 GB", | 20260507 18:41:52.622 | INFO | "pinned" : " 29 GB" | 20260507 18:41:52.622 | INFO | } | 20260507 18:41:52.622 | INFO | Perf report for the last 5.13984 seconds: | 20260507 18:41:52.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:52.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.622 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 18:41:52.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.622 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:41:52.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:52.623 | INFO | Uploaded 36GiB to GPU in 2.5009s (14.3948GiB/s) | 20260507 18:41:52.623 | INFO | Downloaded 34GiB to GPU in 2.41609s (14.0723GiB/s) | 20260507 18:41:57.760 | INFO | Analyzing chunk frames = [1732096,1732608) | 20260507 18:41:57.760 | INFO | MemInfo: { | 20260507 18:41:57.760 | INFO | "gpu" : " 17 GB", | 20260507 18:41:57.761 | INFO | "huge" : "", | 20260507 18:41:57.761 | INFO | "huge_shared" : " 29 GB", | 20260507 18:41:57.761 | INFO | "malloc" : " 14 GB", | 20260507 18:41:57.761 | INFO | "pinned" : " 29 GB" | 20260507 18:41:57.761 | INFO | } | 20260507 18:41:57.761 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:41:57.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:41:57.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.761 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:41:57.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.761 | INFO | PRE_HQ : 37.9%, 1.0, 19.7ms, 0 | 20260507 18:41:57.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:41:57.762 | INFO | Uploaded 36GiB to GPU in 2.52209s (14.2739GiB/s) | 20260507 18:41:57.762 | INFO | Downloaded 34GiB to GPU in 2.41638s (14.0706GiB/s) | 20260507 18:41:57.875 | INFO | Starting a round of Baz IO | 20260507 18:42:01.389 | INFO | Populating RTMetrics report | 20260507 18:42:01.390 | INFO | RTMetrics report populated | 20260507 18:42:01.390 | INFO | Opening tmp RTMetrics json file | 20260507 18:42:01.422 | INFO | Serializing RTMetrics to json | 20260507 18:42:01.443 | INFO | Serialization done, swapping tmp file now... | 20260507 18:42:01.463 | INFO | RTMetrics done | 20260507 18:42:01.482 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.607 seconds | 20260507 18:42:01.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.28% / 0.01% / 53.71% | 20260507 18:42:01.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.81008% / 1.98682e-05% / 3.17891e-05% | 20260507 18:42:01.482 | INFO | Baz IO overall: | 20260507 18:42:01.482 | INFO | BytesWritten : 776869528789 | 20260507 18:42:01.482 | INFO | NumEvents : 376643626158 | 20260507 18:42:02.900 | INFO | Analyzing chunk frames = [1732608,1733120) | 20260507 18:42:02.900 | INFO | MemInfo: { | 20260507 18:42:02.900 | INFO | "gpu" : " 17 GB", | 20260507 18:42:02.900 | INFO | "huge" : "", | 20260507 18:42:02.900 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:02.900 | INFO | "malloc" : " 14 GB", | 20260507 18:42:02.900 | INFO | "pinned" : " 29 GB" | 20260507 18:42:02.900 | INFO | } | 20260507 18:42:02.900 | INFO | Perf report for the last 5.13912 seconds: | 20260507 18:42:02.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:02.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:02.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:02.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:02.900 | INFO | ANALYSIS : 70.1%, 2.9, 105.9ms, 0 | 20260507 18:42:02.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:02.900 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260507 18:42:02.900 | INFO | PRE_HQ : 68.8%, 1.8, 65.3ms, 0 | 20260507 18:42:02.900 | INFO | BAZWRITER : 69.7%, 3.6, 128.9ms, 0 | 20260507 18:42:02.900 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 18:42:02.900 | INFO | Uploaded 36GiB to GPU in 2.64545s (13.6083GiB/s) | 20260507 18:42:02.900 | INFO | Downloaded 38GiB to GPU in 2.70785s (14.0333GiB/s) | 20260507 18:42:08.039 | INFO | Analyzing chunk frames = [1733120,1733632) | 20260507 18:42:08.039 | INFO | MemInfo: { | 20260507 18:42:08.039 | INFO | "gpu" : " 17 GB", | 20260507 18:42:08.039 | INFO | "huge" : "", | 20260507 18:42:08.039 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:08.039 | INFO | "malloc" : " 14 GB", | 20260507 18:42:08.039 | INFO | "pinned" : " 29 GB" | 20260507 18:42:08.040 | INFO | } | 20260507 18:42:08.040 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:42:08.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:08.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:42:08.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:42:08.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:08.040 | INFO | Uploaded 36GiB to GPU in 2.54063s (14.1697GiB/s) | 20260507 18:42:08.040 | INFO | Downloaded 34GiB to GPU in 2.43998s (13.9345GiB/s) | 20260507 18:42:13.179 | INFO | Analyzing chunk frames = [1733632,1734144) | 20260507 18:42:13.179 | INFO | MemInfo: { | 20260507 18:42:13.179 | INFO | "gpu" : " 17 GB", | 20260507 18:42:13.179 | INFO | "huge" : "", | 20260507 18:42:13.179 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:13.179 | INFO | "malloc" : " 14 GB", | 20260507 18:42:13.179 | INFO | "pinned" : " 29 GB" | 20260507 18:42:13.179 | INFO | } | 20260507 18:42:13.179 | INFO | Perf report for the last 5.13925 seconds: | 20260507 18:42:13.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:13.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:42:13.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:42:13.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:13.179 | INFO | Uploaded 36GiB to GPU in 2.56759s (14.0209GiB/s) | 20260507 18:42:13.179 | INFO | Downloaded 34GiB to GPU in 2.42773s (14.0049GiB/s) | 20260507 18:42:18.318 | INFO | Analyzing chunk frames = [1734144,1734656) | 20260507 18:42:18.318 | INFO | MemInfo: { | 20260507 18:42:18.318 | INFO | "gpu" : " 17 GB", | 20260507 18:42:18.318 | INFO | "huge" : "", | 20260507 18:42:18.318 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:18.318 | INFO | "malloc" : " 14 GB", | 20260507 18:42:18.318 | INFO | "pinned" : " 29 GB" | 20260507 18:42:18.318 | INFO | } | 20260507 18:42:18.318 | INFO | Perf report for the last 5.13948 seconds: | 20260507 18:42:18.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:18.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.318 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260507 18:42:18.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.318 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:42:18.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:18.319 | INFO | Uploaded 36GiB to GPU in 2.5043s (14.3753GiB/s) | 20260507 18:42:18.319 | INFO | Downloaded 34GiB to GPU in 2.36008s (14.4063GiB/s) | 20260507 18:42:23.457 | INFO | Analyzing chunk frames = [1734656,1735168) | 20260507 18:42:23.457 | INFO | MemInfo: { | 20260507 18:42:23.457 | INFO | "gpu" : " 17 GB", | 20260507 18:42:23.457 | INFO | "huge" : "", | 20260507 18:42:23.457 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:23.457 | INFO | "malloc" : " 14 GB", | 20260507 18:42:23.457 | INFO | "pinned" : " 29 GB" | 20260507 18:42:23.457 | INFO | } | 20260507 18:42:23.458 | INFO | Perf report for the last 5.13925 seconds: | 20260507 18:42:23.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:23.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:42:23.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:42:23.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:23.458 | INFO | Uploaded 36GiB to GPU in 2.35811s (15.2665GiB/s) | 20260507 18:42:23.458 | INFO | Downloaded 34GiB to GPU in 2.48034s (13.7078GiB/s) | 20260507 18:42:28.597 | INFO | Analyzing chunk frames = [1735168,1735680) | 20260507 18:42:28.597 | INFO | MemInfo: { | 20260507 18:42:28.597 | INFO | "gpu" : " 17 GB", | 20260507 18:42:28.597 | INFO | "huge" : "", | 20260507 18:42:28.597 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:28.597 | INFO | "malloc" : " 14 GB", | 20260507 18:42:28.597 | INFO | "pinned" : " 29 GB" | 20260507 18:42:28.597 | INFO | } | 20260507 18:42:28.597 | INFO | Perf report for the last 5.13922 seconds: | 20260507 18:42:28.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:28.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:42:28.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:42:28.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:28.597 | INFO | Uploaded 36GiB to GPU in 2.46698s (14.5928GiB/s) | 20260507 18:42:28.597 | INFO | Downloaded 34GiB to GPU in 2.49372s (13.6342GiB/s) | 20260507 18:42:33.736 | INFO | Analyzing chunk frames = [1735680,1736192) | 20260507 18:42:33.736 | INFO | MemInfo: { | 20260507 18:42:33.736 | INFO | "gpu" : " 17 GB", | 20260507 18:42:33.736 | INFO | "huge" : "", | 20260507 18:42:33.736 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:33.736 | INFO | "malloc" : " 14 GB", | 20260507 18:42:33.736 | INFO | "pinned" : " 29 GB" | 20260507 18:42:33.736 | INFO | } | 20260507 18:42:33.736 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:42:33.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:33.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.736 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:42:33.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.736 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 18:42:33.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:33.737 | INFO | Uploaded 36GiB to GPU in 2.52327s (14.2672GiB/s) | 20260507 18:42:33.737 | INFO | Downloaded 34GiB to GPU in 2.43559s (13.9596GiB/s) | 20260507 18:42:38.875 | INFO | Analyzing chunk frames = [1736192,1736704) | 20260507 18:42:38.875 | INFO | MemInfo: { | 20260507 18:42:38.875 | INFO | "gpu" : " 17 GB", | 20260507 18:42:38.875 | INFO | "huge" : "", | 20260507 18:42:38.875 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:38.875 | INFO | "malloc" : " 14 GB", | 20260507 18:42:38.875 | INFO | "pinned" : " 29 GB" | 20260507 18:42:38.875 | INFO | } | 20260507 18:42:38.876 | INFO | Perf report for the last 5.13981 seconds: | 20260507 18:42:38.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:38.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:42:38.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:42:38.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:38.876 | INFO | Uploaded 36GiB to GPU in 2.51703s (14.3026GiB/s) | 20260507 18:42:38.876 | INFO | Downloaded 34GiB to GPU in 2.41517s (14.0777GiB/s) | 20260507 18:42:38.876 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:42:38.876 | INFO | Total mem usage: 16.4 GiB | 20260507 18:42:38.876 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:42:38.876 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12%) | 20260507 18:42:38.876 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:42:38.876 | INFO | High Water Breakdown: | 20260507 18:42:38.876 | INFO | 59.5% : 8.72 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:42:38.876 | INFO | 40.3% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:42:38.876 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:42:38.876 | INFO | ------------------------------------------ | 20260507 18:42:38.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:42:38.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:42:38.877 | INFO | Total mem usage: 26.5 GiB | 20260507 18:42:38.877 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:42:38.877 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:42:38.877 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:42:38.877 | INFO | High Water Breakdown: | 20260507 18:42:38.877 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:42:38.877 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:42:38.877 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:42:38.877 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:42:38.877 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:42:38.877 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:42:38.877 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:42:38.877 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:42:38.877 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:42:38.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:42:38.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:42:38.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:42:38.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:42:38.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:42:38.877 | INFO | ------------------------------------------ | 20260507 18:42:38.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:42:38.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:42:38.877 | INFO | Total mem usage: 27.8 GiB | 20260507 18:42:38.877 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:42:38.877 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:42:38.877 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:42:38.877 | INFO | High Water Breakdown: | 20260507 18:42:38.877 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:42:38.877 | INFO | ------------------------------------------ | 20260507 18:42:38.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:42:38.877 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:42:38.877 | INFO | Total mem usage: 15.7 GiB | 20260507 18:42:38.877 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:42:38.877 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:42:38.877 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:42:38.877 | INFO | High Water Breakdown: | 20260507 18:42:38.877 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:42:38.877 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:42:38.877 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:42:38.877 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:42:38.877 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:42:38.877 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:42:38.877 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:42:38.877 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:42:38.877 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:42:38.877 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:42:38.877 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:42:38.877 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:42:38.877 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:42:38.877 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:42:38.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:42:38.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:42:38.877 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:42:38.877 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:42:38.877 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:42:38.877 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:42:38.877 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:42:38.877 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:42:38.877 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:42:38.877 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:42:38.877 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:42:38.877 | INFO | ------------------------------------------ | 20260507 18:42:38.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:42:38.877 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:42:38.877 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:42:38.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:42:38.877 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.63e+01, 1.45e+01 / 1.97e+00 | 20260507 18:42:38.877 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.03e-01, 1.12e-02 / 5.64e-03 | 20260507 18:42:38.877 | INFO | PerfReport: 12.11%, Baseline -- 4.04e+00 / 4.51e+00, 6.14e-01 / 2.54e-01 | 20260507 18:42:38.877 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.48e+00, 2.14e-01 / 1.31e-01 | 20260507 18:42:38.877 | INFO | PerfReport: 15.70%, DME -- 4.20e+01 / 4.49e+01, 3.91e+00 / 1.80e+00 | 20260507 18:42:38.877 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.88e+00 / 1.83e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.39e-01 / 2.60e-01 | 20260507 18:42:38.878 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.40e+00, 3.10e-01 / 1.19e-01 | 20260507 18:42:38.878 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.15e-01, 6.19e-02 / 3.00e-02 | 20260507 18:42:38.878 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:42:38.878 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:42:38.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:42:38.878 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 24.40%, Upload -- 2.62e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 24.00%, Download -- 2.57e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 31.16%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 2.04%, LockUpload -- 2.19e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 2.05%, LockDownload -- 2.19e+00 / 0.00e+00, 3.44e+00 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 16.34%, LockCompute -- 1.75e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:42:38.878 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 18:42:38.973 | INFO | Starting a round of Baz IO | 20260507 18:42:42.479 | INFO | Populating RTMetrics report | 20260507 18:42:42.479 | INFO | RTMetrics report populated | 20260507 18:42:42.479 | INFO | Opening tmp RTMetrics json file | 20260507 18:42:42.581 | INFO | Serializing RTMetrics to json | 20260507 18:42:42.599 | INFO | Serialization done, swapping tmp file now... | 20260507 18:42:42.603 | INFO | RTMetrics done | 20260507 18:42:42.863 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.889 seconds | 20260507 18:42:42.863 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.26% / 0.01% / 53.72% | 20260507 18:42:42.863 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.8008% / 1.98682e-05% / 4.76837e-05% | 20260507 18:42:42.863 | INFO | Baz IO overall: | 20260507 18:42:42.863 | INFO | BytesWritten : 778805505739 | 20260507 18:42:42.863 | INFO | NumEvents : 377583338799 | 20260507 18:42:44.015 | INFO | Analyzing chunk frames = [1736704,1737216) | 20260507 18:42:44.015 | INFO | MemInfo: { | 20260507 18:42:44.015 | INFO | "gpu" : " 17 GB", | 20260507 18:42:44.015 | INFO | "huge" : "", | 20260507 18:42:44.015 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:44.015 | INFO | "malloc" : " 14 GB", | 20260507 18:42:44.015 | INFO | "pinned" : " 29 GB" | 20260507 18:42:44.015 | INFO | } | 20260507 18:42:44.015 | INFO | Perf report for the last 5.13902 seconds: | 20260507 18:42:44.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:44.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:44.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:44.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:44.015 | INFO | ANALYSIS : 69.4%, 3.0, 108.0ms, 0 | 20260507 18:42:44.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:44.015 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 18:42:44.015 | INFO | PRE_HQ : 66.4%, 1.7, 58.8ms, 0 | 20260507 18:42:44.015 | INFO | BAZWRITER : 75.1%, 3.7, 144.2ms, 0 | 20260507 18:42:44.015 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260507 18:42:44.016 | INFO | Uploaded 36GiB to GPU in 2.53024s (14.2279GiB/s) | 20260507 18:42:44.016 | INFO | Downloaded 38GiB to GPU in 2.7174s (13.9839GiB/s) | 20260507 18:42:49.154 | INFO | Analyzing chunk frames = [1737216,1737728) | 20260507 18:42:49.154 | INFO | MemInfo: { | 20260507 18:42:49.154 | INFO | "gpu" : " 17 GB", | 20260507 18:42:49.154 | INFO | "huge" : "", | 20260507 18:42:49.154 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:49.154 | INFO | "malloc" : " 14 GB", | 20260507 18:42:49.154 | INFO | "pinned" : " 29 GB" | 20260507 18:42:49.154 | INFO | } | 20260507 18:42:49.154 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:42:49.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:49.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.154 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:42:49.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.154 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:42:49.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:49.155 | INFO | Uploaded 36GiB to GPU in 2.51313s (14.3248GiB/s) | 20260507 18:42:49.155 | INFO | Downloaded 34GiB to GPU in 2.46368s (13.8005GiB/s) | 20260507 18:42:54.293 | INFO | Analyzing chunk frames = [1737728,1738240) | 20260507 18:42:54.293 | INFO | MemInfo: { | 20260507 18:42:54.293 | INFO | "gpu" : " 17 GB", | 20260507 18:42:54.293 | INFO | "huge" : "", | 20260507 18:42:54.293 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:54.293 | INFO | "malloc" : " 14 GB", | 20260507 18:42:54.293 | INFO | "pinned" : " 29 GB" | 20260507 18:42:54.293 | INFO | } | 20260507 18:42:54.293 | INFO | Perf report for the last 5.1391 seconds: | 20260507 18:42:54.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:54.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:42:54.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:42:54.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:54.294 | INFO | Uploaded 36GiB to GPU in 2.5379s (14.1849GiB/s) | 20260507 18:42:54.294 | INFO | Downloaded 34GiB to GPU in 2.44494s (13.9063GiB/s) | 20260507 18:42:59.433 | INFO | Analyzing chunk frames = [1738240,1738752) | 20260507 18:42:59.433 | INFO | MemInfo: { | 20260507 18:42:59.433 | INFO | "gpu" : " 17 GB", | 20260507 18:42:59.433 | INFO | "huge" : "", | 20260507 18:42:59.433 | INFO | "huge_shared" : " 29 GB", | 20260507 18:42:59.433 | INFO | "malloc" : " 14 GB", | 20260507 18:42:59.433 | INFO | "pinned" : " 29 GB" | 20260507 18:42:59.433 | INFO | } | 20260507 18:42:59.433 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:42:59.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:42:59.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260507 18:42:59.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:42:59.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:42:59.433 | INFO | Uploaded 36GiB to GPU in 2.46858s (14.5833GiB/s) | 20260507 18:42:59.433 | INFO | Downloaded 34GiB to GPU in 2.36628s (14.3686GiB/s) | 20260507 18:43:04.572 | INFO | Analyzing chunk frames = [1738752,1739264) | 20260507 18:43:04.572 | INFO | MemInfo: { | 20260507 18:43:04.572 | INFO | "gpu" : " 17 GB", | 20260507 18:43:04.572 | INFO | "huge" : "", | 20260507 18:43:04.572 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:04.573 | INFO | "malloc" : " 14 GB", | 20260507 18:43:04.573 | INFO | "pinned" : " 29 GB" | 20260507 18:43:04.573 | INFO | } | 20260507 18:43:04.573 | INFO | Perf report for the last 5.13985 seconds: | 20260507 18:43:04.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:04.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:43:04.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:43:04.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:04.573 | INFO | Uploaded 36GiB to GPU in 2.33497s (15.4177GiB/s) | 20260507 18:43:04.573 | INFO | Downloaded 34GiB to GPU in 2.49274s (13.6396GiB/s) | 20260507 18:43:09.711 | INFO | Analyzing chunk frames = [1739264,1739776) | 20260507 18:43:09.711 | INFO | MemInfo: { | 20260507 18:43:09.711 | INFO | "gpu" : " 17 GB", | 20260507 18:43:09.711 | INFO | "huge" : "", | 20260507 18:43:09.711 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:09.711 | INFO | "malloc" : " 14 GB", | 20260507 18:43:09.711 | INFO | "pinned" : " 29 GB" | 20260507 18:43:09.712 | INFO | } | 20260507 18:43:09.712 | INFO | Perf report for the last 5.13886 seconds: | 20260507 18:43:09.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:09.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.712 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:43:09.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.712 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 18:43:09.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:09.713 | INFO | Uploaded 36GiB to GPU in 2.42723s (14.8317GiB/s) | 20260507 18:43:09.713 | INFO | Downloaded 34GiB to GPU in 2.50044s (13.5976GiB/s) | 20260507 18:43:14.851 | INFO | Analyzing chunk frames = [1739776,1740288) | 20260507 18:43:14.851 | INFO | MemInfo: { | 20260507 18:43:14.851 | INFO | "gpu" : " 17 GB", | 20260507 18:43:14.851 | INFO | "huge" : "", | 20260507 18:43:14.851 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:14.851 | INFO | "malloc" : " 14 GB", | 20260507 18:43:14.851 | INFO | "pinned" : " 29 GB" | 20260507 18:43:14.851 | INFO | } | 20260507 18:43:14.851 | INFO | Perf report for the last 5.13981 seconds: | 20260507 18:43:14.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:14.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.852 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:43:14.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.852 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260507 18:43:14.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:14.852 | INFO | Uploaded 36GiB to GPU in 2.50245s (14.3859GiB/s) | 20260507 18:43:14.852 | INFO | Downloaded 34GiB to GPU in 2.44045s (13.9319GiB/s) | 20260507 18:43:19.990 | INFO | Analyzing chunk frames = [1740288,1740800) | 20260507 18:43:19.990 | INFO | MemInfo: { | 20260507 18:43:19.990 | INFO | "gpu" : " 17 GB", | 20260507 18:43:19.990 | INFO | "huge" : "", | 20260507 18:43:19.990 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:19.990 | INFO | "malloc" : " 14 GB", | 20260507 18:43:19.990 | INFO | "pinned" : " 29 GB" | 20260507 18:43:19.990 | INFO | } | 20260507 18:43:19.991 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:43:19.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:19.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:43:19.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:43:19.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:19.991 | INFO | Uploaded 36GiB to GPU in 2.49531s (14.4271GiB/s) | 20260507 18:43:19.991 | INFO | Downloaded 34GiB to GPU in 2.42721s (14.0079GiB/s) | 20260507 18:43:20.088 | INFO | Starting a round of Baz IO | 20260507 18:43:23.586 | INFO | Populating RTMetrics report | 20260507 18:43:23.586 | INFO | RTMetrics report populated | 20260507 18:43:23.586 | INFO | Opening tmp RTMetrics json file | 20260507 18:43:23.604 | INFO | Serializing RTMetrics to json | 20260507 18:43:23.606 | INFO | Serialization done, swapping tmp file now... | 20260507 18:43:23.607 | INFO | RTMetrics done | 20260507 18:43:23.715 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.627 seconds | 20260507 18:43:23.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.25% / 0.01% / 53.74% | 20260507 18:43:23.715 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.79539% / 1.58946e-05% / 4.76837e-05% | 20260507 18:43:23.715 | INFO | Baz IO overall: | 20260507 18:43:23.715 | INFO | BytesWritten : 780740820308 | 20260507 18:43:23.715 | INFO | NumEvents : 378522670417 | 20260507 18:43:25.129 | INFO | Analyzing chunk frames = [1740800,1741312) | 20260507 18:43:25.129 | INFO | MemInfo: { | 20260507 18:43:25.129 | INFO | "gpu" : " 17 GB", | 20260507 18:43:25.129 | INFO | "huge" : "", | 20260507 18:43:25.129 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:25.129 | INFO | "malloc" : " 14 GB", | 20260507 18:43:25.129 | INFO | "pinned" : " 29 GB" | 20260507 18:43:25.129 | INFO | } | 20260507 18:43:25.130 | INFO | Perf report for the last 5.13896 seconds: | 20260507 18:43:25.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:25.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:25.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:25.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:25.130 | INFO | ANALYSIS : 69.2%, 3.0, 107.2ms, 0 | 20260507 18:43:25.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:25.130 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 18:43:25.130 | INFO | PRE_HQ : 66.9%, 1.8, 61.7ms, 0 | 20260507 18:43:25.130 | INFO | BAZWRITER : 69.9%, 3.7, 133.3ms, 0 | 20260507 18:43:25.130 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:43:25.130 | INFO | Uploaded 36GiB to GPU in 2.52022s (14.2844GiB/s) | 20260507 18:43:25.130 | INFO | Downloaded 38GiB to GPU in 2.69917s (14.0784GiB/s) | 20260507 18:43:30.269 | INFO | Analyzing chunk frames = [1741312,1741824) | 20260507 18:43:30.269 | INFO | MemInfo: { | 20260507 18:43:30.269 | INFO | "gpu" : " 17 GB", | 20260507 18:43:30.269 | INFO | "huge" : "", | 20260507 18:43:30.269 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:30.269 | INFO | "malloc" : " 14 GB", | 20260507 18:43:30.269 | INFO | "pinned" : " 29 GB" | 20260507 18:43:30.269 | INFO | } | 20260507 18:43:30.269 | INFO | Perf report for the last 5.13967 seconds: | 20260507 18:43:30.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:30.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.269 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:43:30.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.269 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:43:30.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:30.270 | INFO | Uploaded 36GiB to GPU in 2.56721s (14.023GiB/s) | 20260507 18:43:30.270 | INFO | Downloaded 34GiB to GPU in 2.43862s (13.9423GiB/s) | 20260507 18:43:35.408 | INFO | Analyzing chunk frames = [1741824,1742336) | 20260507 18:43:35.408 | INFO | MemInfo: { | 20260507 18:43:35.408 | INFO | "gpu" : " 17 GB", | 20260507 18:43:35.408 | INFO | "huge" : "", | 20260507 18:43:35.408 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:35.408 | INFO | "malloc" : " 14 GB", | 20260507 18:43:35.408 | INFO | "pinned" : " 29 GB" | 20260507 18:43:35.408 | INFO | } | 20260507 18:43:35.408 | INFO | Perf report for the last 5.13874 seconds: | 20260507 18:43:35.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:35.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:43:35.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:43:35.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:35.408 | INFO | Uploaded 36GiB to GPU in 2.56719s (14.0231GiB/s) | 20260507 18:43:35.408 | INFO | Downloaded 34GiB to GPU in 2.43216s (13.9794GiB/s) | 20260507 18:43:40.547 | INFO | Analyzing chunk frames = [1742336,1742848) | 20260507 18:43:40.547 | INFO | MemInfo: { | 20260507 18:43:40.547 | INFO | "gpu" : " 17 GB", | 20260507 18:43:40.547 | INFO | "huge" : "", | 20260507 18:43:40.547 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:40.547 | INFO | "malloc" : " 14 GB", | 20260507 18:43:40.547 | INFO | "pinned" : " 29 GB" | 20260507 18:43:40.547 | INFO | } | 20260507 18:43:40.547 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:43:40.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:40.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.548 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:43:40.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.548 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:43:40.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:40.548 | INFO | Uploaded 36GiB to GPU in 2.50161s (14.3907GiB/s) | 20260507 18:43:40.548 | INFO | Downloaded 34GiB to GPU in 2.35552s (14.4342GiB/s) | 20260507 18:43:45.687 | INFO | Analyzing chunk frames = [1742848,1743360) | 20260507 18:43:45.687 | INFO | MemInfo: { | 20260507 18:43:45.687 | INFO | "gpu" : " 17 GB", | 20260507 18:43:45.687 | INFO | "huge" : "", | 20260507 18:43:45.687 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:45.687 | INFO | "malloc" : " 14 GB", | 20260507 18:43:45.687 | INFO | "pinned" : " 29 GB" | 20260507 18:43:45.688 | INFO | } | 20260507 18:43:45.688 | INFO | Perf report for the last 5.14016 seconds: | 20260507 18:43:45.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:45.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:43:45.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:43:45.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:45.688 | INFO | Uploaded 36GiB to GPU in 2.35982s (15.2554GiB/s) | 20260507 18:43:45.688 | INFO | Downloaded 34GiB to GPU in 2.47201s (13.754GiB/s) | 20260507 18:43:50.826 | INFO | Analyzing chunk frames = [1743360,1743872) | 20260507 18:43:50.826 | INFO | MemInfo: { | 20260507 18:43:50.826 | INFO | "gpu" : " 17 GB", | 20260507 18:43:50.826 | INFO | "huge" : "", | 20260507 18:43:50.826 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:50.826 | INFO | "malloc" : " 14 GB", | 20260507 18:43:50.826 | INFO | "pinned" : " 29 GB" | 20260507 18:43:50.826 | INFO | } | 20260507 18:43:50.826 | INFO | Perf report for the last 5.13849 seconds: | 20260507 18:43:50.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:50.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.826 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:43:50.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.826 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:43:50.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:50.827 | INFO | Uploaded 36GiB to GPU in 2.43972s (14.7558GiB/s) | 20260507 18:43:50.827 | INFO | Downloaded 34GiB to GPU in 2.49733s (13.6145GiB/s) | 20260507 18:43:55.966 | INFO | Analyzing chunk frames = [1743872,1744384) | 20260507 18:43:55.966 | INFO | MemInfo: { | 20260507 18:43:55.966 | INFO | "gpu" : " 17 GB", | 20260507 18:43:55.966 | INFO | "huge" : "", | 20260507 18:43:55.966 | INFO | "huge_shared" : " 29 GB", | 20260507 18:43:55.966 | INFO | "malloc" : " 14 GB", | 20260507 18:43:55.966 | INFO | "pinned" : " 29 GB" | 20260507 18:43:55.966 | INFO | } | 20260507 18:43:55.966 | INFO | Perf report for the last 5.13994 seconds: | 20260507 18:43:55.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:43:55.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:43:55.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:43:55.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:43:55.966 | INFO | Uploaded 36GiB to GPU in 2.51907s (14.291GiB/s) | 20260507 18:43:55.966 | INFO | Downloaded 34GiB to GPU in 2.42907s (13.9972GiB/s) | 20260507 18:44:01.105 | INFO | Analyzing chunk frames = [1744384,1744896) | 20260507 18:44:01.105 | INFO | MemInfo: { | 20260507 18:44:01.105 | INFO | "gpu" : " 17 GB", | 20260507 18:44:01.105 | INFO | "huge" : "", | 20260507 18:44:01.105 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:01.105 | INFO | "malloc" : " 14 GB", | 20260507 18:44:01.105 | INFO | "pinned" : " 29 GB" | 20260507 18:44:01.105 | INFO | } | 20260507 18:44:01.105 | INFO | Perf report for the last 5.13926 seconds: | 20260507 18:44:01.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:01.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.105 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:44:01.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.105 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:44:01.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:01.106 | INFO | Uploaded 36GiB to GPU in 2.5153s (14.3124GiB/s) | 20260507 18:44:01.106 | INFO | Downloaded 34GiB to GPU in 2.41113s (14.1013GiB/s) | 20260507 18:44:01.106 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:44:01.106 | INFO | Total mem usage: 16.4 GiB | 20260507 18:44:01.106 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:44:01.106 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -6.65%) | 20260507 18:44:01.106 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:44:01.106 | INFO | High Water Breakdown: | 20260507 18:44:01.106 | INFO | 60.2% : 9.26 GiB ( 93.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:44:01.106 | INFO | 39.6% : 6.1 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:44:01.106 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:44:01.106 | INFO | ------------------------------------------ | 20260507 18:44:01.106 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:44:01.106 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:44:01.106 | INFO | Total mem usage: 26.5 GiB | 20260507 18:44:01.106 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:44:01.106 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:44:01.106 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:44:01.106 | INFO | High Water Breakdown: | 20260507 18:44:01.106 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:44:01.106 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:44:01.106 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:44:01.106 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:44:01.106 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:44:01.106 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:44:01.106 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:44:01.106 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:44:01.106 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:44:01.106 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:44:01.106 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:44:01.106 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:44:01.106 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:44:01.106 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:44:01.106 | INFO | ------------------------------------------ | 20260507 18:44:01.106 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:44:01.106 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:44:01.106 | INFO | Total mem usage: 27.8 GiB | 20260507 18:44:01.106 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:44:01.106 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:44:01.106 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:44:01.106 | INFO | High Water Breakdown: | 20260507 18:44:01.106 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:44:01.106 | INFO | ------------------------------------------ | 20260507 18:44:01.106 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:44:01.106 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:44:01.107 | INFO | Total mem usage: 15.7 GiB | 20260507 18:44:01.107 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:44:01.107 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:44:01.107 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:44:01.107 | INFO | High Water Breakdown: | 20260507 18:44:01.107 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:44:01.107 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:44:01.107 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:44:01.107 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:44:01.107 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:44:01.107 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:44:01.107 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:44:01.107 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:44:01.107 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:44:01.107 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:44:01.107 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:44:01.107 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:44:01.107 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:44:01.107 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:44:01.107 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:44:01.107 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:44:01.107 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:44:01.107 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:44:01.107 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:44:01.107 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:44:01.107 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:44:01.107 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:44:01.107 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:44:01.107 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:44:01.107 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:44:01.107 | INFO | ------------------------------------------ | 20260507 18:44:01.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:44:01.107 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:44:01.107 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:44:01.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:44:01.107 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.43e+01 / 2.09e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.04e-01, 1.18e-02 / 6.66e-03 | 20260507 18:44:01.107 | INFO | PerfReport: 12.10%, Baseline -- 4.05e+00 / 4.49e+00, 6.17e-01 / 2.41e-01 | 20260507 18:44:01.107 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.14e-01 / 1.10e-01 | 20260507 18:44:01.107 | INFO | PerfReport: 15.67%, DME -- 4.19e+01 / 4.50e+01, 4.07e+00 / 1.92e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.88e+00 / 2.12e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.51e+00 / 2.80e+00, 4.32e-01 / 2.46e-01 | 20260507 18:44:01.107 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.43e+00, 3.13e-01 / 1.82e-01 | 20260507 18:44:01.107 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.16e-01, 6.30e-02 / 3.44e-02 | 20260507 18:44:01.107 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:44:01.107 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:44:01.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:44:01.107 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 24.14%, Upload -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 24.01%, Download -- 2.58e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 31.15%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 1.91%, LockUpload -- 2.06e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 2.19%, LockDownload -- 2.35e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 16.57%, LockCompute -- 1.78e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:44:01.107 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 18:44:01.202 | INFO | Starting a round of Baz IO | 20260507 18:44:04.706 | INFO | Populating RTMetrics report | 20260507 18:44:04.706 | INFO | RTMetrics report populated | 20260507 18:44:04.706 | INFO | Opening tmp RTMetrics json file | 20260507 18:44:04.707 | INFO | Serializing RTMetrics to json | 20260507 18:44:04.723 | INFO | Serialization done, swapping tmp file now... | 20260507 18:44:04.726 | INFO | RTMetrics done | 20260507 18:44:04.788 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.585 seconds | 20260507 18:44:04.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.23% / 0.01% / 53.75% | 20260507 18:44:04.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.78364% / 1.98682e-05% / 4.76837e-05% | 20260507 18:44:04.788 | INFO | Baz IO overall: | 20260507 18:44:04.788 | INFO | BytesWritten : 782675147429 | 20260507 18:44:04.788 | INFO | NumEvents : 379460993204 | 20260507 18:44:06.244 | INFO | Analyzing chunk frames = [1744896,1745408) | 20260507 18:44:06.244 | INFO | MemInfo: { | 20260507 18:44:06.244 | INFO | "gpu" : " 17 GB", | 20260507 18:44:06.244 | INFO | "huge" : "", | 20260507 18:44:06.244 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:06.244 | INFO | "malloc" : " 14 GB", | 20260507 18:44:06.244 | INFO | "pinned" : " 29 GB" | 20260507 18:44:06.244 | INFO | } | 20260507 18:44:06.244 | INFO | Perf report for the last 5.1391 seconds: | 20260507 18:44:06.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:06.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:06.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:06.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:06.245 | INFO | ANALYSIS : 69.3%, 3.0, 108.2ms, 0 | 20260507 18:44:06.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:06.245 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 18:44:06.245 | INFO | PRE_HQ : 66.7%, 1.7, 59.4ms, 0 | 20260507 18:44:06.245 | INFO | BAZWRITER : 68.8%, 2.9, 103.1ms, 0 | 20260507 18:44:06.245 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:44:06.245 | INFO | Uploaded 36GiB to GPU in 2.53498s (14.2013GiB/s) | 20260507 18:44:06.245 | INFO | Downloaded 38GiB to GPU in 2.72948s (13.9221GiB/s) | 20260507 18:44:11.384 | INFO | Analyzing chunk frames = [1745408,1745920) | 20260507 18:44:11.384 | INFO | MemInfo: { | 20260507 18:44:11.384 | INFO | "gpu" : " 17 GB", | 20260507 18:44:11.384 | INFO | "huge" : "", | 20260507 18:44:11.384 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:11.384 | INFO | "malloc" : " 14 GB", | 20260507 18:44:11.384 | INFO | "pinned" : " 29 GB" | 20260507 18:44:11.384 | INFO | } | 20260507 18:44:11.384 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:44:11.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:11.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:44:11.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:44:11.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:11.384 | INFO | Uploaded 36GiB to GPU in 2.50679s (14.361GiB/s) | 20260507 18:44:11.384 | INFO | Downloaded 34GiB to GPU in 2.45794s (13.8327GiB/s) | 20260507 18:44:16.523 | INFO | Analyzing chunk frames = [1745920,1746432) | 20260507 18:44:16.523 | INFO | MemInfo: { | 20260507 18:44:16.523 | INFO | "gpu" : " 17 GB", | 20260507 18:44:16.523 | INFO | "huge" : "", | 20260507 18:44:16.523 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:16.523 | INFO | "malloc" : " 14 GB", | 20260507 18:44:16.523 | INFO | "pinned" : " 29 GB" | 20260507 18:44:16.523 | INFO | } | 20260507 18:44:16.523 | INFO | Perf report for the last 5.13896 seconds: | 20260507 18:44:16.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:16.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:44:16.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:44:16.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:16.523 | INFO | Uploaded 36GiB to GPU in 2.54959s (14.1199GiB/s) | 20260507 18:44:16.523 | INFO | Downloaded 34GiB to GPU in 2.46741s (13.7797GiB/s) | 20260507 18:44:21.662 | INFO | Analyzing chunk frames = [1746432,1746944) | 20260507 18:44:21.662 | INFO | MemInfo: { | 20260507 18:44:21.662 | INFO | "gpu" : " 17 GB", | 20260507 18:44:21.662 | INFO | "huge" : "", | 20260507 18:44:21.662 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:21.662 | INFO | "malloc" : " 14 GB", | 20260507 18:44:21.662 | INFO | "pinned" : " 29 GB" | 20260507 18:44:21.662 | INFO | } | 20260507 18:44:21.662 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:44:21.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:21.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.662 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:44:21.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.662 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 18:44:21.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:21.663 | INFO | Uploaded 36GiB to GPU in 2.47225s (14.5617GiB/s) | 20260507 18:44:21.663 | INFO | Downloaded 34GiB to GPU in 2.37612s (14.309GiB/s) | 20260507 18:44:26.802 | INFO | Analyzing chunk frames = [1746944,1747456) | 20260507 18:44:26.802 | INFO | MemInfo: { | 20260507 18:44:26.802 | INFO | "gpu" : " 17 GB", | 20260507 18:44:26.802 | INFO | "huge" : "", | 20260507 18:44:26.802 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:26.802 | INFO | "malloc" : " 14 GB", | 20260507 18:44:26.802 | INFO | "pinned" : " 29 GB" | 20260507 18:44:26.802 | INFO | } | 20260507 18:44:26.802 | INFO | Perf report for the last 5.13985 seconds: | 20260507 18:44:26.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:26.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.802 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 18:44:26.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.802 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:44:26.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:26.803 | INFO | Uploaded 36GiB to GPU in 2.31129s (15.5757GiB/s) | 20260507 18:44:26.803 | INFO | Downloaded 34GiB to GPU in 2.48969s (13.6563GiB/s) | 20260507 18:44:31.941 | INFO | Analyzing chunk frames = [1747456,1747968) | 20260507 18:44:31.941 | INFO | MemInfo: { | 20260507 18:44:31.941 | INFO | "gpu" : " 17 GB", | 20260507 18:44:31.941 | INFO | "huge" : "", | 20260507 18:44:31.941 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:31.941 | INFO | "malloc" : " 14 GB", | 20260507 18:44:31.941 | INFO | "pinned" : " 29 GB" | 20260507 18:44:31.941 | INFO | } | 20260507 18:44:31.941 | INFO | Perf report for the last 5.13886 seconds: | 20260507 18:44:31.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:31.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:44:31.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:44:31.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:31.941 | INFO | Uploaded 36GiB to GPU in 2.42295s (14.8579GiB/s) | 20260507 18:44:31.941 | INFO | Downloaded 34GiB to GPU in 2.51158s (13.5373GiB/s) | 20260507 18:44:37.080 | INFO | Analyzing chunk frames = [1747968,1748480) | 20260507 18:44:37.080 | INFO | MemInfo: { | 20260507 18:44:37.080 | INFO | "gpu" : " 17 GB", | 20260507 18:44:37.080 | INFO | "huge" : "", | 20260507 18:44:37.080 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:37.080 | INFO | "malloc" : " 14 GB", | 20260507 18:44:37.080 | INFO | "pinned" : " 29 GB" | 20260507 18:44:37.080 | INFO | } | 20260507 18:44:37.080 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:44:37.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:37.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.080 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:44:37.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.080 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:44:37.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:37.081 | INFO | Uploaded 36GiB to GPU in 2.4765s (14.5366GiB/s) | 20260507 18:44:37.081 | INFO | Downloaded 34GiB to GPU in 2.43906s (13.9398GiB/s) | 20260507 18:44:42.220 | INFO | Analyzing chunk frames = [1748480,1748992) | 20260507 18:44:42.220 | INFO | MemInfo: { | 20260507 18:44:42.220 | INFO | "gpu" : " 17 GB", | 20260507 18:44:42.220 | INFO | "huge" : "", | 20260507 18:44:42.220 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:42.220 | INFO | "malloc" : " 14 GB", | 20260507 18:44:42.220 | INFO | "pinned" : " 29 GB" | 20260507 18:44:42.220 | INFO | } | 20260507 18:44:42.220 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:44:42.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:42.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:44:42.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:44:42.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:42.220 | INFO | Uploaded 36GiB to GPU in 2.48924s (14.4622GiB/s) | 20260507 18:44:42.220 | INFO | Downloaded 34GiB to GPU in 2.42924s (13.9962GiB/s) | 20260507 18:44:42.317 | INFO | Starting a round of Baz IO | 20260507 18:44:45.820 | INFO | Populating RTMetrics report | 20260507 18:44:45.820 | INFO | RTMetrics report populated | 20260507 18:44:45.820 | INFO | Opening tmp RTMetrics json file | 20260507 18:44:45.837 | INFO | Serializing RTMetrics to json | 20260507 18:44:45.839 | INFO | Serialization done, swapping tmp file now... | 20260507 18:44:45.839 | INFO | RTMetrics done | 20260507 18:44:45.899 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.582 seconds | 20260507 18:44:45.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.22% / 0.01% / 53.77% | 20260507 18:44:45.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.77311% / 1.19209e-05% / 4.76837e-05% | 20260507 18:44:45.899 | INFO | Baz IO overall: | 20260507 18:44:45.899 | INFO | BytesWritten : 784609569730 | 20260507 18:44:45.899 | INFO | NumEvents : 380399307121 | 20260507 18:44:47.359 | INFO | Analyzing chunk frames = [1748992,1749504) | 20260507 18:44:47.359 | INFO | MemInfo: { | 20260507 18:44:47.359 | INFO | "gpu" : " 17 GB", | 20260507 18:44:47.359 | INFO | "huge" : "", | 20260507 18:44:47.359 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:47.359 | INFO | "malloc" : " 14 GB", | 20260507 18:44:47.359 | INFO | "pinned" : " 29 GB" | 20260507 18:44:47.359 | INFO | } | 20260507 18:44:47.359 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:44:47.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:47.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:47.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:47.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:47.359 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 18:44:47.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:47.359 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:44:47.359 | INFO | PRE_HQ : 66.7%, 1.7, 59.3ms, 0 | 20260507 18:44:47.359 | INFO | BAZWRITER : 69.0%, 2.9, 103.3ms, 0 | 20260507 18:44:47.359 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:44:47.360 | INFO | Uploaded 36GiB to GPU in 2.57217s (13.996GiB/s) | 20260507 18:44:47.360 | INFO | Downloaded 38GiB to GPU in 2.72607s (13.9395GiB/s) | 20260507 18:44:52.498 | INFO | Analyzing chunk frames = [1749504,1750016) | 20260507 18:44:52.498 | INFO | MemInfo: { | 20260507 18:44:52.498 | INFO | "gpu" : " 17 GB", | 20260507 18:44:52.498 | INFO | "huge" : "", | 20260507 18:44:52.498 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:52.498 | INFO | "malloc" : " 14 GB", | 20260507 18:44:52.498 | INFO | "pinned" : " 29 GB" | 20260507 18:44:52.498 | INFO | } | 20260507 18:44:52.498 | INFO | Perf report for the last 5.13921 seconds: | 20260507 18:44:52.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:52.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.499 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260507 18:44:52.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.499 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:44:52.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:52.499 | INFO | Uploaded 36GiB to GPU in 2.53614s (14.1948GiB/s) | 20260507 18:44:52.499 | INFO | Downloaded 34GiB to GPU in 2.46041s (13.8188GiB/s) | 20260507 18:44:57.638 | INFO | Analyzing chunk frames = [1750016,1750528) | 20260507 18:44:57.638 | INFO | MemInfo: { | 20260507 18:44:57.638 | INFO | "gpu" : " 17 GB", | 20260507 18:44:57.638 | INFO | "huge" : "", | 20260507 18:44:57.638 | INFO | "huge_shared" : " 29 GB", | 20260507 18:44:57.638 | INFO | "malloc" : " 14 GB", | 20260507 18:44:57.638 | INFO | "pinned" : " 29 GB" | 20260507 18:44:57.638 | INFO | } | 20260507 18:44:57.638 | INFO | Perf report for the last 5.13975 seconds: | 20260507 18:44:57.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:44:57.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.638 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:44:57.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.638 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:44:57.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:44:57.639 | INFO | Uploaded 36GiB to GPU in 2.54771s (14.1303GiB/s) | 20260507 18:44:57.639 | INFO | Downloaded 34GiB to GPU in 2.45328s (13.859GiB/s) | 20260507 18:45:02.777 | INFO | Analyzing chunk frames = [1750528,1751040) | 20260507 18:45:02.777 | INFO | MemInfo: { | 20260507 18:45:02.777 | INFO | "gpu" : " 17 GB", | 20260507 18:45:02.777 | INFO | "huge" : "", | 20260507 18:45:02.777 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:02.777 | INFO | "malloc" : " 14 GB", | 20260507 18:45:02.777 | INFO | "pinned" : " 29 GB" | 20260507 18:45:02.777 | INFO | } | 20260507 18:45:02.777 | INFO | Perf report for the last 5.13888 seconds: | 20260507 18:45:02.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:02.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.777 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:45:02.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.777 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:45:02.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:02.778 | INFO | Uploaded 36GiB to GPU in 2.48104s (14.51GiB/s) | 20260507 18:45:02.778 | INFO | Downloaded 34GiB to GPU in 2.36714s (14.3634GiB/s) | 20260507 18:45:07.917 | INFO | Analyzing chunk frames = [1751040,1751552) | 20260507 18:45:07.917 | INFO | MemInfo: { | 20260507 18:45:07.917 | INFO | "gpu" : " 17 GB", | 20260507 18:45:07.917 | INFO | "huge" : "", | 20260507 18:45:07.917 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:07.917 | INFO | "malloc" : " 14 GB", | 20260507 18:45:07.917 | INFO | "pinned" : " 29 GB" | 20260507 18:45:07.917 | INFO | } | 20260507 18:45:07.917 | INFO | Perf report for the last 5.13991 seconds: | 20260507 18:45:07.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:07.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.917 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:45:07.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.917 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:45:07.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:07.918 | INFO | Uploaded 36GiB to GPU in 2.336s (15.411GiB/s) | 20260507 18:45:07.918 | INFO | Downloaded 34GiB to GPU in 2.49613s (13.6211GiB/s) | 20260507 18:45:13.056 | INFO | Analyzing chunk frames = [1751552,1752064) | 20260507 18:45:13.056 | INFO | MemInfo: { | 20260507 18:45:13.056 | INFO | "gpu" : " 17 GB", | 20260507 18:45:13.056 | INFO | "huge" : "", | 20260507 18:45:13.056 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:13.056 | INFO | "malloc" : " 14 GB", | 20260507 18:45:13.056 | INFO | "pinned" : " 29 GB" | 20260507 18:45:13.056 | INFO | } | 20260507 18:45:13.056 | INFO | Perf report for the last 5.13905 seconds: | 20260507 18:45:13.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:13.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.056 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:45:13.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.056 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:45:13.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:13.057 | INFO | Uploaded 36GiB to GPU in 2.45463s (14.6662GiB/s) | 20260507 18:45:13.057 | INFO | Downloaded 34GiB to GPU in 2.52462s (13.4674GiB/s) | 20260507 18:45:18.195 | INFO | Analyzing chunk frames = [1752064,1752576) | 20260507 18:45:18.195 | INFO | MemInfo: { | 20260507 18:45:18.195 | INFO | "gpu" : " 17 GB", | 20260507 18:45:18.195 | INFO | "huge" : "", | 20260507 18:45:18.195 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:18.195 | INFO | "malloc" : " 14 GB", | 20260507 18:45:18.195 | INFO | "pinned" : " 29 GB" | 20260507 18:45:18.195 | INFO | } | 20260507 18:45:18.195 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:45:18.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:18.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.195 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:45:18.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.195 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:45:18.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:18.196 | INFO | Uploaded 36GiB to GPU in 2.48457s (14.4895GiB/s) | 20260507 18:45:18.196 | INFO | Downloaded 34GiB to GPU in 2.43787s (13.9466GiB/s) | 20260507 18:45:23.334 | INFO | Analyzing chunk frames = [1752576,1753088) | 20260507 18:45:23.334 | INFO | MemInfo: { | 20260507 18:45:23.334 | INFO | "gpu" : " 17 GB", | 20260507 18:45:23.334 | INFO | "huge" : "", | 20260507 18:45:23.335 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:23.335 | INFO | "malloc" : " 14 GB", | 20260507 18:45:23.335 | INFO | "pinned" : " 29 GB" | 20260507 18:45:23.335 | INFO | } | 20260507 18:45:23.335 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:45:23.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:23.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:45:23.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:45:23.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:23.335 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:45:23.335 | INFO | Total mem usage: 16.4 GiB | 20260507 18:45:23.335 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:45:23.335 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.6%) | 20260507 18:45:23.335 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:45:23.335 | INFO | High Water Breakdown: | 20260507 18:45:23.335 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:45:23.335 | INFO | 40.7% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:45:23.335 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:45:23.335 | INFO | ------------------------------------------ | 20260507 18:45:23.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:45:23.335 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:45:23.335 | INFO | Total mem usage: 26.5 GiB | 20260507 18:45:23.335 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:45:23.335 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:45:23.335 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:45:23.335 | INFO | High Water Breakdown: | 20260507 18:45:23.335 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:45:23.335 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:45:23.335 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:45:23.335 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:45:23.335 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:45:23.335 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:45:23.335 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:45:23.335 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:45:23.335 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:45:23.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:45:23.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:45:23.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:45:23.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:45:23.335 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:45:23.335 | INFO | ------------------------------------------ | 20260507 18:45:23.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:45:23.335 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:45:23.335 | INFO | Total mem usage: 27.8 GiB | 20260507 18:45:23.335 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:45:23.335 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:45:23.335 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:45:23.335 | INFO | High Water Breakdown: | 20260507 18:45:23.335 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:45:23.335 | INFO | ------------------------------------------ | 20260507 18:45:23.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:45:23.335 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:45:23.335 | INFO | Total mem usage: 15.7 GiB | 20260507 18:45:23.335 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:45:23.335 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:45:23.335 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:45:23.335 | INFO | High Water Breakdown: | 20260507 18:45:23.335 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:45:23.335 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:45:23.335 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:45:23.335 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:45:23.335 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:45:23.335 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:45:23.335 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:45:23.335 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:45:23.335 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:45:23.335 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:45:23.335 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:45:23.335 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:45:23.335 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:45:23.335 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:45:23.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:45:23.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:45:23.335 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:45:23.335 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:45:23.335 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:45:23.335 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:45:23.335 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:45:23.335 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:45:23.335 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:45:23.335 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:45:23.335 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:45:23.335 | INFO | ------------------------------------------ | 20260507 18:45:23.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:45:23.335 | INFO | Uploaded 36GiB to GPU in 2.46889s (14.5815GiB/s) | 20260507 18:45:23.335 | INFO | Downloaded 34GiB to GPU in 2.42812s (14.0026GiB/s) | 20260507 18:45:23.335 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:45:23.335 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:45:23.335 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:45:23.335 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.66e+01, 1.44e+01 / 2.10e+00 | 20260507 18:45:23.335 | INFO | PerfReport: 0.29%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.15e-02 / 5.70e-03 | 20260507 18:45:23.335 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.53e+00, 6.15e-01 / 1.87e-01 | 20260507 18:45:23.335 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 1.02e-01 | 20260507 18:45:23.335 | INFO | PerfReport: 15.71%, DME -- 4.20e+01 / 4.51e+01, 4.09e+00 / 1.70e+00 | 20260507 18:45:23.335 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.89e+00 / 2.09e+00 | 20260507 18:45:23.335 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.31e-01 / 2.55e-01 | 20260507 18:45:23.335 | INFO | PerfReport: 3.68%, Metrics -- 1.23e+00 / 1.42e+00, 3.08e-01 / 1.39e-01 | 20260507 18:45:23.335 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.86e-01 / 4.13e-01, 6.15e-02 / 3.14e-02 | 20260507 18:45:23.335 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:45:23.336 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:45:23.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:45:23.336 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 24.05%, Upload -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 24.14%, Download -- 2.59e+01 / 0.00e+00, 8.60e+00 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 31.12%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 1.71%, LockUpload -- 1.84e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 2.38%, LockDownload -- 2.55e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 16.58%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:45:23.336 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 18:45:23.432 | INFO | Starting a round of Baz IO | 20260507 18:45:26.943 | INFO | Populating RTMetrics report | 20260507 18:45:26.943 | INFO | RTMetrics report populated | 20260507 18:45:26.943 | INFO | Opening tmp RTMetrics json file | 20260507 18:45:26.961 | INFO | Serializing RTMetrics to json | 20260507 18:45:26.962 | INFO | Serialization done, swapping tmp file now... | 20260507 18:45:26.963 | INFO | RTMetrics done | 20260507 18:45:27.023 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.59 seconds | 20260507 18:45:27.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.21% / 0.01% / 53.78% | 20260507 18:45:27.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.77133% / 2.38419e-05% / 3.17891e-05% | 20260507 18:45:27.023 | INFO | Baz IO overall: | 20260507 18:45:27.023 | INFO | BytesWritten : 786544671175 | 20260507 18:45:27.023 | INFO | NumEvents : 381338113923 | 20260507 18:45:28.474 | INFO | Analyzing chunk frames = [1753088,1753600) | 20260507 18:45:28.474 | INFO | MemInfo: { | 20260507 18:45:28.474 | INFO | "gpu" : " 17 GB", | 20260507 18:45:28.474 | INFO | "huge" : "", | 20260507 18:45:28.474 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:28.474 | INFO | "malloc" : " 14 GB", | 20260507 18:45:28.474 | INFO | "pinned" : " 29 GB" | 20260507 18:45:28.474 | INFO | } | 20260507 18:45:28.474 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:45:28.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:28.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:28.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:28.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:28.474 | INFO | ANALYSIS : 69.4%, 3.0, 108.4ms, 0 | 20260507 18:45:28.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:28.474 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 18:45:28.474 | INFO | PRE_HQ : 66.4%, 1.7, 59.3ms, 0 | 20260507 18:45:28.474 | INFO | BAZWRITER : 69.0%, 3.0, 105.1ms, 0 | 20260507 18:45:28.474 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:45:28.475 | INFO | Uploaded 36GiB to GPU in 2.53161s (14.2202GiB/s) | 20260507 18:45:28.475 | INFO | Downloaded 38GiB to GPU in 2.71572s (13.9926GiB/s) | 20260507 18:45:33.613 | INFO | Analyzing chunk frames = [1753600,1754112) | 20260507 18:45:33.613 | INFO | MemInfo: { | 20260507 18:45:33.613 | INFO | "gpu" : " 17 GB", | 20260507 18:45:33.613 | INFO | "huge" : "", | 20260507 18:45:33.613 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:33.613 | INFO | "malloc" : " 14 GB", | 20260507 18:45:33.613 | INFO | "pinned" : " 29 GB" | 20260507 18:45:33.613 | INFO | } | 20260507 18:45:33.613 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:45:33.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:33.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.613 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:45:33.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.614 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:45:33.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:33.614 | INFO | Uploaded 36GiB to GPU in 2.54374s (14.1524GiB/s) | 20260507 18:45:33.614 | INFO | Downloaded 34GiB to GPU in 2.44339s (13.9151GiB/s) | 20260507 18:45:38.752 | INFO | Analyzing chunk frames = [1754112,1754624) | 20260507 18:45:38.752 | INFO | MemInfo: { | 20260507 18:45:38.752 | INFO | "gpu" : " 17 GB", | 20260507 18:45:38.752 | INFO | "huge" : "", | 20260507 18:45:38.753 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:38.753 | INFO | "malloc" : " 14 GB", | 20260507 18:45:38.753 | INFO | "pinned" : " 29 GB" | 20260507 18:45:38.753 | INFO | } | 20260507 18:45:38.753 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:45:38.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:38.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:45:38.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:45:38.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:38.753 | INFO | Uploaded 36GiB to GPU in 2.56712s (14.0235GiB/s) | 20260507 18:45:38.753 | INFO | Downloaded 34GiB to GPU in 2.41993s (14.05GiB/s) | 20260507 18:45:43.892 | INFO | Analyzing chunk frames = [1754624,1755136) | 20260507 18:45:43.892 | INFO | MemInfo: { | 20260507 18:45:43.892 | INFO | "gpu" : " 17 GB", | 20260507 18:45:43.892 | INFO | "huge" : "", | 20260507 18:45:43.892 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:43.892 | INFO | "malloc" : " 14 GB", | 20260507 18:45:43.892 | INFO | "pinned" : " 29 GB" | 20260507 18:45:43.892 | INFO | } | 20260507 18:45:43.892 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:45:43.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:43.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:45:43.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:45:43.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:43.892 | INFO | Uploaded 36GiB to GPU in 2.50221s (14.3873GiB/s) | 20260507 18:45:43.892 | INFO | Downloaded 34GiB to GPU in 2.34929s (14.4724GiB/s) | 20260507 18:45:49.031 | INFO | Analyzing chunk frames = [1755136,1755648) | 20260507 18:45:49.031 | INFO | MemInfo: { | 20260507 18:45:49.031 | INFO | "gpu" : " 17 GB", | 20260507 18:45:49.031 | INFO | "huge" : "", | 20260507 18:45:49.031 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:49.031 | INFO | "malloc" : " 14 GB", | 20260507 18:45:49.031 | INFO | "pinned" : " 29 GB" | 20260507 18:45:49.032 | INFO | } | 20260507 18:45:49.032 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:45:49.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:49.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 18:45:49.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:45:49.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:49.032 | INFO | Uploaded 36GiB to GPU in 2.36812s (15.2019GiB/s) | 20260507 18:45:49.032 | INFO | Downloaded 34GiB to GPU in 2.4777s (13.7224GiB/s) | 20260507 18:45:54.171 | INFO | Analyzing chunk frames = [1755648,1756160) | 20260507 18:45:54.171 | INFO | MemInfo: { | 20260507 18:45:54.171 | INFO | "gpu" : " 17 GB", | 20260507 18:45:54.171 | INFO | "huge" : "", | 20260507 18:45:54.171 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:54.171 | INFO | "malloc" : " 14 GB", | 20260507 18:45:54.171 | INFO | "pinned" : " 29 GB" | 20260507 18:45:54.171 | INFO | } | 20260507 18:45:54.171 | INFO | Perf report for the last 5.13896 seconds: | 20260507 18:45:54.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:54.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:45:54.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:45:54.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:54.171 | INFO | Uploaded 36GiB to GPU in 2.42583s (14.8403GiB/s) | 20260507 18:45:54.171 | INFO | Downloaded 34GiB to GPU in 2.48555s (13.679GiB/s) | 20260507 18:45:59.310 | INFO | Analyzing chunk frames = [1756160,1756672) | 20260507 18:45:59.310 | INFO | MemInfo: { | 20260507 18:45:59.310 | INFO | "gpu" : " 17 GB", | 20260507 18:45:59.310 | INFO | "huge" : "", | 20260507 18:45:59.310 | INFO | "huge_shared" : " 29 GB", | 20260507 18:45:59.310 | INFO | "malloc" : " 14 GB", | 20260507 18:45:59.310 | INFO | "pinned" : " 29 GB" | 20260507 18:45:59.310 | INFO | } | 20260507 18:45:59.311 | INFO | Perf report for the last 5.13989 seconds: | 20260507 18:45:59.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:45:59.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:45:59.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 18:45:59.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:45:59.311 | INFO | Uploaded 36GiB to GPU in 2.51257s (14.3279GiB/s) | 20260507 18:45:59.311 | INFO | Downloaded 34GiB to GPU in 2.43171s (13.9819GiB/s) | 20260507 18:46:04.449 | INFO | Analyzing chunk frames = [1756672,1757184) | 20260507 18:46:04.449 | INFO | MemInfo: { | 20260507 18:46:04.449 | INFO | "gpu" : " 17 GB", | 20260507 18:46:04.449 | INFO | "huge" : "", | 20260507 18:46:04.449 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:04.449 | INFO | "malloc" : " 14 GB", | 20260507 18:46:04.449 | INFO | "pinned" : " 29 GB" | 20260507 18:46:04.449 | INFO | } | 20260507 18:46:04.449 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:46:04.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:04.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.450 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:46:04.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.450 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 18:46:04.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:04.450 | INFO | Uploaded 36GiB to GPU in 2.52112s (14.2793GiB/s) | 20260507 18:46:04.450 | INFO | Downloaded 34GiB to GPU in 2.41755s (14.0639GiB/s) | 20260507 18:46:04.562 | INFO | Starting a round of Baz IO | 20260507 18:46:08.059 | INFO | Populating RTMetrics report | 20260507 18:46:08.059 | INFO | RTMetrics report populated | 20260507 18:46:08.059 | INFO | Opening tmp RTMetrics json file | 20260507 18:46:08.077 | INFO | Serializing RTMetrics to json | 20260507 18:46:08.078 | INFO | Serialization done, swapping tmp file now... | 20260507 18:46:08.079 | INFO | RTMetrics done | 20260507 18:46:08.138 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.576 seconds | 20260507 18:46:08.139 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.19% / 0.01% / 53.79% | 20260507 18:46:08.139 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.76473% / 1.58946e-05% / 6.35783e-05% | 20260507 18:46:08.139 | INFO | Baz IO overall: | 20260507 18:46:08.139 | INFO | BytesWritten : 788479574471 | 20260507 18:46:08.139 | INFO | NumEvents : 382276739407 | 20260507 18:46:09.588 | INFO | Analyzing chunk frames = [1757184,1757696) | 20260507 18:46:09.589 | INFO | MemInfo: { | 20260507 18:46:09.589 | INFO | "gpu" : " 17 GB", | 20260507 18:46:09.589 | INFO | "huge" : "", | 20260507 18:46:09.589 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:09.589 | INFO | "malloc" : " 14 GB", | 20260507 18:46:09.589 | INFO | "pinned" : " 29 GB" | 20260507 18:46:09.589 | INFO | } | 20260507 18:46:09.589 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:46:09.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:09.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:09.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:09.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:09.589 | INFO | ANALYSIS : 69.5%, 2.9, 106.3ms, 0 | 20260507 18:46:09.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:09.589 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 18:46:09.589 | INFO | PRE_HQ : 67.7%, 1.9, 66.2ms, 0 | 20260507 18:46:09.589 | INFO | BAZWRITER : 69.0%, 3.2, 111.9ms, 0 | 20260507 18:46:09.589 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:46:09.589 | INFO | Uploaded 36GiB to GPU in 2.5841s (13.9313GiB/s) | 20260507 18:46:09.589 | INFO | Downloaded 38GiB to GPU in 2.69296s (14.1109GiB/s) | 20260507 18:46:14.728 | INFO | Analyzing chunk frames = [1757696,1758208) | 20260507 18:46:14.728 | INFO | MemInfo: { | 20260507 18:46:14.728 | INFO | "gpu" : " 17 GB", | 20260507 18:46:14.728 | INFO | "huge" : "", | 20260507 18:46:14.728 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:14.728 | INFO | "malloc" : " 14 GB", | 20260507 18:46:14.728 | INFO | "pinned" : " 29 GB" | 20260507 18:46:14.728 | INFO | } | 20260507 18:46:14.728 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:46:14.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:14.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:46:14.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:46:14.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:14.728 | INFO | Uploaded 36GiB to GPU in 2.53305s (14.2121GiB/s) | 20260507 18:46:14.729 | INFO | Downloaded 34GiB to GPU in 2.44633s (13.8983GiB/s) | 20260507 18:46:19.867 | INFO | Analyzing chunk frames = [1758208,1758720) | 20260507 18:46:19.867 | INFO | MemInfo: { | 20260507 18:46:19.867 | INFO | "gpu" : " 17 GB", | 20260507 18:46:19.867 | INFO | "huge" : "", | 20260507 18:46:19.867 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:19.867 | INFO | "malloc" : " 14 GB", | 20260507 18:46:19.867 | INFO | "pinned" : " 29 GB" | 20260507 18:46:19.867 | INFO | } | 20260507 18:46:19.867 | INFO | Perf report for the last 5.13926 seconds: | 20260507 18:46:19.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:19.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.868 | INFO | ANALYSIS : 68.1%, 3.0, 105.5ms, 0 | 20260507 18:46:19.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.868 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:46:19.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:19.868 | INFO | Uploaded 36GiB to GPU in 2.56459s (14.0373GiB/s) | 20260507 18:46:19.868 | INFO | Downloaded 34GiB to GPU in 2.4343s (13.967GiB/s) | 20260507 18:46:25.007 | INFO | Analyzing chunk frames = [1758720,1759232) | 20260507 18:46:25.007 | INFO | MemInfo: { | 20260507 18:46:25.007 | INFO | "gpu" : " 17 GB", | 20260507 18:46:25.007 | INFO | "huge" : "", | 20260507 18:46:25.007 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:25.007 | INFO | "malloc" : " 14 GB", | 20260507 18:46:25.007 | INFO | "pinned" : " 29 GB" | 20260507 18:46:25.007 | INFO | } | 20260507 18:46:25.007 | INFO | Perf report for the last 5.13957 seconds: | 20260507 18:46:25.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:25.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.007 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:46:25.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.007 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:46:25.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:25.008 | INFO | Uploaded 36GiB to GPU in 2.49139s (14.4498GiB/s) | 20260507 18:46:25.008 | INFO | Downloaded 34GiB to GPU in 2.35836s (14.4168GiB/s) | 20260507 18:46:30.146 | INFO | Analyzing chunk frames = [1759232,1759744) | 20260507 18:46:30.146 | INFO | MemInfo: { | 20260507 18:46:30.146 | INFO | "gpu" : " 17 GB", | 20260507 18:46:30.146 | INFO | "huge" : "", | 20260507 18:46:30.146 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:30.146 | INFO | "malloc" : " 14 GB", | 20260507 18:46:30.146 | INFO | "pinned" : " 29 GB" | 20260507 18:46:30.146 | INFO | } | 20260507 18:46:30.147 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:46:30.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:30.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:46:30.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:46:30.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:30.147 | INFO | Uploaded 36GiB to GPU in 2.3527s (15.3016GiB/s) | 20260507 18:46:30.147 | INFO | Downloaded 34GiB to GPU in 2.47939s (13.713GiB/s) | 20260507 18:46:35.285 | INFO | Analyzing chunk frames = [1759744,1760256) | 20260507 18:46:35.285 | INFO | MemInfo: { | 20260507 18:46:35.285 | INFO | "gpu" : " 17 GB", | 20260507 18:46:35.285 | INFO | "huge" : "", | 20260507 18:46:35.285 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:35.285 | INFO | "malloc" : " 14 GB", | 20260507 18:46:35.285 | INFO | "pinned" : " 29 GB" | 20260507 18:46:35.285 | INFO | } | 20260507 18:46:35.285 | INFO | Perf report for the last 5.13875 seconds: | 20260507 18:46:35.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:35.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:46:35.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:46:35.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:35.286 | INFO | Uploaded 36GiB to GPU in 2.41209s (14.9248GiB/s) | 20260507 18:46:35.286 | INFO | Downloaded 34GiB to GPU in 2.49255s (13.6407GiB/s) | 20260507 18:46:40.424 | INFO | Analyzing chunk frames = [1760256,1760768) | 20260507 18:46:40.424 | INFO | MemInfo: { | 20260507 18:46:40.424 | INFO | "gpu" : " 17 GB", | 20260507 18:46:40.424 | INFO | "huge" : "", | 20260507 18:46:40.425 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:40.425 | INFO | "malloc" : " 14 GB", | 20260507 18:46:40.425 | INFO | "pinned" : " 29 GB" | 20260507 18:46:40.425 | INFO | } | 20260507 18:46:40.425 | INFO | Perf report for the last 5.13925 seconds: | 20260507 18:46:40.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:40.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:46:40.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 18:46:40.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:40.425 | INFO | Uploaded 36GiB to GPU in 2.51883s (14.2924GiB/s) | 20260507 18:46:40.425 | INFO | Downloaded 34GiB to GPU in 2.43167s (13.9821GiB/s) | 20260507 18:46:45.564 | INFO | Analyzing chunk frames = [1760768,1761280) | 20260507 18:46:45.564 | INFO | MemInfo: { | 20260507 18:46:45.564 | INFO | "gpu" : " 17 GB", | 20260507 18:46:45.564 | INFO | "huge" : "", | 20260507 18:46:45.564 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:45.564 | INFO | "malloc" : " 14 GB", | 20260507 18:46:45.564 | INFO | "pinned" : " 29 GB" | 20260507 18:46:45.564 | INFO | } | 20260507 18:46:45.564 | INFO | Perf report for the last 5.13963 seconds: | 20260507 18:46:45.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:45.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.564 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:46:45.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.564 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 18:46:45.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:45.565 | INFO | Uploaded 36GiB to GPU in 2.5376s (14.1866GiB/s) | 20260507 18:46:45.565 | INFO | Downloaded 34GiB to GPU in 2.41669s (14.0689GiB/s) | 20260507 18:46:45.565 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:46:45.565 | INFO | Total mem usage: 16.4 GiB | 20260507 18:46:45.565 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:46:45.565 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 18:46:45.565 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:46:45.565 | INFO | High Water Breakdown: | 20260507 18:46:45.565 | INFO | 59.3% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:46:45.565 | INFO | 40.6% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:46:45.565 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:46:45.565 | INFO | ------------------------------------------ | 20260507 18:46:45.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:46:45.565 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:46:45.565 | INFO | Total mem usage: 26.5 GiB | 20260507 18:46:45.565 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:46:45.565 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:46:45.565 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 18:46:45.565 | INFO | High Water Breakdown: | 20260507 18:46:45.565 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:46:45.565 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:46:45.565 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:46:45.565 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:46:45.565 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:46:45.565 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:46:45.565 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:46:45.565 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:46:45.565 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:46:45.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:46:45.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:46:45.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:46:45.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:46:45.565 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:46:45.565 | INFO | ------------------------------------------ | 20260507 18:46:45.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:46:45.565 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:46:45.565 | INFO | Total mem usage: 27.8 GiB | 20260507 18:46:45.565 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:46:45.565 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:46:45.565 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:46:45.565 | INFO | High Water Breakdown: | 20260507 18:46:45.565 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:46:45.565 | INFO | ------------------------------------------ | 20260507 18:46:45.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:46:45.565 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:46:45.565 | INFO | Total mem usage: 15.7 GiB | 20260507 18:46:45.565 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:46:45.565 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:46:45.565 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:46:45.565 | INFO | High Water Breakdown: | 20260507 18:46:45.565 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:46:45.565 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:46:45.565 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:46:45.565 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:46:45.565 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:46:45.565 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:46:45.565 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:46:45.565 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:46:45.565 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:46:45.565 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:46:45.565 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:46:45.565 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:46:45.565 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:46:45.566 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:46:45.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:46:45.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:46:45.566 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:46:45.566 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:46:45.566 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:46:45.566 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:46:45.566 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:46:45.566 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:46:45.566 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:46:45.566 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:46:45.566 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:46:45.566 | INFO | ------------------------------------------ | 20260507 18:46:45.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:46:45.566 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:46:45.566 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:46:45.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:46:45.566 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.43e+01 / 2.25e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 0.30%, AggregateStats -- 9.95e-02 / 1.07e-01, 1.20e-02 / 1.56e-02 | 20260507 18:46:45.566 | INFO | PerfReport: 12.11%, Baseline -- 4.05e+00 / 4.50e+00, 6.15e-01 / 2.72e-01 | 20260507 18:46:45.566 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.02e-01 | 20260507 18:46:45.566 | INFO | PerfReport: 15.68%, DME -- 4.19e+01 / 4.48e+01, 3.95e+00 / 2.09e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.90e+00 / 2.09e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.49e+00 / 2.82e+00, 4.31e-01 / 2.28e-01 | 20260507 18:46:45.566 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.43e+00, 3.12e-01 / 1.75e-01 | 20260507 18:46:45.566 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.21e-01, 6.32e-02 / 4.18e-02 | 20260507 18:46:45.566 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:46:45.566 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:46:45.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:46:45.566 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 24.24%, Upload -- 2.60e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 23.95%, Download -- 2.57e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 31.14%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 2.01%, LockUpload -- 2.16e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 2.08%, LockDownload -- 2.24e+00 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 16.55%, LockCompute -- 1.78e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:46:45.566 | INFO | PerfReport: 0.01%, Misc -- 1.55e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260507 18:46:45.674 | INFO | Starting a round of Baz IO | 20260507 18:46:49.161 | INFO | Populating RTMetrics report | 20260507 18:46:49.162 | INFO | RTMetrics report populated | 20260507 18:46:49.162 | INFO | Opening tmp RTMetrics json file | 20260507 18:46:49.171 | INFO | Serializing RTMetrics to json | 20260507 18:46:49.180 | INFO | Serialization done, swapping tmp file now... | 20260507 18:46:49.180 | INFO | RTMetrics done | 20260507 18:46:49.243 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.569 seconds | 20260507 18:46:49.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.18% / 0.01% / 53.81% | 20260507 18:46:49.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.76262% / 1.58946e-05% / 4.37101e-05% | 20260507 18:46:49.243 | INFO | Baz IO overall: | 20260507 18:46:49.243 | INFO | BytesWritten : 790414799904 | 20260507 18:46:49.243 | INFO | NumEvents : 383215437559 | 20260507 18:46:50.703 | INFO | Analyzing chunk frames = [1761280,1761792) | 20260507 18:46:50.703 | INFO | MemInfo: { | 20260507 18:46:50.703 | INFO | "gpu" : " 17 GB", | 20260507 18:46:50.703 | INFO | "huge" : "", | 20260507 18:46:50.703 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:50.703 | INFO | "malloc" : " 14 GB", | 20260507 18:46:50.703 | INFO | "pinned" : " 29 GB" | 20260507 18:46:50.703 | INFO | } | 20260507 18:46:50.703 | INFO | Perf report for the last 5.13902 seconds: | 20260507 18:46:50.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:50.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:50.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:50.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:50.703 | INFO | ANALYSIS : 69.5%, 2.9, 107.1ms, 0 | 20260507 18:46:50.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:50.703 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 18:46:50.703 | INFO | PRE_HQ : 66.9%, 1.8, 64.1ms, 0 | 20260507 18:46:50.703 | INFO | BAZWRITER : 68.5%, 3.1, 108.8ms, 0 | 20260507 18:46:50.703 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:46:50.704 | INFO | Uploaded 36GiB to GPU in 2.58892s (13.9054GiB/s) | 20260507 18:46:50.704 | INFO | Downloaded 38GiB to GPU in 2.70661s (14.0397GiB/s) | 20260507 18:46:55.843 | INFO | Analyzing chunk frames = [1761792,1762304) | 20260507 18:46:55.843 | INFO | MemInfo: { | 20260507 18:46:55.843 | INFO | "gpu" : " 17 GB", | 20260507 18:46:55.843 | INFO | "huge" : "", | 20260507 18:46:55.843 | INFO | "huge_shared" : " 29 GB", | 20260507 18:46:55.843 | INFO | "malloc" : " 14 GB", | 20260507 18:46:55.843 | INFO | "pinned" : " 29 GB" | 20260507 18:46:55.843 | INFO | } | 20260507 18:46:55.843 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:46:55.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:46:55.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:46:55.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:46:55.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:46:55.843 | INFO | Uploaded 36GiB to GPU in 2.5569s (14.0796GiB/s) | 20260507 18:46:55.843 | INFO | Downloaded 34GiB to GPU in 2.44336s (13.9153GiB/s) | 20260507 18:47:00.982 | INFO | Analyzing chunk frames = [1762304,1762816) | 20260507 18:47:00.982 | INFO | MemInfo: { | 20260507 18:47:00.982 | INFO | "gpu" : " 17 GB", | 20260507 18:47:00.982 | INFO | "huge" : "", | 20260507 18:47:00.982 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:00.982 | INFO | "malloc" : " 14 GB", | 20260507 18:47:00.982 | INFO | "pinned" : " 29 GB" | 20260507 18:47:00.982 | INFO | } | 20260507 18:47:00.982 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:47:00.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:00.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:47:00.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:47:00.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:00.982 | INFO | Uploaded 36GiB to GPU in 2.56602s (14.0295GiB/s) | 20260507 18:47:00.983 | INFO | Downloaded 34GiB to GPU in 2.43162s (13.9825GiB/s) | 20260507 18:47:06.121 | INFO | Analyzing chunk frames = [1762816,1763328) | 20260507 18:47:06.121 | INFO | MemInfo: { | 20260507 18:47:06.121 | INFO | "gpu" : " 17 GB", | 20260507 18:47:06.121 | INFO | "huge" : "", | 20260507 18:47:06.121 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:06.121 | INFO | "malloc" : " 14 GB", | 20260507 18:47:06.121 | INFO | "pinned" : " 29 GB" | 20260507 18:47:06.121 | INFO | } | 20260507 18:47:06.121 | INFO | Perf report for the last 5.13914 seconds: | 20260507 18:47:06.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:06.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.121 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:47:06.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.121 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:47:06.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:06.122 | INFO | Uploaded 36GiB to GPU in 2.50228s (14.3869GiB/s) | 20260507 18:47:06.122 | INFO | Downloaded 34GiB to GPU in 2.35381s (14.4446GiB/s) | 20260507 18:47:11.261 | INFO | Analyzing chunk frames = [1763328,1763840) | 20260507 18:47:11.261 | INFO | MemInfo: { | 20260507 18:47:11.261 | INFO | "gpu" : " 17 GB", | 20260507 18:47:11.261 | INFO | "huge" : "", | 20260507 18:47:11.261 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:11.261 | INFO | "malloc" : " 14 GB", | 20260507 18:47:11.261 | INFO | "pinned" : " 29 GB" | 20260507 18:47:11.261 | INFO | } | 20260507 18:47:11.261 | INFO | Perf report for the last 5.13945 seconds: | 20260507 18:47:11.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:11.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 18:47:11.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:47:11.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:11.261 | INFO | Uploaded 36GiB to GPU in 2.36367s (15.2305GiB/s) | 20260507 18:47:11.261 | INFO | Downloaded 34GiB to GPU in 2.47857s (13.7176GiB/s) | 20260507 18:47:16.400 | INFO | Analyzing chunk frames = [1763840,1764352) | 20260507 18:47:16.400 | INFO | MemInfo: { | 20260507 18:47:16.400 | INFO | "gpu" : " 17 GB", | 20260507 18:47:16.400 | INFO | "huge" : "", | 20260507 18:47:16.400 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:16.400 | INFO | "malloc" : " 14 GB", | 20260507 18:47:16.400 | INFO | "pinned" : " 29 GB" | 20260507 18:47:16.400 | INFO | } | 20260507 18:47:16.400 | INFO | Perf report for the last 5.13952 seconds: | 20260507 18:47:16.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:16.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.400 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:47:16.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.400 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:47:16.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:16.401 | INFO | Uploaded 36GiB to GPU in 2.42553s (14.8421GiB/s) | 20260507 18:47:16.401 | INFO | Downloaded 34GiB to GPU in 2.4969s (13.6169GiB/s) | 20260507 18:47:21.539 | INFO | Analyzing chunk frames = [1764352,1764864) | 20260507 18:47:21.539 | INFO | MemInfo: { | 20260507 18:47:21.539 | INFO | "gpu" : " 17 GB", | 20260507 18:47:21.539 | INFO | "huge" : "", | 20260507 18:47:21.539 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:21.540 | INFO | "malloc" : " 14 GB", | 20260507 18:47:21.540 | INFO | "pinned" : " 29 GB" | 20260507 18:47:21.540 | INFO | } | 20260507 18:47:21.540 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:47:21.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:21.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:47:21.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 18:47:21.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:21.540 | INFO | Uploaded 36GiB to GPU in 2.52839s (14.2383GiB/s) | 20260507 18:47:21.540 | INFO | Downloaded 34GiB to GPU in 2.42967s (13.9937GiB/s) | 20260507 18:47:26.679 | INFO | Analyzing chunk frames = [1764864,1765376) | 20260507 18:47:26.679 | INFO | MemInfo: { | 20260507 18:47:26.679 | INFO | "gpu" : " 17 GB", | 20260507 18:47:26.679 | INFO | "huge" : "", | 20260507 18:47:26.679 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:26.679 | INFO | "malloc" : " 14 GB", | 20260507 18:47:26.679 | INFO | "pinned" : " 29 GB" | 20260507 18:47:26.679 | INFO | } | 20260507 18:47:26.679 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:47:26.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:26.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.679 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:47:26.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.679 | INFO | PRE_HQ : 38.2%, 1.0, 19.8ms, 0 | 20260507 18:47:26.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:26.680 | INFO | Uploaded 36GiB to GPU in 2.53023s (14.228GiB/s) | 20260507 18:47:26.680 | INFO | Downloaded 34GiB to GPU in 2.41822s (14.0599GiB/s) | 20260507 18:47:26.790 | INFO | Starting a round of Baz IO | 20260507 18:47:30.271 | INFO | Populating RTMetrics report | 20260507 18:47:30.272 | INFO | RTMetrics report populated | 20260507 18:47:30.272 | INFO | Opening tmp RTMetrics json file | 20260507 18:47:30.273 | INFO | Serializing RTMetrics to json | 20260507 18:47:30.293 | INFO | Serialization done, swapping tmp file now... | 20260507 18:47:30.297 | INFO | RTMetrics done | 20260507 18:47:30.355 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.564 seconds | 20260507 18:47:30.355 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.16% / 0.01% / 53.82% | 20260507 18:47:30.355 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.75801% / 1.19209e-05% / 4.37101e-05% | 20260507 18:47:30.355 | INFO | Baz IO overall: | 20260507 18:47:30.355 | INFO | BytesWritten : 792350825275 | 20260507 18:47:30.356 | INFO | NumEvents : 384154758660 | 20260507 18:47:31.818 | INFO | Analyzing chunk frames = [1765376,1765888) | 20260507 18:47:31.818 | INFO | MemInfo: { | 20260507 18:47:31.818 | INFO | "gpu" : " 17 GB", | 20260507 18:47:31.818 | INFO | "huge" : "", | 20260507 18:47:31.818 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:31.818 | INFO | "malloc" : " 14 GB", | 20260507 18:47:31.818 | INFO | "pinned" : " 29 GB" | 20260507 18:47:31.818 | INFO | } | 20260507 18:47:31.818 | INFO | Perf report for the last 5.13892 seconds: | 20260507 18:47:31.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:31.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:31.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:31.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:31.818 | INFO | ANALYSIS : 69.2%, 2.9, 106.7ms, 0 | 20260507 18:47:31.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:31.818 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 18:47:31.818 | INFO | PRE_HQ : 67.1%, 1.9, 64.8ms, 0 | 20260507 18:47:31.818 | INFO | BAZWRITER : 68.7%, 3.2, 111.4ms, 0 | 20260507 18:47:31.818 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:47:31.818 | INFO | Uploaded 36GiB to GPU in 2.52411s (14.2625GiB/s) | 20260507 18:47:31.818 | INFO | Downloaded 38GiB to GPU in 2.6966s (14.0918GiB/s) | 20260507 18:47:36.957 | INFO | Analyzing chunk frames = [1765888,1766400) | 20260507 18:47:36.957 | INFO | MemInfo: { | 20260507 18:47:36.957 | INFO | "gpu" : " 17 GB", | 20260507 18:47:36.957 | INFO | "huge" : "", | 20260507 18:47:36.957 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:36.957 | INFO | "malloc" : " 14 GB", | 20260507 18:47:36.957 | INFO | "pinned" : " 29 GB" | 20260507 18:47:36.957 | INFO | } | 20260507 18:47:36.957 | INFO | Perf report for the last 5.13956 seconds: | 20260507 18:47:36.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:36.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 18:47:36.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:47:36.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:36.958 | INFO | Uploaded 36GiB to GPU in 2.49314s (14.4396GiB/s) | 20260507 18:47:36.958 | INFO | Downloaded 34GiB to GPU in 2.44941s (13.8809GiB/s) | 20260507 18:47:42.097 | INFO | Analyzing chunk frames = [1766400,1766912) | 20260507 18:47:42.097 | INFO | MemInfo: { | 20260507 18:47:42.097 | INFO | "gpu" : " 17 GB", | 20260507 18:47:42.097 | INFO | "huge" : "", | 20260507 18:47:42.097 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:42.097 | INFO | "malloc" : " 14 GB", | 20260507 18:47:42.097 | INFO | "pinned" : " 29 GB" | 20260507 18:47:42.097 | INFO | } | 20260507 18:47:42.097 | INFO | Perf report for the last 5.13942 seconds: | 20260507 18:47:42.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:42.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:47:42.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:47:42.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:42.097 | INFO | Uploaded 36GiB to GPU in 2.54682s (14.1353GiB/s) | 20260507 18:47:42.097 | INFO | Downloaded 34GiB to GPU in 2.45406s (13.8546GiB/s) | 20260507 18:47:47.236 | INFO | Analyzing chunk frames = [1766912,1767424) | 20260507 18:47:47.236 | INFO | MemInfo: { | 20260507 18:47:47.236 | INFO | "gpu" : " 17 GB", | 20260507 18:47:47.236 | INFO | "huge" : "", | 20260507 18:47:47.236 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:47.236 | INFO | "malloc" : " 14 GB", | 20260507 18:47:47.236 | INFO | "pinned" : " 29 GB" | 20260507 18:47:47.236 | INFO | } | 20260507 18:47:47.236 | INFO | Perf report for the last 5.13899 seconds: | 20260507 18:47:47.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:47.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:47:47.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:47:47.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:47.236 | INFO | Uploaded 36GiB to GPU in 2.47148s (14.5662GiB/s) | 20260507 18:47:47.236 | INFO | Downloaded 34GiB to GPU in 2.37265s (14.3299GiB/s) | 20260507 18:47:52.375 | INFO | Analyzing chunk frames = [1767424,1767936) | 20260507 18:47:52.375 | INFO | MemInfo: { | 20260507 18:47:52.375 | INFO | "gpu" : " 17 GB", | 20260507 18:47:52.375 | INFO | "huge" : "", | 20260507 18:47:52.375 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:52.375 | INFO | "malloc" : " 14 GB", | 20260507 18:47:52.375 | INFO | "pinned" : " 29 GB" | 20260507 18:47:52.376 | INFO | } | 20260507 18:47:52.376 | INFO | Perf report for the last 5.13965 seconds: | 20260507 18:47:52.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:52.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 18:47:52.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:47:52.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:52.376 | INFO | Uploaded 36GiB to GPU in 2.3133s (15.5622GiB/s) | 20260507 18:47:52.376 | INFO | Downloaded 34GiB to GPU in 2.49028s (13.6531GiB/s) | 20260507 18:47:57.515 | INFO | Analyzing chunk frames = [1767936,1768448) | 20260507 18:47:57.515 | INFO | MemInfo: { | 20260507 18:47:57.515 | INFO | "gpu" : " 17 GB", | 20260507 18:47:57.515 | INFO | "huge" : "", | 20260507 18:47:57.515 | INFO | "huge_shared" : " 29 GB", | 20260507 18:47:57.515 | INFO | "malloc" : " 14 GB", | 20260507 18:47:57.515 | INFO | "pinned" : " 29 GB" | 20260507 18:47:57.515 | INFO | } | 20260507 18:47:57.515 | INFO | Perf report for the last 5.13968 seconds: | 20260507 18:47:57.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:47:57.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.515 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:47:57.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.515 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:47:57.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:47:57.516 | INFO | Uploaded 36GiB to GPU in 2.46843s (14.5842GiB/s) | 20260507 18:47:57.516 | INFO | Downloaded 34GiB to GPU in 2.5116s (13.5372GiB/s) | 20260507 18:48:02.654 | INFO | Analyzing chunk frames = [1768448,1768960) | 20260507 18:48:02.654 | INFO | MemInfo: { | 20260507 18:48:02.654 | INFO | "gpu" : " 17 GB", | 20260507 18:48:02.654 | INFO | "huge" : "", | 20260507 18:48:02.654 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:02.654 | INFO | "malloc" : " 14 GB", | 20260507 18:48:02.654 | INFO | "pinned" : " 29 GB" | 20260507 18:48:02.654 | INFO | } | 20260507 18:48:02.654 | INFO | Perf report for the last 5.13891 seconds: | 20260507 18:48:02.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:02.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.654 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:48:02.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.654 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:48:02.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:02.655 | INFO | Uploaded 36GiB to GPU in 2.48952s (14.4606GiB/s) | 20260507 18:48:02.655 | INFO | Downloaded 34GiB to GPU in 2.44436s (13.9095GiB/s) | 20260507 18:48:07.793 | INFO | Analyzing chunk frames = [1768960,1769472) | 20260507 18:48:07.793 | INFO | MemInfo: { | 20260507 18:48:07.793 | INFO | "gpu" : " 17 GB", | 20260507 18:48:07.793 | INFO | "huge" : "", | 20260507 18:48:07.793 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:07.793 | INFO | "malloc" : " 14 GB", | 20260507 18:48:07.793 | INFO | "pinned" : " 29 GB" | 20260507 18:48:07.793 | INFO | } | 20260507 18:48:07.793 | INFO | Perf report for the last 5.13911 seconds: | 20260507 18:48:07.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:07.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.793 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:48:07.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.794 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 18:48:07.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:07.794 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:48:07.794 | INFO | Total mem usage: 16.4 GiB | 20260507 18:48:07.794 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:48:07.794 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 18:48:07.794 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:48:07.794 | INFO | High Water Breakdown: | 20260507 18:48:07.794 | INFO | 59.3% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:48:07.794 | INFO | 40.5% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:48:07.794 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:48:07.794 | INFO | ------------------------------------------ | 20260507 18:48:07.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:48:07.794 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:48:07.794 | INFO | Total mem usage: 26.5 GiB | 20260507 18:48:07.794 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:48:07.794 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 18:48:07.794 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 18:48:07.794 | INFO | High Water Breakdown: | 20260507 18:48:07.794 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:48:07.794 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:48:07.794 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:48:07.794 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:48:07.794 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:48:07.794 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:48:07.794 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:48:07.794 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:48:07.794 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:48:07.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:48:07.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:48:07.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:48:07.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:48:07.794 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:48:07.794 | INFO | ------------------------------------------ | 20260507 18:48:07.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:48:07.794 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:48:07.794 | INFO | Total mem usage: 27.8 GiB | 20260507 18:48:07.794 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:48:07.794 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 18:48:07.794 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 18:48:07.794 | INFO | High Water Breakdown: | 20260507 18:48:07.794 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 18:48:07.794 | INFO | ------------------------------------------ | 20260507 18:48:07.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:48:07.794 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:48:07.794 | INFO | Total mem usage: 15.7 GiB | 20260507 18:48:07.794 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:48:07.794 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:48:07.794 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:48:07.794 | INFO | High Water Breakdown: | 20260507 18:48:07.794 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:48:07.794 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:48:07.794 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:48:07.794 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:48:07.794 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:48:07.794 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:48:07.794 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:48:07.794 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:48:07.794 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:48:07.794 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:48:07.794 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:48:07.794 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:48:07.794 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:48:07.794 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:48:07.794 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:48:07.794 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:48:07.794 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:48:07.794 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:48:07.794 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:48:07.794 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:48:07.794 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:48:07.794 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:48:07.794 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:48:07.794 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:48:07.794 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:48:07.794 | INFO | ------------------------------------------ | 20260507 18:48:07.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:48:07.794 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:48:07.794 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:48:07.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:48:07.794 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.43e+01 / 1.90e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.04e-01, 1.10e-02 / 4.29e-03 | 20260507 18:48:07.794 | INFO | PerfReport: 12.09%, Baseline -- 4.04e+00 / 4.51e+00, 6.15e-01 / 2.36e-01 | 20260507 18:48:07.794 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.14e-01 / 1.06e-01 | 20260507 18:48:07.794 | INFO | PerfReport: 15.67%, DME -- 4.19e+01 / 4.48e+01, 4.04e+00 / 1.87e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.88e+00 / 1.94e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.50e+00 / 2.80e+00, 4.31e-01 / 2.48e-01 | 20260507 18:48:07.794 | INFO | Uploaded 36GiB to GPU in 2.48037s (14.514GiB/s) | 20260507 18:48:07.794 | INFO | PerfReport: 3.70%, Metrics -- 1.24e+00 / 1.42e+00, 3.15e-01 / 1.72e-01 | 20260507 18:48:07.794 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.88e-01 / 4.18e-01, 6.28e-02 / 3.36e-02 | 20260507 18:48:07.794 | INFO | Downloaded 34GiB to GPU in 2.42691s (14.0096GiB/s) | 20260507 18:48:07.794 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:48:07.794 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:48:07.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:48:07.794 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 24.20%, Upload -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 24.05%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 31.15%, Compute -- 3.34e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 1.91%, LockUpload -- 2.05e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 2.20%, LockDownload -- 2.36e+00 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 16.49%, LockCompute -- 1.77e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 18:48:07.794 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 18:48:07.891 | INFO | Starting a round of Baz IO | 20260507 18:48:11.398 | INFO | Populating RTMetrics report | 20260507 18:48:11.398 | INFO | RTMetrics report populated | 20260507 18:48:11.398 | INFO | Opening tmp RTMetrics json file | 20260507 18:48:11.410 | INFO | Serializing RTMetrics to json | 20260507 18:48:11.418 | INFO | Serialization done, swapping tmp file now... | 20260507 18:48:11.418 | INFO | RTMetrics done | 20260507 18:48:11.480 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.589 seconds | 20260507 18:48:11.480 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.15% / 0.01% / 53.84% | 20260507 18:48:11.480 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.74707% / 1.98682e-05% / 4.76837e-05% | 20260507 18:48:11.480 | INFO | Baz IO overall: | 20260507 18:48:11.480 | INFO | BytesWritten : 794285415289 | 20260507 18:48:11.480 | INFO | NumEvents : 385093018000 | 20260507 18:48:12.933 | INFO | Analyzing chunk frames = [1769472,1769984) | 20260507 18:48:12.933 | INFO | MemInfo: { | 20260507 18:48:12.933 | INFO | "gpu" : " 17 GB", | 20260507 18:48:12.933 | INFO | "huge" : "", | 20260507 18:48:12.933 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:12.933 | INFO | "malloc" : " 14 GB", | 20260507 18:48:12.933 | INFO | "pinned" : " 29 GB" | 20260507 18:48:12.933 | INFO | } | 20260507 18:48:12.933 | INFO | Perf report for the last 5.13977 seconds: | 20260507 18:48:12.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:12.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:12.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:12.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:12.933 | INFO | ANALYSIS : 69.4%, 3.0, 108.2ms, 0 | 20260507 18:48:12.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:12.933 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 18:48:12.933 | INFO | PRE_HQ : 66.4%, 1.7, 59.4ms, 0 | 20260507 18:48:12.933 | INFO | BAZWRITER : 69.1%, 3.0, 106.8ms, 0 | 20260507 18:48:12.933 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:48:12.933 | INFO | Uploaded 36GiB to GPU in 2.52192s (14.2748GiB/s) | 20260507 18:48:12.934 | INFO | Downloaded 38GiB to GPU in 2.72236s (13.9585GiB/s) | 20260507 18:48:18.072 | INFO | Analyzing chunk frames = [1769984,1770496) | 20260507 18:48:18.072 | INFO | MemInfo: { | 20260507 18:48:18.072 | INFO | "gpu" : " 17 GB", | 20260507 18:48:18.072 | INFO | "huge" : "", | 20260507 18:48:18.072 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:18.072 | INFO | "malloc" : " 14 GB", | 20260507 18:48:18.072 | INFO | "pinned" : " 29 GB" | 20260507 18:48:18.072 | INFO | } | 20260507 18:48:18.072 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:48:18.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:18.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 18:48:18.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:48:18.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:18.073 | INFO | Uploaded 36GiB to GPU in 2.49899s (14.4058GiB/s) | 20260507 18:48:18.073 | INFO | Downloaded 34GiB to GPU in 2.46582s (13.7885GiB/s) | 20260507 18:48:23.211 | INFO | Analyzing chunk frames = [1770496,1771008) | 20260507 18:48:23.212 | INFO | MemInfo: { | 20260507 18:48:23.212 | INFO | "gpu" : " 17 GB", | 20260507 18:48:23.212 | INFO | "huge" : "", | 20260507 18:48:23.212 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:23.212 | INFO | "malloc" : " 14 GB", | 20260507 18:48:23.212 | INFO | "pinned" : " 29 GB" | 20260507 18:48:23.212 | INFO | } | 20260507 18:48:23.212 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:48:23.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:23.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 18:48:23.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:48:23.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:23.212 | INFO | Uploaded 36GiB to GPU in 2.54851s (14.1259GiB/s) | 20260507 18:48:23.212 | INFO | Downloaded 34GiB to GPU in 2.45402s (13.8548GiB/s) | 20260507 18:48:28.351 | INFO | Analyzing chunk frames = [1771008,1771520) | 20260507 18:48:28.351 | INFO | MemInfo: { | 20260507 18:48:28.351 | INFO | "gpu" : " 17 GB", | 20260507 18:48:28.351 | INFO | "huge" : "", | 20260507 18:48:28.351 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:28.351 | INFO | "malloc" : " 14 GB", | 20260507 18:48:28.351 | INFO | "pinned" : " 29 GB" | 20260507 18:48:28.351 | INFO | } | 20260507 18:48:28.351 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:48:28.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:28.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 18:48:28.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:48:28.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:28.351 | INFO | Uploaded 36GiB to GPU in 2.46593s (14.5989GiB/s) | 20260507 18:48:28.351 | INFO | Downloaded 34GiB to GPU in 2.35608s (14.4308GiB/s) | 20260507 18:48:33.491 | INFO | Analyzing chunk frames = [1771520,1772032) | 20260507 18:48:33.491 | INFO | MemInfo: { | 20260507 18:48:33.491 | INFO | "gpu" : " 17 GB", | 20260507 18:48:33.491 | INFO | "huge" : "", | 20260507 18:48:33.491 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:33.491 | INFO | "malloc" : " 14 GB", | 20260507 18:48:33.491 | INFO | "pinned" : " 29 GB" | 20260507 18:48:33.491 | INFO | } | 20260507 18:48:33.491 | INFO | Perf report for the last 5.14039 seconds: | 20260507 18:48:33.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:33.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.492 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:48:33.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.492 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 18:48:33.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:33.492 | INFO | Uploaded 36GiB to GPU in 2.30855s (15.5942GiB/s) | 20260507 18:48:33.492 | INFO | Downloaded 34GiB to GPU in 2.49107s (13.6488GiB/s) | 20260507 18:48:38.630 | INFO | Analyzing chunk frames = [1772032,1772544) | 20260507 18:48:38.630 | INFO | MemInfo: { | 20260507 18:48:38.630 | INFO | "gpu" : " 17 GB", | 20260507 18:48:38.630 | INFO | "huge" : "", | 20260507 18:48:38.630 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:38.630 | INFO | "malloc" : " 14 GB", | 20260507 18:48:38.630 | INFO | "pinned" : " 29 GB" | 20260507 18:48:38.630 | INFO | } | 20260507 18:48:38.630 | INFO | Perf report for the last 5.13814 seconds: | 20260507 18:48:38.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:38.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:48:38.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:48:38.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:38.630 | INFO | Uploaded 36GiB to GPU in 2.44364s (14.7321GiB/s) | 20260507 18:48:38.630 | INFO | Downloaded 34GiB to GPU in 2.52084s (13.4876GiB/s) | 20260507 18:48:43.769 | INFO | Analyzing chunk frames = [1772544,1773056) | 20260507 18:48:43.769 | INFO | MemInfo: { | 20260507 18:48:43.769 | INFO | "gpu" : " 17 GB", | 20260507 18:48:43.769 | INFO | "huge" : "", | 20260507 18:48:43.769 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:43.769 | INFO | "malloc" : " 14 GB", | 20260507 18:48:43.769 | INFO | "pinned" : " 29 GB" | 20260507 18:48:43.769 | INFO | } | 20260507 18:48:43.769 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:48:43.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:43.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.769 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:48:43.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.769 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 18:48:43.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:43.770 | INFO | Uploaded 36GiB to GPU in 2.48889s (14.4643GiB/s) | 20260507 18:48:43.770 | INFO | Downloaded 34GiB to GPU in 2.44187s (13.9238GiB/s) | 20260507 18:48:48.908 | INFO | Analyzing chunk frames = [1773056,1773568) | 20260507 18:48:48.908 | INFO | MemInfo: { | 20260507 18:48:48.908 | INFO | "gpu" : " 17 GB", | 20260507 18:48:48.909 | INFO | "huge" : "", | 20260507 18:48:48.909 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:48.909 | INFO | "malloc" : " 14 GB", | 20260507 18:48:48.909 | INFO | "pinned" : " 29 GB" | 20260507 18:48:48.909 | INFO | } | 20260507 18:48:48.909 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:48:48.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:48.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:48:48.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | PRE_HQ : 38.5%, 1.0, 20.0ms, 0 | 20260507 18:48:48.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:48.909 | INFO | Uploaded 36GiB to GPU in 2.49353s (14.4373GiB/s) | 20260507 18:48:48.909 | INFO | Downloaded 34GiB to GPU in 2.43593s (13.9577GiB/s) | 20260507 18:48:49.023 | INFO | Starting a round of Baz IO | 20260507 18:48:52.501 | INFO | Populating RTMetrics report | 20260507 18:48:52.501 | INFO | RTMetrics report populated | 20260507 18:48:52.501 | INFO | Opening tmp RTMetrics json file | 20260507 18:48:52.543 | INFO | Serializing RTMetrics to json | 20260507 18:48:52.545 | INFO | Serialization done, swapping tmp file now... | 20260507 18:48:52.548 | INFO | RTMetrics done | 20260507 18:48:52.690 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.667 seconds | 20260507 18:48:52.690 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.14% / 0.01% / 53.85% | 20260507 18:48:52.690 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.74084% / 7.94729e-06% / 5.96046e-05% | 20260507 18:48:52.690 | INFO | Baz IO overall: | 20260507 18:48:52.690 | INFO | BytesWritten : 796219757732 | 20260507 18:48:52.690 | INFO | NumEvents : 386031671348 | 20260507 18:48:54.047 | INFO | Analyzing chunk frames = [1773568,1774080) | 20260507 18:48:54.047 | INFO | MemInfo: { | 20260507 18:48:54.047 | INFO | "gpu" : " 17 GB", | 20260507 18:48:54.047 | INFO | "huge" : "", | 20260507 18:48:54.047 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:54.047 | INFO | "malloc" : " 14 GB", | 20260507 18:48:54.047 | INFO | "pinned" : " 29 GB" | 20260507 18:48:54.048 | INFO | } | 20260507 18:48:54.048 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:48:54.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:54.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:54.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:54.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:54.048 | INFO | ANALYSIS : 69.4%, 2.9, 106.7ms, 0 | 20260507 18:48:54.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:54.048 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 18:48:54.048 | INFO | PRE_HQ : 67.3%, 1.9, 64.8ms, 0 | 20260507 18:48:54.048 | INFO | BAZWRITER : 70.7%, 3.7, 133.2ms, 0 | 20260507 18:48:54.048 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 18:48:54.048 | INFO | Uploaded 36GiB to GPU in 2.58362s (13.934GiB/s) | 20260507 18:48:54.048 | INFO | Downloaded 38GiB to GPU in 2.71501s (13.9963GiB/s) | 20260507 18:48:59.187 | INFO | Analyzing chunk frames = [1774080,1774592) | 20260507 18:48:59.187 | INFO | MemInfo: { | 20260507 18:48:59.187 | INFO | "gpu" : " 17 GB", | 20260507 18:48:59.187 | INFO | "huge" : "", | 20260507 18:48:59.187 | INFO | "huge_shared" : " 29 GB", | 20260507 18:48:59.187 | INFO | "malloc" : " 14 GB", | 20260507 18:48:59.187 | INFO | "pinned" : " 29 GB" | 20260507 18:48:59.187 | INFO | } | 20260507 18:48:59.187 | INFO | Perf report for the last 5.1398 seconds: | 20260507 18:48:59.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:48:59.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.187 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:48:59.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.188 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:48:59.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:48:59.188 | INFO | Uploaded 36GiB to GPU in 2.50776s (14.3554GiB/s) | 20260507 18:48:59.188 | INFO | Downloaded 34GiB to GPU in 2.46528s (13.7915GiB/s) | 20260507 18:49:04.326 | INFO | Analyzing chunk frames = [1774592,1775104) | 20260507 18:49:04.326 | INFO | MemInfo: { | 20260507 18:49:04.326 | INFO | "gpu" : " 17 GB", | 20260507 18:49:04.326 | INFO | "huge" : "", | 20260507 18:49:04.326 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:04.326 | INFO | "malloc" : " 14 GB", | 20260507 18:49:04.326 | INFO | "pinned" : " 29 GB" | 20260507 18:49:04.326 | INFO | } | 20260507 18:49:04.326 | INFO | Perf report for the last 5.13881 seconds: | 20260507 18:49:04.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:04.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:49:04.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:49:04.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:04.326 | INFO | Uploaded 36GiB to GPU in 2.54208s (14.1616GiB/s) | 20260507 18:49:04.326 | INFO | Downloaded 34GiB to GPU in 2.45456s (13.8517GiB/s) | 20260507 18:49:09.466 | INFO | Analyzing chunk frames = [1775104,1775616) | 20260507 18:49:09.466 | INFO | MemInfo: { | 20260507 18:49:09.466 | INFO | "gpu" : " 17 GB", | 20260507 18:49:09.466 | INFO | "huge" : "", | 20260507 18:49:09.466 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:09.466 | INFO | "malloc" : " 14 GB", | 20260507 18:49:09.466 | INFO | "pinned" : " 29 GB" | 20260507 18:49:09.466 | INFO | } | 20260507 18:49:09.466 | INFO | Perf report for the last 5.13961 seconds: | 20260507 18:49:09.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:09.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:49:09.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:49:09.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:09.466 | INFO | Uploaded 36GiB to GPU in 2.48231s (14.5026GiB/s) | 20260507 18:49:09.466 | INFO | Downloaded 34GiB to GPU in 2.35428s (14.4418GiB/s) | 20260507 18:49:14.606 | INFO | Analyzing chunk frames = [1775616,1776128) | 20260507 18:49:14.606 | INFO | MemInfo: { | 20260507 18:49:14.606 | INFO | "gpu" : " 17 GB", | 20260507 18:49:14.606 | INFO | "huge" : "", | 20260507 18:49:14.606 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:14.606 | INFO | "malloc" : " 14 GB", | 20260507 18:49:14.606 | INFO | "pinned" : " 29 GB" | 20260507 18:49:14.606 | INFO | } | 20260507 18:49:14.606 | INFO | Perf report for the last 5.14007 seconds: | 20260507 18:49:14.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:14.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:49:14.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:49:14.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:14.606 | INFO | Uploaded 36GiB to GPU in 2.30486s (15.6192GiB/s) | 20260507 18:49:14.606 | INFO | Downloaded 34GiB to GPU in 2.48619s (13.6755GiB/s) | 20260507 18:49:19.744 | INFO | Analyzing chunk frames = [1776128,1776640) | 20260507 18:49:19.744 | INFO | MemInfo: { | 20260507 18:49:19.744 | INFO | "gpu" : " 17 GB", | 20260507 18:49:19.744 | INFO | "huge" : "", | 20260507 18:49:19.744 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:19.744 | INFO | "malloc" : " 14 GB", | 20260507 18:49:19.744 | INFO | "pinned" : " 29 GB" | 20260507 18:49:19.744 | INFO | } | 20260507 18:49:19.744 | INFO | Perf report for the last 5.13847 seconds: | 20260507 18:49:19.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:19.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.744 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:49:19.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.745 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:49:19.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:19.745 | INFO | Uploaded 36GiB to GPU in 2.42608s (14.8387GiB/s) | 20260507 18:49:19.745 | INFO | Downloaded 34GiB to GPU in 2.51857s (13.4997GiB/s) | 20260507 18:49:24.884 | INFO | Analyzing chunk frames = [1776640,1777152) | 20260507 18:49:24.884 | INFO | MemInfo: { | 20260507 18:49:24.884 | INFO | "gpu" : " 17 GB", | 20260507 18:49:24.884 | INFO | "huge" : "", | 20260507 18:49:24.884 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:24.884 | INFO | "malloc" : " 14 GB", | 20260507 18:49:24.884 | INFO | "pinned" : " 29 GB" | 20260507 18:49:24.884 | INFO | } | 20260507 18:49:24.884 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:49:24.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:24.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:49:24.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:49:24.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:24.884 | INFO | Uploaded 36GiB to GPU in 2.50292s (14.3832GiB/s) | 20260507 18:49:24.884 | INFO | Downloaded 34GiB to GPU in 2.4457s (13.9019GiB/s) | 20260507 18:49:30.023 | INFO | Analyzing chunk frames = [1777152,1777664) | 20260507 18:49:30.023 | INFO | MemInfo: { | 20260507 18:49:30.023 | INFO | "gpu" : " 17 GB", | 20260507 18:49:30.023 | INFO | "huge" : "", | 20260507 18:49:30.023 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:30.023 | INFO | "malloc" : " 14 GB", | 20260507 18:49:30.023 | INFO | "pinned" : " 29 GB" | 20260507 18:49:30.023 | INFO | } | 20260507 18:49:30.023 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:49:30.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:30.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.023 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:49:30.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.023 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:49:30.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:30.024 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:49:30.024 | INFO | Total mem usage: 16.4 GiB | 20260507 18:49:30.024 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:49:30.024 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8.17%) | 20260507 18:49:30.024 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:49:30.024 | INFO | High Water Breakdown: | 20260507 18:49:30.024 | INFO | 60% : 9.1 GiB ( 91.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:49:30.024 | INFO | 39.9% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:49:30.024 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:49:30.024 | INFO | ------------------------------------------ | 20260507 18:49:30.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:49:30.024 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:49:30.024 | INFO | Total mem usage: 26.5 GiB | 20260507 18:49:30.024 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:49:30.024 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:49:30.024 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:49:30.024 | INFO | High Water Breakdown: | 20260507 18:49:30.024 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:49:30.024 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:49:30.024 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:49:30.024 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:49:30.024 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:49:30.024 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:49:30.024 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:49:30.024 | INFO | Uploaded 36GiB to GPU in 2.49077s (14.4534GiB/s) | 20260507 18:49:30.024 | INFO | Downloaded 34GiB to GPU in 2.42956s (13.9943GiB/s) | 20260507 18:49:30.024 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:49:30.024 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:49:30.024 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:49:30.024 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:49:30.024 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:49:30.024 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:49:30.024 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:49:30.024 | INFO | ------------------------------------------ | 20260507 18:49:30.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:49:30.024 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:49:30.024 | INFO | Total mem usage: 27.8 GiB | 20260507 18:49:30.024 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:49:30.024 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:49:30.024 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:49:30.024 | INFO | High Water Breakdown: | 20260507 18:49:30.024 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:49:30.024 | INFO | ------------------------------------------ | 20260507 18:49:30.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:49:30.024 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:49:30.024 | INFO | Total mem usage: 15.7 GiB | 20260507 18:49:30.024 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:49:30.024 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:49:30.024 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 18:49:30.024 | INFO | High Water Breakdown: | 20260507 18:49:30.024 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:49:30.024 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:49:30.024 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:49:30.024 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:49:30.024 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:49:30.024 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:49:30.024 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:49:30.024 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:49:30.024 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:49:30.024 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:49:30.024 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:49:30.024 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:49:30.024 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:49:30.024 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:49:30.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:49:30.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:49:30.024 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:49:30.024 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:49:30.024 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:49:30.024 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:49:30.024 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:49:30.024 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:49:30.024 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:49:30.024 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:49:30.024 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:49:30.024 | INFO | ------------------------------------------ | 20260507 18:49:30.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:49:30.025 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:49:30.025 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:49:30.025 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:49:30.025 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.43e+01 / 2.15e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.03e-01, 1.11e-02 / 5.11e-03 | 20260507 18:49:30.025 | INFO | PerfReport: 12.11%, Baseline -- 4.04e+00 / 4.54e+00, 6.13e-01 / 1.97e-01 | 20260507 18:49:30.025 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 1.15e-01 | 20260507 18:49:30.025 | INFO | PerfReport: 15.69%, DME -- 4.19e+01 / 4.50e+01, 3.98e+00 / 2.02e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.89e+00 / 2.19e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.49e+00 / 2.84e+00, 4.31e-01 / 2.48e-01 | 20260507 18:49:30.025 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.42e+00, 3.13e-01 / 1.61e-01 | 20260507 18:49:30.025 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.87e-01 / 4.16e-01, 6.24e-02 / 3.00e-02 | 20260507 18:49:30.025 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:49:30.025 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:49:30.025 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:49:30.025 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 24.05%, Upload -- 2.58e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 24.14%, Download -- 2.59e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 31.16%, Compute -- 3.34e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 1.74%, LockUpload -- 1.86e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 2.37%, LockDownload -- 2.54e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 16.53%, LockCompute -- 1.77e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:49:30.025 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 18:49:30.121 | INFO | Starting a round of Baz IO | 20260507 18:49:33.622 | INFO | Populating RTMetrics report | 20260507 18:49:33.622 | INFO | RTMetrics report populated | 20260507 18:49:33.622 | INFO | Opening tmp RTMetrics json file | 20260507 18:49:33.624 | INFO | Serializing RTMetrics to json | 20260507 18:49:33.641 | INFO | Serialization done, swapping tmp file now... | 20260507 18:49:33.641 | INFO | RTMetrics done | 20260507 18:49:33.703 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.581 seconds | 20260507 18:49:33.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.12% / 0.01% / 53.86% | 20260507 18:49:33.703 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.73102% / 3.57628e-05% / 2.38419e-05% | 20260507 18:49:33.703 | INFO | Baz IO overall: | 20260507 18:49:33.703 | INFO | BytesWritten : 798153728620 | 20260507 18:49:33.703 | INFO | NumEvents : 386969681691 | 20260507 18:49:35.162 | INFO | Analyzing chunk frames = [1777664,1778176) | 20260507 18:49:35.162 | INFO | MemInfo: { | 20260507 18:49:35.162 | INFO | "gpu" : " 17 GB", | 20260507 18:49:35.162 | INFO | "huge" : "", | 20260507 18:49:35.162 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:35.162 | INFO | "malloc" : " 14 GB", | 20260507 18:49:35.162 | INFO | "pinned" : " 29 GB" | 20260507 18:49:35.162 | INFO | } | 20260507 18:49:35.162 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:49:35.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:35.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:35.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:35.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:35.162 | INFO | ANALYSIS : 69.6%, 3.0, 108.0ms, 0 | 20260507 18:49:35.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:35.162 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 18:49:35.162 | INFO | PRE_HQ : 67.5%, 1.8, 62.9ms, 0 | 20260507 18:49:35.162 | INFO | BAZWRITER : 68.5%, 3.3, 115.4ms, 0 | 20260507 18:49:35.162 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:49:35.163 | INFO | Uploaded 36GiB to GPU in 2.5663s (14.028GiB/s) | 20260507 18:49:35.163 | INFO | Downloaded 38GiB to GPU in 2.71575s (13.9925GiB/s) | 20260507 18:49:40.301 | INFO | Analyzing chunk frames = [1778176,1778688) | 20260507 18:49:40.301 | INFO | MemInfo: { | 20260507 18:49:40.301 | INFO | "gpu" : " 17 GB", | 20260507 18:49:40.302 | INFO | "huge" : "", | 20260507 18:49:40.302 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:40.302 | INFO | "malloc" : " 14 GB", | 20260507 18:49:40.302 | INFO | "pinned" : " 29 GB" | 20260507 18:49:40.302 | INFO | } | 20260507 18:49:40.302 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:49:40.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:40.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:49:40.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:49:40.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:40.302 | INFO | Uploaded 36GiB to GPU in 2.5154s (14.3119GiB/s) | 20260507 18:49:40.302 | INFO | Downloaded 34GiB to GPU in 2.45661s (13.8402GiB/s) | 20260507 18:49:45.441 | INFO | Analyzing chunk frames = [1778688,1779200) | 20260507 18:49:45.441 | INFO | MemInfo: { | 20260507 18:49:45.441 | INFO | "gpu" : " 17 GB", | 20260507 18:49:45.441 | INFO | "huge" : "", | 20260507 18:49:45.441 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:45.441 | INFO | "malloc" : " 14 GB", | 20260507 18:49:45.441 | INFO | "pinned" : " 29 GB" | 20260507 18:49:45.441 | INFO | } | 20260507 18:49:45.441 | INFO | Perf report for the last 5.13962 seconds: | 20260507 18:49:45.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:45.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.441 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 18:49:45.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.442 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:49:45.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:45.442 | INFO | Uploaded 36GiB to GPU in 2.54437s (14.1489GiB/s) | 20260507 18:49:45.442 | INFO | Downloaded 34GiB to GPU in 2.45595s (13.844GiB/s) | 20260507 18:49:50.580 | INFO | Analyzing chunk frames = [1779200,1779712) | 20260507 18:49:50.580 | INFO | MemInfo: { | 20260507 18:49:50.580 | INFO | "gpu" : " 17 GB", | 20260507 18:49:50.580 | INFO | "huge" : "", | 20260507 18:49:50.580 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:50.581 | INFO | "malloc" : " 14 GB", | 20260507 18:49:50.581 | INFO | "pinned" : " 29 GB" | 20260507 18:49:50.581 | INFO | } | 20260507 18:49:50.581 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:49:50.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:50.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:49:50.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:49:50.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:50.581 | INFO | Uploaded 36GiB to GPU in 2.47119s (14.5679GiB/s) | 20260507 18:49:50.581 | INFO | Downloaded 34GiB to GPU in 2.36822s (14.3568GiB/s) | 20260507 18:49:55.720 | INFO | Analyzing chunk frames = [1779712,1780224) | 20260507 18:49:55.720 | INFO | MemInfo: { | 20260507 18:49:55.720 | INFO | "gpu" : " 17 GB", | 20260507 18:49:55.720 | INFO | "huge" : "", | 20260507 18:49:55.720 | INFO | "huge_shared" : " 29 GB", | 20260507 18:49:55.720 | INFO | "malloc" : " 14 GB", | 20260507 18:49:55.720 | INFO | "pinned" : " 29 GB" | 20260507 18:49:55.720 | INFO | } | 20260507 18:49:55.720 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:49:55.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:49:55.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 18:49:55.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:49:55.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:49:55.720 | INFO | Uploaded 36GiB to GPU in 2.30746s (15.6016GiB/s) | 20260507 18:49:55.720 | INFO | Downloaded 34GiB to GPU in 2.49746s (13.6139GiB/s) | 20260507 18:50:00.859 | INFO | Analyzing chunk frames = [1780224,1780736) | 20260507 18:50:00.859 | INFO | MemInfo: { | 20260507 18:50:00.859 | INFO | "gpu" : " 17 GB", | 20260507 18:50:00.859 | INFO | "huge" : "", | 20260507 18:50:00.859 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:00.859 | INFO | "malloc" : " 14 GB", | 20260507 18:50:00.859 | INFO | "pinned" : " 29 GB" | 20260507 18:50:00.859 | INFO | } | 20260507 18:50:00.859 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:50:00.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:00.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:50:00.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 18:50:00.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:00.859 | INFO | Uploaded 36GiB to GPU in 2.45554s (14.6608GiB/s) | 20260507 18:50:00.859 | INFO | Downloaded 34GiB to GPU in 2.52398s (13.4708GiB/s) | 20260507 18:50:05.998 | INFO | Analyzing chunk frames = [1780736,1781248) | 20260507 18:50:05.998 | INFO | MemInfo: { | 20260507 18:50:05.998 | INFO | "gpu" : " 17 GB", | 20260507 18:50:05.999 | INFO | "huge" : "", | 20260507 18:50:05.999 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:05.999 | INFO | "malloc" : " 14 GB", | 20260507 18:50:05.999 | INFO | "pinned" : " 29 GB" | 20260507 18:50:05.999 | INFO | } | 20260507 18:50:05.999 | INFO | Perf report for the last 5.13929 seconds: | 20260507 18:50:05.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:05.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:50:05.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 18:50:05.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:05.999 | INFO | Uploaded 36GiB to GPU in 2.48319s (14.4975GiB/s) | 20260507 18:50:05.999 | INFO | Downloaded 34GiB to GPU in 2.43957s (13.9369GiB/s) | 20260507 18:50:11.138 | INFO | Analyzing chunk frames = [1781248,1781760) | 20260507 18:50:11.138 | INFO | MemInfo: { | 20260507 18:50:11.138 | INFO | "gpu" : " 17 GB", | 20260507 18:50:11.138 | INFO | "huge" : "", | 20260507 18:50:11.138 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:11.138 | INFO | "malloc" : " 14 GB", | 20260507 18:50:11.138 | INFO | "pinned" : " 29 GB" | 20260507 18:50:11.138 | INFO | } | 20260507 18:50:11.138 | INFO | Perf report for the last 5.13924 seconds: | 20260507 18:50:11.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:11.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:50:11.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:50:11.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:11.138 | INFO | Uploaded 36GiB to GPU in 2.48026s (14.5146GiB/s) | 20260507 18:50:11.138 | INFO | Downloaded 34GiB to GPU in 2.43065s (13.9881GiB/s) | 20260507 18:50:11.236 | INFO | Starting a round of Baz IO | 20260507 18:50:14.735 | INFO | Populating RTMetrics report | 20260507 18:50:14.736 | INFO | RTMetrics report populated | 20260507 18:50:14.736 | INFO | Opening tmp RTMetrics json file | 20260507 18:50:14.751 | INFO | Serializing RTMetrics to json | 20260507 18:50:14.752 | INFO | Serialization done, swapping tmp file now... | 20260507 18:50:14.753 | INFO | RTMetrics done | 20260507 18:50:14.816 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.579 seconds | 20260507 18:50:14.816 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.11% / 0.01% / 53.88% | 20260507 18:50:14.816 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.72786% / 1.98682e-05% / 4.76837e-05% | 20260507 18:50:14.816 | INFO | Baz IO overall: | 20260507 18:50:14.816 | INFO | BytesWritten : 800087526246 | 20260507 18:50:14.816 | INFO | NumEvents : 387907588677 | 20260507 18:50:16.277 | INFO | Analyzing chunk frames = [1781760,1782272) | 20260507 18:50:16.277 | INFO | MemInfo: { | 20260507 18:50:16.277 | INFO | "gpu" : " 17 GB", | 20260507 18:50:16.277 | INFO | "huge" : "", | 20260507 18:50:16.277 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:16.277 | INFO | "malloc" : " 14 GB", | 20260507 18:50:16.277 | INFO | "pinned" : " 29 GB" | 20260507 18:50:16.277 | INFO | } | 20260507 18:50:16.277 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:50:16.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:16.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:16.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:16.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:16.277 | INFO | ANALYSIS : 69.3%, 3.0, 108.2ms, 0 | 20260507 18:50:16.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:16.277 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:50:16.277 | INFO | PRE_HQ : 66.0%, 1.7, 59.4ms, 0 | 20260507 18:50:16.278 | INFO | BAZWRITER : 69.2%, 3.3, 118.0ms, 0 | 20260507 18:50:16.278 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:50:16.278 | INFO | Uploaded 36GiB to GPU in 2.52444s (14.2606GiB/s) | 20260507 18:50:16.278 | INFO | Downloaded 38GiB to GPU in 2.7156s (13.9932GiB/s) | 20260507 18:50:21.416 | INFO | Analyzing chunk frames = [1782272,1782784) | 20260507 18:50:21.416 | INFO | MemInfo: { | 20260507 18:50:21.416 | INFO | "gpu" : " 17 GB", | 20260507 18:50:21.417 | INFO | "huge" : "", | 20260507 18:50:21.417 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:21.417 | INFO | "malloc" : " 14 GB", | 20260507 18:50:21.417 | INFO | "pinned" : " 29 GB" | 20260507 18:50:21.417 | INFO | } | 20260507 18:50:21.417 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:50:21.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:21.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:50:21.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:50:21.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:21.417 | INFO | Uploaded 36GiB to GPU in 2.54243s (14.1597GiB/s) | 20260507 18:50:21.417 | INFO | Downloaded 34GiB to GPU in 2.44174s (13.9245GiB/s) | 20260507 18:50:26.556 | INFO | Analyzing chunk frames = [1782784,1783296) | 20260507 18:50:26.556 | INFO | MemInfo: { | 20260507 18:50:26.556 | INFO | "gpu" : " 17 GB", | 20260507 18:50:26.556 | INFO | "huge" : "", | 20260507 18:50:26.556 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:26.556 | INFO | "malloc" : " 14 GB", | 20260507 18:50:26.556 | INFO | "pinned" : " 29 GB" | 20260507 18:50:26.556 | INFO | } | 20260507 18:50:26.556 | INFO | Perf report for the last 5.13966 seconds: | 20260507 18:50:26.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:26.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.556 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:50:26.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.556 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:50:26.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:26.557 | INFO | Uploaded 36GiB to GPU in 2.54992s (14.1181GiB/s) | 20260507 18:50:26.557 | INFO | Downloaded 34GiB to GPU in 2.43294s (13.9749GiB/s) | 20260507 18:50:31.695 | INFO | Analyzing chunk frames = [1783296,1783808) | 20260507 18:50:31.695 | INFO | MemInfo: { | 20260507 18:50:31.695 | INFO | "gpu" : " 17 GB", | 20260507 18:50:31.695 | INFO | "huge" : "", | 20260507 18:50:31.695 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:31.695 | INFO | "malloc" : " 14 GB", | 20260507 18:50:31.695 | INFO | "pinned" : " 29 GB" | 20260507 18:50:31.696 | INFO | } | 20260507 18:50:31.696 | INFO | Perf report for the last 5.13927 seconds: | 20260507 18:50:31.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:31.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:50:31.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:50:31.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:31.696 | INFO | Uploaded 36GiB to GPU in 2.50678s (14.3611GiB/s) | 20260507 18:50:31.696 | INFO | Downloaded 34GiB to GPU in 2.3621s (14.394GiB/s) | 20260507 18:50:36.835 | INFO | Analyzing chunk frames = [1783808,1784320) | 20260507 18:50:36.835 | INFO | MemInfo: { | 20260507 18:50:36.835 | INFO | "gpu" : " 17 GB", | 20260507 18:50:36.835 | INFO | "huge" : "", | 20260507 18:50:36.835 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:36.835 | INFO | "malloc" : " 14 GB", | 20260507 18:50:36.835 | INFO | "pinned" : " 29 GB" | 20260507 18:50:36.835 | INFO | } | 20260507 18:50:36.835 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:50:36.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:36.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:50:36.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 18:50:36.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:36.835 | INFO | Uploaded 36GiB to GPU in 2.34598s (15.3454GiB/s) | 20260507 18:50:36.835 | INFO | Downloaded 34GiB to GPU in 2.47801s (13.7207GiB/s) | 20260507 18:50:41.974 | INFO | Analyzing chunk frames = [1784320,1784832) | 20260507 18:50:41.974 | INFO | MemInfo: { | 20260507 18:50:41.974 | INFO | "gpu" : " 17 GB", | 20260507 18:50:41.974 | INFO | "huge" : "", | 20260507 18:50:41.974 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:41.974 | INFO | "malloc" : " 14 GB", | 20260507 18:50:41.974 | INFO | "pinned" : " 29 GB" | 20260507 18:50:41.974 | INFO | } | 20260507 18:50:41.974 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:50:41.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:41.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.974 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:50:41.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.974 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 18:50:41.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:41.975 | INFO | Uploaded 36GiB to GPU in 2.41227s (14.9237GiB/s) | 20260507 18:50:41.975 | INFO | Downloaded 34GiB to GPU in 2.47661s (13.7284GiB/s) | 20260507 18:50:47.113 | INFO | Analyzing chunk frames = [1784832,1785344) | 20260507 18:50:47.113 | INFO | MemInfo: { | 20260507 18:50:47.113 | INFO | "gpu" : " 17 GB", | 20260507 18:50:47.113 | INFO | "huge" : "", | 20260507 18:50:47.113 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:47.114 | INFO | "malloc" : " 14 GB", | 20260507 18:50:47.114 | INFO | "pinned" : " 29 GB" | 20260507 18:50:47.114 | INFO | } | 20260507 18:50:47.114 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:50:47.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:47.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:50:47.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:50:47.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:47.114 | INFO | Uploaded 36GiB to GPU in 2.52206s (14.2741GiB/s) | 20260507 18:50:47.114 | INFO | Downloaded 34GiB to GPU in 2.42935s (13.9955GiB/s) | 20260507 18:50:52.253 | INFO | Analyzing chunk frames = [1785344,1785856) | 20260507 18:50:52.253 | INFO | MemInfo: { | 20260507 18:50:52.253 | INFO | "gpu" : " 17 GB", | 20260507 18:50:52.253 | INFO | "huge" : "", | 20260507 18:50:52.253 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:52.253 | INFO | "malloc" : " 14 GB", | 20260507 18:50:52.253 | INFO | "pinned" : " 29 GB" | 20260507 18:50:52.253 | INFO | } | 20260507 18:50:52.253 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:50:52.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:52.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.253 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:50:52.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.253 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:50:52.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:52.254 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:50:52.254 | INFO | Total mem usage: 16.4 GiB | 20260507 18:50:52.254 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:50:52.254 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.63%) | 20260507 18:50:52.254 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:50:52.254 | INFO | High Water Breakdown: | 20260507 18:50:52.254 | INFO | 59.8% : 8.95 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:50:52.254 | INFO | 40.1% : 5.99 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:50:52.254 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:50:52.254 | INFO | ------------------------------------------ | 20260507 18:50:52.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:50:52.254 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:50:52.254 | INFO | Total mem usage: 26.5 GiB | 20260507 18:50:52.254 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:50:52.254 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:50:52.254 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:50:52.254 | INFO | High Water Breakdown: | 20260507 18:50:52.254 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:50:52.254 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:50:52.254 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:50:52.254 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:50:52.254 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:50:52.254 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:50:52.254 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:50:52.254 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:50:52.254 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:50:52.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:50:52.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:50:52.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:50:52.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:50:52.254 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:50:52.254 | INFO | ------------------------------------------ | 20260507 18:50:52.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:50:52.254 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:50:52.254 | INFO | Total mem usage: 27.8 GiB | 20260507 18:50:52.254 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:50:52.254 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:50:52.254 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:50:52.254 | INFO | High Water Breakdown: | 20260507 18:50:52.254 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:50:52.254 | INFO | ------------------------------------------ | 20260507 18:50:52.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:50:52.254 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:50:52.254 | INFO | Total mem usage: 15.7 GiB | 20260507 18:50:52.254 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:50:52.254 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:50:52.254 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:50:52.254 | INFO | High Water Breakdown: | 20260507 18:50:52.254 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:50:52.254 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:50:52.254 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:50:52.254 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:50:52.254 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:50:52.254 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:50:52.254 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:50:52.254 | INFO | Uploaded 36GiB to GPU in 2.51551s (14.3112GiB/s) | 20260507 18:50:52.254 | INFO | Downloaded 34GiB to GPU in 2.41582s (14.0739GiB/s) | 20260507 18:50:52.254 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:50:52.254 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:50:52.254 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:50:52.254 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:50:52.254 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:50:52.254 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:50:52.254 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:50:52.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:50:52.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:50:52.254 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:50:52.254 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:50:52.254 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:50:52.254 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:50:52.254 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:50:52.254 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:50:52.254 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:50:52.254 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:50:52.254 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:50:52.254 | INFO | ------------------------------------------ | 20260507 18:50:52.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:50:52.254 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:50:52.254 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:50:52.254 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:50:52.254 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.67e+01, 1.42e+01 / 2.04e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 0.29%, AggregateStats -- 9.82e-02 / 1.04e-01, 1.10e-02 / 5.04e-03 | 20260507 18:50:52.254 | INFO | PerfReport: 12.10%, Baseline -- 4.04e+00 / 4.52e+00, 6.17e-01 / 1.65e-01 | 20260507 18:50:52.254 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.16e-01 / 8.53e-02 | 20260507 18:50:52.254 | INFO | PerfReport: 15.71%, DME -- 4.20e+01 / 4.51e+01, 4.22e+00 / 1.83e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.89e+00 / 2.27e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.49e+00 / 2.81e+00, 4.27e-01 / 2.49e-01 | 20260507 18:50:52.254 | INFO | PerfReport: 3.68%, Metrics -- 1.23e+00 / 1.41e+00, 3.13e-01 / 1.61e-01 | 20260507 18:50:52.254 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.86e-01 / 4.17e-01, 6.25e-02 / 3.20e-02 | 20260507 18:50:52.254 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:50:52.254 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:50:52.254 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:50:52.254 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 24.10%, Upload -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 24.04%, Download -- 2.58e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 31.11%, Compute -- 3.34e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 1.86%, LockUpload -- 2.00e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 2.25%, LockDownload -- 2.42e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 16.62%, LockCompute -- 1.78e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 18:50:52.254 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260507 18:50:52.351 | INFO | Starting a round of Baz IO | 20260507 18:50:55.864 | INFO | Populating RTMetrics report | 20260507 18:50:55.865 | INFO | RTMetrics report populated | 20260507 18:50:55.865 | INFO | Opening tmp RTMetrics json file | 20260507 18:50:55.872 | INFO | Serializing RTMetrics to json | 20260507 18:50:55.884 | INFO | Serialization done, swapping tmp file now... | 20260507 18:50:55.884 | INFO | RTMetrics done | 20260507 18:50:55.944 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.593 seconds | 20260507 18:50:55.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.10% / 0.01% / 53.89% | 20260507 18:50:55.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.7319% / 3.17891e-05% / 3.17891e-05% | 20260507 18:50:55.944 | INFO | Baz IO overall: | 20260507 18:50:55.944 | INFO | BytesWritten : 802024200640 | 20260507 18:50:55.944 | INFO | NumEvents : 388848005489 | 20260507 18:50:57.392 | INFO | Analyzing chunk frames = [1785856,1786368) | 20260507 18:50:57.392 | INFO | MemInfo: { | 20260507 18:50:57.392 | INFO | "gpu" : " 17 GB", | 20260507 18:50:57.392 | INFO | "huge" : "", | 20260507 18:50:57.392 | INFO | "huge_shared" : " 29 GB", | 20260507 18:50:57.392 | INFO | "malloc" : " 14 GB", | 20260507 18:50:57.392 | INFO | "pinned" : " 29 GB" | 20260507 18:50:57.392 | INFO | } | 20260507 18:50:57.392 | INFO | Perf report for the last 5.13903 seconds: | 20260507 18:50:57.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:50:57.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:57.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:57.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:57.392 | INFO | ANALYSIS : 69.5%, 3.0, 108.6ms, 0 | 20260507 18:50:57.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:50:57.392 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 18:50:57.392 | INFO | PRE_HQ : 67.5%, 1.8, 62.7ms, 0 | 20260507 18:50:57.392 | INFO | BAZWRITER : 69.1%, 3.1, 108.8ms, 0 | 20260507 18:50:57.392 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:50:57.392 | INFO | Uploaded 36GiB to GPU in 2.56805s (14.0184GiB/s) | 20260507 18:50:57.392 | INFO | Downloaded 38GiB to GPU in 2.71572s (13.9926GiB/s) | 20260507 18:51:02.531 | INFO | Analyzing chunk frames = [1786368,1786880) | 20260507 18:51:02.531 | INFO | MemInfo: { | 20260507 18:51:02.531 | INFO | "gpu" : " 17 GB", | 20260507 18:51:02.531 | INFO | "huge" : "", | 20260507 18:51:02.531 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:02.531 | INFO | "malloc" : " 14 GB", | 20260507 18:51:02.531 | INFO | "pinned" : " 29 GB" | 20260507 18:51:02.531 | INFO | } | 20260507 18:51:02.531 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:51:02.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:02.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 18:51:02.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:51:02.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:02.532 | INFO | Uploaded 36GiB to GPU in 2.55706s (14.0787GiB/s) | 20260507 18:51:02.532 | INFO | Downloaded 34GiB to GPU in 2.43943s (13.9377GiB/s) | 20260507 18:51:07.671 | INFO | Analyzing chunk frames = [1786880,1787392) | 20260507 18:51:07.671 | INFO | MemInfo: { | 20260507 18:51:07.671 | INFO | "gpu" : " 17 GB", | 20260507 18:51:07.671 | INFO | "huge" : "", | 20260507 18:51:07.671 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:07.671 | INFO | "malloc" : " 14 GB", | 20260507 18:51:07.671 | INFO | "pinned" : " 29 GB" | 20260507 18:51:07.671 | INFO | } | 20260507 18:51:07.671 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:51:07.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:07.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:51:07.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:51:07.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:07.671 | INFO | Uploaded 36GiB to GPU in 2.57609s (13.9747GiB/s) | 20260507 18:51:07.672 | INFO | Downloaded 34GiB to GPU in 2.44335s (13.9153GiB/s) | 20260507 18:51:12.810 | INFO | Analyzing chunk frames = [1787392,1787904) | 20260507 18:51:12.810 | INFO | MemInfo: { | 20260507 18:51:12.810 | INFO | "gpu" : " 17 GB", | 20260507 18:51:12.810 | INFO | "huge" : "", | 20260507 18:51:12.810 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:12.810 | INFO | "malloc" : " 14 GB", | 20260507 18:51:12.810 | INFO | "pinned" : " 29 GB" | 20260507 18:51:12.810 | INFO | } | 20260507 18:51:12.810 | INFO | Perf report for the last 5.1395 seconds: | 20260507 18:51:12.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:12.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.810 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:51:12.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.810 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:51:12.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:12.811 | INFO | Uploaded 36GiB to GPU in 2.50759s (14.3564GiB/s) | 20260507 18:51:12.811 | INFO | Downloaded 34GiB to GPU in 2.35069s (14.4638GiB/s) | 20260507 18:51:17.950 | INFO | Analyzing chunk frames = [1787904,1788416) | 20260507 18:51:17.950 | INFO | MemInfo: { | 20260507 18:51:17.950 | INFO | "gpu" : " 17 GB", | 20260507 18:51:17.950 | INFO | "huge" : "", | 20260507 18:51:17.950 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:17.950 | INFO | "malloc" : " 14 GB", | 20260507 18:51:17.950 | INFO | "pinned" : " 29 GB" | 20260507 18:51:17.950 | INFO | } | 20260507 18:51:17.950 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:51:17.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:17.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:51:17.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:51:17.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:17.950 | INFO | Uploaded 36GiB to GPU in 2.37723s (15.1437GiB/s) | 20260507 18:51:17.950 | INFO | Downloaded 34GiB to GPU in 2.46154s (13.8125GiB/s) | 20260507 18:51:23.088 | INFO | Analyzing chunk frames = [1788416,1788928) | 20260507 18:51:23.088 | INFO | MemInfo: { | 20260507 18:51:23.088 | INFO | "gpu" : " 17 GB", | 20260507 18:51:23.088 | INFO | "huge" : "", | 20260507 18:51:23.088 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:23.088 | INFO | "malloc" : " 14 GB", | 20260507 18:51:23.088 | INFO | "pinned" : " 29 GB" | 20260507 18:51:23.089 | INFO | } | 20260507 18:51:23.089 | INFO | Perf report for the last 5.13907 seconds: | 20260507 18:51:23.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:23.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:51:23.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:51:23.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:23.089 | INFO | Uploaded 36GiB to GPU in 2.48098s (14.5104GiB/s) | 20260507 18:51:23.089 | INFO | Downloaded 34GiB to GPU in 2.48052s (13.7068GiB/s) | 20260507 18:51:28.228 | INFO | Analyzing chunk frames = [1788928,1789440) | 20260507 18:51:28.228 | INFO | MemInfo: { | 20260507 18:51:28.228 | INFO | "gpu" : " 17 GB", | 20260507 18:51:28.228 | INFO | "huge" : "", | 20260507 18:51:28.228 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:28.228 | INFO | "malloc" : " 14 GB", | 20260507 18:51:28.228 | INFO | "pinned" : " 29 GB" | 20260507 18:51:28.228 | INFO | } | 20260507 18:51:28.228 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:51:28.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:28.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:51:28.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:51:28.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:28.228 | INFO | Uploaded 36GiB to GPU in 2.52325s (14.2673GiB/s) | 20260507 18:51:28.228 | INFO | Downloaded 34GiB to GPU in 2.42851s (14.0003GiB/s) | 20260507 18:51:33.367 | INFO | Analyzing chunk frames = [1789440,1789952) | 20260507 18:51:33.367 | INFO | MemInfo: { | 20260507 18:51:33.367 | INFO | "gpu" : " 17 GB", | 20260507 18:51:33.367 | INFO | "huge" : "", | 20260507 18:51:33.367 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:33.367 | INFO | "malloc" : " 14 GB", | 20260507 18:51:33.367 | INFO | "pinned" : " 29 GB" | 20260507 18:51:33.367 | INFO | } | 20260507 18:51:33.367 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:51:33.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:33.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.367 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 18:51:33.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.368 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 18:51:33.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:33.368 | INFO | Uploaded 36GiB to GPU in 2.52218s (14.2734GiB/s) | 20260507 18:51:33.368 | INFO | Downloaded 34GiB to GPU in 2.41263s (14.0925GiB/s) | 20260507 18:51:33.465 | INFO | Starting a round of Baz IO | 20260507 18:51:36.969 | INFO | Populating RTMetrics report | 20260507 18:51:36.969 | INFO | RTMetrics report populated | 20260507 18:51:36.969 | INFO | Opening tmp RTMetrics json file | 20260507 18:51:36.975 | INFO | Serializing RTMetrics to json | 20260507 18:51:36.987 | INFO | Serialization done, swapping tmp file now... | 20260507 18:51:36.988 | INFO | RTMetrics done | 20260507 18:51:37.049 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.583 seconds | 20260507 18:51:37.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.08% / 0.01% / 53.90% | 20260507 18:51:37.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.72311% / 1.58946e-05% / 3.97364e-05% | 20260507 18:51:37.049 | INFO | Baz IO overall: | 20260507 18:51:37.049 | INFO | BytesWritten : 803958963204 | 20260507 18:51:37.049 | INFO | NumEvents : 389787049581 | 20260507 18:51:38.507 | INFO | Analyzing chunk frames = [1789952,1790464) | 20260507 18:51:38.507 | INFO | MemInfo: { | 20260507 18:51:38.507 | INFO | "gpu" : " 17 GB", | 20260507 18:51:38.507 | INFO | "huge" : "", | 20260507 18:51:38.507 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:38.507 | INFO | "malloc" : " 14 GB", | 20260507 18:51:38.507 | INFO | "pinned" : " 29 GB" | 20260507 18:51:38.507 | INFO | } | 20260507 18:51:38.507 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:51:38.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:38.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:38.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:38.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:38.507 | INFO | ANALYSIS : 69.6%, 3.0, 108.4ms, 0 | 20260507 18:51:38.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:38.507 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:51:38.507 | INFO | PRE_HQ : 66.6%, 1.7, 58.8ms, 0 | 20260507 18:51:38.507 | INFO | BAZWRITER : 68.9%, 2.9, 103.8ms, 0 | 20260507 18:51:38.507 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:51:38.507 | INFO | Uploaded 36GiB to GPU in 2.57942s (13.9566GiB/s) | 20260507 18:51:38.507 | INFO | Downloaded 38GiB to GPU in 2.70012s (14.0734GiB/s) | 20260507 18:51:43.646 | INFO | Analyzing chunk frames = [1790464,1790976) | 20260507 18:51:43.646 | INFO | MemInfo: { | 20260507 18:51:43.646 | INFO | "gpu" : " 17 GB", | 20260507 18:51:43.646 | INFO | "huge" : "", | 20260507 18:51:43.646 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:43.646 | INFO | "malloc" : " 14 GB", | 20260507 18:51:43.646 | INFO | "pinned" : " 29 GB" | 20260507 18:51:43.646 | INFO | } | 20260507 18:51:43.646 | INFO | Perf report for the last 5.13914 seconds: | 20260507 18:51:43.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:43.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.646 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:51:43.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.646 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 18:51:43.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:43.647 | INFO | Uploaded 36GiB to GPU in 2.55148s (14.1094GiB/s) | 20260507 18:51:43.647 | INFO | Downloaded 34GiB to GPU in 2.44272s (13.9189GiB/s) | 20260507 18:51:48.785 | INFO | Analyzing chunk frames = [1790976,1791488) | 20260507 18:51:48.785 | INFO | MemInfo: { | 20260507 18:51:48.785 | INFO | "gpu" : " 17 GB", | 20260507 18:51:48.785 | INFO | "huge" : "", | 20260507 18:51:48.785 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:48.785 | INFO | "malloc" : " 14 GB", | 20260507 18:51:48.785 | INFO | "pinned" : " 29 GB" | 20260507 18:51:48.785 | INFO | } | 20260507 18:51:48.785 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:51:48.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:48.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.785 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:51:48.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.785 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:51:48.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:48.786 | INFO | Uploaded 36GiB to GPU in 2.57627s (13.9737GiB/s) | 20260507 18:51:48.786 | INFO | Downloaded 34GiB to GPU in 2.431s (13.986GiB/s) | 20260507 18:51:53.924 | INFO | Analyzing chunk frames = [1791488,1792000) | 20260507 18:51:53.924 | INFO | MemInfo: { | 20260507 18:51:53.924 | INFO | "gpu" : " 17 GB", | 20260507 18:51:53.924 | INFO | "huge" : "", | 20260507 18:51:53.924 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:53.924 | INFO | "malloc" : " 14 GB", | 20260507 18:51:53.924 | INFO | "pinned" : " 29 GB" | 20260507 18:51:53.924 | INFO | } | 20260507 18:51:53.924 | INFO | Perf report for the last 5.13936 seconds: | 20260507 18:51:53.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:53.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.925 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260507 18:51:53.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.925 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:51:53.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:53.925 | INFO | Uploaded 36GiB to GPU in 2.50826s (14.3526GiB/s) | 20260507 18:51:53.925 | INFO | Downloaded 34GiB to GPU in 2.35041s (14.4655GiB/s) | 20260507 18:51:59.065 | INFO | Analyzing chunk frames = [1792000,1792512) | 20260507 18:51:59.065 | INFO | MemInfo: { | 20260507 18:51:59.065 | INFO | "gpu" : " 17 GB", | 20260507 18:51:59.065 | INFO | "huge" : "", | 20260507 18:51:59.065 | INFO | "huge_shared" : " 29 GB", | 20260507 18:51:59.065 | INFO | "malloc" : " 14 GB", | 20260507 18:51:59.065 | INFO | "pinned" : " 29 GB" | 20260507 18:51:59.065 | INFO | } | 20260507 18:51:59.065 | INFO | Perf report for the last 5.14031 seconds: | 20260507 18:51:59.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:51:59.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:51:59.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:51:59.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:51:59.066 | INFO | Uploaded 36GiB to GPU in 2.35519s (15.2854GiB/s) | 20260507 18:51:59.066 | INFO | Downloaded 34GiB to GPU in 2.47223s (13.7528GiB/s) | 20260507 18:52:04.203 | INFO | Analyzing chunk frames = [1792512,1793024) | 20260507 18:52:04.203 | INFO | MemInfo: { | 20260507 18:52:04.203 | INFO | "gpu" : " 17 GB", | 20260507 18:52:04.203 | INFO | "huge" : "", | 20260507 18:52:04.203 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:04.203 | INFO | "malloc" : " 14 GB", | 20260507 18:52:04.203 | INFO | "pinned" : " 29 GB" | 20260507 18:52:04.203 | INFO | } | 20260507 18:52:04.203 | INFO | Perf report for the last 5.13842 seconds: | 20260507 18:52:04.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:04.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.203 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:52:04.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.203 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:52:04.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:04.204 | INFO | Uploaded 36GiB to GPU in 2.4698s (14.5761GiB/s) | 20260507 18:52:04.204 | INFO | Downloaded 34GiB to GPU in 2.4853s (13.6804GiB/s) | 20260507 18:52:09.342 | INFO | Analyzing chunk frames = [1793024,1793536) | 20260507 18:52:09.343 | INFO | MemInfo: { | 20260507 18:52:09.343 | INFO | "gpu" : " 17 GB", | 20260507 18:52:09.343 | INFO | "huge" : "", | 20260507 18:52:09.343 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:09.343 | INFO | "malloc" : " 14 GB", | 20260507 18:52:09.343 | INFO | "pinned" : " 29 GB" | 20260507 18:52:09.343 | INFO | } | 20260507 18:52:09.343 | INFO | Perf report for the last 5.13938 seconds: | 20260507 18:52:09.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:09.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:52:09.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:52:09.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:09.343 | INFO | Uploaded 36GiB to GPU in 2.5168s (14.3039GiB/s) | 20260507 18:52:09.343 | INFO | Downloaded 34GiB to GPU in 2.42532s (14.0188GiB/s) | 20260507 18:52:14.482 | INFO | Analyzing chunk frames = [1793536,1794048) | 20260507 18:52:14.482 | INFO | MemInfo: { | 20260507 18:52:14.482 | INFO | "gpu" : " 17 GB", | 20260507 18:52:14.482 | INFO | "huge" : "", | 20260507 18:52:14.482 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:14.482 | INFO | "malloc" : " 14 GB", | 20260507 18:52:14.482 | INFO | "pinned" : " 29 GB" | 20260507 18:52:14.482 | INFO | } | 20260507 18:52:14.482 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:52:14.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:14.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:52:14.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:52:14.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:14.482 | INFO | Uploaded 36GiB to GPU in 2.51582s (14.3094GiB/s) | 20260507 18:52:14.482 | INFO | Downloaded 34GiB to GPU in 2.41326s (14.0888GiB/s) | 20260507 18:52:14.483 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:52:14.483 | INFO | Total mem usage: 16.4 GiB | 20260507 18:52:14.483 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:52:14.483 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 18:52:14.483 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:52:14.483 | INFO | High Water Breakdown: | 20260507 18:52:14.483 | INFO | 59.2% : 8.54 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:52:14.483 | INFO | 40.7% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:52:14.483 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:52:14.483 | INFO | ------------------------------------------ | 20260507 18:52:14.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:52:14.483 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:52:14.483 | INFO | Total mem usage: 26.5 GiB | 20260507 18:52:14.483 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:52:14.483 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:52:14.483 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:52:14.483 | INFO | High Water Breakdown: | 20260507 18:52:14.483 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:52:14.483 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:52:14.483 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:52:14.483 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:52:14.483 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:52:14.483 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:52:14.483 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:52:14.483 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:52:14.483 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:52:14.483 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:52:14.483 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:52:14.483 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:52:14.483 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:52:14.483 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:52:14.483 | INFO | ------------------------------------------ | 20260507 18:52:14.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:52:14.483 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:52:14.483 | INFO | Total mem usage: 27.8 GiB | 20260507 18:52:14.483 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:52:14.483 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:52:14.483 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:52:14.483 | INFO | High Water Breakdown: | 20260507 18:52:14.483 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:52:14.483 | INFO | ------------------------------------------ | 20260507 18:52:14.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:52:14.483 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:52:14.483 | INFO | Total mem usage: 15.7 GiB | 20260507 18:52:14.483 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:52:14.483 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:52:14.483 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:52:14.483 | INFO | High Water Breakdown: | 20260507 18:52:14.483 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:52:14.483 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:52:14.483 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:52:14.483 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:52:14.483 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:52:14.483 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:52:14.483 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:52:14.483 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:52:14.483 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:52:14.483 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:52:14.483 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:52:14.483 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:52:14.483 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:52:14.483 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:52:14.483 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:52:14.483 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:52:14.484 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:52:14.484 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:52:14.484 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:52:14.484 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:52:14.484 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:52:14.484 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:52:14.484 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:52:14.484 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:52:14.484 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:52:14.484 | INFO | ------------------------------------------ | 20260507 18:52:14.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:52:14.484 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:52:14.484 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:52:14.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:52:14.484 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.66e+01, 1.42e+01 / 2.02e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.22e-02 / 4.83e-03 | 20260507 18:52:14.484 | INFO | PerfReport: 12.09%, Baseline -- 4.03e+00 / 4.50e+00, 6.14e-01 / 2.04e-01 | 20260507 18:52:14.484 | INFO | PerfReport: 3.43%, Histogram -- 1.14e+00 / 1.51e+00, 2.15e-01 / 1.04e-01 | 20260507 18:52:14.484 | INFO | PerfReport: 15.71%, DME -- 4.19e+01 / 4.51e+01, 4.05e+00 / 1.84e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.88e+00 / 2.25e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.48e+00 / 2.82e+00, 4.32e-01 / 2.42e-01 | 20260507 18:52:14.484 | INFO | PerfReport: 3.68%, Metrics -- 1.23e+00 / 1.42e+00, 3.14e-01 / 1.68e-01 | 20260507 18:52:14.484 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.87e-01 / 4.15e-01, 6.35e-02 / 3.68e-02 | 20260507 18:52:14.484 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:52:14.484 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:52:14.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:52:14.484 | INFO | PerfReport: Full Execution -- 1.08e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 24.33%, Upload -- 2.62e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 23.89%, Download -- 2.57e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 31.01%, Compute -- 3.33e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 2.11%, LockUpload -- 2.27e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 2.08%, LockDownload -- 2.24e+00 / 0.00e+00, 3.59e+00 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 16.55%, LockCompute -- 1.78e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 18:52:14.484 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260507 18:52:14.580 | INFO | Starting a round of Baz IO | 20260507 18:52:18.085 | INFO | Populating RTMetrics report | 20260507 18:52:18.085 | INFO | RTMetrics report populated | 20260507 18:52:18.085 | INFO | Opening tmp RTMetrics json file | 20260507 18:52:18.102 | INFO | Serializing RTMetrics to json | 20260507 18:52:18.104 | INFO | Serialization done, swapping tmp file now... | 20260507 18:52:18.104 | INFO | RTMetrics done | 20260507 18:52:18.166 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.586 seconds | 20260507 18:52:18.166 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.07% / 0.01% / 53.92% | 20260507 18:52:18.166 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70909% / 2.78155e-05% / 1.58946e-05% | 20260507 18:52:18.166 | INFO | Baz IO overall: | 20260507 18:52:18.166 | INFO | BytesWritten : 805892000780 | 20260507 18:52:18.166 | INFO | NumEvents : 390724500958 | 20260507 18:52:19.621 | INFO | Analyzing chunk frames = [1794048,1794560) | 20260507 18:52:19.621 | INFO | MemInfo: { | 20260507 18:52:19.621 | INFO | "gpu" : " 17 GB", | 20260507 18:52:19.621 | INFO | "huge" : "", | 20260507 18:52:19.621 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:19.621 | INFO | "malloc" : " 14 GB", | 20260507 18:52:19.621 | INFO | "pinned" : " 29 GB" | 20260507 18:52:19.621 | INFO | } | 20260507 18:52:19.621 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:52:19.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:19.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:19.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:19.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:19.621 | INFO | ANALYSIS : 69.4%, 3.0, 108.4ms, 0 | 20260507 18:52:19.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:19.621 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:52:19.621 | INFO | PRE_HQ : 66.3%, 1.7, 58.9ms, 0 | 20260507 18:52:19.621 | INFO | BAZWRITER : 68.9%, 2.9, 103.9ms, 0 | 20260507 18:52:19.621 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:52:19.622 | INFO | Uploaded 36GiB to GPU in 2.52074s (14.2815GiB/s) | 20260507 18:52:19.622 | INFO | Downloaded 38GiB to GPU in 2.71077s (14.0182GiB/s) | 20260507 18:52:24.760 | INFO | Analyzing chunk frames = [1794560,1795072) | 20260507 18:52:24.760 | INFO | MemInfo: { | 20260507 18:52:24.760 | INFO | "gpu" : " 17 GB", | 20260507 18:52:24.760 | INFO | "huge" : "", | 20260507 18:52:24.760 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:24.760 | INFO | "malloc" : " 14 GB", | 20260507 18:52:24.760 | INFO | "pinned" : " 29 GB" | 20260507 18:52:24.760 | INFO | } | 20260507 18:52:24.761 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:52:24.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:24.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:52:24.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:52:24.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:24.761 | INFO | Uploaded 36GiB to GPU in 2.52819s (14.2395GiB/s) | 20260507 18:52:24.761 | INFO | Downloaded 34GiB to GPU in 2.43241s (13.9779GiB/s) | 20260507 18:52:29.900 | INFO | Analyzing chunk frames = [1795072,1795584) | 20260507 18:52:29.900 | INFO | MemInfo: { | 20260507 18:52:29.900 | INFO | "gpu" : " 17 GB", | 20260507 18:52:29.900 | INFO | "huge" : "", | 20260507 18:52:29.900 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:29.900 | INFO | "malloc" : " 14 GB", | 20260507 18:52:29.900 | INFO | "pinned" : " 29 GB" | 20260507 18:52:29.900 | INFO | } | 20260507 18:52:29.900 | INFO | Perf report for the last 5.13943 seconds: | 20260507 18:52:29.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:29.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.900 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:52:29.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.900 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:52:29.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:29.901 | INFO | Uploaded 36GiB to GPU in 2.56086s (14.0578GiB/s) | 20260507 18:52:29.901 | INFO | Downloaded 34GiB to GPU in 2.45427s (13.8534GiB/s) | 20260507 18:52:35.039 | INFO | Analyzing chunk frames = [1795584,1796096) | 20260507 18:52:35.039 | INFO | MemInfo: { | 20260507 18:52:35.039 | INFO | "gpu" : " 17 GB", | 20260507 18:52:35.039 | INFO | "huge" : "", | 20260507 18:52:35.039 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:35.039 | INFO | "malloc" : " 14 GB", | 20260507 18:52:35.039 | INFO | "pinned" : " 29 GB" | 20260507 18:52:35.039 | INFO | } | 20260507 18:52:35.039 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:52:35.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:35.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:52:35.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 18:52:35.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:35.040 | INFO | Uploaded 36GiB to GPU in 2.48175s (14.5059GiB/s) | 20260507 18:52:35.040 | INFO | Downloaded 34GiB to GPU in 2.37395s (14.3221GiB/s) | 20260507 18:52:40.179 | INFO | Analyzing chunk frames = [1796096,1796608) | 20260507 18:52:40.179 | INFO | MemInfo: { | 20260507 18:52:40.179 | INFO | "gpu" : " 17 GB", | 20260507 18:52:40.179 | INFO | "huge" : "", | 20260507 18:52:40.179 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:40.179 | INFO | "malloc" : " 14 GB", | 20260507 18:52:40.179 | INFO | "pinned" : " 29 GB" | 20260507 18:52:40.179 | INFO | } | 20260507 18:52:40.179 | INFO | Perf report for the last 5.13963 seconds: | 20260507 18:52:40.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:40.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 18:52:40.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 18:52:40.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:40.179 | INFO | Uploaded 36GiB to GPU in 2.33649s (15.4077GiB/s) | 20260507 18:52:40.180 | INFO | Downloaded 34GiB to GPU in 2.48919s (13.6591GiB/s) | 20260507 18:52:45.318 | INFO | Analyzing chunk frames = [1796608,1797120) | 20260507 18:52:45.318 | INFO | MemInfo: { | 20260507 18:52:45.318 | INFO | "gpu" : " 17 GB", | 20260507 18:52:45.318 | INFO | "huge" : "", | 20260507 18:52:45.318 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:45.318 | INFO | "malloc" : " 14 GB", | 20260507 18:52:45.318 | INFO | "pinned" : " 29 GB" | 20260507 18:52:45.318 | INFO | } | 20260507 18:52:45.318 | INFO | Perf report for the last 5.13911 seconds: | 20260507 18:52:45.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:45.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.318 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:52:45.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.318 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:52:45.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:45.319 | INFO | Uploaded 36GiB to GPU in 2.4289s (14.8215GiB/s) | 20260507 18:52:45.319 | INFO | Downloaded 34GiB to GPU in 2.50679s (13.5631GiB/s) | 20260507 18:52:50.457 | INFO | Analyzing chunk frames = [1797120,1797632) | 20260507 18:52:50.457 | INFO | MemInfo: { | 20260507 18:52:50.457 | INFO | "gpu" : " 17 GB", | 20260507 18:52:50.457 | INFO | "huge" : "", | 20260507 18:52:50.457 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:50.457 | INFO | "malloc" : " 14 GB", | 20260507 18:52:50.457 | INFO | "pinned" : " 29 GB" | 20260507 18:52:50.458 | INFO | } | 20260507 18:52:50.458 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:52:50.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:50.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:52:50.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:52:50.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:50.458 | INFO | Uploaded 36GiB to GPU in 2.48399s (14.4928GiB/s) | 20260507 18:52:50.458 | INFO | Downloaded 34GiB to GPU in 2.44028s (13.9328GiB/s) | 20260507 18:52:55.596 | INFO | Analyzing chunk frames = [1797632,1798144) | 20260507 18:52:55.596 | INFO | MemInfo: { | 20260507 18:52:55.596 | INFO | "gpu" : " 17 GB", | 20260507 18:52:55.596 | INFO | "huge" : "", | 20260507 18:52:55.596 | INFO | "huge_shared" : " 29 GB", | 20260507 18:52:55.597 | INFO | "malloc" : " 14 GB", | 20260507 18:52:55.597 | INFO | "pinned" : " 29 GB" | 20260507 18:52:55.597 | INFO | } | 20260507 18:52:55.597 | INFO | Perf report for the last 5.13892 seconds: | 20260507 18:52:55.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:52:55.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:52:55.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:52:55.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:52:55.597 | INFO | Uploaded 36GiB to GPU in 2.48292s (14.499GiB/s) | 20260507 18:52:55.597 | INFO | Downloaded 34GiB to GPU in 2.42785s (14.0042GiB/s) | 20260507 18:52:55.694 | INFO | Starting a round of Baz IO | 20260507 18:52:59.196 | INFO | Populating RTMetrics report | 20260507 18:52:59.196 | INFO | RTMetrics report populated | 20260507 18:52:59.196 | INFO | Opening tmp RTMetrics json file | 20260507 18:52:59.197 | INFO | Serializing RTMetrics to json | 20260507 18:52:59.198 | INFO | Serialization done, swapping tmp file now... | 20260507 18:52:59.198 | INFO | RTMetrics done | 20260507 18:52:59.281 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.586 seconds | 20260507 18:52:59.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.05% / 0.01% / 53.93% | 20260507 18:52:59.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.71276% / 2.78155e-05% / 3.97364e-05% | 20260507 18:52:59.281 | INFO | Baz IO overall: | 20260507 18:52:59.281 | INFO | BytesWritten : 807827353781 | 20260507 18:52:59.281 | INFO | NumEvents : 391663944497 | 20260507 18:53:00.736 | INFO | Analyzing chunk frames = [1798144,1798656) | 20260507 18:53:00.736 | INFO | MemInfo: { | 20260507 18:53:00.736 | INFO | "gpu" : " 17 GB", | 20260507 18:53:00.736 | INFO | "huge" : "", | 20260507 18:53:00.736 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:00.736 | INFO | "malloc" : " 14 GB", | 20260507 18:53:00.736 | INFO | "pinned" : " 29 GB" | 20260507 18:53:00.736 | INFO | } | 20260507 18:53:00.736 | INFO | Perf report for the last 5.13976 seconds: | 20260507 18:53:00.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:00.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:00.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:00.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:00.736 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 18:53:00.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:00.737 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 18:53:00.737 | INFO | PRE_HQ : 66.6%, 1.7, 59.4ms, 0 | 20260507 18:53:00.737 | INFO | BAZWRITER : 68.7%, 3.0, 104.8ms, 0 | 20260507 18:53:00.737 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 18:53:00.737 | INFO | Uploaded 36GiB to GPU in 2.57099s (14.0024GiB/s) | 20260507 18:53:00.737 | INFO | Downloaded 38GiB to GPU in 2.73417s (13.8982GiB/s) | 20260507 18:53:05.875 | INFO | Analyzing chunk frames = [1798656,1799168) | 20260507 18:53:05.875 | INFO | MemInfo: { | 20260507 18:53:05.875 | INFO | "gpu" : " 17 GB", | 20260507 18:53:05.875 | INFO | "huge" : "", | 20260507 18:53:05.875 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:05.875 | INFO | "malloc" : " 14 GB", | 20260507 18:53:05.876 | INFO | "pinned" : " 29 GB" | 20260507 18:53:05.876 | INFO | } | 20260507 18:53:05.876 | INFO | Perf report for the last 5.13949 seconds: | 20260507 18:53:05.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:05.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.876 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:53:05.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.876 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:53:05.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:05.877 | INFO | Uploaded 36GiB to GPU in 2.53492s (14.2016GiB/s) | 20260507 18:53:05.877 | INFO | Downloaded 34GiB to GPU in 2.46794s (13.7767GiB/s) | 20260507 18:53:11.015 | INFO | Analyzing chunk frames = [1799168,1799680) | 20260507 18:53:11.015 | INFO | MemInfo: { | 20260507 18:53:11.015 | INFO | "gpu" : " 17 GB", | 20260507 18:53:11.015 | INFO | "huge" : "", | 20260507 18:53:11.015 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:11.015 | INFO | "malloc" : " 14 GB", | 20260507 18:53:11.015 | INFO | "pinned" : " 29 GB" | 20260507 18:53:11.015 | INFO | } | 20260507 18:53:11.015 | INFO | Perf report for the last 5.13953 seconds: | 20260507 18:53:11.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:11.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 18:53:11.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 18:53:11.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:11.016 | INFO | Uploaded 36GiB to GPU in 2.54645s (14.1374GiB/s) | 20260507 18:53:11.016 | INFO | Downloaded 34GiB to GPU in 2.45312s (13.8599GiB/s) | 20260507 18:53:16.154 | INFO | Analyzing chunk frames = [1799680,1800192) | 20260507 18:53:16.154 | INFO | MemInfo: { | 20260507 18:53:16.154 | INFO | "gpu" : " 17 GB", | 20260507 18:53:16.154 | INFO | "huge" : "", | 20260507 18:53:16.154 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:16.154 | INFO | "malloc" : " 14 GB", | 20260507 18:53:16.154 | INFO | "pinned" : " 29 GB" | 20260507 18:53:16.154 | INFO | } | 20260507 18:53:16.154 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:53:16.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:16.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:53:16.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:53:16.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:16.155 | INFO | Uploaded 36GiB to GPU in 2.47797s (14.528GiB/s) | 20260507 18:53:16.155 | INFO | Downloaded 34GiB to GPU in 2.38543s (14.2532GiB/s) | 20260507 18:53:21.294 | INFO | Analyzing chunk frames = [1800192,1800704) | 20260507 18:53:21.294 | INFO | MemInfo: { | 20260507 18:53:21.294 | INFO | "gpu" : " 17 GB", | 20260507 18:53:21.294 | INFO | "huge" : "", | 20260507 18:53:21.294 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:21.294 | INFO | "malloc" : " 14 GB", | 20260507 18:53:21.294 | INFO | "pinned" : " 29 GB" | 20260507 18:53:21.294 | INFO | } | 20260507 18:53:21.294 | INFO | Perf report for the last 5.13962 seconds: | 20260507 18:53:21.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:21.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.294 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:53:21.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.294 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:53:21.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:21.295 | INFO | Uploaded 36GiB to GPU in 2.32923s (15.4558GiB/s) | 20260507 18:53:21.295 | INFO | Downloaded 34GiB to GPU in 2.49993s (13.6004GiB/s) | 20260507 18:53:26.433 | INFO | Analyzing chunk frames = [1800704,1801216) | 20260507 18:53:26.433 | INFO | MemInfo: { | 20260507 18:53:26.433 | INFO | "gpu" : " 17 GB", | 20260507 18:53:26.433 | INFO | "huge" : "", | 20260507 18:53:26.433 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:26.433 | INFO | "malloc" : " 14 GB", | 20260507 18:53:26.433 | INFO | "pinned" : " 29 GB" | 20260507 18:53:26.433 | INFO | } | 20260507 18:53:26.433 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:53:26.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:26.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.433 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:53:26.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.433 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 18:53:26.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:26.434 | INFO | Uploaded 36GiB to GPU in 2.38667s (15.0837GiB/s) | 20260507 18:53:26.434 | INFO | Downloaded 34GiB to GPU in 2.49649s (13.6191GiB/s) | 20260507 18:53:31.572 | INFO | Analyzing chunk frames = [1801216,1801728) | 20260507 18:53:31.572 | INFO | MemInfo: { | 20260507 18:53:31.572 | INFO | "gpu" : " 17 GB", | 20260507 18:53:31.572 | INFO | "huge" : "", | 20260507 18:53:31.572 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:31.572 | INFO | "malloc" : " 14 GB", | 20260507 18:53:31.572 | INFO | "pinned" : " 29 GB" | 20260507 18:53:31.572 | INFO | } | 20260507 18:53:31.572 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:53:31.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:31.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:53:31.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 18:53:31.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:31.572 | INFO | Uploaded 36GiB to GPU in 2.50517s (14.3703GiB/s) | 20260507 18:53:31.572 | INFO | Downloaded 34GiB to GPU in 2.44878s (13.8845GiB/s) | 20260507 18:53:36.711 | INFO | Analyzing chunk frames = [1801728,1802240) | 20260507 18:53:36.711 | INFO | MemInfo: { | 20260507 18:53:36.711 | INFO | "gpu" : " 17 GB", | 20260507 18:53:36.711 | INFO | "huge" : "", | 20260507 18:53:36.711 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:36.711 | INFO | "malloc" : " 14 GB", | 20260507 18:53:36.711 | INFO | "pinned" : " 29 GB" | 20260507 18:53:36.711 | INFO | } | 20260507 18:53:36.711 | INFO | Perf report for the last 5.1393 seconds: | 20260507 18:53:36.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:36.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.712 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:53:36.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.712 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 18:53:36.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:36.712 | INFO | Uploaded 36GiB to GPU in 2.50116s (14.3933GiB/s) | 20260507 18:53:36.712 | INFO | Downloaded 34GiB to GPU in 2.4284s (14.001GiB/s) | 20260507 18:53:36.712 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:53:36.712 | INFO | Total mem usage: 16.4 GiB | 20260507 18:53:36.712 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:53:36.712 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 18:53:36.712 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:53:36.712 | INFO | High Water Breakdown: | 20260507 18:53:36.712 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:53:36.712 | INFO | 40.6% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:53:36.712 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:53:36.712 | INFO | ------------------------------------------ | 20260507 18:53:36.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:53:36.712 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:53:36.712 | INFO | Total mem usage: 26.5 GiB | 20260507 18:53:36.712 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:53:36.712 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:53:36.712 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:53:36.712 | INFO | High Water Breakdown: | 20260507 18:53:36.712 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:53:36.712 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:53:36.712 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:53:36.712 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:53:36.712 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:53:36.712 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:53:36.712 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:53:36.712 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:53:36.712 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:53:36.712 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:53:36.712 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:53:36.712 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:53:36.712 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:53:36.712 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:53:36.713 | INFO | ------------------------------------------ | 20260507 18:53:36.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:53:36.713 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:53:36.713 | INFO | Total mem usage: 27.8 GiB | 20260507 18:53:36.713 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:53:36.713 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:53:36.713 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:53:36.713 | INFO | High Water Breakdown: | 20260507 18:53:36.713 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:53:36.713 | INFO | ------------------------------------------ | 20260507 18:53:36.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:53:36.713 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:53:36.713 | INFO | Total mem usage: 15.7 GiB | 20260507 18:53:36.713 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:53:36.713 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:53:36.713 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:53:36.713 | INFO | High Water Breakdown: | 20260507 18:53:36.713 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:53:36.713 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:53:36.713 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:53:36.713 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:53:36.713 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:53:36.713 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:53:36.713 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:53:36.713 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:53:36.713 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:53:36.713 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:53:36.713 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:53:36.713 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:53:36.713 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:53:36.713 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:53:36.713 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:53:36.713 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:53:36.713 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:53:36.713 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:53:36.713 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:53:36.713 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:53:36.713 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:53:36.713 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:53:36.713 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:53:36.713 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:53:36.713 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:53:36.713 | INFO | ------------------------------------------ | 20260507 18:53:36.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:53:36.713 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:53:36.713 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:53:36.713 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:53:36.713 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.65e+01, 1.39e+01 / 2.11e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.17e-02 / 4.22e-03 | 20260507 18:53:36.713 | INFO | PerfReport: 12.10%, Baseline -- 4.04e+00 / 4.52e+00, 6.11e-01 / 1.90e-01 | 20260507 18:53:36.713 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.51e+00, 2.13e-01 / 9.87e-02 | 20260507 18:53:36.713 | INFO | PerfReport: 15.71%, DME -- 4.19e+01 / 4.51e+01, 4.07e+00 / 2.05e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.87e+00 / 2.44e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.49e+00 / 2.78e+00, 4.29e-01 / 2.71e-01 | 20260507 18:53:36.713 | INFO | PerfReport: 3.68%, Metrics -- 1.23e+00 / 1.41e+00, 3.12e-01 / 1.62e-01 | 20260507 18:53:36.713 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.86e-01 / 4.17e-01, 6.27e-02 / 3.78e-02 | 20260507 18:53:36.713 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:53:36.713 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:53:36.713 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:53:36.713 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 24.07%, Upload -- 2.58e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 24.12%, Download -- 2.59e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 31.09%, Compute -- 3.34e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 1.78%, LockUpload -- 1.91e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 2.37%, LockDownload -- 2.54e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 16.56%, LockCompute -- 1.78e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 18:53:36.713 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260507 18:53:36.809 | INFO | Starting a round of Baz IO | 20260507 18:53:40.312 | INFO | Populating RTMetrics report | 20260507 18:53:40.313 | INFO | RTMetrics report populated | 20260507 18:53:40.313 | INFO | Opening tmp RTMetrics json file | 20260507 18:53:40.328 | INFO | Serializing RTMetrics to json | 20260507 18:53:40.329 | INFO | Serialization done, swapping tmp file now... | 20260507 18:53:40.329 | INFO | RTMetrics done | 20260507 18:53:40.401 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.591 seconds | 20260507 18:53:40.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.04% / 0.01% / 53.95% | 20260507 18:53:40.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70883% / 1.58946e-05% / 5.5631e-05% | 20260507 18:53:40.401 | INFO | Baz IO overall: | 20260507 18:53:40.401 | INFO | BytesWritten : 809762490160 | 20260507 18:53:40.401 | INFO | NumEvents : 392603495577 | 20260507 18:53:41.851 | INFO | Analyzing chunk frames = [1802240,1802752) | 20260507 18:53:41.851 | INFO | MemInfo: { | 20260507 18:53:41.851 | INFO | "gpu" : " 17 GB", | 20260507 18:53:41.851 | INFO | "huge" : "", | 20260507 18:53:41.851 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:41.851 | INFO | "malloc" : " 14 GB", | 20260507 18:53:41.851 | INFO | "pinned" : " 29 GB" | 20260507 18:53:41.851 | INFO | } | 20260507 18:53:41.851 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:53:41.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:41.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:41.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:41.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:41.851 | INFO | ANALYSIS : 69.6%, 3.0, 108.3ms, 0 | 20260507 18:53:41.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:41.851 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 18:53:41.851 | INFO | PRE_HQ : 66.8%, 1.8, 61.9ms, 0 | 20260507 18:53:41.851 | INFO | BAZWRITER : 69.0%, 3.0, 106.2ms, 0 | 20260507 18:53:41.851 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:53:41.851 | INFO | Uploaded 36GiB to GPU in 2.56988s (14.0085GiB/s) | 20260507 18:53:41.851 | INFO | Downloaded 38GiB to GPU in 2.73021s (13.9183GiB/s) | 20260507 18:53:46.990 | INFO | Analyzing chunk frames = [1802752,1803264) | 20260507 18:53:46.990 | INFO | MemInfo: { | 20260507 18:53:46.990 | INFO | "gpu" : " 17 GB", | 20260507 18:53:46.990 | INFO | "huge" : "", | 20260507 18:53:46.990 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:46.990 | INFO | "malloc" : " 14 GB", | 20260507 18:53:46.990 | INFO | "pinned" : " 29 GB" | 20260507 18:53:46.990 | INFO | } | 20260507 18:53:46.990 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:53:46.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:46.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.990 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:53:46.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.990 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:53:46.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:46.991 | INFO | Uploaded 36GiB to GPU in 2.51894s (14.2917GiB/s) | 20260507 18:53:46.991 | INFO | Downloaded 34GiB to GPU in 2.46588s (13.7882GiB/s) | 20260507 18:53:52.130 | INFO | Analyzing chunk frames = [1803264,1803776) | 20260507 18:53:52.130 | INFO | MemInfo: { | 20260507 18:53:52.130 | INFO | "gpu" : " 17 GB", | 20260507 18:53:52.130 | INFO | "huge" : "", | 20260507 18:53:52.130 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:52.130 | INFO | "malloc" : " 14 GB", | 20260507 18:53:52.130 | INFO | "pinned" : " 29 GB" | 20260507 18:53:52.130 | INFO | } | 20260507 18:53:52.130 | INFO | Perf report for the last 5.13989 seconds: | 20260507 18:53:52.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:52.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 18:53:52.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:53:52.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:52.130 | INFO | Uploaded 36GiB to GPU in 2.53505s (14.2009GiB/s) | 20260507 18:53:52.130 | INFO | Downloaded 34GiB to GPU in 2.45475s (13.8507GiB/s) | 20260507 18:53:57.269 | INFO | Analyzing chunk frames = [1803776,1804288) | 20260507 18:53:57.269 | INFO | MemInfo: { | 20260507 18:53:57.269 | INFO | "gpu" : " 17 GB", | 20260507 18:53:57.269 | INFO | "huge" : "", | 20260507 18:53:57.269 | INFO | "huge_shared" : " 29 GB", | 20260507 18:53:57.269 | INFO | "malloc" : " 14 GB", | 20260507 18:53:57.269 | INFO | "pinned" : " 29 GB" | 20260507 18:53:57.269 | INFO | } | 20260507 18:53:57.269 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:53:57.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:53:57.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.269 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:53:57.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.269 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:53:57.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:53:57.270 | INFO | Uploaded 36GiB to GPU in 2.47805s (14.5276GiB/s) | 20260507 18:53:57.270 | INFO | Downloaded 34GiB to GPU in 2.37366s (14.3238GiB/s) | 20260507 18:54:02.409 | INFO | Analyzing chunk frames = [1804288,1804800) | 20260507 18:54:02.409 | INFO | MemInfo: { | 20260507 18:54:02.409 | INFO | "gpu" : " 17 GB", | 20260507 18:54:02.409 | INFO | "huge" : "", | 20260507 18:54:02.409 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:02.409 | INFO | "malloc" : " 14 GB", | 20260507 18:54:02.409 | INFO | "pinned" : " 29 GB" | 20260507 18:54:02.409 | INFO | } | 20260507 18:54:02.409 | INFO | Perf report for the last 5.13942 seconds: | 20260507 18:54:02.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:02.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 18:54:02.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 18:54:02.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:02.409 | INFO | Uploaded 36GiB to GPU in 2.30039s (15.6495GiB/s) | 20260507 18:54:02.409 | INFO | Downloaded 34GiB to GPU in 2.4866s (13.6733GiB/s) | 20260507 18:54:07.547 | INFO | Analyzing chunk frames = [1804800,1805312) | 20260507 18:54:07.547 | INFO | MemInfo: { | 20260507 18:54:07.547 | INFO | "gpu" : " 17 GB", | 20260507 18:54:07.547 | INFO | "huge" : "", | 20260507 18:54:07.547 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:07.547 | INFO | "malloc" : " 14 GB", | 20260507 18:54:07.547 | INFO | "pinned" : " 29 GB" | 20260507 18:54:07.547 | INFO | } | 20260507 18:54:07.547 | INFO | Perf report for the last 5.13873 seconds: | 20260507 18:54:07.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:07.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:54:07.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:54:07.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:07.548 | INFO | Uploaded 36GiB to GPU in 2.42834s (14.825GiB/s) | 20260507 18:54:07.548 | INFO | Downloaded 34GiB to GPU in 2.49459s (13.6295GiB/s) | 20260507 18:54:12.687 | INFO | Analyzing chunk frames = [1805312,1805824) | 20260507 18:54:12.687 | INFO | MemInfo: { | 20260507 18:54:12.687 | INFO | "gpu" : " 17 GB", | 20260507 18:54:12.687 | INFO | "huge" : "", | 20260507 18:54:12.687 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:12.687 | INFO | "malloc" : " 14 GB", | 20260507 18:54:12.687 | INFO | "pinned" : " 29 GB" | 20260507 18:54:12.687 | INFO | } | 20260507 18:54:12.687 | INFO | Perf report for the last 5.13999 seconds: | 20260507 18:54:12.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:12.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 18:54:12.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:54:12.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:12.688 | INFO | Uploaded 36GiB to GPU in 2.48572s (14.4827GiB/s) | 20260507 18:54:12.688 | INFO | Downloaded 34GiB to GPU in 2.43968s (13.9363GiB/s) | 20260507 18:54:17.826 | INFO | Analyzing chunk frames = [1805824,1806336) | 20260507 18:54:17.826 | INFO | MemInfo: { | 20260507 18:54:17.826 | INFO | "gpu" : " 17 GB", | 20260507 18:54:17.826 | INFO | "huge" : "", | 20260507 18:54:17.826 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:17.826 | INFO | "malloc" : " 14 GB", | 20260507 18:54:17.826 | INFO | "pinned" : " 29 GB" | 20260507 18:54:17.827 | INFO | } | 20260507 18:54:17.827 | INFO | Perf report for the last 5.1392 seconds: | 20260507 18:54:17.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:17.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:54:17.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 18:54:17.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:17.827 | INFO | Uploaded 36GiB to GPU in 2.50024s (14.3986GiB/s) | 20260507 18:54:17.827 | INFO | Downloaded 34GiB to GPU in 2.43573s (13.9589GiB/s) | 20260507 18:54:17.940 | INFO | Starting a round of Baz IO | 20260507 18:54:21.430 | INFO | Populating RTMetrics report | 20260507 18:54:21.430 | INFO | RTMetrics report populated | 20260507 18:54:21.430 | INFO | Opening tmp RTMetrics json file | 20260507 18:54:21.447 | INFO | Serializing RTMetrics to json | 20260507 18:54:21.448 | INFO | Serialization done, swapping tmp file now... | 20260507 18:54:21.448 | INFO | RTMetrics done | 20260507 18:54:21.510 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.569 seconds | 20260507 18:54:21.510 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.03% / 0.01% / 53.96% | 20260507 18:54:21.510 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70061% / 1.58946e-05% / 3.57628e-05% | 20260507 18:54:21.510 | INFO | Baz IO overall: | 20260507 18:54:21.510 | INFO | BytesWritten : 811697101483 | 20260507 18:54:21.510 | INFO | NumEvents : 393542580417 | 20260507 18:54:22.966 | INFO | Analyzing chunk frames = [1806336,1806848) | 20260507 18:54:22.966 | INFO | MemInfo: { | 20260507 18:54:22.966 | INFO | "gpu" : " 17 GB", | 20260507 18:54:22.966 | INFO | "huge" : "", | 20260507 18:54:22.966 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:22.966 | INFO | "malloc" : " 14 GB", | 20260507 18:54:22.966 | INFO | "pinned" : " 29 GB" | 20260507 18:54:22.966 | INFO | } | 20260507 18:54:22.966 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:54:22.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:22.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:22.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:22.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:22.966 | INFO | ANALYSIS : 69.7%, 2.9, 107.4ms, 0 | 20260507 18:54:22.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:22.966 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 18:54:22.966 | INFO | PRE_HQ : 67.6%, 1.9, 65.3ms, 0 | 20260507 18:54:22.966 | INFO | BAZWRITER : 68.9%, 3.1, 108.4ms, 0 | 20260507 18:54:22.966 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:54:22.966 | INFO | Uploaded 36GiB to GPU in 2.58797s (13.9105GiB/s) | 20260507 18:54:22.966 | INFO | Downloaded 38GiB to GPU in 2.69666s (14.0915GiB/s) | 20260507 18:54:28.105 | INFO | Analyzing chunk frames = [1806848,1807360) | 20260507 18:54:28.105 | INFO | MemInfo: { | 20260507 18:54:28.105 | INFO | "gpu" : " 17 GB", | 20260507 18:54:28.105 | INFO | "huge" : "", | 20260507 18:54:28.105 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:28.105 | INFO | "malloc" : " 14 GB", | 20260507 18:54:28.105 | INFO | "pinned" : " 29 GB" | 20260507 18:54:28.105 | INFO | } | 20260507 18:54:28.105 | INFO | Perf report for the last 5.13868 seconds: | 20260507 18:54:28.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:28.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260507 18:54:28.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:54:28.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:28.105 | INFO | Uploaded 36GiB to GPU in 2.5479s (14.1293GiB/s) | 20260507 18:54:28.105 | INFO | Downloaded 34GiB to GPU in 2.44139s (13.9265GiB/s) | 20260507 18:54:33.244 | INFO | Analyzing chunk frames = [1807360,1807872) | 20260507 18:54:33.244 | INFO | MemInfo: { | 20260507 18:54:33.244 | INFO | "gpu" : " 17 GB", | 20260507 18:54:33.244 | INFO | "huge" : "", | 20260507 18:54:33.244 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:33.244 | INFO | "malloc" : " 14 GB", | 20260507 18:54:33.244 | INFO | "pinned" : " 29 GB" | 20260507 18:54:33.244 | INFO | } | 20260507 18:54:33.244 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:54:33.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:33.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.244 | INFO | ANALYSIS : 68.1%, 3.0, 105.6ms, 0 | 20260507 18:54:33.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.244 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:54:33.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:33.245 | INFO | Uploaded 36GiB to GPU in 2.58s (13.9535GiB/s) | 20260507 18:54:33.245 | INFO | Downloaded 34GiB to GPU in 2.42352s (14.0292GiB/s) | 20260507 18:54:38.383 | INFO | Analyzing chunk frames = [1807872,1808384) | 20260507 18:54:38.383 | INFO | MemInfo: { | 20260507 18:54:38.383 | INFO | "gpu" : " 17 GB", | 20260507 18:54:38.383 | INFO | "huge" : "", | 20260507 18:54:38.383 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:38.383 | INFO | "malloc" : " 14 GB", | 20260507 18:54:38.383 | INFO | "pinned" : " 29 GB" | 20260507 18:54:38.383 | INFO | } | 20260507 18:54:38.384 | INFO | Perf report for the last 5.13938 seconds: | 20260507 18:54:38.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:38.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:54:38.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:54:38.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:38.384 | INFO | Uploaded 36GiB to GPU in 2.51142s (14.3345GiB/s) | 20260507 18:54:38.384 | INFO | Downloaded 34GiB to GPU in 2.35473s (14.439GiB/s) | 20260507 18:54:43.523 | INFO | Analyzing chunk frames = [1808384,1808896) | 20260507 18:54:43.523 | INFO | MemInfo: { | 20260507 18:54:43.523 | INFO | "gpu" : " 17 GB", | 20260507 18:54:43.523 | INFO | "huge" : "", | 20260507 18:54:43.523 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:43.523 | INFO | "malloc" : " 14 GB", | 20260507 18:54:43.523 | INFO | "pinned" : " 29 GB" | 20260507 18:54:43.523 | INFO | } | 20260507 18:54:43.523 | INFO | Perf report for the last 5.13966 seconds: | 20260507 18:54:43.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:43.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.523 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:54:43.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.523 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:54:43.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:43.524 | INFO | Uploaded 36GiB to GPU in 2.36053s (15.2508GiB/s) | 20260507 18:54:43.524 | INFO | Downloaded 34GiB to GPU in 2.47691s (13.7268GiB/s) | 20260507 18:54:48.662 | INFO | Analyzing chunk frames = [1808896,1809408) | 20260507 18:54:48.662 | INFO | MemInfo: { | 20260507 18:54:48.662 | INFO | "gpu" : " 17 GB", | 20260507 18:54:48.662 | INFO | "huge" : "", | 20260507 18:54:48.662 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:48.662 | INFO | "malloc" : " 14 GB", | 20260507 18:54:48.662 | INFO | "pinned" : " 29 GB" | 20260507 18:54:48.662 | INFO | } | 20260507 18:54:48.662 | INFO | Perf report for the last 5.13915 seconds: | 20260507 18:54:48.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:48.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 18:54:48.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:54:48.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:48.663 | INFO | Uploaded 36GiB to GPU in 2.4278s (14.8283GiB/s) | 20260507 18:54:48.663 | INFO | Downloaded 34GiB to GPU in 2.48376s (13.6889GiB/s) | 20260507 18:54:53.802 | INFO | Analyzing chunk frames = [1809408,1809920) | 20260507 18:54:53.802 | INFO | MemInfo: { | 20260507 18:54:53.802 | INFO | "gpu" : " 17 GB", | 20260507 18:54:53.802 | INFO | "huge" : "", | 20260507 18:54:53.802 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:53.802 | INFO | "malloc" : " 14 GB", | 20260507 18:54:53.802 | INFO | "pinned" : " 29 GB" | 20260507 18:54:53.802 | INFO | } | 20260507 18:54:53.803 | INFO | Perf report for the last 5.14013 seconds: | 20260507 18:54:53.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:53.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:54:53.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 18:54:53.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:53.803 | INFO | Uploaded 36GiB to GPU in 2.52069s (14.2818GiB/s) | 20260507 18:54:53.803 | INFO | Downloaded 34GiB to GPU in 2.43078s (13.9873GiB/s) | 20260507 18:54:58.941 | INFO | Analyzing chunk frames = [1809920,1810432) | 20260507 18:54:58.941 | INFO | MemInfo: { | 20260507 18:54:58.941 | INFO | "gpu" : " 17 GB", | 20260507 18:54:58.942 | INFO | "huge" : "", | 20260507 18:54:58.942 | INFO | "huge_shared" : " 29 GB", | 20260507 18:54:58.942 | INFO | "malloc" : " 14 GB", | 20260507 18:54:58.942 | INFO | "pinned" : " 29 GB" | 20260507 18:54:58.942 | INFO | } | 20260507 18:54:58.942 | INFO | Perf report for the last 5.13916 seconds: | 20260507 18:54:58.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:54:58.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 18:54:58.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | PRE_HQ : 37.7%, 1.0, 19.6ms, 0 | 20260507 18:54:58.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:54:58.942 | INFO | Uploaded 36GiB to GPU in 2.54063s (14.1697GiB/s) | 20260507 18:54:58.942 | INFO | Downloaded 34GiB to GPU in 2.41433s (14.0826GiB/s) | 20260507 18:54:58.942 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:54:58.942 | INFO | Total mem usage: 16.4 GiB | 20260507 18:54:58.942 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:54:58.942 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.8%) | 20260507 18:54:58.942 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:54:58.942 | INFO | High Water Breakdown: | 20260507 18:54:58.942 | INFO | 59.3% : 8.63 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:54:58.942 | INFO | 40.5% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:54:58.942 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:54:58.942 | INFO | ------------------------------------------ | 20260507 18:54:58.942 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:54:58.942 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:54:58.942 | INFO | Total mem usage: 26.5 GiB | 20260507 18:54:58.943 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:54:58.943 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:54:58.943 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:54:58.943 | INFO | High Water Breakdown: | 20260507 18:54:58.943 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:54:58.943 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:54:58.943 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:54:58.943 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:54:58.943 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:54:58.943 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:54:58.943 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:54:58.943 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:54:58.943 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:54:58.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:54:58.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:54:58.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:54:58.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:54:58.943 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:54:58.943 | INFO | ------------------------------------------ | 20260507 18:54:58.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:54:58.943 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:54:58.943 | INFO | Total mem usage: 27.8 GiB | 20260507 18:54:58.943 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:54:58.943 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:54:58.943 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:54:58.943 | INFO | High Water Breakdown: | 20260507 18:54:58.943 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:54:58.943 | INFO | ------------------------------------------ | 20260507 18:54:58.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:54:58.943 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:54:58.943 | INFO | Total mem usage: 15.7 GiB | 20260507 18:54:58.943 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:54:58.943 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:54:58.943 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:54:58.943 | INFO | High Water Breakdown: | 20260507 18:54:58.943 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:54:58.943 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:54:58.943 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:54:58.943 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:54:58.943 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:54:58.943 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:54:58.943 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:54:58.943 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:54:58.943 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:54:58.943 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:54:58.943 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:54:58.943 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:54:58.943 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:54:58.943 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:54:58.943 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:54:58.943 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:54:58.943 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:54:58.943 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:54:58.943 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:54:58.943 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:54:58.943 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:54:58.943 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:54:58.943 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:54:58.943 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:54:58.943 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:54:58.943 | INFO | ------------------------------------------ | 20260507 18:54:58.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:54:58.943 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:54:58.943 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:54:58.943 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:54:58.943 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.44e+01 / 2.15e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.05e-01, 1.22e-02 / 7.70e-03 | 20260507 18:54:58.943 | INFO | PerfReport: 12.09%, Baseline -- 4.03e+00 / 4.53e+00, 6.08e-01 / 1.70e-01 | 20260507 18:54:58.943 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.14e-01 / 1.16e-01 | 20260507 18:54:58.943 | INFO | PerfReport: 15.70%, DME -- 4.19e+01 / 4.49e+01, 3.94e+00 / 1.76e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.87e+00 / 2.22e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.32e-01 / 2.49e-01 | 20260507 18:54:58.943 | INFO | PerfReport: 3.68%, Metrics -- 1.23e+00 / 1.41e+00, 3.07e-01 / 1.51e-01 | 20260507 18:54:58.943 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.87e-01 / 4.15e-01, 6.16e-02 / 3.14e-02 | 20260507 18:54:58.943 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:54:58.943 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:54:58.943 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:54:58.943 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 24.20%, Upload -- 2.60e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 31.08%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 1.96%, LockUpload -- 2.10e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 2.21%, LockDownload -- 2.37e+00 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 16.52%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:54:58.943 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 18:54:59.056 | INFO | Starting a round of Baz IO | 20260507 18:55:02.540 | INFO | Populating RTMetrics report | 20260507 18:55:02.541 | INFO | RTMetrics report populated | 20260507 18:55:02.541 | INFO | Opening tmp RTMetrics json file | 20260507 18:55:02.557 | INFO | Serializing RTMetrics to json | 20260507 18:55:02.559 | INFO | Serialization done, swapping tmp file now... | 20260507 18:55:02.559 | INFO | RTMetrics done | 20260507 18:55:02.621 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.564 seconds | 20260507 18:55:02.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.01% / 0.01% / 53.97% | 20260507 18:55:02.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.69349% / 2.38419e-05% / 4.37101e-05% | 20260507 18:55:02.621 | INFO | Baz IO overall: | 20260507 18:55:02.621 | INFO | BytesWritten : 813631774621 | 20260507 18:55:02.621 | INFO | NumEvents : 394481652004 | 20260507 18:55:04.080 | INFO | Analyzing chunk frames = [1810432,1810944) | 20260507 18:55:04.080 | INFO | MemInfo: { | 20260507 18:55:04.080 | INFO | "gpu" : " 17 GB", | 20260507 18:55:04.080 | INFO | "huge" : "", | 20260507 18:55:04.080 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:04.080 | INFO | "malloc" : " 14 GB", | 20260507 18:55:04.080 | INFO | "pinned" : " 29 GB" | 20260507 18:55:04.080 | INFO | } | 20260507 18:55:04.080 | INFO | Perf report for the last 5.13868 seconds: | 20260507 18:55:04.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:04.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:04.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:04.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:04.081 | INFO | ANALYSIS : 69.3%, 2.9, 106.7ms, 0 | 20260507 18:55:04.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:04.081 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 18:55:04.081 | INFO | PRE_HQ : 67.1%, 1.9, 64.4ms, 0 | 20260507 18:55:04.081 | INFO | BAZWRITER : 68.5%, 3.1, 108.6ms, 0 | 20260507 18:55:04.081 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:55:04.081 | INFO | Uploaded 36GiB to GPU in 2.53192s (14.2184GiB/s) | 20260507 18:55:04.081 | INFO | Downloaded 38GiB to GPU in 2.70199s (14.0637GiB/s) | 20260507 18:55:09.220 | INFO | Analyzing chunk frames = [1810944,1811456) | 20260507 18:55:09.220 | INFO | MemInfo: { | 20260507 18:55:09.220 | INFO | "gpu" : " 17 GB", | 20260507 18:55:09.220 | INFO | "huge" : "", | 20260507 18:55:09.220 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:09.220 | INFO | "malloc" : " 14 GB", | 20260507 18:55:09.220 | INFO | "pinned" : " 29 GB" | 20260507 18:55:09.220 | INFO | } | 20260507 18:55:09.220 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:55:09.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:09.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:55:09.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 18:55:09.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:09.220 | INFO | Uploaded 36GiB to GPU in 2.49606s (14.4227GiB/s) | 20260507 18:55:09.220 | INFO | Downloaded 34GiB to GPU in 2.46575s (13.7889GiB/s) | 20260507 18:55:14.360 | INFO | Analyzing chunk frames = [1811456,1811968) | 20260507 18:55:14.360 | INFO | MemInfo: { | 20260507 18:55:14.360 | INFO | "gpu" : " 17 GB", | 20260507 18:55:14.360 | INFO | "huge" : "", | 20260507 18:55:14.360 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:14.360 | INFO | "malloc" : " 14 GB", | 20260507 18:55:14.360 | INFO | "pinned" : " 29 GB" | 20260507 18:55:14.360 | INFO | } | 20260507 18:55:14.360 | INFO | Perf report for the last 5.14015 seconds: | 20260507 18:55:14.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:14.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 18:55:14.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:55:14.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:14.360 | INFO | Uploaded 36GiB to GPU in 2.53468s (14.203GiB/s) | 20260507 18:55:14.360 | INFO | Downloaded 34GiB to GPU in 2.45468s (13.8511GiB/s) | 20260507 18:55:19.499 | INFO | Analyzing chunk frames = [1811968,1812480) | 20260507 18:55:19.499 | INFO | MemInfo: { | 20260507 18:55:19.499 | INFO | "gpu" : " 17 GB", | 20260507 18:55:19.499 | INFO | "huge" : "", | 20260507 18:55:19.499 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:19.499 | INFO | "malloc" : " 14 GB", | 20260507 18:55:19.499 | INFO | "pinned" : " 29 GB" | 20260507 18:55:19.499 | INFO | } | 20260507 18:55:19.499 | INFO | Perf report for the last 5.13885 seconds: | 20260507 18:55:19.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:19.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 18:55:19.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 18:55:19.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:19.499 | INFO | Uploaded 36GiB to GPU in 2.46316s (14.6154GiB/s) | 20260507 18:55:19.499 | INFO | Downloaded 34GiB to GPU in 2.371s (14.3399GiB/s) | 20260507 18:55:24.638 | INFO | Analyzing chunk frames = [1812480,1812992) | 20260507 18:55:24.638 | INFO | MemInfo: { | 20260507 18:55:24.638 | INFO | "gpu" : " 17 GB", | 20260507 18:55:24.638 | INFO | "huge" : "", | 20260507 18:55:24.638 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:24.638 | INFO | "malloc" : " 14 GB", | 20260507 18:55:24.638 | INFO | "pinned" : " 29 GB" | 20260507 18:55:24.638 | INFO | } | 20260507 18:55:24.638 | INFO | Perf report for the last 5.13954 seconds: | 20260507 18:55:24.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:24.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.638 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260507 18:55:24.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.638 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 18:55:24.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:24.639 | INFO | Uploaded 36GiB to GPU in 2.37247s (15.1741GiB/s) | 20260507 18:55:24.639 | INFO | Downloaded 34GiB to GPU in 2.49897s (13.6056GiB/s) | 20260507 18:55:29.777 | INFO | Analyzing chunk frames = [1812992,1813504) | 20260507 18:55:29.777 | INFO | MemInfo: { | 20260507 18:55:29.777 | INFO | "gpu" : " 17 GB", | 20260507 18:55:29.777 | INFO | "huge" : "", | 20260507 18:55:29.777 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:29.777 | INFO | "malloc" : " 14 GB", | 20260507 18:55:29.777 | INFO | "pinned" : " 29 GB" | 20260507 18:55:29.778 | INFO | } | 20260507 18:55:29.778 | INFO | Perf report for the last 5.13934 seconds: | 20260507 18:55:29.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:29.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:55:29.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:55:29.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:29.778 | INFO | Uploaded 36GiB to GPU in 2.41984s (14.877GiB/s) | 20260507 18:55:29.778 | INFO | Downloaded 34GiB to GPU in 2.52134s (13.4849GiB/s) | 20260507 18:55:34.917 | INFO | Analyzing chunk frames = [1813504,1814016) | 20260507 18:55:34.917 | INFO | MemInfo: { | 20260507 18:55:34.917 | INFO | "gpu" : " 17 GB", | 20260507 18:55:34.917 | INFO | "huge" : "", | 20260507 18:55:34.917 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:34.917 | INFO | "malloc" : " 14 GB", | 20260507 18:55:34.917 | INFO | "pinned" : " 29 GB" | 20260507 18:55:34.917 | INFO | } | 20260507 18:55:34.917 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:55:34.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:34.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.917 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:55:34.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.917 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 18:55:34.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:34.918 | INFO | Uploaded 36GiB to GPU in 2.50406s (14.3766GiB/s) | 20260507 18:55:34.918 | INFO | Downloaded 34GiB to GPU in 2.44548s (13.9032GiB/s) | 20260507 18:55:40.056 | INFO | Analyzing chunk frames = [1814016,1814528) | 20260507 18:55:40.056 | INFO | MemInfo: { | 20260507 18:55:40.056 | INFO | "gpu" : " 17 GB", | 20260507 18:55:40.056 | INFO | "huge" : "", | 20260507 18:55:40.056 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:40.056 | INFO | "malloc" : " 14 GB", | 20260507 18:55:40.056 | INFO | "pinned" : " 29 GB" | 20260507 18:55:40.056 | INFO | } | 20260507 18:55:40.056 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:55:40.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:40.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.056 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 18:55:40.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.057 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260507 18:55:40.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:40.057 | INFO | Uploaded 36GiB to GPU in 2.49926s (14.4043GiB/s) | 20260507 18:55:40.057 | INFO | Downloaded 34GiB to GPU in 2.44129s (13.927GiB/s) | 20260507 18:55:40.154 | INFO | Starting a round of Baz IO | 20260507 18:55:43.655 | INFO | Populating RTMetrics report | 20260507 18:55:43.656 | INFO | RTMetrics report populated | 20260507 18:55:43.656 | INFO | Opening tmp RTMetrics json file | 20260507 18:55:43.670 | INFO | Serializing RTMetrics to json | 20260507 18:55:43.674 | INFO | Serialization done, swapping tmp file now... | 20260507 18:55:43.675 | INFO | RTMetrics done | 20260507 18:55:43.745 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.591 seconds | 20260507 18:55:43.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.00% / 0.01% / 53.99% | 20260507 18:55:43.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.68793% / 1.98682e-05% / 3.17891e-05% | 20260507 18:55:43.745 | INFO | Baz IO overall: | 20260507 18:55:43.745 | INFO | BytesWritten : 815566357496 | 20260507 18:55:43.745 | INFO | NumEvents : 395420310312 | 20260507 18:55:45.195 | INFO | Analyzing chunk frames = [1814528,1815040) | 20260507 18:55:45.195 | INFO | MemInfo: { | 20260507 18:55:45.195 | INFO | "gpu" : " 17 GB", | 20260507 18:55:45.195 | INFO | "huge" : "", | 20260507 18:55:45.195 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:45.195 | INFO | "malloc" : " 14 GB", | 20260507 18:55:45.195 | INFO | "pinned" : " 29 GB" | 20260507 18:55:45.195 | INFO | } | 20260507 18:55:45.195 | INFO | Perf report for the last 5.13909 seconds: | 20260507 18:55:45.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:45.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:45.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:45.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:45.196 | INFO | ANALYSIS : 69.3%, 3.0, 108.2ms, 0 | 20260507 18:55:45.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:45.196 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 18:55:45.196 | INFO | PRE_HQ : 66.1%, 1.7, 58.7ms, 0 | 20260507 18:55:45.196 | INFO | BAZWRITER : 69.1%, 2.9, 104.0ms, 0 | 20260507 18:55:45.196 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 18:55:45.196 | INFO | Uploaded 36GiB to GPU in 2.51108s (14.3365GiB/s) | 20260507 18:55:45.196 | INFO | Downloaded 38GiB to GPU in 2.71547s (13.9939GiB/s) | 20260507 18:55:50.335 | INFO | Analyzing chunk frames = [1815040,1815552) | 20260507 18:55:50.335 | INFO | MemInfo: { | 20260507 18:55:50.335 | INFO | "gpu" : " 17 GB", | 20260507 18:55:50.335 | INFO | "huge" : "", | 20260507 18:55:50.335 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:50.335 | INFO | "malloc" : " 14 GB", | 20260507 18:55:50.335 | INFO | "pinned" : " 29 GB" | 20260507 18:55:50.335 | INFO | } | 20260507 18:55:50.335 | INFO | Perf report for the last 5.13939 seconds: | 20260507 18:55:50.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:50.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 18:55:50.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 18:55:50.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:50.335 | INFO | Uploaded 36GiB to GPU in 2.54036s (14.1712GiB/s) | 20260507 18:55:50.335 | INFO | Downloaded 34GiB to GPU in 2.44254s (13.9199GiB/s) | 20260507 18:55:55.474 | INFO | Analyzing chunk frames = [1815552,1816064) | 20260507 18:55:55.474 | INFO | MemInfo: { | 20260507 18:55:55.474 | INFO | "gpu" : " 17 GB", | 20260507 18:55:55.474 | INFO | "huge" : "", | 20260507 18:55:55.474 | INFO | "huge_shared" : " 29 GB", | 20260507 18:55:55.474 | INFO | "malloc" : " 14 GB", | 20260507 18:55:55.474 | INFO | "pinned" : " 29 GB" | 20260507 18:55:55.474 | INFO | } | 20260507 18:55:55.474 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:55:55.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:55:55.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.474 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 18:55:55.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.474 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:55:55.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:55:55.475 | INFO | Uploaded 36GiB to GPU in 2.56491s (14.0356GiB/s) | 20260507 18:55:55.475 | INFO | Downloaded 34GiB to GPU in 2.43184s (13.9812GiB/s) | 20260507 18:56:00.613 | INFO | Analyzing chunk frames = [1816064,1816576) | 20260507 18:56:00.613 | INFO | MemInfo: { | 20260507 18:56:00.613 | INFO | "gpu" : " 17 GB", | 20260507 18:56:00.613 | INFO | "huge" : "", | 20260507 18:56:00.613 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:00.613 | INFO | "malloc" : " 14 GB", | 20260507 18:56:00.613 | INFO | "pinned" : " 29 GB" | 20260507 18:56:00.613 | INFO | } | 20260507 18:56:00.614 | INFO | Perf report for the last 5.13942 seconds: | 20260507 18:56:00.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:00.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 18:56:00.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:56:00.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:00.614 | INFO | Uploaded 36GiB to GPU in 2.5032s (14.3816GiB/s) | 20260507 18:56:00.614 | INFO | Downloaded 34GiB to GPU in 2.35533s (14.4353GiB/s) | 20260507 18:56:05.753 | INFO | Analyzing chunk frames = [1816576,1817088) | 20260507 18:56:05.753 | INFO | MemInfo: { | 20260507 18:56:05.753 | INFO | "gpu" : " 17 GB", | 20260507 18:56:05.753 | INFO | "huge" : "", | 20260507 18:56:05.753 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:05.753 | INFO | "malloc" : " 14 GB", | 20260507 18:56:05.753 | INFO | "pinned" : " 29 GB" | 20260507 18:56:05.753 | INFO | } | 20260507 18:56:05.753 | INFO | Perf report for the last 5.13964 seconds: | 20260507 18:56:05.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:05.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.754 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 18:56:05.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.754 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 18:56:05.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:05.754 | INFO | Uploaded 36GiB to GPU in 2.36177s (15.2428GiB/s) | 20260507 18:56:05.754 | INFO | Downloaded 34GiB to GPU in 2.48425s (13.6862GiB/s) | 20260507 18:56:10.892 | INFO | Analyzing chunk frames = [1817088,1817600) | 20260507 18:56:10.892 | INFO | MemInfo: { | 20260507 18:56:10.892 | INFO | "gpu" : " 17 GB", | 20260507 18:56:10.892 | INFO | "huge" : "", | 20260507 18:56:10.892 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:10.892 | INFO | "malloc" : " 14 GB", | 20260507 18:56:10.892 | INFO | "pinned" : " 29 GB" | 20260507 18:56:10.892 | INFO | } | 20260507 18:56:10.892 | INFO | Perf report for the last 5.13894 seconds: | 20260507 18:56:10.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:10.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:56:10.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:56:10.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:10.892 | INFO | Uploaded 36GiB to GPU in 2.4464s (14.7155GiB/s) | 20260507 18:56:10.893 | INFO | Downloaded 34GiB to GPU in 2.4891s (13.6595GiB/s) | 20260507 18:56:16.031 | INFO | Analyzing chunk frames = [1817600,1818112) | 20260507 18:56:16.031 | INFO | MemInfo: { | 20260507 18:56:16.031 | INFO | "gpu" : " 17 GB", | 20260507 18:56:16.031 | INFO | "huge" : "", | 20260507 18:56:16.031 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:16.031 | INFO | "malloc" : " 14 GB", | 20260507 18:56:16.032 | INFO | "pinned" : " 29 GB" | 20260507 18:56:16.032 | INFO | } | 20260507 18:56:16.032 | INFO | Perf report for the last 5.13966 seconds: | 20260507 18:56:16.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:16.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:56:16.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 18:56:16.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:16.032 | INFO | Uploaded 36GiB to GPU in 2.54448s (14.1483GiB/s) | 20260507 18:56:16.032 | INFO | Downloaded 34GiB to GPU in 2.43622s (13.9561GiB/s) | 20260507 18:56:21.171 | INFO | Analyzing chunk frames = [1818112,1818624) | 20260507 18:56:21.171 | INFO | MemInfo: { | 20260507 18:56:21.171 | INFO | "gpu" : " 17 GB", | 20260507 18:56:21.171 | INFO | "huge" : "", | 20260507 18:56:21.171 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:21.171 | INFO | "malloc" : " 14 GB", | 20260507 18:56:21.171 | INFO | "pinned" : " 29 GB" | 20260507 18:56:21.171 | INFO | } | 20260507 18:56:21.171 | INFO | Perf report for the last 5.13928 seconds: | 20260507 18:56:21.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:21.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.171 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 18:56:21.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.171 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 18:56:21.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:21.172 | INFO | Uploaded 36GiB to GPU in 2.52304s (14.2685GiB/s) | 20260507 18:56:21.172 | INFO | Downloaded 34GiB to GPU in 2.41478s (14.08GiB/s) | 20260507 18:56:21.172 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:56:21.172 | INFO | Total mem usage: 16.4 GiB | 20260507 18:56:21.172 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:56:21.172 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 18:56:21.172 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:56:21.172 | INFO | High Water Breakdown: | 20260507 18:56:21.172 | INFO | 59.3% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:56:21.172 | INFO | 40.5% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:56:21.172 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:56:21.172 | INFO | ------------------------------------------ | 20260507 18:56:21.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:56:21.172 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:56:21.172 | INFO | Total mem usage: 26.5 GiB | 20260507 18:56:21.172 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:56:21.172 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:56:21.172 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:56:21.172 | INFO | High Water Breakdown: | 20260507 18:56:21.172 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:56:21.172 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:56:21.172 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:56:21.172 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:56:21.172 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:56:21.172 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:56:21.172 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:56:21.172 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:56:21.172 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:56:21.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:56:21.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:56:21.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:56:21.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:56:21.172 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:56:21.172 | INFO | ------------------------------------------ | 20260507 18:56:21.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:56:21.172 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:56:21.172 | INFO | Total mem usage: 27.8 GiB | 20260507 18:56:21.172 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:56:21.172 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:56:21.172 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:56:21.172 | INFO | High Water Breakdown: | 20260507 18:56:21.172 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:56:21.172 | INFO | ------------------------------------------ | 20260507 18:56:21.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:56:21.172 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:56:21.172 | INFO | Total mem usage: 15.7 GiB | 20260507 18:56:21.172 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:56:21.172 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:56:21.172 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 18:56:21.172 | INFO | High Water Breakdown: | 20260507 18:56:21.172 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:56:21.172 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:56:21.172 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:56:21.172 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:56:21.172 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:56:21.172 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:56:21.172 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:56:21.172 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:56:21.172 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:56:21.172 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:56:21.172 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:56:21.172 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:56:21.172 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:56:21.172 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:56:21.172 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:56:21.172 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:56:21.172 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:56:21.172 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:56:21.172 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:56:21.172 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:56:21.172 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:56:21.172 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:56:21.172 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:56:21.172 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:56:21.172 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:56:21.172 | INFO | ------------------------------------------ | 20260507 18:56:21.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:56:21.172 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:56:21.172 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:56:21.172 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:56:21.172 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.66e+01, 1.44e+01 / 2.03e+00 | 20260507 18:56:21.172 | INFO | PerfReport: 0.30%, AggregateStats -- 9.96e-02 / 1.05e-01, 1.24e-02 / 9.38e-03 | 20260507 18:56:21.172 | INFO | PerfReport: 12.11%, Baseline -- 4.05e+00 / 4.53e+00, 6.11e-01 / 1.99e-01 | 20260507 18:56:21.172 | INFO | PerfReport: 3.44%, Histogram -- 1.15e+00 / 1.49e+00, 2.14e-01 / 1.21e-01 | 20260507 18:56:21.174 | INFO | PerfReport: 15.69%, DME -- 4.19e+01 / 4.50e+01, 4.08e+00 / 1.84e+00 | 20260507 18:56:21.174 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.88e+01 / 2.08e+01, 1.90e+00 / 2.26e+00 | 20260507 18:56:21.174 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.31e-01 / 2.54e-01 | 20260507 18:56:21.175 | INFO | PerfReport: 3.68%, Metrics -- 1.23e+00 / 1.44e+00, 3.13e-01 / 1.95e-01 | 20260507 18:56:21.175 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.87e-01 / 4.19e-01, 6.18e-02 / 3.51e-02 | 20260507 18:56:21.175 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:56:21.175 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:56:21.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:56:21.175 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 24.15%, Upload -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 24.06%, Download -- 2.58e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 31.14%, Compute -- 3.34e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 1.87%, LockUpload -- 2.00e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 2.25%, LockDownload -- 2.42e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 16.51%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:56:21.175 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 18:56:21.285 | INFO | Starting a round of Baz IO | 20260507 18:56:24.777 | INFO | Populating RTMetrics report | 20260507 18:56:24.777 | INFO | RTMetrics report populated | 20260507 18:56:24.777 | INFO | Opening tmp RTMetrics json file | 20260507 18:56:24.785 | INFO | Serializing RTMetrics to json | 20260507 18:56:24.796 | INFO | Serialization done, swapping tmp file now... | 20260507 18:56:24.797 | INFO | RTMetrics done | 20260507 18:56:24.859 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.573 seconds | 20260507 18:56:24.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.99% / 0.01% / 54.00% | 20260507 18:56:24.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.67508% / 1.98682e-05% / 2.78155e-05% | 20260507 18:56:24.859 | INFO | Baz IO overall: | 20260507 18:56:24.859 | INFO | BytesWritten : 817498934185 | 20260507 18:56:24.859 | INFO | NumEvents : 396357300139 | 20260507 18:56:26.310 | INFO | Analyzing chunk frames = [1818624,1819136) | 20260507 18:56:26.310 | INFO | MemInfo: { | 20260507 18:56:26.310 | INFO | "gpu" : " 17 GB", | 20260507 18:56:26.310 | INFO | "huge" : "", | 20260507 18:56:26.310 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:26.310 | INFO | "malloc" : " 14 GB", | 20260507 18:56:26.310 | INFO | "pinned" : " 29 GB" | 20260507 18:56:26.310 | INFO | } | 20260507 18:56:26.310 | INFO | Perf report for the last 5.13897 seconds: | 20260507 18:56:26.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:26.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:26.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:26.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:26.310 | INFO | ANALYSIS : 69.4%, 2.9, 106.7ms, 0 | 20260507 18:56:26.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:26.310 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 18:56:26.310 | INFO | PRE_HQ : 67.5%, 1.9, 66.2ms, 0 | 20260507 18:56:26.310 | INFO | BAZWRITER : 68.7%, 3.1, 110.8ms, 0 | 20260507 18:56:26.310 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 18:56:26.311 | INFO | Uploaded 36GiB to GPU in 2.56535s (14.0332GiB/s) | 20260507 18:56:26.311 | INFO | Downloaded 38GiB to GPU in 2.69337s (14.1087GiB/s) | 20260507 18:56:31.449 | INFO | Analyzing chunk frames = [1819136,1819648) | 20260507 18:56:31.449 | INFO | MemInfo: { | 20260507 18:56:31.449 | INFO | "gpu" : " 17 GB", | 20260507 18:56:31.449 | INFO | "huge" : "", | 20260507 18:56:31.449 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:31.449 | INFO | "malloc" : " 14 GB", | 20260507 18:56:31.449 | INFO | "pinned" : " 29 GB" | 20260507 18:56:31.449 | INFO | } | 20260507 18:56:31.449 | INFO | Perf report for the last 5.13931 seconds: | 20260507 18:56:31.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:31.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.449 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:56:31.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.450 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 18:56:31.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:31.450 | INFO | Uploaded 36GiB to GPU in 2.50786s (14.3548GiB/s) | 20260507 18:56:31.450 | INFO | Downloaded 34GiB to GPU in 2.46429s (13.7971GiB/s) | 20260507 18:56:36.588 | INFO | Analyzing chunk frames = [1819648,1820160) | 20260507 18:56:36.588 | INFO | MemInfo: { | 20260507 18:56:36.588 | INFO | "gpu" : " 17 GB", | 20260507 18:56:36.588 | INFO | "huge" : "", | 20260507 18:56:36.589 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:36.589 | INFO | "malloc" : " 14 GB", | 20260507 18:56:36.589 | INFO | "pinned" : " 29 GB" | 20260507 18:56:36.589 | INFO | } | 20260507 18:56:36.589 | INFO | Perf report for the last 5.13919 seconds: | 20260507 18:56:36.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:36.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:56:36.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:56:36.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:36.589 | INFO | Uploaded 36GiB to GPU in 2.54433s (14.1491GiB/s) | 20260507 18:56:36.589 | INFO | Downloaded 34GiB to GPU in 2.45158s (13.8686GiB/s) | 20260507 18:56:41.728 | INFO | Analyzing chunk frames = [1820160,1820672) | 20260507 18:56:41.728 | INFO | MemInfo: { | 20260507 18:56:41.728 | INFO | "gpu" : " 17 GB", | 20260507 18:56:41.728 | INFO | "huge" : "", | 20260507 18:56:41.728 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:41.728 | INFO | "malloc" : " 14 GB", | 20260507 18:56:41.728 | INFO | "pinned" : " 29 GB" | 20260507 18:56:41.728 | INFO | } | 20260507 18:56:41.728 | INFO | Perf report for the last 5.13981 seconds: | 20260507 18:56:41.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:41.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.728 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 18:56:41.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.729 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:56:41.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:41.729 | INFO | Uploaded 36GiB to GPU in 2.47667s (14.5356GiB/s) | 20260507 18:56:41.729 | INFO | Downloaded 34GiB to GPU in 2.37308s (14.3274GiB/s) | 20260507 18:56:46.868 | INFO | Analyzing chunk frames = [1820672,1821184) | 20260507 18:56:46.868 | INFO | MemInfo: { | 20260507 18:56:46.868 | INFO | "gpu" : " 17 GB", | 20260507 18:56:46.868 | INFO | "huge" : "", | 20260507 18:56:46.868 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:46.868 | INFO | "malloc" : " 14 GB", | 20260507 18:56:46.868 | INFO | "pinned" : " 29 GB" | 20260507 18:56:46.868 | INFO | } | 20260507 18:56:46.868 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:56:46.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:46.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:56:46.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:56:46.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:46.868 | INFO | Uploaded 36GiB to GPU in 2.31975s (15.5189GiB/s) | 20260507 18:56:46.868 | INFO | Downloaded 34GiB to GPU in 2.48211s (13.698GiB/s) | 20260507 18:56:52.007 | INFO | Analyzing chunk frames = [1821184,1821696) | 20260507 18:56:52.007 | INFO | MemInfo: { | 20260507 18:56:52.007 | INFO | "gpu" : " 17 GB", | 20260507 18:56:52.007 | INFO | "huge" : "", | 20260507 18:56:52.007 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:52.007 | INFO | "malloc" : " 14 GB", | 20260507 18:56:52.007 | INFO | "pinned" : " 29 GB" | 20260507 18:56:52.007 | INFO | } | 20260507 18:56:52.007 | INFO | Perf report for the last 5.13893 seconds: | 20260507 18:56:52.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:52.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:56:52.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:56:52.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:52.007 | INFO | Uploaded 36GiB to GPU in 2.39058s (15.0591GiB/s) | 20260507 18:56:52.007 | INFO | Downloaded 34GiB to GPU in 2.5051s (13.5723GiB/s) | 20260507 18:56:57.146 | INFO | Analyzing chunk frames = [1821696,1822208) | 20260507 18:56:57.146 | INFO | MemInfo: { | 20260507 18:56:57.146 | INFO | "gpu" : " 17 GB", | 20260507 18:56:57.146 | INFO | "huge" : "", | 20260507 18:56:57.146 | INFO | "huge_shared" : " 29 GB", | 20260507 18:56:57.146 | INFO | "malloc" : " 14 GB", | 20260507 18:56:57.146 | INFO | "pinned" : " 29 GB" | 20260507 18:56:57.146 | INFO | } | 20260507 18:56:57.146 | INFO | Perf report for the last 5.13937 seconds: | 20260507 18:56:57.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:56:57.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.146 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:56:57.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.146 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:56:57.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:56:57.147 | INFO | Uploaded 36GiB to GPU in 2.48965s (14.4599GiB/s) | 20260507 18:56:57.147 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9296GiB/s) | 20260507 18:57:02.285 | INFO | Analyzing chunk frames = [1822208,1822720) | 20260507 18:57:02.285 | INFO | MemInfo: { | 20260507 18:57:02.285 | INFO | "gpu" : " 17 GB", | 20260507 18:57:02.285 | INFO | "huge" : "", | 20260507 18:57:02.285 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:02.285 | INFO | "malloc" : " 14 GB", | 20260507 18:57:02.285 | INFO | "pinned" : " 29 GB" | 20260507 18:57:02.285 | INFO | } | 20260507 18:57:02.285 | INFO | Perf report for the last 5.13906 seconds: | 20260507 18:57:02.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:02.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.285 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:57:02.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.286 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:57:02.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:02.286 | INFO | Uploaded 36GiB to GPU in 2.4864s (14.4787GiB/s) | 20260507 18:57:02.286 | INFO | Downloaded 34GiB to GPU in 2.43002s (13.9916GiB/s) | 20260507 18:57:02.382 | INFO | Starting a round of Baz IO | 20260507 18:57:05.880 | INFO | Populating RTMetrics report | 20260507 18:57:05.880 | INFO | RTMetrics report populated | 20260507 18:57:05.880 | INFO | Opening tmp RTMetrics json file | 20260507 18:57:05.881 | INFO | Serializing RTMetrics to json | 20260507 18:57:05.893 | INFO | Serialization done, swapping tmp file now... | 20260507 18:57:05.899 | INFO | RTMetrics done | 20260507 18:57:05.963 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.58 seconds | 20260507 18:57:05.963 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.97% / 0.01% / 54.01% | 20260507 18:57:05.963 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.67023% / 2.38419e-05% / 3.97364e-05% | 20260507 18:57:05.963 | INFO | Baz IO overall: | 20260507 18:57:05.963 | INFO | BytesWritten : 819433071714 | 20260507 18:57:05.963 | INFO | NumEvents : 397295423059 | 20260507 18:57:07.425 | INFO | Analyzing chunk frames = [1822720,1823232) | 20260507 18:57:07.425 | INFO | MemInfo: { | 20260507 18:57:07.425 | INFO | "gpu" : " 17 GB", | 20260507 18:57:07.425 | INFO | "huge" : "", | 20260507 18:57:07.425 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:07.425 | INFO | "malloc" : " 14 GB", | 20260507 18:57:07.425 | INFO | "pinned" : " 29 GB" | 20260507 18:57:07.425 | INFO | } | 20260507 18:57:07.425 | INFO | Perf report for the last 5.13944 seconds: | 20260507 18:57:07.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:07.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:07.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:07.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:07.425 | INFO | ANALYSIS : 69.5%, 3.0, 108.2ms, 0 | 20260507 18:57:07.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:07.425 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 18:57:07.425 | INFO | PRE_HQ : 66.5%, 1.7, 58.7ms, 0 | 20260507 18:57:07.425 | INFO | BAZWRITER : 68.9%, 2.9, 104.3ms, 0 | 20260507 18:57:07.425 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 18:57:07.426 | INFO | Uploaded 36GiB to GPU in 2.56708s (14.0237GiB/s) | 20260507 18:57:07.426 | INFO | Downloaded 38GiB to GPU in 2.7209s (13.966GiB/s) | 20260507 18:57:12.564 | INFO | Analyzing chunk frames = [1823232,1823744) | 20260507 18:57:12.564 | INFO | MemInfo: { | 20260507 18:57:12.564 | INFO | "gpu" : " 17 GB", | 20260507 18:57:12.564 | INFO | "huge" : "", | 20260507 18:57:12.564 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:12.564 | INFO | "malloc" : " 14 GB", | 20260507 18:57:12.564 | INFO | "pinned" : " 29 GB" | 20260507 18:57:12.564 | INFO | } | 20260507 18:57:12.564 | INFO | Perf report for the last 5.13918 seconds: | 20260507 18:57:12.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:12.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 18:57:12.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 18:57:12.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:12.564 | INFO | Uploaded 36GiB to GPU in 2.50994s (14.343GiB/s) | 20260507 18:57:12.564 | INFO | Downloaded 34GiB to GPU in 2.46141s (13.8132GiB/s) | 20260507 18:57:17.703 | INFO | Analyzing chunk frames = [1823744,1824256) | 20260507 18:57:17.703 | INFO | MemInfo: { | 20260507 18:57:17.703 | INFO | "gpu" : " 17 GB", | 20260507 18:57:17.703 | INFO | "huge" : "", | 20260507 18:57:17.703 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:17.704 | INFO | "malloc" : " 14 GB", | 20260507 18:57:17.704 | INFO | "pinned" : " 29 GB" | 20260507 18:57:17.704 | INFO | } | 20260507 18:57:17.704 | INFO | Perf report for the last 5.13951 seconds: | 20260507 18:57:17.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:17.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 18:57:17.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 18:57:17.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:17.704 | INFO | Uploaded 36GiB to GPU in 2.55065s (14.1141GiB/s) | 20260507 18:57:17.704 | INFO | Downloaded 34GiB to GPU in 2.45605s (13.8434GiB/s) | 20260507 18:57:22.843 | INFO | Analyzing chunk frames = [1824256,1824768) | 20260507 18:57:22.843 | INFO | MemInfo: { | 20260507 18:57:22.843 | INFO | "gpu" : " 17 GB", | 20260507 18:57:22.843 | INFO | "huge" : "", | 20260507 18:57:22.843 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:22.843 | INFO | "malloc" : " 14 GB", | 20260507 18:57:22.843 | INFO | "pinned" : " 29 GB" | 20260507 18:57:22.843 | INFO | } | 20260507 18:57:22.843 | INFO | Perf report for the last 5.13958 seconds: | 20260507 18:57:22.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:22.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.843 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:57:22.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.843 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:57:22.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:22.844 | INFO | Uploaded 36GiB to GPU in 2.47942s (14.5195GiB/s) | 20260507 18:57:22.844 | INFO | Downloaded 34GiB to GPU in 2.37947s (14.2889GiB/s) | 20260507 18:57:27.982 | INFO | Analyzing chunk frames = [1824768,1825280) | 20260507 18:57:27.983 | INFO | MemInfo: { | 20260507 18:57:27.983 | INFO | "gpu" : " 17 GB", | 20260507 18:57:27.983 | INFO | "huge" : "", | 20260507 18:57:27.983 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:27.983 | INFO | "malloc" : " 14 GB", | 20260507 18:57:27.983 | INFO | "pinned" : " 29 GB" | 20260507 18:57:27.983 | INFO | } | 20260507 18:57:27.983 | INFO | Perf report for the last 5.13956 seconds: | 20260507 18:57:27.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:27.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 18:57:27.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:57:27.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:27.983 | INFO | Uploaded 36GiB to GPU in 2.34368s (15.3605GiB/s) | 20260507 18:57:27.983 | INFO | Downloaded 34GiB to GPU in 2.48617s (13.6756GiB/s) | 20260507 18:57:33.121 | INFO | Analyzing chunk frames = [1825280,1825792) | 20260507 18:57:33.121 | INFO | MemInfo: { | 20260507 18:57:33.121 | INFO | "gpu" : " 17 GB", | 20260507 18:57:33.122 | INFO | "huge" : "", | 20260507 18:57:33.122 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:33.122 | INFO | "malloc" : " 14 GB", | 20260507 18:57:33.122 | INFO | "pinned" : " 29 GB" | 20260507 18:57:33.122 | INFO | } | 20260507 18:57:33.122 | INFO | Perf report for the last 5.1392 seconds: | 20260507 18:57:33.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:33.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.122 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 18:57:33.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.122 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 18:57:33.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:33.123 | INFO | Uploaded 36GiB to GPU in 2.50978s (14.3439GiB/s) | 20260507 18:57:33.123 | INFO | Downloaded 34GiB to GPU in 2.40252s (14.1518GiB/s) | 20260507 18:57:38.261 | INFO | Analyzing chunk frames = [1825792,1826304) | 20260507 18:57:38.261 | INFO | MemInfo: { | 20260507 18:57:38.261 | INFO | "gpu" : " 17 GB", | 20260507 18:57:38.261 | INFO | "huge" : "", | 20260507 18:57:38.261 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:38.261 | INFO | "malloc" : " 14 GB", | 20260507 18:57:38.261 | INFO | "pinned" : " 29 GB" | 20260507 18:57:38.261 | INFO | } | 20260507 18:57:38.261 | INFO | Perf report for the last 5.13932 seconds: | 20260507 18:57:38.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:38.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.261 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:57:38.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.262 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 18:57:38.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:38.262 | INFO | Uploaded 36GiB to GPU in 2.5284s (14.2382GiB/s) | 20260507 18:57:38.262 | INFO | Downloaded 34GiB to GPU in 2.45233s (13.8644GiB/s) | 20260507 18:57:43.400 | INFO | Analyzing chunk frames = [1826304,1826816) | 20260507 18:57:43.400 | INFO | MemInfo: { | 20260507 18:57:43.400 | INFO | "gpu" : " 17 GB", | 20260507 18:57:43.400 | INFO | "huge" : "", | 20260507 18:57:43.400 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:43.401 | INFO | "malloc" : " 14 GB", | 20260507 18:57:43.401 | INFO | "pinned" : " 29 GB" | 20260507 18:57:43.401 | INFO | } | 20260507 18:57:43.401 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:57:43.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:43.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:57:43.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | PRE_HQ : 36.5%, 1.0, 18.9ms, 0 | 20260507 18:57:43.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:43.401 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:57:43.401 | INFO | Total mem usage: 16.4 GiB | 20260507 18:57:43.401 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:57:43.401 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -13.3%) | 20260507 18:57:43.401 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:57:43.401 | INFO | High Water Breakdown: | 20260507 18:57:43.401 | INFO | 59.3% : 8.58 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:57:43.401 | INFO | 40.6% : 5.87 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:57:43.401 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:57:43.401 | INFO | ------------------------------------------ | 20260507 18:57:43.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:57:43.401 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:57:43.401 | INFO | Total mem usage: 26.5 GiB | 20260507 18:57:43.401 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:57:43.401 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 18:57:43.401 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 18:57:43.401 | INFO | High Water Breakdown: | 20260507 18:57:43.401 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:57:43.401 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:57:43.401 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:57:43.401 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:57:43.401 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:57:43.401 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:57:43.401 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:57:43.401 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:57:43.401 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:57:43.401 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:57:43.401 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:57:43.401 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:57:43.401 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:57:43.401 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:57:43.401 | INFO | ------------------------------------------ | 20260507 18:57:43.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:57:43.401 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:57:43.401 | INFO | Total mem usage: 27.8 GiB | 20260507 18:57:43.401 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:57:43.401 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:57:43.401 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:57:43.401 | INFO | High Water Breakdown: | 20260507 18:57:43.401 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:57:43.401 | INFO | ------------------------------------------ | 20260507 18:57:43.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:57:43.401 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:57:43.401 | INFO | Total mem usage: 15.7 GiB | 20260507 18:57:43.401 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:57:43.401 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:57:43.401 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:57:43.401 | INFO | High Water Breakdown: | 20260507 18:57:43.401 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:57:43.401 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:57:43.401 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:57:43.401 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:57:43.401 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:57:43.401 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:57:43.401 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:57:43.401 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:57:43.401 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:57:43.401 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:57:43.401 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:57:43.401 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:57:43.401 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:57:43.401 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:57:43.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:57:43.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:57:43.401 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:57:43.401 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:57:43.401 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:57:43.401 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:57:43.401 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:57:43.401 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:57:43.401 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:57:43.401 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:57:43.401 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:57:43.401 | INFO | ------------------------------------------ | 20260507 18:57:43.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:57:43.401 | INFO | Uploaded 36GiB to GPU in 2.48547s (14.4842GiB/s) | 20260507 18:57:43.401 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:57:43.401 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:57:43.401 | INFO | Downloaded 34GiB to GPU in 2.43191s (13.9808GiB/s) | 20260507 18:57:43.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:57:43.401 | INFO | PerfReport: Full Execution -- 3.34e+01 / 7.64e+01, 1.45e+01 / 2.19e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 0.30%, AggregateStats -- 9.97e-02 / 1.04e-01, 1.24e-02 / 6.32e-03 | 20260507 18:57:43.401 | INFO | PerfReport: 12.11%, Baseline -- 4.05e+00 / 4.52e+00, 6.12e-01 / 1.82e-01 | 20260507 18:57:43.401 | INFO | PerfReport: 3.43%, Histogram -- 1.15e+00 / 1.50e+00, 2.16e-01 / 1.12e-01 | 20260507 18:57:43.401 | INFO | PerfReport: 15.66%, DME -- 4.19e+01 / 4.49e+01, 4.02e+00 / 1.99e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.93e+00 / 2.05e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.49e+00 / 2.83e+00, 4.36e-01 / 2.20e-01 | 20260507 18:57:43.401 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.43e+00, 3.16e-01 / 1.74e-01 | 20260507 18:57:43.401 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.87e-01 / 4.19e-01, 6.24e-02 / 3.52e-02 | 20260507 18:57:43.401 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:57:43.401 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:57:43.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:57:43.401 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 24.15%, Upload -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 24.07%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 31.18%, Compute -- 3.34e+01 / 0.00e+00, 1.45e+01 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 1.77%, LockUpload -- 1.90e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 2.27%, LockDownload -- 2.43e+00 / 0.00e+00, 3.77e+00 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 16.54%, LockCompute -- 1.77e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 18:57:43.401 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 18:57:43.498 | INFO | Starting a round of Baz IO | 20260507 18:57:46.989 | INFO | Populating RTMetrics report | 20260507 18:57:46.990 | INFO | RTMetrics report populated | 20260507 18:57:46.990 | INFO | Opening tmp RTMetrics json file | 20260507 18:57:46.997 | INFO | Serializing RTMetrics to json | 20260507 18:57:47.008 | INFO | Serialization done, swapping tmp file now... | 20260507 18:57:47.008 | INFO | RTMetrics done | 20260507 18:57:47.071 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.572 seconds | 20260507 18:57:47.071 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.96% / 0.01% / 54.03% | 20260507 18:57:47.071 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.66933% / 2.38419e-05% / 3.97364e-05% | 20260507 18:57:47.071 | INFO | Baz IO overall: | 20260507 18:57:47.071 | INFO | BytesWritten : 821367200045 | 20260507 18:57:47.071 | INFO | NumEvents : 398234066536 | 20260507 18:57:48.540 | INFO | Analyzing chunk frames = [1826816,1827328) | 20260507 18:57:48.540 | INFO | MemInfo: { | 20260507 18:57:48.540 | INFO | "gpu" : " 17 GB", | 20260507 18:57:48.540 | INFO | "huge" : "", | 20260507 18:57:48.540 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:48.540 | INFO | "malloc" : " 14 GB", | 20260507 18:57:48.540 | INFO | "pinned" : " 29 GB" | 20260507 18:57:48.540 | INFO | } | 20260507 18:57:48.540 | INFO | Perf report for the last 5.13903 seconds: | 20260507 18:57:48.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:48.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:48.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:48.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:48.540 | INFO | ANALYSIS : 69.4%, 3.0, 107.9ms, 0 | 20260507 18:57:48.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:48.540 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 18:57:48.540 | INFO | PRE_HQ : 66.4%, 1.7, 58.7ms, 0 | 20260507 18:57:48.540 | INFO | BAZWRITER : 68.7%, 2.9, 103.7ms, 0 | 20260507 18:57:48.540 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:57:48.540 | INFO | Uploaded 36GiB to GPU in 2.55617s (14.0836GiB/s) | 20260507 18:57:48.540 | INFO | Downloaded 38GiB to GPU in 2.73095s (13.9146GiB/s) | 20260507 18:57:53.697 | INFO | Analyzing chunk frames = [1827328,1827840) | 20260507 18:57:53.697 | INFO | MemInfo: { | 20260507 18:57:53.697 | INFO | "gpu" : " 17 GB", | 20260507 18:57:53.697 | INFO | "huge" : "", | 20260507 18:57:53.697 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:53.697 | INFO | "malloc" : " 14 GB", | 20260507 18:57:53.697 | INFO | "pinned" : " 29 GB" | 20260507 18:57:53.697 | INFO | } | 20260507 18:57:53.697 | INFO | Perf report for the last 5.15766 seconds: | 20260507 18:57:53.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:53.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.697 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 18:57:53.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.697 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 18:57:53.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:53.698 | INFO | Uploaded 36GiB to GPU in 2.4968s (14.4185GiB/s) | 20260507 18:57:53.698 | INFO | Downloaded 34GiB to GPU in 2.46077s (13.8168GiB/s) | 20260507 18:57:58.818 | INFO | Analyzing chunk frames = [1827840,1828352) | 20260507 18:57:58.818 | INFO | MemInfo: { | 20260507 18:57:58.818 | INFO | "gpu" : " 17 GB", | 20260507 18:57:58.818 | INFO | "huge" : "", | 20260507 18:57:58.818 | INFO | "huge_shared" : " 29 GB", | 20260507 18:57:58.819 | INFO | "malloc" : " 14 GB", | 20260507 18:57:58.819 | INFO | "pinned" : " 29 GB" | 20260507 18:57:58.819 | INFO | } | 20260507 18:57:58.819 | INFO | Perf report for the last 5.12131 seconds: | 20260507 18:57:58.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:57:58.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | ANALYSIS : 68.1%, 3.0, 105.2ms, 0 | 20260507 18:57:58.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:57:58.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:57:58.819 | INFO | Uploaded 36GiB to GPU in 2.52776s (14.2419GiB/s) | 20260507 18:57:58.819 | INFO | Downloaded 34GiB to GPU in 2.47205s (13.7538GiB/s) | 20260507 18:58:03.958 | INFO | Analyzing chunk frames = [1828352,1828864) | 20260507 18:58:03.958 | INFO | MemInfo: { | 20260507 18:58:03.958 | INFO | "gpu" : " 17 GB", | 20260507 18:58:03.958 | INFO | "huge" : "", | 20260507 18:58:03.958 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:03.958 | INFO | "malloc" : " 14 GB", | 20260507 18:58:03.958 | INFO | "pinned" : " 29 GB" | 20260507 18:58:03.958 | INFO | } | 20260507 18:58:03.958 | INFO | Perf report for the last 5.13946 seconds: | 20260507 18:58:03.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:03.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260507 18:58:03.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 18:58:03.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:03.958 | INFO | Uploaded 36GiB to GPU in 2.4878s (14.4706GiB/s) | 20260507 18:58:03.958 | INFO | Downloaded 34GiB to GPU in 2.37033s (14.344GiB/s) | 20260507 18:58:09.097 | INFO | Analyzing chunk frames = [1828864,1829376) | 20260507 18:58:09.097 | INFO | MemInfo: { | 20260507 18:58:09.097 | INFO | "gpu" : " 17 GB", | 20260507 18:58:09.097 | INFO | "huge" : "", | 20260507 18:58:09.097 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:09.097 | INFO | "malloc" : " 14 GB", | 20260507 18:58:09.097 | INFO | "pinned" : " 29 GB" | 20260507 18:58:09.097 | INFO | } | 20260507 18:58:09.097 | INFO | Perf report for the last 5.1391 seconds: | 20260507 18:58:09.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:09.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.097 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 18:58:09.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.097 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 18:58:09.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:09.098 | INFO | Uploaded 36GiB to GPU in 2.3252s (15.4826GiB/s) | 20260507 18:58:09.098 | INFO | Downloaded 34GiB to GPU in 2.49264s (13.6401GiB/s) | 20260507 18:58:14.236 | INFO | Analyzing chunk frames = [1829376,1829888) | 20260507 18:58:14.236 | INFO | MemInfo: { | 20260507 18:58:14.236 | INFO | "gpu" : " 17 GB", | 20260507 18:58:14.236 | INFO | "huge" : "", | 20260507 18:58:14.236 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:14.236 | INFO | "malloc" : " 14 GB", | 20260507 18:58:14.237 | INFO | "pinned" : " 29 GB" | 20260507 18:58:14.237 | INFO | } | 20260507 18:58:14.237 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:58:14.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:14.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 18:58:14.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:58:14.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:14.237 | INFO | Uploaded 36GiB to GPU in 2.4693s (14.5791GiB/s) | 20260507 18:58:14.237 | INFO | Downloaded 34GiB to GPU in 2.50991s (13.5463GiB/s) | 20260507 18:58:19.376 | INFO | Analyzing chunk frames = [1829888,1830400) | 20260507 18:58:19.376 | INFO | MemInfo: { | 20260507 18:58:19.376 | INFO | "gpu" : " 17 GB", | 20260507 18:58:19.376 | INFO | "huge" : "", | 20260507 18:58:19.376 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:19.376 | INFO | "malloc" : " 14 GB", | 20260507 18:58:19.376 | INFO | "pinned" : " 29 GB" | 20260507 18:58:19.376 | INFO | } | 20260507 18:58:19.376 | INFO | Perf report for the last 5.13933 seconds: | 20260507 18:58:19.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:19.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:58:19.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 18:58:19.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:19.376 | INFO | Uploaded 36GiB to GPU in 2.50814s (14.3532GiB/s) | 20260507 18:58:19.376 | INFO | Downloaded 34GiB to GPU in 2.44518s (13.9049GiB/s) | 20260507 18:58:24.515 | INFO | Analyzing chunk frames = [1830400,1830912) | 20260507 18:58:24.515 | INFO | MemInfo: { | 20260507 18:58:24.515 | INFO | "gpu" : " 17 GB", | 20260507 18:58:24.515 | INFO | "huge" : "", | 20260507 18:58:24.515 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:24.515 | INFO | "malloc" : " 14 GB", | 20260507 18:58:24.515 | INFO | "pinned" : " 29 GB" | 20260507 18:58:24.515 | INFO | } | 20260507 18:58:24.515 | INFO | Perf report for the last 5.13905 seconds: | 20260507 18:58:24.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:24.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 18:58:24.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 18:58:24.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:24.515 | INFO | Uploaded 36GiB to GPU in 2.49825s (14.4101GiB/s) | 20260507 18:58:24.515 | INFO | Downloaded 34GiB to GPU in 2.43676s (13.9529GiB/s) | 20260507 18:58:24.613 | INFO | Starting a round of Baz IO | 20260507 18:58:28.108 | INFO | Populating RTMetrics report | 20260507 18:58:28.108 | INFO | RTMetrics report populated | 20260507 18:58:28.108 | INFO | Opening tmp RTMetrics json file | 20260507 18:58:28.114 | INFO | Serializing RTMetrics to json | 20260507 18:58:28.126 | INFO | Serialization done, swapping tmp file now... | 20260507 18:58:28.127 | INFO | RTMetrics done | 20260507 18:58:28.191 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.578 seconds | 20260507 18:58:28.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.95% / 0.01% / 54.04% | 20260507 18:58:28.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.66117% / 3.17891e-05% / 4.37101e-05% | 20260507 18:58:28.191 | INFO | Baz IO overall: | 20260507 18:58:28.191 | INFO | BytesWritten : 823301925702 | 20260507 18:58:28.191 | INFO | NumEvents : 399172856585 | 20260507 18:58:29.654 | INFO | Analyzing chunk frames = [1830912,1831424) | 20260507 18:58:29.654 | INFO | MemInfo: { | 20260507 18:58:29.654 | INFO | "gpu" : " 17 GB", | 20260507 18:58:29.654 | INFO | "huge" : "", | 20260507 18:58:29.655 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:29.655 | INFO | "malloc" : " 14 GB", | 20260507 18:58:29.655 | INFO | "pinned" : " 29 GB" | 20260507 18:58:29.655 | INFO | } | 20260507 18:58:29.655 | INFO | Perf report for the last 5.13962 seconds: | 20260507 18:58:29.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:29.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:29.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:29.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:29.655 | INFO | ANALYSIS : 69.5%, 3.0, 108.2ms, 0 | 20260507 18:58:29.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:29.655 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 18:58:29.655 | INFO | PRE_HQ : 66.6%, 1.7, 59.0ms, 0 | 20260507 18:58:29.655 | INFO | BAZWRITER : 68.8%, 2.9, 102.8ms, 0 | 20260507 18:58:29.655 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:58:29.655 | INFO | Uploaded 36GiB to GPU in 2.56022s (14.0613GiB/s) | 20260507 18:58:29.655 | INFO | Downloaded 38GiB to GPU in 2.72517s (13.9441GiB/s) | 20260507 18:58:34.793 | INFO | Analyzing chunk frames = [1831424,1831936) | 20260507 18:58:34.793 | INFO | MemInfo: { | 20260507 18:58:34.793 | INFO | "gpu" : " 17 GB", | 20260507 18:58:34.793 | INFO | "huge" : "", | 20260507 18:58:34.794 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:34.794 | INFO | "malloc" : " 14 GB", | 20260507 18:58:34.794 | INFO | "pinned" : " 29 GB" | 20260507 18:58:34.794 | INFO | } | 20260507 18:58:34.794 | INFO | Perf report for the last 5.13889 seconds: | 20260507 18:58:34.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:34.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:58:34.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:58:34.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:34.794 | INFO | Uploaded 36GiB to GPU in 2.52703s (14.246GiB/s) | 20260507 18:58:34.794 | INFO | Downloaded 34GiB to GPU in 2.45431s (13.8532GiB/s) | 20260507 18:58:39.933 | INFO | Analyzing chunk frames = [1831936,1832448) | 20260507 18:58:39.933 | INFO | MemInfo: { | 20260507 18:58:39.933 | INFO | "gpu" : " 17 GB", | 20260507 18:58:39.933 | INFO | "huge" : "", | 20260507 18:58:39.933 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:39.933 | INFO | "malloc" : " 14 GB", | 20260507 18:58:39.933 | INFO | "pinned" : " 29 GB" | 20260507 18:58:39.933 | INFO | } | 20260507 18:58:39.933 | INFO | Perf report for the last 5.13941 seconds: | 20260507 18:58:39.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:39.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 18:58:39.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 18:58:39.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:39.933 | INFO | Uploaded 36GiB to GPU in 2.5379s (14.185GiB/s) | 20260507 18:58:39.933 | INFO | Downloaded 34GiB to GPU in 2.44681s (13.8956GiB/s) | 20260507 18:58:45.072 | INFO | Analyzing chunk frames = [1832448,1832960) | 20260507 18:58:45.072 | INFO | MemInfo: { | 20260507 18:58:45.072 | INFO | "gpu" : " 17 GB", | 20260507 18:58:45.072 | INFO | "huge" : "", | 20260507 18:58:45.072 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:45.072 | INFO | "malloc" : " 14 GB", | 20260507 18:58:45.072 | INFO | "pinned" : " 29 GB" | 20260507 18:58:45.072 | INFO | } | 20260507 18:58:45.072 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:58:45.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:45.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.073 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 18:58:45.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.073 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:58:45.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:45.073 | INFO | Uploaded 36GiB to GPU in 2.47809s (14.5273GiB/s) | 20260507 18:58:45.073 | INFO | Downloaded 34GiB to GPU in 2.3726s (14.3303GiB/s) | 20260507 18:58:50.212 | INFO | Analyzing chunk frames = [1832960,1833472) | 20260507 18:58:50.212 | INFO | MemInfo: { | 20260507 18:58:50.212 | INFO | "gpu" : " 17 GB", | 20260507 18:58:50.212 | INFO | "huge" : "", | 20260507 18:58:50.212 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:50.212 | INFO | "malloc" : " 14 GB", | 20260507 18:58:50.212 | INFO | "pinned" : " 29 GB" | 20260507 18:58:50.212 | INFO | } | 20260507 18:58:50.212 | INFO | Perf report for the last 5.1397 seconds: | 20260507 18:58:50.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:50.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 18:58:50.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 18:58:50.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:50.212 | INFO | Uploaded 36GiB to GPU in 2.37153s (15.18GiB/s) | 20260507 18:58:50.213 | INFO | Downloaded 34GiB to GPU in 2.48265s (13.6951GiB/s) | 20260507 18:58:55.351 | INFO | Analyzing chunk frames = [1833472,1833984) | 20260507 18:58:55.351 | INFO | MemInfo: { | 20260507 18:58:55.351 | INFO | "gpu" : " 17 GB", | 20260507 18:58:55.351 | INFO | "huge" : "", | 20260507 18:58:55.351 | INFO | "huge_shared" : " 29 GB", | 20260507 18:58:55.351 | INFO | "malloc" : " 14 GB", | 20260507 18:58:55.351 | INFO | "pinned" : " 29 GB" | 20260507 18:58:55.351 | INFO | } | 20260507 18:58:55.351 | INFO | Perf report for the last 5.13888 seconds: | 20260507 18:58:55.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:58:55.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.351 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 18:58:55.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.351 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 18:58:55.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:58:55.352 | INFO | Uploaded 36GiB to GPU in 2.40397s (14.9752GiB/s) | 20260507 18:58:55.352 | INFO | Downloaded 34GiB to GPU in 2.52671s (13.4562GiB/s) | 20260507 18:59:00.490 | INFO | Analyzing chunk frames = [1833984,1834496) | 20260507 18:59:00.490 | INFO | MemInfo: { | 20260507 18:59:00.490 | INFO | "gpu" : " 17 GB", | 20260507 18:59:00.490 | INFO | "huge" : "", | 20260507 18:59:00.490 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:00.490 | INFO | "malloc" : " 14 GB", | 20260507 18:59:00.490 | INFO | "pinned" : " 29 GB" | 20260507 18:59:00.490 | INFO | } | 20260507 18:59:00.490 | INFO | Perf report for the last 5.13935 seconds: | 20260507 18:59:00.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:00.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.490 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 18:59:00.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.491 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 18:59:00.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:00.491 | INFO | Uploaded 36GiB to GPU in 2.4554s (14.6615GiB/s) | 20260507 18:59:00.491 | INFO | Downloaded 34GiB to GPU in 2.44975s (13.879GiB/s) | 20260507 18:59:05.629 | INFO | Analyzing chunk frames = [1834496,1835008) | 20260507 18:59:05.629 | INFO | MemInfo: { | 20260507 18:59:05.629 | INFO | "gpu" : " 17 GB", | 20260507 18:59:05.629 | INFO | "huge" : "", | 20260507 18:59:05.629 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:05.629 | INFO | "malloc" : " 14 GB", | 20260507 18:59:05.630 | INFO | "pinned" : " 29 GB" | 20260507 18:59:05.630 | INFO | } | 20260507 18:59:05.630 | INFO | Perf report for the last 5.13926 seconds: | 20260507 18:59:05.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:05.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 18:59:05.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:59:05.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:05.630 | INFO | Uploaded 36GiB to GPU in 2.42622s (14.8379GiB/s) | 20260507 18:59:05.630 | INFO | Downloaded 34GiB to GPU in 2.43867s (13.942GiB/s) | 20260507 18:59:05.630 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 18:59:05.630 | INFO | Total mem usage: 16.4 GiB | 20260507 18:59:05.630 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 18:59:05.630 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 18:59:05.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 18:59:05.630 | INFO | High Water Breakdown: | 20260507 18:59:05.630 | INFO | 59.2% : 8.54 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 18:59:05.630 | INFO | 40.7% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 18:59:05.630 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 18:59:05.630 | INFO | ------------------------------------------ | 20260507 18:59:05.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:59:05.630 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 18:59:05.630 | INFO | Total mem usage: 26.5 GiB | 20260507 18:59:05.630 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 18:59:05.630 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 18:59:05.630 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 18:59:05.630 | INFO | High Water Breakdown: | 20260507 18:59:05.630 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:59:05.630 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:59:05.630 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:59:05.630 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:59:05.630 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:59:05.630 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:59:05.630 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 18:59:05.630 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 18:59:05.630 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:59:05.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 18:59:05.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 18:59:05.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 18:59:05.631 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 18:59:05.631 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 18:59:05.631 | INFO | ------------------------------------------ | 20260507 18:59:05.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:59:05.631 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 18:59:05.631 | INFO | Total mem usage: 27.8 GiB | 20260507 18:59:05.631 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 18:59:05.631 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 18:59:05.631 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 18:59:05.631 | INFO | High Water Breakdown: | 20260507 18:59:05.631 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 18:59:05.631 | INFO | ------------------------------------------ | 20260507 18:59:05.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:59:05.631 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 18:59:05.631 | INFO | Total mem usage: 15.7 GiB | 20260507 18:59:05.631 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 18:59:05.631 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 18:59:05.631 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 18:59:05.631 | INFO | High Water Breakdown: | 20260507 18:59:05.631 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 18:59:05.631 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 18:59:05.631 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 18:59:05.631 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 18:59:05.631 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 18:59:05.631 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 18:59:05.631 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 18:59:05.631 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 18:59:05.631 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 18:59:05.631 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 18:59:05.631 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 18:59:05.631 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 18:59:05.631 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 18:59:05.631 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 18:59:05.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 18:59:05.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 18:59:05.631 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 18:59:05.631 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 18:59:05.631 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 18:59:05.631 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 18:59:05.631 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 18:59:05.631 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 18:59:05.631 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 18:59:05.631 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 18:59:05.631 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 18:59:05.631 | INFO | ------------------------------------------ | 20260507 18:59:05.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 18:59:05.631 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 18:59:05.631 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:59:05.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:59:05.631 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.64e+01, 1.43e+01 / 2.12e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 0.29%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.12e-02 / 4.13e-03 | 20260507 18:59:05.631 | INFO | PerfReport: 12.09%, Baseline -- 4.03e+00 / 4.51e+00, 6.05e-01 / 2.13e-01 | 20260507 18:59:05.631 | INFO | PerfReport: 3.42%, Histogram -- 1.14e+00 / 1.50e+00, 2.13e-01 / 1.11e-01 | 20260507 18:59:05.631 | INFO | PerfReport: 15.69%, DME -- 4.18e+01 / 4.49e+01, 4.01e+00 / 1.81e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.86e+00 / 1.58e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.49e+00 / 2.83e+00, 4.27e-01 / 2.26e-01 | 20260507 18:59:05.631 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.41e+00, 3.11e-01 / 1.62e-01 | 20260507 18:59:05.631 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.84e-01 / 4.12e-01, 6.13e-02 / 3.36e-02 | 20260507 18:59:05.631 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 18:59:05.631 | INFO | PerfReport: Gathered 1536 samples. | 20260507 18:59:05.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 18:59:05.631 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 24.06%, Upload -- 2.58e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 24.18%, Download -- 2.59e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 31.07%, Compute -- 3.33e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 1.68%, LockUpload -- 1.80e+00 / 0.00e+00, 4.71e+00 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 2.46%, LockDownload -- 2.64e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 16.53%, LockCompute -- 1.77e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 18:59:05.631 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 18:59:05.727 | INFO | Starting a round of Baz IO | 20260507 18:59:09.227 | INFO | Populating RTMetrics report | 20260507 18:59:09.227 | INFO | RTMetrics report populated | 20260507 18:59:09.227 | INFO | Opening tmp RTMetrics json file | 20260507 18:59:09.239 | INFO | Serializing RTMetrics to json | 20260507 18:59:09.246 | INFO | Serialization done, swapping tmp file now... | 20260507 18:59:09.255 | INFO | RTMetrics done | 20260507 18:59:09.339 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.612 seconds | 20260507 18:59:09.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.93% / 0.01% / 54.05% | 20260507 18:59:09.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.65638% / 1.58946e-05% / 4.76837e-05% | 20260507 18:59:09.339 | INFO | Baz IO overall: | 20260507 18:59:09.339 | INFO | BytesWritten : 825236637753 | 20260507 18:59:09.339 | INFO | NumEvents : 400111500583 | 20260507 18:59:10.769 | INFO | Analyzing chunk frames = [1835008,1835520) | 20260507 18:59:10.769 | INFO | MemInfo: { | 20260507 18:59:10.769 | INFO | "gpu" : " 17 GB", | 20260507 18:59:10.769 | INFO | "huge" : "", | 20260507 18:59:10.769 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:10.769 | INFO | "malloc" : " 14 GB", | 20260507 18:59:10.769 | INFO | "pinned" : " 29 GB" | 20260507 18:59:10.769 | INFO | } | 20260507 18:59:10.769 | INFO | Perf report for the last 5.1395 seconds: | 20260507 18:59:10.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:10.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:10.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:10.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:10.769 | INFO | ANALYSIS : 69.2%, 3.0, 107.9ms, 0 | 20260507 18:59:10.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:10.769 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 18:59:10.769 | INFO | PRE_HQ : 66.4%, 1.7, 59.1ms, 0 | 20260507 18:59:10.769 | INFO | BAZWRITER : 69.4%, 3.1, 111.4ms, 0 | 20260507 18:59:10.769 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 18:59:10.770 | INFO | Uploaded 36GiB to GPU in 2.48726s (14.4738GiB/s) | 20260507 18:59:10.770 | INFO | Downloaded 38GiB to GPU in 2.72369s (13.9517GiB/s) | 20260507 18:59:15.908 | INFO | Analyzing chunk frames = [1835520,1836032) | 20260507 18:59:15.908 | INFO | MemInfo: { | 20260507 18:59:15.908 | INFO | "gpu" : " 17 GB", | 20260507 18:59:15.908 | INFO | "huge" : "", | 20260507 18:59:15.908 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:15.909 | INFO | "malloc" : " 14 GB", | 20260507 18:59:15.909 | INFO | "pinned" : " 29 GB" | 20260507 18:59:15.909 | INFO | } | 20260507 18:59:15.909 | INFO | Perf report for the last 5.1394 seconds: | 20260507 18:59:15.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:15.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 18:59:15.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:59:15.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:15.909 | INFO | Uploaded 36GiB to GPU in 2.51051s (14.3397GiB/s) | 20260507 18:59:15.909 | INFO | Downloaded 34GiB to GPU in 2.45831s (13.8307GiB/s) | 20260507 18:59:21.066 | INFO | Analyzing chunk frames = [1836032,1836544) | 20260507 18:59:21.066 | INFO | MemInfo: { | 20260507 18:59:21.066 | INFO | "gpu" : " 17 GB", | 20260507 18:59:21.066 | INFO | "huge" : "", | 20260507 18:59:21.066 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:21.066 | INFO | "malloc" : " 14 GB", | 20260507 18:59:21.066 | INFO | "pinned" : " 29 GB" | 20260507 18:59:21.066 | INFO | } | 20260507 18:59:21.066 | INFO | Perf report for the last 5.15762 seconds: | 20260507 18:59:21.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:21.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.066 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 18:59:21.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.066 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 18:59:21.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:21.067 | INFO | Uploaded 36GiB to GPU in 2.52682s (14.2472GiB/s) | 20260507 18:59:21.067 | INFO | Downloaded 34GiB to GPU in 2.44317s (13.9163GiB/s) | 20260507 18:59:26.205 | INFO | Analyzing chunk frames = [1836544,1837056) | 20260507 18:59:26.205 | INFO | MemInfo: { | 20260507 18:59:26.205 | INFO | "gpu" : " 17 GB", | 20260507 18:59:26.205 | INFO | "huge" : "", | 20260507 18:59:26.205 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:26.205 | INFO | "malloc" : " 14 GB", | 20260507 18:59:26.205 | INFO | "pinned" : " 29 GB" | 20260507 18:59:26.205 | INFO | } | 20260507 18:59:26.205 | INFO | Perf report for the last 5.13914 seconds: | 20260507 18:59:26.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:26.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.205 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 18:59:26.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.206 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 18:59:26.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:26.206 | INFO | Uploaded 36GiB to GPU in 2.46668s (14.5945GiB/s) | 20260507 18:59:26.206 | INFO | Downloaded 34GiB to GPU in 2.39796s (14.1787GiB/s) | 20260507 18:59:31.343 | INFO | Analyzing chunk frames = [1837056,1837568) | 20260507 18:59:31.343 | INFO | MemInfo: { | 20260507 18:59:31.343 | INFO | "gpu" : " 17 GB", | 20260507 18:59:31.343 | INFO | "huge" : "", | 20260507 18:59:31.344 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:31.344 | INFO | "malloc" : " 14 GB", | 20260507 18:59:31.344 | INFO | "pinned" : " 29 GB" | 20260507 18:59:31.344 | INFO | } | 20260507 18:59:31.344 | INFO | Perf report for the last 5.13822 seconds: | 20260507 18:59:31.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:31.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 18:59:31.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 18:59:31.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:31.344 | INFO | Uploaded 36GiB to GPU in 2.39709s (15.0182GiB/s) | 20260507 18:59:31.344 | INFO | Downloaded 34GiB to GPU in 2.48821s (13.6644GiB/s) | 20260507 18:59:36.483 | INFO | Analyzing chunk frames = [1837568,1838080) | 20260507 18:59:36.483 | INFO | MemInfo: { | 20260507 18:59:36.483 | INFO | "gpu" : " 17 GB", | 20260507 18:59:36.483 | INFO | "huge" : "", | 20260507 18:59:36.483 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:36.483 | INFO | "malloc" : " 14 GB", | 20260507 18:59:36.483 | INFO | "pinned" : " 29 GB" | 20260507 18:59:36.483 | INFO | } | 20260507 18:59:36.483 | INFO | Perf report for the last 5.13947 seconds: | 20260507 18:59:36.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:36.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.483 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 18:59:36.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.483 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 18:59:36.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:36.484 | INFO | Uploaded 36GiB to GPU in 2.49423s (14.4333GiB/s) | 20260507 18:59:36.484 | INFO | Downloaded 34GiB to GPU in 2.40901s (14.1137GiB/s) | 20260507 18:59:41.622 | INFO | Analyzing chunk frames = [1838080,1838592) | 20260507 18:59:41.622 | INFO | MemInfo: { | 20260507 18:59:41.622 | INFO | "gpu" : " 17 GB", | 20260507 18:59:41.622 | INFO | "huge" : "", | 20260507 18:59:41.622 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:41.622 | INFO | "malloc" : " 14 GB", | 20260507 18:59:41.622 | INFO | "pinned" : " 29 GB" | 20260507 18:59:41.622 | INFO | } | 20260507 18:59:41.622 | INFO | Perf report for the last 5.13861 seconds: | 20260507 18:59:41.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:41.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 18:59:41.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:59:41.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:41.622 | INFO | Uploaded 36GiB to GPU in 2.53336s (14.2104GiB/s) | 20260507 18:59:41.622 | INFO | Downloaded 34GiB to GPU in 2.43632s (13.9555GiB/s) | 20260507 18:59:46.761 | INFO | Analyzing chunk frames = [1838592,1839104) | 20260507 18:59:46.761 | INFO | MemInfo: { | 20260507 18:59:46.761 | INFO | "gpu" : " 17 GB", | 20260507 18:59:46.761 | INFO | "huge" : "", | 20260507 18:59:46.761 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:46.761 | INFO | "malloc" : " 14 GB", | 20260507 18:59:46.761 | INFO | "pinned" : " 29 GB" | 20260507 18:59:46.761 | INFO | } | 20260507 18:59:46.761 | INFO | Perf report for the last 5.13955 seconds: | 20260507 18:59:46.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:46.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 18:59:46.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 18:59:46.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:46.762 | INFO | Uploaded 36GiB to GPU in 2.53087s (14.2244GiB/s) | 20260507 18:59:46.762 | INFO | Downloaded 34GiB to GPU in 2.41964s (14.0517GiB/s) | 20260507 18:59:46.858 | INFO | Starting a round of Baz IO | 20260507 18:59:50.333 | INFO | Populating RTMetrics report | 20260507 18:59:50.334 | INFO | RTMetrics report populated | 20260507 18:59:50.334 | INFO | Opening tmp RTMetrics json file | 20260507 18:59:50.343 | INFO | Serializing RTMetrics to json | 20260507 18:59:50.352 | INFO | Serialization done, swapping tmp file now... | 20260507 18:59:50.352 | INFO | RTMetrics done | 20260507 18:59:50.412 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.554 seconds | 20260507 18:59:50.413 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.92% / 0.01% / 54.07% | 20260507 18:59:50.413 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.64444% / 3.97364e-06% / 1.98682e-05% | 20260507 18:59:50.413 | INFO | Baz IO overall: | 20260507 18:59:50.413 | INFO | BytesWritten : 827170658300 | 20260507 18:59:50.413 | INFO | NumEvents : 401049805712 | 20260507 18:59:51.901 | INFO | Analyzing chunk frames = [1839104,1839616) | 20260507 18:59:51.901 | INFO | MemInfo: { | 20260507 18:59:51.901 | INFO | "gpu" : " 17 GB", | 20260507 18:59:51.901 | INFO | "huge" : "", | 20260507 18:59:51.901 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:51.901 | INFO | "malloc" : " 14 GB", | 20260507 18:59:51.901 | INFO | "pinned" : " 29 GB" | 20260507 18:59:51.901 | INFO | } | 20260507 18:59:51.901 | INFO | Perf report for the last 5.14005 seconds: | 20260507 18:59:51.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:51.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:51.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:51.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:51.901 | INFO | ANALYSIS : 69.0%, 3.0, 107.5ms, 0 | 20260507 18:59:51.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:51.902 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 18:59:51.902 | INFO | PRE_HQ : 66.4%, 1.7, 59.1ms, 0 | 20260507 18:59:51.902 | INFO | BAZWRITER : 68.3%, 2.9, 102.2ms, 0 | 20260507 18:59:51.902 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 18:59:51.902 | INFO | Uploaded 36GiB to GPU in 2.55596s (14.0848GiB/s) | 20260507 18:59:51.902 | INFO | Downloaded 38GiB to GPU in 2.69603s (14.0948GiB/s) | 20260507 18:59:57.039 | INFO | Analyzing chunk frames = [1839616,1840128) | 20260507 18:59:57.039 | INFO | MemInfo: { | 20260507 18:59:57.039 | INFO | "gpu" : " 17 GB", | 20260507 18:59:57.039 | INFO | "huge" : "", | 20260507 18:59:57.039 | INFO | "huge_shared" : " 29 GB", | 20260507 18:59:57.040 | INFO | "malloc" : " 14 GB", | 20260507 18:59:57.040 | INFO | "pinned" : " 29 GB" | 20260507 18:59:57.040 | INFO | } | 20260507 18:59:57.040 | INFO | Perf report for the last 5.13819 seconds: | 20260507 18:59:57.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 18:59:57.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 18:59:57.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 18:59:57.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 18:59:57.040 | INFO | Uploaded 36GiB to GPU in 2.54498s (14.1455GiB/s) | 20260507 18:59:57.040 | INFO | Downloaded 34GiB to GPU in 2.48202s (13.6985GiB/s) | 20260507 19:00:02.181 | INFO | Analyzing chunk frames = [1840128,1840640) | 20260507 19:00:02.181 | INFO | MemInfo: { | 20260507 19:00:02.181 | INFO | "gpu" : " 17 GB", | 20260507 19:00:02.181 | INFO | "huge" : "", | 20260507 19:00:02.181 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:02.182 | INFO | "malloc" : " 14 GB", | 20260507 19:00:02.182 | INFO | "pinned" : " 29 GB" | 20260507 19:00:02.182 | INFO | } | 20260507 19:00:02.182 | INFO | Perf report for the last 5.14203 seconds: | 20260507 19:00:02.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:02.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 19:00:02.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:00:02.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:02.182 | INFO | Uploaded 36GiB to GPU in 2.55022s (14.1164GiB/s) | 20260507 19:00:02.182 | INFO | Downloaded 34GiB to GPU in 2.45232s (13.8644GiB/s) | 20260507 19:00:07.319 | INFO | Analyzing chunk frames = [1840640,1841152) | 20260507 19:00:07.319 | INFO | MemInfo: { | 20260507 19:00:07.319 | INFO | "gpu" : " 17 GB", | 20260507 19:00:07.319 | INFO | "huge" : "", | 20260507 19:00:07.319 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:07.319 | INFO | "malloc" : " 14 GB", | 20260507 19:00:07.319 | INFO | "pinned" : " 29 GB" | 20260507 19:00:07.319 | INFO | } | 20260507 19:00:07.319 | INFO | Perf report for the last 5.13716 seconds: | 20260507 19:00:07.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:07.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:00:07.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:00:07.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:07.319 | INFO | Uploaded 36GiB to GPU in 2.48631s (14.4793GiB/s) | 20260507 19:00:07.320 | INFO | Downloaded 34GiB to GPU in 2.38964s (14.2281GiB/s) | 20260507 19:00:12.459 | INFO | Analyzing chunk frames = [1841152,1841664) | 20260507 19:00:12.459 | INFO | MemInfo: { | 20260507 19:00:12.459 | INFO | "gpu" : " 17 GB", | 20260507 19:00:12.459 | INFO | "huge" : "", | 20260507 19:00:12.459 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:12.459 | INFO | "malloc" : " 14 GB", | 20260507 19:00:12.459 | INFO | "pinned" : " 29 GB" | 20260507 19:00:12.459 | INFO | } | 20260507 19:00:12.460 | INFO | Perf report for the last 5.14067 seconds: | 20260507 19:00:12.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:12.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:00:12.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 19:00:12.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:12.460 | INFO | Uploaded 36GiB to GPU in 2.37352s (15.1673GiB/s) | 20260507 19:00:12.460 | INFO | Downloaded 34GiB to GPU in 2.48521s (13.681GiB/s) | 20260507 19:00:17.597 | INFO | Analyzing chunk frames = [1841664,1842176) | 20260507 19:00:17.597 | INFO | MemInfo: { | 20260507 19:00:17.597 | INFO | "gpu" : " 17 GB", | 20260507 19:00:17.597 | INFO | "huge" : "", | 20260507 19:00:17.597 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:17.598 | INFO | "malloc" : " 14 GB", | 20260507 19:00:17.598 | INFO | "pinned" : " 29 GB" | 20260507 19:00:17.598 | INFO | } | 20260507 19:00:17.598 | INFO | Perf report for the last 5.13809 seconds: | 20260507 19:00:17.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:17.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:00:17.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:00:17.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:17.598 | INFO | Uploaded 36GiB to GPU in 2.48405s (14.4925GiB/s) | 20260507 19:00:17.598 | INFO | Downloaded 34GiB to GPU in 2.41466s (14.0806GiB/s) | 20260507 19:00:22.737 | INFO | Analyzing chunk frames = [1842176,1842688) | 20260507 19:00:22.737 | INFO | MemInfo: { | 20260507 19:00:22.737 | INFO | "gpu" : " 17 GB", | 20260507 19:00:22.737 | INFO | "huge" : "", | 20260507 19:00:22.737 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:22.737 | INFO | "malloc" : " 14 GB", | 20260507 19:00:22.737 | INFO | "pinned" : " 29 GB" | 20260507 19:00:22.737 | INFO | } | 20260507 19:00:22.737 | INFO | Perf report for the last 5.13949 seconds: | 20260507 19:00:22.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:22.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.737 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:00:22.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.737 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:00:22.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:22.738 | INFO | Uploaded 36GiB to GPU in 2.50812s (14.3534GiB/s) | 20260507 19:00:22.738 | INFO | Downloaded 34GiB to GPU in 2.44472s (13.9075GiB/s) | 20260507 19:00:27.876 | INFO | Analyzing chunk frames = [1842688,1843200) | 20260507 19:00:27.876 | INFO | MemInfo: { | 20260507 19:00:27.876 | INFO | "gpu" : " 17 GB", | 20260507 19:00:27.876 | INFO | "huge" : "", | 20260507 19:00:27.876 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:27.876 | INFO | "malloc" : " 14 GB", | 20260507 19:00:27.876 | INFO | "pinned" : " 29 GB" | 20260507 19:00:27.876 | INFO | } | 20260507 19:00:27.876 | INFO | Perf report for the last 5.13928 seconds: | 20260507 19:00:27.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:27.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.876 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:00:27.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.877 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:00:27.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:27.877 | INFO | Uploaded 36GiB to GPU in 2.45639s (14.6557GiB/s) | 20260507 19:00:27.877 | INFO | Downloaded 34GiB to GPU in 2.42564s (14.0169GiB/s) | 20260507 19:00:27.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:00:27.877 | INFO | Total mem usage: 16.4 GiB | 20260507 19:00:27.877 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:00:27.877 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 19:00:27.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:00:27.877 | INFO | High Water Breakdown: | 20260507 19:00:27.877 | INFO | 59.3% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:00:27.877 | INFO | 40.5% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:00:27.877 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:00:27.877 | INFO | ------------------------------------------ | 20260507 19:00:27.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:00:27.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:00:27.877 | INFO | Total mem usage: 26.5 GiB | 20260507 19:00:27.877 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:00:27.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:00:27.877 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:00:27.877 | INFO | High Water Breakdown: | 20260507 19:00:27.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:00:27.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:00:27.877 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:00:27.877 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:00:27.877 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:00:27.877 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:00:27.877 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:00:27.877 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:00:27.877 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:00:27.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:00:27.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:00:27.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:00:27.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:00:27.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:00:27.877 | INFO | ------------------------------------------ | 20260507 19:00:27.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:00:27.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:00:27.877 | INFO | Total mem usage: 27.8 GiB | 20260507 19:00:27.877 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:00:27.877 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:00:27.877 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:00:27.877 | INFO | High Water Breakdown: | 20260507 19:00:27.877 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:00:27.877 | INFO | ------------------------------------------ | 20260507 19:00:27.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:00:27.877 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:00:27.877 | INFO | Total mem usage: 15.7 GiB | 20260507 19:00:27.877 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:00:27.877 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:00:27.877 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:00:27.877 | INFO | High Water Breakdown: | 20260507 19:00:27.877 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:00:27.877 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:00:27.877 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:00:27.877 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:00:27.877 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:00:27.877 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:00:27.877 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:00:27.877 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:00:27.877 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:00:27.877 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:00:27.877 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:00:27.877 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:00:27.877 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:00:27.877 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:00:27.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:00:27.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:00:27.877 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:00:27.877 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:00:27.877 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:00:27.877 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:00:27.877 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:00:27.877 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:00:27.877 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:00:27.877 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:00:27.877 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:00:27.877 | INFO | ------------------------------------------ | 20260507 19:00:27.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:00:27.877 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:00:27.877 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:00:27.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:00:27.877 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.41e+01 / 1.93e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.35e-02 / 5.03e-03 | 20260507 19:00:27.877 | INFO | PerfReport: 12.07%, Baseline -- 4.01e+00 / 4.46e+00, 6.04e-01 / 2.81e-01 | 20260507 19:00:27.877 | INFO | PerfReport: 3.41%, Histogram -- 1.13e+00 / 1.49e+00, 2.11e-01 / 1.23e-01 | 20260507 19:00:27.877 | INFO | PerfReport: 15.67%, DME -- 4.16e+01 / 4.45e+01, 3.74e+00 / 1.62e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.82e+00 / 1.87e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.39e-01 / 2.18e-01 | 20260507 19:00:27.877 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.42e+00, 3.07e-01 / 1.68e-01 | 20260507 19:00:27.877 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.15e-01, 6.13e-02 / 3.68e-02 | 20260507 19:00:27.877 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:00:27.877 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:00:27.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:00:27.877 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 24.26%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 24.05%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 31.02%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 1.91%, LockUpload -- 2.05e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 2.38%, LockDownload -- 2.55e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 16.37%, LockCompute -- 1.75e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:00:27.877 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.12e-02 / 0.00e+00 | 20260507 19:00:27.973 | INFO | Starting a round of Baz IO | 20260507 19:00:31.475 | INFO | Populating RTMetrics report | 20260507 19:00:31.475 | INFO | RTMetrics report populated | 20260507 19:00:31.475 | INFO | Opening tmp RTMetrics json file | 20260507 19:00:31.491 | INFO | Serializing RTMetrics to json | 20260507 19:00:31.492 | INFO | Serialization done, swapping tmp file now... | 20260507 19:00:31.493 | INFO | RTMetrics done | 20260507 19:00:31.547 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.573 seconds | 20260507 19:00:31.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.90% / 0.01% / 54.08% | 20260507 19:00:31.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.63652% / 3.17891e-05% / 3.97364e-05% | 20260507 19:00:31.547 | INFO | Baz IO overall: | 20260507 19:00:31.547 | INFO | BytesWritten : 829101645627 | 20260507 19:00:31.547 | INFO | NumEvents : 401985993603 | 20260507 19:00:33.016 | INFO | Analyzing chunk frames = [1843200,1843712) | 20260507 19:00:33.016 | INFO | MemInfo: { | 20260507 19:00:33.016 | INFO | "gpu" : " 17 GB", | 20260507 19:00:33.016 | INFO | "huge" : "", | 20260507 19:00:33.016 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:33.016 | INFO | "malloc" : " 14 GB", | 20260507 19:00:33.016 | INFO | "pinned" : " 29 GB" | 20260507 19:00:33.016 | INFO | } | 20260507 19:00:33.016 | INFO | Perf report for the last 5.13928 seconds: | 20260507 19:00:33.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:33.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:33.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:33.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:33.016 | INFO | ANALYSIS : 69.0%, 3.0, 107.2ms, 0 | 20260507 19:00:33.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:33.016 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 19:00:33.016 | INFO | PRE_HQ : 66.9%, 1.8, 61.2ms, 0 | 20260507 19:00:33.016 | INFO | BAZWRITER : 68.9%, 2.9, 104.0ms, 0 | 20260507 19:00:33.016 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:00:33.017 | INFO | Uploaded 36GiB to GPU in 2.53831s (14.1827GiB/s) | 20260507 19:00:33.017 | INFO | Downloaded 38GiB to GPU in 2.69648s (14.0924GiB/s) | 20260507 19:00:38.155 | INFO | Analyzing chunk frames = [1843712,1844224) | 20260507 19:00:38.155 | INFO | MemInfo: { | 20260507 19:00:38.155 | INFO | "gpu" : " 17 GB", | 20260507 19:00:38.155 | INFO | "huge" : "", | 20260507 19:00:38.155 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:38.155 | INFO | "malloc" : " 14 GB", | 20260507 19:00:38.155 | INFO | "pinned" : " 29 GB" | 20260507 19:00:38.155 | INFO | } | 20260507 19:00:38.155 | INFO | Perf report for the last 5.13969 seconds: | 20260507 19:00:38.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:38.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.155 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 19:00:38.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.156 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:00:38.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:38.156 | INFO | Uploaded 36GiB to GPU in 2.5114s (14.3346GiB/s) | 20260507 19:00:38.156 | INFO | Downloaded 34GiB to GPU in 2.48861s (13.6623GiB/s) | 20260507 19:00:43.295 | INFO | Analyzing chunk frames = [1844224,1844736) | 20260507 19:00:43.295 | INFO | MemInfo: { | 20260507 19:00:43.295 | INFO | "gpu" : " 17 GB", | 20260507 19:00:43.295 | INFO | "huge" : "", | 20260507 19:00:43.295 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:43.295 | INFO | "malloc" : " 14 GB", | 20260507 19:00:43.295 | INFO | "pinned" : " 29 GB" | 20260507 19:00:43.295 | INFO | } | 20260507 19:00:43.295 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:00:43.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:43.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:00:43.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:00:43.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:43.295 | INFO | Uploaded 36GiB to GPU in 2.53393s (14.2072GiB/s) | 20260507 19:00:43.295 | INFO | Downloaded 34GiB to GPU in 2.46224s (13.8085GiB/s) | 20260507 19:00:48.434 | INFO | Analyzing chunk frames = [1844736,1845248) | 20260507 19:00:48.434 | INFO | MemInfo: { | 20260507 19:00:48.434 | INFO | "gpu" : " 17 GB", | 20260507 19:00:48.434 | INFO | "huge" : "", | 20260507 19:00:48.434 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:48.434 | INFO | "malloc" : " 14 GB", | 20260507 19:00:48.434 | INFO | "pinned" : " 29 GB" | 20260507 19:00:48.434 | INFO | } | 20260507 19:00:48.434 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:00:48.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:48.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:00:48.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:00:48.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:48.434 | INFO | Uploaded 36GiB to GPU in 2.47094s (14.5693GiB/s) | 20260507 19:00:48.434 | INFO | Downloaded 34GiB to GPU in 2.39476s (14.1977GiB/s) | 20260507 19:00:53.573 | INFO | Analyzing chunk frames = [1845248,1845760) | 20260507 19:00:53.573 | INFO | MemInfo: { | 20260507 19:00:53.573 | INFO | "gpu" : " 17 GB", | 20260507 19:00:53.573 | INFO | "huge" : "", | 20260507 19:00:53.573 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:53.573 | INFO | "malloc" : " 14 GB", | 20260507 19:00:53.573 | INFO | "pinned" : " 29 GB" | 20260507 19:00:53.573 | INFO | } | 20260507 19:00:53.573 | INFO | Perf report for the last 5.13939 seconds: | 20260507 19:00:53.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:53.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.573 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 19:00:53.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.573 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:00:53.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:53.574 | INFO | Uploaded 36GiB to GPU in 2.38882s (15.0702GiB/s) | 20260507 19:00:53.574 | INFO | Downloaded 34GiB to GPU in 2.50014s (13.5992GiB/s) | 20260507 19:00:58.712 | INFO | Analyzing chunk frames = [1845760,1846272) | 20260507 19:00:58.712 | INFO | MemInfo: { | 20260507 19:00:58.712 | INFO | "gpu" : " 17 GB", | 20260507 19:00:58.712 | INFO | "huge" : "", | 20260507 19:00:58.712 | INFO | "huge_shared" : " 29 GB", | 20260507 19:00:58.712 | INFO | "malloc" : " 14 GB", | 20260507 19:00:58.712 | INFO | "pinned" : " 29 GB" | 20260507 19:00:58.712 | INFO | } | 20260507 19:00:58.712 | INFO | Perf report for the last 5.13918 seconds: | 20260507 19:00:58.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:00:58.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.713 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:00:58.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.713 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:00:58.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:00:58.713 | INFO | Uploaded 36GiB to GPU in 2.49402s (14.4345GiB/s) | 20260507 19:00:58.713 | INFO | Downloaded 34GiB to GPU in 2.41461s (14.081GiB/s) | 20260507 19:01:03.853 | INFO | Analyzing chunk frames = [1846272,1846784) | 20260507 19:01:03.853 | INFO | MemInfo: { | 20260507 19:01:03.853 | INFO | "gpu" : " 17 GB", | 20260507 19:01:03.853 | INFO | "huge" : "", | 20260507 19:01:03.853 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:03.853 | INFO | "malloc" : " 14 GB", | 20260507 19:01:03.853 | INFO | "pinned" : " 29 GB" | 20260507 19:01:03.853 | INFO | } | 20260507 19:01:03.853 | INFO | Perf report for the last 5.14049 seconds: | 20260507 19:01:03.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:03.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:01:03.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:01:03.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:03.853 | INFO | Uploaded 36GiB to GPU in 2.51323s (14.3242GiB/s) | 20260507 19:01:03.853 | INFO | Downloaded 34GiB to GPU in 2.45251s (13.8633GiB/s) | 20260507 19:01:08.992 | INFO | Analyzing chunk frames = [1846784,1847296) | 20260507 19:01:08.992 | INFO | MemInfo: { | 20260507 19:01:08.992 | INFO | "gpu" : " 17 GB", | 20260507 19:01:08.992 | INFO | "huge" : "", | 20260507 19:01:08.992 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:08.992 | INFO | "malloc" : " 14 GB", | 20260507 19:01:08.992 | INFO | "pinned" : " 29 GB" | 20260507 19:01:08.992 | INFO | } | 20260507 19:01:08.992 | INFO | Perf report for the last 5.13942 seconds: | 20260507 19:01:08.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:08.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.992 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:01:08.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.992 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:01:08.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:08.993 | INFO | Uploaded 36GiB to GPU in 2.47068s (14.5709GiB/s) | 20260507 19:01:08.993 | INFO | Downloaded 34GiB to GPU in 2.43136s (13.984GiB/s) | 20260507 19:01:09.089 | INFO | Starting a round of Baz IO | 20260507 19:01:12.573 | INFO | Populating RTMetrics report | 20260507 19:01:12.573 | INFO | RTMetrics report populated | 20260507 19:01:12.573 | INFO | Opening tmp RTMetrics json file | 20260507 19:01:12.590 | INFO | Serializing RTMetrics to json | 20260507 19:01:12.596 | INFO | Serialization done, swapping tmp file now... | 20260507 19:01:12.597 | INFO | RTMetrics done | 20260507 19:01:12.643 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.554 seconds | 20260507 19:01:12.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.89% / 0.01% / 54.10% | 20260507 19:01:12.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.62883% / 3.97364e-06% / 3.57628e-05% | 20260507 19:01:12.643 | INFO | Baz IO overall: | 20260507 19:01:12.643 | INFO | BytesWritten : 831031885131 | 20260507 19:01:12.643 | INFO | NumEvents : 402921769596 | 20260507 19:01:14.130 | INFO | Analyzing chunk frames = [1847296,1847808) | 20260507 19:01:14.130 | INFO | MemInfo: { | 20260507 19:01:14.130 | INFO | "gpu" : " 17 GB", | 20260507 19:01:14.130 | INFO | "huge" : "", | 20260507 19:01:14.130 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:14.130 | INFO | "malloc" : " 14 GB", | 20260507 19:01:14.130 | INFO | "pinned" : " 29 GB" | 20260507 19:01:14.130 | INFO | } | 20260507 19:01:14.130 | INFO | Perf report for the last 5.13769 seconds: | 20260507 19:01:14.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:14.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:14.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:14.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:14.130 | INFO | ANALYSIS : 68.7%, 3.0, 107.4ms, 0 | 20260507 19:01:14.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:14.130 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:01:14.130 | INFO | PRE_HQ : 65.8%, 1.7, 58.7ms, 0 | 20260507 19:01:14.130 | INFO | BAZWRITER : 68.4%, 2.9, 100.6ms, 0 | 20260507 19:01:14.130 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 19:01:14.131 | INFO | Uploaded 36GiB to GPU in 2.48275s (14.5001GiB/s) | 20260507 19:01:14.131 | INFO | Downloaded 38GiB to GPU in 2.69176s (14.1172GiB/s) | 20260507 19:01:19.269 | INFO | Analyzing chunk frames = [1847808,1848320) | 20260507 19:01:19.269 | INFO | MemInfo: { | 20260507 19:01:19.269 | INFO | "gpu" : " 17 GB", | 20260507 19:01:19.269 | INFO | "huge" : "", | 20260507 19:01:19.269 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:19.269 | INFO | "malloc" : " 14 GB", | 20260507 19:01:19.269 | INFO | "pinned" : " 29 GB" | 20260507 19:01:19.269 | INFO | } | 20260507 19:01:19.270 | INFO | Perf report for the last 5.13943 seconds: | 20260507 19:01:19.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:19.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:01:19.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:01:19.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:19.270 | INFO | Uploaded 36GiB to GPU in 2.47431s (14.5495GiB/s) | 20260507 19:01:19.270 | INFO | Downloaded 34GiB to GPU in 2.50349s (13.581GiB/s) | 20260507 19:01:24.409 | INFO | Analyzing chunk frames = [1848320,1848832) | 20260507 19:01:24.409 | INFO | MemInfo: { | 20260507 19:01:24.409 | INFO | "gpu" : " 17 GB", | 20260507 19:01:24.409 | INFO | "huge" : "", | 20260507 19:01:24.410 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:24.410 | INFO | "malloc" : " 14 GB", | 20260507 19:01:24.410 | INFO | "pinned" : " 29 GB" | 20260507 19:01:24.410 | INFO | } | 20260507 19:01:24.410 | INFO | Perf report for the last 5.14016 seconds: | 20260507 19:01:24.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:24.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 19:01:24.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:01:24.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:24.410 | INFO | Uploaded 36GiB to GPU in 2.49465s (14.4309GiB/s) | 20260507 19:01:24.410 | INFO | Downloaded 34GiB to GPU in 2.47574s (13.7333GiB/s) | 20260507 19:01:29.549 | INFO | Analyzing chunk frames = [1848832,1849344) | 20260507 19:01:29.549 | INFO | MemInfo: { | 20260507 19:01:29.549 | INFO | "gpu" : " 17 GB", | 20260507 19:01:29.549 | INFO | "huge" : "", | 20260507 19:01:29.549 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:29.549 | INFO | "malloc" : " 14 GB", | 20260507 19:01:29.549 | INFO | "pinned" : " 29 GB" | 20260507 19:01:29.549 | INFO | } | 20260507 19:01:29.549 | INFO | Perf report for the last 5.13932 seconds: | 20260507 19:01:29.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:29.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.549 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 19:01:29.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.549 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:01:29.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:29.550 | INFO | Uploaded 36GiB to GPU in 2.43937s (14.7579GiB/s) | 20260507 19:01:29.550 | INFO | Downloaded 34GiB to GPU in 2.41086s (14.1029GiB/s) | 20260507 19:01:34.687 | INFO | Analyzing chunk frames = [1849344,1849856) | 20260507 19:01:34.687 | INFO | MemInfo: { | 20260507 19:01:34.687 | INFO | "gpu" : " 17 GB", | 20260507 19:01:34.687 | INFO | "huge" : "", | 20260507 19:01:34.687 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:34.687 | INFO | "malloc" : " 14 GB", | 20260507 19:01:34.687 | INFO | "pinned" : " 29 GB" | 20260507 19:01:34.687 | INFO | } | 20260507 19:01:34.687 | INFO | Perf report for the last 5.13836 seconds: | 20260507 19:01:34.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:34.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.687 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:01:34.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.688 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:01:34.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:34.688 | INFO | Uploaded 36GiB to GPU in 2.33508s (15.4171GiB/s) | 20260507 19:01:34.688 | INFO | Downloaded 34GiB to GPU in 2.50643s (13.5651GiB/s) | 20260507 19:01:39.826 | INFO | Analyzing chunk frames = [1849856,1850368) | 20260507 19:01:39.826 | INFO | MemInfo: { | 20260507 19:01:39.826 | INFO | "gpu" : " 17 GB", | 20260507 19:01:39.826 | INFO | "huge" : "", | 20260507 19:01:39.826 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:39.826 | INFO | "malloc" : " 14 GB", | 20260507 19:01:39.826 | INFO | "pinned" : " 29 GB" | 20260507 19:01:39.826 | INFO | } | 20260507 19:01:39.826 | INFO | Perf report for the last 5.13892 seconds: | 20260507 19:01:39.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:39.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.826 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:01:39.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.826 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:01:39.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:39.827 | INFO | Uploaded 36GiB to GPU in 2.46347s (14.6136GiB/s) | 20260507 19:01:39.827 | INFO | Downloaded 34GiB to GPU in 2.42053s (14.0465GiB/s) | 20260507 19:01:44.968 | INFO | Analyzing chunk frames = [1850368,1850880) | 20260507 19:01:44.968 | INFO | MemInfo: { | 20260507 19:01:44.968 | INFO | "gpu" : " 17 GB", | 20260507 19:01:44.968 | INFO | "huge" : "", | 20260507 19:01:44.968 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:44.968 | INFO | "malloc" : " 14 GB", | 20260507 19:01:44.968 | INFO | "pinned" : " 29 GB" | 20260507 19:01:44.968 | INFO | } | 20260507 19:01:44.968 | INFO | Perf report for the last 5.1415 seconds: | 20260507 19:01:44.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:44.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 19:01:44.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:01:44.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:44.968 | INFO | Uploaded 36GiB to GPU in 2.47194s (14.5635GiB/s) | 20260507 19:01:44.968 | INFO | Downloaded 34GiB to GPU in 2.45986s (13.8219GiB/s) | 20260507 19:01:50.107 | INFO | Analyzing chunk frames = [1850880,1851392) | 20260507 19:01:50.107 | INFO | MemInfo: { | 20260507 19:01:50.107 | INFO | "gpu" : " 17 GB", | 20260507 19:01:50.107 | INFO | "huge" : "", | 20260507 19:01:50.107 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:50.107 | INFO | "malloc" : " 14 GB", | 20260507 19:01:50.107 | INFO | "pinned" : " 29 GB" | 20260507 19:01:50.107 | INFO | } | 20260507 19:01:50.107 | INFO | Perf report for the last 5.13895 seconds: | 20260507 19:01:50.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:50.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:01:50.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:01:50.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:50.107 | INFO | Uploaded 36GiB to GPU in 2.43542s (14.7819GiB/s) | 20260507 19:01:50.107 | INFO | Downloaded 34GiB to GPU in 2.43755s (13.9484GiB/s) | 20260507 19:01:50.107 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:01:50.107 | INFO | Total mem usage: 16.4 GiB | 20260507 19:01:50.107 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:01:50.107 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 19:01:50.107 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:01:50.107 | INFO | High Water Breakdown: | 20260507 19:01:50.107 | INFO | 59.3% : 8.62 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:01:50.107 | INFO | 40.5% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:01:50.107 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:01:50.107 | INFO | ------------------------------------------ | 20260507 19:01:50.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:01:50.107 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:01:50.107 | INFO | Total mem usage: 26.5 GiB | 20260507 19:01:50.107 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:01:50.108 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:01:50.108 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 19:01:50.108 | INFO | High Water Breakdown: | 20260507 19:01:50.108 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:01:50.108 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:01:50.108 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:01:50.108 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:01:50.108 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:01:50.108 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:01:50.108 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:01:50.108 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:01:50.108 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:01:50.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:01:50.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:01:50.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:01:50.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:01:50.108 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:01:50.108 | INFO | ------------------------------------------ | 20260507 19:01:50.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:01:50.108 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:01:50.108 | INFO | Total mem usage: 27.8 GiB | 20260507 19:01:50.108 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:01:50.108 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:01:50.108 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:01:50.108 | INFO | High Water Breakdown: | 20260507 19:01:50.108 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:01:50.108 | INFO | ------------------------------------------ | 20260507 19:01:50.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:01:50.108 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:01:50.108 | INFO | Total mem usage: 15.7 GiB | 20260507 19:01:50.108 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:01:50.108 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:01:50.108 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:01:50.108 | INFO | High Water Breakdown: | 20260507 19:01:50.108 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:01:50.108 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:01:50.108 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:01:50.108 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:01:50.108 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:01:50.108 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:01:50.108 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:01:50.108 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:01:50.108 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:01:50.108 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:01:50.108 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:01:50.108 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:01:50.108 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:01:50.108 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:01:50.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:01:50.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:01:50.108 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:01:50.108 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:01:50.108 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:01:50.108 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:01:50.108 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:01:50.108 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:01:50.108 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:01:50.108 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:01:50.108 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:01:50.108 | INFO | ------------------------------------------ | 20260507 19:01:50.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:01:50.108 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:01:50.108 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:01:50.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:01:50.108 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.54e+01, 1.41e+01 / 2.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.20e-02 / 5.14e-03 | 20260507 19:01:50.108 | INFO | PerfReport: 12.08%, Baseline -- 4.02e+00 / 4.46e+00, 6.09e-01 / 2.62e-01 | 20260507 19:01:50.108 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.49e+00, 2.13e-01 / 1.12e-01 | 20260507 19:01:50.108 | INFO | PerfReport: 15.74%, DME -- 4.18e+01 / 4.49e+01, 3.83e+00 / 1.45e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.83e+00 / 1.66e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.52e+00 / 2.86e+00, 4.40e-01 / 2.29e-01 | 20260507 19:01:50.108 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.45e+00, 3.11e-01 / 1.80e-01 | 20260507 19:01:50.108 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.83e-01 / 4.16e-01, 6.12e-02 / 3.10e-02 | 20260507 19:01:50.108 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:01:50.108 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:01:50.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:01:50.108 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 24.05%, Upload -- 2.57e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 24.19%, Download -- 2.59e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 31.07%, Compute -- 3.32e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 1.74%, LockUpload -- 1.86e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 2.67%, LockDownload -- 2.85e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 16.26%, LockCompute -- 1.74e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:01:50.108 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260507 19:01:50.203 | INFO | Starting a round of Baz IO | 20260507 19:01:53.667 | INFO | Populating RTMetrics report | 20260507 19:01:53.667 | INFO | RTMetrics report populated | 20260507 19:01:53.667 | INFO | Opening tmp RTMetrics json file | 20260507 19:01:53.674 | INFO | Serializing RTMetrics to json | 20260507 19:01:53.687 | INFO | Serialization done, swapping tmp file now... | 20260507 19:01:53.687 | INFO | RTMetrics done | 20260507 19:01:53.746 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.542 seconds | 20260507 19:01:53.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.88% / 0.01% / 54.11% | 20260507 19:01:53.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.62632% / 3.17891e-05% / 2.78155e-05% | 20260507 19:01:53.746 | INFO | Baz IO overall: | 20260507 19:01:53.746 | INFO | BytesWritten : 832962835622 | 20260507 19:01:53.746 | INFO | NumEvents : 403858132647 | 20260507 19:01:55.246 | INFO | Analyzing chunk frames = [1851392,1851904) | 20260507 19:01:55.246 | INFO | MemInfo: { | 20260507 19:01:55.246 | INFO | "gpu" : " 17 GB", | 20260507 19:01:55.246 | INFO | "huge" : "", | 20260507 19:01:55.246 | INFO | "huge_shared" : " 29 GB", | 20260507 19:01:55.246 | INFO | "malloc" : " 14 GB", | 20260507 19:01:55.246 | INFO | "pinned" : " 29 GB" | 20260507 19:01:55.246 | INFO | } | 20260507 19:01:55.246 | INFO | Perf report for the last 5.13941 seconds: | 20260507 19:01:55.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:01:55.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:55.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:55.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:55.246 | INFO | ANALYSIS : 68.5%, 3.0, 107.1ms, 0 | 20260507 19:01:55.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:01:55.246 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 19:01:55.246 | INFO | PRE_HQ : 66.1%, 1.7, 58.7ms, 0 | 20260507 19:01:55.246 | INFO | BAZWRITER : 68.3%, 2.9, 100.7ms, 0 | 20260507 19:01:55.246 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:01:55.247 | INFO | Uploaded 36GiB to GPU in 2.49055s (14.4546GiB/s) | 20260507 19:01:55.247 | INFO | Downloaded 38GiB to GPU in 2.70376s (14.0545GiB/s) | 20260507 19:02:00.385 | INFO | Analyzing chunk frames = [1851904,1852416) | 20260507 19:02:00.385 | INFO | MemInfo: { | 20260507 19:02:00.385 | INFO | "gpu" : " 17 GB", | 20260507 19:02:00.385 | INFO | "huge" : "", | 20260507 19:02:00.385 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:00.385 | INFO | "malloc" : " 14 GB", | 20260507 19:02:00.385 | INFO | "pinned" : " 29 GB" | 20260507 19:02:00.385 | INFO | } | 20260507 19:02:00.385 | INFO | Perf report for the last 5.13875 seconds: | 20260507 19:02:00.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:00.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 19:02:00.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:02:00.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:00.385 | INFO | Uploaded 36GiB to GPU in 2.52434s (14.2612GiB/s) | 20260507 19:02:00.385 | INFO | Downloaded 34GiB to GPU in 2.4902s (13.6535GiB/s) | 20260507 19:02:05.524 | INFO | Analyzing chunk frames = [1852416,1852928) | 20260507 19:02:05.524 | INFO | MemInfo: { | 20260507 19:02:05.524 | INFO | "gpu" : " 17 GB", | 20260507 19:02:05.524 | INFO | "huge" : "", | 20260507 19:02:05.524 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:05.524 | INFO | "malloc" : " 14 GB", | 20260507 19:02:05.524 | INFO | "pinned" : " 29 GB" | 20260507 19:02:05.524 | INFO | } | 20260507 19:02:05.524 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:02:05.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:05.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:02:05.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:02:05.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:05.524 | INFO | Uploaded 36GiB to GPU in 2.52773s (14.2421GiB/s) | 20260507 19:02:05.524 | INFO | Downloaded 34GiB to GPU in 2.46553s (13.7902GiB/s) | 20260507 19:02:10.663 | INFO | Analyzing chunk frames = [1852928,1853440) | 20260507 19:02:10.663 | INFO | MemInfo: { | 20260507 19:02:10.663 | INFO | "gpu" : " 17 GB", | 20260507 19:02:10.663 | INFO | "huge" : "", | 20260507 19:02:10.663 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:10.663 | INFO | "malloc" : " 14 GB", | 20260507 19:02:10.663 | INFO | "pinned" : " 29 GB" | 20260507 19:02:10.663 | INFO | } | 20260507 19:02:10.663 | INFO | Perf report for the last 5.13888 seconds: | 20260507 19:02:10.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:10.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 19:02:10.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 19:02:10.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:10.663 | INFO | Uploaded 36GiB to GPU in 2.4701s (14.5743GiB/s) | 20260507 19:02:10.663 | INFO | Downloaded 34GiB to GPU in 2.40004s (14.1664GiB/s) | 20260507 19:02:15.802 | INFO | Analyzing chunk frames = [1853440,1853952) | 20260507 19:02:15.802 | INFO | MemInfo: { | 20260507 19:02:15.802 | INFO | "gpu" : " 17 GB", | 20260507 19:02:15.802 | INFO | "huge" : "", | 20260507 19:02:15.802 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:15.802 | INFO | "malloc" : " 14 GB", | 20260507 19:02:15.802 | INFO | "pinned" : " 29 GB" | 20260507 19:02:15.802 | INFO | } | 20260507 19:02:15.802 | INFO | Perf report for the last 5.13941 seconds: | 20260507 19:02:15.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:15.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.802 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:02:15.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.802 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:02:15.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:15.803 | INFO | Uploaded 36GiB to GPU in 2.38095s (15.12GiB/s) | 20260507 19:02:15.803 | INFO | Downloaded 34GiB to GPU in 2.50859s (13.5534GiB/s) | 20260507 19:02:20.942 | INFO | Analyzing chunk frames = [1853952,1854464) | 20260507 19:02:20.942 | INFO | MemInfo: { | 20260507 19:02:20.942 | INFO | "gpu" : " 17 GB", | 20260507 19:02:20.942 | INFO | "huge" : "", | 20260507 19:02:20.942 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:20.942 | INFO | "malloc" : " 14 GB", | 20260507 19:02:20.942 | INFO | "pinned" : " 29 GB" | 20260507 19:02:20.942 | INFO | } | 20260507 19:02:20.942 | INFO | Perf report for the last 5.14014 seconds: | 20260507 19:02:20.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:20.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.943 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:02:20.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.943 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:02:20.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:20.943 | INFO | Uploaded 36GiB to GPU in 2.4741s (14.5507GiB/s) | 20260507 19:02:20.943 | INFO | Downloaded 34GiB to GPU in 2.3889s (14.2325GiB/s) | 20260507 19:02:26.081 | INFO | Analyzing chunk frames = [1854464,1854976) | 20260507 19:02:26.081 | INFO | MemInfo: { | 20260507 19:02:26.081 | INFO | "gpu" : " 17 GB", | 20260507 19:02:26.081 | INFO | "huge" : "", | 20260507 19:02:26.081 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:26.081 | INFO | "malloc" : " 14 GB", | 20260507 19:02:26.081 | INFO | "pinned" : " 29 GB" | 20260507 19:02:26.081 | INFO | } | 20260507 19:02:26.081 | INFO | Perf report for the last 5.13874 seconds: | 20260507 19:02:26.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:26.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:02:26.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:02:26.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:26.081 | INFO | Uploaded 36GiB to GPU in 2.54086s (14.1685GiB/s) | 20260507 19:02:26.081 | INFO | Downloaded 34GiB to GPU in 2.44125s (13.9273GiB/s) | 20260507 19:02:31.221 | INFO | Analyzing chunk frames = [1854976,1855488) | 20260507 19:02:31.221 | INFO | MemInfo: { | 20260507 19:02:31.221 | INFO | "gpu" : " 17 GB", | 20260507 19:02:31.221 | INFO | "huge" : "", | 20260507 19:02:31.221 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:31.221 | INFO | "malloc" : " 14 GB", | 20260507 19:02:31.221 | INFO | "pinned" : " 29 GB" | 20260507 19:02:31.221 | INFO | } | 20260507 19:02:31.221 | INFO | Perf report for the last 5.13992 seconds: | 20260507 19:02:31.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:31.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:02:31.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 19:02:31.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:31.221 | INFO | Uploaded 36GiB to GPU in 2.50888s (14.349GiB/s) | 20260507 19:02:31.221 | INFO | Downloaded 34GiB to GPU in 2.42224s (14.0366GiB/s) | 20260507 19:02:31.319 | INFO | Starting a round of Baz IO | 20260507 19:02:34.798 | INFO | Populating RTMetrics report | 20260507 19:02:34.798 | INFO | RTMetrics report populated | 20260507 19:02:34.798 | INFO | Opening tmp RTMetrics json file | 20260507 19:02:34.814 | INFO | Serializing RTMetrics to json | 20260507 19:02:34.816 | INFO | Serialization done, swapping tmp file now... | 20260507 19:02:34.817 | INFO | RTMetrics done | 20260507 19:02:34.876 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.557 seconds | 20260507 19:02:34.876 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.86% / 0.01% / 54.12% | 20260507 19:02:34.876 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.62425% / 2.38419e-05% / 3.57628e-05% | 20260507 19:02:34.876 | INFO | Baz IO overall: | 20260507 19:02:34.876 | INFO | BytesWritten : 834894289775 | 20260507 19:02:34.876 | INFO | NumEvents : 404795129510 | 20260507 19:02:36.360 | INFO | Analyzing chunk frames = [1855488,1856000) | 20260507 19:02:36.360 | INFO | MemInfo: { | 20260507 19:02:36.360 | INFO | "gpu" : " 17 GB", | 20260507 19:02:36.360 | INFO | "huge" : "", | 20260507 19:02:36.360 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:36.360 | INFO | "malloc" : " 14 GB", | 20260507 19:02:36.360 | INFO | "pinned" : " 29 GB" | 20260507 19:02:36.360 | INFO | } | 20260507 19:02:36.360 | INFO | Perf report for the last 5.13892 seconds: | 20260507 19:02:36.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:36.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:36.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:36.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:36.360 | INFO | ANALYSIS : 68.9%, 3.0, 107.4ms, 0 | 20260507 19:02:36.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:36.360 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:02:36.360 | INFO | PRE_HQ : 65.9%, 1.7, 58.8ms, 0 | 20260507 19:02:36.360 | INFO | BAZWRITER : 68.4%, 2.9, 102.4ms, 0 | 20260507 19:02:36.360 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:02:36.361 | INFO | Uploaded 36GiB to GPU in 2.51523s (14.3128GiB/s) | 20260507 19:02:36.361 | INFO | Downloaded 38GiB to GPU in 2.68064s (14.1757GiB/s) | 20260507 19:02:41.499 | INFO | Analyzing chunk frames = [1856000,1856512) | 20260507 19:02:41.499 | INFO | MemInfo: { | 20260507 19:02:41.499 | INFO | "gpu" : " 17 GB", | 20260507 19:02:41.499 | INFO | "huge" : "", | 20260507 19:02:41.499 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:41.499 | INFO | "malloc" : " 14 GB", | 20260507 19:02:41.499 | INFO | "pinned" : " 29 GB" | 20260507 19:02:41.499 | INFO | } | 20260507 19:02:41.499 | INFO | Perf report for the last 5.13892 seconds: | 20260507 19:02:41.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:41.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:02:41.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:02:41.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:41.499 | INFO | Uploaded 36GiB to GPU in 2.52502s (14.2573GiB/s) | 20260507 19:02:41.499 | INFO | Downloaded 34GiB to GPU in 2.47834s (13.7189GiB/s) | 20260507 19:02:46.638 | INFO | Analyzing chunk frames = [1856512,1857024) | 20260507 19:02:46.638 | INFO | MemInfo: { | 20260507 19:02:46.638 | INFO | "gpu" : " 17 GB", | 20260507 19:02:46.638 | INFO | "huge" : "", | 20260507 19:02:46.638 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:46.638 | INFO | "malloc" : " 14 GB", | 20260507 19:02:46.638 | INFO | "pinned" : " 29 GB" | 20260507 19:02:46.638 | INFO | } | 20260507 19:02:46.638 | INFO | Perf report for the last 5.13866 seconds: | 20260507 19:02:46.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:46.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:02:46.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 19:02:46.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:46.638 | INFO | Uploaded 36GiB to GPU in 2.53591s (14.1961GiB/s) | 20260507 19:02:46.638 | INFO | Downloaded 34GiB to GPU in 2.46829s (13.7747GiB/s) | 20260507 19:02:51.777 | INFO | Analyzing chunk frames = [1857024,1857536) | 20260507 19:02:51.778 | INFO | MemInfo: { | 20260507 19:02:51.778 | INFO | "gpu" : " 17 GB", | 20260507 19:02:51.778 | INFO | "huge" : "", | 20260507 19:02:51.778 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:51.778 | INFO | "malloc" : " 14 GB", | 20260507 19:02:51.778 | INFO | "pinned" : " 29 GB" | 20260507 19:02:51.778 | INFO | } | 20260507 19:02:51.778 | INFO | Perf report for the last 5.13998 seconds: | 20260507 19:02:51.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:51.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:02:51.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:02:51.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:51.778 | INFO | Uploaded 36GiB to GPU in 2.4667s (14.5944GiB/s) | 20260507 19:02:51.778 | INFO | Downloaded 34GiB to GPU in 2.38886s (14.2327GiB/s) | 20260507 19:02:56.919 | INFO | Analyzing chunk frames = [1857536,1858048) | 20260507 19:02:56.919 | INFO | MemInfo: { | 20260507 19:02:56.919 | INFO | "gpu" : " 17 GB", | 20260507 19:02:56.919 | INFO | "huge" : "", | 20260507 19:02:56.919 | INFO | "huge_shared" : " 29 GB", | 20260507 19:02:56.919 | INFO | "malloc" : " 14 GB", | 20260507 19:02:56.919 | INFO | "pinned" : " 29 GB" | 20260507 19:02:56.919 | INFO | } | 20260507 19:02:56.919 | INFO | Perf report for the last 5.14126 seconds: | 20260507 19:02:56.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:02:56.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.919 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:02:56.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.919 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:02:56.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:02:56.920 | INFO | Uploaded 36GiB to GPU in 2.33468s (15.4197GiB/s) | 20260507 19:02:56.920 | INFO | Downloaded 34GiB to GPU in 2.49029s (13.653GiB/s) | 20260507 19:03:02.060 | INFO | Analyzing chunk frames = [1858048,1858560) | 20260507 19:03:02.060 | INFO | MemInfo: { | 20260507 19:03:02.060 | INFO | "gpu" : " 17 GB", | 20260507 19:03:02.060 | INFO | "huge" : "", | 20260507 19:03:02.060 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:02.060 | INFO | "malloc" : " 14 GB", | 20260507 19:03:02.060 | INFO | "pinned" : " 29 GB" | 20260507 19:03:02.060 | INFO | } | 20260507 19:03:02.060 | INFO | Perf report for the last 5.14122 seconds: | 20260507 19:03:02.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:02.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 19:03:02.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:03:02.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:02.060 | INFO | Uploaded 36GiB to GPU in 2.43387s (14.7913GiB/s) | 20260507 19:03:02.061 | INFO | Downloaded 34GiB to GPU in 2.43759s (13.9482GiB/s) | 20260507 19:03:07.196 | INFO | Analyzing chunk frames = [1858560,1859072) | 20260507 19:03:07.196 | INFO | MemInfo: { | 20260507 19:03:07.196 | INFO | "gpu" : " 17 GB", | 20260507 19:03:07.196 | INFO | "huge" : "", | 20260507 19:03:07.196 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:07.196 | INFO | "malloc" : " 14 GB", | 20260507 19:03:07.196 | INFO | "pinned" : " 29 GB" | 20260507 19:03:07.196 | INFO | } | 20260507 19:03:07.196 | INFO | Perf report for the last 5.13603 seconds: | 20260507 19:03:07.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:07.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.196 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 19:03:07.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.196 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:03:07.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:07.197 | INFO | Uploaded 36GiB to GPU in 2.46137s (14.626GiB/s) | 20260507 19:03:07.197 | INFO | Downloaded 34GiB to GPU in 2.45377s (13.8562GiB/s) | 20260507 19:03:12.335 | INFO | Analyzing chunk frames = [1859072,1859584) | 20260507 19:03:12.335 | INFO | MemInfo: { | 20260507 19:03:12.335 | INFO | "gpu" : " 17 GB", | 20260507 19:03:12.335 | INFO | "huge" : "", | 20260507 19:03:12.335 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:12.335 | INFO | "malloc" : " 14 GB", | 20260507 19:03:12.335 | INFO | "pinned" : " 29 GB" | 20260507 19:03:12.335 | INFO | } | 20260507 19:03:12.335 | INFO | Perf report for the last 5.1389 seconds: | 20260507 19:03:12.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:12.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:03:12.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:03:12.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:12.335 | INFO | Uploaded 36GiB to GPU in 2.45193s (14.6823GiB/s) | 20260507 19:03:12.335 | INFO | Downloaded 34GiB to GPU in 2.44653s (13.8972GiB/s) | 20260507 19:03:12.336 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:03:12.336 | INFO | Total mem usage: 16.4 GiB | 20260507 19:03:12.336 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:03:12.336 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.8%) | 20260507 19:03:12.336 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:03:12.336 | INFO | High Water Breakdown: | 20260507 19:03:12.336 | INFO | 59.2% : 8.53 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:03:12.336 | INFO | 40.7% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:03:12.336 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:03:12.336 | INFO | ------------------------------------------ | 20260507 19:03:12.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:03:12.336 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:03:12.336 | INFO | Total mem usage: 26.5 GiB | 20260507 19:03:12.336 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:03:12.336 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:03:12.336 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:03:12.336 | INFO | High Water Breakdown: | 20260507 19:03:12.336 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:03:12.336 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:03:12.336 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:03:12.336 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:03:12.336 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:03:12.336 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:03:12.336 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:03:12.336 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:03:12.336 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:03:12.336 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:03:12.336 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:03:12.336 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:03:12.336 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:03:12.336 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:03:12.336 | INFO | ------------------------------------------ | 20260507 19:03:12.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:03:12.336 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:03:12.336 | INFO | Total mem usage: 27.8 GiB | 20260507 19:03:12.336 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:03:12.336 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:03:12.336 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:03:12.336 | INFO | High Water Breakdown: | 20260507 19:03:12.336 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:03:12.336 | INFO | ------------------------------------------ | 20260507 19:03:12.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:03:12.336 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:03:12.336 | INFO | Total mem usage: 15.7 GiB | 20260507 19:03:12.336 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:03:12.336 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:03:12.336 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:03:12.336 | INFO | High Water Breakdown: | 20260507 19:03:12.336 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:03:12.336 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:03:12.336 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:03:12.336 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:03:12.336 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:03:12.336 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:03:12.336 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:03:12.336 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:03:12.336 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:03:12.336 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:03:12.336 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:03:12.336 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:03:12.336 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:03:12.336 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:03:12.336 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:03:12.336 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:03:12.336 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:03:12.336 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:03:12.336 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:03:12.336 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:03:12.336 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:03:12.336 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:03:12.336 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:03:12.336 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:03:12.336 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:03:12.336 | INFO | ------------------------------------------ | 20260507 19:03:12.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:03:12.336 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:03:12.336 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:03:12.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:03:12.336 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 2.09e+00 | 20260507 19:03:12.336 | INFO | PerfReport: 0.29%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.10e-02 / 6.81e-03 | 20260507 19:03:12.336 | INFO | PerfReport: 12.07%, Baseline -- 4.01e+00 / 4.45e+00, 6.05e-01 / 2.96e-01 | 20260507 19:03:12.336 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.49e+00, 2.12e-01 / 1.27e-01 | 20260507 19:03:12.336 | INFO | PerfReport: 15.75%, DME -- 4.19e+01 / 4.49e+01, 3.91e+00 / 1.60e+00 | 20260507 19:03:12.336 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.82e+00 / 2.02e+00 | 20260507 19:03:12.336 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.38e-01 / 2.51e-01 | 20260507 19:03:12.336 | INFO | PerfReport: 3.69%, Metrics -- 1.23e+00 / 1.44e+00, 3.11e-01 / 1.96e-01 | 20260507 19:03:12.336 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.13e-01, 6.23e-02 / 3.01e-02 | 20260507 19:03:12.337 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:03:12.337 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:03:12.337 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:03:12.337 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 24.11%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 24.12%, Download -- 2.58e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 31.05%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 1.89%, LockUpload -- 2.02e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 2.51%, LockDownload -- 2.69e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 16.31%, LockCompute -- 1.75e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:03:12.337 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 19:03:12.433 | INFO | Starting a round of Baz IO | 20260507 19:03:15.928 | INFO | Populating RTMetrics report | 20260507 19:03:15.928 | INFO | RTMetrics report populated | 20260507 19:03:15.928 | INFO | Opening tmp RTMetrics json file | 20260507 19:03:15.944 | INFO | Serializing RTMetrics to json | 20260507 19:03:15.945 | INFO | Serialization done, swapping tmp file now... | 20260507 19:03:15.946 | INFO | RTMetrics done | 20260507 19:03:16.017 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.584 seconds | 20260507 19:03:16.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.85% / 0.01% / 54.14% | 20260507 19:03:16.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.6197% / 1.19209e-05% / 3.97364e-05% | 20260507 19:03:16.017 | INFO | Baz IO overall: | 20260507 19:03:16.017 | INFO | BytesWritten : 836827204658 | 20260507 19:03:16.017 | INFO | NumEvents : 405733049182 | 20260507 19:03:17.474 | INFO | Analyzing chunk frames = [1859584,1860096) | 20260507 19:03:17.474 | INFO | MemInfo: { | 20260507 19:03:17.474 | INFO | "gpu" : " 17 GB", | 20260507 19:03:17.474 | INFO | "huge" : "", | 20260507 19:03:17.474 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:17.474 | INFO | "malloc" : " 14 GB", | 20260507 19:03:17.474 | INFO | "pinned" : " 29 GB" | 20260507 19:03:17.474 | INFO | } | 20260507 19:03:17.474 | INFO | Perf report for the last 5.13919 seconds: | 20260507 19:03:17.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:17.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:17.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:17.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:17.475 | INFO | ANALYSIS : 69.1%, 3.0, 107.7ms, 0 | 20260507 19:03:17.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:17.475 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:03:17.475 | INFO | PRE_HQ : 66.5%, 1.7, 59.2ms, 0 | 20260507 19:03:17.475 | INFO | BAZWRITER : 68.9%, 2.9, 103.4ms, 0 | 20260507 19:03:17.475 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:03:17.475 | INFO | Uploaded 36GiB to GPU in 2.46926s (14.5793GiB/s) | 20260507 19:03:17.475 | INFO | Downloaded 38GiB to GPU in 2.69026s (14.125GiB/s) | 20260507 19:03:22.614 | INFO | Analyzing chunk frames = [1860096,1860608) | 20260507 19:03:22.614 | INFO | MemInfo: { | 20260507 19:03:22.614 | INFO | "gpu" : " 17 GB", | 20260507 19:03:22.614 | INFO | "huge" : "", | 20260507 19:03:22.614 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:22.614 | INFO | "malloc" : " 14 GB", | 20260507 19:03:22.614 | INFO | "pinned" : " 29 GB" | 20260507 19:03:22.614 | INFO | } | 20260507 19:03:22.614 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:03:22.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:22.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:03:22.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:03:22.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:22.614 | INFO | Uploaded 36GiB to GPU in 2.5358s (14.1967GiB/s) | 20260507 19:03:22.614 | INFO | Downloaded 34GiB to GPU in 2.4942s (13.6316GiB/s) | 20260507 19:03:27.753 | INFO | Analyzing chunk frames = [1860608,1861120) | 20260507 19:03:27.753 | INFO | MemInfo: { | 20260507 19:03:27.753 | INFO | "gpu" : " 17 GB", | 20260507 19:03:27.753 | INFO | "huge" : "", | 20260507 19:03:27.753 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:27.753 | INFO | "malloc" : " 14 GB", | 20260507 19:03:27.753 | INFO | "pinned" : " 29 GB" | 20260507 19:03:27.753 | INFO | } | 20260507 19:03:27.754 | INFO | Perf report for the last 5.13954 seconds: | 20260507 19:03:27.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:27.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:03:27.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:03:27.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:27.754 | INFO | Uploaded 36GiB to GPU in 2.52157s (14.2768GiB/s) | 20260507 19:03:27.754 | INFO | Downloaded 34GiB to GPU in 2.46278s (13.8055GiB/s) | 20260507 19:03:32.892 | INFO | Analyzing chunk frames = [1861120,1861632) | 20260507 19:03:32.892 | INFO | MemInfo: { | 20260507 19:03:32.892 | INFO | "gpu" : " 17 GB", | 20260507 19:03:32.892 | INFO | "huge" : "", | 20260507 19:03:32.893 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:32.893 | INFO | "malloc" : " 14 GB", | 20260507 19:03:32.893 | INFO | "pinned" : " 29 GB" | 20260507 19:03:32.893 | INFO | } | 20260507 19:03:32.893 | INFO | Perf report for the last 5.13914 seconds: | 20260507 19:03:32.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:32.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:03:32.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:03:32.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:32.893 | INFO | Uploaded 36GiB to GPU in 2.4591s (14.6395GiB/s) | 20260507 19:03:32.893 | INFO | Downloaded 34GiB to GPU in 2.38956s (14.2286GiB/s) | 20260507 19:03:38.033 | INFO | Analyzing chunk frames = [1861632,1862144) | 20260507 19:03:38.033 | INFO | MemInfo: { | 20260507 19:03:38.033 | INFO | "gpu" : " 17 GB", | 20260507 19:03:38.033 | INFO | "huge" : "", | 20260507 19:03:38.033 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:38.033 | INFO | "malloc" : " 14 GB", | 20260507 19:03:38.033 | INFO | "pinned" : " 29 GB" | 20260507 19:03:38.033 | INFO | } | 20260507 19:03:38.033 | INFO | Perf report for the last 5.14022 seconds: | 20260507 19:03:38.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:38.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.033 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:03:38.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.033 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:03:38.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:38.034 | INFO | Uploaded 36GiB to GPU in 2.36985s (15.1908GiB/s) | 20260507 19:03:38.034 | INFO | Downloaded 34GiB to GPU in 2.49545s (13.6248GiB/s) | 20260507 19:03:43.171 | INFO | Analyzing chunk frames = [1862144,1862656) | 20260507 19:03:43.171 | INFO | MemInfo: { | 20260507 19:03:43.171 | INFO | "gpu" : " 17 GB", | 20260507 19:03:43.171 | INFO | "huge" : "", | 20260507 19:03:43.171 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:43.171 | INFO | "malloc" : " 14 GB", | 20260507 19:03:43.171 | INFO | "pinned" : " 29 GB" | 20260507 19:03:43.171 | INFO | } | 20260507 19:03:43.172 | INFO | Perf report for the last 5.13857 seconds: | 20260507 19:03:43.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:43.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:03:43.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:03:43.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:43.172 | INFO | Uploaded 36GiB to GPU in 2.49679s (14.4185GiB/s) | 20260507 19:03:43.172 | INFO | Downloaded 34GiB to GPU in 2.41064s (14.1041GiB/s) | 20260507 19:03:48.311 | INFO | Analyzing chunk frames = [1862656,1863168) | 20260507 19:03:48.311 | INFO | MemInfo: { | 20260507 19:03:48.311 | INFO | "gpu" : " 17 GB", | 20260507 19:03:48.311 | INFO | "huge" : "", | 20260507 19:03:48.311 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:48.311 | INFO | "malloc" : " 14 GB", | 20260507 19:03:48.311 | INFO | "pinned" : " 29 GB" | 20260507 19:03:48.311 | INFO | } | 20260507 19:03:48.311 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:03:48.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:48.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.311 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:03:48.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.312 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 19:03:48.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:48.312 | INFO | Uploaded 36GiB to GPU in 2.5437s (14.1526GiB/s) | 20260507 19:03:48.312 | INFO | Downloaded 34GiB to GPU in 2.4373s (13.9498GiB/s) | 20260507 19:03:53.450 | INFO | Analyzing chunk frames = [1863168,1863680) | 20260507 19:03:53.450 | INFO | MemInfo: { | 20260507 19:03:53.450 | INFO | "gpu" : " 17 GB", | 20260507 19:03:53.450 | INFO | "huge" : "", | 20260507 19:03:53.450 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:53.450 | INFO | "malloc" : " 14 GB", | 20260507 19:03:53.450 | INFO | "pinned" : " 29 GB" | 20260507 19:03:53.450 | INFO | } | 20260507 19:03:53.451 | INFO | Perf report for the last 5.13925 seconds: | 20260507 19:03:53.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:53.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:03:53.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 19:03:53.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:53.451 | INFO | Uploaded 36GiB to GPU in 2.52513s (14.2567GiB/s) | 20260507 19:03:53.451 | INFO | Downloaded 34GiB to GPU in 2.4197s (14.0513GiB/s) | 20260507 19:03:53.564 | INFO | Starting a round of Baz IO | 20260507 19:03:57.040 | INFO | Populating RTMetrics report | 20260507 19:03:57.041 | INFO | RTMetrics report populated | 20260507 19:03:57.041 | INFO | Opening tmp RTMetrics json file | 20260507 19:03:57.054 | INFO | Serializing RTMetrics to json | 20260507 19:03:57.059 | INFO | Serialization done, swapping tmp file now... | 20260507 19:03:57.060 | INFO | RTMetrics done | 20260507 19:03:57.117 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.553 seconds | 20260507 19:03:57.117 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.84% / 0.01% / 54.15% | 20260507 19:03:57.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.61076% / 1.58946e-05% / 3.17891e-05% | 20260507 19:03:57.117 | INFO | Baz IO overall: | 20260507 19:03:57.117 | INFO | BytesWritten : 838758964681 | 20260507 19:03:57.117 | INFO | NumEvents : 406670409180 | 20260507 19:03:58.589 | INFO | Analyzing chunk frames = [1863680,1864192) | 20260507 19:03:58.589 | INFO | MemInfo: { | 20260507 19:03:58.589 | INFO | "gpu" : " 17 GB", | 20260507 19:03:58.589 | INFO | "huge" : "", | 20260507 19:03:58.589 | INFO | "huge_shared" : " 29 GB", | 20260507 19:03:58.589 | INFO | "malloc" : " 14 GB", | 20260507 19:03:58.589 | INFO | "pinned" : " 29 GB" | 20260507 19:03:58.589 | INFO | } | 20260507 19:03:58.590 | INFO | Perf report for the last 5.13906 seconds: | 20260507 19:03:58.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:03:58.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:58.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:58.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:58.590 | INFO | ANALYSIS : 69.1%, 2.9, 106.7ms, 0 | 20260507 19:03:58.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:03:58.590 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 19:03:58.590 | INFO | PRE_HQ : 67.0%, 1.8, 64.2ms, 0 | 20260507 19:03:58.590 | INFO | BAZWRITER : 68.6%, 3.1, 107.6ms, 0 | 20260507 19:03:58.590 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:03:58.590 | INFO | Uploaded 36GiB to GPU in 2.50544s (14.3687GiB/s) | 20260507 19:03:58.590 | INFO | Downloaded 38GiB to GPU in 2.67954s (14.1815GiB/s) | 20260507 19:04:03.730 | INFO | Analyzing chunk frames = [1864192,1864704) | 20260507 19:04:03.730 | INFO | MemInfo: { | 20260507 19:04:03.730 | INFO | "gpu" : " 17 GB", | 20260507 19:04:03.730 | INFO | "huge" : "", | 20260507 19:04:03.730 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:03.730 | INFO | "malloc" : " 14 GB", | 20260507 19:04:03.730 | INFO | "pinned" : " 29 GB" | 20260507 19:04:03.730 | INFO | } | 20260507 19:04:03.730 | INFO | Perf report for the last 5.14021 seconds: | 20260507 19:04:03.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:03.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:04:03.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:04:03.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:03.730 | INFO | Uploaded 36GiB to GPU in 2.52038s (14.2836GiB/s) | 20260507 19:04:03.730 | INFO | Downloaded 34GiB to GPU in 2.45432s (13.8531GiB/s) | 20260507 19:04:08.869 | INFO | Analyzing chunk frames = [1864704,1865216) | 20260507 19:04:08.869 | INFO | MemInfo: { | 20260507 19:04:08.869 | INFO | "gpu" : " 17 GB", | 20260507 19:04:08.869 | INFO | "huge" : "", | 20260507 19:04:08.869 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:08.869 | INFO | "malloc" : " 14 GB", | 20260507 19:04:08.869 | INFO | "pinned" : " 29 GB" | 20260507 19:04:08.869 | INFO | } | 20260507 19:04:08.869 | INFO | Perf report for the last 5.13949 seconds: | 20260507 19:04:08.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:08.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.869 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:04:08.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.869 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:04:08.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:08.870 | INFO | Uploaded 36GiB to GPU in 2.56457s (14.0374GiB/s) | 20260507 19:04:08.870 | INFO | Downloaded 34GiB to GPU in 2.44682s (13.8956GiB/s) | 20260507 19:04:14.008 | INFO | Analyzing chunk frames = [1865216,1865728) | 20260507 19:04:14.008 | INFO | MemInfo: { | 20260507 19:04:14.008 | INFO | "gpu" : " 17 GB", | 20260507 19:04:14.008 | INFO | "huge" : "", | 20260507 19:04:14.008 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:14.008 | INFO | "malloc" : " 14 GB", | 20260507 19:04:14.008 | INFO | "pinned" : " 29 GB" | 20260507 19:04:14.008 | INFO | } | 20260507 19:04:14.008 | INFO | Perf report for the last 5.13873 seconds: | 20260507 19:04:14.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:14.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:04:14.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:04:14.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:14.008 | INFO | Uploaded 36GiB to GPU in 2.47896s (14.5222GiB/s) | 20260507 19:04:14.008 | INFO | Downloaded 34GiB to GPU in 2.38291s (14.2683GiB/s) | 20260507 19:04:19.147 | INFO | Analyzing chunk frames = [1865728,1866240) | 20260507 19:04:19.147 | INFO | MemInfo: { | 20260507 19:04:19.147 | INFO | "gpu" : " 17 GB", | 20260507 19:04:19.147 | INFO | "huge" : "", | 20260507 19:04:19.147 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:19.147 | INFO | "malloc" : " 14 GB", | 20260507 19:04:19.147 | INFO | "pinned" : " 29 GB" | 20260507 19:04:19.147 | INFO | } | 20260507 19:04:19.147 | INFO | Perf report for the last 5.13913 seconds: | 20260507 19:04:19.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:19.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260507 19:04:19.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 19:04:19.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:19.147 | INFO | Uploaded 36GiB to GPU in 2.45068s (14.6898GiB/s) | 20260507 19:04:19.148 | INFO | Downloaded 34GiB to GPU in 2.497s (13.6163GiB/s) | 20260507 19:04:24.286 | INFO | Analyzing chunk frames = [1866240,1866752) | 20260507 19:04:24.286 | INFO | MemInfo: { | 20260507 19:04:24.286 | INFO | "gpu" : " 17 GB", | 20260507 19:04:24.286 | INFO | "huge" : "", | 20260507 19:04:24.286 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:24.286 | INFO | "malloc" : " 14 GB", | 20260507 19:04:24.286 | INFO | "pinned" : " 29 GB" | 20260507 19:04:24.286 | INFO | } | 20260507 19:04:24.286 | INFO | Perf report for the last 5.13922 seconds: | 20260507 19:04:24.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:24.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.287 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:04:24.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.287 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:04:24.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:24.287 | INFO | Uploaded 36GiB to GPU in 2.49453s (14.4316GiB/s) | 20260507 19:04:24.287 | INFO | Downloaded 34GiB to GPU in 2.41598s (14.0729GiB/s) | 20260507 19:04:29.425 | INFO | Analyzing chunk frames = [1866752,1867264) | 20260507 19:04:29.425 | INFO | MemInfo: { | 20260507 19:04:29.425 | INFO | "gpu" : " 17 GB", | 20260507 19:04:29.425 | INFO | "huge" : "", | 20260507 19:04:29.426 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:29.426 | INFO | "malloc" : " 14 GB", | 20260507 19:04:29.426 | INFO | "pinned" : " 29 GB" | 20260507 19:04:29.426 | INFO | } | 20260507 19:04:29.426 | INFO | Perf report for the last 5.13919 seconds: | 20260507 19:04:29.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:29.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:04:29.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 19:04:29.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:29.426 | INFO | Uploaded 36GiB to GPU in 2.51099s (14.337GiB/s) | 20260507 19:04:29.426 | INFO | Downloaded 34GiB to GPU in 2.42526s (14.0191GiB/s) | 20260507 19:04:34.565 | INFO | Analyzing chunk frames = [1867264,1867776) | 20260507 19:04:34.565 | INFO | MemInfo: { | 20260507 19:04:34.565 | INFO | "gpu" : " 17 GB", | 20260507 19:04:34.565 | INFO | "huge" : "", | 20260507 19:04:34.565 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:34.565 | INFO | "malloc" : " 14 GB", | 20260507 19:04:34.565 | INFO | "pinned" : " 29 GB" | 20260507 19:04:34.565 | INFO | } | 20260507 19:04:34.565 | INFO | Perf report for the last 5.13969 seconds: | 20260507 19:04:34.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:34.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:04:34.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:04:34.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:34.565 | INFO | Uploaded 36GiB to GPU in 2.50411s (14.3764GiB/s) | 20260507 19:04:34.566 | INFO | Downloaded 34GiB to GPU in 2.42933s (13.9956GiB/s) | 20260507 19:04:34.566 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:04:34.566 | INFO | Total mem usage: 16.4 GiB | 20260507 19:04:34.566 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:04:34.566 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 19:04:34.566 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:04:34.566 | INFO | High Water Breakdown: | 20260507 19:04:34.566 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:04:34.566 | INFO | 40.6% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:04:34.566 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:04:34.566 | INFO | ------------------------------------------ | 20260507 19:04:34.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:04:34.566 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:04:34.566 | INFO | Total mem usage: 26.5 GiB | 20260507 19:04:34.566 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:04:34.566 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 19:04:34.566 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:04:34.566 | INFO | High Water Breakdown: | 20260507 19:04:34.566 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:04:34.566 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:04:34.566 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:04:34.566 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:04:34.566 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:04:34.566 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:04:34.566 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:04:34.566 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:04:34.566 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:04:34.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:04:34.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:04:34.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:04:34.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:04:34.566 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:04:34.566 | INFO | ------------------------------------------ | 20260507 19:04:34.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:04:34.566 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:04:34.566 | INFO | Total mem usage: 27.8 GiB | 20260507 19:04:34.566 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:04:34.566 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:04:34.566 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:04:34.566 | INFO | High Water Breakdown: | 20260507 19:04:34.566 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:04:34.566 | INFO | ------------------------------------------ | 20260507 19:04:34.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:04:34.566 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:04:34.566 | INFO | Total mem usage: 15.7 GiB | 20260507 19:04:34.566 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:04:34.566 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:04:34.566 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 19:04:34.566 | INFO | High Water Breakdown: | 20260507 19:04:34.566 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:04:34.566 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:04:34.566 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:04:34.566 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:04:34.566 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:04:34.566 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:04:34.566 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:04:34.566 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:04:34.566 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:04:34.566 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:04:34.566 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:04:34.566 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:04:34.566 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:04:34.566 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:04:34.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:04:34.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:04:34.566 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:04:34.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:04:34.567 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:04:34.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:04:34.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:04:34.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:04:34.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:04:34.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:04:34.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:04:34.567 | INFO | ------------------------------------------ | 20260507 19:04:34.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:04:34.567 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:04:34.567 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:04:34.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:04:34.567 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.43e+01 / 1.88e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.17e-02 / 5.62e-03 | 20260507 19:04:34.567 | INFO | PerfReport: 12.06%, Baseline -- 4.00e+00 / 4.48e+00, 5.98e-01 / 2.62e-01 | 20260507 19:04:34.567 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.12e-01 / 1.17e-01 | 20260507 19:04:34.567 | INFO | PerfReport: 15.75%, DME -- 4.18e+01 / 4.49e+01, 3.86e+00 / 1.41e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.82e+00 / 1.76e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.51e+00 / 2.87e+00, 4.35e-01 / 2.07e-01 | 20260507 19:04:34.567 | INFO | PerfReport: 3.69%, Metrics -- 1.22e+00 / 1.44e+00, 3.13e-01 / 1.56e-01 | 20260507 19:04:34.567 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.15e-01, 6.21e-02 / 3.47e-02 | 20260507 19:04:34.567 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:04:34.567 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:04:34.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:04:34.567 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 24.28%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 24.02%, Download -- 2.57e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 31.00%, Compute -- 3.32e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 2.06%, LockUpload -- 2.21e+00 / 0.00e+00, 5.20e+00 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 2.38%, LockDownload -- 2.54e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 16.26%, LockCompute -- 1.74e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:04:34.567 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260507 19:04:34.662 | INFO | Starting a round of Baz IO | 20260507 19:04:38.151 | INFO | Populating RTMetrics report | 20260507 19:04:38.151 | INFO | RTMetrics report populated | 20260507 19:04:38.151 | INFO | Opening tmp RTMetrics json file | 20260507 19:04:38.167 | INFO | Serializing RTMetrics to json | 20260507 19:04:38.169 | INFO | Serialization done, swapping tmp file now... | 20260507 19:04:38.170 | INFO | RTMetrics done | 20260507 19:04:38.230 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.567 seconds | 20260507 19:04:38.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.82% / 0.01% / 54.16% | 20260507 19:04:38.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.60427% / 3.17891e-05% / 3.97364e-05% | 20260507 19:04:38.230 | INFO | Baz IO overall: | 20260507 19:04:38.230 | INFO | BytesWritten : 840689750372 | 20260507 19:04:38.230 | INFO | NumEvents : 407606907317 | 20260507 19:04:39.705 | INFO | Analyzing chunk frames = [1867776,1868288) | 20260507 19:04:39.705 | INFO | MemInfo: { | 20260507 19:04:39.705 | INFO | "gpu" : " 17 GB", | 20260507 19:04:39.705 | INFO | "huge" : "", | 20260507 19:04:39.705 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:39.705 | INFO | "malloc" : " 14 GB", | 20260507 19:04:39.705 | INFO | "pinned" : " 29 GB" | 20260507 19:04:39.705 | INFO | } | 20260507 19:04:39.705 | INFO | Perf report for the last 5.13947 seconds: | 20260507 19:04:39.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:39.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:39.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:39.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:39.705 | INFO | ANALYSIS : 69.0%, 3.0, 107.8ms, 0 | 20260507 19:04:39.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:39.705 | INFO | RT_METRICS : 41.9%, 1.0, 21.9ms, 0 | 20260507 19:04:39.705 | INFO | PRE_HQ : 66.0%, 1.7, 59.0ms, 0 | 20260507 19:04:39.705 | INFO | BAZWRITER : 68.6%, 2.9, 103.3ms, 0 | 20260507 19:04:39.705 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:04:39.706 | INFO | Uploaded 36GiB to GPU in 2.53917s (14.1779GiB/s) | 20260507 19:04:39.706 | INFO | Downloaded 38GiB to GPU in 2.68932s (14.13GiB/s) | 20260507 19:04:44.843 | INFO | Analyzing chunk frames = [1868288,1868800) | 20260507 19:04:44.843 | INFO | MemInfo: { | 20260507 19:04:44.843 | INFO | "gpu" : " 17 GB", | 20260507 19:04:44.843 | INFO | "huge" : "", | 20260507 19:04:44.843 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:44.843 | INFO | "malloc" : " 14 GB", | 20260507 19:04:44.843 | INFO | "pinned" : " 29 GB" | 20260507 19:04:44.843 | INFO | } | 20260507 19:04:44.844 | INFO | Perf report for the last 5.13873 seconds: | 20260507 19:04:44.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:44.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 19:04:44.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 19:04:44.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:44.844 | INFO | Uploaded 36GiB to GPU in 2.50591s (14.366GiB/s) | 20260507 19:04:44.844 | INFO | Downloaded 34GiB to GPU in 2.48512s (13.6814GiB/s) | 20260507 19:04:49.985 | INFO | Analyzing chunk frames = [1868800,1869312) | 20260507 19:04:49.985 | INFO | MemInfo: { | 20260507 19:04:49.985 | INFO | "gpu" : " 17 GB", | 20260507 19:04:49.985 | INFO | "huge" : "", | 20260507 19:04:49.985 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:49.985 | INFO | "malloc" : " 14 GB", | 20260507 19:04:49.985 | INFO | "pinned" : " 29 GB" | 20260507 19:04:49.985 | INFO | } | 20260507 19:04:49.985 | INFO | Perf report for the last 5.14138 seconds: | 20260507 19:04:49.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:49.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:04:49.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:04:49.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:49.985 | INFO | Uploaded 36GiB to GPU in 2.53195s (14.2183GiB/s) | 20260507 19:04:49.985 | INFO | Downloaded 34GiB to GPU in 2.45439s (13.8528GiB/s) | 20260507 19:04:55.122 | INFO | Analyzing chunk frames = [1869312,1869824) | 20260507 19:04:55.122 | INFO | MemInfo: { | 20260507 19:04:55.122 | INFO | "gpu" : " 17 GB", | 20260507 19:04:55.122 | INFO | "huge" : "", | 20260507 19:04:55.122 | INFO | "huge_shared" : " 29 GB", | 20260507 19:04:55.122 | INFO | "malloc" : " 14 GB", | 20260507 19:04:55.122 | INFO | "pinned" : " 29 GB" | 20260507 19:04:55.122 | INFO | } | 20260507 19:04:55.122 | INFO | Perf report for the last 5.13742 seconds: | 20260507 19:04:55.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:04:55.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.122 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:04:55.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.122 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:04:55.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:04:55.123 | INFO | Uploaded 36GiB to GPU in 2.44838s (14.7036GiB/s) | 20260507 19:04:55.123 | INFO | Downloaded 34GiB to GPU in 2.38358s (14.2642GiB/s) | 20260507 19:05:00.262 | INFO | Analyzing chunk frames = [1869824,1870336) | 20260507 19:05:00.262 | INFO | MemInfo: { | 20260507 19:05:00.262 | INFO | "gpu" : " 17 GB", | 20260507 19:05:00.262 | INFO | "huge" : "", | 20260507 19:05:00.262 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:00.262 | INFO | "malloc" : " 14 GB", | 20260507 19:05:00.262 | INFO | "pinned" : " 29 GB" | 20260507 19:05:00.262 | INFO | } | 20260507 19:05:00.262 | INFO | Perf report for the last 5.13965 seconds: | 20260507 19:05:00.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:00.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:05:00.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:05:00.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:00.262 | INFO | Uploaded 36GiB to GPU in 2.36847s (15.1997GiB/s) | 20260507 19:05:00.262 | INFO | Downloaded 34GiB to GPU in 2.49016s (13.6537GiB/s) | 20260507 19:05:05.401 | INFO | Analyzing chunk frames = [1870336,1870848) | 20260507 19:05:05.401 | INFO | MemInfo: { | 20260507 19:05:05.401 | INFO | "gpu" : " 17 GB", | 20260507 19:05:05.401 | INFO | "huge" : "", | 20260507 19:05:05.401 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:05.401 | INFO | "malloc" : " 14 GB", | 20260507 19:05:05.401 | INFO | "pinned" : " 29 GB" | 20260507 19:05:05.401 | INFO | } | 20260507 19:05:05.401 | INFO | Perf report for the last 5.13934 seconds: | 20260507 19:05:05.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:05.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.401 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:05:05.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.402 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260507 19:05:05.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:05.402 | INFO | Uploaded 36GiB to GPU in 2.50466s (14.3732GiB/s) | 20260507 19:05:05.402 | INFO | Downloaded 34GiB to GPU in 2.40876s (14.1152GiB/s) | 20260507 19:05:10.541 | INFO | Analyzing chunk frames = [1870848,1871360) | 20260507 19:05:10.541 | INFO | MemInfo: { | 20260507 19:05:10.541 | INFO | "gpu" : " 17 GB", | 20260507 19:05:10.541 | INFO | "huge" : "", | 20260507 19:05:10.541 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:10.541 | INFO | "malloc" : " 14 GB", | 20260507 19:05:10.541 | INFO | "pinned" : " 29 GB" | 20260507 19:05:10.541 | INFO | } | 20260507 19:05:10.541 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:05:10.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:10.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.541 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:05:10.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.541 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:05:10.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:10.542 | INFO | Uploaded 36GiB to GPU in 2.554s (14.0955GiB/s) | 20260507 19:05:10.542 | INFO | Downloaded 34GiB to GPU in 2.436s (13.9573GiB/s) | 20260507 19:05:15.680 | INFO | Analyzing chunk frames = [1871360,1871872) | 20260507 19:05:15.680 | INFO | MemInfo: { | 20260507 19:05:15.680 | INFO | "gpu" : " 17 GB", | 20260507 19:05:15.680 | INFO | "huge" : "", | 20260507 19:05:15.680 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:15.680 | INFO | "malloc" : " 14 GB", | 20260507 19:05:15.680 | INFO | "pinned" : " 29 GB" | 20260507 19:05:15.680 | INFO | } | 20260507 19:05:15.680 | INFO | Perf report for the last 5.13908 seconds: | 20260507 19:05:15.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:15.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.680 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 19:05:15.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.680 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:05:15.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:15.681 | INFO | Uploaded 36GiB to GPU in 2.52601s (14.2517GiB/s) | 20260507 19:05:15.681 | INFO | Downloaded 34GiB to GPU in 2.4273s (14.0073GiB/s) | 20260507 19:05:15.777 | INFO | Starting a round of Baz IO | 20260507 19:05:19.260 | INFO | Populating RTMetrics report | 20260507 19:05:19.260 | INFO | RTMetrics report populated | 20260507 19:05:19.260 | INFO | Opening tmp RTMetrics json file | 20260507 19:05:19.275 | INFO | Serializing RTMetrics to json | 20260507 19:05:19.277 | INFO | Serialization done, swapping tmp file now... | 20260507 19:05:19.278 | INFO | RTMetrics done | 20260507 19:05:19.340 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.562 seconds | 20260507 19:05:19.340 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.81% / 0.01% / 54.18% | 20260507 19:05:19.340 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.59311% / 1.19209e-05% / 3.57628e-05% | 20260507 19:05:19.340 | INFO | Baz IO overall: | 20260507 19:05:19.340 | INFO | BytesWritten : 842620431668 | 20260507 19:05:19.340 | INFO | NumEvents : 408543163719 | 20260507 19:05:20.820 | INFO | Analyzing chunk frames = [1871872,1872384) | 20260507 19:05:20.820 | INFO | MemInfo: { | 20260507 19:05:20.820 | INFO | "gpu" : " 17 GB", | 20260507 19:05:20.820 | INFO | "huge" : "", | 20260507 19:05:20.820 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:20.820 | INFO | "malloc" : " 14 GB", | 20260507 19:05:20.820 | INFO | "pinned" : " 29 GB" | 20260507 19:05:20.820 | INFO | } | 20260507 19:05:20.820 | INFO | Perf report for the last 5.14008 seconds: | 20260507 19:05:20.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:20.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:20.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:20.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:20.820 | INFO | ANALYSIS : 68.9%, 3.0, 107.5ms, 0 | 20260507 19:05:20.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:20.820 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:05:20.820 | INFO | PRE_HQ : 66.0%, 1.7, 58.7ms, 0 | 20260507 19:05:20.820 | INFO | BAZWRITER : 68.5%, 2.9, 103.2ms, 0 | 20260507 19:05:20.820 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:05:20.821 | INFO | Uploaded 36GiB to GPU in 2.52614s (14.251GiB/s) | 20260507 19:05:20.821 | INFO | Downloaded 38GiB to GPU in 2.67881s (14.1854GiB/s) | 20260507 19:05:25.959 | INFO | Analyzing chunk frames = [1872384,1872896) | 20260507 19:05:25.959 | INFO | MemInfo: { | 20260507 19:05:25.959 | INFO | "gpu" : " 17 GB", | 20260507 19:05:25.959 | INFO | "huge" : "", | 20260507 19:05:25.959 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:25.959 | INFO | "malloc" : " 14 GB", | 20260507 19:05:25.959 | INFO | "pinned" : " 29 GB" | 20260507 19:05:25.959 | INFO | } | 20260507 19:05:25.959 | INFO | Perf report for the last 5.13914 seconds: | 20260507 19:05:25.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:25.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.959 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:05:25.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.960 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:05:25.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:25.960 | INFO | Uploaded 36GiB to GPU in 2.5624s (14.0493GiB/s) | 20260507 19:05:25.960 | INFO | Downloaded 34GiB to GPU in 2.48083s (13.7051GiB/s) | 20260507 19:05:31.098 | INFO | Analyzing chunk frames = [1872896,1873408) | 20260507 19:05:31.098 | INFO | MemInfo: { | 20260507 19:05:31.098 | INFO | "gpu" : " 17 GB", | 20260507 19:05:31.098 | INFO | "huge" : "", | 20260507 19:05:31.098 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:31.098 | INFO | "malloc" : " 14 GB", | 20260507 19:05:31.098 | INFO | "pinned" : " 29 GB" | 20260507 19:05:31.098 | INFO | } | 20260507 19:05:31.098 | INFO | Perf report for the last 5.13864 seconds: | 20260507 19:05:31.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:31.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.098 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:05:31.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.098 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:05:31.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:31.099 | INFO | Uploaded 36GiB to GPU in 2.56649s (14.027GiB/s) | 20260507 19:05:31.099 | INFO | Downloaded 34GiB to GPU in 2.45924s (13.8254GiB/s) | 20260507 19:05:36.238 | INFO | Analyzing chunk frames = [1873408,1873920) | 20260507 19:05:36.238 | INFO | MemInfo: { | 20260507 19:05:36.238 | INFO | "gpu" : " 17 GB", | 20260507 19:05:36.238 | INFO | "huge" : "", | 20260507 19:05:36.238 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:36.238 | INFO | "malloc" : " 14 GB", | 20260507 19:05:36.238 | INFO | "pinned" : " 29 GB" | 20260507 19:05:36.238 | INFO | } | 20260507 19:05:36.238 | INFO | Perf report for the last 5.13992 seconds: | 20260507 19:05:36.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:36.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:05:36.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:05:36.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:36.238 | INFO | Uploaded 36GiB to GPU in 2.48572s (14.4827GiB/s) | 20260507 19:05:36.238 | INFO | Downloaded 34GiB to GPU in 2.38939s (14.2295GiB/s) | 20260507 19:05:41.377 | INFO | Analyzing chunk frames = [1873920,1874432) | 20260507 19:05:41.377 | INFO | MemInfo: { | 20260507 19:05:41.377 | INFO | "gpu" : " 17 GB", | 20260507 19:05:41.377 | INFO | "huge" : "", | 20260507 19:05:41.378 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:41.378 | INFO | "malloc" : " 14 GB", | 20260507 19:05:41.378 | INFO | "pinned" : " 29 GB" | 20260507 19:05:41.378 | INFO | } | 20260507 19:05:41.378 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:05:41.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:41.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:05:41.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 19:05:41.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:41.378 | INFO | Uploaded 36GiB to GPU in 2.36519s (15.2208GiB/s) | 20260507 19:05:41.378 | INFO | Downloaded 34GiB to GPU in 2.49142s (13.6468GiB/s) | 20260507 19:05:46.516 | INFO | Analyzing chunk frames = [1874432,1874944) | 20260507 19:05:46.516 | INFO | MemInfo: { | 20260507 19:05:46.516 | INFO | "gpu" : " 17 GB", | 20260507 19:05:46.516 | INFO | "huge" : "", | 20260507 19:05:46.516 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:46.516 | INFO | "malloc" : " 14 GB", | 20260507 19:05:46.516 | INFO | "pinned" : " 29 GB" | 20260507 19:05:46.516 | INFO | } | 20260507 19:05:46.516 | INFO | Perf report for the last 5.13836 seconds: | 20260507 19:05:46.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:46.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:05:46.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:05:46.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:46.516 | INFO | Uploaded 36GiB to GPU in 2.48357s (14.4953GiB/s) | 20260507 19:05:46.516 | INFO | Downloaded 34GiB to GPU in 2.42071s (14.0455GiB/s) | 20260507 19:05:51.655 | INFO | Analyzing chunk frames = [1874944,1875456) | 20260507 19:05:51.655 | INFO | MemInfo: { | 20260507 19:05:51.655 | INFO | "gpu" : " 17 GB", | 20260507 19:05:51.655 | INFO | "huge" : "", | 20260507 19:05:51.655 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:51.655 | INFO | "malloc" : " 14 GB", | 20260507 19:05:51.655 | INFO | "pinned" : " 29 GB" | 20260507 19:05:51.655 | INFO | } | 20260507 19:05:51.655 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:05:51.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:51.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:05:51.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 19:05:51.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:51.655 | INFO | Uploaded 36GiB to GPU in 2.51282s (14.3265GiB/s) | 20260507 19:05:51.656 | INFO | Downloaded 34GiB to GPU in 2.44334s (13.9154GiB/s) | 20260507 19:05:56.795 | INFO | Analyzing chunk frames = [1875456,1875968) | 20260507 19:05:56.795 | INFO | MemInfo: { | 20260507 19:05:56.795 | INFO | "gpu" : " 17 GB", | 20260507 19:05:56.795 | INFO | "huge" : "", | 20260507 19:05:56.795 | INFO | "huge_shared" : " 29 GB", | 20260507 19:05:56.795 | INFO | "malloc" : " 14 GB", | 20260507 19:05:56.795 | INFO | "pinned" : " 29 GB" | 20260507 19:05:56.795 | INFO | } | 20260507 19:05:56.795 | INFO | Perf report for the last 5.14063 seconds: | 20260507 19:05:56.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:05:56.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.796 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:05:56.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.796 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:05:56.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:05:56.796 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:05:56.796 | INFO | Total mem usage: 16.4 GiB | 20260507 19:05:56.796 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:05:56.796 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 19:05:56.796 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:05:56.796 | INFO | High Water Breakdown: | 20260507 19:05:56.796 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:05:56.796 | INFO | 40.6% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:05:56.796 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:05:56.796 | INFO | ------------------------------------------ | 20260507 19:05:56.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:05:56.796 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:05:56.796 | INFO | Total mem usage: 26.5 GiB | 20260507 19:05:56.796 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:05:56.796 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:05:56.796 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:05:56.796 | INFO | High Water Breakdown: | 20260507 19:05:56.796 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:05:56.796 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:05:56.796 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:05:56.796 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:05:56.796 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:05:56.796 | INFO | Uploaded 36GiB to GPU in 2.49217s (14.4452GiB/s) | 20260507 19:05:56.796 | INFO | Downloaded 34GiB to GPU in 2.43058s (13.9884GiB/s) | 20260507 19:05:56.796 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:05:56.796 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:05:56.796 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:05:56.796 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:05:56.796 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:05:56.796 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:05:56.796 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:05:56.796 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:05:56.796 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:05:56.796 | INFO | ------------------------------------------ | 20260507 19:05:56.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:05:56.796 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:05:56.796 | INFO | Total mem usage: 27.8 GiB | 20260507 19:05:56.796 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:05:56.797 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:05:56.797 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:05:56.797 | INFO | High Water Breakdown: | 20260507 19:05:56.797 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:05:56.797 | INFO | ------------------------------------------ | 20260507 19:05:56.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:05:56.797 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:05:56.797 | INFO | Total mem usage: 15.7 GiB | 20260507 19:05:56.797 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:05:56.797 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:05:56.797 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:05:56.797 | INFO | High Water Breakdown: | 20260507 19:05:56.797 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:05:56.797 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:05:56.797 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:05:56.797 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:05:56.797 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:05:56.797 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:05:56.797 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:05:56.797 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:05:56.797 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:05:56.797 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:05:56.797 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:05:56.797 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:05:56.797 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:05:56.797 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:05:56.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:05:56.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:05:56.797 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:05:56.797 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:05:56.797 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:05:56.797 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:05:56.797 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:05:56.797 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:05:56.797 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:05:56.797 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:05:56.797 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:05:56.797 | INFO | ------------------------------------------ | 20260507 19:05:56.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:05:56.797 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:05:56.797 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:05:56.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:05:56.797 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.41e+01 / 1.80e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.12e-02 / 6.71e-03 | 20260507 19:05:56.797 | INFO | PerfReport: 12.07%, Baseline -- 4.01e+00 / 4.47e+00, 6.08e-01 / 3.01e-01 | 20260507 19:05:56.797 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.49e+00, 2.13e-01 / 1.26e-01 | 20260507 19:05:56.797 | INFO | PerfReport: 15.77%, DME -- 4.19e+01 / 4.50e+01, 3.97e+00 / 1.27e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.81e+00 / 1.72e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.52e+00 / 2.89e+00, 4.40e-01 / 1.85e-01 | 20260507 19:05:56.797 | INFO | PerfReport: 3.69%, Metrics -- 1.22e+00 / 1.44e+00, 3.10e-01 / 1.84e-01 | 20260507 19:05:56.797 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.81e-01 / 4.14e-01, 6.20e-02 / 3.22e-02 | 20260507 19:05:56.797 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:05:56.797 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:05:56.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:05:56.797 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 24.29%, Upload -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 24.05%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 30.99%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 2.03%, LockUpload -- 2.18e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 2.38%, LockDownload -- 2.55e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 16.25%, LockCompute -- 1.74e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 19:05:56.797 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.57e-02 / 0.00e+00 | 20260507 19:05:56.893 | INFO | Starting a round of Baz IO | 20260507 19:06:00.382 | INFO | Populating RTMetrics report | 20260507 19:06:00.382 | INFO | RTMetrics report populated | 20260507 19:06:00.382 | INFO | Opening tmp RTMetrics json file | 20260507 19:06:00.398 | INFO | Serializing RTMetrics to json | 20260507 19:06:00.400 | INFO | Serialization done, swapping tmp file now... | 20260507 19:06:00.400 | INFO | RTMetrics done | 20260507 19:06:00.457 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.563 seconds | 20260507 19:06:00.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.80% / 0.01% / 54.19% | 20260507 19:06:00.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.59011% / 1.98682e-05% / 2.38419e-05% | 20260507 19:06:00.457 | INFO | Baz IO overall: | 20260507 19:06:00.457 | INFO | BytesWritten : 844551044301 | 20260507 19:06:00.457 | INFO | NumEvents : 409479411175 | 20260507 19:06:01.935 | INFO | Analyzing chunk frames = [1875968,1876480) | 20260507 19:06:01.935 | INFO | MemInfo: { | 20260507 19:06:01.935 | INFO | "gpu" : " 17 GB", | 20260507 19:06:01.935 | INFO | "huge" : "", | 20260507 19:06:01.935 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:01.935 | INFO | "malloc" : " 14 GB", | 20260507 19:06:01.935 | INFO | "pinned" : " 29 GB" | 20260507 19:06:01.935 | INFO | } | 20260507 19:06:01.935 | INFO | Perf report for the last 5.13929 seconds: | 20260507 19:06:01.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:01.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:01.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:01.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:01.935 | INFO | ANALYSIS : 68.9%, 3.0, 107.6ms, 0 | 20260507 19:06:01.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:01.935 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:06:01.935 | INFO | PRE_HQ : 65.9%, 1.7, 58.7ms, 0 | 20260507 19:06:01.935 | INFO | BAZWRITER : 68.5%, 2.9, 103.4ms, 0 | 20260507 19:06:01.935 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 19:06:01.935 | INFO | Uploaded 36GiB to GPU in 2.49002s (14.4577GiB/s) | 20260507 19:06:01.935 | INFO | Downloaded 38GiB to GPU in 2.67693s (14.1954GiB/s) | 20260507 19:06:07.073 | INFO | Analyzing chunk frames = [1876480,1876992) | 20260507 19:06:07.073 | INFO | MemInfo: { | 20260507 19:06:07.073 | INFO | "gpu" : " 17 GB", | 20260507 19:06:07.073 | INFO | "huge" : "", | 20260507 19:06:07.074 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:07.074 | INFO | "malloc" : " 14 GB", | 20260507 19:06:07.074 | INFO | "pinned" : " 29 GB" | 20260507 19:06:07.074 | INFO | } | 20260507 19:06:07.074 | INFO | Perf report for the last 5.13894 seconds: | 20260507 19:06:07.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:07.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 19:06:07.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:06:07.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:07.074 | INFO | Uploaded 36GiB to GPU in 2.54449s (14.1482GiB/s) | 20260507 19:06:07.074 | INFO | Downloaded 34GiB to GPU in 2.48903s (13.66GiB/s) | 20260507 19:06:12.213 | INFO | Analyzing chunk frames = [1876992,1877504) | 20260507 19:06:12.214 | INFO | MemInfo: { | 20260507 19:06:12.214 | INFO | "gpu" : " 17 GB", | 20260507 19:06:12.214 | INFO | "huge" : "", | 20260507 19:06:12.214 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:12.214 | INFO | "malloc" : " 14 GB", | 20260507 19:06:12.214 | INFO | "pinned" : " 29 GB" | 20260507 19:06:12.214 | INFO | } | 20260507 19:06:12.214 | INFO | Perf report for the last 5.14005 seconds: | 20260507 19:06:12.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:12.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:06:12.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:06:12.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:12.214 | INFO | Uploaded 36GiB to GPU in 2.53554s (14.1981GiB/s) | 20260507 19:06:12.214 | INFO | Downloaded 34GiB to GPU in 2.46351s (13.8015GiB/s) | 20260507 19:06:17.352 | INFO | Analyzing chunk frames = [1877504,1878016) | 20260507 19:06:17.352 | INFO | MemInfo: { | 20260507 19:06:17.352 | INFO | "gpu" : " 17 GB", | 20260507 19:06:17.352 | INFO | "huge" : "", | 20260507 19:06:17.352 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:17.352 | INFO | "malloc" : " 14 GB", | 20260507 19:06:17.352 | INFO | "pinned" : " 29 GB" | 20260507 19:06:17.352 | INFO | } | 20260507 19:06:17.352 | INFO | Perf report for the last 5.13841 seconds: | 20260507 19:06:17.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:17.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.352 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:06:17.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.352 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:06:17.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:17.353 | INFO | Uploaded 36GiB to GPU in 2.45362s (14.6722GiB/s) | 20260507 19:06:17.353 | INFO | Downloaded 34GiB to GPU in 2.38658s (14.2463GiB/s) | 20260507 19:06:22.491 | INFO | Analyzing chunk frames = [1878016,1878528) | 20260507 19:06:22.491 | INFO | MemInfo: { | 20260507 19:06:22.491 | INFO | "gpu" : " 17 GB", | 20260507 19:06:22.491 | INFO | "huge" : "", | 20260507 19:06:22.491 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:22.491 | INFO | "malloc" : " 14 GB", | 20260507 19:06:22.491 | INFO | "pinned" : " 29 GB" | 20260507 19:06:22.491 | INFO | } | 20260507 19:06:22.491 | INFO | Perf report for the last 5.13918 seconds: | 20260507 19:06:22.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:22.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.491 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:06:22.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.492 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:06:22.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:22.492 | INFO | Uploaded 36GiB to GPU in 2.3552s (15.2853GiB/s) | 20260507 19:06:22.492 | INFO | Downloaded 34GiB to GPU in 2.49512s (13.6266GiB/s) | 20260507 19:06:27.630 | INFO | Analyzing chunk frames = [1878528,1879040) | 20260507 19:06:27.630 | INFO | MemInfo: { | 20260507 19:06:27.630 | INFO | "gpu" : " 17 GB", | 20260507 19:06:27.631 | INFO | "huge" : "", | 20260507 19:06:27.631 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:27.631 | INFO | "malloc" : " 14 GB", | 20260507 19:06:27.631 | INFO | "pinned" : " 29 GB" | 20260507 19:06:27.631 | INFO | } | 20260507 19:06:27.631 | INFO | Perf report for the last 5.13921 seconds: | 20260507 19:06:27.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:27.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:06:27.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:06:27.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:27.631 | INFO | Uploaded 36GiB to GPU in 2.51214s (14.3304GiB/s) | 20260507 19:06:27.631 | INFO | Downloaded 34GiB to GPU in 2.40232s (14.153GiB/s) | 20260507 19:06:32.770 | INFO | Analyzing chunk frames = [1879040,1879552) | 20260507 19:06:32.770 | INFO | MemInfo: { | 20260507 19:06:32.770 | INFO | "gpu" : " 17 GB", | 20260507 19:06:32.770 | INFO | "huge" : "", | 20260507 19:06:32.770 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:32.770 | INFO | "malloc" : " 14 GB", | 20260507 19:06:32.770 | INFO | "pinned" : " 29 GB" | 20260507 19:06:32.770 | INFO | } | 20260507 19:06:32.770 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:06:32.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:32.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.770 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:06:32.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.771 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:06:32.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:32.771 | INFO | Uploaded 36GiB to GPU in 2.5528s (14.1022GiB/s) | 20260507 19:06:32.771 | INFO | Downloaded 34GiB to GPU in 2.43383s (13.9698GiB/s) | 20260507 19:06:37.910 | INFO | Analyzing chunk frames = [1879552,1880064) | 20260507 19:06:37.910 | INFO | MemInfo: { | 20260507 19:06:37.910 | INFO | "gpu" : " 17 GB", | 20260507 19:06:37.910 | INFO | "huge" : "", | 20260507 19:06:37.910 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:37.910 | INFO | "malloc" : " 14 GB", | 20260507 19:06:37.910 | INFO | "pinned" : " 29 GB" | 20260507 19:06:37.910 | INFO | } | 20260507 19:06:37.910 | INFO | Perf report for the last 5.13988 seconds: | 20260507 19:06:37.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:37.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.910 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:06:37.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.910 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:06:37.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:37.911 | INFO | Uploaded 36GiB to GPU in 2.52435s (14.2611GiB/s) | 20260507 19:06:37.911 | INFO | Downloaded 34GiB to GPU in 2.41607s (14.0724GiB/s) | 20260507 19:06:38.007 | INFO | Starting a round of Baz IO | 20260507 19:06:41.497 | INFO | Populating RTMetrics report | 20260507 19:06:41.498 | INFO | RTMetrics report populated | 20260507 19:06:41.498 | INFO | Opening tmp RTMetrics json file | 20260507 19:06:41.514 | INFO | Serializing RTMetrics to json | 20260507 19:06:41.515 | INFO | Serialization done, swapping tmp file now... | 20260507 19:06:41.516 | INFO | RTMetrics done | 20260507 19:06:41.575 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.567 seconds | 20260507 19:06:41.575 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.79% / 0.01% / 54.20% | 20260507 19:06:41.575 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.58632% / 7.94729e-06% / 4.76837e-05% | 20260507 19:06:41.575 | INFO | Baz IO overall: | 20260507 19:06:41.575 | INFO | BytesWritten : 846482179410 | 20260507 19:06:41.575 | INFO | NumEvents : 410416047842 | 20260507 19:06:43.051 | INFO | Analyzing chunk frames = [1880064,1880576) | 20260507 19:06:43.051 | INFO | MemInfo: { | 20260507 19:06:43.051 | INFO | "gpu" : " 17 GB", | 20260507 19:06:43.051 | INFO | "huge" : "", | 20260507 19:06:43.051 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:43.051 | INFO | "malloc" : " 14 GB", | 20260507 19:06:43.051 | INFO | "pinned" : " 29 GB" | 20260507 19:06:43.051 | INFO | } | 20260507 19:06:43.051 | INFO | Perf report for the last 5.14058 seconds: | 20260507 19:06:43.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:43.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:43.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:43.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:43.051 | INFO | ANALYSIS : 69.0%, 3.0, 107.8ms, 0 | 20260507 19:06:43.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:43.051 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:06:43.051 | INFO | PRE_HQ : 66.0%, 1.7, 58.8ms, 0 | 20260507 19:06:43.051 | INFO | BAZWRITER : 68.5%, 2.9, 102.6ms, 0 | 20260507 19:06:43.051 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 19:06:43.051 | INFO | Uploaded 36GiB to GPU in 2.55206s (14.1063GiB/s) | 20260507 19:06:43.051 | INFO | Downloaded 38GiB to GPU in 2.6814s (14.1717GiB/s) | 20260507 19:06:48.189 | INFO | Analyzing chunk frames = [1880576,1881088) | 20260507 19:06:48.189 | INFO | MemInfo: { | 20260507 19:06:48.189 | INFO | "gpu" : " 17 GB", | 20260507 19:06:48.189 | INFO | "huge" : "", | 20260507 19:06:48.189 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:48.189 | INFO | "malloc" : " 14 GB", | 20260507 19:06:48.189 | INFO | "pinned" : " 29 GB" | 20260507 19:06:48.189 | INFO | } | 20260507 19:06:48.189 | INFO | Perf report for the last 5.1384 seconds: | 20260507 19:06:48.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:48.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.189 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:06:48.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.189 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 19:06:48.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:48.190 | INFO | Uploaded 36GiB to GPU in 2.54698s (14.1344GiB/s) | 20260507 19:06:48.190 | INFO | Downloaded 34GiB to GPU in 2.48169s (13.7003GiB/s) | 20260507 19:06:53.328 | INFO | Analyzing chunk frames = [1881088,1881600) | 20260507 19:06:53.329 | INFO | MemInfo: { | 20260507 19:06:53.329 | INFO | "gpu" : " 17 GB", | 20260507 19:06:53.329 | INFO | "huge" : "", | 20260507 19:06:53.329 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:53.329 | INFO | "malloc" : " 14 GB", | 20260507 19:06:53.329 | INFO | "pinned" : " 29 GB" | 20260507 19:06:53.329 | INFO | } | 20260507 19:06:53.329 | INFO | Perf report for the last 5.13945 seconds: | 20260507 19:06:53.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:53.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:06:53.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:06:53.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:53.329 | INFO | Uploaded 36GiB to GPU in 2.57235s (13.995GiB/s) | 20260507 19:06:53.329 | INFO | Downloaded 34GiB to GPU in 2.45003s (13.8774GiB/s) | 20260507 19:06:58.466 | INFO | Analyzing chunk frames = [1881600,1882112) | 20260507 19:06:58.466 | INFO | MemInfo: { | 20260507 19:06:58.466 | INFO | "gpu" : " 17 GB", | 20260507 19:06:58.467 | INFO | "huge" : "", | 20260507 19:06:58.467 | INFO | "huge_shared" : " 29 GB", | 20260507 19:06:58.467 | INFO | "malloc" : " 14 GB", | 20260507 19:06:58.467 | INFO | "pinned" : " 29 GB" | 20260507 19:06:58.467 | INFO | } | 20260507 19:06:58.467 | INFO | Perf report for the last 5.13802 seconds: | 20260507 19:06:58.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:06:58.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:06:58.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:06:58.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:06:58.467 | INFO | Uploaded 36GiB to GPU in 2.47383s (14.5523GiB/s) | 20260507 19:06:58.467 | INFO | Downloaded 34GiB to GPU in 2.38243s (14.2711GiB/s) | 20260507 19:07:03.607 | INFO | Analyzing chunk frames = [1882112,1882624) | 20260507 19:07:03.607 | INFO | MemInfo: { | 20260507 19:07:03.607 | INFO | "gpu" : " 17 GB", | 20260507 19:07:03.607 | INFO | "huge" : "", | 20260507 19:07:03.607 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:03.607 | INFO | "malloc" : " 14 GB", | 20260507 19:07:03.607 | INFO | "pinned" : " 29 GB" | 20260507 19:07:03.607 | INFO | } | 20260507 19:07:03.607 | INFO | Perf report for the last 5.14052 seconds: | 20260507 19:07:03.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:03.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.607 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:07:03.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.607 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:07:03.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:03.608 | INFO | Uploaded 36GiB to GPU in 2.37967s (15.1281GiB/s) | 20260507 19:07:03.608 | INFO | Downloaded 34GiB to GPU in 2.49834s (13.6091GiB/s) | 20260507 19:07:08.748 | INFO | Analyzing chunk frames = [1882624,1883136) | 20260507 19:07:08.748 | INFO | MemInfo: { | 20260507 19:07:08.748 | INFO | "gpu" : " 17 GB", | 20260507 19:07:08.748 | INFO | "huge" : "", | 20260507 19:07:08.748 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:08.748 | INFO | "malloc" : " 14 GB", | 20260507 19:07:08.748 | INFO | "pinned" : " 29 GB" | 20260507 19:07:08.748 | INFO | } | 20260507 19:07:08.748 | INFO | Perf report for the last 5.1407 seconds: | 20260507 19:07:08.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:08.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:07:08.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:07:08.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:08.748 | INFO | Uploaded 36GiB to GPU in 2.47554s (14.5423GiB/s) | 20260507 19:07:08.748 | INFO | Downloaded 34GiB to GPU in 2.41491s (14.0792GiB/s) | 20260507 19:07:13.886 | INFO | Analyzing chunk frames = [1883136,1883648) | 20260507 19:07:13.886 | INFO | MemInfo: { | 20260507 19:07:13.886 | INFO | "gpu" : " 17 GB", | 20260507 19:07:13.886 | INFO | "huge" : "", | 20260507 19:07:13.886 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:13.886 | INFO | "malloc" : " 14 GB", | 20260507 19:07:13.886 | INFO | "pinned" : " 29 GB" | 20260507 19:07:13.886 | INFO | } | 20260507 19:07:13.886 | INFO | Perf report for the last 5.13814 seconds: | 20260507 19:07:13.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:13.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:07:13.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:07:13.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:13.886 | INFO | Uploaded 36GiB to GPU in 2.51042s (14.3402GiB/s) | 20260507 19:07:13.886 | INFO | Downloaded 34GiB to GPU in 2.44522s (13.9047GiB/s) | 20260507 19:07:19.025 | INFO | Analyzing chunk frames = [1883648,1884160) | 20260507 19:07:19.025 | INFO | MemInfo: { | 20260507 19:07:19.025 | INFO | "gpu" : " 17 GB", | 20260507 19:07:19.025 | INFO | "huge" : "", | 20260507 19:07:19.025 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:19.026 | INFO | "malloc" : " 14 GB", | 20260507 19:07:19.026 | INFO | "pinned" : " 29 GB" | 20260507 19:07:19.026 | INFO | } | 20260507 19:07:19.026 | INFO | Perf report for the last 5.13942 seconds: | 20260507 19:07:19.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:19.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:07:19.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:07:19.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:19.026 | INFO | Uploaded 36GiB to GPU in 2.48455s (14.4895GiB/s) | 20260507 19:07:19.026 | INFO | Downloaded 34GiB to GPU in 2.42525s (14.0192GiB/s) | 20260507 19:07:19.026 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:07:19.026 | INFO | Total mem usage: 16.4 GiB | 20260507 19:07:19.026 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:07:19.026 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.7%) | 20260507 19:07:19.026 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:07:19.026 | INFO | High Water Breakdown: | 20260507 19:07:19.026 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:07:19.026 | INFO | 40.6% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:07:19.026 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:07:19.026 | INFO | ------------------------------------------ | 20260507 19:07:19.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:07:19.026 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:07:19.026 | INFO | Total mem usage: 26.5 GiB | 20260507 19:07:19.026 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:07:19.026 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:07:19.026 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:07:19.026 | INFO | High Water Breakdown: | 20260507 19:07:19.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:07:19.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:07:19.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:07:19.026 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:07:19.026 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:07:19.026 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:07:19.026 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:07:19.026 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:07:19.027 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:07:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:07:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:07:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:07:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:07:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:07:19.027 | INFO | ------------------------------------------ | 20260507 19:07:19.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:07:19.027 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:07:19.027 | INFO | Total mem usage: 27.8 GiB | 20260507 19:07:19.027 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:07:19.027 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:07:19.027 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:07:19.027 | INFO | High Water Breakdown: | 20260507 19:07:19.027 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:07:19.027 | INFO | ------------------------------------------ | 20260507 19:07:19.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:07:19.027 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:07:19.027 | INFO | Total mem usage: 15.7 GiB | 20260507 19:07:19.027 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:07:19.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:07:19.027 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 19:07:19.027 | INFO | High Water Breakdown: | 20260507 19:07:19.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:07:19.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:07:19.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:07:19.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:07:19.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:07:19.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:07:19.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:07:19.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:07:19.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:07:19.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:07:19.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:07:19.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:07:19.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:07:19.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:07:19.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:07:19.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:07:19.027 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:07:19.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:07:19.027 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:07:19.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:07:19.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:07:19.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:07:19.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:07:19.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:07:19.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:07:19.027 | INFO | ------------------------------------------ | 20260507 19:07:19.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:07:19.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:07:19.027 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:07:19.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:07:19.027 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.42e+01 / 1.79e+00 | 20260507 19:07:19.027 | INFO | PerfReport: 0.29%, AggregateStats -- 9.72e-02 / 1.01e-01, 1.16e-02 / 5.12e-03 | 20260507 19:07:19.027 | INFO | PerfReport: 12.06%, Baseline -- 4.00e+00 / 4.49e+00, 5.98e-01 / 2.77e-01 | 20260507 19:07:19.027 | INFO | PerfReport: 3.41%, Histogram -- 1.13e+00 / 1.51e+00, 2.13e-01 / 1.26e-01 | 20260507 19:07:19.027 | INFO | PerfReport: 15.78%, DME -- 4.19e+01 / 4.50e+01, 3.88e+00 / 1.32e+00 | 20260507 19:07:19.027 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.78e+00 / 1.78e+00 | 20260507 19:07:19.027 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.51e+00 / 2.88e+00, 4.35e-01 / 2.15e-01 | 20260507 19:07:19.027 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.46e+00, 3.08e-01 / 1.98e-01 | 20260507 19:07:19.027 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.81e-01 / 4.15e-01, 6.12e-02 / 3.31e-02 | 20260507 19:07:19.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:07:19.027 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:07:19.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:07:19.027 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 19:07:19.027 | INFO | PerfReport: 24.28%, Upload -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260507 19:07:19.027 | INFO | PerfReport: 24.03%, Download -- 2.57e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 19:07:19.027 | INFO | PerfReport: 30.97%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:07:19.028 | INFO | PerfReport: 2.07%, LockUpload -- 2.21e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 19:07:19.028 | INFO | PerfReport: 2.39%, LockDownload -- 2.56e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260507 19:07:19.028 | INFO | PerfReport: 16.25%, LockCompute -- 1.74e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:07:19.028 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.45e-02 / 0.00e+00 | 20260507 19:07:19.138 | INFO | Starting a round of Baz IO | 20260507 19:07:22.602 | INFO | Populating RTMetrics report | 20260507 19:07:22.603 | INFO | RTMetrics report populated | 20260507 19:07:22.603 | INFO | Opening tmp RTMetrics json file | 20260507 19:07:22.614 | INFO | Serializing RTMetrics to json | 20260507 19:07:22.621 | INFO | Serialization done, swapping tmp file now... | 20260507 19:07:22.622 | INFO | RTMetrics done | 20260507 19:07:22.677 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.538 seconds | 20260507 19:07:22.677 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.77% / 0.01% / 54.21% | 20260507 19:07:22.677 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.57773% / 2.78155e-05% / 5.16574e-05% | 20260507 19:07:22.677 | INFO | Baz IO overall: | 20260507 19:07:22.677 | INFO | BytesWritten : 848412094966 | 20260507 19:07:22.677 | INFO | NumEvents : 411351929800 | 20260507 19:07:24.164 | INFO | Analyzing chunk frames = [1884160,1884672) | 20260507 19:07:24.164 | INFO | MemInfo: { | 20260507 19:07:24.164 | INFO | "gpu" : " 17 GB", | 20260507 19:07:24.164 | INFO | "huge" : "", | 20260507 19:07:24.164 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:24.164 | INFO | "malloc" : " 14 GB", | 20260507 19:07:24.164 | INFO | "pinned" : " 29 GB" | 20260507 19:07:24.164 | INFO | } | 20260507 19:07:24.164 | INFO | Perf report for the last 5.13806 seconds: | 20260507 19:07:24.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:24.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:24.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:24.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:24.164 | INFO | ANALYSIS : 68.8%, 2.9, 106.1ms, 0 | 20260507 19:07:24.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:24.164 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 19:07:24.164 | INFO | PRE_HQ : 66.6%, 1.9, 64.2ms, 0 | 20260507 19:07:24.164 | INFO | BAZWRITER : 68.3%, 3.1, 108.5ms, 0 | 20260507 19:07:24.164 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 19:07:24.164 | INFO | Uploaded 36GiB to GPU in 2.4726s (14.5596GiB/s) | 20260507 19:07:24.164 | INFO | Downloaded 38GiB to GPU in 2.68885s (14.1324GiB/s) | 20260507 19:07:29.303 | INFO | Analyzing chunk frames = [1884672,1885184) | 20260507 19:07:29.303 | INFO | MemInfo: { | 20260507 19:07:29.303 | INFO | "gpu" : " 17 GB", | 20260507 19:07:29.303 | INFO | "huge" : "", | 20260507 19:07:29.303 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:29.303 | INFO | "malloc" : " 14 GB", | 20260507 19:07:29.303 | INFO | "pinned" : " 29 GB" | 20260507 19:07:29.303 | INFO | } | 20260507 19:07:29.303 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:07:29.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:29.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.303 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:07:29.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.303 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:07:29.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:29.304 | INFO | Uploaded 36GiB to GPU in 2.49824s (14.4101GiB/s) | 20260507 19:07:29.304 | INFO | Downloaded 34GiB to GPU in 2.4688s (13.7719GiB/s) | 20260507 19:07:34.442 | INFO | Analyzing chunk frames = [1885184,1885696) | 20260507 19:07:34.442 | INFO | MemInfo: { | 20260507 19:07:34.442 | INFO | "gpu" : " 17 GB", | 20260507 19:07:34.442 | INFO | "huge" : "", | 20260507 19:07:34.442 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:34.442 | INFO | "malloc" : " 14 GB", | 20260507 19:07:34.442 | INFO | "pinned" : " 29 GB" | 20260507 19:07:34.442 | INFO | } | 20260507 19:07:34.442 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:07:34.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:34.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.442 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:07:34.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.442 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:07:34.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:34.443 | INFO | Uploaded 36GiB to GPU in 2.54732s (14.1325GiB/s) | 20260507 19:07:34.443 | INFO | Downloaded 34GiB to GPU in 2.45479s (13.8505GiB/s) | 20260507 19:07:39.582 | INFO | Analyzing chunk frames = [1885696,1886208) | 20260507 19:07:39.582 | INFO | MemInfo: { | 20260507 19:07:39.582 | INFO | "gpu" : " 17 GB", | 20260507 19:07:39.582 | INFO | "huge" : "", | 20260507 19:07:39.582 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:39.582 | INFO | "malloc" : " 14 GB", | 20260507 19:07:39.582 | INFO | "pinned" : " 29 GB" | 20260507 19:07:39.582 | INFO | } | 20260507 19:07:39.582 | INFO | Perf report for the last 5.1394 seconds: | 20260507 19:07:39.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:39.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:07:39.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:07:39.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:39.582 | INFO | Uploaded 36GiB to GPU in 2.46957s (14.5775GiB/s) | 20260507 19:07:39.582 | INFO | Downloaded 34GiB to GPU in 2.38678s (14.2451GiB/s) | 20260507 19:07:44.722 | INFO | Analyzing chunk frames = [1886208,1886720) | 20260507 19:07:44.722 | INFO | MemInfo: { | 20260507 19:07:44.722 | INFO | "gpu" : " 17 GB", | 20260507 19:07:44.723 | INFO | "huge" : "", | 20260507 19:07:44.723 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:44.723 | INFO | "malloc" : " 14 GB", | 20260507 19:07:44.723 | INFO | "pinned" : " 29 GB" | 20260507 19:07:44.723 | INFO | } | 20260507 19:07:44.723 | INFO | Perf report for the last 5.14083 seconds: | 20260507 19:07:44.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:44.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 19:07:44.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:07:44.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:44.723 | INFO | Uploaded 36GiB to GPU in 2.36644s (15.2127GiB/s) | 20260507 19:07:44.723 | INFO | Downloaded 34GiB to GPU in 2.50028s (13.5985GiB/s) | 20260507 19:07:49.863 | INFO | Analyzing chunk frames = [1886720,1887232) | 20260507 19:07:49.863 | INFO | MemInfo: { | 20260507 19:07:49.863 | INFO | "gpu" : " 17 GB", | 20260507 19:07:49.863 | INFO | "huge" : "", | 20260507 19:07:49.863 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:49.863 | INFO | "malloc" : " 14 GB", | 20260507 19:07:49.863 | INFO | "pinned" : " 29 GB" | 20260507 19:07:49.863 | INFO | } | 20260507 19:07:49.863 | INFO | Perf report for the last 5.14015 seconds: | 20260507 19:07:49.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:49.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:07:49.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:07:49.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:49.863 | INFO | Uploaded 36GiB to GPU in 2.52847s (14.2379GiB/s) | 20260507 19:07:49.863 | INFO | Downloaded 34GiB to GPU in 2.41363s (14.0867GiB/s) | 20260507 19:07:55.002 | INFO | Analyzing chunk frames = [1887232,1887744) | 20260507 19:07:55.002 | INFO | MemInfo: { | 20260507 19:07:55.002 | INFO | "gpu" : " 17 GB", | 20260507 19:07:55.002 | INFO | "huge" : "", | 20260507 19:07:55.002 | INFO | "huge_shared" : " 29 GB", | 20260507 19:07:55.002 | INFO | "malloc" : " 14 GB", | 20260507 19:07:55.002 | INFO | "pinned" : " 29 GB" | 20260507 19:07:55.002 | INFO | } | 20260507 19:07:55.002 | INFO | Perf report for the last 5.13943 seconds: | 20260507 19:07:55.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:07:55.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.002 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:07:55.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.002 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 19:07:55.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:07:55.003 | INFO | Uploaded 36GiB to GPU in 2.54438s (14.1488GiB/s) | 20260507 19:07:55.003 | INFO | Downloaded 34GiB to GPU in 2.44133s (13.9268GiB/s) | 20260507 19:08:00.140 | INFO | Analyzing chunk frames = [1887744,1888256) | 20260507 19:08:00.140 | INFO | MemInfo: { | 20260507 19:08:00.140 | INFO | "gpu" : " 17 GB", | 20260507 19:08:00.140 | INFO | "huge" : "", | 20260507 19:08:00.140 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:00.140 | INFO | "malloc" : " 14 GB", | 20260507 19:08:00.140 | INFO | "pinned" : " 29 GB" | 20260507 19:08:00.140 | INFO | } | 20260507 19:08:00.140 | INFO | Perf report for the last 5.13781 seconds: | 20260507 19:08:00.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:00.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 19:08:00.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:08:00.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:00.140 | INFO | Uploaded 36GiB to GPU in 2.53381s (14.2079GiB/s) | 20260507 19:08:00.141 | INFO | Downloaded 34GiB to GPU in 2.41155s (14.0988GiB/s) | 20260507 19:08:00.237 | INFO | Starting a round of Baz IO | 20260507 19:08:03.719 | INFO | Populating RTMetrics report | 20260507 19:08:03.719 | INFO | RTMetrics report populated | 20260507 19:08:03.719 | INFO | Opening tmp RTMetrics json file | 20260507 19:08:03.740 | INFO | Serializing RTMetrics to json | 20260507 19:08:03.742 | INFO | Serialization done, swapping tmp file now... | 20260507 19:08:03.743 | INFO | RTMetrics done | 20260507 19:08:03.816 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.579 seconds | 20260507 19:08:03.816 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.76% / 0.01% / 54.23% | 20260507 19:08:03.816 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.57019% / 2.78155e-05% / 2.78155e-05% | 20260507 19:08:03.816 | INFO | Baz IO overall: | 20260507 19:08:03.816 | INFO | BytesWritten : 850341489583 | 20260507 19:08:03.816 | INFO | NumEvents : 412287607317 | 20260507 19:08:05.279 | INFO | Analyzing chunk frames = [1888256,1888768) | 20260507 19:08:05.279 | INFO | MemInfo: { | 20260507 19:08:05.279 | INFO | "gpu" : " 17 GB", | 20260507 19:08:05.279 | INFO | "huge" : "", | 20260507 19:08:05.279 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:05.279 | INFO | "malloc" : " 14 GB", | 20260507 19:08:05.279 | INFO | "pinned" : " 29 GB" | 20260507 19:08:05.279 | INFO | } | 20260507 19:08:05.279 | INFO | Perf report for the last 5.13871 seconds: | 20260507 19:08:05.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:05.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:05.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:05.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:05.279 | INFO | ANALYSIS : 68.9%, 3.0, 107.6ms, 0 | 20260507 19:08:05.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:05.279 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 19:08:05.279 | INFO | PRE_HQ : 65.8%, 1.7, 58.8ms, 0 | 20260507 19:08:05.279 | INFO | BAZWRITER : 68.8%, 3.0, 107.5ms, 0 | 20260507 19:08:05.279 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:08:05.280 | INFO | Uploaded 36GiB to GPU in 2.51203s (14.331GiB/s) | 20260507 19:08:05.280 | INFO | Downloaded 38GiB to GPU in 2.68602s (14.1473GiB/s) | 20260507 19:08:10.418 | INFO | Analyzing chunk frames = [1888768,1889280) | 20260507 19:08:10.418 | INFO | MemInfo: { | 20260507 19:08:10.418 | INFO | "gpu" : " 17 GB", | 20260507 19:08:10.418 | INFO | "huge" : "", | 20260507 19:08:10.418 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:10.418 | INFO | "malloc" : " 14 GB", | 20260507 19:08:10.418 | INFO | "pinned" : " 29 GB" | 20260507 19:08:10.418 | INFO | } | 20260507 19:08:10.418 | INFO | Perf report for the last 5.13907 seconds: | 20260507 19:08:10.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:10.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 19:08:10.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:08:10.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:10.418 | INFO | Uploaded 36GiB to GPU in 2.58041s (13.9513GiB/s) | 20260507 19:08:10.418 | INFO | Downloaded 34GiB to GPU in 2.46688s (13.7826GiB/s) | 20260507 19:08:15.558 | INFO | Analyzing chunk frames = [1889280,1889792) | 20260507 19:08:15.558 | INFO | MemInfo: { | 20260507 19:08:15.558 | INFO | "gpu" : " 17 GB", | 20260507 19:08:15.558 | INFO | "huge" : "", | 20260507 19:08:15.558 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:15.558 | INFO | "malloc" : " 14 GB", | 20260507 19:08:15.558 | INFO | "pinned" : " 29 GB" | 20260507 19:08:15.558 | INFO | } | 20260507 19:08:15.558 | INFO | Perf report for the last 5.14044 seconds: | 20260507 19:08:15.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:15.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:08:15.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:08:15.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:15.558 | INFO | Uploaded 36GiB to GPU in 2.56581s (14.0307GiB/s) | 20260507 19:08:15.559 | INFO | Downloaded 34GiB to GPU in 2.44927s (13.8817GiB/s) | 20260507 19:08:20.697 | INFO | Analyzing chunk frames = [1889792,1890304) | 20260507 19:08:20.697 | INFO | MemInfo: { | 20260507 19:08:20.697 | INFO | "gpu" : " 17 GB", | 20260507 19:08:20.697 | INFO | "huge" : "", | 20260507 19:08:20.697 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:20.697 | INFO | "malloc" : " 14 GB", | 20260507 19:08:20.697 | INFO | "pinned" : " 29 GB" | 20260507 19:08:20.697 | INFO | } | 20260507 19:08:20.697 | INFO | Perf report for the last 5.13902 seconds: | 20260507 19:08:20.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:20.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.697 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260507 19:08:20.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.697 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:08:20.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:20.698 | INFO | Uploaded 36GiB to GPU in 2.55538s (14.0879GiB/s) | 20260507 19:08:20.698 | INFO | Downloaded 34GiB to GPU in 2.44798s (13.889GiB/s) | 20260507 19:08:25.836 | INFO | Analyzing chunk frames = [1890304,1890816) | 20260507 19:08:25.836 | INFO | MemInfo: { | 20260507 19:08:25.836 | INFO | "gpu" : " 17 GB", | 20260507 19:08:25.836 | INFO | "huge" : "", | 20260507 19:08:25.836 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:25.836 | INFO | "malloc" : " 14 GB", | 20260507 19:08:25.837 | INFO | "pinned" : " 29 GB" | 20260507 19:08:25.837 | INFO | } | 20260507 19:08:25.837 | INFO | Perf report for the last 5.13936 seconds: | 20260507 19:08:25.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:25.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:08:25.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:08:25.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:25.837 | INFO | Uploaded 36GiB to GPU in 2.38061s (15.1222GiB/s) | 20260507 19:08:25.837 | INFO | Downloaded 34GiB to GPU in 2.48456s (13.6845GiB/s) | 20260507 19:08:30.976 | INFO | Analyzing chunk frames = [1890816,1891328) | 20260507 19:08:30.976 | INFO | MemInfo: { | 20260507 19:08:30.976 | INFO | "gpu" : " 17 GB", | 20260507 19:08:30.976 | INFO | "huge" : "", | 20260507 19:08:30.976 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:30.976 | INFO | "malloc" : " 14 GB", | 20260507 19:08:30.976 | INFO | "pinned" : " 29 GB" | 20260507 19:08:30.976 | INFO | } | 20260507 19:08:30.977 | INFO | Perf report for the last 5.1398 seconds: | 20260507 19:08:30.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:30.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:08:30.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 19:08:30.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:30.977 | INFO | Uploaded 36GiB to GPU in 2.46277s (14.6177GiB/s) | 20260507 19:08:30.977 | INFO | Downloaded 34GiB to GPU in 2.41972s (14.0512GiB/s) | 20260507 19:08:36.115 | INFO | Analyzing chunk frames = [1891328,1891840) | 20260507 19:08:36.115 | INFO | MemInfo: { | 20260507 19:08:36.115 | INFO | "gpu" : " 17 GB", | 20260507 19:08:36.115 | INFO | "huge" : "", | 20260507 19:08:36.115 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:36.115 | INFO | "malloc" : " 14 GB", | 20260507 19:08:36.115 | INFO | "pinned" : " 29 GB" | 20260507 19:08:36.115 | INFO | } | 20260507 19:08:36.115 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:08:36.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:36.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.115 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:08:36.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.115 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:08:36.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:36.116 | INFO | Uploaded 36GiB to GPU in 2.5079s (14.3546GiB/s) | 20260507 19:08:36.116 | INFO | Downloaded 34GiB to GPU in 2.44132s (13.9269GiB/s) | 20260507 19:08:41.254 | INFO | Analyzing chunk frames = [1891840,1892352) | 20260507 19:08:41.254 | INFO | MemInfo: { | 20260507 19:08:41.254 | INFO | "gpu" : " 17 GB", | 20260507 19:08:41.254 | INFO | "huge" : "", | 20260507 19:08:41.254 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:41.254 | INFO | "malloc" : " 14 GB", | 20260507 19:08:41.254 | INFO | "pinned" : " 29 GB" | 20260507 19:08:41.254 | INFO | } | 20260507 19:08:41.255 | INFO | Perf report for the last 5.13926 seconds: | 20260507 19:08:41.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:41.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:08:41.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:08:41.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:41.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:08:41.255 | INFO | Total mem usage: 16.4 GiB | 20260507 19:08:41.255 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:08:41.255 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -13.6%) | 20260507 19:08:41.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:08:41.255 | INFO | High Water Breakdown: | 20260507 19:08:41.255 | INFO | 59.2% : 8.55 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:08:41.255 | INFO | 40.6% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:08:41.255 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:08:41.255 | INFO | ------------------------------------------ | 20260507 19:08:41.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:08:41.255 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:08:41.255 | INFO | Total mem usage: 26.5 GiB | 20260507 19:08:41.255 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:08:41.255 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 19:08:41.255 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:08:41.255 | INFO | High Water Breakdown: | 20260507 19:08:41.255 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:08:41.255 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:08:41.255 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:08:41.255 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:08:41.255 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:08:41.255 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:08:41.255 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:08:41.255 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:08:41.255 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:08:41.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:08:41.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:08:41.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:08:41.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:08:41.255 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:08:41.255 | INFO | ------------------------------------------ | 20260507 19:08:41.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:08:41.255 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:08:41.255 | INFO | Total mem usage: 27.8 GiB | 20260507 19:08:41.255 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:08:41.255 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:08:41.255 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:08:41.255 | INFO | High Water Breakdown: | 20260507 19:08:41.255 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:08:41.255 | INFO | ------------------------------------------ | 20260507 19:08:41.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:08:41.255 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:08:41.255 | INFO | Total mem usage: 15.7 GiB | 20260507 19:08:41.255 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:08:41.255 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:08:41.255 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 19:08:41.255 | INFO | High Water Breakdown: | 20260507 19:08:41.255 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:08:41.255 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:08:41.255 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:08:41.255 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:08:41.255 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:08:41.255 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:08:41.255 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:08:41.255 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:08:41.255 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:08:41.255 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:08:41.255 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:08:41.255 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:08:41.255 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:08:41.255 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:08:41.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:08:41.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:08:41.255 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:08:41.255 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:08:41.255 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:08:41.255 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:08:41.255 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:08:41.255 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:08:41.255 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:08:41.255 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:08:41.255 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:08:41.255 | INFO | ------------------------------------------ | 20260507 19:08:41.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:08:41.255 | INFO | Uploaded 36GiB to GPU in 2.49703s (14.4172GiB/s) | 20260507 19:08:41.255 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:08:41.255 | INFO | Downloaded 34GiB to GPU in 2.42929s (13.9959GiB/s) | 20260507 19:08:41.256 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:08:41.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:08:41.256 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.59e+01, 1.39e+01 / 2.01e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 0.29%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.14e-02 / 7.60e-03 | 20260507 19:08:41.256 | INFO | PerfReport: 12.06%, Baseline -- 4.01e+00 / 4.48e+00, 5.96e-01 / 2.27e-01 | 20260507 19:08:41.256 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.12e-01 / 1.20e-01 | 20260507 19:08:41.256 | INFO | PerfReport: 15.74%, DME -- 4.18e+01 / 4.49e+01, 3.81e+00 / 1.34e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.85e+00 / 1.71e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.86e+00, 4.36e-01 / 2.18e-01 | 20260507 19:08:41.256 | INFO | PerfReport: 3.67%, Metrics -- 1.22e+00 / 1.44e+00, 3.08e-01 / 1.64e-01 | 20260507 19:08:41.256 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.15e-01, 6.28e-02 / 3.48e-02 | 20260507 19:08:41.256 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:08:41.256 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:08:41.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:08:41.256 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 24.31%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 24.05%, Download -- 2.58e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 31.00%, Compute -- 3.32e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 2.12%, LockUpload -- 2.27e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 2.31%, LockDownload -- 2.48e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 16.20%, LockCompute -- 1.74e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 19:08:41.256 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260507 19:08:41.353 | INFO | Starting a round of Baz IO | 20260507 19:08:44.830 | INFO | Populating RTMetrics report | 20260507 19:08:44.830 | INFO | RTMetrics report populated | 20260507 19:08:44.830 | INFO | Opening tmp RTMetrics json file | 20260507 19:08:44.860 | INFO | Serializing RTMetrics to json | 20260507 19:08:44.908 | INFO | Serialization done, swapping tmp file now... | 20260507 19:08:44.909 | INFO | RTMetrics done | 20260507 19:08:44.932 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.579 seconds | 20260507 19:08:44.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.75% / 0.01% / 54.24% | 20260507 19:08:44.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.56998% / 3.17891e-05% / 4.37101e-05% | 20260507 19:08:44.933 | INFO | Baz IO overall: | 20260507 19:08:44.933 | INFO | BytesWritten : 852272773090 | 20260507 19:08:44.933 | INFO | NumEvents : 413224912672 | 20260507 19:08:46.394 | INFO | Analyzing chunk frames = [1892352,1892864) | 20260507 19:08:46.394 | INFO | MemInfo: { | 20260507 19:08:46.394 | INFO | "gpu" : " 17 GB", | 20260507 19:08:46.394 | INFO | "huge" : "", | 20260507 19:08:46.394 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:46.394 | INFO | "malloc" : " 14 GB", | 20260507 19:08:46.394 | INFO | "pinned" : " 29 GB" | 20260507 19:08:46.394 | INFO | } | 20260507 19:08:46.394 | INFO | Perf report for the last 5.13936 seconds: | 20260507 19:08:46.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:46.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:46.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:46.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:46.394 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 19:08:46.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:46.394 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 19:08:46.394 | INFO | PRE_HQ : 66.1%, 1.7, 58.9ms, 0 | 20260507 19:08:46.394 | INFO | BAZWRITER : 69.3%, 3.9, 137.4ms, 0 | 20260507 19:08:46.394 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:08:46.395 | INFO | Uploaded 36GiB to GPU in 2.47867s (14.5239GiB/s) | 20260507 19:08:46.395 | INFO | Downloaded 38GiB to GPU in 2.69441s (14.1033GiB/s) | 20260507 19:08:51.533 | INFO | Analyzing chunk frames = [1892864,1893376) | 20260507 19:08:51.533 | INFO | MemInfo: { | 20260507 19:08:51.533 | INFO | "gpu" : " 17 GB", | 20260507 19:08:51.533 | INFO | "huge" : "", | 20260507 19:08:51.533 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:51.533 | INFO | "malloc" : " 14 GB", | 20260507 19:08:51.533 | INFO | "pinned" : " 29 GB" | 20260507 19:08:51.533 | INFO | } | 20260507 19:08:51.533 | INFO | Perf report for the last 5.13933 seconds: | 20260507 19:08:51.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:51.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.533 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:08:51.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.533 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:08:51.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:51.534 | INFO | Uploaded 36GiB to GPU in 2.51306s (14.3252GiB/s) | 20260507 19:08:51.534 | INFO | Downloaded 34GiB to GPU in 2.48555s (13.6791GiB/s) | 20260507 19:08:56.674 | INFO | Analyzing chunk frames = [1893376,1893888) | 20260507 19:08:56.674 | INFO | MemInfo: { | 20260507 19:08:56.674 | INFO | "gpu" : " 17 GB", | 20260507 19:08:56.674 | INFO | "huge" : "", | 20260507 19:08:56.674 | INFO | "huge_shared" : " 29 GB", | 20260507 19:08:56.674 | INFO | "malloc" : " 14 GB", | 20260507 19:08:56.674 | INFO | "pinned" : " 29 GB" | 20260507 19:08:56.674 | INFO | } | 20260507 19:08:56.674 | INFO | Perf report for the last 5.14115 seconds: | 20260507 19:08:56.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:08:56.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.675 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:08:56.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.675 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:08:56.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:08:56.675 | INFO | Uploaded 36GiB to GPU in 2.52211s (14.2738GiB/s) | 20260507 19:08:56.675 | INFO | Downloaded 34GiB to GPU in 2.48076s (13.7055GiB/s) | 20260507 19:09:01.813 | INFO | Analyzing chunk frames = [1893888,1894400) | 20260507 19:09:01.813 | INFO | MemInfo: { | 20260507 19:09:01.813 | INFO | "gpu" : " 17 GB", | 20260507 19:09:01.813 | INFO | "huge" : "", | 20260507 19:09:01.813 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:01.813 | INFO | "malloc" : " 14 GB", | 20260507 19:09:01.813 | INFO | "pinned" : " 29 GB" | 20260507 19:09:01.813 | INFO | } | 20260507 19:09:01.814 | INFO | Perf report for the last 5.13923 seconds: | 20260507 19:09:01.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:01.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:09:01.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 19:09:01.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:01.814 | INFO | Uploaded 36GiB to GPU in 2.45434s (14.6679GiB/s) | 20260507 19:09:01.814 | INFO | Downloaded 34GiB to GPU in 2.42164s (14.0401GiB/s) | 20260507 19:09:06.952 | INFO | Analyzing chunk frames = [1894400,1894912) | 20260507 19:09:06.952 | INFO | MemInfo: { | 20260507 19:09:06.952 | INFO | "gpu" : " 17 GB", | 20260507 19:09:06.952 | INFO | "huge" : "", | 20260507 19:09:06.952 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:06.953 | INFO | "malloc" : " 14 GB", | 20260507 19:09:06.953 | INFO | "pinned" : " 29 GB" | 20260507 19:09:06.953 | INFO | } | 20260507 19:09:06.953 | INFO | Perf report for the last 5.13882 seconds: | 20260507 19:09:06.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:06.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 19:09:06.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:09:06.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:06.953 | INFO | Uploaded 36GiB to GPU in 2.32545s (15.4809GiB/s) | 20260507 19:09:06.953 | INFO | Downloaded 34GiB to GPU in 2.50784s (13.5575GiB/s) | 20260507 19:09:12.091 | INFO | Analyzing chunk frames = [1894912,1895424) | 20260507 19:09:12.091 | INFO | MemInfo: { | 20260507 19:09:12.091 | INFO | "gpu" : " 17 GB", | 20260507 19:09:12.091 | INFO | "huge" : "", | 20260507 19:09:12.091 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:12.091 | INFO | "malloc" : " 14 GB", | 20260507 19:09:12.091 | INFO | "pinned" : " 29 GB" | 20260507 19:09:12.091 | INFO | } | 20260507 19:09:12.091 | INFO | Perf report for the last 5.13838 seconds: | 20260507 19:09:12.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:12.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:09:12.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:09:12.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:12.091 | INFO | Uploaded 36GiB to GPU in 2.4473s (14.7101GiB/s) | 20260507 19:09:12.091 | INFO | Downloaded 34GiB to GPU in 2.42889s (13.9981GiB/s) | 20260507 19:09:17.230 | INFO | Analyzing chunk frames = [1895424,1895936) | 20260507 19:09:17.230 | INFO | MemInfo: { | 20260507 19:09:17.230 | INFO | "gpu" : " 17 GB", | 20260507 19:09:17.230 | INFO | "huge" : "", | 20260507 19:09:17.230 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:17.230 | INFO | "malloc" : " 14 GB", | 20260507 19:09:17.230 | INFO | "pinned" : " 29 GB" | 20260507 19:09:17.230 | INFO | } | 20260507 19:09:17.230 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:09:17.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:17.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 19:09:17.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:09:17.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:17.230 | INFO | Uploaded 36GiB to GPU in 2.45806s (14.6457GiB/s) | 20260507 19:09:17.230 | INFO | Downloaded 34GiB to GPU in 2.44425s (13.9102GiB/s) | 20260507 19:09:22.369 | INFO | Analyzing chunk frames = [1895936,1896448) | 20260507 19:09:22.369 | INFO | MemInfo: { | 20260507 19:09:22.369 | INFO | "gpu" : " 17 GB", | 20260507 19:09:22.369 | INFO | "huge" : "", | 20260507 19:09:22.369 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:22.369 | INFO | "malloc" : " 14 GB", | 20260507 19:09:22.369 | INFO | "pinned" : " 29 GB" | 20260507 19:09:22.369 | INFO | } | 20260507 19:09:22.369 | INFO | Perf report for the last 5.13899 seconds: | 20260507 19:09:22.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:22.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:09:22.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:09:22.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:22.369 | INFO | Uploaded 36GiB to GPU in 2.41907s (14.8817GiB/s) | 20260507 19:09:22.369 | INFO | Downloaded 34GiB to GPU in 2.4441s (13.911GiB/s) | 20260507 19:09:22.467 | INFO | Starting a round of Baz IO | 20260507 19:09:25.945 | INFO | Populating RTMetrics report | 20260507 19:09:25.945 | INFO | RTMetrics report populated | 20260507 19:09:25.945 | INFO | Opening tmp RTMetrics json file | 20260507 19:09:25.968 | INFO | Serializing RTMetrics to json | 20260507 19:09:25.970 | INFO | Serialization done, swapping tmp file now... | 20260507 19:09:25.971 | INFO | RTMetrics done | 20260507 19:09:26.048 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.581 seconds | 20260507 19:09:26.048 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.73% / 0.01% / 54.25% | 20260507 19:09:26.048 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.55909% / 1.58946e-05% / 5.5631e-05% | 20260507 19:09:26.048 | INFO | Baz IO overall: | 20260507 19:09:26.048 | INFO | BytesWritten : 854202325197 | 20260507 19:09:26.048 | INFO | NumEvents : 414160478377 | 20260507 19:09:27.509 | INFO | Analyzing chunk frames = [1896448,1896960) | 20260507 19:09:27.509 | INFO | MemInfo: { | 20260507 19:09:27.509 | INFO | "gpu" : " 17 GB", | 20260507 19:09:27.509 | INFO | "huge" : "", | 20260507 19:09:27.509 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:27.509 | INFO | "malloc" : " 14 GB", | 20260507 19:09:27.509 | INFO | "pinned" : " 29 GB" | 20260507 19:09:27.509 | INFO | } | 20260507 19:09:27.510 | INFO | Perf report for the last 5.14056 seconds: | 20260507 19:09:27.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:27.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:27.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:27.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:27.510 | INFO | ANALYSIS : 68.8%, 3.0, 107.3ms, 0 | 20260507 19:09:27.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:27.510 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 19:09:27.510 | INFO | PRE_HQ : 66.0%, 1.7, 58.7ms, 0 | 20260507 19:09:27.510 | INFO | BAZWRITER : 69.3%, 3.8, 134.4ms, 0 | 20260507 19:09:27.510 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 19:09:27.510 | INFO | Uploaded 36GiB to GPU in 2.50422s (14.3757GiB/s) | 20260507 19:09:27.510 | INFO | Downloaded 38GiB to GPU in 2.68668s (14.1439GiB/s) | 20260507 19:09:32.647 | INFO | Analyzing chunk frames = [1896960,1897472) | 20260507 19:09:32.647 | INFO | MemInfo: { | 20260507 19:09:32.647 | INFO | "gpu" : " 17 GB", | 20260507 19:09:32.647 | INFO | "huge" : "", | 20260507 19:09:32.647 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:32.647 | INFO | "malloc" : " 14 GB", | 20260507 19:09:32.647 | INFO | "pinned" : " 29 GB" | 20260507 19:09:32.647 | INFO | } | 20260507 19:09:32.647 | INFO | Perf report for the last 5.13796 seconds: | 20260507 19:09:32.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:32.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:09:32.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:09:32.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:32.648 | INFO | Uploaded 36GiB to GPU in 2.51629s (14.3068GiB/s) | 20260507 19:09:32.648 | INFO | Downloaded 34GiB to GPU in 2.48905s (13.6598GiB/s) | 20260507 19:09:37.787 | INFO | Analyzing chunk frames = [1897472,1897984) | 20260507 19:09:37.787 | INFO | MemInfo: { | 20260507 19:09:37.787 | INFO | "gpu" : " 17 GB", | 20260507 19:09:37.787 | INFO | "huge" : "", | 20260507 19:09:37.787 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:37.787 | INFO | "malloc" : " 14 GB", | 20260507 19:09:37.787 | INFO | "pinned" : " 29 GB" | 20260507 19:09:37.787 | INFO | } | 20260507 19:09:37.787 | INFO | Perf report for the last 5.13952 seconds: | 20260507 19:09:37.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:37.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.787 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:09:37.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.787 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:09:37.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:37.788 | INFO | Uploaded 36GiB to GPU in 2.54106s (14.1673GiB/s) | 20260507 19:09:37.788 | INFO | Downloaded 34GiB to GPU in 2.47053s (13.7622GiB/s) | 20260507 19:09:42.927 | INFO | Analyzing chunk frames = [1897984,1898496) | 20260507 19:09:42.927 | INFO | MemInfo: { | 20260507 19:09:42.927 | INFO | "gpu" : " 17 GB", | 20260507 19:09:42.927 | INFO | "huge" : "", | 20260507 19:09:42.927 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:42.927 | INFO | "malloc" : " 14 GB", | 20260507 19:09:42.927 | INFO | "pinned" : " 29 GB" | 20260507 19:09:42.927 | INFO | } | 20260507 19:09:42.928 | INFO | Perf report for the last 5.1405 seconds: | 20260507 19:09:42.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:42.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:09:42.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:09:42.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:42.928 | INFO | Uploaded 36GiB to GPU in 2.45692s (14.6525GiB/s) | 20260507 19:09:42.928 | INFO | Downloaded 34GiB to GPU in 2.38262s (14.27GiB/s) | 20260507 19:09:48.067 | INFO | Analyzing chunk frames = [1898496,1899008) | 20260507 19:09:48.067 | INFO | MemInfo: { | 20260507 19:09:48.067 | INFO | "gpu" : " 17 GB", | 20260507 19:09:48.067 | INFO | "huge" : "", | 20260507 19:09:48.067 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:48.067 | INFO | "malloc" : " 14 GB", | 20260507 19:09:48.067 | INFO | "pinned" : " 29 GB" | 20260507 19:09:48.067 | INFO | } | 20260507 19:09:48.067 | INFO | Perf report for the last 5.13985 seconds: | 20260507 19:09:48.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:48.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.067 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:09:48.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.067 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 19:09:48.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:48.068 | INFO | Uploaded 36GiB to GPU in 2.37446s (15.1613GiB/s) | 20260507 19:09:48.068 | INFO | Downloaded 34GiB to GPU in 2.4956s (13.624GiB/s) | 20260507 19:09:53.206 | INFO | Analyzing chunk frames = [1899008,1899520) | 20260507 19:09:53.206 | INFO | MemInfo: { | 20260507 19:09:53.206 | INFO | "gpu" : " 17 GB", | 20260507 19:09:53.206 | INFO | "huge" : "", | 20260507 19:09:53.206 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:53.206 | INFO | "malloc" : " 14 GB", | 20260507 19:09:53.206 | INFO | "pinned" : " 29 GB" | 20260507 19:09:53.206 | INFO | } | 20260507 19:09:53.206 | INFO | Perf report for the last 5.13873 seconds: | 20260507 19:09:53.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:53.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.206 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:09:53.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.206 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:09:53.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:53.207 | INFO | Uploaded 36GiB to GPU in 2.47109s (14.5685GiB/s) | 20260507 19:09:53.207 | INFO | Downloaded 34GiB to GPU in 2.41471s (14.0803GiB/s) | 20260507 19:09:58.345 | INFO | Analyzing chunk frames = [1899520,1900032) | 20260507 19:09:58.345 | INFO | MemInfo: { | 20260507 19:09:58.345 | INFO | "gpu" : " 17 GB", | 20260507 19:09:58.345 | INFO | "huge" : "", | 20260507 19:09:58.345 | INFO | "huge_shared" : " 29 GB", | 20260507 19:09:58.345 | INFO | "malloc" : " 14 GB", | 20260507 19:09:58.345 | INFO | "pinned" : " 29 GB" | 20260507 19:09:58.345 | INFO | } | 20260507 19:09:58.345 | INFO | Perf report for the last 5.1386 seconds: | 20260507 19:09:58.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:09:58.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:09:58.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:09:58.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:09:58.345 | INFO | Uploaded 36GiB to GPU in 2.50581s (14.3666GiB/s) | 20260507 19:09:58.345 | INFO | Downloaded 34GiB to GPU in 2.44494s (13.9063GiB/s) | 20260507 19:10:03.483 | INFO | Analyzing chunk frames = [1900032,1900544) | 20260507 19:10:03.483 | INFO | MemInfo: { | 20260507 19:10:03.483 | INFO | "gpu" : " 17 GB", | 20260507 19:10:03.484 | INFO | "huge" : "", | 20260507 19:10:03.484 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:03.484 | INFO | "malloc" : " 14 GB", | 20260507 19:10:03.484 | INFO | "pinned" : " 29 GB" | 20260507 19:10:03.484 | INFO | } | 20260507 19:10:03.484 | INFO | Perf report for the last 5.13893 seconds: | 20260507 19:10:03.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:03.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 19:10:03.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:10:03.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:03.484 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:10:03.484 | INFO | Total mem usage: 16.4 GiB | 20260507 19:10:03.484 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:10:03.484 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12%) | 20260507 19:10:03.484 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:10:03.484 | INFO | High Water Breakdown: | 20260507 19:10:03.484 | INFO | 59.5% : 8.71 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:10:03.484 | INFO | 40.4% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:10:03.484 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:10:03.484 | INFO | ------------------------------------------ | 20260507 19:10:03.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:10:03.484 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:10:03.484 | INFO | Total mem usage: 26.5 GiB | 20260507 19:10:03.484 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:10:03.484 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:10:03.484 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:10:03.484 | INFO | High Water Breakdown: | 20260507 19:10:03.484 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:10:03.484 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:10:03.484 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:10:03.484 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:10:03.484 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:10:03.484 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:10:03.485 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:10:03.485 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:10:03.485 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:10:03.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:10:03.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:10:03.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:10:03.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:10:03.485 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:10:03.485 | INFO | ------------------------------------------ | 20260507 19:10:03.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:10:03.485 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:10:03.485 | INFO | Total mem usage: 27.8 GiB | 20260507 19:10:03.485 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:10:03.485 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:10:03.485 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:10:03.485 | INFO | High Water Breakdown: | 20260507 19:10:03.485 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:10:03.485 | INFO | ------------------------------------------ | 20260507 19:10:03.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:10:03.485 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:10:03.485 | INFO | Total mem usage: 15.7 GiB | 20260507 19:10:03.485 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:10:03.485 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:10:03.485 | INFO | Uploaded 36GiB to GPU in 2.50263s (14.3849GiB/s) | 20260507 19:10:03.485 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:10:03.485 | INFO | High Water Breakdown: | 20260507 19:10:03.485 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:10:03.485 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:10:03.485 | INFO | Downloaded 34GiB to GPU in 2.429s (13.9975GiB/s) | 20260507 19:10:03.485 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:10:03.485 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:10:03.485 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:10:03.485 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:10:03.485 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:10:03.485 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:10:03.485 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:10:03.485 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:10:03.485 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:10:03.485 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:10:03.485 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:10:03.485 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:10:03.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:10:03.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:10:03.485 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:10:03.485 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:10:03.485 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:10:03.485 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:10:03.485 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:10:03.485 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:10:03.485 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:10:03.485 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:10:03.485 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:10:03.485 | INFO | ------------------------------------------ | 20260507 19:10:03.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:10:03.485 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:10:03.485 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:10:03.485 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:10:03.485 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 1.86e+00 | 20260507 19:10:03.485 | INFO | PerfReport: 0.29%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.29e-02 / 5.65e-03 | 20260507 19:10:03.485 | INFO | PerfReport: 12.10%, Baseline -- 4.02e+00 / 4.48e+00, 6.03e-01 / 2.33e-01 | 20260507 19:10:03.485 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.49e+00, 2.12e-01 / 1.23e-01 | 20260507 19:10:03.485 | INFO | PerfReport: 15.77%, DME -- 4.19e+01 / 4.50e+01, 3.95e+00 / 1.47e+00 | 20260507 19:10:03.485 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.78e+00 / 1.77e+00 | 20260507 19:10:03.485 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.40e-01 / 2.47e-01 | 20260507 19:10:03.485 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.42e+00, 3.07e-01 / 1.83e-01 | 20260507 19:10:03.486 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.80e-01 / 4.11e-01, 6.01e-02 / 3.00e-02 | 20260507 19:10:03.486 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:10:03.486 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:10:03.486 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:10:03.486 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 24.05%, Upload -- 2.57e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 24.19%, Download -- 2.59e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 31.07%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 1.76%, LockUpload -- 1.88e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 2.68%, LockDownload -- 2.87e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 16.24%, LockCompute -- 1.74e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:10:03.486 | INFO | PerfReport: 0.01%, Misc -- 1.54e-02 / 0.00e+00, 3.89e-02 / 0.00e+00 | 20260507 19:10:03.582 | INFO | Starting a round of Baz IO | 20260507 19:10:07.063 | INFO | Populating RTMetrics report | 20260507 19:10:07.064 | INFO | RTMetrics report populated | 20260507 19:10:07.064 | INFO | Opening tmp RTMetrics json file | 20260507 19:10:07.104 | INFO | Serializing RTMetrics to json | 20260507 19:10:07.159 | INFO | Serialization done, swapping tmp file now... | 20260507 19:10:07.163 | INFO | RTMetrics done | 20260507 19:10:07.252 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.669 seconds | 20260507 19:10:07.252 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.72% / 0.01% / 54.27% | 20260507 19:10:07.252 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.54419% / 1.58946e-05% / 7.54992e-05% | 20260507 19:10:07.252 | INFO | Baz IO overall: | 20260507 19:10:07.252 | INFO | BytesWritten : 856130553206 | 20260507 19:10:07.252 | INFO | NumEvents : 415094873273 | 20260507 19:10:08.623 | INFO | Analyzing chunk frames = [1900544,1901056) | 20260507 19:10:08.623 | INFO | MemInfo: { | 20260507 19:10:08.623 | INFO | "gpu" : " 17 GB", | 20260507 19:10:08.623 | INFO | "huge" : "", | 20260507 19:10:08.623 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:08.623 | INFO | "malloc" : " 14 GB", | 20260507 19:10:08.623 | INFO | "pinned" : " 29 GB" | 20260507 19:10:08.623 | INFO | } | 20260507 19:10:08.623 | INFO | Perf report for the last 5.139 seconds: | 20260507 19:10:08.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:08.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:08.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:08.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:08.623 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 19:10:08.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:08.623 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 19:10:08.623 | INFO | PRE_HQ : 66.4%, 1.7, 59.1ms, 0 | 20260507 19:10:08.623 | INFO | BAZWRITER : 71.0%, 3.9, 142.9ms, 0 | 20260507 19:10:08.623 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:10:08.624 | INFO | Uploaded 36GiB to GPU in 2.47403s (14.5511GiB/s) | 20260507 19:10:08.624 | INFO | Downloaded 38GiB to GPU in 2.68877s (14.1328GiB/s) | 20260507 19:10:13.763 | INFO | Analyzing chunk frames = [1901056,1901568) | 20260507 19:10:13.763 | INFO | MemInfo: { | 20260507 19:10:13.763 | INFO | "gpu" : " 17 GB", | 20260507 19:10:13.763 | INFO | "huge" : "", | 20260507 19:10:13.763 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:13.763 | INFO | "malloc" : " 14 GB", | 20260507 19:10:13.763 | INFO | "pinned" : " 29 GB" | 20260507 19:10:13.763 | INFO | } | 20260507 19:10:13.763 | INFO | Perf report for the last 5.14055 seconds: | 20260507 19:10:13.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:13.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.763 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 19:10:13.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.763 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 19:10:13.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:13.764 | INFO | Uploaded 36GiB to GPU in 2.53s (14.2293GiB/s) | 20260507 19:10:13.764 | INFO | Downloaded 34GiB to GPU in 2.49115s (13.6483GiB/s) | 20260507 19:10:18.903 | INFO | Analyzing chunk frames = [1901568,1902080) | 20260507 19:10:18.903 | INFO | MemInfo: { | 20260507 19:10:18.903 | INFO | "gpu" : " 17 GB", | 20260507 19:10:18.903 | INFO | "huge" : "", | 20260507 19:10:18.903 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:18.903 | INFO | "malloc" : " 14 GB", | 20260507 19:10:18.903 | INFO | "pinned" : " 29 GB" | 20260507 19:10:18.903 | INFO | } | 20260507 19:10:18.903 | INFO | Perf report for the last 5.13952 seconds: | 20260507 19:10:18.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:18.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.903 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:10:18.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.903 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:10:18.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:18.904 | INFO | Uploaded 36GiB to GPU in 2.56443s (14.0382GiB/s) | 20260507 19:10:18.904 | INFO | Downloaded 34GiB to GPU in 2.47079s (13.7608GiB/s) | 20260507 19:10:24.042 | INFO | Analyzing chunk frames = [1902080,1902592) | 20260507 19:10:24.042 | INFO | MemInfo: { | 20260507 19:10:24.042 | INFO | "gpu" : " 17 GB", | 20260507 19:10:24.042 | INFO | "huge" : "", | 20260507 19:10:24.042 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:24.042 | INFO | "malloc" : " 14 GB", | 20260507 19:10:24.042 | INFO | "pinned" : " 29 GB" | 20260507 19:10:24.042 | INFO | } | 20260507 19:10:24.042 | INFO | Perf report for the last 5.13917 seconds: | 20260507 19:10:24.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:24.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:10:24.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:10:24.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:24.042 | INFO | Uploaded 36GiB to GPU in 2.47684s (14.5346GiB/s) | 20260507 19:10:24.042 | INFO | Downloaded 34GiB to GPU in 2.39611s (14.1897GiB/s) | 20260507 19:10:29.182 | INFO | Analyzing chunk frames = [1902592,1903104) | 20260507 19:10:29.182 | INFO | MemInfo: { | 20260507 19:10:29.182 | INFO | "gpu" : " 17 GB", | 20260507 19:10:29.182 | INFO | "huge" : "", | 20260507 19:10:29.182 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:29.182 | INFO | "malloc" : " 14 GB", | 20260507 19:10:29.182 | INFO | "pinned" : " 29 GB" | 20260507 19:10:29.182 | INFO | } | 20260507 19:10:29.182 | INFO | Perf report for the last 5.13989 seconds: | 20260507 19:10:29.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:29.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.182 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 19:10:29.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.182 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 19:10:29.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:29.183 | INFO | Uploaded 36GiB to GPU in 2.40165s (14.9897GiB/s) | 20260507 19:10:29.183 | INFO | Downloaded 34GiB to GPU in 2.48727s (13.6696GiB/s) | 20260507 19:10:34.320 | INFO | Analyzing chunk frames = [1903104,1903616) | 20260507 19:10:34.320 | INFO | MemInfo: { | 20260507 19:10:34.320 | INFO | "gpu" : " 17 GB", | 20260507 19:10:34.320 | INFO | "huge" : "", | 20260507 19:10:34.320 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:34.320 | INFO | "malloc" : " 14 GB", | 20260507 19:10:34.320 | INFO | "pinned" : " 29 GB" | 20260507 19:10:34.320 | INFO | } | 20260507 19:10:34.320 | INFO | Perf report for the last 5.13852 seconds: | 20260507 19:10:34.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:34.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.320 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:10:34.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.321 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:10:34.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:34.321 | INFO | Uploaded 36GiB to GPU in 2.50297s (14.3829GiB/s) | 20260507 19:10:34.321 | INFO | Downloaded 34GiB to GPU in 2.40836s (14.1175GiB/s) | 20260507 19:10:39.459 | INFO | Analyzing chunk frames = [1903616,1904128) | 20260507 19:10:39.459 | INFO | MemInfo: { | 20260507 19:10:39.459 | INFO | "gpu" : " 17 GB", | 20260507 19:10:39.459 | INFO | "huge" : "", | 20260507 19:10:39.459 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:39.459 | INFO | "malloc" : " 14 GB", | 20260507 19:10:39.459 | INFO | "pinned" : " 29 GB" | 20260507 19:10:39.459 | INFO | } | 20260507 19:10:39.459 | INFO | Perf report for the last 5.13838 seconds: | 20260507 19:10:39.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:39.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:10:39.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:10:39.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:39.459 | INFO | Uploaded 36GiB to GPU in 2.54265s (14.1584GiB/s) | 20260507 19:10:39.459 | INFO | Downloaded 34GiB to GPU in 2.43586s (13.9581GiB/s) | 20260507 19:10:44.599 | INFO | Analyzing chunk frames = [1904128,1904640) | 20260507 19:10:44.599 | INFO | MemInfo: { | 20260507 19:10:44.599 | INFO | "gpu" : " 17 GB", | 20260507 19:10:44.599 | INFO | "huge" : "", | 20260507 19:10:44.599 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:44.599 | INFO | "malloc" : " 14 GB", | 20260507 19:10:44.599 | INFO | "pinned" : " 29 GB" | 20260507 19:10:44.599 | INFO | } | 20260507 19:10:44.599 | INFO | Perf report for the last 5.1401 seconds: | 20260507 19:10:44.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:44.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 19:10:44.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:10:44.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:44.599 | INFO | Uploaded 36GiB to GPU in 2.53877s (14.1801GiB/s) | 20260507 19:10:44.599 | INFO | Downloaded 34GiB to GPU in 2.41882s (14.0565GiB/s) | 20260507 19:10:44.709 | INFO | Starting a round of Baz IO | 20260507 19:10:48.184 | INFO | Populating RTMetrics report | 20260507 19:10:48.184 | INFO | RTMetrics report populated | 20260507 19:10:48.184 | INFO | Opening tmp RTMetrics json file | 20260507 19:10:48.206 | INFO | Serializing RTMetrics to json | 20260507 19:10:48.208 | INFO | Serialization done, swapping tmp file now... | 20260507 19:10:48.209 | INFO | RTMetrics done | 20260507 19:10:48.277 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.568 seconds | 20260507 19:10:48.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.71% / 0.01% / 54.28% | 20260507 19:10:48.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.54109% / 3.17891e-05% / 3.97364e-05% | 20260507 19:10:48.277 | INFO | Baz IO overall: | 20260507 19:10:48.277 | INFO | BytesWritten : 858058689230 | 20260507 19:10:48.277 | INFO | NumEvents : 416029166289 | 20260507 19:10:49.740 | INFO | Analyzing chunk frames = [1904640,1905152) | 20260507 19:10:49.740 | INFO | MemInfo: { | 20260507 19:10:49.740 | INFO | "gpu" : " 17 GB", | 20260507 19:10:49.740 | INFO | "huge" : "", | 20260507 19:10:49.740 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:49.740 | INFO | "malloc" : " 14 GB", | 20260507 19:10:49.740 | INFO | "pinned" : " 29 GB" | 20260507 19:10:49.740 | INFO | } | 20260507 19:10:49.740 | INFO | Perf report for the last 5.14082 seconds: | 20260507 19:10:49.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:49.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:49.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:49.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:49.740 | INFO | ANALYSIS : 68.9%, 2.9, 105.2ms, 0 | 20260507 19:10:49.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:49.740 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 19:10:49.740 | INFO | PRE_HQ : 67.5%, 1.8, 62.8ms, 0 | 20260507 19:10:49.740 | INFO | BAZWRITER : 69.0%, 4.2, 147.6ms, 0 | 20260507 19:10:49.740 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 19:10:49.740 | INFO | Uploaded 36GiB to GPU in 2.49947s (14.4031GiB/s) | 20260507 19:10:49.740 | INFO | Downloaded 38GiB to GPU in 2.69023s (14.1252GiB/s) | 20260507 19:10:54.878 | INFO | Analyzing chunk frames = [1905152,1905664) | 20260507 19:10:54.878 | INFO | MemInfo: { | 20260507 19:10:54.878 | INFO | "gpu" : " 17 GB", | 20260507 19:10:54.878 | INFO | "huge" : "", | 20260507 19:10:54.878 | INFO | "huge_shared" : " 29 GB", | 20260507 19:10:54.878 | INFO | "malloc" : " 14 GB", | 20260507 19:10:54.878 | INFO | "pinned" : " 29 GB" | 20260507 19:10:54.878 | INFO | } | 20260507 19:10:54.878 | INFO | Perf report for the last 5.13838 seconds: | 20260507 19:10:54.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:10:54.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.878 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:10:54.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.878 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:10:54.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:10:54.879 | INFO | Uploaded 36GiB to GPU in 2.5743s (13.9844GiB/s) | 20260507 19:10:54.879 | INFO | Downloaded 34GiB to GPU in 2.4852s (13.681GiB/s) | 20260507 19:11:00.019 | INFO | Analyzing chunk frames = [1905664,1906176) | 20260507 19:11:00.019 | INFO | MemInfo: { | 20260507 19:11:00.019 | INFO | "gpu" : " 17 GB", | 20260507 19:11:00.019 | INFO | "huge" : "", | 20260507 19:11:00.019 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:00.019 | INFO | "malloc" : " 14 GB", | 20260507 19:11:00.019 | INFO | "pinned" : " 29 GB" | 20260507 19:11:00.019 | INFO | } | 20260507 19:11:00.019 | INFO | Perf report for the last 5.14077 seconds: | 20260507 19:11:00.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:00.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:11:00.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:11:00.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:00.019 | INFO | Uploaded 36GiB to GPU in 2.57141s (14.0001GiB/s) | 20260507 19:11:00.019 | INFO | Downloaded 34GiB to GPU in 2.44835s (13.8869GiB/s) | 20260507 19:11:05.157 | INFO | Analyzing chunk frames = [1906176,1906688) | 20260507 19:11:05.157 | INFO | MemInfo: { | 20260507 19:11:05.157 | INFO | "gpu" : " 17 GB", | 20260507 19:11:05.157 | INFO | "huge" : "", | 20260507 19:11:05.157 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:05.157 | INFO | "malloc" : " 14 GB", | 20260507 19:11:05.157 | INFO | "pinned" : " 29 GB" | 20260507 19:11:05.157 | INFO | } | 20260507 19:11:05.157 | INFO | Perf report for the last 5.13785 seconds: | 20260507 19:11:05.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:05.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:11:05.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:11:05.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:05.157 | INFO | Uploaded 36GiB to GPU in 2.49844s (14.409GiB/s) | 20260507 19:11:05.157 | INFO | Downloaded 34GiB to GPU in 2.38392s (14.2622GiB/s) | 20260507 19:11:10.296 | INFO | Analyzing chunk frames = [1906688,1907200) | 20260507 19:11:10.296 | INFO | MemInfo: { | 20260507 19:11:10.296 | INFO | "gpu" : " 17 GB", | 20260507 19:11:10.296 | INFO | "huge" : "", | 20260507 19:11:10.296 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:10.296 | INFO | "malloc" : " 14 GB", | 20260507 19:11:10.296 | INFO | "pinned" : " 29 GB" | 20260507 19:11:10.296 | INFO | } | 20260507 19:11:10.296 | INFO | Perf report for the last 5.13899 seconds: | 20260507 19:11:10.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:10.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 19:11:10.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 19:11:10.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:10.296 | INFO | Uploaded 36GiB to GPU in 2.41117s (14.9305GiB/s) | 20260507 19:11:10.296 | INFO | Downloaded 34GiB to GPU in 2.49342s (13.6359GiB/s) | 20260507 19:11:15.434 | INFO | Analyzing chunk frames = [1907200,1907712) | 20260507 19:11:15.434 | INFO | MemInfo: { | 20260507 19:11:15.434 | INFO | "gpu" : " 17 GB", | 20260507 19:11:15.434 | INFO | "huge" : "", | 20260507 19:11:15.434 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:15.434 | INFO | "malloc" : " 14 GB", | 20260507 19:11:15.434 | INFO | "pinned" : " 29 GB" | 20260507 19:11:15.434 | INFO | } | 20260507 19:11:15.435 | INFO | Perf report for the last 5.13875 seconds: | 20260507 19:11:15.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:15.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:11:15.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:11:15.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:15.435 | INFO | Uploaded 36GiB to GPU in 2.4657s (14.6003GiB/s) | 20260507 19:11:15.435 | INFO | Downloaded 34GiB to GPU in 2.41309s (14.0898GiB/s) | 20260507 19:11:20.574 | INFO | Analyzing chunk frames = [1907712,1908224) | 20260507 19:11:20.574 | INFO | MemInfo: { | 20260507 19:11:20.574 | INFO | "gpu" : " 17 GB", | 20260507 19:11:20.574 | INFO | "huge" : "", | 20260507 19:11:20.574 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:20.574 | INFO | "malloc" : " 14 GB", | 20260507 19:11:20.574 | INFO | "pinned" : " 29 GB" | 20260507 19:11:20.574 | INFO | } | 20260507 19:11:20.575 | INFO | Perf report for the last 5.13998 seconds: | 20260507 19:11:20.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:20.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.575 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:11:20.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.575 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 19:11:20.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:20.576 | INFO | Uploaded 36GiB to GPU in 2.5165s (14.3056GiB/s) | 20260507 19:11:20.576 | INFO | Downloaded 34GiB to GPU in 2.44426s (13.9102GiB/s) | 20260507 19:11:25.714 | INFO | Analyzing chunk frames = [1908224,1908736) | 20260507 19:11:25.714 | INFO | MemInfo: { | 20260507 19:11:25.714 | INFO | "gpu" : " 17 GB", | 20260507 19:11:25.714 | INFO | "huge" : "", | 20260507 19:11:25.714 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:25.714 | INFO | "malloc" : " 14 GB", | 20260507 19:11:25.714 | INFO | "pinned" : " 29 GB" | 20260507 19:11:25.714 | INFO | } | 20260507 19:11:25.714 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:11:25.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:25.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.714 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:11:25.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.714 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 19:11:25.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:25.715 | INFO | Uploaded 36GiB to GPU in 2.5116s (14.3335GiB/s) | 20260507 19:11:25.715 | INFO | Downloaded 34GiB to GPU in 2.42656s (14.0116GiB/s) | 20260507 19:11:25.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:11:25.715 | INFO | Total mem usage: 16.4 GiB | 20260507 19:11:25.715 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:11:25.715 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 19:11:25.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:11:25.715 | INFO | High Water Breakdown: | 20260507 19:11:25.715 | INFO | 59.7% : 8.8 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:11:25.715 | INFO | 40.2% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:11:25.715 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:11:25.715 | INFO | ------------------------------------------ | 20260507 19:11:25.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:11:25.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:11:25.715 | INFO | Total mem usage: 26.5 GiB | 20260507 19:11:25.715 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:11:25.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:11:25.715 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:11:25.715 | INFO | High Water Breakdown: | 20260507 19:11:25.715 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:11:25.715 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:11:25.715 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:11:25.715 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:11:25.715 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:11:25.715 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:11:25.715 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:11:25.715 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:11:25.716 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:11:25.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:11:25.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:11:25.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:11:25.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:11:25.716 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:11:25.716 | INFO | ------------------------------------------ | 20260507 19:11:25.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:11:25.716 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:11:25.716 | INFO | Total mem usage: 27.8 GiB | 20260507 19:11:25.716 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:11:25.716 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:11:25.716 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:11:25.716 | INFO | High Water Breakdown: | 20260507 19:11:25.716 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:11:25.716 | INFO | ------------------------------------------ | 20260507 19:11:25.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:11:25.716 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:11:25.716 | INFO | Total mem usage: 15.7 GiB | 20260507 19:11:25.716 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:11:25.716 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:11:25.716 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 19:11:25.716 | INFO | High Water Breakdown: | 20260507 19:11:25.716 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:11:25.716 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:11:25.716 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:11:25.716 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:11:25.716 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:11:25.716 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:11:25.716 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:11:25.716 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:11:25.716 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:11:25.716 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:11:25.716 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:11:25.716 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:11:25.716 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:11:25.716 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:11:25.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:11:25.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:11:25.716 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:11:25.716 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:11:25.716 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:11:25.716 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:11:25.716 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:11:25.716 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:11:25.716 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:11:25.716 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:11:25.716 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:11:25.716 | INFO | ------------------------------------------ | 20260507 19:11:25.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:11:25.716 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:11:25.716 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:11:25.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:11:25.716 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 1.93e+00 | 20260507 19:11:25.716 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.12e-02 / 5.99e-03 | 20260507 19:11:25.716 | INFO | PerfReport: 12.10%, Baseline -- 4.02e+00 / 4.47e+00, 6.09e-01 / 2.91e-01 | 20260507 19:11:25.716 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.51e+00, 2.13e-01 / 1.13e-01 | 20260507 19:11:25.716 | INFO | PerfReport: 15.74%, DME -- 4.18e+01 / 4.50e+01, 3.83e+00 / 1.33e+00 | 20260507 19:11:25.716 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.80e+00 / 1.82e+00 | 20260507 19:11:25.716 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.52e+00 / 2.89e+00, 4.44e-01 / 1.92e-01 | 20260507 19:11:25.716 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.44e+00, 3.07e-01 / 1.55e-01 | 20260507 19:11:25.716 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.14e-01, 6.12e-02 / 2.91e-02 | 20260507 19:11:25.717 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:11:25.717 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:11:25.717 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:11:25.717 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 24.39%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 24.09%, Download -- 2.58e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 31.04%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 2.08%, LockUpload -- 2.22e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 2.33%, LockDownload -- 2.50e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 16.07%, LockCompute -- 1.72e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:11:25.717 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 19:11:25.825 | INFO | Starting a round of Baz IO | 20260507 19:11:29.323 | INFO | Populating RTMetrics report | 20260507 19:11:29.323 | INFO | RTMetrics report populated | 20260507 19:11:29.323 | INFO | Opening tmp RTMetrics json file | 20260507 19:11:29.324 | INFO | Serializing RTMetrics to json | 20260507 19:11:29.326 | INFO | Serialization done, swapping tmp file now... | 20260507 19:11:29.327 | INFO | RTMetrics done | 20260507 19:11:29.438 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.613 seconds | 20260507 19:11:29.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.69% / 0.01% / 54.29% | 20260507 19:11:29.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.54311% / 2.38419e-05% / 5.5631e-05% | 20260507 19:11:29.438 | INFO | Baz IO overall: | 20260507 19:11:29.438 | INFO | BytesWritten : 859989032775 | 20260507 19:11:29.438 | INFO | NumEvents : 416965315066 | 20260507 19:11:30.853 | INFO | Analyzing chunk frames = [1908736,1909248) | 20260507 19:11:30.853 | INFO | MemInfo: { | 20260507 19:11:30.853 | INFO | "gpu" : " 17 GB", | 20260507 19:11:30.853 | INFO | "huge" : "", | 20260507 19:11:30.853 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:30.853 | INFO | "malloc" : " 14 GB", | 20260507 19:11:30.853 | INFO | "pinned" : " 29 GB" | 20260507 19:11:30.853 | INFO | } | 20260507 19:11:30.854 | INFO | Perf report for the last 5.13927 seconds: | 20260507 19:11:30.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:30.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:30.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:30.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:30.854 | INFO | ANALYSIS : 69.7%, 2.9, 105.5ms, 0 | 20260507 19:11:30.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:30.854 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260507 19:11:30.854 | INFO | PRE_HQ : 68.8%, 1.8, 64.2ms, 0 | 20260507 19:11:30.854 | INFO | BAZWRITER : 69.9%, 4.3, 155.4ms, 0 | 20260507 19:11:30.854 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 19:11:30.854 | INFO | Uploaded 36GiB to GPU in 2.53123s (14.2223GiB/s) | 20260507 19:11:30.854 | INFO | Downloaded 38GiB to GPU in 2.71368s (14.0031GiB/s) | 20260507 19:11:35.993 | INFO | Analyzing chunk frames = [1909248,1909760) | 20260507 19:11:35.993 | INFO | MemInfo: { | 20260507 19:11:35.993 | INFO | "gpu" : " 17 GB", | 20260507 19:11:35.993 | INFO | "huge" : "", | 20260507 19:11:35.993 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:35.993 | INFO | "malloc" : " 14 GB", | 20260507 19:11:35.993 | INFO | "pinned" : " 29 GB" | 20260507 19:11:35.993 | INFO | } | 20260507 19:11:35.993 | INFO | Perf report for the last 5.13987 seconds: | 20260507 19:11:35.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:35.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:11:35.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:11:35.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:35.994 | INFO | Uploaded 36GiB to GPU in 2.55355s (14.098GiB/s) | 20260507 19:11:35.994 | INFO | Downloaded 34GiB to GPU in 2.48126s (13.7027GiB/s) | 20260507 19:11:41.132 | INFO | Analyzing chunk frames = [1909760,1910272) | 20260507 19:11:41.132 | INFO | MemInfo: { | 20260507 19:11:41.132 | INFO | "gpu" : " 17 GB", | 20260507 19:11:41.132 | INFO | "huge" : "", | 20260507 19:11:41.132 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:41.132 | INFO | "malloc" : " 14 GB", | 20260507 19:11:41.132 | INFO | "pinned" : " 29 GB" | 20260507 19:11:41.132 | INFO | } | 20260507 19:11:41.132 | INFO | Perf report for the last 5.1383 seconds: | 20260507 19:11:41.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:41.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:11:41.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:11:41.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:41.132 | INFO | Uploaded 36GiB to GPU in 2.52575s (14.2532GiB/s) | 20260507 19:11:41.133 | INFO | Downloaded 34GiB to GPU in 2.46625s (13.7861GiB/s) | 20260507 19:11:46.271 | INFO | Analyzing chunk frames = [1910272,1910784) | 20260507 19:11:46.271 | INFO | MemInfo: { | 20260507 19:11:46.271 | INFO | "gpu" : " 17 GB", | 20260507 19:11:46.271 | INFO | "huge" : "", | 20260507 19:11:46.271 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:46.271 | INFO | "malloc" : " 14 GB", | 20260507 19:11:46.272 | INFO | "pinned" : " 29 GB" | 20260507 19:11:46.272 | INFO | } | 20260507 19:11:46.272 | INFO | Perf report for the last 5.14001 seconds: | 20260507 19:11:46.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:46.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 19:11:46.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:11:46.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:46.272 | INFO | Uploaded 36GiB to GPU in 2.46166s (14.6243GiB/s) | 20260507 19:11:46.272 | INFO | Downloaded 34GiB to GPU in 2.38639s (14.2474GiB/s) | 20260507 19:11:51.411 | INFO | Analyzing chunk frames = [1910784,1911296) | 20260507 19:11:51.411 | INFO | MemInfo: { | 20260507 19:11:51.411 | INFO | "gpu" : " 17 GB", | 20260507 19:11:51.411 | INFO | "huge" : "", | 20260507 19:11:51.411 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:51.411 | INFO | "malloc" : " 14 GB", | 20260507 19:11:51.411 | INFO | "pinned" : " 29 GB" | 20260507 19:11:51.411 | INFO | } | 20260507 19:11:51.411 | INFO | Perf report for the last 5.13927 seconds: | 20260507 19:11:51.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:51.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.411 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:11:51.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.411 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:11:51.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:51.412 | INFO | Uploaded 36GiB to GPU in 2.37517s (15.1568GiB/s) | 20260507 19:11:51.412 | INFO | Downloaded 34GiB to GPU in 2.49962s (13.602GiB/s) | 20260507 19:11:56.549 | INFO | Analyzing chunk frames = [1911296,1911808) | 20260507 19:11:56.549 | INFO | MemInfo: { | 20260507 19:11:56.549 | INFO | "gpu" : " 17 GB", | 20260507 19:11:56.549 | INFO | "huge" : "", | 20260507 19:11:56.549 | INFO | "huge_shared" : " 29 GB", | 20260507 19:11:56.549 | INFO | "malloc" : " 14 GB", | 20260507 19:11:56.549 | INFO | "pinned" : " 29 GB" | 20260507 19:11:56.549 | INFO | } | 20260507 19:11:56.550 | INFO | Perf report for the last 5.13846 seconds: | 20260507 19:11:56.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:11:56.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:11:56.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:11:56.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:11:56.550 | INFO | Uploaded 36GiB to GPU in 2.51204s (14.331GiB/s) | 20260507 19:11:56.550 | INFO | Downloaded 34GiB to GPU in 2.41395s (14.0848GiB/s) | 20260507 19:12:01.690 | INFO | Analyzing chunk frames = [1911808,1912320) | 20260507 19:12:01.690 | INFO | MemInfo: { | 20260507 19:12:01.690 | INFO | "gpu" : " 17 GB", | 20260507 19:12:01.690 | INFO | "huge" : "", | 20260507 19:12:01.690 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:01.690 | INFO | "malloc" : " 14 GB", | 20260507 19:12:01.690 | INFO | "pinned" : " 29 GB" | 20260507 19:12:01.690 | INFO | } | 20260507 19:12:01.690 | INFO | Perf report for the last 5.14064 seconds: | 20260507 19:12:01.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:01.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:12:01.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 19:12:01.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:01.690 | INFO | Uploaded 36GiB to GPU in 2.55312s (14.1004GiB/s) | 20260507 19:12:01.691 | INFO | Downloaded 34GiB to GPU in 2.43948s (13.9374GiB/s) | 20260507 19:12:06.830 | INFO | Analyzing chunk frames = [1912320,1912832) | 20260507 19:12:06.830 | INFO | MemInfo: { | 20260507 19:12:06.830 | INFO | "gpu" : " 17 GB", | 20260507 19:12:06.830 | INFO | "huge" : "", | 20260507 19:12:06.830 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:06.830 | INFO | "malloc" : " 14 GB", | 20260507 19:12:06.830 | INFO | "pinned" : " 29 GB" | 20260507 19:12:06.830 | INFO | } | 20260507 19:12:06.831 | INFO | Perf report for the last 5.14042 seconds: | 20260507 19:12:06.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:06.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 19:12:06.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 19:12:06.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:06.831 | INFO | Uploaded 36GiB to GPU in 2.52853s (14.2375GiB/s) | 20260507 19:12:06.831 | INFO | Downloaded 34GiB to GPU in 2.43602s (13.9572GiB/s) | 20260507 19:12:06.941 | INFO | Starting a round of Baz IO | 20260507 19:12:10.435 | INFO | Populating RTMetrics report | 20260507 19:12:10.435 | INFO | RTMetrics report populated | 20260507 19:12:10.435 | INFO | Opening tmp RTMetrics json file | 20260507 19:12:10.458 | INFO | Serializing RTMetrics to json | 20260507 19:12:10.460 | INFO | Serialization done, swapping tmp file now... | 20260507 19:12:10.461 | INFO | RTMetrics done | 20260507 19:12:10.528 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.586 seconds | 20260507 19:12:10.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.68% / 0.01% / 54.31% | 20260507 19:12:10.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.53684% / 1.58946e-05% / 5.16574e-05% | 20260507 19:12:10.529 | INFO | Baz IO overall: | 20260507 19:12:10.529 | INFO | BytesWritten : 861918542011 | 20260507 19:12:10.529 | INFO | NumEvents : 417900895530 | 20260507 19:12:11.969 | INFO | Analyzing chunk frames = [1912832,1913344) | 20260507 19:12:11.969 | INFO | MemInfo: { | 20260507 19:12:11.969 | INFO | "gpu" : " 17 GB", | 20260507 19:12:11.969 | INFO | "huge" : "", | 20260507 19:12:11.969 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:11.969 | INFO | "malloc" : " 14 GB", | 20260507 19:12:11.969 | INFO | "pinned" : " 29 GB" | 20260507 19:12:11.969 | INFO | } | 20260507 19:12:11.969 | INFO | Perf report for the last 5.1382 seconds: | 20260507 19:12:11.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:11.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:11.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:11.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:11.969 | INFO | ANALYSIS : 69.4%, 2.9, 104.5ms, 0 | 20260507 19:12:11.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:11.969 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 19:12:11.969 | INFO | PRE_HQ : 68.2%, 1.8, 65.3ms, 0 | 20260507 19:12:11.969 | INFO | BAZWRITER : 69.4%, 4.4, 156.0ms, 0 | 20260507 19:12:11.969 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:12:11.969 | INFO | Uploaded 36GiB to GPU in 2.54392s (14.1514GiB/s) | 20260507 19:12:11.969 | INFO | Downloaded 38GiB to GPU in 2.69082s (14.1221GiB/s) | 20260507 19:12:17.107 | INFO | Analyzing chunk frames = [1913344,1913856) | 20260507 19:12:17.107 | INFO | MemInfo: { | 20260507 19:12:17.107 | INFO | "gpu" : " 17 GB", | 20260507 19:12:17.107 | INFO | "huge" : "", | 20260507 19:12:17.107 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:17.107 | INFO | "malloc" : " 14 GB", | 20260507 19:12:17.107 | INFO | "pinned" : " 29 GB" | 20260507 19:12:17.107 | INFO | } | 20260507 19:12:17.107 | INFO | Perf report for the last 5.13863 seconds: | 20260507 19:12:17.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:17.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.108 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 19:12:17.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.108 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:12:17.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:17.108 | INFO | Uploaded 36GiB to GPU in 2.55198s (14.1067GiB/s) | 20260507 19:12:17.108 | INFO | Downloaded 34GiB to GPU in 2.49747s (13.6138GiB/s) | 20260507 19:12:22.246 | INFO | Analyzing chunk frames = [1913856,1914368) | 20260507 19:12:22.246 | INFO | MemInfo: { | 20260507 19:12:22.246 | INFO | "gpu" : " 17 GB", | 20260507 19:12:22.246 | INFO | "huge" : "", | 20260507 19:12:22.246 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:22.246 | INFO | "malloc" : " 14 GB", | 20260507 19:12:22.246 | INFO | "pinned" : " 29 GB" | 20260507 19:12:22.246 | INFO | } | 20260507 19:12:22.246 | INFO | Perf report for the last 5.13893 seconds: | 20260507 19:12:22.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:22.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.246 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:12:22.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.247 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:12:22.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:22.247 | INFO | Uploaded 36GiB to GPU in 2.5402s (14.1721GiB/s) | 20260507 19:12:22.247 | INFO | Downloaded 34GiB to GPU in 2.47683s (13.7272GiB/s) | 20260507 19:12:27.386 | INFO | Analyzing chunk frames = [1914368,1914880) | 20260507 19:12:27.386 | INFO | MemInfo: { | 20260507 19:12:27.386 | INFO | "gpu" : " 17 GB", | 20260507 19:12:27.386 | INFO | "huge" : "", | 20260507 19:12:27.386 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:27.386 | INFO | "malloc" : " 14 GB", | 20260507 19:12:27.386 | INFO | "pinned" : " 29 GB" | 20260507 19:12:27.386 | INFO | } | 20260507 19:12:27.386 | INFO | Perf report for the last 5.13942 seconds: | 20260507 19:12:27.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:27.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:12:27.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:12:27.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:27.386 | INFO | Uploaded 36GiB to GPU in 2.43089s (14.8094GiB/s) | 20260507 19:12:27.386 | INFO | Downloaded 34GiB to GPU in 2.39312s (14.2074GiB/s) | 20260507 19:12:32.526 | INFO | Analyzing chunk frames = [1914880,1915392) | 20260507 19:12:32.526 | INFO | MemInfo: { | 20260507 19:12:32.526 | INFO | "gpu" : " 17 GB", | 20260507 19:12:32.526 | INFO | "huge" : "", | 20260507 19:12:32.526 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:32.526 | INFO | "malloc" : " 14 GB", | 20260507 19:12:32.526 | INFO | "pinned" : " 29 GB" | 20260507 19:12:32.526 | INFO | } | 20260507 19:12:32.526 | INFO | Perf report for the last 5.14063 seconds: | 20260507 19:12:32.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:32.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:12:32.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:12:32.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:32.527 | INFO | Uploaded 36GiB to GPU in 2.34906s (15.3253GiB/s) | 20260507 19:12:32.527 | INFO | Downloaded 34GiB to GPU in 2.5056s (13.5696GiB/s) | 20260507 19:12:37.666 | INFO | Analyzing chunk frames = [1915392,1915904) | 20260507 19:12:37.666 | INFO | MemInfo: { | 20260507 19:12:37.666 | INFO | "gpu" : " 17 GB", | 20260507 19:12:37.666 | INFO | "huge" : "", | 20260507 19:12:37.666 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:37.666 | INFO | "malloc" : " 14 GB", | 20260507 19:12:37.666 | INFO | "pinned" : " 29 GB" | 20260507 19:12:37.666 | INFO | } | 20260507 19:12:37.666 | INFO | Perf report for the last 5.13954 seconds: | 20260507 19:12:37.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:37.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:12:37.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:12:37.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:37.666 | INFO | Uploaded 36GiB to GPU in 2.45979s (14.6354GiB/s) | 20260507 19:12:37.666 | INFO | Downloaded 34GiB to GPU in 2.42462s (14.0228GiB/s) | 20260507 19:12:42.804 | INFO | Analyzing chunk frames = [1915904,1916416) | 20260507 19:12:42.804 | INFO | MemInfo: { | 20260507 19:12:42.804 | INFO | "gpu" : " 17 GB", | 20260507 19:12:42.804 | INFO | "huge" : "", | 20260507 19:12:42.804 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:42.805 | INFO | "malloc" : " 14 GB", | 20260507 19:12:42.805 | INFO | "pinned" : " 29 GB" | 20260507 19:12:42.805 | INFO | } | 20260507 19:12:42.805 | INFO | Perf report for the last 5.13859 seconds: | 20260507 19:12:42.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:42.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:12:42.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 19:12:42.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:42.805 | INFO | Uploaded 36GiB to GPU in 2.49361s (14.4369GiB/s) | 20260507 19:12:42.805 | INFO | Downloaded 34GiB to GPU in 2.4488s (13.8844GiB/s) | 20260507 19:12:47.943 | INFO | Analyzing chunk frames = [1916416,1916928) | 20260507 19:12:47.943 | INFO | MemInfo: { | 20260507 19:12:47.943 | INFO | "gpu" : " 17 GB", | 20260507 19:12:47.943 | INFO | "huge" : "", | 20260507 19:12:47.943 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:47.943 | INFO | "malloc" : " 14 GB", | 20260507 19:12:47.943 | INFO | "pinned" : " 29 GB" | 20260507 19:12:47.944 | INFO | } | 20260507 19:12:47.944 | INFO | Perf report for the last 5.13905 seconds: | 20260507 19:12:47.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:47.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:12:47.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 19:12:47.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:47.944 | INFO | Uploaded 36GiB to GPU in 2.49083s (14.453GiB/s) | 20260507 19:12:47.944 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:12:47.944 | INFO | Total mem usage: 16.4 GiB | 20260507 19:12:47.944 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:12:47.944 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8%) | 20260507 19:12:47.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:12:47.944 | INFO | Downloaded 34GiB to GPU in 2.43705s (13.9513GiB/s) | 20260507 19:12:47.944 | INFO | High Water Breakdown: | 20260507 19:12:47.944 | INFO | 60.1% : 9.13 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:12:47.944 | INFO | 39.8% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:12:47.944 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:12:47.944 | INFO | ------------------------------------------ | 20260507 19:12:47.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:12:47.944 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:12:47.944 | INFO | Total mem usage: 26.5 GiB | 20260507 19:12:47.944 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:12:47.944 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 19:12:47.944 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260507 19:12:47.944 | INFO | High Water Breakdown: | 20260507 19:12:47.944 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:12:47.944 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:12:47.944 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:12:47.944 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:12:47.944 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:12:47.944 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:12:47.944 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:12:47.945 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:12:47.945 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:12:47.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:12:47.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:12:47.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:12:47.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:12:47.945 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:12:47.945 | INFO | ------------------------------------------ | 20260507 19:12:47.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:12:47.945 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:12:47.945 | INFO | Total mem usage: 27.8 GiB | 20260507 19:12:47.945 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:12:47.945 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:12:47.945 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:12:47.945 | INFO | High Water Breakdown: | 20260507 19:12:47.945 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:12:47.945 | INFO | ------------------------------------------ | 20260507 19:12:47.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:12:47.945 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:12:47.945 | INFO | Total mem usage: 15.7 GiB | 20260507 19:12:47.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:12:47.945 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:12:47.945 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:12:47.945 | INFO | High Water Breakdown: | 20260507 19:12:47.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:12:47.945 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:12:47.945 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:12:47.945 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:12:47.945 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:12:47.945 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:12:47.946 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:12:47.946 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:12:47.946 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:12:47.946 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:12:47.946 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:12:47.946 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:12:47.946 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:12:47.946 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:12:47.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:12:47.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:12:47.946 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:12:47.946 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:12:47.946 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:12:47.946 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:12:47.946 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:12:47.946 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:12:47.946 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:12:47.946 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:12:47.946 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:12:47.946 | INFO | ------------------------------------------ | 20260507 19:12:47.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:12:47.946 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:12:47.946 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:12:47.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:12:47.946 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.41e+01 / 1.84e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.05e-01, 1.16e-02 / 8.05e-03 | 20260507 19:12:47.946 | INFO | PerfReport: 12.07%, Baseline -- 4.01e+00 / 4.45e+00, 5.96e-01 / 3.16e-01 | 20260507 19:12:47.946 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.13e-01 / 1.31e-01 | 20260507 19:12:47.946 | INFO | PerfReport: 15.74%, DME -- 4.18e+01 / 4.48e+01, 3.90e+00 / 1.36e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.83e+00 / 1.84e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.52e+00 / 2.81e+00, 4.41e-01 / 2.76e-01 | 20260507 19:12:47.946 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.43e+00, 3.11e-01 / 1.99e-01 | 20260507 19:12:47.946 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.14e-01, 6.03e-02 / 3.41e-02 | 20260507 19:12:47.946 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:12:47.946 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:12:47.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:12:47.946 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 24.32%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 24.20%, Download -- 2.59e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 31.11%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 2.33%, LockDownload -- 2.49e+00 / 0.00e+00, 3.72e+00 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 16.05%, LockCompute -- 1.71e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:12:47.946 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260507 19:12:48.059 | INFO | Starting a round of Baz IO | 20260507 19:12:51.561 | INFO | Populating RTMetrics report | 20260507 19:12:51.561 | INFO | RTMetrics report populated | 20260507 19:12:51.561 | INFO | Opening tmp RTMetrics json file | 20260507 19:12:51.562 | INFO | Serializing RTMetrics to json | 20260507 19:12:51.564 | INFO | Serialization done, swapping tmp file now... | 20260507 19:12:51.564 | INFO | RTMetrics done | 20260507 19:12:51.674 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.614 seconds | 20260507 19:12:51.674 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.67% / 0.01% / 54.32% | 20260507 19:12:51.674 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.52366% / 2.78155e-05% / 4.76837e-05% | 20260507 19:12:51.674 | INFO | Baz IO overall: | 20260507 19:12:51.674 | INFO | BytesWritten : 863847209472 | 20260507 19:12:51.674 | INFO | NumEvents : 418835996023 | 20260507 19:12:53.083 | INFO | Analyzing chunk frames = [1916928,1917440) | 20260507 19:12:53.083 | INFO | MemInfo: { | 20260507 19:12:53.083 | INFO | "gpu" : " 17 GB", | 20260507 19:12:53.083 | INFO | "huge" : "", | 20260507 19:12:53.083 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:53.083 | INFO | "malloc" : " 14 GB", | 20260507 19:12:53.083 | INFO | "pinned" : " 29 GB" | 20260507 19:12:53.083 | INFO | } | 20260507 19:12:53.083 | INFO | Perf report for the last 5.13974 seconds: | 20260507 19:12:53.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:53.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:53.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:53.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:53.084 | INFO | ANALYSIS : 69.9%, 2.9, 105.1ms, 0 | 20260507 19:12:53.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:53.084 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260507 19:12:53.084 | INFO | PRE_HQ : 68.9%, 1.8, 65.3ms, 0 | 20260507 19:12:53.084 | INFO | BAZWRITER : 70.2%, 4.1, 147.5ms, 0 | 20260507 19:12:53.084 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 19:12:53.084 | INFO | Uploaded 36GiB to GPU in 2.58832s (13.9087GiB/s) | 20260507 19:12:53.084 | INFO | Downloaded 38GiB to GPU in 2.70238s (14.0617GiB/s) | 20260507 19:12:58.222 | INFO | Analyzing chunk frames = [1917440,1917952) | 20260507 19:12:58.222 | INFO | MemInfo: { | 20260507 19:12:58.222 | INFO | "gpu" : " 17 GB", | 20260507 19:12:58.222 | INFO | "huge" : "", | 20260507 19:12:58.222 | INFO | "huge_shared" : " 29 GB", | 20260507 19:12:58.222 | INFO | "malloc" : " 14 GB", | 20260507 19:12:58.222 | INFO | "pinned" : " 29 GB" | 20260507 19:12:58.222 | INFO | } | 20260507 19:12:58.222 | INFO | Perf report for the last 5.13886 seconds: | 20260507 19:12:58.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:12:58.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:12:58.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:12:58.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:12:58.223 | INFO | Uploaded 36GiB to GPU in 2.49608s (14.4226GiB/s) | 20260507 19:12:58.223 | INFO | Downloaded 34GiB to GPU in 2.45784s (13.8333GiB/s) | 20260507 19:13:03.363 | INFO | Analyzing chunk frames = [1917952,1918464) | 20260507 19:13:03.363 | INFO | MemInfo: { | 20260507 19:13:03.363 | INFO | "gpu" : " 17 GB", | 20260507 19:13:03.363 | INFO | "huge" : "", | 20260507 19:13:03.363 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:03.363 | INFO | "malloc" : " 14 GB", | 20260507 19:13:03.363 | INFO | "pinned" : " 29 GB" | 20260507 19:13:03.363 | INFO | } | 20260507 19:13:03.363 | INFO | Perf report for the last 5.14073 seconds: | 20260507 19:13:03.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:03.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:13:03.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:13:03.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:03.363 | INFO | Uploaded 36GiB to GPU in 2.53004s (14.229GiB/s) | 20260507 19:13:03.363 | INFO | Downloaded 34GiB to GPU in 2.46847s (13.7737GiB/s) | 20260507 19:13:08.501 | INFO | Analyzing chunk frames = [1918464,1918976) | 20260507 19:13:08.501 | INFO | MemInfo: { | 20260507 19:13:08.501 | INFO | "gpu" : " 17 GB", | 20260507 19:13:08.501 | INFO | "huge" : "", | 20260507 19:13:08.501 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:08.501 | INFO | "malloc" : " 14 GB", | 20260507 19:13:08.501 | INFO | "pinned" : " 29 GB" | 20260507 19:13:08.501 | INFO | } | 20260507 19:13:08.501 | INFO | Perf report for the last 5.13775 seconds: | 20260507 19:13:08.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:08.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 19:13:08.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:13:08.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:08.501 | INFO | Uploaded 36GiB to GPU in 2.46146s (14.6255GiB/s) | 20260507 19:13:08.501 | INFO | Downloaded 34GiB to GPU in 2.39395s (14.2025GiB/s) | 20260507 19:13:13.641 | INFO | Analyzing chunk frames = [1918976,1919488) | 20260507 19:13:13.641 | INFO | MemInfo: { | 20260507 19:13:13.641 | INFO | "gpu" : " 17 GB", | 20260507 19:13:13.641 | INFO | "huge" : "", | 20260507 19:13:13.641 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:13.641 | INFO | "malloc" : " 14 GB", | 20260507 19:13:13.641 | INFO | "pinned" : " 29 GB" | 20260507 19:13:13.641 | INFO | } | 20260507 19:13:13.641 | INFO | Perf report for the last 5.14038 seconds: | 20260507 19:13:13.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:13.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.641 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 19:13:13.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.641 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:13:13.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:13.642 | INFO | Uploaded 36GiB to GPU in 2.33249s (15.4342GiB/s) | 20260507 19:13:13.642 | INFO | Downloaded 34GiB to GPU in 2.51111s (13.5398GiB/s) | 20260507 19:13:18.780 | INFO | Analyzing chunk frames = [1919488,1920000) | 20260507 19:13:18.780 | INFO | MemInfo: { | 20260507 19:13:18.780 | INFO | "gpu" : " 17 GB", | 20260507 19:13:18.780 | INFO | "huge" : "", | 20260507 19:13:18.780 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:18.780 | INFO | "malloc" : " 14 GB", | 20260507 19:13:18.780 | INFO | "pinned" : " 29 GB" | 20260507 19:13:18.780 | INFO | } | 20260507 19:13:18.780 | INFO | Perf report for the last 5.13891 seconds: | 20260507 19:13:18.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:18.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.780 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:13:18.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.780 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:13:18.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:18.781 | INFO | Uploaded 36GiB to GPU in 2.43159s (14.8051GiB/s) | 20260507 19:13:18.781 | INFO | Downloaded 34GiB to GPU in 2.43899s (13.9402GiB/s) | 20260507 19:13:23.919 | INFO | Analyzing chunk frames = [1920000,1920512) | 20260507 19:13:23.920 | INFO | MemInfo: { | 20260507 19:13:23.920 | INFO | "gpu" : " 17 GB", | 20260507 19:13:23.920 | INFO | "huge" : "", | 20260507 19:13:23.920 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:23.920 | INFO | "malloc" : " 14 GB", | 20260507 19:13:23.920 | INFO | "pinned" : " 29 GB" | 20260507 19:13:23.920 | INFO | } | 20260507 19:13:23.920 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:13:23.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:23.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:13:23.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:13:23.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:23.920 | INFO | Uploaded 36GiB to GPU in 2.49171s (14.4479GiB/s) | 20260507 19:13:23.920 | INFO | Downloaded 34GiB to GPU in 2.46794s (13.7767GiB/s) | 20260507 19:13:29.058 | INFO | Analyzing chunk frames = [1920512,1921024) | 20260507 19:13:29.058 | INFO | MemInfo: { | 20260507 19:13:29.058 | INFO | "gpu" : " 17 GB", | 20260507 19:13:29.058 | INFO | "huge" : "", | 20260507 19:13:29.058 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:29.058 | INFO | "malloc" : " 14 GB", | 20260507 19:13:29.059 | INFO | "pinned" : " 29 GB" | 20260507 19:13:29.059 | INFO | } | 20260507 19:13:29.059 | INFO | Perf report for the last 5.13908 seconds: | 20260507 19:13:29.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:29.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:13:29.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | PRE_HQ : 36.7%, 1.0, 19.0ms, 0 | 20260507 19:13:29.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:29.059 | INFO | Uploaded 36GiB to GPU in 2.44818s (14.7048GiB/s) | 20260507 19:13:29.059 | INFO | Downloaded 34GiB to GPU in 2.4466s (13.8968GiB/s) | 20260507 19:13:29.173 | INFO | Starting a round of Baz IO | 20260507 19:13:32.644 | INFO | Populating RTMetrics report | 20260507 19:13:32.644 | INFO | RTMetrics report populated | 20260507 19:13:32.644 | INFO | Opening tmp RTMetrics json file | 20260507 19:13:32.659 | INFO | Serializing RTMetrics to json | 20260507 19:13:32.716 | INFO | Serialization done, swapping tmp file now... | 20260507 19:13:32.748 | INFO | RTMetrics done | 20260507 19:13:32.791 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.617 seconds | 20260507 19:13:32.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.66% / 0.01% / 54.33% | 20260507 19:13:32.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.52691% / 1.58946e-05% / 1.58946e-05% | 20260507 19:13:32.791 | INFO | Baz IO overall: | 20260507 19:13:32.791 | INFO | BytesWritten : 865775846812 | 20260507 19:13:32.791 | INFO | NumEvents : 419771423853 | 20260507 19:13:34.198 | INFO | Analyzing chunk frames = [1921024,1921536) | 20260507 19:13:34.198 | INFO | MemInfo: { | 20260507 19:13:34.198 | INFO | "gpu" : " 17 GB", | 20260507 19:13:34.198 | INFO | "huge" : "", | 20260507 19:13:34.198 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:34.198 | INFO | "malloc" : " 14 GB", | 20260507 19:13:34.198 | INFO | "pinned" : " 29 GB" | 20260507 19:13:34.198 | INFO | } | 20260507 19:13:34.198 | INFO | Perf report for the last 5.13911 seconds: | 20260507 19:13:34.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:34.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:34.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:34.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:34.198 | INFO | ANALYSIS : 69.3%, 2.9, 104.1ms, 0 | 20260507 19:13:34.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:34.198 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260507 19:13:34.198 | INFO | PRE_HQ : 68.7%, 1.8, 65.8ms, 0 | 20260507 19:13:34.198 | INFO | BAZWRITER : 70.0%, 4.2, 151.5ms, 0 | 20260507 19:13:34.198 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 19:13:34.198 | INFO | Uploaded 36GiB to GPU in 2.49731s (14.4155GiB/s) | 20260507 19:13:34.198 | INFO | Downloaded 38GiB to GPU in 2.67685s (14.1958GiB/s) | 20260507 19:13:39.337 | INFO | Analyzing chunk frames = [1921536,1922048) | 20260507 19:13:39.337 | INFO | MemInfo: { | 20260507 19:13:39.337 | INFO | "gpu" : " 17 GB", | 20260507 19:13:39.337 | INFO | "huge" : "", | 20260507 19:13:39.337 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:39.337 | INFO | "malloc" : " 14 GB", | 20260507 19:13:39.337 | INFO | "pinned" : " 29 GB" | 20260507 19:13:39.337 | INFO | } | 20260507 19:13:39.337 | INFO | Perf report for the last 5.13911 seconds: | 20260507 19:13:39.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:39.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:13:39.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:13:39.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:39.338 | INFO | Uploaded 36GiB to GPU in 2.50574s (14.367GiB/s) | 20260507 19:13:39.338 | INFO | Downloaded 34GiB to GPU in 2.49021s (13.6535GiB/s) | 20260507 19:13:44.477 | INFO | Analyzing chunk frames = [1922048,1922560) | 20260507 19:13:44.477 | INFO | MemInfo: { | 20260507 19:13:44.477 | INFO | "gpu" : " 17 GB", | 20260507 19:13:44.477 | INFO | "huge" : "", | 20260507 19:13:44.477 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:44.477 | INFO | "malloc" : " 14 GB", | 20260507 19:13:44.477 | INFO | "pinned" : " 29 GB" | 20260507 19:13:44.477 | INFO | } | 20260507 19:13:44.477 | INFO | Perf report for the last 5.13987 seconds: | 20260507 19:13:44.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:44.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:13:44.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:13:44.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:44.477 | INFO | Uploaded 36GiB to GPU in 2.51245s (14.3286GiB/s) | 20260507 19:13:44.477 | INFO | Downloaded 34GiB to GPU in 2.49505s (13.627GiB/s) | 20260507 19:13:49.617 | INFO | Analyzing chunk frames = [1922560,1923072) | 20260507 19:13:49.617 | INFO | MemInfo: { | 20260507 19:13:49.617 | INFO | "gpu" : " 17 GB", | 20260507 19:13:49.617 | INFO | "huge" : "", | 20260507 19:13:49.617 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:49.617 | INFO | "malloc" : " 14 GB", | 20260507 19:13:49.617 | INFO | "pinned" : " 29 GB" | 20260507 19:13:49.617 | INFO | } | 20260507 19:13:49.617 | INFO | Perf report for the last 5.1399 seconds: | 20260507 19:13:49.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:49.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.617 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:13:49.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.617 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:13:49.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:49.618 | INFO | Uploaded 36GiB to GPU in 2.44946s (14.6971GiB/s) | 20260507 19:13:49.618 | INFO | Downloaded 34GiB to GPU in 2.38766s (14.2399GiB/s) | 20260507 19:13:54.755 | INFO | Analyzing chunk frames = [1923072,1923584) | 20260507 19:13:54.755 | INFO | MemInfo: { | 20260507 19:13:54.755 | INFO | "gpu" : " 17 GB", | 20260507 19:13:54.755 | INFO | "huge" : "", | 20260507 19:13:54.755 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:54.755 | INFO | "malloc" : " 14 GB", | 20260507 19:13:54.755 | INFO | "pinned" : " 29 GB" | 20260507 19:13:54.755 | INFO | } | 20260507 19:13:54.755 | INFO | Perf report for the last 5.1381 seconds: | 20260507 19:13:54.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:54.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.755 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:13:54.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.755 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:13:54.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:54.756 | INFO | Uploaded 36GiB to GPU in 2.37045s (15.187GiB/s) | 20260507 19:13:54.756 | INFO | Downloaded 34GiB to GPU in 2.49719s (13.6153GiB/s) | 20260507 19:13:59.894 | INFO | Analyzing chunk frames = [1923584,1924096) | 20260507 19:13:59.894 | INFO | MemInfo: { | 20260507 19:13:59.894 | INFO | "gpu" : " 17 GB", | 20260507 19:13:59.894 | INFO | "huge" : "", | 20260507 19:13:59.894 | INFO | "huge_shared" : " 29 GB", | 20260507 19:13:59.894 | INFO | "malloc" : " 14 GB", | 20260507 19:13:59.894 | INFO | "pinned" : " 29 GB" | 20260507 19:13:59.894 | INFO | } | 20260507 19:13:59.894 | INFO | Perf report for the last 5.13889 seconds: | 20260507 19:13:59.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:13:59.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 19:13:59.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:13:59.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:13:59.894 | INFO | Uploaded 36GiB to GPU in 2.52455s (14.26GiB/s) | 20260507 19:13:59.894 | INFO | Downloaded 34GiB to GPU in 2.41143s (14.0995GiB/s) | 20260507 19:14:05.034 | INFO | Analyzing chunk frames = [1924096,1924608) | 20260507 19:14:05.034 | INFO | MemInfo: { | 20260507 19:14:05.034 | INFO | "gpu" : " 17 GB", | 20260507 19:14:05.034 | INFO | "huge" : "", | 20260507 19:14:05.034 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:05.034 | INFO | "malloc" : " 14 GB", | 20260507 19:14:05.034 | INFO | "pinned" : " 29 GB" | 20260507 19:14:05.035 | INFO | } | 20260507 19:14:05.035 | INFO | Perf report for the last 5.14062 seconds: | 20260507 19:14:05.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:05.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:14:05.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:14:05.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:05.035 | INFO | Uploaded 36GiB to GPU in 2.5498s (14.1187GiB/s) | 20260507 19:14:05.035 | INFO | Downloaded 34GiB to GPU in 2.43575s (13.9587GiB/s) | 20260507 19:14:10.174 | INFO | Analyzing chunk frames = [1924608,1925120) | 20260507 19:14:10.174 | INFO | MemInfo: { | 20260507 19:14:10.174 | INFO | "gpu" : " 17 GB", | 20260507 19:14:10.174 | INFO | "huge" : "", | 20260507 19:14:10.174 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:10.174 | INFO | "malloc" : " 14 GB", | 20260507 19:14:10.174 | INFO | "pinned" : " 29 GB" | 20260507 19:14:10.174 | INFO | } | 20260507 19:14:10.174 | INFO | Perf report for the last 5.13964 seconds: | 20260507 19:14:10.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:10.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.174 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:14:10.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.174 | INFO | PRE_HQ : 37.3%, 1.0, 19.3ms, 0 | 20260507 19:14:10.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:10.175 | INFO | Uploaded 36GiB to GPU in 2.51766s (14.299GiB/s) | 20260507 19:14:10.175 | INFO | Downloaded 34GiB to GPU in 2.42007s (14.0492GiB/s) | 20260507 19:14:10.175 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:14:10.175 | INFO | Total mem usage: 16.4 GiB | 20260507 19:14:10.175 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:14:10.175 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.67%) | 20260507 19:14:10.175 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:14:10.175 | INFO | High Water Breakdown: | 20260507 19:14:10.175 | INFO | 60% : 9.06 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:14:10.175 | INFO | 39.8% : 6.01 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:14:10.175 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:14:10.175 | INFO | ------------------------------------------ | 20260507 19:14:10.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:14:10.175 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:14:10.175 | INFO | Total mem usage: 26.5 GiB | 20260507 19:14:10.175 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:14:10.175 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 19:14:10.175 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:14:10.175 | INFO | High Water Breakdown: | 20260507 19:14:10.175 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:14:10.175 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:14:10.175 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:14:10.175 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:14:10.175 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:14:10.175 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:14:10.175 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:14:10.175 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:14:10.175 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:14:10.175 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:14:10.175 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:14:10.175 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:14:10.175 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:14:10.175 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:14:10.175 | INFO | ------------------------------------------ | 20260507 19:14:10.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:14:10.175 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:14:10.175 | INFO | Total mem usage: 27.8 GiB | 20260507 19:14:10.175 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:14:10.175 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:14:10.175 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:14:10.175 | INFO | High Water Breakdown: | 20260507 19:14:10.175 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:14:10.175 | INFO | ------------------------------------------ | 20260507 19:14:10.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:14:10.175 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:14:10.175 | INFO | Total mem usage: 15.7 GiB | 20260507 19:14:10.175 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:14:10.175 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:14:10.175 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:14:10.175 | INFO | High Water Breakdown: | 20260507 19:14:10.175 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:14:10.175 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:14:10.176 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:14:10.176 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:14:10.176 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:14:10.176 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:14:10.176 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:14:10.176 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:14:10.176 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:14:10.176 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:14:10.176 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:14:10.176 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:14:10.176 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:14:10.176 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:14:10.176 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:14:10.176 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:14:10.176 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:14:10.176 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:14:10.176 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:14:10.176 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:14:10.176 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:14:10.176 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:14:10.176 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:14:10.176 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:14:10.176 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:14:10.176 | INFO | ------------------------------------------ | 20260507 19:14:10.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:14:10.176 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:14:10.176 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:14:10.176 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:14:10.176 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.43e+01 / 1.70e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.02e-01, 1.20e-02 / 6.91e-03 | 20260507 19:14:10.176 | INFO | PerfReport: 12.09%, Baseline -- 4.02e+00 / 4.45e+00, 6.02e-01 / 3.09e-01 | 20260507 19:14:10.176 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.12e-01 / 1.28e-01 | 20260507 19:14:10.176 | INFO | PerfReport: 15.82%, DME -- 4.20e+01 / 4.52e+01, 3.89e+00 / 1.32e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 55.94%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.80e+00 / 1.60e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.53e+00 / 2.85e+00, 4.41e-01 / 2.64e-01 | 20260507 19:14:10.176 | INFO | PerfReport: 3.70%, Metrics -- 1.23e+00 / 1.41e+00, 3.41e-01 / 1.38e-01 | 20260507 19:14:10.176 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.83e-01 / 4.16e-01, 6.21e-02 / 4.02e-02 | 20260507 19:14:10.176 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:14:10.176 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:14:10.176 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:14:10.176 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 24.23%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 24.22%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 31.14%, Compute -- 3.32e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 1.90%, LockUpload -- 2.03e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 2.51%, LockDownload -- 2.68e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 15.99%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 19:14:10.176 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260507 19:14:10.286 | INFO | Starting a round of Baz IO | 20260507 19:14:13.783 | INFO | Populating RTMetrics report | 20260507 19:14:13.783 | INFO | RTMetrics report populated | 20260507 19:14:13.783 | INFO | Opening tmp RTMetrics json file | 20260507 19:14:13.784 | INFO | Serializing RTMetrics to json | 20260507 19:14:13.786 | INFO | Serialization done, swapping tmp file now... | 20260507 19:14:13.786 | INFO | RTMetrics done | 20260507 19:14:13.918 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.631 seconds | 20260507 19:14:13.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.64% / 0.01% / 54.34% | 20260507 19:14:13.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.52145% / 2.78155e-05% / 3.97364e-05% | 20260507 19:14:13.918 | INFO | Baz IO overall: | 20260507 19:14:13.918 | INFO | BytesWritten : 867705003917 | 20260507 19:14:13.918 | INFO | NumEvents : 420706748086 | 20260507 19:14:15.313 | INFO | Analyzing chunk frames = [1925120,1925632) | 20260507 19:14:15.313 | INFO | MemInfo: { | 20260507 19:14:15.313 | INFO | "gpu" : " 17 GB", | 20260507 19:14:15.313 | INFO | "huge" : "", | 20260507 19:14:15.313 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:15.313 | INFO | "malloc" : " 14 GB", | 20260507 19:14:15.313 | INFO | "pinned" : " 29 GB" | 20260507 19:14:15.313 | INFO | } | 20260507 19:14:15.313 | INFO | Perf report for the last 5.13839 seconds: | 20260507 19:14:15.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:15.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:15.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:15.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:15.313 | INFO | ANALYSIS : 69.7%, 2.9, 104.7ms, 0 | 20260507 19:14:15.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:15.313 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 19:14:15.313 | INFO | PRE_HQ : 68.5%, 1.8, 64.5ms, 0 | 20260507 19:14:15.313 | INFO | BAZWRITER : 70.2%, 4.0, 143.4ms, 0 | 20260507 19:14:15.313 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260507 19:14:15.313 | INFO | Uploaded 36GiB to GPU in 2.57115s (14.0015GiB/s) | 20260507 19:14:15.313 | INFO | Downloaded 38GiB to GPU in 2.69459s (14.1023GiB/s) | 20260507 19:14:20.452 | INFO | Analyzing chunk frames = [1925632,1926144) | 20260507 19:14:20.452 | INFO | MemInfo: { | 20260507 19:14:20.452 | INFO | "gpu" : " 17 GB", | 20260507 19:14:20.452 | INFO | "huge" : "", | 20260507 19:14:20.452 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:20.452 | INFO | "malloc" : " 14 GB", | 20260507 19:14:20.452 | INFO | "pinned" : " 29 GB" | 20260507 19:14:20.452 | INFO | } | 20260507 19:14:20.452 | INFO | Perf report for the last 5.13979 seconds: | 20260507 19:14:20.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:20.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.452 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:14:20.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.453 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:14:20.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:20.453 | INFO | Uploaded 36GiB to GPU in 2.55988s (14.0632GiB/s) | 20260507 19:14:20.453 | INFO | Downloaded 34GiB to GPU in 2.48191s (13.6991GiB/s) | 20260507 19:14:25.591 | INFO | Analyzing chunk frames = [1926144,1926656) | 20260507 19:14:25.591 | INFO | MemInfo: { | 20260507 19:14:25.591 | INFO | "gpu" : " 17 GB", | 20260507 19:14:25.591 | INFO | "huge" : "", | 20260507 19:14:25.591 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:25.591 | INFO | "malloc" : " 14 GB", | 20260507 19:14:25.591 | INFO | "pinned" : " 29 GB" | 20260507 19:14:25.591 | INFO | } | 20260507 19:14:25.591 | INFO | Perf report for the last 5.13894 seconds: | 20260507 19:14:25.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:25.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.591 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 19:14:25.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.591 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:14:25.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:25.592 | INFO | Uploaded 36GiB to GPU in 2.59787s (13.8575GiB/s) | 20260507 19:14:25.592 | INFO | Downloaded 34GiB to GPU in 2.4624s (13.8076GiB/s) | 20260507 19:14:30.730 | INFO | Analyzing chunk frames = [1926656,1927168) | 20260507 19:14:30.730 | INFO | MemInfo: { | 20260507 19:14:30.730 | INFO | "gpu" : " 17 GB", | 20260507 19:14:30.730 | INFO | "huge" : "", | 20260507 19:14:30.731 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:30.731 | INFO | "malloc" : " 14 GB", | 20260507 19:14:30.731 | INFO | "pinned" : " 29 GB" | 20260507 19:14:30.731 | INFO | } | 20260507 19:14:30.731 | INFO | Perf report for the last 5.13932 seconds: | 20260507 19:14:30.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:30.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:14:30.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:14:30.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:30.731 | INFO | Uploaded 36GiB to GPU in 2.49786s (14.4124GiB/s) | 20260507 19:14:30.731 | INFO | Downloaded 34GiB to GPU in 2.38943s (14.2293GiB/s) | 20260507 19:14:35.872 | INFO | Analyzing chunk frames = [1927168,1927680) | 20260507 19:14:35.872 | INFO | MemInfo: { | 20260507 19:14:35.872 | INFO | "gpu" : " 17 GB", | 20260507 19:14:35.872 | INFO | "huge" : "", | 20260507 19:14:35.872 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:35.872 | INFO | "malloc" : " 14 GB", | 20260507 19:14:35.872 | INFO | "pinned" : " 29 GB" | 20260507 19:14:35.872 | INFO | } | 20260507 19:14:35.872 | INFO | Perf report for the last 5.14092 seconds: | 20260507 19:14:35.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:35.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:14:35.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:14:35.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:35.872 | INFO | Uploaded 36GiB to GPU in 2.39395s (15.0379GiB/s) | 20260507 19:14:35.872 | INFO | Downloaded 34GiB to GPU in 2.49401s (13.6327GiB/s) | 20260507 19:14:41.009 | INFO | Analyzing chunk frames = [1927680,1928192) | 20260507 19:14:41.009 | INFO | MemInfo: { | 20260507 19:14:41.009 | INFO | "gpu" : " 17 GB", | 20260507 19:14:41.009 | INFO | "huge" : "", | 20260507 19:14:41.009 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:41.009 | INFO | "malloc" : " 14 GB", | 20260507 19:14:41.009 | INFO | "pinned" : " 29 GB" | 20260507 19:14:41.009 | INFO | } | 20260507 19:14:41.009 | INFO | Perf report for the last 5.13752 seconds: | 20260507 19:14:41.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:41.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.009 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:14:41.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.010 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:14:41.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:41.010 | INFO | Uploaded 36GiB to GPU in 2.51338s (14.3234GiB/s) | 20260507 19:14:41.010 | INFO | Downloaded 34GiB to GPU in 2.41164s (14.0983GiB/s) | 20260507 19:14:46.149 | INFO | Analyzing chunk frames = [1928192,1928704) | 20260507 19:14:46.149 | INFO | MemInfo: { | 20260507 19:14:46.149 | INFO | "gpu" : " 17 GB", | 20260507 19:14:46.149 | INFO | "huge" : "", | 20260507 19:14:46.149 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:46.149 | INFO | "malloc" : " 14 GB", | 20260507 19:14:46.149 | INFO | "pinned" : " 29 GB" | 20260507 19:14:46.149 | INFO | } | 20260507 19:14:46.149 | INFO | Perf report for the last 5.14013 seconds: | 20260507 19:14:46.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:46.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.149 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260507 19:14:46.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.149 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 19:14:46.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:46.150 | INFO | Uploaded 36GiB to GPU in 2.53532s (14.1994GiB/s) | 20260507 19:14:46.150 | INFO | Downloaded 34GiB to GPU in 2.43439s (13.9665GiB/s) | 20260507 19:14:51.288 | INFO | Analyzing chunk frames = [1928704,1929216) | 20260507 19:14:51.288 | INFO | MemInfo: { | 20260507 19:14:51.288 | INFO | "gpu" : " 17 GB", | 20260507 19:14:51.288 | INFO | "huge" : "", | 20260507 19:14:51.288 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:51.289 | INFO | "malloc" : " 14 GB", | 20260507 19:14:51.289 | INFO | "pinned" : " 29 GB" | 20260507 19:14:51.289 | INFO | } | 20260507 19:14:51.289 | INFO | Perf report for the last 5.13951 seconds: | 20260507 19:14:51.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:51.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:14:51.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 19:14:51.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:51.289 | INFO | Uploaded 36GiB to GPU in 2.52782s (14.2415GiB/s) | 20260507 19:14:51.289 | INFO | Downloaded 34GiB to GPU in 2.41114s (14.1012GiB/s) | 20260507 19:14:51.401 | INFO | Starting a round of Baz IO | 20260507 19:14:54.888 | INFO | Populating RTMetrics report | 20260507 19:14:54.889 | INFO | RTMetrics report populated | 20260507 19:14:54.889 | INFO | Opening tmp RTMetrics json file | 20260507 19:14:54.899 | INFO | Serializing RTMetrics to json | 20260507 19:14:54.932 | INFO | Serialization done, swapping tmp file now... | 20260507 19:14:54.964 | INFO | RTMetrics done | 20260507 19:14:55.032 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.631 seconds | 20260507 19:14:55.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.63% / 0.01% / 54.36% | 20260507 19:14:55.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.51461% / 1.19209e-05% / 3.97364e-05% | 20260507 19:14:55.032 | INFO | Baz IO overall: | 20260507 19:14:55.032 | INFO | BytesWritten : 869634087967 | 20260507 19:14:55.032 | INFO | NumEvents : 421642131075 | 20260507 19:14:56.427 | INFO | Analyzing chunk frames = [1929216,1929728) | 20260507 19:14:56.427 | INFO | MemInfo: { | 20260507 19:14:56.427 | INFO | "gpu" : " 17 GB", | 20260507 19:14:56.427 | INFO | "huge" : "", | 20260507 19:14:56.427 | INFO | "huge_shared" : " 29 GB", | 20260507 19:14:56.428 | INFO | "malloc" : " 14 GB", | 20260507 19:14:56.428 | INFO | "pinned" : " 29 GB" | 20260507 19:14:56.428 | INFO | } | 20260507 19:14:56.428 | INFO | Perf report for the last 5.13883 seconds: | 20260507 19:14:56.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:14:56.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:56.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:56.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:56.428 | INFO | ANALYSIS : 69.2%, 2.9, 104.9ms, 0 | 20260507 19:14:56.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:14:56.428 | INFO | RT_METRICS : 50.1%, 1.0, 26.3ms, 0 | 20260507 19:14:56.428 | INFO | PRE_HQ : 68.6%, 1.9, 69.2ms, 0 | 20260507 19:14:56.428 | INFO | BAZWRITER : 70.3%, 4.2, 150.6ms, 0 | 20260507 19:14:56.428 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 19:14:56.428 | INFO | Uploaded 36GiB to GPU in 2.56731s (14.0224GiB/s) | 20260507 19:14:56.428 | INFO | Downloaded 38GiB to GPU in 2.68992s (14.1268GiB/s) | 20260507 19:15:01.567 | INFO | Analyzing chunk frames = [1929728,1930240) | 20260507 19:15:01.567 | INFO | MemInfo: { | 20260507 19:15:01.567 | INFO | "gpu" : " 17 GB", | 20260507 19:15:01.567 | INFO | "huge" : "", | 20260507 19:15:01.568 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:01.568 | INFO | "malloc" : " 14 GB", | 20260507 19:15:01.568 | INFO | "pinned" : " 29 GB" | 20260507 19:15:01.568 | INFO | } | 20260507 19:15:01.568 | INFO | Perf report for the last 5.13995 seconds: | 20260507 19:15:01.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:01.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | ANALYSIS : 68.6%, 3.0, 106.9ms, 0 | 20260507 19:15:01.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 19:15:01.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:01.568 | INFO | Uploaded 36GiB to GPU in 2.6055s (13.8169GiB/s) | 20260507 19:15:01.568 | INFO | Downloaded 34GiB to GPU in 2.47153s (13.7566GiB/s) | 20260507 19:15:06.707 | INFO | Analyzing chunk frames = [1930240,1930752) | 20260507 19:15:06.707 | INFO | MemInfo: { | 20260507 19:15:06.707 | INFO | "gpu" : " 17 GB", | 20260507 19:15:06.707 | INFO | "huge" : "", | 20260507 19:15:06.707 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:06.707 | INFO | "malloc" : " 14 GB", | 20260507 19:15:06.707 | INFO | "pinned" : " 29 GB" | 20260507 19:15:06.707 | INFO | } | 20260507 19:15:06.707 | INFO | Perf report for the last 5.13928 seconds: | 20260507 19:15:06.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:06.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 19:15:06.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 19:15:06.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:06.707 | INFO | Uploaded 36GiB to GPU in 2.60633s (13.8125GiB/s) | 20260507 19:15:06.707 | INFO | Downloaded 34GiB to GPU in 2.45857s (13.8292GiB/s) | 20260507 19:15:11.846 | INFO | Analyzing chunk frames = [1930752,1931264) | 20260507 19:15:11.846 | INFO | MemInfo: { | 20260507 19:15:11.846 | INFO | "gpu" : " 17 GB", | 20260507 19:15:11.846 | INFO | "huge" : "", | 20260507 19:15:11.846 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:11.846 | INFO | "malloc" : " 14 GB", | 20260507 19:15:11.846 | INFO | "pinned" : " 29 GB" | 20260507 19:15:11.846 | INFO | } | 20260507 19:15:11.846 | INFO | Perf report for the last 5.13903 seconds: | 20260507 19:15:11.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:11.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | ANALYSIS : 67.9%, 3.0, 105.1ms, 0 | 20260507 19:15:11.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 19:15:11.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:11.846 | INFO | Uploaded 36GiB to GPU in 2.55794s (14.0739GiB/s) | 20260507 19:15:11.847 | INFO | Downloaded 34GiB to GPU in 2.48896s (13.6603GiB/s) | 20260507 19:15:16.985 | INFO | Analyzing chunk frames = [1931264,1931776) | 20260507 19:15:16.985 | INFO | MemInfo: { | 20260507 19:15:16.985 | INFO | "gpu" : " 17 GB", | 20260507 19:15:16.985 | INFO | "huge" : "", | 20260507 19:15:16.985 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:16.986 | INFO | "malloc" : " 14 GB", | 20260507 19:15:16.986 | INFO | "pinned" : " 29 GB" | 20260507 19:15:16.986 | INFO | } | 20260507 19:15:16.986 | INFO | Perf report for the last 5.1395 seconds: | 20260507 19:15:16.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:16.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260507 19:15:16.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 19:15:16.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:16.986 | INFO | Uploaded 36GiB to GPU in 2.44271s (14.7377GiB/s) | 20260507 19:15:16.986 | INFO | Downloaded 34GiB to GPU in 2.4896s (13.6568GiB/s) | 20260507 19:15:22.125 | INFO | Analyzing chunk frames = [1931776,1932288) | 20260507 19:15:22.125 | INFO | MemInfo: { | 20260507 19:15:22.125 | INFO | "gpu" : " 17 GB", | 20260507 19:15:22.125 | INFO | "huge" : "", | 20260507 19:15:22.125 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:22.125 | INFO | "malloc" : " 14 GB", | 20260507 19:15:22.125 | INFO | "pinned" : " 29 GB" | 20260507 19:15:22.125 | INFO | } | 20260507 19:15:22.125 | INFO | Perf report for the last 5.13965 seconds: | 20260507 19:15:22.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:22.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.125 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:15:22.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.125 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:15:22.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:22.126 | INFO | Uploaded 36GiB to GPU in 2.49571s (14.4248GiB/s) | 20260507 19:15:22.126 | INFO | Downloaded 34GiB to GPU in 2.41651s (14.0699GiB/s) | 20260507 19:15:27.263 | INFO | Analyzing chunk frames = [1932288,1932800) | 20260507 19:15:27.263 | INFO | MemInfo: { | 20260507 19:15:27.263 | INFO | "gpu" : " 17 GB", | 20260507 19:15:27.263 | INFO | "huge" : "", | 20260507 19:15:27.263 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:27.263 | INFO | "malloc" : " 14 GB", | 20260507 19:15:27.263 | INFO | "pinned" : " 29 GB" | 20260507 19:15:27.263 | INFO | } | 20260507 19:15:27.263 | INFO | Perf report for the last 5.13823 seconds: | 20260507 19:15:27.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:27.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:15:27.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:15:27.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:27.264 | INFO | Uploaded 36GiB to GPU in 2.51159s (14.3336GiB/s) | 20260507 19:15:27.264 | INFO | Downloaded 34GiB to GPU in 2.44886s (13.884GiB/s) | 20260507 19:15:32.403 | INFO | Analyzing chunk frames = [1932800,1933312) | 20260507 19:15:32.403 | INFO | MemInfo: { | 20260507 19:15:32.403 | INFO | "gpu" : " 17 GB", | 20260507 19:15:32.403 | INFO | "huge" : "", | 20260507 19:15:32.403 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:32.403 | INFO | "malloc" : " 14 GB", | 20260507 19:15:32.403 | INFO | "pinned" : " 29 GB" | 20260507 19:15:32.403 | INFO | } | 20260507 19:15:32.403 | INFO | Perf report for the last 5.13952 seconds: | 20260507 19:15:32.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:32.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:15:32.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:15:32.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:32.403 | INFO | Uploaded 36GiB to GPU in 2.48617s (14.4801GiB/s) | 20260507 19:15:32.403 | INFO | Downloaded 34GiB to GPU in 2.43336s (13.9725GiB/s) | 20260507 19:15:32.403 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:15:32.403 | INFO | Total mem usage: 16.4 GiB | 20260507 19:15:32.403 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:15:32.403 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.87%) | 20260507 19:15:32.403 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:15:32.403 | INFO | High Water Breakdown: | 20260507 19:15:32.403 | INFO | 59.9% : 8.94 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:15:32.403 | INFO | 40% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:15:32.403 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:15:32.403 | INFO | ------------------------------------------ | 20260507 19:15:32.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:15:32.403 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:15:32.403 | INFO | Total mem usage: 26.5 GiB | 20260507 19:15:32.403 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:15:32.403 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 19:15:32.403 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:15:32.403 | INFO | High Water Breakdown: | 20260507 19:15:32.403 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:15:32.403 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:15:32.403 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:15:32.403 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:15:32.403 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:15:32.403 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:15:32.403 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:15:32.403 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:15:32.403 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:15:32.403 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:15:32.403 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:15:32.403 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:15:32.403 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:15:32.403 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:15:32.403 | INFO | ------------------------------------------ | 20260507 19:15:32.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:15:32.403 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:15:32.403 | INFO | Total mem usage: 27.8 GiB | 20260507 19:15:32.403 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:15:32.403 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:15:32.403 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:15:32.403 | INFO | High Water Breakdown: | 20260507 19:15:32.403 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:15:32.403 | INFO | ------------------------------------------ | 20260507 19:15:32.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:15:32.403 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:15:32.403 | INFO | Total mem usage: 15.7 GiB | 20260507 19:15:32.403 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:15:32.403 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:15:32.403 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:15:32.403 | INFO | High Water Breakdown: | 20260507 19:15:32.403 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:15:32.403 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:15:32.403 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:15:32.403 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:15:32.403 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:15:32.403 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:15:32.403 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:15:32.403 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:15:32.403 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:15:32.403 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:15:32.403 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:15:32.403 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:15:32.403 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:15:32.404 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:15:32.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:15:32.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:15:32.404 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:15:32.404 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:15:32.404 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:15:32.404 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:15:32.404 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:15:32.404 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:15:32.404 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:15:32.404 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:15:32.404 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:15:32.404 | INFO | ------------------------------------------ | 20260507 19:15:32.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:15:32.404 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:15:32.404 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:15:32.404 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:15:32.404 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.59e+01, 1.42e+01 / 2.01e+00 | 20260507 19:15:32.404 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.05e-01, 1.14e-02 / 6.17e-03 | 20260507 19:15:32.404 | INFO | PerfReport: 12.08%, Baseline -- 4.02e+00 / 4.44e+00, 5.87e-01 / 2.87e-01 | 20260507 19:15:32.404 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.52e+00, 2.16e-01 / 1.65e-01 | 20260507 19:15:32.404 | INFO | PerfReport: 15.81%, DME -- 4.21e+01 / 4.53e+01, 4.08e+00 / 1.68e+00 | 20260507 19:15:32.404 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.87e+00 | 20260507 19:15:32.404 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.53e+00 / 2.85e+00, 4.43e-01 / 2.22e-01 | 20260507 19:15:32.404 | INFO | PerfReport: 3.67%, Metrics -- 1.22e+00 / 1.41e+00, 3.09e-01 / 1.70e-01 | 20260507 19:15:32.404 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.85e-01 / 4.16e-01, 6.18e-02 / 3.32e-02 | 20260507 19:15:32.404 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:15:32.405 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:15:32.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:15:32.405 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 24.60%, Upload -- 2.63e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 24.12%, Download -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 31.05%, Compute -- 3.33e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 2.23%, LockUpload -- 2.39e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 2.15%, LockDownload -- 2.30e+00 / 0.00e+00, 3.48e+00 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 15.84%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 19:15:32.405 | INFO | PerfReport: 0.01%, Misc -- 1.55e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 19:15:32.500 | INFO | Starting a round of Baz IO | 20260507 19:15:35.981 | INFO | Populating RTMetrics report | 20260507 19:15:35.981 | INFO | RTMetrics report populated | 20260507 19:15:35.981 | INFO | Opening tmp RTMetrics json file | 20260507 19:15:36.055 | INFO | Serializing RTMetrics to json | 20260507 19:15:36.116 | INFO | Serialization done, swapping tmp file now... | 20260507 19:15:36.121 | INFO | RTMetrics done | 20260507 19:15:36.144 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.643 seconds | 20260507 19:15:36.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.62% / 0.01% / 54.37% | 20260507 19:15:36.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.51531% / 3.57628e-05% / 4.76837e-05% | 20260507 19:15:36.144 | INFO | Baz IO overall: | 20260507 19:15:36.144 | INFO | BytesWritten : 871564278928 | 20260507 19:15:36.144 | INFO | NumEvents : 422578590002 | 20260507 19:15:37.544 | INFO | Analyzing chunk frames = [1933312,1933824) | 20260507 19:15:37.544 | INFO | MemInfo: { | 20260507 19:15:37.544 | INFO | "gpu" : " 17 GB", | 20260507 19:15:37.544 | INFO | "huge" : "", | 20260507 19:15:37.544 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:37.544 | INFO | "malloc" : " 14 GB", | 20260507 19:15:37.544 | INFO | "pinned" : " 29 GB" | 20260507 19:15:37.544 | INFO | } | 20260507 19:15:37.544 | INFO | Perf report for the last 5.14063 seconds: | 20260507 19:15:37.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:37.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:37.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:37.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:37.544 | INFO | ANALYSIS : 68.8%, 3.0, 107.3ms, 0 | 20260507 19:15:37.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:37.544 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 19:15:37.544 | INFO | PRE_HQ : 66.0%, 1.7, 58.6ms, 0 | 20260507 19:15:37.544 | INFO | BAZWRITER : 70.4%, 3.9, 141.1ms, 0 | 20260507 19:15:37.544 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 19:15:37.544 | INFO | Uploaded 36GiB to GPU in 2.52941s (14.2326GiB/s) | 20260507 19:15:37.545 | INFO | Downloaded 38GiB to GPU in 2.6939s (14.1059GiB/s) | 20260507 19:15:42.682 | INFO | Analyzing chunk frames = [1933824,1934336) | 20260507 19:15:42.682 | INFO | MemInfo: { | 20260507 19:15:42.682 | INFO | "gpu" : " 17 GB", | 20260507 19:15:42.682 | INFO | "huge" : "", | 20260507 19:15:42.682 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:42.682 | INFO | "malloc" : " 14 GB", | 20260507 19:15:42.682 | INFO | "pinned" : " 29 GB" | 20260507 19:15:42.682 | INFO | } | 20260507 19:15:42.682 | INFO | Perf report for the last 5.13812 seconds: | 20260507 19:15:42.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:42.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.682 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:15:42.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.682 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:15:42.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:42.683 | INFO | Uploaded 36GiB to GPU in 2.55562s (14.0866GiB/s) | 20260507 19:15:42.683 | INFO | Downloaded 34GiB to GPU in 2.48892s (13.6606GiB/s) | 20260507 19:15:47.822 | INFO | Analyzing chunk frames = [1934336,1934848) | 20260507 19:15:47.822 | INFO | MemInfo: { | 20260507 19:15:47.822 | INFO | "gpu" : " 17 GB", | 20260507 19:15:47.822 | INFO | "huge" : "", | 20260507 19:15:47.822 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:47.822 | INFO | "malloc" : " 14 GB", | 20260507 19:15:47.822 | INFO | "pinned" : " 29 GB" | 20260507 19:15:47.822 | INFO | } | 20260507 19:15:47.822 | INFO | Perf report for the last 5.14031 seconds: | 20260507 19:15:47.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:47.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:15:47.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:15:47.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:47.822 | INFO | Uploaded 36GiB to GPU in 2.55594s (14.0849GiB/s) | 20260507 19:15:47.822 | INFO | Downloaded 34GiB to GPU in 2.4718s (13.7552GiB/s) | 20260507 19:15:52.962 | INFO | Analyzing chunk frames = [1934848,1935360) | 20260507 19:15:52.962 | INFO | MemInfo: { | 20260507 19:15:52.962 | INFO | "gpu" : " 17 GB", | 20260507 19:15:52.962 | INFO | "huge" : "", | 20260507 19:15:52.962 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:52.962 | INFO | "malloc" : " 14 GB", | 20260507 19:15:52.962 | INFO | "pinned" : " 29 GB" | 20260507 19:15:52.962 | INFO | } | 20260507 19:15:52.962 | INFO | Perf report for the last 5.13996 seconds: | 20260507 19:15:52.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:52.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.962 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:15:52.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.962 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:15:52.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:52.963 | INFO | Uploaded 36GiB to GPU in 2.43766s (14.7683GiB/s) | 20260507 19:15:52.963 | INFO | Downloaded 34GiB to GPU in 2.39119s (14.2189GiB/s) | 20260507 19:15:58.100 | INFO | Analyzing chunk frames = [1935360,1935872) | 20260507 19:15:58.100 | INFO | MemInfo: { | 20260507 19:15:58.100 | INFO | "gpu" : " 17 GB", | 20260507 19:15:58.100 | INFO | "huge" : "", | 20260507 19:15:58.100 | INFO | "huge_shared" : " 29 GB", | 20260507 19:15:58.100 | INFO | "malloc" : " 14 GB", | 20260507 19:15:58.100 | INFO | "pinned" : " 29 GB" | 20260507 19:15:58.100 | INFO | } | 20260507 19:15:58.100 | INFO | Perf report for the last 5.13816 seconds: | 20260507 19:15:58.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:15:58.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.100 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:15:58.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.101 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260507 19:15:58.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:15:58.101 | INFO | Uploaded 36GiB to GPU in 2.36299s (15.235GiB/s) | 20260507 19:15:58.101 | INFO | Downloaded 34GiB to GPU in 2.49648s (13.6192GiB/s) | 20260507 19:16:03.239 | INFO | Analyzing chunk frames = [1935872,1936384) | 20260507 19:16:03.239 | INFO | MemInfo: { | 20260507 19:16:03.239 | INFO | "gpu" : " 17 GB", | 20260507 19:16:03.239 | INFO | "huge" : "", | 20260507 19:16:03.239 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:03.239 | INFO | "malloc" : " 14 GB", | 20260507 19:16:03.239 | INFO | "pinned" : " 29 GB" | 20260507 19:16:03.239 | INFO | } | 20260507 19:16:03.239 | INFO | Perf report for the last 5.1388 seconds: | 20260507 19:16:03.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:03.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.240 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:16:03.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.240 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:16:03.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:03.240 | INFO | Uploaded 36GiB to GPU in 2.50803s (14.3539GiB/s) | 20260507 19:16:03.240 | INFO | Downloaded 34GiB to GPU in 2.4084s (14.1173GiB/s) | 20260507 19:16:08.379 | INFO | Analyzing chunk frames = [1936384,1936896) | 20260507 19:16:08.379 | INFO | MemInfo: { | 20260507 19:16:08.379 | INFO | "gpu" : " 17 GB", | 20260507 19:16:08.379 | INFO | "huge" : "", | 20260507 19:16:08.379 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:08.379 | INFO | "malloc" : " 14 GB", | 20260507 19:16:08.379 | INFO | "pinned" : " 29 GB" | 20260507 19:16:08.379 | INFO | } | 20260507 19:16:08.379 | INFO | Perf report for the last 5.14005 seconds: | 20260507 19:16:08.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:08.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.379 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:16:08.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.379 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:16:08.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:08.380 | INFO | Uploaded 36GiB to GPU in 2.54666s (14.1362GiB/s) | 20260507 19:16:08.380 | INFO | Downloaded 34GiB to GPU in 2.43625s (13.9558GiB/s) | 20260507 19:16:13.517 | INFO | Analyzing chunk frames = [1936896,1937408) | 20260507 19:16:13.517 | INFO | MemInfo: { | 20260507 19:16:13.517 | INFO | "gpu" : " 17 GB", | 20260507 19:16:13.517 | INFO | "huge" : "", | 20260507 19:16:13.517 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:13.517 | INFO | "malloc" : " 14 GB", | 20260507 19:16:13.517 | INFO | "pinned" : " 29 GB" | 20260507 19:16:13.517 | INFO | } | 20260507 19:16:13.517 | INFO | Perf report for the last 5.13833 seconds: | 20260507 19:16:13.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:13.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:16:13.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:16:13.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:13.518 | INFO | Uploaded 36GiB to GPU in 2.51451s (14.3169GiB/s) | 20260507 19:16:13.518 | INFO | Downloaded 34GiB to GPU in 2.41879s (14.0566GiB/s) | 20260507 19:16:13.615 | INFO | Starting a round of Baz IO | 20260507 19:16:17.106 | INFO | Populating RTMetrics report | 20260507 19:16:17.106 | INFO | RTMetrics report populated | 20260507 19:16:17.106 | INFO | Opening tmp RTMetrics json file | 20260507 19:16:17.108 | INFO | Serializing RTMetrics to json | 20260507 19:16:17.133 | INFO | Serialization done, swapping tmp file now... | 20260507 19:16:17.138 | INFO | RTMetrics done | 20260507 19:16:17.232 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.617 seconds | 20260507 19:16:17.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.61% / 0.01% / 54.38% | 20260507 19:16:17.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.50199% / 1.98682e-05% / 5.16574e-05% | 20260507 19:16:17.232 | INFO | Baz IO overall: | 20260507 19:16:17.232 | INFO | BytesWritten : 873492346450 | 20260507 19:16:17.232 | INFO | NumEvents : 423513202761 | 20260507 19:16:18.658 | INFO | Analyzing chunk frames = [1937408,1937920) | 20260507 19:16:18.658 | INFO | MemInfo: { | 20260507 19:16:18.658 | INFO | "gpu" : " 17 GB", | 20260507 19:16:18.658 | INFO | "huge" : "", | 20260507 19:16:18.658 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:18.658 | INFO | "malloc" : " 14 GB", | 20260507 19:16:18.658 | INFO | "pinned" : " 29 GB" | 20260507 19:16:18.658 | INFO | } | 20260507 19:16:18.658 | INFO | Perf report for the last 5.14024 seconds: | 20260507 19:16:18.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:18.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:18.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:18.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:18.658 | INFO | ANALYSIS : 69.1%, 3.0, 106.9ms, 0 | 20260507 19:16:18.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:18.658 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 19:16:18.658 | INFO | PRE_HQ : 66.3%, 1.7, 58.7ms, 0 | 20260507 19:16:18.658 | INFO | BAZWRITER : 70.0%, 4.0, 144.6ms, 0 | 20260507 19:16:18.658 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 19:16:18.658 | INFO | Uploaded 36GiB to GPU in 2.55198s (14.1067GiB/s) | 20260507 19:16:18.659 | INFO | Downloaded 38GiB to GPU in 2.69979s (14.0752GiB/s) | 20260507 19:16:23.797 | INFO | Analyzing chunk frames = [1937920,1938432) | 20260507 19:16:23.797 | INFO | MemInfo: { | 20260507 19:16:23.797 | INFO | "gpu" : " 17 GB", | 20260507 19:16:23.797 | INFO | "huge" : "", | 20260507 19:16:23.797 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:23.797 | INFO | "malloc" : " 14 GB", | 20260507 19:16:23.797 | INFO | "pinned" : " 29 GB" | 20260507 19:16:23.797 | INFO | } | 20260507 19:16:23.797 | INFO | Perf report for the last 5.1396 seconds: | 20260507 19:16:23.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:23.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:16:23.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 19:16:23.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:23.798 | INFO | Uploaded 36GiB to GPU in 2.49986s (14.4008GiB/s) | 20260507 19:16:23.798 | INFO | Downloaded 34GiB to GPU in 2.474s (13.7429GiB/s) | 20260507 19:16:28.937 | INFO | Analyzing chunk frames = [1938432,1938944) | 20260507 19:16:28.937 | INFO | MemInfo: { | 20260507 19:16:28.937 | INFO | "gpu" : " 17 GB", | 20260507 19:16:28.937 | INFO | "huge" : "", | 20260507 19:16:28.937 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:28.937 | INFO | "malloc" : " 14 GB", | 20260507 19:16:28.937 | INFO | "pinned" : " 29 GB" | 20260507 19:16:28.937 | INFO | } | 20260507 19:16:28.937 | INFO | Perf report for the last 5.13971 seconds: | 20260507 19:16:28.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:28.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:16:28.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:16:28.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:28.937 | INFO | Uploaded 36GiB to GPU in 2.54973s (14.1191GiB/s) | 20260507 19:16:28.937 | INFO | Downloaded 34GiB to GPU in 2.46324s (13.803GiB/s) | 20260507 19:16:34.076 | INFO | Analyzing chunk frames = [1938944,1939456) | 20260507 19:16:34.076 | INFO | MemInfo: { | 20260507 19:16:34.076 | INFO | "gpu" : " 17 GB", | 20260507 19:16:34.076 | INFO | "huge" : "", | 20260507 19:16:34.076 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:34.076 | INFO | "malloc" : " 14 GB", | 20260507 19:16:34.076 | INFO | "pinned" : " 29 GB" | 20260507 19:16:34.076 | INFO | } | 20260507 19:16:34.076 | INFO | Perf report for the last 5.13875 seconds: | 20260507 19:16:34.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:34.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:16:34.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:16:34.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:34.076 | INFO | Uploaded 36GiB to GPU in 2.46261s (14.6186GiB/s) | 20260507 19:16:34.076 | INFO | Downloaded 34GiB to GPU in 2.39764s (14.1806GiB/s) | 20260507 19:16:39.214 | INFO | Analyzing chunk frames = [1939456,1939968) | 20260507 19:16:39.214 | INFO | MemInfo: { | 20260507 19:16:39.214 | INFO | "gpu" : " 17 GB", | 20260507 19:16:39.214 | INFO | "huge" : "", | 20260507 19:16:39.214 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:39.214 | INFO | "malloc" : " 14 GB", | 20260507 19:16:39.214 | INFO | "pinned" : " 29 GB" | 20260507 19:16:39.214 | INFO | } | 20260507 19:16:39.214 | INFO | Perf report for the last 5.13832 seconds: | 20260507 19:16:39.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:39.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.214 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 19:16:39.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.214 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:16:39.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:39.215 | INFO | Uploaded 36GiB to GPU in 2.31041s (15.5817GiB/s) | 20260507 19:16:39.215 | INFO | Downloaded 34GiB to GPU in 2.51339s (13.5275GiB/s) | 20260507 19:16:44.355 | INFO | Analyzing chunk frames = [1939968,1940480) | 20260507 19:16:44.355 | INFO | MemInfo: { | 20260507 19:16:44.355 | INFO | "gpu" : " 17 GB", | 20260507 19:16:44.355 | INFO | "huge" : "", | 20260507 19:16:44.355 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:44.355 | INFO | "malloc" : " 14 GB", | 20260507 19:16:44.355 | INFO | "pinned" : " 29 GB" | 20260507 19:16:44.355 | INFO | } | 20260507 19:16:44.355 | INFO | Perf report for the last 5.14073 seconds: | 20260507 19:16:44.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:44.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.355 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:16:44.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.355 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:16:44.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:44.356 | INFO | Uploaded 36GiB to GPU in 2.4841s (14.4921GiB/s) | 20260507 19:16:44.356 | INFO | Downloaded 34GiB to GPU in 2.41482s (14.0797GiB/s) | 20260507 19:16:49.494 | INFO | Analyzing chunk frames = [1940480,1940992) | 20260507 19:16:49.494 | INFO | MemInfo: { | 20260507 19:16:49.494 | INFO | "gpu" : " 17 GB", | 20260507 19:16:49.494 | INFO | "huge" : "", | 20260507 19:16:49.494 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:49.494 | INFO | "malloc" : " 14 GB", | 20260507 19:16:49.494 | INFO | "pinned" : " 29 GB" | 20260507 19:16:49.494 | INFO | } | 20260507 19:16:49.495 | INFO | Perf report for the last 5.13961 seconds: | 20260507 19:16:49.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:49.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:16:49.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:16:49.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:49.495 | INFO | Uploaded 36GiB to GPU in 2.51981s (14.2868GiB/s) | 20260507 19:16:49.495 | INFO | Downloaded 34GiB to GPU in 2.45038s (13.8754GiB/s) | 20260507 19:16:54.634 | INFO | Analyzing chunk frames = [1940992,1941504) | 20260507 19:16:54.634 | INFO | MemInfo: { | 20260507 19:16:54.634 | INFO | "gpu" : " 17 GB", | 20260507 19:16:54.634 | INFO | "huge" : "", | 20260507 19:16:54.634 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:54.634 | INFO | "malloc" : " 14 GB", | 20260507 19:16:54.634 | INFO | "pinned" : " 29 GB" | 20260507 19:16:54.634 | INFO | } | 20260507 19:16:54.634 | INFO | Perf report for the last 5.13963 seconds: | 20260507 19:16:54.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:54.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.634 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:16:54.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.634 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:16:54.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:54.635 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:16:54.635 | INFO | Total mem usage: 16.4 GiB | 20260507 19:16:54.635 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:16:54.635 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11.1%) | 20260507 19:16:54.635 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:16:54.635 | INFO | High Water Breakdown: | 20260507 19:16:54.635 | INFO | 59.7% : 8.81 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:16:54.635 | INFO | 40.2% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:16:54.635 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:16:54.635 | INFO | ------------------------------------------ | 20260507 19:16:54.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:16:54.635 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:16:54.635 | INFO | Total mem usage: 26.5 GiB | 20260507 19:16:54.635 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:16:54.635 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:16:54.635 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:16:54.635 | INFO | High Water Breakdown: | 20260507 19:16:54.635 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:16:54.635 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:16:54.635 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:16:54.635 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:16:54.635 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:16:54.635 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:16:54.635 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:16:54.635 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:16:54.635 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:16:54.635 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:16:54.635 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:16:54.635 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:16:54.635 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:16:54.635 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:16:54.635 | INFO | ------------------------------------------ | 20260507 19:16:54.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:16:54.635 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:16:54.635 | INFO | Total mem usage: 27.8 GiB | 20260507 19:16:54.635 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:16:54.635 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:16:54.635 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:16:54.635 | INFO | High Water Breakdown: | 20260507 19:16:54.635 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:16:54.635 | INFO | ------------------------------------------ | 20260507 19:16:54.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:16:54.635 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:16:54.635 | INFO | Total mem usage: 15.7 GiB | 20260507 19:16:54.635 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:16:54.635 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:16:54.635 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:16:54.635 | INFO | High Water Breakdown: | 20260507 19:16:54.635 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:16:54.635 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:16:54.635 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:16:54.635 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:16:54.635 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:16:54.635 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:16:54.635 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:16:54.635 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:16:54.635 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:16:54.635 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:16:54.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:16:54.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:16:54.635 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:16:54.635 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:16:54.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:16:54.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:16:54.635 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:16:54.635 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:16:54.635 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:16:54.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:16:54.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:16:54.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:16:54.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:16:54.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:16:54.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:16:54.635 | INFO | ------------------------------------------ | 20260507 19:16:54.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:16:54.635 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:16:54.635 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:16:54.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:16:54.635 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.41e+01 / 1.81e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.16e-02 / 6.31e-03 | 20260507 19:16:54.635 | INFO | PerfReport: 12.08%, Baseline -- 4.01e+00 / 4.47e+00, 6.09e-01 / 3.01e-01 | 20260507 19:16:54.635 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.13e-01 / 1.22e-01 | 20260507 19:16:54.635 | INFO | PerfReport: 15.77%, DME -- 4.19e+01 / 4.49e+01, 3.89e+00 / 1.28e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.95e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.53e+00 / 2.85e+00, 4.44e-01 / 2.56e-01 | 20260507 19:16:54.635 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.42e+00, 3.09e-01 / 1.83e-01 | 20260507 19:16:54.635 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.80e-01 / 4.16e-01, 6.13e-02 / 3.79e-02 | 20260507 19:16:54.635 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:16:54.635 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:16:54.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:16:54.635 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 24.25%, Upload -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 24.12%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 31.00%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 1.98%, LockUpload -- 2.12e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 2.42%, LockDownload -- 2.59e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 16.21%, LockCompute -- 1.74e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:16:54.635 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260507 19:16:54.635 | INFO | Uploaded 36GiB to GPU in 2.48708s (14.4748GiB/s) | 20260507 19:16:54.635 | INFO | Downloaded 34GiB to GPU in 2.43582s (13.9584GiB/s) | 20260507 19:16:54.744 | INFO | Starting a round of Baz IO | 20260507 19:16:58.234 | INFO | Populating RTMetrics report | 20260507 19:16:58.234 | INFO | RTMetrics report populated | 20260507 19:16:58.234 | INFO | Opening tmp RTMetrics json file | 20260507 19:16:58.307 | INFO | Serializing RTMetrics to json | 20260507 19:16:58.332 | INFO | Serialization done, swapping tmp file now... | 20260507 19:16:58.334 | INFO | RTMetrics done | 20260507 19:16:58.363 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.618 seconds | 20260507 19:16:58.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.59% / 0.01% / 54.39% | 20260507 19:16:58.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.50612% / 1.98682e-05% / 3.17891e-05% | 20260507 19:16:58.363 | INFO | Baz IO overall: | 20260507 19:16:58.363 | INFO | BytesWritten : 875422704017 | 20260507 19:16:58.363 | INFO | NumEvents : 424449562223 | 20260507 19:16:59.772 | INFO | Analyzing chunk frames = [1941504,1942016) | 20260507 19:16:59.772 | INFO | MemInfo: { | 20260507 19:16:59.772 | INFO | "gpu" : " 17 GB", | 20260507 19:16:59.772 | INFO | "huge" : "", | 20260507 19:16:59.772 | INFO | "huge_shared" : " 29 GB", | 20260507 19:16:59.772 | INFO | "malloc" : " 14 GB", | 20260507 19:16:59.772 | INFO | "pinned" : " 29 GB" | 20260507 19:16:59.772 | INFO | } | 20260507 19:16:59.772 | INFO | Perf report for the last 5.13796 seconds: | 20260507 19:16:59.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:16:59.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:59.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:59.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:59.772 | INFO | ANALYSIS : 69.0%, 2.9, 105.9ms, 0 | 20260507 19:16:59.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:16:59.772 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 19:16:59.772 | INFO | PRE_HQ : 67.4%, 1.8, 62.5ms, 0 | 20260507 19:16:59.772 | INFO | BAZWRITER : 69.8%, 4.4, 157.7ms, 0 | 20260507 19:16:59.773 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 19:16:59.773 | INFO | Uploaded 36GiB to GPU in 2.57201s (13.9969GiB/s) | 20260507 19:16:59.773 | INFO | Downloaded 38GiB to GPU in 2.6904s (14.1243GiB/s) | 20260507 19:17:04.913 | INFO | Analyzing chunk frames = [1942016,1942528) | 20260507 19:17:04.913 | INFO | MemInfo: { | 20260507 19:17:04.913 | INFO | "gpu" : " 17 GB", | 20260507 19:17:04.913 | INFO | "huge" : "", | 20260507 19:17:04.913 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:04.913 | INFO | "malloc" : " 14 GB", | 20260507 19:17:04.913 | INFO | "pinned" : " 29 GB" | 20260507 19:17:04.913 | INFO | } | 20260507 19:17:04.913 | INFO | Perf report for the last 5.14062 seconds: | 20260507 19:17:04.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:04.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:17:04.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:17:04.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:04.913 | INFO | Uploaded 36GiB to GPU in 2.56135s (14.0551GiB/s) | 20260507 19:17:04.913 | INFO | Downloaded 34GiB to GPU in 2.47846s (13.7182GiB/s) | 20260507 19:17:10.053 | INFO | Analyzing chunk frames = [1942528,1943040) | 20260507 19:17:10.053 | INFO | MemInfo: { | 20260507 19:17:10.053 | INFO | "gpu" : " 17 GB", | 20260507 19:17:10.053 | INFO | "huge" : "", | 20260507 19:17:10.053 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:10.053 | INFO | "malloc" : " 14 GB", | 20260507 19:17:10.053 | INFO | "pinned" : " 29 GB" | 20260507 19:17:10.053 | INFO | } | 20260507 19:17:10.053 | INFO | Perf report for the last 5.14019 seconds: | 20260507 19:17:10.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:10.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:17:10.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:17:10.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:10.053 | INFO | Uploaded 36GiB to GPU in 2.49875s (14.4072GiB/s) | 20260507 19:17:10.053 | INFO | Downloaded 34GiB to GPU in 2.49499s (13.6273GiB/s) | 20260507 19:17:15.191 | INFO | Analyzing chunk frames = [1943040,1943552) | 20260507 19:17:15.191 | INFO | MemInfo: { | 20260507 19:17:15.191 | INFO | "gpu" : " 17 GB", | 20260507 19:17:15.191 | INFO | "huge" : "", | 20260507 19:17:15.191 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:15.191 | INFO | "malloc" : " 14 GB", | 20260507 19:17:15.191 | INFO | "pinned" : " 29 GB" | 20260507 19:17:15.191 | INFO | } | 20260507 19:17:15.191 | INFO | Perf report for the last 5.13767 seconds: | 20260507 19:17:15.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:15.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:17:15.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:17:15.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:15.191 | INFO | Uploaded 36GiB to GPU in 2.46337s (14.6141GiB/s) | 20260507 19:17:15.191 | INFO | Downloaded 34GiB to GPU in 2.38818s (14.2368GiB/s) | 20260507 19:17:20.330 | INFO | Analyzing chunk frames = [1943552,1944064) | 20260507 19:17:20.330 | INFO | MemInfo: { | 20260507 19:17:20.330 | INFO | "gpu" : " 17 GB", | 20260507 19:17:20.330 | INFO | "huge" : "", | 20260507 19:17:20.330 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:20.330 | INFO | "malloc" : " 14 GB", | 20260507 19:17:20.330 | INFO | "pinned" : " 29 GB" | 20260507 19:17:20.330 | INFO | } | 20260507 19:17:20.330 | INFO | Perf report for the last 5.13961 seconds: | 20260507 19:17:20.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:20.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.330 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:17:20.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.330 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:17:20.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:20.331 | INFO | Uploaded 36GiB to GPU in 2.31948s (15.5207GiB/s) | 20260507 19:17:20.331 | INFO | Downloaded 34GiB to GPU in 2.5093s (13.5496GiB/s) | 20260507 19:17:25.469 | INFO | Analyzing chunk frames = [1944064,1944576) | 20260507 19:17:25.469 | INFO | MemInfo: { | 20260507 19:17:25.469 | INFO | "gpu" : " 17 GB", | 20260507 19:17:25.469 | INFO | "huge" : "", | 20260507 19:17:25.469 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:25.469 | INFO | "malloc" : " 14 GB", | 20260507 19:17:25.469 | INFO | "pinned" : " 29 GB" | 20260507 19:17:25.469 | INFO | } | 20260507 19:17:25.469 | INFO | Perf report for the last 5.13876 seconds: | 20260507 19:17:25.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:25.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:17:25.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:17:25.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:25.469 | INFO | Uploaded 36GiB to GPU in 2.43532s (14.7825GiB/s) | 20260507 19:17:25.469 | INFO | Downloaded 34GiB to GPU in 2.43038s (13.9896GiB/s) | 20260507 19:17:30.608 | INFO | Analyzing chunk frames = [1944576,1945088) | 20260507 19:17:30.608 | INFO | MemInfo: { | 20260507 19:17:30.608 | INFO | "gpu" : " 17 GB", | 20260507 19:17:30.608 | INFO | "huge" : "", | 20260507 19:17:30.608 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:30.608 | INFO | "malloc" : " 14 GB", | 20260507 19:17:30.608 | INFO | "pinned" : " 29 GB" | 20260507 19:17:30.609 | INFO | } | 20260507 19:17:30.609 | INFO | Perf report for the last 5.13954 seconds: | 20260507 19:17:30.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:30.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:17:30.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:17:30.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:30.609 | INFO | Uploaded 36GiB to GPU in 2.47906s (14.5216GiB/s) | 20260507 19:17:30.609 | INFO | Downloaded 34GiB to GPU in 2.46187s (13.8106GiB/s) | 20260507 19:17:35.748 | INFO | Analyzing chunk frames = [1945088,1945600) | 20260507 19:17:35.748 | INFO | MemInfo: { | 20260507 19:17:35.748 | INFO | "gpu" : " 17 GB", | 20260507 19:17:35.748 | INFO | "huge" : "", | 20260507 19:17:35.748 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:35.748 | INFO | "malloc" : " 14 GB", | 20260507 19:17:35.748 | INFO | "pinned" : " 29 GB" | 20260507 19:17:35.748 | INFO | } | 20260507 19:17:35.748 | INFO | Perf report for the last 5.13919 seconds: | 20260507 19:17:35.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:35.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 19:17:35.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:17:35.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:35.748 | INFO | Uploaded 36GiB to GPU in 2.4163s (14.8988GiB/s) | 20260507 19:17:35.748 | INFO | Downloaded 34GiB to GPU in 2.44315s (13.9164GiB/s) | 20260507 19:17:35.845 | INFO | Starting a round of Baz IO | 20260507 19:17:39.328 | INFO | Populating RTMetrics report | 20260507 19:17:39.329 | INFO | RTMetrics report populated | 20260507 19:17:39.329 | INFO | Opening tmp RTMetrics json file | 20260507 19:17:39.435 | INFO | Serializing RTMetrics to json | 20260507 19:17:39.465 | INFO | Serialization done, swapping tmp file now... | 20260507 19:17:39.491 | INFO | RTMetrics done | 20260507 19:17:39.774 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.928 seconds | 20260507 19:17:39.774 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.58% / 0.01% / 54.41% | 20260507 19:17:39.774 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.49558% / 1.58946e-05% / 2.78155e-05% | 20260507 19:17:39.774 | INFO | Baz IO overall: | 20260507 19:17:39.774 | INFO | BytesWritten : 877351378167 | 20260507 19:17:39.774 | INFO | NumEvents : 425384373304 | 20260507 19:17:40.887 | INFO | Analyzing chunk frames = [1945600,1946112) | 20260507 19:17:40.887 | INFO | MemInfo: { | 20260507 19:17:40.887 | INFO | "gpu" : " 17 GB", | 20260507 19:17:40.887 | INFO | "huge" : "", | 20260507 19:17:40.887 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:40.887 | INFO | "malloc" : " 14 GB", | 20260507 19:17:40.887 | INFO | "pinned" : " 29 GB" | 20260507 19:17:40.887 | INFO | } | 20260507 19:17:40.887 | INFO | Perf report for the last 5.13947 seconds: | 20260507 19:17:40.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:40.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:40.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:40.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:40.887 | INFO | ANALYSIS : 69.0%, 2.9, 106.6ms, 0 | 20260507 19:17:40.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:40.888 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 19:17:40.888 | INFO | PRE_HQ : 66.5%, 1.7, 58.7ms, 0 | 20260507 19:17:40.888 | INFO | BAZWRITER : 76.0%, 4.4, 170.8ms, 0 | 20260507 19:17:40.888 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 19:17:40.888 | INFO | Uploaded 36GiB to GPU in 2.48517s (14.4859GiB/s) | 20260507 19:17:40.888 | INFO | Downloaded 38GiB to GPU in 2.68979s (14.1275GiB/s) | 20260507 19:17:46.027 | INFO | Analyzing chunk frames = [1946112,1946624) | 20260507 19:17:46.027 | INFO | MemInfo: { | 20260507 19:17:46.027 | INFO | "gpu" : " 17 GB", | 20260507 19:17:46.027 | INFO | "huge" : "", | 20260507 19:17:46.027 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:46.027 | INFO | "malloc" : " 14 GB", | 20260507 19:17:46.027 | INFO | "pinned" : " 29 GB" | 20260507 19:17:46.027 | INFO | } | 20260507 19:17:46.027 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:17:46.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:46.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260507 19:17:46.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:17:46.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:46.027 | INFO | Uploaded 36GiB to GPU in 2.53147s (14.221GiB/s) | 20260507 19:17:46.027 | INFO | Downloaded 34GiB to GPU in 2.45412s (13.8543GiB/s) | 20260507 19:17:51.165 | INFO | Analyzing chunk frames = [1946624,1947136) | 20260507 19:17:51.165 | INFO | MemInfo: { | 20260507 19:17:51.165 | INFO | "gpu" : " 17 GB", | 20260507 19:17:51.165 | INFO | "huge" : "", | 20260507 19:17:51.165 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:51.165 | INFO | "malloc" : " 14 GB", | 20260507 19:17:51.165 | INFO | "pinned" : " 29 GB" | 20260507 19:17:51.165 | INFO | } | 20260507 19:17:51.165 | INFO | Perf report for the last 5.1388 seconds: | 20260507 19:17:51.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:51.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:17:51.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:17:51.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:51.166 | INFO | Uploaded 36GiB to GPU in 2.53538s (14.1991GiB/s) | 20260507 19:17:51.166 | INFO | Downloaded 34GiB to GPU in 2.46472s (13.7947GiB/s) | 20260507 19:17:56.305 | INFO | Analyzing chunk frames = [1947136,1947648) | 20260507 19:17:56.305 | INFO | MemInfo: { | 20260507 19:17:56.305 | INFO | "gpu" : " 17 GB", | 20260507 19:17:56.305 | INFO | "huge" : "", | 20260507 19:17:56.305 | INFO | "huge_shared" : " 29 GB", | 20260507 19:17:56.305 | INFO | "malloc" : " 14 GB", | 20260507 19:17:56.305 | INFO | "pinned" : " 29 GB" | 20260507 19:17:56.306 | INFO | } | 20260507 19:17:56.306 | INFO | Perf report for the last 5.14011 seconds: | 20260507 19:17:56.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:17:56.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:17:56.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:17:56.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:17:56.306 | INFO | Uploaded 36GiB to GPU in 2.46386s (14.6112GiB/s) | 20260507 19:17:56.306 | INFO | Downloaded 34GiB to GPU in 2.38313s (14.267GiB/s) | 20260507 19:18:01.444 | INFO | Analyzing chunk frames = [1947648,1948160) | 20260507 19:18:01.444 | INFO | MemInfo: { | 20260507 19:18:01.444 | INFO | "gpu" : " 17 GB", | 20260507 19:18:01.444 | INFO | "huge" : "", | 20260507 19:18:01.444 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:01.444 | INFO | "malloc" : " 14 GB", | 20260507 19:18:01.444 | INFO | "pinned" : " 29 GB" | 20260507 19:18:01.444 | INFO | } | 20260507 19:18:01.445 | INFO | Perf report for the last 5.13896 seconds: | 20260507 19:18:01.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:01.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:18:01.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:18:01.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:01.445 | INFO | Uploaded 36GiB to GPU in 2.34806s (15.3318GiB/s) | 20260507 19:18:01.445 | INFO | Downloaded 34GiB to GPU in 2.49813s (13.6102GiB/s) | 20260507 19:18:06.583 | INFO | Analyzing chunk frames = [1948160,1948672) | 20260507 19:18:06.583 | INFO | MemInfo: { | 20260507 19:18:06.583 | INFO | "gpu" : " 17 GB", | 20260507 19:18:06.583 | INFO | "huge" : "", | 20260507 19:18:06.583 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:06.583 | INFO | "malloc" : " 14 GB", | 20260507 19:18:06.584 | INFO | "pinned" : " 29 GB" | 20260507 19:18:06.584 | INFO | } | 20260507 19:18:06.584 | INFO | Perf report for the last 5.13913 seconds: | 20260507 19:18:06.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:06.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.584 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:18:06.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.584 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 19:18:06.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:06.585 | INFO | Uploaded 36GiB to GPU in 2.51817s (14.2961GiB/s) | 20260507 19:18:06.585 | INFO | Downloaded 34GiB to GPU in 2.41023s (14.1066GiB/s) | 20260507 19:18:11.723 | INFO | Analyzing chunk frames = [1948672,1949184) | 20260507 19:18:11.723 | INFO | MemInfo: { | 20260507 19:18:11.723 | INFO | "gpu" : " 17 GB", | 20260507 19:18:11.723 | INFO | "huge" : "", | 20260507 19:18:11.723 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:11.723 | INFO | "malloc" : " 14 GB", | 20260507 19:18:11.723 | INFO | "pinned" : " 29 GB" | 20260507 19:18:11.723 | INFO | } | 20260507 19:18:11.723 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:18:11.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:11.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.723 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:18:11.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.724 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 19:18:11.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:11.724 | INFO | Uploaded 36GiB to GPU in 2.55655s (14.0815GiB/s) | 20260507 19:18:11.724 | INFO | Downloaded 34GiB to GPU in 2.44906s (13.8829GiB/s) | 20260507 19:18:16.863 | INFO | Analyzing chunk frames = [1949184,1949696) | 20260507 19:18:16.863 | INFO | MemInfo: { | 20260507 19:18:16.863 | INFO | "gpu" : " 17 GB", | 20260507 19:18:16.863 | INFO | "huge" : "", | 20260507 19:18:16.863 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:16.863 | INFO | "malloc" : " 14 GB", | 20260507 19:18:16.864 | INFO | "pinned" : " 29 GB" | 20260507 19:18:16.864 | INFO | } | 20260507 19:18:16.864 | INFO | Perf report for the last 5.14034 seconds: | 20260507 19:18:16.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:16.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 19:18:16.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | PRE_HQ : 37.8%, 1.0, 19.6ms, 0 | 20260507 19:18:16.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:16.864 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:18:16.864 | INFO | Total mem usage: 16.4 GiB | 20260507 19:18:16.864 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:18:16.864 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.94%) | 20260507 19:18:16.864 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:18:16.864 | INFO | High Water Breakdown: | 20260507 19:18:16.864 | INFO | 60.4% : 9.35 GiB ( 94.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:18:16.864 | INFO | 39.5% : 6.11 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:18:16.864 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:18:16.864 | INFO | ------------------------------------------ | 20260507 19:18:16.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:18:16.864 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:18:16.864 | INFO | Total mem usage: 26.5 GiB | 20260507 19:18:16.864 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:18:16.864 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:18:16.864 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:18:16.864 | INFO | High Water Breakdown: | 20260507 19:18:16.864 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:18:16.864 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:18:16.865 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:18:16.864 | INFO | Uploaded 36GiB to GPU in 2.53268s (14.2142GiB/s) | 20260507 19:18:16.865 | INFO | Downloaded 34GiB to GPU in 2.41938s (14.0532GiB/s) | 20260507 19:18:16.865 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:18:16.865 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:18:16.865 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:18:16.865 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:18:16.865 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:18:16.865 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:18:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:18:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:18:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:18:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:18:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:18:16.865 | INFO | ------------------------------------------ | 20260507 19:18:16.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:18:16.865 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:18:16.865 | INFO | Total mem usage: 27.8 GiB | 20260507 19:18:16.865 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:18:16.865 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:18:16.865 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:18:16.865 | INFO | High Water Breakdown: | 20260507 19:18:16.865 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:18:16.865 | INFO | ------------------------------------------ | 20260507 19:18:16.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:18:16.865 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:18:16.865 | INFO | Total mem usage: 15.7 GiB | 20260507 19:18:16.865 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:18:16.865 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:18:16.865 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:18:16.865 | INFO | High Water Breakdown: | 20260507 19:18:16.865 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:18:16.865 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:18:16.865 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:18:16.865 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:18:16.865 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:18:16.865 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:18:16.865 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:18:16.865 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:18:16.865 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:18:16.865 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:18:16.865 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:18:16.865 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:18:16.865 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:18:16.865 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:18:16.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:18:16.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:18:16.865 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:18:16.865 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:18:16.865 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:18:16.865 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:18:16.865 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:18:16.865 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:18:16.865 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:18:16.865 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:18:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:18:16.865 | INFO | ------------------------------------------ | 20260507 19:18:16.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:18:16.866 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:18:16.866 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:18:16.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:18:16.866 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.43e+01 / 1.84e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.04e-01, 1.34e-02 / 7.81e-03 | 20260507 19:18:16.866 | INFO | PerfReport: 12.12%, Baseline -- 4.03e+00 / 4.49e+00, 6.07e-01 / 2.47e-01 | 20260507 19:18:16.866 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.50e+00, 2.11e-01 / 1.14e-01 | 20260507 19:18:16.866 | INFO | PerfReport: 15.73%, DME -- 4.18e+01 / 4.48e+01, 3.87e+00 / 1.55e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 56.04%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.99e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.52e+00 / 2.83e+00, 4.41e-01 / 2.59e-01 | 20260507 19:18:16.866 | INFO | PerfReport: 3.68%, Metrics -- 1.22e+00 / 1.43e+00, 3.07e-01 / 1.75e-01 | 20260507 19:18:16.866 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.81e-01 / 4.17e-01, 6.07e-02 / 3.46e-02 | 20260507 19:18:16.866 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:18:16.866 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:18:16.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:18:16.866 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 24.19%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 24.16%, Download -- 2.58e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 31.10%, Compute -- 3.32e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 1.88%, LockUpload -- 2.01e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 2.48%, LockDownload -- 2.65e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 16.18%, LockCompute -- 1.73e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 19:18:16.866 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 19:18:16.981 | INFO | Starting a round of Baz IO | 20260507 19:18:20.494 | INFO | Populating RTMetrics report | 20260507 19:18:20.495 | INFO | RTMetrics report populated | 20260507 19:18:20.495 | INFO | Opening tmp RTMetrics json file | 20260507 19:18:20.497 | INFO | Serializing RTMetrics to json | 20260507 19:18:20.499 | INFO | Serialization done, swapping tmp file now... | 20260507 19:18:20.499 | INFO | RTMetrics done | 20260507 19:18:20.692 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.71 seconds | 20260507 19:18:20.692 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.57% / 0.01% / 54.42% | 20260507 19:18:20.692 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.48552% / 4.76837e-05% / 2.78155e-05% | 20260507 19:18:20.692 | INFO | Baz IO overall: | 20260507 19:18:20.692 | INFO | BytesWritten : 879279418309 | 20260507 19:18:20.692 | INFO | NumEvents : 426318792193 | 20260507 19:18:22.003 | INFO | Analyzing chunk frames = [1949696,1950208) | 20260507 19:18:22.003 | INFO | MemInfo: { | 20260507 19:18:22.003 | INFO | "gpu" : " 17 GB", | 20260507 19:18:22.003 | INFO | "huge" : "", | 20260507 19:18:22.003 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:22.003 | INFO | "malloc" : " 14 GB", | 20260507 19:18:22.003 | INFO | "pinned" : " 29 GB" | 20260507 19:18:22.003 | INFO | } | 20260507 19:18:22.003 | INFO | Perf report for the last 5.13916 seconds: | 20260507 19:18:22.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:22.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:22.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:22.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:22.003 | INFO | ANALYSIS : 69.3%, 2.9, 104.3ms, 0 | 20260507 19:18:22.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:22.003 | INFO | RT_METRICS : 50.2%, 1.0, 26.3ms, 0 | 20260507 19:18:22.003 | INFO | PRE_HQ : 69.0%, 2.0, 70.7ms, 0 | 20260507 19:18:22.003 | INFO | BAZWRITER : 71.8%, 4.3, 159.6ms, 0 | 20260507 19:18:22.003 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 19:18:22.003 | INFO | Uploaded 36GiB to GPU in 2.58537s (13.9245GiB/s) | 20260507 19:18:22.003 | INFO | Downloaded 38GiB to GPU in 2.71241s (14.0097GiB/s) | 20260507 19:18:27.142 | INFO | Analyzing chunk frames = [1950208,1950720) | 20260507 19:18:27.142 | INFO | MemInfo: { | 20260507 19:18:27.142 | INFO | "gpu" : " 17 GB", | 20260507 19:18:27.142 | INFO | "huge" : "", | 20260507 19:18:27.142 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:27.142 | INFO | "malloc" : " 14 GB", | 20260507 19:18:27.142 | INFO | "pinned" : " 29 GB" | 20260507 19:18:27.142 | INFO | } | 20260507 19:18:27.142 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:18:27.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:27.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 19:18:27.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | PRE_HQ : 26.2%, 1.0, 13.9ms, 0 | 20260507 19:18:27.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:27.142 | INFO | Uploaded 36GiB to GPU in 2.587s (13.9157GiB/s) | 20260507 19:18:27.142 | INFO | Downloaded 34GiB to GPU in 2.48222s (13.6974GiB/s) | 20260507 19:18:32.280 | INFO | Analyzing chunk frames = [1950720,1951232) | 20260507 19:18:32.280 | INFO | MemInfo: { | 20260507 19:18:32.280 | INFO | "gpu" : " 17 GB", | 20260507 19:18:32.280 | INFO | "huge" : "", | 20260507 19:18:32.280 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:32.280 | INFO | "malloc" : " 14 GB", | 20260507 19:18:32.281 | INFO | "pinned" : " 29 GB" | 20260507 19:18:32.281 | INFO | } | 20260507 19:18:32.281 | INFO | Perf report for the last 5.13871 seconds: | 20260507 19:18:32.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:32.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:18:32.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:18:32.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:32.281 | INFO | Uploaded 36GiB to GPU in 2.55159s (14.1088GiB/s) | 20260507 19:18:32.281 | INFO | Downloaded 34GiB to GPU in 2.45506s (13.8489GiB/s) | 20260507 19:18:37.419 | INFO | Analyzing chunk frames = [1951232,1951744) | 20260507 19:18:37.419 | INFO | MemInfo: { | 20260507 19:18:37.419 | INFO | "gpu" : " 17 GB", | 20260507 19:18:37.419 | INFO | "huge" : "", | 20260507 19:18:37.419 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:37.419 | INFO | "malloc" : " 14 GB", | 20260507 19:18:37.419 | INFO | "pinned" : " 29 GB" | 20260507 19:18:37.419 | INFO | } | 20260507 19:18:37.420 | INFO | Perf report for the last 5.13891 seconds: | 20260507 19:18:37.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:37.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:18:37.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:18:37.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:37.420 | INFO | Uploaded 36GiB to GPU in 2.48194s (14.5048GiB/s) | 20260507 19:18:37.420 | INFO | Downloaded 34GiB to GPU in 2.38984s (14.2269GiB/s) | 20260507 19:18:42.559 | INFO | Analyzing chunk frames = [1951744,1952256) | 20260507 19:18:42.559 | INFO | MemInfo: { | 20260507 19:18:42.559 | INFO | "gpu" : " 17 GB", | 20260507 19:18:42.559 | INFO | "huge" : "", | 20260507 19:18:42.559 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:42.559 | INFO | "malloc" : " 14 GB", | 20260507 19:18:42.559 | INFO | "pinned" : " 29 GB" | 20260507 19:18:42.559 | INFO | } | 20260507 19:18:42.559 | INFO | Perf report for the last 5.13979 seconds: | 20260507 19:18:42.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:42.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:18:42.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:18:42.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:42.560 | INFO | Uploaded 36GiB to GPU in 2.3753s (15.156GiB/s) | 20260507 19:18:42.560 | INFO | Downloaded 34GiB to GPU in 2.50263s (13.5857GiB/s) | 20260507 19:18:47.699 | INFO | Analyzing chunk frames = [1952256,1952768) | 20260507 19:18:47.699 | INFO | MemInfo: { | 20260507 19:18:47.699 | INFO | "gpu" : " 17 GB", | 20260507 19:18:47.699 | INFO | "huge" : "", | 20260507 19:18:47.699 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:47.699 | INFO | "malloc" : " 14 GB", | 20260507 19:18:47.699 | INFO | "pinned" : " 29 GB" | 20260507 19:18:47.699 | INFO | } | 20260507 19:18:47.699 | INFO | Perf report for the last 5.13954 seconds: | 20260507 19:18:47.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:47.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:18:47.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:18:47.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:47.699 | INFO | Uploaded 36GiB to GPU in 2.46965s (14.5769GiB/s) | 20260507 19:18:47.699 | INFO | Downloaded 34GiB to GPU in 2.41724s (14.0656GiB/s) | 20260507 19:18:52.838 | INFO | Analyzing chunk frames = [1952768,1953280) | 20260507 19:18:52.838 | INFO | MemInfo: { | 20260507 19:18:52.838 | INFO | "gpu" : " 17 GB", | 20260507 19:18:52.838 | INFO | "huge" : "", | 20260507 19:18:52.838 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:52.838 | INFO | "malloc" : " 14 GB", | 20260507 19:18:52.838 | INFO | "pinned" : " 29 GB" | 20260507 19:18:52.838 | INFO | } | 20260507 19:18:52.838 | INFO | Perf report for the last 5.13896 seconds: | 20260507 19:18:52.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:52.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:18:52.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:18:52.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:52.838 | INFO | Uploaded 36GiB to GPU in 2.50104s (14.394GiB/s) | 20260507 19:18:52.838 | INFO | Downloaded 34GiB to GPU in 2.44221s (13.9218GiB/s) | 20260507 19:18:57.977 | INFO | Analyzing chunk frames = [1953280,1953792) | 20260507 19:18:57.977 | INFO | MemInfo: { | 20260507 19:18:57.977 | INFO | "gpu" : " 17 GB", | 20260507 19:18:57.978 | INFO | "huge" : "", | 20260507 19:18:57.978 | INFO | "huge_shared" : " 29 GB", | 20260507 19:18:57.978 | INFO | "malloc" : " 14 GB", | 20260507 19:18:57.978 | INFO | "pinned" : " 29 GB" | 20260507 19:18:57.978 | INFO | } | 20260507 19:18:57.978 | INFO | Perf report for the last 5.13981 seconds: | 20260507 19:18:57.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:18:57.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:18:57.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260507 19:18:57.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:18:57.978 | INFO | Uploaded 36GiB to GPU in 2.48198s (14.5045GiB/s) | 20260507 19:18:57.978 | INFO | Downloaded 34GiB to GPU in 2.42606s (14.0145GiB/s) | 20260507 19:18:58.091 | INFO | Starting a round of Baz IO | 20260507 19:19:01.625 | INFO | Populating RTMetrics report | 20260507 19:19:01.626 | INFO | RTMetrics report populated | 20260507 19:19:01.626 | INFO | Opening tmp RTMetrics json file | 20260507 19:19:01.627 | INFO | Serializing RTMetrics to json | 20260507 19:19:01.629 | INFO | Serialization done, swapping tmp file now... | 20260507 19:19:01.630 | INFO | RTMetrics done | 20260507 19:19:01.735 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.644 seconds | 20260507 19:19:01.735 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.55% / 0.01% / 54.43% | 20260507 19:19:01.735 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.48559% / 1.58946e-05% / 3.17891e-05% | 20260507 19:19:01.735 | INFO | Baz IO overall: | 20260507 19:19:01.736 | INFO | BytesWritten : 881208217761 | 20260507 19:19:01.736 | INFO | NumEvents : 427253858749 | 20260507 19:19:03.118 | INFO | Analyzing chunk frames = [1953792,1954304) | 20260507 19:19:03.118 | INFO | MemInfo: { | 20260507 19:19:03.118 | INFO | "gpu" : " 17 GB", | 20260507 19:19:03.118 | INFO | "huge" : "", | 20260507 19:19:03.118 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:03.118 | INFO | "malloc" : " 14 GB", | 20260507 19:19:03.118 | INFO | "pinned" : " 29 GB" | 20260507 19:19:03.118 | INFO | } | 20260507 19:19:03.118 | INFO | Perf report for the last 5.14022 seconds: | 20260507 19:19:03.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:03.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:03.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:03.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:03.118 | INFO | ANALYSIS : 69.5%, 2.9, 105.3ms, 0 | 20260507 19:19:03.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:03.118 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260507 19:19:03.118 | INFO | PRE_HQ : 69.3%, 2.0, 70.9ms, 0 | 20260507 19:19:03.118 | INFO | BAZWRITER : 70.6%, 4.2, 150.8ms, 0 | 20260507 19:19:03.118 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 19:19:03.118 | INFO | Uploaded 36GiB to GPU in 2.58107s (13.9477GiB/s) | 20260507 19:19:03.118 | INFO | Downloaded 38GiB to GPU in 2.71161s (14.0138GiB/s) | 20260507 19:19:08.256 | INFO | Analyzing chunk frames = [1954304,1954816) | 20260507 19:19:08.256 | INFO | MemInfo: { | 20260507 19:19:08.256 | INFO | "gpu" : " 17 GB", | 20260507 19:19:08.256 | INFO | "huge" : "", | 20260507 19:19:08.256 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:08.256 | INFO | "malloc" : " 14 GB", | 20260507 19:19:08.256 | INFO | "pinned" : " 29 GB" | 20260507 19:19:08.256 | INFO | } | 20260507 19:19:08.256 | INFO | Perf report for the last 5.13824 seconds: | 20260507 19:19:08.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:08.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:19:08.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 19:19:08.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:08.256 | INFO | Uploaded 36GiB to GPU in 2.53615s (14.1947GiB/s) | 20260507 19:19:08.256 | INFO | Downloaded 34GiB to GPU in 2.50062s (13.5967GiB/s) | 20260507 19:19:13.394 | INFO | Analyzing chunk frames = [1954816,1955328) | 20260507 19:19:13.394 | INFO | MemInfo: { | 20260507 19:19:13.394 | INFO | "gpu" : " 17 GB", | 20260507 19:19:13.394 | INFO | "huge" : "", | 20260507 19:19:13.394 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:13.394 | INFO | "malloc" : " 14 GB", | 20260507 19:19:13.394 | INFO | "pinned" : " 29 GB" | 20260507 19:19:13.395 | INFO | } | 20260507 19:19:13.395 | INFO | Perf report for the last 5.13841 seconds: | 20260507 19:19:13.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:13.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:19:13.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:19:13.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:13.395 | INFO | Uploaded 36GiB to GPU in 2.52929s (14.2333GiB/s) | 20260507 19:19:13.396 | INFO | Downloaded 34GiB to GPU in 2.4608s (13.8166GiB/s) | 20260507 19:19:18.534 | INFO | Analyzing chunk frames = [1955328,1955840) | 20260507 19:19:18.534 | INFO | MemInfo: { | 20260507 19:19:18.534 | INFO | "gpu" : " 17 GB", | 20260507 19:19:18.534 | INFO | "huge" : "", | 20260507 19:19:18.534 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:18.534 | INFO | "malloc" : " 14 GB", | 20260507 19:19:18.534 | INFO | "pinned" : " 29 GB" | 20260507 19:19:18.534 | INFO | } | 20260507 19:19:18.534 | INFO | Perf report for the last 5.1398 seconds: | 20260507 19:19:18.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:18.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.535 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:19:18.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.535 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:19:18.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:18.535 | INFO | Uploaded 36GiB to GPU in 2.45479s (14.6652GiB/s) | 20260507 19:19:18.535 | INFO | Downloaded 34GiB to GPU in 2.39484s (14.1972GiB/s) | 20260507 19:19:23.674 | INFO | Analyzing chunk frames = [1955840,1956352) | 20260507 19:19:23.674 | INFO | MemInfo: { | 20260507 19:19:23.674 | INFO | "gpu" : " 17 GB", | 20260507 19:19:23.674 | INFO | "huge" : "", | 20260507 19:19:23.674 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:23.674 | INFO | "malloc" : " 14 GB", | 20260507 19:19:23.674 | INFO | "pinned" : " 29 GB" | 20260507 19:19:23.674 | INFO | } | 20260507 19:19:23.674 | INFO | Perf report for the last 5.13957 seconds: | 20260507 19:19:23.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:23.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.674 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:19:23.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.674 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:19:23.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:23.675 | INFO | Uploaded 36GiB to GPU in 2.35138s (15.3102GiB/s) | 20260507 19:19:23.675 | INFO | Downloaded 34GiB to GPU in 2.5019s (13.5897GiB/s) | 20260507 19:19:28.814 | INFO | Analyzing chunk frames = [1956352,1956864) | 20260507 19:19:28.814 | INFO | MemInfo: { | 20260507 19:19:28.814 | INFO | "gpu" : " 17 GB", | 20260507 19:19:28.814 | INFO | "huge" : "", | 20260507 19:19:28.814 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:28.814 | INFO | "malloc" : " 14 GB", | 20260507 19:19:28.814 | INFO | "pinned" : " 29 GB" | 20260507 19:19:28.814 | INFO | } | 20260507 19:19:28.814 | INFO | Perf report for the last 5.14022 seconds: | 20260507 19:19:28.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:28.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.814 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:19:28.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.814 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:19:28.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:28.815 | INFO | Uploaded 36GiB to GPU in 2.5152s (14.313GiB/s) | 20260507 19:19:28.815 | INFO | Downloaded 34GiB to GPU in 2.41222s (14.0949GiB/s) | 20260507 19:19:33.955 | INFO | Analyzing chunk frames = [1956864,1957376) | 20260507 19:19:33.955 | INFO | MemInfo: { | 20260507 19:19:33.955 | INFO | "gpu" : " 17 GB", | 20260507 19:19:33.955 | INFO | "huge" : "", | 20260507 19:19:33.955 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:33.955 | INFO | "malloc" : " 14 GB", | 20260507 19:19:33.955 | INFO | "pinned" : " 29 GB" | 20260507 19:19:33.955 | INFO | } | 20260507 19:19:33.955 | INFO | Perf report for the last 5.14095 seconds: | 20260507 19:19:33.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:33.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:19:33.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:19:33.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:33.955 | INFO | Uploaded 36GiB to GPU in 2.54655s (14.1367GiB/s) | 20260507 19:19:33.955 | INFO | Downloaded 34GiB to GPU in 2.43992s (13.9349GiB/s) | 20260507 19:19:39.091 | INFO | Analyzing chunk frames = [1957376,1957888) | 20260507 19:19:39.091 | INFO | MemInfo: { | 20260507 19:19:39.091 | INFO | "gpu" : " 17 GB", | 20260507 19:19:39.091 | INFO | "huge" : "", | 20260507 19:19:39.091 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:39.092 | INFO | "malloc" : " 14 GB", | 20260507 19:19:39.092 | INFO | "pinned" : " 29 GB" | 20260507 19:19:39.092 | INFO | } | 20260507 19:19:39.092 | INFO | Perf report for the last 5.13635 seconds: | 20260507 19:19:39.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:39.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:19:39.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:19:39.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:39.092 | INFO | Uploaded 36GiB to GPU in 2.52698s (14.2463GiB/s) | 20260507 19:19:39.092 | INFO | Downloaded 34GiB to GPU in 2.41302s (14.0902GiB/s) | 20260507 19:19:39.092 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:19:39.092 | INFO | Total mem usage: 16.4 GiB | 20260507 19:19:39.092 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:19:39.092 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12.6%) | 20260507 19:19:39.092 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:19:39.092 | INFO | High Water Breakdown: | 20260507 19:19:39.092 | INFO | 59.4% : 8.66 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:19:39.092 | INFO | 40.4% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:19:39.092 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:19:39.092 | INFO | ------------------------------------------ | 20260507 19:19:39.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:19:39.092 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:19:39.092 | INFO | Total mem usage: 26.5 GiB | 20260507 19:19:39.092 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:19:39.092 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 19:19:39.092 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 19:19:39.092 | INFO | High Water Breakdown: | 20260507 19:19:39.092 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:19:39.092 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:19:39.092 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:19:39.092 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:19:39.092 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:19:39.092 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:19:39.092 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:19:39.092 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:19:39.092 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:19:39.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:19:39.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:19:39.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:19:39.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:19:39.092 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:19:39.092 | INFO | ------------------------------------------ | 20260507 19:19:39.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:19:39.092 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:19:39.092 | INFO | Total mem usage: 27.8 GiB | 20260507 19:19:39.092 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:19:39.092 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:19:39.092 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:19:39.092 | INFO | High Water Breakdown: | 20260507 19:19:39.092 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:19:39.092 | INFO | ------------------------------------------ | 20260507 19:19:39.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:19:39.092 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:19:39.092 | INFO | Total mem usage: 15.7 GiB | 20260507 19:19:39.092 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:19:39.092 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:19:39.092 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:19:39.092 | INFO | High Water Breakdown: | 20260507 19:19:39.092 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:19:39.092 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:19:39.092 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:19:39.092 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:19:39.092 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:19:39.092 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:19:39.092 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:19:39.092 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:19:39.092 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:19:39.092 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:19:39.092 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:19:39.092 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:19:39.092 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:19:39.092 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:19:39.092 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:19:39.092 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:19:39.092 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:19:39.092 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:19:39.092 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:19:39.092 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:19:39.092 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:19:39.092 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:19:39.092 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:19:39.092 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:19:39.092 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:19:39.092 | INFO | ------------------------------------------ | 20260507 19:19:39.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:19:39.092 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:19:39.092 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:19:39.092 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:19:39.092 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.53e+01, 1.42e+01 / 1.92e+00 | 20260507 19:19:39.092 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.22e-02 / 6.43e-03 | 20260507 19:19:39.092 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.48e+00, 5.95e-01 / 2.91e-01 | 20260507 19:19:39.092 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.49e+00, 2.11e-01 / 1.30e-01 | 20260507 19:19:39.092 | INFO | PerfReport: 15.79%, DME -- 4.19e+01 / 4.49e+01, 3.92e+00 / 1.32e+00 | 20260507 19:19:39.092 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.76e+00 / 1.92e+00 | 20260507 19:19:39.092 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.52e+00 / 2.86e+00, 4.41e-01 / 2.40e-01 | 20260507 19:19:39.092 | INFO | PerfReport: 3.67%, Metrics -- 1.22e+00 / 1.42e+00, 3.08e-01 / 1.57e-01 | 20260507 19:19:39.092 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.80e-01 / 4.12e-01, 6.05e-02 / 3.36e-02 | 20260507 19:19:39.093 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:19:39.093 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:19:39.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:19:39.093 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 24.43%, Upload -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 24.18%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 31.05%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 2.06%, LockUpload -- 2.20e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 2.34%, LockDownload -- 2.49e+00 / 0.00e+00, 3.70e+00 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 15.92%, LockCompute -- 1.70e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 19:19:39.093 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 19:19:39.203 | INFO | Starting a round of Baz IO | 20260507 19:19:42.677 | INFO | Populating RTMetrics report | 20260507 19:19:42.677 | INFO | RTMetrics report populated | 20260507 19:19:42.677 | INFO | Opening tmp RTMetrics json file | 20260507 19:19:42.694 | INFO | Serializing RTMetrics to json | 20260507 19:19:42.701 | INFO | Serialization done, swapping tmp file now... | 20260507 19:19:42.702 | INFO | RTMetrics done | 20260507 19:19:42.775 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.571 seconds | 20260507 19:19:42.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.54% / 0.01% / 54.45% | 20260507 19:19:42.775 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.4755% / 3.17891e-05% / 5.96046e-05% | 20260507 19:19:42.775 | INFO | Baz IO overall: | 20260507 19:19:42.775 | INFO | BytesWritten : 883135408005 | 20260507 19:19:42.775 | INFO | NumEvents : 428187591189 | 20260507 19:19:44.230 | INFO | Analyzing chunk frames = [1957888,1958400) | 20260507 19:19:44.230 | INFO | MemInfo: { | 20260507 19:19:44.230 | INFO | "gpu" : " 17 GB", | 20260507 19:19:44.230 | INFO | "huge" : "", | 20260507 19:19:44.230 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:44.230 | INFO | "malloc" : " 14 GB", | 20260507 19:19:44.230 | INFO | "pinned" : " 29 GB" | 20260507 19:19:44.230 | INFO | } | 20260507 19:19:44.230 | INFO | Perf report for the last 5.13844 seconds: | 20260507 19:19:44.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:44.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:44.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:44.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:44.230 | INFO | ANALYSIS : 69.0%, 2.9, 105.6ms, 0 | 20260507 19:19:44.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:44.230 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 19:19:44.230 | INFO | PRE_HQ : 67.7%, 1.8, 62.4ms, 0 | 20260507 19:19:44.230 | INFO | BAZWRITER : 69.1%, 4.4, 156.9ms, 0 | 20260507 19:19:44.230 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:19:44.231 | INFO | Uploaded 36GiB to GPU in 2.51515s (14.3133GiB/s) | 20260507 19:19:44.231 | INFO | Downloaded 38GiB to GPU in 2.70603s (14.0427GiB/s) | 20260507 19:19:49.369 | INFO | Analyzing chunk frames = [1958400,1958912) | 20260507 19:19:49.369 | INFO | MemInfo: { | 20260507 19:19:49.369 | INFO | "gpu" : " 17 GB", | 20260507 19:19:49.369 | INFO | "huge" : "", | 20260507 19:19:49.369 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:49.369 | INFO | "malloc" : " 14 GB", | 20260507 19:19:49.369 | INFO | "pinned" : " 29 GB" | 20260507 19:19:49.369 | INFO | } | 20260507 19:19:49.369 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:19:49.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:49.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 19:19:49.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:19:49.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:49.370 | INFO | Uploaded 36GiB to GPU in 2.53043s (14.2269GiB/s) | 20260507 19:19:49.370 | INFO | Downloaded 34GiB to GPU in 2.47534s (13.7355GiB/s) | 20260507 19:19:54.509 | INFO | Analyzing chunk frames = [1958912,1959424) | 20260507 19:19:54.509 | INFO | MemInfo: { | 20260507 19:19:54.509 | INFO | "gpu" : " 17 GB", | 20260507 19:19:54.509 | INFO | "huge" : "", | 20260507 19:19:54.509 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:54.509 | INFO | "malloc" : " 14 GB", | 20260507 19:19:54.509 | INFO | "pinned" : " 29 GB" | 20260507 19:19:54.509 | INFO | } | 20260507 19:19:54.509 | INFO | Perf report for the last 5.13953 seconds: | 20260507 19:19:54.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:54.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:19:54.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:19:54.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:54.509 | INFO | Uploaded 36GiB to GPU in 2.56076s (14.0583GiB/s) | 20260507 19:19:54.509 | INFO | Downloaded 34GiB to GPU in 2.47995s (13.7099GiB/s) | 20260507 19:19:59.649 | INFO | Analyzing chunk frames = [1959424,1959936) | 20260507 19:19:59.649 | INFO | MemInfo: { | 20260507 19:19:59.649 | INFO | "gpu" : " 17 GB", | 20260507 19:19:59.649 | INFO | "huge" : "", | 20260507 19:19:59.649 | INFO | "huge_shared" : " 29 GB", | 20260507 19:19:59.649 | INFO | "malloc" : " 14 GB", | 20260507 19:19:59.649 | INFO | "pinned" : " 29 GB" | 20260507 19:19:59.649 | INFO | } | 20260507 19:19:59.649 | INFO | Perf report for the last 5.1397 seconds: | 20260507 19:19:59.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:19:59.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:19:59.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:19:59.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:19:59.649 | INFO | Uploaded 36GiB to GPU in 2.46493s (14.6049GiB/s) | 20260507 19:19:59.649 | INFO | Downloaded 34GiB to GPU in 2.38578s (14.2511GiB/s) | 20260507 19:20:04.788 | INFO | Analyzing chunk frames = [1959936,1960448) | 20260507 19:20:04.788 | INFO | MemInfo: { | 20260507 19:20:04.788 | INFO | "gpu" : " 17 GB", | 20260507 19:20:04.788 | INFO | "huge" : "", | 20260507 19:20:04.788 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:04.788 | INFO | "malloc" : " 14 GB", | 20260507 19:20:04.788 | INFO | "pinned" : " 29 GB" | 20260507 19:20:04.788 | INFO | } | 20260507 19:20:04.788 | INFO | Perf report for the last 5.13942 seconds: | 20260507 19:20:04.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:04.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:20:04.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:20:04.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:04.788 | INFO | Uploaded 36GiB to GPU in 2.33872s (15.393GiB/s) | 20260507 19:20:04.789 | INFO | Downloaded 34GiB to GPU in 2.4995s (13.6027GiB/s) | 20260507 19:20:09.927 | INFO | Analyzing chunk frames = [1960448,1960960) | 20260507 19:20:09.928 | INFO | MemInfo: { | 20260507 19:20:09.928 | INFO | "gpu" : " 17 GB", | 20260507 19:20:09.928 | INFO | "huge" : "", | 20260507 19:20:09.928 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:09.928 | INFO | "malloc" : " 14 GB", | 20260507 19:20:09.928 | INFO | "pinned" : " 29 GB" | 20260507 19:20:09.928 | INFO | } | 20260507 19:20:09.928 | INFO | Perf report for the last 5.13942 seconds: | 20260507 19:20:09.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:09.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 19:20:09.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:20:09.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:09.928 | INFO | Uploaded 36GiB to GPU in 2.41696s (14.8947GiB/s) | 20260507 19:20:09.928 | INFO | Downloaded 34GiB to GPU in 2.43482s (13.9641GiB/s) | 20260507 19:20:15.067 | INFO | Analyzing chunk frames = [1960960,1961472) | 20260507 19:20:15.067 | INFO | MemInfo: { | 20260507 19:20:15.067 | INFO | "gpu" : " 17 GB", | 20260507 19:20:15.067 | INFO | "huge" : "", | 20260507 19:20:15.067 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:15.067 | INFO | "malloc" : " 14 GB", | 20260507 19:20:15.067 | INFO | "pinned" : " 29 GB" | 20260507 19:20:15.067 | INFO | } | 20260507 19:20:15.067 | INFO | Perf report for the last 5.13984 seconds: | 20260507 19:20:15.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:15.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.068 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:20:15.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.068 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:20:15.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:15.068 | INFO | Uploaded 36GiB to GPU in 2.49341s (14.4381GiB/s) | 20260507 19:20:15.068 | INFO | Downloaded 34GiB to GPU in 2.4613s (13.8139GiB/s) | 20260507 19:20:20.207 | INFO | Analyzing chunk frames = [1961472,1961984) | 20260507 19:20:20.207 | INFO | MemInfo: { | 20260507 19:20:20.207 | INFO | "gpu" : " 17 GB", | 20260507 19:20:20.207 | INFO | "huge" : "", | 20260507 19:20:20.207 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:20.207 | INFO | "malloc" : " 14 GB", | 20260507 19:20:20.207 | INFO | "pinned" : " 29 GB" | 20260507 19:20:20.207 | INFO | } | 20260507 19:20:20.207 | INFO | Perf report for the last 5.13984 seconds: | 20260507 19:20:20.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:20.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.207 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:20:20.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.208 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:20:20.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:20.208 | INFO | Uploaded 36GiB to GPU in 2.42095s (14.8702GiB/s) | 20260507 19:20:20.208 | INFO | Downloaded 34GiB to GPU in 2.4454s (13.9037GiB/s) | 20260507 19:20:20.315 | INFO | Starting a round of Baz IO | 20260507 19:20:23.782 | INFO | Populating RTMetrics report | 20260507 19:20:23.782 | INFO | RTMetrics report populated | 20260507 19:20:23.782 | INFO | Opening tmp RTMetrics json file | 20260507 19:20:23.800 | INFO | Serializing RTMetrics to json | 20260507 19:20:23.806 | INFO | Serialization done, swapping tmp file now... | 20260507 19:20:23.807 | INFO | RTMetrics done | 20260507 19:20:23.886 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.57 seconds | 20260507 19:20:23.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.53% / 0.01% / 54.46% | 20260507 19:20:23.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.47371% / 1.58946e-05% / 3.17891e-05% | 20260507 19:20:23.886 | INFO | Baz IO overall: | 20260507 19:20:23.886 | INFO | BytesWritten : 885063834266 | 20260507 19:20:23.886 | INFO | NumEvents : 429122508453 | 20260507 19:20:25.345 | INFO | Analyzing chunk frames = [1961984,1962496) | 20260507 19:20:25.345 | INFO | MemInfo: { | 20260507 19:20:25.345 | INFO | "gpu" : " 17 GB", | 20260507 19:20:25.345 | INFO | "huge" : "", | 20260507 19:20:25.345 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:25.345 | INFO | "malloc" : " 14 GB", | 20260507 19:20:25.345 | INFO | "pinned" : " 29 GB" | 20260507 19:20:25.345 | INFO | } | 20260507 19:20:25.345 | INFO | Perf report for the last 5.13812 seconds: | 20260507 19:20:25.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:25.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:25.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:25.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:25.345 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 19:20:25.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:25.345 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:20:25.346 | INFO | PRE_HQ : 66.2%, 1.7, 58.6ms, 0 | 20260507 19:20:25.346 | INFO | BAZWRITER : 69.5%, 4.2, 151.7ms, 0 | 20260507 19:20:25.346 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:20:25.346 | INFO | Uploaded 36GiB to GPU in 2.45696s (14.6523GiB/s) | 20260507 19:20:25.346 | INFO | Downloaded 38GiB to GPU in 2.68472s (14.1542GiB/s) | 20260507 19:20:30.484 | INFO | Analyzing chunk frames = [1962496,1963008) | 20260507 19:20:30.484 | INFO | MemInfo: { | 20260507 19:20:30.484 | INFO | "gpu" : " 17 GB", | 20260507 19:20:30.484 | INFO | "huge" : "", | 20260507 19:20:30.485 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:30.485 | INFO | "malloc" : " 14 GB", | 20260507 19:20:30.485 | INFO | "pinned" : " 29 GB" | 20260507 19:20:30.485 | INFO | } | 20260507 19:20:30.485 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:20:30.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:30.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:20:30.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 19:20:30.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:30.485 | INFO | Uploaded 36GiB to GPU in 2.53119s (14.2226GiB/s) | 20260507 19:20:30.486 | INFO | Downloaded 34GiB to GPU in 2.49131s (13.6474GiB/s) | 20260507 19:20:35.625 | INFO | Analyzing chunk frames = [1963008,1963520) | 20260507 19:20:35.625 | INFO | MemInfo: { | 20260507 19:20:35.625 | INFO | "gpu" : " 17 GB", | 20260507 19:20:35.625 | INFO | "huge" : "", | 20260507 19:20:35.625 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:35.625 | INFO | "malloc" : " 14 GB", | 20260507 19:20:35.625 | INFO | "pinned" : " 29 GB" | 20260507 19:20:35.625 | INFO | } | 20260507 19:20:35.625 | INFO | Perf report for the last 5.14004 seconds: | 20260507 19:20:35.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:35.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:20:35.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:20:35.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:35.625 | INFO | Uploaded 36GiB to GPU in 2.54607s (14.1395GiB/s) | 20260507 19:20:35.625 | INFO | Downloaded 34GiB to GPU in 2.46772s (13.7779GiB/s) | 20260507 19:20:40.764 | INFO | Analyzing chunk frames = [1963520,1964032) | 20260507 19:20:40.764 | INFO | MemInfo: { | 20260507 19:20:40.764 | INFO | "gpu" : " 17 GB", | 20260507 19:20:40.764 | INFO | "huge" : "", | 20260507 19:20:40.764 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:40.764 | INFO | "malloc" : " 14 GB", | 20260507 19:20:40.764 | INFO | "pinned" : " 29 GB" | 20260507 19:20:40.764 | INFO | } | 20260507 19:20:40.764 | INFO | Perf report for the last 5.13934 seconds: | 20260507 19:20:40.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:40.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.764 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:20:40.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.764 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:20:40.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:40.765 | INFO | Uploaded 36GiB to GPU in 2.46932s (14.5789GiB/s) | 20260507 19:20:40.765 | INFO | Downloaded 34GiB to GPU in 2.40043s (14.1641GiB/s) | 20260507 19:20:45.904 | INFO | Analyzing chunk frames = [1964032,1964544) | 20260507 19:20:45.904 | INFO | MemInfo: { | 20260507 19:20:45.904 | INFO | "gpu" : " 17 GB", | 20260507 19:20:45.904 | INFO | "huge" : "", | 20260507 19:20:45.904 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:45.904 | INFO | "malloc" : " 14 GB", | 20260507 19:20:45.904 | INFO | "pinned" : " 29 GB" | 20260507 19:20:45.904 | INFO | } | 20260507 19:20:45.904 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:20:45.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:45.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:20:45.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:20:45.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:45.904 | INFO | Uploaded 36GiB to GPU in 2.37341s (15.1681GiB/s) | 20260507 19:20:45.904 | INFO | Downloaded 34GiB to GPU in 2.49658s (13.6186GiB/s) | 20260507 19:20:51.042 | INFO | Analyzing chunk frames = [1964544,1965056) | 20260507 19:20:51.042 | INFO | MemInfo: { | 20260507 19:20:51.042 | INFO | "gpu" : " 17 GB", | 20260507 19:20:51.042 | INFO | "huge" : "", | 20260507 19:20:51.042 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:51.042 | INFO | "malloc" : " 14 GB", | 20260507 19:20:51.042 | INFO | "pinned" : " 29 GB" | 20260507 19:20:51.043 | INFO | } | 20260507 19:20:51.043 | INFO | Perf report for the last 5.13866 seconds: | 20260507 19:20:51.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:51.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:20:51.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:20:51.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:51.043 | INFO | Uploaded 36GiB to GPU in 2.50769s (14.3559GiB/s) | 20260507 19:20:51.043 | INFO | Downloaded 34GiB to GPU in 2.41143s (14.0995GiB/s) | 20260507 19:20:56.181 | INFO | Analyzing chunk frames = [1965056,1965568) | 20260507 19:20:56.181 | INFO | MemInfo: { | 20260507 19:20:56.181 | INFO | "gpu" : " 17 GB", | 20260507 19:20:56.181 | INFO | "huge" : "", | 20260507 19:20:56.181 | INFO | "huge_shared" : " 29 GB", | 20260507 19:20:56.181 | INFO | "malloc" : " 14 GB", | 20260507 19:20:56.181 | INFO | "pinned" : " 29 GB" | 20260507 19:20:56.181 | INFO | } | 20260507 19:20:56.181 | INFO | Perf report for the last 5.13869 seconds: | 20260507 19:20:56.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:20:56.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.181 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:20:56.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.181 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:20:56.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:20:56.182 | INFO | Uploaded 36GiB to GPU in 2.56151s (14.0542GiB/s) | 20260507 19:20:56.182 | INFO | Downloaded 34GiB to GPU in 2.43343s (13.972GiB/s) | 20260507 19:21:01.320 | INFO | Analyzing chunk frames = [1965568,1966080) | 20260507 19:21:01.320 | INFO | MemInfo: { | 20260507 19:21:01.320 | INFO | "gpu" : " 17 GB", | 20260507 19:21:01.320 | INFO | "huge" : "", | 20260507 19:21:01.320 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:01.320 | INFO | "malloc" : " 14 GB", | 20260507 19:21:01.320 | INFO | "pinned" : " 29 GB" | 20260507 19:21:01.320 | INFO | } | 20260507 19:21:01.320 | INFO | Perf report for the last 5.13918 seconds: | 20260507 19:21:01.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:01.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:21:01.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 19:21:01.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:01.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:21:01.321 | INFO | Total mem usage: 16.4 GiB | 20260507 19:21:01.321 | INFO | Uploaded 36GiB to GPU in 2.51334s (14.3236GiB/s) | 20260507 19:21:01.321 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:21:01.321 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.05%) | 20260507 19:21:01.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:21:01.321 | INFO | Downloaded 34GiB to GPU in 2.41293s (14.0907GiB/s) | 20260507 19:21:01.321 | INFO | High Water Breakdown: | 20260507 19:21:01.321 | INFO | 60.3% : 9.24 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:21:01.321 | INFO | 39.6% : 6.06 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:21:01.321 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:21:01.321 | INFO | ------------------------------------------ | 20260507 19:21:01.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:21:01.321 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:21:01.321 | INFO | Total mem usage: 26.5 GiB | 20260507 19:21:01.321 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:21:01.321 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 19:21:01.321 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:21:01.321 | INFO | High Water Breakdown: | 20260507 19:21:01.321 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:21:01.321 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:21:01.321 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:21:01.321 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:21:01.321 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:21:01.321 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:21:01.321 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:21:01.321 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:21:01.321 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:21:01.321 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:21:01.321 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:21:01.321 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:21:01.321 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:21:01.321 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:21:01.321 | INFO | ------------------------------------------ | 20260507 19:21:01.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:21:01.321 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:21:01.321 | INFO | Total mem usage: 27.8 GiB | 20260507 19:21:01.321 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:21:01.321 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:21:01.321 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:21:01.321 | INFO | High Water Breakdown: | 20260507 19:21:01.321 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:21:01.321 | INFO | ------------------------------------------ | 20260507 19:21:01.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:21:01.321 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:21:01.321 | INFO | Total mem usage: 15.7 GiB | 20260507 19:21:01.321 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:21:01.321 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:21:01.321 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:21:01.321 | INFO | High Water Breakdown: | 20260507 19:21:01.321 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:21:01.321 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:21:01.321 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:21:01.321 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:21:01.321 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:21:01.321 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:21:01.321 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:21:01.321 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:21:01.321 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:21:01.321 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:21:01.321 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:21:01.321 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:21:01.321 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:21:01.321 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:21:01.321 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:21:01.321 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:21:01.321 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:21:01.321 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:21:01.321 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:21:01.321 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:21:01.321 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:21:01.321 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:21:01.321 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:21:01.321 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:21:01.321 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:21:01.321 | INFO | ------------------------------------------ | 20260507 19:21:01.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:21:01.321 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:21:01.321 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:21:01.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:21:01.321 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.42e+01 / 1.90e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.16e-02 / 6.91e-03 | 20260507 19:21:01.321 | INFO | PerfReport: 12.09%, Baseline -- 4.02e+00 / 4.43e+00, 6.09e-01 / 3.13e-01 | 20260507 19:21:01.321 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.49e+00, 2.12e-01 / 1.19e-01 | 20260507 19:21:01.321 | INFO | PerfReport: 15.76%, DME -- 4.19e+01 / 4.49e+01, 3.86e+00 / 1.36e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.84e+00 / 1.81e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.42e-01 / 2.42e-01 | 20260507 19:21:01.321 | INFO | PerfReport: 3.67%, Metrics -- 1.22e+00 / 1.42e+00, 3.08e-01 / 1.84e-01 | 20260507 19:21:01.321 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.80e-01 / 4.10e-01, 6.06e-02 / 3.35e-02 | 20260507 19:21:01.321 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:21:01.321 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:21:01.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:21:01.321 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 24.18%, Upload -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 24.17%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 31.10%, Compute -- 3.32e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 1.92%, LockUpload -- 2.05e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 2.48%, LockDownload -- 2.65e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 16.14%, LockCompute -- 1.72e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:21:01.321 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260507 19:21:01.418 | INFO | Starting a round of Baz IO | 20260507 19:21:04.900 | INFO | Populating RTMetrics report | 20260507 19:21:04.901 | INFO | RTMetrics report populated | 20260507 19:21:04.901 | INFO | Opening tmp RTMetrics json file | 20260507 19:21:04.908 | INFO | Serializing RTMetrics to json | 20260507 19:21:04.926 | INFO | Serialization done, swapping tmp file now... | 20260507 19:21:04.926 | INFO | RTMetrics done | 20260507 19:21:05.005 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.587 seconds | 20260507 19:21:05.005 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.52% / 0.01% / 54.47% | 20260507 19:21:05.005 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.46408% / 3.97364e-06% / 3.97364e-05% | 20260507 19:21:05.005 | INFO | Baz IO overall: | 20260507 19:21:05.005 | INFO | BytesWritten : 886990418129 | 20260507 19:21:05.005 | INFO | NumEvents : 430056102615 | 20260507 19:21:06.460 | INFO | Analyzing chunk frames = [1966080,1966592) | 20260507 19:21:06.460 | INFO | MemInfo: { | 20260507 19:21:06.460 | INFO | "gpu" : " 17 GB", | 20260507 19:21:06.460 | INFO | "huge" : "", | 20260507 19:21:06.460 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:06.460 | INFO | "malloc" : " 14 GB", | 20260507 19:21:06.460 | INFO | "pinned" : " 29 GB" | 20260507 19:21:06.460 | INFO | } | 20260507 19:21:06.460 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:21:06.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:06.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:06.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:06.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:06.460 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 19:21:06.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:06.460 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:21:06.460 | INFO | PRE_HQ : 66.4%, 1.7, 58.4ms, 0 | 20260507 19:21:06.460 | INFO | BAZWRITER : 69.4%, 4.1, 145.2ms, 0 | 20260507 19:21:06.460 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 19:21:06.460 | INFO | Uploaded 36GiB to GPU in 2.49052s (14.4548GiB/s) | 20260507 19:21:06.460 | INFO | Downloaded 38GiB to GPU in 2.69096s (14.1213GiB/s) | 20260507 19:21:11.599 | INFO | Analyzing chunk frames = [1966592,1967104) | 20260507 19:21:11.599 | INFO | MemInfo: { | 20260507 19:21:11.599 | INFO | "gpu" : " 17 GB", | 20260507 19:21:11.599 | INFO | "huge" : "", | 20260507 19:21:11.599 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:11.599 | INFO | "malloc" : " 14 GB", | 20260507 19:21:11.599 | INFO | "pinned" : " 29 GB" | 20260507 19:21:11.599 | INFO | } | 20260507 19:21:11.600 | INFO | Perf report for the last 5.13976 seconds: | 20260507 19:21:11.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:11.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:21:11.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:21:11.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:11.600 | INFO | Uploaded 36GiB to GPU in 2.49183s (14.4472GiB/s) | 20260507 19:21:11.600 | INFO | Downloaded 34GiB to GPU in 2.49376s (13.6341GiB/s) | 20260507 19:21:16.738 | INFO | Analyzing chunk frames = [1967104,1967616) | 20260507 19:21:16.738 | INFO | MemInfo: { | 20260507 19:21:16.738 | INFO | "gpu" : " 17 GB", | 20260507 19:21:16.738 | INFO | "huge" : "", | 20260507 19:21:16.738 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:16.738 | INFO | "malloc" : " 14 GB", | 20260507 19:21:16.738 | INFO | "pinned" : " 29 GB" | 20260507 19:21:16.738 | INFO | } | 20260507 19:21:16.738 | INFO | Perf report for the last 5.13863 seconds: | 20260507 19:21:16.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:16.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:21:16.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:21:16.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:16.738 | INFO | Uploaded 36GiB to GPU in 2.5382s (14.1833GiB/s) | 20260507 19:21:16.738 | INFO | Downloaded 34GiB to GPU in 2.4721s (13.7535GiB/s) | 20260507 19:21:21.878 | INFO | Analyzing chunk frames = [1967616,1968128) | 20260507 19:21:21.878 | INFO | MemInfo: { | 20260507 19:21:21.878 | INFO | "gpu" : " 17 GB", | 20260507 19:21:21.878 | INFO | "huge" : "", | 20260507 19:21:21.878 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:21.878 | INFO | "malloc" : " 14 GB", | 20260507 19:21:21.878 | INFO | "pinned" : " 29 GB" | 20260507 19:21:21.878 | INFO | } | 20260507 19:21:21.878 | INFO | Perf report for the last 5.13993 seconds: | 20260507 19:21:21.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:21.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:21:21.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260507 19:21:21.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:21.879 | INFO | Uploaded 36GiB to GPU in 2.45367s (14.6719GiB/s) | 20260507 19:21:21.879 | INFO | Downloaded 34GiB to GPU in 2.39392s (14.2026GiB/s) | 20260507 19:21:27.018 | INFO | Analyzing chunk frames = [1968128,1968640) | 20260507 19:21:27.018 | INFO | MemInfo: { | 20260507 19:21:27.018 | INFO | "gpu" : " 17 GB", | 20260507 19:21:27.018 | INFO | "huge" : "", | 20260507 19:21:27.018 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:27.018 | INFO | "malloc" : " 14 GB", | 20260507 19:21:27.018 | INFO | "pinned" : " 29 GB" | 20260507 19:21:27.018 | INFO | } | 20260507 19:21:27.018 | INFO | Perf report for the last 5.13983 seconds: | 20260507 19:21:27.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:27.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.018 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:21:27.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.018 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:21:27.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:27.019 | INFO | Uploaded 36GiB to GPU in 2.35774s (15.2688GiB/s) | 20260507 19:21:27.019 | INFO | Downloaded 34GiB to GPU in 2.50044s (13.5976GiB/s) | 20260507 19:21:32.157 | INFO | Analyzing chunk frames = [1968640,1969152) | 20260507 19:21:32.157 | INFO | MemInfo: { | 20260507 19:21:32.157 | INFO | "gpu" : " 17 GB", | 20260507 19:21:32.157 | INFO | "huge" : "", | 20260507 19:21:32.157 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:32.157 | INFO | "malloc" : " 14 GB", | 20260507 19:21:32.157 | INFO | "pinned" : " 29 GB" | 20260507 19:21:32.157 | INFO | } | 20260507 19:21:32.157 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:21:32.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:32.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.158 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:21:32.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.158 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:21:32.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:32.158 | INFO | Uploaded 36GiB to GPU in 2.48383s (14.4937GiB/s) | 20260507 19:21:32.158 | INFO | Downloaded 34GiB to GPU in 2.42366s (14.0284GiB/s) | 20260507 19:21:37.297 | INFO | Analyzing chunk frames = [1969152,1969664) | 20260507 19:21:37.297 | INFO | MemInfo: { | 20260507 19:21:37.297 | INFO | "gpu" : " 17 GB", | 20260507 19:21:37.297 | INFO | "huge" : "", | 20260507 19:21:37.297 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:37.297 | INFO | "malloc" : " 14 GB", | 20260507 19:21:37.297 | INFO | "pinned" : " 29 GB" | 20260507 19:21:37.297 | INFO | } | 20260507 19:21:37.297 | INFO | Perf report for the last 5.13939 seconds: | 20260507 19:21:37.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:37.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:21:37.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:21:37.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:37.297 | INFO | Uploaded 36GiB to GPU in 2.50658s (14.3622GiB/s) | 20260507 19:21:37.298 | INFO | Downloaded 34GiB to GPU in 2.44607s (13.8998GiB/s) | 20260507 19:21:42.435 | INFO | Analyzing chunk frames = [1969664,1970176) | 20260507 19:21:42.435 | INFO | MemInfo: { | 20260507 19:21:42.435 | INFO | "gpu" : " 17 GB", | 20260507 19:21:42.435 | INFO | "huge" : "", | 20260507 19:21:42.435 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:42.435 | INFO | "malloc" : " 14 GB", | 20260507 19:21:42.435 | INFO | "pinned" : " 29 GB" | 20260507 19:21:42.435 | INFO | } | 20260507 19:21:42.435 | INFO | Perf report for the last 5.13858 seconds: | 20260507 19:21:42.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:42.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:21:42.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:21:42.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:42.436 | INFO | Uploaded 36GiB to GPU in 2.48726s (14.4738GiB/s) | 20260507 19:21:42.436 | INFO | Downloaded 34GiB to GPU in 2.42758s (14.0057GiB/s) | 20260507 19:21:42.533 | INFO | Starting a round of Baz IO | 20260507 19:21:46.006 | INFO | Populating RTMetrics report | 20260507 19:21:46.006 | INFO | RTMetrics report populated | 20260507 19:21:46.006 | INFO | Opening tmp RTMetrics json file | 20260507 19:21:46.007 | INFO | Serializing RTMetrics to json | 20260507 19:21:46.015 | INFO | Serialization done, swapping tmp file now... | 20260507 19:21:46.044 | INFO | RTMetrics done | 20260507 19:21:46.149 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.615 seconds | 20260507 19:21:46.149 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.51% / 0.01% / 54.48% | 20260507 19:21:46.149 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.45529% / 1.58946e-05% / 3.17891e-05% | 20260507 19:21:46.149 | INFO | Baz IO overall: | 20260507 19:21:46.149 | INFO | BytesWritten : 888916825684 | 20260507 19:21:46.149 | INFO | NumEvents : 430989187279 | 20260507 19:21:47.574 | INFO | Analyzing chunk frames = [1970176,1970688) | 20260507 19:21:47.574 | INFO | MemInfo: { | 20260507 19:21:47.574 | INFO | "gpu" : " 17 GB", | 20260507 19:21:47.574 | INFO | "huge" : "", | 20260507 19:21:47.574 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:47.575 | INFO | "malloc" : " 14 GB", | 20260507 19:21:47.575 | INFO | "pinned" : " 29 GB" | 20260507 19:21:47.575 | INFO | } | 20260507 19:21:47.575 | INFO | Perf report for the last 5.13918 seconds: | 20260507 19:21:47.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:47.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:47.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:47.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:47.575 | INFO | ANALYSIS : 69.0%, 3.0, 107.0ms, 0 | 20260507 19:21:47.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:47.575 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 19:21:47.575 | INFO | PRE_HQ : 66.2%, 1.7, 58.7ms, 0 | 20260507 19:21:47.575 | INFO | BAZWRITER : 69.9%, 3.7, 134.6ms, 0 | 20260507 19:21:47.575 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 19:21:47.575 | INFO | Uploaded 36GiB to GPU in 2.57276s (13.9928GiB/s) | 20260507 19:21:47.575 | INFO | Downloaded 38GiB to GPU in 2.69282s (14.1116GiB/s) | 20260507 19:21:52.715 | INFO | Analyzing chunk frames = [1970688,1971200) | 20260507 19:21:52.715 | INFO | MemInfo: { | 20260507 19:21:52.715 | INFO | "gpu" : " 17 GB", | 20260507 19:21:52.715 | INFO | "huge" : "", | 20260507 19:21:52.715 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:52.715 | INFO | "malloc" : " 14 GB", | 20260507 19:21:52.715 | INFO | "pinned" : " 29 GB" | 20260507 19:21:52.715 | INFO | } | 20260507 19:21:52.715 | INFO | Perf report for the last 5.1403 seconds: | 20260507 19:21:52.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:52.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:21:52.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:21:52.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:52.715 | INFO | Uploaded 36GiB to GPU in 2.54034s (14.1713GiB/s) | 20260507 19:21:52.716 | INFO | Downloaded 34GiB to GPU in 2.49648s (13.6192GiB/s) | 20260507 19:21:57.853 | INFO | Analyzing chunk frames = [1971200,1971712) | 20260507 19:21:57.853 | INFO | MemInfo: { | 20260507 19:21:57.853 | INFO | "gpu" : " 17 GB", | 20260507 19:21:57.853 | INFO | "huge" : "", | 20260507 19:21:57.853 | INFO | "huge_shared" : " 29 GB", | 20260507 19:21:57.853 | INFO | "malloc" : " 14 GB", | 20260507 19:21:57.853 | INFO | "pinned" : " 29 GB" | 20260507 19:21:57.853 | INFO | } | 20260507 19:21:57.853 | INFO | Perf report for the last 5.13817 seconds: | 20260507 19:21:57.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:21:57.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 19:21:57.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 19:21:57.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:21:57.853 | INFO | Uploaded 36GiB to GPU in 2.4608s (14.6294GiB/s) | 20260507 19:21:57.853 | INFO | Downloaded 34GiB to GPU in 2.49216s (13.6428GiB/s) | 20260507 19:22:02.993 | INFO | Analyzing chunk frames = [1971712,1972224) | 20260507 19:22:02.993 | INFO | MemInfo: { | 20260507 19:22:02.993 | INFO | "gpu" : " 17 GB", | 20260507 19:22:02.993 | INFO | "huge" : "", | 20260507 19:22:02.993 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:02.993 | INFO | "malloc" : " 14 GB", | 20260507 19:22:02.993 | INFO | "pinned" : " 29 GB" | 20260507 19:22:02.993 | INFO | } | 20260507 19:22:02.993 | INFO | Perf report for the last 5.13968 seconds: | 20260507 19:22:02.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:02.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 19:22:02.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:22:02.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:02.993 | INFO | Uploaded 36GiB to GPU in 2.48318s (14.4975GiB/s) | 20260507 19:22:02.993 | INFO | Downloaded 34GiB to GPU in 2.40907s (14.1133GiB/s) | 20260507 19:22:08.133 | INFO | Analyzing chunk frames = [1972224,1972736) | 20260507 19:22:08.133 | INFO | MemInfo: { | 20260507 19:22:08.133 | INFO | "gpu" : " 17 GB", | 20260507 19:22:08.133 | INFO | "huge" : "", | 20260507 19:22:08.133 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:08.133 | INFO | "malloc" : " 14 GB", | 20260507 19:22:08.133 | INFO | "pinned" : " 29 GB" | 20260507 19:22:08.133 | INFO | } | 20260507 19:22:08.133 | INFO | Perf report for the last 5.14038 seconds: | 20260507 19:22:08.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:08.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 19:22:08.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:22:08.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:08.133 | INFO | Uploaded 36GiB to GPU in 2.33845s (15.3948GiB/s) | 20260507 19:22:08.134 | INFO | Downloaded 34GiB to GPU in 2.49711s (13.6157GiB/s) | 20260507 19:22:13.272 | INFO | Analyzing chunk frames = [1972736,1973248) | 20260507 19:22:13.272 | INFO | MemInfo: { | 20260507 19:22:13.272 | INFO | "gpu" : " 17 GB", | 20260507 19:22:13.272 | INFO | "huge" : "", | 20260507 19:22:13.272 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:13.272 | INFO | "malloc" : " 14 GB", | 20260507 19:22:13.272 | INFO | "pinned" : " 29 GB" | 20260507 19:22:13.272 | INFO | } | 20260507 19:22:13.272 | INFO | Perf report for the last 5.13877 seconds: | 20260507 19:22:13.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:13.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:22:13.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:22:13.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:13.272 | INFO | Uploaded 36GiB to GPU in 2.45956s (14.6368GiB/s) | 20260507 19:22:13.272 | INFO | Downloaded 34GiB to GPU in 2.43643s (13.9548GiB/s) | 20260507 19:22:18.410 | INFO | Analyzing chunk frames = [1973248,1973760) | 20260507 19:22:18.410 | INFO | MemInfo: { | 20260507 19:22:18.410 | INFO | "gpu" : " 17 GB", | 20260507 19:22:18.410 | INFO | "huge" : "", | 20260507 19:22:18.411 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:18.411 | INFO | "malloc" : " 14 GB", | 20260507 19:22:18.411 | INFO | "pinned" : " 29 GB" | 20260507 19:22:18.411 | INFO | } | 20260507 19:22:18.411 | INFO | Perf report for the last 5.13875 seconds: | 20260507 19:22:18.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:18.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 19:22:18.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:22:18.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:18.411 | INFO | Uploaded 36GiB to GPU in 2.49455s (14.4315GiB/s) | 20260507 19:22:18.411 | INFO | Downloaded 34GiB to GPU in 2.46017s (13.8202GiB/s) | 20260507 19:22:23.548 | INFO | Analyzing chunk frames = [1973760,1974272) | 20260507 19:22:23.548 | INFO | MemInfo: { | 20260507 19:22:23.548 | INFO | "gpu" : " 17 GB", | 20260507 19:22:23.548 | INFO | "huge" : "", | 20260507 19:22:23.548 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:23.548 | INFO | "malloc" : " 14 GB", | 20260507 19:22:23.548 | INFO | "pinned" : " 29 GB" | 20260507 19:22:23.549 | INFO | } | 20260507 19:22:23.549 | INFO | Perf report for the last 5.1378 seconds: | 20260507 19:22:23.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:23.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:22:23.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:22:23.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:23.549 | INFO | Uploaded 36GiB to GPU in 2.43743s (14.7697GiB/s) | 20260507 19:22:23.549 | INFO | Downloaded 34GiB to GPU in 2.447s (13.8946GiB/s) | 20260507 19:22:23.549 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:22:23.549 | INFO | Total mem usage: 16.4 GiB | 20260507 19:22:23.549 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:22:23.549 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.89%) | 20260507 19:22:23.549 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:22:23.549 | INFO | High Water Breakdown: | 20260507 19:22:23.549 | INFO | 60.2% : 9.15 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:22:23.549 | INFO | 39.7% : 6.04 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:22:23.549 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:22:23.549 | INFO | ------------------------------------------ | 20260507 19:22:23.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:22:23.549 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:22:23.549 | INFO | Total mem usage: 26.5 GiB | 20260507 19:22:23.549 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:22:23.549 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:22:23.549 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:22:23.549 | INFO | High Water Breakdown: | 20260507 19:22:23.549 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:22:23.549 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:22:23.549 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:22:23.549 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:22:23.549 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:22:23.549 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:22:23.549 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:22:23.549 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:22:23.549 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:22:23.549 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:22:23.549 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:22:23.549 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:22:23.549 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:22:23.549 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:22:23.549 | INFO | ------------------------------------------ | 20260507 19:22:23.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:22:23.549 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:22:23.549 | INFO | Total mem usage: 27.8 GiB | 20260507 19:22:23.549 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:22:23.549 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:22:23.549 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:22:23.549 | INFO | High Water Breakdown: | 20260507 19:22:23.549 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:22:23.549 | INFO | ------------------------------------------ | 20260507 19:22:23.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:22:23.549 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:22:23.549 | INFO | Total mem usage: 15.7 GiB | 20260507 19:22:23.549 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:22:23.549 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:22:23.549 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:22:23.549 | INFO | High Water Breakdown: | 20260507 19:22:23.549 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:22:23.549 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:22:23.549 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:22:23.549 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:22:23.549 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:22:23.549 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:22:23.549 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:22:23.549 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:22:23.549 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:22:23.549 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:22:23.549 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:22:23.549 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:22:23.549 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:22:23.549 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:22:23.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:22:23.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:22:23.549 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:22:23.549 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:22:23.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:22:23.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:22:23.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:22:23.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:22:23.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:22:23.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:22:23.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:22:23.549 | INFO | ------------------------------------------ | 20260507 19:22:23.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:22:23.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:22:23.549 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:22:23.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:22:23.549 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.41e+01 / 1.99e+00 | 20260507 19:22:23.549 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.17e-02 / 5.24e-03 | 20260507 19:22:23.549 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.50e+00, 5.99e-01 / 2.25e-01 | 20260507 19:22:23.550 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.51e+00, 2.11e-01 / 1.09e-01 | 20260507 19:22:23.550 | INFO | PerfReport: 15.88%, DME -- 4.21e+01 / 4.53e+01, 4.03e+00 / 1.65e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 55.90%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.71e+00 / 1.55e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.53e+00 / 2.88e+00, 4.43e-01 / 1.98e-01 | 20260507 19:22:23.550 | INFO | PerfReport: 3.67%, Metrics -- 1.21e+00 / 1.43e+00, 3.05e-01 / 1.90e-01 | 20260507 19:22:23.550 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.78e-01 / 4.12e-01, 6.08e-02 / 3.06e-02 | 20260507 19:22:23.550 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:22:23.550 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:22:23.550 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:22:23.550 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 24.10%, Upload -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 30.98%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 1.88%, LockUpload -- 2.01e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 2.63%, LockDownload -- 2.82e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260507 19:22:23.550 | INFO | PerfReport: 16.18%, LockCompute -- 1.73e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:22:23.551 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 19:22:23.645 | INFO | Starting a round of Baz IO | 20260507 19:22:27.130 | INFO | Populating RTMetrics report | 20260507 19:22:27.130 | INFO | RTMetrics report populated | 20260507 19:22:27.130 | INFO | Opening tmp RTMetrics json file | 20260507 19:22:27.131 | INFO | Serializing RTMetrics to json | 20260507 19:22:27.153 | INFO | Serialization done, swapping tmp file now... | 20260507 19:22:27.177 | INFO | RTMetrics done | 20260507 19:22:27.258 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.612 seconds | 20260507 19:22:27.258 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.49% / 0.01% / 54.49% | 20260507 19:22:27.259 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.45032% / 1.98682e-05% / 3.57628e-05% | 20260507 19:22:27.259 | INFO | Baz IO overall: | 20260507 19:22:27.259 | INFO | BytesWritten : 890843098326 | 20260507 19:22:27.259 | INFO | NumEvents : 431922379149 | 20260507 19:22:28.688 | INFO | Analyzing chunk frames = [1974272,1974784) | 20260507 19:22:28.688 | INFO | MemInfo: { | 20260507 19:22:28.688 | INFO | "gpu" : " 17 GB", | 20260507 19:22:28.688 | INFO | "huge" : "", | 20260507 19:22:28.688 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:28.688 | INFO | "malloc" : " 14 GB", | 20260507 19:22:28.688 | INFO | "pinned" : " 29 GB" | 20260507 19:22:28.688 | INFO | } | 20260507 19:22:28.689 | INFO | Perf report for the last 5.13998 seconds: | 20260507 19:22:28.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:28.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:28.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:28.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:28.689 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 19:22:28.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:28.689 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 19:22:28.689 | INFO | PRE_HQ : 66.6%, 1.8, 61.1ms, 0 | 20260507 19:22:28.689 | INFO | BAZWRITER : 70.1%, 4.3, 153.3ms, 0 | 20260507 19:22:28.689 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 19:22:28.689 | INFO | Uploaded 36GiB to GPU in 2.52926s (14.2334GiB/s) | 20260507 19:22:28.689 | INFO | Downloaded 38GiB to GPU in 2.71663s (13.9879GiB/s) | 20260507 19:22:33.829 | INFO | Analyzing chunk frames = [1974784,1975296) | 20260507 19:22:33.829 | INFO | MemInfo: { | 20260507 19:22:33.829 | INFO | "gpu" : " 17 GB", | 20260507 19:22:33.829 | INFO | "huge" : "", | 20260507 19:22:33.829 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:33.829 | INFO | "malloc" : " 14 GB", | 20260507 19:22:33.829 | INFO | "pinned" : " 29 GB" | 20260507 19:22:33.829 | INFO | } | 20260507 19:22:33.829 | INFO | Perf report for the last 5.14066 seconds: | 20260507 19:22:33.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:33.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.829 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 19:22:33.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.829 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:22:33.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:33.830 | INFO | Uploaded 36GiB to GPU in 2.51826s (14.2956GiB/s) | 20260507 19:22:33.830 | INFO | Downloaded 34GiB to GPU in 2.50021s (13.5989GiB/s) | 20260507 19:22:38.969 | INFO | Analyzing chunk frames = [1975296,1975808) | 20260507 19:22:38.969 | INFO | MemInfo: { | 20260507 19:22:38.969 | INFO | "gpu" : " 17 GB", | 20260507 19:22:38.969 | INFO | "huge" : "", | 20260507 19:22:38.969 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:38.969 | INFO | "malloc" : " 14 GB", | 20260507 19:22:38.969 | INFO | "pinned" : " 29 GB" | 20260507 19:22:38.969 | INFO | } | 20260507 19:22:38.969 | INFO | Perf report for the last 5.14022 seconds: | 20260507 19:22:38.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:38.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.970 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:22:38.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.970 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:22:38.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:38.970 | INFO | Uploaded 36GiB to GPU in 2.5076s (14.3564GiB/s) | 20260507 19:22:38.970 | INFO | Downloaded 34GiB to GPU in 2.49097s (13.6493GiB/s) | 20260507 19:22:44.108 | INFO | Analyzing chunk frames = [1975808,1976320) | 20260507 19:22:44.108 | INFO | MemInfo: { | 20260507 19:22:44.108 | INFO | "gpu" : " 17 GB", | 20260507 19:22:44.108 | INFO | "huge" : "", | 20260507 19:22:44.108 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:44.108 | INFO | "malloc" : " 14 GB", | 20260507 19:22:44.108 | INFO | "pinned" : " 29 GB" | 20260507 19:22:44.108 | INFO | } | 20260507 19:22:44.108 | INFO | Perf report for the last 5.13882 seconds: | 20260507 19:22:44.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:44.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.108 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:22:44.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.108 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:22:44.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:44.109 | INFO | Uploaded 36GiB to GPU in 2.4418s (14.7432GiB/s) | 20260507 19:22:44.109 | INFO | Downloaded 34GiB to GPU in 2.42168s (14.0398GiB/s) | 20260507 19:22:49.247 | INFO | Analyzing chunk frames = [1976320,1976832) | 20260507 19:22:49.247 | INFO | MemInfo: { | 20260507 19:22:49.247 | INFO | "gpu" : " 17 GB", | 20260507 19:22:49.247 | INFO | "huge" : "", | 20260507 19:22:49.247 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:49.247 | INFO | "malloc" : " 14 GB", | 20260507 19:22:49.247 | INFO | "pinned" : " 29 GB" | 20260507 19:22:49.247 | INFO | } | 20260507 19:22:49.247 | INFO | Perf report for the last 5.13835 seconds: | 20260507 19:22:49.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:49.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:22:49.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:22:49.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:49.247 | INFO | Uploaded 36GiB to GPU in 2.34486s (15.3527GiB/s) | 20260507 19:22:49.247 | INFO | Downloaded 34GiB to GPU in 2.50602s (13.5673GiB/s) | 20260507 19:22:54.386 | INFO | Analyzing chunk frames = [1976832,1977344) | 20260507 19:22:54.386 | INFO | MemInfo: { | 20260507 19:22:54.386 | INFO | "gpu" : " 17 GB", | 20260507 19:22:54.386 | INFO | "huge" : "", | 20260507 19:22:54.386 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:54.386 | INFO | "malloc" : " 14 GB", | 20260507 19:22:54.386 | INFO | "pinned" : " 29 GB" | 20260507 19:22:54.386 | INFO | } | 20260507 19:22:54.386 | INFO | Perf report for the last 5.13898 seconds: | 20260507 19:22:54.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:54.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:22:54.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:22:54.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:54.386 | INFO | Uploaded 36GiB to GPU in 2.46853s (14.5836GiB/s) | 20260507 19:22:54.387 | INFO | Downloaded 34GiB to GPU in 2.41425s (14.0831GiB/s) | 20260507 19:22:59.525 | INFO | Analyzing chunk frames = [1977344,1977856) | 20260507 19:22:59.525 | INFO | MemInfo: { | 20260507 19:22:59.525 | INFO | "gpu" : " 17 GB", | 20260507 19:22:59.525 | INFO | "huge" : "", | 20260507 19:22:59.525 | INFO | "huge_shared" : " 29 GB", | 20260507 19:22:59.525 | INFO | "malloc" : " 14 GB", | 20260507 19:22:59.525 | INFO | "pinned" : " 29 GB" | 20260507 19:22:59.525 | INFO | } | 20260507 19:22:59.525 | INFO | Perf report for the last 5.13977 seconds: | 20260507 19:22:59.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:22:59.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.525 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 19:22:59.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.526 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 19:22:59.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:22:59.526 | INFO | Uploaded 36GiB to GPU in 2.48366s (14.4947GiB/s) | 20260507 19:22:59.526 | INFO | Downloaded 34GiB to GPU in 2.42665s (14.0111GiB/s) | 20260507 19:23:04.664 | INFO | Analyzing chunk frames = [1977856,1978368) | 20260507 19:23:04.664 | INFO | MemInfo: { | 20260507 19:23:04.664 | INFO | "gpu" : " 17 GB", | 20260507 19:23:04.664 | INFO | "huge" : "", | 20260507 19:23:04.665 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:04.665 | INFO | "malloc" : " 14 GB", | 20260507 19:23:04.665 | INFO | "pinned" : " 29 GB" | 20260507 19:23:04.665 | INFO | } | 20260507 19:23:04.665 | INFO | Perf report for the last 5.13932 seconds: | 20260507 19:23:04.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:04.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:23:04.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 19:23:04.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:04.665 | INFO | Uploaded 36GiB to GPU in 2.49772s (14.4132GiB/s) | 20260507 19:23:04.665 | INFO | Downloaded 34GiB to GPU in 2.4308s (13.9872GiB/s) | 20260507 19:23:04.762 | INFO | Starting a round of Baz IO | 20260507 19:23:08.267 | INFO | Populating RTMetrics report | 20260507 19:23:08.267 | INFO | RTMetrics report populated | 20260507 19:23:08.267 | INFO | Opening tmp RTMetrics json file | 20260507 19:23:08.289 | INFO | Serializing RTMetrics to json | 20260507 19:23:08.297 | INFO | Serialization done, swapping tmp file now... | 20260507 19:23:08.297 | INFO | RTMetrics done | 20260507 19:23:08.364 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.601 seconds | 20260507 19:23:08.364 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.48% / 0.01% / 54.51% | 20260507 19:23:08.364 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.43225% / 1.98682e-05% / 3.57628e-05% | 20260507 19:23:08.364 | INFO | Baz IO overall: | 20260507 19:23:08.364 | INFO | BytesWritten : 892768456867 | 20260507 19:23:08.364 | INFO | NumEvents : 432855444576 | 20260507 19:23:09.804 | INFO | Analyzing chunk frames = [1978368,1978880) | 20260507 19:23:09.804 | INFO | MemInfo: { | 20260507 19:23:09.804 | INFO | "gpu" : " 17 GB", | 20260507 19:23:09.804 | INFO | "huge" : "", | 20260507 19:23:09.804 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:09.804 | INFO | "malloc" : " 14 GB", | 20260507 19:23:09.804 | INFO | "pinned" : " 29 GB" | 20260507 19:23:09.804 | INFO | } | 20260507 19:23:09.804 | INFO | Perf report for the last 5.13932 seconds: | 20260507 19:23:09.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:09.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:09.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:09.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:09.804 | INFO | ANALYSIS : 69.1%, 3.0, 106.9ms, 0 | 20260507 19:23:09.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:09.804 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:23:09.804 | INFO | PRE_HQ : 66.7%, 1.7, 58.8ms, 0 | 20260507 19:23:09.804 | INFO | BAZWRITER : 69.7%, 4.5, 160.9ms, 0 | 20260507 19:23:09.804 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 19:23:09.805 | INFO | Uploaded 36GiB to GPU in 2.54715s (14.1334GiB/s) | 20260507 19:23:09.805 | INFO | Downloaded 38GiB to GPU in 2.68682s (14.1431GiB/s) | 20260507 19:23:14.943 | INFO | Analyzing chunk frames = [1978880,1979392) | 20260507 19:23:14.943 | INFO | MemInfo: { | 20260507 19:23:14.943 | INFO | "gpu" : " 17 GB", | 20260507 19:23:14.943 | INFO | "huge" : "", | 20260507 19:23:14.944 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:14.944 | INFO | "malloc" : " 14 GB", | 20260507 19:23:14.944 | INFO | "pinned" : " 29 GB" | 20260507 19:23:14.944 | INFO | } | 20260507 19:23:14.944 | INFO | Perf report for the last 5.13963 seconds: | 20260507 19:23:14.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:14.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:23:14.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:23:14.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:14.944 | INFO | Uploaded 36GiB to GPU in 2.54859s (14.1255GiB/s) | 20260507 19:23:14.944 | INFO | Downloaded 34GiB to GPU in 2.49242s (13.6414GiB/s) | 20260507 19:23:20.083 | INFO | Analyzing chunk frames = [1979392,1979904) | 20260507 19:23:20.083 | INFO | MemInfo: { | 20260507 19:23:20.083 | INFO | "gpu" : " 17 GB", | 20260507 19:23:20.083 | INFO | "huge" : "", | 20260507 19:23:20.083 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:20.083 | INFO | "malloc" : " 14 GB", | 20260507 19:23:20.083 | INFO | "pinned" : " 29 GB" | 20260507 19:23:20.083 | INFO | } | 20260507 19:23:20.083 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:23:20.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:20.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:23:20.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:23:20.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:20.084 | INFO | Uploaded 36GiB to GPU in 2.51327s (14.324GiB/s) | 20260507 19:23:20.084 | INFO | Downloaded 34GiB to GPU in 2.49053s (13.6517GiB/s) | 20260507 19:23:25.222 | INFO | Analyzing chunk frames = [1979904,1980416) | 20260507 19:23:25.222 | INFO | MemInfo: { | 20260507 19:23:25.222 | INFO | "gpu" : " 17 GB", | 20260507 19:23:25.222 | INFO | "huge" : "", | 20260507 19:23:25.222 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:25.222 | INFO | "malloc" : " 14 GB", | 20260507 19:23:25.222 | INFO | "pinned" : " 29 GB" | 20260507 19:23:25.222 | INFO | } | 20260507 19:23:25.222 | INFO | Perf report for the last 5.1383 seconds: | 20260507 19:23:25.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:25.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:23:25.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:23:25.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:25.222 | INFO | Uploaded 36GiB to GPU in 2.47803s (14.5276GiB/s) | 20260507 19:23:25.222 | INFO | Downloaded 34GiB to GPU in 2.40505s (14.1369GiB/s) | 20260507 19:23:30.361 | INFO | Analyzing chunk frames = [1980416,1980928) | 20260507 19:23:30.361 | INFO | MemInfo: { | 20260507 19:23:30.361 | INFO | "gpu" : " 17 GB", | 20260507 19:23:30.361 | INFO | "huge" : "", | 20260507 19:23:30.361 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:30.361 | INFO | "malloc" : " 14 GB", | 20260507 19:23:30.361 | INFO | "pinned" : " 29 GB" | 20260507 19:23:30.361 | INFO | } | 20260507 19:23:30.361 | INFO | Perf report for the last 5.13968 seconds: | 20260507 19:23:30.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:30.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.362 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:23:30.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.362 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:23:30.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:30.362 | INFO | Uploaded 36GiB to GPU in 2.33499s (15.4176GiB/s) | 20260507 19:23:30.362 | INFO | Downloaded 34GiB to GPU in 2.49854s (13.6079GiB/s) | 20260507 19:23:35.500 | INFO | Analyzing chunk frames = [1980928,1981440) | 20260507 19:23:35.500 | INFO | MemInfo: { | 20260507 19:23:35.500 | INFO | "gpu" : " 17 GB", | 20260507 19:23:35.500 | INFO | "huge" : "", | 20260507 19:23:35.500 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:35.500 | INFO | "malloc" : " 14 GB", | 20260507 19:23:35.500 | INFO | "pinned" : " 29 GB" | 20260507 19:23:35.500 | INFO | } | 20260507 19:23:35.500 | INFO | Perf report for the last 5.13852 seconds: | 20260507 19:23:35.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:35.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.500 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:23:35.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.500 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 19:23:35.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:35.501 | INFO | Uploaded 36GiB to GPU in 2.49162s (14.4484GiB/s) | 20260507 19:23:35.501 | INFO | Downloaded 34GiB to GPU in 2.42488s (14.0213GiB/s) | 20260507 19:23:40.639 | INFO | Analyzing chunk frames = [1981440,1981952) | 20260507 19:23:40.639 | INFO | MemInfo: { | 20260507 19:23:40.639 | INFO | "gpu" : " 17 GB", | 20260507 19:23:40.639 | INFO | "huge" : "", | 20260507 19:23:40.639 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:40.639 | INFO | "malloc" : " 14 GB", | 20260507 19:23:40.640 | INFO | "pinned" : " 29 GB" | 20260507 19:23:40.640 | INFO | } | 20260507 19:23:40.640 | INFO | Perf report for the last 5.13974 seconds: | 20260507 19:23:40.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:40.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:23:40.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | PRE_HQ : 38.1%, 1.0, 19.8ms, 0 | 20260507 19:23:40.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:40.640 | INFO | Uploaded 36GiB to GPU in 2.53421s (14.2056GiB/s) | 20260507 19:23:40.640 | INFO | Downloaded 34GiB to GPU in 2.47275s (13.7499GiB/s) | 20260507 19:23:45.779 | INFO | Analyzing chunk frames = [1981952,1982464) | 20260507 19:23:45.779 | INFO | MemInfo: { | 20260507 19:23:45.779 | INFO | "gpu" : " 17 GB", | 20260507 19:23:45.779 | INFO | "huge" : "", | 20260507 19:23:45.779 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:45.779 | INFO | "malloc" : " 14 GB", | 20260507 19:23:45.779 | INFO | "pinned" : " 29 GB" | 20260507 19:23:45.780 | INFO | } | 20260507 19:23:45.780 | INFO | Perf report for the last 5.13984 seconds: | 20260507 19:23:45.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:45.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:23:45.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | PRE_HQ : 39.0%, 1.0, 20.2ms, 0 | 20260507 19:23:45.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:45.780 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:23:45.780 | INFO | Uploaded 36GiB to GPU in 2.50052s (14.397GiB/s) | 20260507 19:23:45.780 | INFO | Total mem usage: 16.4 GiB | 20260507 19:23:45.780 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:23:45.780 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.67%) | 20260507 19:23:45.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:23:45.780 | INFO | Downloaded 34GiB to GPU in 2.43754s (13.9485GiB/s) | 20260507 19:23:45.780 | INFO | High Water Breakdown: | 20260507 19:23:45.780 | INFO | 60% : 9.06 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:23:45.780 | INFO | 39.8% : 6.01 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:23:45.780 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:23:45.780 | INFO | ------------------------------------------ | 20260507 19:23:45.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:23:45.780 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:23:45.780 | INFO | Total mem usage: 26.5 GiB | 20260507 19:23:45.780 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:23:45.780 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:23:45.780 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:23:45.780 | INFO | High Water Breakdown: | 20260507 19:23:45.780 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:23:45.780 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:23:45.780 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:23:45.780 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:23:45.780 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:23:45.780 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:23:45.780 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:23:45.780 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:23:45.780 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:23:45.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:23:45.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:23:45.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:23:45.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:23:45.780 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:23:45.780 | INFO | ------------------------------------------ | 20260507 19:23:45.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:23:45.780 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:23:45.780 | INFO | Total mem usage: 27.8 GiB | 20260507 19:23:45.780 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:23:45.780 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:23:45.780 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:23:45.780 | INFO | High Water Breakdown: | 20260507 19:23:45.780 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:23:45.780 | INFO | ------------------------------------------ | 20260507 19:23:45.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:23:45.780 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:23:45.780 | INFO | Total mem usage: 15.7 GiB | 20260507 19:23:45.780 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:23:45.780 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:23:45.780 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:23:45.780 | INFO | High Water Breakdown: | 20260507 19:23:45.780 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:23:45.780 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:23:45.780 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:23:45.780 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:23:45.780 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:23:45.780 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:23:45.780 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:23:45.780 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:23:45.780 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:23:45.780 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:23:45.780 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:23:45.780 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:23:45.780 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:23:45.780 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:23:45.780 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:23:45.780 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:23:45.780 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:23:45.780 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:23:45.780 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:23:45.780 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:23:45.780 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:23:45.780 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:23:45.780 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:23:45.780 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:23:45.780 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:23:45.780 | INFO | ------------------------------------------ | 20260507 19:23:45.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:23:45.781 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:23:45.781 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:23:45.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:23:45.782 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.40e+01 / 1.73e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 0.30%, AggregateStats -- 9.95e-02 / 1.03e-01, 1.32e-02 / 6.12e-03 | 20260507 19:23:45.782 | INFO | PerfReport: 12.11%, Baseline -- 4.02e+00 / 4.52e+00, 6.06e-01 / 2.43e-01 | 20260507 19:23:45.782 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.51e+00, 2.12e-01 / 1.12e-01 | 20260507 19:23:45.782 | INFO | PerfReport: 15.87%, DME -- 4.21e+01 / 4.53e+01, 3.90e+00 / 1.44e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.74e+00 / 1.81e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.53e+00 / 2.87e+00, 4.45e-01 / 2.21e-01 | 20260507 19:23:45.782 | INFO | PerfReport: 3.67%, Metrics -- 1.22e+00 / 1.43e+00, 3.05e-01 / 1.80e-01 | 20260507 19:23:45.782 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.79e-01 / 4.09e-01, 6.11e-02 / 3.01e-02 | 20260507 19:23:45.782 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:23:45.782 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:23:45.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:23:45.782 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 24.18%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 31.01%, Compute -- 3.32e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 1.89%, LockUpload -- 2.02e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 2.57%, LockDownload -- 2.75e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 16.10%, LockCompute -- 1.72e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 19:23:45.782 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260507 19:23:45.896 | INFO | Starting a round of Baz IO | 20260507 19:23:49.398 | INFO | Populating RTMetrics report | 20260507 19:23:49.398 | INFO | RTMetrics report populated | 20260507 19:23:49.398 | INFO | Opening tmp RTMetrics json file | 20260507 19:23:49.404 | INFO | Serializing RTMetrics to json | 20260507 19:23:49.427 | INFO | Serialization done, swapping tmp file now... | 20260507 19:23:49.428 | INFO | RTMetrics done | 20260507 19:23:49.496 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.599 seconds | 20260507 19:23:49.496 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.47% / 0.01% / 54.52% | 20260507 19:23:49.496 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.41473% / 3.17891e-05% / 2.78155e-05% | 20260507 19:23:49.496 | INFO | Baz IO overall: | 20260507 19:23:49.496 | INFO | BytesWritten : 894691377978 | 20260507 19:23:49.496 | INFO | NumEvents : 433787225019 | 20260507 19:23:50.920 | INFO | Analyzing chunk frames = [1982464,1982976) | 20260507 19:23:50.920 | INFO | MemInfo: { | 20260507 19:23:50.920 | INFO | "gpu" : " 17 GB", | 20260507 19:23:50.920 | INFO | "huge" : "", | 20260507 19:23:50.920 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:50.920 | INFO | "malloc" : " 14 GB", | 20260507 19:23:50.920 | INFO | "pinned" : " 29 GB" | 20260507 19:23:50.920 | INFO | } | 20260507 19:23:50.920 | INFO | Perf report for the last 5.14007 seconds: | 20260507 19:23:50.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:50.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:50.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:50.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:50.920 | INFO | ANALYSIS : 70.0%, 2.9, 104.8ms, 0 | 20260507 19:23:50.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:50.920 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260507 19:23:50.920 | INFO | PRE_HQ : 69.1%, 1.8, 65.4ms, 0 | 20260507 19:23:50.920 | INFO | BAZWRITER : 69.6%, 4.3, 153.4ms, 0 | 20260507 19:23:50.920 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 19:23:50.920 | INFO | Uploaded 36GiB to GPU in 2.54928s (14.1216GiB/s) | 20260507 19:23:50.920 | INFO | Downloaded 38GiB to GPU in 2.72108s (13.9651GiB/s) | 20260507 19:23:56.058 | INFO | Analyzing chunk frames = [1982976,1983488) | 20260507 19:23:56.058 | INFO | MemInfo: { | 20260507 19:23:56.058 | INFO | "gpu" : " 17 GB", | 20260507 19:23:56.058 | INFO | "huge" : "", | 20260507 19:23:56.058 | INFO | "huge_shared" : " 29 GB", | 20260507 19:23:56.058 | INFO | "malloc" : " 14 GB", | 20260507 19:23:56.058 | INFO | "pinned" : " 29 GB" | 20260507 19:23:56.058 | INFO | } | 20260507 19:23:56.058 | INFO | Perf report for the last 5.13853 seconds: | 20260507 19:23:56.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:23:56.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.058 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 19:23:56.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.058 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:23:56.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:23:56.059 | INFO | Uploaded 36GiB to GPU in 2.56264s (14.048GiB/s) | 20260507 19:23:56.059 | INFO | Downloaded 34GiB to GPU in 2.4761s (13.7313GiB/s) | 20260507 19:24:01.197 | INFO | Analyzing chunk frames = [1983488,1984000) | 20260507 19:24:01.197 | INFO | MemInfo: { | 20260507 19:24:01.197 | INFO | "gpu" : " 17 GB", | 20260507 19:24:01.197 | INFO | "huge" : "", | 20260507 19:24:01.197 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:01.197 | INFO | "malloc" : " 14 GB", | 20260507 19:24:01.197 | INFO | "pinned" : " 29 GB" | 20260507 19:24:01.197 | INFO | } | 20260507 19:24:01.197 | INFO | Perf report for the last 5.13855 seconds: | 20260507 19:24:01.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:01.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:24:01.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:24:01.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:01.197 | INFO | Uploaded 36GiB to GPU in 2.56113s (14.0563GiB/s) | 20260507 19:24:01.197 | INFO | Downloaded 34GiB to GPU in 2.46889s (13.7714GiB/s) | 20260507 19:24:06.336 | INFO | Analyzing chunk frames = [1984000,1984512) | 20260507 19:24:06.336 | INFO | MemInfo: { | 20260507 19:24:06.336 | INFO | "gpu" : " 17 GB", | 20260507 19:24:06.336 | INFO | "huge" : "", | 20260507 19:24:06.336 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:06.336 | INFO | "malloc" : " 14 GB", | 20260507 19:24:06.336 | INFO | "pinned" : " 29 GB" | 20260507 19:24:06.336 | INFO | } | 20260507 19:24:06.336 | INFO | Perf report for the last 5.13915 seconds: | 20260507 19:24:06.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:06.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 19:24:06.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:24:06.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:06.336 | INFO | Uploaded 36GiB to GPU in 2.45277s (14.6773GiB/s) | 20260507 19:24:06.336 | INFO | Downloaded 34GiB to GPU in 2.39345s (14.2054GiB/s) | 20260507 19:24:11.476 | INFO | Analyzing chunk frames = [1984512,1985024) | 20260507 19:24:11.476 | INFO | MemInfo: { | 20260507 19:24:11.476 | INFO | "gpu" : " 17 GB", | 20260507 19:24:11.476 | INFO | "huge" : "", | 20260507 19:24:11.476 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:11.476 | INFO | "malloc" : " 14 GB", | 20260507 19:24:11.476 | INFO | "pinned" : " 29 GB" | 20260507 19:24:11.476 | INFO | } | 20260507 19:24:11.476 | INFO | Perf report for the last 5.13994 seconds: | 20260507 19:24:11.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:11.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.476 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:24:11.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.476 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 19:24:11.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:11.477 | INFO | Uploaded 36GiB to GPU in 2.34386s (15.3593GiB/s) | 20260507 19:24:11.477 | INFO | Downloaded 34GiB to GPU in 2.50622s (13.5662GiB/s) | 20260507 19:24:16.615 | INFO | Analyzing chunk frames = [1985024,1985536) | 20260507 19:24:16.615 | INFO | MemInfo: { | 20260507 19:24:16.615 | INFO | "gpu" : " 17 GB", | 20260507 19:24:16.615 | INFO | "huge" : "", | 20260507 19:24:16.615 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:16.615 | INFO | "malloc" : " 14 GB", | 20260507 19:24:16.615 | INFO | "pinned" : " 29 GB" | 20260507 19:24:16.615 | INFO | } | 20260507 19:24:16.615 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:24:16.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:16.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.615 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:24:16.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.615 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:24:16.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:16.616 | INFO | Uploaded 36GiB to GPU in 2.44535s (14.7218GiB/s) | 20260507 19:24:16.616 | INFO | Downloaded 34GiB to GPU in 2.43621s (13.9561GiB/s) | 20260507 19:24:21.756 | INFO | Analyzing chunk frames = [1985536,1986048) | 20260507 19:24:21.756 | INFO | MemInfo: { | 20260507 19:24:21.756 | INFO | "gpu" : " 17 GB", | 20260507 19:24:21.756 | INFO | "huge" : "", | 20260507 19:24:21.756 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:21.756 | INFO | "malloc" : " 14 GB", | 20260507 19:24:21.756 | INFO | "pinned" : " 29 GB" | 20260507 19:24:21.756 | INFO | } | 20260507 19:24:21.756 | INFO | Perf report for the last 5.14085 seconds: | 20260507 19:24:21.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:21.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.756 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 19:24:21.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.756 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 19:24:21.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:21.757 | INFO | Uploaded 36GiB to GPU in 2.47901s (14.5219GiB/s) | 20260507 19:24:21.757 | INFO | Downloaded 34GiB to GPU in 2.46082s (13.8165GiB/s) | 20260507 19:24:26.894 | INFO | Analyzing chunk frames = [1986048,1986560) | 20260507 19:24:26.894 | INFO | MemInfo: { | 20260507 19:24:26.894 | INFO | "gpu" : " 17 GB", | 20260507 19:24:26.894 | INFO | "huge" : "", | 20260507 19:24:26.894 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:26.894 | INFO | "malloc" : " 14 GB", | 20260507 19:24:26.894 | INFO | "pinned" : " 29 GB" | 20260507 19:24:26.894 | INFO | } | 20260507 19:24:26.894 | INFO | Perf report for the last 5.13813 seconds: | 20260507 19:24:26.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:26.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:24:26.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 19:24:26.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:26.895 | INFO | Uploaded 36GiB to GPU in 2.46947s (14.578GiB/s) | 20260507 19:24:26.895 | INFO | Downloaded 34GiB to GPU in 2.45401s (13.8549GiB/s) | 20260507 19:24:26.992 | INFO | Starting a round of Baz IO | 20260507 19:24:30.476 | INFO | Populating RTMetrics report | 20260507 19:24:30.477 | INFO | RTMetrics report populated | 20260507 19:24:30.477 | INFO | Opening tmp RTMetrics json file | 20260507 19:24:30.502 | INFO | Serializing RTMetrics to json | 20260507 19:24:30.505 | INFO | Serialization done, swapping tmp file now... | 20260507 19:24:30.505 | INFO | RTMetrics done | 20260507 19:24:30.589 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.596 seconds | 20260507 19:24:30.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.46% / 0.01% / 54.53% | 20260507 19:24:30.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.41094% / 2.38419e-05% / 2.38419e-05% | 20260507 19:24:30.589 | INFO | Baz IO overall: | 20260507 19:24:30.589 | INFO | BytesWritten : 896613245371 | 20260507 19:24:30.589 | INFO | NumEvents : 434718452610 | 20260507 19:24:32.033 | INFO | Analyzing chunk frames = [1986560,1987072) | 20260507 19:24:32.033 | INFO | MemInfo: { | 20260507 19:24:32.033 | INFO | "gpu" : " 17 GB", | 20260507 19:24:32.033 | INFO | "huge" : "", | 20260507 19:24:32.033 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:32.033 | INFO | "malloc" : " 14 GB", | 20260507 19:24:32.033 | INFO | "pinned" : " 29 GB" | 20260507 19:24:32.033 | INFO | } | 20260507 19:24:32.033 | INFO | Perf report for the last 5.13905 seconds: | 20260507 19:24:32.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:32.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:32.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:32.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:32.033 | INFO | ANALYSIS : 68.9%, 2.9, 106.5ms, 0 | 20260507 19:24:32.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:32.034 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 19:24:32.034 | INFO | PRE_HQ : 66.3%, 1.7, 58.7ms, 0 | 20260507 19:24:32.034 | INFO | BAZWRITER : 69.5%, 4.2, 149.7ms, 0 | 20260507 19:24:32.034 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 19:24:32.034 | INFO | Uploaded 36GiB to GPU in 2.47538s (14.5432GiB/s) | 20260507 19:24:32.034 | INFO | Downloaded 38GiB to GPU in 2.69387s (14.1061GiB/s) | 20260507 19:24:37.172 | INFO | Analyzing chunk frames = [1987072,1987584) | 20260507 19:24:37.172 | INFO | MemInfo: { | 20260507 19:24:37.172 | INFO | "gpu" : " 17 GB", | 20260507 19:24:37.172 | INFO | "huge" : "", | 20260507 19:24:37.172 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:37.172 | INFO | "malloc" : " 14 GB", | 20260507 19:24:37.172 | INFO | "pinned" : " 29 GB" | 20260507 19:24:37.172 | INFO | } | 20260507 19:24:37.172 | INFO | Perf report for the last 5.13904 seconds: | 20260507 19:24:37.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:37.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:24:37.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:24:37.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:37.173 | INFO | Uploaded 36GiB to GPU in 2.45135s (14.6858GiB/s) | 20260507 19:24:37.173 | INFO | Downloaded 34GiB to GPU in 2.49294s (13.6385GiB/s) | 20260507 19:24:42.311 | INFO | Analyzing chunk frames = [1987584,1988096) | 20260507 19:24:42.311 | INFO | MemInfo: { | 20260507 19:24:42.311 | INFO | "gpu" : " 17 GB", | 20260507 19:24:42.311 | INFO | "huge" : "", | 20260507 19:24:42.311 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:42.311 | INFO | "malloc" : " 14 GB", | 20260507 19:24:42.311 | INFO | "pinned" : " 29 GB" | 20260507 19:24:42.311 | INFO | } | 20260507 19:24:42.311 | INFO | Perf report for the last 5.13856 seconds: | 20260507 19:24:42.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:42.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:24:42.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:24:42.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:42.311 | INFO | Uploaded 36GiB to GPU in 2.49904s (14.4055GiB/s) | 20260507 19:24:42.311 | INFO | Downloaded 34GiB to GPU in 2.4799s (13.7102GiB/s) | 20260507 19:24:47.451 | INFO | Analyzing chunk frames = [1988096,1988608) | 20260507 19:24:47.451 | INFO | MemInfo: { | 20260507 19:24:47.451 | INFO | "gpu" : " 17 GB", | 20260507 19:24:47.451 | INFO | "huge" : "", | 20260507 19:24:47.451 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:47.451 | INFO | "malloc" : " 14 GB", | 20260507 19:24:47.451 | INFO | "pinned" : " 29 GB" | 20260507 19:24:47.451 | INFO | } | 20260507 19:24:47.451 | INFO | Perf report for the last 5.14012 seconds: | 20260507 19:24:47.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:47.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 19:24:47.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:24:47.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:47.451 | INFO | Uploaded 36GiB to GPU in 2.41399s (14.913GiB/s) | 20260507 19:24:47.452 | INFO | Downloaded 34GiB to GPU in 2.40214s (14.1541GiB/s) | 20260507 19:24:52.590 | INFO | Analyzing chunk frames = [1988608,1989120) | 20260507 19:24:52.590 | INFO | MemInfo: { | 20260507 19:24:52.590 | INFO | "gpu" : " 17 GB", | 20260507 19:24:52.590 | INFO | "huge" : "", | 20260507 19:24:52.591 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:52.591 | INFO | "malloc" : " 14 GB", | 20260507 19:24:52.591 | INFO | "pinned" : " 29 GB" | 20260507 19:24:52.591 | INFO | } | 20260507 19:24:52.591 | INFO | Perf report for the last 5.1395 seconds: | 20260507 19:24:52.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:52.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 19:24:52.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:24:52.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:52.591 | INFO | Uploaded 36GiB to GPU in 2.31776s (15.5323GiB/s) | 20260507 19:24:52.591 | INFO | Downloaded 34GiB to GPU in 2.5154s (13.5167GiB/s) | 20260507 19:24:57.730 | INFO | Analyzing chunk frames = [1989120,1989632) | 20260507 19:24:57.730 | INFO | MemInfo: { | 20260507 19:24:57.730 | INFO | "gpu" : " 17 GB", | 20260507 19:24:57.730 | INFO | "huge" : "", | 20260507 19:24:57.730 | INFO | "huge_shared" : " 29 GB", | 20260507 19:24:57.730 | INFO | "malloc" : " 14 GB", | 20260507 19:24:57.730 | INFO | "pinned" : " 29 GB" | 20260507 19:24:57.730 | INFO | } | 20260507 19:24:57.730 | INFO | Perf report for the last 5.13969 seconds: | 20260507 19:24:57.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:24:57.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.731 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:24:57.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.731 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:24:57.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:24:57.731 | INFO | Uploaded 36GiB to GPU in 2.44674s (14.7134GiB/s) | 20260507 19:24:57.731 | INFO | Downloaded 34GiB to GPU in 2.4352s (13.9619GiB/s) | 20260507 19:25:02.868 | INFO | Analyzing chunk frames = [1989632,1990144) | 20260507 19:25:02.868 | INFO | MemInfo: { | 20260507 19:25:02.868 | INFO | "gpu" : " 17 GB", | 20260507 19:25:02.868 | INFO | "huge" : "", | 20260507 19:25:02.868 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:02.868 | INFO | "malloc" : " 14 GB", | 20260507 19:25:02.869 | INFO | "pinned" : " 29 GB" | 20260507 19:25:02.869 | INFO | } | 20260507 19:25:02.869 | INFO | Perf report for the last 5.13811 seconds: | 20260507 19:25:02.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:02.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 19:25:02.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 19:25:02.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:02.869 | INFO | Uploaded 36GiB to GPU in 2.46679s (14.5939GiB/s) | 20260507 19:25:02.869 | INFO | Downloaded 34GiB to GPU in 2.44688s (13.8952GiB/s) | 20260507 19:25:08.008 | INFO | Analyzing chunk frames = [1990144,1990656) | 20260507 19:25:08.008 | INFO | MemInfo: { | 20260507 19:25:08.008 | INFO | "gpu" : " 17 GB", | 20260507 19:25:08.008 | INFO | "huge" : "", | 20260507 19:25:08.008 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:08.008 | INFO | "malloc" : " 14 GB", | 20260507 19:25:08.008 | INFO | "pinned" : " 29 GB" | 20260507 19:25:08.008 | INFO | } | 20260507 19:25:08.008 | INFO | Perf report for the last 5.13959 seconds: | 20260507 19:25:08.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:08.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.008 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:25:08.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.008 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:25:08.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:08.009 | INFO | Uploaded 36GiB to GPU in 2.43085s (14.8096GiB/s) | 20260507 19:25:08.009 | INFO | Downloaded 34GiB to GPU in 2.43714s (13.9508GiB/s) | 20260507 19:25:08.009 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:25:08.009 | INFO | Total mem usage: 16.4 GiB | 20260507 19:25:08.009 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:25:08.009 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.14%) | 20260507 19:25:08.009 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:25:08.009 | INFO | High Water Breakdown: | 20260507 19:25:08.009 | INFO | 60.3% : 9.23 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:25:08.009 | INFO | 39.6% : 6.06 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:25:08.009 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:25:08.009 | INFO | ------------------------------------------ | 20260507 19:25:08.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:25:08.009 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:25:08.009 | INFO | Total mem usage: 26.5 GiB | 20260507 19:25:08.009 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:25:08.009 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:25:08.009 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 19:25:08.009 | INFO | High Water Breakdown: | 20260507 19:25:08.009 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:25:08.009 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:25:08.009 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:25:08.009 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:25:08.009 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:25:08.009 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:25:08.009 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:25:08.009 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:25:08.009 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:25:08.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:25:08.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:25:08.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:25:08.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:25:08.009 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:25:08.009 | INFO | ------------------------------------------ | 20260507 19:25:08.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:25:08.009 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:25:08.009 | INFO | Total mem usage: 27.8 GiB | 20260507 19:25:08.009 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:25:08.009 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:25:08.009 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:25:08.009 | INFO | High Water Breakdown: | 20260507 19:25:08.009 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:25:08.009 | INFO | ------------------------------------------ | 20260507 19:25:08.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:25:08.009 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:25:08.009 | INFO | Total mem usage: 15.7 GiB | 20260507 19:25:08.009 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:25:08.009 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:25:08.009 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:25:08.009 | INFO | High Water Breakdown: | 20260507 19:25:08.009 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:25:08.009 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:25:08.009 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:25:08.009 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:25:08.009 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:25:08.009 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:25:08.009 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:25:08.009 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:25:08.009 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:25:08.009 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:25:08.009 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:25:08.009 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:25:08.009 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:25:08.009 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:25:08.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:25:08.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:25:08.009 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:25:08.009 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:25:08.009 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:25:08.009 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:25:08.009 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:25:08.009 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:25:08.009 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:25:08.009 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:25:08.009 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:25:08.009 | INFO | ------------------------------------------ | 20260507 19:25:08.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:25:08.010 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:25:08.010 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:25:08.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:25:08.010 | INFO | PerfReport: Full Execution -- 3.33e+01 / 7.56e+01, 1.41e+01 / 1.83e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.03e-01, 1.20e-02 / 6.12e-03 | 20260507 19:25:08.010 | INFO | PerfReport: 12.12%, Baseline -- 4.03e+00 / 4.45e+00, 6.11e-01 / 3.12e-01 | 20260507 19:25:08.010 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.11e-01 / 1.27e-01 | 20260507 19:25:08.010 | INFO | PerfReport: 15.69%, DME -- 4.18e+01 / 4.48e+01, 3.82e+00 / 1.34e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.87e+00 / 1.87e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.51e+00 / 2.78e+00, 4.40e-01 / 2.72e-01 | 20260507 19:25:08.010 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.38e+00, 3.06e-01 / 1.58e-01 | 20260507 19:25:08.010 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.79e-01 / 4.11e-01, 6.23e-02 / 3.37e-02 | 20260507 19:25:08.010 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:25:08.010 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:25:08.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:25:08.010 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 24.01%, Upload -- 2.56e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 31.18%, Compute -- 3.33e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 1.72%, LockUpload -- 1.84e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 2.67%, LockDownload -- 2.85e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 16.11%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:25:08.010 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260507 19:25:08.106 | INFO | Starting a round of Baz IO | 20260507 19:25:11.589 | INFO | Populating RTMetrics report | 20260507 19:25:11.589 | INFO | RTMetrics report populated | 20260507 19:25:11.589 | INFO | Opening tmp RTMetrics json file | 20260507 19:25:11.590 | INFO | Serializing RTMetrics to json | 20260507 19:25:11.591 | INFO | Serialization done, swapping tmp file now... | 20260507 19:25:11.592 | INFO | RTMetrics done | 20260507 19:25:11.687 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.58 seconds | 20260507 19:25:11.687 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.45% / 0.01% / 54.54% | 20260507 19:25:11.687 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.41373% / 3.57628e-05% / 3.57628e-05% | 20260507 19:25:11.687 | INFO | Baz IO overall: | 20260507 19:25:11.687 | INFO | BytesWritten : 898535494595 | 20260507 19:25:11.687 | INFO | NumEvents : 435649997757 | 20260507 19:25:13.148 | INFO | Analyzing chunk frames = [1990656,1991168) | 20260507 19:25:13.148 | INFO | MemInfo: { | 20260507 19:25:13.148 | INFO | "gpu" : " 17 GB", | 20260507 19:25:13.148 | INFO | "huge" : "", | 20260507 19:25:13.148 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:13.148 | INFO | "malloc" : " 14 GB", | 20260507 19:25:13.148 | INFO | "pinned" : " 29 GB" | 20260507 19:25:13.148 | INFO | } | 20260507 19:25:13.148 | INFO | Perf report for the last 5.14028 seconds: | 20260507 19:25:13.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:13.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:13.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:13.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:13.149 | INFO | ANALYSIS : 69.0%, 2.9, 106.7ms, 0 | 20260507 19:25:13.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:13.149 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:25:13.149 | INFO | PRE_HQ : 66.2%, 1.7, 58.8ms, 0 | 20260507 19:25:13.149 | INFO | BAZWRITER : 69.3%, 4.4, 156.0ms, 0 | 20260507 19:25:13.149 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 19:25:13.149 | INFO | Uploaded 36GiB to GPU in 2.48576s (14.4825GiB/s) | 20260507 19:25:13.149 | INFO | Downloaded 38GiB to GPU in 2.69492s (14.1006GiB/s) | 20260507 19:25:18.287 | INFO | Analyzing chunk frames = [1991168,1991680) | 20260507 19:25:18.287 | INFO | MemInfo: { | 20260507 19:25:18.287 | INFO | "gpu" : " 17 GB", | 20260507 19:25:18.287 | INFO | "huge" : "", | 20260507 19:25:18.287 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:18.287 | INFO | "malloc" : " 14 GB", | 20260507 19:25:18.287 | INFO | "pinned" : " 29 GB" | 20260507 19:25:18.287 | INFO | } | 20260507 19:25:18.287 | INFO | Perf report for the last 5.13883 seconds: | 20260507 19:25:18.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:18.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.287 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:25:18.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.287 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:25:18.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:18.288 | INFO | Uploaded 36GiB to GPU in 2.51722s (14.3015GiB/s) | 20260507 19:25:18.288 | INFO | Downloaded 34GiB to GPU in 2.50295s (13.584GiB/s) | 20260507 19:25:23.426 | INFO | Analyzing chunk frames = [1991680,1992192) | 20260507 19:25:23.426 | INFO | MemInfo: { | 20260507 19:25:23.426 | INFO | "gpu" : " 17 GB", | 20260507 19:25:23.426 | INFO | "huge" : "", | 20260507 19:25:23.426 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:23.426 | INFO | "malloc" : " 14 GB", | 20260507 19:25:23.427 | INFO | "pinned" : " 29 GB" | 20260507 19:25:23.427 | INFO | } | 20260507 19:25:23.427 | INFO | Perf report for the last 5.13925 seconds: | 20260507 19:25:23.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:23.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:25:23.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:25:23.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:23.427 | INFO | Uploaded 36GiB to GPU in 2.54228s (14.1605GiB/s) | 20260507 19:25:23.427 | INFO | Downloaded 34GiB to GPU in 2.47219s (13.753GiB/s) | 20260507 19:25:28.565 | INFO | Analyzing chunk frames = [1992192,1992704) | 20260507 19:25:28.565 | INFO | MemInfo: { | 20260507 19:25:28.565 | INFO | "gpu" : " 17 GB", | 20260507 19:25:28.565 | INFO | "huge" : "", | 20260507 19:25:28.565 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:28.566 | INFO | "malloc" : " 14 GB", | 20260507 19:25:28.566 | INFO | "pinned" : " 29 GB" | 20260507 19:25:28.566 | INFO | } | 20260507 19:25:28.566 | INFO | Perf report for the last 5.13903 seconds: | 20260507 19:25:28.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:28.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 19:25:28.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:25:28.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:28.566 | INFO | Uploaded 36GiB to GPU in 2.4372s (14.7711GiB/s) | 20260507 19:25:28.566 | INFO | Downloaded 34GiB to GPU in 2.41177s (14.0975GiB/s) | 20260507 19:25:33.706 | INFO | Analyzing chunk frames = [1992704,1993216) | 20260507 19:25:33.706 | INFO | MemInfo: { | 20260507 19:25:33.706 | INFO | "gpu" : " 17 GB", | 20260507 19:25:33.706 | INFO | "huge" : "", | 20260507 19:25:33.706 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:33.706 | INFO | "malloc" : " 14 GB", | 20260507 19:25:33.706 | INFO | "pinned" : " 29 GB" | 20260507 19:25:33.706 | INFO | } | 20260507 19:25:33.706 | INFO | Perf report for the last 5.14072 seconds: | 20260507 19:25:33.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:33.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.706 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:25:33.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.707 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 19:25:33.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:33.707 | INFO | Uploaded 36GiB to GPU in 2.32332s (15.4951GiB/s) | 20260507 19:25:33.707 | INFO | Downloaded 34GiB to GPU in 2.50398s (13.5784GiB/s) | 20260507 19:25:38.845 | INFO | Analyzing chunk frames = [1993216,1993728) | 20260507 19:25:38.845 | INFO | MemInfo: { | 20260507 19:25:38.845 | INFO | "gpu" : " 17 GB", | 20260507 19:25:38.846 | INFO | "huge" : "", | 20260507 19:25:38.846 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:38.846 | INFO | "malloc" : " 14 GB", | 20260507 19:25:38.846 | INFO | "pinned" : " 29 GB" | 20260507 19:25:38.846 | INFO | } | 20260507 19:25:38.846 | INFO | Perf report for the last 5.13943 seconds: | 20260507 19:25:38.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:38.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:25:38.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:25:38.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:38.846 | INFO | Uploaded 36GiB to GPU in 2.48944s (14.4611GiB/s) | 20260507 19:25:38.846 | INFO | Downloaded 34GiB to GPU in 2.42695s (14.0093GiB/s) | 20260507 19:25:43.987 | INFO | Analyzing chunk frames = [1993728,1994240) | 20260507 19:25:43.987 | INFO | MemInfo: { | 20260507 19:25:43.987 | INFO | "gpu" : " 17 GB", | 20260507 19:25:43.987 | INFO | "huge" : "", | 20260507 19:25:43.987 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:43.987 | INFO | "malloc" : " 14 GB", | 20260507 19:25:43.987 | INFO | "pinned" : " 29 GB" | 20260507 19:25:43.987 | INFO | } | 20260507 19:25:43.987 | INFO | Perf report for the last 5.14103 seconds: | 20260507 19:25:43.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:43.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:25:43.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:25:43.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:43.987 | INFO | Uploaded 36GiB to GPU in 2.51181s (14.3323GiB/s) | 20260507 19:25:43.987 | INFO | Downloaded 34GiB to GPU in 2.44499s (13.906GiB/s) | 20260507 19:25:49.122 | INFO | Analyzing chunk frames = [1994240,1994752) | 20260507 19:25:49.122 | INFO | MemInfo: { | 20260507 19:25:49.122 | INFO | "gpu" : " 17 GB", | 20260507 19:25:49.122 | INFO | "huge" : "", | 20260507 19:25:49.122 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:49.123 | INFO | "malloc" : " 14 GB", | 20260507 19:25:49.123 | INFO | "pinned" : " 29 GB" | 20260507 19:25:49.123 | INFO | } | 20260507 19:25:49.123 | INFO | Perf report for the last 5.1358 seconds: | 20260507 19:25:49.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:49.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 19:25:49.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:25:49.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:49.123 | INFO | Uploaded 36GiB to GPU in 2.50292s (14.3832GiB/s) | 20260507 19:25:49.123 | INFO | Downloaded 34GiB to GPU in 2.42732s (14.0072GiB/s) | 20260507 19:25:49.220 | INFO | Starting a round of Baz IO | 20260507 19:25:52.729 | INFO | Populating RTMetrics report | 20260507 19:25:52.729 | INFO | RTMetrics report populated | 20260507 19:25:52.729 | INFO | Opening tmp RTMetrics json file | 20260507 19:25:52.789 | INFO | Serializing RTMetrics to json | 20260507 19:25:52.850 | INFO | Serialization done, swapping tmp file now... | 20260507 19:25:52.855 | INFO | RTMetrics done | 20260507 19:25:52.874 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.653 seconds | 20260507 19:25:52.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.43% / 0.01% / 54.55% | 20260507 19:25:52.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.41473% / 1.58946e-05% / 3.57628e-05% | 20260507 19:25:52.874 | INFO | Baz IO overall: | 20260507 19:25:52.874 | INFO | BytesWritten : 900459719999 | 20260507 19:25:52.874 | INFO | NumEvents : 436582957244 | 20260507 19:25:54.262 | INFO | Analyzing chunk frames = [1994752,1995264) | 20260507 19:25:54.262 | INFO | MemInfo: { | 20260507 19:25:54.262 | INFO | "gpu" : " 17 GB", | 20260507 19:25:54.262 | INFO | "huge" : "", | 20260507 19:25:54.263 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:54.263 | INFO | "malloc" : " 14 GB", | 20260507 19:25:54.263 | INFO | "pinned" : " 29 GB" | 20260507 19:25:54.263 | INFO | } | 20260507 19:25:54.263 | INFO | Perf report for the last 5.14016 seconds: | 20260507 19:25:54.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:54.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:54.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:54.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:54.263 | INFO | ANALYSIS : 69.4%, 2.9, 105.9ms, 0 | 20260507 19:25:54.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:54.263 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260507 19:25:54.263 | INFO | PRE_HQ : 68.5%, 1.9, 66.1ms, 0 | 20260507 19:25:54.263 | INFO | BAZWRITER : 70.7%, 4.3, 157.0ms, 0 | 20260507 19:25:54.263 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260507 19:25:54.263 | INFO | Uploaded 36GiB to GPU in 2.57331s (13.9898GiB/s) | 20260507 19:25:54.263 | INFO | Downloaded 38GiB to GPU in 2.70675s (14.039GiB/s) | 20260507 19:25:59.402 | INFO | Analyzing chunk frames = [1995264,1995776) | 20260507 19:25:59.402 | INFO | MemInfo: { | 20260507 19:25:59.402 | INFO | "gpu" : " 17 GB", | 20260507 19:25:59.402 | INFO | "huge" : "", | 20260507 19:25:59.403 | INFO | "huge_shared" : " 29 GB", | 20260507 19:25:59.403 | INFO | "malloc" : " 14 GB", | 20260507 19:25:59.403 | INFO | "pinned" : " 29 GB" | 20260507 19:25:59.403 | INFO | } | 20260507 19:25:59.403 | INFO | Perf report for the last 5.13995 seconds: | 20260507 19:25:59.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:25:59.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 19:25:59.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 19:25:59.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:25:59.403 | INFO | Uploaded 36GiB to GPU in 2.59168s (13.8906GiB/s) | 20260507 19:25:59.403 | INFO | Downloaded 34GiB to GPU in 2.46908s (13.7703GiB/s) | 20260507 19:26:04.541 | INFO | Analyzing chunk frames = [1995776,1996288) | 20260507 19:26:04.541 | INFO | MemInfo: { | 20260507 19:26:04.541 | INFO | "gpu" : " 17 GB", | 20260507 19:26:04.541 | INFO | "huge" : "", | 20260507 19:26:04.541 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:04.541 | INFO | "malloc" : " 14 GB", | 20260507 19:26:04.541 | INFO | "pinned" : " 29 GB" | 20260507 19:26:04.541 | INFO | } | 20260507 19:26:04.542 | INFO | Perf report for the last 5.13871 seconds: | 20260507 19:26:04.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:04.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:26:04.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 19:26:04.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:04.542 | INFO | Uploaded 36GiB to GPU in 2.509s (14.3483GiB/s) | 20260507 19:26:04.542 | INFO | Downloaded 34GiB to GPU in 2.47708s (13.7259GiB/s) | 20260507 19:26:09.680 | INFO | Analyzing chunk frames = [1996288,1996800) | 20260507 19:26:09.680 | INFO | MemInfo: { | 20260507 19:26:09.680 | INFO | "gpu" : " 17 GB", | 20260507 19:26:09.680 | INFO | "huge" : "", | 20260507 19:26:09.681 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:09.681 | INFO | "malloc" : " 14 GB", | 20260507 19:26:09.681 | INFO | "pinned" : " 29 GB" | 20260507 19:26:09.681 | INFO | } | 20260507 19:26:09.681 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:26:09.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:09.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260507 19:26:09.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 19:26:09.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:09.681 | INFO | Uploaded 36GiB to GPU in 2.51501s (14.3141GiB/s) | 20260507 19:26:09.681 | INFO | Downloaded 34GiB to GPU in 2.40528s (14.1355GiB/s) | 20260507 19:26:14.820 | INFO | Analyzing chunk frames = [1996800,1997312) | 20260507 19:26:14.820 | INFO | MemInfo: { | 20260507 19:26:14.820 | INFO | "gpu" : " 17 GB", | 20260507 19:26:14.820 | INFO | "huge" : "", | 20260507 19:26:14.820 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:14.820 | INFO | "malloc" : " 14 GB", | 20260507 19:26:14.820 | INFO | "pinned" : " 29 GB" | 20260507 19:26:14.820 | INFO | } | 20260507 19:26:14.820 | INFO | Perf report for the last 5.13928 seconds: | 20260507 19:26:14.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:14.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260507 19:26:14.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 19:26:14.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:14.820 | INFO | Uploaded 36GiB to GPU in 2.46144s (14.6256GiB/s) | 20260507 19:26:14.821 | INFO | Downloaded 34GiB to GPU in 2.49806s (13.6105GiB/s) | 20260507 19:26:19.960 | INFO | Analyzing chunk frames = [1997312,1997824) | 20260507 19:26:19.960 | INFO | MemInfo: { | 20260507 19:26:19.960 | INFO | "gpu" : " 17 GB", | 20260507 19:26:19.960 | INFO | "huge" : "", | 20260507 19:26:19.961 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:19.961 | INFO | "malloc" : " 14 GB", | 20260507 19:26:19.961 | INFO | "pinned" : " 29 GB" | 20260507 19:26:19.961 | INFO | } | 20260507 19:26:19.961 | INFO | Perf report for the last 5.14078 seconds: | 20260507 19:26:19.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:19.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:26:19.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 19:26:19.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:19.961 | INFO | Uploaded 36GiB to GPU in 2.53183s (14.219GiB/s) | 20260507 19:26:19.961 | INFO | Downloaded 34GiB to GPU in 2.41186s (14.097GiB/s) | 20260507 19:26:25.099 | INFO | Analyzing chunk frames = [1997824,1998336) | 20260507 19:26:25.099 | INFO | MemInfo: { | 20260507 19:26:25.099 | INFO | "gpu" : " 17 GB", | 20260507 19:26:25.099 | INFO | "huge" : "", | 20260507 19:26:25.099 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:25.099 | INFO | "malloc" : " 14 GB", | 20260507 19:26:25.099 | INFO | "pinned" : " 29 GB" | 20260507 19:26:25.099 | INFO | } | 20260507 19:26:25.099 | INFO | Perf report for the last 5.13837 seconds: | 20260507 19:26:25.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:25.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.099 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:26:25.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.100 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 19:26:25.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:25.100 | INFO | Uploaded 36GiB to GPU in 2.55941s (14.0657GiB/s) | 20260507 19:26:25.100 | INFO | Downloaded 34GiB to GPU in 2.43738s (13.9494GiB/s) | 20260507 19:26:30.239 | INFO | Analyzing chunk frames = [1998336,1998848) | 20260507 19:26:30.239 | INFO | MemInfo: { | 20260507 19:26:30.239 | INFO | "gpu" : " 17 GB", | 20260507 19:26:30.239 | INFO | "huge" : "", | 20260507 19:26:30.240 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:30.240 | INFO | "malloc" : " 14 GB", | 20260507 19:26:30.240 | INFO | "pinned" : " 29 GB" | 20260507 19:26:30.240 | INFO | } | 20260507 19:26:30.240 | INFO | Perf report for the last 5.14037 seconds: | 20260507 19:26:30.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:30.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:26:30.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 19:26:30.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:30.240 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:26:30.240 | INFO | Total mem usage: 16.4 GiB | 20260507 19:26:30.240 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:26:30.240 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.94%) | 20260507 19:26:30.240 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:26:30.240 | INFO | High Water Breakdown: | 20260507 19:26:30.240 | INFO | 60.1% : 9.13 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:26:30.240 | INFO | 39.8% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:26:30.240 | INFO | Uploaded 36GiB to GPU in 2.53566s (14.1975GiB/s) | 20260507 19:26:30.240 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:26:30.241 | INFO | ------------------------------------------ | 20260507 19:26:30.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:26:30.241 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:26:30.241 | INFO | Total mem usage: 26.5 GiB | 20260507 19:26:30.241 | INFO | Downloaded 34GiB to GPU in 2.41353s (14.0873GiB/s) | 20260507 19:26:30.241 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:26:30.241 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 19:26:30.241 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 19:26:30.241 | INFO | High Water Breakdown: | 20260507 19:26:30.241 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:26:30.241 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:26:30.241 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:26:30.241 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:26:30.241 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:26:30.241 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:26:30.241 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:26:30.241 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:26:30.241 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:26:30.241 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:26:30.241 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:26:30.241 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:26:30.241 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:26:30.241 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:26:30.241 | INFO | ------------------------------------------ | 20260507 19:26:30.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:26:30.241 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:26:30.241 | INFO | Total mem usage: 27.8 GiB | 20260507 19:26:30.241 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:26:30.241 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:26:30.241 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:26:30.241 | INFO | High Water Breakdown: | 20260507 19:26:30.241 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:26:30.241 | INFO | ------------------------------------------ | 20260507 19:26:30.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:26:30.241 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:26:30.241 | INFO | Total mem usage: 15.7 GiB | 20260507 19:26:30.241 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:26:30.241 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:26:30.241 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:26:30.241 | INFO | High Water Breakdown: | 20260507 19:26:30.241 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:26:30.241 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:26:30.241 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:26:30.241 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:26:30.241 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:26:30.241 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:26:30.241 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:26:30.241 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:26:30.241 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:26:30.241 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:26:30.241 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:26:30.241 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:26:30.241 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:26:30.241 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:26:30.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:26:30.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:26:30.241 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:26:30.241 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:26:30.241 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:26:30.241 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:26:30.241 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:26:30.241 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:26:30.241 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:26:30.241 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:26:30.242 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:26:30.242 | INFO | ------------------------------------------ | 20260507 19:26:30.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:26:30.242 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:26:30.242 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:26:30.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:26:30.242 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.41e+01 / 1.80e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.04e-01, 1.15e-02 / 5.80e-03 | 20260507 19:26:30.242 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.48e+00, 6.00e-01 / 2.49e-01 | 20260507 19:26:30.242 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.51e+00, 2.15e-01 / 1.11e-01 | 20260507 19:26:30.242 | INFO | PerfReport: 15.85%, DME -- 4.20e+01 / 4.52e+01, 4.00e+00 / 1.70e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 55.90%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.78e+00 / 1.71e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.53e+00 / 2.84e+00, 4.43e-01 / 2.36e-01 | 20260507 19:26:30.242 | INFO | PerfReport: 3.67%, Metrics -- 1.22e+00 / 1.40e+00, 3.06e-01 / 1.63e-01 | 20260507 19:26:30.242 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.82e-01 / 4.14e-01, 6.09e-02 / 3.40e-02 | 20260507 19:26:30.242 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:26:30.242 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:26:30.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:26:30.242 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 24.37%, Upload -- 2.61e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 24.14%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 30.97%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 2.07%, LockUpload -- 2.22e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 2.39%, LockDownload -- 2.56e+00 / 0.00e+00, 3.85e+00 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 16.03%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:26:30.242 | INFO | PerfReport: 0.02%, Misc -- 1.66e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260507 19:26:30.354 | INFO | Starting a round of Baz IO | 20260507 19:26:33.869 | INFO | Populating RTMetrics report | 20260507 19:26:33.870 | INFO | RTMetrics report populated | 20260507 19:26:33.870 | INFO | Opening tmp RTMetrics json file | 20260507 19:26:33.872 | INFO | Serializing RTMetrics to json | 20260507 19:26:33.880 | INFO | Serialization done, swapping tmp file now... | 20260507 19:26:33.884 | INFO | RTMetrics done | 20260507 19:26:33.977 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.623 seconds | 20260507 19:26:33.977 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.42% / 0.01% / 54.57% | 20260507 19:26:33.977 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.40785% / 1.58946e-05% / 3.97364e-05% | 20260507 19:26:33.977 | INFO | Baz IO overall: | 20260507 19:26:33.977 | INFO | BytesWritten : 902383275822 | 20260507 19:26:33.977 | INFO | NumEvents : 437515368031 | 20260507 19:26:35.377 | INFO | Analyzing chunk frames = [1998848,1999360) | 20260507 19:26:35.377 | INFO | MemInfo: { | 20260507 19:26:35.377 | INFO | "gpu" : " 17 GB", | 20260507 19:26:35.377 | INFO | "huge" : "", | 20260507 19:26:35.377 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:35.377 | INFO | "malloc" : " 14 GB", | 20260507 19:26:35.377 | INFO | "pinned" : " 29 GB" | 20260507 19:26:35.377 | INFO | } | 20260507 19:26:35.377 | INFO | Perf report for the last 5.13764 seconds: | 20260507 19:26:35.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:35.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:35.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:35.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:35.378 | INFO | ANALYSIS : 69.6%, 2.9, 104.9ms, 0 | 20260507 19:26:35.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:35.378 | INFO | RT_METRICS : 49.6%, 1.0, 26.0ms, 0 | 20260507 19:26:35.378 | INFO | PRE_HQ : 68.9%, 2.0, 70.1ms, 0 | 20260507 19:26:35.378 | INFO | BAZWRITER : 70.1%, 4.5, 162.1ms, 0 | 20260507 19:26:35.378 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 19:26:35.378 | INFO | Uploaded 36GiB to GPU in 2.55783s (14.0744GiB/s) | 20260507 19:26:35.378 | INFO | Downloaded 38GiB to GPU in 2.69982s (14.075GiB/s) | 20260507 19:26:40.516 | INFO | Analyzing chunk frames = [1999360,1999872) | 20260507 19:26:40.516 | INFO | MemInfo: { | 20260507 19:26:40.516 | INFO | "gpu" : " 17 GB", | 20260507 19:26:40.516 | INFO | "huge" : "", | 20260507 19:26:40.516 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:40.516 | INFO | "malloc" : " 14 GB", | 20260507 19:26:40.516 | INFO | "pinned" : " 29 GB" | 20260507 19:26:40.516 | INFO | } | 20260507 19:26:40.516 | INFO | Perf report for the last 5.13882 seconds: | 20260507 19:26:40.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:40.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.516 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 19:26:40.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.516 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:26:40.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:40.517 | INFO | Uploaded 36GiB to GPU in 2.58676s (13.917GiB/s) | 20260507 19:26:40.517 | INFO | Downloaded 34GiB to GPU in 2.47369s (13.7447GiB/s) | 20260507 19:26:45.656 | INFO | Analyzing chunk frames = [1999872,2000384) | 20260507 19:26:45.656 | INFO | MemInfo: { | 20260507 19:26:45.656 | INFO | "gpu" : " 17 GB", | 20260507 19:26:45.656 | INFO | "huge" : "", | 20260507 19:26:45.656 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:45.656 | INFO | "malloc" : " 14 GB", | 20260507 19:26:45.657 | INFO | "pinned" : " 29 GB" | 20260507 19:26:45.657 | INFO | } | 20260507 19:26:45.657 | INFO | Perf report for the last 5.14036 seconds: | 20260507 19:26:45.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:45.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 19:26:45.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:26:45.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:45.657 | INFO | Uploaded 36GiB to GPU in 2.59873s (13.8529GiB/s) | 20260507 19:26:45.657 | INFO | Downloaded 34GiB to GPU in 2.45346s (13.858GiB/s) | 20260507 19:26:50.796 | INFO | Analyzing chunk frames = [2000384,2000896) | 20260507 19:26:50.796 | INFO | MemInfo: { | 20260507 19:26:50.796 | INFO | "gpu" : " 17 GB", | 20260507 19:26:50.796 | INFO | "huge" : "", | 20260507 19:26:50.797 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:50.797 | INFO | "malloc" : " 14 GB", | 20260507 19:26:50.797 | INFO | "pinned" : " 29 GB" | 20260507 19:26:50.797 | INFO | } | 20260507 19:26:50.797 | INFO | Perf report for the last 5.14007 seconds: | 20260507 19:26:50.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:50.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:26:50.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:26:50.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:50.797 | INFO | Uploaded 36GiB to GPU in 2.48582s (14.4821GiB/s) | 20260507 19:26:50.797 | INFO | Downloaded 34GiB to GPU in 2.38025s (14.2842GiB/s) | 20260507 19:26:55.935 | INFO | Analyzing chunk frames = [2000896,2001408) | 20260507 19:26:55.935 | INFO | MemInfo: { | 20260507 19:26:55.935 | INFO | "gpu" : " 17 GB", | 20260507 19:26:55.935 | INFO | "huge" : "", | 20260507 19:26:55.936 | INFO | "huge_shared" : " 29 GB", | 20260507 19:26:55.936 | INFO | "malloc" : " 14 GB", | 20260507 19:26:55.936 | INFO | "pinned" : " 29 GB" | 20260507 19:26:55.936 | INFO | } | 20260507 19:26:55.936 | INFO | Perf report for the last 5.1392 seconds: | 20260507 19:26:55.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:26:55.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 19:26:55.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 19:26:55.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:26:55.936 | INFO | Uploaded 36GiB to GPU in 2.43579s (14.7796GiB/s) | 20260507 19:26:55.936 | INFO | Downloaded 34GiB to GPU in 2.48902s (13.66GiB/s) | 20260507 19:27:01.074 | INFO | Analyzing chunk frames = [2001408,2001920) | 20260507 19:27:01.074 | INFO | MemInfo: { | 20260507 19:27:01.074 | INFO | "gpu" : " 17 GB", | 20260507 19:27:01.074 | INFO | "huge" : "", | 20260507 19:27:01.074 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:01.074 | INFO | "malloc" : " 14 GB", | 20260507 19:27:01.074 | INFO | "pinned" : " 29 GB" | 20260507 19:27:01.074 | INFO | } | 20260507 19:27:01.074 | INFO | Perf report for the last 5.13842 seconds: | 20260507 19:27:01.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:01.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.074 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:27:01.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.074 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 19:27:01.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:01.075 | INFO | Uploaded 36GiB to GPU in 2.50808s (14.3536GiB/s) | 20260507 19:27:01.075 | INFO | Downloaded 34GiB to GPU in 2.41283s (14.0913GiB/s) | 20260507 19:27:06.213 | INFO | Analyzing chunk frames = [2001920,2002432) | 20260507 19:27:06.213 | INFO | MemInfo: { | 20260507 19:27:06.213 | INFO | "gpu" : " 17 GB", | 20260507 19:27:06.213 | INFO | "huge" : "", | 20260507 19:27:06.213 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:06.213 | INFO | "malloc" : " 14 GB", | 20260507 19:27:06.213 | INFO | "pinned" : " 29 GB" | 20260507 19:27:06.213 | INFO | } | 20260507 19:27:06.214 | INFO | Perf report for the last 5.13922 seconds: | 20260507 19:27:06.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:06.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:27:06.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:27:06.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:06.214 | INFO | Uploaded 36GiB to GPU in 2.55851s (14.0707GiB/s) | 20260507 19:27:06.214 | INFO | Downloaded 34GiB to GPU in 2.44037s (13.9323GiB/s) | 20260507 19:27:11.352 | INFO | Analyzing chunk frames = [2002432,2002944) | 20260507 19:27:11.352 | INFO | MemInfo: { | 20260507 19:27:11.352 | INFO | "gpu" : " 17 GB", | 20260507 19:27:11.352 | INFO | "huge" : "", | 20260507 19:27:11.352 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:11.352 | INFO | "malloc" : " 14 GB", | 20260507 19:27:11.352 | INFO | "pinned" : " 29 GB" | 20260507 19:27:11.352 | INFO | } | 20260507 19:27:11.352 | INFO | Perf report for the last 5.13888 seconds: | 20260507 19:27:11.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:11.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.352 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:27:11.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.353 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:27:11.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:11.353 | INFO | Uploaded 36GiB to GPU in 2.52565s (14.2537GiB/s) | 20260507 19:27:11.353 | INFO | Downloaded 34GiB to GPU in 2.41413s (14.0837GiB/s) | 20260507 19:27:11.450 | INFO | Starting a round of Baz IO | 20260507 19:27:14.942 | INFO | Populating RTMetrics report | 20260507 19:27:14.943 | INFO | RTMetrics report populated | 20260507 19:27:14.943 | INFO | Opening tmp RTMetrics json file | 20260507 19:27:14.948 | INFO | Serializing RTMetrics to json | 20260507 19:27:14.970 | INFO | Serialization done, swapping tmp file now... | 20260507 19:27:14.971 | INFO | RTMetrics done | 20260507 19:27:15.042 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.592 seconds | 20260507 19:27:15.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.41% / 0.01% / 54.58% | 20260507 19:27:15.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.40146% / 7.94729e-06% / 3.17891e-05% | 20260507 19:27:15.042 | INFO | Baz IO overall: | 20260507 19:27:15.042 | INFO | BytesWritten : 904304681086 | 20260507 19:27:15.042 | INFO | NumEvents : 438446268367 | 20260507 19:27:16.492 | INFO | Analyzing chunk frames = [2002944,2003456) | 20260507 19:27:16.493 | INFO | MemInfo: { | 20260507 19:27:16.493 | INFO | "gpu" : " 17 GB", | 20260507 19:27:16.493 | INFO | "huge" : "", | 20260507 19:27:16.493 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:16.493 | INFO | "malloc" : " 14 GB", | 20260507 19:27:16.493 | INFO | "pinned" : " 29 GB" | 20260507 19:27:16.493 | INFO | } | 20260507 19:27:16.493 | INFO | Perf report for the last 5.14063 seconds: | 20260507 19:27:16.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:16.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:16.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:16.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:16.493 | INFO | ANALYSIS : 69.2%, 2.9, 106.9ms, 0 | 20260507 19:27:16.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:16.493 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 19:27:16.493 | INFO | PRE_HQ : 67.2%, 1.8, 63.6ms, 0 | 20260507 19:27:16.493 | INFO | BAZWRITER : 69.5%, 4.2, 148.6ms, 0 | 20260507 19:27:16.493 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:27:16.494 | INFO | Uploaded 36GiB to GPU in 2.52579s (14.253GiB/s) | 20260507 19:27:16.494 | INFO | Downloaded 38GiB to GPU in 2.69619s (14.094GiB/s) | 20260507 19:27:21.631 | INFO | Analyzing chunk frames = [2003456,2003968) | 20260507 19:27:21.631 | INFO | MemInfo: { | 20260507 19:27:21.631 | INFO | "gpu" : " 17 GB", | 20260507 19:27:21.631 | INFO | "huge" : "", | 20260507 19:27:21.631 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:21.632 | INFO | "malloc" : " 14 GB", | 20260507 19:27:21.632 | INFO | "pinned" : " 29 GB" | 20260507 19:27:21.632 | INFO | } | 20260507 19:27:21.632 | INFO | Perf report for the last 5.13863 seconds: | 20260507 19:27:21.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:21.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:27:21.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 19:27:21.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:21.632 | INFO | Uploaded 36GiB to GPU in 2.52224s (14.273GiB/s) | 20260507 19:27:21.632 | INFO | Downloaded 34GiB to GPU in 2.48204s (13.6984GiB/s) | 20260507 19:27:26.770 | INFO | Analyzing chunk frames = [2003968,2004480) | 20260507 19:27:26.770 | INFO | MemInfo: { | 20260507 19:27:26.770 | INFO | "gpu" : " 17 GB", | 20260507 19:27:26.770 | INFO | "huge" : "", | 20260507 19:27:26.770 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:26.770 | INFO | "malloc" : " 14 GB", | 20260507 19:27:26.770 | INFO | "pinned" : " 29 GB" | 20260507 19:27:26.770 | INFO | } | 20260507 19:27:26.771 | INFO | Perf report for the last 5.13879 seconds: | 20260507 19:27:26.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:26.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 19:27:26.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 19:27:26.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:26.771 | INFO | Uploaded 36GiB to GPU in 2.59641s (13.8653GiB/s) | 20260507 19:27:26.771 | INFO | Downloaded 34GiB to GPU in 2.4654s (13.7909GiB/s) | 20260507 19:27:31.911 | INFO | Analyzing chunk frames = [2004480,2004992) | 20260507 19:27:31.911 | INFO | MemInfo: { | 20260507 19:27:31.911 | INFO | "gpu" : " 17 GB", | 20260507 19:27:31.911 | INFO | "huge" : "", | 20260507 19:27:31.911 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:31.911 | INFO | "malloc" : " 14 GB", | 20260507 19:27:31.911 | INFO | "pinned" : " 29 GB" | 20260507 19:27:31.911 | INFO | } | 20260507 19:27:31.911 | INFO | Perf report for the last 5.14076 seconds: | 20260507 19:27:31.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:31.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.911 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:27:31.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.911 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 19:27:31.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:31.912 | INFO | Uploaded 36GiB to GPU in 2.48888s (14.4643GiB/s) | 20260507 19:27:31.912 | INFO | Downloaded 34GiB to GPU in 2.39793s (14.1789GiB/s) | 20260507 19:27:37.052 | INFO | Analyzing chunk frames = [2004992,2005504) | 20260507 19:27:37.052 | INFO | MemInfo: { | 20260507 19:27:37.052 | INFO | "gpu" : " 17 GB", | 20260507 19:27:37.052 | INFO | "huge" : "", | 20260507 19:27:37.052 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:37.052 | INFO | "malloc" : " 14 GB", | 20260507 19:27:37.052 | INFO | "pinned" : " 29 GB" | 20260507 19:27:37.052 | INFO | } | 20260507 19:27:37.052 | INFO | Perf report for the last 5.14058 seconds: | 20260507 19:27:37.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:37.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260507 19:27:37.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 19:27:37.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:37.052 | INFO | Uploaded 36GiB to GPU in 2.47168s (14.565GiB/s) | 20260507 19:27:37.052 | INFO | Downloaded 34GiB to GPU in 2.48504s (13.6819GiB/s) | 20260507 19:27:42.190 | INFO | Analyzing chunk frames = [2005504,2006016) | 20260507 19:27:42.190 | INFO | MemInfo: { | 20260507 19:27:42.190 | INFO | "gpu" : " 17 GB", | 20260507 19:27:42.190 | INFO | "huge" : "", | 20260507 19:27:42.190 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:42.190 | INFO | "malloc" : " 14 GB", | 20260507 19:27:42.190 | INFO | "pinned" : " 29 GB" | 20260507 19:27:42.190 | INFO | } | 20260507 19:27:42.190 | INFO | Perf report for the last 5.13806 seconds: | 20260507 19:27:42.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:42.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:27:42.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 19:27:42.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:42.190 | INFO | Uploaded 36GiB to GPU in 2.51268s (14.3273GiB/s) | 20260507 19:27:42.190 | INFO | Downloaded 34GiB to GPU in 2.41588s (14.0735GiB/s) | 20260507 19:27:47.330 | INFO | Analyzing chunk frames = [2006016,2006528) | 20260507 19:27:47.330 | INFO | MemInfo: { | 20260507 19:27:47.330 | INFO | "gpu" : " 17 GB", | 20260507 19:27:47.330 | INFO | "huge" : "", | 20260507 19:27:47.330 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:47.330 | INFO | "malloc" : " 14 GB", | 20260507 19:27:47.330 | INFO | "pinned" : " 29 GB" | 20260507 19:27:47.330 | INFO | } | 20260507 19:27:47.331 | INFO | Perf report for the last 5.14057 seconds: | 20260507 19:27:47.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:47.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:27:47.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:27:47.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:47.331 | INFO | Uploaded 36GiB to GPU in 2.54104s (14.1674GiB/s) | 20260507 19:27:47.331 | INFO | Downloaded 34GiB to GPU in 2.43875s (13.9415GiB/s) | 20260507 19:27:52.468 | INFO | Analyzing chunk frames = [2006528,2007040) | 20260507 19:27:52.468 | INFO | MemInfo: { | 20260507 19:27:52.468 | INFO | "gpu" : " 17 GB", | 20260507 19:27:52.468 | INFO | "huge" : "", | 20260507 19:27:52.468 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:52.468 | INFO | "malloc" : " 14 GB", | 20260507 19:27:52.468 | INFO | "pinned" : " 29 GB" | 20260507 19:27:52.468 | INFO | } | 20260507 19:27:52.468 | INFO | Perf report for the last 5.13769 seconds: | 20260507 19:27:52.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:52.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.468 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:27:52.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.468 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:27:52.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:52.469 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:27:52.469 | INFO | Total mem usage: 16.4 GiB | 20260507 19:27:52.469 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:27:52.469 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 19:27:52.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:27:52.469 | INFO | High Water Breakdown: | 20260507 19:27:52.469 | INFO | 59.7% : 8.88 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:27:52.469 | INFO | 40.1% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:27:52.469 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:27:52.469 | INFO | ------------------------------------------ | 20260507 19:27:52.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:27:52.469 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:27:52.469 | INFO | Total mem usage: 26.5 GiB | 20260507 19:27:52.469 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:27:52.469 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 19:27:52.469 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:27:52.469 | INFO | High Water Breakdown: | 20260507 19:27:52.469 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:27:52.469 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:27:52.469 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:27:52.469 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:27:52.469 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:27:52.469 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:27:52.469 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:27:52.469 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:27:52.469 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:27:52.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:27:52.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:27:52.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:27:52.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:27:52.469 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:27:52.469 | INFO | ------------------------------------------ | 20260507 19:27:52.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:27:52.469 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:27:52.469 | INFO | Total mem usage: 27.8 GiB | 20260507 19:27:52.469 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:27:52.469 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:27:52.469 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:27:52.469 | INFO | High Water Breakdown: | 20260507 19:27:52.469 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:27:52.469 | INFO | ------------------------------------------ | 20260507 19:27:52.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:27:52.469 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:27:52.469 | INFO | Total mem usage: 15.7 GiB | 20260507 19:27:52.469 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:27:52.469 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:27:52.469 | INFO | Unused Cache: 858 MiB (Waste: 5.34%) | 20260507 19:27:52.469 | INFO | High Water Breakdown: | 20260507 19:27:52.469 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:27:52.469 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:27:52.469 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:27:52.469 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:27:52.469 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:27:52.469 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:27:52.469 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:27:52.469 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:27:52.469 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:27:52.469 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:27:52.469 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:27:52.469 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:27:52.469 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:27:52.469 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:27:52.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:27:52.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:27:52.469 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:27:52.469 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:27:52.469 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:27:52.469 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:27:52.469 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:27:52.469 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:27:52.469 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:27:52.469 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:27:52.469 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:27:52.469 | INFO | ------------------------------------------ | 20260507 19:27:52.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:27:52.469 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:27:52.469 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:27:52.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:27:52.469 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.42e+01 / 2.01e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 0.30%, AggregateStats -- 1.00e-01 / 1.04e-01, 1.24e-02 / 5.39e-03 | 20260507 19:27:52.469 | INFO | PerfReport: 12.07%, Baseline -- 4.01e+00 / 4.48e+00, 6.05e-01 / 2.72e-01 | 20260507 19:27:52.469 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.52e+00, 2.15e-01 / 8.54e-02 | 20260507 19:27:52.469 | INFO | PerfReport: 15.69%, DME -- 4.17e+01 / 4.47e+01, 3.81e+00 / 1.44e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.85e+00 / 1.78e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.51e+00 / 2.88e+00, 4.42e-01 / 2.11e-01 | 20260507 19:27:52.469 | INFO | PerfReport: 3.65%, Metrics -- 1.21e+00 / 1.43e+00, 3.10e-01 / 1.76e-01 | 20260507 19:27:52.469 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.81e-01 / 4.20e-01, 6.19e-02 / 3.68e-02 | 20260507 19:27:52.469 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:27:52.469 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:27:52.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:27:52.469 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 24.57%, Upload -- 2.63e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 24.04%, Download -- 2.58e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 30.99%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 2.23%, LockUpload -- 2.39e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 2.13%, LockDownload -- 2.29e+00 / 0.00e+00, 3.58e+00 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 16.03%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:27:52.469 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260507 19:27:52.469 | INFO | Uploaded 36GiB to GPU in 2.51819s (14.296GiB/s) | 20260507 19:27:52.469 | INFO | Downloaded 34GiB to GPU in 2.41593s (14.0732GiB/s) | 20260507 19:27:52.566 | INFO | Starting a round of Baz IO | 20260507 19:27:56.067 | INFO | Populating RTMetrics report | 20260507 19:27:56.068 | INFO | RTMetrics report populated | 20260507 19:27:56.068 | INFO | Opening tmp RTMetrics json file | 20260507 19:27:56.069 | INFO | Serializing RTMetrics to json | 20260507 19:27:56.071 | INFO | Serialization done, swapping tmp file now... | 20260507 19:27:56.071 | INFO | RTMetrics done | 20260507 19:27:56.181 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.615 seconds | 20260507 19:27:56.181 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.40% / 0.01% / 54.59% | 20260507 19:27:56.181 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.39463% / 1.19209e-05% / 3.57628e-05% | 20260507 19:27:56.181 | INFO | Baz IO overall: | 20260507 19:27:56.182 | INFO | BytesWritten : 906225969829 | 20260507 19:27:56.182 | INFO | NumEvents : 439377016194 | 20260507 19:27:57.608 | INFO | Analyzing chunk frames = [2007040,2007552) | 20260507 19:27:57.608 | INFO | MemInfo: { | 20260507 19:27:57.608 | INFO | "gpu" : " 17 GB", | 20260507 19:27:57.608 | INFO | "huge" : "", | 20260507 19:27:57.608 | INFO | "huge_shared" : " 29 GB", | 20260507 19:27:57.608 | INFO | "malloc" : " 14 GB", | 20260507 19:27:57.608 | INFO | "pinned" : " 29 GB" | 20260507 19:27:57.608 | INFO | } | 20260507 19:27:57.608 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:27:57.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:27:57.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:57.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:57.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:57.608 | INFO | ANALYSIS : 69.0%, 3.0, 106.8ms, 0 | 20260507 19:27:57.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:27:57.608 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 19:27:57.608 | INFO | PRE_HQ : 66.7%, 1.7, 59.3ms, 0 | 20260507 19:27:57.608 | INFO | BAZWRITER : 69.9%, 4.2, 149.2ms, 0 | 20260507 19:27:57.608 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:27:57.608 | INFO | Uploaded 36GiB to GPU in 2.56878s (14.0144GiB/s) | 20260507 19:27:57.608 | INFO | Downloaded 38GiB to GPU in 2.69755s (14.0869GiB/s) | 20260507 19:28:02.752 | INFO | Analyzing chunk frames = [2007552,2008064) | 20260507 19:28:02.752 | INFO | MemInfo: { | 20260507 19:28:02.752 | INFO | "gpu" : " 17 GB", | 20260507 19:28:02.752 | INFO | "huge" : "", | 20260507 19:28:02.752 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:02.752 | INFO | "malloc" : " 14 GB", | 20260507 19:28:02.752 | INFO | "pinned" : " 29 GB" | 20260507 19:28:02.752 | INFO | } | 20260507 19:28:02.752 | INFO | Perf report for the last 5.14417 seconds: | 20260507 19:28:02.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:02.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260507 19:28:02.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:28:02.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:02.752 | INFO | Uploaded 36GiB to GPU in 2.58017s (13.9526GiB/s) | 20260507 19:28:02.752 | INFO | Downloaded 34GiB to GPU in 2.48434s (13.6857GiB/s) | 20260507 19:28:07.887 | INFO | Analyzing chunk frames = [2008064,2008576) | 20260507 19:28:07.887 | INFO | MemInfo: { | 20260507 19:28:07.887 | INFO | "gpu" : " 17 GB", | 20260507 19:28:07.887 | INFO | "huge" : "", | 20260507 19:28:07.887 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:07.887 | INFO | "malloc" : " 14 GB", | 20260507 19:28:07.887 | INFO | "pinned" : " 29 GB" | 20260507 19:28:07.887 | INFO | } | 20260507 19:28:07.887 | INFO | Perf report for the last 5.13544 seconds: | 20260507 19:28:07.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:07.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.887 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:28:07.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.887 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:28:07.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:07.888 | INFO | Uploaded 36GiB to GPU in 2.5244s (14.2608GiB/s) | 20260507 19:28:07.888 | INFO | Downloaded 34GiB to GPU in 2.47711s (13.7257GiB/s) | 20260507 19:28:13.029 | INFO | Analyzing chunk frames = [2008576,2009088) | 20260507 19:28:13.029 | INFO | MemInfo: { | 20260507 19:28:13.029 | INFO | "gpu" : " 17 GB", | 20260507 19:28:13.029 | INFO | "huge" : "", | 20260507 19:28:13.030 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:13.030 | INFO | "malloc" : " 14 GB", | 20260507 19:28:13.030 | INFO | "pinned" : " 29 GB" | 20260507 19:28:13.030 | INFO | } | 20260507 19:28:13.030 | INFO | Perf report for the last 5.14232 seconds: | 20260507 19:28:13.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:13.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:28:13.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:28:13.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:13.030 | INFO | Uploaded 36GiB to GPU in 2.47555s (14.5422GiB/s) | 20260507 19:28:13.030 | INFO | Downloaded 34GiB to GPU in 2.38032s (14.2838GiB/s) | 20260507 19:28:18.165 | INFO | Analyzing chunk frames = [2009088,2009600) | 20260507 19:28:18.165 | INFO | MemInfo: { | 20260507 19:28:18.165 | INFO | "gpu" : " 17 GB", | 20260507 19:28:18.165 | INFO | "huge" : "", | 20260507 19:28:18.165 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:18.166 | INFO | "malloc" : " 14 GB", | 20260507 19:28:18.166 | INFO | "pinned" : " 29 GB" | 20260507 19:28:18.166 | INFO | } | 20260507 19:28:18.166 | INFO | Perf report for the last 5.13595 seconds: | 20260507 19:28:18.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:18.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260507 19:28:18.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 19:28:18.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:18.166 | INFO | Uploaded 36GiB to GPU in 2.47189s (14.5637GiB/s) | 20260507 19:28:18.166 | INFO | Downloaded 34GiB to GPU in 2.50287s (13.5844GiB/s) | 20260507 19:28:23.304 | INFO | Analyzing chunk frames = [2009600,2010112) | 20260507 19:28:23.304 | INFO | MemInfo: { | 20260507 19:28:23.304 | INFO | "gpu" : " 17 GB", | 20260507 19:28:23.304 | INFO | "huge" : "", | 20260507 19:28:23.304 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:23.304 | INFO | "malloc" : " 14 GB", | 20260507 19:28:23.304 | INFO | "pinned" : " 29 GB" | 20260507 19:28:23.304 | INFO | } | 20260507 19:28:23.304 | INFO | Perf report for the last 5.13848 seconds: | 20260507 19:28:23.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:23.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.304 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:28:23.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.304 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:28:23.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:23.305 | INFO | Uploaded 36GiB to GPU in 2.51411s (14.3192GiB/s) | 20260507 19:28:23.305 | INFO | Downloaded 34GiB to GPU in 2.40915s (14.1129GiB/s) | 20260507 19:28:28.443 | INFO | Analyzing chunk frames = [2010112,2010624) | 20260507 19:28:28.443 | INFO | MemInfo: { | 20260507 19:28:28.443 | INFO | "gpu" : " 17 GB", | 20260507 19:28:28.443 | INFO | "huge" : "", | 20260507 19:28:28.443 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:28.443 | INFO | "malloc" : " 14 GB", | 20260507 19:28:28.443 | INFO | "pinned" : " 29 GB" | 20260507 19:28:28.443 | INFO | } | 20260507 19:28:28.443 | INFO | Perf report for the last 5.13932 seconds: | 20260507 19:28:28.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:28.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.443 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:28:28.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.444 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 19:28:28.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:28.444 | INFO | Uploaded 36GiB to GPU in 2.54984s (14.1185GiB/s) | 20260507 19:28:28.444 | INFO | Downloaded 34GiB to GPU in 2.44132s (13.9269GiB/s) | 20260507 19:28:33.584 | INFO | Analyzing chunk frames = [2010624,2011136) | 20260507 19:28:33.584 | INFO | MemInfo: { | 20260507 19:28:33.584 | INFO | "gpu" : " 17 GB", | 20260507 19:28:33.584 | INFO | "huge" : "", | 20260507 19:28:33.584 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:33.584 | INFO | "malloc" : " 14 GB", | 20260507 19:28:33.584 | INFO | "pinned" : " 29 GB" | 20260507 19:28:33.584 | INFO | } | 20260507 19:28:33.584 | INFO | Perf report for the last 5.14099 seconds: | 20260507 19:28:33.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:33.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.585 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 19:28:33.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.585 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 19:28:33.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:33.585 | INFO | Uploaded 36GiB to GPU in 2.52068s (14.2819GiB/s) | 20260507 19:28:33.585 | INFO | Downloaded 34GiB to GPU in 2.40886s (14.1145GiB/s) | 20260507 19:28:33.682 | INFO | Starting a round of Baz IO | 20260507 19:28:37.160 | INFO | Populating RTMetrics report | 20260507 19:28:37.161 | INFO | RTMetrics report populated | 20260507 19:28:37.161 | INFO | Opening tmp RTMetrics json file | 20260507 19:28:37.183 | INFO | Serializing RTMetrics to json | 20260507 19:28:37.185 | INFO | Serialization done, swapping tmp file now... | 20260507 19:28:37.185 | INFO | RTMetrics done | 20260507 19:28:37.271 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.588 seconds | 20260507 19:28:37.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.39% / 0.01% / 54.60% | 20260507 19:28:37.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.39892% / 3.97364e-06% / 2.38419e-05% | 20260507 19:28:37.271 | INFO | Baz IO overall: | 20260507 19:28:37.271 | INFO | BytesWritten : 908149479169 | 20260507 19:28:37.271 | INFO | NumEvents : 440309502157 | 20260507 19:28:38.723 | INFO | Analyzing chunk frames = [2011136,2011648) | 20260507 19:28:38.723 | INFO | MemInfo: { | 20260507 19:28:38.723 | INFO | "gpu" : " 17 GB", | 20260507 19:28:38.723 | INFO | "huge" : "", | 20260507 19:28:38.723 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:38.723 | INFO | "malloc" : " 14 GB", | 20260507 19:28:38.723 | INFO | "pinned" : " 29 GB" | 20260507 19:28:38.723 | INFO | } | 20260507 19:28:38.723 | INFO | Perf report for the last 5.13868 seconds: | 20260507 19:28:38.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:38.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:38.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:38.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:38.723 | INFO | ANALYSIS : 69.1%, 2.9, 106.8ms, 0 | 20260507 19:28:38.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:38.723 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 19:28:38.723 | INFO | PRE_HQ : 66.6%, 1.7, 58.7ms, 0 | 20260507 19:28:38.723 | INFO | BAZWRITER : 69.4%, 4.0, 144.0ms, 0 | 20260507 19:28:38.723 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 19:28:38.724 | INFO | Uploaded 36GiB to GPU in 2.60112s (13.8402GiB/s) | 20260507 19:28:38.724 | INFO | Downloaded 38GiB to GPU in 2.70836s (14.0307GiB/s) | 20260507 19:28:43.861 | INFO | Analyzing chunk frames = [2011648,2012160) | 20260507 19:28:43.861 | INFO | MemInfo: { | 20260507 19:28:43.861 | INFO | "gpu" : " 17 GB", | 20260507 19:28:43.861 | INFO | "huge" : "", | 20260507 19:28:43.861 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:43.861 | INFO | "malloc" : " 14 GB", | 20260507 19:28:43.861 | INFO | "pinned" : " 29 GB" | 20260507 19:28:43.861 | INFO | } | 20260507 19:28:43.861 | INFO | Perf report for the last 5.13819 seconds: | 20260507 19:28:43.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:43.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 19:28:43.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:28:43.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:43.861 | INFO | Uploaded 36GiB to GPU in 2.58489s (13.9271GiB/s) | 20260507 19:28:43.862 | INFO | Downloaded 34GiB to GPU in 2.46316s (13.8034GiB/s) | 20260507 19:28:49.002 | INFO | Analyzing chunk frames = [2012160,2012672) | 20260507 19:28:49.002 | INFO | MemInfo: { | 20260507 19:28:49.002 | INFO | "gpu" : " 17 GB", | 20260507 19:28:49.002 | INFO | "huge" : "", | 20260507 19:28:49.002 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:49.002 | INFO | "malloc" : " 14 GB", | 20260507 19:28:49.002 | INFO | "pinned" : " 29 GB" | 20260507 19:28:49.002 | INFO | } | 20260507 19:28:49.002 | INFO | Perf report for the last 5.14111 seconds: | 20260507 19:28:49.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:49.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.003 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:28:49.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.003 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:28:49.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:49.003 | INFO | Uploaded 36GiB to GPU in 2.57345s (13.989GiB/s) | 20260507 19:28:49.003 | INFO | Downloaded 34GiB to GPU in 2.45203s (13.866GiB/s) | 20260507 19:28:54.142 | INFO | Analyzing chunk frames = [2012672,2013184) | 20260507 19:28:54.142 | INFO | MemInfo: { | 20260507 19:28:54.142 | INFO | "gpu" : " 17 GB", | 20260507 19:28:54.142 | INFO | "huge" : "", | 20260507 19:28:54.142 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:54.142 | INFO | "malloc" : " 14 GB", | 20260507 19:28:54.142 | INFO | "pinned" : " 29 GB" | 20260507 19:28:54.142 | INFO | } | 20260507 19:28:54.142 | INFO | Perf report for the last 5.13936 seconds: | 20260507 19:28:54.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:54.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:28:54.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:28:54.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:54.142 | INFO | Uploaded 36GiB to GPU in 2.49004s (14.4576GiB/s) | 20260507 19:28:54.142 | INFO | Downloaded 34GiB to GPU in 2.39068s (14.2219GiB/s) | 20260507 19:28:59.281 | INFO | Analyzing chunk frames = [2013184,2013696) | 20260507 19:28:59.282 | INFO | MemInfo: { | 20260507 19:28:59.282 | INFO | "gpu" : " 17 GB", | 20260507 19:28:59.282 | INFO | "huge" : "", | 20260507 19:28:59.282 | INFO | "huge_shared" : " 29 GB", | 20260507 19:28:59.282 | INFO | "malloc" : " 14 GB", | 20260507 19:28:59.282 | INFO | "pinned" : " 29 GB" | 20260507 19:28:59.282 | INFO | } | 20260507 19:28:59.282 | INFO | Perf report for the last 5.13991 seconds: | 20260507 19:28:59.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:28:59.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:28:59.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:28:59.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:28:59.282 | INFO | Uploaded 36GiB to GPU in 2.39807s (15.0121GiB/s) | 20260507 19:28:59.282 | INFO | Downloaded 34GiB to GPU in 2.481s (13.7041GiB/s) | 20260507 19:29:04.421 | INFO | Analyzing chunk frames = [2013696,2014208) | 20260507 19:29:04.421 | INFO | MemInfo: { | 20260507 19:29:04.421 | INFO | "gpu" : " 17 GB", | 20260507 19:29:04.421 | INFO | "huge" : "", | 20260507 19:29:04.421 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:04.421 | INFO | "malloc" : " 14 GB", | 20260507 19:29:04.421 | INFO | "pinned" : " 29 GB" | 20260507 19:29:04.421 | INFO | } | 20260507 19:29:04.421 | INFO | Perf report for the last 5.13904 seconds: | 20260507 19:29:04.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:04.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:29:04.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:29:04.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:04.421 | INFO | Uploaded 36GiB to GPU in 2.50646s (14.3629GiB/s) | 20260507 19:29:04.421 | INFO | Downloaded 34GiB to GPU in 2.41088s (14.1027GiB/s) | 20260507 19:29:09.559 | INFO | Analyzing chunk frames = [2014208,2014720) | 20260507 19:29:09.559 | INFO | MemInfo: { | 20260507 19:29:09.559 | INFO | "gpu" : " 17 GB", | 20260507 19:29:09.559 | INFO | "huge" : "", | 20260507 19:29:09.559 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:09.559 | INFO | "malloc" : " 14 GB", | 20260507 19:29:09.559 | INFO | "pinned" : " 29 GB" | 20260507 19:29:09.559 | INFO | } | 20260507 19:29:09.559 | INFO | Perf report for the last 5.13846 seconds: | 20260507 19:29:09.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:09.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.559 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:29:09.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.559 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260507 19:29:09.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:09.560 | INFO | Uploaded 36GiB to GPU in 2.5504s (14.1154GiB/s) | 20260507 19:29:09.560 | INFO | Downloaded 34GiB to GPU in 2.43556s (13.9599GiB/s) | 20260507 19:29:14.698 | INFO | Analyzing chunk frames = [2014720,2015232) | 20260507 19:29:14.698 | INFO | MemInfo: { | 20260507 19:29:14.698 | INFO | "gpu" : " 17 GB", | 20260507 19:29:14.698 | INFO | "huge" : "", | 20260507 19:29:14.698 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:14.699 | INFO | "malloc" : " 14 GB", | 20260507 19:29:14.699 | INFO | "pinned" : " 29 GB" | 20260507 19:29:14.699 | INFO | } | 20260507 19:29:14.699 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:29:14.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:14.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:29:14.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:29:14.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:14.699 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:29:14.699 | INFO | Total mem usage: 16.4 GiB | 20260507 19:29:14.699 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:29:14.699 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.9%) | 20260507 19:29:14.699 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:29:14.699 | INFO | High Water Breakdown: | 20260507 19:29:14.699 | INFO | 59.5% : 8.73 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:29:14.699 | INFO | 40.3% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:29:14.699 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:29:14.699 | INFO | ------------------------------------------ | 20260507 19:29:14.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:29:14.699 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:29:14.699 | INFO | Total mem usage: 26.5 GiB | 20260507 19:29:14.699 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:29:14.699 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:29:14.699 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:29:14.699 | INFO | High Water Breakdown: | 20260507 19:29:14.699 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:29:14.699 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:29:14.699 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:29:14.699 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:29:14.699 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:29:14.699 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:29:14.699 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:29:14.699 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:29:14.699 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:29:14.699 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:29:14.699 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:29:14.699 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:29:14.699 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:29:14.699 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:29:14.699 | INFO | ------------------------------------------ | 20260507 19:29:14.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:29:14.699 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:29:14.699 | INFO | Total mem usage: 27.8 GiB | 20260507 19:29:14.699 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:29:14.699 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:29:14.699 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:29:14.699 | INFO | High Water Breakdown: | 20260507 19:29:14.699 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:29:14.699 | INFO | ------------------------------------------ | 20260507 19:29:14.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:29:14.699 | INFO | Uploaded 36GiB to GPU in 2.53221s (14.2168GiB/s) | 20260507 19:29:14.699 | INFO | Downloaded 34GiB to GPU in 2.4146s (14.081GiB/s) | 20260507 19:29:14.699 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:29:14.700 | INFO | Total mem usage: 15.7 GiB | 20260507 19:29:14.700 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:29:14.700 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:29:14.700 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:29:14.700 | INFO | High Water Breakdown: | 20260507 19:29:14.700 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:29:14.700 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:29:14.700 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:29:14.700 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:29:14.700 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:29:14.700 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:29:14.700 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:29:14.700 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:29:14.700 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:29:14.700 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:29:14.700 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:29:14.700 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:29:14.700 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:29:14.700 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:29:14.700 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:29:14.700 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:29:14.700 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:29:14.700 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:29:14.700 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:29:14.700 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:29:14.700 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:29:14.700 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:29:14.700 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:29:14.700 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:29:14.700 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:29:14.700 | INFO | ------------------------------------------ | 20260507 19:29:14.700 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:29:14.700 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:29:14.700 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:29:14.700 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:29:14.700 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.42e+01 / 1.81e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.01e-01, 1.17e-02 / 5.10e-03 | 20260507 19:29:14.700 | INFO | PerfReport: 12.07%, Baseline -- 3.99e+00 / 4.50e+00, 5.97e-01 / 2.49e-01 | 20260507 19:29:14.700 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.34e-01 | 20260507 19:29:14.700 | INFO | PerfReport: 15.84%, DME -- 4.19e+01 / 4.51e+01, 4.01e+00 / 1.70e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.73e+00 / 1.58e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.53e+00 / 2.89e+00, 4.41e-01 / 1.87e-01 | 20260507 19:29:14.700 | INFO | PerfReport: 3.66%, Metrics -- 1.21e+00 / 1.43e+00, 3.01e-01 / 1.72e-01 | 20260507 19:29:14.700 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.78e-01 / 4.13e-01, 6.08e-02 / 3.68e-02 | 20260507 19:29:14.700 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:29:14.700 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:29:14.700 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:29:14.700 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 24.56%, Upload -- 2.63e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 24.02%, Download -- 2.58e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 30.86%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 2.32%, LockUpload -- 2.49e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 2.19%, LockDownload -- 2.35e+00 / 0.00e+00, 3.58e+00 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 16.03%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:29:14.700 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260507 19:29:14.797 | INFO | Starting a round of Baz IO | 20260507 19:29:18.263 | INFO | Populating RTMetrics report | 20260507 19:29:18.264 | INFO | RTMetrics report populated | 20260507 19:29:18.264 | INFO | Opening tmp RTMetrics json file | 20260507 19:29:18.301 | INFO | Serializing RTMetrics to json | 20260507 19:29:18.347 | INFO | Serialization done, swapping tmp file now... | 20260507 19:29:18.353 | INFO | RTMetrics done | 20260507 19:29:18.387 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.59 seconds | 20260507 19:29:18.387 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.37% / 0.01% / 54.61% | 20260507 19:29:18.387 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.39561% / 2.78155e-05% / 6.35783e-05% | 20260507 19:29:18.387 | INFO | Baz IO overall: | 20260507 19:29:18.387 | INFO | BytesWritten : 910073776624 | 20260507 19:29:18.387 | INFO | NumEvents : 441242515987 | 20260507 19:29:19.837 | INFO | Analyzing chunk frames = [2015232,2015744) | 20260507 19:29:19.837 | INFO | MemInfo: { | 20260507 19:29:19.837 | INFO | "gpu" : " 17 GB", | 20260507 19:29:19.837 | INFO | "huge" : "", | 20260507 19:29:19.837 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:19.837 | INFO | "malloc" : " 14 GB", | 20260507 19:29:19.837 | INFO | "pinned" : " 29 GB" | 20260507 19:29:19.837 | INFO | } | 20260507 19:29:19.838 | INFO | Perf report for the last 5.13891 seconds: | 20260507 19:29:19.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:19.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:19.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:19.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:19.838 | INFO | ANALYSIS : 68.9%, 3.0, 107.1ms, 0 | 20260507 19:29:19.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:19.838 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 19:29:19.838 | INFO | PRE_HQ : 66.3%, 1.7, 58.5ms, 0 | 20260507 19:29:19.838 | INFO | BAZWRITER : 69.5%, 4.2, 149.9ms, 0 | 20260507 19:29:19.838 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 19:29:19.838 | INFO | Uploaded 36GiB to GPU in 2.55543s (14.0876GiB/s) | 20260507 19:29:19.838 | INFO | Downloaded 38GiB to GPU in 2.68623s (14.1462GiB/s) | 20260507 19:29:24.977 | INFO | Analyzing chunk frames = [2015744,2016256) | 20260507 19:29:24.977 | INFO | MemInfo: { | 20260507 19:29:24.977 | INFO | "gpu" : " 17 GB", | 20260507 19:29:24.977 | INFO | "huge" : "", | 20260507 19:29:24.977 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:24.977 | INFO | "malloc" : " 14 GB", | 20260507 19:29:24.977 | INFO | "pinned" : " 29 GB" | 20260507 19:29:24.977 | INFO | } | 20260507 19:29:24.977 | INFO | Perf report for the last 5.1395 seconds: | 20260507 19:29:24.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:24.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:29:24.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:29:24.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:24.977 | INFO | Uploaded 36GiB to GPU in 2.57179s (13.9981GiB/s) | 20260507 19:29:24.977 | INFO | Downloaded 34GiB to GPU in 2.4592s (13.8256GiB/s) | 20260507 19:29:30.117 | INFO | Analyzing chunk frames = [2016256,2016768) | 20260507 19:29:30.117 | INFO | MemInfo: { | 20260507 19:29:30.117 | INFO | "gpu" : " 17 GB", | 20260507 19:29:30.117 | INFO | "huge" : "", | 20260507 19:29:30.117 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:30.117 | INFO | "malloc" : " 14 GB", | 20260507 19:29:30.117 | INFO | "pinned" : " 29 GB" | 20260507 19:29:30.117 | INFO | } | 20260507 19:29:30.117 | INFO | Perf report for the last 5.1399 seconds: | 20260507 19:29:30.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:30.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 19:29:30.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:29:30.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:30.117 | INFO | Uploaded 36GiB to GPU in 2.56307s (14.0457GiB/s) | 20260507 19:29:30.117 | INFO | Downloaded 34GiB to GPU in 2.44472s (13.9075GiB/s) | 20260507 19:29:35.255 | INFO | Analyzing chunk frames = [2016768,2017280) | 20260507 19:29:35.255 | INFO | MemInfo: { | 20260507 19:29:35.255 | INFO | "gpu" : " 17 GB", | 20260507 19:29:35.255 | INFO | "huge" : "", | 20260507 19:29:35.255 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:35.255 | INFO | "malloc" : " 14 GB", | 20260507 19:29:35.255 | INFO | "pinned" : " 29 GB" | 20260507 19:29:35.255 | INFO | } | 20260507 19:29:35.255 | INFO | Perf report for the last 5.13843 seconds: | 20260507 19:29:35.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:35.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.255 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:29:35.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.255 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:29:35.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:35.256 | INFO | Uploaded 36GiB to GPU in 2.49752s (14.4143GiB/s) | 20260507 19:29:35.256 | INFO | Downloaded 34GiB to GPU in 2.38424s (14.2603GiB/s) | 20260507 19:29:40.395 | INFO | Analyzing chunk frames = [2017280,2017792) | 20260507 19:29:40.395 | INFO | MemInfo: { | 20260507 19:29:40.395 | INFO | "gpu" : " 17 GB", | 20260507 19:29:40.395 | INFO | "huge" : "", | 20260507 19:29:40.396 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:40.396 | INFO | "malloc" : " 14 GB", | 20260507 19:29:40.396 | INFO | "pinned" : " 29 GB" | 20260507 19:29:40.396 | INFO | } | 20260507 19:29:40.396 | INFO | Perf report for the last 5.14026 seconds: | 20260507 19:29:40.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:40.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:29:40.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:29:40.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:40.396 | INFO | Uploaded 36GiB to GPU in 2.40501s (14.9688GiB/s) | 20260507 19:29:40.396 | INFO | Downloaded 34GiB to GPU in 2.48664s (13.6731GiB/s) | 20260507 19:29:45.534 | INFO | Analyzing chunk frames = [2017792,2018304) | 20260507 19:29:45.534 | INFO | MemInfo: { | 20260507 19:29:45.534 | INFO | "gpu" : " 17 GB", | 20260507 19:29:45.534 | INFO | "huge" : "", | 20260507 19:29:45.535 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:45.535 | INFO | "malloc" : " 14 GB", | 20260507 19:29:45.535 | INFO | "pinned" : " 29 GB" | 20260507 19:29:45.535 | INFO | } | 20260507 19:29:45.535 | INFO | Perf report for the last 5.13903 seconds: | 20260507 19:29:45.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:45.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.535 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:29:45.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.535 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 19:29:45.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:45.536 | INFO | Uploaded 36GiB to GPU in 2.51103s (14.3368GiB/s) | 20260507 19:29:45.536 | INFO | Downloaded 34GiB to GPU in 2.40946s (14.1111GiB/s) | 20260507 19:29:50.673 | INFO | Analyzing chunk frames = [2018304,2018816) | 20260507 19:29:50.673 | INFO | MemInfo: { | 20260507 19:29:50.673 | INFO | "gpu" : " 17 GB", | 20260507 19:29:50.673 | INFO | "huge" : "", | 20260507 19:29:50.673 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:50.673 | INFO | "malloc" : " 14 GB", | 20260507 19:29:50.673 | INFO | "pinned" : " 29 GB" | 20260507 19:29:50.673 | INFO | } | 20260507 19:29:50.674 | INFO | Perf report for the last 5.13879 seconds: | 20260507 19:29:50.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:50.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:29:50.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 19:29:50.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:50.674 | INFO | Uploaded 36GiB to GPU in 2.55428s (14.094GiB/s) | 20260507 19:29:50.674 | INFO | Downloaded 34GiB to GPU in 2.43908s (13.9397GiB/s) | 20260507 19:29:55.812 | INFO | Analyzing chunk frames = [2018816,2019328) | 20260507 19:29:55.812 | INFO | MemInfo: { | 20260507 19:29:55.812 | INFO | "gpu" : " 17 GB", | 20260507 19:29:55.813 | INFO | "huge" : "", | 20260507 19:29:55.813 | INFO | "huge_shared" : " 29 GB", | 20260507 19:29:55.813 | INFO | "malloc" : " 14 GB", | 20260507 19:29:55.813 | INFO | "pinned" : " 29 GB" | 20260507 19:29:55.813 | INFO | } | 20260507 19:29:55.813 | INFO | Perf report for the last 5.1394 seconds: | 20260507 19:29:55.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:29:55.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:29:55.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 19:29:55.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:29:55.813 | INFO | Uploaded 36GiB to GPU in 2.53506s (14.2008GiB/s) | 20260507 19:29:55.813 | INFO | Downloaded 34GiB to GPU in 2.41501s (14.0786GiB/s) | 20260507 19:29:55.926 | INFO | Starting a round of Baz IO | 20260507 19:29:59.421 | INFO | Populating RTMetrics report | 20260507 19:29:59.422 | INFO | RTMetrics report populated | 20260507 19:29:59.422 | INFO | Opening tmp RTMetrics json file | 20260507 19:29:59.441 | INFO | Serializing RTMetrics to json | 20260507 19:29:59.450 | INFO | Serialization done, swapping tmp file now... | 20260507 19:29:59.450 | INFO | RTMetrics done | 20260507 19:29:59.523 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.597 seconds | 20260507 19:29:59.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.36% / 0.01% / 54.63% | 20260507 19:29:59.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.3827% / 2.38419e-05% / 5.96046e-05% | 20260507 19:29:59.523 | INFO | Baz IO overall: | 20260507 19:29:59.523 | INFO | BytesWritten : 911996313644 | 20260507 19:29:59.523 | INFO | NumEvents : 442174463641 | 20260507 19:30:00.953 | INFO | Analyzing chunk frames = [2019328,2019840) | 20260507 19:30:00.953 | INFO | MemInfo: { | 20260507 19:30:00.953 | INFO | "gpu" : " 17 GB", | 20260507 19:30:00.953 | INFO | "huge" : "", | 20260507 19:30:00.953 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:00.953 | INFO | "malloc" : " 14 GB", | 20260507 19:30:00.953 | INFO | "pinned" : " 29 GB" | 20260507 19:30:00.953 | INFO | } | 20260507 19:30:00.953 | INFO | Perf report for the last 5.14006 seconds: | 20260507 19:30:00.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:00.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:00.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:00.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:00.953 | INFO | ANALYSIS : 69.4%, 2.9, 104.8ms, 0 | 20260507 19:30:00.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:00.953 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260507 19:30:00.953 | INFO | PRE_HQ : 68.6%, 1.8, 64.9ms, 0 | 20260507 19:30:00.953 | INFO | BAZWRITER : 69.6%, 4.7, 168.2ms, 0 | 20260507 19:30:00.953 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:30:00.954 | INFO | Uploaded 36GiB to GPU in 2.5639s (14.0411GiB/s) | 20260507 19:30:00.954 | INFO | Downloaded 38GiB to GPU in 2.71172s (14.0132GiB/s) | 20260507 19:30:06.092 | INFO | Analyzing chunk frames = [2019840,2020352) | 20260507 19:30:06.092 | INFO | MemInfo: { | 20260507 19:30:06.092 | INFO | "gpu" : " 17 GB", | 20260507 19:30:06.092 | INFO | "huge" : "", | 20260507 19:30:06.092 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:06.092 | INFO | "malloc" : " 14 GB", | 20260507 19:30:06.092 | INFO | "pinned" : " 29 GB" | 20260507 19:30:06.092 | INFO | } | 20260507 19:30:06.093 | INFO | Perf report for the last 5.13949 seconds: | 20260507 19:30:06.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:06.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 19:30:06.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 19:30:06.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:06.093 | INFO | Uploaded 36GiB to GPU in 2.58822s (13.9092GiB/s) | 20260507 19:30:06.093 | INFO | Downloaded 34GiB to GPU in 2.46613s (13.7868GiB/s) | 20260507 19:30:11.231 | INFO | Analyzing chunk frames = [2020352,2020864) | 20260507 19:30:11.231 | INFO | MemInfo: { | 20260507 19:30:11.231 | INFO | "gpu" : " 17 GB", | 20260507 19:30:11.231 | INFO | "huge" : "", | 20260507 19:30:11.231 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:11.231 | INFO | "malloc" : " 14 GB", | 20260507 19:30:11.231 | INFO | "pinned" : " 29 GB" | 20260507 19:30:11.231 | INFO | } | 20260507 19:30:11.231 | INFO | Perf report for the last 5.13887 seconds: | 20260507 19:30:11.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:11.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.232 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:30:11.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.232 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:30:11.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:11.232 | INFO | Uploaded 36GiB to GPU in 2.59901s (13.8514GiB/s) | 20260507 19:30:11.232 | INFO | Downloaded 34GiB to GPU in 2.46228s (13.8083GiB/s) | 20260507 19:30:16.370 | INFO | Analyzing chunk frames = [2020864,2021376) | 20260507 19:30:16.370 | INFO | MemInfo: { | 20260507 19:30:16.370 | INFO | "gpu" : " 17 GB", | 20260507 19:30:16.370 | INFO | "huge" : "", | 20260507 19:30:16.370 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:16.370 | INFO | "malloc" : " 14 GB", | 20260507 19:30:16.370 | INFO | "pinned" : " 29 GB" | 20260507 19:30:16.370 | INFO | } | 20260507 19:30:16.370 | INFO | Perf report for the last 5.13877 seconds: | 20260507 19:30:16.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:16.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.370 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 19:30:16.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.370 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:30:16.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:16.371 | INFO | Uploaded 36GiB to GPU in 2.48449s (14.4899GiB/s) | 20260507 19:30:16.371 | INFO | Downloaded 34GiB to GPU in 2.38373s (14.2633GiB/s) | 20260507 19:30:21.509 | INFO | Analyzing chunk frames = [2021376,2021888) | 20260507 19:30:21.509 | INFO | MemInfo: { | 20260507 19:30:21.509 | INFO | "gpu" : " 17 GB", | 20260507 19:30:21.509 | INFO | "huge" : "", | 20260507 19:30:21.509 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:21.509 | INFO | "malloc" : " 14 GB", | 20260507 19:30:21.509 | INFO | "pinned" : " 29 GB" | 20260507 19:30:21.509 | INFO | } | 20260507 19:30:21.509 | INFO | Perf report for the last 5.13915 seconds: | 20260507 19:30:21.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:21.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 19:30:21.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:30:21.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:21.510 | INFO | Uploaded 36GiB to GPU in 2.45501s (14.6639GiB/s) | 20260507 19:30:21.510 | INFO | Downloaded 34GiB to GPU in 2.50444s (13.5759GiB/s) | 20260507 19:30:26.648 | INFO | Analyzing chunk frames = [2021888,2022400) | 20260507 19:30:26.648 | INFO | MemInfo: { | 20260507 19:30:26.648 | INFO | "gpu" : " 17 GB", | 20260507 19:30:26.648 | INFO | "huge" : "", | 20260507 19:30:26.648 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:26.648 | INFO | "malloc" : " 14 GB", | 20260507 19:30:26.648 | INFO | "pinned" : " 29 GB" | 20260507 19:30:26.648 | INFO | } | 20260507 19:30:26.648 | INFO | Perf report for the last 5.13892 seconds: | 20260507 19:30:26.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:26.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.648 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:30:26.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.648 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:30:26.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:26.649 | INFO | Uploaded 36GiB to GPU in 2.52744s (14.2437GiB/s) | 20260507 19:30:26.649 | INFO | Downloaded 34GiB to GPU in 2.40751s (14.1225GiB/s) | 20260507 19:30:31.789 | INFO | Analyzing chunk frames = [2022400,2022912) | 20260507 19:30:31.789 | INFO | MemInfo: { | 20260507 19:30:31.789 | INFO | "gpu" : " 17 GB", | 20260507 19:30:31.789 | INFO | "huge" : "", | 20260507 19:30:31.789 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:31.789 | INFO | "malloc" : " 14 GB", | 20260507 19:30:31.789 | INFO | "pinned" : " 29 GB" | 20260507 19:30:31.789 | INFO | } | 20260507 19:30:31.789 | INFO | Perf report for the last 5.14084 seconds: | 20260507 19:30:31.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:31.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.789 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 19:30:31.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.789 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 19:30:31.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:31.790 | INFO | Uploaded 36GiB to GPU in 2.53499s (14.2012GiB/s) | 20260507 19:30:31.790 | INFO | Downloaded 34GiB to GPU in 2.42898s (13.9977GiB/s) | 20260507 19:30:36.927 | INFO | Analyzing chunk frames = [2022912,2023424) | 20260507 19:30:36.927 | INFO | MemInfo: { | 20260507 19:30:36.927 | INFO | "gpu" : " 17 GB", | 20260507 19:30:36.927 | INFO | "huge" : "", | 20260507 19:30:36.928 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:36.928 | INFO | "malloc" : " 14 GB", | 20260507 19:30:36.928 | INFO | "pinned" : " 29 GB" | 20260507 19:30:36.928 | INFO | } | 20260507 19:30:36.928 | INFO | Perf report for the last 5.13863 seconds: | 20260507 19:30:36.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:36.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:30:36.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:30:36.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:36.928 | INFO | Uploaded 36GiB to GPU in 2.51939s (14.2892GiB/s) | 20260507 19:30:36.928 | INFO | Downloaded 34GiB to GPU in 2.41753s (14.064GiB/s) | 20260507 19:30:36.928 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:30:36.928 | INFO | Total mem usage: 16.4 GiB | 20260507 19:30:36.928 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:30:36.928 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -4.95%) | 20260507 19:30:36.928 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:30:36.928 | INFO | High Water Breakdown: | 20260507 19:30:36.928 | INFO | 60.6% : 9.47 GiB ( 95.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:30:36.928 | INFO | 39.3% : 6.14 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:30:36.928 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:30:36.928 | INFO | ------------------------------------------ | 20260507 19:30:36.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:30:36.928 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:30:36.928 | INFO | Total mem usage: 26.5 GiB | 20260507 19:30:36.928 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:30:36.928 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:30:36.928 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:30:36.928 | INFO | High Water Breakdown: | 20260507 19:30:36.928 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:30:36.928 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:30:36.928 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:30:36.928 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:30:36.928 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:30:36.928 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:30:36.928 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:30:36.928 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:30:36.928 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:30:36.928 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:30:36.928 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:30:36.928 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:30:36.928 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:30:36.928 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:30:36.928 | INFO | ------------------------------------------ | 20260507 19:30:36.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:30:36.928 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:30:36.928 | INFO | Total mem usage: 27.8 GiB | 20260507 19:30:36.928 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:30:36.928 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:30:36.928 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:30:36.928 | INFO | High Water Breakdown: | 20260507 19:30:36.928 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:30:36.928 | INFO | ------------------------------------------ | 20260507 19:30:36.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:30:36.928 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:30:36.928 | INFO | Total mem usage: 15.7 GiB | 20260507 19:30:36.928 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:30:36.928 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:30:36.928 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:30:36.928 | INFO | High Water Breakdown: | 20260507 19:30:36.928 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:30:36.928 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:30:36.928 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:30:36.928 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:30:36.928 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:30:36.928 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:30:36.928 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:30:36.928 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:30:36.928 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:30:36.928 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:30:36.928 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:30:36.928 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:30:36.928 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:30:36.928 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:30:36.928 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:30:36.928 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:30:36.928 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:30:36.928 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:30:36.928 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:30:36.928 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:30:36.928 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:30:36.928 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:30:36.928 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:30:36.928 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:30:36.928 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:30:36.928 | INFO | ------------------------------------------ | 20260507 19:30:36.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:30:36.928 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:30:36.928 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:30:36.928 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:30:36.928 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.42e+01 / 1.76e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.04e-01, 1.18e-02 / 7.62e-03 | 20260507 19:30:36.928 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.45e+00, 6.03e-01 / 3.33e-01 | 20260507 19:30:36.928 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.48e+00, 2.13e-01 / 1.42e-01 | 20260507 19:30:36.928 | INFO | PerfReport: 15.74%, DME -- 4.18e+01 / 4.49e+01, 3.85e+00 / 1.37e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.84e+00 / 1.87e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.42e-01 / 2.63e-01 | 20260507 19:30:36.928 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.40e+00, 3.07e-01 / 1.83e-01 | 20260507 19:30:36.928 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.79e-01 / 4.11e-01, 6.12e-02 / 3.56e-02 | 20260507 19:30:36.928 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:30:36.928 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:30:36.928 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:30:36.928 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 24.60%, Upload -- 2.63e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 24.02%, Download -- 2.57e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 30.98%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 2.25%, LockUpload -- 2.41e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 2.09%, LockDownload -- 2.24e+00 / 0.00e+00, 3.51e+00 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 16.04%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:30:36.928 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260507 19:30:37.025 | INFO | Starting a round of Baz IO | 20260507 19:30:40.514 | INFO | Populating RTMetrics report | 20260507 19:30:40.514 | INFO | RTMetrics report populated | 20260507 19:30:40.514 | INFO | Opening tmp RTMetrics json file | 20260507 19:30:40.515 | INFO | Serializing RTMetrics to json | 20260507 19:30:40.539 | INFO | Serialization done, swapping tmp file now... | 20260507 19:30:40.541 | INFO | RTMetrics done | 20260507 19:30:40.621 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.596 seconds | 20260507 19:30:40.622 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.35% / 0.01% / 54.64% | 20260507 19:30:40.622 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.37431% / 1.98682e-05% / 3.97364e-05% | 20260507 19:30:40.622 | INFO | Baz IO overall: | 20260507 19:30:40.622 | INFO | BytesWritten : 913917589563 | 20260507 19:30:40.622 | INFO | NumEvents : 443105387470 | 20260507 19:30:42.067 | INFO | Analyzing chunk frames = [2023424,2023936) | 20260507 19:30:42.067 | INFO | MemInfo: { | 20260507 19:30:42.067 | INFO | "gpu" : " 17 GB", | 20260507 19:30:42.067 | INFO | "huge" : "", | 20260507 19:30:42.067 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:42.067 | INFO | "malloc" : " 14 GB", | 20260507 19:30:42.067 | INFO | "pinned" : " 29 GB" | 20260507 19:30:42.067 | INFO | } | 20260507 19:30:42.067 | INFO | Perf report for the last 5.13913 seconds: | 20260507 19:30:42.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:42.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:42.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:42.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:42.067 | INFO | ANALYSIS : 69.1%, 2.9, 106.9ms, 0 | 20260507 19:30:42.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:42.067 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 19:30:42.067 | INFO | PRE_HQ : 66.4%, 1.7, 58.9ms, 0 | 20260507 19:30:42.067 | INFO | BAZWRITER : 69.6%, 3.9, 138.6ms, 0 | 20260507 19:30:42.067 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:30:42.067 | INFO | Uploaded 36GiB to GPU in 2.50805s (14.3538GiB/s) | 20260507 19:30:42.067 | INFO | Downloaded 38GiB to GPU in 2.70125s (14.0676GiB/s) | 20260507 19:30:47.206 | INFO | Analyzing chunk frames = [2023936,2024448) | 20260507 19:30:47.206 | INFO | MemInfo: { | 20260507 19:30:47.206 | INFO | "gpu" : " 17 GB", | 20260507 19:30:47.206 | INFO | "huge" : "", | 20260507 19:30:47.206 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:47.206 | INFO | "malloc" : " 14 GB", | 20260507 19:30:47.206 | INFO | "pinned" : " 29 GB" | 20260507 19:30:47.207 | INFO | } | 20260507 19:30:47.207 | INFO | Perf report for the last 5.13978 seconds: | 20260507 19:30:47.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:47.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 19:30:47.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:30:47.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:47.207 | INFO | Uploaded 36GiB to GPU in 2.51463s (14.3162GiB/s) | 20260507 19:30:47.207 | INFO | Downloaded 34GiB to GPU in 2.44809s (13.8884GiB/s) | 20260507 19:30:52.345 | INFO | Analyzing chunk frames = [2024448,2024960) | 20260507 19:30:52.345 | INFO | MemInfo: { | 20260507 19:30:52.345 | INFO | "gpu" : " 17 GB", | 20260507 19:30:52.345 | INFO | "huge" : "", | 20260507 19:30:52.345 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:52.345 | INFO | "malloc" : " 14 GB", | 20260507 19:30:52.345 | INFO | "pinned" : " 29 GB" | 20260507 19:30:52.345 | INFO | } | 20260507 19:30:52.345 | INFO | Perf report for the last 5.13866 seconds: | 20260507 19:30:52.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:52.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.345 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:30:52.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.346 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:30:52.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:52.346 | INFO | Uploaded 36GiB to GPU in 2.56635s (14.0277GiB/s) | 20260507 19:30:52.346 | INFO | Downloaded 34GiB to GPU in 2.45867s (13.8286GiB/s) | 20260507 19:30:57.485 | INFO | Analyzing chunk frames = [2024960,2025472) | 20260507 19:30:57.485 | INFO | MemInfo: { | 20260507 19:30:57.485 | INFO | "gpu" : " 17 GB", | 20260507 19:30:57.485 | INFO | "huge" : "", | 20260507 19:30:57.485 | INFO | "huge_shared" : " 29 GB", | 20260507 19:30:57.485 | INFO | "malloc" : " 14 GB", | 20260507 19:30:57.485 | INFO | "pinned" : " 29 GB" | 20260507 19:30:57.485 | INFO | } | 20260507 19:30:57.485 | INFO | Perf report for the last 5.13945 seconds: | 20260507 19:30:57.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:30:57.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:30:57.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:30:57.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:30:57.485 | INFO | Uploaded 36GiB to GPU in 2.47599s (14.5396GiB/s) | 20260507 19:30:57.485 | INFO | Downloaded 34GiB to GPU in 2.38559s (14.2522GiB/s) | 20260507 19:31:02.624 | INFO | Analyzing chunk frames = [2025472,2025984) | 20260507 19:31:02.624 | INFO | MemInfo: { | 20260507 19:31:02.624 | INFO | "gpu" : " 17 GB", | 20260507 19:31:02.624 | INFO | "huge" : "", | 20260507 19:31:02.624 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:02.624 | INFO | "malloc" : " 14 GB", | 20260507 19:31:02.624 | INFO | "pinned" : " 29 GB" | 20260507 19:31:02.624 | INFO | } | 20260507 19:31:02.624 | INFO | Perf report for the last 5.1392 seconds: | 20260507 19:31:02.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:02.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.624 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 19:31:02.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.624 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:31:02.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:02.625 | INFO | Uploaded 36GiB to GPU in 2.40572s (14.9643GiB/s) | 20260507 19:31:02.625 | INFO | Downloaded 34GiB to GPU in 2.49609s (13.6213GiB/s) | 20260507 19:31:07.764 | INFO | Analyzing chunk frames = [2025984,2026496) | 20260507 19:31:07.764 | INFO | MemInfo: { | 20260507 19:31:07.764 | INFO | "gpu" : " 17 GB", | 20260507 19:31:07.764 | INFO | "huge" : "", | 20260507 19:31:07.764 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:07.764 | INFO | "malloc" : " 14 GB", | 20260507 19:31:07.764 | INFO | "pinned" : " 29 GB" | 20260507 19:31:07.764 | INFO | } | 20260507 19:31:07.764 | INFO | Perf report for the last 5.1398 seconds: | 20260507 19:31:07.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:07.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.764 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:31:07.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.764 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 19:31:07.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:07.765 | INFO | Uploaded 36GiB to GPU in 2.48939s (14.4614GiB/s) | 20260507 19:31:07.765 | INFO | Downloaded 34GiB to GPU in 2.40939s (14.1115GiB/s) | 20260507 19:31:12.903 | INFO | Analyzing chunk frames = [2026496,2027008) | 20260507 19:31:12.903 | INFO | MemInfo: { | 20260507 19:31:12.903 | INFO | "gpu" : " 17 GB", | 20260507 19:31:12.903 | INFO | "huge" : "", | 20260507 19:31:12.903 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:12.903 | INFO | "malloc" : " 14 GB", | 20260507 19:31:12.903 | INFO | "pinned" : " 29 GB" | 20260507 19:31:12.903 | INFO | } | 20260507 19:31:12.903 | INFO | Perf report for the last 5.13934 seconds: | 20260507 19:31:12.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:12.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.903 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:31:12.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.903 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:31:12.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:12.904 | INFO | Uploaded 36GiB to GPU in 2.54611s (14.1392GiB/s) | 20260507 19:31:12.904 | INFO | Downloaded 34GiB to GPU in 2.43995s (13.9347GiB/s) | 20260507 19:31:18.044 | INFO | Analyzing chunk frames = [2027008,2027520) | 20260507 19:31:18.044 | INFO | MemInfo: { | 20260507 19:31:18.044 | INFO | "gpu" : " 17 GB", | 20260507 19:31:18.044 | INFO | "huge" : "", | 20260507 19:31:18.044 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:18.044 | INFO | "malloc" : " 14 GB", | 20260507 19:31:18.045 | INFO | "pinned" : " 29 GB" | 20260507 19:31:18.045 | INFO | } | 20260507 19:31:18.045 | INFO | Perf report for the last 5.14141 seconds: | 20260507 19:31:18.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:18.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:31:18.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:31:18.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:18.045 | INFO | Uploaded 36GiB to GPU in 2.51227s (14.3297GiB/s) | 20260507 19:31:18.045 | INFO | Downloaded 34GiB to GPU in 2.42186s (14.0388GiB/s) | 20260507 19:31:18.143 | INFO | Starting a round of Baz IO | 20260507 19:31:21.615 | INFO | Populating RTMetrics report | 20260507 19:31:21.615 | INFO | RTMetrics report populated | 20260507 19:31:21.615 | INFO | Opening tmp RTMetrics json file | 20260507 19:31:21.621 | INFO | Serializing RTMetrics to json | 20260507 19:31:21.641 | INFO | Serialization done, swapping tmp file now... | 20260507 19:31:21.642 | INFO | RTMetrics done | 20260507 19:31:21.719 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.576 seconds | 20260507 19:31:21.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.34% / 0.01% / 54.65% | 20260507 19:31:21.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.37482% / 1.98682e-05% / 4.76837e-05% | 20260507 19:31:21.719 | INFO | Baz IO overall: | 20260507 19:31:21.719 | INFO | BytesWritten : 915839905894 | 20260507 19:31:21.719 | INFO | NumEvents : 444037009651 | 20260507 19:31:23.182 | INFO | Analyzing chunk frames = [2027520,2028032) | 20260507 19:31:23.182 | INFO | MemInfo: { | 20260507 19:31:23.182 | INFO | "gpu" : " 17 GB", | 20260507 19:31:23.182 | INFO | "huge" : "", | 20260507 19:31:23.182 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:23.182 | INFO | "malloc" : " 14 GB", | 20260507 19:31:23.182 | INFO | "pinned" : " 29 GB" | 20260507 19:31:23.182 | INFO | } | 20260507 19:31:23.182 | INFO | Perf report for the last 5.13774 seconds: | 20260507 19:31:23.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:23.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:23.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:23.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:23.183 | INFO | ANALYSIS : 69.0%, 3.0, 106.8ms, 0 | 20260507 19:31:23.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:23.183 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 19:31:23.183 | INFO | PRE_HQ : 66.5%, 1.7, 58.6ms, 0 | 20260507 19:31:23.183 | INFO | BAZWRITER : 69.2%, 4.3, 152.9ms, 0 | 20260507 19:31:23.183 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 19:31:23.183 | INFO | Uploaded 36GiB to GPU in 2.57493s (13.981GiB/s) | 20260507 19:31:23.183 | INFO | Downloaded 38GiB to GPU in 2.69925s (14.078GiB/s) | 20260507 19:31:28.320 | INFO | Analyzing chunk frames = [2028032,2028544) | 20260507 19:31:28.320 | INFO | MemInfo: { | 20260507 19:31:28.320 | INFO | "gpu" : " 17 GB", | 20260507 19:31:28.320 | INFO | "huge" : "", | 20260507 19:31:28.320 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:28.320 | INFO | "malloc" : " 14 GB", | 20260507 19:31:28.321 | INFO | "pinned" : " 29 GB" | 20260507 19:31:28.321 | INFO | } | 20260507 19:31:28.321 | INFO | Perf report for the last 5.13815 seconds: | 20260507 19:31:28.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:28.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:31:28.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:31:28.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:28.321 | INFO | Uploaded 36GiB to GPU in 2.56979s (14.0089GiB/s) | 20260507 19:31:28.321 | INFO | Downloaded 34GiB to GPU in 2.46308s (13.8038GiB/s) | 20260507 19:31:33.459 | INFO | Analyzing chunk frames = [2028544,2029056) | 20260507 19:31:33.459 | INFO | MemInfo: { | 20260507 19:31:33.459 | INFO | "gpu" : " 17 GB", | 20260507 19:31:33.459 | INFO | "huge" : "", | 20260507 19:31:33.459 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:33.459 | INFO | "malloc" : " 14 GB", | 20260507 19:31:33.459 | INFO | "pinned" : " 29 GB" | 20260507 19:31:33.459 | INFO | } | 20260507 19:31:33.459 | INFO | Perf report for the last 5.1389 seconds: | 20260507 19:31:33.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:33.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 19:31:33.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:31:33.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:33.460 | INFO | Uploaded 36GiB to GPU in 2.56709s (14.0237GiB/s) | 20260507 19:31:33.460 | INFO | Downloaded 34GiB to GPU in 2.46047s (13.8185GiB/s) | 20260507 19:31:38.598 | INFO | Analyzing chunk frames = [2029056,2029568) | 20260507 19:31:38.598 | INFO | MemInfo: { | 20260507 19:31:38.598 | INFO | "gpu" : " 17 GB", | 20260507 19:31:38.598 | INFO | "huge" : "", | 20260507 19:31:38.598 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:38.598 | INFO | "malloc" : " 14 GB", | 20260507 19:31:38.598 | INFO | "pinned" : " 29 GB" | 20260507 19:31:38.598 | INFO | } | 20260507 19:31:38.599 | INFO | Perf report for the last 5.13911 seconds: | 20260507 19:31:38.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:38.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:31:38.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:31:38.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:38.599 | INFO | Uploaded 36GiB to GPU in 2.48806s (14.4691GiB/s) | 20260507 19:31:38.599 | INFO | Downloaded 34GiB to GPU in 2.39603s (14.1901GiB/s) | 20260507 19:31:43.739 | INFO | Analyzing chunk frames = [2029568,2030080) | 20260507 19:31:43.739 | INFO | MemInfo: { | 20260507 19:31:43.739 | INFO | "gpu" : " 17 GB", | 20260507 19:31:43.739 | INFO | "huge" : "", | 20260507 19:31:43.739 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:43.739 | INFO | "malloc" : " 14 GB", | 20260507 19:31:43.739 | INFO | "pinned" : " 29 GB" | 20260507 19:31:43.739 | INFO | } | 20260507 19:31:43.739 | INFO | Perf report for the last 5.14063 seconds: | 20260507 19:31:43.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:43.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.739 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 19:31:43.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.739 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 19:31:43.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:43.740 | INFO | Uploaded 36GiB to GPU in 2.44445s (14.7272GiB/s) | 20260507 19:31:43.740 | INFO | Downloaded 34GiB to GPU in 2.50742s (13.5597GiB/s) | 20260507 19:31:48.878 | INFO | Analyzing chunk frames = [2030080,2030592) | 20260507 19:31:48.878 | INFO | MemInfo: { | 20260507 19:31:48.878 | INFO | "gpu" : " 17 GB", | 20260507 19:31:48.878 | INFO | "huge" : "", | 20260507 19:31:48.878 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:48.878 | INFO | "malloc" : " 14 GB", | 20260507 19:31:48.879 | INFO | "pinned" : " 29 GB" | 20260507 19:31:48.879 | INFO | } | 20260507 19:31:48.879 | INFO | Perf report for the last 5.13945 seconds: | 20260507 19:31:48.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:48.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:31:48.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:31:48.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:48.879 | INFO | Uploaded 36GiB to GPU in 2.51354s (14.3224GiB/s) | 20260507 19:31:48.879 | INFO | Downloaded 34GiB to GPU in 2.42064s (14.0459GiB/s) | 20260507 19:31:54.018 | INFO | Analyzing chunk frames = [2030592,2031104) | 20260507 19:31:54.018 | INFO | MemInfo: { | 20260507 19:31:54.018 | INFO | "gpu" : " 17 GB", | 20260507 19:31:54.018 | INFO | "huge" : "", | 20260507 19:31:54.018 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:54.018 | INFO | "malloc" : " 14 GB", | 20260507 19:31:54.018 | INFO | "pinned" : " 29 GB" | 20260507 19:31:54.018 | INFO | } | 20260507 19:31:54.018 | INFO | Perf report for the last 5.13916 seconds: | 20260507 19:31:54.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:54.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:31:54.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:31:54.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:54.018 | INFO | Uploaded 36GiB to GPU in 2.5492s (14.1221GiB/s) | 20260507 19:31:54.018 | INFO | Downloaded 34GiB to GPU in 2.43293s (13.9749GiB/s) | 20260507 19:31:59.156 | INFO | Analyzing chunk frames = [2031104,2031616) | 20260507 19:31:59.157 | INFO | MemInfo: { | 20260507 19:31:59.157 | INFO | "gpu" : " 17 GB", | 20260507 19:31:59.157 | INFO | "huge" : "", | 20260507 19:31:59.157 | INFO | "huge_shared" : " 29 GB", | 20260507 19:31:59.157 | INFO | "malloc" : " 14 GB", | 20260507 19:31:59.157 | INFO | "pinned" : " 29 GB" | 20260507 19:31:59.157 | INFO | } | 20260507 19:31:59.157 | INFO | Perf report for the last 5.13891 seconds: | 20260507 19:31:59.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:31:59.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:31:59.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:31:59.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:31:59.157 | INFO | Uploaded 36GiB to GPU in 2.51104s (14.3367GiB/s) | 20260507 19:31:59.157 | INFO | Downloaded 34GiB to GPU in 2.42413s (14.0257GiB/s) | 20260507 19:31:59.157 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:31:59.157 | INFO | Total mem usage: 16.4 GiB | 20260507 19:31:59.157 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:31:59.157 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.69%) | 20260507 19:31:59.157 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:31:59.157 | INFO | High Water Breakdown: | 20260507 19:31:59.157 | INFO | 60% : 9.06 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:31:59.157 | INFO | 39.8% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:31:59.157 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:31:59.157 | INFO | ------------------------------------------ | 20260507 19:31:59.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:31:59.157 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:31:59.157 | INFO | Total mem usage: 26.5 GiB | 20260507 19:31:59.157 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:31:59.157 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:31:59.157 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:31:59.157 | INFO | High Water Breakdown: | 20260507 19:31:59.157 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:31:59.157 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:31:59.157 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:31:59.157 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:31:59.157 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:31:59.157 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:31:59.157 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:31:59.157 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:31:59.157 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:31:59.157 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:31:59.157 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:31:59.157 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:31:59.157 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:31:59.157 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:31:59.157 | INFO | ------------------------------------------ | 20260507 19:31:59.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:31:59.157 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:31:59.157 | INFO | Total mem usage: 27.8 GiB | 20260507 19:31:59.157 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:31:59.157 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:31:59.157 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:31:59.157 | INFO | High Water Breakdown: | 20260507 19:31:59.157 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:31:59.157 | INFO | ------------------------------------------ | 20260507 19:31:59.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:31:59.157 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:31:59.157 | INFO | Total mem usage: 15.7 GiB | 20260507 19:31:59.157 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:31:59.157 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:31:59.157 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:31:59.157 | INFO | High Water Breakdown: | 20260507 19:31:59.157 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:31:59.157 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:31:59.157 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:31:59.157 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:31:59.157 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:31:59.157 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:31:59.157 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:31:59.157 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:31:59.157 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:31:59.157 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:31:59.158 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:31:59.158 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:31:59.158 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:31:59.158 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:31:59.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:31:59.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:31:59.158 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:31:59.158 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:31:59.158 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:31:59.158 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:31:59.158 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:31:59.158 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:31:59.158 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:31:59.158 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:31:59.158 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:31:59.158 | INFO | ------------------------------------------ | 20260507 19:31:59.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:31:59.158 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:31:59.158 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:31:59.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:31:59.158 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.40e+01 / 1.95e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 0.29%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.22e-02 / 7.53e-03 | 20260507 19:31:59.158 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.47e+00, 6.00e-01 / 2.69e-01 | 20260507 19:31:59.158 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.51e+00, 2.10e-01 / 1.15e-01 | 20260507 19:31:59.158 | INFO | PerfReport: 15.73%, DME -- 4.17e+01 / 4.48e+01, 3.77e+00 / 1.42e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.83e+00 / 1.88e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.85e+00, 4.35e-01 / 2.20e-01 | 20260507 19:31:59.158 | INFO | PerfReport: 3.65%, Metrics -- 1.21e+00 / 1.43e+00, 3.09e-01 / 1.96e-01 | 20260507 19:31:59.158 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.78e-01 / 4.12e-01, 6.21e-02 / 3.20e-02 | 20260507 19:31:59.158 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:31:59.158 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:31:59.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:31:59.158 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 24.44%, Upload -- 2.62e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 24.03%, Download -- 2.58e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 30.96%, Compute -- 3.32e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 2.19%, LockUpload -- 2.35e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 2.21%, LockDownload -- 2.37e+00 / 0.00e+00, 3.70e+00 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 16.14%, LockCompute -- 1.73e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:31:59.158 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260507 19:31:59.254 | INFO | Starting a round of Baz IO | 20260507 19:32:02.744 | INFO | Populating RTMetrics report | 20260507 19:32:02.744 | INFO | RTMetrics report populated | 20260507 19:32:02.744 | INFO | Opening tmp RTMetrics json file | 20260507 19:32:02.745 | INFO | Serializing RTMetrics to json | 20260507 19:32:02.748 | INFO | Serialization done, swapping tmp file now... | 20260507 19:32:02.760 | INFO | RTMetrics done | 20260507 19:32:02.847 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.593 seconds | 20260507 19:32:02.847 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.33% / 0.01% / 54.66% | 20260507 19:32:02.847 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.36806% / 1.98682e-05% / 6.35783e-05% | 20260507 19:32:02.847 | INFO | Baz IO overall: | 20260507 19:32:02.847 | INFO | BytesWritten : 917761584600 | 20260507 19:32:02.847 | INFO | NumEvents : 444968351732 | 20260507 19:32:04.296 | INFO | Analyzing chunk frames = [2031616,2032128) | 20260507 19:32:04.296 | INFO | MemInfo: { | 20260507 19:32:04.296 | INFO | "gpu" : " 17 GB", | 20260507 19:32:04.296 | INFO | "huge" : "", | 20260507 19:32:04.296 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:04.296 | INFO | "malloc" : " 14 GB", | 20260507 19:32:04.297 | INFO | "pinned" : " 29 GB" | 20260507 19:32:04.297 | INFO | } | 20260507 19:32:04.297 | INFO | Perf report for the last 5.13988 seconds: | 20260507 19:32:04.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:04.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:04.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:04.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:04.297 | INFO | ANALYSIS : 69.1%, 3.0, 107.0ms, 0 | 20260507 19:32:04.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:04.297 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 19:32:04.297 | INFO | PRE_HQ : 66.5%, 1.7, 58.8ms, 0 | 20260507 19:32:04.297 | INFO | BAZWRITER : 69.5%, 4.2, 149.6ms, 0 | 20260507 19:32:04.297 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 19:32:04.297 | INFO | Uploaded 36GiB to GPU in 2.56494s (14.0354GiB/s) | 20260507 19:32:04.297 | INFO | Downloaded 38GiB to GPU in 2.69051s (14.1237GiB/s) | 20260507 19:32:09.436 | INFO | Analyzing chunk frames = [2032128,2032640) | 20260507 19:32:09.436 | INFO | MemInfo: { | 20260507 19:32:09.436 | INFO | "gpu" : " 17 GB", | 20260507 19:32:09.436 | INFO | "huge" : "", | 20260507 19:32:09.436 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:09.436 | INFO | "malloc" : " 14 GB", | 20260507 19:32:09.436 | INFO | "pinned" : " 29 GB" | 20260507 19:32:09.436 | INFO | } | 20260507 19:32:09.436 | INFO | Perf report for the last 5.13915 seconds: | 20260507 19:32:09.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:09.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.436 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 19:32:09.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.436 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:32:09.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:09.437 | INFO | Uploaded 36GiB to GPU in 2.57293s (13.9919GiB/s) | 20260507 19:32:09.437 | INFO | Downloaded 34GiB to GPU in 2.46325s (13.8029GiB/s) | 20260507 19:32:14.575 | INFO | Analyzing chunk frames = [2032640,2033152) | 20260507 19:32:14.575 | INFO | MemInfo: { | 20260507 19:32:14.575 | INFO | "gpu" : " 17 GB", | 20260507 19:32:14.576 | INFO | "huge" : "", | 20260507 19:32:14.576 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:14.576 | INFO | "malloc" : " 14 GB", | 20260507 19:32:14.576 | INFO | "pinned" : " 29 GB" | 20260507 19:32:14.576 | INFO | } | 20260507 19:32:14.576 | INFO | Perf report for the last 5.14008 seconds: | 20260507 19:32:14.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:14.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 19:32:14.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:32:14.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:14.576 | INFO | Uploaded 36GiB to GPU in 2.58777s (13.9116GiB/s) | 20260507 19:32:14.576 | INFO | Downloaded 34GiB to GPU in 2.45156s (13.8687GiB/s) | 20260507 19:32:19.714 | INFO | Analyzing chunk frames = [2033152,2033664) | 20260507 19:32:19.715 | INFO | MemInfo: { | 20260507 19:32:19.715 | INFO | "gpu" : " 17 GB", | 20260507 19:32:19.715 | INFO | "huge" : "", | 20260507 19:32:19.715 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:19.715 | INFO | "malloc" : " 14 GB", | 20260507 19:32:19.715 | INFO | "pinned" : " 29 GB" | 20260507 19:32:19.715 | INFO | } | 20260507 19:32:19.715 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:32:19.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:19.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:32:19.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 19:32:19.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:19.715 | INFO | Uploaded 36GiB to GPU in 2.5061s (14.3649GiB/s) | 20260507 19:32:19.715 | INFO | Downloaded 34GiB to GPU in 2.39655s (14.187GiB/s) | 20260507 19:32:24.855 | INFO | Analyzing chunk frames = [2033664,2034176) | 20260507 19:32:24.855 | INFO | MemInfo: { | 20260507 19:32:24.855 | INFO | "gpu" : " 17 GB", | 20260507 19:32:24.855 | INFO | "huge" : "", | 20260507 19:32:24.855 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:24.855 | INFO | "malloc" : " 14 GB", | 20260507 19:32:24.855 | INFO | "pinned" : " 29 GB" | 20260507 19:32:24.855 | INFO | } | 20260507 19:32:24.855 | INFO | Perf report for the last 5.14029 seconds: | 20260507 19:32:24.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:24.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:32:24.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:32:24.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:24.855 | INFO | Uploaded 36GiB to GPU in 2.40606s (14.9622GiB/s) | 20260507 19:32:24.855 | INFO | Downloaded 34GiB to GPU in 2.49388s (13.6334GiB/s) | 20260507 19:32:29.993 | INFO | Analyzing chunk frames = [2034176,2034688) | 20260507 19:32:29.993 | INFO | MemInfo: { | 20260507 19:32:29.993 | INFO | "gpu" : " 17 GB", | 20260507 19:32:29.993 | INFO | "huge" : "", | 20260507 19:32:29.993 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:29.993 | INFO | "malloc" : " 14 GB", | 20260507 19:32:29.993 | INFO | "pinned" : " 29 GB" | 20260507 19:32:29.993 | INFO | } | 20260507 19:32:29.993 | INFO | Perf report for the last 5.13845 seconds: | 20260507 19:32:29.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:29.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:32:29.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:32:29.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:29.994 | INFO | Uploaded 36GiB to GPU in 2.52441s (14.2607GiB/s) | 20260507 19:32:29.994 | INFO | Downloaded 34GiB to GPU in 2.41604s (14.0726GiB/s) | 20260507 19:32:35.132 | INFO | Analyzing chunk frames = [2034688,2035200) | 20260507 19:32:35.132 | INFO | MemInfo: { | 20260507 19:32:35.132 | INFO | "gpu" : " 17 GB", | 20260507 19:32:35.132 | INFO | "huge" : "", | 20260507 19:32:35.133 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:35.133 | INFO | "malloc" : " 14 GB", | 20260507 19:32:35.133 | INFO | "pinned" : " 29 GB" | 20260507 19:32:35.133 | INFO | } | 20260507 19:32:35.133 | INFO | Perf report for the last 5.13913 seconds: | 20260507 19:32:35.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:35.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:32:35.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:32:35.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:35.133 | INFO | Uploaded 36GiB to GPU in 2.55222s (14.1053GiB/s) | 20260507 19:32:35.133 | INFO | Downloaded 34GiB to GPU in 2.43448s (13.966GiB/s) | 20260507 19:32:40.272 | INFO | Analyzing chunk frames = [2035200,2035712) | 20260507 19:32:40.272 | INFO | MemInfo: { | 20260507 19:32:40.272 | INFO | "gpu" : " 17 GB", | 20260507 19:32:40.272 | INFO | "huge" : "", | 20260507 19:32:40.272 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:40.272 | INFO | "malloc" : " 14 GB", | 20260507 19:32:40.272 | INFO | "pinned" : " 29 GB" | 20260507 19:32:40.272 | INFO | } | 20260507 19:32:40.272 | INFO | Perf report for the last 5.13969 seconds: | 20260507 19:32:40.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:40.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.272 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:32:40.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.272 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:32:40.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:40.273 | INFO | Uploaded 36GiB to GPU in 2.51341s (14.3232GiB/s) | 20260507 19:32:40.273 | INFO | Downloaded 34GiB to GPU in 2.41161s (14.0984GiB/s) | 20260507 19:32:40.369 | INFO | Starting a round of Baz IO | 20260507 19:32:43.857 | INFO | Populating RTMetrics report | 20260507 19:32:43.857 | INFO | RTMetrics report populated | 20260507 19:32:43.857 | INFO | Opening tmp RTMetrics json file | 20260507 19:32:43.888 | INFO | Serializing RTMetrics to json | 20260507 19:32:43.951 | INFO | Serialization done, swapping tmp file now... | 20260507 19:32:43.956 | INFO | RTMetrics done | 20260507 19:32:43.989 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.619 seconds | 20260507 19:32:43.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.31% / 0.01% / 54.67% | 20260507 19:32:43.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.36106% / 3.57628e-05% / 5.16574e-05% | 20260507 19:32:43.989 | INFO | Baz IO overall: | 20260507 19:32:43.989 | INFO | BytesWritten : 919682607574 | 20260507 19:32:43.989 | INFO | NumEvents : 445899316612 | 20260507 19:32:45.410 | INFO | Analyzing chunk frames = [2035712,2036224) | 20260507 19:32:45.410 | INFO | MemInfo: { | 20260507 19:32:45.410 | INFO | "gpu" : " 17 GB", | 20260507 19:32:45.410 | INFO | "huge" : "", | 20260507 19:32:45.410 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:45.410 | INFO | "malloc" : " 14 GB", | 20260507 19:32:45.410 | INFO | "pinned" : " 29 GB" | 20260507 19:32:45.410 | INFO | } | 20260507 19:32:45.410 | INFO | Perf report for the last 5.1377 seconds: | 20260507 19:32:45.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:45.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:45.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:45.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:45.410 | INFO | ANALYSIS : 68.9%, 3.0, 106.9ms, 0 | 20260507 19:32:45.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:45.410 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 19:32:45.410 | INFO | PRE_HQ : 66.4%, 1.7, 58.4ms, 0 | 20260507 19:32:45.410 | INFO | BAZWRITER : 70.1%, 3.7, 134.7ms, 0 | 20260507 19:32:45.410 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:32:45.411 | INFO | Uploaded 36GiB to GPU in 2.54191s (14.1626GiB/s) | 20260507 19:32:45.411 | INFO | Downloaded 38GiB to GPU in 2.69544s (14.0979GiB/s) | 20260507 19:32:50.550 | INFO | Analyzing chunk frames = [2036224,2036736) | 20260507 19:32:50.550 | INFO | MemInfo: { | 20260507 19:32:50.550 | INFO | "gpu" : " 17 GB", | 20260507 19:32:50.550 | INFO | "huge" : "", | 20260507 19:32:50.550 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:50.550 | INFO | "malloc" : " 14 GB", | 20260507 19:32:50.550 | INFO | "pinned" : " 29 GB" | 20260507 19:32:50.550 | INFO | } | 20260507 19:32:50.550 | INFO | Perf report for the last 5.14001 seconds: | 20260507 19:32:50.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:50.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:32:50.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:32:50.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:50.550 | INFO | Uploaded 36GiB to GPU in 2.56419s (14.0395GiB/s) | 20260507 19:32:50.550 | INFO | Downloaded 34GiB to GPU in 2.46633s (13.7857GiB/s) | 20260507 19:32:55.690 | INFO | Analyzing chunk frames = [2036736,2037248) | 20260507 19:32:55.690 | INFO | MemInfo: { | 20260507 19:32:55.690 | INFO | "gpu" : " 17 GB", | 20260507 19:32:55.690 | INFO | "huge" : "", | 20260507 19:32:55.690 | INFO | "huge_shared" : " 29 GB", | 20260507 19:32:55.690 | INFO | "malloc" : " 14 GB", | 20260507 19:32:55.690 | INFO | "pinned" : " 29 GB" | 20260507 19:32:55.690 | INFO | } | 20260507 19:32:55.690 | INFO | Perf report for the last 5.13996 seconds: | 20260507 19:32:55.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:32:55.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:32:55.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:32:55.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:32:55.690 | INFO | Uploaded 36GiB to GPU in 2.55555s (14.087GiB/s) | 20260507 19:32:55.690 | INFO | Downloaded 34GiB to GPU in 2.45303s (13.8604GiB/s) | 20260507 19:33:00.828 | INFO | Analyzing chunk frames = [2037248,2037760) | 20260507 19:33:00.828 | INFO | MemInfo: { | 20260507 19:33:00.828 | INFO | "gpu" : " 17 GB", | 20260507 19:33:00.828 | INFO | "huge" : "", | 20260507 19:33:00.828 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:00.828 | INFO | "malloc" : " 14 GB", | 20260507 19:33:00.828 | INFO | "pinned" : " 29 GB" | 20260507 19:33:00.828 | INFO | } | 20260507 19:33:00.828 | INFO | Perf report for the last 5.13801 seconds: | 20260507 19:33:00.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:00.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.828 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:33:00.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.828 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:33:00.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:00.829 | INFO | Uploaded 36GiB to GPU in 2.4841s (14.4922GiB/s) | 20260507 19:33:00.829 | INFO | Downloaded 34GiB to GPU in 2.3907s (14.2218GiB/s) | 20260507 19:33:05.968 | INFO | Analyzing chunk frames = [2037760,2038272) | 20260507 19:33:05.968 | INFO | MemInfo: { | 20260507 19:33:05.968 | INFO | "gpu" : " 17 GB", | 20260507 19:33:05.968 | INFO | "huge" : "", | 20260507 19:33:05.968 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:05.968 | INFO | "malloc" : " 14 GB", | 20260507 19:33:05.968 | INFO | "pinned" : " 29 GB" | 20260507 19:33:05.969 | INFO | } | 20260507 19:33:05.969 | INFO | Perf report for the last 5.14051 seconds: | 20260507 19:33:05.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:05.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 19:33:05.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:33:05.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:05.969 | INFO | Uploaded 36GiB to GPU in 2.43669s (14.7741GiB/s) | 20260507 19:33:05.969 | INFO | Downloaded 34GiB to GPU in 2.50118s (13.5936GiB/s) | 20260507 19:33:11.108 | INFO | Analyzing chunk frames = [2038272,2038784) | 20260507 19:33:11.108 | INFO | MemInfo: { | 20260507 19:33:11.108 | INFO | "gpu" : " 17 GB", | 20260507 19:33:11.108 | INFO | "huge" : "", | 20260507 19:33:11.108 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:11.108 | INFO | "malloc" : " 14 GB", | 20260507 19:33:11.108 | INFO | "pinned" : " 29 GB" | 20260507 19:33:11.108 | INFO | } | 20260507 19:33:11.108 | INFO | Perf report for the last 5.1398 seconds: | 20260507 19:33:11.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:11.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.108 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:33:11.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.109 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:33:11.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:11.109 | INFO | Uploaded 36GiB to GPU in 2.49815s (14.4107GiB/s) | 20260507 19:33:11.109 | INFO | Downloaded 34GiB to GPU in 2.4102s (14.1067GiB/s) | 20260507 19:33:16.248 | INFO | Analyzing chunk frames = [2038784,2039296) | 20260507 19:33:16.248 | INFO | MemInfo: { | 20260507 19:33:16.248 | INFO | "gpu" : " 17 GB", | 20260507 19:33:16.248 | INFO | "huge" : "", | 20260507 19:33:16.248 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:16.248 | INFO | "malloc" : " 14 GB", | 20260507 19:33:16.248 | INFO | "pinned" : " 29 GB" | 20260507 19:33:16.248 | INFO | } | 20260507 19:33:16.248 | INFO | Perf report for the last 5.13945 seconds: | 20260507 19:33:16.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:16.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:33:16.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:33:16.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:16.248 | INFO | Uploaded 36GiB to GPU in 2.53739s (14.1878GiB/s) | 20260507 19:33:16.248 | INFO | Downloaded 34GiB to GPU in 2.43792s (13.9463GiB/s) | 20260507 19:33:21.387 | INFO | Analyzing chunk frames = [2039296,2039808) | 20260507 19:33:21.387 | INFO | MemInfo: { | 20260507 19:33:21.387 | INFO | "gpu" : " 17 GB", | 20260507 19:33:21.387 | INFO | "huge" : "", | 20260507 19:33:21.387 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:21.387 | INFO | "malloc" : " 14 GB", | 20260507 19:33:21.387 | INFO | "pinned" : " 29 GB" | 20260507 19:33:21.387 | INFO | } | 20260507 19:33:21.387 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:33:21.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:21.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:33:21.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:33:21.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:21.387 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:33:21.387 | INFO | Total mem usage: 16.4 GiB | 20260507 19:33:21.387 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:33:21.387 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.48%) | 20260507 19:33:21.387 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:33:21.387 | INFO | High Water Breakdown: | 20260507 19:33:21.387 | INFO | 59.9% : 8.98 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:33:21.387 | INFO | 39.9% : 5.98 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:33:21.387 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:33:21.388 | INFO | ------------------------------------------ | 20260507 19:33:21.388 | INFO | Uploaded 36GiB to GPU in 2.51413s (14.319GiB/s) | 20260507 19:33:21.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:33:21.388 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:33:21.388 | INFO | Total mem usage: 26.5 GiB | 20260507 19:33:21.388 | INFO | Downloaded 34GiB to GPU in 2.41551s (14.0757GiB/s) | 20260507 19:33:21.388 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:33:21.388 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:33:21.388 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:33:21.388 | INFO | High Water Breakdown: | 20260507 19:33:21.388 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:33:21.388 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:33:21.388 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:33:21.388 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:33:21.388 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:33:21.388 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:33:21.388 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:33:21.388 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:33:21.388 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:33:21.388 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:33:21.388 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:33:21.388 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:33:21.388 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:33:21.388 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:33:21.388 | INFO | ------------------------------------------ | 20260507 19:33:21.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:33:21.388 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:33:21.388 | INFO | Total mem usage: 27.8 GiB | 20260507 19:33:21.388 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:33:21.388 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:33:21.388 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:33:21.388 | INFO | High Water Breakdown: | 20260507 19:33:21.388 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:33:21.388 | INFO | ------------------------------------------ | 20260507 19:33:21.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:33:21.388 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:33:21.388 | INFO | Total mem usage: 15.7 GiB | 20260507 19:33:21.388 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:33:21.388 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:33:21.388 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:33:21.388 | INFO | High Water Breakdown: | 20260507 19:33:21.388 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:33:21.388 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:33:21.388 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:33:21.388 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:33:21.388 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:33:21.388 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:33:21.388 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:33:21.388 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:33:21.388 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:33:21.388 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:33:21.388 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:33:21.388 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:33:21.388 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:33:21.388 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:33:21.388 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:33:21.388 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:33:21.388 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:33:21.388 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:33:21.388 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:33:21.388 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:33:21.388 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:33:21.388 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:33:21.388 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:33:21.388 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:33:21.388 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:33:21.388 | INFO | ------------------------------------------ | 20260507 19:33:21.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:33:21.389 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:33:21.389 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:33:21.389 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:33:21.389 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 1.78e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 0.29%, AggregateStats -- 9.74e-02 / 1.01e-01, 1.12e-02 / 5.19e-03 | 20260507 19:33:21.389 | INFO | PerfReport: 12.08%, Baseline -- 4.01e+00 / 4.45e+00, 6.09e-01 / 2.94e-01 | 20260507 19:33:21.389 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.48e+00, 2.12e-01 / 1.15e-01 | 20260507 19:33:21.389 | INFO | PerfReport: 15.76%, DME -- 4.18e+01 / 4.50e+01, 3.91e+00 / 1.44e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.84e+00 / 1.94e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.42e-01 / 2.47e-01 | 20260507 19:33:21.389 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.42e+00, 3.07e-01 / 1.79e-01 | 20260507 19:33:21.389 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.77e-01 / 4.07e-01, 6.16e-02 / 3.00e-02 | 20260507 19:33:21.389 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:33:21.389 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:33:21.389 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:33:21.389 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 24.52%, Upload -- 2.63e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 24.01%, Download -- 2.57e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 30.96%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 2.23%, LockUpload -- 2.39e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 2.16%, LockDownload -- 2.32e+00 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 16.10%, LockCompute -- 1.73e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:33:21.389 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260507 19:33:21.484 | INFO | Starting a round of Baz IO | 20260507 19:33:24.979 | INFO | Populating RTMetrics report | 20260507 19:33:24.979 | INFO | RTMetrics report populated | 20260507 19:33:24.979 | INFO | Opening tmp RTMetrics json file | 20260507 19:33:25.189 | INFO | Serializing RTMetrics to json | 20260507 19:33:25.212 | INFO | Serialization done, swapping tmp file now... | 20260507 19:33:25.234 | INFO | RTMetrics done | 20260507 19:33:25.288 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.803 seconds | 20260507 19:33:25.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.30% / 0.01% / 54.69% | 20260507 19:33:25.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.35702% / 1.98682e-05% / 7.15256e-05% | 20260507 19:33:25.288 | INFO | Baz IO overall: | 20260507 19:33:25.288 | INFO | BytesWritten : 921603524336 | 20260507 19:33:25.288 | INFO | NumEvents : 446830151441 | 20260507 19:33:26.526 | INFO | Analyzing chunk frames = [2039808,2040320) | 20260507 19:33:26.526 | INFO | MemInfo: { | 20260507 19:33:26.526 | INFO | "gpu" : " 17 GB", | 20260507 19:33:26.526 | INFO | "huge" : "", | 20260507 19:33:26.526 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:26.526 | INFO | "malloc" : " 14 GB", | 20260507 19:33:26.526 | INFO | "pinned" : " 29 GB" | 20260507 19:33:26.526 | INFO | } | 20260507 19:33:26.526 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:33:26.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:26.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:26.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:26.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:26.526 | INFO | ANALYSIS : 69.0%, 2.9, 106.6ms, 0 | 20260507 19:33:26.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:26.526 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 19:33:26.526 | INFO | PRE_HQ : 66.8%, 1.7, 58.7ms, 0 | 20260507 19:33:26.526 | INFO | BAZWRITER : 73.5%, 4.2, 158.6ms, 0 | 20260507 19:33:26.526 | INFO | BAZ_SAMPLING : 4.3%, 1.0, 2.2ms, 0 | 20260507 19:33:26.526 | INFO | Uploaded 36GiB to GPU in 2.5577s (14.0751GiB/s) | 20260507 19:33:26.526 | INFO | Downloaded 38GiB to GPU in 2.7013s (14.0673GiB/s) | 20260507 19:33:31.664 | INFO | Analyzing chunk frames = [2040320,2040832) | 20260507 19:33:31.664 | INFO | MemInfo: { | 20260507 19:33:31.664 | INFO | "gpu" : " 17 GB", | 20260507 19:33:31.664 | INFO | "huge" : "", | 20260507 19:33:31.664 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:31.664 | INFO | "malloc" : " 14 GB", | 20260507 19:33:31.664 | INFO | "pinned" : " 29 GB" | 20260507 19:33:31.664 | INFO | } | 20260507 19:33:31.664 | INFO | Perf report for the last 5.13857 seconds: | 20260507 19:33:31.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:31.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.664 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260507 19:33:31.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.664 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:33:31.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:31.665 | INFO | Uploaded 36GiB to GPU in 2.59722s (13.861GiB/s) | 20260507 19:33:31.665 | INFO | Downloaded 34GiB to GPU in 2.46244s (13.8074GiB/s) | 20260507 19:33:36.805 | INFO | Analyzing chunk frames = [2040832,2041344) | 20260507 19:33:36.805 | INFO | MemInfo: { | 20260507 19:33:36.805 | INFO | "gpu" : " 17 GB", | 20260507 19:33:36.805 | INFO | "huge" : "", | 20260507 19:33:36.805 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:36.805 | INFO | "malloc" : " 14 GB", | 20260507 19:33:36.805 | INFO | "pinned" : " 29 GB" | 20260507 19:33:36.805 | INFO | } | 20260507 19:33:36.805 | INFO | Perf report for the last 5.1412 seconds: | 20260507 19:33:36.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:36.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.806 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:33:36.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.806 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:33:36.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:36.806 | INFO | Uploaded 36GiB to GPU in 2.56834s (14.0169GiB/s) | 20260507 19:33:36.806 | INFO | Downloaded 34GiB to GPU in 2.44454s (13.9086GiB/s) | 20260507 19:33:41.944 | INFO | Analyzing chunk frames = [2041344,2041856) | 20260507 19:33:41.944 | INFO | MemInfo: { | 20260507 19:33:41.944 | INFO | "gpu" : " 17 GB", | 20260507 19:33:41.944 | INFO | "huge" : "", | 20260507 19:33:41.944 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:41.944 | INFO | "malloc" : " 14 GB", | 20260507 19:33:41.944 | INFO | "pinned" : " 29 GB" | 20260507 19:33:41.944 | INFO | } | 20260507 19:33:41.944 | INFO | Perf report for the last 5.13854 seconds: | 20260507 19:33:41.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:41.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:33:41.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:33:41.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:41.944 | INFO | Uploaded 36GiB to GPU in 2.49278s (14.4417GiB/s) | 20260507 19:33:41.945 | INFO | Downloaded 34GiB to GPU in 2.38589s (14.2504GiB/s) | 20260507 19:33:47.083 | INFO | Analyzing chunk frames = [2041856,2042368) | 20260507 19:33:47.083 | INFO | MemInfo: { | 20260507 19:33:47.083 | INFO | "gpu" : " 17 GB", | 20260507 19:33:47.083 | INFO | "huge" : "", | 20260507 19:33:47.083 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:47.083 | INFO | "malloc" : " 14 GB", | 20260507 19:33:47.083 | INFO | "pinned" : " 29 GB" | 20260507 19:33:47.083 | INFO | } | 20260507 19:33:47.083 | INFO | Perf report for the last 5.13931 seconds: | 20260507 19:33:47.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:47.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260507 19:33:47.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:33:47.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:47.084 | INFO | Uploaded 36GiB to GPU in 2.47245s (14.5604GiB/s) | 20260507 19:33:47.084 | INFO | Downloaded 34GiB to GPU in 2.50047s (13.5975GiB/s) | 20260507 19:33:52.222 | INFO | Analyzing chunk frames = [2042368,2042880) | 20260507 19:33:52.222 | INFO | MemInfo: { | 20260507 19:33:52.222 | INFO | "gpu" : " 17 GB", | 20260507 19:33:52.222 | INFO | "huge" : "", | 20260507 19:33:52.222 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:52.222 | INFO | "malloc" : " 14 GB", | 20260507 19:33:52.222 | INFO | "pinned" : " 29 GB" | 20260507 19:33:52.222 | INFO | } | 20260507 19:33:52.222 | INFO | Perf report for the last 5.13892 seconds: | 20260507 19:33:52.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:52.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.222 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:33:52.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.222 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:33:52.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:52.223 | INFO | Uploaded 36GiB to GPU in 2.523s (14.2687GiB/s) | 20260507 19:33:52.223 | INFO | Downloaded 34GiB to GPU in 2.40876s (14.1151GiB/s) | 20260507 19:33:57.362 | INFO | Analyzing chunk frames = [2042880,2043392) | 20260507 19:33:57.362 | INFO | MemInfo: { | 20260507 19:33:57.362 | INFO | "gpu" : " 17 GB", | 20260507 19:33:57.362 | INFO | "huge" : "", | 20260507 19:33:57.362 | INFO | "huge_shared" : " 29 GB", | 20260507 19:33:57.362 | INFO | "malloc" : " 14 GB", | 20260507 19:33:57.362 | INFO | "pinned" : " 29 GB" | 20260507 19:33:57.362 | INFO | } | 20260507 19:33:57.362 | INFO | Perf report for the last 5.13999 seconds: | 20260507 19:33:57.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:33:57.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.362 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:33:57.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.363 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 19:33:57.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:33:57.363 | INFO | Uploaded 36GiB to GPU in 2.55645s (14.082GiB/s) | 20260507 19:33:57.363 | INFO | Downloaded 34GiB to GPU in 2.4332s (13.9733GiB/s) | 20260507 19:34:02.501 | INFO | Analyzing chunk frames = [2043392,2043904) | 20260507 19:34:02.501 | INFO | MemInfo: { | 20260507 19:34:02.501 | INFO | "gpu" : " 17 GB", | 20260507 19:34:02.501 | INFO | "huge" : "", | 20260507 19:34:02.501 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:02.501 | INFO | "malloc" : " 14 GB", | 20260507 19:34:02.501 | INFO | "pinned" : " 29 GB" | 20260507 19:34:02.501 | INFO | } | 20260507 19:34:02.501 | INFO | Perf report for the last 5.13898 seconds: | 20260507 19:34:02.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:02.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.501 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:34:02.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.501 | INFO | PRE_HQ : 36.7%, 1.0, 19.0ms, 0 | 20260507 19:34:02.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:02.502 | INFO | Uploaded 36GiB to GPU in 2.52685s (14.247GiB/s) | 20260507 19:34:02.502 | INFO | Downloaded 34GiB to GPU in 2.41097s (14.1022GiB/s) | 20260507 19:34:02.614 | INFO | Starting a round of Baz IO | 20260507 19:34:06.108 | INFO | Populating RTMetrics report | 20260507 19:34:06.108 | INFO | RTMetrics report populated | 20260507 19:34:06.108 | INFO | Opening tmp RTMetrics json file | 20260507 19:34:06.131 | INFO | Serializing RTMetrics to json | 20260507 19:34:06.132 | INFO | Serialization done, swapping tmp file now... | 20260507 19:34:06.133 | INFO | RTMetrics done | 20260507 19:34:06.206 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.592 seconds | 20260507 19:34:06.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.29% / 0.01% / 54.70% | 20260507 19:34:06.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.35239% / 1.58946e-05% / 4.37101e-05% | 20260507 19:34:06.206 | INFO | Baz IO overall: | 20260507 19:34:06.206 | INFO | BytesWritten : 923524393160 | 20260507 19:34:06.206 | INFO | NumEvents : 447761182841 | 20260507 19:34:07.640 | INFO | Analyzing chunk frames = [2043904,2044416) | 20260507 19:34:07.640 | INFO | MemInfo: { | 20260507 19:34:07.640 | INFO | "gpu" : " 17 GB", | 20260507 19:34:07.640 | INFO | "huge" : "", | 20260507 19:34:07.640 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:07.640 | INFO | "malloc" : " 14 GB", | 20260507 19:34:07.640 | INFO | "pinned" : " 29 GB" | 20260507 19:34:07.640 | INFO | } | 20260507 19:34:07.640 | INFO | Perf report for the last 5.13881 seconds: | 20260507 19:34:07.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:07.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:07.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:07.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:07.640 | INFO | ANALYSIS : 69.4%, 2.9, 105.2ms, 0 | 20260507 19:34:07.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:07.640 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260507 19:34:07.640 | INFO | PRE_HQ : 68.3%, 1.8, 64.4ms, 0 | 20260507 19:34:07.640 | INFO | BAZWRITER : 69.5%, 4.0, 142.1ms, 0 | 20260507 19:34:07.640 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:34:07.641 | INFO | Uploaded 36GiB to GPU in 2.57119s (14.0013GiB/s) | 20260507 19:34:07.641 | INFO | Downloaded 38GiB to GPU in 2.69365s (14.1072GiB/s) | 20260507 19:34:12.779 | INFO | Analyzing chunk frames = [2044416,2044928) | 20260507 19:34:12.779 | INFO | MemInfo: { | 20260507 19:34:12.779 | INFO | "gpu" : " 17 GB", | 20260507 19:34:12.779 | INFO | "huge" : "", | 20260507 19:34:12.779 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:12.779 | INFO | "malloc" : " 14 GB", | 20260507 19:34:12.779 | INFO | "pinned" : " 29 GB" | 20260507 19:34:12.779 | INFO | } | 20260507 19:34:12.779 | INFO | Perf report for the last 5.13931 seconds: | 20260507 19:34:12.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:12.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.779 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:34:12.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.780 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:34:12.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:12.780 | INFO | Uploaded 36GiB to GPU in 2.48793s (14.4699GiB/s) | 20260507 19:34:12.780 | INFO | Downloaded 34GiB to GPU in 2.47153s (13.7567GiB/s) | 20260507 19:34:17.919 | INFO | Analyzing chunk frames = [2044928,2045440) | 20260507 19:34:17.919 | INFO | MemInfo: { | 20260507 19:34:17.919 | INFO | "gpu" : " 17 GB", | 20260507 19:34:17.919 | INFO | "huge" : "", | 20260507 19:34:17.919 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:17.919 | INFO | "malloc" : " 14 GB", | 20260507 19:34:17.919 | INFO | "pinned" : " 29 GB" | 20260507 19:34:17.919 | INFO | } | 20260507 19:34:17.919 | INFO | Perf report for the last 5.13951 seconds: | 20260507 19:34:17.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:17.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.919 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:34:17.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.919 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:34:17.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:17.920 | INFO | Uploaded 36GiB to GPU in 2.59785s (13.8576GiB/s) | 20260507 19:34:17.920 | INFO | Downloaded 34GiB to GPU in 2.45457s (13.8517GiB/s) | 20260507 19:34:23.060 | INFO | Analyzing chunk frames = [2045440,2045952) | 20260507 19:34:23.060 | INFO | MemInfo: { | 20260507 19:34:23.060 | INFO | "gpu" : " 17 GB", | 20260507 19:34:23.060 | INFO | "huge" : "", | 20260507 19:34:23.060 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:23.060 | INFO | "malloc" : " 14 GB", | 20260507 19:34:23.060 | INFO | "pinned" : " 29 GB" | 20260507 19:34:23.060 | INFO | } | 20260507 19:34:23.060 | INFO | Perf report for the last 5.14098 seconds: | 20260507 19:34:23.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:23.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:34:23.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:34:23.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:23.060 | INFO | Uploaded 36GiB to GPU in 2.48479s (14.4881GiB/s) | 20260507 19:34:23.060 | INFO | Downloaded 34GiB to GPU in 2.38721s (14.2426GiB/s) | 20260507 19:34:28.199 | INFO | Analyzing chunk frames = [2045952,2046464) | 20260507 19:34:28.199 | INFO | MemInfo: { | 20260507 19:34:28.199 | INFO | "gpu" : " 17 GB", | 20260507 19:34:28.199 | INFO | "huge" : "", | 20260507 19:34:28.199 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:28.199 | INFO | "malloc" : " 14 GB", | 20260507 19:34:28.199 | INFO | "pinned" : " 29 GB" | 20260507 19:34:28.199 | INFO | } | 20260507 19:34:28.199 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:34:28.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:28.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.199 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:34:28.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.199 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:34:28.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:28.200 | INFO | Uploaded 36GiB to GPU in 2.39682s (15.0199GiB/s) | 20260507 19:34:28.200 | INFO | Downloaded 34GiB to GPU in 2.48976s (13.6559GiB/s) | 20260507 19:34:33.337 | INFO | Analyzing chunk frames = [2046464,2046976) | 20260507 19:34:33.337 | INFO | MemInfo: { | 20260507 19:34:33.337 | INFO | "gpu" : " 17 GB", | 20260507 19:34:33.337 | INFO | "huge" : "", | 20260507 19:34:33.337 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:33.337 | INFO | "malloc" : " 14 GB", | 20260507 19:34:33.337 | INFO | "pinned" : " 29 GB" | 20260507 19:34:33.337 | INFO | } | 20260507 19:34:33.337 | INFO | Perf report for the last 5.13815 seconds: | 20260507 19:34:33.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:33.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.337 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:34:33.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.338 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:34:33.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:33.338 | INFO | Uploaded 36GiB to GPU in 2.50848s (14.3513GiB/s) | 20260507 19:34:33.338 | INFO | Downloaded 34GiB to GPU in 2.41241s (14.0938GiB/s) | 20260507 19:34:38.475 | INFO | Analyzing chunk frames = [2046976,2047488) | 20260507 19:34:38.475 | INFO | MemInfo: { | 20260507 19:34:38.475 | INFO | "gpu" : " 17 GB", | 20260507 19:34:38.475 | INFO | "huge" : "", | 20260507 19:34:38.475 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:38.475 | INFO | "malloc" : " 14 GB", | 20260507 19:34:38.475 | INFO | "pinned" : " 29 GB" | 20260507 19:34:38.475 | INFO | } | 20260507 19:34:38.475 | INFO | Perf report for the last 5.13803 seconds: | 20260507 19:34:38.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:38.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.475 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 19:34:38.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.476 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:34:38.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:38.476 | INFO | Uploaded 36GiB to GPU in 2.55512s (14.0893GiB/s) | 20260507 19:34:38.476 | INFO | Downloaded 34GiB to GPU in 2.43213s (13.9795GiB/s) | 20260507 19:34:43.614 | INFO | Analyzing chunk frames = [2047488,2048000) | 20260507 19:34:43.614 | INFO | MemInfo: { | 20260507 19:34:43.614 | INFO | "gpu" : " 17 GB", | 20260507 19:34:43.615 | INFO | "huge" : "", | 20260507 19:34:43.615 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:43.615 | INFO | "malloc" : " 14 GB", | 20260507 19:34:43.615 | INFO | "pinned" : " 29 GB" | 20260507 19:34:43.615 | INFO | } | 20260507 19:34:43.615 | INFO | Perf report for the last 5.13928 seconds: | 20260507 19:34:43.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:43.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:34:43.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:34:43.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:43.615 | INFO | Uploaded 36GiB to GPU in 2.52255s (14.2712GiB/s) | 20260507 19:34:43.615 | INFO | Downloaded 34GiB to GPU in 2.40902s (14.1136GiB/s) | 20260507 19:34:43.615 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:34:43.615 | INFO | Total mem usage: 16.4 GiB | 20260507 19:34:43.615 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:34:43.615 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.82%) | 20260507 19:34:43.615 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:34:43.615 | INFO | High Water Breakdown: | 20260507 19:34:43.615 | INFO | 59.8% : 8.93 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:34:43.616 | INFO | 40.1% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:34:43.616 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:34:43.616 | INFO | ------------------------------------------ | 20260507 19:34:43.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:34:43.616 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:34:43.616 | INFO | Total mem usage: 26.5 GiB | 20260507 19:34:43.616 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:34:43.616 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:34:43.616 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:34:43.616 | INFO | High Water Breakdown: | 20260507 19:34:43.616 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:34:43.616 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:34:43.616 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:34:43.616 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:34:43.616 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:34:43.616 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:34:43.616 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:34:43.616 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:34:43.616 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:34:43.616 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:34:43.616 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:34:43.616 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:34:43.616 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:34:43.616 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:34:43.616 | INFO | ------------------------------------------ | 20260507 19:34:43.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:34:43.616 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:34:43.616 | INFO | Total mem usage: 27.8 GiB | 20260507 19:34:43.616 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:34:43.616 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:34:43.616 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:34:43.616 | INFO | High Water Breakdown: | 20260507 19:34:43.616 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:34:43.616 | INFO | ------------------------------------------ | 20260507 19:34:43.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:34:43.616 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:34:43.616 | INFO | Total mem usage: 15.7 GiB | 20260507 19:34:43.616 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:34:43.616 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:34:43.616 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:34:43.616 | INFO | High Water Breakdown: | 20260507 19:34:43.616 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:34:43.616 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:34:43.616 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:34:43.616 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:34:43.616 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:34:43.616 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:34:43.616 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:34:43.616 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:34:43.616 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:34:43.616 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:34:43.616 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:34:43.616 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:34:43.616 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:34:43.616 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:34:43.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:34:43.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:34:43.616 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:34:43.616 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:34:43.616 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:34:43.616 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:34:43.616 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:34:43.616 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:34:43.616 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:34:43.616 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:34:43.616 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:34:43.616 | INFO | ------------------------------------------ | 20260507 19:34:43.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:34:43.617 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:34:43.617 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:34:43.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:34:43.617 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 1.79e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.16e-02 / 5.98e-03 | 20260507 19:34:43.617 | INFO | PerfReport: 12.07%, Baseline -- 4.01e+00 / 4.44e+00, 6.00e-01 / 3.50e-01 | 20260507 19:34:43.617 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.50e+00, 2.14e-01 / 1.24e-01 | 20260507 19:34:43.617 | INFO | PerfReport: 15.75%, DME -- 4.18e+01 / 4.49e+01, 3.83e+00 / 1.43e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.82e+00 / 1.65e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.50e+00 / 2.85e+00, 4.42e-01 / 2.31e-01 | 20260507 19:34:43.617 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.40e+00, 3.06e-01 / 1.89e-01 | 20260507 19:34:43.617 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.78e-01 / 4.08e-01, 6.16e-02 / 3.47e-02 | 20260507 19:34:43.617 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:34:43.617 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:34:43.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:34:43.617 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 24.57%, Upload -- 2.63e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 24.01%, Download -- 2.57e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 31.00%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 2.29%, LockUpload -- 2.45e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 2.08%, LockDownload -- 2.23e+00 / 0.00e+00, 3.47e+00 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 16.03%, LockCompute -- 1.72e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 19:34:43.617 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 19:34:43.712 | INFO | Starting a round of Baz IO | 20260507 19:34:47.199 | INFO | Populating RTMetrics report | 20260507 19:34:47.199 | INFO | RTMetrics report populated | 20260507 19:34:47.199 | INFO | Opening tmp RTMetrics json file | 20260507 19:34:47.201 | INFO | Serializing RTMetrics to json | 20260507 19:34:47.203 | INFO | Serialization done, swapping tmp file now... | 20260507 19:34:47.204 | INFO | RTMetrics done | 20260507 19:34:47.302 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.589 seconds | 20260507 19:34:47.302 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.28% / 0.01% / 54.71% | 20260507 19:34:47.302 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.34756% / 2.78155e-05% / 5.5631e-05% | 20260507 19:34:47.302 | INFO | Baz IO overall: | 20260507 19:34:47.302 | INFO | BytesWritten : 925444520154 | 20260507 19:34:47.302 | INFO | NumEvents : 448691746601 | 20260507 19:34:48.754 | INFO | Analyzing chunk frames = [2048000,2048512) | 20260507 19:34:48.754 | INFO | MemInfo: { | 20260507 19:34:48.754 | INFO | "gpu" : " 17 GB", | 20260507 19:34:48.754 | INFO | "huge" : "", | 20260507 19:34:48.754 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:48.755 | INFO | "malloc" : " 14 GB", | 20260507 19:34:48.755 | INFO | "pinned" : " 29 GB" | 20260507 19:34:48.755 | INFO | } | 20260507 19:34:48.755 | INFO | Perf report for the last 5.13983 seconds: | 20260507 19:34:48.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:48.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:48.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:48.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:48.755 | INFO | ANALYSIS : 69.3%, 2.9, 106.9ms, 0 | 20260507 19:34:48.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:48.755 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 19:34:48.755 | INFO | PRE_HQ : 66.2%, 1.7, 58.2ms, 0 | 20260507 19:34:48.755 | INFO | BAZWRITER : 69.4%, 4.4, 157.1ms, 0 | 20260507 19:34:48.755 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 19:34:48.755 | INFO | Uploaded 36GiB to GPU in 2.50286s (14.3835GiB/s) | 20260507 19:34:48.755 | INFO | Downloaded 38GiB to GPU in 2.69582s (14.0959GiB/s) | 20260507 19:34:53.894 | INFO | Analyzing chunk frames = [2048512,2049024) | 20260507 19:34:53.894 | INFO | MemInfo: { | 20260507 19:34:53.894 | INFO | "gpu" : " 17 GB", | 20260507 19:34:53.894 | INFO | "huge" : "", | 20260507 19:34:53.894 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:53.894 | INFO | "malloc" : " 14 GB", | 20260507 19:34:53.894 | INFO | "pinned" : " 29 GB" | 20260507 19:34:53.894 | INFO | } | 20260507 19:34:53.894 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:34:53.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:53.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.894 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 19:34:53.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.894 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 19:34:53.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:53.895 | INFO | Uploaded 36GiB to GPU in 2.51537s (14.312GiB/s) | 20260507 19:34:53.895 | INFO | Downloaded 34GiB to GPU in 2.49015s (13.6538GiB/s) | 20260507 19:34:59.034 | INFO | Analyzing chunk frames = [2049024,2049536) | 20260507 19:34:59.035 | INFO | MemInfo: { | 20260507 19:34:59.035 | INFO | "gpu" : " 17 GB", | 20260507 19:34:59.035 | INFO | "huge" : "", | 20260507 19:34:59.035 | INFO | "huge_shared" : " 29 GB", | 20260507 19:34:59.035 | INFO | "malloc" : " 14 GB", | 20260507 19:34:59.035 | INFO | "pinned" : " 29 GB" | 20260507 19:34:59.035 | INFO | } | 20260507 19:34:59.035 | INFO | Perf report for the last 5.1407 seconds: | 20260507 19:34:59.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:34:59.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:34:59.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:34:59.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:34:59.035 | INFO | Uploaded 36GiB to GPU in 2.56227s (14.0501GiB/s) | 20260507 19:34:59.035 | INFO | Downloaded 34GiB to GPU in 2.4808s (13.7053GiB/s) | 20260507 19:35:04.173 | INFO | Analyzing chunk frames = [2049536,2050048) | 20260507 19:35:04.173 | INFO | MemInfo: { | 20260507 19:35:04.173 | INFO | "gpu" : " 17 GB", | 20260507 19:35:04.173 | INFO | "huge" : "", | 20260507 19:35:04.173 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:04.173 | INFO | "malloc" : " 14 GB", | 20260507 19:35:04.173 | INFO | "pinned" : " 29 GB" | 20260507 19:35:04.173 | INFO | } | 20260507 19:35:04.173 | INFO | Perf report for the last 5.13856 seconds: | 20260507 19:35:04.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:04.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.173 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 19:35:04.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.173 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:35:04.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:04.174 | INFO | Uploaded 36GiB to GPU in 2.46521s (14.6032GiB/s) | 20260507 19:35:04.174 | INFO | Downloaded 34GiB to GPU in 2.39994s (14.167GiB/s) | 20260507 19:35:09.313 | INFO | Analyzing chunk frames = [2050048,2050560) | 20260507 19:35:09.313 | INFO | MemInfo: { | 20260507 19:35:09.313 | INFO | "gpu" : " 17 GB", | 20260507 19:35:09.313 | INFO | "huge" : "", | 20260507 19:35:09.313 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:09.313 | INFO | "malloc" : " 14 GB", | 20260507 19:35:09.313 | INFO | "pinned" : " 29 GB" | 20260507 19:35:09.313 | INFO | } | 20260507 19:35:09.313 | INFO | Perf report for the last 5.13988 seconds: | 20260507 19:35:09.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:09.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.313 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 19:35:09.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.313 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:35:09.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:09.314 | INFO | Uploaded 36GiB to GPU in 2.41919s (14.881GiB/s) | 20260507 19:35:09.314 | INFO | Downloaded 34GiB to GPU in 2.50637s (13.5655GiB/s) | 20260507 19:35:14.452 | INFO | Analyzing chunk frames = [2050560,2051072) | 20260507 19:35:14.452 | INFO | MemInfo: { | 20260507 19:35:14.452 | INFO | "gpu" : " 17 GB", | 20260507 19:35:14.452 | INFO | "huge" : "", | 20260507 19:35:14.452 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:14.452 | INFO | "malloc" : " 14 GB", | 20260507 19:35:14.452 | INFO | "pinned" : " 29 GB" | 20260507 19:35:14.452 | INFO | } | 20260507 19:35:14.452 | INFO | Perf report for the last 5.13885 seconds: | 20260507 19:35:14.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:14.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.452 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:35:14.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.452 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 19:35:14.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:14.453 | INFO | Uploaded 36GiB to GPU in 2.48992s (14.4583GiB/s) | 20260507 19:35:14.453 | INFO | Downloaded 34GiB to GPU in 2.42969s (13.9936GiB/s) | 20260507 19:35:19.590 | INFO | Analyzing chunk frames = [2051072,2051584) | 20260507 19:35:19.590 | INFO | MemInfo: { | 20260507 19:35:19.590 | INFO | "gpu" : " 17 GB", | 20260507 19:35:19.590 | INFO | "huge" : "", | 20260507 19:35:19.590 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:19.590 | INFO | "malloc" : " 14 GB", | 20260507 19:35:19.590 | INFO | "pinned" : " 29 GB" | 20260507 19:35:19.590 | INFO | } | 20260507 19:35:19.590 | INFO | Perf report for the last 5.13812 seconds: | 20260507 19:35:19.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:19.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.590 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:35:19.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.590 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:35:19.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:19.591 | INFO | Uploaded 36GiB to GPU in 2.52382s (14.2641GiB/s) | 20260507 19:35:19.591 | INFO | Downloaded 34GiB to GPU in 2.44408s (13.9111GiB/s) | 20260507 19:35:24.730 | INFO | Analyzing chunk frames = [2051584,2052096) | 20260507 19:35:24.730 | INFO | MemInfo: { | 20260507 19:35:24.730 | INFO | "gpu" : " 17 GB", | 20260507 19:35:24.730 | INFO | "huge" : "", | 20260507 19:35:24.730 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:24.730 | INFO | "malloc" : " 14 GB", | 20260507 19:35:24.731 | INFO | "pinned" : " 29 GB" | 20260507 19:35:24.731 | INFO | } | 20260507 19:35:24.731 | INFO | Perf report for the last 5.1405 seconds: | 20260507 19:35:24.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:24.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:35:24.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:35:24.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:24.731 | INFO | Uploaded 36GiB to GPU in 2.49049s (14.455GiB/s) | 20260507 19:35:24.731 | INFO | Downloaded 34GiB to GPU in 2.42781s (14.0044GiB/s) | 20260507 19:35:24.828 | INFO | Starting a round of Baz IO | 20260507 19:35:28.300 | INFO | Populating RTMetrics report | 20260507 19:35:28.300 | INFO | RTMetrics report populated | 20260507 19:35:28.300 | INFO | Opening tmp RTMetrics json file | 20260507 19:35:28.321 | INFO | Serializing RTMetrics to json | 20260507 19:35:28.323 | INFO | Serialization done, swapping tmp file now... | 20260507 19:35:28.323 | INFO | RTMetrics done | 20260507 19:35:28.612 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.784 seconds | 20260507 19:35:28.612 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.27% / 0.01% / 54.72% | 20260507 19:35:28.612 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33575% / 1.98682e-05% / 3.57628e-05% | 20260507 19:35:28.613 | INFO | Baz IO overall: | 20260507 19:35:28.613 | INFO | BytesWritten : 927363206357 | 20260507 19:35:28.613 | INFO | NumEvents : 449621379580 | 20260507 19:35:29.871 | INFO | Analyzing chunk frames = [2052096,2052608) | 20260507 19:35:29.871 | INFO | MemInfo: { | 20260507 19:35:29.871 | INFO | "gpu" : " 17 GB", | 20260507 19:35:29.871 | INFO | "huge" : "", | 20260507 19:35:29.871 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:29.871 | INFO | "malloc" : " 14 GB", | 20260507 19:35:29.871 | INFO | "pinned" : " 29 GB" | 20260507 19:35:29.872 | INFO | } | 20260507 19:35:29.872 | INFO | Perf report for the last 5.14083 seconds: | 20260507 19:35:29.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:29.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:29.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:29.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:29.872 | INFO | ANALYSIS : 69.0%, 3.0, 106.8ms, 0 | 20260507 19:35:29.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:29.872 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:35:29.872 | INFO | PRE_HQ : 66.3%, 1.7, 58.2ms, 0 | 20260507 19:35:29.872 | INFO | BAZWRITER : 73.1%, 4.1, 154.8ms, 0 | 20260507 19:35:29.872 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 19:35:29.872 | INFO | Uploaded 36GiB to GPU in 2.56647s (14.027GiB/s) | 20260507 19:35:29.872 | INFO | Downloaded 38GiB to GPU in 2.69935s (14.0774GiB/s) | 20260507 19:35:35.010 | INFO | Analyzing chunk frames = [2052608,2053120) | 20260507 19:35:35.010 | INFO | MemInfo: { | 20260507 19:35:35.010 | INFO | "gpu" : " 17 GB", | 20260507 19:35:35.010 | INFO | "huge" : "", | 20260507 19:35:35.010 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:35.010 | INFO | "malloc" : " 14 GB", | 20260507 19:35:35.010 | INFO | "pinned" : " 29 GB" | 20260507 19:35:35.010 | INFO | } | 20260507 19:35:35.010 | INFO | Perf report for the last 5.13887 seconds: | 20260507 19:35:35.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:35.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.011 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 19:35:35.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.011 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:35:35.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:35.011 | INFO | Uploaded 36GiB to GPU in 2.58968s (13.9013GiB/s) | 20260507 19:35:35.011 | INFO | Downloaded 34GiB to GPU in 2.47298s (13.7486GiB/s) | 20260507 19:35:40.149 | INFO | Analyzing chunk frames = [2053120,2053632) | 20260507 19:35:40.149 | INFO | MemInfo: { | 20260507 19:35:40.149 | INFO | "gpu" : " 17 GB", | 20260507 19:35:40.149 | INFO | "huge" : "", | 20260507 19:35:40.149 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:40.149 | INFO | "malloc" : " 14 GB", | 20260507 19:35:40.149 | INFO | "pinned" : " 29 GB" | 20260507 19:35:40.149 | INFO | } | 20260507 19:35:40.149 | INFO | Perf report for the last 5.13857 seconds: | 20260507 19:35:40.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:40.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 19:35:40.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:35:40.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:40.149 | INFO | Uploaded 36GiB to GPU in 2.59916s (13.8506GiB/s) | 20260507 19:35:40.149 | INFO | Downloaded 34GiB to GPU in 2.45786s (13.8332GiB/s) | 20260507 19:35:45.287 | INFO | Analyzing chunk frames = [2053632,2054144) | 20260507 19:35:45.287 | INFO | MemInfo: { | 20260507 19:35:45.287 | INFO | "gpu" : " 17 GB", | 20260507 19:35:45.287 | INFO | "huge" : "", | 20260507 19:35:45.287 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:45.287 | INFO | "malloc" : " 14 GB", | 20260507 19:35:45.287 | INFO | "pinned" : " 29 GB" | 20260507 19:35:45.287 | INFO | } | 20260507 19:35:45.287 | INFO | Perf report for the last 5.13834 seconds: | 20260507 19:35:45.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:45.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.287 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 19:35:45.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.288 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:35:45.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:45.288 | INFO | Uploaded 36GiB to GPU in 2.48459s (14.4893GiB/s) | 20260507 19:35:45.288 | INFO | Downloaded 34GiB to GPU in 2.38281s (14.2689GiB/s) | 20260507 19:35:50.426 | INFO | Analyzing chunk frames = [2054144,2054656) | 20260507 19:35:50.426 | INFO | MemInfo: { | 20260507 19:35:50.426 | INFO | "gpu" : " 17 GB", | 20260507 19:35:50.426 | INFO | "huge" : "", | 20260507 19:35:50.426 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:50.426 | INFO | "malloc" : " 14 GB", | 20260507 19:35:50.426 | INFO | "pinned" : " 29 GB" | 20260507 19:35:50.426 | INFO | } | 20260507 19:35:50.426 | INFO | Perf report for the last 5.13901 seconds: | 20260507 19:35:50.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:50.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.426 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:35:50.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.427 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:35:50.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:50.427 | INFO | Uploaded 36GiB to GPU in 2.3917s (15.052GiB/s) | 20260507 19:35:50.427 | INFO | Downloaded 34GiB to GPU in 2.49195s (13.6439GiB/s) | 20260507 19:35:55.566 | INFO | Analyzing chunk frames = [2054656,2055168) | 20260507 19:35:55.566 | INFO | MemInfo: { | 20260507 19:35:55.566 | INFO | "gpu" : " 17 GB", | 20260507 19:35:55.566 | INFO | "huge" : "", | 20260507 19:35:55.566 | INFO | "huge_shared" : " 29 GB", | 20260507 19:35:55.566 | INFO | "malloc" : " 14 GB", | 20260507 19:35:55.566 | INFO | "pinned" : " 29 GB" | 20260507 19:35:55.566 | INFO | } | 20260507 19:35:55.566 | INFO | Perf report for the last 5.13943 seconds: | 20260507 19:35:55.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:35:55.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:35:55.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:35:55.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:35:55.566 | INFO | Uploaded 36GiB to GPU in 2.51911s (14.2908GiB/s) | 20260507 19:35:55.566 | INFO | Downloaded 34GiB to GPU in 2.4072s (14.1243GiB/s) | 20260507 19:36:00.705 | INFO | Analyzing chunk frames = [2055168,2055680) | 20260507 19:36:00.705 | INFO | MemInfo: { | 20260507 19:36:00.705 | INFO | "gpu" : " 17 GB", | 20260507 19:36:00.705 | INFO | "huge" : "", | 20260507 19:36:00.705 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:00.705 | INFO | "malloc" : " 14 GB", | 20260507 19:36:00.705 | INFO | "pinned" : " 29 GB" | 20260507 19:36:00.705 | INFO | } | 20260507 19:36:00.705 | INFO | Perf report for the last 5.1395 seconds: | 20260507 19:36:00.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:00.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.705 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:36:00.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.706 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:36:00.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:00.706 | INFO | Uploaded 36GiB to GPU in 2.54828s (14.1272GiB/s) | 20260507 19:36:00.706 | INFO | Downloaded 34GiB to GPU in 2.44015s (13.9336GiB/s) | 20260507 19:36:05.845 | INFO | Analyzing chunk frames = [2055680,2056192) | 20260507 19:36:05.845 | INFO | MemInfo: { | 20260507 19:36:05.845 | INFO | "gpu" : " 17 GB", | 20260507 19:36:05.845 | INFO | "huge" : "", | 20260507 19:36:05.845 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:05.845 | INFO | "malloc" : " 14 GB", | 20260507 19:36:05.846 | INFO | "pinned" : " 29 GB" | 20260507 19:36:05.846 | INFO | } | 20260507 19:36:05.846 | INFO | Perf report for the last 5.14032 seconds: | 20260507 19:36:05.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:05.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:36:05.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:36:05.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:05.846 | INFO | Uploaded 36GiB to GPU in 2.53966s (14.1751GiB/s) | 20260507 19:36:05.846 | INFO | Downloaded 34GiB to GPU in 2.41365s (14.0866GiB/s) | 20260507 19:36:05.846 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:36:05.846 | INFO | Total mem usage: 16.4 GiB | 20260507 19:36:05.846 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:36:05.846 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.9%) | 20260507 19:36:05.846 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:36:05.846 | INFO | High Water Breakdown: | 20260507 19:36:05.846 | INFO | 60.2% : 9.15 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:36:05.846 | INFO | 39.7% : 6.03 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:36:05.846 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:36:05.846 | INFO | ------------------------------------------ | 20260507 19:36:05.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:36:05.846 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:36:05.846 | INFO | Total mem usage: 26.5 GiB | 20260507 19:36:05.846 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:36:05.846 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:36:05.846 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:36:05.846 | INFO | High Water Breakdown: | 20260507 19:36:05.846 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:36:05.846 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:36:05.846 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:36:05.846 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:36:05.846 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:36:05.846 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:36:05.846 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:36:05.846 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:36:05.846 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:36:05.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:36:05.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:36:05.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:36:05.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:36:05.846 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:36:05.846 | INFO | ------------------------------------------ | 20260507 19:36:05.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:36:05.846 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:36:05.846 | INFO | Total mem usage: 27.8 GiB | 20260507 19:36:05.846 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:36:05.846 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:36:05.846 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:36:05.846 | INFO | High Water Breakdown: | 20260507 19:36:05.846 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:36:05.847 | INFO | ------------------------------------------ | 20260507 19:36:05.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:36:05.847 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:36:05.847 | INFO | Total mem usage: 15.7 GiB | 20260507 19:36:05.847 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:36:05.847 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:36:05.847 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:36:05.847 | INFO | High Water Breakdown: | 20260507 19:36:05.847 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:36:05.847 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:36:05.847 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:36:05.847 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:36:05.847 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:36:05.847 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:36:05.847 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:36:05.847 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:36:05.847 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:36:05.847 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:36:05.847 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:36:05.847 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:36:05.847 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:36:05.847 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:36:05.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:36:05.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:36:05.847 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:36:05.847 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:36:05.847 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:36:05.847 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:36:05.847 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:36:05.847 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:36:05.847 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:36:05.847 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:36:05.847 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:36:05.847 | INFO | ------------------------------------------ | 20260507 19:36:05.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:36:05.847 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:36:05.847 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:36:05.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:36:05.847 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.41e+01 / 1.97e+00 | 20260507 19:36:05.847 | INFO | PerfReport: 0.29%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.06e-02 / 4.75e-03 | 20260507 19:36:05.848 | INFO | PerfReport: 12.07%, Baseline -- 4.00e+00 / 4.49e+00, 6.03e-01 / 2.72e-01 | 20260507 19:36:05.848 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.49e+00, 2.13e-01 / 1.23e-01 | 20260507 19:36:05.848 | INFO | PerfReport: 15.77%, DME -- 4.19e+01 / 4.50e+01, 3.76e+00 / 1.44e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.83e+00 / 2.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.43e-01 / 2.49e-01 | 20260507 19:36:05.848 | INFO | PerfReport: 3.65%, Metrics -- 1.21e+00 / 1.43e+00, 3.08e-01 / 1.75e-01 | 20260507 19:36:05.848 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.77e-01 / 4.10e-01, 6.05e-02 / 3.30e-02 | 20260507 19:36:05.848 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:36:05.848 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:36:05.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:36:05.848 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 24.43%, Upload -- 2.62e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 24.09%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 30.97%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 2.13%, LockUpload -- 2.28e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.69e+00 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 16.13%, LockCompute -- 1.73e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 19:36:05.848 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260507 19:36:05.943 | INFO | Starting a round of Baz IO | 20260507 19:36:09.421 | INFO | Populating RTMetrics report | 20260507 19:36:09.421 | INFO | RTMetrics report populated | 20260507 19:36:09.421 | INFO | Opening tmp RTMetrics json file | 20260507 19:36:09.451 | INFO | Serializing RTMetrics to json | 20260507 19:36:09.490 | INFO | Serialization done, swapping tmp file now... | 20260507 19:36:09.510 | INFO | RTMetrics done | 20260507 19:36:09.533 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.59 seconds | 20260507 19:36:09.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.26% / 0.01% / 54.73% | 20260507 19:36:09.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33345% / 2.78155e-05% / 3.17891e-05% | 20260507 19:36:09.533 | INFO | Baz IO overall: | 20260507 19:36:09.533 | INFO | BytesWritten : 929282384315 | 20260507 19:36:09.533 | INFO | NumEvents : 450551037414 | 20260507 19:36:10.984 | INFO | Analyzing chunk frames = [2056192,2056704) | 20260507 19:36:10.984 | INFO | MemInfo: { | 20260507 19:36:10.984 | INFO | "gpu" : " 17 GB", | 20260507 19:36:10.984 | INFO | "huge" : "", | 20260507 19:36:10.984 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:10.984 | INFO | "malloc" : " 14 GB", | 20260507 19:36:10.984 | INFO | "pinned" : " 29 GB" | 20260507 19:36:10.984 | INFO | } | 20260507 19:36:10.984 | INFO | Perf report for the last 5.13815 seconds: | 20260507 19:36:10.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:10.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:10.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:10.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:10.984 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 19:36:10.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:10.984 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 19:36:10.984 | INFO | PRE_HQ : 66.0%, 1.7, 58.2ms, 0 | 20260507 19:36:10.984 | INFO | BAZWRITER : 69.5%, 3.9, 137.5ms, 0 | 20260507 19:36:10.984 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 19:36:10.985 | INFO | Uploaded 36GiB to GPU in 2.55763s (14.0755GiB/s) | 20260507 19:36:10.985 | INFO | Downloaded 38GiB to GPU in 2.69192s (14.1163GiB/s) | 20260507 19:36:16.124 | INFO | Analyzing chunk frames = [2056704,2057216) | 20260507 19:36:16.124 | INFO | MemInfo: { | 20260507 19:36:16.124 | INFO | "gpu" : " 17 GB", | 20260507 19:36:16.124 | INFO | "huge" : "", | 20260507 19:36:16.124 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:16.124 | INFO | "malloc" : " 14 GB", | 20260507 19:36:16.124 | INFO | "pinned" : " 29 GB" | 20260507 19:36:16.124 | INFO | } | 20260507 19:36:16.124 | INFO | Perf report for the last 5.14059 seconds: | 20260507 19:36:16.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:16.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.125 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 19:36:16.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.125 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:36:16.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:16.125 | INFO | Uploaded 36GiB to GPU in 2.59291s (13.884GiB/s) | 20260507 19:36:16.125 | INFO | Downloaded 34GiB to GPU in 2.48464s (13.6841GiB/s) | 20260507 19:36:21.263 | INFO | Analyzing chunk frames = [2057216,2057728) | 20260507 19:36:21.263 | INFO | MemInfo: { | 20260507 19:36:21.263 | INFO | "gpu" : " 17 GB", | 20260507 19:36:21.263 | INFO | "huge" : "", | 20260507 19:36:21.263 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:21.264 | INFO | "malloc" : " 14 GB", | 20260507 19:36:21.264 | INFO | "pinned" : " 29 GB" | 20260507 19:36:21.264 | INFO | } | 20260507 19:36:21.264 | INFO | Perf report for the last 5.13919 seconds: | 20260507 19:36:21.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:21.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:36:21.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260507 19:36:21.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:21.264 | INFO | Uploaded 36GiB to GPU in 2.52526s (14.256GiB/s) | 20260507 19:36:21.264 | INFO | Downloaded 34GiB to GPU in 2.47819s (13.7197GiB/s) | 20260507 19:36:26.403 | INFO | Analyzing chunk frames = [2057728,2058240) | 20260507 19:36:26.403 | INFO | MemInfo: { | 20260507 19:36:26.403 | INFO | "gpu" : " 17 GB", | 20260507 19:36:26.403 | INFO | "huge" : "", | 20260507 19:36:26.403 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:26.403 | INFO | "malloc" : " 14 GB", | 20260507 19:36:26.403 | INFO | "pinned" : " 29 GB" | 20260507 19:36:26.403 | INFO | } | 20260507 19:36:26.403 | INFO | Perf report for the last 5.13929 seconds: | 20260507 19:36:26.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:26.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.403 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:36:26.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.403 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 19:36:26.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:26.404 | INFO | Uploaded 36GiB to GPU in 2.59834s (13.855GiB/s) | 20260507 19:36:26.404 | INFO | Downloaded 34GiB to GPU in 2.45424s (13.8536GiB/s) | 20260507 19:36:31.542 | INFO | Analyzing chunk frames = [2058240,2058752) | 20260507 19:36:31.542 | INFO | MemInfo: { | 20260507 19:36:31.542 | INFO | "gpu" : " 17 GB", | 20260507 19:36:31.542 | INFO | "huge" : "", | 20260507 19:36:31.542 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:31.542 | INFO | "malloc" : " 14 GB", | 20260507 19:36:31.542 | INFO | "pinned" : " 29 GB" | 20260507 19:36:31.542 | INFO | } | 20260507 19:36:31.542 | INFO | Perf report for the last 5.13888 seconds: | 20260507 19:36:31.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:31.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 19:36:31.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | PRE_HQ : 36.1%, 1.0, 18.8ms, 0 | 20260507 19:36:31.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:31.542 | INFO | Uploaded 36GiB to GPU in 2.43278s (14.7979GiB/s) | 20260507 19:36:31.542 | INFO | Downloaded 34GiB to GPU in 2.49402s (13.6326GiB/s) | 20260507 19:36:36.680 | INFO | Analyzing chunk frames = [2058752,2059264) | 20260507 19:36:36.680 | INFO | MemInfo: { | 20260507 19:36:36.680 | INFO | "gpu" : " 17 GB", | 20260507 19:36:36.680 | INFO | "huge" : "", | 20260507 19:36:36.680 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:36.680 | INFO | "malloc" : " 14 GB", | 20260507 19:36:36.680 | INFO | "pinned" : " 29 GB" | 20260507 19:36:36.680 | INFO | } | 20260507 19:36:36.681 | INFO | Perf report for the last 5.13861 seconds: | 20260507 19:36:36.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:36.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 19:36:36.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:36:36.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:36.681 | INFO | Uploaded 36GiB to GPU in 2.51566s (14.3103GiB/s) | 20260507 19:36:36.681 | INFO | Downloaded 34GiB to GPU in 2.41331s (14.0885GiB/s) | 20260507 19:36:41.821 | INFO | Analyzing chunk frames = [2059264,2059776) | 20260507 19:36:41.821 | INFO | MemInfo: { | 20260507 19:36:41.821 | INFO | "gpu" : " 17 GB", | 20260507 19:36:41.821 | INFO | "huge" : "", | 20260507 19:36:41.821 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:41.821 | INFO | "malloc" : " 14 GB", | 20260507 19:36:41.821 | INFO | "pinned" : " 29 GB" | 20260507 19:36:41.822 | INFO | } | 20260507 19:36:41.822 | INFO | Perf report for the last 5.14109 seconds: | 20260507 19:36:41.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:41.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:36:41.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:36:41.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:41.822 | INFO | Uploaded 36GiB to GPU in 2.53839s (14.1822GiB/s) | 20260507 19:36:41.822 | INFO | Downloaded 34GiB to GPU in 2.43391s (13.9693GiB/s) | 20260507 19:36:46.960 | INFO | Analyzing chunk frames = [2059776,2060288) | 20260507 19:36:46.960 | INFO | MemInfo: { | 20260507 19:36:46.960 | INFO | "gpu" : " 17 GB", | 20260507 19:36:46.960 | INFO | "huge" : "", | 20260507 19:36:46.960 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:46.960 | INFO | "malloc" : " 14 GB", | 20260507 19:36:46.960 | INFO | "pinned" : " 29 GB" | 20260507 19:36:46.960 | INFO | } | 20260507 19:36:46.960 | INFO | Perf report for the last 5.13842 seconds: | 20260507 19:36:46.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:46.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:36:46.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:36:46.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:46.960 | INFO | Uploaded 36GiB to GPU in 2.52638s (14.2496GiB/s) | 20260507 19:36:46.960 | INFO | Downloaded 34GiB to GPU in 2.41523s (14.0773GiB/s) | 20260507 19:36:47.057 | INFO | Starting a round of Baz IO | 20260507 19:36:50.543 | INFO | Populating RTMetrics report | 20260507 19:36:50.543 | INFO | RTMetrics report populated | 20260507 19:36:50.543 | INFO | Opening tmp RTMetrics json file | 20260507 19:36:50.565 | INFO | Serializing RTMetrics to json | 20260507 19:36:50.566 | INFO | Serialization done, swapping tmp file now... | 20260507 19:36:50.568 | INFO | RTMetrics done | 20260507 19:36:50.644 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.586 seconds | 20260507 19:36:50.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.24% / 0.01% / 54.74% | 20260507 19:36:50.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.32871% / 2.78155e-05% / 3.57628e-05% | 20260507 19:36:50.644 | INFO | Baz IO overall: | 20260507 19:36:50.644 | INFO | BytesWritten : 931202133063 | 20260507 19:36:50.644 | INFO | NumEvents : 451481338348 | 20260507 19:36:52.098 | INFO | Analyzing chunk frames = [2060288,2060800) | 20260507 19:36:52.098 | INFO | MemInfo: { | 20260507 19:36:52.098 | INFO | "gpu" : " 17 GB", | 20260507 19:36:52.098 | INFO | "huge" : "", | 20260507 19:36:52.098 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:52.098 | INFO | "malloc" : " 14 GB", | 20260507 19:36:52.098 | INFO | "pinned" : " 29 GB" | 20260507 19:36:52.098 | INFO | } | 20260507 19:36:52.098 | INFO | Perf report for the last 5.13844 seconds: | 20260507 19:36:52.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:52.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:52.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:52.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:52.099 | INFO | ANALYSIS : 69.3%, 2.9, 107.2ms, 0 | 20260507 19:36:52.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:52.099 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:36:52.099 | INFO | PRE_HQ : 66.3%, 1.7, 58.4ms, 0 | 20260507 19:36:52.099 | INFO | BAZWRITER : 69.4%, 4.4, 155.7ms, 0 | 20260507 19:36:52.099 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:36:52.099 | INFO | Uploaded 36GiB to GPU in 2.60969s (13.7947GiB/s) | 20260507 19:36:52.099 | INFO | Downloaded 38GiB to GPU in 2.69714s (14.089GiB/s) | 20260507 19:36:57.237 | INFO | Analyzing chunk frames = [2060800,2061312) | 20260507 19:36:57.237 | INFO | MemInfo: { | 20260507 19:36:57.237 | INFO | "gpu" : " 17 GB", | 20260507 19:36:57.237 | INFO | "huge" : "", | 20260507 19:36:57.237 | INFO | "huge_shared" : " 29 GB", | 20260507 19:36:57.237 | INFO | "malloc" : " 14 GB", | 20260507 19:36:57.237 | INFO | "pinned" : " 29 GB" | 20260507 19:36:57.237 | INFO | } | 20260507 19:36:57.237 | INFO | Perf report for the last 5.13885 seconds: | 20260507 19:36:57.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:36:57.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.237 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:36:57.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.237 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:36:57.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:36:57.238 | INFO | Uploaded 36GiB to GPU in 2.55603s (14.0843GiB/s) | 20260507 19:36:57.238 | INFO | Downloaded 34GiB to GPU in 2.47839s (13.7186GiB/s) | 20260507 19:37:02.377 | INFO | Analyzing chunk frames = [2061312,2061824) | 20260507 19:37:02.377 | INFO | MemInfo: { | 20260507 19:37:02.377 | INFO | "gpu" : " 17 GB", | 20260507 19:37:02.377 | INFO | "huge" : "", | 20260507 19:37:02.378 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:02.378 | INFO | "malloc" : " 14 GB", | 20260507 19:37:02.378 | INFO | "pinned" : " 29 GB" | 20260507 19:37:02.378 | INFO | } | 20260507 19:37:02.378 | INFO | Perf report for the last 5.14028 seconds: | 20260507 19:37:02.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:02.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:37:02.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:37:02.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:02.378 | INFO | Uploaded 36GiB to GPU in 2.5498s (14.1188GiB/s) | 20260507 19:37:02.378 | INFO | Downloaded 34GiB to GPU in 2.46418s (13.7977GiB/s) | 20260507 19:37:07.517 | INFO | Analyzing chunk frames = [2061824,2062336) | 20260507 19:37:07.517 | INFO | MemInfo: { | 20260507 19:37:07.517 | INFO | "gpu" : " 17 GB", | 20260507 19:37:07.517 | INFO | "huge" : "", | 20260507 19:37:07.517 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:07.517 | INFO | "malloc" : " 14 GB", | 20260507 19:37:07.517 | INFO | "pinned" : " 29 GB" | 20260507 19:37:07.517 | INFO | } | 20260507 19:37:07.517 | INFO | Perf report for the last 5.13981 seconds: | 20260507 19:37:07.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:07.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.518 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:37:07.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.518 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 19:37:07.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:07.518 | INFO | Uploaded 36GiB to GPU in 2.47911s (14.5213GiB/s) | 20260507 19:37:07.518 | INFO | Downloaded 34GiB to GPU in 2.40007s (14.1662GiB/s) | 20260507 19:37:12.657 | INFO | Analyzing chunk frames = [2062336,2062848) | 20260507 19:37:12.657 | INFO | MemInfo: { | 20260507 19:37:12.657 | INFO | "gpu" : " 17 GB", | 20260507 19:37:12.657 | INFO | "huge" : "", | 20260507 19:37:12.657 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:12.657 | INFO | "malloc" : " 14 GB", | 20260507 19:37:12.657 | INFO | "pinned" : " 29 GB" | 20260507 19:37:12.657 | INFO | } | 20260507 19:37:12.657 | INFO | Perf report for the last 5.13957 seconds: | 20260507 19:37:12.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:12.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 19:37:12.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 19:37:12.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:12.657 | INFO | Uploaded 36GiB to GPU in 2.36665s (15.2114GiB/s) | 20260507 19:37:12.657 | INFO | Downloaded 34GiB to GPU in 2.49685s (13.6172GiB/s) | 20260507 19:37:17.795 | INFO | Analyzing chunk frames = [2062848,2063360) | 20260507 19:37:17.795 | INFO | MemInfo: { | 20260507 19:37:17.795 | INFO | "gpu" : " 17 GB", | 20260507 19:37:17.795 | INFO | "huge" : "", | 20260507 19:37:17.795 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:17.795 | INFO | "malloc" : " 14 GB", | 20260507 19:37:17.795 | INFO | "pinned" : " 29 GB" | 20260507 19:37:17.795 | INFO | } | 20260507 19:37:17.795 | INFO | Perf report for the last 5.13822 seconds: | 20260507 19:37:17.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:17.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.795 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:37:17.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.795 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:37:17.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:17.796 | INFO | Uploaded 36GiB to GPU in 2.46872s (14.5824GiB/s) | 20260507 19:37:17.796 | INFO | Downloaded 34GiB to GPU in 2.4248s (14.0218GiB/s) | 20260507 19:37:22.934 | INFO | Analyzing chunk frames = [2063360,2063872) | 20260507 19:37:22.934 | INFO | MemInfo: { | 20260507 19:37:22.934 | INFO | "gpu" : " 17 GB", | 20260507 19:37:22.934 | INFO | "huge" : "", | 20260507 19:37:22.934 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:22.935 | INFO | "malloc" : " 14 GB", | 20260507 19:37:22.935 | INFO | "pinned" : " 29 GB" | 20260507 19:37:22.935 | INFO | } | 20260507 19:37:22.935 | INFO | Perf report for the last 5.13943 seconds: | 20260507 19:37:22.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:22.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:37:22.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:37:22.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:22.935 | INFO | Uploaded 36GiB to GPU in 2.52003s (14.2855GiB/s) | 20260507 19:37:22.935 | INFO | Downloaded 34GiB to GPU in 2.44761s (13.8911GiB/s) | 20260507 19:37:28.075 | INFO | Analyzing chunk frames = [2063872,2064384) | 20260507 19:37:28.075 | INFO | MemInfo: { | 20260507 19:37:28.075 | INFO | "gpu" : " 17 GB", | 20260507 19:37:28.075 | INFO | "huge" : "", | 20260507 19:37:28.075 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:28.075 | INFO | "malloc" : " 14 GB", | 20260507 19:37:28.075 | INFO | "pinned" : " 29 GB" | 20260507 19:37:28.075 | INFO | } | 20260507 19:37:28.075 | INFO | Perf report for the last 5.14034 seconds: | 20260507 19:37:28.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:28.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.075 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:37:28.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.075 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:37:28.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:28.076 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:37:28.076 | INFO | Total mem usage: 16.4 GiB | 20260507 19:37:28.076 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:37:28.076 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.67%) | 20260507 19:37:28.076 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:37:28.076 | INFO | High Water Breakdown: | 20260507 19:37:28.076 | INFO | 60.1% : 9.07 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:37:28.076 | INFO | 39.8% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:37:28.076 | INFO | Uploaded 36GiB to GPU in 2.48362s (14.495GiB/s) | 20260507 19:37:28.076 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:37:28.076 | INFO | Downloaded 34GiB to GPU in 2.42777s (14.0046GiB/s) | 20260507 19:37:28.076 | INFO | ------------------------------------------ | 20260507 19:37:28.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:37:28.076 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:37:28.076 | INFO | Total mem usage: 26.5 GiB | 20260507 19:37:28.076 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:37:28.076 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:37:28.076 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:37:28.076 | INFO | High Water Breakdown: | 20260507 19:37:28.076 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:37:28.076 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:37:28.076 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:37:28.076 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:37:28.076 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:37:28.076 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:37:28.076 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:37:28.076 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:37:28.076 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:37:28.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:37:28.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:37:28.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:37:28.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:37:28.076 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:37:28.076 | INFO | ------------------------------------------ | 20260507 19:37:28.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:37:28.076 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:37:28.076 | INFO | Total mem usage: 27.8 GiB | 20260507 19:37:28.076 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:37:28.076 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:37:28.076 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:37:28.076 | INFO | High Water Breakdown: | 20260507 19:37:28.076 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:37:28.076 | INFO | ------------------------------------------ | 20260507 19:37:28.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:37:28.076 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:37:28.076 | INFO | Total mem usage: 15.7 GiB | 20260507 19:37:28.076 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:37:28.076 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:37:28.076 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:37:28.076 | INFO | High Water Breakdown: | 20260507 19:37:28.076 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:37:28.076 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:37:28.076 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:37:28.076 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:37:28.076 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:37:28.077 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:37:28.077 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:37:28.077 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:37:28.077 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:37:28.077 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:37:28.077 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:37:28.077 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:37:28.077 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:37:28.077 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:37:28.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:37:28.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:37:28.077 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:37:28.077 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:37:28.077 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:37:28.077 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:37:28.077 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:37:28.077 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:37:28.077 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:37:28.077 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:37:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:37:28.077 | INFO | ------------------------------------------ | 20260507 19:37:28.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:37:28.077 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:37:28.077 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:37:28.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:37:28.077 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.42e+01 / 1.72e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 0.30%, AggregateStats -- 9.96e-02 / 1.04e-01, 1.27e-02 / 6.36e-03 | 20260507 19:37:28.077 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.48e+00, 6.03e-01 / 2.64e-01 | 20260507 19:37:28.077 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.07e-01 | 20260507 19:37:28.077 | INFO | PerfReport: 15.80%, DME -- 4.19e+01 / 4.50e+01, 4.01e+00 / 1.66e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.79e+00 / 1.53e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.52e+00 / 2.88e+00, 4.45e-01 / 2.08e-01 | 20260507 19:37:28.077 | INFO | PerfReport: 3.63%, Metrics -- 1.20e+00 / 1.41e+00, 3.03e-01 / 1.54e-01 | 20260507 19:37:28.077 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.79e-01 / 4.12e-01, 6.17e-02 / 3.08e-02 | 20260507 19:37:28.077 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:37:28.077 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:37:28.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:37:28.077 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 24.48%, Upload -- 2.63e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 24.10%, Download -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 30.92%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 2.21%, LockUpload -- 2.37e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 2.28%, LockDownload -- 2.44e+00 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 16.01%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:37:28.077 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260507 19:37:28.174 | INFO | Starting a round of Baz IO | 20260507 19:37:31.661 | INFO | Populating RTMetrics report | 20260507 19:37:31.662 | INFO | RTMetrics report populated | 20260507 19:37:31.662 | INFO | Opening tmp RTMetrics json file | 20260507 19:37:31.675 | INFO | Serializing RTMetrics to json | 20260507 19:37:31.682 | INFO | Serialization done, swapping tmp file now... | 20260507 19:37:31.683 | INFO | RTMetrics done | 20260507 19:37:31.753 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.579 seconds | 20260507 19:37:31.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.23% / 0.01% / 54.76% | 20260507 19:37:31.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33264% / 1.58946e-05% / 3.57628e-05% | 20260507 19:37:31.753 | INFO | Baz IO overall: | 20260507 19:37:31.753 | INFO | BytesWritten : 933123380934 | 20260507 19:37:31.753 | INFO | NumEvents : 452412900211 | 20260507 19:37:33.214 | INFO | Analyzing chunk frames = [2064384,2064896) | 20260507 19:37:33.214 | INFO | MemInfo: { | 20260507 19:37:33.214 | INFO | "gpu" : " 17 GB", | 20260507 19:37:33.214 | INFO | "huge" : "", | 20260507 19:37:33.214 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:33.214 | INFO | "malloc" : " 14 GB", | 20260507 19:37:33.214 | INFO | "pinned" : " 29 GB" | 20260507 19:37:33.214 | INFO | } | 20260507 19:37:33.214 | INFO | Perf report for the last 5.13918 seconds: | 20260507 19:37:33.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:33.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:33.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:33.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:33.214 | INFO | ANALYSIS : 69.0%, 3.0, 106.8ms, 0 | 20260507 19:37:33.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:33.214 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:37:33.214 | INFO | PRE_HQ : 66.2%, 1.7, 58.3ms, 0 | 20260507 19:37:33.214 | INFO | BAZWRITER : 69.3%, 3.8, 134.4ms, 0 | 20260507 19:37:33.215 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 19:37:33.215 | INFO | Uploaded 36GiB to GPU in 2.58526s (13.9251GiB/s) | 20260507 19:37:33.215 | INFO | Downloaded 38GiB to GPU in 2.6839s (14.1585GiB/s) | 20260507 19:37:38.353 | INFO | Analyzing chunk frames = [2064896,2065408) | 20260507 19:37:38.353 | INFO | MemInfo: { | 20260507 19:37:38.353 | INFO | "gpu" : " 17 GB", | 20260507 19:37:38.353 | INFO | "huge" : "", | 20260507 19:37:38.353 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:38.353 | INFO | "malloc" : " 14 GB", | 20260507 19:37:38.353 | INFO | "pinned" : " 29 GB" | 20260507 19:37:38.353 | INFO | } | 20260507 19:37:38.353 | INFO | Perf report for the last 5.13874 seconds: | 20260507 19:37:38.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:38.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.353 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:37:38.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.353 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:37:38.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:38.354 | INFO | Uploaded 36GiB to GPU in 2.56534s (14.0333GiB/s) | 20260507 19:37:38.354 | INFO | Downloaded 34GiB to GPU in 2.48154s (13.7012GiB/s) | 20260507 19:37:43.492 | INFO | Analyzing chunk frames = [2065408,2065920) | 20260507 19:37:43.492 | INFO | MemInfo: { | 20260507 19:37:43.492 | INFO | "gpu" : " 17 GB", | 20260507 19:37:43.492 | INFO | "huge" : "", | 20260507 19:37:43.492 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:43.492 | INFO | "malloc" : " 14 GB", | 20260507 19:37:43.493 | INFO | "pinned" : " 29 GB" | 20260507 19:37:43.493 | INFO | } | 20260507 19:37:43.493 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:37:43.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:43.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:37:43.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:37:43.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:43.493 | INFO | Uploaded 36GiB to GPU in 2.55753s (14.0761GiB/s) | 20260507 19:37:43.493 | INFO | Downloaded 34GiB to GPU in 2.45512s (13.8486GiB/s) | 20260507 19:37:48.631 | INFO | Analyzing chunk frames = [2065920,2066432) | 20260507 19:37:48.631 | INFO | MemInfo: { | 20260507 19:37:48.631 | INFO | "gpu" : " 17 GB", | 20260507 19:37:48.631 | INFO | "huge" : "", | 20260507 19:37:48.631 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:48.631 | INFO | "malloc" : " 14 GB", | 20260507 19:37:48.631 | INFO | "pinned" : " 29 GB" | 20260507 19:37:48.631 | INFO | } | 20260507 19:37:48.631 | INFO | Perf report for the last 5.13843 seconds: | 20260507 19:37:48.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:48.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:37:48.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:37:48.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:48.631 | INFO | Uploaded 36GiB to GPU in 2.47986s (14.5169GiB/s) | 20260507 19:37:48.631 | INFO | Downloaded 34GiB to GPU in 2.38622s (14.2485GiB/s) | 20260507 19:37:53.771 | INFO | Analyzing chunk frames = [2066432,2066944) | 20260507 19:37:53.771 | INFO | MemInfo: { | 20260507 19:37:53.771 | INFO | "gpu" : " 17 GB", | 20260507 19:37:53.771 | INFO | "huge" : "", | 20260507 19:37:53.771 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:53.771 | INFO | "malloc" : " 14 GB", | 20260507 19:37:53.771 | INFO | "pinned" : " 29 GB" | 20260507 19:37:53.771 | INFO | } | 20260507 19:37:53.771 | INFO | Perf report for the last 5.14024 seconds: | 20260507 19:37:53.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:53.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.771 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 19:37:53.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.771 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:37:53.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:53.772 | INFO | Uploaded 36GiB to GPU in 2.39979s (15.0013GiB/s) | 20260507 19:37:53.772 | INFO | Downloaded 34GiB to GPU in 2.49155s (13.6461GiB/s) | 20260507 19:37:58.910 | INFO | Analyzing chunk frames = [2066944,2067456) | 20260507 19:37:58.910 | INFO | MemInfo: { | 20260507 19:37:58.910 | INFO | "gpu" : " 17 GB", | 20260507 19:37:58.910 | INFO | "huge" : "", | 20260507 19:37:58.910 | INFO | "huge_shared" : " 29 GB", | 20260507 19:37:58.910 | INFO | "malloc" : " 14 GB", | 20260507 19:37:58.910 | INFO | "pinned" : " 29 GB" | 20260507 19:37:58.910 | INFO | } | 20260507 19:37:58.910 | INFO | Perf report for the last 5.13891 seconds: | 20260507 19:37:58.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:37:58.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:37:58.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:37:58.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:37:58.910 | INFO | Uploaded 36GiB to GPU in 2.53387s (14.2075GiB/s) | 20260507 19:37:58.910 | INFO | Downloaded 34GiB to GPU in 2.41504s (14.0784GiB/s) | 20260507 19:38:04.049 | INFO | Analyzing chunk frames = [2067456,2067968) | 20260507 19:38:04.049 | INFO | MemInfo: { | 20260507 19:38:04.049 | INFO | "gpu" : " 17 GB", | 20260507 19:38:04.049 | INFO | "huge" : "", | 20260507 19:38:04.049 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:04.049 | INFO | "malloc" : " 14 GB", | 20260507 19:38:04.049 | INFO | "pinned" : " 29 GB" | 20260507 19:38:04.049 | INFO | } | 20260507 19:38:04.049 | INFO | Perf report for the last 5.13899 seconds: | 20260507 19:38:04.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:04.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.049 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:38:04.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.049 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:38:04.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:04.050 | INFO | Uploaded 36GiB to GPU in 2.56565s (14.0315GiB/s) | 20260507 19:38:04.050 | INFO | Downloaded 34GiB to GPU in 2.43473s (13.9646GiB/s) | 20260507 19:38:09.189 | INFO | Analyzing chunk frames = [2067968,2068480) | 20260507 19:38:09.189 | INFO | MemInfo: { | 20260507 19:38:09.189 | INFO | "gpu" : " 17 GB", | 20260507 19:38:09.189 | INFO | "huge" : "", | 20260507 19:38:09.189 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:09.189 | INFO | "malloc" : " 14 GB", | 20260507 19:38:09.189 | INFO | "pinned" : " 29 GB" | 20260507 19:38:09.189 | INFO | } | 20260507 19:38:09.189 | INFO | Perf report for the last 5.13998 seconds: | 20260507 19:38:09.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:09.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.189 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:38:09.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.189 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:38:09.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:09.190 | INFO | Uploaded 36GiB to GPU in 2.52961s (14.2315GiB/s) | 20260507 19:38:09.190 | INFO | Downloaded 34GiB to GPU in 2.41105s (14.1018GiB/s) | 20260507 19:38:09.286 | INFO | Starting a round of Baz IO | 20260507 19:38:12.768 | INFO | Populating RTMetrics report | 20260507 19:38:12.768 | INFO | RTMetrics report populated | 20260507 19:38:12.768 | INFO | Opening tmp RTMetrics json file | 20260507 19:38:12.787 | INFO | Serializing RTMetrics to json | 20260507 19:38:12.789 | INFO | Serialization done, swapping tmp file now... | 20260507 19:38:12.790 | INFO | RTMetrics done | 20260507 19:38:12.867 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.58 seconds | 20260507 19:38:12.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.22% / 0.01% / 54.77% | 20260507 19:38:12.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33056% / 7.94729e-06% / 5.16574e-05% | 20260507 19:38:12.867 | INFO | Baz IO overall: | 20260507 19:38:12.867 | INFO | BytesWritten : 935046306209 | 20260507 19:38:12.867 | INFO | NumEvents : 453345608534 | 20260507 19:38:14.329 | INFO | Analyzing chunk frames = [2068480,2068992) | 20260507 19:38:14.329 | INFO | MemInfo: { | 20260507 19:38:14.329 | INFO | "gpu" : " 17 GB", | 20260507 19:38:14.329 | INFO | "huge" : "", | 20260507 19:38:14.329 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:14.329 | INFO | "malloc" : " 14 GB", | 20260507 19:38:14.329 | INFO | "pinned" : " 29 GB" | 20260507 19:38:14.329 | INFO | } | 20260507 19:38:14.329 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:38:14.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:14.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:14.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:14.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:14.329 | INFO | ANALYSIS : 68.9%, 3.0, 106.9ms, 0 | 20260507 19:38:14.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:14.329 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 19:38:14.329 | INFO | PRE_HQ : 66.1%, 1.7, 58.1ms, 0 | 20260507 19:38:14.329 | INFO | BAZWRITER : 69.3%, 4.2, 150.7ms, 0 | 20260507 19:38:14.329 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 19:38:14.330 | INFO | Uploaded 36GiB to GPU in 2.54621s (14.1387GiB/s) | 20260507 19:38:14.330 | INFO | Downloaded 38GiB to GPU in 2.69535s (14.0984GiB/s) | 20260507 19:38:19.468 | INFO | Analyzing chunk frames = [2068992,2069504) | 20260507 19:38:19.468 | INFO | MemInfo: { | 20260507 19:38:19.468 | INFO | "gpu" : " 17 GB", | 20260507 19:38:19.468 | INFO | "huge" : "", | 20260507 19:38:19.468 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:19.468 | INFO | "malloc" : " 14 GB", | 20260507 19:38:19.468 | INFO | "pinned" : " 29 GB" | 20260507 19:38:19.468 | INFO | } | 20260507 19:38:19.468 | INFO | Perf report for the last 5.13893 seconds: | 20260507 19:38:19.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:19.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:38:19.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 19:38:19.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:19.468 | INFO | Uploaded 36GiB to GPU in 2.56116s (14.0562GiB/s) | 20260507 19:38:19.468 | INFO | Downloaded 34GiB to GPU in 2.47587s (13.7325GiB/s) | 20260507 19:38:24.606 | INFO | Analyzing chunk frames = [2069504,2070016) | 20260507 19:38:24.606 | INFO | MemInfo: { | 20260507 19:38:24.606 | INFO | "gpu" : " 17 GB", | 20260507 19:38:24.606 | INFO | "huge" : "", | 20260507 19:38:24.607 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:24.607 | INFO | "malloc" : " 14 GB", | 20260507 19:38:24.607 | INFO | "pinned" : " 29 GB" | 20260507 19:38:24.607 | INFO | } | 20260507 19:38:24.607 | INFO | Perf report for the last 5.13872 seconds: | 20260507 19:38:24.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:24.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:38:24.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:38:24.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:24.607 | INFO | Uploaded 36GiB to GPU in 2.58085s (13.9489GiB/s) | 20260507 19:38:24.607 | INFO | Downloaded 34GiB to GPU in 2.47843s (13.7184GiB/s) | 20260507 19:38:29.746 | INFO | Analyzing chunk frames = [2070016,2070528) | 20260507 19:38:29.746 | INFO | MemInfo: { | 20260507 19:38:29.746 | INFO | "gpu" : " 17 GB", | 20260507 19:38:29.746 | INFO | "huge" : "", | 20260507 19:38:29.746 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:29.746 | INFO | "malloc" : " 14 GB", | 20260507 19:38:29.746 | INFO | "pinned" : " 29 GB" | 20260507 19:38:29.746 | INFO | } | 20260507 19:38:29.746 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:38:29.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:29.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:38:29.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:38:29.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:29.746 | INFO | Uploaded 36GiB to GPU in 2.46863s (14.583GiB/s) | 20260507 19:38:29.746 | INFO | Downloaded 34GiB to GPU in 2.39126s (14.2184GiB/s) | 20260507 19:38:34.887 | INFO | Analyzing chunk frames = [2070528,2071040) | 20260507 19:38:34.887 | INFO | MemInfo: { | 20260507 19:38:34.887 | INFO | "gpu" : " 17 GB", | 20260507 19:38:34.887 | INFO | "huge" : "", | 20260507 19:38:34.887 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:34.887 | INFO | "malloc" : " 14 GB", | 20260507 19:38:34.887 | INFO | "pinned" : " 29 GB" | 20260507 19:38:34.887 | INFO | } | 20260507 19:38:34.887 | INFO | Perf report for the last 5.14075 seconds: | 20260507 19:38:34.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:34.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:38:34.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:38:34.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:34.887 | INFO | Uploaded 36GiB to GPU in 2.36442s (15.2257GiB/s) | 20260507 19:38:34.887 | INFO | Downloaded 34GiB to GPU in 2.49751s (13.6135GiB/s) | 20260507 19:38:40.026 | INFO | Analyzing chunk frames = [2071040,2071552) | 20260507 19:38:40.026 | INFO | MemInfo: { | 20260507 19:38:40.026 | INFO | "gpu" : " 17 GB", | 20260507 19:38:40.026 | INFO | "huge" : "", | 20260507 19:38:40.026 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:40.026 | INFO | "malloc" : " 14 GB", | 20260507 19:38:40.026 | INFO | "pinned" : " 29 GB" | 20260507 19:38:40.026 | INFO | } | 20260507 19:38:40.026 | INFO | Perf report for the last 5.13909 seconds: | 20260507 19:38:40.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:40.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:38:40.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:38:40.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:40.026 | INFO | Uploaded 36GiB to GPU in 2.48025s (14.5147GiB/s) | 20260507 19:38:40.026 | INFO | Downloaded 34GiB to GPU in 2.42303s (14.032GiB/s) | 20260507 19:38:45.165 | INFO | Analyzing chunk frames = [2071552,2072064) | 20260507 19:38:45.165 | INFO | MemInfo: { | 20260507 19:38:45.165 | INFO | "gpu" : " 17 GB", | 20260507 19:38:45.165 | INFO | "huge" : "", | 20260507 19:38:45.165 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:45.165 | INFO | "malloc" : " 14 GB", | 20260507 19:38:45.165 | INFO | "pinned" : " 29 GB" | 20260507 19:38:45.165 | INFO | } | 20260507 19:38:45.165 | INFO | Perf report for the last 5.1393 seconds: | 20260507 19:38:45.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:45.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.165 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:38:45.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.165 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:38:45.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:45.166 | INFO | Uploaded 36GiB to GPU in 2.51947s (14.2887GiB/s) | 20260507 19:38:45.166 | INFO | Downloaded 34GiB to GPU in 2.44458s (13.9083GiB/s) | 20260507 19:38:50.304 | INFO | Analyzing chunk frames = [2072064,2072576) | 20260507 19:38:50.304 | INFO | MemInfo: { | 20260507 19:38:50.304 | INFO | "gpu" : " 17 GB", | 20260507 19:38:50.304 | INFO | "huge" : "", | 20260507 19:38:50.304 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:50.304 | INFO | "malloc" : " 14 GB", | 20260507 19:38:50.304 | INFO | "pinned" : " 29 GB" | 20260507 19:38:50.304 | INFO | } | 20260507 19:38:50.305 | INFO | Perf report for the last 5.13942 seconds: | 20260507 19:38:50.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:50.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:38:50.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:38:50.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:50.305 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:38:50.305 | INFO | Total mem usage: 16.4 GiB | 20260507 19:38:50.305 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:38:50.305 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.89%) | 20260507 19:38:50.305 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:38:50.305 | INFO | High Water Breakdown: | 20260507 19:38:50.305 | INFO | 60.2% : 9.15 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:38:50.305 | INFO | 39.7% : 6.03 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:38:50.305 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:38:50.305 | INFO | ------------------------------------------ | 20260507 19:38:50.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:38:50.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:38:50.305 | INFO | Total mem usage: 26.5 GiB | 20260507 19:38:50.305 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:38:50.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:38:50.305 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:38:50.305 | INFO | High Water Breakdown: | 20260507 19:38:50.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:38:50.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:38:50.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:38:50.305 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:38:50.305 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:38:50.305 | INFO | Uploaded 36GiB to GPU in 2.49728s (14.4157GiB/s) | 20260507 19:38:50.305 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:38:50.305 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:38:50.305 | INFO | Downloaded 34GiB to GPU in 2.42964s (13.9938GiB/s) | 20260507 19:38:50.306 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:38:50.306 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:38:50.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:38:50.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:38:50.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:38:50.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:38:50.306 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:38:50.306 | INFO | ------------------------------------------ | 20260507 19:38:50.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:38:50.306 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:38:50.306 | INFO | Total mem usage: 27.8 GiB | 20260507 19:38:50.306 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:38:50.306 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:38:50.306 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:38:50.306 | INFO | High Water Breakdown: | 20260507 19:38:50.306 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:38:50.306 | INFO | ------------------------------------------ | 20260507 19:38:50.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:38:50.306 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:38:50.306 | INFO | Total mem usage: 15.7 GiB | 20260507 19:38:50.306 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:38:50.306 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:38:50.306 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:38:50.306 | INFO | High Water Breakdown: | 20260507 19:38:50.306 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:38:50.306 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:38:50.306 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:38:50.306 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:38:50.306 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:38:50.306 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:38:50.306 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:38:50.306 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:38:50.306 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:38:50.306 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:38:50.306 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:38:50.306 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:38:50.306 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:38:50.306 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:38:50.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:38:50.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:38:50.306 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:38:50.306 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:38:50.306 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:38:50.306 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:38:50.306 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:38:50.307 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:38:50.307 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:38:50.307 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:38:50.307 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:38:50.307 | INFO | ------------------------------------------ | 20260507 19:38:50.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:38:50.307 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:38:50.307 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:38:50.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:38:50.307 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.40e+01 / 2.02e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 0.29%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.15e-02 / 5.38e-03 | 20260507 19:38:50.307 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.48e+00, 6.04e-01 / 2.80e-01 | 20260507 19:38:50.307 | INFO | PerfReport: 3.41%, Histogram -- 1.13e+00 / 1.49e+00, 2.13e-01 / 1.22e-01 | 20260507 19:38:50.307 | INFO | PerfReport: 15.77%, DME -- 4.18e+01 / 4.50e+01, 3.84e+00 / 1.37e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.87e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.42e-01 / 2.17e-01 | 20260507 19:38:50.307 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.43e+00, 3.09e-01 / 1.63e-01 | 20260507 19:38:50.307 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.76e-01 / 4.09e-01, 6.16e-02 / 3.27e-02 | 20260507 19:38:50.307 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:38:50.307 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:38:50.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:38:50.307 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 24.46%, Upload -- 2.62e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 24.07%, Download -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 30.96%, Compute -- 3.32e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 2.14%, LockUpload -- 2.29e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 16.11%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:38:50.307 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260507 19:38:50.413 | INFO | Starting a round of Baz IO | 20260507 19:38:53.892 | INFO | Populating RTMetrics report | 20260507 19:38:53.893 | INFO | RTMetrics report populated | 20260507 19:38:53.893 | INFO | Opening tmp RTMetrics json file | 20260507 19:38:53.919 | INFO | Serializing RTMetrics to json | 20260507 19:38:53.921 | INFO | Serialization done, swapping tmp file now... | 20260507 19:38:53.923 | INFO | RTMetrics done | 20260507 19:38:54.013 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.599 seconds | 20260507 19:38:54.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.21% / 0.01% / 54.78% | 20260507 19:38:54.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31611% / 3.17891e-05% / 5.5631e-05% | 20260507 19:38:54.013 | INFO | Baz IO overall: | 20260507 19:38:54.013 | INFO | BytesWritten : 936966081992 | 20260507 19:38:54.013 | INFO | NumEvents : 454276021347 | 20260507 19:38:55.442 | INFO | Analyzing chunk frames = [2072576,2073088) | 20260507 19:38:55.442 | INFO | MemInfo: { | 20260507 19:38:55.442 | INFO | "gpu" : " 17 GB", | 20260507 19:38:55.442 | INFO | "huge" : "", | 20260507 19:38:55.442 | INFO | "huge_shared" : " 29 GB", | 20260507 19:38:55.442 | INFO | "malloc" : " 14 GB", | 20260507 19:38:55.442 | INFO | "pinned" : " 29 GB" | 20260507 19:38:55.442 | INFO | } | 20260507 19:38:55.442 | INFO | Perf report for the last 5.13724 seconds: | 20260507 19:38:55.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:38:55.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:55.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:55.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:55.442 | INFO | ANALYSIS : 69.1%, 3.0, 106.9ms, 0 | 20260507 19:38:55.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:38:55.442 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 19:38:55.442 | INFO | PRE_HQ : 66.2%, 1.7, 59.8ms, 0 | 20260507 19:38:55.442 | INFO | BAZWRITER : 69.6%, 3.9, 137.6ms, 0 | 20260507 19:38:55.442 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260507 19:38:55.443 | INFO | Uploaded 36GiB to GPU in 2.56525s (14.0337GiB/s) | 20260507 19:38:55.443 | INFO | Downloaded 38GiB to GPU in 2.69114s (14.1204GiB/s) | 20260507 19:39:00.582 | INFO | Analyzing chunk frames = [2073088,2073600) | 20260507 19:39:00.582 | INFO | MemInfo: { | 20260507 19:39:00.582 | INFO | "gpu" : " 17 GB", | 20260507 19:39:00.582 | INFO | "huge" : "", | 20260507 19:39:00.582 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:00.582 | INFO | "malloc" : " 14 GB", | 20260507 19:39:00.582 | INFO | "pinned" : " 29 GB" | 20260507 19:39:00.582 | INFO | } | 20260507 19:39:00.582 | INFO | Perf report for the last 5.14014 seconds: | 20260507 19:39:00.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:00.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.582 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:39:00.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.582 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:39:00.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:00.583 | INFO | Uploaded 36GiB to GPU in 2.55918s (14.067GiB/s) | 20260507 19:39:00.583 | INFO | Downloaded 34GiB to GPU in 2.49539s (13.6251GiB/s) | 20260507 19:39:05.722 | INFO | Analyzing chunk frames = [2073600,2074112) | 20260507 19:39:05.722 | INFO | MemInfo: { | 20260507 19:39:05.722 | INFO | "gpu" : " 17 GB", | 20260507 19:39:05.722 | INFO | "huge" : "", | 20260507 19:39:05.722 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:05.722 | INFO | "malloc" : " 14 GB", | 20260507 19:39:05.722 | INFO | "pinned" : " 29 GB" | 20260507 19:39:05.722 | INFO | } | 20260507 19:39:05.722 | INFO | Perf report for the last 5.14038 seconds: | 20260507 19:39:05.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:05.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 19:39:05.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:39:05.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:05.723 | INFO | Uploaded 36GiB to GPU in 2.53171s (14.2196GiB/s) | 20260507 19:39:05.723 | INFO | Downloaded 34GiB to GPU in 2.47228s (13.7525GiB/s) | 20260507 19:39:10.860 | INFO | Analyzing chunk frames = [2074112,2074624) | 20260507 19:39:10.860 | INFO | MemInfo: { | 20260507 19:39:10.860 | INFO | "gpu" : " 17 GB", | 20260507 19:39:10.860 | INFO | "huge" : "", | 20260507 19:39:10.860 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:10.860 | INFO | "malloc" : " 14 GB", | 20260507 19:39:10.860 | INFO | "pinned" : " 29 GB" | 20260507 19:39:10.860 | INFO | } | 20260507 19:39:10.860 | INFO | Perf report for the last 5.13792 seconds: | 20260507 19:39:10.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:10.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.860 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:39:10.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.860 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:39:10.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:10.861 | INFO | Uploaded 36GiB to GPU in 2.46319s (14.6152GiB/s) | 20260507 19:39:10.861 | INFO | Downloaded 34GiB to GPU in 2.40557s (14.1339GiB/s) | 20260507 19:39:16.000 | INFO | Analyzing chunk frames = [2074624,2075136) | 20260507 19:39:16.000 | INFO | MemInfo: { | 20260507 19:39:16.000 | INFO | "gpu" : " 17 GB", | 20260507 19:39:16.000 | INFO | "huge" : "", | 20260507 19:39:16.000 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:16.001 | INFO | "malloc" : " 14 GB", | 20260507 19:39:16.001 | INFO | "pinned" : " 29 GB" | 20260507 19:39:16.001 | INFO | } | 20260507 19:39:16.001 | INFO | Perf report for the last 5.14025 seconds: | 20260507 19:39:16.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:16.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:39:16.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:39:16.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:16.001 | INFO | Uploaded 36GiB to GPU in 2.35491s (15.2872GiB/s) | 20260507 19:39:16.001 | INFO | Downloaded 34GiB to GPU in 2.49803s (13.6107GiB/s) | 20260507 19:39:21.140 | INFO | Analyzing chunk frames = [2075136,2075648) | 20260507 19:39:21.140 | INFO | MemInfo: { | 20260507 19:39:21.140 | INFO | "gpu" : " 17 GB", | 20260507 19:39:21.140 | INFO | "huge" : "", | 20260507 19:39:21.140 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:21.140 | INFO | "malloc" : " 14 GB", | 20260507 19:39:21.140 | INFO | "pinned" : " 29 GB" | 20260507 19:39:21.140 | INFO | } | 20260507 19:39:21.140 | INFO | Perf report for the last 5.13936 seconds: | 20260507 19:39:21.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:21.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:39:21.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:39:21.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:21.140 | INFO | Uploaded 36GiB to GPU in 2.48474s (14.4884GiB/s) | 20260507 19:39:21.141 | INFO | Downloaded 34GiB to GPU in 2.42156s (14.0405GiB/s) | 20260507 19:39:26.279 | INFO | Analyzing chunk frames = [2075648,2076160) | 20260507 19:39:26.279 | INFO | MemInfo: { | 20260507 19:39:26.279 | INFO | "gpu" : " 17 GB", | 20260507 19:39:26.279 | INFO | "huge" : "", | 20260507 19:39:26.279 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:26.279 | INFO | "malloc" : " 14 GB", | 20260507 19:39:26.279 | INFO | "pinned" : " 29 GB" | 20260507 19:39:26.280 | INFO | } | 20260507 19:39:26.280 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:39:26.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:26.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.280 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:39:26.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.280 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 19:39:26.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:26.281 | INFO | Uploaded 36GiB to GPU in 2.51041s (14.3403GiB/s) | 20260507 19:39:26.281 | INFO | Downloaded 34GiB to GPU in 2.44809s (13.8884GiB/s) | 20260507 19:39:31.418 | INFO | Analyzing chunk frames = [2076160,2076672) | 20260507 19:39:31.418 | INFO | MemInfo: { | 20260507 19:39:31.418 | INFO | "gpu" : " 17 GB", | 20260507 19:39:31.418 | INFO | "huge" : "", | 20260507 19:39:31.418 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:31.418 | INFO | "malloc" : " 14 GB", | 20260507 19:39:31.418 | INFO | "pinned" : " 29 GB" | 20260507 19:39:31.418 | INFO | } | 20260507 19:39:31.418 | INFO | Perf report for the last 5.13883 seconds: | 20260507 19:39:31.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:31.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.418 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:39:31.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.419 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260507 19:39:31.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:31.419 | INFO | Uploaded 36GiB to GPU in 2.49785s (14.4124GiB/s) | 20260507 19:39:31.419 | INFO | Downloaded 34GiB to GPU in 2.44348s (13.9146GiB/s) | 20260507 19:39:31.527 | INFO | Starting a round of Baz IO | 20260507 19:39:34.999 | INFO | Populating RTMetrics report | 20260507 19:39:34.999 | INFO | RTMetrics report populated | 20260507 19:39:34.999 | INFO | Opening tmp RTMetrics json file | 20260507 19:39:35.000 | INFO | Serializing RTMetrics to json | 20260507 19:39:35.007 | INFO | Serialization done, swapping tmp file now... | 20260507 19:39:35.024 | INFO | RTMetrics done | 20260507 19:39:35.098 | INFO | Finished a round of Baz IO: Wrote 1.789 GiB in 3.571 seconds | 20260507 19:39:35.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.20% / 0.01% / 54.79% | 20260507 19:39:35.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31355% / 1.58946e-05% / 4.76837e-05% | 20260507 19:39:35.098 | INFO | Baz IO overall: | 20260507 19:39:35.098 | INFO | BytesWritten : 938887244199 | 20260507 19:39:35.099 | INFO | NumEvents : 455207413170 | 20260507 19:39:36.557 | INFO | Analyzing chunk frames = [2076672,2077184) | 20260507 19:39:36.557 | INFO | MemInfo: { | 20260507 19:39:36.557 | INFO | "gpu" : " 17 GB", | 20260507 19:39:36.557 | INFO | "huge" : "", | 20260507 19:39:36.557 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:36.557 | INFO | "malloc" : " 14 GB", | 20260507 19:39:36.557 | INFO | "pinned" : " 29 GB" | 20260507 19:39:36.557 | INFO | } | 20260507 19:39:36.557 | INFO | Perf report for the last 5.13846 seconds: | 20260507 19:39:36.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:36.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:36.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:36.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:36.557 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 19:39:36.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:36.557 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:39:36.557 | INFO | PRE_HQ : 66.4%, 1.7, 58.8ms, 0 | 20260507 19:39:36.557 | INFO | BAZWRITER : 69.3%, 4.7, 165.7ms, 0 | 20260507 19:39:36.557 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:39:36.557 | INFO | Uploaded 36GiB to GPU in 2.56456s (14.0375GiB/s) | 20260507 19:39:36.557 | INFO | Downloaded 38GiB to GPU in 2.7038s (14.0543GiB/s) | 20260507 19:39:41.697 | INFO | Analyzing chunk frames = [2077184,2077696) | 20260507 19:39:41.697 | INFO | MemInfo: { | 20260507 19:39:41.697 | INFO | "gpu" : " 17 GB", | 20260507 19:39:41.697 | INFO | "huge" : "", | 20260507 19:39:41.697 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:41.697 | INFO | "malloc" : " 14 GB", | 20260507 19:39:41.697 | INFO | "pinned" : " 29 GB" | 20260507 19:39:41.697 | INFO | } | 20260507 19:39:41.697 | INFO | Perf report for the last 5.14014 seconds: | 20260507 19:39:41.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:41.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.697 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:39:41.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.697 | INFO | PRE_HQ : 26.2%, 1.0, 13.7ms, 0 | 20260507 19:39:41.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:41.698 | INFO | Uploaded 36GiB to GPU in 2.55072s (14.1137GiB/s) | 20260507 19:39:41.698 | INFO | Downloaded 34GiB to GPU in 2.48363s (13.6896GiB/s) | 20260507 19:39:46.837 | INFO | Analyzing chunk frames = [2077696,2078208) | 20260507 19:39:46.837 | INFO | MemInfo: { | 20260507 19:39:46.837 | INFO | "gpu" : " 17 GB", | 20260507 19:39:46.837 | INFO | "huge" : "", | 20260507 19:39:46.837 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:46.837 | INFO | "malloc" : " 14 GB", | 20260507 19:39:46.837 | INFO | "pinned" : " 29 GB" | 20260507 19:39:46.837 | INFO | } | 20260507 19:39:46.837 | INFO | Perf report for the last 5.1398 seconds: | 20260507 19:39:46.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:46.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:39:46.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:39:46.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:46.837 | INFO | Uploaded 36GiB to GPU in 2.57387s (13.9867GiB/s) | 20260507 19:39:46.838 | INFO | Downloaded 34GiB to GPU in 2.48141s (13.7019GiB/s) | 20260507 19:39:51.977 | INFO | Analyzing chunk frames = [2078208,2078720) | 20260507 19:39:51.977 | INFO | MemInfo: { | 20260507 19:39:51.977 | INFO | "gpu" : " 17 GB", | 20260507 19:39:51.977 | INFO | "huge" : "", | 20260507 19:39:51.977 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:51.977 | INFO | "malloc" : " 14 GB", | 20260507 19:39:51.977 | INFO | "pinned" : " 29 GB" | 20260507 19:39:51.977 | INFO | } | 20260507 19:39:51.977 | INFO | Perf report for the last 5.13986 seconds: | 20260507 19:39:51.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:51.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 19:39:51.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:39:51.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:51.977 | INFO | Uploaded 36GiB to GPU in 2.47512s (14.5448GiB/s) | 20260507 19:39:51.977 | INFO | Downloaded 34GiB to GPU in 2.40878s (14.115GiB/s) | 20260507 19:39:57.115 | INFO | Analyzing chunk frames = [2078720,2079232) | 20260507 19:39:57.115 | INFO | MemInfo: { | 20260507 19:39:57.115 | INFO | "gpu" : " 17 GB", | 20260507 19:39:57.115 | INFO | "huge" : "", | 20260507 19:39:57.115 | INFO | "huge_shared" : " 29 GB", | 20260507 19:39:57.115 | INFO | "malloc" : " 14 GB", | 20260507 19:39:57.115 | INFO | "pinned" : " 29 GB" | 20260507 19:39:57.115 | INFO | } | 20260507 19:39:57.115 | INFO | Perf report for the last 5.13842 seconds: | 20260507 19:39:57.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:39:57.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:39:57.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:39:57.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:39:57.115 | INFO | Uploaded 36GiB to GPU in 2.35031s (15.3171GiB/s) | 20260507 19:39:57.115 | INFO | Downloaded 34GiB to GPU in 2.49989s (13.6006GiB/s) | 20260507 19:40:02.253 | INFO | Analyzing chunk frames = [2079232,2079744) | 20260507 19:40:02.253 | INFO | MemInfo: { | 20260507 19:40:02.253 | INFO | "gpu" : " 17 GB", | 20260507 19:40:02.253 | INFO | "huge" : "", | 20260507 19:40:02.254 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:02.254 | INFO | "malloc" : " 14 GB", | 20260507 19:40:02.254 | INFO | "pinned" : " 29 GB" | 20260507 19:40:02.254 | INFO | } | 20260507 19:40:02.254 | INFO | Perf report for the last 5.13853 seconds: | 20260507 19:40:02.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:02.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:40:02.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 19:40:02.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:02.254 | INFO | Uploaded 36GiB to GPU in 2.46725s (14.5911GiB/s) | 20260507 19:40:02.254 | INFO | Downloaded 34GiB to GPU in 2.41574s (14.0743GiB/s) | 20260507 19:40:07.393 | INFO | Analyzing chunk frames = [2079744,2080256) | 20260507 19:40:07.393 | INFO | MemInfo: { | 20260507 19:40:07.393 | INFO | "gpu" : " 17 GB", | 20260507 19:40:07.393 | INFO | "huge" : "", | 20260507 19:40:07.393 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:07.393 | INFO | "malloc" : " 14 GB", | 20260507 19:40:07.393 | INFO | "pinned" : " 29 GB" | 20260507 19:40:07.393 | INFO | } | 20260507 19:40:07.393 | INFO | Perf report for the last 5.13965 seconds: | 20260507 19:40:07.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:07.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.393 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:40:07.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.394 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:40:07.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:07.394 | INFO | Uploaded 36GiB to GPU in 2.50032s (14.3981GiB/s) | 20260507 19:40:07.394 | INFO | Downloaded 34GiB to GPU in 2.44659s (13.8969GiB/s) | 20260507 19:40:12.532 | INFO | Analyzing chunk frames = [2080256,2080768) | 20260507 19:40:12.532 | INFO | MemInfo: { | 20260507 19:40:12.532 | INFO | "gpu" : " 17 GB", | 20260507 19:40:12.532 | INFO | "huge" : "", | 20260507 19:40:12.532 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:12.532 | INFO | "malloc" : " 14 GB", | 20260507 19:40:12.532 | INFO | "pinned" : " 29 GB" | 20260507 19:40:12.532 | INFO | } | 20260507 19:40:12.532 | INFO | Perf report for the last 5.13906 seconds: | 20260507 19:40:12.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:12.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.533 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:40:12.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.533 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:40:12.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:12.533 | INFO | Uploaded 36GiB to GPU in 2.48842s (14.467GiB/s) | 20260507 19:40:12.533 | INFO | Downloaded 34GiB to GPU in 2.42609s (14.0143GiB/s) | 20260507 19:40:12.533 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:40:12.533 | INFO | Total mem usage: 16.4 GiB | 20260507 19:40:12.533 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:40:12.533 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -1.97%) | 20260507 19:40:12.533 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:40:12.533 | INFO | High Water Breakdown: | 20260507 19:40:12.533 | INFO | 61.1% : 9.83 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:40:12.533 | INFO | 38.8% : 6.24 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:40:12.533 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:40:12.533 | INFO | ------------------------------------------ | 20260507 19:40:12.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:40:12.533 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:40:12.533 | INFO | Total mem usage: 26.5 GiB | 20260507 19:40:12.533 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:40:12.533 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:40:12.533 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:40:12.533 | INFO | High Water Breakdown: | 20260507 19:40:12.533 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:40:12.533 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:40:12.533 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:40:12.533 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:40:12.533 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:40:12.533 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:40:12.533 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:40:12.533 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:40:12.533 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:40:12.533 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:40:12.533 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:40:12.533 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:40:12.533 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:40:12.533 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:40:12.533 | INFO | ------------------------------------------ | 20260507 19:40:12.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:40:12.533 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:40:12.533 | INFO | Total mem usage: 27.8 GiB | 20260507 19:40:12.533 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:40:12.533 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:40:12.533 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:40:12.533 | INFO | High Water Breakdown: | 20260507 19:40:12.533 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:40:12.533 | INFO | ------------------------------------------ | 20260507 19:40:12.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:40:12.533 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:40:12.533 | INFO | Total mem usage: 15.7 GiB | 20260507 19:40:12.533 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:40:12.533 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:40:12.533 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:40:12.533 | INFO | High Water Breakdown: | 20260507 19:40:12.533 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:40:12.533 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:40:12.533 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:40:12.533 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:40:12.533 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:40:12.533 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:40:12.533 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:40:12.533 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:40:12.533 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:40:12.533 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:40:12.533 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:40:12.533 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:40:12.533 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:40:12.533 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:40:12.533 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:40:12.533 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:40:12.533 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:40:12.533 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:40:12.533 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:40:12.533 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:40:12.533 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:40:12.533 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:40:12.533 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:40:12.533 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:40:12.533 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:40:12.533 | INFO | ------------------------------------------ | 20260507 19:40:12.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:40:12.533 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:40:12.533 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:40:12.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:40:12.533 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 1.88e+00 | 20260507 19:40:12.533 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.03e-01, 1.26e-02 / 5.45e-03 | 20260507 19:40:12.533 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.48e+00, 6.12e-01 / 2.59e-01 | 20260507 19:40:12.533 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.13e-01 / 1.23e-01 | 20260507 19:40:12.534 | INFO | PerfReport: 15.75%, DME -- 4.18e+01 / 4.49e+01, 3.83e+00 / 1.28e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.79e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.46e-01 / 1.99e-01 | 20260507 19:40:12.534 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.43e+00, 3.08e-01 / 1.72e-01 | 20260507 19:40:12.534 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.76e-01 / 4.10e-01, 6.12e-02 / 3.27e-02 | 20260507 19:40:12.534 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:40:12.534 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:40:12.534 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:40:12.534 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 24.30%, Upload -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 24.18%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 31.01%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 1.96%, LockUpload -- 2.10e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 2.39%, LockDownload -- 2.56e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 16.14%, LockCompute -- 1.73e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:40:12.534 | INFO | PerfReport: 0.01%, Misc -- 1.54e-02 / 0.00e+00, 3.25e-02 / 0.00e+00 | 20260507 19:40:12.642 | INFO | Starting a round of Baz IO | 20260507 19:40:16.108 | INFO | Populating RTMetrics report | 20260507 19:40:16.108 | INFO | RTMetrics report populated | 20260507 19:40:16.108 | INFO | Opening tmp RTMetrics json file | 20260507 19:40:16.132 | INFO | Serializing RTMetrics to json | 20260507 19:40:16.141 | INFO | Serialization done, swapping tmp file now... | 20260507 19:40:16.158 | INFO | RTMetrics done | 20260507 19:40:16.220 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.577 seconds | 20260507 19:40:16.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.19% / 0.01% / 54.80% | 20260507 19:40:16.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.30835% / 2.38419e-05% / 4.37101e-05% | 20260507 19:40:16.220 | INFO | Baz IO overall: | 20260507 19:40:16.220 | INFO | BytesWritten : 940807428375 | 20260507 19:40:16.220 | INFO | NumEvents : 456138071491 | 20260507 19:40:17.673 | INFO | Analyzing chunk frames = [2080768,2081280) | 20260507 19:40:17.673 | INFO | MemInfo: { | 20260507 19:40:17.673 | INFO | "gpu" : " 17 GB", | 20260507 19:40:17.673 | INFO | "huge" : "", | 20260507 19:40:17.673 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:17.673 | INFO | "malloc" : " 14 GB", | 20260507 19:40:17.673 | INFO | "pinned" : " 29 GB" | 20260507 19:40:17.673 | INFO | } | 20260507 19:40:17.673 | INFO | Perf report for the last 5.14038 seconds: | 20260507 19:40:17.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:17.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:17.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:17.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:17.673 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 19:40:17.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:17.673 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 19:40:17.673 | INFO | PRE_HQ : 66.4%, 1.7, 59.0ms, 0 | 20260507 19:40:17.673 | INFO | BAZWRITER : 69.5%, 3.9, 140.4ms, 0 | 20260507 19:40:17.673 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 19:40:17.674 | INFO | Uploaded 36GiB to GPU in 2.54763s (14.1308GiB/s) | 20260507 19:40:17.674 | INFO | Downloaded 38GiB to GPU in 2.70244s (14.0614GiB/s) | 20260507 19:40:22.812 | INFO | Analyzing chunk frames = [2081280,2081792) | 20260507 19:40:22.812 | INFO | MemInfo: { | 20260507 19:40:22.812 | INFO | "gpu" : " 17 GB", | 20260507 19:40:22.812 | INFO | "huge" : "", | 20260507 19:40:22.812 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:22.812 | INFO | "malloc" : " 14 GB", | 20260507 19:40:22.812 | INFO | "pinned" : " 29 GB" | 20260507 19:40:22.812 | INFO | } | 20260507 19:40:22.812 | INFO | Perf report for the last 5.13936 seconds: | 20260507 19:40:22.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:22.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.812 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:40:22.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.812 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:40:22.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:22.813 | INFO | Uploaded 36GiB to GPU in 2.54565s (14.1418GiB/s) | 20260507 19:40:22.813 | INFO | Downloaded 34GiB to GPU in 2.48219s (13.6976GiB/s) | 20260507 19:40:27.950 | INFO | Analyzing chunk frames = [2081792,2082304) | 20260507 19:40:27.950 | INFO | MemInfo: { | 20260507 19:40:27.950 | INFO | "gpu" : " 17 GB", | 20260507 19:40:27.950 | INFO | "huge" : "", | 20260507 19:40:27.950 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:27.950 | INFO | "malloc" : " 14 GB", | 20260507 19:40:27.950 | INFO | "pinned" : " 29 GB" | 20260507 19:40:27.950 | INFO | } | 20260507 19:40:27.950 | INFO | Perf report for the last 5.13815 seconds: | 20260507 19:40:27.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:27.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.950 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:40:27.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.951 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:40:27.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:27.951 | INFO | Uploaded 36GiB to GPU in 2.56179s (14.0527GiB/s) | 20260507 19:40:27.951 | INFO | Downloaded 34GiB to GPU in 2.4695s (13.7679GiB/s) | 20260507 19:40:33.090 | INFO | Analyzing chunk frames = [2082304,2082816) | 20260507 19:40:33.090 | INFO | MemInfo: { | 20260507 19:40:33.090 | INFO | "gpu" : " 17 GB", | 20260507 19:40:33.090 | INFO | "huge" : "", | 20260507 19:40:33.090 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:33.090 | INFO | "malloc" : " 14 GB", | 20260507 19:40:33.091 | INFO | "pinned" : " 29 GB" | 20260507 19:40:33.091 | INFO | } | 20260507 19:40:33.091 | INFO | Perf report for the last 5.14018 seconds: | 20260507 19:40:33.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:33.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:40:33.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:40:33.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:33.091 | INFO | Uploaded 36GiB to GPU in 2.4749s (14.546GiB/s) | 20260507 19:40:33.091 | INFO | Downloaded 34GiB to GPU in 2.40022s (14.1654GiB/s) | 20260507 19:40:38.230 | INFO | Analyzing chunk frames = [2082816,2083328) | 20260507 19:40:38.230 | INFO | MemInfo: { | 20260507 19:40:38.230 | INFO | "gpu" : " 17 GB", | 20260507 19:40:38.230 | INFO | "huge" : "", | 20260507 19:40:38.230 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:38.230 | INFO | "malloc" : " 14 GB", | 20260507 19:40:38.231 | INFO | "pinned" : " 29 GB" | 20260507 19:40:38.231 | INFO | } | 20260507 19:40:38.231 | INFO | Perf report for the last 5.14006 seconds: | 20260507 19:40:38.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:38.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 19:40:38.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 19:40:38.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:38.231 | INFO | Uploaded 36GiB to GPU in 2.36659s (15.2118GiB/s) | 20260507 19:40:38.231 | INFO | Downloaded 34GiB to GPU in 2.50138s (13.5925GiB/s) | 20260507 19:40:43.369 | INFO | Analyzing chunk frames = [2083328,2083840) | 20260507 19:40:43.369 | INFO | MemInfo: { | 20260507 19:40:43.369 | INFO | "gpu" : " 17 GB", | 20260507 19:40:43.369 | INFO | "huge" : "", | 20260507 19:40:43.369 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:43.369 | INFO | "malloc" : " 14 GB", | 20260507 19:40:43.369 | INFO | "pinned" : " 29 GB" | 20260507 19:40:43.369 | INFO | } | 20260507 19:40:43.369 | INFO | Perf report for the last 5.13812 seconds: | 20260507 19:40:43.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:43.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:40:43.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:40:43.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:43.369 | INFO | Uploaded 36GiB to GPU in 2.49043s (14.4553GiB/s) | 20260507 19:40:43.369 | INFO | Downloaded 34GiB to GPU in 2.42282s (14.0333GiB/s) | 20260507 19:40:48.509 | INFO | Analyzing chunk frames = [2083840,2084352) | 20260507 19:40:48.509 | INFO | MemInfo: { | 20260507 19:40:48.509 | INFO | "gpu" : " 17 GB", | 20260507 19:40:48.509 | INFO | "huge" : "", | 20260507 19:40:48.509 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:48.509 | INFO | "malloc" : " 14 GB", | 20260507 19:40:48.509 | INFO | "pinned" : " 29 GB" | 20260507 19:40:48.509 | INFO | } | 20260507 19:40:48.509 | INFO | Perf report for the last 5.13993 seconds: | 20260507 19:40:48.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:48.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:40:48.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 19:40:48.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:48.509 | INFO | Uploaded 36GiB to GPU in 2.52067s (14.2819GiB/s) | 20260507 19:40:48.509 | INFO | Downloaded 34GiB to GPU in 2.44951s (13.8803GiB/s) | 20260507 19:40:53.648 | INFO | Analyzing chunk frames = [2084352,2084864) | 20260507 19:40:53.648 | INFO | MemInfo: { | 20260507 19:40:53.648 | INFO | "gpu" : " 17 GB", | 20260507 19:40:53.648 | INFO | "huge" : "", | 20260507 19:40:53.648 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:53.648 | INFO | "malloc" : " 14 GB", | 20260507 19:40:53.648 | INFO | "pinned" : " 29 GB" | 20260507 19:40:53.648 | INFO | } | 20260507 19:40:53.648 | INFO | Perf report for the last 5.13909 seconds: | 20260507 19:40:53.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:53.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:40:53.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:40:53.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:53.648 | INFO | Uploaded 36GiB to GPU in 2.47641s (14.5372GiB/s) | 20260507 19:40:53.648 | INFO | Downloaded 34GiB to GPU in 2.42602s (14.0147GiB/s) | 20260507 19:40:53.745 | INFO | Starting a round of Baz IO | 20260507 19:40:57.218 | INFO | Populating RTMetrics report | 20260507 19:40:57.218 | INFO | RTMetrics report populated | 20260507 19:40:57.218 | INFO | Opening tmp RTMetrics json file | 20260507 19:40:57.229 | INFO | Serializing RTMetrics to json | 20260507 19:40:57.307 | INFO | Serialization done, swapping tmp file now... | 20260507 19:40:57.332 | INFO | RTMetrics done | 20260507 19:40:57.370 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.624 seconds | 20260507 19:40:57.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.18% / 0.01% / 54.81% | 20260507 19:40:57.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.29764% / 1.19209e-05% / 4.37101e-05% | 20260507 19:40:57.370 | INFO | Baz IO overall: | 20260507 19:40:57.370 | INFO | BytesWritten : 942726009220 | 20260507 19:40:57.370 | INFO | NumEvents : 457067313927 | 20260507 19:40:58.787 | INFO | Analyzing chunk frames = [2084864,2085376) | 20260507 19:40:58.787 | INFO | MemInfo: { | 20260507 19:40:58.787 | INFO | "gpu" : " 17 GB", | 20260507 19:40:58.787 | INFO | "huge" : "", | 20260507 19:40:58.787 | INFO | "huge_shared" : " 29 GB", | 20260507 19:40:58.787 | INFO | "malloc" : " 14 GB", | 20260507 19:40:58.787 | INFO | "pinned" : " 29 GB" | 20260507 19:40:58.787 | INFO | } | 20260507 19:40:58.787 | INFO | Perf report for the last 5.1394 seconds: | 20260507 19:40:58.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:40:58.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:58.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:58.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:58.787 | INFO | ANALYSIS : 68.7%, 3.0, 106.4ms, 0 | 20260507 19:40:58.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:40:58.787 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 19:40:58.787 | INFO | PRE_HQ : 66.1%, 1.7, 58.6ms, 0 | 20260507 19:40:58.787 | INFO | BAZWRITER : 70.1%, 3.8, 135.2ms, 0 | 20260507 19:40:58.787 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 19:40:58.788 | INFO | Uploaded 36GiB to GPU in 2.53723s (14.1887GiB/s) | 20260507 19:40:58.788 | INFO | Downloaded 38GiB to GPU in 2.69781s (14.0855GiB/s) | 20260507 19:41:03.926 | INFO | Analyzing chunk frames = [2085376,2085888) | 20260507 19:41:03.926 | INFO | MemInfo: { | 20260507 19:41:03.926 | INFO | "gpu" : " 17 GB", | 20260507 19:41:03.926 | INFO | "huge" : "", | 20260507 19:41:03.926 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:03.926 | INFO | "malloc" : " 14 GB", | 20260507 19:41:03.926 | INFO | "pinned" : " 29 GB" | 20260507 19:41:03.926 | INFO | } | 20260507 19:41:03.926 | INFO | Perf report for the last 5.13922 seconds: | 20260507 19:41:03.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:03.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.927 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:41:03.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.927 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 19:41:03.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:03.927 | INFO | Uploaded 36GiB to GPU in 2.56781s (14.0197GiB/s) | 20260507 19:41:03.927 | INFO | Downloaded 34GiB to GPU in 2.49546s (13.6247GiB/s) | 20260507 19:41:09.066 | INFO | Analyzing chunk frames = [2085888,2086400) | 20260507 19:41:09.066 | INFO | MemInfo: { | 20260507 19:41:09.066 | INFO | "gpu" : " 17 GB", | 20260507 19:41:09.066 | INFO | "huge" : "", | 20260507 19:41:09.066 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:09.066 | INFO | "malloc" : " 14 GB", | 20260507 19:41:09.066 | INFO | "pinned" : " 29 GB" | 20260507 19:41:09.066 | INFO | } | 20260507 19:41:09.066 | INFO | Perf report for the last 5.13917 seconds: | 20260507 19:41:09.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:09.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:41:09.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:41:09.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:09.066 | INFO | Uploaded 36GiB to GPU in 2.55553s (14.0871GiB/s) | 20260507 19:41:09.066 | INFO | Downloaded 34GiB to GPU in 2.47285s (13.7493GiB/s) | 20260507 19:41:14.205 | INFO | Analyzing chunk frames = [2086400,2086912) | 20260507 19:41:14.205 | INFO | MemInfo: { | 20260507 19:41:14.205 | INFO | "gpu" : " 17 GB", | 20260507 19:41:14.205 | INFO | "huge" : "", | 20260507 19:41:14.205 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:14.205 | INFO | "malloc" : " 14 GB", | 20260507 19:41:14.205 | INFO | "pinned" : " 29 GB" | 20260507 19:41:14.205 | INFO | } | 20260507 19:41:14.205 | INFO | Perf report for the last 5.13951 seconds: | 20260507 19:41:14.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:14.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 19:41:14.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:41:14.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:14.205 | INFO | Uploaded 36GiB to GPU in 2.44493s (14.7243GiB/s) | 20260507 19:41:14.205 | INFO | Downloaded 34GiB to GPU in 2.38941s (14.2294GiB/s) | 20260507 19:41:19.344 | INFO | Analyzing chunk frames = [2086912,2087424) | 20260507 19:41:19.344 | INFO | MemInfo: { | 20260507 19:41:19.344 | INFO | "gpu" : " 17 GB", | 20260507 19:41:19.344 | INFO | "huge" : "", | 20260507 19:41:19.344 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:19.344 | INFO | "malloc" : " 14 GB", | 20260507 19:41:19.344 | INFO | "pinned" : " 29 GB" | 20260507 19:41:19.344 | INFO | } | 20260507 19:41:19.344 | INFO | Perf report for the last 5.13904 seconds: | 20260507 19:41:19.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:19.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.344 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:41:19.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.344 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 19:41:19.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:19.345 | INFO | Uploaded 36GiB to GPU in 2.36827s (15.201GiB/s) | 20260507 19:41:19.345 | INFO | Downloaded 34GiB to GPU in 2.50237s (13.5871GiB/s) | 20260507 19:41:24.484 | INFO | Analyzing chunk frames = [2087424,2087936) | 20260507 19:41:24.484 | INFO | MemInfo: { | 20260507 19:41:24.484 | INFO | "gpu" : " 17 GB", | 20260507 19:41:24.484 | INFO | "huge" : "", | 20260507 19:41:24.484 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:24.484 | INFO | "malloc" : " 14 GB", | 20260507 19:41:24.484 | INFO | "pinned" : " 29 GB" | 20260507 19:41:24.484 | INFO | } | 20260507 19:41:24.484 | INFO | Perf report for the last 5.13986 seconds: | 20260507 19:41:24.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:24.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:41:24.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:41:24.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:24.484 | INFO | Uploaded 36GiB to GPU in 2.48229s (14.5027GiB/s) | 20260507 19:41:24.485 | INFO | Downloaded 34GiB to GPU in 2.42366s (14.0284GiB/s) | 20260507 19:41:29.623 | INFO | Analyzing chunk frames = [2087936,2088448) | 20260507 19:41:29.623 | INFO | MemInfo: { | 20260507 19:41:29.623 | INFO | "gpu" : " 17 GB", | 20260507 19:41:29.623 | INFO | "huge" : "", | 20260507 19:41:29.623 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:29.623 | INFO | "malloc" : " 14 GB", | 20260507 19:41:29.623 | INFO | "pinned" : " 29 GB" | 20260507 19:41:29.623 | INFO | } | 20260507 19:41:29.623 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:41:29.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:29.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.623 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:41:29.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.623 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:41:29.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:29.624 | INFO | Uploaded 36GiB to GPU in 2.52771s (14.2421GiB/s) | 20260507 19:41:29.624 | INFO | Downloaded 34GiB to GPU in 2.45233s (13.8644GiB/s) | 20260507 19:41:34.762 | INFO | Analyzing chunk frames = [2088448,2088960) | 20260507 19:41:34.762 | INFO | MemInfo: { | 20260507 19:41:34.762 | INFO | "gpu" : " 17 GB", | 20260507 19:41:34.762 | INFO | "huge" : "", | 20260507 19:41:34.762 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:34.762 | INFO | "malloc" : " 14 GB", | 20260507 19:41:34.762 | INFO | "pinned" : " 29 GB" | 20260507 19:41:34.762 | INFO | } | 20260507 19:41:34.762 | INFO | Perf report for the last 5.13893 seconds: | 20260507 19:41:34.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:34.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.762 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:41:34.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.762 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:41:34.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:34.763 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:41:34.763 | INFO | Total mem usage: 16.4 GiB | 20260507 19:41:34.763 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:41:34.763 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.41%) | 20260507 19:41:34.763 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:41:34.763 | INFO | High Water Breakdown: | 20260507 19:41:34.763 | INFO | 60% : 8.99 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:41:34.763 | INFO | 39.9% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:41:34.763 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:41:34.763 | INFO | ------------------------------------------ | 20260507 19:41:34.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:41:34.763 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:41:34.763 | INFO | Total mem usage: 26.5 GiB | 20260507 19:41:34.763 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:41:34.763 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:41:34.763 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:41:34.763 | INFO | High Water Breakdown: | 20260507 19:41:34.763 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:41:34.763 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:41:34.763 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:41:34.763 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:41:34.763 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:41:34.763 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:41:34.763 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:41:34.763 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:41:34.763 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:41:34.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:41:34.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:41:34.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:41:34.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:41:34.763 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:41:34.763 | INFO | ------------------------------------------ | 20260507 19:41:34.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:41:34.763 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:41:34.763 | INFO | Total mem usage: 27.8 GiB | 20260507 19:41:34.763 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:41:34.763 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:41:34.763 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:41:34.763 | INFO | High Water Breakdown: | 20260507 19:41:34.763 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:41:34.763 | INFO | ------------------------------------------ | 20260507 19:41:34.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:41:34.763 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:41:34.763 | INFO | Total mem usage: 15.7 GiB | 20260507 19:41:34.763 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:41:34.763 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:41:34.763 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:41:34.763 | INFO | High Water Breakdown: | 20260507 19:41:34.763 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:41:34.763 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:41:34.763 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:41:34.763 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:41:34.763 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:41:34.763 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:41:34.763 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:41:34.763 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:41:34.763 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:41:34.763 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:41:34.763 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:41:34.763 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:41:34.763 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:41:34.763 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:41:34.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:41:34.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:41:34.763 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:41:34.763 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:41:34.763 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:41:34.763 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:41:34.763 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:41:34.763 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:41:34.763 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:41:34.763 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:41:34.763 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:41:34.763 | INFO | ------------------------------------------ | 20260507 19:41:34.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:41:34.763 | INFO | Uploaded 36GiB to GPU in 2.48347s (14.4959GiB/s) | 20260507 19:41:34.763 | INFO | Downloaded 34GiB to GPU in 2.4378s (13.947GiB/s) | 20260507 19:41:34.763 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:41:34.763 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:41:34.763 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:41:34.763 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.43e+01 / 1.67e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 0.29%, AggregateStats -- 9.77e-02 / 1.01e-01, 1.16e-02 / 5.61e-03 | 20260507 19:41:34.763 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.44e+00, 6.07e-01 / 3.16e-01 | 20260507 19:41:34.763 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.51e+00, 2.13e-01 / 1.15e-01 | 20260507 19:41:34.763 | INFO | PerfReport: 15.72%, DME -- 4.17e+01 / 4.48e+01, 3.58e+00 / 1.31e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.85e+00 / 1.66e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.40e-01 / 2.27e-01 | 20260507 19:41:34.763 | INFO | PerfReport: 3.63%, Metrics -- 1.21e+00 / 1.41e+00, 3.09e-01 / 1.73e-01 | 20260507 19:41:34.763 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.75e-01 / 4.08e-01, 6.11e-02 / 3.19e-02 | 20260507 19:41:34.763 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:41:34.763 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:41:34.763 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:41:34.763 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 24.32%, Upload -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 24.18%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 31.03%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 1.96%, LockUpload -- 2.10e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 2.39%, LockDownload -- 2.55e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 16.11%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:41:34.763 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 9.90e-03 / 0.00e+00 | 20260507 19:41:34.859 | INFO | Starting a round of Baz IO | 20260507 19:41:38.331 | INFO | Populating RTMetrics report | 20260507 19:41:38.331 | INFO | RTMetrics report populated | 20260507 19:41:38.331 | INFO | Opening tmp RTMetrics json file | 20260507 19:41:38.355 | INFO | Serializing RTMetrics to json | 20260507 19:41:38.393 | INFO | Serialization done, swapping tmp file now... | 20260507 19:41:38.400 | INFO | RTMetrics done | 20260507 19:41:38.453 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.594 seconds | 20260507 19:41:38.453 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.17% / 0.01% / 54.82% | 20260507 19:41:38.453 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.29491% / 1.58946e-05% / 3.17891e-05% | 20260507 19:41:38.453 | INFO | Baz IO overall: | 20260507 19:41:38.453 | INFO | BytesWritten : 944644086373 | 20260507 19:41:38.453 | INFO | NumEvents : 457996317889 | 20260507 19:41:39.901 | INFO | Analyzing chunk frames = [2088960,2089472) | 20260507 19:41:39.901 | INFO | MemInfo: { | 20260507 19:41:39.901 | INFO | "gpu" : " 17 GB", | 20260507 19:41:39.901 | INFO | "huge" : "", | 20260507 19:41:39.901 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:39.901 | INFO | "malloc" : " 14 GB", | 20260507 19:41:39.901 | INFO | "pinned" : " 29 GB" | 20260507 19:41:39.901 | INFO | } | 20260507 19:41:39.902 | INFO | Perf report for the last 5.13945 seconds: | 20260507 19:41:39.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:39.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:39.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:39.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:39.902 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 19:41:39.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:39.902 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 19:41:39.902 | INFO | PRE_HQ : 66.0%, 1.7, 58.2ms, 0 | 20260507 19:41:39.902 | INFO | BAZWRITER : 69.5%, 3.7, 132.4ms, 0 | 20260507 19:41:39.902 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 19:41:39.902 | INFO | Uploaded 36GiB to GPU in 2.52618s (14.2507GiB/s) | 20260507 19:41:39.902 | INFO | Downloaded 38GiB to GPU in 2.69289s (14.1112GiB/s) | 20260507 19:41:45.042 | INFO | Analyzing chunk frames = [2089472,2089984) | 20260507 19:41:45.042 | INFO | MemInfo: { | 20260507 19:41:45.042 | INFO | "gpu" : " 17 GB", | 20260507 19:41:45.042 | INFO | "huge" : "", | 20260507 19:41:45.042 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:45.042 | INFO | "malloc" : " 14 GB", | 20260507 19:41:45.042 | INFO | "pinned" : " 29 GB" | 20260507 19:41:45.042 | INFO | } | 20260507 19:41:45.042 | INFO | Perf report for the last 5.14089 seconds: | 20260507 19:41:45.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:45.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.042 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:41:45.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.043 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 19:41:45.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:45.043 | INFO | Uploaded 36GiB to GPU in 2.52327s (14.2672GiB/s) | 20260507 19:41:45.043 | INFO | Downloaded 34GiB to GPU in 2.50044s (13.5976GiB/s) | 20260507 19:41:50.182 | INFO | Analyzing chunk frames = [2089984,2090496) | 20260507 19:41:50.182 | INFO | MemInfo: { | 20260507 19:41:50.182 | INFO | "gpu" : " 17 GB", | 20260507 19:41:50.182 | INFO | "huge" : "", | 20260507 19:41:50.182 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:50.182 | INFO | "malloc" : " 14 GB", | 20260507 19:41:50.182 | INFO | "pinned" : " 29 GB" | 20260507 19:41:50.182 | INFO | } | 20260507 19:41:50.182 | INFO | Perf report for the last 5.13976 seconds: | 20260507 19:41:50.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:50.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.182 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:41:50.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.182 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:41:50.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:50.183 | INFO | Uploaded 36GiB to GPU in 2.56446s (14.038GiB/s) | 20260507 19:41:50.183 | INFO | Downloaded 34GiB to GPU in 2.48077s (13.7054GiB/s) | 20260507 19:41:55.320 | INFO | Analyzing chunk frames = [2090496,2091008) | 20260507 19:41:55.320 | INFO | MemInfo: { | 20260507 19:41:55.320 | INFO | "gpu" : " 17 GB", | 20260507 19:41:55.320 | INFO | "huge" : "", | 20260507 19:41:55.320 | INFO | "huge_shared" : " 29 GB", | 20260507 19:41:55.320 | INFO | "malloc" : " 14 GB", | 20260507 19:41:55.320 | INFO | "pinned" : " 29 GB" | 20260507 19:41:55.320 | INFO | } | 20260507 19:41:55.320 | INFO | Perf report for the last 5.1379 seconds: | 20260507 19:41:55.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:41:55.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.320 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 19:41:55.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.320 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:41:55.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:41:55.321 | INFO | Uploaded 36GiB to GPU in 2.4511s (14.6873GiB/s) | 20260507 19:41:55.321 | INFO | Downloaded 34GiB to GPU in 2.40042s (14.1642GiB/s) | 20260507 19:42:00.460 | INFO | Analyzing chunk frames = [2091008,2091520) | 20260507 19:42:00.460 | INFO | MemInfo: { | 20260507 19:42:00.460 | INFO | "gpu" : " 17 GB", | 20260507 19:42:00.460 | INFO | "huge" : "", | 20260507 19:42:00.460 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:00.460 | INFO | "malloc" : " 14 GB", | 20260507 19:42:00.460 | INFO | "pinned" : " 29 GB" | 20260507 19:42:00.460 | INFO | } | 20260507 19:42:00.460 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:42:00.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:00.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 19:42:00.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:42:00.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:00.460 | INFO | Uploaded 36GiB to GPU in 2.40396s (14.9753GiB/s) | 20260507 19:42:00.460 | INFO | Downloaded 34GiB to GPU in 2.49203s (13.6435GiB/s) | 20260507 19:42:05.598 | INFO | Analyzing chunk frames = [2091520,2092032) | 20260507 19:42:05.598 | INFO | MemInfo: { | 20260507 19:42:05.598 | INFO | "gpu" : " 17 GB", | 20260507 19:42:05.598 | INFO | "huge" : "", | 20260507 19:42:05.599 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:05.599 | INFO | "malloc" : " 14 GB", | 20260507 19:42:05.599 | INFO | "pinned" : " 29 GB" | 20260507 19:42:05.599 | INFO | } | 20260507 19:42:05.599 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:42:05.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:05.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:42:05.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:42:05.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:05.599 | INFO | Uploaded 36GiB to GPU in 2.45751s (14.649GiB/s) | 20260507 19:42:05.599 | INFO | Downloaded 34GiB to GPU in 2.42396s (14.0266GiB/s) | 20260507 19:42:10.738 | INFO | Analyzing chunk frames = [2092032,2092544) | 20260507 19:42:10.738 | INFO | MemInfo: { | 20260507 19:42:10.738 | INFO | "gpu" : " 17 GB", | 20260507 19:42:10.738 | INFO | "huge" : "", | 20260507 19:42:10.738 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:10.738 | INFO | "malloc" : " 14 GB", | 20260507 19:42:10.738 | INFO | "pinned" : " 29 GB" | 20260507 19:42:10.738 | INFO | } | 20260507 19:42:10.738 | INFO | Perf report for the last 5.13969 seconds: | 20260507 19:42:10.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:10.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.738 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:42:10.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.738 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:42:10.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:10.739 | INFO | Uploaded 36GiB to GPU in 2.52676s (14.2475GiB/s) | 20260507 19:42:10.739 | INFO | Downloaded 34GiB to GPU in 2.44886s (13.884GiB/s) | 20260507 19:42:15.877 | INFO | Analyzing chunk frames = [2092544,2093056) | 20260507 19:42:15.877 | INFO | MemInfo: { | 20260507 19:42:15.877 | INFO | "gpu" : " 17 GB", | 20260507 19:42:15.877 | INFO | "huge" : "", | 20260507 19:42:15.877 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:15.877 | INFO | "malloc" : " 14 GB", | 20260507 19:42:15.877 | INFO | "pinned" : " 29 GB" | 20260507 19:42:15.877 | INFO | } | 20260507 19:42:15.877 | INFO | Perf report for the last 5.13914 seconds: | 20260507 19:42:15.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:15.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.878 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:42:15.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.878 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 19:42:15.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:15.878 | INFO | Uploaded 36GiB to GPU in 2.49586s (14.4239GiB/s) | 20260507 19:42:15.878 | INFO | Downloaded 34GiB to GPU in 2.43832s (13.944GiB/s) | 20260507 19:42:15.992 | INFO | Starting a round of Baz IO | 20260507 19:42:19.478 | INFO | Populating RTMetrics report | 20260507 19:42:19.478 | INFO | RTMetrics report populated | 20260507 19:42:19.478 | INFO | Opening tmp RTMetrics json file | 20260507 19:42:19.496 | INFO | Serializing RTMetrics to json | 20260507 19:42:19.506 | INFO | Serialization done, swapping tmp file now... | 20260507 19:42:19.507 | INFO | RTMetrics done | 20260507 19:42:19.589 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.596 seconds | 20260507 19:42:19.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.15% / 0.01% / 54.83% | 20260507 19:42:19.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2956% / 1.19209e-05% / 3.97364e-05% | 20260507 19:42:19.589 | INFO | Baz IO overall: | 20260507 19:42:19.589 | INFO | BytesWritten : 946564013863 | 20260507 19:42:19.589 | INFO | NumEvents : 458926802876 | 20260507 19:42:21.016 | INFO | Analyzing chunk frames = [2093056,2093568) | 20260507 19:42:21.016 | INFO | MemInfo: { | 20260507 19:42:21.016 | INFO | "gpu" : " 17 GB", | 20260507 19:42:21.016 | INFO | "huge" : "", | 20260507 19:42:21.016 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:21.016 | INFO | "malloc" : " 14 GB", | 20260507 19:42:21.016 | INFO | "pinned" : " 29 GB" | 20260507 19:42:21.016 | INFO | } | 20260507 19:42:21.016 | INFO | Perf report for the last 5.13851 seconds: | 20260507 19:42:21.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:21.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:21.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:21.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:21.016 | INFO | ANALYSIS : 69.4%, 2.9, 104.9ms, 0 | 20260507 19:42:21.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:21.016 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 19:42:21.016 | INFO | PRE_HQ : 68.4%, 1.8, 64.9ms, 0 | 20260507 19:42:21.016 | INFO | BAZWRITER : 69.6%, 4.1, 145.1ms, 0 | 20260507 19:42:21.016 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 19:42:21.017 | INFO | Uploaded 36GiB to GPU in 2.60028s (13.8447GiB/s) | 20260507 19:42:21.017 | INFO | Downloaded 38GiB to GPU in 2.71028s (14.0207GiB/s) | 20260507 19:42:26.157 | INFO | Analyzing chunk frames = [2093568,2094080) | 20260507 19:42:26.157 | INFO | MemInfo: { | 20260507 19:42:26.157 | INFO | "gpu" : " 17 GB", | 20260507 19:42:26.157 | INFO | "huge" : "", | 20260507 19:42:26.157 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:26.157 | INFO | "malloc" : " 14 GB", | 20260507 19:42:26.157 | INFO | "pinned" : " 29 GB" | 20260507 19:42:26.157 | INFO | } | 20260507 19:42:26.157 | INFO | Perf report for the last 5.14081 seconds: | 20260507 19:42:26.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:26.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 19:42:26.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:42:26.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:26.157 | INFO | Uploaded 36GiB to GPU in 2.56323s (14.0448GiB/s) | 20260507 19:42:26.157 | INFO | Downloaded 34GiB to GPU in 2.48271s (13.6947GiB/s) | 20260507 19:42:31.295 | INFO | Analyzing chunk frames = [2094080,2094592) | 20260507 19:42:31.295 | INFO | MemInfo: { | 20260507 19:42:31.295 | INFO | "gpu" : " 17 GB", | 20260507 19:42:31.295 | INFO | "huge" : "", | 20260507 19:42:31.295 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:31.295 | INFO | "malloc" : " 14 GB", | 20260507 19:42:31.295 | INFO | "pinned" : " 29 GB" | 20260507 19:42:31.295 | INFO | } | 20260507 19:42:31.295 | INFO | Perf report for the last 5.13857 seconds: | 20260507 19:42:31.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:31.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:42:31.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:42:31.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:31.296 | INFO | Uploaded 36GiB to GPU in 2.56896s (14.0134GiB/s) | 20260507 19:42:31.296 | INFO | Downloaded 34GiB to GPU in 2.47554s (13.7344GiB/s) | 20260507 19:42:36.435 | INFO | Analyzing chunk frames = [2094592,2095104) | 20260507 19:42:36.435 | INFO | MemInfo: { | 20260507 19:42:36.435 | INFO | "gpu" : " 17 GB", | 20260507 19:42:36.435 | INFO | "huge" : "", | 20260507 19:42:36.435 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:36.435 | INFO | "malloc" : " 14 GB", | 20260507 19:42:36.435 | INFO | "pinned" : " 29 GB" | 20260507 19:42:36.435 | INFO | } | 20260507 19:42:36.435 | INFO | Perf report for the last 5.13984 seconds: | 20260507 19:42:36.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:36.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.435 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:42:36.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.435 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:42:36.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:36.436 | INFO | Uploaded 36GiB to GPU in 2.48748s (14.4725GiB/s) | 20260507 19:42:36.436 | INFO | Downloaded 34GiB to GPU in 2.40857s (14.1163GiB/s) | 20260507 19:42:41.574 | INFO | Analyzing chunk frames = [2095104,2095616) | 20260507 19:42:41.574 | INFO | MemInfo: { | 20260507 19:42:41.574 | INFO | "gpu" : " 17 GB", | 20260507 19:42:41.574 | INFO | "huge" : "", | 20260507 19:42:41.574 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:41.574 | INFO | "malloc" : " 14 GB", | 20260507 19:42:41.574 | INFO | "pinned" : " 29 GB" | 20260507 19:42:41.574 | INFO | } | 20260507 19:42:41.574 | INFO | Perf report for the last 5.13875 seconds: | 20260507 19:42:41.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:41.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:42:41.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:42:41.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:41.574 | INFO | Uploaded 36GiB to GPU in 2.35125s (15.311GiB/s) | 20260507 19:42:41.575 | INFO | Downloaded 34GiB to GPU in 2.50367s (13.5801GiB/s) | 20260507 19:42:46.712 | INFO | Analyzing chunk frames = [2095616,2096128) | 20260507 19:42:46.712 | INFO | MemInfo: { | 20260507 19:42:46.712 | INFO | "gpu" : " 17 GB", | 20260507 19:42:46.712 | INFO | "huge" : "", | 20260507 19:42:46.713 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:46.713 | INFO | "malloc" : " 14 GB", | 20260507 19:42:46.713 | INFO | "pinned" : " 29 GB" | 20260507 19:42:46.713 | INFO | } | 20260507 19:42:46.713 | INFO | Perf report for the last 5.13868 seconds: | 20260507 19:42:46.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:46.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:42:46.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:42:46.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:46.713 | INFO | Uploaded 36GiB to GPU in 2.48204s (14.5042GiB/s) | 20260507 19:42:46.713 | INFO | Downloaded 34GiB to GPU in 2.42798s (14.0034GiB/s) | 20260507 19:42:51.853 | INFO | Analyzing chunk frames = [2096128,2096640) | 20260507 19:42:51.853 | INFO | MemInfo: { | 20260507 19:42:51.853 | INFO | "gpu" : " 17 GB", | 20260507 19:42:51.853 | INFO | "huge" : "", | 20260507 19:42:51.853 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:51.853 | INFO | "malloc" : " 14 GB", | 20260507 19:42:51.853 | INFO | "pinned" : " 29 GB" | 20260507 19:42:51.853 | INFO | } | 20260507 19:42:51.853 | INFO | Perf report for the last 5.14046 seconds: | 20260507 19:42:51.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:51.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.853 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:42:51.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.853 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 19:42:51.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:51.854 | INFO | Uploaded 36GiB to GPU in 2.52106s (14.2797GiB/s) | 20260507 19:42:51.854 | INFO | Downloaded 34GiB to GPU in 2.44646s (13.8976GiB/s) | 20260507 19:42:56.993 | INFO | Analyzing chunk frames = [2096640,2097152) | 20260507 19:42:56.993 | INFO | MemInfo: { | 20260507 19:42:56.993 | INFO | "gpu" : " 17 GB", | 20260507 19:42:56.993 | INFO | "huge" : "", | 20260507 19:42:56.993 | INFO | "huge_shared" : " 29 GB", | 20260507 19:42:56.993 | INFO | "malloc" : " 14 GB", | 20260507 19:42:56.993 | INFO | "pinned" : " 29 GB" | 20260507 19:42:56.993 | INFO | } | 20260507 19:42:56.993 | INFO | Perf report for the last 5.13957 seconds: | 20260507 19:42:56.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:42:56.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:42:56.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 19:42:56.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:42:56.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:42:56.993 | INFO | Total mem usage: 16.4 GiB | 20260507 19:42:56.993 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:42:56.993 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11.1%) | 20260507 19:42:56.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:42:56.993 | INFO | High Water Breakdown: | 20260507 19:42:56.993 | INFO | 59.7% : 8.82 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:42:56.993 | INFO | 40.2% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:42:56.993 | INFO | Uploaded 36GiB to GPU in 2.48013s (14.5153GiB/s) | 20260507 19:42:56.993 | INFO | Downloaded 34GiB to GPU in 2.42534s (14.0186GiB/s) | 20260507 19:42:56.993 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:42:56.993 | INFO | ------------------------------------------ | 20260507 19:42:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:42:56.994 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:42:56.994 | INFO | Total mem usage: 26.5 GiB | 20260507 19:42:56.994 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:42:56.994 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 19:42:56.994 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 19:42:56.994 | INFO | High Water Breakdown: | 20260507 19:42:56.994 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:42:56.994 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:42:56.994 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:42:56.994 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:42:56.994 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:42:56.994 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:42:56.994 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:42:56.994 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:42:56.994 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:42:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:42:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:42:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:42:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:42:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:42:56.994 | INFO | ------------------------------------------ | 20260507 19:42:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:42:56.994 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:42:56.994 | INFO | Total mem usage: 27.8 GiB | 20260507 19:42:56.994 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:42:56.994 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:42:56.994 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:42:56.994 | INFO | High Water Breakdown: | 20260507 19:42:56.994 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:42:56.994 | INFO | ------------------------------------------ | 20260507 19:42:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:42:56.994 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:42:56.994 | INFO | Total mem usage: 15.7 GiB | 20260507 19:42:56.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:42:56.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:42:56.994 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:42:56.994 | INFO | High Water Breakdown: | 20260507 19:42:56.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:42:56.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:42:56.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:42:56.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:42:56.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:42:56.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:42:56.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:42:56.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:42:56.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:42:56.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:42:56.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:42:56.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:42:56.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:42:56.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:42:56.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:42:56.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:42:56.994 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:42:56.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:42:56.994 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:42:56.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:42:56.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:42:56.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:42:56.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:42:56.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:42:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:42:56.994 | INFO | ------------------------------------------ | 20260507 19:42:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:42:56.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:42:56.994 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:42:56.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:42:56.994 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.54e+01, 1.41e+01 / 1.63e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.02e-01, 1.13e-02 / 5.71e-03 | 20260507 19:42:56.994 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.48e+00, 6.02e-01 / 2.77e-01 | 20260507 19:42:56.994 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.49e+00, 2.10e-01 / 1.17e-01 | 20260507 19:42:56.994 | INFO | PerfReport: 15.74%, DME -- 4.17e+01 / 4.48e+01, 3.95e+00 / 1.21e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.82e+00 / 1.82e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.85e+00, 4.40e-01 / 2.12e-01 | 20260507 19:42:56.994 | INFO | PerfReport: 3.64%, Metrics -- 1.21e+00 / 1.41e+00, 3.07e-01 / 1.69e-01 | 20260507 19:42:56.994 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.74e-01 / 4.04e-01, 6.05e-02 / 3.51e-02 | 20260507 19:42:56.994 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:42:56.994 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:42:56.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:42:56.994 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 24.37%, Upload -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 31.02%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 1.94%, LockUpload -- 2.08e+00 / 0.00e+00, 5.05e+00 / 0.00e+00 | 20260507 19:42:56.994 | INFO | PerfReport: 2.40%, LockDownload -- 2.57e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 19:42:56.995 | INFO | PerfReport: 16.04%, LockCompute -- 1.71e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 19:42:56.995 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260507 19:42:57.090 | INFO | Starting a round of Baz IO | 20260507 19:43:00.579 | INFO | Populating RTMetrics report | 20260507 19:43:00.579 | INFO | RTMetrics report populated | 20260507 19:43:00.579 | INFO | Opening tmp RTMetrics json file | 20260507 19:43:00.582 | INFO | Serializing RTMetrics to json | 20260507 19:43:00.606 | INFO | Serialization done, swapping tmp file now... | 20260507 19:43:00.610 | INFO | RTMetrics done | 20260507 19:43:00.697 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.606 seconds | 20260507 19:43:00.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.14% / 0.01% / 54.85% | 20260507 19:43:00.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.28541% / 7.94729e-06% / 5.16574e-05% | 20260507 19:43:00.697 | INFO | Baz IO overall: | 20260507 19:43:00.697 | INFO | BytesWritten : 948481959879 | 20260507 19:43:00.697 | INFO | NumEvents : 459855856644 | 20260507 19:43:02.133 | INFO | Analyzing chunk frames = [2097152,2097664) | 20260507 19:43:02.133 | INFO | MemInfo: { | 20260507 19:43:02.133 | INFO | "gpu" : " 17 GB", | 20260507 19:43:02.133 | INFO | "huge" : "", | 20260507 19:43:02.133 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:02.133 | INFO | "malloc" : " 14 GB", | 20260507 19:43:02.133 | INFO | "pinned" : " 29 GB" | 20260507 19:43:02.133 | INFO | } | 20260507 19:43:02.133 | INFO | Perf report for the last 5.14018 seconds: | 20260507 19:43:02.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:02.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:02.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:02.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:02.133 | INFO | ANALYSIS : 69.0%, 3.0, 107.1ms, 0 | 20260507 19:43:02.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:02.133 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 19:43:02.133 | INFO | PRE_HQ : 66.1%, 1.7, 58.7ms, 0 | 20260507 19:43:02.133 | INFO | BAZWRITER : 69.8%, 4.1, 145.8ms, 0 | 20260507 19:43:02.133 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 19:43:02.133 | INFO | Uploaded 36GiB to GPU in 2.56384s (14.0415GiB/s) | 20260507 19:43:02.133 | INFO | Downloaded 38GiB to GPU in 2.70211s (14.0631GiB/s) | 20260507 19:43:07.271 | INFO | Analyzing chunk frames = [2097664,2098176) | 20260507 19:43:07.271 | INFO | MemInfo: { | 20260507 19:43:07.271 | INFO | "gpu" : " 17 GB", | 20260507 19:43:07.271 | INFO | "huge" : "", | 20260507 19:43:07.271 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:07.271 | INFO | "malloc" : " 14 GB", | 20260507 19:43:07.271 | INFO | "pinned" : " 29 GB" | 20260507 19:43:07.271 | INFO | } | 20260507 19:43:07.271 | INFO | Perf report for the last 5.13844 seconds: | 20260507 19:43:07.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:07.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.271 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:43:07.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.272 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:43:07.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:07.272 | INFO | Uploaded 36GiB to GPU in 2.53703s (14.1898GiB/s) | 20260507 19:43:07.272 | INFO | Downloaded 34GiB to GPU in 2.47555s (13.7343GiB/s) | 20260507 19:43:12.410 | INFO | Analyzing chunk frames = [2098176,2098688) | 20260507 19:43:12.410 | INFO | MemInfo: { | 20260507 19:43:12.410 | INFO | "gpu" : " 17 GB", | 20260507 19:43:12.410 | INFO | "huge" : "", | 20260507 19:43:12.410 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:12.410 | INFO | "malloc" : " 14 GB", | 20260507 19:43:12.410 | INFO | "pinned" : " 29 GB" | 20260507 19:43:12.410 | INFO | } | 20260507 19:43:12.410 | INFO | Perf report for the last 5.13869 seconds: | 20260507 19:43:12.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:12.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.410 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:43:12.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.410 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:43:12.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:12.411 | INFO | Uploaded 36GiB to GPU in 2.57049s (14.0051GiB/s) | 20260507 19:43:12.411 | INFO | Downloaded 34GiB to GPU in 2.45853s (13.8294GiB/s) | 20260507 19:43:17.549 | INFO | Analyzing chunk frames = [2098688,2099200) | 20260507 19:43:17.549 | INFO | MemInfo: { | 20260507 19:43:17.549 | INFO | "gpu" : " 17 GB", | 20260507 19:43:17.549 | INFO | "huge" : "", | 20260507 19:43:17.549 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:17.549 | INFO | "malloc" : " 14 GB", | 20260507 19:43:17.549 | INFO | "pinned" : " 29 GB" | 20260507 19:43:17.549 | INFO | } | 20260507 19:43:17.549 | INFO | Perf report for the last 5.13899 seconds: | 20260507 19:43:17.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:17.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:43:17.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:43:17.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:17.549 | INFO | Uploaded 36GiB to GPU in 2.59379s (13.8793GiB/s) | 20260507 19:43:17.549 | INFO | Downloaded 34GiB to GPU in 2.45836s (13.8303GiB/s) | 20260507 19:43:22.689 | INFO | Analyzing chunk frames = [2099200,2099712) | 20260507 19:43:22.689 | INFO | MemInfo: { | 20260507 19:43:22.689 | INFO | "gpu" : " 17 GB", | 20260507 19:43:22.689 | INFO | "huge" : "", | 20260507 19:43:22.689 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:22.689 | INFO | "malloc" : " 14 GB", | 20260507 19:43:22.689 | INFO | "pinned" : " 29 GB" | 20260507 19:43:22.689 | INFO | } | 20260507 19:43:22.689 | INFO | Perf report for the last 5.13973 seconds: | 20260507 19:43:22.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:22.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:43:22.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:43:22.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:22.689 | INFO | Uploaded 36GiB to GPU in 2.38169s (15.1153GiB/s) | 20260507 19:43:22.689 | INFO | Downloaded 34GiB to GPU in 2.49708s (13.6159GiB/s) | 20260507 19:43:27.828 | INFO | Analyzing chunk frames = [2099712,2100224) | 20260507 19:43:27.828 | INFO | MemInfo: { | 20260507 19:43:27.828 | INFO | "gpu" : " 17 GB", | 20260507 19:43:27.828 | INFO | "huge" : "", | 20260507 19:43:27.828 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:27.828 | INFO | "malloc" : " 14 GB", | 20260507 19:43:27.828 | INFO | "pinned" : " 29 GB" | 20260507 19:43:27.829 | INFO | } | 20260507 19:43:27.829 | INFO | Perf report for the last 5.13974 seconds: | 20260507 19:43:27.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:27.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:43:27.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 19:43:27.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:27.829 | INFO | Uploaded 36GiB to GPU in 2.52553s (14.2544GiB/s) | 20260507 19:43:27.829 | INFO | Downloaded 34GiB to GPU in 2.41151s (14.0991GiB/s) | 20260507 19:43:32.968 | INFO | Analyzing chunk frames = [2100224,2100736) | 20260507 19:43:32.968 | INFO | MemInfo: { | 20260507 19:43:32.968 | INFO | "gpu" : " 17 GB", | 20260507 19:43:32.968 | INFO | "huge" : "", | 20260507 19:43:32.968 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:32.968 | INFO | "malloc" : " 14 GB", | 20260507 19:43:32.968 | INFO | "pinned" : " 29 GB" | 20260507 19:43:32.968 | INFO | } | 20260507 19:43:32.968 | INFO | Perf report for the last 5.1394 seconds: | 20260507 19:43:32.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:32.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.968 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:43:32.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.968 | INFO | PRE_HQ : 37.5%, 1.0, 19.5ms, 0 | 20260507 19:43:32.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:32.969 | INFO | Uploaded 36GiB to GPU in 2.55064s (14.1141GiB/s) | 20260507 19:43:32.969 | INFO | Downloaded 34GiB to GPU in 2.44462s (13.9081GiB/s) | 20260507 19:43:38.107 | INFO | Analyzing chunk frames = [2100736,2101248) | 20260507 19:43:38.107 | INFO | MemInfo: { | 20260507 19:43:38.107 | INFO | "gpu" : " 17 GB", | 20260507 19:43:38.108 | INFO | "huge" : "", | 20260507 19:43:38.108 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:38.108 | INFO | "malloc" : " 14 GB", | 20260507 19:43:38.108 | INFO | "pinned" : " 29 GB" | 20260507 19:43:38.108 | INFO | } | 20260507 19:43:38.108 | INFO | Perf report for the last 5.13974 seconds: | 20260507 19:43:38.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:38.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.108 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:43:38.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.108 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 19:43:38.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:38.109 | INFO | Uploaded 36GiB to GPU in 2.52047s (14.2831GiB/s) | 20260507 19:43:38.109 | INFO | Downloaded 34GiB to GPU in 2.43453s (13.9658GiB/s) | 20260507 19:43:38.220 | INFO | Starting a round of Baz IO | 20260507 19:43:41.703 | INFO | Populating RTMetrics report | 20260507 19:43:41.704 | INFO | RTMetrics report populated | 20260507 19:43:41.704 | INFO | Opening tmp RTMetrics json file | 20260507 19:43:41.715 | INFO | Serializing RTMetrics to json | 20260507 19:43:41.731 | INFO | Serialization done, swapping tmp file now... | 20260507 19:43:41.731 | INFO | RTMetrics done | 20260507 19:43:41.798 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.577 seconds | 20260507 19:43:41.798 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.13% / 0.01% / 54.86% | 20260507 19:43:41.798 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.28315% / 1.98682e-05% / 5.16574e-05% | 20260507 19:43:41.798 | INFO | Baz IO overall: | 20260507 19:43:41.798 | INFO | BytesWritten : 950401516154 | 20260507 19:43:41.798 | INFO | NumEvents : 460786100481 | 20260507 19:43:43.247 | INFO | Analyzing chunk frames = [2101248,2101760) | 20260507 19:43:43.247 | INFO | MemInfo: { | 20260507 19:43:43.247 | INFO | "gpu" : " 17 GB", | 20260507 19:43:43.247 | INFO | "huge" : "", | 20260507 19:43:43.247 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:43.247 | INFO | "malloc" : " 14 GB", | 20260507 19:43:43.247 | INFO | "pinned" : " 29 GB" | 20260507 19:43:43.247 | INFO | } | 20260507 19:43:43.247 | INFO | Perf report for the last 5.13905 seconds: | 20260507 19:43:43.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:43.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:43.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:43.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:43.247 | INFO | ANALYSIS : 69.2%, 2.9, 105.1ms, 0 | 20260507 19:43:43.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:43.247 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 19:43:43.247 | INFO | PRE_HQ : 68.1%, 1.8, 63.1ms, 0 | 20260507 19:43:43.247 | INFO | BAZWRITER : 69.2%, 4.3, 154.1ms, 0 | 20260507 19:43:43.247 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:43:43.247 | INFO | Uploaded 36GiB to GPU in 2.56602s (14.0295GiB/s) | 20260507 19:43:43.247 | INFO | Downloaded 38GiB to GPU in 2.70212s (14.063GiB/s) | 20260507 19:43:48.386 | INFO | Analyzing chunk frames = [2101760,2102272) | 20260507 19:43:48.386 | INFO | MemInfo: { | 20260507 19:43:48.386 | INFO | "gpu" : " 17 GB", | 20260507 19:43:48.386 | INFO | "huge" : "", | 20260507 19:43:48.386 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:48.386 | INFO | "malloc" : " 14 GB", | 20260507 19:43:48.386 | INFO | "pinned" : " 29 GB" | 20260507 19:43:48.386 | INFO | } | 20260507 19:43:48.386 | INFO | Perf report for the last 5.13926 seconds: | 20260507 19:43:48.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:48.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:43:48.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 19:43:48.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:48.386 | INFO | Uploaded 36GiB to GPU in 2.57906s (13.9586GiB/s) | 20260507 19:43:48.386 | INFO | Downloaded 34GiB to GPU in 2.48209s (13.6981GiB/s) | 20260507 19:43:53.525 | INFO | Analyzing chunk frames = [2102272,2102784) | 20260507 19:43:53.526 | INFO | MemInfo: { | 20260507 19:43:53.526 | INFO | "gpu" : " 17 GB", | 20260507 19:43:53.526 | INFO | "huge" : "", | 20260507 19:43:53.526 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:53.526 | INFO | "malloc" : " 14 GB", | 20260507 19:43:53.526 | INFO | "pinned" : " 29 GB" | 20260507 19:43:53.526 | INFO | } | 20260507 19:43:53.526 | INFO | Perf report for the last 5.13968 seconds: | 20260507 19:43:53.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:53.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:43:53.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 19:43:53.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:53.526 | INFO | Uploaded 36GiB to GPU in 2.57062s (14.0044GiB/s) | 20260507 19:43:53.526 | INFO | Downloaded 34GiB to GPU in 2.45362s (13.8571GiB/s) | 20260507 19:43:58.664 | INFO | Analyzing chunk frames = [2102784,2103296) | 20260507 19:43:58.664 | INFO | MemInfo: { | 20260507 19:43:58.664 | INFO | "gpu" : " 17 GB", | 20260507 19:43:58.664 | INFO | "huge" : "", | 20260507 19:43:58.664 | INFO | "huge_shared" : " 29 GB", | 20260507 19:43:58.664 | INFO | "malloc" : " 14 GB", | 20260507 19:43:58.664 | INFO | "pinned" : " 29 GB" | 20260507 19:43:58.664 | INFO | } | 20260507 19:43:58.664 | INFO | Perf report for the last 5.13844 seconds: | 20260507 19:43:58.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:43:58.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.664 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:43:58.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.664 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:43:58.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:43:58.665 | INFO | Uploaded 36GiB to GPU in 2.59106s (13.8939GiB/s) | 20260507 19:43:58.665 | INFO | Downloaded 34GiB to GPU in 2.45192s (13.8667GiB/s) | 20260507 19:44:03.804 | INFO | Analyzing chunk frames = [2103296,2103808) | 20260507 19:44:03.804 | INFO | MemInfo: { | 20260507 19:44:03.804 | INFO | "gpu" : " 17 GB", | 20260507 19:44:03.805 | INFO | "huge" : "", | 20260507 19:44:03.805 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:03.805 | INFO | "malloc" : " 14 GB", | 20260507 19:44:03.805 | INFO | "pinned" : " 29 GB" | 20260507 19:44:03.805 | INFO | } | 20260507 19:44:03.805 | INFO | Perf report for the last 5.14049 seconds: | 20260507 19:44:03.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:03.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 19:44:03.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:44:03.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:03.805 | INFO | Uploaded 36GiB to GPU in 2.40227s (14.9858GiB/s) | 20260507 19:44:03.805 | INFO | Downloaded 34GiB to GPU in 2.49353s (13.6353GiB/s) | 20260507 19:44:08.945 | INFO | Analyzing chunk frames = [2103808,2104320) | 20260507 19:44:08.945 | INFO | MemInfo: { | 20260507 19:44:08.945 | INFO | "gpu" : " 17 GB", | 20260507 19:44:08.945 | INFO | "huge" : "", | 20260507 19:44:08.945 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:08.946 | INFO | "malloc" : " 14 GB", | 20260507 19:44:08.946 | INFO | "pinned" : " 29 GB" | 20260507 19:44:08.946 | INFO | } | 20260507 19:44:08.946 | INFO | Perf report for the last 5.14095 seconds: | 20260507 19:44:08.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:08.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 19:44:08.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:44:08.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:08.946 | INFO | Uploaded 36GiB to GPU in 2.52153s (14.277GiB/s) | 20260507 19:44:08.946 | INFO | Downloaded 34GiB to GPU in 2.41626s (14.0713GiB/s) | 20260507 19:44:14.082 | INFO | Analyzing chunk frames = [2104320,2104832) | 20260507 19:44:14.082 | INFO | MemInfo: { | 20260507 19:44:14.082 | INFO | "gpu" : " 17 GB", | 20260507 19:44:14.082 | INFO | "huge" : "", | 20260507 19:44:14.082 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:14.082 | INFO | "malloc" : " 14 GB", | 20260507 19:44:14.082 | INFO | "pinned" : " 29 GB" | 20260507 19:44:14.083 | INFO | } | 20260507 19:44:14.083 | INFO | Perf report for the last 5.13687 seconds: | 20260507 19:44:14.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:14.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 19:44:14.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 19:44:14.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:14.083 | INFO | Uploaded 36GiB to GPU in 2.53317s (14.2114GiB/s) | 20260507 19:44:14.083 | INFO | Downloaded 34GiB to GPU in 2.4349s (13.9636GiB/s) | 20260507 19:44:19.221 | INFO | Analyzing chunk frames = [2104832,2105344) | 20260507 19:44:19.221 | INFO | MemInfo: { | 20260507 19:44:19.221 | INFO | "gpu" : " 17 GB", | 20260507 19:44:19.221 | INFO | "huge" : "", | 20260507 19:44:19.221 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:19.221 | INFO | "malloc" : " 14 GB", | 20260507 19:44:19.221 | INFO | "pinned" : " 29 GB" | 20260507 19:44:19.221 | INFO | } | 20260507 19:44:19.221 | INFO | Perf report for the last 5.13881 seconds: | 20260507 19:44:19.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:19.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.222 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 19:44:19.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.222 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 19:44:19.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:19.222 | INFO | Uploaded 36GiB to GPU in 2.53943s (14.1764GiB/s) | 20260507 19:44:19.222 | INFO | Downloaded 34GiB to GPU in 2.43189s (13.9809GiB/s) | 20260507 19:44:19.222 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:44:19.222 | INFO | Total mem usage: 16.4 GiB | 20260507 19:44:19.222 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:44:19.222 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.2%) | 20260507 19:44:19.222 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:44:19.222 | INFO | High Water Breakdown: | 20260507 19:44:19.222 | INFO | 59.8% : 8.91 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:44:19.222 | INFO | 40% : 5.96 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:44:19.222 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:44:19.222 | INFO | ------------------------------------------ | 20260507 19:44:19.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:44:19.222 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:44:19.222 | INFO | Total mem usage: 26.5 GiB | 20260507 19:44:19.222 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:44:19.222 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:44:19.222 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:44:19.222 | INFO | High Water Breakdown: | 20260507 19:44:19.222 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:44:19.222 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:44:19.222 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:44:19.222 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:44:19.222 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:44:19.222 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:44:19.222 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:44:19.222 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:44:19.222 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:44:19.222 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:44:19.222 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:44:19.222 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:44:19.222 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:44:19.222 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:44:19.222 | INFO | ------------------------------------------ | 20260507 19:44:19.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:44:19.222 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:44:19.222 | INFO | Total mem usage: 27.8 GiB | 20260507 19:44:19.222 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:44:19.222 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:44:19.222 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:44:19.222 | INFO | High Water Breakdown: | 20260507 19:44:19.222 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:44:19.222 | INFO | ------------------------------------------ | 20260507 19:44:19.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:44:19.222 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:44:19.222 | INFO | Total mem usage: 15.7 GiB | 20260507 19:44:19.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:44:19.223 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:44:19.223 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:44:19.223 | INFO | High Water Breakdown: | 20260507 19:44:19.223 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:44:19.223 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:44:19.223 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:44:19.223 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:44:19.223 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:44:19.223 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:44:19.223 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:44:19.223 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:44:19.223 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:44:19.223 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:44:19.223 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:44:19.223 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:44:19.223 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:44:19.223 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:44:19.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:44:19.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:44:19.223 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:44:19.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:44:19.223 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:44:19.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:44:19.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:44:19.223 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:44:19.223 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:44:19.223 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:44:19.223 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:44:19.223 | INFO | ------------------------------------------ | 20260507 19:44:19.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:44:19.223 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:44:19.223 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:44:19.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:44:19.223 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.60e+01, 1.44e+01 / 1.96e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 0.30%, AggregateStats -- 9.96e-02 / 1.07e-01, 1.28e-02 / 1.97e-02 | 20260507 19:44:19.223 | INFO | PerfReport: 12.08%, Baseline -- 4.01e+00 / 4.47e+00, 6.02e-01 / 3.17e-01 | 20260507 19:44:19.223 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.51e+00, 2.12e-01 / 1.14e-01 | 20260507 19:44:19.223 | INFO | PerfReport: 15.71%, DME -- 4.17e+01 / 4.47e+01, 3.72e+00 / 1.25e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.86e+00 / 1.87e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.49e+00 / 2.83e+00, 4.49e-01 / 2.57e-01 | 20260507 19:44:19.223 | INFO | PerfReport: 3.61%, Metrics -- 1.20e+00 / 1.38e+00, 3.06e-01 / 1.71e-01 | 20260507 19:44:19.223 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.78e-01 / 4.05e-01, 6.26e-02 / 3.36e-02 | 20260507 19:44:19.223 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:44:19.223 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:44:19.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:44:19.223 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 24.61%, Upload -- 2.64e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 24.12%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 30.95%, Compute -- 3.32e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 2.34%, LockUpload -- 2.51e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 2.09%, LockDownload -- 2.24e+00 / 0.00e+00, 3.52e+00 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 15.88%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:44:19.223 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 19:44:19.335 | INFO | Starting a round of Baz IO | 20260507 19:44:22.836 | INFO | Populating RTMetrics report | 20260507 19:44:22.837 | INFO | RTMetrics report populated | 20260507 19:44:22.837 | INFO | Opening tmp RTMetrics json file | 20260507 19:44:22.838 | INFO | Serializing RTMetrics to json | 20260507 19:44:22.839 | INFO | Serialization done, swapping tmp file now... | 20260507 19:44:22.840 | INFO | RTMetrics done | 20260507 19:44:22.961 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.626 seconds | 20260507 19:44:22.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.12% / 0.01% / 54.87% | 20260507 19:44:22.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.28034% / 3.97364e-06% / 2.38419e-05% | 20260507 19:44:22.961 | INFO | Baz IO overall: | 20260507 19:44:22.961 | INFO | BytesWritten : 952320453043 | 20260507 19:44:22.962 | INFO | NumEvents : 461715976816 | 20260507 19:44:24.362 | INFO | Analyzing chunk frames = [2105344,2105856) | 20260507 19:44:24.362 | INFO | MemInfo: { | 20260507 19:44:24.362 | INFO | "gpu" : " 17 GB", | 20260507 19:44:24.362 | INFO | "huge" : "", | 20260507 19:44:24.362 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:24.362 | INFO | "malloc" : " 14 GB", | 20260507 19:44:24.362 | INFO | "pinned" : " 29 GB" | 20260507 19:44:24.362 | INFO | } | 20260507 19:44:24.362 | INFO | Perf report for the last 5.14051 seconds: | 20260507 19:44:24.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:24.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:24.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:24.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:24.362 | INFO | ANALYSIS : 69.8%, 2.9, 105.0ms, 0 | 20260507 19:44:24.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:24.362 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 19:44:24.362 | INFO | PRE_HQ : 68.6%, 1.8, 64.8ms, 0 | 20260507 19:44:24.362 | INFO | BAZWRITER : 70.0%, 4.1, 147.5ms, 0 | 20260507 19:44:24.362 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 19:44:24.362 | INFO | Uploaded 36GiB to GPU in 2.53059s (14.2259GiB/s) | 20260507 19:44:24.362 | INFO | Downloaded 38GiB to GPU in 2.69128s (14.1197GiB/s) | 20260507 19:44:29.501 | INFO | Analyzing chunk frames = [2105856,2106368) | 20260507 19:44:29.501 | INFO | MemInfo: { | 20260507 19:44:29.501 | INFO | "gpu" : " 17 GB", | 20260507 19:44:29.501 | INFO | "huge" : "", | 20260507 19:44:29.501 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:29.501 | INFO | "malloc" : " 14 GB", | 20260507 19:44:29.501 | INFO | "pinned" : " 29 GB" | 20260507 19:44:29.501 | INFO | } | 20260507 19:44:29.501 | INFO | Perf report for the last 5.139 seconds: | 20260507 19:44:29.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:29.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 19:44:29.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 19:44:29.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:29.501 | INFO | Uploaded 36GiB to GPU in 2.59148s (13.8917GiB/s) | 20260507 19:44:29.501 | INFO | Downloaded 34GiB to GPU in 2.46584s (13.7884GiB/s) | 20260507 19:44:34.639 | INFO | Analyzing chunk frames = [2106368,2106880) | 20260507 19:44:34.639 | INFO | MemInfo: { | 20260507 19:44:34.639 | INFO | "gpu" : " 17 GB", | 20260507 19:44:34.639 | INFO | "huge" : "", | 20260507 19:44:34.639 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:34.639 | INFO | "malloc" : " 14 GB", | 20260507 19:44:34.640 | INFO | "pinned" : " 29 GB" | 20260507 19:44:34.640 | INFO | } | 20260507 19:44:34.640 | INFO | Perf report for the last 5.13867 seconds: | 20260507 19:44:34.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:34.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:44:34.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:44:34.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:34.640 | INFO | Uploaded 36GiB to GPU in 2.59134s (13.8924GiB/s) | 20260507 19:44:34.640 | INFO | Downloaded 34GiB to GPU in 2.46387s (13.7994GiB/s) | 20260507 19:44:39.779 | INFO | Analyzing chunk frames = [2106880,2107392) | 20260507 19:44:39.779 | INFO | MemInfo: { | 20260507 19:44:39.779 | INFO | "gpu" : " 17 GB", | 20260507 19:44:39.779 | INFO | "huge" : "", | 20260507 19:44:39.779 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:39.779 | INFO | "malloc" : " 14 GB", | 20260507 19:44:39.779 | INFO | "pinned" : " 29 GB" | 20260507 19:44:39.779 | INFO | } | 20260507 19:44:39.779 | INFO | Perf report for the last 5.13916 seconds: | 20260507 19:44:39.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:39.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:44:39.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:44:39.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:39.779 | INFO | Uploaded 36GiB to GPU in 2.48689s (14.4759GiB/s) | 20260507 19:44:39.779 | INFO | Downloaded 34GiB to GPU in 2.37865s (14.2938GiB/s) | 20260507 19:44:44.918 | INFO | Analyzing chunk frames = [2107392,2107904) | 20260507 19:44:44.918 | INFO | MemInfo: { | 20260507 19:44:44.918 | INFO | "gpu" : " 17 GB", | 20260507 19:44:44.918 | INFO | "huge" : "", | 20260507 19:44:44.918 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:44.919 | INFO | "malloc" : " 14 GB", | 20260507 19:44:44.919 | INFO | "pinned" : " 29 GB" | 20260507 19:44:44.919 | INFO | } | 20260507 19:44:44.919 | INFO | Perf report for the last 5.13983 seconds: | 20260507 19:44:44.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:44.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:44:44.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:44:44.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:44.919 | INFO | Uploaded 36GiB to GPU in 2.40343s (14.9786GiB/s) | 20260507 19:44:44.919 | INFO | Downloaded 34GiB to GPU in 2.48874s (13.6615GiB/s) | 20260507 19:44:50.058 | INFO | Analyzing chunk frames = [2107904,2108416) | 20260507 19:44:50.058 | INFO | MemInfo: { | 20260507 19:44:50.058 | INFO | "gpu" : " 17 GB", | 20260507 19:44:50.058 | INFO | "huge" : "", | 20260507 19:44:50.058 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:50.058 | INFO | "malloc" : " 14 GB", | 20260507 19:44:50.058 | INFO | "pinned" : " 29 GB" | 20260507 19:44:50.058 | INFO | } | 20260507 19:44:50.058 | INFO | Perf report for the last 5.13965 seconds: | 20260507 19:44:50.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:50.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.058 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:44:50.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.058 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:44:50.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:50.059 | INFO | Uploaded 36GiB to GPU in 2.51781s (14.2981GiB/s) | 20260507 19:44:50.059 | INFO | Downloaded 34GiB to GPU in 2.41049s (14.105GiB/s) | 20260507 19:44:55.200 | INFO | Analyzing chunk frames = [2108416,2108928) | 20260507 19:44:55.200 | INFO | MemInfo: { | 20260507 19:44:55.200 | INFO | "gpu" : " 17 GB", | 20260507 19:44:55.200 | INFO | "huge" : "", | 20260507 19:44:55.200 | INFO | "huge_shared" : " 29 GB", | 20260507 19:44:55.200 | INFO | "malloc" : " 14 GB", | 20260507 19:44:55.200 | INFO | "pinned" : " 29 GB" | 20260507 19:44:55.200 | INFO | } | 20260507 19:44:55.200 | INFO | Perf report for the last 5.1418 seconds: | 20260507 19:44:55.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:44:55.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.200 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:44:55.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.200 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 19:44:55.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:44:55.201 | INFO | Uploaded 36GiB to GPU in 2.55318s (14.1001GiB/s) | 20260507 19:44:55.201 | INFO | Downloaded 34GiB to GPU in 2.43589s (13.958GiB/s) | 20260507 19:45:00.337 | INFO | Analyzing chunk frames = [2108928,2109440) | 20260507 19:45:00.337 | INFO | MemInfo: { | 20260507 19:45:00.337 | INFO | "gpu" : " 17 GB", | 20260507 19:45:00.337 | INFO | "huge" : "", | 20260507 19:45:00.337 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:00.337 | INFO | "malloc" : " 14 GB", | 20260507 19:45:00.337 | INFO | "pinned" : " 29 GB" | 20260507 19:45:00.337 | INFO | } | 20260507 19:45:00.337 | INFO | Perf report for the last 5.13681 seconds: | 20260507 19:45:00.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:00.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:45:00.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:45:00.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:00.337 | INFO | Uploaded 36GiB to GPU in 2.53294s (14.2127GiB/s) | 20260507 19:45:00.337 | INFO | Downloaded 34GiB to GPU in 2.41178s (14.0975GiB/s) | 20260507 19:45:00.434 | INFO | Starting a round of Baz IO | 20260507 19:45:03.917 | INFO | Populating RTMetrics report | 20260507 19:45:03.917 | INFO | RTMetrics report populated | 20260507 19:45:03.917 | INFO | Opening tmp RTMetrics json file | 20260507 19:45:03.937 | INFO | Serializing RTMetrics to json | 20260507 19:45:03.939 | INFO | Serialization done, swapping tmp file now... | 20260507 19:45:03.940 | INFO | RTMetrics done | 20260507 19:45:04.007 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.572 seconds | 20260507 19:45:04.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.11% / 0.01% / 54.88% | 20260507 19:45:04.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.27389% / 1.98682e-05% / 3.97364e-05% | 20260507 19:45:04.007 | INFO | Baz IO overall: | 20260507 19:45:04.007 | INFO | BytesWritten : 954238838244 | 20260507 19:45:04.008 | INFO | NumEvents : 462645694637 | 20260507 19:45:05.476 | INFO | Analyzing chunk frames = [2109440,2109952) | 20260507 19:45:05.476 | INFO | MemInfo: { | 20260507 19:45:05.476 | INFO | "gpu" : " 17 GB", | 20260507 19:45:05.476 | INFO | "huge" : "", | 20260507 19:45:05.476 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:05.476 | INFO | "malloc" : " 14 GB", | 20260507 19:45:05.476 | INFO | "pinned" : " 29 GB" | 20260507 19:45:05.476 | INFO | } | 20260507 19:45:05.476 | INFO | Perf report for the last 5.13888 seconds: | 20260507 19:45:05.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:05.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:05.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:05.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:05.476 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 19:45:05.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:05.476 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 19:45:05.476 | INFO | PRE_HQ : 65.9%, 1.7, 58.0ms, 0 | 20260507 19:45:05.476 | INFO | BAZWRITER : 69.1%, 4.3, 151.7ms, 0 | 20260507 19:45:05.476 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 19:45:05.476 | INFO | Uploaded 36GiB to GPU in 2.5624s (14.0493GiB/s) | 20260507 19:45:05.477 | INFO | Downloaded 38GiB to GPU in 2.69347s (14.1082GiB/s) | 20260507 19:45:10.615 | INFO | Analyzing chunk frames = [2109952,2110464) | 20260507 19:45:10.615 | INFO | MemInfo: { | 20260507 19:45:10.615 | INFO | "gpu" : " 17 GB", | 20260507 19:45:10.615 | INFO | "huge" : "", | 20260507 19:45:10.615 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:10.615 | INFO | "malloc" : " 14 GB", | 20260507 19:45:10.615 | INFO | "pinned" : " 29 GB" | 20260507 19:45:10.615 | INFO | } | 20260507 19:45:10.616 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:45:10.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:10.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 19:45:10.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:45:10.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:10.616 | INFO | Uploaded 36GiB to GPU in 2.57975s (13.9548GiB/s) | 20260507 19:45:10.616 | INFO | Downloaded 34GiB to GPU in 2.45995s (13.8214GiB/s) | 20260507 19:45:15.754 | INFO | Analyzing chunk frames = [2110464,2110976) | 20260507 19:45:15.754 | INFO | MemInfo: { | 20260507 19:45:15.754 | INFO | "gpu" : " 17 GB", | 20260507 19:45:15.754 | INFO | "huge" : "", | 20260507 19:45:15.754 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:15.754 | INFO | "malloc" : " 14 GB", | 20260507 19:45:15.754 | INFO | "pinned" : " 29 GB" | 20260507 19:45:15.754 | INFO | } | 20260507 19:45:15.754 | INFO | Perf report for the last 5.13886 seconds: | 20260507 19:45:15.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:15.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.754 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 19:45:15.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.754 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:45:15.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:15.755 | INFO | Uploaded 36GiB to GPU in 2.58899s (13.905GiB/s) | 20260507 19:45:15.755 | INFO | Downloaded 34GiB to GPU in 2.45872s (13.8284GiB/s) | 20260507 19:45:20.894 | INFO | Analyzing chunk frames = [2110976,2111488) | 20260507 19:45:20.894 | INFO | MemInfo: { | 20260507 19:45:20.894 | INFO | "gpu" : " 17 GB", | 20260507 19:45:20.894 | INFO | "huge" : "", | 20260507 19:45:20.894 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:20.894 | INFO | "malloc" : " 14 GB", | 20260507 19:45:20.894 | INFO | "pinned" : " 29 GB" | 20260507 19:45:20.894 | INFO | } | 20260507 19:45:20.894 | INFO | Perf report for the last 5.13963 seconds: | 20260507 19:45:20.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:20.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.894 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:45:20.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.894 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:45:20.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:20.895 | INFO | Uploaded 36GiB to GPU in 2.48865s (14.4657GiB/s) | 20260507 19:45:20.895 | INFO | Downloaded 34GiB to GPU in 2.38232s (14.2718GiB/s) | 20260507 19:45:26.033 | INFO | Analyzing chunk frames = [2111488,2112000) | 20260507 19:45:26.033 | INFO | MemInfo: { | 20260507 19:45:26.033 | INFO | "gpu" : " 17 GB", | 20260507 19:45:26.033 | INFO | "huge" : "", | 20260507 19:45:26.033 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:26.033 | INFO | "malloc" : " 14 GB", | 20260507 19:45:26.033 | INFO | "pinned" : " 29 GB" | 20260507 19:45:26.033 | INFO | } | 20260507 19:45:26.033 | INFO | Perf report for the last 5.13932 seconds: | 20260507 19:45:26.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:26.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.033 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 19:45:26.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.033 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:45:26.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:26.034 | INFO | Uploaded 36GiB to GPU in 2.47049s (14.572GiB/s) | 20260507 19:45:26.034 | INFO | Downloaded 34GiB to GPU in 2.49734s (13.6145GiB/s) | 20260507 19:45:31.172 | INFO | Analyzing chunk frames = [2112000,2112512) | 20260507 19:45:31.172 | INFO | MemInfo: { | 20260507 19:45:31.172 | INFO | "gpu" : " 17 GB", | 20260507 19:45:31.172 | INFO | "huge" : "", | 20260507 19:45:31.173 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:31.173 | INFO | "malloc" : " 14 GB", | 20260507 19:45:31.173 | INFO | "pinned" : " 29 GB" | 20260507 19:45:31.173 | INFO | } | 20260507 19:45:31.173 | INFO | Perf report for the last 5.13934 seconds: | 20260507 19:45:31.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:31.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 19:45:31.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260507 19:45:31.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:31.173 | INFO | Uploaded 36GiB to GPU in 2.53309s (14.2119GiB/s) | 20260507 19:45:31.173 | INFO | Downloaded 34GiB to GPU in 2.41934s (14.0534GiB/s) | 20260507 19:45:36.312 | INFO | Analyzing chunk frames = [2112512,2113024) | 20260507 19:45:36.312 | INFO | MemInfo: { | 20260507 19:45:36.313 | INFO | "gpu" : " 17 GB", | 20260507 19:45:36.313 | INFO | "huge" : "", | 20260507 19:45:36.313 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:36.313 | INFO | "malloc" : " 14 GB", | 20260507 19:45:36.313 | INFO | "pinned" : " 29 GB" | 20260507 19:45:36.313 | INFO | } | 20260507 19:45:36.313 | INFO | Perf report for the last 5.14043 seconds: | 20260507 19:45:36.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:36.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.313 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:45:36.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.313 | INFO | PRE_HQ : 37.4%, 1.0, 19.5ms, 0 | 20260507 19:45:36.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:36.314 | INFO | Uploaded 36GiB to GPU in 2.53519s (14.2001GiB/s) | 20260507 19:45:36.314 | INFO | Downloaded 34GiB to GPU in 2.42436s (14.0243GiB/s) | 20260507 19:45:41.452 | INFO | Analyzing chunk frames = [2113024,2113536) | 20260507 19:45:41.452 | INFO | MemInfo: { | 20260507 19:45:41.452 | INFO | "gpu" : " 17 GB", | 20260507 19:45:41.452 | INFO | "huge" : "", | 20260507 19:45:41.452 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:41.452 | INFO | "malloc" : " 14 GB", | 20260507 19:45:41.452 | INFO | "pinned" : " 29 GB" | 20260507 19:45:41.452 | INFO | } | 20260507 19:45:41.453 | INFO | Perf report for the last 5.13936 seconds: | 20260507 19:45:41.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:41.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:45:41.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | PRE_HQ : 37.6%, 1.0, 19.5ms, 0 | 20260507 19:45:41.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:41.453 | INFO | Uploaded 36GiB to GPU in 2.54163s (14.1641GiB/s) | 20260507 19:45:41.453 | INFO | Downloaded 34GiB to GPU in 2.41688s (14.0677GiB/s) | 20260507 19:45:41.453 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:45:41.453 | INFO | Total mem usage: 16.4 GiB | 20260507 19:45:41.453 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:45:41.453 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.57%) | 20260507 19:45:41.453 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:45:41.453 | INFO | High Water Breakdown: | 20260507 19:45:41.453 | INFO | 60.6% : 9.41 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:45:41.453 | INFO | 39.3% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:45:41.453 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:45:41.453 | INFO | ------------------------------------------ | 20260507 19:45:41.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:45:41.453 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:45:41.453 | INFO | Total mem usage: 26.5 GiB | 20260507 19:45:41.453 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:45:41.453 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:45:41.453 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:45:41.453 | INFO | High Water Breakdown: | 20260507 19:45:41.453 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:45:41.453 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:45:41.453 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:45:41.453 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:45:41.453 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:45:41.453 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:45:41.453 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:45:41.453 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:45:41.453 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:45:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:45:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:45:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:45:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:45:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:45:41.453 | INFO | ------------------------------------------ | 20260507 19:45:41.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:45:41.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:45:41.454 | INFO | Total mem usage: 27.8 GiB | 20260507 19:45:41.454 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:45:41.454 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:45:41.454 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:45:41.454 | INFO | High Water Breakdown: | 20260507 19:45:41.454 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:45:41.454 | INFO | ------------------------------------------ | 20260507 19:45:41.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:45:41.454 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:45:41.454 | INFO | Total mem usage: 15.7 GiB | 20260507 19:45:41.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:45:41.454 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:45:41.454 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:45:41.454 | INFO | High Water Breakdown: | 20260507 19:45:41.454 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:45:41.454 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:45:41.454 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:45:41.454 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:45:41.454 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:45:41.454 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:45:41.454 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:45:41.454 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:45:41.454 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:45:41.454 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:45:41.454 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:45:41.454 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:45:41.454 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:45:41.454 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:45:41.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:45:41.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:45:41.454 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:45:41.454 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:45:41.454 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:45:41.454 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:45:41.454 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:45:41.454 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:45:41.454 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:45:41.454 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:45:41.454 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:45:41.454 | INFO | ------------------------------------------ | 20260507 19:45:41.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:45:41.454 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:45:41.454 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:45:41.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:45:41.454 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.38e+01 / 1.83e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.03e-01, 1.19e-02 / 6.57e-03 | 20260507 19:45:41.454 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.45e+00, 5.94e-01 / 3.02e-01 | 20260507 19:45:41.454 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.27e-01 | 20260507 19:45:41.454 | INFO | PerfReport: 15.75%, DME -- 4.18e+01 / 4.49e+01, 3.81e+00 / 1.32e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.82e+00 / 2.01e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.39e-01 / 2.37e-01 | 20260507 19:45:41.454 | INFO | PerfReport: 3.63%, Metrics -- 1.20e+00 / 1.41e+00, 3.04e-01 / 1.71e-01 | 20260507 19:45:41.454 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.78e-01 / 4.11e-01, 6.17e-02 / 3.51e-02 | 20260507 19:45:41.454 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:45:41.454 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:45:41.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:45:41.454 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 24.63%, Upload -- 2.64e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 24.02%, Download -- 2.57e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 30.98%, Compute -- 3.32e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 2.28%, LockUpload -- 2.44e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 2.06%, LockDownload -- 2.20e+00 / 0.00e+00, 3.42e+00 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 16.02%, LockCompute -- 1.72e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 19:45:41.454 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 19:45:41.563 | INFO | Starting a round of Baz IO | 20260507 19:45:45.044 | INFO | Populating RTMetrics report | 20260507 19:45:45.044 | INFO | RTMetrics report populated | 20260507 19:45:45.044 | INFO | Opening tmp RTMetrics json file | 20260507 19:45:45.064 | INFO | Serializing RTMetrics to json | 20260507 19:45:45.067 | INFO | Serialization done, swapping tmp file now... | 20260507 19:45:45.068 | INFO | RTMetrics done | 20260507 19:45:45.139 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.576 seconds | 20260507 19:45:45.139 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.10% / 0.01% / 54.89% | 20260507 19:45:45.139 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.27293% / 3.17891e-05% / 2.78155e-05% | 20260507 19:45:45.139 | INFO | Baz IO overall: | 20260507 19:45:45.139 | INFO | BytesWritten : 956157843134 | 20260507 19:45:45.139 | INFO | NumEvents : 463575932761 | 20260507 19:45:46.590 | INFO | Analyzing chunk frames = [2113536,2114048) | 20260507 19:45:46.590 | INFO | MemInfo: { | 20260507 19:45:46.590 | INFO | "gpu" : " 17 GB", | 20260507 19:45:46.590 | INFO | "huge" : "", | 20260507 19:45:46.590 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:46.590 | INFO | "malloc" : " 14 GB", | 20260507 19:45:46.590 | INFO | "pinned" : " 29 GB" | 20260507 19:45:46.590 | INFO | } | 20260507 19:45:46.590 | INFO | Perf report for the last 5.13781 seconds: | 20260507 19:45:46.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:46.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:46.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:46.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:46.590 | INFO | ANALYSIS : 69.2%, 2.9, 104.6ms, 0 | 20260507 19:45:46.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:46.590 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 19:45:46.590 | INFO | PRE_HQ : 68.4%, 1.8, 63.7ms, 0 | 20260507 19:45:46.591 | INFO | BAZWRITER : 69.1%, 4.6, 161.8ms, 0 | 20260507 19:45:46.591 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 19:45:46.591 | INFO | Uploaded 36GiB to GPU in 2.53997s (14.1734GiB/s) | 20260507 19:45:46.591 | INFO | Downloaded 38GiB to GPU in 2.70945s (14.025GiB/s) | 20260507 19:45:51.730 | INFO | Analyzing chunk frames = [2114048,2114560) | 20260507 19:45:51.730 | INFO | MemInfo: { | 20260507 19:45:51.730 | INFO | "gpu" : " 17 GB", | 20260507 19:45:51.730 | INFO | "huge" : "", | 20260507 19:45:51.730 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:51.730 | INFO | "malloc" : " 14 GB", | 20260507 19:45:51.730 | INFO | "pinned" : " 29 GB" | 20260507 19:45:51.730 | INFO | } | 20260507 19:45:51.730 | INFO | Perf report for the last 5.13946 seconds: | 20260507 19:45:51.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:51.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:45:51.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:45:51.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:51.730 | INFO | Uploaded 36GiB to GPU in 2.55081s (14.1131GiB/s) | 20260507 19:45:51.730 | INFO | Downloaded 34GiB to GPU in 2.47184s (13.755GiB/s) | 20260507 19:45:56.869 | INFO | Analyzing chunk frames = [2114560,2115072) | 20260507 19:45:56.869 | INFO | MemInfo: { | 20260507 19:45:56.869 | INFO | "gpu" : " 17 GB", | 20260507 19:45:56.869 | INFO | "huge" : "", | 20260507 19:45:56.869 | INFO | "huge_shared" : " 29 GB", | 20260507 19:45:56.869 | INFO | "malloc" : " 14 GB", | 20260507 19:45:56.869 | INFO | "pinned" : " 29 GB" | 20260507 19:45:56.869 | INFO | } | 20260507 19:45:56.869 | INFO | Perf report for the last 5.139 seconds: | 20260507 19:45:56.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:45:56.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.869 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:45:56.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.869 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:45:56.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:45:56.870 | INFO | Uploaded 36GiB to GPU in 2.5916s (13.891GiB/s) | 20260507 19:45:56.870 | INFO | Downloaded 34GiB to GPU in 2.46388s (13.7994GiB/s) | 20260507 19:46:02.009 | INFO | Analyzing chunk frames = [2115072,2115584) | 20260507 19:46:02.009 | INFO | MemInfo: { | 20260507 19:46:02.009 | INFO | "gpu" : " 17 GB", | 20260507 19:46:02.009 | INFO | "huge" : "", | 20260507 19:46:02.009 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:02.009 | INFO | "malloc" : " 14 GB", | 20260507 19:46:02.009 | INFO | "pinned" : " 29 GB" | 20260507 19:46:02.009 | INFO | } | 20260507 19:46:02.009 | INFO | Perf report for the last 5.1403 seconds: | 20260507 19:46:02.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:02.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.009 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:46:02.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.009 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:46:02.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:02.010 | INFO | Uploaded 36GiB to GPU in 2.59917s (13.8506GiB/s) | 20260507 19:46:02.010 | INFO | Downloaded 34GiB to GPU in 2.45357s (13.8574GiB/s) | 20260507 19:46:07.149 | INFO | Analyzing chunk frames = [2115584,2116096) | 20260507 19:46:07.149 | INFO | MemInfo: { | 20260507 19:46:07.149 | INFO | "gpu" : " 17 GB", | 20260507 19:46:07.149 | INFO | "huge" : "", | 20260507 19:46:07.149 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:07.149 | INFO | "malloc" : " 14 GB", | 20260507 19:46:07.149 | INFO | "pinned" : " 29 GB" | 20260507 19:46:07.149 | INFO | } | 20260507 19:46:07.149 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:46:07.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:07.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260507 19:46:07.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:46:07.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:07.149 | INFO | Uploaded 36GiB to GPU in 2.49376s (14.4361GiB/s) | 20260507 19:46:07.149 | INFO | Downloaded 34GiB to GPU in 2.511s (13.5404GiB/s) | 20260507 19:46:12.288 | INFO | Analyzing chunk frames = [2116096,2116608) | 20260507 19:46:12.288 | INFO | MemInfo: { | 20260507 19:46:12.288 | INFO | "gpu" : " 17 GB", | 20260507 19:46:12.288 | INFO | "huge" : "", | 20260507 19:46:12.288 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:12.288 | INFO | "malloc" : " 14 GB", | 20260507 19:46:12.288 | INFO | "pinned" : " 29 GB" | 20260507 19:46:12.288 | INFO | } | 20260507 19:46:12.288 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:46:12.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:12.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.288 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:46:12.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.288 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:46:12.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:12.289 | INFO | Uploaded 36GiB to GPU in 2.50216s (14.3876GiB/s) | 20260507 19:46:12.289 | INFO | Downloaded 34GiB to GPU in 2.41316s (14.0894GiB/s) | 20260507 19:46:17.427 | INFO | Analyzing chunk frames = [2116608,2117120) | 20260507 19:46:17.427 | INFO | MemInfo: { | 20260507 19:46:17.427 | INFO | "gpu" : " 17 GB", | 20260507 19:46:17.427 | INFO | "huge" : "", | 20260507 19:46:17.427 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:17.427 | INFO | "malloc" : " 14 GB", | 20260507 19:46:17.427 | INFO | "pinned" : " 29 GB" | 20260507 19:46:17.427 | INFO | } | 20260507 19:46:17.427 | INFO | Perf report for the last 5.1387 seconds: | 20260507 19:46:17.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:17.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.428 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:46:17.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.428 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 19:46:17.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:17.428 | INFO | Uploaded 36GiB to GPU in 2.55793s (14.0739GiB/s) | 20260507 19:46:17.428 | INFO | Downloaded 34GiB to GPU in 2.44011s (13.9338GiB/s) | 20260507 19:46:22.566 | INFO | Analyzing chunk frames = [2117120,2117632) | 20260507 19:46:22.566 | INFO | MemInfo: { | 20260507 19:46:22.566 | INFO | "gpu" : " 17 GB", | 20260507 19:46:22.566 | INFO | "huge" : "", | 20260507 19:46:22.566 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:22.566 | INFO | "malloc" : " 14 GB", | 20260507 19:46:22.566 | INFO | "pinned" : " 29 GB" | 20260507 19:46:22.566 | INFO | } | 20260507 19:46:22.566 | INFO | Perf report for the last 5.13918 seconds: | 20260507 19:46:22.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:22.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.566 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:46:22.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.567 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 19:46:22.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:22.567 | INFO | Uploaded 36GiB to GPU in 2.51299s (14.3256GiB/s) | 20260507 19:46:22.567 | INFO | Downloaded 34GiB to GPU in 2.43138s (13.9838GiB/s) | 20260507 19:46:22.678 | INFO | Starting a round of Baz IO | 20260507 19:46:26.171 | INFO | Populating RTMetrics report | 20260507 19:46:26.172 | INFO | RTMetrics report populated | 20260507 19:46:26.172 | INFO | Opening tmp RTMetrics json file | 20260507 19:46:26.193 | INFO | Serializing RTMetrics to json | 20260507 19:46:26.194 | INFO | Serialization done, swapping tmp file now... | 20260507 19:46:26.195 | INFO | RTMetrics done | 20260507 19:46:26.255 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.576 seconds | 20260507 19:46:26.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.09% / 0.01% / 54.90% | 20260507 19:46:26.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.26701% / 1.19209e-05% / 5.5631e-05% | 20260507 19:46:26.255 | INFO | Baz IO overall: | 20260507 19:46:26.255 | INFO | BytesWritten : 958076460036 | 20260507 19:46:26.255 | INFO | NumEvents : 464506026563 | 20260507 19:46:27.705 | INFO | Analyzing chunk frames = [2117632,2118144) | 20260507 19:46:27.705 | INFO | MemInfo: { | 20260507 19:46:27.705 | INFO | "gpu" : " 17 GB", | 20260507 19:46:27.705 | INFO | "huge" : "", | 20260507 19:46:27.705 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:27.705 | INFO | "malloc" : " 14 GB", | 20260507 19:46:27.705 | INFO | "pinned" : " 29 GB" | 20260507 19:46:27.705 | INFO | } | 20260507 19:46:27.706 | INFO | Perf report for the last 5.13926 seconds: | 20260507 19:46:27.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:27.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:27.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:27.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:27.706 | INFO | ANALYSIS : 69.4%, 2.9, 104.8ms, 0 | 20260507 19:46:27.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:27.706 | INFO | RT_METRICS : 49.0%, 1.0, 25.7ms, 0 | 20260507 19:46:27.706 | INFO | PRE_HQ : 68.1%, 2.0, 69.2ms, 0 | 20260507 19:46:27.706 | INFO | BAZWRITER : 69.1%, 4.2, 150.5ms, 0 | 20260507 19:46:27.706 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 19:46:27.706 | INFO | Uploaded 36GiB to GPU in 2.52775s (14.2419GiB/s) | 20260507 19:46:27.706 | INFO | Downloaded 38GiB to GPU in 2.69084s (14.122GiB/s) | 20260507 19:46:32.844 | INFO | Analyzing chunk frames = [2118144,2118656) | 20260507 19:46:32.844 | INFO | MemInfo: { | 20260507 19:46:32.844 | INFO | "gpu" : " 17 GB", | 20260507 19:46:32.844 | INFO | "huge" : "", | 20260507 19:46:32.844 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:32.844 | INFO | "malloc" : " 14 GB", | 20260507 19:46:32.844 | INFO | "pinned" : " 29 GB" | 20260507 19:46:32.844 | INFO | } | 20260507 19:46:32.844 | INFO | Perf report for the last 5.13869 seconds: | 20260507 19:46:32.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:32.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.844 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:46:32.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.844 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:46:32.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:32.845 | INFO | Uploaded 36GiB to GPU in 2.54677s (14.1355GiB/s) | 20260507 19:46:32.845 | INFO | Downloaded 34GiB to GPU in 2.49659s (13.6186GiB/s) | 20260507 19:46:37.983 | INFO | Analyzing chunk frames = [2118656,2119168) | 20260507 19:46:37.983 | INFO | MemInfo: { | 20260507 19:46:37.983 | INFO | "gpu" : " 17 GB", | 20260507 19:46:37.983 | INFO | "huge" : "", | 20260507 19:46:37.983 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:37.983 | INFO | "malloc" : " 14 GB", | 20260507 19:46:37.983 | INFO | "pinned" : " 29 GB" | 20260507 19:46:37.983 | INFO | } | 20260507 19:46:37.984 | INFO | Perf report for the last 5.13931 seconds: | 20260507 19:46:37.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:37.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:46:37.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:46:37.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:37.984 | INFO | Uploaded 36GiB to GPU in 2.52095s (14.2803GiB/s) | 20260507 19:46:37.984 | INFO | Downloaded 34GiB to GPU in 2.46412s (13.798GiB/s) | 20260507 19:46:43.125 | INFO | Analyzing chunk frames = [2119168,2119680) | 20260507 19:46:43.125 | INFO | MemInfo: { | 20260507 19:46:43.125 | INFO | "gpu" : " 17 GB", | 20260507 19:46:43.126 | INFO | "huge" : "", | 20260507 19:46:43.126 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:43.126 | INFO | "malloc" : " 14 GB", | 20260507 19:46:43.126 | INFO | "pinned" : " 29 GB" | 20260507 19:46:43.126 | INFO | } | 20260507 19:46:43.126 | INFO | Perf report for the last 5.14215 seconds: | 20260507 19:46:43.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:43.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 19:46:43.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:46:43.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:43.126 | INFO | Uploaded 36GiB to GPU in 2.46415s (14.6095GiB/s) | 20260507 19:46:43.126 | INFO | Downloaded 34GiB to GPU in 2.40792s (14.1201GiB/s) | 20260507 19:46:48.265 | INFO | Analyzing chunk frames = [2119680,2120192) | 20260507 19:46:48.265 | INFO | MemInfo: { | 20260507 19:46:48.265 | INFO | "gpu" : " 17 GB", | 20260507 19:46:48.265 | INFO | "huge" : "", | 20260507 19:46:48.265 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:48.265 | INFO | "malloc" : " 14 GB", | 20260507 19:46:48.265 | INFO | "pinned" : " 29 GB" | 20260507 19:46:48.265 | INFO | } | 20260507 19:46:48.265 | INFO | Perf report for the last 5.13948 seconds: | 20260507 19:46:48.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:48.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 19:46:48.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 19:46:48.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:48.265 | INFO | Uploaded 36GiB to GPU in 2.40906s (14.9436GiB/s) | 20260507 19:46:48.266 | INFO | Downloaded 34GiB to GPU in 2.50165s (13.591GiB/s) | 20260507 19:46:53.401 | INFO | Analyzing chunk frames = [2120192,2120704) | 20260507 19:46:53.401 | INFO | MemInfo: { | 20260507 19:46:53.401 | INFO | "gpu" : " 17 GB", | 20260507 19:46:53.401 | INFO | "huge" : "", | 20260507 19:46:53.401 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:53.401 | INFO | "malloc" : " 14 GB", | 20260507 19:46:53.401 | INFO | "pinned" : " 29 GB" | 20260507 19:46:53.401 | INFO | } | 20260507 19:46:53.402 | INFO | Perf report for the last 5.13639 seconds: | 20260507 19:46:53.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:53.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:46:53.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:46:53.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:53.402 | INFO | Uploaded 36GiB to GPU in 2.48136s (14.5082GiB/s) | 20260507 19:46:53.402 | INFO | Downloaded 34GiB to GPU in 2.42631s (14.013GiB/s) | 20260507 19:46:58.541 | INFO | Analyzing chunk frames = [2120704,2121216) | 20260507 19:46:58.541 | INFO | MemInfo: { | 20260507 19:46:58.541 | INFO | "gpu" : " 17 GB", | 20260507 19:46:58.541 | INFO | "huge" : "", | 20260507 19:46:58.541 | INFO | "huge_shared" : " 29 GB", | 20260507 19:46:58.541 | INFO | "malloc" : " 14 GB", | 20260507 19:46:58.541 | INFO | "pinned" : " 29 GB" | 20260507 19:46:58.541 | INFO | } | 20260507 19:46:58.541 | INFO | Perf report for the last 5.13984 seconds: | 20260507 19:46:58.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:46:58.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:46:58.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:46:58.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:46:58.542 | INFO | Uploaded 36GiB to GPU in 2.49856s (14.4083GiB/s) | 20260507 19:46:58.542 | INFO | Downloaded 34GiB to GPU in 2.44909s (13.8827GiB/s) | 20260507 19:47:03.680 | INFO | Analyzing chunk frames = [2121216,2121728) | 20260507 19:47:03.680 | INFO | MemInfo: { | 20260507 19:47:03.680 | INFO | "gpu" : " 17 GB", | 20260507 19:47:03.681 | INFO | "huge" : "", | 20260507 19:47:03.681 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:03.681 | INFO | "malloc" : " 14 GB", | 20260507 19:47:03.681 | INFO | "pinned" : " 29 GB" | 20260507 19:47:03.681 | INFO | } | 20260507 19:47:03.681 | INFO | Perf report for the last 5.13926 seconds: | 20260507 19:47:03.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:03.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:47:03.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 19:47:03.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:03.681 | INFO | Uploaded 36GiB to GPU in 2.48804s (14.4692GiB/s) | 20260507 19:47:03.681 | INFO | Downloaded 34GiB to GPU in 2.44628s (13.8987GiB/s) | 20260507 19:47:03.682 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:47:03.682 | INFO | Total mem usage: 16.4 GiB | 20260507 19:47:03.682 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:47:03.682 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.57%) | 20260507 19:47:03.682 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:47:03.682 | INFO | High Water Breakdown: | 20260507 19:47:03.682 | INFO | 59.9% : 8.97 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:47:03.682 | INFO | 39.9% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:47:03.682 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:47:03.682 | INFO | ------------------------------------------ | 20260507 19:47:03.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:47:03.682 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:47:03.682 | INFO | Total mem usage: 26.5 GiB | 20260507 19:47:03.682 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:47:03.682 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:47:03.682 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 19:47:03.682 | INFO | High Water Breakdown: | 20260507 19:47:03.682 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:47:03.682 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:47:03.682 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:47:03.682 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:47:03.682 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:47:03.682 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:47:03.682 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:47:03.682 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:47:03.682 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:47:03.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:47:03.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:47:03.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:47:03.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:47:03.682 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:47:03.682 | INFO | ------------------------------------------ | 20260507 19:47:03.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:47:03.682 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:47:03.682 | INFO | Total mem usage: 27.8 GiB | 20260507 19:47:03.682 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:47:03.682 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:47:03.682 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:47:03.682 | INFO | High Water Breakdown: | 20260507 19:47:03.682 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:47:03.682 | INFO | ------------------------------------------ | 20260507 19:47:03.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:47:03.682 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:47:03.682 | INFO | Total mem usage: 15.7 GiB | 20260507 19:47:03.682 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:47:03.682 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:47:03.682 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 19:47:03.682 | INFO | High Water Breakdown: | 20260507 19:47:03.682 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:47:03.682 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:47:03.682 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:47:03.682 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:47:03.682 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:47:03.682 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:47:03.682 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:47:03.682 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:47:03.682 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:47:03.682 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:47:03.682 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:47:03.682 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:47:03.682 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:47:03.682 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:47:03.682 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:47:03.682 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:47:03.682 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:47:03.682 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:47:03.682 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:47:03.683 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:47:03.683 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:47:03.683 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:47:03.683 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:47:03.683 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:47:03.683 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:47:03.683 | INFO | ------------------------------------------ | 20260507 19:47:03.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:47:03.683 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:47:03.683 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:47:03.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:47:03.683 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.42e+01 / 1.73e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.04e-01, 1.24e-02 / 7.26e-03 | 20260507 19:47:03.683 | INFO | PerfReport: 12.08%, Baseline -- 4.01e+00 / 4.47e+00, 5.95e-01 / 2.82e-01 | 20260507 19:47:03.683 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.52e+00, 2.11e-01 / 1.04e-01 | 20260507 19:47:03.683 | INFO | PerfReport: 15.68%, DME -- 4.17e+01 / 4.48e+01, 3.88e+00 / 1.41e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.90e+00 / 1.90e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.49e+00 / 2.82e+00, 4.38e-01 / 2.43e-01 | 20260507 19:47:03.683 | INFO | PerfReport: 3.61%, Metrics -- 1.20e+00 / 1.39e+00, 3.15e-01 / 1.88e-01 | 20260507 19:47:03.683 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.75e-01 / 4.06e-01, 6.13e-02 / 3.43e-02 | 20260507 19:47:03.683 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:47:03.683 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:47:03.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:47:03.683 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 24.52%, Upload -- 2.62e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 31.05%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 2.24%, LockDownload -- 2.40e+00 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 19:47:03.683 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260507 19:47:03.793 | INFO | Starting a round of Baz IO | 20260507 19:47:07.276 | INFO | Populating RTMetrics report | 20260507 19:47:07.277 | INFO | RTMetrics report populated | 20260507 19:47:07.277 | INFO | Opening tmp RTMetrics json file | 20260507 19:47:07.297 | INFO | Serializing RTMetrics to json | 20260507 19:47:07.303 | INFO | Serialization done, swapping tmp file now... | 20260507 19:47:07.303 | INFO | RTMetrics done | 20260507 19:47:07.370 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.576 seconds | 20260507 19:47:07.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.08% / 0.01% / 54.91% | 20260507 19:47:07.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2641% / 1.98682e-05% / 4.76837e-05% | 20260507 19:47:07.370 | INFO | Baz IO overall: | 20260507 19:47:07.370 | INFO | BytesWritten : 959996248186 | 20260507 19:47:07.370 | INFO | NumEvents : 465437035013 | 20260507 19:47:08.820 | INFO | Analyzing chunk frames = [2121728,2122240) | 20260507 19:47:08.820 | INFO | MemInfo: { | 20260507 19:47:08.820 | INFO | "gpu" : " 17 GB", | 20260507 19:47:08.820 | INFO | "huge" : "", | 20260507 19:47:08.820 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:08.820 | INFO | "malloc" : " 14 GB", | 20260507 19:47:08.820 | INFO | "pinned" : " 29 GB" | 20260507 19:47:08.820 | INFO | } | 20260507 19:47:08.820 | INFO | Perf report for the last 5.13971 seconds: | 20260507 19:47:08.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:08.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:08.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:08.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:08.821 | INFO | ANALYSIS : 68.9%, 2.9, 104.2ms, 0 | 20260507 19:47:08.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:08.821 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 19:47:08.821 | INFO | PRE_HQ : 67.1%, 1.9, 64.7ms, 0 | 20260507 19:47:08.821 | INFO | BAZWRITER : 69.2%, 3.9, 138.9ms, 0 | 20260507 19:47:08.821 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:47:08.821 | INFO | Uploaded 36GiB to GPU in 2.58368s (13.9336GiB/s) | 20260507 19:47:08.821 | INFO | Downloaded 38GiB to GPU in 2.68456s (14.155GiB/s) | 20260507 19:47:13.960 | INFO | Analyzing chunk frames = [2122240,2122752) | 20260507 19:47:13.961 | INFO | MemInfo: { | 20260507 19:47:13.961 | INFO | "gpu" : " 17 GB", | 20260507 19:47:13.961 | INFO | "huge" : "", | 20260507 19:47:13.961 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:13.961 | INFO | "malloc" : " 14 GB", | 20260507 19:47:13.961 | INFO | "pinned" : " 29 GB" | 20260507 19:47:13.961 | INFO | } | 20260507 19:47:13.961 | INFO | Perf report for the last 5.14022 seconds: | 20260507 19:47:13.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:13.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:47:13.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:47:13.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:13.961 | INFO | Uploaded 36GiB to GPU in 2.55811s (14.0729GiB/s) | 20260507 19:47:13.961 | INFO | Downloaded 34GiB to GPU in 2.49056s (13.6515GiB/s) | 20260507 19:47:19.099 | INFO | Analyzing chunk frames = [2122752,2123264) | 20260507 19:47:19.099 | INFO | MemInfo: { | 20260507 19:47:19.099 | INFO | "gpu" : " 17 GB", | 20260507 19:47:19.099 | INFO | "huge" : "", | 20260507 19:47:19.099 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:19.099 | INFO | "malloc" : " 14 GB", | 20260507 19:47:19.099 | INFO | "pinned" : " 29 GB" | 20260507 19:47:19.099 | INFO | } | 20260507 19:47:19.099 | INFO | Perf report for the last 5.13844 seconds: | 20260507 19:47:19.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:19.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.099 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:47:19.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.099 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:47:19.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:19.100 | INFO | Uploaded 36GiB to GPU in 2.56557s (14.032GiB/s) | 20260507 19:47:19.100 | INFO | Downloaded 34GiB to GPU in 2.46747s (13.7793GiB/s) | 20260507 19:47:24.238 | INFO | Analyzing chunk frames = [2123264,2123776) | 20260507 19:47:24.238 | INFO | MemInfo: { | 20260507 19:47:24.238 | INFO | "gpu" : " 17 GB", | 20260507 19:47:24.238 | INFO | "huge" : "", | 20260507 19:47:24.238 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:24.238 | INFO | "malloc" : " 14 GB", | 20260507 19:47:24.238 | INFO | "pinned" : " 29 GB" | 20260507 19:47:24.238 | INFO | } | 20260507 19:47:24.238 | INFO | Perf report for the last 5.13864 seconds: | 20260507 19:47:24.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:24.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:47:24.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:47:24.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:24.238 | INFO | Uploaded 36GiB to GPU in 2.456s (14.658GiB/s) | 20260507 19:47:24.238 | INFO | Downloaded 34GiB to GPU in 2.4046s (14.1396GiB/s) | 20260507 19:47:29.378 | INFO | Analyzing chunk frames = [2123776,2124288) | 20260507 19:47:29.378 | INFO | MemInfo: { | 20260507 19:47:29.378 | INFO | "gpu" : " 17 GB", | 20260507 19:47:29.378 | INFO | "huge" : "", | 20260507 19:47:29.378 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:29.378 | INFO | "malloc" : " 14 GB", | 20260507 19:47:29.378 | INFO | "pinned" : " 29 GB" | 20260507 19:47:29.378 | INFO | } | 20260507 19:47:29.378 | INFO | Perf report for the last 5.14005 seconds: | 20260507 19:47:29.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:29.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:47:29.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:47:29.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:29.378 | INFO | Uploaded 36GiB to GPU in 2.35824s (15.2656GiB/s) | 20260507 19:47:29.378 | INFO | Downloaded 34GiB to GPU in 2.5038s (13.5794GiB/s) | 20260507 19:47:34.516 | INFO | Analyzing chunk frames = [2124288,2124800) | 20260507 19:47:34.516 | INFO | MemInfo: { | 20260507 19:47:34.516 | INFO | "gpu" : " 17 GB", | 20260507 19:47:34.516 | INFO | "huge" : "", | 20260507 19:47:34.517 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:34.517 | INFO | "malloc" : " 14 GB", | 20260507 19:47:34.517 | INFO | "pinned" : " 29 GB" | 20260507 19:47:34.517 | INFO | } | 20260507 19:47:34.517 | INFO | Perf report for the last 5.13889 seconds: | 20260507 19:47:34.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:34.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 19:47:34.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:47:34.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:34.517 | INFO | Uploaded 36GiB to GPU in 2.49467s (14.4307GiB/s) | 20260507 19:47:34.517 | INFO | Downloaded 34GiB to GPU in 2.42506s (14.0203GiB/s) | 20260507 19:47:39.656 | INFO | Analyzing chunk frames = [2124800,2125312) | 20260507 19:47:39.656 | INFO | MemInfo: { | 20260507 19:47:39.656 | INFO | "gpu" : " 17 GB", | 20260507 19:47:39.656 | INFO | "huge" : "", | 20260507 19:47:39.656 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:39.656 | INFO | "malloc" : " 14 GB", | 20260507 19:47:39.656 | INFO | "pinned" : " 29 GB" | 20260507 19:47:39.656 | INFO | } | 20260507 19:47:39.656 | INFO | Perf report for the last 5.13951 seconds: | 20260507 19:47:39.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:39.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.656 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:47:39.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.657 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:47:39.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:39.657 | INFO | Uploaded 36GiB to GPU in 2.53247s (14.2154GiB/s) | 20260507 19:47:39.657 | INFO | Downloaded 34GiB to GPU in 2.46268s (13.8061GiB/s) | 20260507 19:47:44.795 | INFO | Analyzing chunk frames = [2125312,2125824) | 20260507 19:47:44.795 | INFO | MemInfo: { | 20260507 19:47:44.795 | INFO | "gpu" : " 17 GB", | 20260507 19:47:44.795 | INFO | "huge" : "", | 20260507 19:47:44.795 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:44.795 | INFO | "malloc" : " 14 GB", | 20260507 19:47:44.796 | INFO | "pinned" : " 29 GB" | 20260507 19:47:44.796 | INFO | } | 20260507 19:47:44.796 | INFO | Perf report for the last 5.13935 seconds: | 20260507 19:47:44.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:44.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:47:44.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 19:47:44.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:44.796 | INFO | Uploaded 36GiB to GPU in 2.49831s (14.4097GiB/s) | 20260507 19:47:44.796 | INFO | Downloaded 34GiB to GPU in 2.43892s (13.9406GiB/s) | 20260507 19:47:44.911 | INFO | Starting a round of Baz IO | 20260507 19:47:48.395 | INFO | Populating RTMetrics report | 20260507 19:47:48.396 | INFO | RTMetrics report populated | 20260507 19:47:48.396 | INFO | Opening tmp RTMetrics json file | 20260507 19:47:48.466 | INFO | Serializing RTMetrics to json | 20260507 19:47:48.472 | INFO | Serialization done, swapping tmp file now... | 20260507 19:47:48.479 | INFO | RTMetrics done | 20260507 19:47:48.693 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.781 seconds | 20260507 19:47:48.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.07% / 0.01% / 54.92% | 20260507 19:47:48.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.25396% / 2.38419e-05% / 3.97364e-05% | 20260507 19:47:48.693 | INFO | Baz IO overall: | 20260507 19:47:48.693 | INFO | BytesWritten : 961913509788 | 20260507 19:47:48.693 | INFO | NumEvents : 466366028459 | 20260507 19:47:49.937 | INFO | Analyzing chunk frames = [2125824,2126336) | 20260507 19:47:49.937 | INFO | MemInfo: { | 20260507 19:47:49.937 | INFO | "gpu" : " 17 GB", | 20260507 19:47:49.937 | INFO | "huge" : "", | 20260507 19:47:49.937 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:49.937 | INFO | "malloc" : " 14 GB", | 20260507 19:47:49.937 | INFO | "pinned" : " 29 GB" | 20260507 19:47:49.937 | INFO | } | 20260507 19:47:49.937 | INFO | Perf report for the last 5.14116 seconds: | 20260507 19:47:49.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:49.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:49.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:49.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:49.937 | INFO | ANALYSIS : 69.3%, 2.9, 104.9ms, 0 | 20260507 19:47:49.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:49.937 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260507 19:47:49.937 | INFO | PRE_HQ : 68.5%, 1.8, 64.9ms, 0 | 20260507 19:47:49.937 | INFO | BAZWRITER : 73.2%, 4.3, 160.7ms, 0 | 20260507 19:47:49.937 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 19:47:49.937 | INFO | Uploaded 36GiB to GPU in 2.60511s (13.819GiB/s) | 20260507 19:47:49.937 | INFO | Downloaded 38GiB to GPU in 2.71475s (13.9976GiB/s) | 20260507 19:47:55.075 | INFO | Analyzing chunk frames = [2126336,2126848) | 20260507 19:47:55.075 | INFO | MemInfo: { | 20260507 19:47:55.075 | INFO | "gpu" : " 17 GB", | 20260507 19:47:55.075 | INFO | "huge" : "", | 20260507 19:47:55.075 | INFO | "huge_shared" : " 29 GB", | 20260507 19:47:55.075 | INFO | "malloc" : " 14 GB", | 20260507 19:47:55.075 | INFO | "pinned" : " 29 GB" | 20260507 19:47:55.075 | INFO | } | 20260507 19:47:55.075 | INFO | Perf report for the last 5.13834 seconds: | 20260507 19:47:55.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:47:55.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:47:55.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:47:55.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:47:55.075 | INFO | Uploaded 36GiB to GPU in 2.56571s (14.0312GiB/s) | 20260507 19:47:55.076 | INFO | Downloaded 34GiB to GPU in 2.50211s (13.5886GiB/s) | 20260507 19:48:00.214 | INFO | Analyzing chunk frames = [2126848,2127360) | 20260507 19:48:00.214 | INFO | MemInfo: { | 20260507 19:48:00.214 | INFO | "gpu" : " 17 GB", | 20260507 19:48:00.214 | INFO | "huge" : "", | 20260507 19:48:00.214 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:00.214 | INFO | "malloc" : " 14 GB", | 20260507 19:48:00.214 | INFO | "pinned" : " 29 GB" | 20260507 19:48:00.214 | INFO | } | 20260507 19:48:00.214 | INFO | Perf report for the last 5.13884 seconds: | 20260507 19:48:00.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:00.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:48:00.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:48:00.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:00.214 | INFO | Uploaded 36GiB to GPU in 2.5656s (14.0318GiB/s) | 20260507 19:48:00.214 | INFO | Downloaded 34GiB to GPU in 2.45918s (13.8257GiB/s) | 20260507 19:48:05.352 | INFO | Analyzing chunk frames = [2127360,2127872) | 20260507 19:48:05.352 | INFO | MemInfo: { | 20260507 19:48:05.352 | INFO | "gpu" : " 17 GB", | 20260507 19:48:05.352 | INFO | "huge" : "", | 20260507 19:48:05.352 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:05.352 | INFO | "malloc" : " 14 GB", | 20260507 19:48:05.353 | INFO | "pinned" : " 29 GB" | 20260507 19:48:05.353 | INFO | } | 20260507 19:48:05.353 | INFO | Perf report for the last 5.1386 seconds: | 20260507 19:48:05.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:05.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:48:05.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:48:05.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:05.353 | INFO | Uploaded 36GiB to GPU in 2.48642s (14.4786GiB/s) | 20260507 19:48:05.353 | INFO | Downloaded 34GiB to GPU in 2.41027s (14.1063GiB/s) | 20260507 19:48:10.492 | INFO | Analyzing chunk frames = [2127872,2128384) | 20260507 19:48:10.492 | INFO | MemInfo: { | 20260507 19:48:10.492 | INFO | "gpu" : " 17 GB", | 20260507 19:48:10.492 | INFO | "huge" : "", | 20260507 19:48:10.492 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:10.492 | INFO | "malloc" : " 14 GB", | 20260507 19:48:10.492 | INFO | "pinned" : " 29 GB" | 20260507 19:48:10.492 | INFO | } | 20260507 19:48:10.492 | INFO | Perf report for the last 5.13963 seconds: | 20260507 19:48:10.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:10.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.492 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 19:48:10.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.492 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:48:10.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:10.493 | INFO | Uploaded 36GiB to GPU in 2.36699s (15.2092GiB/s) | 20260507 19:48:10.493 | INFO | Downloaded 34GiB to GPU in 2.50707s (13.5616GiB/s) | 20260507 19:48:15.632 | INFO | Analyzing chunk frames = [2128384,2128896) | 20260507 19:48:15.632 | INFO | MemInfo: { | 20260507 19:48:15.632 | INFO | "gpu" : " 17 GB", | 20260507 19:48:15.632 | INFO | "huge" : "", | 20260507 19:48:15.632 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:15.632 | INFO | "malloc" : " 14 GB", | 20260507 19:48:15.632 | INFO | "pinned" : " 29 GB" | 20260507 19:48:15.632 | INFO | } | 20260507 19:48:15.632 | INFO | Perf report for the last 5.13981 seconds: | 20260507 19:48:15.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:15.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:48:15.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:48:15.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:15.632 | INFO | Uploaded 36GiB to GPU in 2.48806s (14.4691GiB/s) | 20260507 19:48:15.632 | INFO | Downloaded 34GiB to GPU in 2.42558s (14.0173GiB/s) | 20260507 19:48:20.772 | INFO | Analyzing chunk frames = [2128896,2129408) | 20260507 19:48:20.772 | INFO | MemInfo: { | 20260507 19:48:20.772 | INFO | "gpu" : " 17 GB", | 20260507 19:48:20.772 | INFO | "huge" : "", | 20260507 19:48:20.772 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:20.772 | INFO | "malloc" : " 14 GB", | 20260507 19:48:20.772 | INFO | "pinned" : " 29 GB" | 20260507 19:48:20.773 | INFO | } | 20260507 19:48:20.773 | INFO | Perf report for the last 5.14043 seconds: | 20260507 19:48:20.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:20.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:48:20.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:48:20.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:20.773 | INFO | Uploaded 36GiB to GPU in 2.51954s (14.2883GiB/s) | 20260507 19:48:20.773 | INFO | Downloaded 34GiB to GPU in 2.45006s (13.8772GiB/s) | 20260507 19:48:25.910 | INFO | Analyzing chunk frames = [2129408,2129920) | 20260507 19:48:25.910 | INFO | MemInfo: { | 20260507 19:48:25.910 | INFO | "gpu" : " 17 GB", | 20260507 19:48:25.910 | INFO | "huge" : "", | 20260507 19:48:25.910 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:25.910 | INFO | "malloc" : " 14 GB", | 20260507 19:48:25.910 | INFO | "pinned" : " 29 GB" | 20260507 19:48:25.910 | INFO | } | 20260507 19:48:25.911 | INFO | Perf report for the last 5.13792 seconds: | 20260507 19:48:25.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:25.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:48:25.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 19:48:25.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:25.911 | INFO | Uploaded 36GiB to GPU in 2.50229s (14.3868GiB/s) | 20260507 19:48:25.911 | INFO | Downloaded 34GiB to GPU in 2.44176s (13.9244GiB/s) | 20260507 19:48:25.912 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:48:25.912 | INFO | Total mem usage: 16.4 GiB | 20260507 19:48:25.912 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:48:25.912 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.87%) | 20260507 19:48:25.912 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:48:25.912 | INFO | High Water Breakdown: | 20260507 19:48:25.912 | INFO | 59.8% : 8.93 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:48:25.912 | INFO | 40.1% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:48:25.912 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:48:25.912 | INFO | ------------------------------------------ | 20260507 19:48:25.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:48:25.912 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:48:25.912 | INFO | Total mem usage: 26.5 GiB | 20260507 19:48:25.912 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:48:25.912 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:48:25.912 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:48:25.912 | INFO | High Water Breakdown: | 20260507 19:48:25.912 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:48:25.912 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:48:25.912 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:48:25.912 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:48:25.912 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:48:25.912 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:48:25.912 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:48:25.912 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:48:25.912 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:48:25.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:48:25.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:48:25.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:48:25.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:48:25.912 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:48:25.912 | INFO | ------------------------------------------ | 20260507 19:48:25.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:48:25.912 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:48:25.912 | INFO | Total mem usage: 27.8 GiB | 20260507 19:48:25.912 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:48:25.912 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:48:25.912 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:48:25.912 | INFO | High Water Breakdown: | 20260507 19:48:25.912 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:48:25.912 | INFO | ------------------------------------------ | 20260507 19:48:25.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:48:25.912 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:48:25.912 | INFO | Total mem usage: 15.7 GiB | 20260507 19:48:25.912 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:48:25.912 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:48:25.912 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:48:25.912 | INFO | High Water Breakdown: | 20260507 19:48:25.912 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:48:25.912 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:48:25.912 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:48:25.912 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:48:25.912 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:48:25.912 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:48:25.912 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:48:25.912 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:48:25.912 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:48:25.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:48:25.912 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:48:25.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:48:25.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:48:25.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:48:25.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:48:25.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:48:25.912 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:48:25.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:48:25.912 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:48:25.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:48:25.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:48:25.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:48:25.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:48:25.913 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:48:25.913 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:48:25.913 | INFO | ------------------------------------------ | 20260507 19:48:25.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:48:25.913 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:48:25.913 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:48:25.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:48:25.913 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.42e+01 / 1.79e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.02e-01, 1.30e-02 / 5.63e-03 | 20260507 19:48:25.913 | INFO | PerfReport: 12.09%, Baseline -- 4.00e+00 / 4.48e+00, 5.97e-01 / 2.52e-01 | 20260507 19:48:25.913 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.51e+00, 2.11e-01 / 1.15e-01 | 20260507 19:48:25.913 | INFO | PerfReport: 15.77%, DME -- 4.18e+01 / 4.49e+01, 3.88e+00 / 1.27e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.75e+00 / 1.82e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.46e-01 / 2.37e-01 | 20260507 19:48:25.913 | INFO | PerfReport: 3.63%, Metrics -- 1.20e+00 / 1.37e+00, 3.03e-01 / 1.42e-01 | 20260507 19:48:25.913 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.76e-01 / 4.00e-01, 6.13e-02 / 2.93e-02 | 20260507 19:48:25.913 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:48:25.913 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:48:25.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:48:25.913 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 24.50%, Upload -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 31.05%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 2.34%, LockDownload -- 2.50e+00 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 15.83%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 19:48:25.913 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 19:48:26.023 | INFO | Starting a round of Baz IO | 20260507 19:48:29.559 | INFO | Populating RTMetrics report | 20260507 19:48:29.559 | INFO | RTMetrics report populated | 20260507 19:48:29.559 | INFO | Opening tmp RTMetrics json file | 20260507 19:48:29.560 | INFO | Serializing RTMetrics to json | 20260507 19:48:29.563 | INFO | Serialization done, swapping tmp file now... | 20260507 19:48:29.567 | INFO | RTMetrics done | 20260507 19:48:29.693 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.67 seconds | 20260507 19:48:29.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.05% / 0.01% / 54.93% | 20260507 19:48:29.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.25039% / 1.58946e-05% / 3.57628e-05% | 20260507 19:48:29.693 | INFO | Baz IO overall: | 20260507 19:48:29.693 | INFO | BytesWritten : 963830981407 | 20260507 19:48:29.693 | INFO | NumEvents : 467295315420 | 20260507 19:48:31.049 | INFO | Analyzing chunk frames = [2129920,2130432) | 20260507 19:48:31.049 | INFO | MemInfo: { | 20260507 19:48:31.049 | INFO | "gpu" : " 17 GB", | 20260507 19:48:31.049 | INFO | "huge" : "", | 20260507 19:48:31.049 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:31.049 | INFO | "malloc" : " 14 GB", | 20260507 19:48:31.049 | INFO | "pinned" : " 29 GB" | 20260507 19:48:31.049 | INFO | } | 20260507 19:48:31.049 | INFO | Perf report for the last 5.13821 seconds: | 20260507 19:48:31.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:31.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:31.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:31.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:31.049 | INFO | ANALYSIS : 69.7%, 2.9, 104.8ms, 0 | 20260507 19:48:31.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:31.049 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260507 19:48:31.049 | INFO | PRE_HQ : 68.7%, 2.0, 70.6ms, 0 | 20260507 19:48:31.049 | INFO | BAZWRITER : 71.0%, 4.0, 144.6ms, 0 | 20260507 19:48:31.049 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:48:31.049 | INFO | Uploaded 36GiB to GPU in 2.53626s (14.1941GiB/s) | 20260507 19:48:31.050 | INFO | Downloaded 38GiB to GPU in 2.68458s (14.1549GiB/s) | 20260507 19:48:36.189 | INFO | Analyzing chunk frames = [2130432,2130944) | 20260507 19:48:36.189 | INFO | MemInfo: { | 20260507 19:48:36.189 | INFO | "gpu" : " 17 GB", | 20260507 19:48:36.189 | INFO | "huge" : "", | 20260507 19:48:36.189 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:36.189 | INFO | "malloc" : " 14 GB", | 20260507 19:48:36.189 | INFO | "pinned" : " 29 GB" | 20260507 19:48:36.189 | INFO | } | 20260507 19:48:36.189 | INFO | Perf report for the last 5.14029 seconds: | 20260507 19:48:36.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:36.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:48:36.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:48:36.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:36.189 | INFO | Uploaded 36GiB to GPU in 2.54216s (14.1612GiB/s) | 20260507 19:48:36.189 | INFO | Downloaded 34GiB to GPU in 2.48133s (13.7023GiB/s) | 20260507 19:48:41.327 | INFO | Analyzing chunk frames = [2130944,2131456) | 20260507 19:48:41.327 | INFO | MemInfo: { | 20260507 19:48:41.327 | INFO | "gpu" : " 17 GB", | 20260507 19:48:41.327 | INFO | "huge" : "", | 20260507 19:48:41.327 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:41.328 | INFO | "malloc" : " 14 GB", | 20260507 19:48:41.328 | INFO | "pinned" : " 29 GB" | 20260507 19:48:41.328 | INFO | } | 20260507 19:48:41.328 | INFO | Perf report for the last 5.13851 seconds: | 20260507 19:48:41.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:41.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:48:41.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:48:41.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:41.328 | INFO | Uploaded 36GiB to GPU in 2.58267s (13.9391GiB/s) | 20260507 19:48:41.328 | INFO | Downloaded 34GiB to GPU in 2.46467s (13.795GiB/s) | 20260507 19:48:46.467 | INFO | Analyzing chunk frames = [2131456,2131968) | 20260507 19:48:46.467 | INFO | MemInfo: { | 20260507 19:48:46.467 | INFO | "gpu" : " 17 GB", | 20260507 19:48:46.467 | INFO | "huge" : "", | 20260507 19:48:46.467 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:46.467 | INFO | "malloc" : " 14 GB", | 20260507 19:48:46.468 | INFO | "pinned" : " 29 GB" | 20260507 19:48:46.468 | INFO | } | 20260507 19:48:46.468 | INFO | Perf report for the last 5.13999 seconds: | 20260507 19:48:46.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:46.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:48:46.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:48:46.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:46.468 | INFO | Uploaded 36GiB to GPU in 2.47101s (14.569GiB/s) | 20260507 19:48:46.468 | INFO | Downloaded 34GiB to GPU in 2.38503s (14.2556GiB/s) | 20260507 19:48:51.607 | INFO | Analyzing chunk frames = [2131968,2132480) | 20260507 19:48:51.607 | INFO | MemInfo: { | 20260507 19:48:51.607 | INFO | "gpu" : " 17 GB", | 20260507 19:48:51.607 | INFO | "huge" : "", | 20260507 19:48:51.607 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:51.607 | INFO | "malloc" : " 14 GB", | 20260507 19:48:51.607 | INFO | "pinned" : " 29 GB" | 20260507 19:48:51.607 | INFO | } | 20260507 19:48:51.607 | INFO | Perf report for the last 5.13977 seconds: | 20260507 19:48:51.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:51.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.607 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:48:51.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.607 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:48:51.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:51.608 | INFO | Uploaded 36GiB to GPU in 2.4006s (14.9962GiB/s) | 20260507 19:48:51.608 | INFO | Downloaded 34GiB to GPU in 2.49817s (13.61GiB/s) | 20260507 19:48:56.747 | INFO | Analyzing chunk frames = [2132480,2132992) | 20260507 19:48:56.747 | INFO | MemInfo: { | 20260507 19:48:56.747 | INFO | "gpu" : " 17 GB", | 20260507 19:48:56.747 | INFO | "huge" : "", | 20260507 19:48:56.747 | INFO | "huge_shared" : " 29 GB", | 20260507 19:48:56.747 | INFO | "malloc" : " 14 GB", | 20260507 19:48:56.747 | INFO | "pinned" : " 29 GB" | 20260507 19:48:56.747 | INFO | } | 20260507 19:48:56.747 | INFO | Perf report for the last 5.13966 seconds: | 20260507 19:48:56.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:48:56.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:48:56.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:48:56.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:48:56.747 | INFO | Uploaded 36GiB to GPU in 2.50372s (14.3786GiB/s) | 20260507 19:48:56.747 | INFO | Downloaded 34GiB to GPU in 2.41377s (14.0858GiB/s) | 20260507 19:49:01.888 | INFO | Analyzing chunk frames = [2132992,2133504) | 20260507 19:49:01.888 | INFO | MemInfo: { | 20260507 19:49:01.888 | INFO | "gpu" : " 17 GB", | 20260507 19:49:01.888 | INFO | "huge" : "", | 20260507 19:49:01.888 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:01.888 | INFO | "malloc" : " 14 GB", | 20260507 19:49:01.888 | INFO | "pinned" : " 29 GB" | 20260507 19:49:01.888 | INFO | } | 20260507 19:49:01.888 | INFO | Perf report for the last 5.14118 seconds: | 20260507 19:49:01.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:01.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.888 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:49:01.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.888 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:49:01.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:01.889 | INFO | Uploaded 36GiB to GPU in 2.54662s (14.1364GiB/s) | 20260507 19:49:01.889 | INFO | Downloaded 34GiB to GPU in 2.44241s (13.9207GiB/s) | 20260507 19:49:07.026 | INFO | Analyzing chunk frames = [2133504,2134016) | 20260507 19:49:07.026 | INFO | MemInfo: { | 20260507 19:49:07.026 | INFO | "gpu" : " 17 GB", | 20260507 19:49:07.026 | INFO | "huge" : "", | 20260507 19:49:07.026 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:07.026 | INFO | "malloc" : " 14 GB", | 20260507 19:49:07.026 | INFO | "pinned" : " 29 GB" | 20260507 19:49:07.026 | INFO | } | 20260507 19:49:07.026 | INFO | Perf report for the last 5.13772 seconds: | 20260507 19:49:07.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:07.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:49:07.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260507 19:49:07.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:07.026 | INFO | Uploaded 36GiB to GPU in 2.52442s (14.2607GiB/s) | 20260507 19:49:07.026 | INFO | Downloaded 34GiB to GPU in 2.43001s (13.9917GiB/s) | 20260507 19:49:07.125 | INFO | Starting a round of Baz IO | 20260507 19:49:10.600 | INFO | Populating RTMetrics report | 20260507 19:49:10.600 | INFO | RTMetrics report populated | 20260507 19:49:10.600 | INFO | Opening tmp RTMetrics json file | 20260507 19:49:10.601 | INFO | Serializing RTMetrics to json | 20260507 19:49:10.602 | INFO | Serialization done, swapping tmp file now... | 20260507 19:49:10.603 | INFO | RTMetrics done | 20260507 19:49:10.715 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.59 seconds | 20260507 19:49:10.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.04% / 0.01% / 54.95% | 20260507 19:49:10.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.24804% / 1.58946e-05% / 3.97364e-05% | 20260507 19:49:10.716 | INFO | Baz IO overall: | 20260507 19:49:10.716 | INFO | BytesWritten : 965749363499 | 20260507 19:49:10.716 | INFO | NumEvents : 468225271195 | 20260507 19:49:12.164 | INFO | Analyzing chunk frames = [2134016,2134528) | 20260507 19:49:12.164 | INFO | MemInfo: { | 20260507 19:49:12.164 | INFO | "gpu" : " 17 GB", | 20260507 19:49:12.164 | INFO | "huge" : "", | 20260507 19:49:12.164 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:12.164 | INFO | "malloc" : " 14 GB", | 20260507 19:49:12.164 | INFO | "pinned" : " 29 GB" | 20260507 19:49:12.164 | INFO | } | 20260507 19:49:12.164 | INFO | Perf report for the last 5.13783 seconds: | 20260507 19:49:12.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:12.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:12.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:12.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:12.164 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 19:49:12.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:12.164 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 19:49:12.164 | INFO | PRE_HQ : 66.3%, 1.7, 58.6ms, 0 | 20260507 19:49:12.164 | INFO | BAZWRITER : 69.5%, 3.9, 138.7ms, 0 | 20260507 19:49:12.164 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 19:49:12.164 | INFO | Uploaded 36GiB to GPU in 2.55234s (14.1047GiB/s) | 20260507 19:49:12.164 | INFO | Downloaded 38GiB to GPU in 2.69304s (14.1104GiB/s) | 20260507 19:49:17.303 | INFO | Analyzing chunk frames = [2134528,2135040) | 20260507 19:49:17.303 | INFO | MemInfo: { | 20260507 19:49:17.303 | INFO | "gpu" : " 17 GB", | 20260507 19:49:17.303 | INFO | "huge" : "", | 20260507 19:49:17.303 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:17.303 | INFO | "malloc" : " 14 GB", | 20260507 19:49:17.303 | INFO | "pinned" : " 29 GB" | 20260507 19:49:17.303 | INFO | } | 20260507 19:49:17.303 | INFO | Perf report for the last 5.13957 seconds: | 20260507 19:49:17.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:17.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.303 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 19:49:17.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.304 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 19:49:17.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:17.304 | INFO | Uploaded 36GiB to GPU in 2.50211s (14.3879GiB/s) | 20260507 19:49:17.304 | INFO | Downloaded 34GiB to GPU in 2.46467s (13.7949GiB/s) | 20260507 19:49:22.443 | INFO | Analyzing chunk frames = [2135040,2135552) | 20260507 19:49:22.443 | INFO | MemInfo: { | 20260507 19:49:22.443 | INFO | "gpu" : " 17 GB", | 20260507 19:49:22.443 | INFO | "huge" : "", | 20260507 19:49:22.443 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:22.443 | INFO | "malloc" : " 14 GB", | 20260507 19:49:22.443 | INFO | "pinned" : " 29 GB" | 20260507 19:49:22.443 | INFO | } | 20260507 19:49:22.443 | INFO | Perf report for the last 5.14004 seconds: | 20260507 19:49:22.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:22.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.443 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:49:22.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.444 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:49:22.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:22.444 | INFO | Uploaded 36GiB to GPU in 2.59412s (13.8775GiB/s) | 20260507 19:49:22.444 | INFO | Downloaded 34GiB to GPU in 2.46587s (13.7882GiB/s) | 20260507 19:49:27.583 | INFO | Analyzing chunk frames = [2135552,2136064) | 20260507 19:49:27.583 | INFO | MemInfo: { | 20260507 19:49:27.583 | INFO | "gpu" : " 17 GB", | 20260507 19:49:27.583 | INFO | "huge" : "", | 20260507 19:49:27.583 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:27.583 | INFO | "malloc" : " 14 GB", | 20260507 19:49:27.583 | INFO | "pinned" : " 29 GB" | 20260507 19:49:27.583 | INFO | } | 20260507 19:49:27.583 | INFO | Perf report for the last 5.13979 seconds: | 20260507 19:49:27.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:27.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.583 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:49:27.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.583 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:49:27.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:27.584 | INFO | Uploaded 36GiB to GPU in 2.49104s (14.4518GiB/s) | 20260507 19:49:27.584 | INFO | Downloaded 34GiB to GPU in 2.38907s (14.2315GiB/s) | 20260507 19:49:32.722 | INFO | Analyzing chunk frames = [2136064,2136576) | 20260507 19:49:32.722 | INFO | MemInfo: { | 20260507 19:49:32.722 | INFO | "gpu" : " 17 GB", | 20260507 19:49:32.722 | INFO | "huge" : "", | 20260507 19:49:32.723 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:32.723 | INFO | "malloc" : " 14 GB", | 20260507 19:49:32.723 | INFO | "pinned" : " 29 GB" | 20260507 19:49:32.723 | INFO | } | 20260507 19:49:32.723 | INFO | Perf report for the last 5.13946 seconds: | 20260507 19:49:32.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:32.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 19:49:32.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 19:49:32.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:32.723 | INFO | Uploaded 36GiB to GPU in 2.40513s (14.968GiB/s) | 20260507 19:49:32.723 | INFO | Downloaded 34GiB to GPU in 2.49207s (13.6433GiB/s) | 20260507 19:49:37.861 | INFO | Analyzing chunk frames = [2136576,2137088) | 20260507 19:49:37.861 | INFO | MemInfo: { | 20260507 19:49:37.861 | INFO | "gpu" : " 17 GB", | 20260507 19:49:37.861 | INFO | "huge" : "", | 20260507 19:49:37.861 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:37.861 | INFO | "malloc" : " 14 GB", | 20260507 19:49:37.861 | INFO | "pinned" : " 29 GB" | 20260507 19:49:37.861 | INFO | } | 20260507 19:49:37.861 | INFO | Perf report for the last 5.13822 seconds: | 20260507 19:49:37.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:37.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:49:37.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:49:37.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:37.861 | INFO | Uploaded 36GiB to GPU in 2.51402s (14.3197GiB/s) | 20260507 19:49:37.861 | INFO | Downloaded 34GiB to GPU in 2.41481s (14.0798GiB/s) | 20260507 19:49:43.000 | INFO | Analyzing chunk frames = [2137088,2137600) | 20260507 19:49:43.000 | INFO | MemInfo: { | 20260507 19:49:43.000 | INFO | "gpu" : " 17 GB", | 20260507 19:49:43.000 | INFO | "huge" : "", | 20260507 19:49:43.000 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:43.000 | INFO | "malloc" : " 14 GB", | 20260507 19:49:43.000 | INFO | "pinned" : " 29 GB" | 20260507 19:49:43.000 | INFO | } | 20260507 19:49:43.000 | INFO | Perf report for the last 5.13893 seconds: | 20260507 19:49:43.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:43.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:49:43.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 19:49:43.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:43.000 | INFO | Uploaded 36GiB to GPU in 2.54232s (14.1603GiB/s) | 20260507 19:49:43.001 | INFO | Downloaded 34GiB to GPU in 2.44769s (13.8907GiB/s) | 20260507 19:49:48.140 | INFO | Analyzing chunk frames = [2137600,2138112) | 20260507 19:49:48.140 | INFO | MemInfo: { | 20260507 19:49:48.140 | INFO | "gpu" : " 17 GB", | 20260507 19:49:48.140 | INFO | "huge" : "", | 20260507 19:49:48.140 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:48.141 | INFO | "malloc" : " 14 GB", | 20260507 19:49:48.141 | INFO | "pinned" : " 29 GB" | 20260507 19:49:48.141 | INFO | } | 20260507 19:49:48.141 | INFO | Perf report for the last 5.14069 seconds: | 20260507 19:49:48.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:48.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:49:48.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:49:48.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:48.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:49:48.141 | INFO | Total mem usage: 16.4 GiB | 20260507 19:49:48.141 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:49:48.141 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -11.1%) | 20260507 19:49:48.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:49:48.141 | INFO | High Water Breakdown: | 20260507 19:49:48.141 | INFO | 59.7% : 8.82 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:49:48.141 | INFO | 40.2% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:49:48.141 | INFO | Uploaded 36GiB to GPU in 2.50714s (14.359GiB/s) | 20260507 19:49:48.141 | INFO | Downloaded 34GiB to GPU in 2.42487s (14.0214GiB/s) | 20260507 19:49:48.141 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:49:48.141 | INFO | ------------------------------------------ | 20260507 19:49:48.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:49:48.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:49:48.141 | INFO | Total mem usage: 26.5 GiB | 20260507 19:49:48.141 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:49:48.141 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 19:49:48.141 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 19:49:48.141 | INFO | High Water Breakdown: | 20260507 19:49:48.141 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:49:48.141 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:49:48.141 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:49:48.141 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:49:48.141 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:49:48.141 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:49:48.142 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:49:48.142 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:49:48.142 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:49:48.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:49:48.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:49:48.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:49:48.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:49:48.142 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:49:48.142 | INFO | ------------------------------------------ | 20260507 19:49:48.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:49:48.142 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:49:48.142 | INFO | Total mem usage: 27.8 GiB | 20260507 19:49:48.142 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:49:48.142 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:49:48.142 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:49:48.142 | INFO | High Water Breakdown: | 20260507 19:49:48.142 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:49:48.142 | INFO | ------------------------------------------ | 20260507 19:49:48.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:49:48.142 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:49:48.142 | INFO | Total mem usage: 15.7 GiB | 20260507 19:49:48.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:49:48.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:49:48.142 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:49:48.142 | INFO | High Water Breakdown: | 20260507 19:49:48.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:49:48.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:49:48.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:49:48.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:49:48.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:49:48.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:49:48.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:49:48.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:49:48.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:49:48.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:49:48.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:49:48.142 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:49:48.142 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:49:48.142 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:49:48.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:49:48.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:49:48.142 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:49:48.142 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:49:48.142 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:49:48.142 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:49:48.142 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:49:48.142 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:49:48.142 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:49:48.142 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:49:48.142 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:49:48.142 | INFO | ------------------------------------------ | 20260507 19:49:48.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:49:48.142 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:49:48.142 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:49:48.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:49:48.142 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.40e+01 / 1.83e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.33e-02 / 5.50e-03 | 20260507 19:49:48.142 | INFO | PerfReport: 12.06%, Baseline -- 4.00e+00 / 4.46e+00, 6.05e-01 / 3.08e-01 | 20260507 19:49:48.142 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.10e-01 / 1.15e-01 | 20260507 19:49:48.142 | INFO | PerfReport: 15.72%, DME -- 4.17e+01 / 4.48e+01, 3.71e+00 / 1.19e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.84e+00 / 1.99e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.81e+00, 4.43e-01 / 2.63e-01 | 20260507 19:49:48.142 | INFO | PerfReport: 3.63%, Metrics -- 1.20e+00 / 1.41e+00, 3.07e-01 / 1.80e-01 | 20260507 19:49:48.142 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.75e-01 / 4.09e-01, 6.23e-02 / 3.68e-02 | 20260507 19:49:48.142 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:49:48.142 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:49:48.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:49:48.142 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 24.49%, Upload -- 2.62e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 24.11%, Download -- 2.58e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 2.13%, LockUpload -- 2.28e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 2.21%, LockDownload -- 2.37e+00 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 16.03%, LockCompute -- 1.71e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 19:49:48.142 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260507 19:49:48.238 | INFO | Starting a round of Baz IO | 20260507 19:49:51.719 | INFO | Populating RTMetrics report | 20260507 19:49:51.719 | INFO | RTMetrics report populated | 20260507 19:49:51.719 | INFO | Opening tmp RTMetrics json file | 20260507 19:49:51.721 | INFO | Serializing RTMetrics to json | 20260507 19:49:51.722 | INFO | Serialization done, swapping tmp file now... | 20260507 19:49:51.723 | INFO | RTMetrics done | 20260507 19:49:51.887 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.649 seconds | 20260507 19:49:51.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.03% / 0.01% / 54.96% | 20260507 19:49:51.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.24018% / 2.38419e-05% / 4.76837e-05% | 20260507 19:49:51.887 | INFO | Baz IO overall: | 20260507 19:49:51.887 | INFO | BytesWritten : 967666224729 | 20260507 19:49:51.887 | INFO | NumEvents : 469154076409 | 20260507 19:49:53.279 | INFO | Analyzing chunk frames = [2138112,2138624) | 20260507 19:49:53.279 | INFO | MemInfo: { | 20260507 19:49:53.279 | INFO | "gpu" : " 17 GB", | 20260507 19:49:53.279 | INFO | "huge" : "", | 20260507 19:49:53.279 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:53.279 | INFO | "malloc" : " 14 GB", | 20260507 19:49:53.279 | INFO | "pinned" : " 29 GB" | 20260507 19:49:53.279 | INFO | } | 20260507 19:49:53.279 | INFO | Perf report for the last 5.13877 seconds: | 20260507 19:49:53.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:53.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:53.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:53.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:53.280 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 19:49:53.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:53.280 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 19:49:53.280 | INFO | PRE_HQ : 66.2%, 1.7, 58.2ms, 0 | 20260507 19:49:53.280 | INFO | BAZWRITER : 70.6%, 4.2, 150.9ms, 0 | 20260507 19:49:53.280 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 19:49:53.280 | INFO | Uploaded 36GiB to GPU in 2.55915s (14.0672GiB/s) | 20260507 19:49:53.280 | INFO | Downloaded 38GiB to GPU in 2.69218s (14.1149GiB/s) | 20260507 19:49:58.418 | INFO | Analyzing chunk frames = [2138624,2139136) | 20260507 19:49:58.418 | INFO | MemInfo: { | 20260507 19:49:58.418 | INFO | "gpu" : " 17 GB", | 20260507 19:49:58.418 | INFO | "huge" : "", | 20260507 19:49:58.418 | INFO | "huge_shared" : " 29 GB", | 20260507 19:49:58.418 | INFO | "malloc" : " 14 GB", | 20260507 19:49:58.418 | INFO | "pinned" : " 29 GB" | 20260507 19:49:58.418 | INFO | } | 20260507 19:49:58.418 | INFO | Perf report for the last 5.1389 seconds: | 20260507 19:49:58.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:49:58.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.418 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 19:49:58.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.418 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:49:58.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:49:58.419 | INFO | Uploaded 36GiB to GPU in 2.5801s (13.9529GiB/s) | 20260507 19:49:58.419 | INFO | Downloaded 34GiB to GPU in 2.4671s (13.7814GiB/s) | 20260507 19:50:03.557 | INFO | Analyzing chunk frames = [2139136,2139648) | 20260507 19:50:03.557 | INFO | MemInfo: { | 20260507 19:50:03.557 | INFO | "gpu" : " 17 GB", | 20260507 19:50:03.557 | INFO | "huge" : "", | 20260507 19:50:03.557 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:03.557 | INFO | "malloc" : " 14 GB", | 20260507 19:50:03.557 | INFO | "pinned" : " 29 GB" | 20260507 19:50:03.557 | INFO | } | 20260507 19:50:03.557 | INFO | Perf report for the last 5.13878 seconds: | 20260507 19:50:03.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:03.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.557 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:50:03.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.557 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:50:03.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:03.558 | INFO | Uploaded 36GiB to GPU in 2.57224s (13.9956GiB/s) | 20260507 19:50:03.558 | INFO | Downloaded 34GiB to GPU in 2.45956s (13.8236GiB/s) | 20260507 19:50:08.698 | INFO | Analyzing chunk frames = [2139648,2140160) | 20260507 19:50:08.698 | INFO | MemInfo: { | 20260507 19:50:08.698 | INFO | "gpu" : " 17 GB", | 20260507 19:50:08.698 | INFO | "huge" : "", | 20260507 19:50:08.698 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:08.698 | INFO | "malloc" : " 14 GB", | 20260507 19:50:08.698 | INFO | "pinned" : " 29 GB" | 20260507 19:50:08.698 | INFO | } | 20260507 19:50:08.698 | INFO | Perf report for the last 5.1408 seconds: | 20260507 19:50:08.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:08.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 19:50:08.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:50:08.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:08.698 | INFO | Uploaded 36GiB to GPU in 2.49209s (14.4457GiB/s) | 20260507 19:50:08.698 | INFO | Downloaded 34GiB to GPU in 2.39983s (14.1677GiB/s) | 20260507 19:50:13.838 | INFO | Analyzing chunk frames = [2140160,2140672) | 20260507 19:50:13.838 | INFO | MemInfo: { | 20260507 19:50:13.838 | INFO | "gpu" : " 17 GB", | 20260507 19:50:13.838 | INFO | "huge" : "", | 20260507 19:50:13.838 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:13.838 | INFO | "malloc" : " 14 GB", | 20260507 19:50:13.838 | INFO | "pinned" : " 29 GB" | 20260507 19:50:13.838 | INFO | } | 20260507 19:50:13.838 | INFO | Perf report for the last 5.13998 seconds: | 20260507 19:50:13.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:13.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:50:13.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 19:50:13.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:13.838 | INFO | Uploaded 36GiB to GPU in 2.37513s (15.1571GiB/s) | 20260507 19:50:13.838 | INFO | Downloaded 34GiB to GPU in 2.49178s (13.6449GiB/s) | 20260507 19:50:18.975 | INFO | Analyzing chunk frames = [2140672,2141184) | 20260507 19:50:18.975 | INFO | MemInfo: { | 20260507 19:50:18.975 | INFO | "gpu" : " 17 GB", | 20260507 19:50:18.975 | INFO | "huge" : "", | 20260507 19:50:18.975 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:18.976 | INFO | "malloc" : " 14 GB", | 20260507 19:50:18.976 | INFO | "pinned" : " 29 GB" | 20260507 19:50:18.976 | INFO | } | 20260507 19:50:18.976 | INFO | Perf report for the last 5.13766 seconds: | 20260507 19:50:18.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:18.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 19:50:18.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:50:18.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:18.976 | INFO | Uploaded 36GiB to GPU in 2.51147s (14.3342GiB/s) | 20260507 19:50:18.976 | INFO | Downloaded 34GiB to GPU in 2.41723s (14.0657GiB/s) | 20260507 19:50:24.114 | INFO | Analyzing chunk frames = [2141184,2141696) | 20260507 19:50:24.114 | INFO | MemInfo: { | 20260507 19:50:24.114 | INFO | "gpu" : " 17 GB", | 20260507 19:50:24.114 | INFO | "huge" : "", | 20260507 19:50:24.114 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:24.114 | INFO | "malloc" : " 14 GB", | 20260507 19:50:24.114 | INFO | "pinned" : " 29 GB" | 20260507 19:50:24.114 | INFO | } | 20260507 19:50:24.114 | INFO | Perf report for the last 5.1388 seconds: | 20260507 19:50:24.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:24.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 19:50:24.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:50:24.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:24.115 | INFO | Uploaded 36GiB to GPU in 2.54033s (14.1714GiB/s) | 20260507 19:50:24.115 | INFO | Downloaded 34GiB to GPU in 2.44327s (13.9158GiB/s) | 20260507 19:50:29.254 | INFO | Analyzing chunk frames = [2141696,2142208) | 20260507 19:50:29.254 | INFO | MemInfo: { | 20260507 19:50:29.254 | INFO | "gpu" : " 17 GB", | 20260507 19:50:29.254 | INFO | "huge" : "", | 20260507 19:50:29.254 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:29.254 | INFO | "malloc" : " 14 GB", | 20260507 19:50:29.254 | INFO | "pinned" : " 29 GB" | 20260507 19:50:29.254 | INFO | } | 20260507 19:50:29.254 | INFO | Perf report for the last 5.13966 seconds: | 20260507 19:50:29.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:29.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.254 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:50:29.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.254 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:50:29.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:29.255 | INFO | Uploaded 36GiB to GPU in 2.52033s (14.2838GiB/s) | 20260507 19:50:29.255 | INFO | Downloaded 34GiB to GPU in 2.4213s (14.0421GiB/s) | 20260507 19:50:29.352 | INFO | Starting a round of Baz IO | 20260507 19:50:32.825 | INFO | Populating RTMetrics report | 20260507 19:50:32.826 | INFO | RTMetrics report populated | 20260507 19:50:32.826 | INFO | Opening tmp RTMetrics json file | 20260507 19:50:32.849 | INFO | Serializing RTMetrics to json | 20260507 19:50:32.851 | INFO | Serialization done, swapping tmp file now... | 20260507 19:50:32.851 | INFO | RTMetrics done | 20260507 19:50:32.935 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.583 seconds | 20260507 19:50:32.935 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.02% / 0.01% / 54.97% | 20260507 19:50:32.935 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.23643% / 1.98682e-05% / 1.58946e-05% | 20260507 19:50:32.935 | INFO | Baz IO overall: | 20260507 19:50:32.935 | INFO | BytesWritten : 969583375532 | 20260507 19:50:32.935 | INFO | NumEvents : 470083004130 | 20260507 19:50:34.394 | INFO | Analyzing chunk frames = [2142208,2142720) | 20260507 19:50:34.394 | INFO | MemInfo: { | 20260507 19:50:34.394 | INFO | "gpu" : " 17 GB", | 20260507 19:50:34.394 | INFO | "huge" : "", | 20260507 19:50:34.394 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:34.394 | INFO | "malloc" : " 14 GB", | 20260507 19:50:34.394 | INFO | "pinned" : " 29 GB" | 20260507 19:50:34.394 | INFO | } | 20260507 19:50:34.394 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:50:34.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:34.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:34.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:34.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:34.394 | INFO | ANALYSIS : 69.0%, 2.9, 106.8ms, 0 | 20260507 19:50:34.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:34.394 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 19:50:34.394 | INFO | PRE_HQ : 66.5%, 1.7, 58.4ms, 0 | 20260507 19:50:34.394 | INFO | BAZWRITER : 69.3%, 4.0, 143.8ms, 0 | 20260507 19:50:34.394 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 19:50:34.394 | INFO | Uploaded 36GiB to GPU in 2.57449s (13.9834GiB/s) | 20260507 19:50:34.394 | INFO | Downloaded 38GiB to GPU in 2.70184s (14.0645GiB/s) | 20260507 19:50:39.533 | INFO | Analyzing chunk frames = [2142720,2143232) | 20260507 19:50:39.533 | INFO | MemInfo: { | 20260507 19:50:39.533 | INFO | "gpu" : " 17 GB", | 20260507 19:50:39.533 | INFO | "huge" : "", | 20260507 19:50:39.533 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:39.533 | INFO | "malloc" : " 14 GB", | 20260507 19:50:39.533 | INFO | "pinned" : " 29 GB" | 20260507 19:50:39.533 | INFO | } | 20260507 19:50:39.533 | INFO | Perf report for the last 5.13885 seconds: | 20260507 19:50:39.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:39.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 19:50:39.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:50:39.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:39.533 | INFO | Uploaded 36GiB to GPU in 2.58896s (13.9052GiB/s) | 20260507 19:50:39.533 | INFO | Downloaded 34GiB to GPU in 2.46831s (13.7746GiB/s) | 20260507 19:50:44.672 | INFO | Analyzing chunk frames = [2143232,2143744) | 20260507 19:50:44.672 | INFO | MemInfo: { | 20260507 19:50:44.672 | INFO | "gpu" : " 17 GB", | 20260507 19:50:44.672 | INFO | "huge" : "", | 20260507 19:50:44.672 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:44.672 | INFO | "malloc" : " 14 GB", | 20260507 19:50:44.672 | INFO | "pinned" : " 29 GB" | 20260507 19:50:44.672 | INFO | } | 20260507 19:50:44.672 | INFO | Perf report for the last 5.13967 seconds: | 20260507 19:50:44.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:44.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.672 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:50:44.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.672 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:50:44.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:44.673 | INFO | Uploaded 36GiB to GPU in 2.58005s (13.9532GiB/s) | 20260507 19:50:44.673 | INFO | Downloaded 34GiB to GPU in 2.46132s (13.8137GiB/s) | 20260507 19:50:49.813 | INFO | Analyzing chunk frames = [2143744,2144256) | 20260507 19:50:49.813 | INFO | MemInfo: { | 20260507 19:50:49.813 | INFO | "gpu" : " 17 GB", | 20260507 19:50:49.813 | INFO | "huge" : "", | 20260507 19:50:49.813 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:49.813 | INFO | "malloc" : " 14 GB", | 20260507 19:50:49.813 | INFO | "pinned" : " 29 GB" | 20260507 19:50:49.813 | INFO | } | 20260507 19:50:49.813 | INFO | Perf report for the last 5.14062 seconds: | 20260507 19:50:49.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:49.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:50:49.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 19:50:49.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:49.813 | INFO | Uploaded 36GiB to GPU in 2.57945s (13.9565GiB/s) | 20260507 19:50:49.813 | INFO | Downloaded 34GiB to GPU in 2.4578s (13.8335GiB/s) | 20260507 19:50:54.952 | INFO | Analyzing chunk frames = [2144256,2144768) | 20260507 19:50:54.952 | INFO | MemInfo: { | 20260507 19:50:54.952 | INFO | "gpu" : " 17 GB", | 20260507 19:50:54.952 | INFO | "huge" : "", | 20260507 19:50:54.952 | INFO | "huge_shared" : " 29 GB", | 20260507 19:50:54.952 | INFO | "malloc" : " 14 GB", | 20260507 19:50:54.952 | INFO | "pinned" : " 29 GB" | 20260507 19:50:54.952 | INFO | } | 20260507 19:50:54.952 | INFO | Perf report for the last 5.13912 seconds: | 20260507 19:50:54.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:50:54.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.952 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 19:50:54.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.952 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:50:54.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:50:54.953 | INFO | Uploaded 36GiB to GPU in 2.39585s (15.026GiB/s) | 20260507 19:50:54.953 | INFO | Downloaded 34GiB to GPU in 2.49938s (13.6034GiB/s) | 20260507 19:51:00.090 | INFO | Analyzing chunk frames = [2144768,2145280) | 20260507 19:51:00.090 | INFO | MemInfo: { | 20260507 19:51:00.090 | INFO | "gpu" : " 17 GB", | 20260507 19:51:00.090 | INFO | "huge" : "", | 20260507 19:51:00.090 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:00.090 | INFO | "malloc" : " 14 GB", | 20260507 19:51:00.090 | INFO | "pinned" : " 29 GB" | 20260507 19:51:00.090 | INFO | } | 20260507 19:51:00.090 | INFO | Perf report for the last 5.13803 seconds: | 20260507 19:51:00.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:00.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.090 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:51:00.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.090 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:51:00.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:00.091 | INFO | Uploaded 36GiB to GPU in 2.52726s (14.2447GiB/s) | 20260507 19:51:00.091 | INFO | Downloaded 34GiB to GPU in 2.4139s (14.0851GiB/s) | 20260507 19:51:05.230 | INFO | Analyzing chunk frames = [2145280,2145792) | 20260507 19:51:05.230 | INFO | MemInfo: { | 20260507 19:51:05.230 | INFO | "gpu" : " 17 GB", | 20260507 19:51:05.230 | INFO | "huge" : "", | 20260507 19:51:05.230 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:05.230 | INFO | "malloc" : " 14 GB", | 20260507 19:51:05.230 | INFO | "pinned" : " 29 GB" | 20260507 19:51:05.230 | INFO | } | 20260507 19:51:05.230 | INFO | Perf report for the last 5.13962 seconds: | 20260507 19:51:05.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:05.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:51:05.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:51:05.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:05.230 | INFO | Uploaded 36GiB to GPU in 2.54838s (14.1266GiB/s) | 20260507 19:51:05.230 | INFO | Downloaded 34GiB to GPU in 2.44876s (13.8846GiB/s) | 20260507 19:51:10.368 | INFO | Analyzing chunk frames = [2145792,2146304) | 20260507 19:51:10.368 | INFO | MemInfo: { | 20260507 19:51:10.368 | INFO | "gpu" : " 17 GB", | 20260507 19:51:10.368 | INFO | "huge" : "", | 20260507 19:51:10.368 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:10.368 | INFO | "malloc" : " 14 GB", | 20260507 19:51:10.368 | INFO | "pinned" : " 29 GB" | 20260507 19:51:10.368 | INFO | } | 20260507 19:51:10.368 | INFO | Perf report for the last 5.13804 seconds: | 20260507 19:51:10.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:10.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:51:10.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:51:10.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:10.368 | INFO | Uploaded 36GiB to GPU in 2.50729s (14.3581GiB/s) | 20260507 19:51:10.368 | INFO | Downloaded 34GiB to GPU in 2.42674s (14.0105GiB/s) | 20260507 19:51:10.368 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:51:10.368 | INFO | Total mem usage: 16.4 GiB | 20260507 19:51:10.368 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:51:10.368 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.49%) | 20260507 19:51:10.368 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:51:10.368 | INFO | High Water Breakdown: | 20260507 19:51:10.368 | INFO | 60% : 8.98 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:51:10.368 | INFO | 39.9% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:51:10.368 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:51:10.368 | INFO | ------------------------------------------ | 20260507 19:51:10.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:51:10.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:51:10.368 | INFO | Total mem usage: 26.5 GiB | 20260507 19:51:10.368 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:51:10.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:51:10.368 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:51:10.368 | INFO | High Water Breakdown: | 20260507 19:51:10.368 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:51:10.368 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:51:10.368 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:51:10.368 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:51:10.368 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:51:10.368 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:51:10.368 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:51:10.368 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:51:10.368 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:51:10.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:51:10.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:51:10.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:51:10.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:51:10.368 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:51:10.368 | INFO | ------------------------------------------ | 20260507 19:51:10.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:51:10.368 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:51:10.368 | INFO | Total mem usage: 27.8 GiB | 20260507 19:51:10.368 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:51:10.368 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:51:10.368 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:51:10.368 | INFO | High Water Breakdown: | 20260507 19:51:10.368 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:51:10.368 | INFO | ------------------------------------------ | 20260507 19:51:10.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:51:10.368 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:51:10.368 | INFO | Total mem usage: 15.7 GiB | 20260507 19:51:10.368 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:51:10.368 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:51:10.368 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:51:10.368 | INFO | High Water Breakdown: | 20260507 19:51:10.368 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:51:10.368 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:51:10.368 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:51:10.368 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:51:10.368 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:51:10.368 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:51:10.368 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:51:10.368 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:51:10.368 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:51:10.368 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:51:10.368 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:51:10.368 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:51:10.368 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:51:10.368 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:51:10.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:51:10.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:51:10.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:51:10.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:51:10.369 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:51:10.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:51:10.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:51:10.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:51:10.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:51:10.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:51:10.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:51:10.369 | INFO | ------------------------------------------ | 20260507 19:51:10.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:51:10.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:51:10.369 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:51:10.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:51:10.369 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.59e+01, 1.42e+01 / 1.98e+00 | 20260507 19:51:10.369 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.12e-02 / 6.48e-03 | 20260507 19:51:10.369 | INFO | PerfReport: 12.06%, Baseline -- 4.00e+00 / 4.48e+00, 5.98e-01 / 2.90e-01 | 20260507 19:51:10.369 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.24e-01 | 20260507 19:51:10.369 | INFO | PerfReport: 15.76%, DME -- 4.18e+01 / 4.49e+01, 3.84e+00 / 1.35e+00 | 20260507 19:51:10.369 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.81e+00 / 1.85e+00 | 20260507 19:51:10.369 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.48e+00 / 2.82e+00, 4.41e-01 / 2.42e-01 | 20260507 19:51:10.369 | INFO | PerfReport: 3.61%, Metrics -- 1.20e+00 / 1.38e+00, 3.09e-01 / 1.79e-01 | 20260507 19:51:10.369 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.75e-01 / 4.09e-01, 6.26e-02 / 3.75e-02 | 20260507 19:51:10.369 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:51:10.369 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:51:10.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:51:10.370 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 24.56%, Upload -- 2.63e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 24.08%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 30.89%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 2.33%, LockUpload -- 2.50e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 2.14%, LockDownload -- 2.30e+00 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 15.99%, LockCompute -- 1.71e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:51:10.370 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260507 19:51:10.465 | INFO | Starting a round of Baz IO | 20260507 19:51:13.949 | INFO | Populating RTMetrics report | 20260507 19:51:13.949 | INFO | RTMetrics report populated | 20260507 19:51:13.949 | INFO | Opening tmp RTMetrics json file | 20260507 19:51:13.950 | INFO | Serializing RTMetrics to json | 20260507 19:51:13.952 | INFO | Serialization done, swapping tmp file now... | 20260507 19:51:13.952 | INFO | RTMetrics done | 20260507 19:51:14.078 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.613 seconds | 20260507 19:51:14.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.01% / 0.01% / 54.98% | 20260507 19:51:14.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2342% / 0% / 5.16574e-05% | 20260507 19:51:14.078 | INFO | Baz IO overall: | 20260507 19:51:14.078 | INFO | BytesWritten : 971500605351 | 20260507 19:51:14.078 | INFO | NumEvents : 471012000539 | 20260507 19:51:15.508 | INFO | Analyzing chunk frames = [2146304,2146816) | 20260507 19:51:15.508 | INFO | MemInfo: { | 20260507 19:51:15.508 | INFO | "gpu" : " 17 GB", | 20260507 19:51:15.508 | INFO | "huge" : "", | 20260507 19:51:15.508 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:15.508 | INFO | "malloc" : " 14 GB", | 20260507 19:51:15.508 | INFO | "pinned" : " 29 GB" | 20260507 19:51:15.508 | INFO | } | 20260507 19:51:15.508 | INFO | Perf report for the last 5.1403 seconds: | 20260507 19:51:15.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:15.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:15.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:15.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:15.508 | INFO | ANALYSIS : 68.8%, 3.0, 107.2ms, 0 | 20260507 19:51:15.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:15.508 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 19:51:15.508 | INFO | PRE_HQ : 66.2%, 1.7, 58.6ms, 0 | 20260507 19:51:15.508 | INFO | BAZWRITER : 69.8%, 3.8, 136.9ms, 0 | 20260507 19:51:15.508 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 19:51:15.508 | INFO | Uploaded 36GiB to GPU in 2.56077s (14.0582GiB/s) | 20260507 19:51:15.508 | INFO | Downloaded 38GiB to GPU in 2.68973s (14.1278GiB/s) | 20260507 19:51:20.648 | INFO | Analyzing chunk frames = [2146816,2147328) | 20260507 19:51:20.648 | INFO | MemInfo: { | 20260507 19:51:20.648 | INFO | "gpu" : " 17 GB", | 20260507 19:51:20.648 | INFO | "huge" : "", | 20260507 19:51:20.648 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:20.648 | INFO | "malloc" : " 14 GB", | 20260507 19:51:20.648 | INFO | "pinned" : " 29 GB" | 20260507 19:51:20.648 | INFO | } | 20260507 19:51:20.648 | INFO | Perf report for the last 5.13958 seconds: | 20260507 19:51:20.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:20.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 19:51:20.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:51:20.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:20.648 | INFO | Uploaded 36GiB to GPU in 2.54079s (14.1688GiB/s) | 20260507 19:51:20.648 | INFO | Downloaded 34GiB to GPU in 2.46504s (13.7929GiB/s) | 20260507 19:51:25.787 | INFO | Analyzing chunk frames = [2147328,2147840) | 20260507 19:51:25.787 | INFO | MemInfo: { | 20260507 19:51:25.787 | INFO | "gpu" : " 17 GB", | 20260507 19:51:25.787 | INFO | "huge" : "", | 20260507 19:51:25.787 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:25.787 | INFO | "malloc" : " 14 GB", | 20260507 19:51:25.787 | INFO | "pinned" : " 29 GB" | 20260507 19:51:25.787 | INFO | } | 20260507 19:51:25.787 | INFO | Perf report for the last 5.1396 seconds: | 20260507 19:51:25.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:25.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.787 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 19:51:25.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.787 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:51:25.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:25.788 | INFO | Uploaded 36GiB to GPU in 2.58685s (13.9166GiB/s) | 20260507 19:51:25.788 | INFO | Downloaded 34GiB to GPU in 2.46119s (13.8145GiB/s) | 20260507 19:51:30.927 | INFO | Analyzing chunk frames = [2147840,2148352) | 20260507 19:51:30.927 | INFO | MemInfo: { | 20260507 19:51:30.927 | INFO | "gpu" : " 17 GB", | 20260507 19:51:30.927 | INFO | "huge" : "", | 20260507 19:51:30.927 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:30.927 | INFO | "malloc" : " 14 GB", | 20260507 19:51:30.927 | INFO | "pinned" : " 29 GB" | 20260507 19:51:30.927 | INFO | } | 20260507 19:51:30.927 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:51:30.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:30.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:51:30.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:51:30.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:30.927 | INFO | Uploaded 36GiB to GPU in 2.48005s (14.5158GiB/s) | 20260507 19:51:30.927 | INFO | Downloaded 34GiB to GPU in 2.38579s (14.2511GiB/s) | 20260507 19:51:36.066 | INFO | Analyzing chunk frames = [2148352,2148864) | 20260507 19:51:36.066 | INFO | MemInfo: { | 20260507 19:51:36.066 | INFO | "gpu" : " 17 GB", | 20260507 19:51:36.066 | INFO | "huge" : "", | 20260507 19:51:36.066 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:36.066 | INFO | "malloc" : " 14 GB", | 20260507 19:51:36.066 | INFO | "pinned" : " 29 GB" | 20260507 19:51:36.066 | INFO | } | 20260507 19:51:36.066 | INFO | Perf report for the last 5.13948 seconds: | 20260507 19:51:36.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:36.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.066 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 19:51:36.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.066 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:51:36.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:36.067 | INFO | Uploaded 36GiB to GPU in 2.47603s (14.5394GiB/s) | 20260507 19:51:36.067 | INFO | Downloaded 34GiB to GPU in 2.50205s (13.5889GiB/s) | 20260507 19:51:41.204 | INFO | Analyzing chunk frames = [2148864,2149376) | 20260507 19:51:41.204 | INFO | MemInfo: { | 20260507 19:51:41.204 | INFO | "gpu" : " 17 GB", | 20260507 19:51:41.204 | INFO | "huge" : "", | 20260507 19:51:41.204 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:41.204 | INFO | "malloc" : " 14 GB", | 20260507 19:51:41.204 | INFO | "pinned" : " 29 GB" | 20260507 19:51:41.205 | INFO | } | 20260507 19:51:41.205 | INFO | Perf report for the last 5.13843 seconds: | 20260507 19:51:41.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:41.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 19:51:41.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:51:41.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:41.205 | INFO | Uploaded 36GiB to GPU in 2.5161s (14.3078GiB/s) | 20260507 19:51:41.205 | INFO | Downloaded 34GiB to GPU in 2.41398s (14.0846GiB/s) | 20260507 19:51:46.344 | INFO | Analyzing chunk frames = [2149376,2149888) | 20260507 19:51:46.344 | INFO | MemInfo: { | 20260507 19:51:46.344 | INFO | "gpu" : " 17 GB", | 20260507 19:51:46.344 | INFO | "huge" : "", | 20260507 19:51:46.344 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:46.344 | INFO | "malloc" : " 14 GB", | 20260507 19:51:46.344 | INFO | "pinned" : " 29 GB" | 20260507 19:51:46.344 | INFO | } | 20260507 19:51:46.344 | INFO | Perf report for the last 5.1394 seconds: | 20260507 19:51:46.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:46.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260507 19:51:46.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:51:46.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:46.344 | INFO | Uploaded 36GiB to GPU in 2.53632s (14.1938GiB/s) | 20260507 19:51:46.344 | INFO | Downloaded 34GiB to GPU in 2.44113s (13.928GiB/s) | 20260507 19:51:51.485 | INFO | Analyzing chunk frames = [2149888,2150400) | 20260507 19:51:51.485 | INFO | MemInfo: { | 20260507 19:51:51.485 | INFO | "gpu" : " 17 GB", | 20260507 19:51:51.485 | INFO | "huge" : "", | 20260507 19:51:51.485 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:51.485 | INFO | "malloc" : " 14 GB", | 20260507 19:51:51.485 | INFO | "pinned" : " 29 GB" | 20260507 19:51:51.485 | INFO | } | 20260507 19:51:51.485 | INFO | Perf report for the last 5.14063 seconds: | 20260507 19:51:51.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:51.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:51:51.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:51:51.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:51.485 | INFO | Uploaded 36GiB to GPU in 2.5113s (14.3352GiB/s) | 20260507 19:51:51.485 | INFO | Downloaded 34GiB to GPU in 2.42535s (14.0186GiB/s) | 20260507 19:51:51.584 | INFO | Starting a round of Baz IO | 20260507 19:51:55.086 | INFO | Populating RTMetrics report | 20260507 19:51:55.086 | INFO | RTMetrics report populated | 20260507 19:51:55.086 | INFO | Opening tmp RTMetrics json file | 20260507 19:51:55.087 | INFO | Serializing RTMetrics to json | 20260507 19:51:55.092 | INFO | Serialization done, swapping tmp file now... | 20260507 19:51:55.095 | INFO | RTMetrics done | 20260507 19:51:55.182 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.598 seconds | 20260507 19:51:55.183 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.00% / 0.01% / 54.99% | 20260507 19:51:55.183 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.22701% / 3.97364e-06% / 5.16574e-05% | 20260507 19:51:55.183 | INFO | Baz IO overall: | 20260507 19:51:55.183 | INFO | BytesWritten : 973417671881 | 20260507 19:51:55.183 | INFO | NumEvents : 471940840655 | 20260507 19:51:56.623 | INFO | Analyzing chunk frames = [2150400,2150912) | 20260507 19:51:56.623 | INFO | MemInfo: { | 20260507 19:51:56.623 | INFO | "gpu" : " 17 GB", | 20260507 19:51:56.623 | INFO | "huge" : "", | 20260507 19:51:56.623 | INFO | "huge_shared" : " 29 GB", | 20260507 19:51:56.623 | INFO | "malloc" : " 14 GB", | 20260507 19:51:56.623 | INFO | "pinned" : " 29 GB" | 20260507 19:51:56.623 | INFO | } | 20260507 19:51:56.624 | INFO | Perf report for the last 5.13873 seconds: | 20260507 19:51:56.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:51:56.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:56.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:56.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:56.624 | INFO | ANALYSIS : 69.3%, 2.9, 106.1ms, 0 | 20260507 19:51:56.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:51:56.624 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 19:51:56.624 | INFO | PRE_HQ : 67.9%, 1.9, 66.2ms, 0 | 20260507 19:51:56.624 | INFO | BAZWRITER : 69.7%, 4.0, 143.6ms, 0 | 20260507 19:51:56.624 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 19:51:56.624 | INFO | Uploaded 36GiB to GPU in 2.55228s (14.1051GiB/s) | 20260507 19:51:56.624 | INFO | Downloaded 38GiB to GPU in 2.69202s (14.1158GiB/s) | 20260507 19:52:01.762 | INFO | Analyzing chunk frames = [2150912,2151424) | 20260507 19:52:01.762 | INFO | MemInfo: { | 20260507 19:52:01.762 | INFO | "gpu" : " 17 GB", | 20260507 19:52:01.762 | INFO | "huge" : "", | 20260507 19:52:01.762 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:01.762 | INFO | "malloc" : " 14 GB", | 20260507 19:52:01.762 | INFO | "pinned" : " 29 GB" | 20260507 19:52:01.762 | INFO | } | 20260507 19:52:01.763 | INFO | Perf report for the last 5.13904 seconds: | 20260507 19:52:01.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:01.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 19:52:01.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 19:52:01.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:01.763 | INFO | Uploaded 36GiB to GPU in 2.59693s (13.8625GiB/s) | 20260507 19:52:01.764 | INFO | Downloaded 34GiB to GPU in 2.47866s (13.7171GiB/s) | 20260507 19:52:06.902 | INFO | Analyzing chunk frames = [2151424,2151936) | 20260507 19:52:06.902 | INFO | MemInfo: { | 20260507 19:52:06.902 | INFO | "gpu" : " 17 GB", | 20260507 19:52:06.902 | INFO | "huge" : "", | 20260507 19:52:06.902 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:06.902 | INFO | "malloc" : " 14 GB", | 20260507 19:52:06.902 | INFO | "pinned" : " 29 GB" | 20260507 19:52:06.902 | INFO | } | 20260507 19:52:06.902 | INFO | Perf report for the last 5.13966 seconds: | 20260507 19:52:06.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:06.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.902 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:52:06.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.902 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 19:52:06.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:06.903 | INFO | Uploaded 36GiB to GPU in 2.54795s (14.129GiB/s) | 20260507 19:52:06.903 | INFO | Downloaded 34GiB to GPU in 2.48122s (13.7029GiB/s) | 20260507 19:52:12.040 | INFO | Analyzing chunk frames = [2151936,2152448) | 20260507 19:52:12.040 | INFO | MemInfo: { | 20260507 19:52:12.040 | INFO | "gpu" : " 17 GB", | 20260507 19:52:12.040 | INFO | "huge" : "", | 20260507 19:52:12.040 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:12.040 | INFO | "malloc" : " 14 GB", | 20260507 19:52:12.040 | INFO | "pinned" : " 29 GB" | 20260507 19:52:12.041 | INFO | } | 20260507 19:52:12.041 | INFO | Perf report for the last 5.13842 seconds: | 20260507 19:52:12.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:12.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | ANALYSIS : 67.8%, 2.9, 104.8ms, 0 | 20260507 19:52:12.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 19:52:12.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:12.041 | INFO | Uploaded 36GiB to GPU in 2.51949s (14.2886GiB/s) | 20260507 19:52:12.041 | INFO | Downloaded 34GiB to GPU in 2.48979s (13.6558GiB/s) | 20260507 19:52:17.180 | INFO | Analyzing chunk frames = [2152448,2152960) | 20260507 19:52:17.180 | INFO | MemInfo: { | 20260507 19:52:17.180 | INFO | "gpu" : " 17 GB", | 20260507 19:52:17.180 | INFO | "huge" : "", | 20260507 19:52:17.180 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:17.180 | INFO | "malloc" : " 14 GB", | 20260507 19:52:17.180 | INFO | "pinned" : " 29 GB" | 20260507 19:52:17.180 | INFO | } | 20260507 19:52:17.180 | INFO | Perf report for the last 5.13935 seconds: | 20260507 19:52:17.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:17.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260507 19:52:17.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 19:52:17.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:17.180 | INFO | Uploaded 36GiB to GPU in 2.46665s (14.5947GiB/s) | 20260507 19:52:17.181 | INFO | Downloaded 34GiB to GPU in 2.49564s (13.6238GiB/s) | 20260507 19:52:22.319 | INFO | Analyzing chunk frames = [2152960,2153472) | 20260507 19:52:22.319 | INFO | MemInfo: { | 20260507 19:52:22.319 | INFO | "gpu" : " 17 GB", | 20260507 19:52:22.319 | INFO | "huge" : "", | 20260507 19:52:22.319 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:22.319 | INFO | "malloc" : " 14 GB", | 20260507 19:52:22.319 | INFO | "pinned" : " 29 GB" | 20260507 19:52:22.319 | INFO | } | 20260507 19:52:22.319 | INFO | Perf report for the last 5.13864 seconds: | 20260507 19:52:22.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:22.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:52:22.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:52:22.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:22.319 | INFO | Uploaded 36GiB to GPU in 2.50464s (14.3733GiB/s) | 20260507 19:52:22.319 | INFO | Downloaded 34GiB to GPU in 2.41031s (14.1061GiB/s) | 20260507 19:52:27.458 | INFO | Analyzing chunk frames = [2153472,2153984) | 20260507 19:52:27.458 | INFO | MemInfo: { | 20260507 19:52:27.458 | INFO | "gpu" : " 17 GB", | 20260507 19:52:27.458 | INFO | "huge" : "", | 20260507 19:52:27.458 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:27.458 | INFO | "malloc" : " 14 GB", | 20260507 19:52:27.458 | INFO | "pinned" : " 29 GB" | 20260507 19:52:27.459 | INFO | } | 20260507 19:52:27.459 | INFO | Perf report for the last 5.13991 seconds: | 20260507 19:52:27.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:27.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 19:52:27.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:52:27.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:27.459 | INFO | Uploaded 36GiB to GPU in 2.55445s (14.0931GiB/s) | 20260507 19:52:27.459 | INFO | Downloaded 34GiB to GPU in 2.45239s (13.864GiB/s) | 20260507 19:52:32.599 | INFO | Analyzing chunk frames = [2153984,2154496) | 20260507 19:52:32.599 | INFO | MemInfo: { | 20260507 19:52:32.599 | INFO | "gpu" : " 17 GB", | 20260507 19:52:32.599 | INFO | "huge" : "", | 20260507 19:52:32.599 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:32.599 | INFO | "malloc" : " 14 GB", | 20260507 19:52:32.599 | INFO | "pinned" : " 29 GB" | 20260507 19:52:32.599 | INFO | } | 20260507 19:52:32.599 | INFO | Perf report for the last 5.14027 seconds: | 20260507 19:52:32.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:32.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 19:52:32.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 19:52:32.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:32.599 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:52:32.599 | INFO | Total mem usage: 16.4 GiB | 20260507 19:52:32.599 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:52:32.599 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 19:52:32.599 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:52:32.599 | INFO | High Water Breakdown: | 20260507 19:52:32.599 | INFO | 59.8% : 8.9 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:52:32.599 | INFO | 40% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:52:32.599 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:52:32.599 | INFO | ------------------------------------------ | 20260507 19:52:32.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:52:32.599 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:52:32.599 | INFO | Total mem usage: 26.5 GiB | 20260507 19:52:32.599 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:52:32.599 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:52:32.599 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:52:32.599 | INFO | High Water Breakdown: | 20260507 19:52:32.599 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:52:32.599 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:52:32.599 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:52:32.599 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:52:32.599 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:52:32.599 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:52:32.599 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:52:32.599 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:52:32.599 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:52:32.599 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:52:32.599 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:52:32.599 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:52:32.599 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:52:32.599 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:52:32.599 | INFO | ------------------------------------------ | 20260507 19:52:32.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:52:32.599 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:52:32.599 | INFO | Total mem usage: 27.8 GiB | 20260507 19:52:32.599 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:52:32.599 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:52:32.599 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:52:32.599 | INFO | High Water Breakdown: | 20260507 19:52:32.599 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:52:32.599 | INFO | ------------------------------------------ | 20260507 19:52:32.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:52:32.599 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:52:32.599 | INFO | Total mem usage: 15.7 GiB | 20260507 19:52:32.599 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:52:32.599 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:52:32.599 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:52:32.599 | INFO | High Water Breakdown: | 20260507 19:52:32.599 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:52:32.599 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:52:32.599 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:52:32.599 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:52:32.599 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:52:32.599 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:52:32.599 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:52:32.599 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:52:32.599 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:52:32.599 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:52:32.599 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:52:32.599 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:52:32.599 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:52:32.599 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:52:32.599 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:52:32.599 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:52:32.599 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:52:32.599 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:52:32.599 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:52:32.599 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:52:32.599 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:52:32.599 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:52:32.599 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:52:32.599 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:52:32.599 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:52:32.599 | INFO | ------------------------------------------ | 20260507 19:52:32.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:52:32.599 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:52:32.599 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:52:32.599 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:52:32.599 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.43e+01 / 2.06e+00 | 20260507 19:52:32.599 | INFO | PerfReport: 0.30%, AggregateStats -- 9.95e-02 / 1.03e-01, 1.21e-02 / 4.85e-03 | 20260507 19:52:32.599 | INFO | PerfReport: 12.09%, Baseline -- 4.00e+00 / 4.47e+00, 5.90e-01 / 2.68e-01 | 20260507 19:52:32.599 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.48e+00, 2.09e-01 / 1.24e-01 | 20260507 19:52:32.599 | INFO | PerfReport: 15.87%, DME -- 4.20e+01 / 4.53e+01, 4.09e+00 / 1.84e+00 | 20260507 19:52:32.599 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.74e+00 / 1.51e+00 | 20260507 19:52:32.599 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.38e-01 / 2.02e-01 | 20260507 19:52:32.599 | INFO | PerfReport: 3.62%, Metrics -- 1.20e+00 / 1.42e+00, 3.01e-01 / 1.74e-01 | 20260507 19:52:32.599 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.77e-01 / 4.17e-01, 6.08e-02 / 3.22e-02 | 20260507 19:52:32.599 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:52:32.600 | INFO | Uploaded 36GiB to GPU in 2.52707s (14.2457GiB/s) | 20260507 19:52:32.599 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:52:32.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:52:32.600 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 19:52:32.600 | INFO | Downloaded 34GiB to GPU in 2.42367s (14.0283GiB/s) | 20260507 19:52:32.601 | INFO | PerfReport: 24.55%, Upload -- 2.64e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 19:52:32.601 | INFO | PerfReport: 24.08%, Download -- 2.59e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 19:52:32.601 | INFO | PerfReport: 30.86%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 19:52:32.601 | INFO | PerfReport: 2.32%, LockUpload -- 2.49e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260507 19:52:32.601 | INFO | PerfReport: 2.23%, LockDownload -- 2.39e+00 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260507 19:52:32.601 | INFO | PerfReport: 15.96%, LockCompute -- 1.71e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 19:52:32.601 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 19:52:32.699 | INFO | Starting a round of Baz IO | 20260507 19:52:36.179 | INFO | Populating RTMetrics report | 20260507 19:52:36.179 | INFO | RTMetrics report populated | 20260507 19:52:36.179 | INFO | Opening tmp RTMetrics json file | 20260507 19:52:36.180 | INFO | Serializing RTMetrics to json | 20260507 19:52:36.204 | INFO | Serialization done, swapping tmp file now... | 20260507 19:52:36.207 | INFO | RTMetrics done | 20260507 19:52:36.285 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.585 seconds | 20260507 19:52:36.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.99% / 0.01% / 55.00% | 20260507 19:52:36.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.23082% / 2.78155e-05% / 4.37101e-05% | 20260507 19:52:36.285 | INFO | Baz IO overall: | 20260507 19:52:36.285 | INFO | BytesWritten : 975336336330 | 20260507 19:52:36.285 | INFO | NumEvents : 472871068808 | 20260507 19:52:37.739 | INFO | Analyzing chunk frames = [2154496,2155008) | 20260507 19:52:37.739 | INFO | MemInfo: { | 20260507 19:52:37.739 | INFO | "gpu" : " 17 GB", | 20260507 19:52:37.739 | INFO | "huge" : "", | 20260507 19:52:37.739 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:37.739 | INFO | "malloc" : " 14 GB", | 20260507 19:52:37.739 | INFO | "pinned" : " 29 GB" | 20260507 19:52:37.739 | INFO | } | 20260507 19:52:37.739 | INFO | Perf report for the last 5.13994 seconds: | 20260507 19:52:37.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:37.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:37.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:37.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:37.739 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 19:52:37.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:37.739 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 19:52:37.739 | INFO | PRE_HQ : 66.1%, 1.7, 58.3ms, 0 | 20260507 19:52:37.739 | INFO | BAZWRITER : 69.4%, 3.9, 137.8ms, 0 | 20260507 19:52:37.739 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:52:37.740 | INFO | Uploaded 36GiB to GPU in 2.54625s (14.1384GiB/s) | 20260507 19:52:37.740 | INFO | Downloaded 38GiB to GPU in 2.68865s (14.1335GiB/s) | 20260507 19:52:42.877 | INFO | Analyzing chunk frames = [2155008,2155520) | 20260507 19:52:42.877 | INFO | MemInfo: { | 20260507 19:52:42.877 | INFO | "gpu" : " 17 GB", | 20260507 19:52:42.877 | INFO | "huge" : "", | 20260507 19:52:42.877 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:42.877 | INFO | "malloc" : " 14 GB", | 20260507 19:52:42.877 | INFO | "pinned" : " 29 GB" | 20260507 19:52:42.877 | INFO | } | 20260507 19:52:42.877 | INFO | Perf report for the last 5.1386 seconds: | 20260507 19:52:42.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:42.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.878 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:52:42.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.878 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 19:52:42.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:42.878 | INFO | Uploaded 36GiB to GPU in 2.55281s (14.1021GiB/s) | 20260507 19:52:42.878 | INFO | Downloaded 34GiB to GPU in 2.48562s (13.6787GiB/s) | 20260507 19:52:48.017 | INFO | Analyzing chunk frames = [2155520,2156032) | 20260507 19:52:48.017 | INFO | MemInfo: { | 20260507 19:52:48.017 | INFO | "gpu" : " 17 GB", | 20260507 19:52:48.017 | INFO | "huge" : "", | 20260507 19:52:48.017 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:48.017 | INFO | "malloc" : " 14 GB", | 20260507 19:52:48.017 | INFO | "pinned" : " 29 GB" | 20260507 19:52:48.017 | INFO | } | 20260507 19:52:48.017 | INFO | Perf report for the last 5.1397 seconds: | 20260507 19:52:48.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:48.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 19:52:48.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:52:48.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:48.017 | INFO | Uploaded 36GiB to GPU in 2.59253s (13.8861GiB/s) | 20260507 19:52:48.017 | INFO | Downloaded 34GiB to GPU in 2.45853s (13.8294GiB/s) | 20260507 19:52:53.156 | INFO | Analyzing chunk frames = [2156032,2156544) | 20260507 19:52:53.156 | INFO | MemInfo: { | 20260507 19:52:53.156 | INFO | "gpu" : " 17 GB", | 20260507 19:52:53.156 | INFO | "huge" : "", | 20260507 19:52:53.156 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:53.156 | INFO | "malloc" : " 14 GB", | 20260507 19:52:53.156 | INFO | "pinned" : " 29 GB" | 20260507 19:52:53.156 | INFO | } | 20260507 19:52:53.156 | INFO | Perf report for the last 5.13895 seconds: | 20260507 19:52:53.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:53.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 19:52:53.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 19:52:53.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:53.156 | INFO | Uploaded 36GiB to GPU in 2.50357s (14.3795GiB/s) | 20260507 19:52:53.156 | INFO | Downloaded 34GiB to GPU in 2.39167s (14.216GiB/s) | 20260507 19:52:58.295 | INFO | Analyzing chunk frames = [2156544,2157056) | 20260507 19:52:58.295 | INFO | MemInfo: { | 20260507 19:52:58.295 | INFO | "gpu" : " 17 GB", | 20260507 19:52:58.295 | INFO | "huge" : "", | 20260507 19:52:58.295 | INFO | "huge_shared" : " 29 GB", | 20260507 19:52:58.295 | INFO | "malloc" : " 14 GB", | 20260507 19:52:58.295 | INFO | "pinned" : " 29 GB" | 20260507 19:52:58.295 | INFO | } | 20260507 19:52:58.295 | INFO | Perf report for the last 5.13913 seconds: | 20260507 19:52:58.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:52:58.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.295 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 19:52:58.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.296 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:52:58.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:52:58.296 | INFO | Uploaded 36GiB to GPU in 2.48369s (14.4945GiB/s) | 20260507 19:52:58.296 | INFO | Downloaded 34GiB to GPU in 2.49481s (13.6283GiB/s) | 20260507 19:53:03.434 | INFO | Analyzing chunk frames = [2157056,2157568) | 20260507 19:53:03.434 | INFO | MemInfo: { | 20260507 19:53:03.434 | INFO | "gpu" : " 17 GB", | 20260507 19:53:03.434 | INFO | "huge" : "", | 20260507 19:53:03.434 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:03.434 | INFO | "malloc" : " 14 GB", | 20260507 19:53:03.434 | INFO | "pinned" : " 29 GB" | 20260507 19:53:03.434 | INFO | } | 20260507 19:53:03.434 | INFO | Perf report for the last 5.13847 seconds: | 20260507 19:53:03.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:03.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 19:53:03.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:53:03.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:03.434 | INFO | Uploaded 36GiB to GPU in 2.52992s (14.2297GiB/s) | 20260507 19:53:03.434 | INFO | Downloaded 34GiB to GPU in 2.41543s (14.0761GiB/s) | 20260507 19:53:08.574 | INFO | Analyzing chunk frames = [2157568,2158080) | 20260507 19:53:08.574 | INFO | MemInfo: { | 20260507 19:53:08.574 | INFO | "gpu" : " 17 GB", | 20260507 19:53:08.574 | INFO | "huge" : "", | 20260507 19:53:08.574 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:08.574 | INFO | "malloc" : " 14 GB", | 20260507 19:53:08.574 | INFO | "pinned" : " 29 GB" | 20260507 19:53:08.574 | INFO | } | 20260507 19:53:08.574 | INFO | Perf report for the last 5.14044 seconds: | 20260507 19:53:08.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:08.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.574 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:53:08.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.574 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:53:08.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:08.575 | INFO | Uploaded 36GiB to GPU in 2.55404s (14.0953GiB/s) | 20260507 19:53:08.575 | INFO | Downloaded 34GiB to GPU in 2.44318s (13.9163GiB/s) | 20260507 19:53:13.713 | INFO | Analyzing chunk frames = [2158080,2158592) | 20260507 19:53:13.713 | INFO | MemInfo: { | 20260507 19:53:13.713 | INFO | "gpu" : " 17 GB", | 20260507 19:53:13.713 | INFO | "huge" : "", | 20260507 19:53:13.713 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:13.713 | INFO | "malloc" : " 14 GB", | 20260507 19:53:13.713 | INFO | "pinned" : " 29 GB" | 20260507 19:53:13.713 | INFO | } | 20260507 19:53:13.713 | INFO | Perf report for the last 5.13901 seconds: | 20260507 19:53:13.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:13.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:53:13.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:53:13.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:13.713 | INFO | Uploaded 36GiB to GPU in 2.53262s (14.2145GiB/s) | 20260507 19:53:13.713 | INFO | Downloaded 34GiB to GPU in 2.41478s (14.0799GiB/s) | 20260507 19:53:13.814 | INFO | Starting a round of Baz IO | 20260507 19:53:17.296 | INFO | Populating RTMetrics report | 20260507 19:53:17.297 | INFO | RTMetrics report populated | 20260507 19:53:17.297 | INFO | Opening tmp RTMetrics json file | 20260507 19:53:17.326 | INFO | Serializing RTMetrics to json | 20260507 19:53:17.332 | INFO | Serialization done, swapping tmp file now... | 20260507 19:53:17.333 | INFO | RTMetrics done | 20260507 19:53:17.404 | INFO | Finished a round of Baz IO: Wrote 1.787 GiB in 3.59 seconds | 20260507 19:53:17.404 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.98% / 0.01% / 55.01% | 20260507 19:53:17.404 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2267% / 7.94729e-06% / 4.76837e-05% | 20260507 19:53:17.404 | INFO | Baz IO overall: | 20260507 19:53:17.404 | INFO | BytesWritten : 977254933652 | 20260507 19:53:17.404 | INFO | NumEvents : 473801221725 | 20260507 19:53:18.852 | INFO | Analyzing chunk frames = [2158592,2159104) | 20260507 19:53:18.852 | INFO | MemInfo: { | 20260507 19:53:18.852 | INFO | "gpu" : " 17 GB", | 20260507 19:53:18.852 | INFO | "huge" : "", | 20260507 19:53:18.852 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:18.852 | INFO | "malloc" : " 14 GB", | 20260507 19:53:18.852 | INFO | "pinned" : " 29 GB" | 20260507 19:53:18.852 | INFO | } | 20260507 19:53:18.852 | INFO | Perf report for the last 5.13865 seconds: | 20260507 19:53:18.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:18.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:18.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:18.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:18.852 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 19:53:18.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:18.852 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 19:53:18.852 | INFO | PRE_HQ : 66.4%, 1.7, 59.0ms, 0 | 20260507 19:53:18.852 | INFO | BAZWRITER : 69.3%, 3.9, 137.8ms, 0 | 20260507 19:53:18.852 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 19:53:18.852 | INFO | Uploaded 36GiB to GPU in 2.5556s (14.0867GiB/s) | 20260507 19:53:18.852 | INFO | Downloaded 38GiB to GPU in 2.68764s (14.1388GiB/s) | 20260507 19:53:23.991 | INFO | Analyzing chunk frames = [2159104,2159616) | 20260507 19:53:23.991 | INFO | MemInfo: { | 20260507 19:53:23.991 | INFO | "gpu" : " 17 GB", | 20260507 19:53:23.991 | INFO | "huge" : "", | 20260507 19:53:23.991 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:23.992 | INFO | "malloc" : " 14 GB", | 20260507 19:53:23.992 | INFO | "pinned" : " 29 GB" | 20260507 19:53:23.992 | INFO | } | 20260507 19:53:23.992 | INFO | Perf report for the last 5.13985 seconds: | 20260507 19:53:23.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:23.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 19:53:23.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:53:23.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:23.992 | INFO | Uploaded 36GiB to GPU in 2.58158s (13.9449GiB/s) | 20260507 19:53:23.992 | INFO | Downloaded 34GiB to GPU in 2.46692s (13.7824GiB/s) | 20260507 19:53:29.131 | INFO | Analyzing chunk frames = [2159616,2160128) | 20260507 19:53:29.131 | INFO | MemInfo: { | 20260507 19:53:29.131 | INFO | "gpu" : " 17 GB", | 20260507 19:53:29.131 | INFO | "huge" : "", | 20260507 19:53:29.131 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:29.131 | INFO | "malloc" : " 14 GB", | 20260507 19:53:29.131 | INFO | "pinned" : " 29 GB" | 20260507 19:53:29.131 | INFO | } | 20260507 19:53:29.131 | INFO | Perf report for the last 5.13938 seconds: | 20260507 19:53:29.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:29.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.131 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260507 19:53:29.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.131 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 19:53:29.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:29.132 | INFO | Uploaded 36GiB to GPU in 2.60403s (13.8247GiB/s) | 20260507 19:53:29.132 | INFO | Downloaded 34GiB to GPU in 2.46532s (13.7913GiB/s) | 20260507 19:53:34.271 | INFO | Analyzing chunk frames = [2160128,2160640) | 20260507 19:53:34.271 | INFO | MemInfo: { | 20260507 19:53:34.271 | INFO | "gpu" : " 17 GB", | 20260507 19:53:34.271 | INFO | "huge" : "", | 20260507 19:53:34.271 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:34.271 | INFO | "malloc" : " 14 GB", | 20260507 19:53:34.271 | INFO | "pinned" : " 29 GB" | 20260507 19:53:34.271 | INFO | } | 20260507 19:53:34.271 | INFO | Perf report for the last 5.14001 seconds: | 20260507 19:53:34.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:34.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.271 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 19:53:34.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.271 | INFO | PRE_HQ : 37.3%, 1.0, 19.3ms, 0 | 20260507 19:53:34.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:34.272 | INFO | Uploaded 36GiB to GPU in 2.49115s (14.4512GiB/s) | 20260507 19:53:34.272 | INFO | Downloaded 34GiB to GPU in 2.39673s (14.186GiB/s) | 20260507 19:53:39.411 | INFO | Analyzing chunk frames = [2160640,2161152) | 20260507 19:53:39.411 | INFO | MemInfo: { | 20260507 19:53:39.411 | INFO | "gpu" : " 17 GB", | 20260507 19:53:39.411 | INFO | "huge" : "", | 20260507 19:53:39.411 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:39.411 | INFO | "malloc" : " 14 GB", | 20260507 19:53:39.411 | INFO | "pinned" : " 29 GB" | 20260507 19:53:39.411 | INFO | } | 20260507 19:53:39.411 | INFO | Perf report for the last 5.13947 seconds: | 20260507 19:53:39.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:39.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 19:53:39.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | PRE_HQ : 36.5%, 1.0, 18.9ms, 0 | 20260507 19:53:39.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:39.411 | INFO | Uploaded 36GiB to GPU in 2.40788s (14.9509GiB/s) | 20260507 19:53:39.411 | INFO | Downloaded 34GiB to GPU in 2.4779s (13.7213GiB/s) | 20260507 19:53:44.552 | INFO | Analyzing chunk frames = [2161152,2161664) | 20260507 19:53:44.552 | INFO | MemInfo: { | 20260507 19:53:44.552 | INFO | "gpu" : " 17 GB", | 20260507 19:53:44.552 | INFO | "huge" : "", | 20260507 19:53:44.552 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:44.552 | INFO | "malloc" : " 14 GB", | 20260507 19:53:44.552 | INFO | "pinned" : " 29 GB" | 20260507 19:53:44.552 | INFO | } | 20260507 19:53:44.552 | INFO | Perf report for the last 5.14114 seconds: | 20260507 19:53:44.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:44.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:53:44.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:53:44.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:44.552 | INFO | Uploaded 36GiB to GPU in 2.47909s (14.5215GiB/s) | 20260507 19:53:44.552 | INFO | Downloaded 34GiB to GPU in 2.4257s (14.0166GiB/s) | 20260507 19:53:49.691 | INFO | Analyzing chunk frames = [2161664,2162176) | 20260507 19:53:49.691 | INFO | MemInfo: { | 20260507 19:53:49.691 | INFO | "gpu" : " 17 GB", | 20260507 19:53:49.691 | INFO | "huge" : "", | 20260507 19:53:49.691 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:49.691 | INFO | "malloc" : " 14 GB", | 20260507 19:53:49.691 | INFO | "pinned" : " 29 GB" | 20260507 19:53:49.691 | INFO | } | 20260507 19:53:49.691 | INFO | Perf report for the last 5.13938 seconds: | 20260507 19:53:49.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:49.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.691 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260507 19:53:49.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.691 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:53:49.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:49.692 | INFO | Uploaded 36GiB to GPU in 2.50533s (14.3694GiB/s) | 20260507 19:53:49.692 | INFO | Downloaded 34GiB to GPU in 2.445s (13.9059GiB/s) | 20260507 19:53:54.828 | INFO | Analyzing chunk frames = [2162176,2162688) | 20260507 19:53:54.828 | INFO | MemInfo: { | 20260507 19:53:54.828 | INFO | "gpu" : " 17 GB", | 20260507 19:53:54.828 | INFO | "huge" : "", | 20260507 19:53:54.828 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:54.828 | INFO | "malloc" : " 14 GB", | 20260507 19:53:54.828 | INFO | "pinned" : " 29 GB" | 20260507 19:53:54.828 | INFO | } | 20260507 19:53:54.828 | INFO | Perf report for the last 5.13674 seconds: | 20260507 19:53:54.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:54.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:53:54.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 19:53:54.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:54.828 | INFO | Uploaded 36GiB to GPU in 2.47147s (14.5662GiB/s) | 20260507 19:53:54.828 | INFO | Downloaded 34GiB to GPU in 2.42766s (14.0053GiB/s) | 20260507 19:53:54.828 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:53:54.828 | INFO | Total mem usage: 16.4 GiB | 20260507 19:53:54.828 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:53:54.828 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 19:53:54.828 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:53:54.828 | INFO | High Water Breakdown: | 20260507 19:53:54.828 | INFO | 59.8% : 8.9 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:53:54.828 | INFO | 40% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:53:54.828 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:53:54.828 | INFO | ------------------------------------------ | 20260507 19:53:54.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:53:54.828 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:53:54.828 | INFO | Total mem usage: 26.5 GiB | 20260507 19:53:54.828 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:53:54.828 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:53:54.828 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:53:54.828 | INFO | High Water Breakdown: | 20260507 19:53:54.828 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:53:54.828 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:53:54.828 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:53:54.828 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:53:54.828 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:53:54.828 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:53:54.828 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:53:54.828 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:53:54.828 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:53:54.828 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:53:54.828 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:53:54.828 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:53:54.828 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:53:54.828 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:53:54.828 | INFO | ------------------------------------------ | 20260507 19:53:54.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:53:54.828 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:53:54.828 | INFO | Total mem usage: 27.8 GiB | 20260507 19:53:54.828 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:53:54.828 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:53:54.828 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:53:54.828 | INFO | High Water Breakdown: | 20260507 19:53:54.828 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:53:54.828 | INFO | ------------------------------------------ | 20260507 19:53:54.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:53:54.828 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:53:54.828 | INFO | Total mem usage: 15.7 GiB | 20260507 19:53:54.828 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:53:54.828 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:53:54.828 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:53:54.828 | INFO | High Water Breakdown: | 20260507 19:53:54.828 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:53:54.828 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:53:54.828 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:53:54.828 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:53:54.828 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:53:54.828 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:53:54.828 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:53:54.828 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:53:54.828 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:53:54.829 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:53:54.829 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:53:54.829 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:53:54.829 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:53:54.829 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:53:54.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:53:54.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:53:54.829 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:53:54.829 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:53:54.829 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:53:54.829 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:53:54.829 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:53:54.829 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:53:54.829 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:53:54.829 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:53:54.829 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:53:54.829 | INFO | ------------------------------------------ | 20260507 19:53:54.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:53:54.829 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:53:54.829 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:53:54.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:53:54.829 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.53e+01, 1.41e+01 / 1.77e+00 | 20260507 19:53:54.829 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.03e-01, 1.12e-02 / 5.28e-03 | 20260507 19:53:54.829 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.46e+00, 5.97e-01 / 2.79e-01 | 20260507 19:53:54.829 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.07e-01 | 20260507 19:53:54.829 | INFO | PerfReport: 15.71%, DME -- 4.16e+01 / 4.46e+01, 3.84e+00 / 1.35e+00 | 20260507 19:53:54.829 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.36e+00 | 20260507 19:53:54.829 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.40e-01 / 2.31e-01 | 20260507 19:53:54.829 | INFO | PerfReport: 3.62%, Metrics -- 1.20e+00 / 1.41e+00, 3.05e-01 / 1.92e-01 | 20260507 19:53:54.829 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.76e-01 / 4.07e-01, 6.11e-02 / 3.07e-02 | 20260507 19:53:54.829 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:53:54.830 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:53:54.830 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:53:54.830 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 24.54%, Upload -- 2.63e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 24.06%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 30.91%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 2.26%, LockDownload -- 2.42e+00 / 0.00e+00, 3.69e+00 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 16.05%, LockCompute -- 1.72e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 19:53:54.830 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 5.49e-02 / 0.00e+00 | 20260507 19:53:54.925 | INFO | Starting a round of Baz IO | 20260507 19:53:58.404 | INFO | Populating RTMetrics report | 20260507 19:53:58.405 | INFO | RTMetrics report populated | 20260507 19:53:58.405 | INFO | Opening tmp RTMetrics json file | 20260507 19:53:58.430 | INFO | Serializing RTMetrics to json | 20260507 19:53:58.432 | INFO | Serialization done, swapping tmp file now... | 20260507 19:53:58.433 | INFO | RTMetrics done | 20260507 19:53:58.507 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.581 seconds | 20260507 19:53:58.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.97% / 0.01% / 55.02% | 20260507 19:53:58.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.21323% / 1.19209e-05% / 3.57628e-05% | 20260507 19:53:58.507 | INFO | Baz IO overall: | 20260507 19:53:58.507 | INFO | BytesWritten : 979170910168 | 20260507 19:53:58.507 | INFO | NumEvents : 474729353729 | 20260507 19:53:59.967 | INFO | Analyzing chunk frames = [2162688,2163200) | 20260507 19:53:59.967 | INFO | MemInfo: { | 20260507 19:53:59.967 | INFO | "gpu" : " 17 GB", | 20260507 19:53:59.967 | INFO | "huge" : "", | 20260507 19:53:59.967 | INFO | "huge_shared" : " 29 GB", | 20260507 19:53:59.967 | INFO | "malloc" : " 14 GB", | 20260507 19:53:59.967 | INFO | "pinned" : " 29 GB" | 20260507 19:53:59.967 | INFO | } | 20260507 19:53:59.967 | INFO | Perf report for the last 5.13868 seconds: | 20260507 19:53:59.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:53:59.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:59.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:59.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:59.967 | INFO | ANALYSIS : 68.8%, 3.0, 107.0ms, 0 | 20260507 19:53:59.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:53:59.967 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 19:53:59.967 | INFO | PRE_HQ : 66.0%, 1.7, 58.3ms, 0 | 20260507 19:53:59.967 | INFO | BAZWRITER : 69.3%, 3.8, 136.7ms, 0 | 20260507 19:53:59.967 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 19:53:59.967 | INFO | Uploaded 36GiB to GPU in 2.48391s (14.4933GiB/s) | 20260507 19:53:59.967 | INFO | Downloaded 38GiB to GPU in 2.68917s (14.1307GiB/s) | 20260507 19:54:05.107 | INFO | Analyzing chunk frames = [2163200,2163712) | 20260507 19:54:05.107 | INFO | MemInfo: { | 20260507 19:54:05.107 | INFO | "gpu" : " 17 GB", | 20260507 19:54:05.107 | INFO | "huge" : "", | 20260507 19:54:05.107 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:05.107 | INFO | "malloc" : " 14 GB", | 20260507 19:54:05.107 | INFO | "pinned" : " 29 GB" | 20260507 19:54:05.107 | INFO | } | 20260507 19:54:05.107 | INFO | Perf report for the last 5.14044 seconds: | 20260507 19:54:05.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:05.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 19:54:05.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:54:05.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:05.107 | INFO | Uploaded 36GiB to GPU in 2.55043s (14.1153GiB/s) | 20260507 19:54:05.107 | INFO | Downloaded 34GiB to GPU in 2.47525s (13.736GiB/s) | 20260507 19:54:10.246 | INFO | Analyzing chunk frames = [2163712,2164224) | 20260507 19:54:10.246 | INFO | MemInfo: { | 20260507 19:54:10.246 | INFO | "gpu" : " 17 GB", | 20260507 19:54:10.246 | INFO | "huge" : "", | 20260507 19:54:10.246 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:10.246 | INFO | "malloc" : " 14 GB", | 20260507 19:54:10.246 | INFO | "pinned" : " 29 GB" | 20260507 19:54:10.246 | INFO | } | 20260507 19:54:10.246 | INFO | Perf report for the last 5.13882 seconds: | 20260507 19:54:10.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:10.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 19:54:10.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:54:10.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:10.246 | INFO | Uploaded 36GiB to GPU in 2.55714s (14.0782GiB/s) | 20260507 19:54:10.246 | INFO | Downloaded 34GiB to GPU in 2.45888s (13.8274GiB/s) | 20260507 19:54:15.385 | INFO | Analyzing chunk frames = [2164224,2164736) | 20260507 19:54:15.385 | INFO | MemInfo: { | 20260507 19:54:15.385 | INFO | "gpu" : " 17 GB", | 20260507 19:54:15.385 | INFO | "huge" : "", | 20260507 19:54:15.385 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:15.385 | INFO | "malloc" : " 14 GB", | 20260507 19:54:15.385 | INFO | "pinned" : " 29 GB" | 20260507 19:54:15.385 | INFO | } | 20260507 19:54:15.385 | INFO | Perf report for the last 5.13921 seconds: | 20260507 19:54:15.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:15.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.385 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:54:15.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.385 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 19:54:15.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:15.386 | INFO | Uploaded 36GiB to GPU in 2.46212s (14.6216GiB/s) | 20260507 19:54:15.386 | INFO | Downloaded 34GiB to GPU in 2.39574s (14.1918GiB/s) | 20260507 19:54:20.524 | INFO | Analyzing chunk frames = [2164736,2165248) | 20260507 19:54:20.524 | INFO | MemInfo: { | 20260507 19:54:20.524 | INFO | "gpu" : " 17 GB", | 20260507 19:54:20.524 | INFO | "huge" : "", | 20260507 19:54:20.524 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:20.524 | INFO | "malloc" : " 14 GB", | 20260507 19:54:20.524 | INFO | "pinned" : " 29 GB" | 20260507 19:54:20.524 | INFO | } | 20260507 19:54:20.524 | INFO | Perf report for the last 5.13894 seconds: | 20260507 19:54:20.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:20.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.524 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:54:20.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.524 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 19:54:20.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:20.525 | INFO | Uploaded 36GiB to GPU in 2.39172s (15.0519GiB/s) | 20260507 19:54:20.525 | INFO | Downloaded 34GiB to GPU in 2.50461s (13.5749GiB/s) | 20260507 19:54:25.664 | INFO | Analyzing chunk frames = [2165248,2165760) | 20260507 19:54:25.664 | INFO | MemInfo: { | 20260507 19:54:25.664 | INFO | "gpu" : " 17 GB", | 20260507 19:54:25.664 | INFO | "huge" : "", | 20260507 19:54:25.664 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:25.664 | INFO | "malloc" : " 14 GB", | 20260507 19:54:25.664 | INFO | "pinned" : " 29 GB" | 20260507 19:54:25.664 | INFO | } | 20260507 19:54:25.664 | INFO | Perf report for the last 5.13965 seconds: | 20260507 19:54:25.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:25.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 19:54:25.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 19:54:25.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:25.664 | INFO | Uploaded 36GiB to GPU in 2.54051s (14.1704GiB/s) | 20260507 19:54:25.664 | INFO | Downloaded 34GiB to GPU in 2.41606s (14.0725GiB/s) | 20260507 19:54:30.803 | INFO | Analyzing chunk frames = [2165760,2166272) | 20260507 19:54:30.803 | INFO | MemInfo: { | 20260507 19:54:30.803 | INFO | "gpu" : " 17 GB", | 20260507 19:54:30.803 | INFO | "huge" : "", | 20260507 19:54:30.803 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:30.803 | INFO | "malloc" : " 14 GB", | 20260507 19:54:30.803 | INFO | "pinned" : " 29 GB" | 20260507 19:54:30.803 | INFO | } | 20260507 19:54:30.803 | INFO | Perf report for the last 5.13963 seconds: | 20260507 19:54:30.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:30.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:54:30.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:54:30.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:30.804 | INFO | Uploaded 36GiB to GPU in 2.5624s (14.0493GiB/s) | 20260507 19:54:30.804 | INFO | Downloaded 34GiB to GPU in 2.44639s (13.898GiB/s) | 20260507 19:54:35.943 | INFO | Analyzing chunk frames = [2166272,2166784) | 20260507 19:54:35.943 | INFO | MemInfo: { | 20260507 19:54:35.943 | INFO | "gpu" : " 17 GB", | 20260507 19:54:35.943 | INFO | "huge" : "", | 20260507 19:54:35.943 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:35.943 | INFO | "malloc" : " 14 GB", | 20260507 19:54:35.944 | INFO | "pinned" : " 29 GB" | 20260507 19:54:35.944 | INFO | } | 20260507 19:54:35.944 | INFO | Perf report for the last 5.14018 seconds: | 20260507 19:54:35.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:35.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:54:35.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 19:54:35.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:35.944 | INFO | Uploaded 36GiB to GPU in 2.53077s (14.2249GiB/s) | 20260507 19:54:35.944 | INFO | Downloaded 34GiB to GPU in 2.42654s (14.0117GiB/s) | 20260507 19:54:36.041 | INFO | Starting a round of Baz IO | 20260507 19:54:39.525 | INFO | Populating RTMetrics report | 20260507 19:54:39.526 | INFO | RTMetrics report populated | 20260507 19:54:39.526 | INFO | Opening tmp RTMetrics json file | 20260507 19:54:39.552 | INFO | Serializing RTMetrics to json | 20260507 19:54:39.554 | INFO | Serialization done, swapping tmp file now... | 20260507 19:54:39.555 | INFO | RTMetrics done | 20260507 19:54:39.640 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.599 seconds | 20260507 19:54:39.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.96% / 0.01% / 55.03% | 20260507 19:54:39.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.21058% / 2.38419e-05% / 5.5631e-05% | 20260507 19:54:39.640 | INFO | Baz IO overall: | 20260507 19:54:39.640 | INFO | BytesWritten : 981087319937 | 20260507 19:54:39.641 | INFO | NumEvents : 475657765729 | 20260507 19:54:41.083 | INFO | Analyzing chunk frames = [2166784,2167296) | 20260507 19:54:41.083 | INFO | MemInfo: { | 20260507 19:54:41.083 | INFO | "gpu" : " 17 GB", | 20260507 19:54:41.083 | INFO | "huge" : "", | 20260507 19:54:41.083 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:41.083 | INFO | "malloc" : " 14 GB", | 20260507 19:54:41.083 | INFO | "pinned" : " 29 GB" | 20260507 19:54:41.084 | INFO | } | 20260507 19:54:41.084 | INFO | Perf report for the last 5.14002 seconds: | 20260507 19:54:41.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:41.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:41.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:41.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:41.084 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 19:54:41.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:41.084 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 19:54:41.084 | INFO | PRE_HQ : 66.2%, 1.7, 58.5ms, 0 | 20260507 19:54:41.084 | INFO | BAZWRITER : 69.6%, 4.1, 147.4ms, 0 | 20260507 19:54:41.084 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:54:41.084 | INFO | Uploaded 36GiB to GPU in 2.54s (14.1733GiB/s) | 20260507 19:54:41.084 | INFO | Downloaded 38GiB to GPU in 2.69185s (14.1167GiB/s) | 20260507 19:54:46.221 | INFO | Analyzing chunk frames = [2167296,2167808) | 20260507 19:54:46.221 | INFO | MemInfo: { | 20260507 19:54:46.221 | INFO | "gpu" : " 17 GB", | 20260507 19:54:46.221 | INFO | "huge" : "", | 20260507 19:54:46.221 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:46.221 | INFO | "malloc" : " 14 GB", | 20260507 19:54:46.221 | INFO | "pinned" : " 29 GB" | 20260507 19:54:46.221 | INFO | } | 20260507 19:54:46.221 | INFO | Perf report for the last 5.1374 seconds: | 20260507 19:54:46.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:46.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.221 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:54:46.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.221 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:54:46.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:46.222 | INFO | Uploaded 36GiB to GPU in 2.56784s (14.0195GiB/s) | 20260507 19:54:46.222 | INFO | Downloaded 34GiB to GPU in 2.46555s (13.79GiB/s) | 20260507 19:54:51.361 | INFO | Analyzing chunk frames = [2167808,2168320) | 20260507 19:54:51.361 | INFO | MemInfo: { | 20260507 19:54:51.361 | INFO | "gpu" : " 17 GB", | 20260507 19:54:51.361 | INFO | "huge" : "", | 20260507 19:54:51.361 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:51.361 | INFO | "malloc" : " 14 GB", | 20260507 19:54:51.361 | INFO | "pinned" : " 29 GB" | 20260507 19:54:51.361 | INFO | } | 20260507 19:54:51.361 | INFO | Perf report for the last 5.13984 seconds: | 20260507 19:54:51.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:51.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:54:51.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:54:51.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:51.361 | INFO | Uploaded 36GiB to GPU in 2.59036s (13.8977GiB/s) | 20260507 19:54:51.361 | INFO | Downloaded 34GiB to GPU in 2.46296s (13.8045GiB/s) | 20260507 19:54:56.500 | INFO | Analyzing chunk frames = [2168320,2168832) | 20260507 19:54:56.500 | INFO | MemInfo: { | 20260507 19:54:56.500 | INFO | "gpu" : " 17 GB", | 20260507 19:54:56.500 | INFO | "huge" : "", | 20260507 19:54:56.500 | INFO | "huge_shared" : " 29 GB", | 20260507 19:54:56.500 | INFO | "malloc" : " 14 GB", | 20260507 19:54:56.500 | INFO | "pinned" : " 29 GB" | 20260507 19:54:56.500 | INFO | } | 20260507 19:54:56.500 | INFO | Perf report for the last 5.13953 seconds: | 20260507 19:54:56.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:54:56.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.501 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:54:56.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.501 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:54:56.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:54:56.501 | INFO | Uploaded 36GiB to GPU in 2.49984s (14.4009GiB/s) | 20260507 19:54:56.501 | INFO | Downloaded 34GiB to GPU in 2.39952s (14.1695GiB/s) | 20260507 19:55:01.639 | INFO | Analyzing chunk frames = [2168832,2169344) | 20260507 19:55:01.639 | INFO | MemInfo: { | 20260507 19:55:01.639 | INFO | "gpu" : " 17 GB", | 20260507 19:55:01.640 | INFO | "huge" : "", | 20260507 19:55:01.640 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:01.640 | INFO | "malloc" : " 14 GB", | 20260507 19:55:01.640 | INFO | "pinned" : " 29 GB" | 20260507 19:55:01.640 | INFO | } | 20260507 19:55:01.640 | INFO | Perf report for the last 5.13915 seconds: | 20260507 19:55:01.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:01.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:55:01.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 19:55:01.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:01.640 | INFO | Uploaded 36GiB to GPU in 2.37359s (15.1669GiB/s) | 20260507 19:55:01.640 | INFO | Downloaded 34GiB to GPU in 2.50489s (13.5734GiB/s) | 20260507 19:55:06.779 | INFO | Analyzing chunk frames = [2169344,2169856) | 20260507 19:55:06.779 | INFO | MemInfo: { | 20260507 19:55:06.779 | INFO | "gpu" : " 17 GB", | 20260507 19:55:06.779 | INFO | "huge" : "", | 20260507 19:55:06.779 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:06.779 | INFO | "malloc" : " 14 GB", | 20260507 19:55:06.779 | INFO | "pinned" : " 29 GB" | 20260507 19:55:06.779 | INFO | } | 20260507 19:55:06.779 | INFO | Perf report for the last 5.1395 seconds: | 20260507 19:55:06.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:06.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.779 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:55:06.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.779 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:55:06.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:06.780 | INFO | Uploaded 36GiB to GPU in 2.47605s (14.5393GiB/s) | 20260507 19:55:06.780 | INFO | Downloaded 34GiB to GPU in 2.42357s (14.0289GiB/s) | 20260507 19:55:11.918 | INFO | Analyzing chunk frames = [2169856,2170368) | 20260507 19:55:11.918 | INFO | MemInfo: { | 20260507 19:55:11.918 | INFO | "gpu" : " 17 GB", | 20260507 19:55:11.918 | INFO | "huge" : "", | 20260507 19:55:11.918 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:11.918 | INFO | "malloc" : " 14 GB", | 20260507 19:55:11.918 | INFO | "pinned" : " 29 GB" | 20260507 19:55:11.918 | INFO | } | 20260507 19:55:11.918 | INFO | Perf report for the last 5.13913 seconds: | 20260507 19:55:11.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:11.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.918 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 19:55:11.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.918 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:55:11.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:11.919 | INFO | Uploaded 36GiB to GPU in 2.50665s (14.3618GiB/s) | 20260507 19:55:11.919 | INFO | Downloaded 34GiB to GPU in 2.44838s (13.8867GiB/s) | 20260507 19:55:17.057 | INFO | Analyzing chunk frames = [2170368,2170880) | 20260507 19:55:17.057 | INFO | MemInfo: { | 20260507 19:55:17.057 | INFO | "gpu" : " 17 GB", | 20260507 19:55:17.057 | INFO | "huge" : "", | 20260507 19:55:17.057 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:17.057 | INFO | "malloc" : " 14 GB", | 20260507 19:55:17.057 | INFO | "pinned" : " 29 GB" | 20260507 19:55:17.057 | INFO | } | 20260507 19:55:17.057 | INFO | Perf report for the last 5.13902 seconds: | 20260507 19:55:17.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:17.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.057 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:55:17.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.057 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 19:55:17.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:17.058 | INFO | Uploaded 36GiB to GPU in 2.47188s (14.5638GiB/s) | 20260507 19:55:17.058 | INFO | Downloaded 34GiB to GPU in 2.42698s (14.0092GiB/s) | 20260507 19:55:17.058 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:55:17.058 | INFO | Total mem usage: 16.4 GiB | 20260507 19:55:17.058 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:55:17.058 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -4.88%) | 20260507 19:55:17.058 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:55:17.058 | INFO | High Water Breakdown: | 20260507 19:55:17.058 | INFO | 60.7% : 9.49 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:55:17.058 | INFO | 39.2% : 6.13 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:55:17.058 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:55:17.058 | INFO | ------------------------------------------ | 20260507 19:55:17.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:55:17.058 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:55:17.058 | INFO | Total mem usage: 26.5 GiB | 20260507 19:55:17.058 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:55:17.058 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:55:17.058 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:55:17.058 | INFO | High Water Breakdown: | 20260507 19:55:17.058 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:55:17.058 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:55:17.058 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:55:17.058 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:55:17.058 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:55:17.058 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:55:17.058 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:55:17.058 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:55:17.058 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:55:17.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:55:17.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:55:17.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:55:17.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:55:17.058 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:55:17.058 | INFO | ------------------------------------------ | 20260507 19:55:17.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:55:17.058 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:55:17.058 | INFO | Total mem usage: 27.8 GiB | 20260507 19:55:17.058 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:55:17.059 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 19:55:17.059 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 19:55:17.059 | INFO | High Water Breakdown: | 20260507 19:55:17.059 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 19:55:17.059 | INFO | ------------------------------------------ | 20260507 19:55:17.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:55:17.059 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:55:17.059 | INFO | Total mem usage: 15.7 GiB | 20260507 19:55:17.059 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:55:17.059 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:55:17.059 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:55:17.059 | INFO | High Water Breakdown: | 20260507 19:55:17.059 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:55:17.059 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:55:17.059 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:55:17.059 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:55:17.059 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:55:17.059 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:55:17.059 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:55:17.059 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:55:17.059 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:55:17.059 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:55:17.059 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:55:17.059 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:55:17.059 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:55:17.059 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:55:17.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:55:17.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:55:17.059 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:55:17.059 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:55:17.059 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:55:17.059 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:55:17.059 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:55:17.059 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:55:17.059 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:55:17.059 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:55:17.059 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:55:17.059 | INFO | ------------------------------------------ | 20260507 19:55:17.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:55:17.059 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:55:17.059 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:55:17.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:55:17.059 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.43e+01 / 1.93e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.12e-02 / 5.24e-03 | 20260507 19:55:17.059 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.48e+00, 6.03e-01 / 2.61e-01 | 20260507 19:55:17.059 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.48e+00, 2.13e-01 / 1.32e-01 | 20260507 19:55:17.059 | INFO | PerfReport: 15.74%, DME -- 4.17e+01 / 4.48e+01, 3.83e+00 / 1.36e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.91e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.42e-01 / 2.34e-01 | 20260507 19:55:17.059 | INFO | PerfReport: 3.63%, Metrics -- 1.20e+00 / 1.42e+00, 3.08e-01 / 1.82e-01 | 20260507 19:55:17.059 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.75e-01 / 4.07e-01, 6.16e-02 / 3.39e-02 | 20260507 19:55:17.059 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:55:17.059 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:55:17.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:55:17.059 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 24.11%, Download -- 2.58e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 30.97%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 2.08%, LockUpload -- 2.22e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 2.36%, LockDownload -- 2.52e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 16.07%, LockCompute -- 1.72e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 19:55:17.059 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.16e-02 / 0.00e+00 | 20260507 19:55:17.154 | INFO | Starting a round of Baz IO | 20260507 19:55:20.638 | INFO | Populating RTMetrics report | 20260507 19:55:20.638 | INFO | RTMetrics report populated | 20260507 19:55:20.638 | INFO | Opening tmp RTMetrics json file | 20260507 19:55:20.663 | INFO | Serializing RTMetrics to json | 20260507 19:55:20.665 | INFO | Serialization done, swapping tmp file now... | 20260507 19:55:20.666 | INFO | RTMetrics done | 20260507 19:55:20.737 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.583 seconds | 20260507 19:55:20.737 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.95% / 0.01% / 55.04% | 20260507 19:55:20.737 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.21135% / 1.98682e-05% / 3.17891e-05% | 20260507 19:55:20.737 | INFO | Baz IO overall: | 20260507 19:55:20.737 | INFO | BytesWritten : 983005066523 | 20260507 19:55:20.737 | INFO | NumEvents : 476587206117 | 20260507 19:55:22.196 | INFO | Analyzing chunk frames = [2170880,2171392) | 20260507 19:55:22.196 | INFO | MemInfo: { | 20260507 19:55:22.196 | INFO | "gpu" : " 17 GB", | 20260507 19:55:22.196 | INFO | "huge" : "", | 20260507 19:55:22.196 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:22.196 | INFO | "malloc" : " 14 GB", | 20260507 19:55:22.196 | INFO | "pinned" : " 29 GB" | 20260507 19:55:22.196 | INFO | } | 20260507 19:55:22.196 | INFO | Perf report for the last 5.13867 seconds: | 20260507 19:55:22.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:22.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:22.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:22.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:22.196 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 19:55:22.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:22.196 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 19:55:22.196 | INFO | PRE_HQ : 66.3%, 1.7, 58.2ms, 0 | 20260507 19:55:22.196 | INFO | BAZWRITER : 69.3%, 4.1, 144.7ms, 0 | 20260507 19:55:22.196 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 19:55:22.196 | INFO | Uploaded 36GiB to GPU in 2.51129s (14.3353GiB/s) | 20260507 19:55:22.196 | INFO | Downloaded 38GiB to GPU in 2.6857s (14.149GiB/s) | 20260507 19:55:27.336 | INFO | Analyzing chunk frames = [2171392,2171904) | 20260507 19:55:27.336 | INFO | MemInfo: { | 20260507 19:55:27.336 | INFO | "gpu" : " 17 GB", | 20260507 19:55:27.336 | INFO | "huge" : "", | 20260507 19:55:27.336 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:27.336 | INFO | "malloc" : " 14 GB", | 20260507 19:55:27.336 | INFO | "pinned" : " 29 GB" | 20260507 19:55:27.336 | INFO | } | 20260507 19:55:27.336 | INFO | Perf report for the last 5.13999 seconds: | 20260507 19:55:27.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:27.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 19:55:27.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 19:55:27.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:27.336 | INFO | Uploaded 36GiB to GPU in 2.4847s (14.4887GiB/s) | 20260507 19:55:27.336 | INFO | Downloaded 34GiB to GPU in 2.47948s (13.7126GiB/s) | 20260507 19:55:32.475 | INFO | Analyzing chunk frames = [2171904,2172416) | 20260507 19:55:32.475 | INFO | MemInfo: { | 20260507 19:55:32.475 | INFO | "gpu" : " 17 GB", | 20260507 19:55:32.475 | INFO | "huge" : "", | 20260507 19:55:32.475 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:32.475 | INFO | "malloc" : " 14 GB", | 20260507 19:55:32.475 | INFO | "pinned" : " 29 GB" | 20260507 19:55:32.475 | INFO | } | 20260507 19:55:32.475 | INFO | Perf report for the last 5.13925 seconds: | 20260507 19:55:32.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:32.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.475 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 19:55:32.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.475 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 19:55:32.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:32.476 | INFO | Uploaded 36GiB to GPU in 2.53477s (14.2025GiB/s) | 20260507 19:55:32.476 | INFO | Downloaded 34GiB to GPU in 2.46249s (13.8071GiB/s) | 20260507 19:55:37.616 | INFO | Analyzing chunk frames = [2172416,2172928) | 20260507 19:55:37.616 | INFO | MemInfo: { | 20260507 19:55:37.616 | INFO | "gpu" : " 17 GB", | 20260507 19:55:37.616 | INFO | "huge" : "", | 20260507 19:55:37.616 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:37.616 | INFO | "malloc" : " 14 GB", | 20260507 19:55:37.616 | INFO | "pinned" : " 29 GB" | 20260507 19:55:37.616 | INFO | } | 20260507 19:55:37.616 | INFO | Perf report for the last 5.14112 seconds: | 20260507 19:55:37.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:37.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.616 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 19:55:37.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.616 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:55:37.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:37.617 | INFO | Uploaded 36GiB to GPU in 2.45864s (14.6422GiB/s) | 20260507 19:55:37.617 | INFO | Downloaded 34GiB to GPU in 2.39618s (14.1893GiB/s) | 20260507 19:55:42.754 | INFO | Analyzing chunk frames = [2172928,2173440) | 20260507 19:55:42.754 | INFO | MemInfo: { | 20260507 19:55:42.754 | INFO | "gpu" : " 17 GB", | 20260507 19:55:42.754 | INFO | "huge" : "", | 20260507 19:55:42.754 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:42.754 | INFO | "malloc" : " 14 GB", | 20260507 19:55:42.754 | INFO | "pinned" : " 29 GB" | 20260507 19:55:42.754 | INFO | } | 20260507 19:55:42.754 | INFO | Perf report for the last 5.13808 seconds: | 20260507 19:55:42.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:42.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.755 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 19:55:42.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.755 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 19:55:42.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:42.755 | INFO | Uploaded 36GiB to GPU in 2.35778s (15.2686GiB/s) | 20260507 19:55:42.755 | INFO | Downloaded 34GiB to GPU in 2.49455s (13.6297GiB/s) | 20260507 19:55:47.893 | INFO | Analyzing chunk frames = [2173440,2173952) | 20260507 19:55:47.893 | INFO | MemInfo: { | 20260507 19:55:47.893 | INFO | "gpu" : " 17 GB", | 20260507 19:55:47.893 | INFO | "huge" : "", | 20260507 19:55:47.893 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:47.893 | INFO | "malloc" : " 14 GB", | 20260507 19:55:47.894 | INFO | "pinned" : " 29 GB" | 20260507 19:55:47.894 | INFO | } | 20260507 19:55:47.894 | INFO | Perf report for the last 5.13904 seconds: | 20260507 19:55:47.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:47.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 19:55:47.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:55:47.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:47.894 | INFO | Uploaded 36GiB to GPU in 2.50778s (14.3553GiB/s) | 20260507 19:55:47.894 | INFO | Downloaded 34GiB to GPU in 2.41305s (14.0901GiB/s) | 20260507 19:55:53.033 | INFO | Analyzing chunk frames = [2173952,2174464) | 20260507 19:55:53.033 | INFO | MemInfo: { | 20260507 19:55:53.033 | INFO | "gpu" : " 17 GB", | 20260507 19:55:53.033 | INFO | "huge" : "", | 20260507 19:55:53.033 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:53.033 | INFO | "malloc" : " 14 GB", | 20260507 19:55:53.033 | INFO | "pinned" : " 29 GB" | 20260507 19:55:53.033 | INFO | } | 20260507 19:55:53.033 | INFO | Perf report for the last 5.13921 seconds: | 20260507 19:55:53.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:53.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:55:53.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:55:53.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:53.033 | INFO | Uploaded 36GiB to GPU in 2.55961s (14.0646GiB/s) | 20260507 19:55:53.033 | INFO | Downloaded 34GiB to GPU in 2.44294s (13.9177GiB/s) | 20260507 19:55:58.172 | INFO | Analyzing chunk frames = [2174464,2174976) | 20260507 19:55:58.172 | INFO | MemInfo: { | 20260507 19:55:58.172 | INFO | "gpu" : " 17 GB", | 20260507 19:55:58.172 | INFO | "huge" : "", | 20260507 19:55:58.172 | INFO | "huge_shared" : " 29 GB", | 20260507 19:55:58.172 | INFO | "malloc" : " 14 GB", | 20260507 19:55:58.172 | INFO | "pinned" : " 29 GB" | 20260507 19:55:58.172 | INFO | } | 20260507 19:55:58.172 | INFO | Perf report for the last 5.13917 seconds: | 20260507 19:55:58.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:55:58.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:55:58.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:55:58.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:55:58.172 | INFO | Uploaded 36GiB to GPU in 2.5168s (14.3039GiB/s) | 20260507 19:55:58.172 | INFO | Downloaded 34GiB to GPU in 2.41572s (14.0745GiB/s) | 20260507 19:55:58.269 | INFO | Starting a round of Baz IO | 20260507 19:56:01.755 | INFO | Populating RTMetrics report | 20260507 19:56:01.755 | INFO | RTMetrics report populated | 20260507 19:56:01.755 | INFO | Opening tmp RTMetrics json file | 20260507 19:56:01.777 | INFO | Serializing RTMetrics to json | 20260507 19:56:01.779 | INFO | Serialization done, swapping tmp file now... | 20260507 19:56:01.780 | INFO | RTMetrics done | 20260507 19:56:01.852 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.582 seconds | 20260507 19:56:01.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.94% / 0.01% / 55.05% | 20260507 19:56:01.852 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.20019% / 1.98682e-05% / 3.57628e-05% | 20260507 19:56:01.852 | INFO | Baz IO overall: | 20260507 19:56:01.852 | INFO | BytesWritten : 984920971972 | 20260507 19:56:01.852 | INFO | NumEvents : 477515315211 | 20260507 19:56:03.311 | INFO | Analyzing chunk frames = [2174976,2175488) | 20260507 19:56:03.311 | INFO | MemInfo: { | 20260507 19:56:03.311 | INFO | "gpu" : " 17 GB", | 20260507 19:56:03.311 | INFO | "huge" : "", | 20260507 19:56:03.311 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:03.311 | INFO | "malloc" : " 14 GB", | 20260507 19:56:03.311 | INFO | "pinned" : " 29 GB" | 20260507 19:56:03.311 | INFO | } | 20260507 19:56:03.311 | INFO | Perf report for the last 5.13952 seconds: | 20260507 19:56:03.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:03.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:03.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:03.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:03.312 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 19:56:03.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:03.312 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 19:56:03.312 | INFO | PRE_HQ : 66.1%, 1.7, 58.2ms, 0 | 20260507 19:56:03.312 | INFO | BAZWRITER : 69.3%, 4.6, 164.0ms, 0 | 20260507 19:56:03.312 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 19:56:03.312 | INFO | Uploaded 36GiB to GPU in 2.55079s (14.1132GiB/s) | 20260507 19:56:03.312 | INFO | Downloaded 38GiB to GPU in 2.69271s (14.1122GiB/s) | 20260507 19:56:08.451 | INFO | Analyzing chunk frames = [2175488,2176000) | 20260507 19:56:08.451 | INFO | MemInfo: { | 20260507 19:56:08.451 | INFO | "gpu" : " 17 GB", | 20260507 19:56:08.451 | INFO | "huge" : "", | 20260507 19:56:08.451 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:08.451 | INFO | "malloc" : " 14 GB", | 20260507 19:56:08.451 | INFO | "pinned" : " 29 GB" | 20260507 19:56:08.451 | INFO | } | 20260507 19:56:08.451 | INFO | Perf report for the last 5.1392 seconds: | 20260507 19:56:08.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:08.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 19:56:08.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 19:56:08.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:08.451 | INFO | Uploaded 36GiB to GPU in 2.58659s (13.918GiB/s) | 20260507 19:56:08.451 | INFO | Downloaded 34GiB to GPU in 2.48162s (13.7007GiB/s) | 20260507 19:56:13.590 | INFO | Analyzing chunk frames = [2176000,2176512) | 20260507 19:56:13.590 | INFO | MemInfo: { | 20260507 19:56:13.590 | INFO | "gpu" : " 17 GB", | 20260507 19:56:13.590 | INFO | "huge" : "", | 20260507 19:56:13.590 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:13.590 | INFO | "malloc" : " 14 GB", | 20260507 19:56:13.590 | INFO | "pinned" : " 29 GB" | 20260507 19:56:13.590 | INFO | } | 20260507 19:56:13.590 | INFO | Perf report for the last 5.13972 seconds: | 20260507 19:56:13.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:13.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.590 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260507 19:56:13.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.591 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:56:13.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:13.591 | INFO | Uploaded 36GiB to GPU in 2.59622s (13.8663GiB/s) | 20260507 19:56:13.591 | INFO | Downloaded 34GiB to GPU in 2.46052s (13.8182GiB/s) | 20260507 19:56:18.730 | INFO | Analyzing chunk frames = [2176512,2177024) | 20260507 19:56:18.730 | INFO | MemInfo: { | 20260507 19:56:18.730 | INFO | "gpu" : " 17 GB", | 20260507 19:56:18.730 | INFO | "huge" : "", | 20260507 19:56:18.730 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:18.730 | INFO | "malloc" : " 14 GB", | 20260507 19:56:18.730 | INFO | "pinned" : " 29 GB" | 20260507 19:56:18.730 | INFO | } | 20260507 19:56:18.731 | INFO | Perf report for the last 5.14008 seconds: | 20260507 19:56:18.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:18.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 19:56:18.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:56:18.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:18.731 | INFO | Uploaded 36GiB to GPU in 2.49159s (14.4486GiB/s) | 20260507 19:56:18.731 | INFO | Downloaded 34GiB to GPU in 2.39525s (14.1947GiB/s) | 20260507 19:56:23.870 | INFO | Analyzing chunk frames = [2177024,2177536) | 20260507 19:56:23.870 | INFO | MemInfo: { | 20260507 19:56:23.870 | INFO | "gpu" : " 17 GB", | 20260507 19:56:23.870 | INFO | "huge" : "", | 20260507 19:56:23.870 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:23.870 | INFO | "malloc" : " 14 GB", | 20260507 19:56:23.870 | INFO | "pinned" : " 29 GB" | 20260507 19:56:23.870 | INFO | } | 20260507 19:56:23.870 | INFO | Perf report for the last 5.13974 seconds: | 20260507 19:56:23.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:23.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.870 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:56:23.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.870 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 19:56:23.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:23.871 | INFO | Uploaded 36GiB to GPU in 2.39544s (15.0286GiB/s) | 20260507 19:56:23.871 | INFO | Downloaded 34GiB to GPU in 2.5015s (13.5919GiB/s) | 20260507 19:56:29.008 | INFO | Analyzing chunk frames = [2177536,2178048) | 20260507 19:56:29.008 | INFO | MemInfo: { | 20260507 19:56:29.008 | INFO | "gpu" : " 17 GB", | 20260507 19:56:29.008 | INFO | "huge" : "", | 20260507 19:56:29.008 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:29.008 | INFO | "malloc" : " 14 GB", | 20260507 19:56:29.008 | INFO | "pinned" : " 29 GB" | 20260507 19:56:29.008 | INFO | } | 20260507 19:56:29.008 | INFO | Perf report for the last 5.13814 seconds: | 20260507 19:56:29.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:29.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.008 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 19:56:29.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.008 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:56:29.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:29.009 | INFO | Uploaded 36GiB to GPU in 2.51363s (14.3219GiB/s) | 20260507 19:56:29.009 | INFO | Downloaded 34GiB to GPU in 2.41323s (14.089GiB/s) | 20260507 19:56:34.148 | INFO | Analyzing chunk frames = [2178048,2178560) | 20260507 19:56:34.148 | INFO | MemInfo: { | 20260507 19:56:34.148 | INFO | "gpu" : " 17 GB", | 20260507 19:56:34.148 | INFO | "huge" : "", | 20260507 19:56:34.148 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:34.148 | INFO | "malloc" : " 14 GB", | 20260507 19:56:34.148 | INFO | "pinned" : " 29 GB" | 20260507 19:56:34.148 | INFO | } | 20260507 19:56:34.148 | INFO | Perf report for the last 5.13928 seconds: | 20260507 19:56:34.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:34.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 19:56:34.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:56:34.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:34.148 | INFO | Uploaded 36GiB to GPU in 2.55662s (14.0811GiB/s) | 20260507 19:56:34.148 | INFO | Downloaded 34GiB to GPU in 2.44445s (13.9091GiB/s) | 20260507 19:56:39.286 | INFO | Analyzing chunk frames = [2178560,2179072) | 20260507 19:56:39.286 | INFO | MemInfo: { | 20260507 19:56:39.286 | INFO | "gpu" : " 17 GB", | 20260507 19:56:39.287 | INFO | "huge" : "", | 20260507 19:56:39.287 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:39.287 | INFO | "malloc" : " 14 GB", | 20260507 19:56:39.287 | INFO | "pinned" : " 29 GB" | 20260507 19:56:39.287 | INFO | } | 20260507 19:56:39.287 | INFO | Perf report for the last 5.13902 seconds: | 20260507 19:56:39.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:39.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:56:39.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:56:39.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:39.287 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:56:39.287 | INFO | Total mem usage: 16.4 GiB | 20260507 19:56:39.287 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:56:39.287 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -4.14%) | 20260507 19:56:39.287 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:56:39.287 | INFO | High Water Breakdown: | 20260507 19:56:39.287 | INFO | 60.8% : 9.58 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:56:39.287 | INFO | 39.1% : 6.16 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:56:39.287 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:56:39.287 | INFO | ------------------------------------------ | 20260507 19:56:39.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:56:39.287 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:56:39.287 | INFO | Total mem usage: 26.5 GiB | 20260507 19:56:39.287 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:56:39.287 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 19:56:39.287 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 19:56:39.287 | INFO | High Water Breakdown: | 20260507 19:56:39.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:56:39.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:56:39.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:56:39.287 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:56:39.287 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:56:39.287 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:56:39.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:56:39.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:56:39.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:56:39.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:56:39.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:56:39.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:56:39.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:56:39.287 | INFO | Uploaded 36GiB to GPU in 2.51193s (14.3316GiB/s) | 20260507 19:56:39.287 | INFO | Downloaded 34GiB to GPU in 2.41051s (14.1049GiB/s) | 20260507 19:56:39.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:56:39.288 | INFO | ------------------------------------------ | 20260507 19:56:39.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:56:39.288 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:56:39.288 | INFO | Total mem usage: 27.8 GiB | 20260507 19:56:39.288 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:56:39.288 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 19:56:39.288 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 19:56:39.288 | INFO | High Water Breakdown: | 20260507 19:56:39.288 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 19:56:39.288 | INFO | ------------------------------------------ | 20260507 19:56:39.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:56:39.288 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:56:39.288 | INFO | Total mem usage: 15.7 GiB | 20260507 19:56:39.288 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:56:39.288 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:56:39.288 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:56:39.288 | INFO | High Water Breakdown: | 20260507 19:56:39.288 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:56:39.288 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:56:39.288 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:56:39.288 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:56:39.288 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:56:39.288 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:56:39.288 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:56:39.288 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:56:39.288 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:56:39.288 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:56:39.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:56:39.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:56:39.288 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:56:39.288 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:56:39.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:56:39.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:56:39.288 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:56:39.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:56:39.288 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:56:39.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:56:39.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:56:39.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:56:39.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:56:39.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:56:39.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:56:39.288 | INFO | ------------------------------------------ | 20260507 19:56:39.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:56:39.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:56:39.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:56:39.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:56:39.288 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.56e+01, 1.37e+01 / 1.83e+00 | 20260507 19:56:39.288 | INFO | PerfReport: 0.29%, AggregateStats -- 9.73e-02 / 1.02e-01, 1.25e-02 / 4.72e-03 | 20260507 19:56:39.288 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.46e+00, 6.01e-01 / 2.90e-01 | 20260507 19:56:39.288 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.04e-01 | 20260507 19:56:39.288 | INFO | PerfReport: 15.76%, DME -- 4.17e+01 / 4.49e+01, 3.76e+00 / 1.26e+00 | 20260507 19:56:39.288 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.96e+00 | 20260507 19:56:39.288 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.48e+00 / 2.80e+00, 4.37e-01 / 2.48e-01 | 20260507 19:56:39.288 | INFO | PerfReport: 3.62%, Metrics -- 1.20e+00 / 1.39e+00, 3.04e-01 / 1.84e-01 | 20260507 19:56:39.288 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.73e-01 / 4.02e-01, 6.08e-02 / 3.16e-02 | 20260507 19:56:39.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:56:39.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:56:39.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:56:39.288 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 19:56:39.288 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 19:56:39.288 | INFO | PerfReport: 24.09%, Download -- 2.58e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 19:56:39.289 | INFO | PerfReport: 30.94%, Compute -- 3.31e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260507 19:56:39.289 | INFO | PerfReport: 2.16%, LockUpload -- 2.31e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 19:56:39.289 | INFO | PerfReport: 2.30%, LockDownload -- 2.47e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 19:56:39.289 | INFO | PerfReport: 16.06%, LockCompute -- 1.72e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 19:56:39.289 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260507 19:56:39.402 | INFO | Starting a round of Baz IO | 20260507 19:56:42.909 | INFO | Populating RTMetrics report | 20260507 19:56:42.909 | INFO | RTMetrics report populated | 20260507 19:56:42.909 | INFO | Opening tmp RTMetrics json file | 20260507 19:56:42.965 | INFO | Serializing RTMetrics to json | 20260507 19:56:42.967 | INFO | Serialization done, swapping tmp file now... | 20260507 19:56:42.968 | INFO | RTMetrics done | 20260507 19:56:43.103 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.7 seconds | 20260507 19:56:43.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.93% / 0.01% / 55.06% | 20260507 19:56:43.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.19605% / 1.58946e-05% / 2.38419e-05% | 20260507 19:56:43.103 | INFO | Baz IO overall: | 20260507 19:56:43.103 | INFO | BytesWritten : 986836972657 | 20260507 19:56:43.103 | INFO | NumEvents : 478443598577 | 20260507 19:56:44.427 | INFO | Analyzing chunk frames = [2179072,2179584) | 20260507 19:56:44.427 | INFO | MemInfo: { | 20260507 19:56:44.427 | INFO | "gpu" : " 17 GB", | 20260507 19:56:44.427 | INFO | "huge" : "", | 20260507 19:56:44.427 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:44.427 | INFO | "malloc" : " 14 GB", | 20260507 19:56:44.427 | INFO | "pinned" : " 29 GB" | 20260507 19:56:44.427 | INFO | } | 20260507 19:56:44.427 | INFO | Perf report for the last 5.13993 seconds: | 20260507 19:56:44.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:44.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:44.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:44.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:44.427 | INFO | ANALYSIS : 69.9%, 2.9, 105.0ms, 0 | 20260507 19:56:44.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:44.427 | INFO | RT_METRICS : 50.7%, 1.0, 26.6ms, 0 | 20260507 19:56:44.427 | INFO | PRE_HQ : 69.6%, 1.9, 70.3ms, 0 | 20260507 19:56:44.427 | INFO | BAZWRITER : 71.7%, 4.3, 159.7ms, 0 | 20260507 19:56:44.427 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 19:56:44.427 | INFO | Uploaded 36GiB to GPU in 2.53989s (14.1738GiB/s) | 20260507 19:56:44.427 | INFO | Downloaded 38GiB to GPU in 2.70143s (14.0666GiB/s) | 20260507 19:56:49.566 | INFO | Analyzing chunk frames = [2179584,2180096) | 20260507 19:56:49.566 | INFO | MemInfo: { | 20260507 19:56:49.566 | INFO | "gpu" : " 17 GB", | 20260507 19:56:49.566 | INFO | "huge" : "", | 20260507 19:56:49.566 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:49.566 | INFO | "malloc" : " 14 GB", | 20260507 19:56:49.566 | INFO | "pinned" : " 29 GB" | 20260507 19:56:49.566 | INFO | } | 20260507 19:56:49.566 | INFO | Perf report for the last 5.13909 seconds: | 20260507 19:56:49.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:49.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 19:56:49.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:56:49.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:49.566 | INFO | Uploaded 36GiB to GPU in 2.55525s (14.0887GiB/s) | 20260507 19:56:49.566 | INFO | Downloaded 34GiB to GPU in 2.49526s (13.6258GiB/s) | 20260507 19:56:54.705 | INFO | Analyzing chunk frames = [2180096,2180608) | 20260507 19:56:54.705 | INFO | MemInfo: { | 20260507 19:56:54.705 | INFO | "gpu" : " 17 GB", | 20260507 19:56:54.705 | INFO | "huge" : "", | 20260507 19:56:54.705 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:54.705 | INFO | "malloc" : " 14 GB", | 20260507 19:56:54.705 | INFO | "pinned" : " 29 GB" | 20260507 19:56:54.705 | INFO | } | 20260507 19:56:54.705 | INFO | Perf report for the last 5.13944 seconds: | 20260507 19:56:54.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:54.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 19:56:54.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 19:56:54.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:54.705 | INFO | Uploaded 36GiB to GPU in 2.55143s (14.1097GiB/s) | 20260507 19:56:54.706 | INFO | Downloaded 34GiB to GPU in 2.4672s (13.7808GiB/s) | 20260507 19:56:59.844 | INFO | Analyzing chunk frames = [2180608,2181120) | 20260507 19:56:59.844 | INFO | MemInfo: { | 20260507 19:56:59.844 | INFO | "gpu" : " 17 GB", | 20260507 19:56:59.844 | INFO | "huge" : "", | 20260507 19:56:59.844 | INFO | "huge_shared" : " 29 GB", | 20260507 19:56:59.844 | INFO | "malloc" : " 14 GB", | 20260507 19:56:59.844 | INFO | "pinned" : " 29 GB" | 20260507 19:56:59.844 | INFO | } | 20260507 19:56:59.845 | INFO | Perf report for the last 5.13925 seconds: | 20260507 19:56:59.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:56:59.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 19:56:59.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:56:59.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:56:59.845 | INFO | Uploaded 36GiB to GPU in 2.45959s (14.6366GiB/s) | 20260507 19:56:59.845 | INFO | Downloaded 34GiB to GPU in 2.40273s (14.1505GiB/s) | 20260507 19:57:04.983 | INFO | Analyzing chunk frames = [2181120,2181632) | 20260507 19:57:04.983 | INFO | MemInfo: { | 20260507 19:57:04.983 | INFO | "gpu" : " 17 GB", | 20260507 19:57:04.983 | INFO | "huge" : "", | 20260507 19:57:04.983 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:04.983 | INFO | "malloc" : " 14 GB", | 20260507 19:57:04.983 | INFO | "pinned" : " 29 GB" | 20260507 19:57:04.984 | INFO | } | 20260507 19:57:04.984 | INFO | Perf report for the last 5.13905 seconds: | 20260507 19:57:04.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:04.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:57:04.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 19:57:04.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:04.984 | INFO | Uploaded 36GiB to GPU in 2.35387s (15.294GiB/s) | 20260507 19:57:04.984 | INFO | Downloaded 34GiB to GPU in 2.50956s (13.5482GiB/s) | 20260507 19:57:10.122 | INFO | Analyzing chunk frames = [2181632,2182144) | 20260507 19:57:10.122 | INFO | MemInfo: { | 20260507 19:57:10.122 | INFO | "gpu" : " 17 GB", | 20260507 19:57:10.122 | INFO | "huge" : "", | 20260507 19:57:10.122 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:10.122 | INFO | "malloc" : " 14 GB", | 20260507 19:57:10.122 | INFO | "pinned" : " 29 GB" | 20260507 19:57:10.122 | INFO | } | 20260507 19:57:10.123 | INFO | Perf report for the last 5.13916 seconds: | 20260507 19:57:10.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:10.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 19:57:10.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 19:57:10.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:10.123 | INFO | Uploaded 36GiB to GPU in 2.48701s (14.4752GiB/s) | 20260507 19:57:10.123 | INFO | Downloaded 34GiB to GPU in 2.42484s (14.0216GiB/s) | 20260507 19:57:15.262 | INFO | Analyzing chunk frames = [2182144,2182656) | 20260507 19:57:15.262 | INFO | MemInfo: { | 20260507 19:57:15.262 | INFO | "gpu" : " 17 GB", | 20260507 19:57:15.262 | INFO | "huge" : "", | 20260507 19:57:15.262 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:15.262 | INFO | "malloc" : " 14 GB", | 20260507 19:57:15.262 | INFO | "pinned" : " 29 GB" | 20260507 19:57:15.262 | INFO | } | 20260507 19:57:15.262 | INFO | Perf report for the last 5.13915 seconds: | 20260507 19:57:15.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:15.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:57:15.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 19:57:15.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:15.262 | INFO | Uploaded 36GiB to GPU in 2.51796s (14.2973GiB/s) | 20260507 19:57:15.262 | INFO | Downloaded 34GiB to GPU in 2.45467s (13.8512GiB/s) | 20260507 19:57:20.402 | INFO | Analyzing chunk frames = [2182656,2183168) | 20260507 19:57:20.402 | INFO | MemInfo: { | 20260507 19:57:20.402 | INFO | "gpu" : " 17 GB", | 20260507 19:57:20.402 | INFO | "huge" : "", | 20260507 19:57:20.402 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:20.402 | INFO | "malloc" : " 14 GB", | 20260507 19:57:20.402 | INFO | "pinned" : " 29 GB" | 20260507 19:57:20.402 | INFO | } | 20260507 19:57:20.402 | INFO | Perf report for the last 5.14018 seconds: | 20260507 19:57:20.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:20.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.402 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 19:57:20.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.402 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 19:57:20.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:20.403 | INFO | Uploaded 36GiB to GPU in 2.48856s (14.4662GiB/s) | 20260507 19:57:20.403 | INFO | Downloaded 34GiB to GPU in 2.43231s (13.9785GiB/s) | 20260507 19:57:20.500 | INFO | Starting a round of Baz IO | 20260507 19:57:23.985 | INFO | Populating RTMetrics report | 20260507 19:57:23.986 | INFO | RTMetrics report populated | 20260507 19:57:23.986 | INFO | Opening tmp RTMetrics json file | 20260507 19:57:24.011 | INFO | Serializing RTMetrics to json | 20260507 19:57:24.013 | INFO | Serialization done, swapping tmp file now... | 20260507 19:57:24.014 | INFO | RTMetrics done | 20260507 19:57:24.100 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.6 seconds | 20260507 19:57:24.100 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.91% / 0.01% / 55.07% | 20260507 19:57:24.100 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.19618% / 1.19209e-05% / 3.17891e-05% | 20260507 19:57:24.100 | INFO | Baz IO overall: | 20260507 19:57:24.100 | INFO | BytesWritten : 988753265655 | 20260507 19:57:24.100 | INFO | NumEvents : 479372238697 | 20260507 19:57:25.542 | INFO | Analyzing chunk frames = [2183168,2183680) | 20260507 19:57:25.542 | INFO | MemInfo: { | 20260507 19:57:25.542 | INFO | "gpu" : " 17 GB", | 20260507 19:57:25.542 | INFO | "huge" : "", | 20260507 19:57:25.542 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:25.542 | INFO | "malloc" : " 14 GB", | 20260507 19:57:25.542 | INFO | "pinned" : " 29 GB" | 20260507 19:57:25.542 | INFO | } | 20260507 19:57:25.542 | INFO | Perf report for the last 5.13992 seconds: | 20260507 19:57:25.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:25.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:25.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:25.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:25.542 | INFO | ANALYSIS : 69.0%, 3.0, 107.0ms, 0 | 20260507 19:57:25.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:25.542 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 19:57:25.542 | INFO | PRE_HQ : 66.2%, 1.7, 58.2ms, 0 | 20260507 19:57:25.542 | INFO | BAZWRITER : 69.6%, 3.9, 140.3ms, 0 | 20260507 19:57:25.542 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 19:57:25.542 | INFO | Uploaded 36GiB to GPU in 2.54932s (14.1214GiB/s) | 20260507 19:57:25.542 | INFO | Downloaded 38GiB to GPU in 2.70618s (14.0419GiB/s) | 20260507 19:57:30.683 | INFO | Analyzing chunk frames = [2183680,2184192) | 20260507 19:57:30.683 | INFO | MemInfo: { | 20260507 19:57:30.683 | INFO | "gpu" : " 17 GB", | 20260507 19:57:30.683 | INFO | "huge" : "", | 20260507 19:57:30.683 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:30.683 | INFO | "malloc" : " 14 GB", | 20260507 19:57:30.683 | INFO | "pinned" : " 29 GB" | 20260507 19:57:30.683 | INFO | } | 20260507 19:57:30.683 | INFO | Perf report for the last 5.14097 seconds: | 20260507 19:57:30.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:30.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 19:57:30.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 19:57:30.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:30.683 | INFO | Uploaded 36GiB to GPU in 2.59005s (13.8993GiB/s) | 20260507 19:57:30.683 | INFO | Downloaded 34GiB to GPU in 2.48639s (13.6744GiB/s) | 20260507 19:57:35.820 | INFO | Analyzing chunk frames = [2184192,2184704) | 20260507 19:57:35.820 | INFO | MemInfo: { | 20260507 19:57:35.820 | INFO | "gpu" : " 17 GB", | 20260507 19:57:35.820 | INFO | "huge" : "", | 20260507 19:57:35.820 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:35.820 | INFO | "malloc" : " 14 GB", | 20260507 19:57:35.820 | INFO | "pinned" : " 29 GB" | 20260507 19:57:35.820 | INFO | } | 20260507 19:57:35.820 | INFO | Perf report for the last 5.13707 seconds: | 20260507 19:57:35.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:35.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 19:57:35.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:57:35.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:35.820 | INFO | Uploaded 36GiB to GPU in 2.60583s (13.8152GiB/s) | 20260507 19:57:35.820 | INFO | Downloaded 34GiB to GPU in 2.4634s (13.802GiB/s) | 20260507 19:57:40.959 | INFO | Analyzing chunk frames = [2184704,2185216) | 20260507 19:57:40.959 | INFO | MemInfo: { | 20260507 19:57:40.959 | INFO | "gpu" : " 17 GB", | 20260507 19:57:40.959 | INFO | "huge" : "", | 20260507 19:57:40.959 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:40.959 | INFO | "malloc" : " 14 GB", | 20260507 19:57:40.959 | INFO | "pinned" : " 29 GB" | 20260507 19:57:40.959 | INFO | } | 20260507 19:57:40.959 | INFO | Perf report for the last 5.13911 seconds: | 20260507 19:57:40.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:40.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 19:57:40.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 19:57:40.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:40.959 | INFO | Uploaded 36GiB to GPU in 2.47879s (14.5232GiB/s) | 20260507 19:57:40.959 | INFO | Downloaded 34GiB to GPU in 2.39095s (14.2203GiB/s) | 20260507 19:57:46.099 | INFO | Analyzing chunk frames = [2185216,2185728) | 20260507 19:57:46.099 | INFO | MemInfo: { | 20260507 19:57:46.099 | INFO | "gpu" : " 17 GB", | 20260507 19:57:46.099 | INFO | "huge" : "", | 20260507 19:57:46.099 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:46.099 | INFO | "malloc" : " 14 GB", | 20260507 19:57:46.099 | INFO | "pinned" : " 29 GB" | 20260507 19:57:46.099 | INFO | } | 20260507 19:57:46.099 | INFO | Perf report for the last 5.13947 seconds: | 20260507 19:57:46.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:46.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 19:57:46.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 19:57:46.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:46.099 | INFO | Uploaded 36GiB to GPU in 2.36392s (15.229GiB/s) | 20260507 19:57:46.099 | INFO | Downloaded 34GiB to GPU in 2.4879s (13.6661GiB/s) | 20260507 19:57:51.238 | INFO | Analyzing chunk frames = [2185728,2186240) | 20260507 19:57:51.238 | INFO | MemInfo: { | 20260507 19:57:51.238 | INFO | "gpu" : " 17 GB", | 20260507 19:57:51.238 | INFO | "huge" : "", | 20260507 19:57:51.238 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:51.238 | INFO | "malloc" : " 14 GB", | 20260507 19:57:51.238 | INFO | "pinned" : " 29 GB" | 20260507 19:57:51.238 | INFO | } | 20260507 19:57:51.238 | INFO | Perf report for the last 5.13904 seconds: | 20260507 19:57:51.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:51.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 19:57:51.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:57:51.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:51.238 | INFO | Uploaded 36GiB to GPU in 2.51814s (14.2963GiB/s) | 20260507 19:57:51.238 | INFO | Downloaded 34GiB to GPU in 2.4097s (14.1096GiB/s) | 20260507 19:57:56.378 | INFO | Analyzing chunk frames = [2186240,2186752) | 20260507 19:57:56.378 | INFO | MemInfo: { | 20260507 19:57:56.378 | INFO | "gpu" : " 17 GB", | 20260507 19:57:56.378 | INFO | "huge" : "", | 20260507 19:57:56.378 | INFO | "huge_shared" : " 29 GB", | 20260507 19:57:56.378 | INFO | "malloc" : " 14 GB", | 20260507 19:57:56.378 | INFO | "pinned" : " 29 GB" | 20260507 19:57:56.378 | INFO | } | 20260507 19:57:56.378 | INFO | Perf report for the last 5.14021 seconds: | 20260507 19:57:56.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:57:56.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 19:57:56.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260507 19:57:56.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:57:56.378 | INFO | Uploaded 36GiB to GPU in 2.56853s (14.0158GiB/s) | 20260507 19:57:56.378 | INFO | Downloaded 34GiB to GPU in 2.43904s (13.9399GiB/s) | 20260507 19:58:01.516 | INFO | Analyzing chunk frames = [2186752,2187264) | 20260507 19:58:01.516 | INFO | MemInfo: { | 20260507 19:58:01.516 | INFO | "gpu" : " 17 GB", | 20260507 19:58:01.516 | INFO | "huge" : "", | 20260507 19:58:01.516 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:01.516 | INFO | "malloc" : " 14 GB", | 20260507 19:58:01.516 | INFO | "pinned" : " 29 GB" | 20260507 19:58:01.516 | INFO | } | 20260507 19:58:01.516 | INFO | Perf report for the last 5.13831 seconds: | 20260507 19:58:01.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:01.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.516 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 19:58:01.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.516 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:58:01.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:01.517 | INFO | Uploaded 36GiB to GPU in 2.53525s (14.1998GiB/s) | 20260507 19:58:01.517 | INFO | Downloaded 34GiB to GPU in 2.42404s (14.0262GiB/s) | 20260507 19:58:01.517 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:58:01.517 | INFO | Total mem usage: 16.4 GiB | 20260507 19:58:01.517 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:58:01.517 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.3%) | 20260507 19:58:01.517 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:58:01.517 | INFO | High Water Breakdown: | 20260507 19:58:01.517 | INFO | 59.8% : 8.9 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:58:01.517 | INFO | 40% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:58:01.517 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:58:01.517 | INFO | ------------------------------------------ | 20260507 19:58:01.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:58:01.517 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:58:01.517 | INFO | Total mem usage: 26.5 GiB | 20260507 19:58:01.517 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:58:01.517 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 19:58:01.517 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 19:58:01.517 | INFO | High Water Breakdown: | 20260507 19:58:01.517 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:58:01.517 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:58:01.517 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:58:01.517 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:58:01.517 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:58:01.517 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:58:01.517 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:58:01.517 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:58:01.517 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:58:01.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:58:01.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:58:01.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:58:01.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:58:01.517 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:58:01.517 | INFO | ------------------------------------------ | 20260507 19:58:01.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:58:01.517 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:58:01.517 | INFO | Total mem usage: 27.8 GiB | 20260507 19:58:01.517 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:58:01.517 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:58:01.517 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:58:01.517 | INFO | High Water Breakdown: | 20260507 19:58:01.517 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:58:01.517 | INFO | ------------------------------------------ | 20260507 19:58:01.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:58:01.517 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:58:01.517 | INFO | Total mem usage: 15.7 GiB | 20260507 19:58:01.517 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:58:01.517 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:58:01.517 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 19:58:01.517 | INFO | High Water Breakdown: | 20260507 19:58:01.517 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:58:01.517 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:58:01.517 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:58:01.517 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:58:01.517 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:58:01.517 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:58:01.517 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:58:01.518 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:58:01.518 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:58:01.518 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:58:01.518 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:58:01.518 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:58:01.518 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:58:01.518 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:58:01.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:58:01.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:58:01.518 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:58:01.518 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:58:01.518 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:58:01.518 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:58:01.518 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:58:01.518 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:58:01.518 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:58:01.518 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:58:01.518 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:58:01.518 | INFO | ------------------------------------------ | 20260507 19:58:01.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:58:01.518 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:58:01.518 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:58:01.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:58:01.518 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.39e+01 / 1.71e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.04e-01, 1.14e-02 / 6.79e-03 | 20260507 19:58:01.518 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.47e+00, 6.04e-01 / 2.78e-01 | 20260507 19:58:01.518 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.15e-01 | 20260507 19:58:01.518 | INFO | PerfReport: 15.75%, DME -- 4.17e+01 / 4.49e+01, 3.83e+00 / 1.28e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.84e+00 / 1.97e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.80e+00, 4.41e-01 / 2.69e-01 | 20260507 19:58:01.518 | INFO | PerfReport: 3.62%, Metrics -- 1.20e+00 / 1.40e+00, 3.04e-01 / 1.80e-01 | 20260507 19:58:01.518 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.75e-01 / 4.09e-01, 6.13e-02 / 3.45e-02 | 20260507 19:58:01.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:58:01.518 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:58:01.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:58:01.518 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 24.18%, Download -- 2.58e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 2.23%, LockDownload -- 2.38e+00 / 0.00e+00, 3.68e+00 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 15.95%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 19:58:01.518 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260507 19:58:01.626 | INFO | Starting a round of Baz IO | 20260507 19:58:05.096 | INFO | Populating RTMetrics report | 20260507 19:58:05.097 | INFO | RTMetrics report populated | 20260507 19:58:05.097 | INFO | Opening tmp RTMetrics json file | 20260507 19:58:05.166 | INFO | Serializing RTMetrics to json | 20260507 19:58:05.230 | INFO | Serialization done, swapping tmp file now... | 20260507 19:58:05.257 | INFO | RTMetrics done | 20260507 19:58:05.283 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.657 seconds | 20260507 19:58:05.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.90% / 0.01% / 55.09% | 20260507 19:58:05.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.18856% / 1.98682e-05% / 3.17891e-05% | 20260507 19:58:05.283 | INFO | Baz IO overall: | 20260507 19:58:05.283 | INFO | BytesWritten : 990669149438 | 20260507 19:58:05.283 | INFO | NumEvents : 480300383754 | 20260507 19:58:06.655 | INFO | Analyzing chunk frames = [2187264,2187776) | 20260507 19:58:06.655 | INFO | MemInfo: { | 20260507 19:58:06.655 | INFO | "gpu" : " 17 GB", | 20260507 19:58:06.655 | INFO | "huge" : "", | 20260507 19:58:06.655 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:06.655 | INFO | "malloc" : " 14 GB", | 20260507 19:58:06.655 | INFO | "pinned" : " 29 GB" | 20260507 19:58:06.655 | INFO | } | 20260507 19:58:06.655 | INFO | Perf report for the last 5.1389 seconds: | 20260507 19:58:06.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:06.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:06.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:06.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:06.655 | INFO | ANALYSIS : 68.9%, 3.0, 107.0ms, 0 | 20260507 19:58:06.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:06.655 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 19:58:06.655 | INFO | PRE_HQ : 66.4%, 1.7, 58.8ms, 0 | 20260507 19:58:06.655 | INFO | BAZWRITER : 70.7%, 4.3, 158.0ms, 0 | 20260507 19:58:06.655 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 19:58:06.655 | INFO | Uploaded 36GiB to GPU in 2.519s (14.2914GiB/s) | 20260507 19:58:06.655 | INFO | Downloaded 38GiB to GPU in 2.69171s (14.1174GiB/s) | 20260507 19:58:11.796 | INFO | Analyzing chunk frames = [2187776,2188288) | 20260507 19:58:11.796 | INFO | MemInfo: { | 20260507 19:58:11.796 | INFO | "gpu" : " 17 GB", | 20260507 19:58:11.796 | INFO | "huge" : "", | 20260507 19:58:11.796 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:11.796 | INFO | "malloc" : " 14 GB", | 20260507 19:58:11.796 | INFO | "pinned" : " 29 GB" | 20260507 19:58:11.796 | INFO | } | 20260507 19:58:11.796 | INFO | Perf report for the last 5.14084 seconds: | 20260507 19:58:11.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:11.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:58:11.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:58:11.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:11.796 | INFO | Uploaded 36GiB to GPU in 2.53011s (14.2286GiB/s) | 20260507 19:58:11.796 | INFO | Downloaded 34GiB to GPU in 2.48019s (13.7086GiB/s) | 20260507 19:58:16.935 | INFO | Analyzing chunk frames = [2188288,2188800) | 20260507 19:58:16.935 | INFO | MemInfo: { | 20260507 19:58:16.935 | INFO | "gpu" : " 17 GB", | 20260507 19:58:16.935 | INFO | "huge" : "", | 20260507 19:58:16.935 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:16.935 | INFO | "malloc" : " 14 GB", | 20260507 19:58:16.935 | INFO | "pinned" : " 29 GB" | 20260507 19:58:16.935 | INFO | } | 20260507 19:58:16.935 | INFO | Perf report for the last 5.1388 seconds: | 20260507 19:58:16.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:16.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 19:58:16.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 19:58:16.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:16.935 | INFO | Uploaded 36GiB to GPU in 2.56738s (14.0221GiB/s) | 20260507 19:58:16.935 | INFO | Downloaded 34GiB to GPU in 2.47727s (13.7248GiB/s) | 20260507 19:58:22.074 | INFO | Analyzing chunk frames = [2188800,2189312) | 20260507 19:58:22.074 | INFO | MemInfo: { | 20260507 19:58:22.074 | INFO | "gpu" : " 17 GB", | 20260507 19:58:22.074 | INFO | "huge" : "", | 20260507 19:58:22.074 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:22.074 | INFO | "malloc" : " 14 GB", | 20260507 19:58:22.074 | INFO | "pinned" : " 29 GB" | 20260507 19:58:22.075 | INFO | } | 20260507 19:58:22.075 | INFO | Perf report for the last 5.13956 seconds: | 20260507 19:58:22.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:22.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 19:58:22.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 19:58:22.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:22.075 | INFO | Uploaded 36GiB to GPU in 2.45728s (14.6503GiB/s) | 20260507 19:58:22.075 | INFO | Downloaded 34GiB to GPU in 2.39422s (14.2009GiB/s) | 20260507 19:58:27.214 | INFO | Analyzing chunk frames = [2189312,2189824) | 20260507 19:58:27.214 | INFO | MemInfo: { | 20260507 19:58:27.214 | INFO | "gpu" : " 17 GB", | 20260507 19:58:27.214 | INFO | "huge" : "", | 20260507 19:58:27.214 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:27.214 | INFO | "malloc" : " 14 GB", | 20260507 19:58:27.214 | INFO | "pinned" : " 29 GB" | 20260507 19:58:27.214 | INFO | } | 20260507 19:58:27.214 | INFO | Perf report for the last 5.13926 seconds: | 20260507 19:58:27.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:27.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:58:27.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 19:58:27.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:27.214 | INFO | Uploaded 36GiB to GPU in 2.32524s (15.4823GiB/s) | 20260507 19:58:27.214 | INFO | Downloaded 34GiB to GPU in 2.50966s (13.5476GiB/s) | 20260507 19:58:32.354 | INFO | Analyzing chunk frames = [2189824,2190336) | 20260507 19:58:32.354 | INFO | MemInfo: { | 20260507 19:58:32.354 | INFO | "gpu" : " 17 GB", | 20260507 19:58:32.354 | INFO | "huge" : "", | 20260507 19:58:32.354 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:32.354 | INFO | "malloc" : " 14 GB", | 20260507 19:58:32.354 | INFO | "pinned" : " 29 GB" | 20260507 19:58:32.354 | INFO | } | 20260507 19:58:32.354 | INFO | Perf report for the last 5.14019 seconds: | 20260507 19:58:32.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:32.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:58:32.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 19:58:32.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:32.354 | INFO | Uploaded 36GiB to GPU in 2.47027s (14.5733GiB/s) | 20260507 19:58:32.354 | INFO | Downloaded 34GiB to GPU in 2.43769s (13.9476GiB/s) | 20260507 19:58:37.493 | INFO | Analyzing chunk frames = [2190336,2190848) | 20260507 19:58:37.493 | INFO | MemInfo: { | 20260507 19:58:37.493 | INFO | "gpu" : " 17 GB", | 20260507 19:58:37.493 | INFO | "huge" : "", | 20260507 19:58:37.493 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:37.494 | INFO | "malloc" : " 14 GB", | 20260507 19:58:37.494 | INFO | "pinned" : " 29 GB" | 20260507 19:58:37.494 | INFO | } | 20260507 19:58:37.494 | INFO | Perf report for the last 5.13956 seconds: | 20260507 19:58:37.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:37.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:58:37.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 19:58:37.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:37.494 | INFO | Uploaded 36GiB to GPU in 2.51192s (14.3317GiB/s) | 20260507 19:58:37.494 | INFO | Downloaded 34GiB to GPU in 2.4718s (13.7551GiB/s) | 20260507 19:58:42.631 | INFO | Analyzing chunk frames = [2190848,2191360) | 20260507 19:58:42.631 | INFO | MemInfo: { | 20260507 19:58:42.631 | INFO | "gpu" : " 17 GB", | 20260507 19:58:42.631 | INFO | "huge" : "", | 20260507 19:58:42.631 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:42.631 | INFO | "malloc" : " 14 GB", | 20260507 19:58:42.631 | INFO | "pinned" : " 29 GB" | 20260507 19:58:42.631 | INFO | } | 20260507 19:58:42.632 | INFO | Perf report for the last 5.1379 seconds: | 20260507 19:58:42.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:42.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:58:42.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 19:58:42.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:42.632 | INFO | Uploaded 36GiB to GPU in 2.4613s (14.6264GiB/s) | 20260507 19:58:42.632 | INFO | Downloaded 34GiB to GPU in 2.45561s (13.8459GiB/s) | 20260507 19:58:42.744 | INFO | Starting a round of Baz IO | 20260507 19:58:46.246 | INFO | Populating RTMetrics report | 20260507 19:58:46.247 | INFO | RTMetrics report populated | 20260507 19:58:46.247 | INFO | Opening tmp RTMetrics json file | 20260507 19:58:46.248 | INFO | Serializing RTMetrics to json | 20260507 19:58:46.253 | INFO | Serialization done, swapping tmp file now... | 20260507 19:58:46.258 | INFO | RTMetrics done | 20260507 19:58:46.381 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.636 seconds | 20260507 19:58:46.381 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.89% / 0.01% / 55.10% | 20260507 19:58:46.381 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.18248% / 1.58946e-05% / 5.5631e-05% | 20260507 19:58:46.381 | INFO | Baz IO overall: | 20260507 19:58:46.381 | INFO | BytesWritten : 992583616917 | 20260507 19:58:46.381 | INFO | NumEvents : 481227448159 | 20260507 19:58:47.770 | INFO | Analyzing chunk frames = [2191360,2191872) | 20260507 19:58:47.770 | INFO | MemInfo: { | 20260507 19:58:47.770 | INFO | "gpu" : " 17 GB", | 20260507 19:58:47.770 | INFO | "huge" : "", | 20260507 19:58:47.770 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:47.770 | INFO | "malloc" : " 14 GB", | 20260507 19:58:47.770 | INFO | "pinned" : " 29 GB" | 20260507 19:58:47.770 | INFO | } | 20260507 19:58:47.771 | INFO | Perf report for the last 5.13898 seconds: | 20260507 19:58:47.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:47.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:47.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:47.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:47.771 | INFO | ANALYSIS : 69.1%, 2.9, 105.3ms, 0 | 20260507 19:58:47.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:47.771 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260507 19:58:47.771 | INFO | PRE_HQ : 68.8%, 1.9, 69.6ms, 0 | 20260507 19:58:47.771 | INFO | BAZWRITER : 70.4%, 4.1, 147.9ms, 0 | 20260507 19:58:47.771 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 19:58:47.771 | INFO | Uploaded 36GiB to GPU in 2.49937s (14.4036GiB/s) | 20260507 19:58:47.771 | INFO | Downloaded 38GiB to GPU in 2.6818s (14.1696GiB/s) | 20260507 19:58:52.910 | INFO | Analyzing chunk frames = [2191872,2192384) | 20260507 19:58:52.910 | INFO | MemInfo: { | 20260507 19:58:52.911 | INFO | "gpu" : " 17 GB", | 20260507 19:58:52.911 | INFO | "huge" : "", | 20260507 19:58:52.911 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:52.911 | INFO | "malloc" : " 14 GB", | 20260507 19:58:52.911 | INFO | "pinned" : " 29 GB" | 20260507 19:58:52.911 | INFO | } | 20260507 19:58:52.911 | INFO | Perf report for the last 5.14014 seconds: | 20260507 19:58:52.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:52.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 19:58:52.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:58:52.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:52.911 | INFO | Uploaded 36GiB to GPU in 2.53012s (14.2286GiB/s) | 20260507 19:58:52.911 | INFO | Downloaded 34GiB to GPU in 2.4852s (13.681GiB/s) | 20260507 19:58:58.049 | INFO | Analyzing chunk frames = [2192384,2192896) | 20260507 19:58:58.049 | INFO | MemInfo: { | 20260507 19:58:58.049 | INFO | "gpu" : " 17 GB", | 20260507 19:58:58.049 | INFO | "huge" : "", | 20260507 19:58:58.049 | INFO | "huge_shared" : " 29 GB", | 20260507 19:58:58.049 | INFO | "malloc" : " 14 GB", | 20260507 19:58:58.049 | INFO | "pinned" : " 29 GB" | 20260507 19:58:58.049 | INFO | } | 20260507 19:58:58.049 | INFO | Perf report for the last 5.13852 seconds: | 20260507 19:58:58.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:58:58.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.049 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 19:58:58.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.049 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:58:58.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:58:58.050 | INFO | Uploaded 36GiB to GPU in 2.53263s (14.2145GiB/s) | 20260507 19:58:58.050 | INFO | Downloaded 34GiB to GPU in 2.48956s (13.657GiB/s) | 20260507 19:59:03.189 | INFO | Analyzing chunk frames = [2192896,2193408) | 20260507 19:59:03.189 | INFO | MemInfo: { | 20260507 19:59:03.189 | INFO | "gpu" : " 17 GB", | 20260507 19:59:03.189 | INFO | "huge" : "", | 20260507 19:59:03.189 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:03.189 | INFO | "malloc" : " 14 GB", | 20260507 19:59:03.189 | INFO | "pinned" : " 29 GB" | 20260507 19:59:03.189 | INFO | } | 20260507 19:59:03.189 | INFO | Perf report for the last 5.13981 seconds: | 20260507 19:59:03.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:03.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.189 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 19:59:03.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.189 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 19:59:03.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:03.190 | INFO | Uploaded 36GiB to GPU in 2.41678s (14.8958GiB/s) | 20260507 19:59:03.190 | INFO | Downloaded 34GiB to GPU in 2.40312s (14.1482GiB/s) | 20260507 19:59:08.328 | INFO | Analyzing chunk frames = [2193408,2193920) | 20260507 19:59:08.328 | INFO | MemInfo: { | 20260507 19:59:08.328 | INFO | "gpu" : " 17 GB", | 20260507 19:59:08.328 | INFO | "huge" : "", | 20260507 19:59:08.328 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:08.328 | INFO | "malloc" : " 14 GB", | 20260507 19:59:08.328 | INFO | "pinned" : " 29 GB" | 20260507 19:59:08.328 | INFO | } | 20260507 19:59:08.328 | INFO | Perf report for the last 5.13897 seconds: | 20260507 19:59:08.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:08.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 19:59:08.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:59:08.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:08.328 | INFO | Uploaded 36GiB to GPU in 2.33772s (15.3997GiB/s) | 20260507 19:59:08.328 | INFO | Downloaded 34GiB to GPU in 2.51201s (13.535GiB/s) | 20260507 19:59:13.467 | INFO | Analyzing chunk frames = [2193920,2194432) | 20260507 19:59:13.467 | INFO | MemInfo: { | 20260507 19:59:13.467 | INFO | "gpu" : " 17 GB", | 20260507 19:59:13.467 | INFO | "huge" : "", | 20260507 19:59:13.467 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:13.467 | INFO | "malloc" : " 14 GB", | 20260507 19:59:13.467 | INFO | "pinned" : " 29 GB" | 20260507 19:59:13.467 | INFO | } | 20260507 19:59:13.467 | INFO | Perf report for the last 5.13922 seconds: | 20260507 19:59:13.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:13.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.467 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 19:59:13.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.467 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:59:13.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:13.468 | INFO | Uploaded 36GiB to GPU in 2.47169s (14.565GiB/s) | 20260507 19:59:13.468 | INFO | Downloaded 34GiB to GPU in 2.43676s (13.9529GiB/s) | 20260507 19:59:18.607 | INFO | Analyzing chunk frames = [2194432,2194944) | 20260507 19:59:18.607 | INFO | MemInfo: { | 20260507 19:59:18.607 | INFO | "gpu" : " 17 GB", | 20260507 19:59:18.607 | INFO | "huge" : "", | 20260507 19:59:18.607 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:18.607 | INFO | "malloc" : " 14 GB", | 20260507 19:59:18.607 | INFO | "pinned" : " 29 GB" | 20260507 19:59:18.608 | INFO | } | 20260507 19:59:18.608 | INFO | Perf report for the last 5.14035 seconds: | 20260507 19:59:18.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:18.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 19:59:18.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 19:59:18.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:18.608 | INFO | Uploaded 36GiB to GPU in 2.48141s (14.5079GiB/s) | 20260507 19:59:18.608 | INFO | Downloaded 34GiB to GPU in 2.46005s (13.8208GiB/s) | 20260507 19:59:23.749 | INFO | Analyzing chunk frames = [2194944,2195456) | 20260507 19:59:23.749 | INFO | MemInfo: { | 20260507 19:59:23.749 | INFO | "gpu" : " 17 GB", | 20260507 19:59:23.749 | INFO | "huge" : "", | 20260507 19:59:23.749 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:23.749 | INFO | "malloc" : " 14 GB", | 20260507 19:59:23.749 | INFO | "pinned" : " 29 GB" | 20260507 19:59:23.749 | INFO | } | 20260507 19:59:23.749 | INFO | Perf report for the last 5.14152 seconds: | 20260507 19:59:23.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:23.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.749 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 19:59:23.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.749 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260507 19:59:23.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:23.750 | INFO | Uploaded 36GiB to GPU in 2.43378s (14.7918GiB/s) | 20260507 19:59:23.750 | INFO | Downloaded 34GiB to GPU in 2.44683s (13.8955GiB/s) | 20260507 19:59:23.750 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 19:59:23.750 | INFO | Total mem usage: 16.4 GiB | 20260507 19:59:23.750 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 19:59:23.750 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 19:59:23.750 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 19:59:23.750 | INFO | High Water Breakdown: | 20260507 19:59:23.750 | INFO | 59.7% : 8.86 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 19:59:23.750 | INFO | 40.1% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 19:59:23.750 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 19:59:23.750 | INFO | ------------------------------------------ | 20260507 19:59:23.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:59:23.750 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 19:59:23.750 | INFO | Total mem usage: 26.5 GiB | 20260507 19:59:23.750 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 19:59:23.750 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 19:59:23.750 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 19:59:23.750 | INFO | High Water Breakdown: | 20260507 19:59:23.750 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:59:23.750 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:59:23.750 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:59:23.750 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:59:23.750 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:59:23.750 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:59:23.750 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 19:59:23.750 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 19:59:23.750 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:59:23.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 19:59:23.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 19:59:23.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 19:59:23.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 19:59:23.750 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 19:59:23.750 | INFO | ------------------------------------------ | 20260507 19:59:23.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:59:23.750 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 19:59:23.750 | INFO | Total mem usage: 27.8 GiB | 20260507 19:59:23.751 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 19:59:23.751 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 19:59:23.751 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 19:59:23.751 | INFO | High Water Breakdown: | 20260507 19:59:23.751 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 19:59:23.751 | INFO | ------------------------------------------ | 20260507 19:59:23.751 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:59:23.751 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 19:59:23.751 | INFO | Total mem usage: 15.7 GiB | 20260507 19:59:23.751 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 19:59:23.751 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 19:59:23.751 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 19:59:23.751 | INFO | High Water Breakdown: | 20260507 19:59:23.751 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 19:59:23.751 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 19:59:23.751 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 19:59:23.751 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 19:59:23.751 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 19:59:23.751 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 19:59:23.751 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 19:59:23.751 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 19:59:23.751 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 19:59:23.751 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 19:59:23.751 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 19:59:23.751 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 19:59:23.751 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 19:59:23.751 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 19:59:23.751 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 19:59:23.751 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 19:59:23.751 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 19:59:23.751 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 19:59:23.751 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 19:59:23.751 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 19:59:23.751 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 19:59:23.751 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 19:59:23.751 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 19:59:23.751 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 19:59:23.751 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 19:59:23.751 | INFO | ------------------------------------------ | 20260507 19:59:23.751 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 19:59:23.751 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 19:59:23.751 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:59:23.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:59:23.751 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.42e+01 / 1.81e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.04e-01, 1.28e-02 / 1.07e-02 | 20260507 19:59:23.751 | INFO | PerfReport: 12.12%, Baseline -- 4.02e+00 / 4.50e+00, 6.02e-01 / 2.11e-01 | 20260507 19:59:23.751 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.50e+00, 2.11e-01 / 1.05e-01 | 20260507 19:59:23.751 | INFO | PerfReport: 15.74%, DME -- 4.17e+01 / 4.49e+01, 3.79e+00 / 1.18e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.80e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.49e+00 / 2.82e+00, 4.42e-01 / 2.42e-01 | 20260507 19:59:23.751 | INFO | PerfReport: 3.61%, Metrics -- 1.20e+00 / 1.39e+00, 3.02e-01 / 1.62e-01 | 20260507 19:59:23.751 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.74e-01 / 4.05e-01, 6.13e-02 / 2.91e-02 | 20260507 19:59:23.751 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 19:59:23.751 | INFO | PerfReport: Gathered 1536 samples. | 20260507 19:59:23.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 19:59:23.751 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 24.11%, Upload -- 2.57e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 31.06%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 1.77%, LockUpload -- 1.89e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 2.65%, LockDownload -- 2.83e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 16.10%, LockCompute -- 1.72e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 19:59:23.751 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260507 19:59:23.862 | INFO | Starting a round of Baz IO | 20260507 19:59:27.359 | INFO | Populating RTMetrics report | 20260507 19:59:27.360 | INFO | RTMetrics report populated | 20260507 19:59:27.360 | INFO | Opening tmp RTMetrics json file | 20260507 19:59:27.361 | INFO | Serializing RTMetrics to json | 20260507 19:59:27.362 | INFO | Serialization done, swapping tmp file now... | 20260507 19:59:27.363 | INFO | RTMetrics done | 20260507 19:59:27.458 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.595 seconds | 20260507 19:59:27.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.88% / 0.01% / 55.11% | 20260507 19:59:27.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17756% / 1.58946e-05% / 2.78155e-05% | 20260507 19:59:27.458 | INFO | Baz IO overall: | 20260507 19:59:27.458 | INFO | BytesWritten : 994498452594 | 20260507 19:59:27.458 | INFO | NumEvents : 482154843628 | 20260507 19:59:28.886 | INFO | Analyzing chunk frames = [2195456,2195968) | 20260507 19:59:28.886 | INFO | MemInfo: { | 20260507 19:59:28.886 | INFO | "gpu" : " 17 GB", | 20260507 19:59:28.886 | INFO | "huge" : "", | 20260507 19:59:28.886 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:28.886 | INFO | "malloc" : " 14 GB", | 20260507 19:59:28.886 | INFO | "pinned" : " 29 GB" | 20260507 19:59:28.886 | INFO | } | 20260507 19:59:28.886 | INFO | Perf report for the last 5.13701 seconds: | 20260507 19:59:28.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:28.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:28.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:28.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:28.886 | INFO | ANALYSIS : 69.9%, 2.9, 105.1ms, 0 | 20260507 19:59:28.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:28.886 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260507 19:59:28.886 | INFO | PRE_HQ : 68.8%, 1.8, 65.3ms, 0 | 20260507 19:59:28.886 | INFO | BAZWRITER : 69.6%, 4.3, 153.6ms, 0 | 20260507 19:59:28.886 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 19:59:28.886 | INFO | Uploaded 36GiB to GPU in 2.58623s (13.9199GiB/s) | 20260507 19:59:28.886 | INFO | Downloaded 38GiB to GPU in 2.72222s (13.9592GiB/s) | 20260507 19:59:34.024 | INFO | Analyzing chunk frames = [2195968,2196480) | 20260507 19:59:34.024 | INFO | MemInfo: { | 20260507 19:59:34.024 | INFO | "gpu" : " 17 GB", | 20260507 19:59:34.024 | INFO | "huge" : "", | 20260507 19:59:34.025 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:34.025 | INFO | "malloc" : " 14 GB", | 20260507 19:59:34.025 | INFO | "pinned" : " 29 GB" | 20260507 19:59:34.025 | INFO | } | 20260507 19:59:34.025 | INFO | Perf report for the last 5.13853 seconds: | 20260507 19:59:34.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:34.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 19:59:34.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 19:59:34.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:34.025 | INFO | Uploaded 36GiB to GPU in 2.56403s (14.0404GiB/s) | 20260507 19:59:34.025 | INFO | Downloaded 34GiB to GPU in 2.48976s (13.656GiB/s) | 20260507 19:59:39.164 | INFO | Analyzing chunk frames = [2196480,2196992) | 20260507 19:59:39.164 | INFO | MemInfo: { | 20260507 19:59:39.164 | INFO | "gpu" : " 17 GB", | 20260507 19:59:39.164 | INFO | "huge" : "", | 20260507 19:59:39.164 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:39.164 | INFO | "malloc" : " 14 GB", | 20260507 19:59:39.164 | INFO | "pinned" : " 29 GB" | 20260507 19:59:39.164 | INFO | } | 20260507 19:59:39.164 | INFO | Perf report for the last 5.13933 seconds: | 20260507 19:59:39.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:39.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.164 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 19:59:39.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.164 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 19:59:39.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:39.165 | INFO | Uploaded 36GiB to GPU in 2.56477s (14.0364GiB/s) | 20260507 19:59:39.165 | INFO | Downloaded 34GiB to GPU in 2.46782s (13.7773GiB/s) | 20260507 19:59:44.303 | INFO | Analyzing chunk frames = [2196992,2197504) | 20260507 19:59:44.303 | INFO | MemInfo: { | 20260507 19:59:44.303 | INFO | "gpu" : " 17 GB", | 20260507 19:59:44.303 | INFO | "huge" : "", | 20260507 19:59:44.303 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:44.304 | INFO | "malloc" : " 14 GB", | 20260507 19:59:44.304 | INFO | "pinned" : " 29 GB" | 20260507 19:59:44.304 | INFO | } | 20260507 19:59:44.304 | INFO | Perf report for the last 5.13954 seconds: | 20260507 19:59:44.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:44.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 19:59:44.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 19:59:44.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:44.304 | INFO | Uploaded 36GiB to GPU in 2.47236s (14.561GiB/s) | 20260507 19:59:44.304 | INFO | Downloaded 34GiB to GPU in 2.39282s (14.2092GiB/s) | 20260507 19:59:49.443 | INFO | Analyzing chunk frames = [2197504,2198016) | 20260507 19:59:49.443 | INFO | MemInfo: { | 20260507 19:59:49.443 | INFO | "gpu" : " 17 GB", | 20260507 19:59:49.443 | INFO | "huge" : "", | 20260507 19:59:49.443 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:49.443 | INFO | "malloc" : " 14 GB", | 20260507 19:59:49.443 | INFO | "pinned" : " 29 GB" | 20260507 19:59:49.443 | INFO | } | 20260507 19:59:49.443 | INFO | Perf report for the last 5.13953 seconds: | 20260507 19:59:49.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:49.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 19:59:49.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 19:59:49.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:49.443 | INFO | Uploaded 36GiB to GPU in 2.44655s (14.7146GiB/s) | 20260507 19:59:49.443 | INFO | Downloaded 34GiB to GPU in 2.50769s (13.5583GiB/s) | 20260507 19:59:54.583 | INFO | Analyzing chunk frames = [2198016,2198528) | 20260507 19:59:54.583 | INFO | MemInfo: { | 20260507 19:59:54.583 | INFO | "gpu" : " 17 GB", | 20260507 19:59:54.583 | INFO | "huge" : "", | 20260507 19:59:54.583 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:54.583 | INFO | "malloc" : " 14 GB", | 20260507 19:59:54.583 | INFO | "pinned" : " 29 GB" | 20260507 19:59:54.583 | INFO | } | 20260507 19:59:54.583 | INFO | Perf report for the last 5.13986 seconds: | 20260507 19:59:54.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:54.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 19:59:54.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 19:59:54.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:54.583 | INFO | Uploaded 36GiB to GPU in 2.49363s (14.4368GiB/s) | 20260507 19:59:54.583 | INFO | Downloaded 34GiB to GPU in 2.42301s (14.0322GiB/s) | 20260507 19:59:59.722 | INFO | Analyzing chunk frames = [2198528,2199040) | 20260507 19:59:59.722 | INFO | MemInfo: { | 20260507 19:59:59.722 | INFO | "gpu" : " 17 GB", | 20260507 19:59:59.722 | INFO | "huge" : "", | 20260507 19:59:59.722 | INFO | "huge_shared" : " 29 GB", | 20260507 19:59:59.722 | INFO | "malloc" : " 14 GB", | 20260507 19:59:59.722 | INFO | "pinned" : " 29 GB" | 20260507 19:59:59.722 | INFO | } | 20260507 19:59:59.722 | INFO | Perf report for the last 5.13911 seconds: | 20260507 19:59:59.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 19:59:59.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.722 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 19:59:59.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.722 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 19:59:59.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 19:59:59.723 | INFO | Uploaded 36GiB to GPU in 2.53777s (14.1857GiB/s) | 20260507 19:59:59.723 | INFO | Downloaded 34GiB to GPU in 2.45261s (13.8628GiB/s) | 20260507 20:00:04.861 | INFO | Analyzing chunk frames = [2199040,2199552) | 20260507 20:00:04.861 | INFO | MemInfo: { | 20260507 20:00:04.861 | INFO | "gpu" : " 17 GB", | 20260507 20:00:04.861 | INFO | "huge" : "", | 20260507 20:00:04.861 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:04.861 | INFO | "malloc" : " 14 GB", | 20260507 20:00:04.861 | INFO | "pinned" : " 29 GB" | 20260507 20:00:04.861 | INFO | } | 20260507 20:00:04.861 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:00:04.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:04.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.862 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:00:04.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.862 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 20:00:04.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:04.862 | INFO | Uploaded 36GiB to GPU in 2.51407s (14.3194GiB/s) | 20260507 20:00:04.862 | INFO | Downloaded 34GiB to GPU in 2.43317s (13.9735GiB/s) | 20260507 20:00:04.975 | INFO | Starting a round of Baz IO | 20260507 20:00:08.448 | INFO | Populating RTMetrics report | 20260507 20:00:08.448 | INFO | RTMetrics report populated | 20260507 20:00:08.448 | INFO | Opening tmp RTMetrics json file | 20260507 20:00:08.480 | INFO | Serializing RTMetrics to json | 20260507 20:00:08.547 | INFO | Serialization done, swapping tmp file now... | 20260507 20:00:08.548 | INFO | RTMetrics done | 20260507 20:00:08.572 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.596 seconds | 20260507 20:00:08.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.87% / 0.01% / 55.12% | 20260507 20:00:08.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17173% / 1.58946e-05% / 4.76837e-05% | 20260507 20:00:08.572 | INFO | Baz IO overall: | 20260507 20:00:08.572 | INFO | BytesWritten : 996413034367 | 20260507 20:00:08.572 | INFO | NumEvents : 483082030157 | 20260507 20:00:10.000 | INFO | Analyzing chunk frames = [2199552,2200064) | 20260507 20:00:10.000 | INFO | MemInfo: { | 20260507 20:00:10.000 | INFO | "gpu" : " 17 GB", | 20260507 20:00:10.000 | INFO | "huge" : "", | 20260507 20:00:10.000 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:10.000 | INFO | "malloc" : " 14 GB", | 20260507 20:00:10.000 | INFO | "pinned" : " 29 GB" | 20260507 20:00:10.000 | INFO | } | 20260507 20:00:10.000 | INFO | Perf report for the last 5.13855 seconds: | 20260507 20:00:10.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:10.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:10.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:10.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:10.000 | INFO | ANALYSIS : 69.0%, 2.9, 104.5ms, 0 | 20260507 20:00:10.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:10.000 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260507 20:00:10.000 | INFO | PRE_HQ : 68.0%, 1.8, 64.0ms, 0 | 20260507 20:00:10.000 | INFO | BAZWRITER : 69.6%, 4.1, 147.4ms, 0 | 20260507 20:00:10.000 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 20:00:10.000 | INFO | Uploaded 36GiB to GPU in 2.6112s (13.7868GiB/s) | 20260507 20:00:10.000 | INFO | Downloaded 38GiB to GPU in 2.72687s (13.9354GiB/s) | 20260507 20:00:15.139 | INFO | Analyzing chunk frames = [2200064,2200576) | 20260507 20:00:15.139 | INFO | MemInfo: { | 20260507 20:00:15.139 | INFO | "gpu" : " 17 GB", | 20260507 20:00:15.139 | INFO | "huge" : "", | 20260507 20:00:15.139 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:15.139 | INFO | "malloc" : " 14 GB", | 20260507 20:00:15.139 | INFO | "pinned" : " 29 GB" | 20260507 20:00:15.139 | INFO | } | 20260507 20:00:15.139 | INFO | Perf report for the last 5.13943 seconds: | 20260507 20:00:15.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:15.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.139 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:00:15.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.140 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:00:15.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:15.140 | INFO | Uploaded 36GiB to GPU in 2.5206s (14.2823GiB/s) | 20260507 20:00:15.140 | INFO | Downloaded 34GiB to GPU in 2.49152s (13.6463GiB/s) | 20260507 20:00:20.279 | INFO | Analyzing chunk frames = [2200576,2201088) | 20260507 20:00:20.279 | INFO | MemInfo: { | 20260507 20:00:20.279 | INFO | "gpu" : " 17 GB", | 20260507 20:00:20.279 | INFO | "huge" : "", | 20260507 20:00:20.279 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:20.279 | INFO | "malloc" : " 14 GB", | 20260507 20:00:20.279 | INFO | "pinned" : " 29 GB" | 20260507 20:00:20.279 | INFO | } | 20260507 20:00:20.279 | INFO | Perf report for the last 5.13998 seconds: | 20260507 20:00:20.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:20.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.280 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:00:20.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.280 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:00:20.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:20.280 | INFO | Uploaded 36GiB to GPU in 2.47353s (14.5541GiB/s) | 20260507 20:00:20.280 | INFO | Downloaded 34GiB to GPU in 2.47741s (13.724GiB/s) | 20260507 20:00:25.419 | INFO | Analyzing chunk frames = [2201088,2201600) | 20260507 20:00:25.419 | INFO | MemInfo: { | 20260507 20:00:25.419 | INFO | "gpu" : " 17 GB", | 20260507 20:00:25.419 | INFO | "huge" : "", | 20260507 20:00:25.419 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:25.419 | INFO | "malloc" : " 14 GB", | 20260507 20:00:25.419 | INFO | "pinned" : " 29 GB" | 20260507 20:00:25.419 | INFO | } | 20260507 20:00:25.419 | INFO | Perf report for the last 5.13961 seconds: | 20260507 20:00:25.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:25.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 20:00:25.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:00:25.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:25.419 | INFO | Uploaded 36GiB to GPU in 2.42734s (14.8311GiB/s) | 20260507 20:00:25.419 | INFO | Downloaded 34GiB to GPU in 2.41892s (14.0559GiB/s) | 20260507 20:00:30.558 | INFO | Analyzing chunk frames = [2201600,2202112) | 20260507 20:00:30.558 | INFO | MemInfo: { | 20260507 20:00:30.558 | INFO | "gpu" : " 17 GB", | 20260507 20:00:30.558 | INFO | "huge" : "", | 20260507 20:00:30.558 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:30.558 | INFO | "malloc" : " 14 GB", | 20260507 20:00:30.558 | INFO | "pinned" : " 29 GB" | 20260507 20:00:30.558 | INFO | } | 20260507 20:00:30.558 | INFO | Perf report for the last 5.13919 seconds: | 20260507 20:00:30.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:30.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.558 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 20:00:30.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.558 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 20:00:30.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:30.559 | INFO | Uploaded 36GiB to GPU in 2.34842s (15.3295GiB/s) | 20260507 20:00:30.559 | INFO | Downloaded 34GiB to GPU in 2.51505s (13.5186GiB/s) | 20260507 20:00:35.697 | INFO | Analyzing chunk frames = [2202112,2202624) | 20260507 20:00:35.697 | INFO | MemInfo: { | 20260507 20:00:35.697 | INFO | "gpu" : " 17 GB", | 20260507 20:00:35.697 | INFO | "huge" : "", | 20260507 20:00:35.697 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:35.697 | INFO | "malloc" : " 14 GB", | 20260507 20:00:35.697 | INFO | "pinned" : " 29 GB" | 20260507 20:00:35.697 | INFO | } | 20260507 20:00:35.697 | INFO | Perf report for the last 5.13916 seconds: | 20260507 20:00:35.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:35.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.698 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 20:00:35.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.698 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 20:00:35.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:35.698 | INFO | Uploaded 36GiB to GPU in 2.34461s (15.3544GiB/s) | 20260507 20:00:35.698 | INFO | Downloaded 34GiB to GPU in 2.40494s (14.1376GiB/s) | 20260507 20:00:40.837 | INFO | Analyzing chunk frames = [2202624,2203136) | 20260507 20:00:40.837 | INFO | MemInfo: { | 20260507 20:00:40.837 | INFO | "gpu" : " 17 GB", | 20260507 20:00:40.837 | INFO | "huge" : "", | 20260507 20:00:40.837 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:40.837 | INFO | "malloc" : " 14 GB", | 20260507 20:00:40.837 | INFO | "pinned" : " 29 GB" | 20260507 20:00:40.837 | INFO | } | 20260507 20:00:40.837 | INFO | Perf report for the last 5.13964 seconds: | 20260507 20:00:40.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:40.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 20:00:40.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:00:40.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:40.837 | INFO | Uploaded 36GiB to GPU in 2.46323s (14.6149GiB/s) | 20260507 20:00:40.837 | INFO | Downloaded 34GiB to GPU in 2.46097s (13.8157GiB/s) | 20260507 20:00:45.975 | INFO | Analyzing chunk frames = [2203136,2203648) | 20260507 20:00:45.975 | INFO | MemInfo: { | 20260507 20:00:45.975 | INFO | "gpu" : " 17 GB", | 20260507 20:00:45.975 | INFO | "huge" : "", | 20260507 20:00:45.975 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:45.975 | INFO | "malloc" : " 14 GB", | 20260507 20:00:45.975 | INFO | "pinned" : " 29 GB" | 20260507 20:00:45.975 | INFO | } | 20260507 20:00:45.975 | INFO | Perf report for the last 5.1379 seconds: | 20260507 20:00:45.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:45.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:00:45.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:00:45.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:45.975 | INFO | Uploaded 36GiB to GPU in 2.47312s (14.5565GiB/s) | 20260507 20:00:45.975 | INFO | Downloaded 34GiB to GPU in 2.47131s (13.7579GiB/s) | 20260507 20:00:45.976 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:00:45.976 | INFO | Total mem usage: 16.4 GiB | 20260507 20:00:45.976 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:00:45.976 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.92%) | 20260507 20:00:45.976 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:00:45.976 | INFO | High Water Breakdown: | 20260507 20:00:45.976 | INFO | 60.2% : 9.15 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:00:45.976 | INFO | 39.7% : 6.03 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:00:45.976 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:00:45.976 | INFO | ------------------------------------------ | 20260507 20:00:45.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:00:45.976 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:00:45.976 | INFO | Total mem usage: 26.5 GiB | 20260507 20:00:45.976 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:00:45.976 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:00:45.976 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:00:45.976 | INFO | High Water Breakdown: | 20260507 20:00:45.976 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:00:45.976 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:00:45.976 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:00:45.976 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:00:45.976 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:00:45.976 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:00:45.976 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:00:45.976 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:00:45.976 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:00:45.976 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:00:45.976 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:00:45.976 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:00:45.976 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:00:45.976 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:00:45.976 | INFO | ------------------------------------------ | 20260507 20:00:45.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:00:45.976 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:00:45.976 | INFO | Total mem usage: 27.8 GiB | 20260507 20:00:45.976 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:00:45.976 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:00:45.976 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:00:45.976 | INFO | High Water Breakdown: | 20260507 20:00:45.976 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:00:45.976 | INFO | ------------------------------------------ | 20260507 20:00:45.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:00:45.976 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:00:45.976 | INFO | Total mem usage: 15.7 GiB | 20260507 20:00:45.976 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:00:45.976 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:00:45.976 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:00:45.976 | INFO | High Water Breakdown: | 20260507 20:00:45.976 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:00:45.976 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:00:45.976 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:00:45.976 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:00:45.976 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:00:45.976 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:00:45.976 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:00:45.976 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:00:45.976 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:00:45.977 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:00:45.977 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:00:45.977 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:00:45.977 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:00:45.977 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:00:45.977 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:00:45.977 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:00:45.977 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:00:45.977 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:00:45.977 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:00:45.977 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:00:45.977 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:00:45.977 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:00:45.977 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:00:45.977 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:00:45.977 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:00:45.977 | INFO | ------------------------------------------ | 20260507 20:00:45.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:00:45.977 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:00:45.977 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:00:45.977 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:00:45.977 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.39e+01 / 1.76e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.03e-01, 1.11e-02 / 6.29e-03 | 20260507 20:00:45.977 | INFO | PerfReport: 12.08%, Baseline -- 4.01e+00 / 4.45e+00, 5.99e-01 / 3.04e-01 | 20260507 20:00:45.977 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.08e-01 / 1.21e-01 | 20260507 20:00:45.977 | INFO | PerfReport: 15.76%, DME -- 4.18e+01 / 4.50e+01, 3.89e+00 / 1.28e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.85e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.81e+00, 4.40e-01 / 2.67e-01 | 20260507 20:00:45.977 | INFO | PerfReport: 3.62%, Metrics -- 1.20e+00 / 1.38e+00, 3.06e-01 / 1.71e-01 | 20260507 20:00:45.977 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.74e-01 / 4.03e-01, 6.11e-02 / 3.07e-02 | 20260507 20:00:45.977 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:00:45.977 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:00:45.977 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:00:45.977 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 31.10%, Compute -- 3.32e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 1.80%, LockUpload -- 1.92e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 2.54%, LockDownload -- 2.71e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 15.87%, LockCompute -- 1.69e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260507 20:00:45.977 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260507 20:00:46.073 | INFO | Starting a round of Baz IO | 20260507 20:00:49.549 | INFO | Populating RTMetrics report | 20260507 20:00:49.550 | INFO | RTMetrics report populated | 20260507 20:00:49.550 | INFO | Opening tmp RTMetrics json file | 20260507 20:00:49.573 | INFO | Serializing RTMetrics to json | 20260507 20:00:49.574 | INFO | Serialization done, swapping tmp file now... | 20260507 20:00:49.575 | INFO | RTMetrics done | 20260507 20:00:49.634 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.561 seconds | 20260507 20:00:49.634 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.86% / 0.01% / 55.13% | 20260507 20:00:49.634 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17221% / 1.19209e-05% / 5.96046e-05% | 20260507 20:00:49.634 | INFO | Baz IO overall: | 20260507 20:00:49.634 | INFO | BytesWritten : 998329495224 | 20260507 20:00:49.634 | INFO | NumEvents : 484010759435 | 20260507 20:00:51.115 | INFO | Analyzing chunk frames = [2203648,2204160) | 20260507 20:00:51.115 | INFO | MemInfo: { | 20260507 20:00:51.115 | INFO | "gpu" : " 17 GB", | 20260507 20:00:51.115 | INFO | "huge" : "", | 20260507 20:00:51.115 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:51.115 | INFO | "malloc" : " 14 GB", | 20260507 20:00:51.115 | INFO | "pinned" : " 29 GB" | 20260507 20:00:51.115 | INFO | } | 20260507 20:00:51.115 | INFO | Perf report for the last 5.13981 seconds: | 20260507 20:00:51.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:51.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:51.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:51.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:51.115 | INFO | ANALYSIS : 68.8%, 2.9, 106.2ms, 0 | 20260507 20:00:51.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:51.115 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 20:00:51.115 | INFO | PRE_HQ : 66.1%, 1.7, 58.4ms, 0 | 20260507 20:00:51.115 | INFO | BAZWRITER : 68.9%, 4.0, 141.1ms, 0 | 20260507 20:00:51.115 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 20:00:51.116 | INFO | Uploaded 36GiB to GPU in 2.493s (14.4404GiB/s) | 20260507 20:00:51.116 | INFO | Downloaded 38GiB to GPU in 2.69144s (14.1188GiB/s) | 20260507 20:00:56.254 | INFO | Analyzing chunk frames = [2204160,2204672) | 20260507 20:00:56.254 | INFO | MemInfo: { | 20260507 20:00:56.254 | INFO | "gpu" : " 17 GB", | 20260507 20:00:56.254 | INFO | "huge" : "", | 20260507 20:00:56.254 | INFO | "huge_shared" : " 29 GB", | 20260507 20:00:56.254 | INFO | "malloc" : " 14 GB", | 20260507 20:00:56.254 | INFO | "pinned" : " 29 GB" | 20260507 20:00:56.254 | INFO | } | 20260507 20:00:56.254 | INFO | Perf report for the last 5.13953 seconds: | 20260507 20:00:56.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:00:56.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.254 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 20:00:56.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.255 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 20:00:56.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:00:56.255 | INFO | Uploaded 36GiB to GPU in 2.55241s (14.1043GiB/s) | 20260507 20:00:56.255 | INFO | Downloaded 34GiB to GPU in 2.49771s (13.6125GiB/s) | 20260507 20:01:01.393 | INFO | Analyzing chunk frames = [2204672,2205184) | 20260507 20:01:01.393 | INFO | MemInfo: { | 20260507 20:01:01.393 | INFO | "gpu" : " 17 GB", | 20260507 20:01:01.394 | INFO | "huge" : "", | 20260507 20:01:01.394 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:01.394 | INFO | "malloc" : " 14 GB", | 20260507 20:01:01.394 | INFO | "pinned" : " 29 GB" | 20260507 20:01:01.394 | INFO | } | 20260507 20:01:01.394 | INFO | Perf report for the last 5.13932 seconds: | 20260507 20:01:01.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:01.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 20:01:01.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:01:01.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:01.394 | INFO | Uploaded 36GiB to GPU in 2.52344s (14.2663GiB/s) | 20260507 20:01:01.394 | INFO | Downloaded 34GiB to GPU in 2.49037s (13.6526GiB/s) | 20260507 20:01:06.534 | INFO | Analyzing chunk frames = [2205184,2205696) | 20260507 20:01:06.534 | INFO | MemInfo: { | 20260507 20:01:06.534 | INFO | "gpu" : " 17 GB", | 20260507 20:01:06.534 | INFO | "huge" : "", | 20260507 20:01:06.534 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:06.534 | INFO | "malloc" : " 14 GB", | 20260507 20:01:06.534 | INFO | "pinned" : " 29 GB" | 20260507 20:01:06.534 | INFO | } | 20260507 20:01:06.534 | INFO | Perf report for the last 5.1403 seconds: | 20260507 20:01:06.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:06.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 20:01:06.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:01:06.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:06.534 | INFO | Uploaded 36GiB to GPU in 2.41592s (14.9011GiB/s) | 20260507 20:01:06.534 | INFO | Downloaded 34GiB to GPU in 2.50774s (13.558GiB/s) | 20260507 20:01:11.674 | INFO | Analyzing chunk frames = [2205696,2206208) | 20260507 20:01:11.674 | INFO | MemInfo: { | 20260507 20:01:11.674 | INFO | "gpu" : " 17 GB", | 20260507 20:01:11.674 | INFO | "huge" : "", | 20260507 20:01:11.674 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:11.674 | INFO | "malloc" : " 14 GB", | 20260507 20:01:11.674 | INFO | "pinned" : " 29 GB" | 20260507 20:01:11.674 | INFO | } | 20260507 20:01:11.674 | INFO | Perf report for the last 5.14016 seconds: | 20260507 20:01:11.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:11.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.674 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:01:11.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.674 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:01:11.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:11.675 | INFO | Uploaded 36GiB to GPU in 2.31045s (15.5814GiB/s) | 20260507 20:01:11.675 | INFO | Downloaded 34GiB to GPU in 2.50491s (13.5734GiB/s) | 20260507 20:01:16.813 | INFO | Analyzing chunk frames = [2206208,2206720) | 20260507 20:01:16.813 | INFO | MemInfo: { | 20260507 20:01:16.813 | INFO | "gpu" : " 17 GB", | 20260507 20:01:16.813 | INFO | "huge" : "", | 20260507 20:01:16.813 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:16.813 | INFO | "malloc" : " 14 GB", | 20260507 20:01:16.813 | INFO | "pinned" : " 29 GB" | 20260507 20:01:16.813 | INFO | } | 20260507 20:01:16.813 | INFO | Perf report for the last 5.13899 seconds: | 20260507 20:01:16.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:16.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.813 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:01:16.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.813 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:01:16.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:16.814 | INFO | Uploaded 36GiB to GPU in 2.50457s (14.3737GiB/s) | 20260507 20:01:16.814 | INFO | Downloaded 34GiB to GPU in 2.42666s (14.011GiB/s) | 20260507 20:01:21.951 | INFO | Analyzing chunk frames = [2206720,2207232) | 20260507 20:01:21.951 | INFO | MemInfo: { | 20260507 20:01:21.951 | INFO | "gpu" : " 17 GB", | 20260507 20:01:21.951 | INFO | "huge" : "", | 20260507 20:01:21.951 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:21.951 | INFO | "malloc" : " 14 GB", | 20260507 20:01:21.951 | INFO | "pinned" : " 29 GB" | 20260507 20:01:21.951 | INFO | } | 20260507 20:01:21.951 | INFO | Perf report for the last 5.13805 seconds: | 20260507 20:01:21.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:21.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.951 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:01:21.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.951 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:01:21.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:21.952 | INFO | Uploaded 36GiB to GPU in 2.52676s (14.2475GiB/s) | 20260507 20:01:21.952 | INFO | Downloaded 34GiB to GPU in 2.45471s (13.8509GiB/s) | 20260507 20:01:27.090 | INFO | Analyzing chunk frames = [2207232,2207744) | 20260507 20:01:27.090 | INFO | MemInfo: { | 20260507 20:01:27.090 | INFO | "gpu" : " 17 GB", | 20260507 20:01:27.090 | INFO | "huge" : "", | 20260507 20:01:27.090 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:27.090 | INFO | "malloc" : " 14 GB", | 20260507 20:01:27.090 | INFO | "pinned" : " 29 GB" | 20260507 20:01:27.090 | INFO | } | 20260507 20:01:27.090 | INFO | Perf report for the last 5.13912 seconds: | 20260507 20:01:27.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:27.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.090 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:01:27.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.091 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:01:27.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:27.091 | INFO | Uploaded 36GiB to GPU in 2.50591s (14.366GiB/s) | 20260507 20:01:27.091 | INFO | Downloaded 34GiB to GPU in 2.43643s (13.9548GiB/s) | 20260507 20:01:27.212 | INFO | Starting a round of Baz IO | 20260507 20:01:30.691 | INFO | Populating RTMetrics report | 20260507 20:01:30.692 | INFO | RTMetrics report populated | 20260507 20:01:30.692 | INFO | Opening tmp RTMetrics json file | 20260507 20:01:30.693 | INFO | Serializing RTMetrics to json | 20260507 20:01:30.695 | INFO | Serialization done, swapping tmp file now... | 20260507 20:01:30.696 | INFO | RTMetrics done | 20260507 20:01:30.786 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.574 seconds | 20260507 20:01:30.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.85% / 0.01% / 55.14% | 20260507 20:01:30.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.16282% / 1.19209e-05% / 4.76837e-05% | 20260507 20:01:30.786 | INFO | Baz IO overall: | 20260507 20:01:30.786 | INFO | BytesWritten : 1000245062672 | 20260507 20:01:30.786 | INFO | NumEvents : 484938694789 | 20260507 20:01:32.230 | INFO | Analyzing chunk frames = [2207744,2208256) | 20260507 20:01:32.230 | INFO | MemInfo: { | 20260507 20:01:32.230 | INFO | "gpu" : " 17 GB", | 20260507 20:01:32.230 | INFO | "huge" : "", | 20260507 20:01:32.230 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:32.230 | INFO | "malloc" : " 14 GB", | 20260507 20:01:32.230 | INFO | "pinned" : " 29 GB" | 20260507 20:01:32.230 | INFO | } | 20260507 20:01:32.230 | INFO | Perf report for the last 5.13936 seconds: | 20260507 20:01:32.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:32.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:32.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:32.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:32.230 | INFO | ANALYSIS : 69.6%, 2.9, 105.7ms, 0 | 20260507 20:01:32.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:32.230 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 20:01:32.230 | INFO | PRE_HQ : 68.0%, 1.8, 62.6ms, 0 | 20260507 20:01:32.230 | INFO | BAZWRITER : 69.6%, 3.9, 140.8ms, 0 | 20260507 20:01:32.230 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 20:01:32.231 | INFO | Uploaded 36GiB to GPU in 2.53295s (14.2127GiB/s) | 20260507 20:01:32.231 | INFO | Downloaded 38GiB to GPU in 2.68706s (14.1419GiB/s) | 20260507 20:01:37.369 | INFO | Analyzing chunk frames = [2208256,2208768) | 20260507 20:01:37.369 | INFO | MemInfo: { | 20260507 20:01:37.369 | INFO | "gpu" : " 17 GB", | 20260507 20:01:37.369 | INFO | "huge" : "", | 20260507 20:01:37.369 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:37.369 | INFO | "malloc" : " 14 GB", | 20260507 20:01:37.369 | INFO | "pinned" : " 29 GB" | 20260507 20:01:37.369 | INFO | } | 20260507 20:01:37.369 | INFO | Perf report for the last 5.13904 seconds: | 20260507 20:01:37.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:37.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 20:01:37.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:01:37.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:37.369 | INFO | Uploaded 36GiB to GPU in 2.55891s (14.0685GiB/s) | 20260507 20:01:37.369 | INFO | Downloaded 34GiB to GPU in 2.47025s (13.7638GiB/s) | 20260507 20:01:42.509 | INFO | Analyzing chunk frames = [2208768,2209280) | 20260507 20:01:42.509 | INFO | MemInfo: { | 20260507 20:01:42.509 | INFO | "gpu" : " 17 GB", | 20260507 20:01:42.509 | INFO | "huge" : "", | 20260507 20:01:42.509 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:42.509 | INFO | "malloc" : " 14 GB", | 20260507 20:01:42.509 | INFO | "pinned" : " 29 GB" | 20260507 20:01:42.509 | INFO | } | 20260507 20:01:42.509 | INFO | Perf report for the last 5.14051 seconds: | 20260507 20:01:42.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:42.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.509 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260507 20:01:42.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.509 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:01:42.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:42.510 | INFO | Uploaded 36GiB to GPU in 2.59163s (13.8909GiB/s) | 20260507 20:01:42.510 | INFO | Downloaded 34GiB to GPU in 2.45551s (13.8464GiB/s) | 20260507 20:01:47.648 | INFO | Analyzing chunk frames = [2209280,2209792) | 20260507 20:01:47.648 | INFO | MemInfo: { | 20260507 20:01:47.648 | INFO | "gpu" : " 17 GB", | 20260507 20:01:47.648 | INFO | "huge" : "", | 20260507 20:01:47.648 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:47.648 | INFO | "malloc" : " 14 GB", | 20260507 20:01:47.648 | INFO | "pinned" : " 29 GB" | 20260507 20:01:47.648 | INFO | } | 20260507 20:01:47.648 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:01:47.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:47.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.648 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 20:01:47.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.648 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:01:47.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:47.649 | INFO | Uploaded 36GiB to GPU in 2.48408s (14.4923GiB/s) | 20260507 20:01:47.649 | INFO | Downloaded 34GiB to GPU in 2.38596s (14.25GiB/s) | 20260507 20:01:52.787 | INFO | Analyzing chunk frames = [2209792,2210304) | 20260507 20:01:52.787 | INFO | MemInfo: { | 20260507 20:01:52.787 | INFO | "gpu" : " 17 GB", | 20260507 20:01:52.787 | INFO | "huge" : "", | 20260507 20:01:52.787 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:52.787 | INFO | "malloc" : " 14 GB", | 20260507 20:01:52.787 | INFO | "pinned" : " 29 GB" | 20260507 20:01:52.787 | INFO | } | 20260507 20:01:52.787 | INFO | Perf report for the last 5.13896 seconds: | 20260507 20:01:52.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:52.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.787 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 20:01:52.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.787 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:01:52.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:52.788 | INFO | Uploaded 36GiB to GPU in 2.44688s (14.7126GiB/s) | 20260507 20:01:52.788 | INFO | Downloaded 34GiB to GPU in 2.49484s (13.6281GiB/s) | 20260507 20:01:57.926 | INFO | Analyzing chunk frames = [2210304,2210816) | 20260507 20:01:57.926 | INFO | MemInfo: { | 20260507 20:01:57.926 | INFO | "gpu" : " 17 GB", | 20260507 20:01:57.926 | INFO | "huge" : "", | 20260507 20:01:57.926 | INFO | "huge_shared" : " 29 GB", | 20260507 20:01:57.926 | INFO | "malloc" : " 14 GB", | 20260507 20:01:57.926 | INFO | "pinned" : " 29 GB" | 20260507 20:01:57.926 | INFO | } | 20260507 20:01:57.926 | INFO | Perf report for the last 5.1388 seconds: | 20260507 20:01:57.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:01:57.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.926 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:01:57.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.926 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:01:57.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:01:57.927 | INFO | Uploaded 36GiB to GPU in 2.47413s (14.5506GiB/s) | 20260507 20:01:57.927 | INFO | Downloaded 34GiB to GPU in 2.42438s (14.0242GiB/s) | 20260507 20:02:03.066 | INFO | Analyzing chunk frames = [2210816,2211328) | 20260507 20:02:03.066 | INFO | MemInfo: { | 20260507 20:02:03.066 | INFO | "gpu" : " 17 GB", | 20260507 20:02:03.066 | INFO | "huge" : "", | 20260507 20:02:03.066 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:03.066 | INFO | "malloc" : " 14 GB", | 20260507 20:02:03.066 | INFO | "pinned" : " 29 GB" | 20260507 20:02:03.066 | INFO | } | 20260507 20:02:03.067 | INFO | Perf report for the last 5.14053 seconds: | 20260507 20:02:03.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:03.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:02:03.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 20:02:03.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:03.067 | INFO | Uploaded 36GiB to GPU in 2.53455s (14.2037GiB/s) | 20260507 20:02:03.067 | INFO | Downloaded 34GiB to GPU in 2.4537s (13.8566GiB/s) | 20260507 20:02:08.206 | INFO | Analyzing chunk frames = [2211328,2211840) | 20260507 20:02:08.206 | INFO | MemInfo: { | 20260507 20:02:08.206 | INFO | "gpu" : " 17 GB", | 20260507 20:02:08.206 | INFO | "huge" : "", | 20260507 20:02:08.206 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:08.206 | INFO | "malloc" : " 14 GB", | 20260507 20:02:08.206 | INFO | "pinned" : " 29 GB" | 20260507 20:02:08.206 | INFO | } | 20260507 20:02:08.206 | INFO | Perf report for the last 5.13971 seconds: | 20260507 20:02:08.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:08.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.206 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:02:08.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.207 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 20:02:08.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:08.207 | INFO | Uploaded 36GiB to GPU in 2.49345s (14.4379GiB/s) | 20260507 20:02:08.207 | INFO | Downloaded 34GiB to GPU in 2.42887s (13.9983GiB/s) | 20260507 20:02:08.207 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:02:08.207 | INFO | Total mem usage: 16.4 GiB | 20260507 20:02:08.207 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:02:08.207 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -12%) | 20260507 20:02:08.207 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:02:08.207 | INFO | High Water Breakdown: | 20260507 20:02:08.207 | INFO | 59.6% : 8.73 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:02:08.207 | INFO | 40.3% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:02:08.207 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:02:08.207 | INFO | ------------------------------------------ | 20260507 20:02:08.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:02:08.207 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:02:08.207 | INFO | Total mem usage: 26.5 GiB | 20260507 20:02:08.207 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:02:08.207 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:02:08.207 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:02:08.207 | INFO | High Water Breakdown: | 20260507 20:02:08.207 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:02:08.207 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:02:08.207 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:02:08.207 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:02:08.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:02:08.207 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:02:08.207 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:02:08.207 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:02:08.207 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:02:08.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:02:08.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:02:08.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:02:08.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:02:08.207 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:02:08.207 | INFO | ------------------------------------------ | 20260507 20:02:08.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:02:08.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:02:08.207 | INFO | Total mem usage: 27.8 GiB | 20260507 20:02:08.207 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:02:08.207 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:02:08.207 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:02:08.207 | INFO | High Water Breakdown: | 20260507 20:02:08.207 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:02:08.207 | INFO | ------------------------------------------ | 20260507 20:02:08.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:02:08.207 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:02:08.207 | INFO | Total mem usage: 15.7 GiB | 20260507 20:02:08.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:02:08.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:02:08.207 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:02:08.207 | INFO | High Water Breakdown: | 20260507 20:02:08.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:02:08.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:02:08.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:02:08.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:02:08.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:02:08.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:02:08.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:02:08.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:02:08.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:02:08.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:02:08.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:02:08.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:02:08.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:02:08.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:02:08.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:02:08.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:02:08.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:02:08.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:02:08.208 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:02:08.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:02:08.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:02:08.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:02:08.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:02:08.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:02:08.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:02:08.208 | INFO | ------------------------------------------ | 20260507 20:02:08.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:02:08.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:02:08.208 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:02:08.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:02:08.208 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.58e+01, 1.41e+01 / 2.10e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.03e-01, 1.22e-02 / 5.56e-03 | 20260507 20:02:08.208 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.48e+00, 5.99e-01 / 2.59e-01 | 20260507 20:02:08.208 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.51e+00, 2.10e-01 / 1.08e-01 | 20260507 20:02:08.208 | INFO | PerfReport: 15.84%, DME -- 4.20e+01 / 4.53e+01, 3.85e+00 / 1.71e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.58e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.50e+00 / 2.81e+00, 4.44e-01 / 2.36e-01 | 20260507 20:02:08.208 | INFO | PerfReport: 3.61%, Metrics -- 1.20e+00 / 1.38e+00, 3.05e-01 / 1.61e-01 | 20260507 20:02:08.208 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.74e-01 / 4.07e-01, 6.18e-02 / 3.24e-02 | 20260507 20:02:08.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:02:08.208 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:02:08.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:02:08.208 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 24.31%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 30.99%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 2.00%, LockUpload -- 2.14e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 2.53%, LockDownload -- 2.71e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 20:02:08.208 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260507 20:02:08.318 | INFO | Starting a round of Baz IO | 20260507 20:02:11.815 | INFO | Populating RTMetrics report | 20260507 20:02:11.815 | INFO | RTMetrics report populated | 20260507 20:02:11.815 | INFO | Opening tmp RTMetrics json file | 20260507 20:02:11.817 | INFO | Serializing RTMetrics to json | 20260507 20:02:11.818 | INFO | Serialization done, swapping tmp file now... | 20260507 20:02:11.819 | INFO | RTMetrics done | 20260507 20:02:11.908 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.589 seconds | 20260507 20:02:11.908 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.84% / 0.01% / 55.15% | 20260507 20:02:11.908 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.1593% / 2.78155e-05% / 5.96046e-05% | 20260507 20:02:11.908 | INFO | Baz IO overall: | 20260507 20:02:11.908 | INFO | BytesWritten : 1002160252031 | 20260507 20:02:11.908 | INFO | NumEvents : 485866254978 | 20260507 20:02:13.347 | INFO | Analyzing chunk frames = [2211840,2212352) | 20260507 20:02:13.347 | INFO | MemInfo: { | 20260507 20:02:13.347 | INFO | "gpu" : " 17 GB", | 20260507 20:02:13.347 | INFO | "huge" : "", | 20260507 20:02:13.347 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:13.347 | INFO | "malloc" : " 14 GB", | 20260507 20:02:13.347 | INFO | "pinned" : " 29 GB" | 20260507 20:02:13.347 | INFO | } | 20260507 20:02:13.347 | INFO | Perf report for the last 5.1404 seconds: | 20260507 20:02:13.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:13.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:13.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:13.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:13.347 | INFO | ANALYSIS : 69.6%, 2.9, 104.6ms, 0 | 20260507 20:02:13.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:13.347 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260507 20:02:13.347 | INFO | PRE_HQ : 68.7%, 1.8, 64.9ms, 0 | 20260507 20:02:13.347 | INFO | BAZWRITER : 69.4%, 4.0, 143.8ms, 0 | 20260507 20:02:13.347 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 20:02:13.347 | INFO | Uploaded 36GiB to GPU in 2.58207s (13.9423GiB/s) | 20260507 20:02:13.347 | INFO | Downloaded 38GiB to GPU in 2.72932s (13.9229GiB/s) | 20260507 20:02:18.485 | INFO | Analyzing chunk frames = [2212352,2212864) | 20260507 20:02:18.485 | INFO | MemInfo: { | 20260507 20:02:18.485 | INFO | "gpu" : " 17 GB", | 20260507 20:02:18.485 | INFO | "huge" : "", | 20260507 20:02:18.485 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:18.485 | INFO | "malloc" : " 14 GB", | 20260507 20:02:18.485 | INFO | "pinned" : " 29 GB" | 20260507 20:02:18.485 | INFO | } | 20260507 20:02:18.485 | INFO | Perf report for the last 5.13848 seconds: | 20260507 20:02:18.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:18.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.485 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:02:18.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.485 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:02:18.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:18.486 | INFO | Uploaded 36GiB to GPU in 2.53507s (14.2008GiB/s) | 20260507 20:02:18.486 | INFO | Downloaded 34GiB to GPU in 2.49204s (13.6435GiB/s) | 20260507 20:02:23.623 | INFO | Analyzing chunk frames = [2212864,2213376) | 20260507 20:02:23.623 | INFO | MemInfo: { | 20260507 20:02:23.623 | INFO | "gpu" : " 17 GB", | 20260507 20:02:23.623 | INFO | "huge" : "", | 20260507 20:02:23.623 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:23.623 | INFO | "malloc" : " 14 GB", | 20260507 20:02:23.623 | INFO | "pinned" : " 29 GB" | 20260507 20:02:23.623 | INFO | } | 20260507 20:02:23.623 | INFO | Perf report for the last 5.13793 seconds: | 20260507 20:02:23.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:23.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:02:23.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:02:23.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:23.623 | INFO | Uploaded 36GiB to GPU in 2.53882s (14.1798GiB/s) | 20260507 20:02:23.623 | INFO | Downloaded 34GiB to GPU in 2.48419s (13.6865GiB/s) | 20260507 20:02:28.762 | INFO | Analyzing chunk frames = [2213376,2213888) | 20260507 20:02:28.762 | INFO | MemInfo: { | 20260507 20:02:28.762 | INFO | "gpu" : " 17 GB", | 20260507 20:02:28.762 | INFO | "huge" : "", | 20260507 20:02:28.762 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:28.762 | INFO | "malloc" : " 14 GB", | 20260507 20:02:28.762 | INFO | "pinned" : " 29 GB" | 20260507 20:02:28.762 | INFO | } | 20260507 20:02:28.762 | INFO | Perf report for the last 5.13921 seconds: | 20260507 20:02:28.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:28.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.762 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 20:02:28.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.762 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:02:28.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:28.763 | INFO | Uploaded 36GiB to GPU in 2.44948s (14.697GiB/s) | 20260507 20:02:28.763 | INFO | Downloaded 34GiB to GPU in 2.41496s (14.0789GiB/s) | 20260507 20:02:33.902 | INFO | Analyzing chunk frames = [2213888,2214400) | 20260507 20:02:33.902 | INFO | MemInfo: { | 20260507 20:02:33.902 | INFO | "gpu" : " 17 GB", | 20260507 20:02:33.902 | INFO | "huge" : "", | 20260507 20:02:33.902 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:33.902 | INFO | "malloc" : " 14 GB", | 20260507 20:02:33.902 | INFO | "pinned" : " 29 GB" | 20260507 20:02:33.902 | INFO | } | 20260507 20:02:33.902 | INFO | Perf report for the last 5.13988 seconds: | 20260507 20:02:33.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:33.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.902 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:02:33.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.902 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:02:33.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:33.903 | INFO | Uploaded 36GiB to GPU in 2.34611s (15.3446GiB/s) | 20260507 20:02:33.903 | INFO | Downloaded 34GiB to GPU in 2.50335s (13.5818GiB/s) | 20260507 20:02:39.040 | INFO | Analyzing chunk frames = [2214400,2214912) | 20260507 20:02:39.040 | INFO | MemInfo: { | 20260507 20:02:39.040 | INFO | "gpu" : " 17 GB", | 20260507 20:02:39.041 | INFO | "huge" : "", | 20260507 20:02:39.041 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:39.041 | INFO | "malloc" : " 14 GB", | 20260507 20:02:39.041 | INFO | "pinned" : " 29 GB" | 20260507 20:02:39.041 | INFO | } | 20260507 20:02:39.041 | INFO | Perf report for the last 5.13838 seconds: | 20260507 20:02:39.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:39.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:02:39.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:02:39.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:39.041 | INFO | Uploaded 36GiB to GPU in 2.48728s (14.4736GiB/s) | 20260507 20:02:39.041 | INFO | Downloaded 34GiB to GPU in 2.41597s (14.073GiB/s) | 20260507 20:02:44.181 | INFO | Analyzing chunk frames = [2214912,2215424) | 20260507 20:02:44.181 | INFO | MemInfo: { | 20260507 20:02:44.181 | INFO | "gpu" : " 17 GB", | 20260507 20:02:44.181 | INFO | "huge" : "", | 20260507 20:02:44.181 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:44.181 | INFO | "malloc" : " 14 GB", | 20260507 20:02:44.181 | INFO | "pinned" : " 29 GB" | 20260507 20:02:44.181 | INFO | } | 20260507 20:02:44.181 | INFO | Perf report for the last 5.14049 seconds: | 20260507 20:02:44.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:44.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.181 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:02:44.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.181 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 20:02:44.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:44.182 | INFO | Uploaded 36GiB to GPU in 2.52346s (14.2661GiB/s) | 20260507 20:02:44.182 | INFO | Downloaded 34GiB to GPU in 2.44968s (13.8794GiB/s) | 20260507 20:02:49.320 | INFO | Analyzing chunk frames = [2215424,2215936) | 20260507 20:02:49.320 | INFO | MemInfo: { | 20260507 20:02:49.320 | INFO | "gpu" : " 17 GB", | 20260507 20:02:49.320 | INFO | "huge" : "", | 20260507 20:02:49.320 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:49.320 | INFO | "malloc" : " 14 GB", | 20260507 20:02:49.320 | INFO | "pinned" : " 29 GB" | 20260507 20:02:49.320 | INFO | } | 20260507 20:02:49.321 | INFO | Perf report for the last 5.13938 seconds: | 20260507 20:02:49.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:49.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 20:02:49.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 20:02:49.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:49.321 | INFO | Uploaded 36GiB to GPU in 2.50888s (14.349GiB/s) | 20260507 20:02:49.321 | INFO | Downloaded 34GiB to GPU in 2.44021s (13.9333GiB/s) | 20260507 20:02:49.435 | INFO | Starting a round of Baz IO | 20260507 20:02:52.909 | INFO | Populating RTMetrics report | 20260507 20:02:52.909 | INFO | RTMetrics report populated | 20260507 20:02:52.909 | INFO | Opening tmp RTMetrics json file | 20260507 20:02:52.911 | INFO | Serializing RTMetrics to json | 20260507 20:02:52.913 | INFO | Serialization done, swapping tmp file now... | 20260507 20:02:52.913 | INFO | RTMetrics done | 20260507 20:02:53.013 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.578 seconds | 20260507 20:02:53.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.83% / 0.01% / 55.16% | 20260507 20:02:53.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.15724% / 1.19209e-05% / 5.16574e-05% | 20260507 20:02:53.013 | INFO | Baz IO overall: | 20260507 20:02:53.013 | INFO | BytesWritten : 1004076060562 | 20260507 20:02:53.013 | INFO | NumEvents : 486794267662 | 20260507 20:02:54.460 | INFO | Analyzing chunk frames = [2215936,2216448) | 20260507 20:02:54.460 | INFO | MemInfo: { | 20260507 20:02:54.460 | INFO | "gpu" : " 17 GB", | 20260507 20:02:54.460 | INFO | "huge" : "", | 20260507 20:02:54.460 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:54.460 | INFO | "malloc" : " 14 GB", | 20260507 20:02:54.460 | INFO | "pinned" : " 29 GB" | 20260507 20:02:54.460 | INFO | } | 20260507 20:02:54.460 | INFO | Perf report for the last 5.13949 seconds: | 20260507 20:02:54.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:54.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:54.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:54.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:54.460 | INFO | ANALYSIS : 69.2%, 2.9, 104.4ms, 0 | 20260507 20:02:54.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:54.460 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 20:02:54.460 | INFO | PRE_HQ : 68.2%, 1.8, 64.7ms, 0 | 20260507 20:02:54.460 | INFO | BAZWRITER : 69.0%, 4.3, 151.9ms, 0 | 20260507 20:02:54.460 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 20:02:54.460 | INFO | Uploaded 36GiB to GPU in 2.51498s (14.3142GiB/s) | 20260507 20:02:54.460 | INFO | Downloaded 38GiB to GPU in 2.6694s (14.2354GiB/s) | 20260507 20:02:59.600 | INFO | Analyzing chunk frames = [2216448,2216960) | 20260507 20:02:59.600 | INFO | MemInfo: { | 20260507 20:02:59.600 | INFO | "gpu" : " 17 GB", | 20260507 20:02:59.600 | INFO | "huge" : "", | 20260507 20:02:59.600 | INFO | "huge_shared" : " 29 GB", | 20260507 20:02:59.600 | INFO | "malloc" : " 14 GB", | 20260507 20:02:59.600 | INFO | "pinned" : " 29 GB" | 20260507 20:02:59.600 | INFO | } | 20260507 20:02:59.601 | INFO | Perf report for the last 5.14048 seconds: | 20260507 20:02:59.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:02:59.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 20:02:59.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:02:59.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:02:59.601 | INFO | Uploaded 36GiB to GPU in 2.59432s (13.8765GiB/s) | 20260507 20:02:59.601 | INFO | Downloaded 34GiB to GPU in 2.4648s (13.7942GiB/s) | 20260507 20:03:04.738 | INFO | Analyzing chunk frames = [2216960,2217472) | 20260507 20:03:04.738 | INFO | MemInfo: { | 20260507 20:03:04.738 | INFO | "gpu" : " 17 GB", | 20260507 20:03:04.738 | INFO | "huge" : "", | 20260507 20:03:04.738 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:04.738 | INFO | "malloc" : " 14 GB", | 20260507 20:03:04.738 | INFO | "pinned" : " 29 GB" | 20260507 20:03:04.738 | INFO | } | 20260507 20:03:04.738 | INFO | Perf report for the last 5.13782 seconds: | 20260507 20:03:04.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:04.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.738 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260507 20:03:04.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.738 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:03:04.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:04.739 | INFO | Uploaded 36GiB to GPU in 2.59943s (13.8492GiB/s) | 20260507 20:03:04.739 | INFO | Downloaded 34GiB to GPU in 2.4626s (13.8065GiB/s) | 20260507 20:03:09.878 | INFO | Analyzing chunk frames = [2217472,2217984) | 20260507 20:03:09.878 | INFO | MemInfo: { | 20260507 20:03:09.878 | INFO | "gpu" : " 17 GB", | 20260507 20:03:09.878 | INFO | "huge" : "", | 20260507 20:03:09.878 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:09.878 | INFO | "malloc" : " 14 GB", | 20260507 20:03:09.878 | INFO | "pinned" : " 29 GB" | 20260507 20:03:09.878 | INFO | } | 20260507 20:03:09.878 | INFO | Perf report for the last 5.14 seconds: | 20260507 20:03:09.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:09.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.878 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260507 20:03:09.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.878 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:03:09.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:09.879 | INFO | Uploaded 36GiB to GPU in 2.49642s (14.4206GiB/s) | 20260507 20:03:09.879 | INFO | Downloaded 34GiB to GPU in 2.38927s (14.2303GiB/s) | 20260507 20:03:15.017 | INFO | Analyzing chunk frames = [2217984,2218496) | 20260507 20:03:15.017 | INFO | MemInfo: { | 20260507 20:03:15.017 | INFO | "gpu" : " 17 GB", | 20260507 20:03:15.017 | INFO | "huge" : "", | 20260507 20:03:15.017 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:15.017 | INFO | "malloc" : " 14 GB", | 20260507 20:03:15.017 | INFO | "pinned" : " 29 GB" | 20260507 20:03:15.017 | INFO | } | 20260507 20:03:15.017 | INFO | Perf report for the last 5.13909 seconds: | 20260507 20:03:15.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:15.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 20:03:15.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:03:15.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:15.018 | INFO | Uploaded 36GiB to GPU in 2.4722s (14.562GiB/s) | 20260507 20:03:15.018 | INFO | Downloaded 34GiB to GPU in 2.49982s (13.601GiB/s) | 20260507 20:03:20.157 | INFO | Analyzing chunk frames = [2218496,2219008) | 20260507 20:03:20.157 | INFO | MemInfo: { | 20260507 20:03:20.157 | INFO | "gpu" : " 17 GB", | 20260507 20:03:20.157 | INFO | "huge" : "", | 20260507 20:03:20.157 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:20.157 | INFO | "malloc" : " 14 GB", | 20260507 20:03:20.157 | INFO | "pinned" : " 29 GB" | 20260507 20:03:20.157 | INFO | } | 20260507 20:03:20.157 | INFO | Perf report for the last 5.13969 seconds: | 20260507 20:03:20.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:20.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.157 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:03:20.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.157 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:03:20.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:20.158 | INFO | Uploaded 36GiB to GPU in 2.48183s (14.5054GiB/s) | 20260507 20:03:20.158 | INFO | Downloaded 34GiB to GPU in 2.4203s (14.0478GiB/s) | 20260507 20:03:25.295 | INFO | Analyzing chunk frames = [2219008,2219520) | 20260507 20:03:25.295 | INFO | MemInfo: { | 20260507 20:03:25.295 | INFO | "gpu" : " 17 GB", | 20260507 20:03:25.295 | INFO | "huge" : "", | 20260507 20:03:25.296 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:25.296 | INFO | "malloc" : " 14 GB", | 20260507 20:03:25.296 | INFO | "pinned" : " 29 GB" | 20260507 20:03:25.296 | INFO | } | 20260507 20:03:25.296 | INFO | Perf report for the last 5.13837 seconds: | 20260507 20:03:25.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:25.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:03:25.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 20:03:25.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:25.296 | INFO | Uploaded 36GiB to GPU in 2.51695s (14.303GiB/s) | 20260507 20:03:25.296 | INFO | Downloaded 34GiB to GPU in 2.45169s (13.868GiB/s) | 20260507 20:03:30.435 | INFO | Analyzing chunk frames = [2219520,2220032) | 20260507 20:03:30.435 | INFO | MemInfo: { | 20260507 20:03:30.435 | INFO | "gpu" : " 17 GB", | 20260507 20:03:30.435 | INFO | "huge" : "", | 20260507 20:03:30.435 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:30.435 | INFO | "malloc" : " 14 GB", | 20260507 20:03:30.435 | INFO | "pinned" : " 29 GB" | 20260507 20:03:30.435 | INFO | } | 20260507 20:03:30.435 | INFO | Perf report for the last 5.1395 seconds: | 20260507 20:03:30.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:30.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.435 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:03:30.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.436 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 20:03:30.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:30.436 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:03:30.436 | INFO | Total mem usage: 16.4 GiB | 20260507 20:03:30.436 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:03:30.436 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.4%) | 20260507 20:03:30.436 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:03:30.436 | INFO | High Water Breakdown: | 20260507 20:03:30.436 | INFO | 59.8% : 8.89 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:03:30.436 | INFO | 40% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:03:30.436 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:03:30.436 | INFO | ------------------------------------------ | 20260507 20:03:30.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:03:30.436 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:03:30.436 | INFO | Total mem usage: 26.5 GiB | 20260507 20:03:30.436 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:03:30.436 | INFO | Uploaded 36GiB to GPU in 2.49704s (14.4171GiB/s) | 20260507 20:03:30.436 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:03:30.436 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 20:03:30.436 | INFO | High Water Breakdown: | 20260507 20:03:30.436 | INFO | Downloaded 34GiB to GPU in 2.43041s (13.9894GiB/s) | 20260507 20:03:30.436 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:03:30.436 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:03:30.436 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:03:30.436 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:03:30.436 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:03:30.436 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:03:30.436 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:03:30.436 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:03:30.436 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:03:30.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:03:30.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:03:30.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:03:30.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:03:30.436 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:03:30.436 | INFO | ------------------------------------------ | 20260507 20:03:30.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:03:30.436 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:03:30.436 | INFO | Total mem usage: 27.8 GiB | 20260507 20:03:30.436 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:03:30.436 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:03:30.436 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:03:30.436 | INFO | High Water Breakdown: | 20260507 20:03:30.436 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:03:30.436 | INFO | ------------------------------------------ | 20260507 20:03:30.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:03:30.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:03:30.437 | INFO | Total mem usage: 15.7 GiB | 20260507 20:03:30.437 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:03:30.437 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:03:30.437 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:03:30.437 | INFO | High Water Breakdown: | 20260507 20:03:30.437 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:03:30.437 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:03:30.437 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:03:30.437 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:03:30.437 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:03:30.437 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:03:30.437 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:03:30.437 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:03:30.437 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:03:30.437 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:03:30.437 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:03:30.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:03:30.437 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:03:30.437 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:03:30.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:03:30.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:03:30.437 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:03:30.437 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:03:30.437 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:03:30.437 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:03:30.437 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:03:30.437 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:03:30.437 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:03:30.437 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:03:30.437 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:03:30.437 | INFO | ------------------------------------------ | 20260507 20:03:30.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:03:30.437 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:03:30.437 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:03:30.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:03:30.437 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.42e+01 / 1.79e+00 | 20260507 20:03:30.437 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.04e-01, 1.26e-02 / 6.96e-03 | 20260507 20:03:30.437 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.44e+00, 6.00e-01 / 3.04e-01 | 20260507 20:03:30.437 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.13e-01 / 1.21e-01 | 20260507 20:03:30.437 | INFO | PerfReport: 15.73%, DME -- 4.17e+01 / 4.49e+01, 3.88e+00 / 1.14e+00 | 20260507 20:03:30.437 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.85e+00 / 1.86e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.50e+00 / 2.80e+00, 4.39e-01 / 2.71e-01 | 20260507 20:03:30.438 | INFO | PerfReport: 3.61%, Metrics -- 1.20e+00 / 1.37e+00, 3.05e-01 / 1.59e-01 | 20260507 20:03:30.438 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.74e-01 / 4.06e-01, 6.15e-02 / 3.46e-02 | 20260507 20:03:30.438 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:03:30.438 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:03:30.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:03:30.438 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 31.08%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 2.36%, LockDownload -- 2.52e+00 / 0.00e+00, 3.77e+00 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 15.87%, LockCompute -- 1.69e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:03:30.438 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 20:03:30.551 | INFO | Starting a round of Baz IO | 20260507 20:03:34.018 | INFO | Populating RTMetrics report | 20260507 20:03:34.019 | INFO | RTMetrics report populated | 20260507 20:03:34.019 | INFO | Opening tmp RTMetrics json file | 20260507 20:03:34.032 | INFO | Serializing RTMetrics to json | 20260507 20:03:34.048 | INFO | Serialization done, swapping tmp file now... | 20260507 20:03:34.049 | INFO | RTMetrics done | 20260507 20:03:34.122 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.571 seconds | 20260507 20:03:34.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.82% / 0.01% / 55.17% | 20260507 20:03:34.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.15209% / 7.94729e-06% / 6.35783e-05% | 20260507 20:03:34.123 | INFO | Baz IO overall: | 20260507 20:03:34.123 | INFO | BytesWritten : 1005990486012 | 20260507 20:03:34.123 | INFO | NumEvents : 487721312625 | 20260507 20:03:35.575 | INFO | Analyzing chunk frames = [2220032,2220544) | 20260507 20:03:35.575 | INFO | MemInfo: { | 20260507 20:03:35.575 | INFO | "gpu" : " 17 GB", | 20260507 20:03:35.575 | INFO | "huge" : "", | 20260507 20:03:35.575 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:35.575 | INFO | "malloc" : " 14 GB", | 20260507 20:03:35.575 | INFO | "pinned" : " 29 GB" | 20260507 20:03:35.575 | INFO | } | 20260507 20:03:35.575 | INFO | Perf report for the last 5.14011 seconds: | 20260507 20:03:35.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:35.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:35.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:35.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:35.575 | INFO | ANALYSIS : 69.0%, 2.9, 104.6ms, 0 | 20260507 20:03:35.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:35.575 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 20:03:35.575 | INFO | PRE_HQ : 68.1%, 1.8, 65.0ms, 0 | 20260507 20:03:35.575 | INFO | BAZWRITER : 69.0%, 4.1, 146.0ms, 0 | 20260507 20:03:35.575 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 20:03:35.575 | INFO | Uploaded 36GiB to GPU in 2.568s (14.0187GiB/s) | 20260507 20:03:35.576 | INFO | Downloaded 38GiB to GPU in 2.71474s (13.9977GiB/s) | 20260507 20:03:40.714 | INFO | Analyzing chunk frames = [2220544,2221056) | 20260507 20:03:40.714 | INFO | MemInfo: { | 20260507 20:03:40.714 | INFO | "gpu" : " 17 GB", | 20260507 20:03:40.714 | INFO | "huge" : "", | 20260507 20:03:40.715 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:40.715 | INFO | "malloc" : " 14 GB", | 20260507 20:03:40.715 | INFO | "pinned" : " 29 GB" | 20260507 20:03:40.715 | INFO | } | 20260507 20:03:40.715 | INFO | Perf report for the last 5.13953 seconds: | 20260507 20:03:40.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:40.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 20:03:40.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:03:40.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:40.715 | INFO | Uploaded 36GiB to GPU in 2.51428s (14.3182GiB/s) | 20260507 20:03:40.715 | INFO | Downloaded 34GiB to GPU in 2.47632s (13.7301GiB/s) | 20260507 20:03:45.853 | INFO | Analyzing chunk frames = [2221056,2221568) | 20260507 20:03:45.853 | INFO | MemInfo: { | 20260507 20:03:45.853 | INFO | "gpu" : " 17 GB", | 20260507 20:03:45.853 | INFO | "huge" : "", | 20260507 20:03:45.853 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:45.853 | INFO | "malloc" : " 14 GB", | 20260507 20:03:45.853 | INFO | "pinned" : " 29 GB" | 20260507 20:03:45.853 | INFO | } | 20260507 20:03:45.853 | INFO | Perf report for the last 5.13852 seconds: | 20260507 20:03:45.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:45.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.853 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:03:45.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.853 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:03:45.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:45.854 | INFO | Uploaded 36GiB to GPU in 2.54393s (14.1513GiB/s) | 20260507 20:03:45.854 | INFO | Downloaded 34GiB to GPU in 2.47272s (13.75GiB/s) | 20260507 20:03:50.992 | INFO | Analyzing chunk frames = [2221568,2222080) | 20260507 20:03:50.992 | INFO | MemInfo: { | 20260507 20:03:50.992 | INFO | "gpu" : " 17 GB", | 20260507 20:03:50.992 | INFO | "huge" : "", | 20260507 20:03:50.992 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:50.992 | INFO | "malloc" : " 14 GB", | 20260507 20:03:50.992 | INFO | "pinned" : " 29 GB" | 20260507 20:03:50.992 | INFO | } | 20260507 20:03:50.992 | INFO | Perf report for the last 5.13881 seconds: | 20260507 20:03:50.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:50.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 20:03:50.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:03:50.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:50.992 | INFO | Uploaded 36GiB to GPU in 2.46044s (14.6315GiB/s) | 20260507 20:03:50.992 | INFO | Downloaded 34GiB to GPU in 2.39951s (14.1695GiB/s) | 20260507 20:03:56.131 | INFO | Analyzing chunk frames = [2222080,2222592) | 20260507 20:03:56.131 | INFO | MemInfo: { | 20260507 20:03:56.131 | INFO | "gpu" : " 17 GB", | 20260507 20:03:56.131 | INFO | "huge" : "", | 20260507 20:03:56.131 | INFO | "huge_shared" : " 29 GB", | 20260507 20:03:56.131 | INFO | "malloc" : " 14 GB", | 20260507 20:03:56.132 | INFO | "pinned" : " 29 GB" | 20260507 20:03:56.132 | INFO | } | 20260507 20:03:56.132 | INFO | Perf report for the last 5.13986 seconds: | 20260507 20:03:56.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:03:56.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:03:56.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:03:56.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:03:56.132 | INFO | Uploaded 36GiB to GPU in 2.36846s (15.1997GiB/s) | 20260507 20:03:56.132 | INFO | Downloaded 34GiB to GPU in 2.5078s (13.5577GiB/s) | 20260507 20:04:01.272 | INFO | Analyzing chunk frames = [2222592,2223104) | 20260507 20:04:01.272 | INFO | MemInfo: { | 20260507 20:04:01.272 | INFO | "gpu" : " 17 GB", | 20260507 20:04:01.272 | INFO | "huge" : "", | 20260507 20:04:01.272 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:01.273 | INFO | "malloc" : " 14 GB", | 20260507 20:04:01.273 | INFO | "pinned" : " 29 GB" | 20260507 20:04:01.273 | INFO | } | 20260507 20:04:01.273 | INFO | Perf report for the last 5.14074 seconds: | 20260507 20:04:01.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:01.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 20:04:01.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:04:01.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:01.273 | INFO | Uploaded 36GiB to GPU in 2.52285s (14.2696GiB/s) | 20260507 20:04:01.273 | INFO | Downloaded 34GiB to GPU in 2.41376s (14.0859GiB/s) | 20260507 20:04:06.412 | INFO | Analyzing chunk frames = [2223104,2223616) | 20260507 20:04:06.412 | INFO | MemInfo: { | 20260507 20:04:06.412 | INFO | "gpu" : " 17 GB", | 20260507 20:04:06.412 | INFO | "huge" : "", | 20260507 20:04:06.412 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:06.412 | INFO | "malloc" : " 14 GB", | 20260507 20:04:06.412 | INFO | "pinned" : " 29 GB" | 20260507 20:04:06.412 | INFO | } | 20260507 20:04:06.412 | INFO | Perf report for the last 5.13926 seconds: | 20260507 20:04:06.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:06.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.412 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 20:04:06.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.412 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:04:06.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:06.413 | INFO | Uploaded 36GiB to GPU in 2.53238s (14.2159GiB/s) | 20260507 20:04:06.413 | INFO | Downloaded 34GiB to GPU in 2.43308s (13.974GiB/s) | 20260507 20:04:11.550 | INFO | Analyzing chunk frames = [2223616,2224128) | 20260507 20:04:11.550 | INFO | MemInfo: { | 20260507 20:04:11.550 | INFO | "gpu" : " 17 GB", | 20260507 20:04:11.550 | INFO | "huge" : "", | 20260507 20:04:11.550 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:11.550 | INFO | "malloc" : " 14 GB", | 20260507 20:04:11.551 | INFO | "pinned" : " 29 GB" | 20260507 20:04:11.551 | INFO | } | 20260507 20:04:11.551 | INFO | Perf report for the last 5.1386 seconds: | 20260507 20:04:11.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:11.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:04:11.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:04:11.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:11.551 | INFO | Uploaded 36GiB to GPU in 2.51345s (14.323GiB/s) | 20260507 20:04:11.551 | INFO | Downloaded 34GiB to GPU in 2.4323s (13.9786GiB/s) | 20260507 20:04:11.648 | INFO | Starting a round of Baz IO | 20260507 20:04:15.128 | INFO | Populating RTMetrics report | 20260507 20:04:15.128 | INFO | RTMetrics report populated | 20260507 20:04:15.128 | INFO | Opening tmp RTMetrics json file | 20260507 20:04:15.134 | INFO | Serializing RTMetrics to json | 20260507 20:04:15.220 | INFO | Serialization done, swapping tmp file now... | 20260507 20:04:15.227 | INFO | RTMetrics done | 20260507 20:04:15.244 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.595 seconds | 20260507 20:04:15.244 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.81% / 0.01% / 55.18% | 20260507 20:04:15.244 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.14712% / 7.94729e-06% / 6.75519e-05% | 20260507 20:04:15.244 | INFO | Baz IO overall: | 20260507 20:04:15.244 | INFO | BytesWritten : 1007905021742 | 20260507 20:04:15.244 | INFO | NumEvents : 488648313040 | 20260507 20:04:16.689 | INFO | Analyzing chunk frames = [2224128,2224640) | 20260507 20:04:16.689 | INFO | MemInfo: { | 20260507 20:04:16.689 | INFO | "gpu" : " 17 GB", | 20260507 20:04:16.689 | INFO | "huge" : "", | 20260507 20:04:16.689 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:16.689 | INFO | "malloc" : " 14 GB", | 20260507 20:04:16.689 | INFO | "pinned" : " 29 GB" | 20260507 20:04:16.689 | INFO | } | 20260507 20:04:16.689 | INFO | Perf report for the last 5.13818 seconds: | 20260507 20:04:16.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:16.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:16.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:16.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:16.689 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 20:04:16.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:16.689 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 20:04:16.689 | INFO | PRE_HQ : 66.5%, 1.7, 58.7ms, 0 | 20260507 20:04:16.689 | INFO | BAZWRITER : 69.6%, 3.8, 137.3ms, 0 | 20260507 20:04:16.689 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260507 20:04:16.689 | INFO | Uploaded 36GiB to GPU in 2.49125s (14.4506GiB/s) | 20260507 20:04:16.690 | INFO | Downloaded 38GiB to GPU in 2.67981s (14.1801GiB/s) | 20260507 20:04:21.829 | INFO | Analyzing chunk frames = [2224640,2225152) | 20260507 20:04:21.829 | INFO | MemInfo: { | 20260507 20:04:21.829 | INFO | "gpu" : " 17 GB", | 20260507 20:04:21.829 | INFO | "huge" : "", | 20260507 20:04:21.829 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:21.829 | INFO | "malloc" : " 14 GB", | 20260507 20:04:21.829 | INFO | "pinned" : " 29 GB" | 20260507 20:04:21.829 | INFO | } | 20260507 20:04:21.829 | INFO | Perf report for the last 5.14026 seconds: | 20260507 20:04:21.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:21.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 20:04:21.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:04:21.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:21.829 | INFO | Uploaded 36GiB to GPU in 2.55365s (14.0975GiB/s) | 20260507 20:04:21.829 | INFO | Downloaded 34GiB to GPU in 2.48335s (13.6912GiB/s) | 20260507 20:04:26.968 | INFO | Analyzing chunk frames = [2225152,2225664) | 20260507 20:04:26.968 | INFO | MemInfo: { | 20260507 20:04:26.968 | INFO | "gpu" : " 17 GB", | 20260507 20:04:26.968 | INFO | "huge" : "", | 20260507 20:04:26.968 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:26.968 | INFO | "malloc" : " 14 GB", | 20260507 20:04:26.968 | INFO | "pinned" : " 29 GB" | 20260507 20:04:26.968 | INFO | } | 20260507 20:04:26.968 | INFO | Perf report for the last 5.13879 seconds: | 20260507 20:04:26.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:26.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 20:04:26.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:04:26.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:26.968 | INFO | Uploaded 36GiB to GPU in 2.58336s (13.9354GiB/s) | 20260507 20:04:26.968 | INFO | Downloaded 34GiB to GPU in 2.48329s (13.6915GiB/s) | 20260507 20:04:32.107 | INFO | Analyzing chunk frames = [2225664,2226176) | 20260507 20:04:32.107 | INFO | MemInfo: { | 20260507 20:04:32.107 | INFO | "gpu" : " 17 GB", | 20260507 20:04:32.107 | INFO | "huge" : "", | 20260507 20:04:32.107 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:32.107 | INFO | "malloc" : " 14 GB", | 20260507 20:04:32.107 | INFO | "pinned" : " 29 GB" | 20260507 20:04:32.107 | INFO | } | 20260507 20:04:32.107 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:04:32.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:32.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 20:04:32.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:04:32.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:32.107 | INFO | Uploaded 36GiB to GPU in 2.46764s (14.5888GiB/s) | 20260507 20:04:32.107 | INFO | Downloaded 34GiB to GPU in 2.40383s (14.1441GiB/s) | 20260507 20:04:37.246 | INFO | Analyzing chunk frames = [2226176,2226688) | 20260507 20:04:37.246 | INFO | MemInfo: { | 20260507 20:04:37.246 | INFO | "gpu" : " 17 GB", | 20260507 20:04:37.247 | INFO | "huge" : "", | 20260507 20:04:37.247 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:37.247 | INFO | "malloc" : " 14 GB", | 20260507 20:04:37.247 | INFO | "pinned" : " 29 GB" | 20260507 20:04:37.247 | INFO | } | 20260507 20:04:37.247 | INFO | Perf report for the last 5.14014 seconds: | 20260507 20:04:37.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:37.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:04:37.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:04:37.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:37.247 | INFO | Uploaded 36GiB to GPU in 2.35611s (15.2794GiB/s) | 20260507 20:04:37.247 | INFO | Downloaded 34GiB to GPU in 2.50397s (13.5784GiB/s) | 20260507 20:04:42.386 | INFO | Analyzing chunk frames = [2226688,2227200) | 20260507 20:04:42.386 | INFO | MemInfo: { | 20260507 20:04:42.386 | INFO | "gpu" : " 17 GB", | 20260507 20:04:42.386 | INFO | "huge" : "", | 20260507 20:04:42.386 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:42.386 | INFO | "malloc" : " 14 GB", | 20260507 20:04:42.386 | INFO | "pinned" : " 29 GB" | 20260507 20:04:42.386 | INFO | } | 20260507 20:04:42.386 | INFO | Perf report for the last 5.13919 seconds: | 20260507 20:04:42.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:42.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.386 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:04:42.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.386 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:04:42.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:42.387 | INFO | Uploaded 36GiB to GPU in 2.49807s (14.4111GiB/s) | 20260507 20:04:42.387 | INFO | Downloaded 34GiB to GPU in 2.42678s (14.0103GiB/s) | 20260507 20:04:47.527 | INFO | Analyzing chunk frames = [2227200,2227712) | 20260507 20:04:47.527 | INFO | MemInfo: { | 20260507 20:04:47.527 | INFO | "gpu" : " 17 GB", | 20260507 20:04:47.527 | INFO | "huge" : "", | 20260507 20:04:47.527 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:47.527 | INFO | "malloc" : " 14 GB", | 20260507 20:04:47.527 | INFO | "pinned" : " 29 GB" | 20260507 20:04:47.527 | INFO | } | 20260507 20:04:47.527 | INFO | Perf report for the last 5.14071 seconds: | 20260507 20:04:47.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:47.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.527 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:04:47.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.527 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:04:47.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:47.528 | INFO | Uploaded 36GiB to GPU in 2.51842s (14.2947GiB/s) | 20260507 20:04:47.528 | INFO | Downloaded 34GiB to GPU in 2.44488s (13.9066GiB/s) | 20260507 20:04:52.665 | INFO | Analyzing chunk frames = [2227712,2228224) | 20260507 20:04:52.665 | INFO | MemInfo: { | 20260507 20:04:52.665 | INFO | "gpu" : " 17 GB", | 20260507 20:04:52.665 | INFO | "huge" : "", | 20260507 20:04:52.665 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:52.665 | INFO | "malloc" : " 14 GB", | 20260507 20:04:52.665 | INFO | "pinned" : " 29 GB" | 20260507 20:04:52.665 | INFO | } | 20260507 20:04:52.666 | INFO | Perf report for the last 5.13852 seconds: | 20260507 20:04:52.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:52.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:04:52.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 20:04:52.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:52.666 | INFO | Uploaded 36GiB to GPU in 2.5132s (14.3244GiB/s) | 20260507 20:04:52.666 | INFO | Downloaded 34GiB to GPU in 2.43872s (13.9418GiB/s) | 20260507 20:04:52.666 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:04:52.666 | INFO | Total mem usage: 16.4 GiB | 20260507 20:04:52.666 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:04:52.666 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.71%) | 20260507 20:04:52.666 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:04:52.666 | INFO | High Water Breakdown: | 20260507 20:04:52.666 | INFO | 60.1% : 9.07 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:04:52.666 | INFO | 39.8% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:04:52.666 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:04:52.666 | INFO | ------------------------------------------ | 20260507 20:04:52.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:04:52.666 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:04:52.666 | INFO | Total mem usage: 26.5 GiB | 20260507 20:04:52.666 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:04:52.666 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:04:52.666 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:04:52.666 | INFO | High Water Breakdown: | 20260507 20:04:52.666 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:04:52.666 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:04:52.666 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:04:52.666 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:04:52.666 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:04:52.666 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:04:52.666 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:04:52.666 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:04:52.666 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:04:52.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:04:52.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:04:52.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:04:52.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:04:52.666 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:04:52.666 | INFO | ------------------------------------------ | 20260507 20:04:52.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:04:52.666 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:04:52.666 | INFO | Total mem usage: 27.8 GiB | 20260507 20:04:52.666 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:04:52.666 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:04:52.666 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:04:52.666 | INFO | High Water Breakdown: | 20260507 20:04:52.666 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:04:52.666 | INFO | ------------------------------------------ | 20260507 20:04:52.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:04:52.666 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:04:52.666 | INFO | Total mem usage: 15.7 GiB | 20260507 20:04:52.666 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:04:52.666 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:04:52.666 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:04:52.666 | INFO | High Water Breakdown: | 20260507 20:04:52.666 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:04:52.666 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:04:52.666 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:04:52.666 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:04:52.666 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:04:52.666 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:04:52.666 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:04:52.666 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:04:52.666 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:04:52.666 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:04:52.666 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:04:52.666 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:04:52.666 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:04:52.666 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:04:52.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:04:52.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:04:52.666 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:04:52.666 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:04:52.666 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:04:52.666 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:04:52.666 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:04:52.666 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:04:52.666 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:04:52.666 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:04:52.666 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:04:52.666 | INFO | ------------------------------------------ | 20260507 20:04:52.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:04:52.666 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:04:52.666 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:04:52.666 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:04:52.666 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.54e+01, 1.40e+01 / 1.66e+00 | 20260507 20:04:52.666 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.20e-02 / 4.34e-03 | 20260507 20:04:52.666 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.47e+00, 6.03e-01 / 3.06e-01 | 20260507 20:04:52.667 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.48e+00, 2.08e-01 / 1.34e-01 | 20260507 20:04:52.667 | INFO | PerfReport: 15.75%, DME -- 4.17e+01 / 4.47e+01, 3.83e+00 / 1.25e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.79e+00 / 1.79e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.40e-01 / 2.27e-01 | 20260507 20:04:52.667 | INFO | PerfReport: 3.63%, Metrics -- 1.20e+00 / 1.40e+00, 3.08e-01 / 1.73e-01 | 20260507 20:04:52.667 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.04e-01, 6.09e-02 / 3.23e-02 | 20260507 20:04:52.667 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:04:52.667 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:04:52.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:04:52.667 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 24.20%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 2.06%, LockUpload -- 2.20e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 2.34%, LockDownload -- 2.50e+00 / 0.00e+00, 3.78e+00 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 16.01%, LockCompute -- 1.71e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 20:04:52.667 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 3.29e-02 / 0.00e+00 | 20260507 20:04:52.763 | INFO | Starting a round of Baz IO | 20260507 20:04:56.246 | INFO | Populating RTMetrics report | 20260507 20:04:56.246 | INFO | RTMetrics report populated | 20260507 20:04:56.246 | INFO | Opening tmp RTMetrics json file | 20260507 20:04:56.275 | INFO | Serializing RTMetrics to json | 20260507 20:04:56.336 | INFO | Serialization done, swapping tmp file now... | 20260507 20:04:56.341 | INFO | RTMetrics done | 20260507 20:04:56.366 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.602 seconds | 20260507 20:04:56.366 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.80% / 0.01% / 55.19% | 20260507 20:04:56.366 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.14547% / 1.98682e-05% / 3.57628e-05% | 20260507 20:04:56.366 | INFO | Baz IO overall: | 20260507 20:04:56.366 | INFO | BytesWritten : 1009820603936 | 20260507 20:04:56.366 | INFO | NumEvents : 489576244857 | 20260507 20:04:57.803 | INFO | Analyzing chunk frames = [2228224,2228736) | 20260507 20:04:57.803 | INFO | MemInfo: { | 20260507 20:04:57.803 | INFO | "gpu" : " 17 GB", | 20260507 20:04:57.803 | INFO | "huge" : "", | 20260507 20:04:57.803 | INFO | "huge_shared" : " 29 GB", | 20260507 20:04:57.803 | INFO | "malloc" : " 14 GB", | 20260507 20:04:57.803 | INFO | "pinned" : " 29 GB" | 20260507 20:04:57.803 | INFO | } | 20260507 20:04:57.803 | INFO | Perf report for the last 5.13797 seconds: | 20260507 20:04:57.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:04:57.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:57.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:57.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:57.804 | INFO | ANALYSIS : 68.9%, 3.0, 106.9ms, 0 | 20260507 20:04:57.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:04:57.804 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 20:04:57.804 | INFO | PRE_HQ : 66.5%, 1.7, 58.7ms, 0 | 20260507 20:04:57.804 | INFO | BAZWRITER : 69.7%, 4.5, 162.4ms, 0 | 20260507 20:04:57.804 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 20:04:57.804 | INFO | Uploaded 36GiB to GPU in 2.49302s (14.4403GiB/s) | 20260507 20:04:57.804 | INFO | Downloaded 38GiB to GPU in 2.68644s (14.1451GiB/s) | 20260507 20:05:02.943 | INFO | Analyzing chunk frames = [2228736,2229248) | 20260507 20:05:02.943 | INFO | MemInfo: { | 20260507 20:05:02.943 | INFO | "gpu" : " 17 GB", | 20260507 20:05:02.943 | INFO | "huge" : "", | 20260507 20:05:02.943 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:02.943 | INFO | "malloc" : " 14 GB", | 20260507 20:05:02.943 | INFO | "pinned" : " 29 GB" | 20260507 20:05:02.943 | INFO | } | 20260507 20:05:02.943 | INFO | Perf report for the last 5.13985 seconds: | 20260507 20:05:02.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:02.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.943 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 20:05:02.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.944 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:05:02.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:02.944 | INFO | Uploaded 36GiB to GPU in 2.51043s (14.3402GiB/s) | 20260507 20:05:02.944 | INFO | Downloaded 34GiB to GPU in 2.47749s (13.7236GiB/s) | 20260507 20:05:08.083 | INFO | Analyzing chunk frames = [2229248,2229760) | 20260507 20:05:08.083 | INFO | MemInfo: { | 20260507 20:05:08.083 | INFO | "gpu" : " 17 GB", | 20260507 20:05:08.083 | INFO | "huge" : "", | 20260507 20:05:08.083 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:08.083 | INFO | "malloc" : " 14 GB", | 20260507 20:05:08.083 | INFO | "pinned" : " 29 GB" | 20260507 20:05:08.083 | INFO | } | 20260507 20:05:08.083 | INFO | Perf report for the last 5.13944 seconds: | 20260507 20:05:08.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:08.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 20:05:08.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:05:08.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:08.083 | INFO | Uploaded 36GiB to GPU in 2.5917s (13.8905GiB/s) | 20260507 20:05:08.083 | INFO | Downloaded 34GiB to GPU in 2.46169s (13.8116GiB/s) | 20260507 20:05:13.223 | INFO | Analyzing chunk frames = [2229760,2230272) | 20260507 20:05:13.223 | INFO | MemInfo: { | 20260507 20:05:13.223 | INFO | "gpu" : " 17 GB", | 20260507 20:05:13.223 | INFO | "huge" : "", | 20260507 20:05:13.224 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:13.224 | INFO | "malloc" : " 14 GB", | 20260507 20:05:13.224 | INFO | "pinned" : " 29 GB" | 20260507 20:05:13.224 | INFO | } | 20260507 20:05:13.224 | INFO | Perf report for the last 5.14083 seconds: | 20260507 20:05:13.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:13.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260507 20:05:13.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:05:13.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:13.224 | INFO | Uploaded 36GiB to GPU in 2.57696s (13.9699GiB/s) | 20260507 20:05:13.224 | INFO | Downloaded 34GiB to GPU in 2.46067s (13.8174GiB/s) | 20260507 20:05:18.361 | INFO | Analyzing chunk frames = [2230272,2230784) | 20260507 20:05:18.361 | INFO | MemInfo: { | 20260507 20:05:18.361 | INFO | "gpu" : " 17 GB", | 20260507 20:05:18.361 | INFO | "huge" : "", | 20260507 20:05:18.361 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:18.361 | INFO | "malloc" : " 14 GB", | 20260507 20:05:18.361 | INFO | "pinned" : " 29 GB" | 20260507 20:05:18.361 | INFO | } | 20260507 20:05:18.362 | INFO | Perf report for the last 5.13786 seconds: | 20260507 20:05:18.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:18.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260507 20:05:18.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 20:05:18.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:18.362 | INFO | Uploaded 36GiB to GPU in 2.46321s (14.6151GiB/s) | 20260507 20:05:18.362 | INFO | Downloaded 34GiB to GPU in 2.50152s (13.5918GiB/s) | 20260507 20:05:23.500 | INFO | Analyzing chunk frames = [2230784,2231296) | 20260507 20:05:23.500 | INFO | MemInfo: { | 20260507 20:05:23.500 | INFO | "gpu" : " 17 GB", | 20260507 20:05:23.500 | INFO | "huge" : "", | 20260507 20:05:23.500 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:23.500 | INFO | "malloc" : " 14 GB", | 20260507 20:05:23.500 | INFO | "pinned" : " 29 GB" | 20260507 20:05:23.500 | INFO | } | 20260507 20:05:23.500 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:05:23.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:23.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.500 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:05:23.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.501 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:05:23.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:23.501 | INFO | Uploaded 36GiB to GPU in 2.47116s (14.5681GiB/s) | 20260507 20:05:23.501 | INFO | Downloaded 34GiB to GPU in 2.41897s (14.0556GiB/s) | 20260507 20:05:28.640 | INFO | Analyzing chunk frames = [2231296,2231808) | 20260507 20:05:28.640 | INFO | MemInfo: { | 20260507 20:05:28.640 | INFO | "gpu" : " 17 GB", | 20260507 20:05:28.641 | INFO | "huge" : "", | 20260507 20:05:28.641 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:28.641 | INFO | "malloc" : " 14 GB", | 20260507 20:05:28.641 | INFO | "pinned" : " 29 GB" | 20260507 20:05:28.641 | INFO | } | 20260507 20:05:28.641 | INFO | Perf report for the last 5.14037 seconds: | 20260507 20:05:28.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:28.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:05:28.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:05:28.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:28.641 | INFO | Uploaded 36GiB to GPU in 2.52766s (14.2424GiB/s) | 20260507 20:05:28.641 | INFO | Downloaded 34GiB to GPU in 2.44603s (13.9001GiB/s) | 20260507 20:05:33.780 | INFO | Analyzing chunk frames = [2231808,2232320) | 20260507 20:05:33.780 | INFO | MemInfo: { | 20260507 20:05:33.780 | INFO | "gpu" : " 17 GB", | 20260507 20:05:33.780 | INFO | "huge" : "", | 20260507 20:05:33.780 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:33.780 | INFO | "malloc" : " 14 GB", | 20260507 20:05:33.780 | INFO | "pinned" : " 29 GB" | 20260507 20:05:33.780 | INFO | } | 20260507 20:05:33.780 | INFO | Perf report for the last 5.13903 seconds: | 20260507 20:05:33.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:33.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 20:05:33.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:05:33.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:33.780 | INFO | Uploaded 36GiB to GPU in 2.4919s (14.4468GiB/s) | 20260507 20:05:33.780 | INFO | Downloaded 34GiB to GPU in 2.43815s (13.945GiB/s) | 20260507 20:05:33.877 | INFO | Starting a round of Baz IO | 20260507 20:05:37.358 | INFO | Populating RTMetrics report | 20260507 20:05:37.358 | INFO | RTMetrics report populated | 20260507 20:05:37.358 | INFO | Opening tmp RTMetrics json file | 20260507 20:05:37.382 | INFO | Serializing RTMetrics to json | 20260507 20:05:37.384 | INFO | Serialization done, swapping tmp file now... | 20260507 20:05:37.385 | INFO | RTMetrics done | 20260507 20:05:37.460 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.583 seconds | 20260507 20:05:37.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.79% / 0.01% / 55.20% | 20260507 20:05:37.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.1388% / 1.58946e-05% / 5.96046e-05% | 20260507 20:05:37.460 | INFO | Baz IO overall: | 20260507 20:05:37.460 | INFO | BytesWritten : 1011735118254 | 20260507 20:05:37.460 | INFO | NumEvents : 490503226554 | 20260507 20:05:38.919 | INFO | Analyzing chunk frames = [2232320,2232832) | 20260507 20:05:38.919 | INFO | MemInfo: { | 20260507 20:05:38.919 | INFO | "gpu" : " 17 GB", | 20260507 20:05:38.920 | INFO | "huge" : "", | 20260507 20:05:38.920 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:38.920 | INFO | "malloc" : " 14 GB", | 20260507 20:05:38.920 | INFO | "pinned" : " 29 GB" | 20260507 20:05:38.920 | INFO | } | 20260507 20:05:38.920 | INFO | Perf report for the last 5.13976 seconds: | 20260507 20:05:38.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:38.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:38.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:38.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:38.920 | INFO | ANALYSIS : 68.9%, 2.9, 106.6ms, 0 | 20260507 20:05:38.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:38.920 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 20:05:38.920 | INFO | PRE_HQ : 66.3%, 1.7, 58.6ms, 0 | 20260507 20:05:38.920 | INFO | BAZWRITER : 69.1%, 4.1, 145.4ms, 0 | 20260507 20:05:38.920 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 20:05:38.920 | INFO | Uploaded 36GiB to GPU in 2.45034s (14.6919GiB/s) | 20260507 20:05:38.920 | INFO | Downloaded 38GiB to GPU in 2.69504s (14.1GiB/s) | 20260507 20:05:44.059 | INFO | Analyzing chunk frames = [2232832,2233344) | 20260507 20:05:44.059 | INFO | MemInfo: { | 20260507 20:05:44.059 | INFO | "gpu" : " 17 GB", | 20260507 20:05:44.059 | INFO | "huge" : "", | 20260507 20:05:44.059 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:44.059 | INFO | "malloc" : " 14 GB", | 20260507 20:05:44.059 | INFO | "pinned" : " 29 GB" | 20260507 20:05:44.059 | INFO | } | 20260507 20:05:44.059 | INFO | Perf report for the last 5.13971 seconds: | 20260507 20:05:44.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:44.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.059 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 20:05:44.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.059 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:05:44.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:44.060 | INFO | Uploaded 36GiB to GPU in 2.57085s (14.0031GiB/s) | 20260507 20:05:44.060 | INFO | Downloaded 34GiB to GPU in 2.49062s (13.6512GiB/s) | 20260507 20:05:49.197 | INFO | Analyzing chunk frames = [2233344,2233856) | 20260507 20:05:49.197 | INFO | MemInfo: { | 20260507 20:05:49.197 | INFO | "gpu" : " 17 GB", | 20260507 20:05:49.197 | INFO | "huge" : "", | 20260507 20:05:49.197 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:49.197 | INFO | "malloc" : " 14 GB", | 20260507 20:05:49.197 | INFO | "pinned" : " 29 GB" | 20260507 20:05:49.197 | INFO | } | 20260507 20:05:49.198 | INFO | Perf report for the last 5.13837 seconds: | 20260507 20:05:49.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:49.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 20:05:49.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:05:49.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:49.198 | INFO | Uploaded 36GiB to GPU in 2.52365s (14.2651GiB/s) | 20260507 20:05:49.198 | INFO | Downloaded 34GiB to GPU in 2.48693s (13.6715GiB/s) | 20260507 20:05:54.338 | INFO | Analyzing chunk frames = [2233856,2234368) | 20260507 20:05:54.338 | INFO | MemInfo: { | 20260507 20:05:54.338 | INFO | "gpu" : " 17 GB", | 20260507 20:05:54.338 | INFO | "huge" : "", | 20260507 20:05:54.338 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:54.338 | INFO | "malloc" : " 14 GB", | 20260507 20:05:54.340 | INFO | "pinned" : " 29 GB" | 20260507 20:05:54.340 | INFO | } | 20260507 20:05:54.340 | INFO | Perf report for the last 5.14254 seconds: | 20260507 20:05:54.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:54.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.340 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 20:05:54.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.340 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260507 20:05:54.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:54.341 | INFO | Uploaded 36GiB to GPU in 2.49986s (14.4008GiB/s) | 20260507 20:05:54.341 | INFO | Downloaded 34GiB to GPU in 2.4885s (13.6628GiB/s) | 20260507 20:05:59.478 | INFO | Analyzing chunk frames = [2234368,2234880) | 20260507 20:05:59.478 | INFO | MemInfo: { | 20260507 20:05:59.478 | INFO | "gpu" : " 17 GB", | 20260507 20:05:59.478 | INFO | "huge" : "", | 20260507 20:05:59.478 | INFO | "huge_shared" : " 29 GB", | 20260507 20:05:59.478 | INFO | "malloc" : " 14 GB", | 20260507 20:05:59.478 | INFO | "pinned" : " 29 GB" | 20260507 20:05:59.478 | INFO | } | 20260507 20:05:59.478 | INFO | Perf report for the last 5.13818 seconds: | 20260507 20:05:59.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:05:59.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.478 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 20:05:59.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.479 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 20:05:59.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:05:59.479 | INFO | Uploaded 36GiB to GPU in 2.38938s (15.0667GiB/s) | 20260507 20:05:59.479 | INFO | Downloaded 34GiB to GPU in 2.47724s (13.7249GiB/s) | 20260507 20:06:04.615 | INFO | Analyzing chunk frames = [2234880,2235392) | 20260507 20:06:04.616 | INFO | MemInfo: { | 20260507 20:06:04.616 | INFO | "gpu" : " 17 GB", | 20260507 20:06:04.616 | INFO | "huge" : "", | 20260507 20:06:04.616 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:04.616 | INFO | "malloc" : " 14 GB", | 20260507 20:06:04.616 | INFO | "pinned" : " 29 GB" | 20260507 20:06:04.616 | INFO | } | 20260507 20:06:04.616 | INFO | Perf report for the last 5.13729 seconds: | 20260507 20:06:04.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:04.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:06:04.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:06:04.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:04.616 | INFO | Uploaded 36GiB to GPU in 2.51717s (14.3018GiB/s) | 20260507 20:06:04.616 | INFO | Downloaded 34GiB to GPU in 2.41422s (14.0832GiB/s) | 20260507 20:06:09.754 | INFO | Analyzing chunk frames = [2235392,2235904) | 20260507 20:06:09.754 | INFO | MemInfo: { | 20260507 20:06:09.754 | INFO | "gpu" : " 17 GB", | 20260507 20:06:09.754 | INFO | "huge" : "", | 20260507 20:06:09.754 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:09.754 | INFO | "malloc" : " 14 GB", | 20260507 20:06:09.754 | INFO | "pinned" : " 29 GB" | 20260507 20:06:09.754 | INFO | } | 20260507 20:06:09.754 | INFO | Perf report for the last 5.1386 seconds: | 20260507 20:06:09.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:09.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.754 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:06:09.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.754 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:06:09.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:09.755 | INFO | Uploaded 36GiB to GPU in 2.55076s (14.1134GiB/s) | 20260507 20:06:09.755 | INFO | Downloaded 34GiB to GPU in 2.43387s (13.9695GiB/s) | 20260507 20:06:14.893 | INFO | Analyzing chunk frames = [2235904,2236416) | 20260507 20:06:14.893 | INFO | MemInfo: { | 20260507 20:06:14.893 | INFO | "gpu" : " 17 GB", | 20260507 20:06:14.893 | INFO | "huge" : "", | 20260507 20:06:14.893 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:14.894 | INFO | "malloc" : " 14 GB", | 20260507 20:06:14.894 | INFO | "pinned" : " 29 GB" | 20260507 20:06:14.894 | INFO | } | 20260507 20:06:14.894 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:06:14.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:14.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:06:14.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:06:14.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:14.894 | INFO | Uploaded 36GiB to GPU in 2.53594s (14.1959GiB/s) | 20260507 20:06:14.894 | INFO | Downloaded 34GiB to GPU in 2.42195s (14.0383GiB/s) | 20260507 20:06:14.894 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:06:14.894 | INFO | Total mem usage: 16.4 GiB | 20260507 20:06:14.894 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:06:14.894 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.82%) | 20260507 20:06:14.894 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:06:14.894 | INFO | High Water Breakdown: | 20260507 20:06:14.894 | INFO | 60.1% : 9.06 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:06:14.894 | INFO | 39.8% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:06:14.894 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:06:14.894 | INFO | ------------------------------------------ | 20260507 20:06:14.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:06:14.894 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:06:14.894 | INFO | Total mem usage: 26.5 GiB | 20260507 20:06:14.894 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:06:14.894 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:06:14.894 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:06:14.894 | INFO | High Water Breakdown: | 20260507 20:06:14.894 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:06:14.895 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:06:14.895 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:06:14.895 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:06:14.895 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:06:14.895 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:06:14.895 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:06:14.895 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:06:14.895 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:06:14.895 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:06:14.895 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:06:14.895 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:06:14.895 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:06:14.895 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:06:14.895 | INFO | ------------------------------------------ | 20260507 20:06:14.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:06:14.895 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:06:14.895 | INFO | Total mem usage: 27.8 GiB | 20260507 20:06:14.895 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:06:14.895 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:06:14.895 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:06:14.895 | INFO | High Water Breakdown: | 20260507 20:06:14.895 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:06:14.895 | INFO | ------------------------------------------ | 20260507 20:06:14.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:06:14.895 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:06:14.895 | INFO | Total mem usage: 15.7 GiB | 20260507 20:06:14.895 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:06:14.895 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:06:14.895 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:06:14.895 | INFO | High Water Breakdown: | 20260507 20:06:14.895 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:06:14.895 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:06:14.895 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:06:14.895 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:06:14.895 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:06:14.895 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:06:14.895 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:06:14.895 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:06:14.895 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:06:14.895 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:06:14.895 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:06:14.895 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:06:14.895 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:06:14.895 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:06:14.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:06:14.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:06:14.895 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:06:14.895 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:06:14.895 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:06:14.895 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:06:14.895 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:06:14.895 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:06:14.895 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:06:14.895 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:06:14.895 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:06:14.895 | INFO | ------------------------------------------ | 20260507 20:06:14.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:06:14.895 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:06:14.895 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:06:14.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:06:14.895 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.43e+01 / 2.15e+00 | 20260507 20:06:14.895 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.15e-02 / 6.09e-03 | 20260507 20:06:14.895 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.45e+00, 6.00e-01 / 2.59e-01 | 20260507 20:06:14.895 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.11e-01 / 1.14e-01 | 20260507 20:06:14.895 | INFO | PerfReport: 15.89%, DME -- 4.21e+01 / 4.53e+01, 4.12e+00 / 2.08e+00 | 20260507 20:06:14.895 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.76e+00 / 2.03e+00 | 20260507 20:06:14.895 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.49e+00 / 2.81e+00, 4.43e-01 / 2.48e-01 | 20260507 20:06:14.895 | INFO | PerfReport: 3.58%, Metrics -- 1.19e+00 / 1.41e+00, 3.01e-01 / 1.75e-01 | 20260507 20:06:14.895 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.05e-01, 6.13e-02 / 3.67e-02 | 20260507 20:06:14.896 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:06:14.896 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:06:14.896 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:06:14.896 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 24.37%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 24.15%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 30.85%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 2.18%, LockUpload -- 2.34e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 2.52%, LockDownload -- 2.70e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 15.91%, LockCompute -- 1.71e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 20:06:14.896 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.88e-02 / 0.00e+00 | 20260507 20:06:14.991 | INFO | Starting a round of Baz IO | 20260507 20:06:18.504 | INFO | Populating RTMetrics report | 20260507 20:06:18.504 | INFO | RTMetrics report populated | 20260507 20:06:18.504 | INFO | Opening tmp RTMetrics json file | 20260507 20:06:18.506 | INFO | Serializing RTMetrics to json | 20260507 20:06:18.511 | INFO | Serialization done, swapping tmp file now... | 20260507 20:06:18.516 | INFO | RTMetrics done | 20260507 20:06:18.650 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.659 seconds | 20260507 20:06:18.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.78% / 0.01% / 55.21% | 20260507 20:06:18.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.13611% / 1.19209e-05% / 5.16574e-05% | 20260507 20:06:18.650 | INFO | Baz IO overall: | 20260507 20:06:18.650 | INFO | BytesWritten : 1013649129659 | 20260507 20:06:18.650 | INFO | NumEvents : 491430127470 | 20260507 20:06:20.033 | INFO | Analyzing chunk frames = [2236416,2236928) | 20260507 20:06:20.033 | INFO | MemInfo: { | 20260507 20:06:20.033 | INFO | "gpu" : " 17 GB", | 20260507 20:06:20.033 | INFO | "huge" : "", | 20260507 20:06:20.033 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:20.033 | INFO | "malloc" : " 14 GB", | 20260507 20:06:20.033 | INFO | "pinned" : " 29 GB" | 20260507 20:06:20.033 | INFO | } | 20260507 20:06:20.033 | INFO | Perf report for the last 5.13951 seconds: | 20260507 20:06:20.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:20.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:20.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:20.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:20.033 | INFO | ANALYSIS : 68.9%, 3.0, 107.5ms, 0 | 20260507 20:06:20.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:20.033 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 20:06:20.033 | INFO | PRE_HQ : 66.4%, 1.7, 59.2ms, 0 | 20260507 20:06:20.033 | INFO | BAZWRITER : 70.7%, 3.6, 130.2ms, 0 | 20260507 20:06:20.033 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 20:06:20.034 | INFO | Uploaded 36GiB to GPU in 2.56219s (14.0505GiB/s) | 20260507 20:06:20.034 | INFO | Downloaded 38GiB to GPU in 2.69214s (14.1152GiB/s) | 20260507 20:06:25.172 | INFO | Analyzing chunk frames = [2236928,2237440) | 20260507 20:06:25.172 | INFO | MemInfo: { | 20260507 20:06:25.172 | INFO | "gpu" : " 17 GB", | 20260507 20:06:25.172 | INFO | "huge" : "", | 20260507 20:06:25.172 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:25.172 | INFO | "malloc" : " 14 GB", | 20260507 20:06:25.173 | INFO | "pinned" : " 29 GB" | 20260507 20:06:25.173 | INFO | } | 20260507 20:06:25.173 | INFO | Perf report for the last 5.13952 seconds: | 20260507 20:06:25.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:25.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:06:25.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:06:25.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:25.173 | INFO | Uploaded 36GiB to GPU in 2.52313s (14.268GiB/s) | 20260507 20:06:25.173 | INFO | Downloaded 34GiB to GPU in 2.46268s (13.8061GiB/s) | 20260507 20:06:30.312 | INFO | Analyzing chunk frames = [2237440,2237952) | 20260507 20:06:30.312 | INFO | MemInfo: { | 20260507 20:06:30.312 | INFO | "gpu" : " 17 GB", | 20260507 20:06:30.312 | INFO | "huge" : "", | 20260507 20:06:30.312 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:30.312 | INFO | "malloc" : " 14 GB", | 20260507 20:06:30.312 | INFO | "pinned" : " 29 GB" | 20260507 20:06:30.312 | INFO | } | 20260507 20:06:30.312 | INFO | Perf report for the last 5.13969 seconds: | 20260507 20:06:30.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:30.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.312 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 20:06:30.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.313 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:06:30.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:30.313 | INFO | Uploaded 36GiB to GPU in 2.59051s (13.8969GiB/s) | 20260507 20:06:30.313 | INFO | Downloaded 34GiB to GPU in 2.46219s (13.8088GiB/s) | 20260507 20:06:35.451 | INFO | Analyzing chunk frames = [2237952,2238464) | 20260507 20:06:35.451 | INFO | MemInfo: { | 20260507 20:06:35.451 | INFO | "gpu" : " 17 GB", | 20260507 20:06:35.451 | INFO | "huge" : "", | 20260507 20:06:35.451 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:35.451 | INFO | "malloc" : " 14 GB", | 20260507 20:06:35.451 | INFO | "pinned" : " 29 GB" | 20260507 20:06:35.451 | INFO | } | 20260507 20:06:35.452 | INFO | Perf report for the last 5.13908 seconds: | 20260507 20:06:35.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:35.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | ANALYSIS : 67.7%, 3.0, 104.7ms, 0 | 20260507 20:06:35.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:06:35.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:35.452 | INFO | Uploaded 36GiB to GPU in 2.54003s (14.1731GiB/s) | 20260507 20:06:35.452 | INFO | Downloaded 34GiB to GPU in 2.49235s (13.6417GiB/s) | 20260507 20:06:40.592 | INFO | Analyzing chunk frames = [2238464,2238976) | 20260507 20:06:40.592 | INFO | MemInfo: { | 20260507 20:06:40.592 | INFO | "gpu" : " 17 GB", | 20260507 20:06:40.592 | INFO | "huge" : "", | 20260507 20:06:40.592 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:40.592 | INFO | "malloc" : " 14 GB", | 20260507 20:06:40.592 | INFO | "pinned" : " 29 GB" | 20260507 20:06:40.592 | INFO | } | 20260507 20:06:40.592 | INFO | Perf report for the last 5.14071 seconds: | 20260507 20:06:40.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:40.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 20:06:40.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:06:40.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:40.592 | INFO | Uploaded 36GiB to GPU in 2.4349s (14.785GiB/s) | 20260507 20:06:40.592 | INFO | Downloaded 34GiB to GPU in 2.49648s (13.6192GiB/s) | 20260507 20:06:45.731 | INFO | Analyzing chunk frames = [2238976,2239488) | 20260507 20:06:45.731 | INFO | MemInfo: { | 20260507 20:06:45.731 | INFO | "gpu" : " 17 GB", | 20260507 20:06:45.731 | INFO | "huge" : "", | 20260507 20:06:45.731 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:45.731 | INFO | "malloc" : " 14 GB", | 20260507 20:06:45.731 | INFO | "pinned" : " 29 GB" | 20260507 20:06:45.731 | INFO | } | 20260507 20:06:45.731 | INFO | Perf report for the last 5.13861 seconds: | 20260507 20:06:45.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:45.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:06:45.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:06:45.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:45.731 | INFO | Uploaded 36GiB to GPU in 2.49726s (14.4158GiB/s) | 20260507 20:06:45.731 | INFO | Downloaded 34GiB to GPU in 2.42543s (14.0181GiB/s) | 20260507 20:06:50.869 | INFO | Analyzing chunk frames = [2239488,2240000) | 20260507 20:06:50.869 | INFO | MemInfo: { | 20260507 20:06:50.869 | INFO | "gpu" : " 17 GB", | 20260507 20:06:50.869 | INFO | "huge" : "", | 20260507 20:06:50.869 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:50.870 | INFO | "malloc" : " 14 GB", | 20260507 20:06:50.870 | INFO | "pinned" : " 29 GB" | 20260507 20:06:50.870 | INFO | } | 20260507 20:06:50.870 | INFO | Perf report for the last 5.13884 seconds: | 20260507 20:06:50.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:50.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:06:50.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:06:50.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:50.870 | INFO | Uploaded 36GiB to GPU in 2.51742s (14.3004GiB/s) | 20260507 20:06:50.870 | INFO | Downloaded 34GiB to GPU in 2.45572s (13.8452GiB/s) | 20260507 20:06:56.009 | INFO | Analyzing chunk frames = [2240000,2240512) | 20260507 20:06:56.009 | INFO | MemInfo: { | 20260507 20:06:56.009 | INFO | "gpu" : " 17 GB", | 20260507 20:06:56.009 | INFO | "huge" : "", | 20260507 20:06:56.009 | INFO | "huge_shared" : " 29 GB", | 20260507 20:06:56.009 | INFO | "malloc" : " 14 GB", | 20260507 20:06:56.009 | INFO | "pinned" : " 29 GB" | 20260507 20:06:56.009 | INFO | } | 20260507 20:06:56.009 | INFO | Perf report for the last 5.13953 seconds: | 20260507 20:06:56.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:06:56.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.009 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:06:56.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.010 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260507 20:06:56.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:06:56.010 | INFO | Uploaded 36GiB to GPU in 2.48397s (14.4929GiB/s) | 20260507 20:06:56.010 | INFO | Downloaded 34GiB to GPU in 2.43313s (13.9738GiB/s) | 20260507 20:06:56.122 | INFO | Starting a round of Baz IO | 20260507 20:06:59.610 | INFO | Populating RTMetrics report | 20260507 20:06:59.610 | INFO | RTMetrics report populated | 20260507 20:06:59.610 | INFO | Opening tmp RTMetrics json file | 20260507 20:06:59.614 | INFO | Serializing RTMetrics to json | 20260507 20:06:59.617 | INFO | Serialization done, swapping tmp file now... | 20260507 20:06:59.621 | INFO | RTMetrics done | 20260507 20:06:59.750 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.627 seconds | 20260507 20:06:59.750 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.77% / 0.01% / 55.22% | 20260507 20:06:59.750 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.12488% / 1.19209e-05% / 6.35783e-05% | 20260507 20:06:59.750 | INFO | Baz IO overall: | 20260507 20:06:59.750 | INFO | BytesWritten : 1015560849430 | 20260507 20:06:59.750 | INFO | NumEvents : 492355631043 | 20260507 20:07:01.149 | INFO | Analyzing chunk frames = [2240512,2241024) | 20260507 20:07:01.149 | INFO | MemInfo: { | 20260507 20:07:01.149 | INFO | "gpu" : " 17 GB", | 20260507 20:07:01.149 | INFO | "huge" : "", | 20260507 20:07:01.149 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:01.149 | INFO | "malloc" : " 14 GB", | 20260507 20:07:01.149 | INFO | "pinned" : " 29 GB" | 20260507 20:07:01.149 | INFO | } | 20260507 20:07:01.149 | INFO | Perf report for the last 5.1401 seconds: | 20260507 20:07:01.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:01.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:01.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:01.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:01.149 | INFO | ANALYSIS : 69.6%, 2.9, 104.9ms, 0 | 20260507 20:07:01.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:01.149 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260507 20:07:01.149 | INFO | PRE_HQ : 68.5%, 1.8, 64.2ms, 0 | 20260507 20:07:01.150 | INFO | BAZWRITER : 70.1%, 3.9, 141.3ms, 0 | 20260507 20:07:01.150 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 20:07:01.150 | INFO | Uploaded 36GiB to GPU in 2.5274s (14.2439GiB/s) | 20260507 20:07:01.150 | INFO | Downloaded 38GiB to GPU in 2.70619s (14.0419GiB/s) | 20260507 20:07:06.288 | INFO | Analyzing chunk frames = [2241024,2241536) | 20260507 20:07:06.288 | INFO | MemInfo: { | 20260507 20:07:06.288 | INFO | "gpu" : " 17 GB", | 20260507 20:07:06.288 | INFO | "huge" : "", | 20260507 20:07:06.288 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:06.288 | INFO | "malloc" : " 14 GB", | 20260507 20:07:06.288 | INFO | "pinned" : " 29 GB" | 20260507 20:07:06.288 | INFO | } | 20260507 20:07:06.288 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:07:06.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:06.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.288 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 20:07:06.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.288 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:07:06.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:06.289 | INFO | Uploaded 36GiB to GPU in 2.55406s (14.0952GiB/s) | 20260507 20:07:06.289 | INFO | Downloaded 34GiB to GPU in 2.47791s (13.7212GiB/s) | 20260507 20:07:11.427 | INFO | Analyzing chunk frames = [2241536,2242048) | 20260507 20:07:11.427 | INFO | MemInfo: { | 20260507 20:07:11.427 | INFO | "gpu" : " 17 GB", | 20260507 20:07:11.427 | INFO | "huge" : "", | 20260507 20:07:11.427 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:11.427 | INFO | "malloc" : " 14 GB", | 20260507 20:07:11.427 | INFO | "pinned" : " 29 GB" | 20260507 20:07:11.427 | INFO | } | 20260507 20:07:11.427 | INFO | Perf report for the last 5.13869 seconds: | 20260507 20:07:11.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:11.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.427 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:07:11.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.427 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:07:11.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:11.428 | INFO | Uploaded 36GiB to GPU in 2.5732s (13.9904GiB/s) | 20260507 20:07:11.428 | INFO | Downloaded 34GiB to GPU in 2.47821s (13.7196GiB/s) | 20260507 20:07:16.565 | INFO | Analyzing chunk frames = [2242048,2242560) | 20260507 20:07:16.566 | INFO | MemInfo: { | 20260507 20:07:16.566 | INFO | "gpu" : " 17 GB", | 20260507 20:07:16.566 | INFO | "huge" : "", | 20260507 20:07:16.566 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:16.566 | INFO | "malloc" : " 14 GB", | 20260507 20:07:16.566 | INFO | "pinned" : " 29 GB" | 20260507 20:07:16.566 | INFO | } | 20260507 20:07:16.566 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:07:16.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:16.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 20:07:16.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:07:16.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:16.566 | INFO | Uploaded 36GiB to GPU in 2.48086s (14.5111GiB/s) | 20260507 20:07:16.566 | INFO | Downloaded 34GiB to GPU in 2.40615s (14.1305GiB/s) | 20260507 20:07:21.707 | INFO | Analyzing chunk frames = [2242560,2243072) | 20260507 20:07:21.707 | INFO | MemInfo: { | 20260507 20:07:21.707 | INFO | "gpu" : " 17 GB", | 20260507 20:07:21.707 | INFO | "huge" : "", | 20260507 20:07:21.707 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:21.707 | INFO | "malloc" : " 14 GB", | 20260507 20:07:21.707 | INFO | "pinned" : " 29 GB" | 20260507 20:07:21.707 | INFO | } | 20260507 20:07:21.707 | INFO | Perf report for the last 5.14132 seconds: | 20260507 20:07:21.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:21.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:07:21.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:07:21.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:21.707 | INFO | Uploaded 36GiB to GPU in 2.37153s (15.1801GiB/s) | 20260507 20:07:21.707 | INFO | Downloaded 34GiB to GPU in 2.49948s (13.6028GiB/s) | 20260507 20:07:26.846 | INFO | Analyzing chunk frames = [2243072,2243584) | 20260507 20:07:26.846 | INFO | MemInfo: { | 20260507 20:07:26.846 | INFO | "gpu" : " 17 GB", | 20260507 20:07:26.846 | INFO | "huge" : "", | 20260507 20:07:26.846 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:26.846 | INFO | "malloc" : " 14 GB", | 20260507 20:07:26.846 | INFO | "pinned" : " 29 GB" | 20260507 20:07:26.846 | INFO | } | 20260507 20:07:26.846 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:07:26.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:26.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.846 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:07:26.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.846 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:07:26.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:26.847 | INFO | Uploaded 36GiB to GPU in 2.51463s (14.3162GiB/s) | 20260507 20:07:26.847 | INFO | Downloaded 34GiB to GPU in 2.41444s (14.0819GiB/s) | 20260507 20:07:31.985 | INFO | Analyzing chunk frames = [2243584,2244096) | 20260507 20:07:31.985 | INFO | MemInfo: { | 20260507 20:07:31.985 | INFO | "gpu" : " 17 GB", | 20260507 20:07:31.985 | INFO | "huge" : "", | 20260507 20:07:31.985 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:31.985 | INFO | "malloc" : " 14 GB", | 20260507 20:07:31.985 | INFO | "pinned" : " 29 GB" | 20260507 20:07:31.985 | INFO | } | 20260507 20:07:31.985 | INFO | Perf report for the last 5.13912 seconds: | 20260507 20:07:31.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:31.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 20:07:31.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:07:31.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:31.985 | INFO | Uploaded 36GiB to GPU in 2.51557s (14.3109GiB/s) | 20260507 20:07:31.985 | INFO | Downloaded 34GiB to GPU in 2.43077s (13.9873GiB/s) | 20260507 20:07:37.124 | INFO | Analyzing chunk frames = [2244096,2244608) | 20260507 20:07:37.124 | INFO | MemInfo: { | 20260507 20:07:37.124 | INFO | "gpu" : " 17 GB", | 20260507 20:07:37.124 | INFO | "huge" : "", | 20260507 20:07:37.124 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:37.124 | INFO | "malloc" : " 14 GB", | 20260507 20:07:37.124 | INFO | "pinned" : " 29 GB" | 20260507 20:07:37.124 | INFO | } | 20260507 20:07:37.124 | INFO | Perf report for the last 5.13913 seconds: | 20260507 20:07:37.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:37.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.124 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:07:37.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.124 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:07:37.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:37.125 | INFO | Uploaded 36GiB to GPU in 2.50485s (14.3721GiB/s) | 20260507 20:07:37.125 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:07:37.125 | INFO | Total mem usage: 16.4 GiB | 20260507 20:07:37.125 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:07:37.125 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.8%) | 20260507 20:07:37.125 | INFO | Downloaded 34GiB to GPU in 2.43704s (13.9514GiB/s) | 20260507 20:07:37.125 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:07:37.125 | INFO | High Water Breakdown: | 20260507 20:07:37.125 | INFO | 59.5% : 8.65 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:07:37.125 | INFO | 40.4% : 5.87 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:07:37.125 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:07:37.125 | INFO | ------------------------------------------ | 20260507 20:07:37.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:07:37.125 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:07:37.125 | INFO | Total mem usage: 26.5 GiB | 20260507 20:07:37.125 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:07:37.125 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:07:37.125 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:07:37.125 | INFO | High Water Breakdown: | 20260507 20:07:37.125 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:07:37.125 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:07:37.125 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:07:37.125 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:07:37.125 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:07:37.125 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:07:37.125 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:07:37.125 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:07:37.125 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:07:37.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:07:37.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:07:37.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:07:37.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:07:37.125 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:07:37.125 | INFO | ------------------------------------------ | 20260507 20:07:37.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:07:37.125 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:07:37.125 | INFO | Total mem usage: 27.8 GiB | 20260507 20:07:37.125 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:07:37.125 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:07:37.125 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:07:37.125 | INFO | High Water Breakdown: | 20260507 20:07:37.125 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:07:37.125 | INFO | ------------------------------------------ | 20260507 20:07:37.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:07:37.125 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:07:37.125 | INFO | Total mem usage: 15.7 GiB | 20260507 20:07:37.125 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:07:37.125 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:07:37.125 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:07:37.125 | INFO | High Water Breakdown: | 20260507 20:07:37.125 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:07:37.125 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:07:37.125 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:07:37.125 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:07:37.125 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:07:37.125 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:07:37.125 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:07:37.125 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:07:37.125 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:07:37.125 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:07:37.125 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:07:37.125 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:07:37.125 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:07:37.125 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:07:37.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:07:37.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:07:37.125 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:07:37.125 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:07:37.125 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:07:37.125 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:07:37.125 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:07:37.125 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:07:37.125 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:07:37.125 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:07:37.125 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:07:37.125 | INFO | ------------------------------------------ | 20260507 20:07:37.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:07:37.125 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:07:37.125 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:07:37.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:07:37.125 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.42e+01 / 1.95e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.20e-02 / 6.22e-03 | 20260507 20:07:37.126 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.43e+00, 5.97e-01 / 2.96e-01 | 20260507 20:07:37.126 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.47e+00, 2.08e-01 / 1.24e-01 | 20260507 20:07:37.126 | INFO | PerfReport: 15.82%, DME -- 4.19e+01 / 4.50e+01, 3.89e+00 / 1.64e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.79e+00 / 1.83e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.44e-01 / 2.38e-01 | 20260507 20:07:37.126 | INFO | PerfReport: 3.60%, Metrics -- 1.19e+00 / 1.40e+00, 3.03e-01 / 1.96e-01 | 20260507 20:07:37.126 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.04e-01, 6.10e-02 / 3.40e-02 | 20260507 20:07:37.126 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:07:37.126 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:07:37.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:07:37.126 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 24.45%, Upload -- 2.62e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 24.19%, Download -- 2.59e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 30.93%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 2.12%, LockUpload -- 2.27e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 2.40%, LockDownload -- 2.57e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 15.90%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 20:07:37.126 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260507 20:07:37.221 | INFO | Starting a round of Baz IO | 20260507 20:07:40.715 | INFO | Populating RTMetrics report | 20260507 20:07:40.715 | INFO | RTMetrics report populated | 20260507 20:07:40.715 | INFO | Opening tmp RTMetrics json file | 20260507 20:07:40.716 | INFO | Serializing RTMetrics to json | 20260507 20:07:40.738 | INFO | Serialization done, swapping tmp file now... | 20260507 20:07:40.742 | INFO | RTMetrics done | 20260507 20:07:40.811 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.589 seconds | 20260507 20:07:40.811 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.76% / 0.01% / 55.23% | 20260507 20:07:40.811 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.12675% / 1.19209e-05% / 2.78155e-05% | 20260507 20:07:40.811 | INFO | Baz IO overall: | 20260507 20:07:40.811 | INFO | BytesWritten : 1017475190936 | 20260507 20:07:40.811 | INFO | NumEvents : 493283233314 | 20260507 20:07:42.262 | INFO | Analyzing chunk frames = [2244608,2245120) | 20260507 20:07:42.262 | INFO | MemInfo: { | 20260507 20:07:42.262 | INFO | "gpu" : " 17 GB", | 20260507 20:07:42.262 | INFO | "huge" : "", | 20260507 20:07:42.262 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:42.262 | INFO | "malloc" : " 14 GB", | 20260507 20:07:42.262 | INFO | "pinned" : " 29 GB" | 20260507 20:07:42.262 | INFO | } | 20260507 20:07:42.262 | INFO | Perf report for the last 5.13828 seconds: | 20260507 20:07:42.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:42.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:42.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:42.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:42.263 | INFO | ANALYSIS : 69.2%, 3.0, 107.0ms, 0 | 20260507 20:07:42.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:42.263 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 20:07:42.263 | INFO | PRE_HQ : 66.6%, 1.7, 58.2ms, 0 | 20260507 20:07:42.263 | INFO | BAZWRITER : 69.5%, 4.2, 148.5ms, 0 | 20260507 20:07:42.263 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:07:42.263 | INFO | Uploaded 36GiB to GPU in 2.53687s (14.1907GiB/s) | 20260507 20:07:42.263 | INFO | Downloaded 38GiB to GPU in 2.70416s (14.0524GiB/s) | 20260507 20:07:47.403 | INFO | Analyzing chunk frames = [2245120,2245632) | 20260507 20:07:47.403 | INFO | MemInfo: { | 20260507 20:07:47.403 | INFO | "gpu" : " 17 GB", | 20260507 20:07:47.403 | INFO | "huge" : "", | 20260507 20:07:47.403 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:47.403 | INFO | "malloc" : " 14 GB", | 20260507 20:07:47.403 | INFO | "pinned" : " 29 GB" | 20260507 20:07:47.403 | INFO | } | 20260507 20:07:47.403 | INFO | Perf report for the last 5.14044 seconds: | 20260507 20:07:47.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:47.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:07:47.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:07:47.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:47.403 | INFO | Uploaded 36GiB to GPU in 2.50795s (14.3544GiB/s) | 20260507 20:07:47.403 | INFO | Downloaded 34GiB to GPU in 2.45487s (13.85GiB/s) | 20260507 20:07:52.542 | INFO | Analyzing chunk frames = [2245632,2246144) | 20260507 20:07:52.542 | INFO | MemInfo: { | 20260507 20:07:52.542 | INFO | "gpu" : " 17 GB", | 20260507 20:07:52.542 | INFO | "huge" : "", | 20260507 20:07:52.542 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:52.542 | INFO | "malloc" : " 14 GB", | 20260507 20:07:52.542 | INFO | "pinned" : " 29 GB" | 20260507 20:07:52.542 | INFO | } | 20260507 20:07:52.542 | INFO | Perf report for the last 5.13895 seconds: | 20260507 20:07:52.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:52.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:07:52.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:07:52.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:52.542 | INFO | Uploaded 36GiB to GPU in 2.53671s (14.1916GiB/s) | 20260507 20:07:52.542 | INFO | Downloaded 34GiB to GPU in 2.44377s (13.9129GiB/s) | 20260507 20:07:57.682 | INFO | Analyzing chunk frames = [2246144,2246656) | 20260507 20:07:57.682 | INFO | MemInfo: { | 20260507 20:07:57.682 | INFO | "gpu" : " 17 GB", | 20260507 20:07:57.682 | INFO | "huge" : "", | 20260507 20:07:57.682 | INFO | "huge_shared" : " 29 GB", | 20260507 20:07:57.682 | INFO | "malloc" : " 14 GB", | 20260507 20:07:57.682 | INFO | "pinned" : " 29 GB" | 20260507 20:07:57.682 | INFO | } | 20260507 20:07:57.682 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:07:57.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:07:57.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:07:57.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:07:57.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:07:57.682 | INFO | Uploaded 36GiB to GPU in 2.48275s (14.5GiB/s) | 20260507 20:07:57.682 | INFO | Downloaded 34GiB to GPU in 2.38558s (14.2523GiB/s) | 20260507 20:08:02.820 | INFO | Analyzing chunk frames = [2246656,2247168) | 20260507 20:08:02.821 | INFO | MemInfo: { | 20260507 20:08:02.821 | INFO | "gpu" : " 17 GB", | 20260507 20:08:02.821 | INFO | "huge" : "", | 20260507 20:08:02.821 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:02.821 | INFO | "malloc" : " 14 GB", | 20260507 20:08:02.821 | INFO | "pinned" : " 29 GB" | 20260507 20:08:02.821 | INFO | } | 20260507 20:08:02.821 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:08:02.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:02.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:08:02.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:08:02.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:02.821 | INFO | Uploaded 36GiB to GPU in 2.38443s (15.0979GiB/s) | 20260507 20:08:02.822 | INFO | Downloaded 34GiB to GPU in 2.50258s (13.586GiB/s) | 20260507 20:08:07.960 | INFO | Analyzing chunk frames = [2247168,2247680) | 20260507 20:08:07.960 | INFO | MemInfo: { | 20260507 20:08:07.960 | INFO | "gpu" : " 17 GB", | 20260507 20:08:07.960 | INFO | "huge" : "", | 20260507 20:08:07.960 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:07.960 | INFO | "malloc" : " 14 GB", | 20260507 20:08:07.960 | INFO | "pinned" : " 29 GB" | 20260507 20:08:07.960 | INFO | } | 20260507 20:08:07.960 | INFO | Perf report for the last 5.13915 seconds: | 20260507 20:08:07.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:07.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.960 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:08:07.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.960 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260507 20:08:07.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:07.961 | INFO | Uploaded 36GiB to GPU in 2.41956s (14.8788GiB/s) | 20260507 20:08:07.961 | INFO | Downloaded 34GiB to GPU in 2.38529s (14.254GiB/s) | 20260507 20:08:13.099 | INFO | Analyzing chunk frames = [2247680,2248192) | 20260507 20:08:13.099 | INFO | MemInfo: { | 20260507 20:08:13.099 | INFO | "gpu" : " 17 GB", | 20260507 20:08:13.099 | INFO | "huge" : "", | 20260507 20:08:13.099 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:13.099 | INFO | "malloc" : " 14 GB", | 20260507 20:08:13.099 | INFO | "pinned" : " 29 GB" | 20260507 20:08:13.099 | INFO | } | 20260507 20:08:13.099 | INFO | Perf report for the last 5.13923 seconds: | 20260507 20:08:13.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:13.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.100 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 20:08:13.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.100 | INFO | PRE_HQ : 36.3%, 1.0, 18.8ms, 0 | 20260507 20:08:13.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:13.100 | INFO | Uploaded 36GiB to GPU in 2.53581s (14.1966GiB/s) | 20260507 20:08:13.100 | INFO | Downloaded 34GiB to GPU in 2.43726s (13.9501GiB/s) | 20260507 20:08:18.238 | INFO | Analyzing chunk frames = [2248192,2248704) | 20260507 20:08:18.238 | INFO | MemInfo: { | 20260507 20:08:18.238 | INFO | "gpu" : " 17 GB", | 20260507 20:08:18.238 | INFO | "huge" : "", | 20260507 20:08:18.238 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:18.238 | INFO | "malloc" : " 14 GB", | 20260507 20:08:18.238 | INFO | "pinned" : " 29 GB" | 20260507 20:08:18.238 | INFO | } | 20260507 20:08:18.239 | INFO | Perf report for the last 5.13909 seconds: | 20260507 20:08:18.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:18.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 20:08:18.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 20:08:18.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:18.239 | INFO | Uploaded 36GiB to GPU in 2.53388s (14.2075GiB/s) | 20260507 20:08:18.239 | INFO | Downloaded 34GiB to GPU in 2.43905s (13.9399GiB/s) | 20260507 20:08:18.352 | INFO | Starting a round of Baz IO | 20260507 20:08:21.841 | INFO | Populating RTMetrics report | 20260507 20:08:21.842 | INFO | RTMetrics report populated | 20260507 20:08:21.842 | INFO | Opening tmp RTMetrics json file | 20260507 20:08:21.843 | INFO | Serializing RTMetrics to json | 20260507 20:08:21.845 | INFO | Serialization done, swapping tmp file now... | 20260507 20:08:21.849 | INFO | RTMetrics done | 20260507 20:08:21.951 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.599 seconds | 20260507 20:08:21.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.75% / 0.01% / 55.24% | 20260507 20:08:21.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.11965% / 3.97364e-06% / 4.76837e-05% | 20260507 20:08:21.951 | INFO | Baz IO overall: | 20260507 20:08:21.952 | INFO | BytesWritten : 1019387593275 | 20260507 20:08:21.952 | INFO | NumEvents : 494209585705 | 20260507 20:08:23.378 | INFO | Analyzing chunk frames = [2248704,2249216) | 20260507 20:08:23.378 | INFO | MemInfo: { | 20260507 20:08:23.378 | INFO | "gpu" : " 17 GB", | 20260507 20:08:23.378 | INFO | "huge" : "", | 20260507 20:08:23.378 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:23.378 | INFO | "malloc" : " 14 GB", | 20260507 20:08:23.378 | INFO | "pinned" : " 29 GB" | 20260507 20:08:23.378 | INFO | } | 20260507 20:08:23.378 | INFO | Perf report for the last 5.13947 seconds: | 20260507 20:08:23.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:23.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:23.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:23.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:23.378 | INFO | ANALYSIS : 69.6%, 2.9, 105.1ms, 0 | 20260507 20:08:23.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:23.378 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 20:08:23.378 | INFO | PRE_HQ : 68.4%, 1.8, 64.3ms, 0 | 20260507 20:08:23.378 | INFO | BAZWRITER : 69.6%, 4.1, 145.0ms, 0 | 20260507 20:08:23.378 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 20:08:23.379 | INFO | Uploaded 36GiB to GPU in 2.51979s (14.2869GiB/s) | 20260507 20:08:23.379 | INFO | Downloaded 38GiB to GPU in 2.68954s (14.1288GiB/s) | 20260507 20:08:28.517 | INFO | Analyzing chunk frames = [2249216,2249728) | 20260507 20:08:28.517 | INFO | MemInfo: { | 20260507 20:08:28.517 | INFO | "gpu" : " 17 GB", | 20260507 20:08:28.518 | INFO | "huge" : "", | 20260507 20:08:28.518 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:28.518 | INFO | "malloc" : " 14 GB", | 20260507 20:08:28.518 | INFO | "pinned" : " 29 GB" | 20260507 20:08:28.518 | INFO | } | 20260507 20:08:28.518 | INFO | Perf report for the last 5.1396 seconds: | 20260507 20:08:28.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:28.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 20:08:28.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:08:28.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:28.518 | INFO | Uploaded 36GiB to GPU in 2.53746s (14.1874GiB/s) | 20260507 20:08:28.518 | INFO | Downloaded 34GiB to GPU in 2.50471s (13.5744GiB/s) | 20260507 20:08:33.660 | INFO | Analyzing chunk frames = [2249728,2250240) | 20260507 20:08:33.660 | INFO | MemInfo: { | 20260507 20:08:33.660 | INFO | "gpu" : " 17 GB", | 20260507 20:08:33.661 | INFO | "huge" : "", | 20260507 20:08:33.661 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:33.661 | INFO | "malloc" : " 14 GB", | 20260507 20:08:33.661 | INFO | "pinned" : " 29 GB" | 20260507 20:08:33.661 | INFO | } | 20260507 20:08:33.661 | INFO | Perf report for the last 5.14308 seconds: | 20260507 20:08:33.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:33.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 20:08:33.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:08:33.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:33.661 | INFO | Uploaded 36GiB to GPU in 2.57199s (13.997GiB/s) | 20260507 20:08:33.661 | INFO | Downloaded 34GiB to GPU in 2.48175s (13.7GiB/s) | 20260507 20:08:38.796 | INFO | Analyzing chunk frames = [2250240,2250752) | 20260507 20:08:38.796 | INFO | MemInfo: { | 20260507 20:08:38.796 | INFO | "gpu" : " 17 GB", | 20260507 20:08:38.796 | INFO | "huge" : "", | 20260507 20:08:38.796 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:38.796 | INFO | "malloc" : " 14 GB", | 20260507 20:08:38.796 | INFO | "pinned" : " 29 GB" | 20260507 20:08:38.796 | INFO | } | 20260507 20:08:38.796 | INFO | Perf report for the last 5.13521 seconds: | 20260507 20:08:38.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:38.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.796 | INFO | ANALYSIS : 67.6%, 3.0, 104.5ms, 0 | 20260507 20:08:38.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.796 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:08:38.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:38.797 | INFO | Uploaded 36GiB to GPU in 2.46741s (14.5902GiB/s) | 20260507 20:08:38.797 | INFO | Downloaded 34GiB to GPU in 2.50656s (13.5644GiB/s) | 20260507 20:08:43.935 | INFO | Analyzing chunk frames = [2250752,2251264) | 20260507 20:08:43.935 | INFO | MemInfo: { | 20260507 20:08:43.935 | INFO | "gpu" : " 17 GB", | 20260507 20:08:43.935 | INFO | "huge" : "", | 20260507 20:08:43.935 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:43.935 | INFO | "malloc" : " 14 GB", | 20260507 20:08:43.935 | INFO | "pinned" : " 29 GB" | 20260507 20:08:43.935 | INFO | } | 20260507 20:08:43.935 | INFO | Perf report for the last 5.13902 seconds: | 20260507 20:08:43.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:43.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 20:08:43.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:08:43.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:43.935 | INFO | Uploaded 36GiB to GPU in 2.36199s (15.2414GiB/s) | 20260507 20:08:43.935 | INFO | Downloaded 34GiB to GPU in 2.50904s (13.551GiB/s) | 20260507 20:08:49.074 | INFO | Analyzing chunk frames = [2251264,2251776) | 20260507 20:08:49.074 | INFO | MemInfo: { | 20260507 20:08:49.074 | INFO | "gpu" : " 17 GB", | 20260507 20:08:49.074 | INFO | "huge" : "", | 20260507 20:08:49.074 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:49.074 | INFO | "malloc" : " 14 GB", | 20260507 20:08:49.074 | INFO | "pinned" : " 29 GB" | 20260507 20:08:49.074 | INFO | } | 20260507 20:08:49.074 | INFO | Perf report for the last 5.13931 seconds: | 20260507 20:08:49.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:49.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.074 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:08:49.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.074 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:08:49.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:49.075 | INFO | Uploaded 36GiB to GPU in 2.4579s (14.6467GiB/s) | 20260507 20:08:49.075 | INFO | Downloaded 34GiB to GPU in 2.44293s (13.9177GiB/s) | 20260507 20:08:54.214 | INFO | Analyzing chunk frames = [2251776,2252288) | 20260507 20:08:54.214 | INFO | MemInfo: { | 20260507 20:08:54.214 | INFO | "gpu" : " 17 GB", | 20260507 20:08:54.214 | INFO | "huge" : "", | 20260507 20:08:54.214 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:54.214 | INFO | "malloc" : " 14 GB", | 20260507 20:08:54.214 | INFO | "pinned" : " 29 GB" | 20260507 20:08:54.214 | INFO | } | 20260507 20:08:54.214 | INFO | Perf report for the last 5.13954 seconds: | 20260507 20:08:54.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:54.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 20:08:54.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 20:08:54.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:54.214 | INFO | Uploaded 36GiB to GPU in 2.44269s (14.7378GiB/s) | 20260507 20:08:54.214 | INFO | Downloaded 34GiB to GPU in 2.45386s (13.8557GiB/s) | 20260507 20:08:59.352 | INFO | Analyzing chunk frames = [2252288,2252800) | 20260507 20:08:59.352 | INFO | MemInfo: { | 20260507 20:08:59.352 | INFO | "gpu" : " 17 GB", | 20260507 20:08:59.352 | INFO | "huge" : "", | 20260507 20:08:59.352 | INFO | "huge_shared" : " 29 GB", | 20260507 20:08:59.352 | INFO | "malloc" : " 14 GB", | 20260507 20:08:59.352 | INFO | "pinned" : " 29 GB" | 20260507 20:08:59.352 | INFO | } | 20260507 20:08:59.353 | INFO | Perf report for the last 5.13863 seconds: | 20260507 20:08:59.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:08:59.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:08:59.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:08:59.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:08:59.353 | INFO | Uploaded 36GiB to GPU in 2.43716s (14.7713GiB/s) | 20260507 20:08:59.353 | INFO | Downloaded 34GiB to GPU in 2.44694s (13.8949GiB/s) | 20260507 20:08:59.353 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:08:59.353 | INFO | Total mem usage: 16.4 GiB | 20260507 20:08:59.353 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:08:59.353 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.64%) | 20260507 20:08:59.353 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:08:59.353 | INFO | High Water Breakdown: | 20260507 20:08:59.353 | INFO | 59.9% : 8.97 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:08:59.353 | INFO | 39.9% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:08:59.353 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:08:59.353 | INFO | ------------------------------------------ | 20260507 20:08:59.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:08:59.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:08:59.353 | INFO | Total mem usage: 26.5 GiB | 20260507 20:08:59.353 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:08:59.353 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 20:08:59.353 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:08:59.353 | INFO | High Water Breakdown: | 20260507 20:08:59.353 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:08:59.353 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:08:59.353 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:08:59.353 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:08:59.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:08:59.353 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:08:59.353 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:08:59.353 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:08:59.353 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:08:59.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:08:59.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:08:59.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:08:59.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:08:59.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:08:59.353 | INFO | ------------------------------------------ | 20260507 20:08:59.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:08:59.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:08:59.353 | INFO | Total mem usage: 27.8 GiB | 20260507 20:08:59.353 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:08:59.353 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:08:59.353 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:08:59.353 | INFO | High Water Breakdown: | 20260507 20:08:59.353 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:08:59.353 | INFO | ------------------------------------------ | 20260507 20:08:59.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:08:59.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:08:59.353 | INFO | Total mem usage: 15.7 GiB | 20260507 20:08:59.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:08:59.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:08:59.353 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:08:59.353 | INFO | High Water Breakdown: | 20260507 20:08:59.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:08:59.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:08:59.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:08:59.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:08:59.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:08:59.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:08:59.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:08:59.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:08:59.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:08:59.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:08:59.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:08:59.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:08:59.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:08:59.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:08:59.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:08:59.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:08:59.353 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:08:59.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:08:59.354 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:08:59.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:08:59.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:08:59.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:08:59.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:08:59.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:08:59.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:08:59.354 | INFO | ------------------------------------------ | 20260507 20:08:59.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:08:59.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:08:59.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:08:59.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:08:59.354 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.59e+01, 1.43e+01 / 2.05e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.04e-01, 1.19e-02 / 5.90e-03 | 20260507 20:08:59.354 | INFO | PerfReport: 12.12%, Baseline -- 4.02e+00 / 4.47e+00, 6.05e-01 / 3.01e-01 | 20260507 20:08:59.354 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.10e-01 / 1.29e-01 | 20260507 20:08:59.354 | INFO | PerfReport: 15.80%, DME -- 4.19e+01 / 4.52e+01, 4.04e+00 / 1.91e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.82e+00 / 1.91e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.49e+00 / 2.80e+00, 4.41e-01 / 2.39e-01 | 20260507 20:08:59.354 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.39e+00, 3.03e-01 / 1.82e-01 | 20260507 20:08:59.354 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.73e-01 / 4.08e-01, 6.00e-02 / 3.65e-02 | 20260507 20:08:59.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:08:59.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:08:59.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:08:59.354 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 24.19%, Upload -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 31.03%, Compute -- 3.32e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 2.02%, LockUpload -- 2.17e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 2.48%, LockDownload -- 2.66e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 16.03%, LockCompute -- 1.71e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:08:59.354 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 20:08:59.449 | INFO | Starting a round of Baz IO | 20260507 20:09:02.941 | INFO | Populating RTMetrics report | 20260507 20:09:02.941 | INFO | RTMetrics report populated | 20260507 20:09:02.941 | INFO | Opening tmp RTMetrics json file | 20260507 20:09:02.980 | INFO | Serializing RTMetrics to json | 20260507 20:09:03.025 | INFO | Serialization done, swapping tmp file now... | 20260507 20:09:03.025 | INFO | RTMetrics done | 20260507 20:09:03.052 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.602 seconds | 20260507 20:09:03.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.74% / 0.01% / 55.25% | 20260507 20:09:03.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.12145% / 1.19209e-05% / 3.57628e-05% | 20260507 20:09:03.052 | INFO | Baz IO overall: | 20260507 20:09:03.052 | INFO | BytesWritten : 1021300740330 | 20260507 20:09:03.052 | INFO | NumEvents : 495136772151 | 20260507 20:09:04.492 | INFO | Analyzing chunk frames = [2252800,2253312) | 20260507 20:09:04.492 | INFO | MemInfo: { | 20260507 20:09:04.492 | INFO | "gpu" : " 17 GB", | 20260507 20:09:04.492 | INFO | "huge" : "", | 20260507 20:09:04.492 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:04.492 | INFO | "malloc" : " 14 GB", | 20260507 20:09:04.492 | INFO | "pinned" : " 29 GB" | 20260507 20:09:04.492 | INFO | } | 20260507 20:09:04.492 | INFO | Perf report for the last 5.13977 seconds: | 20260507 20:09:04.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:04.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:04.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:04.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:04.492 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:09:04.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:04.493 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 20:09:04.493 | INFO | PRE_HQ : 66.0%, 1.7, 58.0ms, 0 | 20260507 20:09:04.493 | INFO | BAZWRITER : 69.6%, 3.9, 141.1ms, 0 | 20260507 20:09:04.493 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260507 20:09:04.493 | INFO | Uploaded 36GiB to GPU in 2.53314s (14.2116GiB/s) | 20260507 20:09:04.493 | INFO | Downloaded 38GiB to GPU in 2.70305s (14.0582GiB/s) | 20260507 20:09:09.632 | INFO | Analyzing chunk frames = [2253312,2253824) | 20260507 20:09:09.632 | INFO | MemInfo: { | 20260507 20:09:09.632 | INFO | "gpu" : " 17 GB", | 20260507 20:09:09.632 | INFO | "huge" : "", | 20260507 20:09:09.632 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:09.632 | INFO | "malloc" : " 14 GB", | 20260507 20:09:09.633 | INFO | "pinned" : " 29 GB" | 20260507 20:09:09.633 | INFO | } | 20260507 20:09:09.633 | INFO | Perf report for the last 5.14036 seconds: | 20260507 20:09:09.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:09.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:09:09.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:09:09.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:09.633 | INFO | Uploaded 36GiB to GPU in 2.51835s (14.2951GiB/s) | 20260507 20:09:09.633 | INFO | Downloaded 34GiB to GPU in 2.48708s (13.6707GiB/s) | 20260507 20:09:14.771 | INFO | Analyzing chunk frames = [2253824,2254336) | 20260507 20:09:14.771 | INFO | MemInfo: { | 20260507 20:09:14.771 | INFO | "gpu" : " 17 GB", | 20260507 20:09:14.771 | INFO | "huge" : "", | 20260507 20:09:14.771 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:14.771 | INFO | "malloc" : " 14 GB", | 20260507 20:09:14.771 | INFO | "pinned" : " 29 GB" | 20260507 20:09:14.771 | INFO | } | 20260507 20:09:14.771 | INFO | Perf report for the last 5.13838 seconds: | 20260507 20:09:14.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:14.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.771 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:09:14.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.771 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:09:14.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:14.772 | INFO | Uploaded 36GiB to GPU in 2.55043s (14.1153GiB/s) | 20260507 20:09:14.772 | INFO | Downloaded 34GiB to GPU in 2.48453s (13.6847GiB/s) | 20260507 20:09:19.910 | INFO | Analyzing chunk frames = [2254336,2254848) | 20260507 20:09:19.910 | INFO | MemInfo: { | 20260507 20:09:19.910 | INFO | "gpu" : " 17 GB", | 20260507 20:09:19.910 | INFO | "huge" : "", | 20260507 20:09:19.910 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:19.910 | INFO | "malloc" : " 14 GB", | 20260507 20:09:19.910 | INFO | "pinned" : " 29 GB" | 20260507 20:09:19.910 | INFO | } | 20260507 20:09:19.910 | INFO | Perf report for the last 5.13925 seconds: | 20260507 20:09:19.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:19.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.910 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260507 20:09:19.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.910 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:09:19.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:19.911 | INFO | Uploaded 36GiB to GPU in 2.44356s (14.7326GiB/s) | 20260507 20:09:19.911 | INFO | Downloaded 34GiB to GPU in 2.41179s (14.0974GiB/s) | 20260507 20:09:25.050 | INFO | Analyzing chunk frames = [2254848,2255360) | 20260507 20:09:25.050 | INFO | MemInfo: { | 20260507 20:09:25.050 | INFO | "gpu" : " 17 GB", | 20260507 20:09:25.050 | INFO | "huge" : "", | 20260507 20:09:25.050 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:25.050 | INFO | "malloc" : " 14 GB", | 20260507 20:09:25.050 | INFO | "pinned" : " 29 GB" | 20260507 20:09:25.050 | INFO | } | 20260507 20:09:25.050 | INFO | Perf report for the last 5.14002 seconds: | 20260507 20:09:25.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:25.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.050 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:09:25.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.050 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:09:25.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:25.051 | INFO | Uploaded 36GiB to GPU in 2.35237s (15.3037GiB/s) | 20260507 20:09:25.051 | INFO | Downloaded 34GiB to GPU in 2.5089s (13.5518GiB/s) | 20260507 20:09:30.189 | INFO | Analyzing chunk frames = [2255360,2255872) | 20260507 20:09:30.189 | INFO | MemInfo: { | 20260507 20:09:30.189 | INFO | "gpu" : " 17 GB", | 20260507 20:09:30.189 | INFO | "huge" : "", | 20260507 20:09:30.189 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:30.189 | INFO | "malloc" : " 14 GB", | 20260507 20:09:30.189 | INFO | "pinned" : " 29 GB" | 20260507 20:09:30.189 | INFO | } | 20260507 20:09:30.189 | INFO | Perf report for the last 5.1386 seconds: | 20260507 20:09:30.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:30.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.189 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:09:30.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.189 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260507 20:09:30.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:30.190 | INFO | Uploaded 36GiB to GPU in 2.50206s (14.3881GiB/s) | 20260507 20:09:30.190 | INFO | Downloaded 34GiB to GPU in 2.42616s (14.0139GiB/s) | 20260507 20:09:35.328 | INFO | Analyzing chunk frames = [2255872,2256384) | 20260507 20:09:35.328 | INFO | MemInfo: { | 20260507 20:09:35.328 | INFO | "gpu" : " 17 GB", | 20260507 20:09:35.328 | INFO | "huge" : "", | 20260507 20:09:35.328 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:35.328 | INFO | "malloc" : " 14 GB", | 20260507 20:09:35.328 | INFO | "pinned" : " 29 GB" | 20260507 20:09:35.328 | INFO | } | 20260507 20:09:35.328 | INFO | Perf report for the last 5.1394 seconds: | 20260507 20:09:35.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:35.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.328 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:09:35.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.328 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:09:35.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:35.329 | INFO | Uploaded 36GiB to GPU in 2.52466s (14.2593GiB/s) | 20260507 20:09:35.329 | INFO | Downloaded 34GiB to GPU in 2.45866s (13.8287GiB/s) | 20260507 20:09:40.469 | INFO | Analyzing chunk frames = [2256384,2256896) | 20260507 20:09:40.469 | INFO | MemInfo: { | 20260507 20:09:40.469 | INFO | "gpu" : " 17 GB", | 20260507 20:09:40.469 | INFO | "huge" : "", | 20260507 20:09:40.469 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:40.469 | INFO | "malloc" : " 14 GB", | 20260507 20:09:40.469 | INFO | "pinned" : " 29 GB" | 20260507 20:09:40.469 | INFO | } | 20260507 20:09:40.469 | INFO | Perf report for the last 5.14062 seconds: | 20260507 20:09:40.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:40.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 20:09:40.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:09:40.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:40.469 | INFO | Uploaded 36GiB to GPU in 2.47321s (14.556GiB/s) | 20260507 20:09:40.469 | INFO | Downloaded 34GiB to GPU in 2.4255s (14.0177GiB/s) | 20260507 20:09:40.566 | INFO | Starting a round of Baz IO | 20260507 20:09:44.045 | INFO | Populating RTMetrics report | 20260507 20:09:44.046 | INFO | RTMetrics report populated | 20260507 20:09:44.046 | INFO | Opening tmp RTMetrics json file | 20260507 20:09:44.076 | INFO | Serializing RTMetrics to json | 20260507 20:09:44.135 | INFO | Serialization done, swapping tmp file now... | 20260507 20:09:44.139 | INFO | RTMetrics done | 20260507 20:09:44.175 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.608 seconds | 20260507 20:09:44.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.73% / 0.01% / 55.26% | 20260507 20:09:44.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.11381% / 4.37101e-05% / 5.96046e-05% | 20260507 20:09:44.175 | INFO | Baz IO overall: | 20260507 20:09:44.175 | INFO | BytesWritten : 1023213757942 | 20260507 20:09:44.175 | INFO | NumEvents : 496063667114 | 20260507 20:09:45.607 | INFO | Analyzing chunk frames = [2256896,2257408) | 20260507 20:09:45.607 | INFO | MemInfo: { | 20260507 20:09:45.607 | INFO | "gpu" : " 17 GB", | 20260507 20:09:45.607 | INFO | "huge" : "", | 20260507 20:09:45.608 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:45.608 | INFO | "malloc" : " 14 GB", | 20260507 20:09:45.608 | INFO | "pinned" : " 29 GB" | 20260507 20:09:45.608 | INFO | } | 20260507 20:09:45.608 | INFO | Perf report for the last 5.13858 seconds: | 20260507 20:09:45.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:45.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:45.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:45.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:45.608 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 20:09:45.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:45.608 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 20:09:45.608 | INFO | PRE_HQ : 65.9%, 1.7, 57.8ms, 0 | 20260507 20:09:45.608 | INFO | BAZWRITER : 69.8%, 3.9, 138.7ms, 0 | 20260507 20:09:45.608 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 20:09:45.608 | INFO | Uploaded 36GiB to GPU in 2.54782s (14.1298GiB/s) | 20260507 20:09:45.608 | INFO | Downloaded 38GiB to GPU in 2.69486s (14.1009GiB/s) | 20260507 20:09:50.746 | INFO | Analyzing chunk frames = [2257408,2257920) | 20260507 20:09:50.746 | INFO | MemInfo: { | 20260507 20:09:50.746 | INFO | "gpu" : " 17 GB", | 20260507 20:09:50.746 | INFO | "huge" : "", | 20260507 20:09:50.746 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:50.746 | INFO | "malloc" : " 14 GB", | 20260507 20:09:50.746 | INFO | "pinned" : " 29 GB" | 20260507 20:09:50.746 | INFO | } | 20260507 20:09:50.746 | INFO | Perf report for the last 5.13853 seconds: | 20260507 20:09:50.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:50.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.746 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 20:09:50.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.746 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:09:50.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:50.747 | INFO | Uploaded 36GiB to GPU in 2.589s (13.905GiB/s) | 20260507 20:09:50.747 | INFO | Downloaded 34GiB to GPU in 2.47833s (13.7189GiB/s) | 20260507 20:09:55.885 | INFO | Analyzing chunk frames = [2257920,2258432) | 20260507 20:09:55.885 | INFO | MemInfo: { | 20260507 20:09:55.885 | INFO | "gpu" : " 17 GB", | 20260507 20:09:55.885 | INFO | "huge" : "", | 20260507 20:09:55.885 | INFO | "huge_shared" : " 29 GB", | 20260507 20:09:55.885 | INFO | "malloc" : " 14 GB", | 20260507 20:09:55.885 | INFO | "pinned" : " 29 GB" | 20260507 20:09:55.885 | INFO | } | 20260507 20:09:55.885 | INFO | Perf report for the last 5.1392 seconds: | 20260507 20:09:55.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:09:55.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.885 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 20:09:55.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.885 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:09:55.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:09:55.886 | INFO | Uploaded 36GiB to GPU in 2.5962s (13.8664GiB/s) | 20260507 20:09:55.886 | INFO | Downloaded 34GiB to GPU in 2.46314s (13.8035GiB/s) | 20260507 20:10:01.025 | INFO | Analyzing chunk frames = [2258432,2258944) | 20260507 20:10:01.025 | INFO | MemInfo: { | 20260507 20:10:01.025 | INFO | "gpu" : " 17 GB", | 20260507 20:10:01.025 | INFO | "huge" : "", | 20260507 20:10:01.025 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:01.025 | INFO | "malloc" : " 14 GB", | 20260507 20:10:01.025 | INFO | "pinned" : " 29 GB" | 20260507 20:10:01.025 | INFO | } | 20260507 20:10:01.025 | INFO | Perf report for the last 5.13967 seconds: | 20260507 20:10:01.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:01.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:10:01.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:10:01.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:01.025 | INFO | Uploaded 36GiB to GPU in 2.48681s (14.4764GiB/s) | 20260507 20:10:01.026 | INFO | Downloaded 34GiB to GPU in 2.3932s (14.2069GiB/s) | 20260507 20:10:06.164 | INFO | Analyzing chunk frames = [2258944,2259456) | 20260507 20:10:06.164 | INFO | MemInfo: { | 20260507 20:10:06.164 | INFO | "gpu" : " 17 GB", | 20260507 20:10:06.164 | INFO | "huge" : "", | 20260507 20:10:06.164 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:06.164 | INFO | "malloc" : " 14 GB", | 20260507 20:10:06.164 | INFO | "pinned" : " 29 GB" | 20260507 20:10:06.164 | INFO | } | 20260507 20:10:06.164 | INFO | Perf report for the last 5.13927 seconds: | 20260507 20:10:06.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:06.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.164 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:10:06.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.164 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 20:10:06.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:06.165 | INFO | Uploaded 36GiB to GPU in 2.40982s (14.9389GiB/s) | 20260507 20:10:06.165 | INFO | Downloaded 34GiB to GPU in 2.4829s (13.6937GiB/s) | 20260507 20:10:11.303 | INFO | Analyzing chunk frames = [2259456,2259968) | 20260507 20:10:11.304 | INFO | MemInfo: { | 20260507 20:10:11.304 | INFO | "gpu" : " 17 GB", | 20260507 20:10:11.304 | INFO | "huge" : "", | 20260507 20:10:11.304 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:11.304 | INFO | "malloc" : " 14 GB", | 20260507 20:10:11.304 | INFO | "pinned" : " 29 GB" | 20260507 20:10:11.304 | INFO | } | 20260507 20:10:11.304 | INFO | Perf report for the last 5.13942 seconds: | 20260507 20:10:11.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:11.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 20:10:11.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:10:11.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:11.304 | INFO | Uploaded 36GiB to GPU in 2.50937s (14.3462GiB/s) | 20260507 20:10:11.304 | INFO | Downloaded 34GiB to GPU in 2.42001s (14.0495GiB/s) | 20260507 20:10:16.444 | INFO | Analyzing chunk frames = [2259968,2260480) | 20260507 20:10:16.444 | INFO | MemInfo: { | 20260507 20:10:16.444 | INFO | "gpu" : " 17 GB", | 20260507 20:10:16.444 | INFO | "huge" : "", | 20260507 20:10:16.444 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:16.444 | INFO | "malloc" : " 14 GB", | 20260507 20:10:16.444 | INFO | "pinned" : " 29 GB" | 20260507 20:10:16.444 | INFO | } | 20260507 20:10:16.444 | INFO | Perf report for the last 5.14046 seconds: | 20260507 20:10:16.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:16.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.444 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:10:16.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.444 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:10:16.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:16.445 | INFO | Uploaded 36GiB to GPU in 2.53548s (14.1985GiB/s) | 20260507 20:10:16.445 | INFO | Downloaded 34GiB to GPU in 2.4536s (13.8572GiB/s) | 20260507 20:10:21.583 | INFO | Analyzing chunk frames = [2260480,2260992) | 20260507 20:10:21.583 | INFO | MemInfo: { | 20260507 20:10:21.583 | INFO | "gpu" : " 17 GB", | 20260507 20:10:21.583 | INFO | "huge" : "", | 20260507 20:10:21.583 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:21.583 | INFO | "malloc" : " 14 GB", | 20260507 20:10:21.583 | INFO | "pinned" : " 29 GB" | 20260507 20:10:21.583 | INFO | } | 20260507 20:10:21.583 | INFO | Perf report for the last 5.13855 seconds: | 20260507 20:10:21.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:21.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:10:21.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:10:21.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:21.583 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:10:21.583 | INFO | Total mem usage: 16.4 GiB | 20260507 20:10:21.583 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:10:21.583 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.59%) | 20260507 20:10:21.583 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:10:21.583 | INFO | High Water Breakdown: | 20260507 20:10:21.583 | INFO | 60% : 8.98 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:10:21.583 | INFO | 39.9% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:10:21.583 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:10:21.583 | INFO | ------------------------------------------ | 20260507 20:10:21.583 | INFO | Uploaded 36GiB to GPU in 2.48166s (14.5064GiB/s) | 20260507 20:10:21.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:10:21.584 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:10:21.584 | INFO | Downloaded 34GiB to GPU in 2.42559s (14.0172GiB/s) | 20260507 20:10:21.584 | INFO | Total mem usage: 26.5 GiB | 20260507 20:10:21.584 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:10:21.584 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:10:21.584 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:10:21.584 | INFO | High Water Breakdown: | 20260507 20:10:21.584 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:10:21.584 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:10:21.584 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:10:21.584 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:10:21.584 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:10:21.584 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:10:21.584 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:10:21.584 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:10:21.584 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:10:21.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:10:21.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:10:21.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:10:21.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:10:21.584 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:10:21.584 | INFO | ------------------------------------------ | 20260507 20:10:21.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:10:21.584 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:10:21.584 | INFO | Total mem usage: 27.8 GiB | 20260507 20:10:21.584 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:10:21.584 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:10:21.584 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:10:21.584 | INFO | High Water Breakdown: | 20260507 20:10:21.584 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:10:21.584 | INFO | ------------------------------------------ | 20260507 20:10:21.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:10:21.584 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:10:21.584 | INFO | Total mem usage: 15.7 GiB | 20260507 20:10:21.584 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:10:21.584 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:10:21.584 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:10:21.584 | INFO | High Water Breakdown: | 20260507 20:10:21.584 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:10:21.584 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:10:21.584 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:10:21.584 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:10:21.584 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:10:21.584 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:10:21.584 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:10:21.584 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:10:21.584 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:10:21.584 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:10:21.584 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:10:21.584 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:10:21.584 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:10:21.584 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:10:21.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:10:21.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:10:21.584 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:10:21.584 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:10:21.584 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:10:21.584 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:10:21.584 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:10:21.584 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:10:21.584 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:10:21.584 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:10:21.584 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:10:21.584 | INFO | ------------------------------------------ | 20260507 20:10:21.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:10:21.584 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:10:21.584 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:10:21.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:10:21.584 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.42e+01 / 1.65e+00 | 20260507 20:10:21.584 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.29e-02 / 5.79e-03 | 20260507 20:10:21.584 | INFO | PerfReport: 12.09%, Baseline -- 4.00e+00 / 4.49e+00, 5.98e-01 / 2.45e-01 | 20260507 20:10:21.584 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 9.62e-02 | 20260507 20:10:21.584 | INFO | PerfReport: 15.74%, DME -- 4.17e+01 / 4.49e+01, 3.82e+00 / 1.26e+00 | 20260507 20:10:21.584 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.69e+00 | 20260507 20:10:21.584 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.49e+00 / 2.82e+00, 4.40e-01 / 2.19e-01 | 20260507 20:10:21.584 | INFO | PerfReport: 3.60%, Metrics -- 1.19e+00 / 1.41e+00, 3.04e-01 / 1.98e-01 | 20260507 20:10:21.584 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.04e-01, 6.16e-02 / 3.14e-02 | 20260507 20:10:21.584 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:10:21.584 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:10:21.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:10:21.584 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:10:21.584 | INFO | PerfReport: 24.39%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 20:10:21.584 | INFO | PerfReport: 24.18%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 20:10:21.585 | INFO | PerfReport: 30.95%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:10:21.585 | INFO | PerfReport: 1.98%, LockUpload -- 2.12e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260507 20:10:21.585 | INFO | PerfReport: 2.46%, LockDownload -- 2.64e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260507 20:10:21.585 | INFO | PerfReport: 16.01%, LockCompute -- 1.71e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:10:21.585 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.11e-02 / 0.00e+00 | 20260507 20:10:21.680 | INFO | Starting a round of Baz IO | 20260507 20:10:25.153 | INFO | Populating RTMetrics report | 20260507 20:10:25.154 | INFO | RTMetrics report populated | 20260507 20:10:25.154 | INFO | Opening tmp RTMetrics json file | 20260507 20:10:25.195 | INFO | Serializing RTMetrics to json | 20260507 20:10:25.263 | INFO | Serialization done, swapping tmp file now... | 20260507 20:10:25.265 | INFO | RTMetrics done | 20260507 20:10:25.295 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.615 seconds | 20260507 20:10:25.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.72% / 0.01% / 55.27% | 20260507 20:10:25.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.10831% / 1.58946e-05% / 4.76837e-05% | 20260507 20:10:25.295 | INFO | Baz IO overall: | 20260507 20:10:25.296 | INFO | BytesWritten : 1025126250432 | 20260507 20:10:25.296 | INFO | NumEvents : 496990120332 | 20260507 20:10:26.722 | INFO | Analyzing chunk frames = [2260992,2261504) | 20260507 20:10:26.722 | INFO | MemInfo: { | 20260507 20:10:26.722 | INFO | "gpu" : " 17 GB", | 20260507 20:10:26.722 | INFO | "huge" : "", | 20260507 20:10:26.722 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:26.722 | INFO | "malloc" : " 14 GB", | 20260507 20:10:26.722 | INFO | "pinned" : " 29 GB" | 20260507 20:10:26.722 | INFO | } | 20260507 20:10:26.722 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:10:26.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:26.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:26.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:26.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:26.722 | INFO | ANALYSIS : 68.7%, 3.0, 107.0ms, 0 | 20260507 20:10:26.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:26.722 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 20:10:26.722 | INFO | PRE_HQ : 65.8%, 1.7, 58.3ms, 0 | 20260507 20:10:26.722 | INFO | BAZWRITER : 69.9%, 3.7, 134.4ms, 0 | 20260507 20:10:26.722 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260507 20:10:26.722 | INFO | Uploaded 36GiB to GPU in 2.53489s (14.2018GiB/s) | 20260507 20:10:26.723 | INFO | Downloaded 38GiB to GPU in 2.68361s (14.16GiB/s) | 20260507 20:10:31.861 | INFO | Analyzing chunk frames = [2261504,2262016) | 20260507 20:10:31.861 | INFO | MemInfo: { | 20260507 20:10:31.861 | INFO | "gpu" : " 17 GB", | 20260507 20:10:31.861 | INFO | "huge" : "", | 20260507 20:10:31.861 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:31.861 | INFO | "malloc" : " 14 GB", | 20260507 20:10:31.861 | INFO | "pinned" : " 29 GB" | 20260507 20:10:31.861 | INFO | } | 20260507 20:10:31.861 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:10:31.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:31.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.861 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:10:31.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.862 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:10:31.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:31.862 | INFO | Uploaded 36GiB to GPU in 2.56475s (14.0365GiB/s) | 20260507 20:10:31.862 | INFO | Downloaded 34GiB to GPU in 2.47538s (13.7352GiB/s) | 20260507 20:10:37.000 | INFO | Analyzing chunk frames = [2262016,2262528) | 20260507 20:10:37.000 | INFO | MemInfo: { | 20260507 20:10:37.000 | INFO | "gpu" : " 17 GB", | 20260507 20:10:37.000 | INFO | "huge" : "", | 20260507 20:10:37.001 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:37.001 | INFO | "malloc" : " 14 GB", | 20260507 20:10:37.001 | INFO | "pinned" : " 29 GB" | 20260507 20:10:37.001 | INFO | } | 20260507 20:10:37.001 | INFO | Perf report for the last 5.13934 seconds: | 20260507 20:10:37.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:37.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:10:37.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 20:10:37.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:37.001 | INFO | Uploaded 36GiB to GPU in 2.55247s (14.104GiB/s) | 20260507 20:10:37.001 | INFO | Downloaded 34GiB to GPU in 2.46788s (13.777GiB/s) | 20260507 20:10:42.141 | INFO | Analyzing chunk frames = [2262528,2263040) | 20260507 20:10:42.141 | INFO | MemInfo: { | 20260507 20:10:42.141 | INFO | "gpu" : " 17 GB", | 20260507 20:10:42.141 | INFO | "huge" : "", | 20260507 20:10:42.141 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:42.141 | INFO | "malloc" : " 14 GB", | 20260507 20:10:42.141 | INFO | "pinned" : " 29 GB" | 20260507 20:10:42.141 | INFO | } | 20260507 20:10:42.141 | INFO | Perf report for the last 5.14014 seconds: | 20260507 20:10:42.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:42.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.141 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 20:10:42.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.141 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 20:10:42.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:42.142 | INFO | Uploaded 36GiB to GPU in 2.50943s (14.3459GiB/s) | 20260507 20:10:42.142 | INFO | Downloaded 34GiB to GPU in 2.48989s (13.6552GiB/s) | 20260507 20:10:47.280 | INFO | Analyzing chunk frames = [2263040,2263552) | 20260507 20:10:47.280 | INFO | MemInfo: { | 20260507 20:10:47.280 | INFO | "gpu" : " 17 GB", | 20260507 20:10:47.280 | INFO | "huge" : "", | 20260507 20:10:47.280 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:47.280 | INFO | "malloc" : " 14 GB", | 20260507 20:10:47.280 | INFO | "pinned" : " 29 GB" | 20260507 20:10:47.280 | INFO | } | 20260507 20:10:47.280 | INFO | Perf report for the last 5.13914 seconds: | 20260507 20:10:47.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:47.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:10:47.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:10:47.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:47.280 | INFO | Uploaded 36GiB to GPU in 2.36808s (15.2022GiB/s) | 20260507 20:10:47.280 | INFO | Downloaded 34GiB to GPU in 2.49873s (13.6069GiB/s) | 20260507 20:10:52.418 | INFO | Analyzing chunk frames = [2263552,2264064) | 20260507 20:10:52.419 | INFO | MemInfo: { | 20260507 20:10:52.419 | INFO | "gpu" : " 17 GB", | 20260507 20:10:52.419 | INFO | "huge" : "", | 20260507 20:10:52.419 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:52.419 | INFO | "malloc" : " 14 GB", | 20260507 20:10:52.419 | INFO | "pinned" : " 29 GB" | 20260507 20:10:52.419 | INFO | } | 20260507 20:10:52.419 | INFO | Perf report for the last 5.13868 seconds: | 20260507 20:10:52.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:52.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:10:52.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:10:52.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:52.419 | INFO | Uploaded 36GiB to GPU in 2.50587s (14.3663GiB/s) | 20260507 20:10:52.419 | INFO | Downloaded 34GiB to GPU in 2.41512s (14.078GiB/s) | 20260507 20:10:57.558 | INFO | Analyzing chunk frames = [2264064,2264576) | 20260507 20:10:57.558 | INFO | MemInfo: { | 20260507 20:10:57.558 | INFO | "gpu" : " 17 GB", | 20260507 20:10:57.558 | INFO | "huge" : "", | 20260507 20:10:57.558 | INFO | "huge_shared" : " 29 GB", | 20260507 20:10:57.558 | INFO | "malloc" : " 14 GB", | 20260507 20:10:57.558 | INFO | "pinned" : " 29 GB" | 20260507 20:10:57.558 | INFO | } | 20260507 20:10:57.558 | INFO | Perf report for the last 5.13927 seconds: | 20260507 20:10:57.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:10:57.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 20:10:57.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:10:57.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:10:57.558 | INFO | Uploaded 36GiB to GPU in 2.5204s (14.2834GiB/s) | 20260507 20:10:57.558 | INFO | Downloaded 34GiB to GPU in 2.432s (13.9802GiB/s) | 20260507 20:11:02.696 | INFO | Analyzing chunk frames = [2264576,2265088) | 20260507 20:11:02.696 | INFO | MemInfo: { | 20260507 20:11:02.697 | INFO | "gpu" : " 17 GB", | 20260507 20:11:02.697 | INFO | "huge" : "", | 20260507 20:11:02.697 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:02.697 | INFO | "malloc" : " 14 GB", | 20260507 20:11:02.697 | INFO | "pinned" : " 29 GB" | 20260507 20:11:02.697 | INFO | } | 20260507 20:11:02.697 | INFO | Perf report for the last 5.13864 seconds: | 20260507 20:11:02.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:02.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 20:11:02.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:11:02.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:02.697 | INFO | Uploaded 36GiB to GPU in 2.52753s (14.2431GiB/s) | 20260507 20:11:02.697 | INFO | Downloaded 34GiB to GPU in 2.43703s (13.9514GiB/s) | 20260507 20:11:02.793 | INFO | Starting a round of Baz IO | 20260507 20:11:06.271 | INFO | Populating RTMetrics report | 20260507 20:11:06.272 | INFO | RTMetrics report populated | 20260507 20:11:06.272 | INFO | Opening tmp RTMetrics json file | 20260507 20:11:06.294 | INFO | Serializing RTMetrics to json | 20260507 20:11:06.296 | INFO | Serialization done, swapping tmp file now... | 20260507 20:11:06.296 | INFO | RTMetrics done | 20260507 20:11:06.371 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.578 seconds | 20260507 20:11:06.372 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.71% / 0.01% / 55.28% | 20260507 20:11:06.372 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09923% / 2.78155e-05% / 8.34465e-05% | 20260507 20:11:06.372 | INFO | Baz IO overall: | 20260507 20:11:06.372 | INFO | BytesWritten : 1027037900315 | 20260507 20:11:06.372 | INFO | NumEvents : 497915803351 | 20260507 20:11:07.837 | INFO | Analyzing chunk frames = [2265088,2265600) | 20260507 20:11:07.837 | INFO | MemInfo: { | 20260507 20:11:07.837 | INFO | "gpu" : " 17 GB", | 20260507 20:11:07.837 | INFO | "huge" : "", | 20260507 20:11:07.837 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:07.837 | INFO | "malloc" : " 14 GB", | 20260507 20:11:07.837 | INFO | "pinned" : " 29 GB" | 20260507 20:11:07.837 | INFO | } | 20260507 20:11:07.837 | INFO | Perf report for the last 5.14 seconds: | 20260507 20:11:07.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:07.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:07.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:07.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:07.837 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:11:07.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:07.837 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:11:07.837 | INFO | PRE_HQ : 65.9%, 1.7, 58.1ms, 0 | 20260507 20:11:07.837 | INFO | BAZWRITER : 69.2%, 3.9, 139.9ms, 0 | 20260507 20:11:07.837 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 20:11:07.837 | INFO | Uploaded 36GiB to GPU in 2.55912s (14.0673GiB/s) | 20260507 20:11:07.837 | INFO | Downloaded 38GiB to GPU in 2.69309s (14.1102GiB/s) | 20260507 20:11:12.976 | INFO | Analyzing chunk frames = [2265600,2266112) | 20260507 20:11:12.976 | INFO | MemInfo: { | 20260507 20:11:12.976 | INFO | "gpu" : " 17 GB", | 20260507 20:11:12.976 | INFO | "huge" : "", | 20260507 20:11:12.976 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:12.976 | INFO | "malloc" : " 14 GB", | 20260507 20:11:12.976 | INFO | "pinned" : " 29 GB" | 20260507 20:11:12.976 | INFO | } | 20260507 20:11:12.977 | INFO | Perf report for the last 5.13981 seconds: | 20260507 20:11:12.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:12.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 20:11:12.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 20:11:12.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:12.977 | INFO | Uploaded 36GiB to GPU in 2.53909s (14.1783GiB/s) | 20260507 20:11:12.977 | INFO | Downloaded 34GiB to GPU in 2.4994s (13.6033GiB/s) | 20260507 20:11:18.116 | INFO | Analyzing chunk frames = [2266112,2266624) | 20260507 20:11:18.116 | INFO | MemInfo: { | 20260507 20:11:18.116 | INFO | "gpu" : " 17 GB", | 20260507 20:11:18.116 | INFO | "huge" : "", | 20260507 20:11:18.116 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:18.116 | INFO | "malloc" : " 14 GB", | 20260507 20:11:18.116 | INFO | "pinned" : " 29 GB" | 20260507 20:11:18.116 | INFO | } | 20260507 20:11:18.116 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:11:18.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:18.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 20:11:18.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 20:11:18.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:18.116 | INFO | Uploaded 36GiB to GPU in 2.56328s (14.0445GiB/s) | 20260507 20:11:18.116 | INFO | Downloaded 34GiB to GPU in 2.47703s (13.7261GiB/s) | 20260507 20:11:23.255 | INFO | Analyzing chunk frames = [2266624,2267136) | 20260507 20:11:23.255 | INFO | MemInfo: { | 20260507 20:11:23.255 | INFO | "gpu" : " 17 GB", | 20260507 20:11:23.255 | INFO | "huge" : "", | 20260507 20:11:23.255 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:23.255 | INFO | "malloc" : " 14 GB", | 20260507 20:11:23.255 | INFO | "pinned" : " 29 GB" | 20260507 20:11:23.255 | INFO | } | 20260507 20:11:23.255 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:11:23.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:23.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.255 | INFO | ANALYSIS : 67.7%, 3.0, 104.7ms, 0 | 20260507 20:11:23.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.255 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 20:11:23.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:23.256 | INFO | Uploaded 36GiB to GPU in 2.49513s (14.4281GiB/s) | 20260507 20:11:23.256 | INFO | Downloaded 34GiB to GPU in 2.48406s (13.6873GiB/s) | 20260507 20:11:28.395 | INFO | Analyzing chunk frames = [2267136,2267648) | 20260507 20:11:28.395 | INFO | MemInfo: { | 20260507 20:11:28.395 | INFO | "gpu" : " 17 GB", | 20260507 20:11:28.395 | INFO | "huge" : "", | 20260507 20:11:28.395 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:28.395 | INFO | "malloc" : " 14 GB", | 20260507 20:11:28.395 | INFO | "pinned" : " 29 GB" | 20260507 20:11:28.395 | INFO | } | 20260507 20:11:28.395 | INFO | Perf report for the last 5.13994 seconds: | 20260507 20:11:28.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:28.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.395 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 20:11:28.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.395 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260507 20:11:28.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:28.396 | INFO | Uploaded 36GiB to GPU in 2.37581s (15.1527GiB/s) | 20260507 20:11:28.396 | INFO | Downloaded 34GiB to GPU in 2.48163s (13.7007GiB/s) | 20260507 20:11:33.533 | INFO | Analyzing chunk frames = [2267648,2268160) | 20260507 20:11:33.533 | INFO | MemInfo: { | 20260507 20:11:33.533 | INFO | "gpu" : " 17 GB", | 20260507 20:11:33.533 | INFO | "huge" : "", | 20260507 20:11:33.533 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:33.533 | INFO | "malloc" : " 14 GB", | 20260507 20:11:33.534 | INFO | "pinned" : " 29 GB" | 20260507 20:11:33.534 | INFO | } | 20260507 20:11:33.534 | INFO | Perf report for the last 5.13871 seconds: | 20260507 20:11:33.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:33.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:11:33.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:11:33.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:33.534 | INFO | Uploaded 36GiB to GPU in 2.44904s (14.6996GiB/s) | 20260507 20:11:33.534 | INFO | Downloaded 34GiB to GPU in 2.43826s (13.9444GiB/s) | 20260507 20:11:38.673 | INFO | Analyzing chunk frames = [2268160,2268672) | 20260507 20:11:38.673 | INFO | MemInfo: { | 20260507 20:11:38.673 | INFO | "gpu" : " 17 GB", | 20260507 20:11:38.673 | INFO | "huge" : "", | 20260507 20:11:38.673 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:38.673 | INFO | "malloc" : " 14 GB", | 20260507 20:11:38.673 | INFO | "pinned" : " 29 GB" | 20260507 20:11:38.673 | INFO | } | 20260507 20:11:38.673 | INFO | Perf report for the last 5.13924 seconds: | 20260507 20:11:38.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:38.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.673 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:11:38.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.673 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:11:38.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:38.674 | INFO | Uploaded 36GiB to GPU in 2.50128s (14.3926GiB/s) | 20260507 20:11:38.674 | INFO | Downloaded 34GiB to GPU in 2.47065s (13.7616GiB/s) | 20260507 20:11:43.812 | INFO | Analyzing chunk frames = [2268672,2269184) | 20260507 20:11:43.812 | INFO | MemInfo: { | 20260507 20:11:43.812 | INFO | "gpu" : " 17 GB", | 20260507 20:11:43.812 | INFO | "huge" : "", | 20260507 20:11:43.812 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:43.812 | INFO | "malloc" : " 14 GB", | 20260507 20:11:43.812 | INFO | "pinned" : " 29 GB" | 20260507 20:11:43.812 | INFO | } | 20260507 20:11:43.812 | INFO | Perf report for the last 5.13901 seconds: | 20260507 20:11:43.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:43.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 20:11:43.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:11:43.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:43.812 | INFO | Uploaded 36GiB to GPU in 2.44645s (14.7152GiB/s) | 20260507 20:11:43.812 | INFO | Downloaded 34GiB to GPU in 2.44429s (13.9099GiB/s) | 20260507 20:11:43.812 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:11:43.813 | INFO | Total mem usage: 16.4 GiB | 20260507 20:11:43.813 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:11:43.813 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 20:11:43.813 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:11:43.813 | INFO | High Water Breakdown: | 20260507 20:11:43.813 | INFO | 59.7% : 8.8 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:11:43.813 | INFO | 40.2% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:11:43.813 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:11:43.813 | INFO | ------------------------------------------ | 20260507 20:11:43.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:11:43.813 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:11:43.813 | INFO | Total mem usage: 26.5 GiB | 20260507 20:11:43.813 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:11:43.813 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:11:43.813 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:11:43.813 | INFO | High Water Breakdown: | 20260507 20:11:43.813 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:11:43.813 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:11:43.813 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:11:43.813 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:11:43.813 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:11:43.813 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:11:43.813 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:11:43.813 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:11:43.813 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:11:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:11:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:11:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:11:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:11:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:11:43.813 | INFO | ------------------------------------------ | 20260507 20:11:43.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:11:43.813 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:11:43.813 | INFO | Total mem usage: 27.8 GiB | 20260507 20:11:43.813 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:11:43.813 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:11:43.813 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:11:43.813 | INFO | High Water Breakdown: | 20260507 20:11:43.813 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:11:43.813 | INFO | ------------------------------------------ | 20260507 20:11:43.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:11:43.813 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:11:43.813 | INFO | Total mem usage: 15.7 GiB | 20260507 20:11:43.813 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:11:43.813 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:11:43.813 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:11:43.813 | INFO | High Water Breakdown: | 20260507 20:11:43.813 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:11:43.813 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:11:43.813 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:11:43.813 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:11:43.813 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:11:43.813 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:11:43.813 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:11:43.813 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:11:43.813 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:11:43.813 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:11:43.813 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:11:43.813 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:11:43.813 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:11:43.813 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:11:43.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:11:43.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:11:43.813 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:11:43.813 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:11:43.813 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:11:43.813 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:11:43.813 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:11:43.813 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:11:43.813 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:11:43.813 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:11:43.813 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:11:43.813 | INFO | ------------------------------------------ | 20260507 20:11:43.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:11:43.813 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:11:43.813 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:11:43.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:11:43.813 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.42e+01 / 1.94e+00 | 20260507 20:11:43.813 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.03e-01, 1.30e-02 / 6.98e-03 | 20260507 20:11:43.813 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.46e+00, 5.97e-01 / 2.78e-01 | 20260507 20:11:43.814 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.08e-01 / 1.23e-01 | 20260507 20:11:43.814 | INFO | PerfReport: 15.91%, DME -- 4.21e+01 / 4.54e+01, 4.02e+00 / 1.95e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 55.96%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.70e+00 / 1.80e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.43e-01 / 2.29e-01 | 20260507 20:11:43.814 | INFO | PerfReport: 3.61%, Metrics -- 1.19e+00 / 1.39e+00, 3.00e-01 / 1.82e-01 | 20260507 20:11:43.814 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.72e-01 / 4.06e-01, 6.08e-02 / 3.19e-02 | 20260507 20:11:43.814 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:11:43.814 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:11:43.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:11:43.814 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 24.30%, Upload -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 24.23%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 30.88%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 2.05%, LockUpload -- 2.20e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 2.60%, LockDownload -- 2.79e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 15.93%, LockCompute -- 1.71e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 20:11:43.814 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 20:11:43.911 | INFO | Starting a round of Baz IO | 20260507 20:11:47.385 | INFO | Populating RTMetrics report | 20260507 20:11:47.385 | INFO | RTMetrics report populated | 20260507 20:11:47.385 | INFO | Opening tmp RTMetrics json file | 20260507 20:11:47.390 | INFO | Serializing RTMetrics to json | 20260507 20:11:47.445 | INFO | Serialization done, swapping tmp file now... | 20260507 20:11:47.460 | INFO | RTMetrics done | 20260507 20:11:47.484 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.573 seconds | 20260507 20:11:47.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.70% / 0.01% / 55.29% | 20260507 20:11:47.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09363% / 2.38419e-05% / 3.17891e-05% | 20260507 20:11:47.484 | INFO | Baz IO overall: | 20260507 20:11:47.484 | INFO | BytesWritten : 1028948246496 | 20260507 20:11:47.484 | INFO | NumEvents : 498840472043 | 20260507 20:11:48.952 | INFO | Analyzing chunk frames = [2269184,2269696) | 20260507 20:11:48.952 | INFO | MemInfo: { | 20260507 20:11:48.952 | INFO | "gpu" : " 17 GB", | 20260507 20:11:48.952 | INFO | "huge" : "", | 20260507 20:11:48.952 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:48.952 | INFO | "malloc" : " 14 GB", | 20260507 20:11:48.952 | INFO | "pinned" : " 29 GB" | 20260507 20:11:48.952 | INFO | } | 20260507 20:11:48.952 | INFO | Perf report for the last 5.13989 seconds: | 20260507 20:11:48.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:48.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:48.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:48.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:48.952 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 20:11:48.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:48.952 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 20:11:48.952 | INFO | PRE_HQ : 66.2%, 1.7, 58.2ms, 0 | 20260507 20:11:48.952 | INFO | BAZWRITER : 69.2%, 3.9, 137.5ms, 0 | 20260507 20:11:48.952 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:11:48.953 | INFO | Uploaded 36GiB to GPU in 2.44724s (14.7105GiB/s) | 20260507 20:11:48.953 | INFO | Downloaded 38GiB to GPU in 2.69344s (14.1084GiB/s) | 20260507 20:11:54.090 | INFO | Analyzing chunk frames = [2269696,2270208) | 20260507 20:11:54.091 | INFO | MemInfo: { | 20260507 20:11:54.091 | INFO | "gpu" : " 17 GB", | 20260507 20:11:54.091 | INFO | "huge" : "", | 20260507 20:11:54.091 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:54.091 | INFO | "malloc" : " 14 GB", | 20260507 20:11:54.091 | INFO | "pinned" : " 29 GB" | 20260507 20:11:54.091 | INFO | } | 20260507 20:11:54.091 | INFO | Perf report for the last 5.13887 seconds: | 20260507 20:11:54.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:54.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 20:11:54.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:11:54.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:54.091 | INFO | Uploaded 36GiB to GPU in 2.54427s (14.1494GiB/s) | 20260507 20:11:54.091 | INFO | Downloaded 34GiB to GPU in 2.47791s (13.7213GiB/s) | 20260507 20:11:59.229 | INFO | Analyzing chunk frames = [2270208,2270720) | 20260507 20:11:59.229 | INFO | MemInfo: { | 20260507 20:11:59.229 | INFO | "gpu" : " 17 GB", | 20260507 20:11:59.229 | INFO | "huge" : "", | 20260507 20:11:59.229 | INFO | "huge_shared" : " 29 GB", | 20260507 20:11:59.229 | INFO | "malloc" : " 14 GB", | 20260507 20:11:59.229 | INFO | "pinned" : " 29 GB" | 20260507 20:11:59.229 | INFO | } | 20260507 20:11:59.230 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:11:59.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:11:59.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:11:59.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:11:59.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:11:59.230 | INFO | Uploaded 36GiB to GPU in 2.47936s (14.5199GiB/s) | 20260507 20:11:59.230 | INFO | Downloaded 34GiB to GPU in 2.48966s (13.6565GiB/s) | 20260507 20:12:04.369 | INFO | Analyzing chunk frames = [2270720,2271232) | 20260507 20:12:04.369 | INFO | MemInfo: { | 20260507 20:12:04.369 | INFO | "gpu" : " 17 GB", | 20260507 20:12:04.369 | INFO | "huge" : "", | 20260507 20:12:04.369 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:04.369 | INFO | "malloc" : " 14 GB", | 20260507 20:12:04.369 | INFO | "pinned" : " 29 GB" | 20260507 20:12:04.369 | INFO | } | 20260507 20:12:04.369 | INFO | Perf report for the last 5.13951 seconds: | 20260507 20:12:04.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:04.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 20:12:04.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:12:04.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:04.369 | INFO | Uploaded 36GiB to GPU in 2.46239s (14.62GiB/s) | 20260507 20:12:04.369 | INFO | Downloaded 34GiB to GPU in 2.39581s (14.1915GiB/s) | 20260507 20:12:09.511 | INFO | Analyzing chunk frames = [2271232,2271744) | 20260507 20:12:09.511 | INFO | MemInfo: { | 20260507 20:12:09.511 | INFO | "gpu" : " 17 GB", | 20260507 20:12:09.511 | INFO | "huge" : "", | 20260507 20:12:09.511 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:09.511 | INFO | "malloc" : " 14 GB", | 20260507 20:12:09.511 | INFO | "pinned" : " 29 GB" | 20260507 20:12:09.511 | INFO | } | 20260507 20:12:09.511 | INFO | Perf report for the last 5.14169 seconds: | 20260507 20:12:09.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:09.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:12:09.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:12:09.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:09.511 | INFO | Uploaded 36GiB to GPU in 2.37746s (15.1422GiB/s) | 20260507 20:12:09.511 | INFO | Downloaded 34GiB to GPU in 2.49446s (13.6302GiB/s) | 20260507 20:12:14.648 | INFO | Analyzing chunk frames = [2271744,2272256) | 20260507 20:12:14.648 | INFO | MemInfo: { | 20260507 20:12:14.648 | INFO | "gpu" : " 17 GB", | 20260507 20:12:14.648 | INFO | "huge" : "", | 20260507 20:12:14.648 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:14.648 | INFO | "malloc" : " 14 GB", | 20260507 20:12:14.648 | INFO | "pinned" : " 29 GB" | 20260507 20:12:14.648 | INFO | } | 20260507 20:12:14.648 | INFO | Perf report for the last 5.13756 seconds: | 20260507 20:12:14.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:14.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.648 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:12:14.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.648 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:12:14.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:14.649 | INFO | Uploaded 36GiB to GPU in 2.45293s (14.6763GiB/s) | 20260507 20:12:14.649 | INFO | Downloaded 34GiB to GPU in 2.42239s (14.0357GiB/s) | 20260507 20:12:19.787 | INFO | Analyzing chunk frames = [2272256,2272768) | 20260507 20:12:19.787 | INFO | MemInfo: { | 20260507 20:12:19.787 | INFO | "gpu" : " 17 GB", | 20260507 20:12:19.787 | INFO | "huge" : "", | 20260507 20:12:19.787 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:19.787 | INFO | "malloc" : " 14 GB", | 20260507 20:12:19.787 | INFO | "pinned" : " 29 GB" | 20260507 20:12:19.787 | INFO | } | 20260507 20:12:19.787 | INFO | Perf report for the last 5.13869 seconds: | 20260507 20:12:19.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:19.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 20:12:19.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:12:19.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:19.787 | INFO | Uploaded 36GiB to GPU in 2.51741s (14.3004GiB/s) | 20260507 20:12:19.787 | INFO | Downloaded 34GiB to GPU in 2.44996s (13.8778GiB/s) | 20260507 20:12:24.927 | INFO | Analyzing chunk frames = [2272768,2273280) | 20260507 20:12:24.927 | INFO | MemInfo: { | 20260507 20:12:24.927 | INFO | "gpu" : " 17 GB", | 20260507 20:12:24.927 | INFO | "huge" : "", | 20260507 20:12:24.927 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:24.927 | INFO | "malloc" : " 14 GB", | 20260507 20:12:24.927 | INFO | "pinned" : " 29 GB" | 20260507 20:12:24.927 | INFO | } | 20260507 20:12:24.927 | INFO | Perf report for the last 5.14017 seconds: | 20260507 20:12:24.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:24.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.927 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:12:24.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.927 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:12:24.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:24.928 | INFO | Uploaded 36GiB to GPU in 2.48784s (14.4704GiB/s) | 20260507 20:12:24.928 | INFO | Downloaded 34GiB to GPU in 2.42618s (14.0138GiB/s) | 20260507 20:12:25.025 | INFO | Starting a round of Baz IO | 20260507 20:12:28.502 | INFO | Populating RTMetrics report | 20260507 20:12:28.502 | INFO | RTMetrics report populated | 20260507 20:12:28.502 | INFO | Opening tmp RTMetrics json file | 20260507 20:12:28.508 | INFO | Serializing RTMetrics to json | 20260507 20:12:28.572 | INFO | Serialization done, swapping tmp file now... | 20260507 20:12:28.595 | INFO | RTMetrics done | 20260507 20:12:28.623 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.598 seconds | 20260507 20:12:28.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.69% / 0.01% / 55.30% | 20260507 20:12:28.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09441% / 1.58946e-05% / 5.96046e-05% | 20260507 20:12:28.623 | INFO | Baz IO overall: | 20260507 20:12:28.623 | INFO | BytesWritten : 1030859986135 | 20260507 20:12:28.623 | INFO | NumEvents : 499766269853 | 20260507 20:12:30.065 | INFO | Analyzing chunk frames = [2273280,2273792) | 20260507 20:12:30.065 | INFO | MemInfo: { | 20260507 20:12:30.065 | INFO | "gpu" : " 17 GB", | 20260507 20:12:30.066 | INFO | "huge" : "", | 20260507 20:12:30.066 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:30.066 | INFO | "malloc" : " 14 GB", | 20260507 20:12:30.066 | INFO | "pinned" : " 29 GB" | 20260507 20:12:30.066 | INFO | } | 20260507 20:12:30.066 | INFO | Perf report for the last 5.13841 seconds: | 20260507 20:12:30.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:30.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:30.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:30.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:30.066 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 20:12:30.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:30.066 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 20:12:30.066 | INFO | PRE_HQ : 66.1%, 1.7, 58.0ms, 0 | 20260507 20:12:30.066 | INFO | BAZWRITER : 69.6%, 3.8, 135.0ms, 0 | 20260507 20:12:30.066 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 20:12:30.066 | INFO | Uploaded 36GiB to GPU in 2.52307s (14.2683GiB/s) | 20260507 20:12:30.066 | INFO | Downloaded 38GiB to GPU in 2.70005s (14.0738GiB/s) | 20260507 20:12:35.206 | INFO | Analyzing chunk frames = [2273792,2274304) | 20260507 20:12:35.206 | INFO | MemInfo: { | 20260507 20:12:35.206 | INFO | "gpu" : " 17 GB", | 20260507 20:12:35.206 | INFO | "huge" : "", | 20260507 20:12:35.206 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:35.206 | INFO | "malloc" : " 14 GB", | 20260507 20:12:35.206 | INFO | "pinned" : " 29 GB" | 20260507 20:12:35.206 | INFO | } | 20260507 20:12:35.206 | INFO | Perf report for the last 5.13999 seconds: | 20260507 20:12:35.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:35.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:12:35.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 20:12:35.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:35.206 | INFO | Uploaded 36GiB to GPU in 2.53418s (14.2058GiB/s) | 20260507 20:12:35.206 | INFO | Downloaded 34GiB to GPU in 2.48916s (13.6592GiB/s) | 20260507 20:12:40.344 | INFO | Analyzing chunk frames = [2274304,2274816) | 20260507 20:12:40.345 | INFO | MemInfo: { | 20260507 20:12:40.345 | INFO | "gpu" : " 17 GB", | 20260507 20:12:40.345 | INFO | "huge" : "", | 20260507 20:12:40.345 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:40.345 | INFO | "malloc" : " 14 GB", | 20260507 20:12:40.345 | INFO | "pinned" : " 29 GB" | 20260507 20:12:40.345 | INFO | } | 20260507 20:12:40.345 | INFO | Perf report for the last 5.13893 seconds: | 20260507 20:12:40.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:40.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:12:40.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:12:40.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:40.345 | INFO | Uploaded 36GiB to GPU in 2.55343s (14.0987GiB/s) | 20260507 20:12:40.345 | INFO | Downloaded 34GiB to GPU in 2.48412s (13.6869GiB/s) | 20260507 20:12:45.484 | INFO | Analyzing chunk frames = [2274816,2275328) | 20260507 20:12:45.484 | INFO | MemInfo: { | 20260507 20:12:45.484 | INFO | "gpu" : " 17 GB", | 20260507 20:12:45.484 | INFO | "huge" : "", | 20260507 20:12:45.484 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:45.484 | INFO | "malloc" : " 14 GB", | 20260507 20:12:45.484 | INFO | "pinned" : " 29 GB" | 20260507 20:12:45.484 | INFO | } | 20260507 20:12:45.485 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:12:45.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:45.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 20:12:45.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:12:45.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:45.485 | INFO | Uploaded 36GiB to GPU in 2.42666s (14.8352GiB/s) | 20260507 20:12:45.485 | INFO | Downloaded 34GiB to GPU in 2.41434s (14.0825GiB/s) | 20260507 20:12:50.624 | INFO | Analyzing chunk frames = [2275328,2275840) | 20260507 20:12:50.624 | INFO | MemInfo: { | 20260507 20:12:50.624 | INFO | "gpu" : " 17 GB", | 20260507 20:12:50.624 | INFO | "huge" : "", | 20260507 20:12:50.624 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:50.624 | INFO | "malloc" : " 14 GB", | 20260507 20:12:50.624 | INFO | "pinned" : " 29 GB" | 20260507 20:12:50.624 | INFO | } | 20260507 20:12:50.624 | INFO | Perf report for the last 5.13985 seconds: | 20260507 20:12:50.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:50.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.624 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:12:50.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.625 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:12:50.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:50.625 | INFO | Uploaded 36GiB to GPU in 2.33319s (15.4295GiB/s) | 20260507 20:12:50.625 | INFO | Downloaded 34GiB to GPU in 2.49138s (13.6471GiB/s) | 20260507 20:12:55.763 | INFO | Analyzing chunk frames = [2275840,2276352) | 20260507 20:12:55.763 | INFO | MemInfo: { | 20260507 20:12:55.763 | INFO | "gpu" : " 17 GB", | 20260507 20:12:55.763 | INFO | "huge" : "", | 20260507 20:12:55.763 | INFO | "huge_shared" : " 29 GB", | 20260507 20:12:55.763 | INFO | "malloc" : " 14 GB", | 20260507 20:12:55.763 | INFO | "pinned" : " 29 GB" | 20260507 20:12:55.763 | INFO | } | 20260507 20:12:55.763 | INFO | Perf report for the last 5.13889 seconds: | 20260507 20:12:55.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:12:55.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.763 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 20:12:55.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.763 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:12:55.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:12:55.764 | INFO | Uploaded 36GiB to GPU in 2.48654s (14.478GiB/s) | 20260507 20:12:55.764 | INFO | Downloaded 34GiB to GPU in 2.42135s (14.0417GiB/s) | 20260507 20:13:00.902 | INFO | Analyzing chunk frames = [2276352,2276864) | 20260507 20:13:00.902 | INFO | MemInfo: { | 20260507 20:13:00.902 | INFO | "gpu" : " 17 GB", | 20260507 20:13:00.902 | INFO | "huge" : "", | 20260507 20:13:00.902 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:00.902 | INFO | "malloc" : " 14 GB", | 20260507 20:13:00.902 | INFO | "pinned" : " 29 GB" | 20260507 20:13:00.902 | INFO | } | 20260507 20:13:00.902 | INFO | Perf report for the last 5.1385 seconds: | 20260507 20:13:00.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:00.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:13:00.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:13:00.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:00.902 | INFO | Uploaded 36GiB to GPU in 2.52213s (14.2737GiB/s) | 20260507 20:13:00.902 | INFO | Downloaded 34GiB to GPU in 2.45399s (13.855GiB/s) | 20260507 20:13:06.041 | INFO | Analyzing chunk frames = [2276864,2277376) | 20260507 20:13:06.041 | INFO | MemInfo: { | 20260507 20:13:06.041 | INFO | "gpu" : " 17 GB", | 20260507 20:13:06.041 | INFO | "huge" : "", | 20260507 20:13:06.041 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:06.041 | INFO | "malloc" : " 14 GB", | 20260507 20:13:06.041 | INFO | "pinned" : " 29 GB" | 20260507 20:13:06.041 | INFO | } | 20260507 20:13:06.041 | INFO | Perf report for the last 5.13945 seconds: | 20260507 20:13:06.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:06.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.041 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:13:06.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.042 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:13:06.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:06.042 | INFO | Uploaded 36GiB to GPU in 2.47215s (14.5622GiB/s) | 20260507 20:13:06.042 | INFO | Downloaded 34GiB to GPU in 2.4325s (13.9774GiB/s) | 20260507 20:13:06.042 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:13:06.042 | INFO | Total mem usage: 16.4 GiB | 20260507 20:13:06.042 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:13:06.042 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.6%) | 20260507 20:13:06.042 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:13:06.042 | INFO | High Water Breakdown: | 20260507 20:13:06.042 | INFO | 59.7% : 8.77 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:13:06.042 | INFO | 40.2% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:13:06.042 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:13:06.042 | INFO | ------------------------------------------ | 20260507 20:13:06.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:13:06.042 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:13:06.042 | INFO | Total mem usage: 26.5 GiB | 20260507 20:13:06.042 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:13:06.042 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:13:06.042 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:13:06.042 | INFO | High Water Breakdown: | 20260507 20:13:06.042 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:13:06.042 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:13:06.042 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:13:06.042 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:13:06.042 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:13:06.042 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:13:06.042 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:13:06.042 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:13:06.042 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:13:06.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:13:06.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:13:06.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:13:06.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:13:06.042 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:13:06.042 | INFO | ------------------------------------------ | 20260507 20:13:06.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:13:06.042 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:13:06.042 | INFO | Total mem usage: 27.8 GiB | 20260507 20:13:06.042 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:13:06.042 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:13:06.042 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:13:06.042 | INFO | High Water Breakdown: | 20260507 20:13:06.043 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:13:06.043 | INFO | ------------------------------------------ | 20260507 20:13:06.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:13:06.043 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:13:06.043 | INFO | Total mem usage: 15.7 GiB | 20260507 20:13:06.043 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:13:06.043 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:13:06.043 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 20:13:06.043 | INFO | High Water Breakdown: | 20260507 20:13:06.043 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:13:06.043 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:13:06.043 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:13:06.043 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:13:06.043 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:13:06.043 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:13:06.043 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:13:06.043 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:13:06.043 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:13:06.043 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:13:06.043 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:13:06.043 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:13:06.043 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:13:06.043 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:13:06.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:13:06.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:13:06.043 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:13:06.043 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:13:06.043 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:13:06.043 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:13:06.043 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:13:06.043 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:13:06.043 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:13:06.043 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:13:06.043 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:13:06.043 | INFO | ------------------------------------------ | 20260507 20:13:06.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:13:06.043 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:13:06.043 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:13:06.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:13:06.043 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.42e+01 / 1.83e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.11e-02 / 4.74e-03 | 20260507 20:13:06.043 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.48e+00, 6.04e-01 / 2.20e-01 | 20260507 20:13:06.043 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.12e-01 / 1.10e-01 | 20260507 20:13:06.043 | INFO | PerfReport: 15.86%, DME -- 4.20e+01 / 4.53e+01, 3.87e+00 / 1.44e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.77e+00 / 1.68e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.50e+00 / 2.81e+00, 4.38e-01 / 2.33e-01 | 20260507 20:13:06.043 | INFO | PerfReport: 3.61%, Metrics -- 1.19e+00 / 1.41e+00, 3.04e-01 / 1.80e-01 | 20260507 20:13:06.043 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.71e-01 / 4.06e-01, 6.13e-02 / 3.12e-02 | 20260507 20:13:06.043 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:13:06.043 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:13:06.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:13:06.043 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 24.15%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 1.83%, LockUpload -- 1.95e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 2.69%, LockDownload -- 2.88e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 16.10%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:13:06.043 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260507 20:13:06.139 | INFO | Starting a round of Baz IO | 20260507 20:13:09.632 | INFO | Populating RTMetrics report | 20260507 20:13:09.632 | INFO | RTMetrics report populated | 20260507 20:13:09.632 | INFO | Opening tmp RTMetrics json file | 20260507 20:13:09.663 | INFO | Serializing RTMetrics to json | 20260507 20:13:09.722 | INFO | Serialization done, swapping tmp file now... | 20260507 20:13:09.728 | INFO | RTMetrics done | 20260507 20:13:09.764 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.625 seconds | 20260507 20:13:09.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.68% / 0.01% / 55.31% | 20260507 20:13:09.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09405% / 1.98682e-05% / 2.78155e-05% | 20260507 20:13:09.764 | INFO | Baz IO overall: | 20260507 20:13:09.764 | INFO | BytesWritten : 1032773364975 | 20260507 20:13:09.764 | INFO | NumEvents : 500693127151 | 20260507 20:13:11.181 | INFO | Analyzing chunk frames = [2277376,2277888) | 20260507 20:13:11.181 | INFO | MemInfo: { | 20260507 20:13:11.181 | INFO | "gpu" : " 17 GB", | 20260507 20:13:11.181 | INFO | "huge" : "", | 20260507 20:13:11.181 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:11.181 | INFO | "malloc" : " 14 GB", | 20260507 20:13:11.181 | INFO | "pinned" : " 29 GB" | 20260507 20:13:11.181 | INFO | } | 20260507 20:13:11.181 | INFO | Perf report for the last 5.14002 seconds: | 20260507 20:13:11.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:11.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:11.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:11.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:11.181 | INFO | ANALYSIS : 69.1%, 2.9, 105.4ms, 0 | 20260507 20:13:11.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:11.182 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 20:13:11.182 | INFO | PRE_HQ : 67.7%, 1.8, 63.0ms, 0 | 20260507 20:13:11.182 | INFO | BAZWRITER : 69.8%, 4.0, 143.5ms, 0 | 20260507 20:13:11.182 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:13:11.182 | INFO | Uploaded 36GiB to GPU in 2.53821s (14.1832GiB/s) | 20260507 20:13:11.182 | INFO | Downloaded 38GiB to GPU in 2.69136s (14.1193GiB/s) | 20260507 20:13:16.321 | INFO | Analyzing chunk frames = [2277888,2278400) | 20260507 20:13:16.321 | INFO | MemInfo: { | 20260507 20:13:16.321 | INFO | "gpu" : " 17 GB", | 20260507 20:13:16.321 | INFO | "huge" : "", | 20260507 20:13:16.321 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:16.321 | INFO | "malloc" : " 14 GB", | 20260507 20:13:16.321 | INFO | "pinned" : " 29 GB" | 20260507 20:13:16.321 | INFO | } | 20260507 20:13:16.321 | INFO | Perf report for the last 5.1398 seconds: | 20260507 20:13:16.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:16.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.321 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 20:13:16.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.321 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:13:16.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:16.322 | INFO | Uploaded 36GiB to GPU in 2.56115s (14.0562GiB/s) | 20260507 20:13:16.322 | INFO | Downloaded 34GiB to GPU in 2.48314s (13.6923GiB/s) | 20260507 20:13:21.460 | INFO | Analyzing chunk frames = [2278400,2278912) | 20260507 20:13:21.460 | INFO | MemInfo: { | 20260507 20:13:21.460 | INFO | "gpu" : " 17 GB", | 20260507 20:13:21.460 | INFO | "huge" : "", | 20260507 20:13:21.460 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:21.460 | INFO | "malloc" : " 14 GB", | 20260507 20:13:21.460 | INFO | "pinned" : " 29 GB" | 20260507 20:13:21.460 | INFO | } | 20260507 20:13:21.460 | INFO | Perf report for the last 5.13921 seconds: | 20260507 20:13:21.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:21.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.460 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:13:21.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.461 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:13:21.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:21.461 | INFO | Uploaded 36GiB to GPU in 2.55611s (14.0839GiB/s) | 20260507 20:13:21.461 | INFO | Downloaded 34GiB to GPU in 2.47526s (13.7359GiB/s) | 20260507 20:13:26.599 | INFO | Analyzing chunk frames = [2278912,2279424) | 20260507 20:13:26.599 | INFO | MemInfo: { | 20260507 20:13:26.599 | INFO | "gpu" : " 17 GB", | 20260507 20:13:26.599 | INFO | "huge" : "", | 20260507 20:13:26.599 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:26.599 | INFO | "malloc" : " 14 GB", | 20260507 20:13:26.599 | INFO | "pinned" : " 29 GB" | 20260507 20:13:26.599 | INFO | } | 20260507 20:13:26.599 | INFO | Perf report for the last 5.13854 seconds: | 20260507 20:13:26.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:26.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 20:13:26.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:13:26.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:26.599 | INFO | Uploaded 36GiB to GPU in 2.46848s (14.5839GiB/s) | 20260507 20:13:26.600 | INFO | Downloaded 34GiB to GPU in 2.39586s (14.1911GiB/s) | 20260507 20:13:31.738 | INFO | Analyzing chunk frames = [2279424,2279936) | 20260507 20:13:31.738 | INFO | MemInfo: { | 20260507 20:13:31.738 | INFO | "gpu" : " 17 GB", | 20260507 20:13:31.738 | INFO | "huge" : "", | 20260507 20:13:31.738 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:31.739 | INFO | "malloc" : " 14 GB", | 20260507 20:13:31.739 | INFO | "pinned" : " 29 GB" | 20260507 20:13:31.739 | INFO | } | 20260507 20:13:31.739 | INFO | Perf report for the last 5.13968 seconds: | 20260507 20:13:31.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:31.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:13:31.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:13:31.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:31.739 | INFO | Uploaded 36GiB to GPU in 2.35386s (15.294GiB/s) | 20260507 20:13:31.739 | INFO | Downloaded 34GiB to GPU in 2.51503s (13.5187GiB/s) | 20260507 20:13:36.878 | INFO | Analyzing chunk frames = [2279936,2280448) | 20260507 20:13:36.878 | INFO | MemInfo: { | 20260507 20:13:36.878 | INFO | "gpu" : " 17 GB", | 20260507 20:13:36.878 | INFO | "huge" : "", | 20260507 20:13:36.878 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:36.878 | INFO | "malloc" : " 14 GB", | 20260507 20:13:36.878 | INFO | "pinned" : " 29 GB" | 20260507 20:13:36.878 | INFO | } | 20260507 20:13:36.878 | INFO | Perf report for the last 5.13976 seconds: | 20260507 20:13:36.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:36.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.878 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:13:36.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.879 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:13:36.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:36.879 | INFO | Uploaded 36GiB to GPU in 2.46209s (14.6217GiB/s) | 20260507 20:13:36.879 | INFO | Downloaded 34GiB to GPU in 2.4296s (13.9941GiB/s) | 20260507 20:13:42.017 | INFO | Analyzing chunk frames = [2280448,2280960) | 20260507 20:13:42.017 | INFO | MemInfo: { | 20260507 20:13:42.017 | INFO | "gpu" : " 17 GB", | 20260507 20:13:42.017 | INFO | "huge" : "", | 20260507 20:13:42.017 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:42.017 | INFO | "malloc" : " 14 GB", | 20260507 20:13:42.017 | INFO | "pinned" : " 29 GB" | 20260507 20:13:42.017 | INFO | } | 20260507 20:13:42.018 | INFO | Perf report for the last 5.13906 seconds: | 20260507 20:13:42.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:42.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:13:42.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 20:13:42.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:42.018 | INFO | Uploaded 36GiB to GPU in 2.52545s (14.2549GiB/s) | 20260507 20:13:42.018 | INFO | Downloaded 34GiB to GPU in 2.45413s (13.8542GiB/s) | 20260507 20:13:47.156 | INFO | Analyzing chunk frames = [2280960,2281472) | 20260507 20:13:47.156 | INFO | MemInfo: { | 20260507 20:13:47.156 | INFO | "gpu" : " 17 GB", | 20260507 20:13:47.156 | INFO | "huge" : "", | 20260507 20:13:47.156 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:47.156 | INFO | "malloc" : " 14 GB", | 20260507 20:13:47.156 | INFO | "pinned" : " 29 GB" | 20260507 20:13:47.156 | INFO | } | 20260507 20:13:47.156 | INFO | Perf report for the last 5.13883 seconds: | 20260507 20:13:47.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:47.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.156 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:13:47.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.157 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 20:13:47.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:47.157 | INFO | Uploaded 36GiB to GPU in 2.51364s (14.3219GiB/s) | 20260507 20:13:47.157 | INFO | Downloaded 34GiB to GPU in 2.43638s (13.9551GiB/s) | 20260507 20:13:47.269 | INFO | Starting a round of Baz IO | 20260507 20:13:50.755 | INFO | Populating RTMetrics report | 20260507 20:13:50.756 | INFO | RTMetrics report populated | 20260507 20:13:50.756 | INFO | Opening tmp RTMetrics json file | 20260507 20:13:50.760 | INFO | Serializing RTMetrics to json | 20260507 20:13:50.761 | INFO | Serialization done, swapping tmp file now... | 20260507 20:13:50.762 | INFO | RTMetrics done | 20260507 20:13:50.866 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.597 seconds | 20260507 20:13:50.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.67% / 0.01% / 55.32% | 20260507 20:13:50.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.08407% / 1.98682e-05% / 5.16574e-05% | 20260507 20:13:50.866 | INFO | Baz IO overall: | 20260507 20:13:50.866 | INFO | BytesWritten : 1034684854572 | 20260507 20:13:50.866 | INFO | NumEvents : 501618570985 | 20260507 20:13:52.295 | INFO | Analyzing chunk frames = [2281472,2281984) | 20260507 20:13:52.295 | INFO | MemInfo: { | 20260507 20:13:52.295 | INFO | "gpu" : " 17 GB", | 20260507 20:13:52.296 | INFO | "huge" : "", | 20260507 20:13:52.296 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:52.296 | INFO | "malloc" : " 14 GB", | 20260507 20:13:52.296 | INFO | "pinned" : " 29 GB" | 20260507 20:13:52.296 | INFO | } | 20260507 20:13:52.296 | INFO | Perf report for the last 5.1393 seconds: | 20260507 20:13:52.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:52.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:52.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:52.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:52.296 | INFO | ANALYSIS : 69.6%, 2.9, 104.8ms, 0 | 20260507 20:13:52.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:52.296 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 20:13:52.296 | INFO | PRE_HQ : 68.6%, 1.8, 65.5ms, 0 | 20260507 20:13:52.296 | INFO | BAZWRITER : 69.5%, 4.1, 148.0ms, 0 | 20260507 20:13:52.296 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:13:52.296 | INFO | Uploaded 36GiB to GPU in 2.57912s (13.9583GiB/s) | 20260507 20:13:52.296 | INFO | Downloaded 38GiB to GPU in 2.70066s (14.0706GiB/s) | 20260507 20:13:57.436 | INFO | Analyzing chunk frames = [2281984,2282496) | 20260507 20:13:57.436 | INFO | MemInfo: { | 20260507 20:13:57.436 | INFO | "gpu" : " 17 GB", | 20260507 20:13:57.436 | INFO | "huge" : "", | 20260507 20:13:57.436 | INFO | "huge_shared" : " 29 GB", | 20260507 20:13:57.436 | INFO | "malloc" : " 14 GB", | 20260507 20:13:57.436 | INFO | "pinned" : " 29 GB" | 20260507 20:13:57.436 | INFO | } | 20260507 20:13:57.436 | INFO | Perf report for the last 5.14058 seconds: | 20260507 20:13:57.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:13:57.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.436 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 20:13:57.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.436 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:13:57.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:13:57.437 | INFO | Uploaded 36GiB to GPU in 2.58754s (13.9128GiB/s) | 20260507 20:13:57.437 | INFO | Downloaded 34GiB to GPU in 2.46389s (13.7993GiB/s) | 20260507 20:14:02.575 | INFO | Analyzing chunk frames = [2282496,2283008) | 20260507 20:14:02.575 | INFO | MemInfo: { | 20260507 20:14:02.575 | INFO | "gpu" : " 17 GB", | 20260507 20:14:02.575 | INFO | "huge" : "", | 20260507 20:14:02.575 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:02.575 | INFO | "malloc" : " 14 GB", | 20260507 20:14:02.575 | INFO | "pinned" : " 29 GB" | 20260507 20:14:02.575 | INFO | } | 20260507 20:14:02.575 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:14:02.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:02.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.575 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 20:14:02.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.576 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:14:02.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:02.576 | INFO | Uploaded 36GiB to GPU in 2.60253s (13.8327GiB/s) | 20260507 20:14:02.576 | INFO | Downloaded 34GiB to GPU in 2.46724s (13.7806GiB/s) | 20260507 20:14:07.714 | INFO | Analyzing chunk frames = [2283008,2283520) | 20260507 20:14:07.714 | INFO | MemInfo: { | 20260507 20:14:07.714 | INFO | "gpu" : " 17 GB", | 20260507 20:14:07.714 | INFO | "huge" : "", | 20260507 20:14:07.714 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:07.714 | INFO | "malloc" : " 14 GB", | 20260507 20:14:07.714 | INFO | "pinned" : " 29 GB" | 20260507 20:14:07.714 | INFO | } | 20260507 20:14:07.714 | INFO | Perf report for the last 5.13836 seconds: | 20260507 20:14:07.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:07.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 20:14:07.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:14:07.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:07.714 | INFO | Uploaded 36GiB to GPU in 2.49112s (14.4513GiB/s) | 20260507 20:14:07.714 | INFO | Downloaded 34GiB to GPU in 2.3926s (14.2105GiB/s) | 20260507 20:14:12.853 | INFO | Analyzing chunk frames = [2283520,2284032) | 20260507 20:14:12.853 | INFO | MemInfo: { | 20260507 20:14:12.853 | INFO | "gpu" : " 17 GB", | 20260507 20:14:12.853 | INFO | "huge" : "", | 20260507 20:14:12.853 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:12.853 | INFO | "malloc" : " 14 GB", | 20260507 20:14:12.853 | INFO | "pinned" : " 29 GB" | 20260507 20:14:12.853 | INFO | } | 20260507 20:14:12.853 | INFO | Perf report for the last 5.13919 seconds: | 20260507 20:14:12.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:12.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:14:12.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:14:12.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:12.853 | INFO | Uploaded 36GiB to GPU in 2.41713s (14.8937GiB/s) | 20260507 20:14:12.853 | INFO | Downloaded 34GiB to GPU in 2.50745s (13.5596GiB/s) | 20260507 20:14:17.993 | INFO | Analyzing chunk frames = [2284032,2284544) | 20260507 20:14:17.993 | INFO | MemInfo: { | 20260507 20:14:17.993 | INFO | "gpu" : " 17 GB", | 20260507 20:14:17.993 | INFO | "huge" : "", | 20260507 20:14:17.993 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:17.993 | INFO | "malloc" : " 14 GB", | 20260507 20:14:17.993 | INFO | "pinned" : " 29 GB" | 20260507 20:14:17.993 | INFO | } | 20260507 20:14:17.993 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:14:17.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:17.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 20:14:17.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:14:17.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:17.993 | INFO | Uploaded 36GiB to GPU in 2.40762s (14.9525GiB/s) | 20260507 20:14:17.993 | INFO | Downloaded 34GiB to GPU in 2.39728s (14.1828GiB/s) | 20260507 20:14:23.132 | INFO | Analyzing chunk frames = [2284544,2285056) | 20260507 20:14:23.132 | INFO | MemInfo: { | 20260507 20:14:23.132 | INFO | "gpu" : " 17 GB", | 20260507 20:14:23.133 | INFO | "huge" : "", | 20260507 20:14:23.133 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:23.133 | INFO | "malloc" : " 14 GB", | 20260507 20:14:23.133 | INFO | "pinned" : " 29 GB" | 20260507 20:14:23.133 | INFO | } | 20260507 20:14:23.133 | INFO | Perf report for the last 5.13974 seconds: | 20260507 20:14:23.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:23.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:14:23.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:14:23.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:23.133 | INFO | Uploaded 36GiB to GPU in 2.52372s (14.2647GiB/s) | 20260507 20:14:23.133 | INFO | Downloaded 34GiB to GPU in 2.44832s (13.887GiB/s) | 20260507 20:14:28.273 | INFO | Analyzing chunk frames = [2285056,2285568) | 20260507 20:14:28.273 | INFO | MemInfo: { | 20260507 20:14:28.273 | INFO | "gpu" : " 17 GB", | 20260507 20:14:28.273 | INFO | "huge" : "", | 20260507 20:14:28.273 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:28.273 | INFO | "malloc" : " 14 GB", | 20260507 20:14:28.273 | INFO | "pinned" : " 29 GB" | 20260507 20:14:28.273 | INFO | } | 20260507 20:14:28.273 | INFO | Perf report for the last 5.14033 seconds: | 20260507 20:14:28.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:28.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.273 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:14:28.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.273 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:14:28.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:28.274 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:14:28.274 | INFO | Total mem usage: 16.4 GiB | 20260507 20:14:28.274 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:14:28.274 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.3%) | 20260507 20:14:28.274 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:14:28.274 | INFO | High Water Breakdown: | 20260507 20:14:28.274 | INFO | 60.3% : 9.22 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:14:28.274 | INFO | 39.6% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:14:28.274 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:14:28.274 | INFO | ------------------------------------------ | 20260507 20:14:28.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:14:28.274 | INFO | Uploaded 36GiB to GPU in 2.48405s (14.4924GiB/s) | 20260507 20:14:28.274 | INFO | Downloaded 34GiB to GPU in 2.42407s (14.026GiB/s) | 20260507 20:14:28.274 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:14:28.274 | INFO | Total mem usage: 26.5 GiB | 20260507 20:14:28.274 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:14:28.274 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 20:14:28.274 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:14:28.274 | INFO | High Water Breakdown: | 20260507 20:14:28.274 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:14:28.274 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:14:28.274 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:14:28.274 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:14:28.274 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:14:28.274 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:14:28.274 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:14:28.274 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:14:28.274 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:14:28.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:14:28.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:14:28.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:14:28.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:14:28.274 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:14:28.274 | INFO | ------------------------------------------ | 20260507 20:14:28.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:14:28.274 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:14:28.274 | INFO | Total mem usage: 27.8 GiB | 20260507 20:14:28.274 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:14:28.274 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:14:28.274 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:14:28.274 | INFO | High Water Breakdown: | 20260507 20:14:28.274 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:14:28.274 | INFO | ------------------------------------------ | 20260507 20:14:28.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:14:28.274 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:14:28.274 | INFO | Total mem usage: 15.7 GiB | 20260507 20:14:28.274 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:14:28.274 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:14:28.274 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:14:28.274 | INFO | High Water Breakdown: | 20260507 20:14:28.274 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:14:28.274 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:14:28.274 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:14:28.274 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:14:28.274 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:14:28.274 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:14:28.274 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:14:28.274 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:14:28.274 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:14:28.274 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:14:28.274 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:14:28.274 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:14:28.274 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:14:28.274 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:14:28.274 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:14:28.274 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:14:28.274 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:14:28.274 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:14:28.274 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:14:28.274 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:14:28.274 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:14:28.274 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:14:28.274 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:14:28.274 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:14:28.274 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:14:28.274 | INFO | ------------------------------------------ | 20260507 20:14:28.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:14:28.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:14:28.275 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:14:28.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:14:28.275 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.56e+01, 1.41e+01 / 1.52e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.16e-02 / 7.29e-03 | 20260507 20:14:28.275 | INFO | PerfReport: 12.11%, Baseline -- 4.02e+00 / 4.47e+00, 6.00e-01 / 2.57e-01 | 20260507 20:14:28.275 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.51e+00, 2.10e-01 / 1.15e-01 | 20260507 20:14:28.275 | INFO | PerfReport: 15.74%, DME -- 4.18e+01 / 4.50e+01, 3.81e+00 / 1.11e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.86e+00 / 1.92e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.49e+00 / 2.80e+00, 4.39e-01 / 2.57e-01 | 20260507 20:14:28.275 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.38e+00, 3.05e-01 / 1.71e-01 | 20260507 20:14:28.275 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.73e-01 / 4.05e-01, 6.13e-02 / 3.21e-02 | 20260507 20:14:28.275 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:14:28.275 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:14:28.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:14:28.275 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 24.21%, Download -- 2.58e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 31.08%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 2.07%, LockUpload -- 2.21e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 2.25%, LockDownload -- 2.40e+00 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:14:28.275 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260507 20:14:28.370 | INFO | Starting a round of Baz IO | 20260507 20:14:31.854 | INFO | Populating RTMetrics report | 20260507 20:14:31.854 | INFO | RTMetrics report populated | 20260507 20:14:31.854 | INFO | Opening tmp RTMetrics json file | 20260507 20:14:31.873 | INFO | Serializing RTMetrics to json | 20260507 20:14:31.881 | INFO | Serialization done, swapping tmp file now... | 20260507 20:14:31.882 | INFO | RTMetrics done | 20260507 20:14:31.940 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.57 seconds | 20260507 20:14:31.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.66% / 0.01% / 55.33% | 20260507 20:14:31.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.08303% / 7.94729e-06% / 4.37101e-05% | 20260507 20:14:31.940 | INFO | Baz IO overall: | 20260507 20:14:31.940 | INFO | BytesWritten : 1036595899298 | 20260507 20:14:31.940 | INFO | NumEvents : 502544077479 | 20260507 20:14:33.410 | INFO | Analyzing chunk frames = [2285568,2286080) | 20260507 20:14:33.410 | INFO | MemInfo: { | 20260507 20:14:33.410 | INFO | "gpu" : " 17 GB", | 20260507 20:14:33.410 | INFO | "huge" : "", | 20260507 20:14:33.410 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:33.410 | INFO | "malloc" : " 14 GB", | 20260507 20:14:33.410 | INFO | "pinned" : " 29 GB" | 20260507 20:14:33.410 | INFO | } | 20260507 20:14:33.411 | INFO | Perf report for the last 5.13744 seconds: | 20260507 20:14:33.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:33.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:33.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:33.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:33.411 | INFO | ANALYSIS : 68.7%, 3.0, 106.3ms, 0 | 20260507 20:14:33.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:33.411 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 20:14:33.411 | INFO | PRE_HQ : 65.9%, 1.7, 58.3ms, 0 | 20260507 20:14:33.411 | INFO | BAZWRITER : 69.1%, 4.0, 141.2ms, 0 | 20260507 20:14:33.411 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 20:14:33.411 | INFO | Uploaded 36GiB to GPU in 2.49171s (14.4479GiB/s) | 20260507 20:14:33.411 | INFO | Downloaded 38GiB to GPU in 2.68597s (14.1476GiB/s) | 20260507 20:14:38.549 | INFO | Analyzing chunk frames = [2286080,2286592) | 20260507 20:14:38.549 | INFO | MemInfo: { | 20260507 20:14:38.549 | INFO | "gpu" : " 17 GB", | 20260507 20:14:38.549 | INFO | "huge" : "", | 20260507 20:14:38.549 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:38.549 | INFO | "malloc" : " 14 GB", | 20260507 20:14:38.549 | INFO | "pinned" : " 29 GB" | 20260507 20:14:38.549 | INFO | } | 20260507 20:14:38.549 | INFO | Perf report for the last 5.13888 seconds: | 20260507 20:14:38.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:38.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.549 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:14:38.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.550 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:14:38.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:38.550 | INFO | Uploaded 36GiB to GPU in 2.51694s (14.3031GiB/s) | 20260507 20:14:38.550 | INFO | Downloaded 34GiB to GPU in 2.50272s (13.5852GiB/s) | 20260507 20:14:43.689 | INFO | Analyzing chunk frames = [2286592,2287104) | 20260507 20:14:43.689 | INFO | MemInfo: { | 20260507 20:14:43.689 | INFO | "gpu" : " 17 GB", | 20260507 20:14:43.689 | INFO | "huge" : "", | 20260507 20:14:43.689 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:43.689 | INFO | "malloc" : " 14 GB", | 20260507 20:14:43.689 | INFO | "pinned" : " 29 GB" | 20260507 20:14:43.689 | INFO | } | 20260507 20:14:43.689 | INFO | Perf report for the last 5.1396 seconds: | 20260507 20:14:43.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:43.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 20:14:43.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:14:43.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:43.689 | INFO | Uploaded 36GiB to GPU in 2.51585s (14.3093GiB/s) | 20260507 20:14:43.689 | INFO | Downloaded 34GiB to GPU in 2.47948s (13.7125GiB/s) | 20260507 20:14:48.829 | INFO | Analyzing chunk frames = [2287104,2287616) | 20260507 20:14:48.829 | INFO | MemInfo: { | 20260507 20:14:48.829 | INFO | "gpu" : " 17 GB", | 20260507 20:14:48.829 | INFO | "huge" : "", | 20260507 20:14:48.829 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:48.829 | INFO | "malloc" : " 14 GB", | 20260507 20:14:48.829 | INFO | "pinned" : " 29 GB" | 20260507 20:14:48.829 | INFO | } | 20260507 20:14:48.829 | INFO | Perf report for the last 5.14026 seconds: | 20260507 20:14:48.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:48.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.829 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 20:14:48.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.829 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:14:48.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:48.830 | INFO | Uploaded 36GiB to GPU in 2.44689s (14.7125GiB/s) | 20260507 20:14:48.830 | INFO | Downloaded 34GiB to GPU in 2.41265s (14.0924GiB/s) | 20260507 20:14:53.968 | INFO | Analyzing chunk frames = [2287616,2288128) | 20260507 20:14:53.968 | INFO | MemInfo: { | 20260507 20:14:53.968 | INFO | "gpu" : " 17 GB", | 20260507 20:14:53.968 | INFO | "huge" : "", | 20260507 20:14:53.968 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:53.969 | INFO | "malloc" : " 14 GB", | 20260507 20:14:53.969 | INFO | "pinned" : " 29 GB" | 20260507 20:14:53.969 | INFO | } | 20260507 20:14:53.969 | INFO | Perf report for the last 5.1393 seconds: | 20260507 20:14:53.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:53.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 20:14:53.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:14:53.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:53.969 | INFO | Uploaded 36GiB to GPU in 2.32633s (15.475GiB/s) | 20260507 20:14:53.969 | INFO | Downloaded 34GiB to GPU in 2.51935s (13.4956GiB/s) | 20260507 20:14:59.107 | INFO | Analyzing chunk frames = [2288128,2288640) | 20260507 20:14:59.107 | INFO | MemInfo: { | 20260507 20:14:59.107 | INFO | "gpu" : " 17 GB", | 20260507 20:14:59.108 | INFO | "huge" : "", | 20260507 20:14:59.108 | INFO | "huge_shared" : " 29 GB", | 20260507 20:14:59.108 | INFO | "malloc" : " 14 GB", | 20260507 20:14:59.108 | INFO | "pinned" : " 29 GB" | 20260507 20:14:59.108 | INFO | } | 20260507 20:14:59.108 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:14:59.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:14:59.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.108 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 20:14:59.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.108 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 20:14:59.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:14:59.109 | INFO | Uploaded 36GiB to GPU in 2.51051s (14.3397GiB/s) | 20260507 20:14:59.109 | INFO | Downloaded 34GiB to GPU in 2.42933s (13.9956GiB/s) | 20260507 20:15:04.248 | INFO | Analyzing chunk frames = [2288640,2289152) | 20260507 20:15:04.248 | INFO | MemInfo: { | 20260507 20:15:04.248 | INFO | "gpu" : " 17 GB", | 20260507 20:15:04.248 | INFO | "huge" : "", | 20260507 20:15:04.248 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:04.248 | INFO | "malloc" : " 14 GB", | 20260507 20:15:04.248 | INFO | "pinned" : " 29 GB" | 20260507 20:15:04.248 | INFO | } | 20260507 20:15:04.248 | INFO | Perf report for the last 5.14002 seconds: | 20260507 20:15:04.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:04.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:15:04.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260507 20:15:04.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:04.248 | INFO | Uploaded 36GiB to GPU in 2.53821s (14.1832GiB/s) | 20260507 20:15:04.248 | INFO | Downloaded 34GiB to GPU in 2.46167s (13.8118GiB/s) | 20260507 20:15:09.387 | INFO | Analyzing chunk frames = [2289152,2289664) | 20260507 20:15:09.387 | INFO | MemInfo: { | 20260507 20:15:09.387 | INFO | "gpu" : " 17 GB", | 20260507 20:15:09.387 | INFO | "huge" : "", | 20260507 20:15:09.387 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:09.387 | INFO | "malloc" : " 14 GB", | 20260507 20:15:09.387 | INFO | "pinned" : " 29 GB" | 20260507 20:15:09.388 | INFO | } | 20260507 20:15:09.388 | INFO | Perf report for the last 5.13972 seconds: | 20260507 20:15:09.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:09.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:15:09.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:15:09.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:09.388 | INFO | Uploaded 36GiB to GPU in 2.49213s (14.4455GiB/s) | 20260507 20:15:09.388 | INFO | Downloaded 34GiB to GPU in 2.4323s (13.9786GiB/s) | 20260507 20:15:09.484 | INFO | Starting a round of Baz IO | 20260507 20:15:12.963 | INFO | Populating RTMetrics report | 20260507 20:15:12.964 | INFO | RTMetrics report populated | 20260507 20:15:12.964 | INFO | Opening tmp RTMetrics json file | 20260507 20:15:12.988 | INFO | Serializing RTMetrics to json | 20260507 20:15:12.990 | INFO | Serialization done, swapping tmp file now... | 20260507 20:15:12.991 | INFO | RTMetrics done | 20260507 20:15:13.069 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.584 seconds | 20260507 20:15:13.069 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.65% / 0.01% / 55.34% | 20260507 20:15:13.069 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.08315% / 1.19209e-05% / 3.57628e-05% | 20260507 20:15:13.069 | INFO | Baz IO overall: | 20260507 20:15:13.069 | INFO | BytesWritten : 1038508687789 | 20260507 20:15:13.069 | INFO | NumEvents : 503470561530 | 20260507 20:15:14.527 | INFO | Analyzing chunk frames = [2289664,2290176) | 20260507 20:15:14.527 | INFO | MemInfo: { | 20260507 20:15:14.527 | INFO | "gpu" : " 17 GB", | 20260507 20:15:14.527 | INFO | "huge" : "", | 20260507 20:15:14.527 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:14.527 | INFO | "malloc" : " 14 GB", | 20260507 20:15:14.527 | INFO | "pinned" : " 29 GB" | 20260507 20:15:14.527 | INFO | } | 20260507 20:15:14.527 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:15:14.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:14.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:14.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:14.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:14.527 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 20:15:14.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:14.527 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 20:15:14.527 | INFO | PRE_HQ : 66.0%, 1.7, 58.1ms, 0 | 20260507 20:15:14.527 | INFO | BAZWRITER : 69.3%, 4.0, 143.9ms, 0 | 20260507 20:15:14.527 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 20:15:14.527 | INFO | Uploaded 36GiB to GPU in 2.57817s (13.9634GiB/s) | 20260507 20:15:14.527 | INFO | Downloaded 38GiB to GPU in 2.69489s (14.1007GiB/s) | 20260507 20:15:19.664 | INFO | Analyzing chunk frames = [2290176,2290688) | 20260507 20:15:19.664 | INFO | MemInfo: { | 20260507 20:15:19.664 | INFO | "gpu" : " 17 GB", | 20260507 20:15:19.664 | INFO | "huge" : "", | 20260507 20:15:19.664 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:19.664 | INFO | "malloc" : " 14 GB", | 20260507 20:15:19.664 | INFO | "pinned" : " 29 GB" | 20260507 20:15:19.665 | INFO | } | 20260507 20:15:19.665 | INFO | Perf report for the last 5.13778 seconds: | 20260507 20:15:19.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:19.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:15:19.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:15:19.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:19.665 | INFO | Uploaded 36GiB to GPU in 2.57875s (13.9602GiB/s) | 20260507 20:15:19.665 | INFO | Downloaded 34GiB to GPU in 2.49182s (13.6446GiB/s) | 20260507 20:15:24.805 | INFO | Analyzing chunk frames = [2290688,2291200) | 20260507 20:15:24.805 | INFO | MemInfo: { | 20260507 20:15:24.805 | INFO | "gpu" : " 17 GB", | 20260507 20:15:24.805 | INFO | "huge" : "", | 20260507 20:15:24.805 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:24.805 | INFO | "malloc" : " 14 GB", | 20260507 20:15:24.805 | INFO | "pinned" : " 29 GB" | 20260507 20:15:24.805 | INFO | } | 20260507 20:15:24.805 | INFO | Perf report for the last 5.14046 seconds: | 20260507 20:15:24.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:24.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.805 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 20:15:24.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.805 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:15:24.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:24.806 | INFO | Uploaded 36GiB to GPU in 2.60079s (13.842GiB/s) | 20260507 20:15:24.806 | INFO | Downloaded 34GiB to GPU in 2.46104s (13.8153GiB/s) | 20260507 20:15:29.945 | INFO | Analyzing chunk frames = [2291200,2291712) | 20260507 20:15:29.945 | INFO | MemInfo: { | 20260507 20:15:29.945 | INFO | "gpu" : " 17 GB", | 20260507 20:15:29.945 | INFO | "huge" : "", | 20260507 20:15:29.945 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:29.945 | INFO | "malloc" : " 14 GB", | 20260507 20:15:29.945 | INFO | "pinned" : " 29 GB" | 20260507 20:15:29.945 | INFO | } | 20260507 20:15:29.945 | INFO | Perf report for the last 5.13988 seconds: | 20260507 20:15:29.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:29.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.945 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 20:15:29.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.945 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:15:29.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:29.946 | INFO | Uploaded 36GiB to GPU in 2.52716s (14.2453GiB/s) | 20260507 20:15:29.946 | INFO | Downloaded 34GiB to GPU in 2.39967s (14.1686GiB/s) | 20260507 20:15:35.084 | INFO | Analyzing chunk frames = [2291712,2292224) | 20260507 20:15:35.084 | INFO | MemInfo: { | 20260507 20:15:35.084 | INFO | "gpu" : " 17 GB", | 20260507 20:15:35.085 | INFO | "huge" : "", | 20260507 20:15:35.085 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:35.085 | INFO | "malloc" : " 14 GB", | 20260507 20:15:35.085 | INFO | "pinned" : " 29 GB" | 20260507 20:15:35.085 | INFO | } | 20260507 20:15:35.085 | INFO | Perf report for the last 5.13976 seconds: | 20260507 20:15:35.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:35.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 20:15:35.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 20:15:35.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:35.085 | INFO | Uploaded 36GiB to GPU in 2.39415s (15.0367GiB/s) | 20260507 20:15:35.085 | INFO | Downloaded 34GiB to GPU in 2.48491s (13.6826GiB/s) | 20260507 20:15:40.222 | INFO | Analyzing chunk frames = [2292224,2292736) | 20260507 20:15:40.223 | INFO | MemInfo: { | 20260507 20:15:40.223 | INFO | "gpu" : " 17 GB", | 20260507 20:15:40.223 | INFO | "huge" : "", | 20260507 20:15:40.223 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:40.223 | INFO | "malloc" : " 14 GB", | 20260507 20:15:40.223 | INFO | "pinned" : " 29 GB" | 20260507 20:15:40.223 | INFO | } | 20260507 20:15:40.223 | INFO | Perf report for the last 5.13803 seconds: | 20260507 20:15:40.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:40.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 20:15:40.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:15:40.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:40.223 | INFO | Uploaded 36GiB to GPU in 2.51522s (14.3129GiB/s) | 20260507 20:15:40.223 | INFO | Downloaded 34GiB to GPU in 2.42554s (14.0175GiB/s) | 20260507 20:15:45.361 | INFO | Analyzing chunk frames = [2292736,2293248) | 20260507 20:15:45.361 | INFO | MemInfo: { | 20260507 20:15:45.361 | INFO | "gpu" : " 17 GB", | 20260507 20:15:45.361 | INFO | "huge" : "", | 20260507 20:15:45.361 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:45.361 | INFO | "malloc" : " 14 GB", | 20260507 20:15:45.361 | INFO | "pinned" : " 29 GB" | 20260507 20:15:45.361 | INFO | } | 20260507 20:15:45.362 | INFO | Perf report for the last 5.13876 seconds: | 20260507 20:15:45.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:45.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:15:45.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:15:45.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:45.362 | INFO | Uploaded 36GiB to GPU in 2.52918s (14.2338GiB/s) | 20260507 20:15:45.362 | INFO | Downloaded 34GiB to GPU in 2.44407s (13.9112GiB/s) | 20260507 20:15:50.501 | INFO | Analyzing chunk frames = [2293248,2293760) | 20260507 20:15:50.501 | INFO | MemInfo: { | 20260507 20:15:50.501 | INFO | "gpu" : " 17 GB", | 20260507 20:15:50.501 | INFO | "huge" : "", | 20260507 20:15:50.501 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:50.501 | INFO | "malloc" : " 14 GB", | 20260507 20:15:50.501 | INFO | "pinned" : " 29 GB" | 20260507 20:15:50.501 | INFO | } | 20260507 20:15:50.501 | INFO | Perf report for the last 5.13959 seconds: | 20260507 20:15:50.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:50.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:15:50.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:15:50.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:50.501 | INFO | Uploaded 36GiB to GPU in 2.48877s (14.465GiB/s) | 20260507 20:15:50.501 | INFO | Downloaded 34GiB to GPU in 2.42578s (14.0161GiB/s) | 20260507 20:15:50.502 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:15:50.502 | INFO | Total mem usage: 16.4 GiB | 20260507 20:15:50.502 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:15:50.502 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8.1%) | 20260507 20:15:50.502 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:15:50.502 | INFO | High Water Breakdown: | 20260507 20:15:50.502 | INFO | 60.2% : 9.13 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:15:50.502 | INFO | 39.7% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:15:50.502 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:15:50.502 | INFO | ------------------------------------------ | 20260507 20:15:50.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:15:50.502 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:15:50.502 | INFO | Total mem usage: 26.5 GiB | 20260507 20:15:50.502 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:15:50.502 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:15:50.502 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:15:50.502 | INFO | High Water Breakdown: | 20260507 20:15:50.502 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:15:50.502 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:15:50.502 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:15:50.502 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:15:50.502 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:15:50.502 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:15:50.502 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:15:50.502 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:15:50.502 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:15:50.502 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:15:50.502 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:15:50.502 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:15:50.502 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:15:50.502 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:15:50.502 | INFO | ------------------------------------------ | 20260507 20:15:50.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:15:50.502 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:15:50.502 | INFO | Total mem usage: 27.8 GiB | 20260507 20:15:50.502 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:15:50.502 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:15:50.502 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:15:50.502 | INFO | High Water Breakdown: | 20260507 20:15:50.502 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:15:50.502 | INFO | ------------------------------------------ | 20260507 20:15:50.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:15:50.502 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:15:50.502 | INFO | Total mem usage: 15.7 GiB | 20260507 20:15:50.502 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:15:50.502 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:15:50.502 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:15:50.502 | INFO | High Water Breakdown: | 20260507 20:15:50.502 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:15:50.502 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:15:50.502 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:15:50.502 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:15:50.502 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:15:50.502 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:15:50.502 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:15:50.502 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:15:50.502 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:15:50.502 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:15:50.502 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:15:50.502 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:15:50.502 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:15:50.502 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:15:50.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:15:50.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:15:50.502 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:15:50.502 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:15:50.502 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:15:50.502 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:15:50.502 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:15:50.502 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:15:50.502 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:15:50.502 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:15:50.502 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:15:50.502 | INFO | ------------------------------------------ | 20260507 20:15:50.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:15:50.502 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:15:50.502 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:15:50.502 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:15:50.502 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.55e+01, 1.41e+01 / 1.66e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.11e-02 / 4.27e-03 | 20260507 20:15:50.502 | INFO | PerfReport: 12.09%, Baseline -- 4.00e+00 / 4.51e+00, 5.98e-01 / 2.42e-01 | 20260507 20:15:50.502 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.51e+00, 2.13e-01 / 1.11e-01 | 20260507 20:15:50.502 | INFO | PerfReport: 15.73%, DME -- 4.16e+01 / 4.48e+01, 3.86e+00 / 1.16e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.61e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.50e+00 / 2.85e+00, 4.43e-01 / 2.05e-01 | 20260507 20:15:50.502 | INFO | PerfReport: 3.61%, Metrics -- 1.19e+00 / 1.42e+00, 3.06e-01 / 1.79e-01 | 20260507 20:15:50.502 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.71e-01 / 4.09e-01, 6.09e-02 / 3.40e-02 | 20260507 20:15:50.502 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:15:50.502 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:15:50.502 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:15:50.502 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 24.39%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 30.96%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 1.98%, LockUpload -- 2.12e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 2.47%, LockDownload -- 2.64e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 15.98%, LockCompute -- 1.71e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:15:50.502 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 20:15:50.598 | INFO | Starting a round of Baz IO | 20260507 20:15:54.081 | INFO | Populating RTMetrics report | 20260507 20:15:54.081 | INFO | RTMetrics report populated | 20260507 20:15:54.081 | INFO | Opening tmp RTMetrics json file | 20260507 20:15:54.104 | INFO | Serializing RTMetrics to json | 20260507 20:15:54.105 | INFO | Serialization done, swapping tmp file now... | 20260507 20:15:54.106 | INFO | RTMetrics done | 20260507 20:15:54.176 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.577 seconds | 20260507 20:15:54.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.64% / 0.01% / 55.35% | 20260507 20:15:54.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.07786% / 1.58946e-05% / 4.37101e-05% | 20260507 20:15:54.176 | INFO | Baz IO overall: | 20260507 20:15:54.176 | INFO | BytesWritten : 1040421343564 | 20260507 20:15:54.176 | INFO | NumEvents : 504397134645 | 20260507 20:15:55.641 | INFO | Analyzing chunk frames = [2293760,2294272) | 20260507 20:15:55.641 | INFO | MemInfo: { | 20260507 20:15:55.641 | INFO | "gpu" : " 17 GB", | 20260507 20:15:55.641 | INFO | "huge" : "", | 20260507 20:15:55.641 | INFO | "huge_shared" : " 29 GB", | 20260507 20:15:55.641 | INFO | "malloc" : " 14 GB", | 20260507 20:15:55.641 | INFO | "pinned" : " 29 GB" | 20260507 20:15:55.641 | INFO | } | 20260507 20:15:55.641 | INFO | Perf report for the last 5.13982 seconds: | 20260507 20:15:55.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:15:55.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:55.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:55.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:55.641 | INFO | ANALYSIS : 68.9%, 3.0, 106.9ms, 0 | 20260507 20:15:55.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:15:55.641 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:15:55.641 | INFO | PRE_HQ : 65.9%, 1.7, 58.2ms, 0 | 20260507 20:15:55.641 | INFO | BAZWRITER : 69.2%, 4.4, 156.9ms, 0 | 20260507 20:15:55.641 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260507 20:15:55.642 | INFO | Uploaded 36GiB to GPU in 2.51409s (14.3193GiB/s) | 20260507 20:15:55.642 | INFO | Downloaded 38GiB to GPU in 2.70201s (14.0636GiB/s) | 20260507 20:16:00.780 | INFO | Analyzing chunk frames = [2294272,2294784) | 20260507 20:16:00.780 | INFO | MemInfo: { | 20260507 20:16:00.780 | INFO | "gpu" : " 17 GB", | 20260507 20:16:00.780 | INFO | "huge" : "", | 20260507 20:16:00.780 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:00.780 | INFO | "malloc" : " 14 GB", | 20260507 20:16:00.780 | INFO | "pinned" : " 29 GB" | 20260507 20:16:00.780 | INFO | } | 20260507 20:16:00.780 | INFO | Perf report for the last 5.13908 seconds: | 20260507 20:16:00.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:00.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:16:00.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:16:00.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:00.780 | INFO | Uploaded 36GiB to GPU in 2.53352s (14.2095GiB/s) | 20260507 20:16:00.780 | INFO | Downloaded 34GiB to GPU in 2.50728s (13.5605GiB/s) | 20260507 20:16:05.919 | INFO | Analyzing chunk frames = [2294784,2295296) | 20260507 20:16:05.919 | INFO | MemInfo: { | 20260507 20:16:05.919 | INFO | "gpu" : " 17 GB", | 20260507 20:16:05.919 | INFO | "huge" : "", | 20260507 20:16:05.919 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:05.919 | INFO | "malloc" : " 14 GB", | 20260507 20:16:05.919 | INFO | "pinned" : " 29 GB" | 20260507 20:16:05.919 | INFO | } | 20260507 20:16:05.919 | INFO | Perf report for the last 5.13919 seconds: | 20260507 20:16:05.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:05.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.919 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:16:05.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.919 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:16:05.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:05.920 | INFO | Uploaded 36GiB to GPU in 2.50461s (14.3735GiB/s) | 20260507 20:16:05.920 | INFO | Downloaded 34GiB to GPU in 2.49086s (13.6499GiB/s) | 20260507 20:16:11.059 | INFO | Analyzing chunk frames = [2295296,2295808) | 20260507 20:16:11.059 | INFO | MemInfo: { | 20260507 20:16:11.059 | INFO | "gpu" : " 17 GB", | 20260507 20:16:11.059 | INFO | "huge" : "", | 20260507 20:16:11.059 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:11.059 | INFO | "malloc" : " 14 GB", | 20260507 20:16:11.059 | INFO | "pinned" : " 29 GB" | 20260507 20:16:11.059 | INFO | } | 20260507 20:16:11.059 | INFO | Perf report for the last 5.1399 seconds: | 20260507 20:16:11.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:11.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.059 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 20:16:11.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.059 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 20:16:11.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:11.060 | INFO | Uploaded 36GiB to GPU in 2.4771s (14.5331GiB/s) | 20260507 20:16:11.060 | INFO | Downloaded 34GiB to GPU in 2.50825s (13.5553GiB/s) | 20260507 20:16:16.199 | INFO | Analyzing chunk frames = [2295808,2296320) | 20260507 20:16:16.199 | INFO | MemInfo: { | 20260507 20:16:16.199 | INFO | "gpu" : " 17 GB", | 20260507 20:16:16.199 | INFO | "huge" : "", | 20260507 20:16:16.199 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:16.199 | INFO | "malloc" : " 14 GB", | 20260507 20:16:16.199 | INFO | "pinned" : " 29 GB" | 20260507 20:16:16.199 | INFO | } | 20260507 20:16:16.199 | INFO | Perf report for the last 5.14005 seconds: | 20260507 20:16:16.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:16.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.199 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 20:16:16.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.199 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:16:16.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:16.200 | INFO | Uploaded 36GiB to GPU in 2.3742s (15.163GiB/s) | 20260507 20:16:16.200 | INFO | Downloaded 34GiB to GPU in 2.50331s (13.582GiB/s) | 20260507 20:16:21.338 | INFO | Analyzing chunk frames = [2296320,2296832) | 20260507 20:16:21.338 | INFO | MemInfo: { | 20260507 20:16:21.338 | INFO | "gpu" : " 17 GB", | 20260507 20:16:21.338 | INFO | "huge" : "", | 20260507 20:16:21.338 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:21.338 | INFO | "malloc" : " 14 GB", | 20260507 20:16:21.338 | INFO | "pinned" : " 29 GB" | 20260507 20:16:21.338 | INFO | } | 20260507 20:16:21.339 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:16:21.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:21.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:16:21.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:16:21.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:21.339 | INFO | Uploaded 36GiB to GPU in 2.49283s (14.4414GiB/s) | 20260507 20:16:21.339 | INFO | Downloaded 34GiB to GPU in 2.4293s (13.9958GiB/s) | 20260507 20:16:26.477 | INFO | Analyzing chunk frames = [2296832,2297344) | 20260507 20:16:26.477 | INFO | MemInfo: { | 20260507 20:16:26.477 | INFO | "gpu" : " 17 GB", | 20260507 20:16:26.477 | INFO | "huge" : "", | 20260507 20:16:26.477 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:26.477 | INFO | "malloc" : " 14 GB", | 20260507 20:16:26.477 | INFO | "pinned" : " 29 GB" | 20260507 20:16:26.477 | INFO | } | 20260507 20:16:26.477 | INFO | Perf report for the last 5.13867 seconds: | 20260507 20:16:26.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:26.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.477 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:16:26.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.477 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:16:26.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:26.478 | INFO | Uploaded 36GiB to GPU in 2.5296s (14.2315GiB/s) | 20260507 20:16:26.478 | INFO | Downloaded 34GiB to GPU in 2.45108s (13.8714GiB/s) | 20260507 20:16:31.616 | INFO | Analyzing chunk frames = [2297344,2297856) | 20260507 20:16:31.616 | INFO | MemInfo: { | 20260507 20:16:31.616 | INFO | "gpu" : " 17 GB", | 20260507 20:16:31.616 | INFO | "huge" : "", | 20260507 20:16:31.616 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:31.616 | INFO | "malloc" : " 14 GB", | 20260507 20:16:31.616 | INFO | "pinned" : " 29 GB" | 20260507 20:16:31.616 | INFO | } | 20260507 20:16:31.616 | INFO | Perf report for the last 5.13915 seconds: | 20260507 20:16:31.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:31.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.616 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:16:31.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.616 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:16:31.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:31.617 | INFO | Uploaded 36GiB to GPU in 2.47722s (14.5324GiB/s) | 20260507 20:16:31.617 | INFO | Downloaded 34GiB to GPU in 2.43414s (13.968GiB/s) | 20260507 20:16:31.713 | INFO | Starting a round of Baz IO | 20260507 20:16:35.189 | INFO | Populating RTMetrics report | 20260507 20:16:35.189 | INFO | RTMetrics report populated | 20260507 20:16:35.189 | INFO | Opening tmp RTMetrics json file | 20260507 20:16:35.206 | INFO | Serializing RTMetrics to json | 20260507 20:16:35.217 | INFO | Serialization done, swapping tmp file now... | 20260507 20:16:35.217 | INFO | RTMetrics done | 20260507 20:16:35.306 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.592 seconds | 20260507 20:16:35.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.63% / 0.01% / 55.36% | 20260507 20:16:35.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.07204% / 1.19209e-05% / 5.96046e-05% | 20260507 20:16:35.306 | INFO | Baz IO overall: | 20260507 20:16:35.306 | INFO | BytesWritten : 1042332815169 | 20260507 20:16:35.306 | INFO | NumEvents : 505322765761 | 20260507 20:16:36.756 | INFO | Analyzing chunk frames = [2297856,2298368) | 20260507 20:16:36.756 | INFO | MemInfo: { | 20260507 20:16:36.756 | INFO | "gpu" : " 17 GB", | 20260507 20:16:36.756 | INFO | "huge" : "", | 20260507 20:16:36.756 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:36.756 | INFO | "malloc" : " 14 GB", | 20260507 20:16:36.756 | INFO | "pinned" : " 29 GB" | 20260507 20:16:36.756 | INFO | } | 20260507 20:16:36.757 | INFO | Perf report for the last 5.14022 seconds: | 20260507 20:16:36.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:36.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:36.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:36.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:36.757 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:16:36.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:36.757 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:16:36.757 | INFO | PRE_HQ : 66.0%, 1.7, 58.1ms, 0 | 20260507 20:16:36.757 | INFO | BAZWRITER : 69.5%, 3.6, 129.8ms, 0 | 20260507 20:16:36.757 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:16:36.757 | INFO | Uploaded 36GiB to GPU in 2.53882s (14.1798GiB/s) | 20260507 20:16:36.757 | INFO | Downloaded 38GiB to GPU in 2.68034s (14.1773GiB/s) | 20260507 20:16:41.896 | INFO | Analyzing chunk frames = [2298368,2298880) | 20260507 20:16:41.896 | INFO | MemInfo: { | 20260507 20:16:41.896 | INFO | "gpu" : " 17 GB", | 20260507 20:16:41.896 | INFO | "huge" : "", | 20260507 20:16:41.896 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:41.896 | INFO | "malloc" : " 14 GB", | 20260507 20:16:41.896 | INFO | "pinned" : " 29 GB" | 20260507 20:16:41.896 | INFO | } | 20260507 20:16:41.896 | INFO | Perf report for the last 5.13924 seconds: | 20260507 20:16:41.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:41.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:16:41.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 20:16:41.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:41.896 | INFO | Uploaded 36GiB to GPU in 2.58007s (13.9531GiB/s) | 20260507 20:16:41.896 | INFO | Downloaded 34GiB to GPU in 2.48175s (13.7GiB/s) | 20260507 20:16:47.035 | INFO | Analyzing chunk frames = [2298880,2299392) | 20260507 20:16:47.035 | INFO | MemInfo: { | 20260507 20:16:47.035 | INFO | "gpu" : " 17 GB", | 20260507 20:16:47.035 | INFO | "huge" : "", | 20260507 20:16:47.035 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:47.035 | INFO | "malloc" : " 14 GB", | 20260507 20:16:47.035 | INFO | "pinned" : " 29 GB" | 20260507 20:16:47.035 | INFO | } | 20260507 20:16:47.035 | INFO | Perf report for the last 5.13888 seconds: | 20260507 20:16:47.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:47.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260507 20:16:47.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:16:47.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:47.035 | INFO | Uploaded 36GiB to GPU in 2.59991s (13.8466GiB/s) | 20260507 20:16:47.035 | INFO | Downloaded 34GiB to GPU in 2.46479s (13.7943GiB/s) | 20260507 20:16:52.174 | INFO | Analyzing chunk frames = [2299392,2299904) | 20260507 20:16:52.174 | INFO | MemInfo: { | 20260507 20:16:52.174 | INFO | "gpu" : " 17 GB", | 20260507 20:16:52.174 | INFO | "huge" : "", | 20260507 20:16:52.174 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:52.174 | INFO | "malloc" : " 14 GB", | 20260507 20:16:52.174 | INFO | "pinned" : " 29 GB" | 20260507 20:16:52.174 | INFO | } | 20260507 20:16:52.174 | INFO | Perf report for the last 5.13909 seconds: | 20260507 20:16:52.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:52.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 20:16:52.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:16:52.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:52.174 | INFO | Uploaded 36GiB to GPU in 2.49915s (14.4049GiB/s) | 20260507 20:16:52.174 | INFO | Downloaded 34GiB to GPU in 2.3884s (14.2355GiB/s) | 20260507 20:16:57.313 | INFO | Analyzing chunk frames = [2299904,2300416) | 20260507 20:16:57.313 | INFO | MemInfo: { | 20260507 20:16:57.313 | INFO | "gpu" : " 17 GB", | 20260507 20:16:57.313 | INFO | "huge" : "", | 20260507 20:16:57.313 | INFO | "huge_shared" : " 29 GB", | 20260507 20:16:57.313 | INFO | "malloc" : " 14 GB", | 20260507 20:16:57.313 | INFO | "pinned" : " 29 GB" | 20260507 20:16:57.313 | INFO | } | 20260507 20:16:57.313 | INFO | Perf report for the last 5.13942 seconds: | 20260507 20:16:57.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:16:57.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.313 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 20:16:57.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.313 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:16:57.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:16:57.314 | INFO | Uploaded 36GiB to GPU in 2.43548s (14.7815GiB/s) | 20260507 20:16:57.314 | INFO | Downloaded 34GiB to GPU in 2.49294s (13.6385GiB/s) | 20260507 20:17:02.452 | INFO | Analyzing chunk frames = [2300416,2300928) | 20260507 20:17:02.452 | INFO | MemInfo: { | 20260507 20:17:02.452 | INFO | "gpu" : " 17 GB", | 20260507 20:17:02.452 | INFO | "huge" : "", | 20260507 20:17:02.452 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:02.452 | INFO | "malloc" : " 14 GB", | 20260507 20:17:02.452 | INFO | "pinned" : " 29 GB" | 20260507 20:17:02.452 | INFO | } | 20260507 20:17:02.453 | INFO | Perf report for the last 5.13926 seconds: | 20260507 20:17:02.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:02.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:17:02.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:17:02.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:02.453 | INFO | Uploaded 36GiB to GPU in 2.48082s (14.5113GiB/s) | 20260507 20:17:02.453 | INFO | Downloaded 34GiB to GPU in 2.42127s (14.0422GiB/s) | 20260507 20:17:07.592 | INFO | Analyzing chunk frames = [2300928,2301440) | 20260507 20:17:07.592 | INFO | MemInfo: { | 20260507 20:17:07.592 | INFO | "gpu" : " 17 GB", | 20260507 20:17:07.592 | INFO | "huge" : "", | 20260507 20:17:07.592 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:07.592 | INFO | "malloc" : " 14 GB", | 20260507 20:17:07.592 | INFO | "pinned" : " 29 GB" | 20260507 20:17:07.592 | INFO | } | 20260507 20:17:07.592 | INFO | Perf report for the last 5.13961 seconds: | 20260507 20:17:07.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:07.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.592 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:17:07.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.592 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:17:07.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:07.593 | INFO | Uploaded 36GiB to GPU in 2.5307s (14.2253GiB/s) | 20260507 20:17:07.593 | INFO | Downloaded 34GiB to GPU in 2.45522s (13.848GiB/s) | 20260507 20:17:12.732 | INFO | Analyzing chunk frames = [2301440,2301952) | 20260507 20:17:12.732 | INFO | MemInfo: { | 20260507 20:17:12.732 | INFO | "gpu" : " 17 GB", | 20260507 20:17:12.732 | INFO | "huge" : "", | 20260507 20:17:12.732 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:12.732 | INFO | "malloc" : " 14 GB", | 20260507 20:17:12.732 | INFO | "pinned" : " 29 GB" | 20260507 20:17:12.732 | INFO | } | 20260507 20:17:12.732 | INFO | Perf report for the last 5.13976 seconds: | 20260507 20:17:12.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:12.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.732 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:17:12.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.732 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:17:12.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:12.733 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:17:12.733 | INFO | Total mem usage: 16.4 GiB | 20260507 20:17:12.733 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:17:12.733 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -3.61%) | 20260507 20:17:12.733 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:17:12.733 | INFO | High Water Breakdown: | 20260507 20:17:12.733 | INFO | 60.9% : 9.64 GiB ( 97.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:17:12.733 | INFO | 39% : 6.17 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:17:12.733 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:17:12.733 | INFO | ------------------------------------------ | 20260507 20:17:12.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:17:12.733 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:17:12.733 | INFO | Total mem usage: 26.5 GiB | 20260507 20:17:12.733 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:17:12.733 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:17:12.733 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:17:12.733 | INFO | High Water Breakdown: | 20260507 20:17:12.733 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:17:12.733 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:17:12.733 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:17:12.733 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:17:12.733 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:17:12.733 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:17:12.733 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:17:12.733 | INFO | Uploaded 36GiB to GPU in 2.49581s (14.4242GiB/s) | 20260507 20:17:12.733 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:17:12.733 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:17:12.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:17:12.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:17:12.733 | INFO | Downloaded 34GiB to GPU in 2.42177s (14.0393GiB/s) | 20260507 20:17:12.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:17:12.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:17:12.733 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:17:12.733 | INFO | ------------------------------------------ | 20260507 20:17:12.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:17:12.733 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:17:12.733 | INFO | Total mem usage: 27.8 GiB | 20260507 20:17:12.733 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:17:12.733 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:17:12.733 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:17:12.733 | INFO | High Water Breakdown: | 20260507 20:17:12.733 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:17:12.733 | INFO | ------------------------------------------ | 20260507 20:17:12.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:17:12.733 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:17:12.733 | INFO | Total mem usage: 15.7 GiB | 20260507 20:17:12.733 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:17:12.733 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:17:12.733 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:17:12.733 | INFO | High Water Breakdown: | 20260507 20:17:12.733 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:17:12.733 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:17:12.733 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:17:12.733 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:17:12.733 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:17:12.733 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:17:12.733 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:17:12.733 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:17:12.733 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:17:12.733 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:17:12.733 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:17:12.733 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:17:12.733 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:17:12.734 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:17:12.734 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:17:12.734 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:17:12.734 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:17:12.734 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:17:12.734 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:17:12.734 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:17:12.734 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:17:12.734 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:17:12.734 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:17:12.734 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:17:12.734 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:17:12.734 | INFO | ------------------------------------------ | 20260507 20:17:12.734 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:17:12.734 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:17:12.734 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:17:12.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:17:12.734 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.59e+01, 1.42e+01 / 2.06e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.15e-02 / 4.47e-03 | 20260507 20:17:12.734 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.50e+00, 5.95e-01 / 2.22e-01 | 20260507 20:17:12.734 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.11e-01 | 20260507 20:17:12.734 | INFO | PerfReport: 15.85%, DME -- 4.19e+01 / 4.53e+01, 3.90e+00 / 1.70e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.75e+00 / 1.94e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.86e+00, 4.43e-01 / 2.01e-01 | 20260507 20:17:12.734 | INFO | PerfReport: 3.60%, Metrics -- 1.19e+00 / 1.43e+00, 3.03e-01 / 1.78e-01 | 20260507 20:17:12.734 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.70e-01 / 4.04e-01, 5.97e-02 / 3.31e-02 | 20260507 20:17:12.734 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:17:12.734 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:17:12.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:17:12.734 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 24.36%, Upload -- 2.61e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 30.90%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 1.98%, LockUpload -- 2.12e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 2.54%, LockDownload -- 2.72e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 15.98%, LockCompute -- 1.71e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 20:17:12.734 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260507 20:17:12.846 | INFO | Starting a round of Baz IO | 20260507 20:17:16.312 | INFO | Populating RTMetrics report | 20260507 20:17:16.313 | INFO | RTMetrics report populated | 20260507 20:17:16.313 | INFO | Opening tmp RTMetrics json file | 20260507 20:17:16.333 | INFO | Serializing RTMetrics to json | 20260507 20:17:16.340 | INFO | Serialization done, swapping tmp file now... | 20260507 20:17:16.341 | INFO | RTMetrics done | 20260507 20:17:16.407 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.56 seconds | 20260507 20:17:16.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.62% / 0.01% / 55.37% | 20260507 20:17:16.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.07098% / 1.98682e-05% / 5.96046e-05% | 20260507 20:17:16.407 | INFO | Baz IO overall: | 20260507 20:17:16.407 | INFO | BytesWritten : 1044244219382 | 20260507 20:17:16.407 | INFO | NumEvents : 506248472528 | 20260507 20:17:17.870 | INFO | Analyzing chunk frames = [2301952,2302464) | 20260507 20:17:17.870 | INFO | MemInfo: { | 20260507 20:17:17.870 | INFO | "gpu" : " 17 GB", | 20260507 20:17:17.870 | INFO | "huge" : "", | 20260507 20:17:17.870 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:17.870 | INFO | "malloc" : " 14 GB", | 20260507 20:17:17.870 | INFO | "pinned" : " 29 GB" | 20260507 20:17:17.870 | INFO | } | 20260507 20:17:17.870 | INFO | Perf report for the last 5.13832 seconds: | 20260507 20:17:17.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:17.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:17.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:17.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:17.870 | INFO | ANALYSIS : 68.7%, 2.9, 106.0ms, 0 | 20260507 20:17:17.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:17.870 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 20:17:17.871 | INFO | PRE_HQ : 67.2%, 1.7, 60.5ms, 0 | 20260507 20:17:17.871 | INFO | BAZWRITER : 68.9%, 4.2, 147.6ms, 0 | 20260507 20:17:17.871 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 20:17:17.871 | INFO | Uploaded 36GiB to GPU in 2.55036s (14.1157GiB/s) | 20260507 20:17:17.871 | INFO | Downloaded 38GiB to GPU in 2.69273s (14.1121GiB/s) | 20260507 20:17:23.010 | INFO | Analyzing chunk frames = [2302464,2302976) | 20260507 20:17:23.010 | INFO | MemInfo: { | 20260507 20:17:23.010 | INFO | "gpu" : " 17 GB", | 20260507 20:17:23.010 | INFO | "huge" : "", | 20260507 20:17:23.010 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:23.010 | INFO | "malloc" : " 14 GB", | 20260507 20:17:23.010 | INFO | "pinned" : " 29 GB" | 20260507 20:17:23.010 | INFO | } | 20260507 20:17:23.010 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:17:23.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:23.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.010 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 20:17:23.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.010 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:17:23.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:23.011 | INFO | Uploaded 36GiB to GPU in 2.49921s (14.4045GiB/s) | 20260507 20:17:23.011 | INFO | Downloaded 34GiB to GPU in 2.49626s (13.6204GiB/s) | 20260507 20:17:28.150 | INFO | Analyzing chunk frames = [2302976,2303488) | 20260507 20:17:28.150 | INFO | MemInfo: { | 20260507 20:17:28.150 | INFO | "gpu" : " 17 GB", | 20260507 20:17:28.150 | INFO | "huge" : "", | 20260507 20:17:28.150 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:28.151 | INFO | "malloc" : " 14 GB", | 20260507 20:17:28.151 | INFO | "pinned" : " 29 GB" | 20260507 20:17:28.151 | INFO | } | 20260507 20:17:28.151 | INFO | Perf report for the last 5.14037 seconds: | 20260507 20:17:28.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:28.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 20:17:28.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:17:28.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:28.151 | INFO | Uploaded 36GiB to GPU in 2.50473s (14.3728GiB/s) | 20260507 20:17:28.151 | INFO | Downloaded 34GiB to GPU in 2.48076s (13.7055GiB/s) | 20260507 20:17:33.289 | INFO | Analyzing chunk frames = [2303488,2304000) | 20260507 20:17:33.289 | INFO | MemInfo: { | 20260507 20:17:33.289 | INFO | "gpu" : " 17 GB", | 20260507 20:17:33.290 | INFO | "huge" : "", | 20260507 20:17:33.290 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:33.290 | INFO | "malloc" : " 14 GB", | 20260507 20:17:33.290 | INFO | "pinned" : " 29 GB" | 20260507 20:17:33.290 | INFO | } | 20260507 20:17:33.290 | INFO | Perf report for the last 5.13906 seconds: | 20260507 20:17:33.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:33.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 20:17:33.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:17:33.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:33.290 | INFO | Uploaded 36GiB to GPU in 2.43568s (14.7802GiB/s) | 20260507 20:17:33.290 | INFO | Downloaded 34GiB to GPU in 2.41407s (14.0841GiB/s) | 20260507 20:17:38.429 | INFO | Analyzing chunk frames = [2304000,2304512) | 20260507 20:17:38.429 | INFO | MemInfo: { | 20260507 20:17:38.429 | INFO | "gpu" : " 17 GB", | 20260507 20:17:38.429 | INFO | "huge" : "", | 20260507 20:17:38.429 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:38.429 | INFO | "malloc" : " 14 GB", | 20260507 20:17:38.429 | INFO | "pinned" : " 29 GB" | 20260507 20:17:38.429 | INFO | } | 20260507 20:17:38.429 | INFO | Perf report for the last 5.13911 seconds: | 20260507 20:17:38.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:38.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 20:17:38.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:17:38.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:38.429 | INFO | Uploaded 36GiB to GPU in 2.3625s (15.2381GiB/s) | 20260507 20:17:38.429 | INFO | Downloaded 34GiB to GPU in 2.5124s (13.5329GiB/s) | 20260507 20:17:43.568 | INFO | Analyzing chunk frames = [2304512,2305024) | 20260507 20:17:43.568 | INFO | MemInfo: { | 20260507 20:17:43.568 | INFO | "gpu" : " 17 GB", | 20260507 20:17:43.568 | INFO | "huge" : "", | 20260507 20:17:43.568 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:43.568 | INFO | "malloc" : " 14 GB", | 20260507 20:17:43.568 | INFO | "pinned" : " 29 GB" | 20260507 20:17:43.568 | INFO | } | 20260507 20:17:43.568 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:17:43.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:43.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 20:17:43.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:17:43.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:43.568 | INFO | Uploaded 36GiB to GPU in 2.46589s (14.5992GiB/s) | 20260507 20:17:43.568 | INFO | Downloaded 34GiB to GPU in 2.44144s (13.9262GiB/s) | 20260507 20:17:48.707 | INFO | Analyzing chunk frames = [2305024,2305536) | 20260507 20:17:48.707 | INFO | MemInfo: { | 20260507 20:17:48.707 | INFO | "gpu" : " 17 GB", | 20260507 20:17:48.707 | INFO | "huge" : "", | 20260507 20:17:48.707 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:48.707 | INFO | "malloc" : " 14 GB", | 20260507 20:17:48.707 | INFO | "pinned" : " 29 GB" | 20260507 20:17:48.707 | INFO | } | 20260507 20:17:48.707 | INFO | Perf report for the last 5.13933 seconds: | 20260507 20:17:48.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:48.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.707 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 20:17:48.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.707 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 20:17:48.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:48.708 | INFO | Uploaded 36GiB to GPU in 2.46948s (14.578GiB/s) | 20260507 20:17:48.708 | INFO | Downloaded 34GiB to GPU in 2.45819s (13.8313GiB/s) | 20260507 20:17:53.846 | INFO | Analyzing chunk frames = [2305536,2306048) | 20260507 20:17:53.846 | INFO | MemInfo: { | 20260507 20:17:53.846 | INFO | "gpu" : " 17 GB", | 20260507 20:17:53.846 | INFO | "huge" : "", | 20260507 20:17:53.846 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:53.846 | INFO | "malloc" : " 14 GB", | 20260507 20:17:53.846 | INFO | "pinned" : " 29 GB" | 20260507 20:17:53.846 | INFO | } | 20260507 20:17:53.846 | INFO | Perf report for the last 5.13857 seconds: | 20260507 20:17:53.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:53.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 20:17:53.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:17:53.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:53.846 | INFO | Uploaded 36GiB to GPU in 2.46175s (14.6237GiB/s) | 20260507 20:17:53.846 | INFO | Downloaded 34GiB to GPU in 2.4442s (13.9105GiB/s) | 20260507 20:17:53.943 | INFO | Starting a round of Baz IO | 20260507 20:17:57.403 | INFO | Populating RTMetrics report | 20260507 20:17:57.403 | INFO | RTMetrics report populated | 20260507 20:17:57.403 | INFO | Opening tmp RTMetrics json file | 20260507 20:17:57.435 | INFO | Serializing RTMetrics to json | 20260507 20:17:57.453 | INFO | Serialization done, swapping tmp file now... | 20260507 20:17:57.485 | INFO | RTMetrics done | 20260507 20:17:57.523 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.579 seconds | 20260507 20:17:57.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.61% / 0.01% / 55.38% | 20260507 20:17:57.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.06596% / 1.19209e-05% / 5.96046e-05% | 20260507 20:17:57.523 | INFO | Baz IO overall: | 20260507 20:17:57.523 | INFO | BytesWritten : 1046155952042 | 20260507 20:17:57.523 | INFO | NumEvents : 507174464364 | 20260507 20:17:58.985 | INFO | Analyzing chunk frames = [2306048,2306560) | 20260507 20:17:58.985 | INFO | MemInfo: { | 20260507 20:17:58.985 | INFO | "gpu" : " 17 GB", | 20260507 20:17:58.985 | INFO | "huge" : "", | 20260507 20:17:58.985 | INFO | "huge_shared" : " 29 GB", | 20260507 20:17:58.985 | INFO | "malloc" : " 14 GB", | 20260507 20:17:58.987 | INFO | "pinned" : " 29 GB" | 20260507 20:17:58.987 | INFO | } | 20260507 20:17:58.987 | INFO | Perf report for the last 5.14112 seconds: | 20260507 20:17:58.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:17:58.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:58.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:58.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:58.987 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260507 20:17:58.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:17:58.987 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 20:17:58.987 | INFO | PRE_HQ : 65.9%, 1.7, 57.9ms, 0 | 20260507 20:17:58.987 | INFO | BAZWRITER : 69.2%, 3.7, 130.0ms, 0 | 20260507 20:17:58.987 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:17:58.987 | INFO | Uploaded 36GiB to GPU in 2.50948s (14.3456GiB/s) | 20260507 20:17:58.987 | INFO | Downloaded 38GiB to GPU in 2.69307s (14.1103GiB/s) | 20260507 20:18:04.125 | INFO | Analyzing chunk frames = [2306560,2307072) | 20260507 20:18:04.125 | INFO | MemInfo: { | 20260507 20:18:04.125 | INFO | "gpu" : " 17 GB", | 20260507 20:18:04.125 | INFO | "huge" : "", | 20260507 20:18:04.125 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:04.125 | INFO | "malloc" : " 14 GB", | 20260507 20:18:04.125 | INFO | "pinned" : " 29 GB" | 20260507 20:18:04.125 | INFO | } | 20260507 20:18:04.125 | INFO | Perf report for the last 5.13832 seconds: | 20260507 20:18:04.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:04.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.125 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:18:04.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.125 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:18:04.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:04.126 | INFO | Uploaded 36GiB to GPU in 2.50969s (14.3444GiB/s) | 20260507 20:18:04.126 | INFO | Downloaded 34GiB to GPU in 2.51407s (13.5239GiB/s) | 20260507 20:18:09.265 | INFO | Analyzing chunk frames = [2307072,2307584) | 20260507 20:18:09.265 | INFO | MemInfo: { | 20260507 20:18:09.265 | INFO | "gpu" : " 17 GB", | 20260507 20:18:09.265 | INFO | "huge" : "", | 20260507 20:18:09.265 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:09.265 | INFO | "malloc" : " 14 GB", | 20260507 20:18:09.265 | INFO | "pinned" : " 29 GB" | 20260507 20:18:09.265 | INFO | } | 20260507 20:18:09.265 | INFO | Perf report for the last 5.14012 seconds: | 20260507 20:18:09.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:09.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.265 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 20:18:09.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.265 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:18:09.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:09.266 | INFO | Uploaded 36GiB to GPU in 2.48855s (14.4663GiB/s) | 20260507 20:18:09.266 | INFO | Downloaded 34GiB to GPU in 2.47491s (13.7379GiB/s) | 20260507 20:18:14.405 | INFO | Analyzing chunk frames = [2307584,2308096) | 20260507 20:18:14.405 | INFO | MemInfo: { | 20260507 20:18:14.405 | INFO | "gpu" : " 17 GB", | 20260507 20:18:14.405 | INFO | "huge" : "", | 20260507 20:18:14.405 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:14.405 | INFO | "malloc" : " 14 GB", | 20260507 20:18:14.405 | INFO | "pinned" : " 29 GB" | 20260507 20:18:14.405 | INFO | } | 20260507 20:18:14.405 | INFO | Perf report for the last 5.13964 seconds: | 20260507 20:18:14.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:14.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 20:18:14.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:18:14.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:14.405 | INFO | Uploaded 36GiB to GPU in 2.46326s (14.6148GiB/s) | 20260507 20:18:14.405 | INFO | Downloaded 34GiB to GPU in 2.40739s (14.1232GiB/s) | 20260507 20:18:19.544 | INFO | Analyzing chunk frames = [2308096,2308608) | 20260507 20:18:19.544 | INFO | MemInfo: { | 20260507 20:18:19.544 | INFO | "gpu" : " 17 GB", | 20260507 20:18:19.544 | INFO | "huge" : "", | 20260507 20:18:19.544 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:19.544 | INFO | "malloc" : " 14 GB", | 20260507 20:18:19.544 | INFO | "pinned" : " 29 GB" | 20260507 20:18:19.544 | INFO | } | 20260507 20:18:19.544 | INFO | Perf report for the last 5.13893 seconds: | 20260507 20:18:19.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:19.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:18:19.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:18:19.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:19.544 | INFO | Uploaded 36GiB to GPU in 2.35373s (15.2949GiB/s) | 20260507 20:18:19.544 | INFO | Downloaded 34GiB to GPU in 2.50776s (13.5579GiB/s) | 20260507 20:18:24.682 | INFO | Analyzing chunk frames = [2308608,2309120) | 20260507 20:18:24.682 | INFO | MemInfo: { | 20260507 20:18:24.682 | INFO | "gpu" : " 17 GB", | 20260507 20:18:24.682 | INFO | "huge" : "", | 20260507 20:18:24.682 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:24.682 | INFO | "malloc" : " 14 GB", | 20260507 20:18:24.682 | INFO | "pinned" : " 29 GB" | 20260507 20:18:24.682 | INFO | } | 20260507 20:18:24.682 | INFO | Perf report for the last 5.13804 seconds: | 20260507 20:18:24.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:24.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:18:24.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 20:18:24.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:24.682 | INFO | Uploaded 36GiB to GPU in 2.41124s (14.9301GiB/s) | 20260507 20:18:24.682 | INFO | Downloaded 34GiB to GPU in 2.39185s (14.2149GiB/s) | 20260507 20:18:29.821 | INFO | Analyzing chunk frames = [2309120,2309632) | 20260507 20:18:29.821 | INFO | MemInfo: { | 20260507 20:18:29.821 | INFO | "gpu" : " 17 GB", | 20260507 20:18:29.821 | INFO | "huge" : "", | 20260507 20:18:29.821 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:29.821 | INFO | "malloc" : " 14 GB", | 20260507 20:18:29.821 | INFO | "pinned" : " 29 GB" | 20260507 20:18:29.822 | INFO | } | 20260507 20:18:29.822 | INFO | Perf report for the last 5.13953 seconds: | 20260507 20:18:29.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:29.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:18:29.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 20:18:29.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:29.822 | INFO | Uploaded 36GiB to GPU in 2.53541s (14.1989GiB/s) | 20260507 20:18:29.822 | INFO | Downloaded 34GiB to GPU in 2.45897s (13.8269GiB/s) | 20260507 20:18:34.961 | INFO | Analyzing chunk frames = [2309632,2310144) | 20260507 20:18:34.961 | INFO | MemInfo: { | 20260507 20:18:34.961 | INFO | "gpu" : " 17 GB", | 20260507 20:18:34.961 | INFO | "huge" : "", | 20260507 20:18:34.962 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:34.962 | INFO | "malloc" : " 14 GB", | 20260507 20:18:34.962 | INFO | "pinned" : " 29 GB" | 20260507 20:18:34.962 | INFO | } | 20260507 20:18:34.962 | INFO | Perf report for the last 5.14007 seconds: | 20260507 20:18:34.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:34.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:18:34.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 20:18:34.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:34.962 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:18:34.962 | INFO | Total mem usage: 16.4 GiB | 20260507 20:18:34.962 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:18:34.962 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -8.89%) | 20260507 20:18:34.962 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:18:34.962 | INFO | High Water Breakdown: | 20260507 20:18:34.962 | INFO | 60.1% : 9.05 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:18:34.962 | INFO | 39.8% : 6 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:18:34.962 | INFO | Uploaded 36GiB to GPU in 2.51402s (14.3197GiB/s) | 20260507 20:18:34.962 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:18:34.962 | INFO | ------------------------------------------ | 20260507 20:18:34.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:18:34.962 | INFO | Downloaded 34GiB to GPU in 2.43701s (13.9515GiB/s) | 20260507 20:18:34.962 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:18:34.962 | INFO | Total mem usage: 26.5 GiB | 20260507 20:18:34.962 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:18:34.962 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:18:34.962 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:18:34.962 | INFO | High Water Breakdown: | 20260507 20:18:34.962 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:18:34.963 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:18:34.963 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:18:34.963 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:18:34.963 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:18:34.963 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:18:34.963 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:18:34.963 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:18:34.963 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:18:34.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:18:34.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:18:34.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:18:34.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:18:34.963 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:18:34.963 | INFO | ------------------------------------------ | 20260507 20:18:34.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:18:34.963 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:18:34.963 | INFO | Total mem usage: 27.8 GiB | 20260507 20:18:34.963 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:18:34.963 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:18:34.963 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:18:34.963 | INFO | High Water Breakdown: | 20260507 20:18:34.963 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:18:34.963 | INFO | ------------------------------------------ | 20260507 20:18:34.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:18:34.963 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:18:34.963 | INFO | Total mem usage: 15.7 GiB | 20260507 20:18:34.963 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:18:34.963 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:18:34.963 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:18:34.963 | INFO | High Water Breakdown: | 20260507 20:18:34.963 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:18:34.963 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:18:34.963 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:18:34.963 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:18:34.963 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:18:34.963 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:18:34.963 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:18:34.963 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:18:34.963 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:18:34.963 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:18:34.963 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:18:34.963 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:18:34.963 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:18:34.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:18:34.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:18:34.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:18:34.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:18:34.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:18:34.963 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:18:34.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:18:34.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:18:34.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:18:34.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:18:34.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:18:34.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:18:34.963 | INFO | ------------------------------------------ | 20260507 20:18:34.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:18:34.963 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:18:34.963 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:18:34.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:18:34.963 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.55e+01, 1.41e+01 / 1.70e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.05e-01, 1.15e-02 / 6.11e-03 | 20260507 20:18:34.963 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.49e+00, 6.02e-01 / 2.59e-01 | 20260507 20:18:34.963 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.51e+00, 2.13e-01 / 1.09e-01 | 20260507 20:18:34.963 | INFO | PerfReport: 15.72%, DME -- 4.17e+01 / 4.49e+01, 3.83e+00 / 1.34e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.86e+00 / 1.63e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.49e+00 / 2.82e+00, 4.42e-01 / 2.31e-01 | 20260507 20:18:34.963 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.39e+00, 3.02e-01 / 1.75e-01 | 20260507 20:18:34.963 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.01e-01, 6.00e-02 / 2.91e-02 | 20260507 20:18:34.963 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:18:34.963 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:18:34.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:18:34.963 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 24.13%, Upload -- 2.57e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 31.11%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 1.66%, LockUpload -- 1.77e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 16.09%, LockCompute -- 1.72e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 20:18:34.963 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.74e-02 / 0.00e+00 | 20260507 20:18:35.074 | INFO | Starting a round of Baz IO | 20260507 20:18:38.553 | INFO | Populating RTMetrics report | 20260507 20:18:38.554 | INFO | RTMetrics report populated | 20260507 20:18:38.554 | INFO | Opening tmp RTMetrics json file | 20260507 20:18:38.554 | INFO | Serializing RTMetrics to json | 20260507 20:18:38.556 | INFO | Serialization done, swapping tmp file now... | 20260507 20:18:38.556 | INFO | RTMetrics done | 20260507 20:18:38.645 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.57 seconds | 20260507 20:18:38.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.60% / 0.01% / 55.39% | 20260507 20:18:38.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05752% / 2.38419e-05% / 3.97364e-05% | 20260507 20:18:38.645 | INFO | Baz IO overall: | 20260507 20:18:38.645 | INFO | BytesWritten : 1048065861055 | 20260507 20:18:38.645 | INFO | NumEvents : 508099243950 | 20260507 20:18:40.100 | INFO | Analyzing chunk frames = [2310144,2310656) | 20260507 20:18:40.100 | INFO | MemInfo: { | 20260507 20:18:40.100 | INFO | "gpu" : " 17 GB", | 20260507 20:18:40.100 | INFO | "huge" : "", | 20260507 20:18:40.100 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:40.100 | INFO | "malloc" : " 14 GB", | 20260507 20:18:40.100 | INFO | "pinned" : " 29 GB" | 20260507 20:18:40.100 | INFO | } | 20260507 20:18:40.100 | INFO | Perf report for the last 5.13872 seconds: | 20260507 20:18:40.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:40.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:40.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:40.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:40.101 | INFO | ANALYSIS : 69.5%, 2.9, 104.7ms, 0 | 20260507 20:18:40.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:40.101 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 20:18:40.101 | INFO | PRE_HQ : 68.2%, 1.8, 64.3ms, 0 | 20260507 20:18:40.101 | INFO | BAZWRITER : 69.0%, 3.8, 136.3ms, 0 | 20260507 20:18:40.101 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 20:18:40.101 | INFO | Uploaded 36GiB to GPU in 2.55898s (14.0681GiB/s) | 20260507 20:18:40.101 | INFO | Downloaded 38GiB to GPU in 2.69307s (14.1103GiB/s) | 20260507 20:18:45.241 | INFO | Analyzing chunk frames = [2310656,2311168) | 20260507 20:18:45.241 | INFO | MemInfo: { | 20260507 20:18:45.241 | INFO | "gpu" : " 17 GB", | 20260507 20:18:45.241 | INFO | "huge" : "", | 20260507 20:18:45.241 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:45.241 | INFO | "malloc" : " 14 GB", | 20260507 20:18:45.241 | INFO | "pinned" : " 29 GB" | 20260507 20:18:45.241 | INFO | } | 20260507 20:18:45.241 | INFO | Perf report for the last 5.14097 seconds: | 20260507 20:18:45.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:45.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.241 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 20:18:45.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.242 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 20:18:45.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:45.242 | INFO | Uploaded 36GiB to GPU in 2.57207s (13.9965GiB/s) | 20260507 20:18:45.242 | INFO | Downloaded 34GiB to GPU in 2.48261s (13.6952GiB/s) | 20260507 20:18:50.379 | INFO | Analyzing chunk frames = [2311168,2311680) | 20260507 20:18:50.379 | INFO | MemInfo: { | 20260507 20:18:50.379 | INFO | "gpu" : " 17 GB", | 20260507 20:18:50.379 | INFO | "huge" : "", | 20260507 20:18:50.379 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:50.379 | INFO | "malloc" : " 14 GB", | 20260507 20:18:50.379 | INFO | "pinned" : " 29 GB" | 20260507 20:18:50.379 | INFO | } | 20260507 20:18:50.379 | INFO | Perf report for the last 5.13784 seconds: | 20260507 20:18:50.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:50.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:18:50.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:18:50.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:50.379 | INFO | Uploaded 36GiB to GPU in 2.52263s (14.2708GiB/s) | 20260507 20:18:50.380 | INFO | Downloaded 34GiB to GPU in 2.47656s (13.7287GiB/s) | 20260507 20:18:55.518 | INFO | Analyzing chunk frames = [2311680,2312192) | 20260507 20:18:55.518 | INFO | MemInfo: { | 20260507 20:18:55.518 | INFO | "gpu" : " 17 GB", | 20260507 20:18:55.518 | INFO | "huge" : "", | 20260507 20:18:55.518 | INFO | "huge_shared" : " 29 GB", | 20260507 20:18:55.518 | INFO | "malloc" : " 14 GB", | 20260507 20:18:55.518 | INFO | "pinned" : " 29 GB" | 20260507 20:18:55.518 | INFO | } | 20260507 20:18:55.518 | INFO | Perf report for the last 5.13898 seconds: | 20260507 20:18:55.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:18:55.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260507 20:18:55.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:18:55.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:18:55.518 | INFO | Uploaded 36GiB to GPU in 2.52978s (14.2305GiB/s) | 20260507 20:18:55.519 | INFO | Downloaded 34GiB to GPU in 2.39991s (14.1672GiB/s) | 20260507 20:19:00.658 | INFO | Analyzing chunk frames = [2312192,2312704) | 20260507 20:19:00.658 | INFO | MemInfo: { | 20260507 20:19:00.658 | INFO | "gpu" : " 17 GB", | 20260507 20:19:00.658 | INFO | "huge" : "", | 20260507 20:19:00.658 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:00.658 | INFO | "malloc" : " 14 GB", | 20260507 20:19:00.658 | INFO | "pinned" : " 29 GB" | 20260507 20:19:00.658 | INFO | } | 20260507 20:19:00.658 | INFO | Perf report for the last 5.13973 seconds: | 20260507 20:19:00.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:00.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.658 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 20:19:00.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.658 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:19:00.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:00.659 | INFO | Uploaded 36GiB to GPU in 2.43281s (14.7977GiB/s) | 20260507 20:19:00.659 | INFO | Downloaded 34GiB to GPU in 2.49198s (13.6438GiB/s) | 20260507 20:19:05.797 | INFO | Analyzing chunk frames = [2312704,2313216) | 20260507 20:19:05.797 | INFO | MemInfo: { | 20260507 20:19:05.797 | INFO | "gpu" : " 17 GB", | 20260507 20:19:05.797 | INFO | "huge" : "", | 20260507 20:19:05.797 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:05.797 | INFO | "malloc" : " 14 GB", | 20260507 20:19:05.797 | INFO | "pinned" : " 29 GB" | 20260507 20:19:05.797 | INFO | } | 20260507 20:19:05.797 | INFO | Perf report for the last 5.13933 seconds: | 20260507 20:19:05.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:05.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.797 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:19:05.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.797 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:19:05.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:05.798 | INFO | Uploaded 36GiB to GPU in 2.47778s (14.5291GiB/s) | 20260507 20:19:05.798 | INFO | Downloaded 34GiB to GPU in 2.41912s (14.0547GiB/s) | 20260507 20:19:10.937 | INFO | Analyzing chunk frames = [2313216,2313728) | 20260507 20:19:10.937 | INFO | MemInfo: { | 20260507 20:19:10.937 | INFO | "gpu" : " 17 GB", | 20260507 20:19:10.937 | INFO | "huge" : "", | 20260507 20:19:10.937 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:10.937 | INFO | "malloc" : " 14 GB", | 20260507 20:19:10.937 | INFO | "pinned" : " 29 GB" | 20260507 20:19:10.937 | INFO | } | 20260507 20:19:10.937 | INFO | Perf report for the last 5.13977 seconds: | 20260507 20:19:10.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:10.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.937 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:19:10.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.937 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 20:19:10.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:10.938 | INFO | Uploaded 36GiB to GPU in 2.51788s (14.2978GiB/s) | 20260507 20:19:10.938 | INFO | Downloaded 34GiB to GPU in 2.44513s (13.9052GiB/s) | 20260507 20:19:16.077 | INFO | Analyzing chunk frames = [2313728,2314240) | 20260507 20:19:16.077 | INFO | MemInfo: { | 20260507 20:19:16.077 | INFO | "gpu" : " 17 GB", | 20260507 20:19:16.077 | INFO | "huge" : "", | 20260507 20:19:16.077 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:16.077 | INFO | "malloc" : " 14 GB", | 20260507 20:19:16.077 | INFO | "pinned" : " 29 GB" | 20260507 20:19:16.077 | INFO | } | 20260507 20:19:16.077 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:19:16.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:16.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.077 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 20:19:16.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.077 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 20:19:16.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:16.078 | INFO | Uploaded 36GiB to GPU in 2.52449s (14.2603GiB/s) | 20260507 20:19:16.078 | INFO | Downloaded 34GiB to GPU in 2.43396s (13.969GiB/s) | 20260507 20:19:16.190 | INFO | Starting a round of Baz IO | 20260507 20:19:19.683 | INFO | Populating RTMetrics report | 20260507 20:19:19.683 | INFO | RTMetrics report populated | 20260507 20:19:19.683 | INFO | Opening tmp RTMetrics json file | 20260507 20:19:19.767 | INFO | Serializing RTMetrics to json | 20260507 20:19:19.783 | INFO | Serialization done, swapping tmp file now... | 20260507 20:19:19.791 | INFO | RTMetrics done | 20260507 20:19:19.875 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.684 seconds | 20260507 20:19:19.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.59% / 0.01% / 55.40% | 20260507 20:19:19.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05593% / 7.94729e-06% / 3.97364e-05% | 20260507 20:19:19.875 | INFO | Baz IO overall: | 20260507 20:19:19.875 | INFO | BytesWritten : 1049976251027 | 20260507 20:19:19.875 | INFO | NumEvents : 509024484529 | 20260507 20:19:21.216 | INFO | Analyzing chunk frames = [2314240,2314752) | 20260507 20:19:21.216 | INFO | MemInfo: { | 20260507 20:19:21.216 | INFO | "gpu" : " 17 GB", | 20260507 20:19:21.216 | INFO | "huge" : "", | 20260507 20:19:21.217 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:21.217 | INFO | "malloc" : " 14 GB", | 20260507 20:19:21.217 | INFO | "pinned" : " 29 GB" | 20260507 20:19:21.217 | INFO | } | 20260507 20:19:21.217 | INFO | Perf report for the last 5.13954 seconds: | 20260507 20:19:21.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:21.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:21.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:21.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:21.217 | INFO | ANALYSIS : 69.7%, 2.9, 104.4ms, 0 | 20260507 20:19:21.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:21.217 | INFO | RT_METRICS : 50.3%, 1.0, 26.4ms, 0 | 20260507 20:19:21.217 | INFO | PRE_HQ : 69.2%, 1.9, 68.4ms, 0 | 20260507 20:19:21.217 | INFO | BAZWRITER : 71.2%, 4.7, 170.3ms, 0 | 20260507 20:19:21.217 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:19:21.217 | INFO | Uploaded 36GiB to GPU in 2.53068s (14.2255GiB/s) | 20260507 20:19:21.217 | INFO | Downloaded 38GiB to GPU in 2.70566s (14.0446GiB/s) | 20260507 20:19:26.354 | INFO | Analyzing chunk frames = [2314752,2315264) | 20260507 20:19:26.354 | INFO | MemInfo: { | 20260507 20:19:26.354 | INFO | "gpu" : " 17 GB", | 20260507 20:19:26.354 | INFO | "huge" : "", | 20260507 20:19:26.354 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:26.354 | INFO | "malloc" : " 14 GB", | 20260507 20:19:26.354 | INFO | "pinned" : " 29 GB" | 20260507 20:19:26.354 | INFO | } | 20260507 20:19:26.355 | INFO | Perf report for the last 5.13787 seconds: | 20260507 20:19:26.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:26.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:19:26.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:19:26.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:26.355 | INFO | Uploaded 36GiB to GPU in 2.53434s (14.2049GiB/s) | 20260507 20:19:26.355 | INFO | Downloaded 34GiB to GPU in 2.47958s (13.712GiB/s) | 20260507 20:19:31.493 | INFO | Analyzing chunk frames = [2315264,2315776) | 20260507 20:19:31.493 | INFO | MemInfo: { | 20260507 20:19:31.493 | INFO | "gpu" : " 17 GB", | 20260507 20:19:31.493 | INFO | "huge" : "", | 20260507 20:19:31.493 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:31.493 | INFO | "malloc" : " 14 GB", | 20260507 20:19:31.493 | INFO | "pinned" : " 29 GB" | 20260507 20:19:31.493 | INFO | } | 20260507 20:19:31.493 | INFO | Perf report for the last 5.13873 seconds: | 20260507 20:19:31.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:31.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.493 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:19:31.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.493 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:19:31.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:31.494 | INFO | Uploaded 36GiB to GPU in 2.57344s (13.989GiB/s) | 20260507 20:19:31.494 | INFO | Downloaded 34GiB to GPU in 2.4741s (13.7424GiB/s) | 20260507 20:19:36.633 | INFO | Analyzing chunk frames = [2315776,2316288) | 20260507 20:19:36.633 | INFO | MemInfo: { | 20260507 20:19:36.633 | INFO | "gpu" : " 17 GB", | 20260507 20:19:36.633 | INFO | "huge" : "", | 20260507 20:19:36.633 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:36.633 | INFO | "malloc" : " 14 GB", | 20260507 20:19:36.633 | INFO | "pinned" : " 29 GB" | 20260507 20:19:36.633 | INFO | } | 20260507 20:19:36.633 | INFO | Perf report for the last 5.1396 seconds: | 20260507 20:19:36.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:36.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 20:19:36.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:19:36.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:36.633 | INFO | Uploaded 36GiB to GPU in 2.46781s (14.5878GiB/s) | 20260507 20:19:36.634 | INFO | Downloaded 34GiB to GPU in 2.39194s (14.2144GiB/s) | 20260507 20:19:41.772 | INFO | Analyzing chunk frames = [2316288,2316800) | 20260507 20:19:41.772 | INFO | MemInfo: { | 20260507 20:19:41.772 | INFO | "gpu" : " 17 GB", | 20260507 20:19:41.772 | INFO | "huge" : "", | 20260507 20:19:41.772 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:41.772 | INFO | "malloc" : " 14 GB", | 20260507 20:19:41.772 | INFO | "pinned" : " 29 GB" | 20260507 20:19:41.772 | INFO | } | 20260507 20:19:41.772 | INFO | Perf report for the last 5.13924 seconds: | 20260507 20:19:41.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:41.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.772 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:19:41.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.772 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:19:41.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:41.773 | INFO | Uploaded 36GiB to GPU in 2.36472s (15.2238GiB/s) | 20260507 20:19:41.773 | INFO | Downloaded 34GiB to GPU in 2.49866s (13.6073GiB/s) | 20260507 20:19:46.911 | INFO | Analyzing chunk frames = [2316800,2317312) | 20260507 20:19:46.911 | INFO | MemInfo: { | 20260507 20:19:46.911 | INFO | "gpu" : " 17 GB", | 20260507 20:19:46.911 | INFO | "huge" : "", | 20260507 20:19:46.911 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:46.911 | INFO | "malloc" : " 14 GB", | 20260507 20:19:46.911 | INFO | "pinned" : " 29 GB" | 20260507 20:19:46.911 | INFO | } | 20260507 20:19:46.911 | INFO | Perf report for the last 5.13922 seconds: | 20260507 20:19:46.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:46.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.911 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:19:46.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.912 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:19:46.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:46.912 | INFO | Uploaded 36GiB to GPU in 2.48731s (14.4734GiB/s) | 20260507 20:19:46.912 | INFO | Downloaded 34GiB to GPU in 2.42469s (14.0224GiB/s) | 20260507 20:19:52.052 | INFO | Analyzing chunk frames = [2317312,2317824) | 20260507 20:19:52.052 | INFO | MemInfo: { | 20260507 20:19:52.052 | INFO | "gpu" : " 17 GB", | 20260507 20:19:52.052 | INFO | "huge" : "", | 20260507 20:19:52.052 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:52.052 | INFO | "malloc" : " 14 GB", | 20260507 20:19:52.052 | INFO | "pinned" : " 29 GB" | 20260507 20:19:52.052 | INFO | } | 20260507 20:19:52.052 | INFO | Perf report for the last 5.14098 seconds: | 20260507 20:19:52.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:52.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.052 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:19:52.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.053 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 20:19:52.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:52.053 | INFO | Uploaded 36GiB to GPU in 2.52314s (14.2679GiB/s) | 20260507 20:19:52.053 | INFO | Downloaded 34GiB to GPU in 2.44493s (13.9063GiB/s) | 20260507 20:19:57.191 | INFO | Analyzing chunk frames = [2317824,2318336) | 20260507 20:19:57.191 | INFO | MemInfo: { | 20260507 20:19:57.191 | INFO | "gpu" : " 17 GB", | 20260507 20:19:57.191 | INFO | "huge" : "", | 20260507 20:19:57.191 | INFO | "huge_shared" : " 29 GB", | 20260507 20:19:57.191 | INFO | "malloc" : " 14 GB", | 20260507 20:19:57.191 | INFO | "pinned" : " 29 GB" | 20260507 20:19:57.191 | INFO | } | 20260507 20:19:57.191 | INFO | Perf report for the last 5.13866 seconds: | 20260507 20:19:57.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:19:57.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:19:57.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 20:19:57.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:19:57.191 | INFO | Uploaded 36GiB to GPU in 2.51379s (14.321GiB/s) | 20260507 20:19:57.192 | INFO | Downloaded 34GiB to GPU in 2.43695s (13.9519GiB/s) | 20260507 20:19:57.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:19:57.192 | INFO | Total mem usage: 16.4 GiB | 20260507 20:19:57.192 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:19:57.192 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8.12%) | 20260507 20:19:57.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:19:57.192 | INFO | High Water Breakdown: | 20260507 20:19:57.192 | INFO | 60.2% : 9.13 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:19:57.192 | INFO | 39.7% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:19:57.192 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:19:57.192 | INFO | ------------------------------------------ | 20260507 20:19:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:19:57.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:19:57.192 | INFO | Total mem usage: 26.5 GiB | 20260507 20:19:57.192 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:19:57.192 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:19:57.192 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:19:57.192 | INFO | High Water Breakdown: | 20260507 20:19:57.192 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:19:57.192 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:19:57.192 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:19:57.192 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:19:57.192 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:19:57.192 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:19:57.192 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:19:57.192 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:19:57.192 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:19:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:19:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:19:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:19:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:19:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:19:57.192 | INFO | ------------------------------------------ | 20260507 20:19:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:19:57.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:19:57.192 | INFO | Total mem usage: 27.8 GiB | 20260507 20:19:57.192 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:19:57.192 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:19:57.192 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:19:57.192 | INFO | High Water Breakdown: | 20260507 20:19:57.192 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:19:57.192 | INFO | ------------------------------------------ | 20260507 20:19:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:19:57.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:19:57.192 | INFO | Total mem usage: 15.7 GiB | 20260507 20:19:57.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:19:57.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:19:57.192 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:19:57.192 | INFO | High Water Breakdown: | 20260507 20:19:57.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:19:57.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:19:57.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:19:57.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:19:57.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:19:57.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:19:57.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:19:57.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:19:57.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:19:57.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:19:57.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:19:57.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:19:57.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:19:57.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:19:57.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:19:57.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:19:57.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:19:57.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:19:57.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:19:57.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:19:57.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:19:57.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:19:57.193 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:19:57.193 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:19:57.193 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:19:57.193 | INFO | ------------------------------------------ | 20260507 20:19:57.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:19:57.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:19:57.193 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:19:57.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:19:57.193 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.41e+01 / 1.77e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.03e-01, 1.20e-02 / 5.77e-03 | 20260507 20:19:57.193 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.42e+00, 5.95e-01 / 3.10e-01 | 20260507 20:19:57.193 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.12e-01 / 1.05e-01 | 20260507 20:19:57.193 | INFO | PerfReport: 15.88%, DME -- 4.21e+01 / 4.55e+01, 3.97e+00 / 1.46e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.76e+00 / 1.73e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.79e+00, 4.44e-01 / 2.76e-01 | 20260507 20:19:57.193 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.37e+00, 2.99e-01 / 1.43e-01 | 20260507 20:19:57.193 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.01e-01, 5.97e-02 / 3.72e-02 | 20260507 20:19:57.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:19:57.193 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:19:57.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:19:57.193 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 24.22%, Download -- 2.58e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 31.02%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 2.42%, LockDownload -- 2.59e+00 / 0.00e+00, 3.74e+00 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 15.85%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 20:19:57.193 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260507 20:19:57.302 | INFO | Starting a round of Baz IO | 20260507 20:20:00.809 | INFO | Populating RTMetrics report | 20260507 20:20:00.809 | INFO | RTMetrics report populated | 20260507 20:20:00.809 | INFO | Opening tmp RTMetrics json file | 20260507 20:20:00.811 | INFO | Serializing RTMetrics to json | 20260507 20:20:00.813 | INFO | Serialization done, swapping tmp file now... | 20260507 20:20:00.813 | INFO | RTMetrics done | 20260507 20:20:00.927 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.625 seconds | 20260507 20:20:00.927 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.58% / 0.01% / 55.41% | 20260507 20:20:00.927 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05536% / 1.19209e-05% / 5.5631e-05% | 20260507 20:20:00.927 | INFO | Baz IO overall: | 20260507 20:20:00.927 | INFO | BytesWritten : 1051887095251 | 20260507 20:20:00.927 | INFO | NumEvents : 509950103478 | 20260507 20:20:02.330 | INFO | Analyzing chunk frames = [2318336,2318848) | 20260507 20:20:02.330 | INFO | MemInfo: { | 20260507 20:20:02.330 | INFO | "gpu" : " 17 GB", | 20260507 20:20:02.330 | INFO | "huge" : "", | 20260507 20:20:02.330 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:02.330 | INFO | "malloc" : " 14 GB", | 20260507 20:20:02.330 | INFO | "pinned" : " 29 GB" | 20260507 20:20:02.330 | INFO | } | 20260507 20:20:02.330 | INFO | Perf report for the last 5.13913 seconds: | 20260507 20:20:02.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:02.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:02.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:02.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:02.330 | INFO | ANALYSIS : 70.1%, 2.9, 105.9ms, 0 | 20260507 20:20:02.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:02.330 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 20:20:02.330 | INFO | PRE_HQ : 68.9%, 1.8, 64.3ms, 0 | 20260507 20:20:02.330 | INFO | BAZWRITER : 70.1%, 4.0, 145.7ms, 0 | 20260507 20:20:02.330 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 20:20:02.331 | INFO | Uploaded 36GiB to GPU in 2.55071s (14.1137GiB/s) | 20260507 20:20:02.331 | INFO | Downloaded 38GiB to GPU in 2.70576s (14.0441GiB/s) | 20260507 20:20:07.469 | INFO | Analyzing chunk frames = [2318848,2319360) | 20260507 20:20:07.469 | INFO | MemInfo: { | 20260507 20:20:07.469 | INFO | "gpu" : " 17 GB", | 20260507 20:20:07.469 | INFO | "huge" : "", | 20260507 20:20:07.469 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:07.469 | INFO | "malloc" : " 14 GB", | 20260507 20:20:07.469 | INFO | "pinned" : " 29 GB" | 20260507 20:20:07.469 | INFO | } | 20260507 20:20:07.469 | INFO | Perf report for the last 5.13888 seconds: | 20260507 20:20:07.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:07.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:20:07.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:20:07.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:07.469 | INFO | Uploaded 36GiB to GPU in 2.56s (14.0625GiB/s) | 20260507 20:20:07.469 | INFO | Downloaded 34GiB to GPU in 2.48425s (13.6862GiB/s) | 20260507 20:20:12.608 | INFO | Analyzing chunk frames = [2319360,2319872) | 20260507 20:20:12.608 | INFO | MemInfo: { | 20260507 20:20:12.608 | INFO | "gpu" : " 17 GB", | 20260507 20:20:12.608 | INFO | "huge" : "", | 20260507 20:20:12.608 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:12.608 | INFO | "malloc" : " 14 GB", | 20260507 20:20:12.608 | INFO | "pinned" : " 29 GB" | 20260507 20:20:12.608 | INFO | } | 20260507 20:20:12.608 | INFO | Perf report for the last 5.13937 seconds: | 20260507 20:20:12.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:12.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.609 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:20:12.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.609 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:20:12.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:12.609 | INFO | Uploaded 36GiB to GPU in 2.54979s (14.1188GiB/s) | 20260507 20:20:12.609 | INFO | Downloaded 34GiB to GPU in 2.47354s (13.7455GiB/s) | 20260507 20:20:17.747 | INFO | Analyzing chunk frames = [2319872,2320384) | 20260507 20:20:17.747 | INFO | MemInfo: { | 20260507 20:20:17.747 | INFO | "gpu" : " 17 GB", | 20260507 20:20:17.747 | INFO | "huge" : "", | 20260507 20:20:17.748 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:17.748 | INFO | "malloc" : " 14 GB", | 20260507 20:20:17.748 | INFO | "pinned" : " 29 GB" | 20260507 20:20:17.748 | INFO | } | 20260507 20:20:17.748 | INFO | Perf report for the last 5.1392 seconds: | 20260507 20:20:17.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:17.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:20:17.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:20:17.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:17.748 | INFO | Uploaded 36GiB to GPU in 2.48035s (14.5141GiB/s) | 20260507 20:20:17.748 | INFO | Downloaded 34GiB to GPU in 2.4029s (14.1496GiB/s) | 20260507 20:20:22.888 | INFO | Analyzing chunk frames = [2320384,2320896) | 20260507 20:20:22.889 | INFO | MemInfo: { | 20260507 20:20:22.889 | INFO | "gpu" : " 17 GB", | 20260507 20:20:22.889 | INFO | "huge" : "", | 20260507 20:20:22.889 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:22.889 | INFO | "malloc" : " 14 GB", | 20260507 20:20:22.889 | INFO | "pinned" : " 29 GB" | 20260507 20:20:22.889 | INFO | } | 20260507 20:20:22.889 | INFO | Perf report for the last 5.1411 seconds: | 20260507 20:20:22.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:22.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 20:20:22.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:20:22.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:22.889 | INFO | Uploaded 36GiB to GPU in 2.38879s (15.0704GiB/s) | 20260507 20:20:22.889 | INFO | Downloaded 34GiB to GPU in 2.48999s (13.6547GiB/s) | 20260507 20:20:28.027 | INFO | Analyzing chunk frames = [2320896,2321408) | 20260507 20:20:28.027 | INFO | MemInfo: { | 20260507 20:20:28.027 | INFO | "gpu" : " 17 GB", | 20260507 20:20:28.027 | INFO | "huge" : "", | 20260507 20:20:28.027 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:28.027 | INFO | "malloc" : " 14 GB", | 20260507 20:20:28.027 | INFO | "pinned" : " 29 GB" | 20260507 20:20:28.027 | INFO | } | 20260507 20:20:28.027 | INFO | Perf report for the last 5.13812 seconds: | 20260507 20:20:28.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:28.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260507 20:20:28.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:20:28.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:28.027 | INFO | Uploaded 36GiB to GPU in 2.50999s (14.3427GiB/s) | 20260507 20:20:28.027 | INFO | Downloaded 34GiB to GPU in 2.41475s (14.0802GiB/s) | 20260507 20:20:33.167 | INFO | Analyzing chunk frames = [2321408,2321920) | 20260507 20:20:33.167 | INFO | MemInfo: { | 20260507 20:20:33.167 | INFO | "gpu" : " 17 GB", | 20260507 20:20:33.167 | INFO | "huge" : "", | 20260507 20:20:33.167 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:33.167 | INFO | "malloc" : " 14 GB", | 20260507 20:20:33.167 | INFO | "pinned" : " 29 GB" | 20260507 20:20:33.167 | INFO | } | 20260507 20:20:33.167 | INFO | Perf report for the last 5.14021 seconds: | 20260507 20:20:33.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:33.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.167 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:20:33.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.167 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 20:20:33.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:33.168 | INFO | Uploaded 36GiB to GPU in 2.54319s (14.1554GiB/s) | 20260507 20:20:33.168 | INFO | Downloaded 34GiB to GPU in 2.43786s (13.9467GiB/s) | 20260507 20:20:38.304 | INFO | Analyzing chunk frames = [2321920,2322432) | 20260507 20:20:38.304 | INFO | MemInfo: { | 20260507 20:20:38.305 | INFO | "gpu" : " 17 GB", | 20260507 20:20:38.305 | INFO | "huge" : "", | 20260507 20:20:38.305 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:38.305 | INFO | "malloc" : " 14 GB", | 20260507 20:20:38.305 | INFO | "pinned" : " 29 GB" | 20260507 20:20:38.305 | INFO | } | 20260507 20:20:38.305 | INFO | Perf report for the last 5.13765 seconds: | 20260507 20:20:38.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:38.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:20:38.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 20:20:38.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:38.305 | INFO | Uploaded 36GiB to GPU in 2.55231s (14.1049GiB/s) | 20260507 20:20:38.305 | INFO | Downloaded 34GiB to GPU in 2.44166s (13.925GiB/s) | 20260507 20:20:38.418 | INFO | Starting a round of Baz IO | 20260507 20:20:41.889 | INFO | Populating RTMetrics report | 20260507 20:20:41.889 | INFO | RTMetrics report populated | 20260507 20:20:41.889 | INFO | Opening tmp RTMetrics json file | 20260507 20:20:41.898 | INFO | Serializing RTMetrics to json | 20260507 20:20:41.921 | INFO | Serialization done, swapping tmp file now... | 20260507 20:20:41.930 | INFO | RTMetrics done | 20260507 20:20:41.996 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.578 seconds | 20260507 20:20:41.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.57% / 0.01% / 55.42% | 20260507 20:20:41.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05487% / 1.98682e-05% / 5.16574e-05% | 20260507 20:20:41.996 | INFO | Baz IO overall: | 20260507 20:20:41.996 | INFO | BytesWritten : 1053798532051 | 20260507 20:20:41.996 | INFO | NumEvents : 510876060200 | 20260507 20:20:43.444 | INFO | Analyzing chunk frames = [2322432,2322944) | 20260507 20:20:43.444 | INFO | MemInfo: { | 20260507 20:20:43.444 | INFO | "gpu" : " 17 GB", | 20260507 20:20:43.444 | INFO | "huge" : "", | 20260507 20:20:43.444 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:43.444 | INFO | "malloc" : " 14 GB", | 20260507 20:20:43.444 | INFO | "pinned" : " 29 GB" | 20260507 20:20:43.444 | INFO | } | 20260507 20:20:43.444 | INFO | Perf report for the last 5.13923 seconds: | 20260507 20:20:43.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:43.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:43.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:43.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:43.444 | INFO | ANALYSIS : 69.0%, 2.9, 104.7ms, 0 | 20260507 20:20:43.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:43.444 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260507 20:20:43.444 | INFO | PRE_HQ : 68.3%, 1.9, 66.2ms, 0 | 20260507 20:20:43.444 | INFO | BAZWRITER : 69.2%, 4.5, 160.1ms, 0 | 20260507 20:20:43.444 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 20:20:43.444 | INFO | Uploaded 36GiB to GPU in 2.58753s (13.9129GiB/s) | 20260507 20:20:43.444 | INFO | Downloaded 38GiB to GPU in 2.69019s (14.1254GiB/s) | 20260507 20:20:48.583 | INFO | Analyzing chunk frames = [2322944,2323456) | 20260507 20:20:48.583 | INFO | MemInfo: { | 20260507 20:20:48.583 | INFO | "gpu" : " 17 GB", | 20260507 20:20:48.583 | INFO | "huge" : "", | 20260507 20:20:48.584 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:48.584 | INFO | "malloc" : " 14 GB", | 20260507 20:20:48.584 | INFO | "pinned" : " 29 GB" | 20260507 20:20:48.584 | INFO | } | 20260507 20:20:48.584 | INFO | Perf report for the last 5.13964 seconds: | 20260507 20:20:48.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:48.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.584 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:20:48.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.584 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:20:48.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:48.585 | INFO | Uploaded 36GiB to GPU in 2.57603s (13.975GiB/s) | 20260507 20:20:48.585 | INFO | Downloaded 34GiB to GPU in 2.4663s (13.7858GiB/s) | 20260507 20:20:53.723 | INFO | Analyzing chunk frames = [2323456,2323968) | 20260507 20:20:53.723 | INFO | MemInfo: { | 20260507 20:20:53.723 | INFO | "gpu" : " 17 GB", | 20260507 20:20:53.723 | INFO | "huge" : "", | 20260507 20:20:53.723 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:53.723 | INFO | "malloc" : " 14 GB", | 20260507 20:20:53.723 | INFO | "pinned" : " 29 GB" | 20260507 20:20:53.723 | INFO | } | 20260507 20:20:53.723 | INFO | Perf report for the last 5.13963 seconds: | 20260507 20:20:53.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:53.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.723 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 20:20:53.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.724 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:20:53.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:53.724 | INFO | Uploaded 36GiB to GPU in 2.5935s (13.8809GiB/s) | 20260507 20:20:53.724 | INFO | Downloaded 34GiB to GPU in 2.46082s (13.8165GiB/s) | 20260507 20:20:58.862 | INFO | Analyzing chunk frames = [2323968,2324480) | 20260507 20:20:58.862 | INFO | MemInfo: { | 20260507 20:20:58.862 | INFO | "gpu" : " 17 GB", | 20260507 20:20:58.862 | INFO | "huge" : "", | 20260507 20:20:58.862 | INFO | "huge_shared" : " 29 GB", | 20260507 20:20:58.862 | INFO | "malloc" : " 14 GB", | 20260507 20:20:58.862 | INFO | "pinned" : " 29 GB" | 20260507 20:20:58.862 | INFO | } | 20260507 20:20:58.863 | INFO | Perf report for the last 5.13912 seconds: | 20260507 20:20:58.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:20:58.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 20:20:58.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:20:58.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:20:58.863 | INFO | Uploaded 36GiB to GPU in 2.50229s (14.3868GiB/s) | 20260507 20:20:58.863 | INFO | Downloaded 34GiB to GPU in 2.38658s (14.2463GiB/s) | 20260507 20:21:04.002 | INFO | Analyzing chunk frames = [2324480,2324992) | 20260507 20:21:04.002 | INFO | MemInfo: { | 20260507 20:21:04.002 | INFO | "gpu" : " 17 GB", | 20260507 20:21:04.002 | INFO | "huge" : "", | 20260507 20:21:04.002 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:04.002 | INFO | "malloc" : " 14 GB", | 20260507 20:21:04.002 | INFO | "pinned" : " 29 GB" | 20260507 20:21:04.002 | INFO | } | 20260507 20:21:04.002 | INFO | Perf report for the last 5.13969 seconds: | 20260507 20:21:04.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:04.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.002 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:21:04.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.002 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:21:04.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:04.003 | INFO | Uploaded 36GiB to GPU in 2.41387s (14.9138GiB/s) | 20260507 20:21:04.003 | INFO | Downloaded 34GiB to GPU in 2.49283s (13.6391GiB/s) | 20260507 20:21:09.142 | INFO | Analyzing chunk frames = [2324992,2325504) | 20260507 20:21:09.142 | INFO | MemInfo: { | 20260507 20:21:09.142 | INFO | "gpu" : " 17 GB", | 20260507 20:21:09.142 | INFO | "huge" : "", | 20260507 20:21:09.142 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:09.142 | INFO | "malloc" : " 14 GB", | 20260507 20:21:09.142 | INFO | "pinned" : " 29 GB" | 20260507 20:21:09.142 | INFO | } | 20260507 20:21:09.142 | INFO | Perf report for the last 5.14022 seconds: | 20260507 20:21:09.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:09.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.143 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 20:21:09.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.143 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:21:09.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:09.143 | INFO | Uploaded 36GiB to GPU in 2.39145s (15.0537GiB/s) | 20260507 20:21:09.143 | INFO | Downloaded 34GiB to GPU in 2.38581s (14.2509GiB/s) | 20260507 20:21:14.281 | INFO | Analyzing chunk frames = [2325504,2326016) | 20260507 20:21:14.281 | INFO | MemInfo: { | 20260507 20:21:14.281 | INFO | "gpu" : " 17 GB", | 20260507 20:21:14.281 | INFO | "huge" : "", | 20260507 20:21:14.281 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:14.281 | INFO | "malloc" : " 14 GB", | 20260507 20:21:14.281 | INFO | "pinned" : " 29 GB" | 20260507 20:21:14.281 | INFO | } | 20260507 20:21:14.281 | INFO | Perf report for the last 5.13846 seconds: | 20260507 20:21:14.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:14.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.281 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 20:21:14.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.281 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:21:14.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:14.282 | INFO | Uploaded 36GiB to GPU in 2.50198s (14.3886GiB/s) | 20260507 20:21:14.282 | INFO | Downloaded 34GiB to GPU in 2.42716s (14.0082GiB/s) | 20260507 20:21:19.420 | INFO | Analyzing chunk frames = [2326016,2326528) | 20260507 20:21:19.420 | INFO | MemInfo: { | 20260507 20:21:19.420 | INFO | "gpu" : " 17 GB", | 20260507 20:21:19.420 | INFO | "huge" : "", | 20260507 20:21:19.420 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:19.420 | INFO | "malloc" : " 14 GB", | 20260507 20:21:19.421 | INFO | "pinned" : " 29 GB" | 20260507 20:21:19.421 | INFO | } | 20260507 20:21:19.421 | INFO | Perf report for the last 5.13989 seconds: | 20260507 20:21:19.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:19.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:21:19.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 20:21:19.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:19.421 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:21:19.421 | INFO | Total mem usage: 16.4 GiB | 20260507 20:21:19.421 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:21:19.421 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 20:21:19.421 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:21:19.421 | INFO | High Water Breakdown: | 20260507 20:21:19.421 | INFO | 59.8% : 8.87 GiB ( 89.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:21:19.422 | INFO | 40.1% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:21:19.422 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:21:19.422 | INFO | ------------------------------------------ | 20260507 20:21:19.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:21:19.422 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:21:19.422 | INFO | Total mem usage: 26.5 GiB | 20260507 20:21:19.422 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:21:19.422 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:21:19.422 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:21:19.422 | INFO | High Water Breakdown: | 20260507 20:21:19.422 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:21:19.422 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:21:19.422 | INFO | Uploaded 36GiB to GPU in 2.52924s (14.2335GiB/s) | 20260507 20:21:19.422 | INFO | Downloaded 34GiB to GPU in 2.43399s (13.9688GiB/s) | 20260507 20:21:19.422 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:21:19.422 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:21:19.422 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:21:19.422 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:21:19.422 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:21:19.422 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:21:19.422 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:21:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:21:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:21:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:21:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:21:19.422 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:21:19.422 | INFO | ------------------------------------------ | 20260507 20:21:19.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:21:19.422 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:21:19.422 | INFO | Total mem usage: 27.8 GiB | 20260507 20:21:19.422 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:21:19.422 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:21:19.422 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:21:19.422 | INFO | High Water Breakdown: | 20260507 20:21:19.422 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:21:19.422 | INFO | ------------------------------------------ | 20260507 20:21:19.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:21:19.422 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:21:19.422 | INFO | Total mem usage: 15.7 GiB | 20260507 20:21:19.422 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:21:19.422 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:21:19.422 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:21:19.422 | INFO | High Water Breakdown: | 20260507 20:21:19.422 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:21:19.422 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:21:19.422 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:21:19.422 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:21:19.422 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:21:19.422 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:21:19.422 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:21:19.422 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:21:19.422 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:21:19.422 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:21:19.422 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:21:19.422 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:21:19.422 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:21:19.422 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:21:19.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:21:19.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:21:19.422 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:21:19.422 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:21:19.422 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:21:19.422 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:21:19.422 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:21:19.422 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:21:19.422 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:21:19.423 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:21:19.423 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:21:19.423 | INFO | ------------------------------------------ | 20260507 20:21:19.423 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:21:19.423 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:21:19.423 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:21:19.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:21:19.423 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.57e+01, 1.42e+01 / 1.54e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.04e-01, 1.28e-02 / 5.33e-03 | 20260507 20:21:19.423 | INFO | PerfReport: 12.10%, Baseline -- 4.02e+00 / 4.47e+00, 6.04e-01 / 2.81e-01 | 20260507 20:21:19.423 | INFO | PerfReport: 3.40%, Histogram -- 1.13e+00 / 1.50e+00, 2.12e-01 / 1.16e-01 | 20260507 20:21:19.423 | INFO | PerfReport: 15.72%, DME -- 4.17e+01 / 4.50e+01, 3.82e+00 / 1.13e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.86e+00 / 1.60e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.49e+00 / 2.79e+00, 4.39e-01 / 2.58e-01 | 20260507 20:21:19.423 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.37e+00, 3.02e-01 / 1.70e-01 | 20260507 20:21:19.423 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 3.98e-01, 6.03e-02 / 3.10e-02 | 20260507 20:21:19.423 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:21:19.423 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:21:19.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:21:19.423 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 24.52%, Upload -- 2.62e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 24.14%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 31.09%, Compute -- 3.32e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 2.09%, LockUpload -- 2.23e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 2.20%, LockDownload -- 2.35e+00 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:21:19.423 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 20:21:19.533 | INFO | Starting a round of Baz IO | 20260507 20:21:23.026 | INFO | Populating RTMetrics report | 20260507 20:21:23.027 | INFO | RTMetrics report populated | 20260507 20:21:23.027 | INFO | Opening tmp RTMetrics json file | 20260507 20:21:23.036 | INFO | Serializing RTMetrics to json | 20260507 20:21:23.055 | INFO | Serialization done, swapping tmp file now... | 20260507 20:21:23.055 | INFO | RTMetrics done | 20260507 20:21:23.114 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.581 seconds | 20260507 20:21:23.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.56% / 0.01% / 55.43% | 20260507 20:21:23.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04064% / 1.19209e-05% / 3.57628e-05% | 20260507 20:21:23.114 | INFO | Baz IO overall: | 20260507 20:21:23.114 | INFO | BytesWritten : 1055707208292 | 20260507 20:21:23.114 | INFO | NumEvents : 511799834381 | 20260507 20:21:24.559 | INFO | Analyzing chunk frames = [2326528,2327040) | 20260507 20:21:24.559 | INFO | MemInfo: { | 20260507 20:21:24.559 | INFO | "gpu" : " 17 GB", | 20260507 20:21:24.559 | INFO | "huge" : "", | 20260507 20:21:24.559 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:24.559 | INFO | "malloc" : " 14 GB", | 20260507 20:21:24.559 | INFO | "pinned" : " 29 GB" | 20260507 20:21:24.559 | INFO | } | 20260507 20:21:24.559 | INFO | Perf report for the last 5.13856 seconds: | 20260507 20:21:24.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:24.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:24.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:24.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:24.560 | INFO | ANALYSIS : 69.2%, 2.9, 104.8ms, 0 | 20260507 20:21:24.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:24.560 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 20:21:24.560 | INFO | PRE_HQ : 68.3%, 1.8, 65.3ms, 0 | 20260507 20:21:24.560 | INFO | BAZWRITER : 69.3%, 4.3, 153.8ms, 0 | 20260507 20:21:24.560 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 20:21:24.560 | INFO | Uploaded 36GiB to GPU in 2.60006s (13.8458GiB/s) | 20260507 20:21:24.560 | INFO | Downloaded 38GiB to GPU in 2.72216s (13.9595GiB/s) | 20260507 20:21:29.698 | INFO | Analyzing chunk frames = [2327040,2327552) | 20260507 20:21:29.698 | INFO | MemInfo: { | 20260507 20:21:29.698 | INFO | "gpu" : " 17 GB", | 20260507 20:21:29.698 | INFO | "huge" : "", | 20260507 20:21:29.698 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:29.698 | INFO | "malloc" : " 14 GB", | 20260507 20:21:29.698 | INFO | "pinned" : " 29 GB" | 20260507 20:21:29.698 | INFO | } | 20260507 20:21:29.698 | INFO | Perf report for the last 5.13855 seconds: | 20260507 20:21:29.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:29.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 20:21:29.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:21:29.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:29.698 | INFO | Uploaded 36GiB to GPU in 2.53401s (14.2067GiB/s) | 20260507 20:21:29.698 | INFO | Downloaded 34GiB to GPU in 2.49773s (13.6124GiB/s) | 20260507 20:21:34.838 | INFO | Analyzing chunk frames = [2327552,2328064) | 20260507 20:21:34.838 | INFO | MemInfo: { | 20260507 20:21:34.838 | INFO | "gpu" : " 17 GB", | 20260507 20:21:34.838 | INFO | "huge" : "", | 20260507 20:21:34.838 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:34.838 | INFO | "malloc" : " 14 GB", | 20260507 20:21:34.838 | INFO | "pinned" : " 29 GB" | 20260507 20:21:34.838 | INFO | } | 20260507 20:21:34.838 | INFO | Perf report for the last 5.14051 seconds: | 20260507 20:21:34.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:34.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.839 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:21:34.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.839 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:21:34.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:34.839 | INFO | Uploaded 36GiB to GPU in 2.56248s (14.0489GiB/s) | 20260507 20:21:34.839 | INFO | Downloaded 34GiB to GPU in 2.46953s (13.7678GiB/s) | 20260507 20:21:39.977 | INFO | Analyzing chunk frames = [2328064,2328576) | 20260507 20:21:39.977 | INFO | MemInfo: { | 20260507 20:21:39.977 | INFO | "gpu" : " 17 GB", | 20260507 20:21:39.978 | INFO | "huge" : "", | 20260507 20:21:39.978 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:39.978 | INFO | "malloc" : " 14 GB", | 20260507 20:21:39.978 | INFO | "pinned" : " 29 GB" | 20260507 20:21:39.978 | INFO | } | 20260507 20:21:39.978 | INFO | Perf report for the last 5.13924 seconds: | 20260507 20:21:39.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:39.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260507 20:21:39.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:21:39.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:39.978 | INFO | Uploaded 36GiB to GPU in 2.57197s (13.997GiB/s) | 20260507 20:21:39.978 | INFO | Downloaded 34GiB to GPU in 2.47689s (13.7269GiB/s) | 20260507 20:21:45.116 | INFO | Analyzing chunk frames = [2328576,2329088) | 20260507 20:21:45.117 | INFO | MemInfo: { | 20260507 20:21:45.117 | INFO | "gpu" : " 17 GB", | 20260507 20:21:45.117 | INFO | "huge" : "", | 20260507 20:21:45.117 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:45.117 | INFO | "malloc" : " 14 GB", | 20260507 20:21:45.117 | INFO | "pinned" : " 29 GB" | 20260507 20:21:45.117 | INFO | } | 20260507 20:21:45.117 | INFO | Perf report for the last 5.13899 seconds: | 20260507 20:21:45.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:45.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 20:21:45.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:21:45.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:45.117 | INFO | Uploaded 36GiB to GPU in 2.36682s (15.2103GiB/s) | 20260507 20:21:45.117 | INFO | Downloaded 34GiB to GPU in 2.49286s (13.6389GiB/s) | 20260507 20:21:50.255 | INFO | Analyzing chunk frames = [2329088,2329600) | 20260507 20:21:50.255 | INFO | MemInfo: { | 20260507 20:21:50.255 | INFO | "gpu" : " 17 GB", | 20260507 20:21:50.255 | INFO | "huge" : "", | 20260507 20:21:50.255 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:50.255 | INFO | "malloc" : " 14 GB", | 20260507 20:21:50.255 | INFO | "pinned" : " 29 GB" | 20260507 20:21:50.255 | INFO | } | 20260507 20:21:50.255 | INFO | Perf report for the last 5.13872 seconds: | 20260507 20:21:50.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:50.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.256 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 20:21:50.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.256 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:21:50.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:50.256 | INFO | Uploaded 36GiB to GPU in 2.51767s (14.2989GiB/s) | 20260507 20:21:50.256 | INFO | Downloaded 34GiB to GPU in 2.41934s (14.0534GiB/s) | 20260507 20:21:55.396 | INFO | Analyzing chunk frames = [2329600,2330112) | 20260507 20:21:55.396 | INFO | MemInfo: { | 20260507 20:21:55.396 | INFO | "gpu" : " 17 GB", | 20260507 20:21:55.396 | INFO | "huge" : "", | 20260507 20:21:55.396 | INFO | "huge_shared" : " 29 GB", | 20260507 20:21:55.396 | INFO | "malloc" : " 14 GB", | 20260507 20:21:55.396 | INFO | "pinned" : " 29 GB" | 20260507 20:21:55.396 | INFO | } | 20260507 20:21:55.396 | INFO | Perf report for the last 5.14062 seconds: | 20260507 20:21:55.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:21:55.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.396 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:21:55.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.396 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 20:21:55.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:21:55.397 | INFO | Uploaded 36GiB to GPU in 2.55218s (14.1056GiB/s) | 20260507 20:21:55.397 | INFO | Downloaded 34GiB to GPU in 2.43403s (13.9686GiB/s) | 20260507 20:22:00.534 | INFO | Analyzing chunk frames = [2330112,2330624) | 20260507 20:22:00.534 | INFO | MemInfo: { | 20260507 20:22:00.534 | INFO | "gpu" : " 17 GB", | 20260507 20:22:00.534 | INFO | "huge" : "", | 20260507 20:22:00.534 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:00.534 | INFO | "malloc" : " 14 GB", | 20260507 20:22:00.534 | INFO | "pinned" : " 29 GB" | 20260507 20:22:00.534 | INFO | } | 20260507 20:22:00.534 | INFO | Perf report for the last 5.13834 seconds: | 20260507 20:22:00.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:00.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.535 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:22:00.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.535 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 20:22:00.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:00.535 | INFO | Uploaded 36GiB to GPU in 2.53215s (14.2171GiB/s) | 20260507 20:22:00.535 | INFO | Downloaded 34GiB to GPU in 2.42669s (14.0109GiB/s) | 20260507 20:22:00.631 | INFO | Starting a round of Baz IO | 20260507 20:22:04.114 | INFO | Populating RTMetrics report | 20260507 20:22:04.114 | INFO | RTMetrics report populated | 20260507 20:22:04.114 | INFO | Opening tmp RTMetrics json file | 20260507 20:22:04.149 | INFO | Serializing RTMetrics to json | 20260507 20:22:04.201 | INFO | Serialization done, swapping tmp file now... | 20260507 20:22:04.209 | INFO | RTMetrics done | 20260507 20:22:04.247 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.615 seconds | 20260507 20:22:04.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.55% / 0.01% / 55.44% | 20260507 20:22:04.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04105% / 7.94729e-06% / 3.17891e-05% | 20260507 20:22:04.247 | INFO | Baz IO overall: | 20260507 20:22:04.247 | INFO | BytesWritten : 1057616842630 | 20260507 20:22:04.247 | INFO | NumEvents : 512724297786 | 20260507 20:22:05.673 | INFO | Analyzing chunk frames = [2330624,2331136) | 20260507 20:22:05.673 | INFO | MemInfo: { | 20260507 20:22:05.673 | INFO | "gpu" : " 17 GB", | 20260507 20:22:05.673 | INFO | "huge" : "", | 20260507 20:22:05.673 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:05.673 | INFO | "malloc" : " 14 GB", | 20260507 20:22:05.673 | INFO | "pinned" : " 29 GB" | 20260507 20:22:05.673 | INFO | } | 20260507 20:22:05.673 | INFO | Perf report for the last 5.13877 seconds: | 20260507 20:22:05.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:05.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:05.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:05.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:05.673 | INFO | ANALYSIS : 68.9%, 3.0, 106.6ms, 0 | 20260507 20:22:05.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:05.673 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 20:22:05.673 | INFO | PRE_HQ : 66.0%, 1.7, 58.2ms, 0 | 20260507 20:22:05.673 | INFO | BAZWRITER : 70.0%, 3.9, 141.5ms, 0 | 20260507 20:22:05.673 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:22:05.674 | INFO | Uploaded 36GiB to GPU in 2.55484s (14.0909GiB/s) | 20260507 20:22:05.674 | INFO | Downloaded 38GiB to GPU in 2.6985s (14.0819GiB/s) | 20260507 20:22:10.813 | INFO | Analyzing chunk frames = [2331136,2331648) | 20260507 20:22:10.813 | INFO | MemInfo: { | 20260507 20:22:10.813 | INFO | "gpu" : " 17 GB", | 20260507 20:22:10.813 | INFO | "huge" : "", | 20260507 20:22:10.813 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:10.813 | INFO | "malloc" : " 14 GB", | 20260507 20:22:10.813 | INFO | "pinned" : " 29 GB" | 20260507 20:22:10.813 | INFO | } | 20260507 20:22:10.813 | INFO | Perf report for the last 5.13979 seconds: | 20260507 20:22:10.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:10.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:22:10.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:22:10.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:10.813 | INFO | Uploaded 36GiB to GPU in 2.55561s (14.0867GiB/s) | 20260507 20:22:10.814 | INFO | Downloaded 34GiB to GPU in 2.50884s (13.5521GiB/s) | 20260507 20:22:15.953 | INFO | Analyzing chunk frames = [2331648,2332160) | 20260507 20:22:15.953 | INFO | MemInfo: { | 20260507 20:22:15.953 | INFO | "gpu" : " 17 GB", | 20260507 20:22:15.953 | INFO | "huge" : "", | 20260507 20:22:15.953 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:15.953 | INFO | "malloc" : " 14 GB", | 20260507 20:22:15.953 | INFO | "pinned" : " 29 GB" | 20260507 20:22:15.953 | INFO | } | 20260507 20:22:15.953 | INFO | Perf report for the last 5.13954 seconds: | 20260507 20:22:15.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:15.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:22:15.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:22:15.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:15.953 | INFO | Uploaded 36GiB to GPU in 2.53018s (14.2282GiB/s) | 20260507 20:22:15.953 | INFO | Downloaded 34GiB to GPU in 2.47824s (13.7194GiB/s) | 20260507 20:22:21.093 | INFO | Analyzing chunk frames = [2332160,2332672) | 20260507 20:22:21.093 | INFO | MemInfo: { | 20260507 20:22:21.093 | INFO | "gpu" : " 17 GB", | 20260507 20:22:21.093 | INFO | "huge" : "", | 20260507 20:22:21.093 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:21.093 | INFO | "malloc" : " 14 GB", | 20260507 20:22:21.093 | INFO | "pinned" : " 29 GB" | 20260507 20:22:21.093 | INFO | } | 20260507 20:22:21.093 | INFO | Perf report for the last 5.1404 seconds: | 20260507 20:22:21.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:21.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:22:21.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:22:21.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:21.093 | INFO | Uploaded 36GiB to GPU in 2.49166s (14.4482GiB/s) | 20260507 20:22:21.093 | INFO | Downloaded 34GiB to GPU in 2.40819s (14.1185GiB/s) | 20260507 20:22:26.232 | INFO | Analyzing chunk frames = [2332672,2333184) | 20260507 20:22:26.232 | INFO | MemInfo: { | 20260507 20:22:26.232 | INFO | "gpu" : " 17 GB", | 20260507 20:22:26.232 | INFO | "huge" : "", | 20260507 20:22:26.232 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:26.232 | INFO | "malloc" : " 14 GB", | 20260507 20:22:26.232 | INFO | "pinned" : " 29 GB" | 20260507 20:22:26.232 | INFO | } | 20260507 20:22:26.232 | INFO | Perf report for the last 5.13922 seconds: | 20260507 20:22:26.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:26.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 20:22:26.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:22:26.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:26.232 | INFO | Uploaded 36GiB to GPU in 2.44655s (14.7146GiB/s) | 20260507 20:22:26.233 | INFO | Downloaded 34GiB to GPU in 2.5059s (13.568GiB/s) | 20260507 20:22:31.371 | INFO | Analyzing chunk frames = [2333184,2333696) | 20260507 20:22:31.371 | INFO | MemInfo: { | 20260507 20:22:31.371 | INFO | "gpu" : " 17 GB", | 20260507 20:22:31.371 | INFO | "huge" : "", | 20260507 20:22:31.371 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:31.371 | INFO | "malloc" : " 14 GB", | 20260507 20:22:31.371 | INFO | "pinned" : " 29 GB" | 20260507 20:22:31.371 | INFO | } | 20260507 20:22:31.371 | INFO | Perf report for the last 5.13845 seconds: | 20260507 20:22:31.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:31.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 20:22:31.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:22:31.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:31.371 | INFO | Uploaded 36GiB to GPU in 2.44933s (14.6979GiB/s) | 20260507 20:22:31.371 | INFO | Downloaded 34GiB to GPU in 2.43589s (13.958GiB/s) | 20260507 20:22:36.509 | INFO | Analyzing chunk frames = [2333696,2334208) | 20260507 20:22:36.509 | INFO | MemInfo: { | 20260507 20:22:36.509 | INFO | "gpu" : " 17 GB", | 20260507 20:22:36.509 | INFO | "huge" : "", | 20260507 20:22:36.509 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:36.509 | INFO | "malloc" : " 14 GB", | 20260507 20:22:36.509 | INFO | "pinned" : " 29 GB" | 20260507 20:22:36.510 | INFO | } | 20260507 20:22:36.510 | INFO | Perf report for the last 5.13881 seconds: | 20260507 20:22:36.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:36.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 20:22:36.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 20:22:36.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:36.510 | INFO | Uploaded 36GiB to GPU in 2.49034s (14.4558GiB/s) | 20260507 20:22:36.510 | INFO | Downloaded 34GiB to GPU in 2.47096s (13.7598GiB/s) | 20260507 20:22:41.648 | INFO | Analyzing chunk frames = [2334208,2334720) | 20260507 20:22:41.648 | INFO | MemInfo: { | 20260507 20:22:41.648 | INFO | "gpu" : " 17 GB", | 20260507 20:22:41.648 | INFO | "huge" : "", | 20260507 20:22:41.648 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:41.648 | INFO | "malloc" : " 14 GB", | 20260507 20:22:41.648 | INFO | "pinned" : " 29 GB" | 20260507 20:22:41.648 | INFO | } | 20260507 20:22:41.648 | INFO | Perf report for the last 5.13866 seconds: | 20260507 20:22:41.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:41.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 20:22:41.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 20:22:41.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:41.648 | INFO | Uploaded 36GiB to GPU in 2.46081s (14.6293GiB/s) | 20260507 20:22:41.649 | INFO | Downloaded 34GiB to GPU in 2.44603s (13.9001GiB/s) | 20260507 20:22:41.649 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:22:41.649 | INFO | Total mem usage: 16.4 GiB | 20260507 20:22:41.649 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:22:41.649 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.04%) | 20260507 20:22:41.649 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:22:41.649 | INFO | High Water Breakdown: | 20260507 20:22:41.649 | INFO | 60.1% : 9.05 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:22:41.649 | INFO | 39.7% : 5.98 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:22:41.649 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:22:41.649 | INFO | ------------------------------------------ | 20260507 20:22:41.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:22:41.649 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:22:41.649 | INFO | Total mem usage: 26.5 GiB | 20260507 20:22:41.649 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:22:41.649 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 20:22:41.649 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:22:41.649 | INFO | High Water Breakdown: | 20260507 20:22:41.649 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:22:41.649 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:22:41.649 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:22:41.649 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:22:41.649 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:22:41.649 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:22:41.649 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:22:41.649 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:22:41.649 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:22:41.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:22:41.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:22:41.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:22:41.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:22:41.649 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:22:41.649 | INFO | ------------------------------------------ | 20260507 20:22:41.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:22:41.649 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:22:41.649 | INFO | Total mem usage: 27.8 GiB | 20260507 20:22:41.649 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:22:41.649 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:22:41.649 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:22:41.649 | INFO | High Water Breakdown: | 20260507 20:22:41.649 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:22:41.649 | INFO | ------------------------------------------ | 20260507 20:22:41.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:22:41.649 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:22:41.649 | INFO | Total mem usage: 15.7 GiB | 20260507 20:22:41.649 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:22:41.649 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:22:41.649 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:22:41.649 | INFO | High Water Breakdown: | 20260507 20:22:41.649 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:22:41.649 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:22:41.649 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:22:41.649 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:22:41.649 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:22:41.649 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:22:41.649 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:22:41.649 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:22:41.649 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:22:41.649 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:22:41.649 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:22:41.649 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:22:41.649 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:22:41.649 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:22:41.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:22:41.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:22:41.649 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:22:41.649 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:22:41.649 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:22:41.649 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:22:41.649 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:22:41.649 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:22:41.649 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:22:41.649 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:22:41.649 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:22:41.649 | INFO | ------------------------------------------ | 20260507 20:22:41.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:22:41.649 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:22:41.649 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:22:41.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:22:41.649 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.40e+01 / 1.83e+00 | 20260507 20:22:41.649 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.12e-02 / 5.21e-03 | 20260507 20:22:41.649 | INFO | PerfReport: 12.05%, Baseline -- 3.99e+00 / 4.50e+00, 5.95e-01 / 2.36e-01 | 20260507 20:22:41.649 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.51e+00, 2.10e-01 / 1.09e-01 | 20260507 20:22:41.649 | INFO | PerfReport: 15.70%, DME -- 4.16e+01 / 4.47e+01, 3.78e+00 / 1.32e+00 | 20260507 20:22:41.649 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 2.03e+00 | 20260507 20:22:41.649 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.50e+00 / 2.82e+00, 4.43e-01 / 2.14e-01 | 20260507 20:22:41.649 | INFO | PerfReport: 3.61%, Metrics -- 1.19e+00 / 1.38e+00, 3.05e-01 / 1.78e-01 | 20260507 20:22:41.649 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.71e-01 / 4.05e-01, 6.04e-02 / 3.97e-02 | 20260507 20:22:41.649 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:22:41.650 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:22:41.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:22:41.650 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 24.49%, Upload -- 2.62e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 24.30%, Download -- 2.60e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 30.99%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 1.91%, LockUpload -- 2.04e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 2.46%, LockDownload -- 2.63e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 15.84%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 20:22:41.650 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 20:22:41.746 | INFO | Starting a round of Baz IO | 20260507 20:22:45.232 | INFO | Populating RTMetrics report | 20260507 20:22:45.232 | INFO | RTMetrics report populated | 20260507 20:22:45.232 | INFO | Opening tmp RTMetrics json file | 20260507 20:22:45.274 | INFO | Serializing RTMetrics to json | 20260507 20:22:45.321 | INFO | Serialization done, swapping tmp file now... | 20260507 20:22:45.324 | INFO | RTMetrics done | 20260507 20:22:45.347 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.601 seconds | 20260507 20:22:45.348 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.54% / 0.01% / 55.45% | 20260507 20:22:45.348 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04593% / 1.98682e-05% / 3.57628e-05% | 20260507 20:22:45.348 | INFO | Baz IO overall: | 20260507 20:22:45.348 | INFO | BytesWritten : 1059529227698 | 20260507 20:22:45.348 | INFO | NumEvents : 513650531783 | 20260507 20:22:46.788 | INFO | Analyzing chunk frames = [2334720,2335232) | 20260507 20:22:46.788 | INFO | MemInfo: { | 20260507 20:22:46.788 | INFO | "gpu" : " 17 GB", | 20260507 20:22:46.788 | INFO | "huge" : "", | 20260507 20:22:46.788 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:46.788 | INFO | "malloc" : " 14 GB", | 20260507 20:22:46.788 | INFO | "pinned" : " 29 GB" | 20260507 20:22:46.788 | INFO | } | 20260507 20:22:46.788 | INFO | Perf report for the last 5.13956 seconds: | 20260507 20:22:46.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:46.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:46.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:46.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:46.788 | INFO | ANALYSIS : 68.7%, 3.0, 107.0ms, 0 | 20260507 20:22:46.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:46.788 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 20:22:46.788 | INFO | PRE_HQ : 65.8%, 1.7, 58.2ms, 0 | 20260507 20:22:46.788 | INFO | BAZWRITER : 69.7%, 3.8, 137.6ms, 0 | 20260507 20:22:46.788 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 20:22:46.788 | INFO | Uploaded 36GiB to GPU in 2.50021s (14.3988GiB/s) | 20260507 20:22:46.788 | INFO | Downloaded 38GiB to GPU in 2.6791s (14.1839GiB/s) | 20260507 20:22:51.928 | INFO | Analyzing chunk frames = [2335232,2335744) | 20260507 20:22:51.928 | INFO | MemInfo: { | 20260507 20:22:51.928 | INFO | "gpu" : " 17 GB", | 20260507 20:22:51.928 | INFO | "huge" : "", | 20260507 20:22:51.928 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:51.928 | INFO | "malloc" : " 14 GB", | 20260507 20:22:51.928 | INFO | "pinned" : " 29 GB" | 20260507 20:22:51.928 | INFO | } | 20260507 20:22:51.928 | INFO | Perf report for the last 5.14023 seconds: | 20260507 20:22:51.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:51.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:22:51.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:22:51.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:51.928 | INFO | Uploaded 36GiB to GPU in 2.47453s (14.5482GiB/s) | 20260507 20:22:51.928 | INFO | Downloaded 34GiB to GPU in 2.47591s (13.7323GiB/s) | 20260507 20:22:57.067 | INFO | Analyzing chunk frames = [2335744,2336256) | 20260507 20:22:57.067 | INFO | MemInfo: { | 20260507 20:22:57.067 | INFO | "gpu" : " 17 GB", | 20260507 20:22:57.068 | INFO | "huge" : "", | 20260507 20:22:57.068 | INFO | "huge_shared" : " 29 GB", | 20260507 20:22:57.068 | INFO | "malloc" : " 14 GB", | 20260507 20:22:57.068 | INFO | "pinned" : " 29 GB" | 20260507 20:22:57.068 | INFO | } | 20260507 20:22:57.068 | INFO | Perf report for the last 5.13963 seconds: | 20260507 20:22:57.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:22:57.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 20:22:57.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:22:57.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:22:57.068 | INFO | Uploaded 36GiB to GPU in 2.57279s (13.9926GiB/s) | 20260507 20:22:57.068 | INFO | Downloaded 34GiB to GPU in 2.47334s (13.7466GiB/s) | 20260507 20:23:02.206 | INFO | Analyzing chunk frames = [2336256,2336768) | 20260507 20:23:02.206 | INFO | MemInfo: { | 20260507 20:23:02.206 | INFO | "gpu" : " 17 GB", | 20260507 20:23:02.206 | INFO | "huge" : "", | 20260507 20:23:02.206 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:02.206 | INFO | "malloc" : " 14 GB", | 20260507 20:23:02.206 | INFO | "pinned" : " 29 GB" | 20260507 20:23:02.206 | INFO | } | 20260507 20:23:02.207 | INFO | Perf report for the last 5.13901 seconds: | 20260507 20:23:02.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:02.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:23:02.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:23:02.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:02.207 | INFO | Uploaded 36GiB to GPU in 2.4943s (14.4329GiB/s) | 20260507 20:23:02.207 | INFO | Downloaded 34GiB to GPU in 2.40423s (14.1418GiB/s) | 20260507 20:23:07.346 | INFO | Analyzing chunk frames = [2336768,2337280) | 20260507 20:23:07.346 | INFO | MemInfo: { | 20260507 20:23:07.346 | INFO | "gpu" : " 17 GB", | 20260507 20:23:07.346 | INFO | "huge" : "", | 20260507 20:23:07.346 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:07.346 | INFO | "malloc" : " 14 GB", | 20260507 20:23:07.346 | INFO | "pinned" : " 29 GB" | 20260507 20:23:07.346 | INFO | } | 20260507 20:23:07.346 | INFO | Perf report for the last 5.13932 seconds: | 20260507 20:23:07.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:07.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.346 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 20:23:07.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.346 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:23:07.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:07.347 | INFO | Uploaded 36GiB to GPU in 2.43242s (14.8001GiB/s) | 20260507 20:23:07.347 | INFO | Downloaded 34GiB to GPU in 2.50138s (13.5925GiB/s) | 20260507 20:23:12.486 | INFO | Analyzing chunk frames = [2337280,2337792) | 20260507 20:23:12.486 | INFO | MemInfo: { | 20260507 20:23:12.486 | INFO | "gpu" : " 17 GB", | 20260507 20:23:12.486 | INFO | "huge" : "", | 20260507 20:23:12.486 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:12.486 | INFO | "malloc" : " 14 GB", | 20260507 20:23:12.486 | INFO | "pinned" : " 29 GB" | 20260507 20:23:12.486 | INFO | } | 20260507 20:23:12.486 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:23:12.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:12.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:23:12.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 20:23:12.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:12.486 | INFO | Uploaded 36GiB to GPU in 2.50627s (14.364GiB/s) | 20260507 20:23:12.486 | INFO | Downloaded 34GiB to GPU in 2.41204s (14.096GiB/s) | 20260507 20:23:17.624 | INFO | Analyzing chunk frames = [2337792,2338304) | 20260507 20:23:17.624 | INFO | MemInfo: { | 20260507 20:23:17.624 | INFO | "gpu" : " 17 GB", | 20260507 20:23:17.625 | INFO | "huge" : "", | 20260507 20:23:17.625 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:17.625 | INFO | "malloc" : " 14 GB", | 20260507 20:23:17.625 | INFO | "pinned" : " 29 GB" | 20260507 20:23:17.625 | INFO | } | 20260507 20:23:17.625 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:23:17.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:17.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:23:17.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 20:23:17.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:17.625 | INFO | Uploaded 36GiB to GPU in 2.57279s (13.9926GiB/s) | 20260507 20:23:17.625 | INFO | Downloaded 34GiB to GPU in 2.44843s (13.8865GiB/s) | 20260507 20:23:22.764 | INFO | Analyzing chunk frames = [2338304,2338816) | 20260507 20:23:22.764 | INFO | MemInfo: { | 20260507 20:23:22.764 | INFO | "gpu" : " 17 GB", | 20260507 20:23:22.764 | INFO | "huge" : "", | 20260507 20:23:22.764 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:22.764 | INFO | "malloc" : " 14 GB", | 20260507 20:23:22.764 | INFO | "pinned" : " 29 GB" | 20260507 20:23:22.764 | INFO | } | 20260507 20:23:22.764 | INFO | Perf report for the last 5.13946 seconds: | 20260507 20:23:22.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:22.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.764 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:23:22.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.764 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 20:23:22.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:22.765 | INFO | Uploaded 36GiB to GPU in 2.53089s (14.2242GiB/s) | 20260507 20:23:22.765 | INFO | Downloaded 34GiB to GPU in 2.42797s (14.0035GiB/s) | 20260507 20:23:22.861 | INFO | Starting a round of Baz IO | 20260507 20:23:26.330 | INFO | Populating RTMetrics report | 20260507 20:23:26.331 | INFO | RTMetrics report populated | 20260507 20:23:26.331 | INFO | Opening tmp RTMetrics json file | 20260507 20:23:26.359 | INFO | Serializing RTMetrics to json | 20260507 20:23:26.370 | INFO | Serialization done, swapping tmp file now... | 20260507 20:23:26.390 | INFO | RTMetrics done | 20260507 20:23:26.435 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.573 seconds | 20260507 20:23:26.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.53% / 0.01% / 55.46% | 20260507 20:23:26.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04233% / 7.94729e-06% / 8.34465e-05% | 20260507 20:23:26.435 | INFO | Baz IO overall: | 20260507 20:23:26.435 | INFO | BytesWritten : 1061444087821 | 20260507 20:23:26.435 | INFO | NumEvents : 514577362799 | 20260507 20:23:27.903 | INFO | Analyzing chunk frames = [2338816,2339328) | 20260507 20:23:27.903 | INFO | MemInfo: { | 20260507 20:23:27.903 | INFO | "gpu" : " 17 GB", | 20260507 20:23:27.903 | INFO | "huge" : "", | 20260507 20:23:27.903 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:27.903 | INFO | "malloc" : " 14 GB", | 20260507 20:23:27.903 | INFO | "pinned" : " 29 GB" | 20260507 20:23:27.903 | INFO | } | 20260507 20:23:27.904 | INFO | Perf report for the last 5.13929 seconds: | 20260507 20:23:27.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:27.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:27.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:27.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:27.904 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 20:23:27.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:27.904 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 20:23:27.904 | INFO | PRE_HQ : 66.1%, 1.7, 58.4ms, 0 | 20260507 20:23:27.904 | INFO | BAZWRITER : 69.1%, 4.1, 146.3ms, 0 | 20260507 20:23:27.904 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 20:23:27.904 | INFO | Uploaded 36GiB to GPU in 2.52332s (14.2669GiB/s) | 20260507 20:23:27.904 | INFO | Downloaded 38GiB to GPU in 2.69759s (14.0867GiB/s) | 20260507 20:23:33.044 | INFO | Analyzing chunk frames = [2339328,2339840) | 20260507 20:23:33.044 | INFO | MemInfo: { | 20260507 20:23:33.044 | INFO | "gpu" : " 17 GB", | 20260507 20:23:33.044 | INFO | "huge" : "", | 20260507 20:23:33.044 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:33.044 | INFO | "malloc" : " 14 GB", | 20260507 20:23:33.044 | INFO | "pinned" : " 29 GB" | 20260507 20:23:33.044 | INFO | } | 20260507 20:23:33.044 | INFO | Perf report for the last 5.14048 seconds: | 20260507 20:23:33.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:33.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 20:23:33.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:23:33.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:33.044 | INFO | Uploaded 36GiB to GPU in 2.59105s (13.894GiB/s) | 20260507 20:23:33.045 | INFO | Downloaded 34GiB to GPU in 2.46972s (13.7668GiB/s) | 20260507 20:23:38.182 | INFO | Analyzing chunk frames = [2339840,2340352) | 20260507 20:23:38.182 | INFO | MemInfo: { | 20260507 20:23:38.182 | INFO | "gpu" : " 17 GB", | 20260507 20:23:38.182 | INFO | "huge" : "", | 20260507 20:23:38.182 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:38.182 | INFO | "malloc" : " 14 GB", | 20260507 20:23:38.182 | INFO | "pinned" : " 29 GB" | 20260507 20:23:38.182 | INFO | } | 20260507 20:23:38.182 | INFO | Perf report for the last 5.13771 seconds: | 20260507 20:23:38.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:38.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260507 20:23:38.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:23:38.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:38.182 | INFO | Uploaded 36GiB to GPU in 2.50147s (14.3915GiB/s) | 20260507 20:23:38.182 | INFO | Downloaded 34GiB to GPU in 2.47613s (13.7311GiB/s) | 20260507 20:23:43.321 | INFO | Analyzing chunk frames = [2340352,2340864) | 20260507 20:23:43.321 | INFO | MemInfo: { | 20260507 20:23:43.321 | INFO | "gpu" : " 17 GB", | 20260507 20:23:43.321 | INFO | "huge" : "", | 20260507 20:23:43.321 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:43.321 | INFO | "malloc" : " 14 GB", | 20260507 20:23:43.321 | INFO | "pinned" : " 29 GB" | 20260507 20:23:43.321 | INFO | } | 20260507 20:23:43.321 | INFO | Perf report for the last 5.13915 seconds: | 20260507 20:23:43.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:43.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 20:23:43.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:23:43.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:43.321 | INFO | Uploaded 36GiB to GPU in 2.51199s (14.3312GiB/s) | 20260507 20:23:43.321 | INFO | Downloaded 34GiB to GPU in 2.39895s (14.1729GiB/s) | 20260507 20:23:48.460 | INFO | Analyzing chunk frames = [2340864,2341376) | 20260507 20:23:48.460 | INFO | MemInfo: { | 20260507 20:23:48.460 | INFO | "gpu" : " 17 GB", | 20260507 20:23:48.460 | INFO | "huge" : "", | 20260507 20:23:48.460 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:48.460 | INFO | "malloc" : " 14 GB", | 20260507 20:23:48.460 | INFO | "pinned" : " 29 GB" | 20260507 20:23:48.460 | INFO | } | 20260507 20:23:48.460 | INFO | Perf report for the last 5.13938 seconds: | 20260507 20:23:48.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:48.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.460 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:23:48.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.460 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:23:48.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:48.461 | INFO | Uploaded 36GiB to GPU in 2.40268s (14.9832GiB/s) | 20260507 20:23:48.461 | INFO | Downloaded 34GiB to GPU in 2.49309s (13.6377GiB/s) | 20260507 20:23:53.599 | INFO | Analyzing chunk frames = [2341376,2341888) | 20260507 20:23:53.599 | INFO | MemInfo: { | 20260507 20:23:53.599 | INFO | "gpu" : " 17 GB", | 20260507 20:23:53.599 | INFO | "huge" : "", | 20260507 20:23:53.599 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:53.599 | INFO | "malloc" : " 14 GB", | 20260507 20:23:53.599 | INFO | "pinned" : " 29 GB" | 20260507 20:23:53.600 | INFO | } | 20260507 20:23:53.600 | INFO | Perf report for the last 5.13916 seconds: | 20260507 20:23:53.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:53.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:23:53.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 20:23:53.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:53.600 | INFO | Uploaded 36GiB to GPU in 2.48623s (14.4798GiB/s) | 20260507 20:23:53.600 | INFO | Downloaded 34GiB to GPU in 2.42344s (14.0296GiB/s) | 20260507 20:23:58.739 | INFO | Analyzing chunk frames = [2341888,2342400) | 20260507 20:23:58.739 | INFO | MemInfo: { | 20260507 20:23:58.739 | INFO | "gpu" : " 17 GB", | 20260507 20:23:58.739 | INFO | "huge" : "", | 20260507 20:23:58.739 | INFO | "huge_shared" : " 29 GB", | 20260507 20:23:58.739 | INFO | "malloc" : " 14 GB", | 20260507 20:23:58.739 | INFO | "pinned" : " 29 GB" | 20260507 20:23:58.739 | INFO | } | 20260507 20:23:58.739 | INFO | Perf report for the last 5.13948 seconds: | 20260507 20:23:58.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:23:58.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.739 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:23:58.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.739 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 20:23:58.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:23:58.740 | INFO | Uploaded 36GiB to GPU in 2.4999s (14.4006GiB/s) | 20260507 20:23:58.740 | INFO | Downloaded 34GiB to GPU in 2.44236s (13.9209GiB/s) | 20260507 20:24:03.879 | INFO | Analyzing chunk frames = [2342400,2342912) | 20260507 20:24:03.879 | INFO | MemInfo: { | 20260507 20:24:03.879 | INFO | "gpu" : " 17 GB", | 20260507 20:24:03.879 | INFO | "huge" : "", | 20260507 20:24:03.879 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:03.879 | INFO | "malloc" : " 14 GB", | 20260507 20:24:03.879 | INFO | "pinned" : " 29 GB" | 20260507 20:24:03.879 | INFO | } | 20260507 20:24:03.879 | INFO | Perf report for the last 5.14016 seconds: | 20260507 20:24:03.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:03.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.879 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:24:03.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.879 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 20:24:03.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:03.880 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:24:03.880 | INFO | Total mem usage: 16.4 GiB | 20260507 20:24:03.880 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:24:03.880 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 20:24:03.880 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:24:03.880 | INFO | High Water Breakdown: | 20260507 20:24:03.880 | INFO | 59.7% : 8.8 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:24:03.880 | INFO | 40.2% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:24:03.880 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:24:03.880 | INFO | ------------------------------------------ | 20260507 20:24:03.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:24:03.880 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:24:03.880 | INFO | Total mem usage: 26.5 GiB | 20260507 20:24:03.880 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:24:03.880 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:24:03.880 | INFO | Uploaded 36GiB to GPU in 2.51398s (14.3199GiB/s) | 20260507 20:24:03.880 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:24:03.880 | INFO | Downloaded 34GiB to GPU in 2.432s (13.9802GiB/s) | 20260507 20:24:03.880 | INFO | High Water Breakdown: | 20260507 20:24:03.880 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:24:03.880 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:24:03.880 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:24:03.880 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:24:03.880 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:24:03.880 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:24:03.880 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:24:03.880 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:24:03.880 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:24:03.880 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:24:03.880 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:24:03.880 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:24:03.880 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:24:03.880 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:24:03.880 | INFO | ------------------------------------------ | 20260507 20:24:03.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:24:03.880 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:24:03.880 | INFO | Total mem usage: 27.8 GiB | 20260507 20:24:03.880 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:24:03.880 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:24:03.880 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:24:03.880 | INFO | High Water Breakdown: | 20260507 20:24:03.880 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:24:03.880 | INFO | ------------------------------------------ | 20260507 20:24:03.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:24:03.880 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:24:03.880 | INFO | Total mem usage: 15.7 GiB | 20260507 20:24:03.880 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:24:03.880 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:24:03.880 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:24:03.880 | INFO | High Water Breakdown: | 20260507 20:24:03.880 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:24:03.880 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:24:03.880 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:24:03.880 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:24:03.880 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:24:03.880 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:24:03.880 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:24:03.880 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:24:03.880 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:24:03.881 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:24:03.881 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:24:03.881 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:24:03.881 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:24:03.881 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:24:03.881 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:24:03.881 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:24:03.881 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:24:03.881 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:24:03.881 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:24:03.881 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:24:03.881 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:24:03.881 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:24:03.881 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:24:03.881 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:24:03.881 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:24:03.881 | INFO | ------------------------------------------ | 20260507 20:24:03.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:24:03.881 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:24:03.881 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:24:03.881 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:24:03.881 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.56e+01, 1.40e+01 / 1.70e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.05e-01, 1.23e-02 / 5.76e-03 | 20260507 20:24:03.881 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.47e+00, 6.00e-01 / 2.72e-01 | 20260507 20:24:03.881 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.51e+00, 2.12e-01 / 1.12e-01 | 20260507 20:24:03.881 | INFO | PerfReport: 15.82%, DME -- 4.19e+01 / 4.51e+01, 3.93e+00 / 1.66e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.76e+00 / 1.65e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.45e-01 / 2.18e-01 | 20260507 20:24:03.881 | INFO | PerfReport: 3.60%, Metrics -- 1.19e+00 / 1.41e+00, 2.99e-01 / 1.75e-01 | 20260507 20:24:03.881 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.73e-01 / 4.07e-01, 6.12e-02 / 3.29e-02 | 20260507 20:24:03.881 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:24:03.881 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:24:03.881 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:24:03.881 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 24.12%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 30.93%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 2.09%, LockUpload -- 2.23e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 2.41%, LockDownload -- 2.58e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 16.05%, LockCompute -- 1.72e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:24:03.881 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260507 20:24:03.993 | INFO | Starting a round of Baz IO | 20260507 20:24:07.481 | INFO | Populating RTMetrics report | 20260507 20:24:07.482 | INFO | RTMetrics report populated | 20260507 20:24:07.482 | INFO | Opening tmp RTMetrics json file | 20260507 20:24:07.483 | INFO | Serializing RTMetrics to json | 20260507 20:24:07.485 | INFO | Serialization done, swapping tmp file now... | 20260507 20:24:07.486 | INFO | RTMetrics done | 20260507 20:24:07.583 | INFO | Finished a round of Baz IO: Wrote 1.782 GiB in 3.59 seconds | 20260507 20:24:07.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.52% / 0.01% / 55.47% | 20260507 20:24:07.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.03295% / 1.58946e-05% / 3.57628e-05% | 20260507 20:24:07.583 | INFO | Baz IO overall: | 20260507 20:24:07.583 | INFO | BytesWritten : 1063356961280 | 20260507 20:24:07.583 | INFO | NumEvents : 515502933017 | 20260507 20:24:09.019 | INFO | Analyzing chunk frames = [2342912,2343424) | 20260507 20:24:09.019 | INFO | MemInfo: { | 20260507 20:24:09.019 | INFO | "gpu" : " 17 GB", | 20260507 20:24:09.019 | INFO | "huge" : "", | 20260507 20:24:09.019 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:09.019 | INFO | "malloc" : " 14 GB", | 20260507 20:24:09.019 | INFO | "pinned" : " 29 GB" | 20260507 20:24:09.019 | INFO | } | 20260507 20:24:09.019 | INFO | Perf report for the last 5.14011 seconds: | 20260507 20:24:09.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:09.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:09.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:09.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:09.020 | INFO | ANALYSIS : 69.6%, 2.9, 104.6ms, 0 | 20260507 20:24:09.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:09.020 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 20:24:09.020 | INFO | PRE_HQ : 68.4%, 1.8, 64.1ms, 0 | 20260507 20:24:09.020 | INFO | BAZWRITER : 69.4%, 4.1, 144.7ms, 0 | 20260507 20:24:09.020 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 20:24:09.020 | INFO | Uploaded 36GiB to GPU in 2.49189s (14.4468GiB/s) | 20260507 20:24:09.020 | INFO | Downloaded 38GiB to GPU in 2.68349s (14.1606GiB/s) | 20260507 20:24:14.157 | INFO | Analyzing chunk frames = [2343424,2343936) | 20260507 20:24:14.157 | INFO | MemInfo: { | 20260507 20:24:14.157 | INFO | "gpu" : " 17 GB", | 20260507 20:24:14.157 | INFO | "huge" : "", | 20260507 20:24:14.157 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:14.157 | INFO | "malloc" : " 14 GB", | 20260507 20:24:14.157 | INFO | "pinned" : " 29 GB" | 20260507 20:24:14.157 | INFO | } | 20260507 20:24:14.157 | INFO | Perf report for the last 5.13754 seconds: | 20260507 20:24:14.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:14.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:24:14.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:24:14.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:14.157 | INFO | Uploaded 36GiB to GPU in 2.54024s (14.1719GiB/s) | 20260507 20:24:14.157 | INFO | Downloaded 34GiB to GPU in 2.49716s (13.6155GiB/s) | 20260507 20:24:19.297 | INFO | Analyzing chunk frames = [2343936,2344448) | 20260507 20:24:19.297 | INFO | MemInfo: { | 20260507 20:24:19.297 | INFO | "gpu" : " 17 GB", | 20260507 20:24:19.297 | INFO | "huge" : "", | 20260507 20:24:19.297 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:19.297 | INFO | "malloc" : " 14 GB", | 20260507 20:24:19.297 | INFO | "pinned" : " 29 GB" | 20260507 20:24:19.297 | INFO | } | 20260507 20:24:19.297 | INFO | Perf report for the last 5.13973 seconds: | 20260507 20:24:19.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:19.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:24:19.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:24:19.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:19.297 | INFO | Uploaded 36GiB to GPU in 2.56985s (14.0086GiB/s) | 20260507 20:24:19.297 | INFO | Downloaded 34GiB to GPU in 2.47093s (13.76GiB/s) | 20260507 20:24:24.435 | INFO | Analyzing chunk frames = [2344448,2344960) | 20260507 20:24:24.435 | INFO | MemInfo: { | 20260507 20:24:24.435 | INFO | "gpu" : " 17 GB", | 20260507 20:24:24.435 | INFO | "huge" : "", | 20260507 20:24:24.435 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:24.435 | INFO | "malloc" : " 14 GB", | 20260507 20:24:24.435 | INFO | "pinned" : " 29 GB" | 20260507 20:24:24.435 | INFO | } | 20260507 20:24:24.435 | INFO | Perf report for the last 5.13871 seconds: | 20260507 20:24:24.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:24.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.435 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:24:24.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.436 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:24:24.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:24.436 | INFO | Uploaded 36GiB to GPU in 2.4846s (14.4892GiB/s) | 20260507 20:24:24.436 | INFO | Downloaded 34GiB to GPU in 2.39922s (14.1713GiB/s) | 20260507 20:24:29.575 | INFO | Analyzing chunk frames = [2344960,2345472) | 20260507 20:24:29.575 | INFO | MemInfo: { | 20260507 20:24:29.575 | INFO | "gpu" : " 17 GB", | 20260507 20:24:29.575 | INFO | "huge" : "", | 20260507 20:24:29.575 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:29.575 | INFO | "malloc" : " 14 GB", | 20260507 20:24:29.575 | INFO | "pinned" : " 29 GB" | 20260507 20:24:29.575 | INFO | } | 20260507 20:24:29.575 | INFO | Perf report for the last 5.1399 seconds: | 20260507 20:24:29.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:29.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.575 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:24:29.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.575 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:24:29.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:29.576 | INFO | Uploaded 36GiB to GPU in 2.35885s (15.2617GiB/s) | 20260507 20:24:29.576 | INFO | Downloaded 34GiB to GPU in 2.49905s (13.6052GiB/s) | 20260507 20:24:34.714 | INFO | Analyzing chunk frames = [2345472,2345984) | 20260507 20:24:34.714 | INFO | MemInfo: { | 20260507 20:24:34.714 | INFO | "gpu" : " 17 GB", | 20260507 20:24:34.715 | INFO | "huge" : "", | 20260507 20:24:34.715 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:34.715 | INFO | "malloc" : " 14 GB", | 20260507 20:24:34.715 | INFO | "pinned" : " 29 GB" | 20260507 20:24:34.715 | INFO | } | 20260507 20:24:34.715 | INFO | Perf report for the last 5.13929 seconds: | 20260507 20:24:34.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:34.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:24:34.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:24:34.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:34.715 | INFO | Uploaded 36GiB to GPU in 2.50615s (14.3647GiB/s) | 20260507 20:24:34.715 | INFO | Downloaded 34GiB to GPU in 2.41067s (14.104GiB/s) | 20260507 20:24:39.854 | INFO | Analyzing chunk frames = [2345984,2346496) | 20260507 20:24:39.854 | INFO | MemInfo: { | 20260507 20:24:39.854 | INFO | "gpu" : " 17 GB", | 20260507 20:24:39.854 | INFO | "huge" : "", | 20260507 20:24:39.854 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:39.854 | INFO | "malloc" : " 14 GB", | 20260507 20:24:39.854 | INFO | "pinned" : " 29 GB" | 20260507 20:24:39.854 | INFO | } | 20260507 20:24:39.854 | INFO | Perf report for the last 5.13979 seconds: | 20260507 20:24:39.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:39.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.855 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 20:24:39.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.855 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 20:24:39.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:39.855 | INFO | Uploaded 36GiB to GPU in 2.51978s (14.2869GiB/s) | 20260507 20:24:39.855 | INFO | Downloaded 34GiB to GPU in 2.42921s (13.9963GiB/s) | 20260507 20:24:44.993 | INFO | Analyzing chunk frames = [2346496,2347008) | 20260507 20:24:44.994 | INFO | MemInfo: { | 20260507 20:24:44.994 | INFO | "gpu" : " 17 GB", | 20260507 20:24:44.994 | INFO | "huge" : "", | 20260507 20:24:44.994 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:44.994 | INFO | "malloc" : " 14 GB", | 20260507 20:24:44.994 | INFO | "pinned" : " 29 GB" | 20260507 20:24:44.994 | INFO | } | 20260507 20:24:44.994 | INFO | Perf report for the last 5.13927 seconds: | 20260507 20:24:44.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:44.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 20:24:44.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 20:24:44.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:44.994 | INFO | Uploaded 36GiB to GPU in 2.54047s (14.1706GiB/s) | 20260507 20:24:44.994 | INFO | Downloaded 34GiB to GPU in 2.42473s (14.0222GiB/s) | 20260507 20:24:45.110 | INFO | Starting a round of Baz IO | 20260507 20:24:48.584 | INFO | Populating RTMetrics report | 20260507 20:24:48.584 | INFO | RTMetrics report populated | 20260507 20:24:48.584 | INFO | Opening tmp RTMetrics json file | 20260507 20:24:48.588 | INFO | Serializing RTMetrics to json | 20260507 20:24:48.594 | INFO | Serialization done, swapping tmp file now... | 20260507 20:24:48.600 | INFO | RTMetrics done | 20260507 20:24:48.726 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.615 seconds | 20260507 20:24:48.726 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.51% / 0.01% / 55.48% | 20260507 20:24:48.726 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.03008% / 1.58946e-05% / 4.76837e-05% | 20260507 20:24:48.726 | INFO | Baz IO overall: | 20260507 20:24:48.726 | INFO | BytesWritten : 1065269030589 | 20260507 20:24:48.726 | INFO | NumEvents : 516428325731 | 20260507 20:24:50.134 | INFO | Analyzing chunk frames = [2347008,2347520) | 20260507 20:24:50.134 | INFO | MemInfo: { | 20260507 20:24:50.134 | INFO | "gpu" : " 17 GB", | 20260507 20:24:50.134 | INFO | "huge" : "", | 20260507 20:24:50.134 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:50.134 | INFO | "malloc" : " 14 GB", | 20260507 20:24:50.134 | INFO | "pinned" : " 29 GB" | 20260507 20:24:50.134 | INFO | } | 20260507 20:24:50.134 | INFO | Perf report for the last 5.14042 seconds: | 20260507 20:24:50.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:50.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:50.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:50.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:50.134 | INFO | ANALYSIS : 69.4%, 2.9, 104.3ms, 0 | 20260507 20:24:50.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:50.134 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 20:24:50.134 | INFO | PRE_HQ : 68.2%, 1.8, 65.0ms, 0 | 20260507 20:24:50.134 | INFO | BAZWRITER : 69.8%, 3.9, 139.7ms, 0 | 20260507 20:24:50.134 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 20:24:50.134 | INFO | Uploaded 36GiB to GPU in 2.53299s (14.2125GiB/s) | 20260507 20:24:50.134 | INFO | Downloaded 38GiB to GPU in 2.69309s (14.1102GiB/s) | 20260507 20:24:55.272 | INFO | Analyzing chunk frames = [2347520,2348032) | 20260507 20:24:55.272 | INFO | MemInfo: { | 20260507 20:24:55.272 | INFO | "gpu" : " 17 GB", | 20260507 20:24:55.272 | INFO | "huge" : "", | 20260507 20:24:55.272 | INFO | "huge_shared" : " 29 GB", | 20260507 20:24:55.272 | INFO | "malloc" : " 14 GB", | 20260507 20:24:55.272 | INFO | "pinned" : " 29 GB" | 20260507 20:24:55.272 | INFO | } | 20260507 20:24:55.272 | INFO | Perf report for the last 5.13799 seconds: | 20260507 20:24:55.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:24:55.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:24:55.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:24:55.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:24:55.272 | INFO | Uploaded 36GiB to GPU in 2.57787s (13.965GiB/s) | 20260507 20:24:55.272 | INFO | Downloaded 34GiB to GPU in 2.4916s (13.6458GiB/s) | 20260507 20:25:00.412 | INFO | Analyzing chunk frames = [2348032,2348544) | 20260507 20:25:00.412 | INFO | MemInfo: { | 20260507 20:25:00.412 | INFO | "gpu" : " 17 GB", | 20260507 20:25:00.412 | INFO | "huge" : "", | 20260507 20:25:00.412 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:00.412 | INFO | "malloc" : " 14 GB", | 20260507 20:25:00.412 | INFO | "pinned" : " 29 GB" | 20260507 20:25:00.412 | INFO | } | 20260507 20:25:00.413 | INFO | Perf report for the last 5.14033 seconds: | 20260507 20:25:00.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:00.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:25:00.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:25:00.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:00.413 | INFO | Uploaded 36GiB to GPU in 2.5933s (13.8819GiB/s) | 20260507 20:25:00.413 | INFO | Downloaded 34GiB to GPU in 2.4671s (13.7814GiB/s) | 20260507 20:25:05.551 | INFO | Analyzing chunk frames = [2348544,2349056) | 20260507 20:25:05.551 | INFO | MemInfo: { | 20260507 20:25:05.551 | INFO | "gpu" : " 17 GB", | 20260507 20:25:05.551 | INFO | "huge" : "", | 20260507 20:25:05.551 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:05.551 | INFO | "malloc" : " 14 GB", | 20260507 20:25:05.551 | INFO | "pinned" : " 29 GB" | 20260507 20:25:05.551 | INFO | } | 20260507 20:25:05.551 | INFO | Perf report for the last 5.13839 seconds: | 20260507 20:25:05.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:05.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.551 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 20:25:05.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.551 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:25:05.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:05.552 | INFO | Uploaded 36GiB to GPU in 2.5871s (13.9152GiB/s) | 20260507 20:25:05.552 | INFO | Downloaded 34GiB to GPU in 2.46954s (13.7677GiB/s) | 20260507 20:25:10.692 | INFO | Analyzing chunk frames = [2349056,2349568) | 20260507 20:25:10.692 | INFO | MemInfo: { | 20260507 20:25:10.692 | INFO | "gpu" : " 17 GB", | 20260507 20:25:10.692 | INFO | "huge" : "", | 20260507 20:25:10.692 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:10.692 | INFO | "malloc" : " 14 GB", | 20260507 20:25:10.692 | INFO | "pinned" : " 29 GB" | 20260507 20:25:10.692 | INFO | } | 20260507 20:25:10.692 | INFO | Perf report for the last 5.14098 seconds: | 20260507 20:25:10.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:10.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.692 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:25:10.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.693 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:25:10.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:10.693 | INFO | Uploaded 36GiB to GPU in 2.3779s (15.1394GiB/s) | 20260507 20:25:10.693 | INFO | Downloaded 34GiB to GPU in 2.49054s (13.6517GiB/s) | 20260507 20:25:15.830 | INFO | Analyzing chunk frames = [2349568,2350080) | 20260507 20:25:15.830 | INFO | MemInfo: { | 20260507 20:25:15.831 | INFO | "gpu" : " 17 GB", | 20260507 20:25:15.831 | INFO | "huge" : "", | 20260507 20:25:15.831 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:15.831 | INFO | "malloc" : " 14 GB", | 20260507 20:25:15.831 | INFO | "pinned" : " 29 GB" | 20260507 20:25:15.831 | INFO | } | 20260507 20:25:15.831 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:25:15.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:15.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 20:25:15.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:25:15.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:15.831 | INFO | Uploaded 36GiB to GPU in 2.51244s (14.3287GiB/s) | 20260507 20:25:15.831 | INFO | Downloaded 34GiB to GPU in 2.41515s (14.0778GiB/s) | 20260507 20:25:20.968 | INFO | Analyzing chunk frames = [2350080,2350592) | 20260507 20:25:20.968 | INFO | MemInfo: { | 20260507 20:25:20.968 | INFO | "gpu" : " 17 GB", | 20260507 20:25:20.968 | INFO | "huge" : "", | 20260507 20:25:20.968 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:20.968 | INFO | "malloc" : " 14 GB", | 20260507 20:25:20.968 | INFO | "pinned" : " 29 GB" | 20260507 20:25:20.968 | INFO | } | 20260507 20:25:20.969 | INFO | Perf report for the last 5.1378 seconds: | 20260507 20:25:20.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:20.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:25:20.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 20:25:20.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:20.969 | INFO | Uploaded 36GiB to GPU in 2.56025s (14.0611GiB/s) | 20260507 20:25:20.969 | INFO | Downloaded 34GiB to GPU in 2.4399s (13.935GiB/s) | 20260507 20:25:26.107 | INFO | Analyzing chunk frames = [2350592,2351104) | 20260507 20:25:26.107 | INFO | MemInfo: { | 20260507 20:25:26.107 | INFO | "gpu" : " 17 GB", | 20260507 20:25:26.107 | INFO | "huge" : "", | 20260507 20:25:26.107 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:26.107 | INFO | "malloc" : " 14 GB", | 20260507 20:25:26.108 | INFO | "pinned" : " 29 GB" | 20260507 20:25:26.108 | INFO | } | 20260507 20:25:26.108 | INFO | Perf report for the last 5.13917 seconds: | 20260507 20:25:26.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:26.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 20:25:26.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 20:25:26.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:26.108 | INFO | Uploaded 36GiB to GPU in 2.53462s (14.2033GiB/s) | 20260507 20:25:26.108 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:25:26.108 | INFO | Total mem usage: 16.4 GiB | 20260507 20:25:26.108 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:25:26.108 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.5%) | 20260507 20:25:26.108 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:25:26.108 | INFO | High Water Breakdown: | 20260507 20:25:26.108 | INFO | 59.8% : 8.88 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:25:26.108 | INFO | 40% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:25:26.108 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:25:26.108 | INFO | ------------------------------------------ | 20260507 20:25:26.108 | INFO | Downloaded 34GiB to GPU in 2.44069s (13.9305GiB/s) | 20260507 20:25:26.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:25:26.108 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:25:26.108 | INFO | Total mem usage: 26.5 GiB | 20260507 20:25:26.108 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:25:26.108 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:25:26.108 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 20:25:26.108 | INFO | High Water Breakdown: | 20260507 20:25:26.108 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:25:26.108 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:25:26.108 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:25:26.108 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:25:26.109 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:25:26.109 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:25:26.109 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:25:26.109 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:25:26.109 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:25:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:25:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:25:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:25:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:25:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:25:26.109 | INFO | ------------------------------------------ | 20260507 20:25:26.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:25:26.109 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:25:26.109 | INFO | Total mem usage: 27.8 GiB | 20260507 20:25:26.109 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:25:26.109 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:25:26.109 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:25:26.109 | INFO | High Water Breakdown: | 20260507 20:25:26.109 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:25:26.109 | INFO | ------------------------------------------ | 20260507 20:25:26.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:25:26.109 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:25:26.109 | INFO | Total mem usage: 15.7 GiB | 20260507 20:25:26.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:25:26.109 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:25:26.109 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:25:26.109 | INFO | High Water Breakdown: | 20260507 20:25:26.109 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:25:26.109 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:25:26.109 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:25:26.109 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:25:26.109 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:25:26.109 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:25:26.109 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:25:26.109 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:25:26.109 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:25:26.109 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:25:26.109 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:25:26.109 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:25:26.109 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:25:26.109 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:25:26.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:25:26.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:25:26.109 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:25:26.109 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:25:26.109 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:25:26.109 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:25:26.109 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:25:26.109 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:25:26.109 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:25:26.109 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:25:26.109 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:25:26.109 | INFO | ------------------------------------------ | 20260507 20:25:26.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:25:26.109 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:25:26.109 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:25:26.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:25:26.109 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.59e+01, 1.41e+01 / 1.97e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.05e-01, 1.15e-02 / 7.79e-03 | 20260507 20:25:26.109 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.49e+00, 6.02e-01 / 2.58e-01 | 20260507 20:25:26.109 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.13e-01 / 1.10e-01 | 20260507 20:25:26.109 | INFO | PerfReport: 15.72%, DME -- 4.17e+01 / 4.50e+01, 4.04e+00 / 1.26e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.89e+00 / 2.10e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.49e+00 / 2.79e+00, 4.43e-01 / 2.59e-01 | 20260507 20:25:26.109 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.37e+00, 3.05e-01 / 1.77e-01 | 20260507 20:25:26.109 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.03e-01, 6.09e-02 / 3.77e-02 | 20260507 20:25:26.109 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:25:26.109 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:25:26.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:25:26.109 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 24.55%, Upload -- 2.62e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 31.08%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 2.18%, LockDownload -- 2.33e+00 / 0.00e+00, 3.57e+00 / 0.00e+00 | 20260507 20:25:26.109 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 20:25:26.110 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260507 20:25:26.221 | INFO | Starting a round of Baz IO | 20260507 20:25:29.696 | INFO | Populating RTMetrics report | 20260507 20:25:29.696 | INFO | RTMetrics report populated | 20260507 20:25:29.696 | INFO | Opening tmp RTMetrics json file | 20260507 20:25:29.723 | INFO | Serializing RTMetrics to json | 20260507 20:25:29.726 | INFO | Serialization done, swapping tmp file now... | 20260507 20:25:29.727 | INFO | RTMetrics done | 20260507 20:25:29.819 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.597 seconds | 20260507 20:25:29.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.50% / 0.01% / 55.49% | 20260507 20:25:29.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.02398% / 1.58946e-05% / 3.97364e-05% | 20260507 20:25:29.819 | INFO | Baz IO overall: | 20260507 20:25:29.819 | INFO | BytesWritten : 1067180865766 | 20260507 20:25:29.819 | INFO | NumEvents : 517353629196 | 20260507 20:25:31.247 | INFO | Analyzing chunk frames = [2351104,2351616) | 20260507 20:25:31.247 | INFO | MemInfo: { | 20260507 20:25:31.247 | INFO | "gpu" : " 17 GB", | 20260507 20:25:31.247 | INFO | "huge" : "", | 20260507 20:25:31.247 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:31.247 | INFO | "malloc" : " 14 GB", | 20260507 20:25:31.247 | INFO | "pinned" : " 29 GB" | 20260507 20:25:31.247 | INFO | } | 20260507 20:25:31.247 | INFO | Perf report for the last 5.13965 seconds: | 20260507 20:25:31.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:31.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:31.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:31.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:31.247 | INFO | ANALYSIS : 69.1%, 2.9, 105.1ms, 0 | 20260507 20:25:31.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:31.247 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 20:25:31.247 | INFO | PRE_HQ : 68.0%, 1.8, 63.9ms, 0 | 20260507 20:25:31.247 | INFO | BAZWRITER : 69.4%, 4.1, 144.7ms, 0 | 20260507 20:25:31.247 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:25:31.248 | INFO | Uploaded 36GiB to GPU in 2.55701s (14.0789GiB/s) | 20260507 20:25:31.248 | INFO | Downloaded 38GiB to GPU in 2.7144s (13.9994GiB/s) | 20260507 20:25:36.387 | INFO | Analyzing chunk frames = [2351616,2352128) | 20260507 20:25:36.387 | INFO | MemInfo: { | 20260507 20:25:36.387 | INFO | "gpu" : " 17 GB", | 20260507 20:25:36.387 | INFO | "huge" : "", | 20260507 20:25:36.387 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:36.387 | INFO | "malloc" : " 14 GB", | 20260507 20:25:36.387 | INFO | "pinned" : " 29 GB" | 20260507 20:25:36.387 | INFO | } | 20260507 20:25:36.387 | INFO | Perf report for the last 5.13974 seconds: | 20260507 20:25:36.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:36.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.387 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 20:25:36.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.387 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:25:36.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:36.388 | INFO | Uploaded 36GiB to GPU in 2.58199s (13.9427GiB/s) | 20260507 20:25:36.388 | INFO | Downloaded 34GiB to GPU in 2.49049s (13.6519GiB/s) | 20260507 20:25:41.526 | INFO | Analyzing chunk frames = [2352128,2352640) | 20260507 20:25:41.526 | INFO | MemInfo: { | 20260507 20:25:41.526 | INFO | "gpu" : " 17 GB", | 20260507 20:25:41.526 | INFO | "huge" : "", | 20260507 20:25:41.526 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:41.526 | INFO | "malloc" : " 14 GB", | 20260507 20:25:41.526 | INFO | "pinned" : " 29 GB" | 20260507 20:25:41.526 | INFO | } | 20260507 20:25:41.526 | INFO | Perf report for the last 5.13884 seconds: | 20260507 20:25:41.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:41.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:25:41.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:25:41.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:41.526 | INFO | Uploaded 36GiB to GPU in 2.56877s (14.0145GiB/s) | 20260507 20:25:41.527 | INFO | Downloaded 34GiB to GPU in 2.45988s (13.8218GiB/s) | 20260507 20:25:46.665 | INFO | Analyzing chunk frames = [2352640,2353152) | 20260507 20:25:46.665 | INFO | MemInfo: { | 20260507 20:25:46.665 | INFO | "gpu" : " 17 GB", | 20260507 20:25:46.665 | INFO | "huge" : "", | 20260507 20:25:46.665 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:46.665 | INFO | "malloc" : " 14 GB", | 20260507 20:25:46.665 | INFO | "pinned" : " 29 GB" | 20260507 20:25:46.665 | INFO | } | 20260507 20:25:46.665 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:25:46.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:46.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 20:25:46.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:25:46.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:46.665 | INFO | Uploaded 36GiB to GPU in 2.595s (13.8728GiB/s) | 20260507 20:25:46.665 | INFO | Downloaded 34GiB to GPU in 2.46791s (13.7769GiB/s) | 20260507 20:25:51.806 | INFO | Analyzing chunk frames = [2353152,2353664) | 20260507 20:25:51.806 | INFO | MemInfo: { | 20260507 20:25:51.806 | INFO | "gpu" : " 17 GB", | 20260507 20:25:51.806 | INFO | "huge" : "", | 20260507 20:25:51.806 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:51.806 | INFO | "malloc" : " 14 GB", | 20260507 20:25:51.806 | INFO | "pinned" : " 29 GB" | 20260507 20:25:51.806 | INFO | } | 20260507 20:25:51.806 | INFO | Perf report for the last 5.14086 seconds: | 20260507 20:25:51.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:51.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:25:51.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:25:51.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:51.806 | INFO | Uploaded 36GiB to GPU in 2.36426s (15.2268GiB/s) | 20260507 20:25:51.806 | INFO | Downloaded 34GiB to GPU in 2.49491s (13.6277GiB/s) | 20260507 20:25:56.945 | INFO | Analyzing chunk frames = [2353664,2354176) | 20260507 20:25:56.945 | INFO | MemInfo: { | 20260507 20:25:56.945 | INFO | "gpu" : " 17 GB", | 20260507 20:25:56.945 | INFO | "huge" : "", | 20260507 20:25:56.945 | INFO | "huge_shared" : " 29 GB", | 20260507 20:25:56.945 | INFO | "malloc" : " 14 GB", | 20260507 20:25:56.945 | INFO | "pinned" : " 29 GB" | 20260507 20:25:56.945 | INFO | } | 20260507 20:25:56.945 | INFO | Perf report for the last 5.13946 seconds: | 20260507 20:25:56.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:25:56.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260507 20:25:56.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:25:56.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:25:56.945 | INFO | Uploaded 36GiB to GPU in 2.434s (14.7905GiB/s) | 20260507 20:25:56.945 | INFO | Downloaded 34GiB to GPU in 2.38497s (14.256GiB/s) | 20260507 20:26:02.084 | INFO | Analyzing chunk frames = [2354176,2354688) | 20260507 20:26:02.084 | INFO | MemInfo: { | 20260507 20:26:02.084 | INFO | "gpu" : " 17 GB", | 20260507 20:26:02.084 | INFO | "huge" : "", | 20260507 20:26:02.084 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:02.084 | INFO | "malloc" : " 14 GB", | 20260507 20:26:02.084 | INFO | "pinned" : " 29 GB" | 20260507 20:26:02.084 | INFO | } | 20260507 20:26:02.084 | INFO | Perf report for the last 5.1393 seconds: | 20260507 20:26:02.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:02.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.084 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:26:02.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.084 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 20:26:02.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:02.085 | INFO | Uploaded 36GiB to GPU in 2.55159s (14.1089GiB/s) | 20260507 20:26:02.085 | INFO | Downloaded 34GiB to GPU in 2.44861s (13.8854GiB/s) | 20260507 20:26:07.224 | INFO | Analyzing chunk frames = [2354688,2355200) | 20260507 20:26:07.224 | INFO | MemInfo: { | 20260507 20:26:07.224 | INFO | "gpu" : " 17 GB", | 20260507 20:26:07.224 | INFO | "huge" : "", | 20260507 20:26:07.224 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:07.224 | INFO | "malloc" : " 14 GB", | 20260507 20:26:07.224 | INFO | "pinned" : " 29 GB" | 20260507 20:26:07.224 | INFO | } | 20260507 20:26:07.224 | INFO | Perf report for the last 5.13941 seconds: | 20260507 20:26:07.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:07.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:26:07.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:26:07.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:07.224 | INFO | Uploaded 36GiB to GPU in 2.50104s (14.394GiB/s) | 20260507 20:26:07.224 | INFO | Downloaded 34GiB to GPU in 2.41964s (14.0517GiB/s) | 20260507 20:26:07.321 | INFO | Starting a round of Baz IO | 20260507 20:26:10.784 | INFO | Populating RTMetrics report | 20260507 20:26:10.785 | INFO | RTMetrics report populated | 20260507 20:26:10.785 | INFO | Opening tmp RTMetrics json file | 20260507 20:26:10.795 | INFO | Serializing RTMetrics to json | 20260507 20:26:10.827 | INFO | Serialization done, swapping tmp file now... | 20260507 20:26:10.846 | INFO | RTMetrics done | 20260507 20:26:10.904 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.583 seconds | 20260507 20:26:10.904 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.49% / 0.01% / 55.50% | 20260507 20:26:10.904 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.02672% / 2.38419e-05% / 3.17891e-05% | 20260507 20:26:10.904 | INFO | Baz IO overall: | 20260507 20:26:10.904 | INFO | BytesWritten : 1069093591682 | 20260507 20:26:10.904 | INFO | NumEvents : 518280007125 | 20260507 20:26:12.362 | INFO | Analyzing chunk frames = [2355200,2355712) | 20260507 20:26:12.362 | INFO | MemInfo: { | 20260507 20:26:12.362 | INFO | "gpu" : " 17 GB", | 20260507 20:26:12.362 | INFO | "huge" : "", | 20260507 20:26:12.362 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:12.362 | INFO | "malloc" : " 14 GB", | 20260507 20:26:12.362 | INFO | "pinned" : " 29 GB" | 20260507 20:26:12.363 | INFO | } | 20260507 20:26:12.363 | INFO | Perf report for the last 5.13868 seconds: | 20260507 20:26:12.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:12.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:12.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:12.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:12.363 | INFO | ANALYSIS : 68.8%, 2.9, 106.5ms, 0 | 20260507 20:26:12.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:12.363 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 20:26:12.363 | INFO | PRE_HQ : 66.2%, 1.7, 58.2ms, 0 | 20260507 20:26:12.363 | INFO | BAZWRITER : 69.3%, 3.8, 135.3ms, 0 | 20260507 20:26:12.363 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 20:26:12.363 | INFO | Uploaded 36GiB to GPU in 2.55171s (14.1082GiB/s) | 20260507 20:26:12.363 | INFO | Downloaded 38GiB to GPU in 2.7031s (14.0579GiB/s) | 20260507 20:26:17.501 | INFO | Analyzing chunk frames = [2355712,2356224) | 20260507 20:26:17.501 | INFO | MemInfo: { | 20260507 20:26:17.501 | INFO | "gpu" : " 17 GB", | 20260507 20:26:17.502 | INFO | "huge" : "", | 20260507 20:26:17.502 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:17.502 | INFO | "malloc" : " 14 GB", | 20260507 20:26:17.502 | INFO | "pinned" : " 29 GB" | 20260507 20:26:17.502 | INFO | } | 20260507 20:26:17.502 | INFO | Perf report for the last 5.13905 seconds: | 20260507 20:26:17.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:17.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:26:17.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:26:17.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:17.502 | INFO | Uploaded 36GiB to GPU in 2.54574s (14.1413GiB/s) | 20260507 20:26:17.502 | INFO | Downloaded 34GiB to GPU in 2.50455s (13.5753GiB/s) | 20260507 20:26:22.640 | INFO | Analyzing chunk frames = [2356224,2356736) | 20260507 20:26:22.640 | INFO | MemInfo: { | 20260507 20:26:22.640 | INFO | "gpu" : " 17 GB", | 20260507 20:26:22.641 | INFO | "huge" : "", | 20260507 20:26:22.641 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:22.641 | INFO | "malloc" : " 14 GB", | 20260507 20:26:22.641 | INFO | "pinned" : " 29 GB" | 20260507 20:26:22.641 | INFO | } | 20260507 20:26:22.641 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:26:22.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:22.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 20:26:22.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:26:22.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:22.641 | INFO | Uploaded 36GiB to GPU in 2.4826s (14.5009GiB/s) | 20260507 20:26:22.641 | INFO | Downloaded 34GiB to GPU in 2.48752s (13.6682GiB/s) | 20260507 20:26:27.780 | INFO | Analyzing chunk frames = [2356736,2357248) | 20260507 20:26:27.780 | INFO | MemInfo: { | 20260507 20:26:27.780 | INFO | "gpu" : " 17 GB", | 20260507 20:26:27.780 | INFO | "huge" : "", | 20260507 20:26:27.780 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:27.780 | INFO | "malloc" : " 14 GB", | 20260507 20:26:27.780 | INFO | "pinned" : " 29 GB" | 20260507 20:26:27.780 | INFO | } | 20260507 20:26:27.780 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:26:27.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:27.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 20:26:27.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:26:27.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:27.780 | INFO | Uploaded 36GiB to GPU in 2.45853s (14.6429GiB/s) | 20260507 20:26:27.780 | INFO | Downloaded 34GiB to GPU in 2.39715s (14.1835GiB/s) | 20260507 20:26:32.919 | INFO | Analyzing chunk frames = [2357248,2357760) | 20260507 20:26:32.919 | INFO | MemInfo: { | 20260507 20:26:32.919 | INFO | "gpu" : " 17 GB", | 20260507 20:26:32.919 | INFO | "huge" : "", | 20260507 20:26:32.919 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:32.919 | INFO | "malloc" : " 14 GB", | 20260507 20:26:32.919 | INFO | "pinned" : " 29 GB" | 20260507 20:26:32.919 | INFO | } | 20260507 20:26:32.919 | INFO | Perf report for the last 5.13943 seconds: | 20260507 20:26:32.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:32.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.919 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:26:32.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.919 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:26:32.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:32.920 | INFO | Uploaded 36GiB to GPU in 2.36553s (15.2186GiB/s) | 20260507 20:26:32.920 | INFO | Downloaded 34GiB to GPU in 2.50703s (13.5619GiB/s) | 20260507 20:26:38.060 | INFO | Analyzing chunk frames = [2357760,2358272) | 20260507 20:26:38.060 | INFO | MemInfo: { | 20260507 20:26:38.061 | INFO | "gpu" : " 17 GB", | 20260507 20:26:38.061 | INFO | "huge" : "", | 20260507 20:26:38.061 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:38.061 | INFO | "malloc" : " 14 GB", | 20260507 20:26:38.061 | INFO | "pinned" : " 29 GB" | 20260507 20:26:38.061 | INFO | } | 20260507 20:26:38.061 | INFO | Perf report for the last 5.14141 seconds: | 20260507 20:26:38.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:38.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:26:38.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | PRE_HQ : 31.4%, 1.0, 16.4ms, 0 | 20260507 20:26:38.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:38.061 | INFO | Uploaded 36GiB to GPU in 2.4847s (14.4887GiB/s) | 20260507 20:26:38.062 | INFO | Downloaded 34GiB to GPU in 2.42545s (14.018GiB/s) | 20260507 20:26:43.199 | INFO | Analyzing chunk frames = [2358272,2358784) | 20260507 20:26:43.199 | INFO | MemInfo: { | 20260507 20:26:43.199 | INFO | "gpu" : " 17 GB", | 20260507 20:26:43.199 | INFO | "huge" : "", | 20260507 20:26:43.199 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:43.199 | INFO | "malloc" : " 14 GB", | 20260507 20:26:43.199 | INFO | "pinned" : " 29 GB" | 20260507 20:26:43.199 | INFO | } | 20260507 20:26:43.200 | INFO | Perf report for the last 5.13877 seconds: | 20260507 20:26:43.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:43.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 20:26:43.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 20:26:43.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:43.200 | INFO | Uploaded 36GiB to GPU in 2.54319s (14.1554GiB/s) | 20260507 20:26:43.200 | INFO | Downloaded 34GiB to GPU in 2.46498s (13.7932GiB/s) | 20260507 20:26:48.338 | INFO | Analyzing chunk frames = [2358784,2359296) | 20260507 20:26:48.338 | INFO | MemInfo: { | 20260507 20:26:48.338 | INFO | "gpu" : " 17 GB", | 20260507 20:26:48.338 | INFO | "huge" : "", | 20260507 20:26:48.338 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:48.338 | INFO | "malloc" : " 14 GB", | 20260507 20:26:48.338 | INFO | "pinned" : " 29 GB" | 20260507 20:26:48.338 | INFO | } | 20260507 20:26:48.338 | INFO | Perf report for the last 5.13883 seconds: | 20260507 20:26:48.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:48.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.338 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:26:48.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.338 | INFO | PRE_HQ : 37.2%, 1.0, 19.3ms, 0 | 20260507 20:26:48.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:48.339 | INFO | Uploaded 36GiB to GPU in 2.50455s (14.3739GiB/s) | 20260507 20:26:48.339 | INFO | Downloaded 34GiB to GPU in 2.43054s (13.9887GiB/s) | 20260507 20:26:48.339 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:26:48.339 | INFO | Total mem usage: 16.4 GiB | 20260507 20:26:48.339 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:26:48.339 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 20:26:48.339 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:26:48.339 | INFO | High Water Breakdown: | 20260507 20:26:48.339 | INFO | 59.7% : 8.8 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:26:48.339 | INFO | 40.2% : 5.92 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:26:48.339 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:26:48.339 | INFO | ------------------------------------------ | 20260507 20:26:48.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:26:48.339 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:26:48.339 | INFO | Total mem usage: 26.5 GiB | 20260507 20:26:48.339 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:26:48.339 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:26:48.339 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:26:48.339 | INFO | High Water Breakdown: | 20260507 20:26:48.339 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:26:48.339 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:26:48.339 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:26:48.339 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:26:48.339 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:26:48.339 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:26:48.340 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:26:48.340 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:26:48.340 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:26:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:26:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:26:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:26:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:26:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:26:48.340 | INFO | ------------------------------------------ | 20260507 20:26:48.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:26:48.340 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:26:48.340 | INFO | Total mem usage: 27.8 GiB | 20260507 20:26:48.340 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:26:48.340 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:26:48.340 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:26:48.340 | INFO | High Water Breakdown: | 20260507 20:26:48.340 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:26:48.340 | INFO | ------------------------------------------ | 20260507 20:26:48.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:26:48.340 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:26:48.340 | INFO | Total mem usage: 15.7 GiB | 20260507 20:26:48.340 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:26:48.340 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:26:48.340 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:26:48.340 | INFO | High Water Breakdown: | 20260507 20:26:48.340 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:26:48.340 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:26:48.340 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:26:48.340 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:26:48.340 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:26:48.340 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:26:48.340 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:26:48.340 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:26:48.340 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:26:48.340 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:26:48.340 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:26:48.340 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:26:48.340 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:26:48.340 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:26:48.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:26:48.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:26:48.340 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:26:48.340 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:26:48.340 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:26:48.340 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:26:48.340 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:26:48.340 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:26:48.340 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:26:48.340 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:26:48.340 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:26:48.340 | INFO | ------------------------------------------ | 20260507 20:26:48.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:26:48.340 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:26:48.340 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:26:48.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:26:48.340 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.58e+01, 1.41e+01 / 1.99e+00 | 20260507 20:26:48.340 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.03e-01, 1.19e-02 / 5.13e-03 | 20260507 20:26:48.340 | INFO | PerfReport: 12.08%, Baseline -- 4.00e+00 / 4.51e+00, 5.95e-01 / 2.29e-01 | 20260507 20:26:48.340 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.18e-01 | 20260507 20:26:48.340 | INFO | PerfReport: 15.85%, DME -- 4.20e+01 / 4.52e+01, 3.96e+00 / 1.70e+00 | 20260507 20:26:48.340 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.71e+00 / 1.90e+00 | 20260507 20:26:48.340 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.51e+00 / 2.88e+00, 4.44e-01 / 1.94e-01 | 20260507 20:26:48.340 | INFO | PerfReport: 3.60%, Metrics -- 1.19e+00 / 1.41e+00, 3.02e-01 / 1.80e-01 | 20260507 20:26:48.340 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.71e-01 / 4.07e-01, 6.10e-02 / 3.08e-02 | 20260507 20:26:48.340 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:26:48.340 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:26:48.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:26:48.341 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 30.97%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 2.13%, LockUpload -- 2.28e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 2.33%, LockDownload -- 2.49e+00 / 0.00e+00, 3.71e+00 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 15.88%, LockCompute -- 1.70e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 20:26:48.341 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 20:26:48.451 | INFO | Starting a round of Baz IO | 20260507 20:26:51.937 | INFO | Populating RTMetrics report | 20260507 20:26:51.937 | INFO | RTMetrics report populated | 20260507 20:26:51.937 | INFO | Opening tmp RTMetrics json file | 20260507 20:26:51.960 | INFO | Serializing RTMetrics to json | 20260507 20:26:51.962 | INFO | Serialization done, swapping tmp file now... | 20260507 20:26:51.963 | INFO | RTMetrics done | 20260507 20:26:52.041 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.59 seconds | 20260507 20:26:52.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.48% / 0.01% / 55.51% | 20260507 20:26:52.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.02033% / 3.97364e-06% / 5.16574e-05% | 20260507 20:26:52.041 | INFO | Baz IO overall: | 20260507 20:26:52.041 | INFO | BytesWritten : 1071005063265 | 20260507 20:26:52.041 | INFO | NumEvents : 519205605951 | 20260507 20:26:53.477 | INFO | Analyzing chunk frames = [2359296,2359808) | 20260507 20:26:53.477 | INFO | MemInfo: { | 20260507 20:26:53.477 | INFO | "gpu" : " 17 GB", | 20260507 20:26:53.477 | INFO | "huge" : "", | 20260507 20:26:53.477 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:53.477 | INFO | "malloc" : " 14 GB", | 20260507 20:26:53.477 | INFO | "pinned" : " 29 GB" | 20260507 20:26:53.477 | INFO | } | 20260507 20:26:53.477 | INFO | Perf report for the last 5.13852 seconds: | 20260507 20:26:53.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:53.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:53.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:53.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:53.477 | INFO | ANALYSIS : 69.3%, 2.9, 104.5ms, 0 | 20260507 20:26:53.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:53.477 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260507 20:26:53.477 | INFO | PRE_HQ : 68.2%, 1.8, 64.8ms, 0 | 20260507 20:26:53.477 | INFO | BAZWRITER : 69.2%, 4.3, 151.2ms, 0 | 20260507 20:26:53.477 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 20:26:53.477 | INFO | Uploaded 36GiB to GPU in 2.6036s (13.827GiB/s) | 20260507 20:26:53.477 | INFO | Downloaded 38GiB to GPU in 2.69546s (14.0978GiB/s) | 20260507 20:26:58.616 | INFO | Analyzing chunk frames = [2359808,2360320) | 20260507 20:26:58.616 | INFO | MemInfo: { | 20260507 20:26:58.616 | INFO | "gpu" : " 17 GB", | 20260507 20:26:58.616 | INFO | "huge" : "", | 20260507 20:26:58.616 | INFO | "huge_shared" : " 29 GB", | 20260507 20:26:58.616 | INFO | "malloc" : " 14 GB", | 20260507 20:26:58.616 | INFO | "pinned" : " 29 GB" | 20260507 20:26:58.616 | INFO | } | 20260507 20:26:58.616 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:26:58.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:26:58.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:26:58.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:26:58.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:26:58.616 | INFO | Uploaded 36GiB to GPU in 2.57359s (13.9882GiB/s) | 20260507 20:26:58.616 | INFO | Downloaded 34GiB to GPU in 2.48989s (13.6552GiB/s) | 20260507 20:27:03.756 | INFO | Analyzing chunk frames = [2360320,2360832) | 20260507 20:27:03.756 | INFO | MemInfo: { | 20260507 20:27:03.756 | INFO | "gpu" : " 17 GB", | 20260507 20:27:03.756 | INFO | "huge" : "", | 20260507 20:27:03.756 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:03.756 | INFO | "malloc" : " 14 GB", | 20260507 20:27:03.756 | INFO | "pinned" : " 29 GB" | 20260507 20:27:03.756 | INFO | } | 20260507 20:27:03.756 | INFO | Perf report for the last 5.14052 seconds: | 20260507 20:27:03.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:03.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.756 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 20:27:03.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.756 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:27:03.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:03.757 | INFO | Uploaded 36GiB to GPU in 2.59772s (13.8583GiB/s) | 20260507 20:27:03.757 | INFO | Downloaded 34GiB to GPU in 2.46385s (13.7996GiB/s) | 20260507 20:27:08.896 | INFO | Analyzing chunk frames = [2360832,2361344) | 20260507 20:27:08.896 | INFO | MemInfo: { | 20260507 20:27:08.896 | INFO | "gpu" : " 17 GB", | 20260507 20:27:08.896 | INFO | "huge" : "", | 20260507 20:27:08.896 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:08.896 | INFO | "malloc" : " 14 GB", | 20260507 20:27:08.896 | INFO | "pinned" : " 29 GB" | 20260507 20:27:08.896 | INFO | } | 20260507 20:27:08.896 | INFO | Perf report for the last 5.14008 seconds: | 20260507 20:27:08.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:08.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.896 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 20:27:08.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.896 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:27:08.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:08.897 | INFO | Uploaded 36GiB to GPU in 2.55418s (14.0946GiB/s) | 20260507 20:27:08.897 | INFO | Downloaded 34GiB to GPU in 2.49294s (13.6385GiB/s) | 20260507 20:27:14.036 | INFO | Analyzing chunk frames = [2361344,2361856) | 20260507 20:27:14.036 | INFO | MemInfo: { | 20260507 20:27:14.036 | INFO | "gpu" : " 17 GB", | 20260507 20:27:14.036 | INFO | "huge" : "", | 20260507 20:27:14.036 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:14.036 | INFO | "malloc" : " 14 GB", | 20260507 20:27:14.036 | INFO | "pinned" : " 29 GB" | 20260507 20:27:14.036 | INFO | } | 20260507 20:27:14.036 | INFO | Perf report for the last 5.13937 seconds: | 20260507 20:27:14.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:14.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260507 20:27:14.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:27:14.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:14.036 | INFO | Uploaded 36GiB to GPU in 2.46919s (14.5797GiB/s) | 20260507 20:27:14.036 | INFO | Downloaded 34GiB to GPU in 2.51854s (13.4999GiB/s) | 20260507 20:27:19.174 | INFO | Analyzing chunk frames = [2361856,2362368) | 20260507 20:27:19.174 | INFO | MemInfo: { | 20260507 20:27:19.174 | INFO | "gpu" : " 17 GB", | 20260507 20:27:19.174 | INFO | "huge" : "", | 20260507 20:27:19.174 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:19.174 | INFO | "malloc" : " 14 GB", | 20260507 20:27:19.174 | INFO | "pinned" : " 29 GB" | 20260507 20:27:19.174 | INFO | } | 20260507 20:27:19.174 | INFO | Perf report for the last 5.13806 seconds: | 20260507 20:27:19.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:19.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:27:19.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:27:19.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:19.174 | INFO | Uploaded 36GiB to GPU in 2.43645s (14.7756GiB/s) | 20260507 20:27:19.174 | INFO | Downloaded 34GiB to GPU in 2.38606s (14.2494GiB/s) | 20260507 20:27:24.313 | INFO | Analyzing chunk frames = [2362368,2362880) | 20260507 20:27:24.313 | INFO | MemInfo: { | 20260507 20:27:24.313 | INFO | "gpu" : " 17 GB", | 20260507 20:27:24.313 | INFO | "huge" : "", | 20260507 20:27:24.313 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:24.313 | INFO | "malloc" : " 14 GB", | 20260507 20:27:24.313 | INFO | "pinned" : " 29 GB" | 20260507 20:27:24.313 | INFO | } | 20260507 20:27:24.313 | INFO | Perf report for the last 5.13881 seconds: | 20260507 20:27:24.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:24.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 20:27:24.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 20:27:24.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:24.313 | INFO | Uploaded 36GiB to GPU in 2.53036s (14.2272GiB/s) | 20260507 20:27:24.313 | INFO | Downloaded 34GiB to GPU in 2.4338s (13.9699GiB/s) | 20260507 20:27:29.452 | INFO | Analyzing chunk frames = [2362880,2363392) | 20260507 20:27:29.452 | INFO | MemInfo: { | 20260507 20:27:29.452 | INFO | "gpu" : " 17 GB", | 20260507 20:27:29.452 | INFO | "huge" : "", | 20260507 20:27:29.452 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:29.452 | INFO | "malloc" : " 14 GB", | 20260507 20:27:29.452 | INFO | "pinned" : " 29 GB" | 20260507 20:27:29.452 | INFO | } | 20260507 20:27:29.452 | INFO | Perf report for the last 5.13939 seconds: | 20260507 20:27:29.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:29.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 20:27:29.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:27:29.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:29.452 | INFO | Uploaded 36GiB to GPU in 2.52864s (14.2369GiB/s) | 20260507 20:27:29.452 | INFO | Downloaded 34GiB to GPU in 2.41981s (14.0507GiB/s) | 20260507 20:27:29.550 | INFO | Starting a round of Baz IO | 20260507 20:27:33.035 | INFO | Populating RTMetrics report | 20260507 20:27:33.035 | INFO | RTMetrics report populated | 20260507 20:27:33.035 | INFO | Opening tmp RTMetrics json file | 20260507 20:27:33.045 | INFO | Serializing RTMetrics to json | 20260507 20:27:33.058 | INFO | Serialization done, swapping tmp file now... | 20260507 20:27:33.059 | INFO | RTMetrics done | 20260507 20:27:33.124 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.574 seconds | 20260507 20:27:33.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.47% / 0.01% / 55.52% | 20260507 20:27:33.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01463% / 1.19209e-05% / 7.15256e-05% | 20260507 20:27:33.124 | INFO | Baz IO overall: | 20260507 20:27:33.124 | INFO | BytesWritten : 1072916615319 | 20260507 20:27:33.124 | INFO | NumEvents : 520131355865 | 20260507 20:27:34.592 | INFO | Analyzing chunk frames = [2363392,2363904) | 20260507 20:27:34.592 | INFO | MemInfo: { | 20260507 20:27:34.592 | INFO | "gpu" : " 17 GB", | 20260507 20:27:34.592 | INFO | "huge" : "", | 20260507 20:27:34.592 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:34.592 | INFO | "malloc" : " 14 GB", | 20260507 20:27:34.592 | INFO | "pinned" : " 29 GB" | 20260507 20:27:34.592 | INFO | } | 20260507 20:27:34.592 | INFO | Perf report for the last 5.13972 seconds: | 20260507 20:27:34.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:34.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:34.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:34.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:34.592 | INFO | ANALYSIS : 69.1%, 2.9, 106.8ms, 0 | 20260507 20:27:34.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:34.592 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:27:34.592 | INFO | PRE_HQ : 66.4%, 1.7, 58.0ms, 0 | 20260507 20:27:34.592 | INFO | BAZWRITER : 69.2%, 4.2, 150.7ms, 0 | 20260507 20:27:34.592 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 20:27:34.592 | INFO | Uploaded 36GiB to GPU in 2.49615s (14.4222GiB/s) | 20260507 20:27:34.592 | INFO | Downloaded 38GiB to GPU in 2.69226s (14.1145GiB/s) | 20260507 20:27:39.731 | INFO | Analyzing chunk frames = [2363904,2364416) | 20260507 20:27:39.731 | INFO | MemInfo: { | 20260507 20:27:39.731 | INFO | "gpu" : " 17 GB", | 20260507 20:27:39.731 | INFO | "huge" : "", | 20260507 20:27:39.731 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:39.731 | INFO | "malloc" : " 14 GB", | 20260507 20:27:39.731 | INFO | "pinned" : " 29 GB" | 20260507 20:27:39.731 | INFO | } | 20260507 20:27:39.731 | INFO | Perf report for the last 5.13944 seconds: | 20260507 20:27:39.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:39.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:27:39.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:27:39.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:39.731 | INFO | Uploaded 36GiB to GPU in 2.52796s (14.2407GiB/s) | 20260507 20:27:39.731 | INFO | Downloaded 34GiB to GPU in 2.47887s (13.7159GiB/s) | 20260507 20:27:44.870 | INFO | Analyzing chunk frames = [2364416,2364928) | 20260507 20:27:44.870 | INFO | MemInfo: { | 20260507 20:27:44.870 | INFO | "gpu" : " 17 GB", | 20260507 20:27:44.870 | INFO | "huge" : "", | 20260507 20:27:44.870 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:44.870 | INFO | "malloc" : " 14 GB", | 20260507 20:27:44.870 | INFO | "pinned" : " 29 GB" | 20260507 20:27:44.870 | INFO | } | 20260507 20:27:44.871 | INFO | Perf report for the last 5.13921 seconds: | 20260507 20:27:44.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:44.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 20:27:44.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:27:44.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:44.871 | INFO | Uploaded 36GiB to GPU in 2.58916s (13.9041GiB/s) | 20260507 20:27:44.871 | INFO | Downloaded 34GiB to GPU in 2.46579s (13.7887GiB/s) | 20260507 20:27:50.011 | INFO | Analyzing chunk frames = [2364928,2365440) | 20260507 20:27:50.011 | INFO | MemInfo: { | 20260507 20:27:50.011 | INFO | "gpu" : " 17 GB", | 20260507 20:27:50.011 | INFO | "huge" : "", | 20260507 20:27:50.011 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:50.011 | INFO | "malloc" : " 14 GB", | 20260507 20:27:50.011 | INFO | "pinned" : " 29 GB" | 20260507 20:27:50.011 | INFO | } | 20260507 20:27:50.011 | INFO | Perf report for the last 5.14054 seconds: | 20260507 20:27:50.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:50.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.011 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 20:27:50.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.011 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:27:50.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:50.012 | INFO | Uploaded 36GiB to GPU in 2.51757s (14.2995GiB/s) | 20260507 20:27:50.012 | INFO | Downloaded 34GiB to GPU in 2.48002s (13.7096GiB/s) | 20260507 20:27:55.151 | INFO | Analyzing chunk frames = [2365440,2365952) | 20260507 20:27:55.151 | INFO | MemInfo: { | 20260507 20:27:55.151 | INFO | "gpu" : " 17 GB", | 20260507 20:27:55.151 | INFO | "huge" : "", | 20260507 20:27:55.151 | INFO | "huge_shared" : " 29 GB", | 20260507 20:27:55.151 | INFO | "malloc" : " 14 GB", | 20260507 20:27:55.151 | INFO | "pinned" : " 29 GB" | 20260507 20:27:55.151 | INFO | } | 20260507 20:27:55.151 | INFO | Perf report for the last 5.13975 seconds: | 20260507 20:27:55.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:27:55.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 20:27:55.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:27:55.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:27:55.151 | INFO | Uploaded 36GiB to GPU in 2.45965s (14.6362GiB/s) | 20260507 20:27:55.151 | INFO | Downloaded 34GiB to GPU in 2.49262s (13.6403GiB/s) | 20260507 20:28:00.288 | INFO | Analyzing chunk frames = [2365952,2366464) | 20260507 20:28:00.288 | INFO | MemInfo: { | 20260507 20:28:00.288 | INFO | "gpu" : " 17 GB", | 20260507 20:28:00.288 | INFO | "huge" : "", | 20260507 20:28:00.288 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:00.288 | INFO | "malloc" : " 14 GB", | 20260507 20:28:00.288 | INFO | "pinned" : " 29 GB" | 20260507 20:28:00.288 | INFO | } | 20260507 20:28:00.288 | INFO | Perf report for the last 5.13765 seconds: | 20260507 20:28:00.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:00.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 20:28:00.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260507 20:28:00.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:00.289 | INFO | Uploaded 36GiB to GPU in 2.39558s (15.0277GiB/s) | 20260507 20:28:00.289 | INFO | Downloaded 34GiB to GPU in 2.38564s (14.252GiB/s) | 20260507 20:28:05.427 | INFO | Analyzing chunk frames = [2366464,2366976) | 20260507 20:28:05.427 | INFO | MemInfo: { | 20260507 20:28:05.427 | INFO | "gpu" : " 17 GB", | 20260507 20:28:05.427 | INFO | "huge" : "", | 20260507 20:28:05.427 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:05.427 | INFO | "malloc" : " 14 GB", | 20260507 20:28:05.427 | INFO | "pinned" : " 29 GB" | 20260507 20:28:05.427 | INFO | } | 20260507 20:28:05.427 | INFO | Perf report for the last 5.13827 seconds: | 20260507 20:28:05.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:05.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:28:05.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 20:28:05.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:05.427 | INFO | Uploaded 36GiB to GPU in 2.53706s (14.1896GiB/s) | 20260507 20:28:05.427 | INFO | Downloaded 34GiB to GPU in 2.45293s (13.861GiB/s) | 20260507 20:28:10.567 | INFO | Analyzing chunk frames = [2366976,2367488) | 20260507 20:28:10.567 | INFO | MemInfo: { | 20260507 20:28:10.567 | INFO | "gpu" : " 17 GB", | 20260507 20:28:10.567 | INFO | "huge" : "", | 20260507 20:28:10.567 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:10.567 | INFO | "malloc" : " 14 GB", | 20260507 20:28:10.567 | INFO | "pinned" : " 29 GB" | 20260507 20:28:10.567 | INFO | } | 20260507 20:28:10.567 | INFO | Perf report for the last 5.14025 seconds: | 20260507 20:28:10.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:10.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:28:10.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 20:28:10.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:10.567 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:28:10.567 | INFO | Total mem usage: 16.4 GiB | 20260507 20:28:10.567 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:28:10.568 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.68%) | 20260507 20:28:10.568 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:28:10.568 | INFO | High Water Breakdown: | 20260507 20:28:10.568 | INFO | 60% : 8.97 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:28:10.568 | INFO | 39.9% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:28:10.568 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:28:10.568 | INFO | ------------------------------------------ | 20260507 20:28:10.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:28:10.568 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:28:10.568 | INFO | Total mem usage: 26.5 GiB | 20260507 20:28:10.568 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:28:10.568 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 20:28:10.568 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:28:10.568 | INFO | Uploaded 36GiB to GPU in 2.50317s (14.3818GiB/s) | 20260507 20:28:10.568 | INFO | High Water Breakdown: | 20260507 20:28:10.568 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:28:10.568 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:28:10.568 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:28:10.568 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:28:10.568 | INFO | Downloaded 34GiB to GPU in 2.4309s (13.9866GiB/s) | 20260507 20:28:10.568 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:28:10.568 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:28:10.568 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:28:10.568 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:28:10.568 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:28:10.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:28:10.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:28:10.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:28:10.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:28:10.568 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:28:10.568 | INFO | ------------------------------------------ | 20260507 20:28:10.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:28:10.568 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:28:10.568 | INFO | Total mem usage: 27.8 GiB | 20260507 20:28:10.568 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:28:10.568 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:28:10.568 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:28:10.568 | INFO | High Water Breakdown: | 20260507 20:28:10.568 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:28:10.568 | INFO | ------------------------------------------ | 20260507 20:28:10.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:28:10.568 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:28:10.568 | INFO | Total mem usage: 15.7 GiB | 20260507 20:28:10.568 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:28:10.568 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:28:10.568 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:28:10.568 | INFO | High Water Breakdown: | 20260507 20:28:10.568 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:28:10.568 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:28:10.568 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:28:10.568 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:28:10.568 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:28:10.568 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:28:10.568 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:28:10.568 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:28:10.568 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:28:10.568 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:28:10.568 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:28:10.568 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:28:10.568 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:28:10.568 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:28:10.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:28:10.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:28:10.569 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:28:10.569 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:28:10.569 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:28:10.569 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:28:10.569 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:28:10.569 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:28:10.569 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:28:10.569 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:28:10.569 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:28:10.569 | INFO | ------------------------------------------ | 20260507 20:28:10.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:28:10.569 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:28:10.569 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:28:10.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:28:10.569 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.62e+01, 1.43e+01 / 2.43e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.04e-01, 1.14e-02 / 4.97e-03 | 20260507 20:28:10.569 | INFO | PerfReport: 12.12%, Baseline -- 4.02e+00 / 4.49e+00, 5.97e-01 / 2.48e-01 | 20260507 20:28:10.569 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.06e-01 | 20260507 20:28:10.569 | INFO | PerfReport: 15.89%, DME -- 4.22e+01 / 4.56e+01, 4.28e+00 / 2.17e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.79e+00 / 1.80e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.46e-01 / 2.41e-01 | 20260507 20:28:10.569 | INFO | PerfReport: 3.58%, Metrics -- 1.19e+00 / 1.37e+00, 3.04e-01 / 1.73e-01 | 20260507 20:28:10.569 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.71e-01 / 4.00e-01, 6.17e-02 / 3.20e-02 | 20260507 20:28:10.569 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:28:10.569 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:28:10.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:28:10.569 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 24.51%, Upload -- 2.63e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 24.18%, Download -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 30.99%, Compute -- 3.32e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 2.19%, LockUpload -- 2.34e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 2.29%, LockDownload -- 2.46e+00 / 0.00e+00, 3.66e+00 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 15.83%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:28:10.569 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 20:28:10.681 | INFO | Starting a round of Baz IO | 20260507 20:28:14.181 | INFO | Populating RTMetrics report | 20260507 20:28:14.181 | INFO | RTMetrics report populated | 20260507 20:28:14.181 | INFO | Opening tmp RTMetrics json file | 20260507 20:28:14.196 | INFO | Serializing RTMetrics to json | 20260507 20:28:14.215 | INFO | Serialization done, swapping tmp file now... | 20260507 20:28:14.240 | INFO | RTMetrics done | 20260507 20:28:14.273 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.591 seconds | 20260507 20:28:14.273 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.46% / 0.01% / 55.53% | 20260507 20:28:14.273 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01376% / 1.58946e-05% / 6.75519e-05% | 20260507 20:28:14.273 | INFO | Baz IO overall: | 20260507 20:28:14.273 | INFO | BytesWritten : 1074826795816 | 20260507 20:28:14.273 | INFO | NumEvents : 521056269754 | 20260507 20:28:15.707 | INFO | Analyzing chunk frames = [2367488,2368000) | 20260507 20:28:15.707 | INFO | MemInfo: { | 20260507 20:28:15.707 | INFO | "gpu" : " 17 GB", | 20260507 20:28:15.707 | INFO | "huge" : "", | 20260507 20:28:15.707 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:15.707 | INFO | "malloc" : " 14 GB", | 20260507 20:28:15.707 | INFO | "pinned" : " 29 GB" | 20260507 20:28:15.707 | INFO | } | 20260507 20:28:15.707 | INFO | Perf report for the last 5.13956 seconds: | 20260507 20:28:15.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:15.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:15.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:15.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:15.707 | INFO | ANALYSIS : 69.4%, 2.9, 105.1ms, 0 | 20260507 20:28:15.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:15.707 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 20:28:15.707 | INFO | PRE_HQ : 68.6%, 1.8, 64.6ms, 0 | 20260507 20:28:15.707 | INFO | BAZWRITER : 69.6%, 4.4, 157.3ms, 0 | 20260507 20:28:15.707 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.8ms, 0 | 20260507 20:28:15.707 | INFO | Uploaded 36GiB to GPU in 2.58807s (13.91GiB/s) | 20260507 20:28:15.707 | INFO | Downloaded 38GiB to GPU in 2.72254s (13.9575GiB/s) | 20260507 20:28:20.847 | INFO | Analyzing chunk frames = [2368000,2368512) | 20260507 20:28:20.847 | INFO | MemInfo: { | 20260507 20:28:20.847 | INFO | "gpu" : " 17 GB", | 20260507 20:28:20.847 | INFO | "huge" : "", | 20260507 20:28:20.847 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:20.847 | INFO | "malloc" : " 14 GB", | 20260507 20:28:20.847 | INFO | "pinned" : " 29 GB" | 20260507 20:28:20.847 | INFO | } | 20260507 20:28:20.847 | INFO | Perf report for the last 5.14022 seconds: | 20260507 20:28:20.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:20.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 20:28:20.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:28:20.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:20.847 | INFO | Uploaded 36GiB to GPU in 2.53726s (14.1885GiB/s) | 20260507 20:28:20.847 | INFO | Downloaded 34GiB to GPU in 2.49067s (13.651GiB/s) | 20260507 20:28:25.985 | INFO | Analyzing chunk frames = [2368512,2369024) | 20260507 20:28:25.985 | INFO | MemInfo: { | 20260507 20:28:25.985 | INFO | "gpu" : " 17 GB", | 20260507 20:28:25.985 | INFO | "huge" : "", | 20260507 20:28:25.985 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:25.985 | INFO | "malloc" : " 14 GB", | 20260507 20:28:25.985 | INFO | "pinned" : " 29 GB" | 20260507 20:28:25.985 | INFO | } | 20260507 20:28:25.985 | INFO | Perf report for the last 5.13828 seconds: | 20260507 20:28:25.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:25.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.985 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 20:28:25.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.985 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:28:25.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:25.986 | INFO | Uploaded 36GiB to GPU in 2.45954s (14.6369GiB/s) | 20260507 20:28:25.986 | INFO | Downloaded 34GiB to GPU in 2.48759s (13.6678GiB/s) | 20260507 20:28:31.126 | INFO | Analyzing chunk frames = [2369024,2369536) | 20260507 20:28:31.126 | INFO | MemInfo: { | 20260507 20:28:31.126 | INFO | "gpu" : " 17 GB", | 20260507 20:28:31.126 | INFO | "huge" : "", | 20260507 20:28:31.126 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:31.126 | INFO | "malloc" : " 14 GB", | 20260507 20:28:31.126 | INFO | "pinned" : " 29 GB" | 20260507 20:28:31.126 | INFO | } | 20260507 20:28:31.126 | INFO | Perf report for the last 5.14075 seconds: | 20260507 20:28:31.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:31.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 20:28:31.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:28:31.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:31.126 | INFO | Uploaded 36GiB to GPU in 2.47507s (14.5451GiB/s) | 20260507 20:28:31.126 | INFO | Downloaded 34GiB to GPU in 2.49755s (13.6134GiB/s) | 20260507 20:28:36.264 | INFO | Analyzing chunk frames = [2369536,2370048) | 20260507 20:28:36.264 | INFO | MemInfo: { | 20260507 20:28:36.264 | INFO | "gpu" : " 17 GB", | 20260507 20:28:36.264 | INFO | "huge" : "", | 20260507 20:28:36.264 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:36.264 | INFO | "malloc" : " 14 GB", | 20260507 20:28:36.264 | INFO | "pinned" : " 29 GB" | 20260507 20:28:36.264 | INFO | } | 20260507 20:28:36.264 | INFO | Perf report for the last 5.1381 seconds: | 20260507 20:28:36.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:36.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 20:28:36.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:28:36.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:36.264 | INFO | Uploaded 36GiB to GPU in 2.43302s (14.7964GiB/s) | 20260507 20:28:36.264 | INFO | Downloaded 34GiB to GPU in 2.50215s (13.5883GiB/s) | 20260507 20:28:41.403 | INFO | Analyzing chunk frames = [2370048,2370560) | 20260507 20:28:41.403 | INFO | MemInfo: { | 20260507 20:28:41.403 | INFO | "gpu" : " 17 GB", | 20260507 20:28:41.403 | INFO | "huge" : "", | 20260507 20:28:41.403 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:41.403 | INFO | "malloc" : " 14 GB", | 20260507 20:28:41.403 | INFO | "pinned" : " 29 GB" | 20260507 20:28:41.403 | INFO | } | 20260507 20:28:41.403 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:28:41.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:41.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:28:41.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:28:41.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:41.403 | INFO | Uploaded 36GiB to GPU in 2.48113s (14.5095GiB/s) | 20260507 20:28:41.403 | INFO | Downloaded 34GiB to GPU in 2.42231s (14.0362GiB/s) | 20260507 20:28:46.542 | INFO | Analyzing chunk frames = [2370560,2371072) | 20260507 20:28:46.542 | INFO | MemInfo: { | 20260507 20:28:46.542 | INFO | "gpu" : " 17 GB", | 20260507 20:28:46.542 | INFO | "huge" : "", | 20260507 20:28:46.543 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:46.543 | INFO | "malloc" : " 14 GB", | 20260507 20:28:46.543 | INFO | "pinned" : " 29 GB" | 20260507 20:28:46.543 | INFO | } | 20260507 20:28:46.543 | INFO | Perf report for the last 5.13957 seconds: | 20260507 20:28:46.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:46.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 20:28:46.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 20:28:46.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:46.543 | INFO | Uploaded 36GiB to GPU in 2.52658s (14.2485GiB/s) | 20260507 20:28:46.543 | INFO | Downloaded 34GiB to GPU in 2.45041s (13.8752GiB/s) | 20260507 20:28:51.682 | INFO | Analyzing chunk frames = [2371072,2371584) | 20260507 20:28:51.682 | INFO | MemInfo: { | 20260507 20:28:51.682 | INFO | "gpu" : " 17 GB", | 20260507 20:28:51.682 | INFO | "huge" : "", | 20260507 20:28:51.682 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:51.682 | INFO | "malloc" : " 14 GB", | 20260507 20:28:51.682 | INFO | "pinned" : " 29 GB" | 20260507 20:28:51.682 | INFO | } | 20260507 20:28:51.682 | INFO | Perf report for the last 5.13916 seconds: | 20260507 20:28:51.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:51.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:28:51.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260507 20:28:51.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:51.682 | INFO | Uploaded 36GiB to GPU in 2.51287s (14.3262GiB/s) | 20260507 20:28:51.682 | INFO | Downloaded 34GiB to GPU in 2.43581s (13.9584GiB/s) | 20260507 20:28:51.779 | INFO | Starting a round of Baz IO | 20260507 20:28:55.254 | INFO | Populating RTMetrics report | 20260507 20:28:55.254 | INFO | RTMetrics report populated | 20260507 20:28:55.254 | INFO | Opening tmp RTMetrics json file | 20260507 20:28:55.283 | INFO | Serializing RTMetrics to json | 20260507 20:28:55.309 | INFO | Serialization done, swapping tmp file now... | 20260507 20:28:55.345 | INFO | RTMetrics done | 20260507 20:28:55.370 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.59 seconds | 20260507 20:28:55.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.45% / 0.01% / 55.54% | 20260507 20:28:55.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.00903% / 7.94729e-06% / 3.57628e-05% | 20260507 20:28:55.370 | INFO | Baz IO overall: | 20260507 20:28:55.370 | INFO | BytesWritten : 1076737570289 | 20260507 20:28:55.370 | INFO | NumEvents : 521981887160 | 20260507 20:28:56.823 | INFO | Analyzing chunk frames = [2371584,2372096) | 20260507 20:28:56.823 | INFO | MemInfo: { | 20260507 20:28:56.823 | INFO | "gpu" : " 17 GB", | 20260507 20:28:56.823 | INFO | "huge" : "", | 20260507 20:28:56.823 | INFO | "huge_shared" : " 29 GB", | 20260507 20:28:56.823 | INFO | "malloc" : " 14 GB", | 20260507 20:28:56.823 | INFO | "pinned" : " 29 GB" | 20260507 20:28:56.823 | INFO | } | 20260507 20:28:56.823 | INFO | Perf report for the last 5.14121 seconds: | 20260507 20:28:56.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:28:56.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:56.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:56.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:56.823 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 20:28:56.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:28:56.823 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 20:28:56.823 | INFO | PRE_HQ : 66.0%, 1.7, 57.7ms, 0 | 20260507 20:28:56.823 | INFO | BAZWRITER : 69.4%, 3.7, 132.9ms, 0 | 20260507 20:28:56.823 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 20:28:56.824 | INFO | Uploaded 36GiB to GPU in 2.44645s (14.7152GiB/s) | 20260507 20:28:56.824 | INFO | Downloaded 38GiB to GPU in 2.70332s (14.0568GiB/s) | 20260507 20:29:01.962 | INFO | Analyzing chunk frames = [2372096,2372608) | 20260507 20:29:01.962 | INFO | MemInfo: { | 20260507 20:29:01.962 | INFO | "gpu" : " 17 GB", | 20260507 20:29:01.962 | INFO | "huge" : "", | 20260507 20:29:01.962 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:01.962 | INFO | "malloc" : " 14 GB", | 20260507 20:29:01.962 | INFO | "pinned" : " 29 GB" | 20260507 20:29:01.962 | INFO | } | 20260507 20:29:01.962 | INFO | Perf report for the last 5.13861 seconds: | 20260507 20:29:01.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:01.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:29:01.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:29:01.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:01.962 | INFO | Uploaded 36GiB to GPU in 2.55484s (14.0909GiB/s) | 20260507 20:29:01.962 | INFO | Downloaded 34GiB to GPU in 2.5s (13.6GiB/s) | 20260507 20:29:07.100 | INFO | Analyzing chunk frames = [2372608,2373120) | 20260507 20:29:07.100 | INFO | MemInfo: { | 20260507 20:29:07.100 | INFO | "gpu" : " 17 GB", | 20260507 20:29:07.100 | INFO | "huge" : "", | 20260507 20:29:07.100 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:07.100 | INFO | "malloc" : " 14 GB", | 20260507 20:29:07.100 | INFO | "pinned" : " 29 GB" | 20260507 20:29:07.100 | INFO | } | 20260507 20:29:07.100 | INFO | Perf report for the last 5.13862 seconds: | 20260507 20:29:07.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:07.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:29:07.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:29:07.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:07.100 | INFO | Uploaded 36GiB to GPU in 2.5811s (13.9475GiB/s) | 20260507 20:29:07.101 | INFO | Downloaded 34GiB to GPU in 2.47149s (13.7569GiB/s) | 20260507 20:29:12.238 | INFO | Analyzing chunk frames = [2373120,2373632) | 20260507 20:29:12.238 | INFO | MemInfo: { | 20260507 20:29:12.238 | INFO | "gpu" : " 17 GB", | 20260507 20:29:12.238 | INFO | "huge" : "", | 20260507 20:29:12.239 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:12.239 | INFO | "malloc" : " 14 GB", | 20260507 20:29:12.239 | INFO | "pinned" : " 29 GB" | 20260507 20:29:12.239 | INFO | } | 20260507 20:29:12.239 | INFO | Perf report for the last 5.13833 seconds: | 20260507 20:29:12.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:12.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 20:29:12.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:29:12.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:12.239 | INFO | Uploaded 36GiB to GPU in 2.46893s (14.5812GiB/s) | 20260507 20:29:12.239 | INFO | Downloaded 34GiB to GPU in 2.40594s (14.1317GiB/s) | 20260507 20:29:17.378 | INFO | Analyzing chunk frames = [2373632,2374144) | 20260507 20:29:17.378 | INFO | MemInfo: { | 20260507 20:29:17.378 | INFO | "gpu" : " 17 GB", | 20260507 20:29:17.378 | INFO | "huge" : "", | 20260507 20:29:17.379 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:17.379 | INFO | "malloc" : " 14 GB", | 20260507 20:29:17.379 | INFO | "pinned" : " 29 GB" | 20260507 20:29:17.379 | INFO | } | 20260507 20:29:17.379 | INFO | Perf report for the last 5.13997 seconds: | 20260507 20:29:17.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:17.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 20:29:17.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:29:17.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:17.379 | INFO | Uploaded 36GiB to GPU in 2.3716s (15.1796GiB/s) | 20260507 20:29:17.379 | INFO | Downloaded 34GiB to GPU in 2.49531s (13.6256GiB/s) | 20260507 20:29:22.518 | INFO | Analyzing chunk frames = [2374144,2374656) | 20260507 20:29:22.518 | INFO | MemInfo: { | 20260507 20:29:22.518 | INFO | "gpu" : " 17 GB", | 20260507 20:29:22.518 | INFO | "huge" : "", | 20260507 20:29:22.518 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:22.518 | INFO | "malloc" : " 14 GB", | 20260507 20:29:22.518 | INFO | "pinned" : " 29 GB" | 20260507 20:29:22.518 | INFO | } | 20260507 20:29:22.518 | INFO | Perf report for the last 5.13923 seconds: | 20260507 20:29:22.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:22.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 20:29:22.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:29:22.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:22.518 | INFO | Uploaded 36GiB to GPU in 2.51217s (14.3303GiB/s) | 20260507 20:29:22.518 | INFO | Downloaded 34GiB to GPU in 2.41343s (14.0878GiB/s) | 20260507 20:29:27.661 | INFO | Analyzing chunk frames = [2374656,2375168) | 20260507 20:29:27.661 | INFO | MemInfo: { | 20260507 20:29:27.661 | INFO | "gpu" : " 17 GB", | 20260507 20:29:27.661 | INFO | "huge" : "", | 20260507 20:29:27.661 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:27.661 | INFO | "malloc" : " 14 GB", | 20260507 20:29:27.661 | INFO | "pinned" : " 29 GB" | 20260507 20:29:27.661 | INFO | } | 20260507 20:29:27.661 | INFO | Perf report for the last 5.14284 seconds: | 20260507 20:29:27.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:27.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 20:29:27.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:29:27.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:27.661 | INFO | Uploaded 36GiB to GPU in 2.52156s (14.2769GiB/s) | 20260507 20:29:27.661 | INFO | Downloaded 34GiB to GPU in 2.43386s (13.9696GiB/s) | 20260507 20:29:32.798 | INFO | Analyzing chunk frames = [2375168,2375680) | 20260507 20:29:32.798 | INFO | MemInfo: { | 20260507 20:29:32.798 | INFO | "gpu" : " 17 GB", | 20260507 20:29:32.798 | INFO | "huge" : "", | 20260507 20:29:32.798 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:32.798 | INFO | "malloc" : " 14 GB", | 20260507 20:29:32.798 | INFO | "pinned" : " 29 GB" | 20260507 20:29:32.798 | INFO | } | 20260507 20:29:32.798 | INFO | Perf report for the last 5.13701 seconds: | 20260507 20:29:32.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:32.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:29:32.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:29:32.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:32.798 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:29:32.798 | INFO | Total mem usage: 16.4 GiB | 20260507 20:29:32.798 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:29:32.798 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -5.93%) | 20260507 20:29:32.798 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:29:32.798 | INFO | High Water Breakdown: | 20260507 20:29:32.798 | INFO | 60.5% : 9.37 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:29:32.798 | INFO | 39.4% : 6.09 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:29:32.798 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:29:32.798 | INFO | ------------------------------------------ | 20260507 20:29:32.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:29:32.798 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:29:32.798 | INFO | Total mem usage: 26.5 GiB | 20260507 20:29:32.798 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:29:32.798 | INFO | Uploaded 36GiB to GPU in 2.52578s (14.253GiB/s) | 20260507 20:29:32.798 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:29:32.798 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:29:32.798 | INFO | High Water Breakdown: | 20260507 20:29:32.798 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:29:32.798 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:29:32.798 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:29:32.798 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:29:32.798 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:29:32.798 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:29:32.798 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:29:32.798 | INFO | Downloaded 34GiB to GPU in 2.42009s (14.049GiB/s) | 20260507 20:29:32.798 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:29:32.798 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:29:32.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:29:32.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:29:32.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:29:32.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:29:32.798 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:29:32.798 | INFO | ------------------------------------------ | 20260507 20:29:32.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:29:32.798 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:29:32.798 | INFO | Total mem usage: 27.8 GiB | 20260507 20:29:32.798 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:29:32.798 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:29:32.799 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:29:32.799 | INFO | High Water Breakdown: | 20260507 20:29:32.799 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:29:32.799 | INFO | ------------------------------------------ | 20260507 20:29:32.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:29:32.799 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:29:32.799 | INFO | Total mem usage: 15.7 GiB | 20260507 20:29:32.799 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:29:32.799 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:29:32.799 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:29:32.799 | INFO | High Water Breakdown: | 20260507 20:29:32.799 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:29:32.799 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:29:32.799 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:29:32.799 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:29:32.799 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:29:32.799 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:29:32.799 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:29:32.799 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:29:32.799 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:29:32.799 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:29:32.799 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:29:32.799 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:29:32.799 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:29:32.799 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:29:32.799 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:29:32.799 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:29:32.799 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:29:32.799 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:29:32.799 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:29:32.799 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:29:32.799 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:29:32.799 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:29:32.799 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:29:32.799 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:29:32.799 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:29:32.799 | INFO | ------------------------------------------ | 20260507 20:29:32.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:29:32.799 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:29:32.799 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:29:32.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:29:32.799 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.58e+01, 1.42e+01 / 2.07e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.27e-02 / 5.56e-03 | 20260507 20:29:32.799 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.49e+00, 5.94e-01 / 2.37e-01 | 20260507 20:29:32.799 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.27e-01 | 20260507 20:29:32.799 | INFO | PerfReport: 15.93%, DME -- 4.22e+01 / 4.55e+01, 4.05e+00 / 1.92e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.72e+00 / 1.73e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.46e-01 / 2.07e-01 | 20260507 20:29:32.799 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.38e+00, 3.00e-01 / 1.68e-01 | 20260507 20:29:32.799 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.69e-01 / 4.02e-01, 6.01e-02 / 2.96e-02 | 20260507 20:29:32.799 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:29:32.799 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:29:32.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:29:32.799 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 24.36%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 24.27%, Download -- 2.59e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 30.94%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 2.63%, LockDownload -- 2.82e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 15.80%, LockCompute -- 1.69e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:29:32.799 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260507 20:29:32.895 | INFO | Starting a round of Baz IO | 20260507 20:29:36.372 | INFO | Populating RTMetrics report | 20260507 20:29:36.372 | INFO | RTMetrics report populated | 20260507 20:29:36.372 | INFO | Opening tmp RTMetrics json file | 20260507 20:29:36.398 | INFO | Serializing RTMetrics to json | 20260507 20:29:36.400 | INFO | Serialization done, swapping tmp file now... | 20260507 20:29:36.401 | INFO | RTMetrics done | 20260507 20:29:36.497 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.601 seconds | 20260507 20:29:36.497 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.44% / 0.01% / 55.55% | 20260507 20:29:36.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.00442% / 1.58946e-05% / 4.37101e-05% | 20260507 20:29:36.497 | INFO | Baz IO overall: | 20260507 20:29:36.497 | INFO | BytesWritten : 1078647169323 | 20260507 20:29:36.497 | INFO | NumEvents : 522906911324 | 20260507 20:29:37.936 | INFO | Analyzing chunk frames = [2375680,2376192) | 20260507 20:29:37.936 | INFO | MemInfo: { | 20260507 20:29:37.936 | INFO | "gpu" : " 17 GB", | 20260507 20:29:37.936 | INFO | "huge" : "", | 20260507 20:29:37.936 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:37.936 | INFO | "malloc" : " 14 GB", | 20260507 20:29:37.936 | INFO | "pinned" : " 29 GB" | 20260507 20:29:37.936 | INFO | } | 20260507 20:29:37.936 | INFO | Perf report for the last 5.13844 seconds: | 20260507 20:29:37.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:37.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:37.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:37.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:37.936 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 20:29:37.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:37.936 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 20:29:37.936 | INFO | PRE_HQ : 66.1%, 1.7, 57.9ms, 0 | 20260507 20:29:37.936 | INFO | BAZWRITER : 69.7%, 3.9, 140.7ms, 0 | 20260507 20:29:37.936 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:29:37.936 | INFO | Uploaded 36GiB to GPU in 2.55082s (14.1131GiB/s) | 20260507 20:29:37.936 | INFO | Downloaded 38GiB to GPU in 2.70917s (14.0264GiB/s) | 20260507 20:29:43.077 | INFO | Analyzing chunk frames = [2376192,2376704) | 20260507 20:29:43.077 | INFO | MemInfo: { | 20260507 20:29:43.077 | INFO | "gpu" : " 17 GB", | 20260507 20:29:43.077 | INFO | "huge" : "", | 20260507 20:29:43.077 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:43.077 | INFO | "malloc" : " 14 GB", | 20260507 20:29:43.077 | INFO | "pinned" : " 29 GB" | 20260507 20:29:43.077 | INFO | } | 20260507 20:29:43.077 | INFO | Perf report for the last 5.14055 seconds: | 20260507 20:29:43.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:43.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:29:43.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:29:43.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:43.077 | INFO | Uploaded 36GiB to GPU in 2.47407s (14.551GiB/s) | 20260507 20:29:43.077 | INFO | Downloaded 34GiB to GPU in 2.4913s (13.6475GiB/s) | 20260507 20:29:48.215 | INFO | Analyzing chunk frames = [2376704,2377216) | 20260507 20:29:48.215 | INFO | MemInfo: { | 20260507 20:29:48.215 | INFO | "gpu" : " 17 GB", | 20260507 20:29:48.215 | INFO | "huge" : "", | 20260507 20:29:48.215 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:48.215 | INFO | "malloc" : " 14 GB", | 20260507 20:29:48.215 | INFO | "pinned" : " 29 GB" | 20260507 20:29:48.215 | INFO | } | 20260507 20:29:48.215 | INFO | Perf report for the last 5.13812 seconds: | 20260507 20:29:48.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:48.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 20:29:48.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:29:48.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:48.215 | INFO | Uploaded 36GiB to GPU in 2.50704s (14.3595GiB/s) | 20260507 20:29:48.215 | INFO | Downloaded 34GiB to GPU in 2.48017s (13.7087GiB/s) | 20260507 20:29:53.354 | INFO | Analyzing chunk frames = [2377216,2377728) | 20260507 20:29:53.354 | INFO | MemInfo: { | 20260507 20:29:53.354 | INFO | "gpu" : " 17 GB", | 20260507 20:29:53.354 | INFO | "huge" : "", | 20260507 20:29:53.354 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:53.354 | INFO | "malloc" : " 14 GB", | 20260507 20:29:53.354 | INFO | "pinned" : " 29 GB" | 20260507 20:29:53.354 | INFO | } | 20260507 20:29:53.354 | INFO | Perf report for the last 5.13894 seconds: | 20260507 20:29:53.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:53.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 20:29:53.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 20:29:53.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:53.354 | INFO | Uploaded 36GiB to GPU in 2.49714s (14.4165GiB/s) | 20260507 20:29:53.354 | INFO | Downloaded 34GiB to GPU in 2.50314s (13.5829GiB/s) | 20260507 20:29:58.494 | INFO | Analyzing chunk frames = [2377728,2378240) | 20260507 20:29:58.494 | INFO | MemInfo: { | 20260507 20:29:58.494 | INFO | "gpu" : " 17 GB", | 20260507 20:29:58.494 | INFO | "huge" : "", | 20260507 20:29:58.494 | INFO | "huge_shared" : " 29 GB", | 20260507 20:29:58.494 | INFO | "malloc" : " 14 GB", | 20260507 20:29:58.494 | INFO | "pinned" : " 29 GB" | 20260507 20:29:58.494 | INFO | } | 20260507 20:29:58.494 | INFO | Perf report for the last 5.13989 seconds: | 20260507 20:29:58.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:29:58.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 20:29:58.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:29:58.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:29:58.494 | INFO | Uploaded 36GiB to GPU in 2.35002s (15.319GiB/s) | 20260507 20:29:58.494 | INFO | Downloaded 34GiB to GPU in 2.49998s (13.6001GiB/s) | 20260507 20:30:03.632 | INFO | Analyzing chunk frames = [2378240,2378752) | 20260507 20:30:03.632 | INFO | MemInfo: { | 20260507 20:30:03.632 | INFO | "gpu" : " 17 GB", | 20260507 20:30:03.632 | INFO | "huge" : "", | 20260507 20:30:03.632 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:03.632 | INFO | "malloc" : " 14 GB", | 20260507 20:30:03.632 | INFO | "pinned" : " 29 GB" | 20260507 20:30:03.632 | INFO | } | 20260507 20:30:03.632 | INFO | Perf report for the last 5.13867 seconds: | 20260507 20:30:03.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:03.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.632 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 20:30:03.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.632 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 20:30:03.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:03.633 | INFO | Uploaded 36GiB to GPU in 2.45448s (14.6671GiB/s) | 20260507 20:30:03.633 | INFO | Downloaded 34GiB to GPU in 2.42374s (14.0279GiB/s) | 20260507 20:30:08.772 | INFO | Analyzing chunk frames = [2378752,2379264) | 20260507 20:30:08.772 | INFO | MemInfo: { | 20260507 20:30:08.772 | INFO | "gpu" : " 17 GB", | 20260507 20:30:08.772 | INFO | "huge" : "", | 20260507 20:30:08.772 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:08.772 | INFO | "malloc" : " 14 GB", | 20260507 20:30:08.772 | INFO | "pinned" : " 29 GB" | 20260507 20:30:08.772 | INFO | } | 20260507 20:30:08.772 | INFO | Perf report for the last 5.13967 seconds: | 20260507 20:30:08.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:08.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:30:08.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:30:08.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:08.772 | INFO | Uploaded 36GiB to GPU in 2.52354s (14.2657GiB/s) | 20260507 20:30:08.772 | INFO | Downloaded 34GiB to GPU in 2.45822s (13.8311GiB/s) | 20260507 20:30:13.912 | INFO | Analyzing chunk frames = [2379264,2379776) | 20260507 20:30:13.912 | INFO | MemInfo: { | 20260507 20:30:13.912 | INFO | "gpu" : " 17 GB", | 20260507 20:30:13.912 | INFO | "huge" : "", | 20260507 20:30:13.912 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:13.912 | INFO | "malloc" : " 14 GB", | 20260507 20:30:13.912 | INFO | "pinned" : " 29 GB" | 20260507 20:30:13.912 | INFO | } | 20260507 20:30:13.912 | INFO | Perf report for the last 5.13977 seconds: | 20260507 20:30:13.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:13.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:30:13.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:30:13.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:13.912 | INFO | Uploaded 36GiB to GPU in 2.48566s (14.4831GiB/s) | 20260507 20:30:13.912 | INFO | Downloaded 34GiB to GPU in 2.43094s (13.9864GiB/s) | 20260507 20:30:14.009 | INFO | Starting a round of Baz IO | 20260507 20:30:17.490 | INFO | Populating RTMetrics report | 20260507 20:30:17.490 | INFO | RTMetrics report populated | 20260507 20:30:17.490 | INFO | Opening tmp RTMetrics json file | 20260507 20:30:17.518 | INFO | Serializing RTMetrics to json | 20260507 20:30:17.520 | INFO | Serialization done, swapping tmp file now... | 20260507 20:30:17.520 | INFO | RTMetrics done | 20260507 20:30:17.595 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.586 seconds | 20260507 20:30:17.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.43% / 0.01% / 55.56% | 20260507 20:30:17.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.99643% / 1.19209e-05% / 7.94729e-05% | 20260507 20:30:17.595 | INFO | Baz IO overall: | 20260507 20:30:17.595 | INFO | BytesWritten : 1080555783058 | 20260507 20:30:17.595 | INFO | NumEvents : 523831215968 | 20260507 20:30:19.050 | INFO | Analyzing chunk frames = [2379776,2380288) | 20260507 20:30:19.050 | INFO | MemInfo: { | 20260507 20:30:19.050 | INFO | "gpu" : " 17 GB", | 20260507 20:30:19.051 | INFO | "huge" : "", | 20260507 20:30:19.051 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:19.051 | INFO | "malloc" : " 14 GB", | 20260507 20:30:19.051 | INFO | "pinned" : " 29 GB" | 20260507 20:30:19.051 | INFO | } | 20260507 20:30:19.051 | INFO | Perf report for the last 5.13871 seconds: | 20260507 20:30:19.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:19.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:19.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:19.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:19.051 | INFO | ANALYSIS : 68.9%, 3.0, 107.0ms, 0 | 20260507 20:30:19.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:19.051 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:30:19.051 | INFO | PRE_HQ : 66.1%, 1.7, 58.2ms, 0 | 20260507 20:30:19.051 | INFO | BAZWRITER : 69.4%, 3.7, 132.4ms, 0 | 20260507 20:30:19.051 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:30:19.051 | INFO | Uploaded 36GiB to GPU in 2.56706s (14.0238GiB/s) | 20260507 20:30:19.051 | INFO | Downloaded 38GiB to GPU in 2.70316s (14.0576GiB/s) | 20260507 20:30:24.190 | INFO | Analyzing chunk frames = [2380288,2380800) | 20260507 20:30:24.190 | INFO | MemInfo: { | 20260507 20:30:24.190 | INFO | "gpu" : " 17 GB", | 20260507 20:30:24.190 | INFO | "huge" : "", | 20260507 20:30:24.190 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:24.190 | INFO | "malloc" : " 14 GB", | 20260507 20:30:24.190 | INFO | "pinned" : " 29 GB" | 20260507 20:30:24.190 | INFO | } | 20260507 20:30:24.191 | INFO | Perf report for the last 5.13987 seconds: | 20260507 20:30:24.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:24.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:30:24.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:30:24.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:24.191 | INFO | Uploaded 36GiB to GPU in 2.56594s (14.0299GiB/s) | 20260507 20:30:24.191 | INFO | Downloaded 34GiB to GPU in 2.48724s (13.6697GiB/s) | 20260507 20:30:29.330 | INFO | Analyzing chunk frames = [2380800,2381312) | 20260507 20:30:29.330 | INFO | MemInfo: { | 20260507 20:30:29.330 | INFO | "gpu" : " 17 GB", | 20260507 20:30:29.330 | INFO | "huge" : "", | 20260507 20:30:29.330 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:29.330 | INFO | "malloc" : " 14 GB", | 20260507 20:30:29.330 | INFO | "pinned" : " 29 GB" | 20260507 20:30:29.330 | INFO | } | 20260507 20:30:29.330 | INFO | Perf report for the last 5.13943 seconds: | 20260507 20:30:29.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:29.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 20:30:29.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:30:29.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:29.330 | INFO | Uploaded 36GiB to GPU in 2.56144s (14.0546GiB/s) | 20260507 20:30:29.330 | INFO | Downloaded 34GiB to GPU in 2.47268s (13.7503GiB/s) | 20260507 20:30:34.469 | INFO | Analyzing chunk frames = [2381312,2381824) | 20260507 20:30:34.469 | INFO | MemInfo: { | 20260507 20:30:34.469 | INFO | "gpu" : " 17 GB", | 20260507 20:30:34.469 | INFO | "huge" : "", | 20260507 20:30:34.469 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:34.469 | INFO | "malloc" : " 14 GB", | 20260507 20:30:34.469 | INFO | "pinned" : " 29 GB" | 20260507 20:30:34.469 | INFO | } | 20260507 20:30:34.469 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:30:34.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:34.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:30:34.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:30:34.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:34.469 | INFO | Uploaded 36GiB to GPU in 2.49084s (14.453GiB/s) | 20260507 20:30:34.469 | INFO | Downloaded 34GiB to GPU in 2.41168s (14.0981GiB/s) | 20260507 20:30:39.608 | INFO | Analyzing chunk frames = [2381824,2382336) | 20260507 20:30:39.608 | INFO | MemInfo: { | 20260507 20:30:39.608 | INFO | "gpu" : " 17 GB", | 20260507 20:30:39.608 | INFO | "huge" : "", | 20260507 20:30:39.608 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:39.608 | INFO | "malloc" : " 14 GB", | 20260507 20:30:39.608 | INFO | "pinned" : " 29 GB" | 20260507 20:30:39.608 | INFO | } | 20260507 20:30:39.608 | INFO | Perf report for the last 5.13915 seconds: | 20260507 20:30:39.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:39.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.608 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:30:39.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.608 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:30:39.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:39.609 | INFO | Uploaded 36GiB to GPU in 2.31779s (15.532GiB/s) | 20260507 20:30:39.609 | INFO | Downloaded 34GiB to GPU in 2.50525s (13.5715GiB/s) | 20260507 20:30:44.747 | INFO | Analyzing chunk frames = [2382336,2382848) | 20260507 20:30:44.747 | INFO | MemInfo: { | 20260507 20:30:44.747 | INFO | "gpu" : " 17 GB", | 20260507 20:30:44.747 | INFO | "huge" : "", | 20260507 20:30:44.747 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:44.747 | INFO | "malloc" : " 14 GB", | 20260507 20:30:44.747 | INFO | "pinned" : " 29 GB" | 20260507 20:30:44.747 | INFO | } | 20260507 20:30:44.747 | INFO | Perf report for the last 5.13908 seconds: | 20260507 20:30:44.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:44.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.747 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:30:44.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.748 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:30:44.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:44.748 | INFO | Uploaded 36GiB to GPU in 2.48883s (14.4646GiB/s) | 20260507 20:30:44.748 | INFO | Downloaded 34GiB to GPU in 2.42875s (13.999GiB/s) | 20260507 20:30:49.890 | INFO | Analyzing chunk frames = [2382848,2383360) | 20260507 20:30:49.890 | INFO | MemInfo: { | 20260507 20:30:49.890 | INFO | "gpu" : " 17 GB", | 20260507 20:30:49.890 | INFO | "huge" : "", | 20260507 20:30:49.890 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:49.890 | INFO | "malloc" : " 14 GB", | 20260507 20:30:49.890 | INFO | "pinned" : " 29 GB" | 20260507 20:30:49.890 | INFO | } | 20260507 20:30:49.890 | INFO | Perf report for the last 5.14246 seconds: | 20260507 20:30:49.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:49.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:30:49.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:30:49.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:49.890 | INFO | Uploaded 36GiB to GPU in 2.52864s (14.2369GiB/s) | 20260507 20:30:49.890 | INFO | Downloaded 34GiB to GPU in 2.46089s (13.8161GiB/s) | 20260507 20:30:55.026 | INFO | Analyzing chunk frames = [2383360,2383872) | 20260507 20:30:55.026 | INFO | MemInfo: { | 20260507 20:30:55.026 | INFO | "gpu" : " 17 GB", | 20260507 20:30:55.026 | INFO | "huge" : "", | 20260507 20:30:55.027 | INFO | "huge_shared" : " 29 GB", | 20260507 20:30:55.027 | INFO | "malloc" : " 14 GB", | 20260507 20:30:55.027 | INFO | "pinned" : " 29 GB" | 20260507 20:30:55.027 | INFO | } | 20260507 20:30:55.027 | INFO | Perf report for the last 5.13671 seconds: | 20260507 20:30:55.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:30:55.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:30:55.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:30:55.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:30:55.027 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:30:55.027 | INFO | Total mem usage: 16.4 GiB | 20260507 20:30:55.027 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:30:55.027 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -9.75%) | 20260507 20:30:55.027 | INFO | Uploaded 36GiB to GPU in 2.48545s (14.4843GiB/s) | 20260507 20:30:55.027 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:30:55.027 | INFO | High Water Breakdown: | 20260507 20:30:55.027 | INFO | Downloaded 34GiB to GPU in 2.44041s (13.9321GiB/s) | 20260507 20:30:55.027 | INFO | 59.9% : 8.96 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:30:55.027 | INFO | 39.9% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:30:55.027 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:30:55.027 | INFO | ------------------------------------------ | 20260507 20:30:55.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:30:55.027 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:30:55.027 | INFO | Total mem usage: 26.5 GiB | 20260507 20:30:55.027 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:30:55.027 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:30:55.027 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:30:55.027 | INFO | High Water Breakdown: | 20260507 20:30:55.027 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:30:55.027 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:30:55.027 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:30:55.027 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:30:55.027 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:30:55.027 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:30:55.027 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:30:55.027 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:30:55.027 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:30:55.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:30:55.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:30:55.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:30:55.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:30:55.027 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:30:55.027 | INFO | ------------------------------------------ | 20260507 20:30:55.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:30:55.027 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:30:55.027 | INFO | Total mem usage: 27.8 GiB | 20260507 20:30:55.027 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:30:55.027 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:30:55.027 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:30:55.027 | INFO | High Water Breakdown: | 20260507 20:30:55.027 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:30:55.027 | INFO | ------------------------------------------ | 20260507 20:30:55.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:30:55.027 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:30:55.027 | INFO | Total mem usage: 15.7 GiB | 20260507 20:30:55.027 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:30:55.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:30:55.027 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:30:55.027 | INFO | High Water Breakdown: | 20260507 20:30:55.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:30:55.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:30:55.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:30:55.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:30:55.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:30:55.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:30:55.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:30:55.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:30:55.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:30:55.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:30:55.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:30:55.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:30:55.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:30:55.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:30:55.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:30:55.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:30:55.027 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:30:55.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:30:55.027 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:30:55.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:30:55.028 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:30:55.028 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:30:55.028 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:30:55.028 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:30:55.028 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:30:55.028 | INFO | ------------------------------------------ | 20260507 20:30:55.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:30:55.028 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:30:55.028 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:30:55.028 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:30:55.028 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.58e+01, 1.40e+01 / 1.68e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.19e-02 / 4.31e-03 | 20260507 20:30:55.028 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.49e+00, 6.02e-01 / 2.28e-01 | 20260507 20:30:55.028 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.05e-01 | 20260507 20:30:55.028 | INFO | PerfReport: 15.95%, DME -- 4.21e+01 / 4.55e+01, 3.96e+00 / 1.48e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.67e+00 / 1.70e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.43e-01 / 2.03e-01 | 20260507 20:30:55.028 | INFO | PerfReport: 3.60%, Metrics -- 1.19e+00 / 1.42e+00, 3.00e-01 / 1.52e-01 | 20260507 20:30:55.028 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.68e-01 / 4.04e-01, 5.95e-02 / 2.73e-02 | 20260507 20:30:55.028 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:30:55.028 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:30:55.028 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:30:55.028 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 24.26%, Upload -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 24.30%, Download -- 2.60e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 30.88%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 1.99%, LockUpload -- 2.13e+00 / 0.00e+00, 5.20e+00 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 2.66%, LockDownload -- 2.85e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:30:55.028 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.47e-02 / 0.00e+00 | 20260507 20:30:55.123 | INFO | Starting a round of Baz IO | 20260507 20:30:58.593 | INFO | Populating RTMetrics report | 20260507 20:30:58.594 | INFO | RTMetrics report populated | 20260507 20:30:58.594 | INFO | Opening tmp RTMetrics json file | 20260507 20:30:58.614 | INFO | Serializing RTMetrics to json | 20260507 20:30:58.617 | INFO | Serialization done, swapping tmp file now... | 20260507 20:30:58.618 | INFO | RTMetrics done | 20260507 20:30:58.693 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.569 seconds | 20260507 20:30:58.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.42% / 0.01% / 55.57% | 20260507 20:30:58.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.99342% / 1.19209e-05% / 5.5631e-05% | 20260507 20:30:58.693 | INFO | Baz IO overall: | 20260507 20:30:58.693 | INFO | BytesWritten : 1082464282926 | 20260507 20:30:58.693 | INFO | NumEvents : 524755305572 | 20260507 20:31:00.166 | INFO | Analyzing chunk frames = [2383872,2384384) | 20260507 20:31:00.166 | INFO | MemInfo: { | 20260507 20:31:00.166 | INFO | "gpu" : " 17 GB", | 20260507 20:31:00.166 | INFO | "huge" : "", | 20260507 20:31:00.166 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:00.166 | INFO | "malloc" : " 14 GB", | 20260507 20:31:00.166 | INFO | "pinned" : " 29 GB" | 20260507 20:31:00.166 | INFO | } | 20260507 20:31:00.166 | INFO | Perf report for the last 5.13983 seconds: | 20260507 20:31:00.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:00.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:00.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:00.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:00.167 | INFO | ANALYSIS : 68.6%, 2.9, 105.8ms, 0 | 20260507 20:31:00.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:00.167 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 20:31:00.167 | INFO | PRE_HQ : 66.9%, 1.7, 59.9ms, 0 | 20260507 20:31:00.167 | INFO | BAZWRITER : 69.0%, 3.7, 132.9ms, 0 | 20260507 20:31:00.167 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 20:31:00.167 | INFO | Uploaded 36GiB to GPU in 2.5563s (14.0829GiB/s) | 20260507 20:31:00.167 | INFO | Downloaded 38GiB to GPU in 2.70604s (14.0427GiB/s) | 20260507 20:31:05.305 | INFO | Analyzing chunk frames = [2384384,2384896) | 20260507 20:31:05.305 | INFO | MemInfo: { | 20260507 20:31:05.305 | INFO | "gpu" : " 17 GB", | 20260507 20:31:05.305 | INFO | "huge" : "", | 20260507 20:31:05.305 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:05.305 | INFO | "malloc" : " 14 GB", | 20260507 20:31:05.305 | INFO | "pinned" : " 29 GB" | 20260507 20:31:05.305 | INFO | } | 20260507 20:31:05.305 | INFO | Perf report for the last 5.13826 seconds: | 20260507 20:31:05.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:05.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 20:31:05.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:31:05.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:05.305 | INFO | Uploaded 36GiB to GPU in 2.53797s (14.1845GiB/s) | 20260507 20:31:05.305 | INFO | Downloaded 34GiB to GPU in 2.50386s (13.579GiB/s) | 20260507 20:31:10.443 | INFO | Analyzing chunk frames = [2384896,2385408) | 20260507 20:31:10.443 | INFO | MemInfo: { | 20260507 20:31:10.443 | INFO | "gpu" : " 17 GB", | 20260507 20:31:10.443 | INFO | "huge" : "", | 20260507 20:31:10.443 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:10.443 | INFO | "malloc" : " 14 GB", | 20260507 20:31:10.443 | INFO | "pinned" : " 29 GB" | 20260507 20:31:10.443 | INFO | } | 20260507 20:31:10.443 | INFO | Perf report for the last 5.13849 seconds: | 20260507 20:31:10.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:10.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:31:10.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:31:10.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:10.447 | INFO | Uploaded 36GiB to GPU in 2.53835s (14.1825GiB/s) | 20260507 20:31:10.447 | INFO | Downloaded 34GiB to GPU in 2.46989s (13.7658GiB/s) | 20260507 20:31:15.586 | INFO | Analyzing chunk frames = [2385408,2385920) | 20260507 20:31:15.586 | INFO | MemInfo: { | 20260507 20:31:15.586 | INFO | "gpu" : " 17 GB", | 20260507 20:31:15.586 | INFO | "huge" : "", | 20260507 20:31:15.586 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:15.586 | INFO | "malloc" : " 14 GB", | 20260507 20:31:15.586 | INFO | "pinned" : " 29 GB" | 20260507 20:31:15.586 | INFO | } | 20260507 20:31:15.587 | INFO | Perf report for the last 5.1433 seconds: | 20260507 20:31:15.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:15.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260507 20:31:15.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:31:15.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:15.587 | INFO | Uploaded 36GiB to GPU in 2.57599s (13.9752GiB/s) | 20260507 20:31:15.587 | INFO | Downloaded 34GiB to GPU in 2.46675s (13.7833GiB/s) | 20260507 20:31:20.723 | INFO | Analyzing chunk frames = [2385920,2386432) | 20260507 20:31:20.723 | INFO | MemInfo: { | 20260507 20:31:20.723 | INFO | "gpu" : " 17 GB", | 20260507 20:31:20.723 | INFO | "huge" : "", | 20260507 20:31:20.723 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:20.723 | INFO | "malloc" : " 14 GB", | 20260507 20:31:20.723 | INFO | "pinned" : " 29 GB" | 20260507 20:31:20.723 | INFO | } | 20260507 20:31:20.723 | INFO | Perf report for the last 5.13658 seconds: | 20260507 20:31:20.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:20.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:31:20.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:31:20.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:20.723 | INFO | Uploaded 36GiB to GPU in 2.35s (15.3192GiB/s) | 20260507 20:31:20.723 | INFO | Downloaded 34GiB to GPU in 2.50049s (13.5973GiB/s) | 20260507 20:31:25.863 | INFO | Analyzing chunk frames = [2386432,2386944) | 20260507 20:31:25.863 | INFO | MemInfo: { | 20260507 20:31:25.863 | INFO | "gpu" : " 17 GB", | 20260507 20:31:25.863 | INFO | "huge" : "", | 20260507 20:31:25.863 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:25.863 | INFO | "malloc" : " 14 GB", | 20260507 20:31:25.863 | INFO | "pinned" : " 29 GB" | 20260507 20:31:25.863 | INFO | } | 20260507 20:31:25.863 | INFO | Perf report for the last 5.14003 seconds: | 20260507 20:31:25.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:25.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:31:25.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:31:25.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:25.863 | INFO | Uploaded 36GiB to GPU in 2.46744s (14.59GiB/s) | 20260507 20:31:25.863 | INFO | Downloaded 34GiB to GPU in 2.43667s (13.9534GiB/s) | 20260507 20:31:31.002 | INFO | Analyzing chunk frames = [2386944,2387456) | 20260507 20:31:31.002 | INFO | MemInfo: { | 20260507 20:31:31.002 | INFO | "gpu" : " 17 GB", | 20260507 20:31:31.002 | INFO | "huge" : "", | 20260507 20:31:31.002 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:31.002 | INFO | "malloc" : " 14 GB", | 20260507 20:31:31.002 | INFO | "pinned" : " 29 GB" | 20260507 20:31:31.002 | INFO | } | 20260507 20:31:31.002 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:31:31.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:31.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 20:31:31.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 20:31:31.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:31.002 | INFO | Uploaded 36GiB to GPU in 2.49131s (14.4502GiB/s) | 20260507 20:31:31.002 | INFO | Downloaded 34GiB to GPU in 2.46616s (13.7866GiB/s) | 20260507 20:31:36.141 | INFO | Analyzing chunk frames = [2387456,2387968) | 20260507 20:31:36.141 | INFO | MemInfo: { | 20260507 20:31:36.141 | INFO | "gpu" : " 17 GB", | 20260507 20:31:36.141 | INFO | "huge" : "", | 20260507 20:31:36.142 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:36.142 | INFO | "malloc" : " 14 GB", | 20260507 20:31:36.142 | INFO | "pinned" : " 29 GB" | 20260507 20:31:36.142 | INFO | } | 20260507 20:31:36.142 | INFO | Perf report for the last 5.13968 seconds: | 20260507 20:31:36.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:36.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:31:36.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | PRE_HQ : 37.1%, 1.0, 19.3ms, 0 | 20260507 20:31:36.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:36.142 | INFO | Uploaded 36GiB to GPU in 2.47904s (14.5217GiB/s) | 20260507 20:31:36.142 | INFO | Downloaded 34GiB to GPU in 2.44602s (13.9002GiB/s) | 20260507 20:31:36.255 | INFO | Starting a round of Baz IO | 20260507 20:31:39.752 | INFO | Populating RTMetrics report | 20260507 20:31:39.752 | INFO | RTMetrics report populated | 20260507 20:31:39.752 | INFO | Opening tmp RTMetrics json file | 20260507 20:31:39.757 | INFO | Serializing RTMetrics to json | 20260507 20:31:39.797 | INFO | Serialization done, swapping tmp file now... | 20260507 20:31:39.856 | INFO | RTMetrics done | 20260507 20:31:39.883 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.628 seconds | 20260507 20:31:39.883 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.42% / 0.01% / 55.58% | 20260507 20:31:39.883 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9933% / 7.94729e-06% / 3.97364e-05% | 20260507 20:31:39.883 | INFO | Baz IO overall: | 20260507 20:31:39.883 | INFO | BytesWritten : 1084373781640 | 20260507 20:31:39.883 | INFO | NumEvents : 525680311654 | 20260507 20:31:41.280 | INFO | Analyzing chunk frames = [2387968,2388480) | 20260507 20:31:41.280 | INFO | MemInfo: { | 20260507 20:31:41.280 | INFO | "gpu" : " 17 GB", | 20260507 20:31:41.280 | INFO | "huge" : "", | 20260507 20:31:41.280 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:41.280 | INFO | "malloc" : " 14 GB", | 20260507 20:31:41.280 | INFO | "pinned" : " 29 GB" | 20260507 20:31:41.280 | INFO | } | 20260507 20:31:41.280 | INFO | Perf report for the last 5.13872 seconds: | 20260507 20:31:41.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:41.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:41.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:41.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:41.280 | INFO | ANALYSIS : 69.5%, 2.9, 104.6ms, 0 | 20260507 20:31:41.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:41.280 | INFO | RT_METRICS : 47.6%, 1.0, 24.9ms, 0 | 20260507 20:31:41.280 | INFO | PRE_HQ : 68.4%, 1.8, 64.0ms, 0 | 20260507 20:31:41.280 | INFO | BAZWRITER : 70.2%, 4.1, 146.1ms, 0 | 20260507 20:31:41.280 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 20:31:41.281 | INFO | Uploaded 36GiB to GPU in 2.57675s (13.9711GiB/s) | 20260507 20:31:41.281 | INFO | Downloaded 38GiB to GPU in 2.72072s (13.9669GiB/s) | 20260507 20:31:46.419 | INFO | Analyzing chunk frames = [2388480,2388992) | 20260507 20:31:46.419 | INFO | MemInfo: { | 20260507 20:31:46.419 | INFO | "gpu" : " 17 GB", | 20260507 20:31:46.419 | INFO | "huge" : "", | 20260507 20:31:46.419 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:46.419 | INFO | "malloc" : " 14 GB", | 20260507 20:31:46.419 | INFO | "pinned" : " 29 GB" | 20260507 20:31:46.419 | INFO | } | 20260507 20:31:46.420 | INFO | Perf report for the last 5.13915 seconds: | 20260507 20:31:46.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:46.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:31:46.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:31:46.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:46.420 | INFO | Uploaded 36GiB to GPU in 2.5332s (14.2113GiB/s) | 20260507 20:31:46.420 | INFO | Downloaded 34GiB to GPU in 2.49541s (13.625GiB/s) | 20260507 20:31:51.558 | INFO | Analyzing chunk frames = [2388992,2389504) | 20260507 20:31:51.558 | INFO | MemInfo: { | 20260507 20:31:51.558 | INFO | "gpu" : " 17 GB", | 20260507 20:31:51.558 | INFO | "huge" : "", | 20260507 20:31:51.558 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:51.558 | INFO | "malloc" : " 14 GB", | 20260507 20:31:51.558 | INFO | "pinned" : " 29 GB" | 20260507 20:31:51.558 | INFO | } | 20260507 20:31:51.558 | INFO | Perf report for the last 5.13885 seconds: | 20260507 20:31:51.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:51.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.558 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:31:51.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.558 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:31:51.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:51.559 | INFO | Uploaded 36GiB to GPU in 2.58252s (13.9399GiB/s) | 20260507 20:31:51.559 | INFO | Downloaded 34GiB to GPU in 2.4723s (13.7524GiB/s) | 20260507 20:31:56.698 | INFO | Analyzing chunk frames = [2389504,2390016) | 20260507 20:31:56.698 | INFO | MemInfo: { | 20260507 20:31:56.698 | INFO | "gpu" : " 17 GB", | 20260507 20:31:56.698 | INFO | "huge" : "", | 20260507 20:31:56.698 | INFO | "huge_shared" : " 29 GB", | 20260507 20:31:56.698 | INFO | "malloc" : " 14 GB", | 20260507 20:31:56.698 | INFO | "pinned" : " 29 GB" | 20260507 20:31:56.698 | INFO | } | 20260507 20:31:56.698 | INFO | Perf report for the last 5.13972 seconds: | 20260507 20:31:56.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:31:56.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:31:56.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:31:56.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:31:56.698 | INFO | Uploaded 36GiB to GPU in 2.49731s (14.4155GiB/s) | 20260507 20:31:56.698 | INFO | Downloaded 34GiB to GPU in 2.40267s (14.1509GiB/s) | 20260507 20:32:01.838 | INFO | Analyzing chunk frames = [2390016,2390528) | 20260507 20:32:01.838 | INFO | MemInfo: { | 20260507 20:32:01.838 | INFO | "gpu" : " 17 GB", | 20260507 20:32:01.838 | INFO | "huge" : "", | 20260507 20:32:01.838 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:01.838 | INFO | "malloc" : " 14 GB", | 20260507 20:32:01.838 | INFO | "pinned" : " 29 GB" | 20260507 20:32:01.838 | INFO | } | 20260507 20:32:01.838 | INFO | Perf report for the last 5.14017 seconds: | 20260507 20:32:01.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:01.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.838 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:32:01.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.838 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:32:01.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:01.839 | INFO | Uploaded 36GiB to GPU in 2.40747s (14.9535GiB/s) | 20260507 20:32:01.839 | INFO | Downloaded 34GiB to GPU in 2.49784s (13.6118GiB/s) | 20260507 20:32:06.977 | INFO | Analyzing chunk frames = [2390528,2391040) | 20260507 20:32:06.977 | INFO | MemInfo: { | 20260507 20:32:06.977 | INFO | "gpu" : " 17 GB", | 20260507 20:32:06.977 | INFO | "huge" : "", | 20260507 20:32:06.977 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:06.977 | INFO | "malloc" : " 14 GB", | 20260507 20:32:06.978 | INFO | "pinned" : " 29 GB" | 20260507 20:32:06.978 | INFO | } | 20260507 20:32:06.978 | INFO | Perf report for the last 5.13932 seconds: | 20260507 20:32:06.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:06.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260507 20:32:06.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:32:06.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:06.978 | INFO | Uploaded 36GiB to GPU in 2.42721s (14.8319GiB/s) | 20260507 20:32:06.978 | INFO | Downloaded 34GiB to GPU in 2.38292s (14.2682GiB/s) | 20260507 20:32:12.116 | INFO | Analyzing chunk frames = [2391040,2391552) | 20260507 20:32:12.116 | INFO | MemInfo: { | 20260507 20:32:12.116 | INFO | "gpu" : " 17 GB", | 20260507 20:32:12.116 | INFO | "huge" : "", | 20260507 20:32:12.116 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:12.117 | INFO | "malloc" : " 14 GB", | 20260507 20:32:12.117 | INFO | "pinned" : " 29 GB" | 20260507 20:32:12.117 | INFO | } | 20260507 20:32:12.117 | INFO | Perf report for the last 5.13896 seconds: | 20260507 20:32:12.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:12.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 20:32:12.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:32:12.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:12.117 | INFO | Uploaded 36GiB to GPU in 2.52714s (14.2454GiB/s) | 20260507 20:32:12.117 | INFO | Downloaded 34GiB to GPU in 2.43111s (13.9854GiB/s) | 20260507 20:32:17.256 | INFO | Analyzing chunk frames = [2391552,2392064) | 20260507 20:32:17.256 | INFO | MemInfo: { | 20260507 20:32:17.256 | INFO | "gpu" : " 17 GB", | 20260507 20:32:17.256 | INFO | "huge" : "", | 20260507 20:32:17.256 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:17.256 | INFO | "malloc" : " 14 GB", | 20260507 20:32:17.256 | INFO | "pinned" : " 29 GB" | 20260507 20:32:17.256 | INFO | } | 20260507 20:32:17.256 | INFO | Perf report for the last 5.13933 seconds: | 20260507 20:32:17.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:17.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 20:32:17.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 20:32:17.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:17.256 | INFO | Uploaded 36GiB to GPU in 2.53525s (14.1998GiB/s) | 20260507 20:32:17.256 | INFO | Downloaded 34GiB to GPU in 2.43214s (13.9795GiB/s) | 20260507 20:32:17.256 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:32:17.256 | INFO | Total mem usage: 16.4 GiB | 20260507 20:32:17.256 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:32:17.256 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.5%) | 20260507 20:32:17.256 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:32:17.256 | INFO | High Water Breakdown: | 20260507 20:32:17.256 | INFO | 59.8% : 8.88 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:32:17.256 | INFO | 40% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:32:17.256 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:32:17.256 | INFO | ------------------------------------------ | 20260507 20:32:17.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:32:17.256 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:32:17.256 | INFO | Total mem usage: 26.5 GiB | 20260507 20:32:17.256 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:32:17.256 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 20:32:17.256 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 20:32:17.256 | INFO | High Water Breakdown: | 20260507 20:32:17.256 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:32:17.256 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:32:17.256 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:32:17.257 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:32:17.257 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:32:17.257 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:32:17.257 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:32:17.257 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:32:17.257 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:32:17.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:32:17.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:32:17.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:32:17.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:32:17.257 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:32:17.257 | INFO | ------------------------------------------ | 20260507 20:32:17.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:32:17.257 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:32:17.257 | INFO | Total mem usage: 27.8 GiB | 20260507 20:32:17.257 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:32:17.257 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:32:17.257 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:32:17.257 | INFO | High Water Breakdown: | 20260507 20:32:17.257 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:32:17.257 | INFO | ------------------------------------------ | 20260507 20:32:17.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:32:17.257 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:32:17.257 | INFO | Total mem usage: 15.7 GiB | 20260507 20:32:17.257 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:32:17.257 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:32:17.257 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:32:17.257 | INFO | High Water Breakdown: | 20260507 20:32:17.257 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:32:17.257 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:32:17.257 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:32:17.257 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:32:17.257 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:32:17.257 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:32:17.257 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:32:17.257 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:32:17.257 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:32:17.257 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:32:17.257 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:32:17.257 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:32:17.257 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:32:17.257 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:32:17.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:32:17.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:32:17.257 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:32:17.257 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:32:17.257 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:32:17.257 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:32:17.257 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:32:17.257 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:32:17.257 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:32:17.257 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:32:17.257 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:32:17.257 | INFO | ------------------------------------------ | 20260507 20:32:17.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:32:17.257 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:32:17.257 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:32:17.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:32:17.257 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.56e+01, 1.41e+01 / 1.70e+00 | 20260507 20:32:17.257 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.04e-01, 1.32e-02 / 7.06e-03 | 20260507 20:32:17.257 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.47e+00, 6.03e-01 / 3.06e-01 | 20260507 20:32:17.257 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.11e-01 / 1.31e-01 | 20260507 20:32:17.257 | INFO | PerfReport: 15.72%, DME -- 4.17e+01 / 4.48e+01, 3.86e+00 / 1.19e+00 | 20260507 20:32:17.257 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.83e+00 / 1.70e+00 | 20260507 20:32:17.257 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.49e+00 / 2.80e+00, 4.41e-01 / 2.50e-01 | 20260507 20:32:17.257 | INFO | PerfReport: 3.59%, Metrics -- 1.19e+00 / 1.36e+00, 3.06e-01 / 1.70e-01 | 20260507 20:32:17.257 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.70e-01 / 3.98e-01, 6.10e-02 / 3.15e-02 | 20260507 20:32:17.257 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:32:17.257 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:32:17.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:32:17.257 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:32:17.257 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 20:32:17.257 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.59e+00 / 0.00e+00 | 20260507 20:32:17.257 | INFO | PerfReport: 31.05%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:32:17.258 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 20:32:17.258 | INFO | PerfReport: 2.42%, LockDownload -- 2.58e+00 / 0.00e+00, 3.80e+00 / 0.00e+00 | 20260507 20:32:17.258 | INFO | PerfReport: 15.83%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:32:17.258 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260507 20:32:17.353 | INFO | Starting a round of Baz IO | 20260507 20:32:20.832 | INFO | Populating RTMetrics report | 20260507 20:32:20.833 | INFO | RTMetrics report populated | 20260507 20:32:20.833 | INFO | Opening tmp RTMetrics json file | 20260507 20:32:20.860 | INFO | Serializing RTMetrics to json | 20260507 20:32:20.863 | INFO | Serialization done, swapping tmp file now... | 20260507 20:32:20.864 | INFO | RTMetrics done | 20260507 20:32:20.944 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.59 seconds | 20260507 20:32:20.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.41% / 0.01% / 55.58% | 20260507 20:32:20.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.98163% / 7.94729e-06% / 5.16574e-05% | 20260507 20:32:20.944 | INFO | Baz IO overall: | 20260507 20:32:20.944 | INFO | BytesWritten : 1086281627161 | 20260507 20:32:20.944 | INFO | NumEvents : 526603968943 | 20260507 20:32:22.395 | INFO | Analyzing chunk frames = [2392064,2392576) | 20260507 20:32:22.395 | INFO | MemInfo: { | 20260507 20:32:22.395 | INFO | "gpu" : " 17 GB", | 20260507 20:32:22.395 | INFO | "huge" : "", | 20260507 20:32:22.395 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:22.395 | INFO | "malloc" : " 14 GB", | 20260507 20:32:22.395 | INFO | "pinned" : " 29 GB" | 20260507 20:32:22.395 | INFO | } | 20260507 20:32:22.395 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:32:22.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:22.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:22.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:22.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:22.395 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 20:32:22.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:22.395 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 20:32:22.395 | INFO | PRE_HQ : 66.2%, 1.7, 58.1ms, 0 | 20260507 20:32:22.395 | INFO | BAZWRITER : 69.5%, 3.8, 134.5ms, 0 | 20260507 20:32:22.395 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 20:32:22.396 | INFO | Uploaded 36GiB to GPU in 2.49133s (14.4501GiB/s) | 20260507 20:32:22.396 | INFO | Downloaded 38GiB to GPU in 2.68659s (14.1443GiB/s) | 20260507 20:32:27.535 | INFO | Analyzing chunk frames = [2392576,2393088) | 20260507 20:32:27.535 | INFO | MemInfo: { | 20260507 20:32:27.535 | INFO | "gpu" : " 17 GB", | 20260507 20:32:27.535 | INFO | "huge" : "", | 20260507 20:32:27.535 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:27.535 | INFO | "malloc" : " 14 GB", | 20260507 20:32:27.535 | INFO | "pinned" : " 29 GB" | 20260507 20:32:27.535 | INFO | } | 20260507 20:32:27.535 | INFO | Perf report for the last 5.14012 seconds: | 20260507 20:32:27.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:27.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 20:32:27.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:32:27.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:27.535 | INFO | Uploaded 36GiB to GPU in 2.58329s (13.9357GiB/s) | 20260507 20:32:27.535 | INFO | Downloaded 34GiB to GPU in 2.47389s (13.7436GiB/s) | 20260507 20:32:32.674 | INFO | Analyzing chunk frames = [2393088,2393600) | 20260507 20:32:32.674 | INFO | MemInfo: { | 20260507 20:32:32.674 | INFO | "gpu" : " 17 GB", | 20260507 20:32:32.674 | INFO | "huge" : "", | 20260507 20:32:32.674 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:32.674 | INFO | "malloc" : " 14 GB", | 20260507 20:32:32.674 | INFO | "pinned" : " 29 GB" | 20260507 20:32:32.674 | INFO | } | 20260507 20:32:32.674 | INFO | Perf report for the last 5.13887 seconds: | 20260507 20:32:32.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:32.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:32:32.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:32:32.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:32.674 | INFO | Uploaded 36GiB to GPU in 2.53015s (14.2284GiB/s) | 20260507 20:32:32.674 | INFO | Downloaded 34GiB to GPU in 2.48246s (13.6961GiB/s) | 20260507 20:32:37.813 | INFO | Analyzing chunk frames = [2393600,2394112) | 20260507 20:32:37.813 | INFO | MemInfo: { | 20260507 20:32:37.813 | INFO | "gpu" : " 17 GB", | 20260507 20:32:37.813 | INFO | "huge" : "", | 20260507 20:32:37.813 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:37.813 | INFO | "malloc" : " 14 GB", | 20260507 20:32:37.813 | INFO | "pinned" : " 29 GB" | 20260507 20:32:37.813 | INFO | } | 20260507 20:32:37.813 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:32:37.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:37.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 20:32:37.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:32:37.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:37.813 | INFO | Uploaded 36GiB to GPU in 2.48807s (14.469GiB/s) | 20260507 20:32:37.813 | INFO | Downloaded 34GiB to GPU in 2.398s (14.1785GiB/s) | 20260507 20:32:42.953 | INFO | Analyzing chunk frames = [2394112,2394624) | 20260507 20:32:42.953 | INFO | MemInfo: { | 20260507 20:32:42.953 | INFO | "gpu" : " 17 GB", | 20260507 20:32:42.953 | INFO | "huge" : "", | 20260507 20:32:42.953 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:42.953 | INFO | "malloc" : " 14 GB", | 20260507 20:32:42.953 | INFO | "pinned" : " 29 GB" | 20260507 20:32:42.953 | INFO | } | 20260507 20:32:42.953 | INFO | Perf report for the last 5.14022 seconds: | 20260507 20:32:42.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:42.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 20:32:42.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:32:42.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:42.953 | INFO | Uploaded 36GiB to GPU in 2.39549s (15.0282GiB/s) | 20260507 20:32:42.953 | INFO | Downloaded 34GiB to GPU in 2.48623s (13.6753GiB/s) | 20260507 20:32:48.092 | INFO | Analyzing chunk frames = [2394624,2395136) | 20260507 20:32:48.092 | INFO | MemInfo: { | 20260507 20:32:48.092 | INFO | "gpu" : " 17 GB", | 20260507 20:32:48.092 | INFO | "huge" : "", | 20260507 20:32:48.092 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:48.092 | INFO | "malloc" : " 14 GB", | 20260507 20:32:48.092 | INFO | "pinned" : " 29 GB" | 20260507 20:32:48.092 | INFO | } | 20260507 20:32:48.092 | INFO | Perf report for the last 5.13923 seconds: | 20260507 20:32:48.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:48.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:32:48.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 20:32:48.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:48.093 | INFO | Uploaded 36GiB to GPU in 2.48622s (14.4798GiB/s) | 20260507 20:32:48.093 | INFO | Downloaded 34GiB to GPU in 2.42437s (14.0243GiB/s) | 20260507 20:32:53.232 | INFO | Analyzing chunk frames = [2395136,2395648) | 20260507 20:32:53.232 | INFO | MemInfo: { | 20260507 20:32:53.232 | INFO | "gpu" : " 17 GB", | 20260507 20:32:53.232 | INFO | "huge" : "", | 20260507 20:32:53.232 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:53.232 | INFO | "malloc" : " 14 GB", | 20260507 20:32:53.232 | INFO | "pinned" : " 29 GB" | 20260507 20:32:53.232 | INFO | } | 20260507 20:32:53.233 | INFO | Perf report for the last 5.14008 seconds: | 20260507 20:32:53.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:53.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:32:53.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260507 20:32:53.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:53.233 | INFO | Uploaded 36GiB to GPU in 2.52663s (14.2482GiB/s) | 20260507 20:32:53.233 | INFO | Downloaded 34GiB to GPU in 2.44893s (13.8836GiB/s) | 20260507 20:32:58.371 | INFO | Analyzing chunk frames = [2395648,2396160) | 20260507 20:32:58.371 | INFO | MemInfo: { | 20260507 20:32:58.371 | INFO | "gpu" : " 17 GB", | 20260507 20:32:58.371 | INFO | "huge" : "", | 20260507 20:32:58.371 | INFO | "huge_shared" : " 29 GB", | 20260507 20:32:58.371 | INFO | "malloc" : " 14 GB", | 20260507 20:32:58.371 | INFO | "pinned" : " 29 GB" | 20260507 20:32:58.372 | INFO | } | 20260507 20:32:58.372 | INFO | Perf report for the last 5.13905 seconds: | 20260507 20:32:58.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:32:58.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:32:58.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | PRE_HQ : 36.7%, 1.0, 19.0ms, 0 | 20260507 20:32:58.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:32:58.372 | INFO | Uploaded 36GiB to GPU in 2.51925s (14.29GiB/s) | 20260507 20:32:58.372 | INFO | Downloaded 34GiB to GPU in 2.43804s (13.9456GiB/s) | 20260507 20:32:58.489 | INFO | Starting a round of Baz IO | 20260507 20:33:01.966 | INFO | Populating RTMetrics report | 20260507 20:33:01.966 | INFO | RTMetrics report populated | 20260507 20:33:01.966 | INFO | Opening tmp RTMetrics json file | 20260507 20:33:02.007 | INFO | Serializing RTMetrics to json | 20260507 20:33:02.259 | INFO | Serialization done, swapping tmp file now... | 20260507 20:33:02.267 | INFO | RTMetrics done | 20260507 20:33:02.288 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.799 seconds | 20260507 20:33:02.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.40% / 0.01% / 55.59% | 20260507 20:33:02.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9819% / 1.98682e-05% / 5.5631e-05% | 20260507 20:33:02.288 | INFO | Baz IO overall: | 20260507 20:33:02.288 | INFO | BytesWritten : 1088189963511 | 20260507 20:33:02.289 | INFO | NumEvents : 527528134800 | 20260507 20:33:03.510 | INFO | Analyzing chunk frames = [2396160,2396672) | 20260507 20:33:03.510 | INFO | MemInfo: { | 20260507 20:33:03.510 | INFO | "gpu" : " 17 GB", | 20260507 20:33:03.510 | INFO | "huge" : "", | 20260507 20:33:03.510 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:03.510 | INFO | "malloc" : " 14 GB", | 20260507 20:33:03.510 | INFO | "pinned" : " 29 GB" | 20260507 20:33:03.510 | INFO | } | 20260507 20:33:03.511 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:33:03.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:03.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:03.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:03.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:03.511 | INFO | ANALYSIS : 69.0%, 2.9, 104.2ms, 0 | 20260507 20:33:03.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:03.511 | INFO | RT_METRICS : 51.4%, 1.0, 26.9ms, 0 | 20260507 20:33:03.511 | INFO | PRE_HQ : 67.8%, 1.9, 65.1ms, 0 | 20260507 20:33:03.511 | INFO | BAZWRITER : 73.6%, 4.2, 157.5ms, 0 | 20260507 20:33:03.511 | INFO | BAZ_SAMPLING : 6.0%, 1.0, 3.1ms, 0 | 20260507 20:33:03.511 | INFO | Uploaded 36GiB to GPU in 2.55411s (14.0949GiB/s) | 20260507 20:33:03.511 | INFO | Downloaded 38GiB to GPU in 2.70076s (14.0701GiB/s) | 20260507 20:33:08.649 | INFO | Analyzing chunk frames = [2396672,2397184) | 20260507 20:33:08.649 | INFO | MemInfo: { | 20260507 20:33:08.649 | INFO | "gpu" : " 17 GB", | 20260507 20:33:08.649 | INFO | "huge" : "", | 20260507 20:33:08.649 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:08.649 | INFO | "malloc" : " 14 GB", | 20260507 20:33:08.649 | INFO | "pinned" : " 29 GB" | 20260507 20:33:08.649 | INFO | } | 20260507 20:33:08.649 | INFO | Perf report for the last 5.13863 seconds: | 20260507 20:33:08.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:08.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 20:33:08.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:33:08.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:08.651 | INFO | Uploaded 36GiB to GPU in 2.56449s (14.0379GiB/s) | 20260507 20:33:08.651 | INFO | Downloaded 34GiB to GPU in 2.50478s (13.5741GiB/s) | 20260507 20:33:13.788 | INFO | Analyzing chunk frames = [2397184,2397696) | 20260507 20:33:13.788 | INFO | MemInfo: { | 20260507 20:33:13.788 | INFO | "gpu" : " 17 GB", | 20260507 20:33:13.788 | INFO | "huge" : "", | 20260507 20:33:13.788 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:13.788 | INFO | "malloc" : " 14 GB", | 20260507 20:33:13.788 | INFO | "pinned" : " 29 GB" | 20260507 20:33:13.788 | INFO | } | 20260507 20:33:13.788 | INFO | Perf report for the last 5.13901 seconds: | 20260507 20:33:13.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:13.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.788 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 20:33:13.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.788 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:33:13.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:13.789 | INFO | Uploaded 36GiB to GPU in 2.46748s (14.5898GiB/s) | 20260507 20:33:13.789 | INFO | Downloaded 34GiB to GPU in 2.4862s (13.6755GiB/s) | 20260507 20:33:18.930 | INFO | Analyzing chunk frames = [2397696,2398208) | 20260507 20:33:18.930 | INFO | MemInfo: { | 20260507 20:33:18.930 | INFO | "gpu" : " 17 GB", | 20260507 20:33:18.930 | INFO | "huge" : "", | 20260507 20:33:18.930 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:18.931 | INFO | "malloc" : " 14 GB", | 20260507 20:33:18.931 | INFO | "pinned" : " 29 GB" | 20260507 20:33:18.931 | INFO | } | 20260507 20:33:18.931 | INFO | Perf report for the last 5.14253 seconds: | 20260507 20:33:18.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:18.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 20:33:18.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:33:18.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:18.931 | INFO | Uploaded 36GiB to GPU in 2.53154s (14.2206GiB/s) | 20260507 20:33:18.931 | INFO | Downloaded 34GiB to GPU in 2.49499s (13.6273GiB/s) | 20260507 20:33:24.068 | INFO | Analyzing chunk frames = [2398208,2398720) | 20260507 20:33:24.068 | INFO | MemInfo: { | 20260507 20:33:24.068 | INFO | "gpu" : " 17 GB", | 20260507 20:33:24.068 | INFO | "huge" : "", | 20260507 20:33:24.068 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:24.068 | INFO | "malloc" : " 14 GB", | 20260507 20:33:24.068 | INFO | "pinned" : " 29 GB" | 20260507 20:33:24.068 | INFO | } | 20260507 20:33:24.069 | INFO | Perf report for the last 5.13777 seconds: | 20260507 20:33:24.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:24.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 20:33:24.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 20:33:24.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:24.069 | INFO | Uploaded 36GiB to GPU in 2.43867s (14.7622GiB/s) | 20260507 20:33:24.069 | INFO | Downloaded 34GiB to GPU in 2.503s (13.5837GiB/s) | 20260507 20:33:29.206 | INFO | Analyzing chunk frames = [2398720,2399232) | 20260507 20:33:29.206 | INFO | MemInfo: { | 20260507 20:33:29.206 | INFO | "gpu" : " 17 GB", | 20260507 20:33:29.206 | INFO | "huge" : "", | 20260507 20:33:29.206 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:29.206 | INFO | "malloc" : " 14 GB", | 20260507 20:33:29.206 | INFO | "pinned" : " 29 GB" | 20260507 20:33:29.206 | INFO | } | 20260507 20:33:29.207 | INFO | Perf report for the last 5.13795 seconds: | 20260507 20:33:29.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:29.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 20:33:29.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 20:33:29.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:29.207 | INFO | Uploaded 36GiB to GPU in 2.4308s (14.8099GiB/s) | 20260507 20:33:29.207 | INFO | Downloaded 34GiB to GPU in 2.38197s (14.2739GiB/s) | 20260507 20:33:34.346 | INFO | Analyzing chunk frames = [2399232,2399744) | 20260507 20:33:34.346 | INFO | MemInfo: { | 20260507 20:33:34.346 | INFO | "gpu" : " 17 GB", | 20260507 20:33:34.346 | INFO | "huge" : "", | 20260507 20:33:34.346 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:34.346 | INFO | "malloc" : " 14 GB", | 20260507 20:33:34.346 | INFO | "pinned" : " 29 GB" | 20260507 20:33:34.346 | INFO | } | 20260507 20:33:34.346 | INFO | Perf report for the last 5.13942 seconds: | 20260507 20:33:34.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:34.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.346 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260507 20:33:34.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.346 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 20:33:34.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:34.347 | INFO | Uploaded 36GiB to GPU in 2.54862s (14.1253GiB/s) | 20260507 20:33:34.347 | INFO | Downloaded 34GiB to GPU in 2.44045s (13.9319GiB/s) | 20260507 20:33:39.486 | INFO | Analyzing chunk frames = [2399744,2400256) | 20260507 20:33:39.486 | INFO | MemInfo: { | 20260507 20:33:39.486 | INFO | "gpu" : " 17 GB", | 20260507 20:33:39.486 | INFO | "huge" : "", | 20260507 20:33:39.486 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:39.486 | INFO | "malloc" : " 14 GB", | 20260507 20:33:39.486 | INFO | "pinned" : " 29 GB" | 20260507 20:33:39.486 | INFO | } | 20260507 20:33:39.486 | INFO | Perf report for the last 5.14 seconds: | 20260507 20:33:39.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:39.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:33:39.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 20:33:39.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:39.486 | INFO | Uploaded 36GiB to GPU in 2.54063s (14.1697GiB/s) | 20260507 20:33:39.486 | INFO | Downloaded 34GiB to GPU in 2.43477s (13.9643GiB/s) | 20260507 20:33:39.487 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:33:39.487 | INFO | Total mem usage: 16.4 GiB | 20260507 20:33:39.487 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:33:39.487 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 20:33:39.487 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:33:39.487 | INFO | High Water Breakdown: | 20260507 20:33:39.487 | INFO | 59.7% : 8.8 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:33:39.487 | INFO | 40.1% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:33:39.487 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:33:39.487 | INFO | ------------------------------------------ | 20260507 20:33:39.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:33:39.487 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:33:39.487 | INFO | Total mem usage: 26.5 GiB | 20260507 20:33:39.487 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:33:39.487 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:33:39.487 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 20:33:39.487 | INFO | High Water Breakdown: | 20260507 20:33:39.487 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:33:39.487 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:33:39.487 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:33:39.487 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:33:39.487 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:33:39.487 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:33:39.487 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:33:39.487 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:33:39.487 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:33:39.487 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:33:39.487 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:33:39.487 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:33:39.487 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:33:39.487 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:33:39.487 | INFO | ------------------------------------------ | 20260507 20:33:39.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:33:39.487 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:33:39.487 | INFO | Total mem usage: 27.8 GiB | 20260507 20:33:39.487 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:33:39.487 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:33:39.487 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:33:39.487 | INFO | High Water Breakdown: | 20260507 20:33:39.487 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:33:39.487 | INFO | ------------------------------------------ | 20260507 20:33:39.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:33:39.487 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:33:39.487 | INFO | Total mem usage: 15.7 GiB | 20260507 20:33:39.487 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:33:39.487 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:33:39.487 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 20:33:39.487 | INFO | High Water Breakdown: | 20260507 20:33:39.487 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:33:39.487 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:33:39.487 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:33:39.487 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:33:39.487 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:33:39.487 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:33:39.487 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:33:39.487 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:33:39.487 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:33:39.487 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:33:39.488 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:33:39.488 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:33:39.488 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:33:39.488 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:33:39.488 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:33:39.488 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:33:39.488 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:33:39.488 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:33:39.488 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:33:39.488 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:33:39.488 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:33:39.488 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:33:39.488 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:33:39.488 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:33:39.488 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:33:39.488 | INFO | ------------------------------------------ | 20260507 20:33:39.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:33:39.488 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:33:39.488 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:33:39.488 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:33:39.488 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.44e+01 / 1.87e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 0.30%, AggregateStats -- 9.99e-02 / 1.05e-01, 1.34e-02 / 7.76e-03 | 20260507 20:33:39.488 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.50e+00, 5.91e-01 / 2.23e-01 | 20260507 20:33:39.488 | INFO | PerfReport: 3.35%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 1.18e-01 | 20260507 20:33:39.488 | INFO | PerfReport: 16.03%, DME -- 4.24e+01 / 4.59e+01, 4.21e+00 / 1.82e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 55.87%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.68e+00 / 1.39e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.52e+00 / 2.83e+00, 4.45e-01 / 2.50e-01 | 20260507 20:33:39.488 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.40e+00, 2.91e-01 / 1.60e-01 | 20260507 20:33:39.488 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.71e-01 / 4.08e-01, 6.04e-02 / 3.74e-02 | 20260507 20:33:39.488 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:33:39.488 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:33:39.488 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:33:39.488 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 30.90%, Compute -- 3.31e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 2.16%, LockUpload -- 2.31e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 2.49%, LockDownload -- 2.66e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:33:39.488 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260507 20:33:39.599 | INFO | Starting a round of Baz IO | 20260507 20:33:43.087 | INFO | Populating RTMetrics report | 20260507 20:33:43.087 | INFO | RTMetrics report populated | 20260507 20:33:43.087 | INFO | Opening tmp RTMetrics json file | 20260507 20:33:43.113 | INFO | Serializing RTMetrics to json | 20260507 20:33:43.115 | INFO | Serialization done, swapping tmp file now... | 20260507 20:33:43.116 | INFO | RTMetrics done | 20260507 20:33:43.206 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.607 seconds | 20260507 20:33:43.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.39% / 0.01% / 55.60% | 20260507 20:33:43.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9757% / 1.19209e-05% / 4.37101e-05% | 20260507 20:33:43.206 | INFO | Baz IO overall: | 20260507 20:33:43.206 | INFO | BytesWritten : 1090098276789 | 20260507 20:33:43.206 | INFO | NumEvents : 528452217188 | 20260507 20:33:44.625 | INFO | Analyzing chunk frames = [2400256,2400768) | 20260507 20:33:44.625 | INFO | MemInfo: { | 20260507 20:33:44.625 | INFO | "gpu" : " 17 GB", | 20260507 20:33:44.625 | INFO | "huge" : "", | 20260507 20:33:44.625 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:44.625 | INFO | "malloc" : " 14 GB", | 20260507 20:33:44.625 | INFO | "pinned" : " 29 GB" | 20260507 20:33:44.625 | INFO | } | 20260507 20:33:44.625 | INFO | Perf report for the last 5.13907 seconds: | 20260507 20:33:44.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:44.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:44.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:44.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:44.625 | INFO | ANALYSIS : 69.3%, 2.9, 104.6ms, 0 | 20260507 20:33:44.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:44.625 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 20:33:44.625 | INFO | PRE_HQ : 68.2%, 1.8, 64.4ms, 0 | 20260507 20:33:44.625 | INFO | BAZWRITER : 69.8%, 3.9, 138.6ms, 0 | 20260507 20:33:44.625 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 20:33:44.625 | INFO | Uploaded 36GiB to GPU in 2.5444s (14.1487GiB/s) | 20260507 20:33:44.625 | INFO | Downloaded 38GiB to GPU in 2.69019s (14.1254GiB/s) | 20260507 20:33:49.763 | INFO | Analyzing chunk frames = [2400768,2401280) | 20260507 20:33:49.763 | INFO | MemInfo: { | 20260507 20:33:49.763 | INFO | "gpu" : " 17 GB", | 20260507 20:33:49.763 | INFO | "huge" : "", | 20260507 20:33:49.763 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:49.763 | INFO | "malloc" : " 14 GB", | 20260507 20:33:49.763 | INFO | "pinned" : " 29 GB" | 20260507 20:33:49.763 | INFO | } | 20260507 20:33:49.763 | INFO | Perf report for the last 5.13828 seconds: | 20260507 20:33:49.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:49.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.763 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:33:49.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.763 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:33:49.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:49.764 | INFO | Uploaded 36GiB to GPU in 2.54845s (14.1262GiB/s) | 20260507 20:33:49.764 | INFO | Downloaded 34GiB to GPU in 2.4963s (13.6202GiB/s) | 20260507 20:33:54.903 | INFO | Analyzing chunk frames = [2401280,2401792) | 20260507 20:33:54.903 | INFO | MemInfo: { | 20260507 20:33:54.903 | INFO | "gpu" : " 17 GB", | 20260507 20:33:54.903 | INFO | "huge" : "", | 20260507 20:33:54.903 | INFO | "huge_shared" : " 29 GB", | 20260507 20:33:54.903 | INFO | "malloc" : " 14 GB", | 20260507 20:33:54.903 | INFO | "pinned" : " 29 GB" | 20260507 20:33:54.903 | INFO | } | 20260507 20:33:54.903 | INFO | Perf report for the last 5.13977 seconds: | 20260507 20:33:54.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:33:54.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:33:54.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:33:54.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:33:54.903 | INFO | Uploaded 36GiB to GPU in 2.57367s (13.9878GiB/s) | 20260507 20:33:54.903 | INFO | Downloaded 34GiB to GPU in 2.47486s (13.7382GiB/s) | 20260507 20:34:00.043 | INFO | Analyzing chunk frames = [2401792,2402304) | 20260507 20:34:00.043 | INFO | MemInfo: { | 20260507 20:34:00.043 | INFO | "gpu" : " 17 GB", | 20260507 20:34:00.043 | INFO | "huge" : "", | 20260507 20:34:00.043 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:00.043 | INFO | "malloc" : " 14 GB", | 20260507 20:34:00.044 | INFO | "pinned" : " 29 GB" | 20260507 20:34:00.044 | INFO | } | 20260507 20:34:00.044 | INFO | Perf report for the last 5.14047 seconds: | 20260507 20:34:00.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:00.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:34:00.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:34:00.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:00.044 | INFO | Uploaded 36GiB to GPU in 2.50609s (14.365GiB/s) | 20260507 20:34:00.044 | INFO | Downloaded 34GiB to GPU in 2.41239s (14.0939GiB/s) | 20260507 20:34:05.184 | INFO | Analyzing chunk frames = [2402304,2402816) | 20260507 20:34:05.184 | INFO | MemInfo: { | 20260507 20:34:05.184 | INFO | "gpu" : " 17 GB", | 20260507 20:34:05.184 | INFO | "huge" : "", | 20260507 20:34:05.184 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:05.184 | INFO | "malloc" : " 14 GB", | 20260507 20:34:05.184 | INFO | "pinned" : " 29 GB" | 20260507 20:34:05.184 | INFO | } | 20260507 20:34:05.184 | INFO | Perf report for the last 5.1407 seconds: | 20260507 20:34:05.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:05.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.184 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 20:34:05.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.184 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 20:34:05.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:05.185 | INFO | Uploaded 36GiB to GPU in 2.41069s (14.9335GiB/s) | 20260507 20:34:05.185 | INFO | Downloaded 34GiB to GPU in 2.48895s (13.6604GiB/s) | 20260507 20:34:10.321 | INFO | Analyzing chunk frames = [2402816,2403328) | 20260507 20:34:10.321 | INFO | MemInfo: { | 20260507 20:34:10.321 | INFO | "gpu" : " 17 GB", | 20260507 20:34:10.321 | INFO | "huge" : "", | 20260507 20:34:10.321 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:10.321 | INFO | "malloc" : " 14 GB", | 20260507 20:34:10.321 | INFO | "pinned" : " 29 GB" | 20260507 20:34:10.321 | INFO | } | 20260507 20:34:10.322 | INFO | Perf report for the last 5.13732 seconds: | 20260507 20:34:10.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:10.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260507 20:34:10.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:34:10.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:10.322 | INFO | Uploaded 36GiB to GPU in 2.3704s (15.1873GiB/s) | 20260507 20:34:10.322 | INFO | Downloaded 34GiB to GPU in 2.39989s (14.1673GiB/s) | 20260507 20:34:15.461 | INFO | Analyzing chunk frames = [2403328,2403840) | 20260507 20:34:15.461 | INFO | MemInfo: { | 20260507 20:34:15.461 | INFO | "gpu" : " 17 GB", | 20260507 20:34:15.461 | INFO | "huge" : "", | 20260507 20:34:15.461 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:15.461 | INFO | "malloc" : " 14 GB", | 20260507 20:34:15.461 | INFO | "pinned" : " 29 GB" | 20260507 20:34:15.461 | INFO | } | 20260507 20:34:15.461 | INFO | Perf report for the last 5.13972 seconds: | 20260507 20:34:15.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:15.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.461 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 20:34:15.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.461 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 20:34:15.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:15.462 | INFO | Uploaded 36GiB to GPU in 2.46362s (14.6127GiB/s) | 20260507 20:34:15.462 | INFO | Downloaded 34GiB to GPU in 2.44909s (13.8827GiB/s) | 20260507 20:34:20.600 | INFO | Analyzing chunk frames = [2403840,2404352) | 20260507 20:34:20.600 | INFO | MemInfo: { | 20260507 20:34:20.600 | INFO | "gpu" : " 17 GB", | 20260507 20:34:20.600 | INFO | "huge" : "", | 20260507 20:34:20.600 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:20.600 | INFO | "malloc" : " 14 GB", | 20260507 20:34:20.600 | INFO | "pinned" : " 29 GB" | 20260507 20:34:20.600 | INFO | } | 20260507 20:34:20.600 | INFO | Perf report for the last 5.13884 seconds: | 20260507 20:34:20.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:20.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.600 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 20:34:20.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.600 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 20:34:20.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:20.601 | INFO | Uploaded 36GiB to GPU in 2.44031s (14.7522GiB/s) | 20260507 20:34:20.601 | INFO | Downloaded 34GiB to GPU in 2.44121s (13.9275GiB/s) | 20260507 20:34:20.697 | INFO | Starting a round of Baz IO | 20260507 20:34:24.165 | INFO | Populating RTMetrics report | 20260507 20:34:24.166 | INFO | RTMetrics report populated | 20260507 20:34:24.166 | INFO | Opening tmp RTMetrics json file | 20260507 20:34:24.192 | INFO | Serializing RTMetrics to json | 20260507 20:34:24.194 | INFO | Serialization done, swapping tmp file now... | 20260507 20:34:24.195 | INFO | RTMetrics done | 20260507 20:34:24.285 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.588 seconds | 20260507 20:34:24.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.38% / 0.01% / 55.61% | 20260507 20:34:24.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.96917% / 3.17891e-05% / 5.5631e-05% | 20260507 20:34:24.285 | INFO | Baz IO overall: | 20260507 20:34:24.285 | INFO | BytesWritten : 1092005668163 | 20260507 20:34:24.285 | INFO | NumEvents : 529375687049 | 20260507 20:34:25.739 | INFO | Analyzing chunk frames = [2404352,2404864) | 20260507 20:34:25.739 | INFO | MemInfo: { | 20260507 20:34:25.739 | INFO | "gpu" : " 17 GB", | 20260507 20:34:25.739 | INFO | "huge" : "", | 20260507 20:34:25.739 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:25.739 | INFO | "malloc" : " 14 GB", | 20260507 20:34:25.739 | INFO | "pinned" : " 29 GB" | 20260507 20:34:25.739 | INFO | } | 20260507 20:34:25.739 | INFO | Perf report for the last 5.13889 seconds: | 20260507 20:34:25.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:25.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:25.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:25.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:25.739 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 20:34:25.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:25.739 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 20:34:25.739 | INFO | PRE_HQ : 66.2%, 1.7, 58.2ms, 0 | 20260507 20:34:25.739 | INFO | BAZWRITER : 69.2%, 3.8, 134.6ms, 0 | 20260507 20:34:25.739 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:34:25.739 | INFO | Uploaded 36GiB to GPU in 2.52226s (14.2729GiB/s) | 20260507 20:34:25.739 | INFO | Downloaded 38GiB to GPU in 2.69848s (14.082GiB/s) | 20260507 20:34:30.878 | INFO | Analyzing chunk frames = [2404864,2405376) | 20260507 20:34:30.878 | INFO | MemInfo: { | 20260507 20:34:30.878 | INFO | "gpu" : " 17 GB", | 20260507 20:34:30.878 | INFO | "huge" : "", | 20260507 20:34:30.878 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:30.878 | INFO | "malloc" : " 14 GB", | 20260507 20:34:30.878 | INFO | "pinned" : " 29 GB" | 20260507 20:34:30.878 | INFO | } | 20260507 20:34:30.878 | INFO | Perf report for the last 5.13915 seconds: | 20260507 20:34:30.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:30.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.878 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 20:34:30.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.879 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:34:30.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:30.879 | INFO | Uploaded 36GiB to GPU in 2.53086s (14.2244GiB/s) | 20260507 20:34:30.879 | INFO | Downloaded 34GiB to GPU in 2.50987s (13.5465GiB/s) | 20260507 20:34:36.018 | INFO | Analyzing chunk frames = [2405376,2405888) | 20260507 20:34:36.018 | INFO | MemInfo: { | 20260507 20:34:36.018 | INFO | "gpu" : " 17 GB", | 20260507 20:34:36.018 | INFO | "huge" : "", | 20260507 20:34:36.018 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:36.018 | INFO | "malloc" : " 14 GB", | 20260507 20:34:36.018 | INFO | "pinned" : " 29 GB" | 20260507 20:34:36.018 | INFO | } | 20260507 20:34:36.018 | INFO | Perf report for the last 5.13955 seconds: | 20260507 20:34:36.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:36.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 20:34:36.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:34:36.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:36.018 | INFO | Uploaded 36GiB to GPU in 2.51273s (14.327GiB/s) | 20260507 20:34:36.018 | INFO | Downloaded 34GiB to GPU in 2.48819s (13.6645GiB/s) | 20260507 20:34:41.157 | INFO | Analyzing chunk frames = [2405888,2406400) | 20260507 20:34:41.157 | INFO | MemInfo: { | 20260507 20:34:41.157 | INFO | "gpu" : " 17 GB", | 20260507 20:34:41.157 | INFO | "huge" : "", | 20260507 20:34:41.157 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:41.157 | INFO | "malloc" : " 14 GB", | 20260507 20:34:41.157 | INFO | "pinned" : " 29 GB" | 20260507 20:34:41.157 | INFO | } | 20260507 20:34:41.157 | INFO | Perf report for the last 5.13887 seconds: | 20260507 20:34:41.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:41.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 20:34:41.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:34:41.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:41.157 | INFO | Uploaded 36GiB to GPU in 2.44014s (14.7533GiB/s) | 20260507 20:34:41.157 | INFO | Downloaded 34GiB to GPU in 2.50967s (13.5476GiB/s) | 20260507 20:34:46.299 | INFO | Analyzing chunk frames = [2406400,2406912) | 20260507 20:34:46.299 | INFO | MemInfo: { | 20260507 20:34:46.299 | INFO | "gpu" : " 17 GB", | 20260507 20:34:46.299 | INFO | "huge" : "", | 20260507 20:34:46.299 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:46.299 | INFO | "malloc" : " 14 GB", | 20260507 20:34:46.299 | INFO | "pinned" : " 29 GB" | 20260507 20:34:46.299 | INFO | } | 20260507 20:34:46.299 | INFO | Perf report for the last 5.14204 seconds: | 20260507 20:34:46.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:46.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.299 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 20:34:46.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.299 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:34:46.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:46.300 | INFO | Uploaded 36GiB to GPU in 2.39247s (15.0472GiB/s) | 20260507 20:34:46.300 | INFO | Downloaded 34GiB to GPU in 2.5011s (13.594GiB/s) | 20260507 20:34:51.437 | INFO | Analyzing chunk frames = [2406912,2407424) | 20260507 20:34:51.437 | INFO | MemInfo: { | 20260507 20:34:51.437 | INFO | "gpu" : " 17 GB", | 20260507 20:34:51.437 | INFO | "huge" : "", | 20260507 20:34:51.437 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:51.437 | INFO | "malloc" : " 14 GB", | 20260507 20:34:51.437 | INFO | "pinned" : " 29 GB" | 20260507 20:34:51.437 | INFO | } | 20260507 20:34:51.437 | INFO | Perf report for the last 5.13814 seconds: | 20260507 20:34:51.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:51.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.437 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 20:34:51.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.437 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260507 20:34:51.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:51.438 | INFO | Uploaded 36GiB to GPU in 2.34911s (15.325GiB/s) | 20260507 20:34:51.438 | INFO | Downloaded 34GiB to GPU in 2.40023s (14.1653GiB/s) | 20260507 20:34:56.575 | INFO | Analyzing chunk frames = [2407424,2407936) | 20260507 20:34:56.575 | INFO | MemInfo: { | 20260507 20:34:56.575 | INFO | "gpu" : " 17 GB", | 20260507 20:34:56.575 | INFO | "huge" : "", | 20260507 20:34:56.575 | INFO | "huge_shared" : " 29 GB", | 20260507 20:34:56.575 | INFO | "malloc" : " 14 GB", | 20260507 20:34:56.575 | INFO | "pinned" : " 29 GB" | 20260507 20:34:56.575 | INFO | } | 20260507 20:34:56.575 | INFO | Perf report for the last 5.13789 seconds: | 20260507 20:34:56.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:34:56.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 20:34:56.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:34:56.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:34:56.575 | INFO | Uploaded 36GiB to GPU in 2.45578s (14.6593GiB/s) | 20260507 20:34:56.575 | INFO | Downloaded 34GiB to GPU in 2.44669s (13.8964GiB/s) | 20260507 20:35:01.714 | INFO | Analyzing chunk frames = [2407936,2408448) | 20260507 20:35:01.714 | INFO | MemInfo: { | 20260507 20:35:01.714 | INFO | "gpu" : " 17 GB", | 20260507 20:35:01.714 | INFO | "huge" : "", | 20260507 20:35:01.714 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:01.714 | INFO | "malloc" : " 14 GB", | 20260507 20:35:01.714 | INFO | "pinned" : " 29 GB" | 20260507 20:35:01.714 | INFO | } | 20260507 20:35:01.714 | INFO | Perf report for the last 5.13939 seconds: | 20260507 20:35:01.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:01.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.714 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:35:01.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.714 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:35:01.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:01.715 | INFO | Uploaded 36GiB to GPU in 2.45618s (14.6569GiB/s) | 20260507 20:35:01.715 | INFO | Downloaded 34GiB to GPU in 2.44404s (13.9114GiB/s) | 20260507 20:35:01.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:35:01.715 | INFO | Total mem usage: 16.4 GiB | 20260507 20:35:01.715 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:35:01.715 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.1%) | 20260507 20:35:01.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:35:01.715 | INFO | High Water Breakdown: | 20260507 20:35:01.715 | INFO | 59.9% : 8.93 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:35:01.715 | INFO | 39.9% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:35:01.715 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:35:01.715 | INFO | ------------------------------------------ | 20260507 20:35:01.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:35:01.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:35:01.715 | INFO | Total mem usage: 26.5 GiB | 20260507 20:35:01.715 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:35:01.715 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:35:01.715 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 20:35:01.715 | INFO | High Water Breakdown: | 20260507 20:35:01.715 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:35:01.715 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:35:01.715 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:35:01.715 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:35:01.715 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:35:01.715 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:35:01.715 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:35:01.715 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:35:01.715 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:35:01.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:35:01.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:35:01.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:35:01.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:35:01.716 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:35:01.716 | INFO | ------------------------------------------ | 20260507 20:35:01.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:35:01.716 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:35:01.716 | INFO | Total mem usage: 27.8 GiB | 20260507 20:35:01.716 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:35:01.716 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:35:01.716 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:35:01.716 | INFO | High Water Breakdown: | 20260507 20:35:01.716 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:35:01.716 | INFO | ------------------------------------------ | 20260507 20:35:01.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:35:01.716 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:35:01.716 | INFO | Total mem usage: 15.7 GiB | 20260507 20:35:01.716 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:35:01.716 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:35:01.716 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:35:01.716 | INFO | High Water Breakdown: | 20260507 20:35:01.716 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:35:01.716 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:35:01.716 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:35:01.716 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:35:01.716 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:35:01.716 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:35:01.716 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:35:01.716 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:35:01.716 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:35:01.716 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:35:01.716 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:35:01.716 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:35:01.716 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:35:01.716 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:35:01.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:35:01.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:35:01.716 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:35:01.716 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:35:01.716 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:35:01.716 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:35:01.716 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:35:01.716 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:35:01.716 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:35:01.716 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:35:01.716 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:35:01.716 | INFO | ------------------------------------------ | 20260507 20:35:01.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:35:01.716 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:35:01.716 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:35:01.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:35:01.716 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.60e+01, 1.40e+01 / 1.75e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.02e-01, 1.22e-02 / 5.81e-03 | 20260507 20:35:01.716 | INFO | PerfReport: 12.09%, Baseline -- 4.01e+00 / 4.49e+00, 5.98e-01 / 2.65e-01 | 20260507 20:35:01.716 | INFO | PerfReport: 3.39%, Histogram -- 1.13e+00 / 1.50e+00, 2.11e-01 / 1.29e-01 | 20260507 20:35:01.716 | INFO | PerfReport: 15.80%, DME -- 4.19e+01 / 4.52e+01, 4.11e+00 / 1.58e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.84e+00 / 1.74e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.48e+00 / 2.78e+00, 4.39e-01 / 2.47e-01 | 20260507 20:35:01.716 | INFO | PerfReport: 3.57%, Metrics -- 1.18e+00 / 1.38e+00, 3.05e-01 / 1.72e-01 | 20260507 20:35:01.716 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.67e-01 / 3.96e-01, 6.06e-02 / 2.90e-02 | 20260507 20:35:01.716 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:35:01.716 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:35:01.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:35:01.716 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 24.12%, Upload -- 2.57e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 31.10%, Compute -- 3.32e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 1.75%, LockUpload -- 1.87e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 16.02%, LockCompute -- 1.71e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 20:35:01.716 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260507 20:35:01.812 | INFO | Starting a round of Baz IO | 20260507 20:35:05.291 | INFO | Populating RTMetrics report | 20260507 20:35:05.292 | INFO | RTMetrics report populated | 20260507 20:35:05.292 | INFO | Opening tmp RTMetrics json file | 20260507 20:35:05.385 | INFO | Serializing RTMetrics to json | 20260507 20:35:05.391 | INFO | Serialization done, swapping tmp file now... | 20260507 20:35:05.397 | INFO | RTMetrics done | 20260507 20:35:05.621 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.809 seconds | 20260507 20:35:05.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.37% / 0.01% / 55.62% | 20260507 20:35:05.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.96115% / 2.38419e-05% / 3.57628e-05% | 20260507 20:35:05.621 | INFO | Baz IO overall: | 20260507 20:35:05.621 | INFO | BytesWritten : 1093910366173 | 20260507 20:35:05.621 | INFO | NumEvents : 530297106757 | 20260507 20:35:06.854 | INFO | Analyzing chunk frames = [2408448,2408960) | 20260507 20:35:06.854 | INFO | MemInfo: { | 20260507 20:35:06.854 | INFO | "gpu" : " 17 GB", | 20260507 20:35:06.854 | INFO | "huge" : "", | 20260507 20:35:06.854 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:06.854 | INFO | "malloc" : " 14 GB", | 20260507 20:35:06.854 | INFO | "pinned" : " 29 GB" | 20260507 20:35:06.854 | INFO | } | 20260507 20:35:06.854 | INFO | Perf report for the last 5.1396 seconds: | 20260507 20:35:06.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:06.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:06.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:06.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:06.854 | INFO | ANALYSIS : 68.9%, 3.0, 107.0ms, 0 | 20260507 20:35:06.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:06.854 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 20:35:06.854 | INFO | PRE_HQ : 65.8%, 1.7, 57.9ms, 0 | 20260507 20:35:06.854 | INFO | BAZWRITER : 73.6%, 3.9, 145.8ms, 0 | 20260507 20:35:06.854 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 20:35:06.854 | INFO | Uploaded 36GiB to GPU in 2.49882s (14.4068GiB/s) | 20260507 20:35:06.854 | INFO | Downloaded 38GiB to GPU in 2.69076s (14.1224GiB/s) | 20260507 20:35:11.993 | INFO | Analyzing chunk frames = [2408960,2409472) | 20260507 20:35:11.993 | INFO | MemInfo: { | 20260507 20:35:11.993 | INFO | "gpu" : " 17 GB", | 20260507 20:35:11.993 | INFO | "huge" : "", | 20260507 20:35:11.994 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:11.994 | INFO | "malloc" : " 14 GB", | 20260507 20:35:11.994 | INFO | "pinned" : " 29 GB" | 20260507 20:35:11.994 | INFO | } | 20260507 20:35:11.994 | INFO | Perf report for the last 5.13982 seconds: | 20260507 20:35:11.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:11.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:35:11.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:35:11.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:11.994 | INFO | Uploaded 36GiB to GPU in 2.51278s (14.3268GiB/s) | 20260507 20:35:11.994 | INFO | Downloaded 34GiB to GPU in 2.49375s (13.6341GiB/s) | 20260507 20:35:17.134 | INFO | Analyzing chunk frames = [2409472,2409984) | 20260507 20:35:17.134 | INFO | MemInfo: { | 20260507 20:35:17.134 | INFO | "gpu" : " 17 GB", | 20260507 20:35:17.134 | INFO | "huge" : "", | 20260507 20:35:17.134 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:17.134 | INFO | "malloc" : " 14 GB", | 20260507 20:35:17.134 | INFO | "pinned" : " 29 GB" | 20260507 20:35:17.134 | INFO | } | 20260507 20:35:17.134 | INFO | Perf report for the last 5.14029 seconds: | 20260507 20:35:17.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:17.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.134 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 20:35:17.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.134 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:35:17.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:17.135 | INFO | Uploaded 36GiB to GPU in 2.51384s (14.3207GiB/s) | 20260507 20:35:17.135 | INFO | Downloaded 34GiB to GPU in 2.48806s (13.6653GiB/s) | 20260507 20:35:22.272 | INFO | Analyzing chunk frames = [2409984,2410496) | 20260507 20:35:22.272 | INFO | MemInfo: { | 20260507 20:35:22.272 | INFO | "gpu" : " 17 GB", | 20260507 20:35:22.272 | INFO | "huge" : "", | 20260507 20:35:22.272 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:22.272 | INFO | "malloc" : " 14 GB", | 20260507 20:35:22.272 | INFO | "pinned" : " 29 GB" | 20260507 20:35:22.272 | INFO | } | 20260507 20:35:22.272 | INFO | Perf report for the last 5.13838 seconds: | 20260507 20:35:22.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:22.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.272 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 20:35:22.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.272 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:35:22.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:22.273 | INFO | Uploaded 36GiB to GPU in 2.45621s (14.6567GiB/s) | 20260507 20:35:22.273 | INFO | Downloaded 34GiB to GPU in 2.5135s (13.5269GiB/s) | 20260507 20:35:27.412 | INFO | Analyzing chunk frames = [2410496,2411008) | 20260507 20:35:27.412 | INFO | MemInfo: { | 20260507 20:35:27.412 | INFO | "gpu" : " 17 GB", | 20260507 20:35:27.412 | INFO | "huge" : "", | 20260507 20:35:27.412 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:27.412 | INFO | "malloc" : " 14 GB", | 20260507 20:35:27.412 | INFO | "pinned" : " 29 GB" | 20260507 20:35:27.412 | INFO | } | 20260507 20:35:27.412 | INFO | Perf report for the last 5.13985 seconds: | 20260507 20:35:27.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:27.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.412 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:35:27.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.412 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:35:27.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:27.413 | INFO | Uploaded 36GiB to GPU in 2.32815s (15.4629GiB/s) | 20260507 20:35:27.413 | INFO | Downloaded 34GiB to GPU in 2.50448s (13.5757GiB/s) | 20260507 20:35:32.551 | INFO | Analyzing chunk frames = [2411008,2411520) | 20260507 20:35:32.551 | INFO | MemInfo: { | 20260507 20:35:32.551 | INFO | "gpu" : " 17 GB", | 20260507 20:35:32.551 | INFO | "huge" : "", | 20260507 20:35:32.551 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:32.551 | INFO | "malloc" : " 14 GB", | 20260507 20:35:32.551 | INFO | "pinned" : " 29 GB" | 20260507 20:35:32.551 | INFO | } | 20260507 20:35:32.551 | INFO | Perf report for the last 5.13859 seconds: | 20260507 20:35:32.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:32.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:35:32.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:35:32.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:32.551 | INFO | Uploaded 36GiB to GPU in 2.45433s (14.6679GiB/s) | 20260507 20:35:32.551 | INFO | Downloaded 34GiB to GPU in 2.44187s (13.9238GiB/s) | 20260507 20:35:37.690 | INFO | Analyzing chunk frames = [2411520,2412032) | 20260507 20:35:37.690 | INFO | MemInfo: { | 20260507 20:35:37.690 | INFO | "gpu" : " 17 GB", | 20260507 20:35:37.690 | INFO | "huge" : "", | 20260507 20:35:37.690 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:37.690 | INFO | "malloc" : " 14 GB", | 20260507 20:35:37.690 | INFO | "pinned" : " 29 GB" | 20260507 20:35:37.690 | INFO | } | 20260507 20:35:37.690 | INFO | Perf report for the last 5.13902 seconds: | 20260507 20:35:37.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:37.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 20:35:37.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:35:37.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:37.690 | INFO | Uploaded 36GiB to GPU in 2.46956s (14.5775GiB/s) | 20260507 20:35:37.690 | INFO | Downloaded 34GiB to GPU in 2.46363s (13.8008GiB/s) | 20260507 20:35:42.830 | INFO | Analyzing chunk frames = [2412032,2412544) | 20260507 20:35:42.830 | INFO | MemInfo: { | 20260507 20:35:42.830 | INFO | "gpu" : " 17 GB", | 20260507 20:35:42.830 | INFO | "huge" : "", | 20260507 20:35:42.830 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:42.830 | INFO | "malloc" : " 14 GB", | 20260507 20:35:42.830 | INFO | "pinned" : " 29 GB" | 20260507 20:35:42.830 | INFO | } | 20260507 20:35:42.830 | INFO | Perf report for the last 5.14006 seconds: | 20260507 20:35:42.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:42.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 20:35:42.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:35:42.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:42.830 | INFO | Uploaded 36GiB to GPU in 2.43068s (14.8107GiB/s) | 20260507 20:35:42.831 | INFO | Downloaded 34GiB to GPU in 2.44253s (13.92GiB/s) | 20260507 20:35:42.927 | INFO | Starting a round of Baz IO | 20260507 20:35:46.414 | INFO | Populating RTMetrics report | 20260507 20:35:46.415 | INFO | RTMetrics report populated | 20260507 20:35:46.415 | INFO | Opening tmp RTMetrics json file | 20260507 20:35:46.459 | INFO | Serializing RTMetrics to json | 20260507 20:35:46.507 | INFO | Serialization done, swapping tmp file now... | 20260507 20:35:46.510 | INFO | RTMetrics done | 20260507 20:35:46.531 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.604 seconds | 20260507 20:35:46.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.36% / 0.01% / 55.63% | 20260507 20:35:46.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.96015% / 1.58946e-05% / 4.37101e-05% | 20260507 20:35:46.531 | INFO | Baz IO overall: | 20260507 20:35:46.531 | INFO | BytesWritten : 1095816894046 | 20260507 20:35:46.531 | INFO | NumEvents : 531219992374 | 20260507 20:35:47.969 | INFO | Analyzing chunk frames = [2412544,2413056) | 20260507 20:35:47.969 | INFO | MemInfo: { | 20260507 20:35:47.969 | INFO | "gpu" : " 17 GB", | 20260507 20:35:47.969 | INFO | "huge" : "", | 20260507 20:35:47.969 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:47.969 | INFO | "malloc" : " 14 GB", | 20260507 20:35:47.969 | INFO | "pinned" : " 29 GB" | 20260507 20:35:47.970 | INFO | } | 20260507 20:35:47.970 | INFO | Perf report for the last 5.13961 seconds: | 20260507 20:35:47.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:47.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:47.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:47.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:47.970 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:35:47.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:47.970 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 20:35:47.970 | INFO | PRE_HQ : 66.0%, 1.7, 58.2ms, 0 | 20260507 20:35:47.970 | INFO | BAZWRITER : 69.8%, 3.8, 134.7ms, 0 | 20260507 20:35:47.970 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 20:35:47.970 | INFO | Uploaded 36GiB to GPU in 2.50517s (14.3703GiB/s) | 20260507 20:35:47.970 | INFO | Downloaded 38GiB to GPU in 2.68938s (14.1296GiB/s) | 20260507 20:35:53.109 | INFO | Analyzing chunk frames = [2413056,2413568) | 20260507 20:35:53.109 | INFO | MemInfo: { | 20260507 20:35:53.109 | INFO | "gpu" : " 17 GB", | 20260507 20:35:53.109 | INFO | "huge" : "", | 20260507 20:35:53.109 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:53.109 | INFO | "malloc" : " 14 GB", | 20260507 20:35:53.109 | INFO | "pinned" : " 29 GB" | 20260507 20:35:53.109 | INFO | } | 20260507 20:35:53.109 | INFO | Perf report for the last 5.13989 seconds: | 20260507 20:35:53.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:53.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.110 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:35:53.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.110 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:35:53.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:53.110 | INFO | Uploaded 36GiB to GPU in 2.52301s (14.2687GiB/s) | 20260507 20:35:53.110 | INFO | Downloaded 34GiB to GPU in 2.50102s (13.5945GiB/s) | 20260507 20:35:58.248 | INFO | Analyzing chunk frames = [2413568,2414080) | 20260507 20:35:58.248 | INFO | MemInfo: { | 20260507 20:35:58.248 | INFO | "gpu" : " 17 GB", | 20260507 20:35:58.248 | INFO | "huge" : "", | 20260507 20:35:58.248 | INFO | "huge_shared" : " 29 GB", | 20260507 20:35:58.248 | INFO | "malloc" : " 14 GB", | 20260507 20:35:58.248 | INFO | "pinned" : " 29 GB" | 20260507 20:35:58.248 | INFO | } | 20260507 20:35:58.248 | INFO | Perf report for the last 5.1388 seconds: | 20260507 20:35:58.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:35:58.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.248 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:35:58.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.248 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 20:35:58.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:35:58.249 | INFO | Uploaded 36GiB to GPU in 2.44878s (14.7012GiB/s) | 20260507 20:35:58.249 | INFO | Downloaded 34GiB to GPU in 2.4993s (13.6038GiB/s) | 20260507 20:36:03.387 | INFO | Analyzing chunk frames = [2414080,2414592) | 20260507 20:36:03.387 | INFO | MemInfo: { | 20260507 20:36:03.387 | INFO | "gpu" : " 17 GB", | 20260507 20:36:03.387 | INFO | "huge" : "", | 20260507 20:36:03.387 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:03.387 | INFO | "malloc" : " 14 GB", | 20260507 20:36:03.387 | INFO | "pinned" : " 29 GB" | 20260507 20:36:03.387 | INFO | } | 20260507 20:36:03.387 | INFO | Perf report for the last 5.1389 seconds: | 20260507 20:36:03.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:03.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.387 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 20:36:03.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.387 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:36:03.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:03.388 | INFO | Uploaded 36GiB to GPU in 2.43875s (14.7617GiB/s) | 20260507 20:36:03.388 | INFO | Downloaded 34GiB to GPU in 2.41588s (14.0736GiB/s) | 20260507 20:36:08.526 | INFO | Analyzing chunk frames = [2414592,2415104) | 20260507 20:36:08.526 | INFO | MemInfo: { | 20260507 20:36:08.526 | INFO | "gpu" : " 17 GB", | 20260507 20:36:08.526 | INFO | "huge" : "", | 20260507 20:36:08.526 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:08.526 | INFO | "malloc" : " 14 GB", | 20260507 20:36:08.526 | INFO | "pinned" : " 29 GB" | 20260507 20:36:08.526 | INFO | } | 20260507 20:36:08.526 | INFO | Perf report for the last 5.13866 seconds: | 20260507 20:36:08.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:08.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260507 20:36:08.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:36:08.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:08.526 | INFO | Uploaded 36GiB to GPU in 2.42261s (14.86GiB/s) | 20260507 20:36:08.527 | INFO | Downloaded 34GiB to GPU in 2.50975s (13.5471GiB/s) | 20260507 20:36:13.665 | INFO | Analyzing chunk frames = [2415104,2415616) | 20260507 20:36:13.665 | INFO | MemInfo: { | 20260507 20:36:13.665 | INFO | "gpu" : " 17 GB", | 20260507 20:36:13.665 | INFO | "huge" : "", | 20260507 20:36:13.665 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:13.665 | INFO | "malloc" : " 14 GB", | 20260507 20:36:13.665 | INFO | "pinned" : " 29 GB" | 20260507 20:36:13.665 | INFO | } | 20260507 20:36:13.666 | INFO | Perf report for the last 5.13965 seconds: | 20260507 20:36:13.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:13.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 20:36:13.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:36:13.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:13.666 | INFO | Uploaded 36GiB to GPU in 2.43865s (14.7623GiB/s) | 20260507 20:36:13.666 | INFO | Downloaded 34GiB to GPU in 2.44359s (13.914GiB/s) | 20260507 20:36:18.804 | INFO | Analyzing chunk frames = [2415616,2416128) | 20260507 20:36:18.804 | INFO | MemInfo: { | 20260507 20:36:18.804 | INFO | "gpu" : " 17 GB", | 20260507 20:36:18.804 | INFO | "huge" : "", | 20260507 20:36:18.804 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:18.804 | INFO | "malloc" : " 14 GB", | 20260507 20:36:18.804 | INFO | "pinned" : " 29 GB" | 20260507 20:36:18.805 | INFO | } | 20260507 20:36:18.805 | INFO | Perf report for the last 5.13902 seconds: | 20260507 20:36:18.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:18.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:36:18.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:36:18.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:18.805 | INFO | Uploaded 36GiB to GPU in 2.48571s (14.4828GiB/s) | 20260507 20:36:18.805 | INFO | Downloaded 34GiB to GPU in 2.47137s (13.7576GiB/s) | 20260507 20:36:23.944 | INFO | Analyzing chunk frames = [2416128,2416640) | 20260507 20:36:23.944 | INFO | MemInfo: { | 20260507 20:36:23.944 | INFO | "gpu" : " 17 GB", | 20260507 20:36:23.944 | INFO | "huge" : "", | 20260507 20:36:23.944 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:23.944 | INFO | "malloc" : " 14 GB", | 20260507 20:36:23.944 | INFO | "pinned" : " 29 GB" | 20260507 20:36:23.944 | INFO | } | 20260507 20:36:23.944 | INFO | Perf report for the last 5.13986 seconds: | 20260507 20:36:23.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:23.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.944 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:36:23.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.944 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:36:23.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:23.945 | INFO | Uploaded 36GiB to GPU in 2.44245s (14.7393GiB/s) | 20260507 20:36:23.945 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:36:23.945 | INFO | Downloaded 34GiB to GPU in 2.44571s (13.9019GiB/s) | 20260507 20:36:23.945 | INFO | Total mem usage: 16.4 GiB | 20260507 20:36:23.945 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:36:23.945 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.19%) | 20260507 20:36:23.945 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:36:23.945 | INFO | High Water Breakdown: | 20260507 20:36:23.945 | INFO | 60% : 9.01 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:36:23.945 | INFO | 39.9% : 5.99 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:36:23.945 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:36:23.945 | INFO | ------------------------------------------ | 20260507 20:36:23.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:36:23.945 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:36:23.945 | INFO | Total mem usage: 26.5 GiB | 20260507 20:36:23.945 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:36:23.945 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:36:23.945 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:36:23.945 | INFO | High Water Breakdown: | 20260507 20:36:23.945 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:36:23.945 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:36:23.945 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:36:23.945 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:36:23.945 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:36:23.945 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:36:23.945 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:36:23.945 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:36:23.945 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:36:23.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:36:23.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:36:23.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:36:23.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:36:23.946 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:36:23.946 | INFO | ------------------------------------------ | 20260507 20:36:23.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:36:23.946 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:36:23.946 | INFO | Total mem usage: 27.8 GiB | 20260507 20:36:23.946 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:36:23.946 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:36:23.946 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:36:23.946 | INFO | High Water Breakdown: | 20260507 20:36:23.946 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:36:23.946 | INFO | ------------------------------------------ | 20260507 20:36:23.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:36:23.946 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:36:23.946 | INFO | Total mem usage: 15.7 GiB | 20260507 20:36:23.946 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:36:23.946 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:36:23.946 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:36:23.946 | INFO | High Water Breakdown: | 20260507 20:36:23.946 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:36:23.946 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:36:23.946 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:36:23.946 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:36:23.946 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:36:23.946 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:36:23.946 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:36:23.946 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:36:23.946 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:36:23.946 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:36:23.946 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:36:23.946 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:36:23.946 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:36:23.946 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:36:23.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:36:23.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:36:23.946 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:36:23.946 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:36:23.946 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:36:23.946 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:36:23.946 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:36:23.946 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:36:23.946 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:36:23.946 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:36:23.946 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:36:23.946 | INFO | ------------------------------------------ | 20260507 20:36:23.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:36:23.946 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:36:23.946 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:36:23.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:36:23.946 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.60e+01, 1.38e+01 / 1.92e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.14e-02 / 4.44e-03 | 20260507 20:36:23.946 | INFO | PerfReport: 12.09%, Baseline -- 4.00e+00 / 4.48e+00, 5.98e-01 / 2.48e-01 | 20260507 20:36:23.946 | INFO | PerfReport: 3.37%, Histogram -- 1.12e+00 / 1.50e+00, 2.06e-01 / 1.07e-01 | 20260507 20:36:23.946 | INFO | PerfReport: 15.91%, DME -- 4.21e+01 / 4.56e+01, 4.22e+00 / 1.84e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.75e+00 / 1.43e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.49e+00 / 2.85e+00, 4.39e-01 / 1.88e-01 | 20260507 20:36:23.946 | INFO | PerfReport: 3.56%, Metrics -- 1.18e+00 / 1.41e+00, 3.02e-01 / 1.66e-01 | 20260507 20:36:23.946 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 4.05e-01, 6.08e-02 / 3.32e-02 | 20260507 20:36:23.946 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:36:23.946 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:36:23.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:36:23.946 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 23.97%, Upload -- 2.56e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 24.37%, Download -- 2.61e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 30.97%, Compute -- 3.31e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 1.66%, LockUpload -- 1.78e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 3.00%, LockDownload -- 3.21e+00 / 0.00e+00, 4.44e+00 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 16.01%, LockCompute -- 1.71e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 20:36:23.946 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260507 20:36:24.042 | INFO | Starting a round of Baz IO | 20260507 20:36:27.550 | INFO | Populating RTMetrics report | 20260507 20:36:27.550 | INFO | RTMetrics report populated | 20260507 20:36:27.550 | INFO | Opening tmp RTMetrics json file | 20260507 20:36:27.551 | INFO | Serializing RTMetrics to json | 20260507 20:36:27.553 | INFO | Serialization done, swapping tmp file now... | 20260507 20:36:27.553 | INFO | RTMetrics done | 20260507 20:36:27.657 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.615 seconds | 20260507 20:36:27.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.35% / 0.01% / 55.64% | 20260507 20:36:27.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.95867% / 1.19209e-05% / 3.57628e-05% | 20260507 20:36:27.657 | INFO | Baz IO overall: | 20260507 20:36:27.657 | INFO | BytesWritten : 1097723932714 | 20260507 20:36:27.657 | INFO | NumEvents : 532143298898 | 20260507 20:36:29.083 | INFO | Analyzing chunk frames = [2416640,2417152) | 20260507 20:36:29.083 | INFO | MemInfo: { | 20260507 20:36:29.083 | INFO | "gpu" : " 17 GB", | 20260507 20:36:29.083 | INFO | "huge" : "", | 20260507 20:36:29.083 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:29.083 | INFO | "malloc" : " 14 GB", | 20260507 20:36:29.083 | INFO | "pinned" : " 29 GB" | 20260507 20:36:29.083 | INFO | } | 20260507 20:36:29.083 | INFO | Perf report for the last 5.13885 seconds: | 20260507 20:36:29.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:29.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:29.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:29.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:29.083 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 20:36:29.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:29.084 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 20:36:29.084 | INFO | PRE_HQ : 66.8%, 1.8, 62.8ms, 0 | 20260507 20:36:29.084 | INFO | BAZWRITER : 69.9%, 4.1, 146.9ms, 0 | 20260507 20:36:29.084 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 20:36:29.084 | INFO | Uploaded 36GiB to GPU in 2.50749s (14.357GiB/s) | 20260507 20:36:29.084 | INFO | Downloaded 38GiB to GPU in 2.69062s (14.1231GiB/s) | 20260507 20:36:34.223 | INFO | Analyzing chunk frames = [2417152,2417664) | 20260507 20:36:34.223 | INFO | MemInfo: { | 20260507 20:36:34.223 | INFO | "gpu" : " 17 GB", | 20260507 20:36:34.223 | INFO | "huge" : "", | 20260507 20:36:34.223 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:34.224 | INFO | "malloc" : " 14 GB", | 20260507 20:36:34.224 | INFO | "pinned" : " 29 GB" | 20260507 20:36:34.224 | INFO | } | 20260507 20:36:34.224 | INFO | Perf report for the last 5.14035 seconds: | 20260507 20:36:34.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:34.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 20:36:34.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:36:34.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:34.224 | INFO | Uploaded 36GiB to GPU in 2.50505s (14.371GiB/s) | 20260507 20:36:34.224 | INFO | Downloaded 34GiB to GPU in 2.49552s (13.6244GiB/s) | 20260507 20:36:39.363 | INFO | Analyzing chunk frames = [2417664,2418176) | 20260507 20:36:39.363 | INFO | MemInfo: { | 20260507 20:36:39.363 | INFO | "gpu" : " 17 GB", | 20260507 20:36:39.363 | INFO | "huge" : "", | 20260507 20:36:39.363 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:39.363 | INFO | "malloc" : " 14 GB", | 20260507 20:36:39.363 | INFO | "pinned" : " 29 GB" | 20260507 20:36:39.363 | INFO | } | 20260507 20:36:39.364 | INFO | Perf report for the last 5.13988 seconds: | 20260507 20:36:39.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:39.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:36:39.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:36:39.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:39.364 | INFO | Uploaded 36GiB to GPU in 2.47684s (14.5347GiB/s) | 20260507 20:36:39.364 | INFO | Downloaded 34GiB to GPU in 2.48028s (13.7081GiB/s) | 20260507 20:36:44.502 | INFO | Analyzing chunk frames = [2418176,2418688) | 20260507 20:36:44.502 | INFO | MemInfo: { | 20260507 20:36:44.502 | INFO | "gpu" : " 17 GB", | 20260507 20:36:44.502 | INFO | "huge" : "", | 20260507 20:36:44.502 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:44.502 | INFO | "malloc" : " 14 GB", | 20260507 20:36:44.502 | INFO | "pinned" : " 29 GB" | 20260507 20:36:44.502 | INFO | } | 20260507 20:36:44.502 | INFO | Perf report for the last 5.1388 seconds: | 20260507 20:36:44.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:44.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.502 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 20:36:44.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.502 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:36:44.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:44.503 | INFO | Uploaded 36GiB to GPU in 2.50685s (14.3606GiB/s) | 20260507 20:36:44.503 | INFO | Downloaded 34GiB to GPU in 2.49596s (13.622GiB/s) | 20260507 20:36:49.641 | INFO | Analyzing chunk frames = [2418688,2419200) | 20260507 20:36:49.641 | INFO | MemInfo: { | 20260507 20:36:49.641 | INFO | "gpu" : " 17 GB", | 20260507 20:36:49.641 | INFO | "huge" : "", | 20260507 20:36:49.641 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:49.641 | INFO | "malloc" : " 14 GB", | 20260507 20:36:49.642 | INFO | "pinned" : " 29 GB" | 20260507 20:36:49.642 | INFO | } | 20260507 20:36:49.642 | INFO | Perf report for the last 5.13923 seconds: | 20260507 20:36:49.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:49.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 20:36:49.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:36:49.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:49.642 | INFO | Uploaded 36GiB to GPU in 2.36058s (15.2505GiB/s) | 20260507 20:36:49.642 | INFO | Downloaded 34GiB to GPU in 2.50247s (13.5866GiB/s) | 20260507 20:36:54.780 | INFO | Analyzing chunk frames = [2419200,2419712) | 20260507 20:36:54.780 | INFO | MemInfo: { | 20260507 20:36:54.780 | INFO | "gpu" : " 17 GB", | 20260507 20:36:54.780 | INFO | "huge" : "", | 20260507 20:36:54.780 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:54.780 | INFO | "malloc" : " 14 GB", | 20260507 20:36:54.780 | INFO | "pinned" : " 29 GB" | 20260507 20:36:54.780 | INFO | } | 20260507 20:36:54.780 | INFO | Perf report for the last 5.13864 seconds: | 20260507 20:36:54.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:54.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.780 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 20:36:54.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.780 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260507 20:36:54.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:54.781 | INFO | Uploaded 36GiB to GPU in 2.4124s (14.9229GiB/s) | 20260507 20:36:54.781 | INFO | Downloaded 34GiB to GPU in 2.38383s (14.2628GiB/s) | 20260507 20:36:59.919 | INFO | Analyzing chunk frames = [2419712,2420224) | 20260507 20:36:59.919 | INFO | MemInfo: { | 20260507 20:36:59.919 | INFO | "gpu" : " 17 GB", | 20260507 20:36:59.919 | INFO | "huge" : "", | 20260507 20:36:59.919 | INFO | "huge_shared" : " 29 GB", | 20260507 20:36:59.919 | INFO | "malloc" : " 14 GB", | 20260507 20:36:59.919 | INFO | "pinned" : " 29 GB" | 20260507 20:36:59.919 | INFO | } | 20260507 20:36:59.919 | INFO | Perf report for the last 5.13888 seconds: | 20260507 20:36:59.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:36:59.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.919 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:36:59.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.919 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:36:59.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:36:59.920 | INFO | Uploaded 36GiB to GPU in 2.55283s (14.102GiB/s) | 20260507 20:36:59.920 | INFO | Downloaded 34GiB to GPU in 2.44264s (13.9194GiB/s) | 20260507 20:37:05.060 | INFO | Analyzing chunk frames = [2420224,2420736) | 20260507 20:37:05.060 | INFO | MemInfo: { | 20260507 20:37:05.060 | INFO | "gpu" : " 17 GB", | 20260507 20:37:05.060 | INFO | "huge" : "", | 20260507 20:37:05.060 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:05.060 | INFO | "malloc" : " 14 GB", | 20260507 20:37:05.060 | INFO | "pinned" : " 29 GB" | 20260507 20:37:05.060 | INFO | } | 20260507 20:37:05.060 | INFO | Perf report for the last 5.14102 seconds: | 20260507 20:37:05.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:05.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:37:05.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:37:05.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:05.060 | INFO | Uploaded 36GiB to GPU in 2.50154s (14.3912GiB/s) | 20260507 20:37:05.060 | INFO | Downloaded 34GiB to GPU in 2.42442s (14.024GiB/s) | 20260507 20:37:05.157 | INFO | Starting a round of Baz IO | 20260507 20:37:08.628 | INFO | Populating RTMetrics report | 20260507 20:37:08.628 | INFO | RTMetrics report populated | 20260507 20:37:08.628 | INFO | Opening tmp RTMetrics json file | 20260507 20:37:08.629 | INFO | Serializing RTMetrics to json | 20260507 20:37:08.631 | INFO | Serialization done, swapping tmp file now... | 20260507 20:37:08.631 | INFO | RTMetrics done | 20260507 20:37:08.741 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.583 seconds | 20260507 20:37:08.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.34% / 0.01% / 55.65% | 20260507 20:37:08.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.96033% / 1.19209e-05% / 3.17891e-05% | 20260507 20:37:08.741 | INFO | Baz IO overall: | 20260507 20:37:08.741 | INFO | BytesWritten : 1099632342156 | 20260507 20:37:08.741 | INFO | NumEvents : 533067592526 | 20260507 20:37:10.199 | INFO | Analyzing chunk frames = [2420736,2421248) | 20260507 20:37:10.199 | INFO | MemInfo: { | 20260507 20:37:10.199 | INFO | "gpu" : " 17 GB", | 20260507 20:37:10.199 | INFO | "huge" : "", | 20260507 20:37:10.199 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:10.199 | INFO | "malloc" : " 14 GB", | 20260507 20:37:10.199 | INFO | "pinned" : " 29 GB" | 20260507 20:37:10.199 | INFO | } | 20260507 20:37:10.199 | INFO | Perf report for the last 5.13897 seconds: | 20260507 20:37:10.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:10.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:10.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:10.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:10.199 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 20:37:10.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:10.199 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 20:37:10.199 | INFO | PRE_HQ : 66.1%, 1.7, 58.1ms, 0 | 20260507 20:37:10.199 | INFO | BAZWRITER : 69.3%, 3.8, 135.2ms, 0 | 20260507 20:37:10.199 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 20:37:10.199 | INFO | Uploaded 36GiB to GPU in 2.54336s (14.1545GiB/s) | 20260507 20:37:10.200 | INFO | Downloaded 38GiB to GPU in 2.69927s (14.0779GiB/s) | 20260507 20:37:15.337 | INFO | Analyzing chunk frames = [2421248,2421760) | 20260507 20:37:15.337 | INFO | MemInfo: { | 20260507 20:37:15.337 | INFO | "gpu" : " 17 GB", | 20260507 20:37:15.337 | INFO | "huge" : "", | 20260507 20:37:15.337 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:15.338 | INFO | "malloc" : " 14 GB", | 20260507 20:37:15.338 | INFO | "pinned" : " 29 GB" | 20260507 20:37:15.338 | INFO | } | 20260507 20:37:15.338 | INFO | Perf report for the last 5.13849 seconds: | 20260507 20:37:15.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:15.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 20:37:15.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:37:15.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:15.338 | INFO | Uploaded 36GiB to GPU in 2.58142s (13.9458GiB/s) | 20260507 20:37:15.338 | INFO | Downloaded 34GiB to GPU in 2.48585s (13.6774GiB/s) | 20260507 20:37:20.477 | INFO | Analyzing chunk frames = [2421760,2422272) | 20260507 20:37:20.477 | INFO | MemInfo: { | 20260507 20:37:20.477 | INFO | "gpu" : " 17 GB", | 20260507 20:37:20.477 | INFO | "huge" : "", | 20260507 20:37:20.477 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:20.477 | INFO | "malloc" : " 14 GB", | 20260507 20:37:20.477 | INFO | "pinned" : " 29 GB" | 20260507 20:37:20.477 | INFO | } | 20260507 20:37:20.477 | INFO | Perf report for the last 5.13951 seconds: | 20260507 20:37:20.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:20.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.477 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 20:37:20.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.477 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:37:20.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:20.478 | INFO | Uploaded 36GiB to GPU in 2.59455s (13.8752GiB/s) | 20260507 20:37:20.478 | INFO | Downloaded 34GiB to GPU in 2.46752s (13.779GiB/s) | 20260507 20:37:25.617 | INFO | Analyzing chunk frames = [2422272,2422784) | 20260507 20:37:25.617 | INFO | MemInfo: { | 20260507 20:37:25.617 | INFO | "gpu" : " 17 GB", | 20260507 20:37:25.617 | INFO | "huge" : "", | 20260507 20:37:25.617 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:25.617 | INFO | "malloc" : " 14 GB", | 20260507 20:37:25.617 | INFO | "pinned" : " 29 GB" | 20260507 20:37:25.617 | INFO | } | 20260507 20:37:25.617 | INFO | Perf report for the last 5.1398 seconds: | 20260507 20:37:25.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:25.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 20:37:25.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:37:25.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:25.617 | INFO | Uploaded 36GiB to GPU in 2.50054s (14.3969GiB/s) | 20260507 20:37:25.617 | INFO | Downloaded 34GiB to GPU in 2.40029s (14.165GiB/s) | 20260507 20:37:30.757 | INFO | Analyzing chunk frames = [2422784,2423296) | 20260507 20:37:30.757 | INFO | MemInfo: { | 20260507 20:37:30.757 | INFO | "gpu" : " 17 GB", | 20260507 20:37:30.757 | INFO | "huge" : "", | 20260507 20:37:30.757 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:30.757 | INFO | "malloc" : " 14 GB", | 20260507 20:37:30.757 | INFO | "pinned" : " 29 GB" | 20260507 20:37:30.757 | INFO | } | 20260507 20:37:30.757 | INFO | Perf report for the last 5.13986 seconds: | 20260507 20:37:30.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:30.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.757 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 20:37:30.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.757 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260507 20:37:30.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:30.758 | INFO | Uploaded 36GiB to GPU in 2.41314s (14.9183GiB/s) | 20260507 20:37:30.758 | INFO | Downloaded 34GiB to GPU in 2.48574s (13.678GiB/s) | 20260507 20:37:35.895 | INFO | Analyzing chunk frames = [2423296,2423808) | 20260507 20:37:35.895 | INFO | MemInfo: { | 20260507 20:37:35.895 | INFO | "gpu" : " 17 GB", | 20260507 20:37:35.895 | INFO | "huge" : "", | 20260507 20:37:35.895 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:35.895 | INFO | "malloc" : " 14 GB", | 20260507 20:37:35.895 | INFO | "pinned" : " 29 GB" | 20260507 20:37:35.895 | INFO | } | 20260507 20:37:35.895 | INFO | Perf report for the last 5.13795 seconds: | 20260507 20:37:35.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:35.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:37:35.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:37:35.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:35.895 | INFO | Uploaded 36GiB to GPU in 2.50673s (14.3613GiB/s) | 20260507 20:37:35.895 | INFO | Downloaded 34GiB to GPU in 2.4155s (14.0757GiB/s) | 20260507 20:37:41.035 | INFO | Analyzing chunk frames = [2423808,2424320) | 20260507 20:37:41.035 | INFO | MemInfo: { | 20260507 20:37:41.035 | INFO | "gpu" : " 17 GB", | 20260507 20:37:41.035 | INFO | "huge" : "", | 20260507 20:37:41.035 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:41.035 | INFO | "malloc" : " 14 GB", | 20260507 20:37:41.035 | INFO | "pinned" : " 29 GB" | 20260507 20:37:41.035 | INFO | } | 20260507 20:37:41.035 | INFO | Perf report for the last 5.14004 seconds: | 20260507 20:37:41.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:41.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.035 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:37:41.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.035 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:37:41.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:41.036 | INFO | Uploaded 36GiB to GPU in 2.54906s (14.1228GiB/s) | 20260507 20:37:41.036 | INFO | Downloaded 34GiB to GPU in 2.4515s (13.8691GiB/s) | 20260507 20:37:46.175 | INFO | Analyzing chunk frames = [2424320,2424832) | 20260507 20:37:46.175 | INFO | MemInfo: { | 20260507 20:37:46.175 | INFO | "gpu" : " 17 GB", | 20260507 20:37:46.175 | INFO | "huge" : "", | 20260507 20:37:46.175 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:46.175 | INFO | "malloc" : " 14 GB", | 20260507 20:37:46.175 | INFO | "pinned" : " 29 GB" | 20260507 20:37:46.175 | INFO | } | 20260507 20:37:46.175 | INFO | Perf report for the last 5.14035 seconds: | 20260507 20:37:46.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:46.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.175 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:37:46.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.175 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:37:46.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:46.176 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:37:46.176 | INFO | Total mem usage: 16.4 GiB | 20260507 20:37:46.176 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:37:46.176 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.6%) | 20260507 20:37:46.176 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:37:46.176 | INFO | High Water Breakdown: | 20260507 20:37:46.176 | INFO | 59.8% : 8.87 GiB ( 89.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:37:46.176 | INFO | 40% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:37:46.176 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:37:46.176 | INFO | Uploaded 36GiB to GPU in 2.52357s (14.2655GiB/s) | 20260507 20:37:46.176 | INFO | ------------------------------------------ | 20260507 20:37:46.176 | INFO | Downloaded 34GiB to GPU in 2.42878s (13.9988GiB/s) | 20260507 20:37:46.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:37:46.176 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:37:46.176 | INFO | Total mem usage: 26.5 GiB | 20260507 20:37:46.176 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:37:46.176 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:37:46.176 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:37:46.176 | INFO | High Water Breakdown: | 20260507 20:37:46.176 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:37:46.176 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:37:46.176 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:37:46.176 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:37:46.176 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:37:46.176 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:37:46.176 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:37:46.176 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:37:46.176 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:37:46.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:37:46.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:37:46.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:37:46.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:37:46.176 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:37:46.176 | INFO | ------------------------------------------ | 20260507 20:37:46.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:37:46.176 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:37:46.176 | INFO | Total mem usage: 27.8 GiB | 20260507 20:37:46.176 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:37:46.176 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:37:46.176 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:37:46.176 | INFO | High Water Breakdown: | 20260507 20:37:46.176 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:37:46.177 | INFO | ------------------------------------------ | 20260507 20:37:46.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:37:46.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:37:46.177 | INFO | Total mem usage: 15.7 GiB | 20260507 20:37:46.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:37:46.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:37:46.177 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:37:46.177 | INFO | High Water Breakdown: | 20260507 20:37:46.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:37:46.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:37:46.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:37:46.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:37:46.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:37:46.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:37:46.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:37:46.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:37:46.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:37:46.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:37:46.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:37:46.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:37:46.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:37:46.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:37:46.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:37:46.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:37:46.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:37:46.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:37:46.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:37:46.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:37:46.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:37:46.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:37:46.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:37:46.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:37:46.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:37:46.177 | INFO | ------------------------------------------ | 20260507 20:37:46.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:37:46.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:37:46.177 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:37:46.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:37:46.177 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.59e+01, 1.43e+01 / 1.82e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 0.31%, AggregateStats -- 1.02e-01 / 1.02e-01, 1.53e-01 / 5.40e-03 | 20260507 20:37:46.177 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.53e+00, 5.97e-01 / 1.68e-01 | 20260507 20:37:46.177 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.11e-01 / 1.15e-01 | 20260507 20:37:46.177 | INFO | PerfReport: 15.91%, DME -- 4.21e+01 / 4.55e+01, 4.03e+00 / 1.72e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.72e+00 / 1.46e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.51e+00 / 2.87e+00, 4.43e-01 / 1.73e-01 | 20260507 20:37:46.177 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.40e+00, 2.99e-01 / 1.71e-01 | 20260507 20:37:46.177 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.69e-01 / 4.05e-01, 6.08e-02 / 3.05e-02 | 20260507 20:37:46.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:37:46.177 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:37:46.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:37:46.177 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 24.36%, Upload -- 2.61e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 24.19%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 30.94%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 2.14%, LockUpload -- 2.29e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 2.45%, LockDownload -- 2.62e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 20:37:46.177 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260507 20:37:46.273 | INFO | Starting a round of Baz IO | 20260507 20:37:49.752 | INFO | Populating RTMetrics report | 20260507 20:37:49.752 | INFO | RTMetrics report populated | 20260507 20:37:49.752 | INFO | Opening tmp RTMetrics json file | 20260507 20:37:49.786 | INFO | Serializing RTMetrics to json | 20260507 20:37:49.858 | INFO | Serialization done, swapping tmp file now... | 20260507 20:37:49.860 | INFO | RTMetrics done | 20260507 20:37:49.897 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.623 seconds | 20260507 20:37:49.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.33% / 0.01% / 55.66% | 20260507 20:37:49.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.95427% / 3.97364e-06% / 4.37101e-05% | 20260507 20:37:49.897 | INFO | Baz IO overall: | 20260507 20:37:49.897 | INFO | BytesWritten : 1101539907443 | 20260507 20:37:49.897 | INFO | NumEvents : 533991332692 | 20260507 20:37:51.314 | INFO | Analyzing chunk frames = [2424832,2425344) | 20260507 20:37:51.314 | INFO | MemInfo: { | 20260507 20:37:51.314 | INFO | "gpu" : " 17 GB", | 20260507 20:37:51.314 | INFO | "huge" : "", | 20260507 20:37:51.314 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:51.314 | INFO | "malloc" : " 14 GB", | 20260507 20:37:51.314 | INFO | "pinned" : " 29 GB" | 20260507 20:37:51.314 | INFO | } | 20260507 20:37:51.314 | INFO | Perf report for the last 5.13829 seconds: | 20260507 20:37:51.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:51.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:51.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:51.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:51.314 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 20:37:51.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:51.314 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 20:37:51.314 | INFO | PRE_HQ : 66.2%, 1.7, 58.1ms, 0 | 20260507 20:37:51.314 | INFO | BAZWRITER : 70.1%, 3.7, 132.6ms, 0 | 20260507 20:37:51.314 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 20:37:51.314 | INFO | Uploaded 36GiB to GPU in 2.53471s (14.2028GiB/s) | 20260507 20:37:51.314 | INFO | Downloaded 38GiB to GPU in 2.69243s (14.1137GiB/s) | 20260507 20:37:56.453 | INFO | Analyzing chunk frames = [2425344,2425856) | 20260507 20:37:56.453 | INFO | MemInfo: { | 20260507 20:37:56.453 | INFO | "gpu" : " 17 GB", | 20260507 20:37:56.453 | INFO | "huge" : "", | 20260507 20:37:56.453 | INFO | "huge_shared" : " 29 GB", | 20260507 20:37:56.453 | INFO | "malloc" : " 14 GB", | 20260507 20:37:56.453 | INFO | "pinned" : " 29 GB" | 20260507 20:37:56.453 | INFO | } | 20260507 20:37:56.454 | INFO | Perf report for the last 5.13985 seconds: | 20260507 20:37:56.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:37:56.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 20:37:56.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:37:56.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:37:56.454 | INFO | Uploaded 36GiB to GPU in 2.58476s (13.9278GiB/s) | 20260507 20:37:56.454 | INFO | Downloaded 34GiB to GPU in 2.47873s (13.7167GiB/s) | 20260507 20:38:01.591 | INFO | Analyzing chunk frames = [2425856,2426368) | 20260507 20:38:01.591 | INFO | MemInfo: { | 20260507 20:38:01.591 | INFO | "gpu" : " 17 GB", | 20260507 20:38:01.591 | INFO | "huge" : "", | 20260507 20:38:01.591 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:01.591 | INFO | "malloc" : " 14 GB", | 20260507 20:38:01.592 | INFO | "pinned" : " 29 GB" | 20260507 20:38:01.592 | INFO | } | 20260507 20:38:01.592 | INFO | Perf report for the last 5.13815 seconds: | 20260507 20:38:01.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:01.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:38:01.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:38:01.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:01.592 | INFO | Uploaded 36GiB to GPU in 2.56578s (14.0308GiB/s) | 20260507 20:38:01.592 | INFO | Downloaded 34GiB to GPU in 2.46316s (13.8034GiB/s) | 20260507 20:38:06.731 | INFO | Analyzing chunk frames = [2426368,2426880) | 20260507 20:38:06.731 | INFO | MemInfo: { | 20260507 20:38:06.731 | INFO | "gpu" : " 17 GB", | 20260507 20:38:06.731 | INFO | "huge" : "", | 20260507 20:38:06.731 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:06.731 | INFO | "malloc" : " 14 GB", | 20260507 20:38:06.731 | INFO | "pinned" : " 29 GB" | 20260507 20:38:06.731 | INFO | } | 20260507 20:38:06.731 | INFO | Perf report for the last 5.13932 seconds: | 20260507 20:38:06.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:06.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:38:06.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:38:06.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:06.731 | INFO | Uploaded 36GiB to GPU in 2.48683s (14.4763GiB/s) | 20260507 20:38:06.731 | INFO | Downloaded 34GiB to GPU in 2.38985s (14.2269GiB/s) | 20260507 20:38:11.871 | INFO | Analyzing chunk frames = [2426880,2427392) | 20260507 20:38:11.871 | INFO | MemInfo: { | 20260507 20:38:11.871 | INFO | "gpu" : " 17 GB", | 20260507 20:38:11.871 | INFO | "huge" : "", | 20260507 20:38:11.871 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:11.871 | INFO | "malloc" : " 14 GB", | 20260507 20:38:11.871 | INFO | "pinned" : " 29 GB" | 20260507 20:38:11.871 | INFO | } | 20260507 20:38:11.871 | INFO | Perf report for the last 5.14019 seconds: | 20260507 20:38:11.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:11.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.871 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 20:38:11.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.871 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:38:11.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:11.872 | INFO | Uploaded 36GiB to GPU in 2.38641s (15.0854GiB/s) | 20260507 20:38:11.872 | INFO | Downloaded 34GiB to GPU in 2.5065s (13.5647GiB/s) | 20260507 20:38:17.010 | INFO | Analyzing chunk frames = [2427392,2427904) | 20260507 20:38:17.010 | INFO | MemInfo: { | 20260507 20:38:17.010 | INFO | "gpu" : " 17 GB", | 20260507 20:38:17.010 | INFO | "huge" : "", | 20260507 20:38:17.010 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:17.010 | INFO | "malloc" : " 14 GB", | 20260507 20:38:17.010 | INFO | "pinned" : " 29 GB" | 20260507 20:38:17.010 | INFO | } | 20260507 20:38:17.010 | INFO | Perf report for the last 5.13865 seconds: | 20260507 20:38:17.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:17.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:38:17.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 20:38:17.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:17.010 | INFO | Uploaded 36GiB to GPU in 2.42908s (14.8204GiB/s) | 20260507 20:38:17.010 | INFO | Downloaded 34GiB to GPU in 2.38185s (14.2746GiB/s) | 20260507 20:38:22.149 | INFO | Analyzing chunk frames = [2427904,2428416) | 20260507 20:38:22.149 | INFO | MemInfo: { | 20260507 20:38:22.149 | INFO | "gpu" : " 17 GB", | 20260507 20:38:22.149 | INFO | "huge" : "", | 20260507 20:38:22.149 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:22.149 | INFO | "malloc" : " 14 GB", | 20260507 20:38:22.149 | INFO | "pinned" : " 29 GB" | 20260507 20:38:22.149 | INFO | } | 20260507 20:38:22.149 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:38:22.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:22.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:38:22.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:38:22.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:22.149 | INFO | Uploaded 36GiB to GPU in 2.54152s (14.1647GiB/s) | 20260507 20:38:22.149 | INFO | Downloaded 34GiB to GPU in 2.43917s (13.9391GiB/s) | 20260507 20:38:27.289 | INFO | Analyzing chunk frames = [2428416,2428928) | 20260507 20:38:27.289 | INFO | MemInfo: { | 20260507 20:38:27.289 | INFO | "gpu" : " 17 GB", | 20260507 20:38:27.289 | INFO | "huge" : "", | 20260507 20:38:27.289 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:27.289 | INFO | "malloc" : " 14 GB", | 20260507 20:38:27.289 | INFO | "pinned" : " 29 GB" | 20260507 20:38:27.289 | INFO | } | 20260507 20:38:27.289 | INFO | Perf report for the last 5.13987 seconds: | 20260507 20:38:27.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:27.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:38:27.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:38:27.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:27.289 | INFO | Uploaded 36GiB to GPU in 2.51958s (14.2881GiB/s) | 20260507 20:38:27.289 | INFO | Downloaded 34GiB to GPU in 2.4173s (14.0653GiB/s) | 20260507 20:38:27.386 | INFO | Starting a round of Baz IO | 20260507 20:38:30.864 | INFO | Populating RTMetrics report | 20260507 20:38:30.864 | INFO | RTMetrics report populated | 20260507 20:38:30.864 | INFO | Opening tmp RTMetrics json file | 20260507 20:38:30.893 | INFO | Serializing RTMetrics to json | 20260507 20:38:30.924 | INFO | Serialization done, swapping tmp file now... | 20260507 20:38:30.963 | INFO | RTMetrics done | 20260507 20:38:30.996 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.61 seconds | 20260507 20:38:30.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.32% / 0.01% / 55.67% | 20260507 20:38:30.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94699% / 7.94729e-06% / 4.76837e-05% | 20260507 20:38:30.996 | INFO | Baz IO overall: | 20260507 20:38:30.996 | INFO | BytesWritten : 1103445917639 | 20260507 20:38:30.996 | INFO | NumEvents : 534913978922 | 20260507 20:38:32.430 | INFO | Analyzing chunk frames = [2428928,2429440) | 20260507 20:38:32.430 | INFO | MemInfo: { | 20260507 20:38:32.430 | INFO | "gpu" : " 17 GB", | 20260507 20:38:32.430 | INFO | "huge" : "", | 20260507 20:38:32.430 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:32.430 | INFO | "malloc" : " 14 GB", | 20260507 20:38:32.430 | INFO | "pinned" : " 29 GB" | 20260507 20:38:32.430 | INFO | } | 20260507 20:38:32.430 | INFO | Perf report for the last 5.1416 seconds: | 20260507 20:38:32.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:32.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:32.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:32.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:32.430 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 20:38:32.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:32.430 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 20:38:32.431 | INFO | PRE_HQ : 66.2%, 1.7, 58.5ms, 0 | 20260507 20:38:32.431 | INFO | BAZWRITER : 69.8%, 3.8, 137.3ms, 0 | 20260507 20:38:32.431 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 20:38:32.431 | INFO | Uploaded 36GiB to GPU in 2.52056s (14.2825GiB/s) | 20260507 20:38:32.431 | INFO | Downloaded 38GiB to GPU in 2.68375s (14.1593GiB/s) | 20260507 20:38:37.568 | INFO | Analyzing chunk frames = [2429440,2429952) | 20260507 20:38:37.568 | INFO | MemInfo: { | 20260507 20:38:37.568 | INFO | "gpu" : " 17 GB", | 20260507 20:38:37.568 | INFO | "huge" : "", | 20260507 20:38:37.568 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:37.568 | INFO | "malloc" : " 14 GB", | 20260507 20:38:37.568 | INFO | "pinned" : " 29 GB" | 20260507 20:38:37.568 | INFO | } | 20260507 20:38:37.568 | INFO | Perf report for the last 5.13738 seconds: | 20260507 20:38:37.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:37.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:38:37.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:38:37.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:37.568 | INFO | Uploaded 36GiB to GPU in 2.58609s (13.9206GiB/s) | 20260507 20:38:37.568 | INFO | Downloaded 34GiB to GPU in 2.4906s (13.6513GiB/s) | 20260507 20:38:42.706 | INFO | Analyzing chunk frames = [2429952,2430464) | 20260507 20:38:42.706 | INFO | MemInfo: { | 20260507 20:38:42.706 | INFO | "gpu" : " 17 GB", | 20260507 20:38:42.707 | INFO | "huge" : "", | 20260507 20:38:42.707 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:42.707 | INFO | "malloc" : " 14 GB", | 20260507 20:38:42.707 | INFO | "pinned" : " 29 GB" | 20260507 20:38:42.707 | INFO | } | 20260507 20:38:42.707 | INFO | Perf report for the last 5.13896 seconds: | 20260507 20:38:42.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:42.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260507 20:38:42.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:38:42.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:42.707 | INFO | Uploaded 36GiB to GPU in 2.59693s (13.8625GiB/s) | 20260507 20:38:42.707 | INFO | Downloaded 34GiB to GPU in 2.45906s (13.8264GiB/s) | 20260507 20:38:47.845 | INFO | Analyzing chunk frames = [2430464,2430976) | 20260507 20:38:47.845 | INFO | MemInfo: { | 20260507 20:38:47.845 | INFO | "gpu" : " 17 GB", | 20260507 20:38:47.845 | INFO | "huge" : "", | 20260507 20:38:47.845 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:47.845 | INFO | "malloc" : " 14 GB", | 20260507 20:38:47.846 | INFO | "pinned" : " 29 GB" | 20260507 20:38:47.846 | INFO | } | 20260507 20:38:47.846 | INFO | Perf report for the last 5.13887 seconds: | 20260507 20:38:47.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:47.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | ANALYSIS : 67.8%, 3.0, 104.8ms, 0 | 20260507 20:38:47.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:38:47.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:47.846 | INFO | Uploaded 36GiB to GPU in 2.54426s (14.1495GiB/s) | 20260507 20:38:47.846 | INFO | Downloaded 34GiB to GPU in 2.49979s (13.6012GiB/s) | 20260507 20:38:52.985 | INFO | Analyzing chunk frames = [2430976,2431488) | 20260507 20:38:52.985 | INFO | MemInfo: { | 20260507 20:38:52.985 | INFO | "gpu" : " 17 GB", | 20260507 20:38:52.985 | INFO | "huge" : "", | 20260507 20:38:52.985 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:52.985 | INFO | "malloc" : " 14 GB", | 20260507 20:38:52.985 | INFO | "pinned" : " 29 GB" | 20260507 20:38:52.985 | INFO | } | 20260507 20:38:52.985 | INFO | Perf report for the last 5.13983 seconds: | 20260507 20:38:52.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:52.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.986 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 20:38:52.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.986 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:38:52.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:52.986 | INFO | Uploaded 36GiB to GPU in 2.4049s (14.9694GiB/s) | 20260507 20:38:52.986 | INFO | Downloaded 34GiB to GPU in 2.49949s (13.6028GiB/s) | 20260507 20:38:58.125 | INFO | Analyzing chunk frames = [2431488,2432000) | 20260507 20:38:58.125 | INFO | MemInfo: { | 20260507 20:38:58.125 | INFO | "gpu" : " 17 GB", | 20260507 20:38:58.125 | INFO | "huge" : "", | 20260507 20:38:58.125 | INFO | "huge_shared" : " 29 GB", | 20260507 20:38:58.125 | INFO | "malloc" : " 14 GB", | 20260507 20:38:58.125 | INFO | "pinned" : " 29 GB" | 20260507 20:38:58.125 | INFO | } | 20260507 20:38:58.125 | INFO | Perf report for the last 5.13957 seconds: | 20260507 20:38:58.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:38:58.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 20:38:58.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:38:58.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:38:58.125 | INFO | Uploaded 36GiB to GPU in 2.38961s (15.0652GiB/s) | 20260507 20:38:58.125 | INFO | Downloaded 34GiB to GPU in 2.38197s (14.2739GiB/s) | 20260507 20:39:03.264 | INFO | Analyzing chunk frames = [2432000,2432512) | 20260507 20:39:03.264 | INFO | MemInfo: { | 20260507 20:39:03.264 | INFO | "gpu" : " 17 GB", | 20260507 20:39:03.264 | INFO | "huge" : "", | 20260507 20:39:03.264 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:03.264 | INFO | "malloc" : " 14 GB", | 20260507 20:39:03.264 | INFO | "pinned" : " 29 GB" | 20260507 20:39:03.264 | INFO | } | 20260507 20:39:03.264 | INFO | Perf report for the last 5.13908 seconds: | 20260507 20:39:03.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:03.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:39:03.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 20:39:03.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:03.264 | INFO | Uploaded 36GiB to GPU in 2.52386s (14.2639GiB/s) | 20260507 20:39:03.264 | INFO | Downloaded 34GiB to GPU in 2.4429s (13.9179GiB/s) | 20260507 20:39:08.404 | INFO | Analyzing chunk frames = [2432512,2433024) | 20260507 20:39:08.404 | INFO | MemInfo: { | 20260507 20:39:08.404 | INFO | "gpu" : " 17 GB", | 20260507 20:39:08.404 | INFO | "huge" : "", | 20260507 20:39:08.404 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:08.404 | INFO | "malloc" : " 14 GB", | 20260507 20:39:08.404 | INFO | "pinned" : " 29 GB" | 20260507 20:39:08.404 | INFO | } | 20260507 20:39:08.404 | INFO | Perf report for the last 5.13982 seconds: | 20260507 20:39:08.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:08.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:39:08.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:39:08.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:08.404 | INFO | Uploaded 36GiB to GPU in 2.50501s (14.3712GiB/s) | 20260507 20:39:08.404 | INFO | Downloaded 34GiB to GPU in 2.43759s (13.9482GiB/s) | 20260507 20:39:08.405 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:39:08.405 | INFO | Total mem usage: 16.4 GiB | 20260507 20:39:08.405 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:39:08.405 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.4%) | 20260507 20:39:08.405 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:39:08.405 | INFO | High Water Breakdown: | 20260507 20:39:08.405 | INFO | 59.7% : 8.79 GiB ( 88.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:39:08.405 | INFO | 40.2% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:39:08.405 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:39:08.405 | INFO | ------------------------------------------ | 20260507 20:39:08.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:39:08.405 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:39:08.405 | INFO | Total mem usage: 26.5 GiB | 20260507 20:39:08.405 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:39:08.405 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:39:08.405 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:39:08.405 | INFO | High Water Breakdown: | 20260507 20:39:08.405 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:39:08.405 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:39:08.405 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:39:08.405 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:39:08.405 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:39:08.405 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:39:08.405 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:39:08.405 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:39:08.405 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:39:08.405 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:39:08.405 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:39:08.405 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:39:08.405 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:39:08.405 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:39:08.405 | INFO | ------------------------------------------ | 20260507 20:39:08.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:39:08.405 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:39:08.405 | INFO | Total mem usage: 27.8 GiB | 20260507 20:39:08.405 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:39:08.405 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:39:08.405 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:39:08.405 | INFO | High Water Breakdown: | 20260507 20:39:08.405 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:39:08.405 | INFO | ------------------------------------------ | 20260507 20:39:08.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:39:08.405 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:39:08.405 | INFO | Total mem usage: 15.7 GiB | 20260507 20:39:08.405 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:39:08.405 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:39:08.405 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:39:08.405 | INFO | High Water Breakdown: | 20260507 20:39:08.405 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:39:08.405 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:39:08.405 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:39:08.405 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:39:08.405 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:39:08.405 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:39:08.405 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:39:08.405 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:39:08.405 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:39:08.405 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:39:08.405 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:39:08.405 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:39:08.405 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:39:08.405 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:39:08.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:39:08.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:39:08.405 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:39:08.405 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:39:08.405 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:39:08.405 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:39:08.405 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:39:08.405 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:39:08.405 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:39:08.405 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:39:08.405 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:39:08.405 | INFO | ------------------------------------------ | 20260507 20:39:08.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:39:08.406 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:39:08.406 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:39:08.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:39:08.406 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.41e+01 / 1.90e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 0.29%, AggregateStats -- 9.73e-02 / 1.01e-01, 1.11e-02 / 3.94e-03 | 20260507 20:39:08.406 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.49e+00, 6.01e-01 / 2.23e-01 | 20260507 20:39:08.406 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.20e-01 | 20260507 20:39:08.406 | INFO | PerfReport: 15.87%, DME -- 4.20e+01 / 4.52e+01, 3.96e+00 / 1.65e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.72e+00 / 1.63e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.45e-01 / 2.15e-01 | 20260507 20:39:08.406 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.38e+00, 2.99e-01 / 1.71e-01 | 20260507 20:39:08.406 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.68e-01 / 4.00e-01, 6.01e-02 / 3.05e-02 | 20260507 20:39:08.406 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:39:08.406 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:39:08.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:39:08.406 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 24.14%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 30.92%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 2.20%, LockUpload -- 2.35e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 2.33%, LockDownload -- 2.49e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 15.98%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 20:39:08.406 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260507 20:39:08.501 | INFO | Starting a round of Baz IO | 20260507 20:39:11.983 | INFO | Populating RTMetrics report | 20260507 20:39:11.984 | INFO | RTMetrics report populated | 20260507 20:39:11.984 | INFO | Opening tmp RTMetrics json file | 20260507 20:39:12.016 | INFO | Serializing RTMetrics to json | 20260507 20:39:12.057 | INFO | Serialization done, swapping tmp file now... | 20260507 20:39:12.067 | INFO | RTMetrics done | 20260507 20:39:12.087 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.586 seconds | 20260507 20:39:12.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.31% / 0.01% / 55.68% | 20260507 20:39:12.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94042% / 1.58946e-05% / 6.35783e-05% | 20260507 20:39:12.088 | INFO | Baz IO overall: | 20260507 20:39:12.088 | INFO | BytesWritten : 1105351302085 | 20260507 20:39:12.088 | INFO | NumEvents : 535836072194 | 20260507 20:39:13.543 | INFO | Analyzing chunk frames = [2433024,2433536) | 20260507 20:39:13.543 | INFO | MemInfo: { | 20260507 20:39:13.543 | INFO | "gpu" : " 17 GB", | 20260507 20:39:13.543 | INFO | "huge" : "", | 20260507 20:39:13.543 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:13.543 | INFO | "malloc" : " 14 GB", | 20260507 20:39:13.543 | INFO | "pinned" : " 29 GB" | 20260507 20:39:13.543 | INFO | } | 20260507 20:39:13.543 | INFO | Perf report for the last 5.13938 seconds: | 20260507 20:39:13.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:13.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:13.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:13.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:13.544 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:39:13.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:13.544 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 20:39:13.544 | INFO | PRE_HQ : 66.0%, 1.7, 58.3ms, 0 | 20260507 20:39:13.544 | INFO | BAZWRITER : 69.4%, 4.0, 141.3ms, 0 | 20260507 20:39:13.544 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:39:13.544 | INFO | Uploaded 36GiB to GPU in 2.52237s (14.2723GiB/s) | 20260507 20:39:13.544 | INFO | Downloaded 38GiB to GPU in 2.68544s (14.1504GiB/s) | 20260507 20:39:18.682 | INFO | Analyzing chunk frames = [2433536,2434048) | 20260507 20:39:18.682 | INFO | MemInfo: { | 20260507 20:39:18.682 | INFO | "gpu" : " 17 GB", | 20260507 20:39:18.682 | INFO | "huge" : "", | 20260507 20:39:18.682 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:18.682 | INFO | "malloc" : " 14 GB", | 20260507 20:39:18.682 | INFO | "pinned" : " 29 GB" | 20260507 20:39:18.682 | INFO | } | 20260507 20:39:18.682 | INFO | Perf report for the last 5.13863 seconds: | 20260507 20:39:18.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:18.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 20:39:18.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:39:18.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:18.682 | INFO | Uploaded 36GiB to GPU in 2.55492s (14.0905GiB/s) | 20260507 20:39:18.682 | INFO | Downloaded 34GiB to GPU in 2.50196s (13.5893GiB/s) | 20260507 20:39:23.821 | INFO | Analyzing chunk frames = [2434048,2434560) | 20260507 20:39:23.821 | INFO | MemInfo: { | 20260507 20:39:23.821 | INFO | "gpu" : " 17 GB", | 20260507 20:39:23.821 | INFO | "huge" : "", | 20260507 20:39:23.821 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:23.821 | INFO | "malloc" : " 14 GB", | 20260507 20:39:23.822 | INFO | "pinned" : " 29 GB" | 20260507 20:39:23.822 | INFO | } | 20260507 20:39:23.822 | INFO | Perf report for the last 5.13965 seconds: | 20260507 20:39:23.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:23.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 20:39:23.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:39:23.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:23.822 | INFO | Uploaded 36GiB to GPU in 2.47056s (14.5716GiB/s) | 20260507 20:39:23.822 | INFO | Downloaded 34GiB to GPU in 2.4878s (13.6667GiB/s) | 20260507 20:39:28.960 | INFO | Analyzing chunk frames = [2434560,2435072) | 20260507 20:39:28.960 | INFO | MemInfo: { | 20260507 20:39:28.960 | INFO | "gpu" : " 17 GB", | 20260507 20:39:28.960 | INFO | "huge" : "", | 20260507 20:39:28.960 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:28.960 | INFO | "malloc" : " 14 GB", | 20260507 20:39:28.960 | INFO | "pinned" : " 29 GB" | 20260507 20:39:28.960 | INFO | } | 20260507 20:39:28.960 | INFO | Perf report for the last 5.13851 seconds: | 20260507 20:39:28.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:28.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 20:39:28.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260507 20:39:28.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:28.960 | INFO | Uploaded 36GiB to GPU in 2.46902s (14.5807GiB/s) | 20260507 20:39:28.960 | INFO | Downloaded 34GiB to GPU in 2.4978s (13.612GiB/s) | 20260507 20:39:34.100 | INFO | Analyzing chunk frames = [2435072,2435584) | 20260507 20:39:34.100 | INFO | MemInfo: { | 20260507 20:39:34.100 | INFO | "gpu" : " 17 GB", | 20260507 20:39:34.100 | INFO | "huge" : "", | 20260507 20:39:34.100 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:34.100 | INFO | "malloc" : " 14 GB", | 20260507 20:39:34.101 | INFO | "pinned" : " 29 GB" | 20260507 20:39:34.101 | INFO | } | 20260507 20:39:34.101 | INFO | Perf report for the last 5.14043 seconds: | 20260507 20:39:34.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:34.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:39:34.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:39:34.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:34.101 | INFO | Uploaded 36GiB to GPU in 2.3836s (15.1032GiB/s) | 20260507 20:39:34.102 | INFO | Downloaded 34GiB to GPU in 2.50339s (13.5816GiB/s) | 20260507 20:39:39.240 | INFO | Analyzing chunk frames = [2435584,2436096) | 20260507 20:39:39.240 | INFO | MemInfo: { | 20260507 20:39:39.240 | INFO | "gpu" : " 17 GB", | 20260507 20:39:39.240 | INFO | "huge" : "", | 20260507 20:39:39.240 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:39.240 | INFO | "malloc" : " 14 GB", | 20260507 20:39:39.240 | INFO | "pinned" : " 29 GB" | 20260507 20:39:39.240 | INFO | } | 20260507 20:39:39.240 | INFO | Perf report for the last 5.13955 seconds: | 20260507 20:39:39.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:39.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.240 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:39:39.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.240 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260507 20:39:39.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:39.241 | INFO | Uploaded 36GiB to GPU in 2.40022s (14.9986GiB/s) | 20260507 20:39:39.241 | INFO | Downloaded 34GiB to GPU in 2.39182s (14.2151GiB/s) | 20260507 20:39:44.378 | INFO | Analyzing chunk frames = [2436096,2436608) | 20260507 20:39:44.378 | INFO | MemInfo: { | 20260507 20:39:44.378 | INFO | "gpu" : " 17 GB", | 20260507 20:39:44.378 | INFO | "huge" : "", | 20260507 20:39:44.379 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:44.379 | INFO | "malloc" : " 14 GB", | 20260507 20:39:44.379 | INFO | "pinned" : " 29 GB" | 20260507 20:39:44.379 | INFO | } | 20260507 20:39:44.379 | INFO | Perf report for the last 5.13846 seconds: | 20260507 20:39:44.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:44.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:39:44.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 20:39:44.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:44.379 | INFO | Uploaded 36GiB to GPU in 2.54466s (14.1473GiB/s) | 20260507 20:39:44.379 | INFO | Downloaded 34GiB to GPU in 2.46635s (13.7855GiB/s) | 20260507 20:39:49.518 | INFO | Analyzing chunk frames = [2436608,2437120) | 20260507 20:39:49.519 | INFO | MemInfo: { | 20260507 20:39:49.519 | INFO | "gpu" : " 17 GB", | 20260507 20:39:49.519 | INFO | "huge" : "", | 20260507 20:39:49.519 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:49.519 | INFO | "malloc" : " 14 GB", | 20260507 20:39:49.519 | INFO | "pinned" : " 29 GB" | 20260507 20:39:49.519 | INFO | } | 20260507 20:39:49.519 | INFO | Perf report for the last 5.14009 seconds: | 20260507 20:39:49.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:49.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:39:49.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | PRE_HQ : 37.0%, 1.0, 19.2ms, 0 | 20260507 20:39:49.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:49.519 | INFO | Uploaded 36GiB to GPU in 2.51669s (14.3045GiB/s) | 20260507 20:39:49.519 | INFO | Downloaded 34GiB to GPU in 2.43275s (13.9759GiB/s) | 20260507 20:39:49.616 | INFO | Starting a round of Baz IO | 20260507 20:39:53.084 | INFO | Populating RTMetrics report | 20260507 20:39:53.085 | INFO | RTMetrics report populated | 20260507 20:39:53.085 | INFO | Opening tmp RTMetrics json file | 20260507 20:39:53.086 | INFO | Serializing RTMetrics to json | 20260507 20:39:53.095 | INFO | Serialization done, swapping tmp file now... | 20260507 20:39:53.116 | INFO | RTMetrics done | 20260507 20:39:53.197 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.58 seconds | 20260507 20:39:53.197 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.30% / 0.01% / 55.69% | 20260507 20:39:53.197 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93847% / 1.58946e-05% / 5.16574e-05% | 20260507 20:39:53.197 | INFO | Baz IO overall: | 20260507 20:39:53.197 | INFO | BytesWritten : 1107256977679 | 20260507 20:39:53.197 | INFO | NumEvents : 536758711942 | 20260507 20:39:54.657 | INFO | Analyzing chunk frames = [2437120,2437632) | 20260507 20:39:54.658 | INFO | MemInfo: { | 20260507 20:39:54.658 | INFO | "gpu" : " 17 GB", | 20260507 20:39:54.658 | INFO | "huge" : "", | 20260507 20:39:54.658 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:54.658 | INFO | "malloc" : " 14 GB", | 20260507 20:39:54.658 | INFO | "pinned" : " 29 GB" | 20260507 20:39:54.658 | INFO | } | 20260507 20:39:54.658 | INFO | Perf report for the last 5.13898 seconds: | 20260507 20:39:54.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:54.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:54.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:54.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:54.658 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 20:39:54.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:54.658 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:39:54.658 | INFO | PRE_HQ : 66.3%, 1.7, 58.5ms, 0 | 20260507 20:39:54.658 | INFO | BAZWRITER : 69.2%, 3.9, 139.0ms, 0 | 20260507 20:39:54.658 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:39:54.658 | INFO | Uploaded 36GiB to GPU in 2.49852s (14.4086GiB/s) | 20260507 20:39:54.658 | INFO | Downloaded 38GiB to GPU in 2.68736s (14.1403GiB/s) | 20260507 20:39:59.797 | INFO | Analyzing chunk frames = [2437632,2438144) | 20260507 20:39:59.797 | INFO | MemInfo: { | 20260507 20:39:59.797 | INFO | "gpu" : " 17 GB", | 20260507 20:39:59.797 | INFO | "huge" : "", | 20260507 20:39:59.797 | INFO | "huge_shared" : " 29 GB", | 20260507 20:39:59.797 | INFO | "malloc" : " 14 GB", | 20260507 20:39:59.797 | INFO | "pinned" : " 29 GB" | 20260507 20:39:59.797 | INFO | } | 20260507 20:39:59.797 | INFO | Perf report for the last 5.13937 seconds: | 20260507 20:39:59.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:39:59.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.797 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:39:59.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.797 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:39:59.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:39:59.798 | INFO | Uploaded 36GiB to GPU in 2.56174s (14.053GiB/s) | 20260507 20:39:59.798 | INFO | Downloaded 34GiB to GPU in 2.49048s (13.652GiB/s) | 20260507 20:40:04.937 | INFO | Analyzing chunk frames = [2438144,2438656) | 20260507 20:40:04.937 | INFO | MemInfo: { | 20260507 20:40:04.937 | INFO | "gpu" : " 17 GB", | 20260507 20:40:04.937 | INFO | "huge" : "", | 20260507 20:40:04.937 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:04.937 | INFO | "malloc" : " 14 GB", | 20260507 20:40:04.937 | INFO | "pinned" : " 29 GB" | 20260507 20:40:04.937 | INFO | } | 20260507 20:40:04.937 | INFO | Perf report for the last 5.13969 seconds: | 20260507 20:40:04.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:04.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:40:04.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:40:04.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:04.937 | INFO | Uploaded 36GiB to GPU in 2.57909s (13.9584GiB/s) | 20260507 20:40:04.937 | INFO | Downloaded 34GiB to GPU in 2.47393s (13.7433GiB/s) | 20260507 20:40:10.075 | INFO | Analyzing chunk frames = [2438656,2439168) | 20260507 20:40:10.075 | INFO | MemInfo: { | 20260507 20:40:10.075 | INFO | "gpu" : " 17 GB", | 20260507 20:40:10.075 | INFO | "huge" : "", | 20260507 20:40:10.075 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:10.075 | INFO | "malloc" : " 14 GB", | 20260507 20:40:10.076 | INFO | "pinned" : " 29 GB" | 20260507 20:40:10.076 | INFO | } | 20260507 20:40:10.076 | INFO | Perf report for the last 5.13869 seconds: | 20260507 20:40:10.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:10.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 20:40:10.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:40:10.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:10.076 | INFO | Uploaded 36GiB to GPU in 2.52515s (14.2566GiB/s) | 20260507 20:40:10.076 | INFO | Downloaded 34GiB to GPU in 2.50101s (13.5945GiB/s) | 20260507 20:40:15.215 | INFO | Analyzing chunk frames = [2439168,2439680) | 20260507 20:40:15.215 | INFO | MemInfo: { | 20260507 20:40:15.215 | INFO | "gpu" : " 17 GB", | 20260507 20:40:15.215 | INFO | "huge" : "", | 20260507 20:40:15.215 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:15.215 | INFO | "malloc" : " 14 GB", | 20260507 20:40:15.215 | INFO | "pinned" : " 29 GB" | 20260507 20:40:15.215 | INFO | } | 20260507 20:40:15.215 | INFO | Perf report for the last 5.13972 seconds: | 20260507 20:40:15.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:15.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.215 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:40:15.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.215 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:40:15.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:15.216 | INFO | Uploaded 36GiB to GPU in 2.39504s (15.0311GiB/s) | 20260507 20:40:15.216 | INFO | Downloaded 34GiB to GPU in 2.48592s (13.677GiB/s) | 20260507 20:40:20.354 | INFO | Analyzing chunk frames = [2439680,2440192) | 20260507 20:40:20.354 | INFO | MemInfo: { | 20260507 20:40:20.354 | INFO | "gpu" : " 17 GB", | 20260507 20:40:20.354 | INFO | "huge" : "", | 20260507 20:40:20.354 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:20.354 | INFO | "malloc" : " 14 GB", | 20260507 20:40:20.354 | INFO | "pinned" : " 29 GB" | 20260507 20:40:20.354 | INFO | } | 20260507 20:40:20.354 | INFO | Perf report for the last 5.13844 seconds: | 20260507 20:40:20.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:20.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:40:20.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 20:40:20.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:20.354 | INFO | Uploaded 36GiB to GPU in 2.4264s (14.8368GiB/s) | 20260507 20:40:20.354 | INFO | Downloaded 34GiB to GPU in 2.3785s (14.2947GiB/s) | 20260507 20:40:25.493 | INFO | Analyzing chunk frames = [2440192,2440704) | 20260507 20:40:25.493 | INFO | MemInfo: { | 20260507 20:40:25.493 | INFO | "gpu" : " 17 GB", | 20260507 20:40:25.493 | INFO | "huge" : "", | 20260507 20:40:25.493 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:25.493 | INFO | "malloc" : " 14 GB", | 20260507 20:40:25.493 | INFO | "pinned" : " 29 GB" | 20260507 20:40:25.493 | INFO | } | 20260507 20:40:25.494 | INFO | Perf report for the last 5.13976 seconds: | 20260507 20:40:25.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:25.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:40:25.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | PRE_HQ : 36.3%, 1.0, 18.9ms, 0 | 20260507 20:40:25.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:25.494 | INFO | Uploaded 36GiB to GPU in 2.5538s (14.0967GiB/s) | 20260507 20:40:25.494 | INFO | Downloaded 34GiB to GPU in 2.45052s (13.8746GiB/s) | 20260507 20:40:30.632 | INFO | Analyzing chunk frames = [2440704,2441216) | 20260507 20:40:30.633 | INFO | MemInfo: { | 20260507 20:40:30.633 | INFO | "gpu" : " 17 GB", | 20260507 20:40:30.633 | INFO | "huge" : "", | 20260507 20:40:30.633 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:30.633 | INFO | "malloc" : " 14 GB", | 20260507 20:40:30.633 | INFO | "pinned" : " 29 GB" | 20260507 20:40:30.633 | INFO | } | 20260507 20:40:30.633 | INFO | Perf report for the last 5.13938 seconds: | 20260507 20:40:30.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:30.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.633 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:40:30.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.633 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 20:40:30.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:30.634 | INFO | Uploaded 36GiB to GPU in 2.53384s (14.2077GiB/s) | 20260507 20:40:30.634 | INFO | Downloaded 34GiB to GPU in 2.43272s (13.9761GiB/s) | 20260507 20:40:30.634 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:40:30.634 | INFO | Total mem usage: 16.4 GiB | 20260507 20:40:30.634 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:40:30.634 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.4%) | 20260507 20:40:30.634 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:40:30.634 | INFO | High Water Breakdown: | 20260507 20:40:30.634 | INFO | 59.7% : 8.79 GiB ( 88.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:40:30.634 | INFO | 40.2% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:40:30.634 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:40:30.634 | INFO | ------------------------------------------ | 20260507 20:40:30.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:40:30.634 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:40:30.634 | INFO | Total mem usage: 26.5 GiB | 20260507 20:40:30.634 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:40:30.634 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:40:30.634 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:40:30.634 | INFO | High Water Breakdown: | 20260507 20:40:30.634 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:40:30.634 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:40:30.634 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:40:30.634 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:40:30.634 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:40:30.634 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:40:30.634 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:40:30.634 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:40:30.634 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:40:30.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:40:30.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:40:30.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:40:30.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:40:30.634 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:40:30.634 | INFO | ------------------------------------------ | 20260507 20:40:30.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:40:30.634 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:40:30.634 | INFO | Total mem usage: 27.8 GiB | 20260507 20:40:30.634 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:40:30.634 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:40:30.634 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:40:30.634 | INFO | High Water Breakdown: | 20260507 20:40:30.634 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:40:30.634 | INFO | ------------------------------------------ | 20260507 20:40:30.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:40:30.634 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:40:30.634 | INFO | Total mem usage: 15.7 GiB | 20260507 20:40:30.634 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:40:30.634 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:40:30.635 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:40:30.635 | INFO | High Water Breakdown: | 20260507 20:40:30.635 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:40:30.635 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:40:30.635 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:40:30.635 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:40:30.635 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:40:30.635 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:40:30.635 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:40:30.635 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:40:30.635 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:40:30.635 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:40:30.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:40:30.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:40:30.635 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:40:30.635 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:40:30.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:40:30.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:40:30.635 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:40:30.635 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:40:30.635 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:40:30.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:40:30.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:40:30.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:40:30.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:40:30.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:40:30.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:40:30.635 | INFO | ------------------------------------------ | 20260507 20:40:30.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:40:30.635 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:40:30.635 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:40:30.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:40:30.635 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.62e+01, 1.42e+01 / 2.23e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 0.30%, AggregateStats -- 1.01e-01 / 1.05e-01, 1.31e-02 / 6.42e-03 | 20260507 20:40:30.635 | INFO | PerfReport: 12.13%, Baseline -- 4.02e+00 / 4.46e+00, 5.89e-01 / 2.46e-01 | 20260507 20:40:30.635 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.51e+00, 2.09e-01 / 1.14e-01 | 20260507 20:40:30.635 | INFO | PerfReport: 16.00%, DME -- 4.24e+01 / 4.58e+01, 4.07e+00 / 2.20e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.65e+00 / 1.52e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.52e+00 / 2.84e+00, 4.43e-01 / 2.22e-01 | 20260507 20:40:30.635 | INFO | PerfReport: 3.58%, Metrics -- 1.19e+00 / 1.40e+00, 2.96e-01 / 1.74e-01 | 20260507 20:40:30.635 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.72e-01 / 4.06e-01, 5.99e-02 / 3.30e-02 | 20260507 20:40:30.635 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:40:30.635 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:40:30.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:40:30.635 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 24.29%, Upload -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 24.25%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 30.93%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 2.01%, LockUpload -- 2.15e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 2.62%, LockDownload -- 2.80e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260507 20:40:30.635 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260507 20:40:30.746 | INFO | Starting a round of Baz IO | 20260507 20:40:34.243 | INFO | Populating RTMetrics report | 20260507 20:40:34.244 | INFO | RTMetrics report populated | 20260507 20:40:34.244 | INFO | Opening tmp RTMetrics json file | 20260507 20:40:34.256 | INFO | Serializing RTMetrics to json | 20260507 20:40:34.270 | INFO | Serialization done, swapping tmp file now... | 20260507 20:40:34.271 | INFO | RTMetrics done | 20260507 20:40:34.357 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.61 seconds | 20260507 20:40:34.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.29% / 0.01% / 55.70% | 20260507 20:40:34.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9357% / 1.58946e-05% / 3.97364e-05% | 20260507 20:40:34.357 | INFO | Baz IO overall: | 20260507 20:40:34.357 | INFO | BytesWritten : 1109162834246 | 20260507 20:40:34.357 | INFO | NumEvents : 537681670477 | 20260507 20:40:35.773 | INFO | Analyzing chunk frames = [2441216,2441728) | 20260507 20:40:35.773 | INFO | MemInfo: { | 20260507 20:40:35.773 | INFO | "gpu" : " 17 GB", | 20260507 20:40:35.773 | INFO | "huge" : "", | 20260507 20:40:35.773 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:35.773 | INFO | "malloc" : " 14 GB", | 20260507 20:40:35.773 | INFO | "pinned" : " 29 GB" | 20260507 20:40:35.773 | INFO | } | 20260507 20:40:35.774 | INFO | Perf report for the last 5.14068 seconds: | 20260507 20:40:35.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:35.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:35.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:35.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:35.774 | INFO | ANALYSIS : 69.5%, 2.9, 104.7ms, 0 | 20260507 20:40:35.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:35.774 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260507 20:40:35.774 | INFO | PRE_HQ : 68.6%, 1.8, 65.7ms, 0 | 20260507 20:40:35.774 | INFO | BAZWRITER : 69.9%, 4.7, 168.1ms, 0 | 20260507 20:40:35.774 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:40:35.774 | INFO | Uploaded 36GiB to GPU in 2.58226s (13.9413GiB/s) | 20260507 20:40:35.774 | INFO | Downloaded 38GiB to GPU in 2.70757s (14.0347GiB/s) | 20260507 20:40:40.911 | INFO | Analyzing chunk frames = [2441728,2442240) | 20260507 20:40:40.911 | INFO | MemInfo: { | 20260507 20:40:40.911 | INFO | "gpu" : " 17 GB", | 20260507 20:40:40.911 | INFO | "huge" : "", | 20260507 20:40:40.911 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:40.911 | INFO | "malloc" : " 14 GB", | 20260507 20:40:40.911 | INFO | "pinned" : " 29 GB" | 20260507 20:40:40.912 | INFO | } | 20260507 20:40:40.912 | INFO | Perf report for the last 5.13794 seconds: | 20260507 20:40:40.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:40.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 20:40:40.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:40:40.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:40.912 | INFO | Uploaded 36GiB to GPU in 2.58782s (13.9113GiB/s) | 20260507 20:40:40.912 | INFO | Downloaded 34GiB to GPU in 2.47347s (13.7459GiB/s) | 20260507 20:40:46.050 | INFO | Analyzing chunk frames = [2442240,2442752) | 20260507 20:40:46.050 | INFO | MemInfo: { | 20260507 20:40:46.050 | INFO | "gpu" : " 17 GB", | 20260507 20:40:46.050 | INFO | "huge" : "", | 20260507 20:40:46.050 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:46.050 | INFO | "malloc" : " 14 GB", | 20260507 20:40:46.050 | INFO | "pinned" : " 29 GB" | 20260507 20:40:46.050 | INFO | } | 20260507 20:40:46.050 | INFO | Perf report for the last 5.13859 seconds: | 20260507 20:40:46.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:46.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:40:46.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:40:46.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:46.050 | INFO | Uploaded 36GiB to GPU in 2.5941s (13.8776GiB/s) | 20260507 20:40:46.050 | INFO | Downloaded 34GiB to GPU in 2.46287s (13.805GiB/s) | 20260507 20:40:51.189 | INFO | Analyzing chunk frames = [2442752,2443264) | 20260507 20:40:51.189 | INFO | MemInfo: { | 20260507 20:40:51.189 | INFO | "gpu" : " 17 GB", | 20260507 20:40:51.189 | INFO | "huge" : "", | 20260507 20:40:51.189 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:51.189 | INFO | "malloc" : " 14 GB", | 20260507 20:40:51.190 | INFO | "pinned" : " 29 GB" | 20260507 20:40:51.190 | INFO | } | 20260507 20:40:51.190 | INFO | Perf report for the last 5.13951 seconds: | 20260507 20:40:51.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:51.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260507 20:40:51.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:40:51.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:51.190 | INFO | Uploaded 36GiB to GPU in 2.51475s (14.3155GiB/s) | 20260507 20:40:51.190 | INFO | Downloaded 34GiB to GPU in 2.39643s (14.1878GiB/s) | 20260507 20:40:56.330 | INFO | Analyzing chunk frames = [2443264,2443776) | 20260507 20:40:56.330 | INFO | MemInfo: { | 20260507 20:40:56.330 | INFO | "gpu" : " 17 GB", | 20260507 20:40:56.330 | INFO | "huge" : "", | 20260507 20:40:56.330 | INFO | "huge_shared" : " 29 GB", | 20260507 20:40:56.330 | INFO | "malloc" : " 14 GB", | 20260507 20:40:56.330 | INFO | "pinned" : " 29 GB" | 20260507 20:40:56.330 | INFO | } | 20260507 20:40:56.330 | INFO | Perf report for the last 5.14062 seconds: | 20260507 20:40:56.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:40:56.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.330 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 20:40:56.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.330 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:40:56.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:40:56.331 | INFO | Uploaded 36GiB to GPU in 2.40429s (14.9732GiB/s) | 20260507 20:40:56.331 | INFO | Downloaded 34GiB to GPU in 2.49081s (13.6502GiB/s) | 20260507 20:41:01.469 | INFO | Analyzing chunk frames = [2443776,2444288) | 20260507 20:41:01.469 | INFO | MemInfo: { | 20260507 20:41:01.469 | INFO | "gpu" : " 17 GB", | 20260507 20:41:01.469 | INFO | "huge" : "", | 20260507 20:41:01.469 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:01.469 | INFO | "malloc" : " 14 GB", | 20260507 20:41:01.469 | INFO | "pinned" : " 29 GB" | 20260507 20:41:01.469 | INFO | } | 20260507 20:41:01.469 | INFO | Perf report for the last 5.13889 seconds: | 20260507 20:41:01.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:01.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.469 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 20:41:01.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.469 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:41:01.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:01.470 | INFO | Uploaded 36GiB to GPU in 2.40487s (14.9697GiB/s) | 20260507 20:41:01.470 | INFO | Downloaded 34GiB to GPU in 2.3904s (14.2236GiB/s) | 20260507 20:41:06.610 | INFO | Analyzing chunk frames = [2444288,2444800) | 20260507 20:41:06.610 | INFO | MemInfo: { | 20260507 20:41:06.610 | INFO | "gpu" : " 17 GB", | 20260507 20:41:06.610 | INFO | "huge" : "", | 20260507 20:41:06.610 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:06.610 | INFO | "malloc" : " 14 GB", | 20260507 20:41:06.610 | INFO | "pinned" : " 29 GB" | 20260507 20:41:06.610 | INFO | } | 20260507 20:41:06.610 | INFO | Perf report for the last 5.14097 seconds: | 20260507 20:41:06.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:06.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.610 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:41:06.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.610 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 20:41:06.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:06.611 | INFO | Uploaded 36GiB to GPU in 2.52083s (14.281GiB/s) | 20260507 20:41:06.611 | INFO | Downloaded 34GiB to GPU in 2.45358s (13.8573GiB/s) | 20260507 20:41:11.749 | INFO | Analyzing chunk frames = [2444800,2445312) | 20260507 20:41:11.749 | INFO | MemInfo: { | 20260507 20:41:11.749 | INFO | "gpu" : " 17 GB", | 20260507 20:41:11.749 | INFO | "huge" : "", | 20260507 20:41:11.749 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:11.749 | INFO | "malloc" : " 14 GB", | 20260507 20:41:11.749 | INFO | "pinned" : " 29 GB" | 20260507 20:41:11.749 | INFO | } | 20260507 20:41:11.749 | INFO | Perf report for the last 5.13889 seconds: | 20260507 20:41:11.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:11.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.749 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:41:11.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.749 | INFO | PRE_HQ : 37.4%, 1.0, 19.4ms, 0 | 20260507 20:41:11.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:11.750 | INFO | Uploaded 36GiB to GPU in 2.52349s (14.266GiB/s) | 20260507 20:41:11.750 | INFO | Downloaded 34GiB to GPU in 2.43588s (13.958GiB/s) | 20260507 20:41:11.864 | INFO | Starting a round of Baz IO | 20260507 20:41:15.354 | INFO | Populating RTMetrics report | 20260507 20:41:15.354 | INFO | RTMetrics report populated | 20260507 20:41:15.354 | INFO | Opening tmp RTMetrics json file | 20260507 20:41:15.377 | INFO | Serializing RTMetrics to json | 20260507 20:41:15.380 | INFO | Serialization done, swapping tmp file now... | 20260507 20:41:15.381 | INFO | RTMetrics done | 20260507 20:41:15.457 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.593 seconds | 20260507 20:41:15.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.29% / 0.01% / 55.71% | 20260507 20:41:15.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93032% / 0% / 4.37101e-05% | 20260507 20:41:15.457 | INFO | Baz IO overall: | 20260507 20:41:15.457 | INFO | BytesWritten : 1111067524102 | 20260507 20:41:15.457 | INFO | NumEvents : 538603714375 | 20260507 20:41:16.886 | INFO | Analyzing chunk frames = [2445312,2445824) | 20260507 20:41:16.886 | INFO | MemInfo: { | 20260507 20:41:16.886 | INFO | "gpu" : " 17 GB", | 20260507 20:41:16.886 | INFO | "huge" : "", | 20260507 20:41:16.886 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:16.886 | INFO | "malloc" : " 14 GB", | 20260507 20:41:16.887 | INFO | "pinned" : " 29 GB" | 20260507 20:41:16.887 | INFO | } | 20260507 20:41:16.887 | INFO | Perf report for the last 5.13748 seconds: | 20260507 20:41:16.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:16.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:16.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:16.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:16.887 | INFO | ANALYSIS : 69.5%, 2.9, 105.1ms, 0 | 20260507 20:41:16.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:16.887 | INFO | RT_METRICS : 46.1%, 1.0, 24.1ms, 0 | 20260507 20:41:16.887 | INFO | PRE_HQ : 68.3%, 1.8, 64.7ms, 0 | 20260507 20:41:16.887 | INFO | BAZWRITER : 69.6%, 4.1, 144.9ms, 0 | 20260507 20:41:16.887 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 20:41:16.887 | INFO | Uploaded 36GiB to GPU in 2.53518s (14.2002GiB/s) | 20260507 20:41:16.887 | INFO | Downloaded 38GiB to GPU in 2.72061s (13.9675GiB/s) | 20260507 20:41:22.025 | INFO | Analyzing chunk frames = [2445824,2446336) | 20260507 20:41:22.025 | INFO | MemInfo: { | 20260507 20:41:22.025 | INFO | "gpu" : " 17 GB", | 20260507 20:41:22.025 | INFO | "huge" : "", | 20260507 20:41:22.025 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:22.025 | INFO | "malloc" : " 14 GB", | 20260507 20:41:22.025 | INFO | "pinned" : " 29 GB" | 20260507 20:41:22.025 | INFO | } | 20260507 20:41:22.025 | INFO | Perf report for the last 5.13881 seconds: | 20260507 20:41:22.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:22.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.026 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 20:41:22.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.026 | INFO | PRE_HQ : 25.6%, 1.0, 13.5ms, 0 | 20260507 20:41:22.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:22.026 | INFO | Uploaded 36GiB to GPU in 2.54362s (14.153GiB/s) | 20260507 20:41:22.026 | INFO | Downloaded 34GiB to GPU in 2.49607s (13.6214GiB/s) | 20260507 20:41:27.165 | INFO | Analyzing chunk frames = [2446336,2446848) | 20260507 20:41:27.165 | INFO | MemInfo: { | 20260507 20:41:27.165 | INFO | "gpu" : " 17 GB", | 20260507 20:41:27.165 | INFO | "huge" : "", | 20260507 20:41:27.165 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:27.165 | INFO | "malloc" : " 14 GB", | 20260507 20:41:27.165 | INFO | "pinned" : " 29 GB" | 20260507 20:41:27.165 | INFO | } | 20260507 20:41:27.165 | INFO | Perf report for the last 5.1397 seconds: | 20260507 20:41:27.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:27.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 20:41:27.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:41:27.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:27.165 | INFO | Uploaded 36GiB to GPU in 2.51933s (14.2895GiB/s) | 20260507 20:41:27.166 | INFO | Downloaded 34GiB to GPU in 2.48582s (13.6776GiB/s) | 20260507 20:41:32.304 | INFO | Analyzing chunk frames = [2446848,2447360) | 20260507 20:41:32.304 | INFO | MemInfo: { | 20260507 20:41:32.304 | INFO | "gpu" : " 17 GB", | 20260507 20:41:32.304 | INFO | "huge" : "", | 20260507 20:41:32.304 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:32.304 | INFO | "malloc" : " 14 GB", | 20260507 20:41:32.304 | INFO | "pinned" : " 29 GB" | 20260507 20:41:32.304 | INFO | } | 20260507 20:41:32.305 | INFO | Perf report for the last 5.13935 seconds: | 20260507 20:41:32.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:32.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 20:41:32.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:41:32.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:32.305 | INFO | Uploaded 36GiB to GPU in 2.42587s (14.8401GiB/s) | 20260507 20:41:32.305 | INFO | Downloaded 34GiB to GPU in 2.42128s (14.0422GiB/s) | 20260507 20:41:37.444 | INFO | Analyzing chunk frames = [2447360,2447872) | 20260507 20:41:37.444 | INFO | MemInfo: { | 20260507 20:41:37.444 | INFO | "gpu" : " 17 GB", | 20260507 20:41:37.444 | INFO | "huge" : "", | 20260507 20:41:37.444 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:37.444 | INFO | "malloc" : " 14 GB", | 20260507 20:41:37.445 | INFO | "pinned" : " 29 GB" | 20260507 20:41:37.445 | INFO | } | 20260507 20:41:37.445 | INFO | Perf report for the last 5.1401 seconds: | 20260507 20:41:37.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:37.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 20:41:37.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 20:41:37.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:37.445 | INFO | Uploaded 36GiB to GPU in 2.36743s (15.2064GiB/s) | 20260507 20:41:37.445 | INFO | Downloaded 34GiB to GPU in 2.50038s (13.5979GiB/s) | 20260507 20:41:42.585 | INFO | Analyzing chunk frames = [2447872,2448384) | 20260507 20:41:42.585 | INFO | MemInfo: { | 20260507 20:41:42.585 | INFO | "gpu" : " 17 GB", | 20260507 20:41:42.585 | INFO | "huge" : "", | 20260507 20:41:42.585 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:42.585 | INFO | "malloc" : " 14 GB", | 20260507 20:41:42.585 | INFO | "pinned" : " 29 GB" | 20260507 20:41:42.585 | INFO | } | 20260507 20:41:42.585 | INFO | Perf report for the last 5.14015 seconds: | 20260507 20:41:42.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:42.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 20:41:42.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:41:42.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:42.585 | INFO | Uploaded 36GiB to GPU in 2.35152s (15.3093GiB/s) | 20260507 20:41:42.585 | INFO | Downloaded 34GiB to GPU in 2.41167s (14.0981GiB/s) | 20260507 20:41:47.723 | INFO | Analyzing chunk frames = [2448384,2448896) | 20260507 20:41:47.723 | INFO | MemInfo: { | 20260507 20:41:47.723 | INFO | "gpu" : " 17 GB", | 20260507 20:41:47.723 | INFO | "huge" : "", | 20260507 20:41:47.723 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:47.723 | INFO | "malloc" : " 14 GB", | 20260507 20:41:47.723 | INFO | "pinned" : " 29 GB" | 20260507 20:41:47.723 | INFO | } | 20260507 20:41:47.724 | INFO | Perf report for the last 5.13882 seconds: | 20260507 20:41:47.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:47.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:41:47.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:41:47.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:47.724 | INFO | Uploaded 36GiB to GPU in 2.48393s (14.4932GiB/s) | 20260507 20:41:47.724 | INFO | Downloaded 34GiB to GPU in 2.46762s (13.7785GiB/s) | 20260507 20:41:52.862 | INFO | Analyzing chunk frames = [2448896,2449408) | 20260507 20:41:52.862 | INFO | MemInfo: { | 20260507 20:41:52.862 | INFO | "gpu" : " 17 GB", | 20260507 20:41:52.862 | INFO | "huge" : "", | 20260507 20:41:52.862 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:52.862 | INFO | "malloc" : " 14 GB", | 20260507 20:41:52.862 | INFO | "pinned" : " 29 GB" | 20260507 20:41:52.862 | INFO | } | 20260507 20:41:52.862 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:41:52.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:52.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:41:52.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 20:41:52.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:52.863 | INFO | Uploaded 36GiB to GPU in 2.49439s (14.4324GiB/s) | 20260507 20:41:52.863 | INFO | Downloaded 34GiB to GPU in 2.4624s (13.8077GiB/s) | 20260507 20:41:52.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:41:52.863 | INFO | Total mem usage: 16.4 GiB | 20260507 20:41:52.863 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:41:52.863 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.15%) | 20260507 20:41:52.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:41:52.863 | INFO | High Water Breakdown: | 20260507 20:41:52.863 | INFO | 60.7% : 9.46 GiB ( 95.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:41:52.863 | INFO | 39.2% : 6.12 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:41:52.863 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:41:52.863 | INFO | ------------------------------------------ | 20260507 20:41:52.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:41:52.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:41:52.863 | INFO | Total mem usage: 26.5 GiB | 20260507 20:41:52.863 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:41:52.863 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 20:41:52.863 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:41:52.863 | INFO | High Water Breakdown: | 20260507 20:41:52.863 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:41:52.863 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:41:52.863 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:41:52.863 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:41:52.863 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:41:52.863 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:41:52.863 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:41:52.863 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:41:52.863 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:41:52.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:41:52.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:41:52.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:41:52.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:41:52.863 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:41:52.863 | INFO | ------------------------------------------ | 20260507 20:41:52.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:41:52.863 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:41:52.863 | INFO | Total mem usage: 27.8 GiB | 20260507 20:41:52.864 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:41:52.864 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:41:52.864 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:41:52.864 | INFO | High Water Breakdown: | 20260507 20:41:52.864 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:41:52.864 | INFO | ------------------------------------------ | 20260507 20:41:52.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:41:52.864 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:41:52.864 | INFO | Total mem usage: 15.7 GiB | 20260507 20:41:52.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:41:52.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:41:52.864 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:41:52.864 | INFO | High Water Breakdown: | 20260507 20:41:52.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:41:52.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:41:52.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:41:52.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:41:52.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:41:52.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:41:52.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:41:52.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:41:52.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:41:52.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:41:52.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:41:52.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:41:52.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:41:52.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:41:52.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:41:52.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:41:52.864 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:41:52.864 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:41:52.864 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:41:52.864 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:41:52.864 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:41:52.864 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:41:52.864 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:41:52.864 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:41:52.864 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:41:52.864 | INFO | ------------------------------------------ | 20260507 20:41:52.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:41:52.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:41:52.864 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:41:52.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:41:52.864 | INFO | PerfReport: Full Execution -- 3.32e+01 / 7.59e+01, 1.41e+01 / 1.77e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.07e-01, 1.24e-02 / 1.31e-02 | 20260507 20:41:52.864 | INFO | PerfReport: 12.10%, Baseline -- 4.01e+00 / 4.46e+00, 5.92e-01 / 2.92e-01 | 20260507 20:41:52.864 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.52e+00, 2.13e-01 / 1.02e-01 | 20260507 20:41:52.864 | INFO | PerfReport: 15.71%, DME -- 4.17e+01 / 4.50e+01, 3.87e+00 / 1.16e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.85e+00 / 1.75e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.48e+00 / 2.78e+00, 4.38e-01 / 2.09e-01 | 20260507 20:41:52.864 | INFO | PerfReport: 3.56%, Metrics -- 1.18e+00 / 1.35e+00, 2.99e-01 / 1.53e-01 | 20260507 20:41:52.864 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.68e-01 / 3.96e-01, 5.94e-02 / 2.97e-02 | 20260507 20:41:52.864 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:41:52.864 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:41:52.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:41:52.864 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 24.34%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 31.13%, Compute -- 3.32e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 1.85%, LockUpload -- 1.97e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 2.43%, LockDownload -- 2.59e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 15.94%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:41:52.864 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260507 20:41:52.974 | INFO | Starting a round of Baz IO | 20260507 20:41:56.463 | INFO | Populating RTMetrics report | 20260507 20:41:56.464 | INFO | RTMetrics report populated | 20260507 20:41:56.464 | INFO | Opening tmp RTMetrics json file | 20260507 20:41:56.480 | INFO | Serializing RTMetrics to json | 20260507 20:41:56.486 | INFO | Serialization done, swapping tmp file now... | 20260507 20:41:56.487 | INFO | RTMetrics done | 20260507 20:41:56.551 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.576 seconds | 20260507 20:41:56.551 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.28% / 0.01% / 55.71% | 20260507 20:41:56.551 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93077% / 1.19209e-05% / 5.16574e-05% | 20260507 20:41:56.551 | INFO | Baz IO overall: | 20260507 20:41:56.551 | INFO | BytesWritten : 1112972560398 | 20260507 20:41:56.551 | INFO | NumEvents : 539526120943 | 20260507 20:41:58.001 | INFO | Analyzing chunk frames = [2449408,2449920) | 20260507 20:41:58.001 | INFO | MemInfo: { | 20260507 20:41:58.001 | INFO | "gpu" : " 17 GB", | 20260507 20:41:58.001 | INFO | "huge" : "", | 20260507 20:41:58.001 | INFO | "huge_shared" : " 29 GB", | 20260507 20:41:58.001 | INFO | "malloc" : " 14 GB", | 20260507 20:41:58.001 | INFO | "pinned" : " 29 GB" | 20260507 20:41:58.001 | INFO | } | 20260507 20:41:58.001 | INFO | Perf report for the last 5.1385 seconds: | 20260507 20:41:58.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:41:58.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:58.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:58.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:58.001 | INFO | ANALYSIS : 69.4%, 2.9, 104.8ms, 0 | 20260507 20:41:58.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:41:58.001 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 20:41:58.001 | INFO | PRE_HQ : 68.5%, 1.8, 64.2ms, 0 | 20260507 20:41:58.001 | INFO | BAZWRITER : 69.2%, 4.0, 143.0ms, 0 | 20260507 20:41:58.001 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 20:41:58.002 | INFO | Uploaded 36GiB to GPU in 2.55387s (14.0962GiB/s) | 20260507 20:41:58.002 | INFO | Downloaded 38GiB to GPU in 2.71619s (13.9902GiB/s) | 20260507 20:42:03.141 | INFO | Analyzing chunk frames = [2449920,2450432) | 20260507 20:42:03.141 | INFO | MemInfo: { | 20260507 20:42:03.141 | INFO | "gpu" : " 17 GB", | 20260507 20:42:03.141 | INFO | "huge" : "", | 20260507 20:42:03.141 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:03.141 | INFO | "malloc" : " 14 GB", | 20260507 20:42:03.141 | INFO | "pinned" : " 29 GB" | 20260507 20:42:03.141 | INFO | } | 20260507 20:42:03.141 | INFO | Perf report for the last 5.13983 seconds: | 20260507 20:42:03.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:03.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 20:42:03.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:42:03.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:03.141 | INFO | Uploaded 36GiB to GPU in 2.53946s (14.1762GiB/s) | 20260507 20:42:03.141 | INFO | Downloaded 34GiB to GPU in 2.49874s (13.6069GiB/s) | 20260507 20:42:08.281 | INFO | Analyzing chunk frames = [2450432,2450944) | 20260507 20:42:08.281 | INFO | MemInfo: { | 20260507 20:42:08.281 | INFO | "gpu" : " 17 GB", | 20260507 20:42:08.281 | INFO | "huge" : "", | 20260507 20:42:08.281 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:08.281 | INFO | "malloc" : " 14 GB", | 20260507 20:42:08.281 | INFO | "pinned" : " 29 GB" | 20260507 20:42:08.281 | INFO | } | 20260507 20:42:08.281 | INFO | Perf report for the last 5.14031 seconds: | 20260507 20:42:08.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:08.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.281 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 20:42:08.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.281 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:42:08.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:08.282 | INFO | Uploaded 36GiB to GPU in 2.56465s (14.037GiB/s) | 20260507 20:42:08.282 | INFO | Downloaded 34GiB to GPU in 2.47601s (13.7318GiB/s) | 20260507 20:42:13.420 | INFO | Analyzing chunk frames = [2450944,2451456) | 20260507 20:42:13.420 | INFO | MemInfo: { | 20260507 20:42:13.420 | INFO | "gpu" : " 17 GB", | 20260507 20:42:13.420 | INFO | "huge" : "", | 20260507 20:42:13.420 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:13.420 | INFO | "malloc" : " 14 GB", | 20260507 20:42:13.421 | INFO | "pinned" : " 29 GB" | 20260507 20:42:13.421 | INFO | } | 20260507 20:42:13.421 | INFO | Perf report for the last 5.1395 seconds: | 20260507 20:42:13.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:13.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 20:42:13.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:42:13.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:13.421 | INFO | Uploaded 36GiB to GPU in 2.50816s (14.3532GiB/s) | 20260507 20:42:13.421 | INFO | Downloaded 34GiB to GPU in 2.49498s (13.6274GiB/s) | 20260507 20:42:18.559 | INFO | Analyzing chunk frames = [2451456,2451968) | 20260507 20:42:18.559 | INFO | MemInfo: { | 20260507 20:42:18.559 | INFO | "gpu" : " 17 GB", | 20260507 20:42:18.559 | INFO | "huge" : "", | 20260507 20:42:18.559 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:18.559 | INFO | "malloc" : " 14 GB", | 20260507 20:42:18.559 | INFO | "pinned" : " 29 GB" | 20260507 20:42:18.559 | INFO | } | 20260507 20:42:18.559 | INFO | Perf report for the last 5.1388 seconds: | 20260507 20:42:18.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:18.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.560 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 20:42:18.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.560 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 20:42:18.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:18.560 | INFO | Uploaded 36GiB to GPU in 2.40679s (14.9577GiB/s) | 20260507 20:42:18.560 | INFO | Downloaded 34GiB to GPU in 2.48367s (13.6894GiB/s) | 20260507 20:42:23.698 | INFO | Analyzing chunk frames = [2451968,2452480) | 20260507 20:42:23.698 | INFO | MemInfo: { | 20260507 20:42:23.698 | INFO | "gpu" : " 17 GB", | 20260507 20:42:23.698 | INFO | "huge" : "", | 20260507 20:42:23.698 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:23.698 | INFO | "malloc" : " 14 GB", | 20260507 20:42:23.699 | INFO | "pinned" : " 29 GB" | 20260507 20:42:23.699 | INFO | } | 20260507 20:42:23.699 | INFO | Perf report for the last 5.13928 seconds: | 20260507 20:42:23.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:23.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 20:42:23.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:42:23.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:23.699 | INFO | Uploaded 36GiB to GPU in 2.38932s (15.067GiB/s) | 20260507 20:42:23.699 | INFO | Downloaded 34GiB to GPU in 2.38795s (14.2382GiB/s) | 20260507 20:42:28.838 | INFO | Analyzing chunk frames = [2452480,2452992) | 20260507 20:42:28.838 | INFO | MemInfo: { | 20260507 20:42:28.838 | INFO | "gpu" : " 17 GB", | 20260507 20:42:28.838 | INFO | "huge" : "", | 20260507 20:42:28.838 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:28.838 | INFO | "malloc" : " 14 GB", | 20260507 20:42:28.838 | INFO | "pinned" : " 29 GB" | 20260507 20:42:28.838 | INFO | } | 20260507 20:42:28.838 | INFO | Perf report for the last 5.13946 seconds: | 20260507 20:42:28.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:28.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:42:28.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:42:28.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:28.838 | INFO | Uploaded 36GiB to GPU in 2.50659s (14.3621GiB/s) | 20260507 20:42:28.838 | INFO | Downloaded 34GiB to GPU in 2.45028s (13.8759GiB/s) | 20260507 20:42:33.976 | INFO | Analyzing chunk frames = [2452992,2453504) | 20260507 20:42:33.976 | INFO | MemInfo: { | 20260507 20:42:33.976 | INFO | "gpu" : " 17 GB", | 20260507 20:42:33.976 | INFO | "huge" : "", | 20260507 20:42:33.976 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:33.976 | INFO | "malloc" : " 14 GB", | 20260507 20:42:33.977 | INFO | "pinned" : " 29 GB" | 20260507 20:42:33.977 | INFO | } | 20260507 20:42:33.977 | INFO | Perf report for the last 5.13848 seconds: | 20260507 20:42:33.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:33.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.977 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:42:33.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.977 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 20:42:33.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:33.978 | INFO | Uploaded 36GiB to GPU in 2.5132s (14.3244GiB/s) | 20260507 20:42:33.978 | INFO | Downloaded 34GiB to GPU in 2.43797s (13.946GiB/s) | 20260507 20:42:34.074 | INFO | Starting a round of Baz IO | 20260507 20:42:37.545 | INFO | Populating RTMetrics report | 20260507 20:42:37.546 | INFO | RTMetrics report populated | 20260507 20:42:37.546 | INFO | Opening tmp RTMetrics json file | 20260507 20:42:37.575 | INFO | Serializing RTMetrics to json | 20260507 20:42:37.617 | INFO | Serialization done, swapping tmp file now... | 20260507 20:42:37.634 | INFO | RTMetrics done | 20260507 20:42:37.656 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.581 seconds | 20260507 20:42:37.656 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.27% / 0.01% / 55.72% | 20260507 20:42:37.656 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.92822% / 1.19209e-05% / 2.38419e-05% | 20260507 20:42:37.656 | INFO | Baz IO overall: | 20260507 20:42:37.656 | INFO | BytesWritten : 1114878469045 | 20260507 20:42:37.656 | INFO | NumEvents : 540449304633 | 20260507 20:42:39.117 | INFO | Analyzing chunk frames = [2453504,2454016) | 20260507 20:42:39.117 | INFO | MemInfo: { | 20260507 20:42:39.117 | INFO | "gpu" : " 17 GB", | 20260507 20:42:39.117 | INFO | "huge" : "", | 20260507 20:42:39.117 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:39.117 | INFO | "malloc" : " 14 GB", | 20260507 20:42:39.117 | INFO | "pinned" : " 29 GB" | 20260507 20:42:39.117 | INFO | } | 20260507 20:42:39.117 | INFO | Perf report for the last 5.14034 seconds: | 20260507 20:42:39.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:39.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:39.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:39.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:39.117 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 20:42:39.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:39.117 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 20:42:39.117 | INFO | PRE_HQ : 66.1%, 1.7, 58.1ms, 0 | 20260507 20:42:39.117 | INFO | BAZWRITER : 69.3%, 3.7, 132.3ms, 0 | 20260507 20:42:39.117 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:42:39.118 | INFO | Uploaded 36GiB to GPU in 2.52549s (14.2547GiB/s) | 20260507 20:42:39.118 | INFO | Downloaded 38GiB to GPU in 2.6962s (14.0939GiB/s) | 20260507 20:42:44.256 | INFO | Analyzing chunk frames = [2454016,2454528) | 20260507 20:42:44.256 | INFO | MemInfo: { | 20260507 20:42:44.256 | INFO | "gpu" : " 17 GB", | 20260507 20:42:44.256 | INFO | "huge" : "", | 20260507 20:42:44.256 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:44.256 | INFO | "malloc" : " 14 GB", | 20260507 20:42:44.256 | INFO | "pinned" : " 29 GB" | 20260507 20:42:44.256 | INFO | } | 20260507 20:42:44.256 | INFO | Perf report for the last 5.13911 seconds: | 20260507 20:42:44.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:44.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:42:44.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 20:42:44.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:44.256 | INFO | Uploaded 36GiB to GPU in 2.49345s (14.4379GiB/s) | 20260507 20:42:44.256 | INFO | Downloaded 34GiB to GPU in 2.51605s (13.5132GiB/s) | 20260507 20:42:49.396 | INFO | Analyzing chunk frames = [2454528,2455040) | 20260507 20:42:49.396 | INFO | MemInfo: { | 20260507 20:42:49.396 | INFO | "gpu" : " 17 GB", | 20260507 20:42:49.396 | INFO | "huge" : "", | 20260507 20:42:49.396 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:49.396 | INFO | "malloc" : " 14 GB", | 20260507 20:42:49.396 | INFO | "pinned" : " 29 GB" | 20260507 20:42:49.396 | INFO | } | 20260507 20:42:49.396 | INFO | Perf report for the last 5.13988 seconds: | 20260507 20:42:49.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:49.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 20:42:49.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:42:49.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:49.396 | INFO | Uploaded 36GiB to GPU in 2.42354s (14.8543GiB/s) | 20260507 20:42:49.397 | INFO | Downloaded 34GiB to GPU in 2.50125s (13.5932GiB/s) | 20260507 20:42:54.535 | INFO | Analyzing chunk frames = [2455040,2455552) | 20260507 20:42:54.535 | INFO | MemInfo: { | 20260507 20:42:54.535 | INFO | "gpu" : " 17 GB", | 20260507 20:42:54.535 | INFO | "huge" : "", | 20260507 20:42:54.536 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:54.536 | INFO | "malloc" : " 14 GB", | 20260507 20:42:54.536 | INFO | "pinned" : " 29 GB" | 20260507 20:42:54.536 | INFO | } | 20260507 20:42:54.536 | INFO | Perf report for the last 5.13953 seconds: | 20260507 20:42:54.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:54.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 20:42:54.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:42:54.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:54.536 | INFO | Uploaded 36GiB to GPU in 2.43197s (14.8028GiB/s) | 20260507 20:42:54.536 | INFO | Downloaded 34GiB to GPU in 2.51497s (13.5191GiB/s) | 20260507 20:42:59.674 | INFO | Analyzing chunk frames = [2455552,2456064) | 20260507 20:42:59.674 | INFO | MemInfo: { | 20260507 20:42:59.674 | INFO | "gpu" : " 17 GB", | 20260507 20:42:59.674 | INFO | "huge" : "", | 20260507 20:42:59.674 | INFO | "huge_shared" : " 29 GB", | 20260507 20:42:59.674 | INFO | "malloc" : " 14 GB", | 20260507 20:42:59.674 | INFO | "pinned" : " 29 GB" | 20260507 20:42:59.674 | INFO | } | 20260507 20:42:59.675 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:42:59.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:42:59.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:42:59.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:42:59.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:42:59.675 | INFO | Uploaded 36GiB to GPU in 2.36421s (15.2271GiB/s) | 20260507 20:42:59.675 | INFO | Downloaded 34GiB to GPU in 2.50503s (13.5727GiB/s) | 20260507 20:43:04.812 | INFO | Analyzing chunk frames = [2456064,2456576) | 20260507 20:43:04.812 | INFO | MemInfo: { | 20260507 20:43:04.812 | INFO | "gpu" : " 17 GB", | 20260507 20:43:04.812 | INFO | "huge" : "", | 20260507 20:43:04.812 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:04.813 | INFO | "malloc" : " 14 GB", | 20260507 20:43:04.813 | INFO | "pinned" : " 29 GB" | 20260507 20:43:04.813 | INFO | } | 20260507 20:43:04.813 | INFO | Perf report for the last 5.13804 seconds: | 20260507 20:43:04.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:04.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 20:43:04.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:43:04.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:04.813 | INFO | Uploaded 36GiB to GPU in 2.39799s (15.0125GiB/s) | 20260507 20:43:04.813 | INFO | Downloaded 34GiB to GPU in 2.39635s (14.1882GiB/s) | 20260507 20:43:09.952 | INFO | Analyzing chunk frames = [2456576,2457088) | 20260507 20:43:09.952 | INFO | MemInfo: { | 20260507 20:43:09.952 | INFO | "gpu" : " 17 GB", | 20260507 20:43:09.952 | INFO | "huge" : "", | 20260507 20:43:09.953 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:09.953 | INFO | "malloc" : " 14 GB", | 20260507 20:43:09.953 | INFO | "pinned" : " 29 GB" | 20260507 20:43:09.953 | INFO | } | 20260507 20:43:09.953 | INFO | Perf report for the last 5.14027 seconds: | 20260507 20:43:09.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:09.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:43:09.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 20:43:09.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:09.953 | INFO | Uploaded 36GiB to GPU in 2.50922s (14.3471GiB/s) | 20260507 20:43:09.953 | INFO | Downloaded 34GiB to GPU in 2.45587s (13.8444GiB/s) | 20260507 20:43:15.092 | INFO | Analyzing chunk frames = [2457088,2457600) | 20260507 20:43:15.092 | INFO | MemInfo: { | 20260507 20:43:15.092 | INFO | "gpu" : " 17 GB", | 20260507 20:43:15.092 | INFO | "huge" : "", | 20260507 20:43:15.092 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:15.092 | INFO | "malloc" : " 14 GB", | 20260507 20:43:15.092 | INFO | "pinned" : " 29 GB" | 20260507 20:43:15.092 | INFO | } | 20260507 20:43:15.092 | INFO | Perf report for the last 5.13947 seconds: | 20260507 20:43:15.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:15.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.092 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:43:15.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.093 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 20:43:15.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:15.093 | INFO | Uploaded 36GiB to GPU in 2.50822s (14.3528GiB/s) | 20260507 20:43:15.093 | INFO | Downloaded 34GiB to GPU in 2.43839s (13.9436GiB/s) | 20260507 20:43:15.093 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:43:15.093 | INFO | Total mem usage: 16.4 GiB | 20260507 20:43:15.093 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:43:15.093 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 20:43:15.093 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:43:15.093 | INFO | High Water Breakdown: | 20260507 20:43:15.093 | INFO | 59.7% : 8.8 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:43:15.093 | INFO | 40.1% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:43:15.093 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:43:15.093 | INFO | ------------------------------------------ | 20260507 20:43:15.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:43:15.093 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:43:15.093 | INFO | Total mem usage: 26.5 GiB | 20260507 20:43:15.093 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:43:15.093 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 20:43:15.093 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:43:15.093 | INFO | High Water Breakdown: | 20260507 20:43:15.093 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:43:15.093 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:43:15.093 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:43:15.093 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:43:15.093 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:43:15.093 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:43:15.093 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:43:15.093 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:43:15.093 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:43:15.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:43:15.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:43:15.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:43:15.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:43:15.093 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:43:15.093 | INFO | ------------------------------------------ | 20260507 20:43:15.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:43:15.093 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:43:15.093 | INFO | Total mem usage: 27.8 GiB | 20260507 20:43:15.094 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:43:15.094 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:43:15.094 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:43:15.094 | INFO | High Water Breakdown: | 20260507 20:43:15.094 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:43:15.094 | INFO | ------------------------------------------ | 20260507 20:43:15.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:43:15.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:43:15.094 | INFO | Total mem usage: 15.7 GiB | 20260507 20:43:15.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:43:15.094 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:43:15.094 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:43:15.094 | INFO | High Water Breakdown: | 20260507 20:43:15.094 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:43:15.094 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:43:15.094 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:43:15.094 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:43:15.094 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:43:15.094 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:43:15.094 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:43:15.094 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:43:15.094 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:43:15.094 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:43:15.094 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:43:15.094 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:43:15.094 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:43:15.094 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:43:15.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:43:15.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:43:15.094 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:43:15.094 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:43:15.094 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:43:15.094 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:43:15.094 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:43:15.094 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:43:15.094 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:43:15.094 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:43:15.094 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:43:15.094 | INFO | ------------------------------------------ | 20260507 20:43:15.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:43:15.094 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:43:15.094 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:43:15.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:43:15.094 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.42e+01 / 1.99e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.02e-01, 1.14e-02 / 4.83e-03 | 20260507 20:43:15.094 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.49e+00, 5.92e-01 / 2.17e-01 | 20260507 20:43:15.094 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.52e+00, 2.09e-01 / 9.95e-02 | 20260507 20:43:15.094 | INFO | PerfReport: 16.01%, DME -- 4.24e+01 / 4.59e+01, 4.20e+00 / 2.09e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 55.90%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.67e+00 / 1.50e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.45e-01 / 2.20e-01 | 20260507 20:43:15.094 | INFO | PerfReport: 3.58%, Metrics -- 1.19e+00 / 1.41e+00, 2.99e-01 / 1.54e-01 | 20260507 20:43:15.094 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.69e-01 / 4.06e-01, 6.08e-02 / 3.03e-02 | 20260507 20:43:15.094 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:43:15.094 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:43:15.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:43:15.094 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 24.17%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 24.37%, Download -- 2.60e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 1.85%, LockUpload -- 1.98e+00 / 0.00e+00, 5.10e+00 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 2.80%, LockDownload -- 2.99e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 20:43:15.094 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260507 20:43:15.207 | INFO | Starting a round of Baz IO | 20260507 20:43:18.693 | INFO | Populating RTMetrics report | 20260507 20:43:18.693 | INFO | RTMetrics report populated | 20260507 20:43:18.693 | INFO | Opening tmp RTMetrics json file | 20260507 20:43:18.714 | INFO | Serializing RTMetrics to json | 20260507 20:43:18.716 | INFO | Serialization done, swapping tmp file now... | 20260507 20:43:18.717 | INFO | RTMetrics done | 20260507 20:43:18.796 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.589 seconds | 20260507 20:43:18.796 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.26% / 0.01% / 55.73% | 20260507 20:43:18.796 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.91557% / 2.38419e-05% / 6.35783e-05% | 20260507 20:43:18.796 | INFO | Baz IO overall: | 20260507 20:43:18.796 | INFO | BytesWritten : 1116782260703 | 20260507 20:43:18.796 | INFO | NumEvents : 541370551344 | 20260507 20:43:20.231 | INFO | Analyzing chunk frames = [2457600,2458112) | 20260507 20:43:20.231 | INFO | MemInfo: { | 20260507 20:43:20.231 | INFO | "gpu" : " 17 GB", | 20260507 20:43:20.231 | INFO | "huge" : "", | 20260507 20:43:20.231 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:20.231 | INFO | "malloc" : " 14 GB", | 20260507 20:43:20.231 | INFO | "pinned" : " 29 GB" | 20260507 20:43:20.231 | INFO | } | 20260507 20:43:20.231 | INFO | Perf report for the last 5.139 seconds: | 20260507 20:43:20.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:20.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:20.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:20.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:20.231 | INFO | ANALYSIS : 69.3%, 2.9, 104.7ms, 0 | 20260507 20:43:20.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:20.231 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260507 20:43:20.231 | INFO | PRE_HQ : 68.5%, 1.8, 64.9ms, 0 | 20260507 20:43:20.232 | INFO | BAZWRITER : 69.5%, 4.2, 150.4ms, 0 | 20260507 20:43:20.232 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 20:43:20.232 | INFO | Uploaded 36GiB to GPU in 2.61424s (13.7707GiB/s) | 20260507 20:43:20.232 | INFO | Downloaded 38GiB to GPU in 2.72219s (13.9593GiB/s) | 20260507 20:43:25.370 | INFO | Analyzing chunk frames = [2458112,2458624) | 20260507 20:43:25.370 | INFO | MemInfo: { | 20260507 20:43:25.370 | INFO | "gpu" : " 17 GB", | 20260507 20:43:25.370 | INFO | "huge" : "", | 20260507 20:43:25.370 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:25.370 | INFO | "malloc" : " 14 GB", | 20260507 20:43:25.370 | INFO | "pinned" : " 29 GB" | 20260507 20:43:25.370 | INFO | } | 20260507 20:43:25.370 | INFO | Perf report for the last 5.13848 seconds: | 20260507 20:43:25.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:25.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:43:25.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:43:25.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:25.370 | INFO | Uploaded 36GiB to GPU in 2.55863s (14.07GiB/s) | 20260507 20:43:25.370 | INFO | Downloaded 34GiB to GPU in 2.48896s (13.6603GiB/s) | 20260507 20:43:30.511 | INFO | Analyzing chunk frames = [2458624,2459136) | 20260507 20:43:30.511 | INFO | MemInfo: { | 20260507 20:43:30.511 | INFO | "gpu" : " 17 GB", | 20260507 20:43:30.511 | INFO | "huge" : "", | 20260507 20:43:30.511 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:30.511 | INFO | "malloc" : " 14 GB", | 20260507 20:43:30.511 | INFO | "pinned" : " 29 GB" | 20260507 20:43:30.511 | INFO | } | 20260507 20:43:30.511 | INFO | Perf report for the last 5.14098 seconds: | 20260507 20:43:30.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:30.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:43:30.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:43:30.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:30.511 | INFO | Uploaded 36GiB to GPU in 2.49534s (14.4269GiB/s) | 20260507 20:43:30.511 | INFO | Downloaded 34GiB to GPU in 2.48832s (13.6638GiB/s) | 20260507 20:43:35.650 | INFO | Analyzing chunk frames = [2459136,2459648) | 20260507 20:43:35.650 | INFO | MemInfo: { | 20260507 20:43:35.650 | INFO | "gpu" : " 17 GB", | 20260507 20:43:35.650 | INFO | "huge" : "", | 20260507 20:43:35.650 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:35.650 | INFO | "malloc" : " 14 GB", | 20260507 20:43:35.650 | INFO | "pinned" : " 29 GB" | 20260507 20:43:35.650 | INFO | } | 20260507 20:43:35.650 | INFO | Perf report for the last 5.13938 seconds: | 20260507 20:43:35.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:35.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 20:43:35.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:43:35.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:35.650 | INFO | Uploaded 36GiB to GPU in 2.48848s (14.4667GiB/s) | 20260507 20:43:35.651 | INFO | Downloaded 34GiB to GPU in 2.49704s (13.6161GiB/s) | 20260507 20:43:40.790 | INFO | Analyzing chunk frames = [2459648,2460160) | 20260507 20:43:40.790 | INFO | MemInfo: { | 20260507 20:43:40.790 | INFO | "gpu" : " 17 GB", | 20260507 20:43:40.790 | INFO | "huge" : "", | 20260507 20:43:40.790 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:40.790 | INFO | "malloc" : " 14 GB", | 20260507 20:43:40.790 | INFO | "pinned" : " 29 GB" | 20260507 20:43:40.790 | INFO | } | 20260507 20:43:40.790 | INFO | Perf report for the last 5.14 seconds: | 20260507 20:43:40.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:40.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.790 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:43:40.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.790 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:43:40.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:40.791 | INFO | Uploaded 36GiB to GPU in 2.37749s (15.142GiB/s) | 20260507 20:43:40.791 | INFO | Downloaded 34GiB to GPU in 2.50146s (13.592GiB/s) | 20260507 20:43:45.928 | INFO | Analyzing chunk frames = [2460160,2460672) | 20260507 20:43:45.928 | INFO | MemInfo: { | 20260507 20:43:45.928 | INFO | "gpu" : " 17 GB", | 20260507 20:43:45.928 | INFO | "huge" : "", | 20260507 20:43:45.928 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:45.928 | INFO | "malloc" : " 14 GB", | 20260507 20:43:45.928 | INFO | "pinned" : " 29 GB" | 20260507 20:43:45.928 | INFO | } | 20260507 20:43:45.928 | INFO | Perf report for the last 5.1376 seconds: | 20260507 20:43:45.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:45.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 20:43:45.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:43:45.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:45.928 | INFO | Uploaded 36GiB to GPU in 2.36685s (15.2101GiB/s) | 20260507 20:43:45.928 | INFO | Downloaded 34GiB to GPU in 2.41403s (14.0844GiB/s) | 20260507 20:43:51.071 | INFO | Analyzing chunk frames = [2460672,2461184) | 20260507 20:43:51.071 | INFO | MemInfo: { | 20260507 20:43:51.071 | INFO | "gpu" : " 17 GB", | 20260507 20:43:51.071 | INFO | "huge" : "", | 20260507 20:43:51.071 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:51.071 | INFO | "malloc" : " 14 GB", | 20260507 20:43:51.071 | INFO | "pinned" : " 29 GB" | 20260507 20:43:51.071 | INFO | } | 20260507 20:43:51.071 | INFO | Perf report for the last 5.14354 seconds: | 20260507 20:43:51.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:51.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.072 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 20:43:51.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.072 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 20:43:51.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:51.072 | INFO | Uploaded 36GiB to GPU in 2.48524s (14.4855GiB/s) | 20260507 20:43:51.072 | INFO | Downloaded 34GiB to GPU in 2.46881s (13.7718GiB/s) | 20260507 20:43:56.206 | INFO | Analyzing chunk frames = [2461184,2461696) | 20260507 20:43:56.206 | INFO | MemInfo: { | 20260507 20:43:56.206 | INFO | "gpu" : " 17 GB", | 20260507 20:43:56.206 | INFO | "huge" : "", | 20260507 20:43:56.207 | INFO | "huge_shared" : " 29 GB", | 20260507 20:43:56.207 | INFO | "malloc" : " 14 GB", | 20260507 20:43:56.207 | INFO | "pinned" : " 29 GB" | 20260507 20:43:56.207 | INFO | } | 20260507 20:43:56.207 | INFO | Perf report for the last 5.13529 seconds: | 20260507 20:43:56.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:43:56.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 20:43:56.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260507 20:43:56.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:43:56.207 | INFO | Uploaded 36GiB to GPU in 2.49018s (14.4568GiB/s) | 20260507 20:43:56.207 | INFO | Downloaded 34GiB to GPU in 2.45641s (13.8414GiB/s) | 20260507 20:43:56.303 | INFO | Starting a round of Baz IO | 20260507 20:43:59.771 | INFO | Populating RTMetrics report | 20260507 20:43:59.772 | INFO | RTMetrics report populated | 20260507 20:43:59.772 | INFO | Opening tmp RTMetrics json file | 20260507 20:43:59.783 | INFO | Serializing RTMetrics to json | 20260507 20:43:59.795 | INFO | Serialization done, swapping tmp file now... | 20260507 20:43:59.796 | INFO | RTMetrics done | 20260507 20:43:59.877 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.573 seconds | 20260507 20:43:59.877 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.25% / 0.01% / 55.74% | 20260507 20:43:59.877 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.91006% / 1.58946e-05% / 4.37101e-05% | 20260507 20:43:59.877 | INFO | Baz IO overall: | 20260507 20:43:59.877 | INFO | BytesWritten : 1118684748586 | 20260507 20:43:59.877 | INFO | NumEvents : 542290819274 | 20260507 20:44:01.348 | INFO | Analyzing chunk frames = [2461696,2462208) | 20260507 20:44:01.348 | INFO | MemInfo: { | 20260507 20:44:01.348 | INFO | "gpu" : " 17 GB", | 20260507 20:44:01.348 | INFO | "huge" : "", | 20260507 20:44:01.348 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:01.348 | INFO | "malloc" : " 14 GB", | 20260507 20:44:01.348 | INFO | "pinned" : " 29 GB" | 20260507 20:44:01.348 | INFO | } | 20260507 20:44:01.348 | INFO | Perf report for the last 5.14169 seconds: | 20260507 20:44:01.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:01.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:01.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:01.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:01.349 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 20:44:01.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:01.349 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:44:01.349 | INFO | PRE_HQ : 66.2%, 1.7, 58.5ms, 0 | 20260507 20:44:01.349 | INFO | BAZWRITER : 69.1%, 4.2, 149.5ms, 0 | 20260507 20:44:01.349 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 20:44:01.349 | INFO | Uploaded 36GiB to GPU in 2.50118s (14.3932GiB/s) | 20260507 20:44:01.349 | INFO | Downloaded 38GiB to GPU in 2.69556s (14.0973GiB/s) | 20260507 20:44:06.485 | INFO | Analyzing chunk frames = [2462208,2462720) | 20260507 20:44:06.485 | INFO | MemInfo: { | 20260507 20:44:06.485 | INFO | "gpu" : " 17 GB", | 20260507 20:44:06.485 | INFO | "huge" : "", | 20260507 20:44:06.485 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:06.485 | INFO | "malloc" : " 14 GB", | 20260507 20:44:06.485 | INFO | "pinned" : " 29 GB" | 20260507 20:44:06.485 | INFO | } | 20260507 20:44:06.485 | INFO | Perf report for the last 5.13688 seconds: | 20260507 20:44:06.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:06.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.485 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:44:06.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.485 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:44:06.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:06.486 | INFO | Uploaded 36GiB to GPU in 2.49764s (14.4136GiB/s) | 20260507 20:44:06.486 | INFO | Downloaded 34GiB to GPU in 2.5136s (13.5264GiB/s) | 20260507 20:44:11.624 | INFO | Analyzing chunk frames = [2462720,2463232) | 20260507 20:44:11.624 | INFO | MemInfo: { | 20260507 20:44:11.624 | INFO | "gpu" : " 17 GB", | 20260507 20:44:11.624 | INFO | "huge" : "", | 20260507 20:44:11.624 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:11.624 | INFO | "malloc" : " 14 GB", | 20260507 20:44:11.624 | INFO | "pinned" : " 29 GB" | 20260507 20:44:11.624 | INFO | } | 20260507 20:44:11.624 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:44:11.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:11.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 20:44:11.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:44:11.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:11.624 | INFO | Uploaded 36GiB to GPU in 2.41344s (14.9165GiB/s) | 20260507 20:44:11.625 | INFO | Downloaded 34GiB to GPU in 2.50541s (13.5706GiB/s) | 20260507 20:44:16.765 | INFO | Analyzing chunk frames = [2463232,2463744) | 20260507 20:44:16.765 | INFO | MemInfo: { | 20260507 20:44:16.765 | INFO | "gpu" : " 17 GB", | 20260507 20:44:16.765 | INFO | "huge" : "", | 20260507 20:44:16.765 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:16.765 | INFO | "malloc" : " 14 GB", | 20260507 20:44:16.765 | INFO | "pinned" : " 29 GB" | 20260507 20:44:16.765 | INFO | } | 20260507 20:44:16.765 | INFO | Perf report for the last 5.14059 seconds: | 20260507 20:44:16.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:16.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 20:44:16.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:44:16.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:16.765 | INFO | Uploaded 36GiB to GPU in 2.42429s (14.8497GiB/s) | 20260507 20:44:16.765 | INFO | Downloaded 34GiB to GPU in 2.50778s (13.5578GiB/s) | 20260507 20:44:21.905 | INFO | Analyzing chunk frames = [2463744,2464256) | 20260507 20:44:21.905 | INFO | MemInfo: { | 20260507 20:44:21.905 | INFO | "gpu" : " 17 GB", | 20260507 20:44:21.905 | INFO | "huge" : "", | 20260507 20:44:21.905 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:21.905 | INFO | "malloc" : " 14 GB", | 20260507 20:44:21.905 | INFO | "pinned" : " 29 GB" | 20260507 20:44:21.905 | INFO | } | 20260507 20:44:21.905 | INFO | Perf report for the last 5.1402 seconds: | 20260507 20:44:21.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:21.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:44:21.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:44:21.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:21.905 | INFO | Uploaded 36GiB to GPU in 2.38427s (15.099GiB/s) | 20260507 20:44:21.905 | INFO | Downloaded 34GiB to GPU in 2.49815s (13.6101GiB/s) | 20260507 20:44:27.044 | INFO | Analyzing chunk frames = [2464256,2464768) | 20260507 20:44:27.044 | INFO | MemInfo: { | 20260507 20:44:27.044 | INFO | "gpu" : " 17 GB", | 20260507 20:44:27.044 | INFO | "huge" : "", | 20260507 20:44:27.044 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:27.044 | INFO | "malloc" : " 14 GB", | 20260507 20:44:27.044 | INFO | "pinned" : " 29 GB" | 20260507 20:44:27.044 | INFO | } | 20260507 20:44:27.044 | INFO | Perf report for the last 5.13905 seconds: | 20260507 20:44:27.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:27.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.044 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:44:27.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.044 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:44:27.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:27.045 | INFO | Uploaded 36GiB to GPU in 2.47195s (14.5634GiB/s) | 20260507 20:44:27.045 | INFO | Downloaded 34GiB to GPU in 2.42679s (14.0103GiB/s) | 20260507 20:44:32.182 | INFO | Analyzing chunk frames = [2464768,2465280) | 20260507 20:44:32.182 | INFO | MemInfo: { | 20260507 20:44:32.182 | INFO | "gpu" : " 17 GB", | 20260507 20:44:32.182 | INFO | "huge" : "", | 20260507 20:44:32.182 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:32.182 | INFO | "malloc" : " 14 GB", | 20260507 20:44:32.182 | INFO | "pinned" : " 29 GB" | 20260507 20:44:32.183 | INFO | } | 20260507 20:44:32.183 | INFO | Perf report for the last 5.13852 seconds: | 20260507 20:44:32.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:32.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:44:32.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:44:32.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:32.183 | INFO | Uploaded 36GiB to GPU in 2.54219s (14.161GiB/s) | 20260507 20:44:32.183 | INFO | Downloaded 34GiB to GPU in 2.46365s (13.8007GiB/s) | 20260507 20:44:37.321 | INFO | Analyzing chunk frames = [2465280,2465792) | 20260507 20:44:37.321 | INFO | MemInfo: { | 20260507 20:44:37.321 | INFO | "gpu" : " 17 GB", | 20260507 20:44:37.321 | INFO | "huge" : "", | 20260507 20:44:37.321 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:37.321 | INFO | "malloc" : " 14 GB", | 20260507 20:44:37.321 | INFO | "pinned" : " 29 GB" | 20260507 20:44:37.321 | INFO | } | 20260507 20:44:37.321 | INFO | Perf report for the last 5.13868 seconds: | 20260507 20:44:37.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:37.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.321 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:44:37.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.321 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 20:44:37.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:37.322 | INFO | Uploaded 36GiB to GPU in 2.49119s (14.4509GiB/s) | 20260507 20:44:37.322 | INFO | Downloaded 34GiB to GPU in 2.43756s (13.9484GiB/s) | 20260507 20:44:37.322 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:44:37.322 | INFO | Total mem usage: 16.4 GiB | 20260507 20:44:37.322 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:44:37.322 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -6.54%) | 20260507 20:44:37.322 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:44:37.322 | INFO | High Water Breakdown: | 20260507 20:44:37.322 | INFO | 60.5% : 9.31 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:44:37.322 | INFO | 39.4% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:44:37.322 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:44:37.322 | INFO | ------------------------------------------ | 20260507 20:44:37.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:44:37.322 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:44:37.322 | INFO | Total mem usage: 26.5 GiB | 20260507 20:44:37.322 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:44:37.322 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 20:44:37.322 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 20:44:37.322 | INFO | High Water Breakdown: | 20260507 20:44:37.322 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:44:37.322 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:44:37.322 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:44:37.322 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:44:37.322 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:44:37.322 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:44:37.322 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:44:37.322 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:44:37.322 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:44:37.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:44:37.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:44:37.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:44:37.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:44:37.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:44:37.322 | INFO | ------------------------------------------ | 20260507 20:44:37.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:44:37.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:44:37.322 | INFO | Total mem usage: 27.8 GiB | 20260507 20:44:37.322 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:44:37.322 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:44:37.322 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:44:37.322 | INFO | High Water Breakdown: | 20260507 20:44:37.322 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:44:37.322 | INFO | ------------------------------------------ | 20260507 20:44:37.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:44:37.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:44:37.322 | INFO | Total mem usage: 15.7 GiB | 20260507 20:44:37.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:44:37.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:44:37.322 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:44:37.322 | INFO | High Water Breakdown: | 20260507 20:44:37.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:44:37.323 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:44:37.323 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:44:37.323 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:44:37.323 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:44:37.323 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:44:37.323 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:44:37.323 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:44:37.323 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:44:37.323 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:44:37.323 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:44:37.323 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:44:37.323 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:44:37.323 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:44:37.323 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:44:37.323 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:44:37.323 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:44:37.323 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:44:37.323 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:44:37.323 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:44:37.323 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:44:37.323 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:44:37.323 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:44:37.323 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:44:37.323 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:44:37.323 | INFO | ------------------------------------------ | 20260507 20:44:37.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:44:37.323 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:44:37.323 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:44:37.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:44:37.323 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.41e+01 / 2.22e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.01e-01, 1.33e-02 / 5.12e-03 | 20260507 20:44:37.323 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.46e+00, 5.91e-01 / 2.70e-01 | 20260507 20:44:37.323 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.06e-01 / 1.14e-01 | 20260507 20:44:37.323 | INFO | PerfReport: 16.12%, DME -- 4.26e+01 / 4.60e+01, 4.22e+00 / 2.25e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 55.77%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.40e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.53e+00 / 2.83e+00, 4.45e-01 / 2.27e-01 | 20260507 20:44:37.323 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.39e+00, 2.94e-01 / 1.70e-01 | 20260507 20:44:37.323 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.67e-01 / 4.06e-01, 5.98e-02 / 3.53e-02 | 20260507 20:44:37.323 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:44:37.323 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:44:37.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:44:37.323 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 24.14%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 24.43%, Download -- 2.61e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 30.91%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 1.88%, LockUpload -- 2.01e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 2.91%, LockDownload -- 3.11e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 15.70%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 20:44:37.323 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 20:44:37.419 | INFO | Starting a round of Baz IO | 20260507 20:44:40.896 | INFO | Populating RTMetrics report | 20260507 20:44:40.897 | INFO | RTMetrics report populated | 20260507 20:44:40.897 | INFO | Opening tmp RTMetrics json file | 20260507 20:44:40.898 | INFO | Serializing RTMetrics to json | 20260507 20:44:40.925 | INFO | Serialization done, swapping tmp file now... | 20260507 20:44:40.929 | INFO | RTMetrics done | 20260507 20:44:41.028 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.609 seconds | 20260507 20:44:41.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.24% / 0.01% / 55.75% | 20260507 20:44:41.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90831% / 0% / 4.76837e-05% | 20260507 20:44:41.028 | INFO | Baz IO overall: | 20260507 20:44:41.028 | INFO | BytesWritten : 1120587374288 | 20260507 20:44:41.028 | INFO | NumEvents : 543211356047 | 20260507 20:44:42.462 | INFO | Analyzing chunk frames = [2465792,2466304) | 20260507 20:44:42.462 | INFO | MemInfo: { | 20260507 20:44:42.462 | INFO | "gpu" : " 17 GB", | 20260507 20:44:42.462 | INFO | "huge" : "", | 20260507 20:44:42.462 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:42.462 | INFO | "malloc" : " 14 GB", | 20260507 20:44:42.462 | INFO | "pinned" : " 29 GB" | 20260507 20:44:42.462 | INFO | } | 20260507 20:44:42.462 | INFO | Perf report for the last 5.14029 seconds: | 20260507 20:44:42.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:42.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:42.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:42.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:42.462 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 20:44:42.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:42.462 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 20:44:42.462 | INFO | PRE_HQ : 66.1%, 1.7, 58.2ms, 0 | 20260507 20:44:42.462 | INFO | BAZWRITER : 69.8%, 3.9, 140.9ms, 0 | 20260507 20:44:42.462 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 20:44:42.462 | INFO | Uploaded 36GiB to GPU in 2.54367s (14.1528GiB/s) | 20260507 20:44:42.462 | INFO | Downloaded 38GiB to GPU in 2.68932s (14.13GiB/s) | 20260507 20:44:47.601 | INFO | Analyzing chunk frames = [2466304,2466816) | 20260507 20:44:47.601 | INFO | MemInfo: { | 20260507 20:44:47.601 | INFO | "gpu" : " 17 GB", | 20260507 20:44:47.601 | INFO | "huge" : "", | 20260507 20:44:47.601 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:47.601 | INFO | "malloc" : " 14 GB", | 20260507 20:44:47.601 | INFO | "pinned" : " 29 GB" | 20260507 20:44:47.601 | INFO | } | 20260507 20:44:47.601 | INFO | Perf report for the last 5.13968 seconds: | 20260507 20:44:47.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:47.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.601 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 20:44:47.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.601 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 20:44:47.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:47.602 | INFO | Uploaded 36GiB to GPU in 2.60104s (13.8406GiB/s) | 20260507 20:44:47.602 | INFO | Downloaded 34GiB to GPU in 2.4781s (13.7202GiB/s) | 20260507 20:44:52.740 | INFO | Analyzing chunk frames = [2466816,2467328) | 20260507 20:44:52.740 | INFO | MemInfo: { | 20260507 20:44:52.740 | INFO | "gpu" : " 17 GB", | 20260507 20:44:52.740 | INFO | "huge" : "", | 20260507 20:44:52.740 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:52.740 | INFO | "malloc" : " 14 GB", | 20260507 20:44:52.740 | INFO | "pinned" : " 29 GB" | 20260507 20:44:52.740 | INFO | } | 20260507 20:44:52.740 | INFO | Perf report for the last 5.13887 seconds: | 20260507 20:44:52.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:52.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:44:52.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:44:52.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:52.740 | INFO | Uploaded 36GiB to GPU in 2.52459s (14.2597GiB/s) | 20260507 20:44:52.741 | INFO | Downloaded 34GiB to GPU in 2.47687s (13.727GiB/s) | 20260507 20:44:57.879 | INFO | Analyzing chunk frames = [2467328,2467840) | 20260507 20:44:57.879 | INFO | MemInfo: { | 20260507 20:44:57.879 | INFO | "gpu" : " 17 GB", | 20260507 20:44:57.879 | INFO | "huge" : "", | 20260507 20:44:57.879 | INFO | "huge_shared" : " 29 GB", | 20260507 20:44:57.879 | INFO | "malloc" : " 14 GB", | 20260507 20:44:57.879 | INFO | "pinned" : " 29 GB" | 20260507 20:44:57.879 | INFO | } | 20260507 20:44:57.879 | INFO | Perf report for the last 5.13882 seconds: | 20260507 20:44:57.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:44:57.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:44:57.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:44:57.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:44:57.879 | INFO | Uploaded 36GiB to GPU in 2.48401s (14.4927GiB/s) | 20260507 20:44:57.879 | INFO | Downloaded 34GiB to GPU in 2.38476s (14.2572GiB/s) | 20260507 20:45:03.022 | INFO | Analyzing chunk frames = [2467840,2468352) | 20260507 20:45:03.022 | INFO | MemInfo: { | 20260507 20:45:03.022 | INFO | "gpu" : " 17 GB", | 20260507 20:45:03.022 | INFO | "huge" : "", | 20260507 20:45:03.022 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:03.022 | INFO | "malloc" : " 14 GB", | 20260507 20:45:03.022 | INFO | "pinned" : " 29 GB" | 20260507 20:45:03.022 | INFO | } | 20260507 20:45:03.022 | INFO | Perf report for the last 5.14266 seconds: | 20260507 20:45:03.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:03.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 20:45:03.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 20:45:03.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:03.022 | INFO | Uploaded 36GiB to GPU in 2.4714s (14.5666GiB/s) | 20260507 20:45:03.022 | INFO | Downloaded 34GiB to GPU in 2.49045s (13.6522GiB/s) | 20260507 20:45:08.157 | INFO | Analyzing chunk frames = [2468352,2468864) | 20260507 20:45:08.157 | INFO | MemInfo: { | 20260507 20:45:08.157 | INFO | "gpu" : " 17 GB", | 20260507 20:45:08.157 | INFO | "huge" : "", | 20260507 20:45:08.157 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:08.157 | INFO | "malloc" : " 14 GB", | 20260507 20:45:08.157 | INFO | "pinned" : " 29 GB" | 20260507 20:45:08.157 | INFO | } | 20260507 20:45:08.157 | INFO | Perf report for the last 5.13558 seconds: | 20260507 20:45:08.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:08.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.157 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 20:45:08.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.157 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:45:08.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:08.158 | INFO | Uploaded 36GiB to GPU in 2.39668s (15.0208GiB/s) | 20260507 20:45:08.158 | INFO | Downloaded 34GiB to GPU in 2.3825s (14.2707GiB/s) | 20260507 20:45:13.297 | INFO | Analyzing chunk frames = [2468864,2469376) | 20260507 20:45:13.297 | INFO | MemInfo: { | 20260507 20:45:13.297 | INFO | "gpu" : " 17 GB", | 20260507 20:45:13.297 | INFO | "huge" : "", | 20260507 20:45:13.297 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:13.297 | INFO | "malloc" : " 14 GB", | 20260507 20:45:13.297 | INFO | "pinned" : " 29 GB" | 20260507 20:45:13.297 | INFO | } | 20260507 20:45:13.297 | INFO | Perf report for the last 5.13983 seconds: | 20260507 20:45:13.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:13.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.297 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 20:45:13.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.297 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:45:13.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:13.298 | INFO | Uploaded 36GiB to GPU in 2.53159s (14.2203GiB/s) | 20260507 20:45:13.298 | INFO | Downloaded 34GiB to GPU in 2.4307s (13.9877GiB/s) | 20260507 20:45:18.436 | INFO | Analyzing chunk frames = [2469376,2469888) | 20260507 20:45:18.436 | INFO | MemInfo: { | 20260507 20:45:18.436 | INFO | "gpu" : " 17 GB", | 20260507 20:45:18.436 | INFO | "huge" : "", | 20260507 20:45:18.436 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:18.436 | INFO | "malloc" : " 14 GB", | 20260507 20:45:18.436 | INFO | "pinned" : " 29 GB" | 20260507 20:45:18.436 | INFO | } | 20260507 20:45:18.436 | INFO | Perf report for the last 5.13895 seconds: | 20260507 20:45:18.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:18.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.436 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:45:18.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.436 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:45:18.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:18.437 | INFO | Uploaded 36GiB to GPU in 2.50438s (14.3748GiB/s) | 20260507 20:45:18.437 | INFO | Downloaded 34GiB to GPU in 2.42628s (14.0132GiB/s) | 20260507 20:45:18.534 | INFO | Starting a round of Baz IO | 20260507 20:45:22.019 | INFO | Populating RTMetrics report | 20260507 20:45:22.019 | INFO | RTMetrics report populated | 20260507 20:45:22.019 | INFO | Opening tmp RTMetrics json file | 20260507 20:45:22.041 | INFO | Serializing RTMetrics to json | 20260507 20:45:22.042 | INFO | Serialization done, swapping tmp file now... | 20260507 20:45:22.043 | INFO | RTMetrics done | 20260507 20:45:22.174 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.64 seconds | 20260507 20:45:22.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.23% / 0.01% / 55.76% | 20260507 20:45:22.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90234% / 7.94729e-06% / 2.78155e-05% | 20260507 20:45:22.174 | INFO | Baz IO overall: | 20260507 20:45:22.174 | INFO | BytesWritten : 1122490803770 | 20260507 20:45:22.174 | INFO | NumEvents : 544132401167 | 20260507 20:45:23.576 | INFO | Analyzing chunk frames = [2469888,2470400) | 20260507 20:45:23.576 | INFO | MemInfo: { | 20260507 20:45:23.576 | INFO | "gpu" : " 17 GB", | 20260507 20:45:23.576 | INFO | "huge" : "", | 20260507 20:45:23.576 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:23.576 | INFO | "malloc" : " 14 GB", | 20260507 20:45:23.576 | INFO | "pinned" : " 29 GB" | 20260507 20:45:23.576 | INFO | } | 20260507 20:45:23.576 | INFO | Perf report for the last 5.14014 seconds: | 20260507 20:45:23.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:23.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:23.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:23.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:23.576 | INFO | ANALYSIS : 68.8%, 3.0, 107.0ms, 0 | 20260507 20:45:23.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:23.576 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 20:45:23.576 | INFO | PRE_HQ : 65.9%, 1.7, 58.0ms, 0 | 20260507 20:45:23.576 | INFO | BAZWRITER : 70.4%, 4.0, 145.7ms, 0 | 20260507 20:45:23.576 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 20:45:23.577 | INFO | Uploaded 36GiB to GPU in 2.5385s (14.1816GiB/s) | 20260507 20:45:23.577 | INFO | Downloaded 38GiB to GPU in 2.68262s (14.1652GiB/s) | 20260507 20:45:28.717 | INFO | Analyzing chunk frames = [2470400,2470912) | 20260507 20:45:28.717 | INFO | MemInfo: { | 20260507 20:45:28.717 | INFO | "gpu" : " 17 GB", | 20260507 20:45:28.717 | INFO | "huge" : "", | 20260507 20:45:28.717 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:28.717 | INFO | "malloc" : " 14 GB", | 20260507 20:45:28.718 | INFO | "pinned" : " 29 GB" | 20260507 20:45:28.718 | INFO | } | 20260507 20:45:28.718 | INFO | Perf report for the last 5.14167 seconds: | 20260507 20:45:28.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:28.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 20:45:28.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:45:28.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:28.718 | INFO | Uploaded 36GiB to GPU in 2.51874s (14.2929GiB/s) | 20260507 20:45:28.719 | INFO | Downloaded 34GiB to GPU in 2.49791s (13.6114GiB/s) | 20260507 20:45:33.855 | INFO | Analyzing chunk frames = [2470912,2471424) | 20260507 20:45:33.855 | INFO | MemInfo: { | 20260507 20:45:33.855 | INFO | "gpu" : " 17 GB", | 20260507 20:45:33.855 | INFO | "huge" : "", | 20260507 20:45:33.855 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:33.855 | INFO | "malloc" : " 14 GB", | 20260507 20:45:33.855 | INFO | "pinned" : " 29 GB" | 20260507 20:45:33.855 | INFO | } | 20260507 20:45:33.855 | INFO | Perf report for the last 5.13748 seconds: | 20260507 20:45:33.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:33.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.856 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:45:33.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.856 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:45:33.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:33.856 | INFO | Uploaded 36GiB to GPU in 2.46634s (14.5965GiB/s) | 20260507 20:45:33.856 | INFO | Downloaded 34GiB to GPU in 2.49043s (13.6523GiB/s) | 20260507 20:45:38.994 | INFO | Analyzing chunk frames = [2471424,2471936) | 20260507 20:45:38.994 | INFO | MemInfo: { | 20260507 20:45:38.994 | INFO | "gpu" : " 17 GB", | 20260507 20:45:38.994 | INFO | "huge" : "", | 20260507 20:45:38.994 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:38.995 | INFO | "malloc" : " 14 GB", | 20260507 20:45:38.995 | INFO | "pinned" : " 29 GB" | 20260507 20:45:38.995 | INFO | } | 20260507 20:45:38.995 | INFO | Perf report for the last 5.13921 seconds: | 20260507 20:45:38.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:38.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.995 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 20:45:38.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.995 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260507 20:45:38.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:38.996 | INFO | Uploaded 36GiB to GPU in 2.46261s (14.6186GiB/s) | 20260507 20:45:38.996 | INFO | Downloaded 34GiB to GPU in 2.49664s (13.6183GiB/s) | 20260507 20:45:44.133 | INFO | Analyzing chunk frames = [2471936,2472448) | 20260507 20:45:44.133 | INFO | MemInfo: { | 20260507 20:45:44.133 | INFO | "gpu" : " 17 GB", | 20260507 20:45:44.133 | INFO | "huge" : "", | 20260507 20:45:44.133 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:44.133 | INFO | "malloc" : " 14 GB", | 20260507 20:45:44.133 | INFO | "pinned" : " 29 GB" | 20260507 20:45:44.133 | INFO | } | 20260507 20:45:44.133 | INFO | Perf report for the last 5.13856 seconds: | 20260507 20:45:44.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:44.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.133 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 20:45:44.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.133 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 20:45:44.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:44.134 | INFO | Uploaded 36GiB to GPU in 2.2956s (15.6822GiB/s) | 20260507 20:45:44.134 | INFO | Downloaded 34GiB to GPU in 2.4781s (13.7202GiB/s) | 20260507 20:45:49.273 | INFO | Analyzing chunk frames = [2472448,2472960) | 20260507 20:45:49.273 | INFO | MemInfo: { | 20260507 20:45:49.273 | INFO | "gpu" : " 17 GB", | 20260507 20:45:49.273 | INFO | "huge" : "", | 20260507 20:45:49.273 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:49.273 | INFO | "malloc" : " 14 GB", | 20260507 20:45:49.273 | INFO | "pinned" : " 29 GB" | 20260507 20:45:49.273 | INFO | } | 20260507 20:45:49.273 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:45:49.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:49.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.273 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 20:45:49.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.273 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:45:49.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:49.274 | INFO | Uploaded 36GiB to GPU in 2.41061s (14.934GiB/s) | 20260507 20:45:49.274 | INFO | Downloaded 34GiB to GPU in 2.39404s (14.2019GiB/s) | 20260507 20:45:54.413 | INFO | Analyzing chunk frames = [2472960,2473472) | 20260507 20:45:54.413 | INFO | MemInfo: { | 20260507 20:45:54.413 | INFO | "gpu" : " 17 GB", | 20260507 20:45:54.413 | INFO | "huge" : "", | 20260507 20:45:54.413 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:54.413 | INFO | "malloc" : " 14 GB", | 20260507 20:45:54.413 | INFO | "pinned" : " 29 GB" | 20260507 20:45:54.413 | INFO | } | 20260507 20:45:54.414 | INFO | Perf report for the last 5.14029 seconds: | 20260507 20:45:54.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:54.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:45:54.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:45:54.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:54.414 | INFO | Uploaded 36GiB to GPU in 2.51974s (14.2872GiB/s) | 20260507 20:45:54.414 | INFO | Downloaded 34GiB to GPU in 2.45745s (13.8355GiB/s) | 20260507 20:45:59.552 | INFO | Analyzing chunk frames = [2473472,2473984) | 20260507 20:45:59.552 | INFO | MemInfo: { | 20260507 20:45:59.552 | INFO | "gpu" : " 17 GB", | 20260507 20:45:59.552 | INFO | "huge" : "", | 20260507 20:45:59.552 | INFO | "huge_shared" : " 29 GB", | 20260507 20:45:59.552 | INFO | "malloc" : " 14 GB", | 20260507 20:45:59.552 | INFO | "pinned" : " 29 GB" | 20260507 20:45:59.552 | INFO | } | 20260507 20:45:59.553 | INFO | Perf report for the last 5.13897 seconds: | 20260507 20:45:59.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:45:59.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:45:59.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:45:59.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:45:59.553 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:45:59.553 | INFO | Total mem usage: 16.4 GiB | 20260507 20:45:59.553 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:45:59.553 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.72%) | 20260507 20:45:59.553 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:45:59.553 | INFO | High Water Breakdown: | 20260507 20:45:59.553 | INFO | 60% : 8.97 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:45:59.553 | INFO | 39.9% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:45:59.553 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:45:59.553 | INFO | ------------------------------------------ | 20260507 20:45:59.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:45:59.553 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:45:59.553 | INFO | Total mem usage: 26.5 GiB | 20260507 20:45:59.553 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:45:59.553 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:45:59.553 | INFO | Uploaded 36GiB to GPU in 2.50037s (14.3979GiB/s) | 20260507 20:45:59.553 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:45:59.553 | INFO | Downloaded 34GiB to GPU in 2.44286s (13.9181GiB/s) | 20260507 20:45:59.553 | INFO | High Water Breakdown: | 20260507 20:45:59.553 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:45:59.553 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:45:59.553 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:45:59.553 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:45:59.553 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:45:59.553 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:45:59.553 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:45:59.554 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:45:59.554 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:45:59.554 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:45:59.554 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:45:59.554 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:45:59.554 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:45:59.554 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:45:59.554 | INFO | ------------------------------------------ | 20260507 20:45:59.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:45:59.554 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:45:59.554 | INFO | Total mem usage: 27.8 GiB | 20260507 20:45:59.554 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:45:59.554 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:45:59.554 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:45:59.554 | INFO | High Water Breakdown: | 20260507 20:45:59.554 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:45:59.554 | INFO | ------------------------------------------ | 20260507 20:45:59.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:45:59.554 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:45:59.554 | INFO | Total mem usage: 15.7 GiB | 20260507 20:45:59.554 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:45:59.554 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:45:59.554 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:45:59.554 | INFO | High Water Breakdown: | 20260507 20:45:59.554 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:45:59.554 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:45:59.554 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:45:59.554 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:45:59.554 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:45:59.554 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:45:59.554 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:45:59.554 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:45:59.554 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:45:59.554 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:45:59.554 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:45:59.554 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:45:59.554 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:45:59.554 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:45:59.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:45:59.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:45:59.554 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:45:59.554 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:45:59.554 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:45:59.554 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:45:59.554 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:45:59.554 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:45:59.554 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:45:59.554 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:45:59.554 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:45:59.554 | INFO | ------------------------------------------ | 20260507 20:45:59.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:45:59.554 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:45:59.554 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:45:59.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:45:59.554 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.41e+01 / 1.91e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.20e-02 / 4.19e-03 | 20260507 20:45:59.554 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.49e+00, 5.97e-01 / 2.16e-01 | 20260507 20:45:59.554 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.08e-01 / 1.21e-01 | 20260507 20:45:59.554 | INFO | PerfReport: 16.02%, DME -- 4.24e+01 / 4.60e+01, 4.24e+00 / 1.94e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 55.91%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.67e+00 / 1.33e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.41e-01 / 2.04e-01 | 20260507 20:45:59.554 | INFO | PerfReport: 3.57%, Metrics -- 1.18e+00 / 1.42e+00, 2.94e-01 / 1.60e-01 | 20260507 20:45:59.554 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.68e-01 / 4.08e-01, 6.00e-02 / 3.30e-02 | 20260507 20:45:59.554 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:45:59.554 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:45:59.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:45:59.554 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 24.21%, Upload -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 24.17%, Download -- 2.58e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 30.93%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 2.08%, LockUpload -- 2.22e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 2.60%, LockDownload -- 2.78e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 15.99%, LockCompute -- 1.71e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 20:45:59.554 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 20:45:59.650 | INFO | Starting a round of Baz IO | 20260507 20:46:03.127 | INFO | Populating RTMetrics report | 20260507 20:46:03.127 | INFO | RTMetrics report populated | 20260507 20:46:03.127 | INFO | Opening tmp RTMetrics json file | 20260507 20:46:03.148 | INFO | Serializing RTMetrics to json | 20260507 20:46:03.150 | INFO | Serialization done, swapping tmp file now... | 20260507 20:46:03.151 | INFO | RTMetrics done | 20260507 20:46:03.228 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.577 seconds | 20260507 20:46:03.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.22% / 0.01% / 55.77% | 20260507 20:46:03.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90693% / 1.58946e-05% / 5.16574e-05% | 20260507 20:46:03.228 | INFO | Baz IO overall: | 20260507 20:46:03.228 | INFO | BytesWritten : 1124396559047 | 20260507 20:46:03.228 | INFO | NumEvents : 545055250413 | 20260507 20:46:04.690 | INFO | Analyzing chunk frames = [2473984,2474496) | 20260507 20:46:04.690 | INFO | MemInfo: { | 20260507 20:46:04.690 | INFO | "gpu" : " 17 GB", | 20260507 20:46:04.690 | INFO | "huge" : "", | 20260507 20:46:04.690 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:04.690 | INFO | "malloc" : " 14 GB", | 20260507 20:46:04.690 | INFO | "pinned" : " 29 GB" | 20260507 20:46:04.690 | INFO | } | 20260507 20:46:04.690 | INFO | Perf report for the last 5.13745 seconds: | 20260507 20:46:04.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:04.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:04.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:04.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:04.690 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 20:46:04.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:04.690 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 20:46:04.690 | INFO | PRE_HQ : 66.3%, 1.7, 58.1ms, 0 | 20260507 20:46:04.690 | INFO | BAZWRITER : 69.3%, 4.2, 151.1ms, 0 | 20260507 20:46:04.690 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 20:46:04.690 | INFO | Uploaded 36GiB to GPU in 2.54372s (14.1525GiB/s) | 20260507 20:46:04.690 | INFO | Downloaded 38GiB to GPU in 2.69957s (14.0763GiB/s) | 20260507 20:46:09.830 | INFO | Analyzing chunk frames = [2474496,2475008) | 20260507 20:46:09.830 | INFO | MemInfo: { | 20260507 20:46:09.830 | INFO | "gpu" : " 17 GB", | 20260507 20:46:09.830 | INFO | "huge" : "", | 20260507 20:46:09.830 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:09.830 | INFO | "malloc" : " 14 GB", | 20260507 20:46:09.830 | INFO | "pinned" : " 29 GB" | 20260507 20:46:09.830 | INFO | } | 20260507 20:46:09.830 | INFO | Perf report for the last 5.14029 seconds: | 20260507 20:46:09.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:09.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.830 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:46:09.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.830 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:46:09.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:09.831 | INFO | Uploaded 36GiB to GPU in 2.56305s (14.0457GiB/s) | 20260507 20:46:09.831 | INFO | Downloaded 34GiB to GPU in 2.4983s (13.6092GiB/s) | 20260507 20:46:14.971 | INFO | Analyzing chunk frames = [2475008,2475520) | 20260507 20:46:14.971 | INFO | MemInfo: { | 20260507 20:46:14.971 | INFO | "gpu" : " 17 GB", | 20260507 20:46:14.971 | INFO | "huge" : "", | 20260507 20:46:14.971 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:14.971 | INFO | "malloc" : " 14 GB", | 20260507 20:46:14.971 | INFO | "pinned" : " 29 GB" | 20260507 20:46:14.971 | INFO | } | 20260507 20:46:14.971 | INFO | Perf report for the last 5.14105 seconds: | 20260507 20:46:14.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:14.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.971 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:46:14.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.972 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:46:14.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:14.972 | INFO | Uploaded 36GiB to GPU in 2.50781s (14.3552GiB/s) | 20260507 20:46:14.972 | INFO | Downloaded 34GiB to GPU in 2.49538s (13.6252GiB/s) | 20260507 20:46:20.109 | INFO | Analyzing chunk frames = [2475520,2476032) | 20260507 20:46:20.109 | INFO | MemInfo: { | 20260507 20:46:20.109 | INFO | "gpu" : " 17 GB", | 20260507 20:46:20.109 | INFO | "huge" : "", | 20260507 20:46:20.109 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:20.109 | INFO | "malloc" : " 14 GB", | 20260507 20:46:20.109 | INFO | "pinned" : " 29 GB" | 20260507 20:46:20.109 | INFO | } | 20260507 20:46:20.110 | INFO | Perf report for the last 5.13815 seconds: | 20260507 20:46:20.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:20.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 20:46:20.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | PRE_HQ : 36.3%, 1.0, 18.8ms, 0 | 20260507 20:46:20.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:20.110 | INFO | Uploaded 36GiB to GPU in 2.50739s (14.3576GiB/s) | 20260507 20:46:20.110 | INFO | Downloaded 34GiB to GPU in 2.47968s (13.7114GiB/s) | 20260507 20:46:25.249 | INFO | Analyzing chunk frames = [2476032,2476544) | 20260507 20:46:25.249 | INFO | MemInfo: { | 20260507 20:46:25.249 | INFO | "gpu" : " 17 GB", | 20260507 20:46:25.249 | INFO | "huge" : "", | 20260507 20:46:25.249 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:25.250 | INFO | "malloc" : " 14 GB", | 20260507 20:46:25.250 | INFO | "pinned" : " 29 GB" | 20260507 20:46:25.250 | INFO | } | 20260507 20:46:25.250 | INFO | Perf report for the last 5.1401 seconds: | 20260507 20:46:25.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:25.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 20:46:25.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 20:46:25.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:25.250 | INFO | Uploaded 36GiB to GPU in 2.36828s (15.2009GiB/s) | 20260507 20:46:25.250 | INFO | Downloaded 34GiB to GPU in 2.47705s (13.726GiB/s) | 20260507 20:46:30.387 | INFO | Analyzing chunk frames = [2476544,2477056) | 20260507 20:46:30.387 | INFO | MemInfo: { | 20260507 20:46:30.387 | INFO | "gpu" : " 17 GB", | 20260507 20:46:30.387 | INFO | "huge" : "", | 20260507 20:46:30.387 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:30.387 | INFO | "malloc" : " 14 GB", | 20260507 20:46:30.387 | INFO | "pinned" : " 29 GB" | 20260507 20:46:30.387 | INFO | } | 20260507 20:46:30.388 | INFO | Perf report for the last 5.13785 seconds: | 20260507 20:46:30.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:30.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 20:46:30.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:46:30.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:30.388 | INFO | Uploaded 36GiB to GPU in 2.37312s (15.1699GiB/s) | 20260507 20:46:30.388 | INFO | Downloaded 34GiB to GPU in 2.40387s (14.1438GiB/s) | 20260507 20:46:35.527 | INFO | Analyzing chunk frames = [2477056,2477568) | 20260507 20:46:35.527 | INFO | MemInfo: { | 20260507 20:46:35.527 | INFO | "gpu" : " 17 GB", | 20260507 20:46:35.527 | INFO | "huge" : "", | 20260507 20:46:35.527 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:35.527 | INFO | "malloc" : " 14 GB", | 20260507 20:46:35.527 | INFO | "pinned" : " 29 GB" | 20260507 20:46:35.527 | INFO | } | 20260507 20:46:35.527 | INFO | Perf report for the last 5.13965 seconds: | 20260507 20:46:35.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:35.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.527 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:46:35.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.527 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:46:35.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:35.528 | INFO | Uploaded 36GiB to GPU in 2.5031s (14.3821GiB/s) | 20260507 20:46:35.528 | INFO | Downloaded 34GiB to GPU in 2.47002s (13.7651GiB/s) | 20260507 20:46:40.667 | INFO | Analyzing chunk frames = [2477568,2478080) | 20260507 20:46:40.667 | INFO | MemInfo: { | 20260507 20:46:40.667 | INFO | "gpu" : " 17 GB", | 20260507 20:46:40.667 | INFO | "huge" : "", | 20260507 20:46:40.667 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:40.670 | INFO | "malloc" : " 14 GB", | 20260507 20:46:40.670 | INFO | "pinned" : " 29 GB" | 20260507 20:46:40.670 | INFO | } | 20260507 20:46:40.670 | INFO | Perf report for the last 5.1432 seconds: | 20260507 20:46:40.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:40.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.670 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 20:46:40.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.671 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 20:46:40.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:40.671 | INFO | Uploaded 36GiB to GPU in 2.47351s (14.5542GiB/s) | 20260507 20:46:40.671 | INFO | Downloaded 34GiB to GPU in 2.44384s (13.9125GiB/s) | 20260507 20:46:40.767 | INFO | Starting a round of Baz IO | 20260507 20:46:44.250 | INFO | Populating RTMetrics report | 20260507 20:46:44.250 | INFO | RTMetrics report populated | 20260507 20:46:44.250 | INFO | Opening tmp RTMetrics json file | 20260507 20:46:44.332 | INFO | Serializing RTMetrics to json | 20260507 20:46:44.348 | INFO | Serialization done, swapping tmp file now... | 20260507 20:46:44.349 | INFO | RTMetrics done | 20260507 20:46:44.376 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.609 seconds | 20260507 20:46:44.376 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.21% / 0.01% / 55.78% | 20260507 20:46:44.376 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90398% / 1.58946e-05% / 5.96046e-05% | 20260507 20:46:44.376 | INFO | Baz IO overall: | 20260507 20:46:44.376 | INFO | BytesWritten : 1126302262535 | 20260507 20:46:44.376 | INFO | NumEvents : 545978231127 | 20260507 20:46:45.806 | INFO | Analyzing chunk frames = [2478080,2478592) | 20260507 20:46:45.806 | INFO | MemInfo: { | 20260507 20:46:45.806 | INFO | "gpu" : " 17 GB", | 20260507 20:46:45.806 | INFO | "huge" : "", | 20260507 20:46:45.806 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:45.809 | INFO | "malloc" : " 14 GB", | 20260507 20:46:45.810 | INFO | "pinned" : " 29 GB" | 20260507 20:46:45.810 | INFO | } | 20260507 20:46:45.810 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:46:45.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:45.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:45.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:45.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:45.810 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 20:46:45.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:45.810 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 20:46:45.810 | INFO | PRE_HQ : 65.8%, 1.7, 58.3ms, 0 | 20260507 20:46:45.810 | INFO | BAZWRITER : 69.8%, 3.8, 135.1ms, 0 | 20260507 20:46:45.810 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 20:46:45.810 | INFO | Uploaded 36GiB to GPU in 2.52426s (14.2616GiB/s) | 20260507 20:46:45.810 | INFO | Downloaded 38GiB to GPU in 2.70041s (14.0719GiB/s) | 20260507 20:46:50.944 | INFO | Analyzing chunk frames = [2478592,2479104) | 20260507 20:46:50.944 | INFO | MemInfo: { | 20260507 20:46:50.944 | INFO | "gpu" : " 17 GB", | 20260507 20:46:50.944 | INFO | "huge" : "", | 20260507 20:46:50.944 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:50.944 | INFO | "malloc" : " 14 GB", | 20260507 20:46:50.944 | INFO | "pinned" : " 29 GB" | 20260507 20:46:50.944 | INFO | } | 20260507 20:46:50.944 | INFO | Perf report for the last 5.13475 seconds: | 20260507 20:46:50.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:50.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.944 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 20:46:50.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.944 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:46:50.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:50.945 | INFO | Uploaded 36GiB to GPU in 2.52197s (14.2746GiB/s) | 20260507 20:46:50.945 | INFO | Downloaded 34GiB to GPU in 2.49847s (13.6083GiB/s) | 20260507 20:46:56.085 | INFO | Analyzing chunk frames = [2479104,2479616) | 20260507 20:46:56.085 | INFO | MemInfo: { | 20260507 20:46:56.085 | INFO | "gpu" : " 17 GB", | 20260507 20:46:56.085 | INFO | "huge" : "", | 20260507 20:46:56.085 | INFO | "huge_shared" : " 29 GB", | 20260507 20:46:56.085 | INFO | "malloc" : " 14 GB", | 20260507 20:46:56.085 | INFO | "pinned" : " 29 GB" | 20260507 20:46:56.085 | INFO | } | 20260507 20:46:56.085 | INFO | Perf report for the last 5.1407 seconds: | 20260507 20:46:56.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:46:56.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:46:56.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:46:56.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:46:56.085 | INFO | Uploaded 36GiB to GPU in 2.5282s (14.2394GiB/s) | 20260507 20:46:56.085 | INFO | Downloaded 34GiB to GPU in 2.48557s (13.679GiB/s) | 20260507 20:47:01.224 | INFO | Analyzing chunk frames = [2479616,2480128) | 20260507 20:47:01.224 | INFO | MemInfo: { | 20260507 20:47:01.224 | INFO | "gpu" : " 17 GB", | 20260507 20:47:01.224 | INFO | "huge" : "", | 20260507 20:47:01.224 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:01.224 | INFO | "malloc" : " 14 GB", | 20260507 20:47:01.224 | INFO | "pinned" : " 29 GB" | 20260507 20:47:01.224 | INFO | } | 20260507 20:47:01.224 | INFO | Perf report for the last 5.13879 seconds: | 20260507 20:47:01.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:01.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 20:47:01.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:47:01.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:01.224 | INFO | Uploaded 36GiB to GPU in 2.47753s (14.5306GiB/s) | 20260507 20:47:01.225 | INFO | Downloaded 34GiB to GPU in 2.51034s (13.544GiB/s) | 20260507 20:47:06.363 | INFO | Analyzing chunk frames = [2480128,2480640) | 20260507 20:47:06.363 | INFO | MemInfo: { | 20260507 20:47:06.363 | INFO | "gpu" : " 17 GB", | 20260507 20:47:06.363 | INFO | "huge" : "", | 20260507 20:47:06.363 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:06.363 | INFO | "malloc" : " 14 GB", | 20260507 20:47:06.363 | INFO | "pinned" : " 29 GB" | 20260507 20:47:06.363 | INFO | } | 20260507 20:47:06.363 | INFO | Perf report for the last 5.13948 seconds: | 20260507 20:47:06.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:06.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.363 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 20:47:06.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.363 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:47:06.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:06.364 | INFO | Uploaded 36GiB to GPU in 2.3123s (15.5689GiB/s) | 20260507 20:47:06.364 | INFO | Downloaded 34GiB to GPU in 2.48118s (13.7032GiB/s) | 20260507 20:47:11.502 | INFO | Analyzing chunk frames = [2480640,2481152) | 20260507 20:47:11.502 | INFO | MemInfo: { | 20260507 20:47:11.502 | INFO | "gpu" : " 17 GB", | 20260507 20:47:11.502 | INFO | "huge" : "", | 20260507 20:47:11.502 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:11.502 | INFO | "malloc" : " 14 GB", | 20260507 20:47:11.502 | INFO | "pinned" : " 29 GB" | 20260507 20:47:11.502 | INFO | } | 20260507 20:47:11.502 | INFO | Perf report for the last 5.13857 seconds: | 20260507 20:47:11.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:11.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.502 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 20:47:11.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.502 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:47:11.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:11.503 | INFO | Uploaded 36GiB to GPU in 2.37735s (15.1429GiB/s) | 20260507 20:47:11.503 | INFO | Downloaded 34GiB to GPU in 2.38931s (14.23GiB/s) | 20260507 20:47:16.642 | INFO | Analyzing chunk frames = [2481152,2481664) | 20260507 20:47:16.642 | INFO | MemInfo: { | 20260507 20:47:16.642 | INFO | "gpu" : " 17 GB", | 20260507 20:47:16.642 | INFO | "huge" : "", | 20260507 20:47:16.642 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:16.642 | INFO | "malloc" : " 14 GB", | 20260507 20:47:16.642 | INFO | "pinned" : " 29 GB" | 20260507 20:47:16.642 | INFO | } | 20260507 20:47:16.642 | INFO | Perf report for the last 5.14023 seconds: | 20260507 20:47:16.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:16.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.642 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:47:16.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.642 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:47:16.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:16.643 | INFO | Uploaded 36GiB to GPU in 2.50458s (14.3737GiB/s) | 20260507 20:47:16.643 | INFO | Downloaded 34GiB to GPU in 2.45605s (13.8434GiB/s) | 20260507 20:47:21.782 | INFO | Analyzing chunk frames = [2481664,2482176) | 20260507 20:47:21.782 | INFO | MemInfo: { | 20260507 20:47:21.782 | INFO | "gpu" : " 17 GB", | 20260507 20:47:21.782 | INFO | "huge" : "", | 20260507 20:47:21.782 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:21.782 | INFO | "malloc" : " 14 GB", | 20260507 20:47:21.786 | INFO | "pinned" : " 29 GB" | 20260507 20:47:21.786 | INFO | } | 20260507 20:47:21.786 | INFO | Perf report for the last 5.14381 seconds: | 20260507 20:47:21.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:21.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.786 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:47:21.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.786 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:47:21.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:21.787 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:47:21.787 | INFO | Total mem usage: 16.4 GiB | 20260507 20:47:21.787 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:47:21.787 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -8.21%) | 20260507 20:47:21.787 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:47:21.787 | INFO | High Water Breakdown: | 20260507 20:47:21.787 | INFO | 60.2% : 9.12 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:47:21.787 | INFO | 39.7% : 6.02 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:47:21.787 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:47:21.787 | INFO | ------------------------------------------ | 20260507 20:47:21.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:47:21.787 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:47:21.787 | INFO | Total mem usage: 26.5 GiB | 20260507 20:47:21.787 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:47:21.787 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:47:21.787 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:47:21.787 | INFO | High Water Breakdown: | 20260507 20:47:21.787 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:47:21.787 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:47:21.787 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:47:21.787 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:47:21.787 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:47:21.787 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:47:21.787 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:47:21.787 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:47:21.787 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:47:21.787 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:47:21.787 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:47:21.787 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:47:21.787 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:47:21.787 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:47:21.787 | INFO | ------------------------------------------ | 20260507 20:47:21.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:47:21.787 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:47:21.787 | INFO | Total mem usage: 27.8 GiB | 20260507 20:47:21.787 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:47:21.787 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:47:21.787 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:47:21.787 | INFO | High Water Breakdown: | 20260507 20:47:21.787 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:47:21.787 | INFO | Uploaded 36GiB to GPU in 2.47457s (14.548GiB/s) | 20260507 20:47:21.787 | INFO | ------------------------------------------ | 20260507 20:47:21.787 | INFO | Downloaded 34GiB to GPU in 2.44051s (13.9315GiB/s) | 20260507 20:47:21.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:47:21.787 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:47:21.787 | INFO | Total mem usage: 15.7 GiB | 20260507 20:47:21.787 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:47:21.787 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:47:21.787 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:47:21.787 | INFO | High Water Breakdown: | 20260507 20:47:21.787 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:47:21.787 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:47:21.787 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:47:21.787 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:47:21.787 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:47:21.787 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:47:21.787 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:47:21.787 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:47:21.787 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:47:21.787 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:47:21.787 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:47:21.787 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:47:21.787 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:47:21.787 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:47:21.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:47:21.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:47:21.787 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:47:21.787 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:47:21.787 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:47:21.787 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:47:21.787 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:47:21.787 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:47:21.787 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:47:21.787 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:47:21.787 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:47:21.787 | INFO | ------------------------------------------ | 20260507 20:47:21.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:47:21.787 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:47:21.787 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:47:21.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:47:21.787 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.43e+01 / 1.76e+00 | 20260507 20:47:21.787 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.19e-02 / 5.93e-03 | 20260507 20:47:21.787 | INFO | PerfReport: 12.14%, Baseline -- 4.02e+00 / 4.47e+00, 6.01e-01 / 2.47e-01 | 20260507 20:47:21.787 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.20e-01 | 20260507 20:47:21.787 | INFO | PerfReport: 16.00%, DME -- 4.24e+01 / 4.60e+01, 4.38e+00 / 1.51e+00 | 20260507 20:47:21.787 | INFO | PerfReport: 55.92%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.68e+00 / 1.66e+00 | 20260507 20:47:21.787 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.44e-01 / 2.31e-01 | 20260507 20:47:21.787 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.39e+00, 3.00e-01 / 1.59e-01 | 20260507 20:47:21.787 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.68e-01 / 4.04e-01, 6.07e-02 / 3.17e-02 | 20260507 20:47:21.787 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:47:21.787 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:47:21.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:47:21.787 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 20:47:21.787 | INFO | PerfReport: 24.08%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260507 20:47:21.788 | INFO | PerfReport: 24.31%, Download -- 2.60e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 20:47:21.788 | INFO | PerfReport: 30.94%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:47:21.788 | INFO | PerfReport: 1.91%, LockUpload -- 2.05e+00 / 0.00e+00, 5.23e+00 / 0.00e+00 | 20260507 20:47:21.788 | INFO | PerfReport: 2.79%, LockDownload -- 2.98e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260507 20:47:21.788 | INFO | PerfReport: 15.95%, LockCompute -- 1.71e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260507 20:47:21.788 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260507 20:47:21.883 | INFO | Starting a round of Baz IO | 20260507 20:47:25.357 | INFO | Populating RTMetrics report | 20260507 20:47:25.357 | INFO | RTMetrics report populated | 20260507 20:47:25.357 | INFO | Opening tmp RTMetrics json file | 20260507 20:47:25.387 | INFO | Serializing RTMetrics to json | 20260507 20:47:25.400 | INFO | Serialization done, swapping tmp file now... | 20260507 20:47:25.424 | INFO | RTMetrics done | 20260507 20:47:25.465 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.581 seconds | 20260507 20:47:25.465 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.21% / 0.01% / 55.79% | 20260507 20:47:25.465 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89834% / 1.19209e-05% / 3.17891e-05% | 20260507 20:47:25.465 | INFO | Baz IO overall: | 20260507 20:47:25.465 | INFO | BytesWritten : 1128206896192 | 20260507 20:47:25.465 | INFO | NumEvents : 546900299578 | 20260507 20:47:26.921 | INFO | Analyzing chunk frames = [2482176,2482688) | 20260507 20:47:26.921 | INFO | MemInfo: { | 20260507 20:47:26.921 | INFO | "gpu" : " 17 GB", | 20260507 20:47:26.921 | INFO | "huge" : "", | 20260507 20:47:26.921 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:26.921 | INFO | "malloc" : " 14 GB", | 20260507 20:47:26.921 | INFO | "pinned" : " 29 GB" | 20260507 20:47:26.921 | INFO | } | 20260507 20:47:26.921 | INFO | Perf report for the last 5.13482 seconds: | 20260507 20:47:26.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:26.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:26.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:26.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:26.921 | INFO | ANALYSIS : 68.8%, 3.0, 107.0ms, 0 | 20260507 20:47:26.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:26.921 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 20:47:26.921 | INFO | PRE_HQ : 66.1%, 1.7, 58.0ms, 0 | 20260507 20:47:26.921 | INFO | BAZWRITER : 69.3%, 4.1, 146.6ms, 0 | 20260507 20:47:26.921 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 20:47:26.921 | INFO | Uploaded 36GiB to GPU in 2.54069s (14.1694GiB/s) | 20260507 20:47:26.921 | INFO | Downloaded 38GiB to GPU in 2.70739s (14.0357GiB/s) | 20260507 20:47:32.060 | INFO | Analyzing chunk frames = [2482688,2483200) | 20260507 20:47:32.060 | INFO | MemInfo: { | 20260507 20:47:32.060 | INFO | "gpu" : " 17 GB", | 20260507 20:47:32.060 | INFO | "huge" : "", | 20260507 20:47:32.060 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:32.060 | INFO | "malloc" : " 14 GB", | 20260507 20:47:32.060 | INFO | "pinned" : " 29 GB" | 20260507 20:47:32.060 | INFO | } | 20260507 20:47:32.060 | INFO | Perf report for the last 5.13954 seconds: | 20260507 20:47:32.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:32.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.060 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 20:47:32.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.061 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 20:47:32.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:32.061 | INFO | Uploaded 36GiB to GPU in 2.52381s (14.2642GiB/s) | 20260507 20:47:32.061 | INFO | Downloaded 34GiB to GPU in 2.46722s (13.7807GiB/s) | 20260507 20:47:37.199 | INFO | Analyzing chunk frames = [2483200,2483712) | 20260507 20:47:37.199 | INFO | MemInfo: { | 20260507 20:47:37.199 | INFO | "gpu" : " 17 GB", | 20260507 20:47:37.199 | INFO | "huge" : "", | 20260507 20:47:37.199 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:37.199 | INFO | "malloc" : " 14 GB", | 20260507 20:47:37.199 | INFO | "pinned" : " 29 GB" | 20260507 20:47:37.199 | INFO | } | 20260507 20:47:37.199 | INFO | Perf report for the last 5.13864 seconds: | 20260507 20:47:37.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:37.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:47:37.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:47:37.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:37.199 | INFO | Uploaded 36GiB to GPU in 2.53104s (14.2234GiB/s) | 20260507 20:47:37.199 | INFO | Downloaded 34GiB to GPU in 2.47855s (13.7177GiB/s) | 20260507 20:47:42.338 | INFO | Analyzing chunk frames = [2483712,2484224) | 20260507 20:47:42.338 | INFO | MemInfo: { | 20260507 20:47:42.338 | INFO | "gpu" : " 17 GB", | 20260507 20:47:42.338 | INFO | "huge" : "", | 20260507 20:47:42.338 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:42.338 | INFO | "malloc" : " 14 GB", | 20260507 20:47:42.338 | INFO | "pinned" : " 29 GB" | 20260507 20:47:42.338 | INFO | } | 20260507 20:47:42.338 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:47:42.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:42.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.338 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 20:47:42.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.338 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:47:42.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:42.339 | INFO | Uploaded 36GiB to GPU in 2.53407s (14.2064GiB/s) | 20260507 20:47:42.339 | INFO | Downloaded 34GiB to GPU in 2.48767s (13.6674GiB/s) | 20260507 20:47:47.478 | INFO | Analyzing chunk frames = [2484224,2484736) | 20260507 20:47:47.478 | INFO | MemInfo: { | 20260507 20:47:47.478 | INFO | "gpu" : " 17 GB", | 20260507 20:47:47.478 | INFO | "huge" : "", | 20260507 20:47:47.478 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:47.478 | INFO | "malloc" : " 14 GB", | 20260507 20:47:47.479 | INFO | "pinned" : " 29 GB" | 20260507 20:47:47.479 | INFO | } | 20260507 20:47:47.479 | INFO | Perf report for the last 5.14032 seconds: | 20260507 20:47:47.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:47.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 20:47:47.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:47:47.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:47.479 | INFO | Uploaded 36GiB to GPU in 2.45888s (14.6408GiB/s) | 20260507 20:47:47.479 | INFO | Downloaded 34GiB to GPU in 2.4996s (13.6022GiB/s) | 20260507 20:47:52.617 | INFO | Analyzing chunk frames = [2484736,2485248) | 20260507 20:47:52.617 | INFO | MemInfo: { | 20260507 20:47:52.617 | INFO | "gpu" : " 17 GB", | 20260507 20:47:52.617 | INFO | "huge" : "", | 20260507 20:47:52.617 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:52.617 | INFO | "malloc" : " 14 GB", | 20260507 20:47:52.617 | INFO | "pinned" : " 29 GB" | 20260507 20:47:52.617 | INFO | } | 20260507 20:47:52.617 | INFO | Perf report for the last 5.13853 seconds: | 20260507 20:47:52.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:52.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.617 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 20:47:52.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.617 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:47:52.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:52.618 | INFO | Uploaded 36GiB to GPU in 2.47908s (14.5215GiB/s) | 20260507 20:47:52.618 | INFO | Downloaded 34GiB to GPU in 2.42124s (14.0424GiB/s) | 20260507 20:47:57.757 | INFO | Analyzing chunk frames = [2485248,2485760) | 20260507 20:47:57.757 | INFO | MemInfo: { | 20260507 20:47:57.757 | INFO | "gpu" : " 17 GB", | 20260507 20:47:57.757 | INFO | "huge" : "", | 20260507 20:47:57.757 | INFO | "huge_shared" : " 29 GB", | 20260507 20:47:57.757 | INFO | "malloc" : " 14 GB", | 20260507 20:47:57.757 | INFO | "pinned" : " 29 GB" | 20260507 20:47:57.757 | INFO | } | 20260507 20:47:57.757 | INFO | Perf report for the last 5.14005 seconds: | 20260507 20:47:57.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:47:57.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.757 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:47:57.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.757 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:47:57.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:47:57.758 | INFO | Uploaded 36GiB to GPU in 2.53809s (14.1839GiB/s) | 20260507 20:47:57.758 | INFO | Downloaded 34GiB to GPU in 2.45742s (13.8357GiB/s) | 20260507 20:48:02.896 | INFO | Analyzing chunk frames = [2485760,2486272) | 20260507 20:48:02.896 | INFO | MemInfo: { | 20260507 20:48:02.896 | INFO | "gpu" : " 17 GB", | 20260507 20:48:02.896 | INFO | "huge" : "", | 20260507 20:48:02.897 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:02.897 | INFO | "malloc" : " 14 GB", | 20260507 20:48:02.897 | INFO | "pinned" : " 29 GB" | 20260507 20:48:02.897 | INFO | } | 20260507 20:48:02.897 | INFO | Perf report for the last 5.13957 seconds: | 20260507 20:48:02.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:02.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:48:02.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:48:02.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:02.897 | INFO | Uploaded 36GiB to GPU in 2.49477s (14.4302GiB/s) | 20260507 20:48:02.897 | INFO | Downloaded 34GiB to GPU in 2.43367s (13.9707GiB/s) | 20260507 20:48:03.001 | INFO | Starting a round of Baz IO | 20260507 20:48:06.493 | INFO | Populating RTMetrics report | 20260507 20:48:06.494 | INFO | RTMetrics report populated | 20260507 20:48:06.494 | INFO | Opening tmp RTMetrics json file | 20260507 20:48:06.495 | INFO | Serializing RTMetrics to json | 20260507 20:48:06.497 | INFO | Serialization done, swapping tmp file now... | 20260507 20:48:06.497 | INFO | RTMetrics done | 20260507 20:48:06.600 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.598 seconds | 20260507 20:48:06.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.20% / 0.01% / 55.79% | 20260507 20:48:06.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89162% / 1.19209e-05% / 8.74201e-05% | 20260507 20:48:06.600 | INFO | Baz IO overall: | 20260507 20:48:06.600 | INFO | BytesWritten : 1130110335944 | 20260507 20:48:06.600 | INFO | NumEvents : 547821453755 | 20260507 20:48:08.036 | INFO | Analyzing chunk frames = [2486272,2486784) | 20260507 20:48:08.036 | INFO | MemInfo: { | 20260507 20:48:08.036 | INFO | "gpu" : " 17 GB", | 20260507 20:48:08.036 | INFO | "huge" : "", | 20260507 20:48:08.036 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:08.036 | INFO | "malloc" : " 14 GB", | 20260507 20:48:08.036 | INFO | "pinned" : " 29 GB" | 20260507 20:48:08.036 | INFO | } | 20260507 20:48:08.036 | INFO | Perf report for the last 5.13893 seconds: | 20260507 20:48:08.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:08.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:08.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:08.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:08.036 | INFO | ANALYSIS : 69.2%, 2.9, 105.2ms, 0 | 20260507 20:48:08.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:08.036 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 20:48:08.036 | INFO | PRE_HQ : 68.4%, 1.8, 63.3ms, 0 | 20260507 20:48:08.036 | INFO | BAZWRITER : 69.6%, 4.3, 152.6ms, 0 | 20260507 20:48:08.036 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 20:48:08.036 | INFO | Uploaded 36GiB to GPU in 2.58922s (13.9038GiB/s) | 20260507 20:48:08.036 | INFO | Downloaded 38GiB to GPU in 2.72532s (13.9433GiB/s) | 20260507 20:48:13.174 | INFO | Analyzing chunk frames = [2486784,2487296) | 20260507 20:48:13.174 | INFO | MemInfo: { | 20260507 20:48:13.174 | INFO | "gpu" : " 17 GB", | 20260507 20:48:13.174 | INFO | "huge" : "", | 20260507 20:48:13.174 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:13.174 | INFO | "malloc" : " 14 GB", | 20260507 20:48:13.175 | INFO | "pinned" : " 29 GB" | 20260507 20:48:13.175 | INFO | } | 20260507 20:48:13.175 | INFO | Perf report for the last 5.13882 seconds: | 20260507 20:48:13.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:13.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:48:13.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:48:13.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:13.175 | INFO | Uploaded 36GiB to GPU in 2.44043s (14.7515GiB/s) | 20260507 20:48:13.175 | INFO | Downloaded 34GiB to GPU in 2.50278s (13.5849GiB/s) | 20260507 20:48:18.314 | INFO | Analyzing chunk frames = [2487296,2487808) | 20260507 20:48:18.314 | INFO | MemInfo: { | 20260507 20:48:18.314 | INFO | "gpu" : " 17 GB", | 20260507 20:48:18.314 | INFO | "huge" : "", | 20260507 20:48:18.314 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:18.314 | INFO | "malloc" : " 14 GB", | 20260507 20:48:18.314 | INFO | "pinned" : " 29 GB" | 20260507 20:48:18.314 | INFO | } | 20260507 20:48:18.314 | INFO | Perf report for the last 5.13916 seconds: | 20260507 20:48:18.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:18.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 20:48:18.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:48:18.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:18.314 | INFO | Uploaded 36GiB to GPU in 2.46497s (14.6046GiB/s) | 20260507 20:48:18.314 | INFO | Downloaded 34GiB to GPU in 2.49291s (13.6387GiB/s) | 20260507 20:48:23.453 | INFO | Analyzing chunk frames = [2487808,2488320) | 20260507 20:48:23.453 | INFO | MemInfo: { | 20260507 20:48:23.453 | INFO | "gpu" : " 17 GB", | 20260507 20:48:23.453 | INFO | "huge" : "", | 20260507 20:48:23.453 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:23.453 | INFO | "malloc" : " 14 GB", | 20260507 20:48:23.453 | INFO | "pinned" : " 29 GB" | 20260507 20:48:23.453 | INFO | } | 20260507 20:48:23.454 | INFO | Perf report for the last 5.13981 seconds: | 20260507 20:48:23.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:23.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 20:48:23.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:48:23.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:23.454 | INFO | Uploaded 36GiB to GPU in 2.47283s (14.5582GiB/s) | 20260507 20:48:23.454 | INFO | Downloaded 34GiB to GPU in 2.50948s (13.5486GiB/s) | 20260507 20:48:28.593 | INFO | Analyzing chunk frames = [2488320,2488832) | 20260507 20:48:28.593 | INFO | MemInfo: { | 20260507 20:48:28.593 | INFO | "gpu" : " 17 GB", | 20260507 20:48:28.593 | INFO | "huge" : "", | 20260507 20:48:28.593 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:28.593 | INFO | "malloc" : " 14 GB", | 20260507 20:48:28.593 | INFO | "pinned" : " 29 GB" | 20260507 20:48:28.593 | INFO | } | 20260507 20:48:28.593 | INFO | Perf report for the last 5.1397 seconds: | 20260507 20:48:28.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:28.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.593 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:48:28.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.593 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 20:48:28.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:28.594 | INFO | Uploaded 36GiB to GPU in 2.35211s (15.3054GiB/s) | 20260507 20:48:28.594 | INFO | Downloaded 34GiB to GPU in 2.49621s (13.6206GiB/s) | 20260507 20:48:33.733 | INFO | Analyzing chunk frames = [2488832,2489344) | 20260507 20:48:33.733 | INFO | MemInfo: { | 20260507 20:48:33.733 | INFO | "gpu" : " 17 GB", | 20260507 20:48:33.733 | INFO | "huge" : "", | 20260507 20:48:33.733 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:33.733 | INFO | "malloc" : " 14 GB", | 20260507 20:48:33.733 | INFO | "pinned" : " 29 GB" | 20260507 20:48:33.733 | INFO | } | 20260507 20:48:33.733 | INFO | Perf report for the last 5.13967 seconds: | 20260507 20:48:33.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:33.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 20:48:33.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 20:48:33.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:33.733 | INFO | Uploaded 36GiB to GPU in 2.47368s (14.5532GiB/s) | 20260507 20:48:33.733 | INFO | Downloaded 34GiB to GPU in 2.42623s (14.0135GiB/s) | 20260507 20:48:38.872 | INFO | Analyzing chunk frames = [2489344,2489856) | 20260507 20:48:38.872 | INFO | MemInfo: { | 20260507 20:48:38.872 | INFO | "gpu" : " 17 GB", | 20260507 20:48:38.872 | INFO | "huge" : "", | 20260507 20:48:38.872 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:38.872 | INFO | "malloc" : " 14 GB", | 20260507 20:48:38.872 | INFO | "pinned" : " 29 GB" | 20260507 20:48:38.872 | INFO | } | 20260507 20:48:38.872 | INFO | Perf report for the last 5.13879 seconds: | 20260507 20:48:38.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:38.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 20:48:38.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 20:48:38.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:38.872 | INFO | Uploaded 36GiB to GPU in 2.52161s (14.2766GiB/s) | 20260507 20:48:38.872 | INFO | Downloaded 34GiB to GPU in 2.45698s (13.8381GiB/s) | 20260507 20:48:44.011 | INFO | Analyzing chunk frames = [2489856,2490368) | 20260507 20:48:44.011 | INFO | MemInfo: { | 20260507 20:48:44.011 | INFO | "gpu" : " 17 GB", | 20260507 20:48:44.011 | INFO | "huge" : "", | 20260507 20:48:44.011 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:44.011 | INFO | "malloc" : " 14 GB", | 20260507 20:48:44.011 | INFO | "pinned" : " 29 GB" | 20260507 20:48:44.011 | INFO | } | 20260507 20:48:44.011 | INFO | Perf report for the last 5.13899 seconds: | 20260507 20:48:44.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:44.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:48:44.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:48:44.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:44.011 | INFO | Uploaded 36GiB to GPU in 2.50316s (14.3818GiB/s) | 20260507 20:48:44.011 | INFO | Downloaded 34GiB to GPU in 2.4427s (13.919GiB/s) | 20260507 20:48:44.011 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:48:44.011 | INFO | Total mem usage: 16.4 GiB | 20260507 20:48:44.011 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:48:44.011 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -10.7%) | 20260507 20:48:44.011 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:48:44.011 | INFO | High Water Breakdown: | 20260507 20:48:44.011 | INFO | 59.8% : 8.86 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:48:44.011 | INFO | 40.1% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:48:44.011 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:48:44.011 | INFO | ------------------------------------------ | 20260507 20:48:44.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:48:44.011 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:48:44.011 | INFO | Total mem usage: 26.5 GiB | 20260507 20:48:44.011 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:48:44.011 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 20:48:44.011 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:48:44.011 | INFO | High Water Breakdown: | 20260507 20:48:44.011 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:48:44.011 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:48:44.011 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:48:44.011 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:48:44.011 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:48:44.011 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:48:44.011 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:48:44.011 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:48:44.011 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:48:44.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:48:44.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:48:44.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:48:44.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:48:44.011 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:48:44.011 | INFO | ------------------------------------------ | 20260507 20:48:44.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:48:44.011 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:48:44.011 | INFO | Total mem usage: 27.8 GiB | 20260507 20:48:44.011 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:48:44.011 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:48:44.011 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:48:44.011 | INFO | High Water Breakdown: | 20260507 20:48:44.011 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:48:44.011 | INFO | ------------------------------------------ | 20260507 20:48:44.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:48:44.011 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:48:44.011 | INFO | Total mem usage: 15.7 GiB | 20260507 20:48:44.012 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:48:44.012 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:48:44.012 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:48:44.012 | INFO | High Water Breakdown: | 20260507 20:48:44.012 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:48:44.012 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:48:44.012 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:48:44.012 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:48:44.012 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:48:44.012 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:48:44.012 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:48:44.012 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:48:44.012 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:48:44.012 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:48:44.012 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:48:44.012 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:48:44.012 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:48:44.012 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:48:44.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:48:44.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:48:44.012 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:48:44.012 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:48:44.012 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:48:44.012 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:48:44.012 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:48:44.012 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:48:44.012 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:48:44.012 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:48:44.012 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:48:44.012 | INFO | ------------------------------------------ | 20260507 20:48:44.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:48:44.012 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:48:44.012 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:48:44.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:48:44.012 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.43e+01 / 2.27e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.18e-02 / 4.87e-03 | 20260507 20:48:44.012 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.49e+00, 5.89e-01 / 2.28e-01 | 20260507 20:48:44.012 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.50e+00, 2.10e-01 / 1.00e-01 | 20260507 20:48:44.012 | INFO | PerfReport: 16.12%, DME -- 4.26e+01 / 4.61e+01, 4.29e+00 / 2.22e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 55.76%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.39e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 2.85e+00, 4.42e-01 / 1.97e-01 | 20260507 20:48:44.012 | INFO | PerfReport: 3.57%, Metrics -- 1.18e+00 / 1.39e+00, 2.92e-01 / 1.48e-01 | 20260507 20:48:44.012 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 3.99e-01, 5.88e-02 / 2.65e-02 | 20260507 20:48:44.012 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:48:44.012 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:48:44.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:48:44.012 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 24.28%, Upload -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 24.34%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 30.85%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 1.99%, LockUpload -- 2.13e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 2.79%, LockDownload -- 2.99e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 15.73%, LockCompute -- 1.68e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 20:48:44.012 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260507 20:48:44.108 | INFO | Starting a round of Baz IO | 20260507 20:48:47.582 | INFO | Populating RTMetrics report | 20260507 20:48:47.583 | INFO | RTMetrics report populated | 20260507 20:48:47.583 | INFO | Opening tmp RTMetrics json file | 20260507 20:48:47.607 | INFO | Serializing RTMetrics to json | 20260507 20:48:47.609 | INFO | Serialization done, swapping tmp file now... | 20260507 20:48:47.610 | INFO | RTMetrics done | 20260507 20:48:47.708 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.599 seconds | 20260507 20:48:47.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.19% / 0.01% / 55.80% | 20260507 20:48:47.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89001% / 3.97364e-06% / 5.16574e-05% | 20260507 20:48:47.708 | INFO | Baz IO overall: | 20260507 20:48:47.708 | INFO | BytesWritten : 1132013343554 | 20260507 20:48:47.708 | INFO | NumEvents : 548742292502 | 20260507 20:48:49.149 | INFO | Analyzing chunk frames = [2490368,2490880) | 20260507 20:48:49.149 | INFO | MemInfo: { | 20260507 20:48:49.149 | INFO | "gpu" : " 17 GB", | 20260507 20:48:49.149 | INFO | "huge" : "", | 20260507 20:48:49.149 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:49.149 | INFO | "malloc" : " 14 GB", | 20260507 20:48:49.149 | INFO | "pinned" : " 29 GB" | 20260507 20:48:49.149 | INFO | } | 20260507 20:48:49.149 | INFO | Perf report for the last 5.13821 seconds: | 20260507 20:48:49.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:49.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:49.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:49.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:49.149 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 20:48:49.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:49.149 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 20:48:49.149 | INFO | PRE_HQ : 66.0%, 1.7, 57.8ms, 0 | 20260507 20:48:49.149 | INFO | BAZWRITER : 69.7%, 4.0, 142.7ms, 0 | 20260507 20:48:49.149 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 20:48:49.149 | INFO | Uploaded 36GiB to GPU in 2.54233s (14.1602GiB/s) | 20260507 20:48:49.149 | INFO | Downloaded 38GiB to GPU in 2.70211s (14.0631GiB/s) | 20260507 20:48:54.288 | INFO | Analyzing chunk frames = [2490880,2491392) | 20260507 20:48:54.288 | INFO | MemInfo: { | 20260507 20:48:54.289 | INFO | "gpu" : " 17 GB", | 20260507 20:48:54.289 | INFO | "huge" : "", | 20260507 20:48:54.289 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:54.289 | INFO | "malloc" : " 14 GB", | 20260507 20:48:54.289 | INFO | "pinned" : " 29 GB" | 20260507 20:48:54.289 | INFO | } | 20260507 20:48:54.289 | INFO | Perf report for the last 5.13976 seconds: | 20260507 20:48:54.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:54.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 20:48:54.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 20:48:54.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:54.289 | INFO | Uploaded 36GiB to GPU in 2.56243s (14.0492GiB/s) | 20260507 20:48:54.289 | INFO | Downloaded 34GiB to GPU in 2.48449s (13.6849GiB/s) | 20260507 20:48:59.428 | INFO | Analyzing chunk frames = [2491392,2491904) | 20260507 20:48:59.428 | INFO | MemInfo: { | 20260507 20:48:59.428 | INFO | "gpu" : " 17 GB", | 20260507 20:48:59.428 | INFO | "huge" : "", | 20260507 20:48:59.428 | INFO | "huge_shared" : " 29 GB", | 20260507 20:48:59.428 | INFO | "malloc" : " 14 GB", | 20260507 20:48:59.428 | INFO | "pinned" : " 29 GB" | 20260507 20:48:59.428 | INFO | } | 20260507 20:48:59.428 | INFO | Perf report for the last 5.13969 seconds: | 20260507 20:48:59.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:48:59.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.429 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:48:59.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.429 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:48:59.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:48:59.429 | INFO | Uploaded 36GiB to GPU in 2.51896s (14.2916GiB/s) | 20260507 20:48:59.429 | INFO | Downloaded 34GiB to GPU in 2.49205s (13.6434GiB/s) | 20260507 20:49:04.568 | INFO | Analyzing chunk frames = [2491904,2492416) | 20260507 20:49:04.568 | INFO | MemInfo: { | 20260507 20:49:04.568 | INFO | "gpu" : " 17 GB", | 20260507 20:49:04.568 | INFO | "huge" : "", | 20260507 20:49:04.568 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:04.568 | INFO | "malloc" : " 14 GB", | 20260507 20:49:04.568 | INFO | "pinned" : " 29 GB" | 20260507 20:49:04.568 | INFO | } | 20260507 20:49:04.568 | INFO | Perf report for the last 5.13972 seconds: | 20260507 20:49:04.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:04.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 20:49:04.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:49:04.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:04.568 | INFO | Uploaded 36GiB to GPU in 2.49694s (14.4176GiB/s) | 20260507 20:49:04.568 | INFO | Downloaded 34GiB to GPU in 2.4969s (13.6169GiB/s) | 20260507 20:49:09.709 | INFO | Analyzing chunk frames = [2492416,2492928) | 20260507 20:49:09.709 | INFO | MemInfo: { | 20260507 20:49:09.709 | INFO | "gpu" : " 17 GB", | 20260507 20:49:09.709 | INFO | "huge" : "", | 20260507 20:49:09.709 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:09.709 | INFO | "malloc" : " 14 GB", | 20260507 20:49:09.709 | INFO | "pinned" : " 29 GB" | 20260507 20:49:09.709 | INFO | } | 20260507 20:49:09.709 | INFO | Perf report for the last 5.14075 seconds: | 20260507 20:49:09.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:09.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 20:49:09.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:49:09.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:09.709 | INFO | Uploaded 36GiB to GPU in 2.4382s (14.765GiB/s) | 20260507 20:49:09.709 | INFO | Downloaded 34GiB to GPU in 2.51075s (13.5418GiB/s) | 20260507 20:49:14.847 | INFO | Analyzing chunk frames = [2492928,2493440) | 20260507 20:49:14.847 | INFO | MemInfo: { | 20260507 20:49:14.848 | INFO | "gpu" : " 17 GB", | 20260507 20:49:14.848 | INFO | "huge" : "", | 20260507 20:49:14.848 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:14.848 | INFO | "malloc" : " 14 GB", | 20260507 20:49:14.848 | INFO | "pinned" : " 29 GB" | 20260507 20:49:14.848 | INFO | } | 20260507 20:49:14.848 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:49:14.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:14.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 20:49:14.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 20:49:14.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:14.848 | INFO | Uploaded 36GiB to GPU in 2.36726s (15.2074GiB/s) | 20260507 20:49:14.848 | INFO | Downloaded 34GiB to GPU in 2.41114s (14.1012GiB/s) | 20260507 20:49:19.986 | INFO | Analyzing chunk frames = [2493440,2493952) | 20260507 20:49:19.986 | INFO | MemInfo: { | 20260507 20:49:19.986 | INFO | "gpu" : " 17 GB", | 20260507 20:49:19.986 | INFO | "huge" : "", | 20260507 20:49:19.986 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:19.986 | INFO | "malloc" : " 14 GB", | 20260507 20:49:19.986 | INFO | "pinned" : " 29 GB" | 20260507 20:49:19.986 | INFO | } | 20260507 20:49:19.986 | INFO | Perf report for the last 5.13857 seconds: | 20260507 20:49:19.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:19.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.986 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 20:49:19.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.987 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 20:49:19.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:19.987 | INFO | Uploaded 36GiB to GPU in 2.52893s (14.2353GiB/s) | 20260507 20:49:19.987 | INFO | Downloaded 34GiB to GPU in 2.47366s (13.7448GiB/s) | 20260507 20:49:25.125 | INFO | Analyzing chunk frames = [2493952,2494464) | 20260507 20:49:25.125 | INFO | MemInfo: { | 20260507 20:49:25.125 | INFO | "gpu" : " 17 GB", | 20260507 20:49:25.125 | INFO | "huge" : "", | 20260507 20:49:25.125 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:25.125 | INFO | "malloc" : " 14 GB", | 20260507 20:49:25.125 | INFO | "pinned" : " 29 GB" | 20260507 20:49:25.125 | INFO | } | 20260507 20:49:25.125 | INFO | Perf report for the last 5.13894 seconds: | 20260507 20:49:25.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:25.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.125 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:49:25.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.126 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 20:49:25.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:25.126 | INFO | Uploaded 36GiB to GPU in 2.49253s (14.4431GiB/s) | 20260507 20:49:25.126 | INFO | Downloaded 34GiB to GPU in 2.4605s (13.8184GiB/s) | 20260507 20:49:25.255 | INFO | Starting a round of Baz IO | 20260507 20:49:28.732 | INFO | Populating RTMetrics report | 20260507 20:49:28.732 | INFO | RTMetrics report populated | 20260507 20:49:28.732 | INFO | Opening tmp RTMetrics json file | 20260507 20:49:28.738 | INFO | Serializing RTMetrics to json | 20260507 20:49:28.768 | INFO | Serialization done, swapping tmp file now... | 20260507 20:49:28.772 | INFO | RTMetrics done | 20260507 20:49:28.933 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.677 seconds | 20260507 20:49:28.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.18% / 0.01% / 55.81% | 20260507 20:49:28.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88876% / 7.94729e-06% / 7.15256e-05% | 20260507 20:49:28.933 | INFO | Baz IO overall: | 20260507 20:49:28.933 | INFO | BytesWritten : 1133918221373 | 20260507 20:49:28.933 | INFO | NumEvents : 549664554154 | 20260507 20:49:30.264 | INFO | Analyzing chunk frames = [2494464,2494976) | 20260507 20:49:30.264 | INFO | MemInfo: { | 20260507 20:49:30.264 | INFO | "gpu" : " 17 GB", | 20260507 20:49:30.264 | INFO | "huge" : "", | 20260507 20:49:30.264 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:30.264 | INFO | "malloc" : " 14 GB", | 20260507 20:49:30.264 | INFO | "pinned" : " 29 GB" | 20260507 20:49:30.264 | INFO | } | 20260507 20:49:30.264 | INFO | Perf report for the last 5.13851 seconds: | 20260507 20:49:30.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:30.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:30.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:30.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:30.264 | INFO | ANALYSIS : 69.5%, 2.9, 104.9ms, 0 | 20260507 20:49:30.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:30.264 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 20:49:30.264 | INFO | PRE_HQ : 68.6%, 1.8, 64.6ms, 0 | 20260507 20:49:30.264 | INFO | BAZWRITER : 71.6%, 4.3, 156.9ms, 0 | 20260507 20:49:30.264 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 20:49:30.264 | INFO | Uploaded 36GiB to GPU in 2.57188s (13.9976GiB/s) | 20260507 20:49:30.264 | INFO | Downloaded 38GiB to GPU in 2.7166s (13.9881GiB/s) | 20260507 20:49:35.404 | INFO | Analyzing chunk frames = [2494976,2495488) | 20260507 20:49:35.404 | INFO | MemInfo: { | 20260507 20:49:35.404 | INFO | "gpu" : " 17 GB", | 20260507 20:49:35.404 | INFO | "huge" : "", | 20260507 20:49:35.404 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:35.404 | INFO | "malloc" : " 14 GB", | 20260507 20:49:35.404 | INFO | "pinned" : " 29 GB" | 20260507 20:49:35.404 | INFO | } | 20260507 20:49:35.404 | INFO | Perf report for the last 5.14009 seconds: | 20260507 20:49:35.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:35.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:49:35.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:49:35.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:35.404 | INFO | Uploaded 36GiB to GPU in 2.54428s (14.1494GiB/s) | 20260507 20:49:35.404 | INFO | Downloaded 34GiB to GPU in 2.49657s (13.6187GiB/s) | 20260507 20:49:40.543 | INFO | Analyzing chunk frames = [2495488,2496000) | 20260507 20:49:40.543 | INFO | MemInfo: { | 20260507 20:49:40.543 | INFO | "gpu" : " 17 GB", | 20260507 20:49:40.543 | INFO | "huge" : "", | 20260507 20:49:40.543 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:40.544 | INFO | "malloc" : " 14 GB", | 20260507 20:49:40.544 | INFO | "pinned" : " 29 GB" | 20260507 20:49:40.544 | INFO | } | 20260507 20:49:40.544 | INFO | Perf report for the last 5.13966 seconds: | 20260507 20:49:40.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:40.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 20:49:40.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:49:40.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:40.544 | INFO | Uploaded 36GiB to GPU in 2.55652s (14.0816GiB/s) | 20260507 20:49:40.544 | INFO | Downloaded 34GiB to GPU in 2.4698s (13.7663GiB/s) | 20260507 20:49:45.683 | INFO | Analyzing chunk frames = [2496000,2496512) | 20260507 20:49:45.683 | INFO | MemInfo: { | 20260507 20:49:45.683 | INFO | "gpu" : " 17 GB", | 20260507 20:49:45.683 | INFO | "huge" : "", | 20260507 20:49:45.683 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:45.683 | INFO | "malloc" : " 14 GB", | 20260507 20:49:45.683 | INFO | "pinned" : " 29 GB" | 20260507 20:49:45.683 | INFO | } | 20260507 20:49:45.683 | INFO | Perf report for the last 5.13939 seconds: | 20260507 20:49:45.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:45.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 20:49:45.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:49:45.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:45.683 | INFO | Uploaded 36GiB to GPU in 2.48881s (14.4648GiB/s) | 20260507 20:49:45.683 | INFO | Downloaded 34GiB to GPU in 2.40523s (14.1359GiB/s) | 20260507 20:49:50.822 | INFO | Analyzing chunk frames = [2496512,2497024) | 20260507 20:49:50.822 | INFO | MemInfo: { | 20260507 20:49:50.822 | INFO | "gpu" : " 17 GB", | 20260507 20:49:50.822 | INFO | "huge" : "", | 20260507 20:49:50.822 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:50.822 | INFO | "malloc" : " 14 GB", | 20260507 20:49:50.822 | INFO | "pinned" : " 29 GB" | 20260507 20:49:50.822 | INFO | } | 20260507 20:49:50.823 | INFO | Perf report for the last 5.13947 seconds: | 20260507 20:49:50.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:50.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 20:49:50.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 20:49:50.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:50.823 | INFO | Uploaded 36GiB to GPU in 2.3988s (15.0075GiB/s) | 20260507 20:49:50.823 | INFO | Downloaded 34GiB to GPU in 2.48715s (13.6703GiB/s) | 20260507 20:49:55.961 | INFO | Analyzing chunk frames = [2497024,2497536) | 20260507 20:49:55.961 | INFO | MemInfo: { | 20260507 20:49:55.961 | INFO | "gpu" : " 17 GB", | 20260507 20:49:55.961 | INFO | "huge" : "", | 20260507 20:49:55.961 | INFO | "huge_shared" : " 29 GB", | 20260507 20:49:55.961 | INFO | "malloc" : " 14 GB", | 20260507 20:49:55.961 | INFO | "pinned" : " 29 GB" | 20260507 20:49:55.961 | INFO | } | 20260507 20:49:55.961 | INFO | Perf report for the last 5.13878 seconds: | 20260507 20:49:55.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:49:55.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.961 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 20:49:55.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.961 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:49:55.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:49:55.962 | INFO | Uploaded 36GiB to GPU in 2.40243s (14.9848GiB/s) | 20260507 20:49:55.962 | INFO | Downloaded 34GiB to GPU in 2.39481s (14.1974GiB/s) | 20260507 20:50:01.102 | INFO | Analyzing chunk frames = [2497536,2498048) | 20260507 20:50:01.102 | INFO | MemInfo: { | 20260507 20:50:01.102 | INFO | "gpu" : " 17 GB", | 20260507 20:50:01.102 | INFO | "huge" : "", | 20260507 20:50:01.102 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:01.102 | INFO | "malloc" : " 14 GB", | 20260507 20:50:01.102 | INFO | "pinned" : " 29 GB" | 20260507 20:50:01.102 | INFO | } | 20260507 20:50:01.102 | INFO | Perf report for the last 5.14082 seconds: | 20260507 20:50:01.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:01.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.102 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:50:01.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.102 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 20:50:01.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:01.103 | INFO | Uploaded 36GiB to GPU in 2.52349s (14.266GiB/s) | 20260507 20:50:01.103 | INFO | Downloaded 34GiB to GPU in 2.45067s (13.8738GiB/s) | 20260507 20:50:06.240 | INFO | Analyzing chunk frames = [2498048,2498560) | 20260507 20:50:06.240 | INFO | MemInfo: { | 20260507 20:50:06.240 | INFO | "gpu" : " 17 GB", | 20260507 20:50:06.240 | INFO | "huge" : "", | 20260507 20:50:06.240 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:06.240 | INFO | "malloc" : " 14 GB", | 20260507 20:50:06.240 | INFO | "pinned" : " 29 GB" | 20260507 20:50:06.240 | INFO | } | 20260507 20:50:06.240 | INFO | Perf report for the last 5.13799 seconds: | 20260507 20:50:06.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:06.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:50:06.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 20:50:06.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:06.240 | INFO | Uploaded 36GiB to GPU in 2.5263s (14.2501GiB/s) | 20260507 20:50:06.240 | INFO | Downloaded 34GiB to GPU in 2.4385s (13.943GiB/s) | 20260507 20:50:06.241 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:50:06.241 | INFO | Total mem usage: 16.4 GiB | 20260507 20:50:06.241 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:50:06.241 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -10.4%) | 20260507 20:50:06.241 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:50:06.241 | INFO | High Water Breakdown: | 20260507 20:50:06.241 | INFO | 59.8% : 8.89 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:50:06.241 | INFO | 40% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:50:06.241 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:50:06.241 | INFO | ------------------------------------------ | 20260507 20:50:06.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:50:06.241 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:50:06.241 | INFO | Total mem usage: 26.5 GiB | 20260507 20:50:06.241 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:50:06.241 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:50:06.241 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:50:06.241 | INFO | High Water Breakdown: | 20260507 20:50:06.241 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:50:06.241 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:50:06.241 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:50:06.241 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:50:06.241 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:50:06.241 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:50:06.241 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:50:06.241 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:50:06.241 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:50:06.241 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:50:06.241 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:50:06.241 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:50:06.241 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:50:06.241 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:50:06.241 | INFO | ------------------------------------------ | 20260507 20:50:06.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:50:06.241 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:50:06.241 | INFO | Total mem usage: 27.8 GiB | 20260507 20:50:06.241 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:50:06.241 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:50:06.241 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:50:06.241 | INFO | High Water Breakdown: | 20260507 20:50:06.241 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:50:06.241 | INFO | ------------------------------------------ | 20260507 20:50:06.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:50:06.241 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:50:06.241 | INFO | Total mem usage: 15.7 GiB | 20260507 20:50:06.241 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:50:06.241 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:50:06.241 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:50:06.241 | INFO | High Water Breakdown: | 20260507 20:50:06.241 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:50:06.241 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:50:06.241 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:50:06.241 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:50:06.241 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:50:06.241 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:50:06.241 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:50:06.241 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:50:06.241 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:50:06.241 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:50:06.241 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:50:06.241 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:50:06.241 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:50:06.241 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:50:06.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:50:06.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:50:06.241 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:50:06.241 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:50:06.241 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:50:06.241 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:50:06.241 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:50:06.241 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:50:06.242 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:50:06.242 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:50:06.242 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:50:06.242 | INFO | ------------------------------------------ | 20260507 20:50:06.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:50:06.242 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:50:06.242 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:50:06.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:50:06.242 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.59e+01, 1.38e+01 / 1.76e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 0.30%, AggregateStats -- 9.96e-02 / 1.04e-01, 1.27e-02 / 6.30e-03 | 20260507 20:50:06.242 | INFO | PerfReport: 12.11%, Baseline -- 4.01e+00 / 4.53e+00, 5.94e-01 / 1.85e-01 | 20260507 20:50:06.242 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.52e+00, 2.07e-01 / 9.79e-02 | 20260507 20:50:06.242 | INFO | PerfReport: 15.90%, DME -- 4.21e+01 / 4.55e+01, 4.06e+00 / 1.61e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.75e+00 / 1.76e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.50e+00 / 2.85e+00, 4.39e-01 / 2.03e-01 | 20260507 20:50:06.242 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.39e+00, 2.95e-01 / 1.68e-01 | 20260507 20:50:06.242 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.69e-01 / 4.04e-01, 6.08e-02 / 3.85e-02 | 20260507 20:50:06.242 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:50:06.242 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:50:06.242 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:50:06.242 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 24.35%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 24.31%, Download -- 2.60e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 30.97%, Compute -- 3.31e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 1.94%, LockUpload -- 2.08e+00 / 0.00e+00, 5.14e+00 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 2.57%, LockDownload -- 2.74e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 15.85%, LockCompute -- 1.69e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 20:50:06.242 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 20:50:06.339 | INFO | Starting a round of Baz IO | 20260507 20:50:09.806 | INFO | Populating RTMetrics report | 20260507 20:50:09.806 | INFO | RTMetrics report populated | 20260507 20:50:09.806 | INFO | Opening tmp RTMetrics json file | 20260507 20:50:09.849 | INFO | Serializing RTMetrics to json | 20260507 20:50:09.901 | INFO | Serialization done, swapping tmp file now... | 20260507 20:50:09.908 | INFO | RTMetrics done | 20260507 20:50:09.931 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.591 seconds | 20260507 20:50:09.931 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.17% / 0.01% / 55.82% | 20260507 20:50:09.931 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88535% / 3.97364e-06% / 5.5631e-05% | 20260507 20:50:09.931 | INFO | Baz IO overall: | 20260507 20:50:09.931 | INFO | BytesWritten : 1135821944361 | 20260507 20:50:09.931 | INFO | NumEvents : 550586209231 | 20260507 20:50:11.380 | INFO | Analyzing chunk frames = [2498560,2499072) | 20260507 20:50:11.380 | INFO | MemInfo: { | 20260507 20:50:11.380 | INFO | "gpu" : " 17 GB", | 20260507 20:50:11.380 | INFO | "huge" : "", | 20260507 20:50:11.380 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:11.380 | INFO | "malloc" : " 14 GB", | 20260507 20:50:11.380 | INFO | "pinned" : " 29 GB" | 20260507 20:50:11.380 | INFO | } | 20260507 20:50:11.380 | INFO | Perf report for the last 5.13949 seconds: | 20260507 20:50:11.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:11.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:11.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:11.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:11.380 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 20:50:11.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:11.380 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 20:50:11.380 | INFO | PRE_HQ : 66.0%, 1.7, 57.9ms, 0 | 20260507 20:50:11.380 | INFO | BAZWRITER : 69.5%, 3.9, 138.9ms, 0 | 20260507 20:50:11.380 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 20:50:11.380 | INFO | Uploaded 36GiB to GPU in 2.5257s (14.2535GiB/s) | 20260507 20:50:11.380 | INFO | Downloaded 38GiB to GPU in 2.69704s (14.0895GiB/s) | 20260507 20:50:16.520 | INFO | Analyzing chunk frames = [2499072,2499584) | 20260507 20:50:16.520 | INFO | MemInfo: { | 20260507 20:50:16.520 | INFO | "gpu" : " 17 GB", | 20260507 20:50:16.520 | INFO | "huge" : "", | 20260507 20:50:16.520 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:16.520 | INFO | "malloc" : " 14 GB", | 20260507 20:50:16.520 | INFO | "pinned" : " 29 GB" | 20260507 20:50:16.520 | INFO | } | 20260507 20:50:16.520 | INFO | Perf report for the last 5.14065 seconds: | 20260507 20:50:16.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:16.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.520 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:50:16.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.520 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:50:16.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:16.521 | INFO | Uploaded 36GiB to GPU in 2.5646s (14.0373GiB/s) | 20260507 20:50:16.521 | INFO | Downloaded 34GiB to GPU in 2.4821s (13.6981GiB/s) | 20260507 20:50:21.658 | INFO | Analyzing chunk frames = [2499584,2500096) | 20260507 20:50:21.658 | INFO | MemInfo: { | 20260507 20:50:21.658 | INFO | "gpu" : " 17 GB", | 20260507 20:50:21.658 | INFO | "huge" : "", | 20260507 20:50:21.658 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:21.658 | INFO | "malloc" : " 14 GB", | 20260507 20:50:21.659 | INFO | "pinned" : " 29 GB" | 20260507 20:50:21.659 | INFO | } | 20260507 20:50:21.659 | INFO | Perf report for the last 5.13836 seconds: | 20260507 20:50:21.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:21.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:50:21.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:50:21.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:21.659 | INFO | Uploaded 36GiB to GPU in 2.50226s (14.387GiB/s) | 20260507 20:50:21.659 | INFO | Downloaded 34GiB to GPU in 2.48488s (13.6827GiB/s) | 20260507 20:50:26.797 | INFO | Analyzing chunk frames = [2500096,2500608) | 20260507 20:50:26.797 | INFO | MemInfo: { | 20260507 20:50:26.797 | INFO | "gpu" : " 17 GB", | 20260507 20:50:26.797 | INFO | "huge" : "", | 20260507 20:50:26.797 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:26.798 | INFO | "malloc" : " 14 GB", | 20260507 20:50:26.798 | INFO | "pinned" : " 29 GB" | 20260507 20:50:26.798 | INFO | } | 20260507 20:50:26.798 | INFO | Perf report for the last 5.13903 seconds: | 20260507 20:50:26.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:26.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 20:50:26.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:50:26.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:26.798 | INFO | Uploaded 36GiB to GPU in 2.50149s (14.3914GiB/s) | 20260507 20:50:26.798 | INFO | Downloaded 34GiB to GPU in 2.49254s (13.6407GiB/s) | 20260507 20:50:31.937 | INFO | Analyzing chunk frames = [2500608,2501120) | 20260507 20:50:31.937 | INFO | MemInfo: { | 20260507 20:50:31.937 | INFO | "gpu" : " 17 GB", | 20260507 20:50:31.937 | INFO | "huge" : "", | 20260507 20:50:31.937 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:31.937 | INFO | "malloc" : " 14 GB", | 20260507 20:50:31.937 | INFO | "pinned" : " 29 GB" | 20260507 20:50:31.937 | INFO | } | 20260507 20:50:31.937 | INFO | Perf report for the last 5.13953 seconds: | 20260507 20:50:31.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:31.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.937 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 20:50:31.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.937 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:50:31.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:31.938 | INFO | Uploaded 36GiB to GPU in 2.43991s (14.7546GiB/s) | 20260507 20:50:31.938 | INFO | Downloaded 34GiB to GPU in 2.49413s (13.632GiB/s) | 20260507 20:50:37.076 | INFO | Analyzing chunk frames = [2501120,2501632) | 20260507 20:50:37.076 | INFO | MemInfo: { | 20260507 20:50:37.076 | INFO | "gpu" : " 17 GB", | 20260507 20:50:37.076 | INFO | "huge" : "", | 20260507 20:50:37.076 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:37.076 | INFO | "malloc" : " 14 GB", | 20260507 20:50:37.076 | INFO | "pinned" : " 29 GB" | 20260507 20:50:37.076 | INFO | } | 20260507 20:50:37.076 | INFO | Perf report for the last 5.13896 seconds: | 20260507 20:50:37.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:37.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.076 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 20:50:37.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.076 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:50:37.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:37.077 | INFO | Uploaded 36GiB to GPU in 2.41066s (14.9336GiB/s) | 20260507 20:50:37.077 | INFO | Downloaded 34GiB to GPU in 2.3821s (14.2731GiB/s) | 20260507 20:50:42.215 | INFO | Analyzing chunk frames = [2501632,2502144) | 20260507 20:50:42.215 | INFO | MemInfo: { | 20260507 20:50:42.215 | INFO | "gpu" : " 17 GB", | 20260507 20:50:42.215 | INFO | "huge" : "", | 20260507 20:50:42.215 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:42.215 | INFO | "malloc" : " 14 GB", | 20260507 20:50:42.215 | INFO | "pinned" : " 29 GB" | 20260507 20:50:42.215 | INFO | } | 20260507 20:50:42.215 | INFO | Perf report for the last 5.13886 seconds: | 20260507 20:50:42.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:42.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:50:42.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 20:50:42.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:42.215 | INFO | Uploaded 36GiB to GPU in 2.56109s (14.0565GiB/s) | 20260507 20:50:42.216 | INFO | Downloaded 34GiB to GPU in 2.44053s (13.9314GiB/s) | 20260507 20:50:47.355 | INFO | Analyzing chunk frames = [2502144,2502656) | 20260507 20:50:47.355 | INFO | MemInfo: { | 20260507 20:50:47.355 | INFO | "gpu" : " 17 GB", | 20260507 20:50:47.355 | INFO | "huge" : "", | 20260507 20:50:47.355 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:47.355 | INFO | "malloc" : " 14 GB", | 20260507 20:50:47.355 | INFO | "pinned" : " 29 GB" | 20260507 20:50:47.355 | INFO | } | 20260507 20:50:47.355 | INFO | Perf report for the last 5.13998 seconds: | 20260507 20:50:47.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:47.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:50:47.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:50:47.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:47.355 | INFO | Uploaded 36GiB to GPU in 2.51689s (14.3034GiB/s) | 20260507 20:50:47.355 | INFO | Downloaded 34GiB to GPU in 2.4156s (14.0752GiB/s) | 20260507 20:50:47.452 | INFO | Starting a round of Baz IO | 20260507 20:50:50.932 | INFO | Populating RTMetrics report | 20260507 20:50:50.933 | INFO | RTMetrics report populated | 20260507 20:50:50.933 | INFO | Opening tmp RTMetrics json file | 20260507 20:50:50.956 | INFO | Serializing RTMetrics to json | 20260507 20:50:50.958 | INFO | Serialization done, swapping tmp file now... | 20260507 20:50:50.959 | INFO | RTMetrics done | 20260507 20:50:51.033 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.581 seconds | 20260507 20:50:51.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.16% / 0.01% / 55.83% | 20260507 20:50:51.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88241% / 1.58946e-05% / 6.35783e-05% | 20260507 20:50:51.034 | INFO | Baz IO overall: | 20260507 20:50:51.034 | INFO | BytesWritten : 1137725771843 | 20260507 20:50:51.034 | INFO | NumEvents : 551508216762 | 20260507 20:50:52.494 | INFO | Analyzing chunk frames = [2502656,2503168) | 20260507 20:50:52.494 | INFO | MemInfo: { | 20260507 20:50:52.494 | INFO | "gpu" : " 17 GB", | 20260507 20:50:52.494 | INFO | "huge" : "", | 20260507 20:50:52.494 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:52.494 | INFO | "malloc" : " 14 GB", | 20260507 20:50:52.494 | INFO | "pinned" : " 29 GB" | 20260507 20:50:52.494 | INFO | } | 20260507 20:50:52.494 | INFO | Perf report for the last 5.13883 seconds: | 20260507 20:50:52.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:52.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:52.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:52.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:52.494 | INFO | ANALYSIS : 68.9%, 3.0, 106.7ms, 0 | 20260507 20:50:52.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:52.494 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:50:52.494 | INFO | PRE_HQ : 66.2%, 1.7, 58.1ms, 0 | 20260507 20:50:52.494 | INFO | BAZWRITER : 69.3%, 4.4, 154.9ms, 0 | 20260507 20:50:52.494 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 20:50:52.495 | INFO | Uploaded 36GiB to GPU in 2.57819s (13.9633GiB/s) | 20260507 20:50:52.495 | INFO | Downloaded 38GiB to GPU in 2.70077s (14.0701GiB/s) | 20260507 20:50:57.633 | INFO | Analyzing chunk frames = [2503168,2503680) | 20260507 20:50:57.633 | INFO | MemInfo: { | 20260507 20:50:57.633 | INFO | "gpu" : " 17 GB", | 20260507 20:50:57.633 | INFO | "huge" : "", | 20260507 20:50:57.633 | INFO | "huge_shared" : " 29 GB", | 20260507 20:50:57.633 | INFO | "malloc" : " 14 GB", | 20260507 20:50:57.634 | INFO | "pinned" : " 29 GB" | 20260507 20:50:57.634 | INFO | } | 20260507 20:50:57.634 | INFO | Perf report for the last 5.13964 seconds: | 20260507 20:50:57.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:50:57.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 20:50:57.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 20:50:57.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:50:57.634 | INFO | Uploaded 36GiB to GPU in 2.58766s (13.9122GiB/s) | 20260507 20:50:57.634 | INFO | Downloaded 34GiB to GPU in 2.4814s (13.7019GiB/s) | 20260507 20:51:02.773 | INFO | Analyzing chunk frames = [2503680,2504192) | 20260507 20:51:02.773 | INFO | MemInfo: { | 20260507 20:51:02.773 | INFO | "gpu" : " 17 GB", | 20260507 20:51:02.773 | INFO | "huge" : "", | 20260507 20:51:02.773 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:02.773 | INFO | "malloc" : " 14 GB", | 20260507 20:51:02.773 | INFO | "pinned" : " 29 GB" | 20260507 20:51:02.773 | INFO | } | 20260507 20:51:02.773 | INFO | Perf report for the last 5.13982 seconds: | 20260507 20:51:02.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:02.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.774 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 20:51:02.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.774 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:51:02.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:02.774 | INFO | Uploaded 36GiB to GPU in 2.51278s (14.3268GiB/s) | 20260507 20:51:02.774 | INFO | Downloaded 34GiB to GPU in 2.47803s (13.7206GiB/s) | 20260507 20:51:07.913 | INFO | Analyzing chunk frames = [2504192,2504704) | 20260507 20:51:07.913 | INFO | MemInfo: { | 20260507 20:51:07.913 | INFO | "gpu" : " 17 GB", | 20260507 20:51:07.913 | INFO | "huge" : "", | 20260507 20:51:07.913 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:07.913 | INFO | "malloc" : " 14 GB", | 20260507 20:51:07.913 | INFO | "pinned" : " 29 GB" | 20260507 20:51:07.913 | INFO | } | 20260507 20:51:07.913 | INFO | Perf report for the last 5.13995 seconds: | 20260507 20:51:07.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:07.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.913 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 20:51:07.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.914 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:51:07.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:07.914 | INFO | Uploaded 36GiB to GPU in 2.55814s (14.0727GiB/s) | 20260507 20:51:07.914 | INFO | Downloaded 34GiB to GPU in 2.48982s (13.6556GiB/s) | 20260507 20:51:13.052 | INFO | Analyzing chunk frames = [2504704,2505216) | 20260507 20:51:13.052 | INFO | MemInfo: { | 20260507 20:51:13.052 | INFO | "gpu" : " 17 GB", | 20260507 20:51:13.052 | INFO | "huge" : "", | 20260507 20:51:13.052 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:13.052 | INFO | "malloc" : " 14 GB", | 20260507 20:51:13.052 | INFO | "pinned" : " 29 GB" | 20260507 20:51:13.052 | INFO | } | 20260507 20:51:13.052 | INFO | Perf report for the last 5.13865 seconds: | 20260507 20:51:13.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:13.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 20:51:13.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 20:51:13.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:13.052 | INFO | Uploaded 36GiB to GPU in 2.44312s (14.7353GiB/s) | 20260507 20:51:13.052 | INFO | Downloaded 34GiB to GPU in 2.49602s (13.6217GiB/s) | 20260507 20:51:18.191 | INFO | Analyzing chunk frames = [2505216,2505728) | 20260507 20:51:18.191 | INFO | MemInfo: { | 20260507 20:51:18.191 | INFO | "gpu" : " 17 GB", | 20260507 20:51:18.191 | INFO | "huge" : "", | 20260507 20:51:18.191 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:18.191 | INFO | "malloc" : " 14 GB", | 20260507 20:51:18.191 | INFO | "pinned" : " 29 GB" | 20260507 20:51:18.191 | INFO | } | 20260507 20:51:18.191 | INFO | Perf report for the last 5.13924 seconds: | 20260507 20:51:18.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:18.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.191 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:51:18.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.191 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:51:18.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:18.192 | INFO | Uploaded 36GiB to GPU in 2.48s (14.5161GiB/s) | 20260507 20:51:18.192 | INFO | Downloaded 34GiB to GPU in 2.42253s (14.0349GiB/s) | 20260507 20:51:23.330 | INFO | Analyzing chunk frames = [2505728,2506240) | 20260507 20:51:23.330 | INFO | MemInfo: { | 20260507 20:51:23.330 | INFO | "gpu" : " 17 GB", | 20260507 20:51:23.330 | INFO | "huge" : "", | 20260507 20:51:23.330 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:23.330 | INFO | "malloc" : " 14 GB", | 20260507 20:51:23.330 | INFO | "pinned" : " 29 GB" | 20260507 20:51:23.330 | INFO | } | 20260507 20:51:23.330 | INFO | Perf report for the last 5.13893 seconds: | 20260507 20:51:23.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:23.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.330 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:51:23.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.330 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:51:23.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:23.331 | INFO | Uploaded 36GiB to GPU in 2.52241s (14.272GiB/s) | 20260507 20:51:23.331 | INFO | Downloaded 34GiB to GPU in 2.45303s (13.8604GiB/s) | 20260507 20:51:28.469 | INFO | Analyzing chunk frames = [2506240,2506752) | 20260507 20:51:28.469 | INFO | MemInfo: { | 20260507 20:51:28.469 | INFO | "gpu" : " 17 GB", | 20260507 20:51:28.469 | INFO | "huge" : "", | 20260507 20:51:28.469 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:28.469 | INFO | "malloc" : " 14 GB", | 20260507 20:51:28.469 | INFO | "pinned" : " 29 GB" | 20260507 20:51:28.469 | INFO | } | 20260507 20:51:28.469 | INFO | Perf report for the last 5.13868 seconds: | 20260507 20:51:28.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:28.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:51:28.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:51:28.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:28.469 | INFO | Uploaded 36GiB to GPU in 2.49082s (14.4531GiB/s) | 20260507 20:51:28.469 | INFO | Downloaded 34GiB to GPU in 2.43724s (13.9502GiB/s) | 20260507 20:51:28.470 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:51:28.470 | INFO | Total mem usage: 16.4 GiB | 20260507 20:51:28.470 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:51:28.470 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.07%) | 20260507 20:51:28.470 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:51:28.470 | INFO | High Water Breakdown: | 20260507 20:51:28.470 | INFO | 60% : 9.03 GiB ( 91% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:51:28.470 | INFO | 39.8% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:51:28.470 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:51:28.470 | INFO | ------------------------------------------ | 20260507 20:51:28.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:51:28.470 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:51:28.470 | INFO | Total mem usage: 26.5 GiB | 20260507 20:51:28.470 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:51:28.470 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:51:28.470 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:51:28.470 | INFO | High Water Breakdown: | 20260507 20:51:28.470 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:51:28.470 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:51:28.470 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:51:28.470 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:51:28.470 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:51:28.470 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:51:28.470 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:51:28.470 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:51:28.470 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:51:28.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:51:28.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:51:28.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:51:28.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:51:28.470 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:51:28.470 | INFO | ------------------------------------------ | 20260507 20:51:28.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:51:28.470 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:51:28.470 | INFO | Total mem usage: 27.8 GiB | 20260507 20:51:28.470 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:51:28.470 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:51:28.470 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:51:28.470 | INFO | High Water Breakdown: | 20260507 20:51:28.470 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:51:28.470 | INFO | ------------------------------------------ | 20260507 20:51:28.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:51:28.470 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:51:28.470 | INFO | Total mem usage: 15.7 GiB | 20260507 20:51:28.470 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:51:28.470 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:51:28.470 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:51:28.470 | INFO | High Water Breakdown: | 20260507 20:51:28.470 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:51:28.470 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:51:28.470 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:51:28.470 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:51:28.470 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:51:28.470 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:51:28.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:51:28.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:51:28.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:51:28.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:51:28.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:51:28.470 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:51:28.470 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:51:28.470 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:51:28.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:51:28.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:51:28.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:51:28.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:51:28.470 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:51:28.470 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:51:28.470 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:51:28.470 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:51:28.470 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:51:28.470 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:51:28.470 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:51:28.470 | INFO | ------------------------------------------ | 20260507 20:51:28.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:51:28.470 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:51:28.470 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:51:28.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:51:28.470 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.42e+01 / 2.18e+00 | 20260507 20:51:28.470 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.22e-02 / 4.71e-03 | 20260507 20:51:28.470 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.46e+00, 5.92e-01 / 2.39e-01 | 20260507 20:51:28.470 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.07e-01 / 1.05e-01 | 20260507 20:51:28.470 | INFO | PerfReport: 16.09%, DME -- 4.25e+01 / 4.61e+01, 4.19e+00 / 2.24e+00 | 20260507 20:51:28.470 | INFO | PerfReport: 55.83%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.62e+00 / 1.36e+00 | 20260507 20:51:28.470 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.51e+00 / 2.86e+00, 4.41e-01 / 1.74e-01 | 20260507 20:51:28.470 | INFO | PerfReport: 3.57%, Metrics -- 1.18e+00 / 1.43e+00, 2.99e-01 / 1.61e-01 | 20260507 20:51:28.471 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 4.07e-01, 5.95e-02 / 3.11e-02 | 20260507 20:51:28.471 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:51:28.471 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:51:28.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:51:28.471 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 24.42%, Upload -- 2.62e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 30.80%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 2.20%, LockUpload -- 2.36e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 2.61%, LockDownload -- 2.80e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 15.75%, LockCompute -- 1.69e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 20:51:28.471 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 20:51:28.566 | INFO | Starting a round of Baz IO | 20260507 20:51:32.027 | INFO | Populating RTMetrics report | 20260507 20:51:32.027 | INFO | RTMetrics report populated | 20260507 20:51:32.027 | INFO | Opening tmp RTMetrics json file | 20260507 20:51:32.055 | INFO | Serializing RTMetrics to json | 20260507 20:51:32.057 | INFO | Serialization done, swapping tmp file now... | 20260507 20:51:32.060 | INFO | RTMetrics done | 20260507 20:51:32.153 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.587 seconds | 20260507 20:51:32.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.15% / 0.01% / 55.84% | 20260507 20:51:32.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.87903% / 1.58946e-05% / 7.54992e-05% | 20260507 20:51:32.153 | INFO | Baz IO overall: | 20260507 20:51:32.153 | INFO | BytesWritten : 1139628498248 | 20260507 20:51:32.153 | INFO | NumEvents : 552429468653 | 20260507 20:51:33.608 | INFO | Analyzing chunk frames = [2506752,2507264) | 20260507 20:51:33.608 | INFO | MemInfo: { | 20260507 20:51:33.608 | INFO | "gpu" : " 17 GB", | 20260507 20:51:33.608 | INFO | "huge" : "", | 20260507 20:51:33.608 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:33.608 | INFO | "malloc" : " 14 GB", | 20260507 20:51:33.608 | INFO | "pinned" : " 29 GB" | 20260507 20:51:33.608 | INFO | } | 20260507 20:51:33.608 | INFO | Perf report for the last 5.13903 seconds: | 20260507 20:51:33.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:33.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:33.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:33.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:33.608 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260507 20:51:33.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:33.608 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:51:33.608 | INFO | PRE_HQ : 66.0%, 1.7, 58.0ms, 0 | 20260507 20:51:33.608 | INFO | BAZWRITER : 69.5%, 4.0, 142.8ms, 0 | 20260507 20:51:33.608 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 20:51:33.609 | INFO | Uploaded 36GiB to GPU in 2.51695s (14.303GiB/s) | 20260507 20:51:33.609 | INFO | Downloaded 38GiB to GPU in 2.68934s (14.1299GiB/s) | 20260507 20:51:38.749 | INFO | Analyzing chunk frames = [2507264,2507776) | 20260507 20:51:38.749 | INFO | MemInfo: { | 20260507 20:51:38.749 | INFO | "gpu" : " 17 GB", | 20260507 20:51:38.749 | INFO | "huge" : "", | 20260507 20:51:38.749 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:38.749 | INFO | "malloc" : " 14 GB", | 20260507 20:51:38.749 | INFO | "pinned" : " 29 GB" | 20260507 20:51:38.749 | INFO | } | 20260507 20:51:38.750 | INFO | Perf report for the last 5.14151 seconds: | 20260507 20:51:38.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:38.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 20:51:38.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 20:51:38.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:38.750 | INFO | Uploaded 36GiB to GPU in 2.44364s (14.7321GiB/s) | 20260507 20:51:38.750 | INFO | Downloaded 34GiB to GPU in 2.48979s (13.6558GiB/s) | 20260507 20:51:43.888 | INFO | Analyzing chunk frames = [2507776,2508288) | 20260507 20:51:43.888 | INFO | MemInfo: { | 20260507 20:51:43.888 | INFO | "gpu" : " 17 GB", | 20260507 20:51:43.888 | INFO | "huge" : "", | 20260507 20:51:43.888 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:43.888 | INFO | "malloc" : " 14 GB", | 20260507 20:51:43.888 | INFO | "pinned" : " 29 GB" | 20260507 20:51:43.890 | INFO | } | 20260507 20:51:43.890 | INFO | Perf report for the last 5.14015 seconds: | 20260507 20:51:43.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:43.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 20:51:43.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:51:43.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:43.890 | INFO | Uploaded 36GiB to GPU in 2.45744s (14.6494GiB/s) | 20260507 20:51:43.890 | INFO | Downloaded 34GiB to GPU in 2.482s (13.6986GiB/s) | 20260507 20:51:49.026 | INFO | Analyzing chunk frames = [2508288,2508800) | 20260507 20:51:49.027 | INFO | MemInfo: { | 20260507 20:51:49.027 | INFO | "gpu" : " 17 GB", | 20260507 20:51:49.027 | INFO | "huge" : "", | 20260507 20:51:49.027 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:49.027 | INFO | "malloc" : " 14 GB", | 20260507 20:51:49.027 | INFO | "pinned" : " 29 GB" | 20260507 20:51:49.027 | INFO | } | 20260507 20:51:49.027 | INFO | Perf report for the last 5.13707 seconds: | 20260507 20:51:49.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:49.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 20:51:49.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:51:49.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:49.027 | INFO | Uploaded 36GiB to GPU in 2.46948s (14.578GiB/s) | 20260507 20:51:49.027 | INFO | Downloaded 34GiB to GPU in 2.49309s (13.6377GiB/s) | 20260507 20:51:54.165 | INFO | Analyzing chunk frames = [2508800,2509312) | 20260507 20:51:54.165 | INFO | MemInfo: { | 20260507 20:51:54.165 | INFO | "gpu" : " 17 GB", | 20260507 20:51:54.165 | INFO | "huge" : "", | 20260507 20:51:54.165 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:54.166 | INFO | "malloc" : " 14 GB", | 20260507 20:51:54.166 | INFO | "pinned" : " 29 GB" | 20260507 20:51:54.166 | INFO | } | 20260507 20:51:54.166 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:51:54.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:54.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 20:51:54.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:51:54.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:54.166 | INFO | Uploaded 36GiB to GPU in 2.42357s (14.8541GiB/s) | 20260507 20:51:54.166 | INFO | Downloaded 34GiB to GPU in 2.49208s (13.6432GiB/s) | 20260507 20:51:59.304 | INFO | Analyzing chunk frames = [2509312,2509824) | 20260507 20:51:59.304 | INFO | MemInfo: { | 20260507 20:51:59.304 | INFO | "gpu" : " 17 GB", | 20260507 20:51:59.304 | INFO | "huge" : "", | 20260507 20:51:59.304 | INFO | "huge_shared" : " 29 GB", | 20260507 20:51:59.304 | INFO | "malloc" : " 14 GB", | 20260507 20:51:59.304 | INFO | "pinned" : " 29 GB" | 20260507 20:51:59.304 | INFO | } | 20260507 20:51:59.304 | INFO | Perf report for the last 5.13871 seconds: | 20260507 20:51:59.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:51:59.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.305 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:51:59.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.305 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:51:59.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:51:59.305 | INFO | Uploaded 36GiB to GPU in 2.51514s (14.3133GiB/s) | 20260507 20:51:59.305 | INFO | Downloaded 34GiB to GPU in 2.41656s (14.0696GiB/s) | 20260507 20:52:04.445 | INFO | Analyzing chunk frames = [2509824,2510336) | 20260507 20:52:04.445 | INFO | MemInfo: { | 20260507 20:52:04.445 | INFO | "gpu" : " 17 GB", | 20260507 20:52:04.445 | INFO | "huge" : "", | 20260507 20:52:04.445 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:04.445 | INFO | "malloc" : " 14 GB", | 20260507 20:52:04.445 | INFO | "pinned" : " 29 GB" | 20260507 20:52:04.445 | INFO | } | 20260507 20:52:04.445 | INFO | Perf report for the last 5.14053 seconds: | 20260507 20:52:04.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:04.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.445 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:52:04.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.445 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:52:04.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:04.446 | INFO | Uploaded 36GiB to GPU in 2.5497s (14.1193GiB/s) | 20260507 20:52:04.446 | INFO | Downloaded 34GiB to GPU in 2.44341s (13.915GiB/s) | 20260507 20:52:09.584 | INFO | Analyzing chunk frames = [2510336,2510848) | 20260507 20:52:09.584 | INFO | MemInfo: { | 20260507 20:52:09.584 | INFO | "gpu" : " 17 GB", | 20260507 20:52:09.584 | INFO | "huge" : "", | 20260507 20:52:09.584 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:09.584 | INFO | "malloc" : " 14 GB", | 20260507 20:52:09.584 | INFO | "pinned" : " 29 GB" | 20260507 20:52:09.584 | INFO | } | 20260507 20:52:09.584 | INFO | Perf report for the last 5.13939 seconds: | 20260507 20:52:09.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:09.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.584 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:52:09.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.585 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 20:52:09.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:09.585 | INFO | Uploaded 36GiB to GPU in 2.52302s (14.2686GiB/s) | 20260507 20:52:09.585 | INFO | Downloaded 34GiB to GPU in 2.42952s (13.9945GiB/s) | 20260507 20:52:09.682 | INFO | Starting a round of Baz IO | 20260507 20:52:13.147 | INFO | Populating RTMetrics report | 20260507 20:52:13.147 | INFO | RTMetrics report populated | 20260507 20:52:13.147 | INFO | Opening tmp RTMetrics json file | 20260507 20:52:13.173 | INFO | Serializing RTMetrics to json | 20260507 20:52:13.175 | INFO | Serialization done, swapping tmp file now... | 20260507 20:52:13.175 | INFO | RTMetrics done | 20260507 20:52:13.270 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.587 seconds | 20260507 20:52:13.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.14% / 0.01% / 55.85% | 20260507 20:52:13.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88132% / 7.94729e-06% / 4.76837e-05% | 20260507 20:52:13.270 | INFO | Baz IO overall: | 20260507 20:52:13.270 | INFO | BytesWritten : 1141533799724 | 20260507 20:52:13.270 | INFO | NumEvents : 553352994345 | 20260507 20:52:14.723 | INFO | Analyzing chunk frames = [2510848,2511360) | 20260507 20:52:14.723 | INFO | MemInfo: { | 20260507 20:52:14.723 | INFO | "gpu" : " 17 GB", | 20260507 20:52:14.723 | INFO | "huge" : "", | 20260507 20:52:14.724 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:14.724 | INFO | "malloc" : " 14 GB", | 20260507 20:52:14.724 | INFO | "pinned" : " 29 GB" | 20260507 20:52:14.724 | INFO | } | 20260507 20:52:14.724 | INFO | Perf report for the last 5.1393 seconds: | 20260507 20:52:14.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:14.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:14.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:14.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:14.724 | INFO | ANALYSIS : 68.8%, 3.0, 107.1ms, 0 | 20260507 20:52:14.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:14.724 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 20:52:14.724 | INFO | PRE_HQ : 66.1%, 1.7, 58.3ms, 0 | 20260507 20:52:14.724 | INFO | BAZWRITER : 69.4%, 3.7, 131.1ms, 0 | 20260507 20:52:14.724 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 20:52:14.724 | INFO | Uploaded 36GiB to GPU in 2.58479s (13.9276GiB/s) | 20260507 20:52:14.724 | INFO | Downloaded 38GiB to GPU in 2.69651s (14.0923GiB/s) | 20260507 20:52:19.863 | INFO | Analyzing chunk frames = [2511360,2511872) | 20260507 20:52:19.863 | INFO | MemInfo: { | 20260507 20:52:19.863 | INFO | "gpu" : " 17 GB", | 20260507 20:52:19.863 | INFO | "huge" : "", | 20260507 20:52:19.863 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:19.863 | INFO | "malloc" : " 14 GB", | 20260507 20:52:19.863 | INFO | "pinned" : " 29 GB" | 20260507 20:52:19.863 | INFO | } | 20260507 20:52:19.863 | INFO | Perf report for the last 5.13938 seconds: | 20260507 20:52:19.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:19.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:52:19.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 20:52:19.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:19.863 | INFO | Uploaded 36GiB to GPU in 2.58246s (13.9402GiB/s) | 20260507 20:52:19.863 | INFO | Downloaded 34GiB to GPU in 2.49165s (13.6456GiB/s) | 20260507 20:52:25.002 | INFO | Analyzing chunk frames = [2511872,2512384) | 20260507 20:52:25.002 | INFO | MemInfo: { | 20260507 20:52:25.002 | INFO | "gpu" : " 17 GB", | 20260507 20:52:25.002 | INFO | "huge" : "", | 20260507 20:52:25.002 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:25.002 | INFO | "malloc" : " 14 GB", | 20260507 20:52:25.002 | INFO | "pinned" : " 29 GB" | 20260507 20:52:25.002 | INFO | } | 20260507 20:52:25.002 | INFO | Perf report for the last 5.13903 seconds: | 20260507 20:52:25.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:25.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.002 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 20:52:25.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.002 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:52:25.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:25.003 | INFO | Uploaded 36GiB to GPU in 2.52542s (14.2551GiB/s) | 20260507 20:52:25.003 | INFO | Downloaded 34GiB to GPU in 2.4776s (13.723GiB/s) | 20260507 20:52:30.141 | INFO | Analyzing chunk frames = [2512384,2512896) | 20260507 20:52:30.141 | INFO | MemInfo: { | 20260507 20:52:30.141 | INFO | "gpu" : " 17 GB", | 20260507 20:52:30.141 | INFO | "huge" : "", | 20260507 20:52:30.141 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:30.141 | INFO | "malloc" : " 14 GB", | 20260507 20:52:30.141 | INFO | "pinned" : " 29 GB" | 20260507 20:52:30.141 | INFO | } | 20260507 20:52:30.141 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:52:30.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:30.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 20:52:30.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:52:30.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:30.141 | INFO | Uploaded 36GiB to GPU in 2.5375s (14.1872GiB/s) | 20260507 20:52:30.141 | INFO | Downloaded 34GiB to GPU in 2.48902s (13.66GiB/s) | 20260507 20:52:35.280 | INFO | Analyzing chunk frames = [2512896,2513408) | 20260507 20:52:35.280 | INFO | MemInfo: { | 20260507 20:52:35.280 | INFO | "gpu" : " 17 GB", | 20260507 20:52:35.280 | INFO | "huge" : "", | 20260507 20:52:35.280 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:35.280 | INFO | "malloc" : " 14 GB", | 20260507 20:52:35.280 | INFO | "pinned" : " 29 GB" | 20260507 20:52:35.280 | INFO | } | 20260507 20:52:35.280 | INFO | Perf report for the last 5.13871 seconds: | 20260507 20:52:35.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:35.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:52:35.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:52:35.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:35.280 | INFO | Uploaded 36GiB to GPU in 2.41603s (14.9005GiB/s) | 20260507 20:52:35.280 | INFO | Downloaded 34GiB to GPU in 2.49362s (13.6348GiB/s) | 20260507 20:52:40.420 | INFO | Analyzing chunk frames = [2513408,2513920) | 20260507 20:52:40.420 | INFO | MemInfo: { | 20260507 20:52:40.420 | INFO | "gpu" : " 17 GB", | 20260507 20:52:40.420 | INFO | "huge" : "", | 20260507 20:52:40.420 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:40.420 | INFO | "malloc" : " 14 GB", | 20260507 20:52:40.420 | INFO | "pinned" : " 29 GB" | 20260507 20:52:40.420 | INFO | } | 20260507 20:52:40.420 | INFO | Perf report for the last 5.14004 seconds: | 20260507 20:52:40.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:40.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:52:40.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:52:40.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:40.420 | INFO | Uploaded 36GiB to GPU in 2.48252s (14.5014GiB/s) | 20260507 20:52:40.420 | INFO | Downloaded 34GiB to GPU in 2.42327s (14.0306GiB/s) | 20260507 20:52:45.560 | INFO | Analyzing chunk frames = [2513920,2514432) | 20260507 20:52:45.560 | INFO | MemInfo: { | 20260507 20:52:45.560 | INFO | "gpu" : " 17 GB", | 20260507 20:52:45.560 | INFO | "huge" : "", | 20260507 20:52:45.560 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:45.560 | INFO | "malloc" : " 14 GB", | 20260507 20:52:45.560 | INFO | "pinned" : " 29 GB" | 20260507 20:52:45.560 | INFO | } | 20260507 20:52:45.560 | INFO | Perf report for the last 5.14056 seconds: | 20260507 20:52:45.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:45.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.560 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:52:45.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.561 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:52:45.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:45.561 | INFO | Uploaded 36GiB to GPU in 2.52603s (14.2516GiB/s) | 20260507 20:52:45.561 | INFO | Downloaded 34GiB to GPU in 2.44339s (13.9151GiB/s) | 20260507 20:52:50.700 | INFO | Analyzing chunk frames = [2514432,2514944) | 20260507 20:52:50.700 | INFO | MemInfo: { | 20260507 20:52:50.700 | INFO | "gpu" : " 17 GB", | 20260507 20:52:50.700 | INFO | "huge" : "", | 20260507 20:52:50.700 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:50.700 | INFO | "malloc" : " 14 GB", | 20260507 20:52:50.700 | INFO | "pinned" : " 29 GB" | 20260507 20:52:50.700 | INFO | } | 20260507 20:52:50.700 | INFO | Perf report for the last 5.13962 seconds: | 20260507 20:52:50.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:50.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.700 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:52:50.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.700 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:52:50.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:50.701 | INFO | Uploaded 36GiB to GPU in 2.49383s (14.4356GiB/s) | 20260507 20:52:50.701 | INFO | Downloaded 34GiB to GPU in 2.43316s (13.9736GiB/s) | 20260507 20:52:50.701 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:52:50.701 | INFO | Total mem usage: 16.4 GiB | 20260507 20:52:50.701 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:52:50.701 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -7.71%) | 20260507 20:52:50.701 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:52:50.701 | INFO | High Water Breakdown: | 20260507 20:52:50.701 | INFO | 60.3% : 9.19 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:52:50.701 | INFO | 39.5% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:52:50.701 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:52:50.701 | INFO | ------------------------------------------ | 20260507 20:52:50.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:52:50.701 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:52:50.701 | INFO | Total mem usage: 26.5 GiB | 20260507 20:52:50.701 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:52:50.701 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:52:50.701 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:52:50.701 | INFO | High Water Breakdown: | 20260507 20:52:50.701 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:52:50.701 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:52:50.701 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:52:50.701 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:52:50.701 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:52:50.701 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:52:50.701 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:52:50.701 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:52:50.701 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:52:50.701 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:52:50.701 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:52:50.701 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:52:50.701 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:52:50.701 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:52:50.701 | INFO | ------------------------------------------ | 20260507 20:52:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:52:50.702 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:52:50.702 | INFO | Total mem usage: 27.8 GiB | 20260507 20:52:50.702 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:52:50.702 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:52:50.702 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:52:50.702 | INFO | High Water Breakdown: | 20260507 20:52:50.702 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:52:50.702 | INFO | ------------------------------------------ | 20260507 20:52:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:52:50.702 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:52:50.702 | INFO | Total mem usage: 15.7 GiB | 20260507 20:52:50.702 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:52:50.702 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:52:50.702 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:52:50.702 | INFO | High Water Breakdown: | 20260507 20:52:50.702 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:52:50.702 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:52:50.702 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:52:50.702 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:52:50.702 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:52:50.702 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:52:50.702 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:52:50.702 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:52:50.702 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:52:50.702 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:52:50.702 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:52:50.702 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:52:50.702 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:52:50.702 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:52:50.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:52:50.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:52:50.702 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:52:50.702 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:52:50.702 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:52:50.702 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:52:50.702 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:52:50.702 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:52:50.702 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:52:50.702 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:52:50.702 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:52:50.702 | INFO | ------------------------------------------ | 20260507 20:52:50.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:52:50.702 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:52:50.702 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:52:50.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:52:50.702 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.42e+01 / 2.09e+00 | 20260507 20:52:50.702 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.02e-01, 1.28e-02 / 5.09e-03 | 20260507 20:52:50.702 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.49e+00, 6.01e-01 / 2.54e-01 | 20260507 20:52:50.702 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.09e-01 / 1.11e-01 | 20260507 20:52:50.702 | INFO | PerfReport: 16.12%, DME -- 4.25e+01 / 4.61e+01, 4.31e+00 / 2.17e+00 | 20260507 20:52:50.702 | INFO | PerfReport: 55.71%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.54e+00 / 1.30e+00 | 20260507 20:52:50.702 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.53e+00 / 2.84e+00, 4.47e-01 / 1.95e-01 | 20260507 20:52:50.702 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.40e+00, 2.96e-01 / 1.59e-01 | 20260507 20:52:50.702 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.65e-01 / 4.00e-01, 5.95e-02 / 2.96e-02 | 20260507 20:52:50.702 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:52:50.702 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:52:50.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:52:50.702 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 20:52:50.702 | INFO | PerfReport: 24.34%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 20:52:50.702 | INFO | PerfReport: 24.24%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 20:52:50.703 | INFO | PerfReport: 30.78%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:52:50.703 | INFO | PerfReport: 2.14%, LockUpload -- 2.29e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260507 20:52:50.703 | INFO | PerfReport: 2.69%, LockDownload -- 2.89e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 20:52:50.703 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 20:52:50.703 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.34e-02 / 0.00e+00 | 20260507 20:52:50.813 | INFO | Starting a round of Baz IO | 20260507 20:52:54.292 | INFO | Populating RTMetrics report | 20260507 20:52:54.292 | INFO | RTMetrics report populated | 20260507 20:52:54.292 | INFO | Opening tmp RTMetrics json file | 20260507 20:52:54.335 | INFO | Serializing RTMetrics to json | 20260507 20:52:54.385 | INFO | Serialization done, swapping tmp file now... | 20260507 20:52:54.391 | INFO | RTMetrics done | 20260507 20:52:54.512 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.698 seconds | 20260507 20:52:54.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.13% / 0.01% / 55.86% | 20260507 20:52:54.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.87122% / 3.97364e-06% / 4.76837e-05% | 20260507 20:52:54.512 | INFO | Baz IO overall: | 20260507 20:52:54.513 | INFO | BytesWritten : 1143435990926 | 20260507 20:52:54.513 | INFO | NumEvents : 554274223530 | 20260507 20:52:55.838 | INFO | Analyzing chunk frames = [2514944,2515456) | 20260507 20:52:55.838 | INFO | MemInfo: { | 20260507 20:52:55.838 | INFO | "gpu" : " 17 GB", | 20260507 20:52:55.838 | INFO | "huge" : "", | 20260507 20:52:55.839 | INFO | "huge_shared" : " 29 GB", | 20260507 20:52:55.839 | INFO | "malloc" : " 14 GB", | 20260507 20:52:55.839 | INFO | "pinned" : " 29 GB" | 20260507 20:52:55.839 | INFO | } | 20260507 20:52:55.839 | INFO | Perf report for the last 5.13864 seconds: | 20260507 20:52:55.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:52:55.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:55.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:55.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:55.839 | INFO | ANALYSIS : 69.2%, 2.9, 105.0ms, 0 | 20260507 20:52:55.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:52:55.839 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260507 20:52:55.839 | INFO | PRE_HQ : 68.0%, 1.8, 63.8ms, 0 | 20260507 20:52:55.839 | INFO | BAZWRITER : 71.4%, 4.2, 154.2ms, 0 | 20260507 20:52:55.839 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:52:55.839 | INFO | Uploaded 36GiB to GPU in 2.54706s (14.1339GiB/s) | 20260507 20:52:55.839 | INFO | Downloaded 38GiB to GPU in 2.72561s (13.9418GiB/s) | 20260507 20:53:00.977 | INFO | Analyzing chunk frames = [2515456,2515968) | 20260507 20:53:00.977 | INFO | MemInfo: { | 20260507 20:53:00.977 | INFO | "gpu" : " 17 GB", | 20260507 20:53:00.977 | INFO | "huge" : "", | 20260507 20:53:00.977 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:00.977 | INFO | "malloc" : " 14 GB", | 20260507 20:53:00.977 | INFO | "pinned" : " 29 GB" | 20260507 20:53:00.977 | INFO | } | 20260507 20:53:00.977 | INFO | Perf report for the last 5.13836 seconds: | 20260507 20:53:00.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:00.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.977 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 20:53:00.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.977 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:53:00.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:00.978 | INFO | Uploaded 36GiB to GPU in 2.55655s (14.0815GiB/s) | 20260507 20:53:00.978 | INFO | Downloaded 34GiB to GPU in 2.4815s (13.7014GiB/s) | 20260507 20:53:06.116 | INFO | Analyzing chunk frames = [2515968,2516480) | 20260507 20:53:06.116 | INFO | MemInfo: { | 20260507 20:53:06.116 | INFO | "gpu" : " 17 GB", | 20260507 20:53:06.116 | INFO | "huge" : "", | 20260507 20:53:06.116 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:06.116 | INFO | "malloc" : " 14 GB", | 20260507 20:53:06.116 | INFO | "pinned" : " 29 GB" | 20260507 20:53:06.116 | INFO | } | 20260507 20:53:06.116 | INFO | Perf report for the last 5.13933 seconds: | 20260507 20:53:06.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:06.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.116 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:53:06.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.117 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:53:06.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:06.117 | INFO | Uploaded 36GiB to GPU in 2.47983s (14.5171GiB/s) | 20260507 20:53:06.117 | INFO | Downloaded 34GiB to GPU in 2.48066s (13.706GiB/s) | 20260507 20:53:11.255 | INFO | Analyzing chunk frames = [2516480,2516992) | 20260507 20:53:11.255 | INFO | MemInfo: { | 20260507 20:53:11.255 | INFO | "gpu" : " 17 GB", | 20260507 20:53:11.255 | INFO | "huge" : "", | 20260507 20:53:11.255 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:11.255 | INFO | "malloc" : " 14 GB", | 20260507 20:53:11.255 | INFO | "pinned" : " 29 GB" | 20260507 20:53:11.256 | INFO | } | 20260507 20:53:11.256 | INFO | Perf report for the last 5.13914 seconds: | 20260507 20:53:11.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:11.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 20:53:11.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:53:11.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:11.256 | INFO | Uploaded 36GiB to GPU in 2.52777s (14.2418GiB/s) | 20260507 20:53:11.256 | INFO | Downloaded 34GiB to GPU in 2.48004s (13.7095GiB/s) | 20260507 20:53:16.395 | INFO | Analyzing chunk frames = [2516992,2517504) | 20260507 20:53:16.395 | INFO | MemInfo: { | 20260507 20:53:16.395 | INFO | "gpu" : " 17 GB", | 20260507 20:53:16.395 | INFO | "huge" : "", | 20260507 20:53:16.395 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:16.395 | INFO | "malloc" : " 14 GB", | 20260507 20:53:16.395 | INFO | "pinned" : " 29 GB" | 20260507 20:53:16.395 | INFO | } | 20260507 20:53:16.395 | INFO | Perf report for the last 5.13959 seconds: | 20260507 20:53:16.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:16.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 20:53:16.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 20:53:16.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:16.395 | INFO | Uploaded 36GiB to GPU in 2.35671s (15.2755GiB/s) | 20260507 20:53:16.396 | INFO | Downloaded 34GiB to GPU in 2.50198s (13.5892GiB/s) | 20260507 20:53:21.534 | INFO | Analyzing chunk frames = [2517504,2518016) | 20260507 20:53:21.534 | INFO | MemInfo: { | 20260507 20:53:21.534 | INFO | "gpu" : " 17 GB", | 20260507 20:53:21.534 | INFO | "huge" : "", | 20260507 20:53:21.534 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:21.534 | INFO | "malloc" : " 14 GB", | 20260507 20:53:21.534 | INFO | "pinned" : " 29 GB" | 20260507 20:53:21.534 | INFO | } | 20260507 20:53:21.534 | INFO | Perf report for the last 5.13906 seconds: | 20260507 20:53:21.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:21.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.534 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 20:53:21.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.534 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:53:21.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:21.535 | INFO | Uploaded 36GiB to GPU in 2.52229s (14.2727GiB/s) | 20260507 20:53:21.535 | INFO | Downloaded 34GiB to GPU in 2.40664s (14.1276GiB/s) | 20260507 20:53:26.674 | INFO | Analyzing chunk frames = [2518016,2518528) | 20260507 20:53:26.674 | INFO | MemInfo: { | 20260507 20:53:26.674 | INFO | "gpu" : " 17 GB", | 20260507 20:53:26.674 | INFO | "huge" : "", | 20260507 20:53:26.674 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:26.674 | INFO | "malloc" : " 14 GB", | 20260507 20:53:26.674 | INFO | "pinned" : " 29 GB" | 20260507 20:53:26.675 | INFO | } | 20260507 20:53:26.675 | INFO | Perf report for the last 5.14032 seconds: | 20260507 20:53:26.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:26.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:53:26.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:53:26.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:26.675 | INFO | Uploaded 36GiB to GPU in 2.55324s (14.0997GiB/s) | 20260507 20:53:26.676 | INFO | Downloaded 34GiB to GPU in 2.43644s (13.9548GiB/s) | 20260507 20:53:31.815 | INFO | Analyzing chunk frames = [2518528,2519040) | 20260507 20:53:31.815 | INFO | MemInfo: { | 20260507 20:53:31.815 | INFO | "gpu" : " 17 GB", | 20260507 20:53:31.815 | INFO | "huge" : "", | 20260507 20:53:31.816 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:31.816 | INFO | "malloc" : " 14 GB", | 20260507 20:53:31.816 | INFO | "pinned" : " 29 GB" | 20260507 20:53:31.816 | INFO | } | 20260507 20:53:31.816 | INFO | Perf report for the last 5.14129 seconds: | 20260507 20:53:31.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:31.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 20:53:31.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 20:53:31.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:31.816 | INFO | Uploaded 36GiB to GPU in 2.55971s (14.0641GiB/s) | 20260507 20:53:31.816 | INFO | Downloaded 34GiB to GPU in 2.44491s (13.9064GiB/s) | 20260507 20:53:31.930 | INFO | Starting a round of Baz IO | 20260507 20:53:35.427 | INFO | Populating RTMetrics report | 20260507 20:53:35.427 | INFO | RTMetrics report populated | 20260507 20:53:35.427 | INFO | Opening tmp RTMetrics json file | 20260507 20:53:35.458 | INFO | Serializing RTMetrics to json | 20260507 20:53:35.511 | INFO | Serialization done, swapping tmp file now... | 20260507 20:53:35.512 | INFO | RTMetrics done | 20260507 20:53:35.534 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.604 seconds | 20260507 20:53:35.534 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.13% / 0.01% / 55.87% | 20260507 20:53:35.534 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.86337% / 2.38419e-05% / 3.97364e-05% | 20260507 20:53:35.534 | INFO | Baz IO overall: | 20260507 20:53:35.534 | INFO | BytesWritten : 1145337269403 | 20260507 20:53:35.534 | INFO | NumEvents : 555194395793 | 20260507 20:53:36.952 | INFO | Analyzing chunk frames = [2519040,2519552) | 20260507 20:53:36.952 | INFO | MemInfo: { | 20260507 20:53:36.952 | INFO | "gpu" : " 17 GB", | 20260507 20:53:36.952 | INFO | "huge" : "", | 20260507 20:53:36.952 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:36.952 | INFO | "malloc" : " 14 GB", | 20260507 20:53:36.952 | INFO | "pinned" : " 29 GB" | 20260507 20:53:36.952 | INFO | } | 20260507 20:53:36.952 | INFO | Perf report for the last 5.13651 seconds: | 20260507 20:53:36.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:36.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:36.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:36.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:36.952 | INFO | ANALYSIS : 69.3%, 2.9, 105.1ms, 0 | 20260507 20:53:36.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:36.953 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260507 20:53:36.953 | INFO | PRE_HQ : 68.4%, 1.9, 66.2ms, 0 | 20260507 20:53:36.953 | INFO | BAZWRITER : 69.7%, 4.2, 148.7ms, 0 | 20260507 20:53:36.953 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 20:53:36.953 | INFO | Uploaded 36GiB to GPU in 2.6013s (13.8393GiB/s) | 20260507 20:53:36.953 | INFO | Downloaded 38GiB to GPU in 2.72s (13.9706GiB/s) | 20260507 20:53:42.091 | INFO | Analyzing chunk frames = [2519552,2520064) | 20260507 20:53:42.091 | INFO | MemInfo: { | 20260507 20:53:42.091 | INFO | "gpu" : " 17 GB", | 20260507 20:53:42.091 | INFO | "huge" : "", | 20260507 20:53:42.091 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:42.091 | INFO | "malloc" : " 14 GB", | 20260507 20:53:42.091 | INFO | "pinned" : " 29 GB" | 20260507 20:53:42.091 | INFO | } | 20260507 20:53:42.091 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:53:42.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:42.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.091 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 20:53:42.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.091 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:53:42.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:42.092 | INFO | Uploaded 36GiB to GPU in 2.571s (14.0023GiB/s) | 20260507 20:53:42.092 | INFO | Downloaded 34GiB to GPU in 2.4711s (13.759GiB/s) | 20260507 20:53:47.231 | INFO | Analyzing chunk frames = [2520064,2520576) | 20260507 20:53:47.231 | INFO | MemInfo: { | 20260507 20:53:47.231 | INFO | "gpu" : " 17 GB", | 20260507 20:53:47.231 | INFO | "huge" : "", | 20260507 20:53:47.232 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:47.232 | INFO | "malloc" : " 14 GB", | 20260507 20:53:47.232 | INFO | "pinned" : " 29 GB" | 20260507 20:53:47.232 | INFO | } | 20260507 20:53:47.232 | INFO | Perf report for the last 5.14035 seconds: | 20260507 20:53:47.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:47.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260507 20:53:47.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:53:47.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:47.232 | INFO | Uploaded 36GiB to GPU in 2.60496s (13.8198GiB/s) | 20260507 20:53:47.232 | INFO | Downloaded 34GiB to GPU in 2.46442s (13.7964GiB/s) | 20260507 20:53:52.370 | INFO | Analyzing chunk frames = [2520576,2521088) | 20260507 20:53:52.370 | INFO | MemInfo: { | 20260507 20:53:52.370 | INFO | "gpu" : " 17 GB", | 20260507 20:53:52.370 | INFO | "huge" : "", | 20260507 20:53:52.370 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:52.370 | INFO | "malloc" : " 14 GB", | 20260507 20:53:52.370 | INFO | "pinned" : " 29 GB" | 20260507 20:53:52.370 | INFO | } | 20260507 20:53:52.370 | INFO | Perf report for the last 5.13847 seconds: | 20260507 20:53:52.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:52.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 20:53:52.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:53:52.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:52.370 | INFO | Uploaded 36GiB to GPU in 2.54877s (14.1245GiB/s) | 20260507 20:53:52.370 | INFO | Downloaded 34GiB to GPU in 2.48243s (13.6963GiB/s) | 20260507 20:53:57.512 | INFO | Analyzing chunk frames = [2521088,2521600) | 20260507 20:53:57.512 | INFO | MemInfo: { | 20260507 20:53:57.512 | INFO | "gpu" : " 17 GB", | 20260507 20:53:57.512 | INFO | "huge" : "", | 20260507 20:53:57.512 | INFO | "huge_shared" : " 29 GB", | 20260507 20:53:57.512 | INFO | "malloc" : " 14 GB", | 20260507 20:53:57.512 | INFO | "pinned" : " 29 GB" | 20260507 20:53:57.512 | INFO | } | 20260507 20:53:57.512 | INFO | Perf report for the last 5.14165 seconds: | 20260507 20:53:57.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:53:57.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:53:57.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 20:53:57.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:53:57.512 | INFO | Uploaded 36GiB to GPU in 2.37844s (15.136GiB/s) | 20260507 20:53:57.512 | INFO | Downloaded 34GiB to GPU in 2.47917s (13.7143GiB/s) | 20260507 20:54:02.652 | INFO | Analyzing chunk frames = [2521600,2522112) | 20260507 20:54:02.652 | INFO | MemInfo: { | 20260507 20:54:02.652 | INFO | "gpu" : " 17 GB", | 20260507 20:54:02.652 | INFO | "huge" : "", | 20260507 20:54:02.652 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:02.652 | INFO | "malloc" : " 14 GB", | 20260507 20:54:02.652 | INFO | "pinned" : " 29 GB" | 20260507 20:54:02.652 | INFO | } | 20260507 20:54:02.652 | INFO | Perf report for the last 5.13994 seconds: | 20260507 20:54:02.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:02.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 20:54:02.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:54:02.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:02.652 | INFO | Uploaded 36GiB to GPU in 2.40236s (14.9853GiB/s) | 20260507 20:54:02.652 | INFO | Downloaded 34GiB to GPU in 2.3917s (14.2158GiB/s) | 20260507 20:54:07.788 | INFO | Analyzing chunk frames = [2522112,2522624) | 20260507 20:54:07.789 | INFO | MemInfo: { | 20260507 20:54:07.789 | INFO | "gpu" : " 17 GB", | 20260507 20:54:07.789 | INFO | "huge" : "", | 20260507 20:54:07.789 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:07.789 | INFO | "malloc" : " 14 GB", | 20260507 20:54:07.789 | INFO | "pinned" : " 29 GB" | 20260507 20:54:07.789 | INFO | } | 20260507 20:54:07.789 | INFO | Perf report for the last 5.13719 seconds: | 20260507 20:54:07.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:07.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:54:07.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 20:54:07.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:07.789 | INFO | Uploaded 36GiB to GPU in 2.53416s (14.2059GiB/s) | 20260507 20:54:07.789 | INFO | Downloaded 34GiB to GPU in 2.45858s (13.8291GiB/s) | 20260507 20:54:12.927 | INFO | Analyzing chunk frames = [2522624,2523136) | 20260507 20:54:12.927 | INFO | MemInfo: { | 20260507 20:54:12.927 | INFO | "gpu" : " 17 GB", | 20260507 20:54:12.927 | INFO | "huge" : "", | 20260507 20:54:12.927 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:12.927 | INFO | "malloc" : " 14 GB", | 20260507 20:54:12.927 | INFO | "pinned" : " 29 GB" | 20260507 20:54:12.928 | INFO | } | 20260507 20:54:12.928 | INFO | Perf report for the last 5.13856 seconds: | 20260507 20:54:12.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:12.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 20:54:12.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 20:54:12.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:12.928 | INFO | Uploaded 36GiB to GPU in 2.5378s (14.1855GiB/s) | 20260507 20:54:12.928 | INFO | Downloaded 34GiB to GPU in 2.43834s (13.9439GiB/s) | 20260507 20:54:12.928 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:54:12.928 | INFO | Total mem usage: 16.4 GiB | 20260507 20:54:12.928 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:54:12.928 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -6.54%) | 20260507 20:54:12.928 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:54:12.928 | INFO | High Water Breakdown: | 20260507 20:54:12.928 | INFO | 60.5% : 9.31 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:54:12.928 | INFO | 39.4% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:54:12.928 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:54:12.928 | INFO | ------------------------------------------ | 20260507 20:54:12.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:54:12.928 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:54:12.928 | INFO | Total mem usage: 26.5 GiB | 20260507 20:54:12.928 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:54:12.928 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:54:12.928 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:54:12.929 | INFO | High Water Breakdown: | 20260507 20:54:12.929 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:54:12.929 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:54:12.929 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:54:12.929 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:54:12.929 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:54:12.929 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:54:12.929 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:54:12.929 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:54:12.929 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:54:12.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:54:12.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:54:12.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:54:12.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:54:12.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:54:12.929 | INFO | ------------------------------------------ | 20260507 20:54:12.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:54:12.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:54:12.929 | INFO | Total mem usage: 27.8 GiB | 20260507 20:54:12.929 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:54:12.929 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:54:12.929 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:54:12.929 | INFO | High Water Breakdown: | 20260507 20:54:12.929 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:54:12.929 | INFO | ------------------------------------------ | 20260507 20:54:12.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:54:12.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:54:12.929 | INFO | Total mem usage: 15.7 GiB | 20260507 20:54:12.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:54:12.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:54:12.929 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:54:12.929 | INFO | High Water Breakdown: | 20260507 20:54:12.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:54:12.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:54:12.929 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:54:12.929 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:54:12.929 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:54:12.929 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:54:12.929 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:54:12.929 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:54:12.929 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:54:12.929 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:54:12.929 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:54:12.929 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:54:12.929 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:54:12.929 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:54:12.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:54:12.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:54:12.929 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:54:12.929 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:54:12.929 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:54:12.929 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:54:12.929 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:54:12.929 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:54:12.929 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:54:12.929 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:54:12.929 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:54:12.929 | INFO | ------------------------------------------ | 20260507 20:54:12.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:54:12.929 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:54:12.929 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:54:12.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:54:12.929 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.41e+01 / 2.06e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.03e-01, 1.19e-02 / 7.94e-03 | 20260507 20:54:12.929 | INFO | PerfReport: 12.15%, Baseline -- 4.02e+00 / 4.47e+00, 6.20e-01 / 2.43e-01 | 20260507 20:54:12.929 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.08e-01 / 1.09e-01 | 20260507 20:54:12.929 | INFO | PerfReport: 16.03%, DME -- 4.24e+01 / 4.58e+01, 4.21e+00 / 2.18e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 55.85%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.64e+00 / 1.68e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.52e+00 / 2.85e+00, 4.46e-01 / 2.21e-01 | 20260507 20:54:12.929 | INFO | PerfReport: 3.57%, Metrics -- 1.18e+00 / 1.37e+00, 2.98e-01 / 1.54e-01 | 20260507 20:54:12.929 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.68e-01 / 4.02e-01, 6.03e-02 / 3.39e-02 | 20260507 20:54:12.929 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:54:12.929 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:54:12.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:54:12.929 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 24.53%, Upload -- 2.62e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 24.28%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 30.93%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 20:54:12.929 | INFO | PerfReport: 2.21%, LockUpload -- 2.36e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260507 20:54:12.930 | INFO | PerfReport: 2.46%, LockDownload -- 2.63e+00 / 0.00e+00, 3.75e+00 / 0.00e+00 | 20260507 20:54:12.930 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 20:54:12.930 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 20:54:13.040 | INFO | Starting a round of Baz IO | 20260507 20:54:16.518 | INFO | Populating RTMetrics report | 20260507 20:54:16.519 | INFO | RTMetrics report populated | 20260507 20:54:16.519 | INFO | Opening tmp RTMetrics json file | 20260507 20:54:16.548 | INFO | Serializing RTMetrics to json | 20260507 20:54:16.563 | INFO | Serialization done, swapping tmp file now... | 20260507 20:54:16.589 | INFO | RTMetrics done | 20260507 20:54:16.643 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.603 seconds | 20260507 20:54:16.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.12% / 0.01% / 55.87% | 20260507 20:54:16.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85719% / 1.19209e-05% / 4.76837e-05% | 20260507 20:54:16.643 | INFO | Baz IO overall: | 20260507 20:54:16.643 | INFO | BytesWritten : 1147237391119 | 20260507 20:54:16.643 | INFO | NumEvents : 556113803102 | 20260507 20:54:18.067 | INFO | Analyzing chunk frames = [2523136,2523648) | 20260507 20:54:18.067 | INFO | MemInfo: { | 20260507 20:54:18.067 | INFO | "gpu" : " 17 GB", | 20260507 20:54:18.067 | INFO | "huge" : "", | 20260507 20:54:18.067 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:18.067 | INFO | "malloc" : " 14 GB", | 20260507 20:54:18.067 | INFO | "pinned" : " 29 GB" | 20260507 20:54:18.067 | INFO | } | 20260507 20:54:18.067 | INFO | Perf report for the last 5.13922 seconds: | 20260507 20:54:18.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:18.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:18.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:18.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:18.067 | INFO | ANALYSIS : 69.1%, 2.9, 105.1ms, 0 | 20260507 20:54:18.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:18.067 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 20:54:18.067 | INFO | PRE_HQ : 68.2%, 1.8, 63.7ms, 0 | 20260507 20:54:18.067 | INFO | BAZWRITER : 69.7%, 3.8, 137.7ms, 0 | 20260507 20:54:18.067 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 20:54:18.067 | INFO | Uploaded 36GiB to GPU in 2.58498s (13.9266GiB/s) | 20260507 20:54:18.067 | INFO | Downloaded 38GiB to GPU in 2.71908s (13.9753GiB/s) | 20260507 20:54:23.206 | INFO | Analyzing chunk frames = [2523648,2524160) | 20260507 20:54:23.206 | INFO | MemInfo: { | 20260507 20:54:23.206 | INFO | "gpu" : " 17 GB", | 20260507 20:54:23.206 | INFO | "huge" : "", | 20260507 20:54:23.206 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:23.206 | INFO | "malloc" : " 14 GB", | 20260507 20:54:23.206 | INFO | "pinned" : " 29 GB" | 20260507 20:54:23.206 | INFO | } | 20260507 20:54:23.207 | INFO | Perf report for the last 5.13969 seconds: | 20260507 20:54:23.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:23.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 20:54:23.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:54:23.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:23.207 | INFO | Uploaded 36GiB to GPU in 2.55629s (14.0829GiB/s) | 20260507 20:54:23.207 | INFO | Downloaded 34GiB to GPU in 2.49038s (13.6525GiB/s) | 20260507 20:54:28.346 | INFO | Analyzing chunk frames = [2524160,2524672) | 20260507 20:54:28.346 | INFO | MemInfo: { | 20260507 20:54:28.346 | INFO | "gpu" : " 17 GB", | 20260507 20:54:28.346 | INFO | "huge" : "", | 20260507 20:54:28.346 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:28.346 | INFO | "malloc" : " 14 GB", | 20260507 20:54:28.346 | INFO | "pinned" : " 29 GB" | 20260507 20:54:28.346 | INFO | } | 20260507 20:54:28.346 | INFO | Perf report for the last 5.13943 seconds: | 20260507 20:54:28.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:28.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.346 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:54:28.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.346 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:54:28.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:28.347 | INFO | Uploaded 36GiB to GPU in 2.54869s (14.1249GiB/s) | 20260507 20:54:28.347 | INFO | Downloaded 34GiB to GPU in 2.47489s (13.738GiB/s) | 20260507 20:54:33.486 | INFO | Analyzing chunk frames = [2524672,2525184) | 20260507 20:54:33.486 | INFO | MemInfo: { | 20260507 20:54:33.486 | INFO | "gpu" : " 17 GB", | 20260507 20:54:33.486 | INFO | "huge" : "", | 20260507 20:54:33.486 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:33.486 | INFO | "malloc" : " 14 GB", | 20260507 20:54:33.487 | INFO | "pinned" : " 29 GB" | 20260507 20:54:33.487 | INFO | } | 20260507 20:54:33.487 | INFO | Perf report for the last 5.14073 seconds: | 20260507 20:54:33.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:33.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 20:54:33.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:54:33.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:33.487 | INFO | Uploaded 36GiB to GPU in 2.45381s (14.671GiB/s) | 20260507 20:54:33.487 | INFO | Downloaded 34GiB to GPU in 2.48934s (13.6583GiB/s) | 20260507 20:54:38.625 | INFO | Analyzing chunk frames = [2525184,2525696) | 20260507 20:54:38.625 | INFO | MemInfo: { | 20260507 20:54:38.625 | INFO | "gpu" : " 17 GB", | 20260507 20:54:38.625 | INFO | "huge" : "", | 20260507 20:54:38.625 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:38.625 | INFO | "malloc" : " 14 GB", | 20260507 20:54:38.625 | INFO | "pinned" : " 29 GB" | 20260507 20:54:38.625 | INFO | } | 20260507 20:54:38.626 | INFO | Perf report for the last 5.13879 seconds: | 20260507 20:54:38.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:38.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 20:54:38.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 20:54:38.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:38.626 | INFO | Uploaded 36GiB to GPU in 2.38831s (15.0734GiB/s) | 20260507 20:54:38.626 | INFO | Downloaded 34GiB to GPU in 2.48282s (13.6941GiB/s) | 20260507 20:54:43.764 | INFO | Analyzing chunk frames = [2525696,2526208) | 20260507 20:54:43.764 | INFO | MemInfo: { | 20260507 20:54:43.764 | INFO | "gpu" : " 17 GB", | 20260507 20:54:43.764 | INFO | "huge" : "", | 20260507 20:54:43.764 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:43.764 | INFO | "malloc" : " 14 GB", | 20260507 20:54:43.764 | INFO | "pinned" : " 29 GB" | 20260507 20:54:43.764 | INFO | } | 20260507 20:54:43.764 | INFO | Perf report for the last 5.1388 seconds: | 20260507 20:54:43.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:43.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.764 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 20:54:43.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.764 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:54:43.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:43.765 | INFO | Uploaded 36GiB to GPU in 2.43352s (14.7934GiB/s) | 20260507 20:54:43.765 | INFO | Downloaded 34GiB to GPU in 2.38019s (14.2846GiB/s) | 20260507 20:54:48.903 | INFO | Analyzing chunk frames = [2526208,2526720) | 20260507 20:54:48.903 | INFO | MemInfo: { | 20260507 20:54:48.903 | INFO | "gpu" : " 17 GB", | 20260507 20:54:48.903 | INFO | "huge" : "", | 20260507 20:54:48.903 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:48.903 | INFO | "malloc" : " 14 GB", | 20260507 20:54:48.903 | INFO | "pinned" : " 29 GB" | 20260507 20:54:48.903 | INFO | } | 20260507 20:54:48.903 | INFO | Perf report for the last 5.13905 seconds: | 20260507 20:54:48.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:48.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.903 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260507 20:54:48.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.904 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 20:54:48.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:48.904 | INFO | Uploaded 36GiB to GPU in 2.54134s (14.1658GiB/s) | 20260507 20:54:48.904 | INFO | Downloaded 34GiB to GPU in 2.43856s (13.9426GiB/s) | 20260507 20:54:54.042 | INFO | Analyzing chunk frames = [2526720,2527232) | 20260507 20:54:54.042 | INFO | MemInfo: { | 20260507 20:54:54.042 | INFO | "gpu" : " 17 GB", | 20260507 20:54:54.042 | INFO | "huge" : "", | 20260507 20:54:54.042 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:54.042 | INFO | "malloc" : " 14 GB", | 20260507 20:54:54.042 | INFO | "pinned" : " 29 GB" | 20260507 20:54:54.042 | INFO | } | 20260507 20:54:54.042 | INFO | Perf report for the last 5.13869 seconds: | 20260507 20:54:54.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:54.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.042 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:54:54.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.042 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 20:54:54.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:54.043 | INFO | Uploaded 36GiB to GPU in 2.521s (14.28GiB/s) | 20260507 20:54:54.043 | INFO | Downloaded 34GiB to GPU in 2.42114s (14.043GiB/s) | 20260507 20:54:54.140 | INFO | Starting a round of Baz IO | 20260507 20:54:57.620 | INFO | Populating RTMetrics report | 20260507 20:54:57.620 | INFO | RTMetrics report populated | 20260507 20:54:57.620 | INFO | Opening tmp RTMetrics json file | 20260507 20:54:57.643 | INFO | Serializing RTMetrics to json | 20260507 20:54:57.645 | INFO | Serialization done, swapping tmp file now... | 20260507 20:54:57.646 | INFO | RTMetrics done | 20260507 20:54:57.722 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.582 seconds | 20260507 20:54:57.722 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.11% / 0.01% / 55.88% | 20260507 20:54:57.722 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.86225% / 0% / 2.78155e-05% | 20260507 20:54:57.722 | INFO | Baz IO overall: | 20260507 20:54:57.722 | INFO | BytesWritten : 1149141002509 | 20260507 20:54:57.722 | INFO | NumEvents : 557036136723 | 20260507 20:54:59.182 | INFO | Analyzing chunk frames = [2527232,2527744) | 20260507 20:54:59.182 | INFO | MemInfo: { | 20260507 20:54:59.182 | INFO | "gpu" : " 17 GB", | 20260507 20:54:59.182 | INFO | "huge" : "", | 20260507 20:54:59.182 | INFO | "huge_shared" : " 29 GB", | 20260507 20:54:59.182 | INFO | "malloc" : " 14 GB", | 20260507 20:54:59.182 | INFO | "pinned" : " 29 GB" | 20260507 20:54:59.182 | INFO | } | 20260507 20:54:59.182 | INFO | Perf report for the last 5.13962 seconds: | 20260507 20:54:59.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:54:59.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:59.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:59.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:59.182 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:54:59.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:54:59.182 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 20:54:59.182 | INFO | PRE_HQ : 66.0%, 1.7, 57.9ms, 0 | 20260507 20:54:59.182 | INFO | BAZWRITER : 69.3%, 4.3, 153.4ms, 0 | 20260507 20:54:59.182 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 20:54:59.182 | INFO | Uploaded 36GiB to GPU in 2.54628s (14.1383GiB/s) | 20260507 20:54:59.182 | INFO | Downloaded 38GiB to GPU in 2.684s (14.158GiB/s) | 20260507 20:55:04.322 | INFO | Analyzing chunk frames = [2527744,2528256) | 20260507 20:55:04.323 | INFO | MemInfo: { | 20260507 20:55:04.323 | INFO | "gpu" : " 17 GB", | 20260507 20:55:04.323 | INFO | "huge" : "", | 20260507 20:55:04.323 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:04.323 | INFO | "malloc" : " 14 GB", | 20260507 20:55:04.323 | INFO | "pinned" : " 29 GB" | 20260507 20:55:04.323 | INFO | } | 20260507 20:55:04.323 | INFO | Perf report for the last 5.14104 seconds: | 20260507 20:55:04.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:04.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.323 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 20:55:04.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.323 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 20:55:04.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:04.324 | INFO | Uploaded 36GiB to GPU in 2.57376s (13.9873GiB/s) | 20260507 20:55:04.324 | INFO | Downloaded 34GiB to GPU in 2.49067s (13.651GiB/s) | 20260507 20:55:09.461 | INFO | Analyzing chunk frames = [2528256,2528768) | 20260507 20:55:09.461 | INFO | MemInfo: { | 20260507 20:55:09.461 | INFO | "gpu" : " 17 GB", | 20260507 20:55:09.461 | INFO | "huge" : "", | 20260507 20:55:09.461 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:09.461 | INFO | "malloc" : " 14 GB", | 20260507 20:55:09.461 | INFO | "pinned" : " 29 GB" | 20260507 20:55:09.461 | INFO | } | 20260507 20:55:09.461 | INFO | Perf report for the last 5.13832 seconds: | 20260507 20:55:09.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:09.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.461 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 20:55:09.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.461 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:55:09.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:09.462 | INFO | Uploaded 36GiB to GPU in 2.53307s (14.212GiB/s) | 20260507 20:55:09.462 | INFO | Downloaded 34GiB to GPU in 2.4781s (13.7202GiB/s) | 20260507 20:55:14.601 | INFO | Analyzing chunk frames = [2528768,2529280) | 20260507 20:55:14.601 | INFO | MemInfo: { | 20260507 20:55:14.601 | INFO | "gpu" : " 17 GB", | 20260507 20:55:14.601 | INFO | "huge" : "", | 20260507 20:55:14.601 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:14.601 | INFO | "malloc" : " 14 GB", | 20260507 20:55:14.601 | INFO | "pinned" : " 29 GB" | 20260507 20:55:14.601 | INFO | } | 20260507 20:55:14.601 | INFO | Perf report for the last 5.13989 seconds: | 20260507 20:55:14.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:14.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 20:55:14.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:55:14.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:14.601 | INFO | Uploaded 36GiB to GPU in 2.53075s (14.225GiB/s) | 20260507 20:55:14.601 | INFO | Downloaded 34GiB to GPU in 2.47728s (13.7247GiB/s) | 20260507 20:55:19.740 | INFO | Analyzing chunk frames = [2529280,2529792) | 20260507 20:55:19.740 | INFO | MemInfo: { | 20260507 20:55:19.740 | INFO | "gpu" : " 17 GB", | 20260507 20:55:19.740 | INFO | "huge" : "", | 20260507 20:55:19.740 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:19.741 | INFO | "malloc" : " 14 GB", | 20260507 20:55:19.741 | INFO | "pinned" : " 29 GB" | 20260507 20:55:19.741 | INFO | } | 20260507 20:55:19.741 | INFO | Perf report for the last 5.13957 seconds: | 20260507 20:55:19.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:19.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 20:55:19.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:55:19.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:19.741 | INFO | Uploaded 36GiB to GPU in 2.44348s (14.7331GiB/s) | 20260507 20:55:19.741 | INFO | Downloaded 34GiB to GPU in 2.48378s (13.6888GiB/s) | 20260507 20:55:24.879 | INFO | Analyzing chunk frames = [2529792,2530304) | 20260507 20:55:24.879 | INFO | MemInfo: { | 20260507 20:55:24.879 | INFO | "gpu" : " 17 GB", | 20260507 20:55:24.879 | INFO | "huge" : "", | 20260507 20:55:24.880 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:24.880 | INFO | "malloc" : " 14 GB", | 20260507 20:55:24.880 | INFO | "pinned" : " 29 GB" | 20260507 20:55:24.880 | INFO | } | 20260507 20:55:24.880 | INFO | Perf report for the last 5.13909 seconds: | 20260507 20:55:24.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:24.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 20:55:24.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 20:55:24.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:24.880 | INFO | Uploaded 36GiB to GPU in 2.40384s (14.976GiB/s) | 20260507 20:55:24.880 | INFO | Downloaded 34GiB to GPU in 2.3899s (14.2265GiB/s) | 20260507 20:55:30.018 | INFO | Analyzing chunk frames = [2530304,2530816) | 20260507 20:55:30.018 | INFO | MemInfo: { | 20260507 20:55:30.018 | INFO | "gpu" : " 17 GB", | 20260507 20:55:30.018 | INFO | "huge" : "", | 20260507 20:55:30.018 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:30.018 | INFO | "malloc" : " 14 GB", | 20260507 20:55:30.018 | INFO | "pinned" : " 29 GB" | 20260507 20:55:30.018 | INFO | } | 20260507 20:55:30.018 | INFO | Perf report for the last 5.13862 seconds: | 20260507 20:55:30.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:30.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:55:30.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 20:55:30.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:30.018 | INFO | Uploaded 36GiB to GPU in 2.54646s (14.1373GiB/s) | 20260507 20:55:30.019 | INFO | Downloaded 34GiB to GPU in 2.44966s (13.8795GiB/s) | 20260507 20:55:35.158 | INFO | Analyzing chunk frames = [2530816,2531328) | 20260507 20:55:35.158 | INFO | MemInfo: { | 20260507 20:55:35.158 | INFO | "gpu" : " 17 GB", | 20260507 20:55:35.158 | INFO | "huge" : "", | 20260507 20:55:35.158 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:35.158 | INFO | "malloc" : " 14 GB", | 20260507 20:55:35.158 | INFO | "pinned" : " 29 GB" | 20260507 20:55:35.158 | INFO | } | 20260507 20:55:35.158 | INFO | Perf report for the last 5.13985 seconds: | 20260507 20:55:35.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:35.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.158 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:55:35.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.158 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 20:55:35.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:35.159 | INFO | Uploaded 36GiB to GPU in 2.49134s (14.4501GiB/s) | 20260507 20:55:35.159 | INFO | Downloaded 34GiB to GPU in 2.43431s (13.967GiB/s) | 20260507 20:55:35.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:55:35.159 | INFO | Total mem usage: 16.4 GiB | 20260507 20:55:35.159 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:55:35.159 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -4.29%) | 20260507 20:55:35.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:55:35.159 | INFO | High Water Breakdown: | 20260507 20:55:35.159 | INFO | 60.8% : 9.57 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:55:35.159 | INFO | 39% : 6.14 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:55:35.159 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:55:35.159 | INFO | ------------------------------------------ | 20260507 20:55:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:55:35.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:55:35.159 | INFO | Total mem usage: 26.5 GiB | 20260507 20:55:35.159 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:55:35.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:55:35.159 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:55:35.159 | INFO | High Water Breakdown: | 20260507 20:55:35.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:55:35.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:55:35.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:55:35.159 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:55:35.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:55:35.159 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:55:35.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:55:35.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:55:35.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:55:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:55:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:55:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:55:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:55:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:55:35.159 | INFO | ------------------------------------------ | 20260507 20:55:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:55:35.159 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:55:35.159 | INFO | Total mem usage: 27.8 GiB | 20260507 20:55:35.159 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:55:35.159 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:55:35.159 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:55:35.159 | INFO | High Water Breakdown: | 20260507 20:55:35.159 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:55:35.159 | INFO | ------------------------------------------ | 20260507 20:55:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:55:35.159 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:55:35.159 | INFO | Total mem usage: 15.7 GiB | 20260507 20:55:35.159 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:55:35.159 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:55:35.159 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:55:35.159 | INFO | High Water Breakdown: | 20260507 20:55:35.159 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:55:35.159 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:55:35.159 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:55:35.159 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:55:35.159 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:55:35.159 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:55:35.159 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:55:35.159 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:55:35.159 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:55:35.159 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:55:35.159 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:55:35.159 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:55:35.159 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:55:35.159 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:55:35.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:55:35.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:55:35.159 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:55:35.159 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:55:35.159 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:55:35.159 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:55:35.159 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:55:35.159 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:55:35.159 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:55:35.159 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:55:35.159 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:55:35.159 | INFO | ------------------------------------------ | 20260507 20:55:35.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:55:35.159 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:55:35.159 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:55:35.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:55:35.159 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.63e+01, 1.42e+01 / 1.89e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.00e-01, 1.10e-02 / 4.13e-03 | 20260507 20:55:35.160 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.48e+00, 5.99e-01 / 2.62e-01 | 20260507 20:55:35.160 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.51e+00, 2.11e-01 / 1.01e-01 | 20260507 20:55:35.160 | INFO | PerfReport: 16.05%, DME -- 4.24e+01 / 4.61e+01, 4.35e+00 / 2.01e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 55.86%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.61e+00 / 1.65e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.43e-01 / 2.38e-01 | 20260507 20:55:35.160 | INFO | PerfReport: 3.58%, Metrics -- 1.18e+00 / 1.39e+00, 3.01e-01 / 1.39e-01 | 20260507 20:55:35.160 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 4.00e-01, 5.97e-02 / 3.13e-02 | 20260507 20:55:35.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:55:35.160 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:55:35.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:55:35.160 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 30.90%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 2.53%, LockDownload -- 2.71e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 15.77%, LockCompute -- 1.69e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 20:55:35.160 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260507 20:55:35.255 | INFO | Starting a round of Baz IO | 20260507 20:55:38.719 | INFO | Populating RTMetrics report | 20260507 20:55:38.719 | INFO | RTMetrics report populated | 20260507 20:55:38.719 | INFO | Opening tmp RTMetrics json file | 20260507 20:55:38.748 | INFO | Serializing RTMetrics to json | 20260507 20:55:38.760 | INFO | Serialization done, swapping tmp file now... | 20260507 20:55:38.806 | INFO | RTMetrics done | 20260507 20:55:38.855 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.599 seconds | 20260507 20:55:38.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.10% / 0.01% / 55.89% | 20260507 20:55:38.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85144% / 1.58946e-05% / 4.76837e-05% | 20260507 20:55:38.855 | INFO | Baz IO overall: | 20260507 20:55:38.855 | INFO | BytesWritten : 1151041492810 | 20260507 20:55:38.855 | INFO | NumEvents : 557955957927 | 20260507 20:55:40.297 | INFO | Analyzing chunk frames = [2531328,2531840) | 20260507 20:55:40.297 | INFO | MemInfo: { | 20260507 20:55:40.297 | INFO | "gpu" : " 17 GB", | 20260507 20:55:40.297 | INFO | "huge" : "", | 20260507 20:55:40.297 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:40.297 | INFO | "malloc" : " 14 GB", | 20260507 20:55:40.297 | INFO | "pinned" : " 29 GB" | 20260507 20:55:40.297 | INFO | } | 20260507 20:55:40.297 | INFO | Perf report for the last 5.13847 seconds: | 20260507 20:55:40.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:40.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:40.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:40.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:40.297 | INFO | ANALYSIS : 68.6%, 3.0, 106.9ms, 0 | 20260507 20:55:40.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:40.297 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 20:55:40.297 | INFO | PRE_HQ : 66.0%, 1.7, 58.0ms, 0 | 20260507 20:55:40.297 | INFO | BAZWRITER : 69.7%, 3.6, 128.0ms, 0 | 20260507 20:55:40.297 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 20:55:40.297 | INFO | Uploaded 36GiB to GPU in 2.5314s (14.2214GiB/s) | 20260507 20:55:40.297 | INFO | Downloaded 38GiB to GPU in 2.68788s (14.1375GiB/s) | 20260507 20:55:45.436 | INFO | Analyzing chunk frames = [2531840,2532352) | 20260507 20:55:45.436 | INFO | MemInfo: { | 20260507 20:55:45.436 | INFO | "gpu" : " 17 GB", | 20260507 20:55:45.436 | INFO | "huge" : "", | 20260507 20:55:45.436 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:45.436 | INFO | "malloc" : " 14 GB", | 20260507 20:55:45.437 | INFO | "pinned" : " 29 GB" | 20260507 20:55:45.437 | INFO | } | 20260507 20:55:45.437 | INFO | Perf report for the last 5.13996 seconds: | 20260507 20:55:45.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:45.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 20:55:45.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:55:45.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:45.437 | INFO | Uploaded 36GiB to GPU in 2.5422s (14.1609GiB/s) | 20260507 20:55:45.437 | INFO | Downloaded 34GiB to GPU in 2.49921s (13.6043GiB/s) | 20260507 20:55:50.577 | INFO | Analyzing chunk frames = [2532352,2532864) | 20260507 20:55:50.577 | INFO | MemInfo: { | 20260507 20:55:50.577 | INFO | "gpu" : " 17 GB", | 20260507 20:55:50.577 | INFO | "huge" : "", | 20260507 20:55:50.577 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:50.577 | INFO | "malloc" : " 14 GB", | 20260507 20:55:50.577 | INFO | "pinned" : " 29 GB" | 20260507 20:55:50.577 | INFO | } | 20260507 20:55:50.578 | INFO | Perf report for the last 5.14083 seconds: | 20260507 20:55:50.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:50.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:55:50.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:55:50.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:50.578 | INFO | Uploaded 36GiB to GPU in 2.47502s (14.5454GiB/s) | 20260507 20:55:50.578 | INFO | Downloaded 34GiB to GPU in 2.48593s (13.677GiB/s) | 20260507 20:55:55.716 | INFO | Analyzing chunk frames = [2532864,2533376) | 20260507 20:55:55.716 | INFO | MemInfo: { | 20260507 20:55:55.716 | INFO | "gpu" : " 17 GB", | 20260507 20:55:55.716 | INFO | "huge" : "", | 20260507 20:55:55.716 | INFO | "huge_shared" : " 29 GB", | 20260507 20:55:55.716 | INFO | "malloc" : " 14 GB", | 20260507 20:55:55.716 | INFO | "pinned" : " 29 GB" | 20260507 20:55:55.716 | INFO | } | 20260507 20:55:55.716 | INFO | Perf report for the last 5.13834 seconds: | 20260507 20:55:55.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:55:55.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 20:55:55.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:55:55.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:55:55.716 | INFO | Uploaded 36GiB to GPU in 2.51725s (14.3013GiB/s) | 20260507 20:55:55.716 | INFO | Downloaded 34GiB to GPU in 2.49245s (13.6412GiB/s) | 20260507 20:56:00.855 | INFO | Analyzing chunk frames = [2533376,2533888) | 20260507 20:56:00.855 | INFO | MemInfo: { | 20260507 20:56:00.855 | INFO | "gpu" : " 17 GB", | 20260507 20:56:00.855 | INFO | "huge" : "", | 20260507 20:56:00.855 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:00.855 | INFO | "malloc" : " 14 GB", | 20260507 20:56:00.855 | INFO | "pinned" : " 29 GB" | 20260507 20:56:00.855 | INFO | } | 20260507 20:56:00.855 | INFO | Perf report for the last 5.13892 seconds: | 20260507 20:56:00.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:00.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 20:56:00.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 20:56:00.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:00.855 | INFO | Uploaded 36GiB to GPU in 2.40116s (14.9927GiB/s) | 20260507 20:56:00.855 | INFO | Downloaded 34GiB to GPU in 2.49593s (13.6222GiB/s) | 20260507 20:56:05.994 | INFO | Analyzing chunk frames = [2533888,2534400) | 20260507 20:56:05.994 | INFO | MemInfo: { | 20260507 20:56:05.994 | INFO | "gpu" : " 17 GB", | 20260507 20:56:05.994 | INFO | "huge" : "", | 20260507 20:56:05.994 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:05.994 | INFO | "malloc" : " 14 GB", | 20260507 20:56:05.994 | INFO | "pinned" : " 29 GB" | 20260507 20:56:05.994 | INFO | } | 20260507 20:56:05.994 | INFO | Perf report for the last 5.13919 seconds: | 20260507 20:56:05.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:05.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 20:56:05.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:56:05.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:05.994 | INFO | Uploaded 36GiB to GPU in 2.5205s (14.2829GiB/s) | 20260507 20:56:05.994 | INFO | Downloaded 34GiB to GPU in 2.41072s (14.1036GiB/s) | 20260507 20:56:11.133 | INFO | Analyzing chunk frames = [2534400,2534912) | 20260507 20:56:11.133 | INFO | MemInfo: { | 20260507 20:56:11.133 | INFO | "gpu" : " 17 GB", | 20260507 20:56:11.133 | INFO | "huge" : "", | 20260507 20:56:11.133 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:11.134 | INFO | "malloc" : " 14 GB", | 20260507 20:56:11.134 | INFO | "pinned" : " 29 GB" | 20260507 20:56:11.134 | INFO | } | 20260507 20:56:11.134 | INFO | Perf report for the last 5.13955 seconds: | 20260507 20:56:11.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:11.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 20:56:11.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 20:56:11.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:11.134 | INFO | Uploaded 36GiB to GPU in 2.51003s (14.3425GiB/s) | 20260507 20:56:11.134 | INFO | Downloaded 34GiB to GPU in 2.4292s (13.9964GiB/s) | 20260507 20:56:16.272 | INFO | Analyzing chunk frames = [2534912,2535424) | 20260507 20:56:16.272 | INFO | MemInfo: { | 20260507 20:56:16.272 | INFO | "gpu" : " 17 GB", | 20260507 20:56:16.272 | INFO | "huge" : "", | 20260507 20:56:16.272 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:16.272 | INFO | "malloc" : " 14 GB", | 20260507 20:56:16.272 | INFO | "pinned" : " 29 GB" | 20260507 20:56:16.272 | INFO | } | 20260507 20:56:16.272 | INFO | Perf report for the last 5.13869 seconds: | 20260507 20:56:16.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:16.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.272 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 20:56:16.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.272 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 20:56:16.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:16.273 | INFO | Uploaded 36GiB to GPU in 2.51985s (14.2866GiB/s) | 20260507 20:56:16.273 | INFO | Downloaded 34GiB to GPU in 2.41199s (14.0963GiB/s) | 20260507 20:56:16.369 | INFO | Starting a round of Baz IO | 20260507 20:56:19.853 | INFO | Populating RTMetrics report | 20260507 20:56:19.854 | INFO | RTMetrics report populated | 20260507 20:56:19.854 | INFO | Opening tmp RTMetrics json file | 20260507 20:56:19.877 | INFO | Serializing RTMetrics to json | 20260507 20:56:19.885 | INFO | Serialization done, swapping tmp file now... | 20260507 20:56:19.889 | INFO | RTMetrics done | 20260507 20:56:19.944 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.574 seconds | 20260507 20:56:19.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.09% / 0.01% / 55.90% | 20260507 20:56:19.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85018% / 1.58946e-05% / 3.97364e-05% | 20260507 20:56:19.944 | INFO | Baz IO overall: | 20260507 20:56:19.944 | INFO | BytesWritten : 1152942022372 | 20260507 20:56:19.944 | INFO | NumEvents : 558875804283 | 20260507 20:56:21.412 | INFO | Analyzing chunk frames = [2535424,2535936) | 20260507 20:56:21.412 | INFO | MemInfo: { | 20260507 20:56:21.412 | INFO | "gpu" : " 17 GB", | 20260507 20:56:21.413 | INFO | "huge" : "", | 20260507 20:56:21.413 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:21.413 | INFO | "malloc" : " 14 GB", | 20260507 20:56:21.413 | INFO | "pinned" : " 29 GB" | 20260507 20:56:21.413 | INFO | } | 20260507 20:56:21.413 | INFO | Perf report for the last 5.1404 seconds: | 20260507 20:56:21.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:21.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:21.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:21.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:21.413 | INFO | ANALYSIS : 68.7%, 3.0, 106.9ms, 0 | 20260507 20:56:21.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:21.413 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 20:56:21.413 | INFO | PRE_HQ : 66.1%, 1.7, 58.0ms, 0 | 20260507 20:56:21.413 | INFO | BAZWRITER : 69.2%, 3.8, 135.6ms, 0 | 20260507 20:56:21.413 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 20:56:21.413 | INFO | Uploaded 36GiB to GPU in 2.51883s (14.2923GiB/s) | 20260507 20:56:21.413 | INFO | Downloaded 38GiB to GPU in 2.68027s (14.1777GiB/s) | 20260507 20:56:26.553 | INFO | Analyzing chunk frames = [2535936,2536448) | 20260507 20:56:26.553 | INFO | MemInfo: { | 20260507 20:56:26.553 | INFO | "gpu" : " 17 GB", | 20260507 20:56:26.553 | INFO | "huge" : "", | 20260507 20:56:26.553 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:26.553 | INFO | "malloc" : " 14 GB", | 20260507 20:56:26.553 | INFO | "pinned" : " 29 GB" | 20260507 20:56:26.553 | INFO | } | 20260507 20:56:26.553 | INFO | Perf report for the last 5.14051 seconds: | 20260507 20:56:26.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:26.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.553 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:56:26.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.553 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 20:56:26.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:26.554 | INFO | Uploaded 36GiB to GPU in 2.55863s (14.07GiB/s) | 20260507 20:56:26.554 | INFO | Downloaded 34GiB to GPU in 2.48985s (13.6555GiB/s) | 20260507 20:56:31.691 | INFO | Analyzing chunk frames = [2536448,2536960) | 20260507 20:56:31.691 | INFO | MemInfo: { | 20260507 20:56:31.691 | INFO | "gpu" : " 17 GB", | 20260507 20:56:31.692 | INFO | "huge" : "", | 20260507 20:56:31.692 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:31.692 | INFO | "malloc" : " 14 GB", | 20260507 20:56:31.692 | INFO | "pinned" : " 29 GB" | 20260507 20:56:31.692 | INFO | } | 20260507 20:56:31.692 | INFO | Perf report for the last 5.13842 seconds: | 20260507 20:56:31.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:31.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:56:31.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 20:56:31.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:31.692 | INFO | Uploaded 36GiB to GPU in 2.47236s (14.561GiB/s) | 20260507 20:56:31.692 | INFO | Downloaded 34GiB to GPU in 2.48271s (13.6947GiB/s) | 20260507 20:56:36.831 | INFO | Analyzing chunk frames = [2536960,2537472) | 20260507 20:56:36.831 | INFO | MemInfo: { | 20260507 20:56:36.831 | INFO | "gpu" : " 17 GB", | 20260507 20:56:36.831 | INFO | "huge" : "", | 20260507 20:56:36.831 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:36.831 | INFO | "malloc" : " 14 GB", | 20260507 20:56:36.831 | INFO | "pinned" : " 29 GB" | 20260507 20:56:36.831 | INFO | } | 20260507 20:56:36.831 | INFO | Perf report for the last 5.13978 seconds: | 20260507 20:56:36.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:36.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.832 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 20:56:36.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.832 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:56:36.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:36.832 | INFO | Uploaded 36GiB to GPU in 2.49227s (14.4446GiB/s) | 20260507 20:56:36.832 | INFO | Downloaded 34GiB to GPU in 2.49164s (13.6456GiB/s) | 20260507 20:56:41.970 | INFO | Analyzing chunk frames = [2537472,2537984) | 20260507 20:56:41.970 | INFO | MemInfo: { | 20260507 20:56:41.970 | INFO | "gpu" : " 17 GB", | 20260507 20:56:41.970 | INFO | "huge" : "", | 20260507 20:56:41.970 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:41.971 | INFO | "malloc" : " 14 GB", | 20260507 20:56:41.971 | INFO | "pinned" : " 29 GB" | 20260507 20:56:41.971 | INFO | } | 20260507 20:56:41.971 | INFO | Perf report for the last 5.13917 seconds: | 20260507 20:56:41.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:41.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:56:41.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:56:41.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:41.971 | INFO | Uploaded 36GiB to GPU in 2.35748s (15.2705GiB/s) | 20260507 20:56:41.971 | INFO | Downloaded 34GiB to GPU in 2.50013s (13.5993GiB/s) | 20260507 20:56:47.108 | INFO | Analyzing chunk frames = [2537984,2538496) | 20260507 20:56:47.108 | INFO | MemInfo: { | 20260507 20:56:47.108 | INFO | "gpu" : " 17 GB", | 20260507 20:56:47.108 | INFO | "huge" : "", | 20260507 20:56:47.108 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:47.108 | INFO | "malloc" : " 14 GB", | 20260507 20:56:47.108 | INFO | "pinned" : " 29 GB" | 20260507 20:56:47.108 | INFO | } | 20260507 20:56:47.108 | INFO | Perf report for the last 5.13778 seconds: | 20260507 20:56:47.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:47.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.108 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 20:56:47.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.109 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:56:47.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:47.109 | INFO | Uploaded 36GiB to GPU in 2.47721s (14.5325GiB/s) | 20260507 20:56:47.109 | INFO | Downloaded 34GiB to GPU in 2.42879s (13.9987GiB/s) | 20260507 20:56:52.248 | INFO | Analyzing chunk frames = [2538496,2539008) | 20260507 20:56:52.248 | INFO | MemInfo: { | 20260507 20:56:52.248 | INFO | "gpu" : " 17 GB", | 20260507 20:56:52.248 | INFO | "huge" : "", | 20260507 20:56:52.248 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:52.248 | INFO | "malloc" : " 14 GB", | 20260507 20:56:52.248 | INFO | "pinned" : " 29 GB" | 20260507 20:56:52.248 | INFO | } | 20260507 20:56:52.248 | INFO | Perf report for the last 5.13997 seconds: | 20260507 20:56:52.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:52.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.248 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:56:52.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.249 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:56:52.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:52.249 | INFO | Uploaded 36GiB to GPU in 2.51412s (14.3191GiB/s) | 20260507 20:56:52.249 | INFO | Downloaded 34GiB to GPU in 2.45938s (13.8246GiB/s) | 20260507 20:56:57.389 | INFO | Analyzing chunk frames = [2539008,2539520) | 20260507 20:56:57.389 | INFO | MemInfo: { | 20260507 20:56:57.389 | INFO | "gpu" : " 17 GB", | 20260507 20:56:57.389 | INFO | "huge" : "", | 20260507 20:56:57.389 | INFO | "huge_shared" : " 29 GB", | 20260507 20:56:57.390 | INFO | "malloc" : " 14 GB", | 20260507 20:56:57.390 | INFO | "pinned" : " 29 GB" | 20260507 20:56:57.390 | INFO | } | 20260507 20:56:57.390 | INFO | Perf report for the last 5.1412 seconds: | 20260507 20:56:57.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:56:57.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:56:57.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 20:56:57.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:56:57.390 | INFO | Uploaded 36GiB to GPU in 2.49205s (14.4459GiB/s) | 20260507 20:56:57.390 | INFO | Downloaded 34GiB to GPU in 2.44008s (13.934GiB/s) | 20260507 20:56:57.390 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:56:57.390 | INFO | Total mem usage: 16.4 GiB | 20260507 20:56:57.390 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:56:57.390 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -12.9%) | 20260507 20:56:57.390 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:56:57.390 | INFO | High Water Breakdown: | 20260507 20:56:57.390 | INFO | 59.5% : 8.64 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:56:57.390 | INFO | 40.4% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:56:57.390 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:56:57.390 | INFO | ------------------------------------------ | 20260507 20:56:57.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:56:57.390 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:56:57.390 | INFO | Total mem usage: 26.5 GiB | 20260507 20:56:57.390 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:56:57.390 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:56:57.390 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:56:57.390 | INFO | High Water Breakdown: | 20260507 20:56:57.390 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:56:57.390 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:56:57.390 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:56:57.390 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:56:57.390 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:56:57.390 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:56:57.390 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:56:57.390 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:56:57.390 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:56:57.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:56:57.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:56:57.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:56:57.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:56:57.390 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:56:57.390 | INFO | ------------------------------------------ | 20260507 20:56:57.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:56:57.391 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:56:57.391 | INFO | Total mem usage: 27.8 GiB | 20260507 20:56:57.391 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:56:57.391 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:56:57.391 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:56:57.391 | INFO | High Water Breakdown: | 20260507 20:56:57.391 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:56:57.391 | INFO | ------------------------------------------ | 20260507 20:56:57.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:56:57.391 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:56:57.391 | INFO | Total mem usage: 15.7 GiB | 20260507 20:56:57.391 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:56:57.391 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:56:57.391 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:56:57.391 | INFO | High Water Breakdown: | 20260507 20:56:57.391 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:56:57.391 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:56:57.391 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:56:57.391 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:56:57.391 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:56:57.391 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:56:57.391 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:56:57.391 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:56:57.391 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:56:57.391 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:56:57.391 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:56:57.391 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:56:57.391 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:56:57.391 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:56:57.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:56:57.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:56:57.391 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:56:57.391 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:56:57.391 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:56:57.391 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:56:57.391 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:56:57.391 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:56:57.391 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:56:57.391 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:56:57.391 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:56:57.391 | INFO | ------------------------------------------ | 20260507 20:56:57.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:56:57.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:56:57.391 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:56:57.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:56:57.391 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.63e+01, 1.43e+01 / 2.17e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.01e-01, 1.25e-02 / 5.14e-03 | 20260507 20:56:57.391 | INFO | PerfReport: 12.12%, Baseline -- 4.00e+00 / 4.50e+00, 5.94e-01 / 2.08e-01 | 20260507 20:56:57.391 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.07e-01 | 20260507 20:56:57.391 | INFO | PerfReport: 16.13%, DME -- 4.26e+01 / 4.61e+01, 4.40e+00 / 2.27e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 55.79%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.60e+00 / 1.45e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.43e-01 / 1.96e-01 | 20260507 20:56:57.391 | INFO | PerfReport: 3.56%, Metrics -- 1.18e+00 / 1.41e+00, 2.97e-01 / 1.61e-01 | 20260507 20:56:57.391 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.64e-01 / 4.04e-01, 5.87e-02 / 3.39e-02 | 20260507 20:56:57.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:56:57.391 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:56:57.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:56:57.391 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 24.25%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 24.25%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 30.82%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 2.09%, LockUpload -- 2.24e+00 / 0.00e+00, 5.29e+00 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 2.75%, LockDownload -- 2.95e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 15.83%, LockCompute -- 1.70e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 20:56:57.391 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 9.80e-03 / 0.00e+00 | 20260507 20:56:57.487 | INFO | Starting a round of Baz IO | 20260507 20:57:00.961 | INFO | Populating RTMetrics report | 20260507 20:57:00.961 | INFO | RTMetrics report populated | 20260507 20:57:00.961 | INFO | Opening tmp RTMetrics json file | 20260507 20:57:01.000 | INFO | Serializing RTMetrics to json | 20260507 20:57:01.082 | INFO | Serialization done, swapping tmp file now... | 20260507 20:57:01.083 | INFO | RTMetrics done | 20260507 20:57:01.107 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.619 seconds | 20260507 20:57:01.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.08% / 0.01% / 55.91% | 20260507 20:57:01.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85074% / 0% / 5.96046e-05% | 20260507 20:57:01.107 | INFO | Baz IO overall: | 20260507 20:57:01.107 | INFO | BytesWritten : 1154844486032 | 20260507 20:57:01.107 | INFO | NumEvents : 559797206019 | 20260507 20:57:02.527 | INFO | Analyzing chunk frames = [2539520,2540032) | 20260507 20:57:02.527 | INFO | MemInfo: { | 20260507 20:57:02.527 | INFO | "gpu" : " 17 GB", | 20260507 20:57:02.527 | INFO | "huge" : "", | 20260507 20:57:02.527 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:02.527 | INFO | "malloc" : " 14 GB", | 20260507 20:57:02.527 | INFO | "pinned" : " 29 GB" | 20260507 20:57:02.527 | INFO | } | 20260507 20:57:02.527 | INFO | Perf report for the last 5.13755 seconds: | 20260507 20:57:02.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:02.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:02.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:02.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:02.527 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 20:57:02.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:02.527 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 20:57:02.527 | INFO | PRE_HQ : 66.2%, 1.7, 58.0ms, 0 | 20260507 20:57:02.527 | INFO | BAZWRITER : 70.1%, 3.7, 133.1ms, 0 | 20260507 20:57:02.528 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260507 20:57:02.528 | INFO | Uploaded 36GiB to GPU in 2.56913s (14.0125GiB/s) | 20260507 20:57:02.528 | INFO | Downloaded 38GiB to GPU in 2.69847s (14.0821GiB/s) | 20260507 20:57:07.667 | INFO | Analyzing chunk frames = [2540032,2540544) | 20260507 20:57:07.667 | INFO | MemInfo: { | 20260507 20:57:07.667 | INFO | "gpu" : " 17 GB", | 20260507 20:57:07.667 | INFO | "huge" : "", | 20260507 20:57:07.667 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:07.667 | INFO | "malloc" : " 14 GB", | 20260507 20:57:07.668 | INFO | "pinned" : " 29 GB" | 20260507 20:57:07.668 | INFO | } | 20260507 20:57:07.668 | INFO | Perf report for the last 5.1404 seconds: | 20260507 20:57:07.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:07.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 20:57:07.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 20:57:07.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:07.668 | INFO | Uploaded 36GiB to GPU in 2.51791s (14.2976GiB/s) | 20260507 20:57:07.668 | INFO | Downloaded 34GiB to GPU in 2.4957s (13.6234GiB/s) | 20260507 20:57:12.806 | INFO | Analyzing chunk frames = [2540544,2541056) | 20260507 20:57:12.806 | INFO | MemInfo: { | 20260507 20:57:12.806 | INFO | "gpu" : " 17 GB", | 20260507 20:57:12.806 | INFO | "huge" : "", | 20260507 20:57:12.806 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:12.806 | INFO | "malloc" : " 14 GB", | 20260507 20:57:12.806 | INFO | "pinned" : " 29 GB" | 20260507 20:57:12.806 | INFO | } | 20260507 20:57:12.806 | INFO | Perf report for the last 5.13861 seconds: | 20260507 20:57:12.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:12.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 20:57:12.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 20:57:12.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:12.806 | INFO | Uploaded 36GiB to GPU in 2.45707s (14.6516GiB/s) | 20260507 20:57:12.807 | INFO | Downloaded 34GiB to GPU in 2.48308s (13.6927GiB/s) | 20260507 20:57:17.945 | INFO | Analyzing chunk frames = [2541056,2541568) | 20260507 20:57:17.945 | INFO | MemInfo: { | 20260507 20:57:17.945 | INFO | "gpu" : " 17 GB", | 20260507 20:57:17.945 | INFO | "huge" : "", | 20260507 20:57:17.945 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:17.945 | INFO | "malloc" : " 14 GB", | 20260507 20:57:17.945 | INFO | "pinned" : " 29 GB" | 20260507 20:57:17.945 | INFO | } | 20260507 20:57:17.945 | INFO | Perf report for the last 5.13906 seconds: | 20260507 20:57:17.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:17.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.945 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 20:57:17.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.945 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 20:57:17.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:17.946 | INFO | Uploaded 36GiB to GPU in 2.53865s (14.1808GiB/s) | 20260507 20:57:17.946 | INFO | Downloaded 34GiB to GPU in 2.50691s (13.5625GiB/s) | 20260507 20:57:23.085 | INFO | Analyzing chunk frames = [2541568,2542080) | 20260507 20:57:23.085 | INFO | MemInfo: { | 20260507 20:57:23.085 | INFO | "gpu" : " 17 GB", | 20260507 20:57:23.085 | INFO | "huge" : "", | 20260507 20:57:23.085 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:23.085 | INFO | "malloc" : " 14 GB", | 20260507 20:57:23.085 | INFO | "pinned" : " 29 GB" | 20260507 20:57:23.085 | INFO | } | 20260507 20:57:23.086 | INFO | Perf report for the last 5.14023 seconds: | 20260507 20:57:23.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:23.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 20:57:23.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:57:23.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:23.086 | INFO | Uploaded 36GiB to GPU in 2.42097s (14.8701GiB/s) | 20260507 20:57:23.086 | INFO | Downloaded 34GiB to GPU in 2.49376s (13.634GiB/s) | 20260507 20:57:28.224 | INFO | Analyzing chunk frames = [2542080,2542592) | 20260507 20:57:28.224 | INFO | MemInfo: { | 20260507 20:57:28.224 | INFO | "gpu" : " 17 GB", | 20260507 20:57:28.224 | INFO | "huge" : "", | 20260507 20:57:28.224 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:28.224 | INFO | "malloc" : " 14 GB", | 20260507 20:57:28.224 | INFO | "pinned" : " 29 GB" | 20260507 20:57:28.224 | INFO | } | 20260507 20:57:28.224 | INFO | Perf report for the last 5.13849 seconds: | 20260507 20:57:28.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:28.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.224 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 20:57:28.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.224 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:57:28.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:28.225 | INFO | Uploaded 36GiB to GPU in 2.37973s (15.1278GiB/s) | 20260507 20:57:28.225 | INFO | Downloaded 34GiB to GPU in 2.39502s (14.1961GiB/s) | 20260507 20:57:33.364 | INFO | Analyzing chunk frames = [2542592,2543104) | 20260507 20:57:33.364 | INFO | MemInfo: { | 20260507 20:57:33.364 | INFO | "gpu" : " 17 GB", | 20260507 20:57:33.364 | INFO | "huge" : "", | 20260507 20:57:33.364 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:33.364 | INFO | "malloc" : " 14 GB", | 20260507 20:57:33.364 | INFO | "pinned" : " 29 GB" | 20260507 20:57:33.364 | INFO | } | 20260507 20:57:33.364 | INFO | Perf report for the last 5.13992 seconds: | 20260507 20:57:33.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:33.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:57:33.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:57:33.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:33.364 | INFO | Uploaded 36GiB to GPU in 2.51834s (14.2951GiB/s) | 20260507 20:57:33.364 | INFO | Downloaded 34GiB to GPU in 2.46061s (13.8177GiB/s) | 20260507 20:57:38.503 | INFO | Analyzing chunk frames = [2543104,2543616) | 20260507 20:57:38.503 | INFO | MemInfo: { | 20260507 20:57:38.503 | INFO | "gpu" : " 17 GB", | 20260507 20:57:38.503 | INFO | "huge" : "", | 20260507 20:57:38.503 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:38.504 | INFO | "malloc" : " 14 GB", | 20260507 20:57:38.504 | INFO | "pinned" : " 29 GB" | 20260507 20:57:38.504 | INFO | } | 20260507 20:57:38.504 | INFO | Perf report for the last 5.13963 seconds: | 20260507 20:57:38.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:38.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 20:57:38.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 20:57:38.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:38.504 | INFO | Uploaded 36GiB to GPU in 2.47251s (14.5601GiB/s) | 20260507 20:57:38.504 | INFO | Downloaded 34GiB to GPU in 2.4432s (13.9162GiB/s) | 20260507 20:57:38.601 | INFO | Starting a round of Baz IO | 20260507 20:57:42.084 | INFO | Populating RTMetrics report | 20260507 20:57:42.085 | INFO | RTMetrics report populated | 20260507 20:57:42.085 | INFO | Opening tmp RTMetrics json file | 20260507 20:57:42.096 | INFO | Serializing RTMetrics to json | 20260507 20:57:42.110 | INFO | Serialization done, swapping tmp file now... | 20260507 20:57:42.111 | INFO | RTMetrics done | 20260507 20:57:42.177 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.576 seconds | 20260507 20:57:42.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.07% / 0.01% / 55.92% | 20260507 20:57:42.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.8398% / 1.58946e-05% / 5.96046e-05% | 20260507 20:57:42.178 | INFO | Baz IO overall: | 20260507 20:57:42.178 | INFO | BytesWritten : 1156744664383 | 20260507 20:57:42.178 | INFO | NumEvents : 560716931154 | 20260507 20:57:43.643 | INFO | Analyzing chunk frames = [2543616,2544128) | 20260507 20:57:43.643 | INFO | MemInfo: { | 20260507 20:57:43.643 | INFO | "gpu" : " 17 GB", | 20260507 20:57:43.644 | INFO | "huge" : "", | 20260507 20:57:43.644 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:43.644 | INFO | "malloc" : " 14 GB", | 20260507 20:57:43.644 | INFO | "pinned" : " 29 GB" | 20260507 20:57:43.644 | INFO | } | 20260507 20:57:43.644 | INFO | Perf report for the last 5.14001 seconds: | 20260507 20:57:43.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:43.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:43.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:43.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:43.644 | INFO | ANALYSIS : 69.0%, 2.9, 104.8ms, 0 | 20260507 20:57:43.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:43.644 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 20:57:43.644 | INFO | PRE_HQ : 67.9%, 1.8, 62.6ms, 0 | 20260507 20:57:43.644 | INFO | BAZWRITER : 69.2%, 4.3, 151.3ms, 0 | 20260507 20:57:43.644 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 20:57:43.644 | INFO | Uploaded 36GiB to GPU in 2.55433s (14.0937GiB/s) | 20260507 20:57:43.644 | INFO | Downloaded 38GiB to GPU in 2.72032s (13.9689GiB/s) | 20260507 20:57:48.781 | INFO | Analyzing chunk frames = [2544128,2544640) | 20260507 20:57:48.781 | INFO | MemInfo: { | 20260507 20:57:48.781 | INFO | "gpu" : " 17 GB", | 20260507 20:57:48.781 | INFO | "huge" : "", | 20260507 20:57:48.781 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:48.781 | INFO | "malloc" : " 14 GB", | 20260507 20:57:48.781 | INFO | "pinned" : " 29 GB" | 20260507 20:57:48.781 | INFO | } | 20260507 20:57:48.782 | INFO | Perf report for the last 5.13792 seconds: | 20260507 20:57:48.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:48.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 20:57:48.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:57:48.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:48.782 | INFO | Uploaded 36GiB to GPU in 2.56704s (14.0239GiB/s) | 20260507 20:57:48.782 | INFO | Downloaded 34GiB to GPU in 2.4889s (13.6606GiB/s) | 20260507 20:57:53.921 | INFO | Analyzing chunk frames = [2544640,2545152) | 20260507 20:57:53.921 | INFO | MemInfo: { | 20260507 20:57:53.921 | INFO | "gpu" : " 17 GB", | 20260507 20:57:53.921 | INFO | "huge" : "", | 20260507 20:57:53.921 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:53.921 | INFO | "malloc" : " 14 GB", | 20260507 20:57:53.921 | INFO | "pinned" : " 29 GB" | 20260507 20:57:53.921 | INFO | } | 20260507 20:57:53.921 | INFO | Perf report for the last 5.13952 seconds: | 20260507 20:57:53.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:53.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:57:53.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:57:53.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:53.921 | INFO | Uploaded 36GiB to GPU in 2.56101s (14.0569GiB/s) | 20260507 20:57:53.921 | INFO | Downloaded 34GiB to GPU in 2.48466s (13.684GiB/s) | 20260507 20:57:59.061 | INFO | Analyzing chunk frames = [2545152,2545664) | 20260507 20:57:59.061 | INFO | MemInfo: { | 20260507 20:57:59.061 | INFO | "gpu" : " 17 GB", | 20260507 20:57:59.061 | INFO | "huge" : "", | 20260507 20:57:59.061 | INFO | "huge_shared" : " 29 GB", | 20260507 20:57:59.061 | INFO | "malloc" : " 14 GB", | 20260507 20:57:59.061 | INFO | "pinned" : " 29 GB" | 20260507 20:57:59.061 | INFO | } | 20260507 20:57:59.061 | INFO | Perf report for the last 5.13963 seconds: | 20260507 20:57:59.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:57:59.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 20:57:59.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:57:59.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:57:59.061 | INFO | Uploaded 36GiB to GPU in 2.48135s (14.5082GiB/s) | 20260507 20:57:59.061 | INFO | Downloaded 34GiB to GPU in 2.41239s (14.0939GiB/s) | 20260507 20:58:04.199 | INFO | Analyzing chunk frames = [2545664,2546176) | 20260507 20:58:04.199 | INFO | MemInfo: { | 20260507 20:58:04.199 | INFO | "gpu" : " 17 GB", | 20260507 20:58:04.199 | INFO | "huge" : "", | 20260507 20:58:04.199 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:04.199 | INFO | "malloc" : " 14 GB", | 20260507 20:58:04.199 | INFO | "pinned" : " 29 GB" | 20260507 20:58:04.199 | INFO | } | 20260507 20:58:04.200 | INFO | Perf report for the last 5.13883 seconds: | 20260507 20:58:04.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:04.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:58:04.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:58:04.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:04.200 | INFO | Uploaded 36GiB to GPU in 2.34511s (15.3511GiB/s) | 20260507 20:58:04.200 | INFO | Downloaded 34GiB to GPU in 2.50005s (13.5997GiB/s) | 20260507 20:58:09.339 | INFO | Analyzing chunk frames = [2546176,2546688) | 20260507 20:58:09.339 | INFO | MemInfo: { | 20260507 20:58:09.339 | INFO | "gpu" : " 17 GB", | 20260507 20:58:09.339 | INFO | "huge" : "", | 20260507 20:58:09.339 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:09.339 | INFO | "malloc" : " 14 GB", | 20260507 20:58:09.339 | INFO | "pinned" : " 29 GB" | 20260507 20:58:09.339 | INFO | } | 20260507 20:58:09.339 | INFO | Perf report for the last 5.13974 seconds: | 20260507 20:58:09.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:09.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.339 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 20:58:09.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.339 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 20:58:09.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:09.340 | INFO | Uploaded 36GiB to GPU in 2.44252s (14.7389GiB/s) | 20260507 20:58:09.340 | INFO | Downloaded 34GiB to GPU in 2.4369s (13.9521GiB/s) | 20260507 20:58:14.479 | INFO | Analyzing chunk frames = [2546688,2547200) | 20260507 20:58:14.479 | INFO | MemInfo: { | 20260507 20:58:14.479 | INFO | "gpu" : " 17 GB", | 20260507 20:58:14.479 | INFO | "huge" : "", | 20260507 20:58:14.479 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:14.479 | INFO | "malloc" : " 14 GB", | 20260507 20:58:14.479 | INFO | "pinned" : " 29 GB" | 20260507 20:58:14.479 | INFO | } | 20260507 20:58:14.479 | INFO | Perf report for the last 5.14001 seconds: | 20260507 20:58:14.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:14.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.479 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:58:14.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.480 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 20:58:14.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:14.480 | INFO | Uploaded 36GiB to GPU in 2.50822s (14.3528GiB/s) | 20260507 20:58:14.480 | INFO | Downloaded 34GiB to GPU in 2.47304s (13.7483GiB/s) | 20260507 20:58:19.618 | INFO | Analyzing chunk frames = [2547200,2547712) | 20260507 20:58:19.618 | INFO | MemInfo: { | 20260507 20:58:19.618 | INFO | "gpu" : " 17 GB", | 20260507 20:58:19.618 | INFO | "huge" : "", | 20260507 20:58:19.618 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:19.618 | INFO | "malloc" : " 14 GB", | 20260507 20:58:19.618 | INFO | "pinned" : " 29 GB" | 20260507 20:58:19.618 | INFO | } | 20260507 20:58:19.618 | INFO | Perf report for the last 5.13867 seconds: | 20260507 20:58:19.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:19.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 20:58:19.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 20:58:19.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:19.618 | INFO | Uploaded 36GiB to GPU in 2.49576s (14.4244GiB/s) | 20260507 20:58:19.618 | INFO | Downloaded 34GiB to GPU in 2.45623s (13.8423GiB/s) | 20260507 20:58:19.619 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:58:19.619 | INFO | Total mem usage: 16.4 GiB | 20260507 20:58:19.619 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:58:19.619 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.32%) | 20260507 20:58:19.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:58:19.619 | INFO | High Water Breakdown: | 20260507 20:58:19.619 | INFO | 60.4% : 9.23 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:58:19.619 | INFO | 39.5% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:58:19.619 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:58:19.619 | INFO | ------------------------------------------ | 20260507 20:58:19.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:58:19.619 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:58:19.619 | INFO | Total mem usage: 26.5 GiB | 20260507 20:58:19.619 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:58:19.619 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 20:58:19.619 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 20:58:19.619 | INFO | High Water Breakdown: | 20260507 20:58:19.619 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:58:19.619 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:58:19.619 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:58:19.619 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:58:19.619 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:58:19.619 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:58:19.619 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:58:19.619 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:58:19.619 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:58:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:58:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:58:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:58:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:58:19.619 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:58:19.619 | INFO | ------------------------------------------ | 20260507 20:58:19.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:58:19.619 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:58:19.619 | INFO | Total mem usage: 27.8 GiB | 20260507 20:58:19.619 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:58:19.619 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:58:19.619 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:58:19.619 | INFO | High Water Breakdown: | 20260507 20:58:19.619 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:58:19.619 | INFO | ------------------------------------------ | 20260507 20:58:19.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:58:19.619 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:58:19.619 | INFO | Total mem usage: 15.7 GiB | 20260507 20:58:19.619 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:58:19.619 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:58:19.619 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:58:19.620 | INFO | High Water Breakdown: | 20260507 20:58:19.620 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:58:19.620 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:58:19.620 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:58:19.620 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:58:19.620 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:58:19.620 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:58:19.620 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:58:19.620 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:58:19.620 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:58:19.620 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:58:19.620 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:58:19.620 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:58:19.620 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:58:19.620 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:58:19.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:58:19.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:58:19.620 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:58:19.620 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:58:19.620 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:58:19.620 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:58:19.620 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:58:19.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:58:19.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:58:19.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:58:19.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:58:19.620 | INFO | ------------------------------------------ | 20260507 20:58:19.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:58:19.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:58:19.620 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:58:19.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:58:19.620 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.59e+01, 1.42e+01 / 2.08e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.03e-01, 1.20e-02 / 6.19e-03 | 20260507 20:58:19.620 | INFO | PerfReport: 12.12%, Baseline -- 4.01e+00 / 4.49e+00, 5.89e-01 / 2.48e-01 | 20260507 20:58:19.620 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.08e-01 / 1.18e-01 | 20260507 20:58:19.620 | INFO | PerfReport: 15.90%, DME -- 4.21e+01 / 4.55e+01, 3.99e+00 / 1.94e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.74e+00 / 1.75e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.49e+00 / 2.81e+00, 4.39e-01 / 2.52e-01 | 20260507 20:58:19.620 | INFO | PerfReport: 3.54%, Metrics -- 1.17e+00 / 1.38e+00, 2.95e-01 / 1.72e-01 | 20260507 20:58:19.620 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 4.00e-01, 5.97e-02 / 3.04e-02 | 20260507 20:58:19.620 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:58:19.620 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:58:19.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:58:19.620 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 24.28%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 24.35%, Download -- 2.60e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 30.97%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 1.83%, LockUpload -- 1.95e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 2.67%, LockDownload -- 2.86e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 15.88%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 20:58:19.620 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 20:58:19.733 | INFO | Starting a round of Baz IO | 20260507 20:58:23.215 | INFO | Populating RTMetrics report | 20260507 20:58:23.215 | INFO | RTMetrics report populated | 20260507 20:58:23.215 | INFO | Opening tmp RTMetrics json file | 20260507 20:58:23.295 | INFO | Serializing RTMetrics to json | 20260507 20:58:23.299 | INFO | Serialization done, swapping tmp file now... | 20260507 20:58:23.303 | INFO | RTMetrics done | 20260507 20:58:23.330 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.596 seconds | 20260507 20:58:23.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.06% / 0.01% / 55.93% | 20260507 20:58:23.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83243% / 7.94729e-06% / 5.16574e-05% | 20260507 20:58:23.330 | INFO | Baz IO overall: | 20260507 20:58:23.330 | INFO | BytesWritten : 1158643241003 | 20260507 20:58:23.330 | INFO | NumEvents : 561635312488 | 20260507 20:58:24.757 | INFO | Analyzing chunk frames = [2547712,2548224) | 20260507 20:58:24.757 | INFO | MemInfo: { | 20260507 20:58:24.757 | INFO | "gpu" : " 17 GB", | 20260507 20:58:24.757 | INFO | "huge" : "", | 20260507 20:58:24.757 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:24.757 | INFO | "malloc" : " 14 GB", | 20260507 20:58:24.757 | INFO | "pinned" : " 29 GB" | 20260507 20:58:24.757 | INFO | } | 20260507 20:58:24.757 | INFO | Perf report for the last 5.13918 seconds: | 20260507 20:58:24.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:24.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:24.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:24.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:24.757 | INFO | ANALYSIS : 69.0%, 2.9, 105.1ms, 0 | 20260507 20:58:24.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:24.757 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 20:58:24.757 | INFO | PRE_HQ : 67.9%, 1.8, 63.7ms, 0 | 20260507 20:58:24.757 | INFO | BAZWRITER : 69.8%, 3.8, 136.7ms, 0 | 20260507 20:58:24.757 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 20:58:24.758 | INFO | Uploaded 36GiB to GPU in 2.61125s (13.7865GiB/s) | 20260507 20:58:24.758 | INFO | Downloaded 38GiB to GPU in 2.70066s (14.0706GiB/s) | 20260507 20:58:29.897 | INFO | Analyzing chunk frames = [2548224,2548736) | 20260507 20:58:29.897 | INFO | MemInfo: { | 20260507 20:58:29.897 | INFO | "gpu" : " 17 GB", | 20260507 20:58:29.897 | INFO | "huge" : "", | 20260507 20:58:29.897 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:29.897 | INFO | "malloc" : " 14 GB", | 20260507 20:58:29.897 | INFO | "pinned" : " 29 GB" | 20260507 20:58:29.897 | INFO | } | 20260507 20:58:29.897 | INFO | Perf report for the last 5.13967 seconds: | 20260507 20:58:29.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:29.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 20:58:29.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:58:29.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:29.897 | INFO | Uploaded 36GiB to GPU in 2.52477s (14.2587GiB/s) | 20260507 20:58:29.897 | INFO | Downloaded 34GiB to GPU in 2.50908s (13.5508GiB/s) | 20260507 20:58:35.036 | INFO | Analyzing chunk frames = [2548736,2549248) | 20260507 20:58:35.036 | INFO | MemInfo: { | 20260507 20:58:35.036 | INFO | "gpu" : " 17 GB", | 20260507 20:58:35.036 | INFO | "huge" : "", | 20260507 20:58:35.036 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:35.036 | INFO | "malloc" : " 14 GB", | 20260507 20:58:35.036 | INFO | "pinned" : " 29 GB" | 20260507 20:58:35.036 | INFO | } | 20260507 20:58:35.036 | INFO | Perf report for the last 5.13924 seconds: | 20260507 20:58:35.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:35.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.036 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 20:58:35.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.036 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:58:35.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:35.037 | INFO | Uploaded 36GiB to GPU in 2.41701s (14.8944GiB/s) | 20260507 20:58:35.037 | INFO | Downloaded 34GiB to GPU in 2.49928s (13.6039GiB/s) | 20260507 20:58:40.175 | INFO | Analyzing chunk frames = [2549248,2549760) | 20260507 20:58:40.175 | INFO | MemInfo: { | 20260507 20:58:40.175 | INFO | "gpu" : " 17 GB", | 20260507 20:58:40.175 | INFO | "huge" : "", | 20260507 20:58:40.175 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:40.175 | INFO | "malloc" : " 14 GB", | 20260507 20:58:40.175 | INFO | "pinned" : " 29 GB" | 20260507 20:58:40.175 | INFO | } | 20260507 20:58:40.175 | INFO | Perf report for the last 5.13921 seconds: | 20260507 20:58:40.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:40.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.175 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 20:58:40.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.176 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 20:58:40.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:40.176 | INFO | Uploaded 36GiB to GPU in 2.43581s (14.7795GiB/s) | 20260507 20:58:40.176 | INFO | Downloaded 34GiB to GPU in 2.51239s (13.5329GiB/s) | 20260507 20:58:45.315 | INFO | Analyzing chunk frames = [2549760,2550272) | 20260507 20:58:45.315 | INFO | MemInfo: { | 20260507 20:58:45.315 | INFO | "gpu" : " 17 GB", | 20260507 20:58:45.315 | INFO | "huge" : "", | 20260507 20:58:45.315 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:45.315 | INFO | "malloc" : " 14 GB", | 20260507 20:58:45.316 | INFO | "pinned" : " 29 GB" | 20260507 20:58:45.316 | INFO | } | 20260507 20:58:45.316 | INFO | Perf report for the last 5.14031 seconds: | 20260507 20:58:45.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:45.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 20:58:45.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 20:58:45.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:45.316 | INFO | Uploaded 36GiB to GPU in 2.36014s (15.2533GiB/s) | 20260507 20:58:45.316 | INFO | Downloaded 34GiB to GPU in 2.50073s (13.596GiB/s) | 20260507 20:58:50.454 | INFO | Analyzing chunk frames = [2550272,2550784) | 20260507 20:58:50.454 | INFO | MemInfo: { | 20260507 20:58:50.454 | INFO | "gpu" : " 17 GB", | 20260507 20:58:50.454 | INFO | "huge" : "", | 20260507 20:58:50.454 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:50.454 | INFO | "malloc" : " 14 GB", | 20260507 20:58:50.454 | INFO | "pinned" : " 29 GB" | 20260507 20:58:50.454 | INFO | } | 20260507 20:58:50.454 | INFO | Perf report for the last 5.13812 seconds: | 20260507 20:58:50.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:50.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260507 20:58:50.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:58:50.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:50.454 | INFO | Uploaded 36GiB to GPU in 2.35633s (15.278GiB/s) | 20260507 20:58:50.454 | INFO | Downloaded 34GiB to GPU in 2.4005s (14.1637GiB/s) | 20260507 20:58:55.593 | INFO | Analyzing chunk frames = [2550784,2551296) | 20260507 20:58:55.593 | INFO | MemInfo: { | 20260507 20:58:55.593 | INFO | "gpu" : " 17 GB", | 20260507 20:58:55.593 | INFO | "huge" : "", | 20260507 20:58:55.593 | INFO | "huge_shared" : " 29 GB", | 20260507 20:58:55.593 | INFO | "malloc" : " 14 GB", | 20260507 20:58:55.593 | INFO | "pinned" : " 29 GB" | 20260507 20:58:55.593 | INFO | } | 20260507 20:58:55.593 | INFO | Perf report for the last 5.13925 seconds: | 20260507 20:58:55.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:58:55.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.593 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 20:58:55.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.593 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 20:58:55.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:58:55.594 | INFO | Uploaded 36GiB to GPU in 2.50754s (14.3567GiB/s) | 20260507 20:58:55.594 | INFO | Downloaded 34GiB to GPU in 2.469s (13.7707GiB/s) | 20260507 20:59:00.733 | INFO | Analyzing chunk frames = [2551296,2551808) | 20260507 20:59:00.733 | INFO | MemInfo: { | 20260507 20:59:00.733 | INFO | "gpu" : " 17 GB", | 20260507 20:59:00.733 | INFO | "huge" : "", | 20260507 20:59:00.733 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:00.734 | INFO | "malloc" : " 14 GB", | 20260507 20:59:00.734 | INFO | "pinned" : " 29 GB" | 20260507 20:59:00.734 | INFO | } | 20260507 20:59:00.734 | INFO | Perf report for the last 5.14055 seconds: | 20260507 20:59:00.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:00.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 20:59:00.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 20:59:00.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:00.734 | INFO | Uploaded 36GiB to GPU in 2.49992s (14.4005GiB/s) | 20260507 20:59:00.734 | INFO | Downloaded 34GiB to GPU in 2.45581s (13.8447GiB/s) | 20260507 20:59:00.847 | INFO | Starting a round of Baz IO | 20260507 20:59:04.308 | INFO | Populating RTMetrics report | 20260507 20:59:04.309 | INFO | RTMetrics report populated | 20260507 20:59:04.309 | INFO | Opening tmp RTMetrics json file | 20260507 20:59:04.343 | INFO | Serializing RTMetrics to json | 20260507 20:59:04.405 | INFO | Serialization done, swapping tmp file now... | 20260507 20:59:04.409 | INFO | RTMetrics done | 20260507 20:59:04.439 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.592 seconds | 20260507 20:59:04.440 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.06% / 0.01% / 55.94% | 20260507 20:59:04.440 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83307% / 1.98682e-05% / 6.75519e-05% | 20260507 20:59:04.440 | INFO | Baz IO overall: | 20260507 20:59:04.440 | INFO | BytesWritten : 1160543626126 | 20260507 20:59:04.440 | INFO | NumEvents : 562554999044 | 20260507 20:59:05.871 | INFO | Analyzing chunk frames = [2551808,2552320) | 20260507 20:59:05.871 | INFO | MemInfo: { | 20260507 20:59:05.871 | INFO | "gpu" : " 17 GB", | 20260507 20:59:05.871 | INFO | "huge" : "", | 20260507 20:59:05.871 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:05.871 | INFO | "malloc" : " 14 GB", | 20260507 20:59:05.871 | INFO | "pinned" : " 29 GB" | 20260507 20:59:05.871 | INFO | } | 20260507 20:59:05.871 | INFO | Perf report for the last 5.13762 seconds: | 20260507 20:59:05.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:05.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:05.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:05.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:05.871 | INFO | ANALYSIS : 68.8%, 2.9, 105.2ms, 0 | 20260507 20:59:05.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:05.871 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 20:59:05.871 | INFO | PRE_HQ : 67.7%, 1.8, 63.0ms, 0 | 20260507 20:59:05.871 | INFO | BAZWRITER : 69.5%, 4.0, 141.4ms, 0 | 20260507 20:59:05.872 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 20:59:05.872 | INFO | Uploaded 36GiB to GPU in 2.50407s (14.3766GiB/s) | 20260507 20:59:05.872 | INFO | Downloaded 38GiB to GPU in 2.69339s (14.1086GiB/s) | 20260507 20:59:11.012 | INFO | Analyzing chunk frames = [2552320,2552832) | 20260507 20:59:11.012 | INFO | MemInfo: { | 20260507 20:59:11.012 | INFO | "gpu" : " 17 GB", | 20260507 20:59:11.012 | INFO | "huge" : "", | 20260507 20:59:11.012 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:11.012 | INFO | "malloc" : " 14 GB", | 20260507 20:59:11.013 | INFO | "pinned" : " 29 GB" | 20260507 20:59:11.013 | INFO | } | 20260507 20:59:11.013 | INFO | Perf report for the last 5.14125 seconds: | 20260507 20:59:11.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:11.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 20:59:11.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:59:11.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:11.013 | INFO | Uploaded 36GiB to GPU in 2.54002s (14.1731GiB/s) | 20260507 20:59:11.013 | INFO | Downloaded 34GiB to GPU in 2.49033s (13.6528GiB/s) | 20260507 20:59:16.151 | INFO | Analyzing chunk frames = [2552832,2553344) | 20260507 20:59:16.151 | INFO | MemInfo: { | 20260507 20:59:16.151 | INFO | "gpu" : " 17 GB", | 20260507 20:59:16.151 | INFO | "huge" : "", | 20260507 20:59:16.151 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:16.151 | INFO | "malloc" : " 14 GB", | 20260507 20:59:16.151 | INFO | "pinned" : " 29 GB" | 20260507 20:59:16.151 | INFO | } | 20260507 20:59:16.151 | INFO | Perf report for the last 5.1386 seconds: | 20260507 20:59:16.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:16.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.151 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:59:16.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.151 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:59:16.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:16.152 | INFO | Uploaded 36GiB to GPU in 2.56879s (14.0144GiB/s) | 20260507 20:59:16.152 | INFO | Downloaded 34GiB to GPU in 2.47276s (13.7498GiB/s) | 20260507 20:59:21.290 | INFO | Analyzing chunk frames = [2553344,2553856) | 20260507 20:59:21.290 | INFO | MemInfo: { | 20260507 20:59:21.290 | INFO | "gpu" : " 17 GB", | 20260507 20:59:21.290 | INFO | "huge" : "", | 20260507 20:59:21.290 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:21.290 | INFO | "malloc" : " 14 GB", | 20260507 20:59:21.290 | INFO | "pinned" : " 29 GB" | 20260507 20:59:21.290 | INFO | } | 20260507 20:59:21.290 | INFO | Perf report for the last 5.13914 seconds: | 20260507 20:59:21.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:21.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.290 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 20:59:21.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.290 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 20:59:21.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:21.291 | INFO | Uploaded 36GiB to GPU in 2.51846s (14.2945GiB/s) | 20260507 20:59:21.291 | INFO | Downloaded 34GiB to GPU in 2.49992s (13.6004GiB/s) | 20260507 20:59:26.430 | INFO | Analyzing chunk frames = [2553856,2554368) | 20260507 20:59:26.430 | INFO | MemInfo: { | 20260507 20:59:26.430 | INFO | "gpu" : " 17 GB", | 20260507 20:59:26.430 | INFO | "huge" : "", | 20260507 20:59:26.430 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:26.430 | INFO | "malloc" : " 14 GB", | 20260507 20:59:26.430 | INFO | "pinned" : " 29 GB" | 20260507 20:59:26.430 | INFO | } | 20260507 20:59:26.430 | INFO | Perf report for the last 5.13949 seconds: | 20260507 20:59:26.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:26.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 20:59:26.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 20:59:26.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:26.430 | INFO | Uploaded 36GiB to GPU in 2.40009s (14.9994GiB/s) | 20260507 20:59:26.430 | INFO | Downloaded 34GiB to GPU in 2.48731s (13.6694GiB/s) | 20260507 20:59:31.568 | INFO | Analyzing chunk frames = [2554368,2554880) | 20260507 20:59:31.568 | INFO | MemInfo: { | 20260507 20:59:31.568 | INFO | "gpu" : " 17 GB", | 20260507 20:59:31.568 | INFO | "huge" : "", | 20260507 20:59:31.568 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:31.568 | INFO | "malloc" : " 14 GB", | 20260507 20:59:31.568 | INFO | "pinned" : " 29 GB" | 20260507 20:59:31.568 | INFO | } | 20260507 20:59:31.568 | INFO | Perf report for the last 5.1384 seconds: | 20260507 20:59:31.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:31.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.568 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 20:59:31.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.568 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 20:59:31.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:31.569 | INFO | Uploaded 36GiB to GPU in 2.40194s (14.9879GiB/s) | 20260507 20:59:31.569 | INFO | Downloaded 34GiB to GPU in 2.38199s (14.2738GiB/s) | 20260507 20:59:36.708 | INFO | Analyzing chunk frames = [2554880,2555392) | 20260507 20:59:36.708 | INFO | MemInfo: { | 20260507 20:59:36.708 | INFO | "gpu" : " 17 GB", | 20260507 20:59:36.708 | INFO | "huge" : "", | 20260507 20:59:36.708 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:36.708 | INFO | "malloc" : " 14 GB", | 20260507 20:59:36.708 | INFO | "pinned" : " 29 GB" | 20260507 20:59:36.708 | INFO | } | 20260507 20:59:36.709 | INFO | Perf report for the last 5.14018 seconds: | 20260507 20:59:36.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:36.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:59:36.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 20:59:36.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:36.709 | INFO | Uploaded 36GiB to GPU in 2.54638s (14.1377GiB/s) | 20260507 20:59:36.709 | INFO | Downloaded 34GiB to GPU in 2.44765s (13.8909GiB/s) | 20260507 20:59:41.848 | INFO | Analyzing chunk frames = [2555392,2555904) | 20260507 20:59:41.848 | INFO | MemInfo: { | 20260507 20:59:41.848 | INFO | "gpu" : " 17 GB", | 20260507 20:59:41.848 | INFO | "huge" : "", | 20260507 20:59:41.848 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:41.848 | INFO | "malloc" : " 14 GB", | 20260507 20:59:41.848 | INFO | "pinned" : " 29 GB" | 20260507 20:59:41.848 | INFO | } | 20260507 20:59:41.849 | INFO | Perf report for the last 5.13997 seconds: | 20260507 20:59:41.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:41.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 20:59:41.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 20:59:41.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:41.849 | INFO | Uploaded 36GiB to GPU in 2.54178s (14.1633GiB/s) | 20260507 20:59:41.849 | INFO | Downloaded 34GiB to GPU in 2.42457s (14.0231GiB/s) | 20260507 20:59:41.849 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 20:59:41.849 | INFO | Total mem usage: 16.4 GiB | 20260507 20:59:41.849 | INFO | All Time High Water Marks sum: 16.7 GiB (Savings: 1.58%) | 20260507 20:59:41.849 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -11.3%) | 20260507 20:59:41.849 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 20:59:41.849 | INFO | High Water Breakdown: | 20260507 20:59:41.849 | INFO | 59.8% : 8.81 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 20:59:41.849 | INFO | 40.1% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 20:59:41.849 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 20:59:41.849 | INFO | ------------------------------------------ | 20260507 20:59:41.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:59:41.849 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 20:59:41.849 | INFO | Total mem usage: 26.5 GiB | 20260507 20:59:41.849 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 20:59:41.849 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 20:59:41.849 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 20:59:41.849 | INFO | High Water Breakdown: | 20260507 20:59:41.849 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:59:41.849 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:59:41.849 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:59:41.849 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:59:41.849 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:59:41.849 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:59:41.849 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 20:59:41.849 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 20:59:41.849 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:59:41.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 20:59:41.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 20:59:41.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 20:59:41.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 20:59:41.849 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 20:59:41.849 | INFO | ------------------------------------------ | 20260507 20:59:41.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:59:41.849 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 20:59:41.849 | INFO | Total mem usage: 27.8 GiB | 20260507 20:59:41.850 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 20:59:41.850 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 20:59:41.850 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 20:59:41.850 | INFO | High Water Breakdown: | 20260507 20:59:41.850 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 20:59:41.850 | INFO | ------------------------------------------ | 20260507 20:59:41.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:59:41.850 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 20:59:41.850 | INFO | Total mem usage: 15.7 GiB | 20260507 20:59:41.850 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 20:59:41.850 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 20:59:41.850 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 20:59:41.850 | INFO | High Water Breakdown: | 20260507 20:59:41.850 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 20:59:41.850 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 20:59:41.850 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 20:59:41.850 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 20:59:41.850 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 20:59:41.850 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 20:59:41.850 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 20:59:41.850 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 20:59:41.850 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 20:59:41.850 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 20:59:41.850 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 20:59:41.850 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 20:59:41.850 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 20:59:41.850 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 20:59:41.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 20:59:41.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 20:59:41.850 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 20:59:41.850 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 20:59:41.850 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 20:59:41.850 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 20:59:41.850 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 20:59:41.850 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 20:59:41.850 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 20:59:41.850 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 20:59:41.850 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 20:59:41.850 | INFO | ------------------------------------------ | 20260507 20:59:41.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 20:59:41.851 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 20:59:41.851 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:59:41.851 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:59:41.851 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.44e+01 / 1.83e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.03e-01, 1.24e-02 / 7.10e-03 | 20260507 20:59:41.851 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.45e+00, 5.93e-01 / 2.89e-01 | 20260507 20:59:41.851 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.08e-01 / 1.14e-01 | 20260507 20:59:41.851 | INFO | PerfReport: 16.03%, DME -- 4.24e+01 / 4.59e+01, 4.24e+00 / 1.93e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.70e+00 / 1.66e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.50e+00 / 2.79e+00, 4.43e-01 / 2.76e-01 | 20260507 20:59:41.851 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.36e+00, 2.91e-01 / 1.65e-01 | 20260507 20:59:41.851 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 3.96e-01, 5.94e-02 / 3.13e-02 | 20260507 20:59:41.851 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 20:59:41.851 | INFO | PerfReport: Gathered 1536 samples. | 20260507 20:59:41.851 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 20:59:41.851 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 24.25%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 1.90%, LockUpload -- 2.03e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 2.74%, LockDownload -- 2.92e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 15.71%, LockCompute -- 1.68e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 20:59:41.851 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 20:59:41.960 | INFO | Starting a round of Baz IO | 20260507 20:59:45.428 | INFO | Populating RTMetrics report | 20260507 20:59:45.428 | INFO | RTMetrics report populated | 20260507 20:59:45.428 | INFO | Opening tmp RTMetrics json file | 20260507 20:59:45.452 | INFO | Serializing RTMetrics to json | 20260507 20:59:45.453 | INFO | Serialization done, swapping tmp file now... | 20260507 20:59:45.454 | INFO | RTMetrics done | 20260507 20:59:45.532 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.572 seconds | 20260507 20:59:45.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.05% / 0.01% / 55.94% | 20260507 20:59:45.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.8302% / 1.98682e-05% / 4.76837e-05% | 20260507 20:59:45.532 | INFO | Baz IO overall: | 20260507 20:59:45.533 | INFO | BytesWritten : 1162442925536 | 20260507 20:59:45.533 | INFO | NumEvents : 563473985711 | 20260507 20:59:46.987 | INFO | Analyzing chunk frames = [2555904,2556416) | 20260507 20:59:46.987 | INFO | MemInfo: { | 20260507 20:59:46.987 | INFO | "gpu" : " 17 GB", | 20260507 20:59:46.987 | INFO | "huge" : "", | 20260507 20:59:46.987 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:46.987 | INFO | "malloc" : " 14 GB", | 20260507 20:59:46.987 | INFO | "pinned" : " 29 GB" | 20260507 20:59:46.987 | INFO | } | 20260507 20:59:46.988 | INFO | Perf report for the last 5.13904 seconds: | 20260507 20:59:46.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:46.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:46.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:46.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:46.988 | INFO | ANALYSIS : 68.7%, 2.9, 105.6ms, 0 | 20260507 20:59:46.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:46.988 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 20:59:46.988 | INFO | PRE_HQ : 67.2%, 1.8, 61.8ms, 0 | 20260507 20:59:46.988 | INFO | BAZWRITER : 69.1%, 4.2, 147.6ms, 0 | 20260507 20:59:46.988 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 20:59:46.988 | INFO | Uploaded 36GiB to GPU in 2.58186s (13.9434GiB/s) | 20260507 20:59:46.988 | INFO | Downloaded 38GiB to GPU in 2.69512s (14.0996GiB/s) | 20260507 20:59:52.126 | INFO | Analyzing chunk frames = [2556416,2556928) | 20260507 20:59:52.126 | INFO | MemInfo: { | 20260507 20:59:52.126 | INFO | "gpu" : " 17 GB", | 20260507 20:59:52.126 | INFO | "huge" : "", | 20260507 20:59:52.126 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:52.126 | INFO | "malloc" : " 14 GB", | 20260507 20:59:52.126 | INFO | "pinned" : " 29 GB" | 20260507 20:59:52.126 | INFO | } | 20260507 20:59:52.126 | INFO | Perf report for the last 5.13889 seconds: | 20260507 20:59:52.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:52.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.127 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 20:59:52.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.127 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 20:59:52.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:52.127 | INFO | Uploaded 36GiB to GPU in 2.56521s (14.034GiB/s) | 20260507 20:59:52.127 | INFO | Downloaded 34GiB to GPU in 2.49476s (13.6286GiB/s) | 20260507 20:59:57.265 | INFO | Analyzing chunk frames = [2556928,2557440) | 20260507 20:59:57.265 | INFO | MemInfo: { | 20260507 20:59:57.265 | INFO | "gpu" : " 17 GB", | 20260507 20:59:57.265 | INFO | "huge" : "", | 20260507 20:59:57.265 | INFO | "huge_shared" : " 29 GB", | 20260507 20:59:57.265 | INFO | "malloc" : " 14 GB", | 20260507 20:59:57.265 | INFO | "pinned" : " 29 GB" | 20260507 20:59:57.265 | INFO | } | 20260507 20:59:57.265 | INFO | Perf report for the last 5.13897 seconds: | 20260507 20:59:57.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 20:59:57.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.265 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 20:59:57.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.266 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 20:59:57.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 20:59:57.266 | INFO | Uploaded 36GiB to GPU in 2.48032s (14.5143GiB/s) | 20260507 20:59:57.266 | INFO | Downloaded 34GiB to GPU in 2.49249s (13.641GiB/s) | 20260507 21:00:02.404 | INFO | Analyzing chunk frames = [2557440,2557952) | 20260507 21:00:02.404 | INFO | MemInfo: { | 20260507 21:00:02.404 | INFO | "gpu" : " 17 GB", | 20260507 21:00:02.404 | INFO | "huge" : "", | 20260507 21:00:02.404 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:02.404 | INFO | "malloc" : " 14 GB", | 20260507 21:00:02.404 | INFO | "pinned" : " 29 GB" | 20260507 21:00:02.404 | INFO | } | 20260507 21:00:02.404 | INFO | Perf report for the last 5.13904 seconds: | 20260507 21:00:02.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:02.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:00:02.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:00:02.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:02.405 | INFO | Uploaded 36GiB to GPU in 2.49648s (14.4203GiB/s) | 20260507 21:00:02.405 | INFO | Downloaded 34GiB to GPU in 2.50524s (13.5716GiB/s) | 20260507 21:00:07.544 | INFO | Analyzing chunk frames = [2557952,2558464) | 20260507 21:00:07.544 | INFO | MemInfo: { | 20260507 21:00:07.544 | INFO | "gpu" : " 17 GB", | 20260507 21:00:07.544 | INFO | "huge" : "", | 20260507 21:00:07.544 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:07.544 | INFO | "malloc" : " 14 GB", | 20260507 21:00:07.544 | INFO | "pinned" : " 29 GB" | 20260507 21:00:07.544 | INFO | } | 20260507 21:00:07.544 | INFO | Perf report for the last 5.13996 seconds: | 20260507 21:00:07.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:07.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.545 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:00:07.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.545 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 21:00:07.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:07.545 | INFO | Uploaded 36GiB to GPU in 2.36581s (15.2168GiB/s) | 20260507 21:00:07.545 | INFO | Downloaded 34GiB to GPU in 2.50666s (13.5638GiB/s) | 20260507 21:00:12.686 | INFO | Analyzing chunk frames = [2558464,2558976) | 20260507 21:00:12.686 | INFO | MemInfo: { | 20260507 21:00:12.686 | INFO | "gpu" : " 17 GB", | 20260507 21:00:12.686 | INFO | "huge" : "", | 20260507 21:00:12.686 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:12.686 | INFO | "malloc" : " 14 GB", | 20260507 21:00:12.686 | INFO | "pinned" : " 29 GB" | 20260507 21:00:12.686 | INFO | } | 20260507 21:00:12.686 | INFO | Perf report for the last 5.14184 seconds: | 20260507 21:00:12.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:12.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.686 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:00:12.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.686 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:00:12.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:12.687 | INFO | Uploaded 36GiB to GPU in 2.50669s (14.3616GiB/s) | 20260507 21:00:12.687 | INFO | Downloaded 34GiB to GPU in 2.43269s (13.9763GiB/s) | 20260507 21:00:17.823 | INFO | Analyzing chunk frames = [2558976,2559488) | 20260507 21:00:17.823 | INFO | MemInfo: { | 20260507 21:00:17.823 | INFO | "gpu" : " 17 GB", | 20260507 21:00:17.823 | INFO | "huge" : "", | 20260507 21:00:17.823 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:17.824 | INFO | "malloc" : " 14 GB", | 20260507 21:00:17.824 | INFO | "pinned" : " 29 GB" | 20260507 21:00:17.824 | INFO | } | 20260507 21:00:17.824 | INFO | Perf report for the last 5.13731 seconds: | 20260507 21:00:17.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:17.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:00:17.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 21:00:17.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:17.824 | INFO | Uploaded 36GiB to GPU in 2.50805s (14.3538GiB/s) | 20260507 21:00:17.824 | INFO | Downloaded 34GiB to GPU in 2.45952s (13.8238GiB/s) | 20260507 21:00:22.962 | INFO | Analyzing chunk frames = [2559488,2560000) | 20260507 21:00:22.962 | INFO | MemInfo: { | 20260507 21:00:22.962 | INFO | "gpu" : " 17 GB", | 20260507 21:00:22.962 | INFO | "huge" : "", | 20260507 21:00:22.962 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:22.962 | INFO | "malloc" : " 14 GB", | 20260507 21:00:22.962 | INFO | "pinned" : " 29 GB" | 20260507 21:00:22.962 | INFO | } | 20260507 21:00:22.963 | INFO | Perf report for the last 5.13884 seconds: | 20260507 21:00:22.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:22.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:00:22.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | PRE_HQ : 36.5%, 1.0, 18.9ms, 0 | 20260507 21:00:22.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:22.963 | INFO | Uploaded 36GiB to GPU in 2.50885s (14.3492GiB/s) | 20260507 21:00:22.963 | INFO | Downloaded 34GiB to GPU in 2.44459s (13.9083GiB/s) | 20260507 21:00:23.076 | INFO | Starting a round of Baz IO | 20260507 21:00:26.580 | INFO | Populating RTMetrics report | 20260507 21:00:26.580 | INFO | RTMetrics report populated | 20260507 21:00:26.580 | INFO | Opening tmp RTMetrics json file | 20260507 21:00:26.597 | INFO | Serializing RTMetrics to json | 20260507 21:00:26.625 | INFO | Serialization done, swapping tmp file now... | 20260507 21:00:26.632 | INFO | RTMetrics done | 20260507 21:00:27.103 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 4.026 seconds | 20260507 21:00:27.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.04% / 0.01% / 55.95% | 20260507 21:00:27.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83016% / 3.97364e-06% / 5.5631e-05% | 20260507 21:00:27.103 | INFO | Baz IO overall: | 20260507 21:00:27.103 | INFO | BytesWritten : 1164343890781 | 20260507 21:00:27.103 | INFO | NumEvents : 564394506814 | 20260507 21:00:28.101 | INFO | Analyzing chunk frames = [2560000,2560512) | 20260507 21:00:28.101 | INFO | MemInfo: { | 20260507 21:00:28.101 | INFO | "gpu" : " 17 GB", | 20260507 21:00:28.101 | INFO | "huge" : "", | 20260507 21:00:28.101 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:28.102 | INFO | "malloc" : " 14 GB", | 20260507 21:00:28.102 | INFO | "pinned" : " 29 GB" | 20260507 21:00:28.102 | INFO | } | 20260507 21:00:28.102 | INFO | Perf report for the last 5.13933 seconds: | 20260507 21:00:28.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:28.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:28.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:28.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:28.102 | INFO | ANALYSIS : 69.9%, 2.8, 103.7ms, 0 | 20260507 21:00:28.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:28.102 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260507 21:00:28.102 | INFO | PRE_HQ : 68.9%, 1.9, 67.7ms, 0 | 20260507 21:00:28.102 | INFO | BAZWRITER : 77.9%, 4.8, 193.7ms, 0 | 20260507 21:00:28.102 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:00:28.103 | INFO | Uploaded 36GiB to GPU in 2.52353s (14.2658GiB/s) | 20260507 21:00:28.103 | INFO | Downloaded 38GiB to GPU in 2.66796s (14.2431GiB/s) | 20260507 21:00:33.241 | INFO | Analyzing chunk frames = [2560512,2561024) | 20260507 21:00:33.241 | INFO | MemInfo: { | 20260507 21:00:33.241 | INFO | "gpu" : " 17 GB", | 20260507 21:00:33.241 | INFO | "huge" : "", | 20260507 21:00:33.241 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:33.241 | INFO | "malloc" : " 14 GB", | 20260507 21:00:33.241 | INFO | "pinned" : " 29 GB" | 20260507 21:00:33.241 | INFO | } | 20260507 21:00:33.241 | INFO | Perf report for the last 5.13946 seconds: | 20260507 21:00:33.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:33.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.241 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 21:00:33.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.241 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:00:33.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:33.242 | INFO | Uploaded 36GiB to GPU in 2.59429s (13.8766GiB/s) | 20260507 21:00:33.242 | INFO | Downloaded 34GiB to GPU in 2.47635s (13.7299GiB/s) | 20260507 21:00:38.380 | INFO | Analyzing chunk frames = [2561024,2561536) | 20260507 21:00:38.380 | INFO | MemInfo: { | 20260507 21:00:38.380 | INFO | "gpu" : " 17 GB", | 20260507 21:00:38.380 | INFO | "huge" : "", | 20260507 21:00:38.380 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:38.380 | INFO | "malloc" : " 14 GB", | 20260507 21:00:38.380 | INFO | "pinned" : " 29 GB" | 20260507 21:00:38.380 | INFO | } | 20260507 21:00:38.380 | INFO | Perf report for the last 5.13865 seconds: | 20260507 21:00:38.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:38.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:00:38.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:00:38.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:38.380 | INFO | Uploaded 36GiB to GPU in 2.51643s (14.306GiB/s) | 20260507 21:00:38.380 | INFO | Downloaded 34GiB to GPU in 2.48013s (13.709GiB/s) | 20260507 21:00:43.519 | INFO | Analyzing chunk frames = [2561536,2562048) | 20260507 21:00:43.519 | INFO | MemInfo: { | 20260507 21:00:43.519 | INFO | "gpu" : " 17 GB", | 20260507 21:00:43.519 | INFO | "huge" : "", | 20260507 21:00:43.519 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:43.519 | INFO | "malloc" : " 14 GB", | 20260507 21:00:43.519 | INFO | "pinned" : " 29 GB" | 20260507 21:00:43.519 | INFO | } | 20260507 21:00:43.519 | INFO | Perf report for the last 5.13922 seconds: | 20260507 21:00:43.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:43.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.519 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 21:00:43.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.519 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:00:43.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:43.520 | INFO | Uploaded 36GiB to GPU in 2.5192s (14.2902GiB/s) | 20260507 21:00:43.520 | INFO | Downloaded 34GiB to GPU in 2.49208s (13.6432GiB/s) | 20260507 21:00:48.659 | INFO | Analyzing chunk frames = [2562048,2562560) | 20260507 21:00:48.659 | INFO | MemInfo: { | 20260507 21:00:48.659 | INFO | "gpu" : " 17 GB", | 20260507 21:00:48.659 | INFO | "huge" : "", | 20260507 21:00:48.659 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:48.659 | INFO | "malloc" : " 14 GB", | 20260507 21:00:48.659 | INFO | "pinned" : " 29 GB" | 20260507 21:00:48.659 | INFO | } | 20260507 21:00:48.659 | INFO | Perf report for the last 5.13981 seconds: | 20260507 21:00:48.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:48.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 21:00:48.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 21:00:48.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:48.659 | INFO | Uploaded 36GiB to GPU in 2.4051s (14.9682GiB/s) | 20260507 21:00:48.659 | INFO | Downloaded 34GiB to GPU in 2.48423s (13.6863GiB/s) | 20260507 21:00:53.801 | INFO | Analyzing chunk frames = [2562560,2563072) | 20260507 21:00:53.801 | INFO | MemInfo: { | 20260507 21:00:53.801 | INFO | "gpu" : " 17 GB", | 20260507 21:00:53.801 | INFO | "huge" : "", | 20260507 21:00:53.801 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:53.801 | INFO | "malloc" : " 14 GB", | 20260507 21:00:53.801 | INFO | "pinned" : " 29 GB" | 20260507 21:00:53.801 | INFO | } | 20260507 21:00:53.801 | INFO | Perf report for the last 5.14177 seconds: | 20260507 21:00:53.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:53.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 21:00:53.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:00:53.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:53.801 | INFO | Uploaded 36GiB to GPU in 2.49573s (14.4246GiB/s) | 20260507 21:00:53.801 | INFO | Downloaded 34GiB to GPU in 2.42904s (13.9973GiB/s) | 20260507 21:00:58.938 | INFO | Analyzing chunk frames = [2563072,2563584) | 20260507 21:00:58.938 | INFO | MemInfo: { | 20260507 21:00:58.938 | INFO | "gpu" : " 17 GB", | 20260507 21:00:58.938 | INFO | "huge" : "", | 20260507 21:00:58.939 | INFO | "huge_shared" : " 29 GB", | 20260507 21:00:58.939 | INFO | "malloc" : " 14 GB", | 20260507 21:00:58.939 | INFO | "pinned" : " 29 GB" | 20260507 21:00:58.939 | INFO | } | 20260507 21:00:58.939 | INFO | Perf report for the last 5.13792 seconds: | 20260507 21:00:58.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:00:58.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:00:58.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:00:58.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:00:58.939 | INFO | Uploaded 36GiB to GPU in 2.54623s (14.1386GiB/s) | 20260507 21:00:58.939 | INFO | Downloaded 34GiB to GPU in 2.45796s (13.8326GiB/s) | 20260507 21:01:04.077 | INFO | Analyzing chunk frames = [2563584,2564096) | 20260507 21:01:04.077 | INFO | MemInfo: { | 20260507 21:01:04.077 | INFO | "gpu" : " 17 GB", | 20260507 21:01:04.077 | INFO | "huge" : "", | 20260507 21:01:04.077 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:04.077 | INFO | "malloc" : " 14 GB", | 20260507 21:01:04.077 | INFO | "pinned" : " 29 GB" | 20260507 21:01:04.077 | INFO | } | 20260507 21:01:04.077 | INFO | Perf report for the last 5.13825 seconds: | 20260507 21:01:04.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:04.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:01:04.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:01:04.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:04.077 | INFO | Uploaded 36GiB to GPU in 2.50702s (14.3597GiB/s) | 20260507 21:01:04.077 | INFO | Downloaded 34GiB to GPU in 2.43373s (13.9703GiB/s) | 20260507 21:01:04.077 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:01:04.077 | INFO | Total mem usage: 16.7 GiB | 20260507 21:01:04.077 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:01:04.077 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: 0.0461%) | 20260507 21:01:04.077 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:01:04.077 | INFO | High Water Breakdown: | 20260507 21:01:04.077 | INFO | 61.7% : 10.3 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:01:04.077 | INFO | 38.2% : 6.37 GiB ( 94.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:01:04.077 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:01:04.077 | INFO | ------------------------------------------ | 20260507 21:01:04.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:01:04.077 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:01:04.077 | INFO | Total mem usage: 26.5 GiB | 20260507 21:01:04.077 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:01:04.077 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 21:01:04.077 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 21:01:04.077 | INFO | High Water Breakdown: | 20260507 21:01:04.077 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:01:04.077 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:01:04.077 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:01:04.077 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:01:04.077 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:01:04.077 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:01:04.077 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:01:04.077 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:01:04.077 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:01:04.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:01:04.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:01:04.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:01:04.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:01:04.078 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:01:04.078 | INFO | ------------------------------------------ | 20260507 21:01:04.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:01:04.078 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:01:04.078 | INFO | Total mem usage: 27.8 GiB | 20260507 21:01:04.078 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:01:04.078 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:01:04.078 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:01:04.078 | INFO | High Water Breakdown: | 20260507 21:01:04.078 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:01:04.078 | INFO | ------------------------------------------ | 20260507 21:01:04.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:01:04.078 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:01:04.078 | INFO | Total mem usage: 15.7 GiB | 20260507 21:01:04.078 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:01:04.078 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:01:04.078 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:01:04.078 | INFO | High Water Breakdown: | 20260507 21:01:04.078 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:01:04.078 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:01:04.078 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:01:04.078 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:01:04.078 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:01:04.078 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:01:04.078 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:01:04.078 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:01:04.078 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:01:04.078 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:01:04.078 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:01:04.078 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:01:04.078 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:01:04.078 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:01:04.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:01:04.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:01:04.078 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:01:04.078 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:01:04.078 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:01:04.078 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:01:04.078 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:01:04.078 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:01:04.078 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:01:04.078 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:01:04.078 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:01:04.078 | INFO | ------------------------------------------ | 20260507 21:01:04.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:01:04.078 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:01:04.078 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:01:04.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:01:04.078 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.44e+01 / 2.15e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.04e-01, 1.19e-02 / 7.17e-03 | 20260507 21:01:04.078 | INFO | PerfReport: 12.12%, Baseline -- 4.00e+00 / 4.44e+00, 5.87e-01 / 2.89e-01 | 20260507 21:01:04.078 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.50e+00, 2.07e-01 / 1.15e-01 | 20260507 21:01:04.078 | INFO | PerfReport: 16.14%, DME -- 4.26e+01 / 4.61e+01, 4.15e+00 / 2.21e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 55.78%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.60e+00 / 1.33e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.52e+00 / 2.77e+00, 4.42e-01 / 2.56e-01 | 20260507 21:01:04.078 | INFO | PerfReport: 3.57%, Metrics -- 1.18e+00 / 1.38e+00, 2.92e-01 / 1.67e-01 | 20260507 21:01:04.078 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.65e-01 / 3.99e-01, 5.86e-02 / 2.85e-02 | 20260507 21:01:04.078 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:01:04.078 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:01:04.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:01:04.078 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 24.34%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 30.86%, Compute -- 3.30e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 2.15%, LockUpload -- 2.30e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 2.63%, LockDownload -- 2.81e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 21:01:04.078 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 21:01:04.174 | INFO | Starting a round of Baz IO | 20260507 21:01:07.646 | INFO | Populating RTMetrics report | 20260507 21:01:07.646 | INFO | RTMetrics report populated | 20260507 21:01:07.646 | INFO | Opening tmp RTMetrics json file | 20260507 21:01:07.669 | INFO | Serializing RTMetrics to json | 20260507 21:01:07.673 | INFO | Serialization done, swapping tmp file now... | 20260507 21:01:07.678 | INFO | RTMetrics done | 20260507 21:01:07.744 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.57 seconds | 20260507 21:01:07.744 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.03% / 0.01% / 55.96% | 20260507 21:01:07.744 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.82163% / 2.38419e-05% / 3.17891e-05% | 20260507 21:01:07.744 | INFO | Baz IO overall: | 20260507 21:01:07.745 | INFO | BytesWritten : 1166242701919 | 20260507 21:01:07.745 | INFO | NumEvents : 565313293680 | 20260507 21:01:09.216 | INFO | Analyzing chunk frames = [2564096,2564608) | 20260507 21:01:09.217 | INFO | MemInfo: { | 20260507 21:01:09.217 | INFO | "gpu" : " 17 GB", | 20260507 21:01:09.217 | INFO | "huge" : "", | 20260507 21:01:09.217 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:09.217 | INFO | "malloc" : " 14 GB", | 20260507 21:01:09.217 | INFO | "pinned" : " 29 GB" | 20260507 21:01:09.217 | INFO | } | 20260507 21:01:09.217 | INFO | Perf report for the last 5.13961 seconds: | 20260507 21:01:09.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:09.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:09.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:09.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:09.217 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260507 21:01:09.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:09.217 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 21:01:09.217 | INFO | PRE_HQ : 65.9%, 1.7, 58.2ms, 0 | 20260507 21:01:09.217 | INFO | BAZWRITER : 69.1%, 4.7, 167.8ms, 0 | 20260507 21:01:09.217 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 21:01:09.217 | INFO | Uploaded 36GiB to GPU in 2.51569s (14.3102GiB/s) | 20260507 21:01:09.217 | INFO | Downloaded 38GiB to GPU in 2.68119s (14.1728GiB/s) | 20260507 21:01:14.356 | INFO | Analyzing chunk frames = [2564608,2565120) | 20260507 21:01:14.356 | INFO | MemInfo: { | 20260507 21:01:14.356 | INFO | "gpu" : " 17 GB", | 20260507 21:01:14.356 | INFO | "huge" : "", | 20260507 21:01:14.356 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:14.356 | INFO | "malloc" : " 14 GB", | 20260507 21:01:14.356 | INFO | "pinned" : " 29 GB" | 20260507 21:01:14.356 | INFO | } | 20260507 21:01:14.356 | INFO | Perf report for the last 5.13944 seconds: | 20260507 21:01:14.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:14.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.356 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:01:14.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.356 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:01:14.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:14.357 | INFO | Uploaded 36GiB to GPU in 2.55877s (14.0693GiB/s) | 20260507 21:01:14.357 | INFO | Downloaded 34GiB to GPU in 2.50226s (13.5877GiB/s) | 20260507 21:01:19.496 | INFO | Analyzing chunk frames = [2565120,2565632) | 20260507 21:01:19.496 | INFO | MemInfo: { | 20260507 21:01:19.496 | INFO | "gpu" : " 17 GB", | 20260507 21:01:19.496 | INFO | "huge" : "", | 20260507 21:01:19.496 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:19.496 | INFO | "malloc" : " 14 GB", | 20260507 21:01:19.496 | INFO | "pinned" : " 29 GB" | 20260507 21:01:19.496 | INFO | } | 20260507 21:01:19.496 | INFO | Perf report for the last 5.14011 seconds: | 20260507 21:01:19.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:19.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.496 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:01:19.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.496 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:01:19.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:19.497 | INFO | Uploaded 36GiB to GPU in 2.47715s (14.5328GiB/s) | 20260507 21:01:19.497 | INFO | Downloaded 34GiB to GPU in 2.47573s (13.7333GiB/s) | 20260507 21:01:24.635 | INFO | Analyzing chunk frames = [2565632,2566144) | 20260507 21:01:24.635 | INFO | MemInfo: { | 20260507 21:01:24.635 | INFO | "gpu" : " 17 GB", | 20260507 21:01:24.635 | INFO | "huge" : "", | 20260507 21:01:24.635 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:24.635 | INFO | "malloc" : " 14 GB", | 20260507 21:01:24.635 | INFO | "pinned" : " 29 GB" | 20260507 21:01:24.635 | INFO | } | 20260507 21:01:24.635 | INFO | Perf report for the last 5.139 seconds: | 20260507 21:01:24.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:24.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.635 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:01:24.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.635 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:01:24.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:24.636 | INFO | Uploaded 36GiB to GPU in 2.50527s (14.3697GiB/s) | 20260507 21:01:24.636 | INFO | Downloaded 34GiB to GPU in 2.49336s (13.6362GiB/s) | 20260507 21:01:29.774 | INFO | Analyzing chunk frames = [2566144,2566656) | 20260507 21:01:29.774 | INFO | MemInfo: { | 20260507 21:01:29.774 | INFO | "gpu" : " 17 GB", | 20260507 21:01:29.774 | INFO | "huge" : "", | 20260507 21:01:29.774 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:29.774 | INFO | "malloc" : " 14 GB", | 20260507 21:01:29.774 | INFO | "pinned" : " 29 GB" | 20260507 21:01:29.774 | INFO | } | 20260507 21:01:29.774 | INFO | Perf report for the last 5.13912 seconds: | 20260507 21:01:29.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:29.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.774 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 21:01:29.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.775 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:01:29.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:29.775 | INFO | Uploaded 36GiB to GPU in 2.42975s (14.8163GiB/s) | 20260507 21:01:29.775 | INFO | Downloaded 34GiB to GPU in 2.49316s (13.6373GiB/s) | 20260507 21:01:34.913 | INFO | Analyzing chunk frames = [2566656,2567168) | 20260507 21:01:34.913 | INFO | MemInfo: { | 20260507 21:01:34.913 | INFO | "gpu" : " 17 GB", | 20260507 21:01:34.913 | INFO | "huge" : "", | 20260507 21:01:34.913 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:34.913 | INFO | "malloc" : " 14 GB", | 20260507 21:01:34.913 | INFO | "pinned" : " 29 GB" | 20260507 21:01:34.913 | INFO | } | 20260507 21:01:34.913 | INFO | Perf report for the last 5.13862 seconds: | 20260507 21:01:34.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:34.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.913 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 21:01:34.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.913 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 21:01:34.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:34.914 | INFO | Uploaded 36GiB to GPU in 2.51116s (14.336GiB/s) | 20260507 21:01:34.914 | INFO | Downloaded 34GiB to GPU in 2.42801s (14.0033GiB/s) | 20260507 21:01:40.053 | INFO | Analyzing chunk frames = [2567168,2567680) | 20260507 21:01:40.053 | INFO | MemInfo: { | 20260507 21:01:40.053 | INFO | "gpu" : " 17 GB", | 20260507 21:01:40.053 | INFO | "huge" : "", | 20260507 21:01:40.053 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:40.053 | INFO | "malloc" : " 14 GB", | 20260507 21:01:40.053 | INFO | "pinned" : " 29 GB" | 20260507 21:01:40.053 | INFO | } | 20260507 21:01:40.053 | INFO | Perf report for the last 5.13994 seconds: | 20260507 21:01:40.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:40.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 21:01:40.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | PRE_HQ : 36.8%, 1.0, 19.2ms, 0 | 20260507 21:01:40.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:40.053 | INFO | Uploaded 36GiB to GPU in 2.56005s (14.0623GiB/s) | 20260507 21:01:40.053 | INFO | Downloaded 34GiB to GPU in 2.46277s (13.8056GiB/s) | 20260507 21:01:45.191 | INFO | Analyzing chunk frames = [2567680,2568192) | 20260507 21:01:45.191 | INFO | MemInfo: { | 20260507 21:01:45.191 | INFO | "gpu" : " 17 GB", | 20260507 21:01:45.191 | INFO | "huge" : "", | 20260507 21:01:45.191 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:45.191 | INFO | "malloc" : " 14 GB", | 20260507 21:01:45.191 | INFO | "pinned" : " 29 GB" | 20260507 21:01:45.191 | INFO | } | 20260507 21:01:45.192 | INFO | Perf report for the last 5.13857 seconds: | 20260507 21:01:45.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:45.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:01:45.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 21:01:45.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:45.192 | INFO | Uploaded 36GiB to GPU in 2.52299s (14.2688GiB/s) | 20260507 21:01:45.192 | INFO | Downloaded 34GiB to GPU in 2.43782s (13.9469GiB/s) | 20260507 21:01:45.306 | INFO | Starting a round of Baz IO | 20260507 21:01:48.813 | INFO | Populating RTMetrics report | 20260507 21:01:48.814 | INFO | RTMetrics report populated | 20260507 21:01:48.814 | INFO | Opening tmp RTMetrics json file | 20260507 21:01:48.822 | INFO | Serializing RTMetrics to json | 20260507 21:01:48.850 | INFO | Serialization done, swapping tmp file now... | 20260507 21:01:48.863 | INFO | RTMetrics done | 20260507 21:01:48.974 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.668 seconds | 20260507 21:01:48.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.02% / 0.01% / 55.97% | 20260507 21:01:48.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.81759% / 7.94729e-06% / 3.97364e-05% | 20260507 21:01:48.975 | INFO | Baz IO overall: | 20260507 21:01:48.975 | INFO | BytesWritten : 1168140337230 | 20260507 21:01:48.975 | INFO | NumEvents : 566231458122 | 20260507 21:01:50.330 | INFO | Analyzing chunk frames = [2568192,2568704) | 20260507 21:01:50.330 | INFO | MemInfo: { | 20260507 21:01:50.330 | INFO | "gpu" : " 17 GB", | 20260507 21:01:50.330 | INFO | "huge" : "", | 20260507 21:01:50.330 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:50.330 | INFO | "malloc" : " 14 GB", | 20260507 21:01:50.330 | INFO | "pinned" : " 29 GB" | 20260507 21:01:50.330 | INFO | } | 20260507 21:01:50.330 | INFO | Perf report for the last 5.1388 seconds: | 20260507 21:01:50.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:50.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:50.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:50.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:50.330 | INFO | ANALYSIS : 70.1%, 2.9, 104.9ms, 0 | 20260507 21:01:50.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:50.330 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260507 21:01:50.330 | INFO | PRE_HQ : 69.5%, 1.9, 69.9ms, 0 | 20260507 21:01:50.330 | INFO | BAZWRITER : 71.0%, 4.8, 175.7ms, 0 | 20260507 21:01:50.330 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 21:01:50.331 | INFO | Uploaded 36GiB to GPU in 2.55127s (14.1106GiB/s) | 20260507 21:01:50.331 | INFO | Downloaded 38GiB to GPU in 2.72161s (13.9623GiB/s) | 20260507 21:01:55.470 | INFO | Analyzing chunk frames = [2568704,2569216) | 20260507 21:01:55.470 | INFO | MemInfo: { | 20260507 21:01:55.470 | INFO | "gpu" : " 17 GB", | 20260507 21:01:55.470 | INFO | "huge" : "", | 20260507 21:01:55.470 | INFO | "huge_shared" : " 29 GB", | 20260507 21:01:55.471 | INFO | "malloc" : " 14 GB", | 20260507 21:01:55.471 | INFO | "pinned" : " 29 GB" | 20260507 21:01:55.471 | INFO | } | 20260507 21:01:55.471 | INFO | Perf report for the last 5.14022 seconds: | 20260507 21:01:55.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:01:55.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:01:55.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 21:01:55.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:01:55.471 | INFO | Uploaded 36GiB to GPU in 2.52352s (14.2658GiB/s) | 20260507 21:01:55.471 | INFO | Downloaded 34GiB to GPU in 2.50082s (13.5955GiB/s) | 20260507 21:02:00.610 | INFO | Analyzing chunk frames = [2569216,2569728) | 20260507 21:02:00.610 | INFO | MemInfo: { | 20260507 21:02:00.610 | INFO | "gpu" : " 17 GB", | 20260507 21:02:00.610 | INFO | "huge" : "", | 20260507 21:02:00.610 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:00.611 | INFO | "malloc" : " 14 GB", | 20260507 21:02:00.611 | INFO | "pinned" : " 29 GB" | 20260507 21:02:00.611 | INFO | } | 20260507 21:02:00.611 | INFO | Perf report for the last 5.14002 seconds: | 20260507 21:02:00.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:00.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:02:00.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:02:00.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:00.611 | INFO | Uploaded 36GiB to GPU in 2.45168s (14.6838GiB/s) | 20260507 21:02:00.611 | INFO | Downloaded 34GiB to GPU in 2.49763s (13.6129GiB/s) | 20260507 21:02:05.751 | INFO | Analyzing chunk frames = [2569728,2570240) | 20260507 21:02:05.751 | INFO | MemInfo: { | 20260507 21:02:05.751 | INFO | "gpu" : " 17 GB", | 20260507 21:02:05.751 | INFO | "huge" : "", | 20260507 21:02:05.751 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:05.751 | INFO | "malloc" : " 14 GB", | 20260507 21:02:05.751 | INFO | "pinned" : " 29 GB" | 20260507 21:02:05.751 | INFO | } | 20260507 21:02:05.751 | INFO | Perf report for the last 5.14068 seconds: | 20260507 21:02:05.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:05.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.751 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 21:02:05.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.751 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:02:05.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:05.752 | INFO | Uploaded 36GiB to GPU in 2.45732s (14.6501GiB/s) | 20260507 21:02:05.752 | INFO | Downloaded 34GiB to GPU in 2.50672s (13.5636GiB/s) | 20260507 21:02:10.889 | INFO | Analyzing chunk frames = [2570240,2570752) | 20260507 21:02:10.889 | INFO | MemInfo: { | 20260507 21:02:10.889 | INFO | "gpu" : " 17 GB", | 20260507 21:02:10.889 | INFO | "huge" : "", | 20260507 21:02:10.889 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:10.889 | INFO | "malloc" : " 14 GB", | 20260507 21:02:10.889 | INFO | "pinned" : " 29 GB" | 20260507 21:02:10.889 | INFO | } | 20260507 21:02:10.890 | INFO | Perf report for the last 5.13835 seconds: | 20260507 21:02:10.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:10.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:02:10.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:02:10.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:10.890 | INFO | Uploaded 36GiB to GPU in 2.30675s (15.6063GiB/s) | 20260507 21:02:10.890 | INFO | Downloaded 34GiB to GPU in 2.48299s (13.6931GiB/s) | 20260507 21:02:16.027 | INFO | Analyzing chunk frames = [2570752,2571264) | 20260507 21:02:16.027 | INFO | MemInfo: { | 20260507 21:02:16.027 | INFO | "gpu" : " 17 GB", | 20260507 21:02:16.027 | INFO | "huge" : "", | 20260507 21:02:16.027 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:16.027 | INFO | "malloc" : " 14 GB", | 20260507 21:02:16.027 | INFO | "pinned" : " 29 GB" | 20260507 21:02:16.027 | INFO | } | 20260507 21:02:16.027 | INFO | Perf report for the last 5.13749 seconds: | 20260507 21:02:16.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:16.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.027 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:02:16.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.027 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:02:16.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:16.028 | INFO | Uploaded 36GiB to GPU in 2.39279s (15.0452GiB/s) | 20260507 21:02:16.028 | INFO | Downloaded 34GiB to GPU in 2.39211s (14.2134GiB/s) | 20260507 21:02:21.167 | INFO | Analyzing chunk frames = [2571264,2571776) | 20260507 21:02:21.167 | INFO | MemInfo: { | 20260507 21:02:21.167 | INFO | "gpu" : " 17 GB", | 20260507 21:02:21.167 | INFO | "huge" : "", | 20260507 21:02:21.167 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:21.167 | INFO | "malloc" : " 14 GB", | 20260507 21:02:21.167 | INFO | "pinned" : " 29 GB" | 20260507 21:02:21.167 | INFO | } | 20260507 21:02:21.167 | INFO | Perf report for the last 5.13967 seconds: | 20260507 21:02:21.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:21.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.167 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:02:21.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.167 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:02:21.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:21.168 | INFO | Uploaded 36GiB to GPU in 2.51711s (14.3021GiB/s) | 20260507 21:02:21.168 | INFO | Downloaded 34GiB to GPU in 2.45462s (13.8514GiB/s) | 20260507 21:02:26.306 | INFO | Analyzing chunk frames = [2571776,2572288) | 20260507 21:02:26.306 | INFO | MemInfo: { | 20260507 21:02:26.306 | INFO | "gpu" : " 17 GB", | 20260507 21:02:26.306 | INFO | "huge" : "", | 20260507 21:02:26.306 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:26.306 | INFO | "malloc" : " 14 GB", | 20260507 21:02:26.306 | INFO | "pinned" : " 29 GB" | 20260507 21:02:26.306 | INFO | } | 20260507 21:02:26.306 | INFO | Perf report for the last 5.13951 seconds: | 20260507 21:02:26.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:26.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.306 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:02:26.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.307 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 21:02:26.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:26.307 | INFO | Uploaded 36GiB to GPU in 2.50838s (14.3519GiB/s) | 20260507 21:02:26.307 | INFO | Downloaded 34GiB to GPU in 2.43695s (13.9519GiB/s) | 20260507 21:02:26.307 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:02:26.307 | INFO | Total mem usage: 16.7 GiB | 20260507 21:02:26.307 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:02:26.307 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -3.3%) | 20260507 21:02:26.307 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:02:26.307 | INFO | High Water Breakdown: | 20260507 21:02:26.307 | INFO | 61.2% : 9.89 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:02:26.307 | INFO | 38.6% : 6.24 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:02:26.307 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:02:26.307 | INFO | ------------------------------------------ | 20260507 21:02:26.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:02:26.307 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:02:26.307 | INFO | Total mem usage: 26.5 GiB | 20260507 21:02:26.307 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:02:26.307 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 21:02:26.307 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 21:02:26.307 | INFO | High Water Breakdown: | 20260507 21:02:26.307 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:02:26.307 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:02:26.307 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:02:26.307 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:02:26.307 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:02:26.307 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:02:26.307 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:02:26.307 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:02:26.307 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:02:26.307 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:02:26.307 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:02:26.307 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:02:26.307 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:02:26.307 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:02:26.307 | INFO | ------------------------------------------ | 20260507 21:02:26.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:02:26.307 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:02:26.307 | INFO | Total mem usage: 27.8 GiB | 20260507 21:02:26.307 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:02:26.307 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:02:26.307 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:02:26.307 | INFO | High Water Breakdown: | 20260507 21:02:26.307 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:02:26.307 | INFO | ------------------------------------------ | 20260507 21:02:26.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:02:26.307 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:02:26.307 | INFO | Total mem usage: 15.7 GiB | 20260507 21:02:26.307 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:02:26.307 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:02:26.307 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:02:26.307 | INFO | High Water Breakdown: | 20260507 21:02:26.307 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:02:26.307 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:02:26.307 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:02:26.307 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:02:26.307 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:02:26.307 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:02:26.307 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:02:26.307 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:02:26.307 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:02:26.307 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:02:26.307 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:02:26.307 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:02:26.307 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:02:26.307 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:02:26.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:02:26.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:02:26.307 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:02:26.307 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:02:26.307 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:02:26.307 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:02:26.307 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:02:26.307 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:02:26.307 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:02:26.307 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:02:26.307 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:02:26.307 | INFO | ------------------------------------------ | 20260507 21:02:26.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:02:26.307 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:02:26.307 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:02:26.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:02:26.307 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.43e+01 / 2.04e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 0.30%, AggregateStats -- 9.92e-02 / 1.03e-01, 1.25e-02 / 7.00e-03 | 20260507 21:02:26.307 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.44e+00, 5.94e-01 / 2.91e-01 | 20260507 21:02:26.307 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 1.12e-01 | 20260507 21:02:26.307 | INFO | PerfReport: 16.11%, DME -- 4.25e+01 / 4.61e+01, 4.41e+00 / 2.13e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 55.78%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.59e+00 / 1.35e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 2.82e+00, 4.44e-01 / 2.40e-01 | 20260507 21:02:26.307 | INFO | PerfReport: 3.56%, Metrics -- 1.18e+00 / 1.39e+00, 2.95e-01 / 1.56e-01 | 20260507 21:02:26.307 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.66e-01 / 4.02e-01, 5.96e-02 / 2.96e-02 | 20260507 21:02:26.307 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:02:26.307 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:02:26.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:02:26.307 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 24.24%, Upload -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 24.35%, Download -- 2.60e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 30.89%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 1.99%, LockUpload -- 2.13e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 2.84%, LockDownload -- 3.03e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 15.68%, LockCompute -- 1.68e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 21:02:26.307 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260507 21:02:26.404 | INFO | Starting a round of Baz IO | 20260507 21:02:29.876 | INFO | Populating RTMetrics report | 20260507 21:02:29.877 | INFO | RTMetrics report populated | 20260507 21:02:29.877 | INFO | Opening tmp RTMetrics json file | 20260507 21:02:29.893 | INFO | Serializing RTMetrics to json | 20260507 21:02:29.907 | INFO | Serialization done, swapping tmp file now... | 20260507 21:02:29.916 | INFO | RTMetrics done | 20260507 21:02:29.979 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.574 seconds | 20260507 21:02:29.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.01% / 0.01% / 55.98% | 20260507 21:02:29.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.81557% / 1.19209e-05% / 4.37101e-05% | 20260507 21:02:29.979 | INFO | Baz IO overall: | 20260507 21:02:29.979 | INFO | BytesWritten : 1170038300896 | 20260507 21:02:29.979 | INFO | NumEvents : 567149771469 | 20260507 21:02:31.447 | INFO | Analyzing chunk frames = [2572288,2572800) | 20260507 21:02:31.447 | INFO | MemInfo: { | 20260507 21:02:31.447 | INFO | "gpu" : " 17 GB", | 20260507 21:02:31.447 | INFO | "huge" : "", | 20260507 21:02:31.447 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:31.447 | INFO | "malloc" : " 14 GB", | 20260507 21:02:31.447 | INFO | "pinned" : " 29 GB" | 20260507 21:02:31.447 | INFO | } | 20260507 21:02:31.447 | INFO | Perf report for the last 5.14022 seconds: | 20260507 21:02:31.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:31.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:31.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:31.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:31.447 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260507 21:02:31.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:31.447 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 21:02:31.447 | INFO | PRE_HQ : 65.7%, 1.7, 58.0ms, 0 | 20260507 21:02:31.447 | INFO | BAZWRITER : 69.2%, 3.8, 133.8ms, 0 | 20260507 21:02:31.447 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 21:02:31.447 | INFO | Uploaded 36GiB to GPU in 2.51086s (14.3377GiB/s) | 20260507 21:02:31.447 | INFO | Downloaded 38GiB to GPU in 2.6866s (14.1443GiB/s) | 20260507 21:02:36.584 | INFO | Analyzing chunk frames = [2572800,2573312) | 20260507 21:02:36.584 | INFO | MemInfo: { | 20260507 21:02:36.584 | INFO | "gpu" : " 17 GB", | 20260507 21:02:36.584 | INFO | "huge" : "", | 20260507 21:02:36.584 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:36.584 | INFO | "malloc" : " 14 GB", | 20260507 21:02:36.584 | INFO | "pinned" : " 29 GB" | 20260507 21:02:36.584 | INFO | } | 20260507 21:02:36.584 | INFO | Perf report for the last 5.1378 seconds: | 20260507 21:02:36.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:36.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.585 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:02:36.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.585 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:02:36.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:36.585 | INFO | Uploaded 36GiB to GPU in 2.5774s (13.9675GiB/s) | 20260507 21:02:36.585 | INFO | Downloaded 34GiB to GPU in 2.49277s (13.6394GiB/s) | 20260507 21:02:41.724 | INFO | Analyzing chunk frames = [2573312,2573824) | 20260507 21:02:41.724 | INFO | MemInfo: { | 20260507 21:02:41.724 | INFO | "gpu" : " 17 GB", | 20260507 21:02:41.724 | INFO | "huge" : "", | 20260507 21:02:41.724 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:41.724 | INFO | "malloc" : " 14 GB", | 20260507 21:02:41.724 | INFO | "pinned" : " 29 GB" | 20260507 21:02:41.724 | INFO | } | 20260507 21:02:41.724 | INFO | Perf report for the last 5.13961 seconds: | 20260507 21:02:41.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:41.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:02:41.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:02:41.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:41.724 | INFO | Uploaded 36GiB to GPU in 2.58384s (13.9327GiB/s) | 20260507 21:02:41.724 | INFO | Downloaded 34GiB to GPU in 2.47014s (13.7644GiB/s) | 20260507 21:02:46.865 | INFO | Analyzing chunk frames = [2573824,2574336) | 20260507 21:02:46.865 | INFO | MemInfo: { | 20260507 21:02:46.865 | INFO | "gpu" : " 17 GB", | 20260507 21:02:46.865 | INFO | "huge" : "", | 20260507 21:02:46.865 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:46.865 | INFO | "malloc" : " 14 GB", | 20260507 21:02:46.865 | INFO | "pinned" : " 29 GB" | 20260507 21:02:46.865 | INFO | } | 20260507 21:02:46.865 | INFO | Perf report for the last 5.14051 seconds: | 20260507 21:02:46.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:46.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:02:46.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:02:46.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:46.865 | INFO | Uploaded 36GiB to GPU in 2.50055s (14.3968GiB/s) | 20260507 21:02:46.865 | INFO | Downloaded 34GiB to GPU in 2.48116s (13.7033GiB/s) | 20260507 21:02:52.003 | INFO | Analyzing chunk frames = [2574336,2574848) | 20260507 21:02:52.003 | INFO | MemInfo: { | 20260507 21:02:52.003 | INFO | "gpu" : " 17 GB", | 20260507 21:02:52.003 | INFO | "huge" : "", | 20260507 21:02:52.003 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:52.003 | INFO | "malloc" : " 14 GB", | 20260507 21:02:52.004 | INFO | "pinned" : " 29 GB" | 20260507 21:02:52.004 | INFO | } | 20260507 21:02:52.004 | INFO | Perf report for the last 5.13896 seconds: | 20260507 21:02:52.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:52.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:02:52.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 21:02:52.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:52.004 | INFO | Uploaded 36GiB to GPU in 2.47064s (14.5711GiB/s) | 20260507 21:02:52.004 | INFO | Downloaded 34GiB to GPU in 2.49922s (13.6042GiB/s) | 20260507 21:02:57.143 | INFO | Analyzing chunk frames = [2574848,2575360) | 20260507 21:02:57.143 | INFO | MemInfo: { | 20260507 21:02:57.143 | INFO | "gpu" : " 17 GB", | 20260507 21:02:57.143 | INFO | "huge" : "", | 20260507 21:02:57.143 | INFO | "huge_shared" : " 29 GB", | 20260507 21:02:57.143 | INFO | "malloc" : " 14 GB", | 20260507 21:02:57.143 | INFO | "pinned" : " 29 GB" | 20260507 21:02:57.143 | INFO | } | 20260507 21:02:57.144 | INFO | Perf report for the last 5.14 seconds: | 20260507 21:02:57.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:02:57.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:02:57.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:02:57.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:02:57.144 | INFO | Uploaded 36GiB to GPU in 2.38481s (15.0955GiB/s) | 20260507 21:02:57.144 | INFO | Downloaded 34GiB to GPU in 2.39286s (14.2089GiB/s) | 20260507 21:03:02.281 | INFO | Analyzing chunk frames = [2575360,2575872) | 20260507 21:03:02.281 | INFO | MemInfo: { | 20260507 21:03:02.281 | INFO | "gpu" : " 17 GB", | 20260507 21:03:02.281 | INFO | "huge" : "", | 20260507 21:03:02.281 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:02.281 | INFO | "malloc" : " 14 GB", | 20260507 21:03:02.281 | INFO | "pinned" : " 29 GB" | 20260507 21:03:02.282 | INFO | } | 20260507 21:03:02.282 | INFO | Perf report for the last 5.13823 seconds: | 20260507 21:03:02.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:02.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:03:02.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 21:03:02.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:02.282 | INFO | Uploaded 36GiB to GPU in 2.54036s (14.1712GiB/s) | 20260507 21:03:02.282 | INFO | Downloaded 34GiB to GPU in 2.45942s (13.8244GiB/s) | 20260507 21:03:07.421 | INFO | Analyzing chunk frames = [2575872,2576384) | 20260507 21:03:07.421 | INFO | MemInfo: { | 20260507 21:03:07.421 | INFO | "gpu" : " 17 GB", | 20260507 21:03:07.421 | INFO | "huge" : "", | 20260507 21:03:07.421 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:07.421 | INFO | "malloc" : " 14 GB", | 20260507 21:03:07.421 | INFO | "pinned" : " 29 GB" | 20260507 21:03:07.421 | INFO | } | 20260507 21:03:07.421 | INFO | Perf report for the last 5.1394 seconds: | 20260507 21:03:07.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:07.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.421 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:03:07.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.421 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 21:03:07.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:07.422 | INFO | Uploaded 36GiB to GPU in 2.4862s (14.48GiB/s) | 20260507 21:03:07.422 | INFO | Downloaded 34GiB to GPU in 2.43137s (13.9839GiB/s) | 20260507 21:03:07.518 | INFO | Starting a round of Baz IO | 20260507 21:03:11.006 | INFO | Populating RTMetrics report | 20260507 21:03:11.006 | INFO | RTMetrics report populated | 20260507 21:03:11.006 | INFO | Opening tmp RTMetrics json file | 20260507 21:03:11.027 | INFO | Serializing RTMetrics to json | 20260507 21:03:11.051 | INFO | Serialization done, swapping tmp file now... | 20260507 21:03:11.094 | INFO | RTMetrics done | 20260507 21:03:11.128 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.609 seconds | 20260507 21:03:11.128 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.01% / 0.01% / 55.99% | 20260507 21:03:11.128 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.81555% / 1.58946e-05% / 5.5631e-05% | 20260507 21:03:11.128 | INFO | Baz IO overall: | 20260507 21:03:11.128 | INFO | BytesWritten : 1171936787665 | 20260507 21:03:11.128 | INFO | NumEvents : 568068593492 | 20260507 21:03:12.561 | INFO | Analyzing chunk frames = [2576384,2576896) | 20260507 21:03:12.561 | INFO | MemInfo: { | 20260507 21:03:12.561 | INFO | "gpu" : " 17 GB", | 20260507 21:03:12.561 | INFO | "huge" : "", | 20260507 21:03:12.561 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:12.562 | INFO | "malloc" : " 14 GB", | 20260507 21:03:12.562 | INFO | "pinned" : " 29 GB" | 20260507 21:03:12.562 | INFO | } | 20260507 21:03:12.562 | INFO | Perf report for the last 5.14044 seconds: | 20260507 21:03:12.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:12.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:12.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:12.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:12.562 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 21:03:12.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:12.562 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 21:03:12.562 | INFO | PRE_HQ : 66.2%, 1.7, 57.9ms, 0 | 20260507 21:03:12.562 | INFO | BAZWRITER : 69.8%, 3.9, 139.1ms, 0 | 20260507 21:03:12.562 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260507 21:03:12.562 | INFO | Uploaded 36GiB to GPU in 2.55249s (14.1039GiB/s) | 20260507 21:03:12.562 | INFO | Downloaded 38GiB to GPU in 2.68144s (14.1715GiB/s) | 20260507 21:03:17.700 | INFO | Analyzing chunk frames = [2576896,2577408) | 20260507 21:03:17.700 | INFO | MemInfo: { | 20260507 21:03:17.700 | INFO | "gpu" : " 17 GB", | 20260507 21:03:17.700 | INFO | "huge" : "", | 20260507 21:03:17.700 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:17.700 | INFO | "malloc" : " 14 GB", | 20260507 21:03:17.700 | INFO | "pinned" : " 29 GB" | 20260507 21:03:17.700 | INFO | } | 20260507 21:03:17.700 | INFO | Perf report for the last 5.13834 seconds: | 20260507 21:03:17.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:17.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.700 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:03:17.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.700 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:03:17.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:17.701 | INFO | Uploaded 36GiB to GPU in 2.57649s (13.9725GiB/s) | 20260507 21:03:17.701 | INFO | Downloaded 34GiB to GPU in 2.50056s (13.597GiB/s) | 20260507 21:03:22.838 | INFO | Analyzing chunk frames = [2577408,2577920) | 20260507 21:03:22.838 | INFO | MemInfo: { | 20260507 21:03:22.838 | INFO | "gpu" : " 17 GB", | 20260507 21:03:22.838 | INFO | "huge" : "", | 20260507 21:03:22.838 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:22.838 | INFO | "malloc" : " 14 GB", | 20260507 21:03:22.838 | INFO | "pinned" : " 29 GB" | 20260507 21:03:22.838 | INFO | } | 20260507 21:03:22.838 | INFO | Perf report for the last 5.13843 seconds: | 20260507 21:03:22.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:22.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:03:22.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:03:22.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:22.839 | INFO | Uploaded 36GiB to GPU in 2.49264s (14.4425GiB/s) | 20260507 21:03:22.839 | INFO | Downloaded 34GiB to GPU in 2.47895s (13.7155GiB/s) | 20260507 21:03:27.979 | INFO | Analyzing chunk frames = [2577920,2578432) | 20260507 21:03:27.979 | INFO | MemInfo: { | 20260507 21:03:27.979 | INFO | "gpu" : " 17 GB", | 20260507 21:03:27.979 | INFO | "huge" : "", | 20260507 21:03:27.979 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:27.979 | INFO | "malloc" : " 14 GB", | 20260507 21:03:27.979 | INFO | "pinned" : " 29 GB" | 20260507 21:03:27.979 | INFO | } | 20260507 21:03:27.979 | INFO | Perf report for the last 5.14072 seconds: | 20260507 21:03:27.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:27.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.979 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:03:27.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.979 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:03:27.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:27.980 | INFO | Uploaded 36GiB to GPU in 2.52206s (14.2741GiB/s) | 20260507 21:03:27.980 | INFO | Downloaded 34GiB to GPU in 2.49596s (13.622GiB/s) | 20260507 21:03:33.119 | INFO | Analyzing chunk frames = [2578432,2578944) | 20260507 21:03:33.119 | INFO | MemInfo: { | 20260507 21:03:33.119 | INFO | "gpu" : " 17 GB", | 20260507 21:03:33.119 | INFO | "huge" : "", | 20260507 21:03:33.119 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:33.119 | INFO | "malloc" : " 14 GB", | 20260507 21:03:33.119 | INFO | "pinned" : " 29 GB" | 20260507 21:03:33.119 | INFO | } | 20260507 21:03:33.119 | INFO | Perf report for the last 5.13982 seconds: | 20260507 21:03:33.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:33.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:03:33.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 21:03:33.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:33.119 | INFO | Uploaded 36GiB to GPU in 2.43826s (14.7646GiB/s) | 20260507 21:03:33.120 | INFO | Downloaded 34GiB to GPU in 2.49484s (13.6281GiB/s) | 20260507 21:03:38.257 | INFO | Analyzing chunk frames = [2578944,2579456) | 20260507 21:03:38.257 | INFO | MemInfo: { | 20260507 21:03:38.257 | INFO | "gpu" : " 17 GB", | 20260507 21:03:38.257 | INFO | "huge" : "", | 20260507 21:03:38.257 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:38.257 | INFO | "malloc" : " 14 GB", | 20260507 21:03:38.257 | INFO | "pinned" : " 29 GB" | 20260507 21:03:38.257 | INFO | } | 20260507 21:03:38.257 | INFO | Perf report for the last 5.13822 seconds: | 20260507 21:03:38.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:38.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.257 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:03:38.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.257 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 21:03:38.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:38.258 | INFO | Uploaded 36GiB to GPU in 2.39822s (15.0111GiB/s) | 20260507 21:03:38.258 | INFO | Downloaded 34GiB to GPU in 2.39707s (14.184GiB/s) | 20260507 21:03:43.395 | INFO | Analyzing chunk frames = [2579456,2579968) | 20260507 21:03:43.395 | INFO | MemInfo: { | 20260507 21:03:43.395 | INFO | "gpu" : " 17 GB", | 20260507 21:03:43.395 | INFO | "huge" : "", | 20260507 21:03:43.395 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:43.395 | INFO | "malloc" : " 14 GB", | 20260507 21:03:43.395 | INFO | "pinned" : " 29 GB" | 20260507 21:03:43.395 | INFO | } | 20260507 21:03:43.395 | INFO | Perf report for the last 5.138 seconds: | 20260507 21:03:43.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:43.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.395 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:03:43.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.395 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:03:43.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:43.396 | INFO | Uploaded 36GiB to GPU in 2.54051s (14.1704GiB/s) | 20260507 21:03:43.396 | INFO | Downloaded 34GiB to GPU in 2.45325s (13.8592GiB/s) | 20260507 21:03:48.535 | INFO | Analyzing chunk frames = [2579968,2580480) | 20260507 21:03:48.535 | INFO | MemInfo: { | 20260507 21:03:48.535 | INFO | "gpu" : " 17 GB", | 20260507 21:03:48.535 | INFO | "huge" : "", | 20260507 21:03:48.535 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:48.535 | INFO | "malloc" : " 14 GB", | 20260507 21:03:48.535 | INFO | "pinned" : " 29 GB" | 20260507 21:03:48.535 | INFO | } | 20260507 21:03:48.535 | INFO | Perf report for the last 5.14007 seconds: | 20260507 21:03:48.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:48.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.535 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:03:48.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.535 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:03:48.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:48.536 | INFO | Uploaded 36GiB to GPU in 2.48715s (14.4744GiB/s) | 20260507 21:03:48.536 | INFO | Downloaded 34GiB to GPU in 2.44059s (13.9311GiB/s) | 20260507 21:03:48.536 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:03:48.536 | INFO | Total mem usage: 16.7 GiB | 20260507 21:03:48.536 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:03:48.536 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:03:48.536 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:03:48.536 | INFO | High Water Breakdown: | 20260507 21:03:48.536 | INFO | 59.9% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:03:48.536 | INFO | 40% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:03:48.536 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:03:48.536 | INFO | ------------------------------------------ | 20260507 21:03:48.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:03:48.536 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:03:48.536 | INFO | Total mem usage: 26.5 GiB | 20260507 21:03:48.536 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:03:48.536 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:03:48.536 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:03:48.536 | INFO | High Water Breakdown: | 20260507 21:03:48.536 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:03:48.536 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:03:48.536 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:03:48.536 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:03:48.536 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:03:48.536 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:03:48.536 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:03:48.536 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:03:48.536 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:03:48.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:03:48.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:03:48.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:03:48.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:03:48.536 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:03:48.536 | INFO | ------------------------------------------ | 20260507 21:03:48.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:03:48.536 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:03:48.536 | INFO | Total mem usage: 27.8 GiB | 20260507 21:03:48.536 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:03:48.536 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:03:48.536 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:03:48.536 | INFO | High Water Breakdown: | 20260507 21:03:48.536 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:03:48.536 | INFO | ------------------------------------------ | 20260507 21:03:48.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:03:48.536 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:03:48.536 | INFO | Total mem usage: 15.7 GiB | 20260507 21:03:48.536 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:03:48.536 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:03:48.536 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:03:48.536 | INFO | High Water Breakdown: | 20260507 21:03:48.536 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:03:48.536 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:03:48.536 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:03:48.537 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:03:48.537 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:03:48.537 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:03:48.537 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:03:48.537 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:03:48.537 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:03:48.537 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:03:48.537 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:03:48.537 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:03:48.537 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:03:48.537 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:03:48.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:03:48.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:03:48.537 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:03:48.537 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:03:48.537 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:03:48.537 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:03:48.537 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:03:48.537 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:03:48.537 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:03:48.537 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:03:48.537 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:03:48.537 | INFO | ------------------------------------------ | 20260507 21:03:48.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:03:48.537 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:03:48.537 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:03:48.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:03:48.537 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.43e+01 / 2.07e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.19e-02 / 6.13e-03 | 20260507 21:03:48.537 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.50e+00, 5.93e-01 / 2.37e-01 | 20260507 21:03:48.537 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.14e-01 | 20260507 21:03:48.537 | INFO | PerfReport: 16.01%, DME -- 4.23e+01 / 4.57e+01, 4.06e+00 / 1.99e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.72e+00 / 1.72e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.49e+00 / 2.83e+00, 4.35e-01 / 2.25e-01 | 20260507 21:03:48.537 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.40e+00, 2.97e-01 / 1.61e-01 | 20260507 21:03:48.537 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.64e-01 / 4.00e-01, 5.96e-02 / 2.85e-02 | 20260507 21:03:48.537 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:03:48.537 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:03:48.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:03:48.537 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 24.37%, Upload -- 2.61e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 30.86%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 2.07%, LockUpload -- 2.21e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 2.61%, LockDownload -- 2.80e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 15.83%, LockCompute -- 1.69e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260507 21:03:48.537 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260507 21:03:48.632 | INFO | Starting a round of Baz IO | 20260507 21:03:52.092 | INFO | Populating RTMetrics report | 20260507 21:03:52.092 | INFO | RTMetrics report populated | 20260507 21:03:52.092 | INFO | Opening tmp RTMetrics json file | 20260507 21:03:52.316 | INFO | Serializing RTMetrics to json | 20260507 21:03:52.452 | INFO | Serialization done, swapping tmp file now... | 20260507 21:03:52.460 | INFO | RTMetrics done | 20260507 21:03:52.475 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.842 seconds | 20260507 21:03:52.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.00% / 0.01% / 55.99% | 20260507 21:03:52.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.80889% / 2.78155e-05% / 7.94729e-05% | 20260507 21:03:52.475 | INFO | Baz IO overall: | 20260507 21:03:52.475 | INFO | BytesWritten : 1173833800048 | 20260507 21:03:52.475 | INFO | NumEvents : 568986504255 | 20260507 21:03:53.675 | INFO | Analyzing chunk frames = [2580480,2580992) | 20260507 21:03:53.675 | INFO | MemInfo: { | 20260507 21:03:53.675 | INFO | "gpu" : " 17 GB", | 20260507 21:03:53.675 | INFO | "huge" : "", | 20260507 21:03:53.675 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:53.675 | INFO | "malloc" : " 14 GB", | 20260507 21:03:53.675 | INFO | "pinned" : " 29 GB" | 20260507 21:03:53.675 | INFO | } | 20260507 21:03:53.675 | INFO | Perf report for the last 5.13998 seconds: | 20260507 21:03:53.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:53.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:53.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:53.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:53.675 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260507 21:03:53.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:53.675 | INFO | RT_METRICS : 48.4%, 1.0, 25.4ms, 0 | 20260507 21:03:53.675 | INFO | PRE_HQ : 65.8%, 1.7, 57.9ms, 0 | 20260507 21:03:53.675 | INFO | BAZWRITER : 74.3%, 3.9, 150.0ms, 0 | 20260507 21:03:53.675 | INFO | BAZ_SAMPLING : 5.4%, 1.0, 2.8ms, 0 | 20260507 21:03:53.676 | INFO | Uploaded 36GiB to GPU in 2.5189s (14.2919GiB/s) | 20260507 21:03:53.676 | INFO | Downloaded 38GiB to GPU in 2.68701s (14.1421GiB/s) | 20260507 21:03:58.816 | INFO | Analyzing chunk frames = [2580992,2581504) | 20260507 21:03:58.816 | INFO | MemInfo: { | 20260507 21:03:58.816 | INFO | "gpu" : " 17 GB", | 20260507 21:03:58.816 | INFO | "huge" : "", | 20260507 21:03:58.816 | INFO | "huge_shared" : " 29 GB", | 20260507 21:03:58.816 | INFO | "malloc" : " 14 GB", | 20260507 21:03:58.816 | INFO | "pinned" : " 29 GB" | 20260507 21:03:58.816 | INFO | } | 20260507 21:03:58.816 | INFO | Perf report for the last 5.14066 seconds: | 20260507 21:03:58.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:03:58.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.816 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:03:58.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.816 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:03:58.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:03:58.817 | INFO | Uploaded 36GiB to GPU in 2.56063s (14.0591GiB/s) | 20260507 21:03:58.817 | INFO | Downloaded 34GiB to GPU in 2.50564s (13.5694GiB/s) | 20260507 21:04:03.954 | INFO | Analyzing chunk frames = [2581504,2582016) | 20260507 21:04:03.954 | INFO | MemInfo: { | 20260507 21:04:03.954 | INFO | "gpu" : " 17 GB", | 20260507 21:04:03.954 | INFO | "huge" : "", | 20260507 21:04:03.954 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:03.955 | INFO | "malloc" : " 14 GB", | 20260507 21:04:03.955 | INFO | "pinned" : " 29 GB" | 20260507 21:04:03.955 | INFO | } | 20260507 21:04:03.955 | INFO | Perf report for the last 5.13855 seconds: | 20260507 21:04:03.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:03.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:04:03.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:04:03.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:03.955 | INFO | Uploaded 36GiB to GPU in 2.49195s (14.4465GiB/s) | 20260507 21:04:03.955 | INFO | Downloaded 34GiB to GPU in 2.47861s (13.7174GiB/s) | 20260507 21:04:09.092 | INFO | Analyzing chunk frames = [2582016,2582528) | 20260507 21:04:09.092 | INFO | MemInfo: { | 20260507 21:04:09.092 | INFO | "gpu" : " 17 GB", | 20260507 21:04:09.092 | INFO | "huge" : "", | 20260507 21:04:09.092 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:09.092 | INFO | "malloc" : " 14 GB", | 20260507 21:04:09.092 | INFO | "pinned" : " 29 GB" | 20260507 21:04:09.092 | INFO | } | 20260507 21:04:09.093 | INFO | Perf report for the last 5.13794 seconds: | 20260507 21:04:09.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:09.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:04:09.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:04:09.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:09.093 | INFO | Uploaded 36GiB to GPU in 2.5026s (14.385GiB/s) | 20260507 21:04:09.093 | INFO | Downloaded 34GiB to GPU in 2.49638s (13.6197GiB/s) | 20260507 21:04:14.233 | INFO | Analyzing chunk frames = [2582528,2583040) | 20260507 21:04:14.233 | INFO | MemInfo: { | 20260507 21:04:14.233 | INFO | "gpu" : " 17 GB", | 20260507 21:04:14.233 | INFO | "huge" : "", | 20260507 21:04:14.233 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:14.233 | INFO | "malloc" : " 14 GB", | 20260507 21:04:14.233 | INFO | "pinned" : " 29 GB" | 20260507 21:04:14.233 | INFO | } | 20260507 21:04:14.233 | INFO | Perf report for the last 5.14026 seconds: | 20260507 21:04:14.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:14.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 21:04:14.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 21:04:14.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:14.233 | INFO | Uploaded 36GiB to GPU in 2.39876s (15.0078GiB/s) | 20260507 21:04:14.233 | INFO | Downloaded 34GiB to GPU in 2.4812s (13.7031GiB/s) | 20260507 21:04:19.371 | INFO | Analyzing chunk frames = [2583040,2583552) | 20260507 21:04:19.371 | INFO | MemInfo: { | 20260507 21:04:19.371 | INFO | "gpu" : " 17 GB", | 20260507 21:04:19.371 | INFO | "huge" : "", | 20260507 21:04:19.371 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:19.371 | INFO | "malloc" : " 14 GB", | 20260507 21:04:19.371 | INFO | "pinned" : " 29 GB" | 20260507 21:04:19.371 | INFO | } | 20260507 21:04:19.371 | INFO | Perf report for the last 5.13862 seconds: | 20260507 21:04:19.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:19.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.371 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:04:19.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.372 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:04:19.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:19.372 | INFO | Uploaded 36GiB to GPU in 2.51859s (14.2937GiB/s) | 20260507 21:04:19.372 | INFO | Downloaded 34GiB to GPU in 2.41368s (14.0864GiB/s) | 20260507 21:04:24.510 | INFO | Analyzing chunk frames = [2583552,2584064) | 20260507 21:04:24.510 | INFO | MemInfo: { | 20260507 21:04:24.510 | INFO | "gpu" : " 17 GB", | 20260507 21:04:24.510 | INFO | "huge" : "", | 20260507 21:04:24.510 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:24.510 | INFO | "malloc" : " 14 GB", | 20260507 21:04:24.510 | INFO | "pinned" : " 29 GB" | 20260507 21:04:24.510 | INFO | } | 20260507 21:04:24.510 | INFO | Perf report for the last 5.1389 seconds: | 20260507 21:04:24.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:24.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.510 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260507 21:04:24.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.511 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:04:24.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:24.511 | INFO | Uploaded 36GiB to GPU in 2.55187s (14.1073GiB/s) | 20260507 21:04:24.511 | INFO | Downloaded 34GiB to GPU in 2.43971s (13.9361GiB/s) | 20260507 21:04:29.650 | INFO | Analyzing chunk frames = [2584064,2584576) | 20260507 21:04:29.650 | INFO | MemInfo: { | 20260507 21:04:29.650 | INFO | "gpu" : " 17 GB", | 20260507 21:04:29.650 | INFO | "huge" : "", | 20260507 21:04:29.650 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:29.650 | INFO | "malloc" : " 14 GB", | 20260507 21:04:29.650 | INFO | "pinned" : " 29 GB" | 20260507 21:04:29.650 | INFO | } | 20260507 21:04:29.650 | INFO | Perf report for the last 5.13974 seconds: | 20260507 21:04:29.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:29.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:04:29.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:04:29.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:29.650 | INFO | Uploaded 36GiB to GPU in 2.51224s (14.3299GiB/s) | 20260507 21:04:29.651 | INFO | Downloaded 34GiB to GPU in 2.41727s (14.0654GiB/s) | 20260507 21:04:29.748 | INFO | Starting a round of Baz IO | 20260507 21:04:33.220 | INFO | Populating RTMetrics report | 20260507 21:04:33.221 | INFO | RTMetrics report populated | 20260507 21:04:33.221 | INFO | Opening tmp RTMetrics json file | 20260507 21:04:33.222 | INFO | Serializing RTMetrics to json | 20260507 21:04:33.223 | INFO | Serialization done, swapping tmp file now... | 20260507 21:04:33.224 | INFO | RTMetrics done | 20260507 21:04:33.318 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.569 seconds | 20260507 21:04:33.318 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.99% / 0.01% / 56.00% | 20260507 21:04:33.318 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.80767% / 1.19209e-05% / 7.15256e-05% | 20260507 21:04:33.318 | INFO | Baz IO overall: | 20260507 21:04:33.318 | INFO | BytesWritten : 1175731470550 | 20260507 21:04:33.318 | INFO | NumEvents : 569904875682 | 20260507 21:04:34.789 | INFO | Analyzing chunk frames = [2584576,2585088) | 20260507 21:04:34.789 | INFO | MemInfo: { | 20260507 21:04:34.789 | INFO | "gpu" : " 17 GB", | 20260507 21:04:34.789 | INFO | "huge" : "", | 20260507 21:04:34.789 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:34.789 | INFO | "malloc" : " 14 GB", | 20260507 21:04:34.789 | INFO | "pinned" : " 29 GB" | 20260507 21:04:34.789 | INFO | } | 20260507 21:04:34.789 | INFO | Perf report for the last 5.13931 seconds: | 20260507 21:04:34.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:34.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:34.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:34.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:34.790 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 21:04:34.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:34.790 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 21:04:34.790 | INFO | PRE_HQ : 66.3%, 1.7, 58.3ms, 0 | 20260507 21:04:34.790 | INFO | BAZWRITER : 69.0%, 3.9, 137.7ms, 0 | 20260507 21:04:34.790 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:04:34.790 | INFO | Uploaded 36GiB to GPU in 2.54382s (14.152GiB/s) | 20260507 21:04:34.790 | INFO | Downloaded 38GiB to GPU in 2.68569s (14.1491GiB/s) | 20260507 21:04:39.930 | INFO | Analyzing chunk frames = [2585088,2585600) | 20260507 21:04:39.930 | INFO | MemInfo: { | 20260507 21:04:39.930 | INFO | "gpu" : " 17 GB", | 20260507 21:04:39.930 | INFO | "huge" : "", | 20260507 21:04:39.930 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:39.930 | INFO | "malloc" : " 14 GB", | 20260507 21:04:39.930 | INFO | "pinned" : " 29 GB" | 20260507 21:04:39.930 | INFO | } | 20260507 21:04:39.930 | INFO | Perf report for the last 5.14049 seconds: | 20260507 21:04:39.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:39.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.930 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:04:39.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.930 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:04:39.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:39.931 | INFO | Uploaded 36GiB to GPU in 2.5634s (14.0438GiB/s) | 20260507 21:04:39.931 | INFO | Downloaded 34GiB to GPU in 2.5026s (13.5859GiB/s) | 20260507 21:04:45.068 | INFO | Analyzing chunk frames = [2585600,2586112) | 20260507 21:04:45.068 | INFO | MemInfo: { | 20260507 21:04:45.068 | INFO | "gpu" : " 17 GB", | 20260507 21:04:45.068 | INFO | "huge" : "", | 20260507 21:04:45.068 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:45.068 | INFO | "malloc" : " 14 GB", | 20260507 21:04:45.069 | INFO | "pinned" : " 29 GB" | 20260507 21:04:45.069 | INFO | } | 20260507 21:04:45.069 | INFO | Perf report for the last 5.13863 seconds: | 20260507 21:04:45.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:45.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:04:45.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:04:45.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:45.069 | INFO | Uploaded 36GiB to GPU in 2.49512s (14.4281GiB/s) | 20260507 21:04:45.069 | INFO | Downloaded 34GiB to GPU in 2.48234s (13.6968GiB/s) | 20260507 21:04:50.208 | INFO | Analyzing chunk frames = [2586112,2586624) | 20260507 21:04:50.208 | INFO | MemInfo: { | 20260507 21:04:50.208 | INFO | "gpu" : " 17 GB", | 20260507 21:04:50.208 | INFO | "huge" : "", | 20260507 21:04:50.208 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:50.208 | INFO | "malloc" : " 14 GB", | 20260507 21:04:50.208 | INFO | "pinned" : " 29 GB" | 20260507 21:04:50.208 | INFO | } | 20260507 21:04:50.208 | INFO | Perf report for the last 5.13932 seconds: | 20260507 21:04:50.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:50.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 21:04:50.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:04:50.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:50.208 | INFO | Uploaded 36GiB to GPU in 2.48557s (14.4836GiB/s) | 20260507 21:04:50.208 | INFO | Downloaded 34GiB to GPU in 2.41033s (14.106GiB/s) | 20260507 21:04:55.347 | INFO | Analyzing chunk frames = [2586624,2587136) | 20260507 21:04:55.347 | INFO | MemInfo: { | 20260507 21:04:55.347 | INFO | "gpu" : " 17 GB", | 20260507 21:04:55.347 | INFO | "huge" : "", | 20260507 21:04:55.347 | INFO | "huge_shared" : " 29 GB", | 20260507 21:04:55.347 | INFO | "malloc" : " 14 GB", | 20260507 21:04:55.347 | INFO | "pinned" : " 29 GB" | 20260507 21:04:55.347 | INFO | } | 20260507 21:04:55.347 | INFO | Perf report for the last 5.13898 seconds: | 20260507 21:04:55.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:04:55.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 21:04:55.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 21:04:55.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:04:55.347 | INFO | Uploaded 36GiB to GPU in 2.3621s (15.2407GiB/s) | 20260507 21:04:55.347 | INFO | Downloaded 34GiB to GPU in 2.51801s (13.5027GiB/s) | 20260507 21:05:00.486 | INFO | Analyzing chunk frames = [2587136,2587648) | 20260507 21:05:00.486 | INFO | MemInfo: { | 20260507 21:05:00.486 | INFO | "gpu" : " 17 GB", | 20260507 21:05:00.486 | INFO | "huge" : "", | 20260507 21:05:00.486 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:00.486 | INFO | "malloc" : " 14 GB", | 20260507 21:05:00.486 | INFO | "pinned" : " 29 GB" | 20260507 21:05:00.486 | INFO | } | 20260507 21:05:00.486 | INFO | Perf report for the last 5.13935 seconds: | 20260507 21:05:00.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:00.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.486 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:05:00.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.486 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:05:00.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:00.487 | INFO | Uploaded 36GiB to GPU in 2.39667s (15.0209GiB/s) | 20260507 21:05:00.487 | INFO | Downloaded 34GiB to GPU in 2.4031s (14.1484GiB/s) | 20260507 21:05:05.625 | INFO | Analyzing chunk frames = [2587648,2588160) | 20260507 21:05:05.625 | INFO | MemInfo: { | 20260507 21:05:05.625 | INFO | "gpu" : " 17 GB", | 20260507 21:05:05.625 | INFO | "huge" : "", | 20260507 21:05:05.625 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:05.625 | INFO | "malloc" : " 14 GB", | 20260507 21:05:05.626 | INFO | "pinned" : " 29 GB" | 20260507 21:05:05.626 | INFO | } | 20260507 21:05:05.626 | INFO | Perf report for the last 5.13933 seconds: | 20260507 21:05:05.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:05.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.626 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:05:05.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.626 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:05:05.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:05.627 | INFO | Uploaded 36GiB to GPU in 2.52821s (14.2393GiB/s) | 20260507 21:05:05.627 | INFO | Downloaded 34GiB to GPU in 2.45407s (13.8545GiB/s) | 20260507 21:05:10.765 | INFO | Analyzing chunk frames = [2588160,2588672) | 20260507 21:05:10.765 | INFO | MemInfo: { | 20260507 21:05:10.765 | INFO | "gpu" : " 17 GB", | 20260507 21:05:10.765 | INFO | "huge" : "", | 20260507 21:05:10.765 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:10.765 | INFO | "malloc" : " 14 GB", | 20260507 21:05:10.765 | INFO | "pinned" : " 29 GB" | 20260507 21:05:10.765 | INFO | } | 20260507 21:05:10.765 | INFO | Perf report for the last 5.13942 seconds: | 20260507 21:05:10.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:10.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:05:10.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:05:10.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:10.765 | INFO | Uploaded 36GiB to GPU in 2.47773s (14.5294GiB/s) | 20260507 21:05:10.765 | INFO | Downloaded 34GiB to GPU in 2.43404s (13.9685GiB/s) | 20260507 21:05:10.766 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:05:10.766 | INFO | Total mem usage: 16.7 GiB | 20260507 21:05:10.766 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:05:10.766 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260507 21:05:10.766 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:05:10.766 | INFO | High Water Breakdown: | 20260507 21:05:10.766 | INFO | 59.7% : 8.84 GiB ( 85.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:05:10.766 | INFO | 40.1% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:05:10.766 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:05:10.766 | INFO | ------------------------------------------ | 20260507 21:05:10.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:05:10.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:05:10.766 | INFO | Total mem usage: 26.5 GiB | 20260507 21:05:10.766 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:05:10.766 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:05:10.766 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:05:10.766 | INFO | High Water Breakdown: | 20260507 21:05:10.766 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:05:10.766 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:05:10.766 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:05:10.766 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:05:10.766 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:05:10.766 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:05:10.766 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:05:10.766 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:05:10.766 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:05:10.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:05:10.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:05:10.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:05:10.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:05:10.766 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:05:10.766 | INFO | ------------------------------------------ | 20260507 21:05:10.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:05:10.766 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:05:10.766 | INFO | Total mem usage: 27.8 GiB | 20260507 21:05:10.766 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:05:10.766 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:05:10.766 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:05:10.766 | INFO | High Water Breakdown: | 20260507 21:05:10.766 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:05:10.766 | INFO | ------------------------------------------ | 20260507 21:05:10.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:05:10.766 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:05:10.766 | INFO | Total mem usage: 15.7 GiB | 20260507 21:05:10.766 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:05:10.766 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:05:10.766 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:05:10.766 | INFO | High Water Breakdown: | 20260507 21:05:10.766 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:05:10.766 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:05:10.766 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:05:10.766 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:05:10.766 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:05:10.766 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:05:10.766 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:05:10.766 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:05:10.766 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:05:10.766 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:05:10.766 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:05:10.766 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:05:10.766 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:05:10.766 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:05:10.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:05:10.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:05:10.766 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:05:10.766 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:05:10.766 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:05:10.766 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:05:10.766 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:05:10.766 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:05:10.766 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:05:10.766 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:05:10.766 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:05:10.766 | INFO | ------------------------------------------ | 20260507 21:05:10.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:05:10.766 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:05:10.766 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:05:10.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:05:10.766 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.58e+01, 1.42e+01 / 1.88e+00 | 20260507 21:05:10.766 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.19e-02 / 4.98e-03 | 20260507 21:05:10.766 | INFO | PerfReport: 12.10%, Baseline -- 3.99e+00 / 4.48e+00, 5.91e-01 / 2.54e-01 | 20260507 21:05:10.766 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.08e-01 / 1.07e-01 | 20260507 21:05:10.766 | INFO | PerfReport: 16.03%, DME -- 4.23e+01 / 4.57e+01, 4.06e+00 / 1.87e+00 | 20260507 21:05:10.766 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.66e+00 / 1.54e+00 | 20260507 21:05:10.766 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.38e-01 / 2.16e-01 | 20260507 21:05:10.766 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.40e+00, 2.96e-01 / 1.69e-01 | 20260507 21:05:10.766 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.63e-01 / 4.03e-01, 5.95e-02 / 3.29e-02 | 20260507 21:05:10.767 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:05:10.767 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:05:10.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:05:10.767 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 24.29%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 30.81%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 2.10%, LockUpload -- 2.25e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 2.70%, LockDownload -- 2.89e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 15.85%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 21:05:10.767 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 21:05:10.862 | INFO | Starting a round of Baz IO | 20260507 21:05:14.364 | INFO | Populating RTMetrics report | 20260507 21:05:14.364 | INFO | RTMetrics report populated | 20260507 21:05:14.364 | INFO | Opening tmp RTMetrics json file | 20260507 21:05:14.366 | INFO | Serializing RTMetrics to json | 20260507 21:05:14.368 | INFO | Serialization done, swapping tmp file now... | 20260507 21:05:14.368 | INFO | RTMetrics done | 20260507 21:05:14.485 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.623 seconds | 20260507 21:05:14.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.98% / 0.01% / 56.01% | 20260507 21:05:14.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.79573% / 1.19209e-05% / 5.5631e-05% | 20260507 21:05:14.486 | INFO | Baz IO overall: | 20260507 21:05:14.486 | INFO | BytesWritten : 1177626333100 | 20260507 21:05:14.486 | INFO | NumEvents : 570820970748 | 20260507 21:05:15.904 | INFO | Analyzing chunk frames = [2588672,2589184) | 20260507 21:05:15.904 | INFO | MemInfo: { | 20260507 21:05:15.904 | INFO | "gpu" : " 17 GB", | 20260507 21:05:15.904 | INFO | "huge" : "", | 20260507 21:05:15.904 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:15.904 | INFO | "malloc" : " 14 GB", | 20260507 21:05:15.904 | INFO | "pinned" : " 29 GB" | 20260507 21:05:15.904 | INFO | } | 20260507 21:05:15.904 | INFO | Perf report for the last 5.13849 seconds: | 20260507 21:05:15.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:15.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:15.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:15.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:15.904 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 21:05:15.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:15.904 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 21:05:15.904 | INFO | PRE_HQ : 66.7%, 1.8, 61.9ms, 0 | 20260507 21:05:15.904 | INFO | BAZWRITER : 70.1%, 4.0, 143.7ms, 0 | 20260507 21:05:15.904 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 21:05:15.904 | INFO | Uploaded 36GiB to GPU in 2.53987s (14.1739GiB/s) | 20260507 21:05:15.904 | INFO | Downloaded 38GiB to GPU in 2.69339s (14.1086GiB/s) | 20260507 21:05:21.043 | INFO | Analyzing chunk frames = [2589184,2589696) | 20260507 21:05:21.043 | INFO | MemInfo: { | 20260507 21:05:21.043 | INFO | "gpu" : " 17 GB", | 20260507 21:05:21.043 | INFO | "huge" : "", | 20260507 21:05:21.043 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:21.043 | INFO | "malloc" : " 14 GB", | 20260507 21:05:21.043 | INFO | "pinned" : " 29 GB" | 20260507 21:05:21.043 | INFO | } | 20260507 21:05:21.043 | INFO | Perf report for the last 5.13961 seconds: | 20260507 21:05:21.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:21.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.043 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 21:05:21.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.043 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:05:21.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:21.044 | INFO | Uploaded 36GiB to GPU in 2.57229s (13.9953GiB/s) | 20260507 21:05:21.044 | INFO | Downloaded 34GiB to GPU in 2.49521s (13.6261GiB/s) | 20260507 21:05:26.184 | INFO | Analyzing chunk frames = [2589696,2590208) | 20260507 21:05:26.184 | INFO | MemInfo: { | 20260507 21:05:26.184 | INFO | "gpu" : " 17 GB", | 20260507 21:05:26.184 | INFO | "huge" : "", | 20260507 21:05:26.184 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:26.184 | INFO | "malloc" : " 14 GB", | 20260507 21:05:26.185 | INFO | "pinned" : " 29 GB" | 20260507 21:05:26.185 | INFO | } | 20260507 21:05:26.185 | INFO | Perf report for the last 5.14137 seconds: | 20260507 21:05:26.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:26.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:05:26.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:05:26.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:26.185 | INFO | Uploaded 36GiB to GPU in 2.5125s (14.3284GiB/s) | 20260507 21:05:26.185 | INFO | Downloaded 34GiB to GPU in 2.49022s (13.6534GiB/s) | 20260507 21:05:31.323 | INFO | Analyzing chunk frames = [2590208,2590720) | 20260507 21:05:31.323 | INFO | MemInfo: { | 20260507 21:05:31.323 | INFO | "gpu" : " 17 GB", | 20260507 21:05:31.323 | INFO | "huge" : "", | 20260507 21:05:31.323 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:31.323 | INFO | "malloc" : " 14 GB", | 20260507 21:05:31.323 | INFO | "pinned" : " 29 GB" | 20260507 21:05:31.323 | INFO | } | 20260507 21:05:31.323 | INFO | Perf report for the last 5.1386 seconds: | 20260507 21:05:31.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:31.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.323 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:05:31.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.323 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:05:31.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:31.324 | INFO | Uploaded 36GiB to GPU in 2.49672s (14.4189GiB/s) | 20260507 21:05:31.324 | INFO | Downloaded 34GiB to GPU in 2.49278s (13.6394GiB/s) | 20260507 21:05:36.462 | INFO | Analyzing chunk frames = [2590720,2591232) | 20260507 21:05:36.462 | INFO | MemInfo: { | 20260507 21:05:36.462 | INFO | "gpu" : " 17 GB", | 20260507 21:05:36.462 | INFO | "huge" : "", | 20260507 21:05:36.462 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:36.462 | INFO | "malloc" : " 14 GB", | 20260507 21:05:36.462 | INFO | "pinned" : " 29 GB" | 20260507 21:05:36.462 | INFO | } | 20260507 21:05:36.462 | INFO | Perf report for the last 5.13892 seconds: | 20260507 21:05:36.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:36.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 21:05:36.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:05:36.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:36.462 | INFO | Uploaded 36GiB to GPU in 2.37582s (15.1526GiB/s) | 20260507 21:05:36.462 | INFO | Downloaded 34GiB to GPU in 2.50401s (13.5782GiB/s) | 20260507 21:05:41.601 | INFO | Analyzing chunk frames = [2591232,2591744) | 20260507 21:05:41.601 | INFO | MemInfo: { | 20260507 21:05:41.601 | INFO | "gpu" : " 17 GB", | 20260507 21:05:41.601 | INFO | "huge" : "", | 20260507 21:05:41.601 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:41.601 | INFO | "malloc" : " 14 GB", | 20260507 21:05:41.601 | INFO | "pinned" : " 29 GB" | 20260507 21:05:41.601 | INFO | } | 20260507 21:05:41.601 | INFO | Perf report for the last 5.13868 seconds: | 20260507 21:05:41.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:41.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:05:41.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:05:41.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:41.601 | INFO | Uploaded 36GiB to GPU in 2.38912s (15.0683GiB/s) | 20260507 21:05:41.601 | INFO | Downloaded 34GiB to GPU in 2.39384s (14.2031GiB/s) | 20260507 21:05:46.740 | INFO | Analyzing chunk frames = [2591744,2592256) | 20260507 21:05:46.740 | INFO | MemInfo: { | 20260507 21:05:46.740 | INFO | "gpu" : " 17 GB", | 20260507 21:05:46.740 | INFO | "huge" : "", | 20260507 21:05:46.740 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:46.741 | INFO | "malloc" : " 14 GB", | 20260507 21:05:46.741 | INFO | "pinned" : " 29 GB" | 20260507 21:05:46.741 | INFO | } | 20260507 21:05:46.741 | INFO | Perf report for the last 5.1399 seconds: | 20260507 21:05:46.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:46.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:05:46.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 21:05:46.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:46.741 | INFO | Uploaded 36GiB to GPU in 2.52208s (14.274GiB/s) | 20260507 21:05:46.742 | INFO | Downloaded 34GiB to GPU in 2.45833s (13.8305GiB/s) | 20260507 21:05:51.881 | INFO | Analyzing chunk frames = [2592256,2592768) | 20260507 21:05:51.881 | INFO | MemInfo: { | 20260507 21:05:51.881 | INFO | "gpu" : " 17 GB", | 20260507 21:05:51.881 | INFO | "huge" : "", | 20260507 21:05:51.881 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:51.881 | INFO | "malloc" : " 14 GB", | 20260507 21:05:51.881 | INFO | "pinned" : " 29 GB" | 20260507 21:05:51.881 | INFO | } | 20260507 21:05:51.881 | INFO | Perf report for the last 5.14027 seconds: | 20260507 21:05:51.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:51.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.881 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:05:51.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.881 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260507 21:05:51.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:51.882 | INFO | Uploaded 36GiB to GPU in 2.51656s (14.3052GiB/s) | 20260507 21:05:51.882 | INFO | Downloaded 34GiB to GPU in 2.44549s (13.9031GiB/s) | 20260507 21:05:51.993 | INFO | Starting a round of Baz IO | 20260507 21:05:55.466 | INFO | Populating RTMetrics report | 20260507 21:05:55.467 | INFO | RTMetrics report populated | 20260507 21:05:55.467 | INFO | Opening tmp RTMetrics json file | 20260507 21:05:55.483 | INFO | Serializing RTMetrics to json | 20260507 21:05:55.489 | INFO | Serialization done, swapping tmp file now... | 20260507 21:05:55.493 | INFO | RTMetrics done | 20260507 21:05:55.706 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.712 seconds | 20260507 21:05:55.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.97% / 0.01% / 56.02% | 20260507 21:05:55.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.79698% / 1.19209e-05% / 3.17891e-05% | 20260507 21:05:55.706 | INFO | Baz IO overall: | 20260507 21:05:55.706 | INFO | BytesWritten : 1179523692574 | 20260507 21:05:55.706 | INFO | NumEvents : 571739024710 | 20260507 21:05:57.020 | INFO | Analyzing chunk frames = [2592768,2593280) | 20260507 21:05:57.020 | INFO | MemInfo: { | 20260507 21:05:57.020 | INFO | "gpu" : " 17 GB", | 20260507 21:05:57.020 | INFO | "huge" : "", | 20260507 21:05:57.020 | INFO | "huge_shared" : " 29 GB", | 20260507 21:05:57.020 | INFO | "malloc" : " 14 GB", | 20260507 21:05:57.020 | INFO | "pinned" : " 29 GB" | 20260507 21:05:57.020 | INFO | } | 20260507 21:05:57.020 | INFO | Perf report for the last 5.13877 seconds: | 20260507 21:05:57.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:05:57.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:57.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:57.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:57.020 | INFO | ANALYSIS : 69.0%, 2.9, 105.0ms, 0 | 20260507 21:05:57.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:05:57.020 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 21:05:57.020 | INFO | PRE_HQ : 68.0%, 1.8, 63.4ms, 0 | 20260507 21:05:57.020 | INFO | BAZWRITER : 71.8%, 3.9, 143.3ms, 0 | 20260507 21:05:57.020 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:05:57.021 | INFO | Uploaded 36GiB to GPU in 2.6334s (13.6705GiB/s) | 20260507 21:05:57.021 | INFO | Downloaded 38GiB to GPU in 2.71125s (14.0157GiB/s) | 20260507 21:06:02.158 | INFO | Analyzing chunk frames = [2593280,2593792) | 20260507 21:06:02.158 | INFO | MemInfo: { | 20260507 21:06:02.158 | INFO | "gpu" : " 17 GB", | 20260507 21:06:02.158 | INFO | "huge" : "", | 20260507 21:06:02.158 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:02.158 | INFO | "malloc" : " 14 GB", | 20260507 21:06:02.158 | INFO | "pinned" : " 29 GB" | 20260507 21:06:02.158 | INFO | } | 20260507 21:06:02.158 | INFO | Perf report for the last 5.13817 seconds: | 20260507 21:06:02.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:02.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260507 21:06:02.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:06:02.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:02.158 | INFO | Uploaded 36GiB to GPU in 2.59011s (13.899GiB/s) | 20260507 21:06:02.158 | INFO | Downloaded 34GiB to GPU in 2.47731s (13.7246GiB/s) | 20260507 21:06:07.297 | INFO | Analyzing chunk frames = [2593792,2594304) | 20260507 21:06:07.297 | INFO | MemInfo: { | 20260507 21:06:07.297 | INFO | "gpu" : " 17 GB", | 20260507 21:06:07.297 | INFO | "huge" : "", | 20260507 21:06:07.297 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:07.297 | INFO | "malloc" : " 14 GB", | 20260507 21:06:07.297 | INFO | "pinned" : " 29 GB" | 20260507 21:06:07.297 | INFO | } | 20260507 21:06:07.297 | INFO | Perf report for the last 5.1393 seconds: | 20260507 21:06:07.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:07.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.297 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:06:07.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.297 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 21:06:07.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:07.298 | INFO | Uploaded 36GiB to GPU in 2.51693s (14.3032GiB/s) | 20260507 21:06:07.298 | INFO | Downloaded 34GiB to GPU in 2.47465s (13.7393GiB/s) | 20260507 21:06:12.438 | INFO | Analyzing chunk frames = [2594304,2594816) | 20260507 21:06:12.438 | INFO | MemInfo: { | 20260507 21:06:12.438 | INFO | "gpu" : " 17 GB", | 20260507 21:06:12.438 | INFO | "huge" : "", | 20260507 21:06:12.438 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:12.438 | INFO | "malloc" : " 14 GB", | 20260507 21:06:12.438 | INFO | "pinned" : " 29 GB" | 20260507 21:06:12.438 | INFO | } | 20260507 21:06:12.438 | INFO | Perf report for the last 5.14049 seconds: | 20260507 21:06:12.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:12.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | ANALYSIS : 67.7%, 3.0, 104.7ms, 0 | 20260507 21:06:12.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:06:12.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:12.438 | INFO | Uploaded 36GiB to GPU in 2.53407s (14.2064GiB/s) | 20260507 21:06:12.438 | INFO | Downloaded 34GiB to GPU in 2.48272s (13.6947GiB/s) | 20260507 21:06:17.576 | INFO | Analyzing chunk frames = [2594816,2595328) | 20260507 21:06:17.576 | INFO | MemInfo: { | 20260507 21:06:17.576 | INFO | "gpu" : " 17 GB", | 20260507 21:06:17.576 | INFO | "huge" : "", | 20260507 21:06:17.576 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:17.576 | INFO | "malloc" : " 14 GB", | 20260507 21:06:17.576 | INFO | "pinned" : " 29 GB" | 20260507 21:06:17.576 | INFO | } | 20260507 21:06:17.576 | INFO | Perf report for the last 5.13851 seconds: | 20260507 21:06:17.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:17.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.576 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 21:06:17.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.576 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:06:17.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:17.577 | INFO | Uploaded 36GiB to GPU in 2.39275s (15.0454GiB/s) | 20260507 21:06:17.577 | INFO | Downloaded 34GiB to GPU in 2.49434s (13.6308GiB/s) | 20260507 21:06:22.715 | INFO | Analyzing chunk frames = [2595328,2595840) | 20260507 21:06:22.715 | INFO | MemInfo: { | 20260507 21:06:22.715 | INFO | "gpu" : " 17 GB", | 20260507 21:06:22.715 | INFO | "huge" : "", | 20260507 21:06:22.715 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:22.715 | INFO | "malloc" : " 14 GB", | 20260507 21:06:22.715 | INFO | "pinned" : " 29 GB" | 20260507 21:06:22.715 | INFO | } | 20260507 21:06:22.715 | INFO | Perf report for the last 5.13874 seconds: | 20260507 21:06:22.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:22.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:06:22.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:06:22.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:22.715 | INFO | Uploaded 36GiB to GPU in 2.50526s (14.3698GiB/s) | 20260507 21:06:22.715 | INFO | Downloaded 34GiB to GPU in 2.4247s (14.0223GiB/s) | 20260507 21:06:27.855 | INFO | Analyzing chunk frames = [2595840,2596352) | 20260507 21:06:27.855 | INFO | MemInfo: { | 20260507 21:06:27.855 | INFO | "gpu" : " 17 GB", | 20260507 21:06:27.855 | INFO | "huge" : "", | 20260507 21:06:27.855 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:27.855 | INFO | "malloc" : " 14 GB", | 20260507 21:06:27.855 | INFO | "pinned" : " 29 GB" | 20260507 21:06:27.855 | INFO | } | 20260507 21:06:27.855 | INFO | Perf report for the last 5.13991 seconds: | 20260507 21:06:27.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:27.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.855 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:06:27.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.855 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 21:06:27.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:27.856 | INFO | Uploaded 36GiB to GPU in 2.52439s (14.2609GiB/s) | 20260507 21:06:27.856 | INFO | Downloaded 34GiB to GPU in 2.44753s (13.8916GiB/s) | 20260507 21:06:32.995 | INFO | Analyzing chunk frames = [2596352,2596864) | 20260507 21:06:32.995 | INFO | MemInfo: { | 20260507 21:06:32.995 | INFO | "gpu" : " 17 GB", | 20260507 21:06:32.995 | INFO | "huge" : "", | 20260507 21:06:32.995 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:32.995 | INFO | "malloc" : " 14 GB", | 20260507 21:06:32.995 | INFO | "pinned" : " 29 GB" | 20260507 21:06:32.995 | INFO | } | 20260507 21:06:32.995 | INFO | Perf report for the last 5.14036 seconds: | 20260507 21:06:32.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:32.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.996 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:06:32.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.996 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 21:06:32.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:32.996 | INFO | Uploaded 36GiB to GPU in 2.53923s (14.1775GiB/s) | 20260507 21:06:32.996 | INFO | Downloaded 34GiB to GPU in 2.4445s (13.9088GiB/s) | 20260507 21:06:32.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:06:32.996 | INFO | Total mem usage: 16.7 GiB | 20260507 21:06:32.996 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:06:32.996 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:06:32.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:06:32.996 | INFO | High Water Breakdown: | 20260507 21:06:32.996 | INFO | 59.9% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:06:32.996 | INFO | 40% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:06:32.996 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:06:32.996 | INFO | ------------------------------------------ | 20260507 21:06:32.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:06:32.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:06:32.996 | INFO | Total mem usage: 26.5 GiB | 20260507 21:06:32.996 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:06:32.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:06:32.996 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:06:32.996 | INFO | High Water Breakdown: | 20260507 21:06:32.997 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:06:32.997 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:06:32.997 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:06:32.997 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:06:32.997 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:06:32.997 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:06:32.997 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:06:32.997 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:06:32.997 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:06:32.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:06:32.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:06:32.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:06:32.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:06:32.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:06:32.997 | INFO | ------------------------------------------ | 20260507 21:06:32.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:06:32.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:06:32.997 | INFO | Total mem usage: 27.8 GiB | 20260507 21:06:32.997 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:06:32.997 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:06:32.997 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:06:32.997 | INFO | High Water Breakdown: | 20260507 21:06:32.997 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:06:32.997 | INFO | ------------------------------------------ | 20260507 21:06:32.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:06:32.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:06:32.997 | INFO | Total mem usage: 15.7 GiB | 20260507 21:06:32.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:06:32.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:06:32.997 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:06:32.997 | INFO | High Water Breakdown: | 20260507 21:06:32.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:06:32.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:06:32.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:06:32.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:06:32.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:06:32.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:06:32.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:06:32.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:06:32.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:06:32.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:06:32.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:06:32.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:06:32.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:06:32.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:06:32.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:06:32.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:06:32.997 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:06:32.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:06:32.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:06:32.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:06:32.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:06:32.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:06:32.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:06:32.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:06:32.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:06:32.997 | INFO | ------------------------------------------ | 20260507 21:06:32.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:06:32.998 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:06:32.998 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:06:32.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:06:32.998 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.43e+01 / 2.02e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.17e-02 / 7.15e-03 | 20260507 21:06:32.998 | INFO | PerfReport: 12.14%, Baseline -- 4.00e+00 / 4.48e+00, 5.98e-01 / 2.37e-01 | 20260507 21:06:32.998 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.50e+00, 2.09e-01 / 1.26e-01 | 20260507 21:06:32.998 | INFO | PerfReport: 16.12%, DME -- 4.25e+01 / 4.62e+01, 4.18e+00 / 2.24e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 55.77%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.56e+00 / 1.34e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.52e+00 / 2.85e+00, 4.42e-01 / 2.06e-01 | 20260507 21:06:32.998 | INFO | PerfReport: 3.56%, Metrics -- 1.17e+00 / 1.39e+00, 2.94e-01 / 1.67e-01 | 20260507 21:06:32.998 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.65e-01 / 4.02e-01, 5.91e-02 / 3.16e-02 | 20260507 21:06:32.998 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:06:32.998 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:06:32.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:06:32.998 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 24.30%, Download -- 2.60e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 30.79%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 2.24%, LockUpload -- 2.39e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 2.59%, LockDownload -- 2.77e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:06:32.998 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260507 21:06:33.109 | INFO | Starting a round of Baz IO | 20260507 21:06:36.600 | INFO | Populating RTMetrics report | 20260507 21:06:36.601 | INFO | RTMetrics report populated | 20260507 21:06:36.601 | INFO | Opening tmp RTMetrics json file | 20260507 21:06:36.607 | INFO | Serializing RTMetrics to json | 20260507 21:06:36.630 | INFO | Serialization done, swapping tmp file now... | 20260507 21:06:36.631 | INFO | RTMetrics done | 20260507 21:06:36.839 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.729 seconds | 20260507 21:06:36.839 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.96% / 0.01% / 56.03% | 20260507 21:06:36.839 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.78869% / 1.98682e-05% / 5.96046e-05% | 20260507 21:06:36.839 | INFO | Baz IO overall: | 20260507 21:06:36.839 | INFO | BytesWritten : 1181418580230 | 20260507 21:06:36.839 | INFO | NumEvents : 572655233824 | 20260507 21:06:38.135 | INFO | Analyzing chunk frames = [2596864,2597376) | 20260507 21:06:38.135 | INFO | MemInfo: { | 20260507 21:06:38.135 | INFO | "gpu" : " 17 GB", | 20260507 21:06:38.135 | INFO | "huge" : "", | 20260507 21:06:38.135 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:38.135 | INFO | "malloc" : " 14 GB", | 20260507 21:06:38.135 | INFO | "pinned" : " 29 GB" | 20260507 21:06:38.135 | INFO | } | 20260507 21:06:38.135 | INFO | Perf report for the last 5.13981 seconds: | 20260507 21:06:38.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:38.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:38.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:38.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:38.135 | INFO | ANALYSIS : 69.2%, 2.9, 105.3ms, 0 | 20260507 21:06:38.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:38.135 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 21:06:38.135 | INFO | PRE_HQ : 68.0%, 1.8, 63.7ms, 0 | 20260507 21:06:38.135 | INFO | BAZWRITER : 72.1%, 4.1, 151.5ms, 0 | 20260507 21:06:38.135 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 21:06:38.136 | INFO | Uploaded 36GiB to GPU in 2.58769s (13.912GiB/s) | 20260507 21:06:38.136 | INFO | Downloaded 38GiB to GPU in 2.73046s (13.9171GiB/s) | 20260507 21:06:43.274 | INFO | Analyzing chunk frames = [2597376,2597888) | 20260507 21:06:43.274 | INFO | MemInfo: { | 20260507 21:06:43.274 | INFO | "gpu" : " 17 GB", | 20260507 21:06:43.274 | INFO | "huge" : "", | 20260507 21:06:43.274 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:43.274 | INFO | "malloc" : " 14 GB", | 20260507 21:06:43.274 | INFO | "pinned" : " 29 GB" | 20260507 21:06:43.274 | INFO | } | 20260507 21:06:43.274 | INFO | Perf report for the last 5.1385 seconds: | 20260507 21:06:43.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:43.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:06:43.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:06:43.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:43.274 | INFO | Uploaded 36GiB to GPU in 2.56917s (14.0123GiB/s) | 20260507 21:06:43.274 | INFO | Downloaded 34GiB to GPU in 2.50113s (13.5939GiB/s) | 20260507 21:06:48.413 | INFO | Analyzing chunk frames = [2597888,2598400) | 20260507 21:06:48.413 | INFO | MemInfo: { | 20260507 21:06:48.413 | INFO | "gpu" : " 17 GB", | 20260507 21:06:48.413 | INFO | "huge" : "", | 20260507 21:06:48.413 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:48.413 | INFO | "malloc" : " 14 GB", | 20260507 21:06:48.413 | INFO | "pinned" : " 29 GB" | 20260507 21:06:48.413 | INFO | } | 20260507 21:06:48.413 | INFO | Perf report for the last 5.13936 seconds: | 20260507 21:06:48.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:48.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:06:48.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:06:48.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:48.413 | INFO | Uploaded 36GiB to GPU in 2.47829s (14.5261GiB/s) | 20260507 21:06:48.413 | INFO | Downloaded 34GiB to GPU in 2.48219s (13.6976GiB/s) | 20260507 21:06:53.551 | INFO | Analyzing chunk frames = [2598400,2598912) | 20260507 21:06:53.551 | INFO | MemInfo: { | 20260507 21:06:53.551 | INFO | "gpu" : " 17 GB", | 20260507 21:06:53.551 | INFO | "huge" : "", | 20260507 21:06:53.551 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:53.551 | INFO | "malloc" : " 14 GB", | 20260507 21:06:53.551 | INFO | "pinned" : " 29 GB" | 20260507 21:06:53.551 | INFO | } | 20260507 21:06:53.551 | INFO | Perf report for the last 5.13814 seconds: | 20260507 21:06:53.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:53.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.551 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:06:53.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.551 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:06:53.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:53.552 | INFO | Uploaded 36GiB to GPU in 2.49628s (14.4215GiB/s) | 20260507 21:06:53.552 | INFO | Downloaded 34GiB to GPU in 2.48483s (13.683GiB/s) | 20260507 21:06:58.691 | INFO | Analyzing chunk frames = [2598912,2599424) | 20260507 21:06:58.691 | INFO | MemInfo: { | 20260507 21:06:58.691 | INFO | "gpu" : " 17 GB", | 20260507 21:06:58.691 | INFO | "huge" : "", | 20260507 21:06:58.691 | INFO | "huge_shared" : " 29 GB", | 20260507 21:06:58.691 | INFO | "malloc" : " 14 GB", | 20260507 21:06:58.691 | INFO | "pinned" : " 29 GB" | 20260507 21:06:58.691 | INFO | } | 20260507 21:06:58.691 | INFO | Perf report for the last 5.13982 seconds: | 20260507 21:06:58.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:06:58.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260507 21:06:58.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:06:58.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:06:58.691 | INFO | Uploaded 36GiB to GPU in 2.43633s (14.7763GiB/s) | 20260507 21:06:58.691 | INFO | Downloaded 34GiB to GPU in 2.49487s (13.628GiB/s) | 20260507 21:07:03.830 | INFO | Analyzing chunk frames = [2599424,2599936) | 20260507 21:07:03.830 | INFO | MemInfo: { | 20260507 21:07:03.830 | INFO | "gpu" : " 17 GB", | 20260507 21:07:03.830 | INFO | "huge" : "", | 20260507 21:07:03.830 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:03.830 | INFO | "malloc" : " 14 GB", | 20260507 21:07:03.830 | INFO | "pinned" : " 29 GB" | 20260507 21:07:03.830 | INFO | } | 20260507 21:07:03.831 | INFO | Perf report for the last 5.13941 seconds: | 20260507 21:07:03.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:03.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:07:03.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:07:03.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:03.831 | INFO | Uploaded 36GiB to GPU in 2.39979s (15.0013GiB/s) | 20260507 21:07:03.831 | INFO | Downloaded 34GiB to GPU in 2.39383s (14.2032GiB/s) | 20260507 21:07:08.970 | INFO | Analyzing chunk frames = [2599936,2600448) | 20260507 21:07:08.970 | INFO | MemInfo: { | 20260507 21:07:08.970 | INFO | "gpu" : " 17 GB", | 20260507 21:07:08.970 | INFO | "huge" : "", | 20260507 21:07:08.970 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:08.970 | INFO | "malloc" : " 14 GB", | 20260507 21:07:08.970 | INFO | "pinned" : " 29 GB" | 20260507 21:07:08.970 | INFO | } | 20260507 21:07:08.970 | INFO | Perf report for the last 5.13927 seconds: | 20260507 21:07:08.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:08.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.970 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:07:08.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.970 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 21:07:08.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:08.971 | INFO | Uploaded 36GiB to GPU in 2.5333s (14.2107GiB/s) | 20260507 21:07:08.971 | INFO | Downloaded 34GiB to GPU in 2.45513s (13.8485GiB/s) | 20260507 21:07:14.109 | INFO | Analyzing chunk frames = [2600448,2600960) | 20260507 21:07:14.109 | INFO | MemInfo: { | 20260507 21:07:14.109 | INFO | "gpu" : " 17 GB", | 20260507 21:07:14.109 | INFO | "huge" : "", | 20260507 21:07:14.109 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:14.109 | INFO | "malloc" : " 14 GB", | 20260507 21:07:14.109 | INFO | "pinned" : " 29 GB" | 20260507 21:07:14.109 | INFO | } | 20260507 21:07:14.109 | INFO | Perf report for the last 5.13915 seconds: | 20260507 21:07:14.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:14.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:07:14.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 21:07:14.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:14.109 | INFO | Uploaded 36GiB to GPU in 2.52938s (14.2327GiB/s) | 20260507 21:07:14.110 | INFO | Downloaded 34GiB to GPU in 2.44235s (13.921GiB/s) | 20260507 21:07:14.222 | INFO | Starting a round of Baz IO | 20260507 21:07:17.716 | INFO | Populating RTMetrics report | 20260507 21:07:17.716 | INFO | RTMetrics report populated | 20260507 21:07:17.716 | INFO | Opening tmp RTMetrics json file | 20260507 21:07:17.740 | INFO | Serializing RTMetrics to json | 20260507 21:07:17.751 | INFO | Serialization done, swapping tmp file now... | 20260507 21:07:17.771 | INFO | RTMetrics done | 20260507 21:07:17.814 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.591 seconds | 20260507 21:07:17.814 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.96% / 0.01% / 56.04% | 20260507 21:07:17.814 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7891% / 1.19209e-05% / 3.57628e-05% | 20260507 21:07:17.814 | INFO | Baz IO overall: | 20260507 21:07:17.814 | INFO | BytesWritten : 1183315131840 | 20260507 21:07:17.814 | INFO | NumEvents : 573572738941 | 20260507 21:07:19.250 | INFO | Analyzing chunk frames = [2600960,2601472) | 20260507 21:07:19.250 | INFO | MemInfo: { | 20260507 21:07:19.250 | INFO | "gpu" : " 17 GB", | 20260507 21:07:19.250 | INFO | "huge" : "", | 20260507 21:07:19.250 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:19.250 | INFO | "malloc" : " 14 GB", | 20260507 21:07:19.250 | INFO | "pinned" : " 29 GB" | 20260507 21:07:19.250 | INFO | } | 20260507 21:07:19.250 | INFO | Perf report for the last 5.14122 seconds: | 20260507 21:07:19.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:19.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:19.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:19.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:19.250 | INFO | ANALYSIS : 69.2%, 2.9, 104.9ms, 0 | 20260507 21:07:19.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:19.250 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260507 21:07:19.250 | INFO | PRE_HQ : 68.3%, 1.9, 67.3ms, 0 | 20260507 21:07:19.250 | INFO | BAZWRITER : 69.4%, 3.9, 138.9ms, 0 | 20260507 21:07:19.250 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:07:19.250 | INFO | Uploaded 36GiB to GPU in 2.59912s (13.8508GiB/s) | 20260507 21:07:19.251 | INFO | Downloaded 38GiB to GPU in 2.72229s (13.9588GiB/s) | 20260507 21:07:24.389 | INFO | Analyzing chunk frames = [2601472,2601984) | 20260507 21:07:24.389 | INFO | MemInfo: { | 20260507 21:07:24.389 | INFO | "gpu" : " 17 GB", | 20260507 21:07:24.389 | INFO | "huge" : "", | 20260507 21:07:24.389 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:24.389 | INFO | "malloc" : " 14 GB", | 20260507 21:07:24.389 | INFO | "pinned" : " 29 GB" | 20260507 21:07:24.389 | INFO | } | 20260507 21:07:24.389 | INFO | Perf report for the last 5.13869 seconds: | 20260507 21:07:24.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:24.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:07:24.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:07:24.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:24.389 | INFO | Uploaded 36GiB to GPU in 2.56709s (14.0237GiB/s) | 20260507 21:07:24.389 | INFO | Downloaded 34GiB to GPU in 2.47927s (13.7137GiB/s) | 20260507 21:07:29.527 | INFO | Analyzing chunk frames = [2601984,2602496) | 20260507 21:07:29.527 | INFO | MemInfo: { | 20260507 21:07:29.527 | INFO | "gpu" : " 17 GB", | 20260507 21:07:29.527 | INFO | "huge" : "", | 20260507 21:07:29.527 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:29.527 | INFO | "malloc" : " 14 GB", | 20260507 21:07:29.527 | INFO | "pinned" : " 29 GB" | 20260507 21:07:29.527 | INFO | } | 20260507 21:07:29.527 | INFO | Perf report for the last 5.13836 seconds: | 20260507 21:07:29.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:29.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.527 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:07:29.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.527 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:07:29.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:29.528 | INFO | Uploaded 36GiB to GPU in 2.50754s (14.3567GiB/s) | 20260507 21:07:29.528 | INFO | Downloaded 34GiB to GPU in 2.47739s (13.7241GiB/s) | 20260507 21:07:34.666 | INFO | Analyzing chunk frames = [2602496,2603008) | 20260507 21:07:34.666 | INFO | MemInfo: { | 20260507 21:07:34.666 | INFO | "gpu" : " 17 GB", | 20260507 21:07:34.666 | INFO | "huge" : "", | 20260507 21:07:34.666 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:34.666 | INFO | "malloc" : " 14 GB", | 20260507 21:07:34.666 | INFO | "pinned" : " 29 GB" | 20260507 21:07:34.666 | INFO | } | 20260507 21:07:34.666 | INFO | Perf report for the last 5.13864 seconds: | 20260507 21:07:34.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:34.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.666 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 21:07:34.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.666 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:07:34.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:34.667 | INFO | Uploaded 36GiB to GPU in 2.48635s (14.4791GiB/s) | 20260507 21:07:34.667 | INFO | Downloaded 34GiB to GPU in 2.49699s (13.6164GiB/s) | 20260507 21:07:39.806 | INFO | Analyzing chunk frames = [2603008,2603520) | 20260507 21:07:39.806 | INFO | MemInfo: { | 20260507 21:07:39.806 | INFO | "gpu" : " 17 GB", | 20260507 21:07:39.806 | INFO | "huge" : "", | 20260507 21:07:39.806 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:39.806 | INFO | "malloc" : " 14 GB", | 20260507 21:07:39.806 | INFO | "pinned" : " 29 GB" | 20260507 21:07:39.806 | INFO | } | 20260507 21:07:39.806 | INFO | Perf report for the last 5.14014 seconds: | 20260507 21:07:39.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:39.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 21:07:39.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 21:07:39.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:39.806 | INFO | Uploaded 36GiB to GPU in 2.39708s (15.0183GiB/s) | 20260507 21:07:39.806 | INFO | Downloaded 34GiB to GPU in 2.48776s (13.6669GiB/s) | 20260507 21:07:44.945 | INFO | Analyzing chunk frames = [2603520,2604032) | 20260507 21:07:44.945 | INFO | MemInfo: { | 20260507 21:07:44.945 | INFO | "gpu" : " 17 GB", | 20260507 21:07:44.945 | INFO | "huge" : "", | 20260507 21:07:44.945 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:44.945 | INFO | "malloc" : " 14 GB", | 20260507 21:07:44.945 | INFO | "pinned" : " 29 GB" | 20260507 21:07:44.946 | INFO | } | 20260507 21:07:44.946 | INFO | Perf report for the last 5.13957 seconds: | 20260507 21:07:44.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:44.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260507 21:07:44.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:07:44.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:44.946 | INFO | Uploaded 36GiB to GPU in 2.42969s (14.8167GiB/s) | 20260507 21:07:44.946 | INFO | Downloaded 34GiB to GPU in 2.38086s (14.2805GiB/s) | 20260507 21:07:50.086 | INFO | Analyzing chunk frames = [2604032,2604544) | 20260507 21:07:50.086 | INFO | MemInfo: { | 20260507 21:07:50.086 | INFO | "gpu" : " 17 GB", | 20260507 21:07:50.086 | INFO | "huge" : "", | 20260507 21:07:50.086 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:50.086 | INFO | "malloc" : " 14 GB", | 20260507 21:07:50.086 | INFO | "pinned" : " 29 GB" | 20260507 21:07:50.086 | INFO | } | 20260507 21:07:50.087 | INFO | Perf report for the last 5.14079 seconds: | 20260507 21:07:50.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:50.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:07:50.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:07:50.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:50.087 | INFO | Uploaded 36GiB to GPU in 2.55686s (14.0798GiB/s) | 20260507 21:07:50.087 | INFO | Downloaded 34GiB to GPU in 2.43529s (13.9614GiB/s) | 20260507 21:07:55.224 | INFO | Analyzing chunk frames = [2604544,2605056) | 20260507 21:07:55.224 | INFO | MemInfo: { | 20260507 21:07:55.224 | INFO | "gpu" : " 17 GB", | 20260507 21:07:55.224 | INFO | "huge" : "", | 20260507 21:07:55.224 | INFO | "huge_shared" : " 29 GB", | 20260507 21:07:55.224 | INFO | "malloc" : " 14 GB", | 20260507 21:07:55.225 | INFO | "pinned" : " 29 GB" | 20260507 21:07:55.225 | INFO | } | 20260507 21:07:55.225 | INFO | Perf report for the last 5.13827 seconds: | 20260507 21:07:55.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:07:55.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:07:55.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | PRE_HQ : 35.7%, 1.0, 18.6ms, 0 | 20260507 21:07:55.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:07:55.225 | INFO | Uploaded 36GiB to GPU in 2.55309s (14.1006GiB/s) | 20260507 21:07:55.225 | INFO | Downloaded 34GiB to GPU in 2.44063s (13.9308GiB/s) | 20260507 21:07:55.226 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:07:55.226 | INFO | Total mem usage: 16.7 GiB | 20260507 21:07:55.226 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:07:55.226 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260507 21:07:55.226 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:07:55.226 | INFO | High Water Breakdown: | 20260507 21:07:55.226 | INFO | 59.8% : 8.85 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:07:55.226 | INFO | 40.1% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:07:55.226 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:07:55.226 | INFO | ------------------------------------------ | 20260507 21:07:55.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:07:55.226 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:07:55.226 | INFO | Total mem usage: 26.5 GiB | 20260507 21:07:55.226 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:07:55.226 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:07:55.226 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:07:55.226 | INFO | High Water Breakdown: | 20260507 21:07:55.226 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:07:55.226 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:07:55.226 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:07:55.226 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:07:55.226 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:07:55.226 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:07:55.226 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:07:55.226 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:07:55.226 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:07:55.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:07:55.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:07:55.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:07:55.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:07:55.226 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:07:55.226 | INFO | ------------------------------------------ | 20260507 21:07:55.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:07:55.226 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:07:55.226 | INFO | Total mem usage: 27.8 GiB | 20260507 21:07:55.226 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:07:55.226 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:07:55.226 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:07:55.226 | INFO | High Water Breakdown: | 20260507 21:07:55.226 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:07:55.226 | INFO | ------------------------------------------ | 20260507 21:07:55.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:07:55.226 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:07:55.226 | INFO | Total mem usage: 15.7 GiB | 20260507 21:07:55.226 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:07:55.226 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:07:55.226 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:07:55.226 | INFO | High Water Breakdown: | 20260507 21:07:55.226 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:07:55.226 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:07:55.226 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:07:55.226 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:07:55.226 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:07:55.226 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:07:55.226 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:07:55.226 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:07:55.226 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:07:55.226 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:07:55.226 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:07:55.226 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:07:55.227 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:07:55.227 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:07:55.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:07:55.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:07:55.227 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:07:55.227 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:07:55.227 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:07:55.227 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:07:55.227 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:07:55.227 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:07:55.227 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:07:55.227 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:07:55.227 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:07:55.227 | INFO | ------------------------------------------ | 20260507 21:07:55.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:07:55.227 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:07:55.227 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:07:55.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:07:55.227 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.40e+01 / 1.91e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.03e-01, 1.27e-02 / 5.91e-03 | 20260507 21:07:55.227 | INFO | PerfReport: 12.10%, Baseline -- 3.99e+00 / 4.47e+00, 5.87e-01 / 2.69e-01 | 20260507 21:07:55.227 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.08e-01 / 1.11e-01 | 20260507 21:07:55.227 | INFO | PerfReport: 16.10%, DME -- 4.25e+01 / 4.60e+01, 4.28e+00 / 2.03e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 55.85%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.64e+00 / 1.38e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.52e+00 / 2.84e+00, 4.42e-01 / 2.29e-01 | 20260507 21:07:55.227 | INFO | PerfReport: 3.54%, Metrics -- 1.17e+00 / 1.36e+00, 2.90e-01 / 1.37e-01 | 20260507 21:07:55.227 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.65e-01 / 3.96e-01, 5.97e-02 / 2.82e-02 | 20260507 21:07:55.227 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:07:55.227 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:07:55.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:07:55.227 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 24.32%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 30.90%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 2.12%, LockUpload -- 2.27e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:07:55.227 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.77e-02 / 0.00e+00 | 20260507 21:07:55.337 | INFO | Starting a round of Baz IO | 20260507 21:07:58.833 | INFO | Populating RTMetrics report | 20260507 21:07:58.834 | INFO | RTMetrics report populated | 20260507 21:07:58.834 | INFO | Opening tmp RTMetrics json file | 20260507 21:07:58.862 | INFO | Serializing RTMetrics to json | 20260507 21:07:58.873 | INFO | Serialization done, swapping tmp file now... | 20260507 21:07:58.893 | INFO | RTMetrics done | 20260507 21:07:59.048 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.711 seconds | 20260507 21:07:59.048 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.95% / 0.01% / 56.04% | 20260507 21:07:59.048 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.78887% / 2.78155e-05% / 2.78155e-05% | 20260507 21:07:59.048 | INFO | Baz IO overall: | 20260507 21:07:59.048 | INFO | BytesWritten : 1185212742697 | 20260507 21:07:59.048 | INFO | NumEvents : 574491117515 | 20260507 21:08:00.363 | INFO | Analyzing chunk frames = [2605056,2605568) | 20260507 21:08:00.363 | INFO | MemInfo: { | 20260507 21:08:00.363 | INFO | "gpu" : " 17 GB", | 20260507 21:08:00.363 | INFO | "huge" : "", | 20260507 21:08:00.363 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:00.363 | INFO | "malloc" : " 14 GB", | 20260507 21:08:00.363 | INFO | "pinned" : " 29 GB" | 20260507 21:08:00.363 | INFO | } | 20260507 21:08:00.363 | INFO | Perf report for the last 5.13831 seconds: | 20260507 21:08:00.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:00.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:00.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:00.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:00.363 | INFO | ANALYSIS : 69.5%, 2.9, 105.4ms, 0 | 20260507 21:08:00.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:00.363 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 21:08:00.363 | INFO | PRE_HQ : 68.4%, 1.8, 63.2ms, 0 | 20260507 21:08:00.363 | INFO | BAZWRITER : 71.8%, 4.2, 156.0ms, 0 | 20260507 21:08:00.363 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:08:00.363 | INFO | Uploaded 36GiB to GPU in 2.60754s (13.8061GiB/s) | 20260507 21:08:00.364 | INFO | Downloaded 38GiB to GPU in 2.71247s (14.0093GiB/s) | 20260507 21:08:05.502 | INFO | Analyzing chunk frames = [2605568,2606080) | 20260507 21:08:05.502 | INFO | MemInfo: { | 20260507 21:08:05.502 | INFO | "gpu" : " 17 GB", | 20260507 21:08:05.502 | INFO | "huge" : "", | 20260507 21:08:05.502 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:05.502 | INFO | "malloc" : " 14 GB", | 20260507 21:08:05.502 | INFO | "pinned" : " 29 GB" | 20260507 21:08:05.502 | INFO | } | 20260507 21:08:05.502 | INFO | Perf report for the last 5.13928 seconds: | 20260507 21:08:05.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:05.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.503 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:08:05.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.503 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:08:05.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:05.503 | INFO | Uploaded 36GiB to GPU in 2.55249s (14.1039GiB/s) | 20260507 21:08:05.503 | INFO | Downloaded 34GiB to GPU in 2.50825s (13.5553GiB/s) | 20260507 21:08:10.642 | INFO | Analyzing chunk frames = [2606080,2606592) | 20260507 21:08:10.642 | INFO | MemInfo: { | 20260507 21:08:10.642 | INFO | "gpu" : " 17 GB", | 20260507 21:08:10.642 | INFO | "huge" : "", | 20260507 21:08:10.642 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:10.642 | INFO | "malloc" : " 14 GB", | 20260507 21:08:10.642 | INFO | "pinned" : " 29 GB" | 20260507 21:08:10.642 | INFO | } | 20260507 21:08:10.642 | INFO | Perf report for the last 5.13944 seconds: | 20260507 21:08:10.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:10.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:08:10.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:08:10.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:10.642 | INFO | Uploaded 36GiB to GPU in 2.49495s (14.4292GiB/s) | 20260507 21:08:10.642 | INFO | Downloaded 34GiB to GPU in 2.48758s (13.6679GiB/s) | 20260507 21:08:15.781 | INFO | Analyzing chunk frames = [2606592,2607104) | 20260507 21:08:15.781 | INFO | MemInfo: { | 20260507 21:08:15.781 | INFO | "gpu" : " 17 GB", | 20260507 21:08:15.781 | INFO | "huge" : "", | 20260507 21:08:15.781 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:15.781 | INFO | "malloc" : " 14 GB", | 20260507 21:08:15.781 | INFO | "pinned" : " 29 GB" | 20260507 21:08:15.781 | INFO | } | 20260507 21:08:15.781 | INFO | Perf report for the last 5.1394 seconds: | 20260507 21:08:15.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:15.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:08:15.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:08:15.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:15.782 | INFO | Uploaded 36GiB to GPU in 2.45201s (14.6818GiB/s) | 20260507 21:08:15.782 | INFO | Downloaded 34GiB to GPU in 2.40931s (14.1119GiB/s) | 20260507 21:08:20.921 | INFO | Analyzing chunk frames = [2607104,2607616) | 20260507 21:08:20.921 | INFO | MemInfo: { | 20260507 21:08:20.921 | INFO | "gpu" : " 17 GB", | 20260507 21:08:20.921 | INFO | "huge" : "", | 20260507 21:08:20.921 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:20.921 | INFO | "malloc" : " 14 GB", | 20260507 21:08:20.921 | INFO | "pinned" : " 29 GB" | 20260507 21:08:20.921 | INFO | } | 20260507 21:08:20.922 | INFO | Perf report for the last 5.14022 seconds: | 20260507 21:08:20.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:20.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:08:20.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:08:20.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:20.922 | INFO | Uploaded 36GiB to GPU in 2.30462s (15.6208GiB/s) | 20260507 21:08:20.922 | INFO | Downloaded 34GiB to GPU in 2.48272s (13.6947GiB/s) | 20260507 21:08:26.060 | INFO | Analyzing chunk frames = [2607616,2608128) | 20260507 21:08:26.060 | INFO | MemInfo: { | 20260507 21:08:26.060 | INFO | "gpu" : " 17 GB", | 20260507 21:08:26.060 | INFO | "huge" : "", | 20260507 21:08:26.061 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:26.061 | INFO | "malloc" : " 14 GB", | 20260507 21:08:26.061 | INFO | "pinned" : " 29 GB" | 20260507 21:08:26.061 | INFO | } | 20260507 21:08:26.061 | INFO | Perf report for the last 5.13916 seconds: | 20260507 21:08:26.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:26.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260507 21:08:26.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:08:26.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:26.061 | INFO | Uploaded 36GiB to GPU in 2.34927s (15.3239GiB/s) | 20260507 21:08:26.061 | INFO | Downloaded 34GiB to GPU in 2.40964s (14.11GiB/s) | 20260507 21:08:31.200 | INFO | Analyzing chunk frames = [2608128,2608640) | 20260507 21:08:31.200 | INFO | MemInfo: { | 20260507 21:08:31.200 | INFO | "gpu" : " 17 GB", | 20260507 21:08:31.200 | INFO | "huge" : "", | 20260507 21:08:31.200 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:31.200 | INFO | "malloc" : " 14 GB", | 20260507 21:08:31.200 | INFO | "pinned" : " 29 GB" | 20260507 21:08:31.201 | INFO | } | 20260507 21:08:31.201 | INFO | Perf report for the last 5.13992 seconds: | 20260507 21:08:31.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:31.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 21:08:31.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 21:08:31.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:31.201 | INFO | Uploaded 36GiB to GPU in 2.47276s (14.5587GiB/s) | 20260507 21:08:31.201 | INFO | Downloaded 34GiB to GPU in 2.46531s (13.7914GiB/s) | 20260507 21:08:36.340 | INFO | Analyzing chunk frames = [2608640,2609152) | 20260507 21:08:36.340 | INFO | MemInfo: { | 20260507 21:08:36.340 | INFO | "gpu" : " 17 GB", | 20260507 21:08:36.340 | INFO | "huge" : "", | 20260507 21:08:36.340 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:36.340 | INFO | "malloc" : " 14 GB", | 20260507 21:08:36.340 | INFO | "pinned" : " 29 GB" | 20260507 21:08:36.340 | INFO | } | 20260507 21:08:36.340 | INFO | Perf report for the last 5.13965 seconds: | 20260507 21:08:36.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:36.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.340 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:08:36.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.340 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 21:08:36.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:36.341 | INFO | Uploaded 36GiB to GPU in 2.47282s (14.5583GiB/s) | 20260507 21:08:36.341 | INFO | Downloaded 34GiB to GPU in 2.44974s (13.879GiB/s) | 20260507 21:08:36.454 | INFO | Starting a round of Baz IO | 20260507 21:08:39.940 | INFO | Populating RTMetrics report | 20260507 21:08:39.941 | INFO | RTMetrics report populated | 20260507 21:08:39.941 | INFO | Opening tmp RTMetrics json file | 20260507 21:08:39.953 | INFO | Serializing RTMetrics to json | 20260507 21:08:39.965 | INFO | Serialization done, swapping tmp file now... | 20260507 21:08:39.966 | INFO | RTMetrics done | 20260507 21:08:40.037 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.583 seconds | 20260507 21:08:40.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.94% / 0.01% / 56.05% | 20260507 21:08:40.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.78704% / 3.57628e-05% / 4.37101e-05% | 20260507 21:08:40.037 | INFO | Baz IO overall: | 20260507 21:08:40.037 | INFO | BytesWritten : 1187110088814 | 20260507 21:08:40.037 | INFO | NumEvents : 575409216650 | 20260507 21:08:41.477 | INFO | Analyzing chunk frames = [2609152,2609664) | 20260507 21:08:41.477 | INFO | MemInfo: { | 20260507 21:08:41.477 | INFO | "gpu" : " 17 GB", | 20260507 21:08:41.477 | INFO | "huge" : "", | 20260507 21:08:41.477 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:41.477 | INFO | "malloc" : " 14 GB", | 20260507 21:08:41.477 | INFO | "pinned" : " 29 GB" | 20260507 21:08:41.477 | INFO | } | 20260507 21:08:41.477 | INFO | Perf report for the last 5.13704 seconds: | 20260507 21:08:41.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:41.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:41.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:41.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:41.477 | INFO | ANALYSIS : 69.3%, 2.9, 105.0ms, 0 | 20260507 21:08:41.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:41.477 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 21:08:41.477 | INFO | PRE_HQ : 68.3%, 1.8, 64.0ms, 0 | 20260507 21:08:41.478 | INFO | BAZWRITER : 69.3%, 4.3, 152.6ms, 0 | 20260507 21:08:41.478 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 21:08:41.478 | INFO | Uploaded 36GiB to GPU in 2.58916s (13.9041GiB/s) | 20260507 21:08:41.478 | INFO | Downloaded 38GiB to GPU in 2.71266s (14.0084GiB/s) | 20260507 21:08:46.617 | INFO | Analyzing chunk frames = [2609664,2610176) | 20260507 21:08:46.617 | INFO | MemInfo: { | 20260507 21:08:46.617 | INFO | "gpu" : " 17 GB", | 20260507 21:08:46.617 | INFO | "huge" : "", | 20260507 21:08:46.617 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:46.617 | INFO | "malloc" : " 14 GB", | 20260507 21:08:46.617 | INFO | "pinned" : " 29 GB" | 20260507 21:08:46.617 | INFO | } | 20260507 21:08:46.617 | INFO | Perf report for the last 5.13949 seconds: | 20260507 21:08:46.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:46.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 21:08:46.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:08:46.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:46.617 | INFO | Uploaded 36GiB to GPU in 2.51055s (14.3395GiB/s) | 20260507 21:08:46.618 | INFO | Downloaded 34GiB to GPU in 2.4786s (13.7174GiB/s) | 20260507 21:08:51.757 | INFO | Analyzing chunk frames = [2610176,2610688) | 20260507 21:08:51.757 | INFO | MemInfo: { | 20260507 21:08:51.757 | INFO | "gpu" : " 17 GB", | 20260507 21:08:51.757 | INFO | "huge" : "", | 20260507 21:08:51.757 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:51.757 | INFO | "malloc" : " 14 GB", | 20260507 21:08:51.757 | INFO | "pinned" : " 29 GB" | 20260507 21:08:51.757 | INFO | } | 20260507 21:08:51.757 | INFO | Perf report for the last 5.14 seconds: | 20260507 21:08:51.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:51.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:08:51.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:08:51.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:51.757 | INFO | Uploaded 36GiB to GPU in 2.48541s (14.4845GiB/s) | 20260507 21:08:51.758 | INFO | Downloaded 34GiB to GPU in 2.48555s (13.679GiB/s) | 20260507 21:08:56.896 | INFO | Analyzing chunk frames = [2610688,2611200) | 20260507 21:08:56.896 | INFO | MemInfo: { | 20260507 21:08:56.896 | INFO | "gpu" : " 17 GB", | 20260507 21:08:56.896 | INFO | "huge" : "", | 20260507 21:08:56.896 | INFO | "huge_shared" : " 29 GB", | 20260507 21:08:56.896 | INFO | "malloc" : " 14 GB", | 20260507 21:08:56.896 | INFO | "pinned" : " 29 GB" | 20260507 21:08:56.896 | INFO | } | 20260507 21:08:56.897 | INFO | Perf report for the last 5.13963 seconds: | 20260507 21:08:56.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:08:56.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 21:08:56.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:08:56.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:08:56.897 | INFO | Uploaded 36GiB to GPU in 2.47307s (14.5568GiB/s) | 20260507 21:08:56.897 | INFO | Downloaded 34GiB to GPU in 2.49793s (13.6113GiB/s) | 20260507 21:09:02.036 | INFO | Analyzing chunk frames = [2611200,2611712) | 20260507 21:09:02.036 | INFO | MemInfo: { | 20260507 21:09:02.036 | INFO | "gpu" : " 17 GB", | 20260507 21:09:02.036 | INFO | "huge" : "", | 20260507 21:09:02.036 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:02.036 | INFO | "malloc" : " 14 GB", | 20260507 21:09:02.036 | INFO | "pinned" : " 29 GB" | 20260507 21:09:02.036 | INFO | } | 20260507 21:09:02.036 | INFO | Perf report for the last 5.13958 seconds: | 20260507 21:09:02.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:02.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 21:09:02.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:09:02.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:02.036 | INFO | Uploaded 36GiB to GPU in 2.40717s (14.9553GiB/s) | 20260507 21:09:02.036 | INFO | Downloaded 34GiB to GPU in 2.4867s (13.6728GiB/s) | 20260507 21:09:07.174 | INFO | Analyzing chunk frames = [2611712,2612224) | 20260507 21:09:07.174 | INFO | MemInfo: { | 20260507 21:09:07.174 | INFO | "gpu" : " 17 GB", | 20260507 21:09:07.174 | INFO | "huge" : "", | 20260507 21:09:07.174 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:07.174 | INFO | "malloc" : " 14 GB", | 20260507 21:09:07.174 | INFO | "pinned" : " 29 GB" | 20260507 21:09:07.174 | INFO | } | 20260507 21:09:07.174 | INFO | Perf report for the last 5.1379 seconds: | 20260507 21:09:07.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:07.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.174 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:09:07.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.174 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:09:07.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:07.175 | INFO | Uploaded 36GiB to GPU in 2.43234s (14.8006GiB/s) | 20260507 21:09:07.175 | INFO | Downloaded 34GiB to GPU in 2.38181s (14.2749GiB/s) | 20260507 21:09:12.314 | INFO | Analyzing chunk frames = [2612224,2612736) | 20260507 21:09:12.314 | INFO | MemInfo: { | 20260507 21:09:12.314 | INFO | "gpu" : " 17 GB", | 20260507 21:09:12.314 | INFO | "huge" : "", | 20260507 21:09:12.314 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:12.314 | INFO | "malloc" : " 14 GB", | 20260507 21:09:12.314 | INFO | "pinned" : " 29 GB" | 20260507 21:09:12.314 | INFO | } | 20260507 21:09:12.314 | INFO | Perf report for the last 5.14039 seconds: | 20260507 21:09:12.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:12.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.314 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 21:09:12.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.315 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:09:12.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:12.315 | INFO | Uploaded 36GiB to GPU in 2.56881s (14.0143GiB/s) | 20260507 21:09:12.315 | INFO | Downloaded 34GiB to GPU in 2.46029s (13.8195GiB/s) | 20260507 21:09:17.454 | INFO | Analyzing chunk frames = [2612736,2613248) | 20260507 21:09:17.454 | INFO | MemInfo: { | 20260507 21:09:17.454 | INFO | "gpu" : " 17 GB", | 20260507 21:09:17.454 | INFO | "huge" : "", | 20260507 21:09:17.454 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:17.454 | INFO | "malloc" : " 14 GB", | 20260507 21:09:17.454 | INFO | "pinned" : " 29 GB" | 20260507 21:09:17.454 | INFO | } | 20260507 21:09:17.454 | INFO | Perf report for the last 5.13942 seconds: | 20260507 21:09:17.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:17.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:09:17.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 21:09:17.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:17.454 | INFO | Uploaded 36GiB to GPU in 2.53832s (14.1826GiB/s) | 20260507 21:09:17.454 | INFO | Downloaded 34GiB to GPU in 2.42805s (14.003GiB/s) | 20260507 21:09:17.455 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:09:17.455 | INFO | Total mem usage: 16.7 GiB | 20260507 21:09:17.455 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:09:17.455 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:09:17.455 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:09:17.455 | INFO | High Water Breakdown: | 20260507 21:09:17.455 | INFO | 59.9% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:09:17.455 | INFO | 40% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:09:17.455 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:09:17.455 | INFO | ------------------------------------------ | 20260507 21:09:17.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:09:17.455 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:09:17.455 | INFO | Total mem usage: 26.5 GiB | 20260507 21:09:17.455 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:09:17.455 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:09:17.455 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:09:17.455 | INFO | High Water Breakdown: | 20260507 21:09:17.455 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:09:17.455 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:09:17.455 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:09:17.455 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:09:17.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:09:17.455 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:09:17.455 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:09:17.455 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:09:17.455 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:09:17.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:09:17.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:09:17.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:09:17.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:09:17.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:09:17.455 | INFO | ------------------------------------------ | 20260507 21:09:17.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:09:17.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:09:17.455 | INFO | Total mem usage: 27.8 GiB | 20260507 21:09:17.455 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:09:17.455 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:09:17.455 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:09:17.455 | INFO | High Water Breakdown: | 20260507 21:09:17.455 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:09:17.455 | INFO | ------------------------------------------ | 20260507 21:09:17.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:09:17.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:09:17.455 | INFO | Total mem usage: 15.7 GiB | 20260507 21:09:17.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:09:17.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:09:17.455 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 21:09:17.455 | INFO | High Water Breakdown: | 20260507 21:09:17.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:09:17.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:09:17.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:09:17.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:09:17.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:09:17.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:09:17.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:09:17.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:09:17.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:09:17.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:09:17.456 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:09:17.456 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:09:17.456 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:09:17.456 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:09:17.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:09:17.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:09:17.456 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:09:17.456 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:09:17.456 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:09:17.456 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:09:17.456 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:09:17.456 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:09:17.456 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:09:17.456 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:09:17.456 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:09:17.456 | INFO | ------------------------------------------ | 20260507 21:09:17.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:09:17.456 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:09:17.456 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:09:17.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:09:17.456 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.59e+01, 1.42e+01 / 1.97e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.02e-01, 1.21e-02 / 6.29e-03 | 20260507 21:09:17.456 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.49e+00, 5.93e-01 / 2.05e-01 | 20260507 21:09:17.456 | INFO | PerfReport: 3.37%, Histogram -- 1.12e+00 / 1.49e+00, 2.08e-01 / 1.25e-01 | 20260507 21:09:17.456 | INFO | PerfReport: 16.00%, DME -- 4.23e+01 / 4.58e+01, 4.38e+00 / 1.96e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 55.95%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.68e+00 / 1.50e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.79e+00, 4.41e-01 / 2.86e-01 | 20260507 21:09:17.456 | INFO | PerfReport: 3.54%, Metrics -- 1.17e+00 / 1.34e+00, 2.93e-01 / 1.62e-01 | 20260507 21:09:17.456 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.65e-01 / 4.00e-01, 5.95e-02 / 3.32e-02 | 20260507 21:09:17.456 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:09:17.456 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:09:17.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:09:17.456 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 24.24%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 31.01%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 2.01%, LockUpload -- 2.14e+00 / 0.00e+00, 5.23e+00 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 2.66%, LockDownload -- 2.83e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 15.74%, LockCompute -- 1.68e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 21:09:17.456 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 21:09:17.551 | INFO | Starting a round of Baz IO | 20260507 21:09:21.030 | INFO | Populating RTMetrics report | 20260507 21:09:21.030 | INFO | RTMetrics report populated | 20260507 21:09:21.030 | INFO | Opening tmp RTMetrics json file | 20260507 21:09:21.048 | INFO | Serializing RTMetrics to json | 20260507 21:09:21.050 | INFO | Serialization done, swapping tmp file now... | 20260507 21:09:21.050 | INFO | RTMetrics done | 20260507 21:09:21.130 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.579 seconds | 20260507 21:09:21.131 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.93% / 0.01% / 56.06% | 20260507 21:09:21.131 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77546% / 0% / 3.57628e-05% | 20260507 21:09:21.131 | INFO | Baz IO overall: | 20260507 21:09:21.131 | INFO | BytesWritten : 1189004894434 | 20260507 21:09:21.131 | INFO | NumEvents : 576325276264 | 20260507 21:09:22.593 | INFO | Analyzing chunk frames = [2613248,2613760) | 20260507 21:09:22.593 | INFO | MemInfo: { | 20260507 21:09:22.593 | INFO | "gpu" : " 17 GB", | 20260507 21:09:22.593 | INFO | "huge" : "", | 20260507 21:09:22.593 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:22.593 | INFO | "malloc" : " 14 GB", | 20260507 21:09:22.593 | INFO | "pinned" : " 29 GB" | 20260507 21:09:22.593 | INFO | } | 20260507 21:09:22.593 | INFO | Perf report for the last 5.13931 seconds: | 20260507 21:09:22.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:22.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:22.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:22.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:22.593 | INFO | ANALYSIS : 69.0%, 3.0, 106.7ms, 0 | 20260507 21:09:22.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:22.593 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 21:09:22.593 | INFO | PRE_HQ : 66.3%, 1.7, 57.8ms, 0 | 20260507 21:09:22.593 | INFO | BAZWRITER : 69.2%, 3.7, 133.2ms, 0 | 20260507 21:09:22.593 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 21:09:22.594 | INFO | Uploaded 36GiB to GPU in 2.50997s (14.3428GiB/s) | 20260507 21:09:22.594 | INFO | Downloaded 38GiB to GPU in 2.68113s (14.1731GiB/s) | 20260507 21:09:27.732 | INFO | Analyzing chunk frames = [2613760,2614272) | 20260507 21:09:27.732 | INFO | MemInfo: { | 20260507 21:09:27.732 | INFO | "gpu" : " 17 GB", | 20260507 21:09:27.732 | INFO | "huge" : "", | 20260507 21:09:27.732 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:27.732 | INFO | "malloc" : " 14 GB", | 20260507 21:09:27.732 | INFO | "pinned" : " 29 GB" | 20260507 21:09:27.732 | INFO | } | 20260507 21:09:27.732 | INFO | Perf report for the last 5.13905 seconds: | 20260507 21:09:27.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:27.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.732 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 21:09:27.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.732 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:09:27.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:27.733 | INFO | Uploaded 36GiB to GPU in 2.57627s (13.9737GiB/s) | 20260507 21:09:27.733 | INFO | Downloaded 34GiB to GPU in 2.49251s (13.6409GiB/s) | 20260507 21:09:32.870 | INFO | Analyzing chunk frames = [2614272,2614784) | 20260507 21:09:32.870 | INFO | MemInfo: { | 20260507 21:09:32.870 | INFO | "gpu" : " 17 GB", | 20260507 21:09:32.870 | INFO | "huge" : "", | 20260507 21:09:32.870 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:32.870 | INFO | "malloc" : " 14 GB", | 20260507 21:09:32.870 | INFO | "pinned" : " 29 GB" | 20260507 21:09:32.870 | INFO | } | 20260507 21:09:32.871 | INFO | Perf report for the last 5.1383 seconds: | 20260507 21:09:32.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:32.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:09:32.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:09:32.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:32.871 | INFO | Uploaded 36GiB to GPU in 2.48753s (14.4722GiB/s) | 20260507 21:09:32.871 | INFO | Downloaded 34GiB to GPU in 2.49209s (13.6432GiB/s) | 20260507 21:09:38.010 | INFO | Analyzing chunk frames = [2614784,2615296) | 20260507 21:09:38.010 | INFO | MemInfo: { | 20260507 21:09:38.010 | INFO | "gpu" : " 17 GB", | 20260507 21:09:38.010 | INFO | "huge" : "", | 20260507 21:09:38.010 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:38.010 | INFO | "malloc" : " 14 GB", | 20260507 21:09:38.011 | INFO | "pinned" : " 29 GB" | 20260507 21:09:38.011 | INFO | } | 20260507 21:09:38.011 | INFO | Perf report for the last 5.14018 seconds: | 20260507 21:09:38.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:38.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:09:38.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:09:38.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:38.011 | INFO | Uploaded 36GiB to GPU in 2.50264s (14.3848GiB/s) | 20260507 21:09:38.011 | INFO | Downloaded 34GiB to GPU in 2.50011s (13.5994GiB/s) | 20260507 21:09:43.151 | INFO | Analyzing chunk frames = [2615296,2615808) | 20260507 21:09:43.151 | INFO | MemInfo: { | 20260507 21:09:43.151 | INFO | "gpu" : " 17 GB", | 20260507 21:09:43.151 | INFO | "huge" : "", | 20260507 21:09:43.151 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:43.151 | INFO | "malloc" : " 14 GB", | 20260507 21:09:43.151 | INFO | "pinned" : " 29 GB" | 20260507 21:09:43.151 | INFO | } | 20260507 21:09:43.151 | INFO | Perf report for the last 5.14029 seconds: | 20260507 21:09:43.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:43.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 21:09:43.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:09:43.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:43.151 | INFO | Uploaded 36GiB to GPU in 2.37674s (15.1468GiB/s) | 20260507 21:09:43.152 | INFO | Downloaded 34GiB to GPU in 2.50366s (13.5801GiB/s) | 20260507 21:09:48.289 | INFO | Analyzing chunk frames = [2615808,2616320) | 20260507 21:09:48.289 | INFO | MemInfo: { | 20260507 21:09:48.289 | INFO | "gpu" : " 17 GB", | 20260507 21:09:48.289 | INFO | "huge" : "", | 20260507 21:09:48.289 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:48.289 | INFO | "malloc" : " 14 GB", | 20260507 21:09:48.289 | INFO | "pinned" : " 29 GB" | 20260507 21:09:48.289 | INFO | } | 20260507 21:09:48.289 | INFO | Perf report for the last 5.13812 seconds: | 20260507 21:09:48.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:48.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.289 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 21:09:48.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.289 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 21:09:48.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:48.290 | INFO | Uploaded 36GiB to GPU in 2.40664s (14.9586GiB/s) | 20260507 21:09:48.290 | INFO | Downloaded 34GiB to GPU in 2.39942s (14.1701GiB/s) | 20260507 21:09:53.428 | INFO | Analyzing chunk frames = [2616320,2616832) | 20260507 21:09:53.428 | INFO | MemInfo: { | 20260507 21:09:53.428 | INFO | "gpu" : " 17 GB", | 20260507 21:09:53.428 | INFO | "huge" : "", | 20260507 21:09:53.428 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:53.429 | INFO | "malloc" : " 14 GB", | 20260507 21:09:53.429 | INFO | "pinned" : " 29 GB" | 20260507 21:09:53.429 | INFO | } | 20260507 21:09:53.429 | INFO | Perf report for the last 5.13954 seconds: | 20260507 21:09:53.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:53.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:09:53.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | PRE_HQ : 36.1%, 1.0, 18.8ms, 0 | 20260507 21:09:53.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:53.429 | INFO | Uploaded 36GiB to GPU in 2.53308s (14.2119GiB/s) | 20260507 21:09:53.429 | INFO | Downloaded 34GiB to GPU in 2.46401s (13.7986GiB/s) | 20260507 21:09:58.570 | INFO | Analyzing chunk frames = [2616832,2617344) | 20260507 21:09:58.570 | INFO | MemInfo: { | 20260507 21:09:58.570 | INFO | "gpu" : " 17 GB", | 20260507 21:09:58.570 | INFO | "huge" : "", | 20260507 21:09:58.570 | INFO | "huge_shared" : " 29 GB", | 20260507 21:09:58.570 | INFO | "malloc" : " 14 GB", | 20260507 21:09:58.570 | INFO | "pinned" : " 29 GB" | 20260507 21:09:58.570 | INFO | } | 20260507 21:09:58.570 | INFO | Perf report for the last 5.14154 seconds: | 20260507 21:09:58.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:09:58.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.570 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:09:58.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.571 | INFO | PRE_HQ : 35.5%, 1.0, 18.4ms, 0 | 20260507 21:09:58.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:09:58.571 | INFO | Uploaded 36GiB to GPU in 2.53947s (14.1762GiB/s) | 20260507 21:09:58.571 | INFO | Downloaded 34GiB to GPU in 2.45384s (13.8558GiB/s) | 20260507 21:09:58.696 | INFO | Starting a round of Baz IO | 20260507 21:10:02.174 | INFO | Populating RTMetrics report | 20260507 21:10:02.175 | INFO | RTMetrics report populated | 20260507 21:10:02.175 | INFO | Opening tmp RTMetrics json file | 20260507 21:10:02.194 | INFO | Serializing RTMetrics to json | 20260507 21:10:02.195 | INFO | Serialization done, swapping tmp file now... | 20260507 21:10:02.195 | INFO | RTMetrics done | 20260507 21:10:02.264 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.568 seconds | 20260507 21:10:02.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.92% / 0.01% / 56.07% | 20260507 21:10:02.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77619% / 3.97364e-06% / 3.57628e-05% | 20260507 21:10:02.264 | INFO | Baz IO overall: | 20260507 21:10:02.264 | INFO | BytesWritten : 1190901131010 | 20260507 21:10:02.264 | INFO | NumEvents : 577242683164 | 20260507 21:10:03.708 | INFO | Analyzing chunk frames = [2617344,2617856) | 20260507 21:10:03.708 | INFO | MemInfo: { | 20260507 21:10:03.708 | INFO | "gpu" : " 17 GB", | 20260507 21:10:03.708 | INFO | "huge" : "", | 20260507 21:10:03.708 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:03.708 | INFO | "malloc" : " 14 GB", | 20260507 21:10:03.708 | INFO | "pinned" : " 29 GB" | 20260507 21:10:03.708 | INFO | } | 20260507 21:10:03.708 | INFO | Perf report for the last 5.13807 seconds: | 20260507 21:10:03.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:03.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:03.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:03.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:03.708 | INFO | ANALYSIS : 69.2%, 2.9, 105.0ms, 0 | 20260507 21:10:03.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:03.709 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 21:10:03.709 | INFO | PRE_HQ : 68.3%, 1.9, 65.9ms, 0 | 20260507 21:10:03.709 | INFO | BAZWRITER : 69.5%, 4.1, 145.9ms, 0 | 20260507 21:10:03.709 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 21:10:03.709 | INFO | Uploaded 36GiB to GPU in 2.63248s (13.6753GiB/s) | 20260507 21:10:03.709 | INFO | Downloaded 38GiB to GPU in 2.72601s (13.9398GiB/s) | 20260507 21:10:08.847 | INFO | Analyzing chunk frames = [2617856,2618368) | 20260507 21:10:08.847 | INFO | MemInfo: { | 20260507 21:10:08.847 | INFO | "gpu" : " 17 GB", | 20260507 21:10:08.847 | INFO | "huge" : "", | 20260507 21:10:08.847 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:08.847 | INFO | "malloc" : " 14 GB", | 20260507 21:10:08.847 | INFO | "pinned" : " 29 GB" | 20260507 21:10:08.847 | INFO | } | 20260507 21:10:08.847 | INFO | Perf report for the last 5.13899 seconds: | 20260507 21:10:08.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:08.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.847 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:10:08.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.848 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:10:08.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:08.848 | INFO | Uploaded 36GiB to GPU in 2.58313s (13.9366GiB/s) | 20260507 21:10:08.848 | INFO | Downloaded 34GiB to GPU in 2.49251s (13.6409GiB/s) | 20260507 21:10:13.986 | INFO | Analyzing chunk frames = [2618368,2618880) | 20260507 21:10:13.986 | INFO | MemInfo: { | 20260507 21:10:13.986 | INFO | "gpu" : " 17 GB", | 20260507 21:10:13.986 | INFO | "huge" : "", | 20260507 21:10:13.986 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:13.986 | INFO | "malloc" : " 14 GB", | 20260507 21:10:13.986 | INFO | "pinned" : " 29 GB" | 20260507 21:10:13.986 | INFO | } | 20260507 21:10:13.986 | INFO | Perf report for the last 5.13868 seconds: | 20260507 21:10:13.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:13.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 21:10:13.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:10:13.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:13.986 | INFO | Uploaded 36GiB to GPU in 2.53206s (14.2177GiB/s) | 20260507 21:10:13.987 | INFO | Downloaded 34GiB to GPU in 2.4761s (13.7313GiB/s) | 20260507 21:10:19.125 | INFO | Analyzing chunk frames = [2618880,2619392) | 20260507 21:10:19.125 | INFO | MemInfo: { | 20260507 21:10:19.125 | INFO | "gpu" : " 17 GB", | 20260507 21:10:19.125 | INFO | "huge" : "", | 20260507 21:10:19.125 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:19.125 | INFO | "malloc" : " 14 GB", | 20260507 21:10:19.125 | INFO | "pinned" : " 29 GB" | 20260507 21:10:19.125 | INFO | } | 20260507 21:10:19.125 | INFO | Perf report for the last 5.13897 seconds: | 20260507 21:10:19.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:19.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:10:19.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:10:19.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:19.125 | INFO | Uploaded 36GiB to GPU in 2.52909s (14.2344GiB/s) | 20260507 21:10:19.125 | INFO | Downloaded 34GiB to GPU in 2.47876s (13.7165GiB/s) | 20260507 21:10:24.265 | INFO | Analyzing chunk frames = [2619392,2619904) | 20260507 21:10:24.265 | INFO | MemInfo: { | 20260507 21:10:24.265 | INFO | "gpu" : " 17 GB", | 20260507 21:10:24.265 | INFO | "huge" : "", | 20260507 21:10:24.265 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:24.265 | INFO | "malloc" : " 14 GB", | 20260507 21:10:24.265 | INFO | "pinned" : " 29 GB" | 20260507 21:10:24.265 | INFO | } | 20260507 21:10:24.265 | INFO | Perf report for the last 5.14032 seconds: | 20260507 21:10:24.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:24.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.265 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 21:10:24.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.266 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:10:24.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:24.266 | INFO | Uploaded 36GiB to GPU in 2.41914s (14.8813GiB/s) | 20260507 21:10:24.266 | INFO | Downloaded 34GiB to GPU in 2.47854s (13.7177GiB/s) | 20260507 21:10:29.405 | INFO | Analyzing chunk frames = [2619904,2620416) | 20260507 21:10:29.405 | INFO | MemInfo: { | 20260507 21:10:29.405 | INFO | "gpu" : " 17 GB", | 20260507 21:10:29.405 | INFO | "huge" : "", | 20260507 21:10:29.405 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:29.405 | INFO | "malloc" : " 14 GB", | 20260507 21:10:29.405 | INFO | "pinned" : " 29 GB" | 20260507 21:10:29.405 | INFO | } | 20260507 21:10:29.405 | INFO | Perf report for the last 5.13997 seconds: | 20260507 21:10:29.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:29.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.405 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:10:29.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.406 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:10:29.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:29.406 | INFO | Uploaded 36GiB to GPU in 2.39582s (15.0262GiB/s) | 20260507 21:10:29.406 | INFO | Downloaded 34GiB to GPU in 2.38672s (14.2455GiB/s) | 20260507 21:10:34.543 | INFO | Analyzing chunk frames = [2620416,2620928) | 20260507 21:10:34.543 | INFO | MemInfo: { | 20260507 21:10:34.543 | INFO | "gpu" : " 17 GB", | 20260507 21:10:34.543 | INFO | "huge" : "", | 20260507 21:10:34.543 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:34.543 | INFO | "malloc" : " 14 GB", | 20260507 21:10:34.543 | INFO | "pinned" : " 29 GB" | 20260507 21:10:34.543 | INFO | } | 20260507 21:10:34.543 | INFO | Perf report for the last 5.13797 seconds: | 20260507 21:10:34.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:34.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.544 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:10:34.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.544 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260507 21:10:34.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:34.544 | INFO | Uploaded 36GiB to GPU in 2.5382s (14.1833GiB/s) | 20260507 21:10:34.544 | INFO | Downloaded 34GiB to GPU in 2.45017s (13.8766GiB/s) | 20260507 21:10:39.682 | INFO | Analyzing chunk frames = [2620928,2621440) | 20260507 21:10:39.682 | INFO | MemInfo: { | 20260507 21:10:39.682 | INFO | "gpu" : " 17 GB", | 20260507 21:10:39.682 | INFO | "huge" : "", | 20260507 21:10:39.682 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:39.682 | INFO | "malloc" : " 14 GB", | 20260507 21:10:39.682 | INFO | "pinned" : " 29 GB" | 20260507 21:10:39.682 | INFO | } | 20260507 21:10:39.683 | INFO | Perf report for the last 5.13934 seconds: | 20260507 21:10:39.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:39.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:10:39.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:10:39.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:39.683 | INFO | Uploaded 36GiB to GPU in 2.48646s (14.4784GiB/s) | 20260507 21:10:39.683 | INFO | Downloaded 34GiB to GPU in 2.43662s (13.9537GiB/s) | 20260507 21:10:39.683 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:10:39.683 | INFO | Total mem usage: 16.7 GiB | 20260507 21:10:39.683 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:10:39.683 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.7%) | 20260507 21:10:39.683 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:10:39.683 | INFO | High Water Breakdown: | 20260507 21:10:39.683 | INFO | 60.1% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:10:39.683 | INFO | 39.7% : 5.99 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:10:39.683 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:10:39.683 | INFO | ------------------------------------------ | 20260507 21:10:39.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:10:39.683 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:10:39.683 | INFO | Total mem usage: 26.5 GiB | 20260507 21:10:39.683 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:10:39.684 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 21:10:39.684 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 21:10:39.684 | INFO | High Water Breakdown: | 20260507 21:10:39.684 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:10:39.684 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:10:39.684 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:10:39.684 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:10:39.684 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:10:39.684 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:10:39.684 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:10:39.684 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:10:39.684 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:10:39.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:10:39.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:10:39.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:10:39.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:10:39.684 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:10:39.684 | INFO | ------------------------------------------ | 20260507 21:10:39.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:10:39.684 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:10:39.684 | INFO | Total mem usage: 27.8 GiB | 20260507 21:10:39.684 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:10:39.684 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:10:39.684 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:10:39.684 | INFO | High Water Breakdown: | 20260507 21:10:39.684 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:10:39.684 | INFO | ------------------------------------------ | 20260507 21:10:39.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:10:39.684 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:10:39.684 | INFO | Total mem usage: 15.7 GiB | 20260507 21:10:39.684 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:10:39.684 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:10:39.684 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:10:39.684 | INFO | High Water Breakdown: | 20260507 21:10:39.684 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:10:39.684 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:10:39.684 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:10:39.684 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:10:39.684 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:10:39.684 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:10:39.684 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:10:39.684 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:10:39.684 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:10:39.684 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:10:39.684 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:10:39.684 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:10:39.684 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:10:39.684 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:10:39.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:10:39.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:10:39.684 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:10:39.684 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:10:39.684 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:10:39.684 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:10:39.684 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:10:39.684 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:10:39.684 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:10:39.684 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:10:39.684 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:10:39.684 | INFO | ------------------------------------------ | 20260507 21:10:39.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:10:39.684 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:10:39.684 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:10:39.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:10:39.684 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.63e+01, 1.43e+01 / 2.18e+00 | 20260507 21:10:39.684 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.01e-01, 1.16e-02 / 5.27e-03 | 20260507 21:10:39.684 | INFO | PerfReport: 12.12%, Baseline -- 4.00e+00 / 4.48e+00, 5.93e-01 / 2.51e-01 | 20260507 21:10:39.684 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.51e+00, 2.10e-01 / 9.67e-02 | 20260507 21:10:39.684 | INFO | PerfReport: 16.12%, DME -- 4.26e+01 / 4.62e+01, 4.24e+00 / 2.13e+00 | 20260507 21:10:39.684 | INFO | PerfReport: 55.82%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.61e+00 / 1.38e+00 | 20260507 21:10:39.684 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.41e-01 / 1.95e-01 | 20260507 21:10:39.684 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.39e+00, 2.95e-01 / 1.53e-01 | 20260507 21:10:39.684 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.65e-01 / 4.05e-01, 5.95e-02 / 3.39e-02 | 20260507 21:10:39.684 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:10:39.684 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:10:39.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:10:39.684 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 21:10:39.684 | INFO | PerfReport: 24.38%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 21:10:39.684 | INFO | PerfReport: 24.30%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 21:10:39.684 | INFO | PerfReport: 30.89%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:10:39.685 | INFO | PerfReport: 2.18%, LockUpload -- 2.33e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260507 21:10:39.685 | INFO | PerfReport: 2.54%, LockDownload -- 2.72e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 21:10:39.685 | INFO | PerfReport: 15.71%, LockCompute -- 1.68e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 21:10:39.685 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260507 21:10:39.780 | INFO | Starting a round of Baz IO | 20260507 21:10:43.259 | INFO | Populating RTMetrics report | 20260507 21:10:43.259 | INFO | RTMetrics report populated | 20260507 21:10:43.259 | INFO | Opening tmp RTMetrics json file | 20260507 21:10:43.283 | INFO | Serializing RTMetrics to json | 20260507 21:10:43.312 | INFO | Serialization done, swapping tmp file now... | 20260507 21:10:43.338 | INFO | RTMetrics done | 20260507 21:10:43.363 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.583 seconds | 20260507 21:10:43.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.91% / 0.01% / 56.08% | 20260507 21:10:43.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77668% / 7.94729e-06% / 1.98682e-05% | 20260507 21:10:43.363 | INFO | Baz IO overall: | 20260507 21:10:43.364 | INFO | BytesWritten : 1192798777192 | 20260507 21:10:43.364 | INFO | NumEvents : 578161215963 | 20260507 21:10:44.823 | INFO | Analyzing chunk frames = [2621440,2621952) | 20260507 21:10:44.823 | INFO | MemInfo: { | 20260507 21:10:44.823 | INFO | "gpu" : " 17 GB", | 20260507 21:10:44.823 | INFO | "huge" : "", | 20260507 21:10:44.823 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:44.823 | INFO | "malloc" : " 14 GB", | 20260507 21:10:44.823 | INFO | "pinned" : " 29 GB" | 20260507 21:10:44.823 | INFO | } | 20260507 21:10:44.823 | INFO | Perf report for the last 5.13997 seconds: | 20260507 21:10:44.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:44.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:44.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:44.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:44.823 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 21:10:44.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:44.823 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 21:10:44.823 | INFO | PRE_HQ : 66.0%, 1.7, 58.1ms, 0 | 20260507 21:10:44.823 | INFO | BAZWRITER : 69.3%, 4.3, 152.8ms, 0 | 20260507 21:10:44.823 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 21:10:44.823 | INFO | Uploaded 36GiB to GPU in 2.53031s (14.2275GiB/s) | 20260507 21:10:44.823 | INFO | Downloaded 38GiB to GPU in 2.69671s (14.0912GiB/s) | 20260507 21:10:49.961 | INFO | Analyzing chunk frames = [2621952,2622464) | 20260507 21:10:49.961 | INFO | MemInfo: { | 20260507 21:10:49.961 | INFO | "gpu" : " 17 GB", | 20260507 21:10:49.961 | INFO | "huge" : "", | 20260507 21:10:49.961 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:49.961 | INFO | "malloc" : " 14 GB", | 20260507 21:10:49.961 | INFO | "pinned" : " 29 GB" | 20260507 21:10:49.961 | INFO | } | 20260507 21:10:49.961 | INFO | Perf report for the last 5.13866 seconds: | 20260507 21:10:49.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:49.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.961 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:10:49.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.961 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:10:49.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:49.962 | INFO | Uploaded 36GiB to GPU in 2.56465s (14.037GiB/s) | 20260507 21:10:49.962 | INFO | Downloaded 34GiB to GPU in 2.50286s (13.5845GiB/s) | 20260507 21:10:55.101 | INFO | Analyzing chunk frames = [2622464,2622976) | 20260507 21:10:55.101 | INFO | MemInfo: { | 20260507 21:10:55.101 | INFO | "gpu" : " 17 GB", | 20260507 21:10:55.101 | INFO | "huge" : "", | 20260507 21:10:55.101 | INFO | "huge_shared" : " 29 GB", | 20260507 21:10:55.101 | INFO | "malloc" : " 14 GB", | 20260507 21:10:55.101 | INFO | "pinned" : " 29 GB" | 20260507 21:10:55.101 | INFO | } | 20260507 21:10:55.101 | INFO | Perf report for the last 5.1398 seconds: | 20260507 21:10:55.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:10:55.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:10:55.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:10:55.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:10:55.101 | INFO | Uploaded 36GiB to GPU in 2.50241s (14.3862GiB/s) | 20260507 21:10:55.102 | INFO | Downloaded 34GiB to GPU in 2.47802s (13.7206GiB/s) | 20260507 21:11:00.240 | INFO | Analyzing chunk frames = [2622976,2623488) | 20260507 21:11:00.240 | INFO | MemInfo: { | 20260507 21:11:00.240 | INFO | "gpu" : " 17 GB", | 20260507 21:11:00.240 | INFO | "huge" : "", | 20260507 21:11:00.240 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:00.240 | INFO | "malloc" : " 14 GB", | 20260507 21:11:00.240 | INFO | "pinned" : " 29 GB" | 20260507 21:11:00.240 | INFO | } | 20260507 21:11:00.240 | INFO | Perf report for the last 5.13901 seconds: | 20260507 21:11:00.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:00.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.240 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:11:00.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.241 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:11:00.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:00.241 | INFO | Uploaded 36GiB to GPU in 2.48737s (14.4731GiB/s) | 20260507 21:11:00.241 | INFO | Downloaded 34GiB to GPU in 2.48285s (13.6939GiB/s) | 20260507 21:11:05.380 | INFO | Analyzing chunk frames = [2623488,2624000) | 20260507 21:11:05.380 | INFO | MemInfo: { | 20260507 21:11:05.380 | INFO | "gpu" : " 17 GB", | 20260507 21:11:05.380 | INFO | "huge" : "", | 20260507 21:11:05.380 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:05.380 | INFO | "malloc" : " 14 GB", | 20260507 21:11:05.380 | INFO | "pinned" : " 29 GB" | 20260507 21:11:05.380 | INFO | } | 20260507 21:11:05.380 | INFO | Perf report for the last 5.13988 seconds: | 20260507 21:11:05.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:05.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.380 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 21:11:05.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.380 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:11:05.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:05.381 | INFO | Uploaded 36GiB to GPU in 2.46959s (14.5773GiB/s) | 20260507 21:11:05.381 | INFO | Downloaded 34GiB to GPU in 2.4944s (13.6305GiB/s) | 20260507 21:11:10.520 | INFO | Analyzing chunk frames = [2624000,2624512) | 20260507 21:11:10.520 | INFO | MemInfo: { | 20260507 21:11:10.520 | INFO | "gpu" : " 17 GB", | 20260507 21:11:10.520 | INFO | "huge" : "", | 20260507 21:11:10.520 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:10.520 | INFO | "malloc" : " 14 GB", | 20260507 21:11:10.520 | INFO | "pinned" : " 29 GB" | 20260507 21:11:10.520 | INFO | } | 20260507 21:11:10.520 | INFO | Perf report for the last 5.13987 seconds: | 20260507 21:11:10.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:10.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.520 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 21:11:10.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.520 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260507 21:11:10.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:10.521 | INFO | Uploaded 36GiB to GPU in 2.42446s (14.8487GiB/s) | 20260507 21:11:10.521 | INFO | Downloaded 34GiB to GPU in 2.3806s (14.2821GiB/s) | 20260507 21:11:15.659 | INFO | Analyzing chunk frames = [2624512,2625024) | 20260507 21:11:15.659 | INFO | MemInfo: { | 20260507 21:11:15.659 | INFO | "gpu" : " 17 GB", | 20260507 21:11:15.659 | INFO | "huge" : "", | 20260507 21:11:15.660 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:15.660 | INFO | "malloc" : " 14 GB", | 20260507 21:11:15.660 | INFO | "pinned" : " 29 GB" | 20260507 21:11:15.660 | INFO | } | 20260507 21:11:15.660 | INFO | Perf report for the last 5.13964 seconds: | 20260507 21:11:15.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:15.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260507 21:11:15.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:11:15.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:15.660 | INFO | Uploaded 36GiB to GPU in 2.54394s (14.1513GiB/s) | 20260507 21:11:15.660 | INFO | Downloaded 34GiB to GPU in 2.44291s (13.9178GiB/s) | 20260507 21:11:20.798 | INFO | Analyzing chunk frames = [2625024,2625536) | 20260507 21:11:20.798 | INFO | MemInfo: { | 20260507 21:11:20.798 | INFO | "gpu" : " 17 GB", | 20260507 21:11:20.798 | INFO | "huge" : "", | 20260507 21:11:20.798 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:20.798 | INFO | "malloc" : " 14 GB", | 20260507 21:11:20.798 | INFO | "pinned" : " 29 GB" | 20260507 21:11:20.798 | INFO | } | 20260507 21:11:20.798 | INFO | Perf report for the last 5.13875 seconds: | 20260507 21:11:20.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:20.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.798 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:11:20.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.799 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:11:20.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:20.799 | INFO | Uploaded 36GiB to GPU in 2.5477s (14.1304GiB/s) | 20260507 21:11:20.799 | INFO | Downloaded 34GiB to GPU in 2.43763s (13.9479GiB/s) | 20260507 21:11:20.896 | INFO | Starting a round of Baz IO | 20260507 21:11:24.362 | INFO | Populating RTMetrics report | 20260507 21:11:24.362 | INFO | RTMetrics report populated | 20260507 21:11:24.362 | INFO | Opening tmp RTMetrics json file | 20260507 21:11:24.368 | INFO | Serializing RTMetrics to json | 20260507 21:11:24.390 | INFO | Serialization done, swapping tmp file now... | 20260507 21:11:24.390 | INFO | RTMetrics done | 20260507 21:11:24.527 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.631 seconds | 20260507 21:11:24.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.91% / 0.01% / 56.09% | 20260507 21:11:24.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77422% / 1.98682e-05% / 4.37101e-05% | 20260507 21:11:24.527 | INFO | Baz IO overall: | 20260507 21:11:24.527 | INFO | BytesWritten : 1194696690505 | 20260507 21:11:24.527 | INFO | NumEvents : 579080013629 | 20260507 21:11:25.936 | INFO | Analyzing chunk frames = [2625536,2626048) | 20260507 21:11:25.936 | INFO | MemInfo: { | 20260507 21:11:25.936 | INFO | "gpu" : " 17 GB", | 20260507 21:11:25.936 | INFO | "huge" : "", | 20260507 21:11:25.936 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:25.936 | INFO | "malloc" : " 14 GB", | 20260507 21:11:25.936 | INFO | "pinned" : " 29 GB" | 20260507 21:11:25.936 | INFO | } | 20260507 21:11:25.936 | INFO | Perf report for the last 5.13786 seconds: | 20260507 21:11:25.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:25.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:25.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:25.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:25.937 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 21:11:25.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:25.937 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 21:11:25.937 | INFO | PRE_HQ : 65.8%, 1.7, 57.7ms, 0 | 20260507 21:11:25.937 | INFO | BAZWRITER : 70.3%, 4.1, 146.9ms, 0 | 20260507 21:11:25.937 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 21:11:25.937 | INFO | Uploaded 36GiB to GPU in 2.53456s (14.2037GiB/s) | 20260507 21:11:25.937 | INFO | Downloaded 38GiB to GPU in 2.68552s (14.15GiB/s) | 20260507 21:11:31.077 | INFO | Analyzing chunk frames = [2626048,2626560) | 20260507 21:11:31.077 | INFO | MemInfo: { | 20260507 21:11:31.077 | INFO | "gpu" : " 17 GB", | 20260507 21:11:31.077 | INFO | "huge" : "", | 20260507 21:11:31.077 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:31.077 | INFO | "malloc" : " 14 GB", | 20260507 21:11:31.077 | INFO | "pinned" : " 29 GB" | 20260507 21:11:31.077 | INFO | } | 20260507 21:11:31.077 | INFO | Perf report for the last 5.14053 seconds: | 20260507 21:11:31.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:31.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 21:11:31.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | PRE_HQ : 25.8%, 1.0, 13.6ms, 0 | 20260507 21:11:31.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:31.077 | INFO | Uploaded 36GiB to GPU in 2.59242s (13.8867GiB/s) | 20260507 21:11:31.077 | INFO | Downloaded 34GiB to GPU in 2.49539s (13.6251GiB/s) | 20260507 21:11:36.216 | INFO | Analyzing chunk frames = [2626560,2627072) | 20260507 21:11:36.216 | INFO | MemInfo: { | 20260507 21:11:36.216 | INFO | "gpu" : " 17 GB", | 20260507 21:11:36.216 | INFO | "huge" : "", | 20260507 21:11:36.216 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:36.216 | INFO | "malloc" : " 14 GB", | 20260507 21:11:36.216 | INFO | "pinned" : " 29 GB" | 20260507 21:11:36.216 | INFO | } | 20260507 21:11:36.216 | INFO | Perf report for the last 5.13924 seconds: | 20260507 21:11:36.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:36.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:11:36.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:11:36.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:36.216 | INFO | Uploaded 36GiB to GPU in 2.51906s (14.291GiB/s) | 20260507 21:11:36.216 | INFO | Downloaded 34GiB to GPU in 2.48042s (13.7073GiB/s) | 20260507 21:11:41.355 | INFO | Analyzing chunk frames = [2627072,2627584) | 20260507 21:11:41.355 | INFO | MemInfo: { | 20260507 21:11:41.355 | INFO | "gpu" : " 17 GB", | 20260507 21:11:41.355 | INFO | "huge" : "", | 20260507 21:11:41.355 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:41.355 | INFO | "malloc" : " 14 GB", | 20260507 21:11:41.355 | INFO | "pinned" : " 29 GB" | 20260507 21:11:41.355 | INFO | } | 20260507 21:11:41.355 | INFO | Perf report for the last 5.13903 seconds: | 20260507 21:11:41.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:41.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:11:41.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:11:41.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:41.355 | INFO | Uploaded 36GiB to GPU in 2.53817s (14.1835GiB/s) | 20260507 21:11:41.355 | INFO | Downloaded 34GiB to GPU in 2.4878s (13.6667GiB/s) | 20260507 21:11:46.495 | INFO | Analyzing chunk frames = [2627584,2628096) | 20260507 21:11:46.495 | INFO | MemInfo: { | 20260507 21:11:46.495 | INFO | "gpu" : " 17 GB", | 20260507 21:11:46.495 | INFO | "huge" : "", | 20260507 21:11:46.495 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:46.495 | INFO | "malloc" : " 14 GB", | 20260507 21:11:46.495 | INFO | "pinned" : " 29 GB" | 20260507 21:11:46.495 | INFO | } | 20260507 21:11:46.495 | INFO | Perf report for the last 5.14021 seconds: | 20260507 21:11:46.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:46.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.495 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 21:11:46.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.495 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 21:11:46.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:46.496 | INFO | Uploaded 36GiB to GPU in 2.41189s (14.9261GiB/s) | 20260507 21:11:46.496 | INFO | Downloaded 34GiB to GPU in 2.49133s (13.6473GiB/s) | 20260507 21:11:51.633 | INFO | Analyzing chunk frames = [2628096,2628608) | 20260507 21:11:51.633 | INFO | MemInfo: { | 20260507 21:11:51.633 | INFO | "gpu" : " 17 GB", | 20260507 21:11:51.633 | INFO | "huge" : "", | 20260507 21:11:51.633 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:51.633 | INFO | "malloc" : " 14 GB", | 20260507 21:11:51.633 | INFO | "pinned" : " 29 GB" | 20260507 21:11:51.633 | INFO | } | 20260507 21:11:51.633 | INFO | Perf report for the last 5.13807 seconds: | 20260507 21:11:51.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:51.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.633 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:11:51.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.634 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:11:51.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:51.634 | INFO | Uploaded 36GiB to GPU in 2.40105s (14.9934GiB/s) | 20260507 21:11:51.634 | INFO | Downloaded 34GiB to GPU in 2.38411s (14.2611GiB/s) | 20260507 21:11:56.774 | INFO | Analyzing chunk frames = [2628608,2629120) | 20260507 21:11:56.774 | INFO | MemInfo: { | 20260507 21:11:56.774 | INFO | "gpu" : " 17 GB", | 20260507 21:11:56.774 | INFO | "huge" : "", | 20260507 21:11:56.774 | INFO | "huge_shared" : " 29 GB", | 20260507 21:11:56.774 | INFO | "malloc" : " 14 GB", | 20260507 21:11:56.774 | INFO | "pinned" : " 29 GB" | 20260507 21:11:56.774 | INFO | } | 20260507 21:11:56.774 | INFO | Perf report for the last 5.14078 seconds: | 20260507 21:11:56.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:11:56.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.774 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 21:11:56.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.774 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:11:56.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:11:56.775 | INFO | Uploaded 36GiB to GPU in 2.55114s (14.1113GiB/s) | 20260507 21:11:56.775 | INFO | Downloaded 34GiB to GPU in 2.45139s (13.8697GiB/s) | 20260507 21:12:01.912 | INFO | Analyzing chunk frames = [2629120,2629632) | 20260507 21:12:01.912 | INFO | MemInfo: { | 20260507 21:12:01.912 | INFO | "gpu" : " 17 GB", | 20260507 21:12:01.912 | INFO | "huge" : "", | 20260507 21:12:01.912 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:01.912 | INFO | "malloc" : " 14 GB", | 20260507 21:12:01.912 | INFO | "pinned" : " 29 GB" | 20260507 21:12:01.912 | INFO | } | 20260507 21:12:01.913 | INFO | Perf report for the last 5.13832 seconds: | 20260507 21:12:01.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:01.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:12:01.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 21:12:01.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:01.913 | INFO | Uploaded 36GiB to GPU in 2.52875s (14.2363GiB/s) | 20260507 21:12:01.913 | INFO | Downloaded 34GiB to GPU in 2.43146s (13.9833GiB/s) | 20260507 21:12:01.913 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:12:01.913 | INFO | Total mem usage: 16.7 GiB | 20260507 21:12:01.913 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:12:01.913 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.1%) | 20260507 21:12:01.913 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:12:01.913 | INFO | High Water Breakdown: | 20260507 21:12:01.913 | INFO | 59.7% : 8.81 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:12:01.913 | INFO | 40.1% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:12:01.913 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:12:01.913 | INFO | ------------------------------------------ | 20260507 21:12:01.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:12:01.913 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:12:01.913 | INFO | Total mem usage: 26.5 GiB | 20260507 21:12:01.913 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:12:01.913 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:12:01.913 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:12:01.913 | INFO | High Water Breakdown: | 20260507 21:12:01.913 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:12:01.913 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:12:01.913 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:12:01.913 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:12:01.913 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:12:01.913 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:12:01.913 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:12:01.913 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:12:01.913 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:12:01.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:12:01.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:12:01.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:12:01.914 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:12:01.914 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:12:01.914 | INFO | ------------------------------------------ | 20260507 21:12:01.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:12:01.914 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:12:01.914 | INFO | Total mem usage: 27.8 GiB | 20260507 21:12:01.914 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:12:01.914 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:12:01.914 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:12:01.914 | INFO | High Water Breakdown: | 20260507 21:12:01.914 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:12:01.914 | INFO | ------------------------------------------ | 20260507 21:12:01.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:12:01.914 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:12:01.914 | INFO | Total mem usage: 15.7 GiB | 20260507 21:12:01.914 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:12:01.914 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:12:01.914 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:12:01.914 | INFO | High Water Breakdown: | 20260507 21:12:01.914 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:12:01.914 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:12:01.914 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:12:01.914 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:12:01.914 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:12:01.914 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:12:01.914 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:12:01.914 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:12:01.914 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:12:01.914 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:12:01.914 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:12:01.914 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:12:01.914 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:12:01.914 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:12:01.914 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:12:01.914 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:12:01.914 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:12:01.914 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:12:01.914 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:12:01.914 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:12:01.914 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:12:01.914 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:12:01.914 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:12:01.914 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:12:01.914 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:12:01.914 | INFO | ------------------------------------------ | 20260507 21:12:01.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:12:01.914 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:12:01.914 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:12:01.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:12:01.914 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.42e+01 / 2.08e+00 | 20260507 21:12:01.914 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.29e-02 / 5.87e-03 | 20260507 21:12:01.914 | INFO | PerfReport: 12.12%, Baseline -- 4.00e+00 / 4.50e+00, 5.90e-01 / 2.09e-01 | 20260507 21:12:01.914 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.50e+00, 2.09e-01 / 1.24e-01 | 20260507 21:12:01.914 | INFO | PerfReport: 16.10%, DME -- 4.25e+01 / 4.60e+01, 4.27e+00 / 2.19e+00 | 20260507 21:12:01.914 | INFO | PerfReport: 55.84%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.61e+00 / 1.51e+00 | 20260507 21:12:01.914 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.51e+00 / 2.82e+00, 4.39e-01 / 2.41e-01 | 20260507 21:12:01.914 | INFO | PerfReport: 3.56%, Metrics -- 1.17e+00 / 1.40e+00, 2.95e-01 / 1.57e-01 | 20260507 21:12:01.914 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.63e-01 / 4.02e-01, 5.90e-02 / 2.97e-02 | 20260507 21:12:01.914 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:12:01.914 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:12:01.914 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:12:01.914 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 21:12:01.914 | INFO | PerfReport: 24.39%, Upload -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 21:12:01.914 | INFO | PerfReport: 24.19%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 21:12:01.915 | INFO | PerfReport: 30.79%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:12:01.915 | INFO | PerfReport: 2.28%, LockUpload -- 2.44e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260507 21:12:01.915 | INFO | PerfReport: 2.52%, LockDownload -- 2.70e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260507 21:12:01.915 | INFO | PerfReport: 15.82%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:12:01.915 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 21:12:02.012 | INFO | Starting a round of Baz IO | 20260507 21:12:05.484 | INFO | Populating RTMetrics report | 20260507 21:12:05.484 | INFO | RTMetrics report populated | 20260507 21:12:05.484 | INFO | Opening tmp RTMetrics json file | 20260507 21:12:05.529 | INFO | Serializing RTMetrics to json | 20260507 21:12:05.606 | INFO | Serialization done, swapping tmp file now... | 20260507 21:12:05.611 | INFO | RTMetrics done | 20260507 21:12:05.644 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.632 seconds | 20260507 21:12:05.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.90% / 0.01% / 56.09% | 20260507 21:12:05.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76869% / 1.19209e-05% / 5.16574e-05% | 20260507 21:12:05.644 | INFO | Baz IO overall: | 20260507 21:12:05.644 | INFO | BytesWritten : 1196592894849 | 20260507 21:12:05.644 | INFO | NumEvents : 579997503333 | 20260507 21:12:07.052 | INFO | Analyzing chunk frames = [2629632,2630144) | 20260507 21:12:07.052 | INFO | MemInfo: { | 20260507 21:12:07.052 | INFO | "gpu" : " 17 GB", | 20260507 21:12:07.052 | INFO | "huge" : "", | 20260507 21:12:07.052 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:07.052 | INFO | "malloc" : " 14 GB", | 20260507 21:12:07.052 | INFO | "pinned" : " 29 GB" | 20260507 21:12:07.052 | INFO | } | 20260507 21:12:07.052 | INFO | Perf report for the last 5.13945 seconds: | 20260507 21:12:07.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:07.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:07.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:07.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:07.052 | INFO | ANALYSIS : 68.7%, 3.0, 107.1ms, 0 | 20260507 21:12:07.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:07.052 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 21:12:07.052 | INFO | PRE_HQ : 65.9%, 1.7, 58.2ms, 0 | 20260507 21:12:07.052 | INFO | BAZWRITER : 70.2%, 3.7, 131.9ms, 0 | 20260507 21:12:07.052 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 21:12:07.052 | INFO | Uploaded 36GiB to GPU in 2.54525s (14.144GiB/s) | 20260507 21:12:07.052 | INFO | Downloaded 38GiB to GPU in 2.6937s (14.107GiB/s) | 20260507 21:12:12.191 | INFO | Analyzing chunk frames = [2630144,2630656) | 20260507 21:12:12.191 | INFO | MemInfo: { | 20260507 21:12:12.191 | INFO | "gpu" : " 17 GB", | 20260507 21:12:12.191 | INFO | "huge" : "", | 20260507 21:12:12.191 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:12.191 | INFO | "malloc" : " 14 GB", | 20260507 21:12:12.191 | INFO | "pinned" : " 29 GB" | 20260507 21:12:12.191 | INFO | } | 20260507 21:12:12.191 | INFO | Perf report for the last 5.13892 seconds: | 20260507 21:12:12.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:12.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:12:12.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:12:12.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:12.191 | INFO | Uploaded 36GiB to GPU in 2.56962s (14.0098GiB/s) | 20260507 21:12:12.191 | INFO | Downloaded 34GiB to GPU in 2.49032s (13.6529GiB/s) | 20260507 21:12:17.330 | INFO | Analyzing chunk frames = [2630656,2631168) | 20260507 21:12:17.330 | INFO | MemInfo: { | 20260507 21:12:17.330 | INFO | "gpu" : " 17 GB", | 20260507 21:12:17.330 | INFO | "huge" : "", | 20260507 21:12:17.330 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:17.330 | INFO | "malloc" : " 14 GB", | 20260507 21:12:17.330 | INFO | "pinned" : " 29 GB" | 20260507 21:12:17.330 | INFO | } | 20260507 21:12:17.330 | INFO | Perf report for the last 5.13901 seconds: | 20260507 21:12:17.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:17.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:12:17.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:12:17.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:17.330 | INFO | Uploaded 36GiB to GPU in 2.50714s (14.359GiB/s) | 20260507 21:12:17.330 | INFO | Downloaded 34GiB to GPU in 2.47199s (13.7541GiB/s) | 20260507 21:12:22.469 | INFO | Analyzing chunk frames = [2631168,2631680) | 20260507 21:12:22.469 | INFO | MemInfo: { | 20260507 21:12:22.469 | INFO | "gpu" : " 17 GB", | 20260507 21:12:22.469 | INFO | "huge" : "", | 20260507 21:12:22.469 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:22.469 | INFO | "malloc" : " 14 GB", | 20260507 21:12:22.469 | INFO | "pinned" : " 29 GB" | 20260507 21:12:22.470 | INFO | } | 20260507 21:12:22.470 | INFO | Perf report for the last 5.13967 seconds: | 20260507 21:12:22.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:22.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:12:22.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:12:22.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:22.470 | INFO | Uploaded 36GiB to GPU in 2.52003s (14.2855GiB/s) | 20260507 21:12:22.470 | INFO | Downloaded 34GiB to GPU in 2.48583s (13.6775GiB/s) | 20260507 21:12:27.609 | INFO | Analyzing chunk frames = [2631680,2632192) | 20260507 21:12:27.609 | INFO | MemInfo: { | 20260507 21:12:27.609 | INFO | "gpu" : " 17 GB", | 20260507 21:12:27.609 | INFO | "huge" : "", | 20260507 21:12:27.609 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:27.609 | INFO | "malloc" : " 14 GB", | 20260507 21:12:27.609 | INFO | "pinned" : " 29 GB" | 20260507 21:12:27.609 | INFO | } | 20260507 21:12:27.609 | INFO | Perf report for the last 5.13971 seconds: | 20260507 21:12:27.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:27.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.609 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:12:27.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.610 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:12:27.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:27.610 | INFO | Uploaded 36GiB to GPU in 2.46543s (14.6019GiB/s) | 20260507 21:12:27.610 | INFO | Downloaded 34GiB to GPU in 2.48043s (13.7073GiB/s) | 20260507 21:12:32.749 | INFO | Analyzing chunk frames = [2632192,2632704) | 20260507 21:12:32.749 | INFO | MemInfo: { | 20260507 21:12:32.749 | INFO | "gpu" : " 17 GB", | 20260507 21:12:32.749 | INFO | "huge" : "", | 20260507 21:12:32.749 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:32.749 | INFO | "malloc" : " 14 GB", | 20260507 21:12:32.749 | INFO | "pinned" : " 29 GB" | 20260507 21:12:32.749 | INFO | } | 20260507 21:12:32.749 | INFO | Perf report for the last 5.13979 seconds: | 20260507 21:12:32.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:32.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:12:32.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:12:32.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:32.749 | INFO | Uploaded 36GiB to GPU in 2.39945s (15.0035GiB/s) | 20260507 21:12:32.749 | INFO | Downloaded 34GiB to GPU in 2.39171s (14.2158GiB/s) | 20260507 21:12:37.889 | INFO | Analyzing chunk frames = [2632704,2633216) | 20260507 21:12:37.889 | INFO | MemInfo: { | 20260507 21:12:37.889 | INFO | "gpu" : " 17 GB", | 20260507 21:12:37.889 | INFO | "huge" : "", | 20260507 21:12:37.889 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:37.889 | INFO | "malloc" : " 14 GB", | 20260507 21:12:37.889 | INFO | "pinned" : " 29 GB" | 20260507 21:12:37.889 | INFO | } | 20260507 21:12:37.889 | INFO | Perf report for the last 5.13957 seconds: | 20260507 21:12:37.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:37.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:12:37.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:12:37.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:37.889 | INFO | Uploaded 36GiB to GPU in 2.54599s (14.1399GiB/s) | 20260507 21:12:37.889 | INFO | Downloaded 34GiB to GPU in 2.45837s (13.8303GiB/s) | 20260507 21:12:43.028 | INFO | Analyzing chunk frames = [2633216,2633728) | 20260507 21:12:43.028 | INFO | MemInfo: { | 20260507 21:12:43.028 | INFO | "gpu" : " 17 GB", | 20260507 21:12:43.028 | INFO | "huge" : "", | 20260507 21:12:43.028 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:43.028 | INFO | "malloc" : " 14 GB", | 20260507 21:12:43.028 | INFO | "pinned" : " 29 GB" | 20260507 21:12:43.028 | INFO | } | 20260507 21:12:43.028 | INFO | Perf report for the last 5.13961 seconds: | 20260507 21:12:43.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:43.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.028 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:12:43.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.029 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:12:43.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:43.029 | INFO | Uploaded 36GiB to GPU in 2.4894s (14.4613GiB/s) | 20260507 21:12:43.029 | INFO | Downloaded 34GiB to GPU in 2.42555s (14.0175GiB/s) | 20260507 21:12:43.125 | INFO | Starting a round of Baz IO | 20260507 21:12:46.598 | INFO | Populating RTMetrics report | 20260507 21:12:46.598 | INFO | RTMetrics report populated | 20260507 21:12:46.598 | INFO | Opening tmp RTMetrics json file | 20260507 21:12:46.660 | INFO | Serializing RTMetrics to json | 20260507 21:12:46.720 | INFO | Serialization done, swapping tmp file now... | 20260507 21:12:46.721 | INFO | RTMetrics done | 20260507 21:12:46.735 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.609 seconds | 20260507 21:12:46.735 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.89% / 0.01% / 56.10% | 20260507 21:12:46.735 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76361% / 3.97364e-06% / 4.76837e-05% | 20260507 21:12:46.735 | INFO | Baz IO overall: | 20260507 21:12:46.736 | INFO | BytesWritten : 1198488205563 | 20260507 21:12:46.736 | INFO | NumEvents : 580914735452 | 20260507 21:12:48.166 | INFO | Analyzing chunk frames = [2633728,2634240) | 20260507 21:12:48.166 | INFO | MemInfo: { | 20260507 21:12:48.166 | INFO | "gpu" : " 17 GB", | 20260507 21:12:48.166 | INFO | "huge" : "", | 20260507 21:12:48.166 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:48.166 | INFO | "malloc" : " 14 GB", | 20260507 21:12:48.166 | INFO | "pinned" : " 29 GB" | 20260507 21:12:48.167 | INFO | } | 20260507 21:12:48.167 | INFO | Perf report for the last 5.13819 seconds: | 20260507 21:12:48.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:48.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:48.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:48.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:48.167 | INFO | ANALYSIS : 68.7%, 3.0, 106.9ms, 0 | 20260507 21:12:48.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:48.167 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 21:12:48.167 | INFO | PRE_HQ : 66.0%, 1.7, 58.1ms, 0 | 20260507 21:12:48.167 | INFO | BAZWRITER : 69.7%, 3.9, 138.0ms, 0 | 20260507 21:12:48.167 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260507 21:12:48.167 | INFO | Uploaded 36GiB to GPU in 2.54069s (14.1694GiB/s) | 20260507 21:12:48.167 | INFO | Downloaded 38GiB to GPU in 2.69467s (14.1019GiB/s) | 20260507 21:12:53.305 | INFO | Analyzing chunk frames = [2634240,2634752) | 20260507 21:12:53.305 | INFO | MemInfo: { | 20260507 21:12:53.305 | INFO | "gpu" : " 17 GB", | 20260507 21:12:53.305 | INFO | "huge" : "", | 20260507 21:12:53.305 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:53.305 | INFO | "malloc" : " 14 GB", | 20260507 21:12:53.305 | INFO | "pinned" : " 29 GB" | 20260507 21:12:53.305 | INFO | } | 20260507 21:12:53.306 | INFO | Perf report for the last 5.13899 seconds: | 20260507 21:12:53.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:53.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:12:53.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 21:12:53.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:53.306 | INFO | Uploaded 36GiB to GPU in 2.56253s (14.0486GiB/s) | 20260507 21:12:53.306 | INFO | Downloaded 34GiB to GPU in 2.50331s (13.582GiB/s) | 20260507 21:12:58.445 | INFO | Analyzing chunk frames = [2634752,2635264) | 20260507 21:12:58.445 | INFO | MemInfo: { | 20260507 21:12:58.445 | INFO | "gpu" : " 17 GB", | 20260507 21:12:58.445 | INFO | "huge" : "", | 20260507 21:12:58.445 | INFO | "huge_shared" : " 29 GB", | 20260507 21:12:58.445 | INFO | "malloc" : " 14 GB", | 20260507 21:12:58.445 | INFO | "pinned" : " 29 GB" | 20260507 21:12:58.445 | INFO | } | 20260507 21:12:58.445 | INFO | Perf report for the last 5.1394 seconds: | 20260507 21:12:58.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:12:58.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:12:58.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:12:58.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:12:58.445 | INFO | Uploaded 36GiB to GPU in 2.47463s (14.5476GiB/s) | 20260507 21:12:58.445 | INFO | Downloaded 34GiB to GPU in 2.48927s (13.6586GiB/s) | 20260507 21:13:03.584 | INFO | Analyzing chunk frames = [2635264,2635776) | 20260507 21:13:03.584 | INFO | MemInfo: { | 20260507 21:13:03.584 | INFO | "gpu" : " 17 GB", | 20260507 21:13:03.584 | INFO | "huge" : "", | 20260507 21:13:03.584 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:03.584 | INFO | "malloc" : " 14 GB", | 20260507 21:13:03.584 | INFO | "pinned" : " 29 GB" | 20260507 21:13:03.584 | INFO | } | 20260507 21:13:03.584 | INFO | Perf report for the last 5.13943 seconds: | 20260507 21:13:03.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:03.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.585 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 21:13:03.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.585 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:13:03.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:03.585 | INFO | Uploaded 36GiB to GPU in 2.46048s (14.6313GiB/s) | 20260507 21:13:03.585 | INFO | Downloaded 34GiB to GPU in 2.49124s (13.6478GiB/s) | 20260507 21:13:08.724 | INFO | Analyzing chunk frames = [2635776,2636288) | 20260507 21:13:08.725 | INFO | MemInfo: { | 20260507 21:13:08.725 | INFO | "gpu" : " 17 GB", | 20260507 21:13:08.725 | INFO | "huge" : "", | 20260507 21:13:08.725 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:08.725 | INFO | "malloc" : " 14 GB", | 20260507 21:13:08.725 | INFO | "pinned" : " 29 GB" | 20260507 21:13:08.725 | INFO | } | 20260507 21:13:08.725 | INFO | Perf report for the last 5.14036 seconds: | 20260507 21:13:08.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:08.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.725 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:13:08.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.725 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 21:13:08.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:08.726 | INFO | Uploaded 36GiB to GPU in 2.42868s (14.8229GiB/s) | 20260507 21:13:08.726 | INFO | Downloaded 34GiB to GPU in 2.5035s (13.581GiB/s) | 20260507 21:13:13.864 | INFO | Analyzing chunk frames = [2636288,2636800) | 20260507 21:13:13.864 | INFO | MemInfo: { | 20260507 21:13:13.864 | INFO | "gpu" : " 17 GB", | 20260507 21:13:13.864 | INFO | "huge" : "", | 20260507 21:13:13.864 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:13.865 | INFO | "malloc" : " 14 GB", | 20260507 21:13:13.865 | INFO | "pinned" : " 29 GB" | 20260507 21:13:13.865 | INFO | } | 20260507 21:13:13.865 | INFO | Perf report for the last 5.13984 seconds: | 20260507 21:13:13.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:13.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260507 21:13:13.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:13:13.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:13.865 | INFO | Uploaded 36GiB to GPU in 2.43061s (14.8111GiB/s) | 20260507 21:13:13.865 | INFO | Downloaded 34GiB to GPU in 2.38213s (14.2729GiB/s) | 20260507 21:13:19.003 | INFO | Analyzing chunk frames = [2636800,2637312) | 20260507 21:13:19.003 | INFO | MemInfo: { | 20260507 21:13:19.003 | INFO | "gpu" : " 17 GB", | 20260507 21:13:19.003 | INFO | "huge" : "", | 20260507 21:13:19.003 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:19.003 | INFO | "malloc" : " 14 GB", | 20260507 21:13:19.003 | INFO | "pinned" : " 29 GB" | 20260507 21:13:19.003 | INFO | } | 20260507 21:13:19.003 | INFO | Perf report for the last 5.13839 seconds: | 20260507 21:13:19.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:19.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:13:19.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:13:19.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:19.003 | INFO | Uploaded 36GiB to GPU in 2.5544s (14.0934GiB/s) | 20260507 21:13:19.003 | INFO | Downloaded 34GiB to GPU in 2.4391s (13.9395GiB/s) | 20260507 21:13:24.143 | INFO | Analyzing chunk frames = [2637312,2637824) | 20260507 21:13:24.143 | INFO | MemInfo: { | 20260507 21:13:24.143 | INFO | "gpu" : " 17 GB", | 20260507 21:13:24.143 | INFO | "huge" : "", | 20260507 21:13:24.143 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:24.143 | INFO | "malloc" : " 14 GB", | 20260507 21:13:24.143 | INFO | "pinned" : " 29 GB" | 20260507 21:13:24.143 | INFO | } | 20260507 21:13:24.143 | INFO | Perf report for the last 5.13987 seconds: | 20260507 21:13:24.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:24.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:13:24.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:13:24.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:24.143 | INFO | Uploaded 36GiB to GPU in 2.52003s (14.2855GiB/s) | 20260507 21:13:24.143 | INFO | Downloaded 34GiB to GPU in 2.43429s (13.9671GiB/s) | 20260507 21:13:24.143 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:13:24.143 | INFO | Total mem usage: 16.7 GiB | 20260507 21:13:24.143 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:13:24.144 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.8%) | 20260507 21:13:24.144 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:13:24.144 | INFO | High Water Breakdown: | 20260507 21:13:24.144 | INFO | 59.5% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:13:24.144 | INFO | 40.3% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:13:24.144 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:13:24.144 | INFO | ------------------------------------------ | 20260507 21:13:24.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:13:24.144 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:13:24.144 | INFO | Total mem usage: 26.5 GiB | 20260507 21:13:24.144 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:13:24.144 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:13:24.144 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:13:24.144 | INFO | High Water Breakdown: | 20260507 21:13:24.144 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:13:24.144 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:13:24.144 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:13:24.144 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:13:24.144 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:13:24.144 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:13:24.144 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:13:24.144 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:13:24.144 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:13:24.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:13:24.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:13:24.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:13:24.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:13:24.144 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:13:24.144 | INFO | ------------------------------------------ | 20260507 21:13:24.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:13:24.144 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:13:24.144 | INFO | Total mem usage: 27.8 GiB | 20260507 21:13:24.144 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:13:24.144 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:13:24.144 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:13:24.144 | INFO | High Water Breakdown: | 20260507 21:13:24.144 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:13:24.144 | INFO | ------------------------------------------ | 20260507 21:13:24.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:13:24.144 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:13:24.144 | INFO | Total mem usage: 15.7 GiB | 20260507 21:13:24.144 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:13:24.144 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:13:24.144 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:13:24.144 | INFO | High Water Breakdown: | 20260507 21:13:24.144 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:13:24.144 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:13:24.144 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:13:24.144 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:13:24.144 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:13:24.144 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:13:24.144 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:13:24.144 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:13:24.144 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:13:24.144 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:13:24.144 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:13:24.144 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:13:24.144 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:13:24.144 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:13:24.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:13:24.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:13:24.144 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:13:24.144 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:13:24.144 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:13:24.144 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:13:24.144 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:13:24.144 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:13:24.144 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:13:24.144 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:13:24.144 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:13:24.144 | INFO | ------------------------------------------ | 20260507 21:13:24.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:13:24.144 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:13:24.144 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:13:24.144 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:13:24.144 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.44e+01 / 1.79e+00 | 20260507 21:13:24.144 | INFO | PerfReport: 0.29%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.21e-02 / 7.37e-03 | 20260507 21:13:24.145 | INFO | PerfReport: 12.10%, Baseline -- 4.00e+00 / 4.47e+00, 5.93e-01 / 2.73e-01 | 20260507 21:13:24.145 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 1.21e-01 | 20260507 21:13:24.145 | INFO | PerfReport: 16.14%, DME -- 4.26e+01 / 4.62e+01, 4.32e+00 / 1.96e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 55.85%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.62e+00 / 1.38e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.51e+00 / 2.84e+00, 4.39e-01 / 2.19e-01 | 20260507 21:13:24.145 | INFO | PerfReport: 3.54%, Metrics -- 1.17e+00 / 1.38e+00, 2.93e-01 / 1.44e-01 | 20260507 21:13:24.145 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.61e-01 / 3.97e-01, 5.81e-02 / 2.92e-02 | 20260507 21:13:24.145 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:13:24.145 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:13:24.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:13:24.145 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 24.32%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 30.82%, Compute -- 3.30e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 2.13%, LockUpload -- 2.28e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 2.64%, LockDownload -- 2.83e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 15.86%, LockCompute -- 1.70e+01 / 0.00e+00, 1.99e+01 / 0.00e+00 | 20260507 21:13:24.145 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260507 21:13:24.242 | INFO | Starting a round of Baz IO | 20260507 21:13:27.722 | INFO | Populating RTMetrics report | 20260507 21:13:27.722 | INFO | RTMetrics report populated | 20260507 21:13:27.722 | INFO | Opening tmp RTMetrics json file | 20260507 21:13:27.739 | INFO | Serializing RTMetrics to json | 20260507 21:13:27.747 | INFO | Serialization done, swapping tmp file now... | 20260507 21:13:27.748 | INFO | RTMetrics done | 20260507 21:13:27.819 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.577 seconds | 20260507 21:13:27.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.88% / 0.01% / 56.11% | 20260507 21:13:27.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.75539% / 1.98682e-05% / 3.97364e-05% | 20260507 21:13:27.819 | INFO | Baz IO overall: | 20260507 21:13:27.819 | INFO | BytesWritten : 1200381433076 | 20260507 21:13:27.819 | INFO | NumEvents : 581830414482 | 20260507 21:13:29.281 | INFO | Analyzing chunk frames = [2637824,2638336) | 20260507 21:13:29.281 | INFO | MemInfo: { | 20260507 21:13:29.281 | INFO | "gpu" : " 17 GB", | 20260507 21:13:29.281 | INFO | "huge" : "", | 20260507 21:13:29.281 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:29.281 | INFO | "malloc" : " 14 GB", | 20260507 21:13:29.281 | INFO | "pinned" : " 29 GB" | 20260507 21:13:29.281 | INFO | } | 20260507 21:13:29.281 | INFO | Perf report for the last 5.13851 seconds: | 20260507 21:13:29.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:29.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:29.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:29.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:29.282 | INFO | ANALYSIS : 68.9%, 2.9, 105.6ms, 0 | 20260507 21:13:29.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:29.282 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 21:13:29.282 | INFO | PRE_HQ : 67.2%, 1.8, 61.2ms, 0 | 20260507 21:13:29.282 | INFO | BAZWRITER : 68.8%, 4.3, 153.1ms, 0 | 20260507 21:13:29.282 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 21:13:29.282 | INFO | Uploaded 36GiB to GPU in 2.5999s (13.8467GiB/s) | 20260507 21:13:29.282 | INFO | Downloaded 38GiB to GPU in 2.70032s (14.0724GiB/s) | 20260507 21:13:34.421 | INFO | Analyzing chunk frames = [2638336,2638848) | 20260507 21:13:34.421 | INFO | MemInfo: { | 20260507 21:13:34.421 | INFO | "gpu" : " 17 GB", | 20260507 21:13:34.421 | INFO | "huge" : "", | 20260507 21:13:34.421 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:34.421 | INFO | "malloc" : " 14 GB", | 20260507 21:13:34.421 | INFO | "pinned" : " 29 GB" | 20260507 21:13:34.421 | INFO | } | 20260507 21:13:34.422 | INFO | Perf report for the last 5.14007 seconds: | 20260507 21:13:34.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:34.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 21:13:34.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:13:34.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:34.422 | INFO | Uploaded 36GiB to GPU in 2.59762s (13.8589GiB/s) | 20260507 21:13:34.422 | INFO | Downloaded 34GiB to GPU in 2.4734s (13.7462GiB/s) | 20260507 21:13:39.561 | INFO | Analyzing chunk frames = [2638848,2639360) | 20260507 21:13:39.561 | INFO | MemInfo: { | 20260507 21:13:39.561 | INFO | "gpu" : " 17 GB", | 20260507 21:13:39.561 | INFO | "huge" : "", | 20260507 21:13:39.561 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:39.561 | INFO | "malloc" : " 14 GB", | 20260507 21:13:39.561 | INFO | "pinned" : " 29 GB" | 20260507 21:13:39.561 | INFO | } | 20260507 21:13:39.561 | INFO | Perf report for the last 5.13964 seconds: | 20260507 21:13:39.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:39.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.561 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:13:39.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.561 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:13:39.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:39.562 | INFO | Uploaded 36GiB to GPU in 2.53752s (14.1871GiB/s) | 20260507 21:13:39.562 | INFO | Downloaded 34GiB to GPU in 2.47047s (13.7626GiB/s) | 20260507 21:13:44.701 | INFO | Analyzing chunk frames = [2639360,2639872) | 20260507 21:13:44.701 | INFO | MemInfo: { | 20260507 21:13:44.701 | INFO | "gpu" : " 17 GB", | 20260507 21:13:44.701 | INFO | "huge" : "", | 20260507 21:13:44.701 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:44.701 | INFO | "malloc" : " 14 GB", | 20260507 21:13:44.701 | INFO | "pinned" : " 29 GB" | 20260507 21:13:44.701 | INFO | } | 20260507 21:13:44.701 | INFO | Perf report for the last 5.13997 seconds: | 20260507 21:13:44.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:44.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 21:13:44.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:13:44.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:44.701 | INFO | Uploaded 36GiB to GPU in 2.53274s (14.2138GiB/s) | 20260507 21:13:44.702 | INFO | Downloaded 34GiB to GPU in 2.4779s (13.7213GiB/s) | 20260507 21:13:49.839 | INFO | Analyzing chunk frames = [2639872,2640384) | 20260507 21:13:49.839 | INFO | MemInfo: { | 20260507 21:13:49.839 | INFO | "gpu" : " 17 GB", | 20260507 21:13:49.839 | INFO | "huge" : "", | 20260507 21:13:49.839 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:49.839 | INFO | "malloc" : " 14 GB", | 20260507 21:13:49.839 | INFO | "pinned" : " 29 GB" | 20260507 21:13:49.839 | INFO | } | 20260507 21:13:49.839 | INFO | Perf report for the last 5.13824 seconds: | 20260507 21:13:49.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:49.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.840 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:13:49.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.840 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:13:49.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:49.840 | INFO | Uploaded 36GiB to GPU in 2.46251s (14.6193GiB/s) | 20260507 21:13:49.840 | INFO | Downloaded 34GiB to GPU in 2.49434s (13.6308GiB/s) | 20260507 21:13:54.978 | INFO | Analyzing chunk frames = [2640384,2640896) | 20260507 21:13:54.978 | INFO | MemInfo: { | 20260507 21:13:54.978 | INFO | "gpu" : " 17 GB", | 20260507 21:13:54.978 | INFO | "huge" : "", | 20260507 21:13:54.978 | INFO | "huge_shared" : " 29 GB", | 20260507 21:13:54.978 | INFO | "malloc" : " 14 GB", | 20260507 21:13:54.978 | INFO | "pinned" : " 29 GB" | 20260507 21:13:54.978 | INFO | } | 20260507 21:13:54.978 | INFO | Perf report for the last 5.13879 seconds: | 20260507 21:13:54.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:13:54.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.978 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260507 21:13:54.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.978 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:13:54.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:13:54.979 | INFO | Uploaded 36GiB to GPU in 2.42482s (14.8465GiB/s) | 20260507 21:13:54.979 | INFO | Downloaded 34GiB to GPU in 2.38001s (14.2856GiB/s) | 20260507 21:14:00.117 | INFO | Analyzing chunk frames = [2640896,2641408) | 20260507 21:14:00.117 | INFO | MemInfo: { | 20260507 21:14:00.117 | INFO | "gpu" : " 17 GB", | 20260507 21:14:00.117 | INFO | "huge" : "", | 20260507 21:14:00.117 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:00.117 | INFO | "malloc" : " 14 GB", | 20260507 21:14:00.117 | INFO | "pinned" : " 29 GB" | 20260507 21:14:00.117 | INFO | } | 20260507 21:14:00.117 | INFO | Perf report for the last 5.13915 seconds: | 20260507 21:14:00.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:00.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:14:00.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 21:14:00.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:00.118 | INFO | Uploaded 36GiB to GPU in 2.57471s (13.9822GiB/s) | 20260507 21:14:00.118 | INFO | Downloaded 34GiB to GPU in 2.45525s (13.8479GiB/s) | 20260507 21:14:05.257 | INFO | Analyzing chunk frames = [2641408,2641920) | 20260507 21:14:05.257 | INFO | MemInfo: { | 20260507 21:14:05.257 | INFO | "gpu" : " 17 GB", | 20260507 21:14:05.257 | INFO | "huge" : "", | 20260507 21:14:05.257 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:05.257 | INFO | "malloc" : " 14 GB", | 20260507 21:14:05.257 | INFO | "pinned" : " 29 GB" | 20260507 21:14:05.257 | INFO | } | 20260507 21:14:05.257 | INFO | Perf report for the last 5.13943 seconds: | 20260507 21:14:05.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:05.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 21:14:05.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:14:05.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:05.257 | INFO | Uploaded 36GiB to GPU in 2.53469s (14.2029GiB/s) | 20260507 21:14:05.258 | INFO | Downloaded 34GiB to GPU in 2.43229s (13.9786GiB/s) | 20260507 21:14:05.354 | INFO | Starting a round of Baz IO | 20260507 21:14:08.823 | INFO | Populating RTMetrics report | 20260507 21:14:08.824 | INFO | RTMetrics report populated | 20260507 21:14:08.824 | INFO | Opening tmp RTMetrics json file | 20260507 21:14:08.846 | INFO | Serializing RTMetrics to json | 20260507 21:14:08.848 | INFO | Serialization done, swapping tmp file now... | 20260507 21:14:08.849 | INFO | RTMetrics done | 20260507 21:14:08.930 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.575 seconds | 20260507 21:14:08.930 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.87% / 0.01% / 56.12% | 20260507 21:14:08.930 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74986% / 7.94729e-06% / 5.16574e-05% | 20260507 21:14:08.930 | INFO | Baz IO overall: | 20260507 21:14:08.930 | INFO | BytesWritten : 1202273116680 | 20260507 21:14:08.930 | INFO | NumEvents : 582744919914 | 20260507 21:14:10.397 | INFO | Analyzing chunk frames = [2641920,2642432) | 20260507 21:14:10.397 | INFO | MemInfo: { | 20260507 21:14:10.397 | INFO | "gpu" : " 17 GB", | 20260507 21:14:10.397 | INFO | "huge" : "", | 20260507 21:14:10.397 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:10.397 | INFO | "malloc" : " 14 GB", | 20260507 21:14:10.397 | INFO | "pinned" : " 29 GB" | 20260507 21:14:10.397 | INFO | } | 20260507 21:14:10.397 | INFO | Perf report for the last 5.14056 seconds: | 20260507 21:14:10.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:10.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:10.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:10.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:10.397 | INFO | ANALYSIS : 68.6%, 3.0, 106.9ms, 0 | 20260507 21:14:10.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:10.398 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 21:14:10.398 | INFO | PRE_HQ : 65.7%, 1.7, 57.7ms, 0 | 20260507 21:14:10.398 | INFO | BAZWRITER : 69.2%, 3.9, 137.4ms, 0 | 20260507 21:14:10.398 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 21:14:10.398 | INFO | Uploaded 36GiB to GPU in 2.57087s (14.003GiB/s) | 20260507 21:14:10.398 | INFO | Downloaded 38GiB to GPU in 2.68443s (14.1557GiB/s) | 20260507 21:14:15.536 | INFO | Analyzing chunk frames = [2642432,2642944) | 20260507 21:14:15.536 | INFO | MemInfo: { | 20260507 21:14:15.536 | INFO | "gpu" : " 17 GB", | 20260507 21:14:15.536 | INFO | "huge" : "", | 20260507 21:14:15.536 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:15.536 | INFO | "malloc" : " 14 GB", | 20260507 21:14:15.536 | INFO | "pinned" : " 29 GB" | 20260507 21:14:15.536 | INFO | } | 20260507 21:14:15.536 | INFO | Perf report for the last 5.1385 seconds: | 20260507 21:14:15.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:15.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.536 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 21:14:15.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.536 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 21:14:15.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:15.537 | INFO | Uploaded 36GiB to GPU in 2.59227s (13.8874GiB/s) | 20260507 21:14:15.537 | INFO | Downloaded 34GiB to GPU in 2.47566s (13.7337GiB/s) | 20260507 21:14:20.674 | INFO | Analyzing chunk frames = [2642944,2643456) | 20260507 21:14:20.674 | INFO | MemInfo: { | 20260507 21:14:20.674 | INFO | "gpu" : " 17 GB", | 20260507 21:14:20.674 | INFO | "huge" : "", | 20260507 21:14:20.674 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:20.674 | INFO | "malloc" : " 14 GB", | 20260507 21:14:20.674 | INFO | "pinned" : " 29 GB" | 20260507 21:14:20.674 | INFO | } | 20260507 21:14:20.674 | INFO | Perf report for the last 5.13811 seconds: | 20260507 21:14:20.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:20.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.674 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:14:20.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.674 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:14:20.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:20.675 | INFO | Uploaded 36GiB to GPU in 2.50714s (14.359GiB/s) | 20260507 21:14:20.675 | INFO | Downloaded 34GiB to GPU in 2.47501s (13.7373GiB/s) | 20260507 21:14:25.816 | INFO | Analyzing chunk frames = [2643456,2643968) | 20260507 21:14:25.816 | INFO | MemInfo: { | 20260507 21:14:25.816 | INFO | "gpu" : " 17 GB", | 20260507 21:14:25.816 | INFO | "huge" : "", | 20260507 21:14:25.816 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:25.816 | INFO | "malloc" : " 14 GB", | 20260507 21:14:25.816 | INFO | "pinned" : " 29 GB" | 20260507 21:14:25.816 | INFO | } | 20260507 21:14:25.816 | INFO | Perf report for the last 5.14183 seconds: | 20260507 21:14:25.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:25.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.816 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 21:14:25.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.816 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:14:25.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:25.817 | INFO | Uploaded 36GiB to GPU in 2.51963s (14.2878GiB/s) | 20260507 21:14:25.817 | INFO | Downloaded 34GiB to GPU in 2.47641s (13.7295GiB/s) | 20260507 21:14:30.954 | INFO | Analyzing chunk frames = [2643968,2644480) | 20260507 21:14:30.954 | INFO | MemInfo: { | 20260507 21:14:30.954 | INFO | "gpu" : " 17 GB", | 20260507 21:14:30.954 | INFO | "huge" : "", | 20260507 21:14:30.954 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:30.954 | INFO | "malloc" : " 14 GB", | 20260507 21:14:30.954 | INFO | "pinned" : " 29 GB" | 20260507 21:14:30.954 | INFO | } | 20260507 21:14:30.954 | INFO | Perf report for the last 5.13852 seconds: | 20260507 21:14:30.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:30.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.955 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 21:14:30.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.955 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 21:14:30.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:30.955 | INFO | Uploaded 36GiB to GPU in 2.38295s (15.1073GiB/s) | 20260507 21:14:30.955 | INFO | Downloaded 34GiB to GPU in 2.48086s (13.7049GiB/s) | 20260507 21:14:36.093 | INFO | Analyzing chunk frames = [2644480,2644992) | 20260507 21:14:36.093 | INFO | MemInfo: { | 20260507 21:14:36.093 | INFO | "gpu" : " 17 GB", | 20260507 21:14:36.093 | INFO | "huge" : "", | 20260507 21:14:36.093 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:36.093 | INFO | "malloc" : " 14 GB", | 20260507 21:14:36.093 | INFO | "pinned" : " 29 GB" | 20260507 21:14:36.093 | INFO | } | 20260507 21:14:36.093 | INFO | Perf report for the last 5.13827 seconds: | 20260507 21:14:36.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:36.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:14:36.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:14:36.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:36.093 | INFO | Uploaded 36GiB to GPU in 2.40287s (14.9821GiB/s) | 20260507 21:14:36.093 | INFO | Downloaded 34GiB to GPU in 2.39114s (14.2192GiB/s) | 20260507 21:14:41.233 | INFO | Analyzing chunk frames = [2644992,2645504) | 20260507 21:14:41.233 | INFO | MemInfo: { | 20260507 21:14:41.233 | INFO | "gpu" : " 17 GB", | 20260507 21:14:41.233 | INFO | "huge" : "", | 20260507 21:14:41.233 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:41.233 | INFO | "malloc" : " 14 GB", | 20260507 21:14:41.233 | INFO | "pinned" : " 29 GB" | 20260507 21:14:41.233 | INFO | } | 20260507 21:14:41.233 | INFO | Perf report for the last 5.14025 seconds: | 20260507 21:14:41.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:41.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:14:41.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:14:41.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:41.233 | INFO | Uploaded 36GiB to GPU in 2.53207s (14.2176GiB/s) | 20260507 21:14:41.233 | INFO | Downloaded 34GiB to GPU in 2.44878s (13.8845GiB/s) | 20260507 21:14:46.372 | INFO | Analyzing chunk frames = [2645504,2646016) | 20260507 21:14:46.372 | INFO | MemInfo: { | 20260507 21:14:46.372 | INFO | "gpu" : " 17 GB", | 20260507 21:14:46.372 | INFO | "huge" : "", | 20260507 21:14:46.372 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:46.372 | INFO | "malloc" : " 14 GB", | 20260507 21:14:46.372 | INFO | "pinned" : " 29 GB" | 20260507 21:14:46.372 | INFO | } | 20260507 21:14:46.372 | INFO | Perf report for the last 5.13928 seconds: | 20260507 21:14:46.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:46.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.372 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:14:46.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.372 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:14:46.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:46.373 | INFO | Uploaded 36GiB to GPU in 2.51642s (14.3061GiB/s) | 20260507 21:14:46.373 | INFO | Downloaded 34GiB to GPU in 2.43531s (13.9613GiB/s) | 20260507 21:14:46.373 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:14:46.373 | INFO | Total mem usage: 16.7 GiB | 20260507 21:14:46.373 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:14:46.373 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.5%) | 20260507 21:14:46.373 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:14:46.373 | INFO | High Water Breakdown: | 20260507 21:14:46.373 | INFO | 60% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:14:46.373 | INFO | 39.8% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:14:46.373 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:14:46.373 | INFO | ------------------------------------------ | 20260507 21:14:46.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:14:46.373 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:14:46.373 | INFO | Total mem usage: 26.5 GiB | 20260507 21:14:46.373 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:14:46.373 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:14:46.373 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:14:46.373 | INFO | High Water Breakdown: | 20260507 21:14:46.373 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:14:46.373 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:14:46.373 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:14:46.373 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:14:46.373 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:14:46.373 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:14:46.373 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:14:46.373 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:14:46.373 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:14:46.373 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:14:46.373 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:14:46.373 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:14:46.373 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:14:46.373 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:14:46.374 | INFO | ------------------------------------------ | 20260507 21:14:46.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:14:46.374 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:14:46.374 | INFO | Total mem usage: 27.8 GiB | 20260507 21:14:46.374 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:14:46.374 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:14:46.374 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:14:46.374 | INFO | High Water Breakdown: | 20260507 21:14:46.374 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:14:46.374 | INFO | ------------------------------------------ | 20260507 21:14:46.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:14:46.374 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:14:46.374 | INFO | Total mem usage: 15.7 GiB | 20260507 21:14:46.374 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:14:46.374 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:14:46.374 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:14:46.374 | INFO | High Water Breakdown: | 20260507 21:14:46.374 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:14:46.374 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:14:46.374 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:14:46.374 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:14:46.374 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:14:46.374 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:14:46.374 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:14:46.374 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:14:46.374 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:14:46.374 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:14:46.374 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:14:46.374 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:14:46.374 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:14:46.374 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:14:46.374 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:14:46.374 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:14:46.374 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:14:46.374 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:14:46.374 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:14:46.374 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:14:46.374 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:14:46.374 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:14:46.374 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:14:46.374 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:14:46.374 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:14:46.374 | INFO | ------------------------------------------ | 20260507 21:14:46.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:14:46.374 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:14:46.374 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:14:46.374 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:14:46.374 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.43e+01 / 1.95e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.22e-02 / 5.35e-03 | 20260507 21:14:46.374 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.49e+00, 5.90e-01 / 2.11e-01 | 20260507 21:14:46.374 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.51e+00, 2.08e-01 / 9.67e-02 | 20260507 21:14:46.374 | INFO | PerfReport: 16.10%, DME -- 4.25e+01 / 4.61e+01, 4.08e+00 / 1.90e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 55.87%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.62e+00 / 1.25e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.51e+00 / 2.85e+00, 4.40e-01 / 1.86e-01 | 20260507 21:14:46.374 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.39e+00, 2.94e-01 / 1.52e-01 | 20260507 21:14:46.374 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.62e-01 / 4.02e-01, 5.93e-02 / 3.15e-02 | 20260507 21:14:46.374 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:14:46.374 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:14:46.374 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:14:46.374 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 24.49%, Upload -- 2.62e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 24.17%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 30.83%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 2.38%, LockUpload -- 2.55e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 2.38%, LockDownload -- 2.55e+00 / 0.00e+00, 3.70e+00 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 15.74%, LockCompute -- 1.69e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260507 21:14:46.374 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260507 21:14:46.480 | INFO | Starting a round of Baz IO | 20260507 21:14:49.944 | INFO | Populating RTMetrics report | 20260507 21:14:49.945 | INFO | RTMetrics report populated | 20260507 21:14:49.945 | INFO | Opening tmp RTMetrics json file | 20260507 21:14:49.948 | INFO | Serializing RTMetrics to json | 20260507 21:14:49.990 | INFO | Serialization done, swapping tmp file now... | 20260507 21:14:49.990 | INFO | RTMetrics done | 20260507 21:14:50.108 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.627 seconds | 20260507 21:14:50.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.87% / 0.01% / 56.13% | 20260507 21:14:50.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.75119% / 3.97364e-06% / 5.5631e-05% | 20260507 21:14:50.108 | INFO | Baz IO overall: | 20260507 21:14:50.108 | INFO | BytesWritten : 1204167030623 | 20260507 21:14:50.108 | INFO | NumEvents : 583661241099 | 20260507 21:14:51.510 | INFO | Analyzing chunk frames = [2646016,2646528) | 20260507 21:14:51.510 | INFO | MemInfo: { | 20260507 21:14:51.510 | INFO | "gpu" : " 17 GB", | 20260507 21:14:51.510 | INFO | "huge" : "", | 20260507 21:14:51.510 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:51.510 | INFO | "malloc" : " 14 GB", | 20260507 21:14:51.510 | INFO | "pinned" : " 29 GB" | 20260507 21:14:51.510 | INFO | } | 20260507 21:14:51.510 | INFO | Perf report for the last 5.13784 seconds: | 20260507 21:14:51.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:51.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:51.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:51.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:51.510 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 21:14:51.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:51.510 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 21:14:51.510 | INFO | PRE_HQ : 66.6%, 1.7, 58.0ms, 0 | 20260507 21:14:51.510 | INFO | BAZWRITER : 70.4%, 4.5, 161.9ms, 0 | 20260507 21:14:51.510 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 21:14:51.511 | INFO | Uploaded 36GiB to GPU in 2.51034s (14.3407GiB/s) | 20260507 21:14:51.511 | INFO | Downloaded 38GiB to GPU in 2.71123s (14.0158GiB/s) | 20260507 21:14:56.650 | INFO | Analyzing chunk frames = [2646528,2647040) | 20260507 21:14:56.650 | INFO | MemInfo: { | 20260507 21:14:56.650 | INFO | "gpu" : " 17 GB", | 20260507 21:14:56.650 | INFO | "huge" : "", | 20260507 21:14:56.650 | INFO | "huge_shared" : " 29 GB", | 20260507 21:14:56.650 | INFO | "malloc" : " 14 GB", | 20260507 21:14:56.650 | INFO | "pinned" : " 29 GB" | 20260507 21:14:56.650 | INFO | } | 20260507 21:14:56.650 | INFO | Perf report for the last 5.13965 seconds: | 20260507 21:14:56.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:14:56.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.650 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:14:56.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.650 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:14:56.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:14:56.651 | INFO | Uploaded 36GiB to GPU in 2.52854s (14.2375GiB/s) | 20260507 21:14:56.651 | INFO | Downloaded 34GiB to GPU in 2.48444s (13.6852GiB/s) | 20260507 21:15:01.789 | INFO | Analyzing chunk frames = [2647040,2647552) | 20260507 21:15:01.789 | INFO | MemInfo: { | 20260507 21:15:01.789 | INFO | "gpu" : " 17 GB", | 20260507 21:15:01.789 | INFO | "huge" : "", | 20260507 21:15:01.789 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:01.789 | INFO | "malloc" : " 14 GB", | 20260507 21:15:01.789 | INFO | "pinned" : " 29 GB" | 20260507 21:15:01.789 | INFO | } | 20260507 21:15:01.789 | INFO | Perf report for the last 5.13963 seconds: | 20260507 21:15:01.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:01.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:15:01.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:15:01.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:01.790 | INFO | Uploaded 36GiB to GPU in 2.57846s (13.9618GiB/s) | 20260507 21:15:01.790 | INFO | Downloaded 34GiB to GPU in 2.47308s (13.748GiB/s) | 20260507 21:15:06.929 | INFO | Analyzing chunk frames = [2647552,2648064) | 20260507 21:15:06.929 | INFO | MemInfo: { | 20260507 21:15:06.929 | INFO | "gpu" : " 17 GB", | 20260507 21:15:06.929 | INFO | "huge" : "", | 20260507 21:15:06.929 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:06.929 | INFO | "malloc" : " 14 GB", | 20260507 21:15:06.929 | INFO | "pinned" : " 29 GB" | 20260507 21:15:06.929 | INFO | } | 20260507 21:15:06.929 | INFO | Perf report for the last 5.13986 seconds: | 20260507 21:15:06.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:06.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.929 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:15:06.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.929 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 21:15:06.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:06.930 | INFO | Uploaded 36GiB to GPU in 2.48589s (14.4818GiB/s) | 20260507 21:15:06.930 | INFO | Downloaded 34GiB to GPU in 2.49544s (13.6249GiB/s) | 20260507 21:15:12.071 | INFO | Analyzing chunk frames = [2648064,2648576) | 20260507 21:15:12.071 | INFO | MemInfo: { | 20260507 21:15:12.071 | INFO | "gpu" : " 17 GB", | 20260507 21:15:12.071 | INFO | "huge" : "", | 20260507 21:15:12.071 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:12.071 | INFO | "malloc" : " 14 GB", | 20260507 21:15:12.071 | INFO | "pinned" : " 29 GB" | 20260507 21:15:12.071 | INFO | } | 20260507 21:15:12.071 | INFO | Perf report for the last 5.14158 seconds: | 20260507 21:15:12.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:12.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:15:12.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:15:12.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:12.071 | INFO | Uploaded 36GiB to GPU in 2.37075s (15.185GiB/s) | 20260507 21:15:12.071 | INFO | Downloaded 34GiB to GPU in 2.49034s (13.6528GiB/s) | 20260507 21:15:17.208 | INFO | Analyzing chunk frames = [2648576,2649088) | 20260507 21:15:17.208 | INFO | MemInfo: { | 20260507 21:15:17.208 | INFO | "gpu" : " 17 GB", | 20260507 21:15:17.208 | INFO | "huge" : "", | 20260507 21:15:17.208 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:17.208 | INFO | "malloc" : " 14 GB", | 20260507 21:15:17.208 | INFO | "pinned" : " 29 GB" | 20260507 21:15:17.208 | INFO | } | 20260507 21:15:17.208 | INFO | Perf report for the last 5.13742 seconds: | 20260507 21:15:17.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:17.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.208 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260507 21:15:17.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.209 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:15:17.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:17.209 | INFO | Uploaded 36GiB to GPU in 2.3972s (15.0175GiB/s) | 20260507 21:15:17.209 | INFO | Downloaded 34GiB to GPU in 2.38536s (14.2536GiB/s) | 20260507 21:15:22.347 | INFO | Analyzing chunk frames = [2649088,2649600) | 20260507 21:15:22.347 | INFO | MemInfo: { | 20260507 21:15:22.347 | INFO | "gpu" : " 17 GB", | 20260507 21:15:22.347 | INFO | "huge" : "", | 20260507 21:15:22.347 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:22.347 | INFO | "malloc" : " 14 GB", | 20260507 21:15:22.347 | INFO | "pinned" : " 29 GB" | 20260507 21:15:22.347 | INFO | } | 20260507 21:15:22.347 | INFO | Perf report for the last 5.13844 seconds: | 20260507 21:15:22.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:22.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 21:15:22.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 21:15:22.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:22.347 | INFO | Uploaded 36GiB to GPU in 2.51843s (14.2946GiB/s) | 20260507 21:15:22.348 | INFO | Downloaded 34GiB to GPU in 2.43262s (13.9767GiB/s) | 20260507 21:15:27.485 | INFO | Analyzing chunk frames = [2649600,2650112) | 20260507 21:15:27.486 | INFO | MemInfo: { | 20260507 21:15:27.486 | INFO | "gpu" : " 17 GB", | 20260507 21:15:27.486 | INFO | "huge" : "", | 20260507 21:15:27.486 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:27.486 | INFO | "malloc" : " 14 GB", | 20260507 21:15:27.486 | INFO | "pinned" : " 29 GB" | 20260507 21:15:27.486 | INFO | } | 20260507 21:15:27.486 | INFO | Perf report for the last 5.13885 seconds: | 20260507 21:15:27.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:27.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:15:27.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 21:15:27.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:27.486 | INFO | Uploaded 36GiB to GPU in 2.50372s (14.3786GiB/s) | 20260507 21:15:27.486 | INFO | Downloaded 34GiB to GPU in 2.42633s (14.0129GiB/s) | 20260507 21:15:27.583 | INFO | Starting a round of Baz IO | 20260507 21:15:31.067 | INFO | Populating RTMetrics report | 20260507 21:15:31.068 | INFO | RTMetrics report populated | 20260507 21:15:31.068 | INFO | Opening tmp RTMetrics json file | 20260507 21:15:31.154 | INFO | Serializing RTMetrics to json | 20260507 21:15:31.192 | INFO | Serialization done, swapping tmp file now... | 20260507 21:15:31.195 | INFO | RTMetrics done | 20260507 21:15:31.227 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.644 seconds | 20260507 21:15:31.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.86% / 0.01% / 56.13% | 20260507 21:15:31.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74631% / 1.58946e-05% / 5.16574e-05% | 20260507 21:15:31.227 | INFO | Baz IO overall: | 20260507 21:15:31.227 | INFO | BytesWritten : 1206059952115 | 20260507 21:15:31.227 | INFO | NumEvents : 584576778104 | 20260507 21:15:32.625 | INFO | Analyzing chunk frames = [2650112,2650624) | 20260507 21:15:32.625 | INFO | MemInfo: { | 20260507 21:15:32.625 | INFO | "gpu" : " 17 GB", | 20260507 21:15:32.625 | INFO | "huge" : "", | 20260507 21:15:32.625 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:32.625 | INFO | "malloc" : " 14 GB", | 20260507 21:15:32.626 | INFO | "pinned" : " 29 GB" | 20260507 21:15:32.626 | INFO | } | 20260507 21:15:32.626 | INFO | Perf report for the last 5.13993 seconds: | 20260507 21:15:32.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:32.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:32.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:32.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:32.626 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 21:15:32.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:32.626 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260507 21:15:32.626 | INFO | PRE_HQ : 66.8%, 1.8, 62.6ms, 0 | 20260507 21:15:32.626 | INFO | BAZWRITER : 70.5%, 3.9, 140.4ms, 0 | 20260507 21:15:32.626 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260507 21:15:32.626 | INFO | Uploaded 36GiB to GPU in 2.48607s (14.4807GiB/s) | 20260507 21:15:32.626 | INFO | Downloaded 38GiB to GPU in 2.68126s (14.1725GiB/s) | 20260507 21:15:37.764 | INFO | Analyzing chunk frames = [2650624,2651136) | 20260507 21:15:37.764 | INFO | MemInfo: { | 20260507 21:15:37.764 | INFO | "gpu" : " 17 GB", | 20260507 21:15:37.764 | INFO | "huge" : "", | 20260507 21:15:37.764 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:37.764 | INFO | "malloc" : " 14 GB", | 20260507 21:15:37.764 | INFO | "pinned" : " 29 GB" | 20260507 21:15:37.764 | INFO | } | 20260507 21:15:37.765 | INFO | Perf report for the last 5.1389 seconds: | 20260507 21:15:37.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:37.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:15:37.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:15:37.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:37.765 | INFO | Uploaded 36GiB to GPU in 2.56258s (14.0483GiB/s) | 20260507 21:15:37.765 | INFO | Downloaded 34GiB to GPU in 2.49887s (13.6062GiB/s) | 20260507 21:15:42.904 | INFO | Analyzing chunk frames = [2651136,2651648) | 20260507 21:15:42.904 | INFO | MemInfo: { | 20260507 21:15:42.904 | INFO | "gpu" : " 17 GB", | 20260507 21:15:42.904 | INFO | "huge" : "", | 20260507 21:15:42.904 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:42.905 | INFO | "malloc" : " 14 GB", | 20260507 21:15:42.905 | INFO | "pinned" : " 29 GB" | 20260507 21:15:42.905 | INFO | } | 20260507 21:15:42.905 | INFO | Perf report for the last 5.14018 seconds: | 20260507 21:15:42.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:42.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:15:42.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:15:42.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:42.905 | INFO | Uploaded 36GiB to GPU in 2.46665s (14.5947GiB/s) | 20260507 21:15:42.905 | INFO | Downloaded 34GiB to GPU in 2.48166s (13.7005GiB/s) | 20260507 21:15:48.045 | INFO | Analyzing chunk frames = [2651648,2652160) | 20260507 21:15:48.045 | INFO | MemInfo: { | 20260507 21:15:48.045 | INFO | "gpu" : " 17 GB", | 20260507 21:15:48.045 | INFO | "huge" : "", | 20260507 21:15:48.045 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:48.045 | INFO | "malloc" : " 14 GB", | 20260507 21:15:48.045 | INFO | "pinned" : " 29 GB" | 20260507 21:15:48.045 | INFO | } | 20260507 21:15:48.045 | INFO | Perf report for the last 5.14024 seconds: | 20260507 21:15:48.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:48.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:15:48.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:15:48.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:48.045 | INFO | Uploaded 36GiB to GPU in 2.49757s (14.414GiB/s) | 20260507 21:15:48.045 | INFO | Downloaded 34GiB to GPU in 2.48632s (13.6748GiB/s) | 20260507 21:15:53.185 | INFO | Analyzing chunk frames = [2652160,2652672) | 20260507 21:15:53.185 | INFO | MemInfo: { | 20260507 21:15:53.185 | INFO | "gpu" : " 17 GB", | 20260507 21:15:53.185 | INFO | "huge" : "", | 20260507 21:15:53.185 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:53.185 | INFO | "malloc" : " 14 GB", | 20260507 21:15:53.185 | INFO | "pinned" : " 29 GB" | 20260507 21:15:53.185 | INFO | } | 20260507 21:15:53.185 | INFO | Perf report for the last 5.14012 seconds: | 20260507 21:15:53.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:53.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 21:15:53.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 21:15:53.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:53.185 | INFO | Uploaded 36GiB to GPU in 2.36016s (15.2532GiB/s) | 20260507 21:15:53.185 | INFO | Downloaded 34GiB to GPU in 2.48956s (13.6571GiB/s) | 20260507 21:15:58.323 | INFO | Analyzing chunk frames = [2652672,2653184) | 20260507 21:15:58.323 | INFO | MemInfo: { | 20260507 21:15:58.323 | INFO | "gpu" : " 17 GB", | 20260507 21:15:58.323 | INFO | "huge" : "", | 20260507 21:15:58.323 | INFO | "huge_shared" : " 29 GB", | 20260507 21:15:58.323 | INFO | "malloc" : " 14 GB", | 20260507 21:15:58.323 | INFO | "pinned" : " 29 GB" | 20260507 21:15:58.323 | INFO | } | 20260507 21:15:58.323 | INFO | Perf report for the last 5.13805 seconds: | 20260507 21:15:58.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:15:58.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.323 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:15:58.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.323 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260507 21:15:58.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:15:58.324 | INFO | Uploaded 36GiB to GPU in 2.43437s (14.7882GiB/s) | 20260507 21:15:58.324 | INFO | Downloaded 34GiB to GPU in 2.37563s (14.312GiB/s) | 20260507 21:16:03.462 | INFO | Analyzing chunk frames = [2653184,2653696) | 20260507 21:16:03.462 | INFO | MemInfo: { | 20260507 21:16:03.462 | INFO | "gpu" : " 17 GB", | 20260507 21:16:03.462 | INFO | "huge" : "", | 20260507 21:16:03.462 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:03.462 | INFO | "malloc" : " 14 GB", | 20260507 21:16:03.462 | INFO | "pinned" : " 29 GB" | 20260507 21:16:03.463 | INFO | } | 20260507 21:16:03.463 | INFO | Perf report for the last 5.13936 seconds: | 20260507 21:16:03.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:03.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260507 21:16:03.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 21:16:03.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:03.463 | INFO | Uploaded 36GiB to GPU in 2.56303s (14.0459GiB/s) | 20260507 21:16:03.463 | INFO | Downloaded 34GiB to GPU in 2.47356s (13.7454GiB/s) | 20260507 21:16:08.601 | INFO | Analyzing chunk frames = [2653696,2654208) | 20260507 21:16:08.601 | INFO | MemInfo: { | 20260507 21:16:08.601 | INFO | "gpu" : " 17 GB", | 20260507 21:16:08.601 | INFO | "huge" : "", | 20260507 21:16:08.601 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:08.601 | INFO | "malloc" : " 14 GB", | 20260507 21:16:08.601 | INFO | "pinned" : " 29 GB" | 20260507 21:16:08.602 | INFO | } | 20260507 21:16:08.602 | INFO | Perf report for the last 5.13901 seconds: | 20260507 21:16:08.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:08.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:16:08.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 21:16:08.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:08.602 | INFO | Uploaded 36GiB to GPU in 2.53872s (14.1804GiB/s) | 20260507 21:16:08.602 | INFO | Downloaded 34GiB to GPU in 2.46288s (13.805GiB/s) | 20260507 21:16:08.602 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:16:08.602 | INFO | Total mem usage: 16.7 GiB | 20260507 21:16:08.602 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:16:08.602 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.7%) | 20260507 21:16:08.602 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:16:08.602 | INFO | High Water Breakdown: | 20260507 21:16:08.602 | INFO | 60.2% : 9.07 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:16:08.602 | INFO | 39.7% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:16:08.602 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:16:08.602 | INFO | ------------------------------------------ | 20260507 21:16:08.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:16:08.602 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:16:08.602 | INFO | Total mem usage: 26.5 GiB | 20260507 21:16:08.602 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:16:08.602 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:16:08.602 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:16:08.602 | INFO | High Water Breakdown: | 20260507 21:16:08.602 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:16:08.602 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:16:08.602 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:16:08.602 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:16:08.602 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:16:08.602 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:16:08.602 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:16:08.602 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:16:08.603 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:16:08.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:16:08.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:16:08.603 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:16:08.603 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:16:08.603 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:16:08.603 | INFO | ------------------------------------------ | 20260507 21:16:08.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:16:08.603 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:16:08.603 | INFO | Total mem usage: 27.8 GiB | 20260507 21:16:08.603 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:16:08.603 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:16:08.603 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:16:08.603 | INFO | High Water Breakdown: | 20260507 21:16:08.603 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:16:08.603 | INFO | ------------------------------------------ | 20260507 21:16:08.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:16:08.603 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:16:08.603 | INFO | Total mem usage: 15.7 GiB | 20260507 21:16:08.603 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:16:08.603 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:16:08.603 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:16:08.603 | INFO | High Water Breakdown: | 20260507 21:16:08.603 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:16:08.603 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:16:08.603 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:16:08.603 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:16:08.603 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:16:08.603 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:16:08.603 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:16:08.603 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:16:08.603 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:16:08.603 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:16:08.603 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:16:08.603 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:16:08.603 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:16:08.603 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:16:08.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:16:08.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:16:08.603 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:16:08.603 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:16:08.603 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:16:08.603 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:16:08.603 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:16:08.603 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:16:08.603 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:16:08.603 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:16:08.603 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:16:08.603 | INFO | ------------------------------------------ | 20260507 21:16:08.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:16:08.603 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:16:08.603 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:16:08.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:16:08.603 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.65e+01, 1.41e+01 / 2.26e+00 | 20260507 21:16:08.603 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.03e-01, 1.18e-02 / 8.65e-03 | 20260507 21:16:08.603 | INFO | PerfReport: 12.16%, Baseline -- 4.03e+00 / 4.48e+00, 6.00e-01 / 2.69e-01 | 20260507 21:16:08.603 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.11e-01 / 1.28e-01 | 20260507 21:16:08.603 | INFO | PerfReport: 16.07%, DME -- 4.26e+01 / 4.62e+01, 4.58e+00 / 2.09e+00 | 20260507 21:16:08.603 | INFO | PerfReport: 55.90%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.57e+00 | 20260507 21:16:08.603 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.43e-01 / 2.04e-01 | 20260507 21:16:08.604 | INFO | PerfReport: 3.55%, Metrics -- 1.18e+00 / 1.38e+00, 2.95e-01 / 1.51e-01 | 20260507 21:16:08.604 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.65e-01 / 3.97e-01, 5.97e-02 / 2.74e-02 | 20260507 21:16:08.604 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:16:08.604 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:16:08.604 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:16:08.604 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 24.20%, Upload -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 30.93%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 2.06%, LockUpload -- 2.20e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 2.68%, LockDownload -- 2.87e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:16:08.604 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260507 21:16:08.717 | INFO | Starting a round of Baz IO | 20260507 21:16:12.192 | INFO | Populating RTMetrics report | 20260507 21:16:12.193 | INFO | RTMetrics report populated | 20260507 21:16:12.193 | INFO | Opening tmp RTMetrics json file | 20260507 21:16:12.218 | INFO | Serializing RTMetrics to json | 20260507 21:16:12.221 | INFO | Serialization done, swapping tmp file now... | 20260507 21:16:12.221 | INFO | RTMetrics done | 20260507 21:16:12.285 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.568 seconds | 20260507 21:16:12.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.85% / 0.01% / 56.14% | 20260507 21:16:12.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74458% / 1.19209e-05% / 4.37101e-05% | 20260507 21:16:12.285 | INFO | Baz IO overall: | 20260507 21:16:12.285 | INFO | BytesWritten : 1207953161977 | 20260507 21:16:12.285 | INFO | NumEvents : 585492740751 | 20260507 21:16:13.740 | INFO | Analyzing chunk frames = [2654208,2654720) | 20260507 21:16:13.740 | INFO | MemInfo: { | 20260507 21:16:13.740 | INFO | "gpu" : " 17 GB", | 20260507 21:16:13.740 | INFO | "huge" : "", | 20260507 21:16:13.740 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:13.740 | INFO | "malloc" : " 14 GB", | 20260507 21:16:13.741 | INFO | "pinned" : " 29 GB" | 20260507 21:16:13.741 | INFO | } | 20260507 21:16:13.741 | INFO | Perf report for the last 5.13892 seconds: | 20260507 21:16:13.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:13.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:13.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:13.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:13.741 | INFO | ANALYSIS : 69.2%, 2.9, 105.0ms, 0 | 20260507 21:16:13.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:13.741 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260507 21:16:13.741 | INFO | PRE_HQ : 68.1%, 1.8, 63.0ms, 0 | 20260507 21:16:13.741 | INFO | BAZWRITER : 69.1%, 4.3, 151.0ms, 0 | 20260507 21:16:13.741 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:16:13.741 | INFO | Uploaded 36GiB to GPU in 2.59566s (13.8693GiB/s) | 20260507 21:16:13.741 | INFO | Downloaded 38GiB to GPU in 2.70197s (14.0638GiB/s) | 20260507 21:16:18.882 | INFO | Analyzing chunk frames = [2654720,2655232) | 20260507 21:16:18.882 | INFO | MemInfo: { | 20260507 21:16:18.882 | INFO | "gpu" : " 17 GB", | 20260507 21:16:18.882 | INFO | "huge" : "", | 20260507 21:16:18.882 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:18.882 | INFO | "malloc" : " 14 GB", | 20260507 21:16:18.882 | INFO | "pinned" : " 29 GB" | 20260507 21:16:18.882 | INFO | } | 20260507 21:16:18.882 | INFO | Perf report for the last 5.14126 seconds: | 20260507 21:16:18.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:18.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:16:18.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:16:18.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:18.882 | INFO | Uploaded 36GiB to GPU in 2.57914s (13.9581GiB/s) | 20260507 21:16:18.882 | INFO | Downloaded 34GiB to GPU in 2.49173s (13.6452GiB/s) | 20260507 21:16:24.020 | INFO | Analyzing chunk frames = [2655232,2655744) | 20260507 21:16:24.020 | INFO | MemInfo: { | 20260507 21:16:24.020 | INFO | "gpu" : " 17 GB", | 20260507 21:16:24.020 | INFO | "huge" : "", | 20260507 21:16:24.020 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:24.020 | INFO | "malloc" : " 14 GB", | 20260507 21:16:24.020 | INFO | "pinned" : " 29 GB" | 20260507 21:16:24.020 | INFO | } | 20260507 21:16:24.020 | INFO | Perf report for the last 5.13845 seconds: | 20260507 21:16:24.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:24.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.020 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:16:24.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.020 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:16:24.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:24.021 | INFO | Uploaded 36GiB to GPU in 2.52874s (14.2363GiB/s) | 20260507 21:16:24.021 | INFO | Downloaded 34GiB to GPU in 2.47177s (13.7553GiB/s) | 20260507 21:16:29.158 | INFO | Analyzing chunk frames = [2655744,2656256) | 20260507 21:16:29.158 | INFO | MemInfo: { | 20260507 21:16:29.158 | INFO | "gpu" : " 17 GB", | 20260507 21:16:29.158 | INFO | "huge" : "", | 20260507 21:16:29.158 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:29.158 | INFO | "malloc" : " 14 GB", | 20260507 21:16:29.159 | INFO | "pinned" : " 29 GB" | 20260507 21:16:29.159 | INFO | } | 20260507 21:16:29.159 | INFO | Perf report for the last 5.13833 seconds: | 20260507 21:16:29.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:29.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:16:29.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:16:29.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:29.159 | INFO | Uploaded 36GiB to GPU in 2.53351s (14.2095GiB/s) | 20260507 21:16:29.159 | INFO | Downloaded 34GiB to GPU in 2.4767s (13.728GiB/s) | 20260507 21:16:34.299 | INFO | Analyzing chunk frames = [2656256,2656768) | 20260507 21:16:34.299 | INFO | MemInfo: { | 20260507 21:16:34.299 | INFO | "gpu" : " 17 GB", | 20260507 21:16:34.299 | INFO | "huge" : "", | 20260507 21:16:34.299 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:34.299 | INFO | "malloc" : " 14 GB", | 20260507 21:16:34.299 | INFO | "pinned" : " 29 GB" | 20260507 21:16:34.299 | INFO | } | 20260507 21:16:34.299 | INFO | Perf report for the last 5.14017 seconds: | 20260507 21:16:34.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:34.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 21:16:34.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:16:34.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:34.299 | INFO | Uploaded 36GiB to GPU in 2.41117s (14.9305GiB/s) | 20260507 21:16:34.299 | INFO | Downloaded 34GiB to GPU in 2.49582s (13.6228GiB/s) | 20260507 21:16:39.437 | INFO | Analyzing chunk frames = [2656768,2657280) | 20260507 21:16:39.437 | INFO | MemInfo: { | 20260507 21:16:39.437 | INFO | "gpu" : " 17 GB", | 20260507 21:16:39.437 | INFO | "huge" : "", | 20260507 21:16:39.437 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:39.437 | INFO | "malloc" : " 14 GB", | 20260507 21:16:39.438 | INFO | "pinned" : " 29 GB" | 20260507 21:16:39.438 | INFO | } | 20260507 21:16:39.438 | INFO | Perf report for the last 5.13884 seconds: | 20260507 21:16:39.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:39.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:16:39.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:16:39.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:39.438 | INFO | Uploaded 36GiB to GPU in 2.403s (14.9813GiB/s) | 20260507 21:16:39.438 | INFO | Downloaded 34GiB to GPU in 2.39375s (14.2036GiB/s) | 20260507 21:16:44.577 | INFO | Analyzing chunk frames = [2657280,2657792) | 20260507 21:16:44.577 | INFO | MemInfo: { | 20260507 21:16:44.577 | INFO | "gpu" : " 17 GB", | 20260507 21:16:44.577 | INFO | "huge" : "", | 20260507 21:16:44.577 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:44.577 | INFO | "malloc" : " 14 GB", | 20260507 21:16:44.578 | INFO | "pinned" : " 29 GB" | 20260507 21:16:44.578 | INFO | } | 20260507 21:16:44.578 | INFO | Perf report for the last 5.13989 seconds: | 20260507 21:16:44.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:44.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:16:44.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:16:44.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:44.578 | INFO | Uploaded 36GiB to GPU in 2.52053s (14.2827GiB/s) | 20260507 21:16:44.578 | INFO | Downloaded 34GiB to GPU in 2.44802s (13.8888GiB/s) | 20260507 21:16:49.716 | INFO | Analyzing chunk frames = [2657792,2658304) | 20260507 21:16:49.716 | INFO | MemInfo: { | 20260507 21:16:49.716 | INFO | "gpu" : " 17 GB", | 20260507 21:16:49.716 | INFO | "huge" : "", | 20260507 21:16:49.716 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:49.716 | INFO | "malloc" : " 14 GB", | 20260507 21:16:49.716 | INFO | "pinned" : " 29 GB" | 20260507 21:16:49.716 | INFO | } | 20260507 21:16:49.716 | INFO | Perf report for the last 5.13879 seconds: | 20260507 21:16:49.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:49.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:16:49.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:16:49.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:49.716 | INFO | Uploaded 36GiB to GPU in 2.50434s (14.3751GiB/s) | 20260507 21:16:49.717 | INFO | Downloaded 34GiB to GPU in 2.4306s (13.9883GiB/s) | 20260507 21:16:49.814 | INFO | Starting a round of Baz IO | 20260507 21:16:53.288 | INFO | Populating RTMetrics report | 20260507 21:16:53.288 | INFO | RTMetrics report populated | 20260507 21:16:53.288 | INFO | Opening tmp RTMetrics json file | 20260507 21:16:53.291 | INFO | Serializing RTMetrics to json | 20260507 21:16:53.316 | INFO | Serialization done, swapping tmp file now... | 20260507 21:16:53.345 | INFO | RTMetrics done | 20260507 21:16:53.526 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.712 seconds | 20260507 21:16:53.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.84% / 0.01% / 56.15% | 20260507 21:16:53.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74115% / 1.19209e-05% / 3.97364e-05% | 20260507 21:16:53.527 | INFO | Baz IO overall: | 20260507 21:16:53.527 | INFO | BytesWritten : 1209845902169 | 20260507 21:16:53.527 | INFO | NumEvents : 586408218644 | 20260507 21:16:54.857 | INFO | Analyzing chunk frames = [2658304,2658816) | 20260507 21:16:54.857 | INFO | MemInfo: { | 20260507 21:16:54.857 | INFO | "gpu" : " 17 GB", | 20260507 21:16:54.857 | INFO | "huge" : "", | 20260507 21:16:54.857 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:54.857 | INFO | "malloc" : " 14 GB", | 20260507 21:16:54.857 | INFO | "pinned" : " 29 GB" | 20260507 21:16:54.857 | INFO | } | 20260507 21:16:54.857 | INFO | Perf report for the last 5.14101 seconds: | 20260507 21:16:54.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:54.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:54.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:54.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:54.857 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 21:16:54.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:54.857 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 21:16:54.858 | INFO | PRE_HQ : 66.0%, 1.7, 57.5ms, 0 | 20260507 21:16:54.858 | INFO | BAZWRITER : 71.8%, 4.8, 175.9ms, 0 | 20260507 21:16:54.858 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:16:54.858 | INFO | Uploaded 36GiB to GPU in 2.52055s (14.2826GiB/s) | 20260507 21:16:54.858 | INFO | Downloaded 38GiB to GPU in 2.69078s (14.1223GiB/s) | 20260507 21:16:59.996 | INFO | Analyzing chunk frames = [2658816,2659328) | 20260507 21:16:59.996 | INFO | MemInfo: { | 20260507 21:16:59.996 | INFO | "gpu" : " 17 GB", | 20260507 21:16:59.996 | INFO | "huge" : "", | 20260507 21:16:59.996 | INFO | "huge_shared" : " 29 GB", | 20260507 21:16:59.996 | INFO | "malloc" : " 14 GB", | 20260507 21:16:59.996 | INFO | "pinned" : " 29 GB" | 20260507 21:16:59.996 | INFO | } | 20260507 21:16:59.996 | INFO | Perf report for the last 5.13864 seconds: | 20260507 21:16:59.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:16:59.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:16:59.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:16:59.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:16:59.996 | INFO | Uploaded 36GiB to GPU in 2.5703s (14.0061GiB/s) | 20260507 21:16:59.996 | INFO | Downloaded 34GiB to GPU in 2.50371s (13.5798GiB/s) | 20260507 21:17:05.136 | INFO | Analyzing chunk frames = [2659328,2659840) | 20260507 21:17:05.136 | INFO | MemInfo: { | 20260507 21:17:05.136 | INFO | "gpu" : " 17 GB", | 20260507 21:17:05.136 | INFO | "huge" : "", | 20260507 21:17:05.136 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:05.136 | INFO | "malloc" : " 14 GB", | 20260507 21:17:05.136 | INFO | "pinned" : " 29 GB" | 20260507 21:17:05.136 | INFO | } | 20260507 21:17:05.136 | INFO | Perf report for the last 5.13987 seconds: | 20260507 21:17:05.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:05.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 21:17:05.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:17:05.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:05.136 | INFO | Uploaded 36GiB to GPU in 2.45939s (14.6378GiB/s) | 20260507 21:17:05.136 | INFO | Downloaded 34GiB to GPU in 2.47498s (13.7375GiB/s) | 20260507 21:17:10.273 | INFO | Analyzing chunk frames = [2659840,2660352) | 20260507 21:17:10.273 | INFO | MemInfo: { | 20260507 21:17:10.273 | INFO | "gpu" : " 17 GB", | 20260507 21:17:10.273 | INFO | "huge" : "", | 20260507 21:17:10.273 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:10.273 | INFO | "malloc" : " 14 GB", | 20260507 21:17:10.273 | INFO | "pinned" : " 29 GB" | 20260507 21:17:10.273 | INFO | } | 20260507 21:17:10.273 | INFO | Perf report for the last 5.13748 seconds: | 20260507 21:17:10.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:10.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.273 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 21:17:10.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.274 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:17:10.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:10.274 | INFO | Uploaded 36GiB to GPU in 2.48961s (14.4601GiB/s) | 20260507 21:17:10.274 | INFO | Downloaded 34GiB to GPU in 2.48049s (13.7069GiB/s) | 20260507 21:17:15.413 | INFO | Analyzing chunk frames = [2660352,2660864) | 20260507 21:17:15.413 | INFO | MemInfo: { | 20260507 21:17:15.413 | INFO | "gpu" : " 17 GB", | 20260507 21:17:15.413 | INFO | "huge" : "", | 20260507 21:17:15.413 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:15.413 | INFO | "malloc" : " 14 GB", | 20260507 21:17:15.413 | INFO | "pinned" : " 29 GB" | 20260507 21:17:15.413 | INFO | } | 20260507 21:17:15.413 | INFO | Perf report for the last 5.13951 seconds: | 20260507 21:17:15.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:15.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 21:17:15.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:17:15.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:15.413 | INFO | Uploaded 36GiB to GPU in 2.37208s (15.1766GiB/s) | 20260507 21:17:15.413 | INFO | Downloaded 34GiB to GPU in 2.50297s (13.5839GiB/s) | 20260507 21:17:20.552 | INFO | Analyzing chunk frames = [2660864,2661376) | 20260507 21:17:20.552 | INFO | MemInfo: { | 20260507 21:17:20.552 | INFO | "gpu" : " 17 GB", | 20260507 21:17:20.552 | INFO | "huge" : "", | 20260507 21:17:20.552 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:20.552 | INFO | "malloc" : " 14 GB", | 20260507 21:17:20.553 | INFO | "pinned" : " 29 GB" | 20260507 21:17:20.553 | INFO | } | 20260507 21:17:20.553 | INFO | Perf report for the last 5.13961 seconds: | 20260507 21:17:20.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:20.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:17:20.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 21:17:20.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:20.553 | INFO | Uploaded 36GiB to GPU in 2.39671s (15.0206GiB/s) | 20260507 21:17:20.553 | INFO | Downloaded 34GiB to GPU in 2.39711s (14.1837GiB/s) | 20260507 21:17:25.692 | INFO | Analyzing chunk frames = [2661376,2661888) | 20260507 21:17:25.692 | INFO | MemInfo: { | 20260507 21:17:25.692 | INFO | "gpu" : " 17 GB", | 20260507 21:17:25.692 | INFO | "huge" : "", | 20260507 21:17:25.692 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:25.692 | INFO | "malloc" : " 14 GB", | 20260507 21:17:25.692 | INFO | "pinned" : " 29 GB" | 20260507 21:17:25.692 | INFO | } | 20260507 21:17:25.693 | INFO | Perf report for the last 5.14 seconds: | 20260507 21:17:25.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:25.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:17:25.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | PRE_HQ : 37.3%, 1.0, 19.4ms, 0 | 20260507 21:17:25.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:25.693 | INFO | Uploaded 36GiB to GPU in 2.54954s (14.1202GiB/s) | 20260507 21:17:25.693 | INFO | Downloaded 34GiB to GPU in 2.46102s (13.8154GiB/s) | 20260507 21:17:30.831 | INFO | Analyzing chunk frames = [2661888,2662400) | 20260507 21:17:30.831 | INFO | MemInfo: { | 20260507 21:17:30.831 | INFO | "gpu" : " 17 GB", | 20260507 21:17:30.831 | INFO | "huge" : "", | 20260507 21:17:30.831 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:30.831 | INFO | "malloc" : " 14 GB", | 20260507 21:17:30.831 | INFO | "pinned" : " 29 GB" | 20260507 21:17:30.831 | INFO | } | 20260507 21:17:30.831 | INFO | Perf report for the last 5.13872 seconds: | 20260507 21:17:30.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:30.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.831 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:17:30.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.831 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 21:17:30.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:30.832 | INFO | Uploaded 36GiB to GPU in 2.51855s (14.294GiB/s) | 20260507 21:17:30.832 | INFO | Downloaded 34GiB to GPU in 2.44413s (13.9109GiB/s) | 20260507 21:17:30.832 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:17:30.832 | INFO | Total mem usage: 16.7 GiB | 20260507 21:17:30.832 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:17:30.832 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -3.24%) | 20260507 21:17:30.832 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:17:30.832 | INFO | High Water Breakdown: | 20260507 21:17:30.832 | INFO | 61.3% : 9.9 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:17:30.832 | INFO | 38.6% : 6.24 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:17:30.832 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:17:30.832 | INFO | ------------------------------------------ | 20260507 21:17:30.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:17:30.832 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:17:30.832 | INFO | Total mem usage: 26.5 GiB | 20260507 21:17:30.832 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:17:30.832 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:17:30.832 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:17:30.832 | INFO | High Water Breakdown: | 20260507 21:17:30.832 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:17:30.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:17:30.832 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:17:30.832 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:17:30.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:17:30.832 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:17:30.832 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:17:30.832 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:17:30.832 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:17:30.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:17:30.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:17:30.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:17:30.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:17:30.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:17:30.832 | INFO | ------------------------------------------ | 20260507 21:17:30.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:17:30.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:17:30.832 | INFO | Total mem usage: 27.8 GiB | 20260507 21:17:30.832 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:17:30.832 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:17:30.832 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:17:30.832 | INFO | High Water Breakdown: | 20260507 21:17:30.832 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:17:30.832 | INFO | ------------------------------------------ | 20260507 21:17:30.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:17:30.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:17:30.832 | INFO | Total mem usage: 15.7 GiB | 20260507 21:17:30.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:17:30.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:17:30.832 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 21:17:30.832 | INFO | High Water Breakdown: | 20260507 21:17:30.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:17:30.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:17:30.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:17:30.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:17:30.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:17:30.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:17:30.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:17:30.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:17:30.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:17:30.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:17:30.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:17:30.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:17:30.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:17:30.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:17:30.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:17:30.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:17:30.833 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:17:30.833 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:17:30.833 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:17:30.833 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:17:30.833 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:17:30.833 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:17:30.833 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:17:30.833 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:17:30.833 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:17:30.833 | INFO | ------------------------------------------ | 20260507 21:17:30.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:17:30.833 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:17:30.833 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:17:30.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:17:30.833 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.43e+01 / 1.86e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.02e-01, 1.18e-02 / 5.51e-03 | 20260507 21:17:30.833 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.50e+00, 5.95e-01 / 1.61e-01 | 20260507 21:17:30.833 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.11e-01 / 1.17e-01 | 20260507 21:17:30.833 | INFO | PerfReport: 16.13%, DME -- 4.26e+01 / 4.62e+01, 4.21e+00 / 1.89e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 55.76%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.56e+00 / 1.26e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.52e+00 / 2.84e+00, 4.40e-01 / 2.15e-01 | 20260507 21:17:30.833 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.37e+00, 2.90e-01 / 1.28e-01 | 20260507 21:17:30.833 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.65e-01 / 4.04e-01, 5.93e-02 / 3.49e-02 | 20260507 21:17:30.833 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:17:30.833 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:17:30.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:17:30.833 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 24.34%, Upload -- 2.60e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 24.29%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 30.88%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 2.15%, LockUpload -- 2.30e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 2.66%, LockDownload -- 2.84e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:17:30.833 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 21:17:30.928 | INFO | Starting a round of Baz IO | 20260507 21:17:34.410 | INFO | Populating RTMetrics report | 20260507 21:17:34.411 | INFO | RTMetrics report populated | 20260507 21:17:34.411 | INFO | Opening tmp RTMetrics json file | 20260507 21:17:34.432 | INFO | Serializing RTMetrics to json | 20260507 21:17:34.440 | INFO | Serialization done, swapping tmp file now... | 20260507 21:17:34.441 | INFO | RTMetrics done | 20260507 21:17:34.506 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.577 seconds | 20260507 21:17:34.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.83% / 0.01% / 56.16% | 20260507 21:17:34.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7368% / 7.94729e-06% / 6.35783e-05% | 20260507 21:17:34.506 | INFO | Baz IO overall: | 20260507 21:17:34.506 | INFO | BytesWritten : 1211737521445 | 20260507 21:17:34.506 | INFO | NumEvents : 587322953785 | 20260507 21:17:35.970 | INFO | Analyzing chunk frames = [2662400,2662912) | 20260507 21:17:35.970 | INFO | MemInfo: { | 20260507 21:17:35.970 | INFO | "gpu" : " 17 GB", | 20260507 21:17:35.970 | INFO | "huge" : "", | 20260507 21:17:35.970 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:35.970 | INFO | "malloc" : " 14 GB", | 20260507 21:17:35.970 | INFO | "pinned" : " 29 GB" | 20260507 21:17:35.970 | INFO | } | 20260507 21:17:35.970 | INFO | Perf report for the last 5.13896 seconds: | 20260507 21:17:35.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:35.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:35.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:35.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:35.970 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 21:17:35.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:35.970 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 21:17:35.970 | INFO | PRE_HQ : 65.9%, 1.7, 57.5ms, 0 | 20260507 21:17:35.970 | INFO | BAZWRITER : 69.2%, 4.1, 147.6ms, 0 | 20260507 21:17:35.970 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 21:17:35.971 | INFO | Uploaded 36GiB to GPU in 2.53025s (14.2279GiB/s) | 20260507 21:17:35.971 | INFO | Downloaded 38GiB to GPU in 2.68203s (14.1684GiB/s) | 20260507 21:17:41.110 | INFO | Analyzing chunk frames = [2662912,2663424) | 20260507 21:17:41.110 | INFO | MemInfo: { | 20260507 21:17:41.110 | INFO | "gpu" : " 17 GB", | 20260507 21:17:41.110 | INFO | "huge" : "", | 20260507 21:17:41.110 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:41.110 | INFO | "malloc" : " 14 GB", | 20260507 21:17:41.110 | INFO | "pinned" : " 29 GB" | 20260507 21:17:41.110 | INFO | } | 20260507 21:17:41.111 | INFO | Perf report for the last 5.14028 seconds: | 20260507 21:17:41.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:41.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:17:41.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 21:17:41.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:41.111 | INFO | Uploaded 36GiB to GPU in 2.56689s (14.0248GiB/s) | 20260507 21:17:41.111 | INFO | Downloaded 34GiB to GPU in 2.50159s (13.5914GiB/s) | 20260507 21:17:46.250 | INFO | Analyzing chunk frames = [2663424,2663936) | 20260507 21:17:46.250 | INFO | MemInfo: { | 20260507 21:17:46.250 | INFO | "gpu" : " 17 GB", | 20260507 21:17:46.250 | INFO | "huge" : "", | 20260507 21:17:46.250 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:46.250 | INFO | "malloc" : " 14 GB", | 20260507 21:17:46.250 | INFO | "pinned" : " 29 GB" | 20260507 21:17:46.250 | INFO | } | 20260507 21:17:46.250 | INFO | Perf report for the last 5.13946 seconds: | 20260507 21:17:46.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:46.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:17:46.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:17:46.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:46.250 | INFO | Uploaded 36GiB to GPU in 2.49064s (14.4541GiB/s) | 20260507 21:17:46.250 | INFO | Downloaded 34GiB to GPU in 2.47891s (13.7157GiB/s) | 20260507 21:17:51.389 | INFO | Analyzing chunk frames = [2663936,2664448) | 20260507 21:17:51.389 | INFO | MemInfo: { | 20260507 21:17:51.389 | INFO | "gpu" : " 17 GB", | 20260507 21:17:51.389 | INFO | "huge" : "", | 20260507 21:17:51.389 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:51.389 | INFO | "malloc" : " 14 GB", | 20260507 21:17:51.389 | INFO | "pinned" : " 29 GB" | 20260507 21:17:51.389 | INFO | } | 20260507 21:17:51.389 | INFO | Perf report for the last 5.13896 seconds: | 20260507 21:17:51.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:51.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 21:17:51.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:17:51.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:51.389 | INFO | Uploaded 36GiB to GPU in 2.49425s (14.4332GiB/s) | 20260507 21:17:51.389 | INFO | Downloaded 34GiB to GPU in 2.48386s (13.6884GiB/s) | 20260507 21:17:56.528 | INFO | Analyzing chunk frames = [2664448,2664960) | 20260507 21:17:56.528 | INFO | MemInfo: { | 20260507 21:17:56.528 | INFO | "gpu" : " 17 GB", | 20260507 21:17:56.528 | INFO | "huge" : "", | 20260507 21:17:56.528 | INFO | "huge_shared" : " 29 GB", | 20260507 21:17:56.529 | INFO | "malloc" : " 14 GB", | 20260507 21:17:56.529 | INFO | "pinned" : " 29 GB" | 20260507 21:17:56.529 | INFO | } | 20260507 21:17:56.529 | INFO | Perf report for the last 5.1397 seconds: | 20260507 21:17:56.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:17:56.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 21:17:56.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:17:56.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:17:56.529 | INFO | Uploaded 36GiB to GPU in 2.41669s (14.8964GiB/s) | 20260507 21:17:56.529 | INFO | Downloaded 34GiB to GPU in 2.49064s (13.6511GiB/s) | 20260507 21:18:01.667 | INFO | Analyzing chunk frames = [2664960,2665472) | 20260507 21:18:01.667 | INFO | MemInfo: { | 20260507 21:18:01.667 | INFO | "gpu" : " 17 GB", | 20260507 21:18:01.667 | INFO | "huge" : "", | 20260507 21:18:01.667 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:01.667 | INFO | "malloc" : " 14 GB", | 20260507 21:18:01.667 | INFO | "pinned" : " 29 GB" | 20260507 21:18:01.667 | INFO | } | 20260507 21:18:01.667 | INFO | Perf report for the last 5.1386 seconds: | 20260507 21:18:01.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:01.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.667 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:18:01.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.667 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:18:01.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:01.668 | INFO | Uploaded 36GiB to GPU in 2.40266s (14.9834GiB/s) | 20260507 21:18:01.668 | INFO | Downloaded 34GiB to GPU in 2.38908s (14.2314GiB/s) | 20260507 21:18:06.806 | INFO | Analyzing chunk frames = [2665472,2665984) | 20260507 21:18:06.806 | INFO | MemInfo: { | 20260507 21:18:06.806 | INFO | "gpu" : " 17 GB", | 20260507 21:18:06.806 | INFO | "huge" : "", | 20260507 21:18:06.806 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:06.806 | INFO | "malloc" : " 14 GB", | 20260507 21:18:06.807 | INFO | "pinned" : " 29 GB" | 20260507 21:18:06.807 | INFO | } | 20260507 21:18:06.807 | INFO | Perf report for the last 5.13934 seconds: | 20260507 21:18:06.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:06.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 21:18:06.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 21:18:06.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:06.807 | INFO | Uploaded 36GiB to GPU in 2.52512s (14.2568GiB/s) | 20260507 21:18:06.807 | INFO | Downloaded 34GiB to GPU in 2.44493s (13.9063GiB/s) | 20260507 21:18:11.945 | INFO | Analyzing chunk frames = [2665984,2666496) | 20260507 21:18:11.945 | INFO | MemInfo: { | 20260507 21:18:11.945 | INFO | "gpu" : " 17 GB", | 20260507 21:18:11.945 | INFO | "huge" : "", | 20260507 21:18:11.945 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:11.945 | INFO | "malloc" : " 14 GB", | 20260507 21:18:11.945 | INFO | "pinned" : " 29 GB" | 20260507 21:18:11.945 | INFO | } | 20260507 21:18:11.945 | INFO | Perf report for the last 5.13878 seconds: | 20260507 21:18:11.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:11.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:18:11.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:18:11.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:11.946 | INFO | Uploaded 36GiB to GPU in 2.50096s (14.3945GiB/s) | 20260507 21:18:11.946 | INFO | Downloaded 34GiB to GPU in 2.43558s (13.9597GiB/s) | 20260507 21:18:12.042 | INFO | Starting a round of Baz IO | 20260507 21:18:15.534 | INFO | Populating RTMetrics report | 20260507 21:18:15.534 | INFO | RTMetrics report populated | 20260507 21:18:15.534 | INFO | Opening tmp RTMetrics json file | 20260507 21:18:15.556 | INFO | Serializing RTMetrics to json | 20260507 21:18:15.558 | INFO | Serialization done, swapping tmp file now... | 20260507 21:18:15.559 | INFO | RTMetrics done | 20260507 21:18:15.631 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.588 seconds | 20260507 21:18:15.631 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.83% / 0.01% / 56.17% | 20260507 21:18:15.631 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7321% / 3.97364e-06% / 7.15256e-05% | 20260507 21:18:15.631 | INFO | Baz IO overall: | 20260507 21:18:15.631 | INFO | BytesWritten : 1213629704788 | 20260507 21:18:15.631 | INFO | NumEvents : 588237736399 | 20260507 21:18:17.085 | INFO | Analyzing chunk frames = [2666496,2667008) | 20260507 21:18:17.085 | INFO | MemInfo: { | 20260507 21:18:17.085 | INFO | "gpu" : " 17 GB", | 20260507 21:18:17.085 | INFO | "huge" : "", | 20260507 21:18:17.085 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:17.085 | INFO | "malloc" : " 14 GB", | 20260507 21:18:17.085 | INFO | "pinned" : " 29 GB" | 20260507 21:18:17.085 | INFO | } | 20260507 21:18:17.085 | INFO | Perf report for the last 5.13927 seconds: | 20260507 21:18:17.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:17.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:17.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:17.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:17.085 | INFO | ANALYSIS : 69.1%, 3.0, 107.0ms, 0 | 20260507 21:18:17.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:17.085 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 21:18:17.085 | INFO | PRE_HQ : 66.5%, 1.7, 57.6ms, 0 | 20260507 21:18:17.085 | INFO | BAZWRITER : 69.4%, 3.8, 136.5ms, 0 | 20260507 21:18:17.085 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 21:18:17.086 | INFO | Uploaded 36GiB to GPU in 2.55724s (14.0777GiB/s) | 20260507 21:18:17.086 | INFO | Downloaded 38GiB to GPU in 2.72566s (13.9416GiB/s) | 20260507 21:18:22.224 | INFO | Analyzing chunk frames = [2667008,2667520) | 20260507 21:18:22.224 | INFO | MemInfo: { | 20260507 21:18:22.224 | INFO | "gpu" : " 17 GB", | 20260507 21:18:22.224 | INFO | "huge" : "", | 20260507 21:18:22.224 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:22.224 | INFO | "malloc" : " 14 GB", | 20260507 21:18:22.224 | INFO | "pinned" : " 29 GB" | 20260507 21:18:22.225 | INFO | } | 20260507 21:18:22.225 | INFO | Perf report for the last 5.13986 seconds: | 20260507 21:18:22.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:22.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 21:18:22.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 21:18:22.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:22.225 | INFO | Uploaded 36GiB to GPU in 2.53751s (14.1871GiB/s) | 20260507 21:18:22.225 | INFO | Downloaded 34GiB to GPU in 2.50143s (13.5922GiB/s) | 20260507 21:18:27.364 | INFO | Analyzing chunk frames = [2667520,2668032) | 20260507 21:18:27.364 | INFO | MemInfo: { | 20260507 21:18:27.364 | INFO | "gpu" : " 17 GB", | 20260507 21:18:27.364 | INFO | "huge" : "", | 20260507 21:18:27.364 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:27.364 | INFO | "malloc" : " 14 GB", | 20260507 21:18:27.364 | INFO | "pinned" : " 29 GB" | 20260507 21:18:27.364 | INFO | } | 20260507 21:18:27.364 | INFO | Perf report for the last 5.13975 seconds: | 20260507 21:18:27.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:27.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.364 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:18:27.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.365 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:18:27.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:27.365 | INFO | Uploaded 36GiB to GPU in 2.46876s (14.5822GiB/s) | 20260507 21:18:27.365 | INFO | Downloaded 34GiB to GPU in 2.48574s (13.678GiB/s) | 20260507 21:18:32.504 | INFO | Analyzing chunk frames = [2668032,2668544) | 20260507 21:18:32.504 | INFO | MemInfo: { | 20260507 21:18:32.504 | INFO | "gpu" : " 17 GB", | 20260507 21:18:32.504 | INFO | "huge" : "", | 20260507 21:18:32.504 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:32.504 | INFO | "malloc" : " 14 GB", | 20260507 21:18:32.504 | INFO | "pinned" : " 29 GB" | 20260507 21:18:32.504 | INFO | } | 20260507 21:18:32.504 | INFO | Perf report for the last 5.14006 seconds: | 20260507 21:18:32.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:32.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 21:18:32.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:18:32.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:32.505 | INFO | Uploaded 36GiB to GPU in 2.46543s (14.6019GiB/s) | 20260507 21:18:32.505 | INFO | Downloaded 34GiB to GPU in 2.4874s (13.6689GiB/s) | 20260507 21:18:37.643 | INFO | Analyzing chunk frames = [2668544,2669056) | 20260507 21:18:37.643 | INFO | MemInfo: { | 20260507 21:18:37.643 | INFO | "gpu" : " 17 GB", | 20260507 21:18:37.643 | INFO | "huge" : "", | 20260507 21:18:37.643 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:37.643 | INFO | "malloc" : " 14 GB", | 20260507 21:18:37.644 | INFO | "pinned" : " 29 GB" | 20260507 21:18:37.644 | INFO | } | 20260507 21:18:37.644 | INFO | Perf report for the last 5.13932 seconds: | 20260507 21:18:37.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:37.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:18:37.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:18:37.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:37.644 | INFO | Uploaded 36GiB to GPU in 2.35822s (15.2657GiB/s) | 20260507 21:18:37.644 | INFO | Downloaded 34GiB to GPU in 2.47153s (13.7567GiB/s) | 20260507 21:18:42.782 | INFO | Analyzing chunk frames = [2669056,2669568) | 20260507 21:18:42.782 | INFO | MemInfo: { | 20260507 21:18:42.782 | INFO | "gpu" : " 17 GB", | 20260507 21:18:42.782 | INFO | "huge" : "", | 20260507 21:18:42.782 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:42.782 | INFO | "malloc" : " 14 GB", | 20260507 21:18:42.782 | INFO | "pinned" : " 29 GB" | 20260507 21:18:42.782 | INFO | } | 20260507 21:18:42.782 | INFO | Perf report for the last 5.13805 seconds: | 20260507 21:18:42.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:42.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:18:42.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:18:42.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:42.782 | INFO | Uploaded 36GiB to GPU in 2.40651s (14.9594GiB/s) | 20260507 21:18:42.782 | INFO | Downloaded 34GiB to GPU in 2.39158s (14.2165GiB/s) | 20260507 21:18:47.920 | INFO | Analyzing chunk frames = [2669568,2670080) | 20260507 21:18:47.920 | INFO | MemInfo: { | 20260507 21:18:47.920 | INFO | "gpu" : " 17 GB", | 20260507 21:18:47.920 | INFO | "huge" : "", | 20260507 21:18:47.920 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:47.920 | INFO | "malloc" : " 14 GB", | 20260507 21:18:47.920 | INFO | "pinned" : " 29 GB" | 20260507 21:18:47.921 | INFO | } | 20260507 21:18:47.921 | INFO | Perf report for the last 5.13866 seconds: | 20260507 21:18:47.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:47.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:18:47.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:18:47.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:47.921 | INFO | Uploaded 36GiB to GPU in 2.55139s (14.11GiB/s) | 20260507 21:18:47.921 | INFO | Downloaded 34GiB to GPU in 2.44655s (13.8971GiB/s) | 20260507 21:18:53.060 | INFO | Analyzing chunk frames = [2670080,2670592) | 20260507 21:18:53.060 | INFO | MemInfo: { | 20260507 21:18:53.060 | INFO | "gpu" : " 17 GB", | 20260507 21:18:53.060 | INFO | "huge" : "", | 20260507 21:18:53.060 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:53.060 | INFO | "malloc" : " 14 GB", | 20260507 21:18:53.060 | INFO | "pinned" : " 29 GB" | 20260507 21:18:53.060 | INFO | } | 20260507 21:18:53.061 | INFO | Perf report for the last 5.13998 seconds: | 20260507 21:18:53.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:53.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:18:53.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:18:53.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:53.061 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:18:53.061 | INFO | Total mem usage: 16.7 GiB | 20260507 21:18:53.061 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:18:53.061 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.7%) | 20260507 21:18:53.061 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:18:53.061 | INFO | High Water Breakdown: | 20260507 21:18:53.061 | INFO | 60.2% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:18:53.061 | INFO | 39.7% : 5.98 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:18:53.061 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:18:53.061 | INFO | ------------------------------------------ | 20260507 21:18:53.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:18:53.061 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:18:53.061 | INFO | Total mem usage: 26.5 GiB | 20260507 21:18:53.061 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:18:53.061 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:18:53.061 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:18:53.061 | INFO | High Water Breakdown: | 20260507 21:18:53.061 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:18:53.061 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:18:53.061 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:18:53.061 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:18:53.061 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:18:53.061 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:18:53.061 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:18:53.061 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:18:53.061 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:18:53.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:18:53.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:18:53.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:18:53.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:18:53.061 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:18:53.061 | INFO | ------------------------------------------ | 20260507 21:18:53.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:18:53.061 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:18:53.061 | INFO | Total mem usage: 27.8 GiB | 20260507 21:18:53.061 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:18:53.061 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:18:53.061 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:18:53.061 | INFO | High Water Breakdown: | 20260507 21:18:53.061 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:18:53.061 | INFO | ------------------------------------------ | 20260507 21:18:53.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:18:53.061 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:18:53.061 | INFO | Total mem usage: 15.7 GiB | 20260507 21:18:53.061 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:18:53.061 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:18:53.061 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:18:53.061 | INFO | High Water Breakdown: | 20260507 21:18:53.061 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:18:53.061 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:18:53.061 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:18:53.061 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:18:53.061 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:18:53.061 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:18:53.061 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:18:53.061 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:18:53.061 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:18:53.061 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:18:53.061 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:18:53.061 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:18:53.061 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:18:53.061 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:18:53.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:18:53.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:18:53.061 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:18:53.061 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:18:53.061 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:18:53.061 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:18:53.061 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:18:53.061 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:18:53.061 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:18:53.061 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:18:53.061 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:18:53.061 | INFO | ------------------------------------------ | 20260507 21:18:53.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:18:53.061 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:18:53.061 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:18:53.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:18:53.061 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.41e+01 / 2.01e+00 | 20260507 21:18:53.061 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.23e-02 / 5.10e-03 | 20260507 21:18:53.061 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.50e+00, 5.96e-01 / 1.64e-01 | 20260507 21:18:53.061 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.51e+00, 2.07e-01 / 9.50e-02 | 20260507 21:18:53.061 | INFO | PerfReport: 16.13%, DME -- 4.26e+01 / 4.63e+01, 4.14e+00 / 2.19e+00 | 20260507 21:18:53.061 | INFO | PerfReport: 55.78%, FrameLabeling -- 1.84e+01 / 1.92e+01, 1.56e+00 / 1.21e+00 | 20260507 21:18:53.061 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.51e+00 / 2.87e+00, 4.38e-01 / 1.77e-01 | 20260507 21:18:53.061 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.40e+00, 2.94e-01 / 1.63e-01 | 20260507 21:18:53.061 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.62e-01 / 3.98e-01, 5.90e-02 / 2.66e-02 | 20260507 21:18:53.061 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:18:53.061 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:18:53.061 | INFO | Uploaded 36GiB to GPU in 2.52743s (14.2437GiB/s) | 20260507 21:18:53.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:18:53.062 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260507 21:18:53.062 | INFO | Downloaded 34GiB to GPU in 2.4261s (14.0142GiB/s) | 20260507 21:18:53.062 | INFO | PerfReport: 24.22%, Upload -- 2.59e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260507 21:18:53.062 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:18:53.062 | INFO | PerfReport: 30.85%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:18:53.062 | INFO | PerfReport: 2.07%, LockUpload -- 2.22e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260507 21:18:53.062 | INFO | PerfReport: 2.78%, LockDownload -- 2.97e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260507 21:18:53.062 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 21:18:53.062 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260507 21:18:53.159 | INFO | Starting a round of Baz IO | 20260507 21:18:56.639 | INFO | Populating RTMetrics report | 20260507 21:18:56.639 | INFO | RTMetrics report populated | 20260507 21:18:56.639 | INFO | Opening tmp RTMetrics json file | 20260507 21:18:56.668 | INFO | Serializing RTMetrics to json | 20260507 21:18:56.689 | INFO | Serialization done, swapping tmp file now... | 20260507 21:18:56.730 | INFO | RTMetrics done | 20260507 21:18:56.771 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.611 seconds | 20260507 21:18:56.771 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.82% / 0.01% / 56.17% | 20260507 21:18:56.771 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.72971% / 0% / 3.97364e-05% | 20260507 21:18:56.771 | INFO | Baz IO overall: | 20260507 21:18:56.771 | INFO | BytesWritten : 1215522157714 | 20260507 21:18:56.771 | INFO | NumEvents : 589153064473 | 20260507 21:18:58.200 | INFO | Analyzing chunk frames = [2670592,2671104) | 20260507 21:18:58.200 | INFO | MemInfo: { | 20260507 21:18:58.200 | INFO | "gpu" : " 17 GB", | 20260507 21:18:58.200 | INFO | "huge" : "", | 20260507 21:18:58.200 | INFO | "huge_shared" : " 29 GB", | 20260507 21:18:58.200 | INFO | "malloc" : " 14 GB", | 20260507 21:18:58.200 | INFO | "pinned" : " 29 GB" | 20260507 21:18:58.200 | INFO | } | 20260507 21:18:58.200 | INFO | Perf report for the last 5.13971 seconds: | 20260507 21:18:58.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:18:58.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:58.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:58.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:58.200 | INFO | ANALYSIS : 68.9%, 3.0, 107.0ms, 0 | 20260507 21:18:58.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:18:58.200 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 21:18:58.200 | INFO | PRE_HQ : 66.0%, 1.7, 57.3ms, 0 | 20260507 21:18:58.201 | INFO | BAZWRITER : 69.7%, 4.0, 142.9ms, 0 | 20260507 21:18:58.201 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 21:18:58.201 | INFO | Uploaded 36GiB to GPU in 2.57957s (13.9558GiB/s) | 20260507 21:18:58.201 | INFO | Downloaded 38GiB to GPU in 2.69321s (14.1096GiB/s) | 20260507 21:19:03.340 | INFO | Analyzing chunk frames = [2671104,2671616) | 20260507 21:19:03.340 | INFO | MemInfo: { | 20260507 21:19:03.340 | INFO | "gpu" : " 17 GB", | 20260507 21:19:03.340 | INFO | "huge" : "", | 20260507 21:19:03.340 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:03.340 | INFO | "malloc" : " 14 GB", | 20260507 21:19:03.340 | INFO | "pinned" : " 29 GB" | 20260507 21:19:03.340 | INFO | } | 20260507 21:19:03.340 | INFO | Perf report for the last 5.14012 seconds: | 20260507 21:19:03.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:03.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.340 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 21:19:03.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.341 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:19:03.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:03.341 | INFO | Uploaded 36GiB to GPU in 2.58705s (13.9155GiB/s) | 20260507 21:19:03.341 | INFO | Downloaded 34GiB to GPU in 2.47633s (13.73GiB/s) | 20260507 21:19:08.478 | INFO | Analyzing chunk frames = [2671616,2672128) | 20260507 21:19:08.479 | INFO | MemInfo: { | 20260507 21:19:08.479 | INFO | "gpu" : " 17 GB", | 20260507 21:19:08.479 | INFO | "huge" : "", | 20260507 21:19:08.479 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:08.479 | INFO | "malloc" : " 14 GB", | 20260507 21:19:08.479 | INFO | "pinned" : " 29 GB" | 20260507 21:19:08.479 | INFO | } | 20260507 21:19:08.479 | INFO | Perf report for the last 5.13823 seconds: | 20260507 21:19:08.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:08.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260507 21:19:08.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:19:08.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:08.479 | INFO | Uploaded 36GiB to GPU in 2.53364s (14.2088GiB/s) | 20260507 21:19:08.479 | INFO | Downloaded 34GiB to GPU in 2.46657s (13.7843GiB/s) | 20260507 21:19:13.618 | INFO | Analyzing chunk frames = [2672128,2672640) | 20260507 21:19:13.618 | INFO | MemInfo: { | 20260507 21:19:13.618 | INFO | "gpu" : " 17 GB", | 20260507 21:19:13.618 | INFO | "huge" : "", | 20260507 21:19:13.618 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:13.618 | INFO | "malloc" : " 14 GB", | 20260507 21:19:13.618 | INFO | "pinned" : " 29 GB" | 20260507 21:19:13.618 | INFO | } | 20260507 21:19:13.618 | INFO | Perf report for the last 5.13922 seconds: | 20260507 21:19:13.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:13.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.618 | INFO | ANALYSIS : 67.6%, 3.0, 104.5ms, 0 | 20260507 21:19:13.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.618 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 21:19:13.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:13.619 | INFO | Uploaded 36GiB to GPU in 2.52003s (14.2855GiB/s) | 20260507 21:19:13.619 | INFO | Downloaded 34GiB to GPU in 2.4734s (13.7462GiB/s) | 20260507 21:19:18.758 | INFO | Analyzing chunk frames = [2672640,2673152) | 20260507 21:19:18.758 | INFO | MemInfo: { | 20260507 21:19:18.758 | INFO | "gpu" : " 17 GB", | 20260507 21:19:18.758 | INFO | "huge" : "", | 20260507 21:19:18.758 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:18.758 | INFO | "malloc" : " 14 GB", | 20260507 21:19:18.758 | INFO | "pinned" : " 29 GB" | 20260507 21:19:18.758 | INFO | } | 20260507 21:19:18.758 | INFO | Perf report for the last 5.14012 seconds: | 20260507 21:19:18.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:18.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.758 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:19:18.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.758 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 21:19:18.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:18.759 | INFO | Uploaded 36GiB to GPU in 2.39172s (15.0519GiB/s) | 20260507 21:19:18.759 | INFO | Downloaded 34GiB to GPU in 2.48988s (13.6553GiB/s) | 20260507 21:19:23.897 | INFO | Analyzing chunk frames = [2673152,2673664) | 20260507 21:19:23.897 | INFO | MemInfo: { | 20260507 21:19:23.897 | INFO | "gpu" : " 17 GB", | 20260507 21:19:23.897 | INFO | "huge" : "", | 20260507 21:19:23.897 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:23.897 | INFO | "malloc" : " 14 GB", | 20260507 21:19:23.897 | INFO | "pinned" : " 29 GB" | 20260507 21:19:23.897 | INFO | } | 20260507 21:19:23.897 | INFO | Perf report for the last 5.13933 seconds: | 20260507 21:19:23.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:23.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.897 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260507 21:19:23.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.897 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:19:23.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:23.898 | INFO | Uploaded 36GiB to GPU in 2.38998s (15.0629GiB/s) | 20260507 21:19:23.898 | INFO | Downloaded 34GiB to GPU in 2.37686s (14.3046GiB/s) | 20260507 21:19:29.037 | INFO | Analyzing chunk frames = [2673664,2674176) | 20260507 21:19:29.037 | INFO | MemInfo: { | 20260507 21:19:29.037 | INFO | "gpu" : " 17 GB", | 20260507 21:19:29.037 | INFO | "huge" : "", | 20260507 21:19:29.037 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:29.037 | INFO | "malloc" : " 14 GB", | 20260507 21:19:29.037 | INFO | "pinned" : " 29 GB" | 20260507 21:19:29.037 | INFO | } | 20260507 21:19:29.037 | INFO | Perf report for the last 5.13954 seconds: | 20260507 21:19:29.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:29.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 21:19:29.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:19:29.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:29.037 | INFO | Uploaded 36GiB to GPU in 2.51581s (14.3095GiB/s) | 20260507 21:19:29.037 | INFO | Downloaded 34GiB to GPU in 2.4376s (13.9482GiB/s) | 20260507 21:19:34.175 | INFO | Analyzing chunk frames = [2674176,2674688) | 20260507 21:19:34.175 | INFO | MemInfo: { | 20260507 21:19:34.175 | INFO | "gpu" : " 17 GB", | 20260507 21:19:34.175 | INFO | "huge" : "", | 20260507 21:19:34.175 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:34.175 | INFO | "malloc" : " 14 GB", | 20260507 21:19:34.176 | INFO | "pinned" : " 29 GB" | 20260507 21:19:34.176 | INFO | } | 20260507 21:19:34.176 | INFO | Perf report for the last 5.13872 seconds: | 20260507 21:19:34.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:34.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:19:34.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:19:34.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:34.176 | INFO | Uploaded 36GiB to GPU in 2.53917s (14.1779GiB/s) | 20260507 21:19:34.176 | INFO | Downloaded 34GiB to GPU in 2.42051s (14.0467GiB/s) | 20260507 21:19:34.273 | INFO | Starting a round of Baz IO | 20260507 21:19:37.739 | INFO | Populating RTMetrics report | 20260507 21:19:37.739 | INFO | RTMetrics report populated | 20260507 21:19:37.739 | INFO | Opening tmp RTMetrics json file | 20260507 21:19:37.770 | INFO | Serializing RTMetrics to json | 20260507 21:19:37.810 | INFO | Serialization done, swapping tmp file now... | 20260507 21:19:37.831 | INFO | RTMetrics done | 20260507 21:19:37.859 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.586 seconds | 20260507 21:19:37.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.81% / 0.01% / 56.18% | 20260507 21:19:37.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.72806% / 0% / 3.57628e-05% | 20260507 21:19:37.859 | INFO | Baz IO overall: | 20260507 21:19:37.859 | INFO | BytesWritten : 1217415829482 | 20260507 21:19:37.859 | INFO | NumEvents : 590069405825 | 20260507 21:19:39.314 | INFO | Analyzing chunk frames = [2674688,2675200) | 20260507 21:19:39.314 | INFO | MemInfo: { | 20260507 21:19:39.314 | INFO | "gpu" : " 17 GB", | 20260507 21:19:39.314 | INFO | "huge" : "", | 20260507 21:19:39.314 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:39.314 | INFO | "malloc" : " 14 GB", | 20260507 21:19:39.314 | INFO | "pinned" : " 29 GB" | 20260507 21:19:39.314 | INFO | } | 20260507 21:19:39.315 | INFO | Perf report for the last 5.13887 seconds: | 20260507 21:19:39.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:39.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:39.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:39.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:39.315 | INFO | ANALYSIS : 68.6%, 3.0, 107.0ms, 0 | 20260507 21:19:39.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:39.315 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 21:19:39.315 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260507 21:19:39.315 | INFO | BAZWRITER : 69.4%, 3.8, 137.0ms, 0 | 20260507 21:19:39.315 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 21:19:39.315 | INFO | Uploaded 36GiB to GPU in 2.557s (14.079GiB/s) | 20260507 21:19:39.315 | INFO | Downloaded 38GiB to GPU in 2.68741s (14.14GiB/s) | 20260507 21:19:44.453 | INFO | Analyzing chunk frames = [2675200,2675712) | 20260507 21:19:44.453 | INFO | MemInfo: { | 20260507 21:19:44.454 | INFO | "gpu" : " 17 GB", | 20260507 21:19:44.454 | INFO | "huge" : "", | 20260507 21:19:44.454 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:44.454 | INFO | "malloc" : " 14 GB", | 20260507 21:19:44.454 | INFO | "pinned" : " 29 GB" | 20260507 21:19:44.454 | INFO | } | 20260507 21:19:44.454 | INFO | Perf report for the last 5.13919 seconds: | 20260507 21:19:44.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:44.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 21:19:44.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:19:44.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:44.454 | INFO | Uploaded 36GiB to GPU in 2.58457s (13.9288GiB/s) | 20260507 21:19:44.454 | INFO | Downloaded 34GiB to GPU in 2.47825s (13.7194GiB/s) | 20260507 21:19:49.593 | INFO | Analyzing chunk frames = [2675712,2676224) | 20260507 21:19:49.594 | INFO | MemInfo: { | 20260507 21:19:49.594 | INFO | "gpu" : " 17 GB", | 20260507 21:19:49.594 | INFO | "huge" : "", | 20260507 21:19:49.594 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:49.594 | INFO | "malloc" : " 14 GB", | 20260507 21:19:49.594 | INFO | "pinned" : " 29 GB" | 20260507 21:19:49.594 | INFO | } | 20260507 21:19:49.594 | INFO | Perf report for the last 5.13983 seconds: | 20260507 21:19:49.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:49.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:19:49.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:19:49.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:49.594 | INFO | Uploaded 36GiB to GPU in 2.52773s (14.242GiB/s) | 20260507 21:19:49.594 | INFO | Downloaded 34GiB to GPU in 2.46444s (13.7962GiB/s) | 20260507 21:19:54.732 | INFO | Analyzing chunk frames = [2676224,2676736) | 20260507 21:19:54.732 | INFO | MemInfo: { | 20260507 21:19:54.732 | INFO | "gpu" : " 17 GB", | 20260507 21:19:54.732 | INFO | "huge" : "", | 20260507 21:19:54.732 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:54.732 | INFO | "malloc" : " 14 GB", | 20260507 21:19:54.732 | INFO | "pinned" : " 29 GB" | 20260507 21:19:54.732 | INFO | } | 20260507 21:19:54.732 | INFO | Perf report for the last 5.13863 seconds: | 20260507 21:19:54.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:54.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.732 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:19:54.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.732 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:19:54.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:54.733 | INFO | Uploaded 36GiB to GPU in 2.53633s (14.1937GiB/s) | 20260507 21:19:54.733 | INFO | Downloaded 34GiB to GPU in 2.47746s (13.7237GiB/s) | 20260507 21:19:59.872 | INFO | Analyzing chunk frames = [2676736,2677248) | 20260507 21:19:59.872 | INFO | MemInfo: { | 20260507 21:19:59.872 | INFO | "gpu" : " 17 GB", | 20260507 21:19:59.872 | INFO | "huge" : "", | 20260507 21:19:59.872 | INFO | "huge_shared" : " 29 GB", | 20260507 21:19:59.872 | INFO | "malloc" : " 14 GB", | 20260507 21:19:59.872 | INFO | "pinned" : " 29 GB" | 20260507 21:19:59.872 | INFO | } | 20260507 21:19:59.872 | INFO | Perf report for the last 5.14014 seconds: | 20260507 21:19:59.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:19:59.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.872 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:19:59.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.873 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:19:59.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:19:59.873 | INFO | Uploaded 36GiB to GPU in 2.44257s (14.7386GiB/s) | 20260507 21:19:59.873 | INFO | Downloaded 34GiB to GPU in 2.48317s (13.6922GiB/s) | 20260507 21:20:05.012 | INFO | Analyzing chunk frames = [2677248,2677760) | 20260507 21:20:05.012 | INFO | MemInfo: { | 20260507 21:20:05.012 | INFO | "gpu" : " 17 GB", | 20260507 21:20:05.012 | INFO | "huge" : "", | 20260507 21:20:05.012 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:05.012 | INFO | "malloc" : " 14 GB", | 20260507 21:20:05.012 | INFO | "pinned" : " 29 GB" | 20260507 21:20:05.012 | INFO | } | 20260507 21:20:05.012 | INFO | Perf report for the last 5.13998 seconds: | 20260507 21:20:05.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:05.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260507 21:20:05.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:20:05.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:05.013 | INFO | Uploaded 36GiB to GPU in 2.32993s (15.4511GiB/s) | 20260507 21:20:05.013 | INFO | Downloaded 34GiB to GPU in 2.38072s (14.2814GiB/s) | 20260507 21:20:10.151 | INFO | Analyzing chunk frames = [2677760,2678272) | 20260507 21:20:10.152 | INFO | MemInfo: { | 20260507 21:20:10.152 | INFO | "gpu" : " 17 GB", | 20260507 21:20:10.152 | INFO | "huge" : "", | 20260507 21:20:10.152 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:10.152 | INFO | "malloc" : " 14 GB", | 20260507 21:20:10.152 | INFO | "pinned" : " 29 GB" | 20260507 21:20:10.152 | INFO | } | 20260507 21:20:10.152 | INFO | Perf report for the last 5.13918 seconds: | 20260507 21:20:10.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:10.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:20:10.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:20:10.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:10.152 | INFO | Uploaded 36GiB to GPU in 2.53924s (14.1775GiB/s) | 20260507 21:20:10.152 | INFO | Downloaded 34GiB to GPU in 2.46048s (13.8185GiB/s) | 20260507 21:20:15.290 | INFO | Analyzing chunk frames = [2678272,2678784) | 20260507 21:20:15.290 | INFO | MemInfo: { | 20260507 21:20:15.290 | INFO | "gpu" : " 17 GB", | 20260507 21:20:15.290 | INFO | "huge" : "", | 20260507 21:20:15.290 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:15.290 | INFO | "malloc" : " 14 GB", | 20260507 21:20:15.290 | INFO | "pinned" : " 29 GB" | 20260507 21:20:15.290 | INFO | } | 20260507 21:20:15.291 | INFO | Perf report for the last 5.13886 seconds: | 20260507 21:20:15.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:15.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:20:15.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:20:15.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:15.291 | INFO | Uploaded 36GiB to GPU in 2.52579s (14.2529GiB/s) | 20260507 21:20:15.291 | INFO | Downloaded 34GiB to GPU in 2.43541s (13.9607GiB/s) | 20260507 21:20:15.291 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:20:15.291 | INFO | Total mem usage: 16.7 GiB | 20260507 21:20:15.291 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:20:15.291 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.7%) | 20260507 21:20:15.291 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:20:15.291 | INFO | High Water Breakdown: | 20260507 21:20:15.291 | INFO | 60.2% : 9.07 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:20:15.291 | INFO | 39.7% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:20:15.291 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:20:15.291 | INFO | ------------------------------------------ | 20260507 21:20:15.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:20:15.291 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:20:15.291 | INFO | Total mem usage: 26.5 GiB | 20260507 21:20:15.291 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:20:15.291 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:20:15.291 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:20:15.291 | INFO | High Water Breakdown: | 20260507 21:20:15.291 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:20:15.291 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:20:15.291 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:20:15.292 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:20:15.292 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:20:15.292 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:20:15.292 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:20:15.292 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:20:15.292 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:20:15.292 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:20:15.292 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:20:15.292 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:20:15.292 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:20:15.292 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:20:15.292 | INFO | ------------------------------------------ | 20260507 21:20:15.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:20:15.292 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:20:15.292 | INFO | Total mem usage: 27.8 GiB | 20260507 21:20:15.292 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:20:15.292 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 21:20:15.292 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 21:20:15.292 | INFO | High Water Breakdown: | 20260507 21:20:15.292 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 21:20:15.292 | INFO | ------------------------------------------ | 20260507 21:20:15.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:20:15.292 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:20:15.292 | INFO | Total mem usage: 15.7 GiB | 20260507 21:20:15.292 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:20:15.292 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:20:15.292 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:20:15.292 | INFO | High Water Breakdown: | 20260507 21:20:15.292 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:20:15.292 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:20:15.292 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:20:15.292 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:20:15.292 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:20:15.292 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:20:15.292 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:20:15.292 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:20:15.292 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:20:15.292 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:20:15.292 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:20:15.292 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:20:15.292 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:20:15.292 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:20:15.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:20:15.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:20:15.293 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:20:15.293 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:20:15.293 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:20:15.293 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:20:15.293 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:20:15.293 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:20:15.293 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:20:15.293 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:20:15.293 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:20:15.293 | INFO | ------------------------------------------ | 20260507 21:20:15.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:20:15.293 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:20:15.293 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:20:15.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:20:15.293 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.64e+01, 1.44e+01 / 1.63e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.21e-02 / 5.71e-03 | 20260507 21:20:15.293 | INFO | PerfReport: 12.16%, Baseline -- 4.02e+00 / 4.48e+00, 6.00e-01 / 2.30e-01 | 20260507 21:20:15.293 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.10e-01 / 1.04e-01 | 20260507 21:20:15.293 | INFO | PerfReport: 16.14%, DME -- 4.26e+01 / 4.64e+01, 4.37e+00 / 1.57e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 55.80%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.61e+00 / 1.41e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.42e-01 / 2.04e-01 | 20260507 21:20:15.293 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.40e+00, 2.94e-01 / 1.47e-01 | 20260507 21:20:15.293 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.62e-01 / 3.99e-01, 5.93e-02 / 2.58e-02 | 20260507 21:20:15.293 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:20:15.293 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:20:15.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:20:15.293 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 24.36%, Upload -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 24.12%, Download -- 2.58e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 30.82%, Compute -- 3.30e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 2.33%, LockUpload -- 2.49e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 2.50%, LockDownload -- 2.68e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 15.86%, LockCompute -- 1.70e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260507 21:20:15.293 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 4.21e-02 / 0.00e+00 | 20260507 21:20:15.403 | INFO | Starting a round of Baz IO | 20260507 21:20:18.872 | INFO | Populating RTMetrics report | 20260507 21:20:18.872 | INFO | RTMetrics report populated | 20260507 21:20:18.872 | INFO | Opening tmp RTMetrics json file | 20260507 21:20:18.880 | INFO | Serializing RTMetrics to json | 20260507 21:20:18.892 | INFO | Serialization done, swapping tmp file now... | 20260507 21:20:18.892 | INFO | RTMetrics done | 20260507 21:20:18.961 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.557 seconds | 20260507 21:20:18.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.80% / 0.01% / 56.19% | 20260507 21:20:18.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.72369% / 7.94729e-06% / 5.16574e-05% | 20260507 21:20:18.961 | INFO | Baz IO overall: | 20260507 21:20:18.961 | INFO | BytesWritten : 1219309335081 | 20260507 21:20:18.961 | INFO | NumEvents : 590985739566 | 20260507 21:20:20.428 | INFO | Analyzing chunk frames = [2678784,2679296) | 20260507 21:20:20.428 | INFO | MemInfo: { | 20260507 21:20:20.428 | INFO | "gpu" : " 17 GB", | 20260507 21:20:20.428 | INFO | "huge" : "", | 20260507 21:20:20.428 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:20.428 | INFO | "malloc" : " 14 GB", | 20260507 21:20:20.428 | INFO | "pinned" : " 29 GB" | 20260507 21:20:20.428 | INFO | } | 20260507 21:20:20.428 | INFO | Perf report for the last 5.13799 seconds: | 20260507 21:20:20.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:20.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:20.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:20.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:20.429 | INFO | ANALYSIS : 69.0%, 2.9, 104.9ms, 0 | 20260507 21:20:20.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:20.429 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 21:20:20.429 | INFO | PRE_HQ : 67.8%, 1.8, 62.5ms, 0 | 20260507 21:20:20.429 | INFO | BAZWRITER : 68.7%, 4.2, 149.7ms, 0 | 20260507 21:20:20.429 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 21:20:20.429 | INFO | Uploaded 36GiB to GPU in 2.60285s (13.831GiB/s) | 20260507 21:20:20.429 | INFO | Downloaded 38GiB to GPU in 2.70667s (14.0394GiB/s) | 20260507 21:20:25.569 | INFO | Analyzing chunk frames = [2679296,2679808) | 20260507 21:20:25.569 | INFO | MemInfo: { | 20260507 21:20:25.569 | INFO | "gpu" : " 17 GB", | 20260507 21:20:25.570 | INFO | "huge" : "", | 20260507 21:20:25.570 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:25.570 | INFO | "malloc" : " 14 GB", | 20260507 21:20:25.570 | INFO | "pinned" : " 29 GB" | 20260507 21:20:25.570 | INFO | } | 20260507 21:20:25.570 | INFO | Perf report for the last 5.14122 seconds: | 20260507 21:20:25.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:25.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 21:20:25.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:20:25.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:25.570 | INFO | Uploaded 36GiB to GPU in 2.56592s (14.0301GiB/s) | 20260507 21:20:25.570 | INFO | Downloaded 34GiB to GPU in 2.48911s (13.6595GiB/s) | 20260507 21:20:30.708 | INFO | Analyzing chunk frames = [2679808,2680320) | 20260507 21:20:30.708 | INFO | MemInfo: { | 20260507 21:20:30.708 | INFO | "gpu" : " 17 GB", | 20260507 21:20:30.708 | INFO | "huge" : "", | 20260507 21:20:30.708 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:30.708 | INFO | "malloc" : " 14 GB", | 20260507 21:20:30.708 | INFO | "pinned" : " 29 GB" | 20260507 21:20:30.708 | INFO | } | 20260507 21:20:30.708 | INFO | Perf report for the last 5.1385 seconds: | 20260507 21:20:30.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:30.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.708 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:20:30.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.708 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:20:30.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:30.709 | INFO | Uploaded 36GiB to GPU in 2.47772s (14.5295GiB/s) | 20260507 21:20:30.709 | INFO | Downloaded 34GiB to GPU in 2.48339s (13.691GiB/s) | 20260507 21:20:35.847 | INFO | Analyzing chunk frames = [2680320,2680832) | 20260507 21:20:35.847 | INFO | MemInfo: { | 20260507 21:20:35.847 | INFO | "gpu" : " 17 GB", | 20260507 21:20:35.847 | INFO | "huge" : "", | 20260507 21:20:35.847 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:35.847 | INFO | "malloc" : " 14 GB", | 20260507 21:20:35.847 | INFO | "pinned" : " 29 GB" | 20260507 21:20:35.847 | INFO | } | 20260507 21:20:35.847 | INFO | Perf report for the last 5.13912 seconds: | 20260507 21:20:35.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:35.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.847 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 21:20:35.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.847 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:20:35.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:35.848 | INFO | Uploaded 36GiB to GPU in 2.47502s (14.5454GiB/s) | 20260507 21:20:35.848 | INFO | Downloaded 34GiB to GPU in 2.48712s (13.6704GiB/s) | 20260507 21:20:40.987 | INFO | Analyzing chunk frames = [2680832,2681344) | 20260507 21:20:40.987 | INFO | MemInfo: { | 20260507 21:20:40.987 | INFO | "gpu" : " 17 GB", | 20260507 21:20:40.987 | INFO | "huge" : "", | 20260507 21:20:40.987 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:40.987 | INFO | "malloc" : " 14 GB", | 20260507 21:20:40.987 | INFO | "pinned" : " 29 GB" | 20260507 21:20:40.987 | INFO | } | 20260507 21:20:40.987 | INFO | Perf report for the last 5.13953 seconds: | 20260507 21:20:40.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:40.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 21:20:40.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:20:40.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:40.987 | INFO | Uploaded 36GiB to GPU in 2.39235s (15.048GiB/s) | 20260507 21:20:40.988 | INFO | Downloaded 34GiB to GPU in 2.48442s (13.6853GiB/s) | 20260507 21:20:46.126 | INFO | Analyzing chunk frames = [2681344,2681856) | 20260507 21:20:46.126 | INFO | MemInfo: { | 20260507 21:20:46.126 | INFO | "gpu" : " 17 GB", | 20260507 21:20:46.126 | INFO | "huge" : "", | 20260507 21:20:46.126 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:46.126 | INFO | "malloc" : " 14 GB", | 20260507 21:20:46.126 | INFO | "pinned" : " 29 GB" | 20260507 21:20:46.126 | INFO | } | 20260507 21:20:46.126 | INFO | Perf report for the last 5.13935 seconds: | 20260507 21:20:46.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:46.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.126 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:20:46.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.126 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:20:46.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:46.127 | INFO | Uploaded 36GiB to GPU in 2.39828s (15.0108GiB/s) | 20260507 21:20:46.127 | INFO | Downloaded 34GiB to GPU in 2.3742s (14.3206GiB/s) | 20260507 21:20:51.266 | INFO | Analyzing chunk frames = [2681856,2682368) | 20260507 21:20:51.266 | INFO | MemInfo: { | 20260507 21:20:51.266 | INFO | "gpu" : " 17 GB", | 20260507 21:20:51.266 | INFO | "huge" : "", | 20260507 21:20:51.266 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:51.266 | INFO | "malloc" : " 14 GB", | 20260507 21:20:51.266 | INFO | "pinned" : " 29 GB" | 20260507 21:20:51.266 | INFO | } | 20260507 21:20:51.266 | INFO | Perf report for the last 5.13992 seconds: | 20260507 21:20:51.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:51.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 21:20:51.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 21:20:51.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:51.266 | INFO | Uploaded 36GiB to GPU in 2.53719s (14.1889GiB/s) | 20260507 21:20:51.266 | INFO | Downloaded 34GiB to GPU in 2.42763s (14.0054GiB/s) | 20260507 21:20:56.406 | INFO | Analyzing chunk frames = [2682368,2682880) | 20260507 21:20:56.406 | INFO | MemInfo: { | 20260507 21:20:56.406 | INFO | "gpu" : " 17 GB", | 20260507 21:20:56.406 | INFO | "huge" : "", | 20260507 21:20:56.406 | INFO | "huge_shared" : " 29 GB", | 20260507 21:20:56.406 | INFO | "malloc" : " 14 GB", | 20260507 21:20:56.406 | INFO | "pinned" : " 29 GB" | 20260507 21:20:56.406 | INFO | } | 20260507 21:20:56.406 | INFO | Perf report for the last 5.13976 seconds: | 20260507 21:20:56.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:20:56.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:20:56.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:20:56.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:20:56.406 | INFO | Uploaded 36GiB to GPU in 2.53469s (14.2029GiB/s) | 20260507 21:20:56.406 | INFO | Downloaded 34GiB to GPU in 2.42759s (14.0056GiB/s) | 20260507 21:20:56.503 | INFO | Starting a round of Baz IO | 20260507 21:20:59.978 | INFO | Populating RTMetrics report | 20260507 21:20:59.979 | INFO | RTMetrics report populated | 20260507 21:20:59.979 | INFO | Opening tmp RTMetrics json file | 20260507 21:21:00.000 | INFO | Serializing RTMetrics to json | 20260507 21:21:00.002 | INFO | Serialization done, swapping tmp file now... | 20260507 21:21:00.003 | INFO | RTMetrics done | 20260507 21:21:00.075 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.571 seconds | 20260507 21:21:00.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.79% / 0.01% / 56.20% | 20260507 21:21:00.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.71986% / 1.19209e-05% / 6.35783e-05% | 20260507 21:21:00.075 | INFO | Baz IO overall: | 20260507 21:21:00.075 | INFO | BytesWritten : 1221201141695 | 20260507 21:21:00.075 | INFO | NumEvents : 591900594464 | 20260507 21:21:01.544 | INFO | Analyzing chunk frames = [2682880,2683392) | 20260507 21:21:01.544 | INFO | MemInfo: { | 20260507 21:21:01.544 | INFO | "gpu" : " 17 GB", | 20260507 21:21:01.544 | INFO | "huge" : "", | 20260507 21:21:01.544 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:01.544 | INFO | "malloc" : " 14 GB", | 20260507 21:21:01.544 | INFO | "pinned" : " 29 GB" | 20260507 21:21:01.544 | INFO | } | 20260507 21:21:01.544 | INFO | Perf report for the last 5.13835 seconds: | 20260507 21:21:01.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:01.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:01.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:01.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:01.544 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 21:21:01.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:01.544 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 21:21:01.544 | INFO | PRE_HQ : 66.2%, 1.7, 58.4ms, 0 | 20260507 21:21:01.544 | INFO | BAZWRITER : 69.1%, 4.1, 145.3ms, 0 | 20260507 21:21:01.545 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 21:21:01.545 | INFO | Uploaded 36GiB to GPU in 2.59877s (13.8527GiB/s) | 20260507 21:21:01.545 | INFO | Downloaded 38GiB to GPU in 2.67474s (14.207GiB/s) | 20260507 21:21:06.684 | INFO | Analyzing chunk frames = [2683392,2683904) | 20260507 21:21:06.684 | INFO | MemInfo: { | 20260507 21:21:06.684 | INFO | "gpu" : " 17 GB", | 20260507 21:21:06.684 | INFO | "huge" : "", | 20260507 21:21:06.684 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:06.684 | INFO | "malloc" : " 14 GB", | 20260507 21:21:06.684 | INFO | "pinned" : " 29 GB" | 20260507 21:21:06.684 | INFO | } | 20260507 21:21:06.684 | INFO | Perf report for the last 5.13948 seconds: | 20260507 21:21:06.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:06.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:21:06.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:21:06.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:06.684 | INFO | Uploaded 36GiB to GPU in 2.57765s (13.9662GiB/s) | 20260507 21:21:06.684 | INFO | Downloaded 34GiB to GPU in 2.49489s (13.6279GiB/s) | 20260507 21:21:11.823 | INFO | Analyzing chunk frames = [2683904,2684416) | 20260507 21:21:11.823 | INFO | MemInfo: { | 20260507 21:21:11.823 | INFO | "gpu" : " 17 GB", | 20260507 21:21:11.823 | INFO | "huge" : "", | 20260507 21:21:11.823 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:11.823 | INFO | "malloc" : " 14 GB", | 20260507 21:21:11.824 | INFO | "pinned" : " 29 GB" | 20260507 21:21:11.824 | INFO | } | 20260507 21:21:11.824 | INFO | Perf report for the last 5.13994 seconds: | 20260507 21:21:11.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:11.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:21:11.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:21:11.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:11.824 | INFO | Uploaded 36GiB to GPU in 2.51246s (14.3286GiB/s) | 20260507 21:21:11.824 | INFO | Downloaded 34GiB to GPU in 2.48137s (13.7021GiB/s) | 20260507 21:21:16.962 | INFO | Analyzing chunk frames = [2684416,2684928) | 20260507 21:21:16.962 | INFO | MemInfo: { | 20260507 21:21:16.962 | INFO | "gpu" : " 17 GB", | 20260507 21:21:16.962 | INFO | "huge" : "", | 20260507 21:21:16.962 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:16.962 | INFO | "malloc" : " 14 GB", | 20260507 21:21:16.962 | INFO | "pinned" : " 29 GB" | 20260507 21:21:16.962 | INFO | } | 20260507 21:21:16.962 | INFO | Perf report for the last 5.1387 seconds: | 20260507 21:21:16.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:16.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.963 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:21:16.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.963 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:21:16.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:16.963 | INFO | Uploaded 36GiB to GPU in 2.53136s (14.2216GiB/s) | 20260507 21:21:16.963 | INFO | Downloaded 34GiB to GPU in 2.47542s (13.735GiB/s) | 20260507 21:21:22.101 | INFO | Analyzing chunk frames = [2684928,2685440) | 20260507 21:21:22.101 | INFO | MemInfo: { | 20260507 21:21:22.101 | INFO | "gpu" : " 17 GB", | 20260507 21:21:22.101 | INFO | "huge" : "", | 20260507 21:21:22.101 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:22.101 | INFO | "malloc" : " 14 GB", | 20260507 21:21:22.101 | INFO | "pinned" : " 29 GB" | 20260507 21:21:22.101 | INFO | } | 20260507 21:21:22.101 | INFO | Perf report for the last 5.13875 seconds: | 20260507 21:21:22.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:22.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:21:22.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:21:22.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:22.101 | INFO | Uploaded 36GiB to GPU in 2.37266s (15.1729GiB/s) | 20260507 21:21:22.102 | INFO | Downloaded 34GiB to GPU in 2.48894s (13.6604GiB/s) | 20260507 21:21:27.241 | INFO | Analyzing chunk frames = [2685440,2685952) | 20260507 21:21:27.241 | INFO | MemInfo: { | 20260507 21:21:27.241 | INFO | "gpu" : " 17 GB", | 20260507 21:21:27.241 | INFO | "huge" : "", | 20260507 21:21:27.241 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:27.241 | INFO | "malloc" : " 14 GB", | 20260507 21:21:27.241 | INFO | "pinned" : " 29 GB" | 20260507 21:21:27.241 | INFO | } | 20260507 21:21:27.241 | INFO | Perf report for the last 5.13982 seconds: | 20260507 21:21:27.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:27.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:21:27.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 21:21:27.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:27.241 | INFO | Uploaded 36GiB to GPU in 2.3834s (15.1045GiB/s) | 20260507 21:21:27.241 | INFO | Downloaded 34GiB to GPU in 2.39083s (14.221GiB/s) | 20260507 21:21:32.380 | INFO | Analyzing chunk frames = [2685952,2686464) | 20260507 21:21:32.380 | INFO | MemInfo: { | 20260507 21:21:32.380 | INFO | "gpu" : " 17 GB", | 20260507 21:21:32.380 | INFO | "huge" : "", | 20260507 21:21:32.380 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:32.380 | INFO | "malloc" : " 14 GB", | 20260507 21:21:32.380 | INFO | "pinned" : " 29 GB" | 20260507 21:21:32.380 | INFO | } | 20260507 21:21:32.381 | INFO | Perf report for the last 5.13947 seconds: | 20260507 21:21:32.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:32.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:21:32.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:21:32.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:32.381 | INFO | Uploaded 36GiB to GPU in 2.54819s (14.1277GiB/s) | 20260507 21:21:32.381 | INFO | Downloaded 34GiB to GPU in 2.45752s (13.8351GiB/s) | 20260507 21:21:37.520 | INFO | Analyzing chunk frames = [2686464,2686976) | 20260507 21:21:37.520 | INFO | MemInfo: { | 20260507 21:21:37.520 | INFO | "gpu" : " 17 GB", | 20260507 21:21:37.520 | INFO | "huge" : "", | 20260507 21:21:37.520 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:37.520 | INFO | "malloc" : " 14 GB", | 20260507 21:21:37.520 | INFO | "pinned" : " 29 GB" | 20260507 21:21:37.520 | INFO | } | 20260507 21:21:37.520 | INFO | Perf report for the last 5.13919 seconds: | 20260507 21:21:37.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:37.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:21:37.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:21:37.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:37.520 | INFO | Uploaded 36GiB to GPU in 2.51559s (14.3108GiB/s) | 20260507 21:21:37.520 | INFO | Downloaded 34GiB to GPU in 2.43453s (13.9657GiB/s) | 20260507 21:21:37.520 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:21:37.520 | INFO | Total mem usage: 16.7 GiB | 20260507 21:21:37.520 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:21:37.520 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:21:37.520 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:21:37.520 | INFO | High Water Breakdown: | 20260507 21:21:37.520 | INFO | 59.9% : 8.9 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:21:37.520 | INFO | 40% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:21:37.520 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:21:37.520 | INFO | ------------------------------------------ | 20260507 21:21:37.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:21:37.520 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:21:37.520 | INFO | Total mem usage: 26.5 GiB | 20260507 21:21:37.520 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:21:37.520 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:21:37.520 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:21:37.520 | INFO | High Water Breakdown: | 20260507 21:21:37.520 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:21:37.520 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:21:37.520 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:21:37.520 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:21:37.520 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:21:37.520 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:21:37.521 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:21:37.521 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:21:37.521 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:21:37.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:21:37.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:21:37.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:21:37.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:21:37.521 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:21:37.521 | INFO | ------------------------------------------ | 20260507 21:21:37.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:21:37.521 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:21:37.521 | INFO | Total mem usage: 27.8 GiB | 20260507 21:21:37.521 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:21:37.521 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:21:37.521 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:21:37.521 | INFO | High Water Breakdown: | 20260507 21:21:37.521 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:21:37.521 | INFO | ------------------------------------------ | 20260507 21:21:37.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:21:37.521 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:21:37.521 | INFO | Total mem usage: 15.7 GiB | 20260507 21:21:37.521 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:21:37.521 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:21:37.521 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:21:37.521 | INFO | High Water Breakdown: | 20260507 21:21:37.521 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:21:37.521 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:21:37.521 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:21:37.521 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:21:37.521 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:21:37.521 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:21:37.521 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:21:37.521 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:21:37.521 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:21:37.521 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:21:37.521 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:21:37.521 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:21:37.521 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:21:37.521 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:21:37.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:21:37.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:21:37.521 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:21:37.521 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:21:37.521 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:21:37.521 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:21:37.521 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:21:37.521 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:21:37.521 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:21:37.521 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:21:37.521 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:21:37.521 | INFO | ------------------------------------------ | 20260507 21:21:37.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:21:37.521 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:21:37.521 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:21:37.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:21:37.521 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.64e+01, 1.42e+01 / 1.98e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.27e-02 / 5.95e-03 | 20260507 21:21:37.521 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.51e+00, 6.00e-01 / 1.99e-01 | 20260507 21:21:37.521 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.51e+00, 2.11e-01 / 1.05e-01 | 20260507 21:21:37.521 | INFO | PerfReport: 16.17%, DME -- 4.27e+01 / 4.64e+01, 4.19e+00 / 2.04e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 55.73%, FrameLabeling -- 1.84e+01 / 1.92e+01, 1.55e+00 / 1.33e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.52e+00 / 2.86e+00, 4.45e-01 / 1.91e-01 | 20260507 21:21:37.521 | INFO | PerfReport: 3.54%, Metrics -- 1.17e+00 / 1.39e+00, 2.92e-01 / 1.56e-01 | 20260507 21:21:37.521 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.62e-01 / 3.98e-01, 5.88e-02 / 2.53e-02 | 20260507 21:21:37.521 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:21:37.521 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:21:37.521 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:21:37.521 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 24.38%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 30.87%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 2.19%, LockUpload -- 2.34e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 2.60%, LockDownload -- 2.77e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 15.71%, LockCompute -- 1.68e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:21:37.521 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 21:21:37.617 | INFO | Starting a round of Baz IO | 20260507 21:21:41.104 | INFO | Populating RTMetrics report | 20260507 21:21:41.104 | INFO | RTMetrics report populated | 20260507 21:21:41.104 | INFO | Opening tmp RTMetrics json file | 20260507 21:21:41.138 | INFO | Serializing RTMetrics to json | 20260507 21:21:41.191 | INFO | Serialization done, swapping tmp file now... | 20260507 21:21:41.196 | INFO | RTMetrics done | 20260507 21:21:41.218 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.601 seconds | 20260507 21:21:41.219 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.79% / 0.01% / 56.21% | 20260507 21:21:41.219 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.71261% / 1.19209e-05% / 7.94729e-05% | 20260507 21:21:41.219 | INFO | Baz IO overall: | 20260507 21:21:41.219 | INFO | BytesWritten : 1223091765164 | 20260507 21:21:41.219 | INFO | NumEvents : 592814390926 | 20260507 21:21:42.660 | INFO | Analyzing chunk frames = [2686976,2687488) | 20260507 21:21:42.660 | INFO | MemInfo: { | 20260507 21:21:42.660 | INFO | "gpu" : " 17 GB", | 20260507 21:21:42.660 | INFO | "huge" : "", | 20260507 21:21:42.660 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:42.660 | INFO | "malloc" : " 14 GB", | 20260507 21:21:42.660 | INFO | "pinned" : " 29 GB" | 20260507 21:21:42.660 | INFO | } | 20260507 21:21:42.660 | INFO | Perf report for the last 5.1406 seconds: | 20260507 21:21:42.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:42.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:42.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:42.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:42.660 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 21:21:42.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:42.660 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 21:21:42.660 | INFO | PRE_HQ : 65.9%, 1.7, 58.1ms, 0 | 20260507 21:21:42.660 | INFO | BAZWRITER : 69.6%, 3.7, 130.9ms, 0 | 20260507 21:21:42.660 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 21:21:42.661 | INFO | Uploaded 36GiB to GPU in 2.57608s (13.9747GiB/s) | 20260507 21:21:42.661 | INFO | Downloaded 38GiB to GPU in 2.68828s (14.1354GiB/s) | 20260507 21:21:47.797 | INFO | Analyzing chunk frames = [2687488,2688000) | 20260507 21:21:47.797 | INFO | MemInfo: { | 20260507 21:21:47.797 | INFO | "gpu" : " 17 GB", | 20260507 21:21:47.797 | INFO | "huge" : "", | 20260507 21:21:47.797 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:47.797 | INFO | "malloc" : " 14 GB", | 20260507 21:21:47.797 | INFO | "pinned" : " 29 GB" | 20260507 21:21:47.797 | INFO | } | 20260507 21:21:47.797 | INFO | Perf report for the last 5.13698 seconds: | 20260507 21:21:47.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:47.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.797 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:21:47.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.797 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:21:47.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:47.798 | INFO | Uploaded 36GiB to GPU in 2.55824s (14.0722GiB/s) | 20260507 21:21:47.798 | INFO | Downloaded 34GiB to GPU in 2.50638s (13.5654GiB/s) | 20260507 21:21:52.937 | INFO | Analyzing chunk frames = [2688000,2688512) | 20260507 21:21:52.937 | INFO | MemInfo: { | 20260507 21:21:52.937 | INFO | "gpu" : " 17 GB", | 20260507 21:21:52.937 | INFO | "huge" : "", | 20260507 21:21:52.937 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:52.937 | INFO | "malloc" : " 14 GB", | 20260507 21:21:52.937 | INFO | "pinned" : " 29 GB" | 20260507 21:21:52.937 | INFO | } | 20260507 21:21:52.937 | INFO | Perf report for the last 5.13978 seconds: | 20260507 21:21:52.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:52.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:21:52.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:21:52.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:52.937 | INFO | Uploaded 36GiB to GPU in 2.49404s (14.4344GiB/s) | 20260507 21:21:52.937 | INFO | Downloaded 34GiB to GPU in 2.48257s (13.6955GiB/s) | 20260507 21:21:58.076 | INFO | Analyzing chunk frames = [2688512,2689024) | 20260507 21:21:58.076 | INFO | MemInfo: { | 20260507 21:21:58.076 | INFO | "gpu" : " 17 GB", | 20260507 21:21:58.076 | INFO | "huge" : "", | 20260507 21:21:58.077 | INFO | "huge_shared" : " 29 GB", | 20260507 21:21:58.077 | INFO | "malloc" : " 14 GB", | 20260507 21:21:58.077 | INFO | "pinned" : " 29 GB" | 20260507 21:21:58.077 | INFO | } | 20260507 21:21:58.077 | INFO | Perf report for the last 5.13955 seconds: | 20260507 21:21:58.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:21:58.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:21:58.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:21:58.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:21:58.077 | INFO | Uploaded 36GiB to GPU in 2.50097s (14.3944GiB/s) | 20260507 21:21:58.077 | INFO | Downloaded 34GiB to GPU in 2.49547s (13.6247GiB/s) | 20260507 21:22:03.216 | INFO | Analyzing chunk frames = [2689024,2689536) | 20260507 21:22:03.216 | INFO | MemInfo: { | 20260507 21:22:03.216 | INFO | "gpu" : " 17 GB", | 20260507 21:22:03.216 | INFO | "huge" : "", | 20260507 21:22:03.216 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:03.216 | INFO | "malloc" : " 14 GB", | 20260507 21:22:03.216 | INFO | "pinned" : " 29 GB" | 20260507 21:22:03.216 | INFO | } | 20260507 21:22:03.216 | INFO | Perf report for the last 5.13956 seconds: | 20260507 21:22:03.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:03.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.216 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 21:22:03.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.216 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260507 21:22:03.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:03.217 | INFO | Uploaded 36GiB to GPU in 2.35558s (15.2829GiB/s) | 20260507 21:22:03.217 | INFO | Downloaded 34GiB to GPU in 2.48362s (13.6897GiB/s) | 20260507 21:22:08.355 | INFO | Analyzing chunk frames = [2689536,2690048) | 20260507 21:22:08.355 | INFO | MemInfo: { | 20260507 21:22:08.355 | INFO | "gpu" : " 17 GB", | 20260507 21:22:08.355 | INFO | "huge" : "", | 20260507 21:22:08.355 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:08.355 | INFO | "malloc" : " 14 GB", | 20260507 21:22:08.355 | INFO | "pinned" : " 29 GB" | 20260507 21:22:08.355 | INFO | } | 20260507 21:22:08.355 | INFO | Perf report for the last 5.13901 seconds: | 20260507 21:22:08.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:08.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.355 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:22:08.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.355 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:22:08.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:08.356 | INFO | Uploaded 36GiB to GPU in 2.38988s (15.0635GiB/s) | 20260507 21:22:08.356 | INFO | Downloaded 34GiB to GPU in 2.39972s (14.1683GiB/s) | 20260507 21:22:13.494 | INFO | Analyzing chunk frames = [2690048,2690560) | 20260507 21:22:13.494 | INFO | MemInfo: { | 20260507 21:22:13.495 | INFO | "gpu" : " 17 GB", | 20260507 21:22:13.495 | INFO | "huge" : "", | 20260507 21:22:13.495 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:13.495 | INFO | "malloc" : " 14 GB", | 20260507 21:22:13.495 | INFO | "pinned" : " 29 GB" | 20260507 21:22:13.495 | INFO | } | 20260507 21:22:13.495 | INFO | Perf report for the last 5.13937 seconds: | 20260507 21:22:13.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:13.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:22:13.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:22:13.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:13.495 | INFO | Uploaded 36GiB to GPU in 2.53295s (14.2127GiB/s) | 20260507 21:22:13.495 | INFO | Downloaded 34GiB to GPU in 2.45693s (13.8384GiB/s) | 20260507 21:22:18.634 | INFO | Analyzing chunk frames = [2690560,2691072) | 20260507 21:22:18.634 | INFO | MemInfo: { | 20260507 21:22:18.634 | INFO | "gpu" : " 17 GB", | 20260507 21:22:18.634 | INFO | "huge" : "", | 20260507 21:22:18.634 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:18.634 | INFO | "malloc" : " 14 GB", | 20260507 21:22:18.634 | INFO | "pinned" : " 29 GB" | 20260507 21:22:18.635 | INFO | } | 20260507 21:22:18.635 | INFO | Perf report for the last 5.13989 seconds: | 20260507 21:22:18.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:18.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:22:18.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 21:22:18.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:18.635 | INFO | Uploaded 36GiB to GPU in 2.50676s (14.3612GiB/s) | 20260507 21:22:18.635 | INFO | Downloaded 34GiB to GPU in 2.4376s (13.9481GiB/s) | 20260507 21:22:18.732 | INFO | Starting a round of Baz IO | 20260507 21:22:22.211 | INFO | Populating RTMetrics report | 20260507 21:22:22.211 | INFO | RTMetrics report populated | 20260507 21:22:22.211 | INFO | Opening tmp RTMetrics json file | 20260507 21:22:22.374 | INFO | Serializing RTMetrics to json | 20260507 21:22:22.417 | INFO | Serialization done, swapping tmp file now... | 20260507 21:22:22.421 | INFO | RTMetrics done | 20260507 21:22:22.520 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.788 seconds | 20260507 21:22:22.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.78% / 0.01% / 56.21% | 20260507 21:22:22.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70945% / 1.58946e-05% / 4.37101e-05% | 20260507 21:22:22.520 | INFO | Baz IO overall: | 20260507 21:22:22.520 | INFO | BytesWritten : 1224982179588 | 20260507 21:22:22.520 | INFO | NumEvents : 593728082739 | 20260507 21:22:23.774 | INFO | Analyzing chunk frames = [2691072,2691584) | 20260507 21:22:23.774 | INFO | MemInfo: { | 20260507 21:22:23.774 | INFO | "gpu" : " 17 GB", | 20260507 21:22:23.774 | INFO | "huge" : "", | 20260507 21:22:23.774 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:23.774 | INFO | "malloc" : " 14 GB", | 20260507 21:22:23.774 | INFO | "pinned" : " 29 GB" | 20260507 21:22:23.774 | INFO | } | 20260507 21:22:23.774 | INFO | Perf report for the last 5.13961 seconds: | 20260507 21:22:23.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:23.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:23.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:23.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:23.774 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 21:22:23.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:23.774 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 21:22:23.774 | INFO | PRE_HQ : 66.2%, 1.7, 57.7ms, 0 | 20260507 21:22:23.774 | INFO | BAZWRITER : 73.3%, 4.0, 152.2ms, 0 | 20260507 21:22:23.774 | INFO | BAZ_SAMPLING : 3.5%, 1.0, 1.8ms, 0 | 20260507 21:22:23.775 | INFO | Uploaded 36GiB to GPU in 2.55201s (14.1065GiB/s) | 20260507 21:22:23.775 | INFO | Downloaded 38GiB to GPU in 2.68355s (14.1604GiB/s) | 20260507 21:22:28.913 | INFO | Analyzing chunk frames = [2691584,2692096) | 20260507 21:22:28.913 | INFO | MemInfo: { | 20260507 21:22:28.913 | INFO | "gpu" : " 17 GB", | 20260507 21:22:28.913 | INFO | "huge" : "", | 20260507 21:22:28.913 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:28.913 | INFO | "malloc" : " 14 GB", | 20260507 21:22:28.913 | INFO | "pinned" : " 29 GB" | 20260507 21:22:28.913 | INFO | } | 20260507 21:22:28.913 | INFO | Perf report for the last 5.13874 seconds: | 20260507 21:22:28.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:28.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:22:28.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 21:22:28.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:28.913 | INFO | Uploaded 36GiB to GPU in 2.56257s (14.0484GiB/s) | 20260507 21:22:28.913 | INFO | Downloaded 34GiB to GPU in 2.50504s (13.5726GiB/s) | 20260507 21:22:34.052 | INFO | Analyzing chunk frames = [2692096,2692608) | 20260507 21:22:34.052 | INFO | MemInfo: { | 20260507 21:22:34.052 | INFO | "gpu" : " 17 GB", | 20260507 21:22:34.052 | INFO | "huge" : "", | 20260507 21:22:34.052 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:34.052 | INFO | "malloc" : " 14 GB", | 20260507 21:22:34.052 | INFO | "pinned" : " 29 GB" | 20260507 21:22:34.053 | INFO | } | 20260507 21:22:34.053 | INFO | Perf report for the last 5.13986 seconds: | 20260507 21:22:34.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:34.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.053 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:22:34.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.053 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:22:34.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:34.054 | INFO | Uploaded 36GiB to GPU in 2.49097s (14.4522GiB/s) | 20260507 21:22:34.054 | INFO | Downloaded 34GiB to GPU in 2.47658s (13.7286GiB/s) | 20260507 21:22:39.191 | INFO | Analyzing chunk frames = [2692608,2693120) | 20260507 21:22:39.191 | INFO | MemInfo: { | 20260507 21:22:39.191 | INFO | "gpu" : " 17 GB", | 20260507 21:22:39.192 | INFO | "huge" : "", | 20260507 21:22:39.192 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:39.192 | INFO | "malloc" : " 14 GB", | 20260507 21:22:39.192 | INFO | "pinned" : " 29 GB" | 20260507 21:22:39.192 | INFO | } | 20260507 21:22:39.192 | INFO | Perf report for the last 5.13893 seconds: | 20260507 21:22:39.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:39.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 21:22:39.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 21:22:39.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:39.192 | INFO | Uploaded 36GiB to GPU in 2.51829s (14.2954GiB/s) | 20260507 21:22:39.192 | INFO | Downloaded 34GiB to GPU in 2.47767s (13.7226GiB/s) | 20260507 21:22:44.331 | INFO | Analyzing chunk frames = [2693120,2693632) | 20260507 21:22:44.331 | INFO | MemInfo: { | 20260507 21:22:44.331 | INFO | "gpu" : " 17 GB", | 20260507 21:22:44.331 | INFO | "huge" : "", | 20260507 21:22:44.332 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:44.332 | INFO | "malloc" : " 14 GB", | 20260507 21:22:44.332 | INFO | "pinned" : " 29 GB" | 20260507 21:22:44.332 | INFO | } | 20260507 21:22:44.332 | INFO | Perf report for the last 5.13976 seconds: | 20260507 21:22:44.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:44.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:22:44.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260507 21:22:44.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:44.332 | INFO | Uploaded 36GiB to GPU in 2.31064s (15.5801GiB/s) | 20260507 21:22:44.332 | INFO | Downloaded 34GiB to GPU in 2.47673s (13.7278GiB/s) | 20260507 21:22:49.471 | INFO | Analyzing chunk frames = [2693632,2694144) | 20260507 21:22:49.471 | INFO | MemInfo: { | 20260507 21:22:49.471 | INFO | "gpu" : " 17 GB", | 20260507 21:22:49.471 | INFO | "huge" : "", | 20260507 21:22:49.471 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:49.471 | INFO | "malloc" : " 14 GB", | 20260507 21:22:49.471 | INFO | "pinned" : " 29 GB" | 20260507 21:22:49.471 | INFO | } | 20260507 21:22:49.471 | INFO | Perf report for the last 5.1397 seconds: | 20260507 21:22:49.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:49.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.471 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260507 21:22:49.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.471 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:22:49.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:49.472 | INFO | Uploaded 36GiB to GPU in 2.42236s (14.8615GiB/s) | 20260507 21:22:49.472 | INFO | Downloaded 34GiB to GPU in 2.38774s (14.2394GiB/s) | 20260507 21:22:54.609 | INFO | Analyzing chunk frames = [2694144,2694656) | 20260507 21:22:54.609 | INFO | MemInfo: { | 20260507 21:22:54.609 | INFO | "gpu" : " 17 GB", | 20260507 21:22:54.609 | INFO | "huge" : "", | 20260507 21:22:54.609 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:54.609 | INFO | "malloc" : " 14 GB", | 20260507 21:22:54.609 | INFO | "pinned" : " 29 GB" | 20260507 21:22:54.609 | INFO | } | 20260507 21:22:54.609 | INFO | Perf report for the last 5.1377 seconds: | 20260507 21:22:54.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:54.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.609 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260507 21:22:54.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.609 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:22:54.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:54.610 | INFO | Uploaded 36GiB to GPU in 2.5383s (14.1827GiB/s) | 20260507 21:22:54.610 | INFO | Downloaded 34GiB to GPU in 2.43798s (13.9459GiB/s) | 20260507 21:22:59.748 | INFO | Analyzing chunk frames = [2694656,2695168) | 20260507 21:22:59.748 | INFO | MemInfo: { | 20260507 21:22:59.748 | INFO | "gpu" : " 17 GB", | 20260507 21:22:59.748 | INFO | "huge" : "", | 20260507 21:22:59.748 | INFO | "huge_shared" : " 29 GB", | 20260507 21:22:59.748 | INFO | "malloc" : " 14 GB", | 20260507 21:22:59.748 | INFO | "pinned" : " 29 GB" | 20260507 21:22:59.748 | INFO | } | 20260507 21:22:59.749 | INFO | Perf report for the last 5.13973 seconds: | 20260507 21:22:59.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:22:59.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 21:22:59.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:22:59.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:22:59.749 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:22:59.749 | INFO | Total mem usage: 16.7 GiB | 20260507 21:22:59.749 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:22:59.749 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.5%) | 20260507 21:22:59.749 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:22:59.749 | INFO | High Water Breakdown: | 20260507 21:22:59.749 | INFO | 60% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:22:59.749 | INFO | 39.9% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:22:59.749 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:22:59.749 | INFO | ------------------------------------------ | 20260507 21:22:59.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:22:59.749 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:22:59.749 | INFO | Total mem usage: 26.5 GiB | 20260507 21:22:59.749 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:22:59.749 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:22:59.749 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:22:59.749 | INFO | High Water Breakdown: | 20260507 21:22:59.749 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:22:59.749 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:22:59.749 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:22:59.749 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:22:59.749 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:22:59.749 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:22:59.749 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:22:59.749 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:22:59.749 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:22:59.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:22:59.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:22:59.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:22:59.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:22:59.749 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:22:59.749 | INFO | ------------------------------------------ | 20260507 21:22:59.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:22:59.749 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:22:59.749 | INFO | Total mem usage: 27.8 GiB | 20260507 21:22:59.749 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:22:59.749 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:22:59.749 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:22:59.749 | INFO | High Water Breakdown: | 20260507 21:22:59.749 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:22:59.749 | INFO | ------------------------------------------ | 20260507 21:22:59.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:22:59.749 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:22:59.749 | INFO | Total mem usage: 15.7 GiB | 20260507 21:22:59.749 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:22:59.749 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:22:59.749 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:22:59.749 | INFO | High Water Breakdown: | 20260507 21:22:59.749 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:22:59.749 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:22:59.749 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:22:59.749 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:22:59.749 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:22:59.749 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:22:59.749 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:22:59.749 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:22:59.749 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:22:59.749 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:22:59.749 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:22:59.749 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:22:59.749 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:22:59.749 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:22:59.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:22:59.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:22:59.749 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:22:59.749 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:22:59.749 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:22:59.749 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:22:59.749 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:22:59.749 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:22:59.749 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:22:59.749 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:22:59.749 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:22:59.749 | INFO | ------------------------------------------ | 20260507 21:22:59.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:22:59.750 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:22:59.750 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:22:59.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:22:59.750 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.43e+01 / 1.83e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.03e-01, 1.35e-02 / 8.31e-03 | 20260507 21:22:59.750 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.51e+00, 6.00e-01 / 2.11e-01 | 20260507 21:22:59.750 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.10e-01 / 1.14e-01 | 20260507 21:22:59.750 | INFO | PerfReport: 16.14%, DME -- 4.26e+01 / 4.62e+01, 4.42e+00 / 1.98e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 55.73%, FrameLabeling -- 1.84e+01 / 1.91e+01, 1.57e+00 / 1.36e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.52e+00 / 2.88e+00, 4.38e-01 / 1.69e-01 | 20260507 21:22:59.750 | INFO | PerfReport: 3.56%, Metrics -- 1.17e+00 / 1.39e+00, 2.94e-01 / 1.37e-01 | 20260507 21:22:59.750 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.63e-01 / 4.02e-01, 5.92e-02 / 3.10e-02 | 20260507 21:22:59.750 | INFO | Uploaded 36GiB to GPU in 2.50802s (14.354GiB/s) | 20260507 21:22:59.750 | INFO | Downloaded 34GiB to GPU in 2.40796s (14.1198GiB/s) | 20260507 21:22:59.750 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:22:59.750 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:22:59.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:22:59.750 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 24.24%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 30.84%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 2.08%, LockUpload -- 2.23e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 2.75%, LockDownload -- 2.94e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 15.85%, LockCompute -- 1.69e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:22:59.750 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 21:22:59.846 | INFO | Starting a round of Baz IO | 20260507 21:23:03.314 | INFO | Populating RTMetrics report | 20260507 21:23:03.315 | INFO | RTMetrics report populated | 20260507 21:23:03.315 | INFO | Opening tmp RTMetrics json file | 20260507 21:23:03.319 | INFO | Serializing RTMetrics to json | 20260507 21:23:03.338 | INFO | Serialization done, swapping tmp file now... | 20260507 21:23:03.339 | INFO | RTMetrics done | 20260507 21:23:03.412 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.565 seconds | 20260507 21:23:03.412 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.77% / 0.01% / 56.22% | 20260507 21:23:03.412 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.69117% / 3.97364e-06% / 5.16574e-05% | 20260507 21:23:03.412 | INFO | Baz IO overall: | 20260507 21:23:03.412 | INFO | BytesWritten : 1226872214356 | 20260507 21:23:03.412 | INFO | NumEvents : 594641498197 | 20260507 21:23:04.888 | INFO | Analyzing chunk frames = [2695168,2695680) | 20260507 21:23:04.888 | INFO | MemInfo: { | 20260507 21:23:04.888 | INFO | "gpu" : " 17 GB", | 20260507 21:23:04.888 | INFO | "huge" : "", | 20260507 21:23:04.888 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:04.888 | INFO | "malloc" : " 14 GB", | 20260507 21:23:04.888 | INFO | "pinned" : " 29 GB" | 20260507 21:23:04.888 | INFO | } | 20260507 21:23:04.888 | INFO | Perf report for the last 5.1396 seconds: | 20260507 21:23:04.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:04.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:04.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:04.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:04.889 | INFO | ANALYSIS : 68.7%, 3.0, 107.0ms, 0 | 20260507 21:23:04.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:04.889 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 21:23:04.889 | INFO | PRE_HQ : 65.9%, 1.7, 57.6ms, 0 | 20260507 21:23:04.889 | INFO | BAZWRITER : 69.0%, 3.8, 135.6ms, 0 | 20260507 21:23:04.889 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 21:23:04.889 | INFO | Uploaded 36GiB to GPU in 2.56253s (14.0486GiB/s) | 20260507 21:23:04.889 | INFO | Downloaded 38GiB to GPU in 2.68631s (14.1458GiB/s) | 20260507 21:23:10.027 | INFO | Analyzing chunk frames = [2695680,2696192) | 20260507 21:23:10.027 | INFO | MemInfo: { | 20260507 21:23:10.027 | INFO | "gpu" : " 17 GB", | 20260507 21:23:10.027 | INFO | "huge" : "", | 20260507 21:23:10.027 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:10.027 | INFO | "malloc" : " 14 GB", | 20260507 21:23:10.027 | INFO | "pinned" : " 29 GB" | 20260507 21:23:10.027 | INFO | } | 20260507 21:23:10.028 | INFO | Perf report for the last 5.13933 seconds: | 20260507 21:23:10.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:10.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 21:23:10.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:23:10.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:10.028 | INFO | Uploaded 36GiB to GPU in 2.59012s (13.899GiB/s) | 20260507 21:23:10.028 | INFO | Downloaded 34GiB to GPU in 2.49104s (13.6489GiB/s) | 20260507 21:23:15.167 | INFO | Analyzing chunk frames = [2696192,2696704) | 20260507 21:23:15.167 | INFO | MemInfo: { | 20260507 21:23:15.167 | INFO | "gpu" : " 17 GB", | 20260507 21:23:15.167 | INFO | "huge" : "", | 20260507 21:23:15.167 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:15.167 | INFO | "malloc" : " 14 GB", | 20260507 21:23:15.167 | INFO | "pinned" : " 29 GB" | 20260507 21:23:15.167 | INFO | } | 20260507 21:23:15.167 | INFO | Perf report for the last 5.1395 seconds: | 20260507 21:23:15.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:15.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.167 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:23:15.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.167 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:23:15.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:15.168 | INFO | Uploaded 36GiB to GPU in 2.52221s (14.2732GiB/s) | 20260507 21:23:15.168 | INFO | Downloaded 34GiB to GPU in 2.47896s (13.7154GiB/s) | 20260507 21:23:20.306 | INFO | Analyzing chunk frames = [2696704,2697216) | 20260507 21:23:20.306 | INFO | MemInfo: { | 20260507 21:23:20.306 | INFO | "gpu" : " 17 GB", | 20260507 21:23:20.306 | INFO | "huge" : "", | 20260507 21:23:20.306 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:20.306 | INFO | "malloc" : " 14 GB", | 20260507 21:23:20.306 | INFO | "pinned" : " 29 GB" | 20260507 21:23:20.306 | INFO | } | 20260507 21:23:20.306 | INFO | Perf report for the last 5.139 seconds: | 20260507 21:23:20.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:20.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.306 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 21:23:20.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.306 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:23:20.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:20.307 | INFO | Uploaded 36GiB to GPU in 2.52354s (14.2657GiB/s) | 20260507 21:23:20.307 | INFO | Downloaded 34GiB to GPU in 2.48076s (13.7055GiB/s) | 20260507 21:23:25.445 | INFO | Analyzing chunk frames = [2697216,2697728) | 20260507 21:23:25.445 | INFO | MemInfo: { | 20260507 21:23:25.445 | INFO | "gpu" : " 17 GB", | 20260507 21:23:25.445 | INFO | "huge" : "", | 20260507 21:23:25.445 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:25.445 | INFO | "malloc" : " 14 GB", | 20260507 21:23:25.445 | INFO | "pinned" : " 29 GB" | 20260507 21:23:25.445 | INFO | } | 20260507 21:23:25.445 | INFO | Perf report for the last 5.13889 seconds: | 20260507 21:23:25.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:25.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 21:23:25.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 21:23:25.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:25.445 | INFO | Uploaded 36GiB to GPU in 2.37142s (15.1808GiB/s) | 20260507 21:23:25.445 | INFO | Downloaded 34GiB to GPU in 2.48564s (13.6786GiB/s) | 20260507 21:23:30.584 | INFO | Analyzing chunk frames = [2697728,2698240) | 20260507 21:23:30.584 | INFO | MemInfo: { | 20260507 21:23:30.584 | INFO | "gpu" : " 17 GB", | 20260507 21:23:30.584 | INFO | "huge" : "", | 20260507 21:23:30.584 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:30.585 | INFO | "malloc" : " 14 GB", | 20260507 21:23:30.585 | INFO | "pinned" : " 29 GB" | 20260507 21:23:30.585 | INFO | } | 20260507 21:23:30.585 | INFO | Perf report for the last 5.13952 seconds: | 20260507 21:23:30.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:30.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:23:30.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:23:30.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:30.585 | INFO | Uploaded 36GiB to GPU in 2.38625s (15.0864GiB/s) | 20260507 21:23:30.585 | INFO | Downloaded 34GiB to GPU in 2.39004s (14.2257GiB/s) | 20260507 21:23:35.726 | INFO | Analyzing chunk frames = [2698240,2698752) | 20260507 21:23:35.726 | INFO | MemInfo: { | 20260507 21:23:35.726 | INFO | "gpu" : " 17 GB", | 20260507 21:23:35.726 | INFO | "huge" : "", | 20260507 21:23:35.726 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:35.726 | INFO | "malloc" : " 14 GB", | 20260507 21:23:35.726 | INFO | "pinned" : " 29 GB" | 20260507 21:23:35.726 | INFO | } | 20260507 21:23:35.726 | INFO | Perf report for the last 5.14104 seconds: | 20260507 21:23:35.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:35.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:23:35.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:23:35.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:35.726 | INFO | Uploaded 36GiB to GPU in 2.5424s (14.1599GiB/s) | 20260507 21:23:35.726 | INFO | Downloaded 34GiB to GPU in 2.45611s (13.843GiB/s) | 20260507 21:23:40.863 | INFO | Analyzing chunk frames = [2698752,2699264) | 20260507 21:23:40.863 | INFO | MemInfo: { | 20260507 21:23:40.863 | INFO | "gpu" : " 17 GB", | 20260507 21:23:40.863 | INFO | "huge" : "", | 20260507 21:23:40.863 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:40.863 | INFO | "malloc" : " 14 GB", | 20260507 21:23:40.863 | INFO | "pinned" : " 29 GB" | 20260507 21:23:40.863 | INFO | } | 20260507 21:23:40.864 | INFO | Perf report for the last 5.1378 seconds: | 20260507 21:23:40.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:40.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:23:40.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:23:40.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:40.864 | INFO | Uploaded 36GiB to GPU in 2.50819s (14.353GiB/s) | 20260507 21:23:40.864 | INFO | Downloaded 34GiB to GPU in 2.43489s (13.9637GiB/s) | 20260507 21:23:40.960 | INFO | Starting a round of Baz IO | 20260507 21:23:44.441 | INFO | Populating RTMetrics report | 20260507 21:23:44.441 | INFO | RTMetrics report populated | 20260507 21:23:44.441 | INFO | Opening tmp RTMetrics json file | 20260507 21:23:44.465 | INFO | Serializing RTMetrics to json | 20260507 21:23:44.473 | INFO | Serialization done, swapping tmp file now... | 20260507 21:23:44.474 | INFO | RTMetrics done | 20260507 21:23:44.537 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.576 seconds | 20260507 21:23:44.537 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.76% / 0.01% / 56.23% | 20260507 21:23:44.537 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6789% / 3.97364e-06% / 8.34465e-05% | 20260507 21:23:44.537 | INFO | Baz IO overall: | 20260507 21:23:44.537 | INFO | BytesWritten : 1228761708195 | 20260507 21:23:44.537 | INFO | NumEvents : 595554370848 | 20260507 21:23:46.003 | INFO | Analyzing chunk frames = [2699264,2699776) | 20260507 21:23:46.003 | INFO | MemInfo: { | 20260507 21:23:46.003 | INFO | "gpu" : " 17 GB", | 20260507 21:23:46.003 | INFO | "huge" : "", | 20260507 21:23:46.003 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:46.003 | INFO | "malloc" : " 14 GB", | 20260507 21:23:46.003 | INFO | "pinned" : " 29 GB" | 20260507 21:23:46.003 | INFO | } | 20260507 21:23:46.003 | INFO | Perf report for the last 5.1393 seconds: | 20260507 21:23:46.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:46.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:46.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:46.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:46.003 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 21:23:46.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:46.003 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 21:23:46.003 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260507 21:23:46.003 | INFO | BAZWRITER : 69.2%, 4.1, 146.3ms, 0 | 20260507 21:23:46.003 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 21:23:46.003 | INFO | Uploaded 36GiB to GPU in 2.54554s (14.1424GiB/s) | 20260507 21:23:46.003 | INFO | Downloaded 38GiB to GPU in 2.70074s (14.0702GiB/s) | 20260507 21:23:51.142 | INFO | Analyzing chunk frames = [2699776,2700288) | 20260507 21:23:51.142 | INFO | MemInfo: { | 20260507 21:23:51.142 | INFO | "gpu" : " 17 GB", | 20260507 21:23:51.143 | INFO | "huge" : "", | 20260507 21:23:51.143 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:51.143 | INFO | "malloc" : " 14 GB", | 20260507 21:23:51.143 | INFO | "pinned" : " 29 GB" | 20260507 21:23:51.143 | INFO | } | 20260507 21:23:51.143 | INFO | Perf report for the last 5.14007 seconds: | 20260507 21:23:51.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:51.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 21:23:51.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260507 21:23:51.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:51.143 | INFO | Uploaded 36GiB to GPU in 2.56677s (14.0254GiB/s) | 20260507 21:23:51.144 | INFO | Downloaded 34GiB to GPU in 2.48839s (13.6634GiB/s) | 20260507 21:23:56.281 | INFO | Analyzing chunk frames = [2700288,2700800) | 20260507 21:23:56.281 | INFO | MemInfo: { | 20260507 21:23:56.281 | INFO | "gpu" : " 17 GB", | 20260507 21:23:56.281 | INFO | "huge" : "", | 20260507 21:23:56.281 | INFO | "huge_shared" : " 29 GB", | 20260507 21:23:56.281 | INFO | "malloc" : " 14 GB", | 20260507 21:23:56.281 | INFO | "pinned" : " 29 GB" | 20260507 21:23:56.282 | INFO | } | 20260507 21:23:56.282 | INFO | Perf report for the last 5.13886 seconds: | 20260507 21:23:56.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:23:56.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:23:56.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 21:23:56.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:23:56.282 | INFO | Uploaded 36GiB to GPU in 2.45217s (14.6809GiB/s) | 20260507 21:23:56.282 | INFO | Downloaded 34GiB to GPU in 2.47748s (13.7236GiB/s) | 20260507 21:24:01.421 | INFO | Analyzing chunk frames = [2700800,2701312) | 20260507 21:24:01.421 | INFO | MemInfo: { | 20260507 21:24:01.421 | INFO | "gpu" : " 17 GB", | 20260507 21:24:01.421 | INFO | "huge" : "", | 20260507 21:24:01.421 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:01.421 | INFO | "malloc" : " 14 GB", | 20260507 21:24:01.421 | INFO | "pinned" : " 29 GB" | 20260507 21:24:01.421 | INFO | } | 20260507 21:24:01.421 | INFO | Perf report for the last 5.13918 seconds: | 20260507 21:24:01.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:01.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:24:01.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 21:24:01.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:01.421 | INFO | Uploaded 36GiB to GPU in 2.51196s (14.3315GiB/s) | 20260507 21:24:01.422 | INFO | Downloaded 34GiB to GPU in 2.48017s (13.7087GiB/s) | 20260507 21:24:06.559 | INFO | Analyzing chunk frames = [2701312,2701824) | 20260507 21:24:06.559 | INFO | MemInfo: { | 20260507 21:24:06.559 | INFO | "gpu" : " 17 GB", | 20260507 21:24:06.559 | INFO | "huge" : "", | 20260507 21:24:06.560 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:06.560 | INFO | "malloc" : " 14 GB", | 20260507 21:24:06.560 | INFO | "pinned" : " 29 GB" | 20260507 21:24:06.560 | INFO | } | 20260507 21:24:06.560 | INFO | Perf report for the last 5.13873 seconds: | 20260507 21:24:06.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:06.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:24:06.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 21:24:06.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:06.560 | INFO | Uploaded 36GiB to GPU in 2.3359s (15.4116GiB/s) | 20260507 21:24:06.560 | INFO | Downloaded 34GiB to GPU in 2.49232s (13.6419GiB/s) | 20260507 21:24:11.700 | INFO | Analyzing chunk frames = [2701824,2702336) | 20260507 21:24:11.700 | INFO | MemInfo: { | 20260507 21:24:11.700 | INFO | "gpu" : " 17 GB", | 20260507 21:24:11.700 | INFO | "huge" : "", | 20260507 21:24:11.700 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:11.700 | INFO | "malloc" : " 14 GB", | 20260507 21:24:11.700 | INFO | "pinned" : " 29 GB" | 20260507 21:24:11.700 | INFO | } | 20260507 21:24:11.700 | INFO | Perf report for the last 5.14012 seconds: | 20260507 21:24:11.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:11.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:24:11.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:24:11.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:11.700 | INFO | Uploaded 36GiB to GPU in 2.42358s (14.8541GiB/s) | 20260507 21:24:11.700 | INFO | Downloaded 34GiB to GPU in 2.38342s (14.2652GiB/s) | 20260507 21:24:16.839 | INFO | Analyzing chunk frames = [2702336,2702848) | 20260507 21:24:16.839 | INFO | MemInfo: { | 20260507 21:24:16.839 | INFO | "gpu" : " 17 GB", | 20260507 21:24:16.839 | INFO | "huge" : "", | 20260507 21:24:16.839 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:16.839 | INFO | "malloc" : " 14 GB", | 20260507 21:24:16.839 | INFO | "pinned" : " 29 GB" | 20260507 21:24:16.839 | INFO | } | 20260507 21:24:16.839 | INFO | Perf report for the last 5.1396 seconds: | 20260507 21:24:16.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:16.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.840 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:24:16.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.840 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:24:16.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:16.840 | INFO | Uploaded 36GiB to GPU in 2.54971s (14.1193GiB/s) | 20260507 21:24:16.840 | INFO | Downloaded 34GiB to GPU in 2.45154s (13.8688GiB/s) | 20260507 21:24:21.979 | INFO | Analyzing chunk frames = [2702848,2703360) | 20260507 21:24:21.979 | INFO | MemInfo: { | 20260507 21:24:21.979 | INFO | "gpu" : " 17 GB", | 20260507 21:24:21.979 | INFO | "huge" : "", | 20260507 21:24:21.979 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:21.979 | INFO | "malloc" : " 14 GB", | 20260507 21:24:21.979 | INFO | "pinned" : " 29 GB" | 20260507 21:24:21.979 | INFO | } | 20260507 21:24:21.979 | INFO | Perf report for the last 5.13975 seconds: | 20260507 21:24:21.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:21.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.979 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:24:21.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.979 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:24:21.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:21.980 | INFO | Uploaded 36GiB to GPU in 2.5379s (14.185GiB/s) | 20260507 21:24:21.980 | INFO | Downloaded 34GiB to GPU in 2.42739s (14.0068GiB/s) | 20260507 21:24:21.980 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:24:21.980 | INFO | Total mem usage: 16.7 GiB | 20260507 21:24:21.980 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:24:21.980 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.4%) | 20260507 21:24:21.980 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:24:21.980 | INFO | High Water Breakdown: | 20260507 21:24:21.980 | INFO | 60.2% : 9.11 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:24:21.980 | INFO | 39.6% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:24:21.980 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:24:21.980 | INFO | ------------------------------------------ | 20260507 21:24:21.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:24:21.980 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:24:21.980 | INFO | Total mem usage: 26.5 GiB | 20260507 21:24:21.980 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:24:21.980 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:24:21.980 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:24:21.980 | INFO | High Water Breakdown: | 20260507 21:24:21.980 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:24:21.980 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:24:21.980 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:24:21.980 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:24:21.980 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:24:21.980 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:24:21.980 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:24:21.980 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:24:21.980 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:24:21.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:24:21.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:24:21.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:24:21.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:24:21.980 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:24:21.980 | INFO | ------------------------------------------ | 20260507 21:24:21.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:24:21.980 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:24:21.980 | INFO | Total mem usage: 27.8 GiB | 20260507 21:24:21.980 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:24:21.980 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:24:21.980 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:24:21.980 | INFO | High Water Breakdown: | 20260507 21:24:21.980 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:24:21.980 | INFO | ------------------------------------------ | 20260507 21:24:21.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:24:21.980 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:24:21.980 | INFO | Total mem usage: 15.7 GiB | 20260507 21:24:21.980 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:24:21.980 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:24:21.980 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:24:21.980 | INFO | High Water Breakdown: | 20260507 21:24:21.980 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:24:21.981 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:24:21.981 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:24:21.981 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:24:21.981 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:24:21.981 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:24:21.981 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:24:21.981 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:24:21.981 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:24:21.981 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:24:21.981 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:24:21.981 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:24:21.981 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:24:21.981 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:24:21.981 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:24:21.981 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:24:21.981 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:24:21.981 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:24:21.981 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:24:21.981 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:24:21.981 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:24:21.981 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:24:21.981 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:24:21.981 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:24:21.981 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:24:21.981 | INFO | ------------------------------------------ | 20260507 21:24:21.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:24:21.981 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:24:21.981 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:24:21.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:24:21.981 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.42e+01 / 2.07e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.27e-02 / 6.04e-03 | 20260507 21:24:21.981 | INFO | PerfReport: 12.18%, Baseline -- 4.02e+00 / 4.48e+00, 5.93e-01 / 2.53e-01 | 20260507 21:24:21.981 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 1.06e-01 | 20260507 21:24:21.981 | INFO | PerfReport: 16.14%, DME -- 4.26e+01 / 4.62e+01, 4.46e+00 / 2.25e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 55.74%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.55e+00 / 1.46e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.51e+00 / 2.83e+00, 4.36e-01 / 2.24e-01 | 20260507 21:24:21.981 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.39e+00, 2.92e-01 / 1.59e-01 | 20260507 21:24:21.981 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.62e-01 / 3.98e-01, 5.90e-02 / 2.53e-02 | 20260507 21:24:21.981 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:24:21.981 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:24:21.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:24:21.981 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 24.29%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 30.82%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 2.18%, LockUpload -- 2.34e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 2.64%, LockDownload -- 2.83e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 15.83%, LockCompute -- 1.69e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 21:24:21.981 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260507 21:24:22.076 | INFO | Starting a round of Baz IO | 20260507 21:24:25.544 | INFO | Populating RTMetrics report | 20260507 21:24:25.545 | INFO | RTMetrics report populated | 20260507 21:24:25.545 | INFO | Opening tmp RTMetrics json file | 20260507 21:24:25.546 | INFO | Serializing RTMetrics to json | 20260507 21:24:25.568 | INFO | Serialization done, swapping tmp file now... | 20260507 21:24:25.570 | INFO | RTMetrics done | 20260507 21:24:25.652 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.575 seconds | 20260507 21:24:25.652 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.76% / 0.01% / 56.24% | 20260507 21:24:25.652 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67729% / 7.94729e-06% / 4.76837e-05% | 20260507 21:24:25.652 | INFO | Baz IO overall: | 20260507 21:24:25.652 | INFO | BytesWritten : 1230650491846 | 20260507 21:24:25.652 | INFO | NumEvents : 596467077652 | 20260507 21:24:27.117 | INFO | Analyzing chunk frames = [2703360,2703872) | 20260507 21:24:27.117 | INFO | MemInfo: { | 20260507 21:24:27.117 | INFO | "gpu" : " 17 GB", | 20260507 21:24:27.117 | INFO | "huge" : "", | 20260507 21:24:27.117 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:27.117 | INFO | "malloc" : " 14 GB", | 20260507 21:24:27.117 | INFO | "pinned" : " 29 GB" | 20260507 21:24:27.117 | INFO | } | 20260507 21:24:27.117 | INFO | Perf report for the last 5.13814 seconds: | 20260507 21:24:27.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:27.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:27.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:27.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:27.118 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 21:24:27.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:27.118 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 21:24:27.118 | INFO | PRE_HQ : 66.0%, 1.7, 57.7ms, 0 | 20260507 21:24:27.118 | INFO | BAZWRITER : 69.2%, 4.2, 148.8ms, 0 | 20260507 21:24:27.118 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 21:24:27.118 | INFO | Uploaded 36GiB to GPU in 2.53547s (14.1986GiB/s) | 20260507 21:24:27.118 | INFO | Downloaded 38GiB to GPU in 2.69383s (14.1063GiB/s) | 20260507 21:24:32.256 | INFO | Analyzing chunk frames = [2703872,2704384) | 20260507 21:24:32.256 | INFO | MemInfo: { | 20260507 21:24:32.256 | INFO | "gpu" : " 17 GB", | 20260507 21:24:32.256 | INFO | "huge" : "", | 20260507 21:24:32.256 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:32.256 | INFO | "malloc" : " 14 GB", | 20260507 21:24:32.256 | INFO | "pinned" : " 29 GB" | 20260507 21:24:32.256 | INFO | } | 20260507 21:24:32.257 | INFO | Perf report for the last 5.13913 seconds: | 20260507 21:24:32.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:32.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:24:32.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:24:32.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:32.257 | INFO | Uploaded 36GiB to GPU in 2.58264s (13.9392GiB/s) | 20260507 21:24:32.257 | INFO | Downloaded 34GiB to GPU in 2.49256s (13.6406GiB/s) | 20260507 21:24:37.397 | INFO | Analyzing chunk frames = [2704384,2704896) | 20260507 21:24:37.397 | INFO | MemInfo: { | 20260507 21:24:37.397 | INFO | "gpu" : " 17 GB", | 20260507 21:24:37.397 | INFO | "huge" : "", | 20260507 21:24:37.397 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:37.397 | INFO | "malloc" : " 14 GB", | 20260507 21:24:37.397 | INFO | "pinned" : " 29 GB" | 20260507 21:24:37.397 | INFO | } | 20260507 21:24:37.397 | INFO | Perf report for the last 5.14086 seconds: | 20260507 21:24:37.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:37.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.397 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:24:37.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.398 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:24:37.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:37.398 | INFO | Uploaded 36GiB to GPU in 2.5495s (14.1204GiB/s) | 20260507 21:24:37.398 | INFO | Downloaded 34GiB to GPU in 2.47029s (13.7635GiB/s) | 20260507 21:24:42.536 | INFO | Analyzing chunk frames = [2704896,2705408) | 20260507 21:24:42.536 | INFO | MemInfo: { | 20260507 21:24:42.536 | INFO | "gpu" : " 17 GB", | 20260507 21:24:42.536 | INFO | "huge" : "", | 20260507 21:24:42.536 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:42.536 | INFO | "malloc" : " 14 GB", | 20260507 21:24:42.536 | INFO | "pinned" : " 29 GB" | 20260507 21:24:42.536 | INFO | } | 20260507 21:24:42.536 | INFO | Perf report for the last 5.13845 seconds: | 20260507 21:24:42.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:42.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:24:42.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:24:42.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:42.536 | INFO | Uploaded 36GiB to GPU in 2.52041s (14.2834GiB/s) | 20260507 21:24:42.536 | INFO | Downloaded 34GiB to GPU in 2.47033s (13.7634GiB/s) | 20260507 21:24:47.675 | INFO | Analyzing chunk frames = [2705408,2705920) | 20260507 21:24:47.675 | INFO | MemInfo: { | 20260507 21:24:47.675 | INFO | "gpu" : " 17 GB", | 20260507 21:24:47.675 | INFO | "huge" : "", | 20260507 21:24:47.675 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:47.675 | INFO | "malloc" : " 14 GB", | 20260507 21:24:47.676 | INFO | "pinned" : " 29 GB" | 20260507 21:24:47.676 | INFO | } | 20260507 21:24:47.676 | INFO | Perf report for the last 5.1398 seconds: | 20260507 21:24:47.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:47.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:24:47.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:24:47.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:47.676 | INFO | Uploaded 36GiB to GPU in 2.43245s (14.7999GiB/s) | 20260507 21:24:47.676 | INFO | Downloaded 34GiB to GPU in 2.49929s (13.6039GiB/s) | 20260507 21:24:52.814 | INFO | Analyzing chunk frames = [2705920,2706432) | 20260507 21:24:52.814 | INFO | MemInfo: { | 20260507 21:24:52.814 | INFO | "gpu" : " 17 GB", | 20260507 21:24:52.814 | INFO | "huge" : "", | 20260507 21:24:52.814 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:52.814 | INFO | "malloc" : " 14 GB", | 20260507 21:24:52.814 | INFO | "pinned" : " 29 GB" | 20260507 21:24:52.815 | INFO | } | 20260507 21:24:52.815 | INFO | Perf report for the last 5.13884 seconds: | 20260507 21:24:52.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:52.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:24:52.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:24:52.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:52.815 | INFO | Uploaded 36GiB to GPU in 2.41606s (14.9003GiB/s) | 20260507 21:24:52.815 | INFO | Downloaded 34GiB to GPU in 2.37937s (14.2895GiB/s) | 20260507 21:24:57.953 | INFO | Analyzing chunk frames = [2706432,2706944) | 20260507 21:24:57.953 | INFO | MemInfo: { | 20260507 21:24:57.953 | INFO | "gpu" : " 17 GB", | 20260507 21:24:57.953 | INFO | "huge" : "", | 20260507 21:24:57.953 | INFO | "huge_shared" : " 29 GB", | 20260507 21:24:57.953 | INFO | "malloc" : " 14 GB", | 20260507 21:24:57.953 | INFO | "pinned" : " 29 GB" | 20260507 21:24:57.953 | INFO | } | 20260507 21:24:57.953 | INFO | Perf report for the last 5.1385 seconds: | 20260507 21:24:57.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:24:57.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:24:57.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:24:57.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:24:57.953 | INFO | Uploaded 36GiB to GPU in 2.55199s (14.1067GiB/s) | 20260507 21:24:57.954 | INFO | Downloaded 34GiB to GPU in 2.44227s (13.9215GiB/s) | 20260507 21:25:03.092 | INFO | Analyzing chunk frames = [2706944,2707456) | 20260507 21:25:03.092 | INFO | MemInfo: { | 20260507 21:25:03.092 | INFO | "gpu" : " 17 GB", | 20260507 21:25:03.092 | INFO | "huge" : "", | 20260507 21:25:03.092 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:03.092 | INFO | "malloc" : " 14 GB", | 20260507 21:25:03.093 | INFO | "pinned" : " 29 GB" | 20260507 21:25:03.093 | INFO | } | 20260507 21:25:03.093 | INFO | Perf report for the last 5.13965 seconds: | 20260507 21:25:03.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:03.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 21:25:03.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:25:03.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:03.093 | INFO | Uploaded 36GiB to GPU in 2.53881s (14.1799GiB/s) | 20260507 21:25:03.093 | INFO | Downloaded 34GiB to GPU in 2.43791s (13.9464GiB/s) | 20260507 21:25:03.191 | INFO | Starting a round of Baz IO | 20260507 21:25:06.664 | INFO | Populating RTMetrics report | 20260507 21:25:06.665 | INFO | RTMetrics report populated | 20260507 21:25:06.665 | INFO | Opening tmp RTMetrics json file | 20260507 21:25:06.687 | INFO | Serializing RTMetrics to json | 20260507 21:25:06.693 | INFO | Serialization done, swapping tmp file now... | 20260507 21:25:06.694 | INFO | RTMetrics done | 20260507 21:25:06.766 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.574 seconds | 20260507 21:25:06.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.75% / 0.01% / 56.24% | 20260507 21:25:06.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67958% / 7.94729e-06% / 7.54992e-05% | 20260507 21:25:06.766 | INFO | Baz IO overall: | 20260507 21:25:06.766 | INFO | BytesWritten : 1232538989994 | 20260507 21:25:06.766 | INFO | NumEvents : 597380073188 | 20260507 21:25:08.232 | INFO | Analyzing chunk frames = [2707456,2707968) | 20260507 21:25:08.232 | INFO | MemInfo: { | 20260507 21:25:08.232 | INFO | "gpu" : " 17 GB", | 20260507 21:25:08.232 | INFO | "huge" : "", | 20260507 21:25:08.232 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:08.232 | INFO | "malloc" : " 14 GB", | 20260507 21:25:08.232 | INFO | "pinned" : " 29 GB" | 20260507 21:25:08.233 | INFO | } | 20260507 21:25:08.233 | INFO | Perf report for the last 5.13979 seconds: | 20260507 21:25:08.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:08.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:08.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:08.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:08.233 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 21:25:08.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:08.233 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 21:25:08.233 | INFO | PRE_HQ : 66.0%, 1.7, 57.5ms, 0 | 20260507 21:25:08.233 | INFO | BAZWRITER : 69.2%, 3.8, 136.2ms, 0 | 20260507 21:25:08.233 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 21:25:08.233 | INFO | Uploaded 36GiB to GPU in 2.55251s (14.1037GiB/s) | 20260507 21:25:08.233 | INFO | Downloaded 38GiB to GPU in 2.6762s (14.1992GiB/s) | 20260507 21:25:13.370 | INFO | Analyzing chunk frames = [2707968,2708480) | 20260507 21:25:13.371 | INFO | MemInfo: { | 20260507 21:25:13.371 | INFO | "gpu" : " 17 GB", | 20260507 21:25:13.371 | INFO | "huge" : "", | 20260507 21:25:13.371 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:13.371 | INFO | "malloc" : " 14 GB", | 20260507 21:25:13.371 | INFO | "pinned" : " 29 GB" | 20260507 21:25:13.371 | INFO | } | 20260507 21:25:13.371 | INFO | Perf report for the last 5.13821 seconds: | 20260507 21:25:13.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:13.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:25:13.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 21:25:13.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:13.371 | INFO | Uploaded 36GiB to GPU in 2.58646s (13.9186GiB/s) | 20260507 21:25:13.371 | INFO | Downloaded 34GiB to GPU in 2.4911s (13.6486GiB/s) | 20260507 21:25:18.510 | INFO | Analyzing chunk frames = [2708480,2708992) | 20260507 21:25:18.511 | INFO | MemInfo: { | 20260507 21:25:18.511 | INFO | "gpu" : " 17 GB", | 20260507 21:25:18.511 | INFO | "huge" : "", | 20260507 21:25:18.511 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:18.511 | INFO | "malloc" : " 14 GB", | 20260507 21:25:18.511 | INFO | "pinned" : " 29 GB" | 20260507 21:25:18.511 | INFO | } | 20260507 21:25:18.511 | INFO | Perf report for the last 5.14002 seconds: | 20260507 21:25:18.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:18.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:25:18.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:25:18.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:18.511 | INFO | Uploaded 36GiB to GPU in 2.50106s (14.3939GiB/s) | 20260507 21:25:18.511 | INFO | Downloaded 34GiB to GPU in 2.47393s (13.7433GiB/s) | 20260507 21:25:23.651 | INFO | Analyzing chunk frames = [2708992,2709504) | 20260507 21:25:23.651 | INFO | MemInfo: { | 20260507 21:25:23.651 | INFO | "gpu" : " 17 GB", | 20260507 21:25:23.651 | INFO | "huge" : "", | 20260507 21:25:23.651 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:23.651 | INFO | "malloc" : " 14 GB", | 20260507 21:25:23.651 | INFO | "pinned" : " 29 GB" | 20260507 21:25:23.651 | INFO | } | 20260507 21:25:23.652 | INFO | Perf report for the last 5.14069 seconds: | 20260507 21:25:23.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:23.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 21:25:23.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:25:23.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:23.652 | INFO | Uploaded 36GiB to GPU in 2.51336s (14.3235GiB/s) | 20260507 21:25:23.652 | INFO | Downloaded 34GiB to GPU in 2.46175s (13.8113GiB/s) | 20260507 21:25:28.791 | INFO | Analyzing chunk frames = [2709504,2710016) | 20260507 21:25:28.791 | INFO | MemInfo: { | 20260507 21:25:28.791 | INFO | "gpu" : " 17 GB", | 20260507 21:25:28.791 | INFO | "huge" : "", | 20260507 21:25:28.791 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:28.791 | INFO | "malloc" : " 14 GB", | 20260507 21:25:28.791 | INFO | "pinned" : " 29 GB" | 20260507 21:25:28.791 | INFO | } | 20260507 21:25:28.791 | INFO | Perf report for the last 5.1397 seconds: | 20260507 21:25:28.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:28.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.791 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 21:25:28.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.791 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:25:28.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:28.792 | INFO | Uploaded 36GiB to GPU in 2.44464s (14.7261GiB/s) | 20260507 21:25:28.792 | INFO | Downloaded 34GiB to GPU in 2.47566s (13.7337GiB/s) | 20260507 21:25:33.928 | INFO | Analyzing chunk frames = [2710016,2710528) | 20260507 21:25:33.928 | INFO | MemInfo: { | 20260507 21:25:33.928 | INFO | "gpu" : " 17 GB", | 20260507 21:25:33.928 | INFO | "huge" : "", | 20260507 21:25:33.928 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:33.928 | INFO | "malloc" : " 14 GB", | 20260507 21:25:33.928 | INFO | "pinned" : " 29 GB" | 20260507 21:25:33.929 | INFO | } | 20260507 21:25:33.929 | INFO | Perf report for the last 5.1375 seconds: | 20260507 21:25:33.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:33.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:25:33.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:25:33.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:33.929 | INFO | Uploaded 36GiB to GPU in 2.40639s (14.9602GiB/s) | 20260507 21:25:33.929 | INFO | Downloaded 34GiB to GPU in 2.38762s (14.2401GiB/s) | 20260507 21:25:39.068 | INFO | Analyzing chunk frames = [2710528,2711040) | 20260507 21:25:39.068 | INFO | MemInfo: { | 20260507 21:25:39.068 | INFO | "gpu" : " 17 GB", | 20260507 21:25:39.068 | INFO | "huge" : "", | 20260507 21:25:39.068 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:39.068 | INFO | "malloc" : " 14 GB", | 20260507 21:25:39.068 | INFO | "pinned" : " 29 GB" | 20260507 21:25:39.068 | INFO | } | 20260507 21:25:39.068 | INFO | Perf report for the last 5.13916 seconds: | 20260507 21:25:39.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:39.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:25:39.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:25:39.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:39.068 | INFO | Uploaded 36GiB to GPU in 2.54759s (14.131GiB/s) | 20260507 21:25:39.068 | INFO | Downloaded 34GiB to GPU in 2.45457s (13.8517GiB/s) | 20260507 21:25:44.207 | INFO | Analyzing chunk frames = [2711040,2711552) | 20260507 21:25:44.207 | INFO | MemInfo: { | 20260507 21:25:44.207 | INFO | "gpu" : " 17 GB", | 20260507 21:25:44.207 | INFO | "huge" : "", | 20260507 21:25:44.207 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:44.207 | INFO | "malloc" : " 14 GB", | 20260507 21:25:44.207 | INFO | "pinned" : " 29 GB" | 20260507 21:25:44.207 | INFO | } | 20260507 21:25:44.207 | INFO | Perf report for the last 5.13938 seconds: | 20260507 21:25:44.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:44.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.207 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:25:44.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.207 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:25:44.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:44.208 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:25:44.208 | INFO | Total mem usage: 16.7 GiB | 20260507 21:25:44.208 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:25:44.208 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:25:44.208 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:25:44.208 | INFO | High Water Breakdown: | 20260507 21:25:44.208 | INFO | 59.9% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:25:44.208 | INFO | 40% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:25:44.208 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:25:44.208 | INFO | ------------------------------------------ | 20260507 21:25:44.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:25:44.208 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:25:44.208 | INFO | Total mem usage: 26.5 GiB | 20260507 21:25:44.208 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:25:44.208 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:25:44.208 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:25:44.208 | INFO | High Water Breakdown: | 20260507 21:25:44.208 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:25:44.208 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:25:44.208 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:25:44.208 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:25:44.208 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:25:44.208 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:25:44.208 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:25:44.208 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:25:44.208 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:25:44.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:25:44.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:25:44.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:25:44.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:25:44.208 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:25:44.208 | INFO | ------------------------------------------ | 20260507 21:25:44.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:25:44.208 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:25:44.208 | INFO | Total mem usage: 27.8 GiB | 20260507 21:25:44.208 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:25:44.208 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:25:44.208 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:25:44.208 | INFO | High Water Breakdown: | 20260507 21:25:44.208 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:25:44.208 | INFO | ------------------------------------------ | 20260507 21:25:44.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:25:44.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:25:44.208 | INFO | Total mem usage: 15.7 GiB | 20260507 21:25:44.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:25:44.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:25:44.208 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:25:44.208 | INFO | High Water Breakdown: | 20260507 21:25:44.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:25:44.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:25:44.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:25:44.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:25:44.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:25:44.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:25:44.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:25:44.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:25:44.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:25:44.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:25:44.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:25:44.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:25:44.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:25:44.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:25:44.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:25:44.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:25:44.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:25:44.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:25:44.208 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:25:44.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:25:44.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:25:44.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:25:44.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:25:44.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:25:44.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:25:44.208 | INFO | ------------------------------------------ | 20260507 21:25:44.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:25:44.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:25:44.208 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:25:44.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:25:44.208 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.59e+01, 1.43e+01 / 1.77e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.01e-01, 1.17e-02 / 4.86e-03 | 20260507 21:25:44.208 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.49e+00, 5.98e-01 / 2.08e-01 | 20260507 21:25:44.208 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.08e-01 | 20260507 21:25:44.208 | INFO | PerfReport: 16.12%, DME -- 4.25e+01 / 4.60e+01, 4.13e+00 / 1.98e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 55.83%, FrameLabeling -- 1.84e+01 / 1.92e+01, 1.57e+00 / 1.36e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.40e-01 / 1.95e-01 | 20260507 21:25:44.208 | INFO | PerfReport: 3.54%, Metrics -- 1.16e+00 / 1.41e+00, 2.95e-01 / 1.64e-01 | 20260507 21:25:44.208 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 4.00e-01, 5.94e-02 / 2.95e-02 | 20260507 21:25:44.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:25:44.208 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:25:44.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:25:44.208 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 24.44%, Upload -- 2.62e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 24.17%, Download -- 2.59e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 30.76%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 2.38%, LockUpload -- 2.55e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 2.49%, LockDownload -- 2.66e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 15.75%, LockCompute -- 1.69e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 21:25:44.208 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 1.01e-02 / 0.00e+00 | 20260507 21:25:44.208 | INFO | Uploaded 36GiB to GPU in 2.49858s (14.4082GiB/s) | 20260507 21:25:44.208 | INFO | Downloaded 34GiB to GPU in 2.43381s (13.9699GiB/s) | 20260507 21:25:44.305 | INFO | Starting a round of Baz IO | 20260507 21:25:47.779 | INFO | Populating RTMetrics report | 20260507 21:25:47.779 | INFO | RTMetrics report populated | 20260507 21:25:47.779 | INFO | Opening tmp RTMetrics json file | 20260507 21:25:47.804 | INFO | Serializing RTMetrics to json | 20260507 21:25:47.806 | INFO | Serialization done, swapping tmp file now... | 20260507 21:25:47.807 | INFO | RTMetrics done | 20260507 21:25:47.904 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.599 seconds | 20260507 21:25:47.904 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.74% / 0.01% / 56.25% | 20260507 21:25:47.904 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6754% / 2.38419e-05% / 6.75519e-05% | 20260507 21:25:47.904 | INFO | Baz IO overall: | 20260507 21:25:47.904 | INFO | BytesWritten : 1234425352415 | 20260507 21:25:47.904 | INFO | NumEvents : 598291513393 | 20260507 21:25:49.347 | INFO | Analyzing chunk frames = [2711552,2712064) | 20260507 21:25:49.347 | INFO | MemInfo: { | 20260507 21:25:49.347 | INFO | "gpu" : " 17 GB", | 20260507 21:25:49.347 | INFO | "huge" : "", | 20260507 21:25:49.347 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:49.347 | INFO | "malloc" : " 14 GB", | 20260507 21:25:49.347 | INFO | "pinned" : " 29 GB" | 20260507 21:25:49.347 | INFO | } | 20260507 21:25:49.347 | INFO | Perf report for the last 5.13928 seconds: | 20260507 21:25:49.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:49.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:49.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:49.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:49.347 | INFO | ANALYSIS : 68.7%, 3.0, 107.3ms, 0 | 20260507 21:25:49.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:49.347 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 21:25:49.347 | INFO | PRE_HQ : 66.5%, 1.8, 62.0ms, 0 | 20260507 21:25:49.347 | INFO | BAZWRITER : 69.6%, 3.7, 133.9ms, 0 | 20260507 21:25:49.347 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 21:25:49.347 | INFO | Uploaded 36GiB to GPU in 2.54523s (14.1441GiB/s) | 20260507 21:25:49.347 | INFO | Downloaded 38GiB to GPU in 2.69005s (14.1261GiB/s) | 20260507 21:25:54.486 | INFO | Analyzing chunk frames = [2712064,2712576) | 20260507 21:25:54.487 | INFO | MemInfo: { | 20260507 21:25:54.487 | INFO | "gpu" : " 17 GB", | 20260507 21:25:54.487 | INFO | "huge" : "", | 20260507 21:25:54.487 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:54.487 | INFO | "malloc" : " 14 GB", | 20260507 21:25:54.487 | INFO | "pinned" : " 29 GB" | 20260507 21:25:54.487 | INFO | } | 20260507 21:25:54.487 | INFO | Perf report for the last 5.13998 seconds: | 20260507 21:25:54.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:54.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:25:54.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:25:54.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:54.487 | INFO | Uploaded 36GiB to GPU in 2.55598s (14.0846GiB/s) | 20260507 21:25:54.487 | INFO | Downloaded 34GiB to GPU in 2.50504s (13.5726GiB/s) | 20260507 21:25:59.626 | INFO | Analyzing chunk frames = [2712576,2713088) | 20260507 21:25:59.626 | INFO | MemInfo: { | 20260507 21:25:59.626 | INFO | "gpu" : " 17 GB", | 20260507 21:25:59.626 | INFO | "huge" : "", | 20260507 21:25:59.626 | INFO | "huge_shared" : " 29 GB", | 20260507 21:25:59.626 | INFO | "malloc" : " 14 GB", | 20260507 21:25:59.626 | INFO | "pinned" : " 29 GB" | 20260507 21:25:59.626 | INFO | } | 20260507 21:25:59.626 | INFO | Perf report for the last 5.13919 seconds: | 20260507 21:25:59.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:25:59.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 21:25:59.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:25:59.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:25:59.626 | INFO | Uploaded 36GiB to GPU in 2.48075s (14.5117GiB/s) | 20260507 21:25:59.626 | INFO | Downloaded 34GiB to GPU in 2.48221s (13.6975GiB/s) | 20260507 21:26:04.765 | INFO | Analyzing chunk frames = [2713088,2713600) | 20260507 21:26:04.765 | INFO | MemInfo: { | 20260507 21:26:04.765 | INFO | "gpu" : " 17 GB", | 20260507 21:26:04.765 | INFO | "huge" : "", | 20260507 21:26:04.765 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:04.765 | INFO | "malloc" : " 14 GB", | 20260507 21:26:04.765 | INFO | "pinned" : " 29 GB" | 20260507 21:26:04.765 | INFO | } | 20260507 21:26:04.765 | INFO | Perf report for the last 5.13968 seconds: | 20260507 21:26:04.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:04.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.766 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:26:04.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.766 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:26:04.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:04.766 | INFO | Uploaded 36GiB to GPU in 2.48414s (14.4919GiB/s) | 20260507 21:26:04.766 | INFO | Downloaded 34GiB to GPU in 2.41072s (14.1036GiB/s) | 20260507 21:26:09.905 | INFO | Analyzing chunk frames = [2713600,2714112) | 20260507 21:26:09.905 | INFO | MemInfo: { | 20260507 21:26:09.905 | INFO | "gpu" : " 17 GB", | 20260507 21:26:09.905 | INFO | "huge" : "", | 20260507 21:26:09.905 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:09.905 | INFO | "malloc" : " 14 GB", | 20260507 21:26:09.905 | INFO | "pinned" : " 29 GB" | 20260507 21:26:09.905 | INFO | } | 20260507 21:26:09.905 | INFO | Perf report for the last 5.13972 seconds: | 20260507 21:26:09.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:09.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.905 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 21:26:09.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.905 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 21:26:09.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:09.906 | INFO | Uploaded 36GiB to GPU in 2.39865s (15.0085GiB/s) | 20260507 21:26:09.906 | INFO | Downloaded 34GiB to GPU in 2.49176s (13.645GiB/s) | 20260507 21:26:15.043 | INFO | Analyzing chunk frames = [2714112,2714624) | 20260507 21:26:15.043 | INFO | MemInfo: { | 20260507 21:26:15.043 | INFO | "gpu" : " 17 GB", | 20260507 21:26:15.043 | INFO | "huge" : "", | 20260507 21:26:15.043 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:15.043 | INFO | "malloc" : " 14 GB", | 20260507 21:26:15.043 | INFO | "pinned" : " 29 GB" | 20260507 21:26:15.043 | INFO | } | 20260507 21:26:15.043 | INFO | Perf report for the last 5.13778 seconds: | 20260507 21:26:15.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:15.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:26:15.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:26:15.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:15.043 | INFO | Uploaded 36GiB to GPU in 2.39148s (15.0534GiB/s) | 20260507 21:26:15.043 | INFO | Downloaded 34GiB to GPU in 2.40093s (14.1612GiB/s) | 20260507 21:26:20.183 | INFO | Analyzing chunk frames = [2714624,2715136) | 20260507 21:26:20.183 | INFO | MemInfo: { | 20260507 21:26:20.183 | INFO | "gpu" : " 17 GB", | 20260507 21:26:20.183 | INFO | "huge" : "", | 20260507 21:26:20.183 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:20.183 | INFO | "malloc" : " 14 GB", | 20260507 21:26:20.183 | INFO | "pinned" : " 29 GB" | 20260507 21:26:20.183 | INFO | } | 20260507 21:26:20.183 | INFO | Perf report for the last 5.13999 seconds: | 20260507 21:26:20.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:20.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.183 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:26:20.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.183 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:26:20.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:20.184 | INFO | Uploaded 36GiB to GPU in 2.52777s (14.2418GiB/s) | 20260507 21:26:20.184 | INFO | Downloaded 34GiB to GPU in 2.45641s (13.8414GiB/s) | 20260507 21:26:25.322 | INFO | Analyzing chunk frames = [2715136,2715648) | 20260507 21:26:25.322 | INFO | MemInfo: { | 20260507 21:26:25.322 | INFO | "gpu" : " 17 GB", | 20260507 21:26:25.322 | INFO | "huge" : "", | 20260507 21:26:25.322 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:25.322 | INFO | "malloc" : " 14 GB", | 20260507 21:26:25.322 | INFO | "pinned" : " 29 GB" | 20260507 21:26:25.322 | INFO | } | 20260507 21:26:25.322 | INFO | Perf report for the last 5.13908 seconds: | 20260507 21:26:25.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:25.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.322 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:26:25.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.322 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:26:25.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:25.323 | INFO | Uploaded 36GiB to GPU in 2.4903s (14.4561GiB/s) | 20260507 21:26:25.323 | INFO | Downloaded 34GiB to GPU in 2.43418s (13.9677GiB/s) | 20260507 21:26:25.420 | INFO | Starting a round of Baz IO | 20260507 21:26:28.901 | INFO | Populating RTMetrics report | 20260507 21:26:28.901 | INFO | RTMetrics report populated | 20260507 21:26:28.901 | INFO | Opening tmp RTMetrics json file | 20260507 21:26:28.951 | INFO | Serializing RTMetrics to json | 20260507 21:26:29.025 | INFO | Serialization done, swapping tmp file now... | 20260507 21:26:29.029 | INFO | RTMetrics done | 20260507 21:26:29.053 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.633 seconds | 20260507 21:26:29.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.73% / 0.01% / 56.26% | 20260507 21:26:29.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67669% / 1.98682e-05% / 5.96046e-05% | 20260507 21:26:29.054 | INFO | Baz IO overall: | 20260507 21:26:29.054 | INFO | BytesWritten : 1236312175387 | 20260507 21:26:29.054 | INFO | NumEvents : 599203182360 | 20260507 21:26:30.462 | INFO | Analyzing chunk frames = [2715648,2716160) | 20260507 21:26:30.462 | INFO | MemInfo: { | 20260507 21:26:30.462 | INFO | "gpu" : " 17 GB", | 20260507 21:26:30.462 | INFO | "huge" : "", | 20260507 21:26:30.462 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:30.462 | INFO | "malloc" : " 14 GB", | 20260507 21:26:30.462 | INFO | "pinned" : " 29 GB" | 20260507 21:26:30.462 | INFO | } | 20260507 21:26:30.463 | INFO | Perf report for the last 5.14053 seconds: | 20260507 21:26:30.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:30.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:30.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:30.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:30.463 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 21:26:30.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:30.463 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 21:26:30.463 | INFO | PRE_HQ : 65.9%, 1.7, 57.6ms, 0 | 20260507 21:26:30.463 | INFO | BAZWRITER : 70.1%, 3.8, 138.5ms, 0 | 20260507 21:26:30.463 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260507 21:26:30.463 | INFO | Uploaded 36GiB to GPU in 2.54424s (14.1496GiB/s) | 20260507 21:26:30.463 | INFO | Downloaded 38GiB to GPU in 2.68714s (14.1415GiB/s) | 20260507 21:26:35.601 | INFO | Analyzing chunk frames = [2716160,2716672) | 20260507 21:26:35.601 | INFO | MemInfo: { | 20260507 21:26:35.601 | INFO | "gpu" : " 17 GB", | 20260507 21:26:35.601 | INFO | "huge" : "", | 20260507 21:26:35.601 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:35.601 | INFO | "malloc" : " 14 GB", | 20260507 21:26:35.601 | INFO | "pinned" : " 29 GB" | 20260507 21:26:35.601 | INFO | } | 20260507 21:26:35.602 | INFO | Perf report for the last 5.13883 seconds: | 20260507 21:26:35.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:35.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:26:35.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:26:35.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:35.602 | INFO | Uploaded 36GiB to GPU in 2.55691s (14.0795GiB/s) | 20260507 21:26:35.602 | INFO | Downloaded 34GiB to GPU in 2.4787s (13.7169GiB/s) | 20260507 21:26:40.740 | INFO | Analyzing chunk frames = [2716672,2717184) | 20260507 21:26:40.740 | INFO | MemInfo: { | 20260507 21:26:40.740 | INFO | "gpu" : " 17 GB", | 20260507 21:26:40.740 | INFO | "huge" : "", | 20260507 21:26:40.740 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:40.741 | INFO | "malloc" : " 14 GB", | 20260507 21:26:40.741 | INFO | "pinned" : " 29 GB" | 20260507 21:26:40.741 | INFO | } | 20260507 21:26:40.741 | INFO | Perf report for the last 5.13909 seconds: | 20260507 21:26:40.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:40.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:26:40.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:26:40.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:40.741 | INFO | Uploaded 36GiB to GPU in 2.47702s (14.5336GiB/s) | 20260507 21:26:40.741 | INFO | Downloaded 34GiB to GPU in 2.48519s (13.6811GiB/s) | 20260507 21:26:45.879 | INFO | Analyzing chunk frames = [2717184,2717696) | 20260507 21:26:45.879 | INFO | MemInfo: { | 20260507 21:26:45.879 | INFO | "gpu" : " 17 GB", | 20260507 21:26:45.879 | INFO | "huge" : "", | 20260507 21:26:45.879 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:45.879 | INFO | "malloc" : " 14 GB", | 20260507 21:26:45.879 | INFO | "pinned" : " 29 GB" | 20260507 21:26:45.879 | INFO | } | 20260507 21:26:45.879 | INFO | Perf report for the last 5.1388 seconds: | 20260507 21:26:45.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:45.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.879 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:26:45.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.880 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 21:26:45.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:45.880 | INFO | Uploaded 36GiB to GPU in 2.51035s (14.3406GiB/s) | 20260507 21:26:45.880 | INFO | Downloaded 34GiB to GPU in 2.47893s (13.7156GiB/s) | 20260507 21:26:51.019 | INFO | Analyzing chunk frames = [2717696,2718208) | 20260507 21:26:51.019 | INFO | MemInfo: { | 20260507 21:26:51.019 | INFO | "gpu" : " 17 GB", | 20260507 21:26:51.019 | INFO | "huge" : "", | 20260507 21:26:51.019 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:51.019 | INFO | "malloc" : " 14 GB", | 20260507 21:26:51.019 | INFO | "pinned" : " 29 GB" | 20260507 21:26:51.019 | INFO | } | 20260507 21:26:51.020 | INFO | Perf report for the last 5.14008 seconds: | 20260507 21:26:51.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:51.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 21:26:51.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:26:51.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:51.020 | INFO | Uploaded 36GiB to GPU in 2.39421s (15.0363GiB/s) | 20260507 21:26:51.020 | INFO | Downloaded 34GiB to GPU in 2.48206s (13.6983GiB/s) | 20260507 21:26:56.158 | INFO | Analyzing chunk frames = [2718208,2718720) | 20260507 21:26:56.158 | INFO | MemInfo: { | 20260507 21:26:56.158 | INFO | "gpu" : " 17 GB", | 20260507 21:26:56.158 | INFO | "huge" : "", | 20260507 21:26:56.158 | INFO | "huge_shared" : " 29 GB", | 20260507 21:26:56.158 | INFO | "malloc" : " 14 GB", | 20260507 21:26:56.158 | INFO | "pinned" : " 29 GB" | 20260507 21:26:56.159 | INFO | } | 20260507 21:26:56.159 | INFO | Perf report for the last 5.13921 seconds: | 20260507 21:26:56.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:26:56.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 21:26:56.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:26:56.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:26:56.159 | INFO | Uploaded 36GiB to GPU in 2.43505s (14.7841GiB/s) | 20260507 21:26:56.159 | INFO | Downloaded 34GiB to GPU in 2.37608s (14.3093GiB/s) | 20260507 21:27:01.299 | INFO | Analyzing chunk frames = [2718720,2719232) | 20260507 21:27:01.299 | INFO | MemInfo: { | 20260507 21:27:01.299 | INFO | "gpu" : " 17 GB", | 20260507 21:27:01.299 | INFO | "huge" : "", | 20260507 21:27:01.299 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:01.299 | INFO | "malloc" : " 14 GB", | 20260507 21:27:01.299 | INFO | "pinned" : " 29 GB" | 20260507 21:27:01.299 | INFO | } | 20260507 21:27:01.299 | INFO | Perf report for the last 5.14033 seconds: | 20260507 21:27:01.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:01.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.299 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 21:27:01.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.299 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:27:01.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:01.300 | INFO | Uploaded 36GiB to GPU in 2.54928s (14.1216GiB/s) | 20260507 21:27:01.300 | INFO | Downloaded 34GiB to GPU in 2.43853s (13.9428GiB/s) | 20260507 21:27:06.439 | INFO | Analyzing chunk frames = [2719232,2719744) | 20260507 21:27:06.439 | INFO | MemInfo: { | 20260507 21:27:06.439 | INFO | "gpu" : " 17 GB", | 20260507 21:27:06.439 | INFO | "huge" : "", | 20260507 21:27:06.439 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:06.439 | INFO | "malloc" : " 14 GB", | 20260507 21:27:06.439 | INFO | "pinned" : " 29 GB" | 20260507 21:27:06.439 | INFO | } | 20260507 21:27:06.439 | INFO | Perf report for the last 5.13987 seconds: | 20260507 21:27:06.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:06.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 21:27:06.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:27:06.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:06.439 | INFO | Uploaded 36GiB to GPU in 2.55768s (14.0753GiB/s) | 20260507 21:27:06.439 | INFO | Downloaded 34GiB to GPU in 2.43757s (13.9483GiB/s) | 20260507 21:27:06.440 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:27:06.440 | INFO | Total mem usage: 16.7 GiB | 20260507 21:27:06.440 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:27:06.440 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260507 21:27:06.440 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:27:06.440 | INFO | High Water Breakdown: | 20260507 21:27:06.440 | INFO | 59.7% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:27:06.440 | INFO | 40.2% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:27:06.440 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:27:06.440 | INFO | ------------------------------------------ | 20260507 21:27:06.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:27:06.440 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:27:06.440 | INFO | Total mem usage: 26.5 GiB | 20260507 21:27:06.440 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:27:06.440 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:27:06.440 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:27:06.440 | INFO | High Water Breakdown: | 20260507 21:27:06.440 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:27:06.440 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:27:06.440 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:27:06.440 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:27:06.440 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:27:06.440 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:27:06.440 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:27:06.440 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:27:06.440 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:27:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:27:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:27:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:27:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:27:06.440 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:27:06.440 | INFO | ------------------------------------------ | 20260507 21:27:06.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:27:06.440 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:27:06.440 | INFO | Total mem usage: 27.8 GiB | 20260507 21:27:06.440 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:27:06.440 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:27:06.440 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:27:06.440 | INFO | High Water Breakdown: | 20260507 21:27:06.440 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:27:06.440 | INFO | ------------------------------------------ | 20260507 21:27:06.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:27:06.440 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:27:06.440 | INFO | Total mem usage: 15.7 GiB | 20260507 21:27:06.440 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:27:06.440 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:27:06.440 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:27:06.440 | INFO | High Water Breakdown: | 20260507 21:27:06.440 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:27:06.440 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:27:06.440 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:27:06.440 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:27:06.440 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:27:06.440 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:27:06.440 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:27:06.440 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:27:06.440 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:27:06.440 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:27:06.440 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:27:06.440 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:27:06.440 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:27:06.440 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:27:06.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:27:06.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:27:06.440 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:27:06.440 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:27:06.440 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:27:06.440 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:27:06.441 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:27:06.441 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:27:06.441 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:27:06.441 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:27:06.441 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:27:06.441 | INFO | ------------------------------------------ | 20260507 21:27:06.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:27:06.441 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:27:06.441 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:27:06.441 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:27:06.441 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.42e+01 / 1.69e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.13e-02 / 4.81e-03 | 20260507 21:27:06.441 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.50e+00, 5.93e-01 / 2.31e-01 | 20260507 21:27:06.441 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.50e+00, 2.07e-01 / 1.16e-01 | 20260507 21:27:06.441 | INFO | PerfReport: 16.00%, DME -- 4.22e+01 / 4.57e+01, 4.20e+00 / 1.79e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.65e+00 / 1.36e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.50e+00 / 2.83e+00, 4.35e-01 / 1.75e-01 | 20260507 21:27:06.441 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.40e+00, 2.93e-01 / 1.47e-01 | 20260507 21:27:06.441 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.96e-01, 6.02e-02 / 2.64e-02 | 20260507 21:27:06.441 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:27:06.441 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:27:06.441 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:27:06.441 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 24.31%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 30.85%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 2.06%, LockUpload -- 2.20e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 21:27:06.441 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260507 21:27:06.537 | INFO | Starting a round of Baz IO | 20260507 21:27:10.004 | INFO | Populating RTMetrics report | 20260507 21:27:10.004 | INFO | RTMetrics report populated | 20260507 21:27:10.004 | INFO | Opening tmp RTMetrics json file | 20260507 21:27:10.034 | INFO | Serializing RTMetrics to json | 20260507 21:27:10.075 | INFO | Serialization done, swapping tmp file now... | 20260507 21:27:10.094 | INFO | RTMetrics done | 20260507 21:27:10.124 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.587 seconds | 20260507 21:27:10.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.73% / 0.01% / 56.27% | 20260507 21:27:10.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67007% / 7.94729e-06% / 6.75519e-05% | 20260507 21:27:10.124 | INFO | Baz IO overall: | 20260507 21:27:10.124 | INFO | BytesWritten : 1238197847753 | 20260507 21:27:10.124 | INFO | NumEvents : 600114265625 | 20260507 21:27:11.576 | INFO | Analyzing chunk frames = [2719744,2720256) | 20260507 21:27:11.576 | INFO | MemInfo: { | 20260507 21:27:11.576 | INFO | "gpu" : " 17 GB", | 20260507 21:27:11.576 | INFO | "huge" : "", | 20260507 21:27:11.576 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:11.576 | INFO | "malloc" : " 14 GB", | 20260507 21:27:11.576 | INFO | "pinned" : " 29 GB" | 20260507 21:27:11.576 | INFO | } | 20260507 21:27:11.576 | INFO | Perf report for the last 5.13708 seconds: | 20260507 21:27:11.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:11.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:11.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:11.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:11.576 | INFO | ANALYSIS : 68.7%, 3.0, 106.4ms, 0 | 20260507 21:27:11.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:11.576 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 21:27:11.576 | INFO | PRE_HQ : 65.9%, 1.7, 57.5ms, 0 | 20260507 21:27:11.576 | INFO | BAZWRITER : 69.4%, 3.9, 139.9ms, 0 | 20260507 21:27:11.576 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 21:27:11.577 | INFO | Uploaded 36GiB to GPU in 2.55143s (14.1098GiB/s) | 20260507 21:27:11.577 | INFO | Downloaded 38GiB to GPU in 2.67905s (14.1841GiB/s) | 20260507 21:27:16.715 | INFO | Analyzing chunk frames = [2720256,2720768) | 20260507 21:27:16.715 | INFO | MemInfo: { | 20260507 21:27:16.715 | INFO | "gpu" : " 17 GB", | 20260507 21:27:16.715 | INFO | "huge" : "", | 20260507 21:27:16.715 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:16.715 | INFO | "malloc" : " 14 GB", | 20260507 21:27:16.715 | INFO | "pinned" : " 29 GB" | 20260507 21:27:16.716 | INFO | } | 20260507 21:27:16.716 | INFO | Perf report for the last 5.13964 seconds: | 20260507 21:27:16.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:16.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 21:27:16.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:27:16.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:16.716 | INFO | Uploaded 36GiB to GPU in 2.58488s (13.9271GiB/s) | 20260507 21:27:16.716 | INFO | Downloaded 34GiB to GPU in 2.47203s (13.7539GiB/s) | 20260507 21:27:21.855 | INFO | Analyzing chunk frames = [2720768,2721280) | 20260507 21:27:21.855 | INFO | MemInfo: { | 20260507 21:27:21.855 | INFO | "gpu" : " 17 GB", | 20260507 21:27:21.855 | INFO | "huge" : "", | 20260507 21:27:21.855 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:21.855 | INFO | "malloc" : " 14 GB", | 20260507 21:27:21.855 | INFO | "pinned" : " 29 GB" | 20260507 21:27:21.855 | INFO | } | 20260507 21:27:21.855 | INFO | Perf report for the last 5.13972 seconds: | 20260507 21:27:21.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:21.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.856 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:27:21.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.856 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 21:27:21.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:21.856 | INFO | Uploaded 36GiB to GPU in 2.55141s (14.1098GiB/s) | 20260507 21:27:21.856 | INFO | Downloaded 34GiB to GPU in 2.47006s (13.7648GiB/s) | 20260507 21:27:26.995 | INFO | Analyzing chunk frames = [2721280,2721792) | 20260507 21:27:26.995 | INFO | MemInfo: { | 20260507 21:27:26.995 | INFO | "gpu" : " 17 GB", | 20260507 21:27:26.995 | INFO | "huge" : "", | 20260507 21:27:26.995 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:26.995 | INFO | "malloc" : " 14 GB", | 20260507 21:27:26.995 | INFO | "pinned" : " 29 GB" | 20260507 21:27:26.995 | INFO | } | 20260507 21:27:26.995 | INFO | Perf report for the last 5.13976 seconds: | 20260507 21:27:26.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:26.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.995 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 21:27:26.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.996 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 21:27:26.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:26.996 | INFO | Uploaded 36GiB to GPU in 2.55373s (14.097GiB/s) | 20260507 21:27:26.996 | INFO | Downloaded 34GiB to GPU in 2.47361s (13.7451GiB/s) | 20260507 21:27:32.136 | INFO | Analyzing chunk frames = [2721792,2722304) | 20260507 21:27:32.136 | INFO | MemInfo: { | 20260507 21:27:32.136 | INFO | "gpu" : " 17 GB", | 20260507 21:27:32.136 | INFO | "huge" : "", | 20260507 21:27:32.136 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:32.136 | INFO | "malloc" : " 14 GB", | 20260507 21:27:32.136 | INFO | "pinned" : " 29 GB" | 20260507 21:27:32.136 | INFO | } | 20260507 21:27:32.136 | INFO | Perf report for the last 5.1407 seconds: | 20260507 21:27:32.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:32.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.136 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 21:27:32.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.136 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 21:27:32.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:32.137 | INFO | Uploaded 36GiB to GPU in 2.42976s (14.8163GiB/s) | 20260507 21:27:32.137 | INFO | Downloaded 34GiB to GPU in 2.47762s (13.7229GiB/s) | 20260507 21:27:37.273 | INFO | Analyzing chunk frames = [2722304,2722816) | 20260507 21:27:37.273 | INFO | MemInfo: { | 20260507 21:27:37.273 | INFO | "gpu" : " 17 GB", | 20260507 21:27:37.273 | INFO | "huge" : "", | 20260507 21:27:37.273 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:37.273 | INFO | "malloc" : " 14 GB", | 20260507 21:27:37.273 | INFO | "pinned" : " 29 GB" | 20260507 21:27:37.273 | INFO | } | 20260507 21:27:37.273 | INFO | Perf report for the last 5.13746 seconds: | 20260507 21:27:37.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:37.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.273 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 21:27:37.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.274 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:27:37.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:37.274 | INFO | Uploaded 36GiB to GPU in 2.40325s (14.9797GiB/s) | 20260507 21:27:37.274 | INFO | Downloaded 34GiB to GPU in 2.38971s (14.2277GiB/s) | 20260507 21:27:42.412 | INFO | Analyzing chunk frames = [2722816,2723328) | 20260507 21:27:42.412 | INFO | MemInfo: { | 20260507 21:27:42.412 | INFO | "gpu" : " 17 GB", | 20260507 21:27:42.412 | INFO | "huge" : "", | 20260507 21:27:42.412 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:42.412 | INFO | "malloc" : " 14 GB", | 20260507 21:27:42.412 | INFO | "pinned" : " 29 GB" | 20260507 21:27:42.412 | INFO | } | 20260507 21:27:42.412 | INFO | Perf report for the last 5.1389 seconds: | 20260507 21:27:42.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:42.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.412 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:27:42.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.412 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:27:42.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:42.413 | INFO | Uploaded 36GiB to GPU in 2.54497s (14.1456GiB/s) | 20260507 21:27:42.413 | INFO | Downloaded 34GiB to GPU in 2.45458s (13.8517GiB/s) | 20260507 21:27:47.552 | INFO | Analyzing chunk frames = [2723328,2723840) | 20260507 21:27:47.552 | INFO | MemInfo: { | 20260507 21:27:47.552 | INFO | "gpu" : " 17 GB", | 20260507 21:27:47.552 | INFO | "huge" : "", | 20260507 21:27:47.552 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:47.552 | INFO | "malloc" : " 14 GB", | 20260507 21:27:47.552 | INFO | "pinned" : " 29 GB" | 20260507 21:27:47.552 | INFO | } | 20260507 21:27:47.552 | INFO | Perf report for the last 5.13984 seconds: | 20260507 21:27:47.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:47.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:27:47.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:27:47.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:47.552 | INFO | Uploaded 36GiB to GPU in 2.51693s (14.3032GiB/s) | 20260507 21:27:47.553 | INFO | Downloaded 34GiB to GPU in 2.43183s (13.9812GiB/s) | 20260507 21:27:47.650 | INFO | Starting a round of Baz IO | 20260507 21:27:51.118 | INFO | Populating RTMetrics report | 20260507 21:27:51.119 | INFO | RTMetrics report populated | 20260507 21:27:51.119 | INFO | Opening tmp RTMetrics json file | 20260507 21:27:51.129 | INFO | Serializing RTMetrics to json | 20260507 21:27:51.189 | INFO | Serialization done, swapping tmp file now... | 20260507 21:27:51.247 | INFO | RTMetrics done | 20260507 21:27:51.285 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.635 seconds | 20260507 21:27:51.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.72% / 0.01% / 56.27% | 20260507 21:27:51.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67362% / 1.58946e-05% / 5.96046e-05% | 20260507 21:27:51.285 | INFO | Baz IO overall: | 20260507 21:27:51.285 | INFO | BytesWritten : 1240085006318 | 20260507 21:27:51.285 | INFO | NumEvents : 601026578837 | 20260507 21:27:52.691 | INFO | Analyzing chunk frames = [2723840,2724352) | 20260507 21:27:52.691 | INFO | MemInfo: { | 20260507 21:27:52.691 | INFO | "gpu" : " 17 GB", | 20260507 21:27:52.691 | INFO | "huge" : "", | 20260507 21:27:52.691 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:52.691 | INFO | "malloc" : " 14 GB", | 20260507 21:27:52.691 | INFO | "pinned" : " 29 GB" | 20260507 21:27:52.691 | INFO | } | 20260507 21:27:52.691 | INFO | Perf report for the last 5.13915 seconds: | 20260507 21:27:52.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:52.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:52.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:52.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:52.691 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 21:27:52.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:52.691 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 21:27:52.691 | INFO | PRE_HQ : 66.0%, 1.7, 57.8ms, 0 | 20260507 21:27:52.691 | INFO | BAZWRITER : 70.3%, 3.9, 141.4ms, 0 | 20260507 21:27:52.691 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 21:27:52.692 | INFO | Uploaded 36GiB to GPU in 2.5619s (14.0521GiB/s) | 20260507 21:27:52.692 | INFO | Downloaded 38GiB to GPU in 2.68568s (14.1491GiB/s) | 20260507 21:27:57.830 | INFO | Analyzing chunk frames = [2724352,2724864) | 20260507 21:27:57.830 | INFO | MemInfo: { | 20260507 21:27:57.830 | INFO | "gpu" : " 17 GB", | 20260507 21:27:57.830 | INFO | "huge" : "", | 20260507 21:27:57.830 | INFO | "huge_shared" : " 29 GB", | 20260507 21:27:57.830 | INFO | "malloc" : " 14 GB", | 20260507 21:27:57.831 | INFO | "pinned" : " 29 GB" | 20260507 21:27:57.831 | INFO | } | 20260507 21:27:57.831 | INFO | Perf report for the last 5.13943 seconds: | 20260507 21:27:57.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:27:57.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:27:57.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 21:27:57.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:27:57.831 | INFO | Uploaded 36GiB to GPU in 2.55752s (14.0761GiB/s) | 20260507 21:27:57.831 | INFO | Downloaded 34GiB to GPU in 2.4993s (13.6038GiB/s) | 20260507 21:28:02.970 | INFO | Analyzing chunk frames = [2724864,2725376) | 20260507 21:28:02.970 | INFO | MemInfo: { | 20260507 21:28:02.970 | INFO | "gpu" : " 17 GB", | 20260507 21:28:02.970 | INFO | "huge" : "", | 20260507 21:28:02.970 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:02.970 | INFO | "malloc" : " 14 GB", | 20260507 21:28:02.970 | INFO | "pinned" : " 29 GB" | 20260507 21:28:02.970 | INFO | } | 20260507 21:28:02.970 | INFO | Perf report for the last 5.13937 seconds: | 20260507 21:28:02.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:02.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:28:02.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:28:02.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:02.970 | INFO | Uploaded 36GiB to GPU in 2.51128s (14.3353GiB/s) | 20260507 21:28:02.971 | INFO | Downloaded 34GiB to GPU in 2.47947s (13.7126GiB/s) | 20260507 21:28:08.160 | INFO | Analyzing chunk frames = [2725376,2725888) | 20260507 21:28:08.160 | INFO | MemInfo: { | 20260507 21:28:08.160 | INFO | "gpu" : " 17 GB", | 20260507 21:28:08.160 | INFO | "huge" : "", | 20260507 21:28:08.160 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:08.160 | INFO | "malloc" : " 14 GB", | 20260507 21:28:08.160 | INFO | "pinned" : " 29 GB" | 20260507 21:28:08.160 | INFO | } | 20260507 21:28:08.161 | INFO | Perf report for the last 5.19039 seconds: | 20260507 21:28:08.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:08.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | ANALYSIS : 65.8%, 3.0, 103.2ms, 0 | 20260507 21:28:08.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | PRE_HQ : 35.8%, 1.0, 18.8ms, 0 | 20260507 21:28:08.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:08.161 | INFO | Uploaded 36GiB to GPU in 2.484s (14.4928GiB/s) | 20260507 21:28:08.161 | INFO | Downloaded 34GiB to GPU in 2.37726s (14.3022GiB/s) | 20260507 21:28:13.250 | INFO | Analyzing chunk frames = [2725888,2726400) | 20260507 21:28:13.250 | INFO | MemInfo: { | 20260507 21:28:13.250 | INFO | "gpu" : " 17 GB", | 20260507 21:28:13.250 | INFO | "huge" : "", | 20260507 21:28:13.250 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:13.250 | INFO | "malloc" : " 14 GB", | 20260507 21:28:13.250 | INFO | "pinned" : " 29 GB" | 20260507 21:28:13.250 | INFO | } | 20260507 21:28:13.250 | INFO | Perf report for the last 5.08924 seconds: | 20260507 21:28:13.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:13.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | ANALYSIS : 66.9%, 3.0, 103.2ms, 0 | 20260507 21:28:13.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | PRE_HQ : 32.2%, 1.0, 16.5ms, 0 | 20260507 21:28:13.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:13.250 | INFO | Uploaded 36GiB to GPU in 2.41669s (14.8964GiB/s) | 20260507 21:28:13.250 | INFO | Downloaded 34GiB to GPU in 2.49226s (13.6422GiB/s) | 20260507 21:28:18.388 | INFO | Analyzing chunk frames = [2726400,2726912) | 20260507 21:28:18.388 | INFO | MemInfo: { | 20260507 21:28:18.388 | INFO | "gpu" : " 17 GB", | 20260507 21:28:18.388 | INFO | "huge" : "", | 20260507 21:28:18.388 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:18.388 | INFO | "malloc" : " 14 GB", | 20260507 21:28:18.388 | INFO | "pinned" : " 29 GB" | 20260507 21:28:18.388 | INFO | } | 20260507 21:28:18.388 | INFO | Perf report for the last 5.13833 seconds: | 20260507 21:28:18.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:18.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 21:28:18.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:28:18.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:18.388 | INFO | Uploaded 36GiB to GPU in 2.40113s (14.9929GiB/s) | 20260507 21:28:18.389 | INFO | Downloaded 34GiB to GPU in 2.39206s (14.2137GiB/s) | 20260507 21:28:23.527 | INFO | Analyzing chunk frames = [2726912,2727424) | 20260507 21:28:23.527 | INFO | MemInfo: { | 20260507 21:28:23.527 | INFO | "gpu" : " 17 GB", | 20260507 21:28:23.527 | INFO | "huge" : "", | 20260507 21:28:23.527 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:23.527 | INFO | "malloc" : " 14 GB", | 20260507 21:28:23.527 | INFO | "pinned" : " 29 GB" | 20260507 21:28:23.528 | INFO | } | 20260507 21:28:23.528 | INFO | Perf report for the last 5.13938 seconds: | 20260507 21:28:23.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:23.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 21:28:23.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:28:23.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:23.528 | INFO | Uploaded 36GiB to GPU in 2.50698s (14.3599GiB/s) | 20260507 21:28:23.528 | INFO | Downloaded 34GiB to GPU in 2.43892s (13.9406GiB/s) | 20260507 21:28:28.667 | INFO | Analyzing chunk frames = [2727424,2727936) | 20260507 21:28:28.667 | INFO | MemInfo: { | 20260507 21:28:28.667 | INFO | "gpu" : " 17 GB", | 20260507 21:28:28.667 | INFO | "huge" : "", | 20260507 21:28:28.667 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:28.667 | INFO | "malloc" : " 14 GB", | 20260507 21:28:28.667 | INFO | "pinned" : " 29 GB" | 20260507 21:28:28.667 | INFO | } | 20260507 21:28:28.668 | INFO | Perf report for the last 5.13995 seconds: | 20260507 21:28:28.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:28.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:28:28.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:28:28.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:28.668 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:28:28.668 | INFO | Total mem usage: 16.7 GiB | 20260507 21:28:28.668 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:28:28.668 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.54%) | 20260507 21:28:28.668 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:28:28.668 | INFO | High Water Breakdown: | 20260507 21:28:28.668 | INFO | 60.4% : 9.2 GiB ( 89.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:28:28.668 | INFO | 39.5% : 6.01 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:28:28.668 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:28:28.668 | INFO | ------------------------------------------ | 20260507 21:28:28.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:28:28.668 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:28:28.668 | INFO | Total mem usage: 26.5 GiB | 20260507 21:28:28.668 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:28:28.668 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:28:28.668 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:28:28.668 | INFO | High Water Breakdown: | 20260507 21:28:28.668 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:28:28.668 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:28:28.668 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:28:28.668 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:28:28.668 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:28:28.668 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:28:28.668 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:28:28.668 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:28:28.668 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:28:28.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:28:28.668 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:28:28.668 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:28:28.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:28:28.668 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:28:28.668 | INFO | ------------------------------------------ | 20260507 21:28:28.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:28:28.668 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:28:28.668 | INFO | Total mem usage: 27.8 GiB | 20260507 21:28:28.668 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:28:28.668 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:28:28.668 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:28:28.668 | INFO | High Water Breakdown: | 20260507 21:28:28.668 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:28:28.668 | INFO | ------------------------------------------ | 20260507 21:28:28.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:28:28.668 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:28:28.668 | INFO | Total mem usage: 15.7 GiB | 20260507 21:28:28.668 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:28:28.668 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:28:28.668 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:28:28.668 | INFO | High Water Breakdown: | 20260507 21:28:28.668 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:28:28.668 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:28:28.668 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:28:28.668 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:28:28.668 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:28:28.668 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:28:28.668 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:28:28.668 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:28:28.668 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:28:28.668 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:28:28.668 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:28:28.668 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:28:28.668 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:28:28.668 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:28:28.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:28:28.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:28:28.668 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:28:28.668 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:28:28.668 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:28:28.668 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:28:28.668 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:28:28.668 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:28:28.668 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:28:28.668 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:28:28.668 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:28:28.668 | INFO | ------------------------------------------ | 20260507 21:28:28.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:28:28.668 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:28:28.668 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:28:28.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:28:28.668 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.40e+01 / 1.50e+00 | 20260507 21:28:28.668 | INFO | PerfReport: 0.30%, AggregateStats -- 9.94e-02 / 1.03e-01, 1.27e-02 / 5.41e-03 | 20260507 21:28:28.668 | INFO | PerfReport: 12.18%, Baseline -- 4.02e+00 / 4.55e+00, 5.98e-01 / 1.33e-01 | 20260507 21:28:28.668 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.51e+00, 2.10e-01 / 1.08e-01 | 20260507 21:28:28.668 | INFO | PerfReport: 16.00%, DME -- 4.22e+01 / 4.58e+01, 4.07e+00 / 1.45e+00 | 20260507 21:28:28.668 | INFO | PerfReport: 55.90%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.65e+00 / 1.48e+00 | 20260507 21:28:28.668 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.50e+00 / 2.86e+00, 4.34e-01 / 1.80e-01 | 20260507 21:28:28.668 | INFO | PerfReport: 3.55%, Metrics -- 1.17e+00 / 1.41e+00, 2.96e-01 / 1.46e-01 | 20260507 21:28:28.668 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.63e-01 / 4.01e-01, 5.92e-02 / 2.45e-02 | 20260507 21:28:28.668 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:28:28.668 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:28:28.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:28:28.668 | INFO | Uploaded 36GiB to GPU in 2.51087s (14.3376GiB/s) | 20260507 21:28:28.668 | INFO | Downloaded 34GiB to GPU in 2.43196s (13.9805GiB/s) | 20260507 21:28:28.668 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 24.15%, Download -- 2.58e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 30.86%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 2.12%, LockUpload -- 2.27e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 2.56%, LockDownload -- 2.74e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 15.87%, LockCompute -- 1.70e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 21:28:28.669 | INFO | PerfReport: 0.01%, Misc -- 1.60e-02 / 0.00e+00, 2.69e-02 / 0.00e+00 | 20260507 21:28:28.765 | INFO | Starting a round of Baz IO | 20260507 21:28:32.242 | INFO | Populating RTMetrics report | 20260507 21:28:32.243 | INFO | RTMetrics report populated | 20260507 21:28:32.243 | INFO | Opening tmp RTMetrics json file | 20260507 21:28:32.306 | INFO | Serializing RTMetrics to json | 20260507 21:28:32.367 | INFO | Serialization done, swapping tmp file now... | 20260507 21:28:32.370 | INFO | RTMetrics done | 20260507 21:28:32.401 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.636 seconds | 20260507 21:28:32.402 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.71% / 0.01% / 56.28% | 20260507 21:28:32.402 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6759% / 7.94729e-06% / 5.5631e-05% | 20260507 21:28:32.402 | INFO | Baz IO overall: | 20260507 21:28:32.402 | INFO | BytesWritten : 1241973703013 | 20260507 21:28:32.402 | INFO | NumEvents : 601940126135 | 20260507 21:28:33.807 | INFO | Analyzing chunk frames = [2727936,2728448) | 20260507 21:28:33.807 | INFO | MemInfo: { | 20260507 21:28:33.807 | INFO | "gpu" : " 17 GB", | 20260507 21:28:33.807 | INFO | "huge" : "", | 20260507 21:28:33.807 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:33.807 | INFO | "malloc" : " 14 GB", | 20260507 21:28:33.807 | INFO | "pinned" : " 29 GB" | 20260507 21:28:33.807 | INFO | } | 20260507 21:28:33.807 | INFO | Perf report for the last 5.13907 seconds: | 20260507 21:28:33.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:33.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:33.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:33.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:33.807 | INFO | ANALYSIS : 68.6%, 3.0, 107.0ms, 0 | 20260507 21:28:33.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:33.807 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 21:28:33.807 | INFO | PRE_HQ : 65.7%, 1.7, 58.0ms, 0 | 20260507 21:28:33.807 | INFO | BAZWRITER : 70.4%, 4.0, 143.6ms, 0 | 20260507 21:28:33.807 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 21:28:33.807 | INFO | Uploaded 36GiB to GPU in 2.53145s (14.2211GiB/s) | 20260507 21:28:33.807 | INFO | Downloaded 38GiB to GPU in 2.69764s (14.0864GiB/s) | 20260507 21:28:38.945 | INFO | Analyzing chunk frames = [2728448,2728960) | 20260507 21:28:38.945 | INFO | MemInfo: { | 20260507 21:28:38.945 | INFO | "gpu" : " 17 GB", | 20260507 21:28:38.945 | INFO | "huge" : "", | 20260507 21:28:38.945 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:38.945 | INFO | "malloc" : " 14 GB", | 20260507 21:28:38.945 | INFO | "pinned" : " 29 GB" | 20260507 21:28:38.945 | INFO | } | 20260507 21:28:38.945 | INFO | Perf report for the last 5.13846 seconds: | 20260507 21:28:38.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:38.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:28:38.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:28:38.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:38.945 | INFO | Uploaded 36GiB to GPU in 2.56461s (14.0372GiB/s) | 20260507 21:28:38.945 | INFO | Downloaded 34GiB to GPU in 2.49855s (13.6079GiB/s) | 20260507 21:28:44.084 | INFO | Analyzing chunk frames = [2728960,2729472) | 20260507 21:28:44.084 | INFO | MemInfo: { | 20260507 21:28:44.084 | INFO | "gpu" : " 17 GB", | 20260507 21:28:44.084 | INFO | "huge" : "", | 20260507 21:28:44.084 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:44.084 | INFO | "malloc" : " 14 GB", | 20260507 21:28:44.084 | INFO | "pinned" : " 29 GB" | 20260507 21:28:44.084 | INFO | } | 20260507 21:28:44.084 | INFO | Perf report for the last 5.13934 seconds: | 20260507 21:28:44.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:44.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.084 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:28:44.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.085 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:28:44.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:44.085 | INFO | Uploaded 36GiB to GPU in 2.4667s (14.5944GiB/s) | 20260507 21:28:44.085 | INFO | Downloaded 34GiB to GPU in 2.48388s (13.6883GiB/s) | 20260507 21:28:49.224 | INFO | Analyzing chunk frames = [2729472,2729984) | 20260507 21:28:49.224 | INFO | MemInfo: { | 20260507 21:28:49.224 | INFO | "gpu" : " 17 GB", | 20260507 21:28:49.224 | INFO | "huge" : "", | 20260507 21:28:49.224 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:49.224 | INFO | "malloc" : " 14 GB", | 20260507 21:28:49.224 | INFO | "pinned" : " 29 GB" | 20260507 21:28:49.224 | INFO | } | 20260507 21:28:49.225 | INFO | Perf report for the last 5.14009 seconds: | 20260507 21:28:49.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:49.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 21:28:49.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:28:49.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:49.225 | INFO | Uploaded 36GiB to GPU in 2.50276s (14.3841GiB/s) | 20260507 21:28:49.225 | INFO | Downloaded 34GiB to GPU in 2.49594s (13.6221GiB/s) | 20260507 21:28:54.365 | INFO | Analyzing chunk frames = [2729984,2730496) | 20260507 21:28:54.365 | INFO | MemInfo: { | 20260507 21:28:54.365 | INFO | "gpu" : " 17 GB", | 20260507 21:28:54.365 | INFO | "huge" : "", | 20260507 21:28:54.365 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:54.365 | INFO | "malloc" : " 14 GB", | 20260507 21:28:54.365 | INFO | "pinned" : " 29 GB" | 20260507 21:28:54.365 | INFO | } | 20260507 21:28:54.365 | INFO | Perf report for the last 5.14066 seconds: | 20260507 21:28:54.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:54.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:28:54.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:28:54.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:54.365 | INFO | Uploaded 36GiB to GPU in 2.32775s (15.4656GiB/s) | 20260507 21:28:54.366 | INFO | Downloaded 34GiB to GPU in 2.47231s (13.7523GiB/s) | 20260507 21:28:59.504 | INFO | Analyzing chunk frames = [2730496,2731008) | 20260507 21:28:59.504 | INFO | MemInfo: { | 20260507 21:28:59.504 | INFO | "gpu" : " 17 GB", | 20260507 21:28:59.504 | INFO | "huge" : "", | 20260507 21:28:59.504 | INFO | "huge_shared" : " 29 GB", | 20260507 21:28:59.504 | INFO | "malloc" : " 14 GB", | 20260507 21:28:59.504 | INFO | "pinned" : " 29 GB" | 20260507 21:28:59.504 | INFO | } | 20260507 21:28:59.504 | INFO | Perf report for the last 5.13881 seconds: | 20260507 21:28:59.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:28:59.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:28:59.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:28:59.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:28:59.504 | INFO | Uploaded 36GiB to GPU in 2.42938s (14.8186GiB/s) | 20260507 21:28:59.504 | INFO | Downloaded 34GiB to GPU in 2.38411s (14.2611GiB/s) | 20260507 21:29:04.642 | INFO | Analyzing chunk frames = [2731008,2731520) | 20260507 21:29:04.642 | INFO | MemInfo: { | 20260507 21:29:04.642 | INFO | "gpu" : " 17 GB", | 20260507 21:29:04.642 | INFO | "huge" : "", | 20260507 21:29:04.642 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:04.642 | INFO | "malloc" : " 14 GB", | 20260507 21:29:04.642 | INFO | "pinned" : " 29 GB" | 20260507 21:29:04.642 | INFO | } | 20260507 21:29:04.642 | INFO | Perf report for the last 5.13783 seconds: | 20260507 21:29:04.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:04.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 21:29:04.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:29:04.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:04.642 | INFO | Uploaded 36GiB to GPU in 2.5338s (14.2079GiB/s) | 20260507 21:29:04.642 | INFO | Downloaded 34GiB to GPU in 2.43271s (13.9762GiB/s) | 20260507 21:29:09.781 | INFO | Analyzing chunk frames = [2731520,2732032) | 20260507 21:29:09.781 | INFO | MemInfo: { | 20260507 21:29:09.781 | INFO | "gpu" : " 17 GB", | 20260507 21:29:09.781 | INFO | "huge" : "", | 20260507 21:29:09.781 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:09.781 | INFO | "malloc" : " 14 GB", | 20260507 21:29:09.781 | INFO | "pinned" : " 29 GB" | 20260507 21:29:09.781 | INFO | } | 20260507 21:29:09.781 | INFO | Perf report for the last 5.13925 seconds: | 20260507 21:29:09.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:09.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.781 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:29:09.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.781 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:29:09.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:09.782 | INFO | Uploaded 36GiB to GPU in 2.51843s (14.2946GiB/s) | 20260507 21:29:09.782 | INFO | Downloaded 34GiB to GPU in 2.42862s (13.9997GiB/s) | 20260507 21:29:09.878 | INFO | Starting a round of Baz IO | 20260507 21:29:13.354 | INFO | Populating RTMetrics report | 20260507 21:29:13.354 | INFO | RTMetrics report populated | 20260507 21:29:13.354 | INFO | Opening tmp RTMetrics json file | 20260507 21:29:13.382 | INFO | Serializing RTMetrics to json | 20260507 21:29:13.384 | INFO | Serialization done, swapping tmp file now... | 20260507 21:29:13.386 | INFO | RTMetrics done | 20260507 21:29:13.461 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.582 seconds | 20260507 21:29:13.461 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.70% / 0.01% / 56.29% | 20260507 21:29:13.461 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67533% / 1.98682e-05% / 4.37101e-05% | 20260507 21:29:13.461 | INFO | Baz IO overall: | 20260507 21:29:13.461 | INFO | BytesWritten : 1243863563909 | 20260507 21:29:13.461 | INFO | NumEvents : 602854645009 | 20260507 21:29:14.921 | INFO | Analyzing chunk frames = [2732032,2732544) | 20260507 21:29:14.921 | INFO | MemInfo: { | 20260507 21:29:14.921 | INFO | "gpu" : " 17 GB", | 20260507 21:29:14.921 | INFO | "huge" : "", | 20260507 21:29:14.921 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:14.921 | INFO | "malloc" : " 14 GB", | 20260507 21:29:14.921 | INFO | "pinned" : " 29 GB" | 20260507 21:29:14.921 | INFO | } | 20260507 21:29:14.921 | INFO | Perf report for the last 5.13997 seconds: | 20260507 21:29:14.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:14.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:14.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:14.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:14.921 | INFO | ANALYSIS : 68.8%, 3.0, 106.9ms, 0 | 20260507 21:29:14.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:14.921 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 21:29:14.921 | INFO | PRE_HQ : 66.1%, 1.7, 57.8ms, 0 | 20260507 21:29:14.921 | INFO | BAZWRITER : 69.3%, 3.7, 131.4ms, 0 | 20260507 21:29:14.921 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:29:14.922 | INFO | Uploaded 36GiB to GPU in 2.55613s (14.0838GiB/s) | 20260507 21:29:14.922 | INFO | Downloaded 38GiB to GPU in 2.68375s (14.1593GiB/s) | 20260507 21:29:20.060 | INFO | Analyzing chunk frames = [2732544,2733056) | 20260507 21:29:20.060 | INFO | MemInfo: { | 20260507 21:29:20.060 | INFO | "gpu" : " 17 GB", | 20260507 21:29:20.060 | INFO | "huge" : "", | 20260507 21:29:20.060 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:20.060 | INFO | "malloc" : " 14 GB", | 20260507 21:29:20.060 | INFO | "pinned" : " 29 GB" | 20260507 21:29:20.060 | INFO | } | 20260507 21:29:20.060 | INFO | Perf report for the last 5.13921 seconds: | 20260507 21:29:20.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:20.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:29:20.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 21:29:20.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:20.061 | INFO | Uploaded 36GiB to GPU in 2.58463s (13.9285GiB/s) | 20260507 21:29:20.061 | INFO | Downloaded 34GiB to GPU in 2.49251s (13.6409GiB/s) | 20260507 21:29:25.200 | INFO | Analyzing chunk frames = [2733056,2733568) | 20260507 21:29:25.200 | INFO | MemInfo: { | 20260507 21:29:25.200 | INFO | "gpu" : " 17 GB", | 20260507 21:29:25.200 | INFO | "huge" : "", | 20260507 21:29:25.200 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:25.200 | INFO | "malloc" : " 14 GB", | 20260507 21:29:25.200 | INFO | "pinned" : " 29 GB" | 20260507 21:29:25.200 | INFO | } | 20260507 21:29:25.200 | INFO | Perf report for the last 5.13994 seconds: | 20260507 21:29:25.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:25.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.200 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:29:25.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.201 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:29:25.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:25.201 | INFO | Uploaded 36GiB to GPU in 2.54567s (14.1417GiB/s) | 20260507 21:29:25.201 | INFO | Downloaded 34GiB to GPU in 2.47409s (13.7424GiB/s) | 20260507 21:29:30.339 | INFO | Analyzing chunk frames = [2733568,2734080) | 20260507 21:29:30.339 | INFO | MemInfo: { | 20260507 21:29:30.339 | INFO | "gpu" : " 17 GB", | 20260507 21:29:30.339 | INFO | "huge" : "", | 20260507 21:29:30.339 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:30.339 | INFO | "malloc" : " 14 GB", | 20260507 21:29:30.340 | INFO | "pinned" : " 29 GB" | 20260507 21:29:30.340 | INFO | } | 20260507 21:29:30.340 | INFO | Perf report for the last 5.13928 seconds: | 20260507 21:29:30.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:30.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 21:29:30.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:29:30.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:30.340 | INFO | Uploaded 36GiB to GPU in 2.55603s (14.0843GiB/s) | 20260507 21:29:30.340 | INFO | Downloaded 34GiB to GPU in 2.47453s (13.74GiB/s) | 20260507 21:29:35.481 | INFO | Analyzing chunk frames = [2734080,2734592) | 20260507 21:29:35.481 | INFO | MemInfo: { | 20260507 21:29:35.481 | INFO | "gpu" : " 17 GB", | 20260507 21:29:35.481 | INFO | "huge" : "", | 20260507 21:29:35.481 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:35.481 | INFO | "malloc" : " 14 GB", | 20260507 21:29:35.481 | INFO | "pinned" : " 29 GB" | 20260507 21:29:35.481 | INFO | } | 20260507 21:29:35.481 | INFO | Perf report for the last 5.14155 seconds: | 20260507 21:29:35.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:35.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 21:29:35.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 21:29:35.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:35.481 | INFO | Uploaded 36GiB to GPU in 2.465s (14.6045GiB/s) | 20260507 21:29:35.481 | INFO | Downloaded 34GiB to GPU in 2.48576s (13.6779GiB/s) | 20260507 21:29:40.618 | INFO | Analyzing chunk frames = [2734592,2735104) | 20260507 21:29:40.618 | INFO | MemInfo: { | 20260507 21:29:40.618 | INFO | "gpu" : " 17 GB", | 20260507 21:29:40.618 | INFO | "huge" : "", | 20260507 21:29:40.618 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:40.618 | INFO | "malloc" : " 14 GB", | 20260507 21:29:40.618 | INFO | "pinned" : " 29 GB" | 20260507 21:29:40.618 | INFO | } | 20260507 21:29:40.618 | INFO | Perf report for the last 5.13687 seconds: | 20260507 21:29:40.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:40.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:29:40.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:29:40.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:40.618 | INFO | Uploaded 36GiB to GPU in 2.40269s (14.9832GiB/s) | 20260507 21:29:40.618 | INFO | Downloaded 34GiB to GPU in 2.39433s (14.2002GiB/s) | 20260507 21:29:45.795 | INFO | Analyzing chunk frames = [2735104,2735616) | 20260507 21:29:45.795 | INFO | MemInfo: { | 20260507 21:29:45.795 | INFO | "gpu" : " 17 GB", | 20260507 21:29:45.795 | INFO | "huge" : "", | 20260507 21:29:45.795 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:45.795 | INFO | "malloc" : " 14 GB", | 20260507 21:29:45.795 | INFO | "pinned" : " 29 GB" | 20260507 21:29:45.795 | INFO | } | 20260507 21:29:45.795 | INFO | Perf report for the last 5.17717 seconds: | 20260507 21:29:45.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:45.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.795 | INFO | ANALYSIS : 66.9%, 3.0, 105.0ms, 0 | 20260507 21:29:45.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.795 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:29:45.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:45.796 | INFO | Uploaded 36GiB to GPU in 2.54207s (14.1617GiB/s) | 20260507 21:29:45.796 | INFO | Downloaded 34GiB to GPU in 2.45185s (13.8671GiB/s) | 20260507 21:29:50.933 | INFO | Analyzing chunk frames = [2735616,2736128) | 20260507 21:29:50.933 | INFO | MemInfo: { | 20260507 21:29:50.933 | INFO | "gpu" : " 17 GB", | 20260507 21:29:50.933 | INFO | "huge" : "", | 20260507 21:29:50.933 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:50.933 | INFO | "malloc" : " 14 GB", | 20260507 21:29:50.933 | INFO | "pinned" : " 29 GB" | 20260507 21:29:50.933 | INFO | } | 20260507 21:29:50.933 | INFO | Perf report for the last 5.13792 seconds: | 20260507 21:29:50.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:50.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.933 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:29:50.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.933 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:29:50.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:50.934 | INFO | Uploaded 36GiB to GPU in 2.50574s (14.367GiB/s) | 20260507 21:29:50.934 | INFO | Downloaded 34GiB to GPU in 2.4384s (13.9435GiB/s) | 20260507 21:29:50.934 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:29:50.934 | INFO | Total mem usage: 16.7 GiB | 20260507 21:29:50.934 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:29:50.934 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260507 21:29:50.934 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:29:50.934 | INFO | High Water Breakdown: | 20260507 21:29:50.934 | INFO | 59.8% : 8.86 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:29:50.934 | INFO | 40.1% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:29:50.934 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:29:50.934 | INFO | ------------------------------------------ | 20260507 21:29:50.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:29:50.934 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:29:50.934 | INFO | Total mem usage: 26.5 GiB | 20260507 21:29:50.934 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:29:50.934 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:29:50.934 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:29:50.934 | INFO | High Water Breakdown: | 20260507 21:29:50.934 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:29:50.934 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:29:50.934 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:29:50.934 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:29:50.934 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:29:50.934 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:29:50.934 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:29:50.934 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:29:50.934 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:29:50.934 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:29:50.934 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:29:50.934 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:29:50.934 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:29:50.934 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:29:50.934 | INFO | ------------------------------------------ | 20260507 21:29:50.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:29:50.934 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:29:50.935 | INFO | Total mem usage: 27.8 GiB | 20260507 21:29:50.935 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:29:50.935 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:29:50.935 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:29:50.935 | INFO | High Water Breakdown: | 20260507 21:29:50.935 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:29:50.935 | INFO | ------------------------------------------ | 20260507 21:29:50.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:29:50.935 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:29:50.935 | INFO | Total mem usage: 15.7 GiB | 20260507 21:29:50.935 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:29:50.935 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:29:50.935 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:29:50.935 | INFO | High Water Breakdown: | 20260507 21:29:50.935 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:29:50.935 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:29:50.935 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:29:50.935 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:29:50.935 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:29:50.935 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:29:50.935 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:29:50.935 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:29:50.935 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:29:50.935 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:29:50.935 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:29:50.935 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:29:50.935 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:29:50.935 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:29:50.935 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:29:50.935 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:29:50.935 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:29:50.935 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:29:50.935 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:29:50.935 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:29:50.935 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:29:50.935 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:29:50.935 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:29:50.935 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:29:50.935 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:29:50.935 | INFO | ------------------------------------------ | 20260507 21:29:50.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:29:50.935 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:29:50.935 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:29:50.935 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:29:50.936 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.63e+01, 1.43e+01 / 2.01e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.21e-02 / 4.57e-03 | 20260507 21:29:50.936 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.48e+00, 5.94e-01 / 2.16e-01 | 20260507 21:29:50.936 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 9.66e-02 | 20260507 21:29:50.936 | INFO | PerfReport: 16.13%, DME -- 4.25e+01 / 4.62e+01, 4.37e+00 / 2.26e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 55.83%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.60e+00 / 1.40e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.50e+00 / 2.84e+00, 4.38e-01 / 2.11e-01 | 20260507 21:29:50.936 | INFO | PerfReport: 3.54%, Metrics -- 1.17e+00 / 1.40e+00, 2.95e-01 / 1.69e-01 | 20260507 21:29:50.936 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 4.00e-01, 5.97e-02 / 2.67e-02 | 20260507 21:29:50.936 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:29:50.936 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:29:50.936 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:29:50.936 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 24.35%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 30.78%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 2.24%, LockUpload -- 2.39e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 2.62%, LockDownload -- 2.80e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 21:29:50.936 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 21:29:51.046 | INFO | Starting a round of Baz IO | 20260507 21:29:54.513 | INFO | Populating RTMetrics report | 20260507 21:29:54.514 | INFO | RTMetrics report populated | 20260507 21:29:54.514 | INFO | Opening tmp RTMetrics json file | 20260507 21:29:54.536 | INFO | Serializing RTMetrics to json | 20260507 21:29:54.545 | INFO | Serialization done, swapping tmp file now... | 20260507 21:29:54.554 | INFO | RTMetrics done | 20260507 21:29:54.610 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.563 seconds | 20260507 21:29:54.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.69% / 0.01% / 56.30% | 20260507 21:29:54.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.66553% / 1.19209e-05% / 3.17891e-05% | 20260507 21:29:54.610 | INFO | Baz IO overall: | 20260507 21:29:54.610 | INFO | BytesWritten : 1245751048949 | 20260507 21:29:54.610 | INFO | NumEvents : 603767303940 | 20260507 21:29:56.074 | INFO | Analyzing chunk frames = [2736128,2736640) | 20260507 21:29:56.074 | INFO | MemInfo: { | 20260507 21:29:56.074 | INFO | "gpu" : " 17 GB", | 20260507 21:29:56.074 | INFO | "huge" : "", | 20260507 21:29:56.074 | INFO | "huge_shared" : " 29 GB", | 20260507 21:29:56.074 | INFO | "malloc" : " 14 GB", | 20260507 21:29:56.074 | INFO | "pinned" : " 29 GB" | 20260507 21:29:56.074 | INFO | } | 20260507 21:29:56.074 | INFO | Perf report for the last 5.14045 seconds: | 20260507 21:29:56.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:29:56.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:56.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:56.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:56.074 | INFO | ANALYSIS : 68.7%, 2.9, 105.1ms, 0 | 20260507 21:29:56.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:29:56.074 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 21:29:56.074 | INFO | PRE_HQ : 66.8%, 1.8, 62.1ms, 0 | 20260507 21:29:56.074 | INFO | BAZWRITER : 68.8%, 4.2, 149.0ms, 0 | 20260507 21:29:56.074 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:29:56.074 | INFO | Uploaded 36GiB to GPU in 2.56047s (14.0599GiB/s) | 20260507 21:29:56.074 | INFO | Downloaded 38GiB to GPU in 2.67549s (14.203GiB/s) | 20260507 21:30:01.212 | INFO | Analyzing chunk frames = [2736640,2737152) | 20260507 21:30:01.212 | INFO | MemInfo: { | 20260507 21:30:01.212 | INFO | "gpu" : " 17 GB", | 20260507 21:30:01.212 | INFO | "huge" : "", | 20260507 21:30:01.212 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:01.212 | INFO | "malloc" : " 14 GB", | 20260507 21:30:01.213 | INFO | "pinned" : " 29 GB" | 20260507 21:30:01.213 | INFO | } | 20260507 21:30:01.213 | INFO | Perf report for the last 5.13899 seconds: | 20260507 21:30:01.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:01.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:30:01.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:30:01.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:01.213 | INFO | Uploaded 36GiB to GPU in 2.52348s (14.266GiB/s) | 20260507 21:30:01.213 | INFO | Downloaded 34GiB to GPU in 2.4778s (13.7218GiB/s) | 20260507 21:30:06.352 | INFO | Analyzing chunk frames = [2737152,2737664) | 20260507 21:30:06.352 | INFO | MemInfo: { | 20260507 21:30:06.352 | INFO | "gpu" : " 17 GB", | 20260507 21:30:06.352 | INFO | "huge" : "", | 20260507 21:30:06.352 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:06.352 | INFO | "malloc" : " 14 GB", | 20260507 21:30:06.352 | INFO | "pinned" : " 29 GB" | 20260507 21:30:06.352 | INFO | } | 20260507 21:30:06.352 | INFO | Perf report for the last 5.13952 seconds: | 20260507 21:30:06.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:06.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:30:06.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:30:06.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:06.352 | INFO | Uploaded 36GiB to GPU in 2.53085s (14.2245GiB/s) | 20260507 21:30:06.353 | INFO | Downloaded 34GiB to GPU in 2.45313s (13.8599GiB/s) | 20260507 21:30:11.492 | INFO | Analyzing chunk frames = [2737664,2738176) | 20260507 21:30:11.492 | INFO | MemInfo: { | 20260507 21:30:11.492 | INFO | "gpu" : " 17 GB", | 20260507 21:30:11.492 | INFO | "huge" : "", | 20260507 21:30:11.492 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:11.492 | INFO | "malloc" : " 14 GB", | 20260507 21:30:11.492 | INFO | "pinned" : " 29 GB" | 20260507 21:30:11.492 | INFO | } | 20260507 21:30:11.492 | INFO | Perf report for the last 5.13989 seconds: | 20260507 21:30:11.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:11.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.492 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:30:11.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.492 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:30:11.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:11.493 | INFO | Uploaded 36GiB to GPU in 2.46263s (14.6185GiB/s) | 20260507 21:30:11.493 | INFO | Downloaded 34GiB to GPU in 2.38407s (14.2613GiB/s) | 20260507 21:30:16.632 | INFO | Analyzing chunk frames = [2738176,2738688) | 20260507 21:30:16.632 | INFO | MemInfo: { | 20260507 21:30:16.632 | INFO | "gpu" : " 17 GB", | 20260507 21:30:16.632 | INFO | "huge" : "", | 20260507 21:30:16.632 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:16.633 | INFO | "malloc" : " 14 GB", | 20260507 21:30:16.633 | INFO | "pinned" : " 29 GB" | 20260507 21:30:16.633 | INFO | } | 20260507 21:30:16.633 | INFO | Perf report for the last 5.14069 seconds: | 20260507 21:30:16.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:16.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:30:16.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 21:30:16.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:16.633 | INFO | Uploaded 36GiB to GPU in 2.35846s (15.2642GiB/s) | 20260507 21:30:16.633 | INFO | Downloaded 34GiB to GPU in 2.48866s (13.662GiB/s) | 20260507 21:30:21.773 | INFO | Analyzing chunk frames = [2738688,2739200) | 20260507 21:30:21.773 | INFO | MemInfo: { | 20260507 21:30:21.773 | INFO | "gpu" : " 17 GB", | 20260507 21:30:21.773 | INFO | "huge" : "", | 20260507 21:30:21.774 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:21.774 | INFO | "malloc" : " 14 GB", | 20260507 21:30:21.774 | INFO | "pinned" : " 29 GB" | 20260507 21:30:21.774 | INFO | } | 20260507 21:30:21.774 | INFO | Perf report for the last 5.14098 seconds: | 20260507 21:30:21.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:21.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 21:30:21.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:30:21.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:21.774 | INFO | Uploaded 36GiB to GPU in 2.52638s (14.2496GiB/s) | 20260507 21:30:21.774 | INFO | Downloaded 34GiB to GPU in 2.41156s (14.0987GiB/s) | 20260507 21:30:26.910 | INFO | Analyzing chunk frames = [2739200,2739712) | 20260507 21:30:26.910 | INFO | MemInfo: { | 20260507 21:30:26.910 | INFO | "gpu" : " 17 GB", | 20260507 21:30:26.910 | INFO | "huge" : "", | 20260507 21:30:26.910 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:26.910 | INFO | "malloc" : " 14 GB", | 20260507 21:30:26.911 | INFO | "pinned" : " 29 GB" | 20260507 21:30:26.911 | INFO | } | 20260507 21:30:26.911 | INFO | Perf report for the last 5.13687 seconds: | 20260507 21:30:26.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:26.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.911 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:30:26.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.911 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 21:30:26.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:26.912 | INFO | Uploaded 36GiB to GPU in 2.55853s (14.0706GiB/s) | 20260507 21:30:26.912 | INFO | Downloaded 34GiB to GPU in 2.44567s (13.9021GiB/s) | 20260507 21:30:32.050 | INFO | Analyzing chunk frames = [2739712,2740224) | 20260507 21:30:32.050 | INFO | MemInfo: { | 20260507 21:30:32.050 | INFO | "gpu" : " 17 GB", | 20260507 21:30:32.050 | INFO | "huge" : "", | 20260507 21:30:32.051 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:32.051 | INFO | "malloc" : " 14 GB", | 20260507 21:30:32.051 | INFO | "pinned" : " 29 GB" | 20260507 21:30:32.051 | INFO | } | 20260507 21:30:32.051 | INFO | Perf report for the last 5.14008 seconds: | 20260507 21:30:32.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:32.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:30:32.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 21:30:32.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:32.051 | INFO | Uploaded 36GiB to GPU in 2.53034s (14.2273GiB/s) | 20260507 21:30:32.051 | INFO | Downloaded 34GiB to GPU in 2.44413s (13.9109GiB/s) | 20260507 21:30:32.167 | INFO | Starting a round of Baz IO | 20260507 21:30:35.635 | INFO | Populating RTMetrics report | 20260507 21:30:35.635 | INFO | RTMetrics report populated | 20260507 21:30:35.635 | INFO | Opening tmp RTMetrics json file | 20260507 21:30:35.662 | INFO | Serializing RTMetrics to json | 20260507 21:30:35.664 | INFO | Serialization done, swapping tmp file now... | 20260507 21:30:35.665 | INFO | RTMetrics done | 20260507 21:30:35.746 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.579 seconds | 20260507 21:30:35.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.69% / 0.01% / 56.31% | 20260507 21:30:35.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6594% / 1.58946e-05% / 5.16574e-05% | 20260507 21:30:35.746 | INFO | Baz IO overall: | 20260507 21:30:35.746 | INFO | BytesWritten : 1247637340536 | 20260507 21:30:35.746 | INFO | NumEvents : 604679040460 | 20260507 21:30:37.186 | INFO | Analyzing chunk frames = [2740224,2740736) | 20260507 21:30:37.186 | INFO | MemInfo: { | 20260507 21:30:37.186 | INFO | "gpu" : " 17 GB", | 20260507 21:30:37.186 | INFO | "huge" : "", | 20260507 21:30:37.186 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:37.186 | INFO | "malloc" : " 14 GB", | 20260507 21:30:37.186 | INFO | "pinned" : " 29 GB" | 20260507 21:30:37.187 | INFO | } | 20260507 21:30:37.187 | INFO | Perf report for the last 5.13584 seconds: | 20260507 21:30:37.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:37.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:37.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:37.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:37.187 | INFO | ANALYSIS : 69.0%, 2.9, 104.7ms, 0 | 20260507 21:30:37.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:37.187 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 21:30:37.187 | INFO | PRE_HQ : 67.9%, 1.8, 63.7ms, 0 | 20260507 21:30:37.187 | INFO | BAZWRITER : 69.0%, 4.0, 141.2ms, 0 | 20260507 21:30:37.187 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:30:37.187 | INFO | Uploaded 36GiB to GPU in 2.60433s (13.8232GiB/s) | 20260507 21:30:37.187 | INFO | Downloaded 38GiB to GPU in 2.71514s (13.9956GiB/s) | 20260507 21:30:42.325 | INFO | Analyzing chunk frames = [2740736,2741248) | 20260507 21:30:42.325 | INFO | MemInfo: { | 20260507 21:30:42.325 | INFO | "gpu" : " 17 GB", | 20260507 21:30:42.325 | INFO | "huge" : "", | 20260507 21:30:42.325 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:42.325 | INFO | "malloc" : " 14 GB", | 20260507 21:30:42.325 | INFO | "pinned" : " 29 GB" | 20260507 21:30:42.326 | INFO | } | 20260507 21:30:42.326 | INFO | Perf report for the last 5.13909 seconds: | 20260507 21:30:42.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:42.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:30:42.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:30:42.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:42.326 | INFO | Uploaded 36GiB to GPU in 2.5165s (14.3056GiB/s) | 20260507 21:30:42.326 | INFO | Downloaded 34GiB to GPU in 2.44844s (13.8864GiB/s) | 20260507 21:30:47.470 | INFO | Analyzing chunk frames = [2741248,2741760) | 20260507 21:30:47.470 | INFO | MemInfo: { | 20260507 21:30:47.470 | INFO | "gpu" : " 17 GB", | 20260507 21:30:47.470 | INFO | "huge" : "", | 20260507 21:30:47.470 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:47.470 | INFO | "malloc" : " 14 GB", | 20260507 21:30:47.470 | INFO | "pinned" : " 29 GB" | 20260507 21:30:47.470 | INFO | } | 20260507 21:30:47.470 | INFO | Perf report for the last 5.14469 seconds: | 20260507 21:30:47.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:47.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.470 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260507 21:30:47.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.471 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:30:47.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:47.471 | INFO | Uploaded 36GiB to GPU in 2.57601s (13.9751GiB/s) | 20260507 21:30:47.471 | INFO | Downloaded 34GiB to GPU in 2.44168s (13.9248GiB/s) | 20260507 21:30:52.615 | INFO | Analyzing chunk frames = [2741760,2742272) | 20260507 21:30:52.615 | INFO | MemInfo: { | 20260507 21:30:52.615 | INFO | "gpu" : " 17 GB", | 20260507 21:30:52.615 | INFO | "huge" : "", | 20260507 21:30:52.615 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:52.615 | INFO | "malloc" : " 14 GB", | 20260507 21:30:52.615 | INFO | "pinned" : " 29 GB" | 20260507 21:30:52.615 | INFO | } | 20260507 21:30:52.615 | INFO | Perf report for the last 5.14493 seconds: | 20260507 21:30:52.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:52.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.615 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260507 21:30:52.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.616 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:30:52.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:52.616 | INFO | Uploaded 36GiB to GPU in 2.50762s (14.3563GiB/s) | 20260507 21:30:52.616 | INFO | Downloaded 34GiB to GPU in 2.37287s (14.3286GiB/s) | 20260507 21:30:57.749 | INFO | Analyzing chunk frames = [2742272,2742784) | 20260507 21:30:57.749 | INFO | MemInfo: { | 20260507 21:30:57.749 | INFO | "gpu" : " 17 GB", | 20260507 21:30:57.749 | INFO | "huge" : "", | 20260507 21:30:57.749 | INFO | "huge_shared" : " 29 GB", | 20260507 21:30:57.749 | INFO | "malloc" : " 14 GB", | 20260507 21:30:57.749 | INFO | "pinned" : " 29 GB" | 20260507 21:30:57.749 | INFO | } | 20260507 21:30:57.749 | INFO | Perf report for the last 5.13382 seconds: | 20260507 21:30:57.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:30:57.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.749 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 21:30:57.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.749 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:30:57.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:30:57.750 | INFO | Uploaded 36GiB to GPU in 2.3996s (15.0025GiB/s) | 20260507 21:30:57.750 | INFO | Downloaded 34GiB to GPU in 2.49338s (13.6361GiB/s) | 20260507 21:31:02.888 | INFO | Analyzing chunk frames = [2742784,2743296) | 20260507 21:31:02.888 | INFO | MemInfo: { | 20260507 21:31:02.888 | INFO | "gpu" : " 17 GB", | 20260507 21:31:02.888 | INFO | "huge" : "", | 20260507 21:31:02.888 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:02.888 | INFO | "malloc" : " 14 GB", | 20260507 21:31:02.888 | INFO | "pinned" : " 29 GB" | 20260507 21:31:02.888 | INFO | } | 20260507 21:31:02.889 | INFO | Perf report for the last 5.13942 seconds: | 20260507 21:31:02.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:02.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:31:02.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:31:02.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:02.889 | INFO | Uploaded 36GiB to GPU in 2.52263s (14.2708GiB/s) | 20260507 21:31:02.889 | INFO | Downloaded 34GiB to GPU in 2.4216s (14.0403GiB/s) | 20260507 21:31:08.025 | INFO | Analyzing chunk frames = [2743296,2743808) | 20260507 21:31:08.025 | INFO | MemInfo: { | 20260507 21:31:08.025 | INFO | "gpu" : " 17 GB", | 20260507 21:31:08.025 | INFO | "huge" : "", | 20260507 21:31:08.025 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:08.026 | INFO | "malloc" : " 14 GB", | 20260507 21:31:08.026 | INFO | "pinned" : " 29 GB" | 20260507 21:31:08.026 | INFO | } | 20260507 21:31:08.026 | INFO | Perf report for the last 5.13721 seconds: | 20260507 21:31:08.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:08.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.026 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:31:08.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.026 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 21:31:08.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:08.027 | INFO | Uploaded 36GiB to GPU in 2.52645s (14.2492GiB/s) | 20260507 21:31:08.027 | INFO | Downloaded 34GiB to GPU in 2.45337s (13.8585GiB/s) | 20260507 21:31:13.163 | INFO | Analyzing chunk frames = [2743808,2744320) | 20260507 21:31:13.163 | INFO | MemInfo: { | 20260507 21:31:13.163 | INFO | "gpu" : " 17 GB", | 20260507 21:31:13.163 | INFO | "huge" : "", | 20260507 21:31:13.163 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:13.163 | INFO | "malloc" : " 14 GB", | 20260507 21:31:13.163 | INFO | "pinned" : " 29 GB" | 20260507 21:31:13.163 | INFO | } | 20260507 21:31:13.163 | INFO | Perf report for the last 5.13742 seconds: | 20260507 21:31:13.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:13.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.163 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:31:13.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.163 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 21:31:13.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:13.164 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:31:13.164 | INFO | Total mem usage: 16.7 GiB | 20260507 21:31:13.164 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:31:13.164 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.6%) | 20260507 21:31:13.164 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:31:13.164 | INFO | High Water Breakdown: | 20260507 21:31:13.164 | INFO | 60% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:31:13.164 | INFO | 39.8% : 5.96 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:31:13.164 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:31:13.164 | INFO | ------------------------------------------ | 20260507 21:31:13.164 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:31:13.164 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:31:13.164 | INFO | Total mem usage: 26.5 GiB | 20260507 21:31:13.164 | INFO | Uploaded 36GiB to GPU in 2.49662s (14.4195GiB/s) | 20260507 21:31:13.164 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:31:13.164 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 21:31:13.164 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 21:31:13.164 | INFO | Downloaded 34GiB to GPU in 2.44062s (13.9309GiB/s) | 20260507 21:31:13.164 | INFO | High Water Breakdown: | 20260507 21:31:13.164 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:31:13.164 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:31:13.164 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:31:13.164 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:31:13.164 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:31:13.164 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:31:13.164 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:31:13.164 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:31:13.164 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:31:13.164 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:31:13.164 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:31:13.165 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:31:13.165 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:31:13.165 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:31:13.165 | INFO | ------------------------------------------ | 20260507 21:31:13.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:31:13.165 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:31:13.165 | INFO | Total mem usage: 27.8 GiB | 20260507 21:31:13.165 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:31:13.165 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:31:13.165 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:31:13.165 | INFO | High Water Breakdown: | 20260507 21:31:13.165 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:31:13.165 | INFO | ------------------------------------------ | 20260507 21:31:13.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:31:13.165 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:31:13.165 | INFO | Total mem usage: 15.7 GiB | 20260507 21:31:13.165 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:31:13.165 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:31:13.165 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:31:13.165 | INFO | High Water Breakdown: | 20260507 21:31:13.165 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:31:13.165 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:31:13.165 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:31:13.165 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:31:13.165 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:31:13.165 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:31:13.165 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:31:13.165 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:31:13.165 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:31:13.165 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:31:13.165 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:31:13.165 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:31:13.165 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:31:13.165 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:31:13.165 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:31:13.165 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:31:13.165 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:31:13.165 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:31:13.165 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:31:13.165 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:31:13.165 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:31:13.165 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:31:13.165 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:31:13.165 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:31:13.165 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:31:13.165 | INFO | ------------------------------------------ | 20260507 21:31:13.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:31:13.165 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:31:13.165 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:31:13.165 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:31:13.165 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.41e+01 / 1.73e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.23e-02 / 6.44e-03 | 20260507 21:31:13.165 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.47e+00, 5.97e-01 / 2.41e-01 | 20260507 21:31:13.165 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.48e+00, 2.06e-01 / 1.20e-01 | 20260507 21:31:13.165 | INFO | PerfReport: 15.79%, DME -- 4.17e+01 / 4.50e+01, 3.82e+00 / 1.49e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.80e+00 / 2.02e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.32e-01 / 2.61e-01 | 20260507 21:31:13.165 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.34e+00, 2.96e-01 / 1.74e-01 | 20260507 21:31:13.165 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.63e-01 / 3.90e-01, 6.07e-02 / 3.08e-02 | 20260507 21:31:13.165 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:31:13.165 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:31:13.165 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:31:13.165 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 24.54%, Upload -- 2.62e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 24.16%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 30.97%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 2.10%, LockUpload -- 2.24e+00 / 0.00e+00, 5.29e+00 / 0.00e+00 | 20260507 21:31:13.165 | INFO | PerfReport: 2.33%, LockDownload -- 2.48e+00 / 0.00e+00, 3.74e+00 / 0.00e+00 | 20260507 21:31:13.166 | INFO | PerfReport: 15.89%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:31:13.166 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 21:31:13.276 | INFO | Starting a round of Baz IO | 20260507 21:31:16.758 | INFO | Populating RTMetrics report | 20260507 21:31:16.758 | INFO | RTMetrics report populated | 20260507 21:31:16.758 | INFO | Opening tmp RTMetrics json file | 20260507 21:31:16.849 | INFO | Serializing RTMetrics to json | 20260507 21:31:16.855 | INFO | Serialization done, swapping tmp file now... | 20260507 21:31:16.860 | INFO | RTMetrics done | 20260507 21:31:17.275 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.999 seconds | 20260507 21:31:17.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.68% / 0.01% / 56.31% | 20260507 21:31:17.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.65648% / 1.58946e-05% / 5.96046e-05% | 20260507 21:31:17.276 | INFO | Baz IO overall: | 20260507 21:31:17.276 | INFO | BytesWritten : 1249523217011 | 20260507 21:31:17.276 | INFO | NumEvents : 605590175017 | 20260507 21:31:18.300 | INFO | Analyzing chunk frames = [2744320,2744832) | 20260507 21:31:18.300 | INFO | MemInfo: { | 20260507 21:31:18.300 | INFO | "gpu" : " 17 GB", | 20260507 21:31:18.300 | INFO | "huge" : "", | 20260507 21:31:18.300 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:18.300 | INFO | "malloc" : " 14 GB", | 20260507 21:31:18.300 | INFO | "pinned" : " 29 GB" | 20260507 21:31:18.300 | INFO | } | 20260507 21:31:18.300 | INFO | Perf report for the last 5.13653 seconds: | 20260507 21:31:18.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:18.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:18.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:18.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:18.300 | INFO | ANALYSIS : 69.0%, 2.9, 104.3ms, 0 | 20260507 21:31:18.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:18.300 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260507 21:31:18.300 | INFO | PRE_HQ : 67.9%, 1.8, 64.9ms, 0 | 20260507 21:31:18.300 | INFO | BAZWRITER : 77.4%, 4.8, 190.7ms, 0 | 20260507 21:31:18.300 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260507 21:31:18.301 | INFO | Uploaded 36GiB to GPU in 2.56905s (14.013GiB/s) | 20260507 21:31:18.301 | INFO | Downloaded 38GiB to GPU in 2.70431s (14.0517GiB/s) | 20260507 21:31:23.442 | INFO | Analyzing chunk frames = [2744832,2745344) | 20260507 21:31:23.442 | INFO | MemInfo: { | 20260507 21:31:23.442 | INFO | "gpu" : " 17 GB", | 20260507 21:31:23.442 | INFO | "huge" : "", | 20260507 21:31:23.442 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:23.442 | INFO | "malloc" : " 14 GB", | 20260507 21:31:23.442 | INFO | "pinned" : " 29 GB" | 20260507 21:31:23.442 | INFO | } | 20260507 21:31:23.442 | INFO | Perf report for the last 5.14198 seconds: | 20260507 21:31:23.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:23.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 21:31:23.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 21:31:23.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:23.442 | INFO | Uploaded 36GiB to GPU in 2.49095s (14.4523GiB/s) | 20260507 21:31:23.442 | INFO | Downloaded 34GiB to GPU in 2.49098s (13.6492GiB/s) | 20260507 21:31:28.582 | INFO | Analyzing chunk frames = [2745344,2745856) | 20260507 21:31:28.582 | INFO | MemInfo: { | 20260507 21:31:28.582 | INFO | "gpu" : " 17 GB", | 20260507 21:31:28.582 | INFO | "huge" : "", | 20260507 21:31:28.582 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:28.582 | INFO | "malloc" : " 14 GB", | 20260507 21:31:28.582 | INFO | "pinned" : " 29 GB" | 20260507 21:31:28.582 | INFO | } | 20260507 21:31:28.583 | INFO | Perf report for the last 5.14063 seconds: | 20260507 21:31:28.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:28.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 21:31:28.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:31:28.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:28.583 | INFO | Uploaded 36GiB to GPU in 2.47684s (14.5347GiB/s) | 20260507 21:31:28.583 | INFO | Downloaded 34GiB to GPU in 2.46398s (13.7988GiB/s) | 20260507 21:31:33.726 | INFO | Analyzing chunk frames = [2745856,2746368) | 20260507 21:31:33.726 | INFO | MemInfo: { | 20260507 21:31:33.726 | INFO | "gpu" : " 17 GB", | 20260507 21:31:33.726 | INFO | "huge" : "", | 20260507 21:31:33.726 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:33.726 | INFO | "malloc" : " 14 GB", | 20260507 21:31:33.726 | INFO | "pinned" : " 29 GB" | 20260507 21:31:33.726 | INFO | } | 20260507 21:31:33.726 | INFO | Perf report for the last 5.14369 seconds: | 20260507 21:31:33.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:33.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.726 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 21:31:33.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.726 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:31:33.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:33.727 | INFO | Uploaded 36GiB to GPU in 2.42221s (14.8625GiB/s) | 20260507 21:31:33.727 | INFO | Downloaded 34GiB to GPU in 2.38996s (14.2262GiB/s) | 20260507 21:31:38.864 | INFO | Analyzing chunk frames = [2746368,2746880) | 20260507 21:31:38.864 | INFO | MemInfo: { | 20260507 21:31:38.864 | INFO | "gpu" : " 17 GB", | 20260507 21:31:38.864 | INFO | "huge" : "", | 20260507 21:31:38.864 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:38.864 | INFO | "malloc" : " 14 GB", | 20260507 21:31:38.864 | INFO | "pinned" : " 29 GB" | 20260507 21:31:38.864 | INFO | } | 20260507 21:31:38.864 | INFO | Perf report for the last 5.13803 seconds: | 20260507 21:31:38.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:38.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.864 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:31:38.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.864 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:31:38.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:38.865 | INFO | Uploaded 36GiB to GPU in 2.35417s (15.292GiB/s) | 20260507 21:31:38.865 | INFO | Downloaded 34GiB to GPU in 2.51328s (13.5281GiB/s) | 20260507 21:31:43.998 | INFO | Analyzing chunk frames = [2746880,2747392) | 20260507 21:31:43.998 | INFO | MemInfo: { | 20260507 21:31:43.998 | INFO | "gpu" : " 17 GB", | 20260507 21:31:43.998 | INFO | "huge" : "", | 20260507 21:31:43.998 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:43.998 | INFO | "malloc" : " 14 GB", | 20260507 21:31:43.998 | INFO | "pinned" : " 29 GB" | 20260507 21:31:43.998 | INFO | } | 20260507 21:31:43.998 | INFO | Perf report for the last 5.13357 seconds: | 20260507 21:31:43.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:43.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:31:43.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:31:43.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:43.998 | INFO | Uploaded 36GiB to GPU in 2.46948s (14.578GiB/s) | 20260507 21:31:43.998 | INFO | Downloaded 34GiB to GPU in 2.43632s (13.9555GiB/s) | 20260507 21:31:49.140 | INFO | Analyzing chunk frames = [2747392,2747904) | 20260507 21:31:49.140 | INFO | MemInfo: { | 20260507 21:31:49.140 | INFO | "gpu" : " 17 GB", | 20260507 21:31:49.140 | INFO | "huge" : "", | 20260507 21:31:49.140 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:49.140 | INFO | "malloc" : " 14 GB", | 20260507 21:31:49.140 | INFO | "pinned" : " 29 GB" | 20260507 21:31:49.140 | INFO | } | 20260507 21:31:49.141 | INFO | Perf report for the last 5.14272 seconds: | 20260507 21:31:49.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:49.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:31:49.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:31:49.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:49.141 | INFO | Uploaded 36GiB to GPU in 2.49117s (14.451GiB/s) | 20260507 21:31:49.141 | INFO | Downloaded 34GiB to GPU in 2.46884s (13.7717GiB/s) | 20260507 21:31:54.279 | INFO | Analyzing chunk frames = [2747904,2748416) | 20260507 21:31:54.279 | INFO | MemInfo: { | 20260507 21:31:54.279 | INFO | "gpu" : " 17 GB", | 20260507 21:31:54.279 | INFO | "huge" : "", | 20260507 21:31:54.279 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:54.279 | INFO | "malloc" : " 14 GB", | 20260507 21:31:54.279 | INFO | "pinned" : " 29 GB" | 20260507 21:31:54.279 | INFO | } | 20260507 21:31:54.279 | INFO | Perf report for the last 5.13851 seconds: | 20260507 21:31:54.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:54.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:31:54.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 21:31:54.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:54.279 | INFO | Uploaded 36GiB to GPU in 2.47133s (14.567GiB/s) | 20260507 21:31:54.280 | INFO | Downloaded 34GiB to GPU in 2.45787s (13.8331GiB/s) | 20260507 21:31:54.392 | INFO | Starting a round of Baz IO | 20260507 21:31:57.872 | INFO | Populating RTMetrics report | 20260507 21:31:57.873 | INFO | RTMetrics report populated | 20260507 21:31:57.873 | INFO | Opening tmp RTMetrics json file | 20260507 21:31:57.891 | INFO | Serializing RTMetrics to json | 20260507 21:31:57.897 | INFO | Serialization done, swapping tmp file now... | 20260507 21:31:57.898 | INFO | RTMetrics done | 20260507 21:31:57.971 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.578 seconds | 20260507 21:31:57.971 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.67% / 0.01% / 56.32% | 20260507 21:31:57.971 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.66262% / 2.38419e-05% / 4.76837e-05% | 20260507 21:31:57.971 | INFO | Baz IO overall: | 20260507 21:31:57.971 | INFO | BytesWritten : 1251411979265 | 20260507 21:31:57.971 | INFO | NumEvents : 606503793801 | 20260507 21:31:59.420 | INFO | Analyzing chunk frames = [2748416,2748928) | 20260507 21:31:59.420 | INFO | MemInfo: { | 20260507 21:31:59.420 | INFO | "gpu" : " 17 GB", | 20260507 21:31:59.420 | INFO | "huge" : "", | 20260507 21:31:59.420 | INFO | "huge_shared" : " 29 GB", | 20260507 21:31:59.420 | INFO | "malloc" : " 14 GB", | 20260507 21:31:59.420 | INFO | "pinned" : " 29 GB" | 20260507 21:31:59.420 | INFO | } | 20260507 21:31:59.420 | INFO | Perf report for the last 5.1414 seconds: | 20260507 21:31:59.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:31:59.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:59.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:59.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:59.421 | INFO | ANALYSIS : 69.2%, 2.9, 104.6ms, 0 | 20260507 21:31:59.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:31:59.421 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260507 21:31:59.421 | INFO | PRE_HQ : 68.1%, 1.8, 63.6ms, 0 | 20260507 21:31:59.421 | INFO | BAZWRITER : 69.2%, 4.5, 158.4ms, 0 | 20260507 21:31:59.421 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 21:31:59.421 | INFO | Uploaded 36GiB to GPU in 2.59063s (13.8962GiB/s) | 20260507 21:31:59.421 | INFO | Downloaded 38GiB to GPU in 2.71044s (14.0199GiB/s) | 20260507 21:32:04.561 | INFO | Analyzing chunk frames = [2748928,2749440) | 20260507 21:32:04.561 | INFO | MemInfo: { | 20260507 21:32:04.561 | INFO | "gpu" : " 17 GB", | 20260507 21:32:04.561 | INFO | "huge" : "", | 20260507 21:32:04.561 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:04.561 | INFO | "malloc" : " 14 GB", | 20260507 21:32:04.561 | INFO | "pinned" : " 29 GB" | 20260507 21:32:04.561 | INFO | } | 20260507 21:32:04.561 | INFO | Perf report for the last 5.14053 seconds: | 20260507 21:32:04.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:04.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:32:04.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:32:04.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:04.561 | INFO | Uploaded 36GiB to GPU in 2.51821s (14.2959GiB/s) | 20260507 21:32:04.561 | INFO | Downloaded 34GiB to GPU in 2.48052s (13.7068GiB/s) | 20260507 21:32:09.698 | INFO | Analyzing chunk frames = [2749440,2749952) | 20260507 21:32:09.698 | INFO | MemInfo: { | 20260507 21:32:09.698 | INFO | "gpu" : " 17 GB", | 20260507 21:32:09.698 | INFO | "huge" : "", | 20260507 21:32:09.698 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:09.698 | INFO | "malloc" : " 14 GB", | 20260507 21:32:09.698 | INFO | "pinned" : " 29 GB" | 20260507 21:32:09.698 | INFO | } | 20260507 21:32:09.698 | INFO | Perf report for the last 5.13695 seconds: | 20260507 21:32:09.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:09.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.698 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:32:09.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.698 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:32:09.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:09.699 | INFO | Uploaded 36GiB to GPU in 2.52649s (14.249GiB/s) | 20260507 21:32:09.699 | INFO | Downloaded 34GiB to GPU in 2.46369s (13.8004GiB/s) | 20260507 21:32:14.842 | INFO | Analyzing chunk frames = [2749952,2750464) | 20260507 21:32:14.842 | INFO | MemInfo: { | 20260507 21:32:14.842 | INFO | "gpu" : " 17 GB", | 20260507 21:32:14.842 | INFO | "huge" : "", | 20260507 21:32:14.842 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:14.842 | INFO | "malloc" : " 14 GB", | 20260507 21:32:14.842 | INFO | "pinned" : " 29 GB" | 20260507 21:32:14.842 | INFO | } | 20260507 21:32:14.842 | INFO | Perf report for the last 5.14439 seconds: | 20260507 21:32:14.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:14.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 21:32:14.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:32:14.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:14.843 | INFO | Uploaded 36GiB to GPU in 2.46365s (14.6125GiB/s) | 20260507 21:32:14.843 | INFO | Downloaded 34GiB to GPU in 2.38596s (14.25GiB/s) | 20260507 21:32:19.977 | INFO | Analyzing chunk frames = [2750464,2750976) | 20260507 21:32:19.977 | INFO | MemInfo: { | 20260507 21:32:19.977 | INFO | "gpu" : " 17 GB", | 20260507 21:32:19.977 | INFO | "huge" : "", | 20260507 21:32:19.977 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:19.977 | INFO | "malloc" : " 14 GB", | 20260507 21:32:19.977 | INFO | "pinned" : " 29 GB" | 20260507 21:32:19.977 | INFO | } | 20260507 21:32:19.977 | INFO | Perf report for the last 5.13478 seconds: | 20260507 21:32:19.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:19.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 21:32:19.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:32:19.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:19.977 | INFO | Uploaded 36GiB to GPU in 2.37101s (15.1834GiB/s) | 20260507 21:32:19.977 | INFO | Downloaded 34GiB to GPU in 2.49331s (13.6365GiB/s) | 20260507 21:32:25.113 | INFO | Analyzing chunk frames = [2750976,2751488) | 20260507 21:32:25.113 | INFO | MemInfo: { | 20260507 21:32:25.113 | INFO | "gpu" : " 17 GB", | 20260507 21:32:25.113 | INFO | "huge" : "", | 20260507 21:32:25.113 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:25.113 | INFO | "malloc" : " 14 GB", | 20260507 21:32:25.113 | INFO | "pinned" : " 29 GB" | 20260507 21:32:25.113 | INFO | } | 20260507 21:32:25.113 | INFO | Perf report for the last 5.13623 seconds: | 20260507 21:32:25.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:25.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.113 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:32:25.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.113 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:32:25.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:25.114 | INFO | Uploaded 36GiB to GPU in 2.52018s (14.2847GiB/s) | 20260507 21:32:25.114 | INFO | Downloaded 34GiB to GPU in 2.41335s (14.0883GiB/s) | 20260507 21:32:30.250 | INFO | Analyzing chunk frames = [2751488,2752000) | 20260507 21:32:30.250 | INFO | MemInfo: { | 20260507 21:32:30.250 | INFO | "gpu" : " 17 GB", | 20260507 21:32:30.250 | INFO | "huge" : "", | 20260507 21:32:30.250 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:30.250 | INFO | "malloc" : " 14 GB", | 20260507 21:32:30.250 | INFO | "pinned" : " 29 GB" | 20260507 21:32:30.250 | INFO | } | 20260507 21:32:30.250 | INFO | Perf report for the last 5.13686 seconds: | 20260507 21:32:30.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:30.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.250 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 21:32:30.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.251 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 21:32:30.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:30.251 | INFO | Uploaded 36GiB to GPU in 2.52355s (14.2656GiB/s) | 20260507 21:32:30.251 | INFO | Downloaded 34GiB to GPU in 2.43675s (13.953GiB/s) | 20260507 21:32:35.392 | INFO | Analyzing chunk frames = [2752000,2752512) | 20260507 21:32:35.392 | INFO | MemInfo: { | 20260507 21:32:35.392 | INFO | "gpu" : " 17 GB", | 20260507 21:32:35.392 | INFO | "huge" : "", | 20260507 21:32:35.392 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:35.392 | INFO | "malloc" : " 14 GB", | 20260507 21:32:35.392 | INFO | "pinned" : " 29 GB" | 20260507 21:32:35.392 | INFO | } | 20260507 21:32:35.392 | INFO | Perf report for the last 5.14191 seconds: | 20260507 21:32:35.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:35.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.392 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 21:32:35.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.392 | INFO | PRE_HQ : 36.3%, 1.0, 18.8ms, 0 | 20260507 21:32:35.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:35.393 | INFO | Uploaded 36GiB to GPU in 2.53414s (14.206GiB/s) | 20260507 21:32:35.393 | INFO | Downloaded 34GiB to GPU in 2.43706s (13.9512GiB/s) | 20260507 21:32:35.393 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:32:35.393 | INFO | Total mem usage: 16.7 GiB | 20260507 21:32:35.393 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:32:35.393 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -5.7%) | 20260507 21:32:35.393 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:32:35.393 | INFO | High Water Breakdown: | 20260507 21:32:35.393 | INFO | 60.9% : 9.61 GiB ( 93.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:32:35.393 | INFO | 39% : 6.16 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:32:35.393 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:32:35.393 | INFO | ------------------------------------------ | 20260507 21:32:35.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:32:35.393 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:32:35.393 | INFO | Total mem usage: 26.5 GiB | 20260507 21:32:35.393 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:32:35.393 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 21:32:35.393 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 21:32:35.393 | INFO | High Water Breakdown: | 20260507 21:32:35.393 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:32:35.393 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:32:35.393 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:32:35.393 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:32:35.393 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:32:35.393 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:32:35.393 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:32:35.393 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:32:35.393 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:32:35.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:32:35.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:32:35.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:32:35.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:32:35.393 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:32:35.393 | INFO | ------------------------------------------ | 20260507 21:32:35.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:32:35.393 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:32:35.393 | INFO | Total mem usage: 27.8 GiB | 20260507 21:32:35.393 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:32:35.393 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:32:35.393 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:32:35.394 | INFO | High Water Breakdown: | 20260507 21:32:35.394 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:32:35.394 | INFO | ------------------------------------------ | 20260507 21:32:35.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:32:35.394 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:32:35.394 | INFO | Total mem usage: 15.7 GiB | 20260507 21:32:35.394 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:32:35.394 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:32:35.394 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:32:35.394 | INFO | High Water Breakdown: | 20260507 21:32:35.394 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:32:35.394 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:32:35.394 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:32:35.394 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:32:35.394 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:32:35.394 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:32:35.394 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:32:35.394 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:32:35.394 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:32:35.394 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:32:35.394 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:32:35.394 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:32:35.394 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:32:35.394 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:32:35.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:32:35.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:32:35.394 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:32:35.394 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:32:35.394 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:32:35.394 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:32:35.394 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:32:35.394 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:32:35.394 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:32:35.394 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:32:35.394 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:32:35.394 | INFO | ------------------------------------------ | 20260507 21:32:35.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:32:35.394 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:32:35.394 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:32:35.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:32:35.394 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.57e+01, 1.41e+01 / 1.62e+00 | 20260507 21:32:35.394 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.03e-01, 1.15e-02 / 5.68e-03 | 20260507 21:32:35.394 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.46e+00, 5.89e-01 / 2.61e-01 | 20260507 21:32:35.394 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.49e+00, 2.10e-01 / 1.27e-01 | 20260507 21:32:35.394 | INFO | PerfReport: 15.74%, DME -- 4.16e+01 / 4.48e+01, 3.87e+00 / 1.35e+00 | 20260507 21:32:35.394 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.85e+00 / 1.93e+00 | 20260507 21:32:35.394 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.46e+00 / 2.73e+00, 4.29e-01 / 2.77e-01 | 20260507 21:32:35.394 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.35e+00, 2.97e-01 / 1.41e-01 | 20260507 21:32:35.394 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.63e-01 / 3.96e-01, 6.05e-02 / 3.20e-02 | 20260507 21:32:35.394 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:32:35.394 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:32:35.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:32:35.394 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 21:32:35.394 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260507 21:32:35.394 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 21:32:35.395 | INFO | PerfReport: 31.07%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:32:35.395 | INFO | PerfReport: 1.80%, LockUpload -- 1.92e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 21:32:35.395 | INFO | PerfReport: 2.60%, LockDownload -- 2.77e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260507 21:32:35.395 | INFO | PerfReport: 15.89%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:32:35.395 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260507 21:32:35.507 | INFO | Starting a round of Baz IO | 20260507 21:32:38.977 | INFO | Populating RTMetrics report | 20260507 21:32:38.978 | INFO | RTMetrics report populated | 20260507 21:32:38.978 | INFO | Opening tmp RTMetrics json file | 20260507 21:32:39.002 | INFO | Serializing RTMetrics to json | 20260507 21:32:39.005 | INFO | Serialization done, swapping tmp file now... | 20260507 21:32:39.006 | INFO | RTMetrics done | 20260507 21:32:39.075 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.567 seconds | 20260507 21:32:39.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.66% / 0.01% / 56.33% | 20260507 21:32:39.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.661% / 2.38419e-05% / 5.5631e-05% | 20260507 21:32:39.075 | INFO | Baz IO overall: | 20260507 21:32:39.075 | INFO | BytesWritten : 1253301218946 | 20260507 21:32:39.075 | INFO | NumEvents : 607417693164 | 20260507 21:32:40.534 | INFO | Analyzing chunk frames = [2752512,2753024) | 20260507 21:32:40.534 | INFO | MemInfo: { | 20260507 21:32:40.534 | INFO | "gpu" : " 17 GB", | 20260507 21:32:40.534 | INFO | "huge" : "", | 20260507 21:32:40.534 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:40.534 | INFO | "malloc" : " 14 GB", | 20260507 21:32:40.534 | INFO | "pinned" : " 29 GB" | 20260507 21:32:40.534 | INFO | } | 20260507 21:32:40.534 | INFO | Perf report for the last 5.14161 seconds: | 20260507 21:32:40.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:40.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:40.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:40.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:40.534 | INFO | ANALYSIS : 69.0%, 2.9, 105.5ms, 0 | 20260507 21:32:40.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:40.534 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 21:32:40.534 | INFO | PRE_HQ : 67.5%, 1.8, 62.1ms, 0 | 20260507 21:32:40.534 | INFO | BAZWRITER : 69.0%, 4.4, 155.8ms, 0 | 20260507 21:32:40.534 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:32:40.535 | INFO | Uploaded 36GiB to GPU in 2.56721s (14.023GiB/s) | 20260507 21:32:40.535 | INFO | Downloaded 38GiB to GPU in 2.69898s (14.0794GiB/s) | 20260507 21:32:45.676 | INFO | Analyzing chunk frames = [2753024,2753536) | 20260507 21:32:45.676 | INFO | MemInfo: { | 20260507 21:32:45.676 | INFO | "gpu" : " 17 GB", | 20260507 21:32:45.676 | INFO | "huge" : "", | 20260507 21:32:45.676 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:45.676 | INFO | "malloc" : " 14 GB", | 20260507 21:32:45.676 | INFO | "pinned" : " 29 GB" | 20260507 21:32:45.676 | INFO | } | 20260507 21:32:45.676 | INFO | Perf report for the last 5.14224 seconds: | 20260507 21:32:45.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:45.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.676 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260507 21:32:45.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.676 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 21:32:45.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:45.677 | INFO | Uploaded 36GiB to GPU in 2.53041s (14.227GiB/s) | 20260507 21:32:45.677 | INFO | Downloaded 34GiB to GPU in 2.47908s (13.7147GiB/s) | 20260507 21:32:50.814 | INFO | Analyzing chunk frames = [2753536,2754048) | 20260507 21:32:50.814 | INFO | MemInfo: { | 20260507 21:32:50.814 | INFO | "gpu" : " 17 GB", | 20260507 21:32:50.814 | INFO | "huge" : "", | 20260507 21:32:50.814 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:50.814 | INFO | "malloc" : " 14 GB", | 20260507 21:32:50.814 | INFO | "pinned" : " 29 GB" | 20260507 21:32:50.814 | INFO | } | 20260507 21:32:50.814 | INFO | Perf report for the last 5.13833 seconds: | 20260507 21:32:50.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:50.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.815 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:32:50.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.815 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:32:50.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:50.815 | INFO | Uploaded 36GiB to GPU in 2.55039s (14.1155GiB/s) | 20260507 21:32:50.815 | INFO | Downloaded 34GiB to GPU in 2.48364s (13.6896GiB/s) | 20260507 21:32:55.948 | INFO | Analyzing chunk frames = [2754048,2754560) | 20260507 21:32:55.948 | INFO | MemInfo: { | 20260507 21:32:55.948 | INFO | "gpu" : " 17 GB", | 20260507 21:32:55.948 | INFO | "huge" : "", | 20260507 21:32:55.948 | INFO | "huge_shared" : " 29 GB", | 20260507 21:32:55.948 | INFO | "malloc" : " 14 GB", | 20260507 21:32:55.948 | INFO | "pinned" : " 29 GB" | 20260507 21:32:55.948 | INFO | } | 20260507 21:32:55.948 | INFO | Perf report for the last 5.13397 seconds: | 20260507 21:32:55.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:32:55.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.948 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260507 21:32:55.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.948 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:32:55.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:32:55.949 | INFO | Uploaded 36GiB to GPU in 2.4519s (14.6825GiB/s) | 20260507 21:32:55.949 | INFO | Downloaded 34GiB to GPU in 2.38619s (14.2487GiB/s) | 20260507 21:33:01.091 | INFO | Analyzing chunk frames = [2754560,2755072) | 20260507 21:33:01.091 | INFO | MemInfo: { | 20260507 21:33:01.091 | INFO | "gpu" : " 17 GB", | 20260507 21:33:01.091 | INFO | "huge" : "", | 20260507 21:33:01.091 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:01.091 | INFO | "malloc" : " 14 GB", | 20260507 21:33:01.091 | INFO | "pinned" : " 29 GB" | 20260507 21:33:01.091 | INFO | } | 20260507 21:33:01.091 | INFO | Perf report for the last 5.14233 seconds: | 20260507 21:33:01.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:01.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 21:33:01.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:33:01.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:01.091 | INFO | Uploaded 36GiB to GPU in 2.33505s (15.4172GiB/s) | 20260507 21:33:01.091 | INFO | Downloaded 34GiB to GPU in 2.50282s (13.5847GiB/s) | 20260507 21:33:06.230 | INFO | Analyzing chunk frames = [2755072,2755584) | 20260507 21:33:06.230 | INFO | MemInfo: { | 20260507 21:33:06.230 | INFO | "gpu" : " 17 GB", | 20260507 21:33:06.230 | INFO | "huge" : "", | 20260507 21:33:06.230 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:06.230 | INFO | "malloc" : " 14 GB", | 20260507 21:33:06.230 | INFO | "pinned" : " 29 GB" | 20260507 21:33:06.230 | INFO | } | 20260507 21:33:06.230 | INFO | Perf report for the last 5.13968 seconds: | 20260507 21:33:06.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:06.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.230 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 21:33:06.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.231 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:33:06.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:06.231 | INFO | Uploaded 36GiB to GPU in 2.49974s (14.4015GiB/s) | 20260507 21:33:06.231 | INFO | Downloaded 34GiB to GPU in 2.42599s (14.0149GiB/s) | 20260507 21:33:11.370 | INFO | Analyzing chunk frames = [2755584,2756096) | 20260507 21:33:11.370 | INFO | MemInfo: { | 20260507 21:33:11.370 | INFO | "gpu" : " 17 GB", | 20260507 21:33:11.370 | INFO | "huge" : "", | 20260507 21:33:11.370 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:11.370 | INFO | "malloc" : " 14 GB", | 20260507 21:33:11.370 | INFO | "pinned" : " 29 GB" | 20260507 21:33:11.370 | INFO | } | 20260507 21:33:11.370 | INFO | Perf report for the last 5.1396 seconds: | 20260507 21:33:11.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:11.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.370 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:33:11.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.370 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 21:33:11.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:11.371 | INFO | Uploaded 36GiB to GPU in 2.50133s (14.3924GiB/s) | 20260507 21:33:11.371 | INFO | Downloaded 34GiB to GPU in 2.46216s (13.809GiB/s) | 20260507 21:33:16.511 | INFO | Analyzing chunk frames = [2756096,2756608) | 20260507 21:33:16.511 | INFO | MemInfo: { | 20260507 21:33:16.511 | INFO | "gpu" : " 17 GB", | 20260507 21:33:16.511 | INFO | "huge" : "", | 20260507 21:33:16.511 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:16.511 | INFO | "malloc" : " 14 GB", | 20260507 21:33:16.511 | INFO | "pinned" : " 29 GB" | 20260507 21:33:16.511 | INFO | } | 20260507 21:33:16.511 | INFO | Perf report for the last 5.14081 seconds: | 20260507 21:33:16.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:16.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.511 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:33:16.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.511 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 21:33:16.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:16.512 | INFO | Uploaded 36GiB to GPU in 2.49106s (14.4517GiB/s) | 20260507 21:33:16.512 | INFO | Downloaded 34GiB to GPU in 2.44127s (13.9272GiB/s) | 20260507 21:33:16.626 | INFO | Starting a round of Baz IO | 20260507 21:33:20.115 | INFO | Populating RTMetrics report | 20260507 21:33:20.115 | INFO | RTMetrics report populated | 20260507 21:33:20.115 | INFO | Opening tmp RTMetrics json file | 20260507 21:33:20.208 | INFO | Serializing RTMetrics to json | 20260507 21:33:20.281 | INFO | Serialization done, swapping tmp file now... | 20260507 21:33:20.286 | INFO | RTMetrics done | 20260507 21:33:20.315 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.688 seconds | 20260507 21:33:20.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.66% / 0.01% / 56.34% | 20260507 21:33:20.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.65454% / 1.58946e-05% / 4.37101e-05% | 20260507 21:33:20.315 | INFO | Baz IO overall: | 20260507 21:33:20.315 | INFO | BytesWritten : 1255188473086 | 20260507 21:33:20.315 | INFO | NumEvents : 608330102313 | 20260507 21:33:21.651 | INFO | Analyzing chunk frames = [2756608,2757120) | 20260507 21:33:21.651 | INFO | MemInfo: { | 20260507 21:33:21.651 | INFO | "gpu" : " 17 GB", | 20260507 21:33:21.651 | INFO | "huge" : "", | 20260507 21:33:21.651 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:21.651 | INFO | "malloc" : " 14 GB", | 20260507 21:33:21.651 | INFO | "pinned" : " 29 GB" | 20260507 21:33:21.651 | INFO | } | 20260507 21:33:21.651 | INFO | Perf report for the last 5.14013 seconds: | 20260507 21:33:21.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:21.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:21.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:21.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:21.651 | INFO | ANALYSIS : 69.1%, 2.9, 104.6ms, 0 | 20260507 21:33:21.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:21.651 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260507 21:33:21.651 | INFO | PRE_HQ : 68.0%, 1.8, 63.8ms, 0 | 20260507 21:33:21.651 | INFO | BAZWRITER : 71.4%, 3.9, 143.7ms, 0 | 20260507 21:33:21.651 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.9ms, 0 | 20260507 21:33:21.651 | INFO | Uploaded 36GiB to GPU in 2.59391s (13.8786GiB/s) | 20260507 21:33:21.651 | INFO | Downloaded 38GiB to GPU in 2.70091s (14.0693GiB/s) | 20260507 21:33:26.792 | INFO | Analyzing chunk frames = [2757120,2757632) | 20260507 21:33:26.792 | INFO | MemInfo: { | 20260507 21:33:26.792 | INFO | "gpu" : " 17 GB", | 20260507 21:33:26.792 | INFO | "huge" : "", | 20260507 21:33:26.792 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:26.792 | INFO | "malloc" : " 14 GB", | 20260507 21:33:26.792 | INFO | "pinned" : " 29 GB" | 20260507 21:33:26.792 | INFO | } | 20260507 21:33:26.792 | INFO | Perf report for the last 5.14062 seconds: | 20260507 21:33:26.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:26.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:33:26.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:33:26.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:26.792 | INFO | Uploaded 36GiB to GPU in 2.54112s (14.167GiB/s) | 20260507 21:33:26.792 | INFO | Downloaded 34GiB to GPU in 2.48296s (13.6933GiB/s) | 20260507 21:33:31.930 | INFO | Analyzing chunk frames = [2757632,2758144) | 20260507 21:33:31.930 | INFO | MemInfo: { | 20260507 21:33:31.930 | INFO | "gpu" : " 17 GB", | 20260507 21:33:31.930 | INFO | "huge" : "", | 20260507 21:33:31.930 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:31.930 | INFO | "malloc" : " 14 GB", | 20260507 21:33:31.930 | INFO | "pinned" : " 29 GB" | 20260507 21:33:31.930 | INFO | } | 20260507 21:33:31.930 | INFO | Perf report for the last 5.13814 seconds: | 20260507 21:33:31.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:31.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260507 21:33:31.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:33:31.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:31.930 | INFO | Uploaded 36GiB to GPU in 2.56641s (14.0274GiB/s) | 20260507 21:33:31.930 | INFO | Downloaded 34GiB to GPU in 2.45421s (13.8537GiB/s) | 20260507 21:33:37.061 | INFO | Analyzing chunk frames = [2758144,2758656) | 20260507 21:33:37.061 | INFO | MemInfo: { | 20260507 21:33:37.061 | INFO | "gpu" : " 17 GB", | 20260507 21:33:37.061 | INFO | "huge" : "", | 20260507 21:33:37.061 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:37.061 | INFO | "malloc" : " 14 GB", | 20260507 21:33:37.061 | INFO | "pinned" : " 29 GB" | 20260507 21:33:37.061 | INFO | } | 20260507 21:33:37.061 | INFO | Perf report for the last 5.13149 seconds: | 20260507 21:33:37.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:37.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.061 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260507 21:33:37.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.061 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:33:37.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:37.062 | INFO | Uploaded 36GiB to GPU in 2.50309s (14.3822GiB/s) | 20260507 21:33:37.062 | INFO | Downloaded 34GiB to GPU in 2.38565s (14.2519GiB/s) | 20260507 21:33:42.202 | INFO | Analyzing chunk frames = [2758656,2759168) | 20260507 21:33:42.202 | INFO | MemInfo: { | 20260507 21:33:42.202 | INFO | "gpu" : " 17 GB", | 20260507 21:33:42.202 | INFO | "huge" : "", | 20260507 21:33:42.202 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:42.202 | INFO | "malloc" : " 14 GB", | 20260507 21:33:42.202 | INFO | "pinned" : " 29 GB" | 20260507 21:33:42.202 | INFO | } | 20260507 21:33:42.202 | INFO | Perf report for the last 5.1404 seconds: | 20260507 21:33:42.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:42.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 21:33:42.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:33:42.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:42.202 | INFO | Uploaded 36GiB to GPU in 2.3678s (15.204GiB/s) | 20260507 21:33:42.202 | INFO | Downloaded 34GiB to GPU in 2.4835s (13.6903GiB/s) | 20260507 21:33:47.347 | INFO | Analyzing chunk frames = [2759168,2759680) | 20260507 21:33:47.347 | INFO | MemInfo: { | 20260507 21:33:47.347 | INFO | "gpu" : " 17 GB", | 20260507 21:33:47.347 | INFO | "huge" : "", | 20260507 21:33:47.347 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:47.347 | INFO | "malloc" : " 14 GB", | 20260507 21:33:47.347 | INFO | "pinned" : " 29 GB" | 20260507 21:33:47.347 | INFO | } | 20260507 21:33:47.347 | INFO | Perf report for the last 5.14542 seconds: | 20260507 21:33:47.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:47.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 21:33:47.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:33:47.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:47.347 | INFO | Uploaded 36GiB to GPU in 2.45906s (14.6397GiB/s) | 20260507 21:33:47.347 | INFO | Downloaded 34GiB to GPU in 2.50145s (13.5921GiB/s) | 20260507 21:33:52.488 | INFO | Analyzing chunk frames = [2759680,2760192) | 20260507 21:33:52.488 | INFO | MemInfo: { | 20260507 21:33:52.489 | INFO | "gpu" : " 17 GB", | 20260507 21:33:52.489 | INFO | "huge" : "", | 20260507 21:33:52.489 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:52.489 | INFO | "malloc" : " 14 GB", | 20260507 21:33:52.489 | INFO | "pinned" : " 29 GB" | 20260507 21:33:52.489 | INFO | } | 20260507 21:33:52.489 | INFO | Perf report for the last 5.14148 seconds: | 20260507 21:33:52.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:52.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.489 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:33:52.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.489 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 21:33:52.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:52.490 | INFO | Uploaded 36GiB to GPU in 2.52012s (14.2851GiB/s) | 20260507 21:33:52.490 | INFO | Downloaded 34GiB to GPU in 2.44609s (13.8997GiB/s) | 20260507 21:33:57.631 | INFO | Analyzing chunk frames = [2760192,2760704) | 20260507 21:33:57.631 | INFO | MemInfo: { | 20260507 21:33:57.631 | INFO | "gpu" : " 17 GB", | 20260507 21:33:57.631 | INFO | "huge" : "", | 20260507 21:33:57.631 | INFO | "huge_shared" : " 29 GB", | 20260507 21:33:57.631 | INFO | "malloc" : " 14 GB", | 20260507 21:33:57.631 | INFO | "pinned" : " 29 GB" | 20260507 21:33:57.631 | INFO | } | 20260507 21:33:57.632 | INFO | Perf report for the last 5.14276 seconds: | 20260507 21:33:57.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:33:57.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:33:57.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | PRE_HQ : 36.1%, 1.0, 18.8ms, 0 | 20260507 21:33:57.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:33:57.632 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:33:57.632 | INFO | Total mem usage: 16.7 GiB | 20260507 21:33:57.632 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:33:57.632 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -6.92%) | 20260507 21:33:57.632 | INFO | Uploaded 36GiB to GPU in 2.51767s (14.2989GiB/s) | 20260507 21:33:57.632 | INFO | Downloaded 34GiB to GPU in 2.44126s (13.9272GiB/s) | 20260507 21:33:57.632 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:33:57.632 | INFO | High Water Breakdown: | 20260507 21:33:57.632 | INFO | 60.7% : 9.48 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:33:57.632 | INFO | 39.1% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:33:57.632 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:33:57.632 | INFO | ------------------------------------------ | 20260507 21:33:57.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:33:57.632 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:33:57.633 | INFO | Total mem usage: 26.5 GiB | 20260507 21:33:57.633 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:33:57.633 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 21:33:57.633 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 21:33:57.633 | INFO | High Water Breakdown: | 20260507 21:33:57.633 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:33:57.633 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:33:57.633 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:33:57.633 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:33:57.633 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:33:57.633 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:33:57.633 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:33:57.633 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:33:57.633 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:33:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:33:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:33:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:33:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:33:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:33:57.633 | INFO | ------------------------------------------ | 20260507 21:33:57.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:33:57.633 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:33:57.633 | INFO | Total mem usage: 27.8 GiB | 20260507 21:33:57.633 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:33:57.633 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:33:57.633 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:33:57.633 | INFO | High Water Breakdown: | 20260507 21:33:57.633 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:33:57.633 | INFO | ------------------------------------------ | 20260507 21:33:57.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:33:57.633 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:33:57.633 | INFO | Total mem usage: 15.7 GiB | 20260507 21:33:57.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:33:57.633 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:33:57.633 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:33:57.633 | INFO | High Water Breakdown: | 20260507 21:33:57.633 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:33:57.633 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:33:57.633 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:33:57.633 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:33:57.633 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:33:57.633 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:33:57.633 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:33:57.633 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:33:57.633 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:33:57.633 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:33:57.633 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:33:57.633 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:33:57.633 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:33:57.633 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:33:57.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:33:57.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:33:57.633 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:33:57.633 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:33:57.633 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:33:57.633 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:33:57.633 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:33:57.633 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:33:57.633 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:33:57.633 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:33:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:33:57.633 | INFO | ------------------------------------------ | 20260507 21:33:57.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:33:57.633 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:33:57.633 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:33:57.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:33:57.633 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.42e+01 / 1.46e+00 | 20260507 21:33:57.633 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.17e-02 / 6.62e-03 | 20260507 21:33:57.633 | INFO | PerfReport: 12.12%, Baseline -- 4.01e+00 / 4.46e+00, 6.02e-01 / 2.67e-01 | 20260507 21:33:57.633 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.16e-01 | 20260507 21:33:57.633 | INFO | PerfReport: 15.84%, DME -- 4.19e+01 / 4.54e+01, 4.00e+00 / 1.43e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 2.14e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.46e+00 / 2.75e+00, 4.36e-01 / 2.88e-01 | 20260507 21:33:57.634 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.37e+00, 2.97e-01 / 1.79e-01 | 20260507 21:33:57.634 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.62e-01 / 3.91e-01, 6.01e-02 / 2.61e-02 | 20260507 21:33:57.634 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:33:57.634 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:33:57.634 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:33:57.634 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 30.99%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 1.99%, LockUpload -- 2.12e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 2.42%, LockDownload -- 2.59e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 15.89%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:33:57.634 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 21:33:57.743 | INFO | Starting a round of Baz IO | 20260507 21:34:01.198 | INFO | Populating RTMetrics report | 20260507 21:34:01.199 | INFO | RTMetrics report populated | 20260507 21:34:01.199 | INFO | Opening tmp RTMetrics json file | 20260507 21:34:01.206 | INFO | Serializing RTMetrics to json | 20260507 21:34:01.215 | INFO | Serialization done, swapping tmp file now... | 20260507 21:34:01.242 | INFO | RTMetrics done | 20260507 21:34:01.322 | INFO | Finished a round of Baz IO: Wrote 1.757 GiB in 3.579 seconds | 20260507 21:34:01.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.65% / 0.01% / 56.34% | 20260507 21:34:01.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64918% / 1.58946e-05% / 2.38419e-05% | 20260507 21:34:01.322 | INFO | Baz IO overall: | 20260507 21:34:01.322 | INFO | BytesWritten : 1257075098516 | 20260507 21:34:01.322 | INFO | NumEvents : 609241868385 | 20260507 21:34:02.770 | INFO | Analyzing chunk frames = [2760704,2761216) | 20260507 21:34:02.770 | INFO | MemInfo: { | 20260507 21:34:02.770 | INFO | "gpu" : " 17 GB", | 20260507 21:34:02.770 | INFO | "huge" : "", | 20260507 21:34:02.770 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:02.770 | INFO | "malloc" : " 14 GB", | 20260507 21:34:02.770 | INFO | "pinned" : " 29 GB" | 20260507 21:34:02.770 | INFO | } | 20260507 21:34:02.770 | INFO | Perf report for the last 5.13843 seconds: | 20260507 21:34:02.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:02.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:02.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:02.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:02.770 | INFO | ANALYSIS : 68.9%, 2.9, 103.9ms, 0 | 20260507 21:34:02.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:02.770 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 21:34:02.770 | INFO | PRE_HQ : 67.8%, 1.8, 63.8ms, 0 | 20260507 21:34:02.770 | INFO | BAZWRITER : 69.2%, 3.9, 140.2ms, 0 | 20260507 21:34:02.770 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 21:34:02.770 | INFO | Uploaded 36GiB to GPU in 2.45185s (14.6828GiB/s) | 20260507 21:34:02.770 | INFO | Downloaded 38GiB to GPU in 2.67333s (14.2145GiB/s) | 20260507 21:34:07.904 | INFO | Analyzing chunk frames = [2761216,2761728) | 20260507 21:34:07.904 | INFO | MemInfo: { | 20260507 21:34:07.904 | INFO | "gpu" : " 17 GB", | 20260507 21:34:07.904 | INFO | "huge" : "", | 20260507 21:34:07.904 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:07.904 | INFO | "malloc" : " 14 GB", | 20260507 21:34:07.904 | INFO | "pinned" : " 29 GB" | 20260507 21:34:07.904 | INFO | } | 20260507 21:34:07.904 | INFO | Perf report for the last 5.13437 seconds: | 20260507 21:34:07.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:07.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.904 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 21:34:07.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.904 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:34:07.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:07.905 | INFO | Uploaded 36GiB to GPU in 2.55884s (14.0689GiB/s) | 20260507 21:34:07.905 | INFO | Downloaded 34GiB to GPU in 2.48753s (13.6682GiB/s) | 20260507 21:34:13.040 | INFO | Analyzing chunk frames = [2761728,2762240) | 20260507 21:34:13.040 | INFO | MemInfo: { | 20260507 21:34:13.040 | INFO | "gpu" : " 17 GB", | 20260507 21:34:13.040 | INFO | "huge" : "", | 20260507 21:34:13.041 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:13.041 | INFO | "malloc" : " 14 GB", | 20260507 21:34:13.041 | INFO | "pinned" : " 29 GB" | 20260507 21:34:13.041 | INFO | } | 20260507 21:34:13.041 | INFO | Perf report for the last 5.13623 seconds: | 20260507 21:34:13.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:13.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:34:13.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:34:13.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:13.041 | INFO | Uploaded 36GiB to GPU in 2.5499s (14.1182GiB/s) | 20260507 21:34:13.041 | INFO | Downloaded 34GiB to GPU in 2.46816s (13.7755GiB/s) | 20260507 21:34:18.183 | INFO | Analyzing chunk frames = [2762240,2762752) | 20260507 21:34:18.183 | INFO | MemInfo: { | 20260507 21:34:18.183 | INFO | "gpu" : " 17 GB", | 20260507 21:34:18.183 | INFO | "huge" : "", | 20260507 21:34:18.183 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:18.183 | INFO | "malloc" : " 14 GB", | 20260507 21:34:18.183 | INFO | "pinned" : " 29 GB" | 20260507 21:34:18.183 | INFO | } | 20260507 21:34:18.183 | INFO | Perf report for the last 5.14243 seconds: | 20260507 21:34:18.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:18.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.183 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 21:34:18.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.183 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:34:18.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:18.184 | INFO | Uploaded 36GiB to GPU in 2.46047s (14.6313GiB/s) | 20260507 21:34:18.184 | INFO | Downloaded 34GiB to GPU in 2.38653s (14.2466GiB/s) | 20260507 21:34:23.322 | INFO | Analyzing chunk frames = [2762752,2763264) | 20260507 21:34:23.322 | INFO | MemInfo: { | 20260507 21:34:23.322 | INFO | "gpu" : " 17 GB", | 20260507 21:34:23.322 | INFO | "huge" : "", | 20260507 21:34:23.322 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:23.322 | INFO | "malloc" : " 14 GB", | 20260507 21:34:23.322 | INFO | "pinned" : " 29 GB" | 20260507 21:34:23.322 | INFO | } | 20260507 21:34:23.322 | INFO | Perf report for the last 5.13923 seconds: | 20260507 21:34:23.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:23.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.322 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:34:23.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.322 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:34:23.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:23.323 | INFO | Uploaded 36GiB to GPU in 2.35179s (15.3075GiB/s) | 20260507 21:34:23.323 | INFO | Downloaded 34GiB to GPU in 2.48251s (13.6958GiB/s) | 20260507 21:34:28.461 | INFO | Analyzing chunk frames = [2763264,2763776) | 20260507 21:34:28.461 | INFO | MemInfo: { | 20260507 21:34:28.461 | INFO | "gpu" : " 17 GB", | 20260507 21:34:28.461 | INFO | "huge" : "", | 20260507 21:34:28.461 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:28.461 | INFO | "malloc" : " 14 GB", | 20260507 21:34:28.461 | INFO | "pinned" : " 29 GB" | 20260507 21:34:28.461 | INFO | } | 20260507 21:34:28.461 | INFO | Perf report for the last 5.13873 seconds: | 20260507 21:34:28.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:28.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 21:34:28.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:34:28.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:28.461 | INFO | Uploaded 36GiB to GPU in 2.52716s (14.2452GiB/s) | 20260507 21:34:28.461 | INFO | Downloaded 34GiB to GPU in 2.40263s (14.1512GiB/s) | 20260507 21:34:33.602 | INFO | Analyzing chunk frames = [2763776,2764288) | 20260507 21:34:33.602 | INFO | MemInfo: { | 20260507 21:34:33.602 | INFO | "gpu" : " 17 GB", | 20260507 21:34:33.602 | INFO | "huge" : "", | 20260507 21:34:33.602 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:33.602 | INFO | "malloc" : " 14 GB", | 20260507 21:34:33.602 | INFO | "pinned" : " 29 GB" | 20260507 21:34:33.602 | INFO | } | 20260507 21:34:33.602 | INFO | Perf report for the last 5.14097 seconds: | 20260507 21:34:33.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:33.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:34:33.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 21:34:33.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:33.602 | INFO | Uploaded 36GiB to GPU in 2.54751s (14.1314GiB/s) | 20260507 21:34:33.603 | INFO | Downloaded 34GiB to GPU in 2.46086s (13.8163GiB/s) | 20260507 21:34:38.742 | INFO | Analyzing chunk frames = [2764288,2764800) | 20260507 21:34:38.742 | INFO | MemInfo: { | 20260507 21:34:38.742 | INFO | "gpu" : " 17 GB", | 20260507 21:34:38.742 | INFO | "huge" : "", | 20260507 21:34:38.742 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:38.742 | INFO | "malloc" : " 14 GB", | 20260507 21:34:38.742 | INFO | "pinned" : " 29 GB" | 20260507 21:34:38.742 | INFO | } | 20260507 21:34:38.742 | INFO | Perf report for the last 5.14012 seconds: | 20260507 21:34:38.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:38.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:34:38.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260507 21:34:38.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:38.742 | INFO | Uploaded 36GiB to GPU in 2.5222s (14.2733GiB/s) | 20260507 21:34:38.742 | INFO | Downloaded 34GiB to GPU in 2.44103s (13.9286GiB/s) | 20260507 21:34:38.839 | INFO | Starting a round of Baz IO | 20260507 21:34:42.316 | INFO | Populating RTMetrics report | 20260507 21:34:42.317 | INFO | RTMetrics report populated | 20260507 21:34:42.317 | INFO | Opening tmp RTMetrics json file | 20260507 21:34:42.345 | INFO | Serializing RTMetrics to json | 20260507 21:34:42.365 | INFO | Serialization done, swapping tmp file now... | 20260507 21:34:42.408 | INFO | RTMetrics done | 20260507 21:34:42.440 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.601 seconds | 20260507 21:34:42.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.64% / 0.01% / 56.35% | 20260507 21:34:42.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64844% / 7.94729e-06% / 4.76837e-05% | 20260507 21:34:42.441 | INFO | Baz IO overall: | 20260507 21:34:42.441 | INFO | BytesWritten : 1258960892520 | 20260507 21:34:42.441 | INFO | NumEvents : 610153669604 | 20260507 21:34:43.880 | INFO | Analyzing chunk frames = [2764800,2765312) | 20260507 21:34:43.880 | INFO | MemInfo: { | 20260507 21:34:43.880 | INFO | "gpu" : " 17 GB", | 20260507 21:34:43.880 | INFO | "huge" : "", | 20260507 21:34:43.880 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:43.880 | INFO | "malloc" : " 14 GB", | 20260507 21:34:43.880 | INFO | "pinned" : " 29 GB" | 20260507 21:34:43.880 | INFO | } | 20260507 21:34:43.880 | INFO | Perf report for the last 5.13834 seconds: | 20260507 21:34:43.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:43.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:43.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:43.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:43.880 | INFO | ANALYSIS : 68.8%, 3.0, 107.0ms, 0 | 20260507 21:34:43.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:43.881 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 21:34:43.881 | INFO | PRE_HQ : 65.7%, 1.7, 57.7ms, 0 | 20260507 21:34:43.881 | INFO | BAZWRITER : 69.6%, 3.7, 133.6ms, 0 | 20260507 21:34:43.881 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 21:34:43.881 | INFO | Uploaded 36GiB to GPU in 2.53146s (14.2211GiB/s) | 20260507 21:34:43.881 | INFO | Downloaded 38GiB to GPU in 2.69124s (14.1199GiB/s) | 20260507 21:34:49.016 | INFO | Analyzing chunk frames = [2765312,2765824) | 20260507 21:34:49.016 | INFO | MemInfo: { | 20260507 21:34:49.016 | INFO | "gpu" : " 17 GB", | 20260507 21:34:49.016 | INFO | "huge" : "", | 20260507 21:34:49.016 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:49.016 | INFO | "malloc" : " 14 GB", | 20260507 21:34:49.016 | INFO | "pinned" : " 29 GB" | 20260507 21:34:49.016 | INFO | } | 20260507 21:34:49.016 | INFO | Perf report for the last 5.13532 seconds: | 20260507 21:34:49.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:49.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:34:49.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:34:49.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:49.016 | INFO | Uploaded 36GiB to GPU in 2.55076s (14.1134GiB/s) | 20260507 21:34:49.016 | INFO | Downloaded 34GiB to GPU in 2.48062s (13.7062GiB/s) | 20260507 21:34:54.153 | INFO | Analyzing chunk frames = [2765824,2766336) | 20260507 21:34:54.153 | INFO | MemInfo: { | 20260507 21:34:54.154 | INFO | "gpu" : " 17 GB", | 20260507 21:34:54.154 | INFO | "huge" : "", | 20260507 21:34:54.154 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:54.154 | INFO | "malloc" : " 14 GB", | 20260507 21:34:54.154 | INFO | "pinned" : " 29 GB" | 20260507 21:34:54.154 | INFO | } | 20260507 21:34:54.154 | INFO | Perf report for the last 5.13786 seconds: | 20260507 21:34:54.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:54.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:34:54.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:34:54.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:54.154 | INFO | Uploaded 36GiB to GPU in 2.51966s (14.2877GiB/s) | 20260507 21:34:54.154 | INFO | Downloaded 34GiB to GPU in 2.4516s (13.8685GiB/s) | 20260507 21:34:59.296 | INFO | Analyzing chunk frames = [2766336,2766848) | 20260507 21:34:59.296 | INFO | MemInfo: { | 20260507 21:34:59.296 | INFO | "gpu" : " 17 GB", | 20260507 21:34:59.296 | INFO | "huge" : "", | 20260507 21:34:59.296 | INFO | "huge_shared" : " 29 GB", | 20260507 21:34:59.296 | INFO | "malloc" : " 14 GB", | 20260507 21:34:59.296 | INFO | "pinned" : " 29 GB" | 20260507 21:34:59.296 | INFO | } | 20260507 21:34:59.296 | INFO | Perf report for the last 5.14244 seconds: | 20260507 21:34:59.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:34:59.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:34:59.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:34:59.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:34:59.296 | INFO | Uploaded 36GiB to GPU in 2.48573s (14.4827GiB/s) | 20260507 21:34:59.296 | INFO | Downloaded 34GiB to GPU in 2.37506s (14.3154GiB/s) | 20260507 21:35:04.443 | INFO | Analyzing chunk frames = [2766848,2767360) | 20260507 21:35:04.443 | INFO | MemInfo: { | 20260507 21:35:04.443 | INFO | "gpu" : " 17 GB", | 20260507 21:35:04.443 | INFO | "huge" : "", | 20260507 21:35:04.443 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:04.443 | INFO | "malloc" : " 14 GB", | 20260507 21:35:04.443 | INFO | "pinned" : " 29 GB" | 20260507 21:35:04.443 | INFO | } | 20260507 21:35:04.443 | INFO | Perf report for the last 5.14671 seconds: | 20260507 21:35:04.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:04.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 21:35:04.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:35:04.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:04.443 | INFO | Uploaded 36GiB to GPU in 2.3362s (15.4096GiB/s) | 20260507 21:35:04.443 | INFO | Downloaded 34GiB to GPU in 2.49308s (13.6377GiB/s) | 20260507 21:35:09.578 | INFO | Analyzing chunk frames = [2767360,2767872) | 20260507 21:35:09.578 | INFO | MemInfo: { | 20260507 21:35:09.578 | INFO | "gpu" : " 17 GB", | 20260507 21:35:09.578 | INFO | "huge" : "", | 20260507 21:35:09.578 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:09.578 | INFO | "malloc" : " 14 GB", | 20260507 21:35:09.578 | INFO | "pinned" : " 29 GB" | 20260507 21:35:09.578 | INFO | } | 20260507 21:35:09.578 | INFO | Perf report for the last 5.13556 seconds: | 20260507 21:35:09.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:09.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.578 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 21:35:09.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.578 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 21:35:09.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:09.579 | INFO | Uploaded 36GiB to GPU in 2.49715s (14.4164GiB/s) | 20260507 21:35:09.579 | INFO | Downloaded 34GiB to GPU in 2.43293s (13.9749GiB/s) | 20260507 21:35:14.714 | INFO | Analyzing chunk frames = [2767872,2768384) | 20260507 21:35:14.714 | INFO | MemInfo: { | 20260507 21:35:14.714 | INFO | "gpu" : " 17 GB", | 20260507 21:35:14.714 | INFO | "huge" : "", | 20260507 21:35:14.714 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:14.714 | INFO | "malloc" : " 14 GB", | 20260507 21:35:14.714 | INFO | "pinned" : " 29 GB" | 20260507 21:35:14.714 | INFO | } | 20260507 21:35:14.714 | INFO | Perf report for the last 5.13579 seconds: | 20260507 21:35:14.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:14.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 21:35:14.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:35:14.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:14.714 | INFO | Uploaded 36GiB to GPU in 2.54919s (14.1221GiB/s) | 20260507 21:35:14.714 | INFO | Downloaded 34GiB to GPU in 2.44703s (13.8944GiB/s) | 20260507 21:35:19.856 | INFO | Analyzing chunk frames = [2768384,2768896) | 20260507 21:35:19.856 | INFO | MemInfo: { | 20260507 21:35:19.856 | INFO | "gpu" : " 17 GB", | 20260507 21:35:19.856 | INFO | "huge" : "", | 20260507 21:35:19.856 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:19.856 | INFO | "malloc" : " 14 GB", | 20260507 21:35:19.856 | INFO | "pinned" : " 29 GB" | 20260507 21:35:19.856 | INFO | } | 20260507 21:35:19.856 | INFO | Perf report for the last 5.14168 seconds: | 20260507 21:35:19.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:19.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 21:35:19.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:35:19.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:19.856 | INFO | Uploaded 36GiB to GPU in 2.51644s (14.3059GiB/s) | 20260507 21:35:19.856 | INFO | Downloaded 34GiB to GPU in 2.43578s (13.9586GiB/s) | 20260507 21:35:19.856 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:35:19.856 | INFO | Total mem usage: 16.7 GiB | 20260507 21:35:19.856 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:35:19.856 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260507 21:35:19.856 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:35:19.856 | INFO | High Water Breakdown: | 20260507 21:35:19.856 | INFO | 59.7% : 8.72 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:35:19.856 | INFO | 40.2% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:35:19.856 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:35:19.856 | INFO | ------------------------------------------ | 20260507 21:35:19.856 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:35:19.856 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:35:19.856 | INFO | Total mem usage: 26.5 GiB | 20260507 21:35:19.856 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:35:19.856 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:35:19.856 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:35:19.856 | INFO | High Water Breakdown: | 20260507 21:35:19.856 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:35:19.856 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:35:19.856 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:35:19.856 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:35:19.856 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:35:19.856 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:35:19.857 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:35:19.857 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:35:19.857 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:35:19.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:35:19.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:35:19.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:35:19.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:35:19.857 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:35:19.857 | INFO | ------------------------------------------ | 20260507 21:35:19.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:35:19.857 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:35:19.857 | INFO | Total mem usage: 27.8 GiB | 20260507 21:35:19.857 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:35:19.857 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:35:19.857 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:35:19.857 | INFO | High Water Breakdown: | 20260507 21:35:19.857 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:35:19.857 | INFO | ------------------------------------------ | 20260507 21:35:19.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:35:19.857 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:35:19.857 | INFO | Total mem usage: 15.7 GiB | 20260507 21:35:19.857 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:35:19.857 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:35:19.857 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:35:19.857 | INFO | High Water Breakdown: | 20260507 21:35:19.857 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:35:19.857 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:35:19.857 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:35:19.857 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:35:19.857 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:35:19.857 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:35:19.857 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:35:19.857 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:35:19.857 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:35:19.857 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:35:19.857 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:35:19.857 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:35:19.857 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:35:19.857 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:35:19.857 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:35:19.857 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:35:19.857 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:35:19.857 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:35:19.857 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:35:19.857 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:35:19.857 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:35:19.857 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:35:19.857 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:35:19.857 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:35:19.857 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:35:19.857 | INFO | ------------------------------------------ | 20260507 21:35:19.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:35:19.857 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:35:19.857 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:35:19.857 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:35:19.857 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.41e+01 / 1.65e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 0.29%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.10e-02 / 5.39e-03 | 20260507 21:35:19.857 | INFO | PerfReport: 12.12%, Baseline -- 4.01e+00 / 4.46e+00, 6.01e-01 / 2.65e-01 | 20260507 21:35:19.857 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.04e-01 | 20260507 21:35:19.857 | INFO | PerfReport: 15.80%, DME -- 4.18e+01 / 4.52e+01, 4.03e+00 / 1.49e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.85e+00 / 1.96e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.45e+00 / 2.76e+00, 4.28e-01 / 2.36e-01 | 20260507 21:35:19.857 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.35e+00, 2.96e-01 / 1.73e-01 | 20260507 21:35:19.857 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.89e-01, 6.01e-02 / 2.98e-02 | 20260507 21:35:19.857 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:35:19.857 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:35:19.857 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:35:19.857 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 24.37%, Upload -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 24.16%, Download -- 2.58e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 2.05%, LockUpload -- 2.19e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 2.38%, LockDownload -- 2.54e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 16.02%, LockCompute -- 1.71e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 21:35:19.857 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260507 21:35:19.953 | INFO | Starting a round of Baz IO | 20260507 21:35:23.420 | INFO | Populating RTMetrics report | 20260507 21:35:23.420 | INFO | RTMetrics report populated | 20260507 21:35:23.420 | INFO | Opening tmp RTMetrics json file | 20260507 21:35:23.447 | INFO | Serializing RTMetrics to json | 20260507 21:35:23.511 | INFO | Serialization done, swapping tmp file now... | 20260507 21:35:23.515 | INFO | RTMetrics done | 20260507 21:35:23.546 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.592 seconds | 20260507 21:35:23.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.63% / 0.01% / 56.36% | 20260507 21:35:23.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64401% / 7.94729e-06% / 2.38419e-05% | 20260507 21:35:23.546 | INFO | Baz IO overall: | 20260507 21:35:23.546 | INFO | BytesWritten : 1260846070624 | 20260507 21:35:23.546 | INFO | NumEvents : 611065208497 | 20260507 21:35:24.995 | INFO | Analyzing chunk frames = [2768896,2769408) | 20260507 21:35:24.995 | INFO | MemInfo: { | 20260507 21:35:24.995 | INFO | "gpu" : " 17 GB", | 20260507 21:35:24.995 | INFO | "huge" : "", | 20260507 21:35:24.995 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:24.995 | INFO | "malloc" : " 14 GB", | 20260507 21:35:24.995 | INFO | "pinned" : " 29 GB" | 20260507 21:35:24.995 | INFO | } | 20260507 21:35:24.995 | INFO | Perf report for the last 5.13926 seconds: | 20260507 21:35:24.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:24.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:24.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:24.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:24.995 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 21:35:24.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:24.995 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 21:35:24.995 | INFO | PRE_HQ : 65.9%, 1.7, 57.5ms, 0 | 20260507 21:35:24.995 | INFO | BAZWRITER : 69.5%, 4.1, 146.3ms, 0 | 20260507 21:35:24.995 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 21:35:24.995 | INFO | Uploaded 36GiB to GPU in 2.5398s (14.1743GiB/s) | 20260507 21:35:24.996 | INFO | Downloaded 38GiB to GPU in 2.69202s (14.1158GiB/s) | 20260507 21:35:30.131 | INFO | Analyzing chunk frames = [2769408,2769920) | 20260507 21:35:30.131 | INFO | MemInfo: { | 20260507 21:35:30.131 | INFO | "gpu" : " 17 GB", | 20260507 21:35:30.131 | INFO | "huge" : "", | 20260507 21:35:30.132 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:30.132 | INFO | "malloc" : " 14 GB", | 20260507 21:35:30.132 | INFO | "pinned" : " 29 GB" | 20260507 21:35:30.132 | INFO | } | 20260507 21:35:30.132 | INFO | Perf report for the last 5.13637 seconds: | 20260507 21:35:30.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:30.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 21:35:30.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:35:30.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:30.132 | INFO | Uploaded 36GiB to GPU in 2.52766s (14.2424GiB/s) | 20260507 21:35:30.132 | INFO | Downloaded 34GiB to GPU in 2.49774s (13.6123GiB/s) | 20260507 21:35:35.271 | INFO | Analyzing chunk frames = [2769920,2770432) | 20260507 21:35:35.271 | INFO | MemInfo: { | 20260507 21:35:35.271 | INFO | "gpu" : " 17 GB", | 20260507 21:35:35.271 | INFO | "huge" : "", | 20260507 21:35:35.271 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:35.271 | INFO | "malloc" : " 14 GB", | 20260507 21:35:35.271 | INFO | "pinned" : " 29 GB" | 20260507 21:35:35.271 | INFO | } | 20260507 21:35:35.271 | INFO | Perf report for the last 5.13933 seconds: | 20260507 21:35:35.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:35.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 21:35:35.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:35:35.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:35.271 | INFO | Uploaded 36GiB to GPU in 2.50955s (14.3452GiB/s) | 20260507 21:35:35.271 | INFO | Downloaded 34GiB to GPU in 2.46455s (13.7956GiB/s) | 20260507 21:35:40.416 | INFO | Analyzing chunk frames = [2770432,2770944) | 20260507 21:35:40.416 | INFO | MemInfo: { | 20260507 21:35:40.416 | INFO | "gpu" : " 17 GB", | 20260507 21:35:40.416 | INFO | "huge" : "", | 20260507 21:35:40.416 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:40.416 | INFO | "malloc" : " 14 GB", | 20260507 21:35:40.416 | INFO | "pinned" : " 29 GB" | 20260507 21:35:40.416 | INFO | } | 20260507 21:35:40.416 | INFO | Perf report for the last 5.1455 seconds: | 20260507 21:35:40.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:40.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.417 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 21:35:40.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.417 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:35:40.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:40.417 | INFO | Uploaded 36GiB to GPU in 2.45072s (14.6895GiB/s) | 20260507 21:35:40.417 | INFO | Downloaded 34GiB to GPU in 2.3909s (14.2206GiB/s) | 20260507 21:35:45.557 | INFO | Analyzing chunk frames = [2770944,2771456) | 20260507 21:35:45.557 | INFO | MemInfo: { | 20260507 21:35:45.557 | INFO | "gpu" : " 17 GB", | 20260507 21:35:45.557 | INFO | "huge" : "", | 20260507 21:35:45.557 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:45.557 | INFO | "malloc" : " 14 GB", | 20260507 21:35:45.557 | INFO | "pinned" : " 29 GB" | 20260507 21:35:45.557 | INFO | } | 20260507 21:35:45.557 | INFO | Perf report for the last 5.14055 seconds: | 20260507 21:35:45.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:45.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 21:35:45.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 21:35:45.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:45.557 | INFO | Uploaded 36GiB to GPU in 2.37252s (15.1737GiB/s) | 20260507 21:35:45.557 | INFO | Downloaded 34GiB to GPU in 2.50147s (13.592GiB/s) | 20260507 21:35:50.695 | INFO | Analyzing chunk frames = [2771456,2771968) | 20260507 21:35:50.695 | INFO | MemInfo: { | 20260507 21:35:50.695 | INFO | "gpu" : " 17 GB", | 20260507 21:35:50.695 | INFO | "huge" : "", | 20260507 21:35:50.696 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:50.696 | INFO | "malloc" : " 14 GB", | 20260507 21:35:50.696 | INFO | "pinned" : " 29 GB" | 20260507 21:35:50.696 | INFO | } | 20260507 21:35:50.696 | INFO | Perf report for the last 5.13857 seconds: | 20260507 21:35:50.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:50.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 21:35:50.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:35:50.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:50.696 | INFO | Uploaded 36GiB to GPU in 2.48961s (14.4601GiB/s) | 20260507 21:35:50.696 | INFO | Downloaded 34GiB to GPU in 2.41499s (14.0787GiB/s) | 20260507 21:35:55.829 | INFO | Analyzing chunk frames = [2771968,2772480) | 20260507 21:35:55.829 | INFO | MemInfo: { | 20260507 21:35:55.829 | INFO | "gpu" : " 17 GB", | 20260507 21:35:55.829 | INFO | "huge" : "", | 20260507 21:35:55.829 | INFO | "huge_shared" : " 29 GB", | 20260507 21:35:55.829 | INFO | "malloc" : " 14 GB", | 20260507 21:35:55.829 | INFO | "pinned" : " 29 GB" | 20260507 21:35:55.829 | INFO | } | 20260507 21:35:55.829 | INFO | Perf report for the last 5.13327 seconds: | 20260507 21:35:55.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:35:55.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 21:35:55.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:35:55.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:35:55.829 | INFO | Uploaded 36GiB to GPU in 2.48942s (14.4612GiB/s) | 20260507 21:35:55.829 | INFO | Downloaded 34GiB to GPU in 2.44559s (13.9026GiB/s) | 20260507 21:36:00.964 | INFO | Analyzing chunk frames = [2772480,2772992) | 20260507 21:36:00.964 | INFO | MemInfo: { | 20260507 21:36:00.964 | INFO | "gpu" : " 17 GB", | 20260507 21:36:00.964 | INFO | "huge" : "", | 20260507 21:36:00.964 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:00.964 | INFO | "malloc" : " 14 GB", | 20260507 21:36:00.964 | INFO | "pinned" : " 29 GB" | 20260507 21:36:00.964 | INFO | } | 20260507 21:36:00.964 | INFO | Perf report for the last 5.13561 seconds: | 20260507 21:36:00.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:00.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.965 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:36:00.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.965 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:36:00.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:00.965 | INFO | Uploaded 36GiB to GPU in 2.46809s (14.5862GiB/s) | 20260507 21:36:00.965 | INFO | Downloaded 34GiB to GPU in 2.44417s (13.9107GiB/s) | 20260507 21:36:01.062 | INFO | Starting a round of Baz IO | 20260507 21:36:04.531 | INFO | Populating RTMetrics report | 20260507 21:36:04.532 | INFO | RTMetrics report populated | 20260507 21:36:04.532 | INFO | Opening tmp RTMetrics json file | 20260507 21:36:04.562 | INFO | Serializing RTMetrics to json | 20260507 21:36:04.599 | INFO | Serialization done, swapping tmp file now... | 20260507 21:36:04.622 | INFO | RTMetrics done | 20260507 21:36:04.654 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.592 seconds | 20260507 21:36:04.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.63% / 0.01% / 56.37% | 20260507 21:36:04.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63982% / 2.38419e-05% / 4.76837e-05% | 20260507 21:36:04.654 | INFO | Baz IO overall: | 20260507 21:36:04.654 | INFO | BytesWritten : 1262731228388 | 20260507 21:36:04.654 | INFO | NumEvents : 611976553570 | 20260507 21:36:06.106 | INFO | Analyzing chunk frames = [2772992,2773504) | 20260507 21:36:06.106 | INFO | MemInfo: { | 20260507 21:36:06.107 | INFO | "gpu" : " 17 GB", | 20260507 21:36:06.107 | INFO | "huge" : "", | 20260507 21:36:06.107 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:06.107 | INFO | "malloc" : " 14 GB", | 20260507 21:36:06.107 | INFO | "pinned" : " 29 GB" | 20260507 21:36:06.107 | INFO | } | 20260507 21:36:06.107 | INFO | Perf report for the last 5.14217 seconds: | 20260507 21:36:06.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:06.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:06.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:06.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:06.107 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 21:36:06.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:06.107 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 21:36:06.107 | INFO | PRE_HQ : 65.4%, 1.7, 57.4ms, 0 | 20260507 21:36:06.107 | INFO | BAZWRITER : 69.4%, 4.2, 149.4ms, 0 | 20260507 21:36:06.107 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 21:36:06.107 | INFO | Uploaded 36GiB to GPU in 2.53037s (14.2271GiB/s) | 20260507 21:36:06.107 | INFO | Downloaded 38GiB to GPU in 2.69825s (14.0832GiB/s) | 20260507 21:36:11.248 | INFO | Analyzing chunk frames = [2773504,2774016) | 20260507 21:36:11.248 | INFO | MemInfo: { | 20260507 21:36:11.248 | INFO | "gpu" : " 17 GB", | 20260507 21:36:11.248 | INFO | "huge" : "", | 20260507 21:36:11.248 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:11.248 | INFO | "malloc" : " 14 GB", | 20260507 21:36:11.248 | INFO | "pinned" : " 29 GB" | 20260507 21:36:11.248 | INFO | } | 20260507 21:36:11.248 | INFO | Perf report for the last 5.14133 seconds: | 20260507 21:36:11.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:11.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.248 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:36:11.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.248 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:36:11.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:11.249 | INFO | Uploaded 36GiB to GPU in 2.50551s (14.3683GiB/s) | 20260507 21:36:11.249 | INFO | Downloaded 34GiB to GPU in 2.47246s (13.7515GiB/s) | 20260507 21:36:16.389 | INFO | Analyzing chunk frames = [2774016,2774528) | 20260507 21:36:16.389 | INFO | MemInfo: { | 20260507 21:36:16.389 | INFO | "gpu" : " 17 GB", | 20260507 21:36:16.389 | INFO | "huge" : "", | 20260507 21:36:16.389 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:16.389 | INFO | "malloc" : " 14 GB", | 20260507 21:36:16.389 | INFO | "pinned" : " 29 GB" | 20260507 21:36:16.389 | INFO | } | 20260507 21:36:16.390 | INFO | Perf report for the last 5.14152 seconds: | 20260507 21:36:16.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:16.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:36:16.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:36:16.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:16.390 | INFO | Uploaded 36GiB to GPU in 2.51156s (14.3337GiB/s) | 20260507 21:36:16.390 | INFO | Downloaded 34GiB to GPU in 2.47106s (13.7593GiB/s) | 20260507 21:36:21.529 | INFO | Analyzing chunk frames = [2774528,2775040) | 20260507 21:36:21.529 | INFO | MemInfo: { | 20260507 21:36:21.529 | INFO | "gpu" : " 17 GB", | 20260507 21:36:21.530 | INFO | "huge" : "", | 20260507 21:36:21.530 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:21.530 | INFO | "malloc" : " 14 GB", | 20260507 21:36:21.530 | INFO | "pinned" : " 29 GB" | 20260507 21:36:21.530 | INFO | } | 20260507 21:36:21.530 | INFO | Perf report for the last 5.14004 seconds: | 20260507 21:36:21.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:21.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:36:21.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:36:21.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:21.530 | INFO | Uploaded 36GiB to GPU in 2.45806s (14.6457GiB/s) | 20260507 21:36:21.530 | INFO | Downloaded 34GiB to GPU in 2.39186s (14.2149GiB/s) | 20260507 21:36:26.667 | INFO | Analyzing chunk frames = [2775040,2775552) | 20260507 21:36:26.667 | INFO | MemInfo: { | 20260507 21:36:26.667 | INFO | "gpu" : " 17 GB", | 20260507 21:36:26.667 | INFO | "huge" : "", | 20260507 21:36:26.667 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:26.667 | INFO | "malloc" : " 14 GB", | 20260507 21:36:26.667 | INFO | "pinned" : " 29 GB" | 20260507 21:36:26.667 | INFO | } | 20260507 21:36:26.667 | INFO | Perf report for the last 5.13723 seconds: | 20260507 21:36:26.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:26.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:36:26.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:36:26.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:26.667 | INFO | Uploaded 36GiB to GPU in 2.31613s (15.5432GiB/s) | 20260507 21:36:26.667 | INFO | Downloaded 34GiB to GPU in 2.51229s (13.5335GiB/s) | 20260507 21:36:31.807 | INFO | Analyzing chunk frames = [2775552,2776064) | 20260507 21:36:31.807 | INFO | MemInfo: { | 20260507 21:36:31.807 | INFO | "gpu" : " 17 GB", | 20260507 21:36:31.807 | INFO | "huge" : "", | 20260507 21:36:31.807 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:31.807 | INFO | "malloc" : " 14 GB", | 20260507 21:36:31.807 | INFO | "pinned" : " 29 GB" | 20260507 21:36:31.807 | INFO | } | 20260507 21:36:31.807 | INFO | Perf report for the last 5.14025 seconds: | 20260507 21:36:31.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:31.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:36:31.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:36:31.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:31.807 | INFO | Uploaded 36GiB to GPU in 2.46264s (14.6184GiB/s) | 20260507 21:36:31.807 | INFO | Downloaded 34GiB to GPU in 2.41953s (14.0523GiB/s) | 20260507 21:36:36.942 | INFO | Analyzing chunk frames = [2776064,2776576) | 20260507 21:36:36.942 | INFO | MemInfo: { | 20260507 21:36:36.942 | INFO | "gpu" : " 17 GB", | 20260507 21:36:36.942 | INFO | "huge" : "", | 20260507 21:36:36.942 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:36.942 | INFO | "malloc" : " 14 GB", | 20260507 21:36:36.942 | INFO | "pinned" : " 29 GB" | 20260507 21:36:36.942 | INFO | } | 20260507 21:36:36.942 | INFO | Perf report for the last 5.13454 seconds: | 20260507 21:36:36.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:36.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:36:36.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:36:36.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:36.942 | INFO | Uploaded 36GiB to GPU in 2.52875s (14.2363GiB/s) | 20260507 21:36:36.942 | INFO | Downloaded 34GiB to GPU in 2.45734s (13.8361GiB/s) | 20260507 21:36:42.081 | INFO | Analyzing chunk frames = [2776576,2777088) | 20260507 21:36:42.081 | INFO | MemInfo: { | 20260507 21:36:42.081 | INFO | "gpu" : " 17 GB", | 20260507 21:36:42.081 | INFO | "huge" : "", | 20260507 21:36:42.081 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:42.081 | INFO | "malloc" : " 14 GB", | 20260507 21:36:42.081 | INFO | "pinned" : " 29 GB" | 20260507 21:36:42.081 | INFO | } | 20260507 21:36:42.081 | INFO | Perf report for the last 5.13979 seconds: | 20260507 21:36:42.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:42.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.082 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:36:42.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.082 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:36:42.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:42.082 | INFO | Uploaded 36GiB to GPU in 2.47889s (14.5226GiB/s) | 20260507 21:36:42.082 | INFO | Downloaded 34GiB to GPU in 2.44226s (13.9215GiB/s) | 20260507 21:36:42.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:36:42.082 | INFO | Total mem usage: 16.7 GiB | 20260507 21:36:42.082 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:36:42.082 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.74%) | 20260507 21:36:42.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:36:42.082 | INFO | High Water Breakdown: | 20260507 21:36:42.082 | INFO | 60.7% : 9.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:36:42.082 | INFO | 39.2% : 6.06 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:36:42.082 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:36:42.082 | INFO | ------------------------------------------ | 20260507 21:36:42.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:36:42.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:36:42.082 | INFO | Total mem usage: 26.5 GiB | 20260507 21:36:42.082 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:36:42.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:36:42.082 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:36:42.082 | INFO | High Water Breakdown: | 20260507 21:36:42.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:36:42.082 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:36:42.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:36:42.082 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:36:42.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:36:42.082 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:36:42.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:36:42.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:36:42.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:36:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:36:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:36:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:36:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:36:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:36:42.082 | INFO | ------------------------------------------ | 20260507 21:36:42.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:36:42.082 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:36:42.082 | INFO | Total mem usage: 27.8 GiB | 20260507 21:36:42.082 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:36:42.082 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:36:42.082 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:36:42.082 | INFO | High Water Breakdown: | 20260507 21:36:42.082 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:36:42.082 | INFO | ------------------------------------------ | 20260507 21:36:42.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:36:42.082 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:36:42.082 | INFO | Total mem usage: 15.7 GiB | 20260507 21:36:42.082 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:36:42.082 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:36:42.082 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 21:36:42.082 | INFO | High Water Breakdown: | 20260507 21:36:42.082 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:36:42.082 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:36:42.082 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:36:42.082 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:36:42.082 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:36:42.082 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:36:42.082 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:36:42.082 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:36:42.082 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:36:42.082 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:36:42.082 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:36:42.082 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:36:42.082 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:36:42.082 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:36:42.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:36:42.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:36:42.082 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:36:42.082 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:36:42.082 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:36:42.082 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:36:42.082 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:36:42.082 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:36:42.082 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:36:42.082 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:36:42.082 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:36:42.082 | INFO | ------------------------------------------ | 20260507 21:36:42.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:36:42.082 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:36:42.082 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:36:42.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:36:42.082 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.58e+01, 1.41e+01 / 1.74e+00 | 20260507 21:36:42.082 | INFO | PerfReport: 0.29%, AggregateStats -- 9.73e-02 / 1.02e-01, 1.11e-02 / 5.52e-03 | 20260507 21:36:42.082 | INFO | PerfReport: 12.14%, Baseline -- 4.02e+00 / 4.48e+00, 6.08e-01 / 2.67e-01 | 20260507 21:36:42.082 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.48e+00, 2.10e-01 / 1.28e-01 | 20260507 21:36:42.082 | INFO | PerfReport: 15.71%, DME -- 4.16e+01 / 4.47e+01, 3.72e+00 / 1.45e+00 | 20260507 21:36:42.082 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.84e+00 / 2.17e+00 | 20260507 21:36:42.082 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.45e+00 / 2.76e+00, 4.27e-01 / 2.29e-01 | 20260507 21:36:42.082 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.38e+00, 2.97e-01 / 1.65e-01 | 20260507 21:36:42.083 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.99e-01, 6.04e-02 / 3.32e-02 | 20260507 21:36:42.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:36:42.083 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:36:42.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:36:42.083 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 24.19%, Upload -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 31.01%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 1.81%, LockUpload -- 1.93e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 2.59%, LockDownload -- 2.77e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 16.15%, LockCompute -- 1.72e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:36:42.083 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260507 21:36:42.179 | INFO | Starting a round of Baz IO | 20260507 21:36:45.649 | INFO | Populating RTMetrics report | 20260507 21:36:45.649 | INFO | RTMetrics report populated | 20260507 21:36:45.649 | INFO | Opening tmp RTMetrics json file | 20260507 21:36:45.698 | INFO | Serializing RTMetrics to json | 20260507 21:36:45.762 | INFO | Serialization done, swapping tmp file now... | 20260507 21:36:45.766 | INFO | RTMetrics done | 20260507 21:36:45.790 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3.61 seconds | 20260507 21:36:45.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.62% / 0.01% / 56.37% | 20260507 21:36:45.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63545% / 2.38419e-05% / 7.15256e-05% | 20260507 21:36:45.790 | INFO | Baz IO overall: | 20260507 21:36:45.790 | INFO | BytesWritten : 1264616016414 | 20260507 21:36:45.790 | INFO | NumEvents : 612887897352 | 20260507 21:36:47.225 | INFO | Analyzing chunk frames = [2777088,2777600) | 20260507 21:36:47.225 | INFO | MemInfo: { | 20260507 21:36:47.225 | INFO | "gpu" : " 17 GB", | 20260507 21:36:47.225 | INFO | "huge" : "", | 20260507 21:36:47.225 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:47.225 | INFO | "malloc" : " 14 GB", | 20260507 21:36:47.225 | INFO | "pinned" : " 29 GB" | 20260507 21:36:47.225 | INFO | } | 20260507 21:36:47.225 | INFO | Perf report for the last 5.14324 seconds: | 20260507 21:36:47.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:47.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:47.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:47.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:47.225 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 21:36:47.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:47.225 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 21:36:47.225 | INFO | PRE_HQ : 65.7%, 1.7, 57.8ms, 0 | 20260507 21:36:47.225 | INFO | BAZWRITER : 69.5%, 3.8, 135.7ms, 0 | 20260507 21:36:47.225 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 21:36:47.225 | INFO | Uploaded 36GiB to GPU in 2.51593s (14.3088GiB/s) | 20260507 21:36:47.225 | INFO | Downloaded 38GiB to GPU in 2.6962s (14.0939GiB/s) | 20260507 21:36:52.365 | INFO | Analyzing chunk frames = [2777600,2778112) | 20260507 21:36:52.365 | INFO | MemInfo: { | 20260507 21:36:52.365 | INFO | "gpu" : " 17 GB", | 20260507 21:36:52.366 | INFO | "huge" : "", | 20260507 21:36:52.366 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:52.366 | INFO | "malloc" : " 14 GB", | 20260507 21:36:52.366 | INFO | "pinned" : " 29 GB" | 20260507 21:36:52.366 | INFO | } | 20260507 21:36:52.366 | INFO | Perf report for the last 5.14085 seconds: | 20260507 21:36:52.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:52.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:36:52.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 21:36:52.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:52.366 | INFO | Uploaded 36GiB to GPU in 2.55162s (14.1087GiB/s) | 20260507 21:36:52.366 | INFO | Downloaded 34GiB to GPU in 2.46694s (13.7823GiB/s) | 20260507 21:36:57.506 | INFO | Analyzing chunk frames = [2778112,2778624) | 20260507 21:36:57.506 | INFO | MemInfo: { | 20260507 21:36:57.506 | INFO | "gpu" : " 17 GB", | 20260507 21:36:57.506 | INFO | "huge" : "", | 20260507 21:36:57.506 | INFO | "huge_shared" : " 29 GB", | 20260507 21:36:57.506 | INFO | "malloc" : " 14 GB", | 20260507 21:36:57.506 | INFO | "pinned" : " 29 GB" | 20260507 21:36:57.506 | INFO | } | 20260507 21:36:57.506 | INFO | Perf report for the last 5.14067 seconds: | 20260507 21:36:57.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:36:57.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.506 | INFO | ANALYSIS : 67.7%, 3.0, 105.0ms, 0 | 20260507 21:36:57.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.506 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:36:57.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:36:57.507 | INFO | Uploaded 36GiB to GPU in 2.57798s (13.9644GiB/s) | 20260507 21:36:57.507 | INFO | Downloaded 34GiB to GPU in 2.46291s (13.8048GiB/s) | 20260507 21:37:02.643 | INFO | Analyzing chunk frames = [2778624,2779136) | 20260507 21:37:02.643 | INFO | MemInfo: { | 20260507 21:37:02.643 | INFO | "gpu" : " 17 GB", | 20260507 21:37:02.643 | INFO | "huge" : "", | 20260507 21:37:02.643 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:02.643 | INFO | "malloc" : " 14 GB", | 20260507 21:37:02.643 | INFO | "pinned" : " 29 GB" | 20260507 21:37:02.643 | INFO | } | 20260507 21:37:02.643 | INFO | Perf report for the last 5.1372 seconds: | 20260507 21:37:02.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:02.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.644 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 21:37:02.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.644 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:37:02.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:02.644 | INFO | Uploaded 36GiB to GPU in 2.49771s (14.4132GiB/s) | 20260507 21:37:02.644 | INFO | Downloaded 34GiB to GPU in 2.3993s (14.1708GiB/s) | 20260507 21:37:07.778 | INFO | Analyzing chunk frames = [2779136,2779648) | 20260507 21:37:07.778 | INFO | MemInfo: { | 20260507 21:37:07.778 | INFO | "gpu" : " 17 GB", | 20260507 21:37:07.778 | INFO | "huge" : "", | 20260507 21:37:07.778 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:07.778 | INFO | "malloc" : " 14 GB", | 20260507 21:37:07.778 | INFO | "pinned" : " 29 GB" | 20260507 21:37:07.778 | INFO | } | 20260507 21:37:07.778 | INFO | Perf report for the last 5.13494 seconds: | 20260507 21:37:07.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:07.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.778 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 21:37:07.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.779 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:37:07.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:07.779 | INFO | Uploaded 36GiB to GPU in 2.36538s (15.2195GiB/s) | 20260507 21:37:07.779 | INFO | Downloaded 34GiB to GPU in 2.49586s (13.6226GiB/s) | 20260507 21:37:12.917 | INFO | Analyzing chunk frames = [2779648,2780160) | 20260507 21:37:12.917 | INFO | MemInfo: { | 20260507 21:37:12.917 | INFO | "gpu" : " 17 GB", | 20260507 21:37:12.917 | INFO | "huge" : "", | 20260507 21:37:12.917 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:12.917 | INFO | "malloc" : " 14 GB", | 20260507 21:37:12.917 | INFO | "pinned" : " 29 GB" | 20260507 21:37:12.917 | INFO | } | 20260507 21:37:12.917 | INFO | Perf report for the last 5.13887 seconds: | 20260507 21:37:12.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:12.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.917 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:37:12.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.917 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:37:12.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:12.918 | INFO | Uploaded 36GiB to GPU in 2.48972s (14.4595GiB/s) | 20260507 21:37:12.918 | INFO | Downloaded 34GiB to GPU in 2.41972s (14.0512GiB/s) | 20260507 21:37:18.054 | INFO | Analyzing chunk frames = [2780160,2780672) | 20260507 21:37:18.054 | INFO | MemInfo: { | 20260507 21:37:18.054 | INFO | "gpu" : " 17 GB", | 20260507 21:37:18.054 | INFO | "huge" : "", | 20260507 21:37:18.054 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:18.054 | INFO | "malloc" : " 14 GB", | 20260507 21:37:18.054 | INFO | "pinned" : " 29 GB" | 20260507 21:37:18.054 | INFO | } | 20260507 21:37:18.054 | INFO | Perf report for the last 5.13696 seconds: | 20260507 21:37:18.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:18.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.054 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:37:18.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.054 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:37:18.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:18.055 | INFO | Uploaded 36GiB to GPU in 2.53638s (14.1935GiB/s) | 20260507 21:37:18.055 | INFO | Downloaded 34GiB to GPU in 2.45518s (13.8483GiB/s) | 20260507 21:37:23.200 | INFO | Analyzing chunk frames = [2780672,2781184) | 20260507 21:37:23.200 | INFO | MemInfo: { | 20260507 21:37:23.200 | INFO | "gpu" : " 17 GB", | 20260507 21:37:23.200 | INFO | "huge" : "", | 20260507 21:37:23.200 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:23.200 | INFO | "malloc" : " 14 GB", | 20260507 21:37:23.200 | INFO | "pinned" : " 29 GB" | 20260507 21:37:23.200 | INFO | } | 20260507 21:37:23.200 | INFO | Perf report for the last 5.1455 seconds: | 20260507 21:37:23.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:23.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 21:37:23.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:37:23.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:23.200 | INFO | Uploaded 36GiB to GPU in 2.50574s (14.367GiB/s) | 20260507 21:37:23.200 | INFO | Downloaded 34GiB to GPU in 2.43313s (13.9738GiB/s) | 20260507 21:37:23.306 | INFO | Starting a round of Baz IO | 20260507 21:37:26.818 | INFO | Populating RTMetrics report | 20260507 21:37:26.819 | INFO | RTMetrics report populated | 20260507 21:37:26.819 | INFO | Opening tmp RTMetrics json file | 20260507 21:37:26.820 | INFO | Serializing RTMetrics to json | 20260507 21:37:26.822 | INFO | Serialization done, swapping tmp file now... | 20260507 21:37:26.823 | INFO | RTMetrics done | 20260507 21:37:26.933 | INFO | Finished a round of Baz IO: Wrote 1.755 GiB in 3.626 seconds | 20260507 21:37:26.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.61% / 0.01% / 56.38% | 20260507 21:37:26.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63297% / 7.94729e-06% / 6.35783e-05% | 20260507 21:37:26.933 | INFO | Baz IO overall: | 20260507 21:37:26.933 | INFO | BytesWritten : 1266500720951 | 20260507 21:37:26.933 | INFO | NumEvents : 613799097863 | 20260507 21:37:28.337 | INFO | Analyzing chunk frames = [2781184,2781696) | 20260507 21:37:28.337 | INFO | MemInfo: { | 20260507 21:37:28.337 | INFO | "gpu" : " 17 GB", | 20260507 21:37:28.337 | INFO | "huge" : "", | 20260507 21:37:28.337 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:28.337 | INFO | "malloc" : " 14 GB", | 20260507 21:37:28.337 | INFO | "pinned" : " 29 GB" | 20260507 21:37:28.337 | INFO | } | 20260507 21:37:28.337 | INFO | Perf report for the last 5.13763 seconds: | 20260507 21:37:28.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:28.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:28.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:28.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:28.338 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260507 21:37:28.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:28.338 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260507 21:37:28.338 | INFO | PRE_HQ : 68.4%, 2.0, 69.5ms, 0 | 20260507 21:37:28.338 | INFO | BAZWRITER : 70.0%, 4.2, 151.1ms, 0 | 20260507 21:37:28.338 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 21:37:28.338 | INFO | Uploaded 36GiB to GPU in 2.56881s (14.0143GiB/s) | 20260507 21:37:28.338 | INFO | Downloaded 38GiB to GPU in 2.69329s (14.1091GiB/s) | 20260507 21:37:33.475 | INFO | Analyzing chunk frames = [2781696,2782208) | 20260507 21:37:33.475 | INFO | MemInfo: { | 20260507 21:37:33.475 | INFO | "gpu" : " 17 GB", | 20260507 21:37:33.475 | INFO | "huge" : "", | 20260507 21:37:33.475 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:33.475 | INFO | "malloc" : " 14 GB", | 20260507 21:37:33.475 | INFO | "pinned" : " 29 GB" | 20260507 21:37:33.475 | INFO | } | 20260507 21:37:33.475 | INFO | Perf report for the last 5.13796 seconds: | 20260507 21:37:33.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:33.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.476 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260507 21:37:33.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.476 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:37:33.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:33.476 | INFO | Uploaded 36GiB to GPU in 2.44422s (14.7286GiB/s) | 20260507 21:37:33.476 | INFO | Downloaded 34GiB to GPU in 2.48325s (13.6918GiB/s) | 20260507 21:37:38.619 | INFO | Analyzing chunk frames = [2782208,2782720) | 20260507 21:37:38.619 | INFO | MemInfo: { | 20260507 21:37:38.619 | INFO | "gpu" : " 17 GB", | 20260507 21:37:38.619 | INFO | "huge" : "", | 20260507 21:37:38.619 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:38.619 | INFO | "malloc" : " 14 GB", | 20260507 21:37:38.619 | INFO | "pinned" : " 29 GB" | 20260507 21:37:38.619 | INFO | } | 20260507 21:37:38.619 | INFO | Perf report for the last 5.14325 seconds: | 20260507 21:37:38.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:38.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260507 21:37:38.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:37:38.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:38.619 | INFO | Uploaded 36GiB to GPU in 2.47157s (14.5657GiB/s) | 20260507 21:37:38.619 | INFO | Downloaded 34GiB to GPU in 2.47434s (13.741GiB/s) | 20260507 21:37:43.753 | INFO | Analyzing chunk frames = [2782720,2783232) | 20260507 21:37:43.753 | INFO | MemInfo: { | 20260507 21:37:43.753 | INFO | "gpu" : " 17 GB", | 20260507 21:37:43.753 | INFO | "huge" : "", | 20260507 21:37:43.753 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:43.753 | INFO | "malloc" : " 14 GB", | 20260507 21:37:43.753 | INFO | "pinned" : " 29 GB" | 20260507 21:37:43.753 | INFO | } | 20260507 21:37:43.753 | INFO | Perf report for the last 5.13436 seconds: | 20260507 21:37:43.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:43.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.753 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260507 21:37:43.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.753 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:37:43.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:43.754 | INFO | Uploaded 36GiB to GPU in 2.45823s (14.6447GiB/s) | 20260507 21:37:43.754 | INFO | Downloaded 34GiB to GPU in 2.41557s (14.0753GiB/s) | 20260507 21:37:48.893 | INFO | Analyzing chunk frames = [2783232,2783744) | 20260507 21:37:48.893 | INFO | MemInfo: { | 20260507 21:37:48.893 | INFO | "gpu" : " 17 GB", | 20260507 21:37:48.893 | INFO | "huge" : "", | 20260507 21:37:48.893 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:48.893 | INFO | "malloc" : " 14 GB", | 20260507 21:37:48.893 | INFO | "pinned" : " 29 GB" | 20260507 21:37:48.893 | INFO | } | 20260507 21:37:48.894 | INFO | Perf report for the last 5.14042 seconds: | 20260507 21:37:48.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:48.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 21:37:48.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:37:48.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:48.894 | INFO | Uploaded 36GiB to GPU in 2.29089s (15.7144GiB/s) | 20260507 21:37:48.894 | INFO | Downloaded 34GiB to GPU in 2.49216s (13.6428GiB/s) | 20260507 21:37:54.029 | INFO | Analyzing chunk frames = [2783744,2784256) | 20260507 21:37:54.029 | INFO | MemInfo: { | 20260507 21:37:54.029 | INFO | "gpu" : " 17 GB", | 20260507 21:37:54.029 | INFO | "huge" : "", | 20260507 21:37:54.029 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:54.029 | INFO | "malloc" : " 14 GB", | 20260507 21:37:54.029 | INFO | "pinned" : " 29 GB" | 20260507 21:37:54.029 | INFO | } | 20260507 21:37:54.029 | INFO | Perf report for the last 5.13534 seconds: | 20260507 21:37:54.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:54.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260507 21:37:54.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 21:37:54.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:54.029 | INFO | Uploaded 36GiB to GPU in 2.44285s (14.7369GiB/s) | 20260507 21:37:54.029 | INFO | Downloaded 34GiB to GPU in 2.42286s (14.033GiB/s) | 20260507 21:37:59.171 | INFO | Analyzing chunk frames = [2784256,2784768) | 20260507 21:37:59.171 | INFO | MemInfo: { | 20260507 21:37:59.171 | INFO | "gpu" : " 17 GB", | 20260507 21:37:59.171 | INFO | "huge" : "", | 20260507 21:37:59.171 | INFO | "huge_shared" : " 29 GB", | 20260507 21:37:59.171 | INFO | "malloc" : " 14 GB", | 20260507 21:37:59.171 | INFO | "pinned" : " 29 GB" | 20260507 21:37:59.171 | INFO | } | 20260507 21:37:59.172 | INFO | Perf report for the last 5.14267 seconds: | 20260507 21:37:59.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:37:59.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:37:59.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 21:37:59.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:37:59.172 | INFO | Uploaded 36GiB to GPU in 2.50452s (14.374GiB/s) | 20260507 21:37:59.172 | INFO | Downloaded 34GiB to GPU in 2.46846s (13.7738GiB/s) | 20260507 21:38:04.316 | INFO | Analyzing chunk frames = [2784768,2785280) | 20260507 21:38:04.316 | INFO | MemInfo: { | 20260507 21:38:04.316 | INFO | "gpu" : " 17 GB", | 20260507 21:38:04.316 | INFO | "huge" : "", | 20260507 21:38:04.316 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:04.316 | INFO | "malloc" : " 14 GB", | 20260507 21:38:04.316 | INFO | "pinned" : " 29 GB" | 20260507 21:38:04.316 | INFO | } | 20260507 21:38:04.316 | INFO | Perf report for the last 5.14419 seconds: | 20260507 21:38:04.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:04.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 21:38:04.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 21:38:04.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:04.316 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:38:04.316 | INFO | Total mem usage: 16.7 GiB | 20260507 21:38:04.316 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:38:04.316 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260507 21:38:04.316 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:38:04.316 | INFO | High Water Breakdown: | 20260507 21:38:04.316 | INFO | 59.7% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:38:04.316 | INFO | 40.2% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:38:04.316 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:38:04.316 | INFO | ------------------------------------------ | 20260507 21:38:04.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:38:04.316 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:38:04.316 | INFO | Total mem usage: 26.5 GiB | 20260507 21:38:04.316 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:38:04.316 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:38:04.316 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:38:04.316 | INFO | High Water Breakdown: | 20260507 21:38:04.317 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:38:04.317 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:38:04.317 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:38:04.317 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:38:04.317 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:38:04.317 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:38:04.317 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:38:04.317 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:38:04.317 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:38:04.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:38:04.317 | INFO | Uploaded 36GiB to GPU in 2.46991s (14.5754GiB/s) | 20260507 21:38:04.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:38:04.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:38:04.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:38:04.317 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:38:04.317 | INFO | Downloaded 34GiB to GPU in 2.45682s (13.839GiB/s) | 20260507 21:38:04.317 | INFO | ------------------------------------------ | 20260507 21:38:04.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:38:04.317 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:38:04.317 | INFO | Total mem usage: 27.8 GiB | 20260507 21:38:04.317 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:38:04.317 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:38:04.317 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:38:04.317 | INFO | High Water Breakdown: | 20260507 21:38:04.317 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:38:04.317 | INFO | ------------------------------------------ | 20260507 21:38:04.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:38:04.317 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:38:04.317 | INFO | Total mem usage: 15.7 GiB | 20260507 21:38:04.317 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:38:04.317 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:38:04.317 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:38:04.317 | INFO | High Water Breakdown: | 20260507 21:38:04.317 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:38:04.317 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:38:04.317 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:38:04.317 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:38:04.317 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:38:04.317 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:38:04.317 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:38:04.317 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:38:04.317 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:38:04.317 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:38:04.317 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:38:04.317 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:38:04.317 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:38:04.317 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:38:04.317 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:38:04.317 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:38:04.317 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:38:04.317 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:38:04.317 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:38:04.317 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:38:04.317 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:38:04.317 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:38:04.317 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:38:04.317 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:38:04.317 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:38:04.317 | INFO | ------------------------------------------ | 20260507 21:38:04.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:38:04.318 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:38:04.318 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:38:04.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:38:04.318 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.62e+01, 1.42e+01 / 1.41e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.04e-01, 1.15e-02 / 4.90e-03 | 20260507 21:38:04.318 | INFO | PerfReport: 12.15%, Baseline -- 4.02e+00 / 4.48e+00, 5.97e-01 / 2.27e-01 | 20260507 21:38:04.318 | INFO | PerfReport: 3.41%, Histogram -- 1.13e+00 / 1.48e+00, 2.10e-01 / 1.22e-01 | 20260507 21:38:04.318 | INFO | PerfReport: 15.71%, DME -- 4.16e+01 / 4.49e+01, 3.73e+00 / 1.43e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.85e+00 / 1.73e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.27e-01 / 2.33e-01 | 20260507 21:38:04.318 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.39e+00, 2.95e-01 / 1.79e-01 | 20260507 21:38:04.318 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.93e-01, 5.95e-02 / 3.13e-02 | 20260507 21:38:04.318 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:38:04.318 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:38:04.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:38:04.318 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 24.27%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 31.04%, Compute -- 3.31e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 1.80%, LockUpload -- 1.92e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 16.06%, LockCompute -- 1.71e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:38:04.318 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 21:38:04.431 | INFO | Starting a round of Baz IO | 20260507 21:38:07.912 | INFO | Populating RTMetrics report | 20260507 21:38:07.912 | INFO | RTMetrics report populated | 20260507 21:38:07.912 | INFO | Opening tmp RTMetrics json file | 20260507 21:38:07.914 | INFO | Serializing RTMetrics to json | 20260507 21:38:07.916 | INFO | Serialization done, swapping tmp file now... | 20260507 21:38:07.917 | INFO | RTMetrics done | 20260507 21:38:08.013 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.581 seconds | 20260507 21:38:08.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.60% / 0.01% / 56.39% | 20260507 21:38:08.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63178% / 3.97364e-06% / 7.15256e-05% | 20260507 21:38:08.013 | INFO | Baz IO overall: | 20260507 21:38:08.013 | INFO | BytesWritten : 1268386337472 | 20260507 21:38:08.013 | INFO | NumEvents : 614711054621 | 20260507 21:38:09.458 | INFO | Analyzing chunk frames = [2785280,2785792) | 20260507 21:38:09.458 | INFO | MemInfo: { | 20260507 21:38:09.458 | INFO | "gpu" : " 17 GB", | 20260507 21:38:09.458 | INFO | "huge" : "", | 20260507 21:38:09.458 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:09.458 | INFO | "malloc" : " 14 GB", | 20260507 21:38:09.458 | INFO | "pinned" : " 29 GB" | 20260507 21:38:09.458 | INFO | } | 20260507 21:38:09.458 | INFO | Perf report for the last 5.14204 seconds: | 20260507 21:38:09.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:09.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:09.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:09.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:09.458 | INFO | ANALYSIS : 69.5%, 2.9, 105.1ms, 0 | 20260507 21:38:09.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:09.458 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260507 21:38:09.458 | INFO | PRE_HQ : 68.1%, 1.8, 64.0ms, 0 | 20260507 21:38:09.458 | INFO | BAZWRITER : 69.2%, 4.4, 157.7ms, 0 | 20260507 21:38:09.458 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:38:09.458 | INFO | Uploaded 36GiB to GPU in 2.55482s (14.091GiB/s) | 20260507 21:38:09.458 | INFO | Downloaded 38GiB to GPU in 2.70554s (14.0452GiB/s) | 20260507 21:38:14.593 | INFO | Analyzing chunk frames = [2785792,2786304) | 20260507 21:38:14.593 | INFO | MemInfo: { | 20260507 21:38:14.593 | INFO | "gpu" : " 17 GB", | 20260507 21:38:14.593 | INFO | "huge" : "", | 20260507 21:38:14.593 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:14.593 | INFO | "malloc" : " 14 GB", | 20260507 21:38:14.593 | INFO | "pinned" : " 29 GB" | 20260507 21:38:14.593 | INFO | } | 20260507 21:38:14.593 | INFO | Perf report for the last 5.13486 seconds: | 20260507 21:38:14.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:14.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:38:14.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:38:14.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:14.593 | INFO | Uploaded 36GiB to GPU in 2.56315s (14.0452GiB/s) | 20260507 21:38:14.593 | INFO | Downloaded 34GiB to GPU in 2.48325s (13.6917GiB/s) | 20260507 21:38:19.728 | INFO | Analyzing chunk frames = [2786304,2786816) | 20260507 21:38:19.728 | INFO | MemInfo: { | 20260507 21:38:19.728 | INFO | "gpu" : " 17 GB", | 20260507 21:38:19.728 | INFO | "huge" : "", | 20260507 21:38:19.728 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:19.728 | INFO | "malloc" : " 14 GB", | 20260507 21:38:19.728 | INFO | "pinned" : " 29 GB" | 20260507 21:38:19.728 | INFO | } | 20260507 21:38:19.728 | INFO | Perf report for the last 5.13576 seconds: | 20260507 21:38:19.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:19.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.728 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:38:19.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.729 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:38:19.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:19.729 | INFO | Uploaded 36GiB to GPU in 2.54311s (14.1559GiB/s) | 20260507 21:38:19.729 | INFO | Downloaded 34GiB to GPU in 2.45697s (13.8382GiB/s) | 20260507 21:38:24.869 | INFO | Analyzing chunk frames = [2786816,2787328) | 20260507 21:38:24.869 | INFO | MemInfo: { | 20260507 21:38:24.869 | INFO | "gpu" : " 17 GB", | 20260507 21:38:24.869 | INFO | "huge" : "", | 20260507 21:38:24.869 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:24.869 | INFO | "malloc" : " 14 GB", | 20260507 21:38:24.869 | INFO | "pinned" : " 29 GB" | 20260507 21:38:24.869 | INFO | } | 20260507 21:38:24.869 | INFO | Perf report for the last 5.1408 seconds: | 20260507 21:38:24.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:24.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.869 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:38:24.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.869 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:38:24.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:24.870 | INFO | Uploaded 36GiB to GPU in 2.47841s (14.5254GiB/s) | 20260507 21:38:24.870 | INFO | Downloaded 34GiB to GPU in 2.38299s (14.2678GiB/s) | 20260507 21:38:30.006 | INFO | Analyzing chunk frames = [2787328,2787840) | 20260507 21:38:30.006 | INFO | MemInfo: { | 20260507 21:38:30.006 | INFO | "gpu" : " 17 GB", | 20260507 21:38:30.006 | INFO | "huge" : "", | 20260507 21:38:30.007 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:30.007 | INFO | "malloc" : " 14 GB", | 20260507 21:38:30.007 | INFO | "pinned" : " 29 GB" | 20260507 21:38:30.007 | INFO | } | 20260507 21:38:30.007 | INFO | Perf report for the last 5.13731 seconds: | 20260507 21:38:30.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:30.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 21:38:30.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:38:30.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:30.007 | INFO | Uploaded 36GiB to GPU in 2.39146s (15.0536GiB/s) | 20260507 21:38:30.007 | INFO | Downloaded 34GiB to GPU in 2.49993s (13.6004GiB/s) | 20260507 21:38:35.148 | INFO | Analyzing chunk frames = [2787840,2788352) | 20260507 21:38:35.148 | INFO | MemInfo: { | 20260507 21:38:35.148 | INFO | "gpu" : " 17 GB", | 20260507 21:38:35.148 | INFO | "huge" : "", | 20260507 21:38:35.148 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:35.148 | INFO | "malloc" : " 14 GB", | 20260507 21:38:35.148 | INFO | "pinned" : " 29 GB" | 20260507 21:38:35.148 | INFO | } | 20260507 21:38:35.148 | INFO | Perf report for the last 5.14171 seconds: | 20260507 21:38:35.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:35.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.148 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 21:38:35.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.148 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:38:35.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:35.149 | INFO | Uploaded 36GiB to GPU in 2.54793s (14.1291GiB/s) | 20260507 21:38:35.149 | INFO | Downloaded 34GiB to GPU in 2.39947s (14.1698GiB/s) | 20260507 21:38:40.286 | INFO | Analyzing chunk frames = [2788352,2788864) | 20260507 21:38:40.286 | INFO | MemInfo: { | 20260507 21:38:40.286 | INFO | "gpu" : " 17 GB", | 20260507 21:38:40.286 | INFO | "huge" : "", | 20260507 21:38:40.286 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:40.286 | INFO | "malloc" : " 14 GB", | 20260507 21:38:40.286 | INFO | "pinned" : " 29 GB" | 20260507 21:38:40.286 | INFO | } | 20260507 21:38:40.286 | INFO | Perf report for the last 5.13793 seconds: | 20260507 21:38:40.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:40.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.286 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:38:40.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.286 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 21:38:40.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:40.287 | INFO | Uploaded 36GiB to GPU in 2.55275s (14.1024GiB/s) | 20260507 21:38:40.287 | INFO | Downloaded 34GiB to GPU in 2.43365s (13.9708GiB/s) | 20260507 21:38:45.429 | INFO | Analyzing chunk frames = [2788864,2789376) | 20260507 21:38:45.429 | INFO | MemInfo: { | 20260507 21:38:45.430 | INFO | "gpu" : " 17 GB", | 20260507 21:38:45.430 | INFO | "huge" : "", | 20260507 21:38:45.430 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:45.430 | INFO | "malloc" : " 14 GB", | 20260507 21:38:45.430 | INFO | "pinned" : " 29 GB" | 20260507 21:38:45.430 | INFO | } | 20260507 21:38:45.430 | INFO | Perf report for the last 5.14365 seconds: | 20260507 21:38:45.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:45.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:38:45.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 21:38:45.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:45.430 | INFO | Uploaded 36GiB to GPU in 2.54262s (14.1586GiB/s) | 20260507 21:38:45.431 | INFO | Downloaded 34GiB to GPU in 2.44465s (13.9079GiB/s) | 20260507 21:38:45.544 | INFO | Starting a round of Baz IO | 20260507 21:38:49.013 | INFO | Populating RTMetrics report | 20260507 21:38:49.014 | INFO | RTMetrics report populated | 20260507 21:38:49.014 | INFO | Opening tmp RTMetrics json file | 20260507 21:38:49.021 | INFO | Serializing RTMetrics to json | 20260507 21:38:49.037 | INFO | Serialization done, swapping tmp file now... | 20260507 21:38:49.038 | INFO | RTMetrics done | 20260507 21:38:49.111 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.567 seconds | 20260507 21:38:49.111 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.60% / 0.01% / 56.40% | 20260507 21:38:49.111 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6237% / 7.94729e-06% / 4.76837e-05% | 20260507 21:38:49.111 | INFO | Baz IO overall: | 20260507 21:38:49.111 | INFO | BytesWritten : 1270269869090 | 20260507 21:38:49.111 | INFO | NumEvents : 615621542774 | 20260507 21:38:50.569 | INFO | Analyzing chunk frames = [2789376,2789888) | 20260507 21:38:50.569 | INFO | MemInfo: { | 20260507 21:38:50.569 | INFO | "gpu" : " 17 GB", | 20260507 21:38:50.569 | INFO | "huge" : "", | 20260507 21:38:50.569 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:50.569 | INFO | "malloc" : " 14 GB", | 20260507 21:38:50.569 | INFO | "pinned" : " 29 GB" | 20260507 21:38:50.569 | INFO | } | 20260507 21:38:50.569 | INFO | Perf report for the last 5.13921 seconds: | 20260507 21:38:50.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:50.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:50.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:50.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:50.569 | INFO | ANALYSIS : 69.0%, 2.9, 104.9ms, 0 | 20260507 21:38:50.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:50.569 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 21:38:50.569 | INFO | PRE_HQ : 67.7%, 1.8, 63.7ms, 0 | 20260507 21:38:50.569 | INFO | BAZWRITER : 69.0%, 4.1, 144.4ms, 0 | 20260507 21:38:50.569 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 21:38:50.569 | INFO | Uploaded 36GiB to GPU in 2.5787s (13.9605GiB/s) | 20260507 21:38:50.570 | INFO | Downloaded 38GiB to GPU in 2.70109s (14.0684GiB/s) | 20260507 21:38:55.707 | INFO | Analyzing chunk frames = [2789888,2790400) | 20260507 21:38:55.707 | INFO | MemInfo: { | 20260507 21:38:55.707 | INFO | "gpu" : " 17 GB", | 20260507 21:38:55.707 | INFO | "huge" : "", | 20260507 21:38:55.707 | INFO | "huge_shared" : " 29 GB", | 20260507 21:38:55.707 | INFO | "malloc" : " 14 GB", | 20260507 21:38:55.707 | INFO | "pinned" : " 29 GB" | 20260507 21:38:55.707 | INFO | } | 20260507 21:38:55.707 | INFO | Perf report for the last 5.13792 seconds: | 20260507 21:38:55.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:38:55.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.707 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 21:38:55.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.707 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:38:55.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:38:55.708 | INFO | Uploaded 36GiB to GPU in 2.55741s (14.0767GiB/s) | 20260507 21:38:55.708 | INFO | Downloaded 34GiB to GPU in 2.4838s (13.6887GiB/s) | 20260507 21:39:00.841 | INFO | Analyzing chunk frames = [2790400,2790912) | 20260507 21:39:00.841 | INFO | MemInfo: { | 20260507 21:39:00.841 | INFO | "gpu" : " 17 GB", | 20260507 21:39:00.841 | INFO | "huge" : "", | 20260507 21:39:00.841 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:00.842 | INFO | "malloc" : " 14 GB", | 20260507 21:39:00.842 | INFO | "pinned" : " 29 GB" | 20260507 21:39:00.842 | INFO | } | 20260507 21:39:00.842 | INFO | Perf report for the last 5.13455 seconds: | 20260507 21:39:00.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:00.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 21:39:00.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:39:00.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:00.842 | INFO | Uploaded 36GiB to GPU in 2.5179s (14.2976GiB/s) | 20260507 21:39:00.842 | INFO | Downloaded 34GiB to GPU in 2.4674s (13.7797GiB/s) | 20260507 21:39:05.983 | INFO | Analyzing chunk frames = [2790912,2791424) | 20260507 21:39:05.983 | INFO | MemInfo: { | 20260507 21:39:05.983 | INFO | "gpu" : " 17 GB", | 20260507 21:39:05.983 | INFO | "huge" : "", | 20260507 21:39:05.983 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:05.983 | INFO | "malloc" : " 14 GB", | 20260507 21:39:05.983 | INFO | "pinned" : " 29 GB" | 20260507 21:39:05.983 | INFO | } | 20260507 21:39:05.983 | INFO | Perf report for the last 5.1412 seconds: | 20260507 21:39:05.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:05.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.983 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:39:05.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.983 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:39:05.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:05.984 | INFO | Uploaded 36GiB to GPU in 2.45442s (14.6674GiB/s) | 20260507 21:39:05.984 | INFO | Downloaded 34GiB to GPU in 2.39699s (14.1845GiB/s) | 20260507 21:39:11.127 | INFO | Analyzing chunk frames = [2791424,2791936) | 20260507 21:39:11.127 | INFO | MemInfo: { | 20260507 21:39:11.127 | INFO | "gpu" : " 17 GB", | 20260507 21:39:11.127 | INFO | "huge" : "", | 20260507 21:39:11.127 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:11.127 | INFO | "malloc" : " 14 GB", | 20260507 21:39:11.127 | INFO | "pinned" : " 29 GB" | 20260507 21:39:11.127 | INFO | } | 20260507 21:39:11.128 | INFO | Perf report for the last 5.14476 seconds: | 20260507 21:39:11.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:11.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 21:39:11.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:39:11.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:11.128 | INFO | Uploaded 36GiB to GPU in 2.33786s (15.3987GiB/s) | 20260507 21:39:11.128 | INFO | Downloaded 34GiB to GPU in 2.49573s (13.6232GiB/s) | 20260507 21:39:16.269 | INFO | Analyzing chunk frames = [2791936,2792448) | 20260507 21:39:16.269 | INFO | MemInfo: { | 20260507 21:39:16.269 | INFO | "gpu" : " 17 GB", | 20260507 21:39:16.269 | INFO | "huge" : "", | 20260507 21:39:16.269 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:16.269 | INFO | "malloc" : " 14 GB", | 20260507 21:39:16.269 | INFO | "pinned" : " 29 GB" | 20260507 21:39:16.269 | INFO | } | 20260507 21:39:16.269 | INFO | Perf report for the last 5.14102 seconds: | 20260507 21:39:16.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:16.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260507 21:39:16.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:39:16.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:16.269 | INFO | Uploaded 36GiB to GPU in 2.42704s (14.8329GiB/s) | 20260507 21:39:16.269 | INFO | Downloaded 34GiB to GPU in 2.43572s (13.9589GiB/s) | 20260507 21:39:21.405 | INFO | Analyzing chunk frames = [2792448,2792960) | 20260507 21:39:21.405 | INFO | MemInfo: { | 20260507 21:39:21.405 | INFO | "gpu" : " 17 GB", | 20260507 21:39:21.405 | INFO | "huge" : "", | 20260507 21:39:21.405 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:21.405 | INFO | "malloc" : " 14 GB", | 20260507 21:39:21.405 | INFO | "pinned" : " 29 GB" | 20260507 21:39:21.405 | INFO | } | 20260507 21:39:21.405 | INFO | Perf report for the last 5.13654 seconds: | 20260507 21:39:21.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:21.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.405 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:39:21.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.405 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 21:39:21.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:21.406 | INFO | Uploaded 36GiB to GPU in 2.5004s (14.3977GiB/s) | 20260507 21:39:21.406 | INFO | Downloaded 34GiB to GPU in 2.47439s (13.7408GiB/s) | 20260507 21:39:26.539 | INFO | Analyzing chunk frames = [2792960,2793472) | 20260507 21:39:26.539 | INFO | MemInfo: { | 20260507 21:39:26.539 | INFO | "gpu" : " 17 GB", | 20260507 21:39:26.539 | INFO | "huge" : "", | 20260507 21:39:26.539 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:26.539 | INFO | "malloc" : " 14 GB", | 20260507 21:39:26.539 | INFO | "pinned" : " 29 GB" | 20260507 21:39:26.539 | INFO | } | 20260507 21:39:26.540 | INFO | Perf report for the last 5.13421 seconds: | 20260507 21:39:26.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:26.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:39:26.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:39:26.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:26.540 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:39:26.540 | INFO | Total mem usage: 16.7 GiB | 20260507 21:39:26.540 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:39:26.540 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.14%) | 20260507 21:39:26.540 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:39:26.540 | INFO | High Water Breakdown: | 20260507 21:39:26.540 | INFO | 60.4% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:39:26.540 | INFO | 39.5% : 6.03 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:39:26.540 | INFO | Uploaded 36GiB to GPU in 2.45822s (14.6447GiB/s) | 20260507 21:39:26.540 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:39:26.540 | INFO | Downloaded 34GiB to GPU in 2.45553s (13.8463GiB/s) | 20260507 21:39:26.540 | INFO | ------------------------------------------ | 20260507 21:39:26.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:39:26.540 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:39:26.540 | INFO | Total mem usage: 26.5 GiB | 20260507 21:39:26.540 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:39:26.540 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 21:39:26.540 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 21:39:26.540 | INFO | High Water Breakdown: | 20260507 21:39:26.540 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:39:26.540 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:39:26.541 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:39:26.541 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:39:26.541 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:39:26.541 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:39:26.541 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:39:26.541 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:39:26.541 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:39:26.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:39:26.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:39:26.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:39:26.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:39:26.541 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:39:26.541 | INFO | ------------------------------------------ | 20260507 21:39:26.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:39:26.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:39:26.541 | INFO | Total mem usage: 27.8 GiB | 20260507 21:39:26.541 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:39:26.541 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:39:26.541 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:39:26.541 | INFO | High Water Breakdown: | 20260507 21:39:26.541 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:39:26.541 | INFO | ------------------------------------------ | 20260507 21:39:26.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:39:26.541 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:39:26.541 | INFO | Total mem usage: 15.7 GiB | 20260507 21:39:26.541 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:39:26.541 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:39:26.541 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:39:26.541 | INFO | High Water Breakdown: | 20260507 21:39:26.541 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:39:26.541 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:39:26.541 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:39:26.541 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:39:26.541 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:39:26.541 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:39:26.541 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:39:26.541 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:39:26.541 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:39:26.541 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:39:26.541 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:39:26.541 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:39:26.541 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:39:26.541 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:39:26.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:39:26.541 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:39:26.541 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:39:26.541 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:39:26.541 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:39:26.541 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:39:26.541 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:39:26.541 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:39:26.541 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:39:26.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:39:26.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:39:26.541 | INFO | ------------------------------------------ | 20260507 21:39:26.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:39:26.541 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:39:26.541 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:39:26.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:39:26.541 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.58e+01, 1.40e+01 / 1.63e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.04e-01, 1.15e-02 / 5.92e-03 | 20260507 21:39:26.541 | INFO | PerfReport: 12.12%, Baseline -- 4.01e+00 / 4.46e+00, 6.02e-01 / 2.75e-01 | 20260507 21:39:26.541 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.25e-01 | 20260507 21:39:26.541 | INFO | PerfReport: 15.79%, DME -- 4.17e+01 / 4.51e+01, 3.77e+00 / 1.37e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.86e+00 / 1.63e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.33e-01 / 2.83e-01 | 20260507 21:39:26.541 | INFO | PerfReport: 3.50%, Metrics -- 1.16e+00 / 1.36e+00, 2.97e-01 / 1.51e-01 | 20260507 21:39:26.541 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.61e-01 / 3.96e-01, 6.05e-02 / 3.65e-02 | 20260507 21:39:26.541 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:39:26.541 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:39:26.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:39:26.541 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 24.42%, Upload -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 31.00%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 2.04%, LockUpload -- 2.17e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 2.41%, LockDownload -- 2.57e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 15.87%, LockCompute -- 1.69e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260507 21:39:26.541 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 21:39:26.637 | INFO | Starting a round of Baz IO | 20260507 21:39:30.124 | INFO | Populating RTMetrics report | 20260507 21:39:30.124 | INFO | RTMetrics report populated | 20260507 21:39:30.124 | INFO | Opening tmp RTMetrics json file | 20260507 21:39:30.149 | INFO | Serializing RTMetrics to json | 20260507 21:39:30.152 | INFO | Serialization done, swapping tmp file now... | 20260507 21:39:30.153 | INFO | RTMetrics done | 20260507 21:39:30.222 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.585 seconds | 20260507 21:39:30.222 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.59% / 0.01% / 56.40% | 20260507 21:39:30.222 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62284% / 3.97364e-06% / 5.96046e-05% | 20260507 21:39:30.222 | INFO | Baz IO overall: | 20260507 21:39:30.222 | INFO | BytesWritten : 1272153328653 | 20260507 21:39:30.222 | INFO | NumEvents : 616532178375 | 20260507 21:39:31.679 | INFO | Analyzing chunk frames = [2793472,2793984) | 20260507 21:39:31.679 | INFO | MemInfo: { | 20260507 21:39:31.679 | INFO | "gpu" : " 17 GB", | 20260507 21:39:31.679 | INFO | "huge" : "", | 20260507 21:39:31.679 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:31.679 | INFO | "malloc" : " 14 GB", | 20260507 21:39:31.679 | INFO | "pinned" : " 29 GB" | 20260507 21:39:31.679 | INFO | } | 20260507 21:39:31.679 | INFO | Perf report for the last 5.13977 seconds: | 20260507 21:39:31.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:31.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:31.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:31.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:31.679 | INFO | ANALYSIS : 69.0%, 3.0, 107.0ms, 0 | 20260507 21:39:31.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:31.680 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 21:39:31.680 | INFO | PRE_HQ : 66.0%, 1.7, 57.9ms, 0 | 20260507 21:39:31.680 | INFO | BAZWRITER : 69.3%, 4.1, 146.6ms, 0 | 20260507 21:39:31.680 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 21:39:31.680 | INFO | Uploaded 36GiB to GPU in 2.52085s (14.2809GiB/s) | 20260507 21:39:31.680 | INFO | Downloaded 38GiB to GPU in 2.701s (14.0689GiB/s) | 20260507 21:39:36.817 | INFO | Analyzing chunk frames = [2793984,2794496) | 20260507 21:39:36.817 | INFO | MemInfo: { | 20260507 21:39:36.817 | INFO | "gpu" : " 17 GB", | 20260507 21:39:36.817 | INFO | "huge" : "", | 20260507 21:39:36.817 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:36.817 | INFO | "malloc" : " 14 GB", | 20260507 21:39:36.817 | INFO | "pinned" : " 29 GB" | 20260507 21:39:36.817 | INFO | } | 20260507 21:39:36.817 | INFO | Perf report for the last 5.13778 seconds: | 20260507 21:39:36.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:36.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 21:39:36.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:39:36.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:36.817 | INFO | Uploaded 36GiB to GPU in 2.53726s (14.1885GiB/s) | 20260507 21:39:36.817 | INFO | Downloaded 34GiB to GPU in 2.49353s (13.6353GiB/s) | 20260507 21:39:41.956 | INFO | Analyzing chunk frames = [2794496,2795008) | 20260507 21:39:41.956 | INFO | MemInfo: { | 20260507 21:39:41.956 | INFO | "gpu" : " 17 GB", | 20260507 21:39:41.956 | INFO | "huge" : "", | 20260507 21:39:41.956 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:41.956 | INFO | "malloc" : " 14 GB", | 20260507 21:39:41.956 | INFO | "pinned" : " 29 GB" | 20260507 21:39:41.956 | INFO | } | 20260507 21:39:41.956 | INFO | Perf report for the last 5.13897 seconds: | 20260507 21:39:41.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:41.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 21:39:41.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:39:41.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:41.956 | INFO | Uploaded 36GiB to GPU in 2.54081s (14.1687GiB/s) | 20260507 21:39:41.956 | INFO | Downloaded 34GiB to GPU in 2.46294s (13.8046GiB/s) | 20260507 21:39:47.097 | INFO | Analyzing chunk frames = [2795008,2795520) | 20260507 21:39:47.097 | INFO | MemInfo: { | 20260507 21:39:47.097 | INFO | "gpu" : " 17 GB", | 20260507 21:39:47.097 | INFO | "huge" : "", | 20260507 21:39:47.097 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:47.097 | INFO | "malloc" : " 14 GB", | 20260507 21:39:47.097 | INFO | "pinned" : " 29 GB" | 20260507 21:39:47.097 | INFO | } | 20260507 21:39:47.097 | INFO | Perf report for the last 5.14121 seconds: | 20260507 21:39:47.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:47.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.097 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 21:39:47.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.097 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:39:47.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:47.098 | INFO | Uploaded 36GiB to GPU in 2.48214s (14.5036GiB/s) | 20260507 21:39:47.098 | INFO | Downloaded 34GiB to GPU in 2.38325s (14.2662GiB/s) | 20260507 21:39:52.238 | INFO | Analyzing chunk frames = [2795520,2796032) | 20260507 21:39:52.238 | INFO | MemInfo: { | 20260507 21:39:52.238 | INFO | "gpu" : " 17 GB", | 20260507 21:39:52.238 | INFO | "huge" : "", | 20260507 21:39:52.238 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:52.238 | INFO | "malloc" : " 14 GB", | 20260507 21:39:52.238 | INFO | "pinned" : " 29 GB" | 20260507 21:39:52.238 | INFO | } | 20260507 21:39:52.238 | INFO | Perf report for the last 5.14076 seconds: | 20260507 21:39:52.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:52.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.238 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:39:52.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.238 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:39:52.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:52.239 | INFO | Uploaded 36GiB to GPU in 2.36856s (15.1991GiB/s) | 20260507 21:39:52.239 | INFO | Downloaded 34GiB to GPU in 2.49746s (13.6138GiB/s) | 20260507 21:39:57.382 | INFO | Analyzing chunk frames = [2796032,2796544) | 20260507 21:39:57.382 | INFO | MemInfo: { | 20260507 21:39:57.382 | INFO | "gpu" : " 17 GB", | 20260507 21:39:57.382 | INFO | "huge" : "", | 20260507 21:39:57.382 | INFO | "huge_shared" : " 29 GB", | 20260507 21:39:57.382 | INFO | "malloc" : " 14 GB", | 20260507 21:39:57.382 | INFO | "pinned" : " 29 GB" | 20260507 21:39:57.382 | INFO | } | 20260507 21:39:57.382 | INFO | Perf report for the last 5.14434 seconds: | 20260507 21:39:57.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:39:57.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.383 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 21:39:57.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.383 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260507 21:39:57.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:39:57.383 | INFO | Uploaded 36GiB to GPU in 2.52232s (14.2726GiB/s) | 20260507 21:39:57.383 | INFO | Downloaded 34GiB to GPU in 2.42639s (14.0126GiB/s) | 20260507 21:40:02.520 | INFO | Analyzing chunk frames = [2796544,2797056) | 20260507 21:40:02.520 | INFO | MemInfo: { | 20260507 21:40:02.520 | INFO | "gpu" : " 17 GB", | 20260507 21:40:02.520 | INFO | "huge" : "", | 20260507 21:40:02.521 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:02.521 | INFO | "malloc" : " 14 GB", | 20260507 21:40:02.521 | INFO | "pinned" : " 29 GB" | 20260507 21:40:02.521 | INFO | } | 20260507 21:40:02.521 | INFO | Perf report for the last 5.13843 seconds: | 20260507 21:40:02.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:02.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.521 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 21:40:02.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.521 | INFO | PRE_HQ : 36.7%, 1.0, 19.0ms, 0 | 20260507 21:40:02.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:02.522 | INFO | Uploaded 36GiB to GPU in 2.54027s (14.1717GiB/s) | 20260507 21:40:02.522 | INFO | Downloaded 34GiB to GPU in 2.44309s (13.9168GiB/s) | 20260507 21:40:07.654 | INFO | Analyzing chunk frames = [2797056,2797568) | 20260507 21:40:07.654 | INFO | MemInfo: { | 20260507 21:40:07.654 | INFO | "gpu" : " 17 GB", | 20260507 21:40:07.654 | INFO | "huge" : "", | 20260507 21:40:07.655 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:07.655 | INFO | "malloc" : " 14 GB", | 20260507 21:40:07.655 | INFO | "pinned" : " 29 GB" | 20260507 21:40:07.655 | INFO | } | 20260507 21:40:07.655 | INFO | Perf report for the last 5.13402 seconds: | 20260507 21:40:07.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:07.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260507 21:40:07.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 21:40:07.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:07.655 | INFO | Uploaded 36GiB to GPU in 2.52093s (14.2805GiB/s) | 20260507 21:40:07.655 | INFO | Downloaded 34GiB to GPU in 2.42975s (13.9932GiB/s) | 20260507 21:40:07.768 | INFO | Starting a round of Baz IO | 20260507 21:40:11.260 | INFO | Populating RTMetrics report | 20260507 21:40:11.260 | INFO | RTMetrics report populated | 20260507 21:40:11.260 | INFO | Opening tmp RTMetrics json file | 20260507 21:40:11.262 | INFO | Serializing RTMetrics to json | 20260507 21:40:11.267 | INFO | Serialization done, swapping tmp file now... | 20260507 21:40:11.268 | INFO | RTMetrics done | 20260507 21:40:11.364 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.596 seconds | 20260507 21:40:11.364 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.58% / 0.01% / 56.41% | 20260507 21:40:11.364 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61747% / 3.97364e-06% / 5.16574e-05% | 20260507 21:40:11.364 | INFO | Baz IO overall: | 20260507 21:40:11.364 | INFO | BytesWritten : 1274036176851 | 20260507 21:40:11.364 | INFO | NumEvents : 617442122903 | 20260507 21:40:12.792 | INFO | Analyzing chunk frames = [2797568,2798080) | 20260507 21:40:12.792 | INFO | MemInfo: { | 20260507 21:40:12.792 | INFO | "gpu" : " 17 GB", | 20260507 21:40:12.792 | INFO | "huge" : "", | 20260507 21:40:12.792 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:12.792 | INFO | "malloc" : " 14 GB", | 20260507 21:40:12.792 | INFO | "pinned" : " 29 GB" | 20260507 21:40:12.792 | INFO | } | 20260507 21:40:12.792 | INFO | Perf report for the last 5.13703 seconds: | 20260507 21:40:12.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:12.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:12.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:12.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:12.792 | INFO | ANALYSIS : 69.7%, 2.9, 104.4ms, 0 | 20260507 21:40:12.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:12.792 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260507 21:40:12.792 | INFO | PRE_HQ : 68.7%, 1.9, 66.0ms, 0 | 20260507 21:40:12.792 | INFO | BAZWRITER : 70.0%, 4.8, 172.2ms, 0 | 20260507 21:40:12.792 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 21:40:12.792 | INFO | Uploaded 36GiB to GPU in 2.52227s (14.2728GiB/s) | 20260507 21:40:12.792 | INFO | Downloaded 38GiB to GPU in 2.668s (14.2429GiB/s) | 20260507 21:40:17.933 | INFO | Analyzing chunk frames = [2798080,2798592) | 20260507 21:40:17.933 | INFO | MemInfo: { | 20260507 21:40:17.933 | INFO | "gpu" : " 17 GB", | 20260507 21:40:17.933 | INFO | "huge" : "", | 20260507 21:40:17.933 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:17.933 | INFO | "malloc" : " 14 GB", | 20260507 21:40:17.933 | INFO | "pinned" : " 29 GB" | 20260507 21:40:17.933 | INFO | } | 20260507 21:40:17.933 | INFO | Perf report for the last 5.14117 seconds: | 20260507 21:40:17.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:17.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 21:40:17.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:40:17.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:17.933 | INFO | Uploaded 36GiB to GPU in 2.50661s (14.362GiB/s) | 20260507 21:40:17.933 | INFO | Downloaded 34GiB to GPU in 2.50074s (13.596GiB/s) | 20260507 21:40:23.079 | INFO | Analyzing chunk frames = [2798592,2799104) | 20260507 21:40:23.079 | INFO | MemInfo: { | 20260507 21:40:23.079 | INFO | "gpu" : " 17 GB", | 20260507 21:40:23.079 | INFO | "huge" : "", | 20260507 21:40:23.079 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:23.079 | INFO | "malloc" : " 14 GB", | 20260507 21:40:23.079 | INFO | "pinned" : " 29 GB" | 20260507 21:40:23.079 | INFO | } | 20260507 21:40:23.079 | INFO | Perf report for the last 5.14564 seconds: | 20260507 21:40:23.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:23.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.079 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260507 21:40:23.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.079 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:40:23.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:23.080 | INFO | Uploaded 36GiB to GPU in 2.51314s (14.3247GiB/s) | 20260507 21:40:23.080 | INFO | Downloaded 34GiB to GPU in 2.46486s (13.7939GiB/s) | 20260507 21:40:28.217 | INFO | Analyzing chunk frames = [2799104,2799616) | 20260507 21:40:28.217 | INFO | MemInfo: { | 20260507 21:40:28.217 | INFO | "gpu" : " 17 GB", | 20260507 21:40:28.217 | INFO | "huge" : "", | 20260507 21:40:28.217 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:28.217 | INFO | "malloc" : " 14 GB", | 20260507 21:40:28.217 | INFO | "pinned" : " 29 GB" | 20260507 21:40:28.217 | INFO | } | 20260507 21:40:28.217 | INFO | Perf report for the last 5.13818 seconds: | 20260507 21:40:28.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:28.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 21:40:28.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:40:28.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:28.217 | INFO | Uploaded 36GiB to GPU in 2.46588s (14.5993GiB/s) | 20260507 21:40:28.218 | INFO | Downloaded 34GiB to GPU in 2.39628s (14.1887GiB/s) | 20260507 21:40:33.355 | INFO | Analyzing chunk frames = [2799616,2800128) | 20260507 21:40:33.355 | INFO | MemInfo: { | 20260507 21:40:33.355 | INFO | "gpu" : " 17 GB", | 20260507 21:40:33.355 | INFO | "huge" : "", | 20260507 21:40:33.355 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:33.355 | INFO | "malloc" : " 14 GB", | 20260507 21:40:33.355 | INFO | "pinned" : " 29 GB" | 20260507 21:40:33.355 | INFO | } | 20260507 21:40:33.355 | INFO | Perf report for the last 5.1384 seconds: | 20260507 21:40:33.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:33.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.355 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:40:33.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.356 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:40:33.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:33.356 | INFO | Uploaded 36GiB to GPU in 2.33469s (15.4196GiB/s) | 20260507 21:40:33.356 | INFO | Downloaded 34GiB to GPU in 2.51141s (13.5382GiB/s) | 20260507 21:40:38.492 | INFO | Analyzing chunk frames = [2800128,2800640) | 20260507 21:40:38.492 | INFO | MemInfo: { | 20260507 21:40:38.492 | INFO | "gpu" : " 17 GB", | 20260507 21:40:38.492 | INFO | "huge" : "", | 20260507 21:40:38.492 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:38.492 | INFO | "malloc" : " 14 GB", | 20260507 21:40:38.492 | INFO | "pinned" : " 29 GB" | 20260507 21:40:38.492 | INFO | } | 20260507 21:40:38.492 | INFO | Perf report for the last 5.13705 seconds: | 20260507 21:40:38.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:38.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:40:38.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:40:38.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:38.493 | INFO | Uploaded 36GiB to GPU in 2.49131s (14.4502GiB/s) | 20260507 21:40:38.493 | INFO | Downloaded 34GiB to GPU in 2.43341s (13.9722GiB/s) | 20260507 21:40:43.635 | INFO | Analyzing chunk frames = [2800640,2801152) | 20260507 21:40:43.635 | INFO | MemInfo: { | 20260507 21:40:43.635 | INFO | "gpu" : " 17 GB", | 20260507 21:40:43.635 | INFO | "huge" : "", | 20260507 21:40:43.635 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:43.635 | INFO | "malloc" : " 14 GB", | 20260507 21:40:43.635 | INFO | "pinned" : " 29 GB" | 20260507 21:40:43.635 | INFO | } | 20260507 21:40:43.635 | INFO | Perf report for the last 5.14269 seconds: | 20260507 21:40:43.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:43.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:40:43.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260507 21:40:43.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:43.635 | INFO | Uploaded 36GiB to GPU in 2.52839s (14.2383GiB/s) | 20260507 21:40:43.635 | INFO | Downloaded 34GiB to GPU in 2.46835s (13.7744GiB/s) | 20260507 21:40:48.769 | INFO | Analyzing chunk frames = [2801152,2801664) | 20260507 21:40:48.769 | INFO | MemInfo: { | 20260507 21:40:48.769 | INFO | "gpu" : " 17 GB", | 20260507 21:40:48.769 | INFO | "huge" : "", | 20260507 21:40:48.769 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:48.769 | INFO | "malloc" : " 14 GB", | 20260507 21:40:48.769 | INFO | "pinned" : " 29 GB" | 20260507 21:40:48.770 | INFO | } | 20260507 21:40:48.770 | INFO | Perf report for the last 5.13452 seconds: | 20260507 21:40:48.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:48.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:40:48.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:40:48.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:48.770 | INFO | Uploaded 36GiB to GPU in 2.49253s (14.4431GiB/s) | 20260507 21:40:48.770 | INFO | Downloaded 34GiB to GPU in 2.44607s (13.8998GiB/s) | 20260507 21:40:48.770 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:40:48.770 | INFO | Total mem usage: 16.7 GiB | 20260507 21:40:48.770 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:40:48.770 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.13%) | 20260507 21:40:48.770 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:40:48.770 | INFO | High Water Breakdown: | 20260507 21:40:48.770 | INFO | 60.9% : 9.57 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:40:48.770 | INFO | 39% : 6.13 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:40:48.770 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:40:48.770 | INFO | ------------------------------------------ | 20260507 21:40:48.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:40:48.770 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:40:48.770 | INFO | Total mem usage: 26.5 GiB | 20260507 21:40:48.770 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:40:48.770 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:40:48.770 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:40:48.770 | INFO | High Water Breakdown: | 20260507 21:40:48.770 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:40:48.770 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:40:48.770 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:40:48.770 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:40:48.770 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:40:48.770 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:40:48.770 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:40:48.770 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:40:48.770 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:40:48.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:40:48.770 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:40:48.770 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:40:48.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:40:48.770 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:40:48.770 | INFO | ------------------------------------------ | 20260507 21:40:48.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:40:48.770 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:40:48.770 | INFO | Total mem usage: 27.8 GiB | 20260507 21:40:48.770 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:40:48.770 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:40:48.770 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:40:48.770 | INFO | High Water Breakdown: | 20260507 21:40:48.770 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:40:48.770 | INFO | ------------------------------------------ | 20260507 21:40:48.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:40:48.770 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:40:48.770 | INFO | Total mem usage: 15.7 GiB | 20260507 21:40:48.770 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:40:48.770 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:40:48.770 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:40:48.770 | INFO | High Water Breakdown: | 20260507 21:40:48.770 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:40:48.770 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:40:48.770 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:40:48.770 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:40:48.770 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:40:48.770 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:40:48.770 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:40:48.770 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:40:48.770 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:40:48.770 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:40:48.770 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:40:48.770 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:40:48.770 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:40:48.770 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:40:48.770 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:40:48.770 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:40:48.770 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:40:48.770 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:40:48.770 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:40:48.770 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:40:48.770 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:40:48.770 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:40:48.770 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:40:48.770 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:40:48.770 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:40:48.770 | INFO | ------------------------------------------ | 20260507 21:40:48.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:40:48.770 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:40:48.770 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:40:48.770 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:40:48.770 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.42e+01 / 1.68e+00 | 20260507 21:40:48.770 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.15e-02 / 4.45e-03 | 20260507 21:40:48.770 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.49e+00, 5.96e-01 / 2.62e-01 | 20260507 21:40:48.770 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.07e-01 | 20260507 21:40:48.770 | INFO | PerfReport: 15.83%, DME -- 4.18e+01 / 4.52e+01, 3.82e+00 / 1.56e+00 | 20260507 21:40:48.770 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.80e+00 / 2.16e+00 | 20260507 21:40:48.770 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.46e+00 / 2.76e+00, 4.31e-01 / 2.51e-01 | 20260507 21:40:48.770 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.35e+00, 2.98e-01 / 1.72e-01 | 20260507 21:40:48.770 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.61e-01 / 3.88e-01, 5.81e-02 / 3.29e-02 | 20260507 21:40:48.771 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:40:48.771 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:40:48.771 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:40:48.771 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 24.34%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 30.96%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 2.00%, LockUpload -- 2.14e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 2.51%, LockDownload -- 2.68e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 21:40:48.771 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 21:40:48.867 | INFO | Starting a round of Baz IO | 20260507 21:40:52.343 | INFO | Populating RTMetrics report | 20260507 21:40:52.343 | INFO | RTMetrics report populated | 20260507 21:40:52.343 | INFO | Opening tmp RTMetrics json file | 20260507 21:40:52.356 | INFO | Serializing RTMetrics to json | 20260507 21:40:52.370 | INFO | Serialization done, swapping tmp file now... | 20260507 21:40:52.370 | INFO | RTMetrics done | 20260507 21:40:52.447 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.579 seconds | 20260507 21:40:52.447 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.57% / 0.01% / 56.42% | 20260507 21:40:52.447 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61097% / 1.58946e-05% / 4.37101e-05% | 20260507 21:40:52.447 | INFO | Baz IO overall: | 20260507 21:40:52.447 | INFO | BytesWritten : 1275917953311 | 20260507 21:40:52.447 | INFO | NumEvents : 618351313636 | 20260507 21:40:53.906 | INFO | Analyzing chunk frames = [2801664,2802176) | 20260507 21:40:53.906 | INFO | MemInfo: { | 20260507 21:40:53.906 | INFO | "gpu" : " 17 GB", | 20260507 21:40:53.906 | INFO | "huge" : "", | 20260507 21:40:53.906 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:53.906 | INFO | "malloc" : " 14 GB", | 20260507 21:40:53.906 | INFO | "pinned" : " 29 GB" | 20260507 21:40:53.906 | INFO | } | 20260507 21:40:53.906 | INFO | Perf report for the last 5.13634 seconds: | 20260507 21:40:53.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:53.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:53.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:53.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:53.906 | INFO | ANALYSIS : 68.8%, 2.9, 106.3ms, 0 | 20260507 21:40:53.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:53.906 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 21:40:53.906 | INFO | PRE_HQ : 66.2%, 1.7, 59.9ms, 0 | 20260507 21:40:53.906 | INFO | BAZWRITER : 69.3%, 4.2, 148.7ms, 0 | 20260507 21:40:53.906 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 21:40:53.906 | INFO | Uploaded 36GiB to GPU in 2.55434s (14.0936GiB/s) | 20260507 21:40:53.906 | INFO | Downloaded 38GiB to GPU in 2.69695s (14.09GiB/s) | 20260507 21:40:59.049 | INFO | Analyzing chunk frames = [2802176,2802688) | 20260507 21:40:59.049 | INFO | MemInfo: { | 20260507 21:40:59.049 | INFO | "gpu" : " 17 GB", | 20260507 21:40:59.049 | INFO | "huge" : "", | 20260507 21:40:59.049 | INFO | "huge_shared" : " 29 GB", | 20260507 21:40:59.049 | INFO | "malloc" : " 14 GB", | 20260507 21:40:59.049 | INFO | "pinned" : " 29 GB" | 20260507 21:40:59.049 | INFO | } | 20260507 21:40:59.049 | INFO | Perf report for the last 5.14313 seconds: | 20260507 21:40:59.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:40:59.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:40:59.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 21:40:59.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:40:59.049 | INFO | Uploaded 36GiB to GPU in 2.5603s (14.0609GiB/s) | 20260507 21:40:59.049 | INFO | Downloaded 34GiB to GPU in 2.45326s (13.8591GiB/s) | 20260507 21:41:04.192 | INFO | Analyzing chunk frames = [2802688,2803200) | 20260507 21:41:04.192 | INFO | MemInfo: { | 20260507 21:41:04.192 | INFO | "gpu" : " 17 GB", | 20260507 21:41:04.192 | INFO | "huge" : "", | 20260507 21:41:04.192 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:04.192 | INFO | "malloc" : " 14 GB", | 20260507 21:41:04.192 | INFO | "pinned" : " 29 GB" | 20260507 21:41:04.192 | INFO | } | 20260507 21:41:04.192 | INFO | Perf report for the last 5.14283 seconds: | 20260507 21:41:04.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:04.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.192 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:41:04.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.192 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:41:04.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:04.193 | INFO | Uploaded 36GiB to GPU in 2.56903s (14.0131GiB/s) | 20260507 21:41:04.193 | INFO | Downloaded 34GiB to GPU in 2.43794s (13.9462GiB/s) | 20260507 21:41:09.334 | INFO | Analyzing chunk frames = [2803200,2803712) | 20260507 21:41:09.334 | INFO | MemInfo: { | 20260507 21:41:09.334 | INFO | "gpu" : " 17 GB", | 20260507 21:41:09.334 | INFO | "huge" : "", | 20260507 21:41:09.334 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:09.334 | INFO | "malloc" : " 14 GB", | 20260507 21:41:09.334 | INFO | "pinned" : " 29 GB" | 20260507 21:41:09.334 | INFO | } | 20260507 21:41:09.334 | INFO | Perf report for the last 5.14207 seconds: | 20260507 21:41:09.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:09.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260507 21:41:09.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:41:09.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:09.334 | INFO | Uploaded 36GiB to GPU in 2.48287s (14.4994GiB/s) | 20260507 21:41:09.334 | INFO | Downloaded 34GiB to GPU in 2.38455s (14.2585GiB/s) | 20260507 21:41:14.471 | INFO | Analyzing chunk frames = [2803712,2804224) | 20260507 21:41:14.471 | INFO | MemInfo: { | 20260507 21:41:14.471 | INFO | "gpu" : " 17 GB", | 20260507 21:41:14.471 | INFO | "huge" : "", | 20260507 21:41:14.471 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:14.471 | INFO | "malloc" : " 14 GB", | 20260507 21:41:14.471 | INFO | "pinned" : " 29 GB" | 20260507 21:41:14.471 | INFO | } | 20260507 21:41:14.472 | INFO | Perf report for the last 5.13739 seconds: | 20260507 21:41:14.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:14.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:41:14.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:41:14.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:14.472 | INFO | Uploaded 36GiB to GPU in 2.37638s (15.1491GiB/s) | 20260507 21:41:14.472 | INFO | Downloaded 34GiB to GPU in 2.50171s (13.5907GiB/s) | 20260507 21:41:19.606 | INFO | Analyzing chunk frames = [2804224,2804736) | 20260507 21:41:19.606 | INFO | MemInfo: { | 20260507 21:41:19.606 | INFO | "gpu" : " 17 GB", | 20260507 21:41:19.606 | INFO | "huge" : "", | 20260507 21:41:19.606 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:19.606 | INFO | "malloc" : " 14 GB", | 20260507 21:41:19.606 | INFO | "pinned" : " 29 GB" | 20260507 21:41:19.606 | INFO | } | 20260507 21:41:19.606 | INFO | Perf report for the last 5.13435 seconds: | 20260507 21:41:19.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:19.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 21:41:19.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:41:19.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:19.606 | INFO | Uploaded 36GiB to GPU in 2.51212s (14.3305GiB/s) | 20260507 21:41:19.606 | INFO | Downloaded 34GiB to GPU in 2.41142s (14.0996GiB/s) | 20260507 21:41:24.743 | INFO | Analyzing chunk frames = [2804736,2805248) | 20260507 21:41:24.743 | INFO | MemInfo: { | 20260507 21:41:24.743 | INFO | "gpu" : " 17 GB", | 20260507 21:41:24.743 | INFO | "huge" : "", | 20260507 21:41:24.743 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:24.743 | INFO | "malloc" : " 14 GB", | 20260507 21:41:24.743 | INFO | "pinned" : " 29 GB" | 20260507 21:41:24.743 | INFO | } | 20260507 21:41:24.743 | INFO | Perf report for the last 5.13746 seconds: | 20260507 21:41:24.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:24.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:41:24.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:41:24.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:24.744 | INFO | Uploaded 36GiB to GPU in 2.55753s (14.0761GiB/s) | 20260507 21:41:24.744 | INFO | Downloaded 34GiB to GPU in 2.43901s (13.9401GiB/s) | 20260507 21:41:29.885 | INFO | Analyzing chunk frames = [2805248,2805760) | 20260507 21:41:29.885 | INFO | MemInfo: { | 20260507 21:41:29.885 | INFO | "gpu" : " 17 GB", | 20260507 21:41:29.885 | INFO | "huge" : "", | 20260507 21:41:29.885 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:29.885 | INFO | "malloc" : " 14 GB", | 20260507 21:41:29.885 | INFO | "pinned" : " 29 GB" | 20260507 21:41:29.885 | INFO | } | 20260507 21:41:29.885 | INFO | Perf report for the last 5.14178 seconds: | 20260507 21:41:29.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:29.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 21:41:29.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:41:29.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:29.885 | INFO | Uploaded 36GiB to GPU in 2.53214s (14.2172GiB/s) | 20260507 21:41:29.885 | INFO | Downloaded 34GiB to GPU in 2.42752s (14.0061GiB/s) | 20260507 21:41:29.982 | INFO | Starting a round of Baz IO | 20260507 21:41:33.456 | INFO | Populating RTMetrics report | 20260507 21:41:33.456 | INFO | RTMetrics report populated | 20260507 21:41:33.456 | INFO | Opening tmp RTMetrics json file | 20260507 21:41:33.485 | INFO | Serializing RTMetrics to json | 20260507 21:41:33.489 | INFO | Serialization done, swapping tmp file now... | 20260507 21:41:33.622 | INFO | RTMetrics done | 20260507 21:41:33.686 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.703 seconds | 20260507 21:41:33.686 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.57% / 0.01% / 56.43% | 20260507 21:41:33.686 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61347% / 7.94729e-06% / 5.5631e-05% | 20260507 21:41:33.686 | INFO | Baz IO overall: | 20260507 21:41:33.686 | INFO | BytesWritten : 1277801423599 | 20260507 21:41:33.686 | INFO | NumEvents : 619261834833 | 20260507 21:41:35.028 | INFO | Analyzing chunk frames = [2805760,2806272) | 20260507 21:41:35.028 | INFO | MemInfo: { | 20260507 21:41:35.028 | INFO | "gpu" : " 17 GB", | 20260507 21:41:35.028 | INFO | "huge" : "", | 20260507 21:41:35.028 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:35.028 | INFO | "malloc" : " 14 GB", | 20260507 21:41:35.028 | INFO | "pinned" : " 29 GB" | 20260507 21:41:35.028 | INFO | } | 20260507 21:41:35.028 | INFO | Perf report for the last 5.14275 seconds: | 20260507 21:41:35.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:35.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:35.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:35.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:35.028 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 21:41:35.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:35.028 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 21:41:35.028 | INFO | PRE_HQ : 65.4%, 1.7, 57.5ms, 0 | 20260507 21:41:35.028 | INFO | BAZWRITER : 71.6%, 3.9, 143.9ms, 0 | 20260507 21:41:35.028 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260507 21:41:35.028 | INFO | Uploaded 36GiB to GPU in 2.54238s (14.16GiB/s) | 20260507 21:41:35.028 | INFO | Downloaded 38GiB to GPU in 2.69283s (14.1115GiB/s) | 20260507 21:41:40.170 | INFO | Analyzing chunk frames = [2806272,2806784) | 20260507 21:41:40.170 | INFO | MemInfo: { | 20260507 21:41:40.170 | INFO | "gpu" : " 17 GB", | 20260507 21:41:40.170 | INFO | "huge" : "", | 20260507 21:41:40.170 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:40.170 | INFO | "malloc" : " 14 GB", | 20260507 21:41:40.170 | INFO | "pinned" : " 29 GB" | 20260507 21:41:40.170 | INFO | } | 20260507 21:41:40.170 | INFO | Perf report for the last 5.14247 seconds: | 20260507 21:41:40.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:40.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.170 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:41:40.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.170 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 21:41:40.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:40.171 | INFO | Uploaded 36GiB to GPU in 2.54545s (14.1429GiB/s) | 20260507 21:41:40.171 | INFO | Downloaded 34GiB to GPU in 2.46994s (13.7655GiB/s) | 20260507 21:41:45.307 | INFO | Analyzing chunk frames = [2806784,2807296) | 20260507 21:41:45.307 | INFO | MemInfo: { | 20260507 21:41:45.307 | INFO | "gpu" : " 17 GB", | 20260507 21:41:45.307 | INFO | "huge" : "", | 20260507 21:41:45.307 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:45.307 | INFO | "malloc" : " 14 GB", | 20260507 21:41:45.307 | INFO | "pinned" : " 29 GB" | 20260507 21:41:45.307 | INFO | } | 20260507 21:41:45.307 | INFO | Perf report for the last 5.13697 seconds: | 20260507 21:41:45.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:45.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.307 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 21:41:45.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.307 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:41:45.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:45.308 | INFO | Uploaded 36GiB to GPU in 2.5922s (13.8878GiB/s) | 20260507 21:41:45.308 | INFO | Downloaded 34GiB to GPU in 2.46591s (13.788GiB/s) | 20260507 21:41:50.444 | INFO | Analyzing chunk frames = [2807296,2807808) | 20260507 21:41:50.444 | INFO | MemInfo: { | 20260507 21:41:50.444 | INFO | "gpu" : " 17 GB", | 20260507 21:41:50.444 | INFO | "huge" : "", | 20260507 21:41:50.444 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:50.444 | INFO | "malloc" : " 14 GB", | 20260507 21:41:50.444 | INFO | "pinned" : " 29 GB" | 20260507 21:41:50.444 | INFO | } | 20260507 21:41:50.444 | INFO | Perf report for the last 5.13705 seconds: | 20260507 21:41:50.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:50.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260507 21:41:50.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 21:41:50.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:50.445 | INFO | Uploaded 36GiB to GPU in 2.57044s (14.0054GiB/s) | 20260507 21:41:50.445 | INFO | Downloaded 34GiB to GPU in 2.45581s (13.8447GiB/s) | 20260507 21:41:55.582 | INFO | Analyzing chunk frames = [2807808,2808320) | 20260507 21:41:55.582 | INFO | MemInfo: { | 20260507 21:41:55.582 | INFO | "gpu" : " 17 GB", | 20260507 21:41:55.582 | INFO | "huge" : "", | 20260507 21:41:55.582 | INFO | "huge_shared" : " 29 GB", | 20260507 21:41:55.582 | INFO | "malloc" : " 14 GB", | 20260507 21:41:55.582 | INFO | "pinned" : " 29 GB" | 20260507 21:41:55.582 | INFO | } | 20260507 21:41:55.582 | INFO | Perf report for the last 5.1372 seconds: | 20260507 21:41:55.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:41:55.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 21:41:55.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 21:41:55.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:41:55.582 | INFO | Uploaded 36GiB to GPU in 2.40662s (14.9587GiB/s) | 20260507 21:41:55.582 | INFO | Downloaded 34GiB to GPU in 2.48785s (13.6664GiB/s) | 20260507 21:42:00.719 | INFO | Analyzing chunk frames = [2808320,2808832) | 20260507 21:42:00.719 | INFO | MemInfo: { | 20260507 21:42:00.719 | INFO | "gpu" : " 17 GB", | 20260507 21:42:00.719 | INFO | "huge" : "", | 20260507 21:42:00.719 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:00.719 | INFO | "malloc" : " 14 GB", | 20260507 21:42:00.719 | INFO | "pinned" : " 29 GB" | 20260507 21:42:00.719 | INFO | } | 20260507 21:42:00.719 | INFO | Perf report for the last 5.13768 seconds: | 20260507 21:42:00.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:00.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 21:42:00.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:42:00.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:00.720 | INFO | Uploaded 36GiB to GPU in 2.49331s (14.4386GiB/s) | 20260507 21:42:00.720 | INFO | Downloaded 34GiB to GPU in 2.42053s (14.0465GiB/s) | 20260507 21:42:05.856 | INFO | Analyzing chunk frames = [2808832,2809344) | 20260507 21:42:05.856 | INFO | MemInfo: { | 20260507 21:42:05.856 | INFO | "gpu" : " 17 GB", | 20260507 21:42:05.856 | INFO | "huge" : "", | 20260507 21:42:05.856 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:05.856 | INFO | "malloc" : " 14 GB", | 20260507 21:42:05.856 | INFO | "pinned" : " 29 GB" | 20260507 21:42:05.856 | INFO | } | 20260507 21:42:05.856 | INFO | Perf report for the last 5.13705 seconds: | 20260507 21:42:05.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:05.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:42:05.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:42:05.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:05.857 | INFO | Uploaded 36GiB to GPU in 2.52004s (14.2855GiB/s) | 20260507 21:42:05.857 | INFO | Downloaded 34GiB to GPU in 2.45837s (13.8303GiB/s) | 20260507 21:42:10.995 | INFO | Analyzing chunk frames = [2809344,2809856) | 20260507 21:42:10.995 | INFO | MemInfo: { | 20260507 21:42:10.995 | INFO | "gpu" : " 17 GB", | 20260507 21:42:10.995 | INFO | "huge" : "", | 20260507 21:42:10.995 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:10.995 | INFO | "malloc" : " 14 GB", | 20260507 21:42:10.995 | INFO | "pinned" : " 29 GB" | 20260507 21:42:10.995 | INFO | } | 20260507 21:42:10.995 | INFO | Perf report for the last 5.13856 seconds: | 20260507 21:42:10.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:10.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:42:10.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:42:10.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:10.995 | INFO | Uploaded 36GiB to GPU in 2.47326s (14.5557GiB/s) | 20260507 21:42:10.995 | INFO | Downloaded 34GiB to GPU in 2.43716s (13.9506GiB/s) | 20260507 21:42:10.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:42:10.996 | INFO | Total mem usage: 16.7 GiB | 20260507 21:42:10.996 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:42:10.996 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.2%) | 20260507 21:42:10.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:42:10.996 | INFO | High Water Breakdown: | 20260507 21:42:10.996 | INFO | 60.1% : 9.03 GiB ( 87.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:42:10.996 | INFO | 39.7% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:42:10.996 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:42:10.996 | INFO | ------------------------------------------ | 20260507 21:42:10.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:42:10.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:42:10.996 | INFO | Total mem usage: 26.5 GiB | 20260507 21:42:10.996 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:42:10.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:42:10.996 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:42:10.996 | INFO | High Water Breakdown: | 20260507 21:42:10.996 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:42:10.996 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:42:10.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:42:10.996 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:42:10.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:42:10.996 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:42:10.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:42:10.996 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:42:10.996 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:42:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:42:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:42:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:42:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:42:10.996 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:42:10.996 | INFO | ------------------------------------------ | 20260507 21:42:10.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:42:10.996 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:42:10.996 | INFO | Total mem usage: 27.8 GiB | 20260507 21:42:10.996 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:42:10.996 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:42:10.996 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:42:10.996 | INFO | High Water Breakdown: | 20260507 21:42:10.996 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:42:10.996 | INFO | ------------------------------------------ | 20260507 21:42:10.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:42:10.996 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:42:10.996 | INFO | Total mem usage: 15.7 GiB | 20260507 21:42:10.996 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:42:10.996 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:42:10.996 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:42:10.996 | INFO | High Water Breakdown: | 20260507 21:42:10.996 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:42:10.996 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:42:10.996 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:42:10.996 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:42:10.996 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:42:10.996 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:42:10.996 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:42:10.996 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:42:10.996 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:42:10.996 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:42:10.996 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:42:10.996 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:42:10.996 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:42:10.996 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:42:10.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:42:10.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:42:10.996 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:42:10.996 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:42:10.996 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:42:10.996 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:42:10.996 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:42:10.996 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:42:10.996 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:42:10.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:42:10.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:42:10.997 | INFO | ------------------------------------------ | 20260507 21:42:10.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:42:10.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:42:10.997 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:42:10.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:42:10.997 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.64e+01, 1.41e+01 / 1.87e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.22e-02 / 4.98e-03 | 20260507 21:42:10.997 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.42e+00, 5.97e-01 / 2.83e-01 | 20260507 21:42:10.997 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.48e+00, 2.10e-01 / 1.20e-01 | 20260507 21:42:10.997 | INFO | PerfReport: 15.73%, DME -- 4.16e+01 / 4.49e+01, 3.77e+00 / 1.40e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.87e+01 / 2.10e+01, 1.87e+00 / 1.94e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.44e+00 / 2.76e+00, 4.29e-01 / 2.09e-01 | 20260507 21:42:10.997 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.36e+00, 2.98e-01 / 1.88e-01 | 20260507 21:42:10.997 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 3.89e-01, 5.99e-02 / 3.43e-02 | 20260507 21:42:10.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:42:10.997 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:42:10.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:42:10.997 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 24.53%, Upload -- 2.62e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 24.14%, Download -- 2.58e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 30.92%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 2.08%, LockUpload -- 2.22e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 2.30%, LockDownload -- 2.46e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 16.02%, LockCompute -- 1.71e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 21:42:10.997 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260507 21:42:11.095 | INFO | Starting a round of Baz IO | 20260507 21:42:14.561 | INFO | Populating RTMetrics report | 20260507 21:42:14.561 | INFO | RTMetrics report populated | 20260507 21:42:14.561 | INFO | Opening tmp RTMetrics json file | 20260507 21:42:14.563 | INFO | Serializing RTMetrics to json | 20260507 21:42:14.581 | INFO | Serialization done, swapping tmp file now... | 20260507 21:42:14.584 | INFO | RTMetrics done | 20260507 21:42:14.671 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.575 seconds | 20260507 21:42:14.671 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.56% / 0.01% / 56.43% | 20260507 21:42:14.671 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60862% / 1.19209e-05% / 6.75519e-05% | 20260507 21:42:14.671 | INFO | Baz IO overall: | 20260507 21:42:14.671 | INFO | BytesWritten : 1279684852301 | 20260507 21:42:14.671 | INFO | NumEvents : 620172407032 | 20260507 21:42:16.141 | INFO | Analyzing chunk frames = [2809856,2810368) | 20260507 21:42:16.141 | INFO | MemInfo: { | 20260507 21:42:16.141 | INFO | "gpu" : " 17 GB", | 20260507 21:42:16.141 | INFO | "huge" : "", | 20260507 21:42:16.141 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:16.141 | INFO | "malloc" : " 14 GB", | 20260507 21:42:16.141 | INFO | "pinned" : " 29 GB" | 20260507 21:42:16.141 | INFO | } | 20260507 21:42:16.141 | INFO | Perf report for the last 5.14577 seconds: | 20260507 21:42:16.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:16.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:16.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:16.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:16.141 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 21:42:16.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:16.141 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 21:42:16.141 | INFO | PRE_HQ : 65.6%, 1.7, 57.5ms, 0 | 20260507 21:42:16.141 | INFO | BAZWRITER : 69.2%, 3.9, 139.4ms, 0 | 20260507 21:42:16.141 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 21:42:16.141 | INFO | Uploaded 36GiB to GPU in 2.50586s (14.3663GiB/s) | 20260507 21:42:16.142 | INFO | Downloaded 38GiB to GPU in 2.70265s (14.0603GiB/s) | 20260507 21:42:21.286 | INFO | Analyzing chunk frames = [2810368,2810880) | 20260507 21:42:21.286 | INFO | MemInfo: { | 20260507 21:42:21.286 | INFO | "gpu" : " 17 GB", | 20260507 21:42:21.286 | INFO | "huge" : "", | 20260507 21:42:21.286 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:21.286 | INFO | "malloc" : " 14 GB", | 20260507 21:42:21.286 | INFO | "pinned" : " 29 GB" | 20260507 21:42:21.286 | INFO | } | 20260507 21:42:21.286 | INFO | Perf report for the last 5.14534 seconds: | 20260507 21:42:21.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:21.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.286 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:42:21.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.286 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 21:42:21.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:21.287 | INFO | Uploaded 36GiB to GPU in 2.53652s (14.1927GiB/s) | 20260507 21:42:21.287 | INFO | Downloaded 34GiB to GPU in 2.46584s (13.7884GiB/s) | 20260507 21:42:26.421 | INFO | Analyzing chunk frames = [2810880,2811392) | 20260507 21:42:26.421 | INFO | MemInfo: { | 20260507 21:42:26.421 | INFO | "gpu" : " 17 GB", | 20260507 21:42:26.421 | INFO | "huge" : "", | 20260507 21:42:26.421 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:26.421 | INFO | "malloc" : " 14 GB", | 20260507 21:42:26.421 | INFO | "pinned" : " 29 GB" | 20260507 21:42:26.421 | INFO | } | 20260507 21:42:26.421 | INFO | Perf report for the last 5.1349 seconds: | 20260507 21:42:26.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:26.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.421 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:42:26.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.421 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:42:26.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:26.422 | INFO | Uploaded 36GiB to GPU in 2.57749s (13.9671GiB/s) | 20260507 21:42:26.422 | INFO | Downloaded 34GiB to GPU in 2.4731s (13.748GiB/s) | 20260507 21:42:31.557 | INFO | Analyzing chunk frames = [2811392,2811904) | 20260507 21:42:31.557 | INFO | MemInfo: { | 20260507 21:42:31.557 | INFO | "gpu" : " 17 GB", | 20260507 21:42:31.557 | INFO | "huge" : "", | 20260507 21:42:31.557 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:31.557 | INFO | "malloc" : " 14 GB", | 20260507 21:42:31.557 | INFO | "pinned" : " 29 GB" | 20260507 21:42:31.557 | INFO | } | 20260507 21:42:31.557 | INFO | Perf report for the last 5.13631 seconds: | 20260507 21:42:31.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:31.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:42:31.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:42:31.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:31.558 | INFO | Uploaded 36GiB to GPU in 2.48509s (14.4864GiB/s) | 20260507 21:42:31.558 | INFO | Downloaded 34GiB to GPU in 2.40731s (14.1237GiB/s) | 20260507 21:42:36.701 | INFO | Analyzing chunk frames = [2811904,2812416) | 20260507 21:42:36.701 | INFO | MemInfo: { | 20260507 21:42:36.701 | INFO | "gpu" : " 17 GB", | 20260507 21:42:36.701 | INFO | "huge" : "", | 20260507 21:42:36.701 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:36.701 | INFO | "malloc" : " 14 GB", | 20260507 21:42:36.701 | INFO | "pinned" : " 29 GB" | 20260507 21:42:36.701 | INFO | } | 20260507 21:42:36.701 | INFO | Perf report for the last 5.14353 seconds: | 20260507 21:42:36.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:36.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 21:42:36.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:42:36.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:36.701 | INFO | Uploaded 36GiB to GPU in 2.35715s (15.2727GiB/s) | 20260507 21:42:36.701 | INFO | Downloaded 34GiB to GPU in 2.49196s (13.6439GiB/s) | 20260507 21:42:41.841 | INFO | Analyzing chunk frames = [2812416,2812928) | 20260507 21:42:41.841 | INFO | MemInfo: { | 20260507 21:42:41.841 | INFO | "gpu" : " 17 GB", | 20260507 21:42:41.841 | INFO | "huge" : "", | 20260507 21:42:41.841 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:41.841 | INFO | "malloc" : " 14 GB", | 20260507 21:42:41.841 | INFO | "pinned" : " 29 GB" | 20260507 21:42:41.841 | INFO | } | 20260507 21:42:41.841 | INFO | Perf report for the last 5.14009 seconds: | 20260507 21:42:41.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:41.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.841 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 21:42:41.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.841 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:42:41.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:41.842 | INFO | Uploaded 36GiB to GPU in 2.49093s (14.4525GiB/s) | 20260507 21:42:41.842 | INFO | Downloaded 34GiB to GPU in 2.41547s (14.0759GiB/s) | 20260507 21:42:46.981 | INFO | Analyzing chunk frames = [2812928,2813440) | 20260507 21:42:46.981 | INFO | MemInfo: { | 20260507 21:42:46.981 | INFO | "gpu" : " 17 GB", | 20260507 21:42:46.981 | INFO | "huge" : "", | 20260507 21:42:46.981 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:46.981 | INFO | "malloc" : " 14 GB", | 20260507 21:42:46.981 | INFO | "pinned" : " 29 GB" | 20260507 21:42:46.981 | INFO | } | 20260507 21:42:46.981 | INFO | Perf report for the last 5.14028 seconds: | 20260507 21:42:46.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:46.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.981 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:42:46.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.981 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:42:46.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:46.982 | INFO | Uploaded 36GiB to GPU in 2.53809s (14.1839GiB/s) | 20260507 21:42:46.982 | INFO | Downloaded 34GiB to GPU in 2.45168s (13.868GiB/s) | 20260507 21:42:52.120 | INFO | Analyzing chunk frames = [2813440,2813952) | 20260507 21:42:52.120 | INFO | MemInfo: { | 20260507 21:42:52.120 | INFO | "gpu" : " 17 GB", | 20260507 21:42:52.120 | INFO | "huge" : "", | 20260507 21:42:52.120 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:52.120 | INFO | "malloc" : " 14 GB", | 20260507 21:42:52.120 | INFO | "pinned" : " 29 GB" | 20260507 21:42:52.120 | INFO | } | 20260507 21:42:52.120 | INFO | Perf report for the last 5.13831 seconds: | 20260507 21:42:52.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:52.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:42:52.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:42:52.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:52.120 | INFO | Uploaded 36GiB to GPU in 2.52351s (14.2658GiB/s) | 20260507 21:42:52.120 | INFO | Downloaded 34GiB to GPU in 2.4381s (13.9453GiB/s) | 20260507 21:42:52.217 | INFO | Starting a round of Baz IO | 20260507 21:42:55.684 | INFO | Populating RTMetrics report | 20260507 21:42:55.684 | INFO | RTMetrics report populated | 20260507 21:42:55.684 | INFO | Opening tmp RTMetrics json file | 20260507 21:42:55.714 | INFO | Serializing RTMetrics to json | 20260507 21:42:55.737 | INFO | Serialization done, swapping tmp file now... | 20260507 21:42:55.779 | INFO | RTMetrics done | 20260507 21:42:55.808 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.591 seconds | 20260507 21:42:55.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.55% / 0.01% / 56.44% | 20260507 21:42:55.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60584% / 7.94729e-06% / 3.97364e-05% | 20260507 21:42:55.808 | INFO | Baz IO overall: | 20260507 21:42:55.808 | INFO | BytesWritten : 1281566787751 | 20260507 21:42:55.808 | INFO | NumEvents : 621081936510 | 20260507 21:42:57.257 | INFO | Analyzing chunk frames = [2813952,2814464) | 20260507 21:42:57.257 | INFO | MemInfo: { | 20260507 21:42:57.257 | INFO | "gpu" : " 17 GB", | 20260507 21:42:57.257 | INFO | "huge" : "", | 20260507 21:42:57.257 | INFO | "huge_shared" : " 29 GB", | 20260507 21:42:57.257 | INFO | "malloc" : " 14 GB", | 20260507 21:42:57.257 | INFO | "pinned" : " 29 GB" | 20260507 21:42:57.257 | INFO | } | 20260507 21:42:57.257 | INFO | Perf report for the last 5.13787 seconds: | 20260507 21:42:57.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:42:57.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:57.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:57.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:57.258 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 21:42:57.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:42:57.258 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 21:42:57.258 | INFO | PRE_HQ : 65.5%, 1.7, 57.5ms, 0 | 20260507 21:42:57.258 | INFO | BAZWRITER : 69.5%, 3.6, 129.3ms, 0 | 20260507 21:42:57.258 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 21:42:57.258 | INFO | Uploaded 36GiB to GPU in 2.49059s (14.4544GiB/s) | 20260507 21:42:57.258 | INFO | Downloaded 38GiB to GPU in 2.70732s (14.036GiB/s) | 20260507 21:43:02.399 | INFO | Analyzing chunk frames = [2814464,2814976) | 20260507 21:43:02.399 | INFO | MemInfo: { | 20260507 21:43:02.399 | INFO | "gpu" : " 17 GB", | 20260507 21:43:02.399 | INFO | "huge" : "", | 20260507 21:43:02.399 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:02.399 | INFO | "malloc" : " 14 GB", | 20260507 21:43:02.399 | INFO | "pinned" : " 29 GB" | 20260507 21:43:02.399 | INFO | } | 20260507 21:43:02.399 | INFO | Perf report for the last 5.14146 seconds: | 20260507 21:43:02.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:02.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.399 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:43:02.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.399 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 21:43:02.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:02.400 | INFO | Uploaded 36GiB to GPU in 2.5221s (14.2738GiB/s) | 20260507 21:43:02.400 | INFO | Downloaded 34GiB to GPU in 2.49181s (13.6447GiB/s) | 20260507 21:43:07.538 | INFO | Analyzing chunk frames = [2814976,2815488) | 20260507 21:43:07.538 | INFO | MemInfo: { | 20260507 21:43:07.538 | INFO | "gpu" : " 17 GB", | 20260507 21:43:07.538 | INFO | "huge" : "", | 20260507 21:43:07.538 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:07.539 | INFO | "malloc" : " 14 GB", | 20260507 21:43:07.539 | INFO | "pinned" : " 29 GB" | 20260507 21:43:07.539 | INFO | } | 20260507 21:43:07.539 | INFO | Perf report for the last 5.13963 seconds: | 20260507 21:43:07.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:07.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:43:07.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 21:43:07.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:07.539 | INFO | Uploaded 36GiB to GPU in 2.52487s (14.2582GiB/s) | 20260507 21:43:07.539 | INFO | Downloaded 34GiB to GPU in 2.49219s (13.6426GiB/s) | 20260507 21:43:12.670 | INFO | Analyzing chunk frames = [2815488,2816000) | 20260507 21:43:12.670 | INFO | MemInfo: { | 20260507 21:43:12.670 | INFO | "gpu" : " 17 GB", | 20260507 21:43:12.670 | INFO | "huge" : "", | 20260507 21:43:12.670 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:12.670 | INFO | "malloc" : " 14 GB", | 20260507 21:43:12.670 | INFO | "pinned" : " 29 GB" | 20260507 21:43:12.670 | INFO | } | 20260507 21:43:12.670 | INFO | Perf report for the last 5.13169 seconds: | 20260507 21:43:12.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:12.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.670 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260507 21:43:12.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.670 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:43:12.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:12.671 | INFO | Uploaded 36GiB to GPU in 2.42275s (14.8592GiB/s) | 20260507 21:43:12.671 | INFO | Downloaded 34GiB to GPU in 2.4017s (14.1567GiB/s) | 20260507 21:43:17.810 | INFO | Analyzing chunk frames = [2816000,2816512) | 20260507 21:43:17.810 | INFO | MemInfo: { | 20260507 21:43:17.810 | INFO | "gpu" : " 17 GB", | 20260507 21:43:17.810 | INFO | "huge" : "", | 20260507 21:43:17.810 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:17.810 | INFO | "malloc" : " 14 GB", | 20260507 21:43:17.810 | INFO | "pinned" : " 29 GB" | 20260507 21:43:17.810 | INFO | } | 20260507 21:43:17.810 | INFO | Perf report for the last 5.1395 seconds: | 20260507 21:43:17.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:17.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:43:17.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:43:17.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:17.810 | INFO | Uploaded 36GiB to GPU in 2.30833s (15.5957GiB/s) | 20260507 21:43:17.810 | INFO | Downloaded 34GiB to GPU in 2.50075s (13.5959GiB/s) | 20260507 21:43:22.949 | INFO | Analyzing chunk frames = [2816512,2817024) | 20260507 21:43:22.949 | INFO | MemInfo: { | 20260507 21:43:22.949 | INFO | "gpu" : " 17 GB", | 20260507 21:43:22.949 | INFO | "huge" : "", | 20260507 21:43:22.949 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:22.949 | INFO | "malloc" : " 14 GB", | 20260507 21:43:22.949 | INFO | "pinned" : " 29 GB" | 20260507 21:43:22.949 | INFO | } | 20260507 21:43:22.949 | INFO | Perf report for the last 5.13908 seconds: | 20260507 21:43:22.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:22.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.949 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:43:22.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.949 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:43:22.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:22.950 | INFO | Uploaded 36GiB to GPU in 2.4848s (14.4881GiB/s) | 20260507 21:43:22.950 | INFO | Downloaded 34GiB to GPU in 2.51811s (13.5022GiB/s) | 20260507 21:43:28.092 | INFO | Analyzing chunk frames = [2817024,2817536) | 20260507 21:43:28.092 | INFO | MemInfo: { | 20260507 21:43:28.092 | INFO | "gpu" : " 17 GB", | 20260507 21:43:28.092 | INFO | "huge" : "", | 20260507 21:43:28.092 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:28.092 | INFO | "malloc" : " 14 GB", | 20260507 21:43:28.092 | INFO | "pinned" : " 29 GB" | 20260507 21:43:28.092 | INFO | } | 20260507 21:43:28.092 | INFO | Perf report for the last 5.14302 seconds: | 20260507 21:43:28.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:28.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:43:28.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:43:28.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:28.092 | INFO | Uploaded 36GiB to GPU in 2.50212s (14.3878GiB/s) | 20260507 21:43:28.093 | INFO | Downloaded 34GiB to GPU in 2.46978s (13.7664GiB/s) | 20260507 21:43:33.238 | INFO | Analyzing chunk frames = [2817536,2818048) | 20260507 21:43:33.238 | INFO | MemInfo: { | 20260507 21:43:33.238 | INFO | "gpu" : " 17 GB", | 20260507 21:43:33.238 | INFO | "huge" : "", | 20260507 21:43:33.238 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:33.238 | INFO | "malloc" : " 14 GB", | 20260507 21:43:33.238 | INFO | "pinned" : " 29 GB" | 20260507 21:43:33.238 | INFO | } | 20260507 21:43:33.239 | INFO | Perf report for the last 5.14652 seconds: | 20260507 21:43:33.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:33.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:43:33.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:43:33.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:33.239 | INFO | Uploaded 36GiB to GPU in 2.50007s (14.3996GiB/s) | 20260507 21:43:33.239 | INFO | Downloaded 34GiB to GPU in 2.44794s (13.8893GiB/s) | 20260507 21:43:33.239 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:43:33.239 | INFO | Total mem usage: 16.7 GiB | 20260507 21:43:33.239 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:43:33.239 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.8%) | 20260507 21:43:33.239 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:43:33.239 | INFO | High Water Breakdown: | 20260507 21:43:33.239 | INFO | 60.2% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:43:33.239 | INFO | 39.7% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:43:33.239 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:43:33.239 | INFO | ------------------------------------------ | 20260507 21:43:33.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:43:33.239 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:43:33.239 | INFO | Total mem usage: 26.5 GiB | 20260507 21:43:33.239 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:43:33.239 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:43:33.239 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:43:33.239 | INFO | High Water Breakdown: | 20260507 21:43:33.239 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:43:33.239 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:43:33.239 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:43:33.239 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:43:33.239 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:43:33.239 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:43:33.239 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:43:33.239 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:43:33.239 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:43:33.239 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:43:33.239 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:43:33.239 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:43:33.239 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:43:33.239 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:43:33.240 | INFO | ------------------------------------------ | 20260507 21:43:33.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:43:33.240 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:43:33.240 | INFO | Total mem usage: 27.8 GiB | 20260507 21:43:33.240 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:43:33.240 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:43:33.240 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:43:33.240 | INFO | High Water Breakdown: | 20260507 21:43:33.240 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:43:33.240 | INFO | ------------------------------------------ | 20260507 21:43:33.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:43:33.240 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:43:33.240 | INFO | Total mem usage: 15.7 GiB | 20260507 21:43:33.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:43:33.240 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:43:33.240 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:43:33.240 | INFO | High Water Breakdown: | 20260507 21:43:33.240 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:43:33.240 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:43:33.240 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:43:33.240 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:43:33.240 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:43:33.240 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:43:33.240 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:43:33.240 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:43:33.240 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:43:33.240 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:43:33.240 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:43:33.240 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:43:33.240 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:43:33.240 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:43:33.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:43:33.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:43:33.240 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:43:33.240 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:43:33.240 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:43:33.240 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:43:33.240 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:43:33.240 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:43:33.240 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:43:33.240 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:43:33.240 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:43:33.240 | INFO | ------------------------------------------ | 20260507 21:43:33.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:43:33.240 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:43:33.240 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:43:33.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:43:33.240 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.42e+01 / 1.70e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.17e-02 / 7.83e-03 | 20260507 21:43:33.240 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.48e+00, 5.97e-01 / 2.16e-01 | 20260507 21:43:33.240 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.05e-01 | 20260507 21:43:33.240 | INFO | PerfReport: 15.81%, DME -- 4.17e+01 / 4.52e+01, 3.76e+00 / 1.57e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.77e+00 / 1.91e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.45e+00 / 2.74e+00, 4.27e-01 / 2.21e-01 | 20260507 21:43:33.240 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.36e+00, 2.90e-01 / 1.75e-01 | 20260507 21:43:33.240 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 3.86e-01, 5.91e-02 / 2.95e-02 | 20260507 21:43:33.240 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:43:33.240 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:43:33.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:43:33.240 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 24.26%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 24.32%, Download -- 2.60e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 30.91%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 1.80%, LockUpload -- 1.92e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 2.70%, LockDownload -- 2.88e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 15.99%, LockCompute -- 1.71e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 21:43:33.240 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260507 21:43:33.335 | INFO | Starting a round of Baz IO | 20260507 21:43:36.797 | INFO | Populating RTMetrics report | 20260507 21:43:36.798 | INFO | RTMetrics report populated | 20260507 21:43:36.798 | INFO | Opening tmp RTMetrics json file | 20260507 21:43:36.827 | INFO | Serializing RTMetrics to json | 20260507 21:43:36.857 | INFO | Serialization done, swapping tmp file now... | 20260507 21:43:36.887 | INFO | RTMetrics done | 20260507 21:43:36.918 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.582 seconds | 20260507 21:43:36.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.54% / 0.01% / 56.45% | 20260507 21:43:36.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59998% / 1.19209e-05% / 4.76837e-05% | 20260507 21:43:36.918 | INFO | Baz IO overall: | 20260507 21:43:36.918 | INFO | BytesWritten : 1283448283941 | 20260507 21:43:36.918 | INFO | NumEvents : 621990872455 | 20260507 21:43:38.376 | INFO | Analyzing chunk frames = [2818048,2818560) | 20260507 21:43:38.376 | INFO | MemInfo: { | 20260507 21:43:38.376 | INFO | "gpu" : " 17 GB", | 20260507 21:43:38.376 | INFO | "huge" : "", | 20260507 21:43:38.376 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:38.376 | INFO | "malloc" : " 14 GB", | 20260507 21:43:38.376 | INFO | "pinned" : " 29 GB" | 20260507 21:43:38.376 | INFO | } | 20260507 21:43:38.376 | INFO | Perf report for the last 5.13708 seconds: | 20260507 21:43:38.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:38.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:38.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:38.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:38.376 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260507 21:43:38.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:38.376 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 21:43:38.376 | INFO | PRE_HQ : 65.6%, 1.7, 57.6ms, 0 | 20260507 21:43:38.376 | INFO | BAZWRITER : 69.3%, 3.6, 128.3ms, 0 | 20260507 21:43:38.376 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 21:43:38.376 | INFO | Uploaded 36GiB to GPU in 2.50251s (14.3856GiB/s) | 20260507 21:43:38.376 | INFO | Downloaded 38GiB to GPU in 2.6883s (14.1353GiB/s) | 20260507 21:43:43.510 | INFO | Analyzing chunk frames = [2818560,2819072) | 20260507 21:43:43.510 | INFO | MemInfo: { | 20260507 21:43:43.510 | INFO | "gpu" : " 17 GB", | 20260507 21:43:43.510 | INFO | "huge" : "", | 20260507 21:43:43.510 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:43.510 | INFO | "malloc" : " 14 GB", | 20260507 21:43:43.510 | INFO | "pinned" : " 29 GB" | 20260507 21:43:43.510 | INFO | } | 20260507 21:43:43.510 | INFO | Perf report for the last 5.13447 seconds: | 20260507 21:43:43.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:43.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 21:43:43.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 21:43:43.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:43.510 | INFO | Uploaded 36GiB to GPU in 2.57926s (13.9575GiB/s) | 20260507 21:43:43.510 | INFO | Downloaded 34GiB to GPU in 2.4661s (13.7869GiB/s) | 20260507 21:43:48.653 | INFO | Analyzing chunk frames = [2819072,2819584) | 20260507 21:43:48.653 | INFO | MemInfo: { | 20260507 21:43:48.653 | INFO | "gpu" : " 17 GB", | 20260507 21:43:48.653 | INFO | "huge" : "", | 20260507 21:43:48.653 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:48.653 | INFO | "malloc" : " 14 GB", | 20260507 21:43:48.653 | INFO | "pinned" : " 29 GB" | 20260507 21:43:48.653 | INFO | } | 20260507 21:43:48.653 | INFO | Perf report for the last 5.14289 seconds: | 20260507 21:43:48.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:48.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.653 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260507 21:43:48.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.653 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:43:48.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:48.654 | INFO | Uploaded 36GiB to GPU in 2.5757s (13.9768GiB/s) | 20260507 21:43:48.654 | INFO | Downloaded 34GiB to GPU in 2.4648s (13.7942GiB/s) | 20260507 21:43:53.790 | INFO | Analyzing chunk frames = [2819584,2820096) | 20260507 21:43:53.790 | INFO | MemInfo: { | 20260507 21:43:53.790 | INFO | "gpu" : " 17 GB", | 20260507 21:43:53.790 | INFO | "huge" : "", | 20260507 21:43:53.790 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:53.790 | INFO | "malloc" : " 14 GB", | 20260507 21:43:53.790 | INFO | "pinned" : " 29 GB" | 20260507 21:43:53.790 | INFO | } | 20260507 21:43:53.790 | INFO | Perf report for the last 5.13724 seconds: | 20260507 21:43:53.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:53.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.790 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:43:53.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.790 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:43:53.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:53.791 | INFO | Uploaded 36GiB to GPU in 2.48514s (14.4861GiB/s) | 20260507 21:43:53.791 | INFO | Downloaded 34GiB to GPU in 2.39374s (14.2037GiB/s) | 20260507 21:43:58.929 | INFO | Analyzing chunk frames = [2820096,2820608) | 20260507 21:43:58.929 | INFO | MemInfo: { | 20260507 21:43:58.929 | INFO | "gpu" : " 17 GB", | 20260507 21:43:58.929 | INFO | "huge" : "", | 20260507 21:43:58.929 | INFO | "huge_shared" : " 29 GB", | 20260507 21:43:58.929 | INFO | "malloc" : " 14 GB", | 20260507 21:43:58.929 | INFO | "pinned" : " 29 GB" | 20260507 21:43:58.929 | INFO | } | 20260507 21:43:58.930 | INFO | Perf report for the last 5.13942 seconds: | 20260507 21:43:58.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:43:58.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:43:58.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:43:58.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:43:58.930 | INFO | Uploaded 36GiB to GPU in 2.35716s (15.2726GiB/s) | 20260507 21:43:58.930 | INFO | Downloaded 34GiB to GPU in 2.48172s (13.7002GiB/s) | 20260507 21:44:04.065 | INFO | Analyzing chunk frames = [2820608,2821120) | 20260507 21:44:04.065 | INFO | MemInfo: { | 20260507 21:44:04.065 | INFO | "gpu" : " 17 GB", | 20260507 21:44:04.065 | INFO | "huge" : "", | 20260507 21:44:04.065 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:04.065 | INFO | "malloc" : " 14 GB", | 20260507 21:44:04.065 | INFO | "pinned" : " 29 GB" | 20260507 21:44:04.065 | INFO | } | 20260507 21:44:04.065 | INFO | Perf report for the last 5.13557 seconds: | 20260507 21:44:04.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:04.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.065 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:44:04.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.065 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:44:04.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:04.066 | INFO | Uploaded 36GiB to GPU in 2.46983s (14.5759GiB/s) | 20260507 21:44:04.066 | INFO | Downloaded 34GiB to GPU in 2.42015s (14.0487GiB/s) | 20260507 21:44:09.209 | INFO | Analyzing chunk frames = [2821120,2821632) | 20260507 21:44:09.209 | INFO | MemInfo: { | 20260507 21:44:09.209 | INFO | "gpu" : " 17 GB", | 20260507 21:44:09.209 | INFO | "huge" : "", | 20260507 21:44:09.209 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:09.210 | INFO | "malloc" : " 14 GB", | 20260507 21:44:09.210 | INFO | "pinned" : " 29 GB" | 20260507 21:44:09.210 | INFO | } | 20260507 21:44:09.210 | INFO | Perf report for the last 5.14443 seconds: | 20260507 21:44:09.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:09.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:44:09.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:44:09.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:09.210 | INFO | Uploaded 36GiB to GPU in 2.50713s (14.3591GiB/s) | 20260507 21:44:09.210 | INFO | Downloaded 34GiB to GPU in 2.45038s (13.8754GiB/s) | 20260507 21:44:14.349 | INFO | Analyzing chunk frames = [2821632,2822144) | 20260507 21:44:14.350 | INFO | MemInfo: { | 20260507 21:44:14.350 | INFO | "gpu" : " 17 GB", | 20260507 21:44:14.350 | INFO | "huge" : "", | 20260507 21:44:14.350 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:14.350 | INFO | "malloc" : " 14 GB", | 20260507 21:44:14.350 | INFO | "pinned" : " 29 GB" | 20260507 21:44:14.350 | INFO | } | 20260507 21:44:14.350 | INFO | Perf report for the last 5.14005 seconds: | 20260507 21:44:14.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:14.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:44:14.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:44:14.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:14.350 | INFO | Uploaded 36GiB to GPU in 2.50169s (14.3903GiB/s) | 20260507 21:44:14.350 | INFO | Downloaded 34GiB to GPU in 2.43691s (13.9521GiB/s) | 20260507 21:44:14.456 | INFO | Starting a round of Baz IO | 20260507 21:44:17.918 | INFO | Populating RTMetrics report | 20260507 21:44:17.918 | INFO | RTMetrics report populated | 20260507 21:44:17.918 | INFO | Opening tmp RTMetrics json file | 20260507 21:44:17.938 | INFO | Serializing RTMetrics to json | 20260507 21:44:17.943 | INFO | Serialization done, swapping tmp file now... | 20260507 21:44:17.944 | INFO | RTMetrics done | 20260507 21:44:18.025 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.569 seconds | 20260507 21:44:18.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.54% / 0.01% / 56.46% | 20260507 21:44:18.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60135% / 1.58946e-05% / 6.35783e-05% | 20260507 21:44:18.025 | INFO | Baz IO overall: | 20260507 21:44:18.025 | INFO | BytesWritten : 1285331233244 | 20260507 21:44:18.025 | INFO | NumEvents : 622901351953 | 20260507 21:44:19.489 | INFO | Analyzing chunk frames = [2822144,2822656) | 20260507 21:44:19.489 | INFO | MemInfo: { | 20260507 21:44:19.489 | INFO | "gpu" : " 17 GB", | 20260507 21:44:19.489 | INFO | "huge" : "", | 20260507 21:44:19.489 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:19.490 | INFO | "malloc" : " 14 GB", | 20260507 21:44:19.490 | INFO | "pinned" : " 29 GB" | 20260507 21:44:19.490 | INFO | } | 20260507 21:44:19.490 | INFO | Perf report for the last 5.13988 seconds: | 20260507 21:44:19.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:19.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:19.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:19.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:19.490 | INFO | ANALYSIS : 68.7%, 2.9, 106.1ms, 0 | 20260507 21:44:19.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:19.490 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260507 21:44:19.490 | INFO | PRE_HQ : 66.1%, 1.7, 59.9ms, 0 | 20260507 21:44:19.490 | INFO | BAZWRITER : 69.0%, 3.7, 130.8ms, 0 | 20260507 21:44:19.490 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 21:44:19.490 | INFO | Uploaded 36GiB to GPU in 2.52131s (14.2783GiB/s) | 20260507 21:44:19.490 | INFO | Downloaded 38GiB to GPU in 2.68311s (14.1627GiB/s) | 20260507 21:44:24.622 | INFO | Analyzing chunk frames = [2822656,2823168) | 20260507 21:44:24.622 | INFO | MemInfo: { | 20260507 21:44:24.622 | INFO | "gpu" : " 17 GB", | 20260507 21:44:24.622 | INFO | "huge" : "", | 20260507 21:44:24.622 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:24.622 | INFO | "malloc" : " 14 GB", | 20260507 21:44:24.622 | INFO | "pinned" : " 29 GB" | 20260507 21:44:24.622 | INFO | } | 20260507 21:44:24.622 | INFO | Perf report for the last 5.13275 seconds: | 20260507 21:44:24.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:24.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.623 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 21:44:24.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.623 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:44:24.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:24.623 | INFO | Uploaded 36GiB to GPU in 2.52376s (14.2644GiB/s) | 20260507 21:44:24.623 | INFO | Downloaded 34GiB to GPU in 2.49705s (13.6161GiB/s) | 20260507 21:44:29.763 | INFO | Analyzing chunk frames = [2823168,2823680) | 20260507 21:44:29.763 | INFO | MemInfo: { | 20260507 21:44:29.763 | INFO | "gpu" : " 17 GB", | 20260507 21:44:29.763 | INFO | "huge" : "", | 20260507 21:44:29.763 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:29.763 | INFO | "malloc" : " 14 GB", | 20260507 21:44:29.763 | INFO | "pinned" : " 29 GB" | 20260507 21:44:29.763 | INFO | } | 20260507 21:44:29.763 | INFO | Perf report for the last 5.14058 seconds: | 20260507 21:44:29.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:29.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 21:44:29.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:44:29.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:29.763 | INFO | Uploaded 36GiB to GPU in 2.48679s (14.4765GiB/s) | 20260507 21:44:29.763 | INFO | Downloaded 34GiB to GPU in 2.46825s (13.775GiB/s) | 20260507 21:44:34.908 | INFO | Analyzing chunk frames = [2823680,2824192) | 20260507 21:44:34.908 | INFO | MemInfo: { | 20260507 21:44:34.908 | INFO | "gpu" : " 17 GB", | 20260507 21:44:34.908 | INFO | "huge" : "", | 20260507 21:44:34.908 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:34.908 | INFO | "malloc" : " 14 GB", | 20260507 21:44:34.908 | INFO | "pinned" : " 29 GB" | 20260507 21:44:34.908 | INFO | } | 20260507 21:44:34.908 | INFO | Perf report for the last 5.14535 seconds: | 20260507 21:44:34.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:34.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.908 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 21:44:34.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.909 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:44:34.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:34.909 | INFO | Uploaded 36GiB to GPU in 2.45801s (14.646GiB/s) | 20260507 21:44:34.909 | INFO | Downloaded 34GiB to GPU in 2.38759s (14.2403GiB/s) | 20260507 21:44:40.051 | INFO | Analyzing chunk frames = [2824192,2824704) | 20260507 21:44:40.051 | INFO | MemInfo: { | 20260507 21:44:40.051 | INFO | "gpu" : " 17 GB", | 20260507 21:44:40.051 | INFO | "huge" : "", | 20260507 21:44:40.051 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:40.051 | INFO | "malloc" : " 14 GB", | 20260507 21:44:40.051 | INFO | "pinned" : " 29 GB" | 20260507 21:44:40.051 | INFO | } | 20260507 21:44:40.051 | INFO | Perf report for the last 5.14266 seconds: | 20260507 21:44:40.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:40.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 21:44:40.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:44:40.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:40.051 | INFO | Uploaded 36GiB to GPU in 2.34249s (15.3683GiB/s) | 20260507 21:44:40.051 | INFO | Downloaded 34GiB to GPU in 2.50711s (13.5614GiB/s) | 20260507 21:44:45.186 | INFO | Analyzing chunk frames = [2824704,2825216) | 20260507 21:44:45.186 | INFO | MemInfo: { | 20260507 21:44:45.186 | INFO | "gpu" : " 17 GB", | 20260507 21:44:45.186 | INFO | "huge" : "", | 20260507 21:44:45.187 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:45.187 | INFO | "malloc" : " 14 GB", | 20260507 21:44:45.187 | INFO | "pinned" : " 29 GB" | 20260507 21:44:45.187 | INFO | } | 20260507 21:44:45.187 | INFO | Perf report for the last 5.13559 seconds: | 20260507 21:44:45.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:45.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 21:44:45.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:44:45.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:45.187 | INFO | Uploaded 36GiB to GPU in 2.35392s (15.2936GiB/s) | 20260507 21:44:45.187 | INFO | Downloaded 34GiB to GPU in 2.41091s (14.1026GiB/s) | 20260507 21:44:50.323 | INFO | Analyzing chunk frames = [2825216,2825728) | 20260507 21:44:50.323 | INFO | MemInfo: { | 20260507 21:44:50.323 | INFO | "gpu" : " 17 GB", | 20260507 21:44:50.323 | INFO | "huge" : "", | 20260507 21:44:50.323 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:50.323 | INFO | "malloc" : " 14 GB", | 20260507 21:44:50.323 | INFO | "pinned" : " 29 GB" | 20260507 21:44:50.323 | INFO | } | 20260507 21:44:50.323 | INFO | Perf report for the last 5.13661 seconds: | 20260507 21:44:50.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:50.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.323 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:44:50.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.323 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 21:44:50.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:50.324 | INFO | Uploaded 36GiB to GPU in 2.52016s (14.2848GiB/s) | 20260507 21:44:50.324 | INFO | Downloaded 34GiB to GPU in 2.47317s (13.7475GiB/s) | 20260507 21:44:55.464 | INFO | Analyzing chunk frames = [2825728,2826240) | 20260507 21:44:55.464 | INFO | MemInfo: { | 20260507 21:44:55.464 | INFO | "gpu" : " 17 GB", | 20260507 21:44:55.464 | INFO | "huge" : "", | 20260507 21:44:55.464 | INFO | "huge_shared" : " 29 GB", | 20260507 21:44:55.464 | INFO | "malloc" : " 14 GB", | 20260507 21:44:55.464 | INFO | "pinned" : " 29 GB" | 20260507 21:44:55.464 | INFO | } | 20260507 21:44:55.464 | INFO | Perf report for the last 5.14089 seconds: | 20260507 21:44:55.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:44:55.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.464 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:44:55.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.464 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 21:44:55.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:44:55.465 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:44:55.465 | INFO | Total mem usage: 16.7 GiB | 20260507 21:44:55.465 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:44:55.465 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260507 21:44:55.465 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:44:55.465 | INFO | High Water Breakdown: | 20260507 21:44:55.465 | INFO | 59.8% : 8.78 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:44:55.465 | INFO | 40.1% : 5.88 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:44:55.465 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:44:55.465 | INFO | ------------------------------------------ | 20260507 21:44:55.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:44:55.465 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:44:55.465 | INFO | Total mem usage: 26.5 GiB | 20260507 21:44:55.465 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:44:55.465 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:44:55.465 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:44:55.465 | INFO | High Water Breakdown: | 20260507 21:44:55.465 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:44:55.465 | INFO | Uploaded 36GiB to GPU in 2.46179s (14.6235GiB/s) | 20260507 21:44:55.465 | INFO | Downloaded 34GiB to GPU in 2.45873s (13.8283GiB/s) | 20260507 21:44:55.465 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:44:55.465 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:44:55.465 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:44:55.465 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:44:55.465 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:44:55.465 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:44:55.465 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:44:55.465 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:44:55.465 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:44:55.465 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:44:55.465 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:44:55.465 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:44:55.465 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:44:55.465 | INFO | ------------------------------------------ | 20260507 21:44:55.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:44:55.465 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:44:55.465 | INFO | Total mem usage: 27.8 GiB | 20260507 21:44:55.465 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:44:55.465 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:44:55.465 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:44:55.465 | INFO | High Water Breakdown: | 20260507 21:44:55.465 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:44:55.465 | INFO | ------------------------------------------ | 20260507 21:44:55.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:44:55.465 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:44:55.465 | INFO | Total mem usage: 15.7 GiB | 20260507 21:44:55.465 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:44:55.465 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:44:55.465 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:44:55.466 | INFO | High Water Breakdown: | 20260507 21:44:55.466 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:44:55.466 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:44:55.466 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:44:55.466 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:44:55.466 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:44:55.466 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:44:55.466 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:44:55.466 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:44:55.466 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:44:55.466 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:44:55.466 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:44:55.466 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:44:55.466 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:44:55.466 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:44:55.466 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:44:55.466 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:44:55.466 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:44:55.466 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:44:55.466 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:44:55.466 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:44:55.466 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:44:55.466 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:44:55.466 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:44:55.466 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:44:55.466 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:44:55.466 | INFO | ------------------------------------------ | 20260507 21:44:55.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:44:55.466 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:44:55.466 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:44:55.466 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:44:55.466 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.43e+01 / 1.69e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.04e-01, 1.17e-02 / 5.07e-03 | 20260507 21:44:55.466 | INFO | PerfReport: 12.15%, Baseline -- 4.02e+00 / 4.48e+00, 6.04e-01 / 2.08e-01 | 20260507 21:44:55.466 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.07e-01 | 20260507 21:44:55.466 | INFO | PerfReport: 15.79%, DME -- 4.17e+01 / 4.51e+01, 3.80e+00 / 1.41e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.81e+00 / 1.97e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.32e-01 / 2.61e-01 | 20260507 21:44:55.466 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.38e+00, 2.97e-01 / 1.96e-01 | 20260507 21:44:55.466 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 3.93e-01, 5.93e-02 / 3.20e-02 | 20260507 21:44:55.466 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:44:55.466 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:44:55.466 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:44:55.466 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 24.20%, Upload -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 30.99%, Compute -- 3.31e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 1.84%, LockUpload -- 1.96e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 2.61%, LockDownload -- 2.79e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 16.12%, LockCompute -- 1.72e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 21:44:55.466 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260507 21:44:55.577 | INFO | Starting a round of Baz IO | 20260507 21:44:59.057 | INFO | Populating RTMetrics report | 20260507 21:44:59.057 | INFO | RTMetrics report populated | 20260507 21:44:59.057 | INFO | Opening tmp RTMetrics json file | 20260507 21:44:59.068 | INFO | Serializing RTMetrics to json | 20260507 21:44:59.083 | INFO | Serialization done, swapping tmp file now... | 20260507 21:44:59.084 | INFO | RTMetrics done | 20260507 21:44:59.173 | INFO | Finished a round of Baz IO: Wrote 1.754 GiB in 3.596 seconds | 20260507 21:44:59.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.53% / 0.01% / 56.46% | 20260507 21:44:59.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59963% / 1.19209e-05% / 7.54992e-05% | 20260507 21:44:59.173 | INFO | Baz IO overall: | 20260507 21:44:59.173 | INFO | BytesWritten : 1287214756460 | 20260507 21:44:59.173 | INFO | NumEvents : 623812297655 | 20260507 21:45:00.602 | INFO | Analyzing chunk frames = [2826240,2826752) | 20260507 21:45:00.602 | INFO | MemInfo: { | 20260507 21:45:00.602 | INFO | "gpu" : " 17 GB", | 20260507 21:45:00.602 | INFO | "huge" : "", | 20260507 21:45:00.602 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:00.602 | INFO | "malloc" : " 14 GB", | 20260507 21:45:00.602 | INFO | "pinned" : " 29 GB" | 20260507 21:45:00.602 | INFO | } | 20260507 21:45:00.602 | INFO | Perf report for the last 5.13769 seconds: | 20260507 21:45:00.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:00.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:00.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:00.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:00.602 | INFO | ANALYSIS : 69.2%, 2.9, 104.6ms, 0 | 20260507 21:45:00.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:00.602 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 21:45:00.602 | INFO | PRE_HQ : 68.4%, 1.8, 64.8ms, 0 | 20260507 21:45:00.602 | INFO | BAZWRITER : 69.6%, 4.1, 147.3ms, 0 | 20260507 21:45:00.602 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 21:45:00.603 | INFO | Uploaded 36GiB to GPU in 2.58459s (13.9287GiB/s) | 20260507 21:45:00.603 | INFO | Downloaded 38GiB to GPU in 2.72957s (13.9216GiB/s) | 20260507 21:45:05.741 | INFO | Analyzing chunk frames = [2826752,2827264) | 20260507 21:45:05.741 | INFO | MemInfo: { | 20260507 21:45:05.741 | INFO | "gpu" : " 17 GB", | 20260507 21:45:05.741 | INFO | "huge" : "", | 20260507 21:45:05.741 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:05.741 | INFO | "malloc" : " 14 GB", | 20260507 21:45:05.741 | INFO | "pinned" : " 29 GB" | 20260507 21:45:05.741 | INFO | } | 20260507 21:45:05.741 | INFO | Perf report for the last 5.13953 seconds: | 20260507 21:45:05.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:05.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 21:45:05.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:45:05.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:05.742 | INFO | Uploaded 36GiB to GPU in 2.53145s (14.2211GiB/s) | 20260507 21:45:05.742 | INFO | Downloaded 34GiB to GPU in 2.47863s (13.7173GiB/s) | 20260507 21:45:10.879 | INFO | Analyzing chunk frames = [2827264,2827776) | 20260507 21:45:10.879 | INFO | MemInfo: { | 20260507 21:45:10.879 | INFO | "gpu" : " 17 GB", | 20260507 21:45:10.879 | INFO | "huge" : "", | 20260507 21:45:10.879 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:10.879 | INFO | "malloc" : " 14 GB", | 20260507 21:45:10.879 | INFO | "pinned" : " 29 GB" | 20260507 21:45:10.879 | INFO | } | 20260507 21:45:10.879 | INFO | Perf report for the last 5.13791 seconds: | 20260507 21:45:10.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:10.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.879 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:45:10.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.879 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:45:10.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:10.880 | INFO | Uploaded 36GiB to GPU in 2.53861s (14.181GiB/s) | 20260507 21:45:10.880 | INFO | Downloaded 34GiB to GPU in 2.45649s (13.8409GiB/s) | 20260507 21:45:16.020 | INFO | Analyzing chunk frames = [2827776,2828288) | 20260507 21:45:16.020 | INFO | MemInfo: { | 20260507 21:45:16.020 | INFO | "gpu" : " 17 GB", | 20260507 21:45:16.020 | INFO | "huge" : "", | 20260507 21:45:16.020 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:16.020 | INFO | "malloc" : " 14 GB", | 20260507 21:45:16.020 | INFO | "pinned" : " 29 GB" | 20260507 21:45:16.020 | INFO | } | 20260507 21:45:16.020 | INFO | Perf report for the last 5.14036 seconds: | 20260507 21:45:16.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:16.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 21:45:16.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 21:45:16.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:16.020 | INFO | Uploaded 36GiB to GPU in 2.50281s (14.3838GiB/s) | 20260507 21:45:16.020 | INFO | Downloaded 34GiB to GPU in 2.37561s (14.3121GiB/s) | 20260507 21:45:21.160 | INFO | Analyzing chunk frames = [2828288,2828800) | 20260507 21:45:21.160 | INFO | MemInfo: { | 20260507 21:45:21.160 | INFO | "gpu" : " 17 GB", | 20260507 21:45:21.160 | INFO | "huge" : "", | 20260507 21:45:21.160 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:21.160 | INFO | "malloc" : " 14 GB", | 20260507 21:45:21.160 | INFO | "pinned" : " 29 GB" | 20260507 21:45:21.160 | INFO | } | 20260507 21:45:21.161 | INFO | Perf report for the last 5.1408 seconds: | 20260507 21:45:21.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:21.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:45:21.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 21:45:21.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:21.161 | INFO | Uploaded 36GiB to GPU in 2.37988s (15.1268GiB/s) | 20260507 21:45:21.161 | INFO | Downloaded 34GiB to GPU in 2.48547s (13.6795GiB/s) | 20260507 21:45:26.303 | INFO | Analyzing chunk frames = [2828800,2829312) | 20260507 21:45:26.303 | INFO | MemInfo: { | 20260507 21:45:26.303 | INFO | "gpu" : " 17 GB", | 20260507 21:45:26.303 | INFO | "huge" : "", | 20260507 21:45:26.303 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:26.303 | INFO | "malloc" : " 14 GB", | 20260507 21:45:26.303 | INFO | "pinned" : " 29 GB" | 20260507 21:45:26.303 | INFO | } | 20260507 21:45:26.304 | INFO | Perf report for the last 5.14293 seconds: | 20260507 21:45:26.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:26.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 21:45:26.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:45:26.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:26.304 | INFO | Uploaded 36GiB to GPU in 2.53952s (14.1759GiB/s) | 20260507 21:45:26.304 | INFO | Downloaded 34GiB to GPU in 2.42144s (14.0412GiB/s) | 20260507 21:45:31.443 | INFO | Analyzing chunk frames = [2829312,2829824) | 20260507 21:45:31.443 | INFO | MemInfo: { | 20260507 21:45:31.443 | INFO | "gpu" : " 17 GB", | 20260507 21:45:31.443 | INFO | "huge" : "", | 20260507 21:45:31.443 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:31.443 | INFO | "malloc" : " 14 GB", | 20260507 21:45:31.443 | INFO | "pinned" : " 29 GB" | 20260507 21:45:31.443 | INFO | } | 20260507 21:45:31.443 | INFO | Perf report for the last 5.1397 seconds: | 20260507 21:45:31.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:31.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.443 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:45:31.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.443 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 21:45:31.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:31.444 | INFO | Uploaded 36GiB to GPU in 2.55289s (14.1017GiB/s) | 20260507 21:45:31.444 | INFO | Downloaded 34GiB to GPU in 2.44513s (13.9052GiB/s) | 20260507 21:45:36.575 | INFO | Analyzing chunk frames = [2829824,2830336) | 20260507 21:45:36.575 | INFO | MemInfo: { | 20260507 21:45:36.575 | INFO | "gpu" : " 17 GB", | 20260507 21:45:36.575 | INFO | "huge" : "", | 20260507 21:45:36.575 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:36.575 | INFO | "malloc" : " 14 GB", | 20260507 21:45:36.575 | INFO | "pinned" : " 29 GB" | 20260507 21:45:36.575 | INFO | } | 20260507 21:45:36.575 | INFO | Perf report for the last 5.13186 seconds: | 20260507 21:45:36.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:36.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.575 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 21:45:36.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.575 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:45:36.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:36.576 | INFO | Uploaded 36GiB to GPU in 2.51613s (14.3077GiB/s) | 20260507 21:45:36.576 | INFO | Downloaded 34GiB to GPU in 2.43392s (13.9693GiB/s) | 20260507 21:45:36.673 | INFO | Starting a round of Baz IO | 20260507 21:45:40.149 | INFO | Populating RTMetrics report | 20260507 21:45:40.149 | INFO | RTMetrics report populated | 20260507 21:45:40.149 | INFO | Opening tmp RTMetrics json file | 20260507 21:45:40.230 | INFO | Serializing RTMetrics to json | 20260507 21:45:40.243 | INFO | Serialization done, swapping tmp file now... | 20260507 21:45:40.244 | INFO | RTMetrics done | 20260507 21:45:40.272 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.599 seconds | 20260507 21:45:40.272 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.52% / 0.01% / 56.47% | 20260507 21:45:40.272 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59268% / 3.97364e-06% / 7.15256e-05% | 20260507 21:45:40.273 | INFO | Baz IO overall: | 20260507 21:45:40.273 | INFO | BytesWritten : 1289096469705 | 20260507 21:45:40.273 | INFO | NumEvents : 624721848745 | 20260507 21:45:41.717 | INFO | Analyzing chunk frames = [2830336,2830848) | 20260507 21:45:41.717 | INFO | MemInfo: { | 20260507 21:45:41.717 | INFO | "gpu" : " 17 GB", | 20260507 21:45:41.717 | INFO | "huge" : "", | 20260507 21:45:41.718 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:41.718 | INFO | "malloc" : " 14 GB", | 20260507 21:45:41.718 | INFO | "pinned" : " 29 GB" | 20260507 21:45:41.718 | INFO | } | 20260507 21:45:41.718 | INFO | Perf report for the last 5.14258 seconds: | 20260507 21:45:41.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:41.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:41.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:41.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:41.718 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 21:45:41.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:41.718 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 21:45:41.718 | INFO | PRE_HQ : 65.8%, 1.7, 57.5ms, 0 | 20260507 21:45:41.718 | INFO | BAZWRITER : 69.6%, 3.9, 140.1ms, 0 | 20260507 21:45:41.718 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 21:45:41.718 | INFO | Uploaded 36GiB to GPU in 2.54619s (14.1388GiB/s) | 20260507 21:45:41.718 | INFO | Downloaded 38GiB to GPU in 2.70746s (14.0353GiB/s) | 20260507 21:45:46.853 | INFO | Analyzing chunk frames = [2830848,2831360) | 20260507 21:45:46.853 | INFO | MemInfo: { | 20260507 21:45:46.853 | INFO | "gpu" : " 17 GB", | 20260507 21:45:46.853 | INFO | "huge" : "", | 20260507 21:45:46.853 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:46.853 | INFO | "malloc" : " 14 GB", | 20260507 21:45:46.854 | INFO | "pinned" : " 29 GB" | 20260507 21:45:46.854 | INFO | } | 20260507 21:45:46.854 | INFO | Perf report for the last 5.13591 seconds: | 20260507 21:45:46.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:46.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 21:45:46.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:45:46.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:46.854 | INFO | Uploaded 36GiB to GPU in 2.55102s (14.112GiB/s) | 20260507 21:45:46.854 | INFO | Downloaded 34GiB to GPU in 2.46861s (13.773GiB/s) | 20260507 21:45:51.995 | INFO | Analyzing chunk frames = [2831360,2831872) | 20260507 21:45:51.995 | INFO | MemInfo: { | 20260507 21:45:51.995 | INFO | "gpu" : " 17 GB", | 20260507 21:45:51.995 | INFO | "huge" : "", | 20260507 21:45:51.995 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:51.995 | INFO | "malloc" : " 14 GB", | 20260507 21:45:51.995 | INFO | "pinned" : " 29 GB" | 20260507 21:45:51.995 | INFO | } | 20260507 21:45:51.995 | INFO | Perf report for the last 5.1416 seconds: | 20260507 21:45:51.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:51.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.995 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:45:51.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.995 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:45:51.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:51.996 | INFO | Uploaded 36GiB to GPU in 2.55395s (14.0958GiB/s) | 20260507 21:45:51.996 | INFO | Downloaded 34GiB to GPU in 2.44054s (13.9313GiB/s) | 20260507 21:45:57.136 | INFO | Analyzing chunk frames = [2831872,2832384) | 20260507 21:45:57.136 | INFO | MemInfo: { | 20260507 21:45:57.136 | INFO | "gpu" : " 17 GB", | 20260507 21:45:57.136 | INFO | "huge" : "", | 20260507 21:45:57.136 | INFO | "huge_shared" : " 29 GB", | 20260507 21:45:57.136 | INFO | "malloc" : " 14 GB", | 20260507 21:45:57.136 | INFO | "pinned" : " 29 GB" | 20260507 21:45:57.136 | INFO | } | 20260507 21:45:57.136 | INFO | Perf report for the last 5.14098 seconds: | 20260507 21:45:57.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:45:57.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.136 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:45:57.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.136 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:45:57.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:45:57.137 | INFO | Uploaded 36GiB to GPU in 2.48569s (14.4829GiB/s) | 20260507 21:45:57.137 | INFO | Downloaded 34GiB to GPU in 2.37347s (14.325GiB/s) | 20260507 21:46:02.274 | INFO | Analyzing chunk frames = [2832384,2832896) | 20260507 21:46:02.274 | INFO | MemInfo: { | 20260507 21:46:02.274 | INFO | "gpu" : " 17 GB", | 20260507 21:46:02.274 | INFO | "huge" : "", | 20260507 21:46:02.274 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:02.274 | INFO | "malloc" : " 14 GB", | 20260507 21:46:02.274 | INFO | "pinned" : " 29 GB" | 20260507 21:46:02.274 | INFO | } | 20260507 21:46:02.274 | INFO | Perf report for the last 5.13806 seconds: | 20260507 21:46:02.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:02.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.274 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 21:46:02.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.274 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:46:02.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:02.275 | INFO | Uploaded 36GiB to GPU in 2.41544s (14.9041GiB/s) | 20260507 21:46:02.275 | INFO | Downloaded 34GiB to GPU in 2.49671s (13.6179GiB/s) | 20260507 21:46:07.414 | INFO | Analyzing chunk frames = [2832896,2833408) | 20260507 21:46:07.414 | INFO | MemInfo: { | 20260507 21:46:07.414 | INFO | "gpu" : " 17 GB", | 20260507 21:46:07.414 | INFO | "huge" : "", | 20260507 21:46:07.414 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:07.414 | INFO | "malloc" : " 14 GB", | 20260507 21:46:07.414 | INFO | "pinned" : " 29 GB" | 20260507 21:46:07.414 | INFO | } | 20260507 21:46:07.414 | INFO | Perf report for the last 5.14011 seconds: | 20260507 21:46:07.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:07.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.415 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 21:46:07.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.415 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 21:46:07.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:07.415 | INFO | Uploaded 36GiB to GPU in 2.51671s (14.3044GiB/s) | 20260507 21:46:07.415 | INFO | Downloaded 34GiB to GPU in 2.43163s (13.9824GiB/s) | 20260507 21:46:12.554 | INFO | Analyzing chunk frames = [2833408,2833920) | 20260507 21:46:12.555 | INFO | MemInfo: { | 20260507 21:46:12.555 | INFO | "gpu" : " 17 GB", | 20260507 21:46:12.555 | INFO | "huge" : "", | 20260507 21:46:12.555 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:12.555 | INFO | "malloc" : " 14 GB", | 20260507 21:46:12.555 | INFO | "pinned" : " 29 GB" | 20260507 21:46:12.555 | INFO | } | 20260507 21:46:12.555 | INFO | Perf report for the last 5.14015 seconds: | 20260507 21:46:12.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:12.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:46:12.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:46:12.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:12.555 | INFO | Uploaded 36GiB to GPU in 2.5232s (14.2676GiB/s) | 20260507 21:46:12.555 | INFO | Downloaded 34GiB to GPU in 2.45457s (13.8517GiB/s) | 20260507 21:46:17.689 | INFO | Analyzing chunk frames = [2833920,2834432) | 20260507 21:46:17.689 | INFO | MemInfo: { | 20260507 21:46:17.689 | INFO | "gpu" : " 17 GB", | 20260507 21:46:17.689 | INFO | "huge" : "", | 20260507 21:46:17.689 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:17.689 | INFO | "malloc" : " 14 GB", | 20260507 21:46:17.689 | INFO | "pinned" : " 29 GB" | 20260507 21:46:17.689 | INFO | } | 20260507 21:46:17.690 | INFO | Perf report for the last 5.13495 seconds: | 20260507 21:46:17.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:17.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:46:17.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:46:17.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:17.690 | INFO | Uploaded 36GiB to GPU in 2.50884s (14.3492GiB/s) | 20260507 21:46:17.690 | INFO | Downloaded 34GiB to GPU in 2.43836s (13.9438GiB/s) | 20260507 21:46:17.690 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:46:17.690 | INFO | Total mem usage: 16.7 GiB | 20260507 21:46:17.690 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:46:17.690 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.34%) | 20260507 21:46:17.690 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:46:17.690 | INFO | High Water Breakdown: | 20260507 21:46:17.690 | INFO | 60.5% : 9.32 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:46:17.690 | INFO | 39.3% : 6.06 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:46:17.690 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:46:17.690 | INFO | ------------------------------------------ | 20260507 21:46:17.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:46:17.690 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:46:17.690 | INFO | Total mem usage: 26.5 GiB | 20260507 21:46:17.690 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:46:17.690 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 21:46:17.690 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 21:46:17.690 | INFO | High Water Breakdown: | 20260507 21:46:17.690 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:46:17.690 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:46:17.690 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:46:17.690 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:46:17.691 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:46:17.691 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:46:17.691 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:46:17.691 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:46:17.691 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:46:17.691 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:46:17.691 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:46:17.691 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:46:17.691 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:46:17.691 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:46:17.691 | INFO | ------------------------------------------ | 20260507 21:46:17.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:46:17.691 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:46:17.691 | INFO | Total mem usage: 27.8 GiB | 20260507 21:46:17.691 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:46:17.691 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:46:17.691 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:46:17.691 | INFO | High Water Breakdown: | 20260507 21:46:17.691 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:46:17.691 | INFO | ------------------------------------------ | 20260507 21:46:17.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:46:17.691 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:46:17.691 | INFO | Total mem usage: 15.7 GiB | 20260507 21:46:17.691 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:46:17.691 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:46:17.691 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:46:17.691 | INFO | High Water Breakdown: | 20260507 21:46:17.691 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:46:17.691 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:46:17.691 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:46:17.691 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:46:17.691 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:46:17.691 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:46:17.691 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:46:17.691 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:46:17.691 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:46:17.691 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:46:17.691 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:46:17.691 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:46:17.691 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:46:17.691 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:46:17.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:46:17.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:46:17.691 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:46:17.691 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:46:17.691 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:46:17.691 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:46:17.691 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:46:17.691 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:46:17.691 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:46:17.691 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:46:17.691 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:46:17.691 | INFO | ------------------------------------------ | 20260507 21:46:17.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:46:17.691 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:46:17.691 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:46:17.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:46:17.691 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.41e+01 / 1.59e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.20e-02 / 7.31e-03 | 20260507 21:46:17.691 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.48e+00, 5.95e-01 / 2.18e-01 | 20260507 21:46:17.691 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.07e-01 / 1.04e-01 | 20260507 21:46:17.691 | INFO | PerfReport: 15.80%, DME -- 4.17e+01 / 4.51e+01, 3.94e+00 / 1.28e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.68e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.29e-01 / 2.57e-01 | 20260507 21:46:17.691 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.36e+00, 2.97e-01 / 1.96e-01 | 20260507 21:46:17.691 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.94e-01, 6.11e-02 / 3.54e-02 | 20260507 21:46:17.691 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:46:17.691 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:46:17.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:46:17.691 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 24.54%, Upload -- 2.62e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 24.17%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 30.92%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 2.11%, LockUpload -- 2.25e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 2.34%, LockDownload -- 2.50e+00 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 15.90%, LockCompute -- 1.70e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 21:46:17.691 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 21:46:17.787 | INFO | Starting a round of Baz IO | 20260507 21:46:21.282 | INFO | Populating RTMetrics report | 20260507 21:46:21.282 | INFO | RTMetrics report populated | 20260507 21:46:21.282 | INFO | Opening tmp RTMetrics json file | 20260507 21:46:21.322 | INFO | Serializing RTMetrics to json | 20260507 21:46:21.595 | INFO | Serialization done, swapping tmp file now... | 20260507 21:46:21.596 | INFO | RTMetrics done | 20260507 21:46:21.630 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.843 seconds | 20260507 21:46:21.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.52% / 0.01% / 56.48% | 20260507 21:46:21.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59003% / 0% / 5.16574e-05% | 20260507 21:46:21.630 | INFO | Baz IO overall: | 20260507 21:46:21.630 | INFO | BytesWritten : 1290977554563 | 20260507 21:46:21.630 | INFO | NumEvents : 625631154135 | 20260507 21:46:22.829 | INFO | Analyzing chunk frames = [2834432,2834944) | 20260507 21:46:22.829 | INFO | MemInfo: { | 20260507 21:46:22.829 | INFO | "gpu" : " 17 GB", | 20260507 21:46:22.829 | INFO | "huge" : "", | 20260507 21:46:22.829 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:22.829 | INFO | "malloc" : " 14 GB", | 20260507 21:46:22.829 | INFO | "pinned" : " 29 GB" | 20260507 21:46:22.829 | INFO | } | 20260507 21:46:22.829 | INFO | Perf report for the last 5.13928 seconds: | 20260507 21:46:22.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:22.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:22.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:22.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:22.829 | INFO | ANALYSIS : 69.0%, 3.0, 107.0ms, 0 | 20260507 21:46:22.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:22.829 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 21:46:22.829 | INFO | PRE_HQ : 65.7%, 1.7, 57.5ms, 0 | 20260507 21:46:22.829 | INFO | BAZWRITER : 74.2%, 3.8, 144.0ms, 0 | 20260507 21:46:22.829 | INFO | BAZ_SAMPLING : 6.3%, 1.0, 3.3ms, 0 | 20260507 21:46:22.829 | INFO | Uploaded 36GiB to GPU in 2.54868s (14.1249GiB/s) | 20260507 21:46:22.829 | INFO | Downloaded 38GiB to GPU in 2.69924s (14.0781GiB/s) | 20260507 21:46:27.972 | INFO | Analyzing chunk frames = [2834944,2835456) | 20260507 21:46:27.972 | INFO | MemInfo: { | 20260507 21:46:27.972 | INFO | "gpu" : " 17 GB", | 20260507 21:46:27.972 | INFO | "huge" : "", | 20260507 21:46:27.972 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:27.972 | INFO | "malloc" : " 14 GB", | 20260507 21:46:27.972 | INFO | "pinned" : " 29 GB" | 20260507 21:46:27.972 | INFO | } | 20260507 21:46:27.972 | INFO | Perf report for the last 5.14301 seconds: | 20260507 21:46:27.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:27.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:46:27.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:46:27.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:27.972 | INFO | Uploaded 36GiB to GPU in 2.53202s (14.2179GiB/s) | 20260507 21:46:27.973 | INFO | Downloaded 34GiB to GPU in 2.45927s (13.8252GiB/s) | 20260507 21:46:33.114 | INFO | Analyzing chunk frames = [2835456,2835968) | 20260507 21:46:33.114 | INFO | MemInfo: { | 20260507 21:46:33.114 | INFO | "gpu" : " 17 GB", | 20260507 21:46:33.114 | INFO | "huge" : "", | 20260507 21:46:33.115 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:33.115 | INFO | "malloc" : " 14 GB", | 20260507 21:46:33.115 | INFO | "pinned" : " 29 GB" | 20260507 21:46:33.115 | INFO | } | 20260507 21:46:33.115 | INFO | Perf report for the last 5.14323 seconds: | 20260507 21:46:33.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:33.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:46:33.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:46:33.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:33.115 | INFO | Uploaded 36GiB to GPU in 2.5345s (14.204GiB/s) | 20260507 21:46:33.115 | INFO | Downloaded 34GiB to GPU in 2.45868s (13.8285GiB/s) | 20260507 21:46:38.251 | INFO | Analyzing chunk frames = [2835968,2836480) | 20260507 21:46:38.251 | INFO | MemInfo: { | 20260507 21:46:38.251 | INFO | "gpu" : " 17 GB", | 20260507 21:46:38.251 | INFO | "huge" : "", | 20260507 21:46:38.251 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:38.251 | INFO | "malloc" : " 14 GB", | 20260507 21:46:38.251 | INFO | "pinned" : " 29 GB" | 20260507 21:46:38.251 | INFO | } | 20260507 21:46:38.252 | INFO | Perf report for the last 5.13636 seconds: | 20260507 21:46:38.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:38.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:46:38.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:46:38.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:38.252 | INFO | Uploaded 36GiB to GPU in 2.48707s (14.4748GiB/s) | 20260507 21:46:38.252 | INFO | Downloaded 34GiB to GPU in 2.39683s (14.1854GiB/s) | 20260507 21:46:43.387 | INFO | Analyzing chunk frames = [2836480,2836992) | 20260507 21:46:43.387 | INFO | MemInfo: { | 20260507 21:46:43.387 | INFO | "gpu" : " 17 GB", | 20260507 21:46:43.387 | INFO | "huge" : "", | 20260507 21:46:43.387 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:43.387 | INFO | "malloc" : " 14 GB", | 20260507 21:46:43.387 | INFO | "pinned" : " 29 GB" | 20260507 21:46:43.387 | INFO | } | 20260507 21:46:43.387 | INFO | Perf report for the last 5.13552 seconds: | 20260507 21:46:43.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:43.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 21:46:43.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260507 21:46:43.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:43.387 | INFO | Uploaded 36GiB to GPU in 2.43327s (14.7949GiB/s) | 20260507 21:46:43.388 | INFO | Downloaded 34GiB to GPU in 2.50359s (13.5805GiB/s) | 20260507 21:46:48.525 | INFO | Analyzing chunk frames = [2836992,2837504) | 20260507 21:46:48.525 | INFO | MemInfo: { | 20260507 21:46:48.525 | INFO | "gpu" : " 17 GB", | 20260507 21:46:48.526 | INFO | "huge" : "", | 20260507 21:46:48.526 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:48.526 | INFO | "malloc" : " 14 GB", | 20260507 21:46:48.526 | INFO | "pinned" : " 29 GB" | 20260507 21:46:48.526 | INFO | } | 20260507 21:46:48.526 | INFO | Perf report for the last 5.13872 seconds: | 20260507 21:46:48.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:48.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:46:48.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:46:48.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:48.526 | INFO | Uploaded 36GiB to GPU in 2.51699s (14.3028GiB/s) | 20260507 21:46:48.526 | INFO | Downloaded 34GiB to GPU in 2.40453s (14.14GiB/s) | 20260507 21:46:53.666 | INFO | Analyzing chunk frames = [2837504,2838016) | 20260507 21:46:53.666 | INFO | MemInfo: { | 20260507 21:46:53.666 | INFO | "gpu" : " 17 GB", | 20260507 21:46:53.666 | INFO | "huge" : "", | 20260507 21:46:53.666 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:53.666 | INFO | "malloc" : " 14 GB", | 20260507 21:46:53.666 | INFO | "pinned" : " 29 GB" | 20260507 21:46:53.667 | INFO | } | 20260507 21:46:53.667 | INFO | Perf report for the last 5.1408 seconds: | 20260507 21:46:53.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:53.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 21:46:53.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:46:53.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:53.667 | INFO | Uploaded 36GiB to GPU in 2.54257s (14.1589GiB/s) | 20260507 21:46:53.667 | INFO | Downloaded 34GiB to GPU in 2.44023s (13.9331GiB/s) | 20260507 21:46:58.806 | INFO | Analyzing chunk frames = [2838016,2838528) | 20260507 21:46:58.806 | INFO | MemInfo: { | 20260507 21:46:58.806 | INFO | "gpu" : " 17 GB", | 20260507 21:46:58.806 | INFO | "huge" : "", | 20260507 21:46:58.806 | INFO | "huge_shared" : " 29 GB", | 20260507 21:46:58.806 | INFO | "malloc" : " 14 GB", | 20260507 21:46:58.806 | INFO | "pinned" : " 29 GB" | 20260507 21:46:58.806 | INFO | } | 20260507 21:46:58.806 | INFO | Perf report for the last 5.13985 seconds: | 20260507 21:46:58.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:46:58.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.807 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 21:46:58.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.807 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:46:58.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:46:58.807 | INFO | Uploaded 36GiB to GPU in 2.53875s (14.1802GiB/s) | 20260507 21:46:58.807 | INFO | Downloaded 34GiB to GPU in 2.42648s (14.012GiB/s) | 20260507 21:46:58.907 | INFO | Starting a round of Baz IO | 20260507 21:47:02.381 | INFO | Populating RTMetrics report | 20260507 21:47:02.381 | INFO | RTMetrics report populated | 20260507 21:47:02.381 | INFO | Opening tmp RTMetrics json file | 20260507 21:47:02.403 | INFO | Serializing RTMetrics to json | 20260507 21:47:02.405 | INFO | Serialization done, swapping tmp file now... | 20260507 21:47:02.406 | INFO | RTMetrics done | 20260507 21:47:02.486 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.578 seconds | 20260507 21:47:02.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.51% / 0.01% / 56.48% | 20260507 21:47:02.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58274% / 2.78155e-05% / 4.76837e-05% | 20260507 21:47:02.486 | INFO | Baz IO overall: | 20260507 21:47:02.486 | INFO | BytesWritten : 1292856770863 | 20260507 21:47:02.486 | INFO | NumEvents : 626539087445 | 20260507 21:47:03.943 | INFO | Analyzing chunk frames = [2838528,2839040) | 20260507 21:47:03.943 | INFO | MemInfo: { | 20260507 21:47:03.943 | INFO | "gpu" : " 17 GB", | 20260507 21:47:03.943 | INFO | "huge" : "", | 20260507 21:47:03.943 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:03.943 | INFO | "malloc" : " 14 GB", | 20260507 21:47:03.943 | INFO | "pinned" : " 29 GB" | 20260507 21:47:03.943 | INFO | } | 20260507 21:47:03.943 | INFO | Perf report for the last 5.13682 seconds: | 20260507 21:47:03.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:03.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:03.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:03.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:03.943 | INFO | ANALYSIS : 68.9%, 3.0, 106.8ms, 0 | 20260507 21:47:03.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:03.943 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 21:47:03.943 | INFO | PRE_HQ : 66.0%, 1.7, 57.6ms, 0 | 20260507 21:47:03.943 | INFO | BAZWRITER : 69.3%, 3.9, 137.8ms, 0 | 20260507 21:47:03.943 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 21:47:03.944 | INFO | Uploaded 36GiB to GPU in 2.54083s (14.1686GiB/s) | 20260507 21:47:03.944 | INFO | Downloaded 38GiB to GPU in 2.70136s (14.067GiB/s) | 20260507 21:47:09.087 | INFO | Analyzing chunk frames = [2839040,2839552) | 20260507 21:47:09.087 | INFO | MemInfo: { | 20260507 21:47:09.087 | INFO | "gpu" : " 17 GB", | 20260507 21:47:09.087 | INFO | "huge" : "", | 20260507 21:47:09.087 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:09.087 | INFO | "malloc" : " 14 GB", | 20260507 21:47:09.087 | INFO | "pinned" : " 29 GB" | 20260507 21:47:09.087 | INFO | } | 20260507 21:47:09.087 | INFO | Perf report for the last 5.14372 seconds: | 20260507 21:47:09.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:09.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 21:47:09.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | PRE_HQ : 25.2%, 1.0, 13.2ms, 0 | 20260507 21:47:09.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:09.087 | INFO | Uploaded 36GiB to GPU in 2.51933s (14.2895GiB/s) | 20260507 21:47:09.087 | INFO | Downloaded 34GiB to GPU in 2.47129s (13.758GiB/s) | 20260507 21:47:14.228 | INFO | Analyzing chunk frames = [2839552,2840064) | 20260507 21:47:14.228 | INFO | MemInfo: { | 20260507 21:47:14.228 | INFO | "gpu" : " 17 GB", | 20260507 21:47:14.228 | INFO | "huge" : "", | 20260507 21:47:14.228 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:14.228 | INFO | "malloc" : " 14 GB", | 20260507 21:47:14.228 | INFO | "pinned" : " 29 GB" | 20260507 21:47:14.228 | INFO | } | 20260507 21:47:14.228 | INFO | Perf report for the last 5.14103 seconds: | 20260507 21:47:14.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:14.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.228 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 21:47:14.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.228 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:47:14.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:14.229 | INFO | Uploaded 36GiB to GPU in 2.56446s (14.038GiB/s) | 20260507 21:47:14.229 | INFO | Downloaded 34GiB to GPU in 2.47826s (13.7193GiB/s) | 20260507 21:47:19.370 | INFO | Analyzing chunk frames = [2840064,2840576) | 20260507 21:47:19.370 | INFO | MemInfo: { | 20260507 21:47:19.370 | INFO | "gpu" : " 17 GB", | 20260507 21:47:19.370 | INFO | "huge" : "", | 20260507 21:47:19.370 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:19.370 | INFO | "malloc" : " 14 GB", | 20260507 21:47:19.370 | INFO | "pinned" : " 29 GB" | 20260507 21:47:19.370 | INFO | } | 20260507 21:47:19.370 | INFO | Perf report for the last 5.1421 seconds: | 20260507 21:47:19.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:19.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 21:47:19.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | PRE_HQ : 36.1%, 1.0, 18.8ms, 0 | 20260507 21:47:19.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:19.370 | INFO | Uploaded 36GiB to GPU in 2.52045s (14.2832GiB/s) | 20260507 21:47:19.371 | INFO | Downloaded 34GiB to GPU in 2.49995s (13.6003GiB/s) | 20260507 21:47:24.506 | INFO | Analyzing chunk frames = [2840576,2841088) | 20260507 21:47:24.506 | INFO | MemInfo: { | 20260507 21:47:24.506 | INFO | "gpu" : " 17 GB", | 20260507 21:47:24.506 | INFO | "huge" : "", | 20260507 21:47:24.506 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:24.506 | INFO | "malloc" : " 14 GB", | 20260507 21:47:24.506 | INFO | "pinned" : " 29 GB" | 20260507 21:47:24.506 | INFO | } | 20260507 21:47:24.507 | INFO | Perf report for the last 5.13636 seconds: | 20260507 21:47:24.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:24.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260507 21:47:24.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 21:47:24.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:24.507 | INFO | Uploaded 36GiB to GPU in 2.43266s (14.7986GiB/s) | 20260507 21:47:24.507 | INFO | Downloaded 34GiB to GPU in 2.50678s (13.5632GiB/s) | 20260507 21:47:29.643 | INFO | Analyzing chunk frames = [2841088,2841600) | 20260507 21:47:29.643 | INFO | MemInfo: { | 20260507 21:47:29.643 | INFO | "gpu" : " 17 GB", | 20260507 21:47:29.643 | INFO | "huge" : "", | 20260507 21:47:29.643 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:29.643 | INFO | "malloc" : " 14 GB", | 20260507 21:47:29.643 | INFO | "pinned" : " 29 GB" | 20260507 21:47:29.643 | INFO | } | 20260507 21:47:29.643 | INFO | Perf report for the last 5.13672 seconds: | 20260507 21:47:29.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:29.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.644 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 21:47:29.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.644 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:47:29.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:29.644 | INFO | Uploaded 36GiB to GPU in 2.49059s (14.4544GiB/s) | 20260507 21:47:29.644 | INFO | Downloaded 34GiB to GPU in 2.42939s (13.9953GiB/s) | 20260507 21:47:34.782 | INFO | Analyzing chunk frames = [2841600,2842112) | 20260507 21:47:34.782 | INFO | MemInfo: { | 20260507 21:47:34.782 | INFO | "gpu" : " 17 GB", | 20260507 21:47:34.782 | INFO | "huge" : "", | 20260507 21:47:34.782 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:34.782 | INFO | "malloc" : " 14 GB", | 20260507 21:47:34.782 | INFO | "pinned" : " 29 GB" | 20260507 21:47:34.782 | INFO | } | 20260507 21:47:34.782 | INFO | Perf report for the last 5.13871 seconds: | 20260507 21:47:34.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:34.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:47:34.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:47:34.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:34.782 | INFO | Uploaded 36GiB to GPU in 2.51646s (14.3058GiB/s) | 20260507 21:47:34.783 | INFO | Downloaded 34GiB to GPU in 2.45805s (13.8321GiB/s) | 20260507 21:47:39.921 | INFO | Analyzing chunk frames = [2842112,2842624) | 20260507 21:47:39.921 | INFO | MemInfo: { | 20260507 21:47:39.921 | INFO | "gpu" : " 17 GB", | 20260507 21:47:39.921 | INFO | "huge" : "", | 20260507 21:47:39.921 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:39.921 | INFO | "malloc" : " 14 GB", | 20260507 21:47:39.921 | INFO | "pinned" : " 29 GB" | 20260507 21:47:39.921 | INFO | } | 20260507 21:47:39.921 | INFO | Perf report for the last 5.13905 seconds: | 20260507 21:47:39.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:39.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.921 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:47:39.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.921 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 21:47:39.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:39.922 | INFO | Uploaded 36GiB to GPU in 2.49838s (14.4093GiB/s) | 20260507 21:47:39.922 | INFO | Downloaded 34GiB to GPU in 2.44557s (13.9027GiB/s) | 20260507 21:47:39.922 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:47:39.922 | INFO | Total mem usage: 16.7 GiB | 20260507 21:47:39.922 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:47:39.922 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:47:39.922 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:47:39.922 | INFO | High Water Breakdown: | 20260507 21:47:39.922 | INFO | 59.9% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:47:39.922 | INFO | 39.9% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:47:39.922 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:47:39.922 | INFO | ------------------------------------------ | 20260507 21:47:39.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:47:39.922 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:47:39.922 | INFO | Total mem usage: 26.5 GiB | 20260507 21:47:39.922 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:47:39.922 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:47:39.922 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:47:39.922 | INFO | High Water Breakdown: | 20260507 21:47:39.922 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:47:39.922 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:47:39.922 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:47:39.922 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:47:39.922 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:47:39.922 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:47:39.922 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:47:39.922 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:47:39.922 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:47:39.922 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:47:39.922 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:47:39.922 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:47:39.922 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:47:39.922 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:47:39.922 | INFO | ------------------------------------------ | 20260507 21:47:39.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:47:39.922 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:47:39.922 | INFO | Total mem usage: 27.8 GiB | 20260507 21:47:39.922 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:47:39.922 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:47:39.922 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:47:39.922 | INFO | High Water Breakdown: | 20260507 21:47:39.922 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:47:39.922 | INFO | ------------------------------------------ | 20260507 21:47:39.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:47:39.922 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:47:39.922 | INFO | Total mem usage: 15.7 GiB | 20260507 21:47:39.922 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:47:39.922 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:47:39.922 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:47:39.922 | INFO | High Water Breakdown: | 20260507 21:47:39.922 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:47:39.923 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:47:39.923 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:47:39.923 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:47:39.923 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:47:39.923 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:47:39.923 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:47:39.923 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:47:39.923 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:47:39.923 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:47:39.923 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:47:39.923 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:47:39.923 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:47:39.923 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:47:39.923 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:47:39.923 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:47:39.923 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:47:39.923 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:47:39.923 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:47:39.923 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:47:39.923 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:47:39.923 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:47:39.923 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:47:39.923 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:47:39.923 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:47:39.923 | INFO | ------------------------------------------ | 20260507 21:47:39.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:47:39.923 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:47:39.923 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:47:39.923 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:47:39.923 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.40e+01 / 1.94e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.02e-01, 1.16e-02 / 6.05e-03 | 20260507 21:47:39.923 | INFO | PerfReport: 12.12%, Baseline -- 4.01e+00 / 4.49e+00, 6.00e-01 / 2.60e-01 | 20260507 21:47:39.923 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.08e-01 / 1.11e-01 | 20260507 21:47:39.923 | INFO | PerfReport: 15.67%, DME -- 4.15e+01 / 4.46e+01, 3.80e+00 / 1.70e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.87e+01 / 2.09e+01, 1.89e+00 / 2.08e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.25e-01 / 2.30e-01 | 20260507 21:47:39.923 | INFO | PerfReport: 3.48%, Metrics -- 1.15e+00 / 1.35e+00, 2.93e-01 / 1.45e-01 | 20260507 21:47:39.923 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.96e-01, 5.97e-02 / 3.34e-02 | 20260507 21:47:39.923 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:47:39.923 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:47:39.923 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:47:39.923 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 24.47%, Upload -- 2.62e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.57e+00 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 30.91%, Compute -- 3.31e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 2.45%, LockDownload -- 2.62e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 15.99%, LockCompute -- 1.71e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 21:47:39.923 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260507 21:47:40.019 | INFO | Starting a round of Baz IO | 20260507 21:47:43.495 | INFO | Populating RTMetrics report | 20260507 21:47:43.495 | INFO | RTMetrics report populated | 20260507 21:47:43.495 | INFO | Opening tmp RTMetrics json file | 20260507 21:47:43.518 | INFO | Serializing RTMetrics to json | 20260507 21:47:43.520 | INFO | Serialization done, swapping tmp file now... | 20260507 21:47:43.520 | INFO | RTMetrics done | 20260507 21:47:43.607 | INFO | Finished a round of Baz IO: Wrote 1.751 GiB in 3.588 seconds | 20260507 21:47:43.607 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.50% / 0.01% / 56.49% | 20260507 21:47:43.607 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5838% / 3.97364e-06% / 5.16574e-05% | 20260507 21:47:43.607 | INFO | Baz IO overall: | 20260507 21:47:43.607 | INFO | BytesWritten : 1294737349103 | 20260507 21:47:43.607 | INFO | NumEvents : 627448137056 | 20260507 21:47:45.064 | INFO | Analyzing chunk frames = [2842624,2843136) | 20260507 21:47:45.064 | INFO | MemInfo: { | 20260507 21:47:45.064 | INFO | "gpu" : " 17 GB", | 20260507 21:47:45.064 | INFO | "huge" : "", | 20260507 21:47:45.064 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:45.064 | INFO | "malloc" : " 14 GB", | 20260507 21:47:45.064 | INFO | "pinned" : " 29 GB" | 20260507 21:47:45.064 | INFO | } | 20260507 21:47:45.064 | INFO | Perf report for the last 5.14308 seconds: | 20260507 21:47:45.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:45.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:45.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:45.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:45.064 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 21:47:45.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:45.064 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 21:47:45.064 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260507 21:47:45.064 | INFO | BAZWRITER : 69.4%, 4.3, 155.0ms, 0 | 20260507 21:47:45.064 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 21:47:45.065 | INFO | Uploaded 36GiB to GPU in 2.56722s (14.0229GiB/s) | 20260507 21:47:45.065 | INFO | Downloaded 38GiB to GPU in 2.70364s (14.0551GiB/s) | 20260507 21:47:50.203 | INFO | Analyzing chunk frames = [2843136,2843648) | 20260507 21:47:50.203 | INFO | MemInfo: { | 20260507 21:47:50.203 | INFO | "gpu" : " 17 GB", | 20260507 21:47:50.203 | INFO | "huge" : "", | 20260507 21:47:50.203 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:50.203 | INFO | "malloc" : " 14 GB", | 20260507 21:47:50.203 | INFO | "pinned" : " 29 GB" | 20260507 21:47:50.203 | INFO | } | 20260507 21:47:50.203 | INFO | Perf report for the last 5.13893 seconds: | 20260507 21:47:50.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:50.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.203 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:47:50.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.203 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:47:50.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:50.204 | INFO | Uploaded 36GiB to GPU in 2.53801s (14.1843GiB/s) | 20260507 21:47:50.204 | INFO | Downloaded 34GiB to GPU in 2.48357s (13.69GiB/s) | 20260507 21:47:55.337 | INFO | Analyzing chunk frames = [2843648,2844160) | 20260507 21:47:55.337 | INFO | MemInfo: { | 20260507 21:47:55.337 | INFO | "gpu" : " 17 GB", | 20260507 21:47:55.338 | INFO | "huge" : "", | 20260507 21:47:55.338 | INFO | "huge_shared" : " 29 GB", | 20260507 21:47:55.338 | INFO | "malloc" : " 14 GB", | 20260507 21:47:55.338 | INFO | "pinned" : " 29 GB" | 20260507 21:47:55.338 | INFO | } | 20260507 21:47:55.338 | INFO | Perf report for the last 5.13449 seconds: | 20260507 21:47:55.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:47:55.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:47:55.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:47:55.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:47:55.338 | INFO | Uploaded 36GiB to GPU in 2.52929s (14.2332GiB/s) | 20260507 21:47:55.338 | INFO | Downloaded 34GiB to GPU in 2.47s (13.7652GiB/s) | 20260507 21:48:00.476 | INFO | Analyzing chunk frames = [2844160,2844672) | 20260507 21:48:00.476 | INFO | MemInfo: { | 20260507 21:48:00.476 | INFO | "gpu" : " 17 GB", | 20260507 21:48:00.476 | INFO | "huge" : "", | 20260507 21:48:00.476 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:00.476 | INFO | "malloc" : " 14 GB", | 20260507 21:48:00.476 | INFO | "pinned" : " 29 GB" | 20260507 21:48:00.476 | INFO | } | 20260507 21:48:00.476 | INFO | Perf report for the last 5.13826 seconds: | 20260507 21:48:00.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:00.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:48:00.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:48:00.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:00.476 | INFO | Uploaded 36GiB to GPU in 2.47793s (14.5282GiB/s) | 20260507 21:48:00.476 | INFO | Downloaded 34GiB to GPU in 2.39295s (14.2084GiB/s) | 20260507 21:48:05.616 | INFO | Analyzing chunk frames = [2844672,2845184) | 20260507 21:48:05.616 | INFO | MemInfo: { | 20260507 21:48:05.616 | INFO | "gpu" : " 17 GB", | 20260507 21:48:05.616 | INFO | "huge" : "", | 20260507 21:48:05.616 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:05.616 | INFO | "malloc" : " 14 GB", | 20260507 21:48:05.616 | INFO | "pinned" : " 29 GB" | 20260507 21:48:05.616 | INFO | } | 20260507 21:48:05.616 | INFO | Perf report for the last 5.14032 seconds: | 20260507 21:48:05.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:05.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.616 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:48:05.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.616 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:48:05.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:05.617 | INFO | Uploaded 36GiB to GPU in 2.32836s (15.4615GiB/s) | 20260507 21:48:05.617 | INFO | Downloaded 34GiB to GPU in 2.49983s (13.6009GiB/s) | 20260507 21:48:10.756 | INFO | Analyzing chunk frames = [2845184,2845696) | 20260507 21:48:10.756 | INFO | MemInfo: { | 20260507 21:48:10.756 | INFO | "gpu" : " 17 GB", | 20260507 21:48:10.756 | INFO | "huge" : "", | 20260507 21:48:10.756 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:10.757 | INFO | "malloc" : " 14 GB", | 20260507 21:48:10.757 | INFO | "pinned" : " 29 GB" | 20260507 21:48:10.757 | INFO | } | 20260507 21:48:10.757 | INFO | Perf report for the last 5.14042 seconds: | 20260507 21:48:10.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:10.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:48:10.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:48:10.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:10.757 | INFO | Uploaded 36GiB to GPU in 2.43244s (14.8GiB/s) | 20260507 21:48:10.757 | INFO | Downloaded 34GiB to GPU in 2.52649s (13.4574GiB/s) | 20260507 21:48:15.894 | INFO | Analyzing chunk frames = [2845696,2846208) | 20260507 21:48:15.894 | INFO | MemInfo: { | 20260507 21:48:15.894 | INFO | "gpu" : " 17 GB", | 20260507 21:48:15.894 | INFO | "huge" : "", | 20260507 21:48:15.894 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:15.894 | INFO | "malloc" : " 14 GB", | 20260507 21:48:15.894 | INFO | "pinned" : " 29 GB" | 20260507 21:48:15.894 | INFO | } | 20260507 21:48:15.894 | INFO | Perf report for the last 5.13734 seconds: | 20260507 21:48:15.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:15.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 21:48:15.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:48:15.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:15.894 | INFO | Uploaded 36GiB to GPU in 2.46365s (14.6125GiB/s) | 20260507 21:48:15.894 | INFO | Downloaded 34GiB to GPU in 2.45598s (13.8437GiB/s) | 20260507 21:48:21.038 | INFO | Analyzing chunk frames = [2846208,2846720) | 20260507 21:48:21.038 | INFO | MemInfo: { | 20260507 21:48:21.038 | INFO | "gpu" : " 17 GB", | 20260507 21:48:21.038 | INFO | "huge" : "", | 20260507 21:48:21.038 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:21.038 | INFO | "malloc" : " 14 GB", | 20260507 21:48:21.038 | INFO | "pinned" : " 29 GB" | 20260507 21:48:21.038 | INFO | } | 20260507 21:48:21.038 | INFO | Perf report for the last 5.14432 seconds: | 20260507 21:48:21.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:21.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.038 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:48:21.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.038 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:48:21.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:21.039 | INFO | Uploaded 36GiB to GPU in 2.44204s (14.7418GiB/s) | 20260507 21:48:21.039 | INFO | Downloaded 34GiB to GPU in 2.45034s (13.8756GiB/s) | 20260507 21:48:21.136 | INFO | Starting a round of Baz IO | 20260507 21:48:24.598 | INFO | Populating RTMetrics report | 20260507 21:48:24.599 | INFO | RTMetrics report populated | 20260507 21:48:24.599 | INFO | Opening tmp RTMetrics json file | 20260507 21:48:24.632 | INFO | Serializing RTMetrics to json | 20260507 21:48:24.685 | INFO | Serialization done, swapping tmp file now... | 20260507 21:48:24.690 | INFO | RTMetrics done | 20260507 21:48:24.716 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.58 seconds | 20260507 21:48:24.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.49% / 0.01% / 56.50% | 20260507 21:48:24.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58488% / 7.94729e-06% / 5.16574e-05% | 20260507 21:48:24.716 | INFO | Baz IO overall: | 20260507 21:48:24.716 | INFO | BytesWritten : 1296618364525 | 20260507 21:48:24.716 | INFO | NumEvents : 628357485844 | 20260507 21:48:26.179 | INFO | Analyzing chunk frames = [2846720,2847232) | 20260507 21:48:26.179 | INFO | MemInfo: { | 20260507 21:48:26.179 | INFO | "gpu" : " 17 GB", | 20260507 21:48:26.179 | INFO | "huge" : "", | 20260507 21:48:26.179 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:26.179 | INFO | "malloc" : " 14 GB", | 20260507 21:48:26.179 | INFO | "pinned" : " 29 GB" | 20260507 21:48:26.179 | INFO | } | 20260507 21:48:26.179 | INFO | Perf report for the last 5.14089 seconds: | 20260507 21:48:26.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:26.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:26.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:26.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:26.179 | INFO | ANALYSIS : 68.5%, 3.0, 106.3ms, 0 | 20260507 21:48:26.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:26.179 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260507 21:48:26.179 | INFO | PRE_HQ : 65.7%, 1.7, 57.6ms, 0 | 20260507 21:48:26.179 | INFO | BAZWRITER : 69.3%, 4.0, 143.8ms, 0 | 20260507 21:48:26.179 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260507 21:48:26.180 | INFO | Uploaded 36GiB to GPU in 2.5263s (14.2501GiB/s) | 20260507 21:48:26.180 | INFO | Downloaded 38GiB to GPU in 2.69386s (14.1061GiB/s) | 20260507 21:48:31.319 | INFO | Analyzing chunk frames = [2847232,2847744) | 20260507 21:48:31.319 | INFO | MemInfo: { | 20260507 21:48:31.319 | INFO | "gpu" : " 17 GB", | 20260507 21:48:31.319 | INFO | "huge" : "", | 20260507 21:48:31.319 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:31.319 | INFO | "malloc" : " 14 GB", | 20260507 21:48:31.319 | INFO | "pinned" : " 29 GB" | 20260507 21:48:31.319 | INFO | } | 20260507 21:48:31.319 | INFO | Perf report for the last 5.14001 seconds: | 20260507 21:48:31.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:31.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.319 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:48:31.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.320 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:48:31.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:31.320 | INFO | Uploaded 36GiB to GPU in 2.51628s (14.3069GiB/s) | 20260507 21:48:31.320 | INFO | Downloaded 34GiB to GPU in 2.49144s (13.6467GiB/s) | 20260507 21:48:36.454 | INFO | Analyzing chunk frames = [2847744,2848256) | 20260507 21:48:36.454 | INFO | MemInfo: { | 20260507 21:48:36.454 | INFO | "gpu" : " 17 GB", | 20260507 21:48:36.454 | INFO | "huge" : "", | 20260507 21:48:36.454 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:36.454 | INFO | "malloc" : " 14 GB", | 20260507 21:48:36.454 | INFO | "pinned" : " 29 GB" | 20260507 21:48:36.454 | INFO | } | 20260507 21:48:36.454 | INFO | Perf report for the last 5.13481 seconds: | 20260507 21:48:36.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:36.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 21:48:36.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:48:36.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:36.454 | INFO | Uploaded 36GiB to GPU in 2.51677s (14.3041GiB/s) | 20260507 21:48:36.454 | INFO | Downloaded 34GiB to GPU in 2.49186s (13.6444GiB/s) | 20260507 21:48:41.591 | INFO | Analyzing chunk frames = [2848256,2848768) | 20260507 21:48:41.591 | INFO | MemInfo: { | 20260507 21:48:41.591 | INFO | "gpu" : " 17 GB", | 20260507 21:48:41.591 | INFO | "huge" : "", | 20260507 21:48:41.592 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:41.592 | INFO | "malloc" : " 14 GB", | 20260507 21:48:41.592 | INFO | "pinned" : " 29 GB" | 20260507 21:48:41.592 | INFO | } | 20260507 21:48:41.592 | INFO | Perf report for the last 5.13744 seconds: | 20260507 21:48:41.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:41.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 21:48:41.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:48:41.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:41.592 | INFO | Uploaded 36GiB to GPU in 2.42407s (14.8511GiB/s) | 20260507 21:48:41.592 | INFO | Downloaded 34GiB to GPU in 2.39381s (14.2033GiB/s) | 20260507 21:48:46.733 | INFO | Analyzing chunk frames = [2848768,2849280) | 20260507 21:48:46.733 | INFO | MemInfo: { | 20260507 21:48:46.733 | INFO | "gpu" : " 17 GB", | 20260507 21:48:46.733 | INFO | "huge" : "", | 20260507 21:48:46.733 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:46.733 | INFO | "malloc" : " 14 GB", | 20260507 21:48:46.733 | INFO | "pinned" : " 29 GB" | 20260507 21:48:46.733 | INFO | } | 20260507 21:48:46.733 | INFO | Perf report for the last 5.14165 seconds: | 20260507 21:48:46.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:46.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.733 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 21:48:46.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.733 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 21:48:46.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:46.734 | INFO | Uploaded 36GiB to GPU in 2.31366s (15.5598GiB/s) | 20260507 21:48:46.734 | INFO | Downloaded 34GiB to GPU in 2.50341s (13.5815GiB/s) | 20260507 21:48:51.875 | INFO | Analyzing chunk frames = [2849280,2849792) | 20260507 21:48:51.875 | INFO | MemInfo: { | 20260507 21:48:51.875 | INFO | "gpu" : " 17 GB", | 20260507 21:48:51.875 | INFO | "huge" : "", | 20260507 21:48:51.875 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:51.875 | INFO | "malloc" : " 14 GB", | 20260507 21:48:51.875 | INFO | "pinned" : " 29 GB" | 20260507 21:48:51.875 | INFO | } | 20260507 21:48:51.875 | INFO | Perf report for the last 5.142 seconds: | 20260507 21:48:51.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:51.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.875 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 21:48:51.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.875 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 21:48:51.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:51.876 | INFO | Uploaded 36GiB to GPU in 2.51885s (14.2922GiB/s) | 20260507 21:48:51.876 | INFO | Downloaded 34GiB to GPU in 2.42118s (14.0427GiB/s) | 20260507 21:48:57.009 | INFO | Analyzing chunk frames = [2849792,2850304) | 20260507 21:48:57.009 | INFO | MemInfo: { | 20260507 21:48:57.009 | INFO | "gpu" : " 17 GB", | 20260507 21:48:57.009 | INFO | "huge" : "", | 20260507 21:48:57.009 | INFO | "huge_shared" : " 29 GB", | 20260507 21:48:57.009 | INFO | "malloc" : " 14 GB", | 20260507 21:48:57.009 | INFO | "pinned" : " 29 GB" | 20260507 21:48:57.009 | INFO | } | 20260507 21:48:57.009 | INFO | Perf report for the last 5.13402 seconds: | 20260507 21:48:57.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:48:57.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.009 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 21:48:57.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.009 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:48:57.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:48:57.010 | INFO | Uploaded 36GiB to GPU in 2.51985s (14.2866GiB/s) | 20260507 21:48:57.010 | INFO | Downloaded 34GiB to GPU in 2.45471s (13.8509GiB/s) | 20260507 21:49:02.147 | INFO | Analyzing chunk frames = [2850304,2850816) | 20260507 21:49:02.147 | INFO | MemInfo: { | 20260507 21:49:02.147 | INFO | "gpu" : " 17 GB", | 20260507 21:49:02.147 | INFO | "huge" : "", | 20260507 21:49:02.147 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:02.147 | INFO | "malloc" : " 14 GB", | 20260507 21:49:02.147 | INFO | "pinned" : " 29 GB" | 20260507 21:49:02.147 | INFO | } | 20260507 21:49:02.147 | INFO | Perf report for the last 5.1379 seconds: | 20260507 21:49:02.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:02.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.147 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:49:02.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.147 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 21:49:02.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:02.148 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:49:02.148 | INFO | Total mem usage: 16.7 GiB | 20260507 21:49:02.148 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:49:02.148 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.8%) | 20260507 21:49:02.148 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:49:02.148 | INFO | High Water Breakdown: | 20260507 21:49:02.148 | INFO | 60.2% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:49:02.148 | INFO | 39.7% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:49:02.148 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:49:02.148 | INFO | ------------------------------------------ | 20260507 21:49:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:49:02.148 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:49:02.148 | INFO | Total mem usage: 26.5 GiB | 20260507 21:49:02.148 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:49:02.148 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:49:02.148 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:49:02.148 | INFO | High Water Breakdown: | 20260507 21:49:02.148 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:49:02.148 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:49:02.148 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:49:02.148 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:49:02.148 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:49:02.148 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:49:02.148 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:49:02.148 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:49:02.148 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:49:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:49:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:49:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:49:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:49:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:49:02.148 | INFO | ------------------------------------------ | 20260507 21:49:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:49:02.148 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:49:02.148 | INFO | Total mem usage: 27.8 GiB | 20260507 21:49:02.148 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:49:02.148 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:49:02.148 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:49:02.148 | INFO | High Water Breakdown: | 20260507 21:49:02.148 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:49:02.148 | INFO | ------------------------------------------ | 20260507 21:49:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:49:02.148 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:49:02.148 | INFO | Total mem usage: 15.7 GiB | 20260507 21:49:02.148 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:49:02.148 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:49:02.148 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:49:02.148 | INFO | High Water Breakdown: | 20260507 21:49:02.148 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:49:02.148 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:49:02.148 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:49:02.148 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:49:02.148 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:49:02.148 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:49:02.148 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:49:02.148 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:49:02.148 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:49:02.148 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:49:02.148 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:49:02.148 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:49:02.148 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:49:02.148 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:49:02.148 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:49:02.148 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:49:02.148 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:49:02.148 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:49:02.148 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:49:02.148 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:49:02.148 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:49:02.148 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:49:02.148 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:49:02.148 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:49:02.148 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:49:02.148 | INFO | ------------------------------------------ | 20260507 21:49:02.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:49:02.148 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:49:02.148 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:49:02.148 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:49:02.148 | INFO | Uploaded 36GiB to GPU in 2.4968s (14.4185GiB/s) | 20260507 21:49:02.148 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.40e+01 / 1.64e+00 | 20260507 21:49:02.148 | INFO | Downloaded 34GiB to GPU in 2.44274s (13.9188GiB/s) | 20260507 21:49:02.148 | INFO | PerfReport: 0.29%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.10e-02 / 5.93e-03 | 20260507 21:49:02.148 | INFO | PerfReport: 12.17%, Baseline -- 4.02e+00 / 4.50e+00, 6.04e-01 / 2.38e-01 | 20260507 21:49:02.149 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.11e-01 / 1.01e-01 | 20260507 21:49:02.149 | INFO | PerfReport: 15.85%, DME -- 4.19e+01 / 4.53e+01, 3.96e+00 / 1.63e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 2.17e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.30e-01 / 2.05e-01 | 20260507 21:49:02.149 | INFO | PerfReport: 3.50%, Metrics -- 1.16e+00 / 1.37e+00, 2.96e-01 / 1.93e-01 | 20260507 21:49:02.149 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.58e-01 / 3.92e-01, 5.91e-02 / 3.47e-02 | 20260507 21:49:02.149 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:49:02.149 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:49:02.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:49:02.149 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 24.16%, Upload -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 24.32%, Download -- 2.60e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 30.96%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 1.80%, LockUpload -- 1.93e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 2.73%, LockDownload -- 2.92e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 16.00%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 21:49:02.149 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 21:49:02.248 | INFO | Starting a round of Baz IO | 20260507 21:49:05.712 | INFO | Populating RTMetrics report | 20260507 21:49:05.713 | INFO | RTMetrics report populated | 20260507 21:49:05.713 | INFO | Opening tmp RTMetrics json file | 20260507 21:49:05.714 | INFO | Serializing RTMetrics to json | 20260507 21:49:05.730 | INFO | Serialization done, swapping tmp file now... | 20260507 21:49:05.748 | INFO | RTMetrics done | 20260507 21:49:05.837 | INFO | Finished a round of Baz IO: Wrote 1.751 GiB in 3.589 seconds | 20260507 21:49:05.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.49% / 0.01% / 56.51% | 20260507 21:49:05.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58049% / 1.19209e-05% / 7.15256e-05% | 20260507 21:49:05.837 | INFO | Baz IO overall: | 20260507 21:49:05.837 | INFO | BytesWritten : 1298498598774 | 20260507 21:49:05.837 | INFO | NumEvents : 629266129065 | 20260507 21:49:07.292 | INFO | Analyzing chunk frames = [2850816,2851328) | 20260507 21:49:07.292 | INFO | MemInfo: { | 20260507 21:49:07.292 | INFO | "gpu" : " 17 GB", | 20260507 21:49:07.292 | INFO | "huge" : "", | 20260507 21:49:07.292 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:07.292 | INFO | "malloc" : " 14 GB", | 20260507 21:49:07.292 | INFO | "pinned" : " 29 GB" | 20260507 21:49:07.292 | INFO | } | 20260507 21:49:07.292 | INFO | Perf report for the last 5.14441 seconds: | 20260507 21:49:07.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:07.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:07.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:07.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:07.292 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 21:49:07.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:07.292 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 21:49:07.292 | INFO | PRE_HQ : 65.9%, 1.7, 57.8ms, 0 | 20260507 21:49:07.292 | INFO | BAZWRITER : 69.4%, 3.7, 131.4ms, 0 | 20260507 21:49:07.292 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 21:49:07.292 | INFO | Uploaded 36GiB to GPU in 2.56409s (14.0401GiB/s) | 20260507 21:49:07.292 | INFO | Downloaded 38GiB to GPU in 2.70221s (14.0626GiB/s) | 20260507 21:49:12.431 | INFO | Analyzing chunk frames = [2851328,2851840) | 20260507 21:49:12.431 | INFO | MemInfo: { | 20260507 21:49:12.431 | INFO | "gpu" : " 17 GB", | 20260507 21:49:12.431 | INFO | "huge" : "", | 20260507 21:49:12.431 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:12.431 | INFO | "malloc" : " 14 GB", | 20260507 21:49:12.431 | INFO | "pinned" : " 29 GB" | 20260507 21:49:12.431 | INFO | } | 20260507 21:49:12.431 | INFO | Perf report for the last 5.13984 seconds: | 20260507 21:49:12.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:12.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.432 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 21:49:12.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.432 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:49:12.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:12.432 | INFO | Uploaded 36GiB to GPU in 2.53986s (14.174GiB/s) | 20260507 21:49:12.432 | INFO | Downloaded 34GiB to GPU in 2.4836s (13.6898GiB/s) | 20260507 21:49:17.567 | INFO | Analyzing chunk frames = [2851840,2852352) | 20260507 21:49:17.567 | INFO | MemInfo: { | 20260507 21:49:17.567 | INFO | "gpu" : " 17 GB", | 20260507 21:49:17.567 | INFO | "huge" : "", | 20260507 21:49:17.567 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:17.567 | INFO | "malloc" : " 14 GB", | 20260507 21:49:17.567 | INFO | "pinned" : " 29 GB" | 20260507 21:49:17.567 | INFO | } | 20260507 21:49:17.567 | INFO | Perf report for the last 5.13587 seconds: | 20260507 21:49:17.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:17.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.567 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:49:17.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.567 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:49:17.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:17.568 | INFO | Uploaded 36GiB to GPU in 2.52217s (14.2734GiB/s) | 20260507 21:49:17.568 | INFO | Downloaded 34GiB to GPU in 2.4567s (13.8397GiB/s) | 20260507 21:49:22.706 | INFO | Analyzing chunk frames = [2852352,2852864) | 20260507 21:49:22.706 | INFO | MemInfo: { | 20260507 21:49:22.706 | INFO | "gpu" : " 17 GB", | 20260507 21:49:22.706 | INFO | "huge" : "", | 20260507 21:49:22.706 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:22.706 | INFO | "malloc" : " 14 GB", | 20260507 21:49:22.706 | INFO | "pinned" : " 29 GB" | 20260507 21:49:22.706 | INFO | } | 20260507 21:49:22.706 | INFO | Perf report for the last 5.13871 seconds: | 20260507 21:49:22.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:22.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:49:22.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:49:22.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:22.706 | INFO | Uploaded 36GiB to GPU in 2.47545s (14.5428GiB/s) | 20260507 21:49:22.706 | INFO | Downloaded 34GiB to GPU in 2.38947s (14.2291GiB/s) | 20260507 21:49:27.846 | INFO | Analyzing chunk frames = [2852864,2853376) | 20260507 21:49:27.846 | INFO | MemInfo: { | 20260507 21:49:27.846 | INFO | "gpu" : " 17 GB", | 20260507 21:49:27.846 | INFO | "huge" : "", | 20260507 21:49:27.846 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:27.846 | INFO | "malloc" : " 14 GB", | 20260507 21:49:27.846 | INFO | "pinned" : " 29 GB" | 20260507 21:49:27.846 | INFO | } | 20260507 21:49:27.846 | INFO | Perf report for the last 5.14024 seconds: | 20260507 21:49:27.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:27.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.846 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 21:49:27.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.846 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 21:49:27.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:27.847 | INFO | Uploaded 36GiB to GPU in 2.32461s (15.4865GiB/s) | 20260507 21:49:27.847 | INFO | Downloaded 34GiB to GPU in 2.50314s (13.5829GiB/s) | 20260507 21:49:32.984 | INFO | Analyzing chunk frames = [2853376,2853888) | 20260507 21:49:32.984 | INFO | MemInfo: { | 20260507 21:49:32.984 | INFO | "gpu" : " 17 GB", | 20260507 21:49:32.984 | INFO | "huge" : "", | 20260507 21:49:32.984 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:32.984 | INFO | "malloc" : " 14 GB", | 20260507 21:49:32.984 | INFO | "pinned" : " 29 GB" | 20260507 21:49:32.985 | INFO | } | 20260507 21:49:32.985 | INFO | Perf report for the last 5.13838 seconds: | 20260507 21:49:32.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:32.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 21:49:32.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 21:49:32.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:32.985 | INFO | Uploaded 36GiB to GPU in 2.49814s (14.4107GiB/s) | 20260507 21:49:32.985 | INFO | Downloaded 34GiB to GPU in 2.42324s (14.0308GiB/s) | 20260507 21:49:38.133 | INFO | Analyzing chunk frames = [2853888,2854400) | 20260507 21:49:38.133 | INFO | MemInfo: { | 20260507 21:49:38.133 | INFO | "gpu" : " 17 GB", | 20260507 21:49:38.133 | INFO | "huge" : "", | 20260507 21:49:38.133 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:38.133 | INFO | "malloc" : " 14 GB", | 20260507 21:49:38.133 | INFO | "pinned" : " 29 GB" | 20260507 21:49:38.133 | INFO | } | 20260507 21:49:38.133 | INFO | Perf report for the last 5.14854 seconds: | 20260507 21:49:38.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:38.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.133 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 21:49:38.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.134 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 21:49:38.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:38.134 | INFO | Uploaded 36GiB to GPU in 2.52297s (14.2689GiB/s) | 20260507 21:49:38.134 | INFO | Downloaded 34GiB to GPU in 2.45856s (13.8292GiB/s) | 20260507 21:49:43.270 | INFO | Analyzing chunk frames = [2854400,2854912) | 20260507 21:49:43.270 | INFO | MemInfo: { | 20260507 21:49:43.270 | INFO | "gpu" : " 17 GB", | 20260507 21:49:43.270 | INFO | "huge" : "", | 20260507 21:49:43.270 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:43.270 | INFO | "malloc" : " 14 GB", | 20260507 21:49:43.270 | INFO | "pinned" : " 29 GB" | 20260507 21:49:43.270 | INFO | } | 20260507 21:49:43.271 | INFO | Perf report for the last 5.13731 seconds: | 20260507 21:49:43.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:43.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:49:43.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 21:49:43.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:43.271 | INFO | Uploaded 36GiB to GPU in 2.50608s (14.365GiB/s) | 20260507 21:49:43.271 | INFO | Downloaded 34GiB to GPU in 2.44651s (13.8974GiB/s) | 20260507 21:49:43.383 | INFO | Starting a round of Baz IO | 20260507 21:49:46.886 | INFO | Populating RTMetrics report | 20260507 21:49:46.887 | INFO | RTMetrics report populated | 20260507 21:49:46.887 | INFO | Opening tmp RTMetrics json file | 20260507 21:49:46.928 | INFO | Serializing RTMetrics to json | 20260507 21:49:46.999 | INFO | Serialization done, swapping tmp file now... | 20260507 21:49:47.000 | INFO | RTMetrics done | 20260507 21:49:47.028 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.644 seconds | 20260507 21:49:47.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.48% / 0.01% / 56.51% | 20260507 21:49:47.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5747% / 7.94729e-06% / 5.5631e-05% | 20260507 21:49:47.028 | INFO | Baz IO overall: | 20260507 21:49:47.028 | INFO | BytesWritten : 1300377448430 | 20260507 21:49:47.028 | INFO | NumEvents : 630173752080 | 20260507 21:49:48.408 | INFO | Analyzing chunk frames = [2854912,2855424) | 20260507 21:49:48.408 | INFO | MemInfo: { | 20260507 21:49:48.408 | INFO | "gpu" : " 17 GB", | 20260507 21:49:48.408 | INFO | "huge" : "", | 20260507 21:49:48.408 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:48.408 | INFO | "malloc" : " 14 GB", | 20260507 21:49:48.408 | INFO | "pinned" : " 29 GB" | 20260507 21:49:48.408 | INFO | } | 20260507 21:49:48.408 | INFO | Perf report for the last 5.13727 seconds: | 20260507 21:49:48.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:48.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:48.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:48.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:48.408 | INFO | ANALYSIS : 69.3%, 2.9, 104.4ms, 0 | 20260507 21:49:48.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:48.408 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260507 21:49:48.408 | INFO | PRE_HQ : 68.4%, 1.9, 65.9ms, 0 | 20260507 21:49:48.408 | INFO | BAZWRITER : 70.5%, 4.7, 169.7ms, 0 | 20260507 21:49:48.408 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 21:49:48.408 | INFO | Uploaded 36GiB to GPU in 2.55579s (14.0856GiB/s) | 20260507 21:49:48.409 | INFO | Downloaded 38GiB to GPU in 2.68896s (14.1319GiB/s) | 20260507 21:49:53.545 | INFO | Analyzing chunk frames = [2855424,2855936) | 20260507 21:49:53.545 | INFO | MemInfo: { | 20260507 21:49:53.545 | INFO | "gpu" : " 17 GB", | 20260507 21:49:53.545 | INFO | "huge" : "", | 20260507 21:49:53.545 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:53.545 | INFO | "malloc" : " 14 GB", | 20260507 21:49:53.545 | INFO | "pinned" : " 29 GB" | 20260507 21:49:53.545 | INFO | } | 20260507 21:49:53.545 | INFO | Perf report for the last 5.1371 seconds: | 20260507 21:49:53.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:53.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.545 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260507 21:49:53.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.545 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 21:49:53.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:53.546 | INFO | Uploaded 36GiB to GPU in 2.59616s (13.8666GiB/s) | 20260507 21:49:53.546 | INFO | Downloaded 34GiB to GPU in 2.47196s (13.7543GiB/s) | 20260507 21:49:58.687 | INFO | Analyzing chunk frames = [2855936,2856448) | 20260507 21:49:58.687 | INFO | MemInfo: { | 20260507 21:49:58.687 | INFO | "gpu" : " 17 GB", | 20260507 21:49:58.687 | INFO | "huge" : "", | 20260507 21:49:58.687 | INFO | "huge_shared" : " 29 GB", | 20260507 21:49:58.687 | INFO | "malloc" : " 14 GB", | 20260507 21:49:58.687 | INFO | "pinned" : " 29 GB" | 20260507 21:49:58.687 | INFO | } | 20260507 21:49:58.687 | INFO | Perf report for the last 5.14235 seconds: | 20260507 21:49:58.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:49:58.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.687 | INFO | ANALYSIS : 67.7%, 3.0, 105.1ms, 0 | 20260507 21:49:58.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.688 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 21:49:58.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:49:58.688 | INFO | Uploaded 36GiB to GPU in 2.60397s (13.825GiB/s) | 20260507 21:49:58.688 | INFO | Downloaded 34GiB to GPU in 2.46736s (13.7799GiB/s) | 20260507 21:50:03.828 | INFO | Analyzing chunk frames = [2856448,2856960) | 20260507 21:50:03.828 | INFO | MemInfo: { | 20260507 21:50:03.828 | INFO | "gpu" : " 17 GB", | 20260507 21:50:03.828 | INFO | "huge" : "", | 20260507 21:50:03.828 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:03.828 | INFO | "malloc" : " 14 GB", | 20260507 21:50:03.828 | INFO | "pinned" : " 29 GB" | 20260507 21:50:03.828 | INFO | } | 20260507 21:50:03.828 | INFO | Perf report for the last 5.14099 seconds: | 20260507 21:50:03.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:03.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.828 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 21:50:03.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.829 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 21:50:03.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:03.829 | INFO | Uploaded 36GiB to GPU in 2.55489s (14.0906GiB/s) | 20260507 21:50:03.829 | INFO | Downloaded 34GiB to GPU in 2.48601s (13.6766GiB/s) | 20260507 21:50:08.960 | INFO | Analyzing chunk frames = [2856960,2857472) | 20260507 21:50:08.960 | INFO | MemInfo: { | 20260507 21:50:08.960 | INFO | "gpu" : " 17 GB", | 20260507 21:50:08.960 | INFO | "huge" : "", | 20260507 21:50:08.960 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:08.960 | INFO | "malloc" : " 14 GB", | 20260507 21:50:08.960 | INFO | "pinned" : " 29 GB" | 20260507 21:50:08.960 | INFO | } | 20260507 21:50:08.960 | INFO | Perf report for the last 5.13173 seconds: | 20260507 21:50:08.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:08.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.960 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260507 21:50:08.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.960 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260507 21:50:08.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:08.961 | INFO | Uploaded 36GiB to GPU in 2.42474s (14.8469GiB/s) | 20260507 21:50:08.961 | INFO | Downloaded 34GiB to GPU in 2.50995s (13.5461GiB/s) | 20260507 21:50:14.101 | INFO | Analyzing chunk frames = [2857472,2857984) | 20260507 21:50:14.101 | INFO | MemInfo: { | 20260507 21:50:14.101 | INFO | "gpu" : " 17 GB", | 20260507 21:50:14.101 | INFO | "huge" : "", | 20260507 21:50:14.101 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:14.101 | INFO | "malloc" : " 14 GB", | 20260507 21:50:14.101 | INFO | "pinned" : " 29 GB" | 20260507 21:50:14.101 | INFO | } | 20260507 21:50:14.101 | INFO | Perf report for the last 5.14103 seconds: | 20260507 21:50:14.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:14.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:50:14.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:50:14.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:14.101 | INFO | Uploaded 36GiB to GPU in 2.49062s (14.4542GiB/s) | 20260507 21:50:14.102 | INFO | Downloaded 34GiB to GPU in 2.40427s (14.1415GiB/s) | 20260507 21:50:19.240 | INFO | Analyzing chunk frames = [2857984,2858496) | 20260507 21:50:19.240 | INFO | MemInfo: { | 20260507 21:50:19.240 | INFO | "gpu" : " 17 GB", | 20260507 21:50:19.240 | INFO | "huge" : "", | 20260507 21:50:19.240 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:19.240 | INFO | "malloc" : " 14 GB", | 20260507 21:50:19.240 | INFO | "pinned" : " 29 GB" | 20260507 21:50:19.240 | INFO | } | 20260507 21:50:19.240 | INFO | Perf report for the last 5.13875 seconds: | 20260507 21:50:19.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:19.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 21:50:19.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:50:19.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:19.240 | INFO | Uploaded 36GiB to GPU in 2.52381s (14.2642GiB/s) | 20260507 21:50:19.240 | INFO | Downloaded 34GiB to GPU in 2.43179s (13.9815GiB/s) | 20260507 21:50:24.379 | INFO | Analyzing chunk frames = [2858496,2859008) | 20260507 21:50:24.379 | INFO | MemInfo: { | 20260507 21:50:24.379 | INFO | "gpu" : " 17 GB", | 20260507 21:50:24.379 | INFO | "huge" : "", | 20260507 21:50:24.379 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:24.379 | INFO | "malloc" : " 14 GB", | 20260507 21:50:24.379 | INFO | "pinned" : " 29 GB" | 20260507 21:50:24.379 | INFO | } | 20260507 21:50:24.379 | INFO | Perf report for the last 5.13926 seconds: | 20260507 21:50:24.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:24.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:50:24.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:50:24.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:24.379 | INFO | Uploaded 36GiB to GPU in 2.52669s (14.2479GiB/s) | 20260507 21:50:24.380 | INFO | Downloaded 34GiB to GPU in 2.4322s (13.9791GiB/s) | 20260507 21:50:24.380 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:50:24.380 | INFO | Total mem usage: 16.7 GiB | 20260507 21:50:24.380 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:50:24.380 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.56%) | 20260507 21:50:24.380 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:50:24.380 | INFO | High Water Breakdown: | 20260507 21:50:24.380 | INFO | 60.5% : 9.29 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:50:24.380 | INFO | 39.4% : 6.06 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:50:24.380 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:50:24.380 | INFO | ------------------------------------------ | 20260507 21:50:24.380 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:50:24.380 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:50:24.380 | INFO | Total mem usage: 26.5 GiB | 20260507 21:50:24.380 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:50:24.380 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 21:50:24.380 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 21:50:24.380 | INFO | High Water Breakdown: | 20260507 21:50:24.380 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:50:24.380 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:50:24.380 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:50:24.380 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:50:24.380 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:50:24.380 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:50:24.380 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:50:24.380 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:50:24.380 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:50:24.380 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:50:24.380 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:50:24.380 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:50:24.380 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:50:24.380 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:50:24.380 | INFO | ------------------------------------------ | 20260507 21:50:24.380 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:50:24.380 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:50:24.380 | INFO | Total mem usage: 27.8 GiB | 20260507 21:50:24.380 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:50:24.381 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:50:24.381 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:50:24.381 | INFO | High Water Breakdown: | 20260507 21:50:24.381 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:50:24.381 | INFO | ------------------------------------------ | 20260507 21:50:24.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:50:24.381 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:50:24.381 | INFO | Total mem usage: 15.7 GiB | 20260507 21:50:24.381 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:50:24.381 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:50:24.381 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:50:24.381 | INFO | High Water Breakdown: | 20260507 21:50:24.381 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:50:24.381 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:50:24.381 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:50:24.381 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:50:24.381 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:50:24.381 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:50:24.381 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:50:24.381 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:50:24.381 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:50:24.381 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:50:24.381 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:50:24.381 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:50:24.381 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:50:24.381 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:50:24.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:50:24.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:50:24.381 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:50:24.381 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:50:24.381 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:50:24.381 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:50:24.381 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:50:24.381 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:50:24.381 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:50:24.381 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:50:24.381 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:50:24.381 | INFO | ------------------------------------------ | 20260507 21:50:24.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:50:24.381 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:50:24.381 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:50:24.381 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:50:24.381 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.61e+01, 1.41e+01 / 1.73e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 0.30%, AggregateStats -- 9.97e-02 / 1.05e-01, 1.12e-02 / 5.30e-03 | 20260507 21:50:24.381 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.48e+00, 5.94e-01 / 2.19e-01 | 20260507 21:50:24.381 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.10e-01 / 1.27e-01 | 20260507 21:50:24.381 | INFO | PerfReport: 15.87%, DME -- 4.20e+01 / 4.54e+01, 4.03e+00 / 1.87e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.82e+00 / 2.05e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.45e+00 / 2.76e+00, 4.35e-01 / 2.71e-01 | 20260507 21:50:24.381 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.35e+00, 2.97e-01 / 1.63e-01 | 20260507 21:50:24.381 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.61e-01 / 3.91e-01, 5.98e-02 / 2.78e-02 | 20260507 21:50:24.381 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:50:24.381 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:50:24.381 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:50:24.381 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 24.49%, Upload -- 2.62e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 24.20%, Download -- 2.59e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 30.92%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 2.20%, LockUpload -- 2.36e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 2.36%, LockDownload -- 2.53e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 21:50:24.381 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260507 21:50:24.476 | INFO | Starting a round of Baz IO | 20260507 21:50:27.946 | INFO | Populating RTMetrics report | 20260507 21:50:27.946 | INFO | RTMetrics report populated | 20260507 21:50:27.946 | INFO | Opening tmp RTMetrics json file | 20260507 21:50:28.016 | INFO | Serializing RTMetrics to json | 20260507 21:50:28.050 | INFO | Serialization done, swapping tmp file now... | 20260507 21:50:28.055 | INFO | RTMetrics done | 20260507 21:50:28.085 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.608 seconds | 20260507 21:50:28.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.47% / 0.01% / 56.52% | 20260507 21:50:28.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.57241% / 3.17891e-05% / 5.16574e-05% | 20260507 21:50:28.085 | INFO | Baz IO overall: | 20260507 21:50:28.085 | INFO | BytesWritten : 1302256730349 | 20260507 21:50:28.085 | INFO | NumEvents : 631081577111 | 20260507 21:50:29.522 | INFO | Analyzing chunk frames = [2859008,2859520) | 20260507 21:50:29.522 | INFO | MemInfo: { | 20260507 21:50:29.522 | INFO | "gpu" : " 17 GB", | 20260507 21:50:29.522 | INFO | "huge" : "", | 20260507 21:50:29.522 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:29.522 | INFO | "malloc" : " 14 GB", | 20260507 21:50:29.522 | INFO | "pinned" : " 29 GB" | 20260507 21:50:29.522 | INFO | } | 20260507 21:50:29.522 | INFO | Perf report for the last 5.14236 seconds: | 20260507 21:50:29.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:29.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:29.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:29.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:29.522 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 21:50:29.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:29.522 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 21:50:29.522 | INFO | PRE_HQ : 65.6%, 1.7, 57.7ms, 0 | 20260507 21:50:29.522 | INFO | BAZWRITER : 69.8%, 3.9, 139.6ms, 0 | 20260507 21:50:29.522 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 21:50:29.522 | INFO | Uploaded 36GiB to GPU in 2.53356s (14.2093GiB/s) | 20260507 21:50:29.522 | INFO | Downloaded 38GiB to GPU in 2.69257s (14.1129GiB/s) | 20260507 21:50:34.657 | INFO | Analyzing chunk frames = [2859520,2860032) | 20260507 21:50:34.657 | INFO | MemInfo: { | 20260507 21:50:34.657 | INFO | "gpu" : " 17 GB", | 20260507 21:50:34.657 | INFO | "huge" : "", | 20260507 21:50:34.657 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:34.657 | INFO | "malloc" : " 14 GB", | 20260507 21:50:34.657 | INFO | "pinned" : " 29 GB" | 20260507 21:50:34.657 | INFO | } | 20260507 21:50:34.657 | INFO | Perf report for the last 5.13586 seconds: | 20260507 21:50:34.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:34.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.658 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:50:34.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.658 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:50:34.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:34.658 | INFO | Uploaded 36GiB to GPU in 2.55108s (14.1117GiB/s) | 20260507 21:50:34.658 | INFO | Downloaded 34GiB to GPU in 2.4826s (13.6953GiB/s) | 20260507 21:50:39.796 | INFO | Analyzing chunk frames = [2860032,2860544) | 20260507 21:50:39.796 | INFO | MemInfo: { | 20260507 21:50:39.796 | INFO | "gpu" : " 17 GB", | 20260507 21:50:39.796 | INFO | "huge" : "", | 20260507 21:50:39.796 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:39.796 | INFO | "malloc" : " 14 GB", | 20260507 21:50:39.796 | INFO | "pinned" : " 29 GB" | 20260507 21:50:39.796 | INFO | } | 20260507 21:50:39.797 | INFO | Perf report for the last 5.13922 seconds: | 20260507 21:50:39.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:39.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 21:50:39.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:50:39.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:39.797 | INFO | Uploaded 36GiB to GPU in 2.56078s (14.0582GiB/s) | 20260507 21:50:39.797 | INFO | Downloaded 34GiB to GPU in 2.47644s (13.7294GiB/s) | 20260507 21:50:44.936 | INFO | Analyzing chunk frames = [2860544,2861056) | 20260507 21:50:44.936 | INFO | MemInfo: { | 20260507 21:50:44.936 | INFO | "gpu" : " 17 GB", | 20260507 21:50:44.936 | INFO | "huge" : "", | 20260507 21:50:44.936 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:44.936 | INFO | "malloc" : " 14 GB", | 20260507 21:50:44.936 | INFO | "pinned" : " 29 GB" | 20260507 21:50:44.936 | INFO | } | 20260507 21:50:44.936 | INFO | Perf report for the last 5.13967 seconds: | 20260507 21:50:44.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:44.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.936 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:50:44.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.936 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:50:44.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:44.937 | INFO | Uploaded 36GiB to GPU in 2.47433s (14.5494GiB/s) | 20260507 21:50:44.937 | INFO | Downloaded 34GiB to GPU in 2.39084s (14.221GiB/s) | 20260507 21:50:50.079 | INFO | Analyzing chunk frames = [2861056,2861568) | 20260507 21:50:50.079 | INFO | MemInfo: { | 20260507 21:50:50.079 | INFO | "gpu" : " 17 GB", | 20260507 21:50:50.079 | INFO | "huge" : "", | 20260507 21:50:50.079 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:50.079 | INFO | "malloc" : " 14 GB", | 20260507 21:50:50.079 | INFO | "pinned" : " 29 GB" | 20260507 21:50:50.079 | INFO | } | 20260507 21:50:50.079 | INFO | Perf report for the last 5.14255 seconds: | 20260507 21:50:50.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:50.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 21:50:50.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 21:50:50.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:50.079 | INFO | Uploaded 36GiB to GPU in 2.32152s (15.5071GiB/s) | 20260507 21:50:50.079 | INFO | Downloaded 34GiB to GPU in 2.50206s (13.5888GiB/s) | 20260507 21:50:55.216 | INFO | Analyzing chunk frames = [2861568,2862080) | 20260507 21:50:55.216 | INFO | MemInfo: { | 20260507 21:50:55.216 | INFO | "gpu" : " 17 GB", | 20260507 21:50:55.216 | INFO | "huge" : "", | 20260507 21:50:55.216 | INFO | "huge_shared" : " 29 GB", | 20260507 21:50:55.216 | INFO | "malloc" : " 14 GB", | 20260507 21:50:55.216 | INFO | "pinned" : " 29 GB" | 20260507 21:50:55.216 | INFO | } | 20260507 21:50:55.216 | INFO | Perf report for the last 5.13754 seconds: | 20260507 21:50:55.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:50:55.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:50:55.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:50:55.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:50:55.217 | INFO | Uploaded 36GiB to GPU in 2.48268s (14.5005GiB/s) | 20260507 21:50:55.217 | INFO | Downloaded 34GiB to GPU in 2.4441s (13.911GiB/s) | 20260507 21:51:00.357 | INFO | Analyzing chunk frames = [2862080,2862592) | 20260507 21:51:00.357 | INFO | MemInfo: { | 20260507 21:51:00.357 | INFO | "gpu" : " 17 GB", | 20260507 21:51:00.357 | INFO | "huge" : "", | 20260507 21:51:00.357 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:00.357 | INFO | "malloc" : " 14 GB", | 20260507 21:51:00.357 | INFO | "pinned" : " 29 GB" | 20260507 21:51:00.357 | INFO | } | 20260507 21:51:00.357 | INFO | Perf report for the last 5.14084 seconds: | 20260507 21:51:00.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:00.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.357 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 21:51:00.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.357 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:51:00.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:00.358 | INFO | Uploaded 36GiB to GPU in 2.48991s (14.4584GiB/s) | 20260507 21:51:00.358 | INFO | Downloaded 34GiB to GPU in 2.46091s (13.816GiB/s) | 20260507 21:51:05.499 | INFO | Analyzing chunk frames = [2862592,2863104) | 20260507 21:51:05.499 | INFO | MemInfo: { | 20260507 21:51:05.499 | INFO | "gpu" : " 17 GB", | 20260507 21:51:05.499 | INFO | "huge" : "", | 20260507 21:51:05.499 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:05.499 | INFO | "malloc" : " 14 GB", | 20260507 21:51:05.499 | INFO | "pinned" : " 29 GB" | 20260507 21:51:05.499 | INFO | } | 20260507 21:51:05.499 | INFO | Perf report for the last 5.14145 seconds: | 20260507 21:51:05.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:05.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.499 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:51:05.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.499 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:51:05.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:05.500 | INFO | Uploaded 36GiB to GPU in 2.47991s (14.5167GiB/s) | 20260507 21:51:05.500 | INFO | Downloaded 34GiB to GPU in 2.44908s (13.8828GiB/s) | 20260507 21:51:05.597 | INFO | Starting a round of Baz IO | 20260507 21:51:09.065 | INFO | Populating RTMetrics report | 20260507 21:51:09.066 | INFO | RTMetrics report populated | 20260507 21:51:09.066 | INFO | Opening tmp RTMetrics json file | 20260507 21:51:09.098 | INFO | Serializing RTMetrics to json | 20260507 21:51:09.174 | INFO | Serialization done, swapping tmp file now... | 20260507 21:51:09.180 | INFO | RTMetrics done | 20260507 21:51:09.212 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.614 seconds | 20260507 21:51:09.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.47% / 0.01% / 56.53% | 20260507 21:51:09.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56972% / 1.58946e-05% / 4.37101e-05% | 20260507 21:51:09.212 | INFO | Baz IO overall: | 20260507 21:51:09.212 | INFO | BytesWritten : 1304135501323 | 20260507 21:51:09.212 | INFO | NumEvents : 631989264947 | 20260507 21:51:10.635 | INFO | Analyzing chunk frames = [2863104,2863616) | 20260507 21:51:10.635 | INFO | MemInfo: { | 20260507 21:51:10.635 | INFO | "gpu" : " 17 GB", | 20260507 21:51:10.635 | INFO | "huge" : "", | 20260507 21:51:10.635 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:10.635 | INFO | "malloc" : " 14 GB", | 20260507 21:51:10.635 | INFO | "pinned" : " 29 GB" | 20260507 21:51:10.635 | INFO | } | 20260507 21:51:10.635 | INFO | Perf report for the last 5.13658 seconds: | 20260507 21:51:10.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:10.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:10.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:10.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:10.635 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260507 21:51:10.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:10.636 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260507 21:51:10.636 | INFO | PRE_HQ : 65.8%, 1.7, 57.3ms, 0 | 20260507 21:51:10.636 | INFO | BAZWRITER : 70.0%, 4.2, 149.5ms, 0 | 20260507 21:51:10.636 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 21:51:10.636 | INFO | Uploaded 36GiB to GPU in 2.50256s (14.3853GiB/s) | 20260507 21:51:10.636 | INFO | Downloaded 38GiB to GPU in 2.68501s (14.1527GiB/s) | 20260507 21:51:15.775 | INFO | Analyzing chunk frames = [2863616,2864128) | 20260507 21:51:15.775 | INFO | MemInfo: { | 20260507 21:51:15.775 | INFO | "gpu" : " 17 GB", | 20260507 21:51:15.775 | INFO | "huge" : "", | 20260507 21:51:15.775 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:15.775 | INFO | "malloc" : " 14 GB", | 20260507 21:51:15.775 | INFO | "pinned" : " 29 GB" | 20260507 21:51:15.775 | INFO | } | 20260507 21:51:15.775 | INFO | Perf report for the last 5.1394 seconds: | 20260507 21:51:15.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:15.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 21:51:15.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 21:51:15.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:15.775 | INFO | Uploaded 36GiB to GPU in 2.5252s (14.2563GiB/s) | 20260507 21:51:15.775 | INFO | Downloaded 34GiB to GPU in 2.49s (13.6546GiB/s) | 20260507 21:51:20.909 | INFO | Analyzing chunk frames = [2864128,2864640) | 20260507 21:51:20.909 | INFO | MemInfo: { | 20260507 21:51:20.909 | INFO | "gpu" : " 17 GB", | 20260507 21:51:20.909 | INFO | "huge" : "", | 20260507 21:51:20.909 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:20.909 | INFO | "malloc" : " 14 GB", | 20260507 21:51:20.909 | INFO | "pinned" : " 29 GB" | 20260507 21:51:20.910 | INFO | } | 20260507 21:51:20.910 | INFO | Perf report for the last 5.13491 seconds: | 20260507 21:51:20.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:20.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 21:51:20.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:51:20.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:20.910 | INFO | Uploaded 36GiB to GPU in 2.51106s (14.3366GiB/s) | 20260507 21:51:20.910 | INFO | Downloaded 34GiB to GPU in 2.47555s (13.7343GiB/s) | 20260507 21:51:26.049 | INFO | Analyzing chunk frames = [2864640,2865152) | 20260507 21:51:26.049 | INFO | MemInfo: { | 20260507 21:51:26.049 | INFO | "gpu" : " 17 GB", | 20260507 21:51:26.049 | INFO | "huge" : "", | 20260507 21:51:26.049 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:26.049 | INFO | "malloc" : " 14 GB", | 20260507 21:51:26.049 | INFO | "pinned" : " 29 GB" | 20260507 21:51:26.049 | INFO | } | 20260507 21:51:26.049 | INFO | Perf report for the last 5.13912 seconds: | 20260507 21:51:26.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:26.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 21:51:26.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:51:26.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:26.049 | INFO | Uploaded 36GiB to GPU in 2.43743s (14.7696GiB/s) | 20260507 21:51:26.050 | INFO | Downloaded 34GiB to GPU in 2.41314s (14.0895GiB/s) | 20260507 21:51:31.195 | INFO | Analyzing chunk frames = [2865152,2865664) | 20260507 21:51:31.195 | INFO | MemInfo: { | 20260507 21:51:31.195 | INFO | "gpu" : " 17 GB", | 20260507 21:51:31.195 | INFO | "huge" : "", | 20260507 21:51:31.195 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:31.195 | INFO | "malloc" : " 14 GB", | 20260507 21:51:31.195 | INFO | "pinned" : " 29 GB" | 20260507 21:51:31.195 | INFO | } | 20260507 21:51:31.195 | INFO | Perf report for the last 5.14649 seconds: | 20260507 21:51:31.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:31.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.195 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 21:51:31.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.196 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 21:51:31.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:31.196 | INFO | Uploaded 36GiB to GPU in 2.32507s (15.4834GiB/s) | 20260507 21:51:31.196 | INFO | Downloaded 34GiB to GPU in 2.50347s (13.5811GiB/s) | 20260507 21:51:36.333 | INFO | Analyzing chunk frames = [2865664,2866176) | 20260507 21:51:36.333 | INFO | MemInfo: { | 20260507 21:51:36.333 | INFO | "gpu" : " 17 GB", | 20260507 21:51:36.334 | INFO | "huge" : "", | 20260507 21:51:36.334 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:36.334 | INFO | "malloc" : " 14 GB", | 20260507 21:51:36.334 | INFO | "pinned" : " 29 GB" | 20260507 21:51:36.334 | INFO | } | 20260507 21:51:36.334 | INFO | Perf report for the last 5.13832 seconds: | 20260507 21:51:36.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:36.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 21:51:36.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:51:36.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:36.334 | INFO | Uploaded 36GiB to GPU in 2.46359s (14.6128GiB/s) | 20260507 21:51:36.334 | INFO | Downloaded 34GiB to GPU in 2.4272s (14.0079GiB/s) | 20260507 21:51:41.472 | INFO | Analyzing chunk frames = [2866176,2866688) | 20260507 21:51:41.472 | INFO | MemInfo: { | 20260507 21:51:41.472 | INFO | "gpu" : " 17 GB", | 20260507 21:51:41.472 | INFO | "huge" : "", | 20260507 21:51:41.472 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:41.472 | INFO | "malloc" : " 14 GB", | 20260507 21:51:41.472 | INFO | "pinned" : " 29 GB" | 20260507 21:51:41.472 | INFO | } | 20260507 21:51:41.472 | INFO | Perf report for the last 5.13852 seconds: | 20260507 21:51:41.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:41.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.472 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 21:51:41.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.472 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:51:41.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:41.473 | INFO | Uploaded 36GiB to GPU in 2.52146s (14.2775GiB/s) | 20260507 21:51:41.473 | INFO | Downloaded 34GiB to GPU in 2.45898s (13.8269GiB/s) | 20260507 21:51:46.612 | INFO | Analyzing chunk frames = [2866688,2867200) | 20260507 21:51:46.612 | INFO | MemInfo: { | 20260507 21:51:46.612 | INFO | "gpu" : " 17 GB", | 20260507 21:51:46.612 | INFO | "huge" : "", | 20260507 21:51:46.612 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:46.612 | INFO | "malloc" : " 14 GB", | 20260507 21:51:46.612 | INFO | "pinned" : " 29 GB" | 20260507 21:51:46.612 | INFO | } | 20260507 21:51:46.612 | INFO | Perf report for the last 5.13957 seconds: | 20260507 21:51:46.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:46.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 21:51:46.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:51:46.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:46.612 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:51:46.612 | INFO | Total mem usage: 16.7 GiB | 20260507 21:51:46.612 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:51:46.612 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.3%) | 20260507 21:51:46.612 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:51:46.612 | INFO | High Water Breakdown: | 20260507 21:51:46.612 | INFO | 59.9% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:51:46.612 | INFO | 39.9% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:51:46.612 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:51:46.612 | INFO | ------------------------------------------ | 20260507 21:51:46.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:51:46.612 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:51:46.612 | INFO | Total mem usage: 26.5 GiB | 20260507 21:51:46.612 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:51:46.612 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:51:46.612 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:51:46.612 | INFO | High Water Breakdown: | 20260507 21:51:46.612 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:51:46.612 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:51:46.612 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:51:46.612 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:51:46.612 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:51:46.612 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:51:46.612 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:51:46.612 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:51:46.612 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:51:46.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:51:46.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:51:46.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:51:46.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:51:46.612 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:51:46.612 | INFO | ------------------------------------------ | 20260507 21:51:46.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:51:46.612 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:51:46.612 | INFO | Total mem usage: 27.8 GiB | 20260507 21:51:46.612 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:51:46.612 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:51:46.612 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:51:46.612 | INFO | High Water Breakdown: | 20260507 21:51:46.612 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:51:46.612 | INFO | ------------------------------------------ | 20260507 21:51:46.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:51:46.612 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:51:46.612 | INFO | Total mem usage: 15.7 GiB | 20260507 21:51:46.612 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:51:46.612 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:51:46.612 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:51:46.612 | INFO | High Water Breakdown: | 20260507 21:51:46.612 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:51:46.612 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:51:46.612 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:51:46.612 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:51:46.612 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:51:46.612 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:51:46.612 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:51:46.612 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:51:46.612 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:51:46.612 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:51:46.612 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:51:46.612 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:51:46.612 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:51:46.612 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:51:46.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:51:46.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:51:46.612 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:51:46.612 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:51:46.612 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:51:46.612 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:51:46.612 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:51:46.612 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:51:46.612 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:51:46.612 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:51:46.612 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:51:46.612 | INFO | ------------------------------------------ | 20260507 21:51:46.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:51:46.612 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:51:46.612 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:51:46.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:51:46.612 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.58e+01, 1.39e+01 / 1.49e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 0.29%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.29e-02 / 4.57e-03 | 20260507 21:51:46.612 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.48e+00, 6.03e-01 / 2.33e-01 | 20260507 21:51:46.612 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 1.06e-01 | 20260507 21:51:46.612 | INFO | PerfReport: 15.79%, DME -- 4.18e+01 / 4.51e+01, 3.82e+00 / 1.33e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.83e+00 / 1.91e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.30e-01 / 2.28e-01 | 20260507 21:51:46.612 | INFO | PerfReport: 3.50%, Metrics -- 1.16e+00 / 1.38e+00, 2.95e-01 / 1.89e-01 | 20260507 21:51:46.612 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.58e-01 / 3.96e-01, 6.03e-02 / 3.86e-02 | 20260507 21:51:46.612 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:51:46.612 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:51:46.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:51:46.612 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 24.20%, Upload -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 30.98%, Compute -- 3.30e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 1.81%, LockUpload -- 1.93e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 2.67%, LockDownload -- 2.85e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 16.04%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 21:51:46.612 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260507 21:51:46.612 | INFO | Uploaded 36GiB to GPU in 2.46444s (14.6078GiB/s) | 20260507 21:51:46.614 | INFO | Downloaded 34GiB to GPU in 2.44011s (13.9338GiB/s) | 20260507 21:51:46.711 | INFO | Starting a round of Baz IO | 20260507 21:51:50.184 | INFO | Populating RTMetrics report | 20260507 21:51:50.185 | INFO | RTMetrics report populated | 20260507 21:51:50.185 | INFO | Opening tmp RTMetrics json file | 20260507 21:51:50.215 | INFO | Serializing RTMetrics to json | 20260507 21:51:50.234 | INFO | Serialization done, swapping tmp file now... | 20260507 21:51:50.260 | INFO | RTMetrics done | 20260507 21:51:50.290 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.578 seconds | 20260507 21:51:50.290 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.46% / 0.01% / 56.53% | 20260507 21:51:50.290 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56618% / 3.97364e-06% / 3.97364e-05% | 20260507 21:51:50.290 | INFO | Baz IO overall: | 20260507 21:51:50.290 | INFO | BytesWritten : 1306014919373 | 20260507 21:51:50.290 | INFO | NumEvents : 632897217945 | 20260507 21:51:51.749 | INFO | Analyzing chunk frames = [2867200,2867712) | 20260507 21:51:51.749 | INFO | MemInfo: { | 20260507 21:51:51.749 | INFO | "gpu" : " 17 GB", | 20260507 21:51:51.749 | INFO | "huge" : "", | 20260507 21:51:51.749 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:51.749 | INFO | "malloc" : " 14 GB", | 20260507 21:51:51.749 | INFO | "pinned" : " 29 GB" | 20260507 21:51:51.749 | INFO | } | 20260507 21:51:51.749 | INFO | Perf report for the last 5.13764 seconds: | 20260507 21:51:51.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:51.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:51.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:51.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:51.750 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 21:51:51.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:51.750 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 21:51:51.750 | INFO | PRE_HQ : 65.7%, 1.7, 57.4ms, 0 | 20260507 21:51:51.750 | INFO | BAZWRITER : 69.3%, 4.1, 145.4ms, 0 | 20260507 21:51:51.750 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:51:51.750 | INFO | Uploaded 36GiB to GPU in 2.55264s (14.103GiB/s) | 20260507 21:51:51.750 | INFO | Downloaded 38GiB to GPU in 2.68199s (14.1686GiB/s) | 20260507 21:51:56.887 | INFO | Analyzing chunk frames = [2867712,2868224) | 20260507 21:51:56.887 | INFO | MemInfo: { | 20260507 21:51:56.887 | INFO | "gpu" : " 17 GB", | 20260507 21:51:56.887 | INFO | "huge" : "", | 20260507 21:51:56.887 | INFO | "huge_shared" : " 29 GB", | 20260507 21:51:56.887 | INFO | "malloc" : " 14 GB", | 20260507 21:51:56.887 | INFO | "pinned" : " 29 GB" | 20260507 21:51:56.887 | INFO | } | 20260507 21:51:56.887 | INFO | Perf report for the last 5.13774 seconds: | 20260507 21:51:56.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:51:56.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 21:51:56.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 21:51:56.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:51:56.887 | INFO | Uploaded 36GiB to GPU in 2.53034s (14.2273GiB/s) | 20260507 21:51:56.888 | INFO | Downloaded 34GiB to GPU in 2.49559s (13.6241GiB/s) | 20260507 21:52:02.026 | INFO | Analyzing chunk frames = [2868224,2868736) | 20260507 21:52:02.027 | INFO | MemInfo: { | 20260507 21:52:02.027 | INFO | "gpu" : " 17 GB", | 20260507 21:52:02.027 | INFO | "huge" : "", | 20260507 21:52:02.027 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:02.027 | INFO | "malloc" : " 14 GB", | 20260507 21:52:02.027 | INFO | "pinned" : " 29 GB" | 20260507 21:52:02.027 | INFO | } | 20260507 21:52:02.027 | INFO | Perf report for the last 5.13954 seconds: | 20260507 21:52:02.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:02.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:52:02.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:52:02.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:02.027 | INFO | Uploaded 36GiB to GPU in 2.53078s (14.2249GiB/s) | 20260507 21:52:02.027 | INFO | Downloaded 34GiB to GPU in 2.46809s (13.7759GiB/s) | 20260507 21:52:07.170 | INFO | Analyzing chunk frames = [2868736,2869248) | 20260507 21:52:07.170 | INFO | MemInfo: { | 20260507 21:52:07.170 | INFO | "gpu" : " 17 GB", | 20260507 21:52:07.170 | INFO | "huge" : "", | 20260507 21:52:07.170 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:07.170 | INFO | "malloc" : " 14 GB", | 20260507 21:52:07.171 | INFO | "pinned" : " 29 GB" | 20260507 21:52:07.171 | INFO | } | 20260507 21:52:07.171 | INFO | Perf report for the last 5.14385 seconds: | 20260507 21:52:07.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:07.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 21:52:07.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:52:07.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:07.171 | INFO | Uploaded 36GiB to GPU in 2.45776s (14.6475GiB/s) | 20260507 21:52:07.171 | INFO | Downloaded 34GiB to GPU in 2.39631s (14.1885GiB/s) | 20260507 21:52:12.308 | INFO | Analyzing chunk frames = [2869248,2869760) | 20260507 21:52:12.308 | INFO | MemInfo: { | 20260507 21:52:12.308 | INFO | "gpu" : " 17 GB", | 20260507 21:52:12.308 | INFO | "huge" : "", | 20260507 21:52:12.308 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:12.308 | INFO | "malloc" : " 14 GB", | 20260507 21:52:12.308 | INFO | "pinned" : " 29 GB" | 20260507 21:52:12.308 | INFO | } | 20260507 21:52:12.308 | INFO | Perf report for the last 5.13728 seconds: | 20260507 21:52:12.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:12.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 21:52:12.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 21:52:12.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:12.308 | INFO | Uploaded 36GiB to GPU in 2.32467s (15.4861GiB/s) | 20260507 21:52:12.308 | INFO | Downloaded 34GiB to GPU in 2.50309s (13.5832GiB/s) | 20260507 21:52:17.447 | INFO | Analyzing chunk frames = [2869760,2870272) | 20260507 21:52:17.447 | INFO | MemInfo: { | 20260507 21:52:17.447 | INFO | "gpu" : " 17 GB", | 20260507 21:52:17.447 | INFO | "huge" : "", | 20260507 21:52:17.447 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:17.447 | INFO | "malloc" : " 14 GB", | 20260507 21:52:17.447 | INFO | "pinned" : " 29 GB" | 20260507 21:52:17.447 | INFO | } | 20260507 21:52:17.447 | INFO | Perf report for the last 5.13912 seconds: | 20260507 21:52:17.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:17.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:52:17.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:52:17.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:17.447 | INFO | Uploaded 36GiB to GPU in 2.50397s (14.3771GiB/s) | 20260507 21:52:17.447 | INFO | Downloaded 34GiB to GPU in 2.43001s (13.9917GiB/s) | 20260507 21:52:22.590 | INFO | Analyzing chunk frames = [2870272,2870784) | 20260507 21:52:22.590 | INFO | MemInfo: { | 20260507 21:52:22.590 | INFO | "gpu" : " 17 GB", | 20260507 21:52:22.590 | INFO | "huge" : "", | 20260507 21:52:22.590 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:22.590 | INFO | "malloc" : " 14 GB", | 20260507 21:52:22.590 | INFO | "pinned" : " 29 GB" | 20260507 21:52:22.590 | INFO | } | 20260507 21:52:22.590 | INFO | Perf report for the last 5.1432 seconds: | 20260507 21:52:22.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:22.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.590 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 21:52:22.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.591 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 21:52:22.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:22.591 | INFO | Uploaded 36GiB to GPU in 2.50713s (14.359GiB/s) | 20260507 21:52:22.591 | INFO | Downloaded 34GiB to GPU in 2.4622s (13.8088GiB/s) | 20260507 21:52:27.724 | INFO | Analyzing chunk frames = [2870784,2871296) | 20260507 21:52:27.724 | INFO | MemInfo: { | 20260507 21:52:27.724 | INFO | "gpu" : " 17 GB", | 20260507 21:52:27.725 | INFO | "huge" : "", | 20260507 21:52:27.725 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:27.725 | INFO | "malloc" : " 14 GB", | 20260507 21:52:27.725 | INFO | "pinned" : " 29 GB" | 20260507 21:52:27.725 | INFO | } | 20260507 21:52:27.725 | INFO | Perf report for the last 5.13476 seconds: | 20260507 21:52:27.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:27.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.725 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:52:27.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.725 | INFO | PRE_HQ : 36.7%, 1.0, 19.1ms, 0 | 20260507 21:52:27.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:27.726 | INFO | Uploaded 36GiB to GPU in 2.51968s (14.2875GiB/s) | 20260507 21:52:27.726 | INFO | Downloaded 34GiB to GPU in 2.45102s (13.8718GiB/s) | 20260507 21:52:27.840 | INFO | Starting a round of Baz IO | 20260507 21:52:31.316 | INFO | Populating RTMetrics report | 20260507 21:52:31.317 | INFO | RTMetrics report populated | 20260507 21:52:31.317 | INFO | Opening tmp RTMetrics json file | 20260507 21:52:31.320 | INFO | Serializing RTMetrics to json | 20260507 21:52:31.325 | INFO | Serialization done, swapping tmp file now... | 20260507 21:52:31.326 | INFO | RTMetrics done | 20260507 21:52:31.429 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.588 seconds | 20260507 21:52:31.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.45% / 0.01% / 56.54% | 20260507 21:52:31.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5627% / 7.94729e-06% / 7.15256e-05% | 20260507 21:52:31.429 | INFO | Baz IO overall: | 20260507 21:52:31.429 | INFO | BytesWritten : 1307893590277 | 20260507 21:52:31.429 | INFO | NumEvents : 633804668975 | 20260507 21:52:32.859 | INFO | Analyzing chunk frames = [2871296,2871808) | 20260507 21:52:32.859 | INFO | MemInfo: { | 20260507 21:52:32.859 | INFO | "gpu" : " 17 GB", | 20260507 21:52:32.859 | INFO | "huge" : "", | 20260507 21:52:32.859 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:32.859 | INFO | "malloc" : " 14 GB", | 20260507 21:52:32.859 | INFO | "pinned" : " 29 GB" | 20260507 21:52:32.859 | INFO | } | 20260507 21:52:32.859 | INFO | Perf report for the last 5.13411 seconds: | 20260507 21:52:32.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:32.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:32.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:32.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:32.859 | INFO | ANALYSIS : 69.4%, 2.9, 104.3ms, 0 | 20260507 21:52:32.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:32.859 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 21:52:32.859 | INFO | PRE_HQ : 68.7%, 1.8, 65.9ms, 0 | 20260507 21:52:32.859 | INFO | BAZWRITER : 69.5%, 4.6, 164.9ms, 0 | 20260507 21:52:32.859 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 21:52:32.859 | INFO | Uploaded 36GiB to GPU in 2.53676s (14.1913GiB/s) | 20260507 21:52:32.859 | INFO | Downloaded 38GiB to GPU in 2.69322s (14.1095GiB/s) | 20260507 21:52:38.000 | INFO | Analyzing chunk frames = [2871808,2872320) | 20260507 21:52:38.000 | INFO | MemInfo: { | 20260507 21:52:38.000 | INFO | "gpu" : " 17 GB", | 20260507 21:52:38.000 | INFO | "huge" : "", | 20260507 21:52:38.000 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:38.000 | INFO | "malloc" : " 14 GB", | 20260507 21:52:38.000 | INFO | "pinned" : " 29 GB" | 20260507 21:52:38.000 | INFO | } | 20260507 21:52:38.000 | INFO | Perf report for the last 5.14113 seconds: | 20260507 21:52:38.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:38.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.000 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 21:52:38.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.000 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 21:52:38.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:38.001 | INFO | Uploaded 36GiB to GPU in 2.5547s (14.0917GiB/s) | 20260507 21:52:38.001 | INFO | Downloaded 34GiB to GPU in 2.45009s (13.8771GiB/s) | 20260507 21:52:43.144 | INFO | Analyzing chunk frames = [2872320,2872832) | 20260507 21:52:43.144 | INFO | MemInfo: { | 20260507 21:52:43.144 | INFO | "gpu" : " 17 GB", | 20260507 21:52:43.144 | INFO | "huge" : "", | 20260507 21:52:43.144 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:43.144 | INFO | "malloc" : " 14 GB", | 20260507 21:52:43.144 | INFO | "pinned" : " 29 GB" | 20260507 21:52:43.144 | INFO | } | 20260507 21:52:43.144 | INFO | Perf report for the last 5.14374 seconds: | 20260507 21:52:43.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:43.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.144 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 21:52:43.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.144 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:52:43.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:43.145 | INFO | Uploaded 36GiB to GPU in 2.56641s (14.0274GiB/s) | 20260507 21:52:43.145 | INFO | Downloaded 34GiB to GPU in 2.43774s (13.9474GiB/s) | 20260507 21:52:48.282 | INFO | Analyzing chunk frames = [2872832,2873344) | 20260507 21:52:48.282 | INFO | MemInfo: { | 20260507 21:52:48.282 | INFO | "gpu" : " 17 GB", | 20260507 21:52:48.282 | INFO | "huge" : "", | 20260507 21:52:48.282 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:48.282 | INFO | "malloc" : " 14 GB", | 20260507 21:52:48.282 | INFO | "pinned" : " 29 GB" | 20260507 21:52:48.282 | INFO | } | 20260507 21:52:48.282 | INFO | Perf report for the last 5.13807 seconds: | 20260507 21:52:48.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:48.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.282 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 21:52:48.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.282 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:52:48.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:48.283 | INFO | Uploaded 36GiB to GPU in 2.49682s (14.4183GiB/s) | 20260507 21:52:48.283 | INFO | Downloaded 34GiB to GPU in 2.3894s (14.2295GiB/s) | 20260507 21:52:53.422 | INFO | Analyzing chunk frames = [2873344,2873856) | 20260507 21:52:53.422 | INFO | MemInfo: { | 20260507 21:52:53.422 | INFO | "gpu" : " 17 GB", | 20260507 21:52:53.422 | INFO | "huge" : "", | 20260507 21:52:53.422 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:53.422 | INFO | "malloc" : " 14 GB", | 20260507 21:52:53.422 | INFO | "pinned" : " 29 GB" | 20260507 21:52:53.422 | INFO | } | 20260507 21:52:53.422 | INFO | Perf report for the last 5.13998 seconds: | 20260507 21:52:53.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:53.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.422 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:52:53.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.422 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:52:53.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:53.423 | INFO | Uploaded 36GiB to GPU in 2.3795s (15.1292GiB/s) | 20260507 21:52:53.423 | INFO | Downloaded 34GiB to GPU in 2.49177s (13.6449GiB/s) | 20260507 21:52:58.563 | INFO | Analyzing chunk frames = [2873856,2874368) | 20260507 21:52:58.563 | INFO | MemInfo: { | 20260507 21:52:58.563 | INFO | "gpu" : " 17 GB", | 20260507 21:52:58.563 | INFO | "huge" : "", | 20260507 21:52:58.563 | INFO | "huge_shared" : " 29 GB", | 20260507 21:52:58.563 | INFO | "malloc" : " 14 GB", | 20260507 21:52:58.563 | INFO | "pinned" : " 29 GB" | 20260507 21:52:58.563 | INFO | } | 20260507 21:52:58.563 | INFO | Perf report for the last 5.14109 seconds: | 20260507 21:52:58.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:52:58.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.564 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 21:52:58.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.564 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 21:52:58.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:52:58.564 | INFO | Uploaded 36GiB to GPU in 2.52158s (14.2767GiB/s) | 20260507 21:52:58.564 | INFO | Downloaded 34GiB to GPU in 2.41272s (14.092GiB/s) | 20260507 21:53:03.701 | INFO | Analyzing chunk frames = [2874368,2874880) | 20260507 21:53:03.701 | INFO | MemInfo: { | 20260507 21:53:03.701 | INFO | "gpu" : " 17 GB", | 20260507 21:53:03.701 | INFO | "huge" : "", | 20260507 21:53:03.701 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:03.701 | INFO | "malloc" : " 14 GB", | 20260507 21:53:03.701 | INFO | "pinned" : " 29 GB" | 20260507 21:53:03.701 | INFO | } | 20260507 21:53:03.702 | INFO | Perf report for the last 5.13814 seconds: | 20260507 21:53:03.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:03.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 21:53:03.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:53:03.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:03.702 | INFO | Uploaded 36GiB to GPU in 2.53712s (14.1893GiB/s) | 20260507 21:53:03.702 | INFO | Downloaded 34GiB to GPU in 2.43154s (13.9829GiB/s) | 20260507 21:53:08.839 | INFO | Analyzing chunk frames = [2874880,2875392) | 20260507 21:53:08.839 | INFO | MemInfo: { | 20260507 21:53:08.839 | INFO | "gpu" : " 17 GB", | 20260507 21:53:08.839 | INFO | "huge" : "", | 20260507 21:53:08.839 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:08.839 | INFO | "malloc" : " 14 GB", | 20260507 21:53:08.839 | INFO | "pinned" : " 29 GB" | 20260507 21:53:08.839 | INFO | } | 20260507 21:53:08.840 | INFO | Perf report for the last 5.13802 seconds: | 20260507 21:53:08.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:08.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 21:53:08.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | PRE_HQ : 36.5%, 1.0, 18.9ms, 0 | 20260507 21:53:08.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:08.840 | INFO | Uploaded 36GiB to GPU in 2.53295s (14.2127GiB/s) | 20260507 21:53:08.840 | INFO | Downloaded 34GiB to GPU in 2.43897s (13.9403GiB/s) | 20260507 21:53:08.840 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:53:08.840 | INFO | Total mem usage: 16.7 GiB | 20260507 21:53:08.840 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:53:08.840 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.54%) | 20260507 21:53:08.840 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:53:08.840 | INFO | High Water Breakdown: | 20260507 21:53:08.840 | INFO | 60.5% : 9.3 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:53:08.840 | INFO | 39.4% : 6.05 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:53:08.840 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:53:08.840 | INFO | ------------------------------------------ | 20260507 21:53:08.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:53:08.840 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:53:08.840 | INFO | Total mem usage: 26.5 GiB | 20260507 21:53:08.840 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:53:08.840 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:53:08.840 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:53:08.840 | INFO | High Water Breakdown: | 20260507 21:53:08.840 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:53:08.840 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:53:08.840 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:53:08.840 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:53:08.840 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:53:08.840 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:53:08.840 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:53:08.840 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:53:08.840 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:53:08.840 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:53:08.840 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:53:08.840 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:53:08.840 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:53:08.840 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:53:08.840 | INFO | ------------------------------------------ | 20260507 21:53:08.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:53:08.840 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:53:08.840 | INFO | Total mem usage: 27.8 GiB | 20260507 21:53:08.840 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:53:08.840 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:53:08.840 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:53:08.840 | INFO | High Water Breakdown: | 20260507 21:53:08.840 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:53:08.840 | INFO | ------------------------------------------ | 20260507 21:53:08.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:53:08.840 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:53:08.840 | INFO | Total mem usage: 15.7 GiB | 20260507 21:53:08.840 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:53:08.841 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:53:08.841 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:53:08.841 | INFO | High Water Breakdown: | 20260507 21:53:08.841 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:53:08.841 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:53:08.841 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:53:08.841 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:53:08.841 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:53:08.841 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:53:08.841 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:53:08.841 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:53:08.841 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:53:08.841 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:53:08.841 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:53:08.841 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:53:08.841 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:53:08.841 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:53:08.841 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:53:08.841 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:53:08.841 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:53:08.841 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:53:08.841 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:53:08.841 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:53:08.841 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:53:08.841 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:53:08.841 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:53:08.841 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:53:08.841 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:53:08.841 | INFO | ------------------------------------------ | 20260507 21:53:08.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:53:08.841 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:53:08.841 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:53:08.841 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:53:08.841 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.42e+01 / 1.66e+00 | 20260507 21:53:08.841 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.26e-02 / 5.81e-03 | 20260507 21:53:08.841 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.49e+00, 6.02e-01 / 2.29e-01 | 20260507 21:53:08.841 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.19e-01 | 20260507 21:53:08.841 | INFO | PerfReport: 15.71%, DME -- 4.15e+01 / 4.46e+01, 3.71e+00 / 1.27e+00 | 20260507 21:53:08.841 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.86e+01 / 2.07e+01, 1.81e+00 / 1.98e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.46e+00 / 2.71e+00, 4.31e-01 / 2.34e-01 | 20260507 21:53:08.842 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.34e+00, 2.98e-01 / 1.64e-01 | 20260507 21:53:08.842 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.93e-01, 5.97e-02 / 3.23e-02 | 20260507 21:53:08.842 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:53:08.842 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:53:08.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:53:08.842 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 24.20%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 30.95%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 2.07%, LockUpload -- 2.21e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 2.33%, LockDownload -- 2.49e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 15.97%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:53:08.842 | INFO | PerfReport: 0.02%, Misc -- 1.63e-02 / 0.00e+00, 5.30e-02 / 0.00e+00 | 20260507 21:53:08.952 | INFO | Starting a round of Baz IO | 20260507 21:53:12.409 | INFO | Populating RTMetrics report | 20260507 21:53:12.410 | INFO | RTMetrics report populated | 20260507 21:53:12.410 | INFO | Opening tmp RTMetrics json file | 20260507 21:53:12.411 | INFO | Serializing RTMetrics to json | 20260507 21:53:12.414 | INFO | Serialization done, swapping tmp file now... | 20260507 21:53:12.415 | INFO | RTMetrics done | 20260507 21:53:12.512 | INFO | Finished a round of Baz IO: Wrote 1.749 GiB in 3.559 seconds | 20260507 21:53:12.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.45% / 0.01% / 56.55% | 20260507 21:53:12.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55863% / 1.58946e-05% / 3.97364e-05% | 20260507 21:53:12.512 | INFO | Baz IO overall: | 20260507 21:53:12.512 | INFO | BytesWritten : 1309771631701 | 20260507 21:53:12.512 | INFO | NumEvents : 634711790238 | 20260507 21:53:13.976 | INFO | Analyzing chunk frames = [2875392,2875904) | 20260507 21:53:13.976 | INFO | MemInfo: { | 20260507 21:53:13.976 | INFO | "gpu" : " 17 GB", | 20260507 21:53:13.976 | INFO | "huge" : "", | 20260507 21:53:13.976 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:13.976 | INFO | "malloc" : " 14 GB", | 20260507 21:53:13.976 | INFO | "pinned" : " 29 GB" | 20260507 21:53:13.976 | INFO | } | 20260507 21:53:13.976 | INFO | Perf report for the last 5.13666 seconds: | 20260507 21:53:13.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:13.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:13.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:13.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:13.976 | INFO | ANALYSIS : 69.0%, 2.9, 103.8ms, 0 | 20260507 21:53:13.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:13.976 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 21:53:13.976 | INFO | PRE_HQ : 68.0%, 1.8, 64.1ms, 0 | 20260507 21:53:13.976 | INFO | BAZWRITER : 68.9%, 4.0, 142.4ms, 0 | 20260507 21:53:13.976 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 21:53:13.976 | INFO | Uploaded 36GiB to GPU in 2.50737s (14.3576GiB/s) | 20260507 21:53:13.976 | INFO | Downloaded 38GiB to GPU in 2.67156s (14.2239GiB/s) | 20260507 21:53:19.118 | INFO | Analyzing chunk frames = [2875904,2876416) | 20260507 21:53:19.118 | INFO | MemInfo: { | 20260507 21:53:19.118 | INFO | "gpu" : " 17 GB", | 20260507 21:53:19.118 | INFO | "huge" : "", | 20260507 21:53:19.118 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:19.118 | INFO | "malloc" : " 14 GB", | 20260507 21:53:19.118 | INFO | "pinned" : " 29 GB" | 20260507 21:53:19.118 | INFO | } | 20260507 21:53:19.118 | INFO | Perf report for the last 5.14207 seconds: | 20260507 21:53:19.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:19.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.118 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:53:19.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.118 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:53:19.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:19.119 | INFO | Uploaded 36GiB to GPU in 2.52263s (14.2708GiB/s) | 20260507 21:53:19.119 | INFO | Downloaded 34GiB to GPU in 2.45343s (13.8582GiB/s) | 20260507 21:53:24.260 | INFO | Analyzing chunk frames = [2876416,2876928) | 20260507 21:53:24.260 | INFO | MemInfo: { | 20260507 21:53:24.260 | INFO | "gpu" : " 17 GB", | 20260507 21:53:24.260 | INFO | "huge" : "", | 20260507 21:53:24.260 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:24.260 | INFO | "malloc" : " 14 GB", | 20260507 21:53:24.260 | INFO | "pinned" : " 29 GB" | 20260507 21:53:24.260 | INFO | } | 20260507 21:53:24.260 | INFO | Perf report for the last 5.14181 seconds: | 20260507 21:53:24.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:24.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.260 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 21:53:24.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.260 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:53:24.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:24.261 | INFO | Uploaded 36GiB to GPU in 2.56779s (14.0198GiB/s) | 20260507 21:53:24.261 | INFO | Downloaded 34GiB to GPU in 2.45203s (13.8661GiB/s) | 20260507 21:53:29.401 | INFO | Analyzing chunk frames = [2876928,2877440) | 20260507 21:53:29.401 | INFO | MemInfo: { | 20260507 21:53:29.401 | INFO | "gpu" : " 17 GB", | 20260507 21:53:29.401 | INFO | "huge" : "", | 20260507 21:53:29.401 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:29.401 | INFO | "malloc" : " 14 GB", | 20260507 21:53:29.401 | INFO | "pinned" : " 29 GB" | 20260507 21:53:29.401 | INFO | } | 20260507 21:53:29.401 | INFO | Perf report for the last 5.14126 seconds: | 20260507 21:53:29.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:29.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.401 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 21:53:29.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.401 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:53:29.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:29.402 | INFO | Uploaded 36GiB to GPU in 2.4916s (14.4485GiB/s) | 20260507 21:53:29.402 | INFO | Downloaded 34GiB to GPU in 2.37541s (14.3133GiB/s) | 20260507 21:53:34.540 | INFO | Analyzing chunk frames = [2877440,2877952) | 20260507 21:53:34.540 | INFO | MemInfo: { | 20260507 21:53:34.540 | INFO | "gpu" : " 17 GB", | 20260507 21:53:34.540 | INFO | "huge" : "", | 20260507 21:53:34.540 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:34.540 | INFO | "malloc" : " 14 GB", | 20260507 21:53:34.540 | INFO | "pinned" : " 29 GB" | 20260507 21:53:34.540 | INFO | } | 20260507 21:53:34.541 | INFO | Perf report for the last 5.13914 seconds: | 20260507 21:53:34.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:34.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 21:53:34.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 21:53:34.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:34.541 | INFO | Uploaded 36GiB to GPU in 2.42488s (14.8461GiB/s) | 20260507 21:53:34.541 | INFO | Downloaded 34GiB to GPU in 2.49233s (13.6418GiB/s) | 20260507 21:53:39.672 | INFO | Analyzing chunk frames = [2877952,2878464) | 20260507 21:53:39.672 | INFO | MemInfo: { | 20260507 21:53:39.672 | INFO | "gpu" : " 17 GB", | 20260507 21:53:39.672 | INFO | "huge" : "", | 20260507 21:53:39.672 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:39.672 | INFO | "malloc" : " 14 GB", | 20260507 21:53:39.672 | INFO | "pinned" : " 29 GB" | 20260507 21:53:39.672 | INFO | } | 20260507 21:53:39.673 | INFO | Perf report for the last 5.13195 seconds: | 20260507 21:53:39.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:39.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260507 21:53:39.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:53:39.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:39.673 | INFO | Uploaded 36GiB to GPU in 2.5444s (14.1487GiB/s) | 20260507 21:53:39.673 | INFO | Downloaded 34GiB to GPU in 2.41749s (14.0642GiB/s) | 20260507 21:53:44.813 | INFO | Analyzing chunk frames = [2878464,2878976) | 20260507 21:53:44.813 | INFO | MemInfo: { | 20260507 21:53:44.813 | INFO | "gpu" : " 17 GB", | 20260507 21:53:44.813 | INFO | "huge" : "", | 20260507 21:53:44.813 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:44.813 | INFO | "malloc" : " 14 GB", | 20260507 21:53:44.813 | INFO | "pinned" : " 29 GB" | 20260507 21:53:44.813 | INFO | } | 20260507 21:53:44.813 | INFO | Perf report for the last 5.14093 seconds: | 20260507 21:53:44.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:44.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.813 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 21:53:44.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.813 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 21:53:44.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:44.814 | INFO | Uploaded 36GiB to GPU in 2.5441s (14.1504GiB/s) | 20260507 21:53:44.814 | INFO | Downloaded 34GiB to GPU in 2.43904s (13.9399GiB/s) | 20260507 21:53:49.952 | INFO | Analyzing chunk frames = [2878976,2879488) | 20260507 21:53:49.952 | INFO | MemInfo: { | 20260507 21:53:49.952 | INFO | "gpu" : " 17 GB", | 20260507 21:53:49.952 | INFO | "huge" : "", | 20260507 21:53:49.952 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:49.952 | INFO | "malloc" : " 14 GB", | 20260507 21:53:49.952 | INFO | "pinned" : " 29 GB" | 20260507 21:53:49.952 | INFO | } | 20260507 21:53:49.952 | INFO | Perf report for the last 5.13871 seconds: | 20260507 21:53:49.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:49.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:53:49.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:53:49.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:49.952 | INFO | Uploaded 36GiB to GPU in 2.53936s (14.1768GiB/s) | 20260507 21:53:49.953 | INFO | Downloaded 34GiB to GPU in 2.43211s (13.9796GiB/s) | 20260507 21:53:50.049 | INFO | Starting a round of Baz IO | 20260507 21:53:53.521 | INFO | Populating RTMetrics report | 20260507 21:53:53.521 | INFO | RTMetrics report populated | 20260507 21:53:53.521 | INFO | Opening tmp RTMetrics json file | 20260507 21:53:53.556 | INFO | Serializing RTMetrics to json | 20260507 21:53:53.612 | INFO | Serialization done, swapping tmp file now... | 20260507 21:53:53.613 | INFO | RTMetrics done | 20260507 21:53:53.643 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.593 seconds | 20260507 21:53:53.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.44% / 0.01% / 56.55% | 20260507 21:53:53.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56243% / 1.58946e-05% / 5.16574e-05% | 20260507 21:53:53.643 | INFO | Baz IO overall: | 20260507 21:53:53.643 | INFO | BytesWritten : 1311652518475 | 20260507 21:53:53.643 | INFO | NumEvents : 635621199946 | 20260507 21:53:55.097 | INFO | Analyzing chunk frames = [2879488,2880000) | 20260507 21:53:55.097 | INFO | MemInfo: { | 20260507 21:53:55.097 | INFO | "gpu" : " 17 GB", | 20260507 21:53:55.097 | INFO | "huge" : "", | 20260507 21:53:55.097 | INFO | "huge_shared" : " 29 GB", | 20260507 21:53:55.097 | INFO | "malloc" : " 14 GB", | 20260507 21:53:55.097 | INFO | "pinned" : " 29 GB" | 20260507 21:53:55.097 | INFO | } | 20260507 21:53:55.097 | INFO | Perf report for the last 5.14507 seconds: | 20260507 21:53:55.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:53:55.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:55.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:55.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:55.097 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 21:53:55.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:53:55.097 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 21:53:55.097 | INFO | PRE_HQ : 65.8%, 1.7, 57.7ms, 0 | 20260507 21:53:55.097 | INFO | BAZWRITER : 69.5%, 3.7, 130.9ms, 0 | 20260507 21:53:55.097 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 21:53:55.098 | INFO | Uploaded 36GiB to GPU in 2.55547s (14.0874GiB/s) | 20260507 21:53:55.098 | INFO | Downloaded 38GiB to GPU in 2.69722s (14.0886GiB/s) | 20260507 21:54:00.232 | INFO | Analyzing chunk frames = [2880000,2880512) | 20260507 21:54:00.232 | INFO | MemInfo: { | 20260507 21:54:00.232 | INFO | "gpu" : " 17 GB", | 20260507 21:54:00.232 | INFO | "huge" : "", | 20260507 21:54:00.232 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:00.232 | INFO | "malloc" : " 14 GB", | 20260507 21:54:00.232 | INFO | "pinned" : " 29 GB" | 20260507 21:54:00.232 | INFO | } | 20260507 21:54:00.232 | INFO | Perf report for the last 5.13513 seconds: | 20260507 21:54:00.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:00.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.232 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 21:54:00.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.233 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:54:00.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:00.233 | INFO | Uploaded 36GiB to GPU in 2.58679s (13.9169GiB/s) | 20260507 21:54:00.233 | INFO | Downloaded 34GiB to GPU in 2.46019s (13.8201GiB/s) | 20260507 21:54:05.370 | INFO | Analyzing chunk frames = [2880512,2881024) | 20260507 21:54:05.370 | INFO | MemInfo: { | 20260507 21:54:05.370 | INFO | "gpu" : " 17 GB", | 20260507 21:54:05.370 | INFO | "huge" : "", | 20260507 21:54:05.370 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:05.370 | INFO | "malloc" : " 14 GB", | 20260507 21:54:05.370 | INFO | "pinned" : " 29 GB" | 20260507 21:54:05.370 | INFO | } | 20260507 21:54:05.371 | INFO | Perf report for the last 5.1381 seconds: | 20260507 21:54:05.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:05.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 21:54:05.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 21:54:05.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:05.371 | INFO | Uploaded 36GiB to GPU in 2.55761s (14.0756GiB/s) | 20260507 21:54:05.371 | INFO | Downloaded 34GiB to GPU in 2.45086s (13.8727GiB/s) | 20260507 21:54:10.511 | INFO | Analyzing chunk frames = [2881024,2881536) | 20260507 21:54:10.511 | INFO | MemInfo: { | 20260507 21:54:10.511 | INFO | "gpu" : " 17 GB", | 20260507 21:54:10.511 | INFO | "huge" : "", | 20260507 21:54:10.511 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:10.511 | INFO | "malloc" : " 14 GB", | 20260507 21:54:10.511 | INFO | "pinned" : " 29 GB" | 20260507 21:54:10.511 | INFO | } | 20260507 21:54:10.511 | INFO | Perf report for the last 5.14059 seconds: | 20260507 21:54:10.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:10.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 21:54:10.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:54:10.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:10.511 | INFO | Uploaded 36GiB to GPU in 2.48734s (14.4733GiB/s) | 20260507 21:54:10.511 | INFO | Downloaded 34GiB to GPU in 2.37972s (14.2874GiB/s) | 20260507 21:54:15.652 | INFO | Analyzing chunk frames = [2881536,2882048) | 20260507 21:54:15.652 | INFO | MemInfo: { | 20260507 21:54:15.652 | INFO | "gpu" : " 17 GB", | 20260507 21:54:15.652 | INFO | "huge" : "", | 20260507 21:54:15.652 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:15.652 | INFO | "malloc" : " 14 GB", | 20260507 21:54:15.652 | INFO | "pinned" : " 29 GB" | 20260507 21:54:15.652 | INFO | } | 20260507 21:54:15.652 | INFO | Perf report for the last 5.14129 seconds: | 20260507 21:54:15.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:15.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.652 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 21:54:15.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.653 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 21:54:15.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:15.653 | INFO | Uploaded 36GiB to GPU in 2.41378s (14.9144GiB/s) | 20260507 21:54:15.653 | INFO | Downloaded 34GiB to GPU in 2.4892s (13.659GiB/s) | 20260507 21:54:20.785 | INFO | Analyzing chunk frames = [2882048,2882560) | 20260507 21:54:20.785 | INFO | MemInfo: { | 20260507 21:54:20.785 | INFO | "gpu" : " 17 GB", | 20260507 21:54:20.785 | INFO | "huge" : "", | 20260507 21:54:20.785 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:20.785 | INFO | "malloc" : " 14 GB", | 20260507 21:54:20.785 | INFO | "pinned" : " 29 GB" | 20260507 21:54:20.786 | INFO | } | 20260507 21:54:20.786 | INFO | Perf report for the last 5.13314 seconds: | 20260507 21:54:20.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:20.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260507 21:54:20.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260507 21:54:20.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:20.786 | INFO | Uploaded 36GiB to GPU in 2.52347s (14.2661GiB/s) | 20260507 21:54:20.786 | INFO | Downloaded 34GiB to GPU in 2.42314s (14.0314GiB/s) | 20260507 21:54:25.926 | INFO | Analyzing chunk frames = [2882560,2883072) | 20260507 21:54:25.926 | INFO | MemInfo: { | 20260507 21:54:25.926 | INFO | "gpu" : " 17 GB", | 20260507 21:54:25.926 | INFO | "huge" : "", | 20260507 21:54:25.926 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:25.926 | INFO | "malloc" : " 14 GB", | 20260507 21:54:25.926 | INFO | "pinned" : " 29 GB" | 20260507 21:54:25.926 | INFO | } | 20260507 21:54:25.926 | INFO | Perf report for the last 5.14062 seconds: | 20260507 21:54:25.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:25.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.926 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 21:54:25.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.926 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 21:54:25.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:25.927 | INFO | Uploaded 36GiB to GPU in 2.52109s (14.2795GiB/s) | 20260507 21:54:25.927 | INFO | Downloaded 34GiB to GPU in 2.4482s (13.8877GiB/s) | 20260507 21:54:31.073 | INFO | Analyzing chunk frames = [2883072,2883584) | 20260507 21:54:31.073 | INFO | MemInfo: { | 20260507 21:54:31.073 | INFO | "gpu" : " 17 GB", | 20260507 21:54:31.073 | INFO | "huge" : "", | 20260507 21:54:31.073 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:31.073 | INFO | "malloc" : " 14 GB", | 20260507 21:54:31.073 | INFO | "pinned" : " 29 GB" | 20260507 21:54:31.073 | INFO | } | 20260507 21:54:31.073 | INFO | Perf report for the last 5.14697 seconds: | 20260507 21:54:31.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:31.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.073 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 21:54:31.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.074 | INFO | PRE_HQ : 36.2%, 1.0, 18.9ms, 0 | 20260507 21:54:31.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:31.074 | INFO | Uploaded 36GiB to GPU in 2.49599s (14.4231GiB/s) | 20260507 21:54:31.074 | INFO | Downloaded 34GiB to GPU in 2.44801s (13.8888GiB/s) | 20260507 21:54:31.074 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:54:31.074 | INFO | Total mem usage: 16.7 GiB | 20260507 21:54:31.074 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:54:31.074 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260507 21:54:31.074 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:54:31.074 | INFO | High Water Breakdown: | 20260507 21:54:31.074 | INFO | 59.9% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:54:31.074 | INFO | 39.9% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:54:31.074 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:54:31.074 | INFO | ------------------------------------------ | 20260507 21:54:31.074 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:54:31.074 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:54:31.074 | INFO | Total mem usage: 26.5 GiB | 20260507 21:54:31.074 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:54:31.074 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:54:31.074 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:54:31.074 | INFO | High Water Breakdown: | 20260507 21:54:31.074 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:54:31.074 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:54:31.074 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:54:31.074 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:54:31.074 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:54:31.074 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:54:31.074 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:54:31.074 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:54:31.074 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:54:31.074 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:54:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:54:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:54:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:54:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:54:31.075 | INFO | ------------------------------------------ | 20260507 21:54:31.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:54:31.075 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:54:31.075 | INFO | Total mem usage: 27.8 GiB | 20260507 21:54:31.075 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:54:31.075 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:54:31.075 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:54:31.075 | INFO | High Water Breakdown: | 20260507 21:54:31.075 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:54:31.075 | INFO | ------------------------------------------ | 20260507 21:54:31.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:54:31.075 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:54:31.075 | INFO | Total mem usage: 15.7 GiB | 20260507 21:54:31.075 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:54:31.075 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:54:31.075 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:54:31.075 | INFO | High Water Breakdown: | 20260507 21:54:31.075 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:54:31.075 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:54:31.075 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:54:31.075 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:54:31.075 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:54:31.075 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:54:31.075 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:54:31.075 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:54:31.075 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:54:31.075 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:54:31.075 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:54:31.075 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:54:31.075 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:54:31.075 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:54:31.075 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:54:31.075 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:54:31.075 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:54:31.075 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:54:31.075 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:54:31.075 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:54:31.075 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:54:31.075 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:54:31.076 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:54:31.076 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:54:31.076 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:54:31.076 | INFO | ------------------------------------------ | 20260507 21:54:31.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:54:31.076 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:54:31.076 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:54:31.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:54:31.076 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.58e+01, 1.39e+01 / 1.63e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.04e-01, 1.28e-02 / 8.21e-03 | 20260507 21:54:31.076 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.44e+00, 5.97e-01 / 3.11e-01 | 20260507 21:54:31.076 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.11e-01 / 1.28e-01 | 20260507 21:54:31.076 | INFO | PerfReport: 15.73%, DME -- 4.16e+01 / 4.49e+01, 3.80e+00 / 1.13e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.90e+00 / 2.13e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.30e-01 / 2.52e-01 | 20260507 21:54:31.076 | INFO | PerfReport: 3.50%, Metrics -- 1.16e+00 / 1.38e+00, 2.97e-01 / 1.92e-01 | 20260507 21:54:31.076 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.92e-01, 6.06e-02 / 2.57e-02 | 20260507 21:54:31.076 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:54:31.076 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:54:31.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:54:31.076 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 24.55%, Upload -- 2.62e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 24.09%, Download -- 2.57e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 30.95%, Compute -- 3.31e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 2.21%, LockUpload -- 2.36e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 2.20%, LockDownload -- 2.35e+00 / 0.00e+00, 3.65e+00 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 15.99%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 21:54:31.076 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 21:54:31.188 | INFO | Starting a round of Baz IO | 20260507 21:54:34.680 | INFO | Populating RTMetrics report | 20260507 21:54:34.681 | INFO | RTMetrics report populated | 20260507 21:54:34.681 | INFO | Opening tmp RTMetrics json file | 20260507 21:54:34.706 | INFO | Serializing RTMetrics to json | 20260507 21:54:34.710 | INFO | Serialization done, swapping tmp file now... | 20260507 21:54:34.710 | INFO | RTMetrics done | 20260507 21:54:34.773 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.585 seconds | 20260507 21:54:34.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.43% / 0.01% / 56.56% | 20260507 21:54:34.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56108% / 0% / 4.76837e-05% | 20260507 21:54:34.773 | INFO | Baz IO overall: | 20260507 21:54:34.773 | INFO | BytesWritten : 1313533844326 | 20260507 21:54:34.773 | INFO | NumEvents : 636530942854 | 20260507 21:54:36.212 | INFO | Analyzing chunk frames = [2883584,2884096) | 20260507 21:54:36.212 | INFO | MemInfo: { | 20260507 21:54:36.212 | INFO | "gpu" : " 17 GB", | 20260507 21:54:36.212 | INFO | "huge" : "", | 20260507 21:54:36.212 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:36.212 | INFO | "malloc" : " 14 GB", | 20260507 21:54:36.212 | INFO | "pinned" : " 29 GB" | 20260507 21:54:36.212 | INFO | } | 20260507 21:54:36.212 | INFO | Perf report for the last 5.13854 seconds: | 20260507 21:54:36.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:36.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:36.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:36.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:36.212 | INFO | ANALYSIS : 69.2%, 2.9, 104.5ms, 0 | 20260507 21:54:36.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:36.212 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 21:54:36.212 | INFO | PRE_HQ : 68.2%, 1.8, 64.5ms, 0 | 20260507 21:54:36.212 | INFO | BAZWRITER : 69.4%, 3.9, 137.7ms, 0 | 20260507 21:54:36.212 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 21:54:36.212 | INFO | Uploaded 36GiB to GPU in 2.55732s (14.0772GiB/s) | 20260507 21:54:36.212 | INFO | Downloaded 38GiB to GPU in 2.71102s (14.0169GiB/s) | 20260507 21:54:41.351 | INFO | Analyzing chunk frames = [2884096,2884608) | 20260507 21:54:41.351 | INFO | MemInfo: { | 20260507 21:54:41.351 | INFO | "gpu" : " 17 GB", | 20260507 21:54:41.351 | INFO | "huge" : "", | 20260507 21:54:41.351 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:41.351 | INFO | "malloc" : " 14 GB", | 20260507 21:54:41.351 | INFO | "pinned" : " 29 GB" | 20260507 21:54:41.351 | INFO | } | 20260507 21:54:41.351 | INFO | Perf report for the last 5.13932 seconds: | 20260507 21:54:41.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:41.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.351 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 21:54:41.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.351 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:54:41.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:41.352 | INFO | Uploaded 36GiB to GPU in 2.52354s (14.2657GiB/s) | 20260507 21:54:41.352 | INFO | Downloaded 34GiB to GPU in 2.4902s (13.6535GiB/s) | 20260507 21:54:46.487 | INFO | Analyzing chunk frames = [2884608,2885120) | 20260507 21:54:46.487 | INFO | MemInfo: { | 20260507 21:54:46.487 | INFO | "gpu" : " 17 GB", | 20260507 21:54:46.487 | INFO | "huge" : "", | 20260507 21:54:46.487 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:46.487 | INFO | "malloc" : " 14 GB", | 20260507 21:54:46.487 | INFO | "pinned" : " 29 GB" | 20260507 21:54:46.487 | INFO | } | 20260507 21:54:46.487 | INFO | Perf report for the last 5.1357 seconds: | 20260507 21:54:46.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:46.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 21:54:46.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:54:46.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:46.487 | INFO | Uploaded 36GiB to GPU in 2.5577s (14.0752GiB/s) | 20260507 21:54:46.487 | INFO | Downloaded 34GiB to GPU in 2.46654s (13.7845GiB/s) | 20260507 21:54:51.626 | INFO | Analyzing chunk frames = [2885120,2885632) | 20260507 21:54:51.626 | INFO | MemInfo: { | 20260507 21:54:51.626 | INFO | "gpu" : " 17 GB", | 20260507 21:54:51.626 | INFO | "huge" : "", | 20260507 21:54:51.626 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:51.626 | INFO | "malloc" : " 14 GB", | 20260507 21:54:51.626 | INFO | "pinned" : " 29 GB" | 20260507 21:54:51.626 | INFO | } | 20260507 21:54:51.626 | INFO | Perf report for the last 5.13954 seconds: | 20260507 21:54:51.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:51.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 21:54:51.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:54:51.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:51.627 | INFO | Uploaded 36GiB to GPU in 2.47614s (14.5388GiB/s) | 20260507 21:54:51.627 | INFO | Downloaded 34GiB to GPU in 2.39576s (14.1917GiB/s) | 20260507 21:54:56.766 | INFO | Analyzing chunk frames = [2885632,2886144) | 20260507 21:54:56.766 | INFO | MemInfo: { | 20260507 21:54:56.766 | INFO | "gpu" : " 17 GB", | 20260507 21:54:56.766 | INFO | "huge" : "", | 20260507 21:54:56.766 | INFO | "huge_shared" : " 29 GB", | 20260507 21:54:56.766 | INFO | "malloc" : " 14 GB", | 20260507 21:54:56.766 | INFO | "pinned" : " 29 GB" | 20260507 21:54:56.766 | INFO | } | 20260507 21:54:56.766 | INFO | Perf report for the last 5.13993 seconds: | 20260507 21:54:56.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:54:56.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.766 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:54:56.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.766 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 21:54:56.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:54:56.767 | INFO | Uploaded 36GiB to GPU in 2.36593s (15.216GiB/s) | 20260507 21:54:56.767 | INFO | Downloaded 34GiB to GPU in 2.49886s (13.6062GiB/s) | 20260507 21:55:01.906 | INFO | Analyzing chunk frames = [2886144,2886656) | 20260507 21:55:01.906 | INFO | MemInfo: { | 20260507 21:55:01.906 | INFO | "gpu" : " 17 GB", | 20260507 21:55:01.906 | INFO | "huge" : "", | 20260507 21:55:01.906 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:01.906 | INFO | "malloc" : " 14 GB", | 20260507 21:55:01.906 | INFO | "pinned" : " 29 GB" | 20260507 21:55:01.906 | INFO | } | 20260507 21:55:01.906 | INFO | Perf report for the last 5.13983 seconds: | 20260507 21:55:01.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:01.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:55:01.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:55:01.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:01.906 | INFO | Uploaded 36GiB to GPU in 2.50935s (14.3463GiB/s) | 20260507 21:55:01.907 | INFO | Downloaded 34GiB to GPU in 2.41621s (14.0716GiB/s) | 20260507 21:55:07.045 | INFO | Analyzing chunk frames = [2886656,2887168) | 20260507 21:55:07.045 | INFO | MemInfo: { | 20260507 21:55:07.045 | INFO | "gpu" : " 17 GB", | 20260507 21:55:07.045 | INFO | "huge" : "", | 20260507 21:55:07.045 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:07.045 | INFO | "malloc" : " 14 GB", | 20260507 21:55:07.045 | INFO | "pinned" : " 29 GB" | 20260507 21:55:07.045 | INFO | } | 20260507 21:55:07.045 | INFO | Perf report for the last 5.1391 seconds: | 20260507 21:55:07.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:07.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.045 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 21:55:07.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.045 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:55:07.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:07.046 | INFO | Uploaded 36GiB to GPU in 2.53323s (14.2111GiB/s) | 20260507 21:55:07.046 | INFO | Downloaded 34GiB to GPU in 2.43409s (13.9683GiB/s) | 20260507 21:55:12.181 | INFO | Analyzing chunk frames = [2887168,2887680) | 20260507 21:55:12.181 | INFO | MemInfo: { | 20260507 21:55:12.181 | INFO | "gpu" : " 17 GB", | 20260507 21:55:12.181 | INFO | "huge" : "", | 20260507 21:55:12.182 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:12.182 | INFO | "malloc" : " 14 GB", | 20260507 21:55:12.182 | INFO | "pinned" : " 29 GB" | 20260507 21:55:12.182 | INFO | } | 20260507 21:55:12.182 | INFO | Perf report for the last 5.13655 seconds: | 20260507 21:55:12.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:12.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.182 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:55:12.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.182 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 21:55:12.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:12.183 | INFO | Uploaded 36GiB to GPU in 2.51271s (14.3272GiB/s) | 20260507 21:55:12.183 | INFO | Downloaded 34GiB to GPU in 2.43926s (13.9387GiB/s) | 20260507 21:55:12.293 | INFO | Starting a round of Baz IO | 20260507 21:55:15.766 | INFO | Populating RTMetrics report | 20260507 21:55:15.766 | INFO | RTMetrics report populated | 20260507 21:55:15.766 | INFO | Opening tmp RTMetrics json file | 20260507 21:55:15.770 | INFO | Serializing RTMetrics to json | 20260507 21:55:15.795 | INFO | Serialization done, swapping tmp file now... | 20260507 21:55:15.796 | INFO | RTMetrics done | 20260507 21:55:15.866 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.572 seconds | 20260507 21:55:15.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.42% / 0.01% / 56.57% | 20260507 21:55:15.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55462% / 0% / 6.75519e-05% | 20260507 21:55:15.866 | INFO | Baz IO overall: | 20260507 21:55:15.866 | INFO | BytesWritten : 1315412592154 | 20260507 21:55:15.866 | INFO | NumEvents : 637438742898 | 20260507 21:55:17.324 | INFO | Analyzing chunk frames = [2887680,2888192) | 20260507 21:55:17.324 | INFO | MemInfo: { | 20260507 21:55:17.324 | INFO | "gpu" : " 17 GB", | 20260507 21:55:17.324 | INFO | "huge" : "", | 20260507 21:55:17.324 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:17.324 | INFO | "malloc" : " 14 GB", | 20260507 21:55:17.324 | INFO | "pinned" : " 29 GB" | 20260507 21:55:17.324 | INFO | } | 20260507 21:55:17.324 | INFO | Perf report for the last 5.14247 seconds: | 20260507 21:55:17.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:17.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:17.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:17.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:17.324 | INFO | ANALYSIS : 69.0%, 2.9, 104.8ms, 0 | 20260507 21:55:17.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:17.324 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 21:55:17.324 | INFO | PRE_HQ : 68.1%, 1.8, 63.1ms, 0 | 20260507 21:55:17.324 | INFO | BAZWRITER : 69.1%, 4.0, 142.2ms, 0 | 20260507 21:55:17.324 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 21:55:17.324 | INFO | Uploaded 36GiB to GPU in 2.53327s (14.2109GiB/s) | 20260507 21:55:17.325 | INFO | Downloaded 38GiB to GPU in 2.6799s (14.1796GiB/s) | 20260507 21:55:22.464 | INFO | Analyzing chunk frames = [2888192,2888704) | 20260507 21:55:22.464 | INFO | MemInfo: { | 20260507 21:55:22.464 | INFO | "gpu" : " 17 GB", | 20260507 21:55:22.464 | INFO | "huge" : "", | 20260507 21:55:22.464 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:22.464 | INFO | "malloc" : " 14 GB", | 20260507 21:55:22.464 | INFO | "pinned" : " 29 GB" | 20260507 21:55:22.464 | INFO | } | 20260507 21:55:22.464 | INFO | Perf report for the last 5.13942 seconds: | 20260507 21:55:22.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:22.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 21:55:22.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 21:55:22.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:22.464 | INFO | Uploaded 36GiB to GPU in 2.54252s (14.1592GiB/s) | 20260507 21:55:22.464 | INFO | Downloaded 34GiB to GPU in 2.48304s (13.6929GiB/s) | 20260507 21:55:27.602 | INFO | Analyzing chunk frames = [2888704,2889216) | 20260507 21:55:27.602 | INFO | MemInfo: { | 20260507 21:55:27.602 | INFO | "gpu" : " 17 GB", | 20260507 21:55:27.602 | INFO | "huge" : "", | 20260507 21:55:27.602 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:27.602 | INFO | "malloc" : " 14 GB", | 20260507 21:55:27.602 | INFO | "pinned" : " 29 GB" | 20260507 21:55:27.602 | INFO | } | 20260507 21:55:27.602 | INFO | Perf report for the last 5.13782 seconds: | 20260507 21:55:27.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:27.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 21:55:27.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:55:27.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:27.602 | INFO | Uploaded 36GiB to GPU in 2.50101s (14.3942GiB/s) | 20260507 21:55:27.602 | INFO | Downloaded 34GiB to GPU in 2.47544s (13.7349GiB/s) | 20260507 21:55:32.736 | INFO | Analyzing chunk frames = [2889216,2889728) | 20260507 21:55:32.736 | INFO | MemInfo: { | 20260507 21:55:32.736 | INFO | "gpu" : " 17 GB", | 20260507 21:55:32.736 | INFO | "huge" : "", | 20260507 21:55:32.737 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:32.737 | INFO | "malloc" : " 14 GB", | 20260507 21:55:32.737 | INFO | "pinned" : " 29 GB" | 20260507 21:55:32.737 | INFO | } | 20260507 21:55:32.737 | INFO | Perf report for the last 5.13515 seconds: | 20260507 21:55:32.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:32.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 21:55:32.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:55:32.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:32.737 | INFO | Uploaded 36GiB to GPU in 2.47306s (14.5568GiB/s) | 20260507 21:55:32.737 | INFO | Downloaded 34GiB to GPU in 2.39531s (14.1944GiB/s) | 20260507 21:55:37.877 | INFO | Analyzing chunk frames = [2889728,2890240) | 20260507 21:55:37.877 | INFO | MemInfo: { | 20260507 21:55:37.877 | INFO | "gpu" : " 17 GB", | 20260507 21:55:37.877 | INFO | "huge" : "", | 20260507 21:55:37.877 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:37.877 | INFO | "malloc" : " 14 GB", | 20260507 21:55:37.877 | INFO | "pinned" : " 29 GB" | 20260507 21:55:37.877 | INFO | } | 20260507 21:55:37.877 | INFO | Perf report for the last 5.14052 seconds: | 20260507 21:55:37.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:37.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.877 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:55:37.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.877 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 21:55:37.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:37.878 | INFO | Uploaded 36GiB to GPU in 2.32063s (15.513GiB/s) | 20260507 21:55:37.878 | INFO | Downloaded 34GiB to GPU in 2.49749s (13.6137GiB/s) | 20260507 21:55:43.025 | INFO | Analyzing chunk frames = [2890240,2890752) | 20260507 21:55:43.025 | INFO | MemInfo: { | 20260507 21:55:43.025 | INFO | "gpu" : " 17 GB", | 20260507 21:55:43.025 | INFO | "huge" : "", | 20260507 21:55:43.025 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:43.026 | INFO | "malloc" : " 14 GB", | 20260507 21:55:43.026 | INFO | "pinned" : " 29 GB" | 20260507 21:55:43.026 | INFO | } | 20260507 21:55:43.026 | INFO | Perf report for the last 5.1484 seconds: | 20260507 21:55:43.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:43.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260507 21:55:43.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:55:43.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:43.026 | INFO | Uploaded 36GiB to GPU in 2.44894s (14.7002GiB/s) | 20260507 21:55:43.026 | INFO | Downloaded 34GiB to GPU in 2.44194s (13.9234GiB/s) | 20260507 21:55:48.168 | INFO | Analyzing chunk frames = [2890752,2891264) | 20260507 21:55:48.168 | INFO | MemInfo: { | 20260507 21:55:48.168 | INFO | "gpu" : " 17 GB", | 20260507 21:55:48.168 | INFO | "huge" : "", | 20260507 21:55:48.168 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:48.168 | INFO | "malloc" : " 14 GB", | 20260507 21:55:48.168 | INFO | "pinned" : " 29 GB" | 20260507 21:55:48.168 | INFO | } | 20260507 21:55:48.168 | INFO | Perf report for the last 5.14218 seconds: | 20260507 21:55:48.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:48.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.168 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 21:55:48.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.168 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:55:48.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:48.169 | INFO | Uploaded 36GiB to GPU in 2.48473s (14.4885GiB/s) | 20260507 21:55:48.169 | INFO | Downloaded 34GiB to GPU in 2.45814s (13.8316GiB/s) | 20260507 21:55:53.301 | INFO | Analyzing chunk frames = [2891264,2891776) | 20260507 21:55:53.301 | INFO | MemInfo: { | 20260507 21:55:53.301 | INFO | "gpu" : " 17 GB", | 20260507 21:55:53.301 | INFO | "huge" : "", | 20260507 21:55:53.301 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:53.301 | INFO | "malloc" : " 14 GB", | 20260507 21:55:53.301 | INFO | "pinned" : " 29 GB" | 20260507 21:55:53.301 | INFO | } | 20260507 21:55:53.302 | INFO | Perf report for the last 5.13361 seconds: | 20260507 21:55:53.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:53.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 21:55:53.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:55:53.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:53.302 | INFO | Uploaded 36GiB to GPU in 2.45415s (14.669GiB/s) | 20260507 21:55:53.302 | INFO | Downloaded 34GiB to GPU in 2.45729s (13.8364GiB/s) | 20260507 21:55:53.302 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:55:53.302 | INFO | Total mem usage: 16.7 GiB | 20260507 21:55:53.302 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:55:53.302 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.6%) | 20260507 21:55:53.302 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:55:53.302 | INFO | High Water Breakdown: | 20260507 21:55:53.302 | INFO | 60.1% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:55:53.302 | INFO | 39.8% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:55:53.302 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:55:53.302 | INFO | ------------------------------------------ | 20260507 21:55:53.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:55:53.302 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:55:53.302 | INFO | Total mem usage: 26.5 GiB | 20260507 21:55:53.302 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:55:53.302 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 21:55:53.302 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 21:55:53.302 | INFO | High Water Breakdown: | 20260507 21:55:53.302 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:55:53.302 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:55:53.302 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:55:53.302 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:55:53.302 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:55:53.302 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:55:53.302 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:55:53.302 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:55:53.302 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:55:53.302 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:55:53.302 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:55:53.302 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:55:53.302 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:55:53.302 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:55:53.302 | INFO | ------------------------------------------ | 20260507 21:55:53.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:55:53.303 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:55:53.303 | INFO | Total mem usage: 27.8 GiB | 20260507 21:55:53.303 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:55:53.303 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:55:53.303 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:55:53.303 | INFO | High Water Breakdown: | 20260507 21:55:53.303 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:55:53.303 | INFO | ------------------------------------------ | 20260507 21:55:53.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:55:53.303 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:55:53.303 | INFO | Total mem usage: 15.7 GiB | 20260507 21:55:53.303 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:55:53.303 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:55:53.303 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:55:53.303 | INFO | High Water Breakdown: | 20260507 21:55:53.303 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:55:53.303 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:55:53.303 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:55:53.303 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:55:53.303 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:55:53.303 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:55:53.303 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:55:53.303 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:55:53.303 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:55:53.303 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:55:53.303 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:55:53.303 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:55:53.303 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:55:53.303 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:55:53.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:55:53.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:55:53.303 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:55:53.303 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:55:53.303 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:55:53.303 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:55:53.303 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:55:53.303 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:55:53.303 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:55:53.303 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:55:53.303 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:55:53.303 | INFO | ------------------------------------------ | 20260507 21:55:53.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:55:53.303 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:55:53.303 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:55:53.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:55:53.303 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.42e+01 / 1.60e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.04e-01, 1.16e-02 / 7.49e-03 | 20260507 21:55:53.303 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.48e+00, 6.01e-01 / 2.20e-01 | 20260507 21:55:53.303 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.11e-01 / 1.20e-01 | 20260507 21:55:53.303 | INFO | PerfReport: 15.83%, DME -- 4.18e+01 / 4.54e+01, 3.83e+00 / 1.33e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.66e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.28e-01 / 2.16e-01 | 20260507 21:55:53.303 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.37e+00, 2.91e-01 / 1.53e-01 | 20260507 21:55:53.303 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.90e-01, 5.92e-02 / 3.19e-02 | 20260507 21:55:53.303 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:55:53.303 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:55:53.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:55:53.303 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 31.00%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.14e+00 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 15.86%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 21:55:53.303 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 21:55:53.398 | INFO | Starting a round of Baz IO | 20260507 21:55:56.869 | INFO | Populating RTMetrics report | 20260507 21:55:56.869 | INFO | RTMetrics report populated | 20260507 21:55:56.869 | INFO | Opening tmp RTMetrics json file | 20260507 21:55:56.893 | INFO | Serializing RTMetrics to json | 20260507 21:55:56.895 | INFO | Serialization done, swapping tmp file now... | 20260507 21:55:56.896 | INFO | RTMetrics done | 20260507 21:55:56.967 | INFO | Finished a round of Baz IO: Wrote 1.751 GiB in 3.568 seconds | 20260507 21:55:56.967 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.42% / 0.01% / 56.58% | 20260507 21:55:56.967 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55548% / 7.94729e-06% / 6.75519e-05% | 20260507 21:55:56.967 | INFO | Baz IO overall: | 20260507 21:55:56.967 | INFO | BytesWritten : 1317292827699 | 20260507 21:55:56.967 | INFO | NumEvents : 638347707664 | 20260507 21:55:58.438 | INFO | Analyzing chunk frames = [2891776,2892288) | 20260507 21:55:58.438 | INFO | MemInfo: { | 20260507 21:55:58.438 | INFO | "gpu" : " 17 GB", | 20260507 21:55:58.438 | INFO | "huge" : "", | 20260507 21:55:58.438 | INFO | "huge_shared" : " 29 GB", | 20260507 21:55:58.438 | INFO | "malloc" : " 14 GB", | 20260507 21:55:58.438 | INFO | "pinned" : " 29 GB" | 20260507 21:55:58.439 | INFO | } | 20260507 21:55:58.439 | INFO | Perf report for the last 5.13726 seconds: | 20260507 21:55:58.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:55:58.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:58.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:58.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:58.439 | INFO | ANALYSIS : 68.7%, 3.0, 106.3ms, 0 | 20260507 21:55:58.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:55:58.439 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 21:55:58.439 | INFO | PRE_HQ : 66.7%, 1.8, 60.6ms, 0 | 20260507 21:55:58.439 | INFO | BAZWRITER : 68.9%, 4.0, 142.5ms, 0 | 20260507 21:55:58.439 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 21:55:58.439 | INFO | Uploaded 36GiB to GPU in 2.53579s (14.1968GiB/s) | 20260507 21:55:58.439 | INFO | Downloaded 38GiB to GPU in 2.70187s (14.0643GiB/s) | 20260507 21:56:03.578 | INFO | Analyzing chunk frames = [2892288,2892800) | 20260507 21:56:03.578 | INFO | MemInfo: { | 20260507 21:56:03.578 | INFO | "gpu" : " 17 GB", | 20260507 21:56:03.578 | INFO | "huge" : "", | 20260507 21:56:03.578 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:03.578 | INFO | "malloc" : " 14 GB", | 20260507 21:56:03.578 | INFO | "pinned" : " 29 GB" | 20260507 21:56:03.578 | INFO | } | 20260507 21:56:03.578 | INFO | Perf report for the last 5.13964 seconds: | 20260507 21:56:03.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:03.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 21:56:03.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | PRE_HQ : 35.5%, 1.0, 18.5ms, 0 | 20260507 21:56:03.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:03.579 | INFO | Uploaded 36GiB to GPU in 2.53561s (14.1978GiB/s) | 20260507 21:56:03.579 | INFO | Downloaded 34GiB to GPU in 2.51169s (13.5367GiB/s) | 20260507 21:56:08.713 | INFO | Analyzing chunk frames = [2892800,2893312) | 20260507 21:56:08.713 | INFO | MemInfo: { | 20260507 21:56:08.713 | INFO | "gpu" : " 17 GB", | 20260507 21:56:08.713 | INFO | "huge" : "", | 20260507 21:56:08.713 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:08.713 | INFO | "malloc" : " 14 GB", | 20260507 21:56:08.713 | INFO | "pinned" : " 29 GB" | 20260507 21:56:08.713 | INFO | } | 20260507 21:56:08.714 | INFO | Perf report for the last 5.13503 seconds: | 20260507 21:56:08.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:08.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.714 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 21:56:08.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.714 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 21:56:08.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:08.715 | INFO | Uploaded 36GiB to GPU in 2.51553s (14.3111GiB/s) | 20260507 21:56:08.715 | INFO | Downloaded 34GiB to GPU in 2.48972s (13.6561GiB/s) | 20260507 21:56:13.852 | INFO | Analyzing chunk frames = [2893312,2893824) | 20260507 21:56:13.852 | INFO | MemInfo: { | 20260507 21:56:13.852 | INFO | "gpu" : " 17 GB", | 20260507 21:56:13.852 | INFO | "huge" : "", | 20260507 21:56:13.852 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:13.852 | INFO | "malloc" : " 14 GB", | 20260507 21:56:13.852 | INFO | "pinned" : " 29 GB" | 20260507 21:56:13.852 | INFO | } | 20260507 21:56:13.852 | INFO | Perf report for the last 5.13875 seconds: | 20260507 21:56:13.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:13.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.852 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 21:56:13.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.852 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 21:56:13.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:13.853 | INFO | Uploaded 36GiB to GPU in 2.43545s (14.7817GiB/s) | 20260507 21:56:13.853 | INFO | Downloaded 34GiB to GPU in 2.40347s (14.1462GiB/s) | 20260507 21:56:18.998 | INFO | Analyzing chunk frames = [2893824,2894336) | 20260507 21:56:18.998 | INFO | MemInfo: { | 20260507 21:56:18.998 | INFO | "gpu" : " 17 GB", | 20260507 21:56:18.998 | INFO | "huge" : "", | 20260507 21:56:18.998 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:18.998 | INFO | "malloc" : " 14 GB", | 20260507 21:56:18.998 | INFO | "pinned" : " 29 GB" | 20260507 21:56:18.998 | INFO | } | 20260507 21:56:18.998 | INFO | Perf report for the last 5.14597 seconds: | 20260507 21:56:18.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:18.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.998 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 21:56:18.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.998 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260507 21:56:18.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:18.999 | INFO | Uploaded 36GiB to GPU in 2.33981s (15.3859GiB/s) | 20260507 21:56:18.999 | INFO | Downloaded 34GiB to GPU in 2.50751s (13.5593GiB/s) | 20260507 21:56:24.132 | INFO | Analyzing chunk frames = [2894336,2894848) | 20260507 21:56:24.132 | INFO | MemInfo: { | 20260507 21:56:24.132 | INFO | "gpu" : " 17 GB", | 20260507 21:56:24.132 | INFO | "huge" : "", | 20260507 21:56:24.132 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:24.132 | INFO | "malloc" : " 14 GB", | 20260507 21:56:24.132 | INFO | "pinned" : " 29 GB" | 20260507 21:56:24.132 | INFO | } | 20260507 21:56:24.133 | INFO | Perf report for the last 5.13449 seconds: | 20260507 21:56:24.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:24.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:56:24.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 21:56:24.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:24.133 | INFO | Uploaded 36GiB to GPU in 2.47373s (14.5529GiB/s) | 20260507 21:56:24.133 | INFO | Downloaded 34GiB to GPU in 2.43498s (13.9631GiB/s) | 20260507 21:56:29.276 | INFO | Analyzing chunk frames = [2894848,2895360) | 20260507 21:56:29.276 | INFO | MemInfo: { | 20260507 21:56:29.276 | INFO | "gpu" : " 17 GB", | 20260507 21:56:29.276 | INFO | "huge" : "", | 20260507 21:56:29.276 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:29.276 | INFO | "malloc" : " 14 GB", | 20260507 21:56:29.276 | INFO | "pinned" : " 29 GB" | 20260507 21:56:29.276 | INFO | } | 20260507 21:56:29.276 | INFO | Perf report for the last 5.14343 seconds: | 20260507 21:56:29.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:29.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:56:29.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:56:29.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:29.276 | INFO | Uploaded 36GiB to GPU in 2.4962s (14.4219GiB/s) | 20260507 21:56:29.277 | INFO | Downloaded 34GiB to GPU in 2.4674s (13.7797GiB/s) | 20260507 21:56:34.415 | INFO | Analyzing chunk frames = [2895360,2895872) | 20260507 21:56:34.415 | INFO | MemInfo: { | 20260507 21:56:34.415 | INFO | "gpu" : " 17 GB", | 20260507 21:56:34.415 | INFO | "huge" : "", | 20260507 21:56:34.415 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:34.415 | INFO | "malloc" : " 14 GB", | 20260507 21:56:34.415 | INFO | "pinned" : " 29 GB" | 20260507 21:56:34.415 | INFO | } | 20260507 21:56:34.415 | INFO | Perf report for the last 5.13856 seconds: | 20260507 21:56:34.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:34.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:56:34.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 21:56:34.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:34.415 | INFO | Uploaded 36GiB to GPU in 2.47384s (14.5523GiB/s) | 20260507 21:56:34.415 | INFO | Downloaded 34GiB to GPU in 2.45536s (13.8473GiB/s) | 20260507 21:56:34.512 | INFO | Starting a round of Baz IO | 20260507 21:56:37.982 | INFO | Populating RTMetrics report | 20260507 21:56:37.982 | INFO | RTMetrics report populated | 20260507 21:56:37.982 | INFO | Opening tmp RTMetrics json file | 20260507 21:56:38.018 | INFO | Serializing RTMetrics to json | 20260507 21:56:38.079 | INFO | Serialization done, swapping tmp file now... | 20260507 21:56:38.088 | INFO | RTMetrics done | 20260507 21:56:38.117 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.604 seconds | 20260507 21:56:38.117 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.41% / 0.01% / 56.58% | 20260507 21:56:38.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5451% / 3.97364e-06% / 5.16574e-05% | 20260507 21:56:38.117 | INFO | Baz IO overall: | 20260507 21:56:38.117 | INFO | BytesWritten : 1319169426010 | 20260507 21:56:38.117 | INFO | NumEvents : 639254261753 | 20260507 21:56:39.551 | INFO | Analyzing chunk frames = [2895872,2896384) | 20260507 21:56:39.551 | INFO | MemInfo: { | 20260507 21:56:39.551 | INFO | "gpu" : " 17 GB", | 20260507 21:56:39.551 | INFO | "huge" : "", | 20260507 21:56:39.551 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:39.551 | INFO | "malloc" : " 14 GB", | 20260507 21:56:39.551 | INFO | "pinned" : " 29 GB" | 20260507 21:56:39.551 | INFO | } | 20260507 21:56:39.551 | INFO | Perf report for the last 5.13661 seconds: | 20260507 21:56:39.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:39.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:39.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:39.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:39.551 | INFO | ANALYSIS : 68.7%, 2.9, 105.5ms, 0 | 20260507 21:56:39.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:39.552 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260507 21:56:39.552 | INFO | PRE_HQ : 67.2%, 1.8, 62.5ms, 0 | 20260507 21:56:39.552 | INFO | BAZWRITER : 69.7%, 3.9, 139.4ms, 0 | 20260507 21:56:39.552 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 21:56:39.552 | INFO | Uploaded 36GiB to GPU in 2.55299s (14.1011GiB/s) | 20260507 21:56:39.552 | INFO | Downloaded 38GiB to GPU in 2.70091s (14.0693GiB/s) | 20260507 21:56:44.688 | INFO | Analyzing chunk frames = [2896384,2896896) | 20260507 21:56:44.688 | INFO | MemInfo: { | 20260507 21:56:44.688 | INFO | "gpu" : " 17 GB", | 20260507 21:56:44.688 | INFO | "huge" : "", | 20260507 21:56:44.689 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:44.689 | INFO | "malloc" : " 14 GB", | 20260507 21:56:44.689 | INFO | "pinned" : " 29 GB" | 20260507 21:56:44.689 | INFO | } | 20260507 21:56:44.689 | INFO | Perf report for the last 5.13727 seconds: | 20260507 21:56:44.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:44.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 21:56:44.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:56:44.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:44.689 | INFO | Uploaded 36GiB to GPU in 2.48167s (14.5064GiB/s) | 20260507 21:56:44.689 | INFO | Downloaded 34GiB to GPU in 2.46861s (13.7729GiB/s) | 20260507 21:56:49.832 | INFO | Analyzing chunk frames = [2896896,2897408) | 20260507 21:56:49.832 | INFO | MemInfo: { | 20260507 21:56:49.832 | INFO | "gpu" : " 17 GB", | 20260507 21:56:49.832 | INFO | "huge" : "", | 20260507 21:56:49.832 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:49.832 | INFO | "malloc" : " 14 GB", | 20260507 21:56:49.832 | INFO | "pinned" : " 29 GB" | 20260507 21:56:49.832 | INFO | } | 20260507 21:56:49.832 | INFO | Perf report for the last 5.14322 seconds: | 20260507 21:56:49.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:49.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.832 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 21:56:49.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.832 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 21:56:49.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:49.833 | INFO | Uploaded 36GiB to GPU in 2.50623s (14.3642GiB/s) | 20260507 21:56:49.833 | INFO | Downloaded 34GiB to GPU in 2.45474s (13.8508GiB/s) | 20260507 21:56:54.976 | INFO | Analyzing chunk frames = [2897408,2897920) | 20260507 21:56:54.976 | INFO | MemInfo: { | 20260507 21:56:54.976 | INFO | "gpu" : " 17 GB", | 20260507 21:56:54.976 | INFO | "huge" : "", | 20260507 21:56:54.976 | INFO | "huge_shared" : " 29 GB", | 20260507 21:56:54.976 | INFO | "malloc" : " 14 GB", | 20260507 21:56:54.976 | INFO | "pinned" : " 29 GB" | 20260507 21:56:54.976 | INFO | } | 20260507 21:56:54.976 | INFO | Perf report for the last 5.14416 seconds: | 20260507 21:56:54.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:56:54.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 21:56:54.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:56:54.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:56:54.976 | INFO | Uploaded 36GiB to GPU in 2.47055s (14.5717GiB/s) | 20260507 21:56:54.976 | INFO | Downloaded 34GiB to GPU in 2.38679s (14.2451GiB/s) | 20260507 21:57:00.113 | INFO | Analyzing chunk frames = [2897920,2898432) | 20260507 21:57:00.113 | INFO | MemInfo: { | 20260507 21:57:00.113 | INFO | "gpu" : " 17 GB", | 20260507 21:57:00.113 | INFO | "huge" : "", | 20260507 21:57:00.113 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:00.113 | INFO | "malloc" : " 14 GB", | 20260507 21:57:00.113 | INFO | "pinned" : " 29 GB" | 20260507 21:57:00.113 | INFO | } | 20260507 21:57:00.113 | INFO | Perf report for the last 5.13689 seconds: | 20260507 21:57:00.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:00.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.113 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:57:00.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.113 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 21:57:00.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:00.114 | INFO | Uploaded 36GiB to GPU in 2.36865s (15.1985GiB/s) | 20260507 21:57:00.114 | INFO | Downloaded 34GiB to GPU in 2.49895s (13.6057GiB/s) | 20260507 21:57:05.249 | INFO | Analyzing chunk frames = [2898432,2898944) | 20260507 21:57:05.249 | INFO | MemInfo: { | 20260507 21:57:05.249 | INFO | "gpu" : " 17 GB", | 20260507 21:57:05.249 | INFO | "huge" : "", | 20260507 21:57:05.249 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:05.249 | INFO | "malloc" : " 14 GB", | 20260507 21:57:05.249 | INFO | "pinned" : " 29 GB" | 20260507 21:57:05.249 | INFO | } | 20260507 21:57:05.249 | INFO | Perf report for the last 5.13652 seconds: | 20260507 21:57:05.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:05.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:57:05.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:57:05.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:05.250 | INFO | Uploaded 36GiB to GPU in 2.46655s (14.5953GiB/s) | 20260507 21:57:05.250 | INFO | Downloaded 34GiB to GPU in 2.41883s (14.0564GiB/s) | 20260507 21:57:10.391 | INFO | Analyzing chunk frames = [2898944,2899456) | 20260507 21:57:10.391 | INFO | MemInfo: { | 20260507 21:57:10.391 | INFO | "gpu" : " 17 GB", | 20260507 21:57:10.391 | INFO | "huge" : "", | 20260507 21:57:10.391 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:10.391 | INFO | "malloc" : " 14 GB", | 20260507 21:57:10.391 | INFO | "pinned" : " 29 GB" | 20260507 21:57:10.391 | INFO | } | 20260507 21:57:10.391 | INFO | Perf report for the last 5.14134 seconds: | 20260507 21:57:10.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:10.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.391 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:57:10.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.391 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 21:57:10.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:10.392 | INFO | Uploaded 36GiB to GPU in 2.52039s (14.2835GiB/s) | 20260507 21:57:10.392 | INFO | Downloaded 34GiB to GPU in 2.45525s (13.8479GiB/s) | 20260507 21:57:15.531 | INFO | Analyzing chunk frames = [2899456,2899968) | 20260507 21:57:15.531 | INFO | MemInfo: { | 20260507 21:57:15.531 | INFO | "gpu" : " 17 GB", | 20260507 21:57:15.531 | INFO | "huge" : "", | 20260507 21:57:15.531 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:15.531 | INFO | "malloc" : " 14 GB", | 20260507 21:57:15.531 | INFO | "pinned" : " 29 GB" | 20260507 21:57:15.531 | INFO | } | 20260507 21:57:15.531 | INFO | Perf report for the last 5.14034 seconds: | 20260507 21:57:15.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:15.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 21:57:15.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:57:15.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:15.531 | INFO | Uploaded 36GiB to GPU in 2.4892s (14.4625GiB/s) | 20260507 21:57:15.531 | INFO | Downloaded 34GiB to GPU in 2.44032s (13.9326GiB/s) | 20260507 21:57:15.532 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:57:15.532 | INFO | Total mem usage: 16.7 GiB | 20260507 21:57:15.532 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:57:15.532 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.5%) | 20260507 21:57:15.532 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:57:15.532 | INFO | High Water Breakdown: | 20260507 21:57:15.532 | INFO | 60.1% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:57:15.532 | INFO | 39.8% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:57:15.532 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:57:15.532 | INFO | ------------------------------------------ | 20260507 21:57:15.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:57:15.532 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:57:15.532 | INFO | Total mem usage: 26.5 GiB | 20260507 21:57:15.532 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:57:15.532 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:57:15.532 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:57:15.532 | INFO | High Water Breakdown: | 20260507 21:57:15.532 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:57:15.532 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:57:15.532 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:57:15.532 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:57:15.532 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:57:15.532 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:57:15.532 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:57:15.532 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:57:15.532 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:57:15.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:57:15.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:57:15.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:57:15.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:57:15.532 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:57:15.532 | INFO | ------------------------------------------ | 20260507 21:57:15.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:57:15.532 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:57:15.532 | INFO | Total mem usage: 27.8 GiB | 20260507 21:57:15.532 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:57:15.532 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:57:15.532 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:57:15.532 | INFO | High Water Breakdown: | 20260507 21:57:15.532 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:57:15.532 | INFO | ------------------------------------------ | 20260507 21:57:15.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:57:15.532 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:57:15.532 | INFO | Total mem usage: 15.7 GiB | 20260507 21:57:15.532 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:57:15.532 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:57:15.532 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:57:15.532 | INFO | High Water Breakdown: | 20260507 21:57:15.532 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:57:15.532 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:57:15.532 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:57:15.532 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:57:15.532 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:57:15.532 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:57:15.532 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:57:15.532 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:57:15.532 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:57:15.532 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:57:15.532 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:57:15.532 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:57:15.532 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:57:15.532 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:57:15.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:57:15.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:57:15.532 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:57:15.532 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:57:15.533 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:57:15.533 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:57:15.533 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:57:15.533 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:57:15.533 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:57:15.533 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:57:15.533 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:57:15.533 | INFO | ------------------------------------------ | 20260507 21:57:15.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:57:15.533 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:57:15.533 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:57:15.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:57:15.533 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.58e+01, 1.41e+01 / 1.54e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 0.30%, AggregateStats -- 9.95e-02 / 1.05e-01, 1.16e-02 / 4.77e-03 | 20260507 21:57:15.533 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.45e+00, 5.98e-01 / 2.74e-01 | 20260507 21:57:15.533 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.48e+00, 2.09e-01 / 1.28e-01 | 20260507 21:57:15.533 | INFO | PerfReport: 15.78%, DME -- 4.16e+01 / 4.49e+01, 3.82e+00 / 1.28e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 1.86e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.46e+00 / 2.72e+00, 4.32e-01 / 2.52e-01 | 20260507 21:57:15.533 | INFO | PerfReport: 3.52%, Metrics -- 1.16e+00 / 1.37e+00, 2.93e-01 / 1.75e-01 | 20260507 21:57:15.533 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.95e-01, 5.84e-02 / 3.40e-02 | 20260507 21:57:15.533 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:57:15.533 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:57:15.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:57:15.533 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 24.22%, Upload -- 2.58e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 30.95%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 1.77%, LockUpload -- 1.88e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 2.76%, LockDownload -- 2.94e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 15.98%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 21:57:15.533 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260507 21:57:15.630 | INFO | Starting a round of Baz IO | 20260507 21:57:19.102 | INFO | Populating RTMetrics report | 20260507 21:57:19.103 | INFO | RTMetrics report populated | 20260507 21:57:19.103 | INFO | Opening tmp RTMetrics json file | 20260507 21:57:19.261 | INFO | Serializing RTMetrics to json | 20260507 21:57:19.269 | INFO | Serialization done, swapping tmp file now... | 20260507 21:57:19.274 | INFO | RTMetrics done | 20260507 21:57:19.470 | INFO | Finished a round of Baz IO: Wrote 1.749 GiB in 3.839 seconds | 20260507 21:57:19.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.40% / 0.01% / 56.59% | 20260507 21:57:19.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.54204% / 7.94729e-06% / 3.97364e-05% | 20260507 21:57:19.470 | INFO | Baz IO overall: | 20260507 21:57:19.470 | INFO | BytesWritten : 1321046941631 | 20260507 21:57:19.470 | INFO | NumEvents : 640161591804 | 20260507 21:57:20.667 | INFO | Analyzing chunk frames = [2899968,2900480) | 20260507 21:57:20.667 | INFO | MemInfo: { | 20260507 21:57:20.667 | INFO | "gpu" : " 17 GB", | 20260507 21:57:20.667 | INFO | "huge" : "", | 20260507 21:57:20.667 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:20.667 | INFO | "malloc" : " 14 GB", | 20260507 21:57:20.667 | INFO | "pinned" : " 29 GB" | 20260507 21:57:20.667 | INFO | } | 20260507 21:57:20.667 | INFO | Perf report for the last 5.13623 seconds: | 20260507 21:57:20.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:20.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:20.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:20.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:20.668 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 21:57:20.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:20.668 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 21:57:20.668 | INFO | PRE_HQ : 66.6%, 1.8, 61.3ms, 0 | 20260507 21:57:20.668 | INFO | BAZWRITER : 74.3%, 4.3, 165.3ms, 0 | 20260507 21:57:20.668 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260507 21:57:20.668 | INFO | Uploaded 36GiB to GPU in 2.51131s (14.3351GiB/s) | 20260507 21:57:20.668 | INFO | Downloaded 38GiB to GPU in 2.69648s (14.0925GiB/s) | 20260507 21:57:25.803 | INFO | Analyzing chunk frames = [2900480,2900992) | 20260507 21:57:25.803 | INFO | MemInfo: { | 20260507 21:57:25.803 | INFO | "gpu" : " 17 GB", | 20260507 21:57:25.803 | INFO | "huge" : "", | 20260507 21:57:25.803 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:25.803 | INFO | "malloc" : " 14 GB", | 20260507 21:57:25.803 | INFO | "pinned" : " 29 GB" | 20260507 21:57:25.803 | INFO | } | 20260507 21:57:25.803 | INFO | Perf report for the last 5.13558 seconds: | 20260507 21:57:25.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:25.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.803 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 21:57:25.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.803 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 21:57:25.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:25.804 | INFO | Uploaded 36GiB to GPU in 2.51706s (14.3024GiB/s) | 20260507 21:57:25.804 | INFO | Downloaded 34GiB to GPU in 2.47909s (13.7147GiB/s) | 20260507 21:57:30.946 | INFO | Analyzing chunk frames = [2900992,2901504) | 20260507 21:57:30.946 | INFO | MemInfo: { | 20260507 21:57:30.946 | INFO | "gpu" : " 17 GB", | 20260507 21:57:30.946 | INFO | "huge" : "", | 20260507 21:57:30.946 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:30.946 | INFO | "malloc" : " 14 GB", | 20260507 21:57:30.946 | INFO | "pinned" : " 29 GB" | 20260507 21:57:30.946 | INFO | } | 20260507 21:57:30.946 | INFO | Perf report for the last 5.14281 seconds: | 20260507 21:57:30.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:30.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 21:57:30.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 21:57:30.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:30.946 | INFO | Uploaded 36GiB to GPU in 2.51507s (14.3137GiB/s) | 20260507 21:57:30.946 | INFO | Downloaded 34GiB to GPU in 2.45185s (13.8671GiB/s) | 20260507 21:57:36.088 | INFO | Analyzing chunk frames = [2901504,2902016) | 20260507 21:57:36.088 | INFO | MemInfo: { | 20260507 21:57:36.088 | INFO | "gpu" : " 17 GB", | 20260507 21:57:36.088 | INFO | "huge" : "", | 20260507 21:57:36.088 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:36.088 | INFO | "malloc" : " 14 GB", | 20260507 21:57:36.088 | INFO | "pinned" : " 29 GB" | 20260507 21:57:36.088 | INFO | } | 20260507 21:57:36.088 | INFO | Perf report for the last 5.14242 seconds: | 20260507 21:57:36.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:36.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.088 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 21:57:36.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.088 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:57:36.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:36.089 | INFO | Uploaded 36GiB to GPU in 2.47192s (14.5636GiB/s) | 20260507 21:57:36.089 | INFO | Downloaded 34GiB to GPU in 2.38483s (14.2568GiB/s) | 20260507 21:57:41.227 | INFO | Analyzing chunk frames = [2902016,2902528) | 20260507 21:57:41.227 | INFO | MemInfo: { | 20260507 21:57:41.227 | INFO | "gpu" : " 17 GB", | 20260507 21:57:41.227 | INFO | "huge" : "", | 20260507 21:57:41.227 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:41.227 | INFO | "malloc" : " 14 GB", | 20260507 21:57:41.227 | INFO | "pinned" : " 29 GB" | 20260507 21:57:41.227 | INFO | } | 20260507 21:57:41.227 | INFO | Perf report for the last 5.13898 seconds: | 20260507 21:57:41.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:41.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.227 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 21:57:41.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.227 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 21:57:41.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:41.228 | INFO | Uploaded 36GiB to GPU in 2.35617s (15.279GiB/s) | 20260507 21:57:41.228 | INFO | Downloaded 34GiB to GPU in 2.49767s (13.6127GiB/s) | 20260507 21:57:46.365 | INFO | Analyzing chunk frames = [2902528,2903040) | 20260507 21:57:46.365 | INFO | MemInfo: { | 20260507 21:57:46.365 | INFO | "gpu" : " 17 GB", | 20260507 21:57:46.365 | INFO | "huge" : "", | 20260507 21:57:46.365 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:46.365 | INFO | "malloc" : " 14 GB", | 20260507 21:57:46.365 | INFO | "pinned" : " 29 GB" | 20260507 21:57:46.365 | INFO | } | 20260507 21:57:46.365 | INFO | Perf report for the last 5.13774 seconds: | 20260507 21:57:46.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:46.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.365 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 21:57:46.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.365 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:57:46.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:46.366 | INFO | Uploaded 36GiB to GPU in 2.47712s (14.533GiB/s) | 20260507 21:57:46.366 | INFO | Downloaded 34GiB to GPU in 2.41008s (14.1074GiB/s) | 20260507 21:57:51.504 | INFO | Analyzing chunk frames = [2903040,2903552) | 20260507 21:57:51.504 | INFO | MemInfo: { | 20260507 21:57:51.504 | INFO | "gpu" : " 17 GB", | 20260507 21:57:51.504 | INFO | "huge" : "", | 20260507 21:57:51.504 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:51.504 | INFO | "malloc" : " 14 GB", | 20260507 21:57:51.504 | INFO | "pinned" : " 29 GB" | 20260507 21:57:51.504 | INFO | } | 20260507 21:57:51.505 | INFO | Perf report for the last 5.13948 seconds: | 20260507 21:57:51.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:51.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 21:57:51.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 21:57:51.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:51.505 | INFO | Uploaded 36GiB to GPU in 2.55106s (14.1118GiB/s) | 20260507 21:57:51.505 | INFO | Downloaded 34GiB to GPU in 2.4555s (13.8465GiB/s) | 20260507 21:57:56.639 | INFO | Analyzing chunk frames = [2903552,2904064) | 20260507 21:57:56.639 | INFO | MemInfo: { | 20260507 21:57:56.639 | INFO | "gpu" : " 17 GB", | 20260507 21:57:56.640 | INFO | "huge" : "", | 20260507 21:57:56.640 | INFO | "huge_shared" : " 29 GB", | 20260507 21:57:56.640 | INFO | "malloc" : " 14 GB", | 20260507 21:57:56.640 | INFO | "pinned" : " 29 GB" | 20260507 21:57:56.640 | INFO | } | 20260507 21:57:56.640 | INFO | Perf report for the last 5.13522 seconds: | 20260507 21:57:56.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:57:56.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 21:57:56.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 21:57:56.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:57:56.640 | INFO | Uploaded 36GiB to GPU in 2.52824s (14.2391GiB/s) | 20260507 21:57:56.641 | INFO | Downloaded 34GiB to GPU in 2.43883s (13.9411GiB/s) | 20260507 21:57:56.754 | INFO | Starting a round of Baz IO | 20260507 21:58:00.209 | INFO | Populating RTMetrics report | 20260507 21:58:00.209 | INFO | RTMetrics report populated | 20260507 21:58:00.209 | INFO | Opening tmp RTMetrics json file | 20260507 21:58:00.213 | INFO | Serializing RTMetrics to json | 20260507 21:58:00.250 | INFO | Serialization done, swapping tmp file now... | 20260507 21:58:00.297 | INFO | RTMetrics done | 20260507 21:58:00.345 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.591 seconds | 20260507 21:58:00.345 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.40% / 0.01% / 56.60% | 20260507 21:58:00.346 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53812% / 1.19209e-05% / 6.35783e-05% | 20260507 21:58:00.346 | INFO | Baz IO overall: | 20260507 21:58:00.346 | INFO | BytesWritten : 1322923959949 | 20260507 21:58:00.346 | INFO | NumEvents : 641068821797 | 20260507 21:58:01.778 | INFO | Analyzing chunk frames = [2904064,2904576) | 20260507 21:58:01.778 | INFO | MemInfo: { | 20260507 21:58:01.778 | INFO | "gpu" : " 17 GB", | 20260507 21:58:01.778 | INFO | "huge" : "", | 20260507 21:58:01.778 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:01.778 | INFO | "malloc" : " 14 GB", | 20260507 21:58:01.778 | INFO | "pinned" : " 29 GB" | 20260507 21:58:01.778 | INFO | } | 20260507 21:58:01.778 | INFO | Perf report for the last 5.13864 seconds: | 20260507 21:58:01.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:01.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:01.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:01.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:01.779 | INFO | ANALYSIS : 68.7%, 2.9, 105.5ms, 0 | 20260507 21:58:01.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:01.779 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260507 21:58:01.779 | INFO | PRE_HQ : 67.8%, 1.9, 65.4ms, 0 | 20260507 21:58:01.779 | INFO | BAZWRITER : 69.5%, 4.1, 147.9ms, 0 | 20260507 21:58:01.779 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260507 21:58:01.779 | INFO | Uploaded 36GiB to GPU in 2.56498s (14.0352GiB/s) | 20260507 21:58:01.779 | INFO | Downloaded 38GiB to GPU in 2.67981s (14.1801GiB/s) | 20260507 21:58:06.926 | INFO | Analyzing chunk frames = [2904576,2905088) | 20260507 21:58:06.926 | INFO | MemInfo: { | 20260507 21:58:06.926 | INFO | "gpu" : " 17 GB", | 20260507 21:58:06.926 | INFO | "huge" : "", | 20260507 21:58:06.926 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:06.926 | INFO | "malloc" : " 14 GB", | 20260507 21:58:06.926 | INFO | "pinned" : " 29 GB" | 20260507 21:58:06.926 | INFO | } | 20260507 21:58:06.926 | INFO | Perf report for the last 5.14772 seconds: | 20260507 21:58:06.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:06.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.926 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 21:58:06.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.926 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:58:06.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:06.927 | INFO | Uploaded 36GiB to GPU in 2.51406s (14.3194GiB/s) | 20260507 21:58:06.927 | INFO | Downloaded 34GiB to GPU in 2.47613s (13.7311GiB/s) | 20260507 21:58:12.064 | INFO | Analyzing chunk frames = [2905088,2905600) | 20260507 21:58:12.064 | INFO | MemInfo: { | 20260507 21:58:12.064 | INFO | "gpu" : " 17 GB", | 20260507 21:58:12.064 | INFO | "huge" : "", | 20260507 21:58:12.065 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:12.065 | INFO | "malloc" : " 14 GB", | 20260507 21:58:12.065 | INFO | "pinned" : " 29 GB" | 20260507 21:58:12.065 | INFO | } | 20260507 21:58:12.065 | INFO | Perf report for the last 5.13849 seconds: | 20260507 21:58:12.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:12.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 21:58:12.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 21:58:12.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:12.065 | INFO | Uploaded 36GiB to GPU in 2.50198s (14.3886GiB/s) | 20260507 21:58:12.065 | INFO | Downloaded 34GiB to GPU in 2.45455s (13.8518GiB/s) | 20260507 21:58:17.201 | INFO | Analyzing chunk frames = [2905600,2906112) | 20260507 21:58:17.201 | INFO | MemInfo: { | 20260507 21:58:17.201 | INFO | "gpu" : " 17 GB", | 20260507 21:58:17.201 | INFO | "huge" : "", | 20260507 21:58:17.201 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:17.201 | INFO | "malloc" : " 14 GB", | 20260507 21:58:17.201 | INFO | "pinned" : " 29 GB" | 20260507 21:58:17.201 | INFO | } | 20260507 21:58:17.201 | INFO | Perf report for the last 5.13638 seconds: | 20260507 21:58:17.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:17.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.201 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 21:58:17.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.201 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:58:17.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:17.202 | INFO | Uploaded 36GiB to GPU in 2.48281s (14.4997GiB/s) | 20260507 21:58:17.202 | INFO | Downloaded 34GiB to GPU in 2.40017s (14.1657GiB/s) | 20260507 21:58:22.345 | INFO | Analyzing chunk frames = [2906112,2906624) | 20260507 21:58:22.345 | INFO | MemInfo: { | 20260507 21:58:22.345 | INFO | "gpu" : " 17 GB", | 20260507 21:58:22.345 | INFO | "huge" : "", | 20260507 21:58:22.345 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:22.345 | INFO | "malloc" : " 14 GB", | 20260507 21:58:22.345 | INFO | "pinned" : " 29 GB" | 20260507 21:58:22.345 | INFO | } | 20260507 21:58:22.345 | INFO | Perf report for the last 5.14389 seconds: | 20260507 21:58:22.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:22.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.345 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 21:58:22.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.345 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 21:58:22.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:22.346 | INFO | Uploaded 36GiB to GPU in 2.34288s (15.3657GiB/s) | 20260507 21:58:22.346 | INFO | Downloaded 34GiB to GPU in 2.51337s (13.5277GiB/s) | 20260507 21:58:27.479 | INFO | Analyzing chunk frames = [2906624,2907136) | 20260507 21:58:27.479 | INFO | MemInfo: { | 20260507 21:58:27.479 | INFO | "gpu" : " 17 GB", | 20260507 21:58:27.479 | INFO | "huge" : "", | 20260507 21:58:27.479 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:27.479 | INFO | "malloc" : " 14 GB", | 20260507 21:58:27.479 | INFO | "pinned" : " 29 GB" | 20260507 21:58:27.479 | INFO | } | 20260507 21:58:27.479 | INFO | Perf report for the last 5.13393 seconds: | 20260507 21:58:27.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:27.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.479 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260507 21:58:27.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.479 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260507 21:58:27.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:27.480 | INFO | Uploaded 36GiB to GPU in 2.49749s (14.4145GiB/s) | 20260507 21:58:27.480 | INFO | Downloaded 34GiB to GPU in 2.43016s (13.9909GiB/s) | 20260507 21:58:32.614 | INFO | Analyzing chunk frames = [2907136,2907648) | 20260507 21:58:32.614 | INFO | MemInfo: { | 20260507 21:58:32.614 | INFO | "gpu" : " 17 GB", | 20260507 21:58:32.614 | INFO | "huge" : "", | 20260507 21:58:32.614 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:32.614 | INFO | "malloc" : " 14 GB", | 20260507 21:58:32.614 | INFO | "pinned" : " 29 GB" | 20260507 21:58:32.614 | INFO | } | 20260507 21:58:32.614 | INFO | Perf report for the last 5.13544 seconds: | 20260507 21:58:32.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:32.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.615 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 21:58:32.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.615 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 21:58:32.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:32.615 | INFO | Uploaded 36GiB to GPU in 2.54039s (14.171GiB/s) | 20260507 21:58:32.615 | INFO | Downloaded 34GiB to GPU in 2.46756s (13.7788GiB/s) | 20260507 21:58:37.755 | INFO | Analyzing chunk frames = [2907648,2908160) | 20260507 21:58:37.755 | INFO | MemInfo: { | 20260507 21:58:37.755 | INFO | "gpu" : " 17 GB", | 20260507 21:58:37.755 | INFO | "huge" : "", | 20260507 21:58:37.755 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:37.755 | INFO | "malloc" : " 14 GB", | 20260507 21:58:37.755 | INFO | "pinned" : " 29 GB" | 20260507 21:58:37.755 | INFO | } | 20260507 21:58:37.756 | INFO | Perf report for the last 5.14117 seconds: | 20260507 21:58:37.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:37.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:58:37.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 21:58:37.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:37.756 | INFO | Uploaded 36GiB to GPU in 2.48951s (14.4606GiB/s) | 20260507 21:58:37.756 | INFO | Downloaded 34GiB to GPU in 2.44211s (13.9224GiB/s) | 20260507 21:58:37.756 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:58:37.756 | INFO | Total mem usage: 16.7 GiB | 20260507 21:58:37.756 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:58:37.756 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.7%) | 20260507 21:58:37.756 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:58:37.756 | INFO | High Water Breakdown: | 20260507 21:58:37.756 | INFO | 60.2% : 9.07 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:58:37.756 | INFO | 39.7% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:58:37.756 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:58:37.756 | INFO | ------------------------------------------ | 20260507 21:58:37.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:58:37.757 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:58:37.757 | INFO | Total mem usage: 26.5 GiB | 20260507 21:58:37.757 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:58:37.757 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 21:58:37.757 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 21:58:37.757 | INFO | High Water Breakdown: | 20260507 21:58:37.757 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:58:37.757 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:58:37.757 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:58:37.757 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:58:37.757 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:58:37.757 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:58:37.757 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:58:37.757 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:58:37.757 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:58:37.757 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:58:37.757 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:58:37.757 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:58:37.757 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:58:37.757 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:58:37.757 | INFO | ------------------------------------------ | 20260507 21:58:37.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:58:37.757 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:58:37.757 | INFO | Total mem usage: 27.8 GiB | 20260507 21:58:37.757 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:58:37.757 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:58:37.757 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:58:37.757 | INFO | High Water Breakdown: | 20260507 21:58:37.757 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:58:37.757 | INFO | ------------------------------------------ | 20260507 21:58:37.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:58:37.757 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:58:37.757 | INFO | Total mem usage: 15.7 GiB | 20260507 21:58:37.757 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:58:37.757 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:58:37.757 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:58:37.757 | INFO | High Water Breakdown: | 20260507 21:58:37.757 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:58:37.757 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:58:37.757 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:58:37.757 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:58:37.757 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:58:37.757 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:58:37.757 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:58:37.757 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:58:37.757 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:58:37.757 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:58:37.757 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:58:37.757 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:58:37.757 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:58:37.757 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:58:37.757 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:58:37.757 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:58:37.757 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:58:37.757 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:58:37.757 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:58:37.757 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:58:37.757 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:58:37.757 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:58:37.758 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:58:37.758 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:58:37.758 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:58:37.758 | INFO | ------------------------------------------ | 20260507 21:58:37.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:58:37.758 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:58:37.758 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:58:37.758 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:58:37.758 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.41e+01 / 1.53e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 0.30%, AggregateStats -- 9.95e-02 / 1.04e-01, 1.23e-02 / 6.94e-03 | 20260507 21:58:37.758 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.49e+00, 6.03e-01 / 2.39e-01 | 20260507 21:58:37.758 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.18e-01 | 20260507 21:58:37.758 | INFO | PerfReport: 15.71%, DME -- 4.15e+01 / 4.46e+01, 3.68e+00 / 1.49e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.86e+01 / 2.08e+01, 1.83e+00 / 2.07e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.44e+00 / 2.71e+00, 4.27e-01 / 2.28e-01 | 20260507 21:58:37.758 | INFO | PerfReport: 3.51%, Metrics -- 1.16e+00 / 1.34e+00, 2.96e-01 / 1.66e-01 | 20260507 21:58:37.758 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.61e-01 / 3.92e-01, 5.94e-02 / 3.25e-02 | 20260507 21:58:37.758 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:58:37.758 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:58:37.758 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:58:37.758 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 24.34%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 30.97%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 1.93%, LockUpload -- 2.06e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 2.48%, LockDownload -- 2.65e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 16.04%, LockCompute -- 1.71e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 21:58:37.758 | INFO | PerfReport: 0.01%, Misc -- 1.58e-02 / 0.00e+00, 2.54e-02 / 0.00e+00 | 20260507 21:58:37.868 | INFO | Starting a round of Baz IO | 20260507 21:58:41.333 | INFO | Populating RTMetrics report | 20260507 21:58:41.334 | INFO | RTMetrics report populated | 20260507 21:58:41.334 | INFO | Opening tmp RTMetrics json file | 20260507 21:58:41.362 | INFO | Serializing RTMetrics to json | 20260507 21:58:41.394 | INFO | Serialization done, swapping tmp file now... | 20260507 21:58:41.421 | INFO | RTMetrics done | 20260507 21:58:41.455 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.587 seconds | 20260507 21:58:41.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.39% / 0.01% / 56.60% | 20260507 21:58:41.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53521% / 1.19209e-05% / 5.96046e-05% | 20260507 21:58:41.456 | INFO | Baz IO overall: | 20260507 21:58:41.456 | INFO | BytesWritten : 1324799240917 | 20260507 21:58:41.456 | INFO | NumEvents : 641974690475 | 20260507 21:58:42.896 | INFO | Analyzing chunk frames = [2908160,2908672) | 20260507 21:58:42.896 | INFO | MemInfo: { | 20260507 21:58:42.896 | INFO | "gpu" : " 17 GB", | 20260507 21:58:42.896 | INFO | "huge" : "", | 20260507 21:58:42.896 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:42.896 | INFO | "malloc" : " 14 GB", | 20260507 21:58:42.896 | INFO | "pinned" : " 29 GB" | 20260507 21:58:42.896 | INFO | } | 20260507 21:58:42.896 | INFO | Perf report for the last 5.14045 seconds: | 20260507 21:58:42.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:42.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:42.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:42.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:42.896 | INFO | ANALYSIS : 68.9%, 2.9, 106.4ms, 0 | 20260507 21:58:42.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:42.896 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260507 21:58:42.896 | INFO | PRE_HQ : 67.1%, 1.8, 63.8ms, 0 | 20260507 21:58:42.896 | INFO | BAZWRITER : 69.2%, 3.9, 138.0ms, 0 | 20260507 21:58:42.896 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 21:58:42.897 | INFO | Uploaded 36GiB to GPU in 2.53958s (14.1756GiB/s) | 20260507 21:58:42.897 | INFO | Downloaded 38GiB to GPU in 2.70003s (14.0739GiB/s) | 20260507 21:58:48.038 | INFO | Analyzing chunk frames = [2908672,2909184) | 20260507 21:58:48.038 | INFO | MemInfo: { | 20260507 21:58:48.038 | INFO | "gpu" : " 17 GB", | 20260507 21:58:48.038 | INFO | "huge" : "", | 20260507 21:58:48.038 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:48.038 | INFO | "malloc" : " 14 GB", | 20260507 21:58:48.038 | INFO | "pinned" : " 29 GB" | 20260507 21:58:48.038 | INFO | } | 20260507 21:58:48.038 | INFO | Perf report for the last 5.14223 seconds: | 20260507 21:58:48.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:48.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.038 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 21:58:48.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.038 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 21:58:48.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:48.039 | INFO | Uploaded 36GiB to GPU in 2.5219s (14.2749GiB/s) | 20260507 21:58:48.039 | INFO | Downloaded 34GiB to GPU in 2.4594s (13.8245GiB/s) | 20260507 21:58:53.180 | INFO | Analyzing chunk frames = [2909184,2909696) | 20260507 21:58:53.180 | INFO | MemInfo: { | 20260507 21:58:53.180 | INFO | "gpu" : " 17 GB", | 20260507 21:58:53.180 | INFO | "huge" : "", | 20260507 21:58:53.180 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:53.180 | INFO | "malloc" : " 14 GB", | 20260507 21:58:53.180 | INFO | "pinned" : " 29 GB" | 20260507 21:58:53.181 | INFO | } | 20260507 21:58:53.181 | INFO | Perf report for the last 5.14242 seconds: | 20260507 21:58:53.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:53.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 21:58:53.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 21:58:53.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:53.181 | INFO | Uploaded 36GiB to GPU in 2.55124s (14.1108GiB/s) | 20260507 21:58:53.181 | INFO | Downloaded 34GiB to GPU in 2.44792s (13.8893GiB/s) | 20260507 21:58:58.315 | INFO | Analyzing chunk frames = [2909696,2910208) | 20260507 21:58:58.315 | INFO | MemInfo: { | 20260507 21:58:58.315 | INFO | "gpu" : " 17 GB", | 20260507 21:58:58.315 | INFO | "huge" : "", | 20260507 21:58:58.315 | INFO | "huge_shared" : " 29 GB", | 20260507 21:58:58.315 | INFO | "malloc" : " 14 GB", | 20260507 21:58:58.315 | INFO | "pinned" : " 29 GB" | 20260507 21:58:58.315 | INFO | } | 20260507 21:58:58.315 | INFO | Perf report for the last 5.13419 seconds: | 20260507 21:58:58.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:58:58.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260507 21:58:58.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 21:58:58.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:58:58.315 | INFO | Uploaded 36GiB to GPU in 2.49812s (14.4108GiB/s) | 20260507 21:58:58.315 | INFO | Downloaded 34GiB to GPU in 2.4019s (14.1555GiB/s) | 20260507 21:59:03.453 | INFO | Analyzing chunk frames = [2910208,2910720) | 20260507 21:59:03.453 | INFO | MemInfo: { | 20260507 21:59:03.453 | INFO | "gpu" : " 17 GB", | 20260507 21:59:03.453 | INFO | "huge" : "", | 20260507 21:59:03.453 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:03.453 | INFO | "malloc" : " 14 GB", | 20260507 21:59:03.453 | INFO | "pinned" : " 29 GB" | 20260507 21:59:03.453 | INFO | } | 20260507 21:59:03.454 | INFO | Perf report for the last 5.13854 seconds: | 20260507 21:59:03.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:03.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:59:03.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 21:59:03.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:03.454 | INFO | Uploaded 36GiB to GPU in 2.37361s (15.1668GiB/s) | 20260507 21:59:03.454 | INFO | Downloaded 34GiB to GPU in 2.49823s (13.6096GiB/s) | 20260507 21:59:08.593 | INFO | Analyzing chunk frames = [2910720,2911232) | 20260507 21:59:08.593 | INFO | MemInfo: { | 20260507 21:59:08.593 | INFO | "gpu" : " 17 GB", | 20260507 21:59:08.593 | INFO | "huge" : "", | 20260507 21:59:08.593 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:08.593 | INFO | "malloc" : " 14 GB", | 20260507 21:59:08.593 | INFO | "pinned" : " 29 GB" | 20260507 21:59:08.594 | INFO | } | 20260507 21:59:08.594 | INFO | Perf report for the last 5.14007 seconds: | 20260507 21:59:08.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:08.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 21:59:08.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:59:08.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:08.594 | INFO | Uploaded 36GiB to GPU in 2.49463s (14.431GiB/s) | 20260507 21:59:08.594 | INFO | Downloaded 34GiB to GPU in 2.4231s (14.0316GiB/s) | 20260507 21:59:13.733 | INFO | Analyzing chunk frames = [2911232,2911744) | 20260507 21:59:13.733 | INFO | MemInfo: { | 20260507 21:59:13.733 | INFO | "gpu" : " 17 GB", | 20260507 21:59:13.733 | INFO | "huge" : "", | 20260507 21:59:13.733 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:13.733 | INFO | "malloc" : " 14 GB", | 20260507 21:59:13.733 | INFO | "pinned" : " 29 GB" | 20260507 21:59:13.733 | INFO | } | 20260507 21:59:13.733 | INFO | Perf report for the last 5.13939 seconds: | 20260507 21:59:13.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:13.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 21:59:13.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 21:59:13.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:13.733 | INFO | Uploaded 36GiB to GPU in 2.53444s (14.2043GiB/s) | 20260507 21:59:13.733 | INFO | Downloaded 34GiB to GPU in 2.45853s (13.8294GiB/s) | 20260507 21:59:18.877 | INFO | Analyzing chunk frames = [2911744,2912256) | 20260507 21:59:18.877 | INFO | MemInfo: { | 20260507 21:59:18.877 | INFO | "gpu" : " 17 GB", | 20260507 21:59:18.877 | INFO | "huge" : "", | 20260507 21:59:18.877 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:18.877 | INFO | "malloc" : " 14 GB", | 20260507 21:59:18.877 | INFO | "pinned" : " 29 GB" | 20260507 21:59:18.877 | INFO | } | 20260507 21:59:18.877 | INFO | Perf report for the last 5.14433 seconds: | 20260507 21:59:18.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:18.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.877 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 21:59:18.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.878 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 21:59:18.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:18.878 | INFO | Uploaded 36GiB to GPU in 2.49669s (14.4191GiB/s) | 20260507 21:59:18.878 | INFO | Downloaded 34GiB to GPU in 2.44108s (13.9283GiB/s) | 20260507 21:59:18.990 | INFO | Starting a round of Baz IO | 20260507 21:59:22.449 | INFO | Populating RTMetrics report | 20260507 21:59:22.449 | INFO | RTMetrics report populated | 20260507 21:59:22.449 | INFO | Opening tmp RTMetrics json file | 20260507 21:59:22.472 | INFO | Serializing RTMetrics to json | 20260507 21:59:22.497 | INFO | Serialization done, swapping tmp file now... | 20260507 21:59:22.519 | INFO | RTMetrics done | 20260507 21:59:22.553 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.563 seconds | 20260507 21:59:22.553 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.38% / 0.01% / 56.61% | 20260507 21:59:22.553 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53251% / 3.97364e-06% / 4.37101e-05% | 20260507 21:59:22.553 | INFO | Baz IO overall: | 20260507 21:59:22.554 | INFO | BytesWritten : 1326675305232 | 20260507 21:59:22.554 | INFO | NumEvents : 642881096720 | 20260507 21:59:24.017 | INFO | Analyzing chunk frames = [2912256,2912768) | 20260507 21:59:24.017 | INFO | MemInfo: { | 20260507 21:59:24.017 | INFO | "gpu" : " 17 GB", | 20260507 21:59:24.017 | INFO | "huge" : "", | 20260507 21:59:24.017 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:24.017 | INFO | "malloc" : " 14 GB", | 20260507 21:59:24.017 | INFO | "pinned" : " 29 GB" | 20260507 21:59:24.017 | INFO | } | 20260507 21:59:24.017 | INFO | Perf report for the last 5.13945 seconds: | 20260507 21:59:24.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:24.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:24.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:24.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:24.017 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 21:59:24.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:24.017 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 21:59:24.017 | INFO | PRE_HQ : 65.5%, 1.7, 58.6ms, 0 | 20260507 21:59:24.017 | INFO | BAZWRITER : 68.6%, 3.6, 127.1ms, 0 | 20260507 21:59:24.017 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 21:59:24.017 | INFO | Uploaded 36GiB to GPU in 2.52829s (14.2389GiB/s) | 20260507 21:59:24.017 | INFO | Downloaded 38GiB to GPU in 2.69316s (14.1098GiB/s) | 20260507 21:59:29.155 | INFO | Analyzing chunk frames = [2912768,2913280) | 20260507 21:59:29.155 | INFO | MemInfo: { | 20260507 21:59:29.155 | INFO | "gpu" : " 17 GB", | 20260507 21:59:29.155 | INFO | "huge" : "", | 20260507 21:59:29.155 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:29.155 | INFO | "malloc" : " 14 GB", | 20260507 21:59:29.155 | INFO | "pinned" : " 29 GB" | 20260507 21:59:29.155 | INFO | } | 20260507 21:59:29.155 | INFO | Perf report for the last 5.13797 seconds: | 20260507 21:59:29.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:29.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 21:59:29.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 21:59:29.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:29.155 | INFO | Uploaded 36GiB to GPU in 2.52544s (14.255GiB/s) | 20260507 21:59:29.155 | INFO | Downloaded 34GiB to GPU in 2.50119s (13.5935GiB/s) | 20260507 21:59:34.295 | INFO | Analyzing chunk frames = [2913280,2913792) | 20260507 21:59:34.295 | INFO | MemInfo: { | 20260507 21:59:34.295 | INFO | "gpu" : " 17 GB", | 20260507 21:59:34.295 | INFO | "huge" : "", | 20260507 21:59:34.295 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:34.295 | INFO | "malloc" : " 14 GB", | 20260507 21:59:34.295 | INFO | "pinned" : " 29 GB" | 20260507 21:59:34.295 | INFO | } | 20260507 21:59:34.295 | INFO | Perf report for the last 5.1401 seconds: | 20260507 21:59:34.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:34.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 21:59:34.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 21:59:34.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:34.295 | INFO | Uploaded 36GiB to GPU in 2.55804s (14.0733GiB/s) | 20260507 21:59:34.295 | INFO | Downloaded 34GiB to GPU in 2.47452s (13.7401GiB/s) | 20260507 21:59:39.431 | INFO | Analyzing chunk frames = [2913792,2914304) | 20260507 21:59:39.431 | INFO | MemInfo: { | 20260507 21:59:39.431 | INFO | "gpu" : " 17 GB", | 20260507 21:59:39.431 | INFO | "huge" : "", | 20260507 21:59:39.431 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:39.431 | INFO | "malloc" : " 14 GB", | 20260507 21:59:39.431 | INFO | "pinned" : " 29 GB" | 20260507 21:59:39.432 | INFO | } | 20260507 21:59:39.432 | INFO | Perf report for the last 5.13664 seconds: | 20260507 21:59:39.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:39.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 21:59:39.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 21:59:39.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:39.432 | INFO | Uploaded 36GiB to GPU in 2.51626s (14.3069GiB/s) | 20260507 21:59:39.432 | INFO | Downloaded 34GiB to GPU in 2.50842s (13.5544GiB/s) | 20260507 21:59:44.568 | INFO | Analyzing chunk frames = [2914304,2914816) | 20260507 21:59:44.568 | INFO | MemInfo: { | 20260507 21:59:44.568 | INFO | "gpu" : " 17 GB", | 20260507 21:59:44.568 | INFO | "huge" : "", | 20260507 21:59:44.568 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:44.568 | INFO | "malloc" : " 14 GB", | 20260507 21:59:44.568 | INFO | "pinned" : " 29 GB" | 20260507 21:59:44.568 | INFO | } | 20260507 21:59:44.568 | INFO | Perf report for the last 5.13674 seconds: | 20260507 21:59:44.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:44.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.568 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 21:59:44.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.568 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 21:59:44.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:44.569 | INFO | Uploaded 36GiB to GPU in 2.36859s (15.1989GiB/s) | 20260507 21:59:44.569 | INFO | Downloaded 34GiB to GPU in 2.49443s (13.6304GiB/s) | 20260507 21:59:49.701 | INFO | Analyzing chunk frames = [2914816,2915328) | 20260507 21:59:49.701 | INFO | MemInfo: { | 20260507 21:59:49.701 | INFO | "gpu" : " 17 GB", | 20260507 21:59:49.701 | INFO | "huge" : "", | 20260507 21:59:49.701 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:49.701 | INFO | "malloc" : " 14 GB", | 20260507 21:59:49.701 | INFO | "pinned" : " 29 GB" | 20260507 21:59:49.701 | INFO | } | 20260507 21:59:49.701 | INFO | Perf report for the last 5.13303 seconds: | 20260507 21:59:49.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:49.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.701 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 21:59:49.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.701 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 21:59:49.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:49.702 | INFO | Uploaded 36GiB to GPU in 2.51465s (14.3161GiB/s) | 20260507 21:59:49.702 | INFO | Downloaded 34GiB to GPU in 2.42131s (14.042GiB/s) | 20260507 21:59:54.847 | INFO | Analyzing chunk frames = [2915328,2915840) | 20260507 21:59:54.847 | INFO | MemInfo: { | 20260507 21:59:54.847 | INFO | "gpu" : " 17 GB", | 20260507 21:59:54.847 | INFO | "huge" : "", | 20260507 21:59:54.847 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:54.847 | INFO | "malloc" : " 14 GB", | 20260507 21:59:54.847 | INFO | "pinned" : " 29 GB" | 20260507 21:59:54.847 | INFO | } | 20260507 21:59:54.847 | INFO | Perf report for the last 5.14549 seconds: | 20260507 21:59:54.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:54.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 21:59:54.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 21:59:54.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:54.847 | INFO | Uploaded 36GiB to GPU in 2.5271s (14.2456GiB/s) | 20260507 21:59:54.847 | INFO | Downloaded 34GiB to GPU in 2.45923s (13.8255GiB/s) | 20260507 21:59:59.989 | INFO | Analyzing chunk frames = [2915840,2916352) | 20260507 21:59:59.989 | INFO | MemInfo: { | 20260507 21:59:59.989 | INFO | "gpu" : " 17 GB", | 20260507 21:59:59.990 | INFO | "huge" : "", | 20260507 21:59:59.990 | INFO | "huge_shared" : " 29 GB", | 20260507 21:59:59.990 | INFO | "malloc" : " 14 GB", | 20260507 21:59:59.990 | INFO | "pinned" : " 29 GB" | 20260507 21:59:59.990 | INFO | } | 20260507 21:59:59.990 | INFO | Perf report for the last 5.14287 seconds: | 20260507 21:59:59.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 21:59:59.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 21:59:59.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 21:59:59.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 21:59:59.990 | INFO | Uploaded 36GiB to GPU in 2.50811s (14.3534GiB/s) | 20260507 21:59:59.990 | INFO | Downloaded 34GiB to GPU in 2.44422s (13.9104GiB/s) | 20260507 21:59:59.990 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 21:59:59.990 | INFO | Total mem usage: 16.7 GiB | 20260507 21:59:59.990 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 21:59:59.990 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14%) | 20260507 21:59:59.990 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 21:59:59.990 | INFO | High Water Breakdown: | 20260507 21:59:59.990 | INFO | 59.7% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 21:59:59.990 | INFO | 40.1% : 5.88 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 21:59:59.990 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 21:59:59.991 | INFO | ------------------------------------------ | 20260507 21:59:59.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:59:59.991 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 21:59:59.991 | INFO | Total mem usage: 26.5 GiB | 20260507 21:59:59.991 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 21:59:59.991 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 21:59:59.991 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 21:59:59.991 | INFO | High Water Breakdown: | 20260507 21:59:59.991 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:59:59.991 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:59:59.991 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:59:59.991 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:59:59.991 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:59:59.991 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:59:59.991 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 21:59:59.991 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 21:59:59.991 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:59:59.991 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 21:59:59.991 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 21:59:59.991 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 21:59:59.991 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 21:59:59.991 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 21:59:59.991 | INFO | ------------------------------------------ | 20260507 21:59:59.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:59:59.991 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 21:59:59.991 | INFO | Total mem usage: 27.8 GiB | 20260507 21:59:59.991 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 21:59:59.991 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 21:59:59.991 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 21:59:59.991 | INFO | High Water Breakdown: | 20260507 21:59:59.991 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 21:59:59.991 | INFO | ------------------------------------------ | 20260507 21:59:59.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:59:59.991 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 21:59:59.991 | INFO | Total mem usage: 15.7 GiB | 20260507 21:59:59.991 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 21:59:59.991 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 21:59:59.991 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 21:59:59.991 | INFO | High Water Breakdown: | 20260507 21:59:59.991 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 21:59:59.991 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 21:59:59.991 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 21:59:59.991 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 21:59:59.991 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 21:59:59.991 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 21:59:59.991 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 21:59:59.991 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 21:59:59.991 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 21:59:59.991 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 21:59:59.991 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 21:59:59.991 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 21:59:59.991 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 21:59:59.991 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 21:59:59.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 21:59:59.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 21:59:59.991 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 21:59:59.991 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 21:59:59.991 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 21:59:59.991 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 21:59:59.991 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 21:59:59.991 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 21:59:59.991 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 21:59:59.991 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 21:59:59.991 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 21:59:59.991 | INFO | ------------------------------------------ | 20260507 21:59:59.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 21:59:59.991 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 21:59:59.991 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:59:59.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:59:59.991 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.64e+01, 1.43e+01 / 1.55e+00 | 20260507 21:59:59.991 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.27e-02 / 5.86e-03 | 20260507 21:59:59.991 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.47e+00, 5.96e-01 / 2.30e-01 | 20260507 21:59:59.991 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.47e+00, 2.08e-01 / 1.17e-01 | 20260507 21:59:59.991 | INFO | PerfReport: 15.89%, DME -- 4.19e+01 / 4.55e+01, 3.86e+00 / 1.64e+00 | 20260507 21:59:59.991 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.76e+00 / 1.86e+00 | 20260507 21:59:59.991 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.46e+00 / 2.75e+00, 4.33e-01 / 2.31e-01 | 20260507 21:59:59.991 | INFO | PerfReport: 3.50%, Metrics -- 1.15e+00 / 1.37e+00, 2.91e-01 / 1.44e-01 | 20260507 21:59:59.991 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 3.92e-01, 5.90e-02 / 3.14e-02 | 20260507 21:59:59.991 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 21:59:59.991 | INFO | PerfReport: Gathered 1536 samples. | 20260507 21:59:59.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 21:59:59.992 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 24.39%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 30.83%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 1.99%, LockUpload -- 2.13e+00 / 0.00e+00, 5.12e+00 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 2.62%, LockDownload -- 2.80e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 21:59:59.992 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.61e-02 / 0.00e+00 | 20260507 22:00:00.086 | INFO | Starting a round of Baz IO | 20260507 22:00:03.559 | INFO | Populating RTMetrics report | 20260507 22:00:03.560 | INFO | RTMetrics report populated | 20260507 22:00:03.560 | INFO | Opening tmp RTMetrics json file | 20260507 22:00:03.590 | INFO | Serializing RTMetrics to json | 20260507 22:00:03.654 | INFO | Serialization done, swapping tmp file now... | 20260507 22:00:03.663 | INFO | RTMetrics done | 20260507 22:00:03.681 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.594 seconds | 20260507 22:00:03.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.38% / 0.01% / 56.62% | 20260507 22:00:03.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53106% / 2.38419e-05% / 6.75519e-05% | 20260507 22:00:03.681 | INFO | Baz IO overall: | 20260507 22:00:03.681 | INFO | BytesWritten : 1328551963072 | 20260507 22:00:03.681 | INFO | NumEvents : 643788290132 | 20260507 22:00:05.132 | INFO | Analyzing chunk frames = [2916352,2916864) | 20260507 22:00:05.132 | INFO | MemInfo: { | 20260507 22:00:05.132 | INFO | "gpu" : " 17 GB", | 20260507 22:00:05.132 | INFO | "huge" : "", | 20260507 22:00:05.132 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:05.132 | INFO | "malloc" : " 14 GB", | 20260507 22:00:05.132 | INFO | "pinned" : " 29 GB" | 20260507 22:00:05.132 | INFO | } | 20260507 22:00:05.132 | INFO | Perf report for the last 5.14257 seconds: | 20260507 22:00:05.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:05.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:05.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:05.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:05.133 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 22:00:05.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:05.133 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 22:00:05.133 | INFO | PRE_HQ : 65.6%, 1.7, 57.4ms, 0 | 20260507 22:00:05.133 | INFO | BAZWRITER : 69.5%, 4.0, 143.9ms, 0 | 20260507 22:00:05.133 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260507 22:00:05.133 | INFO | Uploaded 36GiB to GPU in 2.46194s (14.6226GiB/s) | 20260507 22:00:05.133 | INFO | Downloaded 38GiB to GPU in 2.68528s (14.1512GiB/s) | 20260507 22:00:10.268 | INFO | Analyzing chunk frames = [2916864,2917376) | 20260507 22:00:10.268 | INFO | MemInfo: { | 20260507 22:00:10.268 | INFO | "gpu" : " 17 GB", | 20260507 22:00:10.268 | INFO | "huge" : "", | 20260507 22:00:10.268 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:10.268 | INFO | "malloc" : " 14 GB", | 20260507 22:00:10.268 | INFO | "pinned" : " 29 GB" | 20260507 22:00:10.268 | INFO | } | 20260507 22:00:10.268 | INFO | Perf report for the last 5.13571 seconds: | 20260507 22:00:10.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:10.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:00:10.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:00:10.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:10.268 | INFO | Uploaded 36GiB to GPU in 2.44836s (14.7037GiB/s) | 20260507 22:00:10.268 | INFO | Downloaded 34GiB to GPU in 2.5003s (13.5984GiB/s) | 20260507 22:00:15.408 | INFO | Analyzing chunk frames = [2917376,2917888) | 20260507 22:00:15.408 | INFO | MemInfo: { | 20260507 22:00:15.408 | INFO | "gpu" : " 17 GB", | 20260507 22:00:15.408 | INFO | "huge" : "", | 20260507 22:00:15.408 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:15.408 | INFO | "malloc" : " 14 GB", | 20260507 22:00:15.408 | INFO | "pinned" : " 29 GB" | 20260507 22:00:15.408 | INFO | } | 20260507 22:00:15.408 | INFO | Perf report for the last 5.14012 seconds: | 20260507 22:00:15.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:15.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260507 22:00:15.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:00:15.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:15.408 | INFO | Uploaded 36GiB to GPU in 2.48327s (14.497GiB/s) | 20260507 22:00:15.409 | INFO | Downloaded 34GiB to GPU in 2.48132s (13.7024GiB/s) | 20260507 22:00:20.545 | INFO | Analyzing chunk frames = [2917888,2918400) | 20260507 22:00:20.545 | INFO | MemInfo: { | 20260507 22:00:20.545 | INFO | "gpu" : " 17 GB", | 20260507 22:00:20.545 | INFO | "huge" : "", | 20260507 22:00:20.545 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:20.545 | INFO | "malloc" : " 14 GB", | 20260507 22:00:20.545 | INFO | "pinned" : " 29 GB" | 20260507 22:00:20.545 | INFO | } | 20260507 22:00:20.545 | INFO | Perf report for the last 5.13672 seconds: | 20260507 22:00:20.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:20.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.545 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260507 22:00:20.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.545 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:00:20.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:20.546 | INFO | Uploaded 36GiB to GPU in 2.42193s (14.8642GiB/s) | 20260507 22:00:20.546 | INFO | Downloaded 34GiB to GPU in 2.41441s (14.0821GiB/s) | 20260507 22:00:25.684 | INFO | Analyzing chunk frames = [2918400,2918912) | 20260507 22:00:25.684 | INFO | MemInfo: { | 20260507 22:00:25.684 | INFO | "gpu" : " 17 GB", | 20260507 22:00:25.684 | INFO | "huge" : "", | 20260507 22:00:25.684 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:25.684 | INFO | "malloc" : " 14 GB", | 20260507 22:00:25.684 | INFO | "pinned" : " 29 GB" | 20260507 22:00:25.684 | INFO | } | 20260507 22:00:25.684 | INFO | Perf report for the last 5.13943 seconds: | 20260507 22:00:25.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:25.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.684 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:00:25.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.684 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 22:00:25.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:25.685 | INFO | Uploaded 36GiB to GPU in 2.32567s (15.4794GiB/s) | 20260507 22:00:25.685 | INFO | Downloaded 34GiB to GPU in 2.52306s (13.4757GiB/s) | 20260507 22:00:30.829 | INFO | Analyzing chunk frames = [2918912,2919424) | 20260507 22:00:30.829 | INFO | MemInfo: { | 20260507 22:00:30.829 | INFO | "gpu" : " 17 GB", | 20260507 22:00:30.829 | INFO | "huge" : "", | 20260507 22:00:30.829 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:30.829 | INFO | "malloc" : " 14 GB", | 20260507 22:00:30.830 | INFO | "pinned" : " 29 GB" | 20260507 22:00:30.830 | INFO | } | 20260507 22:00:30.830 | INFO | Perf report for the last 5.14522 seconds: | 20260507 22:00:30.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:30.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260507 22:00:30.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:00:30.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:30.830 | INFO | Uploaded 36GiB to GPU in 2.4446s (14.7263GiB/s) | 20260507 22:00:30.830 | INFO | Downloaded 34GiB to GPU in 2.43861s (13.9424GiB/s) | 20260507 22:00:35.966 | INFO | Analyzing chunk frames = [2919424,2919936) | 20260507 22:00:35.966 | INFO | MemInfo: { | 20260507 22:00:35.966 | INFO | "gpu" : " 17 GB", | 20260507 22:00:35.966 | INFO | "huge" : "", | 20260507 22:00:35.966 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:35.966 | INFO | "malloc" : " 14 GB", | 20260507 22:00:35.966 | INFO | "pinned" : " 29 GB" | 20260507 22:00:35.966 | INFO | } | 20260507 22:00:35.966 | INFO | Perf report for the last 5.13662 seconds: | 20260507 22:00:35.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:35.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.966 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 22:00:35.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.966 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260507 22:00:35.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:35.967 | INFO | Uploaded 36GiB to GPU in 2.47154s (14.5658GiB/s) | 20260507 22:00:35.967 | INFO | Downloaded 34GiB to GPU in 2.45797s (13.8325GiB/s) | 20260507 22:00:41.106 | INFO | Analyzing chunk frames = [2919936,2920448) | 20260507 22:00:41.106 | INFO | MemInfo: { | 20260507 22:00:41.106 | INFO | "gpu" : " 17 GB", | 20260507 22:00:41.106 | INFO | "huge" : "", | 20260507 22:00:41.106 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:41.106 | INFO | "malloc" : " 14 GB", | 20260507 22:00:41.106 | INFO | "pinned" : " 29 GB" | 20260507 22:00:41.106 | INFO | } | 20260507 22:00:41.106 | INFO | Perf report for the last 5.14005 seconds: | 20260507 22:00:41.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:41.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.106 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:00:41.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.107 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:00:41.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:41.107 | INFO | Uploaded 36GiB to GPU in 2.46424s (14.609GiB/s) | 20260507 22:00:41.107 | INFO | Downloaded 34GiB to GPU in 2.4616s (13.8122GiB/s) | 20260507 22:00:41.220 | INFO | Starting a round of Baz IO | 20260507 22:00:44.693 | INFO | Populating RTMetrics report | 20260507 22:00:44.693 | INFO | RTMetrics report populated | 20260507 22:00:44.693 | INFO | Opening tmp RTMetrics json file | 20260507 22:00:44.761 | INFO | Serializing RTMetrics to json | 20260507 22:00:44.850 | INFO | Serialization done, swapping tmp file now... | 20260507 22:00:44.858 | INFO | RTMetrics done | 20260507 22:00:44.866 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.645 seconds | 20260507 22:00:44.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.37% / 0.01% / 56.62% | 20260507 22:00:44.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.52926% / 3.97364e-06% / 6.35783e-05% | 20260507 22:00:44.866 | INFO | Baz IO overall: | 20260507 22:00:44.866 | INFO | BytesWritten : 1330428789013 | 20260507 22:00:44.866 | INFO | NumEvents : 644695581241 | 20260507 22:00:46.245 | INFO | Analyzing chunk frames = [2920448,2920960) | 20260507 22:00:46.245 | INFO | MemInfo: { | 20260507 22:00:46.245 | INFO | "gpu" : " 17 GB", | 20260507 22:00:46.245 | INFO | "huge" : "", | 20260507 22:00:46.245 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:46.245 | INFO | "malloc" : " 14 GB", | 20260507 22:00:46.245 | INFO | "pinned" : " 29 GB" | 20260507 22:00:46.245 | INFO | } | 20260507 22:00:46.245 | INFO | Perf report for the last 5.13894 seconds: | 20260507 22:00:46.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:46.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:46.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:46.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:46.245 | INFO | ANALYSIS : 68.9%, 2.9, 105.0ms, 0 | 20260507 22:00:46.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:46.245 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260507 22:00:46.245 | INFO | PRE_HQ : 67.7%, 1.8, 62.7ms, 0 | 20260507 22:00:46.245 | INFO | BAZWRITER : 70.6%, 4.1, 148.9ms, 0 | 20260507 22:00:46.245 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260507 22:00:46.246 | INFO | Uploaded 36GiB to GPU in 2.57644s (13.9727GiB/s) | 20260507 22:00:46.246 | INFO | Downloaded 38GiB to GPU in 2.69505s (14.0999GiB/s) | 20260507 22:00:51.379 | INFO | Analyzing chunk frames = [2920960,2921472) | 20260507 22:00:51.379 | INFO | MemInfo: { | 20260507 22:00:51.379 | INFO | "gpu" : " 17 GB", | 20260507 22:00:51.379 | INFO | "huge" : "", | 20260507 22:00:51.379 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:51.380 | INFO | "malloc" : " 14 GB", | 20260507 22:00:51.380 | INFO | "pinned" : " 29 GB" | 20260507 22:00:51.380 | INFO | } | 20260507 22:00:51.380 | INFO | Perf report for the last 5.13449 seconds: | 20260507 22:00:51.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:51.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 22:00:51.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:00:51.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:51.380 | INFO | Uploaded 36GiB to GPU in 2.53717s (14.189GiB/s) | 20260507 22:00:51.380 | INFO | Downloaded 34GiB to GPU in 2.48609s (13.6761GiB/s) | 20260507 22:00:56.515 | INFO | Analyzing chunk frames = [2921472,2921984) | 20260507 22:00:56.515 | INFO | MemInfo: { | 20260507 22:00:56.515 | INFO | "gpu" : " 17 GB", | 20260507 22:00:56.515 | INFO | "huge" : "", | 20260507 22:00:56.515 | INFO | "huge_shared" : " 29 GB", | 20260507 22:00:56.515 | INFO | "malloc" : " 14 GB", | 20260507 22:00:56.515 | INFO | "pinned" : " 29 GB" | 20260507 22:00:56.515 | INFO | } | 20260507 22:00:56.515 | INFO | Perf report for the last 5.1353 seconds: | 20260507 22:00:56.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:00:56.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 22:00:56.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:00:56.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:00:56.515 | INFO | Uploaded 36GiB to GPU in 2.50549s (14.3684GiB/s) | 20260507 22:00:56.515 | INFO | Downloaded 34GiB to GPU in 2.47997s (13.7099GiB/s) | 20260507 22:01:01.657 | INFO | Analyzing chunk frames = [2921984,2922496) | 20260507 22:01:01.657 | INFO | MemInfo: { | 20260507 22:01:01.657 | INFO | "gpu" : " 17 GB", | 20260507 22:01:01.657 | INFO | "huge" : "", | 20260507 22:01:01.657 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:01.657 | INFO | "malloc" : " 14 GB", | 20260507 22:01:01.657 | INFO | "pinned" : " 29 GB" | 20260507 22:01:01.657 | INFO | } | 20260507 22:01:01.657 | INFO | Perf report for the last 5.14213 seconds: | 20260507 22:01:01.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:01.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 22:01:01.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:01:01.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:01.657 | INFO | Uploaded 36GiB to GPU in 2.42357s (14.8541GiB/s) | 20260507 22:01:01.658 | INFO | Downloaded 34GiB to GPU in 2.3999s (14.1673GiB/s) | 20260507 22:01:06.798 | INFO | Analyzing chunk frames = [2922496,2923008) | 20260507 22:01:06.798 | INFO | MemInfo: { | 20260507 22:01:06.798 | INFO | "gpu" : " 17 GB", | 20260507 22:01:06.798 | INFO | "huge" : "", | 20260507 22:01:06.798 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:06.798 | INFO | "malloc" : " 14 GB", | 20260507 22:01:06.798 | INFO | "pinned" : " 29 GB" | 20260507 22:01:06.798 | INFO | } | 20260507 22:01:06.798 | INFO | Perf report for the last 5.14114 seconds: | 20260507 22:01:06.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:06.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.798 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:01:06.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.798 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:01:06.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:06.799 | INFO | Uploaded 36GiB to GPU in 2.3146s (15.5534GiB/s) | 20260507 22:01:06.799 | INFO | Downloaded 34GiB to GPU in 2.48961s (13.6568GiB/s) | 20260507 22:01:11.943 | INFO | Analyzing chunk frames = [2923008,2923520) | 20260507 22:01:11.943 | INFO | MemInfo: { | 20260507 22:01:11.943 | INFO | "gpu" : " 17 GB", | 20260507 22:01:11.943 | INFO | "huge" : "", | 20260507 22:01:11.943 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:11.943 | INFO | "malloc" : " 14 GB", | 20260507 22:01:11.943 | INFO | "pinned" : " 29 GB" | 20260507 22:01:11.943 | INFO | } | 20260507 22:01:11.943 | INFO | Perf report for the last 5.14463 seconds: | 20260507 22:01:11.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:11.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260507 22:01:11.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:01:11.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:11.943 | INFO | Uploaded 36GiB to GPU in 2.45268s (14.6778GiB/s) | 20260507 22:01:11.943 | INFO | Downloaded 34GiB to GPU in 2.41594s (14.0732GiB/s) | 20260507 22:01:17.082 | INFO | Analyzing chunk frames = [2923520,2924032) | 20260507 22:01:17.082 | INFO | MemInfo: { | 20260507 22:01:17.082 | INFO | "gpu" : " 17 GB", | 20260507 22:01:17.082 | INFO | "huge" : "", | 20260507 22:01:17.082 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:17.082 | INFO | "malloc" : " 14 GB", | 20260507 22:01:17.082 | INFO | "pinned" : " 29 GB" | 20260507 22:01:17.082 | INFO | } | 20260507 22:01:17.082 | INFO | Perf report for the last 5.13912 seconds: | 20260507 22:01:17.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:17.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:01:17.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 22:01:17.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:17.082 | INFO | Uploaded 36GiB to GPU in 2.5368s (14.1911GiB/s) | 20260507 22:01:17.083 | INFO | Downloaded 34GiB to GPU in 2.46358s (13.8011GiB/s) | 20260507 22:01:22.218 | INFO | Analyzing chunk frames = [2924032,2924544) | 20260507 22:01:22.218 | INFO | MemInfo: { | 20260507 22:01:22.218 | INFO | "gpu" : " 17 GB", | 20260507 22:01:22.218 | INFO | "huge" : "", | 20260507 22:01:22.218 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:22.218 | INFO | "malloc" : " 14 GB", | 20260507 22:01:22.218 | INFO | "pinned" : " 29 GB" | 20260507 22:01:22.218 | INFO | } | 20260507 22:01:22.218 | INFO | Perf report for the last 5.13568 seconds: | 20260507 22:01:22.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:22.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:01:22.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:01:22.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:22.218 | INFO | Uploaded 36GiB to GPU in 2.50542s (14.3689GiB/s) | 20260507 22:01:22.218 | INFO | Downloaded 34GiB to GPU in 2.44597s (13.9004GiB/s) | 20260507 22:01:22.219 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:01:22.219 | INFO | Total mem usage: 16.7 GiB | 20260507 22:01:22.219 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:01:22.219 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260507 22:01:22.219 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:01:22.219 | INFO | High Water Breakdown: | 20260507 22:01:22.219 | INFO | 59.8% : 8.79 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:01:22.219 | INFO | 40.1% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:01:22.219 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:01:22.219 | INFO | ------------------------------------------ | 20260507 22:01:22.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:01:22.219 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:01:22.219 | INFO | Total mem usage: 26.5 GiB | 20260507 22:01:22.219 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:01:22.219 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:01:22.219 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:01:22.219 | INFO | High Water Breakdown: | 20260507 22:01:22.219 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:01:22.219 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:01:22.219 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:01:22.219 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:01:22.219 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:01:22.219 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:01:22.219 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:01:22.219 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:01:22.219 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:01:22.219 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:01:22.219 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:01:22.219 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:01:22.219 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:01:22.219 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:01:22.219 | INFO | ------------------------------------------ | 20260507 22:01:22.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:01:22.219 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:01:22.219 | INFO | Total mem usage: 27.8 GiB | 20260507 22:01:22.219 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:01:22.219 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:01:22.219 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:01:22.219 | INFO | High Water Breakdown: | 20260507 22:01:22.219 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:01:22.219 | INFO | ------------------------------------------ | 20260507 22:01:22.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:01:22.219 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:01:22.219 | INFO | Total mem usage: 15.7 GiB | 20260507 22:01:22.219 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:01:22.219 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:01:22.219 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:01:22.219 | INFO | High Water Breakdown: | 20260507 22:01:22.219 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:01:22.219 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:01:22.219 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:01:22.220 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:01:22.220 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:01:22.220 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:01:22.220 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:01:22.220 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:01:22.220 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:01:22.220 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:01:22.220 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:01:22.220 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:01:22.220 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:01:22.220 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:01:22.220 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:01:22.220 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:01:22.220 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:01:22.220 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:01:22.220 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:01:22.220 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:01:22.220 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:01:22.220 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:01:22.220 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:01:22.220 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:01:22.220 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:01:22.220 | INFO | ------------------------------------------ | 20260507 22:01:22.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:01:22.220 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:01:22.220 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:01:22.220 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:01:22.220 | INFO | PerfReport: Full Execution -- 3.31e+01 / 7.60e+01, 1.41e+01 / 1.61e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.04e-01, 1.17e-02 / 5.46e-03 | 20260507 22:01:22.220 | INFO | PerfReport: 12.16%, Baseline -- 4.02e+00 / 4.50e+00, 6.04e-01 / 2.18e-01 | 20260507 22:01:22.220 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.11e-01 / 1.07e-01 | 20260507 22:01:22.220 | INFO | PerfReport: 15.78%, DME -- 4.18e+01 / 4.52e+01, 3.78e+00 / 1.41e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.84e+00 / 1.96e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.33e-01 / 2.66e-01 | 20260507 22:01:22.220 | INFO | PerfReport: 3.50%, Metrics -- 1.16e+00 / 1.36e+00, 2.93e-01 / 1.69e-01 | 20260507 22:01:22.220 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 3.88e-01, 5.92e-02 / 3.13e-02 | 20260507 22:01:22.220 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:01:22.220 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:01:22.220 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:01:22.220 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 24.07%, Upload -- 2.56e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 24.36%, Download -- 2.59e+01 / 0.00e+00, 8.52e+00 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 31.07%, Compute -- 3.31e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 1.66%, LockUpload -- 1.77e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 2.80%, LockDownload -- 2.98e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 16.03%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:01:22.220 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260507 22:01:22.339 | INFO | Starting a round of Baz IO | 20260507 22:01:25.817 | INFO | Populating RTMetrics report | 20260507 22:01:25.817 | INFO | RTMetrics report populated | 20260507 22:01:25.817 | INFO | Opening tmp RTMetrics json file | 20260507 22:01:25.844 | INFO | Serializing RTMetrics to json | 20260507 22:01:25.846 | INFO | Serialization done, swapping tmp file now... | 20260507 22:01:25.847 | INFO | RTMetrics done | 20260507 22:01:25.924 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.585 seconds | 20260507 22:01:25.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.36% / 0.01% / 56.63% | 20260507 22:01:25.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.52473% / 0% / 4.76837e-05% | 20260507 22:01:25.924 | INFO | Baz IO overall: | 20260507 22:01:25.924 | INFO | BytesWritten : 1332304679307 | 20260507 22:01:25.924 | INFO | NumEvents : 645602241081 | 20260507 22:01:27.360 | INFO | Analyzing chunk frames = [2924544,2925056) | 20260507 22:01:27.360 | INFO | MemInfo: { | 20260507 22:01:27.360 | INFO | "gpu" : " 17 GB", | 20260507 22:01:27.360 | INFO | "huge" : "", | 20260507 22:01:27.360 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:27.360 | INFO | "malloc" : " 14 GB", | 20260507 22:01:27.360 | INFO | "pinned" : " 29 GB" | 20260507 22:01:27.360 | INFO | } | 20260507 22:01:27.360 | INFO | Perf report for the last 5.14184 seconds: | 20260507 22:01:27.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:27.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:27.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:27.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:27.360 | INFO | ANALYSIS : 69.3%, 2.9, 105.3ms, 0 | 20260507 22:01:27.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:27.360 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 22:01:27.360 | INFO | PRE_HQ : 68.0%, 1.8, 63.4ms, 0 | 20260507 22:01:27.360 | INFO | BAZWRITER : 69.5%, 4.2, 148.9ms, 0 | 20260507 22:01:27.360 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:01:27.360 | INFO | Uploaded 36GiB to GPU in 2.57149s (13.9997GiB/s) | 20260507 22:01:27.360 | INFO | Downloaded 38GiB to GPU in 2.7037s (14.0548GiB/s) | 20260507 22:01:32.496 | INFO | Analyzing chunk frames = [2925056,2925568) | 20260507 22:01:32.496 | INFO | MemInfo: { | 20260507 22:01:32.496 | INFO | "gpu" : " 17 GB", | 20260507 22:01:32.496 | INFO | "huge" : "", | 20260507 22:01:32.496 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:32.496 | INFO | "malloc" : " 14 GB", | 20260507 22:01:32.496 | INFO | "pinned" : " 29 GB" | 20260507 22:01:32.496 | INFO | } | 20260507 22:01:32.496 | INFO | Perf report for the last 5.13662 seconds: | 20260507 22:01:32.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:32.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.496 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 22:01:32.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.496 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:01:32.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:32.497 | INFO | Uploaded 36GiB to GPU in 2.56613s (14.0289GiB/s) | 20260507 22:01:32.497 | INFO | Downloaded 34GiB to GPU in 2.4891s (13.6596GiB/s) | 20260507 22:01:37.636 | INFO | Analyzing chunk frames = [2925568,2926080) | 20260507 22:01:37.636 | INFO | MemInfo: { | 20260507 22:01:37.636 | INFO | "gpu" : " 17 GB", | 20260507 22:01:37.636 | INFO | "huge" : "", | 20260507 22:01:37.636 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:37.636 | INFO | "malloc" : " 14 GB", | 20260507 22:01:37.636 | INFO | "pinned" : " 29 GB" | 20260507 22:01:37.636 | INFO | } | 20260507 22:01:37.636 | INFO | Perf report for the last 5.14016 seconds: | 20260507 22:01:37.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:37.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.637 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:01:37.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.637 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:01:37.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:37.637 | INFO | Uploaded 36GiB to GPU in 2.50842s (14.3517GiB/s) | 20260507 22:01:37.637 | INFO | Downloaded 34GiB to GPU in 2.46856s (13.7732GiB/s) | 20260507 22:01:42.776 | INFO | Analyzing chunk frames = [2926080,2926592) | 20260507 22:01:42.776 | INFO | MemInfo: { | 20260507 22:01:42.776 | INFO | "gpu" : " 17 GB", | 20260507 22:01:42.776 | INFO | "huge" : "", | 20260507 22:01:42.776 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:42.776 | INFO | "malloc" : " 14 GB", | 20260507 22:01:42.776 | INFO | "pinned" : " 29 GB" | 20260507 22:01:42.776 | INFO | } | 20260507 22:01:42.776 | INFO | Perf report for the last 5.13961 seconds: | 20260507 22:01:42.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:42.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.776 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:01:42.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.776 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:01:42.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:42.777 | INFO | Uploaded 36GiB to GPU in 2.48067s (14.5122GiB/s) | 20260507 22:01:42.777 | INFO | Downloaded 34GiB to GPU in 2.4048s (14.1384GiB/s) | 20260507 22:01:47.913 | INFO | Analyzing chunk frames = [2926592,2927104) | 20260507 22:01:47.913 | INFO | MemInfo: { | 20260507 22:01:47.913 | INFO | "gpu" : " 17 GB", | 20260507 22:01:47.913 | INFO | "huge" : "", | 20260507 22:01:47.913 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:47.913 | INFO | "malloc" : " 14 GB", | 20260507 22:01:47.913 | INFO | "pinned" : " 29 GB" | 20260507 22:01:47.913 | INFO | } | 20260507 22:01:47.913 | INFO | Perf report for the last 5.13729 seconds: | 20260507 22:01:47.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:47.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.913 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:01:47.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.914 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:01:47.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:47.914 | INFO | Uploaded 36GiB to GPU in 2.34912s (15.3249GiB/s) | 20260507 22:01:47.914 | INFO | Downloaded 34GiB to GPU in 2.50669s (13.5637GiB/s) | 20260507 22:01:53.055 | INFO | Analyzing chunk frames = [2927104,2927616) | 20260507 22:01:53.055 | INFO | MemInfo: { | 20260507 22:01:53.055 | INFO | "gpu" : " 17 GB", | 20260507 22:01:53.055 | INFO | "huge" : "", | 20260507 22:01:53.055 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:53.055 | INFO | "malloc" : " 14 GB", | 20260507 22:01:53.055 | INFO | "pinned" : " 29 GB" | 20260507 22:01:53.055 | INFO | } | 20260507 22:01:53.055 | INFO | Perf report for the last 5.14184 seconds: | 20260507 22:01:53.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:53.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.055 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:01:53.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.055 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 22:01:53.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:53.056 | INFO | Uploaded 36GiB to GPU in 2.49967s (14.4019GiB/s) | 20260507 22:01:53.056 | INFO | Downloaded 34GiB to GPU in 2.43s (13.9918GiB/s) | 20260507 22:01:58.200 | INFO | Analyzing chunk frames = [2927616,2928128) | 20260507 22:01:58.200 | INFO | MemInfo: { | 20260507 22:01:58.200 | INFO | "gpu" : " 17 GB", | 20260507 22:01:58.200 | INFO | "huge" : "", | 20260507 22:01:58.200 | INFO | "huge_shared" : " 29 GB", | 20260507 22:01:58.200 | INFO | "malloc" : " 14 GB", | 20260507 22:01:58.200 | INFO | "pinned" : " 29 GB" | 20260507 22:01:58.200 | INFO | } | 20260507 22:01:58.200 | INFO | Perf report for the last 5.14467 seconds: | 20260507 22:01:58.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:01:58.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 22:01:58.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:01:58.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:01:58.200 | INFO | Uploaded 36GiB to GPU in 2.5187s (14.2931GiB/s) | 20260507 22:01:58.201 | INFO | Downloaded 34GiB to GPU in 2.46125s (13.8141GiB/s) | 20260507 22:02:03.332 | INFO | Analyzing chunk frames = [2928128,2928640) | 20260507 22:02:03.332 | INFO | MemInfo: { | 20260507 22:02:03.332 | INFO | "gpu" : " 17 GB", | 20260507 22:02:03.332 | INFO | "huge" : "", | 20260507 22:02:03.332 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:03.332 | INFO | "malloc" : " 14 GB", | 20260507 22:02:03.332 | INFO | "pinned" : " 29 GB" | 20260507 22:02:03.332 | INFO | } | 20260507 22:02:03.332 | INFO | Perf report for the last 5.13247 seconds: | 20260507 22:02:03.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:03.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.333 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:02:03.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.333 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:02:03.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:03.333 | INFO | Uploaded 36GiB to GPU in 2.49062s (14.4542GiB/s) | 20260507 22:02:03.333 | INFO | Downloaded 34GiB to GPU in 2.43748s (13.9488GiB/s) | 20260507 22:02:03.430 | INFO | Starting a round of Baz IO | 20260507 22:02:06.908 | INFO | Populating RTMetrics report | 20260507 22:02:06.908 | INFO | RTMetrics report populated | 20260507 22:02:06.908 | INFO | Opening tmp RTMetrics json file | 20260507 22:02:06.928 | INFO | Serializing RTMetrics to json | 20260507 22:02:06.931 | INFO | Serialization done, swapping tmp file now... | 20260507 22:02:06.937 | INFO | RTMetrics done | 20260507 22:02:07.002 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.571 seconds | 20260507 22:02:07.002 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.36% / 0.01% / 56.64% | 20260507 22:02:07.002 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.52342% / 7.94729e-06% / 5.5631e-05% | 20260507 22:02:07.002 | INFO | Baz IO overall: | 20260507 22:02:07.002 | INFO | BytesWritten : 1334181584800 | 20260507 22:02:07.002 | INFO | NumEvents : 646509616502 | 20260507 22:02:08.470 | INFO | Analyzing chunk frames = [2928640,2929152) | 20260507 22:02:08.470 | INFO | MemInfo: { | 20260507 22:02:08.470 | INFO | "gpu" : " 17 GB", | 20260507 22:02:08.470 | INFO | "huge" : "", | 20260507 22:02:08.470 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:08.470 | INFO | "malloc" : " 14 GB", | 20260507 22:02:08.470 | INFO | "pinned" : " 29 GB" | 20260507 22:02:08.470 | INFO | } | 20260507 22:02:08.470 | INFO | Perf report for the last 5.13761 seconds: | 20260507 22:02:08.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:08.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:08.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:08.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:08.470 | INFO | ANALYSIS : 68.7%, 3.0, 107.1ms, 0 | 20260507 22:02:08.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:08.470 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 22:02:08.470 | INFO | PRE_HQ : 65.4%, 1.7, 57.8ms, 0 | 20260507 22:02:08.470 | INFO | BAZWRITER : 69.1%, 3.9, 136.9ms, 0 | 20260507 22:02:08.470 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 22:02:08.471 | INFO | Uploaded 36GiB to GPU in 2.54829s (14.1271GiB/s) | 20260507 22:02:08.471 | INFO | Downloaded 38GiB to GPU in 2.68356s (14.1603GiB/s) | 20260507 22:02:13.612 | INFO | Analyzing chunk frames = [2929152,2929664) | 20260507 22:02:13.612 | INFO | MemInfo: { | 20260507 22:02:13.612 | INFO | "gpu" : " 17 GB", | 20260507 22:02:13.612 | INFO | "huge" : "", | 20260507 22:02:13.612 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:13.612 | INFO | "malloc" : " 14 GB", | 20260507 22:02:13.612 | INFO | "pinned" : " 29 GB" | 20260507 22:02:13.612 | INFO | } | 20260507 22:02:13.612 | INFO | Perf report for the last 5.14184 seconds: | 20260507 22:02:13.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:13.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:02:13.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:02:13.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:13.612 | INFO | Uploaded 36GiB to GPU in 2.54952s (14.1203GiB/s) | 20260507 22:02:13.612 | INFO | Downloaded 34GiB to GPU in 2.48363s (13.6896GiB/s) | 20260507 22:02:18.750 | INFO | Analyzing chunk frames = [2929664,2930176) | 20260507 22:02:18.750 | INFO | MemInfo: { | 20260507 22:02:18.750 | INFO | "gpu" : " 17 GB", | 20260507 22:02:18.750 | INFO | "huge" : "", | 20260507 22:02:18.750 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:18.750 | INFO | "malloc" : " 14 GB", | 20260507 22:02:18.750 | INFO | "pinned" : " 29 GB" | 20260507 22:02:18.750 | INFO | } | 20260507 22:02:18.751 | INFO | Perf report for the last 5.1386 seconds: | 20260507 22:02:18.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:18.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:02:18.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:02:18.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:18.751 | INFO | Uploaded 36GiB to GPU in 2.51675s (14.3042GiB/s) | 20260507 22:02:18.751 | INFO | Downloaded 34GiB to GPU in 2.46856s (13.7732GiB/s) | 20260507 22:02:23.891 | INFO | Analyzing chunk frames = [2930176,2930688) | 20260507 22:02:23.891 | INFO | MemInfo: { | 20260507 22:02:23.891 | INFO | "gpu" : " 17 GB", | 20260507 22:02:23.891 | INFO | "huge" : "", | 20260507 22:02:23.891 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:23.891 | INFO | "malloc" : " 14 GB", | 20260507 22:02:23.891 | INFO | "pinned" : " 29 GB" | 20260507 22:02:23.891 | INFO | } | 20260507 22:02:23.892 | INFO | Perf report for the last 5.14097 seconds: | 20260507 22:02:23.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:23.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:02:23.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:02:23.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:23.892 | INFO | Uploaded 36GiB to GPU in 2.46909s (14.5803GiB/s) | 20260507 22:02:23.892 | INFO | Downloaded 34GiB to GPU in 2.4007s (14.1625GiB/s) | 20260507 22:02:29.033 | INFO | Analyzing chunk frames = [2930688,2931200) | 20260507 22:02:29.033 | INFO | MemInfo: { | 20260507 22:02:29.033 | INFO | "gpu" : " 17 GB", | 20260507 22:02:29.033 | INFO | "huge" : "", | 20260507 22:02:29.033 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:29.033 | INFO | "malloc" : " 14 GB", | 20260507 22:02:29.033 | INFO | "pinned" : " 29 GB" | 20260507 22:02:29.034 | INFO | } | 20260507 22:02:29.034 | INFO | Perf report for the last 5.14204 seconds: | 20260507 22:02:29.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:29.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260507 22:02:29.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:02:29.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:29.034 | INFO | Uploaded 36GiB to GPU in 2.33015s (15.4497GiB/s) | 20260507 22:02:29.034 | INFO | Downloaded 34GiB to GPU in 2.51266s (13.5315GiB/s) | 20260507 22:02:34.171 | INFO | Analyzing chunk frames = [2931200,2931712) | 20260507 22:02:34.171 | INFO | MemInfo: { | 20260507 22:02:34.171 | INFO | "gpu" : " 17 GB", | 20260507 22:02:34.171 | INFO | "huge" : "", | 20260507 22:02:34.171 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:34.172 | INFO | "malloc" : " 14 GB", | 20260507 22:02:34.172 | INFO | "pinned" : " 29 GB" | 20260507 22:02:34.172 | INFO | } | 20260507 22:02:34.172 | INFO | Perf report for the last 5.13814 seconds: | 20260507 22:02:34.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:34.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 22:02:34.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 22:02:34.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:34.172 | INFO | Uploaded 36GiB to GPU in 2.4516s (14.6843GiB/s) | 20260507 22:02:34.172 | INFO | Downloaded 34GiB to GPU in 2.43532s (13.9612GiB/s) | 20260507 22:02:39.310 | INFO | Analyzing chunk frames = [2931712,2932224) | 20260507 22:02:39.310 | INFO | MemInfo: { | 20260507 22:02:39.310 | INFO | "gpu" : " 17 GB", | 20260507 22:02:39.310 | INFO | "huge" : "", | 20260507 22:02:39.310 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:39.310 | INFO | "malloc" : " 14 GB", | 20260507 22:02:39.310 | INFO | "pinned" : " 29 GB" | 20260507 22:02:39.310 | INFO | } | 20260507 22:02:39.311 | INFO | Perf report for the last 5.13877 seconds: | 20260507 22:02:39.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:39.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:02:39.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | PRE_HQ : 36.5%, 1.0, 18.9ms, 0 | 20260507 22:02:39.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:39.311 | INFO | Uploaded 36GiB to GPU in 2.50933s (14.3465GiB/s) | 20260507 22:02:39.311 | INFO | Downloaded 34GiB to GPU in 2.47057s (13.762GiB/s) | 20260507 22:02:44.445 | INFO | Analyzing chunk frames = [2932224,2932736) | 20260507 22:02:44.445 | INFO | MemInfo: { | 20260507 22:02:44.445 | INFO | "gpu" : " 17 GB", | 20260507 22:02:44.445 | INFO | "huge" : "", | 20260507 22:02:44.445 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:44.445 | INFO | "malloc" : " 14 GB", | 20260507 22:02:44.445 | INFO | "pinned" : " 29 GB" | 20260507 22:02:44.445 | INFO | } | 20260507 22:02:44.445 | INFO | Perf report for the last 5.13465 seconds: | 20260507 22:02:44.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:44.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.445 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:02:44.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.445 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 22:02:44.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:44.446 | INFO | Uploaded 36GiB to GPU in 2.45216s (14.6809GiB/s) | 20260507 22:02:44.446 | INFO | Downloaded 34GiB to GPU in 2.45024s (13.8762GiB/s) | 20260507 22:02:44.446 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:02:44.446 | INFO | Total mem usage: 16.7 GiB | 20260507 22:02:44.446 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:02:44.446 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14%) | 20260507 22:02:44.446 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:02:44.446 | INFO | High Water Breakdown: | 20260507 22:02:44.446 | INFO | 59.7% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:02:44.446 | INFO | 40.2% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:02:44.446 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:02:44.446 | INFO | ------------------------------------------ | 20260507 22:02:44.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:02:44.446 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:02:44.446 | INFO | Total mem usage: 26.5 GiB | 20260507 22:02:44.446 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:02:44.446 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 22:02:44.446 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 22:02:44.446 | INFO | High Water Breakdown: | 20260507 22:02:44.446 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:02:44.446 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:02:44.446 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:02:44.446 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:02:44.446 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:02:44.446 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:02:44.446 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:02:44.446 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:02:44.446 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:02:44.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:02:44.446 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:02:44.446 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:02:44.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:02:44.446 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:02:44.446 | INFO | ------------------------------------------ | 20260507 22:02:44.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:02:44.446 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:02:44.446 | INFO | Total mem usage: 27.8 GiB | 20260507 22:02:44.446 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:02:44.446 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:02:44.446 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:02:44.446 | INFO | High Water Breakdown: | 20260507 22:02:44.446 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:02:44.446 | INFO | ------------------------------------------ | 20260507 22:02:44.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:02:44.446 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:02:44.446 | INFO | Total mem usage: 15.7 GiB | 20260507 22:02:44.446 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:02:44.446 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:02:44.446 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:02:44.446 | INFO | High Water Breakdown: | 20260507 22:02:44.446 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:02:44.446 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:02:44.446 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:02:44.446 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:02:44.446 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:02:44.446 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:02:44.446 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:02:44.446 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:02:44.446 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:02:44.446 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:02:44.446 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:02:44.446 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:02:44.446 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:02:44.446 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:02:44.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:02:44.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:02:44.446 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:02:44.446 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:02:44.446 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:02:44.446 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:02:44.446 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:02:44.446 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:02:44.446 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:02:44.446 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:02:44.446 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:02:44.446 | INFO | ------------------------------------------ | 20260507 22:02:44.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:02:44.447 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:02:44.447 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:02:44.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:02:44.447 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.39e+01 / 1.61e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.16e-02 / 5.05e-03 | 20260507 22:02:44.447 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.49e+00, 6.00e-01 / 2.02e-01 | 20260507 22:02:44.447 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.52e+00, 2.08e-01 / 9.11e-02 | 20260507 22:02:44.447 | INFO | PerfReport: 15.80%, DME -- 4.17e+01 / 4.52e+01, 3.68e+00 / 1.15e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.81e+00 / 1.94e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.30e-01 / 2.50e-01 | 20260507 22:02:44.447 | INFO | PerfReport: 3.50%, Metrics -- 1.16e+00 / 1.38e+00, 2.91e-01 / 1.71e-01 | 20260507 22:02:44.447 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.59e-01 / 3.88e-01, 5.86e-02 / 3.17e-02 | 20260507 22:02:44.447 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:02:44.447 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:02:44.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:02:44.447 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 24.30%, Upload -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 30.95%, Compute -- 3.30e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 1.90%, LockUpload -- 2.03e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 2.57%, LockDownload -- 2.74e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 15.97%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:02:44.447 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.54e-02 / 0.00e+00 | 20260507 22:02:44.560 | INFO | Starting a round of Baz IO | 20260507 22:02:48.040 | INFO | Populating RTMetrics report | 20260507 22:02:48.040 | INFO | RTMetrics report populated | 20260507 22:02:48.040 | INFO | Opening tmp RTMetrics json file | 20260507 22:02:48.057 | INFO | Serializing RTMetrics to json | 20260507 22:02:48.065 | INFO | Serialization done, swapping tmp file now... | 20260507 22:02:48.065 | INFO | RTMetrics done | 20260507 22:02:48.134 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.574 seconds | 20260507 22:02:48.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.35% / 0.01% / 56.64% | 20260507 22:02:48.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51807% / 1.19209e-05% / 4.76837e-05% | 20260507 22:02:48.134 | INFO | Baz IO overall: | 20260507 22:02:48.134 | INFO | BytesWritten : 1336057159151 | 20260507 22:02:48.134 | INFO | NumEvents : 647415894206 | 20260507 22:02:49.582 | INFO | Analyzing chunk frames = [2932736,2933248) | 20260507 22:02:49.582 | INFO | MemInfo: { | 20260507 22:02:49.582 | INFO | "gpu" : " 17 GB", | 20260507 22:02:49.582 | INFO | "huge" : "", | 20260507 22:02:49.582 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:49.582 | INFO | "malloc" : " 14 GB", | 20260507 22:02:49.582 | INFO | "pinned" : " 29 GB" | 20260507 22:02:49.582 | INFO | } | 20260507 22:02:49.582 | INFO | Perf report for the last 5.13697 seconds: | 20260507 22:02:49.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:49.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:49.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:49.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:49.583 | INFO | ANALYSIS : 69.3%, 2.9, 104.7ms, 0 | 20260507 22:02:49.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:49.583 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 22:02:49.583 | INFO | PRE_HQ : 68.1%, 1.8, 64.0ms, 0 | 20260507 22:02:49.583 | INFO | BAZWRITER : 69.2%, 4.1, 145.4ms, 0 | 20260507 22:02:49.583 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 22:02:49.583 | INFO | Uploaded 36GiB to GPU in 2.54434s (14.149GiB/s) | 20260507 22:02:49.583 | INFO | Downloaded 38GiB to GPU in 2.70118s (14.0679GiB/s) | 20260507 22:02:54.722 | INFO | Analyzing chunk frames = [2933248,2933760) | 20260507 22:02:54.722 | INFO | MemInfo: { | 20260507 22:02:54.722 | INFO | "gpu" : " 17 GB", | 20260507 22:02:54.722 | INFO | "huge" : "", | 20260507 22:02:54.722 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:54.722 | INFO | "malloc" : " 14 GB", | 20260507 22:02:54.722 | INFO | "pinned" : " 29 GB" | 20260507 22:02:54.722 | INFO | } | 20260507 22:02:54.722 | INFO | Perf report for the last 5.13995 seconds: | 20260507 22:02:54.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:54.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:02:54.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:02:54.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:54.722 | INFO | Uploaded 36GiB to GPU in 2.52867s (14.2367GiB/s) | 20260507 22:02:54.723 | INFO | Downloaded 34GiB to GPU in 2.46256s (13.8068GiB/s) | 20260507 22:02:59.867 | INFO | Analyzing chunk frames = [2933760,2934272) | 20260507 22:02:59.867 | INFO | MemInfo: { | 20260507 22:02:59.867 | INFO | "gpu" : " 17 GB", | 20260507 22:02:59.867 | INFO | "huge" : "", | 20260507 22:02:59.867 | INFO | "huge_shared" : " 29 GB", | 20260507 22:02:59.867 | INFO | "malloc" : " 14 GB", | 20260507 22:02:59.867 | INFO | "pinned" : " 29 GB" | 20260507 22:02:59.867 | INFO | } | 20260507 22:02:59.867 | INFO | Perf report for the last 5.14458 seconds: | 20260507 22:02:59.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:02:59.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:02:59.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:02:59.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:02:59.867 | INFO | Uploaded 36GiB to GPU in 2.53602s (14.1955GiB/s) | 20260507 22:02:59.867 | INFO | Downloaded 34GiB to GPU in 2.45s (13.8776GiB/s) | 20260507 22:03:05.010 | INFO | Analyzing chunk frames = [2934272,2934784) | 20260507 22:03:05.010 | INFO | MemInfo: { | 20260507 22:03:05.010 | INFO | "gpu" : " 17 GB", | 20260507 22:03:05.010 | INFO | "huge" : "", | 20260507 22:03:05.010 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:05.010 | INFO | "malloc" : " 14 GB", | 20260507 22:03:05.010 | INFO | "pinned" : " 29 GB" | 20260507 22:03:05.010 | INFO | } | 20260507 22:03:05.010 | INFO | Perf report for the last 5.1437 seconds: | 20260507 22:03:05.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:05.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.010 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:03:05.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.011 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:03:05.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:05.011 | INFO | Uploaded 36GiB to GPU in 2.49028s (14.4562GiB/s) | 20260507 22:03:05.011 | INFO | Downloaded 34GiB to GPU in 2.40576s (14.1328GiB/s) | 20260507 22:03:10.147 | INFO | Analyzing chunk frames = [2934784,2935296) | 20260507 22:03:10.147 | INFO | MemInfo: { | 20260507 22:03:10.147 | INFO | "gpu" : " 17 GB", | 20260507 22:03:10.147 | INFO | "huge" : "", | 20260507 22:03:10.147 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:10.147 | INFO | "malloc" : " 14 GB", | 20260507 22:03:10.148 | INFO | "pinned" : " 29 GB" | 20260507 22:03:10.148 | INFO | } | 20260507 22:03:10.148 | INFO | Perf report for the last 5.13719 seconds: | 20260507 22:03:10.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:10.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 22:03:10.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:03:10.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:10.148 | INFO | Uploaded 36GiB to GPU in 2.3966s (15.0213GiB/s) | 20260507 22:03:10.148 | INFO | Downloaded 34GiB to GPU in 2.49286s (13.6389GiB/s) | 20260507 22:03:15.281 | INFO | Analyzing chunk frames = [2935296,2935808) | 20260507 22:03:15.281 | INFO | MemInfo: { | 20260507 22:03:15.281 | INFO | "gpu" : " 17 GB", | 20260507 22:03:15.281 | INFO | "huge" : "", | 20260507 22:03:15.281 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:15.281 | INFO | "malloc" : " 14 GB", | 20260507 22:03:15.281 | INFO | "pinned" : " 29 GB" | 20260507 22:03:15.281 | INFO | } | 20260507 22:03:15.281 | INFO | Perf report for the last 5.13322 seconds: | 20260507 22:03:15.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:15.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260507 22:03:15.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:03:15.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:15.281 | INFO | Uploaded 36GiB to GPU in 2.50351s (14.3798GiB/s) | 20260507 22:03:15.281 | INFO | Downloaded 34GiB to GPU in 2.42634s (14.0129GiB/s) | 20260507 22:03:20.423 | INFO | Analyzing chunk frames = [2935808,2936320) | 20260507 22:03:20.423 | INFO | MemInfo: { | 20260507 22:03:20.423 | INFO | "gpu" : " 17 GB", | 20260507 22:03:20.423 | INFO | "huge" : "", | 20260507 22:03:20.423 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:20.423 | INFO | "malloc" : " 14 GB", | 20260507 22:03:20.423 | INFO | "pinned" : " 29 GB" | 20260507 22:03:20.423 | INFO | } | 20260507 22:03:20.423 | INFO | Perf report for the last 5.14231 seconds: | 20260507 22:03:20.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:20.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.423 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:03:20.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.423 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 22:03:20.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:20.424 | INFO | Uploaded 36GiB to GPU in 2.51435s (14.3178GiB/s) | 20260507 22:03:20.424 | INFO | Downloaded 34GiB to GPU in 2.44288s (13.918GiB/s) | 20260507 22:03:25.560 | INFO | Analyzing chunk frames = [2936320,2936832) | 20260507 22:03:25.560 | INFO | MemInfo: { | 20260507 22:03:25.560 | INFO | "gpu" : " 17 GB", | 20260507 22:03:25.560 | INFO | "huge" : "", | 20260507 22:03:25.560 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:25.560 | INFO | "malloc" : " 14 GB", | 20260507 22:03:25.560 | INFO | "pinned" : " 29 GB" | 20260507 22:03:25.560 | INFO | } | 20260507 22:03:25.561 | INFO | Perf report for the last 5.13734 seconds: | 20260507 22:03:25.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:25.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:03:25.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260507 22:03:25.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:25.561 | INFO | Uploaded 36GiB to GPU in 2.48905s (14.4634GiB/s) | 20260507 22:03:25.561 | INFO | Downloaded 34GiB to GPU in 2.44005s (13.9342GiB/s) | 20260507 22:03:25.658 | INFO | Starting a round of Baz IO | 20260507 22:03:29.140 | INFO | Populating RTMetrics report | 20260507 22:03:29.140 | INFO | RTMetrics report populated | 20260507 22:03:29.140 | INFO | Opening tmp RTMetrics json file | 20260507 22:03:29.458 | INFO | Serializing RTMetrics to json | 20260507 22:03:29.579 | INFO | Serialization done, swapping tmp file now... | 20260507 22:03:29.594 | INFO | RTMetrics done | 20260507 22:03:29.610 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.951 seconds | 20260507 22:03:29.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.34% / 0.01% / 56.65% | 20260507 22:03:29.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5126% / 0% / 5.96046e-05% | 20260507 22:03:29.610 | INFO | Baz IO overall: | 20260507 22:03:29.610 | INFO | BytesWritten : 1337930964836 | 20260507 22:03:29.610 | INFO | NumEvents : 648320784432 | 20260507 22:03:30.702 | INFO | Analyzing chunk frames = [2936832,2937344) | 20260507 22:03:30.702 | INFO | MemInfo: { | 20260507 22:03:30.702 | INFO | "gpu" : " 17 GB", | 20260507 22:03:30.702 | INFO | "huge" : "", | 20260507 22:03:30.702 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:30.702 | INFO | "malloc" : " 14 GB", | 20260507 22:03:30.702 | INFO | "pinned" : " 29 GB" | 20260507 22:03:30.702 | INFO | } | 20260507 22:03:30.702 | INFO | Perf report for the last 5.14137 seconds: | 20260507 22:03:30.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:30.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:30.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:30.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:30.702 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 22:03:30.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:30.702 | INFO | RT_METRICS : 49.6%, 1.0, 26.0ms, 0 | 20260507 22:03:30.702 | INFO | PRE_HQ : 65.3%, 1.7, 57.4ms, 0 | 20260507 22:03:30.702 | INFO | BAZWRITER : 76.4%, 4.0, 158.6ms, 0 | 20260507 22:03:30.702 | INFO | BAZ_SAMPLING : 8.1%, 1.0, 4.2ms, 0 | 20260507 22:03:30.702 | INFO | Uploaded 36GiB to GPU in 2.51939s (14.2891GiB/s) | 20260507 22:03:30.702 | INFO | Downloaded 38GiB to GPU in 2.68048s (14.1766GiB/s) | 20260507 22:03:35.841 | INFO | Analyzing chunk frames = [2937344,2937856) | 20260507 22:03:35.841 | INFO | MemInfo: { | 20260507 22:03:35.841 | INFO | "gpu" : " 17 GB", | 20260507 22:03:35.841 | INFO | "huge" : "", | 20260507 22:03:35.841 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:35.841 | INFO | "malloc" : " 14 GB", | 20260507 22:03:35.841 | INFO | "pinned" : " 29 GB" | 20260507 22:03:35.841 | INFO | } | 20260507 22:03:35.841 | INFO | Perf report for the last 5.13924 seconds: | 20260507 22:03:35.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:35.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.841 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 22:03:35.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.841 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:03:35.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:35.842 | INFO | Uploaded 36GiB to GPU in 2.52835s (14.2386GiB/s) | 20260507 22:03:35.842 | INFO | Downloaded 34GiB to GPU in 2.47824s (13.7194GiB/s) | 20260507 22:03:40.981 | INFO | Analyzing chunk frames = [2937856,2938368) | 20260507 22:03:40.981 | INFO | MemInfo: { | 20260507 22:03:40.981 | INFO | "gpu" : " 17 GB", | 20260507 22:03:40.981 | INFO | "huge" : "", | 20260507 22:03:40.981 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:40.981 | INFO | "malloc" : " 14 GB", | 20260507 22:03:40.981 | INFO | "pinned" : " 29 GB" | 20260507 22:03:40.981 | INFO | } | 20260507 22:03:40.981 | INFO | Perf report for the last 5.14007 seconds: | 20260507 22:03:40.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:40.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.981 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:03:40.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.981 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:03:40.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:40.982 | INFO | Uploaded 36GiB to GPU in 2.53125s (14.2222GiB/s) | 20260507 22:03:40.982 | INFO | Downloaded 34GiB to GPU in 2.45887s (13.8275GiB/s) | 20260507 22:03:46.119 | INFO | Analyzing chunk frames = [2938368,2938880) | 20260507 22:03:46.119 | INFO | MemInfo: { | 20260507 22:03:46.119 | INFO | "gpu" : " 17 GB", | 20260507 22:03:46.119 | INFO | "huge" : "", | 20260507 22:03:46.119 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:46.119 | INFO | "malloc" : " 14 GB", | 20260507 22:03:46.119 | INFO | "pinned" : " 29 GB" | 20260507 22:03:46.119 | INFO | } | 20260507 22:03:46.119 | INFO | Perf report for the last 5.13819 seconds: | 20260507 22:03:46.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:46.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.120 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:03:46.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.120 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:03:46.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:46.120 | INFO | Uploaded 36GiB to GPU in 2.47973s (14.5177GiB/s) | 20260507 22:03:46.120 | INFO | Downloaded 34GiB to GPU in 2.39715s (14.1835GiB/s) | 20260507 22:03:51.263 | INFO | Analyzing chunk frames = [2938880,2939392) | 20260507 22:03:51.263 | INFO | MemInfo: { | 20260507 22:03:51.263 | INFO | "gpu" : " 17 GB", | 20260507 22:03:51.263 | INFO | "huge" : "", | 20260507 22:03:51.263 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:51.263 | INFO | "malloc" : " 14 GB", | 20260507 22:03:51.263 | INFO | "pinned" : " 29 GB" | 20260507 22:03:51.263 | INFO | } | 20260507 22:03:51.263 | INFO | Perf report for the last 5.14358 seconds: | 20260507 22:03:51.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:51.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.263 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:03:51.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.263 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:03:51.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:51.264 | INFO | Uploaded 36GiB to GPU in 2.36686s (15.21GiB/s) | 20260507 22:03:51.264 | INFO | Downloaded 34GiB to GPU in 2.49784s (13.6118GiB/s) | 20260507 22:03:56.400 | INFO | Analyzing chunk frames = [2939392,2939904) | 20260507 22:03:56.400 | INFO | MemInfo: { | 20260507 22:03:56.400 | INFO | "gpu" : " 17 GB", | 20260507 22:03:56.400 | INFO | "huge" : "", | 20260507 22:03:56.400 | INFO | "huge_shared" : " 29 GB", | 20260507 22:03:56.400 | INFO | "malloc" : " 14 GB", | 20260507 22:03:56.400 | INFO | "pinned" : " 29 GB" | 20260507 22:03:56.401 | INFO | } | 20260507 22:03:56.401 | INFO | Perf report for the last 5.13753 seconds: | 20260507 22:03:56.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:03:56.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.401 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:03:56.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.401 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:03:56.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:03:56.402 | INFO | Uploaded 36GiB to GPU in 2.53702s (14.1899GiB/s) | 20260507 22:03:56.402 | INFO | Downloaded 34GiB to GPU in 2.4136s (14.0868GiB/s) | 20260507 22:04:01.537 | INFO | Analyzing chunk frames = [2939904,2940416) | 20260507 22:04:01.537 | INFO | MemInfo: { | 20260507 22:04:01.537 | INFO | "gpu" : " 17 GB", | 20260507 22:04:01.537 | INFO | "huge" : "", | 20260507 22:04:01.537 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:01.537 | INFO | "malloc" : " 14 GB", | 20260507 22:04:01.537 | INFO | "pinned" : " 29 GB" | 20260507 22:04:01.537 | INFO | } | 20260507 22:04:01.538 | INFO | Perf report for the last 5.13687 seconds: | 20260507 22:04:01.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:01.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:04:01.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260507 22:04:01.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:01.538 | INFO | Uploaded 36GiB to GPU in 2.56023s (14.0612GiB/s) | 20260507 22:04:01.538 | INFO | Downloaded 34GiB to GPU in 2.4597s (13.8228GiB/s) | 20260507 22:04:06.676 | INFO | Analyzing chunk frames = [2940416,2940928) | 20260507 22:04:06.676 | INFO | MemInfo: { | 20260507 22:04:06.676 | INFO | "gpu" : " 17 GB", | 20260507 22:04:06.676 | INFO | "huge" : "", | 20260507 22:04:06.676 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:06.676 | INFO | "malloc" : " 14 GB", | 20260507 22:04:06.676 | INFO | "pinned" : " 29 GB" | 20260507 22:04:06.676 | INFO | } | 20260507 22:04:06.676 | INFO | Perf report for the last 5.13885 seconds: | 20260507 22:04:06.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:06.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.676 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:04:06.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.677 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 22:04:06.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:06.677 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:04:06.677 | INFO | Total mem usage: 16.7 GiB | 20260507 22:04:06.677 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:04:06.677 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12%) | 20260507 22:04:06.677 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:04:06.677 | INFO | High Water Breakdown: | 20260507 22:04:06.677 | INFO | 59.9% : 8.93 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:04:06.677 | INFO | 39.9% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:04:06.677 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:04:06.677 | INFO | ------------------------------------------ | 20260507 22:04:06.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:04:06.677 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:04:06.677 | INFO | Total mem usage: 26.5 GiB | 20260507 22:04:06.677 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:04:06.677 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 22:04:06.677 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:04:06.677 | INFO | High Water Breakdown: | 20260507 22:04:06.677 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:04:06.677 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:04:06.677 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:04:06.677 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:04:06.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:04:06.677 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:04:06.677 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:04:06.677 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:04:06.677 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:04:06.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:04:06.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:04:06.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:04:06.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:04:06.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:04:06.677 | INFO | ------------------------------------------ | 20260507 22:04:06.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:04:06.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:04:06.677 | INFO | Total mem usage: 27.8 GiB | 20260507 22:04:06.677 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:04:06.677 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:04:06.677 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:04:06.677 | INFO | High Water Breakdown: | 20260507 22:04:06.677 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:04:06.677 | INFO | ------------------------------------------ | 20260507 22:04:06.677 | INFO | Uploaded 36GiB to GPU in 2.49777s (14.4129GiB/s) | 20260507 22:04:06.678 | INFO | Downloaded 34GiB to GPU in 2.43612s (13.9566GiB/s) | 20260507 22:04:06.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:04:06.678 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:04:06.678 | INFO | Total mem usage: 15.7 GiB | 20260507 22:04:06.678 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:04:06.678 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:04:06.678 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:04:06.678 | INFO | High Water Breakdown: | 20260507 22:04:06.678 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:04:06.678 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:04:06.678 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:04:06.678 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:04:06.678 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:04:06.678 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:04:06.678 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:04:06.678 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:04:06.678 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:04:06.678 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:04:06.678 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:04:06.678 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:04:06.678 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:04:06.678 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:04:06.678 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:04:06.678 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:04:06.678 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:04:06.678 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:04:06.678 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:04:06.678 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:04:06.678 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:04:06.678 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:04:06.678 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:04:06.678 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:04:06.678 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:04:06.678 | INFO | ------------------------------------------ | 20260507 22:04:06.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:04:06.678 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:04:06.678 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:04:06.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:04:06.678 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.41e+01 / 1.64e+00 | 20260507 22:04:06.678 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.06e-01, 1.23e-02 / 1.85e-02 | 20260507 22:04:06.678 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.48e+00, 5.96e-01 / 2.40e-01 | 20260507 22:04:06.678 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.48e+00, 2.08e-01 / 1.24e-01 | 20260507 22:04:06.678 | INFO | PerfReport: 15.70%, DME -- 4.15e+01 / 4.47e+01, 3.70e+00 / 1.33e+00 | 20260507 22:04:06.678 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.85e+00 / 1.97e+00 | 20260507 22:04:06.678 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.43e+00 / 2.71e+00, 4.25e-01 / 2.81e-01 | 20260507 22:04:06.678 | INFO | PerfReport: 3.47%, Metrics -- 1.15e+00 / 1.36e+00, 2.90e-01 / 1.64e-01 | 20260507 22:04:06.678 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.58e-01 / 3.87e-01, 5.88e-02 / 3.26e-02 | 20260507 22:04:06.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:04:06.678 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:04:06.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:04:06.678 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 22:04:06.678 | INFO | PerfReport: 24.43%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 22:04:06.679 | INFO | PerfReport: 24.20%, Download -- 2.58e+01 / 0.00e+00, 8.54e+00 / 0.00e+00 | 20260507 22:04:06.679 | INFO | PerfReport: 30.99%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:04:06.679 | INFO | PerfReport: 1.91%, LockUpload -- 2.04e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 22:04:06.679 | INFO | PerfReport: 2.47%, LockDownload -- 2.64e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260507 22:04:06.679 | INFO | PerfReport: 15.98%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:04:06.679 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.96e-02 / 0.00e+00 | 20260507 22:04:06.793 | INFO | Starting a round of Baz IO | 20260507 22:04:10.280 | INFO | Populating RTMetrics report | 20260507 22:04:10.280 | INFO | RTMetrics report populated | 20260507 22:04:10.280 | INFO | Opening tmp RTMetrics json file | 20260507 22:04:10.305 | INFO | Serializing RTMetrics to json | 20260507 22:04:10.307 | INFO | Serialization done, swapping tmp file now... | 20260507 22:04:10.308 | INFO | RTMetrics done | 20260507 22:04:10.391 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.597 seconds | 20260507 22:04:10.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.33% / 0.01% / 56.66% | 20260507 22:04:10.392 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51154% / 3.97364e-06% / 6.75519e-05% | 20260507 22:04:10.392 | INFO | Baz IO overall: | 20260507 22:04:10.392 | INFO | BytesWritten : 1339805215409 | 20260507 22:04:10.392 | INFO | NumEvents : 649225902624 | 20260507 22:04:11.816 | INFO | Analyzing chunk frames = [2940928,2941440) | 20260507 22:04:11.816 | INFO | MemInfo: { | 20260507 22:04:11.816 | INFO | "gpu" : " 17 GB", | 20260507 22:04:11.816 | INFO | "huge" : "", | 20260507 22:04:11.816 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:11.816 | INFO | "malloc" : " 14 GB", | 20260507 22:04:11.816 | INFO | "pinned" : " 29 GB" | 20260507 22:04:11.816 | INFO | } | 20260507 22:04:11.816 | INFO | Perf report for the last 5.13965 seconds: | 20260507 22:04:11.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:11.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:11.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:11.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:11.816 | INFO | ANALYSIS : 69.4%, 2.9, 104.8ms, 0 | 20260507 22:04:11.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:11.816 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260507 22:04:11.816 | INFO | PRE_HQ : 68.2%, 1.8, 63.1ms, 0 | 20260507 22:04:11.816 | INFO | BAZWRITER : 69.7%, 4.1, 145.1ms, 0 | 20260507 22:04:11.816 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 22:04:11.816 | INFO | Uploaded 36GiB to GPU in 2.61157s (13.7848GiB/s) | 20260507 22:04:11.816 | INFO | Downloaded 38GiB to GPU in 2.71292s (14.007GiB/s) | 20260507 22:04:16.961 | INFO | Analyzing chunk frames = [2941440,2941952) | 20260507 22:04:16.961 | INFO | MemInfo: { | 20260507 22:04:16.961 | INFO | "gpu" : " 17 GB", | 20260507 22:04:16.961 | INFO | "huge" : "", | 20260507 22:04:16.961 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:16.961 | INFO | "malloc" : " 14 GB", | 20260507 22:04:16.961 | INFO | "pinned" : " 29 GB" | 20260507 22:04:16.961 | INFO | } | 20260507 22:04:16.961 | INFO | Perf report for the last 5.14528 seconds: | 20260507 22:04:16.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:16.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.961 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 22:04:16.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.961 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:04:16.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:16.962 | INFO | Uploaded 36GiB to GPU in 2.51433s (14.318GiB/s) | 20260507 22:04:16.962 | INFO | Downloaded 34GiB to GPU in 2.47806s (13.7204GiB/s) | 20260507 22:04:22.097 | INFO | Analyzing chunk frames = [2941952,2942464) | 20260507 22:04:22.097 | INFO | MemInfo: { | 20260507 22:04:22.097 | INFO | "gpu" : " 17 GB", | 20260507 22:04:22.097 | INFO | "huge" : "", | 20260507 22:04:22.097 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:22.097 | INFO | "malloc" : " 14 GB", | 20260507 22:04:22.097 | INFO | "pinned" : " 29 GB" | 20260507 22:04:22.097 | INFO | } | 20260507 22:04:22.097 | INFO | Perf report for the last 5.13593 seconds: | 20260507 22:04:22.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:22.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.097 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 22:04:22.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.097 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:04:22.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:22.098 | INFO | Uploaded 36GiB to GPU in 2.46532s (14.6026GiB/s) | 20260507 22:04:22.098 | INFO | Downloaded 34GiB to GPU in 2.48752s (13.6682GiB/s) | 20260507 22:04:27.234 | INFO | Analyzing chunk frames = [2942464,2942976) | 20260507 22:04:27.234 | INFO | MemInfo: { | 20260507 22:04:27.234 | INFO | "gpu" : " 17 GB", | 20260507 22:04:27.234 | INFO | "huge" : "", | 20260507 22:04:27.234 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:27.234 | INFO | "malloc" : " 14 GB", | 20260507 22:04:27.234 | INFO | "pinned" : " 29 GB" | 20260507 22:04:27.234 | INFO | } | 20260507 22:04:27.234 | INFO | Perf report for the last 5.13686 seconds: | 20260507 22:04:27.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:27.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.234 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:04:27.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.234 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:04:27.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:27.235 | INFO | Uploaded 36GiB to GPU in 2.46859s (14.5833GiB/s) | 20260507 22:04:27.235 | INFO | Downloaded 34GiB to GPU in 2.3966s (14.1868GiB/s) | 20260507 22:04:32.374 | INFO | Analyzing chunk frames = [2942976,2943488) | 20260507 22:04:32.374 | INFO | MemInfo: { | 20260507 22:04:32.374 | INFO | "gpu" : " 17 GB", | 20260507 22:04:32.374 | INFO | "huge" : "", | 20260507 22:04:32.374 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:32.374 | INFO | "malloc" : " 14 GB", | 20260507 22:04:32.374 | INFO | "pinned" : " 29 GB" | 20260507 22:04:32.374 | INFO | } | 20260507 22:04:32.374 | INFO | Perf report for the last 5.13964 seconds: | 20260507 22:04:32.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:32.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 22:04:32.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 22:04:32.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:32.374 | INFO | Uploaded 36GiB to GPU in 2.36073s (15.2495GiB/s) | 20260507 22:04:32.374 | INFO | Downloaded 34GiB to GPU in 2.49171s (13.6453GiB/s) | 20260507 22:04:37.511 | INFO | Analyzing chunk frames = [2943488,2944000) | 20260507 22:04:37.511 | INFO | MemInfo: { | 20260507 22:04:37.511 | INFO | "gpu" : " 17 GB", | 20260507 22:04:37.511 | INFO | "huge" : "", | 20260507 22:04:37.511 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:37.511 | INFO | "malloc" : " 14 GB", | 20260507 22:04:37.511 | INFO | "pinned" : " 29 GB" | 20260507 22:04:37.511 | INFO | } | 20260507 22:04:37.511 | INFO | Perf report for the last 5.13743 seconds: | 20260507 22:04:37.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:37.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.511 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:04:37.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.511 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:04:37.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:37.512 | INFO | Uploaded 36GiB to GPU in 2.51143s (14.3345GiB/s) | 20260507 22:04:37.512 | INFO | Downloaded 34GiB to GPU in 2.4303s (13.99GiB/s) | 20260507 22:04:42.652 | INFO | Analyzing chunk frames = [2944000,2944512) | 20260507 22:04:42.652 | INFO | MemInfo: { | 20260507 22:04:42.652 | INFO | "gpu" : " 17 GB", | 20260507 22:04:42.652 | INFO | "huge" : "", | 20260507 22:04:42.652 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:42.652 | INFO | "malloc" : " 14 GB", | 20260507 22:04:42.652 | INFO | "pinned" : " 29 GB" | 20260507 22:04:42.652 | INFO | } | 20260507 22:04:42.652 | INFO | Perf report for the last 5.14054 seconds: | 20260507 22:04:42.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:42.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:04:42.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 22:04:42.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:42.652 | INFO | Uploaded 36GiB to GPU in 2.52443s (14.2606GiB/s) | 20260507 22:04:42.652 | INFO | Downloaded 34GiB to GPU in 2.46093s (13.8159GiB/s) | 20260507 22:04:47.787 | INFO | Analyzing chunk frames = [2944512,2945024) | 20260507 22:04:47.787 | INFO | MemInfo: { | 20260507 22:04:47.787 | INFO | "gpu" : " 17 GB", | 20260507 22:04:47.787 | INFO | "huge" : "", | 20260507 22:04:47.787 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:47.787 | INFO | "malloc" : " 14 GB", | 20260507 22:04:47.788 | INFO | "pinned" : " 29 GB" | 20260507 22:04:47.788 | INFO | } | 20260507 22:04:47.788 | INFO | Perf report for the last 5.13581 seconds: | 20260507 22:04:47.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:47.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:04:47.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:04:47.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:47.788 | INFO | Uploaded 36GiB to GPU in 2.50474s (14.3727GiB/s) | 20260507 22:04:47.788 | INFO | Downloaded 34GiB to GPU in 2.44277s (13.9186GiB/s) | 20260507 22:04:47.885 | INFO | Starting a round of Baz IO | 20260507 22:04:51.348 | INFO | Populating RTMetrics report | 20260507 22:04:51.349 | INFO | RTMetrics report populated | 20260507 22:04:51.349 | INFO | Opening tmp RTMetrics json file | 20260507 22:04:51.376 | INFO | Serializing RTMetrics to json | 20260507 22:04:51.378 | INFO | Serialization done, swapping tmp file now... | 20260507 22:04:51.379 | INFO | RTMetrics done | 20260507 22:04:51.462 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.577 seconds | 20260507 22:04:51.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.33% / 0.01% / 56.67% | 20260507 22:04:51.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51074% / 7.94729e-06% / 4.37101e-05% | 20260507 22:04:51.462 | INFO | Baz IO overall: | 20260507 22:04:51.462 | INFO | BytesWritten : 1341679754277 | 20260507 22:04:51.463 | INFO | NumEvents : 650131403777 | 20260507 22:04:52.931 | INFO | Analyzing chunk frames = [2945024,2945536) | 20260507 22:04:52.931 | INFO | MemInfo: { | 20260507 22:04:52.931 | INFO | "gpu" : " 17 GB", | 20260507 22:04:52.931 | INFO | "huge" : "", | 20260507 22:04:52.931 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:52.931 | INFO | "malloc" : " 14 GB", | 20260507 22:04:52.931 | INFO | "pinned" : " 29 GB" | 20260507 22:04:52.931 | INFO | } | 20260507 22:04:52.931 | INFO | Perf report for the last 5.1436 seconds: | 20260507 22:04:52.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:52.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:52.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:52.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:52.931 | INFO | ANALYSIS : 68.8%, 2.9, 106.5ms, 0 | 20260507 22:04:52.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:52.931 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 22:04:52.931 | INFO | PRE_HQ : 65.7%, 1.7, 57.5ms, 0 | 20260507 22:04:52.931 | INFO | BAZWRITER : 69.1%, 4.7, 167.4ms, 0 | 20260507 22:04:52.931 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:04:52.932 | INFO | Uploaded 36GiB to GPU in 2.56864s (14.0152GiB/s) | 20260507 22:04:52.932 | INFO | Downloaded 38GiB to GPU in 2.70339s (14.0564GiB/s) | 20260507 22:04:58.075 | INFO | Analyzing chunk frames = [2945536,2946048) | 20260507 22:04:58.075 | INFO | MemInfo: { | 20260507 22:04:58.075 | INFO | "gpu" : " 17 GB", | 20260507 22:04:58.075 | INFO | "huge" : "", | 20260507 22:04:58.075 | INFO | "huge_shared" : " 29 GB", | 20260507 22:04:58.075 | INFO | "malloc" : " 14 GB", | 20260507 22:04:58.075 | INFO | "pinned" : " 29 GB" | 20260507 22:04:58.075 | INFO | } | 20260507 22:04:58.075 | INFO | Perf report for the last 5.14386 seconds: | 20260507 22:04:58.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:04:58.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:04:58.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:04:58.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:04:58.075 | INFO | Uploaded 36GiB to GPU in 2.4949s (14.4295GiB/s) | 20260507 22:04:58.076 | INFO | Downloaded 34GiB to GPU in 2.47078s (13.7608GiB/s) | 20260507 22:05:03.215 | INFO | Analyzing chunk frames = [2946048,2946560) | 20260507 22:05:03.215 | INFO | MemInfo: { | 20260507 22:05:03.215 | INFO | "gpu" : " 17 GB", | 20260507 22:05:03.215 | INFO | "huge" : "", | 20260507 22:05:03.215 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:03.215 | INFO | "malloc" : " 14 GB", | 20260507 22:05:03.215 | INFO | "pinned" : " 29 GB" | 20260507 22:05:03.215 | INFO | } | 20260507 22:05:03.215 | INFO | Perf report for the last 5.13991 seconds: | 20260507 22:05:03.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:03.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.215 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:05:03.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.215 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:05:03.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:03.216 | INFO | Uploaded 36GiB to GPU in 2.50424s (14.3756GiB/s) | 20260507 22:05:03.216 | INFO | Downloaded 34GiB to GPU in 2.48808s (13.6651GiB/s) | 20260507 22:05:08.348 | INFO | Analyzing chunk frames = [2946560,2947072) | 20260507 22:05:08.348 | INFO | MemInfo: { | 20260507 22:05:08.348 | INFO | "gpu" : " 17 GB", | 20260507 22:05:08.348 | INFO | "huge" : "", | 20260507 22:05:08.348 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:08.348 | INFO | "malloc" : " 14 GB", | 20260507 22:05:08.348 | INFO | "pinned" : " 29 GB" | 20260507 22:05:08.348 | INFO | } | 20260507 22:05:08.348 | INFO | Perf report for the last 5.13291 seconds: | 20260507 22:05:08.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:08.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 22:05:08.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:05:08.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:08.348 | INFO | Uploaded 36GiB to GPU in 2.48016s (14.5152GiB/s) | 20260507 22:05:08.348 | INFO | Downloaded 34GiB to GPU in 2.47575s (13.7332GiB/s) | 20260507 22:05:13.489 | INFO | Analyzing chunk frames = [2947072,2947584) | 20260507 22:05:13.489 | INFO | MemInfo: { | 20260507 22:05:13.489 | INFO | "gpu" : " 17 GB", | 20260507 22:05:13.489 | INFO | "huge" : "", | 20260507 22:05:13.489 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:13.489 | INFO | "malloc" : " 14 GB", | 20260507 22:05:13.489 | INFO | "pinned" : " 29 GB" | 20260507 22:05:13.489 | INFO | } | 20260507 22:05:13.489 | INFO | Perf report for the last 5.14104 seconds: | 20260507 22:05:13.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:13.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:05:13.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:05:13.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:13.489 | INFO | Uploaded 36GiB to GPU in 2.36367s (15.2305GiB/s) | 20260507 22:05:13.489 | INFO | Downloaded 34GiB to GPU in 2.49495s (13.6275GiB/s) | 20260507 22:05:18.631 | INFO | Analyzing chunk frames = [2947584,2948096) | 20260507 22:05:18.631 | INFO | MemInfo: { | 20260507 22:05:18.631 | INFO | "gpu" : " 17 GB", | 20260507 22:05:18.631 | INFO | "huge" : "", | 20260507 22:05:18.631 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:18.631 | INFO | "malloc" : " 14 GB", | 20260507 22:05:18.631 | INFO | "pinned" : " 29 GB" | 20260507 22:05:18.631 | INFO | } | 20260507 22:05:18.632 | INFO | Perf report for the last 5.14245 seconds: | 20260507 22:05:18.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:18.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 22:05:18.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260507 22:05:18.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:18.632 | INFO | Uploaded 36GiB to GPU in 2.49969s (14.4018GiB/s) | 20260507 22:05:18.632 | INFO | Downloaded 34GiB to GPU in 2.41423s (14.0832GiB/s) | 20260507 22:05:23.772 | INFO | Analyzing chunk frames = [2948096,2948608) | 20260507 22:05:23.772 | INFO | MemInfo: { | 20260507 22:05:23.772 | INFO | "gpu" : " 17 GB", | 20260507 22:05:23.772 | INFO | "huge" : "", | 20260507 22:05:23.772 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:23.772 | INFO | "malloc" : " 14 GB", | 20260507 22:05:23.772 | INFO | "pinned" : " 29 GB" | 20260507 22:05:23.772 | INFO | } | 20260507 22:05:23.772 | INFO | Perf report for the last 5.14091 seconds: | 20260507 22:05:23.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:23.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.773 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260507 22:05:23.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.773 | INFO | PRE_HQ : 35.7%, 1.0, 18.6ms, 0 | 20260507 22:05:23.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:23.773 | INFO | Uploaded 36GiB to GPU in 2.55416s (14.0947GiB/s) | 20260507 22:05:23.773 | INFO | Downloaded 34GiB to GPU in 2.45007s (13.8772GiB/s) | 20260507 22:05:28.908 | INFO | Analyzing chunk frames = [2948608,2949120) | 20260507 22:05:28.908 | INFO | MemInfo: { | 20260507 22:05:28.908 | INFO | "gpu" : " 17 GB", | 20260507 22:05:28.908 | INFO | "huge" : "", | 20260507 22:05:28.908 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:28.908 | INFO | "malloc" : " 14 GB", | 20260507 22:05:28.908 | INFO | "pinned" : " 29 GB" | 20260507 22:05:28.908 | INFO | } | 20260507 22:05:28.908 | INFO | Perf report for the last 5.13591 seconds: | 20260507 22:05:28.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:28.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.908 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 22:05:28.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.909 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:05:28.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:28.909 | INFO | Uploaded 36GiB to GPU in 2.55596s (14.0847GiB/s) | 20260507 22:05:28.909 | INFO | Downloaded 34GiB to GPU in 2.44696s (13.8948GiB/s) | 20260507 22:05:28.909 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:05:28.909 | INFO | Total mem usage: 16.7 GiB | 20260507 22:05:28.909 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:05:28.909 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -5.48%) | 20260507 22:05:28.909 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:05:28.909 | INFO | High Water Breakdown: | 20260507 22:05:28.909 | INFO | 61% : 9.64 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:05:28.909 | INFO | 38.9% : 6.15 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:05:28.909 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:05:28.909 | INFO | ------------------------------------------ | 20260507 22:05:28.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:05:28.909 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:05:28.909 | INFO | Total mem usage: 26.5 GiB | 20260507 22:05:28.909 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:05:28.909 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:05:28.909 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 22:05:28.909 | INFO | High Water Breakdown: | 20260507 22:05:28.909 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:05:28.909 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:05:28.909 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:05:28.909 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:05:28.909 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:05:28.909 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:05:28.909 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:05:28.909 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:05:28.909 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:05:28.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:05:28.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:05:28.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:05:28.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:05:28.909 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:05:28.909 | INFO | ------------------------------------------ | 20260507 22:05:28.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:05:28.909 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:05:28.909 | INFO | Total mem usage: 27.8 GiB | 20260507 22:05:28.909 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:05:28.909 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:05:28.909 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:05:28.909 | INFO | High Water Breakdown: | 20260507 22:05:28.909 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:05:28.909 | INFO | ------------------------------------------ | 20260507 22:05:28.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:05:28.909 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:05:28.909 | INFO | Total mem usage: 15.7 GiB | 20260507 22:05:28.909 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:05:28.909 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:05:28.909 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:05:28.909 | INFO | High Water Breakdown: | 20260507 22:05:28.909 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:05:28.909 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:05:28.909 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:05:28.909 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:05:28.909 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:05:28.909 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:05:28.909 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:05:28.909 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:05:28.910 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:05:28.910 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:05:28.910 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:05:28.910 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:05:28.910 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:05:28.910 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:05:28.910 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:05:28.910 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:05:28.910 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:05:28.910 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:05:28.910 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:05:28.910 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:05:28.910 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:05:28.910 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:05:28.910 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:05:28.910 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:05:28.910 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:05:28.910 | INFO | ------------------------------------------ | 20260507 22:05:28.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:05:28.910 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:05:28.910 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:05:28.910 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:05:28.910 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.44e+01 / 1.91e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.14e-02 / 6.49e-03 | 20260507 22:05:28.910 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.46e+00, 5.84e-01 / 1.96e-01 | 20260507 22:05:28.910 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.48e+00, 2.07e-01 / 1.22e-01 | 20260507 22:05:28.910 | INFO | PerfReport: 16.12%, DME -- 4.24e+01 / 4.63e+01, 4.22e+00 / 1.75e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 55.89%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.34e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.49e+00 / 2.79e+00, 4.36e-01 / 2.33e-01 | 20260507 22:05:28.910 | INFO | PerfReport: 3.50%, Metrics -- 1.15e+00 / 1.35e+00, 2.85e-01 / 1.43e-01 | 20260507 22:05:28.910 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.58e-01 / 3.90e-01, 5.90e-02 / 3.16e-02 | 20260507 22:05:28.910 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:05:28.910 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:05:28.910 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:05:28.910 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 30.80%, Compute -- 3.29e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 2.07%, LockUpload -- 2.21e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 2.73%, LockDownload -- 2.92e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 15.69%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:05:28.910 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260507 22:05:29.028 | INFO | Starting a round of Baz IO | 20260507 22:05:32.501 | INFO | Populating RTMetrics report | 20260507 22:05:32.501 | INFO | RTMetrics report populated | 20260507 22:05:32.501 | INFO | Opening tmp RTMetrics json file | 20260507 22:05:32.522 | INFO | Serializing RTMetrics to json | 20260507 22:05:32.525 | INFO | Serialization done, swapping tmp file now... | 20260507 22:05:32.526 | INFO | RTMetrics done | 20260507 22:05:32.605 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.577 seconds | 20260507 22:05:32.605 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.32% / 0.01% / 56.67% | 20260507 22:05:32.605 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51198% / 1.19209e-05% / 5.5631e-05% | 20260507 22:05:32.605 | INFO | Baz IO overall: | 20260507 22:05:32.605 | INFO | BytesWritten : 1343555810143 | 20260507 22:05:32.605 | INFO | NumEvents : 651038109071 | 20260507 22:05:34.038 | INFO | Analyzing chunk frames = [2949120,2949632) | 20260507 22:05:34.038 | INFO | MemInfo: { | 20260507 22:05:34.038 | INFO | "gpu" : " 17 GB", | 20260507 22:05:34.038 | INFO | "huge" : "", | 20260507 22:05:34.038 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:34.038 | INFO | "malloc" : " 14 GB", | 20260507 22:05:34.038 | INFO | "pinned" : " 29 GB" | 20260507 22:05:34.038 | INFO | } | 20260507 22:05:34.038 | INFO | Perf report for the last 5.13009 seconds: | 20260507 22:05:34.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:34.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:34.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:34.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:34.038 | INFO | ANALYSIS : 69.3%, 2.9, 105.2ms, 0 | 20260507 22:05:34.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:34.039 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260507 22:05:34.039 | INFO | PRE_HQ : 68.2%, 1.8, 64.0ms, 0 | 20260507 22:05:34.039 | INFO | BAZWRITER : 69.5%, 4.0, 142.0ms, 0 | 20260507 22:05:34.039 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:05:34.039 | INFO | Uploaded 36GiB to GPU in 2.61692s (13.7566GiB/s) | 20260507 22:05:34.039 | INFO | Downloaded 38GiB to GPU in 2.69974s (14.0754GiB/s) | 20260507 22:05:39.182 | INFO | Analyzing chunk frames = [2949632,2950144) | 20260507 22:05:39.182 | INFO | MemInfo: { | 20260507 22:05:39.182 | INFO | "gpu" : " 17 GB", | 20260507 22:05:39.182 | INFO | "huge" : "", | 20260507 22:05:39.182 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:39.182 | INFO | "malloc" : " 14 GB", | 20260507 22:05:39.182 | INFO | "pinned" : " 29 GB" | 20260507 22:05:39.183 | INFO | } | 20260507 22:05:39.183 | INFO | Perf report for the last 5.14413 seconds: | 20260507 22:05:39.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:39.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:05:39.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:05:39.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:39.183 | INFO | Uploaded 36GiB to GPU in 2.5299s (14.2298GiB/s) | 20260507 22:05:39.183 | INFO | Downloaded 34GiB to GPU in 2.464s (13.7987GiB/s) | 20260507 22:05:44.330 | INFO | Analyzing chunk frames = [2950144,2950656) | 20260507 22:05:44.330 | INFO | MemInfo: { | 20260507 22:05:44.330 | INFO | "gpu" : " 17 GB", | 20260507 22:05:44.330 | INFO | "huge" : "", | 20260507 22:05:44.330 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:44.330 | INFO | "malloc" : " 14 GB", | 20260507 22:05:44.330 | INFO | "pinned" : " 29 GB" | 20260507 22:05:44.330 | INFO | } | 20260507 22:05:44.330 | INFO | Perf report for the last 5.14728 seconds: | 20260507 22:05:44.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:44.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 22:05:44.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:05:44.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:44.330 | INFO | Uploaded 36GiB to GPU in 2.55235s (14.1047GiB/s) | 20260507 22:05:44.330 | INFO | Downloaded 34GiB to GPU in 2.43947s (13.9374GiB/s) | 20260507 22:05:49.467 | INFO | Analyzing chunk frames = [2950656,2951168) | 20260507 22:05:49.467 | INFO | MemInfo: { | 20260507 22:05:49.467 | INFO | "gpu" : " 17 GB", | 20260507 22:05:49.467 | INFO | "huge" : "", | 20260507 22:05:49.467 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:49.467 | INFO | "malloc" : " 14 GB", | 20260507 22:05:49.467 | INFO | "pinned" : " 29 GB" | 20260507 22:05:49.467 | INFO | } | 20260507 22:05:49.467 | INFO | Perf report for the last 5.1372 seconds: | 20260507 22:05:49.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:49.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.467 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 22:05:49.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.467 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:05:49.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:49.468 | INFO | Uploaded 36GiB to GPU in 2.48114s (14.5095GiB/s) | 20260507 22:05:49.468 | INFO | Downloaded 34GiB to GPU in 2.38695s (14.2441GiB/s) | 20260507 22:05:54.606 | INFO | Analyzing chunk frames = [2951168,2951680) | 20260507 22:05:54.606 | INFO | MemInfo: { | 20260507 22:05:54.606 | INFO | "gpu" : " 17 GB", | 20260507 22:05:54.606 | INFO | "huge" : "", | 20260507 22:05:54.606 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:54.606 | INFO | "malloc" : " 14 GB", | 20260507 22:05:54.606 | INFO | "pinned" : " 29 GB" | 20260507 22:05:54.606 | INFO | } | 20260507 22:05:54.606 | INFO | Perf report for the last 5.13878 seconds: | 20260507 22:05:54.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:54.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:05:54.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:05:54.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:54.606 | INFO | Uploaded 36GiB to GPU in 2.36405s (15.2281GiB/s) | 20260507 22:05:54.606 | INFO | Downloaded 34GiB to GPU in 2.50457s (13.5752GiB/s) | 20260507 22:05:59.743 | INFO | Analyzing chunk frames = [2951680,2952192) | 20260507 22:05:59.743 | INFO | MemInfo: { | 20260507 22:05:59.743 | INFO | "gpu" : " 17 GB", | 20260507 22:05:59.743 | INFO | "huge" : "", | 20260507 22:05:59.743 | INFO | "huge_shared" : " 29 GB", | 20260507 22:05:59.743 | INFO | "malloc" : " 14 GB", | 20260507 22:05:59.743 | INFO | "pinned" : " 29 GB" | 20260507 22:05:59.743 | INFO | } | 20260507 22:05:59.743 | INFO | Perf report for the last 5.13748 seconds: | 20260507 22:05:59.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:05:59.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.743 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:05:59.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.743 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:05:59.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:05:59.744 | INFO | Uploaded 36GiB to GPU in 2.51982s (14.2868GiB/s) | 20260507 22:05:59.744 | INFO | Downloaded 34GiB to GPU in 2.41959s (14.0519GiB/s) | 20260507 22:06:04.884 | INFO | Analyzing chunk frames = [2952192,2952704) | 20260507 22:06:04.884 | INFO | MemInfo: { | 20260507 22:06:04.884 | INFO | "gpu" : " 17 GB", | 20260507 22:06:04.884 | INFO | "huge" : "", | 20260507 22:06:04.884 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:04.884 | INFO | "malloc" : " 14 GB", | 20260507 22:06:04.884 | INFO | "pinned" : " 29 GB" | 20260507 22:06:04.884 | INFO | } | 20260507 22:06:04.884 | INFO | Perf report for the last 5.14103 seconds: | 20260507 22:06:04.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:04.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.884 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260507 22:06:04.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.884 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 22:06:04.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:04.885 | INFO | Uploaded 36GiB to GPU in 2.53983s (14.1742GiB/s) | 20260507 22:06:04.885 | INFO | Downloaded 34GiB to GPU in 2.43719s (13.9505GiB/s) | 20260507 22:06:10.025 | INFO | Analyzing chunk frames = [2952704,2953216) | 20260507 22:06:10.025 | INFO | MemInfo: { | 20260507 22:06:10.025 | INFO | "gpu" : " 17 GB", | 20260507 22:06:10.025 | INFO | "huge" : "", | 20260507 22:06:10.025 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:10.025 | INFO | "malloc" : " 14 GB", | 20260507 22:06:10.025 | INFO | "pinned" : " 29 GB" | 20260507 22:06:10.025 | INFO | } | 20260507 22:06:10.025 | INFO | Perf report for the last 5.14056 seconds: | 20260507 22:06:10.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:10.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:06:10.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:06:10.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:10.025 | INFO | Uploaded 36GiB to GPU in 2.52825s (14.2391GiB/s) | 20260507 22:06:10.025 | INFO | Downloaded 34GiB to GPU in 2.43961s (13.9367GiB/s) | 20260507 22:06:10.123 | INFO | Starting a round of Baz IO | 20260507 22:06:13.583 | INFO | Populating RTMetrics report | 20260507 22:06:13.584 | INFO | RTMetrics report populated | 20260507 22:06:13.584 | INFO | Opening tmp RTMetrics json file | 20260507 22:06:13.607 | INFO | Serializing RTMetrics to json | 20260507 22:06:13.609 | INFO | Serialization done, swapping tmp file now... | 20260507 22:06:13.610 | INFO | RTMetrics done | 20260507 22:06:13.685 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.562 seconds | 20260507 22:06:13.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.32% / 0.01% / 56.68% | 20260507 22:06:13.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.50472% / 1.19209e-05% / 5.16574e-05% | 20260507 22:06:13.685 | INFO | Baz IO overall: | 20260507 22:06:13.685 | INFO | BytesWritten : 1345429808361 | 20260507 22:06:13.685 | INFO | NumEvents : 651943224748 | 20260507 22:06:15.161 | INFO | Analyzing chunk frames = [2953216,2953728) | 20260507 22:06:15.161 | INFO | MemInfo: { | 20260507 22:06:15.161 | INFO | "gpu" : " 17 GB", | 20260507 22:06:15.161 | INFO | "huge" : "", | 20260507 22:06:15.161 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:15.161 | INFO | "malloc" : " 14 GB", | 20260507 22:06:15.161 | INFO | "pinned" : " 29 GB" | 20260507 22:06:15.161 | INFO | } | 20260507 22:06:15.161 | INFO | Perf report for the last 5.13641 seconds: | 20260507 22:06:15.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:15.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:15.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:15.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:15.161 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 22:06:15.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:15.162 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 22:06:15.162 | INFO | PRE_HQ : 66.1%, 1.7, 57.6ms, 0 | 20260507 22:06:15.162 | INFO | BAZWRITER : 68.8%, 4.1, 143.5ms, 0 | 20260507 22:06:15.162 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 22:06:15.162 | INFO | Uploaded 36GiB to GPU in 2.5296s (14.2315GiB/s) | 20260507 22:06:15.162 | INFO | Downloaded 38GiB to GPU in 2.67919s (14.1834GiB/s) | 20260507 22:06:20.305 | INFO | Analyzing chunk frames = [2953728,2954240) | 20260507 22:06:20.305 | INFO | MemInfo: { | 20260507 22:06:20.306 | INFO | "gpu" : " 17 GB", | 20260507 22:06:20.306 | INFO | "huge" : "", | 20260507 22:06:20.306 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:20.306 | INFO | "malloc" : " 14 GB", | 20260507 22:06:20.306 | INFO | "pinned" : " 29 GB" | 20260507 22:06:20.306 | INFO | } | 20260507 22:06:20.306 | INFO | Perf report for the last 5.14434 seconds: | 20260507 22:06:20.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:20.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:06:20.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:06:20.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:20.306 | INFO | Uploaded 36GiB to GPU in 2.52877s (14.2362GiB/s) | 20260507 22:06:20.306 | INFO | Downloaded 34GiB to GPU in 2.47097s (13.7598GiB/s) | 20260507 22:06:25.442 | INFO | Analyzing chunk frames = [2954240,2954752) | 20260507 22:06:25.442 | INFO | MemInfo: { | 20260507 22:06:25.442 | INFO | "gpu" : " 17 GB", | 20260507 22:06:25.442 | INFO | "huge" : "", | 20260507 22:06:25.442 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:25.442 | INFO | "malloc" : " 14 GB", | 20260507 22:06:25.442 | INFO | "pinned" : " 29 GB" | 20260507 22:06:25.442 | INFO | } | 20260507 22:06:25.442 | INFO | Perf report for the last 5.13648 seconds: | 20260507 22:06:25.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:25.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.442 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:06:25.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.442 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:06:25.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:25.443 | INFO | Uploaded 36GiB to GPU in 2.55366s (14.0974GiB/s) | 20260507 22:06:25.443 | INFO | Downloaded 34GiB to GPU in 2.48595s (13.6768GiB/s) | 20260507 22:06:30.580 | INFO | Analyzing chunk frames = [2954752,2955264) | 20260507 22:06:30.581 | INFO | MemInfo: { | 20260507 22:06:30.581 | INFO | "gpu" : " 17 GB", | 20260507 22:06:30.581 | INFO | "huge" : "", | 20260507 22:06:30.581 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:30.581 | INFO | "malloc" : " 14 GB", | 20260507 22:06:30.581 | INFO | "pinned" : " 29 GB" | 20260507 22:06:30.581 | INFO | } | 20260507 22:06:30.581 | INFO | Perf report for the last 5.1385 seconds: | 20260507 22:06:30.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:30.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:06:30.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:06:30.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:30.581 | INFO | Uploaded 36GiB to GPU in 2.45551s (14.6609GiB/s) | 20260507 22:06:30.581 | INFO | Downloaded 34GiB to GPU in 2.39994s (14.167GiB/s) | 20260507 22:06:35.718 | INFO | Analyzing chunk frames = [2955264,2955776) | 20260507 22:06:35.718 | INFO | MemInfo: { | 20260507 22:06:35.718 | INFO | "gpu" : " 17 GB", | 20260507 22:06:35.718 | INFO | "huge" : "", | 20260507 22:06:35.718 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:35.718 | INFO | "malloc" : " 14 GB", | 20260507 22:06:35.718 | INFO | "pinned" : " 29 GB" | 20260507 22:06:35.718 | INFO | } | 20260507 22:06:35.718 | INFO | Perf report for the last 5.13707 seconds: | 20260507 22:06:35.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:35.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:06:35.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:06:35.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:35.718 | INFO | Uploaded 36GiB to GPU in 2.3365s (15.4077GiB/s) | 20260507 22:06:35.718 | INFO | Downloaded 34GiB to GPU in 2.5173s (13.5065GiB/s) | 20260507 22:06:40.852 | INFO | Analyzing chunk frames = [2955776,2956288) | 20260507 22:06:40.852 | INFO | MemInfo: { | 20260507 22:06:40.852 | INFO | "gpu" : " 17 GB", | 20260507 22:06:40.852 | INFO | "huge" : "", | 20260507 22:06:40.852 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:40.852 | INFO | "malloc" : " 14 GB", | 20260507 22:06:40.852 | INFO | "pinned" : " 29 GB" | 20260507 22:06:40.852 | INFO | } | 20260507 22:06:40.852 | INFO | Perf report for the last 5.13428 seconds: | 20260507 22:06:40.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:40.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:06:40.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 22:06:40.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:40.852 | INFO | Uploaded 36GiB to GPU in 2.49313s (14.4397GiB/s) | 20260507 22:06:40.853 | INFO | Downloaded 34GiB to GPU in 2.43336s (13.9724GiB/s) | 20260507 22:06:45.991 | INFO | Analyzing chunk frames = [2956288,2956800) | 20260507 22:06:45.991 | INFO | MemInfo: { | 20260507 22:06:45.991 | INFO | "gpu" : " 17 GB", | 20260507 22:06:45.991 | INFO | "huge" : "", | 20260507 22:06:45.991 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:45.991 | INFO | "malloc" : " 14 GB", | 20260507 22:06:45.991 | INFO | "pinned" : " 29 GB" | 20260507 22:06:45.991 | INFO | } | 20260507 22:06:45.991 | INFO | Perf report for the last 5.13922 seconds: | 20260507 22:06:45.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:45.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.991 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260507 22:06:45.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.991 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:06:45.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:45.992 | INFO | Uploaded 36GiB to GPU in 2.50377s (14.3783GiB/s) | 20260507 22:06:45.992 | INFO | Downloaded 34GiB to GPU in 2.45933s (13.8249GiB/s) | 20260507 22:06:51.136 | INFO | Analyzing chunk frames = [2956800,2957312) | 20260507 22:06:51.136 | INFO | MemInfo: { | 20260507 22:06:51.136 | INFO | "gpu" : " 17 GB", | 20260507 22:06:51.136 | INFO | "huge" : "", | 20260507 22:06:51.136 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:51.136 | INFO | "malloc" : " 14 GB", | 20260507 22:06:51.136 | INFO | "pinned" : " 29 GB" | 20260507 22:06:51.136 | INFO | } | 20260507 22:06:51.136 | INFO | Perf report for the last 5.14486 seconds: | 20260507 22:06:51.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:51.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.136 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 22:06:51.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.136 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:06:51.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:51.137 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:06:51.137 | INFO | Total mem usage: 16.7 GiB | 20260507 22:06:51.137 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:06:51.137 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.6%) | 20260507 22:06:51.137 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:06:51.137 | INFO | High Water Breakdown: | 20260507 22:06:51.137 | INFO | 60% : 8.97 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:06:51.137 | INFO | 39.8% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:06:51.137 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:06:51.137 | INFO | ------------------------------------------ | 20260507 22:06:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:06:51.137 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:06:51.137 | INFO | Total mem usage: 26.5 GiB | 20260507 22:06:51.137 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:06:51.137 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:06:51.137 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:06:51.137 | INFO | High Water Breakdown: | 20260507 22:06:51.137 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:06:51.137 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:06:51.137 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:06:51.137 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:06:51.137 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:06:51.137 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:06:51.137 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:06:51.137 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:06:51.137 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:06:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:06:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:06:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:06:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:06:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:06:51.137 | INFO | ------------------------------------------ | 20260507 22:06:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:06:51.137 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:06:51.137 | INFO | Total mem usage: 27.8 GiB | 20260507 22:06:51.137 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:06:51.137 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:06:51.137 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:06:51.137 | INFO | High Water Breakdown: | 20260507 22:06:51.137 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:06:51.137 | INFO | ------------------------------------------ | 20260507 22:06:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:06:51.137 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:06:51.137 | INFO | Total mem usage: 15.7 GiB | 20260507 22:06:51.137 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:06:51.137 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:06:51.137 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:06:51.137 | INFO | High Water Breakdown: | 20260507 22:06:51.137 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:06:51.137 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:06:51.137 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:06:51.137 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:06:51.137 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:06:51.137 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:06:51.137 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:06:51.137 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:06:51.137 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:06:51.137 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:06:51.137 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:06:51.137 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:06:51.137 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:06:51.137 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:06:51.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:06:51.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:06:51.137 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:06:51.137 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:06:51.137 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:06:51.137 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:06:51.137 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:06:51.137 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:06:51.137 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:06:51.137 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:06:51.137 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:06:51.137 | INFO | Uploaded 36GiB to GPU in 2.48845s (14.4669GiB/s) | 20260507 22:06:51.137 | INFO | ------------------------------------------ | 20260507 22:06:51.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:06:51.137 | INFO | Downloaded 34GiB to GPU in 2.45003s (13.8774GiB/s) | 20260507 22:06:51.137 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:06:51.137 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:06:51.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:06:51.137 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.42e+01 / 1.43e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.15e-02 / 5.20e-03 | 20260507 22:06:51.138 | INFO | PerfReport: 12.12%, Baseline -- 4.00e+00 / 4.48e+00, 5.97e-01 / 2.32e-01 | 20260507 22:06:51.138 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.11e-01 / 1.08e-01 | 20260507 22:06:51.138 | INFO | PerfReport: 15.79%, DME -- 4.16e+01 / 4.50e+01, 3.77e+00 / 1.27e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.68e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.45e+00 / 2.72e+00, 4.29e-01 / 2.49e-01 | 20260507 22:06:51.138 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.33e+00, 2.94e-01 / 1.61e-01 | 20260507 22:06:51.138 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.86e-01, 5.96e-02 / 2.84e-02 | 20260507 22:06:51.138 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:06:51.138 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:06:51.138 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:06:51.138 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 24.21%, Download -- 2.58e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 30.90%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 2.06%, LockUpload -- 2.19e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 2.42%, LockDownload -- 2.58e+00 / 0.00e+00, 3.84e+00 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 15.97%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:06:51.138 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260507 22:06:51.234 | INFO | Starting a round of Baz IO | 20260507 22:06:54.698 | INFO | Populating RTMetrics report | 20260507 22:06:54.698 | INFO | RTMetrics report populated | 20260507 22:06:54.698 | INFO | Opening tmp RTMetrics json file | 20260507 22:06:54.725 | INFO | Serializing RTMetrics to json | 20260507 22:06:54.728 | INFO | Serialization done, swapping tmp file now... | 20260507 22:06:54.729 | INFO | RTMetrics done | 20260507 22:06:54.826 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.592 seconds | 20260507 22:06:54.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.31% / 0.01% / 56.68% | 20260507 22:06:54.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49733% / 7.94729e-06% / 7.94729e-05% | 20260507 22:06:54.826 | INFO | Baz IO overall: | 20260507 22:06:54.826 | INFO | BytesWritten : 1347301864619 | 20260507 22:06:54.826 | INFO | NumEvents : 652846966481 | 20260507 22:06:56.277 | INFO | Analyzing chunk frames = [2957312,2957824) | 20260507 22:06:56.277 | INFO | MemInfo: { | 20260507 22:06:56.277 | INFO | "gpu" : " 17 GB", | 20260507 22:06:56.277 | INFO | "huge" : "", | 20260507 22:06:56.277 | INFO | "huge_shared" : " 29 GB", | 20260507 22:06:56.277 | INFO | "malloc" : " 14 GB", | 20260507 22:06:56.277 | INFO | "pinned" : " 29 GB" | 20260507 22:06:56.277 | INFO | } | 20260507 22:06:56.277 | INFO | Perf report for the last 5.14092 seconds: | 20260507 22:06:56.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:06:56.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:56.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:56.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:56.277 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 22:06:56.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:06:56.277 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 22:06:56.277 | INFO | PRE_HQ : 65.4%, 1.7, 57.5ms, 0 | 20260507 22:06:56.277 | INFO | BAZWRITER : 69.4%, 3.7, 132.7ms, 0 | 20260507 22:06:56.277 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 22:06:56.278 | INFO | Uploaded 36GiB to GPU in 2.51532s (14.3123GiB/s) | 20260507 22:06:56.278 | INFO | Downloaded 38GiB to GPU in 2.68583s (14.1483GiB/s) | 20260507 22:07:01.418 | INFO | Analyzing chunk frames = [2957824,2958336) | 20260507 22:07:01.418 | INFO | MemInfo: { | 20260507 22:07:01.418 | INFO | "gpu" : " 17 GB", | 20260507 22:07:01.418 | INFO | "huge" : "", | 20260507 22:07:01.418 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:01.418 | INFO | "malloc" : " 14 GB", | 20260507 22:07:01.418 | INFO | "pinned" : " 29 GB" | 20260507 22:07:01.418 | INFO | } | 20260507 22:07:01.418 | INFO | Perf report for the last 5.14045 seconds: | 20260507 22:07:01.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:01.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:07:01.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:07:01.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:01.418 | INFO | Uploaded 36GiB to GPU in 2.55229s (14.105GiB/s) | 20260507 22:07:01.418 | INFO | Downloaded 34GiB to GPU in 2.48453s (13.6847GiB/s) | 20260507 22:07:06.554 | INFO | Analyzing chunk frames = [2958336,2958848) | 20260507 22:07:06.554 | INFO | MemInfo: { | 20260507 22:07:06.554 | INFO | "gpu" : " 17 GB", | 20260507 22:07:06.554 | INFO | "huge" : "", | 20260507 22:07:06.554 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:06.554 | INFO | "malloc" : " 14 GB", | 20260507 22:07:06.554 | INFO | "pinned" : " 29 GB" | 20260507 22:07:06.554 | INFO | } | 20260507 22:07:06.555 | INFO | Perf report for the last 5.13694 seconds: | 20260507 22:07:06.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:06.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 22:07:06.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:07:06.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:06.555 | INFO | Uploaded 36GiB to GPU in 2.5467s (14.136GiB/s) | 20260507 22:07:06.555 | INFO | Downloaded 34GiB to GPU in 2.4747s (13.7391GiB/s) | 20260507 22:07:11.694 | INFO | Analyzing chunk frames = [2958848,2959360) | 20260507 22:07:11.694 | INFO | MemInfo: { | 20260507 22:07:11.694 | INFO | "gpu" : " 17 GB", | 20260507 22:07:11.694 | INFO | "huge" : "", | 20260507 22:07:11.694 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:11.694 | INFO | "malloc" : " 14 GB", | 20260507 22:07:11.694 | INFO | "pinned" : " 29 GB" | 20260507 22:07:11.694 | INFO | } | 20260507 22:07:11.694 | INFO | Perf report for the last 5.1396 seconds: | 20260507 22:07:11.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:11.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.694 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:07:11.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.694 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:07:11.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:11.695 | INFO | Uploaded 36GiB to GPU in 2.46921s (14.5796GiB/s) | 20260507 22:07:11.695 | INFO | Downloaded 34GiB to GPU in 2.39776s (14.1799GiB/s) | 20260507 22:07:16.832 | INFO | Analyzing chunk frames = [2959360,2959872) | 20260507 22:07:16.832 | INFO | MemInfo: { | 20260507 22:07:16.832 | INFO | "gpu" : " 17 GB", | 20260507 22:07:16.832 | INFO | "huge" : "", | 20260507 22:07:16.832 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:16.832 | INFO | "malloc" : " 14 GB", | 20260507 22:07:16.832 | INFO | "pinned" : " 29 GB" | 20260507 22:07:16.832 | INFO | } | 20260507 22:07:16.832 | INFO | Perf report for the last 5.13798 seconds: | 20260507 22:07:16.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:16.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.832 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:07:16.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.832 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260507 22:07:16.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:16.833 | INFO | Uploaded 36GiB to GPU in 2.31688s (15.5382GiB/s) | 20260507 22:07:16.833 | INFO | Downloaded 34GiB to GPU in 2.51059s (13.5426GiB/s) | 20260507 22:07:21.970 | INFO | Analyzing chunk frames = [2959872,2960384) | 20260507 22:07:21.970 | INFO | MemInfo: { | 20260507 22:07:21.970 | INFO | "gpu" : " 17 GB", | 20260507 22:07:21.970 | INFO | "huge" : "", | 20260507 22:07:21.970 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:21.970 | INFO | "malloc" : " 14 GB", | 20260507 22:07:21.970 | INFO | "pinned" : " 29 GB" | 20260507 22:07:21.970 | INFO | } | 20260507 22:07:21.970 | INFO | Perf report for the last 5.13822 seconds: | 20260507 22:07:21.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:21.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.970 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:07:21.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.971 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:07:21.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:21.971 | INFO | Uploaded 36GiB to GPU in 2.51933s (14.2895GiB/s) | 20260507 22:07:21.971 | INFO | Downloaded 34GiB to GPU in 2.42148s (14.041GiB/s) | 20260507 22:07:27.110 | INFO | Analyzing chunk frames = [2960384,2960896) | 20260507 22:07:27.110 | INFO | MemInfo: { | 20260507 22:07:27.110 | INFO | "gpu" : " 17 GB", | 20260507 22:07:27.110 | INFO | "huge" : "", | 20260507 22:07:27.110 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:27.110 | INFO | "malloc" : " 14 GB", | 20260507 22:07:27.110 | INFO | "pinned" : " 29 GB" | 20260507 22:07:27.110 | INFO | } | 20260507 22:07:27.110 | INFO | Perf report for the last 5.13998 seconds: | 20260507 22:07:27.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:27.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.110 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:07:27.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.111 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:07:27.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:27.111 | INFO | Uploaded 36GiB to GPU in 2.53073s (14.2251GiB/s) | 20260507 22:07:27.111 | INFO | Downloaded 34GiB to GPU in 2.46226s (13.8085GiB/s) | 20260507 22:07:32.249 | INFO | Analyzing chunk frames = [2960896,2961408) | 20260507 22:07:32.249 | INFO | MemInfo: { | 20260507 22:07:32.249 | INFO | "gpu" : " 17 GB", | 20260507 22:07:32.249 | INFO | "huge" : "", | 20260507 22:07:32.250 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:32.250 | INFO | "malloc" : " 14 GB", | 20260507 22:07:32.250 | INFO | "pinned" : " 29 GB" | 20260507 22:07:32.250 | INFO | } | 20260507 22:07:32.250 | INFO | Perf report for the last 5.1392 seconds: | 20260507 22:07:32.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:32.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:07:32.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:07:32.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:32.250 | INFO | Uploaded 36GiB to GPU in 2.48726s (14.4738GiB/s) | 20260507 22:07:32.250 | INFO | Downloaded 34GiB to GPU in 2.44812s (13.8882GiB/s) | 20260507 22:07:32.348 | INFO | Starting a round of Baz IO | 20260507 22:07:35.815 | INFO | Populating RTMetrics report | 20260507 22:07:35.815 | INFO | RTMetrics report populated | 20260507 22:07:35.815 | INFO | Opening tmp RTMetrics json file | 20260507 22:07:35.821 | INFO | Serializing RTMetrics to json | 20260507 22:07:35.840 | INFO | Serialization done, swapping tmp file now... | 20260507 22:07:35.840 | INFO | RTMetrics done | 20260507 22:07:35.936 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.588 seconds | 20260507 22:07:35.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.30% / 0.01% / 56.69% | 20260507 22:07:35.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49467% / 1.58946e-05% / 5.96046e-05% | 20260507 22:07:35.937 | INFO | Baz IO overall: | 20260507 22:07:35.937 | INFO | BytesWritten : 1349173518535 | 20260507 22:07:35.937 | INFO | NumEvents : 653750558127 | 20260507 22:07:37.392 | INFO | Analyzing chunk frames = [2961408,2961920) | 20260507 22:07:37.392 | INFO | MemInfo: { | 20260507 22:07:37.392 | INFO | "gpu" : " 17 GB", | 20260507 22:07:37.392 | INFO | "huge" : "", | 20260507 22:07:37.392 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:37.392 | INFO | "malloc" : " 14 GB", | 20260507 22:07:37.392 | INFO | "pinned" : " 29 GB" | 20260507 22:07:37.392 | INFO | } | 20260507 22:07:37.392 | INFO | Perf report for the last 5.14252 seconds: | 20260507 22:07:37.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:37.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:37.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:37.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:37.392 | INFO | ANALYSIS : 68.9%, 2.9, 106.5ms, 0 | 20260507 22:07:37.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:37.392 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 22:07:37.392 | INFO | PRE_HQ : 65.9%, 1.7, 57.9ms, 0 | 20260507 22:07:37.392 | INFO | BAZWRITER : 69.3%, 3.9, 139.1ms, 0 | 20260507 22:07:37.392 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:07:37.392 | INFO | Uploaded 36GiB to GPU in 2.56731s (14.0224GiB/s) | 20260507 22:07:37.392 | INFO | Downloaded 38GiB to GPU in 2.69124s (14.1199GiB/s) | 20260507 22:07:42.537 | INFO | Analyzing chunk frames = [2961920,2962432) | 20260507 22:07:42.537 | INFO | MemInfo: { | 20260507 22:07:42.537 | INFO | "gpu" : " 17 GB", | 20260507 22:07:42.537 | INFO | "huge" : "", | 20260507 22:07:42.537 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:42.537 | INFO | "malloc" : " 14 GB", | 20260507 22:07:42.537 | INFO | "pinned" : " 29 GB" | 20260507 22:07:42.537 | INFO | } | 20260507 22:07:42.537 | INFO | Perf report for the last 5.14476 seconds: | 20260507 22:07:42.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:42.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.537 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260507 22:07:42.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.537 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:07:42.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:42.538 | INFO | Uploaded 36GiB to GPU in 2.5389s (14.1794GiB/s) | 20260507 22:07:42.538 | INFO | Downloaded 34GiB to GPU in 2.49712s (13.6157GiB/s) | 20260507 22:07:47.670 | INFO | Analyzing chunk frames = [2962432,2962944) | 20260507 22:07:47.670 | INFO | MemInfo: { | 20260507 22:07:47.670 | INFO | "gpu" : " 17 GB", | 20260507 22:07:47.670 | INFO | "huge" : "", | 20260507 22:07:47.670 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:47.670 | INFO | "malloc" : " 14 GB", | 20260507 22:07:47.670 | INFO | "pinned" : " 29 GB" | 20260507 22:07:47.670 | INFO | } | 20260507 22:07:47.670 | INFO | Perf report for the last 5.1331 seconds: | 20260507 22:07:47.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:47.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.670 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 22:07:47.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.670 | INFO | PRE_HQ : 25.4%, 1.0, 13.1ms, 0 | 20260507 22:07:47.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:47.671 | INFO | Uploaded 36GiB to GPU in 2.4869s (14.4758GiB/s) | 20260507 22:07:47.671 | INFO | Downloaded 34GiB to GPU in 2.48683s (13.672GiB/s) | 20260507 22:07:52.804 | INFO | Analyzing chunk frames = [2962944,2963456) | 20260507 22:07:52.804 | INFO | MemInfo: { | 20260507 22:07:52.804 | INFO | "gpu" : " 17 GB", | 20260507 22:07:52.804 | INFO | "huge" : "", | 20260507 22:07:52.804 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:52.804 | INFO | "malloc" : " 14 GB", | 20260507 22:07:52.804 | INFO | "pinned" : " 29 GB" | 20260507 22:07:52.805 | INFO | } | 20260507 22:07:52.805 | INFO | Perf report for the last 5.13451 seconds: | 20260507 22:07:52.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:52.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260507 22:07:52.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 22:07:52.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:52.805 | INFO | Uploaded 36GiB to GPU in 2.47462s (14.5477GiB/s) | 20260507 22:07:52.805 | INFO | Downloaded 34GiB to GPU in 2.40764s (14.1217GiB/s) | 20260507 22:07:57.946 | INFO | Analyzing chunk frames = [2963456,2963968) | 20260507 22:07:57.946 | INFO | MemInfo: { | 20260507 22:07:57.946 | INFO | "gpu" : " 17 GB", | 20260507 22:07:57.946 | INFO | "huge" : "", | 20260507 22:07:57.946 | INFO | "huge_shared" : " 29 GB", | 20260507 22:07:57.946 | INFO | "malloc" : " 14 GB", | 20260507 22:07:57.946 | INFO | "pinned" : " 29 GB" | 20260507 22:07:57.946 | INFO | } | 20260507 22:07:57.946 | INFO | Perf report for the last 5.14177 seconds: | 20260507 22:07:57.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:07:57.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.946 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:07:57.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.946 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:07:57.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:07:57.947 | INFO | Uploaded 36GiB to GPU in 2.32613s (15.4763GiB/s) | 20260507 22:07:57.947 | INFO | Downloaded 34GiB to GPU in 2.50139s (13.5924GiB/s) | 20260507 22:08:03.090 | INFO | Analyzing chunk frames = [2963968,2964480) | 20260507 22:08:03.090 | INFO | MemInfo: { | 20260507 22:08:03.090 | INFO | "gpu" : " 17 GB", | 20260507 22:08:03.090 | INFO | "huge" : "", | 20260507 22:08:03.090 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:03.090 | INFO | "malloc" : " 14 GB", | 20260507 22:08:03.090 | INFO | "pinned" : " 29 GB" | 20260507 22:08:03.090 | INFO | } | 20260507 22:08:03.090 | INFO | Perf report for the last 5.14398 seconds: | 20260507 22:08:03.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:03.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.090 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 22:08:03.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.090 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 22:08:03.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:03.091 | INFO | Uploaded 36GiB to GPU in 2.49792s (14.412GiB/s) | 20260507 22:08:03.091 | INFO | Downloaded 34GiB to GPU in 2.42641s (14.0125GiB/s) | 20260507 22:08:08.231 | INFO | Analyzing chunk frames = [2964480,2964992) | 20260507 22:08:08.231 | INFO | MemInfo: { | 20260507 22:08:08.231 | INFO | "gpu" : " 17 GB", | 20260507 22:08:08.231 | INFO | "huge" : "", | 20260507 22:08:08.231 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:08.231 | INFO | "malloc" : " 14 GB", | 20260507 22:08:08.231 | INFO | "pinned" : " 29 GB" | 20260507 22:08:08.231 | INFO | } | 20260507 22:08:08.232 | INFO | Perf report for the last 5.14116 seconds: | 20260507 22:08:08.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:08.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 22:08:08.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:08:08.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:08.232 | INFO | Uploaded 36GiB to GPU in 2.50975s (14.3441GiB/s) | 20260507 22:08:08.232 | INFO | Downloaded 34GiB to GPU in 2.4605s (13.8183GiB/s) | 20260507 22:08:13.367 | INFO | Analyzing chunk frames = [2964992,2965504) | 20260507 22:08:13.367 | INFO | MemInfo: { | 20260507 22:08:13.367 | INFO | "gpu" : " 17 GB", | 20260507 22:08:13.367 | INFO | "huge" : "", | 20260507 22:08:13.367 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:13.367 | INFO | "malloc" : " 14 GB", | 20260507 22:08:13.367 | INFO | "pinned" : " 29 GB" | 20260507 22:08:13.367 | INFO | } | 20260507 22:08:13.367 | INFO | Perf report for the last 5.13594 seconds: | 20260507 22:08:13.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:13.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:08:13.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:08:13.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:13.368 | INFO | Uploaded 36GiB to GPU in 2.48448s (14.49GiB/s) | 20260507 22:08:13.368 | INFO | Downloaded 34GiB to GPU in 2.4433s (13.9156GiB/s) | 20260507 22:08:13.368 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:08:13.368 | INFO | Total mem usage: 16.7 GiB | 20260507 22:08:13.368 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:08:13.368 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.1%) | 20260507 22:08:13.368 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:08:13.368 | INFO | High Water Breakdown: | 20260507 22:08:13.368 | INFO | 60.2% : 9.03 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:08:13.368 | INFO | 39.7% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:08:13.368 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:08:13.368 | INFO | ------------------------------------------ | 20260507 22:08:13.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:08:13.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:08:13.368 | INFO | Total mem usage: 26.5 GiB | 20260507 22:08:13.368 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:08:13.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:08:13.368 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:08:13.368 | INFO | High Water Breakdown: | 20260507 22:08:13.368 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:08:13.368 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:08:13.368 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:08:13.368 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:08:13.368 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:08:13.368 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:08:13.368 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:08:13.368 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:08:13.368 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:08:13.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:08:13.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:08:13.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:08:13.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:08:13.368 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:08:13.368 | INFO | ------------------------------------------ | 20260507 22:08:13.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:08:13.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:08:13.369 | INFO | Total mem usage: 27.8 GiB | 20260507 22:08:13.369 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:08:13.369 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 22:08:13.369 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 22:08:13.369 | INFO | High Water Breakdown: | 20260507 22:08:13.369 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 22:08:13.369 | INFO | ------------------------------------------ | 20260507 22:08:13.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:08:13.369 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:08:13.369 | INFO | Total mem usage: 15.7 GiB | 20260507 22:08:13.369 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:08:13.369 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:08:13.369 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:08:13.369 | INFO | High Water Breakdown: | 20260507 22:08:13.369 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:08:13.369 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:08:13.369 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:08:13.369 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:08:13.369 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:08:13.369 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:08:13.369 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:08:13.369 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:08:13.369 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:08:13.369 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:08:13.369 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:08:13.369 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:08:13.369 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:08:13.369 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:08:13.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:08:13.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:08:13.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:08:13.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:08:13.369 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:08:13.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:08:13.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:08:13.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:08:13.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:08:13.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:08:13.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:08:13.369 | INFO | ------------------------------------------ | 20260507 22:08:13.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:08:13.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:08:13.369 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:08:13.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:08:13.369 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.40e+01 / 1.56e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.24e-02 / 5.41e-03 | 20260507 22:08:13.369 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.47e+00, 6.01e-01 / 2.21e-01 | 20260507 22:08:13.369 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.51e+00, 2.09e-01 / 9.43e-02 | 20260507 22:08:13.369 | INFO | PerfReport: 15.97%, DME -- 4.21e+01 / 4.57e+01, 3.95e+00 / 1.34e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.71e+00 / 1.76e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.47e+00 / 2.80e+00, 4.34e-01 / 2.28e-01 | 20260507 22:08:13.369 | INFO | PerfReport: 3.50%, Metrics -- 1.15e+00 / 1.39e+00, 2.88e-01 / 1.65e-01 | 20260507 22:08:13.369 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.90e-01, 5.91e-02 / 3.25e-02 | 20260507 22:08:13.369 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:08:13.369 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:08:13.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:08:13.369 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 24.29%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 24.27%, Download -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 30.86%, Compute -- 3.29e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 1.97%, LockUpload -- 2.10e+00 / 0.00e+00, 5.12e+00 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 2.60%, LockDownload -- 2.78e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 16.00%, LockCompute -- 1.71e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:08:13.369 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 22:08:13.465 | INFO | Starting a round of Baz IO | 20260507 22:08:16.940 | INFO | Populating RTMetrics report | 20260507 22:08:16.940 | INFO | RTMetrics report populated | 20260507 22:08:16.940 | INFO | Opening tmp RTMetrics json file | 20260507 22:08:16.964 | INFO | Serializing RTMetrics to json | 20260507 22:08:16.966 | INFO | Serialization done, swapping tmp file now... | 20260507 22:08:16.966 | INFO | RTMetrics done | 20260507 22:08:17.039 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.574 seconds | 20260507 22:08:17.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.29% / 0.01% / 56.70% | 20260507 22:08:17.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49704% / 1.98682e-05% / 5.5631e-05% | 20260507 22:08:17.039 | INFO | Baz IO overall: | 20260507 22:08:17.039 | INFO | BytesWritten : 1351047628652 | 20260507 22:08:17.039 | INFO | NumEvents : 654656291851 | 20260507 22:08:18.505 | INFO | Analyzing chunk frames = [2965504,2966016) | 20260507 22:08:18.505 | INFO | MemInfo: { | 20260507 22:08:18.505 | INFO | "gpu" : " 17 GB", | 20260507 22:08:18.505 | INFO | "huge" : "", | 20260507 22:08:18.505 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:18.505 | INFO | "malloc" : " 14 GB", | 20260507 22:08:18.505 | INFO | "pinned" : " 29 GB" | 20260507 22:08:18.505 | INFO | } | 20260507 22:08:18.505 | INFO | Perf report for the last 5.13769 seconds: | 20260507 22:08:18.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:18.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:18.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:18.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:18.505 | INFO | ANALYSIS : 68.8%, 3.0, 106.4ms, 0 | 20260507 22:08:18.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:18.505 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 22:08:18.505 | INFO | PRE_HQ : 65.9%, 1.7, 57.7ms, 0 | 20260507 22:08:18.505 | INFO | BAZWRITER : 69.2%, 4.3, 153.9ms, 0 | 20260507 22:08:18.505 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 22:08:18.506 | INFO | Uploaded 36GiB to GPU in 2.56086s (14.0578GiB/s) | 20260507 22:08:18.506 | INFO | Downloaded 38GiB to GPU in 2.70126s (14.0675GiB/s) | 20260507 22:08:23.645 | INFO | Analyzing chunk frames = [2966016,2966528) | 20260507 22:08:23.645 | INFO | MemInfo: { | 20260507 22:08:23.645 | INFO | "gpu" : " 17 GB", | 20260507 22:08:23.645 | INFO | "huge" : "", | 20260507 22:08:23.645 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:23.645 | INFO | "malloc" : " 14 GB", | 20260507 22:08:23.645 | INFO | "pinned" : " 29 GB" | 20260507 22:08:23.645 | INFO | } | 20260507 22:08:23.645 | INFO | Perf report for the last 5.14022 seconds: | 20260507 22:08:23.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:23.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:08:23.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:08:23.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:23.646 | INFO | Uploaded 36GiB to GPU in 2.54757s (14.1311GiB/s) | 20260507 22:08:23.646 | INFO | Downloaded 34GiB to GPU in 2.48588s (13.6772GiB/s) | 20260507 22:08:28.785 | INFO | Analyzing chunk frames = [2966528,2967040) | 20260507 22:08:28.785 | INFO | MemInfo: { | 20260507 22:08:28.785 | INFO | "gpu" : " 17 GB", | 20260507 22:08:28.785 | INFO | "huge" : "", | 20260507 22:08:28.785 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:28.785 | INFO | "malloc" : " 14 GB", | 20260507 22:08:28.785 | INFO | "pinned" : " 29 GB" | 20260507 22:08:28.785 | INFO | } | 20260507 22:08:28.786 | INFO | Perf report for the last 5.1401 seconds: | 20260507 22:08:28.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:28.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:08:28.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:08:28.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:28.786 | INFO | Uploaded 36GiB to GPU in 2.54896s (14.1234GiB/s) | 20260507 22:08:28.786 | INFO | Downloaded 34GiB to GPU in 2.47738s (13.7242GiB/s) | 20260507 22:08:33.919 | INFO | Analyzing chunk frames = [2967040,2967552) | 20260507 22:08:33.919 | INFO | MemInfo: { | 20260507 22:08:33.919 | INFO | "gpu" : " 17 GB", | 20260507 22:08:33.919 | INFO | "huge" : "", | 20260507 22:08:33.919 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:33.919 | INFO | "malloc" : " 14 GB", | 20260507 22:08:33.919 | INFO | "pinned" : " 29 GB" | 20260507 22:08:33.919 | INFO | } | 20260507 22:08:33.919 | INFO | Perf report for the last 5.13331 seconds: | 20260507 22:08:33.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:33.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260507 22:08:33.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:08:33.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:33.919 | INFO | Uploaded 36GiB to GPU in 2.48226s (14.5029GiB/s) | 20260507 22:08:33.919 | INFO | Downloaded 34GiB to GPU in 2.41521s (14.0774GiB/s) | 20260507 22:08:39.059 | INFO | Analyzing chunk frames = [2967552,2968064) | 20260507 22:08:39.059 | INFO | MemInfo: { | 20260507 22:08:39.059 | INFO | "gpu" : " 17 GB", | 20260507 22:08:39.059 | INFO | "huge" : "", | 20260507 22:08:39.059 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:39.059 | INFO | "malloc" : " 14 GB", | 20260507 22:08:39.059 | INFO | "pinned" : " 29 GB" | 20260507 22:08:39.059 | INFO | } | 20260507 22:08:39.059 | INFO | Perf report for the last 5.14024 seconds: | 20260507 22:08:39.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:39.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:08:39.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 22:08:39.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:39.059 | INFO | Uploaded 36GiB to GPU in 2.33207s (15.4369GiB/s) | 20260507 22:08:39.059 | INFO | Downloaded 34GiB to GPU in 2.49823s (13.6096GiB/s) | 20260507 22:08:44.205 | INFO | Analyzing chunk frames = [2968064,2968576) | 20260507 22:08:44.205 | INFO | MemInfo: { | 20260507 22:08:44.205 | INFO | "gpu" : " 17 GB", | 20260507 22:08:44.205 | INFO | "huge" : "", | 20260507 22:08:44.205 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:44.205 | INFO | "malloc" : " 14 GB", | 20260507 22:08:44.205 | INFO | "pinned" : " 29 GB" | 20260507 22:08:44.205 | INFO | } | 20260507 22:08:44.205 | INFO | Perf report for the last 5.14633 seconds: | 20260507 22:08:44.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:44.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.206 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 22:08:44.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.206 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 22:08:44.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:44.206 | INFO | Uploaded 36GiB to GPU in 2.49324s (14.4391GiB/s) | 20260507 22:08:44.206 | INFO | Downloaded 34GiB to GPU in 2.41047s (14.1051GiB/s) | 20260507 22:08:49.347 | INFO | Analyzing chunk frames = [2968576,2969088) | 20260507 22:08:49.347 | INFO | MemInfo: { | 20260507 22:08:49.347 | INFO | "gpu" : " 17 GB", | 20260507 22:08:49.347 | INFO | "huge" : "", | 20260507 22:08:49.347 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:49.347 | INFO | "malloc" : " 14 GB", | 20260507 22:08:49.347 | INFO | "pinned" : " 29 GB" | 20260507 22:08:49.347 | INFO | } | 20260507 22:08:49.347 | INFO | Perf report for the last 5.14134 seconds: | 20260507 22:08:49.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:49.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 22:08:49.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:08:49.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:49.347 | INFO | Uploaded 36GiB to GPU in 2.49778s (14.4128GiB/s) | 20260507 22:08:49.347 | INFO | Downloaded 34GiB to GPU in 2.45197s (13.8664GiB/s) | 20260507 22:08:54.483 | INFO | Analyzing chunk frames = [2969088,2969600) | 20260507 22:08:54.483 | INFO | MemInfo: { | 20260507 22:08:54.483 | INFO | "gpu" : " 17 GB", | 20260507 22:08:54.483 | INFO | "huge" : "", | 20260507 22:08:54.483 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:54.483 | INFO | "malloc" : " 14 GB", | 20260507 22:08:54.483 | INFO | "pinned" : " 29 GB" | 20260507 22:08:54.483 | INFO | } | 20260507 22:08:54.484 | INFO | Perf report for the last 5.1367 seconds: | 20260507 22:08:54.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:54.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:08:54.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:08:54.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:54.484 | INFO | Uploaded 36GiB to GPU in 2.47807s (14.5275GiB/s) | 20260507 22:08:54.484 | INFO | Downloaded 34GiB to GPU in 2.4359s (13.9579GiB/s) | 20260507 22:08:54.597 | INFO | Starting a round of Baz IO | 20260507 22:08:58.087 | INFO | Populating RTMetrics report | 20260507 22:08:58.088 | INFO | RTMetrics report populated | 20260507 22:08:58.088 | INFO | Opening tmp RTMetrics json file | 20260507 22:08:58.337 | INFO | Serializing RTMetrics to json | 20260507 22:08:58.367 | INFO | Serialization done, swapping tmp file now... | 20260507 22:08:58.369 | INFO | RTMetrics done | 20260507 22:08:58.470 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.873 seconds | 20260507 22:08:58.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.29% / 0.01% / 56.71% | 20260507 22:08:58.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49298% / 0% / 5.5631e-05% | 20260507 22:08:58.470 | INFO | Baz IO overall: | 20260507 22:08:58.470 | INFO | BytesWritten : 1352920137382 | 20260507 22:08:58.470 | INFO | NumEvents : 655560689725 | 20260507 22:08:59.619 | INFO | Analyzing chunk frames = [2969600,2970112) | 20260507 22:08:59.619 | INFO | MemInfo: { | 20260507 22:08:59.619 | INFO | "gpu" : " 17 GB", | 20260507 22:08:59.619 | INFO | "huge" : "", | 20260507 22:08:59.619 | INFO | "huge_shared" : " 29 GB", | 20260507 22:08:59.619 | INFO | "malloc" : " 14 GB", | 20260507 22:08:59.619 | INFO | "pinned" : " 29 GB" | 20260507 22:08:59.619 | INFO | } | 20260507 22:08:59.619 | INFO | Perf report for the last 5.13551 seconds: | 20260507 22:08:59.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:08:59.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:59.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:59.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:59.619 | INFO | ANALYSIS : 68.9%, 2.9, 105.5ms, 0 | 20260507 22:08:59.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:08:59.619 | INFO | RT_METRICS : 53.2%, 1.0, 27.9ms, 0 | 20260507 22:08:59.619 | INFO | PRE_HQ : 68.4%, 1.9, 68.0ms, 0 | 20260507 22:08:59.619 | INFO | BAZWRITER : 74.9%, 4.3, 165.2ms, 0 | 20260507 22:08:59.619 | INFO | BAZ_SAMPLING : 5.6%, 1.0, 2.9ms, 0 | 20260507 22:08:59.619 | INFO | Uploaded 36GiB to GPU in 2.54986s (14.1184GiB/s) | 20260507 22:08:59.619 | INFO | Downloaded 38GiB to GPU in 2.70302s (14.0584GiB/s) | 20260507 22:09:04.758 | INFO | Analyzing chunk frames = [2970112,2970624) | 20260507 22:09:04.758 | INFO | MemInfo: { | 20260507 22:09:04.758 | INFO | "gpu" : " 17 GB", | 20260507 22:09:04.758 | INFO | "huge" : "", | 20260507 22:09:04.758 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:04.758 | INFO | "malloc" : " 14 GB", | 20260507 22:09:04.758 | INFO | "pinned" : " 29 GB" | 20260507 22:09:04.758 | INFO | } | 20260507 22:09:04.758 | INFO | Perf report for the last 5.13907 seconds: | 20260507 22:09:04.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:04.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 22:09:04.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:09:04.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:04.758 | INFO | Uploaded 36GiB to GPU in 2.56182s (14.0525GiB/s) | 20260507 22:09:04.758 | INFO | Downloaded 34GiB to GPU in 2.48124s (13.7028GiB/s) | 20260507 22:09:09.897 | INFO | Analyzing chunk frames = [2970624,2971136) | 20260507 22:09:09.897 | INFO | MemInfo: { | 20260507 22:09:09.897 | INFO | "gpu" : " 17 GB", | 20260507 22:09:09.897 | INFO | "huge" : "", | 20260507 22:09:09.897 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:09.897 | INFO | "malloc" : " 14 GB", | 20260507 22:09:09.897 | INFO | "pinned" : " 29 GB" | 20260507 22:09:09.897 | INFO | } | 20260507 22:09:09.897 | INFO | Perf report for the last 5.13895 seconds: | 20260507 22:09:09.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:09.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:09:09.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:09:09.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:09.897 | INFO | Uploaded 36GiB to GPU in 2.49678s (14.4186GiB/s) | 20260507 22:09:09.898 | INFO | Downloaded 34GiB to GPU in 2.45637s (13.8416GiB/s) | 20260507 22:09:15.036 | INFO | Analyzing chunk frames = [2971136,2971648) | 20260507 22:09:15.036 | INFO | MemInfo: { | 20260507 22:09:15.036 | INFO | "gpu" : " 17 GB", | 20260507 22:09:15.036 | INFO | "huge" : "", | 20260507 22:09:15.036 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:15.036 | INFO | "malloc" : " 14 GB", | 20260507 22:09:15.036 | INFO | "pinned" : " 29 GB" | 20260507 22:09:15.036 | INFO | } | 20260507 22:09:15.036 | INFO | Perf report for the last 5.13917 seconds: | 20260507 22:09:15.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:15.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.036 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:09:15.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.036 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:09:15.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:15.037 | INFO | Uploaded 36GiB to GPU in 2.47363s (14.5535GiB/s) | 20260507 22:09:15.037 | INFO | Downloaded 34GiB to GPU in 2.39159s (14.2165GiB/s) | 20260507 22:09:20.178 | INFO | Analyzing chunk frames = [2971648,2972160) | 20260507 22:09:20.178 | INFO | MemInfo: { | 20260507 22:09:20.178 | INFO | "gpu" : " 17 GB", | 20260507 22:09:20.178 | INFO | "huge" : "", | 20260507 22:09:20.178 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:20.178 | INFO | "malloc" : " 14 GB", | 20260507 22:09:20.178 | INFO | "pinned" : " 29 GB" | 20260507 22:09:20.178 | INFO | } | 20260507 22:09:20.178 | INFO | Perf report for the last 5.14166 seconds: | 20260507 22:09:20.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:20.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.178 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:09:20.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.178 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:09:20.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:20.179 | INFO | Uploaded 36GiB to GPU in 2.35291s (15.3002GiB/s) | 20260507 22:09:20.179 | INFO | Downloaded 34GiB to GPU in 2.51186s (13.5358GiB/s) | 20260507 22:09:25.318 | INFO | Analyzing chunk frames = [2972160,2972672) | 20260507 22:09:25.318 | INFO | MemInfo: { | 20260507 22:09:25.318 | INFO | "gpu" : " 17 GB", | 20260507 22:09:25.318 | INFO | "huge" : "", | 20260507 22:09:25.318 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:25.318 | INFO | "malloc" : " 14 GB", | 20260507 22:09:25.318 | INFO | "pinned" : " 29 GB" | 20260507 22:09:25.318 | INFO | } | 20260507 22:09:25.318 | INFO | Perf report for the last 5.13998 seconds: | 20260507 22:09:25.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:25.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.318 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:09:25.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.318 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:09:25.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:25.319 | INFO | Uploaded 36GiB to GPU in 2.50422s (14.3757GiB/s) | 20260507 22:09:25.319 | INFO | Downloaded 34GiB to GPU in 2.43174s (13.9818GiB/s) | 20260507 22:09:30.459 | INFO | Analyzing chunk frames = [2972672,2973184) | 20260507 22:09:30.459 | INFO | MemInfo: { | 20260507 22:09:30.459 | INFO | "gpu" : " 17 GB", | 20260507 22:09:30.459 | INFO | "huge" : "", | 20260507 22:09:30.459 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:30.459 | INFO | "malloc" : " 14 GB", | 20260507 22:09:30.459 | INFO | "pinned" : " 29 GB" | 20260507 22:09:30.459 | INFO | } | 20260507 22:09:30.459 | INFO | Perf report for the last 5.1412 seconds: | 20260507 22:09:30.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:30.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 22:09:30.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:09:30.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:30.459 | INFO | Uploaded 36GiB to GPU in 2.51974s (14.2872GiB/s) | 20260507 22:09:30.460 | INFO | Downloaded 34GiB to GPU in 2.457s (13.838GiB/s) | 20260507 22:09:35.598 | INFO | Analyzing chunk frames = [2973184,2973696) | 20260507 22:09:35.598 | INFO | MemInfo: { | 20260507 22:09:35.598 | INFO | "gpu" : " 17 GB", | 20260507 22:09:35.598 | INFO | "huge" : "", | 20260507 22:09:35.598 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:35.598 | INFO | "malloc" : " 14 GB", | 20260507 22:09:35.598 | INFO | "pinned" : " 29 GB" | 20260507 22:09:35.598 | INFO | } | 20260507 22:09:35.598 | INFO | Perf report for the last 5.13921 seconds: | 20260507 22:09:35.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:35.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.599 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:09:35.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.599 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 22:09:35.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:35.599 | INFO | Uploaded 36GiB to GPU in 2.49717s (14.4163GiB/s) | 20260507 22:09:35.599 | INFO | Downloaded 34GiB to GPU in 2.447s (13.8945GiB/s) | 20260507 22:09:35.599 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:09:35.599 | INFO | Total mem usage: 16.7 GiB | 20260507 22:09:35.599 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:09:35.599 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -6.9%) | 20260507 22:09:35.599 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:09:35.599 | INFO | High Water Breakdown: | 20260507 22:09:35.599 | INFO | 60.8% : 9.48 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:09:35.599 | INFO | 39.1% : 6.1 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:09:35.599 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:09:35.599 | INFO | ------------------------------------------ | 20260507 22:09:35.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:09:35.599 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:09:35.599 | INFO | Total mem usage: 26.5 GiB | 20260507 22:09:35.599 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:09:35.599 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 22:09:35.599 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 22:09:35.599 | INFO | High Water Breakdown: | 20260507 22:09:35.599 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:09:35.599 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:09:35.599 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:09:35.599 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:09:35.599 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:09:35.599 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:09:35.599 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:09:35.599 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:09:35.599 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:09:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:09:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:09:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:09:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:09:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:09:35.600 | INFO | ------------------------------------------ | 20260507 22:09:35.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:09:35.600 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:09:35.600 | INFO | Total mem usage: 27.8 GiB | 20260507 22:09:35.600 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:09:35.600 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:09:35.600 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:09:35.600 | INFO | High Water Breakdown: | 20260507 22:09:35.600 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:09:35.600 | INFO | ------------------------------------------ | 20260507 22:09:35.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:09:35.600 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:09:35.600 | INFO | Total mem usage: 15.7 GiB | 20260507 22:09:35.600 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:09:35.600 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:09:35.600 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:09:35.600 | INFO | High Water Breakdown: | 20260507 22:09:35.600 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:09:35.600 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:09:35.600 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:09:35.600 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:09:35.600 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:09:35.600 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:09:35.600 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:09:35.600 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:09:35.600 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:09:35.600 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:09:35.600 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:09:35.600 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:09:35.600 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:09:35.600 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:09:35.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:09:35.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:09:35.600 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:09:35.600 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:09:35.600 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:09:35.600 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:09:35.600 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:09:35.600 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:09:35.600 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:09:35.600 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:09:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:09:35.600 | INFO | ------------------------------------------ | 20260507 22:09:35.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:09:35.600 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:09:35.600 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:09:35.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:09:35.600 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.42e+01 / 1.45e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.04e-01, 1.15e-02 / 6.34e-03 | 20260507 22:09:35.600 | INFO | PerfReport: 12.16%, Baseline -- 4.02e+00 / 4.48e+00, 6.00e-01 / 2.18e-01 | 20260507 22:09:35.600 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.13e-01 | 20260507 22:09:35.600 | INFO | PerfReport: 15.81%, DME -- 4.18e+01 / 4.52e+01, 3.88e+00 / 1.39e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.84e+00 / 1.65e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.44e+00 / 2.71e+00, 4.31e-01 / 2.39e-01 | 20260507 22:09:35.600 | INFO | PerfReport: 3.48%, Metrics -- 1.15e+00 / 1.37e+00, 2.98e-01 / 1.82e-01 | 20260507 22:09:35.600 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.88e-01, 6.00e-02 / 2.66e-02 | 20260507 22:09:35.600 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:09:35.600 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:09:35.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:09:35.600 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 24.35%, Upload -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 30.97%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 2.53%, LockDownload -- 2.70e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 22:09:35.600 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260507 22:09:35.711 | INFO | Starting a round of Baz IO | 20260507 22:09:39.196 | INFO | Populating RTMetrics report | 20260507 22:09:39.197 | INFO | RTMetrics report populated | 20260507 22:09:39.197 | INFO | Opening tmp RTMetrics json file | 20260507 22:09:39.210 | INFO | Serializing RTMetrics to json | 20260507 22:09:39.225 | INFO | Serialization done, swapping tmp file now... | 20260507 22:09:39.226 | INFO | RTMetrics done | 20260507 22:09:39.298 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.586 seconds | 20260507 22:09:39.298 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.28% / 0.01% / 56.71% | 20260507 22:09:39.298 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49162% / 3.97364e-06% / 7.15256e-05% | 20260507 22:09:39.298 | INFO | Baz IO overall: | 20260507 22:09:39.298 | INFO | BytesWritten : 1354793441757 | 20260507 22:09:39.298 | INFO | NumEvents : 656465836550 | 20260507 22:09:40.735 | INFO | Analyzing chunk frames = [2973696,2974208) | 20260507 22:09:40.735 | INFO | MemInfo: { | 20260507 22:09:40.735 | INFO | "gpu" : " 17 GB", | 20260507 22:09:40.735 | INFO | "huge" : "", | 20260507 22:09:40.735 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:40.735 | INFO | "malloc" : " 14 GB", | 20260507 22:09:40.735 | INFO | "pinned" : " 29 GB" | 20260507 22:09:40.735 | INFO | } | 20260507 22:09:40.735 | INFO | Perf report for the last 5.13678 seconds: | 20260507 22:09:40.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:40.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:40.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:40.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:40.735 | INFO | ANALYSIS : 69.2%, 2.9, 104.7ms, 0 | 20260507 22:09:40.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:40.735 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260507 22:09:40.735 | INFO | PRE_HQ : 68.7%, 1.9, 67.1ms, 0 | 20260507 22:09:40.735 | INFO | BAZWRITER : 69.4%, 4.4, 156.8ms, 0 | 20260507 22:09:40.735 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 22:09:40.736 | INFO | Uploaded 36GiB to GPU in 2.60046s (13.8437GiB/s) | 20260507 22:09:40.736 | INFO | Downloaded 38GiB to GPU in 2.6973s (14.0881GiB/s) | 20260507 22:09:45.874 | INFO | Analyzing chunk frames = [2974208,2974720) | 20260507 22:09:45.874 | INFO | MemInfo: { | 20260507 22:09:45.874 | INFO | "gpu" : " 17 GB", | 20260507 22:09:45.874 | INFO | "huge" : "", | 20260507 22:09:45.874 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:45.874 | INFO | "malloc" : " 14 GB", | 20260507 22:09:45.874 | INFO | "pinned" : " 29 GB" | 20260507 22:09:45.874 | INFO | } | 20260507 22:09:45.874 | INFO | Perf report for the last 5.13861 seconds: | 20260507 22:09:45.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:45.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:09:45.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:09:45.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:45.874 | INFO | Uploaded 36GiB to GPU in 2.57403s (13.9858GiB/s) | 20260507 22:09:45.874 | INFO | Downloaded 34GiB to GPU in 2.48586s (13.6774GiB/s) | 20260507 22:09:51.013 | INFO | Analyzing chunk frames = [2974720,2975232) | 20260507 22:09:51.013 | INFO | MemInfo: { | 20260507 22:09:51.013 | INFO | "gpu" : " 17 GB", | 20260507 22:09:51.013 | INFO | "huge" : "", | 20260507 22:09:51.013 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:51.013 | INFO | "malloc" : " 14 GB", | 20260507 22:09:51.013 | INFO | "pinned" : " 29 GB" | 20260507 22:09:51.013 | INFO | } | 20260507 22:09:51.013 | INFO | Perf report for the last 5.13961 seconds: | 20260507 22:09:51.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:51.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.014 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 22:09:51.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.014 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:09:51.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:51.014 | INFO | Uploaded 36GiB to GPU in 2.55876s (14.0693GiB/s) | 20260507 22:09:51.014 | INFO | Downloaded 34GiB to GPU in 2.44939s (13.881GiB/s) | 20260507 22:09:56.156 | INFO | Analyzing chunk frames = [2975232,2975744) | 20260507 22:09:56.156 | INFO | MemInfo: { | 20260507 22:09:56.156 | INFO | "gpu" : " 17 GB", | 20260507 22:09:56.156 | INFO | "huge" : "", | 20260507 22:09:56.156 | INFO | "huge_shared" : " 29 GB", | 20260507 22:09:56.156 | INFO | "malloc" : " 14 GB", | 20260507 22:09:56.156 | INFO | "pinned" : " 29 GB" | 20260507 22:09:56.156 | INFO | } | 20260507 22:09:56.156 | INFO | Perf report for the last 5.14266 seconds: | 20260507 22:09:56.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:09:56.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:09:56.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:09:56.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:09:56.156 | INFO | Uploaded 36GiB to GPU in 2.49561s (14.4253GiB/s) | 20260507 22:09:56.157 | INFO | Downloaded 34GiB to GPU in 2.37022s (14.3447GiB/s) | 20260507 22:10:01.301 | INFO | Analyzing chunk frames = [2975744,2976256) | 20260507 22:10:01.301 | INFO | MemInfo: { | 20260507 22:10:01.301 | INFO | "gpu" : " 17 GB", | 20260507 22:10:01.301 | INFO | "huge" : "", | 20260507 22:10:01.301 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:01.301 | INFO | "malloc" : " 14 GB", | 20260507 22:10:01.301 | INFO | "pinned" : " 29 GB" | 20260507 22:10:01.301 | INFO | } | 20260507 22:10:01.301 | INFO | Perf report for the last 5.14514 seconds: | 20260507 22:10:01.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:01.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.301 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 22:10:01.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.301 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:10:01.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:01.302 | INFO | Uploaded 36GiB to GPU in 2.4649s (14.6051GiB/s) | 20260507 22:10:01.302 | INFO | Downloaded 34GiB to GPU in 2.49764s (13.6129GiB/s) | 20260507 22:10:06.436 | INFO | Analyzing chunk frames = [2976256,2976768) | 20260507 22:10:06.436 | INFO | MemInfo: { | 20260507 22:10:06.436 | INFO | "gpu" : " 17 GB", | 20260507 22:10:06.436 | INFO | "huge" : "", | 20260507 22:10:06.436 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:06.436 | INFO | "malloc" : " 14 GB", | 20260507 22:10:06.436 | INFO | "pinned" : " 29 GB" | 20260507 22:10:06.436 | INFO | } | 20260507 22:10:06.436 | INFO | Perf report for the last 5.13486 seconds: | 20260507 22:10:06.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:06.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.436 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260507 22:10:06.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.436 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:10:06.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:06.437 | INFO | Uploaded 36GiB to GPU in 2.43778s (14.7675GiB/s) | 20260507 22:10:06.437 | INFO | Downloaded 34GiB to GPU in 2.38327s (14.2661GiB/s) | 20260507 22:10:11.570 | INFO | Analyzing chunk frames = [2976768,2977280) | 20260507 22:10:11.570 | INFO | MemInfo: { | 20260507 22:10:11.570 | INFO | "gpu" : " 17 GB", | 20260507 22:10:11.570 | INFO | "huge" : "", | 20260507 22:10:11.570 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:11.570 | INFO | "malloc" : " 14 GB", | 20260507 22:10:11.571 | INFO | "pinned" : " 29 GB" | 20260507 22:10:11.571 | INFO | } | 20260507 22:10:11.571 | INFO | Perf report for the last 5.13454 seconds: | 20260507 22:10:11.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:11.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 22:10:11.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:10:11.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:11.571 | INFO | Uploaded 36GiB to GPU in 2.52764s (14.2426GiB/s) | 20260507 22:10:11.571 | INFO | Downloaded 34GiB to GPU in 2.4277s (14.005GiB/s) | 20260507 22:10:16.711 | INFO | Analyzing chunk frames = [2977280,2977792) | 20260507 22:10:16.711 | INFO | MemInfo: { | 20260507 22:10:16.711 | INFO | "gpu" : " 17 GB", | 20260507 22:10:16.711 | INFO | "huge" : "", | 20260507 22:10:16.711 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:16.711 | INFO | "malloc" : " 14 GB", | 20260507 22:10:16.711 | INFO | "pinned" : " 29 GB" | 20260507 22:10:16.711 | INFO | } | 20260507 22:10:16.711 | INFO | Perf report for the last 5.14006 seconds: | 20260507 22:10:16.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:16.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 22:10:16.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:10:16.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:16.711 | INFO | Uploaded 36GiB to GPU in 2.52952s (14.2319GiB/s) | 20260507 22:10:16.711 | INFO | Downloaded 34GiB to GPU in 2.43916s (13.9392GiB/s) | 20260507 22:10:16.809 | INFO | Starting a round of Baz IO | 20260507 22:10:20.275 | INFO | Populating RTMetrics report | 20260507 22:10:20.275 | INFO | RTMetrics report populated | 20260507 22:10:20.275 | INFO | Opening tmp RTMetrics json file | 20260507 22:10:20.316 | INFO | Serializing RTMetrics to json | 20260507 22:10:20.385 | INFO | Serialization done, swapping tmp file now... | 20260507 22:10:20.390 | INFO | RTMetrics done | 20260507 22:10:20.409 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.599 seconds | 20260507 22:10:20.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.27% / 0.01% / 56.72% | 20260507 22:10:20.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49258% / 7.94729e-06% / 7.54992e-05% | 20260507 22:10:20.409 | INFO | Baz IO overall: | 20260507 22:10:20.409 | INFO | BytesWritten : 1356667850016 | 20260507 22:10:20.409 | INFO | NumEvents : 657371632405 | 20260507 22:10:21.849 | INFO | Analyzing chunk frames = [2977792,2978304) | 20260507 22:10:21.849 | INFO | MemInfo: { | 20260507 22:10:21.849 | INFO | "gpu" : " 17 GB", | 20260507 22:10:21.849 | INFO | "huge" : "", | 20260507 22:10:21.849 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:21.849 | INFO | "malloc" : " 14 GB", | 20260507 22:10:21.849 | INFO | "pinned" : " 29 GB" | 20260507 22:10:21.849 | INFO | } | 20260507 22:10:21.849 | INFO | Perf report for the last 5.13832 seconds: | 20260507 22:10:21.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:21.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:21.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:21.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:21.849 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 22:10:21.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:21.849 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 22:10:21.849 | INFO | PRE_HQ : 65.7%, 1.7, 57.6ms, 0 | 20260507 22:10:21.849 | INFO | BAZWRITER : 69.6%, 3.9, 140.6ms, 0 | 20260507 22:10:21.849 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 22:10:21.849 | INFO | Uploaded 36GiB to GPU in 2.51093s (14.3373GiB/s) | 20260507 22:10:21.849 | INFO | Downloaded 38GiB to GPU in 2.69524s (14.0989GiB/s) | 20260507 22:10:26.988 | INFO | Analyzing chunk frames = [2978304,2978816) | 20260507 22:10:26.988 | INFO | MemInfo: { | 20260507 22:10:26.988 | INFO | "gpu" : " 17 GB", | 20260507 22:10:26.988 | INFO | "huge" : "", | 20260507 22:10:26.988 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:26.988 | INFO | "malloc" : " 14 GB", | 20260507 22:10:26.988 | INFO | "pinned" : " 29 GB" | 20260507 22:10:26.988 | INFO | } | 20260507 22:10:26.988 | INFO | Perf report for the last 5.13878 seconds: | 20260507 22:10:26.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:26.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:10:26.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:10:26.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:26.988 | INFO | Uploaded 36GiB to GPU in 2.51025s (14.3412GiB/s) | 20260507 22:10:26.988 | INFO | Downloaded 34GiB to GPU in 2.47248s (13.7514GiB/s) | 20260507 22:10:32.124 | INFO | Analyzing chunk frames = [2978816,2979328) | 20260507 22:10:32.124 | INFO | MemInfo: { | 20260507 22:10:32.124 | INFO | "gpu" : " 17 GB", | 20260507 22:10:32.124 | INFO | "huge" : "", | 20260507 22:10:32.124 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:32.124 | INFO | "malloc" : " 14 GB", | 20260507 22:10:32.124 | INFO | "pinned" : " 29 GB" | 20260507 22:10:32.124 | INFO | } | 20260507 22:10:32.124 | INFO | Perf report for the last 5.13646 seconds: | 20260507 22:10:32.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:32.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.124 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:10:32.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.124 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:10:32.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:32.125 | INFO | Uploaded 36GiB to GPU in 2.52405s (14.2628GiB/s) | 20260507 22:10:32.125 | INFO | Downloaded 34GiB to GPU in 2.4768s (13.7274GiB/s) | 20260507 22:10:37.271 | INFO | Analyzing chunk frames = [2979328,2979840) | 20260507 22:10:37.271 | INFO | MemInfo: { | 20260507 22:10:37.271 | INFO | "gpu" : " 17 GB", | 20260507 22:10:37.271 | INFO | "huge" : "", | 20260507 22:10:37.271 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:37.271 | INFO | "malloc" : " 14 GB", | 20260507 22:10:37.271 | INFO | "pinned" : " 29 GB" | 20260507 22:10:37.271 | INFO | } | 20260507 22:10:37.271 | INFO | Perf report for the last 5.14666 seconds: | 20260507 22:10:37.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:37.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.271 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 22:10:37.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.271 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:10:37.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:37.272 | INFO | Uploaded 36GiB to GPU in 2.47684s (14.5346GiB/s) | 20260507 22:10:37.272 | INFO | Downloaded 34GiB to GPU in 2.39201s (14.214GiB/s) | 20260507 22:10:42.410 | INFO | Analyzing chunk frames = [2979840,2980352) | 20260507 22:10:42.410 | INFO | MemInfo: { | 20260507 22:10:42.410 | INFO | "gpu" : " 17 GB", | 20260507 22:10:42.410 | INFO | "huge" : "", | 20260507 22:10:42.410 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:42.410 | INFO | "malloc" : " 14 GB", | 20260507 22:10:42.410 | INFO | "pinned" : " 29 GB" | 20260507 22:10:42.410 | INFO | } | 20260507 22:10:42.410 | INFO | Perf report for the last 5.13899 seconds: | 20260507 22:10:42.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:42.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:10:42.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 22:10:42.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:42.410 | INFO | Uploaded 36GiB to GPU in 2.32174s (15.5056GiB/s) | 20260507 22:10:42.410 | INFO | Downloaded 34GiB to GPU in 2.50441s (13.576GiB/s) | 20260507 22:10:47.548 | INFO | Analyzing chunk frames = [2980352,2980864) | 20260507 22:10:47.548 | INFO | MemInfo: { | 20260507 22:10:47.548 | INFO | "gpu" : " 17 GB", | 20260507 22:10:47.548 | INFO | "huge" : "", | 20260507 22:10:47.548 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:47.549 | INFO | "malloc" : " 14 GB", | 20260507 22:10:47.549 | INFO | "pinned" : " 29 GB" | 20260507 22:10:47.549 | INFO | } | 20260507 22:10:47.549 | INFO | Perf report for the last 5.13853 seconds: | 20260507 22:10:47.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:47.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:10:47.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 22:10:47.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:47.549 | INFO | Uploaded 36GiB to GPU in 2.50188s (14.3892GiB/s) | 20260507 22:10:47.549 | INFO | Downloaded 34GiB to GPU in 2.43337s (13.9724GiB/s) | 20260507 22:10:52.685 | INFO | Analyzing chunk frames = [2980864,2981376) | 20260507 22:10:52.685 | INFO | MemInfo: { | 20260507 22:10:52.685 | INFO | "gpu" : " 17 GB", | 20260507 22:10:52.685 | INFO | "huge" : "", | 20260507 22:10:52.685 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:52.685 | INFO | "malloc" : " 14 GB", | 20260507 22:10:52.685 | INFO | "pinned" : " 29 GB" | 20260507 22:10:52.685 | INFO | } | 20260507 22:10:52.685 | INFO | Perf report for the last 5.13646 seconds: | 20260507 22:10:52.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:52.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.685 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:10:52.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.685 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:10:52.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:52.686 | INFO | Uploaded 36GiB to GPU in 2.51727s (14.3012GiB/s) | 20260507 22:10:52.686 | INFO | Downloaded 34GiB to GPU in 2.46015s (13.8203GiB/s) | 20260507 22:10:57.823 | INFO | Analyzing chunk frames = [2981376,2981888) | 20260507 22:10:57.823 | INFO | MemInfo: { | 20260507 22:10:57.823 | INFO | "gpu" : " 17 GB", | 20260507 22:10:57.823 | INFO | "huge" : "", | 20260507 22:10:57.823 | INFO | "huge_shared" : " 29 GB", | 20260507 22:10:57.823 | INFO | "malloc" : " 14 GB", | 20260507 22:10:57.823 | INFO | "pinned" : " 29 GB" | 20260507 22:10:57.823 | INFO | } | 20260507 22:10:57.823 | INFO | Perf report for the last 5.13818 seconds: | 20260507 22:10:57.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:10:57.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.823 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:10:57.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.823 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 22:10:57.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:10:57.824 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:10:57.824 | INFO | Total mem usage: 16.7 GiB | 20260507 22:10:57.824 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:10:57.824 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -6.82%) | 20260507 22:10:57.824 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:10:57.824 | INFO | High Water Breakdown: | 20260507 22:10:57.824 | INFO | 60.8% : 9.49 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:10:57.824 | INFO | 39.1% : 6.1 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:10:57.824 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:10:57.824 | INFO | ------------------------------------------ | 20260507 22:10:57.824 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:10:57.824 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:10:57.824 | INFO | Total mem usage: 26.5 GiB | 20260507 22:10:57.824 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:10:57.824 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:10:57.824 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 22:10:57.824 | INFO | High Water Breakdown: | 20260507 22:10:57.824 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:10:57.824 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:10:57.824 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:10:57.824 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:10:57.824 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:10:57.824 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:10:57.824 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:10:57.824 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:10:57.824 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:10:57.824 | INFO | Uploaded 36GiB to GPU in 2.48035s (14.5141GiB/s) | 20260507 22:10:57.824 | INFO | Downloaded 34GiB to GPU in 2.43802s (13.9457GiB/s) | 20260507 22:10:57.824 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:10:57.824 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:10:57.824 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:10:57.824 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:10:57.824 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:10:57.824 | INFO | ------------------------------------------ | 20260507 22:10:57.824 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:10:57.824 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:10:57.824 | INFO | Total mem usage: 27.8 GiB | 20260507 22:10:57.824 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:10:57.824 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:10:57.824 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:10:57.824 | INFO | High Water Breakdown: | 20260507 22:10:57.824 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:10:57.824 | INFO | ------------------------------------------ | 20260507 22:10:57.824 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:10:57.824 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:10:57.824 | INFO | Total mem usage: 15.7 GiB | 20260507 22:10:57.824 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:10:57.824 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:10:57.824 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:10:57.824 | INFO | High Water Breakdown: | 20260507 22:10:57.824 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:10:57.824 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:10:57.824 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:10:57.825 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:10:57.825 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:10:57.825 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:10:57.825 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:10:57.825 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:10:57.825 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:10:57.825 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:10:57.825 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:10:57.825 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:10:57.825 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:10:57.825 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:10:57.825 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:10:57.825 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:10:57.825 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:10:57.825 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:10:57.825 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:10:57.825 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:10:57.825 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:10:57.825 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:10:57.825 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:10:57.825 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:10:57.825 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:10:57.825 | INFO | ------------------------------------------ | 20260507 22:10:57.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:10:57.825 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:10:57.825 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:10:57.825 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:10:57.825 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.41e+01 / 1.67e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.22e-02 / 5.20e-03 | 20260507 22:10:57.825 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.48e+00, 6.01e-01 / 2.85e-01 | 20260507 22:10:57.825 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.47e+00, 2.08e-01 / 1.26e-01 | 20260507 22:10:57.825 | INFO | PerfReport: 15.73%, DME -- 4.15e+01 / 4.48e+01, 3.71e+00 / 1.22e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.83e+00 / 1.95e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.44e+00 / 2.69e+00, 4.27e-01 / 2.70e-01 | 20260507 22:10:57.825 | INFO | PerfReport: 3.48%, Metrics -- 1.15e+00 / 1.36e+00, 2.99e-01 / 1.72e-01 | 20260507 22:10:57.825 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.84e-01, 6.00e-02 / 2.52e-02 | 20260507 22:10:57.825 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:10:57.825 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:10:57.825 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:10:57.825 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 24.43%, Upload -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 24.18%, Download -- 2.58e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 30.95%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.14e+00 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 2.35%, LockDownload -- 2.50e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 16.03%, LockCompute -- 1.71e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 22:10:57.825 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 22:10:57.922 | INFO | Starting a round of Baz IO | 20260507 22:11:01.391 | INFO | Populating RTMetrics report | 20260507 22:11:01.391 | INFO | RTMetrics report populated | 20260507 22:11:01.391 | INFO | Opening tmp RTMetrics json file | 20260507 22:11:01.419 | INFO | Serializing RTMetrics to json | 20260507 22:11:01.420 | INFO | Serialization done, swapping tmp file now... | 20260507 22:11:01.421 | INFO | RTMetrics done | 20260507 22:11:01.511 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.589 seconds | 20260507 22:11:01.511 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.27% / 0.01% / 56.73% | 20260507 22:11:01.511 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48422% / 3.97364e-06% / 5.96046e-05% | 20260507 22:11:01.511 | INFO | Baz IO overall: | 20260507 22:11:01.511 | INFO | BytesWritten : 1358540010172 | 20260507 22:11:01.511 | INFO | NumEvents : 658275896900 | 20260507 22:11:02.965 | INFO | Analyzing chunk frames = [2981888,2982400) | 20260507 22:11:02.965 | INFO | MemInfo: { | 20260507 22:11:02.965 | INFO | "gpu" : " 17 GB", | 20260507 22:11:02.965 | INFO | "huge" : "", | 20260507 22:11:02.965 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:02.965 | INFO | "malloc" : " 14 GB", | 20260507 22:11:02.965 | INFO | "pinned" : " 29 GB" | 20260507 22:11:02.965 | INFO | } | 20260507 22:11:02.965 | INFO | Perf report for the last 5.14151 seconds: | 20260507 22:11:02.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:02.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:02.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:02.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:02.965 | INFO | ANALYSIS : 69.0%, 3.0, 107.0ms, 0 | 20260507 22:11:02.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:02.965 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 22:11:02.965 | INFO | PRE_HQ : 65.8%, 1.7, 57.8ms, 0 | 20260507 22:11:02.965 | INFO | BAZWRITER : 69.3%, 3.8, 135.9ms, 0 | 20260507 22:11:02.965 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:11:02.966 | INFO | Uploaded 36GiB to GPU in 2.5668s (14.0252GiB/s) | 20260507 22:11:02.966 | INFO | Downloaded 38GiB to GPU in 2.69261s (14.1127GiB/s) | 20260507 22:11:08.104 | INFO | Analyzing chunk frames = [2982400,2982912) | 20260507 22:11:08.104 | INFO | MemInfo: { | 20260507 22:11:08.104 | INFO | "gpu" : " 17 GB", | 20260507 22:11:08.104 | INFO | "huge" : "", | 20260507 22:11:08.104 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:08.104 | INFO | "malloc" : " 14 GB", | 20260507 22:11:08.104 | INFO | "pinned" : " 29 GB" | 20260507 22:11:08.104 | INFO | } | 20260507 22:11:08.104 | INFO | Perf report for the last 5.13925 seconds: | 20260507 22:11:08.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:08.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 22:11:08.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:11:08.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:08.104 | INFO | Uploaded 36GiB to GPU in 2.54488s (14.146GiB/s) | 20260507 22:11:08.104 | INFO | Downloaded 34GiB to GPU in 2.48533s (13.6803GiB/s) | 20260507 22:11:13.246 | INFO | Analyzing chunk frames = [2982912,2983424) | 20260507 22:11:13.246 | INFO | MemInfo: { | 20260507 22:11:13.246 | INFO | "gpu" : " 17 GB", | 20260507 22:11:13.246 | INFO | "huge" : "", | 20260507 22:11:13.246 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:13.246 | INFO | "malloc" : " 14 GB", | 20260507 22:11:13.246 | INFO | "pinned" : " 29 GB" | 20260507 22:11:13.246 | INFO | } | 20260507 22:11:13.246 | INFO | Perf report for the last 5.14208 seconds: | 20260507 22:11:13.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:13.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:11:13.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:11:13.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:13.246 | INFO | Uploaded 36GiB to GPU in 2.50886s (14.3491GiB/s) | 20260507 22:11:13.247 | INFO | Downloaded 34GiB to GPU in 2.46669s (13.7837GiB/s) | 20260507 22:11:18.383 | INFO | Analyzing chunk frames = [2983424,2983936) | 20260507 22:11:18.383 | INFO | MemInfo: { | 20260507 22:11:18.383 | INFO | "gpu" : " 17 GB", | 20260507 22:11:18.383 | INFO | "huge" : "", | 20260507 22:11:18.383 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:18.383 | INFO | "malloc" : " 14 GB", | 20260507 22:11:18.383 | INFO | "pinned" : " 29 GB" | 20260507 22:11:18.383 | INFO | } | 20260507 22:11:18.383 | INFO | Perf report for the last 5.13717 seconds: | 20260507 22:11:18.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:18.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.383 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:11:18.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.384 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:11:18.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:18.384 | INFO | Uploaded 36GiB to GPU in 2.46754s (14.5895GiB/s) | 20260507 22:11:18.384 | INFO | Downloaded 34GiB to GPU in 2.40234s (14.1529GiB/s) | 20260507 22:11:23.524 | INFO | Analyzing chunk frames = [2983936,2984448) | 20260507 22:11:23.524 | INFO | MemInfo: { | 20260507 22:11:23.524 | INFO | "gpu" : " 17 GB", | 20260507 22:11:23.524 | INFO | "huge" : "", | 20260507 22:11:23.524 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:23.524 | INFO | "malloc" : " 14 GB", | 20260507 22:11:23.524 | INFO | "pinned" : " 29 GB" | 20260507 22:11:23.524 | INFO | } | 20260507 22:11:23.524 | INFO | Perf report for the last 5.14056 seconds: | 20260507 22:11:23.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:23.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 22:11:23.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 22:11:23.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:23.524 | INFO | Uploaded 36GiB to GPU in 2.37781s (15.14GiB/s) | 20260507 22:11:23.525 | INFO | Downloaded 34GiB to GPU in 2.49264s (13.6402GiB/s) | 20260507 22:11:28.665 | INFO | Analyzing chunk frames = [2984448,2984960) | 20260507 22:11:28.665 | INFO | MemInfo: { | 20260507 22:11:28.665 | INFO | "gpu" : " 17 GB", | 20260507 22:11:28.665 | INFO | "huge" : "", | 20260507 22:11:28.665 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:28.665 | INFO | "malloc" : " 14 GB", | 20260507 22:11:28.665 | INFO | "pinned" : " 29 GB" | 20260507 22:11:28.665 | INFO | } | 20260507 22:11:28.665 | INFO | Perf report for the last 5.14118 seconds: | 20260507 22:11:28.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:28.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.665 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:11:28.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.665 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:11:28.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:28.666 | INFO | Uploaded 36GiB to GPU in 2.51405s (14.3195GiB/s) | 20260507 22:11:28.666 | INFO | Downloaded 34GiB to GPU in 2.4299s (13.9924GiB/s) | 20260507 22:11:33.800 | INFO | Analyzing chunk frames = [2984960,2985472) | 20260507 22:11:33.800 | INFO | MemInfo: { | 20260507 22:11:33.800 | INFO | "gpu" : " 17 GB", | 20260507 22:11:33.801 | INFO | "huge" : "", | 20260507 22:11:33.801 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:33.801 | INFO | "malloc" : " 14 GB", | 20260507 22:11:33.801 | INFO | "pinned" : " 29 GB" | 20260507 22:11:33.801 | INFO | } | 20260507 22:11:33.801 | INFO | Perf report for the last 5.13547 seconds: | 20260507 22:11:33.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:33.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:11:33.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 22:11:33.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:33.801 | INFO | Uploaded 36GiB to GPU in 2.53817s (14.1834GiB/s) | 20260507 22:11:33.801 | INFO | Downloaded 34GiB to GPU in 2.46608s (13.7871GiB/s) | 20260507 22:11:38.941 | INFO | Analyzing chunk frames = [2985472,2985984) | 20260507 22:11:38.941 | INFO | MemInfo: { | 20260507 22:11:38.941 | INFO | "gpu" : " 17 GB", | 20260507 22:11:38.941 | INFO | "huge" : "", | 20260507 22:11:38.941 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:38.941 | INFO | "malloc" : " 14 GB", | 20260507 22:11:38.941 | INFO | "pinned" : " 29 GB" | 20260507 22:11:38.941 | INFO | } | 20260507 22:11:38.941 | INFO | Perf report for the last 5.1401 seconds: | 20260507 22:11:38.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:38.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:11:38.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:11:38.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:38.941 | INFO | Uploaded 36GiB to GPU in 2.51098s (14.337GiB/s) | 20260507 22:11:38.941 | INFO | Downloaded 34GiB to GPU in 2.44543s (13.9035GiB/s) | 20260507 22:11:39.038 | INFO | Starting a round of Baz IO | 20260507 22:11:42.495 | INFO | Populating RTMetrics report | 20260507 22:11:42.495 | INFO | RTMetrics report populated | 20260507 22:11:42.495 | INFO | Opening tmp RTMetrics json file | 20260507 22:11:42.559 | INFO | Serializing RTMetrics to json | 20260507 22:11:42.610 | INFO | Serialization done, swapping tmp file now... | 20260507 22:11:42.619 | INFO | RTMetrics done | 20260507 22:11:42.645 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.606 seconds | 20260507 22:11:42.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.26% / 0.01% / 56.73% | 20260507 22:11:42.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48334% / 1.19209e-05% / 4.37101e-05% | 20260507 22:11:42.645 | INFO | Baz IO overall: | 20260507 22:11:42.645 | INFO | BytesWritten : 1360412004555 | 20260507 22:11:42.645 | INFO | NumEvents : 659180037030 | 20260507 22:11:44.075 | INFO | Analyzing chunk frames = [2985984,2986496) | 20260507 22:11:44.075 | INFO | MemInfo: { | 20260507 22:11:44.075 | INFO | "gpu" : " 17 GB", | 20260507 22:11:44.075 | INFO | "huge" : "", | 20260507 22:11:44.075 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:44.075 | INFO | "malloc" : " 14 GB", | 20260507 22:11:44.075 | INFO | "pinned" : " 29 GB" | 20260507 22:11:44.075 | INFO | } | 20260507 22:11:44.075 | INFO | Perf report for the last 5.13443 seconds: | 20260507 22:11:44.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:44.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:44.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:44.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:44.075 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 22:11:44.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:44.075 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 22:11:44.075 | INFO | PRE_HQ : 65.7%, 1.7, 57.8ms, 0 | 20260507 22:11:44.075 | INFO | BAZWRITER : 69.8%, 4.0, 145.1ms, 0 | 20260507 22:11:44.075 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 22:11:44.075 | INFO | Uploaded 36GiB to GPU in 2.55264s (14.103GiB/s) | 20260507 22:11:44.075 | INFO | Downloaded 38GiB to GPU in 2.69129s (14.1196GiB/s) | 20260507 22:11:49.218 | INFO | Analyzing chunk frames = [2986496,2987008) | 20260507 22:11:49.218 | INFO | MemInfo: { | 20260507 22:11:49.218 | INFO | "gpu" : " 17 GB", | 20260507 22:11:49.218 | INFO | "huge" : "", | 20260507 22:11:49.218 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:49.218 | INFO | "malloc" : " 14 GB", | 20260507 22:11:49.218 | INFO | "pinned" : " 29 GB" | 20260507 22:11:49.218 | INFO | } | 20260507 22:11:49.218 | INFO | Perf report for the last 5.14272 seconds: | 20260507 22:11:49.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:49.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:11:49.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:11:49.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:49.218 | INFO | Uploaded 36GiB to GPU in 2.5169s (14.3033GiB/s) | 20260507 22:11:49.218 | INFO | Downloaded 34GiB to GPU in 2.46826s (13.7749GiB/s) | 20260507 22:11:54.363 | INFO | Analyzing chunk frames = [2987008,2987520) | 20260507 22:11:54.363 | INFO | MemInfo: { | 20260507 22:11:54.363 | INFO | "gpu" : " 17 GB", | 20260507 22:11:54.363 | INFO | "huge" : "", | 20260507 22:11:54.363 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:54.363 | INFO | "malloc" : " 14 GB", | 20260507 22:11:54.363 | INFO | "pinned" : " 29 GB" | 20260507 22:11:54.363 | INFO | } | 20260507 22:11:54.363 | INFO | Perf report for the last 5.1453 seconds: | 20260507 22:11:54.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:54.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:11:54.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 22:11:54.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:54.363 | INFO | Uploaded 36GiB to GPU in 2.52454s (14.26GiB/s) | 20260507 22:11:54.364 | INFO | Downloaded 34GiB to GPU in 2.4592s (13.8256GiB/s) | 20260507 22:11:59.503 | INFO | Analyzing chunk frames = [2987520,2988032) | 20260507 22:11:59.503 | INFO | MemInfo: { | 20260507 22:11:59.503 | INFO | "gpu" : " 17 GB", | 20260507 22:11:59.503 | INFO | "huge" : "", | 20260507 22:11:59.503 | INFO | "huge_shared" : " 29 GB", | 20260507 22:11:59.503 | INFO | "malloc" : " 14 GB", | 20260507 22:11:59.503 | INFO | "pinned" : " 29 GB" | 20260507 22:11:59.503 | INFO | } | 20260507 22:11:59.503 | INFO | Perf report for the last 5.13978 seconds: | 20260507 22:11:59.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:11:59.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:11:59.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:11:59.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:11:59.503 | INFO | Uploaded 36GiB to GPU in 2.49901s (14.4057GiB/s) | 20260507 22:11:59.503 | INFO | Downloaded 34GiB to GPU in 2.41246s (14.0935GiB/s) | 20260507 22:12:04.639 | INFO | Analyzing chunk frames = [2988032,2988544) | 20260507 22:12:04.639 | INFO | MemInfo: { | 20260507 22:12:04.639 | INFO | "gpu" : " 17 GB", | 20260507 22:12:04.639 | INFO | "huge" : "", | 20260507 22:12:04.639 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:04.639 | INFO | "malloc" : " 14 GB", | 20260507 22:12:04.639 | INFO | "pinned" : " 29 GB" | 20260507 22:12:04.639 | INFO | } | 20260507 22:12:04.639 | INFO | Perf report for the last 5.13623 seconds: | 20260507 22:12:04.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:04.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.639 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 22:12:04.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.639 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 22:12:04.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:04.640 | INFO | Uploaded 36GiB to GPU in 2.32395s (15.4908GiB/s) | 20260507 22:12:04.640 | INFO | Downloaded 34GiB to GPU in 2.50398s (13.5784GiB/s) | 20260507 22:12:09.779 | INFO | Analyzing chunk frames = [2988544,2989056) | 20260507 22:12:09.779 | INFO | MemInfo: { | 20260507 22:12:09.779 | INFO | "gpu" : " 17 GB", | 20260507 22:12:09.779 | INFO | "huge" : "", | 20260507 22:12:09.780 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:09.780 | INFO | "malloc" : " 14 GB", | 20260507 22:12:09.780 | INFO | "pinned" : " 29 GB" | 20260507 22:12:09.780 | INFO | } | 20260507 22:12:09.780 | INFO | Perf report for the last 5.14049 seconds: | 20260507 22:12:09.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:09.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260507 22:12:09.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:12:09.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:09.780 | INFO | Uploaded 36GiB to GPU in 2.46399s (14.6105GiB/s) | 20260507 22:12:09.780 | INFO | Downloaded 34GiB to GPU in 2.42831s (14.0015GiB/s) | 20260507 22:12:14.916 | INFO | Analyzing chunk frames = [2989056,2989568) | 20260507 22:12:14.916 | INFO | MemInfo: { | 20260507 22:12:14.916 | INFO | "gpu" : " 17 GB", | 20260507 22:12:14.916 | INFO | "huge" : "", | 20260507 22:12:14.916 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:14.916 | INFO | "malloc" : " 14 GB", | 20260507 22:12:14.916 | INFO | "pinned" : " 29 GB" | 20260507 22:12:14.916 | INFO | } | 20260507 22:12:14.916 | INFO | Perf report for the last 5.13653 seconds: | 20260507 22:12:14.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:14.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.916 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:12:14.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.916 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:12:14.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:14.917 | INFO | Uploaded 36GiB to GPU in 2.53702s (14.1899GiB/s) | 20260507 22:12:14.917 | INFO | Downloaded 34GiB to GPU in 2.46098s (13.8157GiB/s) | 20260507 22:12:20.057 | INFO | Analyzing chunk frames = [2989568,2990080) | 20260507 22:12:20.057 | INFO | MemInfo: { | 20260507 22:12:20.057 | INFO | "gpu" : " 17 GB", | 20260507 22:12:20.057 | INFO | "huge" : "", | 20260507 22:12:20.057 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:20.057 | INFO | "malloc" : " 14 GB", | 20260507 22:12:20.057 | INFO | "pinned" : " 29 GB" | 20260507 22:12:20.057 | INFO | } | 20260507 22:12:20.057 | INFO | Perf report for the last 5.14115 seconds: | 20260507 22:12:20.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:20.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.057 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:12:20.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.057 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:12:20.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:20.058 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:12:20.058 | INFO | Uploaded 36GiB to GPU in 2.50032s (14.3981GiB/s) | 20260507 22:12:20.058 | INFO | Total mem usage: 16.7 GiB | 20260507 22:12:20.058 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:12:20.058 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.5%) | 20260507 22:12:20.058 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:12:20.058 | INFO | High Water Breakdown: | 20260507 22:12:20.058 | INFO | 59.6% : 8.68 GiB ( 84.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:12:20.058 | INFO | 40.3% : 5.88 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:12:20.058 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:12:20.058 | INFO | Downloaded 34GiB to GPU in 2.43952s (13.9371GiB/s) | 20260507 22:12:20.058 | INFO | ------------------------------------------ | 20260507 22:12:20.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:12:20.059 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:12:20.059 | INFO | Total mem usage: 26.5 GiB | 20260507 22:12:20.059 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:12:20.059 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:12:20.059 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:12:20.059 | INFO | High Water Breakdown: | 20260507 22:12:20.059 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:12:20.059 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:12:20.059 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:12:20.059 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:12:20.059 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:12:20.059 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:12:20.059 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:12:20.059 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:12:20.059 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:12:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:12:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:12:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:12:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:12:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:12:20.059 | INFO | ------------------------------------------ | 20260507 22:12:20.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:12:20.059 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:12:20.059 | INFO | Total mem usage: 27.8 GiB | 20260507 22:12:20.059 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:12:20.059 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:12:20.059 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:12:20.059 | INFO | High Water Breakdown: | 20260507 22:12:20.059 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:12:20.059 | INFO | ------------------------------------------ | 20260507 22:12:20.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:12:20.059 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:12:20.059 | INFO | Total mem usage: 15.7 GiB | 20260507 22:12:20.059 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:12:20.059 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:12:20.059 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:12:20.059 | INFO | High Water Breakdown: | 20260507 22:12:20.059 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:12:20.059 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:12:20.059 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:12:20.059 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:12:20.059 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:12:20.059 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:12:20.059 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:12:20.059 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:12:20.059 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:12:20.059 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:12:20.059 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:12:20.059 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:12:20.059 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:12:20.059 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:12:20.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:12:20.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:12:20.059 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:12:20.059 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:12:20.059 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:12:20.059 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:12:20.059 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:12:20.059 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:12:20.059 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:12:20.059 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:12:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:12:20.059 | INFO | ------------------------------------------ | 20260507 22:12:20.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:12:20.059 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:12:20.059 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:12:20.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:12:20.059 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.41e+01 / 1.78e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.13e-02 / 4.81e-03 | 20260507 22:12:20.059 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.46e+00, 6.00e-01 / 2.56e-01 | 20260507 22:12:20.059 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.00e-01 | 20260507 22:12:20.059 | INFO | PerfReport: 15.72%, DME -- 4.15e+01 / 4.47e+01, 3.62e+00 / 1.53e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.85e+00 / 2.32e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.43e+00 / 2.73e+00, 4.26e-01 / 2.49e-01 | 20260507 22:12:20.059 | INFO | PerfReport: 3.48%, Metrics -- 1.15e+00 / 1.34e+00, 2.93e-01 / 1.75e-01 | 20260507 22:12:20.059 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.86e-01, 5.96e-02 / 3.33e-02 | 20260507 22:12:20.059 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:12:20.059 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:12:20.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:12:20.059 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 24.37%, Upload -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 30.92%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 1.86%, LockUpload -- 1.98e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 2.49%, LockDownload -- 2.66e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 16.10%, LockCompute -- 1.72e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 22:12:20.059 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.89e-02 / 0.00e+00 | 20260507 22:12:20.156 | INFO | Starting a round of Baz IO | 20260507 22:12:23.623 | INFO | Populating RTMetrics report | 20260507 22:12:23.623 | INFO | RTMetrics report populated | 20260507 22:12:23.623 | INFO | Opening tmp RTMetrics json file | 20260507 22:12:23.644 | INFO | Serializing RTMetrics to json | 20260507 22:12:23.646 | INFO | Serialization done, swapping tmp file now... | 20260507 22:12:23.646 | INFO | RTMetrics done | 20260507 22:12:23.718 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.562 seconds | 20260507 22:12:23.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.25% / 0.01% / 56.74% | 20260507 22:12:23.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48121% / 7.94729e-06% / 5.5631e-05% | 20260507 22:12:23.718 | INFO | Baz IO overall: | 20260507 22:12:23.718 | INFO | BytesWritten : 1362284645744 | 20260507 22:12:23.718 | INFO | NumEvents : 660084865088 | 20260507 22:12:25.193 | INFO | Analyzing chunk frames = [2990080,2990592) | 20260507 22:12:25.193 | INFO | MemInfo: { | 20260507 22:12:25.193 | INFO | "gpu" : " 17 GB", | 20260507 22:12:25.193 | INFO | "huge" : "", | 20260507 22:12:25.193 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:25.193 | INFO | "malloc" : " 14 GB", | 20260507 22:12:25.193 | INFO | "pinned" : " 29 GB" | 20260507 22:12:25.193 | INFO | } | 20260507 22:12:25.193 | INFO | Perf report for the last 5.13575 seconds: | 20260507 22:12:25.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:25.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:25.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:25.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:25.193 | INFO | ANALYSIS : 68.7%, 3.0, 106.9ms, 0 | 20260507 22:12:25.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:25.193 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 22:12:25.193 | INFO | PRE_HQ : 65.5%, 1.7, 57.5ms, 0 | 20260507 22:12:25.193 | INFO | BAZWRITER : 68.9%, 3.6, 125.8ms, 0 | 20260507 22:12:25.193 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:12:25.194 | INFO | Uploaded 36GiB to GPU in 2.55941s (14.0658GiB/s) | 20260507 22:12:25.194 | INFO | Downloaded 38GiB to GPU in 2.69066s (14.1229GiB/s) | 20260507 22:12:30.331 | INFO | Analyzing chunk frames = [2990592,2991104) | 20260507 22:12:30.331 | INFO | MemInfo: { | 20260507 22:12:30.331 | INFO | "gpu" : " 17 GB", | 20260507 22:12:30.331 | INFO | "huge" : "", | 20260507 22:12:30.331 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:30.332 | INFO | "malloc" : " 14 GB", | 20260507 22:12:30.332 | INFO | "pinned" : " 29 GB" | 20260507 22:12:30.332 | INFO | } | 20260507 22:12:30.332 | INFO | Perf report for the last 5.1385 seconds: | 20260507 22:12:30.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:30.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 22:12:30.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 22:12:30.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:30.332 | INFO | Uploaded 36GiB to GPU in 2.51489s (14.3147GiB/s) | 20260507 22:12:30.332 | INFO | Downloaded 34GiB to GPU in 2.4789s (13.7158GiB/s) | 20260507 22:12:35.475 | INFO | Analyzing chunk frames = [2991104,2991616) | 20260507 22:12:35.475 | INFO | MemInfo: { | 20260507 22:12:35.475 | INFO | "gpu" : " 17 GB", | 20260507 22:12:35.475 | INFO | "huge" : "", | 20260507 22:12:35.475 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:35.475 | INFO | "malloc" : " 14 GB", | 20260507 22:12:35.475 | INFO | "pinned" : " 29 GB" | 20260507 22:12:35.475 | INFO | } | 20260507 22:12:35.476 | INFO | Perf report for the last 5.14383 seconds: | 20260507 22:12:35.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:35.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 22:12:35.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:12:35.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:35.476 | INFO | Uploaded 36GiB to GPU in 2.52379s (14.2642GiB/s) | 20260507 22:12:35.476 | INFO | Downloaded 34GiB to GPU in 2.46349s (13.8016GiB/s) | 20260507 22:12:40.617 | INFO | Analyzing chunk frames = [2991616,2992128) | 20260507 22:12:40.617 | INFO | MemInfo: { | 20260507 22:12:40.617 | INFO | "gpu" : " 17 GB", | 20260507 22:12:40.617 | INFO | "huge" : "", | 20260507 22:12:40.617 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:40.617 | INFO | "malloc" : " 14 GB", | 20260507 22:12:40.617 | INFO | "pinned" : " 29 GB" | 20260507 22:12:40.617 | INFO | } | 20260507 22:12:40.617 | INFO | Perf report for the last 5.14184 seconds: | 20260507 22:12:40.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:40.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.617 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:12:40.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.618 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:12:40.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:40.618 | INFO | Uploaded 36GiB to GPU in 2.45513s (14.6632GiB/s) | 20260507 22:12:40.618 | INFO | Downloaded 34GiB to GPU in 2.3909s (14.2206GiB/s) | 20260507 22:12:45.751 | INFO | Analyzing chunk frames = [2992128,2992640) | 20260507 22:12:45.751 | INFO | MemInfo: { | 20260507 22:12:45.751 | INFO | "gpu" : " 17 GB", | 20260507 22:12:45.751 | INFO | "huge" : "", | 20260507 22:12:45.751 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:45.751 | INFO | "malloc" : " 14 GB", | 20260507 22:12:45.751 | INFO | "pinned" : " 29 GB" | 20260507 22:12:45.751 | INFO | } | 20260507 22:12:45.751 | INFO | Perf report for the last 5.13342 seconds: | 20260507 22:12:45.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:45.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 22:12:45.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:12:45.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:45.751 | INFO | Uploaded 36GiB to GPU in 2.35467s (15.2888GiB/s) | 20260507 22:12:45.751 | INFO | Downloaded 34GiB to GPU in 2.51057s (13.5427GiB/s) | 20260507 22:12:50.891 | INFO | Analyzing chunk frames = [2992640,2993152) | 20260507 22:12:50.891 | INFO | MemInfo: { | 20260507 22:12:50.891 | INFO | "gpu" : " 17 GB", | 20260507 22:12:50.891 | INFO | "huge" : "", | 20260507 22:12:50.891 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:50.891 | INFO | "malloc" : " 14 GB", | 20260507 22:12:50.891 | INFO | "pinned" : " 29 GB" | 20260507 22:12:50.891 | INFO | } | 20260507 22:12:50.891 | INFO | Perf report for the last 5.14022 seconds: | 20260507 22:12:50.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:50.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 22:12:50.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:12:50.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:50.891 | INFO | Uploaded 36GiB to GPU in 2.46693s (14.593GiB/s) | 20260507 22:12:50.891 | INFO | Downloaded 34GiB to GPU in 2.4196s (14.0519GiB/s) | 20260507 22:12:56.033 | INFO | Analyzing chunk frames = [2993152,2993664) | 20260507 22:12:56.033 | INFO | MemInfo: { | 20260507 22:12:56.033 | INFO | "gpu" : " 17 GB", | 20260507 22:12:56.033 | INFO | "huge" : "", | 20260507 22:12:56.033 | INFO | "huge_shared" : " 29 GB", | 20260507 22:12:56.033 | INFO | "malloc" : " 14 GB", | 20260507 22:12:56.033 | INFO | "pinned" : " 29 GB" | 20260507 22:12:56.033 | INFO | } | 20260507 22:12:56.033 | INFO | Perf report for the last 5.14177 seconds: | 20260507 22:12:56.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:12:56.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 22:12:56.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:12:56.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:12:56.033 | INFO | Uploaded 36GiB to GPU in 2.5211s (14.2795GiB/s) | 20260507 22:12:56.033 | INFO | Downloaded 34GiB to GPU in 2.45844s (13.8299GiB/s) | 20260507 22:13:01.172 | INFO | Analyzing chunk frames = [2993664,2994176) | 20260507 22:13:01.172 | INFO | MemInfo: { | 20260507 22:13:01.172 | INFO | "gpu" : " 17 GB", | 20260507 22:13:01.172 | INFO | "huge" : "", | 20260507 22:13:01.172 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:01.172 | INFO | "malloc" : " 14 GB", | 20260507 22:13:01.172 | INFO | "pinned" : " 29 GB" | 20260507 22:13:01.172 | INFO | } | 20260507 22:13:01.172 | INFO | Perf report for the last 5.13925 seconds: | 20260507 22:13:01.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:01.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.172 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:13:01.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.172 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:13:01.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:01.173 | INFO | Uploaded 36GiB to GPU in 2.48979s (14.4591GiB/s) | 20260507 22:13:01.173 | INFO | Downloaded 34GiB to GPU in 2.44351s (13.9144GiB/s) | 20260507 22:13:01.280 | INFO | Starting a round of Baz IO | 20260507 22:13:04.747 | INFO | Populating RTMetrics report | 20260507 22:13:04.748 | INFO | RTMetrics report populated | 20260507 22:13:04.748 | INFO | Opening tmp RTMetrics json file | 20260507 22:13:04.753 | INFO | Serializing RTMetrics to json | 20260507 22:13:04.774 | INFO | Serialization done, swapping tmp file now... | 20260507 22:13:04.775 | INFO | RTMetrics done | 20260507 22:13:04.855 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.575 seconds | 20260507 22:13:04.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.25% / 0.01% / 56.75% | 20260507 22:13:04.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47937% / 1.98682e-05% / 5.16574e-05% | 20260507 22:13:04.855 | INFO | Baz IO overall: | 20260507 22:13:04.855 | INFO | BytesWritten : 1364158057583 | 20260507 22:13:04.855 | INFO | NumEvents : 660990017582 | 20260507 22:13:06.313 | INFO | Analyzing chunk frames = [2994176,2994688) | 20260507 22:13:06.313 | INFO | MemInfo: { | 20260507 22:13:06.313 | INFO | "gpu" : " 17 GB", | 20260507 22:13:06.313 | INFO | "huge" : "", | 20260507 22:13:06.313 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:06.313 | INFO | "malloc" : " 14 GB", | 20260507 22:13:06.313 | INFO | "pinned" : " 29 GB" | 20260507 22:13:06.313 | INFO | } | 20260507 22:13:06.313 | INFO | Perf report for the last 5.14134 seconds: | 20260507 22:13:06.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:06.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:06.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:06.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:06.314 | INFO | ANALYSIS : 68.8%, 2.9, 105.0ms, 0 | 20260507 22:13:06.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:06.314 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260507 22:13:06.314 | INFO | PRE_HQ : 67.9%, 1.9, 67.1ms, 0 | 20260507 22:13:06.314 | INFO | BAZWRITER : 69.0%, 4.1, 144.8ms, 0 | 20260507 22:13:06.314 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 22:13:06.314 | INFO | Uploaded 36GiB to GPU in 2.5812s (13.947GiB/s) | 20260507 22:13:06.314 | INFO | Downloaded 38GiB to GPU in 2.69368s (14.1071GiB/s) | 20260507 22:13:11.453 | INFO | Analyzing chunk frames = [2994688,2995200) | 20260507 22:13:11.453 | INFO | MemInfo: { | 20260507 22:13:11.453 | INFO | "gpu" : " 17 GB", | 20260507 22:13:11.453 | INFO | "huge" : "", | 20260507 22:13:11.453 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:11.453 | INFO | "malloc" : " 14 GB", | 20260507 22:13:11.453 | INFO | "pinned" : " 29 GB" | 20260507 22:13:11.453 | INFO | } | 20260507 22:13:11.453 | INFO | Perf report for the last 5.13983 seconds: | 20260507 22:13:11.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:11.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.453 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:13:11.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.454 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260507 22:13:11.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:11.454 | INFO | Uploaded 36GiB to GPU in 2.57732s (13.968GiB/s) | 20260507 22:13:11.454 | INFO | Downloaded 34GiB to GPU in 2.50844s (13.5543GiB/s) | 20260507 22:13:16.589 | INFO | Analyzing chunk frames = [2995200,2995712) | 20260507 22:13:16.589 | INFO | MemInfo: { | 20260507 22:13:16.590 | INFO | "gpu" : " 17 GB", | 20260507 22:13:16.590 | INFO | "huge" : "", | 20260507 22:13:16.590 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:16.590 | INFO | "malloc" : " 14 GB", | 20260507 22:13:16.590 | INFO | "pinned" : " 29 GB" | 20260507 22:13:16.590 | INFO | } | 20260507 22:13:16.590 | INFO | Perf report for the last 5.13642 seconds: | 20260507 22:13:16.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:16.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260507 22:13:16.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 22:13:16.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:16.590 | INFO | Uploaded 36GiB to GPU in 2.54151s (14.1648GiB/s) | 20260507 22:13:16.590 | INFO | Downloaded 34GiB to GPU in 2.48057s (13.7065GiB/s) | 20260507 22:13:21.730 | INFO | Analyzing chunk frames = [2995712,2996224) | 20260507 22:13:21.730 | INFO | MemInfo: { | 20260507 22:13:21.730 | INFO | "gpu" : " 17 GB", | 20260507 22:13:21.730 | INFO | "huge" : "", | 20260507 22:13:21.730 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:21.730 | INFO | "malloc" : " 14 GB", | 20260507 22:13:21.730 | INFO | "pinned" : " 29 GB" | 20260507 22:13:21.731 | INFO | } | 20260507 22:13:21.731 | INFO | Perf report for the last 5.14092 seconds: | 20260507 22:13:21.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:21.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.731 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 22:13:21.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.731 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 22:13:21.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:21.732 | INFO | Uploaded 36GiB to GPU in 2.48766s (14.4714GiB/s) | 20260507 22:13:21.732 | INFO | Downloaded 34GiB to GPU in 2.40358s (14.1456GiB/s) | 20260507 22:13:26.866 | INFO | Analyzing chunk frames = [2996224,2996736) | 20260507 22:13:26.866 | INFO | MemInfo: { | 20260507 22:13:26.866 | INFO | "gpu" : " 17 GB", | 20260507 22:13:26.866 | INFO | "huge" : "", | 20260507 22:13:26.866 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:26.866 | INFO | "malloc" : " 14 GB", | 20260507 22:13:26.866 | INFO | "pinned" : " 29 GB" | 20260507 22:13:26.866 | INFO | } | 20260507 22:13:26.866 | INFO | Perf report for the last 5.13554 seconds: | 20260507 22:13:26.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:26.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:13:26.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:13:26.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:26.866 | INFO | Uploaded 36GiB to GPU in 2.31368s (15.5596GiB/s) | 20260507 22:13:26.867 | INFO | Downloaded 34GiB to GPU in 2.47389s (13.7435GiB/s) | 20260507 22:13:32.013 | INFO | Analyzing chunk frames = [2996736,2997248) | 20260507 22:13:32.013 | INFO | MemInfo: { | 20260507 22:13:32.013 | INFO | "gpu" : " 17 GB", | 20260507 22:13:32.013 | INFO | "huge" : "", | 20260507 22:13:32.014 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:32.014 | INFO | "malloc" : " 14 GB", | 20260507 22:13:32.014 | INFO | "pinned" : " 29 GB" | 20260507 22:13:32.014 | INFO | } | 20260507 22:13:32.014 | INFO | Perf report for the last 5.14747 seconds: | 20260507 22:13:32.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:32.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 22:13:32.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 22:13:32.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:32.014 | INFO | Uploaded 36GiB to GPU in 2.50504s (14.371GiB/s) | 20260507 22:13:32.014 | INFO | Downloaded 34GiB to GPU in 2.43598s (13.9574GiB/s) | 20260507 22:13:37.141 | INFO | Analyzing chunk frames = [2997248,2997760) | 20260507 22:13:37.141 | INFO | MemInfo: { | 20260507 22:13:37.141 | INFO | "gpu" : " 17 GB", | 20260507 22:13:37.141 | INFO | "huge" : "", | 20260507 22:13:37.141 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:37.141 | INFO | "malloc" : " 14 GB", | 20260507 22:13:37.141 | INFO | "pinned" : " 29 GB" | 20260507 22:13:37.141 | INFO | } | 20260507 22:13:37.141 | INFO | Perf report for the last 5.12687 seconds: | 20260507 22:13:37.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:37.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 22:13:37.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:13:37.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:37.141 | INFO | Uploaded 36GiB to GPU in 2.5237s (14.2648GiB/s) | 20260507 22:13:37.141 | INFO | Downloaded 34GiB to GPU in 2.45708s (13.8376GiB/s) | 20260507 22:13:42.278 | INFO | Analyzing chunk frames = [2997760,2998272) | 20260507 22:13:42.278 | INFO | MemInfo: { | 20260507 22:13:42.278 | INFO | "gpu" : " 17 GB", | 20260507 22:13:42.278 | INFO | "huge" : "", | 20260507 22:13:42.278 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:42.279 | INFO | "malloc" : " 14 GB", | 20260507 22:13:42.279 | INFO | "pinned" : " 29 GB" | 20260507 22:13:42.279 | INFO | } | 20260507 22:13:42.279 | INFO | Perf report for the last 5.13797 seconds: | 20260507 22:13:42.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:42.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:13:42.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:13:42.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:42.279 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:13:42.279 | INFO | Total mem usage: 16.7 GiB | 20260507 22:13:42.279 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:13:42.279 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.2%) | 20260507 22:13:42.279 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:13:42.279 | INFO | High Water Breakdown: | 20260507 22:13:42.279 | INFO | 59.8% : 8.82 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:13:42.279 | INFO | 40% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:13:42.279 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:13:42.279 | INFO | ------------------------------------------ | 20260507 22:13:42.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:13:42.279 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:13:42.279 | INFO | Total mem usage: 26.5 GiB | 20260507 22:13:42.279 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:13:42.279 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 22:13:42.279 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 22:13:42.279 | INFO | High Water Breakdown: | 20260507 22:13:42.279 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:13:42.279 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:13:42.279 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:13:42.279 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:13:42.279 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:13:42.279 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:13:42.279 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:13:42.279 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:13:42.279 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:13:42.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:13:42.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:13:42.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:13:42.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:13:42.279 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:13:42.279 | INFO | ------------------------------------------ | 20260507 22:13:42.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:13:42.279 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:13:42.279 | INFO | Total mem usage: 27.8 GiB | 20260507 22:13:42.279 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:13:42.279 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:13:42.279 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:13:42.279 | INFO | High Water Breakdown: | 20260507 22:13:42.279 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:13:42.279 | INFO | ------------------------------------------ | 20260507 22:13:42.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:13:42.279 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:13:42.279 | INFO | Total mem usage: 15.7 GiB | 20260507 22:13:42.279 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:13:42.279 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:13:42.279 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:13:42.279 | INFO | High Water Breakdown: | 20260507 22:13:42.279 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:13:42.279 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:13:42.279 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:13:42.279 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:13:42.279 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:13:42.279 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:13:42.279 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:13:42.279 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:13:42.279 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:13:42.279 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:13:42.279 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:13:42.279 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:13:42.279 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:13:42.279 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:13:42.279 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:13:42.279 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:13:42.279 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:13:42.279 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:13:42.279 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:13:42.279 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:13:42.279 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:13:42.279 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:13:42.279 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:13:42.279 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:13:42.279 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:13:42.279 | INFO | ------------------------------------------ | 20260507 22:13:42.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:13:42.279 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:13:42.279 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:13:42.279 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:13:42.279 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.58e+01, 1.40e+01 / 1.42e+00 | 20260507 22:13:42.279 | INFO | PerfReport: 0.30%, AggregateStats -- 9.99e-02 / 1.04e-01, 1.31e-02 / 5.64e-03 | 20260507 22:13:42.279 | INFO | PerfReport: 12.17%, Baseline -- 4.01e+00 / 4.50e+00, 6.00e-01 / 2.14e-01 | 20260507 22:13:42.279 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.51e+00, 2.07e-01 / 9.68e-02 | 20260507 22:13:42.279 | INFO | PerfReport: 15.86%, DME -- 4.18e+01 / 4.54e+01, 3.75e+00 / 1.02e+00 | 20260507 22:13:42.279 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.72e+00 / 1.74e+00 | 20260507 22:13:42.279 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.47e+00 / 2.80e+00, 4.34e-01 / 2.06e-01 | 20260507 22:13:42.279 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.36e+00, 2.90e-01 / 1.42e-01 | 20260507 22:13:42.279 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.60e-01 / 3.96e-01, 6.05e-02 / 2.58e-02 | 20260507 22:13:42.279 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:13:42.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:13:42.279 | INFO | Uploaded 36GiB to GPU in 2.5065s (14.3627GiB/s) | 20260507 22:13:42.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:13:42.280 | INFO | Downloaded 34GiB to GPU in 2.44765s (13.8909GiB/s) | 20260507 22:13:42.280 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 24.36%, Upload -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 30.87%, Compute -- 3.29e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 2.02%, LockUpload -- 2.15e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 2.54%, LockDownload -- 2.71e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 15.93%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:13:42.280 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 4.04e-02 / 0.00e+00 | 20260507 22:13:42.377 | INFO | Starting a round of Baz IO | 20260507 22:13:45.845 | INFO | Populating RTMetrics report | 20260507 22:13:45.846 | INFO | RTMetrics report populated | 20260507 22:13:45.846 | INFO | Opening tmp RTMetrics json file | 20260507 22:13:46.062 | INFO | Serializing RTMetrics to json | 20260507 22:13:46.070 | INFO | Serialization done, swapping tmp file now... | 20260507 22:13:46.071 | INFO | RTMetrics done | 20260507 22:13:46.163 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.785 seconds | 20260507 22:13:46.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.24% / 0.01% / 56.75% | 20260507 22:13:46.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47517% / 3.97364e-06% / 8.74201e-05% | 20260507 22:13:46.163 | INFO | Baz IO overall: | 20260507 22:13:46.163 | INFO | BytesWritten : 1366030407539 | 20260507 22:13:46.163 | INFO | NumEvents : 661894536354 | 20260507 22:13:47.426 | INFO | Analyzing chunk frames = [2998272,2998784) | 20260507 22:13:47.426 | INFO | MemInfo: { | 20260507 22:13:47.426 | INFO | "gpu" : " 17 GB", | 20260507 22:13:47.426 | INFO | "huge" : "", | 20260507 22:13:47.426 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:47.426 | INFO | "malloc" : " 14 GB", | 20260507 22:13:47.426 | INFO | "pinned" : " 29 GB" | 20260507 22:13:47.426 | INFO | } | 20260507 22:13:47.426 | INFO | Perf report for the last 5.14767 seconds: | 20260507 22:13:47.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:47.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:47.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:47.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:47.426 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 22:13:47.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:47.426 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 22:13:47.426 | INFO | PRE_HQ : 65.4%, 1.7, 57.9ms, 0 | 20260507 22:13:47.426 | INFO | BAZWRITER : 73.1%, 3.8, 144.9ms, 0 | 20260507 22:13:47.426 | INFO | BAZ_SAMPLING : 4.3%, 1.0, 2.2ms, 0 | 20260507 22:13:47.427 | INFO | Uploaded 36GiB to GPU in 2.5276s (14.2428GiB/s) | 20260507 22:13:47.427 | INFO | Downloaded 38GiB to GPU in 2.68037s (14.1771GiB/s) | 20260507 22:13:52.568 | INFO | Analyzing chunk frames = [2998784,2999296) | 20260507 22:13:52.568 | INFO | MemInfo: { | 20260507 22:13:52.568 | INFO | "gpu" : " 17 GB", | 20260507 22:13:52.568 | INFO | "huge" : "", | 20260507 22:13:52.568 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:52.568 | INFO | "malloc" : " 14 GB", | 20260507 22:13:52.568 | INFO | "pinned" : " 29 GB" | 20260507 22:13:52.568 | INFO | } | 20260507 22:13:52.568 | INFO | Perf report for the last 5.14181 seconds: | 20260507 22:13:52.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:52.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:13:52.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:13:52.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:52.568 | INFO | Uploaded 36GiB to GPU in 2.55357s (14.0979GiB/s) | 20260507 22:13:52.568 | INFO | Downloaded 34GiB to GPU in 2.44804s (13.8886GiB/s) | 20260507 22:13:57.703 | INFO | Analyzing chunk frames = [2999296,2999808) | 20260507 22:13:57.703 | INFO | MemInfo: { | 20260507 22:13:57.703 | INFO | "gpu" : " 17 GB", | 20260507 22:13:57.703 | INFO | "huge" : "", | 20260507 22:13:57.703 | INFO | "huge_shared" : " 29 GB", | 20260507 22:13:57.703 | INFO | "malloc" : " 14 GB", | 20260507 22:13:57.703 | INFO | "pinned" : " 29 GB" | 20260507 22:13:57.703 | INFO | } | 20260507 22:13:57.703 | INFO | Perf report for the last 5.13491 seconds: | 20260507 22:13:57.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:13:57.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 22:13:57.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:13:57.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:13:57.703 | INFO | Uploaded 36GiB to GPU in 2.55825s (14.0721GiB/s) | 20260507 22:13:57.704 | INFO | Downloaded 34GiB to GPU in 2.4526s (13.8628GiB/s) | 20260507 22:14:02.842 | INFO | Analyzing chunk frames = [2999808,3000320) | 20260507 22:14:02.842 | INFO | MemInfo: { | 20260507 22:14:02.842 | INFO | "gpu" : " 17 GB", | 20260507 22:14:02.842 | INFO | "huge" : "", | 20260507 22:14:02.842 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:02.842 | INFO | "malloc" : " 14 GB", | 20260507 22:14:02.842 | INFO | "pinned" : " 29 GB" | 20260507 22:14:02.842 | INFO | } | 20260507 22:14:02.842 | INFO | Perf report for the last 5.13939 seconds: | 20260507 22:14:02.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:02.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.843 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 22:14:02.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.843 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 22:14:02.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:02.843 | INFO | Uploaded 36GiB to GPU in 2.50728s (14.3582GiB/s) | 20260507 22:14:02.843 | INFO | Downloaded 34GiB to GPU in 2.39697s (14.1846GiB/s) | 20260507 22:14:07.984 | INFO | Analyzing chunk frames = [3000320,3000832) | 20260507 22:14:07.984 | INFO | MemInfo: { | 20260507 22:14:07.984 | INFO | "gpu" : " 17 GB", | 20260507 22:14:07.984 | INFO | "huge" : "", | 20260507 22:14:07.984 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:07.984 | INFO | "malloc" : " 14 GB", | 20260507 22:14:07.984 | INFO | "pinned" : " 29 GB" | 20260507 22:14:07.984 | INFO | } | 20260507 22:14:07.984 | INFO | Perf report for the last 5.14133 seconds: | 20260507 22:14:07.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:07.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:14:07.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:14:07.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:07.984 | INFO | Uploaded 36GiB to GPU in 2.38102s (15.1196GiB/s) | 20260507 22:14:07.984 | INFO | Downloaded 34GiB to GPU in 2.49802s (13.6108GiB/s) | 20260507 22:14:13.122 | INFO | Analyzing chunk frames = [3000832,3001344) | 20260507 22:14:13.122 | INFO | MemInfo: { | 20260507 22:14:13.122 | INFO | "gpu" : " 17 GB", | 20260507 22:14:13.122 | INFO | "huge" : "", | 20260507 22:14:13.122 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:13.122 | INFO | "malloc" : " 14 GB", | 20260507 22:14:13.122 | INFO | "pinned" : " 29 GB" | 20260507 22:14:13.122 | INFO | } | 20260507 22:14:13.123 | INFO | Perf report for the last 5.13875 seconds: | 20260507 22:14:13.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:13.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:14:13.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:14:13.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:13.123 | INFO | Uploaded 36GiB to GPU in 2.53538s (14.1991GiB/s) | 20260507 22:14:13.124 | INFO | Downloaded 34GiB to GPU in 2.41182s (14.0972GiB/s) | 20260507 22:14:18.262 | INFO | Analyzing chunk frames = [3001344,3001856) | 20260507 22:14:18.262 | INFO | MemInfo: { | 20260507 22:14:18.262 | INFO | "gpu" : " 17 GB", | 20260507 22:14:18.262 | INFO | "huge" : "", | 20260507 22:14:18.262 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:18.262 | INFO | "malloc" : " 14 GB", | 20260507 22:14:18.262 | INFO | "pinned" : " 29 GB" | 20260507 22:14:18.262 | INFO | } | 20260507 22:14:18.262 | INFO | Perf report for the last 5.13992 seconds: | 20260507 22:14:18.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:18.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.263 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 22:14:18.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.263 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:14:18.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:18.263 | INFO | Uploaded 36GiB to GPU in 2.55208s (14.1061GiB/s) | 20260507 22:14:18.263 | INFO | Downloaded 34GiB to GPU in 2.44263s (13.9194GiB/s) | 20260507 22:14:23.402 | INFO | Analyzing chunk frames = [3001856,3002368) | 20260507 22:14:23.402 | INFO | MemInfo: { | 20260507 22:14:23.402 | INFO | "gpu" : " 17 GB", | 20260507 22:14:23.402 | INFO | "huge" : "", | 20260507 22:14:23.402 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:23.402 | INFO | "malloc" : " 14 GB", | 20260507 22:14:23.403 | INFO | "pinned" : " 29 GB" | 20260507 22:14:23.403 | INFO | } | 20260507 22:14:23.403 | INFO | Perf report for the last 5.14007 seconds: | 20260507 22:14:23.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:23.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:14:23.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 22:14:23.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:23.403 | INFO | Uploaded 36GiB to GPU in 2.52598s (14.2519GiB/s) | 20260507 22:14:23.403 | INFO | Downloaded 34GiB to GPU in 2.43783s (13.9469GiB/s) | 20260507 22:14:23.501 | INFO | Starting a round of Baz IO | 20260507 22:14:26.973 | INFO | Populating RTMetrics report | 20260507 22:14:26.974 | INFO | RTMetrics report populated | 20260507 22:14:26.974 | INFO | Opening tmp RTMetrics json file | 20260507 22:14:27.145 | INFO | Serializing RTMetrics to json | 20260507 22:14:27.152 | INFO | Serialization done, swapping tmp file now... | 20260507 22:14:27.154 | INFO | RTMetrics done | 20260507 22:14:27.170 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.668 seconds | 20260507 22:14:27.170 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.23% / 0.01% / 56.76% | 20260507 22:14:27.170 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46496% / 1.19209e-05% / 5.5631e-05% | 20260507 22:14:27.170 | INFO | Baz IO overall: | 20260507 22:14:27.170 | INFO | BytesWritten : 1367899211125 | 20260507 22:14:27.170 | INFO | NumEvents : 662796410156 | 20260507 22:14:28.542 | INFO | Analyzing chunk frames = [3002368,3002880) | 20260507 22:14:28.542 | INFO | MemInfo: { | 20260507 22:14:28.542 | INFO | "gpu" : " 17 GB", | 20260507 22:14:28.542 | INFO | "huge" : "", | 20260507 22:14:28.542 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:28.542 | INFO | "malloc" : " 14 GB", | 20260507 22:14:28.542 | INFO | "pinned" : " 29 GB" | 20260507 22:14:28.542 | INFO | } | 20260507 22:14:28.542 | INFO | Perf report for the last 5.13928 seconds: | 20260507 22:14:28.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:28.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:28.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:28.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:28.542 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 22:14:28.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:28.542 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 22:14:28.542 | INFO | PRE_HQ : 65.8%, 1.7, 57.5ms, 0 | 20260507 22:14:28.542 | INFO | BAZWRITER : 71.0%, 3.8, 138.2ms, 0 | 20260507 22:14:28.542 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.7ms, 0 | 20260507 22:14:28.542 | INFO | Uploaded 36GiB to GPU in 2.5582s (14.0724GiB/s) | 20260507 22:14:28.542 | INFO | Downloaded 38GiB to GPU in 2.68229s (14.167GiB/s) | 20260507 22:14:33.681 | INFO | Analyzing chunk frames = [3002880,3003392) | 20260507 22:14:33.681 | INFO | MemInfo: { | 20260507 22:14:33.681 | INFO | "gpu" : " 17 GB", | 20260507 22:14:33.681 | INFO | "huge" : "", | 20260507 22:14:33.681 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:33.681 | INFO | "malloc" : " 14 GB", | 20260507 22:14:33.681 | INFO | "pinned" : " 29 GB" | 20260507 22:14:33.681 | INFO | } | 20260507 22:14:33.681 | INFO | Perf report for the last 5.13922 seconds: | 20260507 22:14:33.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:33.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:14:33.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:14:33.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:33.681 | INFO | Uploaded 36GiB to GPU in 2.5557s (14.0861GiB/s) | 20260507 22:14:33.681 | INFO | Downloaded 34GiB to GPU in 2.46971s (13.7668GiB/s) | 20260507 22:14:38.820 | INFO | Analyzing chunk frames = [3003392,3003904) | 20260507 22:14:38.820 | INFO | MemInfo: { | 20260507 22:14:38.820 | INFO | "gpu" : " 17 GB", | 20260507 22:14:38.820 | INFO | "huge" : "", | 20260507 22:14:38.820 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:38.820 | INFO | "malloc" : " 14 GB", | 20260507 22:14:38.820 | INFO | "pinned" : " 29 GB" | 20260507 22:14:38.820 | INFO | } | 20260507 22:14:38.821 | INFO | Perf report for the last 5.13941 seconds: | 20260507 22:14:38.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:38.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | ANALYSIS : 67.9%, 3.0, 105.3ms, 0 | 20260507 22:14:38.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 22:14:38.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:38.821 | INFO | Uploaded 36GiB to GPU in 2.57784s (13.9652GiB/s) | 20260507 22:14:38.821 | INFO | Downloaded 34GiB to GPU in 2.48583s (13.6775GiB/s) | 20260507 22:14:43.956 | INFO | Analyzing chunk frames = [3003904,3004416) | 20260507 22:14:43.956 | INFO | MemInfo: { | 20260507 22:14:43.956 | INFO | "gpu" : " 17 GB", | 20260507 22:14:43.956 | INFO | "huge" : "", | 20260507 22:14:43.956 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:43.956 | INFO | "malloc" : " 14 GB", | 20260507 22:14:43.956 | INFO | "pinned" : " 29 GB" | 20260507 22:14:43.956 | INFO | } | 20260507 22:14:43.956 | INFO | Perf report for the last 5.1354 seconds: | 20260507 22:14:43.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:43.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:14:43.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:14:43.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:43.956 | INFO | Uploaded 36GiB to GPU in 2.58654s (13.9182GiB/s) | 20260507 22:14:43.956 | INFO | Downloaded 34GiB to GPU in 2.46722s (13.7807GiB/s) | 20260507 22:14:49.095 | INFO | Analyzing chunk frames = [3004416,3004928) | 20260507 22:14:49.095 | INFO | MemInfo: { | 20260507 22:14:49.095 | INFO | "gpu" : " 17 GB", | 20260507 22:14:49.095 | INFO | "huge" : "", | 20260507 22:14:49.095 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:49.095 | INFO | "malloc" : " 14 GB", | 20260507 22:14:49.095 | INFO | "pinned" : " 29 GB" | 20260507 22:14:49.095 | INFO | } | 20260507 22:14:49.095 | INFO | Perf report for the last 5.13953 seconds: | 20260507 22:14:49.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:49.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.096 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:14:49.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.096 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:14:49.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:49.096 | INFO | Uploaded 36GiB to GPU in 2.46448s (14.6075GiB/s) | 20260507 22:14:49.096 | INFO | Downloaded 34GiB to GPU in 2.48868s (13.6619GiB/s) | 20260507 22:14:54.234 | INFO | Analyzing chunk frames = [3004928,3005440) | 20260507 22:14:54.234 | INFO | MemInfo: { | 20260507 22:14:54.234 | INFO | "gpu" : " 17 GB", | 20260507 22:14:54.234 | INFO | "huge" : "", | 20260507 22:14:54.235 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:54.235 | INFO | "malloc" : " 14 GB", | 20260507 22:14:54.235 | INFO | "pinned" : " 29 GB" | 20260507 22:14:54.235 | INFO | } | 20260507 22:14:54.235 | INFO | Perf report for the last 5.13928 seconds: | 20260507 22:14:54.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:54.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:14:54.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:14:54.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:54.235 | INFO | Uploaded 36GiB to GPU in 2.48981s (14.4589GiB/s) | 20260507 22:14:54.235 | INFO | Downloaded 34GiB to GPU in 2.41754s (14.0639GiB/s) | 20260507 22:14:59.379 | INFO | Analyzing chunk frames = [3005440,3005952) | 20260507 22:14:59.379 | INFO | MemInfo: { | 20260507 22:14:59.379 | INFO | "gpu" : " 17 GB", | 20260507 22:14:59.379 | INFO | "huge" : "", | 20260507 22:14:59.379 | INFO | "huge_shared" : " 29 GB", | 20260507 22:14:59.379 | INFO | "malloc" : " 14 GB", | 20260507 22:14:59.379 | INFO | "pinned" : " 29 GB" | 20260507 22:14:59.379 | INFO | } | 20260507 22:14:59.379 | INFO | Perf report for the last 5.14423 seconds: | 20260507 22:14:59.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:14:59.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 22:14:59.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:14:59.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:14:59.379 | INFO | Uploaded 36GiB to GPU in 2.53027s (14.2278GiB/s) | 20260507 22:14:59.379 | INFO | Downloaded 34GiB to GPU in 2.44744s (13.8921GiB/s) | 20260507 22:15:04.516 | INFO | Analyzing chunk frames = [3005952,3006464) | 20260507 22:15:04.516 | INFO | MemInfo: { | 20260507 22:15:04.516 | INFO | "gpu" : " 17 GB", | 20260507 22:15:04.516 | INFO | "huge" : "", | 20260507 22:15:04.516 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:04.516 | INFO | "malloc" : " 14 GB", | 20260507 22:15:04.516 | INFO | "pinned" : " 29 GB" | 20260507 22:15:04.516 | INFO | } | 20260507 22:15:04.516 | INFO | Perf report for the last 5.13699 seconds: | 20260507 22:15:04.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:04.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:15:04.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:15:04.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:04.516 | INFO | Uploaded 36GiB to GPU in 2.487s (14.4753GiB/s) | 20260507 22:15:04.516 | INFO | Downloaded 34GiB to GPU in 2.43971s (13.9361GiB/s) | 20260507 22:15:04.517 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:15:04.517 | INFO | Total mem usage: 16.7 GiB | 20260507 22:15:04.517 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:15:04.517 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.2%) | 20260507 22:15:04.517 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:15:04.517 | INFO | High Water Breakdown: | 20260507 22:15:04.517 | INFO | 59.8% : 8.82 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:15:04.517 | INFO | 40% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:15:04.517 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:15:04.517 | INFO | ------------------------------------------ | 20260507 22:15:04.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:15:04.517 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:15:04.517 | INFO | Total mem usage: 26.5 GiB | 20260507 22:15:04.517 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:15:04.517 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:15:04.517 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:15:04.517 | INFO | High Water Breakdown: | 20260507 22:15:04.517 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:15:04.517 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:15:04.517 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:15:04.517 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:15:04.517 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:15:04.517 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:15:04.517 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:15:04.517 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:15:04.517 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:15:04.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:15:04.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:15:04.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:15:04.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:15:04.517 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:15:04.517 | INFO | ------------------------------------------ | 20260507 22:15:04.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:15:04.517 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:15:04.517 | INFO | Total mem usage: 27.8 GiB | 20260507 22:15:04.517 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:15:04.517 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:15:04.517 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:15:04.518 | INFO | High Water Breakdown: | 20260507 22:15:04.518 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:15:04.518 | INFO | ------------------------------------------ | 20260507 22:15:04.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:15:04.518 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:15:04.518 | INFO | Total mem usage: 15.7 GiB | 20260507 22:15:04.518 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:15:04.518 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:15:04.518 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:15:04.518 | INFO | High Water Breakdown: | 20260507 22:15:04.518 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:15:04.518 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:15:04.518 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:15:04.518 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:15:04.518 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:15:04.518 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:15:04.518 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:15:04.518 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:15:04.518 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:15:04.518 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:15:04.518 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:15:04.518 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:15:04.518 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:15:04.518 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:15:04.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:15:04.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:15:04.518 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:15:04.518 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:15:04.518 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:15:04.518 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:15:04.518 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:15:04.518 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:15:04.518 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:15:04.518 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:15:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:15:04.518 | INFO | ------------------------------------------ | 20260507 22:15:04.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:15:04.518 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:15:04.518 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:15:04.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:15:04.518 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.65e+01, 1.39e+01 / 1.56e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.10e-02 / 5.05e-03 | 20260507 22:15:04.518 | INFO | PerfReport: 12.11%, Baseline -- 3.99e+00 / 4.47e+00, 5.93e-01 / 1.98e-01 | 20260507 22:15:04.518 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.08e-01 / 1.12e-01 | 20260507 22:15:04.518 | INFO | PerfReport: 15.80%, DME -- 4.16e+01 / 4.52e+01, 3.89e+00 / 1.35e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.86e+01 / 2.08e+01, 1.82e+00 / 1.97e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.43e+00 / 2.72e+00, 4.26e-01 / 2.08e-01 | 20260507 22:15:04.518 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.36e+00, 2.91e-01 / 1.86e-01 | 20260507 22:15:04.518 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.86e-01, 6.05e-02 / 2.34e-02 | 20260507 22:15:04.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:15:04.518 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:15:04.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:15:04.518 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 24.57%, Upload -- 2.63e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 24.13%, Download -- 2.58e+01 / 0.00e+00, 8.56e+00 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 30.79%, Compute -- 3.30e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 2.24%, LockUpload -- 2.39e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 2.36%, LockDownload -- 2.52e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 15.91%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:15:04.518 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 3.15e-02 / 0.00e+00 | 20260507 22:15:04.616 | INFO | Starting a round of Baz IO | 20260507 22:15:08.110 | INFO | Populating RTMetrics report | 20260507 22:15:08.110 | INFO | RTMetrics report populated | 20260507 22:15:08.110 | INFO | Opening tmp RTMetrics json file | 20260507 22:15:08.111 | INFO | Serializing RTMetrics to json | 20260507 22:15:08.113 | INFO | Serialization done, swapping tmp file now... | 20260507 22:15:08.114 | INFO | RTMetrics done | 20260507 22:15:08.247 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.631 seconds | 20260507 22:15:08.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.23% / 0.01% / 56.77% | 20260507 22:15:08.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46604% / 2.38419e-05% / 6.35783e-05% | 20260507 22:15:08.247 | INFO | Baz IO overall: | 20260507 22:15:08.247 | INFO | BytesWritten : 1369769466995 | 20260507 22:15:08.247 | INFO | NumEvents : 663699429456 | 20260507 22:15:09.655 | INFO | Analyzing chunk frames = [3006464,3006976) | 20260507 22:15:09.655 | INFO | MemInfo: { | 20260507 22:15:09.655 | INFO | "gpu" : " 17 GB", | 20260507 22:15:09.655 | INFO | "huge" : "", | 20260507 22:15:09.655 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:09.655 | INFO | "malloc" : " 14 GB", | 20260507 22:15:09.655 | INFO | "pinned" : " 29 GB" | 20260507 22:15:09.655 | INFO | } | 20260507 22:15:09.655 | INFO | Perf report for the last 5.13884 seconds: | 20260507 22:15:09.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:09.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:09.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:09.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:09.655 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260507 22:15:09.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:09.655 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 22:15:09.655 | INFO | PRE_HQ : 66.0%, 1.7, 59.4ms, 0 | 20260507 22:15:09.655 | INFO | BAZWRITER : 70.3%, 4.2, 150.3ms, 0 | 20260507 22:15:09.655 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:15:09.655 | INFO | Uploaded 36GiB to GPU in 2.50026s (14.3985GiB/s) | 20260507 22:15:09.655 | INFO | Downloaded 38GiB to GPU in 2.68882s (14.1326GiB/s) | 20260507 22:15:14.793 | INFO | Analyzing chunk frames = [3006976,3007488) | 20260507 22:15:14.793 | INFO | MemInfo: { | 20260507 22:15:14.793 | INFO | "gpu" : " 17 GB", | 20260507 22:15:14.793 | INFO | "huge" : "", | 20260507 22:15:14.793 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:14.793 | INFO | "malloc" : " 14 GB", | 20260507 22:15:14.793 | INFO | "pinned" : " 29 GB" | 20260507 22:15:14.793 | INFO | } | 20260507 22:15:14.793 | INFO | Perf report for the last 5.1386 seconds: | 20260507 22:15:14.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:14.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 22:15:14.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:15:14.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:14.794 | INFO | Uploaded 36GiB to GPU in 2.52137s (14.278GiB/s) | 20260507 22:15:14.794 | INFO | Downloaded 34GiB to GPU in 2.49286s (13.6389GiB/s) | 20260507 22:15:19.933 | INFO | Analyzing chunk frames = [3007488,3008000) | 20260507 22:15:19.933 | INFO | MemInfo: { | 20260507 22:15:19.933 | INFO | "gpu" : " 17 GB", | 20260507 22:15:19.933 | INFO | "huge" : "", | 20260507 22:15:19.933 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:19.933 | INFO | "malloc" : " 14 GB", | 20260507 22:15:19.933 | INFO | "pinned" : " 29 GB" | 20260507 22:15:19.933 | INFO | } | 20260507 22:15:19.933 | INFO | Perf report for the last 5.13938 seconds: | 20260507 22:15:19.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:19.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.933 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:15:19.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.933 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:15:19.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:19.934 | INFO | Uploaded 36GiB to GPU in 2.56925s (14.0119GiB/s) | 20260507 22:15:19.934 | INFO | Downloaded 34GiB to GPU in 2.47355s (13.7454GiB/s) | 20260507 22:15:25.073 | INFO | Analyzing chunk frames = [3008000,3008512) | 20260507 22:15:25.073 | INFO | MemInfo: { | 20260507 22:15:25.073 | INFO | "gpu" : " 17 GB", | 20260507 22:15:25.073 | INFO | "huge" : "", | 20260507 22:15:25.073 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:25.073 | INFO | "malloc" : " 14 GB", | 20260507 22:15:25.073 | INFO | "pinned" : " 29 GB" | 20260507 22:15:25.073 | INFO | } | 20260507 22:15:25.074 | INFO | Perf report for the last 5.14064 seconds: | 20260507 22:15:25.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:25.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:15:25.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:15:25.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:25.074 | INFO | Uploaded 36GiB to GPU in 2.46478s (14.6058GiB/s) | 20260507 22:15:25.074 | INFO | Downloaded 34GiB to GPU in 2.38743s (14.2412GiB/s) | 20260507 22:15:30.209 | INFO | Analyzing chunk frames = [3008512,3009024) | 20260507 22:15:30.209 | INFO | MemInfo: { | 20260507 22:15:30.209 | INFO | "gpu" : " 17 GB", | 20260507 22:15:30.209 | INFO | "huge" : "", | 20260507 22:15:30.209 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:30.209 | INFO | "malloc" : " 14 GB", | 20260507 22:15:30.209 | INFO | "pinned" : " 29 GB" | 20260507 22:15:30.209 | INFO | } | 20260507 22:15:30.209 | INFO | Perf report for the last 5.13549 seconds: | 20260507 22:15:30.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:30.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.209 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:15:30.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.209 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:15:30.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:30.210 | INFO | Uploaded 36GiB to GPU in 2.3352s (15.4162GiB/s) | 20260507 22:15:30.210 | INFO | Downloaded 34GiB to GPU in 2.50122s (13.5934GiB/s) | 20260507 22:15:35.351 | INFO | Analyzing chunk frames = [3009024,3009536) | 20260507 22:15:35.351 | INFO | MemInfo: { | 20260507 22:15:35.351 | INFO | "gpu" : " 17 GB", | 20260507 22:15:35.351 | INFO | "huge" : "", | 20260507 22:15:35.351 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:35.351 | INFO | "malloc" : " 14 GB", | 20260507 22:15:35.351 | INFO | "pinned" : " 29 GB" | 20260507 22:15:35.351 | INFO | } | 20260507 22:15:35.351 | INFO | Perf report for the last 5.14209 seconds: | 20260507 22:15:35.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:35.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 22:15:35.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:15:35.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:35.351 | INFO | Uploaded 36GiB to GPU in 2.51649s (14.3056GiB/s) | 20260507 22:15:35.351 | INFO | Downloaded 34GiB to GPU in 2.41628s (14.0712GiB/s) | 20260507 22:15:40.493 | INFO | Analyzing chunk frames = [3009536,3010048) | 20260507 22:15:40.493 | INFO | MemInfo: { | 20260507 22:15:40.493 | INFO | "gpu" : " 17 GB", | 20260507 22:15:40.493 | INFO | "huge" : "", | 20260507 22:15:40.493 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:40.493 | INFO | "malloc" : " 14 GB", | 20260507 22:15:40.493 | INFO | "pinned" : " 29 GB" | 20260507 22:15:40.493 | INFO | } | 20260507 22:15:40.493 | INFO | Perf report for the last 5.14191 seconds: | 20260507 22:15:40.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:40.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 22:15:40.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:15:40.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:40.493 | INFO | Uploaded 36GiB to GPU in 2.53404s (14.2065GiB/s) | 20260507 22:15:40.493 | INFO | Downloaded 34GiB to GPU in 2.43928s (13.9385GiB/s) | 20260507 22:15:45.637 | INFO | Analyzing chunk frames = [3010048,3010560) | 20260507 22:15:45.637 | INFO | MemInfo: { | 20260507 22:15:45.637 | INFO | "gpu" : " 17 GB", | 20260507 22:15:45.637 | INFO | "huge" : "", | 20260507 22:15:45.637 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:45.637 | INFO | "malloc" : " 14 GB", | 20260507 22:15:45.637 | INFO | "pinned" : " 29 GB" | 20260507 22:15:45.637 | INFO | } | 20260507 22:15:45.637 | INFO | Perf report for the last 5.14373 seconds: | 20260507 22:15:45.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:45.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.637 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 22:15:45.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.637 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 22:15:45.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:45.638 | INFO | Uploaded 36GiB to GPU in 2.5341s (14.2062GiB/s) | 20260507 22:15:45.638 | INFO | Downloaded 34GiB to GPU in 2.43436s (13.9667GiB/s) | 20260507 22:15:45.734 | INFO | Starting a round of Baz IO | 20260507 22:15:49.200 | INFO | Populating RTMetrics report | 20260507 22:15:49.200 | INFO | RTMetrics report populated | 20260507 22:15:49.200 | INFO | Opening tmp RTMetrics json file | 20260507 22:15:49.201 | INFO | Serializing RTMetrics to json | 20260507 22:15:49.237 | INFO | Serialization done, swapping tmp file now... | 20260507 22:15:49.273 | INFO | RTMetrics done | 20260507 22:15:49.322 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.587 seconds | 20260507 22:15:49.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.22% / 0.01% / 56.77% | 20260507 22:15:49.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46408% / 1.58946e-05% / 5.16574e-05% | 20260507 22:15:49.322 | INFO | Baz IO overall: | 20260507 22:15:49.322 | INFO | BytesWritten : 1371640144967 | 20260507 22:15:49.322 | INFO | NumEvents : 664602804900 | 20260507 22:15:50.771 | INFO | Analyzing chunk frames = [3010560,3011072) | 20260507 22:15:50.771 | INFO | MemInfo: { | 20260507 22:15:50.771 | INFO | "gpu" : " 17 GB", | 20260507 22:15:50.771 | INFO | "huge" : "", | 20260507 22:15:50.771 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:50.771 | INFO | "malloc" : " 14 GB", | 20260507 22:15:50.771 | INFO | "pinned" : " 29 GB" | 20260507 22:15:50.771 | INFO | } | 20260507 22:15:50.771 | INFO | Perf report for the last 5.13434 seconds: | 20260507 22:15:50.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:50.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:50.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:50.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:50.771 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 22:15:50.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:50.771 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 22:15:50.771 | INFO | PRE_HQ : 65.8%, 1.7, 57.7ms, 0 | 20260507 22:15:50.771 | INFO | BAZWRITER : 69.1%, 3.6, 129.3ms, 0 | 20260507 22:15:50.771 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 22:15:50.771 | INFO | Uploaded 36GiB to GPU in 2.53977s (14.1745GiB/s) | 20260507 22:15:50.771 | INFO | Downloaded 38GiB to GPU in 2.68524s (14.1514GiB/s) | 20260507 22:15:55.906 | INFO | Analyzing chunk frames = [3011072,3011584) | 20260507 22:15:55.906 | INFO | MemInfo: { | 20260507 22:15:55.906 | INFO | "gpu" : " 17 GB", | 20260507 22:15:55.906 | INFO | "huge" : "", | 20260507 22:15:55.906 | INFO | "huge_shared" : " 29 GB", | 20260507 22:15:55.906 | INFO | "malloc" : " 14 GB", | 20260507 22:15:55.906 | INFO | "pinned" : " 29 GB" | 20260507 22:15:55.906 | INFO | } | 20260507 22:15:55.906 | INFO | Perf report for the last 5.13462 seconds: | 20260507 22:15:55.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:15:55.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 22:15:55.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:15:55.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:15:55.906 | INFO | Uploaded 36GiB to GPU in 2.58627s (13.9197GiB/s) | 20260507 22:15:55.906 | INFO | Downloaded 34GiB to GPU in 2.47344s (13.7461GiB/s) | 20260507 22:16:01.046 | INFO | Analyzing chunk frames = [3011584,3012096) | 20260507 22:16:01.046 | INFO | MemInfo: { | 20260507 22:16:01.046 | INFO | "gpu" : " 17 GB", | 20260507 22:16:01.046 | INFO | "huge" : "", | 20260507 22:16:01.046 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:01.046 | INFO | "malloc" : " 14 GB", | 20260507 22:16:01.046 | INFO | "pinned" : " 29 GB" | 20260507 22:16:01.046 | INFO | } | 20260507 22:16:01.046 | INFO | Perf report for the last 5.14001 seconds: | 20260507 22:16:01.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:01.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 22:16:01.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:16:01.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:01.046 | INFO | Uploaded 36GiB to GPU in 2.54443s (14.1486GiB/s) | 20260507 22:16:01.046 | INFO | Downloaded 34GiB to GPU in 2.44491s (13.9064GiB/s) | 20260507 22:16:06.185 | INFO | Analyzing chunk frames = [3012096,3012608) | 20260507 22:16:06.185 | INFO | MemInfo: { | 20260507 22:16:06.185 | INFO | "gpu" : " 17 GB", | 20260507 22:16:06.185 | INFO | "huge" : "", | 20260507 22:16:06.185 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:06.185 | INFO | "malloc" : " 14 GB", | 20260507 22:16:06.186 | INFO | "pinned" : " 29 GB" | 20260507 22:16:06.186 | INFO | } | 20260507 22:16:06.186 | INFO | Perf report for the last 5.13978 seconds: | 20260507 22:16:06.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:06.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 22:16:06.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:16:06.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:06.186 | INFO | Uploaded 36GiB to GPU in 2.4871s (14.4747GiB/s) | 20260507 22:16:06.186 | INFO | Downloaded 34GiB to GPU in 2.38681s (14.2449GiB/s) | 20260507 22:16:11.326 | INFO | Analyzing chunk frames = [3012608,3013120) | 20260507 22:16:11.326 | INFO | MemInfo: { | 20260507 22:16:11.326 | INFO | "gpu" : " 17 GB", | 20260507 22:16:11.326 | INFO | "huge" : "", | 20260507 22:16:11.326 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:11.326 | INFO | "malloc" : " 14 GB", | 20260507 22:16:11.326 | INFO | "pinned" : " 29 GB" | 20260507 22:16:11.326 | INFO | } | 20260507 22:16:11.326 | INFO | Perf report for the last 5.14067 seconds: | 20260507 22:16:11.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:11.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.326 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:16:11.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.326 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 22:16:11.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:11.327 | INFO | Uploaded 36GiB to GPU in 2.37295s (15.171GiB/s) | 20260507 22:16:11.327 | INFO | Downloaded 34GiB to GPU in 2.49382s (13.6337GiB/s) | 20260507 22:16:16.466 | INFO | Analyzing chunk frames = [3013120,3013632) | 20260507 22:16:16.466 | INFO | MemInfo: { | 20260507 22:16:16.466 | INFO | "gpu" : " 17 GB", | 20260507 22:16:16.466 | INFO | "huge" : "", | 20260507 22:16:16.466 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:16.466 | INFO | "malloc" : " 14 GB", | 20260507 22:16:16.466 | INFO | "pinned" : " 29 GB" | 20260507 22:16:16.466 | INFO | } | 20260507 22:16:16.466 | INFO | Perf report for the last 5.14014 seconds: | 20260507 22:16:16.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:16.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.467 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:16:16.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.467 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:16:16.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:16.467 | INFO | Uploaded 36GiB to GPU in 2.5113s (14.3352GiB/s) | 20260507 22:16:16.467 | INFO | Downloaded 34GiB to GPU in 2.42661s (14.0113GiB/s) | 20260507 22:16:21.607 | INFO | Analyzing chunk frames = [3013632,3014144) | 20260507 22:16:21.607 | INFO | MemInfo: { | 20260507 22:16:21.607 | INFO | "gpu" : " 17 GB", | 20260507 22:16:21.607 | INFO | "huge" : "", | 20260507 22:16:21.607 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:21.607 | INFO | "malloc" : " 14 GB", | 20260507 22:16:21.607 | INFO | "pinned" : " 29 GB" | 20260507 22:16:21.607 | INFO | } | 20260507 22:16:21.607 | INFO | Perf report for the last 5.14094 seconds: | 20260507 22:16:21.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:21.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 22:16:21.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 22:16:21.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:21.608 | INFO | Uploaded 36GiB to GPU in 2.52035s (14.2837GiB/s) | 20260507 22:16:21.608 | INFO | Downloaded 34GiB to GPU in 2.44762s (13.8911GiB/s) | 20260507 22:16:26.748 | INFO | Analyzing chunk frames = [3014144,3014656) | 20260507 22:16:26.748 | INFO | MemInfo: { | 20260507 22:16:26.748 | INFO | "gpu" : " 17 GB", | 20260507 22:16:26.748 | INFO | "huge" : "", | 20260507 22:16:26.748 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:26.748 | INFO | "malloc" : " 14 GB", | 20260507 22:16:26.748 | INFO | "pinned" : " 29 GB" | 20260507 22:16:26.748 | INFO | } | 20260507 22:16:26.749 | INFO | Perf report for the last 5.14126 seconds: | 20260507 22:16:26.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:26.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:16:26.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:16:26.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:26.749 | INFO | Uploaded 36GiB to GPU in 2.49338s (14.4382GiB/s) | 20260507 22:16:26.749 | INFO | Downloaded 34GiB to GPU in 2.43581s (13.9584GiB/s) | 20260507 22:16:26.749 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:16:26.749 | INFO | Total mem usage: 16.7 GiB | 20260507 22:16:26.749 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:16:26.749 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260507 22:16:26.749 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:16:26.749 | INFO | High Water Breakdown: | 20260507 22:16:26.749 | INFO | 59.8% : 8.79 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:16:26.749 | INFO | 40.1% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:16:26.749 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:16:26.749 | INFO | ------------------------------------------ | 20260507 22:16:26.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:16:26.749 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:16:26.749 | INFO | Total mem usage: 26.5 GiB | 20260507 22:16:26.749 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:16:26.749 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:16:26.749 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:16:26.749 | INFO | High Water Breakdown: | 20260507 22:16:26.749 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:16:26.749 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:16:26.749 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:16:26.749 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:16:26.749 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:16:26.749 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:16:26.749 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:16:26.749 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:16:26.749 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:16:26.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:16:26.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:16:26.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:16:26.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:16:26.749 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:16:26.749 | INFO | ------------------------------------------ | 20260507 22:16:26.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:16:26.749 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:16:26.749 | INFO | Total mem usage: 27.8 GiB | 20260507 22:16:26.749 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:16:26.749 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:16:26.749 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:16:26.749 | INFO | High Water Breakdown: | 20260507 22:16:26.750 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:16:26.750 | INFO | ------------------------------------------ | 20260507 22:16:26.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:16:26.750 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:16:26.750 | INFO | Total mem usage: 15.7 GiB | 20260507 22:16:26.750 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:16:26.750 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:16:26.750 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:16:26.750 | INFO | High Water Breakdown: | 20260507 22:16:26.750 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:16:26.750 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:16:26.750 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:16:26.750 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:16:26.750 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:16:26.750 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:16:26.750 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:16:26.750 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:16:26.750 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:16:26.750 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:16:26.750 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:16:26.750 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:16:26.750 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:16:26.750 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:16:26.750 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:16:26.750 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:16:26.750 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:16:26.750 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:16:26.750 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:16:26.750 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:16:26.750 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:16:26.750 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:16:26.750 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:16:26.750 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:16:26.750 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:16:26.750 | INFO | ------------------------------------------ | 20260507 22:16:26.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:16:26.750 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:16:26.750 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:16:26.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:16:26.750 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.41e+01 / 1.58e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.18e-02 / 6.60e-03 | 20260507 22:16:26.750 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.50e+00, 5.96e-01 / 2.12e-01 | 20260507 22:16:26.750 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.48e+00, 2.10e-01 / 1.32e-01 | 20260507 22:16:26.750 | INFO | PerfReport: 15.81%, DME -- 4.17e+01 / 4.50e+01, 3.75e+00 / 1.32e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.77e+00 / 1.92e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.31e-01 / 1.98e-01 | 20260507 22:16:26.750 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.36e+00, 2.94e-01 / 1.60e-01 | 20260507 22:16:26.750 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.88e-01, 5.98e-02 / 2.93e-02 | 20260507 22:16:26.750 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:16:26.750 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:16:26.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:16:26.750 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 24.15%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 30.85%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 2.11%, LockUpload -- 2.25e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 2.45%, LockDownload -- 2.62e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 16.02%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:16:26.750 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 22:16:26.846 | INFO | Starting a round of Baz IO | 20260507 22:16:30.301 | INFO | Populating RTMetrics report | 20260507 22:16:30.301 | INFO | RTMetrics report populated | 20260507 22:16:30.301 | INFO | Opening tmp RTMetrics json file | 20260507 22:16:30.357 | INFO | Serializing RTMetrics to json | 20260507 22:16:30.411 | INFO | Serialization done, swapping tmp file now... | 20260507 22:16:30.415 | INFO | RTMetrics done | 20260507 22:16:30.440 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.594 seconds | 20260507 22:16:30.440 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.21% / 0.01% / 56.78% | 20260507 22:16:30.440 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46407% / 3.97364e-06% / 5.96046e-05% | 20260507 22:16:30.440 | INFO | Baz IO overall: | 20260507 22:16:30.440 | INFO | BytesWritten : 1373511715902 | 20260507 22:16:30.440 | INFO | NumEvents : 665506991656 | 20260507 22:16:31.884 | INFO | Analyzing chunk frames = [3014656,3015168) | 20260507 22:16:31.884 | INFO | MemInfo: { | 20260507 22:16:31.884 | INFO | "gpu" : " 17 GB", | 20260507 22:16:31.884 | INFO | "huge" : "", | 20260507 22:16:31.884 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:31.884 | INFO | "malloc" : " 14 GB", | 20260507 22:16:31.884 | INFO | "pinned" : " 29 GB" | 20260507 22:16:31.884 | INFO | } | 20260507 22:16:31.884 | INFO | Perf report for the last 5.13507 seconds: | 20260507 22:16:31.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:31.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:31.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:31.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:31.884 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260507 22:16:31.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:31.884 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 22:16:31.884 | INFO | PRE_HQ : 65.9%, 1.7, 57.7ms, 0 | 20260507 22:16:31.884 | INFO | BAZWRITER : 69.6%, 3.9, 139.1ms, 0 | 20260507 22:16:31.884 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 22:16:31.884 | INFO | Uploaded 36GiB to GPU in 2.52089s (14.2807GiB/s) | 20260507 22:16:31.884 | INFO | Downloaded 38GiB to GPU in 2.69285s (14.1114GiB/s) | 20260507 22:16:37.021 | INFO | Analyzing chunk frames = [3015168,3015680) | 20260507 22:16:37.021 | INFO | MemInfo: { | 20260507 22:16:37.021 | INFO | "gpu" : " 17 GB", | 20260507 22:16:37.021 | INFO | "huge" : "", | 20260507 22:16:37.021 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:37.021 | INFO | "malloc" : " 14 GB", | 20260507 22:16:37.021 | INFO | "pinned" : " 29 GB" | 20260507 22:16:37.021 | INFO | } | 20260507 22:16:37.021 | INFO | Perf report for the last 5.13723 seconds: | 20260507 22:16:37.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:37.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 22:16:37.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:16:37.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:37.021 | INFO | Uploaded 36GiB to GPU in 2.56157s (14.0539GiB/s) | 20260507 22:16:37.021 | INFO | Downloaded 34GiB to GPU in 2.48207s (13.6983GiB/s) | 20260507 22:16:42.162 | INFO | Analyzing chunk frames = [3015680,3016192) | 20260507 22:16:42.162 | INFO | MemInfo: { | 20260507 22:16:42.162 | INFO | "gpu" : " 17 GB", | 20260507 22:16:42.162 | INFO | "huge" : "", | 20260507 22:16:42.162 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:42.162 | INFO | "malloc" : " 14 GB", | 20260507 22:16:42.162 | INFO | "pinned" : " 29 GB" | 20260507 22:16:42.162 | INFO | } | 20260507 22:16:42.162 | INFO | Perf report for the last 5.14109 seconds: | 20260507 22:16:42.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:42.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.162 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 22:16:42.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.162 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:16:42.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:42.163 | INFO | Uploaded 36GiB to GPU in 2.56034s (14.0607GiB/s) | 20260507 22:16:42.163 | INFO | Downloaded 34GiB to GPU in 2.46769s (13.778GiB/s) | 20260507 22:16:47.303 | INFO | Analyzing chunk frames = [3016192,3016704) | 20260507 22:16:47.303 | INFO | MemInfo: { | 20260507 22:16:47.303 | INFO | "gpu" : " 17 GB", | 20260507 22:16:47.303 | INFO | "huge" : "", | 20260507 22:16:47.303 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:47.303 | INFO | "malloc" : " 14 GB", | 20260507 22:16:47.303 | INFO | "pinned" : " 29 GB" | 20260507 22:16:47.303 | INFO | } | 20260507 22:16:47.303 | INFO | Perf report for the last 5.14122 seconds: | 20260507 22:16:47.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:47.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.303 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 22:16:47.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.303 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:16:47.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:47.304 | INFO | Uploaded 36GiB to GPU in 2.46484s (14.6054GiB/s) | 20260507 22:16:47.304 | INFO | Downloaded 34GiB to GPU in 2.38457s (14.2583GiB/s) | 20260507 22:16:52.448 | INFO | Analyzing chunk frames = [3016704,3017216) | 20260507 22:16:52.448 | INFO | MemInfo: { | 20260507 22:16:52.448 | INFO | "gpu" : " 17 GB", | 20260507 22:16:52.448 | INFO | "huge" : "", | 20260507 22:16:52.448 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:52.448 | INFO | "malloc" : " 14 GB", | 20260507 22:16:52.448 | INFO | "pinned" : " 29 GB" | 20260507 22:16:52.448 | INFO | } | 20260507 22:16:52.448 | INFO | Perf report for the last 5.14469 seconds: | 20260507 22:16:52.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:52.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 22:16:52.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 22:16:52.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:52.448 | INFO | Uploaded 36GiB to GPU in 2.39361s (15.04GiB/s) | 20260507 22:16:52.448 | INFO | Downloaded 34GiB to GPU in 2.48679s (13.6723GiB/s) | 20260507 22:16:57.586 | INFO | Analyzing chunk frames = [3017216,3017728) | 20260507 22:16:57.586 | INFO | MemInfo: { | 20260507 22:16:57.586 | INFO | "gpu" : " 17 GB", | 20260507 22:16:57.586 | INFO | "huge" : "", | 20260507 22:16:57.586 | INFO | "huge_shared" : " 29 GB", | 20260507 22:16:57.586 | INFO | "malloc" : " 14 GB", | 20260507 22:16:57.586 | INFO | "pinned" : " 29 GB" | 20260507 22:16:57.586 | INFO | } | 20260507 22:16:57.586 | INFO | Perf report for the last 5.13807 seconds: | 20260507 22:16:57.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:16:57.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.586 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:16:57.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.586 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:16:57.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:16:57.587 | INFO | Uploaded 36GiB to GPU in 2.40832s (14.9482GiB/s) | 20260507 22:16:57.587 | INFO | Downloaded 34GiB to GPU in 2.39909s (14.172GiB/s) | 20260507 22:17:02.717 | INFO | Analyzing chunk frames = [3017728,3018240) | 20260507 22:17:02.717 | INFO | MemInfo: { | 20260507 22:17:02.717 | INFO | "gpu" : " 17 GB", | 20260507 22:17:02.717 | INFO | "huge" : "", | 20260507 22:17:02.717 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:02.717 | INFO | "malloc" : " 14 GB", | 20260507 22:17:02.717 | INFO | "pinned" : " 29 GB" | 20260507 22:17:02.717 | INFO | } | 20260507 22:17:02.717 | INFO | Perf report for the last 5.13128 seconds: | 20260507 22:17:02.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:02.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260507 22:17:02.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:17:02.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:02.718 | INFO | Uploaded 36GiB to GPU in 2.53465s (14.2031GiB/s) | 20260507 22:17:02.718 | INFO | Downloaded 34GiB to GPU in 2.4561s (13.8431GiB/s) | 20260507 22:17:07.857 | INFO | Analyzing chunk frames = [3018240,3018752) | 20260507 22:17:07.857 | INFO | MemInfo: { | 20260507 22:17:07.857 | INFO | "gpu" : " 17 GB", | 20260507 22:17:07.857 | INFO | "huge" : "", | 20260507 22:17:07.857 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:07.857 | INFO | "malloc" : " 14 GB", | 20260507 22:17:07.857 | INFO | "pinned" : " 29 GB" | 20260507 22:17:07.857 | INFO | } | 20260507 22:17:07.857 | INFO | Perf report for the last 5.13947 seconds: | 20260507 22:17:07.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:07.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:17:07.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 22:17:07.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:07.857 | INFO | Uploaded 36GiB to GPU in 2.47965s (14.5182GiB/s) | 20260507 22:17:07.857 | INFO | Downloaded 34GiB to GPU in 2.44115s (13.9279GiB/s) | 20260507 22:17:07.955 | INFO | Starting a round of Baz IO | 20260507 22:17:11.408 | INFO | Populating RTMetrics report | 20260507 22:17:11.409 | INFO | RTMetrics report populated | 20260507 22:17:11.409 | INFO | Opening tmp RTMetrics json file | 20260507 22:17:11.409 | INFO | Serializing RTMetrics to json | 20260507 22:17:11.428 | INFO | Serialization done, swapping tmp file now... | 20260507 22:17:11.438 | INFO | RTMetrics done | 20260507 22:17:11.505 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.55 seconds | 20260507 22:17:11.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.21% / 0.01% / 56.79% | 20260507 22:17:11.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45645% / 3.97364e-06% / 3.17891e-05% | 20260507 22:17:11.505 | INFO | Baz IO overall: | 20260507 22:17:11.505 | INFO | BytesWritten : 1375381106191 | 20260507 22:17:11.505 | INFO | NumEvents : 666409370698 | 20260507 22:17:12.995 | INFO | Analyzing chunk frames = [3018752,3019264) | 20260507 22:17:12.995 | INFO | MemInfo: { | 20260507 22:17:12.995 | INFO | "gpu" : " 17 GB", | 20260507 22:17:12.995 | INFO | "huge" : "", | 20260507 22:17:12.995 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:12.995 | INFO | "malloc" : " 14 GB", | 20260507 22:17:12.995 | INFO | "pinned" : " 29 GB" | 20260507 22:17:12.995 | INFO | } | 20260507 22:17:12.995 | INFO | Perf report for the last 5.13821 seconds: | 20260507 22:17:12.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:12.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:12.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:12.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:12.995 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260507 22:17:12.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:12.995 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 22:17:12.995 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260507 22:17:12.995 | INFO | BAZWRITER : 68.7%, 3.8, 133.8ms, 0 | 20260507 22:17:12.995 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 22:17:12.996 | INFO | Uploaded 36GiB to GPU in 2.49123s (14.4507GiB/s) | 20260507 22:17:12.996 | INFO | Downloaded 38GiB to GPU in 2.69284s (14.1115GiB/s) | 20260507 22:17:18.134 | INFO | Analyzing chunk frames = [3019264,3019776) | 20260507 22:17:18.134 | INFO | MemInfo: { | 20260507 22:17:18.134 | INFO | "gpu" : " 17 GB", | 20260507 22:17:18.134 | INFO | "huge" : "", | 20260507 22:17:18.134 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:18.134 | INFO | "malloc" : " 14 GB", | 20260507 22:17:18.134 | INFO | "pinned" : " 29 GB" | 20260507 22:17:18.134 | INFO | } | 20260507 22:17:18.134 | INFO | Perf report for the last 5.13916 seconds: | 20260507 22:17:18.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:18.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.134 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 22:17:18.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.134 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:17:18.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:18.135 | INFO | Uploaded 36GiB to GPU in 2.55689s (14.0796GiB/s) | 20260507 22:17:18.135 | INFO | Downloaded 34GiB to GPU in 2.47025s (13.7638GiB/s) | 20260507 22:17:23.275 | INFO | Analyzing chunk frames = [3019776,3020288) | 20260507 22:17:23.275 | INFO | MemInfo: { | 20260507 22:17:23.275 | INFO | "gpu" : " 17 GB", | 20260507 22:17:23.275 | INFO | "huge" : "", | 20260507 22:17:23.275 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:23.275 | INFO | "malloc" : " 14 GB", | 20260507 22:17:23.275 | INFO | "pinned" : " 29 GB" | 20260507 22:17:23.275 | INFO | } | 20260507 22:17:23.275 | INFO | Perf report for the last 5.14104 seconds: | 20260507 22:17:23.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:23.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.275 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:17:23.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.275 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:17:23.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:23.276 | INFO | Uploaded 36GiB to GPU in 2.54309s (14.156GiB/s) | 20260507 22:17:23.276 | INFO | Downloaded 34GiB to GPU in 2.44351s (13.9144GiB/s) | 20260507 22:17:28.415 | INFO | Analyzing chunk frames = [3020288,3020800) | 20260507 22:17:28.415 | INFO | MemInfo: { | 20260507 22:17:28.415 | INFO | "gpu" : " 17 GB", | 20260507 22:17:28.415 | INFO | "huge" : "", | 20260507 22:17:28.415 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:28.415 | INFO | "malloc" : " 14 GB", | 20260507 22:17:28.415 | INFO | "pinned" : " 29 GB" | 20260507 22:17:28.415 | INFO | } | 20260507 22:17:28.415 | INFO | Perf report for the last 5.13956 seconds: | 20260507 22:17:28.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:28.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.415 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:17:28.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.415 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 22:17:28.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:28.416 | INFO | Uploaded 36GiB to GPU in 2.46356s (14.613GiB/s) | 20260507 22:17:28.416 | INFO | Downloaded 34GiB to GPU in 2.38261s (14.27GiB/s) | 20260507 22:17:33.559 | INFO | Analyzing chunk frames = [3020800,3021312) | 20260507 22:17:33.559 | INFO | MemInfo: { | 20260507 22:17:33.559 | INFO | "gpu" : " 17 GB", | 20260507 22:17:33.559 | INFO | "huge" : "", | 20260507 22:17:33.559 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:33.559 | INFO | "malloc" : " 14 GB", | 20260507 22:17:33.559 | INFO | "pinned" : " 29 GB" | 20260507 22:17:33.559 | INFO | } | 20260507 22:17:33.559 | INFO | Perf report for the last 5.14393 seconds: | 20260507 22:17:33.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:33.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 22:17:33.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:17:33.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:33.559 | INFO | Uploaded 36GiB to GPU in 2.39737s (15.0165GiB/s) | 20260507 22:17:33.559 | INFO | Downloaded 34GiB to GPU in 2.49544s (13.6248GiB/s) | 20260507 22:17:38.699 | INFO | Analyzing chunk frames = [3021312,3021824) | 20260507 22:17:38.699 | INFO | MemInfo: { | 20260507 22:17:38.699 | INFO | "gpu" : " 17 GB", | 20260507 22:17:38.699 | INFO | "huge" : "", | 20260507 22:17:38.699 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:38.699 | INFO | "malloc" : " 14 GB", | 20260507 22:17:38.699 | INFO | "pinned" : " 29 GB" | 20260507 22:17:38.699 | INFO | } | 20260507 22:17:38.699 | INFO | Perf report for the last 5.14026 seconds: | 20260507 22:17:38.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:38.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:17:38.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:17:38.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:38.699 | INFO | Uploaded 36GiB to GPU in 2.50289s (14.3834GiB/s) | 20260507 22:17:38.699 | INFO | Downloaded 34GiB to GPU in 2.41281s (14.0915GiB/s) | 20260507 22:17:43.835 | INFO | Analyzing chunk frames = [3021824,3022336) | 20260507 22:17:43.835 | INFO | MemInfo: { | 20260507 22:17:43.835 | INFO | "gpu" : " 17 GB", | 20260507 22:17:43.835 | INFO | "huge" : "", | 20260507 22:17:43.835 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:43.835 | INFO | "malloc" : " 14 GB", | 20260507 22:17:43.835 | INFO | "pinned" : " 29 GB" | 20260507 22:17:43.835 | INFO | } | 20260507 22:17:43.835 | INFO | Perf report for the last 5.13587 seconds: | 20260507 22:17:43.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:43.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 22:17:43.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:17:43.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:43.835 | INFO | Uploaded 36GiB to GPU in 2.55084s (14.113GiB/s) | 20260507 22:17:43.835 | INFO | Downloaded 34GiB to GPU in 2.44425s (13.9102GiB/s) | 20260507 22:17:48.970 | INFO | Analyzing chunk frames = [3022336,3022848) | 20260507 22:17:48.970 | INFO | MemInfo: { | 20260507 22:17:48.970 | INFO | "gpu" : " 17 GB", | 20260507 22:17:48.970 | INFO | "huge" : "", | 20260507 22:17:48.970 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:48.971 | INFO | "malloc" : " 14 GB", | 20260507 22:17:48.971 | INFO | "pinned" : " 29 GB" | 20260507 22:17:48.971 | INFO | } | 20260507 22:17:48.971 | INFO | Perf report for the last 5.13596 seconds: | 20260507 22:17:48.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:48.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 22:17:48.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:17:48.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:48.971 | INFO | Uploaded 36GiB to GPU in 2.51849s (14.2943GiB/s) | 20260507 22:17:48.971 | INFO | Downloaded 34GiB to GPU in 2.4423s (13.9213GiB/s) | 20260507 22:17:48.971 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:17:48.971 | INFO | Total mem usage: 16.7 GiB | 20260507 22:17:48.971 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:17:48.971 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.6%) | 20260507 22:17:48.971 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:17:48.971 | INFO | High Water Breakdown: | 20260507 22:17:48.971 | INFO | 60.1% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:17:48.971 | INFO | 39.8% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:17:48.971 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:17:48.971 | INFO | ------------------------------------------ | 20260507 22:17:48.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:17:48.971 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:17:48.971 | INFO | Total mem usage: 26.5 GiB | 20260507 22:17:48.971 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:17:48.971 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:17:48.971 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:17:48.971 | INFO | High Water Breakdown: | 20260507 22:17:48.971 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:17:48.971 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:17:48.971 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:17:48.971 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:17:48.971 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:17:48.971 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:17:48.971 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:17:48.971 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:17:48.971 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:17:48.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:17:48.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:17:48.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:17:48.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:17:48.972 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:17:48.972 | INFO | ------------------------------------------ | 20260507 22:17:48.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:17:48.972 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:17:48.972 | INFO | Total mem usage: 27.8 GiB | 20260507 22:17:48.972 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:17:48.972 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:17:48.972 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:17:48.972 | INFO | High Water Breakdown: | 20260507 22:17:48.972 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:17:48.972 | INFO | ------------------------------------------ | 20260507 22:17:48.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:17:48.972 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:17:48.972 | INFO | Total mem usage: 15.7 GiB | 20260507 22:17:48.972 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:17:48.972 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:17:48.972 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:17:48.972 | INFO | High Water Breakdown: | 20260507 22:17:48.972 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:17:48.972 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:17:48.972 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:17:48.972 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:17:48.972 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:17:48.972 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:17:48.972 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:17:48.972 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:17:48.972 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:17:48.972 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:17:48.972 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:17:48.972 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:17:48.972 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:17:48.972 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:17:48.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:17:48.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:17:48.972 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:17:48.972 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:17:48.972 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:17:48.972 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:17:48.972 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:17:48.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:17:48.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:17:48.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:17:48.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:17:48.972 | INFO | ------------------------------------------ | 20260507 22:17:48.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:17:48.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:17:48.972 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:17:48.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:17:48.972 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.43e+01 / 1.42e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 0.29%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.22e-02 / 8.39e-03 | 20260507 22:17:48.972 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.46e+00, 5.97e-01 / 2.31e-01 | 20260507 22:17:48.972 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.48e+00, 2.09e-01 / 1.18e-01 | 20260507 22:17:48.972 | INFO | PerfReport: 15.71%, DME -- 4.15e+01 / 4.45e+01, 3.55e+00 / 1.17e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.87e+01 / 2.09e+01, 1.85e+00 / 1.87e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.24e-01 / 2.43e-01 | 20260507 22:17:48.972 | INFO | PerfReport: 3.47%, Metrics -- 1.15e+00 / 1.35e+00, 2.93e-01 / 1.39e-01 | 20260507 22:17:48.972 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.86e-01, 5.89e-02 / 3.13e-02 | 20260507 22:17:48.972 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:17:48.972 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:17:48.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:17:48.972 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 24.16%, Download -- 2.58e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 30.94%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 1.97%, LockUpload -- 2.10e+00 / 0.00e+00, 5.14e+00 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 2.43%, LockDownload -- 2.60e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 16.11%, LockCompute -- 1.72e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:17:48.972 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260507 22:17:49.068 | INFO | Starting a round of Baz IO | 20260507 22:17:52.531 | INFO | Populating RTMetrics report | 20260507 22:17:52.532 | INFO | RTMetrics report populated | 20260507 22:17:52.532 | INFO | Opening tmp RTMetrics json file | 20260507 22:17:52.540 | INFO | Serializing RTMetrics to json | 20260507 22:17:52.556 | INFO | Serialization done, swapping tmp file now... | 20260507 22:17:52.556 | INFO | RTMetrics done | 20260507 22:17:52.633 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.564 seconds | 20260507 22:17:52.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.20% / 0.01% / 56.79% | 20260507 22:17:52.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45389% / 1.19209e-05% / 4.37101e-05% | 20260507 22:17:52.633 | INFO | Baz IO overall: | 20260507 22:17:52.633 | INFO | BytesWritten : 1377250053193 | 20260507 22:17:52.633 | INFO | NumEvents : 667311517095 | 20260507 22:17:54.112 | INFO | Analyzing chunk frames = [3022848,3023360) | 20260507 22:17:54.112 | INFO | MemInfo: { | 20260507 22:17:54.112 | INFO | "gpu" : " 17 GB", | 20260507 22:17:54.112 | INFO | "huge" : "", | 20260507 22:17:54.112 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:54.112 | INFO | "malloc" : " 14 GB", | 20260507 22:17:54.112 | INFO | "pinned" : " 29 GB" | 20260507 22:17:54.112 | INFO | } | 20260507 22:17:54.112 | INFO | Perf report for the last 5.14072 seconds: | 20260507 22:17:54.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:54.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:54.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:54.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:54.112 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260507 22:17:54.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:54.112 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 22:17:54.112 | INFO | PRE_HQ : 65.7%, 1.7, 57.4ms, 0 | 20260507 22:17:54.112 | INFO | BAZWRITER : 68.9%, 4.0, 141.7ms, 0 | 20260507 22:17:54.112 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 22:17:54.112 | INFO | Uploaded 36GiB to GPU in 2.55404s (14.0953GiB/s) | 20260507 22:17:54.112 | INFO | Downloaded 38GiB to GPU in 2.68667s (14.1439GiB/s) | 20260507 22:17:59.255 | INFO | Analyzing chunk frames = [3023360,3023872) | 20260507 22:17:59.255 | INFO | MemInfo: { | 20260507 22:17:59.255 | INFO | "gpu" : " 17 GB", | 20260507 22:17:59.255 | INFO | "huge" : "", | 20260507 22:17:59.255 | INFO | "huge_shared" : " 29 GB", | 20260507 22:17:59.255 | INFO | "malloc" : " 14 GB", | 20260507 22:17:59.255 | INFO | "pinned" : " 29 GB" | 20260507 22:17:59.255 | INFO | } | 20260507 22:17:59.255 | INFO | Perf report for the last 5.14378 seconds: | 20260507 22:17:59.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:17:59.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 22:17:59.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 22:17:59.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:17:59.256 | INFO | Uploaded 36GiB to GPU in 2.57618s (13.9742GiB/s) | 20260507 22:17:59.256 | INFO | Downloaded 34GiB to GPU in 2.46677s (13.7832GiB/s) | 20260507 22:18:04.397 | INFO | Analyzing chunk frames = [3023872,3024384) | 20260507 22:18:04.397 | INFO | MemInfo: { | 20260507 22:18:04.397 | INFO | "gpu" : " 17 GB", | 20260507 22:18:04.397 | INFO | "huge" : "", | 20260507 22:18:04.397 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:04.397 | INFO | "malloc" : " 14 GB", | 20260507 22:18:04.397 | INFO | "pinned" : " 29 GB" | 20260507 22:18:04.397 | INFO | } | 20260507 22:18:04.397 | INFO | Perf report for the last 5.14174 seconds: | 20260507 22:18:04.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:04.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 22:18:04.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:18:04.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:04.397 | INFO | Uploaded 36GiB to GPU in 2.55584s (14.0854GiB/s) | 20260507 22:18:04.398 | INFO | Downloaded 34GiB to GPU in 2.44623s (13.8989GiB/s) | 20260507 22:18:09.531 | INFO | Analyzing chunk frames = [3024384,3024896) | 20260507 22:18:09.531 | INFO | MemInfo: { | 20260507 22:18:09.531 | INFO | "gpu" : " 17 GB", | 20260507 22:18:09.531 | INFO | "huge" : "", | 20260507 22:18:09.532 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:09.532 | INFO | "malloc" : " 14 GB", | 20260507 22:18:09.532 | INFO | "pinned" : " 29 GB" | 20260507 22:18:09.532 | INFO | } | 20260507 22:18:09.532 | INFO | Perf report for the last 5.13457 seconds: | 20260507 22:18:09.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:09.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | ANALYSIS : 67.7%, 3.0, 104.6ms, 0 | 20260507 22:18:09.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:18:09.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:09.532 | INFO | Uploaded 36GiB to GPU in 2.55495s (14.0903GiB/s) | 20260507 22:18:09.532 | INFO | Downloaded 34GiB to GPU in 2.49097s (13.6493GiB/s) | 20260507 22:18:14.670 | INFO | Analyzing chunk frames = [3024896,3025408) | 20260507 22:18:14.670 | INFO | MemInfo: { | 20260507 22:18:14.670 | INFO | "gpu" : " 17 GB", | 20260507 22:18:14.670 | INFO | "huge" : "", | 20260507 22:18:14.670 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:14.670 | INFO | "malloc" : " 14 GB", | 20260507 22:18:14.670 | INFO | "pinned" : " 29 GB" | 20260507 22:18:14.670 | INFO | } | 20260507 22:18:14.670 | INFO | Perf report for the last 5.13852 seconds: | 20260507 22:18:14.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:14.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.670 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 22:18:14.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.670 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:18:14.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:14.671 | INFO | Uploaded 36GiB to GPU in 2.38503s (15.0941GiB/s) | 20260507 22:18:14.671 | INFO | Downloaded 34GiB to GPU in 2.48291s (13.6936GiB/s) | 20260507 22:18:19.811 | INFO | Analyzing chunk frames = [3025408,3025920) | 20260507 22:18:19.811 | INFO | MemInfo: { | 20260507 22:18:19.811 | INFO | "gpu" : " 17 GB", | 20260507 22:18:19.811 | INFO | "huge" : "", | 20260507 22:18:19.812 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:19.812 | INFO | "malloc" : " 14 GB", | 20260507 22:18:19.812 | INFO | "pinned" : " 29 GB" | 20260507 22:18:19.812 | INFO | } | 20260507 22:18:19.812 | INFO | Perf report for the last 5.14133 seconds: | 20260507 22:18:19.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:19.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:18:19.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:18:19.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:19.812 | INFO | Uploaded 36GiB to GPU in 2.52669s (14.2479GiB/s) | 20260507 22:18:19.812 | INFO | Downloaded 34GiB to GPU in 2.40963s (14.1101GiB/s) | 20260507 22:18:24.947 | INFO | Analyzing chunk frames = [3025920,3026432) | 20260507 22:18:24.947 | INFO | MemInfo: { | 20260507 22:18:24.947 | INFO | "gpu" : " 17 GB", | 20260507 22:18:24.947 | INFO | "huge" : "", | 20260507 22:18:24.947 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:24.947 | INFO | "malloc" : " 14 GB", | 20260507 22:18:24.947 | INFO | "pinned" : " 29 GB" | 20260507 22:18:24.947 | INFO | } | 20260507 22:18:24.947 | INFO | Perf report for the last 5.13542 seconds: | 20260507 22:18:24.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:24.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 22:18:24.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:18:24.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:24.947 | INFO | Uploaded 36GiB to GPU in 2.54943s (14.1208GiB/s) | 20260507 22:18:24.947 | INFO | Downloaded 34GiB to GPU in 2.44249s (13.9202GiB/s) | 20260507 22:18:30.085 | INFO | Analyzing chunk frames = [3026432,3026944) | 20260507 22:18:30.085 | INFO | MemInfo: { | 20260507 22:18:30.085 | INFO | "gpu" : " 17 GB", | 20260507 22:18:30.085 | INFO | "huge" : "", | 20260507 22:18:30.085 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:30.086 | INFO | "malloc" : " 14 GB", | 20260507 22:18:30.086 | INFO | "pinned" : " 29 GB" | 20260507 22:18:30.086 | INFO | } | 20260507 22:18:30.086 | INFO | Perf report for the last 5.13844 seconds: | 20260507 22:18:30.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:30.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 22:18:30.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:18:30.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:30.086 | INFO | Uploaded 36GiB to GPU in 2.52238s (14.2723GiB/s) | 20260507 22:18:30.086 | INFO | Downloaded 34GiB to GPU in 2.43519s (13.962GiB/s) | 20260507 22:18:30.183 | INFO | Starting a round of Baz IO | 20260507 22:18:33.672 | INFO | Populating RTMetrics report | 20260507 22:18:33.672 | INFO | RTMetrics report populated | 20260507 22:18:33.672 | INFO | Opening tmp RTMetrics json file | 20260507 22:18:33.693 | INFO | Serializing RTMetrics to json | 20260507 22:18:33.695 | INFO | Serialization done, swapping tmp file now... | 20260507 22:18:33.696 | INFO | RTMetrics done | 20260507 22:18:33.756 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.572 seconds | 20260507 22:18:33.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.20% / 0.01% / 56.80% | 20260507 22:18:33.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45546% / 7.94729e-06% / 7.94729e-05% | 20260507 22:18:33.756 | INFO | Baz IO overall: | 20260507 22:18:33.756 | INFO | BytesWritten : 1379120054033 | 20260507 22:18:33.756 | INFO | NumEvents : 668214696189 | 20260507 22:18:35.223 | INFO | Analyzing chunk frames = [3026944,3027456) | 20260507 22:18:35.223 | INFO | MemInfo: { | 20260507 22:18:35.223 | INFO | "gpu" : " 17 GB", | 20260507 22:18:35.223 | INFO | "huge" : "", | 20260507 22:18:35.224 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:35.224 | INFO | "malloc" : " 14 GB", | 20260507 22:18:35.224 | INFO | "pinned" : " 29 GB" | 20260507 22:18:35.224 | INFO | } | 20260507 22:18:35.224 | INFO | Perf report for the last 5.13807 seconds: | 20260507 22:18:35.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:35.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:35.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:35.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:35.224 | INFO | ANALYSIS : 69.1%, 2.9, 106.6ms, 0 | 20260507 22:18:35.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:35.224 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 22:18:35.224 | INFO | PRE_HQ : 67.1%, 1.7, 58.6ms, 0 | 20260507 22:18:35.224 | INFO | BAZWRITER : 69.1%, 4.1, 144.6ms, 0 | 20260507 22:18:35.224 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 22:18:35.224 | INFO | Uploaded 36GiB to GPU in 2.53269s (14.2141GiB/s) | 20260507 22:18:35.224 | INFO | Downloaded 38GiB to GPU in 2.69193s (14.1162GiB/s) | 20260507 22:18:40.367 | INFO | Analyzing chunk frames = [3027456,3027968) | 20260507 22:18:40.367 | INFO | MemInfo: { | 20260507 22:18:40.367 | INFO | "gpu" : " 17 GB", | 20260507 22:18:40.367 | INFO | "huge" : "", | 20260507 22:18:40.367 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:40.367 | INFO | "malloc" : " 14 GB", | 20260507 22:18:40.367 | INFO | "pinned" : " 29 GB" | 20260507 22:18:40.367 | INFO | } | 20260507 22:18:40.367 | INFO | Perf report for the last 5.14302 seconds: | 20260507 22:18:40.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:40.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:18:40.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:18:40.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:40.367 | INFO | Uploaded 36GiB to GPU in 2.50957s (14.3451GiB/s) | 20260507 22:18:40.367 | INFO | Downloaded 34GiB to GPU in 2.47886s (13.716GiB/s) | 20260507 22:18:45.510 | INFO | Analyzing chunk frames = [3027968,3028480) | 20260507 22:18:45.510 | INFO | MemInfo: { | 20260507 22:18:45.510 | INFO | "gpu" : " 17 GB", | 20260507 22:18:45.510 | INFO | "huge" : "", | 20260507 22:18:45.510 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:45.510 | INFO | "malloc" : " 14 GB", | 20260507 22:18:45.510 | INFO | "pinned" : " 29 GB" | 20260507 22:18:45.510 | INFO | } | 20260507 22:18:45.510 | INFO | Perf report for the last 5.14334 seconds: | 20260507 22:18:45.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:45.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:18:45.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:18:45.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:45.510 | INFO | Uploaded 36GiB to GPU in 2.52938s (14.2328GiB/s) | 20260507 22:18:45.510 | INFO | Downloaded 34GiB to GPU in 2.48478s (13.6833GiB/s) | 20260507 22:18:50.651 | INFO | Analyzing chunk frames = [3028480,3028992) | 20260507 22:18:50.651 | INFO | MemInfo: { | 20260507 22:18:50.651 | INFO | "gpu" : " 17 GB", | 20260507 22:18:50.651 | INFO | "huge" : "", | 20260507 22:18:50.651 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:50.651 | INFO | "malloc" : " 14 GB", | 20260507 22:18:50.651 | INFO | "pinned" : " 29 GB" | 20260507 22:18:50.651 | INFO | } | 20260507 22:18:50.651 | INFO | Perf report for the last 5.14129 seconds: | 20260507 22:18:50.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:50.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.651 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:18:50.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.651 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:18:50.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:50.652 | INFO | Uploaded 36GiB to GPU in 2.47975s (14.5176GiB/s) | 20260507 22:18:50.652 | INFO | Downloaded 34GiB to GPU in 2.41285s (14.0912GiB/s) | 20260507 22:18:55.786 | INFO | Analyzing chunk frames = [3028992,3029504) | 20260507 22:18:55.786 | INFO | MemInfo: { | 20260507 22:18:55.786 | INFO | "gpu" : " 17 GB", | 20260507 22:18:55.786 | INFO | "huge" : "", | 20260507 22:18:55.786 | INFO | "huge_shared" : " 29 GB", | 20260507 22:18:55.786 | INFO | "malloc" : " 14 GB", | 20260507 22:18:55.786 | INFO | "pinned" : " 29 GB" | 20260507 22:18:55.786 | INFO | } | 20260507 22:18:55.786 | INFO | Perf report for the last 5.13478 seconds: | 20260507 22:18:55.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:18:55.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 22:18:55.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:18:55.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:18:55.786 | INFO | Uploaded 36GiB to GPU in 2.31782s (15.5319GiB/s) | 20260507 22:18:55.787 | INFO | Downloaded 34GiB to GPU in 2.50099s (13.5946GiB/s) | 20260507 22:19:00.922 | INFO | Analyzing chunk frames = [3029504,3030016) | 20260507 22:19:00.922 | INFO | MemInfo: { | 20260507 22:19:00.922 | INFO | "gpu" : " 17 GB", | 20260507 22:19:00.922 | INFO | "huge" : "", | 20260507 22:19:00.922 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:00.922 | INFO | "malloc" : " 14 GB", | 20260507 22:19:00.922 | INFO | "pinned" : " 29 GB" | 20260507 22:19:00.923 | INFO | } | 20260507 22:19:00.923 | INFO | Perf report for the last 5.13639 seconds: | 20260507 22:19:00.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:00.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 22:19:00.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:19:00.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:00.923 | INFO | Uploaded 36GiB to GPU in 2.44216s (14.7411GiB/s) | 20260507 22:19:00.923 | INFO | Downloaded 34GiB to GPU in 2.43733s (13.9497GiB/s) | 20260507 22:19:06.065 | INFO | Analyzing chunk frames = [3030016,3030528) | 20260507 22:19:06.065 | INFO | MemInfo: { | 20260507 22:19:06.065 | INFO | "gpu" : " 17 GB", | 20260507 22:19:06.065 | INFO | "huge" : "", | 20260507 22:19:06.065 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:06.065 | INFO | "malloc" : " 14 GB", | 20260507 22:19:06.065 | INFO | "pinned" : " 29 GB" | 20260507 22:19:06.065 | INFO | } | 20260507 22:19:06.065 | INFO | Perf report for the last 5.14252 seconds: | 20260507 22:19:06.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:06.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 22:19:06.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 22:19:06.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:06.065 | INFO | Uploaded 36GiB to GPU in 2.50977s (14.3439GiB/s) | 20260507 22:19:06.065 | INFO | Downloaded 34GiB to GPU in 2.47063s (13.7617GiB/s) | 20260507 22:19:11.202 | INFO | Analyzing chunk frames = [3030528,3031040) | 20260507 22:19:11.202 | INFO | MemInfo: { | 20260507 22:19:11.202 | INFO | "gpu" : " 17 GB", | 20260507 22:19:11.202 | INFO | "huge" : "", | 20260507 22:19:11.202 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:11.202 | INFO | "malloc" : " 14 GB", | 20260507 22:19:11.202 | INFO | "pinned" : " 29 GB" | 20260507 22:19:11.202 | INFO | } | 20260507 22:19:11.202 | INFO | Perf report for the last 5.13678 seconds: | 20260507 22:19:11.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:11.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:19:11.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:19:11.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:11.202 | INFO | Uploaded 36GiB to GPU in 2.47181s (14.5642GiB/s) | 20260507 22:19:11.202 | INFO | Downloaded 34GiB to GPU in 2.45425s (13.8535GiB/s) | 20260507 22:19:11.202 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:19:11.202 | INFO | Total mem usage: 16.7 GiB | 20260507 22:19:11.202 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:19:11.202 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.54%) | 20260507 22:19:11.202 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:19:11.202 | INFO | High Water Breakdown: | 20260507 22:19:11.202 | INFO | 60.7% : 9.42 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:19:11.202 | INFO | 39.2% : 6.08 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:19:11.202 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:19:11.202 | INFO | ------------------------------------------ | 20260507 22:19:11.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:19:11.202 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:19:11.202 | INFO | Total mem usage: 26.5 GiB | 20260507 22:19:11.202 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:19:11.202 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:19:11.202 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:19:11.202 | INFO | High Water Breakdown: | 20260507 22:19:11.202 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:19:11.202 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:19:11.202 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:19:11.202 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:19:11.202 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:19:11.202 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:19:11.202 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:19:11.202 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:19:11.202 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:19:11.202 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:19:11.202 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:19:11.202 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:19:11.202 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:19:11.202 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:19:11.202 | INFO | ------------------------------------------ | 20260507 22:19:11.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:19:11.202 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:19:11.202 | INFO | Total mem usage: 27.8 GiB | 20260507 22:19:11.202 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:19:11.202 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:19:11.202 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:19:11.202 | INFO | High Water Breakdown: | 20260507 22:19:11.202 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:19:11.202 | INFO | ------------------------------------------ | 20260507 22:19:11.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:19:11.202 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:19:11.202 | INFO | Total mem usage: 15.7 GiB | 20260507 22:19:11.202 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:19:11.202 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:19:11.202 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:19:11.202 | INFO | High Water Breakdown: | 20260507 22:19:11.202 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:19:11.202 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:19:11.202 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:19:11.202 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:19:11.202 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:19:11.202 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:19:11.202 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:19:11.202 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:19:11.202 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:19:11.202 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:19:11.202 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:19:11.202 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:19:11.202 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:19:11.202 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:19:11.202 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:19:11.202 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:19:11.202 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:19:11.202 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:19:11.202 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:19:11.202 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:19:11.202 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:19:11.202 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:19:11.202 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:19:11.202 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:19:11.202 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:19:11.202 | INFO | ------------------------------------------ | 20260507 22:19:11.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:19:11.202 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:19:11.202 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:19:11.202 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:19:11.202 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.65e+01, 1.40e+01 / 1.79e+00 | 20260507 22:19:11.202 | INFO | PerfReport: 0.29%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.07e-02 / 4.67e-03 | 20260507 22:19:11.202 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.53e+00, 5.98e-01 / 1.49e-01 | 20260507 22:19:11.202 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.04e-01 | 20260507 22:19:11.202 | INFO | PerfReport: 15.76%, DME -- 4.16e+01 / 4.51e+01, 3.98e+00 / 1.64e+00 | 20260507 22:19:11.202 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.85e+00 / 1.53e+00 | 20260507 22:19:11.202 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.43e+00 / 2.71e+00, 4.28e-01 / 1.88e-01 | 20260507 22:19:11.202 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.37e+00, 2.95e-01 / 1.74e-01 | 20260507 22:19:11.202 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.55e-01 / 3.88e-01, 5.91e-02 / 3.00e-02 | 20260507 22:19:11.203 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:19:11.203 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:19:11.203 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:19:11.203 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 24.38%, Upload -- 2.61e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 30.90%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 2.52%, LockDownload -- 2.69e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 15.92%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:19:11.203 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 22:19:11.300 | INFO | Starting a round of Baz IO | 20260507 22:19:14.780 | INFO | Populating RTMetrics report | 20260507 22:19:14.780 | INFO | RTMetrics report populated | 20260507 22:19:14.780 | INFO | Opening tmp RTMetrics json file | 20260507 22:19:14.802 | INFO | Serializing RTMetrics to json | 20260507 22:19:14.811 | INFO | Serialization done, swapping tmp file now... | 20260507 22:19:14.814 | INFO | RTMetrics done | 20260507 22:19:14.869 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.569 seconds | 20260507 22:19:14.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.19% / 0.01% / 56.80% | 20260507 22:19:14.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45011% / 3.97364e-06% / 5.96046e-05% | 20260507 22:19:14.869 | INFO | Baz IO overall: | 20260507 22:19:14.869 | INFO | BytesWritten : 1380988413927 | 20260507 22:19:14.869 | INFO | NumEvents : 669116966015 | 20260507 22:19:16.344 | INFO | Analyzing chunk frames = [3031040,3031552) | 20260507 22:19:16.344 | INFO | MemInfo: { | 20260507 22:19:16.344 | INFO | "gpu" : " 17 GB", | 20260507 22:19:16.344 | INFO | "huge" : "", | 20260507 22:19:16.344 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:16.344 | INFO | "malloc" : " 14 GB", | 20260507 22:19:16.344 | INFO | "pinned" : " 29 GB" | 20260507 22:19:16.344 | INFO | } | 20260507 22:19:16.344 | INFO | Perf report for the last 5.14209 seconds: | 20260507 22:19:16.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:16.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:16.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:16.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:16.344 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 22:19:16.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:16.344 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 22:19:16.344 | INFO | PRE_HQ : 65.4%, 1.7, 57.7ms, 0 | 20260507 22:19:16.344 | INFO | BAZWRITER : 69.0%, 4.1, 146.0ms, 0 | 20260507 22:19:16.344 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 22:19:16.344 | INFO | Uploaded 36GiB to GPU in 2.49759s (14.4139GiB/s) | 20260507 22:19:16.344 | INFO | Downloaded 38GiB to GPU in 2.69112s (14.1205GiB/s) | 20260507 22:19:21.483 | INFO | Analyzing chunk frames = [3031552,3032064) | 20260507 22:19:21.483 | INFO | MemInfo: { | 20260507 22:19:21.483 | INFO | "gpu" : " 17 GB", | 20260507 22:19:21.483 | INFO | "huge" : "", | 20260507 22:19:21.483 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:21.483 | INFO | "malloc" : " 14 GB", | 20260507 22:19:21.483 | INFO | "pinned" : " 29 GB" | 20260507 22:19:21.483 | INFO | } | 20260507 22:19:21.483 | INFO | Perf report for the last 5.13911 seconds: | 20260507 22:19:21.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:21.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 22:19:21.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:19:21.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:21.483 | INFO | Uploaded 36GiB to GPU in 2.53647s (14.193GiB/s) | 20260507 22:19:21.484 | INFO | Downloaded 34GiB to GPU in 2.48641s (13.6743GiB/s) | 20260507 22:19:26.623 | INFO | Analyzing chunk frames = [3032064,3032576) | 20260507 22:19:26.623 | INFO | MemInfo: { | 20260507 22:19:26.623 | INFO | "gpu" : " 17 GB", | 20260507 22:19:26.623 | INFO | "huge" : "", | 20260507 22:19:26.623 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:26.623 | INFO | "malloc" : " 14 GB", | 20260507 22:19:26.623 | INFO | "pinned" : " 29 GB" | 20260507 22:19:26.623 | INFO | } | 20260507 22:19:26.623 | INFO | Perf report for the last 5.14015 seconds: | 20260507 22:19:26.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:26.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.623 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 22:19:26.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.623 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:19:26.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:26.624 | INFO | Uploaded 36GiB to GPU in 2.53491s (14.2017GiB/s) | 20260507 22:19:26.624 | INFO | Downloaded 34GiB to GPU in 2.48398s (13.6877GiB/s) | 20260507 22:19:31.762 | INFO | Analyzing chunk frames = [3032576,3033088) | 20260507 22:19:31.762 | INFO | MemInfo: { | 20260507 22:19:31.762 | INFO | "gpu" : " 17 GB", | 20260507 22:19:31.762 | INFO | "huge" : "", | 20260507 22:19:31.762 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:31.762 | INFO | "malloc" : " 14 GB", | 20260507 22:19:31.762 | INFO | "pinned" : " 29 GB" | 20260507 22:19:31.762 | INFO | } | 20260507 22:19:31.763 | INFO | Perf report for the last 5.13925 seconds: | 20260507 22:19:31.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:31.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 22:19:31.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:19:31.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:31.763 | INFO | Uploaded 36GiB to GPU in 2.467s (14.5926GiB/s) | 20260507 22:19:31.763 | INFO | Downloaded 34GiB to GPU in 2.50961s (13.5479GiB/s) | 20260507 22:19:36.898 | INFO | Analyzing chunk frames = [3033088,3033600) | 20260507 22:19:36.898 | INFO | MemInfo: { | 20260507 22:19:36.898 | INFO | "gpu" : " 17 GB", | 20260507 22:19:36.898 | INFO | "huge" : "", | 20260507 22:19:36.898 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:36.898 | INFO | "malloc" : " 14 GB", | 20260507 22:19:36.898 | INFO | "pinned" : " 29 GB" | 20260507 22:19:36.898 | INFO | } | 20260507 22:19:36.899 | INFO | Perf report for the last 5.136 seconds: | 20260507 22:19:36.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:36.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:19:36.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 22:19:36.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:36.899 | INFO | Uploaded 36GiB to GPU in 2.31859s (15.5267GiB/s) | 20260507 22:19:36.899 | INFO | Downloaded 34GiB to GPU in 2.49723s (13.6151GiB/s) | 20260507 22:19:42.035 | INFO | Analyzing chunk frames = [3033600,3034112) | 20260507 22:19:42.035 | INFO | MemInfo: { | 20260507 22:19:42.035 | INFO | "gpu" : " 17 GB", | 20260507 22:19:42.035 | INFO | "huge" : "", | 20260507 22:19:42.036 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:42.036 | INFO | "malloc" : " 14 GB", | 20260507 22:19:42.036 | INFO | "pinned" : " 29 GB" | 20260507 22:19:42.036 | INFO | } | 20260507 22:19:42.036 | INFO | Perf report for the last 5.1372 seconds: | 20260507 22:19:42.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:42.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.036 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:19:42.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.036 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 22:19:42.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:42.037 | INFO | Uploaded 36GiB to GPU in 2.48455s (14.4895GiB/s) | 20260507 22:19:42.037 | INFO | Downloaded 34GiB to GPU in 2.42517s (14.0196GiB/s) | 20260507 22:19:47.176 | INFO | Analyzing chunk frames = [3034112,3034624) | 20260507 22:19:47.176 | INFO | MemInfo: { | 20260507 22:19:47.176 | INFO | "gpu" : " 17 GB", | 20260507 22:19:47.176 | INFO | "huge" : "", | 20260507 22:19:47.176 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:47.176 | INFO | "malloc" : " 14 GB", | 20260507 22:19:47.176 | INFO | "pinned" : " 29 GB" | 20260507 22:19:47.176 | INFO | } | 20260507 22:19:47.176 | INFO | Perf report for the last 5.14045 seconds: | 20260507 22:19:47.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:47.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.176 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:19:47.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.176 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260507 22:19:47.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:47.177 | INFO | Uploaded 36GiB to GPU in 2.54091s (14.1682GiB/s) | 20260507 22:19:47.177 | INFO | Downloaded 34GiB to GPU in 2.46917s (13.7698GiB/s) | 20260507 22:19:52.317 | INFO | Analyzing chunk frames = [3034624,3035136) | 20260507 22:19:52.317 | INFO | MemInfo: { | 20260507 22:19:52.317 | INFO | "gpu" : " 17 GB", | 20260507 22:19:52.317 | INFO | "huge" : "", | 20260507 22:19:52.317 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:52.317 | INFO | "malloc" : " 14 GB", | 20260507 22:19:52.317 | INFO | "pinned" : " 29 GB" | 20260507 22:19:52.317 | INFO | } | 20260507 22:19:52.318 | INFO | Perf report for the last 5.14127 seconds: | 20260507 22:19:52.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:52.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:19:52.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 22:19:52.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:52.318 | INFO | Uploaded 36GiB to GPU in 2.50107s (14.3938GiB/s) | 20260507 22:19:52.318 | INFO | Downloaded 34GiB to GPU in 2.44947s (13.8806GiB/s) | 20260507 22:19:52.431 | INFO | Starting a round of Baz IO | 20260507 22:19:55.934 | INFO | Populating RTMetrics report | 20260507 22:19:55.934 | INFO | RTMetrics report populated | 20260507 22:19:55.934 | INFO | Opening tmp RTMetrics json file | 20260507 22:19:55.935 | INFO | Serializing RTMetrics to json | 20260507 22:19:55.937 | INFO | Serialization done, swapping tmp file now... | 20260507 22:19:55.938 | INFO | RTMetrics done | 20260507 22:19:56.032 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.6 seconds | 20260507 22:19:56.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.18% / 0.01% / 56.81% | 20260507 22:19:56.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45177% / 3.97364e-06% / 7.54992e-05% | 20260507 22:19:56.032 | INFO | Baz IO overall: | 20260507 22:19:56.032 | INFO | BytesWritten : 1382859046870 | 20260507 22:19:56.032 | INFO | NumEvents : 670021107084 | 20260507 22:19:57.461 | INFO | Analyzing chunk frames = [3035136,3035648) | 20260507 22:19:57.461 | INFO | MemInfo: { | 20260507 22:19:57.461 | INFO | "gpu" : " 17 GB", | 20260507 22:19:57.461 | INFO | "huge" : "", | 20260507 22:19:57.461 | INFO | "huge_shared" : " 29 GB", | 20260507 22:19:57.461 | INFO | "malloc" : " 14 GB", | 20260507 22:19:57.461 | INFO | "pinned" : " 29 GB" | 20260507 22:19:57.461 | INFO | } | 20260507 22:19:57.461 | INFO | Perf report for the last 5.14352 seconds: | 20260507 22:19:57.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:19:57.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:57.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:57.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:57.461 | INFO | ANALYSIS : 69.5%, 2.9, 105.0ms, 0 | 20260507 22:19:57.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:19:57.461 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260507 22:19:57.461 | INFO | PRE_HQ : 69.1%, 1.9, 69.3ms, 0 | 20260507 22:19:57.461 | INFO | BAZWRITER : 69.6%, 4.2, 150.1ms, 0 | 20260507 22:19:57.461 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 22:19:57.461 | INFO | Uploaded 36GiB to GPU in 2.56852s (14.0158GiB/s) | 20260507 22:19:57.461 | INFO | Downloaded 38GiB to GPU in 2.71185s (14.0126GiB/s) | 20260507 22:20:02.602 | INFO | Analyzing chunk frames = [3035648,3036160) | 20260507 22:20:02.602 | INFO | MemInfo: { | 20260507 22:20:02.602 | INFO | "gpu" : " 17 GB", | 20260507 22:20:02.602 | INFO | "huge" : "", | 20260507 22:20:02.602 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:02.602 | INFO | "malloc" : " 14 GB", | 20260507 22:20:02.602 | INFO | "pinned" : " 29 GB" | 20260507 22:20:02.602 | INFO | } | 20260507 22:20:02.602 | INFO | Perf report for the last 5.14077 seconds: | 20260507 22:20:02.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:02.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:20:02.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:20:02.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:02.602 | INFO | Uploaded 36GiB to GPU in 2.53005s (14.2289GiB/s) | 20260507 22:20:02.602 | INFO | Downloaded 34GiB to GPU in 2.45429s (13.8533GiB/s) | 20260507 22:20:07.736 | INFO | Analyzing chunk frames = [3036160,3036672) | 20260507 22:20:07.736 | INFO | MemInfo: { | 20260507 22:20:07.736 | INFO | "gpu" : " 17 GB", | 20260507 22:20:07.736 | INFO | "huge" : "", | 20260507 22:20:07.736 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:07.736 | INFO | "malloc" : " 14 GB", | 20260507 22:20:07.736 | INFO | "pinned" : " 29 GB" | 20260507 22:20:07.736 | INFO | } | 20260507 22:20:07.736 | INFO | Perf report for the last 5.13429 seconds: | 20260507 22:20:07.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:07.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 22:20:07.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:20:07.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:07.736 | INFO | Uploaded 36GiB to GPU in 2.52298s (14.2689GiB/s) | 20260507 22:20:07.737 | INFO | Downloaded 34GiB to GPU in 2.47261s (13.7507GiB/s) | 20260507 22:20:12.879 | INFO | Analyzing chunk frames = [3036672,3037184) | 20260507 22:20:12.879 | INFO | MemInfo: { | 20260507 22:20:12.879 | INFO | "gpu" : " 17 GB", | 20260507 22:20:12.879 | INFO | "huge" : "", | 20260507 22:20:12.879 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:12.879 | INFO | "malloc" : " 14 GB", | 20260507 22:20:12.879 | INFO | "pinned" : " 29 GB" | 20260507 22:20:12.879 | INFO | } | 20260507 22:20:12.879 | INFO | Perf report for the last 5.14273 seconds: | 20260507 22:20:12.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:12.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 22:20:12.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:20:12.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:12.879 | INFO | Uploaded 36GiB to GPU in 2.46946s (14.5781GiB/s) | 20260507 22:20:12.879 | INFO | Downloaded 34GiB to GPU in 2.37705s (14.3035GiB/s) | 20260507 22:20:18.018 | INFO | Analyzing chunk frames = [3037184,3037696) | 20260507 22:20:18.018 | INFO | MemInfo: { | 20260507 22:20:18.018 | INFO | "gpu" : " 17 GB", | 20260507 22:20:18.018 | INFO | "huge" : "", | 20260507 22:20:18.018 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:18.018 | INFO | "malloc" : " 14 GB", | 20260507 22:20:18.018 | INFO | "pinned" : " 29 GB" | 20260507 22:20:18.018 | INFO | } | 20260507 22:20:18.018 | INFO | Perf report for the last 5.13948 seconds: | 20260507 22:20:18.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:18.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.018 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 22:20:18.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.018 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:20:18.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:18.019 | INFO | Uploaded 36GiB to GPU in 2.37949s (15.1293GiB/s) | 20260507 22:20:18.019 | INFO | Downloaded 34GiB to GPU in 2.49225s (13.6423GiB/s) | 20260507 22:20:23.156 | INFO | Analyzing chunk frames = [3037696,3038208) | 20260507 22:20:23.156 | INFO | MemInfo: { | 20260507 22:20:23.156 | INFO | "gpu" : " 17 GB", | 20260507 22:20:23.156 | INFO | "huge" : "", | 20260507 22:20:23.156 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:23.156 | INFO | "malloc" : " 14 GB", | 20260507 22:20:23.156 | INFO | "pinned" : " 29 GB" | 20260507 22:20:23.156 | INFO | } | 20260507 22:20:23.157 | INFO | Perf report for the last 5.13816 seconds: | 20260507 22:20:23.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:23.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 22:20:23.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:20:23.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:23.157 | INFO | Uploaded 36GiB to GPU in 2.52491s (14.2579GiB/s) | 20260507 22:20:23.157 | INFO | Downloaded 34GiB to GPU in 2.41786s (14.062GiB/s) | 20260507 22:20:28.294 | INFO | Analyzing chunk frames = [3038208,3038720) | 20260507 22:20:28.294 | INFO | MemInfo: { | 20260507 22:20:28.294 | INFO | "gpu" : " 17 GB", | 20260507 22:20:28.294 | INFO | "huge" : "", | 20260507 22:20:28.294 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:28.294 | INFO | "malloc" : " 14 GB", | 20260507 22:20:28.294 | INFO | "pinned" : " 29 GB" | 20260507 22:20:28.294 | INFO | } | 20260507 22:20:28.294 | INFO | Perf report for the last 5.13751 seconds: | 20260507 22:20:28.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:28.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:20:28.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:20:28.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:28.294 | INFO | Uploaded 36GiB to GPU in 2.55715s (14.0782GiB/s) | 20260507 22:20:28.294 | INFO | Downloaded 34GiB to GPU in 2.46278s (13.8055GiB/s) | 20260507 22:20:33.434 | INFO | Analyzing chunk frames = [3038720,3039232) | 20260507 22:20:33.434 | INFO | MemInfo: { | 20260507 22:20:33.434 | INFO | "gpu" : " 17 GB", | 20260507 22:20:33.434 | INFO | "huge" : "", | 20260507 22:20:33.434 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:33.435 | INFO | "malloc" : " 14 GB", | 20260507 22:20:33.435 | INFO | "pinned" : " 29 GB" | 20260507 22:20:33.435 | INFO | } | 20260507 22:20:33.435 | INFO | Perf report for the last 5.14061 seconds: | 20260507 22:20:33.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:33.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:20:33.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 22:20:33.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:33.435 | INFO | Uploaded 36GiB to GPU in 2.52761s (14.2427GiB/s) | 20260507 22:20:33.435 | INFO | Downloaded 34GiB to GPU in 2.44062s (13.9309GiB/s) | 20260507 22:20:33.435 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:20:33.435 | INFO | Total mem usage: 16.7 GiB | 20260507 22:20:33.435 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:20:33.435 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.5%) | 20260507 22:20:33.435 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:20:33.435 | INFO | High Water Breakdown: | 20260507 22:20:33.435 | INFO | 60.1% : 8.99 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:20:33.435 | INFO | 39.8% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:20:33.435 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:20:33.435 | INFO | ------------------------------------------ | 20260507 22:20:33.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:20:33.436 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:20:33.436 | INFO | Total mem usage: 26.5 GiB | 20260507 22:20:33.436 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:20:33.436 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:20:33.436 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:20:33.436 | INFO | High Water Breakdown: | 20260507 22:20:33.436 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:20:33.436 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:20:33.436 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:20:33.436 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:20:33.436 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:20:33.436 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:20:33.436 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:20:33.436 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:20:33.436 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:20:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:20:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:20:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:20:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:20:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:20:33.436 | INFO | ------------------------------------------ | 20260507 22:20:33.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:20:33.436 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:20:33.436 | INFO | Total mem usage: 27.8 GiB | 20260507 22:20:33.436 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:20:33.436 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:20:33.436 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:20:33.436 | INFO | High Water Breakdown: | 20260507 22:20:33.436 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:20:33.436 | INFO | ------------------------------------------ | 20260507 22:20:33.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:20:33.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:20:33.436 | INFO | Total mem usage: 15.7 GiB | 20260507 22:20:33.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:20:33.436 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:20:33.436 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:20:33.436 | INFO | High Water Breakdown: | 20260507 22:20:33.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:20:33.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:20:33.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:20:33.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:20:33.436 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:20:33.436 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:20:33.436 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:20:33.436 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:20:33.436 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:20:33.436 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:20:33.436 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:20:33.436 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:20:33.436 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:20:33.436 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:20:33.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:20:33.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:20:33.436 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:20:33.436 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:20:33.436 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:20:33.436 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:20:33.436 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:20:33.436 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:20:33.436 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:20:33.436 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:20:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:20:33.436 | INFO | ------------------------------------------ | 20260507 22:20:33.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:20:33.436 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:20:33.436 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:20:33.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:20:33.436 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.64e+01, 1.42e+01 / 1.67e+00 | 20260507 22:20:33.436 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.04e-01, 1.22e-02 / 1.66e-02 | 20260507 22:20:33.437 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.45e+00, 5.92e-01 / 2.42e-01 | 20260507 22:20:33.437 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.10e-01 / 1.02e-01 | 20260507 22:20:33.437 | INFO | PerfReport: 16.00%, DME -- 4.21e+01 / 4.59e+01, 3.74e+00 / 1.33e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.71e+00 / 1.60e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.47e+00 / 2.73e+00, 4.35e-01 / 2.34e-01 | 20260507 22:20:33.437 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.39e+00, 2.90e-01 / 1.84e-01 | 20260507 22:20:33.437 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.57e-01 / 3.93e-01, 5.85e-02 / 3.34e-02 | 20260507 22:20:33.437 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:20:33.437 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:20:33.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:20:33.437 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 24.36%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 30.84%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 2.08%, LockUpload -- 2.23e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 2.66%, LockDownload -- 2.84e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 15.75%, LockCompute -- 1.68e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 22:20:33.437 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 22:20:33.552 | INFO | Starting a round of Baz IO | 20260507 22:20:37.064 | INFO | Populating RTMetrics report | 20260507 22:20:37.064 | INFO | RTMetrics report populated | 20260507 22:20:37.064 | INFO | Opening tmp RTMetrics json file | 20260507 22:20:37.066 | INFO | Serializing RTMetrics to json | 20260507 22:20:37.074 | INFO | Serialization done, swapping tmp file now... | 20260507 22:20:37.087 | INFO | RTMetrics done | 20260507 22:20:37.168 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.615 seconds | 20260507 22:20:37.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.18% / 0.01% / 56.82% | 20260507 22:20:37.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44687% / 3.97364e-06% / 5.16574e-05% | 20260507 22:20:37.168 | INFO | Baz IO overall: | 20260507 22:20:37.168 | INFO | BytesWritten : 1384727689580 | 20260507 22:20:37.168 | INFO | NumEvents : 670923768225 | 20260507 22:20:38.571 | INFO | Analyzing chunk frames = [3039232,3039744) | 20260507 22:20:38.571 | INFO | MemInfo: { | 20260507 22:20:38.571 | INFO | "gpu" : " 17 GB", | 20260507 22:20:38.571 | INFO | "huge" : "", | 20260507 22:20:38.571 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:38.571 | INFO | "malloc" : " 14 GB", | 20260507 22:20:38.571 | INFO | "pinned" : " 29 GB" | 20260507 22:20:38.571 | INFO | } | 20260507 22:20:38.571 | INFO | Perf report for the last 5.13636 seconds: | 20260507 22:20:38.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:38.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:38.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:38.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:38.571 | INFO | ANALYSIS : 69.3%, 2.9, 104.4ms, 0 | 20260507 22:20:38.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:38.571 | INFO | RT_METRICS : 49.4%, 1.0, 25.9ms, 0 | 20260507 22:20:38.571 | INFO | PRE_HQ : 68.6%, 1.9, 69.0ms, 0 | 20260507 22:20:38.571 | INFO | BAZWRITER : 70.1%, 4.2, 150.7ms, 0 | 20260507 22:20:38.571 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 22:20:38.571 | INFO | Uploaded 36GiB to GPU in 2.53432s (14.205GiB/s) | 20260507 22:20:38.571 | INFO | Downloaded 38GiB to GPU in 2.71065s (14.0188GiB/s) | 20260507 22:20:43.714 | INFO | Analyzing chunk frames = [3039744,3040256) | 20260507 22:20:43.714 | INFO | MemInfo: { | 20260507 22:20:43.714 | INFO | "gpu" : " 17 GB", | 20260507 22:20:43.714 | INFO | "huge" : "", | 20260507 22:20:43.714 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:43.714 | INFO | "malloc" : " 14 GB", | 20260507 22:20:43.714 | INFO | "pinned" : " 29 GB" | 20260507 22:20:43.714 | INFO | } | 20260507 22:20:43.715 | INFO | Perf report for the last 5.14344 seconds: | 20260507 22:20:43.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:43.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:20:43.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:20:43.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:43.715 | INFO | Uploaded 36GiB to GPU in 2.51402s (14.3197GiB/s) | 20260507 22:20:43.715 | INFO | Downloaded 34GiB to GPU in 2.44786s (13.8897GiB/s) | 20260507 22:20:48.850 | INFO | Analyzing chunk frames = [3040256,3040768) | 20260507 22:20:48.850 | INFO | MemInfo: { | 20260507 22:20:48.850 | INFO | "gpu" : " 17 GB", | 20260507 22:20:48.850 | INFO | "huge" : "", | 20260507 22:20:48.850 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:48.850 | INFO | "malloc" : " 14 GB", | 20260507 22:20:48.850 | INFO | "pinned" : " 29 GB" | 20260507 22:20:48.850 | INFO | } | 20260507 22:20:48.850 | INFO | Perf report for the last 5.13566 seconds: | 20260507 22:20:48.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:48.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.850 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 22:20:48.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.850 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:20:48.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:48.851 | INFO | Uploaded 36GiB to GPU in 2.58429s (13.9303GiB/s) | 20260507 22:20:48.851 | INFO | Downloaded 34GiB to GPU in 2.46135s (13.8136GiB/s) | 20260507 22:20:53.984 | INFO | Analyzing chunk frames = [3040768,3041280) | 20260507 22:20:53.984 | INFO | MemInfo: { | 20260507 22:20:53.984 | INFO | "gpu" : " 17 GB", | 20260507 22:20:53.984 | INFO | "huge" : "", | 20260507 22:20:53.984 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:53.984 | INFO | "malloc" : " 14 GB", | 20260507 22:20:53.984 | INFO | "pinned" : " 29 GB" | 20260507 22:20:53.984 | INFO | } | 20260507 22:20:53.984 | INFO | Perf report for the last 5.13415 seconds: | 20260507 22:20:53.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:53.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.984 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260507 22:20:53.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.984 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:20:53.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:53.985 | INFO | Uploaded 36GiB to GPU in 2.50691s (14.3603GiB/s) | 20260507 22:20:53.985 | INFO | Downloaded 34GiB to GPU in 2.38985s (14.2268GiB/s) | 20260507 22:20:59.127 | INFO | Analyzing chunk frames = [3041280,3041792) | 20260507 22:20:59.127 | INFO | MemInfo: { | 20260507 22:20:59.127 | INFO | "gpu" : " 17 GB", | 20260507 22:20:59.127 | INFO | "huge" : "", | 20260507 22:20:59.127 | INFO | "huge_shared" : " 29 GB", | 20260507 22:20:59.128 | INFO | "malloc" : " 14 GB", | 20260507 22:20:59.128 | INFO | "pinned" : " 29 GB" | 20260507 22:20:59.128 | INFO | } | 20260507 22:20:59.128 | INFO | Perf report for the last 5.14326 seconds: | 20260507 22:20:59.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:20:59.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 22:20:59.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:20:59.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:20:59.128 | INFO | Uploaded 36GiB to GPU in 2.36694s (15.2095GiB/s) | 20260507 22:20:59.128 | INFO | Downloaded 34GiB to GPU in 2.47475s (13.7388GiB/s) | 20260507 22:21:04.267 | INFO | Analyzing chunk frames = [3041792,3042304) | 20260507 22:21:04.267 | INFO | MemInfo: { | 20260507 22:21:04.267 | INFO | "gpu" : " 17 GB", | 20260507 22:21:04.267 | INFO | "huge" : "", | 20260507 22:21:04.267 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:04.267 | INFO | "malloc" : " 14 GB", | 20260507 22:21:04.267 | INFO | "pinned" : " 29 GB" | 20260507 22:21:04.267 | INFO | } | 20260507 22:21:04.267 | INFO | Perf report for the last 5.13949 seconds: | 20260507 22:21:04.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:04.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.267 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:21:04.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.267 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:21:04.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:04.268 | INFO | Uploaded 36GiB to GPU in 2.50581s (14.3666GiB/s) | 20260507 22:21:04.268 | INFO | Downloaded 34GiB to GPU in 2.42418s (14.0254GiB/s) | 20260507 22:21:09.416 | INFO | Analyzing chunk frames = [3042304,3042816) | 20260507 22:21:09.416 | INFO | MemInfo: { | 20260507 22:21:09.416 | INFO | "gpu" : " 17 GB", | 20260507 22:21:09.416 | INFO | "huge" : "", | 20260507 22:21:09.416 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:09.416 | INFO | "malloc" : " 14 GB", | 20260507 22:21:09.416 | INFO | "pinned" : " 29 GB" | 20260507 22:21:09.416 | INFO | } | 20260507 22:21:09.416 | INFO | Perf report for the last 5.14899 seconds: | 20260507 22:21:09.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:09.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.416 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:21:09.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.416 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:21:09.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:09.417 | INFO | Uploaded 36GiB to GPU in 2.54577s (14.1411GiB/s) | 20260507 22:21:09.417 | INFO | Downloaded 34GiB to GPU in 2.45821s (13.8312GiB/s) | 20260507 22:21:14.553 | INFO | Analyzing chunk frames = [3042816,3043328) | 20260507 22:21:14.553 | INFO | MemInfo: { | 20260507 22:21:14.553 | INFO | "gpu" : " 17 GB", | 20260507 22:21:14.553 | INFO | "huge" : "", | 20260507 22:21:14.553 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:14.553 | INFO | "malloc" : " 14 GB", | 20260507 22:21:14.553 | INFO | "pinned" : " 29 GB" | 20260507 22:21:14.553 | INFO | } | 20260507 22:21:14.553 | INFO | Perf report for the last 5.13712 seconds: | 20260507 22:21:14.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:14.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.553 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:21:14.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.554 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 22:21:14.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:14.554 | INFO | Uploaded 36GiB to GPU in 2.5028s (14.3839GiB/s) | 20260507 22:21:14.554 | INFO | Downloaded 34GiB to GPU in 2.43785s (13.9467GiB/s) | 20260507 22:21:14.667 | INFO | Starting a round of Baz IO | 20260507 22:21:18.167 | INFO | Populating RTMetrics report | 20260507 22:21:18.167 | INFO | RTMetrics report populated | 20260507 22:21:18.167 | INFO | Opening tmp RTMetrics json file | 20260507 22:21:18.174 | INFO | Serializing RTMetrics to json | 20260507 22:21:18.178 | INFO | Serialization done, swapping tmp file now... | 20260507 22:21:18.179 | INFO | RTMetrics done | 20260507 22:21:18.354 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.687 seconds | 20260507 22:21:18.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.17% / 0.01% / 56.82% | 20260507 22:21:18.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44441% / 1.98682e-05% / 3.97364e-05% | 20260507 22:21:18.354 | INFO | Baz IO overall: | 20260507 22:21:18.355 | INFO | BytesWritten : 1386596331214 | 20260507 22:21:18.355 | INFO | NumEvents : 671826542423 | 20260507 22:21:19.687 | INFO | Analyzing chunk frames = [3043328,3043840) | 20260507 22:21:19.687 | INFO | MemInfo: { | 20260507 22:21:19.687 | INFO | "gpu" : " 17 GB", | 20260507 22:21:19.687 | INFO | "huge" : "", | 20260507 22:21:19.687 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:19.687 | INFO | "malloc" : " 14 GB", | 20260507 22:21:19.687 | INFO | "pinned" : " 29 GB" | 20260507 22:21:19.687 | INFO | } | 20260507 22:21:19.687 | INFO | Perf report for the last 5.13407 seconds: | 20260507 22:21:19.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:19.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:19.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:19.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:19.687 | INFO | ANALYSIS : 69.4%, 2.9, 104.9ms, 0 | 20260507 22:21:19.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:19.687 | INFO | RT_METRICS : 49.3%, 1.0, 25.8ms, 0 | 20260507 22:21:19.687 | INFO | PRE_HQ : 69.2%, 1.9, 69.2ms, 0 | 20260507 22:21:19.687 | INFO | BAZWRITER : 71.4%, 4.3, 157.3ms, 0 | 20260507 22:21:19.688 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260507 22:21:19.688 | INFO | Uploaded 36GiB to GPU in 2.53867s (14.1806GiB/s) | 20260507 22:21:19.688 | INFO | Downloaded 38GiB to GPU in 2.7165s (13.9886GiB/s) | 20260507 22:21:24.831 | INFO | Analyzing chunk frames = [3043840,3044352) | 20260507 22:21:24.832 | INFO | MemInfo: { | 20260507 22:21:24.832 | INFO | "gpu" : " 17 GB", | 20260507 22:21:24.832 | INFO | "huge" : "", | 20260507 22:21:24.832 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:24.832 | INFO | "malloc" : " 14 GB", | 20260507 22:21:24.832 | INFO | "pinned" : " 29 GB" | 20260507 22:21:24.832 | INFO | } | 20260507 22:21:24.832 | INFO | Perf report for the last 5.14444 seconds: | 20260507 22:21:24.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:24.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:21:24.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:21:24.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:24.832 | INFO | Uploaded 36GiB to GPU in 2.51s (14.3426GiB/s) | 20260507 22:21:24.832 | INFO | Downloaded 34GiB to GPU in 2.5101s (13.5453GiB/s) | 20260507 22:21:29.967 | INFO | Analyzing chunk frames = [3044352,3044864) | 20260507 22:21:29.967 | INFO | MemInfo: { | 20260507 22:21:29.967 | INFO | "gpu" : " 17 GB", | 20260507 22:21:29.967 | INFO | "huge" : "", | 20260507 22:21:29.967 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:29.967 | INFO | "malloc" : " 14 GB", | 20260507 22:21:29.967 | INFO | "pinned" : " 29 GB" | 20260507 22:21:29.967 | INFO | } | 20260507 22:21:29.967 | INFO | Perf report for the last 5.13496 seconds: | 20260507 22:21:29.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:29.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:21:29.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:21:29.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:29.967 | INFO | Uploaded 36GiB to GPU in 2.47153s (14.5659GiB/s) | 20260507 22:21:29.967 | INFO | Downloaded 34GiB to GPU in 2.50027s (13.5985GiB/s) | 20260507 22:21:35.107 | INFO | Analyzing chunk frames = [3044864,3045376) | 20260507 22:21:35.107 | INFO | MemInfo: { | 20260507 22:21:35.107 | INFO | "gpu" : " 17 GB", | 20260507 22:21:35.107 | INFO | "huge" : "", | 20260507 22:21:35.107 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:35.107 | INFO | "malloc" : " 14 GB", | 20260507 22:21:35.107 | INFO | "pinned" : " 29 GB" | 20260507 22:21:35.107 | INFO | } | 20260507 22:21:35.107 | INFO | Perf report for the last 5.14023 seconds: | 20260507 22:21:35.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:35.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260507 22:21:35.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:21:35.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:35.107 | INFO | Uploaded 36GiB to GPU in 2.40751s (14.9532GiB/s) | 20260507 22:21:35.107 | INFO | Downloaded 34GiB to GPU in 2.40042s (14.1642GiB/s) | 20260507 22:21:40.245 | INFO | Analyzing chunk frames = [3045376,3045888) | 20260507 22:21:40.245 | INFO | MemInfo: { | 20260507 22:21:40.245 | INFO | "gpu" : " 17 GB", | 20260507 22:21:40.245 | INFO | "huge" : "", | 20260507 22:21:40.245 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:40.245 | INFO | "malloc" : " 14 GB", | 20260507 22:21:40.245 | INFO | "pinned" : " 29 GB" | 20260507 22:21:40.245 | INFO | } | 20260507 22:21:40.245 | INFO | Perf report for the last 5.13834 seconds: | 20260507 22:21:40.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:40.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.245 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:21:40.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.245 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:21:40.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:40.246 | INFO | Uploaded 36GiB to GPU in 2.32544s (15.4809GiB/s) | 20260507 22:21:40.246 | INFO | Downloaded 34GiB to GPU in 2.50838s (13.5546GiB/s) | 20260507 22:21:45.383 | INFO | Analyzing chunk frames = [3045888,3046400) | 20260507 22:21:45.383 | INFO | MemInfo: { | 20260507 22:21:45.383 | INFO | "gpu" : " 17 GB", | 20260507 22:21:45.383 | INFO | "huge" : "", | 20260507 22:21:45.383 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:45.383 | INFO | "malloc" : " 14 GB", | 20260507 22:21:45.383 | INFO | "pinned" : " 29 GB" | 20260507 22:21:45.383 | INFO | } | 20260507 22:21:45.383 | INFO | Perf report for the last 5.13811 seconds: | 20260507 22:21:45.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:45.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.383 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260507 22:21:45.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.384 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:21:45.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:45.384 | INFO | Uploaded 36GiB to GPU in 2.49513s (14.4281GiB/s) | 20260507 22:21:45.384 | INFO | Downloaded 34GiB to GPU in 2.42704s (14.0088GiB/s) | 20260507 22:21:50.523 | INFO | Analyzing chunk frames = [3046400,3046912) | 20260507 22:21:50.523 | INFO | MemInfo: { | 20260507 22:21:50.523 | INFO | "gpu" : " 17 GB", | 20260507 22:21:50.523 | INFO | "huge" : "", | 20260507 22:21:50.523 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:50.523 | INFO | "malloc" : " 14 GB", | 20260507 22:21:50.523 | INFO | "pinned" : " 29 GB" | 20260507 22:21:50.523 | INFO | } | 20260507 22:21:50.523 | INFO | Perf report for the last 5.13961 seconds: | 20260507 22:21:50.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:50.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.523 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 22:21:50.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.523 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:21:50.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:50.524 | INFO | Uploaded 36GiB to GPU in 2.50499s (14.3713GiB/s) | 20260507 22:21:50.524 | INFO | Downloaded 34GiB to GPU in 2.4623s (13.8082GiB/s) | 20260507 22:21:55.665 | INFO | Analyzing chunk frames = [3046912,3047424) | 20260507 22:21:55.665 | INFO | MemInfo: { | 20260507 22:21:55.665 | INFO | "gpu" : " 17 GB", | 20260507 22:21:55.665 | INFO | "huge" : "", | 20260507 22:21:55.665 | INFO | "huge_shared" : " 29 GB", | 20260507 22:21:55.665 | INFO | "malloc" : " 14 GB", | 20260507 22:21:55.665 | INFO | "pinned" : " 29 GB" | 20260507 22:21:55.665 | INFO | } | 20260507 22:21:55.665 | INFO | Perf report for the last 5.14174 seconds: | 20260507 22:21:55.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:21:55.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:21:55.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 22:21:55.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:21:55.665 | INFO | Uploaded 36GiB to GPU in 2.48006s (14.5158GiB/s) | 20260507 22:21:55.665 | INFO | Downloaded 34GiB to GPU in 2.44202s (13.9229GiB/s) | 20260507 22:21:55.665 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:21:55.665 | INFO | Total mem usage: 16.7 GiB | 20260507 22:21:55.665 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:21:55.665 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.2%) | 20260507 22:21:55.665 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:21:55.665 | INFO | High Water Breakdown: | 20260507 22:21:55.665 | INFO | 59.8% : 8.81 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:21:55.665 | INFO | 40% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:21:55.665 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:21:55.665 | INFO | ------------------------------------------ | 20260507 22:21:55.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:21:55.665 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:21:55.665 | INFO | Total mem usage: 26.5 GiB | 20260507 22:21:55.665 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:21:55.665 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:21:55.665 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:21:55.665 | INFO | High Water Breakdown: | 20260507 22:21:55.665 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:21:55.665 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:21:55.665 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:21:55.665 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:21:55.665 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:21:55.665 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:21:55.666 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:21:55.666 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:21:55.666 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:21:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:21:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:21:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:21:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:21:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:21:55.666 | INFO | ------------------------------------------ | 20260507 22:21:55.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:21:55.666 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:21:55.666 | INFO | Total mem usage: 27.8 GiB | 20260507 22:21:55.666 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:21:55.666 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:21:55.666 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:21:55.666 | INFO | High Water Breakdown: | 20260507 22:21:55.666 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:21:55.666 | INFO | ------------------------------------------ | 20260507 22:21:55.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:21:55.666 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:21:55.666 | INFO | Total mem usage: 15.7 GiB | 20260507 22:21:55.666 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:21:55.666 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:21:55.666 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:21:55.666 | INFO | High Water Breakdown: | 20260507 22:21:55.666 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:21:55.666 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:21:55.666 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:21:55.666 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:21:55.666 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:21:55.666 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:21:55.666 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:21:55.666 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:21:55.666 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:21:55.666 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:21:55.666 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:21:55.666 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:21:55.666 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:21:55.666 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:21:55.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:21:55.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:21:55.666 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:21:55.666 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:21:55.666 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:21:55.667 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:21:55.667 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:21:55.667 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:21:55.667 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:21:55.667 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:21:55.667 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:21:55.667 | INFO | ------------------------------------------ | 20260507 22:21:55.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:21:55.667 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:21:55.667 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:21:55.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:21:55.667 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.59e+01, 1.41e+01 / 1.62e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.04e-01, 1.13e-02 / 6.63e-03 | 20260507 22:21:55.667 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.49e+00, 6.00e-01 / 2.17e-01 | 20260507 22:21:55.667 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 9.98e-02 | 20260507 22:21:55.667 | INFO | PerfReport: 15.90%, DME -- 4.19e+01 / 4.56e+01, 3.94e+00 / 1.45e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.76e+00 / 1.63e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.46e+00 / 2.73e+00, 4.33e-01 / 2.27e-01 | 20260507 22:21:55.667 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.34e+00, 2.94e-01 / 1.55e-01 | 20260507 22:21:55.667 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.85e-01, 5.90e-02 / 2.80e-02 | 20260507 22:21:55.667 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:21:55.667 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:21:55.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:21:55.667 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 24.34%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 30.99%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 1.93%, LockUpload -- 2.05e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 2.63%, LockDownload -- 2.80e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 15.77%, LockCompute -- 1.68e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260507 22:21:55.667 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260507 22:21:55.762 | INFO | Starting a round of Baz IO | 20260507 22:21:59.238 | INFO | Populating RTMetrics report | 20260507 22:21:59.238 | INFO | RTMetrics report populated | 20260507 22:21:59.238 | INFO | Opening tmp RTMetrics json file | 20260507 22:21:59.263 | INFO | Serializing RTMetrics to json | 20260507 22:21:59.265 | INFO | Serialization done, swapping tmp file now... | 20260507 22:21:59.265 | INFO | RTMetrics done | 20260507 22:21:59.341 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 3.578 seconds | 20260507 22:21:59.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.16% / 0.01% / 56.83% | 20260507 22:21:59.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44075% / 3.97364e-06% / 5.96046e-05% | 20260507 22:21:59.341 | INFO | Baz IO overall: | 20260507 22:21:59.341 | INFO | BytesWritten : 1388463800126 | 20260507 22:21:59.341 | INFO | NumEvents : 672728389628 | 20260507 22:22:00.802 | INFO | Analyzing chunk frames = [3047424,3047936) | 20260507 22:22:00.802 | INFO | MemInfo: { | 20260507 22:22:00.802 | INFO | "gpu" : " 17 GB", | 20260507 22:22:00.802 | INFO | "huge" : "", | 20260507 22:22:00.802 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:00.802 | INFO | "malloc" : " 14 GB", | 20260507 22:22:00.802 | INFO | "pinned" : " 29 GB" | 20260507 22:22:00.802 | INFO | } | 20260507 22:22:00.802 | INFO | Perf report for the last 5.13711 seconds: | 20260507 22:22:00.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:00.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:00.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:00.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:00.802 | INFO | ANALYSIS : 68.8%, 3.0, 107.1ms, 0 | 20260507 22:22:00.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:00.802 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 22:22:00.802 | INFO | PRE_HQ : 65.6%, 1.7, 57.6ms, 0 | 20260507 22:22:00.802 | INFO | BAZWRITER : 69.2%, 3.8, 133.8ms, 0 | 20260507 22:22:00.802 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 22:22:00.802 | INFO | Uploaded 36GiB to GPU in 2.53713s (14.1893GiB/s) | 20260507 22:22:00.803 | INFO | Downloaded 38GiB to GPU in 2.69172s (14.1174GiB/s) | 20260507 22:22:05.940 | INFO | Analyzing chunk frames = [3047936,3048448) | 20260507 22:22:05.940 | INFO | MemInfo: { | 20260507 22:22:05.940 | INFO | "gpu" : " 17 GB", | 20260507 22:22:05.940 | INFO | "huge" : "", | 20260507 22:22:05.941 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:05.941 | INFO | "malloc" : " 14 GB", | 20260507 22:22:05.941 | INFO | "pinned" : " 29 GB" | 20260507 22:22:05.941 | INFO | } | 20260507 22:22:05.941 | INFO | Perf report for the last 5.1388 seconds: | 20260507 22:22:05.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:05.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 22:22:05.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:22:05.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:05.941 | INFO | Uploaded 36GiB to GPU in 2.5287s (14.2365GiB/s) | 20260507 22:22:05.941 | INFO | Downloaded 34GiB to GPU in 2.48056s (13.7066GiB/s) | 20260507 22:22:11.082 | INFO | Analyzing chunk frames = [3048448,3048960) | 20260507 22:22:11.082 | INFO | MemInfo: { | 20260507 22:22:11.082 | INFO | "gpu" : " 17 GB", | 20260507 22:22:11.082 | INFO | "huge" : "", | 20260507 22:22:11.082 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:11.082 | INFO | "malloc" : " 14 GB", | 20260507 22:22:11.082 | INFO | "pinned" : " 29 GB" | 20260507 22:22:11.082 | INFO | } | 20260507 22:22:11.082 | INFO | Perf report for the last 5.14149 seconds: | 20260507 22:22:11.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:11.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.082 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:22:11.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.082 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:22:11.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:11.083 | INFO | Uploaded 36GiB to GPU in 2.52923s (14.2336GiB/s) | 20260507 22:22:11.083 | INFO | Downloaded 34GiB to GPU in 2.46577s (13.7888GiB/s) | 20260507 22:22:16.222 | INFO | Analyzing chunk frames = [3048960,3049472) | 20260507 22:22:16.222 | INFO | MemInfo: { | 20260507 22:22:16.222 | INFO | "gpu" : " 17 GB", | 20260507 22:22:16.222 | INFO | "huge" : "", | 20260507 22:22:16.222 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:16.222 | INFO | "malloc" : " 14 GB", | 20260507 22:22:16.222 | INFO | "pinned" : " 29 GB" | 20260507 22:22:16.222 | INFO | } | 20260507 22:22:16.222 | INFO | Perf report for the last 5.1399 seconds: | 20260507 22:22:16.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:16.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.222 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:22:16.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.222 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:22:16.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:16.223 | INFO | Uploaded 36GiB to GPU in 2.46624s (14.5971GiB/s) | 20260507 22:22:16.223 | INFO | Downloaded 34GiB to GPU in 2.40558s (14.1338GiB/s) | 20260507 22:22:21.362 | INFO | Analyzing chunk frames = [3049472,3049984) | 20260507 22:22:21.362 | INFO | MemInfo: { | 20260507 22:22:21.362 | INFO | "gpu" : " 17 GB", | 20260507 22:22:21.362 | INFO | "huge" : "", | 20260507 22:22:21.362 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:21.362 | INFO | "malloc" : " 14 GB", | 20260507 22:22:21.362 | INFO | "pinned" : " 29 GB" | 20260507 22:22:21.362 | INFO | } | 20260507 22:22:21.362 | INFO | Perf report for the last 5.14022 seconds: | 20260507 22:22:21.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:21.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.363 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 22:22:21.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.363 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:22:21.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:21.363 | INFO | Uploaded 36GiB to GPU in 2.37325s (15.1691GiB/s) | 20260507 22:22:21.363 | INFO | Downloaded 34GiB to GPU in 2.49814s (13.6101GiB/s) | 20260507 22:22:26.498 | INFO | Analyzing chunk frames = [3049984,3050496) | 20260507 22:22:26.498 | INFO | MemInfo: { | 20260507 22:22:26.498 | INFO | "gpu" : " 17 GB", | 20260507 22:22:26.498 | INFO | "huge" : "", | 20260507 22:22:26.498 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:26.498 | INFO | "malloc" : " 14 GB", | 20260507 22:22:26.499 | INFO | "pinned" : " 29 GB" | 20260507 22:22:26.499 | INFO | } | 20260507 22:22:26.499 | INFO | Perf report for the last 5.13623 seconds: | 20260507 22:22:26.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:26.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:22:26.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:22:26.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:26.499 | INFO | Uploaded 36GiB to GPU in 2.4947s (14.4306GiB/s) | 20260507 22:22:26.499 | INFO | Downloaded 34GiB to GPU in 2.44689s (13.8952GiB/s) | 20260507 22:22:31.637 | INFO | Analyzing chunk frames = [3050496,3051008) | 20260507 22:22:31.637 | INFO | MemInfo: { | 20260507 22:22:31.637 | INFO | "gpu" : " 17 GB", | 20260507 22:22:31.637 | INFO | "huge" : "", | 20260507 22:22:31.637 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:31.637 | INFO | "malloc" : " 14 GB", | 20260507 22:22:31.637 | INFO | "pinned" : " 29 GB" | 20260507 22:22:31.637 | INFO | } | 20260507 22:22:31.637 | INFO | Perf report for the last 5.13848 seconds: | 20260507 22:22:31.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:31.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:22:31.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260507 22:22:31.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:31.637 | INFO | Uploaded 36GiB to GPU in 2.50968s (14.3445GiB/s) | 20260507 22:22:31.638 | INFO | Downloaded 34GiB to GPU in 2.47195s (13.7543GiB/s) | 20260507 22:22:36.780 | INFO | Analyzing chunk frames = [3051008,3051520) | 20260507 22:22:36.780 | INFO | MemInfo: { | 20260507 22:22:36.780 | INFO | "gpu" : " 17 GB", | 20260507 22:22:36.780 | INFO | "huge" : "", | 20260507 22:22:36.780 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:36.780 | INFO | "malloc" : " 14 GB", | 20260507 22:22:36.781 | INFO | "pinned" : " 29 GB" | 20260507 22:22:36.781 | INFO | } | 20260507 22:22:36.781 | INFO | Perf report for the last 5.14343 seconds: | 20260507 22:22:36.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:36.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 22:22:36.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:22:36.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:36.781 | INFO | Uploaded 36GiB to GPU in 2.48886s (14.4644GiB/s) | 20260507 22:22:36.781 | INFO | Downloaded 34GiB to GPU in 2.45924s (13.8254GiB/s) | 20260507 22:22:36.893 | INFO | Starting a round of Baz IO | 20260507 22:22:40.388 | INFO | Populating RTMetrics report | 20260507 22:22:40.388 | INFO | RTMetrics report populated | 20260507 22:22:40.388 | INFO | Opening tmp RTMetrics json file | 20260507 22:22:40.446 | INFO | Serializing RTMetrics to json | 20260507 22:22:40.509 | INFO | Serialization done, swapping tmp file now... | 20260507 22:22:40.522 | INFO | RTMetrics done | 20260507 22:22:40.538 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 3.644 seconds | 20260507 22:22:40.538 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.16% / 0.01% / 56.84% | 20260507 22:22:40.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43344% / 7.94729e-06% / 5.16574e-05% | 20260507 22:22:40.538 | INFO | Baz IO overall: | 20260507 22:22:40.538 | INFO | BytesWritten : 1390330549760 | 20260507 22:22:40.538 | INFO | NumEvents : 673629483729 | 20260507 22:22:41.920 | INFO | Analyzing chunk frames = [3051520,3052032) | 20260507 22:22:41.920 | INFO | MemInfo: { | 20260507 22:22:41.920 | INFO | "gpu" : " 17 GB", | 20260507 22:22:41.920 | INFO | "huge" : "", | 20260507 22:22:41.920 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:41.920 | INFO | "malloc" : " 14 GB", | 20260507 22:22:41.920 | INFO | "pinned" : " 29 GB" | 20260507 22:22:41.920 | INFO | } | 20260507 22:22:41.920 | INFO | Perf report for the last 5.13963 seconds: | 20260507 22:22:41.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:41.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:41.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:41.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:41.920 | INFO | ANALYSIS : 69.0%, 2.9, 105.0ms, 0 | 20260507 22:22:41.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:41.920 | INFO | RT_METRICS : 50.9%, 1.0, 26.7ms, 0 | 20260507 22:22:41.920 | INFO | PRE_HQ : 68.5%, 1.9, 69.0ms, 0 | 20260507 22:22:41.920 | INFO | BAZWRITER : 70.5%, 4.6, 167.0ms, 0 | 20260507 22:22:41.920 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 22:22:41.920 | INFO | Uploaded 36GiB to GPU in 2.59452s (13.8754GiB/s) | 20260507 22:22:41.920 | INFO | Downloaded 38GiB to GPU in 2.70301s (14.0584GiB/s) | 20260507 22:22:47.053 | INFO | Analyzing chunk frames = [3052032,3052544) | 20260507 22:22:47.053 | INFO | MemInfo: { | 20260507 22:22:47.053 | INFO | "gpu" : " 17 GB", | 20260507 22:22:47.053 | INFO | "huge" : "", | 20260507 22:22:47.053 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:47.053 | INFO | "malloc" : " 14 GB", | 20260507 22:22:47.053 | INFO | "pinned" : " 29 GB" | 20260507 22:22:47.053 | INFO | } | 20260507 22:22:47.053 | INFO | Perf report for the last 5.13288 seconds: | 20260507 22:22:47.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:47.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260507 22:22:47.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:22:47.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:47.053 | INFO | Uploaded 36GiB to GPU in 2.5627s (14.0477GiB/s) | 20260507 22:22:47.053 | INFO | Downloaded 34GiB to GPU in 2.48764s (13.6675GiB/s) | 20260507 22:22:52.191 | INFO | Analyzing chunk frames = [3052544,3053056) | 20260507 22:22:52.191 | INFO | MemInfo: { | 20260507 22:22:52.191 | INFO | "gpu" : " 17 GB", | 20260507 22:22:52.191 | INFO | "huge" : "", | 20260507 22:22:52.191 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:52.191 | INFO | "malloc" : " 14 GB", | 20260507 22:22:52.191 | INFO | "pinned" : " 29 GB" | 20260507 22:22:52.191 | INFO | } | 20260507 22:22:52.191 | INFO | Perf report for the last 5.1379 seconds: | 20260507 22:22:52.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:52.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:22:52.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:22:52.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:52.191 | INFO | Uploaded 36GiB to GPU in 2.53336s (14.2104GiB/s) | 20260507 22:22:52.191 | INFO | Downloaded 34GiB to GPU in 2.45378s (13.8562GiB/s) | 20260507 22:22:57.336 | INFO | Analyzing chunk frames = [3053056,3053568) | 20260507 22:22:57.336 | INFO | MemInfo: { | 20260507 22:22:57.336 | INFO | "gpu" : " 17 GB", | 20260507 22:22:57.336 | INFO | "huge" : "", | 20260507 22:22:57.336 | INFO | "huge_shared" : " 29 GB", | 20260507 22:22:57.336 | INFO | "malloc" : " 14 GB", | 20260507 22:22:57.336 | INFO | "pinned" : " 29 GB" | 20260507 22:22:57.336 | INFO | } | 20260507 22:22:57.336 | INFO | Perf report for the last 5.14485 seconds: | 20260507 22:22:57.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:22:57.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260507 22:22:57.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:22:57.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:22:57.336 | INFO | Uploaded 36GiB to GPU in 2.47612s (14.5389GiB/s) | 20260507 22:22:57.336 | INFO | Downloaded 34GiB to GPU in 2.39215s (14.2131GiB/s) | 20260507 22:23:02.476 | INFO | Analyzing chunk frames = [3053568,3054080) | 20260507 22:23:02.476 | INFO | MemInfo: { | 20260507 22:23:02.476 | INFO | "gpu" : " 17 GB", | 20260507 22:23:02.476 | INFO | "huge" : "", | 20260507 22:23:02.476 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:02.476 | INFO | "malloc" : " 14 GB", | 20260507 22:23:02.476 | INFO | "pinned" : " 29 GB" | 20260507 22:23:02.476 | INFO | } | 20260507 22:23:02.476 | INFO | Perf report for the last 5.13982 seconds: | 20260507 22:23:02.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:02.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 22:23:02.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 22:23:02.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:02.476 | INFO | Uploaded 36GiB to GPU in 2.4219s (14.8644GiB/s) | 20260507 22:23:02.476 | INFO | Downloaded 34GiB to GPU in 2.49846s (13.6084GiB/s) | 20260507 22:23:07.615 | INFO | Analyzing chunk frames = [3054080,3054592) | 20260507 22:23:07.615 | INFO | MemInfo: { | 20260507 22:23:07.615 | INFO | "gpu" : " 17 GB", | 20260507 22:23:07.615 | INFO | "huge" : "", | 20260507 22:23:07.615 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:07.615 | INFO | "malloc" : " 14 GB", | 20260507 22:23:07.615 | INFO | "pinned" : " 29 GB" | 20260507 22:23:07.615 | INFO | } | 20260507 22:23:07.615 | INFO | Perf report for the last 5.13932 seconds: | 20260507 22:23:07.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:07.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.615 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:23:07.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.615 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:23:07.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:07.616 | INFO | Uploaded 36GiB to GPU in 2.51939s (14.2892GiB/s) | 20260507 22:23:07.616 | INFO | Downloaded 34GiB to GPU in 2.41186s (14.097GiB/s) | 20260507 22:23:12.752 | INFO | Analyzing chunk frames = [3054592,3055104) | 20260507 22:23:12.752 | INFO | MemInfo: { | 20260507 22:23:12.752 | INFO | "gpu" : " 17 GB", | 20260507 22:23:12.752 | INFO | "huge" : "", | 20260507 22:23:12.752 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:12.752 | INFO | "malloc" : " 14 GB", | 20260507 22:23:12.752 | INFO | "pinned" : " 29 GB" | 20260507 22:23:12.753 | INFO | } | 20260507 22:23:12.753 | INFO | Perf report for the last 5.13744 seconds: | 20260507 22:23:12.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:12.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:23:12.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:23:12.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:12.753 | INFO | Uploaded 36GiB to GPU in 2.53237s (14.2159GiB/s) | 20260507 22:23:12.753 | INFO | Downloaded 34GiB to GPU in 2.42807s (14.0029GiB/s) | 20260507 22:23:17.895 | INFO | Analyzing chunk frames = [3055104,3055616) | 20260507 22:23:17.895 | INFO | MemInfo: { | 20260507 22:23:17.895 | INFO | "gpu" : " 17 GB", | 20260507 22:23:17.895 | INFO | "huge" : "", | 20260507 22:23:17.896 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:17.896 | INFO | "malloc" : " 14 GB", | 20260507 22:23:17.896 | INFO | "pinned" : " 29 GB" | 20260507 22:23:17.896 | INFO | } | 20260507 22:23:17.896 | INFO | Perf report for the last 5.14327 seconds: | 20260507 22:23:17.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:17.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 22:23:17.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 22:23:17.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:17.896 | INFO | Uploaded 36GiB to GPU in 2.54425s (14.1496GiB/s) | 20260507 22:23:17.896 | INFO | Downloaded 34GiB to GPU in 2.43305s (13.9742GiB/s) | 20260507 22:23:17.896 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:23:17.896 | INFO | Total mem usage: 16.7 GiB | 20260507 22:23:17.896 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:23:17.896 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.8%) | 20260507 22:23:17.896 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:23:17.896 | INFO | High Water Breakdown: | 20260507 22:23:17.896 | INFO | 60.2% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:23:17.896 | INFO | 39.7% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:23:17.897 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:23:17.897 | INFO | ------------------------------------------ | 20260507 22:23:17.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:23:17.897 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:23:17.897 | INFO | Total mem usage: 26.5 GiB | 20260507 22:23:17.897 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:23:17.897 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:23:17.897 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:23:17.897 | INFO | High Water Breakdown: | 20260507 22:23:17.897 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:23:17.897 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:23:17.897 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:23:17.897 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:23:17.897 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:23:17.897 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:23:17.897 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:23:17.897 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:23:17.897 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:23:17.897 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:23:17.897 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:23:17.897 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:23:17.897 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:23:17.897 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:23:17.897 | INFO | ------------------------------------------ | 20260507 22:23:17.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:23:17.897 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:23:17.897 | INFO | Total mem usage: 27.8 GiB | 20260507 22:23:17.897 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:23:17.897 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:23:17.897 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:23:17.897 | INFO | High Water Breakdown: | 20260507 22:23:17.897 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:23:17.897 | INFO | ------------------------------------------ | 20260507 22:23:17.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:23:17.897 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:23:17.897 | INFO | Total mem usage: 15.7 GiB | 20260507 22:23:17.897 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:23:17.897 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:23:17.897 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:23:17.897 | INFO | High Water Breakdown: | 20260507 22:23:17.897 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:23:17.897 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:23:17.897 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:23:17.897 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:23:17.897 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:23:17.897 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:23:17.897 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:23:17.897 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:23:17.897 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:23:17.897 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:23:17.897 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:23:17.897 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:23:17.897 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:23:17.897 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:23:17.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:23:17.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:23:17.897 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:23:17.897 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:23:17.897 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:23:17.897 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:23:17.897 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:23:17.897 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:23:17.897 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:23:17.897 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:23:17.897 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:23:17.897 | INFO | ------------------------------------------ | 20260507 22:23:17.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:23:17.897 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:23:17.897 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:23:17.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:23:17.898 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.40e+01 / 1.83e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.04e-01, 1.18e-02 / 6.80e-03 | 20260507 22:23:17.898 | INFO | PerfReport: 12.13%, Baseline -- 4.01e+00 / 4.47e+00, 5.95e-01 / 2.58e-01 | 20260507 22:23:17.898 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.07e-01 / 1.18e-01 | 20260507 22:23:17.898 | INFO | PerfReport: 15.79%, DME -- 4.17e+01 / 4.52e+01, 3.80e+00 / 1.47e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.84e+00 / 2.05e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.27e-01 / 2.22e-01 | 20260507 22:23:17.898 | INFO | PerfReport: 3.46%, Metrics -- 1.14e+00 / 1.36e+00, 2.92e-01 / 1.90e-01 | 20260507 22:23:17.898 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.87e-01, 5.95e-02 / 3.25e-02 | 20260507 22:23:17.898 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:23:17.898 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:23:17.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:23:17.898 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 30.92%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 1.99%, LockUpload -- 2.12e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 2.45%, LockDownload -- 2.61e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 15.96%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:23:17.898 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 22:23:18.010 | INFO | Starting a round of Baz IO | 20260507 22:23:21.511 | INFO | Populating RTMetrics report | 20260507 22:23:21.512 | INFO | RTMetrics report populated | 20260507 22:23:21.512 | INFO | Opening tmp RTMetrics json file | 20260507 22:23:21.517 | INFO | Serializing RTMetrics to json | 20260507 22:23:21.521 | INFO | Serialization done, swapping tmp file now... | 20260507 22:23:21.531 | INFO | RTMetrics done | 20260507 22:23:21.642 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.632 seconds | 20260507 22:23:21.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.15% / 0.01% / 56.84% | 20260507 22:23:21.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40709% / 0% / 5.96046e-05% | 20260507 22:23:21.643 | INFO | Baz IO overall: | 20260507 22:23:21.643 | INFO | BytesWritten : 1392196374624 | 20260507 22:23:21.643 | INFO | NumEvents : 674528718718 | 20260507 22:23:23.036 | INFO | Analyzing chunk frames = [3055616,3056128) | 20260507 22:23:23.036 | INFO | MemInfo: { | 20260507 22:23:23.036 | INFO | "gpu" : " 17 GB", | 20260507 22:23:23.036 | INFO | "huge" : "", | 20260507 22:23:23.036 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:23.036 | INFO | "malloc" : " 14 GB", | 20260507 22:23:23.036 | INFO | "pinned" : " 29 GB" | 20260507 22:23:23.036 | INFO | } | 20260507 22:23:23.036 | INFO | Perf report for the last 5.13972 seconds: | 20260507 22:23:23.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:23.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:23.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:23.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:23.036 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260507 22:23:23.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:23.036 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260507 22:23:23.036 | INFO | PRE_HQ : 68.5%, 1.9, 69.2ms, 0 | 20260507 22:23:23.036 | INFO | BAZWRITER : 70.3%, 4.1, 146.8ms, 0 | 20260507 22:23:23.036 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 22:23:23.036 | INFO | Uploaded 36GiB to GPU in 2.58381s (13.9329GiB/s) | 20260507 22:23:23.036 | INFO | Downloaded 38GiB to GPU in 2.7038s (14.0543GiB/s) | 20260507 22:23:28.176 | INFO | Analyzing chunk frames = [3056128,3056640) | 20260507 22:23:28.176 | INFO | MemInfo: { | 20260507 22:23:28.176 | INFO | "gpu" : " 17 GB", | 20260507 22:23:28.176 | INFO | "huge" : "", | 20260507 22:23:28.176 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:28.176 | INFO | "malloc" : " 14 GB", | 20260507 22:23:28.176 | INFO | "pinned" : " 29 GB" | 20260507 22:23:28.176 | INFO | } | 20260507 22:23:28.176 | INFO | Perf report for the last 5.14045 seconds: | 20260507 22:23:28.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:28.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 22:23:28.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:23:28.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:28.176 | INFO | Uploaded 36GiB to GPU in 2.5905s (13.897GiB/s) | 20260507 22:23:28.176 | INFO | Downloaded 34GiB to GPU in 2.47663s (13.7283GiB/s) | 20260507 22:23:33.311 | INFO | Analyzing chunk frames = [3056640,3057152) | 20260507 22:23:33.311 | INFO | MemInfo: { | 20260507 22:23:33.311 | INFO | "gpu" : " 17 GB", | 20260507 22:23:33.311 | INFO | "huge" : "", | 20260507 22:23:33.311 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:33.311 | INFO | "malloc" : " 14 GB", | 20260507 22:23:33.311 | INFO | "pinned" : " 29 GB" | 20260507 22:23:33.311 | INFO | } | 20260507 22:23:33.311 | INFO | Perf report for the last 5.13484 seconds: | 20260507 22:23:33.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:33.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260507 22:23:33.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:23:33.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:33.311 | INFO | Uploaded 36GiB to GPU in 2.59311s (13.8829GiB/s) | 20260507 22:23:33.311 | INFO | Downloaded 34GiB to GPU in 2.45915s (13.8259GiB/s) | 20260507 22:23:38.446 | INFO | Analyzing chunk frames = [3057152,3057664) | 20260507 22:23:38.446 | INFO | MemInfo: { | 20260507 22:23:38.446 | INFO | "gpu" : " 17 GB", | 20260507 22:23:38.447 | INFO | "huge" : "", | 20260507 22:23:38.447 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:38.447 | INFO | "malloc" : " 14 GB", | 20260507 22:23:38.447 | INFO | "pinned" : " 29 GB" | 20260507 22:23:38.447 | INFO | } | 20260507 22:23:38.447 | INFO | Perf report for the last 5.1357 seconds: | 20260507 22:23:38.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:38.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:23:38.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:23:38.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:38.447 | INFO | Uploaded 36GiB to GPU in 2.59255s (13.886GiB/s) | 20260507 22:23:38.447 | INFO | Downloaded 34GiB to GPU in 2.4678s (13.7775GiB/s) | 20260507 22:23:43.590 | INFO | Analyzing chunk frames = [3057664,3058176) | 20260507 22:23:43.590 | INFO | MemInfo: { | 20260507 22:23:43.590 | INFO | "gpu" : " 17 GB", | 20260507 22:23:43.590 | INFO | "huge" : "", | 20260507 22:23:43.590 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:43.590 | INFO | "malloc" : " 14 GB", | 20260507 22:23:43.590 | INFO | "pinned" : " 29 GB" | 20260507 22:23:43.590 | INFO | } | 20260507 22:23:43.590 | INFO | Perf report for the last 5.14379 seconds: | 20260507 22:23:43.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:43.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.591 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:23:43.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.591 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:23:43.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:43.591 | INFO | Uploaded 36GiB to GPU in 2.4051s (14.9682GiB/s) | 20260507 22:23:43.591 | INFO | Downloaded 34GiB to GPU in 2.46379s (13.7999GiB/s) | 20260507 22:23:48.734 | INFO | Analyzing chunk frames = [3058176,3058688) | 20260507 22:23:48.734 | INFO | MemInfo: { | 20260507 22:23:48.734 | INFO | "gpu" : " 17 GB", | 20260507 22:23:48.734 | INFO | "huge" : "", | 20260507 22:23:48.734 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:48.734 | INFO | "malloc" : " 14 GB", | 20260507 22:23:48.734 | INFO | "pinned" : " 29 GB" | 20260507 22:23:48.734 | INFO | } | 20260507 22:23:48.734 | INFO | Perf report for the last 5.14341 seconds: | 20260507 22:23:48.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:48.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 22:23:48.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:23:48.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:48.734 | INFO | Uploaded 36GiB to GPU in 2.51878s (14.2926GiB/s) | 20260507 22:23:48.734 | INFO | Downloaded 34GiB to GPU in 2.4188s (14.0566GiB/s) | 20260507 22:23:53.873 | INFO | Analyzing chunk frames = [3058688,3059200) | 20260507 22:23:53.873 | INFO | MemInfo: { | 20260507 22:23:53.873 | INFO | "gpu" : " 17 GB", | 20260507 22:23:53.873 | INFO | "huge" : "", | 20260507 22:23:53.873 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:53.873 | INFO | "malloc" : " 14 GB", | 20260507 22:23:53.873 | INFO | "pinned" : " 29 GB" | 20260507 22:23:53.873 | INFO | } | 20260507 22:23:53.874 | INFO | Perf report for the last 5.13961 seconds: | 20260507 22:23:53.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:53.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:23:53.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:23:53.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:53.874 | INFO | Uploaded 36GiB to GPU in 2.55745s (14.0765GiB/s) | 20260507 22:23:53.874 | INFO | Downloaded 34GiB to GPU in 2.46318s (13.8033GiB/s) | 20260507 22:23:59.008 | INFO | Analyzing chunk frames = [3059200,3059712) | 20260507 22:23:59.008 | INFO | MemInfo: { | 20260507 22:23:59.008 | INFO | "gpu" : " 17 GB", | 20260507 22:23:59.008 | INFO | "huge" : "", | 20260507 22:23:59.008 | INFO | "huge_shared" : " 29 GB", | 20260507 22:23:59.008 | INFO | "malloc" : " 14 GB", | 20260507 22:23:59.008 | INFO | "pinned" : " 29 GB" | 20260507 22:23:59.008 | INFO | } | 20260507 22:23:59.008 | INFO | Perf report for the last 5.13445 seconds: | 20260507 22:23:59.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:23:59.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:23:59.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 22:23:59.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:23:59.008 | INFO | Uploaded 36GiB to GPU in 2.51133s (14.335GiB/s) | 20260507 22:23:59.008 | INFO | Downloaded 34GiB to GPU in 2.43541s (13.9607GiB/s) | 20260507 22:23:59.106 | INFO | Starting a round of Baz IO | 20260507 22:24:02.559 | INFO | Populating RTMetrics report | 20260507 22:24:02.560 | INFO | RTMetrics report populated | 20260507 22:24:02.560 | INFO | Opening tmp RTMetrics json file | 20260507 22:24:02.593 | INFO | Serializing RTMetrics to json | 20260507 22:24:02.716 | INFO | Serialization done, swapping tmp file now... | 20260507 22:24:02.718 | INFO | RTMetrics done | 20260507 22:24:02.781 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.675 seconds | 20260507 22:24:02.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.14% / 0.01% / 56.85% | 20260507 22:24:02.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40728% / 1.19209e-05% / 6.35783e-05% | 20260507 22:24:02.781 | INFO | Baz IO overall: | 20260507 22:24:02.781 | INFO | BytesWritten : 1394062761508 | 20260507 22:24:02.781 | INFO | NumEvents : 675428237635 | 20260507 22:24:04.149 | INFO | Analyzing chunk frames = [3059712,3060224) | 20260507 22:24:04.149 | INFO | MemInfo: { | 20260507 22:24:04.149 | INFO | "gpu" : " 17 GB", | 20260507 22:24:04.149 | INFO | "huge" : "", | 20260507 22:24:04.149 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:04.149 | INFO | "malloc" : " 14 GB", | 20260507 22:24:04.149 | INFO | "pinned" : " 29 GB" | 20260507 22:24:04.149 | INFO | } | 20260507 22:24:04.149 | INFO | Perf report for the last 5.14121 seconds: | 20260507 22:24:04.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:04.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:04.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:04.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:04.149 | INFO | ANALYSIS : 68.4%, 3.0, 106.3ms, 0 | 20260507 22:24:04.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:04.149 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 22:24:04.149 | INFO | PRE_HQ : 66.0%, 1.7, 57.9ms, 0 | 20260507 22:24:04.149 | INFO | BAZWRITER : 71.1%, 4.1, 148.1ms, 0 | 20260507 22:24:04.149 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260507 22:24:04.149 | INFO | Uploaded 36GiB to GPU in 2.49153s (14.449GiB/s) | 20260507 22:24:04.150 | INFO | Downloaded 38GiB to GPU in 2.6973s (14.0882GiB/s) | 20260507 22:24:09.285 | INFO | Analyzing chunk frames = [3060224,3060736) | 20260507 22:24:09.285 | INFO | MemInfo: { | 20260507 22:24:09.285 | INFO | "gpu" : " 17 GB", | 20260507 22:24:09.285 | INFO | "huge" : "", | 20260507 22:24:09.285 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:09.285 | INFO | "malloc" : " 14 GB", | 20260507 22:24:09.285 | INFO | "pinned" : " 29 GB" | 20260507 22:24:09.285 | INFO | } | 20260507 22:24:09.285 | INFO | Perf report for the last 5.13618 seconds: | 20260507 22:24:09.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:09.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.285 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 22:24:09.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.285 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:24:09.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:09.286 | INFO | Uploaded 36GiB to GPU in 2.55765s (14.0754GiB/s) | 20260507 22:24:09.286 | INFO | Downloaded 34GiB to GPU in 2.48158s (13.7009GiB/s) | 20260507 22:24:14.422 | INFO | Analyzing chunk frames = [3060736,3061248) | 20260507 22:24:14.422 | INFO | MemInfo: { | 20260507 22:24:14.422 | INFO | "gpu" : " 17 GB", | 20260507 22:24:14.422 | INFO | "huge" : "", | 20260507 22:24:14.422 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:14.422 | INFO | "malloc" : " 14 GB", | 20260507 22:24:14.422 | INFO | "pinned" : " 29 GB" | 20260507 22:24:14.422 | INFO | } | 20260507 22:24:14.422 | INFO | Perf report for the last 5.13667 seconds: | 20260507 22:24:14.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:14.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:24:14.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:24:14.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:14.422 | INFO | Uploaded 36GiB to GPU in 2.56653s (14.0267GiB/s) | 20260507 22:24:14.422 | INFO | Downloaded 34GiB to GPU in 2.47192s (13.7545GiB/s) | 20260507 22:24:19.569 | INFO | Analyzing chunk frames = [3061248,3061760) | 20260507 22:24:19.569 | INFO | MemInfo: { | 20260507 22:24:19.569 | INFO | "gpu" : " 17 GB", | 20260507 22:24:19.569 | INFO | "huge" : "", | 20260507 22:24:19.569 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:19.569 | INFO | "malloc" : " 14 GB", | 20260507 22:24:19.569 | INFO | "pinned" : " 29 GB" | 20260507 22:24:19.569 | INFO | } | 20260507 22:24:19.569 | INFO | Perf report for the last 5.14679 seconds: | 20260507 22:24:19.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:19.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 22:24:19.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:24:19.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:19.569 | INFO | Uploaded 36GiB to GPU in 2.46096s (14.6284GiB/s) | 20260507 22:24:19.569 | INFO | Downloaded 34GiB to GPU in 2.38485s (14.2566GiB/s) | 20260507 22:24:24.704 | INFO | Analyzing chunk frames = [3061760,3062272) | 20260507 22:24:24.704 | INFO | MemInfo: { | 20260507 22:24:24.704 | INFO | "gpu" : " 17 GB", | 20260507 22:24:24.704 | INFO | "huge" : "", | 20260507 22:24:24.704 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:24.704 | INFO | "malloc" : " 14 GB", | 20260507 22:24:24.704 | INFO | "pinned" : " 29 GB" | 20260507 22:24:24.704 | INFO | } | 20260507 22:24:24.704 | INFO | Perf report for the last 5.13497 seconds: | 20260507 22:24:24.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:24.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 22:24:24.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 22:24:24.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:24.704 | INFO | Uploaded 36GiB to GPU in 2.3871s (15.0811GiB/s) | 20260507 22:24:24.705 | INFO | Downloaded 34GiB to GPU in 2.50125s (13.5932GiB/s) | 20260507 22:24:29.845 | INFO | Analyzing chunk frames = [3062272,3062784) | 20260507 22:24:29.845 | INFO | MemInfo: { | 20260507 22:24:29.845 | INFO | "gpu" : " 17 GB", | 20260507 22:24:29.845 | INFO | "huge" : "", | 20260507 22:24:29.845 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:29.845 | INFO | "malloc" : " 14 GB", | 20260507 22:24:29.845 | INFO | "pinned" : " 29 GB" | 20260507 22:24:29.845 | INFO | } | 20260507 22:24:29.845 | INFO | Perf report for the last 5.1411 seconds: | 20260507 22:24:29.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:29.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 22:24:29.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 22:24:29.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:29.845 | INFO | Uploaded 36GiB to GPU in 2.54766s (14.1306GiB/s) | 20260507 22:24:29.846 | INFO | Downloaded 34GiB to GPU in 2.41535s (14.0766GiB/s) | 20260507 22:24:34.985 | INFO | Analyzing chunk frames = [3062784,3063296) | 20260507 22:24:34.985 | INFO | MemInfo: { | 20260507 22:24:34.985 | INFO | "gpu" : " 17 GB", | 20260507 22:24:34.985 | INFO | "huge" : "", | 20260507 22:24:34.985 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:34.985 | INFO | "malloc" : " 14 GB", | 20260507 22:24:34.985 | INFO | "pinned" : " 29 GB" | 20260507 22:24:34.985 | INFO | } | 20260507 22:24:34.985 | INFO | Perf report for the last 5.14046 seconds: | 20260507 22:24:34.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:34.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.985 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 22:24:34.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.986 | INFO | PRE_HQ : 35.8%, 1.0, 18.4ms, 0 | 20260507 22:24:34.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:34.986 | INFO | Uploaded 36GiB to GPU in 2.55335s (14.0991GiB/s) | 20260507 22:24:34.986 | INFO | Downloaded 34GiB to GPU in 2.45363s (13.857GiB/s) | 20260507 22:24:40.119 | INFO | Analyzing chunk frames = [3063296,3063808) | 20260507 22:24:40.119 | INFO | MemInfo: { | 20260507 22:24:40.119 | INFO | "gpu" : " 17 GB", | 20260507 22:24:40.119 | INFO | "huge" : "", | 20260507 22:24:40.119 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:40.119 | INFO | "malloc" : " 14 GB", | 20260507 22:24:40.119 | INFO | "pinned" : " 29 GB" | 20260507 22:24:40.119 | INFO | } | 20260507 22:24:40.119 | INFO | Perf report for the last 5.13361 seconds: | 20260507 22:24:40.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:40.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.119 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 22:24:40.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.119 | INFO | PRE_HQ : 36.8%, 1.0, 19.1ms, 0 | 20260507 22:24:40.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:40.120 | INFO | Uploaded 36GiB to GPU in 2.54693s (14.1347GiB/s) | 20260507 22:24:40.120 | INFO | Downloaded 34GiB to GPU in 2.44314s (13.9165GiB/s) | 20260507 22:24:40.120 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:24:40.120 | INFO | Total mem usage: 16.7 GiB | 20260507 22:24:40.120 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:24:40.120 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.95%) | 20260507 22:24:40.120 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:24:40.120 | INFO | High Water Breakdown: | 20260507 22:24:40.120 | INFO | 60.3% : 9.16 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:24:40.120 | INFO | 39.5% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:24:40.120 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:24:40.120 | INFO | ------------------------------------------ | 20260507 22:24:40.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:24:40.120 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:24:40.120 | INFO | Total mem usage: 26.5 GiB | 20260507 22:24:40.120 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:24:40.120 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:24:40.120 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:24:40.120 | INFO | High Water Breakdown: | 20260507 22:24:40.120 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:24:40.120 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:24:40.120 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:24:40.120 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:24:40.120 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:24:40.120 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:24:40.120 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:24:40.120 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:24:40.120 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:24:40.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:24:40.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:24:40.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:24:40.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:24:40.120 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:24:40.120 | INFO | ------------------------------------------ | 20260507 22:24:40.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:24:40.120 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:24:40.120 | INFO | Total mem usage: 27.8 GiB | 20260507 22:24:40.120 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:24:40.120 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:24:40.120 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:24:40.120 | INFO | High Water Breakdown: | 20260507 22:24:40.120 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:24:40.120 | INFO | ------------------------------------------ | 20260507 22:24:40.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:24:40.120 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:24:40.120 | INFO | Total mem usage: 15.7 GiB | 20260507 22:24:40.120 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:24:40.120 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:24:40.120 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:24:40.120 | INFO | High Water Breakdown: | 20260507 22:24:40.120 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:24:40.120 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:24:40.120 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:24:40.120 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:24:40.120 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:24:40.120 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:24:40.120 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:24:40.120 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:24:40.120 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:24:40.120 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:24:40.120 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:24:40.120 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:24:40.120 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:24:40.120 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:24:40.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:24:40.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:24:40.120 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:24:40.120 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:24:40.121 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:24:40.121 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:24:40.121 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:24:40.121 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:24:40.121 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:24:40.121 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:24:40.121 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:24:40.121 | INFO | ------------------------------------------ | 20260507 22:24:40.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:24:40.121 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:24:40.121 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:24:40.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:24:40.121 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.41e+01 / 1.56e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.04e-01, 1.20e-02 / 5.88e-03 | 20260507 22:24:40.121 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.46e+00, 5.94e-01 / 2.27e-01 | 20260507 22:24:40.121 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.08e-01 / 1.19e-01 | 20260507 22:24:40.121 | INFO | PerfReport: 15.79%, DME -- 4.17e+01 / 4.50e+01, 3.83e+00 / 1.23e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.86e+01 / 2.07e+01, 1.83e+00 / 1.60e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.44e+00 / 2.72e+00, 4.34e-01 / 2.30e-01 | 20260507 22:24:40.121 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.34e+00, 2.88e-01 / 1.74e-01 | 20260507 22:24:40.121 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.85e-01, 5.98e-02 / 3.15e-02 | 20260507 22:24:40.121 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:24:40.121 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:24:40.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:24:40.121 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 24.65%, Upload -- 2.63e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.50e+00 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 30.87%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 2.19%, LockUpload -- 2.34e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 2.27%, LockDownload -- 2.43e+00 / 0.00e+00, 3.71e+00 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:24:40.121 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260507 22:24:40.230 | INFO | Starting a round of Baz IO | 20260507 22:24:43.706 | INFO | Populating RTMetrics report | 20260507 22:24:43.706 | INFO | RTMetrics report populated | 20260507 22:24:43.706 | INFO | Opening tmp RTMetrics json file | 20260507 22:24:43.707 | INFO | Serializing RTMetrics to json | 20260507 22:24:43.715 | INFO | Serialization done, swapping tmp file now... | 20260507 22:24:43.732 | INFO | RTMetrics done | 20260507 22:24:43.825 | INFO | Finished a round of Baz IO: Wrote 1.737 GiB in 3.594 seconds | 20260507 22:24:43.825 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.14% / 0.01% / 56.86% | 20260507 22:24:43.825 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40546% / 7.94729e-06% / 5.5631e-05% | 20260507 22:24:43.825 | INFO | Baz IO overall: | 20260507 22:24:43.825 | INFO | BytesWritten : 1395927478639 | 20260507 22:24:43.825 | INFO | NumEvents : 676327311576 | 20260507 22:24:45.259 | INFO | Analyzing chunk frames = [3063808,3064320) | 20260507 22:24:45.259 | INFO | MemInfo: { | 20260507 22:24:45.259 | INFO | "gpu" : " 17 GB", | 20260507 22:24:45.259 | INFO | "huge" : "", | 20260507 22:24:45.259 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:45.259 | INFO | "malloc" : " 14 GB", | 20260507 22:24:45.259 | INFO | "pinned" : " 29 GB" | 20260507 22:24:45.259 | INFO | } | 20260507 22:24:45.259 | INFO | Perf report for the last 5.13993 seconds: | 20260507 22:24:45.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:45.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:45.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:45.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:45.259 | INFO | ANALYSIS : 69.0%, 2.9, 105.0ms, 0 | 20260507 22:24:45.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:45.259 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260507 22:24:45.259 | INFO | PRE_HQ : 68.2%, 1.9, 66.4ms, 0 | 20260507 22:24:45.259 | INFO | BAZWRITER : 69.5%, 4.0, 142.4ms, 0 | 20260507 22:24:45.259 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 22:24:45.260 | INFO | Uploaded 36GiB to GPU in 2.54775s (14.1301GiB/s) | 20260507 22:24:45.260 | INFO | Downloaded 38GiB to GPU in 2.6751s (14.2051GiB/s) | 20260507 22:24:50.398 | INFO | Analyzing chunk frames = [3064320,3064832) | 20260507 22:24:50.398 | INFO | MemInfo: { | 20260507 22:24:50.399 | INFO | "gpu" : " 17 GB", | 20260507 22:24:50.399 | INFO | "huge" : "", | 20260507 22:24:50.399 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:50.399 | INFO | "malloc" : " 14 GB", | 20260507 22:24:50.399 | INFO | "pinned" : " 29 GB" | 20260507 22:24:50.399 | INFO | } | 20260507 22:24:50.399 | INFO | Perf report for the last 5.13965 seconds: | 20260507 22:24:50.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:50.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 22:24:50.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:24:50.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:50.399 | INFO | Uploaded 36GiB to GPU in 2.54598s (14.1399GiB/s) | 20260507 22:24:50.399 | INFO | Downloaded 34GiB to GPU in 2.46834s (13.7745GiB/s) | 20260507 22:24:55.543 | INFO | Analyzing chunk frames = [3064832,3065344) | 20260507 22:24:55.543 | INFO | MemInfo: { | 20260507 22:24:55.543 | INFO | "gpu" : " 17 GB", | 20260507 22:24:55.543 | INFO | "huge" : "", | 20260507 22:24:55.543 | INFO | "huge_shared" : " 29 GB", | 20260507 22:24:55.543 | INFO | "malloc" : " 14 GB", | 20260507 22:24:55.543 | INFO | "pinned" : " 29 GB" | 20260507 22:24:55.543 | INFO | } | 20260507 22:24:55.543 | INFO | Perf report for the last 5.14431 seconds: | 20260507 22:24:55.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:24:55.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 22:24:55.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:24:55.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:24:55.543 | INFO | Uploaded 36GiB to GPU in 2.57502s (13.9804GiB/s) | 20260507 22:24:55.543 | INFO | Downloaded 34GiB to GPU in 2.4502s (13.8764GiB/s) | 20260507 22:25:00.688 | INFO | Analyzing chunk frames = [3065344,3065856) | 20260507 22:25:00.688 | INFO | MemInfo: { | 20260507 22:25:00.688 | INFO | "gpu" : " 17 GB", | 20260507 22:25:00.688 | INFO | "huge" : "", | 20260507 22:25:00.688 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:00.688 | INFO | "malloc" : " 14 GB", | 20260507 22:25:00.689 | INFO | "pinned" : " 29 GB" | 20260507 22:25:00.689 | INFO | } | 20260507 22:25:00.689 | INFO | Perf report for the last 5.14558 seconds: | 20260507 22:25:00.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:00.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 22:25:00.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:25:00.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:00.689 | INFO | Uploaded 36GiB to GPU in 2.47823s (14.5265GiB/s) | 20260507 22:25:00.689 | INFO | Downloaded 34GiB to GPU in 2.37938s (14.2894GiB/s) | 20260507 22:25:05.823 | INFO | Analyzing chunk frames = [3065856,3066368) | 20260507 22:25:05.823 | INFO | MemInfo: { | 20260507 22:25:05.823 | INFO | "gpu" : " 17 GB", | 20260507 22:25:05.823 | INFO | "huge" : "", | 20260507 22:25:05.823 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:05.823 | INFO | "malloc" : " 14 GB", | 20260507 22:25:05.823 | INFO | "pinned" : " 29 GB" | 20260507 22:25:05.823 | INFO | } | 20260507 22:25:05.823 | INFO | Perf report for the last 5.13463 seconds: | 20260507 22:25:05.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:05.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.823 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 22:25:05.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.823 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:25:05.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:05.824 | INFO | Uploaded 36GiB to GPU in 2.45124s (14.6864GiB/s) | 20260507 22:25:05.824 | INFO | Downloaded 34GiB to GPU in 2.4876s (13.6678GiB/s) | 20260507 22:25:10.956 | INFO | Analyzing chunk frames = [3066368,3066880) | 20260507 22:25:10.956 | INFO | MemInfo: { | 20260507 22:25:10.956 | INFO | "gpu" : " 17 GB", | 20260507 22:25:10.956 | INFO | "huge" : "", | 20260507 22:25:10.956 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:10.956 | INFO | "malloc" : " 14 GB", | 20260507 22:25:10.956 | INFO | "pinned" : " 29 GB" | 20260507 22:25:10.956 | INFO | } | 20260507 22:25:10.956 | INFO | Perf report for the last 5.13323 seconds: | 20260507 22:25:10.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:10.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:25:10.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:25:10.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:10.957 | INFO | Uploaded 36GiB to GPU in 2.51541s (14.3118GiB/s) | 20260507 22:25:10.957 | INFO | Downloaded 34GiB to GPU in 2.41356s (14.0871GiB/s) | 20260507 22:25:16.100 | INFO | Analyzing chunk frames = [3066880,3067392) | 20260507 22:25:16.100 | INFO | MemInfo: { | 20260507 22:25:16.100 | INFO | "gpu" : " 17 GB", | 20260507 22:25:16.100 | INFO | "huge" : "", | 20260507 22:25:16.100 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:16.100 | INFO | "malloc" : " 14 GB", | 20260507 22:25:16.100 | INFO | "pinned" : " 29 GB" | 20260507 22:25:16.100 | INFO | } | 20260507 22:25:16.100 | INFO | Perf report for the last 5.14371 seconds: | 20260507 22:25:16.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:16.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.101 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260507 22:25:16.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.101 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:25:16.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:16.101 | INFO | Uploaded 36GiB to GPU in 2.53893s (14.1792GiB/s) | 20260507 22:25:16.101 | INFO | Downloaded 34GiB to GPU in 2.43659s (13.9539GiB/s) | 20260507 22:25:21.237 | INFO | Analyzing chunk frames = [3067392,3067904) | 20260507 22:25:21.237 | INFO | MemInfo: { | 20260507 22:25:21.237 | INFO | "gpu" : " 17 GB", | 20260507 22:25:21.237 | INFO | "huge" : "", | 20260507 22:25:21.237 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:21.237 | INFO | "malloc" : " 14 GB", | 20260507 22:25:21.237 | INFO | "pinned" : " 29 GB" | 20260507 22:25:21.237 | INFO | } | 20260507 22:25:21.237 | INFO | Perf report for the last 5.1372 seconds: | 20260507 22:25:21.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:21.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.238 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 22:25:21.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.238 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 22:25:21.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:21.238 | INFO | Uploaded 36GiB to GPU in 2.54155s (14.1646GiB/s) | 20260507 22:25:21.238 | INFO | Downloaded 34GiB to GPU in 2.44173s (13.9246GiB/s) | 20260507 22:25:21.352 | INFO | Starting a round of Baz IO | 20260507 22:25:24.847 | INFO | Populating RTMetrics report | 20260507 22:25:24.847 | INFO | RTMetrics report populated | 20260507 22:25:24.847 | INFO | Opening tmp RTMetrics json file | 20260507 22:25:24.849 | INFO | Serializing RTMetrics to json | 20260507 22:25:24.853 | INFO | Serialization done, swapping tmp file now... | 20260507 22:25:24.855 | INFO | RTMetrics done | 20260507 22:25:24.950 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.597 seconds | 20260507 22:25:24.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.13% / 0.01% / 56.86% | 20260507 22:25:24.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40644% / 7.94729e-06% / 3.97364e-05% | 20260507 22:25:24.950 | INFO | Baz IO overall: | 20260507 22:25:24.950 | INFO | BytesWritten : 1397790620606 | 20260507 22:25:24.950 | INFO | NumEvents : 677225765245 | 20260507 22:25:26.374 | INFO | Analyzing chunk frames = [3067904,3068416) | 20260507 22:25:26.374 | INFO | MemInfo: { | 20260507 22:25:26.374 | INFO | "gpu" : " 17 GB", | 20260507 22:25:26.374 | INFO | "huge" : "", | 20260507 22:25:26.374 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:26.374 | INFO | "malloc" : " 14 GB", | 20260507 22:25:26.374 | INFO | "pinned" : " 29 GB" | 20260507 22:25:26.374 | INFO | } | 20260507 22:25:26.374 | INFO | Perf report for the last 5.13671 seconds: | 20260507 22:25:26.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:26.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:26.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:26.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:26.374 | INFO | ANALYSIS : 69.1%, 2.9, 104.5ms, 0 | 20260507 22:25:26.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:26.374 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260507 22:25:26.374 | INFO | PRE_HQ : 68.7%, 1.9, 69.5ms, 0 | 20260507 22:25:26.374 | INFO | BAZWRITER : 69.7%, 4.0, 141.9ms, 0 | 20260507 22:25:26.374 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 22:25:26.375 | INFO | Uploaded 36GiB to GPU in 2.61547s (13.7643GiB/s) | 20260507 22:25:26.375 | INFO | Downloaded 38GiB to GPU in 2.70269s (14.0601GiB/s) | 20260507 22:25:31.510 | INFO | Analyzing chunk frames = [3068416,3068928) | 20260507 22:25:31.510 | INFO | MemInfo: { | 20260507 22:25:31.510 | INFO | "gpu" : " 17 GB", | 20260507 22:25:31.510 | INFO | "huge" : "", | 20260507 22:25:31.510 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:31.510 | INFO | "malloc" : " 14 GB", | 20260507 22:25:31.510 | INFO | "pinned" : " 29 GB" | 20260507 22:25:31.510 | INFO | } | 20260507 22:25:31.510 | INFO | Perf report for the last 5.1363 seconds: | 20260507 22:25:31.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:31.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.511 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 22:25:31.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.511 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 22:25:31.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:31.511 | INFO | Uploaded 36GiB to GPU in 2.5564s (14.0823GiB/s) | 20260507 22:25:31.511 | INFO | Downloaded 34GiB to GPU in 2.48507s (13.6817GiB/s) | 20260507 22:25:36.655 | INFO | Analyzing chunk frames = [3068928,3069440) | 20260507 22:25:36.655 | INFO | MemInfo: { | 20260507 22:25:36.655 | INFO | "gpu" : " 17 GB", | 20260507 22:25:36.655 | INFO | "huge" : "", | 20260507 22:25:36.655 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:36.655 | INFO | "malloc" : " 14 GB", | 20260507 22:25:36.655 | INFO | "pinned" : " 29 GB" | 20260507 22:25:36.655 | INFO | } | 20260507 22:25:36.655 | INFO | Perf report for the last 5.14458 seconds: | 20260507 22:25:36.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:36.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:25:36.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:25:36.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:36.655 | INFO | Uploaded 36GiB to GPU in 2.52001s (14.2857GiB/s) | 20260507 22:25:36.655 | INFO | Downloaded 34GiB to GPU in 2.45533s (13.8474GiB/s) | 20260507 22:25:41.799 | INFO | Analyzing chunk frames = [3069440,3069952) | 20260507 22:25:41.799 | INFO | MemInfo: { | 20260507 22:25:41.799 | INFO | "gpu" : " 17 GB", | 20260507 22:25:41.799 | INFO | "huge" : "", | 20260507 22:25:41.799 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:41.799 | INFO | "malloc" : " 14 GB", | 20260507 22:25:41.799 | INFO | "pinned" : " 29 GB" | 20260507 22:25:41.799 | INFO | } | 20260507 22:25:41.799 | INFO | Perf report for the last 5.14429 seconds: | 20260507 22:25:41.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:41.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.799 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:25:41.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.800 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:25:41.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:41.800 | INFO | Uploaded 36GiB to GPU in 2.46956s (14.5775GiB/s) | 20260507 22:25:41.800 | INFO | Downloaded 34GiB to GPU in 2.40804s (14.1194GiB/s) | 20260507 22:25:46.940 | INFO | Analyzing chunk frames = [3069952,3070464) | 20260507 22:25:46.940 | INFO | MemInfo: { | 20260507 22:25:46.940 | INFO | "gpu" : " 17 GB", | 20260507 22:25:46.940 | INFO | "huge" : "", | 20260507 22:25:46.940 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:46.940 | INFO | "malloc" : " 14 GB", | 20260507 22:25:46.940 | INFO | "pinned" : " 29 GB" | 20260507 22:25:46.940 | INFO | } | 20260507 22:25:46.940 | INFO | Perf report for the last 5.14041 seconds: | 20260507 22:25:46.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:46.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:25:46.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 22:25:46.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:46.940 | INFO | Uploaded 36GiB to GPU in 2.31894s (15.5243GiB/s) | 20260507 22:25:46.940 | INFO | Downloaded 34GiB to GPU in 2.47807s (13.7204GiB/s) | 20260507 22:25:52.073 | INFO | Analyzing chunk frames = [3070464,3070976) | 20260507 22:25:52.073 | INFO | MemInfo: { | 20260507 22:25:52.073 | INFO | "gpu" : " 17 GB", | 20260507 22:25:52.073 | INFO | "huge" : "", | 20260507 22:25:52.073 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:52.073 | INFO | "malloc" : " 14 GB", | 20260507 22:25:52.073 | INFO | "pinned" : " 29 GB" | 20260507 22:25:52.073 | INFO | } | 20260507 22:25:52.073 | INFO | Perf report for the last 5.13337 seconds: | 20260507 22:25:52.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:52.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.073 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:25:52.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.073 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:25:52.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:52.074 | INFO | Uploaded 36GiB to GPU in 2.4945s (14.4318GiB/s) | 20260507 22:25:52.074 | INFO | Downloaded 34GiB to GPU in 2.44265s (13.9193GiB/s) | 20260507 22:25:57.212 | INFO | Analyzing chunk frames = [3070976,3071488) | 20260507 22:25:57.212 | INFO | MemInfo: { | 20260507 22:25:57.212 | INFO | "gpu" : " 17 GB", | 20260507 22:25:57.212 | INFO | "huge" : "", | 20260507 22:25:57.212 | INFO | "huge_shared" : " 29 GB", | 20260507 22:25:57.212 | INFO | "malloc" : " 14 GB", | 20260507 22:25:57.212 | INFO | "pinned" : " 29 GB" | 20260507 22:25:57.212 | INFO | } | 20260507 22:25:57.212 | INFO | Perf report for the last 5.13886 seconds: | 20260507 22:25:57.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:25:57.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:25:57.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 22:25:57.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:25:57.212 | INFO | Uploaded 36GiB to GPU in 2.49768s (14.4134GiB/s) | 20260507 22:25:57.212 | INFO | Downloaded 34GiB to GPU in 2.46473s (13.7946GiB/s) | 20260507 22:26:02.353 | INFO | Analyzing chunk frames = [3071488,3072000) | 20260507 22:26:02.353 | INFO | MemInfo: { | 20260507 22:26:02.353 | INFO | "gpu" : " 17 GB", | 20260507 22:26:02.353 | INFO | "huge" : "", | 20260507 22:26:02.353 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:02.353 | INFO | "malloc" : " 14 GB", | 20260507 22:26:02.353 | INFO | "pinned" : " 29 GB" | 20260507 22:26:02.353 | INFO | } | 20260507 22:26:02.354 | INFO | Perf report for the last 5.14143 seconds: | 20260507 22:26:02.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:02.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:26:02.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 22:26:02.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:02.354 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:26:02.354 | INFO | Total mem usage: 16.7 GiB | 20260507 22:26:02.354 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:26:02.354 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.6%) | 20260507 22:26:02.354 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:26:02.354 | INFO | High Water Breakdown: | 20260507 22:26:02.354 | INFO | 60.1% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:26:02.354 | INFO | 39.8% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:26:02.354 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:26:02.354 | INFO | ------------------------------------------ | 20260507 22:26:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:26:02.354 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:26:02.354 | INFO | Total mem usage: 26.5 GiB | 20260507 22:26:02.354 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:26:02.354 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 22:26:02.354 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:26:02.354 | INFO | High Water Breakdown: | 20260507 22:26:02.354 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:26:02.354 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:26:02.354 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:26:02.354 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:26:02.354 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:26:02.354 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:26:02.354 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:26:02.354 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:26:02.354 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:26:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:26:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:26:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:26:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:26:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:26:02.354 | INFO | ------------------------------------------ | 20260507 22:26:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:26:02.354 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:26:02.354 | INFO | Total mem usage: 27.8 GiB | 20260507 22:26:02.354 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:26:02.354 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:26:02.354 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:26:02.354 | INFO | High Water Breakdown: | 20260507 22:26:02.354 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:26:02.354 | INFO | ------------------------------------------ | 20260507 22:26:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:26:02.354 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:26:02.354 | INFO | Total mem usage: 15.7 GiB | 20260507 22:26:02.354 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:26:02.354 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:26:02.354 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:26:02.354 | INFO | High Water Breakdown: | 20260507 22:26:02.354 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:26:02.354 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:26:02.354 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:26:02.354 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:26:02.354 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:26:02.354 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:26:02.354 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:26:02.354 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:26:02.354 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:26:02.354 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:26:02.354 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:26:02.354 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:26:02.354 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:26:02.354 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:26:02.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:26:02.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:26:02.354 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:26:02.354 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:26:02.354 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:26:02.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:26:02.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:26:02.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:26:02.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:26:02.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:26:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:26:02.354 | INFO | ------------------------------------------ | 20260507 22:26:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:26:02.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:26:02.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:26:02.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:26:02.354 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.41e+01 / 1.37e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.10e-02 / 5.59e-03 | 20260507 22:26:02.354 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.42e+00, 5.93e-01 / 2.94e-01 | 20260507 22:26:02.354 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.47e+00, 2.30e-01 / 1.26e-01 | 20260507 22:26:02.354 | INFO | PerfReport: 15.75%, DME -- 4.16e+01 / 4.51e+01, 3.93e+00 / 1.56e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.85e+00 / 1.78e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.29e-01 / 2.79e-01 | 20260507 22:26:02.354 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.31e+00, 2.90e-01 / 1.64e-01 | 20260507 22:26:02.354 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.83e-01, 5.93e-02 / 3.15e-02 | 20260507 22:26:02.354 | INFO | Uploaded 36GiB to GPU in 2.45934s (14.6381GiB/s) | 20260507 22:26:02.354 | INFO | Downloaded 34GiB to GPU in 2.45793s (13.8328GiB/s) | 20260507 22:26:02.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:26:02.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:26:02.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:26:02.354 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 24.50%, Upload -- 2.61e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 24.20%, Download -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 30.97%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 2.10%, LockUpload -- 2.24e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 2.35%, LockDownload -- 2.51e+00 / 0.00e+00, 3.73e+00 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 15.87%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:26:02.354 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 22:26:02.451 | INFO | Starting a round of Baz IO | 20260507 22:26:05.928 | INFO | Populating RTMetrics report | 20260507 22:26:05.928 | INFO | RTMetrics report populated | 20260507 22:26:05.928 | INFO | Opening tmp RTMetrics json file | 20260507 22:26:05.951 | INFO | Serializing RTMetrics to json | 20260507 22:26:05.952 | INFO | Serialization done, swapping tmp file now... | 20260507 22:26:05.953 | INFO | RTMetrics done | 20260507 22:26:06.025 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.574 seconds | 20260507 22:26:06.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.12% / 0.01% / 56.87% | 20260507 22:26:06.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40693% / 1.58946e-05% / 3.97364e-05% | 20260507 22:26:06.025 | INFO | Baz IO overall: | 20260507 22:26:06.025 | INFO | BytesWritten : 1399654808253 | 20260507 22:26:06.025 | INFO | NumEvents : 678125300935 | 20260507 22:26:07.495 | INFO | Analyzing chunk frames = [3072000,3072512) | 20260507 22:26:07.495 | INFO | MemInfo: { | 20260507 22:26:07.495 | INFO | "gpu" : " 17 GB", | 20260507 22:26:07.495 | INFO | "huge" : "", | 20260507 22:26:07.495 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:07.495 | INFO | "malloc" : " 14 GB", | 20260507 22:26:07.495 | INFO | "pinned" : " 29 GB" | 20260507 22:26:07.495 | INFO | } | 20260507 22:26:07.495 | INFO | Perf report for the last 5.1418 seconds: | 20260507 22:26:07.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:07.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:07.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:07.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:07.495 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 22:26:07.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:07.495 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 22:26:07.496 | INFO | PRE_HQ : 66.0%, 1.7, 57.3ms, 0 | 20260507 22:26:07.496 | INFO | BAZWRITER : 69.1%, 4.1, 144.5ms, 0 | 20260507 22:26:07.496 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 22:26:07.496 | INFO | Uploaded 36GiB to GPU in 2.50576s (14.3669GiB/s) | 20260507 22:26:07.496 | INFO | Downloaded 38GiB to GPU in 2.69357s (14.1076GiB/s) | 20260507 22:26:12.639 | INFO | Analyzing chunk frames = [3072512,3073024) | 20260507 22:26:12.639 | INFO | MemInfo: { | 20260507 22:26:12.639 | INFO | "gpu" : " 17 GB", | 20260507 22:26:12.639 | INFO | "huge" : "", | 20260507 22:26:12.639 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:12.639 | INFO | "malloc" : " 14 GB", | 20260507 22:26:12.639 | INFO | "pinned" : " 29 GB" | 20260507 22:26:12.639 | INFO | } | 20260507 22:26:12.639 | INFO | Perf report for the last 5.14389 seconds: | 20260507 22:26:12.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:12.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.639 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:26:12.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.639 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:26:12.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:12.640 | INFO | Uploaded 36GiB to GPU in 2.54402s (14.1508GiB/s) | 20260507 22:26:12.640 | INFO | Downloaded 34GiB to GPU in 2.49632s (13.62GiB/s) | 20260507 22:26:17.769 | INFO | Analyzing chunk frames = [3073024,3073536) | 20260507 22:26:17.769 | INFO | MemInfo: { | 20260507 22:26:17.769 | INFO | "gpu" : " 17 GB", | 20260507 22:26:17.769 | INFO | "huge" : "", | 20260507 22:26:17.769 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:17.770 | INFO | "malloc" : " 14 GB", | 20260507 22:26:17.770 | INFO | "pinned" : " 29 GB" | 20260507 22:26:17.770 | INFO | } | 20260507 22:26:17.770 | INFO | Perf report for the last 5.13031 seconds: | 20260507 22:26:17.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:17.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 22:26:17.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:26:17.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:17.770 | INFO | Uploaded 36GiB to GPU in 2.46698s (14.5927GiB/s) | 20260507 22:26:17.770 | INFO | Downloaded 34GiB to GPU in 2.49401s (13.6327GiB/s) | 20260507 22:26:22.912 | INFO | Analyzing chunk frames = [3073536,3074048) | 20260507 22:26:22.912 | INFO | MemInfo: { | 20260507 22:26:22.912 | INFO | "gpu" : " 17 GB", | 20260507 22:26:22.912 | INFO | "huge" : "", | 20260507 22:26:22.912 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:22.912 | INFO | "malloc" : " 14 GB", | 20260507 22:26:22.912 | INFO | "pinned" : " 29 GB" | 20260507 22:26:22.912 | INFO | } | 20260507 22:26:22.912 | INFO | Perf report for the last 5.14256 seconds: | 20260507 22:26:22.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:22.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.912 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 22:26:22.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.912 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:26:22.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:22.913 | INFO | Uploaded 36GiB to GPU in 2.42514s (14.8445GiB/s) | 20260507 22:26:22.913 | INFO | Downloaded 34GiB to GPU in 2.40284s (14.1499GiB/s) | 20260507 22:26:28.048 | INFO | Analyzing chunk frames = [3074048,3074560) | 20260507 22:26:28.048 | INFO | MemInfo: { | 20260507 22:26:28.048 | INFO | "gpu" : " 17 GB", | 20260507 22:26:28.048 | INFO | "huge" : "", | 20260507 22:26:28.048 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:28.048 | INFO | "malloc" : " 14 GB", | 20260507 22:26:28.048 | INFO | "pinned" : " 29 GB" | 20260507 22:26:28.048 | INFO | } | 20260507 22:26:28.048 | INFO | Perf report for the last 5.1363 seconds: | 20260507 22:26:28.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:28.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.048 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:26:28.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.048 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 22:26:28.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:28.049 | INFO | Uploaded 36GiB to GPU in 2.31101s (15.5776GiB/s) | 20260507 22:26:28.049 | INFO | Downloaded 34GiB to GPU in 2.513s (13.5297GiB/s) | 20260507 22:26:33.186 | INFO | Analyzing chunk frames = [3074560,3075072) | 20260507 22:26:33.186 | INFO | MemInfo: { | 20260507 22:26:33.186 | INFO | "gpu" : " 17 GB", | 20260507 22:26:33.186 | INFO | "huge" : "", | 20260507 22:26:33.186 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:33.186 | INFO | "malloc" : " 14 GB", | 20260507 22:26:33.186 | INFO | "pinned" : " 29 GB" | 20260507 22:26:33.186 | INFO | } | 20260507 22:26:33.186 | INFO | Perf report for the last 5.13731 seconds: | 20260507 22:26:33.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:33.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:26:33.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:26:33.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:33.186 | INFO | Uploaded 36GiB to GPU in 2.51259s (14.3278GiB/s) | 20260507 22:26:33.186 | INFO | Downloaded 34GiB to GPU in 2.43353s (13.9715GiB/s) | 20260507 22:26:38.324 | INFO | Analyzing chunk frames = [3075072,3075584) | 20260507 22:26:38.324 | INFO | MemInfo: { | 20260507 22:26:38.324 | INFO | "gpu" : " 17 GB", | 20260507 22:26:38.324 | INFO | "huge" : "", | 20260507 22:26:38.324 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:38.324 | INFO | "malloc" : " 14 GB", | 20260507 22:26:38.324 | INFO | "pinned" : " 29 GB" | 20260507 22:26:38.324 | INFO | } | 20260507 22:26:38.324 | INFO | Perf report for the last 5.13825 seconds: | 20260507 22:26:38.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:38.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.324 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:26:38.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.324 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:26:38.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:38.325 | INFO | Uploaded 36GiB to GPU in 2.51701s (14.3027GiB/s) | 20260507 22:26:38.325 | INFO | Downloaded 34GiB to GPU in 2.45671s (13.8397GiB/s) | 20260507 22:26:43.464 | INFO | Analyzing chunk frames = [3075584,3076096) | 20260507 22:26:43.464 | INFO | MemInfo: { | 20260507 22:26:43.464 | INFO | "gpu" : " 17 GB", | 20260507 22:26:43.464 | INFO | "huge" : "", | 20260507 22:26:43.464 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:43.464 | INFO | "malloc" : " 14 GB", | 20260507 22:26:43.464 | INFO | "pinned" : " 29 GB" | 20260507 22:26:43.464 | INFO | } | 20260507 22:26:43.464 | INFO | Perf report for the last 5.14036 seconds: | 20260507 22:26:43.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:43.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.464 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:26:43.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.465 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:26:43.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:43.465 | INFO | Uploaded 36GiB to GPU in 2.50849s (14.3513GiB/s) | 20260507 22:26:43.465 | INFO | Downloaded 34GiB to GPU in 2.44901s (13.8832GiB/s) | 20260507 22:26:43.562 | INFO | Starting a round of Baz IO | 20260507 22:26:47.037 | INFO | Populating RTMetrics report | 20260507 22:26:47.037 | INFO | RTMetrics report populated | 20260507 22:26:47.037 | INFO | Opening tmp RTMetrics json file | 20260507 22:26:47.039 | INFO | Serializing RTMetrics to json | 20260507 22:26:47.066 | INFO | Serialization done, swapping tmp file now... | 20260507 22:26:47.067 | INFO | RTMetrics done | 20260507 22:26:47.151 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.588 seconds | 20260507 22:26:47.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.12% / 0.01% / 56.88% | 20260507 22:26:47.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40177% / 1.19209e-05% / 5.96046e-05% | 20260507 22:26:47.151 | INFO | Baz IO overall: | 20260507 22:26:47.151 | INFO | BytesWritten : 1401517841238 | 20260507 22:26:47.151 | INFO | NumEvents : 679024105668 | 20260507 22:26:48.605 | INFO | Analyzing chunk frames = [3076096,3076608) | 20260507 22:26:48.605 | INFO | MemInfo: { | 20260507 22:26:48.605 | INFO | "gpu" : " 17 GB", | 20260507 22:26:48.605 | INFO | "huge" : "", | 20260507 22:26:48.605 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:48.605 | INFO | "malloc" : " 14 GB", | 20260507 22:26:48.605 | INFO | "pinned" : " 29 GB" | 20260507 22:26:48.605 | INFO | } | 20260507 22:26:48.605 | INFO | Perf report for the last 5.14077 seconds: | 20260507 22:26:48.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:48.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:48.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:48.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:48.605 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 22:26:48.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:48.605 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 22:26:48.605 | INFO | PRE_HQ : 66.0%, 1.7, 57.4ms, 0 | 20260507 22:26:48.605 | INFO | BAZWRITER : 69.4%, 4.7, 169.3ms, 0 | 20260507 22:26:48.605 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:26:48.605 | INFO | Uploaded 36GiB to GPU in 2.57004s (14.0075GiB/s) | 20260507 22:26:48.605 | INFO | Downloaded 38GiB to GPU in 2.71455s (13.9986GiB/s) | 20260507 22:26:53.754 | INFO | Analyzing chunk frames = [3076608,3077120) | 20260507 22:26:53.754 | INFO | MemInfo: { | 20260507 22:26:53.754 | INFO | "gpu" : " 17 GB", | 20260507 22:26:53.754 | INFO | "huge" : "", | 20260507 22:26:53.754 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:53.754 | INFO | "malloc" : " 14 GB", | 20260507 22:26:53.754 | INFO | "pinned" : " 29 GB" | 20260507 22:26:53.754 | INFO | } | 20260507 22:26:53.754 | INFO | Perf report for the last 5.14865 seconds: | 20260507 22:26:53.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:53.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260507 22:26:53.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:26:53.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:53.754 | INFO | Uploaded 36GiB to GPU in 2.47924s (14.5206GiB/s) | 20260507 22:26:53.754 | INFO | Downloaded 34GiB to GPU in 2.46251s (13.8071GiB/s) | 20260507 22:26:58.888 | INFO | Analyzing chunk frames = [3077120,3077632) | 20260507 22:26:58.888 | INFO | MemInfo: { | 20260507 22:26:58.888 | INFO | "gpu" : " 17 GB", | 20260507 22:26:58.888 | INFO | "huge" : "", | 20260507 22:26:58.888 | INFO | "huge_shared" : " 29 GB", | 20260507 22:26:58.888 | INFO | "malloc" : " 14 GB", | 20260507 22:26:58.888 | INFO | "pinned" : " 29 GB" | 20260507 22:26:58.888 | INFO | } | 20260507 22:26:58.888 | INFO | Perf report for the last 5.13442 seconds: | 20260507 22:26:58.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:26:58.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 22:26:58.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:26:58.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:26:58.888 | INFO | Uploaded 36GiB to GPU in 2.49883s (14.4067GiB/s) | 20260507 22:26:58.888 | INFO | Downloaded 34GiB to GPU in 2.48406s (13.6872GiB/s) | 20260507 22:27:04.022 | INFO | Analyzing chunk frames = [3077632,3078144) | 20260507 22:27:04.022 | INFO | MemInfo: { | 20260507 22:27:04.022 | INFO | "gpu" : " 17 GB", | 20260507 22:27:04.022 | INFO | "huge" : "", | 20260507 22:27:04.022 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:04.022 | INFO | "malloc" : " 14 GB", | 20260507 22:27:04.022 | INFO | "pinned" : " 29 GB" | 20260507 22:27:04.022 | INFO | } | 20260507 22:27:04.022 | INFO | Perf report for the last 5.13384 seconds: | 20260507 22:27:04.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:04.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 22:27:04.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:27:04.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:04.022 | INFO | Uploaded 36GiB to GPU in 2.49458s (14.4313GiB/s) | 20260507 22:27:04.022 | INFO | Downloaded 34GiB to GPU in 2.41097s (14.1022GiB/s) | 20260507 22:27:09.162 | INFO | Analyzing chunk frames = [3078144,3078656) | 20260507 22:27:09.162 | INFO | MemInfo: { | 20260507 22:27:09.162 | INFO | "gpu" : " 17 GB", | 20260507 22:27:09.162 | INFO | "huge" : "", | 20260507 22:27:09.162 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:09.162 | INFO | "malloc" : " 14 GB", | 20260507 22:27:09.162 | INFO | "pinned" : " 29 GB" | 20260507 22:27:09.162 | INFO | } | 20260507 22:27:09.162 | INFO | Perf report for the last 5.14034 seconds: | 20260507 22:27:09.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:09.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.163 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260507 22:27:09.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.163 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 22:27:09.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:09.163 | INFO | Uploaded 36GiB to GPU in 2.32916s (15.4562GiB/s) | 20260507 22:27:09.163 | INFO | Downloaded 34GiB to GPU in 2.49896s (13.6057GiB/s) | 20260507 22:27:14.303 | INFO | Analyzing chunk frames = [3078656,3079168) | 20260507 22:27:14.303 | INFO | MemInfo: { | 20260507 22:27:14.303 | INFO | "gpu" : " 17 GB", | 20260507 22:27:14.303 | INFO | "huge" : "", | 20260507 22:27:14.303 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:14.303 | INFO | "malloc" : " 14 GB", | 20260507 22:27:14.303 | INFO | "pinned" : " 29 GB" | 20260507 22:27:14.303 | INFO | } | 20260507 22:27:14.303 | INFO | Perf report for the last 5.14076 seconds: | 20260507 22:27:14.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:14.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:27:14.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 22:27:14.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:14.303 | INFO | Uploaded 36GiB to GPU in 2.50391s (14.3775GiB/s) | 20260507 22:27:14.303 | INFO | Downloaded 34GiB to GPU in 2.43249s (13.9775GiB/s) | 20260507 22:27:19.442 | INFO | Analyzing chunk frames = [3079168,3079680) | 20260507 22:27:19.442 | INFO | MemInfo: { | 20260507 22:27:19.442 | INFO | "gpu" : " 17 GB", | 20260507 22:27:19.442 | INFO | "huge" : "", | 20260507 22:27:19.442 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:19.442 | INFO | "malloc" : " 14 GB", | 20260507 22:27:19.442 | INFO | "pinned" : " 29 GB" | 20260507 22:27:19.443 | INFO | } | 20260507 22:27:19.443 | INFO | Perf report for the last 5.13954 seconds: | 20260507 22:27:19.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:19.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 22:27:19.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:27:19.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:19.443 | INFO | Uploaded 36GiB to GPU in 2.51604s (14.3082GiB/s) | 20260507 22:27:19.443 | INFO | Downloaded 34GiB to GPU in 2.45202s (13.8661GiB/s) | 20260507 22:27:24.583 | INFO | Analyzing chunk frames = [3079680,3080192) | 20260507 22:27:24.583 | INFO | MemInfo: { | 20260507 22:27:24.583 | INFO | "gpu" : " 17 GB", | 20260507 22:27:24.583 | INFO | "huge" : "", | 20260507 22:27:24.583 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:24.583 | INFO | "malloc" : " 14 GB", | 20260507 22:27:24.583 | INFO | "pinned" : " 29 GB" | 20260507 22:27:24.583 | INFO | } | 20260507 22:27:24.583 | INFO | Perf report for the last 5.14015 seconds: | 20260507 22:27:24.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:24.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 22:27:24.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 22:27:24.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:24.583 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:27:24.583 | INFO | Total mem usage: 16.7 GiB | 20260507 22:27:24.583 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:27:24.584 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.8%) | 20260507 22:27:24.584 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:27:24.584 | INFO | High Water Breakdown: | 20260507 22:27:24.584 | INFO | 60.2% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:27:24.584 | INFO | 39.7% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:27:24.584 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:27:24.584 | INFO | ------------------------------------------ | 20260507 22:27:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:27:24.584 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:27:24.584 | INFO | Total mem usage: 26.5 GiB | 20260507 22:27:24.584 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:27:24.584 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:27:24.584 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:27:24.584 | INFO | Uploaded 36GiB to GPU in 2.50397s (14.3772GiB/s) | 20260507 22:27:24.584 | INFO | Downloaded 34GiB to GPU in 2.44306s (13.917GiB/s) | 20260507 22:27:24.584 | INFO | High Water Breakdown: | 20260507 22:27:24.584 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:27:24.584 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:27:24.584 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:27:24.584 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:27:24.584 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:27:24.584 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:27:24.584 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:27:24.584 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:27:24.584 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:27:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:27:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:27:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:27:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:27:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:27:24.584 | INFO | ------------------------------------------ | 20260507 22:27:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:27:24.584 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:27:24.584 | INFO | Total mem usage: 27.8 GiB | 20260507 22:27:24.584 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:27:24.584 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:27:24.584 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:27:24.584 | INFO | High Water Breakdown: | 20260507 22:27:24.584 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:27:24.584 | INFO | ------------------------------------------ | 20260507 22:27:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:27:24.584 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:27:24.584 | INFO | Total mem usage: 15.7 GiB | 20260507 22:27:24.584 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:27:24.584 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:27:24.584 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:27:24.584 | INFO | High Water Breakdown: | 20260507 22:27:24.584 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:27:24.584 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:27:24.584 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:27:24.584 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:27:24.584 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:27:24.584 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:27:24.584 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:27:24.584 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:27:24.584 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:27:24.584 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:27:24.584 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:27:24.584 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:27:24.584 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:27:24.584 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:27:24.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:27:24.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:27:24.584 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:27:24.584 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:27:24.584 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:27:24.584 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:27:24.584 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:27:24.584 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:27:24.584 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:27:24.584 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:27:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:27:24.584 | INFO | ------------------------------------------ | 20260507 22:27:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:27:24.584 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:27:24.584 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:27:24.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:27:24.584 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.41e+01 / 1.47e+00 | 20260507 22:27:24.584 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.13e-02 / 4.29e-03 | 20260507 22:27:24.585 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.49e+00, 5.97e-01 / 1.98e-01 | 20260507 22:27:24.585 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.52e+00, 2.09e-01 / 8.28e-02 | 20260507 22:27:24.585 | INFO | PerfReport: 16.00%, DME -- 4.21e+01 / 4.58e+01, 3.88e+00 / 1.50e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.63e+00 / 1.53e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.47e+00 / 2.77e+00, 4.33e-01 / 2.11e-01 | 20260507 22:27:24.585 | INFO | PerfReport: 3.49%, Metrics -- 1.15e+00 / 1.40e+00, 2.90e-01 / 1.69e-01 | 20260507 22:27:24.585 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.91e-01, 5.70e-02 / 3.03e-02 | 20260507 22:27:24.585 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:27:24.585 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:27:24.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:27:24.585 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 24.22%, Upload -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 30.81%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 1.95%, LockUpload -- 2.08e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 15.88%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:27:24.585 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 22:27:24.680 | INFO | Starting a round of Baz IO | 20260507 22:27:28.161 | INFO | Populating RTMetrics report | 20260507 22:27:28.161 | INFO | RTMetrics report populated | 20260507 22:27:28.161 | INFO | Opening tmp RTMetrics json file | 20260507 22:27:28.183 | INFO | Serializing RTMetrics to json | 20260507 22:27:28.186 | INFO | Serialization done, swapping tmp file now... | 20260507 22:27:28.187 | INFO | RTMetrics done | 20260507 22:27:28.249 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.568 seconds | 20260507 22:27:28.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.11% / 0.01% / 56.88% | 20260507 22:27:28.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40105% / 1.58946e-05% / 4.76837e-05% | 20260507 22:27:28.249 | INFO | Baz IO overall: | 20260507 22:27:28.249 | INFO | BytesWritten : 1403379922678 | 20260507 22:27:28.249 | INFO | NumEvents : 679922249536 | 20260507 22:27:29.719 | INFO | Analyzing chunk frames = [3080192,3080704) | 20260507 22:27:29.719 | INFO | MemInfo: { | 20260507 22:27:29.719 | INFO | "gpu" : " 17 GB", | 20260507 22:27:29.719 | INFO | "huge" : "", | 20260507 22:27:29.719 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:29.719 | INFO | "malloc" : " 14 GB", | 20260507 22:27:29.719 | INFO | "pinned" : " 29 GB" | 20260507 22:27:29.719 | INFO | } | 20260507 22:27:29.719 | INFO | Perf report for the last 5.13646 seconds: | 20260507 22:27:29.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:29.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:29.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:29.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:29.720 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260507 22:27:29.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:29.720 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 22:27:29.720 | INFO | PRE_HQ : 65.9%, 1.7, 57.3ms, 0 | 20260507 22:27:29.720 | INFO | BAZWRITER : 69.0%, 4.4, 155.2ms, 0 | 20260507 22:27:29.720 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 22:27:29.720 | INFO | Uploaded 36GiB to GPU in 2.54827s (14.1272GiB/s) | 20260507 22:27:29.720 | INFO | Downloaded 38GiB to GPU in 2.67032s (14.2305GiB/s) | 20260507 22:27:34.862 | INFO | Analyzing chunk frames = [3080704,3081216) | 20260507 22:27:34.862 | INFO | MemInfo: { | 20260507 22:27:34.862 | INFO | "gpu" : " 17 GB", | 20260507 22:27:34.862 | INFO | "huge" : "", | 20260507 22:27:34.862 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:34.862 | INFO | "malloc" : " 14 GB", | 20260507 22:27:34.862 | INFO | "pinned" : " 29 GB" | 20260507 22:27:34.862 | INFO | } | 20260507 22:27:34.862 | INFO | Perf report for the last 5.14239 seconds: | 20260507 22:27:34.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:34.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.862 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:27:34.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.862 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:27:34.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:34.863 | INFO | Uploaded 36GiB to GPU in 2.56676s (14.0255GiB/s) | 20260507 22:27:34.863 | INFO | Downloaded 34GiB to GPU in 2.49174s (13.6451GiB/s) | 20260507 22:27:40.004 | INFO | Analyzing chunk frames = [3081216,3081728) | 20260507 22:27:40.004 | INFO | MemInfo: { | 20260507 22:27:40.004 | INFO | "gpu" : " 17 GB", | 20260507 22:27:40.005 | INFO | "huge" : "", | 20260507 22:27:40.005 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:40.005 | INFO | "malloc" : " 14 GB", | 20260507 22:27:40.005 | INFO | "pinned" : " 29 GB" | 20260507 22:27:40.005 | INFO | } | 20260507 22:27:40.005 | INFO | Perf report for the last 5.14277 seconds: | 20260507 22:27:40.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:40.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:27:40.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:27:40.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:40.005 | INFO | Uploaded 36GiB to GPU in 2.55262s (14.1032GiB/s) | 20260507 22:27:40.005 | INFO | Downloaded 34GiB to GPU in 2.481s (13.7042GiB/s) | 20260507 22:27:45.138 | INFO | Analyzing chunk frames = [3081728,3082240) | 20260507 22:27:45.138 | INFO | MemInfo: { | 20260507 22:27:45.138 | INFO | "gpu" : " 17 GB", | 20260507 22:27:45.138 | INFO | "huge" : "", | 20260507 22:27:45.138 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:45.138 | INFO | "malloc" : " 14 GB", | 20260507 22:27:45.138 | INFO | "pinned" : " 29 GB" | 20260507 22:27:45.138 | INFO | } | 20260507 22:27:45.138 | INFO | Perf report for the last 5.1336 seconds: | 20260507 22:27:45.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:45.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.138 | INFO | ANALYSIS : 67.7%, 3.0, 104.7ms, 0 | 20260507 22:27:45.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.138 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:27:45.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:45.139 | INFO | Uploaded 36GiB to GPU in 2.536s (14.1956GiB/s) | 20260507 22:27:45.139 | INFO | Downloaded 34GiB to GPU in 2.50517s (13.5719GiB/s) | 20260507 22:27:50.275 | INFO | Analyzing chunk frames = [3082240,3082752) | 20260507 22:27:50.275 | INFO | MemInfo: { | 20260507 22:27:50.275 | INFO | "gpu" : " 17 GB", | 20260507 22:27:50.275 | INFO | "huge" : "", | 20260507 22:27:50.276 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:50.276 | INFO | "malloc" : " 14 GB", | 20260507 22:27:50.276 | INFO | "pinned" : " 29 GB" | 20260507 22:27:50.276 | INFO | } | 20260507 22:27:50.276 | INFO | Perf report for the last 5.13754 seconds: | 20260507 22:27:50.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:50.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 22:27:50.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:27:50.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:50.276 | INFO | Uploaded 36GiB to GPU in 2.32479s (15.4853GiB/s) | 20260507 22:27:50.276 | INFO | Downloaded 34GiB to GPU in 2.50928s (13.5497GiB/s) | 20260507 22:27:55.418 | INFO | Analyzing chunk frames = [3082752,3083264) | 20260507 22:27:55.418 | INFO | MemInfo: { | 20260507 22:27:55.418 | INFO | "gpu" : " 17 GB", | 20260507 22:27:55.419 | INFO | "huge" : "", | 20260507 22:27:55.419 | INFO | "huge_shared" : " 29 GB", | 20260507 22:27:55.419 | INFO | "malloc" : " 14 GB", | 20260507 22:27:55.419 | INFO | "pinned" : " 29 GB" | 20260507 22:27:55.419 | INFO | } | 20260507 22:27:55.419 | INFO | Perf report for the last 5.14285 seconds: | 20260507 22:27:55.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:27:55.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 22:27:55.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:27:55.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:27:55.419 | INFO | Uploaded 36GiB to GPU in 2.51092s (14.3374GiB/s) | 20260507 22:27:55.419 | INFO | Downloaded 34GiB to GPU in 2.43195s (13.9806GiB/s) | 20260507 22:28:00.560 | INFO | Analyzing chunk frames = [3083264,3083776) | 20260507 22:28:00.560 | INFO | MemInfo: { | 20260507 22:28:00.560 | INFO | "gpu" : " 17 GB", | 20260507 22:28:00.560 | INFO | "huge" : "", | 20260507 22:28:00.560 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:00.560 | INFO | "malloc" : " 14 GB", | 20260507 22:28:00.560 | INFO | "pinned" : " 29 GB" | 20260507 22:28:00.560 | INFO | } | 20260507 22:28:00.560 | INFO | Perf report for the last 5.14126 seconds: | 20260507 22:28:00.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:00.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 22:28:00.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 22:28:00.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:00.560 | INFO | Uploaded 36GiB to GPU in 2.51192s (14.3317GiB/s) | 20260507 22:28:00.561 | INFO | Downloaded 34GiB to GPU in 2.45128s (13.8703GiB/s) | 20260507 22:28:05.699 | INFO | Analyzing chunk frames = [3083776,3084288) | 20260507 22:28:05.699 | INFO | MemInfo: { | 20260507 22:28:05.699 | INFO | "gpu" : " 17 GB", | 20260507 22:28:05.699 | INFO | "huge" : "", | 20260507 22:28:05.699 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:05.699 | INFO | "malloc" : " 14 GB", | 20260507 22:28:05.699 | INFO | "pinned" : " 29 GB" | 20260507 22:28:05.699 | INFO | } | 20260507 22:28:05.699 | INFO | Perf report for the last 5.13925 seconds: | 20260507 22:28:05.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:05.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:28:05.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:28:05.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:05.699 | INFO | Uploaded 36GiB to GPU in 2.50579s (14.3667GiB/s) | 20260507 22:28:05.700 | INFO | Downloaded 34GiB to GPU in 2.44821s (13.8877GiB/s) | 20260507 22:28:05.796 | INFO | Starting a round of Baz IO | 20260507 22:28:09.259 | INFO | Populating RTMetrics report | 20260507 22:28:09.260 | INFO | RTMetrics report populated | 20260507 22:28:09.260 | INFO | Opening tmp RTMetrics json file | 20260507 22:28:09.271 | INFO | Serializing RTMetrics to json | 20260507 22:28:09.292 | INFO | Serialization done, swapping tmp file now... | 20260507 22:28:09.312 | INFO | RTMetrics done | 20260507 22:28:09.478 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.681 seconds | 20260507 22:28:09.478 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.11% / 0.01% / 56.89% | 20260507 22:28:09.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40124% / 3.97364e-06% / 5.5631e-05% | 20260507 22:28:09.478 | INFO | Baz IO overall: | 20260507 22:28:09.478 | INFO | BytesWritten : 1405244256327 | 20260507 22:28:09.478 | INFO | NumEvents : 680822179836 | 20260507 22:28:10.835 | INFO | Analyzing chunk frames = [3084288,3084800) | 20260507 22:28:10.835 | INFO | MemInfo: { | 20260507 22:28:10.835 | INFO | "gpu" : " 17 GB", | 20260507 22:28:10.836 | INFO | "huge" : "", | 20260507 22:28:10.836 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:10.836 | INFO | "malloc" : " 14 GB", | 20260507 22:28:10.836 | INFO | "pinned" : " 29 GB" | 20260507 22:28:10.836 | INFO | } | 20260507 22:28:10.836 | INFO | Perf report for the last 5.13654 seconds: | 20260507 22:28:10.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:10.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:10.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:10.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:10.836 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 22:28:10.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:10.836 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 22:28:10.836 | INFO | PRE_HQ : 65.9%, 1.7, 57.5ms, 0 | 20260507 22:28:10.836 | INFO | BAZWRITER : 71.2%, 3.6, 133.3ms, 0 | 20260507 22:28:10.836 | INFO | BAZ_SAMPLING : 3.9%, 1.0, 2.0ms, 0 | 20260507 22:28:10.836 | INFO | Uploaded 36GiB to GPU in 2.55747s (14.0764GiB/s) | 20260507 22:28:10.836 | INFO | Downloaded 38GiB to GPU in 2.68532s (14.151GiB/s) | 20260507 22:28:15.974 | INFO | Analyzing chunk frames = [3084800,3085312) | 20260507 22:28:15.974 | INFO | MemInfo: { | 20260507 22:28:15.974 | INFO | "gpu" : " 17 GB", | 20260507 22:28:15.974 | INFO | "huge" : "", | 20260507 22:28:15.974 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:15.974 | INFO | "malloc" : " 14 GB", | 20260507 22:28:15.974 | INFO | "pinned" : " 29 GB" | 20260507 22:28:15.974 | INFO | } | 20260507 22:28:15.974 | INFO | Perf report for the last 5.13799 seconds: | 20260507 22:28:15.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:15.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:28:15.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 22:28:15.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:15.974 | INFO | Uploaded 36GiB to GPU in 2.57399s (13.9861GiB/s) | 20260507 22:28:15.974 | INFO | Downloaded 34GiB to GPU in 2.4834s (13.6909GiB/s) | 20260507 22:28:21.115 | INFO | Analyzing chunk frames = [3085312,3085824) | 20260507 22:28:21.116 | INFO | MemInfo: { | 20260507 22:28:21.116 | INFO | "gpu" : " 17 GB", | 20260507 22:28:21.116 | INFO | "huge" : "", | 20260507 22:28:21.116 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:21.116 | INFO | "malloc" : " 14 GB", | 20260507 22:28:21.116 | INFO | "pinned" : " 29 GB" | 20260507 22:28:21.116 | INFO | } | 20260507 22:28:21.116 | INFO | Perf report for the last 5.14185 seconds: | 20260507 22:28:21.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:21.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 22:28:21.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | PRE_HQ : 35.9%, 1.0, 18.7ms, 0 | 20260507 22:28:21.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:21.116 | INFO | Uploaded 36GiB to GPU in 2.51692s (14.3032GiB/s) | 20260507 22:28:21.116 | INFO | Downloaded 34GiB to GPU in 2.4785s (13.718GiB/s) | 20260507 22:28:26.254 | INFO | Analyzing chunk frames = [3085824,3086336) | 20260507 22:28:26.254 | INFO | MemInfo: { | 20260507 22:28:26.254 | INFO | "gpu" : " 17 GB", | 20260507 22:28:26.255 | INFO | "huge" : "", | 20260507 22:28:26.255 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:26.255 | INFO | "malloc" : " 14 GB", | 20260507 22:28:26.255 | INFO | "pinned" : " 29 GB" | 20260507 22:28:26.255 | INFO | } | 20260507 22:28:26.255 | INFO | Perf report for the last 5.13902 seconds: | 20260507 22:28:26.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:26.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | ANALYSIS : 67.8%, 3.0, 104.9ms, 0 | 20260507 22:28:26.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:28:26.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:26.255 | INFO | Uploaded 36GiB to GPU in 2.54594s (14.1401GiB/s) | 20260507 22:28:26.255 | INFO | Downloaded 34GiB to GPU in 2.48375s (13.689GiB/s) | 20260507 22:28:31.392 | INFO | Analyzing chunk frames = [3086336,3086848) | 20260507 22:28:31.392 | INFO | MemInfo: { | 20260507 22:28:31.392 | INFO | "gpu" : " 17 GB", | 20260507 22:28:31.392 | INFO | "huge" : "", | 20260507 22:28:31.392 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:31.392 | INFO | "malloc" : " 14 GB", | 20260507 22:28:31.392 | INFO | "pinned" : " 29 GB" | 20260507 22:28:31.392 | INFO | } | 20260507 22:28:31.392 | INFO | Perf report for the last 5.13753 seconds: | 20260507 22:28:31.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:31.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.392 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:28:31.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.392 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 22:28:31.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:31.393 | INFO | Uploaded 36GiB to GPU in 2.32969s (15.4527GiB/s) | 20260507 22:28:31.393 | INFO | Downloaded 34GiB to GPU in 2.50144s (13.5921GiB/s) | 20260507 22:28:36.531 | INFO | Analyzing chunk frames = [3086848,3087360) | 20260507 22:28:36.531 | INFO | MemInfo: { | 20260507 22:28:36.531 | INFO | "gpu" : " 17 GB", | 20260507 22:28:36.531 | INFO | "huge" : "", | 20260507 22:28:36.531 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:36.531 | INFO | "malloc" : " 14 GB", | 20260507 22:28:36.531 | INFO | "pinned" : " 29 GB" | 20260507 22:28:36.531 | INFO | } | 20260507 22:28:36.531 | INFO | Perf report for the last 5.13881 seconds: | 20260507 22:28:36.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:36.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 22:28:36.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:28:36.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:36.531 | INFO | Uploaded 36GiB to GPU in 2.47845s (14.5252GiB/s) | 20260507 22:28:36.531 | INFO | Downloaded 34GiB to GPU in 2.44068s (13.9306GiB/s) | 20260507 22:28:41.675 | INFO | Analyzing chunk frames = [3087360,3087872) | 20260507 22:28:41.675 | INFO | MemInfo: { | 20260507 22:28:41.675 | INFO | "gpu" : " 17 GB", | 20260507 22:28:41.675 | INFO | "huge" : "", | 20260507 22:28:41.675 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:41.675 | INFO | "malloc" : " 14 GB", | 20260507 22:28:41.675 | INFO | "pinned" : " 29 GB" | 20260507 22:28:41.675 | INFO | } | 20260507 22:28:41.675 | INFO | Perf report for the last 5.14421 seconds: | 20260507 22:28:41.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:41.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.675 | INFO | ANALYSIS : 67.0%, 3.0, 104.8ms, 0 | 20260507 22:28:41.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.675 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 22:28:41.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:41.676 | INFO | Uploaded 36GiB to GPU in 2.46769s (14.5885GiB/s) | 20260507 22:28:41.676 | INFO | Downloaded 34GiB to GPU in 2.4591s (13.8262GiB/s) | 20260507 22:28:46.813 | INFO | Analyzing chunk frames = [3087872,3088384) | 20260507 22:28:46.813 | INFO | MemInfo: { | 20260507 22:28:46.813 | INFO | "gpu" : " 17 GB", | 20260507 22:28:46.813 | INFO | "huge" : "", | 20260507 22:28:46.813 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:46.813 | INFO | "malloc" : " 14 GB", | 20260507 22:28:46.813 | INFO | "pinned" : " 29 GB" | 20260507 22:28:46.813 | INFO | } | 20260507 22:28:46.813 | INFO | Perf report for the last 5.13757 seconds: | 20260507 22:28:46.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:46.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:28:46.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:28:46.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:46.813 | INFO | Uploaded 36GiB to GPU in 2.45278s (14.6772GiB/s) | 20260507 22:28:46.813 | INFO | Downloaded 34GiB to GPU in 2.45412s (13.8542GiB/s) | 20260507 22:28:46.813 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:28:46.813 | INFO | Total mem usage: 16.7 GiB | 20260507 22:28:46.813 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:28:46.813 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260507 22:28:46.813 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:28:46.813 | INFO | High Water Breakdown: | 20260507 22:28:46.813 | INFO | 60.1% : 9.03 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:28:46.813 | INFO | 39.8% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:28:46.813 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:28:46.813 | INFO | ------------------------------------------ | 20260507 22:28:46.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:28:46.813 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:28:46.813 | INFO | Total mem usage: 26.5 GiB | 20260507 22:28:46.813 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:28:46.814 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:28:46.814 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:28:46.814 | INFO | High Water Breakdown: | 20260507 22:28:46.814 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:28:46.814 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:28:46.814 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:28:46.814 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:28:46.814 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:28:46.814 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:28:46.814 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:28:46.814 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:28:46.814 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:28:46.814 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:28:46.814 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:28:46.814 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:28:46.814 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:28:46.814 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:28:46.814 | INFO | ------------------------------------------ | 20260507 22:28:46.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:28:46.814 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:28:46.814 | INFO | Total mem usage: 27.8 GiB | 20260507 22:28:46.814 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:28:46.814 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:28:46.814 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:28:46.814 | INFO | High Water Breakdown: | 20260507 22:28:46.814 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:28:46.814 | INFO | ------------------------------------------ | 20260507 22:28:46.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:28:46.814 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:28:46.814 | INFO | Total mem usage: 15.7 GiB | 20260507 22:28:46.814 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:28:46.814 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:28:46.814 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:28:46.814 | INFO | High Water Breakdown: | 20260507 22:28:46.814 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:28:46.814 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:28:46.814 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:28:46.814 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:28:46.814 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:28:46.814 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:28:46.814 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:28:46.814 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:28:46.814 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:28:46.814 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:28:46.814 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:28:46.814 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:28:46.814 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:28:46.814 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:28:46.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:28:46.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:28:46.814 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:28:46.814 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:28:46.814 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:28:46.814 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:28:46.814 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:28:46.814 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:28:46.814 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:28:46.814 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:28:46.814 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:28:46.814 | INFO | ------------------------------------------ | 20260507 22:28:46.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:28:46.814 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:28:46.814 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:28:46.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:28:46.814 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.65e+01, 1.42e+01 / 1.68e+00 | 20260507 22:28:46.814 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.03e-01, 1.25e-02 / 6.68e-03 | 20260507 22:28:46.814 | INFO | PerfReport: 12.18%, Baseline -- 4.01e+00 / 4.48e+00, 5.95e-01 / 2.17e-01 | 20260507 22:28:46.814 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 1.10e-01 | 20260507 22:28:46.814 | INFO | PerfReport: 15.97%, DME -- 4.21e+01 / 4.56e+01, 3.85e+00 / 1.63e+00 | 20260507 22:28:46.814 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.72e+00 / 1.99e+00 | 20260507 22:28:46.814 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.71e+00, 4.35e-01 / 2.55e-01 | 20260507 22:28:46.815 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.35e+00, 2.87e-01 / 1.74e-01 | 20260507 22:28:46.815 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.56e-01 / 3.87e-01, 5.76e-02 / 2.61e-02 | 20260507 22:28:46.815 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:28:46.815 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:28:46.815 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:28:46.815 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 24.33%, Upload -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 24.32%, Download -- 2.60e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 30.78%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 2.04%, LockUpload -- 2.19e+00 / 0.00e+00, 5.23e+00 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 2.70%, LockDownload -- 2.89e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 22:28:46.815 | INFO | PerfReport: 0.02%, Misc -- 1.61e-02 / 0.00e+00, 3.32e-02 / 0.00e+00 | 20260507 22:28:46.910 | INFO | Starting a round of Baz IO | 20260507 22:28:50.370 | INFO | Populating RTMetrics report | 20260507 22:28:50.371 | INFO | RTMetrics report populated | 20260507 22:28:50.371 | INFO | Opening tmp RTMetrics json file | 20260507 22:28:50.402 | INFO | Serializing RTMetrics to json | 20260507 22:28:50.428 | INFO | Serialization done, swapping tmp file now... | 20260507 22:28:50.444 | INFO | RTMetrics done | 20260507 22:28:50.531 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.62 seconds | 20260507 22:28:50.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.10% / 0.01% / 56.89% | 20260507 22:28:50.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.39855% / 1.58946e-05% / 3.17891e-05% | 20260507 22:28:50.531 | INFO | Baz IO overall: | 20260507 22:28:50.531 | INFO | BytesWritten : 1407107368792 | 20260507 22:28:50.531 | INFO | NumEvents : 681721450720 | 20260507 22:28:51.953 | INFO | Analyzing chunk frames = [3088384,3088896) | 20260507 22:28:51.953 | INFO | MemInfo: { | 20260507 22:28:51.953 | INFO | "gpu" : " 17 GB", | 20260507 22:28:51.953 | INFO | "huge" : "", | 20260507 22:28:51.953 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:51.953 | INFO | "malloc" : " 14 GB", | 20260507 22:28:51.953 | INFO | "pinned" : " 29 GB" | 20260507 22:28:51.953 | INFO | } | 20260507 22:28:51.953 | INFO | Perf report for the last 5.14011 seconds: | 20260507 22:28:51.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:51.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:51.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:51.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:51.953 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260507 22:28:51.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:51.953 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 22:28:51.953 | INFO | PRE_HQ : 65.7%, 1.7, 57.7ms, 0 | 20260507 22:28:51.953 | INFO | BAZWRITER : 69.9%, 3.9, 140.7ms, 0 | 20260507 22:28:51.953 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:28:51.953 | INFO | Uploaded 36GiB to GPU in 2.51281s (14.3266GiB/s) | 20260507 22:28:51.953 | INFO | Downloaded 38GiB to GPU in 2.69189s (14.1165GiB/s) | 20260507 22:28:57.087 | INFO | Analyzing chunk frames = [3088896,3089408) | 20260507 22:28:57.087 | INFO | MemInfo: { | 20260507 22:28:57.087 | INFO | "gpu" : " 17 GB", | 20260507 22:28:57.087 | INFO | "huge" : "", | 20260507 22:28:57.087 | INFO | "huge_shared" : " 29 GB", | 20260507 22:28:57.088 | INFO | "malloc" : " 14 GB", | 20260507 22:28:57.088 | INFO | "pinned" : " 29 GB" | 20260507 22:28:57.088 | INFO | } | 20260507 22:28:57.088 | INFO | Perf report for the last 5.13467 seconds: | 20260507 22:28:57.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:28:57.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.088 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 22:28:57.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.088 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:28:57.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:28:57.089 | INFO | Uploaded 36GiB to GPU in 2.52996s (14.2295GiB/s) | 20260507 22:28:57.089 | INFO | Downloaded 34GiB to GPU in 2.47179s (13.7552GiB/s) | 20260507 22:29:02.228 | INFO | Analyzing chunk frames = [3089408,3089920) | 20260507 22:29:02.228 | INFO | MemInfo: { | 20260507 22:29:02.228 | INFO | "gpu" : " 17 GB", | 20260507 22:29:02.228 | INFO | "huge" : "", | 20260507 22:29:02.228 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:02.228 | INFO | "malloc" : " 14 GB", | 20260507 22:29:02.228 | INFO | "pinned" : " 29 GB" | 20260507 22:29:02.228 | INFO | } | 20260507 22:29:02.228 | INFO | Perf report for the last 5.14039 seconds: | 20260507 22:29:02.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:02.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:29:02.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:29:02.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:02.228 | INFO | Uploaded 36GiB to GPU in 2.52624s (14.2504GiB/s) | 20260507 22:29:02.228 | INFO | Downloaded 34GiB to GPU in 2.45697s (13.8382GiB/s) | 20260507 22:29:07.368 | INFO | Analyzing chunk frames = [3089920,3090432) | 20260507 22:29:07.368 | INFO | MemInfo: { | 20260507 22:29:07.368 | INFO | "gpu" : " 17 GB", | 20260507 22:29:07.368 | INFO | "huge" : "", | 20260507 22:29:07.368 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:07.368 | INFO | "malloc" : " 14 GB", | 20260507 22:29:07.368 | INFO | "pinned" : " 29 GB" | 20260507 22:29:07.368 | INFO | } | 20260507 22:29:07.369 | INFO | Perf report for the last 5.14042 seconds: | 20260507 22:29:07.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:07.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:29:07.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260507 22:29:07.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:07.369 | INFO | Uploaded 36GiB to GPU in 2.47767s (14.5298GiB/s) | 20260507 22:29:07.369 | INFO | Downloaded 34GiB to GPU in 2.4017s (14.1566GiB/s) | 20260507 22:29:12.509 | INFO | Analyzing chunk frames = [3090432,3090944) | 20260507 22:29:12.509 | INFO | MemInfo: { | 20260507 22:29:12.509 | INFO | "gpu" : " 17 GB", | 20260507 22:29:12.509 | INFO | "huge" : "", | 20260507 22:29:12.509 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:12.509 | INFO | "malloc" : " 14 GB", | 20260507 22:29:12.509 | INFO | "pinned" : " 29 GB" | 20260507 22:29:12.509 | INFO | } | 20260507 22:29:12.509 | INFO | Perf report for the last 5.14097 seconds: | 20260507 22:29:12.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:12.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 22:29:12.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 22:29:12.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:12.510 | INFO | Uploaded 36GiB to GPU in 2.41502s (14.9067GiB/s) | 20260507 22:29:12.510 | INFO | Downloaded 34GiB to GPU in 2.49738s (13.6142GiB/s) | 20260507 22:29:17.651 | INFO | Analyzing chunk frames = [3090944,3091456) | 20260507 22:29:17.651 | INFO | MemInfo: { | 20260507 22:29:17.651 | INFO | "gpu" : " 17 GB", | 20260507 22:29:17.651 | INFO | "huge" : "", | 20260507 22:29:17.651 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:17.651 | INFO | "malloc" : " 14 GB", | 20260507 22:29:17.651 | INFO | "pinned" : " 29 GB" | 20260507 22:29:17.651 | INFO | } | 20260507 22:29:17.651 | INFO | Perf report for the last 5.14182 seconds: | 20260507 22:29:17.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:17.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.651 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:29:17.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.651 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 22:29:17.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:17.652 | INFO | Uploaded 36GiB to GPU in 2.51588s (14.3091GiB/s) | 20260507 22:29:17.652 | INFO | Downloaded 34GiB to GPU in 2.42337s (14.03GiB/s) | 20260507 22:29:22.785 | INFO | Analyzing chunk frames = [3091456,3091968) | 20260507 22:29:22.785 | INFO | MemInfo: { | 20260507 22:29:22.785 | INFO | "gpu" : " 17 GB", | 20260507 22:29:22.785 | INFO | "huge" : "", | 20260507 22:29:22.785 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:22.785 | INFO | "malloc" : " 14 GB", | 20260507 22:29:22.785 | INFO | "pinned" : " 29 GB" | 20260507 22:29:22.785 | INFO | } | 20260507 22:29:22.785 | INFO | Perf report for the last 5.13412 seconds: | 20260507 22:29:22.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:22.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.786 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260507 22:29:22.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.786 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:29:22.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:22.786 | INFO | Uploaded 36GiB to GPU in 2.54433s (14.1491GiB/s) | 20260507 22:29:22.786 | INFO | Downloaded 34GiB to GPU in 2.45852s (13.8295GiB/s) | 20260507 22:29:27.927 | INFO | Analyzing chunk frames = [3091968,3092480) | 20260507 22:29:27.927 | INFO | MemInfo: { | 20260507 22:29:27.927 | INFO | "gpu" : " 17 GB", | 20260507 22:29:27.927 | INFO | "huge" : "", | 20260507 22:29:27.927 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:27.927 | INFO | "malloc" : " 14 GB", | 20260507 22:29:27.927 | INFO | "pinned" : " 29 GB" | 20260507 22:29:27.927 | INFO | } | 20260507 22:29:27.928 | INFO | Perf report for the last 5.1421 seconds: | 20260507 22:29:27.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:27.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:29:27.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:29:27.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:27.928 | INFO | Uploaded 36GiB to GPU in 2.51137s (14.3348GiB/s) | 20260507 22:29:27.928 | INFO | Downloaded 34GiB to GPU in 2.44097s (13.9289GiB/s) | 20260507 22:29:28.025 | INFO | Starting a round of Baz IO | 20260507 22:29:31.481 | INFO | Populating RTMetrics report | 20260507 22:29:31.481 | INFO | RTMetrics report populated | 20260507 22:29:31.481 | INFO | Opening tmp RTMetrics json file | 20260507 22:29:31.498 | INFO | Serializing RTMetrics to json | 20260507 22:29:31.547 | INFO | Serialization done, swapping tmp file now... | 20260507 22:29:31.571 | INFO | RTMetrics done | 20260507 22:29:31.595 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.569 seconds | 20260507 22:29:31.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.09% / 0.01% / 56.90% | 20260507 22:29:31.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.39283% / 7.94729e-06% / 5.16574e-05% | 20260507 22:29:31.595 | INFO | Baz IO overall: | 20260507 22:29:31.595 | INFO | BytesWritten : 1408968062480 | 20260507 22:29:31.595 | INFO | NumEvents : 682619007238 | 20260507 22:29:33.072 | INFO | Analyzing chunk frames = [3092480,3092992) | 20260507 22:29:33.072 | INFO | MemInfo: { | 20260507 22:29:33.072 | INFO | "gpu" : " 17 GB", | 20260507 22:29:33.072 | INFO | "huge" : "", | 20260507 22:29:33.072 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:33.072 | INFO | "malloc" : " 14 GB", | 20260507 22:29:33.072 | INFO | "pinned" : " 29 GB" | 20260507 22:29:33.072 | INFO | } | 20260507 22:29:33.072 | INFO | Perf report for the last 5.14452 seconds: | 20260507 22:29:33.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:33.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:33.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:33.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:33.072 | INFO | ANALYSIS : 68.4%, 3.0, 106.3ms, 0 | 20260507 22:29:33.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:33.072 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 22:29:33.072 | INFO | PRE_HQ : 65.8%, 1.7, 57.5ms, 0 | 20260507 22:29:33.072 | INFO | BAZWRITER : 69.0%, 3.9, 138.2ms, 0 | 20260507 22:29:33.072 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 22:29:33.072 | INFO | Uploaded 36GiB to GPU in 2.49922s (14.4045GiB/s) | 20260507 22:29:33.072 | INFO | Downloaded 38GiB to GPU in 2.69478s (14.1014GiB/s) | 20260507 22:29:38.207 | INFO | Analyzing chunk frames = [3092992,3093504) | 20260507 22:29:38.207 | INFO | MemInfo: { | 20260507 22:29:38.207 | INFO | "gpu" : " 17 GB", | 20260507 22:29:38.207 | INFO | "huge" : "", | 20260507 22:29:38.207 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:38.207 | INFO | "malloc" : " 14 GB", | 20260507 22:29:38.207 | INFO | "pinned" : " 29 GB" | 20260507 22:29:38.207 | INFO | } | 20260507 22:29:38.207 | INFO | Perf report for the last 5.13475 seconds: | 20260507 22:29:38.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:38.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.207 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 22:29:38.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.207 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:29:38.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:38.208 | INFO | Uploaded 36GiB to GPU in 2.55368s (14.0973GiB/s) | 20260507 22:29:38.208 | INFO | Downloaded 34GiB to GPU in 2.48693s (13.6715GiB/s) | 20260507 22:29:43.345 | INFO | Analyzing chunk frames = [3093504,3094016) | 20260507 22:29:43.345 | INFO | MemInfo: { | 20260507 22:29:43.345 | INFO | "gpu" : " 17 GB", | 20260507 22:29:43.345 | INFO | "huge" : "", | 20260507 22:29:43.345 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:43.345 | INFO | "malloc" : " 14 GB", | 20260507 22:29:43.345 | INFO | "pinned" : " 29 GB" | 20260507 22:29:43.345 | INFO | } | 20260507 22:29:43.345 | INFO | Perf report for the last 5.1385 seconds: | 20260507 22:29:43.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:43.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.345 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:29:43.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.345 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:29:43.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:43.346 | INFO | Uploaded 36GiB to GPU in 2.54197s (14.1623GiB/s) | 20260507 22:29:43.346 | INFO | Downloaded 34GiB to GPU in 2.4637s (13.8004GiB/s) | 20260507 22:29:48.484 | INFO | Analyzing chunk frames = [3094016,3094528) | 20260507 22:29:48.484 | INFO | MemInfo: { | 20260507 22:29:48.484 | INFO | "gpu" : " 17 GB", | 20260507 22:29:48.484 | INFO | "huge" : "", | 20260507 22:29:48.484 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:48.484 | INFO | "malloc" : " 14 GB", | 20260507 22:29:48.484 | INFO | "pinned" : " 29 GB" | 20260507 22:29:48.484 | INFO | } | 20260507 22:29:48.484 | INFO | Perf report for the last 5.13898 seconds: | 20260507 22:29:48.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:48.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.484 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 22:29:48.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.485 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:29:48.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:48.485 | INFO | Uploaded 36GiB to GPU in 2.49757s (14.414GiB/s) | 20260507 22:29:48.485 | INFO | Downloaded 34GiB to GPU in 2.4978s (13.612GiB/s) | 20260507 22:29:53.622 | INFO | Analyzing chunk frames = [3094528,3095040) | 20260507 22:29:53.622 | INFO | MemInfo: { | 20260507 22:29:53.622 | INFO | "gpu" : " 17 GB", | 20260507 22:29:53.622 | INFO | "huge" : "", | 20260507 22:29:53.622 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:53.622 | INFO | "malloc" : " 14 GB", | 20260507 22:29:53.622 | INFO | "pinned" : " 29 GB" | 20260507 22:29:53.622 | INFO | } | 20260507 22:29:53.622 | INFO | Perf report for the last 5.13761 seconds: | 20260507 22:29:53.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:53.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 22:29:53.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:29:53.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:53.622 | INFO | Uploaded 36GiB to GPU in 2.37178s (15.1785GiB/s) | 20260507 22:29:53.622 | INFO | Downloaded 34GiB to GPU in 2.49381s (13.6337GiB/s) | 20260507 22:29:58.764 | INFO | Analyzing chunk frames = [3095040,3095552) | 20260507 22:29:58.764 | INFO | MemInfo: { | 20260507 22:29:58.764 | INFO | "gpu" : " 17 GB", | 20260507 22:29:58.764 | INFO | "huge" : "", | 20260507 22:29:58.764 | INFO | "huge_shared" : " 29 GB", | 20260507 22:29:58.764 | INFO | "malloc" : " 14 GB", | 20260507 22:29:58.764 | INFO | "pinned" : " 29 GB" | 20260507 22:29:58.764 | INFO | } | 20260507 22:29:58.764 | INFO | Perf report for the last 5.14224 seconds: | 20260507 22:29:58.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:29:58.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.764 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:29:58.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.764 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 22:29:58.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:29:58.765 | INFO | Uploaded 36GiB to GPU in 2.52167s (14.2763GiB/s) | 20260507 22:29:58.765 | INFO | Downloaded 34GiB to GPU in 2.41855s (14.058GiB/s) | 20260507 22:30:03.904 | INFO | Analyzing chunk frames = [3095552,3096064) | 20260507 22:30:03.904 | INFO | MemInfo: { | 20260507 22:30:03.904 | INFO | "gpu" : " 17 GB", | 20260507 22:30:03.904 | INFO | "huge" : "", | 20260507 22:30:03.904 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:03.904 | INFO | "malloc" : " 14 GB", | 20260507 22:30:03.904 | INFO | "pinned" : " 29 GB" | 20260507 22:30:03.904 | INFO | } | 20260507 22:30:03.904 | INFO | Perf report for the last 5.13991 seconds: | 20260507 22:30:03.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:03.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.904 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 22:30:03.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.904 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:30:03.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:03.905 | INFO | Uploaded 36GiB to GPU in 2.53845s (14.1819GiB/s) | 20260507 22:30:03.905 | INFO | Downloaded 34GiB to GPU in 2.43857s (13.9426GiB/s) | 20260507 22:30:09.040 | INFO | Analyzing chunk frames = [3096064,3096576) | 20260507 22:30:09.040 | INFO | MemInfo: { | 20260507 22:30:09.040 | INFO | "gpu" : " 17 GB", | 20260507 22:30:09.040 | INFO | "huge" : "", | 20260507 22:30:09.040 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:09.040 | INFO | "malloc" : " 14 GB", | 20260507 22:30:09.040 | INFO | "pinned" : " 29 GB" | 20260507 22:30:09.040 | INFO | } | 20260507 22:30:09.040 | INFO | Perf report for the last 5.13587 seconds: | 20260507 22:30:09.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:09.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:30:09.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:30:09.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:09.040 | INFO | Uploaded 36GiB to GPU in 2.52226s (14.2729GiB/s) | 20260507 22:30:09.040 | INFO | Downloaded 34GiB to GPU in 2.42719s (14.008GiB/s) | 20260507 22:30:09.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:30:09.041 | INFO | Total mem usage: 16.7 GiB | 20260507 22:30:09.041 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:30:09.041 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260507 22:30:09.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:30:09.041 | INFO | High Water Breakdown: | 20260507 22:30:09.041 | INFO | 59.7% : 8.77 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:30:09.041 | INFO | 40.1% : 5.9 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:30:09.041 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:30:09.041 | INFO | ------------------------------------------ | 20260507 22:30:09.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:30:09.041 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:30:09.041 | INFO | Total mem usage: 26.5 GiB | 20260507 22:30:09.041 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:30:09.041 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:30:09.041 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:30:09.041 | INFO | High Water Breakdown: | 20260507 22:30:09.041 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:30:09.041 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:30:09.041 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:30:09.041 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:30:09.041 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:30:09.041 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:30:09.041 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:30:09.041 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:30:09.041 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:30:09.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:30:09.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:30:09.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:30:09.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:30:09.041 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:30:09.041 | INFO | ------------------------------------------ | 20260507 22:30:09.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:30:09.041 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:30:09.041 | INFO | Total mem usage: 27.8 GiB | 20260507 22:30:09.041 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:30:09.041 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:30:09.041 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:30:09.041 | INFO | High Water Breakdown: | 20260507 22:30:09.041 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:30:09.041 | INFO | ------------------------------------------ | 20260507 22:30:09.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:30:09.041 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:30:09.041 | INFO | Total mem usage: 15.7 GiB | 20260507 22:30:09.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:30:09.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:30:09.041 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:30:09.041 | INFO | High Water Breakdown: | 20260507 22:30:09.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:30:09.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:30:09.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:30:09.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:30:09.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:30:09.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:30:09.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:30:09.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:30:09.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:30:09.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:30:09.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:30:09.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:30:09.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:30:09.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:30:09.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:30:09.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:30:09.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:30:09.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:30:09.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:30:09.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:30:09.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:30:09.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:30:09.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:30:09.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:30:09.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:30:09.041 | INFO | ------------------------------------------ | 20260507 22:30:09.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:30:09.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:30:09.041 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:30:09.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:30:09.041 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.63e+01, 1.40e+01 / 1.95e+00 | 20260507 22:30:09.041 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.13e-02 / 4.88e-03 | 20260507 22:30:09.041 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.48e+00, 5.93e-01 / 2.14e-01 | 20260507 22:30:09.041 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.10e-01 / 1.18e-01 | 20260507 22:30:09.042 | INFO | PerfReport: 15.85%, DME -- 4.18e+01 / 4.54e+01, 4.00e+00 / 1.89e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.81e+00 / 2.07e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.43e+00 / 2.72e+00, 4.26e-01 / 2.51e-01 | 20260507 22:30:09.042 | INFO | PerfReport: 3.45%, Metrics -- 1.14e+00 / 1.35e+00, 2.85e-01 / 1.57e-01 | 20260507 22:30:09.042 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.86e-01, 5.70e-02 / 3.49e-02 | 20260507 22:30:09.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:30:09.042 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:30:09.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:30:09.042 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 30.83%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 2.02%, LockUpload -- 2.16e+00 / 0.00e+00, 5.20e+00 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 2.60%, LockDownload -- 2.78e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 15.92%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:30:09.042 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260507 22:30:09.137 | INFO | Starting a round of Baz IO | 20260507 22:30:12.598 | INFO | Populating RTMetrics report | 20260507 22:30:12.598 | INFO | RTMetrics report populated | 20260507 22:30:12.598 | INFO | Opening tmp RTMetrics json file | 20260507 22:30:12.612 | INFO | Serializing RTMetrics to json | 20260507 22:30:12.617 | INFO | Serialization done, swapping tmp file now... | 20260507 22:30:12.623 | INFO | RTMetrics done | 20260507 22:30:12.920 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.782 seconds | 20260507 22:30:12.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.09% / 0.01% / 56.91% | 20260507 22:30:12.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40022% / 1.19209e-05% / 7.94729e-05% | 20260507 22:30:12.920 | INFO | Baz IO overall: | 20260507 22:30:12.920 | INFO | BytesWritten : 1410832115031 | 20260507 22:30:12.920 | INFO | NumEvents : 683519444834 | 20260507 22:30:14.182 | INFO | Analyzing chunk frames = [3096576,3097088) | 20260507 22:30:14.182 | INFO | MemInfo: { | 20260507 22:30:14.182 | INFO | "gpu" : " 17 GB", | 20260507 22:30:14.182 | INFO | "huge" : "", | 20260507 22:30:14.182 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:14.182 | INFO | "malloc" : " 14 GB", | 20260507 22:30:14.182 | INFO | "pinned" : " 29 GB" | 20260507 22:30:14.182 | INFO | } | 20260507 22:30:14.182 | INFO | Perf report for the last 5.14189 seconds: | 20260507 22:30:14.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:14.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:14.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:14.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:14.182 | INFO | ANALYSIS : 68.8%, 3.0, 106.8ms, 0 | 20260507 22:30:14.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:14.182 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 22:30:14.182 | INFO | PRE_HQ : 65.7%, 1.7, 57.3ms, 0 | 20260507 22:30:14.182 | INFO | BAZWRITER : 73.1%, 4.1, 154.0ms, 0 | 20260507 22:30:14.182 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 22:30:14.183 | INFO | Uploaded 36GiB to GPU in 2.58043s (13.9512GiB/s) | 20260507 22:30:14.183 | INFO | Downloaded 38GiB to GPU in 2.68972s (14.1279GiB/s) | 20260507 22:30:19.320 | INFO | Analyzing chunk frames = [3097088,3097600) | 20260507 22:30:19.320 | INFO | MemInfo: { | 20260507 22:30:19.320 | INFO | "gpu" : " 17 GB", | 20260507 22:30:19.320 | INFO | "huge" : "", | 20260507 22:30:19.320 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:19.320 | INFO | "malloc" : " 14 GB", | 20260507 22:30:19.320 | INFO | "pinned" : " 29 GB" | 20260507 22:30:19.320 | INFO | } | 20260507 22:30:19.320 | INFO | Perf report for the last 5.13782 seconds: | 20260507 22:30:19.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:19.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 22:30:19.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:30:19.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:19.320 | INFO | Uploaded 36GiB to GPU in 2.58219s (13.9417GiB/s) | 20260507 22:30:19.320 | INFO | Downloaded 34GiB to GPU in 2.47135s (13.7577GiB/s) | 20260507 22:30:24.458 | INFO | Analyzing chunk frames = [3097600,3098112) | 20260507 22:30:24.458 | INFO | MemInfo: { | 20260507 22:30:24.458 | INFO | "gpu" : " 17 GB", | 20260507 22:30:24.458 | INFO | "huge" : "", | 20260507 22:30:24.458 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:24.458 | INFO | "malloc" : " 14 GB", | 20260507 22:30:24.458 | INFO | "pinned" : " 29 GB" | 20260507 22:30:24.458 | INFO | } | 20260507 22:30:24.458 | INFO | Perf report for the last 5.138 seconds: | 20260507 22:30:24.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:24.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 22:30:24.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:30:24.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:24.458 | INFO | Uploaded 36GiB to GPU in 2.55417s (14.0946GiB/s) | 20260507 22:30:24.458 | INFO | Downloaded 34GiB to GPU in 2.45008s (13.8771GiB/s) | 20260507 22:30:29.596 | INFO | Analyzing chunk frames = [3098112,3098624) | 20260507 22:30:29.596 | INFO | MemInfo: { | 20260507 22:30:29.596 | INFO | "gpu" : " 17 GB", | 20260507 22:30:29.596 | INFO | "huge" : "", | 20260507 22:30:29.596 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:29.596 | INFO | "malloc" : " 14 GB", | 20260507 22:30:29.596 | INFO | "pinned" : " 29 GB" | 20260507 22:30:29.596 | INFO | } | 20260507 22:30:29.596 | INFO | Perf report for the last 5.1385 seconds: | 20260507 22:30:29.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:29.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.596 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:30:29.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.597 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:30:29.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:29.597 | INFO | Uploaded 36GiB to GPU in 2.4922s (14.4451GiB/s) | 20260507 22:30:29.597 | INFO | Downloaded 34GiB to GPU in 2.38394s (14.2621GiB/s) | 20260507 22:30:34.735 | INFO | Analyzing chunk frames = [3098624,3099136) | 20260507 22:30:34.735 | INFO | MemInfo: { | 20260507 22:30:34.735 | INFO | "gpu" : " 17 GB", | 20260507 22:30:34.735 | INFO | "huge" : "", | 20260507 22:30:34.735 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:34.736 | INFO | "malloc" : " 14 GB", | 20260507 22:30:34.736 | INFO | "pinned" : " 29 GB" | 20260507 22:30:34.736 | INFO | } | 20260507 22:30:34.736 | INFO | Perf report for the last 5.13928 seconds: | 20260507 22:30:34.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:34.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 22:30:34.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 22:30:34.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:34.736 | INFO | Uploaded 36GiB to GPU in 2.34484s (15.3528GiB/s) | 20260507 22:30:34.736 | INFO | Downloaded 34GiB to GPU in 2.47586s (13.7326GiB/s) | 20260507 22:30:39.880 | INFO | Analyzing chunk frames = [3099136,3099648) | 20260507 22:30:39.880 | INFO | MemInfo: { | 20260507 22:30:39.880 | INFO | "gpu" : " 17 GB", | 20260507 22:30:39.880 | INFO | "huge" : "", | 20260507 22:30:39.880 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:39.880 | INFO | "malloc" : " 14 GB", | 20260507 22:30:39.880 | INFO | "pinned" : " 29 GB" | 20260507 22:30:39.880 | INFO | } | 20260507 22:30:39.880 | INFO | Perf report for the last 5.14464 seconds: | 20260507 22:30:39.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:39.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.880 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 22:30:39.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.880 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:30:39.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:39.881 | INFO | Uploaded 36GiB to GPU in 2.54133s (14.1658GiB/s) | 20260507 22:30:39.881 | INFO | Downloaded 34GiB to GPU in 2.42036s (14.0475GiB/s) | 20260507 22:30:45.023 | INFO | Analyzing chunk frames = [3099648,3100160) | 20260507 22:30:45.023 | INFO | MemInfo: { | 20260507 22:30:45.023 | INFO | "gpu" : " 17 GB", | 20260507 22:30:45.023 | INFO | "huge" : "", | 20260507 22:30:45.023 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:45.023 | INFO | "malloc" : " 14 GB", | 20260507 22:30:45.023 | INFO | "pinned" : " 29 GB" | 20260507 22:30:45.023 | INFO | } | 20260507 22:30:45.023 | INFO | Perf report for the last 5.14269 seconds: | 20260507 22:30:45.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:45.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.023 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260507 22:30:45.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.023 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:30:45.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:45.024 | INFO | Uploaded 36GiB to GPU in 2.55167s (14.1084GiB/s) | 20260507 22:30:45.024 | INFO | Downloaded 34GiB to GPU in 2.44425s (13.9102GiB/s) | 20260507 22:30:50.155 | INFO | Analyzing chunk frames = [3100160,3100672) | 20260507 22:30:50.155 | INFO | MemInfo: { | 20260507 22:30:50.155 | INFO | "gpu" : " 17 GB", | 20260507 22:30:50.155 | INFO | "huge" : "", | 20260507 22:30:50.155 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:50.155 | INFO | "malloc" : " 14 GB", | 20260507 22:30:50.155 | INFO | "pinned" : " 29 GB" | 20260507 22:30:50.155 | INFO | } | 20260507 22:30:50.155 | INFO | Perf report for the last 5.13191 seconds: | 20260507 22:30:50.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:50.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:30:50.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:30:50.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:50.155 | INFO | Uploaded 36GiB to GPU in 2.53174s (14.2195GiB/s) | 20260507 22:30:50.155 | INFO | Downloaded 34GiB to GPU in 2.43678s (13.9528GiB/s) | 20260507 22:30:50.252 | INFO | Starting a round of Baz IO | 20260507 22:30:53.724 | INFO | Populating RTMetrics report | 20260507 22:30:53.725 | INFO | RTMetrics report populated | 20260507 22:30:53.725 | INFO | Opening tmp RTMetrics json file | 20260507 22:30:53.769 | INFO | Serializing RTMetrics to json | 20260507 22:30:53.844 | INFO | Serialization done, swapping tmp file now... | 20260507 22:30:53.854 | INFO | RTMetrics done | 20260507 22:30:53.897 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.644 seconds | 20260507 22:30:53.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.08% / 0.01% / 56.91% | 20260507 22:30:53.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38821% / 3.97364e-06% / 4.37101e-05% | 20260507 22:30:53.897 | INFO | Baz IO overall: | 20260507 22:30:53.897 | INFO | BytesWritten : 1412691703101 | 20260507 22:30:53.897 | INFO | NumEvents : 684416490986 | 20260507 22:30:55.292 | INFO | Analyzing chunk frames = [3100672,3101184) | 20260507 22:30:55.292 | INFO | MemInfo: { | 20260507 22:30:55.292 | INFO | "gpu" : " 17 GB", | 20260507 22:30:55.292 | INFO | "huge" : "", | 20260507 22:30:55.292 | INFO | "huge_shared" : " 29 GB", | 20260507 22:30:55.292 | INFO | "malloc" : " 14 GB", | 20260507 22:30:55.292 | INFO | "pinned" : " 29 GB" | 20260507 22:30:55.292 | INFO | } | 20260507 22:30:55.292 | INFO | Perf report for the last 5.13739 seconds: | 20260507 22:30:55.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:30:55.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:55.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:55.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:55.292 | INFO | ANALYSIS : 68.8%, 3.0, 106.6ms, 0 | 20260507 22:30:55.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:30:55.292 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 22:30:55.292 | INFO | PRE_HQ : 65.8%, 1.7, 57.0ms, 0 | 20260507 22:30:55.292 | INFO | BAZWRITER : 70.5%, 4.8, 173.5ms, 0 | 20260507 22:30:55.292 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 22:30:55.293 | INFO | Uploaded 36GiB to GPU in 2.57045s (14.0053GiB/s) | 20260507 22:30:55.293 | INFO | Downloaded 38GiB to GPU in 2.70186s (14.0644GiB/s) | 20260507 22:31:00.437 | INFO | Analyzing chunk frames = [3101184,3101696) | 20260507 22:31:00.437 | INFO | MemInfo: { | 20260507 22:31:00.437 | INFO | "gpu" : " 17 GB", | 20260507 22:31:00.437 | INFO | "huge" : "", | 20260507 22:31:00.437 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:00.437 | INFO | "malloc" : " 14 GB", | 20260507 22:31:00.437 | INFO | "pinned" : " 29 GB" | 20260507 22:31:00.437 | INFO | } | 20260507 22:31:00.437 | INFO | Perf report for the last 5.14493 seconds: | 20260507 22:31:00.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:00.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.437 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:31:00.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.437 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:31:00.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:00.438 | INFO | Uploaded 36GiB to GPU in 2.54085s (14.1685GiB/s) | 20260507 22:31:00.438 | INFO | Downloaded 34GiB to GPU in 2.48234s (13.6968GiB/s) | 20260507 22:31:05.570 | INFO | Analyzing chunk frames = [3101696,3102208) | 20260507 22:31:05.570 | INFO | MemInfo: { | 20260507 22:31:05.570 | INFO | "gpu" : " 17 GB", | 20260507 22:31:05.570 | INFO | "huge" : "", | 20260507 22:31:05.570 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:05.570 | INFO | "malloc" : " 14 GB", | 20260507 22:31:05.570 | INFO | "pinned" : " 29 GB" | 20260507 22:31:05.570 | INFO | } | 20260507 22:31:05.570 | INFO | Perf report for the last 5.13308 seconds: | 20260507 22:31:05.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:05.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 22:31:05.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:31:05.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:05.571 | INFO | Uploaded 36GiB to GPU in 2.5083s (14.3523GiB/s) | 20260507 22:31:05.571 | INFO | Downloaded 34GiB to GPU in 2.49015s (13.6538GiB/s) | 20260507 22:31:10.712 | INFO | Analyzing chunk frames = [3102208,3102720) | 20260507 22:31:10.712 | INFO | MemInfo: { | 20260507 22:31:10.712 | INFO | "gpu" : " 17 GB", | 20260507 22:31:10.712 | INFO | "huge" : "", | 20260507 22:31:10.712 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:10.712 | INFO | "malloc" : " 14 GB", | 20260507 22:31:10.712 | INFO | "pinned" : " 29 GB" | 20260507 22:31:10.712 | INFO | } | 20260507 22:31:10.712 | INFO | Perf report for the last 5.14163 seconds: | 20260507 22:31:10.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:10.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:31:10.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:31:10.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:10.712 | INFO | Uploaded 36GiB to GPU in 2.47072s (14.5706GiB/s) | 20260507 22:31:10.712 | INFO | Downloaded 34GiB to GPU in 2.39765s (14.1805GiB/s) | 20260507 22:31:15.852 | INFO | Analyzing chunk frames = [3102720,3103232) | 20260507 22:31:15.852 | INFO | MemInfo: { | 20260507 22:31:15.852 | INFO | "gpu" : " 17 GB", | 20260507 22:31:15.852 | INFO | "huge" : "", | 20260507 22:31:15.852 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:15.852 | INFO | "malloc" : " 14 GB", | 20260507 22:31:15.852 | INFO | "pinned" : " 29 GB" | 20260507 22:31:15.852 | INFO | } | 20260507 22:31:15.852 | INFO | Perf report for the last 5.14044 seconds: | 20260507 22:31:15.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:15.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.852 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 22:31:15.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.852 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:31:15.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:15.853 | INFO | Uploaded 36GiB to GPU in 2.32727s (15.4688GiB/s) | 20260507 22:31:15.853 | INFO | Downloaded 34GiB to GPU in 2.50973s (13.5473GiB/s) | 20260507 22:31:20.989 | INFO | Analyzing chunk frames = [3103232,3103744) | 20260507 22:31:20.989 | INFO | MemInfo: { | 20260507 22:31:20.989 | INFO | "gpu" : " 17 GB", | 20260507 22:31:20.989 | INFO | "huge" : "", | 20260507 22:31:20.989 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:20.989 | INFO | "malloc" : " 14 GB", | 20260507 22:31:20.989 | INFO | "pinned" : " 29 GB" | 20260507 22:31:20.989 | INFO | } | 20260507 22:31:20.990 | INFO | Perf report for the last 5.13711 seconds: | 20260507 22:31:20.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:20.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 22:31:20.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:31:20.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:20.990 | INFO | Uploaded 36GiB to GPU in 2.46595s (14.5988GiB/s) | 20260507 22:31:20.990 | INFO | Downloaded 34GiB to GPU in 2.44426s (13.9101GiB/s) | 20260507 22:31:26.134 | INFO | Analyzing chunk frames = [3103744,3104256) | 20260507 22:31:26.134 | INFO | MemInfo: { | 20260507 22:31:26.134 | INFO | "gpu" : " 17 GB", | 20260507 22:31:26.134 | INFO | "huge" : "", | 20260507 22:31:26.134 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:26.134 | INFO | "malloc" : " 14 GB", | 20260507 22:31:26.134 | INFO | "pinned" : " 29 GB" | 20260507 22:31:26.134 | INFO | } | 20260507 22:31:26.134 | INFO | Perf report for the last 5.14425 seconds: | 20260507 22:31:26.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:26.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 22:31:26.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:31:26.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:26.134 | INFO | Uploaded 36GiB to GPU in 2.481s (14.5103GiB/s) | 20260507 22:31:26.134 | INFO | Downloaded 34GiB to GPU in 2.47229s (13.7525GiB/s) | 20260507 22:31:31.273 | INFO | Analyzing chunk frames = [3104256,3104768) | 20260507 22:31:31.273 | INFO | MemInfo: { | 20260507 22:31:31.273 | INFO | "gpu" : " 17 GB", | 20260507 22:31:31.273 | INFO | "huge" : "", | 20260507 22:31:31.273 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:31.273 | INFO | "malloc" : " 14 GB", | 20260507 22:31:31.273 | INFO | "pinned" : " 29 GB" | 20260507 22:31:31.273 | INFO | } | 20260507 22:31:31.273 | INFO | Perf report for the last 5.13935 seconds: | 20260507 22:31:31.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:31.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.273 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260507 22:31:31.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.273 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:31:31.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:31.274 | INFO | Uploaded 36GiB to GPU in 2.45051s (14.6908GiB/s) | 20260507 22:31:31.274 | INFO | Downloaded 34GiB to GPU in 2.45175s (13.8676GiB/s) | 20260507 22:31:31.274 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:31:31.274 | INFO | Total mem usage: 16.7 GiB | 20260507 22:31:31.274 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:31:31.274 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -4.71%) | 20260507 22:31:31.274 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:31:31.274 | INFO | High Water Breakdown: | 20260507 22:31:31.274 | INFO | 61.1% : 9.74 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:31:31.274 | INFO | 38.8% : 6.18 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:31:31.274 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:31:31.274 | INFO | ------------------------------------------ | 20260507 22:31:31.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:31:31.274 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:31:31.274 | INFO | Total mem usage: 26.5 GiB | 20260507 22:31:31.274 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:31:31.274 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:31:31.274 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:31:31.274 | INFO | High Water Breakdown: | 20260507 22:31:31.274 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:31:31.274 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:31:31.274 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:31:31.274 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:31:31.274 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:31:31.274 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:31:31.274 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:31:31.274 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:31:31.274 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:31:31.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:31:31.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:31:31.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:31:31.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:31:31.274 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:31:31.274 | INFO | ------------------------------------------ | 20260507 22:31:31.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:31:31.274 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:31:31.274 | INFO | Total mem usage: 27.8 GiB | 20260507 22:31:31.274 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:31:31.274 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:31:31.274 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:31:31.274 | INFO | High Water Breakdown: | 20260507 22:31:31.274 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:31:31.274 | INFO | ------------------------------------------ | 20260507 22:31:31.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:31:31.274 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:31:31.274 | INFO | Total mem usage: 15.7 GiB | 20260507 22:31:31.274 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:31:31.274 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:31:31.274 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:31:31.274 | INFO | High Water Breakdown: | 20260507 22:31:31.274 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:31:31.274 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:31:31.274 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:31:31.274 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:31:31.274 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:31:31.274 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:31:31.274 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:31:31.274 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:31:31.275 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:31:31.275 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:31:31.275 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:31:31.275 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:31:31.275 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:31:31.275 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:31:31.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:31:31.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:31:31.275 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:31:31.275 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:31:31.275 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:31:31.275 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:31:31.275 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:31:31.275 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:31:31.275 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:31:31.275 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:31:31.275 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:31:31.275 | INFO | ------------------------------------------ | 20260507 22:31:31.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:31:31.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:31:31.275 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:31:31.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:31:31.275 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.41e+01 / 1.60e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.14e-02 / 5.69e-03 | 20260507 22:31:31.275 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.45e+00, 5.94e-01 / 2.14e-01 | 20260507 22:31:31.275 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.47e+00, 2.06e-01 / 1.20e-01 | 20260507 22:31:31.275 | INFO | PerfReport: 15.89%, DME -- 4.18e+01 / 4.56e+01, 3.99e+00 / 1.57e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.74e+00 / 1.69e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.46e+00 / 2.77e+00, 4.27e-01 / 1.69e-01 | 20260507 22:31:31.275 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.37e+00, 2.89e-01 / 1.37e-01 | 20260507 22:31:31.275 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.85e-01, 5.69e-02 / 2.88e-02 | 20260507 22:31:31.275 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:31:31.275 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:31:31.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:31:31.275 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 30.80%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 2.09%, LockUpload -- 2.23e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 15.95%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:31:31.275 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 22:31:31.371 | INFO | Starting a round of Baz IO | 20260507 22:31:34.835 | INFO | Populating RTMetrics report | 20260507 22:31:34.836 | INFO | RTMetrics report populated | 20260507 22:31:34.836 | INFO | Opening tmp RTMetrics json file | 20260507 22:31:34.859 | INFO | Serializing RTMetrics to json | 20260507 22:31:34.860 | INFO | Serialization done, swapping tmp file now... | 20260507 22:31:34.861 | INFO | RTMetrics done | 20260507 22:31:34.950 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.579 seconds | 20260507 22:31:34.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.07% / 0.01% / 56.92% | 20260507 22:31:34.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.3872% / 1.58946e-05% / 5.16574e-05% | 20260507 22:31:34.950 | INFO | Baz IO overall: | 20260507 22:31:34.950 | INFO | BytesWritten : 1414552121838 | 20260507 22:31:34.950 | INFO | NumEvents : 685314054195 | 20260507 22:31:36.408 | INFO | Analyzing chunk frames = [3104768,3105280) | 20260507 22:31:36.408 | INFO | MemInfo: { | 20260507 22:31:36.408 | INFO | "gpu" : " 17 GB", | 20260507 22:31:36.408 | INFO | "huge" : "", | 20260507 22:31:36.408 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:36.408 | INFO | "malloc" : " 14 GB", | 20260507 22:31:36.408 | INFO | "pinned" : " 29 GB" | 20260507 22:31:36.408 | INFO | } | 20260507 22:31:36.409 | INFO | Perf report for the last 5.13531 seconds: | 20260507 22:31:36.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:36.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:36.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:36.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:36.409 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 22:31:36.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:36.409 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 22:31:36.409 | INFO | PRE_HQ : 65.7%, 1.7, 57.2ms, 0 | 20260507 22:31:36.409 | INFO | BAZWRITER : 69.3%, 3.9, 140.3ms, 0 | 20260507 22:31:36.409 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 22:31:36.409 | INFO | Uploaded 36GiB to GPU in 2.54511s (14.1448GiB/s) | 20260507 22:31:36.409 | INFO | Downloaded 38GiB to GPU in 2.71083s (14.0178GiB/s) | 20260507 22:31:41.548 | INFO | Analyzing chunk frames = [3105280,3105792) | 20260507 22:31:41.548 | INFO | MemInfo: { | 20260507 22:31:41.548 | INFO | "gpu" : " 17 GB", | 20260507 22:31:41.548 | INFO | "huge" : "", | 20260507 22:31:41.548 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:41.548 | INFO | "malloc" : " 14 GB", | 20260507 22:31:41.548 | INFO | "pinned" : " 29 GB" | 20260507 22:31:41.548 | INFO | } | 20260507 22:31:41.548 | INFO | Perf report for the last 5.13931 seconds: | 20260507 22:31:41.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:41.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 22:31:41.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:31:41.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:41.548 | INFO | Uploaded 36GiB to GPU in 2.47522s (14.5442GiB/s) | 20260507 22:31:41.548 | INFO | Downloaded 34GiB to GPU in 2.48084s (13.705GiB/s) | 20260507 22:31:46.686 | INFO | Analyzing chunk frames = [3105792,3106304) | 20260507 22:31:46.686 | INFO | MemInfo: { | 20260507 22:31:46.686 | INFO | "gpu" : " 17 GB", | 20260507 22:31:46.686 | INFO | "huge" : "", | 20260507 22:31:46.686 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:46.686 | INFO | "malloc" : " 14 GB", | 20260507 22:31:46.686 | INFO | "pinned" : " 29 GB" | 20260507 22:31:46.686 | INFO | } | 20260507 22:31:46.686 | INFO | Perf report for the last 5.13827 seconds: | 20260507 22:31:46.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:46.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 22:31:46.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:31:46.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:46.686 | INFO | Uploaded 36GiB to GPU in 2.50666s (14.3617GiB/s) | 20260507 22:31:46.686 | INFO | Downloaded 34GiB to GPU in 2.48514s (13.6813GiB/s) | 20260507 22:31:51.833 | INFO | Analyzing chunk frames = [3106304,3106816) | 20260507 22:31:51.833 | INFO | MemInfo: { | 20260507 22:31:51.833 | INFO | "gpu" : " 17 GB", | 20260507 22:31:51.833 | INFO | "huge" : "", | 20260507 22:31:51.833 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:51.833 | INFO | "malloc" : " 14 GB", | 20260507 22:31:51.833 | INFO | "pinned" : " 29 GB" | 20260507 22:31:51.833 | INFO | } | 20260507 22:31:51.833 | INFO | Perf report for the last 5.14691 seconds: | 20260507 22:31:51.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:51.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.833 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 22:31:51.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.833 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:31:51.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:51.834 | INFO | Uploaded 36GiB to GPU in 2.44247s (14.7391GiB/s) | 20260507 22:31:51.834 | INFO | Downloaded 34GiB to GPU in 2.40716s (14.1246GiB/s) | 20260507 22:31:56.970 | INFO | Analyzing chunk frames = [3106816,3107328) | 20260507 22:31:56.970 | INFO | MemInfo: { | 20260507 22:31:56.970 | INFO | "gpu" : " 17 GB", | 20260507 22:31:56.970 | INFO | "huge" : "", | 20260507 22:31:56.971 | INFO | "huge_shared" : " 29 GB", | 20260507 22:31:56.971 | INFO | "malloc" : " 14 GB", | 20260507 22:31:56.971 | INFO | "pinned" : " 29 GB" | 20260507 22:31:56.971 | INFO | } | 20260507 22:31:56.971 | INFO | Perf report for the last 5.13764 seconds: | 20260507 22:31:56.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:31:56.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:31:56.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 22:31:56.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:31:56.971 | INFO | Uploaded 36GiB to GPU in 2.32926s (15.4555GiB/s) | 20260507 22:31:56.971 | INFO | Downloaded 34GiB to GPU in 2.51432s (13.5226GiB/s) | 20260507 22:32:02.109 | INFO | Analyzing chunk frames = [3107328,3107840) | 20260507 22:32:02.109 | INFO | MemInfo: { | 20260507 22:32:02.109 | INFO | "gpu" : " 17 GB", | 20260507 22:32:02.109 | INFO | "huge" : "", | 20260507 22:32:02.109 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:02.109 | INFO | "malloc" : " 14 GB", | 20260507 22:32:02.109 | INFO | "pinned" : " 29 GB" | 20260507 22:32:02.109 | INFO | } | 20260507 22:32:02.109 | INFO | Perf report for the last 5.13852 seconds: | 20260507 22:32:02.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:02.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:32:02.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:32:02.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:02.109 | INFO | Uploaded 36GiB to GPU in 2.50953s (14.3453GiB/s) | 20260507 22:32:02.110 | INFO | Downloaded 34GiB to GPU in 2.42799s (14.0034GiB/s) | 20260507 22:32:07.246 | INFO | Analyzing chunk frames = [3107840,3108352) | 20260507 22:32:07.246 | INFO | MemInfo: { | 20260507 22:32:07.246 | INFO | "gpu" : " 17 GB", | 20260507 22:32:07.246 | INFO | "huge" : "", | 20260507 22:32:07.246 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:07.246 | INFO | "malloc" : " 14 GB", | 20260507 22:32:07.246 | INFO | "pinned" : " 29 GB" | 20260507 22:32:07.246 | INFO | } | 20260507 22:32:07.246 | INFO | Perf report for the last 5.13719 seconds: | 20260507 22:32:07.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:07.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.247 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:32:07.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.247 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:32:07.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:07.247 | INFO | Uploaded 36GiB to GPU in 2.53049s (14.2265GiB/s) | 20260507 22:32:07.247 | INFO | Downloaded 34GiB to GPU in 2.46118s (13.8145GiB/s) | 20260507 22:32:12.388 | INFO | Analyzing chunk frames = [3108352,3108864) | 20260507 22:32:12.388 | INFO | MemInfo: { | 20260507 22:32:12.388 | INFO | "gpu" : " 17 GB", | 20260507 22:32:12.388 | INFO | "huge" : "", | 20260507 22:32:12.388 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:12.388 | INFO | "malloc" : " 14 GB", | 20260507 22:32:12.388 | INFO | "pinned" : " 29 GB" | 20260507 22:32:12.388 | INFO | } | 20260507 22:32:12.388 | INFO | Perf report for the last 5.14176 seconds: | 20260507 22:32:12.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:12.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.388 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 22:32:12.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.388 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:32:12.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:12.389 | INFO | Uploaded 36GiB to GPU in 2.48664s (14.4774GiB/s) | 20260507 22:32:12.389 | INFO | Downloaded 34GiB to GPU in 2.44116s (13.9278GiB/s) | 20260507 22:32:12.486 | INFO | Starting a round of Baz IO | 20260507 22:32:15.968 | INFO | Populating RTMetrics report | 20260507 22:32:15.968 | INFO | RTMetrics report populated | 20260507 22:32:15.968 | INFO | Opening tmp RTMetrics json file | 20260507 22:32:15.984 | INFO | Serializing RTMetrics to json | 20260507 22:32:15.987 | INFO | Serialization done, swapping tmp file now... | 20260507 22:32:15.988 | INFO | RTMetrics done | 20260507 22:32:16.053 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.567 seconds | 20260507 22:32:16.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.07% / 0.01% / 56.93% | 20260507 22:32:16.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38301% / 3.97364e-06% / 5.96046e-05% | 20260507 22:32:16.053 | INFO | Baz IO overall: | 20260507 22:32:16.053 | INFO | BytesWritten : 1416410918570 | 20260507 22:32:16.053 | INFO | NumEvents : 686210378549 | 20260507 22:32:17.522 | INFO | Analyzing chunk frames = [3108864,3109376) | 20260507 22:32:17.522 | INFO | MemInfo: { | 20260507 22:32:17.522 | INFO | "gpu" : " 17 GB", | 20260507 22:32:17.522 | INFO | "huge" : "", | 20260507 22:32:17.522 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:17.522 | INFO | "malloc" : " 14 GB", | 20260507 22:32:17.522 | INFO | "pinned" : " 29 GB" | 20260507 22:32:17.522 | INFO | } | 20260507 22:32:17.522 | INFO | Perf report for the last 5.13402 seconds: | 20260507 22:32:17.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:17.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:17.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:17.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:17.522 | INFO | ANALYSIS : 68.8%, 2.9, 106.2ms, 0 | 20260507 22:32:17.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:17.522 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 22:32:17.522 | INFO | PRE_HQ : 66.3%, 1.7, 59.0ms, 0 | 20260507 22:32:17.522 | INFO | BAZWRITER : 69.1%, 4.1, 147.1ms, 0 | 20260507 22:32:17.522 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 22:32:17.523 | INFO | Uploaded 36GiB to GPU in 2.54642s (14.1375GiB/s) | 20260507 22:32:17.523 | INFO | Downloaded 38GiB to GPU in 2.69751s (14.087GiB/s) | 20260507 22:32:22.659 | INFO | Analyzing chunk frames = [3109376,3109888) | 20260507 22:32:22.659 | INFO | MemInfo: { | 20260507 22:32:22.659 | INFO | "gpu" : " 17 GB", | 20260507 22:32:22.659 | INFO | "huge" : "", | 20260507 22:32:22.659 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:22.659 | INFO | "malloc" : " 14 GB", | 20260507 22:32:22.659 | INFO | "pinned" : " 29 GB" | 20260507 22:32:22.659 | INFO | } | 20260507 22:32:22.659 | INFO | Perf report for the last 5.13716 seconds: | 20260507 22:32:22.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:22.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.660 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 22:32:22.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.660 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:32:22.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:22.660 | INFO | Uploaded 36GiB to GPU in 2.57875s (13.9603GiB/s) | 20260507 22:32:22.660 | INFO | Downloaded 34GiB to GPU in 2.49466s (13.6291GiB/s) | 20260507 22:32:27.804 | INFO | Analyzing chunk frames = [3109888,3110400) | 20260507 22:32:27.804 | INFO | MemInfo: { | 20260507 22:32:27.804 | INFO | "gpu" : " 17 GB", | 20260507 22:32:27.804 | INFO | "huge" : "", | 20260507 22:32:27.804 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:27.804 | INFO | "malloc" : " 14 GB", | 20260507 22:32:27.804 | INFO | "pinned" : " 29 GB" | 20260507 22:32:27.804 | INFO | } | 20260507 22:32:27.804 | INFO | Perf report for the last 5.1448 seconds: | 20260507 22:32:27.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:27.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.804 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260507 22:32:27.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.804 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:32:27.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:27.805 | INFO | Uploaded 36GiB to GPU in 2.50768s (14.3559GiB/s) | 20260507 22:32:27.805 | INFO | Downloaded 34GiB to GPU in 2.46426s (13.7972GiB/s) | 20260507 22:32:32.943 | INFO | Analyzing chunk frames = [3110400,3110912) | 20260507 22:32:32.943 | INFO | MemInfo: { | 20260507 22:32:32.943 | INFO | "gpu" : " 17 GB", | 20260507 22:32:32.943 | INFO | "huge" : "", | 20260507 22:32:32.943 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:32.943 | INFO | "malloc" : " 14 GB", | 20260507 22:32:32.943 | INFO | "pinned" : " 29 GB" | 20260507 22:32:32.943 | INFO | } | 20260507 22:32:32.943 | INFO | Perf report for the last 5.13893 seconds: | 20260507 22:32:32.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:32.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.943 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:32:32.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.943 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:32:32.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:32.944 | INFO | Uploaded 36GiB to GPU in 2.46496s (14.6047GiB/s) | 20260507 22:32:32.944 | INFO | Downloaded 34GiB to GPU in 2.39115s (14.2191GiB/s) | 20260507 22:32:38.088 | INFO | Analyzing chunk frames = [3110912,3111424) | 20260507 22:32:38.088 | INFO | MemInfo: { | 20260507 22:32:38.088 | INFO | "gpu" : " 17 GB", | 20260507 22:32:38.088 | INFO | "huge" : "", | 20260507 22:32:38.088 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:38.088 | INFO | "malloc" : " 14 GB", | 20260507 22:32:38.088 | INFO | "pinned" : " 29 GB" | 20260507 22:32:38.088 | INFO | } | 20260507 22:32:38.088 | INFO | Perf report for the last 5.1452 seconds: | 20260507 22:32:38.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:38.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.089 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:32:38.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.089 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:32:38.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:38.089 | INFO | Uploaded 36GiB to GPU in 2.33748s (15.4012GiB/s) | 20260507 22:32:38.089 | INFO | Downloaded 34GiB to GPU in 2.50487s (13.5736GiB/s) | 20260507 22:32:43.224 | INFO | Analyzing chunk frames = [3111424,3111936) | 20260507 22:32:43.224 | INFO | MemInfo: { | 20260507 22:32:43.224 | INFO | "gpu" : " 17 GB", | 20260507 22:32:43.224 | INFO | "huge" : "", | 20260507 22:32:43.224 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:43.224 | INFO | "malloc" : " 14 GB", | 20260507 22:32:43.224 | INFO | "pinned" : " 29 GB" | 20260507 22:32:43.224 | INFO | } | 20260507 22:32:43.224 | INFO | Perf report for the last 5.1359 seconds: | 20260507 22:32:43.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:43.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.224 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260507 22:32:43.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.224 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:32:43.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:43.225 | INFO | Uploaded 36GiB to GPU in 2.46864s (14.5829GiB/s) | 20260507 22:32:43.225 | INFO | Downloaded 34GiB to GPU in 2.44543s (13.9035GiB/s) | 20260507 22:32:48.357 | INFO | Analyzing chunk frames = [3111936,3112448) | 20260507 22:32:48.357 | INFO | MemInfo: { | 20260507 22:32:48.357 | INFO | "gpu" : " 17 GB", | 20260507 22:32:48.357 | INFO | "huge" : "", | 20260507 22:32:48.357 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:48.357 | INFO | "malloc" : " 14 GB", | 20260507 22:32:48.357 | INFO | "pinned" : " 29 GB" | 20260507 22:32:48.357 | INFO | } | 20260507 22:32:48.357 | INFO | Perf report for the last 5.13263 seconds: | 20260507 22:32:48.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:48.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.357 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:32:48.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.357 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 22:32:48.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:48.358 | INFO | Uploaded 36GiB to GPU in 2.48508s (14.4864GiB/s) | 20260507 22:32:48.358 | INFO | Downloaded 34GiB to GPU in 2.45554s (13.8462GiB/s) | 20260507 22:32:53.497 | INFO | Analyzing chunk frames = [3112448,3112960) | 20260507 22:32:53.497 | INFO | MemInfo: { | 20260507 22:32:53.497 | INFO | "gpu" : " 17 GB", | 20260507 22:32:53.497 | INFO | "huge" : "", | 20260507 22:32:53.497 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:53.497 | INFO | "malloc" : " 14 GB", | 20260507 22:32:53.497 | INFO | "pinned" : " 29 GB" | 20260507 22:32:53.497 | INFO | } | 20260507 22:32:53.498 | INFO | Perf report for the last 5.14055 seconds: | 20260507 22:32:53.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:53.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:32:53.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:32:53.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:53.498 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:32:53.498 | INFO | Total mem usage: 16.7 GiB | 20260507 22:32:53.498 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:32:53.498 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260507 22:32:53.498 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:32:53.498 | INFO | High Water Breakdown: | 20260507 22:32:53.498 | INFO | 60% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:32:53.498 | INFO | 39.8% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:32:53.498 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:32:53.498 | INFO | ------------------------------------------ | 20260507 22:32:53.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:32:53.498 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:32:53.498 | INFO | Total mem usage: 26.5 GiB | 20260507 22:32:53.498 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:32:53.498 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:32:53.498 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:32:53.498 | INFO | High Water Breakdown: | 20260507 22:32:53.498 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:32:53.498 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:32:53.498 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:32:53.498 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:32:53.498 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:32:53.498 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:32:53.498 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:32:53.498 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:32:53.498 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:32:53.498 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:32:53.498 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:32:53.498 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:32:53.498 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:32:53.498 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:32:53.498 | INFO | ------------------------------------------ | 20260507 22:32:53.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:32:53.498 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:32:53.498 | INFO | Total mem usage: 27.8 GiB | 20260507 22:32:53.498 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:32:53.498 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 22:32:53.498 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 22:32:53.498 | INFO | Uploaded 36GiB to GPU in 2.48976s (14.4592GiB/s) | 20260507 22:32:53.498 | INFO | High Water Breakdown: | 20260507 22:32:53.498 | INFO | Downloaded 34GiB to GPU in 2.47457s (13.7398GiB/s) | 20260507 22:32:53.498 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 22:32:53.499 | INFO | ------------------------------------------ | 20260507 22:32:53.499 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:32:53.499 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:32:53.499 | INFO | Total mem usage: 15.7 GiB | 20260507 22:32:53.499 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:32:53.499 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:32:53.499 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:32:53.499 | INFO | High Water Breakdown: | 20260507 22:32:53.499 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:32:53.499 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:32:53.499 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:32:53.499 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:32:53.499 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:32:53.499 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:32:53.499 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:32:53.499 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:32:53.499 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:32:53.499 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:32:53.499 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:32:53.499 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:32:53.499 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:32:53.499 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:32:53.499 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:32:53.499 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:32:53.499 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:32:53.499 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:32:53.499 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:32:53.499 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:32:53.499 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:32:53.499 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:32:53.499 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:32:53.499 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:32:53.499 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:32:53.499 | INFO | ------------------------------------------ | 20260507 22:32:53.499 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:32:53.499 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:32:53.499 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:32:53.499 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:32:53.499 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.41e+01 / 1.64e+00 | 20260507 22:32:53.499 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.13e-02 / 4.80e-03 | 20260507 22:32:53.499 | INFO | PerfReport: 12.15%, Baseline -- 4.01e+00 / 4.52e+00, 6.07e-01 / 1.88e-01 | 20260507 22:32:53.499 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.50e+00, 2.11e-01 / 9.65e-02 | 20260507 22:32:53.499 | INFO | PerfReport: 15.72%, DME -- 4.15e+01 / 4.47e+01, 3.66e+00 / 1.20e+00 | 20260507 22:32:53.499 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.86e+01 / 2.09e+01, 1.85e+00 / 2.16e+00 | 20260507 22:32:53.499 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.42e+00 / 2.68e+00, 4.28e-01 / 2.47e-01 | 20260507 22:32:53.499 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.33e+00, 3.17e-01 / 1.75e-01 | 20260507 22:32:53.500 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.54e-01 / 3.80e-01, 5.85e-02 / 3.40e-02 | 20260507 22:32:53.500 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:32:53.500 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:32:53.500 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:32:53.500 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 30.93%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 1.78%, LockUpload -- 1.89e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 16.02%, LockCompute -- 1.71e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:32:53.500 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 22:32:53.626 | INFO | Starting a round of Baz IO | 20260507 22:32:57.088 | INFO | Populating RTMetrics report | 20260507 22:32:57.088 | INFO | RTMetrics report populated | 20260507 22:32:57.088 | INFO | Opening tmp RTMetrics json file | 20260507 22:32:57.126 | INFO | Serializing RTMetrics to json | 20260507 22:32:57.166 | INFO | Serialization done, swapping tmp file now... | 20260507 22:32:57.167 | INFO | RTMetrics done | 20260507 22:32:57.184 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.558 seconds | 20260507 22:32:57.184 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.06% / 0.01% / 56.93% | 20260507 22:32:57.184 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38656% / 1.58946e-05% / 5.96046e-05% | 20260507 22:32:57.184 | INFO | Baz IO overall: | 20260507 22:32:57.184 | INFO | BytesWritten : 1418272106936 | 20260507 22:32:57.184 | INFO | NumEvents : 687108562341 | 20260507 22:32:58.637 | INFO | Analyzing chunk frames = [3112960,3113472) | 20260507 22:32:58.637 | INFO | MemInfo: { | 20260507 22:32:58.637 | INFO | "gpu" : " 17 GB", | 20260507 22:32:58.637 | INFO | "huge" : "", | 20260507 22:32:58.637 | INFO | "huge_shared" : " 29 GB", | 20260507 22:32:58.637 | INFO | "malloc" : " 14 GB", | 20260507 22:32:58.637 | INFO | "pinned" : " 29 GB" | 20260507 22:32:58.637 | INFO | } | 20260507 22:32:58.637 | INFO | Perf report for the last 5.13979 seconds: | 20260507 22:32:58.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:32:58.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:58.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:58.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:58.638 | INFO | ANALYSIS : 69.2%, 2.9, 104.9ms, 0 | 20260507 22:32:58.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:32:58.638 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260507 22:32:58.638 | INFO | PRE_HQ : 68.6%, 1.9, 66.5ms, 0 | 20260507 22:32:58.638 | INFO | BAZWRITER : 69.2%, 4.2, 150.3ms, 0 | 20260507 22:32:58.638 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 22:32:58.638 | INFO | Uploaded 36GiB to GPU in 2.54663s (14.1363GiB/s) | 20260507 22:32:58.638 | INFO | Downloaded 38GiB to GPU in 2.71656s (13.9883GiB/s) | 20260507 22:33:03.783 | INFO | Analyzing chunk frames = [3113472,3113984) | 20260507 22:33:03.783 | INFO | MemInfo: { | 20260507 22:33:03.783 | INFO | "gpu" : " 17 GB", | 20260507 22:33:03.783 | INFO | "huge" : "", | 20260507 22:33:03.783 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:03.783 | INFO | "malloc" : " 14 GB", | 20260507 22:33:03.783 | INFO | "pinned" : " 29 GB" | 20260507 22:33:03.783 | INFO | } | 20260507 22:33:03.783 | INFO | Perf report for the last 5.14554 seconds: | 20260507 22:33:03.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:03.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260507 22:33:03.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:33:03.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:03.783 | INFO | Uploaded 36GiB to GPU in 2.45181s (14.683GiB/s) | 20260507 22:33:03.783 | INFO | Downloaded 34GiB to GPU in 2.48298s (13.6932GiB/s) | 20260507 22:33:08.922 | INFO | Analyzing chunk frames = [3113984,3114496) | 20260507 22:33:08.922 | INFO | MemInfo: { | 20260507 22:33:08.922 | INFO | "gpu" : " 17 GB", | 20260507 22:33:08.922 | INFO | "huge" : "", | 20260507 22:33:08.922 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:08.922 | INFO | "malloc" : " 14 GB", | 20260507 22:33:08.922 | INFO | "pinned" : " 29 GB" | 20260507 22:33:08.922 | INFO | } | 20260507 22:33:08.922 | INFO | Perf report for the last 5.13947 seconds: | 20260507 22:33:08.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:08.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.922 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260507 22:33:08.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.923 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:33:08.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:08.923 | INFO | Uploaded 36GiB to GPU in 2.49806s (14.4112GiB/s) | 20260507 22:33:08.923 | INFO | Downloaded 34GiB to GPU in 2.47964s (13.7117GiB/s) | 20260507 22:33:14.058 | INFO | Analyzing chunk frames = [3114496,3115008) | 20260507 22:33:14.058 | INFO | MemInfo: { | 20260507 22:33:14.058 | INFO | "gpu" : " 17 GB", | 20260507 22:33:14.058 | INFO | "huge" : "", | 20260507 22:33:14.058 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:14.058 | INFO | "malloc" : " 14 GB", | 20260507 22:33:14.058 | INFO | "pinned" : " 29 GB" | 20260507 22:33:14.058 | INFO | } | 20260507 22:33:14.058 | INFO | Perf report for the last 5.13561 seconds: | 20260507 22:33:14.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:14.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260507 22:33:14.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:33:14.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:14.058 | INFO | Uploaded 36GiB to GPU in 2.43919s (14.759GiB/s) | 20260507 22:33:14.058 | INFO | Downloaded 34GiB to GPU in 2.51286s (13.5304GiB/s) | 20260507 22:33:19.200 | INFO | Analyzing chunk frames = [3115008,3115520) | 20260507 22:33:19.200 | INFO | MemInfo: { | 20260507 22:33:19.200 | INFO | "gpu" : " 17 GB", | 20260507 22:33:19.200 | INFO | "huge" : "", | 20260507 22:33:19.200 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:19.200 | INFO | "malloc" : " 14 GB", | 20260507 22:33:19.200 | INFO | "pinned" : " 29 GB" | 20260507 22:33:19.200 | INFO | } | 20260507 22:33:19.200 | INFO | Perf report for the last 5.14201 seconds: | 20260507 22:33:19.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:19.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:33:19.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:33:19.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:19.200 | INFO | Uploaded 36GiB to GPU in 2.32328s (15.4953GiB/s) | 20260507 22:33:19.200 | INFO | Downloaded 34GiB to GPU in 2.51467s (13.5206GiB/s) | 20260507 22:33:24.338 | INFO | Analyzing chunk frames = [3115520,3116032) | 20260507 22:33:24.338 | INFO | MemInfo: { | 20260507 22:33:24.338 | INFO | "gpu" : " 17 GB", | 20260507 22:33:24.338 | INFO | "huge" : "", | 20260507 22:33:24.338 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:24.338 | INFO | "malloc" : " 14 GB", | 20260507 22:33:24.338 | INFO | "pinned" : " 29 GB" | 20260507 22:33:24.338 | INFO | } | 20260507 22:33:24.338 | INFO | Perf report for the last 5.13797 seconds: | 20260507 22:33:24.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:24.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:33:24.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:33:24.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:24.338 | INFO | Uploaded 36GiB to GPU in 2.50965s (14.3447GiB/s) | 20260507 22:33:24.338 | INFO | Downloaded 34GiB to GPU in 2.43465s (13.9651GiB/s) | 20260507 22:33:29.473 | INFO | Analyzing chunk frames = [3116032,3116544) | 20260507 22:33:29.473 | INFO | MemInfo: { | 20260507 22:33:29.473 | INFO | "gpu" : " 17 GB", | 20260507 22:33:29.473 | INFO | "huge" : "", | 20260507 22:33:29.474 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:29.474 | INFO | "malloc" : " 14 GB", | 20260507 22:33:29.474 | INFO | "pinned" : " 29 GB" | 20260507 22:33:29.474 | INFO | } | 20260507 22:33:29.474 | INFO | Perf report for the last 5.13581 seconds: | 20260507 22:33:29.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:29.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260507 22:33:29.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:33:29.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:29.474 | INFO | Uploaded 36GiB to GPU in 2.48598s (14.4812GiB/s) | 20260507 22:33:29.474 | INFO | Downloaded 34GiB to GPU in 2.45701s (13.838GiB/s) | 20260507 22:33:34.612 | INFO | Analyzing chunk frames = [3116544,3117056) | 20260507 22:33:34.612 | INFO | MemInfo: { | 20260507 22:33:34.612 | INFO | "gpu" : " 17 GB", | 20260507 22:33:34.612 | INFO | "huge" : "", | 20260507 22:33:34.612 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:34.612 | INFO | "malloc" : " 14 GB", | 20260507 22:33:34.612 | INFO | "pinned" : " 29 GB" | 20260507 22:33:34.612 | INFO | } | 20260507 22:33:34.612 | INFO | Perf report for the last 5.13843 seconds: | 20260507 22:33:34.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:34.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.612 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:33:34.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.612 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 22:33:34.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:34.613 | INFO | Uploaded 36GiB to GPU in 2.47934s (14.52GiB/s) | 20260507 22:33:34.613 | INFO | Downloaded 34GiB to GPU in 2.44729s (13.8929GiB/s) | 20260507 22:33:34.710 | INFO | Starting a round of Baz IO | 20260507 22:33:38.171 | INFO | Populating RTMetrics report | 20260507 22:33:38.172 | INFO | RTMetrics report populated | 20260507 22:33:38.172 | INFO | Opening tmp RTMetrics json file | 20260507 22:33:38.185 | INFO | Serializing RTMetrics to json | 20260507 22:33:38.204 | INFO | Serialization done, swapping tmp file now... | 20260507 22:33:38.216 | INFO | RTMetrics done | 20260507 22:33:38.283 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.573 seconds | 20260507 22:33:38.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.06% / 0.01% / 56.94% | 20260507 22:33:38.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38846% / 7.94729e-06% / 7.54992e-05% | 20260507 22:33:38.283 | INFO | Baz IO overall: | 20260507 22:33:38.283 | INFO | BytesWritten : 1420135411252 | 20260507 22:33:38.283 | INFO | NumEvents : 688008408826 | 20260507 22:33:39.753 | INFO | Analyzing chunk frames = [3117056,3117568) | 20260507 22:33:39.753 | INFO | MemInfo: { | 20260507 22:33:39.753 | INFO | "gpu" : " 17 GB", | 20260507 22:33:39.753 | INFO | "huge" : "", | 20260507 22:33:39.753 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:39.753 | INFO | "malloc" : " 14 GB", | 20260507 22:33:39.753 | INFO | "pinned" : " 29 GB" | 20260507 22:33:39.753 | INFO | } | 20260507 22:33:39.753 | INFO | Perf report for the last 5.14072 seconds: | 20260507 22:33:39.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:39.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:39.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:39.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:39.753 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260507 22:33:39.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:39.753 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 22:33:39.753 | INFO | PRE_HQ : 65.5%, 1.7, 57.3ms, 0 | 20260507 22:33:39.753 | INFO | BAZWRITER : 69.0%, 4.0, 140.6ms, 0 | 20260507 22:33:39.753 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 22:33:39.753 | INFO | Uploaded 36GiB to GPU in 2.50822s (14.3528GiB/s) | 20260507 22:33:39.753 | INFO | Downloaded 38GiB to GPU in 2.69992s (14.0745GiB/s) | 20260507 22:33:44.892 | INFO | Analyzing chunk frames = [3117568,3118080) | 20260507 22:33:44.892 | INFO | MemInfo: { | 20260507 22:33:44.892 | INFO | "gpu" : " 17 GB", | 20260507 22:33:44.892 | INFO | "huge" : "", | 20260507 22:33:44.892 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:44.892 | INFO | "malloc" : " 14 GB", | 20260507 22:33:44.892 | INFO | "pinned" : " 29 GB" | 20260507 22:33:44.892 | INFO | } | 20260507 22:33:44.892 | INFO | Perf report for the last 5.13906 seconds: | 20260507 22:33:44.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:44.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:33:44.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | PRE_HQ : 25.0%, 1.0, 13.1ms, 0 | 20260507 22:33:44.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:44.892 | INFO | Uploaded 36GiB to GPU in 2.5696s (14.01GiB/s) | 20260507 22:33:44.892 | INFO | Downloaded 34GiB to GPU in 2.46362s (13.8008GiB/s) | 20260507 22:33:50.036 | INFO | Analyzing chunk frames = [3118080,3118592) | 20260507 22:33:50.036 | INFO | MemInfo: { | 20260507 22:33:50.036 | INFO | "gpu" : " 17 GB", | 20260507 22:33:50.036 | INFO | "huge" : "", | 20260507 22:33:50.037 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:50.037 | INFO | "malloc" : " 14 GB", | 20260507 22:33:50.037 | INFO | "pinned" : " 29 GB" | 20260507 22:33:50.037 | INFO | } | 20260507 22:33:50.037 | INFO | Perf report for the last 5.14458 seconds: | 20260507 22:33:50.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:50.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 22:33:50.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:33:50.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:50.037 | INFO | Uploaded 36GiB to GPU in 2.55522s (14.0888GiB/s) | 20260507 22:33:50.037 | INFO | Downloaded 34GiB to GPU in 2.44975s (13.8789GiB/s) | 20260507 22:33:55.176 | INFO | Analyzing chunk frames = [3118592,3119104) | 20260507 22:33:55.176 | INFO | MemInfo: { | 20260507 22:33:55.176 | INFO | "gpu" : " 17 GB", | 20260507 22:33:55.176 | INFO | "huge" : "", | 20260507 22:33:55.176 | INFO | "huge_shared" : " 29 GB", | 20260507 22:33:55.176 | INFO | "malloc" : " 14 GB", | 20260507 22:33:55.176 | INFO | "pinned" : " 29 GB" | 20260507 22:33:55.176 | INFO | } | 20260507 22:33:55.176 | INFO | Perf report for the last 5.13938 seconds: | 20260507 22:33:55.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:33:55.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.176 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 22:33:55.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.176 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:33:55.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:33:55.177 | INFO | Uploaded 36GiB to GPU in 2.50217s (14.3875GiB/s) | 20260507 22:33:55.177 | INFO | Downloaded 34GiB to GPU in 2.39964s (14.1688GiB/s) | 20260507 22:34:00.313 | INFO | Analyzing chunk frames = [3119104,3119616) | 20260507 22:34:00.313 | INFO | MemInfo: { | 20260507 22:34:00.313 | INFO | "gpu" : " 17 GB", | 20260507 22:34:00.313 | INFO | "huge" : "", | 20260507 22:34:00.313 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:00.313 | INFO | "malloc" : " 14 GB", | 20260507 22:34:00.313 | INFO | "pinned" : " 29 GB" | 20260507 22:34:00.313 | INFO | } | 20260507 22:34:00.313 | INFO | Perf report for the last 5.13713 seconds: | 20260507 22:34:00.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:00.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.313 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 22:34:00.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.313 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:34:00.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:00.314 | INFO | Uploaded 36GiB to GPU in 2.40527s (14.9671GiB/s) | 20260507 22:34:00.314 | INFO | Downloaded 34GiB to GPU in 2.49437s (13.6307GiB/s) | 20260507 22:34:05.454 | INFO | Analyzing chunk frames = [3119616,3120128) | 20260507 22:34:05.454 | INFO | MemInfo: { | 20260507 22:34:05.454 | INFO | "gpu" : " 17 GB", | 20260507 22:34:05.454 | INFO | "huge" : "", | 20260507 22:34:05.454 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:05.454 | INFO | "malloc" : " 14 GB", | 20260507 22:34:05.454 | INFO | "pinned" : " 29 GB" | 20260507 22:34:05.454 | INFO | } | 20260507 22:34:05.454 | INFO | Perf report for the last 5.14106 seconds: | 20260507 22:34:05.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:05.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:34:05.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:34:05.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:05.455 | INFO | Uploaded 36GiB to GPU in 2.50301s (14.3827GiB/s) | 20260507 22:34:05.455 | INFO | Downloaded 34GiB to GPU in 2.42695s (14.0093GiB/s) | 20260507 22:34:10.591 | INFO | Analyzing chunk frames = [3120128,3120640) | 20260507 22:34:10.591 | INFO | MemInfo: { | 20260507 22:34:10.591 | INFO | "gpu" : " 17 GB", | 20260507 22:34:10.591 | INFO | "huge" : "", | 20260507 22:34:10.591 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:10.591 | INFO | "malloc" : " 14 GB", | 20260507 22:34:10.591 | INFO | "pinned" : " 29 GB" | 20260507 22:34:10.591 | INFO | } | 20260507 22:34:10.592 | INFO | Perf report for the last 5.13717 seconds: | 20260507 22:34:10.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:10.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:34:10.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 22:34:10.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:10.592 | INFO | Uploaded 36GiB to GPU in 2.54546s (14.1428GiB/s) | 20260507 22:34:10.592 | INFO | Downloaded 34GiB to GPU in 2.46321s (13.8031GiB/s) | 20260507 22:34:15.731 | INFO | Analyzing chunk frames = [3120640,3121152) | 20260507 22:34:15.731 | INFO | MemInfo: { | 20260507 22:34:15.731 | INFO | "gpu" : " 17 GB", | 20260507 22:34:15.731 | INFO | "huge" : "", | 20260507 22:34:15.731 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:15.731 | INFO | "malloc" : " 14 GB", | 20260507 22:34:15.731 | INFO | "pinned" : " 29 GB" | 20260507 22:34:15.731 | INFO | } | 20260507 22:34:15.731 | INFO | Perf report for the last 5.1397 seconds: | 20260507 22:34:15.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:15.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:34:15.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 22:34:15.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:15.731 | INFO | Uploaded 36GiB to GPU in 2.51169s (14.333GiB/s) | 20260507 22:34:15.732 | INFO | Downloaded 34GiB to GPU in 2.45047s (13.8749GiB/s) | 20260507 22:34:15.732 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:34:15.732 | INFO | Total mem usage: 16.7 GiB | 20260507 22:34:15.732 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:34:15.732 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260507 22:34:15.732 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:34:15.732 | INFO | High Water Breakdown: | 20260507 22:34:15.732 | INFO | 60.1% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:34:15.732 | INFO | 39.7% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:34:15.732 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:34:15.732 | INFO | ------------------------------------------ | 20260507 22:34:15.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:34:15.732 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:34:15.732 | INFO | Total mem usage: 26.5 GiB | 20260507 22:34:15.732 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:34:15.732 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 22:34:15.732 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:34:15.732 | INFO | High Water Breakdown: | 20260507 22:34:15.732 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:34:15.732 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:34:15.732 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:34:15.732 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:34:15.732 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:34:15.732 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:34:15.732 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:34:15.732 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:34:15.732 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:34:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:34:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:34:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:34:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:34:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:34:15.732 | INFO | ------------------------------------------ | 20260507 22:34:15.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:34:15.732 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:34:15.732 | INFO | Total mem usage: 27.8 GiB | 20260507 22:34:15.732 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:34:15.732 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:34:15.732 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:34:15.732 | INFO | High Water Breakdown: | 20260507 22:34:15.732 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:34:15.732 | INFO | ------------------------------------------ | 20260507 22:34:15.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:34:15.732 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:34:15.732 | INFO | Total mem usage: 15.7 GiB | 20260507 22:34:15.732 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:34:15.732 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:34:15.732 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:34:15.732 | INFO | High Water Breakdown: | 20260507 22:34:15.732 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:34:15.732 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:34:15.732 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:34:15.732 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:34:15.732 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:34:15.732 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:34:15.732 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:34:15.732 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:34:15.732 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:34:15.732 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:34:15.732 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:34:15.732 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:34:15.732 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:34:15.732 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:34:15.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:34:15.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:34:15.732 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:34:15.732 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:34:15.732 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:34:15.732 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:34:15.732 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:34:15.732 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:34:15.732 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:34:15.732 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:34:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:34:15.732 | INFO | ------------------------------------------ | 20260507 22:34:15.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:34:15.732 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:34:15.732 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:34:15.732 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:34:15.732 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.64e+01, 1.41e+01 / 1.65e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.04e-01, 1.18e-02 / 5.99e-03 | 20260507 22:34:15.733 | INFO | PerfReport: 12.14%, Baseline -- 4.00e+00 / 4.46e+00, 5.94e-01 / 2.14e-01 | 20260507 22:34:15.733 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.06e-01 | 20260507 22:34:15.733 | INFO | PerfReport: 15.81%, DME -- 4.17e+01 / 4.52e+01, 3.74e+00 / 1.66e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.86e+01 / 2.08e+01, 1.76e+00 / 1.90e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.66e+00, 4.31e-01 / 2.41e-01 | 20260507 22:34:15.733 | INFO | PerfReport: 3.48%, Metrics -- 1.14e+00 / 1.34e+00, 2.88e-01 / 1.87e-01 | 20260507 22:34:15.733 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.81e-01, 5.80e-02 / 2.46e-02 | 20260507 22:34:15.733 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:34:15.733 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:34:15.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:34:15.733 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 24.36%, Download -- 2.60e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 30.90%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 1.89%, LockUpload -- 2.01e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 2.71%, LockDownload -- 2.89e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 15.80%, LockCompute -- 1.68e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 22:34:15.733 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.79e-02 / 0.00e+00 | 20260507 22:34:15.843 | INFO | Starting a round of Baz IO | 20260507 22:34:19.391 | INFO | Populating RTMetrics report | 20260507 22:34:19.392 | INFO | RTMetrics report populated | 20260507 22:34:19.392 | INFO | Opening tmp RTMetrics json file | 20260507 22:34:19.394 | INFO | Serializing RTMetrics to json | 20260507 22:34:19.395 | INFO | Serialization done, swapping tmp file now... | 20260507 22:34:19.396 | INFO | RTMetrics done | 20260507 22:34:19.498 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.654 seconds | 20260507 22:34:19.498 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.05% / 0.01% / 56.94% | 20260507 22:34:19.498 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38301% / 7.94729e-06% / 7.94729e-05% | 20260507 22:34:19.498 | INFO | Baz IO overall: | 20260507 22:34:19.498 | INFO | BytesWritten : 1421997525837 | 20260507 22:34:19.498 | INFO | NumEvents : 688907245585 | 20260507 22:34:20.872 | INFO | Analyzing chunk frames = [3121152,3121664) | 20260507 22:34:20.872 | INFO | MemInfo: { | 20260507 22:34:20.872 | INFO | "gpu" : " 17 GB", | 20260507 22:34:20.872 | INFO | "huge" : "", | 20260507 22:34:20.872 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:20.872 | INFO | "malloc" : " 14 GB", | 20260507 22:34:20.872 | INFO | "pinned" : " 29 GB" | 20260507 22:34:20.872 | INFO | } | 20260507 22:34:20.872 | INFO | Perf report for the last 5.14111 seconds: | 20260507 22:34:20.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:20.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:20.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:20.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:20.872 | INFO | ANALYSIS : 70.1%, 2.9, 105.1ms, 0 | 20260507 22:34:20.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:20.872 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260507 22:34:20.872 | INFO | PRE_HQ : 69.8%, 1.9, 70.7ms, 0 | 20260507 22:34:20.873 | INFO | BAZWRITER : 70.7%, 4.8, 172.8ms, 0 | 20260507 22:34:20.873 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 22:34:20.873 | INFO | Uploaded 36GiB to GPU in 2.53884s (14.1797GiB/s) | 20260507 22:34:20.873 | INFO | Downloaded 38GiB to GPU in 2.70178s (14.0648GiB/s) | 20260507 22:34:26.008 | INFO | Analyzing chunk frames = [3121664,3122176) | 20260507 22:34:26.008 | INFO | MemInfo: { | 20260507 22:34:26.008 | INFO | "gpu" : " 17 GB", | 20260507 22:34:26.008 | INFO | "huge" : "", | 20260507 22:34:26.008 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:26.008 | INFO | "malloc" : " 14 GB", | 20260507 22:34:26.008 | INFO | "pinned" : " 29 GB" | 20260507 22:34:26.009 | INFO | } | 20260507 22:34:26.009 | INFO | Perf report for the last 5.13622 seconds: | 20260507 22:34:26.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:26.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 22:34:26.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:34:26.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:26.009 | INFO | Uploaded 36GiB to GPU in 2.52096s (14.2803GiB/s) | 20260507 22:34:26.009 | INFO | Downloaded 34GiB to GPU in 2.50269s (13.5854GiB/s) | 20260507 22:34:31.151 | INFO | Analyzing chunk frames = [3122176,3122688) | 20260507 22:34:31.151 | INFO | MemInfo: { | 20260507 22:34:31.151 | INFO | "gpu" : " 17 GB", | 20260507 22:34:31.151 | INFO | "huge" : "", | 20260507 22:34:31.151 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:31.151 | INFO | "malloc" : " 14 GB", | 20260507 22:34:31.151 | INFO | "pinned" : " 29 GB" | 20260507 22:34:31.151 | INFO | } | 20260507 22:34:31.151 | INFO | Perf report for the last 5.14277 seconds: | 20260507 22:34:31.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:31.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.151 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 22:34:31.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.151 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:34:31.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:31.152 | INFO | Uploaded 36GiB to GPU in 2.51294s (14.3258GiB/s) | 20260507 22:34:31.152 | INFO | Downloaded 34GiB to GPU in 2.49526s (13.6258GiB/s) | 20260507 22:34:36.290 | INFO | Analyzing chunk frames = [3122688,3123200) | 20260507 22:34:36.290 | INFO | MemInfo: { | 20260507 22:34:36.290 | INFO | "gpu" : " 17 GB", | 20260507 22:34:36.290 | INFO | "huge" : "", | 20260507 22:34:36.290 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:36.290 | INFO | "malloc" : " 14 GB", | 20260507 22:34:36.290 | INFO | "pinned" : " 29 GB" | 20260507 22:34:36.290 | INFO | } | 20260507 22:34:36.290 | INFO | Perf report for the last 5.13873 seconds: | 20260507 22:34:36.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:36.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260507 22:34:36.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:34:36.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:36.290 | INFO | Uploaded 36GiB to GPU in 2.44692s (14.7123GiB/s) | 20260507 22:34:36.291 | INFO | Downloaded 34GiB to GPU in 2.41627s (14.0713GiB/s) | 20260507 22:34:41.422 | INFO | Analyzing chunk frames = [3123200,3123712) | 20260507 22:34:41.422 | INFO | MemInfo: { | 20260507 22:34:41.422 | INFO | "gpu" : " 17 GB", | 20260507 22:34:41.422 | INFO | "huge" : "", | 20260507 22:34:41.422 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:41.422 | INFO | "malloc" : " 14 GB", | 20260507 22:34:41.422 | INFO | "pinned" : " 29 GB" | 20260507 22:34:41.422 | INFO | } | 20260507 22:34:41.422 | INFO | Perf report for the last 5.13176 seconds: | 20260507 22:34:41.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:41.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 22:34:41.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:34:41.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:41.422 | INFO | Uploaded 36GiB to GPU in 2.37148s (15.1804GiB/s) | 20260507 22:34:41.422 | INFO | Downloaded 34GiB to GPU in 2.50442s (13.576GiB/s) | 20260507 22:34:46.563 | INFO | Analyzing chunk frames = [3123712,3124224) | 20260507 22:34:46.563 | INFO | MemInfo: { | 20260507 22:34:46.563 | INFO | "gpu" : " 17 GB", | 20260507 22:34:46.563 | INFO | "huge" : "", | 20260507 22:34:46.563 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:46.563 | INFO | "malloc" : " 14 GB", | 20260507 22:34:46.563 | INFO | "pinned" : " 29 GB" | 20260507 22:34:46.563 | INFO | } | 20260507 22:34:46.563 | INFO | Perf report for the last 5.14083 seconds: | 20260507 22:34:46.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:46.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 22:34:46.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:34:46.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:46.563 | INFO | Uploaded 36GiB to GPU in 2.46129s (14.6264GiB/s) | 20260507 22:34:46.563 | INFO | Downloaded 34GiB to GPU in 2.44885s (13.8841GiB/s) | 20260507 22:34:51.705 | INFO | Analyzing chunk frames = [3124224,3124736) | 20260507 22:34:51.705 | INFO | MemInfo: { | 20260507 22:34:51.705 | INFO | "gpu" : " 17 GB", | 20260507 22:34:51.705 | INFO | "huge" : "", | 20260507 22:34:51.705 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:51.705 | INFO | "malloc" : " 14 GB", | 20260507 22:34:51.705 | INFO | "pinned" : " 29 GB" | 20260507 22:34:51.706 | INFO | } | 20260507 22:34:51.706 | INFO | Perf report for the last 5.14285 seconds: | 20260507 22:34:51.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:51.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 22:34:51.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:34:51.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:51.706 | INFO | Uploaded 36GiB to GPU in 2.44899s (14.6999GiB/s) | 20260507 22:34:51.706 | INFO | Downloaded 34GiB to GPU in 2.46134s (13.8136GiB/s) | 20260507 22:34:56.845 | INFO | Analyzing chunk frames = [3124736,3125248) | 20260507 22:34:56.845 | INFO | MemInfo: { | 20260507 22:34:56.845 | INFO | "gpu" : " 17 GB", | 20260507 22:34:56.845 | INFO | "huge" : "", | 20260507 22:34:56.845 | INFO | "huge_shared" : " 29 GB", | 20260507 22:34:56.845 | INFO | "malloc" : " 14 GB", | 20260507 22:34:56.845 | INFO | "pinned" : " 29 GB" | 20260507 22:34:56.846 | INFO | } | 20260507 22:34:56.846 | INFO | Perf report for the last 5.14003 seconds: | 20260507 22:34:56.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:34:56.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:34:56.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 22:34:56.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:34:56.846 | INFO | Uploaded 36GiB to GPU in 2.46688s (14.5933GiB/s) | 20260507 22:34:56.846 | INFO | Downloaded 34GiB to GPU in 2.4542s (13.8538GiB/s) | 20260507 22:34:56.959 | INFO | Starting a round of Baz IO | 20260507 22:35:00.442 | INFO | Populating RTMetrics report | 20260507 22:35:00.442 | INFO | RTMetrics report populated | 20260507 22:35:00.442 | INFO | Opening tmp RTMetrics json file | 20260507 22:35:00.499 | INFO | Serializing RTMetrics to json | 20260507 22:35:00.571 | INFO | Serialization done, swapping tmp file now... | 20260507 22:35:00.574 | INFO | RTMetrics done | 20260507 22:35:00.719 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.759 seconds | 20260507 22:35:00.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.04% / 0.01% / 56.95% | 20260507 22:35:00.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37863% / 0% / 5.96046e-05% | 20260507 22:35:00.719 | INFO | Baz IO overall: | 20260507 22:35:00.719 | INFO | BytesWritten : 1423857614363 | 20260507 22:35:00.719 | INFO | NumEvents : 689804555269 | 20260507 22:35:01.990 | INFO | Analyzing chunk frames = [3125248,3125760) | 20260507 22:35:01.990 | INFO | MemInfo: { | 20260507 22:35:01.990 | INFO | "gpu" : " 17 GB", | 20260507 22:35:01.990 | INFO | "huge" : "", | 20260507 22:35:01.990 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:01.990 | INFO | "malloc" : " 14 GB", | 20260507 22:35:01.990 | INFO | "pinned" : " 29 GB" | 20260507 22:35:01.990 | INFO | } | 20260507 22:35:01.990 | INFO | Perf report for the last 5.14442 seconds: | 20260507 22:35:01.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:01.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:01.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:01.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:01.990 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260507 22:35:01.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:01.990 | INFO | RT_METRICS : 51.6%, 1.0, 27.1ms, 0 | 20260507 22:35:01.990 | INFO | PRE_HQ : 68.4%, 2.0, 69.4ms, 0 | 20260507 22:35:01.990 | INFO | BAZWRITER : 72.8%, 4.5, 168.4ms, 0 | 20260507 22:35:01.990 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 22:35:01.991 | INFO | Uploaded 36GiB to GPU in 2.59229s (13.8873GiB/s) | 20260507 22:35:01.991 | INFO | Downloaded 38GiB to GPU in 2.70923s (14.0261GiB/s) | 20260507 22:35:07.124 | INFO | Analyzing chunk frames = [3125760,3126272) | 20260507 22:35:07.124 | INFO | MemInfo: { | 20260507 22:35:07.124 | INFO | "gpu" : " 17 GB", | 20260507 22:35:07.124 | INFO | "huge" : "", | 20260507 22:35:07.124 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:07.124 | INFO | "malloc" : " 14 GB", | 20260507 22:35:07.124 | INFO | "pinned" : " 29 GB" | 20260507 22:35:07.124 | INFO | } | 20260507 22:35:07.125 | INFO | Perf report for the last 5.13456 seconds: | 20260507 22:35:07.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:07.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260507 22:35:07.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:35:07.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:07.125 | INFO | Uploaded 36GiB to GPU in 2.56862s (14.0153GiB/s) | 20260507 22:35:07.125 | INFO | Downloaded 34GiB to GPU in 2.48535s (13.6802GiB/s) | 20260507 22:35:12.266 | INFO | Analyzing chunk frames = [3126272,3126784) | 20260507 22:35:12.266 | INFO | MemInfo: { | 20260507 22:35:12.266 | INFO | "gpu" : " 17 GB", | 20260507 22:35:12.266 | INFO | "huge" : "", | 20260507 22:35:12.266 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:12.266 | INFO | "malloc" : " 14 GB", | 20260507 22:35:12.266 | INFO | "pinned" : " 29 GB" | 20260507 22:35:12.266 | INFO | } | 20260507 22:35:12.266 | INFO | Perf report for the last 5.14131 seconds: | 20260507 22:35:12.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:12.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:35:12.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:35:12.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:12.266 | INFO | Uploaded 36GiB to GPU in 2.5584s (14.0713GiB/s) | 20260507 22:35:12.266 | INFO | Downloaded 34GiB to GPU in 2.47658s (13.7286GiB/s) | 20260507 22:35:17.404 | INFO | Analyzing chunk frames = [3126784,3127296) | 20260507 22:35:17.404 | INFO | MemInfo: { | 20260507 22:35:17.404 | INFO | "gpu" : " 17 GB", | 20260507 22:35:17.404 | INFO | "huge" : "", | 20260507 22:35:17.405 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:17.405 | INFO | "malloc" : " 14 GB", | 20260507 22:35:17.405 | INFO | "pinned" : " 29 GB" | 20260507 22:35:17.405 | INFO | } | 20260507 22:35:17.405 | INFO | Perf report for the last 5.13871 seconds: | 20260507 22:35:17.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:17.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260507 22:35:17.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:35:17.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:17.405 | INFO | Uploaded 36GiB to GPU in 2.5675s (14.0214GiB/s) | 20260507 22:35:17.405 | INFO | Downloaded 34GiB to GPU in 2.46388s (13.7994GiB/s) | 20260507 22:35:22.542 | INFO | Analyzing chunk frames = [3127296,3127808) | 20260507 22:35:22.542 | INFO | MemInfo: { | 20260507 22:35:22.542 | INFO | "gpu" : " 17 GB", | 20260507 22:35:22.542 | INFO | "huge" : "", | 20260507 22:35:22.542 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:22.542 | INFO | "malloc" : " 14 GB", | 20260507 22:35:22.543 | INFO | "pinned" : " 29 GB" | 20260507 22:35:22.543 | INFO | } | 20260507 22:35:22.543 | INFO | Perf report for the last 5.13803 seconds: | 20260507 22:35:22.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:22.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:35:22.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:35:22.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:22.543 | INFO | Uploaded 36GiB to GPU in 2.35965s (15.2565GiB/s) | 20260507 22:35:22.543 | INFO | Downloaded 34GiB to GPU in 2.50242s (13.5868GiB/s) | 20260507 22:35:27.683 | INFO | Analyzing chunk frames = [3127808,3128320) | 20260507 22:35:27.683 | INFO | MemInfo: { | 20260507 22:35:27.683 | INFO | "gpu" : " 17 GB", | 20260507 22:35:27.683 | INFO | "huge" : "", | 20260507 22:35:27.683 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:27.683 | INFO | "malloc" : " 14 GB", | 20260507 22:35:27.683 | INFO | "pinned" : " 29 GB" | 20260507 22:35:27.683 | INFO | } | 20260507 22:35:27.683 | INFO | Perf report for the last 5.14018 seconds: | 20260507 22:35:27.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:27.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:35:27.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:35:27.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:27.683 | INFO | Uploaded 36GiB to GPU in 2.52824s (14.2392GiB/s) | 20260507 22:35:27.683 | INFO | Downloaded 34GiB to GPU in 2.41365s (14.0865GiB/s) | 20260507 22:35:32.820 | INFO | Analyzing chunk frames = [3128320,3128832) | 20260507 22:35:32.820 | INFO | MemInfo: { | 20260507 22:35:32.820 | INFO | "gpu" : " 17 GB", | 20260507 22:35:32.820 | INFO | "huge" : "", | 20260507 22:35:32.820 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:32.820 | INFO | "malloc" : " 14 GB", | 20260507 22:35:32.820 | INFO | "pinned" : " 29 GB" | 20260507 22:35:32.820 | INFO | } | 20260507 22:35:32.820 | INFO | Perf report for the last 5.13736 seconds: | 20260507 22:35:32.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:32.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.820 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 22:35:32.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.820 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:35:32.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:32.821 | INFO | Uploaded 36GiB to GPU in 2.55407s (14.0952GiB/s) | 20260507 22:35:32.821 | INFO | Downloaded 34GiB to GPU in 2.4454s (13.9036GiB/s) | 20260507 22:35:37.960 | INFO | Analyzing chunk frames = [3128832,3129344) | 20260507 22:35:37.960 | INFO | MemInfo: { | 20260507 22:35:37.960 | INFO | "gpu" : " 17 GB", | 20260507 22:35:37.960 | INFO | "huge" : "", | 20260507 22:35:37.960 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:37.960 | INFO | "malloc" : " 14 GB", | 20260507 22:35:37.960 | INFO | "pinned" : " 29 GB" | 20260507 22:35:37.960 | INFO | } | 20260507 22:35:37.960 | INFO | Perf report for the last 5.13953 seconds: | 20260507 22:35:37.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:37.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:35:37.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 22:35:37.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:37.960 | INFO | Uploaded 36GiB to GPU in 2.51857s (14.2938GiB/s) | 20260507 22:35:37.960 | INFO | Downloaded 34GiB to GPU in 2.42759s (14.0056GiB/s) | 20260507 22:35:37.960 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:35:37.960 | INFO | Total mem usage: 16.7 GiB | 20260507 22:35:37.960 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:35:37.960 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.52%) | 20260507 22:35:37.960 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:35:37.960 | INFO | High Water Breakdown: | 20260507 22:35:37.960 | INFO | 60.5% : 9.31 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:35:37.960 | INFO | 39.3% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:35:37.960 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:35:37.960 | INFO | ------------------------------------------ | 20260507 22:35:37.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:35:37.960 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:35:37.960 | INFO | Total mem usage: 26.5 GiB | 20260507 22:35:37.960 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:35:37.960 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 22:35:37.960 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:35:37.960 | INFO | High Water Breakdown: | 20260507 22:35:37.960 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:35:37.960 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:35:37.960 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:35:37.960 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:35:37.960 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:35:37.960 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:35:37.960 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:35:37.960 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:35:37.960 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:35:37.960 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:35:37.960 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:35:37.960 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:35:37.960 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:35:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:35:37.961 | INFO | ------------------------------------------ | 20260507 22:35:37.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:35:37.961 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:35:37.961 | INFO | Total mem usage: 27.8 GiB | 20260507 22:35:37.961 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:35:37.961 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:35:37.961 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:35:37.961 | INFO | High Water Breakdown: | 20260507 22:35:37.961 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:35:37.961 | INFO | ------------------------------------------ | 20260507 22:35:37.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:35:37.961 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:35:37.961 | INFO | Total mem usage: 15.7 GiB | 20260507 22:35:37.961 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:35:37.961 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:35:37.961 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:35:37.961 | INFO | High Water Breakdown: | 20260507 22:35:37.961 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:35:37.961 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:35:37.961 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:35:37.961 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:35:37.961 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:35:37.961 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:35:37.961 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:35:37.961 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:35:37.961 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:35:37.961 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:35:37.961 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:35:37.961 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:35:37.961 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:35:37.961 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:35:37.961 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:35:37.961 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:35:37.961 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:35:37.961 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:35:37.961 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:35:37.961 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:35:37.961 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:35:37.961 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:35:37.961 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:35:37.961 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:35:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:35:37.961 | INFO | ------------------------------------------ | 20260507 22:35:37.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:35:37.961 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:35:37.961 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:35:37.961 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:35:37.961 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.43e+01 / 1.61e+00 | 20260507 22:35:37.961 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.24e-02 / 4.93e-03 | 20260507 22:35:37.961 | INFO | PerfReport: 12.11%, Baseline -- 4.00e+00 / 4.50e+00, 5.94e-01 / 2.04e-01 | 20260507 22:35:37.961 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.51e+00, 2.12e-01 / 1.07e-01 | 20260507 22:35:37.961 | INFO | PerfReport: 15.76%, DME -- 4.16e+01 / 4.51e+01, 3.78e+00 / 1.22e+00 | 20260507 22:35:37.961 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.86e+00 / 2.08e+00 | 20260507 22:35:37.961 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.42e+00 / 2.69e+00, 4.34e-01 / 2.86e-01 | 20260507 22:35:37.961 | INFO | PerfReport: 3.44%, Metrics -- 1.14e+00 / 1.32e+00, 2.92e-01 / 1.90e-01 | 20260507 22:35:37.962 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.81e-01, 5.82e-02 / 3.33e-02 | 20260507 22:35:37.962 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:35:37.962 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:35:37.962 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:35:37.962 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.58e+00 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 30.99%, Compute -- 3.30e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 1.89%, LockUpload -- 2.01e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 15.73%, LockCompute -- 1.68e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:35:37.962 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 22:35:38.058 | INFO | Starting a round of Baz IO | 20260507 22:35:41.522 | INFO | Populating RTMetrics report | 20260507 22:35:41.523 | INFO | RTMetrics report populated | 20260507 22:35:41.523 | INFO | Opening tmp RTMetrics json file | 20260507 22:35:41.546 | INFO | Serializing RTMetrics to json | 20260507 22:35:41.547 | INFO | Serialization done, swapping tmp file now... | 20260507 22:35:41.548 | INFO | RTMetrics done | 20260507 22:35:41.617 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.559 seconds | 20260507 22:35:41.617 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.04% / 0.01% / 56.96% | 20260507 22:35:41.617 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37988% / 1.98682e-05% / 6.75519e-05% | 20260507 22:35:41.617 | INFO | Baz IO overall: | 20260507 22:35:41.617 | INFO | BytesWritten : 1425718483173 | 20260507 22:35:41.617 | INFO | NumEvents : 690702456917 | 20260507 22:35:43.103 | INFO | Analyzing chunk frames = [3129344,3129856) | 20260507 22:35:43.103 | INFO | MemInfo: { | 20260507 22:35:43.103 | INFO | "gpu" : " 17 GB", | 20260507 22:35:43.103 | INFO | "huge" : "", | 20260507 22:35:43.103 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:43.103 | INFO | "malloc" : " 14 GB", | 20260507 22:35:43.103 | INFO | "pinned" : " 29 GB" | 20260507 22:35:43.103 | INFO | } | 20260507 22:35:43.103 | INFO | Perf report for the last 5.14289 seconds: | 20260507 22:35:43.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:43.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:43.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:43.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:43.103 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260507 22:35:43.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:43.103 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 22:35:43.103 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260507 22:35:43.103 | INFO | BAZWRITER : 68.8%, 4.6, 161.2ms, 0 | 20260507 22:35:43.103 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 22:35:43.103 | INFO | Uploaded 36GiB to GPU in 2.56096s (14.0573GiB/s) | 20260507 22:35:43.104 | INFO | Downloaded 38GiB to GPU in 2.69717s (14.0888GiB/s) | 20260507 22:35:48.242 | INFO | Analyzing chunk frames = [3129856,3130368) | 20260507 22:35:48.242 | INFO | MemInfo: { | 20260507 22:35:48.242 | INFO | "gpu" : " 17 GB", | 20260507 22:35:48.242 | INFO | "huge" : "", | 20260507 22:35:48.242 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:48.242 | INFO | "malloc" : " 14 GB", | 20260507 22:35:48.242 | INFO | "pinned" : " 29 GB" | 20260507 22:35:48.242 | INFO | } | 20260507 22:35:48.242 | INFO | Perf report for the last 5.13925 seconds: | 20260507 22:35:48.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:48.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.242 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:35:48.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.242 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:35:48.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:48.243 | INFO | Uploaded 36GiB to GPU in 2.59019s (13.8986GiB/s) | 20260507 22:35:48.243 | INFO | Downloaded 34GiB to GPU in 2.47418s (13.7419GiB/s) | 20260507 22:35:53.377 | INFO | Analyzing chunk frames = [3130368,3130880) | 20260507 22:35:53.377 | INFO | MemInfo: { | 20260507 22:35:53.377 | INFO | "gpu" : " 17 GB", | 20260507 22:35:53.377 | INFO | "huge" : "", | 20260507 22:35:53.377 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:53.377 | INFO | "malloc" : " 14 GB", | 20260507 22:35:53.377 | INFO | "pinned" : " 29 GB" | 20260507 22:35:53.377 | INFO | } | 20260507 22:35:53.377 | INFO | Perf report for the last 5.13477 seconds: | 20260507 22:35:53.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:53.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 22:35:53.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:35:53.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:53.377 | INFO | Uploaded 36GiB to GPU in 2.56308s (14.0456GiB/s) | 20260507 22:35:53.377 | INFO | Downloaded 34GiB to GPU in 2.44993s (13.878GiB/s) | 20260507 22:35:58.517 | INFO | Analyzing chunk frames = [3130880,3131392) | 20260507 22:35:58.517 | INFO | MemInfo: { | 20260507 22:35:58.517 | INFO | "gpu" : " 17 GB", | 20260507 22:35:58.517 | INFO | "huge" : "", | 20260507 22:35:58.518 | INFO | "huge_shared" : " 29 GB", | 20260507 22:35:58.518 | INFO | "malloc" : " 14 GB", | 20260507 22:35:58.518 | INFO | "pinned" : " 29 GB" | 20260507 22:35:58.518 | INFO | } | 20260507 22:35:58.518 | INFO | Perf report for the last 5.14098 seconds: | 20260507 22:35:58.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:35:58.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:35:58.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:35:58.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:35:58.518 | INFO | Uploaded 36GiB to GPU in 2.47806s (14.5275GiB/s) | 20260507 22:35:58.518 | INFO | Downloaded 34GiB to GPU in 2.38001s (14.2856GiB/s) | 20260507 22:36:03.656 | INFO | Analyzing chunk frames = [3131392,3131904) | 20260507 22:36:03.656 | INFO | MemInfo: { | 20260507 22:36:03.656 | INFO | "gpu" : " 17 GB", | 20260507 22:36:03.656 | INFO | "huge" : "", | 20260507 22:36:03.656 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:03.657 | INFO | "malloc" : " 14 GB", | 20260507 22:36:03.657 | INFO | "pinned" : " 29 GB" | 20260507 22:36:03.657 | INFO | } | 20260507 22:36:03.657 | INFO | Perf report for the last 5.13895 seconds: | 20260507 22:36:03.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:03.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:36:03.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:36:03.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:03.657 | INFO | Uploaded 36GiB to GPU in 2.45399s (14.67GiB/s) | 20260507 22:36:03.657 | INFO | Downloaded 34GiB to GPU in 2.49218s (13.6426GiB/s) | 20260507 22:36:08.796 | INFO | Analyzing chunk frames = [3131904,3132416) | 20260507 22:36:08.796 | INFO | MemInfo: { | 20260507 22:36:08.796 | INFO | "gpu" : " 17 GB", | 20260507 22:36:08.796 | INFO | "huge" : "", | 20260507 22:36:08.796 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:08.796 | INFO | "malloc" : " 14 GB", | 20260507 22:36:08.796 | INFO | "pinned" : " 29 GB" | 20260507 22:36:08.796 | INFO | } | 20260507 22:36:08.796 | INFO | Perf report for the last 5.13932 seconds: | 20260507 22:36:08.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:08.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.796 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 22:36:08.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.796 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:36:08.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:08.797 | INFO | Uploaded 36GiB to GPU in 2.49206s (14.4459GiB/s) | 20260507 22:36:08.797 | INFO | Downloaded 34GiB to GPU in 2.41297s (14.0905GiB/s) | 20260507 22:36:13.940 | INFO | Analyzing chunk frames = [3132416,3132928) | 20260507 22:36:13.940 | INFO | MemInfo: { | 20260507 22:36:13.940 | INFO | "gpu" : " 17 GB", | 20260507 22:36:13.940 | INFO | "huge" : "", | 20260507 22:36:13.941 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:13.941 | INFO | "malloc" : " 14 GB", | 20260507 22:36:13.941 | INFO | "pinned" : " 29 GB" | 20260507 22:36:13.941 | INFO | } | 20260507 22:36:13.941 | INFO | Perf report for the last 5.14462 seconds: | 20260507 22:36:13.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:13.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260507 22:36:13.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 22:36:13.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:13.941 | INFO | Uploaded 36GiB to GPU in 2.54892s (14.1236GiB/s) | 20260507 22:36:13.941 | INFO | Downloaded 34GiB to GPU in 2.4391s (13.9396GiB/s) | 20260507 22:36:19.075 | INFO | Analyzing chunk frames = [3132928,3133440) | 20260507 22:36:19.075 | INFO | MemInfo: { | 20260507 22:36:19.075 | INFO | "gpu" : " 17 GB", | 20260507 22:36:19.075 | INFO | "huge" : "", | 20260507 22:36:19.075 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:19.075 | INFO | "malloc" : " 14 GB", | 20260507 22:36:19.075 | INFO | "pinned" : " 29 GB" | 20260507 22:36:19.075 | INFO | } | 20260507 22:36:19.075 | INFO | Perf report for the last 5.13463 seconds: | 20260507 22:36:19.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:19.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.075 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:36:19.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.075 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:36:19.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:19.076 | INFO | Uploaded 36GiB to GPU in 2.52431s (14.2613GiB/s) | 20260507 22:36:19.076 | INFO | Downloaded 34GiB to GPU in 2.4382s (13.9447GiB/s) | 20260507 22:36:19.173 | INFO | Starting a round of Baz IO | 20260507 22:36:22.649 | INFO | Populating RTMetrics report | 20260507 22:36:22.649 | INFO | RTMetrics report populated | 20260507 22:36:22.649 | INFO | Opening tmp RTMetrics json file | 20260507 22:36:22.668 | INFO | Serializing RTMetrics to json | 20260507 22:36:22.672 | INFO | Serialization done, swapping tmp file now... | 20260507 22:36:22.677 | INFO | RTMetrics done | 20260507 22:36:22.751 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.577 seconds | 20260507 22:36:22.751 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.03% / 0.01% / 56.96% | 20260507 22:36:22.751 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37869% / 3.97364e-06% / 7.54992e-05% | 20260507 22:36:22.751 | INFO | Baz IO overall: | 20260507 22:36:22.751 | INFO | BytesWritten : 1427580802827 | 20260507 22:36:22.751 | INFO | NumEvents : 691601421703 | 20260507 22:36:24.215 | INFO | Analyzing chunk frames = [3133440,3133952) | 20260507 22:36:24.215 | INFO | MemInfo: { | 20260507 22:36:24.215 | INFO | "gpu" : " 17 GB", | 20260507 22:36:24.215 | INFO | "huge" : "", | 20260507 22:36:24.215 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:24.215 | INFO | "malloc" : " 14 GB", | 20260507 22:36:24.216 | INFO | "pinned" : " 29 GB" | 20260507 22:36:24.216 | INFO | } | 20260507 22:36:24.216 | INFO | Perf report for the last 5.14019 seconds: | 20260507 22:36:24.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:24.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:24.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:24.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:24.216 | INFO | ANALYSIS : 68.6%, 3.0, 106.9ms, 0 | 20260507 22:36:24.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:24.216 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 22:36:24.216 | INFO | PRE_HQ : 65.7%, 1.7, 57.4ms, 0 | 20260507 22:36:24.216 | INFO | BAZWRITER : 69.2%, 3.9, 138.3ms, 0 | 20260507 22:36:24.216 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 22:36:24.216 | INFO | Uploaded 36GiB to GPU in 2.56279s (14.0472GiB/s) | 20260507 22:36:24.216 | INFO | Downloaded 38GiB to GPU in 2.68529s (14.1512GiB/s) | 20260507 22:36:29.355 | INFO | Analyzing chunk frames = [3133952,3134464) | 20260507 22:36:29.355 | INFO | MemInfo: { | 20260507 22:36:29.355 | INFO | "gpu" : " 17 GB", | 20260507 22:36:29.355 | INFO | "huge" : "", | 20260507 22:36:29.355 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:29.355 | INFO | "malloc" : " 14 GB", | 20260507 22:36:29.355 | INFO | "pinned" : " 29 GB" | 20260507 22:36:29.356 | INFO | } | 20260507 22:36:29.356 | INFO | Perf report for the last 5.14 seconds: | 20260507 22:36:29.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:29.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 22:36:29.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:36:29.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:29.356 | INFO | Uploaded 36GiB to GPU in 2.58532s (13.9248GiB/s) | 20260507 22:36:29.356 | INFO | Downloaded 34GiB to GPU in 2.47377s (13.7442GiB/s) | 20260507 22:36:34.490 | INFO | Analyzing chunk frames = [3134464,3134976) | 20260507 22:36:34.490 | INFO | MemInfo: { | 20260507 22:36:34.490 | INFO | "gpu" : " 17 GB", | 20260507 22:36:34.490 | INFO | "huge" : "", | 20260507 22:36:34.490 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:34.490 | INFO | "malloc" : " 14 GB", | 20260507 22:36:34.490 | INFO | "pinned" : " 29 GB" | 20260507 22:36:34.490 | INFO | } | 20260507 22:36:34.490 | INFO | Perf report for the last 5.13451 seconds: | 20260507 22:36:34.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:34.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.490 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260507 22:36:34.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.490 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:36:34.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:34.491 | INFO | Uploaded 36GiB to GPU in 2.53561s (14.1978GiB/s) | 20260507 22:36:34.491 | INFO | Downloaded 34GiB to GPU in 2.47636s (13.7299GiB/s) | 20260507 22:36:39.630 | INFO | Analyzing chunk frames = [3134976,3135488) | 20260507 22:36:39.630 | INFO | MemInfo: { | 20260507 22:36:39.630 | INFO | "gpu" : " 17 GB", | 20260507 22:36:39.630 | INFO | "huge" : "", | 20260507 22:36:39.630 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:39.630 | INFO | "malloc" : " 14 GB", | 20260507 22:36:39.630 | INFO | "pinned" : " 29 GB" | 20260507 22:36:39.630 | INFO | } | 20260507 22:36:39.630 | INFO | Perf report for the last 5.14023 seconds: | 20260507 22:36:39.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:39.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.630 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260507 22:36:39.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.630 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:36:39.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:39.631 | INFO | Uploaded 36GiB to GPU in 2.51142s (14.3345GiB/s) | 20260507 22:36:39.631 | INFO | Downloaded 34GiB to GPU in 2.3642s (14.3812GiB/s) | 20260507 22:36:44.771 | INFO | Analyzing chunk frames = [3135488,3136000) | 20260507 22:36:44.771 | INFO | MemInfo: { | 20260507 22:36:44.771 | INFO | "gpu" : " 17 GB", | 20260507 22:36:44.771 | INFO | "huge" : "", | 20260507 22:36:44.771 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:44.771 | INFO | "malloc" : " 14 GB", | 20260507 22:36:44.771 | INFO | "pinned" : " 29 GB" | 20260507 22:36:44.771 | INFO | } | 20260507 22:36:44.771 | INFO | Perf report for the last 5.14067 seconds: | 20260507 22:36:44.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:44.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:36:44.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 22:36:44.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:44.771 | INFO | Uploaded 36GiB to GPU in 2.37243s (15.1743GiB/s) | 20260507 22:36:44.772 | INFO | Downloaded 34GiB to GPU in 2.50393s (13.5787GiB/s) | 20260507 22:36:49.911 | INFO | Analyzing chunk frames = [3136000,3136512) | 20260507 22:36:49.911 | INFO | MemInfo: { | 20260507 22:36:49.911 | INFO | "gpu" : " 17 GB", | 20260507 22:36:49.911 | INFO | "huge" : "", | 20260507 22:36:49.911 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:49.911 | INFO | "malloc" : " 14 GB", | 20260507 22:36:49.911 | INFO | "pinned" : " 29 GB" | 20260507 22:36:49.911 | INFO | } | 20260507 22:36:49.911 | INFO | Perf report for the last 5.14024 seconds: | 20260507 22:36:49.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:49.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.911 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 22:36:49.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.911 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:36:49.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:49.912 | INFO | Uploaded 36GiB to GPU in 2.43633s (14.7763GiB/s) | 20260507 22:36:49.912 | INFO | Downloaded 34GiB to GPU in 2.39322s (14.2068GiB/s) | 20260507 22:36:55.053 | INFO | Analyzing chunk frames = [3136512,3137024) | 20260507 22:36:55.053 | INFO | MemInfo: { | 20260507 22:36:55.053 | INFO | "gpu" : " 17 GB", | 20260507 22:36:55.053 | INFO | "huge" : "", | 20260507 22:36:55.053 | INFO | "huge_shared" : " 29 GB", | 20260507 22:36:55.053 | INFO | "malloc" : " 14 GB", | 20260507 22:36:55.053 | INFO | "pinned" : " 29 GB" | 20260507 22:36:55.053 | INFO | } | 20260507 22:36:55.053 | INFO | Perf report for the last 5.142 seconds: | 20260507 22:36:55.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:36:55.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.054 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 22:36:55.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.054 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:36:55.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:36:55.054 | INFO | Uploaded 36GiB to GPU in 2.53551s (14.1983GiB/s) | 20260507 22:36:55.054 | INFO | Downloaded 34GiB to GPU in 2.43678s (13.9528GiB/s) | 20260507 22:37:00.187 | INFO | Analyzing chunk frames = [3137024,3137536) | 20260507 22:37:00.187 | INFO | MemInfo: { | 20260507 22:37:00.187 | INFO | "gpu" : " 17 GB", | 20260507 22:37:00.187 | INFO | "huge" : "", | 20260507 22:37:00.187 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:00.187 | INFO | "malloc" : " 14 GB", | 20260507 22:37:00.188 | INFO | "pinned" : " 29 GB" | 20260507 22:37:00.188 | INFO | } | 20260507 22:37:00.188 | INFO | Perf report for the last 5.13429 seconds: | 20260507 22:37:00.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:00.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:37:00.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:37:00.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:00.188 | INFO | Uploaded 36GiB to GPU in 2.53042s (14.2269GiB/s) | 20260507 22:37:00.188 | INFO | Downloaded 34GiB to GPU in 2.44023s (13.9331GiB/s) | 20260507 22:37:00.188 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:37:00.188 | INFO | Total mem usage: 16.7 GiB | 20260507 22:37:00.188 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:37:00.188 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.78%) | 20260507 22:37:00.188 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:37:00.188 | INFO | High Water Breakdown: | 20260507 22:37:00.188 | INFO | 60.6% : 9.39 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:37:00.188 | INFO | 39.2% : 6.07 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:37:00.188 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:37:00.188 | INFO | ------------------------------------------ | 20260507 22:37:00.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:37:00.188 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:37:00.188 | INFO | Total mem usage: 26.5 GiB | 20260507 22:37:00.188 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:37:00.188 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:37:00.188 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:37:00.188 | INFO | High Water Breakdown: | 20260507 22:37:00.188 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:37:00.188 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:37:00.188 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:37:00.188 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:37:00.189 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:37:00.189 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:37:00.189 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:37:00.189 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:37:00.189 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:37:00.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:37:00.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:37:00.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:37:00.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:37:00.189 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:37:00.189 | INFO | ------------------------------------------ | 20260507 22:37:00.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:37:00.189 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:37:00.189 | INFO | Total mem usage: 27.8 GiB | 20260507 22:37:00.189 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:37:00.189 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:37:00.189 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:37:00.189 | INFO | High Water Breakdown: | 20260507 22:37:00.189 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:37:00.189 | INFO | ------------------------------------------ | 20260507 22:37:00.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:37:00.189 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:37:00.189 | INFO | Total mem usage: 15.7 GiB | 20260507 22:37:00.189 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:37:00.189 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:37:00.189 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:37:00.189 | INFO | High Water Breakdown: | 20260507 22:37:00.189 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:37:00.189 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:37:00.189 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:37:00.189 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:37:00.189 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:37:00.189 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:37:00.189 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:37:00.189 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:37:00.189 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:37:00.189 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:37:00.189 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:37:00.189 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:37:00.189 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:37:00.189 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:37:00.189 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:37:00.189 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:37:00.189 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:37:00.189 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:37:00.189 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:37:00.189 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:37:00.189 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:37:00.189 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:37:00.189 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:37:00.189 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:37:00.189 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:37:00.189 | INFO | ------------------------------------------ | 20260507 22:37:00.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:37:00.189 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:37:00.189 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:37:00.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:37:00.189 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.38e+01 / 1.42e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 0.29%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.17e-02 / 5.83e-03 | 20260507 22:37:00.189 | INFO | PerfReport: 12.14%, Baseline -- 3.99e+00 / 4.49e+00, 5.94e-01 / 2.16e-01 | 20260507 22:37:00.189 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.07e-01 / 1.04e-01 | 20260507 22:37:00.189 | INFO | PerfReport: 15.89%, DME -- 4.18e+01 / 4.54e+01, 3.88e+00 / 1.47e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.76e+00 / 1.63e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.76e+00, 4.27e-01 / 1.97e-01 | 20260507 22:37:00.189 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.38e+00, 2.87e-01 / 1.58e-01 | 20260507 22:37:00.189 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.52e-01 / 3.88e-01, 5.69e-02 / 2.49e-02 | 20260507 22:37:00.189 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:37:00.189 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:37:00.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:37:00.189 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 24.53%, Upload -- 2.62e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 24.09%, Download -- 2.58e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 30.77%, Compute -- 3.29e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 2.37%, LockUpload -- 2.53e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260507 22:37:00.189 | INFO | PerfReport: 2.30%, LockDownload -- 2.46e+00 / 0.00e+00, 3.74e+00 / 0.00e+00 | 20260507 22:37:00.190 | INFO | PerfReport: 15.94%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:37:00.190 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 1.04e-02 / 0.00e+00 | 20260507 22:37:00.284 | INFO | Starting a round of Baz IO | 20260507 22:37:03.760 | INFO | Populating RTMetrics report | 20260507 22:37:03.760 | INFO | RTMetrics report populated | 20260507 22:37:03.760 | INFO | Opening tmp RTMetrics json file | 20260507 22:37:03.919 | INFO | Serializing RTMetrics to json | 20260507 22:37:03.921 | INFO | Serialization done, swapping tmp file now... | 20260507 22:37:03.921 | INFO | RTMetrics done | 20260507 22:37:03.933 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.649 seconds | 20260507 22:37:03.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.02% / 0.01% / 56.97% | 20260507 22:37:03.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37536% / 7.94729e-06% / 4.76837e-05% | 20260507 22:37:03.933 | INFO | Baz IO overall: | 20260507 22:37:03.934 | INFO | BytesWritten : 1429442179851 | 20260507 22:37:03.934 | INFO | NumEvents : 692499675514 | 20260507 22:37:05.324 | INFO | Analyzing chunk frames = [3137536,3138048) | 20260507 22:37:05.324 | INFO | MemInfo: { | 20260507 22:37:05.324 | INFO | "gpu" : " 17 GB", | 20260507 22:37:05.325 | INFO | "huge" : "", | 20260507 22:37:05.325 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:05.325 | INFO | "malloc" : " 14 GB", | 20260507 22:37:05.325 | INFO | "pinned" : " 29 GB" | 20260507 22:37:05.325 | INFO | } | 20260507 22:37:05.325 | INFO | Perf report for the last 5.1371 seconds: | 20260507 22:37:05.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:05.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:05.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:05.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:05.325 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 22:37:05.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:05.325 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 22:37:05.325 | INFO | PRE_HQ : 65.7%, 1.7, 57.1ms, 0 | 20260507 22:37:05.325 | INFO | BAZWRITER : 70.4%, 3.8, 136.3ms, 0 | 20260507 22:37:05.325 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260507 22:37:05.325 | INFO | Uploaded 36GiB to GPU in 2.58528s (13.925GiB/s) | 20260507 22:37:05.325 | INFO | Downloaded 38GiB to GPU in 2.6943s (14.1038GiB/s) | 20260507 22:37:10.466 | INFO | Analyzing chunk frames = [3138048,3138560) | 20260507 22:37:10.466 | INFO | MemInfo: { | 20260507 22:37:10.466 | INFO | "gpu" : " 17 GB", | 20260507 22:37:10.466 | INFO | "huge" : "", | 20260507 22:37:10.466 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:10.467 | INFO | "malloc" : " 14 GB", | 20260507 22:37:10.467 | INFO | "pinned" : " 29 GB" | 20260507 22:37:10.467 | INFO | } | 20260507 22:37:10.467 | INFO | Perf report for the last 5.14191 seconds: | 20260507 22:37:10.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:10.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:37:10.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:37:10.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:10.467 | INFO | Uploaded 36GiB to GPU in 2.56756s (14.0211GiB/s) | 20260507 22:37:10.467 | INFO | Downloaded 34GiB to GPU in 2.46802s (13.7762GiB/s) | 20260507 22:37:15.608 | INFO | Analyzing chunk frames = [3138560,3139072) | 20260507 22:37:15.608 | INFO | MemInfo: { | 20260507 22:37:15.608 | INFO | "gpu" : " 17 GB", | 20260507 22:37:15.608 | INFO | "huge" : "", | 20260507 22:37:15.608 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:15.608 | INFO | "malloc" : " 14 GB", | 20260507 22:37:15.608 | INFO | "pinned" : " 29 GB" | 20260507 22:37:15.608 | INFO | } | 20260507 22:37:15.608 | INFO | Perf report for the last 5.14151 seconds: | 20260507 22:37:15.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:15.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.608 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 22:37:15.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.608 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:37:15.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:15.609 | INFO | Uploaded 36GiB to GPU in 2.55556s (14.0869GiB/s) | 20260507 22:37:15.609 | INFO | Downloaded 34GiB to GPU in 2.44275s (13.9188GiB/s) | 20260507 22:37:20.747 | INFO | Analyzing chunk frames = [3139072,3139584) | 20260507 22:37:20.747 | INFO | MemInfo: { | 20260507 22:37:20.747 | INFO | "gpu" : " 17 GB", | 20260507 22:37:20.747 | INFO | "huge" : "", | 20260507 22:37:20.747 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:20.747 | INFO | "malloc" : " 14 GB", | 20260507 22:37:20.747 | INFO | "pinned" : " 29 GB" | 20260507 22:37:20.747 | INFO | } | 20260507 22:37:20.747 | INFO | Perf report for the last 5.13926 seconds: | 20260507 22:37:20.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:20.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 22:37:20.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:37:20.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:20.748 | INFO | Uploaded 36GiB to GPU in 2.4996s (14.4023GiB/s) | 20260507 22:37:20.748 | INFO | Downloaded 34GiB to GPU in 2.39481s (14.1973GiB/s) | 20260507 22:37:25.885 | INFO | Analyzing chunk frames = [3139584,3140096) | 20260507 22:37:25.885 | INFO | MemInfo: { | 20260507 22:37:25.885 | INFO | "gpu" : " 17 GB", | 20260507 22:37:25.885 | INFO | "huge" : "", | 20260507 22:37:25.885 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:25.885 | INFO | "malloc" : " 14 GB", | 20260507 22:37:25.885 | INFO | "pinned" : " 29 GB" | 20260507 22:37:25.885 | INFO | } | 20260507 22:37:25.885 | INFO | Perf report for the last 5.13776 seconds: | 20260507 22:37:25.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:25.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.885 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260507 22:37:25.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.885 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:37:25.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:25.886 | INFO | Uploaded 36GiB to GPU in 2.47145s (14.5664GiB/s) | 20260507 22:37:25.886 | INFO | Downloaded 34GiB to GPU in 2.48914s (13.6593GiB/s) | 20260507 22:37:31.024 | INFO | Analyzing chunk frames = [3140096,3140608) | 20260507 22:37:31.024 | INFO | MemInfo: { | 20260507 22:37:31.024 | INFO | "gpu" : " 17 GB", | 20260507 22:37:31.024 | INFO | "huge" : "", | 20260507 22:37:31.024 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:31.024 | INFO | "malloc" : " 14 GB", | 20260507 22:37:31.024 | INFO | "pinned" : " 29 GB" | 20260507 22:37:31.024 | INFO | } | 20260507 22:37:31.024 | INFO | Perf report for the last 5.13885 seconds: | 20260507 22:37:31.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:31.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:37:31.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:37:31.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:31.024 | INFO | Uploaded 36GiB to GPU in 2.48938s (14.4614GiB/s) | 20260507 22:37:31.024 | INFO | Downloaded 34GiB to GPU in 2.42803s (14.0031GiB/s) | 20260507 22:37:36.169 | INFO | Analyzing chunk frames = [3140608,3141120) | 20260507 22:37:36.169 | INFO | MemInfo: { | 20260507 22:37:36.169 | INFO | "gpu" : " 17 GB", | 20260507 22:37:36.169 | INFO | "huge" : "", | 20260507 22:37:36.169 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:36.169 | INFO | "malloc" : " 14 GB", | 20260507 22:37:36.169 | INFO | "pinned" : " 29 GB" | 20260507 22:37:36.169 | INFO | } | 20260507 22:37:36.169 | INFO | Perf report for the last 5.14474 seconds: | 20260507 22:37:36.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:36.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | ANALYSIS : 67.1%, 3.0, 105.0ms, 0 | 20260507 22:37:36.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:37:36.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:36.169 | INFO | Uploaded 36GiB to GPU in 2.51517s (14.3131GiB/s) | 20260507 22:37:36.169 | INFO | Downloaded 34GiB to GPU in 2.44769s (13.8907GiB/s) | 20260507 22:37:41.308 | INFO | Analyzing chunk frames = [3141120,3141632) | 20260507 22:37:41.308 | INFO | MemInfo: { | 20260507 22:37:41.308 | INFO | "gpu" : " 17 GB", | 20260507 22:37:41.308 | INFO | "huge" : "", | 20260507 22:37:41.308 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:41.308 | INFO | "malloc" : " 14 GB", | 20260507 22:37:41.308 | INFO | "pinned" : " 29 GB" | 20260507 22:37:41.308 | INFO | } | 20260507 22:37:41.308 | INFO | Perf report for the last 5.1393 seconds: | 20260507 22:37:41.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:41.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:37:41.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:37:41.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:41.308 | INFO | Uploaded 36GiB to GPU in 2.49951s (14.4028GiB/s) | 20260507 22:37:41.308 | INFO | Downloaded 34GiB to GPU in 2.43993s (13.9348GiB/s) | 20260507 22:37:41.406 | INFO | Starting a round of Baz IO | 20260507 22:37:44.881 | INFO | Populating RTMetrics report | 20260507 22:37:44.881 | INFO | RTMetrics report populated | 20260507 22:37:44.881 | INFO | Opening tmp RTMetrics json file | 20260507 22:37:44.893 | INFO | Serializing RTMetrics to json | 20260507 22:37:44.916 | INFO | Serialization done, swapping tmp file now... | 20260507 22:37:44.952 | INFO | RTMetrics done | 20260507 22:37:44.977 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.571 seconds | 20260507 22:37:44.977 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.02% / 0.01% / 56.98% | 20260507 22:37:44.977 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36938% / 3.97364e-06% / 3.97364e-05% | 20260507 22:37:44.977 | INFO | Baz IO overall: | 20260507 22:37:44.977 | INFO | BytesWritten : 1431301051708 | 20260507 22:37:44.977 | INFO | NumEvents : 693396208399 | 20260507 22:37:46.442 | INFO | Analyzing chunk frames = [3141632,3142144) | 20260507 22:37:46.442 | INFO | MemInfo: { | 20260507 22:37:46.442 | INFO | "gpu" : " 17 GB", | 20260507 22:37:46.442 | INFO | "huge" : "", | 20260507 22:37:46.442 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:46.442 | INFO | "malloc" : " 14 GB", | 20260507 22:37:46.442 | INFO | "pinned" : " 29 GB" | 20260507 22:37:46.442 | INFO | } | 20260507 22:37:46.442 | INFO | Perf report for the last 5.13373 seconds: | 20260507 22:37:46.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:46.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:46.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:46.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:46.442 | INFO | ANALYSIS : 69.0%, 3.0, 106.6ms, 0 | 20260507 22:37:46.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:46.442 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 22:37:46.442 | INFO | PRE_HQ : 66.2%, 1.7, 57.3ms, 0 | 20260507 22:37:46.442 | INFO | BAZWRITER : 69.1%, 4.4, 155.8ms, 0 | 20260507 22:37:46.442 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 22:37:46.442 | INFO | Uploaded 36GiB to GPU in 2.50616s (14.3646GiB/s) | 20260507 22:37:46.442 | INFO | Downloaded 38GiB to GPU in 2.70837s (14.0306GiB/s) | 20260507 22:37:51.585 | INFO | Analyzing chunk frames = [3142144,3142656) | 20260507 22:37:51.585 | INFO | MemInfo: { | 20260507 22:37:51.585 | INFO | "gpu" : " 17 GB", | 20260507 22:37:51.585 | INFO | "huge" : "", | 20260507 22:37:51.585 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:51.585 | INFO | "malloc" : " 14 GB", | 20260507 22:37:51.585 | INFO | "pinned" : " 29 GB" | 20260507 22:37:51.585 | INFO | } | 20260507 22:37:51.585 | INFO | Perf report for the last 5.14328 seconds: | 20260507 22:37:51.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:51.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.585 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:37:51.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.585 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 22:37:51.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:51.586 | INFO | Uploaded 36GiB to GPU in 2.56495s (14.0354GiB/s) | 20260507 22:37:51.586 | INFO | Downloaded 34GiB to GPU in 2.47906s (13.7149GiB/s) | 20260507 22:37:56.725 | INFO | Analyzing chunk frames = [3142656,3143168) | 20260507 22:37:56.725 | INFO | MemInfo: { | 20260507 22:37:56.725 | INFO | "gpu" : " 17 GB", | 20260507 22:37:56.725 | INFO | "huge" : "", | 20260507 22:37:56.726 | INFO | "huge_shared" : " 29 GB", | 20260507 22:37:56.726 | INFO | "malloc" : " 14 GB", | 20260507 22:37:56.726 | INFO | "pinned" : " 29 GB" | 20260507 22:37:56.726 | INFO | } | 20260507 22:37:56.726 | INFO | Perf report for the last 5.14054 seconds: | 20260507 22:37:56.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:37:56.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.726 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:37:56.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.726 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 22:37:56.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:37:56.727 | INFO | Uploaded 36GiB to GPU in 2.50235s (14.3865GiB/s) | 20260507 22:37:56.727 | INFO | Downloaded 34GiB to GPU in 2.48615s (13.6758GiB/s) | 20260507 22:38:01.861 | INFO | Analyzing chunk frames = [3143168,3143680) | 20260507 22:38:01.861 | INFO | MemInfo: { | 20260507 22:38:01.861 | INFO | "gpu" : " 17 GB", | 20260507 22:38:01.861 | INFO | "huge" : "", | 20260507 22:38:01.862 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:01.862 | INFO | "malloc" : " 14 GB", | 20260507 22:38:01.862 | INFO | "pinned" : " 29 GB" | 20260507 22:38:01.862 | INFO | } | 20260507 22:38:01.862 | INFO | Perf report for the last 5.13592 seconds: | 20260507 22:38:01.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:01.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 22:38:01.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:38:01.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:01.862 | INFO | Uploaded 36GiB to GPU in 2.50843s (14.3516GiB/s) | 20260507 22:38:01.862 | INFO | Downloaded 34GiB to GPU in 2.48902s (13.66GiB/s) | 20260507 22:38:07.004 | INFO | Analyzing chunk frames = [3143680,3144192) | 20260507 22:38:07.004 | INFO | MemInfo: { | 20260507 22:38:07.004 | INFO | "gpu" : " 17 GB", | 20260507 22:38:07.004 | INFO | "huge" : "", | 20260507 22:38:07.004 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:07.004 | INFO | "malloc" : " 14 GB", | 20260507 22:38:07.004 | INFO | "pinned" : " 29 GB" | 20260507 22:38:07.004 | INFO | } | 20260507 22:38:07.004 | INFO | Perf report for the last 5.14226 seconds: | 20260507 22:38:07.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:07.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.004 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 22:38:07.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.004 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:38:07.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:07.005 | INFO | Uploaded 36GiB to GPU in 2.42677s (14.8345GiB/s) | 20260507 22:38:07.005 | INFO | Downloaded 34GiB to GPU in 2.49561s (13.6239GiB/s) | 20260507 22:38:12.141 | INFO | Analyzing chunk frames = [3144192,3144704) | 20260507 22:38:12.141 | INFO | MemInfo: { | 20260507 22:38:12.141 | INFO | "gpu" : " 17 GB", | 20260507 22:38:12.141 | INFO | "huge" : "", | 20260507 22:38:12.141 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:12.141 | INFO | "malloc" : " 14 GB", | 20260507 22:38:12.141 | INFO | "pinned" : " 29 GB" | 20260507 22:38:12.141 | INFO | } | 20260507 22:38:12.141 | INFO | Perf report for the last 5.13693 seconds: | 20260507 22:38:12.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:12.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.141 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:38:12.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.141 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:38:12.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:12.142 | INFO | Uploaded 36GiB to GPU in 2.50208s (14.388GiB/s) | 20260507 22:38:12.142 | INFO | Downloaded 34GiB to GPU in 2.42919s (13.9964GiB/s) | 20260507 22:38:17.278 | INFO | Analyzing chunk frames = [3144704,3145216) | 20260507 22:38:17.279 | INFO | MemInfo: { | 20260507 22:38:17.279 | INFO | "gpu" : " 17 GB", | 20260507 22:38:17.279 | INFO | "huge" : "", | 20260507 22:38:17.279 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:17.279 | INFO | "malloc" : " 14 GB", | 20260507 22:38:17.279 | INFO | "pinned" : " 29 GB" | 20260507 22:38:17.279 | INFO | } | 20260507 22:38:17.279 | INFO | Perf report for the last 5.1379 seconds: | 20260507 22:38:17.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:17.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:38:17.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:38:17.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:17.279 | INFO | Uploaded 36GiB to GPU in 2.53572s (14.1972GiB/s) | 20260507 22:38:17.279 | INFO | Downloaded 34GiB to GPU in 2.45752s (13.8351GiB/s) | 20260507 22:38:22.417 | INFO | Analyzing chunk frames = [3145216,3145728) | 20260507 22:38:22.417 | INFO | MemInfo: { | 20260507 22:38:22.417 | INFO | "gpu" : " 17 GB", | 20260507 22:38:22.417 | INFO | "huge" : "", | 20260507 22:38:22.417 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:22.417 | INFO | "malloc" : " 14 GB", | 20260507 22:38:22.417 | INFO | "pinned" : " 29 GB" | 20260507 22:38:22.417 | INFO | } | 20260507 22:38:22.418 | INFO | Perf report for the last 5.1389 seconds: | 20260507 22:38:22.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:22.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:38:22.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:38:22.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:22.418 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:38:22.418 | INFO | Total mem usage: 16.7 GiB | 20260507 22:38:22.418 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:38:22.418 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.63%) | 20260507 22:38:22.418 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:38:22.418 | INFO | High Water Breakdown: | 20260507 22:38:22.418 | INFO | 60.5% : 9.29 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:38:22.418 | INFO | 39.4% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:38:22.418 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:38:22.418 | INFO | ------------------------------------------ | 20260507 22:38:22.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:38:22.418 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:38:22.418 | INFO | Total mem usage: 26.5 GiB | 20260507 22:38:22.418 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:38:22.418 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:38:22.418 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:38:22.418 | INFO | High Water Breakdown: | 20260507 22:38:22.418 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:38:22.418 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:38:22.418 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:38:22.418 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:38:22.418 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:38:22.418 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:38:22.418 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:38:22.418 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:38:22.418 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:38:22.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:38:22.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:38:22.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:38:22.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:38:22.418 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:38:22.418 | INFO | ------------------------------------------ | 20260507 22:38:22.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:38:22.418 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:38:22.418 | INFO | Total mem usage: 27.8 GiB | 20260507 22:38:22.418 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:38:22.418 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:38:22.418 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:38:22.418 | INFO | High Water Breakdown: | 20260507 22:38:22.418 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:38:22.418 | INFO | ------------------------------------------ | 20260507 22:38:22.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:38:22.418 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:38:22.418 | INFO | Total mem usage: 15.7 GiB | 20260507 22:38:22.418 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:38:22.418 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:38:22.418 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:38:22.418 | INFO | High Water Breakdown: | 20260507 22:38:22.418 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:38:22.418 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:38:22.418 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:38:22.418 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:38:22.418 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:38:22.418 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:38:22.418 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:38:22.418 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:38:22.418 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:38:22.418 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:38:22.418 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:38:22.418 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:38:22.418 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:38:22.418 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:38:22.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:38:22.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:38:22.418 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:38:22.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:38:22.418 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:38:22.418 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:38:22.418 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:38:22.418 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:38:22.418 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:38:22.418 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:38:22.418 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:38:22.418 | INFO | ------------------------------------------ | 20260507 22:38:22.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:38:22.418 | INFO | Uploaded 36GiB to GPU in 2.49258s (14.4429GiB/s) | 20260507 22:38:22.418 | INFO | Downloaded 34GiB to GPU in 2.44281s (13.9184GiB/s) | 20260507 22:38:22.418 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:38:22.418 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:38:22.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:38:22.418 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.65e+01, 1.41e+01 / 1.89e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.04e-01, 1.25e-02 / 6.60e-03 | 20260507 22:38:22.418 | INFO | PerfReport: 12.13%, Baseline -- 3.99e+00 / 4.48e+00, 5.86e-01 / 2.11e-01 | 20260507 22:38:22.418 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.07e-01 / 1.18e-01 | 20260507 22:38:22.418 | INFO | PerfReport: 15.94%, DME -- 4.20e+01 / 4.59e+01, 4.02e+00 / 1.84e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.75e+00 / 1.85e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.25e-01 / 2.14e-01 | 20260507 22:38:22.418 | INFO | PerfReport: 3.45%, Metrics -- 1.14e+00 / 1.37e+00, 2.86e-01 / 1.70e-01 | 20260507 22:38:22.418 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.85e-01, 5.69e-02 / 2.36e-02 | 20260507 22:38:22.418 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:38:22.418 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:38:22.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:38:22.418 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 24.48%, Upload -- 2.62e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 30.78%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 2.08%, LockUpload -- 2.23e+00 / 0.00e+00, 5.29e+00 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 2.62%, LockDownload -- 2.80e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 15.82%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:38:22.418 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 22:38:22.515 | INFO | Starting a round of Baz IO | 20260507 22:38:25.974 | INFO | Populating RTMetrics report | 20260507 22:38:25.974 | INFO | RTMetrics report populated | 20260507 22:38:25.974 | INFO | Opening tmp RTMetrics json file | 20260507 22:38:25.998 | INFO | Serializing RTMetrics to json | 20260507 22:38:25.999 | INFO | Serialization done, swapping tmp file now... | 20260507 22:38:26.000 | INFO | RTMetrics done | 20260507 22:38:26.078 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.562 seconds | 20260507 22:38:26.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.01% / 0.01% / 56.98% | 20260507 22:38:26.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36857% / 7.94729e-06% / 5.16574e-05% | 20260507 22:38:26.078 | INFO | Baz IO overall: | 20260507 22:38:26.078 | INFO | BytesWritten : 1433160880701 | 20260507 22:38:26.078 | INFO | NumEvents : 694293483094 | 20260507 22:38:27.558 | INFO | Analyzing chunk frames = [3145728,3146240) | 20260507 22:38:27.558 | INFO | MemInfo: { | 20260507 22:38:27.558 | INFO | "gpu" : " 17 GB", | 20260507 22:38:27.558 | INFO | "huge" : "", | 20260507 22:38:27.558 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:27.558 | INFO | "malloc" : " 14 GB", | 20260507 22:38:27.558 | INFO | "pinned" : " 29 GB" | 20260507 22:38:27.558 | INFO | } | 20260507 22:38:27.558 | INFO | Perf report for the last 5.14035 seconds: | 20260507 22:38:27.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:27.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:27.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:27.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:27.558 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260507 22:38:27.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:27.558 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 22:38:27.558 | INFO | PRE_HQ : 65.8%, 1.7, 57.4ms, 0 | 20260507 22:38:27.558 | INFO | BAZWRITER : 68.7%, 4.0, 142.1ms, 0 | 20260507 22:38:27.558 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 22:38:27.559 | INFO | Uploaded 36GiB to GPU in 2.52475s (14.2588GiB/s) | 20260507 22:38:27.559 | INFO | Downloaded 38GiB to GPU in 2.69513s (14.0995GiB/s) | 20260507 22:38:32.696 | INFO | Analyzing chunk frames = [3146240,3146752) | 20260507 22:38:32.696 | INFO | MemInfo: { | 20260507 22:38:32.696 | INFO | "gpu" : " 17 GB", | 20260507 22:38:32.696 | INFO | "huge" : "", | 20260507 22:38:32.696 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:32.696 | INFO | "malloc" : " 14 GB", | 20260507 22:38:32.696 | INFO | "pinned" : " 29 GB" | 20260507 22:38:32.696 | INFO | } | 20260507 22:38:32.696 | INFO | Perf report for the last 5.13842 seconds: | 20260507 22:38:32.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:32.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 22:38:32.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:38:32.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:32.697 | INFO | Uploaded 36GiB to GPU in 2.48811s (14.4688GiB/s) | 20260507 22:38:32.697 | INFO | Downloaded 34GiB to GPU in 2.46129s (13.8139GiB/s) | 20260507 22:38:37.836 | INFO | Analyzing chunk frames = [3146752,3147264) | 20260507 22:38:37.836 | INFO | MemInfo: { | 20260507 22:38:37.836 | INFO | "gpu" : " 17 GB", | 20260507 22:38:37.836 | INFO | "huge" : "", | 20260507 22:38:37.836 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:37.836 | INFO | "malloc" : " 14 GB", | 20260507 22:38:37.836 | INFO | "pinned" : " 29 GB" | 20260507 22:38:37.836 | INFO | } | 20260507 22:38:37.836 | INFO | Perf report for the last 5.13984 seconds: | 20260507 22:38:37.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:37.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.836 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:38:37.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.836 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:38:37.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:37.837 | INFO | Uploaded 36GiB to GPU in 2.58337s (13.9353GiB/s) | 20260507 22:38:37.837 | INFO | Downloaded 34GiB to GPU in 2.47148s (13.7569GiB/s) | 20260507 22:38:42.976 | INFO | Analyzing chunk frames = [3147264,3147776) | 20260507 22:38:42.976 | INFO | MemInfo: { | 20260507 22:38:42.976 | INFO | "gpu" : " 17 GB", | 20260507 22:38:42.976 | INFO | "huge" : "", | 20260507 22:38:42.976 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:42.976 | INFO | "malloc" : " 14 GB", | 20260507 22:38:42.976 | INFO | "pinned" : " 29 GB" | 20260507 22:38:42.976 | INFO | } | 20260507 22:38:42.976 | INFO | Perf report for the last 5.13946 seconds: | 20260507 22:38:42.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:42.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:38:42.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:38:42.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:42.976 | INFO | Uploaded 36GiB to GPU in 2.48885s (14.4645GiB/s) | 20260507 22:38:42.976 | INFO | Downloaded 34GiB to GPU in 2.4058s (14.1325GiB/s) | 20260507 22:38:48.124 | INFO | Analyzing chunk frames = [3147776,3148288) | 20260507 22:38:48.124 | INFO | MemInfo: { | 20260507 22:38:48.124 | INFO | "gpu" : " 17 GB", | 20260507 22:38:48.124 | INFO | "huge" : "", | 20260507 22:38:48.124 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:48.124 | INFO | "malloc" : " 14 GB", | 20260507 22:38:48.124 | INFO | "pinned" : " 29 GB" | 20260507 22:38:48.124 | INFO | } | 20260507 22:38:48.124 | INFO | Perf report for the last 5.14863 seconds: | 20260507 22:38:48.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:48.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.125 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 22:38:48.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.125 | INFO | PRE_HQ : 26.4%, 1.0, 13.8ms, 0 | 20260507 22:38:48.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:48.125 | INFO | Uploaded 36GiB to GPU in 2.36159s (15.244GiB/s) | 20260507 22:38:48.125 | INFO | Downloaded 34GiB to GPU in 2.50262s (13.5858GiB/s) | 20260507 22:38:53.258 | INFO | Analyzing chunk frames = [3148288,3148800) | 20260507 22:38:53.258 | INFO | MemInfo: { | 20260507 22:38:53.258 | INFO | "gpu" : " 17 GB", | 20260507 22:38:53.258 | INFO | "huge" : "", | 20260507 22:38:53.258 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:53.258 | INFO | "malloc" : " 14 GB", | 20260507 22:38:53.258 | INFO | "pinned" : " 29 GB" | 20260507 22:38:53.258 | INFO | } | 20260507 22:38:53.258 | INFO | Perf report for the last 5.13398 seconds: | 20260507 22:38:53.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:53.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.259 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 22:38:53.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.259 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:38:53.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:53.259 | INFO | Uploaded 36GiB to GPU in 2.41477s (14.9082GiB/s) | 20260507 22:38:53.259 | INFO | Downloaded 34GiB to GPU in 2.3771s (14.3031GiB/s) | 20260507 22:38:58.394 | INFO | Analyzing chunk frames = [3148800,3149312) | 20260507 22:38:58.394 | INFO | MemInfo: { | 20260507 22:38:58.394 | INFO | "gpu" : " 17 GB", | 20260507 22:38:58.394 | INFO | "huge" : "", | 20260507 22:38:58.394 | INFO | "huge_shared" : " 29 GB", | 20260507 22:38:58.394 | INFO | "malloc" : " 14 GB", | 20260507 22:38:58.394 | INFO | "pinned" : " 29 GB" | 20260507 22:38:58.394 | INFO | } | 20260507 22:38:58.394 | INFO | Perf report for the last 5.13578 seconds: | 20260507 22:38:58.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:38:58.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.394 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:38:58.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.394 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:38:58.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:38:58.395 | INFO | Uploaded 36GiB to GPU in 2.52569s (14.2535GiB/s) | 20260507 22:38:58.395 | INFO | Downloaded 34GiB to GPU in 2.4332s (13.9734GiB/s) | 20260507 22:39:03.534 | INFO | Analyzing chunk frames = [3149312,3149824) | 20260507 22:39:03.534 | INFO | MemInfo: { | 20260507 22:39:03.534 | INFO | "gpu" : " 17 GB", | 20260507 22:39:03.534 | INFO | "huge" : "", | 20260507 22:39:03.534 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:03.534 | INFO | "malloc" : " 14 GB", | 20260507 22:39:03.534 | INFO | "pinned" : " 29 GB" | 20260507 22:39:03.534 | INFO | } | 20260507 22:39:03.534 | INFO | Perf report for the last 5.1398 seconds: | 20260507 22:39:03.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:03.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.534 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 22:39:03.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.534 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 22:39:03.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:03.535 | INFO | Uploaded 36GiB to GPU in 2.52955s (14.2318GiB/s) | 20260507 22:39:03.535 | INFO | Downloaded 34GiB to GPU in 2.44171s (13.9247GiB/s) | 20260507 22:39:03.647 | INFO | Starting a round of Baz IO | 20260507 22:39:07.120 | INFO | Populating RTMetrics report | 20260507 22:39:07.120 | INFO | RTMetrics report populated | 20260507 22:39:07.120 | INFO | Opening tmp RTMetrics json file | 20260507 22:39:07.127 | INFO | Serializing RTMetrics to json | 20260507 22:39:07.142 | INFO | Serialization done, swapping tmp file now... | 20260507 22:39:07.143 | INFO | RTMetrics done | 20260507 22:39:07.207 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.56 seconds | 20260507 22:39:07.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.01% / 0.01% / 56.99% | 20260507 22:39:07.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36595% / 7.94729e-06% / 4.76837e-05% | 20260507 22:39:07.207 | INFO | Baz IO overall: | 20260507 22:39:07.207 | INFO | BytesWritten : 1435020789222 | 20260507 22:39:07.207 | INFO | NumEvents : 695190815170 | 20260507 22:39:08.676 | INFO | Analyzing chunk frames = [3149824,3150336) | 20260507 22:39:08.676 | INFO | MemInfo: { | 20260507 22:39:08.676 | INFO | "gpu" : " 17 GB", | 20260507 22:39:08.676 | INFO | "huge" : "", | 20260507 22:39:08.676 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:08.676 | INFO | "malloc" : " 14 GB", | 20260507 22:39:08.676 | INFO | "pinned" : " 29 GB" | 20260507 22:39:08.676 | INFO | } | 20260507 22:39:08.676 | INFO | Perf report for the last 5.142 seconds: | 20260507 22:39:08.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:08.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:08.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:08.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:08.676 | INFO | ANALYSIS : 68.9%, 2.9, 104.3ms, 0 | 20260507 22:39:08.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:08.676 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 22:39:08.676 | INFO | PRE_HQ : 67.9%, 1.8, 64.1ms, 0 | 20260507 22:39:08.676 | INFO | BAZWRITER : 68.9%, 4.2, 150.2ms, 0 | 20260507 22:39:08.676 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 22:39:08.677 | INFO | Uploaded 36GiB to GPU in 2.55556s (14.0869GiB/s) | 20260507 22:39:08.677 | INFO | Downloaded 38GiB to GPU in 2.68497s (14.1529GiB/s) | 20260507 22:39:13.816 | INFO | Analyzing chunk frames = [3150336,3150848) | 20260507 22:39:13.816 | INFO | MemInfo: { | 20260507 22:39:13.816 | INFO | "gpu" : " 17 GB", | 20260507 22:39:13.816 | INFO | "huge" : "", | 20260507 22:39:13.816 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:13.816 | INFO | "malloc" : " 14 GB", | 20260507 22:39:13.816 | INFO | "pinned" : " 29 GB" | 20260507 22:39:13.816 | INFO | } | 20260507 22:39:13.816 | INFO | Perf report for the last 5.13994 seconds: | 20260507 22:39:13.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:13.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:39:13.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:39:13.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:13.816 | INFO | Uploaded 36GiB to GPU in 2.58109s (13.9476GiB/s) | 20260507 22:39:13.816 | INFO | Downloaded 34GiB to GPU in 2.46668s (13.7837GiB/s) | 20260507 22:39:18.949 | INFO | Analyzing chunk frames = [3150848,3151360) | 20260507 22:39:18.949 | INFO | MemInfo: { | 20260507 22:39:18.949 | INFO | "gpu" : " 17 GB", | 20260507 22:39:18.949 | INFO | "huge" : "", | 20260507 22:39:18.949 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:18.949 | INFO | "malloc" : " 14 GB", | 20260507 22:39:18.949 | INFO | "pinned" : " 29 GB" | 20260507 22:39:18.949 | INFO | } | 20260507 22:39:18.949 | INFO | Perf report for the last 5.13287 seconds: | 20260507 22:39:18.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:18.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260507 22:39:18.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:39:18.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:18.949 | INFO | Uploaded 36GiB to GPU in 2.54891s (14.1237GiB/s) | 20260507 22:39:18.949 | INFO | Downloaded 34GiB to GPU in 2.47617s (13.7309GiB/s) | 20260507 22:39:24.087 | INFO | Analyzing chunk frames = [3151360,3151872) | 20260507 22:39:24.087 | INFO | MemInfo: { | 20260507 22:39:24.087 | INFO | "gpu" : " 17 GB", | 20260507 22:39:24.087 | INFO | "huge" : "", | 20260507 22:39:24.087 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:24.087 | INFO | "malloc" : " 14 GB", | 20260507 22:39:24.088 | INFO | "pinned" : " 29 GB" | 20260507 22:39:24.088 | INFO | } | 20260507 22:39:24.088 | INFO | Perf report for the last 5.13869 seconds: | 20260507 22:39:24.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:24.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 22:39:24.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:39:24.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:24.088 | INFO | Uploaded 36GiB to GPU in 2.47874s (14.5235GiB/s) | 20260507 22:39:24.088 | INFO | Downloaded 34GiB to GPU in 2.37684s (14.3047GiB/s) | 20260507 22:39:29.230 | INFO | Analyzing chunk frames = [3151872,3152384) | 20260507 22:39:29.230 | INFO | MemInfo: { | 20260507 22:39:29.230 | INFO | "gpu" : " 17 GB", | 20260507 22:39:29.230 | INFO | "huge" : "", | 20260507 22:39:29.230 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:29.230 | INFO | "malloc" : " 14 GB", | 20260507 22:39:29.231 | INFO | "pinned" : " 29 GB" | 20260507 22:39:29.231 | INFO | } | 20260507 22:39:29.231 | INFO | Perf report for the last 5.14294 seconds: | 20260507 22:39:29.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:29.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 22:39:29.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 22:39:29.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:29.231 | INFO | Uploaded 36GiB to GPU in 2.39554s (15.0279GiB/s) | 20260507 22:39:29.231 | INFO | Downloaded 34GiB to GPU in 2.49291s (13.6387GiB/s) | 20260507 22:39:34.372 | INFO | Analyzing chunk frames = [3152384,3152896) | 20260507 22:39:34.372 | INFO | MemInfo: { | 20260507 22:39:34.372 | INFO | "gpu" : " 17 GB", | 20260507 22:39:34.372 | INFO | "huge" : "", | 20260507 22:39:34.372 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:34.372 | INFO | "malloc" : " 14 GB", | 20260507 22:39:34.372 | INFO | "pinned" : " 29 GB" | 20260507 22:39:34.372 | INFO | } | 20260507 22:39:34.372 | INFO | Perf report for the last 5.14173 seconds: | 20260507 22:39:34.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:34.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.372 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 22:39:34.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.373 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:39:34.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:34.373 | INFO | Uploaded 36GiB to GPU in 2.46628s (14.5969GiB/s) | 20260507 22:39:34.373 | INFO | Downloaded 34GiB to GPU in 2.41686s (14.0678GiB/s) | 20260507 22:39:39.508 | INFO | Analyzing chunk frames = [3152896,3153408) | 20260507 22:39:39.508 | INFO | MemInfo: { | 20260507 22:39:39.508 | INFO | "gpu" : " 17 GB", | 20260507 22:39:39.508 | INFO | "huge" : "", | 20260507 22:39:39.508 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:39.508 | INFO | "malloc" : " 14 GB", | 20260507 22:39:39.508 | INFO | "pinned" : " 29 GB" | 20260507 22:39:39.508 | INFO | } | 20260507 22:39:39.508 | INFO | Perf report for the last 5.13588 seconds: | 20260507 22:39:39.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:39.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:39:39.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 22:39:39.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:39.508 | INFO | Uploaded 36GiB to GPU in 2.53806s (14.1841GiB/s) | 20260507 22:39:39.509 | INFO | Downloaded 34GiB to GPU in 2.45566s (13.8456GiB/s) | 20260507 22:39:44.646 | INFO | Analyzing chunk frames = [3153408,3153920) | 20260507 22:39:44.646 | INFO | MemInfo: { | 20260507 22:39:44.646 | INFO | "gpu" : " 17 GB", | 20260507 22:39:44.647 | INFO | "huge" : "", | 20260507 22:39:44.647 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:44.647 | INFO | "malloc" : " 14 GB", | 20260507 22:39:44.647 | INFO | "pinned" : " 29 GB" | 20260507 22:39:44.647 | INFO | } | 20260507 22:39:44.647 | INFO | Perf report for the last 5.1386 seconds: | 20260507 22:39:44.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:44.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.647 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:39:44.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.647 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 22:39:44.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:44.648 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:39:44.648 | INFO | Total mem usage: 16.7 GiB | 20260507 22:39:44.648 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:39:44.648 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -5.44%) | 20260507 22:39:44.648 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:39:44.648 | INFO | High Water Breakdown: | 20260507 22:39:44.648 | INFO | 61% : 9.65 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:39:44.648 | INFO | 38.9% : 6.15 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:39:44.648 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:39:44.648 | INFO | ------------------------------------------ | 20260507 22:39:44.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:39:44.648 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:39:44.648 | INFO | Total mem usage: 26.5 GiB | 20260507 22:39:44.648 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:39:44.648 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:39:44.648 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:39:44.648 | INFO | Uploaded 36GiB to GPU in 2.52761s (14.2427GiB/s) | 20260507 22:39:44.648 | INFO | High Water Breakdown: | 20260507 22:39:44.648 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:39:44.648 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:39:44.648 | INFO | Downloaded 34GiB to GPU in 2.44614s (13.8995GiB/s) | 20260507 22:39:44.648 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:39:44.648 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:39:44.648 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:39:44.648 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:39:44.648 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:39:44.648 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:39:44.648 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:39:44.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:39:44.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:39:44.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:39:44.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:39:44.648 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:39:44.648 | INFO | ------------------------------------------ | 20260507 22:39:44.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:39:44.648 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:39:44.648 | INFO | Total mem usage: 27.8 GiB | 20260507 22:39:44.648 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:39:44.648 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:39:44.648 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:39:44.648 | INFO | High Water Breakdown: | 20260507 22:39:44.648 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:39:44.648 | INFO | ------------------------------------------ | 20260507 22:39:44.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:39:44.648 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:39:44.648 | INFO | Total mem usage: 15.7 GiB | 20260507 22:39:44.649 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:39:44.649 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:39:44.649 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:39:44.649 | INFO | High Water Breakdown: | 20260507 22:39:44.649 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:39:44.649 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:39:44.649 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:39:44.649 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:39:44.649 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:39:44.649 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:39:44.649 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:39:44.649 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:39:44.649 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:39:44.649 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:39:44.649 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:39:44.649 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:39:44.649 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:39:44.649 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:39:44.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:39:44.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:39:44.649 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:39:44.649 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:39:44.649 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:39:44.649 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:39:44.649 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:39:44.649 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:39:44.649 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:39:44.649 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:39:44.649 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:39:44.649 | INFO | ------------------------------------------ | 20260507 22:39:44.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:39:44.649 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:39:44.649 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:39:44.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:39:44.649 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.57e+01, 1.39e+01 / 1.43e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.12e-02 / 6.05e-03 | 20260507 22:39:44.649 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.47e+00, 5.95e-01 / 2.45e-01 | 20260507 22:39:44.649 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.49e+00, 2.08e-01 / 1.13e-01 | 20260507 22:39:44.649 | INFO | PerfReport: 15.88%, DME -- 4.19e+01 / 4.52e+01, 3.67e+00 / 1.30e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.74e+00 / 1.72e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.71e+00, 4.29e-01 / 2.61e-01 | 20260507 22:39:44.649 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.32e+00, 2.88e-01 / 1.81e-01 | 20260507 22:39:44.649 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.54e-01 / 3.81e-01, 5.71e-02 / 2.51e-02 | 20260507 22:39:44.649 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:39:44.649 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:39:44.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:39:44.649 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 24.19%, Download -- 2.58e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 30.91%, Compute -- 3.30e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 2.14%, LockUpload -- 2.28e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 2.46%, LockDownload -- 2.62e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 15.87%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:39:44.649 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 22:39:44.762 | INFO | Starting a round of Baz IO | 20260507 22:39:48.275 | INFO | Populating RTMetrics report | 20260507 22:39:48.275 | INFO | RTMetrics report populated | 20260507 22:39:48.275 | INFO | Opening tmp RTMetrics json file | 20260507 22:39:48.279 | INFO | Serializing RTMetrics to json | 20260507 22:39:48.281 | INFO | Serialization done, swapping tmp file now... | 20260507 22:39:48.281 | INFO | RTMetrics done | 20260507 22:39:48.367 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.604 seconds | 20260507 22:39:48.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.00% / 0.01% / 56.99% | 20260507 22:39:48.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36761% / 3.97364e-06% / 4.76837e-05% | 20260507 22:39:48.367 | INFO | Baz IO overall: | 20260507 22:39:48.367 | INFO | BytesWritten : 1436882289302 | 20260507 22:39:48.367 | INFO | NumEvents : 696089663370 | 20260507 22:39:49.787 | INFO | Analyzing chunk frames = [3153920,3154432) | 20260507 22:39:49.787 | INFO | MemInfo: { | 20260507 22:39:49.787 | INFO | "gpu" : " 17 GB", | 20260507 22:39:49.787 | INFO | "huge" : "", | 20260507 22:39:49.787 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:49.787 | INFO | "malloc" : " 14 GB", | 20260507 22:39:49.787 | INFO | "pinned" : " 29 GB" | 20260507 22:39:49.787 | INFO | } | 20260507 22:39:49.787 | INFO | Perf report for the last 5.13977 seconds: | 20260507 22:39:49.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:49.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:49.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:49.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:49.787 | INFO | ANALYSIS : 69.7%, 2.9, 105.0ms, 0 | 20260507 22:39:49.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:49.787 | INFO | RT_METRICS : 49.0%, 1.0, 25.7ms, 0 | 20260507 22:39:49.787 | INFO | PRE_HQ : 69.2%, 1.9, 69.8ms, 0 | 20260507 22:39:49.787 | INFO | BAZWRITER : 69.7%, 4.5, 162.1ms, 0 | 20260507 22:39:49.787 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 22:39:49.787 | INFO | Uploaded 36GiB to GPU in 2.55111s (14.1115GiB/s) | 20260507 22:39:49.787 | INFO | Downloaded 38GiB to GPU in 2.71922s (13.9746GiB/s) | 20260507 22:39:54.929 | INFO | Analyzing chunk frames = [3154432,3154944) | 20260507 22:39:54.929 | INFO | MemInfo: { | 20260507 22:39:54.929 | INFO | "gpu" : " 17 GB", | 20260507 22:39:54.929 | INFO | "huge" : "", | 20260507 22:39:54.929 | INFO | "huge_shared" : " 29 GB", | 20260507 22:39:54.929 | INFO | "malloc" : " 14 GB", | 20260507 22:39:54.929 | INFO | "pinned" : " 29 GB" | 20260507 22:39:54.929 | INFO | } | 20260507 22:39:54.930 | INFO | Perf report for the last 5.14293 seconds: | 20260507 22:39:54.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:39:54.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 22:39:54.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:39:54.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:39:54.930 | INFO | Uploaded 36GiB to GPU in 2.54731s (14.1325GiB/s) | 20260507 22:39:54.930 | INFO | Downloaded 34GiB to GPU in 2.49393s (13.6331GiB/s) | 20260507 22:40:00.075 | INFO | Analyzing chunk frames = [3154944,3155456) | 20260507 22:40:00.075 | INFO | MemInfo: { | 20260507 22:40:00.075 | INFO | "gpu" : " 17 GB", | 20260507 22:40:00.075 | INFO | "huge" : "", | 20260507 22:40:00.075 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:00.075 | INFO | "malloc" : " 14 GB", | 20260507 22:40:00.075 | INFO | "pinned" : " 29 GB" | 20260507 22:40:00.075 | INFO | } | 20260507 22:40:00.075 | INFO | Perf report for the last 5.14524 seconds: | 20260507 22:40:00.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:00.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:40:00.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 22:40:00.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:00.075 | INFO | Uploaded 36GiB to GPU in 2.50915s (14.3475GiB/s) | 20260507 22:40:00.075 | INFO | Downloaded 34GiB to GPU in 2.48746s (13.6686GiB/s) | 20260507 22:40:05.208 | INFO | Analyzing chunk frames = [3155456,3155968) | 20260507 22:40:05.208 | INFO | MemInfo: { | 20260507 22:40:05.208 | INFO | "gpu" : " 17 GB", | 20260507 22:40:05.208 | INFO | "huge" : "", | 20260507 22:40:05.208 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:05.208 | INFO | "malloc" : " 14 GB", | 20260507 22:40:05.208 | INFO | "pinned" : " 29 GB" | 20260507 22:40:05.208 | INFO | } | 20260507 22:40:05.208 | INFO | Perf report for the last 5.13284 seconds: | 20260507 22:40:05.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:05.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260507 22:40:05.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:40:05.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:05.208 | INFO | Uploaded 36GiB to GPU in 2.50662s (14.362GiB/s) | 20260507 22:40:05.208 | INFO | Downloaded 34GiB to GPU in 2.4719s (13.7546GiB/s) | 20260507 22:40:10.348 | INFO | Analyzing chunk frames = [3155968,3156480) | 20260507 22:40:10.348 | INFO | MemInfo: { | 20260507 22:40:10.348 | INFO | "gpu" : " 17 GB", | 20260507 22:40:10.348 | INFO | "huge" : "", | 20260507 22:40:10.348 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:10.348 | INFO | "malloc" : " 14 GB", | 20260507 22:40:10.348 | INFO | "pinned" : " 29 GB" | 20260507 22:40:10.348 | INFO | } | 20260507 22:40:10.348 | INFO | Perf report for the last 5.14023 seconds: | 20260507 22:40:10.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:10.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.348 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:40:10.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.348 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:40:10.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:10.349 | INFO | Uploaded 36GiB to GPU in 2.36634s (15.2133GiB/s) | 20260507 22:40:10.349 | INFO | Downloaded 34GiB to GPU in 2.49741s (13.6141GiB/s) | 20260507 22:40:15.486 | INFO | Analyzing chunk frames = [3156480,3156992) | 20260507 22:40:15.486 | INFO | MemInfo: { | 20260507 22:40:15.486 | INFO | "gpu" : " 17 GB", | 20260507 22:40:15.486 | INFO | "huge" : "", | 20260507 22:40:15.486 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:15.486 | INFO | "malloc" : " 14 GB", | 20260507 22:40:15.486 | INFO | "pinned" : " 29 GB" | 20260507 22:40:15.486 | INFO | } | 20260507 22:40:15.486 | INFO | Perf report for the last 5.13822 seconds: | 20260507 22:40:15.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:15.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 22:40:15.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:40:15.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:15.486 | INFO | Uploaded 36GiB to GPU in 2.43567s (14.7803GiB/s) | 20260507 22:40:15.486 | INFO | Downloaded 34GiB to GPU in 2.39421s (14.2009GiB/s) | 20260507 22:40:20.626 | INFO | Analyzing chunk frames = [3156992,3157504) | 20260507 22:40:20.626 | INFO | MemInfo: { | 20260507 22:40:20.626 | INFO | "gpu" : " 17 GB", | 20260507 22:40:20.626 | INFO | "huge" : "", | 20260507 22:40:20.626 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:20.626 | INFO | "malloc" : " 14 GB", | 20260507 22:40:20.626 | INFO | "pinned" : " 29 GB" | 20260507 22:40:20.626 | INFO | } | 20260507 22:40:20.626 | INFO | Perf report for the last 5.14023 seconds: | 20260507 22:40:20.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:20.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.626 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:40:20.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.626 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:40:20.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:20.627 | INFO | Uploaded 36GiB to GPU in 2.5219s (14.275GiB/s) | 20260507 22:40:20.627 | INFO | Downloaded 34GiB to GPU in 2.43549s (13.9602GiB/s) | 20260507 22:40:25.762 | INFO | Analyzing chunk frames = [3157504,3158016) | 20260507 22:40:25.762 | INFO | MemInfo: { | 20260507 22:40:25.762 | INFO | "gpu" : " 17 GB", | 20260507 22:40:25.762 | INFO | "huge" : "", | 20260507 22:40:25.762 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:25.763 | INFO | "malloc" : " 14 GB", | 20260507 22:40:25.763 | INFO | "pinned" : " 29 GB" | 20260507 22:40:25.763 | INFO | } | 20260507 22:40:25.763 | INFO | Perf report for the last 5.13637 seconds: | 20260507 22:40:25.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:25.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:40:25.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 22:40:25.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:25.763 | INFO | Uploaded 36GiB to GPU in 2.5423s (14.1604GiB/s) | 20260507 22:40:25.763 | INFO | Downloaded 34GiB to GPU in 2.44013s (13.9337GiB/s) | 20260507 22:40:25.877 | INFO | Starting a round of Baz IO | 20260507 22:40:29.336 | INFO | Populating RTMetrics report | 20260507 22:40:29.337 | INFO | RTMetrics report populated | 20260507 22:40:29.337 | INFO | Opening tmp RTMetrics json file | 20260507 22:40:29.344 | INFO | Serializing RTMetrics to json | 20260507 22:40:29.357 | INFO | Serialization done, swapping tmp file now... | 20260507 22:40:29.357 | INFO | RTMetrics done | 20260507 22:40:29.436 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.558 seconds | 20260507 22:40:29.436 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.99% / 0.01% / 57.00% | 20260507 22:40:29.436 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36063% / 0% / 5.16574e-05% | 20260507 22:40:29.436 | INFO | Baz IO overall: | 20260507 22:40:29.436 | INFO | BytesWritten : 1438740667373 | 20260507 22:40:29.436 | INFO | NumEvents : 696986479146 | 20260507 22:40:30.900 | INFO | Analyzing chunk frames = [3158016,3158528) | 20260507 22:40:30.900 | INFO | MemInfo: { | 20260507 22:40:30.900 | INFO | "gpu" : " 17 GB", | 20260507 22:40:30.900 | INFO | "huge" : "", | 20260507 22:40:30.900 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:30.900 | INFO | "malloc" : " 14 GB", | 20260507 22:40:30.900 | INFO | "pinned" : " 29 GB" | 20260507 22:40:30.900 | INFO | } | 20260507 22:40:30.900 | INFO | Perf report for the last 5.13725 seconds: | 20260507 22:40:30.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:30.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:30.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:30.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:30.900 | INFO | ANALYSIS : 68.8%, 2.9, 105.0ms, 0 | 20260507 22:40:30.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:30.900 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 22:40:30.900 | INFO | PRE_HQ : 67.6%, 1.8, 62.8ms, 0 | 20260507 22:40:30.900 | INFO | BAZWRITER : 68.8%, 3.7, 131.7ms, 0 | 20260507 22:40:30.900 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:40:30.900 | INFO | Uploaded 36GiB to GPU in 2.57218s (13.9959GiB/s) | 20260507 22:40:30.900 | INFO | Downloaded 38GiB to GPU in 2.67662s (14.197GiB/s) | 20260507 22:40:36.042 | INFO | Analyzing chunk frames = [3158528,3159040) | 20260507 22:40:36.042 | INFO | MemInfo: { | 20260507 22:40:36.042 | INFO | "gpu" : " 17 GB", | 20260507 22:40:36.042 | INFO | "huge" : "", | 20260507 22:40:36.042 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:36.042 | INFO | "malloc" : " 14 GB", | 20260507 22:40:36.042 | INFO | "pinned" : " 29 GB" | 20260507 22:40:36.042 | INFO | } | 20260507 22:40:36.042 | INFO | Perf report for the last 5.14185 seconds: | 20260507 22:40:36.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:36.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 22:40:36.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:40:36.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:36.042 | INFO | Uploaded 36GiB to GPU in 2.57021s (14.0066GiB/s) | 20260507 22:40:36.043 | INFO | Downloaded 34GiB to GPU in 2.47042s (13.7628GiB/s) | 20260507 22:40:41.183 | INFO | Analyzing chunk frames = [3159040,3159552) | 20260507 22:40:41.183 | INFO | MemInfo: { | 20260507 22:40:41.183 | INFO | "gpu" : " 17 GB", | 20260507 22:40:41.183 | INFO | "huge" : "", | 20260507 22:40:41.183 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:41.183 | INFO | "malloc" : " 14 GB", | 20260507 22:40:41.183 | INFO | "pinned" : " 29 GB" | 20260507 22:40:41.183 | INFO | } | 20260507 22:40:41.183 | INFO | Perf report for the last 5.14148 seconds: | 20260507 22:40:41.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:41.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.183 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:40:41.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.184 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:40:41.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:41.184 | INFO | Uploaded 36GiB to GPU in 2.55569s (14.0862GiB/s) | 20260507 22:40:41.184 | INFO | Downloaded 34GiB to GPU in 2.45411s (13.8543GiB/s) | 20260507 22:40:46.322 | INFO | Analyzing chunk frames = [3159552,3160064) | 20260507 22:40:46.322 | INFO | MemInfo: { | 20260507 22:40:46.322 | INFO | "gpu" : " 17 GB", | 20260507 22:40:46.322 | INFO | "huge" : "", | 20260507 22:40:46.322 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:46.322 | INFO | "malloc" : " 14 GB", | 20260507 22:40:46.322 | INFO | "pinned" : " 29 GB" | 20260507 22:40:46.322 | INFO | } | 20260507 22:40:46.322 | INFO | Perf report for the last 5.13903 seconds: | 20260507 22:40:46.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:46.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260507 22:40:46.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:40:46.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:46.323 | INFO | Uploaded 36GiB to GPU in 2.51082s (14.3379GiB/s) | 20260507 22:40:46.323 | INFO | Downloaded 34GiB to GPU in 2.40391s (14.1436GiB/s) | 20260507 22:40:51.460 | INFO | Analyzing chunk frames = [3160064,3160576) | 20260507 22:40:51.460 | INFO | MemInfo: { | 20260507 22:40:51.460 | INFO | "gpu" : " 17 GB", | 20260507 22:40:51.460 | INFO | "huge" : "", | 20260507 22:40:51.460 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:51.460 | INFO | "malloc" : " 14 GB", | 20260507 22:40:51.460 | INFO | "pinned" : " 29 GB" | 20260507 22:40:51.460 | INFO | } | 20260507 22:40:51.460 | INFO | Perf report for the last 5.13752 seconds: | 20260507 22:40:51.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:51.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.460 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:40:51.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.460 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 22:40:51.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:51.461 | INFO | Uploaded 36GiB to GPU in 2.35917s (15.2596GiB/s) | 20260507 22:40:51.461 | INFO | Downloaded 34GiB to GPU in 2.49529s (13.6256GiB/s) | 20260507 22:40:56.601 | INFO | Analyzing chunk frames = [3160576,3161088) | 20260507 22:40:56.601 | INFO | MemInfo: { | 20260507 22:40:56.601 | INFO | "gpu" : " 17 GB", | 20260507 22:40:56.601 | INFO | "huge" : "", | 20260507 22:40:56.601 | INFO | "huge_shared" : " 29 GB", | 20260507 22:40:56.601 | INFO | "malloc" : " 14 GB", | 20260507 22:40:56.601 | INFO | "pinned" : " 29 GB" | 20260507 22:40:56.601 | INFO | } | 20260507 22:40:56.601 | INFO | Perf report for the last 5.14112 seconds: | 20260507 22:40:56.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:40:56.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.601 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:40:56.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.601 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:40:56.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:40:56.602 | INFO | Uploaded 36GiB to GPU in 2.5137s (14.3215GiB/s) | 20260507 22:40:56.602 | INFO | Downloaded 34GiB to GPU in 2.42082s (14.0448GiB/s) | 20260507 22:41:01.742 | INFO | Analyzing chunk frames = [3161088,3161600) | 20260507 22:41:01.742 | INFO | MemInfo: { | 20260507 22:41:01.742 | INFO | "gpu" : " 17 GB", | 20260507 22:41:01.742 | INFO | "huge" : "", | 20260507 22:41:01.742 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:01.742 | INFO | "malloc" : " 14 GB", | 20260507 22:41:01.742 | INFO | "pinned" : " 29 GB" | 20260507 22:41:01.742 | INFO | } | 20260507 22:41:01.742 | INFO | Perf report for the last 5.14095 seconds: | 20260507 22:41:01.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:01.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.742 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:41:01.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.742 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:41:01.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:01.743 | INFO | Uploaded 36GiB to GPU in 2.55375s (14.0969GiB/s) | 20260507 22:41:01.743 | INFO | Downloaded 34GiB to GPU in 2.45441s (13.8526GiB/s) | 20260507 22:41:06.884 | INFO | Analyzing chunk frames = [3161600,3162112) | 20260507 22:41:06.884 | INFO | MemInfo: { | 20260507 22:41:06.884 | INFO | "gpu" : " 17 GB", | 20260507 22:41:06.884 | INFO | "huge" : "", | 20260507 22:41:06.884 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:06.885 | INFO | "malloc" : " 14 GB", | 20260507 22:41:06.885 | INFO | "pinned" : " 29 GB" | 20260507 22:41:06.885 | INFO | } | 20260507 22:41:06.885 | INFO | Perf report for the last 5.14257 seconds: | 20260507 22:41:06.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:06.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 22:41:06.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260507 22:41:06.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:06.885 | INFO | Uploaded 36GiB to GPU in 2.54947s (14.1206GiB/s) | 20260507 22:41:06.885 | INFO | Downloaded 34GiB to GPU in 2.44653s (13.8973GiB/s) | 20260507 22:41:06.885 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:41:06.885 | INFO | Total mem usage: 16.7 GiB | 20260507 22:41:06.885 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:41:06.885 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.53%) | 20260507 22:41:06.885 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:41:06.885 | INFO | High Water Breakdown: | 20260507 22:41:06.885 | INFO | 60.5% : 9.3 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:41:06.885 | INFO | 39.3% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:41:06.885 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:41:06.885 | INFO | ------------------------------------------ | 20260507 22:41:06.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:41:06.885 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:41:06.885 | INFO | Total mem usage: 26.5 GiB | 20260507 22:41:06.886 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:41:06.886 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260507 22:41:06.886 | INFO | Unused Cache: 1.74 GiB (Waste: 6.56%) | 20260507 22:41:06.886 | INFO | High Water Breakdown: | 20260507 22:41:06.886 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:41:06.886 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:41:06.886 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:41:06.886 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:41:06.886 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:41:06.886 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:41:06.886 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:41:06.886 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:41:06.886 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:41:06.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:41:06.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:41:06.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:41:06.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:41:06.886 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:41:06.886 | INFO | ------------------------------------------ | 20260507 22:41:06.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:41:06.886 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:41:06.886 | INFO | Total mem usage: 27.8 GiB | 20260507 22:41:06.886 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:41:06.886 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:41:06.886 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:41:06.886 | INFO | High Water Breakdown: | 20260507 22:41:06.886 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:41:06.886 | INFO | ------------------------------------------ | 20260507 22:41:06.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:41:06.886 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:41:06.886 | INFO | Total mem usage: 15.7 GiB | 20260507 22:41:06.886 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:41:06.886 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:41:06.886 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:41:06.886 | INFO | High Water Breakdown: | 20260507 22:41:06.886 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:41:06.886 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:41:06.886 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:41:06.886 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:41:06.886 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:41:06.886 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:41:06.886 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:41:06.886 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:41:06.886 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:41:06.886 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:41:06.886 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:41:06.886 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:41:06.886 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:41:06.886 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:41:06.886 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:41:06.886 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:41:06.886 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:41:06.886 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:41:06.886 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:41:06.886 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:41:06.886 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:41:06.886 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:41:06.886 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:41:06.886 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:41:06.886 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:41:06.886 | INFO | ------------------------------------------ | 20260507 22:41:06.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:41:06.886 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:41:06.886 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:41:06.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:41:06.886 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.61e+01, 1.40e+01 / 1.74e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.04e-01, 1.12e-02 / 1.06e-02 | 20260507 22:41:06.886 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.49e+00, 5.93e-01 / 2.27e-01 | 20260507 22:41:06.886 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.07e-01 | 20260507 22:41:06.886 | INFO | PerfReport: 15.99%, DME -- 4.21e+01 / 4.58e+01, 3.97e+00 / 1.65e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.69e+00 / 1.60e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.47e+00 / 2.76e+00, 4.38e-01 / 2.35e-01 | 20260507 22:41:06.886 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.34e+00, 2.90e-01 / 1.21e-01 | 20260507 22:41:06.886 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.54e-01 / 3.87e-01, 5.84e-02 / 3.31e-02 | 20260507 22:41:06.886 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:41:06.886 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:41:06.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:41:06.886 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 24.52%, Upload -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 30.85%, Compute -- 3.29e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 2.22%, LockUpload -- 2.37e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 2.46%, LockDownload -- 2.62e+00 / 0.00e+00, 3.79e+00 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:41:06.886 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260507 22:41:06.985 | INFO | Starting a round of Baz IO | 20260507 22:41:10.446 | INFO | Populating RTMetrics report | 20260507 22:41:10.446 | INFO | RTMetrics report populated | 20260507 22:41:10.446 | INFO | Opening tmp RTMetrics json file | 20260507 22:41:10.467 | INFO | Serializing RTMetrics to json | 20260507 22:41:10.468 | INFO | Serialization done, swapping tmp file now... | 20260507 22:41:10.468 | INFO | RTMetrics done | 20260507 22:41:10.542 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.556 seconds | 20260507 22:41:10.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.99% / 0.01% / 57.01% | 20260507 22:41:10.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.3593% / 0% / 4.76837e-05% | 20260507 22:41:10.542 | INFO | Baz IO overall: | 20260507 22:41:10.542 | INFO | BytesWritten : 1440599096534 | 20260507 22:41:10.542 | INFO | NumEvents : 697883297374 | 20260507 22:41:12.021 | INFO | Analyzing chunk frames = [3162112,3162624) | 20260507 22:41:12.021 | INFO | MemInfo: { | 20260507 22:41:12.021 | INFO | "gpu" : " 17 GB", | 20260507 22:41:12.021 | INFO | "huge" : "", | 20260507 22:41:12.021 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:12.021 | INFO | "malloc" : " 14 GB", | 20260507 22:41:12.021 | INFO | "pinned" : " 29 GB" | 20260507 22:41:12.021 | INFO | } | 20260507 22:41:12.021 | INFO | Perf report for the last 5.13612 seconds: | 20260507 22:41:12.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:12.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:12.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:12.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:12.021 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 22:41:12.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:12.021 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 22:41:12.021 | INFO | PRE_HQ : 65.7%, 1.7, 57.3ms, 0 | 20260507 22:41:12.021 | INFO | BAZWRITER : 68.9%, 4.3, 153.8ms, 0 | 20260507 22:41:12.021 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:41:12.021 | INFO | Uploaded 36GiB to GPU in 2.56749s (14.0215GiB/s) | 20260507 22:41:12.021 | INFO | Downloaded 38GiB to GPU in 2.69958s (14.0762GiB/s) | 20260507 22:41:17.159 | INFO | Analyzing chunk frames = [3162624,3163136) | 20260507 22:41:17.159 | INFO | MemInfo: { | 20260507 22:41:17.159 | INFO | "gpu" : " 17 GB", | 20260507 22:41:17.159 | INFO | "huge" : "", | 20260507 22:41:17.159 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:17.159 | INFO | "malloc" : " 14 GB", | 20260507 22:41:17.159 | INFO | "pinned" : " 29 GB" | 20260507 22:41:17.159 | INFO | } | 20260507 22:41:17.159 | INFO | Perf report for the last 5.13844 seconds: | 20260507 22:41:17.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:17.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.159 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 22:41:17.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.159 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:41:17.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:17.160 | INFO | Uploaded 36GiB to GPU in 2.49229s (14.4445GiB/s) | 20260507 22:41:17.160 | INFO | Downloaded 34GiB to GPU in 2.49537s (13.6252GiB/s) | 20260507 22:41:22.299 | INFO | Analyzing chunk frames = [3163136,3163648) | 20260507 22:41:22.299 | INFO | MemInfo: { | 20260507 22:41:22.299 | INFO | "gpu" : " 17 GB", | 20260507 22:41:22.299 | INFO | "huge" : "", | 20260507 22:41:22.299 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:22.299 | INFO | "malloc" : " 14 GB", | 20260507 22:41:22.299 | INFO | "pinned" : " 29 GB" | 20260507 22:41:22.299 | INFO | } | 20260507 22:41:22.299 | INFO | Perf report for the last 5.14012 seconds: | 20260507 22:41:22.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:22.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.299 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:41:22.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.299 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:41:22.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:22.300 | INFO | Uploaded 36GiB to GPU in 2.55175s (14.1079GiB/s) | 20260507 22:41:22.300 | INFO | Downloaded 34GiB to GPU in 2.47498s (13.7375GiB/s) | 20260507 22:41:27.437 | INFO | Analyzing chunk frames = [3163648,3164160) | 20260507 22:41:27.437 | INFO | MemInfo: { | 20260507 22:41:27.437 | INFO | "gpu" : " 17 GB", | 20260507 22:41:27.437 | INFO | "huge" : "", | 20260507 22:41:27.437 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:27.437 | INFO | "malloc" : " 14 GB", | 20260507 22:41:27.437 | INFO | "pinned" : " 29 GB" | 20260507 22:41:27.437 | INFO | } | 20260507 22:41:27.437 | INFO | Perf report for the last 5.13798 seconds: | 20260507 22:41:27.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:27.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.437 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 22:41:27.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.437 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 22:41:27.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:27.438 | INFO | Uploaded 36GiB to GPU in 2.50809s (14.3535GiB/s) | 20260507 22:41:27.438 | INFO | Downloaded 34GiB to GPU in 2.50559s (13.5697GiB/s) | 20260507 22:41:32.581 | INFO | Analyzing chunk frames = [3164160,3164672) | 20260507 22:41:32.581 | INFO | MemInfo: { | 20260507 22:41:32.581 | INFO | "gpu" : " 17 GB", | 20260507 22:41:32.581 | INFO | "huge" : "", | 20260507 22:41:32.581 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:32.581 | INFO | "malloc" : " 14 GB", | 20260507 22:41:32.581 | INFO | "pinned" : " 29 GB" | 20260507 22:41:32.581 | INFO | } | 20260507 22:41:32.581 | INFO | Perf report for the last 5.14365 seconds: | 20260507 22:41:32.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:32.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.581 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:41:32.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.581 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:41:32.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:32.582 | INFO | Uploaded 36GiB to GPU in 2.32988s (15.4514GiB/s) | 20260507 22:41:32.582 | INFO | Downloaded 34GiB to GPU in 2.50518s (13.5719GiB/s) | 20260507 22:41:37.708 | INFO | Analyzing chunk frames = [3164672,3165184) | 20260507 22:41:37.708 | INFO | MemInfo: { | 20260507 22:41:37.708 | INFO | "gpu" : " 17 GB", | 20260507 22:41:37.708 | INFO | "huge" : "", | 20260507 22:41:37.708 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:37.708 | INFO | "malloc" : " 14 GB", | 20260507 22:41:37.708 | INFO | "pinned" : " 29 GB" | 20260507 22:41:37.708 | INFO | } | 20260507 22:41:37.708 | INFO | Perf report for the last 5.12729 seconds: | 20260507 22:41:37.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:37.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.708 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260507 22:41:37.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.709 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:41:37.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:37.709 | INFO | Uploaded 36GiB to GPU in 2.38597s (15.0882GiB/s) | 20260507 22:41:37.709 | INFO | Downloaded 34GiB to GPU in 2.40142s (14.1583GiB/s) | 20260507 22:41:42.848 | INFO | Analyzing chunk frames = [3165184,3165696) | 20260507 22:41:42.848 | INFO | MemInfo: { | 20260507 22:41:42.848 | INFO | "gpu" : " 17 GB", | 20260507 22:41:42.848 | INFO | "huge" : "", | 20260507 22:41:42.848 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:42.848 | INFO | "malloc" : " 14 GB", | 20260507 22:41:42.848 | INFO | "pinned" : " 29 GB" | 20260507 22:41:42.848 | INFO | } | 20260507 22:41:42.848 | INFO | Perf report for the last 5.13993 seconds: | 20260507 22:41:42.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:42.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.848 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:41:42.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.848 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260507 22:41:42.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:42.849 | INFO | Uploaded 36GiB to GPU in 2.51922s (14.2902GiB/s) | 20260507 22:41:42.849 | INFO | Downloaded 34GiB to GPU in 2.4652s (13.792GiB/s) | 20260507 22:41:47.993 | INFO | Analyzing chunk frames = [3165696,3166208) | 20260507 22:41:47.993 | INFO | MemInfo: { | 20260507 22:41:47.993 | INFO | "gpu" : " 17 GB", | 20260507 22:41:47.993 | INFO | "huge" : "", | 20260507 22:41:47.994 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:47.994 | INFO | "malloc" : " 14 GB", | 20260507 22:41:47.994 | INFO | "pinned" : " 29 GB" | 20260507 22:41:47.994 | INFO | } | 20260507 22:41:47.994 | INFO | Perf report for the last 5.14537 seconds: | 20260507 22:41:47.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:47.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 22:41:47.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:41:47.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:47.994 | INFO | Uploaded 36GiB to GPU in 2.48341s (14.4962GiB/s) | 20260507 22:41:47.994 | INFO | Downloaded 34GiB to GPU in 2.44699s (13.8946GiB/s) | 20260507 22:41:48.091 | INFO | Starting a round of Baz IO | 20260507 22:41:51.561 | INFO | Populating RTMetrics report | 20260507 22:41:51.562 | INFO | RTMetrics report populated | 20260507 22:41:51.562 | INFO | Opening tmp RTMetrics json file | 20260507 22:41:51.569 | INFO | Serializing RTMetrics to json | 20260507 22:41:51.585 | INFO | Serialization done, swapping tmp file now... | 20260507 22:41:51.586 | INFO | RTMetrics done | 20260507 22:41:51.664 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.572 seconds | 20260507 22:41:51.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.98% / 0.01% / 57.01% | 20260507 22:41:51.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35631% / 1.19209e-05% / 5.16574e-05% | 20260507 22:41:51.664 | INFO | Baz IO overall: | 20260507 22:41:51.664 | INFO | BytesWritten : 1442458191537 | 20260507 22:41:51.664 | INFO | NumEvents : 698780633328 | 20260507 22:41:53.136 | INFO | Analyzing chunk frames = [3166208,3166720) | 20260507 22:41:53.136 | INFO | MemInfo: { | 20260507 22:41:53.136 | INFO | "gpu" : " 17 GB", | 20260507 22:41:53.136 | INFO | "huge" : "", | 20260507 22:41:53.136 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:53.136 | INFO | "malloc" : " 14 GB", | 20260507 22:41:53.136 | INFO | "pinned" : " 29 GB" | 20260507 22:41:53.136 | INFO | } | 20260507 22:41:53.136 | INFO | Perf report for the last 5.1422 seconds: | 20260507 22:41:53.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:53.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:53.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:53.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:53.136 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 22:41:53.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:53.136 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 22:41:53.136 | INFO | PRE_HQ : 65.8%, 1.7, 57.2ms, 0 | 20260507 22:41:53.136 | INFO | BAZWRITER : 69.1%, 3.9, 140.3ms, 0 | 20260507 22:41:53.136 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 22:41:53.136 | INFO | Uploaded 36GiB to GPU in 2.56673s (14.0257GiB/s) | 20260507 22:41:53.136 | INFO | Downloaded 38GiB to GPU in 2.68705s (14.1419GiB/s) | 20260507 22:41:58.272 | INFO | Analyzing chunk frames = [3166720,3167232) | 20260507 22:41:58.272 | INFO | MemInfo: { | 20260507 22:41:58.272 | INFO | "gpu" : " 17 GB", | 20260507 22:41:58.272 | INFO | "huge" : "", | 20260507 22:41:58.272 | INFO | "huge_shared" : " 29 GB", | 20260507 22:41:58.272 | INFO | "malloc" : " 14 GB", | 20260507 22:41:58.272 | INFO | "pinned" : " 29 GB" | 20260507 22:41:58.272 | INFO | } | 20260507 22:41:58.272 | INFO | Perf report for the last 5.13623 seconds: | 20260507 22:41:58.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:41:58.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.272 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 22:41:58.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.272 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:41:58.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:41:58.273 | INFO | Uploaded 36GiB to GPU in 2.58761s (13.9125GiB/s) | 20260507 22:41:58.273 | INFO | Downloaded 34GiB to GPU in 2.4791s (13.7146GiB/s) | 20260507 22:42:03.412 | INFO | Analyzing chunk frames = [3167232,3167744) | 20260507 22:42:03.412 | INFO | MemInfo: { | 20260507 22:42:03.412 | INFO | "gpu" : " 17 GB", | 20260507 22:42:03.412 | INFO | "huge" : "", | 20260507 22:42:03.412 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:03.412 | INFO | "malloc" : " 14 GB", | 20260507 22:42:03.412 | INFO | "pinned" : " 29 GB" | 20260507 22:42:03.412 | INFO | } | 20260507 22:42:03.412 | INFO | Perf report for the last 5.13996 seconds: | 20260507 22:42:03.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:03.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 22:42:03.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:42:03.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:03.412 | INFO | Uploaded 36GiB to GPU in 2.57707s (13.9694GiB/s) | 20260507 22:42:03.412 | INFO | Downloaded 34GiB to GPU in 2.46416s (13.7978GiB/s) | 20260507 22:42:08.549 | INFO | Analyzing chunk frames = [3167744,3168256) | 20260507 22:42:08.549 | INFO | MemInfo: { | 20260507 22:42:08.549 | INFO | "gpu" : " 17 GB", | 20260507 22:42:08.549 | INFO | "huge" : "", | 20260507 22:42:08.549 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:08.549 | INFO | "malloc" : " 14 GB", | 20260507 22:42:08.549 | INFO | "pinned" : " 29 GB" | 20260507 22:42:08.549 | INFO | } | 20260507 22:42:08.549 | INFO | Perf report for the last 5.13678 seconds: | 20260507 22:42:08.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:08.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:42:08.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:42:08.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:08.549 | INFO | Uploaded 36GiB to GPU in 2.48099s (14.5103GiB/s) | 20260507 22:42:08.549 | INFO | Downloaded 34GiB to GPU in 2.37993s (14.2862GiB/s) | 20260507 22:42:13.689 | INFO | Analyzing chunk frames = [3168256,3168768) | 20260507 22:42:13.689 | INFO | MemInfo: { | 20260507 22:42:13.689 | INFO | "gpu" : " 17 GB", | 20260507 22:42:13.689 | INFO | "huge" : "", | 20260507 22:42:13.689 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:13.689 | INFO | "malloc" : " 14 GB", | 20260507 22:42:13.689 | INFO | "pinned" : " 29 GB" | 20260507 22:42:13.689 | INFO | } | 20260507 22:42:13.689 | INFO | Perf report for the last 5.14001 seconds: | 20260507 22:42:13.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:13.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 22:42:13.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:42:13.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:13.689 | INFO | Uploaded 36GiB to GPU in 2.37592s (15.152GiB/s) | 20260507 22:42:13.689 | INFO | Downloaded 34GiB to GPU in 2.49711s (13.6157GiB/s) | 20260507 22:42:18.829 | INFO | Analyzing chunk frames = [3168768,3169280) | 20260507 22:42:18.829 | INFO | MemInfo: { | 20260507 22:42:18.829 | INFO | "gpu" : " 17 GB", | 20260507 22:42:18.829 | INFO | "huge" : "", | 20260507 22:42:18.829 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:18.830 | INFO | "malloc" : " 14 GB", | 20260507 22:42:18.830 | INFO | "pinned" : " 29 GB" | 20260507 22:42:18.830 | INFO | } | 20260507 22:42:18.830 | INFO | Perf report for the last 5.14083 seconds: | 20260507 22:42:18.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:18.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 22:42:18.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260507 22:42:18.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:18.830 | INFO | Uploaded 36GiB to GPU in 2.52039s (14.2835GiB/s) | 20260507 22:42:18.830 | INFO | Downloaded 34GiB to GPU in 2.43833s (13.944GiB/s) | 20260507 22:42:23.969 | INFO | Analyzing chunk frames = [3169280,3169792) | 20260507 22:42:23.969 | INFO | MemInfo: { | 20260507 22:42:23.969 | INFO | "gpu" : " 17 GB", | 20260507 22:42:23.969 | INFO | "huge" : "", | 20260507 22:42:23.969 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:23.969 | INFO | "malloc" : " 14 GB", | 20260507 22:42:23.969 | INFO | "pinned" : " 29 GB" | 20260507 22:42:23.969 | INFO | } | 20260507 22:42:23.969 | INFO | Perf report for the last 5.13962 seconds: | 20260507 22:42:23.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:23.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.969 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:42:23.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.970 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 22:42:23.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:23.970 | INFO | Uploaded 36GiB to GPU in 2.5182s (14.2959GiB/s) | 20260507 22:42:23.970 | INFO | Downloaded 34GiB to GPU in 2.45056s (13.8744GiB/s) | 20260507 22:42:29.111 | INFO | Analyzing chunk frames = [3169792,3170304) | 20260507 22:42:29.111 | INFO | MemInfo: { | 20260507 22:42:29.111 | INFO | "gpu" : " 17 GB", | 20260507 22:42:29.111 | INFO | "huge" : "", | 20260507 22:42:29.111 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:29.111 | INFO | "malloc" : " 14 GB", | 20260507 22:42:29.111 | INFO | "pinned" : " 29 GB" | 20260507 22:42:29.111 | INFO | } | 20260507 22:42:29.112 | INFO | Perf report for the last 5.14227 seconds: | 20260507 22:42:29.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:29.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 22:42:29.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:42:29.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:29.112 | INFO | Uploaded 36GiB to GPU in 2.51621s (14.3072GiB/s) | 20260507 22:42:29.112 | INFO | Downloaded 34GiB to GPU in 2.45016s (13.8766GiB/s) | 20260507 22:42:29.112 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:42:29.112 | INFO | Total mem usage: 16.7 GiB | 20260507 22:42:29.112 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:42:29.112 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.26%) | 20260507 22:42:29.112 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:42:29.112 | INFO | High Water Breakdown: | 20260507 22:42:29.112 | INFO | 60.9% : 9.56 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:42:29.112 | INFO | 39% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:42:29.112 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:42:29.112 | INFO | ------------------------------------------ | 20260507 22:42:29.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:42:29.112 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:42:29.112 | INFO | Total mem usage: 26.5 GiB | 20260507 22:42:29.113 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:42:29.113 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:42:29.113 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:42:29.113 | INFO | High Water Breakdown: | 20260507 22:42:29.113 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:42:29.113 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:42:29.113 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:42:29.113 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:42:29.113 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:42:29.113 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:42:29.113 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:42:29.113 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:42:29.113 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:42:29.113 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:42:29.113 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:42:29.113 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:42:29.113 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:42:29.113 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:42:29.113 | INFO | ------------------------------------------ | 20260507 22:42:29.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:42:29.113 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:42:29.113 | INFO | Total mem usage: 27.8 GiB | 20260507 22:42:29.113 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:42:29.113 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:42:29.113 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:42:29.113 | INFO | High Water Breakdown: | 20260507 22:42:29.113 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:42:29.113 | INFO | ------------------------------------------ | 20260507 22:42:29.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:42:29.113 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:42:29.113 | INFO | Total mem usage: 15.7 GiB | 20260507 22:42:29.113 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:42:29.113 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:42:29.113 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:42:29.113 | INFO | High Water Breakdown: | 20260507 22:42:29.113 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:42:29.113 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:42:29.113 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:42:29.113 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:42:29.113 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:42:29.113 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:42:29.113 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:42:29.113 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:42:29.113 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:42:29.114 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:42:29.114 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:42:29.114 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:42:29.114 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:42:29.114 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:42:29.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:42:29.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:42:29.114 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:42:29.114 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:42:29.114 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:42:29.114 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:42:29.114 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:42:29.114 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:42:29.114 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:42:29.114 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:42:29.114 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:42:29.114 | INFO | ------------------------------------------ | 20260507 22:42:29.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:42:29.114 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:42:29.114 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:42:29.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:42:29.114 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.42e+01 / 1.58e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.02e-01, 1.14e-02 / 4.64e-03 | 20260507 22:42:29.114 | INFO | PerfReport: 12.17%, Baseline -- 4.01e+00 / 4.52e+00, 6.03e-01 / 1.75e-01 | 20260507 22:42:29.114 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.50e+00, 2.09e-01 / 1.18e-01 | 20260507 22:42:29.114 | INFO | PerfReport: 15.90%, DME -- 4.19e+01 / 4.54e+01, 4.00e+00 / 1.43e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.75e+00 / 1.87e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.34e-01 / 2.00e-01 | 20260507 22:42:29.114 | INFO | PerfReport: 3.48%, Metrics -- 1.15e+00 / 1.35e+00, 2.95e-01 / 1.74e-01 | 20260507 22:42:29.114 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.55e-01 / 3.85e-01, 5.77e-02 / 2.69e-02 | 20260507 22:42:29.114 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:42:29.114 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:42:29.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:42:29.114 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 24.37%, Upload -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 30.87%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 2.01%, LockUpload -- 2.15e+00 / 0.00e+00, 5.20e+00 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 15.89%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 22:42:29.114 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260507 22:42:29.228 | INFO | Starting a round of Baz IO | 20260507 22:42:32.699 | INFO | Populating RTMetrics report | 20260507 22:42:32.700 | INFO | RTMetrics report populated | 20260507 22:42:32.700 | INFO | Opening tmp RTMetrics json file | 20260507 22:42:32.727 | INFO | Serializing RTMetrics to json | 20260507 22:42:32.729 | INFO | Serialization done, swapping tmp file now... | 20260507 22:42:32.730 | INFO | RTMetrics done | 20260507 22:42:32.811 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.582 seconds | 20260507 22:42:32.811 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.98% / 0.01% / 57.02% | 20260507 22:42:32.811 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35527% / 1.19209e-05% / 8.74201e-05% | 20260507 22:42:32.811 | INFO | Baz IO overall: | 20260507 22:42:32.811 | INFO | BytesWritten : 1444316966566 | 20260507 22:42:32.811 | INFO | NumEvents : 699677708387 | 20260507 22:42:34.247 | INFO | Analyzing chunk frames = [3170304,3170816) | 20260507 22:42:34.247 | INFO | MemInfo: { | 20260507 22:42:34.247 | INFO | "gpu" : " 17 GB", | 20260507 22:42:34.247 | INFO | "huge" : "", | 20260507 22:42:34.247 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:34.247 | INFO | "malloc" : " 14 GB", | 20260507 22:42:34.247 | INFO | "pinned" : " 29 GB" | 20260507 22:42:34.247 | INFO | } | 20260507 22:42:34.247 | INFO | Perf report for the last 5.13547 seconds: | 20260507 22:42:34.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:34.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:34.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:34.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:34.247 | INFO | ANALYSIS : 69.1%, 2.9, 104.6ms, 0 | 20260507 22:42:34.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:34.247 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260507 22:42:34.247 | INFO | PRE_HQ : 68.2%, 1.8, 64.6ms, 0 | 20260507 22:42:34.247 | INFO | BAZWRITER : 69.3%, 4.4, 156.4ms, 0 | 20260507 22:42:34.247 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 22:42:34.247 | INFO | Uploaded 36GiB to GPU in 2.62609s (13.7086GiB/s) | 20260507 22:42:34.247 | INFO | Downloaded 38GiB to GPU in 2.71588s (13.9918GiB/s) | 20260507 22:42:39.393 | INFO | Analyzing chunk frames = [3170816,3171328) | 20260507 22:42:39.393 | INFO | MemInfo: { | 20260507 22:42:39.393 | INFO | "gpu" : " 17 GB", | 20260507 22:42:39.393 | INFO | "huge" : "", | 20260507 22:42:39.393 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:39.393 | INFO | "malloc" : " 14 GB", | 20260507 22:42:39.393 | INFO | "pinned" : " 29 GB" | 20260507 22:42:39.393 | INFO | } | 20260507 22:42:39.393 | INFO | Perf report for the last 5.14606 seconds: | 20260507 22:42:39.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:39.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.393 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260507 22:42:39.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.393 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:42:39.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:39.394 | INFO | Uploaded 36GiB to GPU in 2.52847s (14.2378GiB/s) | 20260507 22:42:39.394 | INFO | Downloaded 34GiB to GPU in 2.46628s (13.786GiB/s) | 20260507 22:42:44.525 | INFO | Analyzing chunk frames = [3171328,3171840) | 20260507 22:42:44.525 | INFO | MemInfo: { | 20260507 22:42:44.525 | INFO | "gpu" : " 17 GB", | 20260507 22:42:44.525 | INFO | "huge" : "", | 20260507 22:42:44.525 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:44.525 | INFO | "malloc" : " 14 GB", | 20260507 22:42:44.525 | INFO | "pinned" : " 29 GB" | 20260507 22:42:44.525 | INFO | } | 20260507 22:42:44.525 | INFO | Perf report for the last 5.1316 seconds: | 20260507 22:42:44.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:44.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 22:42:44.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:42:44.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:44.525 | INFO | Uploaded 36GiB to GPU in 2.48926s (14.4621GiB/s) | 20260507 22:42:44.525 | INFO | Downloaded 34GiB to GPU in 2.48084s (13.705GiB/s) | 20260507 22:42:49.661 | INFO | Analyzing chunk frames = [3171840,3172352) | 20260507 22:42:49.661 | INFO | MemInfo: { | 20260507 22:42:49.661 | INFO | "gpu" : " 17 GB", | 20260507 22:42:49.661 | INFO | "huge" : "", | 20260507 22:42:49.661 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:49.661 | INFO | "malloc" : " 14 GB", | 20260507 22:42:49.661 | INFO | "pinned" : " 29 GB" | 20260507 22:42:49.661 | INFO | } | 20260507 22:42:49.661 | INFO | Perf report for the last 5.13663 seconds: | 20260507 22:42:49.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:49.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.661 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260507 22:42:49.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.662 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:42:49.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:49.662 | INFO | Uploaded 36GiB to GPU in 2.46162s (14.6245GiB/s) | 20260507 22:42:49.662 | INFO | Downloaded 34GiB to GPU in 2.39491s (14.1968GiB/s) | 20260507 22:42:54.803 | INFO | Analyzing chunk frames = [3172352,3172864) | 20260507 22:42:54.803 | INFO | MemInfo: { | 20260507 22:42:54.803 | INFO | "gpu" : " 17 GB", | 20260507 22:42:54.803 | INFO | "huge" : "", | 20260507 22:42:54.803 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:54.803 | INFO | "malloc" : " 14 GB", | 20260507 22:42:54.803 | INFO | "pinned" : " 29 GB" | 20260507 22:42:54.803 | INFO | } | 20260507 22:42:54.803 | INFO | Perf report for the last 5.14164 seconds: | 20260507 22:42:54.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:54.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:42:54.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:42:54.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:54.803 | INFO | Uploaded 36GiB to GPU in 2.3512s (15.3114GiB/s) | 20260507 22:42:54.803 | INFO | Downloaded 34GiB to GPU in 2.49866s (13.6073GiB/s) | 20260507 22:42:59.945 | INFO | Analyzing chunk frames = [3172864,3173376) | 20260507 22:42:59.945 | INFO | MemInfo: { | 20260507 22:42:59.945 | INFO | "gpu" : " 17 GB", | 20260507 22:42:59.945 | INFO | "huge" : "", | 20260507 22:42:59.945 | INFO | "huge_shared" : " 29 GB", | 20260507 22:42:59.945 | INFO | "malloc" : " 14 GB", | 20260507 22:42:59.945 | INFO | "pinned" : " 29 GB" | 20260507 22:42:59.945 | INFO | } | 20260507 22:42:59.946 | INFO | Perf report for the last 5.14244 seconds: | 20260507 22:42:59.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:42:59.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | ANALYSIS : 66.8%, 3.0, 104.5ms, 0 | 20260507 22:42:59.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:42:59.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:42:59.946 | INFO | Uploaded 36GiB to GPU in 2.5195s (14.2885GiB/s) | 20260507 22:42:59.946 | INFO | Downloaded 34GiB to GPU in 2.42323s (14.0308GiB/s) | 20260507 22:43:05.087 | INFO | Analyzing chunk frames = [3173376,3173888) | 20260507 22:43:05.087 | INFO | MemInfo: { | 20260507 22:43:05.087 | INFO | "gpu" : " 17 GB", | 20260507 22:43:05.087 | INFO | "huge" : "", | 20260507 22:43:05.087 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:05.087 | INFO | "malloc" : " 14 GB", | 20260507 22:43:05.087 | INFO | "pinned" : " 29 GB" | 20260507 22:43:05.087 | INFO | } | 20260507 22:43:05.088 | INFO | Perf report for the last 5.14199 seconds: | 20260507 22:43:05.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:05.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:43:05.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 22:43:05.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:05.088 | INFO | Uploaded 36GiB to GPU in 2.53543s (14.1988GiB/s) | 20260507 22:43:05.088 | INFO | Downloaded 34GiB to GPU in 2.4566s (13.8403GiB/s) | 20260507 22:43:10.223 | INFO | Analyzing chunk frames = [3173888,3174400) | 20260507 22:43:10.223 | INFO | MemInfo: { | 20260507 22:43:10.223 | INFO | "gpu" : " 17 GB", | 20260507 22:43:10.223 | INFO | "huge" : "", | 20260507 22:43:10.223 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:10.223 | INFO | "malloc" : " 14 GB", | 20260507 22:43:10.223 | INFO | "pinned" : " 29 GB" | 20260507 22:43:10.223 | INFO | } | 20260507 22:43:10.224 | INFO | Perf report for the last 5.13596 seconds: | 20260507 22:43:10.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:10.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:43:10.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260507 22:43:10.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:10.224 | INFO | Uploaded 36GiB to GPU in 2.48672s (14.4769GiB/s) | 20260507 22:43:10.224 | INFO | Downloaded 34GiB to GPU in 2.43322s (13.9733GiB/s) | 20260507 22:43:10.339 | INFO | Starting a round of Baz IO | 20260507 22:43:13.810 | INFO | Populating RTMetrics report | 20260507 22:43:13.811 | INFO | RTMetrics report populated | 20260507 22:43:13.811 | INFO | Opening tmp RTMetrics json file | 20260507 22:43:13.812 | INFO | Serializing RTMetrics to json | 20260507 22:43:13.815 | INFO | Serialization done, swapping tmp file now... | 20260507 22:43:13.815 | INFO | RTMetrics done | 20260507 22:43:13.905 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.566 seconds | 20260507 22:43:13.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.97% / 0.01% / 57.02% | 20260507 22:43:13.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35094% / 1.19209e-05% / 5.5631e-05% | 20260507 22:43:13.905 | INFO | Baz IO overall: | 20260507 22:43:13.905 | INFO | BytesWritten : 1446175215063 | 20260507 22:43:13.905 | INFO | NumEvents : 700574504594 | 20260507 22:43:15.361 | INFO | Analyzing chunk frames = [3174400,3174912) | 20260507 22:43:15.361 | INFO | MemInfo: { | 20260507 22:43:15.361 | INFO | "gpu" : " 17 GB", | 20260507 22:43:15.361 | INFO | "huge" : "", | 20260507 22:43:15.361 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:15.361 | INFO | "malloc" : " 14 GB", | 20260507 22:43:15.361 | INFO | "pinned" : " 29 GB" | 20260507 22:43:15.361 | INFO | } | 20260507 22:43:15.362 | INFO | Perf report for the last 5.13803 seconds: | 20260507 22:43:15.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:15.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:15.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:15.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:15.362 | INFO | ANALYSIS : 69.4%, 2.9, 104.7ms, 0 | 20260507 22:43:15.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:15.362 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 22:43:15.362 | INFO | PRE_HQ : 68.2%, 1.8, 64.9ms, 0 | 20260507 22:43:15.362 | INFO | BAZWRITER : 69.0%, 4.3, 153.7ms, 0 | 20260507 22:43:15.362 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 22:43:15.362 | INFO | Uploaded 36GiB to GPU in 2.52061s (14.2822GiB/s) | 20260507 22:43:15.362 | INFO | Downloaded 38GiB to GPU in 2.69832s (14.0828GiB/s) | 20260507 22:43:20.501 | INFO | Analyzing chunk frames = [3174912,3175424) | 20260507 22:43:20.501 | INFO | MemInfo: { | 20260507 22:43:20.501 | INFO | "gpu" : " 17 GB", | 20260507 22:43:20.501 | INFO | "huge" : "", | 20260507 22:43:20.501 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:20.501 | INFO | "malloc" : " 14 GB", | 20260507 22:43:20.501 | INFO | "pinned" : " 29 GB" | 20260507 22:43:20.502 | INFO | } | 20260507 22:43:20.502 | INFO | Perf report for the last 5.14008 seconds: | 20260507 22:43:20.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:20.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:43:20.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:43:20.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:20.502 | INFO | Uploaded 36GiB to GPU in 2.5551s (14.0895GiB/s) | 20260507 22:43:20.502 | INFO | Downloaded 34GiB to GPU in 2.48669s (13.6728GiB/s) | 20260507 22:43:25.643 | INFO | Analyzing chunk frames = [3175424,3175936) | 20260507 22:43:25.643 | INFO | MemInfo: { | 20260507 22:43:25.643 | INFO | "gpu" : " 17 GB", | 20260507 22:43:25.643 | INFO | "huge" : "", | 20260507 22:43:25.643 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:25.643 | INFO | "malloc" : " 14 GB", | 20260507 22:43:25.643 | INFO | "pinned" : " 29 GB" | 20260507 22:43:25.643 | INFO | } | 20260507 22:43:25.644 | INFO | Perf report for the last 5.14187 seconds: | 20260507 22:43:25.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:25.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:43:25.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:43:25.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:25.644 | INFO | Uploaded 36GiB to GPU in 2.51155s (14.3338GiB/s) | 20260507 22:43:25.644 | INFO | Downloaded 34GiB to GPU in 2.45675s (13.8394GiB/s) | 20260507 22:43:30.777 | INFO | Analyzing chunk frames = [3175936,3176448) | 20260507 22:43:30.777 | INFO | MemInfo: { | 20260507 22:43:30.777 | INFO | "gpu" : " 17 GB", | 20260507 22:43:30.777 | INFO | "huge" : "", | 20260507 22:43:30.777 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:30.777 | INFO | "malloc" : " 14 GB", | 20260507 22:43:30.777 | INFO | "pinned" : " 29 GB" | 20260507 22:43:30.777 | INFO | } | 20260507 22:43:30.777 | INFO | Perf report for the last 5.1336 seconds: | 20260507 22:43:30.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:30.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.777 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260507 22:43:30.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.777 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:43:30.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:30.778 | INFO | Uploaded 36GiB to GPU in 2.50546s (14.3686GiB/s) | 20260507 22:43:30.778 | INFO | Downloaded 34GiB to GPU in 2.49067s (13.6509GiB/s) | 20260507 22:43:35.918 | INFO | Analyzing chunk frames = [3176448,3176960) | 20260507 22:43:35.918 | INFO | MemInfo: { | 20260507 22:43:35.918 | INFO | "gpu" : " 17 GB", | 20260507 22:43:35.918 | INFO | "huge" : "", | 20260507 22:43:35.918 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:35.918 | INFO | "malloc" : " 14 GB", | 20260507 22:43:35.918 | INFO | "pinned" : " 29 GB" | 20260507 22:43:35.918 | INFO | } | 20260507 22:43:35.918 | INFO | Perf report for the last 5.14125 seconds: | 20260507 22:43:35.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:35.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.918 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:43:35.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.919 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:43:35.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:35.919 | INFO | Uploaded 36GiB to GPU in 2.36456s (15.2248GiB/s) | 20260507 22:43:35.919 | INFO | Downloaded 34GiB to GPU in 2.50316s (13.5828GiB/s) | 20260507 22:43:41.060 | INFO | Analyzing chunk frames = [3176960,3177472) | 20260507 22:43:41.060 | INFO | MemInfo: { | 20260507 22:43:41.060 | INFO | "gpu" : " 17 GB", | 20260507 22:43:41.060 | INFO | "huge" : "", | 20260507 22:43:41.060 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:41.060 | INFO | "malloc" : " 14 GB", | 20260507 22:43:41.060 | INFO | "pinned" : " 29 GB" | 20260507 22:43:41.060 | INFO | } | 20260507 22:43:41.061 | INFO | Perf report for the last 5.14208 seconds: | 20260507 22:43:41.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:41.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:43:41.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 22:43:41.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:41.061 | INFO | Uploaded 36GiB to GPU in 2.51223s (14.3299GiB/s) | 20260507 22:43:41.061 | INFO | Downloaded 34GiB to GPU in 2.41875s (14.0569GiB/s) | 20260507 22:43:46.204 | INFO | Analyzing chunk frames = [3177472,3177984) | 20260507 22:43:46.204 | INFO | MemInfo: { | 20260507 22:43:46.204 | INFO | "gpu" : " 17 GB", | 20260507 22:43:46.204 | INFO | "huge" : "", | 20260507 22:43:46.204 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:46.204 | INFO | "malloc" : " 14 GB", | 20260507 22:43:46.204 | INFO | "pinned" : " 29 GB" | 20260507 22:43:46.204 | INFO | } | 20260507 22:43:46.204 | INFO | Perf report for the last 5.1437 seconds: | 20260507 22:43:46.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:46.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.204 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 22:43:46.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.204 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 22:43:46.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:46.205 | INFO | Uploaded 36GiB to GPU in 2.54262s (14.1586GiB/s) | 20260507 22:43:46.205 | INFO | Downloaded 34GiB to GPU in 2.43641s (13.955GiB/s) | 20260507 22:43:51.344 | INFO | Analyzing chunk frames = [3177984,3178496) | 20260507 22:43:51.344 | INFO | MemInfo: { | 20260507 22:43:51.344 | INFO | "gpu" : " 17 GB", | 20260507 22:43:51.344 | INFO | "huge" : "", | 20260507 22:43:51.344 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:51.344 | INFO | "malloc" : " 14 GB", | 20260507 22:43:51.344 | INFO | "pinned" : " 29 GB" | 20260507 22:43:51.344 | INFO | } | 20260507 22:43:51.344 | INFO | Perf report for the last 5.1402 seconds: | 20260507 22:43:51.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:51.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.345 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 22:43:51.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.345 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:43:51.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:51.345 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:43:51.345 | INFO | Total mem usage: 16.7 GiB | 20260507 22:43:51.345 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:43:51.345 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.21%) | 20260507 22:43:51.345 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:43:51.345 | INFO | High Water Breakdown: | 20260507 22:43:51.345 | INFO | 60.9% : 9.57 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:43:51.345 | INFO | 39% : 6.12 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:43:51.345 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:43:51.345 | INFO | ------------------------------------------ | 20260507 22:43:51.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:43:51.345 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:43:51.345 | INFO | Total mem usage: 26.5 GiB | 20260507 22:43:51.345 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:43:51.345 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:43:51.345 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:43:51.345 | INFO | High Water Breakdown: | 20260507 22:43:51.345 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:43:51.345 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:43:51.345 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:43:51.345 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:43:51.345 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:43:51.345 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:43:51.345 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:43:51.345 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:43:51.345 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:43:51.345 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:43:51.345 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:43:51.345 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:43:51.345 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:43:51.345 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:43:51.345 | INFO | ------------------------------------------ | 20260507 22:43:51.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:43:51.345 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:43:51.345 | INFO | Total mem usage: 27.8 GiB | 20260507 22:43:51.345 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:43:51.345 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:43:51.345 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:43:51.345 | INFO | High Water Breakdown: | 20260507 22:43:51.345 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:43:51.345 | INFO | ------------------------------------------ | 20260507 22:43:51.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:43:51.345 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:43:51.345 | INFO | Total mem usage: 15.7 GiB | 20260507 22:43:51.345 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:43:51.345 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:43:51.345 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:43:51.345 | INFO | High Water Breakdown: | 20260507 22:43:51.345 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:43:51.345 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:43:51.345 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:43:51.345 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:43:51.345 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:43:51.345 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:43:51.345 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:43:51.345 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:43:51.345 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:43:51.345 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:43:51.345 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:43:51.345 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:43:51.345 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:43:51.345 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:43:51.345 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:43:51.345 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:43:51.345 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:43:51.345 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:43:51.345 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:43:51.345 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:43:51.345 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:43:51.345 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:43:51.345 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:43:51.345 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:43:51.345 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:43:51.345 | INFO | ------------------------------------------ | 20260507 22:43:51.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:43:51.345 | INFO | Uploaded 36GiB to GPU in 2.52808s (14.2401GiB/s) | 20260507 22:43:51.345 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:43:51.346 | INFO | Downloaded 34GiB to GPU in 2.43798s (13.946GiB/s) | 20260507 22:43:51.346 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:43:51.346 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:43:51.346 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.43e+01 / 1.54e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.28e-02 / 5.28e-03 | 20260507 22:43:51.346 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.45e+00, 5.93e-01 / 2.49e-01 | 20260507 22:43:51.346 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.08e-01 / 1.03e-01 | 20260507 22:43:51.346 | INFO | PerfReport: 15.98%, DME -- 4.21e+01 / 4.57e+01, 3.83e+00 / 1.53e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.70e+00 / 1.54e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.46e+00 / 2.68e+00, 4.37e-01 / 2.75e-01 | 20260507 22:43:51.346 | INFO | PerfReport: 3.45%, Metrics -- 1.14e+00 / 1.30e+00, 2.85e-01 / 1.50e-01 | 20260507 22:43:51.346 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.54e-01 / 3.84e-01, 5.90e-02 / 3.78e-02 | 20260507 22:43:51.346 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:43:51.346 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:43:51.346 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:43:51.346 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 30.91%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 2.08%, LockUpload -- 2.21e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 2.63%, LockDownload -- 2.80e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:43:51.346 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260507 22:43:51.444 | INFO | Starting a round of Baz IO | 20260507 22:43:54.909 | INFO | Populating RTMetrics report | 20260507 22:43:54.910 | INFO | RTMetrics report populated | 20260507 22:43:54.910 | INFO | Opening tmp RTMetrics json file | 20260507 22:43:54.965 | INFO | Serializing RTMetrics to json | 20260507 22:43:55.016 | INFO | Serialization done, swapping tmp file now... | 20260507 22:43:55.020 | INFO | RTMetrics done | 20260507 22:43:55.053 | INFO | Finished a round of Baz IO: Wrote 1.73 GiB in 3.609 seconds | 20260507 22:43:55.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.96% / 0.01% / 57.03% | 20260507 22:43:55.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34743% / 1.19209e-05% / 5.5631e-05% | 20260507 22:43:55.053 | INFO | Baz IO overall: | 20260507 22:43:55.053 | INFO | BytesWritten : 1448032280809 | 20260507 22:43:55.053 | INFO | NumEvents : 701470480650 | 20260507 22:43:56.476 | INFO | Analyzing chunk frames = [3178496,3179008) | 20260507 22:43:56.476 | INFO | MemInfo: { | 20260507 22:43:56.476 | INFO | "gpu" : " 17 GB", | 20260507 22:43:56.476 | INFO | "huge" : "", | 20260507 22:43:56.476 | INFO | "huge_shared" : " 29 GB", | 20260507 22:43:56.476 | INFO | "malloc" : " 14 GB", | 20260507 22:43:56.476 | INFO | "pinned" : " 29 GB" | 20260507 22:43:56.476 | INFO | } | 20260507 22:43:56.476 | INFO | Perf report for the last 5.1316 seconds: | 20260507 22:43:56.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:43:56.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:56.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:56.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:56.476 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 22:43:56.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:43:56.476 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 22:43:56.476 | INFO | PRE_HQ : 66.0%, 1.7, 57.3ms, 0 | 20260507 22:43:56.476 | INFO | BAZWRITER : 69.9%, 4.0, 142.2ms, 0 | 20260507 22:43:56.476 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260507 22:43:56.476 | INFO | Uploaded 36GiB to GPU in 2.56453s (14.0376GiB/s) | 20260507 22:43:56.477 | INFO | Downloaded 38GiB to GPU in 2.69055s (14.1235GiB/s) | 20260507 22:44:01.613 | INFO | Analyzing chunk frames = [3179008,3179520) | 20260507 22:44:01.613 | INFO | MemInfo: { | 20260507 22:44:01.613 | INFO | "gpu" : " 17 GB", | 20260507 22:44:01.613 | INFO | "huge" : "", | 20260507 22:44:01.613 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:01.613 | INFO | "malloc" : " 14 GB", | 20260507 22:44:01.613 | INFO | "pinned" : " 29 GB" | 20260507 22:44:01.613 | INFO | } | 20260507 22:44:01.613 | INFO | Perf report for the last 5.13726 seconds: | 20260507 22:44:01.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:01.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.613 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 22:44:01.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.613 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:44:01.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:01.614 | INFO | Uploaded 36GiB to GPU in 2.58216s (13.9418GiB/s) | 20260507 22:44:01.614 | INFO | Downloaded 34GiB to GPU in 2.47537s (13.7353GiB/s) | 20260507 22:44:06.754 | INFO | Analyzing chunk frames = [3179520,3180032) | 20260507 22:44:06.754 | INFO | MemInfo: { | 20260507 22:44:06.754 | INFO | "gpu" : " 17 GB", | 20260507 22:44:06.754 | INFO | "huge" : "", | 20260507 22:44:06.754 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:06.754 | INFO | "malloc" : " 14 GB", | 20260507 22:44:06.754 | INFO | "pinned" : " 29 GB" | 20260507 22:44:06.755 | INFO | } | 20260507 22:44:06.755 | INFO | Perf report for the last 5.14137 seconds: | 20260507 22:44:06.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:06.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:44:06.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:44:06.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:06.755 | INFO | Uploaded 36GiB to GPU in 2.5714s (14.0001GiB/s) | 20260507 22:44:06.755 | INFO | Downloaded 34GiB to GPU in 2.46381s (13.7998GiB/s) | 20260507 22:44:11.896 | INFO | Analyzing chunk frames = [3180032,3180544) | 20260507 22:44:11.896 | INFO | MemInfo: { | 20260507 22:44:11.896 | INFO | "gpu" : " 17 GB", | 20260507 22:44:11.896 | INFO | "huge" : "", | 20260507 22:44:11.896 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:11.896 | INFO | "malloc" : " 14 GB", | 20260507 22:44:11.896 | INFO | "pinned" : " 29 GB" | 20260507 22:44:11.896 | INFO | } | 20260507 22:44:11.896 | INFO | Perf report for the last 5.14178 seconds: | 20260507 22:44:11.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:11.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:44:11.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:44:11.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:11.897 | INFO | Uploaded 36GiB to GPU in 2.49159s (14.4486GiB/s) | 20260507 22:44:11.897 | INFO | Downloaded 34GiB to GPU in 2.37756s (14.3004GiB/s) | 20260507 22:44:17.033 | INFO | Analyzing chunk frames = [3180544,3181056) | 20260507 22:44:17.033 | INFO | MemInfo: { | 20260507 22:44:17.033 | INFO | "gpu" : " 17 GB", | 20260507 22:44:17.033 | INFO | "huge" : "", | 20260507 22:44:17.033 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:17.033 | INFO | "malloc" : " 14 GB", | 20260507 22:44:17.033 | INFO | "pinned" : " 29 GB" | 20260507 22:44:17.033 | INFO | } | 20260507 22:44:17.033 | INFO | Perf report for the last 5.13682 seconds: | 20260507 22:44:17.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:17.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.033 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260507 22:44:17.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.033 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 22:44:17.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:17.034 | INFO | Uploaded 36GiB to GPU in 2.3562s (15.2789GiB/s) | 20260507 22:44:17.034 | INFO | Downloaded 34GiB to GPU in 2.50207s (13.5888GiB/s) | 20260507 22:44:22.175 | INFO | Analyzing chunk frames = [3181056,3181568) | 20260507 22:44:22.175 | INFO | MemInfo: { | 20260507 22:44:22.175 | INFO | "gpu" : " 17 GB", | 20260507 22:44:22.175 | INFO | "huge" : "", | 20260507 22:44:22.175 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:22.175 | INFO | "malloc" : " 14 GB", | 20260507 22:44:22.175 | INFO | "pinned" : " 29 GB" | 20260507 22:44:22.175 | INFO | } | 20260507 22:44:22.175 | INFO | Perf report for the last 5.14165 seconds: | 20260507 22:44:22.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:22.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 22:44:22.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 22:44:22.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:22.175 | INFO | Uploaded 36GiB to GPU in 2.50826s (14.3526GiB/s) | 20260507 22:44:22.175 | INFO | Downloaded 34GiB to GPU in 2.43247s (13.9776GiB/s) | 20260507 22:44:27.317 | INFO | Analyzing chunk frames = [3181568,3182080) | 20260507 22:44:27.317 | INFO | MemInfo: { | 20260507 22:44:27.317 | INFO | "gpu" : " 17 GB", | 20260507 22:44:27.317 | INFO | "huge" : "", | 20260507 22:44:27.317 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:27.317 | INFO | "malloc" : " 14 GB", | 20260507 22:44:27.317 | INFO | "pinned" : " 29 GB" | 20260507 22:44:27.317 | INFO | } | 20260507 22:44:27.317 | INFO | Perf report for the last 5.14207 seconds: | 20260507 22:44:27.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:27.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 22:44:27.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:44:27.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:27.317 | INFO | Uploaded 36GiB to GPU in 2.4888s (14.4648GiB/s) | 20260507 22:44:27.317 | INFO | Downloaded 34GiB to GPU in 2.44134s (13.9268GiB/s) | 20260507 22:44:32.452 | INFO | Analyzing chunk frames = [3182080,3182592) | 20260507 22:44:32.452 | INFO | MemInfo: { | 20260507 22:44:32.452 | INFO | "gpu" : " 17 GB", | 20260507 22:44:32.452 | INFO | "huge" : "", | 20260507 22:44:32.452 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:32.452 | INFO | "malloc" : " 14 GB", | 20260507 22:44:32.452 | INFO | "pinned" : " 29 GB" | 20260507 22:44:32.452 | INFO | } | 20260507 22:44:32.452 | INFO | Perf report for the last 5.13501 seconds: | 20260507 22:44:32.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:32.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:44:32.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:44:32.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:32.452 | INFO | Uploaded 36GiB to GPU in 2.48954s (14.4605GiB/s) | 20260507 22:44:32.452 | INFO | Downloaded 34GiB to GPU in 2.44546s (13.9033GiB/s) | 20260507 22:44:32.552 | INFO | Starting a round of Baz IO | 20260507 22:44:36.016 | INFO | Populating RTMetrics report | 20260507 22:44:36.017 | INFO | RTMetrics report populated | 20260507 22:44:36.017 | INFO | Opening tmp RTMetrics json file | 20260507 22:44:36.039 | INFO | Serializing RTMetrics to json | 20260507 22:44:36.105 | INFO | Serialization done, swapping tmp file now... | 20260507 22:44:36.145 | INFO | RTMetrics done | 20260507 22:44:36.173 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.621 seconds | 20260507 22:44:36.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.96% / 0.01% / 57.04% | 20260507 22:44:36.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34677% / 7.94729e-06% / 5.5631e-05% | 20260507 22:44:36.173 | INFO | Baz IO overall: | 20260507 22:44:36.173 | INFO | BytesWritten : 1449890770538 | 20260507 22:44:36.173 | INFO | NumEvents : 702367280877 | 20260507 22:44:37.591 | INFO | Analyzing chunk frames = [3182592,3183104) | 20260507 22:44:37.591 | INFO | MemInfo: { | 20260507 22:44:37.591 | INFO | "gpu" : " 17 GB", | 20260507 22:44:37.591 | INFO | "huge" : "", | 20260507 22:44:37.591 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:37.591 | INFO | "malloc" : " 14 GB", | 20260507 22:44:37.591 | INFO | "pinned" : " 29 GB" | 20260507 22:44:37.592 | INFO | } | 20260507 22:44:37.592 | INFO | Perf report for the last 5.13944 seconds: | 20260507 22:44:37.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:37.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:37.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:37.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:37.592 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 22:44:37.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:37.592 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 22:44:37.592 | INFO | PRE_HQ : 65.5%, 1.7, 57.4ms, 0 | 20260507 22:44:37.592 | INFO | BAZWRITER : 70.1%, 3.6, 128.8ms, 0 | 20260507 22:44:37.592 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 22:44:37.592 | INFO | Uploaded 36GiB to GPU in 2.55838s (14.0714GiB/s) | 20260507 22:44:37.592 | INFO | Downloaded 38GiB to GPU in 2.68398s (14.1581GiB/s) | 20260507 22:44:42.730 | INFO | Analyzing chunk frames = [3183104,3183616) | 20260507 22:44:42.730 | INFO | MemInfo: { | 20260507 22:44:42.730 | INFO | "gpu" : " 17 GB", | 20260507 22:44:42.730 | INFO | "huge" : "", | 20260507 22:44:42.730 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:42.730 | INFO | "malloc" : " 14 GB", | 20260507 22:44:42.730 | INFO | "pinned" : " 29 GB" | 20260507 22:44:42.730 | INFO | } | 20260507 22:44:42.730 | INFO | Perf report for the last 5.13875 seconds: | 20260507 22:44:42.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:42.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.730 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:44:42.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.730 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:44:42.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:42.731 | INFO | Uploaded 36GiB to GPU in 2.5539s (14.0961GiB/s) | 20260507 22:44:42.731 | INFO | Downloaded 34GiB to GPU in 2.49403s (13.6326GiB/s) | 20260507 22:44:47.873 | INFO | Analyzing chunk frames = [3183616,3184128) | 20260507 22:44:47.873 | INFO | MemInfo: { | 20260507 22:44:47.873 | INFO | "gpu" : " 17 GB", | 20260507 22:44:47.873 | INFO | "huge" : "", | 20260507 22:44:47.873 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:47.873 | INFO | "malloc" : " 14 GB", | 20260507 22:44:47.873 | INFO | "pinned" : " 29 GB" | 20260507 22:44:47.873 | INFO | } | 20260507 22:44:47.873 | INFO | Perf report for the last 5.14287 seconds: | 20260507 22:44:47.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:47.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260507 22:44:47.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:44:47.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:47.873 | INFO | Uploaded 36GiB to GPU in 2.53709s (14.1895GiB/s) | 20260507 22:44:47.873 | INFO | Downloaded 34GiB to GPU in 2.45104s (13.8716GiB/s) | 20260507 22:44:53.013 | INFO | Analyzing chunk frames = [3184128,3184640) | 20260507 22:44:53.013 | INFO | MemInfo: { | 20260507 22:44:53.013 | INFO | "gpu" : " 17 GB", | 20260507 22:44:53.013 | INFO | "huge" : "", | 20260507 22:44:53.013 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:53.013 | INFO | "malloc" : " 14 GB", | 20260507 22:44:53.013 | INFO | "pinned" : " 29 GB" | 20260507 22:44:53.013 | INFO | } | 20260507 22:44:53.013 | INFO | Perf report for the last 5.13965 seconds: | 20260507 22:44:53.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:53.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:44:53.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:44:53.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:53.013 | INFO | Uploaded 36GiB to GPU in 2.48575s (14.4826GiB/s) | 20260507 22:44:53.014 | INFO | Downloaded 34GiB to GPU in 2.39691s (14.1849GiB/s) | 20260507 22:44:58.154 | INFO | Analyzing chunk frames = [3184640,3185152) | 20260507 22:44:58.154 | INFO | MemInfo: { | 20260507 22:44:58.154 | INFO | "gpu" : " 17 GB", | 20260507 22:44:58.154 | INFO | "huge" : "", | 20260507 22:44:58.154 | INFO | "huge_shared" : " 29 GB", | 20260507 22:44:58.154 | INFO | "malloc" : " 14 GB", | 20260507 22:44:58.154 | INFO | "pinned" : " 29 GB" | 20260507 22:44:58.155 | INFO | } | 20260507 22:44:58.155 | INFO | Perf report for the last 5.14183 seconds: | 20260507 22:44:58.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:44:58.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 22:44:58.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:44:58.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:44:58.155 | INFO | Uploaded 36GiB to GPU in 2.39726s (15.0171GiB/s) | 20260507 22:44:58.155 | INFO | Downloaded 34GiB to GPU in 2.48581s (13.6777GiB/s) | 20260507 22:45:03.291 | INFO | Analyzing chunk frames = [3185152,3185664) | 20260507 22:45:03.291 | INFO | MemInfo: { | 20260507 22:45:03.291 | INFO | "gpu" : " 17 GB", | 20260507 22:45:03.291 | INFO | "huge" : "", | 20260507 22:45:03.291 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:03.291 | INFO | "malloc" : " 14 GB", | 20260507 22:45:03.291 | INFO | "pinned" : " 29 GB" | 20260507 22:45:03.291 | INFO | } | 20260507 22:45:03.291 | INFO | Perf report for the last 5.13643 seconds: | 20260507 22:45:03.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:03.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.291 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 22:45:03.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.291 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:45:03.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:03.292 | INFO | Uploaded 36GiB to GPU in 2.41263s (14.9215GiB/s) | 20260507 22:45:03.292 | INFO | Downloaded 34GiB to GPU in 2.39411s (14.2015GiB/s) | 20260507 22:45:08.425 | INFO | Analyzing chunk frames = [3185664,3186176) | 20260507 22:45:08.425 | INFO | MemInfo: { | 20260507 22:45:08.425 | INFO | "gpu" : " 17 GB", | 20260507 22:45:08.425 | INFO | "huge" : "", | 20260507 22:45:08.425 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:08.425 | INFO | "malloc" : " 14 GB", | 20260507 22:45:08.425 | INFO | "pinned" : " 29 GB" | 20260507 22:45:08.425 | INFO | } | 20260507 22:45:08.425 | INFO | Perf report for the last 5.13422 seconds: | 20260507 22:45:08.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:08.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.425 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 22:45:08.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.425 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:45:08.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:08.426 | INFO | Uploaded 36GiB to GPU in 2.50141s (14.3919GiB/s) | 20260507 22:45:08.426 | INFO | Downloaded 34GiB to GPU in 2.44485s (13.9068GiB/s) | 20260507 22:45:13.566 | INFO | Analyzing chunk frames = [3186176,3186688) | 20260507 22:45:13.566 | INFO | MemInfo: { | 20260507 22:45:13.566 | INFO | "gpu" : " 17 GB", | 20260507 22:45:13.566 | INFO | "huge" : "", | 20260507 22:45:13.566 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:13.566 | INFO | "malloc" : " 14 GB", | 20260507 22:45:13.566 | INFO | "pinned" : " 29 GB" | 20260507 22:45:13.566 | INFO | } | 20260507 22:45:13.566 | INFO | Perf report for the last 5.14087 seconds: | 20260507 22:45:13.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:13.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:45:13.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:45:13.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:13.566 | INFO | Uploaded 36GiB to GPU in 2.50986s (14.3434GiB/s) | 20260507 22:45:13.567 | INFO | Downloaded 34GiB to GPU in 2.44334s (13.9154GiB/s) | 20260507 22:45:13.567 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:45:13.567 | INFO | Total mem usage: 16.7 GiB | 20260507 22:45:13.567 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:45:13.567 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.23%) | 20260507 22:45:13.567 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:45:13.567 | INFO | High Water Breakdown: | 20260507 22:45:13.567 | INFO | 60.4% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:45:13.567 | INFO | 39.4% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:45:13.567 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:45:13.567 | INFO | ------------------------------------------ | 20260507 22:45:13.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:45:13.567 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:45:13.567 | INFO | Total mem usage: 26.5 GiB | 20260507 22:45:13.567 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:45:13.567 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:45:13.567 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:45:13.567 | INFO | High Water Breakdown: | 20260507 22:45:13.567 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:45:13.567 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:45:13.567 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:45:13.567 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:45:13.567 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:45:13.567 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:45:13.567 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:45:13.567 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:45:13.567 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:45:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:45:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:45:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:45:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:45:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:45:13.567 | INFO | ------------------------------------------ | 20260507 22:45:13.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:45:13.567 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:45:13.567 | INFO | Total mem usage: 27.8 GiB | 20260507 22:45:13.567 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:45:13.567 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:45:13.567 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:45:13.567 | INFO | High Water Breakdown: | 20260507 22:45:13.567 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:45:13.567 | INFO | ------------------------------------------ | 20260507 22:45:13.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:45:13.567 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:45:13.567 | INFO | Total mem usage: 15.7 GiB | 20260507 22:45:13.567 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:45:13.567 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:45:13.567 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:45:13.567 | INFO | High Water Breakdown: | 20260507 22:45:13.567 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:45:13.567 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:45:13.567 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:45:13.567 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:45:13.567 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:45:13.567 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:45:13.567 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:45:13.567 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:45:13.567 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:45:13.567 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:45:13.567 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:45:13.567 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:45:13.567 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:45:13.567 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:45:13.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:45:13.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:45:13.567 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:45:13.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:45:13.567 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:45:13.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:45:13.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:45:13.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:45:13.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:45:13.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:45:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:45:13.567 | INFO | ------------------------------------------ | 20260507 22:45:13.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:45:13.568 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:45:13.568 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:45:13.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:45:13.568 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.40e+01 / 1.45e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 0.29%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.13e-02 / 4.65e-03 | 20260507 22:45:13.568 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.51e+00, 5.98e-01 / 1.76e-01 | 20260507 22:45:13.568 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.07e-01 | 20260507 22:45:13.568 | INFO | PerfReport: 15.77%, DME -- 4.16e+01 / 4.50e+01, 3.84e+00 / 1.27e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.84e+00 / 1.68e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.43e+00 / 2.75e+00, 4.26e-01 / 2.02e-01 | 20260507 22:45:13.568 | INFO | PerfReport: 3.45%, Metrics -- 1.14e+00 / 1.36e+00, 2.87e-01 / 1.69e-01 | 20260507 22:45:13.568 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.52e-01 / 3.88e-01, 5.81e-02 / 3.30e-02 | 20260507 22:45:13.568 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:45:13.568 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:45:13.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:45:13.568 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 24.17%, Download -- 2.58e+01 / 0.00e+00, 8.55e+00 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 30.88%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 2.01%, LockUpload -- 2.14e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 2.50%, LockDownload -- 2.67e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 16.03%, LockCompute -- 1.71e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:45:13.568 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 4.76e-02 / 0.00e+00 | 20260507 22:45:13.664 | INFO | Starting a round of Baz IO | 20260507 22:45:17.139 | INFO | Populating RTMetrics report | 20260507 22:45:17.140 | INFO | RTMetrics report populated | 20260507 22:45:17.140 | INFO | Opening tmp RTMetrics json file | 20260507 22:45:17.247 | INFO | Serializing RTMetrics to json | 20260507 22:45:17.256 | INFO | Serialization done, swapping tmp file now... | 20260507 22:45:17.256 | INFO | RTMetrics done | 20260507 22:45:17.657 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.992 seconds | 20260507 22:45:17.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.95% / 0.01% / 57.04% | 20260507 22:45:17.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34126% / 7.94729e-06% / 6.35783e-05% | 20260507 22:45:17.657 | INFO | Baz IO overall: | 20260507 22:45:17.657 | INFO | BytesWritten : 1451747301998 | 20260507 22:45:17.657 | INFO | NumEvents : 703262801008 | 20260507 22:45:18.704 | INFO | Analyzing chunk frames = [3186688,3187200) | 20260507 22:45:18.704 | INFO | MemInfo: { | 20260507 22:45:18.704 | INFO | "gpu" : " 17 GB", | 20260507 22:45:18.704 | INFO | "huge" : "", | 20260507 22:45:18.704 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:18.704 | INFO | "malloc" : " 14 GB", | 20260507 22:45:18.704 | INFO | "pinned" : " 29 GB" | 20260507 22:45:18.704 | INFO | } | 20260507 22:45:18.704 | INFO | Perf report for the last 5.13743 seconds: | 20260507 22:45:18.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:18.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:18.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:18.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:18.704 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 22:45:18.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:18.704 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260507 22:45:18.704 | INFO | PRE_HQ : 65.4%, 1.7, 57.6ms, 0 | 20260507 22:45:18.704 | INFO | BAZWRITER : 77.3%, 4.3, 172.5ms, 0 | 20260507 22:45:18.704 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 22:45:18.704 | INFO | Uploaded 36GiB to GPU in 2.54201s (14.162GiB/s) | 20260507 22:45:18.704 | INFO | Downloaded 38GiB to GPU in 2.69077s (14.1223GiB/s) | 20260507 22:45:23.844 | INFO | Analyzing chunk frames = [3187200,3187712) | 20260507 22:45:23.844 | INFO | MemInfo: { | 20260507 22:45:23.844 | INFO | "gpu" : " 17 GB", | 20260507 22:45:23.844 | INFO | "huge" : "", | 20260507 22:45:23.844 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:23.844 | INFO | "malloc" : " 14 GB", | 20260507 22:45:23.844 | INFO | "pinned" : " 29 GB" | 20260507 22:45:23.844 | INFO | } | 20260507 22:45:23.844 | INFO | Perf report for the last 5.14024 seconds: | 20260507 22:45:23.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:23.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:45:23.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:45:23.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:23.844 | INFO | Uploaded 36GiB to GPU in 2.53824s (14.1831GiB/s) | 20260507 22:45:23.844 | INFO | Downloaded 34GiB to GPU in 2.46231s (13.8082GiB/s) | 20260507 22:45:28.983 | INFO | Analyzing chunk frames = [3187712,3188224) | 20260507 22:45:28.983 | INFO | MemInfo: { | 20260507 22:45:28.983 | INFO | "gpu" : " 17 GB", | 20260507 22:45:28.983 | INFO | "huge" : "", | 20260507 22:45:28.983 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:28.983 | INFO | "malloc" : " 14 GB", | 20260507 22:45:28.983 | INFO | "pinned" : " 29 GB" | 20260507 22:45:28.983 | INFO | } | 20260507 22:45:28.983 | INFO | Perf report for the last 5.13938 seconds: | 20260507 22:45:28.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:28.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.983 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:45:28.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.983 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 22:45:28.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:28.984 | INFO | Uploaded 36GiB to GPU in 2.51393s (14.3202GiB/s) | 20260507 22:45:28.984 | INFO | Downloaded 34GiB to GPU in 2.45294s (13.8609GiB/s) | 20260507 22:45:34.127 | INFO | Analyzing chunk frames = [3188224,3188736) | 20260507 22:45:34.127 | INFO | MemInfo: { | 20260507 22:45:34.127 | INFO | "gpu" : " 17 GB", | 20260507 22:45:34.127 | INFO | "huge" : "", | 20260507 22:45:34.127 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:34.127 | INFO | "malloc" : " 14 GB", | 20260507 22:45:34.127 | INFO | "pinned" : " 29 GB" | 20260507 22:45:34.127 | INFO | } | 20260507 22:45:34.127 | INFO | Perf report for the last 5.14403 seconds: | 20260507 22:45:34.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:34.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.127 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 22:45:34.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.128 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:45:34.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:34.128 | INFO | Uploaded 36GiB to GPU in 2.45949s (14.6372GiB/s) | 20260507 22:45:34.128 | INFO | Downloaded 34GiB to GPU in 2.38547s (14.253GiB/s) | 20260507 22:45:39.267 | INFO | Analyzing chunk frames = [3188736,3189248) | 20260507 22:45:39.267 | INFO | MemInfo: { | 20260507 22:45:39.267 | INFO | "gpu" : " 17 GB", | 20260507 22:45:39.267 | INFO | "huge" : "", | 20260507 22:45:39.267 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:39.267 | INFO | "malloc" : " 14 GB", | 20260507 22:45:39.267 | INFO | "pinned" : " 29 GB" | 20260507 22:45:39.267 | INFO | } | 20260507 22:45:39.267 | INFO | Perf report for the last 5.13944 seconds: | 20260507 22:45:39.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:39.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260507 22:45:39.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:45:39.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:39.267 | INFO | Uploaded 36GiB to GPU in 2.36095s (15.2481GiB/s) | 20260507 22:45:39.267 | INFO | Downloaded 34GiB to GPU in 2.50457s (13.5752GiB/s) | 20260507 22:45:44.409 | INFO | Analyzing chunk frames = [3189248,3189760) | 20260507 22:45:44.409 | INFO | MemInfo: { | 20260507 22:45:44.409 | INFO | "gpu" : " 17 GB", | 20260507 22:45:44.409 | INFO | "huge" : "", | 20260507 22:45:44.409 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:44.409 | INFO | "malloc" : " 14 GB", | 20260507 22:45:44.409 | INFO | "pinned" : " 29 GB" | 20260507 22:45:44.409 | INFO | } | 20260507 22:45:44.409 | INFO | Perf report for the last 5.14214 seconds: | 20260507 22:45:44.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:44.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.409 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 22:45:44.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.409 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:45:44.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:44.410 | INFO | Uploaded 36GiB to GPU in 2.51071s (14.3386GiB/s) | 20260507 22:45:44.410 | INFO | Downloaded 34GiB to GPU in 2.41395s (14.0848GiB/s) | 20260507 22:45:49.542 | INFO | Analyzing chunk frames = [3189760,3190272) | 20260507 22:45:49.542 | INFO | MemInfo: { | 20260507 22:45:49.542 | INFO | "gpu" : " 17 GB", | 20260507 22:45:49.542 | INFO | "huge" : "", | 20260507 22:45:49.542 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:49.542 | INFO | "malloc" : " 14 GB", | 20260507 22:45:49.542 | INFO | "pinned" : " 29 GB" | 20260507 22:45:49.542 | INFO | } | 20260507 22:45:49.542 | INFO | Perf report for the last 5.13341 seconds: | 20260507 22:45:49.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:49.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.542 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 22:45:49.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.543 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 22:45:49.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:49.543 | INFO | Uploaded 36GiB to GPU in 2.54527s (14.1439GiB/s) | 20260507 22:45:49.543 | INFO | Downloaded 34GiB to GPU in 2.43787s (13.9466GiB/s) | 20260507 22:45:54.679 | INFO | Analyzing chunk frames = [3190272,3190784) | 20260507 22:45:54.679 | INFO | MemInfo: { | 20260507 22:45:54.679 | INFO | "gpu" : " 17 GB", | 20260507 22:45:54.679 | INFO | "huge" : "", | 20260507 22:45:54.679 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:54.679 | INFO | "malloc" : " 14 GB", | 20260507 22:45:54.680 | INFO | "pinned" : " 29 GB" | 20260507 22:45:54.680 | INFO | } | 20260507 22:45:54.680 | INFO | Perf report for the last 5.13727 seconds: | 20260507 22:45:54.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:54.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 22:45:54.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:45:54.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:54.680 | INFO | Uploaded 36GiB to GPU in 2.53901s (14.1788GiB/s) | 20260507 22:45:54.680 | INFO | Downloaded 34GiB to GPU in 2.44255s (13.9199GiB/s) | 20260507 22:45:54.777 | INFO | Starting a round of Baz IO | 20260507 22:45:58.258 | INFO | Populating RTMetrics report | 20260507 22:45:58.258 | INFO | RTMetrics report populated | 20260507 22:45:58.258 | INFO | Opening tmp RTMetrics json file | 20260507 22:45:58.259 | INFO | Serializing RTMetrics to json | 20260507 22:45:58.261 | INFO | Serialization done, swapping tmp file now... | 20260507 22:45:58.265 | INFO | RTMetrics done | 20260507 22:45:58.379 | INFO | Finished a round of Baz IO: Wrote 1.73 GiB in 3.602 seconds | 20260507 22:45:58.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.95% / 0.01% / 57.05% | 20260507 22:45:58.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34079% / 1.58946e-05% / 5.5631e-05% | 20260507 22:45:58.379 | INFO | Baz IO overall: | 20260507 22:45:58.379 | INFO | BytesWritten : 1453605182079 | 20260507 22:45:58.379 | INFO | NumEvents : 704159297064 | 20260507 22:45:59.821 | INFO | Analyzing chunk frames = [3190784,3191296) | 20260507 22:45:59.821 | INFO | MemInfo: { | 20260507 22:45:59.821 | INFO | "gpu" : " 17 GB", | 20260507 22:45:59.821 | INFO | "huge" : "", | 20260507 22:45:59.821 | INFO | "huge_shared" : " 29 GB", | 20260507 22:45:59.821 | INFO | "malloc" : " 14 GB", | 20260507 22:45:59.821 | INFO | "pinned" : " 29 GB" | 20260507 22:45:59.821 | INFO | } | 20260507 22:45:59.822 | INFO | Perf report for the last 5.14177 seconds: | 20260507 22:45:59.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:45:59.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:59.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:59.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:59.822 | INFO | ANALYSIS : 68.8%, 3.0, 107.4ms, 0 | 20260507 22:45:59.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:45:59.822 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 22:45:59.822 | INFO | PRE_HQ : 65.8%, 1.8, 60.0ms, 0 | 20260507 22:45:59.822 | INFO | BAZWRITER : 69.6%, 3.9, 140.4ms, 0 | 20260507 22:45:59.822 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 22:45:59.822 | INFO | Uploaded 36GiB to GPU in 2.5886s (13.9071GiB/s) | 20260507 22:45:59.822 | INFO | Downloaded 38GiB to GPU in 2.68807s (14.1365GiB/s) | 20260507 22:46:04.964 | INFO | Analyzing chunk frames = [3191296,3191808) | 20260507 22:46:04.964 | INFO | MemInfo: { | 20260507 22:46:04.964 | INFO | "gpu" : " 17 GB", | 20260507 22:46:04.964 | INFO | "huge" : "", | 20260507 22:46:04.964 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:04.964 | INFO | "malloc" : " 14 GB", | 20260507 22:46:04.964 | INFO | "pinned" : " 29 GB" | 20260507 22:46:04.964 | INFO | } | 20260507 22:46:04.964 | INFO | Perf report for the last 5.14267 seconds: | 20260507 22:46:04.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:04.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:46:04.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:46:04.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:04.964 | INFO | Uploaded 36GiB to GPU in 2.54793s (14.1291GiB/s) | 20260507 22:46:04.964 | INFO | Downloaded 34GiB to GPU in 2.4515s (13.8691GiB/s) | 20260507 22:46:10.103 | INFO | Analyzing chunk frames = [3191808,3192320) | 20260507 22:46:10.103 | INFO | MemInfo: { | 20260507 22:46:10.103 | INFO | "gpu" : " 17 GB", | 20260507 22:46:10.103 | INFO | "huge" : "", | 20260507 22:46:10.103 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:10.103 | INFO | "malloc" : " 14 GB", | 20260507 22:46:10.103 | INFO | "pinned" : " 29 GB" | 20260507 22:46:10.103 | INFO | } | 20260507 22:46:10.103 | INFO | Perf report for the last 5.13875 seconds: | 20260507 22:46:10.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:10.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:46:10.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:46:10.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:10.103 | INFO | Uploaded 36GiB to GPU in 2.56868s (14.015GiB/s) | 20260507 22:46:10.103 | INFO | Downloaded 34GiB to GPU in 2.45617s (13.8427GiB/s) | 20260507 22:46:15.241 | INFO | Analyzing chunk frames = [3192320,3192832) | 20260507 22:46:15.241 | INFO | MemInfo: { | 20260507 22:46:15.241 | INFO | "gpu" : " 17 GB", | 20260507 22:46:15.241 | INFO | "huge" : "", | 20260507 22:46:15.241 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:15.241 | INFO | "malloc" : " 14 GB", | 20260507 22:46:15.241 | INFO | "pinned" : " 29 GB" | 20260507 22:46:15.241 | INFO | } | 20260507 22:46:15.241 | INFO | Perf report for the last 5.1383 seconds: | 20260507 22:46:15.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:15.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.241 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:46:15.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.241 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:46:15.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:15.242 | INFO | Uploaded 36GiB to GPU in 2.51344s (14.323GiB/s) | 20260507 22:46:15.242 | INFO | Downloaded 34GiB to GPU in 2.39896s (14.1728GiB/s) | 20260507 22:46:20.379 | INFO | Analyzing chunk frames = [3192832,3193344) | 20260507 22:46:20.379 | INFO | MemInfo: { | 20260507 22:46:20.379 | INFO | "gpu" : " 17 GB", | 20260507 22:46:20.379 | INFO | "huge" : "", | 20260507 22:46:20.379 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:20.379 | INFO | "malloc" : " 14 GB", | 20260507 22:46:20.379 | INFO | "pinned" : " 29 GB" | 20260507 22:46:20.379 | INFO | } | 20260507 22:46:20.379 | INFO | Perf report for the last 5.13807 seconds: | 20260507 22:46:20.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:20.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.379 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 22:46:20.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.379 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:46:20.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:20.380 | INFO | Uploaded 36GiB to GPU in 2.37392s (15.1648GiB/s) | 20260507 22:46:20.380 | INFO | Downloaded 34GiB to GPU in 2.4826s (13.6953GiB/s) | 20260507 22:46:25.519 | INFO | Analyzing chunk frames = [3193344,3193856) | 20260507 22:46:25.519 | INFO | MemInfo: { | 20260507 22:46:25.519 | INFO | "gpu" : " 17 GB", | 20260507 22:46:25.519 | INFO | "huge" : "", | 20260507 22:46:25.519 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:25.519 | INFO | "malloc" : " 14 GB", | 20260507 22:46:25.519 | INFO | "pinned" : " 29 GB" | 20260507 22:46:25.519 | INFO | } | 20260507 22:46:25.519 | INFO | Perf report for the last 5.14003 seconds: | 20260507 22:46:25.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:25.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.519 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:46:25.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.520 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:46:25.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:25.520 | INFO | Uploaded 36GiB to GPU in 2.51069s (14.3387GiB/s) | 20260507 22:46:25.520 | INFO | Downloaded 34GiB to GPU in 2.42333s (14.0303GiB/s) | 20260507 22:46:30.658 | INFO | Analyzing chunk frames = [3193856,3194368) | 20260507 22:46:30.658 | INFO | MemInfo: { | 20260507 22:46:30.658 | INFO | "gpu" : " 17 GB", | 20260507 22:46:30.658 | INFO | "huge" : "", | 20260507 22:46:30.658 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:30.658 | INFO | "malloc" : " 14 GB", | 20260507 22:46:30.658 | INFO | "pinned" : " 29 GB" | 20260507 22:46:30.658 | INFO | } | 20260507 22:46:30.658 | INFO | Perf report for the last 5.13857 seconds: | 20260507 22:46:30.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:30.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:46:30.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:46:30.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:30.658 | INFO | Uploaded 36GiB to GPU in 2.52815s (14.2397GiB/s) | 20260507 22:46:30.658 | INFO | Downloaded 34GiB to GPU in 2.44735s (13.8926GiB/s) | 20260507 22:46:35.793 | INFO | Analyzing chunk frames = [3194368,3194880) | 20260507 22:46:35.793 | INFO | MemInfo: { | 20260507 22:46:35.793 | INFO | "gpu" : " 17 GB", | 20260507 22:46:35.793 | INFO | "huge" : "", | 20260507 22:46:35.793 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:35.794 | INFO | "malloc" : " 14 GB", | 20260507 22:46:35.794 | INFO | "pinned" : " 29 GB" | 20260507 22:46:35.794 | INFO | } | 20260507 22:46:35.794 | INFO | Perf report for the last 5.13563 seconds: | 20260507 22:46:35.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:35.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:46:35.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 22:46:35.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:35.794 | INFO | Uploaded 36GiB to GPU in 2.48775s (14.4709GiB/s) | 20260507 22:46:35.794 | INFO | Downloaded 34GiB to GPU in 2.44074s (13.9302GiB/s) | 20260507 22:46:35.794 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:46:35.794 | INFO | Total mem usage: 16.7 GiB | 20260507 22:46:35.794 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:46:35.794 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -5.01%) | 20260507 22:46:35.794 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:46:35.794 | INFO | High Water Breakdown: | 20260507 22:46:35.794 | INFO | 61% : 9.69 GiB ( 94.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:46:35.794 | INFO | 38.9% : 6.17 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:46:35.794 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:46:35.794 | INFO | ------------------------------------------ | 20260507 22:46:35.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:46:35.794 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:46:35.794 | INFO | Total mem usage: 26.5 GiB | 20260507 22:46:35.794 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:46:35.794 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:46:35.794 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:46:35.795 | INFO | High Water Breakdown: | 20260507 22:46:35.795 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:46:35.795 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:46:35.795 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:46:35.795 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:46:35.795 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:46:35.795 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:46:35.795 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:46:35.795 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:46:35.795 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:46:35.795 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:46:35.795 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:46:35.795 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:46:35.795 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:46:35.795 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:46:35.795 | INFO | ------------------------------------------ | 20260507 22:46:35.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:46:35.795 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:46:35.795 | INFO | Total mem usage: 27.8 GiB | 20260507 22:46:35.795 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:46:35.795 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:46:35.795 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:46:35.795 | INFO | High Water Breakdown: | 20260507 22:46:35.795 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:46:35.795 | INFO | ------------------------------------------ | 20260507 22:46:35.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:46:35.795 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:46:35.795 | INFO | Total mem usage: 15.7 GiB | 20260507 22:46:35.795 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:46:35.795 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:46:35.795 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:46:35.795 | INFO | High Water Breakdown: | 20260507 22:46:35.795 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:46:35.795 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:46:35.795 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:46:35.795 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:46:35.795 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:46:35.795 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:46:35.795 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:46:35.795 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:46:35.795 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:46:35.795 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:46:35.795 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:46:35.795 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:46:35.795 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:46:35.795 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:46:35.795 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:46:35.795 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:46:35.795 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:46:35.795 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:46:35.795 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:46:35.795 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:46:35.795 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:46:35.795 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:46:35.795 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:46:35.795 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:46:35.795 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:46:35.795 | INFO | ------------------------------------------ | 20260507 22:46:35.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:46:35.795 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:46:35.795 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:46:35.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:46:35.795 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.63e+01, 1.40e+01 / 1.75e+00 | 20260507 22:46:35.795 | INFO | PerfReport: 0.29%, AggregateStats -- 9.71e-02 / 1.03e-01, 1.06e-02 / 4.88e-03 | 20260507 22:46:35.795 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.49e+00, 6.00e-01 / 2.02e-01 | 20260507 22:46:35.795 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.49e+00, 2.10e-01 / 1.21e-01 | 20260507 22:46:35.795 | INFO | PerfReport: 15.64%, DME -- 4.13e+01 / 4.45e+01, 3.85e+00 / 1.13e+00 | 20260507 22:46:35.795 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.87e+01 / 2.13e+01, 1.86e+00 / 1.97e+00 | 20260507 22:46:35.795 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.41e+00 / 2.71e+00, 4.22e-01 / 2.18e-01 | 20260507 22:46:35.795 | INFO | PerfReport: 3.44%, Metrics -- 1.14e+00 / 1.32e+00, 2.91e-01 / 1.42e-01 | 20260507 22:46:35.795 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.52e-01 / 3.83e-01, 5.79e-02 / 3.36e-02 | 20260507 22:46:35.795 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:46:35.795 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:46:35.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:46:35.795 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 22:46:35.795 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260507 22:46:35.796 | INFO | PerfReport: 24.11%, Download -- 2.58e+01 / 0.00e+00, 8.62e+00 / 0.00e+00 | 20260507 22:46:35.796 | INFO | PerfReport: 30.87%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:46:35.796 | INFO | PerfReport: 1.97%, LockUpload -- 2.10e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 22:46:35.796 | INFO | PerfReport: 2.46%, LockDownload -- 2.63e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260507 22:46:35.796 | INFO | PerfReport: 16.13%, LockCompute -- 1.72e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 22:46:35.796 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 22:46:35.904 | INFO | Starting a round of Baz IO | 20260507 22:46:39.360 | INFO | Populating RTMetrics report | 20260507 22:46:39.360 | INFO | RTMetrics report populated | 20260507 22:46:39.360 | INFO | Opening tmp RTMetrics json file | 20260507 22:46:39.386 | INFO | Serializing RTMetrics to json | 20260507 22:46:39.388 | INFO | Serialization done, swapping tmp file now... | 20260507 22:46:39.388 | INFO | RTMetrics done | 20260507 22:46:39.463 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.559 seconds | 20260507 22:46:39.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.94% / 0.01% / 57.05% | 20260507 22:46:39.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33592% / 1.19209e-05% / 3.97364e-05% | 20260507 22:46:39.463 | INFO | Baz IO overall: | 20260507 22:46:39.463 | INFO | BytesWritten : 1455460815251 | 20260507 22:46:39.463 | INFO | NumEvents : 705054110421 | 20260507 22:46:40.938 | INFO | Analyzing chunk frames = [3194880,3195392) | 20260507 22:46:40.938 | INFO | MemInfo: { | 20260507 22:46:40.938 | INFO | "gpu" : " 17 GB", | 20260507 22:46:40.938 | INFO | "huge" : "", | 20260507 22:46:40.938 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:40.938 | INFO | "malloc" : " 14 GB", | 20260507 22:46:40.938 | INFO | "pinned" : " 29 GB" | 20260507 22:46:40.938 | INFO | } | 20260507 22:46:40.938 | INFO | Perf report for the last 5.14441 seconds: | 20260507 22:46:40.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:40.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:40.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:40.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:40.938 | INFO | ANALYSIS : 68.6%, 3.0, 107.1ms, 0 | 20260507 22:46:40.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:40.938 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 22:46:40.938 | INFO | PRE_HQ : 65.6%, 1.7, 57.7ms, 0 | 20260507 22:46:40.938 | INFO | BAZWRITER : 69.0%, 3.7, 131.5ms, 0 | 20260507 22:46:40.938 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 22:46:40.938 | INFO | Uploaded 36GiB to GPU in 2.48717s (14.4743GiB/s) | 20260507 22:46:40.938 | INFO | Downloaded 38GiB to GPU in 2.69686s (14.0904GiB/s) | 20260507 22:46:46.081 | INFO | Analyzing chunk frames = [3195392,3195904) | 20260507 22:46:46.081 | INFO | MemInfo: { | 20260507 22:46:46.081 | INFO | "gpu" : " 17 GB", | 20260507 22:46:46.081 | INFO | "huge" : "", | 20260507 22:46:46.081 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:46.081 | INFO | "malloc" : " 14 GB", | 20260507 22:46:46.081 | INFO | "pinned" : " 29 GB" | 20260507 22:46:46.081 | INFO | } | 20260507 22:46:46.082 | INFO | Perf report for the last 5.14354 seconds: | 20260507 22:46:46.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:46.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:46:46.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 22:46:46.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:46.082 | INFO | Uploaded 36GiB to GPU in 2.56335s (14.0441GiB/s) | 20260507 22:46:46.082 | INFO | Downloaded 34GiB to GPU in 2.47706s (13.7259GiB/s) | 20260507 22:46:51.218 | INFO | Analyzing chunk frames = [3195904,3196416) | 20260507 22:46:51.218 | INFO | MemInfo: { | 20260507 22:46:51.218 | INFO | "gpu" : " 17 GB", | 20260507 22:46:51.218 | INFO | "huge" : "", | 20260507 22:46:51.218 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:51.218 | INFO | "malloc" : " 14 GB", | 20260507 22:46:51.219 | INFO | "pinned" : " 29 GB" | 20260507 22:46:51.219 | INFO | } | 20260507 22:46:51.219 | INFO | Perf report for the last 5.13699 seconds: | 20260507 22:46:51.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:51.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:46:51.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:46:51.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:51.219 | INFO | Uploaded 36GiB to GPU in 2.49169s (14.448GiB/s) | 20260507 22:46:51.219 | INFO | Downloaded 34GiB to GPU in 2.48506s (13.6817GiB/s) | 20260507 22:46:56.359 | INFO | Analyzing chunk frames = [3196416,3196928) | 20260507 22:46:56.359 | INFO | MemInfo: { | 20260507 22:46:56.359 | INFO | "gpu" : " 17 GB", | 20260507 22:46:56.359 | INFO | "huge" : "", | 20260507 22:46:56.359 | INFO | "huge_shared" : " 29 GB", | 20260507 22:46:56.359 | INFO | "malloc" : " 14 GB", | 20260507 22:46:56.359 | INFO | "pinned" : " 29 GB" | 20260507 22:46:56.359 | INFO | } | 20260507 22:46:56.359 | INFO | Perf report for the last 5.14023 seconds: | 20260507 22:46:56.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:46:56.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.359 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:46:56.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.359 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:46:56.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:46:56.360 | INFO | Uploaded 36GiB to GPU in 2.48251s (14.5015GiB/s) | 20260507 22:46:56.360 | INFO | Downloaded 34GiB to GPU in 2.40586s (14.1322GiB/s) | 20260507 22:47:01.495 | INFO | Analyzing chunk frames = [3196928,3197440) | 20260507 22:47:01.495 | INFO | MemInfo: { | 20260507 22:47:01.495 | INFO | "gpu" : " 17 GB", | 20260507 22:47:01.495 | INFO | "huge" : "", | 20260507 22:47:01.495 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:01.495 | INFO | "malloc" : " 14 GB", | 20260507 22:47:01.495 | INFO | "pinned" : " 29 GB" | 20260507 22:47:01.495 | INFO | } | 20260507 22:47:01.495 | INFO | Perf report for the last 5.13639 seconds: | 20260507 22:47:01.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:01.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.495 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 22:47:01.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.495 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 22:47:01.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:01.496 | INFO | Uploaded 36GiB to GPU in 2.36481s (15.2232GiB/s) | 20260507 22:47:01.496 | INFO | Downloaded 34GiB to GPU in 2.48958s (13.6569GiB/s) | 20260507 22:47:06.635 | INFO | Analyzing chunk frames = [3197440,3197952) | 20260507 22:47:06.635 | INFO | MemInfo: { | 20260507 22:47:06.635 | INFO | "gpu" : " 17 GB", | 20260507 22:47:06.636 | INFO | "huge" : "", | 20260507 22:47:06.636 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:06.636 | INFO | "malloc" : " 14 GB", | 20260507 22:47:06.636 | INFO | "pinned" : " 29 GB" | 20260507 22:47:06.636 | INFO | } | 20260507 22:47:06.636 | INFO | Perf report for the last 5.14068 seconds: | 20260507 22:47:06.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:06.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.636 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 22:47:06.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.636 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:47:06.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:06.637 | INFO | Uploaded 36GiB to GPU in 2.49107s (14.4516GiB/s) | 20260507 22:47:06.637 | INFO | Downloaded 34GiB to GPU in 2.41425s (14.0831GiB/s) | 20260507 22:47:11.775 | INFO | Analyzing chunk frames = [3197952,3198464) | 20260507 22:47:11.775 | INFO | MemInfo: { | 20260507 22:47:11.775 | INFO | "gpu" : " 17 GB", | 20260507 22:47:11.775 | INFO | "huge" : "", | 20260507 22:47:11.775 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:11.775 | INFO | "malloc" : " 14 GB", | 20260507 22:47:11.775 | INFO | "pinned" : " 29 GB" | 20260507 22:47:11.775 | INFO | } | 20260507 22:47:11.775 | INFO | Perf report for the last 5.1394 seconds: | 20260507 22:47:11.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:11.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.775 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 22:47:11.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.775 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:47:11.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:11.776 | INFO | Uploaded 36GiB to GPU in 2.54915s (14.1224GiB/s) | 20260507 22:47:11.776 | INFO | Downloaded 34GiB to GPU in 2.45406s (13.8546GiB/s) | 20260507 22:47:16.913 | INFO | Analyzing chunk frames = [3198464,3198976) | 20260507 22:47:16.913 | INFO | MemInfo: { | 20260507 22:47:16.913 | INFO | "gpu" : " 17 GB", | 20260507 22:47:16.914 | INFO | "huge" : "", | 20260507 22:47:16.914 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:16.914 | INFO | "malloc" : " 14 GB", | 20260507 22:47:16.914 | INFO | "pinned" : " 29 GB" | 20260507 22:47:16.914 | INFO | } | 20260507 22:47:16.914 | INFO | Perf report for the last 5.13842 seconds: | 20260507 22:47:16.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:16.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 22:47:16.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 22:47:16.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:16.914 | INFO | Uploaded 36GiB to GPU in 2.52974s (14.2307GiB/s) | 20260507 22:47:16.914 | INFO | Downloaded 34GiB to GPU in 2.44621s (13.8991GiB/s) | 20260507 22:47:17.028 | INFO | Starting a round of Baz IO | 20260507 22:47:20.488 | INFO | Populating RTMetrics report | 20260507 22:47:20.488 | INFO | RTMetrics report populated | 20260507 22:47:20.488 | INFO | Opening tmp RTMetrics json file | 20260507 22:47:20.490 | INFO | Serializing RTMetrics to json | 20260507 22:47:20.492 | INFO | Serialization done, swapping tmp file now... | 20260507 22:47:20.493 | INFO | RTMetrics done | 20260507 22:47:20.633 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.604 seconds | 20260507 22:47:20.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.93% / 0.01% / 57.06% | 20260507 22:47:20.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33359% / 1.58946e-05% / 4.76837e-05% | 20260507 22:47:20.633 | INFO | Baz IO overall: | 20260507 22:47:20.633 | INFO | BytesWritten : 1457315711549 | 20260507 22:47:20.633 | INFO | NumEvents : 705948431286 | 20260507 22:47:22.054 | INFO | Analyzing chunk frames = [3198976,3199488) | 20260507 22:47:22.054 | INFO | MemInfo: { | 20260507 22:47:22.054 | INFO | "gpu" : " 17 GB", | 20260507 22:47:22.054 | INFO | "huge" : "", | 20260507 22:47:22.054 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:22.054 | INFO | "malloc" : " 14 GB", | 20260507 22:47:22.054 | INFO | "pinned" : " 29 GB" | 20260507 22:47:22.054 | INFO | } | 20260507 22:47:22.054 | INFO | Perf report for the last 5.14013 seconds: | 20260507 22:47:22.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:22.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:22.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:22.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:22.054 | INFO | ANALYSIS : 69.1%, 2.9, 105.2ms, 0 | 20260507 22:47:22.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:22.054 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 22:47:22.054 | INFO | PRE_HQ : 67.2%, 1.8, 62.4ms, 0 | 20260507 22:47:22.054 | INFO | BAZWRITER : 69.7%, 3.7, 132.3ms, 0 | 20260507 22:47:22.054 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 22:47:22.055 | INFO | Uploaded 36GiB to GPU in 2.54195s (14.1623GiB/s) | 20260507 22:47:22.055 | INFO | Downloaded 38GiB to GPU in 2.68409s (14.1575GiB/s) | 20260507 22:47:27.194 | INFO | Analyzing chunk frames = [3199488,3200000) | 20260507 22:47:27.194 | INFO | MemInfo: { | 20260507 22:47:27.194 | INFO | "gpu" : " 17 GB", | 20260507 22:47:27.194 | INFO | "huge" : "", | 20260507 22:47:27.194 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:27.194 | INFO | "malloc" : " 14 GB", | 20260507 22:47:27.194 | INFO | "pinned" : " 29 GB" | 20260507 22:47:27.194 | INFO | } | 20260507 22:47:27.194 | INFO | Perf report for the last 5.14022 seconds: | 20260507 22:47:27.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:27.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 22:47:27.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:47:27.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:27.194 | INFO | Uploaded 36GiB to GPU in 2.58666s (13.9176GiB/s) | 20260507 22:47:27.195 | INFO | Downloaded 34GiB to GPU in 2.47531s (13.7356GiB/s) | 20260507 22:47:32.331 | INFO | Analyzing chunk frames = [3200000,3200512) | 20260507 22:47:32.331 | INFO | MemInfo: { | 20260507 22:47:32.331 | INFO | "gpu" : " 17 GB", | 20260507 22:47:32.331 | INFO | "huge" : "", | 20260507 22:47:32.331 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:32.331 | INFO | "malloc" : " 14 GB", | 20260507 22:47:32.332 | INFO | "pinned" : " 29 GB" | 20260507 22:47:32.332 | INFO | } | 20260507 22:47:32.332 | INFO | Perf report for the last 5.13748 seconds: | 20260507 22:47:32.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:32.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 22:47:32.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:47:32.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:32.332 | INFO | Uploaded 36GiB to GPU in 2.53662s (14.1921GiB/s) | 20260507 22:47:32.332 | INFO | Downloaded 34GiB to GPU in 2.48451s (13.6848GiB/s) | 20260507 22:47:37.470 | INFO | Analyzing chunk frames = [3200512,3201024) | 20260507 22:47:37.470 | INFO | MemInfo: { | 20260507 22:47:37.470 | INFO | "gpu" : " 17 GB", | 20260507 22:47:37.470 | INFO | "huge" : "", | 20260507 22:47:37.470 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:37.470 | INFO | "malloc" : " 14 GB", | 20260507 22:47:37.470 | INFO | "pinned" : " 29 GB" | 20260507 22:47:37.470 | INFO | } | 20260507 22:47:37.470 | INFO | Perf report for the last 5.13826 seconds: | 20260507 22:47:37.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:37.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 22:47:37.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:47:37.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:37.470 | INFO | Uploaded 36GiB to GPU in 2.49019s (14.4567GiB/s) | 20260507 22:47:37.471 | INFO | Downloaded 34GiB to GPU in 2.39346s (14.2054GiB/s) | 20260507 22:47:42.606 | INFO | Analyzing chunk frames = [3201024,3201536) | 20260507 22:47:42.606 | INFO | MemInfo: { | 20260507 22:47:42.606 | INFO | "gpu" : " 17 GB", | 20260507 22:47:42.606 | INFO | "huge" : "", | 20260507 22:47:42.606 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:42.606 | INFO | "malloc" : " 14 GB", | 20260507 22:47:42.606 | INFO | "pinned" : " 29 GB" | 20260507 22:47:42.606 | INFO | } | 20260507 22:47:42.607 | INFO | Perf report for the last 5.13659 seconds: | 20260507 22:47:42.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:42.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260507 22:47:42.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:47:42.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:42.607 | INFO | Uploaded 36GiB to GPU in 2.35412s (15.2923GiB/s) | 20260507 22:47:42.607 | INFO | Downloaded 34GiB to GPU in 2.49918s (13.6045GiB/s) | 20260507 22:47:47.747 | INFO | Analyzing chunk frames = [3201536,3202048) | 20260507 22:47:47.747 | INFO | MemInfo: { | 20260507 22:47:47.747 | INFO | "gpu" : " 17 GB", | 20260507 22:47:47.747 | INFO | "huge" : "", | 20260507 22:47:47.747 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:47.747 | INFO | "malloc" : " 14 GB", | 20260507 22:47:47.748 | INFO | "pinned" : " 29 GB" | 20260507 22:47:47.748 | INFO | } | 20260507 22:47:47.748 | INFO | Perf report for the last 5.14115 seconds: | 20260507 22:47:47.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:47.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:47:47.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 22:47:47.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:47.748 | INFO | Uploaded 36GiB to GPU in 2.5045s (14.3741GiB/s) | 20260507 22:47:47.748 | INFO | Downloaded 34GiB to GPU in 2.42519s (14.0195GiB/s) | 20260507 22:47:52.887 | INFO | Analyzing chunk frames = [3202048,3202560) | 20260507 22:47:52.887 | INFO | MemInfo: { | 20260507 22:47:52.887 | INFO | "gpu" : " 17 GB", | 20260507 22:47:52.887 | INFO | "huge" : "", | 20260507 22:47:52.888 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:52.888 | INFO | "malloc" : " 14 GB", | 20260507 22:47:52.888 | INFO | "pinned" : " 29 GB" | 20260507 22:47:52.888 | INFO | } | 20260507 22:47:52.888 | INFO | Perf report for the last 5.13996 seconds: | 20260507 22:47:52.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:52.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:47:52.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 22:47:52.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:52.888 | INFO | Uploaded 36GiB to GPU in 2.51897s (14.2915GiB/s) | 20260507 22:47:52.888 | INFO | Downloaded 34GiB to GPU in 2.4517s (13.8679GiB/s) | 20260507 22:47:58.032 | INFO | Analyzing chunk frames = [3202560,3203072) | 20260507 22:47:58.032 | INFO | MemInfo: { | 20260507 22:47:58.032 | INFO | "gpu" : " 17 GB", | 20260507 22:47:58.032 | INFO | "huge" : "", | 20260507 22:47:58.032 | INFO | "huge_shared" : " 29 GB", | 20260507 22:47:58.032 | INFO | "malloc" : " 14 GB", | 20260507 22:47:58.032 | INFO | "pinned" : " 29 GB" | 20260507 22:47:58.032 | INFO | } | 20260507 22:47:58.033 | INFO | Perf report for the last 5.14477 seconds: | 20260507 22:47:58.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:47:58.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 22:47:58.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | PRE_HQ : 36.6%, 1.0, 19.0ms, 0 | 20260507 22:47:58.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:47:58.033 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:47:58.033 | INFO | Total mem usage: 16.7 GiB | 20260507 22:47:58.033 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:47:58.033 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260507 22:47:58.033 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:47:58.033 | INFO | High Water Breakdown: | 20260507 22:47:58.033 | INFO | 59.7% : 8.73 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:47:58.033 | INFO | 40.1% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:47:58.033 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:47:58.033 | INFO | ------------------------------------------ | 20260507 22:47:58.033 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:47:58.033 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:47:58.033 | INFO | Total mem usage: 26.5 GiB | 20260507 22:47:58.033 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:47:58.033 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 22:47:58.033 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:47:58.033 | INFO | High Water Breakdown: | 20260507 22:47:58.033 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:47:58.033 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:47:58.033 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:47:58.033 | INFO | Uploaded 36GiB to GPU in 2.52773s (14.242GiB/s) | 20260507 22:47:58.033 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:47:58.034 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:47:58.034 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:47:58.034 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:47:58.034 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:47:58.034 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:47:58.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:47:58.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:47:58.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:47:58.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:47:58.034 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:47:58.034 | INFO | ------------------------------------------ | 20260507 22:47:58.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:47:58.034 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:47:58.034 | INFO | Total mem usage: 27.8 GiB | 20260507 22:47:58.034 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:47:58.034 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:47:58.034 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:47:58.034 | INFO | High Water Breakdown: | 20260507 22:47:58.034 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:47:58.034 | INFO | ------------------------------------------ | 20260507 22:47:58.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:47:58.034 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:47:58.034 | INFO | Total mem usage: 15.7 GiB | 20260507 22:47:58.034 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:47:58.034 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:47:58.034 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:47:58.034 | INFO | High Water Breakdown: | 20260507 22:47:58.034 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:47:58.034 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:47:58.034 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:47:58.034 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:47:58.034 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:47:58.034 | INFO | Downloaded 34GiB to GPU in 2.45832s (13.8306GiB/s) | 20260507 22:47:58.034 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:47:58.034 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:47:58.034 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:47:58.034 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:47:58.034 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:47:58.034 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:47:58.034 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:47:58.034 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:47:58.034 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:47:58.034 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:47:58.034 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:47:58.034 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:47:58.034 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:47:58.034 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:47:58.034 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:47:58.034 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:47:58.034 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:47:58.034 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:47:58.034 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:47:58.034 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:47:58.034 | INFO | ------------------------------------------ | 20260507 22:47:58.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:47:58.034 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:47:58.034 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:47:58.034 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:47:58.034 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.59e+01, 1.42e+01 / 1.54e+00 | 20260507 22:47:58.034 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.14e-02 / 7.73e-03 | 20260507 22:47:58.034 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.48e+00, 5.91e-01 / 2.10e-01 | 20260507 22:47:58.034 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.10e-01 / 1.22e-01 | 20260507 22:47:58.034 | INFO | PerfReport: 15.98%, DME -- 4.21e+01 / 4.57e+01, 3.99e+00 / 1.55e+00 | 20260507 22:47:58.034 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.73e+00 / 1.49e+00 | 20260507 22:47:58.034 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.33e-01 / 2.33e-01 | 20260507 22:47:58.035 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.35e+00, 2.83e-01 / 1.56e-01 | 20260507 22:47:58.035 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.86e-01, 5.73e-02 / 2.92e-02 | 20260507 22:47:58.035 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:47:58.035 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:47:58.035 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:47:58.035 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 30.81%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 2.18%, LockUpload -- 2.32e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 2.59%, LockDownload -- 2.76e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 15.78%, LockCompute -- 1.69e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 22:47:58.035 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 22:47:58.145 | INFO | Starting a round of Baz IO | 20260507 22:48:01.624 | INFO | Populating RTMetrics report | 20260507 22:48:01.625 | INFO | RTMetrics report populated | 20260507 22:48:01.625 | INFO | Opening tmp RTMetrics json file | 20260507 22:48:01.655 | INFO | Serializing RTMetrics to json | 20260507 22:48:01.708 | INFO | Serialization done, swapping tmp file now... | 20260507 22:48:01.714 | INFO | RTMetrics done | 20260507 22:48:01.729 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.583 seconds | 20260507 22:48:01.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.93% / 0.01% / 57.07% | 20260507 22:48:01.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33423% / 1.19209e-05% / 4.37101e-05% | 20260507 22:48:01.729 | INFO | Baz IO overall: | 20260507 22:48:01.729 | INFO | BytesWritten : 1459172171950 | 20260507 22:48:01.729 | INFO | NumEvents : 706844136966 | 20260507 22:48:03.177 | INFO | Analyzing chunk frames = [3203072,3203584) | 20260507 22:48:03.177 | INFO | MemInfo: { | 20260507 22:48:03.177 | INFO | "gpu" : " 17 GB", | 20260507 22:48:03.177 | INFO | "huge" : "", | 20260507 22:48:03.177 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:03.177 | INFO | "malloc" : " 14 GB", | 20260507 22:48:03.177 | INFO | "pinned" : " 29 GB" | 20260507 22:48:03.177 | INFO | } | 20260507 22:48:03.177 | INFO | Perf report for the last 5.14469 seconds: | 20260507 22:48:03.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:03.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:03.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:03.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:03.177 | INFO | ANALYSIS : 69.2%, 2.9, 104.9ms, 0 | 20260507 22:48:03.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:03.177 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260507 22:48:03.177 | INFO | PRE_HQ : 67.7%, 1.8, 64.4ms, 0 | 20260507 22:48:03.177 | INFO | BAZWRITER : 69.1%, 3.6, 129.2ms, 0 | 20260507 22:48:03.177 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 22:48:03.178 | INFO | Uploaded 36GiB to GPU in 2.54217s (14.1611GiB/s) | 20260507 22:48:03.178 | INFO | Downloaded 38GiB to GPU in 2.71749s (13.9835GiB/s) | 20260507 22:48:08.310 | INFO | Analyzing chunk frames = [3203584,3204096) | 20260507 22:48:08.310 | INFO | MemInfo: { | 20260507 22:48:08.310 | INFO | "gpu" : " 17 GB", | 20260507 22:48:08.310 | INFO | "huge" : "", | 20260507 22:48:08.310 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:08.310 | INFO | "malloc" : " 14 GB", | 20260507 22:48:08.310 | INFO | "pinned" : " 29 GB" | 20260507 22:48:08.310 | INFO | } | 20260507 22:48:08.310 | INFO | Perf report for the last 5.1332 seconds: | 20260507 22:48:08.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:08.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.311 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:48:08.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.311 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:48:08.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:08.311 | INFO | Uploaded 36GiB to GPU in 2.49815s (14.4106GiB/s) | 20260507 22:48:08.311 | INFO | Downloaded 34GiB to GPU in 2.42986s (13.9926GiB/s) | 20260507 22:48:13.445 | INFO | Analyzing chunk frames = [3204096,3204608) | 20260507 22:48:13.445 | INFO | MemInfo: { | 20260507 22:48:13.445 | INFO | "gpu" : " 17 GB", | 20260507 22:48:13.445 | INFO | "huge" : "", | 20260507 22:48:13.445 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:13.445 | INFO | "malloc" : " 14 GB", | 20260507 22:48:13.445 | INFO | "pinned" : " 29 GB" | 20260507 22:48:13.445 | INFO | } | 20260507 22:48:13.445 | INFO | Perf report for the last 5.13475 seconds: | 20260507 22:48:13.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:13.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.445 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 22:48:13.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.445 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:48:13.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:13.446 | INFO | Uploaded 36GiB to GPU in 2.4529s (14.6765GiB/s) | 20260507 22:48:13.446 | INFO | Downloaded 34GiB to GPU in 2.49451s (13.6299GiB/s) | 20260507 22:48:18.587 | INFO | Analyzing chunk frames = [3204608,3205120) | 20260507 22:48:18.587 | INFO | MemInfo: { | 20260507 22:48:18.587 | INFO | "gpu" : " 17 GB", | 20260507 22:48:18.588 | INFO | "huge" : "", | 20260507 22:48:18.588 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:18.588 | INFO | "malloc" : " 14 GB", | 20260507 22:48:18.588 | INFO | "pinned" : " 29 GB" | 20260507 22:48:18.588 | INFO | } | 20260507 22:48:18.588 | INFO | Perf report for the last 5.14253 seconds: | 20260507 22:48:18.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:18.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260507 22:48:18.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:48:18.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:18.588 | INFO | Uploaded 36GiB to GPU in 2.54329s (14.1549GiB/s) | 20260507 22:48:18.588 | INFO | Downloaded 34GiB to GPU in 2.49161s (13.6458GiB/s) | 20260507 22:48:23.727 | INFO | Analyzing chunk frames = [3205120,3205632) | 20260507 22:48:23.727 | INFO | MemInfo: { | 20260507 22:48:23.727 | INFO | "gpu" : " 17 GB", | 20260507 22:48:23.728 | INFO | "huge" : "", | 20260507 22:48:23.728 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:23.728 | INFO | "malloc" : " 14 GB", | 20260507 22:48:23.728 | INFO | "pinned" : " 29 GB" | 20260507 22:48:23.728 | INFO | } | 20260507 22:48:23.728 | INFO | Perf report for the last 5.14 seconds: | 20260507 22:48:23.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:23.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:48:23.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 22:48:23.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:23.728 | INFO | Uploaded 36GiB to GPU in 2.32193s (15.5043GiB/s) | 20260507 22:48:23.728 | INFO | Downloaded 34GiB to GPU in 2.49927s (13.604GiB/s) | 20260507 22:48:28.862 | INFO | Analyzing chunk frames = [3205632,3206144) | 20260507 22:48:28.862 | INFO | MemInfo: { | 20260507 22:48:28.862 | INFO | "gpu" : " 17 GB", | 20260507 22:48:28.862 | INFO | "huge" : "", | 20260507 22:48:28.862 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:28.862 | INFO | "malloc" : " 14 GB", | 20260507 22:48:28.862 | INFO | "pinned" : " 29 GB" | 20260507 22:48:28.862 | INFO | } | 20260507 22:48:28.862 | INFO | Perf report for the last 5.13468 seconds: | 20260507 22:48:28.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:28.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:48:28.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:48:28.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:28.863 | INFO | Uploaded 36GiB to GPU in 2.49954s (14.4027GiB/s) | 20260507 22:48:28.863 | INFO | Downloaded 34GiB to GPU in 2.43217s (13.9793GiB/s) | 20260507 22:48:34.005 | INFO | Analyzing chunk frames = [3206144,3206656) | 20260507 22:48:34.005 | INFO | MemInfo: { | 20260507 22:48:34.005 | INFO | "gpu" : " 17 GB", | 20260507 22:48:34.005 | INFO | "huge" : "", | 20260507 22:48:34.005 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:34.005 | INFO | "malloc" : " 14 GB", | 20260507 22:48:34.005 | INFO | "pinned" : " 29 GB" | 20260507 22:48:34.005 | INFO | } | 20260507 22:48:34.005 | INFO | Perf report for the last 5.14243 seconds: | 20260507 22:48:34.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:34.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 22:48:34.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 22:48:34.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:34.005 | INFO | Uploaded 36GiB to GPU in 2.49602s (14.423GiB/s) | 20260507 22:48:34.005 | INFO | Downloaded 34GiB to GPU in 2.44638s (13.8981GiB/s) | 20260507 22:48:39.140 | INFO | Analyzing chunk frames = [3206656,3207168) | 20260507 22:48:39.140 | INFO | MemInfo: { | 20260507 22:48:39.140 | INFO | "gpu" : " 17 GB", | 20260507 22:48:39.140 | INFO | "huge" : "", | 20260507 22:48:39.140 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:39.140 | INFO | "malloc" : " 14 GB", | 20260507 22:48:39.140 | INFO | "pinned" : " 29 GB" | 20260507 22:48:39.140 | INFO | } | 20260507 22:48:39.140 | INFO | Perf report for the last 5.13528 seconds: | 20260507 22:48:39.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:39.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.140 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:48:39.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.140 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260507 22:48:39.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:39.141 | INFO | Uploaded 36GiB to GPU in 2.5411s (14.1671GiB/s) | 20260507 22:48:39.141 | INFO | Downloaded 34GiB to GPU in 2.46183s (13.8109GiB/s) | 20260507 22:48:39.238 | INFO | Starting a round of Baz IO | 20260507 22:48:42.704 | INFO | Populating RTMetrics report | 20260507 22:48:42.704 | INFO | RTMetrics report populated | 20260507 22:48:42.704 | INFO | Opening tmp RTMetrics json file | 20260507 22:48:42.713 | INFO | Serializing RTMetrics to json | 20260507 22:48:42.723 | INFO | Serialization done, swapping tmp file now... | 20260507 22:48:42.724 | INFO | RTMetrics done | 20260507 22:48:42.803 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.564 seconds | 20260507 22:48:42.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.92% / 0.01% / 57.07% | 20260507 22:48:42.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33105% / 7.94729e-06% / 5.5631e-05% | 20260507 22:48:42.803 | INFO | Baz IO overall: | 20260507 22:48:42.803 | INFO | BytesWritten : 1461028127309 | 20260507 22:48:42.803 | INFO | NumEvents : 707739390213 | 20260507 22:48:44.282 | INFO | Analyzing chunk frames = [3207168,3207680) | 20260507 22:48:44.282 | INFO | MemInfo: { | 20260507 22:48:44.282 | INFO | "gpu" : " 17 GB", | 20260507 22:48:44.282 | INFO | "huge" : "", | 20260507 22:48:44.282 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:44.282 | INFO | "malloc" : " 14 GB", | 20260507 22:48:44.282 | INFO | "pinned" : " 29 GB" | 20260507 22:48:44.282 | INFO | } | 20260507 22:48:44.282 | INFO | Perf report for the last 5.14156 seconds: | 20260507 22:48:44.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:44.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:44.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:44.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:44.282 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 22:48:44.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:44.282 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 22:48:44.282 | INFO | PRE_HQ : 65.6%, 1.7, 57.3ms, 0 | 20260507 22:48:44.282 | INFO | BAZWRITER : 68.9%, 3.6, 126.4ms, 0 | 20260507 22:48:44.282 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 22:48:44.282 | INFO | Uploaded 36GiB to GPU in 2.54078s (14.1689GiB/s) | 20260507 22:48:44.282 | INFO | Downloaded 38GiB to GPU in 2.68684s (14.143GiB/s) | 20260507 22:48:49.424 | INFO | Analyzing chunk frames = [3207680,3208192) | 20260507 22:48:49.424 | INFO | MemInfo: { | 20260507 22:48:49.424 | INFO | "gpu" : " 17 GB", | 20260507 22:48:49.424 | INFO | "huge" : "", | 20260507 22:48:49.424 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:49.424 | INFO | "malloc" : " 14 GB", | 20260507 22:48:49.424 | INFO | "pinned" : " 29 GB" | 20260507 22:48:49.424 | INFO | } | 20260507 22:48:49.424 | INFO | Perf report for the last 5.14258 seconds: | 20260507 22:48:49.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:49.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.424 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 22:48:49.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.424 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:48:49.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:49.425 | INFO | Uploaded 36GiB to GPU in 2.52335s (14.2667GiB/s) | 20260507 22:48:49.425 | INFO | Downloaded 34GiB to GPU in 2.45618s (13.8426GiB/s) | 20260507 22:48:54.557 | INFO | Analyzing chunk frames = [3208192,3208704) | 20260507 22:48:54.557 | INFO | MemInfo: { | 20260507 22:48:54.557 | INFO | "gpu" : " 17 GB", | 20260507 22:48:54.557 | INFO | "huge" : "", | 20260507 22:48:54.557 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:54.557 | INFO | "malloc" : " 14 GB", | 20260507 22:48:54.557 | INFO | "pinned" : " 29 GB" | 20260507 22:48:54.557 | INFO | } | 20260507 22:48:54.557 | INFO | Perf report for the last 5.13286 seconds: | 20260507 22:48:54.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:54.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 22:48:54.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 22:48:54.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:54.557 | INFO | Uploaded 36GiB to GPU in 2.51975s (14.2872GiB/s) | 20260507 22:48:54.558 | INFO | Downloaded 34GiB to GPU in 2.47842s (13.7184GiB/s) | 20260507 22:48:59.697 | INFO | Analyzing chunk frames = [3208704,3209216) | 20260507 22:48:59.697 | INFO | MemInfo: { | 20260507 22:48:59.697 | INFO | "gpu" : " 17 GB", | 20260507 22:48:59.697 | INFO | "huge" : "", | 20260507 22:48:59.697 | INFO | "huge_shared" : " 29 GB", | 20260507 22:48:59.698 | INFO | "malloc" : " 14 GB", | 20260507 22:48:59.698 | INFO | "pinned" : " 29 GB" | 20260507 22:48:59.698 | INFO | } | 20260507 22:48:59.698 | INFO | Perf report for the last 5.14037 seconds: | 20260507 22:48:59.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:48:59.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:48:59.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:48:59.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:48:59.698 | INFO | Uploaded 36GiB to GPU in 2.49133s (14.4501GiB/s) | 20260507 22:48:59.698 | INFO | Downloaded 34GiB to GPU in 2.39891s (14.1731GiB/s) | 20260507 22:49:04.839 | INFO | Analyzing chunk frames = [3209216,3209728) | 20260507 22:49:04.839 | INFO | MemInfo: { | 20260507 22:49:04.839 | INFO | "gpu" : " 17 GB", | 20260507 22:49:04.839 | INFO | "huge" : "", | 20260507 22:49:04.839 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:04.839 | INFO | "malloc" : " 14 GB", | 20260507 22:49:04.839 | INFO | "pinned" : " 29 GB" | 20260507 22:49:04.839 | INFO | } | 20260507 22:49:04.840 | INFO | Perf report for the last 5.1419 seconds: | 20260507 22:49:04.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:04.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 22:49:04.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 22:49:04.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:04.840 | INFO | Uploaded 36GiB to GPU in 2.40501s (14.9688GiB/s) | 20260507 22:49:04.840 | INFO | Downloaded 34GiB to GPU in 2.49575s (13.6232GiB/s) | 20260507 22:49:09.978 | INFO | Analyzing chunk frames = [3209728,3210240) | 20260507 22:49:09.978 | INFO | MemInfo: { | 20260507 22:49:09.978 | INFO | "gpu" : " 17 GB", | 20260507 22:49:09.978 | INFO | "huge" : "", | 20260507 22:49:09.978 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:09.978 | INFO | "malloc" : " 14 GB", | 20260507 22:49:09.978 | INFO | "pinned" : " 29 GB" | 20260507 22:49:09.978 | INFO | } | 20260507 22:49:09.978 | INFO | Perf report for the last 5.1388 seconds: | 20260507 22:49:09.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:09.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:49:09.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:49:09.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:09.979 | INFO | Uploaded 36GiB to GPU in 2.48379s (14.494GiB/s) | 20260507 22:49:09.979 | INFO | Downloaded 34GiB to GPU in 2.43084s (13.987GiB/s) | 20260507 22:49:15.120 | INFO | Analyzing chunk frames = [3210240,3210752) | 20260507 22:49:15.120 | INFO | MemInfo: { | 20260507 22:49:15.120 | INFO | "gpu" : " 17 GB", | 20260507 22:49:15.120 | INFO | "huge" : "", | 20260507 22:49:15.120 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:15.120 | INFO | "malloc" : " 14 GB", | 20260507 22:49:15.120 | INFO | "pinned" : " 29 GB" | 20260507 22:49:15.120 | INFO | } | 20260507 22:49:15.121 | INFO | Perf report for the last 5.14217 seconds: | 20260507 22:49:15.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:15.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:49:15.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | PRE_HQ : 35.5%, 1.0, 18.4ms, 0 | 20260507 22:49:15.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:15.121 | INFO | Uploaded 36GiB to GPU in 2.51626s (14.3069GiB/s) | 20260507 22:49:15.121 | INFO | Downloaded 34GiB to GPU in 2.45293s (13.861GiB/s) | 20260507 22:49:20.259 | INFO | Analyzing chunk frames = [3210752,3211264) | 20260507 22:49:20.259 | INFO | MemInfo: { | 20260507 22:49:20.259 | INFO | "gpu" : " 17 GB", | 20260507 22:49:20.259 | INFO | "huge" : "", | 20260507 22:49:20.259 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:20.259 | INFO | "malloc" : " 14 GB", | 20260507 22:49:20.259 | INFO | "pinned" : " 29 GB" | 20260507 22:49:20.259 | INFO | } | 20260507 22:49:20.259 | INFO | Perf report for the last 5.13851 seconds: | 20260507 22:49:20.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:20.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.259 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:49:20.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.259 | INFO | PRE_HQ : 36.9%, 1.0, 19.2ms, 0 | 20260507 22:49:20.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:20.260 | INFO | Uploaded 36GiB to GPU in 2.55023s (14.1163GiB/s) | 20260507 22:49:20.260 | INFO | Downloaded 34GiB to GPU in 2.44913s (13.8825GiB/s) | 20260507 22:49:20.260 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:49:20.260 | INFO | Total mem usage: 16.7 GiB | 20260507 22:49:20.260 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:49:20.260 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260507 22:49:20.260 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:49:20.260 | INFO | High Water Breakdown: | 20260507 22:49:20.260 | INFO | 59.7% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:49:20.260 | INFO | 40.2% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:49:20.260 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:49:20.260 | INFO | ------------------------------------------ | 20260507 22:49:20.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:49:20.260 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:49:20.260 | INFO | Total mem usage: 26.5 GiB | 20260507 22:49:20.260 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:49:20.260 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 22:49:20.260 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 22:49:20.260 | INFO | High Water Breakdown: | 20260507 22:49:20.260 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:49:20.260 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:49:20.260 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:49:20.260 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:49:20.260 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:49:20.260 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:49:20.260 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:49:20.260 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:49:20.260 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:49:20.260 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:49:20.260 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:49:20.260 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:49:20.260 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:49:20.260 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:49:20.260 | INFO | ------------------------------------------ | 20260507 22:49:20.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:49:20.260 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:49:20.260 | INFO | Total mem usage: 27.8 GiB | 20260507 22:49:20.260 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:49:20.260 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:49:20.260 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:49:20.260 | INFO | High Water Breakdown: | 20260507 22:49:20.260 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:49:20.260 | INFO | ------------------------------------------ | 20260507 22:49:20.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:49:20.260 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:49:20.260 | INFO | Total mem usage: 15.7 GiB | 20260507 22:49:20.260 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:49:20.260 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:49:20.260 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:49:20.260 | INFO | High Water Breakdown: | 20260507 22:49:20.260 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:49:20.260 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:49:20.260 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:49:20.260 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:49:20.260 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:49:20.260 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:49:20.260 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:49:20.260 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:49:20.260 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:49:20.260 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:49:20.260 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:49:20.261 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:49:20.261 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:49:20.261 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:49:20.261 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:49:20.261 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:49:20.261 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:49:20.261 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:49:20.261 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:49:20.261 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:49:20.261 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:49:20.261 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:49:20.261 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:49:20.261 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:49:20.261 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:49:20.261 | INFO | ------------------------------------------ | 20260507 22:49:20.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:49:20.261 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:49:20.261 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:49:20.261 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:49:20.261 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.41e+01 / 1.77e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.04e-01, 1.18e-02 / 7.02e-03 | 20260507 22:49:20.261 | INFO | PerfReport: 12.13%, Baseline -- 4.00e+00 / 4.50e+00, 5.90e-01 / 1.68e-01 | 20260507 22:49:20.261 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.09e-01 / 1.14e-01 | 20260507 22:49:20.261 | INFO | PerfReport: 15.85%, DME -- 4.18e+01 / 4.53e+01, 3.99e+00 / 1.51e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.78e+00 / 2.02e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.45e+00 / 2.69e+00, 4.30e-01 / 2.67e-01 | 20260507 22:49:20.261 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.29e+00, 2.84e-01 / 1.41e-01 | 20260507 22:49:20.261 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.85e-01, 5.69e-02 / 2.75e-02 | 20260507 22:49:20.261 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:49:20.261 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:49:20.261 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:49:20.261 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 24.37%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 30.90%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 2.63%, LockDownload -- 2.80e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 15.82%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:49:20.261 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260507 22:49:20.372 | INFO | Starting a round of Baz IO | 20260507 22:49:23.840 | INFO | Populating RTMetrics report | 20260507 22:49:23.840 | INFO | RTMetrics report populated | 20260507 22:49:23.840 | INFO | Opening tmp RTMetrics json file | 20260507 22:49:23.846 | INFO | Serializing RTMetrics to json | 20260507 22:49:23.867 | INFO | Serialization done, swapping tmp file now... | 20260507 22:49:23.879 | INFO | RTMetrics done | 20260507 22:49:23.954 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.581 seconds | 20260507 22:49:23.954 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.92% / 0.01% / 57.08% | 20260507 22:49:23.954 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32583% / 3.97364e-06% / 7.54992e-05% | 20260507 22:49:23.954 | INFO | Baz IO overall: | 20260507 22:49:23.954 | INFO | BytesWritten : 1462883445084 | 20260507 22:49:23.954 | INFO | NumEvents : 708634235513 | 20260507 22:49:25.398 | INFO | Analyzing chunk frames = [3211264,3211776) | 20260507 22:49:25.398 | INFO | MemInfo: { | 20260507 22:49:25.398 | INFO | "gpu" : " 17 GB", | 20260507 22:49:25.398 | INFO | "huge" : "", | 20260507 22:49:25.398 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:25.398 | INFO | "malloc" : " 14 GB", | 20260507 22:49:25.398 | INFO | "pinned" : " 29 GB" | 20260507 22:49:25.398 | INFO | } | 20260507 22:49:25.398 | INFO | Perf report for the last 5.13907 seconds: | 20260507 22:49:25.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:25.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:25.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:25.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:25.398 | INFO | ANALYSIS : 69.3%, 2.9, 104.7ms, 0 | 20260507 22:49:25.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:25.398 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260507 22:49:25.398 | INFO | PRE_HQ : 68.3%, 1.8, 63.7ms, 0 | 20260507 22:49:25.398 | INFO | BAZWRITER : 69.3%, 3.9, 139.0ms, 0 | 20260507 22:49:25.398 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 22:49:25.399 | INFO | Uploaded 36GiB to GPU in 2.51875s (14.2928GiB/s) | 20260507 22:49:25.399 | INFO | Downloaded 38GiB to GPU in 2.68682s (14.1431GiB/s) | 20260507 22:49:30.537 | INFO | Analyzing chunk frames = [3211776,3212288) | 20260507 22:49:30.537 | INFO | MemInfo: { | 20260507 22:49:30.537 | INFO | "gpu" : " 17 GB", | 20260507 22:49:30.537 | INFO | "huge" : "", | 20260507 22:49:30.537 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:30.537 | INFO | "malloc" : " 14 GB", | 20260507 22:49:30.537 | INFO | "pinned" : " 29 GB" | 20260507 22:49:30.537 | INFO | } | 20260507 22:49:30.537 | INFO | Perf report for the last 5.13886 seconds: | 20260507 22:49:30.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:30.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 22:49:30.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:49:30.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:30.537 | INFO | Uploaded 36GiB to GPU in 2.56874s (14.0146GiB/s) | 20260507 22:49:30.537 | INFO | Downloaded 34GiB to GPU in 2.47928s (13.7137GiB/s) | 20260507 22:49:35.673 | INFO | Analyzing chunk frames = [3212288,3212800) | 20260507 22:49:35.673 | INFO | MemInfo: { | 20260507 22:49:35.673 | INFO | "gpu" : " 17 GB", | 20260507 22:49:35.673 | INFO | "huge" : "", | 20260507 22:49:35.673 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:35.673 | INFO | "malloc" : " 14 GB", | 20260507 22:49:35.673 | INFO | "pinned" : " 29 GB" | 20260507 22:49:35.673 | INFO | } | 20260507 22:49:35.674 | INFO | Perf report for the last 5.13652 seconds: | 20260507 22:49:35.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:35.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 22:49:35.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:49:35.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:35.674 | INFO | Uploaded 36GiB to GPU in 2.57192s (13.9973GiB/s) | 20260507 22:49:35.674 | INFO | Downloaded 34GiB to GPU in 2.47423s (13.7417GiB/s) | 20260507 22:49:40.813 | INFO | Analyzing chunk frames = [3212800,3213312) | 20260507 22:49:40.813 | INFO | MemInfo: { | 20260507 22:49:40.813 | INFO | "gpu" : " 17 GB", | 20260507 22:49:40.813 | INFO | "huge" : "", | 20260507 22:49:40.813 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:40.813 | INFO | "malloc" : " 14 GB", | 20260507 22:49:40.813 | INFO | "pinned" : " 29 GB" | 20260507 22:49:40.813 | INFO | } | 20260507 22:49:40.813 | INFO | Perf report for the last 5.13917 seconds: | 20260507 22:49:40.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:40.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.813 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260507 22:49:40.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.813 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:49:40.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:40.814 | INFO | Uploaded 36GiB to GPU in 2.47352s (14.5542GiB/s) | 20260507 22:49:40.814 | INFO | Downloaded 34GiB to GPU in 2.39526s (14.1947GiB/s) | 20260507 22:49:45.952 | INFO | Analyzing chunk frames = [3213312,3213824) | 20260507 22:49:45.952 | INFO | MemInfo: { | 20260507 22:49:45.952 | INFO | "gpu" : " 17 GB", | 20260507 22:49:45.952 | INFO | "huge" : "", | 20260507 22:49:45.952 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:45.952 | INFO | "malloc" : " 14 GB", | 20260507 22:49:45.952 | INFO | "pinned" : " 29 GB" | 20260507 22:49:45.952 | INFO | } | 20260507 22:49:45.952 | INFO | Perf report for the last 5.13904 seconds: | 20260507 22:49:45.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:45.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 22:49:45.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:49:45.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:45.952 | INFO | Uploaded 36GiB to GPU in 2.37672s (15.1469GiB/s) | 20260507 22:49:45.952 | INFO | Downloaded 34GiB to GPU in 2.49234s (13.6418GiB/s) | 20260507 22:49:51.090 | INFO | Analyzing chunk frames = [3213824,3214336) | 20260507 22:49:51.090 | INFO | MemInfo: { | 20260507 22:49:51.090 | INFO | "gpu" : " 17 GB", | 20260507 22:49:51.090 | INFO | "huge" : "", | 20260507 22:49:51.090 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:51.090 | INFO | "malloc" : " 14 GB", | 20260507 22:49:51.090 | INFO | "pinned" : " 29 GB" | 20260507 22:49:51.090 | INFO | } | 20260507 22:49:51.090 | INFO | Perf report for the last 5.13846 seconds: | 20260507 22:49:51.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:51.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.090 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 22:49:51.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.090 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:49:51.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:51.091 | INFO | Uploaded 36GiB to GPU in 2.49177s (14.4476GiB/s) | 20260507 22:49:51.091 | INFO | Downloaded 34GiB to GPU in 2.42119s (14.0427GiB/s) | 20260507 22:49:56.235 | INFO | Analyzing chunk frames = [3214336,3214848) | 20260507 22:49:56.235 | INFO | MemInfo: { | 20260507 22:49:56.235 | INFO | "gpu" : " 17 GB", | 20260507 22:49:56.235 | INFO | "huge" : "", | 20260507 22:49:56.235 | INFO | "huge_shared" : " 29 GB", | 20260507 22:49:56.235 | INFO | "malloc" : " 14 GB", | 20260507 22:49:56.235 | INFO | "pinned" : " 29 GB" | 20260507 22:49:56.235 | INFO | } | 20260507 22:49:56.236 | INFO | Perf report for the last 5.14526 seconds: | 20260507 22:49:56.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:49:56.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | ANALYSIS : 67.2%, 3.0, 105.1ms, 0 | 20260507 22:49:56.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:49:56.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:49:56.236 | INFO | Uploaded 36GiB to GPU in 2.51408s (14.3193GiB/s) | 20260507 22:49:56.236 | INFO | Downloaded 34GiB to GPU in 2.4467s (13.8963GiB/s) | 20260507 22:50:01.374 | INFO | Analyzing chunk frames = [3214848,3215360) | 20260507 22:50:01.374 | INFO | MemInfo: { | 20260507 22:50:01.374 | INFO | "gpu" : " 17 GB", | 20260507 22:50:01.374 | INFO | "huge" : "", | 20260507 22:50:01.374 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:01.374 | INFO | "malloc" : " 14 GB", | 20260507 22:50:01.374 | INFO | "pinned" : " 29 GB" | 20260507 22:50:01.374 | INFO | } | 20260507 22:50:01.374 | INFO | Perf report for the last 5.13842 seconds: | 20260507 22:50:01.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:01.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:50:01.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:50:01.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:01.374 | INFO | Uploaded 36GiB to GPU in 2.52553s (14.2544GiB/s) | 20260507 22:50:01.374 | INFO | Downloaded 34GiB to GPU in 2.44572s (13.9018GiB/s) | 20260507 22:50:01.471 | INFO | Starting a round of Baz IO | 20260507 22:50:04.947 | INFO | Populating RTMetrics report | 20260507 22:50:04.948 | INFO | RTMetrics report populated | 20260507 22:50:04.948 | INFO | Opening tmp RTMetrics json file | 20260507 22:50:04.966 | INFO | Serializing RTMetrics to json | 20260507 22:50:04.993 | INFO | Serialization done, swapping tmp file now... | 20260507 22:50:05.062 | INFO | RTMetrics done | 20260507 22:50:05.092 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.62 seconds | 20260507 22:50:05.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.91% / 0.01% / 57.08% | 20260507 22:50:05.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32864% / 1.19209e-05% / 5.5631e-05% | 20260507 22:50:05.092 | INFO | Baz IO overall: | 20260507 22:50:05.092 | INFO | BytesWritten : 1464739925831 | 20260507 22:50:05.092 | INFO | NumEvents : 709530104663 | 20260507 22:50:06.508 | INFO | Analyzing chunk frames = [3215360,3215872) | 20260507 22:50:06.508 | INFO | MemInfo: { | 20260507 22:50:06.508 | INFO | "gpu" : " 17 GB", | 20260507 22:50:06.508 | INFO | "huge" : "", | 20260507 22:50:06.508 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:06.508 | INFO | "malloc" : " 14 GB", | 20260507 22:50:06.508 | INFO | "pinned" : " 29 GB" | 20260507 22:50:06.508 | INFO | } | 20260507 22:50:06.508 | INFO | Perf report for the last 5.13379 seconds: | 20260507 22:50:06.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:06.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:06.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:06.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:06.508 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 22:50:06.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:06.508 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 22:50:06.508 | INFO | PRE_HQ : 65.6%, 1.7, 57.1ms, 0 | 20260507 22:50:06.508 | INFO | BAZWRITER : 70.1%, 3.6, 128.8ms, 0 | 20260507 22:50:06.508 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 22:50:06.508 | INFO | Uploaded 36GiB to GPU in 2.54061s (14.1698GiB/s) | 20260507 22:50:06.508 | INFO | Downloaded 38GiB to GPU in 2.68523s (14.1515GiB/s) | 20260507 22:50:11.648 | INFO | Analyzing chunk frames = [3215872,3216384) | 20260507 22:50:11.648 | INFO | MemInfo: { | 20260507 22:50:11.648 | INFO | "gpu" : " 17 GB", | 20260507 22:50:11.648 | INFO | "huge" : "", | 20260507 22:50:11.648 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:11.649 | INFO | "malloc" : " 14 GB", | 20260507 22:50:11.649 | INFO | "pinned" : " 29 GB" | 20260507 22:50:11.649 | INFO | } | 20260507 22:50:11.649 | INFO | Perf report for the last 5.14089 seconds: | 20260507 22:50:11.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:11.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 22:50:11.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 22:50:11.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:11.649 | INFO | Uploaded 36GiB to GPU in 2.51341s (14.3232GiB/s) | 20260507 22:50:11.649 | INFO | Downloaded 34GiB to GPU in 2.47052s (13.7623GiB/s) | 20260507 22:50:16.788 | INFO | Analyzing chunk frames = [3216384,3216896) | 20260507 22:50:16.788 | INFO | MemInfo: { | 20260507 22:50:16.788 | INFO | "gpu" : " 17 GB", | 20260507 22:50:16.788 | INFO | "huge" : "", | 20260507 22:50:16.788 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:16.788 | INFO | "malloc" : " 14 GB", | 20260507 22:50:16.788 | INFO | "pinned" : " 29 GB" | 20260507 22:50:16.788 | INFO | } | 20260507 22:50:16.788 | INFO | Perf report for the last 5.13958 seconds: | 20260507 22:50:16.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:16.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:50:16.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:50:16.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:16.788 | INFO | Uploaded 36GiB to GPU in 2.50395s (14.3773GiB/s) | 20260507 22:50:16.788 | INFO | Downloaded 34GiB to GPU in 2.48734s (13.6692GiB/s) | 20260507 22:50:21.929 | INFO | Analyzing chunk frames = [3216896,3217408) | 20260507 22:50:21.929 | INFO | MemInfo: { | 20260507 22:50:21.929 | INFO | "gpu" : " 17 GB", | 20260507 22:50:21.929 | INFO | "huge" : "", | 20260507 22:50:21.929 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:21.929 | INFO | "malloc" : " 14 GB", | 20260507 22:50:21.929 | INFO | "pinned" : " 29 GB" | 20260507 22:50:21.929 | INFO | } | 20260507 22:50:21.929 | INFO | Perf report for the last 5.141 seconds: | 20260507 22:50:21.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:21.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.929 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:50:21.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.929 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:50:21.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:21.930 | INFO | Uploaded 36GiB to GPU in 2.46755s (14.5894GiB/s) | 20260507 22:50:21.930 | INFO | Downloaded 34GiB to GPU in 2.3811s (14.2791GiB/s) | 20260507 22:50:27.070 | INFO | Analyzing chunk frames = [3217408,3217920) | 20260507 22:50:27.070 | INFO | MemInfo: { | 20260507 22:50:27.070 | INFO | "gpu" : " 17 GB", | 20260507 22:50:27.070 | INFO | "huge" : "", | 20260507 22:50:27.070 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:27.070 | INFO | "malloc" : " 14 GB", | 20260507 22:50:27.070 | INFO | "pinned" : " 29 GB" | 20260507 22:50:27.070 | INFO | } | 20260507 22:50:27.070 | INFO | Perf report for the last 5.141 seconds: | 20260507 22:50:27.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:27.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 22:50:27.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:50:27.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:27.070 | INFO | Uploaded 36GiB to GPU in 2.39394s (15.0379GiB/s) | 20260507 22:50:27.071 | INFO | Downloaded 34GiB to GPU in 2.49479s (13.6284GiB/s) | 20260507 22:50:32.207 | INFO | Analyzing chunk frames = [3217920,3218432) | 20260507 22:50:32.207 | INFO | MemInfo: { | 20260507 22:50:32.207 | INFO | "gpu" : " 17 GB", | 20260507 22:50:32.207 | INFO | "huge" : "", | 20260507 22:50:32.207 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:32.207 | INFO | "malloc" : " 14 GB", | 20260507 22:50:32.207 | INFO | "pinned" : " 29 GB" | 20260507 22:50:32.207 | INFO | } | 20260507 22:50:32.207 | INFO | Perf report for the last 5.1367 seconds: | 20260507 22:50:32.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:32.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.207 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:50:32.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.207 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:50:32.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:32.208 | INFO | Uploaded 36GiB to GPU in 2.42657s (14.8358GiB/s) | 20260507 22:50:32.208 | INFO | Downloaded 34GiB to GPU in 2.38445s (14.2591GiB/s) | 20260507 22:50:37.351 | INFO | Analyzing chunk frames = [3218432,3218944) | 20260507 22:50:37.351 | INFO | MemInfo: { | 20260507 22:50:37.351 | INFO | "gpu" : " 17 GB", | 20260507 22:50:37.351 | INFO | "huge" : "", | 20260507 22:50:37.351 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:37.351 | INFO | "malloc" : " 14 GB", | 20260507 22:50:37.351 | INFO | "pinned" : " 29 GB" | 20260507 22:50:37.351 | INFO | } | 20260507 22:50:37.351 | INFO | Perf report for the last 5.14408 seconds: | 20260507 22:50:37.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:37.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260507 22:50:37.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:50:37.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:37.351 | INFO | Uploaded 36GiB to GPU in 2.54175s (14.1635GiB/s) | 20260507 22:50:37.351 | INFO | Downloaded 34GiB to GPU in 2.43897s (13.9403GiB/s) | 20260507 22:50:42.491 | INFO | Analyzing chunk frames = [3218944,3219456) | 20260507 22:50:42.491 | INFO | MemInfo: { | 20260507 22:50:42.491 | INFO | "gpu" : " 17 GB", | 20260507 22:50:42.491 | INFO | "huge" : "", | 20260507 22:50:42.491 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:42.491 | INFO | "malloc" : " 14 GB", | 20260507 22:50:42.491 | INFO | "pinned" : " 29 GB" | 20260507 22:50:42.491 | INFO | } | 20260507 22:50:42.491 | INFO | Perf report for the last 5.13992 seconds: | 20260507 22:50:42.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:42.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.491 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 22:50:42.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.491 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:50:42.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:42.492 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:50:42.492 | INFO | Total mem usage: 16.7 GiB | 20260507 22:50:42.492 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:50:42.492 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.38%) | 20260507 22:50:42.492 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:50:42.492 | INFO | Uploaded 36GiB to GPU in 2.52392s (14.2635GiB/s) | 20260507 22:50:42.492 | INFO | High Water Breakdown: | 20260507 22:50:42.492 | INFO | Downloaded 34GiB to GPU in 2.43617s (13.9563GiB/s) | 20260507 22:50:42.492 | INFO | 60.6% : 9.33 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:50:42.492 | INFO | 39.3% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:50:42.492 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:50:42.492 | INFO | ------------------------------------------ | 20260507 22:50:42.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:50:42.492 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:50:42.492 | INFO | Total mem usage: 26.5 GiB | 20260507 22:50:42.492 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:50:42.492 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 22:50:42.492 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:50:42.492 | INFO | High Water Breakdown: | 20260507 22:50:42.492 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:50:42.492 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:50:42.492 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:50:42.492 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:50:42.492 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:50:42.492 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:50:42.492 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:50:42.492 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:50:42.492 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:50:42.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:50:42.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:50:42.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:50:42.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:50:42.492 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:50:42.492 | INFO | ------------------------------------------ | 20260507 22:50:42.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:50:42.492 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:50:42.492 | INFO | Total mem usage: 27.8 GiB | 20260507 22:50:42.492 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:50:42.492 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:50:42.492 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:50:42.492 | INFO | High Water Breakdown: | 20260507 22:50:42.492 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:50:42.492 | INFO | ------------------------------------------ | 20260507 22:50:42.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:50:42.492 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:50:42.492 | INFO | Total mem usage: 15.7 GiB | 20260507 22:50:42.492 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:50:42.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:50:42.492 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:50:42.492 | INFO | High Water Breakdown: | 20260507 22:50:42.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:50:42.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:50:42.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:50:42.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:50:42.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:50:42.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:50:42.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:50:42.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:50:42.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:50:42.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:50:42.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:50:42.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:50:42.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:50:42.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:50:42.493 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:50:42.493 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:50:42.493 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:50:42.493 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:50:42.493 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:50:42.493 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:50:42.493 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:50:42.493 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:50:42.493 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:50:42.493 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:50:42.493 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:50:42.493 | INFO | ------------------------------------------ | 20260507 22:50:42.493 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:50:42.493 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:50:42.493 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:50:42.493 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:50:42.493 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.61e+01, 1.41e+01 / 1.62e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.15e-02 / 5.11e-03 | 20260507 22:50:42.493 | INFO | PerfReport: 12.14%, Baseline -- 4.01e+00 / 4.45e+00, 5.94e-01 / 2.73e-01 | 20260507 22:50:42.493 | INFO | PerfReport: 3.38%, Histogram -- 1.12e+00 / 1.48e+00, 2.08e-01 / 1.33e-01 | 20260507 22:50:42.493 | INFO | PerfReport: 15.84%, DME -- 4.18e+01 / 4.54e+01, 3.91e+00 / 1.63e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.84e+00 / 1.78e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.29e-01 / 2.21e-01 | 20260507 22:50:42.493 | INFO | PerfReport: 3.44%, Metrics -- 1.14e+00 / 1.36e+00, 2.89e-01 / 1.57e-01 | 20260507 22:50:42.493 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.89e-01, 5.71e-02 / 3.35e-02 | 20260507 22:50:42.493 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:50:42.493 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:50:42.493 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:50:42.493 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 24.39%, Upload -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 24.19%, Download -- 2.58e+01 / 0.00e+00, 8.51e+00 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 30.94%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 2.03%, LockUpload -- 2.16e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 2.52%, LockDownload -- 2.69e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 15.92%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:50:42.493 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 22:50:42.601 | INFO | Starting a round of Baz IO | 20260507 22:50:46.053 | INFO | Populating RTMetrics report | 20260507 22:50:46.053 | INFO | RTMetrics report populated | 20260507 22:50:46.053 | INFO | Opening tmp RTMetrics json file | 20260507 22:50:46.058 | INFO | Serializing RTMetrics to json | 20260507 22:50:46.088 | INFO | Serialization done, swapping tmp file now... | 20260507 22:50:46.117 | INFO | RTMetrics done | 20260507 22:50:46.176 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.575 seconds | 20260507 22:50:46.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.90% / 0.01% / 57.09% | 20260507 22:50:46.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32343% / 1.19209e-05% / 7.54992e-05% | 20260507 22:50:46.176 | INFO | Baz IO overall: | 20260507 22:50:46.176 | INFO | BytesWritten : 1466596467578 | 20260507 22:50:46.176 | INFO | NumEvents : 710426020238 | 20260507 22:50:47.628 | INFO | Analyzing chunk frames = [3219456,3219968) | 20260507 22:50:47.628 | INFO | MemInfo: { | 20260507 22:50:47.628 | INFO | "gpu" : " 17 GB", | 20260507 22:50:47.628 | INFO | "huge" : "", | 20260507 22:50:47.628 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:47.628 | INFO | "malloc" : " 14 GB", | 20260507 22:50:47.628 | INFO | "pinned" : " 29 GB" | 20260507 22:50:47.628 | INFO | } | 20260507 22:50:47.628 | INFO | Perf report for the last 5.1374 seconds: | 20260507 22:50:47.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:47.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:47.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:47.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:47.629 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 22:50:47.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:47.629 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 22:50:47.629 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260507 22:50:47.629 | INFO | BAZWRITER : 69.6%, 3.9, 139.6ms, 0 | 20260507 22:50:47.629 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260507 22:50:47.629 | INFO | Uploaded 36GiB to GPU in 2.55221s (14.1054GiB/s) | 20260507 22:50:47.629 | INFO | Downloaded 38GiB to GPU in 2.68778s (14.1381GiB/s) | 20260507 22:50:52.763 | INFO | Analyzing chunk frames = [3219968,3220480) | 20260507 22:50:52.763 | INFO | MemInfo: { | 20260507 22:50:52.763 | INFO | "gpu" : " 17 GB", | 20260507 22:50:52.763 | INFO | "huge" : "", | 20260507 22:50:52.763 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:52.763 | INFO | "malloc" : " 14 GB", | 20260507 22:50:52.763 | INFO | "pinned" : " 29 GB" | 20260507 22:50:52.763 | INFO | } | 20260507 22:50:52.763 | INFO | Perf report for the last 5.13474 seconds: | 20260507 22:50:52.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:52.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 22:50:52.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:50:52.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:52.763 | INFO | Uploaded 36GiB to GPU in 2.6028s (13.8312GiB/s) | 20260507 22:50:52.763 | INFO | Downloaded 34GiB to GPU in 2.47643s (13.7295GiB/s) | 20260507 22:50:57.902 | INFO | Analyzing chunk frames = [3220480,3220992) | 20260507 22:50:57.902 | INFO | MemInfo: { | 20260507 22:50:57.902 | INFO | "gpu" : " 17 GB", | 20260507 22:50:57.902 | INFO | "huge" : "", | 20260507 22:50:57.902 | INFO | "huge_shared" : " 29 GB", | 20260507 22:50:57.902 | INFO | "malloc" : " 14 GB", | 20260507 22:50:57.902 | INFO | "pinned" : " 29 GB" | 20260507 22:50:57.902 | INFO | } | 20260507 22:50:57.902 | INFO | Perf report for the last 5.13879 seconds: | 20260507 22:50:57.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:50:57.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 22:50:57.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:50:57.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:50:57.902 | INFO | Uploaded 36GiB to GPU in 2.59142s (13.892GiB/s) | 20260507 22:50:57.902 | INFO | Downloaded 34GiB to GPU in 2.4641s (13.7981GiB/s) | 20260507 22:51:03.042 | INFO | Analyzing chunk frames = [3220992,3221504) | 20260507 22:51:03.042 | INFO | MemInfo: { | 20260507 22:51:03.042 | INFO | "gpu" : " 17 GB", | 20260507 22:51:03.042 | INFO | "huge" : "", | 20260507 22:51:03.042 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:03.042 | INFO | "malloc" : " 14 GB", | 20260507 22:51:03.042 | INFO | "pinned" : " 29 GB" | 20260507 22:51:03.042 | INFO | } | 20260507 22:51:03.042 | INFO | Perf report for the last 5.14023 seconds: | 20260507 22:51:03.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:03.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.042 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:51:03.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.042 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 22:51:03.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:03.043 | INFO | Uploaded 36GiB to GPU in 2.48849s (14.4666GiB/s) | 20260507 22:51:03.043 | INFO | Downloaded 34GiB to GPU in 2.38537s (14.2536GiB/s) | 20260507 22:51:08.186 | INFO | Analyzing chunk frames = [3221504,3222016) | 20260507 22:51:08.186 | INFO | MemInfo: { | 20260507 22:51:08.186 | INFO | "gpu" : " 17 GB", | 20260507 22:51:08.186 | INFO | "huge" : "", | 20260507 22:51:08.186 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:08.187 | INFO | "malloc" : " 14 GB", | 20260507 22:51:08.187 | INFO | "pinned" : " 29 GB" | 20260507 22:51:08.187 | INFO | } | 20260507 22:51:08.187 | INFO | Perf report for the last 5.14437 seconds: | 20260507 22:51:08.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:08.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260507 22:51:08.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 22:51:08.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:08.187 | INFO | Uploaded 36GiB to GPU in 2.42344s (14.8549GiB/s) | 20260507 22:51:08.187 | INFO | Downloaded 34GiB to GPU in 2.49214s (13.6429GiB/s) | 20260507 22:51:13.325 | INFO | Analyzing chunk frames = [3222016,3222528) | 20260507 22:51:13.325 | INFO | MemInfo: { | 20260507 22:51:13.325 | INFO | "gpu" : " 17 GB", | 20260507 22:51:13.325 | INFO | "huge" : "", | 20260507 22:51:13.325 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:13.325 | INFO | "malloc" : " 14 GB", | 20260507 22:51:13.325 | INFO | "pinned" : " 29 GB" | 20260507 22:51:13.325 | INFO | } | 20260507 22:51:13.325 | INFO | Perf report for the last 5.13885 seconds: | 20260507 22:51:13.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:13.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.326 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 22:51:13.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.326 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:51:13.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:13.326 | INFO | Uploaded 36GiB to GPU in 2.41165s (14.9276GiB/s) | 20260507 22:51:13.326 | INFO | Downloaded 34GiB to GPU in 2.39333s (14.2061GiB/s) | 20260507 22:51:18.461 | INFO | Analyzing chunk frames = [3222528,3223040) | 20260507 22:51:18.461 | INFO | MemInfo: { | 20260507 22:51:18.461 | INFO | "gpu" : " 17 GB", | 20260507 22:51:18.461 | INFO | "huge" : "", | 20260507 22:51:18.461 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:18.461 | INFO | "malloc" : " 14 GB", | 20260507 22:51:18.461 | INFO | "pinned" : " 29 GB" | 20260507 22:51:18.461 | INFO | } | 20260507 22:51:18.461 | INFO | Perf report for the last 5.13571 seconds: | 20260507 22:51:18.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:18.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:51:18.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:51:18.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:18.461 | INFO | Uploaded 36GiB to GPU in 2.52438s (14.2609GiB/s) | 20260507 22:51:18.462 | INFO | Downloaded 34GiB to GPU in 2.44836s (13.8868GiB/s) | 20260507 22:51:23.606 | INFO | Analyzing chunk frames = [3223040,3223552) | 20260507 22:51:23.606 | INFO | MemInfo: { | 20260507 22:51:23.606 | INFO | "gpu" : " 17 GB", | 20260507 22:51:23.606 | INFO | "huge" : "", | 20260507 22:51:23.606 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:23.606 | INFO | "malloc" : " 14 GB", | 20260507 22:51:23.606 | INFO | "pinned" : " 29 GB" | 20260507 22:51:23.606 | INFO | } | 20260507 22:51:23.606 | INFO | Perf report for the last 5.14502 seconds: | 20260507 22:51:23.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:23.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.606 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:51:23.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.606 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:51:23.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:23.607 | INFO | Uploaded 36GiB to GPU in 2.50183s (14.3895GiB/s) | 20260507 22:51:23.607 | INFO | Downloaded 34GiB to GPU in 2.44116s (13.9278GiB/s) | 20260507 22:51:23.703 | INFO | Starting a round of Baz IO | 20260507 22:51:27.156 | INFO | Populating RTMetrics report | 20260507 22:51:27.157 | INFO | RTMetrics report populated | 20260507 22:51:27.157 | INFO | Opening tmp RTMetrics json file | 20260507 22:51:27.164 | INFO | Serializing RTMetrics to json | 20260507 22:51:27.187 | INFO | Serialization done, swapping tmp file now... | 20260507 22:51:27.195 | INFO | RTMetrics done | 20260507 22:51:27.271 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.567 seconds | 20260507 22:51:27.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.90% / 0.01% / 57.10% | 20260507 22:51:27.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32206% / 1.98682e-05% / 5.16574e-05% | 20260507 22:51:27.271 | INFO | Baz IO overall: | 20260507 22:51:27.271 | INFO | BytesWritten : 1468452795164 | 20260507 22:51:27.271 | INFO | NumEvents : 711322058223 | 20260507 22:51:28.741 | INFO | Analyzing chunk frames = [3223552,3224064) | 20260507 22:51:28.741 | INFO | MemInfo: { | 20260507 22:51:28.741 | INFO | "gpu" : " 17 GB", | 20260507 22:51:28.741 | INFO | "huge" : "", | 20260507 22:51:28.741 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:28.741 | INFO | "malloc" : " 14 GB", | 20260507 22:51:28.741 | INFO | "pinned" : " 29 GB" | 20260507 22:51:28.741 | INFO | } | 20260507 22:51:28.741 | INFO | Perf report for the last 5.13498 seconds: | 20260507 22:51:28.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:28.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:28.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:28.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:28.741 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260507 22:51:28.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:28.741 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 22:51:28.741 | INFO | PRE_HQ : 65.7%, 1.7, 57.4ms, 0 | 20260507 22:51:28.741 | INFO | BAZWRITER : 69.1%, 3.9, 138.0ms, 0 | 20260507 22:51:28.741 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 22:51:28.741 | INFO | Uploaded 36GiB to GPU in 2.54833s (14.1269GiB/s) | 20260507 22:51:28.742 | INFO | Downloaded 38GiB to GPU in 2.67981s (14.1801GiB/s) | 20260507 22:51:33.878 | INFO | Analyzing chunk frames = [3224064,3224576) | 20260507 22:51:33.878 | INFO | MemInfo: { | 20260507 22:51:33.878 | INFO | "gpu" : " 17 GB", | 20260507 22:51:33.878 | INFO | "huge" : "", | 20260507 22:51:33.878 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:33.878 | INFO | "malloc" : " 14 GB", | 20260507 22:51:33.878 | INFO | "pinned" : " 29 GB" | 20260507 22:51:33.878 | INFO | } | 20260507 22:51:33.878 | INFO | Perf report for the last 5.13687 seconds: | 20260507 22:51:33.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:33.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.878 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 22:51:33.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.878 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 22:51:33.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:33.879 | INFO | Uploaded 36GiB to GPU in 2.56378s (14.0418GiB/s) | 20260507 22:51:33.879 | INFO | Downloaded 34GiB to GPU in 2.50293s (13.5841GiB/s) | 20260507 22:51:39.017 | INFO | Analyzing chunk frames = [3224576,3225088) | 20260507 22:51:39.017 | INFO | MemInfo: { | 20260507 22:51:39.017 | INFO | "gpu" : " 17 GB", | 20260507 22:51:39.017 | INFO | "huge" : "", | 20260507 22:51:39.017 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:39.017 | INFO | "malloc" : " 14 GB", | 20260507 22:51:39.017 | INFO | "pinned" : " 29 GB" | 20260507 22:51:39.017 | INFO | } | 20260507 22:51:39.017 | INFO | Perf report for the last 5.13868 seconds: | 20260507 22:51:39.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:39.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 22:51:39.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:51:39.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:39.017 | INFO | Uploaded 36GiB to GPU in 2.49352s (14.4374GiB/s) | 20260507 22:51:39.017 | INFO | Downloaded 34GiB to GPU in 2.48062s (13.7063GiB/s) | 20260507 22:51:44.161 | INFO | Analyzing chunk frames = [3225088,3225600) | 20260507 22:51:44.161 | INFO | MemInfo: { | 20260507 22:51:44.161 | INFO | "gpu" : " 17 GB", | 20260507 22:51:44.161 | INFO | "huge" : "", | 20260507 22:51:44.161 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:44.161 | INFO | "malloc" : " 14 GB", | 20260507 22:51:44.161 | INFO | "pinned" : " 29 GB" | 20260507 22:51:44.161 | INFO | } | 20260507 22:51:44.161 | INFO | Perf report for the last 5.14446 seconds: | 20260507 22:51:44.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:44.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.161 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 22:51:44.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.161 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:51:44.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:44.162 | INFO | Uploaded 36GiB to GPU in 2.47568s (14.5415GiB/s) | 20260507 22:51:44.162 | INFO | Downloaded 34GiB to GPU in 2.38862s (14.2342GiB/s) | 20260507 22:51:49.302 | INFO | Analyzing chunk frames = [3225600,3226112) | 20260507 22:51:49.302 | INFO | MemInfo: { | 20260507 22:51:49.302 | INFO | "gpu" : " 17 GB", | 20260507 22:51:49.302 | INFO | "huge" : "", | 20260507 22:51:49.302 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:49.303 | INFO | "malloc" : " 14 GB", | 20260507 22:51:49.303 | INFO | "pinned" : " 29 GB" | 20260507 22:51:49.303 | INFO | } | 20260507 22:51:49.303 | INFO | Perf report for the last 5.14126 seconds: | 20260507 22:51:49.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:49.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:51:49.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:51:49.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:49.303 | INFO | Uploaded 36GiB to GPU in 2.35113s (15.3118GiB/s) | 20260507 22:51:49.303 | INFO | Downloaded 34GiB to GPU in 2.49915s (13.6046GiB/s) | 20260507 22:51:54.438 | INFO | Analyzing chunk frames = [3226112,3226624) | 20260507 22:51:54.438 | INFO | MemInfo: { | 20260507 22:51:54.438 | INFO | "gpu" : " 17 GB", | 20260507 22:51:54.438 | INFO | "huge" : "", | 20260507 22:51:54.438 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:54.438 | INFO | "malloc" : " 14 GB", | 20260507 22:51:54.438 | INFO | "pinned" : " 29 GB" | 20260507 22:51:54.438 | INFO | } | 20260507 22:51:54.438 | INFO | Perf report for the last 5.1356 seconds: | 20260507 22:51:54.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:54.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.438 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:51:54.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.438 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 22:51:54.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:54.439 | INFO | Uploaded 36GiB to GPU in 2.43515s (14.7835GiB/s) | 20260507 22:51:54.439 | INFO | Downloaded 34GiB to GPU in 2.40067s (14.1627GiB/s) | 20260507 22:51:59.576 | INFO | Analyzing chunk frames = [3226624,3227136) | 20260507 22:51:59.576 | INFO | MemInfo: { | 20260507 22:51:59.576 | INFO | "gpu" : " 17 GB", | 20260507 22:51:59.576 | INFO | "huge" : "", | 20260507 22:51:59.577 | INFO | "huge_shared" : " 29 GB", | 20260507 22:51:59.577 | INFO | "malloc" : " 14 GB", | 20260507 22:51:59.577 | INFO | "pinned" : " 29 GB" | 20260507 22:51:59.577 | INFO | } | 20260507 22:51:59.577 | INFO | Perf report for the last 5.1385 seconds: | 20260507 22:51:59.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:51:59.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:51:59.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:51:59.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:51:59.577 | INFO | Uploaded 36GiB to GPU in 2.53404s (14.2066GiB/s) | 20260507 22:51:59.577 | INFO | Downloaded 34GiB to GPU in 2.43449s (13.966GiB/s) | 20260507 22:52:04.716 | INFO | Analyzing chunk frames = [3227136,3227648) | 20260507 22:52:04.716 | INFO | MemInfo: { | 20260507 22:52:04.716 | INFO | "gpu" : " 17 GB", | 20260507 22:52:04.716 | INFO | "huge" : "", | 20260507 22:52:04.716 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:04.716 | INFO | "malloc" : " 14 GB", | 20260507 22:52:04.717 | INFO | "pinned" : " 29 GB" | 20260507 22:52:04.717 | INFO | } | 20260507 22:52:04.717 | INFO | Perf report for the last 5.13989 seconds: | 20260507 22:52:04.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:04.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 22:52:04.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:52:04.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:04.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:52:04.717 | INFO | Total mem usage: 16.7 GiB | 20260507 22:52:04.717 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:52:04.717 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.5%) | 20260507 22:52:04.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:52:04.717 | INFO | High Water Breakdown: | 20260507 22:52:04.717 | INFO | 59.9% : 8.89 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:52:04.717 | INFO | 40% : 5.92 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:52:04.717 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:52:04.717 | INFO | ------------------------------------------ | 20260507 22:52:04.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:52:04.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:52:04.717 | INFO | Total mem usage: 26.5 GiB | 20260507 22:52:04.717 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:52:04.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:52:04.717 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:52:04.717 | INFO | High Water Breakdown: | 20260507 22:52:04.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:52:04.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:52:04.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:52:04.717 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:52:04.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:52:04.717 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:52:04.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:52:04.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:52:04.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:52:04.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:52:04.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:52:04.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:52:04.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:52:04.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:52:04.717 | INFO | ------------------------------------------ | 20260507 22:52:04.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:52:04.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:52:04.717 | INFO | Total mem usage: 27.8 GiB | 20260507 22:52:04.717 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:52:04.717 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260507 22:52:04.717 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260507 22:52:04.717 | INFO | High Water Breakdown: | 20260507 22:52:04.717 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260507 22:52:04.717 | INFO | ------------------------------------------ | 20260507 22:52:04.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:52:04.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:52:04.717 | INFO | Total mem usage: 15.7 GiB | 20260507 22:52:04.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:52:04.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:52:04.717 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:52:04.717 | INFO | High Water Breakdown: | 20260507 22:52:04.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:52:04.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:52:04.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:52:04.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:52:04.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:52:04.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:52:04.717 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:52:04.717 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:52:04.717 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:52:04.717 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:52:04.717 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:52:04.717 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:52:04.717 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:52:04.717 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:52:04.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:52:04.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:52:04.717 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:52:04.717 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:52:04.717 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:52:04.717 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:52:04.717 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:52:04.717 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:52:04.717 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:52:04.717 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:52:04.717 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:52:04.717 | INFO | ------------------------------------------ | 20260507 22:52:04.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:52:04.717 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:52:04.717 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:52:04.717 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:52:04.717 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.59e+01, 1.41e+01 / 1.64e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.03e-01, 1.20e-02 / 4.52e-03 | 20260507 22:52:04.717 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.51e+00, 6.00e-01 / 1.99e-01 | 20260507 22:52:04.717 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.11e-01 / 9.94e-02 | 20260507 22:52:04.717 | INFO | PerfReport: 15.89%, DME -- 4.18e+01 / 4.54e+01, 3.73e+00 / 1.20e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.76e+00 / 1.66e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.30e-01 / 1.91e-01 | 20260507 22:52:04.717 | INFO | PerfReport: 3.47%, Metrics -- 1.14e+00 / 1.35e+00, 2.89e-01 / 1.65e-01 | 20260507 22:52:04.717 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.52e-01 / 3.88e-01, 5.83e-02 / 3.20e-02 | 20260507 22:52:04.717 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:52:04.717 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:52:04.717 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:52:04.717 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 24.18%, Download -- 2.58e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 30.85%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 2.19%, LockUpload -- 2.34e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 2.40%, LockDownload -- 2.56e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 15.94%, LockCompute -- 1.70e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 22:52:04.717 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.44e-02 / 0.00e+00 | 20260507 22:52:04.717 | INFO | Uploaded 36GiB to GPU in 2.55119s (14.1111GiB/s) | 20260507 22:52:04.717 | INFO | Downloaded 34GiB to GPU in 2.43924s (13.9388GiB/s) | 20260507 22:52:04.814 | INFO | Starting a round of Baz IO | 20260507 22:52:08.283 | INFO | Populating RTMetrics report | 20260507 22:52:08.284 | INFO | RTMetrics report populated | 20260507 22:52:08.284 | INFO | Opening tmp RTMetrics json file | 20260507 22:52:08.327 | INFO | Serializing RTMetrics to json | 20260507 22:52:08.362 | INFO | Serialization done, swapping tmp file now... | 20260507 22:52:08.367 | INFO | RTMetrics done | 20260507 22:52:08.392 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.577 seconds | 20260507 22:52:08.392 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.89% / 0.01% / 57.10% | 20260507 22:52:08.392 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31927% / 7.94729e-06% / 4.76837e-05% | 20260507 22:52:08.392 | INFO | Baz IO overall: | 20260507 22:52:08.392 | INFO | BytesWritten : 1470308493219 | 20260507 22:52:08.392 | INFO | NumEvents : 712217592242 | 20260507 22:52:09.853 | INFO | Analyzing chunk frames = [3227648,3228160) | 20260507 22:52:09.853 | INFO | MemInfo: { | 20260507 22:52:09.853 | INFO | "gpu" : " 17 GB", | 20260507 22:52:09.853 | INFO | "huge" : "", | 20260507 22:52:09.853 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:09.853 | INFO | "malloc" : " 14 GB", | 20260507 22:52:09.853 | INFO | "pinned" : " 29 GB" | 20260507 22:52:09.853 | INFO | } | 20260507 22:52:09.853 | INFO | Perf report for the last 5.13647 seconds: | 20260507 22:52:09.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:09.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:09.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:09.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:09.853 | INFO | ANALYSIS : 68.7%, 3.0, 107.0ms, 0 | 20260507 22:52:09.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:09.853 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260507 22:52:09.853 | INFO | PRE_HQ : 66.3%, 1.8, 60.3ms, 0 | 20260507 22:52:09.853 | INFO | BAZWRITER : 69.3%, 3.8, 134.2ms, 0 | 20260507 22:52:09.853 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 22:52:09.854 | INFO | Uploaded 36GiB to GPU in 2.57138s (14.0002GiB/s) | 20260507 22:52:09.854 | INFO | Downloaded 38GiB to GPU in 2.6833s (14.1617GiB/s) | 20260507 22:52:14.994 | INFO | Analyzing chunk frames = [3228160,3228672) | 20260507 22:52:14.994 | INFO | MemInfo: { | 20260507 22:52:14.994 | INFO | "gpu" : " 17 GB", | 20260507 22:52:14.994 | INFO | "huge" : "", | 20260507 22:52:14.994 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:14.994 | INFO | "malloc" : " 14 GB", | 20260507 22:52:14.994 | INFO | "pinned" : " 29 GB" | 20260507 22:52:14.994 | INFO | } | 20260507 22:52:14.994 | INFO | Perf report for the last 5.14109 seconds: | 20260507 22:52:14.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:14.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.994 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 22:52:14.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.994 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 22:52:14.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:14.995 | INFO | Uploaded 36GiB to GPU in 2.57876s (13.9602GiB/s) | 20260507 22:52:14.995 | INFO | Downloaded 34GiB to GPU in 2.47475s (13.7388GiB/s) | 20260507 22:52:20.133 | INFO | Analyzing chunk frames = [3228672,3229184) | 20260507 22:52:20.133 | INFO | MemInfo: { | 20260507 22:52:20.133 | INFO | "gpu" : " 17 GB", | 20260507 22:52:20.134 | INFO | "huge" : "", | 20260507 22:52:20.134 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:20.134 | INFO | "malloc" : " 14 GB", | 20260507 22:52:20.134 | INFO | "pinned" : " 29 GB" | 20260507 22:52:20.134 | INFO | } | 20260507 22:52:20.134 | INFO | Perf report for the last 5.13945 seconds: | 20260507 22:52:20.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:20.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 22:52:20.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:52:20.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:20.134 | INFO | Uploaded 36GiB to GPU in 2.57286s (13.9922GiB/s) | 20260507 22:52:20.134 | INFO | Downloaded 34GiB to GPU in 2.46193s (13.8103GiB/s) | 20260507 22:52:25.275 | INFO | Analyzing chunk frames = [3229184,3229696) | 20260507 22:52:25.275 | INFO | MemInfo: { | 20260507 22:52:25.275 | INFO | "gpu" : " 17 GB", | 20260507 22:52:25.275 | INFO | "huge" : "", | 20260507 22:52:25.275 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:25.275 | INFO | "malloc" : " 14 GB", | 20260507 22:52:25.275 | INFO | "pinned" : " 29 GB" | 20260507 22:52:25.275 | INFO | } | 20260507 22:52:25.275 | INFO | Perf report for the last 5.14159 seconds: | 20260507 22:52:25.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:25.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.275 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 22:52:25.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.275 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:52:25.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:25.276 | INFO | Uploaded 36GiB to GPU in 2.48715s (14.4744GiB/s) | 20260507 22:52:25.276 | INFO | Downloaded 34GiB to GPU in 2.38875s (14.2334GiB/s) | 20260507 22:52:30.416 | INFO | Analyzing chunk frames = [3229696,3230208) | 20260507 22:52:30.416 | INFO | MemInfo: { | 20260507 22:52:30.416 | INFO | "gpu" : " 17 GB", | 20260507 22:52:30.416 | INFO | "huge" : "", | 20260507 22:52:30.416 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:30.416 | INFO | "malloc" : " 14 GB", | 20260507 22:52:30.416 | INFO | "pinned" : " 29 GB" | 20260507 22:52:30.416 | INFO | } | 20260507 22:52:30.416 | INFO | Perf report for the last 5.14117 seconds: | 20260507 22:52:30.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:30.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.416 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 22:52:30.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.417 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:52:30.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:30.417 | INFO | Uploaded 36GiB to GPU in 2.41143s (14.9289GiB/s) | 20260507 22:52:30.417 | INFO | Downloaded 34GiB to GPU in 2.48865s (13.662GiB/s) | 20260507 22:52:35.555 | INFO | Analyzing chunk frames = [3230208,3230720) | 20260507 22:52:35.555 | INFO | MemInfo: { | 20260507 22:52:35.555 | INFO | "gpu" : " 17 GB", | 20260507 22:52:35.555 | INFO | "huge" : "", | 20260507 22:52:35.555 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:35.555 | INFO | "malloc" : " 14 GB", | 20260507 22:52:35.555 | INFO | "pinned" : " 29 GB" | 20260507 22:52:35.555 | INFO | } | 20260507 22:52:35.555 | INFO | Perf report for the last 5.13903 seconds: | 20260507 22:52:35.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:35.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 22:52:35.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:52:35.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:35.556 | INFO | Uploaded 36GiB to GPU in 2.40948s (14.941GiB/s) | 20260507 22:52:35.556 | INFO | Downloaded 34GiB to GPU in 2.3974s (14.1821GiB/s) | 20260507 22:52:40.693 | INFO | Analyzing chunk frames = [3230720,3231232) | 20260507 22:52:40.693 | INFO | MemInfo: { | 20260507 22:52:40.693 | INFO | "gpu" : " 17 GB", | 20260507 22:52:40.694 | INFO | "huge" : "", | 20260507 22:52:40.694 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:40.694 | INFO | "malloc" : " 14 GB", | 20260507 22:52:40.694 | INFO | "pinned" : " 29 GB" | 20260507 22:52:40.694 | INFO | } | 20260507 22:52:40.694 | INFO | Perf report for the last 5.13828 seconds: | 20260507 22:52:40.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:40.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 22:52:40.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:52:40.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:40.694 | INFO | Uploaded 36GiB to GPU in 2.53847s (14.1818GiB/s) | 20260507 22:52:40.694 | INFO | Downloaded 34GiB to GPU in 2.44815s (13.888GiB/s) | 20260507 22:52:45.828 | INFO | Analyzing chunk frames = [3231232,3231744) | 20260507 22:52:45.828 | INFO | MemInfo: { | 20260507 22:52:45.828 | INFO | "gpu" : " 17 GB", | 20260507 22:52:45.829 | INFO | "huge" : "", | 20260507 22:52:45.829 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:45.829 | INFO | "malloc" : " 14 GB", | 20260507 22:52:45.829 | INFO | "pinned" : " 29 GB" | 20260507 22:52:45.829 | INFO | } | 20260507 22:52:45.829 | INFO | Perf report for the last 5.13498 seconds: | 20260507 22:52:45.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:45.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:52:45.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 22:52:45.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:45.829 | INFO | Uploaded 36GiB to GPU in 2.5202s (14.2846GiB/s) | 20260507 22:52:45.829 | INFO | Downloaded 34GiB to GPU in 2.44751s (13.8917GiB/s) | 20260507 22:52:45.926 | INFO | Starting a round of Baz IO | 20260507 22:52:49.395 | INFO | Populating RTMetrics report | 20260507 22:52:49.396 | INFO | RTMetrics report populated | 20260507 22:52:49.396 | INFO | Opening tmp RTMetrics json file | 20260507 22:52:49.410 | INFO | Serializing RTMetrics to json | 20260507 22:52:49.415 | INFO | Serialization done, swapping tmp file now... | 20260507 22:52:49.415 | INFO | RTMetrics done | 20260507 22:52:49.492 | INFO | Finished a round of Baz IO: Wrote 1.727 GiB in 3.565 seconds | 20260507 22:52:49.492 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.89% / 0.01% / 57.11% | 20260507 22:52:49.492 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31624% / 3.97364e-06% / 6.35783e-05% | 20260507 22:52:49.492 | INFO | Baz IO overall: | 20260507 22:52:49.492 | INFO | BytesWritten : 1472163226448 | 20260507 22:52:49.492 | INFO | NumEvents : 713112449925 | 20260507 22:52:50.967 | INFO | Analyzing chunk frames = [3231744,3232256) | 20260507 22:52:50.967 | INFO | MemInfo: { | 20260507 22:52:50.967 | INFO | "gpu" : " 17 GB", | 20260507 22:52:50.967 | INFO | "huge" : "", | 20260507 22:52:50.967 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:50.967 | INFO | "malloc" : " 14 GB", | 20260507 22:52:50.968 | INFO | "pinned" : " 29 GB" | 20260507 22:52:50.968 | INFO | } | 20260507 22:52:50.968 | INFO | Perf report for the last 5.13887 seconds: | 20260507 22:52:50.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:50.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:50.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:50.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:50.968 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 22:52:50.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:50.968 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 22:52:50.968 | INFO | PRE_HQ : 66.0%, 1.7, 57.5ms, 0 | 20260507 22:52:50.968 | INFO | BAZWRITER : 69.0%, 4.2, 147.1ms, 0 | 20260507 22:52:50.968 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 22:52:50.968 | INFO | Uploaded 36GiB to GPU in 2.52083s (14.281GiB/s) | 20260507 22:52:50.968 | INFO | Downloaded 38GiB to GPU in 2.69757s (14.0867GiB/s) | 20260507 22:52:56.110 | INFO | Analyzing chunk frames = [3232256,3232768) | 20260507 22:52:56.110 | INFO | MemInfo: { | 20260507 22:52:56.110 | INFO | "gpu" : " 17 GB", | 20260507 22:52:56.110 | INFO | "huge" : "", | 20260507 22:52:56.110 | INFO | "huge_shared" : " 29 GB", | 20260507 22:52:56.110 | INFO | "malloc" : " 14 GB", | 20260507 22:52:56.110 | INFO | "pinned" : " 29 GB" | 20260507 22:52:56.110 | INFO | } | 20260507 22:52:56.110 | INFO | Perf report for the last 5.1428 seconds: | 20260507 22:52:56.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:52:56.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.111 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 22:52:56.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.111 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:52:56.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:52:56.111 | INFO | Uploaded 36GiB to GPU in 2.527s (14.2461GiB/s) | 20260507 22:52:56.111 | INFO | Downloaded 34GiB to GPU in 2.47846s (13.7182GiB/s) | 20260507 22:53:01.252 | INFO | Analyzing chunk frames = [3232768,3233280) | 20260507 22:53:01.252 | INFO | MemInfo: { | 20260507 22:53:01.252 | INFO | "gpu" : " 17 GB", | 20260507 22:53:01.252 | INFO | "huge" : "", | 20260507 22:53:01.253 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:01.253 | INFO | "malloc" : " 14 GB", | 20260507 22:53:01.253 | INFO | "pinned" : " 29 GB" | 20260507 22:53:01.253 | INFO | } | 20260507 22:53:01.253 | INFO | Perf report for the last 5.14231 seconds: | 20260507 22:53:01.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:01.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:53:01.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:53:01.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:01.253 | INFO | Uploaded 36GiB to GPU in 2.51784s (14.2979GiB/s) | 20260507 22:53:01.253 | INFO | Downloaded 34GiB to GPU in 2.46077s (13.8168GiB/s) | 20260507 22:53:06.387 | INFO | Analyzing chunk frames = [3233280,3233792) | 20260507 22:53:06.387 | INFO | MemInfo: { | 20260507 22:53:06.387 | INFO | "gpu" : " 17 GB", | 20260507 22:53:06.387 | INFO | "huge" : "", | 20260507 22:53:06.387 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:06.387 | INFO | "malloc" : " 14 GB", | 20260507 22:53:06.387 | INFO | "pinned" : " 29 GB" | 20260507 22:53:06.387 | INFO | } | 20260507 22:53:06.387 | INFO | Perf report for the last 5.13464 seconds: | 20260507 22:53:06.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:06.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.387 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 22:53:06.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.387 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:53:06.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:06.388 | INFO | Uploaded 36GiB to GPU in 2.47798s (14.528GiB/s) | 20260507 22:53:06.388 | INFO | Downloaded 34GiB to GPU in 2.40539s (14.1349GiB/s) | 20260507 22:53:11.529 | INFO | Analyzing chunk frames = [3233792,3234304) | 20260507 22:53:11.529 | INFO | MemInfo: { | 20260507 22:53:11.529 | INFO | "gpu" : " 17 GB", | 20260507 22:53:11.529 | INFO | "huge" : "", | 20260507 22:53:11.529 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:11.529 | INFO | "malloc" : " 14 GB", | 20260507 22:53:11.529 | INFO | "pinned" : " 29 GB" | 20260507 22:53:11.529 | INFO | } | 20260507 22:53:11.529 | INFO | Perf report for the last 5.14155 seconds: | 20260507 22:53:11.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:11.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:53:11.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 22:53:11.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:11.529 | INFO | Uploaded 36GiB to GPU in 2.34875s (15.3273GiB/s) | 20260507 22:53:11.529 | INFO | Downloaded 34GiB to GPU in 2.4997s (13.6016GiB/s) | 20260507 22:53:16.664 | INFO | Analyzing chunk frames = [3234304,3234816) | 20260507 22:53:16.664 | INFO | MemInfo: { | 20260507 22:53:16.664 | INFO | "gpu" : " 17 GB", | 20260507 22:53:16.664 | INFO | "huge" : "", | 20260507 22:53:16.665 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:16.665 | INFO | "malloc" : " 14 GB", | 20260507 22:53:16.665 | INFO | "pinned" : " 29 GB" | 20260507 22:53:16.665 | INFO | } | 20260507 22:53:16.665 | INFO | Perf report for the last 5.13574 seconds: | 20260507 22:53:16.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:16.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:53:16.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 22:53:16.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:16.665 | INFO | Uploaded 36GiB to GPU in 2.51736s (14.3007GiB/s) | 20260507 22:53:16.665 | INFO | Downloaded 34GiB to GPU in 2.43175s (13.9817GiB/s) | 20260507 22:53:21.803 | INFO | Analyzing chunk frames = [3234816,3235328) | 20260507 22:53:21.803 | INFO | MemInfo: { | 20260507 22:53:21.803 | INFO | "gpu" : " 17 GB", | 20260507 22:53:21.803 | INFO | "huge" : "", | 20260507 22:53:21.803 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:21.803 | INFO | "malloc" : " 14 GB", | 20260507 22:53:21.804 | INFO | "pinned" : " 29 GB" | 20260507 22:53:21.804 | INFO | } | 20260507 22:53:21.804 | INFO | Perf report for the last 5.139 seconds: | 20260507 22:53:21.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:21.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 22:53:21.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:53:21.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:21.804 | INFO | Uploaded 36GiB to GPU in 2.53744s (14.1875GiB/s) | 20260507 22:53:21.804 | INFO | Downloaded 34GiB to GPU in 2.45795s (13.8327GiB/s) | 20260507 22:53:26.943 | INFO | Analyzing chunk frames = [3235328,3235840) | 20260507 22:53:26.943 | INFO | MemInfo: { | 20260507 22:53:26.943 | INFO | "gpu" : " 17 GB", | 20260507 22:53:26.943 | INFO | "huge" : "", | 20260507 22:53:26.943 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:26.943 | INFO | "malloc" : " 14 GB", | 20260507 22:53:26.943 | INFO | "pinned" : " 29 GB" | 20260507 22:53:26.943 | INFO | } | 20260507 22:53:26.943 | INFO | Perf report for the last 5.1395 seconds: | 20260507 22:53:26.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:26.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.943 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:53:26.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.943 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:53:26.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:26.944 | INFO | Uploaded 36GiB to GPU in 2.50154s (14.3911GiB/s) | 20260507 22:53:26.944 | INFO | Downloaded 34GiB to GPU in 2.43814s (13.945GiB/s) | 20260507 22:53:26.944 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:53:26.944 | INFO | Total mem usage: 16.7 GiB | 20260507 22:53:26.944 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:53:26.944 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260507 22:53:26.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:53:26.944 | INFO | High Water Breakdown: | 20260507 22:53:26.944 | INFO | 60% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:53:26.944 | INFO | 39.8% : 5.94 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:53:26.944 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:53:26.944 | INFO | ------------------------------------------ | 20260507 22:53:26.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:53:26.944 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:53:26.944 | INFO | Total mem usage: 26.5 GiB | 20260507 22:53:26.944 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:53:26.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:53:26.944 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:53:26.944 | INFO | High Water Breakdown: | 20260507 22:53:26.944 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:53:26.944 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:53:26.944 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:53:26.944 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:53:26.944 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:53:26.944 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:53:26.944 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:53:26.944 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:53:26.944 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:53:26.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:53:26.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:53:26.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:53:26.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:53:26.944 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:53:26.944 | INFO | ------------------------------------------ | 20260507 22:53:26.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:53:26.944 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:53:26.944 | INFO | Total mem usage: 27.8 GiB | 20260507 22:53:26.944 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:53:26.944 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:53:26.944 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:53:26.944 | INFO | High Water Breakdown: | 20260507 22:53:26.944 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:53:26.944 | INFO | ------------------------------------------ | 20260507 22:53:26.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:53:26.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:53:26.944 | INFO | Total mem usage: 15.7 GiB | 20260507 22:53:26.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:53:26.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:53:26.944 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:53:26.944 | INFO | High Water Breakdown: | 20260507 22:53:26.944 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:53:26.944 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:53:26.944 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:53:26.944 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:53:26.944 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:53:26.944 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:53:26.944 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:53:26.945 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:53:26.945 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:53:26.945 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:53:26.945 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:53:26.945 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:53:26.945 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:53:26.945 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:53:26.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:53:26.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:53:26.945 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:53:26.945 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:53:26.945 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:53:26.945 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:53:26.945 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:53:26.945 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:53:26.945 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:53:26.945 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:53:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:53:26.945 | INFO | ------------------------------------------ | 20260507 22:53:26.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:53:26.945 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:53:26.945 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:53:26.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:53:26.945 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.63e+01, 1.40e+01 / 1.65e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.14e-02 / 5.48e-03 | 20260507 22:53:26.945 | INFO | PerfReport: 12.14%, Baseline -- 4.00e+00 / 4.50e+00, 5.99e-01 / 2.30e-01 | 20260507 22:53:26.945 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.49e+00, 2.09e-01 / 1.06e-01 | 20260507 22:53:26.945 | INFO | PerfReport: 15.78%, DME -- 4.16e+01 / 4.52e+01, 3.76e+00 / 1.31e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.82e+00 / 1.64e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.43e+00 / 2.73e+00, 4.27e-01 / 1.90e-01 | 20260507 22:53:26.945 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.35e+00, 2.88e-01 / 1.47e-01 | 20260507 22:53:26.945 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.51e-01 / 3.80e-01, 5.84e-02 / 3.07e-02 | 20260507 22:53:26.945 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:53:26.945 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:53:26.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:53:26.945 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 24.19%, Download -- 2.58e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 30.87%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 2.02%, LockUpload -- 2.16e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 2.49%, LockDownload -- 2.66e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260507 22:53:26.945 | INFO | PerfReport: 16.00%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 22:53:26.946 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260507 22:53:27.040 | INFO | Starting a round of Baz IO | 20260507 22:53:30.520 | INFO | Populating RTMetrics report | 20260507 22:53:30.520 | INFO | RTMetrics report populated | 20260507 22:53:30.520 | INFO | Opening tmp RTMetrics json file | 20260507 22:53:30.532 | INFO | Serializing RTMetrics to json | 20260507 22:53:30.533 | INFO | Serialization done, swapping tmp file now... | 20260507 22:53:30.534 | INFO | RTMetrics done | 20260507 22:53:30.703 | INFO | Finished a round of Baz IO: Wrote 1.727 GiB in 3.663 seconds | 20260507 22:53:30.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.88% / 0.01% / 57.11% | 20260507 22:53:30.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31391% / 0% / 7.15256e-05% | 20260507 22:53:30.704 | INFO | Baz IO overall: | 20260507 22:53:30.704 | INFO | BytesWritten : 1474017650651 | 20260507 22:53:30.704 | INFO | NumEvents : 714007172376 | 20260507 22:53:32.080 | INFO | Analyzing chunk frames = [3235840,3236352) | 20260507 22:53:32.080 | INFO | MemInfo: { | 20260507 22:53:32.080 | INFO | "gpu" : " 17 GB", | 20260507 22:53:32.080 | INFO | "huge" : "", | 20260507 22:53:32.080 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:32.080 | INFO | "malloc" : " 14 GB", | 20260507 22:53:32.080 | INFO | "pinned" : " 29 GB" | 20260507 22:53:32.080 | INFO | } | 20260507 22:53:32.080 | INFO | Perf report for the last 5.13656 seconds: | 20260507 22:53:32.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:32.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:32.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:32.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:32.080 | INFO | ANALYSIS : 68.9%, 3.0, 107.3ms, 0 | 20260507 22:53:32.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:32.080 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 22:53:32.080 | INFO | PRE_HQ : 65.8%, 1.7, 57.4ms, 0 | 20260507 22:53:32.080 | INFO | BAZWRITER : 70.9%, 3.9, 141.9ms, 0 | 20260507 22:53:32.080 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 22:53:32.081 | INFO | Uploaded 36GiB to GPU in 2.53152s (14.2207GiB/s) | 20260507 22:53:32.081 | INFO | Downloaded 38GiB to GPU in 2.68435s (14.1561GiB/s) | 20260507 22:53:37.225 | INFO | Analyzing chunk frames = [3236352,3236864) | 20260507 22:53:37.225 | INFO | MemInfo: { | 20260507 22:53:37.225 | INFO | "gpu" : " 17 GB", | 20260507 22:53:37.225 | INFO | "huge" : "", | 20260507 22:53:37.225 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:37.225 | INFO | "malloc" : " 14 GB", | 20260507 22:53:37.225 | INFO | "pinned" : " 29 GB" | 20260507 22:53:37.225 | INFO | } | 20260507 22:53:37.225 | INFO | Perf report for the last 5.14498 seconds: | 20260507 22:53:37.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:37.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 22:53:37.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 22:53:37.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:37.225 | INFO | Uploaded 36GiB to GPU in 2.54878s (14.1244GiB/s) | 20260507 22:53:37.225 | INFO | Downloaded 34GiB to GPU in 2.48242s (13.6963GiB/s) | 20260507 22:53:42.370 | INFO | Analyzing chunk frames = [3236864,3237376) | 20260507 22:53:42.370 | INFO | MemInfo: { | 20260507 22:53:42.370 | INFO | "gpu" : " 17 GB", | 20260507 22:53:42.370 | INFO | "huge" : "", | 20260507 22:53:42.370 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:42.370 | INFO | "malloc" : " 14 GB", | 20260507 22:53:42.370 | INFO | "pinned" : " 29 GB" | 20260507 22:53:42.370 | INFO | } | 20260507 22:53:42.371 | INFO | Perf report for the last 5.14568 seconds: | 20260507 22:53:42.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:42.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260507 22:53:42.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:53:42.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:42.371 | INFO | Uploaded 36GiB to GPU in 2.56328s (14.0445GiB/s) | 20260507 22:53:42.371 | INFO | Downloaded 34GiB to GPU in 2.47325s (13.7471GiB/s) | 20260507 22:53:47.506 | INFO | Analyzing chunk frames = [3237376,3237888) | 20260507 22:53:47.506 | INFO | MemInfo: { | 20260507 22:53:47.506 | INFO | "gpu" : " 17 GB", | 20260507 22:53:47.506 | INFO | "huge" : "", | 20260507 22:53:47.506 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:47.506 | INFO | "malloc" : " 14 GB", | 20260507 22:53:47.506 | INFO | "pinned" : " 29 GB" | 20260507 22:53:47.506 | INFO | } | 20260507 22:53:47.506 | INFO | Perf report for the last 5.13537 seconds: | 20260507 22:53:47.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:47.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 22:53:47.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:53:47.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:47.506 | INFO | Uploaded 36GiB to GPU in 2.50015s (14.3991GiB/s) | 20260507 22:53:47.506 | INFO | Downloaded 34GiB to GPU in 2.48851s (13.6628GiB/s) | 20260507 22:53:52.643 | INFO | Analyzing chunk frames = [3237888,3238400) | 20260507 22:53:52.643 | INFO | MemInfo: { | 20260507 22:53:52.643 | INFO | "gpu" : " 17 GB", | 20260507 22:53:52.643 | INFO | "huge" : "", | 20260507 22:53:52.644 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:52.644 | INFO | "malloc" : " 14 GB", | 20260507 22:53:52.644 | INFO | "pinned" : " 29 GB" | 20260507 22:53:52.644 | INFO | } | 20260507 22:53:52.644 | INFO | Perf report for the last 5.13808 seconds: | 20260507 22:53:52.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:52.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.644 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:53:52.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.644 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 22:53:52.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:52.645 | INFO | Uploaded 36GiB to GPU in 2.32644s (15.4743GiB/s) | 20260507 22:53:52.645 | INFO | Downloaded 34GiB to GPU in 2.46276s (13.8056GiB/s) | 20260507 22:53:57.781 | INFO | Analyzing chunk frames = [3238400,3238912) | 20260507 22:53:57.781 | INFO | MemInfo: { | 20260507 22:53:57.781 | INFO | "gpu" : " 17 GB", | 20260507 22:53:57.781 | INFO | "huge" : "", | 20260507 22:53:57.781 | INFO | "huge_shared" : " 29 GB", | 20260507 22:53:57.781 | INFO | "malloc" : " 14 GB", | 20260507 22:53:57.781 | INFO | "pinned" : " 29 GB" | 20260507 22:53:57.781 | INFO | } | 20260507 22:53:57.781 | INFO | Perf report for the last 5.13718 seconds: | 20260507 22:53:57.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:53:57.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.781 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:53:57.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.781 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:53:57.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:53:57.782 | INFO | Uploaded 36GiB to GPU in 2.4187s (14.884GiB/s) | 20260507 22:53:57.782 | INFO | Downloaded 34GiB to GPU in 2.39183s (14.2151GiB/s) | 20260507 22:54:02.924 | INFO | Analyzing chunk frames = [3238912,3239424) | 20260507 22:54:02.924 | INFO | MemInfo: { | 20260507 22:54:02.924 | INFO | "gpu" : " 17 GB", | 20260507 22:54:02.924 | INFO | "huge" : "", | 20260507 22:54:02.924 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:02.924 | INFO | "malloc" : " 14 GB", | 20260507 22:54:02.924 | INFO | "pinned" : " 29 GB" | 20260507 22:54:02.924 | INFO | } | 20260507 22:54:02.925 | INFO | Perf report for the last 5.1433 seconds: | 20260507 22:54:02.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:02.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 22:54:02.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 22:54:02.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:02.925 | INFO | Uploaded 36GiB to GPU in 2.53274s (14.2139GiB/s) | 20260507 22:54:02.925 | INFO | Downloaded 34GiB to GPU in 2.43881s (13.9412GiB/s) | 20260507 22:54:08.063 | INFO | Analyzing chunk frames = [3239424,3239936) | 20260507 22:54:08.063 | INFO | MemInfo: { | 20260507 22:54:08.063 | INFO | "gpu" : " 17 GB", | 20260507 22:54:08.063 | INFO | "huge" : "", | 20260507 22:54:08.063 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:08.063 | INFO | "malloc" : " 14 GB", | 20260507 22:54:08.063 | INFO | "pinned" : " 29 GB" | 20260507 22:54:08.063 | INFO | } | 20260507 22:54:08.063 | INFO | Perf report for the last 5.13876 seconds: | 20260507 22:54:08.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:08.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.063 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:54:08.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.063 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:54:08.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:08.064 | INFO | Uploaded 36GiB to GPU in 2.50593s (14.3659GiB/s) | 20260507 22:54:08.064 | INFO | Downloaded 34GiB to GPU in 2.43018s (13.9907GiB/s) | 20260507 22:54:08.161 | INFO | Starting a round of Baz IO | 20260507 22:54:11.627 | INFO | Populating RTMetrics report | 20260507 22:54:11.628 | INFO | RTMetrics report populated | 20260507 22:54:11.628 | INFO | Opening tmp RTMetrics json file | 20260507 22:54:11.629 | INFO | Serializing RTMetrics to json | 20260507 22:54:11.652 | INFO | Serialization done, swapping tmp file now... | 20260507 22:54:11.655 | INFO | RTMetrics done | 20260507 22:54:11.730 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.569 seconds | 20260507 22:54:11.730 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.88% / 0.01% / 57.12% | 20260507 22:54:11.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31585% / 3.97364e-06% / 5.96046e-05% | 20260507 22:54:11.731 | INFO | Baz IO overall: | 20260507 22:54:11.731 | INFO | BytesWritten : 1475874136321 | 20260507 22:54:11.731 | INFO | NumEvents : 714903412037 | 20260507 22:54:13.201 | INFO | Analyzing chunk frames = [3239936,3240448) | 20260507 22:54:13.201 | INFO | MemInfo: { | 20260507 22:54:13.201 | INFO | "gpu" : " 17 GB", | 20260507 22:54:13.201 | INFO | "huge" : "", | 20260507 22:54:13.201 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:13.201 | INFO | "malloc" : " 14 GB", | 20260507 22:54:13.201 | INFO | "pinned" : " 29 GB" | 20260507 22:54:13.201 | INFO | } | 20260507 22:54:13.201 | INFO | Perf report for the last 5.13811 seconds: | 20260507 22:54:13.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:13.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:13.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:13.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:13.201 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 22:54:13.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:13.202 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 22:54:13.202 | INFO | PRE_HQ : 65.6%, 1.7, 56.9ms, 0 | 20260507 22:54:13.202 | INFO | BAZWRITER : 69.1%, 4.5, 158.1ms, 0 | 20260507 22:54:13.202 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 22:54:13.202 | INFO | Uploaded 36GiB to GPU in 2.58455s (13.9289GiB/s) | 20260507 22:54:13.202 | INFO | Downloaded 38GiB to GPU in 2.69739s (14.0877GiB/s) | 20260507 22:54:18.340 | INFO | Analyzing chunk frames = [3240448,3240960) | 20260507 22:54:18.340 | INFO | MemInfo: { | 20260507 22:54:18.340 | INFO | "gpu" : " 17 GB", | 20260507 22:54:18.340 | INFO | "huge" : "", | 20260507 22:54:18.340 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:18.340 | INFO | "malloc" : " 14 GB", | 20260507 22:54:18.340 | INFO | "pinned" : " 29 GB" | 20260507 22:54:18.340 | INFO | } | 20260507 22:54:18.340 | INFO | Perf report for the last 5.139 seconds: | 20260507 22:54:18.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:18.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.340 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 22:54:18.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.341 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:54:18.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:18.341 | INFO | Uploaded 36GiB to GPU in 2.58751s (13.913GiB/s) | 20260507 22:54:18.341 | INFO | Downloaded 34GiB to GPU in 2.47152s (13.7567GiB/s) | 20260507 22:54:23.481 | INFO | Analyzing chunk frames = [3240960,3241472) | 20260507 22:54:23.481 | INFO | MemInfo: { | 20260507 22:54:23.481 | INFO | "gpu" : " 17 GB", | 20260507 22:54:23.481 | INFO | "huge" : "", | 20260507 22:54:23.481 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:23.481 | INFO | "malloc" : " 14 GB", | 20260507 22:54:23.481 | INFO | "pinned" : " 29 GB" | 20260507 22:54:23.481 | INFO | } | 20260507 22:54:23.481 | INFO | Perf report for the last 5.14099 seconds: | 20260507 22:54:23.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:23.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 22:54:23.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 22:54:23.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:23.482 | INFO | Uploaded 36GiB to GPU in 2.58096s (13.9483GiB/s) | 20260507 22:54:23.482 | INFO | Downloaded 34GiB to GPU in 2.4664s (13.7853GiB/s) | 20260507 22:54:28.619 | INFO | Analyzing chunk frames = [3241472,3241984) | 20260507 22:54:28.619 | INFO | MemInfo: { | 20260507 22:54:28.619 | INFO | "gpu" : " 17 GB", | 20260507 22:54:28.619 | INFO | "huge" : "", | 20260507 22:54:28.619 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:28.619 | INFO | "malloc" : " 14 GB", | 20260507 22:54:28.619 | INFO | "pinned" : " 29 GB" | 20260507 22:54:28.619 | INFO | } | 20260507 22:54:28.619 | INFO | Perf report for the last 5.13754 seconds: | 20260507 22:54:28.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:28.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.619 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 22:54:28.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.619 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 22:54:28.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:28.620 | INFO | Uploaded 36GiB to GPU in 2.53241s (14.2157GiB/s) | 20260507 22:54:28.620 | INFO | Downloaded 34GiB to GPU in 2.48071s (13.7058GiB/s) | 20260507 22:54:33.757 | INFO | Analyzing chunk frames = [3241984,3242496) | 20260507 22:54:33.757 | INFO | MemInfo: { | 20260507 22:54:33.757 | INFO | "gpu" : " 17 GB", | 20260507 22:54:33.757 | INFO | "huge" : "", | 20260507 22:54:33.757 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:33.757 | INFO | "malloc" : " 14 GB", | 20260507 22:54:33.757 | INFO | "pinned" : " 29 GB" | 20260507 22:54:33.757 | INFO | } | 20260507 22:54:33.757 | INFO | Perf report for the last 5.13803 seconds: | 20260507 22:54:33.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:33.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.757 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:54:33.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.757 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:54:33.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:33.758 | INFO | Uploaded 36GiB to GPU in 2.3833s (15.1051GiB/s) | 20260507 22:54:33.758 | INFO | Downloaded 34GiB to GPU in 2.4978s (13.612GiB/s) | 20260507 22:54:38.893 | INFO | Analyzing chunk frames = [3242496,3243008) | 20260507 22:54:38.893 | INFO | MemInfo: { | 20260507 22:54:38.893 | INFO | "gpu" : " 17 GB", | 20260507 22:54:38.893 | INFO | "huge" : "", | 20260507 22:54:38.893 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:38.893 | INFO | "malloc" : " 14 GB", | 20260507 22:54:38.893 | INFO | "pinned" : " 29 GB" | 20260507 22:54:38.893 | INFO | } | 20260507 22:54:38.893 | INFO | Perf report for the last 5.13612 seconds: | 20260507 22:54:38.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:38.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.893 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 22:54:38.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.893 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:54:38.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:38.894 | INFO | Uploaded 36GiB to GPU in 2.50556s (14.368GiB/s) | 20260507 22:54:38.894 | INFO | Downloaded 34GiB to GPU in 2.43586s (13.9581GiB/s) | 20260507 22:54:44.034 | INFO | Analyzing chunk frames = [3243008,3243520) | 20260507 22:54:44.034 | INFO | MemInfo: { | 20260507 22:54:44.034 | INFO | "gpu" : " 17 GB", | 20260507 22:54:44.034 | INFO | "huge" : "", | 20260507 22:54:44.034 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:44.034 | INFO | "malloc" : " 14 GB", | 20260507 22:54:44.034 | INFO | "pinned" : " 29 GB" | 20260507 22:54:44.034 | INFO | } | 20260507 22:54:44.034 | INFO | Perf report for the last 5.14123 seconds: | 20260507 22:54:44.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:44.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.034 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 22:54:44.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.035 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:54:44.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:44.035 | INFO | Uploaded 36GiB to GPU in 2.51539s (14.3119GiB/s) | 20260507 22:54:44.035 | INFO | Downloaded 34GiB to GPU in 2.45133s (13.87GiB/s) | 20260507 22:54:49.177 | INFO | Analyzing chunk frames = [3243520,3244032) | 20260507 22:54:49.177 | INFO | MemInfo: { | 20260507 22:54:49.177 | INFO | "gpu" : " 17 GB", | 20260507 22:54:49.177 | INFO | "huge" : "", | 20260507 22:54:49.177 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:49.177 | INFO | "malloc" : " 14 GB", | 20260507 22:54:49.177 | INFO | "pinned" : " 29 GB" | 20260507 22:54:49.177 | INFO | } | 20260507 22:54:49.177 | INFO | Perf report for the last 5.14265 seconds: | 20260507 22:54:49.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:49.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 22:54:49.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:54:49.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:49.177 | INFO | Uploaded 36GiB to GPU in 2.51599s (14.3085GiB/s) | 20260507 22:54:49.177 | INFO | Downloaded 34GiB to GPU in 2.44413s (13.9109GiB/s) | 20260507 22:54:49.178 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:54:49.178 | INFO | Total mem usage: 16.7 GiB | 20260507 22:54:49.178 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:54:49.178 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260507 22:54:49.178 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:54:49.178 | INFO | High Water Breakdown: | 20260507 22:54:49.178 | INFO | 60.1% : 8.97 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:54:49.178 | INFO | 39.8% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:54:49.178 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:54:49.178 | INFO | ------------------------------------------ | 20260507 22:54:49.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:54:49.178 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:54:49.178 | INFO | Total mem usage: 26.5 GiB | 20260507 22:54:49.178 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:54:49.178 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:54:49.178 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:54:49.178 | INFO | High Water Breakdown: | 20260507 22:54:49.178 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:54:49.178 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:54:49.178 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:54:49.178 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:54:49.178 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:54:49.178 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:54:49.178 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:54:49.178 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:54:49.178 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:54:49.178 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:54:49.178 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:54:49.178 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:54:49.178 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:54:49.178 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:54:49.178 | INFO | ------------------------------------------ | 20260507 22:54:49.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:54:49.178 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:54:49.178 | INFO | Total mem usage: 27.8 GiB | 20260507 22:54:49.178 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:54:49.178 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:54:49.178 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:54:49.178 | INFO | High Water Breakdown: | 20260507 22:54:49.178 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:54:49.178 | INFO | ------------------------------------------ | 20260507 22:54:49.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:54:49.178 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:54:49.178 | INFO | Total mem usage: 15.7 GiB | 20260507 22:54:49.178 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:54:49.178 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:54:49.178 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:54:49.178 | INFO | High Water Breakdown: | 20260507 22:54:49.178 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:54:49.178 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:54:49.178 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:54:49.178 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:54:49.178 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:54:49.178 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:54:49.178 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:54:49.178 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:54:49.178 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:54:49.178 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:54:49.178 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:54:49.178 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:54:49.178 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:54:49.179 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:54:49.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:54:49.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:54:49.179 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:54:49.179 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:54:49.179 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:54:49.179 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:54:49.179 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:54:49.179 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:54:49.179 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:54:49.179 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:54:49.179 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:54:49.179 | INFO | ------------------------------------------ | 20260507 22:54:49.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:54:49.179 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:54:49.179 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:54:49.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:54:49.179 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.67e+01, 1.43e+01 / 1.83e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.12e-02 / 5.48e-03 | 20260507 22:54:49.179 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.48e+00, 5.92e-01 / 2.24e-01 | 20260507 22:54:49.179 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.47e+00, 2.09e-01 / 1.17e-01 | 20260507 22:54:49.179 | INFO | PerfReport: 15.99%, DME -- 4.21e+01 / 4.60e+01, 4.12e+00 / 1.78e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.85e+01 / 2.02e+01, 1.71e+00 / 1.72e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.32e-01 / 2.25e-01 | 20260507 22:54:49.179 | INFO | PerfReport: 3.46%, Metrics -- 1.14e+00 / 1.37e+00, 2.86e-01 / 1.50e-01 | 20260507 22:54:49.179 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.51e-01 / 3.87e-01, 5.75e-02 / 3.37e-02 | 20260507 22:54:49.179 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:54:49.179 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:54:49.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:54:49.179 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 24.43%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 30.77%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 2.58%, LockDownload -- 2.75e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 22:54:49.179 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.21e-02 / 0.00e+00 | 20260507 22:54:49.274 | INFO | Starting a round of Baz IO | 20260507 22:54:52.746 | INFO | Populating RTMetrics report | 20260507 22:54:52.746 | INFO | RTMetrics report populated | 20260507 22:54:52.746 | INFO | Opening tmp RTMetrics json file | 20260507 22:54:52.777 | INFO | Serializing RTMetrics to json | 20260507 22:54:52.785 | INFO | Serialization done, swapping tmp file now... | 20260507 22:54:52.801 | INFO | RTMetrics done | 20260507 22:54:52.847 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.572 seconds | 20260507 22:54:52.847 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.87% / 0.01% / 57.12% | 20260507 22:54:52.847 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31205% / 7.94729e-06% / 0.000111262% | 20260507 22:54:52.847 | INFO | Baz IO overall: | 20260507 22:54:52.847 | INFO | BytesWritten : 1477729759278 | 20260507 22:54:52.847 | INFO | NumEvents : 715799119972 | 20260507 22:54:54.321 | INFO | Analyzing chunk frames = [3244032,3244544) | 20260507 22:54:54.321 | INFO | MemInfo: { | 20260507 22:54:54.321 | INFO | "gpu" : " 17 GB", | 20260507 22:54:54.321 | INFO | "huge" : "", | 20260507 22:54:54.321 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:54.321 | INFO | "malloc" : " 14 GB", | 20260507 22:54:54.321 | INFO | "pinned" : " 29 GB" | 20260507 22:54:54.321 | INFO | } | 20260507 22:54:54.321 | INFO | Perf report for the last 5.14402 seconds: | 20260507 22:54:54.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:54.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:54.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:54.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:54.321 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 22:54:54.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:54.321 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 22:54:54.321 | INFO | PRE_HQ : 65.7%, 1.7, 57.0ms, 0 | 20260507 22:54:54.321 | INFO | BAZWRITER : 69.0%, 3.9, 139.0ms, 0 | 20260507 22:54:54.321 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 22:54:54.322 | INFO | Uploaded 36GiB to GPU in 2.51691s (14.3032GiB/s) | 20260507 22:54:54.322 | INFO | Downloaded 38GiB to GPU in 2.67859s (14.1866GiB/s) | 20260507 22:54:59.462 | INFO | Analyzing chunk frames = [3244544,3245056) | 20260507 22:54:59.462 | INFO | MemInfo: { | 20260507 22:54:59.462 | INFO | "gpu" : " 17 GB", | 20260507 22:54:59.462 | INFO | "huge" : "", | 20260507 22:54:59.462 | INFO | "huge_shared" : " 29 GB", | 20260507 22:54:59.462 | INFO | "malloc" : " 14 GB", | 20260507 22:54:59.462 | INFO | "pinned" : " 29 GB" | 20260507 22:54:59.462 | INFO | } | 20260507 22:54:59.462 | INFO | Perf report for the last 5.1413 seconds: | 20260507 22:54:59.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:54:59.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.462 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 22:54:59.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.463 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:54:59.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:54:59.464 | INFO | Uploaded 36GiB to GPU in 2.5669s (14.0247GiB/s) | 20260507 22:54:59.464 | INFO | Downloaded 34GiB to GPU in 2.48731s (13.6694GiB/s) | 20260507 22:55:04.595 | INFO | Analyzing chunk frames = [3245056,3245568) | 20260507 22:55:04.595 | INFO | MemInfo: { | 20260507 22:55:04.595 | INFO | "gpu" : " 17 GB", | 20260507 22:55:04.595 | INFO | "huge" : "", | 20260507 22:55:04.595 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:04.595 | INFO | "malloc" : " 14 GB", | 20260507 22:55:04.595 | INFO | "pinned" : " 29 GB" | 20260507 22:55:04.595 | INFO | } | 20260507 22:55:04.595 | INFO | Perf report for the last 5.13257 seconds: | 20260507 22:55:04.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:04.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.595 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 22:55:04.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.595 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:55:04.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:04.596 | INFO | Uploaded 36GiB to GPU in 2.53626s (14.1941GiB/s) | 20260507 22:55:04.596 | INFO | Downloaded 34GiB to GPU in 2.49171s (13.6452GiB/s) | 20260507 22:55:09.734 | INFO | Analyzing chunk frames = [3245568,3246080) | 20260507 22:55:09.734 | INFO | MemInfo: { | 20260507 22:55:09.734 | INFO | "gpu" : " 17 GB", | 20260507 22:55:09.734 | INFO | "huge" : "", | 20260507 22:55:09.734 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:09.734 | INFO | "malloc" : " 14 GB", | 20260507 22:55:09.734 | INFO | "pinned" : " 29 GB" | 20260507 22:55:09.734 | INFO | } | 20260507 22:55:09.734 | INFO | Perf report for the last 5.13943 seconds: | 20260507 22:55:09.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:09.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 22:55:09.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:55:09.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:09.735 | INFO | Uploaded 36GiB to GPU in 2.52054s (14.2826GiB/s) | 20260507 22:55:09.735 | INFO | Downloaded 34GiB to GPU in 2.48421s (13.6865GiB/s) | 20260507 22:55:14.874 | INFO | Analyzing chunk frames = [3246080,3246592) | 20260507 22:55:14.874 | INFO | MemInfo: { | 20260507 22:55:14.874 | INFO | "gpu" : " 17 GB", | 20260507 22:55:14.874 | INFO | "huge" : "", | 20260507 22:55:14.874 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:14.875 | INFO | "malloc" : " 14 GB", | 20260507 22:55:14.875 | INFO | "pinned" : " 29 GB" | 20260507 22:55:14.875 | INFO | } | 20260507 22:55:14.875 | INFO | Perf report for the last 5.14014 seconds: | 20260507 22:55:14.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:14.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 22:55:14.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260507 22:55:14.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:14.875 | INFO | Uploaded 36GiB to GPU in 2.34436s (15.356GiB/s) | 20260507 22:55:14.875 | INFO | Downloaded 34GiB to GPU in 2.46941s (13.7685GiB/s) | 20260507 22:55:20.013 | INFO | Analyzing chunk frames = [3246592,3247104) | 20260507 22:55:20.013 | INFO | MemInfo: { | 20260507 22:55:20.013 | INFO | "gpu" : " 17 GB", | 20260507 22:55:20.013 | INFO | "huge" : "", | 20260507 22:55:20.013 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:20.013 | INFO | "malloc" : " 14 GB", | 20260507 22:55:20.013 | INFO | "pinned" : " 29 GB" | 20260507 22:55:20.013 | INFO | } | 20260507 22:55:20.013 | INFO | Perf report for the last 5.13877 seconds: | 20260507 22:55:20.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:20.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.013 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 22:55:20.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.013 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:55:20.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:20.014 | INFO | Uploaded 36GiB to GPU in 2.50199s (14.3886GiB/s) | 20260507 22:55:20.014 | INFO | Downloaded 34GiB to GPU in 2.4245s (14.0235GiB/s) | 20260507 22:55:25.152 | INFO | Analyzing chunk frames = [3247104,3247616) | 20260507 22:55:25.152 | INFO | MemInfo: { | 20260507 22:55:25.152 | INFO | "gpu" : " 17 GB", | 20260507 22:55:25.152 | INFO | "huge" : "", | 20260507 22:55:25.152 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:25.152 | INFO | "malloc" : " 14 GB", | 20260507 22:55:25.152 | INFO | "pinned" : " 29 GB" | 20260507 22:55:25.152 | INFO | } | 20260507 22:55:25.152 | INFO | Perf report for the last 5.1388 seconds: | 20260507 22:55:25.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:25.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.152 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 22:55:25.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.152 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:55:25.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:25.153 | INFO | Uploaded 36GiB to GPU in 2.49993s (14.4004GiB/s) | 20260507 22:55:25.153 | INFO | Downloaded 34GiB to GPU in 2.44486s (13.9067GiB/s) | 20260507 22:55:30.288 | INFO | Analyzing chunk frames = [3247616,3248128) | 20260507 22:55:30.288 | INFO | MemInfo: { | 20260507 22:55:30.288 | INFO | "gpu" : " 17 GB", | 20260507 22:55:30.288 | INFO | "huge" : "", | 20260507 22:55:30.288 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:30.288 | INFO | "malloc" : " 14 GB", | 20260507 22:55:30.288 | INFO | "pinned" : " 29 GB" | 20260507 22:55:30.288 | INFO | } | 20260507 22:55:30.288 | INFO | Perf report for the last 5.13588 seconds: | 20260507 22:55:30.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:30.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:55:30.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:55:30.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:30.288 | INFO | Uploaded 36GiB to GPU in 2.51356s (14.3223GiB/s) | 20260507 22:55:30.288 | INFO | Downloaded 34GiB to GPU in 2.45088s (13.8726GiB/s) | 20260507 22:55:30.385 | INFO | Starting a round of Baz IO | 20260507 22:55:33.845 | INFO | Populating RTMetrics report | 20260507 22:55:33.845 | INFO | RTMetrics report populated | 20260507 22:55:33.845 | INFO | Opening tmp RTMetrics json file | 20260507 22:55:33.855 | INFO | Serializing RTMetrics to json | 20260507 22:55:33.870 | INFO | Serialization done, swapping tmp file now... | 20260507 22:55:33.871 | INFO | RTMetrics done | 20260507 22:55:33.946 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.56 seconds | 20260507 22:55:33.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.86% / 0.01% / 57.13% | 20260507 22:55:33.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30848% / 1.19209e-05% / 7.94729e-05% | 20260507 22:55:33.946 | INFO | Baz IO overall: | 20260507 22:55:33.946 | INFO | BytesWritten : 1479584649663 | 20260507 22:55:33.946 | INFO | NumEvents : 716694096420 | 20260507 22:55:35.429 | INFO | Analyzing chunk frames = [3248128,3248640) | 20260507 22:55:35.429 | INFO | MemInfo: { | 20260507 22:55:35.429 | INFO | "gpu" : " 17 GB", | 20260507 22:55:35.429 | INFO | "huge" : "", | 20260507 22:55:35.429 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:35.429 | INFO | "malloc" : " 14 GB", | 20260507 22:55:35.429 | INFO | "pinned" : " 29 GB" | 20260507 22:55:35.429 | INFO | } | 20260507 22:55:35.429 | INFO | Perf report for the last 5.14104 seconds: | 20260507 22:55:35.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:35.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:35.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:35.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:35.429 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260507 22:55:35.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:35.429 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 22:55:35.429 | INFO | PRE_HQ : 65.6%, 1.7, 57.0ms, 0 | 20260507 22:55:35.429 | INFO | BAZWRITER : 68.9%, 4.0, 139.9ms, 0 | 20260507 22:55:35.429 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260507 22:55:35.430 | INFO | Uploaded 36GiB to GPU in 2.5292s (14.2338GiB/s) | 20260507 22:55:35.430 | INFO | Downloaded 38GiB to GPU in 2.6902s (14.1253GiB/s) | 20260507 22:55:40.572 | INFO | Analyzing chunk frames = [3248640,3249152) | 20260507 22:55:40.572 | INFO | MemInfo: { | 20260507 22:55:40.572 | INFO | "gpu" : " 17 GB", | 20260507 22:55:40.572 | INFO | "huge" : "", | 20260507 22:55:40.573 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:40.573 | INFO | "malloc" : " 14 GB", | 20260507 22:55:40.573 | INFO | "pinned" : " 29 GB" | 20260507 22:55:40.573 | INFO | } | 20260507 22:55:40.573 | INFO | Perf report for the last 5.14348 seconds: | 20260507 22:55:40.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:40.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 22:55:40.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 22:55:40.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:40.573 | INFO | Uploaded 36GiB to GPU in 2.55337s (14.099GiB/s) | 20260507 22:55:40.573 | INFO | Downloaded 34GiB to GPU in 2.47713s (13.7256GiB/s) | 20260507 22:55:45.707 | INFO | Analyzing chunk frames = [3249152,3249664) | 20260507 22:55:45.707 | INFO | MemInfo: { | 20260507 22:55:45.707 | INFO | "gpu" : " 17 GB", | 20260507 22:55:45.707 | INFO | "huge" : "", | 20260507 22:55:45.707 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:45.707 | INFO | "malloc" : " 14 GB", | 20260507 22:55:45.707 | INFO | "pinned" : " 29 GB" | 20260507 22:55:45.707 | INFO | } | 20260507 22:55:45.707 | INFO | Perf report for the last 5.13456 seconds: | 20260507 22:55:45.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:45.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 22:55:45.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:55:45.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:45.707 | INFO | Uploaded 36GiB to GPU in 2.49938s (14.4035GiB/s) | 20260507 22:55:45.708 | INFO | Downloaded 34GiB to GPU in 2.49337s (13.6362GiB/s) | 20260507 22:55:50.845 | INFO | Analyzing chunk frames = [3249664,3250176) | 20260507 22:55:50.845 | INFO | MemInfo: { | 20260507 22:55:50.845 | INFO | "gpu" : " 17 GB", | 20260507 22:55:50.845 | INFO | "huge" : "", | 20260507 22:55:50.845 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:50.845 | INFO | "malloc" : " 14 GB", | 20260507 22:55:50.845 | INFO | "pinned" : " 29 GB" | 20260507 22:55:50.845 | INFO | } | 20260507 22:55:50.845 | INFO | Perf report for the last 5.13807 seconds: | 20260507 22:55:50.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:50.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.845 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260507 22:55:50.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.845 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:55:50.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:50.846 | INFO | Uploaded 36GiB to GPU in 2.46705s (14.5923GiB/s) | 20260507 22:55:50.846 | INFO | Downloaded 34GiB to GPU in 2.38795s (14.2381GiB/s) | 20260507 22:55:55.986 | INFO | Analyzing chunk frames = [3250176,3250688) | 20260507 22:55:55.986 | INFO | MemInfo: { | 20260507 22:55:55.986 | INFO | "gpu" : " 17 GB", | 20260507 22:55:55.986 | INFO | "huge" : "", | 20260507 22:55:55.986 | INFO | "huge_shared" : " 29 GB", | 20260507 22:55:55.986 | INFO | "malloc" : " 14 GB", | 20260507 22:55:55.986 | INFO | "pinned" : " 29 GB" | 20260507 22:55:55.986 | INFO | } | 20260507 22:55:55.987 | INFO | Perf report for the last 5.14127 seconds: | 20260507 22:55:55.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:55:55.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 22:55:55.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 22:55:55.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:55:55.987 | INFO | Uploaded 36GiB to GPU in 2.4016s (14.99GiB/s) | 20260507 22:55:55.987 | INFO | Downloaded 34GiB to GPU in 2.48617s (13.6757GiB/s) | 20260507 22:56:01.127 | INFO | Analyzing chunk frames = [3250688,3251200) | 20260507 22:56:01.127 | INFO | MemInfo: { | 20260507 22:56:01.127 | INFO | "gpu" : " 17 GB", | 20260507 22:56:01.127 | INFO | "huge" : "", | 20260507 22:56:01.127 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:01.127 | INFO | "malloc" : " 14 GB", | 20260507 22:56:01.127 | INFO | "pinned" : " 29 GB" | 20260507 22:56:01.127 | INFO | } | 20260507 22:56:01.127 | INFO | Perf report for the last 5.14062 seconds: | 20260507 22:56:01.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:01.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.127 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:56:01.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.127 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 22:56:01.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:01.128 | INFO | Uploaded 36GiB to GPU in 2.53324s (14.211GiB/s) | 20260507 22:56:01.128 | INFO | Downloaded 34GiB to GPU in 2.4155s (14.0758GiB/s) | 20260507 22:56:06.271 | INFO | Analyzing chunk frames = [3251200,3251712) | 20260507 22:56:06.271 | INFO | MemInfo: { | 20260507 22:56:06.271 | INFO | "gpu" : " 17 GB", | 20260507 22:56:06.271 | INFO | "huge" : "", | 20260507 22:56:06.271 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:06.271 | INFO | "malloc" : " 14 GB", | 20260507 22:56:06.271 | INFO | "pinned" : " 29 GB" | 20260507 22:56:06.271 | INFO | } | 20260507 22:56:06.271 | INFO | Perf report for the last 5.14399 seconds: | 20260507 22:56:06.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:06.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 22:56:06.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:56:06.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:06.271 | INFO | Uploaded 36GiB to GPU in 2.5408s (14.1688GiB/s) | 20260507 22:56:06.272 | INFO | Downloaded 34GiB to GPU in 2.43941s (13.9378GiB/s) | 20260507 22:56:11.408 | INFO | Analyzing chunk frames = [3251712,3252224) | 20260507 22:56:11.408 | INFO | MemInfo: { | 20260507 22:56:11.408 | INFO | "gpu" : " 17 GB", | 20260507 22:56:11.408 | INFO | "huge" : "", | 20260507 22:56:11.408 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:11.408 | INFO | "malloc" : " 14 GB", | 20260507 22:56:11.408 | INFO | "pinned" : " 29 GB" | 20260507 22:56:11.408 | INFO | } | 20260507 22:56:11.408 | INFO | Perf report for the last 5.13708 seconds: | 20260507 22:56:11.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:11.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.408 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:56:11.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.408 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 22:56:11.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:11.409 | INFO | Uploaded 36GiB to GPU in 2.53841s (14.1821GiB/s) | 20260507 22:56:11.409 | INFO | Downloaded 34GiB to GPU in 2.44272s (13.9189GiB/s) | 20260507 22:56:11.409 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:56:11.409 | INFO | Total mem usage: 16.7 GiB | 20260507 22:56:11.409 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:56:11.409 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260507 22:56:11.409 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:56:11.409 | INFO | High Water Breakdown: | 20260507 22:56:11.409 | INFO | 59.9% : 8.87 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:56:11.409 | INFO | 40% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:56:11.409 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:56:11.409 | INFO | ------------------------------------------ | 20260507 22:56:11.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:56:11.409 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:56:11.409 | INFO | Total mem usage: 26.5 GiB | 20260507 22:56:11.409 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:56:11.409 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 22:56:11.409 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 22:56:11.409 | INFO | High Water Breakdown: | 20260507 22:56:11.409 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:56:11.409 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:56:11.409 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:56:11.409 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:56:11.409 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:56:11.409 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:56:11.409 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:56:11.409 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:56:11.409 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:56:11.409 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:56:11.409 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:56:11.409 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:56:11.409 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:56:11.409 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:56:11.409 | INFO | ------------------------------------------ | 20260507 22:56:11.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:56:11.409 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:56:11.409 | INFO | Total mem usage: 27.8 GiB | 20260507 22:56:11.409 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:56:11.409 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:56:11.409 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:56:11.409 | INFO | High Water Breakdown: | 20260507 22:56:11.409 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:56:11.409 | INFO | ------------------------------------------ | 20260507 22:56:11.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:56:11.409 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:56:11.409 | INFO | Total mem usage: 15.7 GiB | 20260507 22:56:11.409 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:56:11.409 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:56:11.409 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:56:11.409 | INFO | High Water Breakdown: | 20260507 22:56:11.409 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:56:11.409 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:56:11.409 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:56:11.409 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:56:11.409 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:56:11.409 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:56:11.409 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:56:11.409 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:56:11.409 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:56:11.409 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:56:11.409 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:56:11.409 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:56:11.409 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:56:11.409 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:56:11.409 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:56:11.409 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:56:11.409 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:56:11.409 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:56:11.409 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:56:11.409 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:56:11.409 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:56:11.409 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:56:11.409 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:56:11.409 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:56:11.409 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:56:11.409 | INFO | ------------------------------------------ | 20260507 22:56:11.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:56:11.409 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:56:11.409 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:56:11.409 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:56:11.409 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.41e+01 / 1.39e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.01e-01, 1.13e-02 / 4.86e-03 | 20260507 22:56:11.409 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.49e+00, 5.84e-01 / 1.95e-01 | 20260507 22:56:11.409 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.09e-01 / 1.07e-01 | 20260507 22:56:11.409 | INFO | PerfReport: 16.01%, DME -- 4.21e+01 / 4.59e+01, 4.02e+00 / 1.65e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.64e+00 / 1.52e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.47e+00 / 2.78e+00, 4.30e-01 / 2.04e-01 | 20260507 22:56:11.409 | INFO | PerfReport: 3.48%, Metrics -- 1.14e+00 / 1.39e+00, 2.88e-01 / 1.75e-01 | 20260507 22:56:11.409 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.53e-01 / 3.93e-01, 5.68e-02 / 3.05e-02 | 20260507 22:56:11.409 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:56:11.409 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:56:11.409 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:56:11.409 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 24.40%, Upload -- 2.61e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 24.22%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 30.74%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 2.70%, LockDownload -- 2.89e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 15.76%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 22:56:11.409 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260507 22:56:11.506 | INFO | Starting a round of Baz IO | 20260507 22:56:14.969 | INFO | Populating RTMetrics report | 20260507 22:56:14.969 | INFO | RTMetrics report populated | 20260507 22:56:14.969 | INFO | Opening tmp RTMetrics json file | 20260507 22:56:14.970 | INFO | Serializing RTMetrics to json | 20260507 22:56:14.978 | INFO | Serialization done, swapping tmp file now... | 20260507 22:56:14.991 | INFO | RTMetrics done | 20260507 22:56:15.057 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.551 seconds | 20260507 22:56:15.057 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.86% / 0.01% / 57.14% | 20260507 22:56:15.057 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30767% / 7.94729e-06% / 5.5631e-05% | 20260507 22:56:15.057 | INFO | Baz IO overall: | 20260507 22:56:15.057 | INFO | BytesWritten : 1481440487119 | 20260507 22:56:15.057 | INFO | NumEvents : 717589917223 | 20260507 22:56:16.545 | INFO | Analyzing chunk frames = [3252224,3252736) | 20260507 22:56:16.545 | INFO | MemInfo: { | 20260507 22:56:16.545 | INFO | "gpu" : " 17 GB", | 20260507 22:56:16.545 | INFO | "huge" : "", | 20260507 22:56:16.545 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:16.545 | INFO | "malloc" : " 14 GB", | 20260507 22:56:16.545 | INFO | "pinned" : " 29 GB" | 20260507 22:56:16.545 | INFO | } | 20260507 22:56:16.545 | INFO | Perf report for the last 5.13664 seconds: | 20260507 22:56:16.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:16.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:16.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:16.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:16.545 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 22:56:16.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:16.545 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 22:56:16.545 | INFO | PRE_HQ : 65.8%, 1.7, 57.3ms, 0 | 20260507 22:56:16.545 | INFO | BAZWRITER : 68.7%, 4.3, 151.1ms, 0 | 20260507 22:56:16.545 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:56:16.546 | INFO | Uploaded 36GiB to GPU in 2.56788s (14.0193GiB/s) | 20260507 22:56:16.546 | INFO | Downloaded 38GiB to GPU in 2.68141s (14.1717GiB/s) | 20260507 22:56:21.688 | INFO | Analyzing chunk frames = [3252736,3253248) | 20260507 22:56:21.688 | INFO | MemInfo: { | 20260507 22:56:21.688 | INFO | "gpu" : " 17 GB", | 20260507 22:56:21.688 | INFO | "huge" : "", | 20260507 22:56:21.688 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:21.688 | INFO | "malloc" : " 14 GB", | 20260507 22:56:21.688 | INFO | "pinned" : " 29 GB" | 20260507 22:56:21.688 | INFO | } | 20260507 22:56:21.688 | INFO | Perf report for the last 5.14294 seconds: | 20260507 22:56:21.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:21.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.688 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260507 22:56:21.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.688 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:56:21.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:21.689 | INFO | Uploaded 36GiB to GPU in 2.58208s (13.9422GiB/s) | 20260507 22:56:21.689 | INFO | Downloaded 34GiB to GPU in 2.47014s (13.7644GiB/s) | 20260507 22:56:26.820 | INFO | Analyzing chunk frames = [3253248,3253760) | 20260507 22:56:26.820 | INFO | MemInfo: { | 20260507 22:56:26.820 | INFO | "gpu" : " 17 GB", | 20260507 22:56:26.821 | INFO | "huge" : "", | 20260507 22:56:26.821 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:26.821 | INFO | "malloc" : " 14 GB", | 20260507 22:56:26.821 | INFO | "pinned" : " 29 GB" | 20260507 22:56:26.821 | INFO | } | 20260507 22:56:26.821 | INFO | Perf report for the last 5.13275 seconds: | 20260507 22:56:26.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:26.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 22:56:26.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:56:26.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:26.821 | INFO | Uploaded 36GiB to GPU in 2.52594s (14.2521GiB/s) | 20260507 22:56:26.821 | INFO | Downloaded 34GiB to GPU in 2.47428s (13.7414GiB/s) | 20260507 22:56:31.960 | INFO | Analyzing chunk frames = [3253760,3254272) | 20260507 22:56:31.960 | INFO | MemInfo: { | 20260507 22:56:31.960 | INFO | "gpu" : " 17 GB", | 20260507 22:56:31.960 | INFO | "huge" : "", | 20260507 22:56:31.960 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:31.960 | INFO | "malloc" : " 14 GB", | 20260507 22:56:31.961 | INFO | "pinned" : " 29 GB" | 20260507 22:56:31.961 | INFO | } | 20260507 22:56:31.961 | INFO | Perf report for the last 5.13993 seconds: | 20260507 22:56:31.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:31.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 22:56:31.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:56:31.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:31.961 | INFO | Uploaded 36GiB to GPU in 2.48271s (14.5003GiB/s) | 20260507 22:56:31.961 | INFO | Downloaded 34GiB to GPU in 2.3758s (14.311GiB/s) | 20260507 22:56:37.098 | INFO | Analyzing chunk frames = [3254272,3254784) | 20260507 22:56:37.098 | INFO | MemInfo: { | 20260507 22:56:37.098 | INFO | "gpu" : " 17 GB", | 20260507 22:56:37.098 | INFO | "huge" : "", | 20260507 22:56:37.098 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:37.098 | INFO | "malloc" : " 14 GB", | 20260507 22:56:37.098 | INFO | "pinned" : " 29 GB" | 20260507 22:56:37.098 | INFO | } | 20260507 22:56:37.098 | INFO | Perf report for the last 5.13731 seconds: | 20260507 22:56:37.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:37.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 22:56:37.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 22:56:37.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:37.098 | INFO | Uploaded 36GiB to GPU in 2.38368s (15.1027GiB/s) | 20260507 22:56:37.098 | INFO | Downloaded 34GiB to GPU in 2.4973s (13.6147GiB/s) | 20260507 22:56:42.241 | INFO | Analyzing chunk frames = [3254784,3255296) | 20260507 22:56:42.241 | INFO | MemInfo: { | 20260507 22:56:42.241 | INFO | "gpu" : " 17 GB", | 20260507 22:56:42.241 | INFO | "huge" : "", | 20260507 22:56:42.241 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:42.241 | INFO | "malloc" : " 14 GB", | 20260507 22:56:42.241 | INFO | "pinned" : " 29 GB" | 20260507 22:56:42.241 | INFO | } | 20260507 22:56:42.241 | INFO | Perf report for the last 5.14328 seconds: | 20260507 22:56:42.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:42.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 22:56:42.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:56:42.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:42.241 | INFO | Uploaded 36GiB to GPU in 2.49806s (14.4112GiB/s) | 20260507 22:56:42.241 | INFO | Downloaded 34GiB to GPU in 2.40535s (14.1352GiB/s) | 20260507 22:56:47.386 | INFO | Analyzing chunk frames = [3255296,3255808) | 20260507 22:56:47.386 | INFO | MemInfo: { | 20260507 22:56:47.386 | INFO | "gpu" : " 17 GB", | 20260507 22:56:47.386 | INFO | "huge" : "", | 20260507 22:56:47.386 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:47.386 | INFO | "malloc" : " 14 GB", | 20260507 22:56:47.386 | INFO | "pinned" : " 29 GB" | 20260507 22:56:47.386 | INFO | } | 20260507 22:56:47.386 | INFO | Perf report for the last 5.14482 seconds: | 20260507 22:56:47.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:47.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.386 | INFO | ANALYSIS : 67.2%, 3.0, 105.1ms, 0 | 20260507 22:56:47.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.386 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 22:56:47.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:47.387 | INFO | Uploaded 36GiB to GPU in 2.51491s (14.3146GiB/s) | 20260507 22:56:47.387 | INFO | Downloaded 34GiB to GPU in 2.43436s (13.9667GiB/s) | 20260507 22:56:52.521 | INFO | Analyzing chunk frames = [3255808,3256320) | 20260507 22:56:52.521 | INFO | MemInfo: { | 20260507 22:56:52.521 | INFO | "gpu" : " 17 GB", | 20260507 22:56:52.521 | INFO | "huge" : "", | 20260507 22:56:52.521 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:52.521 | INFO | "malloc" : " 14 GB", | 20260507 22:56:52.521 | INFO | "pinned" : " 29 GB" | 20260507 22:56:52.521 | INFO | } | 20260507 22:56:52.522 | INFO | Perf report for the last 5.13541 seconds: | 20260507 22:56:52.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:52.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 22:56:52.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 22:56:52.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:52.522 | INFO | Uploaded 36GiB to GPU in 2.50247s (14.3858GiB/s) | 20260507 22:56:52.522 | INFO | Downloaded 34GiB to GPU in 2.42829s (14.0016GiB/s) | 20260507 22:56:52.637 | INFO | Starting a round of Baz IO | 20260507 22:56:56.117 | INFO | Populating RTMetrics report | 20260507 22:56:56.117 | INFO | RTMetrics report populated | 20260507 22:56:56.117 | INFO | Opening tmp RTMetrics json file | 20260507 22:56:56.151 | INFO | Serializing RTMetrics to json | 20260507 22:56:56.245 | INFO | Serialization done, swapping tmp file now... | 20260507 22:56:56.250 | INFO | RTMetrics done | 20260507 22:56:56.276 | INFO | Finished a round of Baz IO: Wrote 1.727 GiB in 3.639 seconds | 20260507 22:56:56.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.85% / 0.01% / 57.14% | 20260507 22:56:56.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30314% / 0% / 5.16574e-05% | 20260507 22:56:56.276 | INFO | Baz IO overall: | 20260507 22:56:56.276 | INFO | BytesWritten : 1483295066119 | 20260507 22:56:56.276 | INFO | NumEvents : 718484650572 | 20260507 22:56:57.655 | INFO | Analyzing chunk frames = [3256320,3256832) | 20260507 22:56:57.655 | INFO | MemInfo: { | 20260507 22:56:57.655 | INFO | "gpu" : " 17 GB", | 20260507 22:56:57.656 | INFO | "huge" : "", | 20260507 22:56:57.656 | INFO | "huge_shared" : " 29 GB", | 20260507 22:56:57.656 | INFO | "malloc" : " 14 GB", | 20260507 22:56:57.656 | INFO | "pinned" : " 29 GB" | 20260507 22:56:57.656 | INFO | } | 20260507 22:56:57.656 | INFO | Perf report for the last 5.1342 seconds: | 20260507 22:56:57.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:56:57.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:57.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:57.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:57.656 | INFO | ANALYSIS : 69.1%, 2.9, 105.0ms, 0 | 20260507 22:56:57.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:56:57.656 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260507 22:56:57.656 | INFO | PRE_HQ : 67.7%, 1.8, 62.2ms, 0 | 20260507 22:56:57.656 | INFO | BAZWRITER : 70.6%, 3.8, 136.2ms, 0 | 20260507 22:56:57.656 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.4ms, 0 | 20260507 22:56:57.656 | INFO | Uploaded 36GiB to GPU in 2.6174s (13.7541GiB/s) | 20260507 22:56:57.656 | INFO | Downloaded 38GiB to GPU in 2.69319s (14.1097GiB/s) | 20260507 22:57:02.796 | INFO | Analyzing chunk frames = [3256832,3257344) | 20260507 22:57:02.796 | INFO | MemInfo: { | 20260507 22:57:02.796 | INFO | "gpu" : " 17 GB", | 20260507 22:57:02.796 | INFO | "huge" : "", | 20260507 22:57:02.796 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:02.796 | INFO | "malloc" : " 14 GB", | 20260507 22:57:02.796 | INFO | "pinned" : " 29 GB" | 20260507 22:57:02.796 | INFO | } | 20260507 22:57:02.796 | INFO | Perf report for the last 5.14052 seconds: | 20260507 22:57:02.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:02.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.796 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 22:57:02.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.796 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 22:57:02.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:02.797 | INFO | Uploaded 36GiB to GPU in 2.54579s (14.141GiB/s) | 20260507 22:57:02.797 | INFO | Downloaded 34GiB to GPU in 2.45558s (13.846GiB/s) | 20260507 22:57:07.937 | INFO | Analyzing chunk frames = [3257344,3257856) | 20260507 22:57:07.937 | INFO | MemInfo: { | 20260507 22:57:07.937 | INFO | "gpu" : " 17 GB", | 20260507 22:57:07.937 | INFO | "huge" : "", | 20260507 22:57:07.937 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:07.937 | INFO | "malloc" : " 14 GB", | 20260507 22:57:07.937 | INFO | "pinned" : " 29 GB" | 20260507 22:57:07.937 | INFO | } | 20260507 22:57:07.937 | INFO | Perf report for the last 5.14105 seconds: | 20260507 22:57:07.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:07.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.937 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 22:57:07.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.937 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:57:07.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:07.938 | INFO | Uploaded 36GiB to GPU in 2.56997s (14.0079GiB/s) | 20260507 22:57:07.938 | INFO | Downloaded 34GiB to GPU in 2.45194s (13.8666GiB/s) | 20260507 22:57:13.079 | INFO | Analyzing chunk frames = [3257856,3258368) | 20260507 22:57:13.079 | INFO | MemInfo: { | 20260507 22:57:13.079 | INFO | "gpu" : " 17 GB", | 20260507 22:57:13.080 | INFO | "huge" : "", | 20260507 22:57:13.080 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:13.080 | INFO | "malloc" : " 14 GB", | 20260507 22:57:13.080 | INFO | "pinned" : " 29 GB" | 20260507 22:57:13.080 | INFO | } | 20260507 22:57:13.080 | INFO | Perf report for the last 5.14246 seconds: | 20260507 22:57:13.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:13.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 22:57:13.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:57:13.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:13.080 | INFO | Uploaded 36GiB to GPU in 2.52916s (14.234GiB/s) | 20260507 22:57:13.080 | INFO | Downloaded 34GiB to GPU in 2.48582s (13.6776GiB/s) | 20260507 22:57:18.222 | INFO | Analyzing chunk frames = [3258368,3258880) | 20260507 22:57:18.222 | INFO | MemInfo: { | 20260507 22:57:18.222 | INFO | "gpu" : " 17 GB", | 20260507 22:57:18.223 | INFO | "huge" : "", | 20260507 22:57:18.223 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:18.223 | INFO | "malloc" : " 14 GB", | 20260507 22:57:18.223 | INFO | "pinned" : " 29 GB" | 20260507 22:57:18.223 | INFO | } | 20260507 22:57:18.223 | INFO | Perf report for the last 5.14303 seconds: | 20260507 22:57:18.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:18.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 22:57:18.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 22:57:18.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:18.223 | INFO | Uploaded 36GiB to GPU in 2.38539s (15.0919GiB/s) | 20260507 22:57:18.223 | INFO | Downloaded 34GiB to GPU in 2.50141s (13.5923GiB/s) | 20260507 22:57:23.356 | INFO | Analyzing chunk frames = [3258880,3259392) | 20260507 22:57:23.356 | INFO | MemInfo: { | 20260507 22:57:23.356 | INFO | "gpu" : " 17 GB", | 20260507 22:57:23.356 | INFO | "huge" : "", | 20260507 22:57:23.357 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:23.357 | INFO | "malloc" : " 14 GB", | 20260507 22:57:23.357 | INFO | "pinned" : " 29 GB" | 20260507 22:57:23.357 | INFO | } | 20260507 22:57:23.357 | INFO | Perf report for the last 5.1339 seconds: | 20260507 22:57:23.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:23.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 22:57:23.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:57:23.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:23.357 | INFO | Uploaded 36GiB to GPU in 2.42457s (14.848GiB/s) | 20260507 22:57:23.357 | INFO | Downloaded 34GiB to GPU in 2.39235s (14.212GiB/s) | 20260507 22:57:28.496 | INFO | Analyzing chunk frames = [3259392,3259904) | 20260507 22:57:28.496 | INFO | MemInfo: { | 20260507 22:57:28.496 | INFO | "gpu" : " 17 GB", | 20260507 22:57:28.496 | INFO | "huge" : "", | 20260507 22:57:28.496 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:28.496 | INFO | "malloc" : " 14 GB", | 20260507 22:57:28.497 | INFO | "pinned" : " 29 GB" | 20260507 22:57:28.497 | INFO | } | 20260507 22:57:28.497 | INFO | Perf report for the last 5.13992 seconds: | 20260507 22:57:28.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:28.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 22:57:28.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 22:57:28.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:28.497 | INFO | Uploaded 36GiB to GPU in 2.56852s (14.0159GiB/s) | 20260507 22:57:28.497 | INFO | Downloaded 34GiB to GPU in 2.45148s (13.8692GiB/s) | 20260507 22:57:33.639 | INFO | Analyzing chunk frames = [3259904,3260416) | 20260507 22:57:33.639 | INFO | MemInfo: { | 20260507 22:57:33.639 | INFO | "gpu" : " 17 GB", | 20260507 22:57:33.639 | INFO | "huge" : "", | 20260507 22:57:33.640 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:33.640 | INFO | "malloc" : " 14 GB", | 20260507 22:57:33.640 | INFO | "pinned" : " 29 GB" | 20260507 22:57:33.640 | INFO | } | 20260507 22:57:33.640 | INFO | Perf report for the last 5.14313 seconds: | 20260507 22:57:33.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:33.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 22:57:33.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260507 22:57:33.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:33.640 | INFO | Uploaded 36GiB to GPU in 2.5215s (14.2772GiB/s) | 20260507 22:57:33.640 | INFO | Downloaded 34GiB to GPU in 2.44145s (13.9261GiB/s) | 20260507 22:57:33.640 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:57:33.640 | INFO | Total mem usage: 16.7 GiB | 20260507 22:57:33.640 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:57:33.640 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260507 22:57:33.640 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:57:33.640 | INFO | High Water Breakdown: | 20260507 22:57:33.640 | INFO | 60.2% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:57:33.640 | INFO | 39.7% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:57:33.640 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:57:33.640 | INFO | ------------------------------------------ | 20260507 22:57:33.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:57:33.641 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:57:33.641 | INFO | Total mem usage: 26.5 GiB | 20260507 22:57:33.641 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:57:33.641 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 22:57:33.641 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 22:57:33.641 | INFO | High Water Breakdown: | 20260507 22:57:33.641 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:57:33.641 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:57:33.641 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:57:33.641 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:57:33.641 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:57:33.641 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:57:33.641 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:57:33.641 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:57:33.641 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:57:33.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:57:33.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:57:33.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:57:33.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:57:33.641 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:57:33.641 | INFO | ------------------------------------------ | 20260507 22:57:33.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:57:33.641 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:57:33.641 | INFO | Total mem usage: 27.8 GiB | 20260507 22:57:33.641 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:57:33.641 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:57:33.641 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:57:33.641 | INFO | High Water Breakdown: | 20260507 22:57:33.641 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:57:33.641 | INFO | ------------------------------------------ | 20260507 22:57:33.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:57:33.641 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:57:33.641 | INFO | Total mem usage: 15.7 GiB | 20260507 22:57:33.641 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:57:33.641 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:57:33.641 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:57:33.641 | INFO | High Water Breakdown: | 20260507 22:57:33.641 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:57:33.641 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:57:33.641 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:57:33.641 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:57:33.641 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:57:33.641 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:57:33.641 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:57:33.641 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:57:33.641 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:57:33.641 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:57:33.641 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:57:33.641 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:57:33.641 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:57:33.641 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:57:33.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:57:33.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:57:33.641 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:57:33.641 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:57:33.641 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:57:33.641 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:57:33.641 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:57:33.641 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:57:33.641 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:57:33.641 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:57:33.641 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:57:33.641 | INFO | ------------------------------------------ | 20260507 22:57:33.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:57:33.641 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:57:33.641 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:57:33.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:57:33.641 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.64e+01, 1.42e+01 / 1.76e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.09e-02 / 5.96e-03 | 20260507 22:57:33.641 | INFO | PerfReport: 12.14%, Baseline -- 4.00e+00 / 4.48e+00, 5.92e-01 / 2.30e-01 | 20260507 22:57:33.641 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.07e-01 / 1.05e-01 | 20260507 22:57:33.641 | INFO | PerfReport: 15.91%, DME -- 4.19e+01 / 4.54e+01, 4.01e+00 / 1.67e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.74e+00 / 1.77e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.43e+00 / 2.74e+00, 4.29e-01 / 2.26e-01 | 20260507 22:57:33.641 | INFO | PerfReport: 3.45%, Metrics -- 1.14e+00 / 1.33e+00, 2.85e-01 / 1.59e-01 | 20260507 22:57:33.641 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.52e-01 / 3.87e-01, 5.77e-02 / 2.94e-02 | 20260507 22:57:33.641 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:57:33.641 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:57:33.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:57:33.641 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 24.52%, Upload -- 2.62e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 24.17%, Download -- 2.58e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 30.83%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 2.27%, LockUpload -- 2.42e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 2.39%, LockDownload -- 2.55e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260507 22:57:33.641 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260507 22:57:33.737 | INFO | Starting a round of Baz IO | 20260507 22:57:37.203 | INFO | Populating RTMetrics report | 20260507 22:57:37.203 | INFO | RTMetrics report populated | 20260507 22:57:37.203 | INFO | Opening tmp RTMetrics json file | 20260507 22:57:37.226 | INFO | Serializing RTMetrics to json | 20260507 22:57:37.227 | INFO | Serialization done, swapping tmp file now... | 20260507 22:57:37.228 | INFO | RTMetrics done | 20260507 22:57:37.301 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.564 seconds | 20260507 22:57:37.301 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.85% / 0.01% / 57.15% | 20260507 22:57:37.301 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.2973% / 0% / 6.75519e-05% | 20260507 22:57:37.301 | INFO | Baz IO overall: | 20260507 22:57:37.301 | INFO | BytesWritten : 1485147117044 | 20260507 22:57:37.301 | INFO | NumEvents : 719377430095 | 20260507 22:57:38.777 | INFO | Analyzing chunk frames = [3260416,3260928) | 20260507 22:57:38.777 | INFO | MemInfo: { | 20260507 22:57:38.777 | INFO | "gpu" : " 17 GB", | 20260507 22:57:38.777 | INFO | "huge" : "", | 20260507 22:57:38.777 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:38.777 | INFO | "malloc" : " 14 GB", | 20260507 22:57:38.777 | INFO | "pinned" : " 29 GB" | 20260507 22:57:38.777 | INFO | } | 20260507 22:57:38.777 | INFO | Perf report for the last 5.13736 seconds: | 20260507 22:57:38.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:38.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:38.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:38.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:38.777 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 22:57:38.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:38.777 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 22:57:38.777 | INFO | PRE_HQ : 65.8%, 1.7, 56.9ms, 0 | 20260507 22:57:38.777 | INFO | BAZWRITER : 69.4%, 3.8, 136.1ms, 0 | 20260507 22:57:38.777 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 22:57:38.777 | INFO | Uploaded 36GiB to GPU in 2.56954s (14.0103GiB/s) | 20260507 22:57:38.778 | INFO | Downloaded 38GiB to GPU in 2.68727s (14.1408GiB/s) | 20260507 22:57:43.911 | INFO | Analyzing chunk frames = [3260928,3261440) | 20260507 22:57:43.911 | INFO | MemInfo: { | 20260507 22:57:43.911 | INFO | "gpu" : " 17 GB", | 20260507 22:57:43.911 | INFO | "huge" : "", | 20260507 22:57:43.911 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:43.911 | INFO | "malloc" : " 14 GB", | 20260507 22:57:43.911 | INFO | "pinned" : " 29 GB" | 20260507 22:57:43.911 | INFO | } | 20260507 22:57:43.911 | INFO | Perf report for the last 5.13391 seconds: | 20260507 22:57:43.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:43.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 22:57:43.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 22:57:43.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:43.911 | INFO | Uploaded 36GiB to GPU in 2.57332s (13.9897GiB/s) | 20260507 22:57:43.911 | INFO | Downloaded 34GiB to GPU in 2.46428s (13.7971GiB/s) | 20260507 22:57:49.049 | INFO | Analyzing chunk frames = [3261440,3261952) | 20260507 22:57:49.049 | INFO | MemInfo: { | 20260507 22:57:49.049 | INFO | "gpu" : " 17 GB", | 20260507 22:57:49.049 | INFO | "huge" : "", | 20260507 22:57:49.049 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:49.050 | INFO | "malloc" : " 14 GB", | 20260507 22:57:49.050 | INFO | "pinned" : " 29 GB" | 20260507 22:57:49.050 | INFO | } | 20260507 22:57:49.050 | INFO | Perf report for the last 5.13853 seconds: | 20260507 22:57:49.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:49.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 22:57:49.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:57:49.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:49.050 | INFO | Uploaded 36GiB to GPU in 2.55425s (14.0941GiB/s) | 20260507 22:57:49.050 | INFO | Downloaded 34GiB to GPU in 2.45212s (13.8655GiB/s) | 20260507 22:57:54.192 | INFO | Analyzing chunk frames = [3261952,3262464) | 20260507 22:57:54.192 | INFO | MemInfo: { | 20260507 22:57:54.192 | INFO | "gpu" : " 17 GB", | 20260507 22:57:54.192 | INFO | "huge" : "", | 20260507 22:57:54.192 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:54.192 | INFO | "malloc" : " 14 GB", | 20260507 22:57:54.192 | INFO | "pinned" : " 29 GB" | 20260507 22:57:54.193 | INFO | } | 20260507 22:57:54.193 | INFO | Perf report for the last 5.14294 seconds: | 20260507 22:57:54.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:54.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260507 22:57:54.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 22:57:54.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:54.193 | INFO | Uploaded 36GiB to GPU in 2.58017s (13.9526GiB/s) | 20260507 22:57:54.193 | INFO | Downloaded 34GiB to GPU in 2.46959s (13.7675GiB/s) | 20260507 22:57:59.339 | INFO | Analyzing chunk frames = [3262464,3262976) | 20260507 22:57:59.339 | INFO | MemInfo: { | 20260507 22:57:59.339 | INFO | "gpu" : " 17 GB", | 20260507 22:57:59.340 | INFO | "huge" : "", | 20260507 22:57:59.340 | INFO | "huge_shared" : " 29 GB", | 20260507 22:57:59.340 | INFO | "malloc" : " 14 GB", | 20260507 22:57:59.340 | INFO | "pinned" : " 29 GB" | 20260507 22:57:59.340 | INFO | } | 20260507 22:57:59.340 | INFO | Perf report for the last 5.14706 seconds: | 20260507 22:57:59.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:57:59.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260507 22:57:59.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:57:59.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:57:59.340 | INFO | Uploaded 36GiB to GPU in 2.35013s (15.3183GiB/s) | 20260507 22:57:59.340 | INFO | Downloaded 34GiB to GPU in 2.49642s (13.6195GiB/s) | 20260507 22:58:04.475 | INFO | Analyzing chunk frames = [3262976,3263488) | 20260507 22:58:04.475 | INFO | MemInfo: { | 20260507 22:58:04.475 | INFO | "gpu" : " 17 GB", | 20260507 22:58:04.475 | INFO | "huge" : "", | 20260507 22:58:04.475 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:04.476 | INFO | "malloc" : " 14 GB", | 20260507 22:58:04.476 | INFO | "pinned" : " 29 GB" | 20260507 22:58:04.476 | INFO | } | 20260507 22:58:04.476 | INFO | Perf report for the last 5.13599 seconds: | 20260507 22:58:04.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:04.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 22:58:04.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:58:04.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:04.476 | INFO | Uploaded 36GiB to GPU in 2.38172s (15.1151GiB/s) | 20260507 22:58:04.476 | INFO | Downloaded 34GiB to GPU in 2.40109s (14.1602GiB/s) | 20260507 22:58:09.608 | INFO | Analyzing chunk frames = [3263488,3264000) | 20260507 22:58:09.608 | INFO | MemInfo: { | 20260507 22:58:09.608 | INFO | "gpu" : " 17 GB", | 20260507 22:58:09.608 | INFO | "huge" : "", | 20260507 22:58:09.608 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:09.608 | INFO | "malloc" : " 14 GB", | 20260507 22:58:09.608 | INFO | "pinned" : " 29 GB" | 20260507 22:58:09.608 | INFO | } | 20260507 22:58:09.608 | INFO | Perf report for the last 5.13248 seconds: | 20260507 22:58:09.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:09.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.608 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 22:58:09.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.608 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 22:58:09.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:09.609 | INFO | Uploaded 36GiB to GPU in 2.54514s (14.1446GiB/s) | 20260507 22:58:09.609 | INFO | Downloaded 34GiB to GPU in 2.45782s (13.8334GiB/s) | 20260507 22:58:14.748 | INFO | Analyzing chunk frames = [3264000,3264512) | 20260507 22:58:14.748 | INFO | MemInfo: { | 20260507 22:58:14.748 | INFO | "gpu" : " 17 GB", | 20260507 22:58:14.748 | INFO | "huge" : "", | 20260507 22:58:14.748 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:14.748 | INFO | "malloc" : " 14 GB", | 20260507 22:58:14.748 | INFO | "pinned" : " 29 GB" | 20260507 22:58:14.748 | INFO | } | 20260507 22:58:14.748 | INFO | Perf report for the last 5.13954 seconds: | 20260507 22:58:14.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:14.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:58:14.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 22:58:14.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:14.748 | INFO | Uploaded 36GiB to GPU in 2.52649s (14.249GiB/s) | 20260507 22:58:14.748 | INFO | Downloaded 34GiB to GPU in 2.441s (13.9287GiB/s) | 20260507 22:58:14.861 | INFO | Starting a round of Baz IO | 20260507 22:58:18.342 | INFO | Populating RTMetrics report | 20260507 22:58:18.342 | INFO | RTMetrics report populated | 20260507 22:58:18.342 | INFO | Opening tmp RTMetrics json file | 20260507 22:58:18.375 | INFO | Serializing RTMetrics to json | 20260507 22:58:18.431 | INFO | Serialization done, swapping tmp file now... | 20260507 22:58:18.457 | INFO | RTMetrics done | 20260507 22:58:18.484 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.622 seconds | 20260507 22:58:18.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.84% / 0.01% / 57.15% | 20260507 22:58:18.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.29596% / 0% / 5.5631e-05% | 20260507 22:58:18.484 | INFO | Baz IO overall: | 20260507 22:58:18.484 | INFO | BytesWritten : 1487000244857 | 20260507 22:58:18.484 | INFO | NumEvents : 720271140959 | 20260507 22:58:19.891 | INFO | Analyzing chunk frames = [3264512,3265024) | 20260507 22:58:19.891 | INFO | MemInfo: { | 20260507 22:58:19.891 | INFO | "gpu" : " 17 GB", | 20260507 22:58:19.891 | INFO | "huge" : "", | 20260507 22:58:19.891 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:19.891 | INFO | "malloc" : " 14 GB", | 20260507 22:58:19.891 | INFO | "pinned" : " 29 GB" | 20260507 22:58:19.891 | INFO | } | 20260507 22:58:19.891 | INFO | Perf report for the last 5.143 seconds: | 20260507 22:58:19.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:19.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:19.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:19.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:19.891 | INFO | ANALYSIS : 69.0%, 2.9, 104.8ms, 0 | 20260507 22:58:19.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:19.891 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260507 22:58:19.891 | INFO | PRE_HQ : 67.8%, 1.8, 63.6ms, 0 | 20260507 22:58:19.891 | INFO | BAZWRITER : 70.0%, 4.2, 150.6ms, 0 | 20260507 22:58:19.891 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260507 22:58:19.891 | INFO | Uploaded 36GiB to GPU in 2.58264s (13.9392GiB/s) | 20260507 22:58:19.891 | INFO | Downloaded 38GiB to GPU in 2.70222s (14.0625GiB/s) | 20260507 22:58:25.030 | INFO | Analyzing chunk frames = [3265024,3265536) | 20260507 22:58:25.030 | INFO | MemInfo: { | 20260507 22:58:25.030 | INFO | "gpu" : " 17 GB", | 20260507 22:58:25.030 | INFO | "huge" : "", | 20260507 22:58:25.030 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:25.030 | INFO | "malloc" : " 14 GB", | 20260507 22:58:25.030 | INFO | "pinned" : " 29 GB" | 20260507 22:58:25.030 | INFO | } | 20260507 22:58:25.030 | INFO | Perf report for the last 5.13926 seconds: | 20260507 22:58:25.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:25.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.030 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 22:58:25.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.030 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 22:58:25.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:25.031 | INFO | Uploaded 36GiB to GPU in 2.56147s (14.0544GiB/s) | 20260507 22:58:25.031 | INFO | Downloaded 34GiB to GPU in 2.48915s (13.6593GiB/s) | 20260507 22:58:30.172 | INFO | Analyzing chunk frames = [3265536,3266048) | 20260507 22:58:30.172 | INFO | MemInfo: { | 20260507 22:58:30.172 | INFO | "gpu" : " 17 GB", | 20260507 22:58:30.172 | INFO | "huge" : "", | 20260507 22:58:30.173 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:30.173 | INFO | "malloc" : " 14 GB", | 20260507 22:58:30.173 | INFO | "pinned" : " 29 GB" | 20260507 22:58:30.173 | INFO | } | 20260507 22:58:30.173 | INFO | Perf report for the last 5.1426 seconds: | 20260507 22:58:30.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:30.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.173 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 22:58:30.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.173 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 22:58:30.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:30.174 | INFO | Uploaded 36GiB to GPU in 2.4614s (14.6258GiB/s) | 20260507 22:58:30.174 | INFO | Downloaded 34GiB to GPU in 2.49041s (13.6523GiB/s) | 20260507 22:58:35.308 | INFO | Analyzing chunk frames = [3266048,3266560) | 20260507 22:58:35.308 | INFO | MemInfo: { | 20260507 22:58:35.308 | INFO | "gpu" : " 17 GB", | 20260507 22:58:35.308 | INFO | "huge" : "", | 20260507 22:58:35.308 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:35.308 | INFO | "malloc" : " 14 GB", | 20260507 22:58:35.308 | INFO | "pinned" : " 29 GB" | 20260507 22:58:35.308 | INFO | } | 20260507 22:58:35.308 | INFO | Perf report for the last 5.13522 seconds: | 20260507 22:58:35.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:35.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 22:58:35.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:58:35.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:35.308 | INFO | Uploaded 36GiB to GPU in 2.4861s (14.4805GiB/s) | 20260507 22:58:35.308 | INFO | Downloaded 34GiB to GPU in 2.48517s (13.6812GiB/s) | 20260507 22:58:40.451 | INFO | Analyzing chunk frames = [3266560,3267072) | 20260507 22:58:40.451 | INFO | MemInfo: { | 20260507 22:58:40.451 | INFO | "gpu" : " 17 GB", | 20260507 22:58:40.451 | INFO | "huge" : "", | 20260507 22:58:40.451 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:40.451 | INFO | "malloc" : " 14 GB", | 20260507 22:58:40.451 | INFO | "pinned" : " 29 GB" | 20260507 22:58:40.451 | INFO | } | 20260507 22:58:40.451 | INFO | Perf report for the last 5.14278 seconds: | 20260507 22:58:40.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:40.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 22:58:40.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 22:58:40.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:40.451 | INFO | Uploaded 36GiB to GPU in 2.38246s (15.1104GiB/s) | 20260507 22:58:40.451 | INFO | Downloaded 34GiB to GPU in 2.49515s (13.6264GiB/s) | 20260507 22:58:45.590 | INFO | Analyzing chunk frames = [3267072,3267584) | 20260507 22:58:45.590 | INFO | MemInfo: { | 20260507 22:58:45.590 | INFO | "gpu" : " 17 GB", | 20260507 22:58:45.590 | INFO | "huge" : "", | 20260507 22:58:45.590 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:45.590 | INFO | "malloc" : " 14 GB", | 20260507 22:58:45.590 | INFO | "pinned" : " 29 GB" | 20260507 22:58:45.590 | INFO | } | 20260507 22:58:45.590 | INFO | Perf report for the last 5.13935 seconds: | 20260507 22:58:45.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:45.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.590 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260507 22:58:45.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.590 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 22:58:45.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:45.591 | INFO | Uploaded 36GiB to GPU in 2.42359s (14.854GiB/s) | 20260507 22:58:45.591 | INFO | Downloaded 34GiB to GPU in 2.38591s (14.2503GiB/s) | 20260507 22:58:50.726 | INFO | Analyzing chunk frames = [3267584,3268096) | 20260507 22:58:50.726 | INFO | MemInfo: { | 20260507 22:58:50.726 | INFO | "gpu" : " 17 GB", | 20260507 22:58:50.726 | INFO | "huge" : "", | 20260507 22:58:50.726 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:50.726 | INFO | "malloc" : " 14 GB", | 20260507 22:58:50.726 | INFO | "pinned" : " 29 GB" | 20260507 22:58:50.726 | INFO | } | 20260507 22:58:50.726 | INFO | Perf report for the last 5.136 seconds: | 20260507 22:58:50.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:50.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.726 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 22:58:50.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.726 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 22:58:50.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:50.727 | INFO | Uploaded 36GiB to GPU in 2.55129s (14.1105GiB/s) | 20260507 22:58:50.727 | INFO | Downloaded 34GiB to GPU in 2.45549s (13.8465GiB/s) | 20260507 22:58:55.860 | INFO | Analyzing chunk frames = [3268096,3268608) | 20260507 22:58:55.860 | INFO | MemInfo: { | 20260507 22:58:55.860 | INFO | "gpu" : " 17 GB", | 20260507 22:58:55.860 | INFO | "huge" : "", | 20260507 22:58:55.860 | INFO | "huge_shared" : " 29 GB", | 20260507 22:58:55.860 | INFO | "malloc" : " 14 GB", | 20260507 22:58:55.860 | INFO | "pinned" : " 29 GB" | 20260507 22:58:55.860 | INFO | } | 20260507 22:58:55.860 | INFO | Perf report for the last 5.13399 seconds: | 20260507 22:58:55.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:58:55.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.860 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 22:58:55.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.860 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 22:58:55.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:58:55.861 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 22:58:55.861 | INFO | Total mem usage: 16.7 GiB | 20260507 22:58:55.861 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 22:58:55.861 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260507 22:58:55.861 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 22:58:55.861 | INFO | High Water Breakdown: | 20260507 22:58:55.861 | INFO | 60.2% : 9.05 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 22:58:55.861 | INFO | Uploaded 36GiB to GPU in 2.5329s (14.2129GiB/s) | 20260507 22:58:55.861 | INFO | 39.7% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 22:58:55.861 | INFO | Downloaded 34GiB to GPU in 2.44728s (13.893GiB/s) | 20260507 22:58:55.861 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 22:58:55.861 | INFO | ------------------------------------------ | 20260507 22:58:55.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:58:55.861 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 22:58:55.861 | INFO | Total mem usage: 26.5 GiB | 20260507 22:58:55.861 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 22:58:55.861 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 22:58:55.861 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 22:58:55.861 | INFO | High Water Breakdown: | 20260507 22:58:55.861 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:58:55.861 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:58:55.861 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:58:55.861 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:58:55.861 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:58:55.861 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:58:55.861 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 22:58:55.861 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 22:58:55.861 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:58:55.862 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 22:58:55.862 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 22:58:55.862 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 22:58:55.862 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 22:58:55.862 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 22:58:55.862 | INFO | ------------------------------------------ | 20260507 22:58:55.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:58:55.862 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 22:58:55.862 | INFO | Total mem usage: 27.8 GiB | 20260507 22:58:55.862 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 22:58:55.862 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 22:58:55.862 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 22:58:55.862 | INFO | High Water Breakdown: | 20260507 22:58:55.862 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 22:58:55.862 | INFO | ------------------------------------------ | 20260507 22:58:55.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:58:55.862 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 22:58:55.862 | INFO | Total mem usage: 15.7 GiB | 20260507 22:58:55.862 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 22:58:55.862 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 22:58:55.862 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 22:58:55.862 | INFO | High Water Breakdown: | 20260507 22:58:55.862 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 22:58:55.862 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 22:58:55.862 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 22:58:55.862 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 22:58:55.862 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 22:58:55.862 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 22:58:55.862 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 22:58:55.862 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 22:58:55.862 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 22:58:55.862 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 22:58:55.862 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 22:58:55.862 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 22:58:55.862 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 22:58:55.862 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 22:58:55.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 22:58:55.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 22:58:55.862 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 22:58:55.862 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 22:58:55.862 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 22:58:55.862 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 22:58:55.862 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 22:58:55.862 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 22:58:55.862 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 22:58:55.862 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 22:58:55.862 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 22:58:55.862 | INFO | ------------------------------------------ | 20260507 22:58:55.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 22:58:55.862 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 22:58:55.862 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:58:55.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:58:55.862 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.64e+01, 1.42e+01 / 1.77e+00 | 20260507 22:58:55.862 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.02e-01, 1.13e-02 / 6.07e-03 | 20260507 22:58:55.862 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.48e+00, 5.90e-01 / 2.02e-01 | 20260507 22:58:55.862 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.09e-01 / 1.09e-01 | 20260507 22:58:55.862 | INFO | PerfReport: 15.94%, DME -- 4.20e+01 / 4.56e+01, 3.90e+00 / 1.49e+00 | 20260507 22:58:55.862 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.75e+00 / 1.91e+00 | 20260507 22:58:55.862 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.72e+00, 4.32e-01 / 2.31e-01 | 20260507 22:58:55.862 | INFO | PerfReport: 3.46%, Metrics -- 1.14e+00 / 1.33e+00, 2.91e-01 / 1.54e-01 | 20260507 22:58:55.862 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.52e-01 / 3.81e-01, 5.75e-02 / 3.13e-02 | 20260507 22:58:55.862 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 22:58:55.862 | INFO | PerfReport: Gathered 1536 samples. | 20260507 22:58:55.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 22:58:55.863 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 24.43%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 30.88%, Compute -- 3.30e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 2.14%, LockUpload -- 2.29e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 2.53%, LockDownload -- 2.70e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 15.73%, LockCompute -- 1.68e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 22:58:55.863 | INFO | PerfReport: 0.01%, Misc -- 1.59e-02 / 0.00e+00, 2.84e-02 / 0.00e+00 | 20260507 22:58:55.974 | INFO | Starting a round of Baz IO | 20260507 22:58:59.448 | INFO | Populating RTMetrics report | 20260507 22:58:59.449 | INFO | RTMetrics report populated | 20260507 22:58:59.449 | INFO | Opening tmp RTMetrics json file | 20260507 22:58:59.469 | INFO | Serializing RTMetrics to json | 20260507 22:58:59.471 | INFO | Serialization done, swapping tmp file now... | 20260507 22:58:59.471 | INFO | RTMetrics done | 20260507 22:58:59.532 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.558 seconds | 20260507 22:58:59.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.84% / 0.01% / 57.16% | 20260507 22:58:59.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.29352% / 3.97364e-06% / 5.96046e-05% | 20260507 22:58:59.532 | INFO | Baz IO overall: | 20260507 22:58:59.532 | INFO | BytesWritten : 1488853217819 | 20260507 22:58:59.532 | INFO | NumEvents : 721164853071 | 20260507 22:59:01.003 | INFO | Analyzing chunk frames = [3268608,3269120) | 20260507 22:59:01.003 | INFO | MemInfo: { | 20260507 22:59:01.003 | INFO | "gpu" : " 17 GB", | 20260507 22:59:01.003 | INFO | "huge" : "", | 20260507 22:59:01.003 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:01.003 | INFO | "malloc" : " 14 GB", | 20260507 22:59:01.003 | INFO | "pinned" : " 29 GB" | 20260507 22:59:01.003 | INFO | } | 20260507 22:59:01.003 | INFO | Perf report for the last 5.14336 seconds: | 20260507 22:59:01.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:01.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:01.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:01.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:01.003 | INFO | ANALYSIS : 69.1%, 2.9, 104.7ms, 0 | 20260507 22:59:01.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:01.004 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 22:59:01.004 | INFO | PRE_HQ : 68.2%, 1.8, 64.2ms, 0 | 20260507 22:59:01.004 | INFO | BAZWRITER : 68.7%, 3.9, 136.6ms, 0 | 20260507 22:59:01.004 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 22:59:01.004 | INFO | Uploaded 36GiB to GPU in 2.60326s (13.8288GiB/s) | 20260507 22:59:01.004 | INFO | Downloaded 38GiB to GPU in 2.69432s (14.1038GiB/s) | 20260507 22:59:06.145 | INFO | Analyzing chunk frames = [3269120,3269632) | 20260507 22:59:06.145 | INFO | MemInfo: { | 20260507 22:59:06.145 | INFO | "gpu" : " 17 GB", | 20260507 22:59:06.146 | INFO | "huge" : "", | 20260507 22:59:06.146 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:06.146 | INFO | "malloc" : " 14 GB", | 20260507 22:59:06.146 | INFO | "pinned" : " 29 GB" | 20260507 22:59:06.146 | INFO | } | 20260507 22:59:06.146 | INFO | Perf report for the last 5.14225 seconds: | 20260507 22:59:06.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:06.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 22:59:06.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 22:59:06.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:06.146 | INFO | Uploaded 36GiB to GPU in 2.5859s (13.9217GiB/s) | 20260507 22:59:06.146 | INFO | Downloaded 34GiB to GPU in 2.4756s (13.734GiB/s) | 20260507 22:59:11.285 | INFO | Analyzing chunk frames = [3269632,3270144) | 20260507 22:59:11.286 | INFO | MemInfo: { | 20260507 22:59:11.286 | INFO | "gpu" : " 17 GB", | 20260507 22:59:11.286 | INFO | "huge" : "", | 20260507 22:59:11.286 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:11.286 | INFO | "malloc" : " 14 GB", | 20260507 22:59:11.286 | INFO | "pinned" : " 29 GB" | 20260507 22:59:11.286 | INFO | } | 20260507 22:59:11.286 | INFO | Perf report for the last 5.14005 seconds: | 20260507 22:59:11.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:11.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 22:59:11.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:59:11.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:11.286 | INFO | Uploaded 36GiB to GPU in 2.59264s (13.8855GiB/s) | 20260507 22:59:11.286 | INFO | Downloaded 34GiB to GPU in 2.45951s (13.8239GiB/s) | 20260507 22:59:16.423 | INFO | Analyzing chunk frames = [3270144,3270656) | 20260507 22:59:16.423 | INFO | MemInfo: { | 20260507 22:59:16.423 | INFO | "gpu" : " 17 GB", | 20260507 22:59:16.423 | INFO | "huge" : "", | 20260507 22:59:16.423 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:16.423 | INFO | "malloc" : " 14 GB", | 20260507 22:59:16.423 | INFO | "pinned" : " 29 GB" | 20260507 22:59:16.423 | INFO | } | 20260507 22:59:16.423 | INFO | Perf report for the last 5.13751 seconds: | 20260507 22:59:16.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:16.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.423 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 22:59:16.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.423 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:59:16.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:16.424 | INFO | Uploaded 36GiB to GPU in 2.53609s (14.1951GiB/s) | 20260507 22:59:16.424 | INFO | Downloaded 34GiB to GPU in 2.48098s (13.7043GiB/s) | 20260507 22:59:21.563 | INFO | Analyzing chunk frames = [3270656,3271168) | 20260507 22:59:21.563 | INFO | MemInfo: { | 20260507 22:59:21.563 | INFO | "gpu" : " 17 GB", | 20260507 22:59:21.563 | INFO | "huge" : "", | 20260507 22:59:21.563 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:21.563 | INFO | "malloc" : " 14 GB", | 20260507 22:59:21.563 | INFO | "pinned" : " 29 GB" | 20260507 22:59:21.563 | INFO | } | 20260507 22:59:21.563 | INFO | Perf report for the last 5.13976 seconds: | 20260507 22:59:21.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:21.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.563 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 22:59:21.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.563 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:59:21.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:21.564 | INFO | Uploaded 36GiB to GPU in 2.39408s (15.0371GiB/s) | 20260507 22:59:21.564 | INFO | Downloaded 34GiB to GPU in 2.49075s (13.6505GiB/s) | 20260507 22:59:26.701 | INFO | Analyzing chunk frames = [3271168,3271680) | 20260507 22:59:26.701 | INFO | MemInfo: { | 20260507 22:59:26.701 | INFO | "gpu" : " 17 GB", | 20260507 22:59:26.701 | INFO | "huge" : "", | 20260507 22:59:26.701 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:26.701 | INFO | "malloc" : " 14 GB", | 20260507 22:59:26.701 | INFO | "pinned" : " 29 GB" | 20260507 22:59:26.701 | INFO | } | 20260507 22:59:26.701 | INFO | Perf report for the last 5.13841 seconds: | 20260507 22:59:26.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:26.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 22:59:26.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 22:59:26.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:26.702 | INFO | Uploaded 36GiB to GPU in 2.49231s (14.4444GiB/s) | 20260507 22:59:26.702 | INFO | Downloaded 34GiB to GPU in 2.41867s (14.0573GiB/s) | 20260507 22:59:31.839 | INFO | Analyzing chunk frames = [3271680,3272192) | 20260507 22:59:31.839 | INFO | MemInfo: { | 20260507 22:59:31.839 | INFO | "gpu" : " 17 GB", | 20260507 22:59:31.840 | INFO | "huge" : "", | 20260507 22:59:31.840 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:31.840 | INFO | "malloc" : " 14 GB", | 20260507 22:59:31.840 | INFO | "pinned" : " 29 GB" | 20260507 22:59:31.840 | INFO | } | 20260507 22:59:31.840 | INFO | Perf report for the last 5.13825 seconds: | 20260507 22:59:31.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:31.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.840 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 22:59:31.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.840 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 22:59:31.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:31.841 | INFO | Uploaded 36GiB to GPU in 2.54409s (14.1504GiB/s) | 20260507 22:59:31.841 | INFO | Downloaded 34GiB to GPU in 2.4595s (13.8239GiB/s) | 20260507 22:59:36.980 | INFO | Analyzing chunk frames = [3272192,3272704) | 20260507 22:59:36.980 | INFO | MemInfo: { | 20260507 22:59:36.980 | INFO | "gpu" : " 17 GB", | 20260507 22:59:36.980 | INFO | "huge" : "", | 20260507 22:59:36.980 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:36.980 | INFO | "malloc" : " 14 GB", | 20260507 22:59:36.980 | INFO | "pinned" : " 29 GB" | 20260507 22:59:36.980 | INFO | } | 20260507 22:59:36.980 | INFO | Perf report for the last 5.14032 seconds: | 20260507 22:59:36.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:36.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 22:59:36.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 22:59:36.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:36.980 | INFO | Uploaded 36GiB to GPU in 2.52053s (14.2827GiB/s) | 20260507 22:59:36.980 | INFO | Downloaded 34GiB to GPU in 2.44839s (13.8867GiB/s) | 20260507 22:59:37.077 | INFO | Starting a round of Baz IO | 20260507 22:59:40.544 | INFO | Populating RTMetrics report | 20260507 22:59:40.545 | INFO | RTMetrics report populated | 20260507 22:59:40.545 | INFO | Opening tmp RTMetrics json file | 20260507 22:59:40.566 | INFO | Serializing RTMetrics to json | 20260507 22:59:40.615 | INFO | Serialization done, swapping tmp file now... | 20260507 22:59:40.626 | INFO | RTMetrics done | 20260507 22:59:40.643 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.565 seconds | 20260507 22:59:40.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.83% / 0.01% / 57.16% | 20260507 22:59:40.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28953% / 3.97364e-06% / 5.16574e-05% | 20260507 22:59:40.643 | INFO | Baz IO overall: | 20260507 22:59:40.643 | INFO | BytesWritten : 1490705450122 | 20260507 22:59:40.643 | INFO | NumEvents : 722057945594 | 20260507 22:59:42.117 | INFO | Analyzing chunk frames = [3272704,3273216) | 20260507 22:59:42.117 | INFO | MemInfo: { | 20260507 22:59:42.117 | INFO | "gpu" : " 17 GB", | 20260507 22:59:42.117 | INFO | "huge" : "", | 20260507 22:59:42.117 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:42.117 | INFO | "malloc" : " 14 GB", | 20260507 22:59:42.117 | INFO | "pinned" : " 29 GB" | 20260507 22:59:42.117 | INFO | } | 20260507 22:59:42.117 | INFO | Perf report for the last 5.13708 seconds: | 20260507 22:59:42.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:42.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:42.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:42.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:42.117 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 22:59:42.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:42.117 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 22:59:42.117 | INFO | PRE_HQ : 66.1%, 1.7, 57.0ms, 0 | 20260507 22:59:42.117 | INFO | BAZWRITER : 69.0%, 4.0, 141.7ms, 0 | 20260507 22:59:42.117 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 22:59:42.117 | INFO | Uploaded 36GiB to GPU in 2.5068s (14.3609GiB/s) | 20260507 22:59:42.117 | INFO | Downloaded 38GiB to GPU in 2.70272s (14.0599GiB/s) | 20260507 22:59:47.259 | INFO | Analyzing chunk frames = [3273216,3273728) | 20260507 22:59:47.259 | INFO | MemInfo: { | 20260507 22:59:47.259 | INFO | "gpu" : " 17 GB", | 20260507 22:59:47.259 | INFO | "huge" : "", | 20260507 22:59:47.259 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:47.259 | INFO | "malloc" : " 14 GB", | 20260507 22:59:47.259 | INFO | "pinned" : " 29 GB" | 20260507 22:59:47.259 | INFO | } | 20260507 22:59:47.259 | INFO | Perf report for the last 5.14218 seconds: | 20260507 22:59:47.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:47.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.259 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 22:59:47.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.259 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 22:59:47.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:47.260 | INFO | Uploaded 36GiB to GPU in 2.55203s (14.1064GiB/s) | 20260507 22:59:47.260 | INFO | Downloaded 34GiB to GPU in 2.49133s (13.6473GiB/s) | 20260507 22:59:52.402 | INFO | Analyzing chunk frames = [3273728,3274240) | 20260507 22:59:52.402 | INFO | MemInfo: { | 20260507 22:59:52.402 | INFO | "gpu" : " 17 GB", | 20260507 22:59:52.402 | INFO | "huge" : "", | 20260507 22:59:52.402 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:52.403 | INFO | "malloc" : " 14 GB", | 20260507 22:59:52.403 | INFO | "pinned" : " 29 GB" | 20260507 22:59:52.403 | INFO | } | 20260507 22:59:52.403 | INFO | Perf report for the last 5.14321 seconds: | 20260507 22:59:52.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:52.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 22:59:52.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 22:59:52.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:52.403 | INFO | Uploaded 36GiB to GPU in 2.55766s (14.0753GiB/s) | 20260507 22:59:52.403 | INFO | Downloaded 34GiB to GPU in 2.4702s (13.7641GiB/s) | 20260507 22:59:57.542 | INFO | Analyzing chunk frames = [3274240,3274752) | 20260507 22:59:57.542 | INFO | MemInfo: { | 20260507 22:59:57.542 | INFO | "gpu" : " 17 GB", | 20260507 22:59:57.542 | INFO | "huge" : "", | 20260507 22:59:57.542 | INFO | "huge_shared" : " 29 GB", | 20260507 22:59:57.542 | INFO | "malloc" : " 14 GB", | 20260507 22:59:57.542 | INFO | "pinned" : " 29 GB" | 20260507 22:59:57.542 | INFO | } | 20260507 22:59:57.542 | INFO | Perf report for the last 5.13948 seconds: | 20260507 22:59:57.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 22:59:57.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 22:59:57.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 22:59:57.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 22:59:57.542 | INFO | Uploaded 36GiB to GPU in 2.51765s (14.2991GiB/s) | 20260507 22:59:57.542 | INFO | Downloaded 34GiB to GPU in 2.49458s (13.6296GiB/s) | 20260507 23:00:02.676 | INFO | Analyzing chunk frames = [3274752,3275264) | 20260507 23:00:02.676 | INFO | MemInfo: { | 20260507 23:00:02.676 | INFO | "gpu" : " 17 GB", | 20260507 23:00:02.676 | INFO | "huge" : "", | 20260507 23:00:02.676 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:02.676 | INFO | "malloc" : " 14 GB", | 20260507 23:00:02.676 | INFO | "pinned" : " 29 GB" | 20260507 23:00:02.676 | INFO | } | 20260507 23:00:02.676 | INFO | Perf report for the last 5.13386 seconds: | 20260507 23:00:02.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:02.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 23:00:02.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:00:02.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:02.676 | INFO | Uploaded 36GiB to GPU in 2.43307s (14.7961GiB/s) | 20260507 23:00:02.676 | INFO | Downloaded 34GiB to GPU in 2.50291s (13.5842GiB/s) | 20260507 23:00:07.813 | INFO | Analyzing chunk frames = [3275264,3275776) | 20260507 23:00:07.813 | INFO | MemInfo: { | 20260507 23:00:07.813 | INFO | "gpu" : " 17 GB", | 20260507 23:00:07.813 | INFO | "huge" : "", | 20260507 23:00:07.813 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:07.813 | INFO | "malloc" : " 14 GB", | 20260507 23:00:07.813 | INFO | "pinned" : " 29 GB" | 20260507 23:00:07.813 | INFO | } | 20260507 23:00:07.813 | INFO | Perf report for the last 5.13742 seconds: | 20260507 23:00:07.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:07.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.813 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 23:00:07.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.814 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:00:07.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:07.814 | INFO | Uploaded 36GiB to GPU in 2.51706s (14.3024GiB/s) | 20260507 23:00:07.814 | INFO | Downloaded 34GiB to GPU in 2.41518s (14.0776GiB/s) | 20260507 23:00:12.955 | INFO | Analyzing chunk frames = [3275776,3276288) | 20260507 23:00:12.955 | INFO | MemInfo: { | 20260507 23:00:12.955 | INFO | "gpu" : " 17 GB", | 20260507 23:00:12.955 | INFO | "huge" : "", | 20260507 23:00:12.955 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:12.955 | INFO | "malloc" : " 14 GB", | 20260507 23:00:12.955 | INFO | "pinned" : " 29 GB" | 20260507 23:00:12.955 | INFO | } | 20260507 23:00:12.956 | INFO | Perf report for the last 5.14213 seconds: | 20260507 23:00:12.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:12.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:00:12.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 23:00:12.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:12.956 | INFO | Uploaded 36GiB to GPU in 2.53023s (14.228GiB/s) | 20260507 23:00:12.956 | INFO | Downloaded 34GiB to GPU in 2.43959s (13.9368GiB/s) | 20260507 23:00:18.098 | INFO | Analyzing chunk frames = [3276288,3276800) | 20260507 23:00:18.098 | INFO | MemInfo: { | 20260507 23:00:18.098 | INFO | "gpu" : " 17 GB", | 20260507 23:00:18.098 | INFO | "huge" : "", | 20260507 23:00:18.098 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:18.098 | INFO | "malloc" : " 14 GB", | 20260507 23:00:18.098 | INFO | "pinned" : " 29 GB" | 20260507 23:00:18.098 | INFO | } | 20260507 23:00:18.099 | INFO | Perf report for the last 5.143 seconds: | 20260507 23:00:18.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:18.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:00:18.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:00:18.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:18.099 | INFO | Uploaded 36GiB to GPU in 2.51919s (14.2903GiB/s) | 20260507 23:00:18.099 | INFO | Downloaded 34GiB to GPU in 2.4371s (13.951GiB/s) | 20260507 23:00:18.099 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:00:18.099 | INFO | Total mem usage: 16.7 GiB | 20260507 23:00:18.099 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:00:18.099 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260507 23:00:18.099 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:00:18.099 | INFO | High Water Breakdown: | 20260507 23:00:18.099 | INFO | 60.2% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:00:18.099 | INFO | 39.7% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:00:18.099 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:00:18.099 | INFO | ------------------------------------------ | 20260507 23:00:18.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:00:18.099 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:00:18.099 | INFO | Total mem usage: 26.5 GiB | 20260507 23:00:18.099 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:00:18.099 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 23:00:18.099 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:00:18.099 | INFO | High Water Breakdown: | 20260507 23:00:18.099 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:00:18.099 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:00:18.099 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:00:18.099 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:00:18.099 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:00:18.099 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:00:18.099 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:00:18.099 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:00:18.099 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:00:18.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:00:18.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:00:18.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:00:18.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:00:18.100 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:00:18.100 | INFO | ------------------------------------------ | 20260507 23:00:18.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:00:18.100 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:00:18.100 | INFO | Total mem usage: 27.8 GiB | 20260507 23:00:18.100 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:00:18.100 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:00:18.100 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:00:18.100 | INFO | High Water Breakdown: | 20260507 23:00:18.100 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:00:18.100 | INFO | ------------------------------------------ | 20260507 23:00:18.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:00:18.100 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:00:18.100 | INFO | Total mem usage: 15.7 GiB | 20260507 23:00:18.100 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:00:18.100 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:00:18.100 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:00:18.100 | INFO | High Water Breakdown: | 20260507 23:00:18.100 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:00:18.100 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:00:18.100 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:00:18.100 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:00:18.100 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:00:18.100 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:00:18.100 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:00:18.100 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:00:18.100 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:00:18.100 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:00:18.100 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:00:18.100 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:00:18.100 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:00:18.100 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:00:18.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:00:18.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:00:18.100 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:00:18.100 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:00:18.100 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:00:18.100 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:00:18.100 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:00:18.100 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:00:18.100 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:00:18.100 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:00:18.100 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:00:18.100 | INFO | ------------------------------------------ | 20260507 23:00:18.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:00:18.100 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:00:18.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:00:18.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:00:18.100 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.64e+01, 1.42e+01 / 1.93e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.21e-02 / 6.15e-03 | 20260507 23:00:18.100 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.46e+00, 5.79e-01 / 2.20e-01 | 20260507 23:00:18.100 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.48e+00, 2.07e-01 / 1.16e-01 | 20260507 23:00:18.100 | INFO | PerfReport: 15.95%, DME -- 4.20e+01 / 4.57e+01, 4.10e+00 / 1.81e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.71e+00 / 1.66e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.45e+00 / 2.71e+00, 4.31e-01 / 2.30e-01 | 20260507 23:00:18.100 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.30e+00, 2.84e-01 / 1.69e-01 | 20260507 23:00:18.100 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.78e-01, 5.76e-02 / 2.87e-02 | 20260507 23:00:18.100 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:00:18.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:00:18.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:00:18.100 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 24.61%, Upload -- 2.63e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 24.29%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 30.77%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 2.22%, LockUpload -- 2.38e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 2.51%, LockDownload -- 2.68e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 15.58%, LockCompute -- 1.67e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:00:18.100 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 23:00:18.196 | INFO | Starting a round of Baz IO | 20260507 23:00:21.655 | INFO | Populating RTMetrics report | 20260507 23:00:21.655 | INFO | RTMetrics report populated | 20260507 23:00:21.655 | INFO | Opening tmp RTMetrics json file | 20260507 23:00:21.681 | INFO | Serializing RTMetrics to json | 20260507 23:00:21.683 | INFO | Serialization done, swapping tmp file now... | 20260507 23:00:21.683 | INFO | RTMetrics done | 20260507 23:00:21.770 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.574 seconds | 20260507 23:00:21.771 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.82% / 0.01% / 57.17% | 20260507 23:00:21.771 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.2892% / 1.19209e-05% / 5.5631e-05% | 20260507 23:00:21.771 | INFO | Baz IO overall: | 20260507 23:00:21.771 | INFO | BytesWritten : 1492558152212 | 20260507 23:00:21.771 | INFO | NumEvents : 722951424877 | 20260507 23:00:23.236 | INFO | Analyzing chunk frames = [3276800,3277312) | 20260507 23:00:23.236 | INFO | MemInfo: { | 20260507 23:00:23.236 | INFO | "gpu" : " 17 GB", | 20260507 23:00:23.236 | INFO | "huge" : "", | 20260507 23:00:23.236 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:23.236 | INFO | "malloc" : " 14 GB", | 20260507 23:00:23.236 | INFO | "pinned" : " 29 GB" | 20260507 23:00:23.236 | INFO | } | 20260507 23:00:23.236 | INFO | Perf report for the last 5.13731 seconds: | 20260507 23:00:23.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:23.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:23.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:23.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:23.236 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260507 23:00:23.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:23.236 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 23:00:23.236 | INFO | PRE_HQ : 65.8%, 1.7, 57.2ms, 0 | 20260507 23:00:23.236 | INFO | BAZWRITER : 69.2%, 4.1, 146.1ms, 0 | 20260507 23:00:23.236 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:00:23.236 | INFO | Uploaded 36GiB to GPU in 2.52159s (14.2767GiB/s) | 20260507 23:00:23.236 | INFO | Downloaded 38GiB to GPU in 2.68957s (14.1287GiB/s) | 20260507 23:00:28.370 | INFO | Analyzing chunk frames = [3277312,3277824) | 20260507 23:00:28.370 | INFO | MemInfo: { | 20260507 23:00:28.370 | INFO | "gpu" : " 17 GB", | 20260507 23:00:28.370 | INFO | "huge" : "", | 20260507 23:00:28.370 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:28.370 | INFO | "malloc" : " 14 GB", | 20260507 23:00:28.370 | INFO | "pinned" : " 29 GB" | 20260507 23:00:28.370 | INFO | } | 20260507 23:00:28.370 | INFO | Perf report for the last 5.13412 seconds: | 20260507 23:00:28.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:28.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 23:00:28.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:00:28.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:28.370 | INFO | Uploaded 36GiB to GPU in 2.5823s (13.9411GiB/s) | 20260507 23:00:28.370 | INFO | Downloaded 34GiB to GPU in 2.47043s (13.7628GiB/s) | 20260507 23:00:33.511 | INFO | Analyzing chunk frames = [3277824,3278336) | 20260507 23:00:33.511 | INFO | MemInfo: { | 20260507 23:00:33.512 | INFO | "gpu" : " 17 GB", | 20260507 23:00:33.512 | INFO | "huge" : "", | 20260507 23:00:33.512 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:33.512 | INFO | "malloc" : " 14 GB", | 20260507 23:00:33.512 | INFO | "pinned" : " 29 GB" | 20260507 23:00:33.512 | INFO | } | 20260507 23:00:33.512 | INFO | Perf report for the last 5.14175 seconds: | 20260507 23:00:33.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:33.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.512 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260507 23:00:33.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.512 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:00:33.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:33.513 | INFO | Uploaded 36GiB to GPU in 2.56303s (14.0459GiB/s) | 20260507 23:00:33.513 | INFO | Downloaded 34GiB to GPU in 2.44837s (13.8868GiB/s) | 20260507 23:00:38.652 | INFO | Analyzing chunk frames = [3278336,3278848) | 20260507 23:00:38.652 | INFO | MemInfo: { | 20260507 23:00:38.652 | INFO | "gpu" : " 17 GB", | 20260507 23:00:38.652 | INFO | "huge" : "", | 20260507 23:00:38.652 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:38.652 | INFO | "malloc" : " 14 GB", | 20260507 23:00:38.652 | INFO | "pinned" : " 29 GB" | 20260507 23:00:38.652 | INFO | } | 20260507 23:00:38.652 | INFO | Perf report for the last 5.14057 seconds: | 20260507 23:00:38.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:38.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.652 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 23:00:38.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.652 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:00:38.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:38.653 | INFO | Uploaded 36GiB to GPU in 2.49497s (14.429GiB/s) | 20260507 23:00:38.653 | INFO | Downloaded 34GiB to GPU in 2.39349s (14.2052GiB/s) | 20260507 23:00:43.797 | INFO | Analyzing chunk frames = [3278848,3279360) | 20260507 23:00:43.797 | INFO | MemInfo: { | 20260507 23:00:43.797 | INFO | "gpu" : " 17 GB", | 20260507 23:00:43.797 | INFO | "huge" : "", | 20260507 23:00:43.797 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:43.797 | INFO | "malloc" : " 14 GB", | 20260507 23:00:43.797 | INFO | "pinned" : " 29 GB" | 20260507 23:00:43.797 | INFO | } | 20260507 23:00:43.797 | INFO | Perf report for the last 5.14474 seconds: | 20260507 23:00:43.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:43.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.797 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 23:00:43.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.797 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 23:00:43.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:43.798 | INFO | Uploaded 36GiB to GPU in 2.3965s (15.0219GiB/s) | 20260507 23:00:43.798 | INFO | Downloaded 34GiB to GPU in 2.49497s (13.6274GiB/s) | 20260507 23:00:48.928 | INFO | Analyzing chunk frames = [3279360,3279872) | 20260507 23:00:48.928 | INFO | MemInfo: { | 20260507 23:00:48.928 | INFO | "gpu" : " 17 GB", | 20260507 23:00:48.928 | INFO | "huge" : "", | 20260507 23:00:48.928 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:48.928 | INFO | "malloc" : " 14 GB", | 20260507 23:00:48.928 | INFO | "pinned" : " 29 GB" | 20260507 23:00:48.928 | INFO | } | 20260507 23:00:48.928 | INFO | Perf report for the last 5.13128 seconds: | 20260507 23:00:48.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:48.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.928 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260507 23:00:48.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.928 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 23:00:48.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:48.929 | INFO | Uploaded 36GiB to GPU in 2.39294s (15.0443GiB/s) | 20260507 23:00:48.929 | INFO | Downloaded 34GiB to GPU in 2.39469s (14.1981GiB/s) | 20260507 23:00:54.066 | INFO | Analyzing chunk frames = [3279872,3280384) | 20260507 23:00:54.066 | INFO | MemInfo: { | 20260507 23:00:54.066 | INFO | "gpu" : " 17 GB", | 20260507 23:00:54.066 | INFO | "huge" : "", | 20260507 23:00:54.066 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:54.066 | INFO | "malloc" : " 14 GB", | 20260507 23:00:54.066 | INFO | "pinned" : " 29 GB" | 20260507 23:00:54.067 | INFO | } | 20260507 23:00:54.067 | INFO | Perf report for the last 5.13831 seconds: | 20260507 23:00:54.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:54.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:00:54.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:00:54.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:54.067 | INFO | Uploaded 36GiB to GPU in 2.52955s (14.2318GiB/s) | 20260507 23:00:54.067 | INFO | Downloaded 34GiB to GPU in 2.45895s (13.827GiB/s) | 20260507 23:00:59.208 | INFO | Analyzing chunk frames = [3280384,3280896) | 20260507 23:00:59.209 | INFO | MemInfo: { | 20260507 23:00:59.209 | INFO | "gpu" : " 17 GB", | 20260507 23:00:59.209 | INFO | "huge" : "", | 20260507 23:00:59.209 | INFO | "huge_shared" : " 29 GB", | 20260507 23:00:59.209 | INFO | "malloc" : " 14 GB", | 20260507 23:00:59.209 | INFO | "pinned" : " 29 GB" | 20260507 23:00:59.209 | INFO | } | 20260507 23:00:59.209 | INFO | Perf report for the last 5.142 seconds: | 20260507 23:00:59.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:00:59.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:00:59.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:00:59.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:00:59.209 | INFO | Uploaded 36GiB to GPU in 2.4947s (14.4306GiB/s) | 20260507 23:00:59.209 | INFO | Downloaded 34GiB to GPU in 2.44797s (13.8891GiB/s) | 20260507 23:00:59.306 | INFO | Starting a round of Baz IO | 20260507 23:01:02.775 | INFO | Populating RTMetrics report | 20260507 23:01:02.776 | INFO | RTMetrics report populated | 20260507 23:01:02.776 | INFO | Opening tmp RTMetrics json file | 20260507 23:01:02.792 | INFO | Serializing RTMetrics to json | 20260507 23:01:02.794 | INFO | Serialization done, swapping tmp file now... | 20260507 23:01:02.795 | INFO | RTMetrics done | 20260507 23:01:02.866 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.56 seconds | 20260507 23:01:02.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.82% / 0.01% / 57.18% | 20260507 23:01:02.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28724% / 0% / 7.94729e-05% | 20260507 23:01:02.866 | INFO | Baz IO overall: | 20260507 23:01:02.866 | INFO | BytesWritten : 1494411089945 | 20260507 23:01:02.866 | INFO | NumEvents : 723845179278 | 20260507 23:01:04.355 | INFO | Analyzing chunk frames = [3280896,3281408) | 20260507 23:01:04.355 | INFO | MemInfo: { | 20260507 23:01:04.355 | INFO | "gpu" : " 17 GB", | 20260507 23:01:04.355 | INFO | "huge" : "", | 20260507 23:01:04.355 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:04.355 | INFO | "malloc" : " 14 GB", | 20260507 23:01:04.355 | INFO | "pinned" : " 29 GB" | 20260507 23:01:04.355 | INFO | } | 20260507 23:01:04.355 | INFO | Perf report for the last 5.14671 seconds: | 20260507 23:01:04.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:04.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:04.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:04.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:04.355 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 23:01:04.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:04.355 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 23:01:04.356 | INFO | PRE_HQ : 65.5%, 1.7, 57.0ms, 0 | 20260507 23:01:04.356 | INFO | BAZWRITER : 68.8%, 4.2, 147.8ms, 0 | 20260507 23:01:04.356 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 23:01:04.356 | INFO | Uploaded 36GiB to GPU in 2.52647s (14.2491GiB/s) | 20260507 23:01:04.356 | INFO | Downloaded 38GiB to GPU in 2.68435s (14.1561GiB/s) | 20260507 23:01:09.491 | INFO | Analyzing chunk frames = [3281408,3281920) | 20260507 23:01:09.491 | INFO | MemInfo: { | 20260507 23:01:09.491 | INFO | "gpu" : " 17 GB", | 20260507 23:01:09.491 | INFO | "huge" : "", | 20260507 23:01:09.491 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:09.491 | INFO | "malloc" : " 14 GB", | 20260507 23:01:09.491 | INFO | "pinned" : " 29 GB" | 20260507 23:01:09.491 | INFO | } | 20260507 23:01:09.491 | INFO | Perf report for the last 5.13597 seconds: | 20260507 23:01:09.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:09.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.491 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 23:01:09.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.492 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:01:09.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:09.492 | INFO | Uploaded 36GiB to GPU in 2.5575s (14.0762GiB/s) | 20260507 23:01:09.492 | INFO | Downloaded 34GiB to GPU in 2.48983s (13.6555GiB/s) | 20260507 23:01:14.627 | INFO | Analyzing chunk frames = [3281920,3282432) | 20260507 23:01:14.627 | INFO | MemInfo: { | 20260507 23:01:14.627 | INFO | "gpu" : " 17 GB", | 20260507 23:01:14.627 | INFO | "huge" : "", | 20260507 23:01:14.627 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:14.627 | INFO | "malloc" : " 14 GB", | 20260507 23:01:14.627 | INFO | "pinned" : " 29 GB" | 20260507 23:01:14.627 | INFO | } | 20260507 23:01:14.627 | INFO | Perf report for the last 5.1355 seconds: | 20260507 23:01:14.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:14.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260507 23:01:14.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:01:14.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:14.627 | INFO | Uploaded 36GiB to GPU in 2.49017s (14.4568GiB/s) | 20260507 23:01:14.627 | INFO | Downloaded 34GiB to GPU in 2.48586s (13.6774GiB/s) | 20260507 23:01:19.765 | INFO | Analyzing chunk frames = [3282432,3282944) | 20260507 23:01:19.765 | INFO | MemInfo: { | 20260507 23:01:19.765 | INFO | "gpu" : " 17 GB", | 20260507 23:01:19.765 | INFO | "huge" : "", | 20260507 23:01:19.765 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:19.765 | INFO | "malloc" : " 14 GB", | 20260507 23:01:19.765 | INFO | "pinned" : " 29 GB" | 20260507 23:01:19.765 | INFO | } | 20260507 23:01:19.765 | INFO | Perf report for the last 5.13831 seconds: | 20260507 23:01:19.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:19.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:01:19.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 23:01:19.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:19.765 | INFO | Uploaded 36GiB to GPU in 2.46731s (14.5908GiB/s) | 20260507 23:01:19.766 | INFO | Downloaded 34GiB to GPU in 2.40659s (14.1278GiB/s) | 20260507 23:01:24.909 | INFO | Analyzing chunk frames = [3282944,3283456) | 20260507 23:01:24.909 | INFO | MemInfo: { | 20260507 23:01:24.909 | INFO | "gpu" : " 17 GB", | 20260507 23:01:24.909 | INFO | "huge" : "", | 20260507 23:01:24.909 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:24.909 | INFO | "malloc" : " 14 GB", | 20260507 23:01:24.909 | INFO | "pinned" : " 29 GB" | 20260507 23:01:24.909 | INFO | } | 20260507 23:01:24.909 | INFO | Perf report for the last 5.14379 seconds: | 20260507 23:01:24.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:24.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 23:01:24.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:01:24.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:24.909 | INFO | Uploaded 36GiB to GPU in 2.35857s (15.2635GiB/s) | 20260507 23:01:24.909 | INFO | Downloaded 34GiB to GPU in 2.49831s (13.6092GiB/s) | 20260507 23:01:30.044 | INFO | Analyzing chunk frames = [3283456,3283968) | 20260507 23:01:30.044 | INFO | MemInfo: { | 20260507 23:01:30.044 | INFO | "gpu" : " 17 GB", | 20260507 23:01:30.044 | INFO | "huge" : "", | 20260507 23:01:30.045 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:30.045 | INFO | "malloc" : " 14 GB", | 20260507 23:01:30.045 | INFO | "pinned" : " 29 GB" | 20260507 23:01:30.045 | INFO | } | 20260507 23:01:30.045 | INFO | Perf report for the last 5.13568 seconds: | 20260507 23:01:30.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:30.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260507 23:01:30.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:01:30.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:30.045 | INFO | Uploaded 36GiB to GPU in 2.52149s (14.2773GiB/s) | 20260507 23:01:30.045 | INFO | Downloaded 34GiB to GPU in 2.42862s (13.9997GiB/s) | 20260507 23:01:35.181 | INFO | Analyzing chunk frames = [3283968,3284480) | 20260507 23:01:35.181 | INFO | MemInfo: { | 20260507 23:01:35.181 | INFO | "gpu" : " 17 GB", | 20260507 23:01:35.181 | INFO | "huge" : "", | 20260507 23:01:35.181 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:35.181 | INFO | "malloc" : " 14 GB", | 20260507 23:01:35.181 | INFO | "pinned" : " 29 GB" | 20260507 23:01:35.181 | INFO | } | 20260507 23:01:35.181 | INFO | Perf report for the last 5.13615 seconds: | 20260507 23:01:35.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:35.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 23:01:35.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:01:35.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:35.181 | INFO | Uploaded 36GiB to GPU in 2.55008s (14.1172GiB/s) | 20260507 23:01:35.181 | INFO | Downloaded 34GiB to GPU in 2.4543s (13.8532GiB/s) | 20260507 23:01:40.324 | INFO | Analyzing chunk frames = [3284480,3284992) | 20260507 23:01:40.324 | INFO | MemInfo: { | 20260507 23:01:40.324 | INFO | "gpu" : " 17 GB", | 20260507 23:01:40.324 | INFO | "huge" : "", | 20260507 23:01:40.324 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:40.324 | INFO | "malloc" : " 14 GB", | 20260507 23:01:40.324 | INFO | "pinned" : " 29 GB" | 20260507 23:01:40.324 | INFO | } | 20260507 23:01:40.324 | INFO | Perf report for the last 5.14317 seconds: | 20260507 23:01:40.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:40.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.324 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 23:01:40.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.324 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:01:40.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:40.325 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:01:40.325 | INFO | Total mem usage: 16.7 GiB | 20260507 23:01:40.325 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:01:40.325 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260507 23:01:40.325 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:01:40.325 | INFO | High Water Breakdown: | 20260507 23:01:40.325 | INFO | 60.1% : 9.04 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:01:40.325 | INFO | 39.7% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:01:40.325 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:01:40.325 | INFO | ------------------------------------------ | 20260507 23:01:40.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:01:40.325 | INFO | Uploaded 36GiB to GPU in 2.54474s (14.1468GiB/s) | 20260507 23:01:40.325 | INFO | Downloaded 34GiB to GPU in 2.44136s (13.9267GiB/s) | 20260507 23:01:40.325 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:01:40.325 | INFO | Total mem usage: 26.5 GiB | 20260507 23:01:40.325 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:01:40.325 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:01:40.325 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:01:40.325 | INFO | High Water Breakdown: | 20260507 23:01:40.325 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:01:40.325 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:01:40.325 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:01:40.325 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:01:40.325 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:01:40.325 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:01:40.325 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:01:40.325 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:01:40.325 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:01:40.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:01:40.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:01:40.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:01:40.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:01:40.325 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:01:40.325 | INFO | ------------------------------------------ | 20260507 23:01:40.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:01:40.325 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:01:40.325 | INFO | Total mem usage: 27.8 GiB | 20260507 23:01:40.325 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:01:40.325 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:01:40.325 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:01:40.325 | INFO | High Water Breakdown: | 20260507 23:01:40.325 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:01:40.325 | INFO | ------------------------------------------ | 20260507 23:01:40.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:01:40.325 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:01:40.325 | INFO | Total mem usage: 15.7 GiB | 20260507 23:01:40.325 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:01:40.325 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:01:40.325 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:01:40.325 | INFO | High Water Breakdown: | 20260507 23:01:40.325 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:01:40.325 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:01:40.325 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:01:40.325 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:01:40.325 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:01:40.325 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:01:40.325 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:01:40.325 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:01:40.325 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:01:40.325 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:01:40.325 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:01:40.325 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:01:40.325 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:01:40.325 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:01:40.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:01:40.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:01:40.325 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:01:40.325 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:01:40.325 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:01:40.325 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:01:40.325 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:01:40.325 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:01:40.325 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:01:40.325 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:01:40.325 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:01:40.325 | INFO | ------------------------------------------ | 20260507 23:01:40.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:01:40.326 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:01:40.326 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:01:40.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:01:40.326 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.59e+01, 1.40e+01 / 1.58e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.18e-02 / 1.13e-02 | 20260507 23:01:40.326 | INFO | PerfReport: 12.13%, Baseline -- 3.99e+00 / 4.49e+00, 5.93e-01 / 1.79e-01 | 20260507 23:01:40.326 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.06e-01 / 9.95e-02 | 20260507 23:01:40.326 | INFO | PerfReport: 15.91%, DME -- 4.18e+01 / 4.53e+01, 3.79e+00 / 1.52e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.72e+00 / 1.78e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.45e+00 / 2.70e+00, 4.27e-01 / 2.31e-01 | 20260507 23:01:40.326 | INFO | PerfReport: 3.45%, Metrics -- 1.13e+00 / 1.34e+00, 2.83e-01 / 1.85e-01 | 20260507 23:01:40.326 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.82e-01, 5.68e-02 / 2.45e-02 | 20260507 23:01:40.326 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:01:40.326 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:01:40.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:01:40.326 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 24.22%, Download -- 2.58e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 30.81%, Compute -- 3.29e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 2.18%, LockUpload -- 2.33e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 2.50%, LockDownload -- 2.67e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 15.87%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:01:40.326 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 23:01:40.421 | INFO | Starting a round of Baz IO | 20260507 23:01:43.897 | INFO | Populating RTMetrics report | 20260507 23:01:43.897 | INFO | RTMetrics report populated | 20260507 23:01:43.897 | INFO | Opening tmp RTMetrics json file | 20260507 23:01:43.898 | INFO | Serializing RTMetrics to json | 20260507 23:01:43.899 | INFO | Serialization done, swapping tmp file now... | 20260507 23:01:43.900 | INFO | RTMetrics done | 20260507 23:01:44.000 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.578 seconds | 20260507 23:01:44.000 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.81% / 0.01% / 57.18% | 20260507 23:01:44.000 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28517% / 7.94729e-06% / 6.35783e-05% | 20260507 23:01:44.000 | INFO | Baz IO overall: | 20260507 23:01:44.000 | INFO | BytesWritten : 1496263479352 | 20260507 23:01:44.000 | INFO | NumEvents : 724738634912 | 20260507 23:01:45.464 | INFO | Analyzing chunk frames = [3284992,3285504) | 20260507 23:01:45.464 | INFO | MemInfo: { | 20260507 23:01:45.464 | INFO | "gpu" : " 17 GB", | 20260507 23:01:45.464 | INFO | "huge" : "", | 20260507 23:01:45.464 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:45.464 | INFO | "malloc" : " 14 GB", | 20260507 23:01:45.464 | INFO | "pinned" : " 29 GB" | 20260507 23:01:45.464 | INFO | } | 20260507 23:01:45.464 | INFO | Perf report for the last 5.13961 seconds: | 20260507 23:01:45.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:45.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:45.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:45.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:45.464 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 23:01:45.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:45.464 | INFO | RT_METRICS : 40.5%, 1.0, 21.2ms, 0 | 20260507 23:01:45.464 | INFO | PRE_HQ : 65.5%, 1.7, 57.0ms, 0 | 20260507 23:01:45.464 | INFO | BAZWRITER : 69.2%, 4.1, 146.7ms, 0 | 20260507 23:01:45.464 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:01:45.464 | INFO | Uploaded 36GiB to GPU in 2.56282s (14.047GiB/s) | 20260507 23:01:45.464 | INFO | Downloaded 38GiB to GPU in 2.69297s (14.1108GiB/s) | 20260507 23:01:50.605 | INFO | Analyzing chunk frames = [3285504,3286016) | 20260507 23:01:50.605 | INFO | MemInfo: { | 20260507 23:01:50.605 | INFO | "gpu" : " 17 GB", | 20260507 23:01:50.605 | INFO | "huge" : "", | 20260507 23:01:50.605 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:50.605 | INFO | "malloc" : " 14 GB", | 20260507 23:01:50.605 | INFO | "pinned" : " 29 GB" | 20260507 23:01:50.605 | INFO | } | 20260507 23:01:50.605 | INFO | Perf report for the last 5.14161 seconds: | 20260507 23:01:50.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:50.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.605 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:01:50.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.605 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:01:50.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:50.606 | INFO | Uploaded 36GiB to GPU in 2.58365s (13.9338GiB/s) | 20260507 23:01:50.606 | INFO | Downloaded 34GiB to GPU in 2.47621s (13.7307GiB/s) | 20260507 23:01:55.743 | INFO | Analyzing chunk frames = [3286016,3286528) | 20260507 23:01:55.743 | INFO | MemInfo: { | 20260507 23:01:55.743 | INFO | "gpu" : " 17 GB", | 20260507 23:01:55.743 | INFO | "huge" : "", | 20260507 23:01:55.743 | INFO | "huge_shared" : " 29 GB", | 20260507 23:01:55.743 | INFO | "malloc" : " 14 GB", | 20260507 23:01:55.743 | INFO | "pinned" : " 29 GB" | 20260507 23:01:55.743 | INFO | } | 20260507 23:01:55.743 | INFO | Perf report for the last 5.13806 seconds: | 20260507 23:01:55.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:01:55.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.744 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:01:55.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.744 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260507 23:01:55.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:01:55.744 | INFO | Uploaded 36GiB to GPU in 2.52715s (14.2453GiB/s) | 20260507 23:01:55.744 | INFO | Downloaded 34GiB to GPU in 2.46954s (13.7678GiB/s) | 20260507 23:02:00.882 | INFO | Analyzing chunk frames = [3286528,3287040) | 20260507 23:02:00.882 | INFO | MemInfo: { | 20260507 23:02:00.882 | INFO | "gpu" : " 17 GB", | 20260507 23:02:00.882 | INFO | "huge" : "", | 20260507 23:02:00.882 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:00.882 | INFO | "malloc" : " 14 GB", | 20260507 23:02:00.882 | INFO | "pinned" : " 29 GB" | 20260507 23:02:00.882 | INFO | } | 20260507 23:02:00.882 | INFO | Perf report for the last 5.13893 seconds: | 20260507 23:02:00.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:00.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 23:02:00.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:02:00.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:00.882 | INFO | Uploaded 36GiB to GPU in 2.52833s (14.2386GiB/s) | 20260507 23:02:00.882 | INFO | Downloaded 34GiB to GPU in 2.47789s (13.7213GiB/s) | 20260507 23:02:06.021 | INFO | Analyzing chunk frames = [3287040,3287552) | 20260507 23:02:06.021 | INFO | MemInfo: { | 20260507 23:02:06.021 | INFO | "gpu" : " 17 GB", | 20260507 23:02:06.021 | INFO | "huge" : "", | 20260507 23:02:06.021 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:06.021 | INFO | "malloc" : " 14 GB", | 20260507 23:02:06.022 | INFO | "pinned" : " 29 GB" | 20260507 23:02:06.022 | INFO | } | 20260507 23:02:06.022 | INFO | Perf report for the last 5.1392 seconds: | 20260507 23:02:06.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:06.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260507 23:02:06.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:02:06.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:06.022 | INFO | Uploaded 36GiB to GPU in 2.39281s (15.045GiB/s) | 20260507 23:02:06.022 | INFO | Downloaded 34GiB to GPU in 2.49384s (13.6336GiB/s) | 20260507 23:02:11.158 | INFO | Analyzing chunk frames = [3287552,3288064) | 20260507 23:02:11.158 | INFO | MemInfo: { | 20260507 23:02:11.158 | INFO | "gpu" : " 17 GB", | 20260507 23:02:11.158 | INFO | "huge" : "", | 20260507 23:02:11.158 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:11.158 | INFO | "malloc" : " 14 GB", | 20260507 23:02:11.158 | INFO | "pinned" : " 29 GB" | 20260507 23:02:11.158 | INFO | } | 20260507 23:02:11.158 | INFO | Perf report for the last 5.13677 seconds: | 20260507 23:02:11.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:11.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.158 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 23:02:11.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.158 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:02:11.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:11.159 | INFO | Uploaded 36GiB to GPU in 2.50987s (14.3434GiB/s) | 20260507 23:02:11.159 | INFO | Downloaded 34GiB to GPU in 2.43567s (13.9592GiB/s) | 20260507 23:02:16.303 | INFO | Analyzing chunk frames = [3288064,3288576) | 20260507 23:02:16.303 | INFO | MemInfo: { | 20260507 23:02:16.303 | INFO | "gpu" : " 17 GB", | 20260507 23:02:16.303 | INFO | "huge" : "", | 20260507 23:02:16.303 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:16.303 | INFO | "malloc" : " 14 GB", | 20260507 23:02:16.303 | INFO | "pinned" : " 29 GB" | 20260507 23:02:16.303 | INFO | } | 20260507 23:02:16.303 | INFO | Perf report for the last 5.14498 seconds: | 20260507 23:02:16.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:16.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.303 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:02:16.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.303 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:02:16.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:16.304 | INFO | Uploaded 36GiB to GPU in 2.55042s (14.1153GiB/s) | 20260507 23:02:16.304 | INFO | Downloaded 34GiB to GPU in 2.46765s (13.7783GiB/s) | 20260507 23:02:21.440 | INFO | Analyzing chunk frames = [3288576,3289088) | 20260507 23:02:21.440 | INFO | MemInfo: { | 20260507 23:02:21.440 | INFO | "gpu" : " 17 GB", | 20260507 23:02:21.440 | INFO | "huge" : "", | 20260507 23:02:21.440 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:21.440 | INFO | "malloc" : " 14 GB", | 20260507 23:02:21.440 | INFO | "pinned" : " 29 GB" | 20260507 23:02:21.441 | INFO | } | 20260507 23:02:21.441 | INFO | Perf report for the last 5.13721 seconds: | 20260507 23:02:21.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:21.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:02:21.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:02:21.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:21.441 | INFO | Uploaded 36GiB to GPU in 2.4879s (14.47GiB/s) | 20260507 23:02:21.441 | INFO | Downloaded 34GiB to GPU in 2.44332s (13.9155GiB/s) | 20260507 23:02:21.537 | INFO | Starting a round of Baz IO | 20260507 23:02:24.996 | INFO | Populating RTMetrics report | 20260507 23:02:24.997 | INFO | RTMetrics report populated | 20260507 23:02:24.997 | INFO | Opening tmp RTMetrics json file | 20260507 23:02:25.004 | INFO | Serializing RTMetrics to json | 20260507 23:02:25.020 | INFO | Serialization done, swapping tmp file now... | 20260507 23:02:25.020 | INFO | RTMetrics done | 20260507 23:02:25.099 | INFO | Finished a round of Baz IO: Wrote 1.723 GiB in 3.561 seconds | 20260507 23:02:25.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.81% / 0.01% / 57.19% | 20260507 23:02:25.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27914% / 0% / 7.94729e-05% | 20260507 23:02:25.099 | INFO | Baz IO overall: | 20260507 23:02:25.099 | INFO | BytesWritten : 1498113243422 | 20260507 23:02:25.099 | INFO | NumEvents : 725630585051 | 20260507 23:02:26.578 | INFO | Analyzing chunk frames = [3289088,3289600) | 20260507 23:02:26.578 | INFO | MemInfo: { | 20260507 23:02:26.578 | INFO | "gpu" : " 17 GB", | 20260507 23:02:26.578 | INFO | "huge" : "", | 20260507 23:02:26.578 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:26.578 | INFO | "malloc" : " 14 GB", | 20260507 23:02:26.578 | INFO | "pinned" : " 29 GB" | 20260507 23:02:26.578 | INFO | } | 20260507 23:02:26.578 | INFO | Perf report for the last 5.13735 seconds: | 20260507 23:02:26.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:26.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:26.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:26.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:26.578 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260507 23:02:26.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:26.578 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 23:02:26.578 | INFO | PRE_HQ : 65.6%, 1.7, 57.0ms, 0 | 20260507 23:02:26.578 | INFO | BAZWRITER : 68.9%, 3.9, 139.3ms, 0 | 20260507 23:02:26.578 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:02:26.578 | INFO | Uploaded 36GiB to GPU in 2.55966s (14.0644GiB/s) | 20260507 23:02:26.578 | INFO | Downloaded 38GiB to GPU in 2.70154s (14.066GiB/s) | 20260507 23:02:31.723 | INFO | Analyzing chunk frames = [3289600,3290112) | 20260507 23:02:31.723 | INFO | MemInfo: { | 20260507 23:02:31.723 | INFO | "gpu" : " 17 GB", | 20260507 23:02:31.723 | INFO | "huge" : "", | 20260507 23:02:31.723 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:31.723 | INFO | "malloc" : " 14 GB", | 20260507 23:02:31.723 | INFO | "pinned" : " 29 GB" | 20260507 23:02:31.723 | INFO | } | 20260507 23:02:31.723 | INFO | Perf report for the last 5.1455 seconds: | 20260507 23:02:31.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:31.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 23:02:31.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260507 23:02:31.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:31.724 | INFO | Uploaded 36GiB to GPU in 2.55867s (14.0698GiB/s) | 20260507 23:02:31.724 | INFO | Downloaded 34GiB to GPU in 2.48904s (13.6599GiB/s) | 20260507 23:02:36.858 | INFO | Analyzing chunk frames = [3290112,3290624) | 20260507 23:02:36.858 | INFO | MemInfo: { | 20260507 23:02:36.859 | INFO | "gpu" : " 17 GB", | 20260507 23:02:36.859 | INFO | "huge" : "", | 20260507 23:02:36.859 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:36.859 | INFO | "malloc" : " 14 GB", | 20260507 23:02:36.859 | INFO | "pinned" : " 29 GB" | 20260507 23:02:36.859 | INFO | } | 20260507 23:02:36.859 | INFO | Perf report for the last 5.13553 seconds: | 20260507 23:02:36.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:36.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:02:36.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 23:02:36.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:36.859 | INFO | Uploaded 36GiB to GPU in 2.50293s (14.3831GiB/s) | 20260507 23:02:36.859 | INFO | Downloaded 34GiB to GPU in 2.49548s (13.6246GiB/s) | 20260507 23:02:41.993 | INFO | Analyzing chunk frames = [3290624,3291136) | 20260507 23:02:41.993 | INFO | MemInfo: { | 20260507 23:02:41.993 | INFO | "gpu" : " 17 GB", | 20260507 23:02:41.993 | INFO | "huge" : "", | 20260507 23:02:41.993 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:41.993 | INFO | "malloc" : " 14 GB", | 20260507 23:02:41.993 | INFO | "pinned" : " 29 GB" | 20260507 23:02:41.993 | INFO | } | 20260507 23:02:41.993 | INFO | Perf report for the last 5.13441 seconds: | 20260507 23:02:41.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:41.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260507 23:02:41.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260507 23:02:41.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:41.994 | INFO | Uploaded 36GiB to GPU in 2.52776s (14.2419GiB/s) | 20260507 23:02:41.994 | INFO | Downloaded 34GiB to GPU in 2.46627s (13.786GiB/s) | 20260507 23:02:47.131 | INFO | Analyzing chunk frames = [3291136,3291648) | 20260507 23:02:47.131 | INFO | MemInfo: { | 20260507 23:02:47.131 | INFO | "gpu" : " 17 GB", | 20260507 23:02:47.131 | INFO | "huge" : "", | 20260507 23:02:47.131 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:47.131 | INFO | "malloc" : " 14 GB", | 20260507 23:02:47.131 | INFO | "pinned" : " 29 GB" | 20260507 23:02:47.131 | INFO | } | 20260507 23:02:47.131 | INFO | Perf report for the last 5.13793 seconds: | 20260507 23:02:47.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:47.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.131 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 23:02:47.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.131 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260507 23:02:47.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:47.132 | INFO | Uploaded 36GiB to GPU in 2.3613s (15.2458GiB/s) | 20260507 23:02:47.132 | INFO | Downloaded 34GiB to GPU in 2.50075s (13.5959GiB/s) | 20260507 23:02:52.272 | INFO | Analyzing chunk frames = [3291648,3292160) | 20260507 23:02:52.272 | INFO | MemInfo: { | 20260507 23:02:52.272 | INFO | "gpu" : " 17 GB", | 20260507 23:02:52.272 | INFO | "huge" : "", | 20260507 23:02:52.272 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:52.272 | INFO | "malloc" : " 14 GB", | 20260507 23:02:52.272 | INFO | "pinned" : " 29 GB" | 20260507 23:02:52.272 | INFO | } | 20260507 23:02:52.272 | INFO | Perf report for the last 5.14057 seconds: | 20260507 23:02:52.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:52.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 23:02:52.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 23:02:52.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:52.272 | INFO | Uploaded 36GiB to GPU in 2.54453s (14.148GiB/s) | 20260507 23:02:52.273 | INFO | Downloaded 34GiB to GPU in 2.41389s (14.0852GiB/s) | 20260507 23:02:57.417 | INFO | Analyzing chunk frames = [3292160,3292672) | 20260507 23:02:57.417 | INFO | MemInfo: { | 20260507 23:02:57.417 | INFO | "gpu" : " 17 GB", | 20260507 23:02:57.417 | INFO | "huge" : "", | 20260507 23:02:57.417 | INFO | "huge_shared" : " 29 GB", | 20260507 23:02:57.417 | INFO | "malloc" : " 14 GB", | 20260507 23:02:57.417 | INFO | "pinned" : " 29 GB" | 20260507 23:02:57.417 | INFO | } | 20260507 23:02:57.417 | INFO | Perf report for the last 5.14484 seconds: | 20260507 23:02:57.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:02:57.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 23:02:57.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:02:57.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:02:57.417 | INFO | Uploaded 36GiB to GPU in 2.53846s (14.1818GiB/s) | 20260507 23:02:57.417 | INFO | Downloaded 34GiB to GPU in 2.44587s (13.901GiB/s) | 20260507 23:03:02.558 | INFO | Analyzing chunk frames = [3292672,3293184) | 20260507 23:03:02.558 | INFO | MemInfo: { | 20260507 23:03:02.558 | INFO | "gpu" : " 17 GB", | 20260507 23:03:02.558 | INFO | "huge" : "", | 20260507 23:03:02.558 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:02.558 | INFO | "malloc" : " 14 GB", | 20260507 23:03:02.558 | INFO | "pinned" : " 29 GB" | 20260507 23:03:02.558 | INFO | } | 20260507 23:03:02.558 | INFO | Perf report for the last 5.14102 seconds: | 20260507 23:03:02.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:02.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 23:03:02.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:03:02.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:02.558 | INFO | Uploaded 36GiB to GPU in 2.53806s (14.1841GiB/s) | 20260507 23:03:02.558 | INFO | Downloaded 34GiB to GPU in 2.44241s (13.9207GiB/s) | 20260507 23:03:02.558 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:03:02.558 | INFO | Total mem usage: 16.7 GiB | 20260507 23:03:02.558 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:03:02.558 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260507 23:03:02.558 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:03:02.558 | INFO | High Water Breakdown: | 20260507 23:03:02.558 | INFO | 60.2% : 9.05 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:03:02.558 | INFO | 39.7% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:03:02.558 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:03:02.558 | INFO | ------------------------------------------ | 20260507 23:03:02.558 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:03:02.558 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:03:02.558 | INFO | Total mem usage: 26.5 GiB | 20260507 23:03:02.558 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:03:02.558 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:03:02.558 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:03:02.558 | INFO | High Water Breakdown: | 20260507 23:03:02.558 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:03:02.558 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:03:02.558 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:03:02.558 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:03:02.558 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:03:02.558 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:03:02.558 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:03:02.558 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:03:02.558 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:03:02.558 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:03:02.558 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:03:02.558 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:03:02.558 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:03:02.558 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:03:02.558 | INFO | ------------------------------------------ | 20260507 23:03:02.558 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:03:02.558 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:03:02.558 | INFO | Total mem usage: 27.8 GiB | 20260507 23:03:02.558 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:03:02.558 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:03:02.558 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:03:02.558 | INFO | High Water Breakdown: | 20260507 23:03:02.558 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:03:02.558 | INFO | ------------------------------------------ | 20260507 23:03:02.558 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:03:02.558 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:03:02.558 | INFO | Total mem usage: 15.7 GiB | 20260507 23:03:02.558 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:03:02.558 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:03:02.558 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:03:02.558 | INFO | High Water Breakdown: | 20260507 23:03:02.558 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:03:02.558 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:03:02.558 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:03:02.558 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:03:02.558 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:03:02.558 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:03:02.558 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:03:02.558 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:03:02.558 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:03:02.558 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:03:02.558 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:03:02.558 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:03:02.558 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:03:02.558 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:03:02.558 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:03:02.558 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:03:02.558 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:03:02.558 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:03:02.558 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:03:02.558 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:03:02.558 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:03:02.558 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:03:02.558 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:03:02.558 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:03:02.558 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:03:02.558 | INFO | ------------------------------------------ | 20260507 23:03:02.558 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:03:02.558 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:03:02.560 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:03:02.560 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:03:02.560 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.63e+01, 1.42e+01 / 1.45e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.02e-01, 1.18e-02 / 4.68e-03 | 20260507 23:03:02.560 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.46e+00, 5.80e-01 / 2.02e-01 | 20260507 23:03:02.560 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.10e-01 | 20260507 23:03:02.560 | INFO | PerfReport: 16.16%, DME -- 4.24e+01 / 4.63e+01, 3.99e+00 / 1.78e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 55.92%, FrameLabeling -- 1.83e+01 / 1.94e+01, 1.53e+00 / 1.66e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.48e+00 / 2.78e+00, 4.31e-01 / 2.03e-01 | 20260507 23:03:02.560 | INFO | PerfReport: 3.46%, Metrics -- 1.14e+00 / 1.37e+00, 2.83e-01 / 1.76e-01 | 20260507 23:03:02.560 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.53e-01 / 3.92e-01, 5.68e-02 / 3.26e-02 | 20260507 23:03:02.560 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:03:02.560 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:03:02.560 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:03:02.560 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 24.50%, Upload -- 2.62e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 24.29%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 30.63%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 2.29%, LockUpload -- 2.45e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 2.70%, LockDownload -- 2.88e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 15.58%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 23:03:02.560 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260507 23:03:02.655 | INFO | Starting a round of Baz IO | 20260507 23:03:06.125 | INFO | Populating RTMetrics report | 20260507 23:03:06.125 | INFO | RTMetrics report populated | 20260507 23:03:06.125 | INFO | Opening tmp RTMetrics json file | 20260507 23:03:06.148 | INFO | Serializing RTMetrics to json | 20260507 23:03:06.150 | INFO | Serialization done, swapping tmp file now... | 20260507 23:03:06.150 | INFO | RTMetrics done | 20260507 23:03:06.220 | INFO | Finished a round of Baz IO: Wrote 1.724 GiB in 3.565 seconds | 20260507 23:03:06.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.80% / 0.01% / 57.19% | 20260507 23:03:06.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27897% / 3.97364e-06% / 6.75519e-05% | 20260507 23:03:06.220 | INFO | Baz IO overall: | 20260507 23:03:06.220 | INFO | BytesWritten : 1499964190807 | 20260507 23:03:06.220 | INFO | NumEvents : 726523674871 | 20260507 23:03:07.691 | INFO | Analyzing chunk frames = [3293184,3293696) | 20260507 23:03:07.691 | INFO | MemInfo: { | 20260507 23:03:07.691 | INFO | "gpu" : " 17 GB", | 20260507 23:03:07.691 | INFO | "huge" : "", | 20260507 23:03:07.691 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:07.691 | INFO | "malloc" : " 14 GB", | 20260507 23:03:07.691 | INFO | "pinned" : " 29 GB" | 20260507 23:03:07.691 | INFO | } | 20260507 23:03:07.691 | INFO | Perf report for the last 5.13356 seconds: | 20260507 23:03:07.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:07.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:07.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:07.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:07.692 | INFO | ANALYSIS : 68.8%, 3.0, 107.0ms, 0 | 20260507 23:03:07.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:07.692 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:03:07.692 | INFO | PRE_HQ : 65.8%, 1.7, 57.1ms, 0 | 20260507 23:03:07.692 | INFO | BAZWRITER : 69.0%, 4.3, 151.5ms, 0 | 20260507 23:03:07.692 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 23:03:07.692 | INFO | Uploaded 36GiB to GPU in 2.5803s (13.9519GiB/s) | 20260507 23:03:07.692 | INFO | Downloaded 38GiB to GPU in 2.6985s (14.0819GiB/s) | 20260507 23:03:12.831 | INFO | Analyzing chunk frames = [3293696,3294208) | 20260507 23:03:12.831 | INFO | MemInfo: { | 20260507 23:03:12.831 | INFO | "gpu" : " 17 GB", | 20260507 23:03:12.831 | INFO | "huge" : "", | 20260507 23:03:12.831 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:12.831 | INFO | "malloc" : " 14 GB", | 20260507 23:03:12.831 | INFO | "pinned" : " 29 GB" | 20260507 23:03:12.831 | INFO | } | 20260507 23:03:12.831 | INFO | Perf report for the last 5.13946 seconds: | 20260507 23:03:12.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:12.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 23:03:12.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:03:12.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:12.831 | INFO | Uploaded 36GiB to GPU in 2.58853s (13.9075GiB/s) | 20260507 23:03:12.831 | INFO | Downloaded 34GiB to GPU in 2.48158s (13.701GiB/s) | 20260507 23:03:17.971 | INFO | Analyzing chunk frames = [3294208,3294720) | 20260507 23:03:17.971 | INFO | MemInfo: { | 20260507 23:03:17.971 | INFO | "gpu" : " 17 GB", | 20260507 23:03:17.971 | INFO | "huge" : "", | 20260507 23:03:17.971 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:17.971 | INFO | "malloc" : " 14 GB", | 20260507 23:03:17.971 | INFO | "pinned" : " 29 GB" | 20260507 23:03:17.971 | INFO | } | 20260507 23:03:17.971 | INFO | Perf report for the last 5.14034 seconds: | 20260507 23:03:17.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:17.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.971 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260507 23:03:17.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.971 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:03:17.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:17.972 | INFO | Uploaded 36GiB to GPU in 2.60111s (13.8403GiB/s) | 20260507 23:03:17.972 | INFO | Downloaded 34GiB to GPU in 2.46603s (13.7874GiB/s) | 20260507 23:03:23.112 | INFO | Analyzing chunk frames = [3294720,3295232) | 20260507 23:03:23.112 | INFO | MemInfo: { | 20260507 23:03:23.112 | INFO | "gpu" : " 17 GB", | 20260507 23:03:23.112 | INFO | "huge" : "", | 20260507 23:03:23.112 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:23.112 | INFO | "malloc" : " 14 GB", | 20260507 23:03:23.112 | INFO | "pinned" : " 29 GB" | 20260507 23:03:23.112 | INFO | } | 20260507 23:03:23.112 | INFO | Perf report for the last 5.14112 seconds: | 20260507 23:03:23.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:23.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.112 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 23:03:23.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.113 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260507 23:03:23.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:23.113 | INFO | Uploaded 36GiB to GPU in 2.5176s (14.2993GiB/s) | 20260507 23:03:23.113 | INFO | Downloaded 34GiB to GPU in 2.40168s (14.1568GiB/s) | 20260507 23:03:28.253 | INFO | Analyzing chunk frames = [3295232,3295744) | 20260507 23:03:28.253 | INFO | MemInfo: { | 20260507 23:03:28.253 | INFO | "gpu" : " 17 GB", | 20260507 23:03:28.253 | INFO | "huge" : "", | 20260507 23:03:28.253 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:28.254 | INFO | "malloc" : " 14 GB", | 20260507 23:03:28.254 | INFO | "pinned" : " 29 GB" | 20260507 23:03:28.254 | INFO | } | 20260507 23:03:28.254 | INFO | Perf report for the last 5.14127 seconds: | 20260507 23:03:28.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:28.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 23:03:28.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260507 23:03:28.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:28.254 | INFO | Uploaded 36GiB to GPU in 2.42859s (14.8234GiB/s) | 20260507 23:03:28.254 | INFO | Downloaded 34GiB to GPU in 2.49363s (13.6347GiB/s) | 20260507 23:03:33.388 | INFO | Analyzing chunk frames = [3295744,3296256) | 20260507 23:03:33.388 | INFO | MemInfo: { | 20260507 23:03:33.388 | INFO | "gpu" : " 17 GB", | 20260507 23:03:33.388 | INFO | "huge" : "", | 20260507 23:03:33.388 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:33.388 | INFO | "malloc" : " 14 GB", | 20260507 23:03:33.388 | INFO | "pinned" : " 29 GB" | 20260507 23:03:33.388 | INFO | } | 20260507 23:03:33.388 | INFO | Perf report for the last 5.13479 seconds: | 20260507 23:03:33.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:33.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.389 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 23:03:33.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.389 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:03:33.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:33.389 | INFO | Uploaded 36GiB to GPU in 2.43389s (14.7911GiB/s) | 20260507 23:03:33.389 | INFO | Downloaded 34GiB to GPU in 2.39457s (14.1988GiB/s) | 20260507 23:03:38.534 | INFO | Analyzing chunk frames = [3296256,3296768) | 20260507 23:03:38.534 | INFO | MemInfo: { | 20260507 23:03:38.534 | INFO | "gpu" : " 17 GB", | 20260507 23:03:38.534 | INFO | "huge" : "", | 20260507 23:03:38.534 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:38.534 | INFO | "malloc" : " 14 GB", | 20260507 23:03:38.534 | INFO | "pinned" : " 29 GB" | 20260507 23:03:38.534 | INFO | } | 20260507 23:03:38.534 | INFO | Perf report for the last 5.14553 seconds: | 20260507 23:03:38.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:38.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 23:03:38.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:03:38.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:38.534 | INFO | Uploaded 36GiB to GPU in 2.53397s (14.2069GiB/s) | 20260507 23:03:38.534 | INFO | Downloaded 34GiB to GPU in 2.443s (13.9173GiB/s) | 20260507 23:03:43.673 | INFO | Analyzing chunk frames = [3296768,3297280) | 20260507 23:03:43.673 | INFO | MemInfo: { | 20260507 23:03:43.673 | INFO | "gpu" : " 17 GB", | 20260507 23:03:43.673 | INFO | "huge" : "", | 20260507 23:03:43.673 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:43.673 | INFO | "malloc" : " 14 GB", | 20260507 23:03:43.673 | INFO | "pinned" : " 29 GB" | 20260507 23:03:43.673 | INFO | } | 20260507 23:03:43.673 | INFO | Perf report for the last 5.13898 seconds: | 20260507 23:03:43.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:43.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.673 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:03:43.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.673 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:03:43.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:43.674 | INFO | Uploaded 36GiB to GPU in 2.51317s (14.3246GiB/s) | 20260507 23:03:43.674 | INFO | Downloaded 34GiB to GPU in 2.43911s (13.9395GiB/s) | 20260507 23:03:43.771 | INFO | Starting a round of Baz IO | 20260507 23:03:47.249 | INFO | Populating RTMetrics report | 20260507 23:03:47.249 | INFO | RTMetrics report populated | 20260507 23:03:47.249 | INFO | Opening tmp RTMetrics json file | 20260507 23:03:47.319 | INFO | Serializing RTMetrics to json | 20260507 23:03:47.346 | INFO | Serialization done, swapping tmp file now... | 20260507 23:03:47.348 | INFO | RTMetrics done | 20260507 23:03:47.779 | INFO | Finished a round of Baz IO: Wrote 1.723 GiB in 4.007 seconds | 20260507 23:03:47.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.80% / 0.01% / 57.20% | 20260507 23:03:47.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27398% / 7.94729e-06% / 7.54992e-05% | 20260507 23:03:47.779 | INFO | Baz IO overall: | 20260507 23:03:47.779 | INFO | BytesWritten : 1501813767580 | 20260507 23:03:47.779 | INFO | NumEvents : 727415786650 | 20260507 23:03:48.809 | INFO | Analyzing chunk frames = [3297280,3297792) | 20260507 23:03:48.809 | INFO | MemInfo: { | 20260507 23:03:48.809 | INFO | "gpu" : " 17 GB", | 20260507 23:03:48.809 | INFO | "huge" : "", | 20260507 23:03:48.809 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:48.809 | INFO | "malloc" : " 14 GB", | 20260507 23:03:48.809 | INFO | "pinned" : " 29 GB" | 20260507 23:03:48.809 | INFO | } | 20260507 23:03:48.809 | INFO | Perf report for the last 5.13603 seconds: | 20260507 23:03:48.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:48.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:48.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:48.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:48.809 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 23:03:48.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:48.809 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 23:03:48.809 | INFO | PRE_HQ : 65.9%, 1.7, 57.3ms, 0 | 20260507 23:03:48.809 | INFO | BAZWRITER : 77.4%, 4.5, 177.6ms, 0 | 20260507 23:03:48.809 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 23:03:48.810 | INFO | Uploaded 36GiB to GPU in 2.56944s (14.0108GiB/s) | 20260507 23:03:48.810 | INFO | Downloaded 38GiB to GPU in 2.69863s (14.0812GiB/s) | 20260507 23:03:53.943 | INFO | Analyzing chunk frames = [3297792,3298304) | 20260507 23:03:53.943 | INFO | MemInfo: { | 20260507 23:03:53.943 | INFO | "gpu" : " 17 GB", | 20260507 23:03:53.943 | INFO | "huge" : "", | 20260507 23:03:53.943 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:53.943 | INFO | "malloc" : " 14 GB", | 20260507 23:03:53.943 | INFO | "pinned" : " 29 GB" | 20260507 23:03:53.943 | INFO | } | 20260507 23:03:53.943 | INFO | Perf report for the last 5.13379 seconds: | 20260507 23:03:53.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:53.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 23:03:53.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:03:53.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:53.943 | INFO | Uploaded 36GiB to GPU in 2.57966s (13.9553GiB/s) | 20260507 23:03:53.943 | INFO | Downloaded 34GiB to GPU in 2.476s (13.7318GiB/s) | 20260507 23:03:59.081 | INFO | Analyzing chunk frames = [3298304,3298816) | 20260507 23:03:59.081 | INFO | MemInfo: { | 20260507 23:03:59.081 | INFO | "gpu" : " 17 GB", | 20260507 23:03:59.081 | INFO | "huge" : "", | 20260507 23:03:59.081 | INFO | "huge_shared" : " 29 GB", | 20260507 23:03:59.081 | INFO | "malloc" : " 14 GB", | 20260507 23:03:59.081 | INFO | "pinned" : " 29 GB" | 20260507 23:03:59.081 | INFO | } | 20260507 23:03:59.081 | INFO | Perf report for the last 5.13837 seconds: | 20260507 23:03:59.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:03:59.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:03:59.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:03:59.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:03:59.081 | INFO | Uploaded 36GiB to GPU in 2.52305s (14.2685GiB/s) | 20260507 23:03:59.081 | INFO | Downloaded 34GiB to GPU in 2.47759s (13.723GiB/s) | 20260507 23:04:04.224 | INFO | Analyzing chunk frames = [3298816,3299328) | 20260507 23:04:04.224 | INFO | MemInfo: { | 20260507 23:04:04.224 | INFO | "gpu" : " 17 GB", | 20260507 23:04:04.224 | INFO | "huge" : "", | 20260507 23:04:04.224 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:04.224 | INFO | "malloc" : " 14 GB", | 20260507 23:04:04.224 | INFO | "pinned" : " 29 GB" | 20260507 23:04:04.224 | INFO | } | 20260507 23:04:04.224 | INFO | Perf report for the last 5.14316 seconds: | 20260507 23:04:04.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:04.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.225 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260507 23:04:04.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.225 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 23:04:04.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:04.225 | INFO | Uploaded 36GiB to GPU in 2.60486s (13.8203GiB/s) | 20260507 23:04:04.225 | INFO | Downloaded 34GiB to GPU in 2.47142s (13.7573GiB/s) | 20260507 23:04:09.370 | INFO | Analyzing chunk frames = [3299328,3299840) | 20260507 23:04:09.370 | INFO | MemInfo: { | 20260507 23:04:09.370 | INFO | "gpu" : " 17 GB", | 20260507 23:04:09.370 | INFO | "huge" : "", | 20260507 23:04:09.370 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:09.370 | INFO | "malloc" : " 14 GB", | 20260507 23:04:09.370 | INFO | "pinned" : " 29 GB" | 20260507 23:04:09.370 | INFO | } | 20260507 23:04:09.370 | INFO | Perf report for the last 5.14553 seconds: | 20260507 23:04:09.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:09.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 23:04:09.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260507 23:04:09.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:09.370 | INFO | Uploaded 36GiB to GPU in 2.37975s (15.1276GiB/s) | 20260507 23:04:09.371 | INFO | Downloaded 34GiB to GPU in 2.49589s (13.6224GiB/s) | 20260507 23:04:14.508 | INFO | Analyzing chunk frames = [3299840,3300352) | 20260507 23:04:14.508 | INFO | MemInfo: { | 20260507 23:04:14.508 | INFO | "gpu" : " 17 GB", | 20260507 23:04:14.508 | INFO | "huge" : "", | 20260507 23:04:14.508 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:14.508 | INFO | "malloc" : " 14 GB", | 20260507 23:04:14.508 | INFO | "pinned" : " 29 GB" | 20260507 23:04:14.508 | INFO | } | 20260507 23:04:14.509 | INFO | Perf report for the last 5.13857 seconds: | 20260507 23:04:14.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:14.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 23:04:14.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:04:14.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:14.509 | INFO | Uploaded 36GiB to GPU in 2.43419s (14.7893GiB/s) | 20260507 23:04:14.509 | INFO | Downloaded 34GiB to GPU in 2.39341s (14.2057GiB/s) | 20260507 23:04:19.645 | INFO | Analyzing chunk frames = [3300352,3300864) | 20260507 23:04:19.645 | INFO | MemInfo: { | 20260507 23:04:19.645 | INFO | "gpu" : " 17 GB", | 20260507 23:04:19.645 | INFO | "huge" : "", | 20260507 23:04:19.645 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:19.645 | INFO | "malloc" : " 14 GB", | 20260507 23:04:19.645 | INFO | "pinned" : " 29 GB" | 20260507 23:04:19.645 | INFO | } | 20260507 23:04:19.645 | INFO | Perf report for the last 5.13668 seconds: | 20260507 23:04:19.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:19.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.645 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:04:19.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.645 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:04:19.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:19.646 | INFO | Uploaded 36GiB to GPU in 2.53623s (14.1943GiB/s) | 20260507 23:04:19.646 | INFO | Downloaded 34GiB to GPU in 2.43663s (13.9537GiB/s) | 20260507 23:04:24.785 | INFO | Analyzing chunk frames = [3300864,3301376) | 20260507 23:04:24.785 | INFO | MemInfo: { | 20260507 23:04:24.785 | INFO | "gpu" : " 17 GB", | 20260507 23:04:24.785 | INFO | "huge" : "", | 20260507 23:04:24.785 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:24.785 | INFO | "malloc" : " 14 GB", | 20260507 23:04:24.785 | INFO | "pinned" : " 29 GB" | 20260507 23:04:24.785 | INFO | } | 20260507 23:04:24.785 | INFO | Perf report for the last 5.13966 seconds: | 20260507 23:04:24.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:24.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:04:24.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:04:24.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:24.785 | INFO | Uploaded 36GiB to GPU in 2.51341s (14.3232GiB/s) | 20260507 23:04:24.785 | INFO | Downloaded 34GiB to GPU in 2.43491s (13.9635GiB/s) | 20260507 23:04:24.786 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:04:24.786 | INFO | Total mem usage: 16.7 GiB | 20260507 23:04:24.786 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:04:24.786 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -4.3%) | 20260507 23:04:24.786 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:04:24.786 | INFO | High Water Breakdown: | 20260507 23:04:24.786 | INFO | 61.1% : 9.78 GiB ( 94.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:04:24.786 | INFO | 38.7% : 6.2 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:04:24.786 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:04:24.786 | INFO | ------------------------------------------ | 20260507 23:04:24.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:04:24.786 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:04:24.786 | INFO | Total mem usage: 26.5 GiB | 20260507 23:04:24.786 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:04:24.786 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:04:24.786 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:04:24.786 | INFO | High Water Breakdown: | 20260507 23:04:24.786 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:04:24.786 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:04:24.786 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:04:24.786 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:04:24.786 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:04:24.786 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:04:24.786 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:04:24.786 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:04:24.786 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:04:24.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:04:24.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:04:24.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:04:24.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:04:24.786 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:04:24.786 | INFO | ------------------------------------------ | 20260507 23:04:24.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:04:24.786 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:04:24.786 | INFO | Total mem usage: 27.8 GiB | 20260507 23:04:24.786 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:04:24.786 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:04:24.786 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:04:24.786 | INFO | High Water Breakdown: | 20260507 23:04:24.786 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:04:24.786 | INFO | ------------------------------------------ | 20260507 23:04:24.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:04:24.786 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:04:24.786 | INFO | Total mem usage: 15.7 GiB | 20260507 23:04:24.786 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:04:24.786 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:04:24.786 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:04:24.786 | INFO | High Water Breakdown: | 20260507 23:04:24.786 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:04:24.786 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:04:24.786 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:04:24.786 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:04:24.786 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:04:24.786 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:04:24.786 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:04:24.786 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:04:24.786 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:04:24.786 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:04:24.786 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:04:24.786 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:04:24.786 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:04:24.786 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:04:24.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:04:24.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:04:24.786 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:04:24.786 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:04:24.786 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:04:24.786 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:04:24.786 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:04:24.786 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:04:24.786 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:04:24.786 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:04:24.786 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:04:24.786 | INFO | ------------------------------------------ | 20260507 23:04:24.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:04:24.786 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:04:24.786 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:04:24.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:04:24.786 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.61e+01, 1.41e+01 / 1.57e+00 | 20260507 23:04:24.786 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.03e-01, 1.17e-02 / 4.95e-03 | 20260507 23:04:24.787 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.49e+00, 5.84e-01 / 1.93e-01 | 20260507 23:04:24.787 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.11e-01 | 20260507 23:04:24.787 | INFO | PerfReport: 15.86%, DME -- 4.18e+01 / 4.52e+01, 3.90e+00 / 1.62e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.78e+00 / 1.88e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.25e-01 / 2.37e-01 | 20260507 23:04:24.787 | INFO | PerfReport: 3.45%, Metrics -- 1.14e+00 / 1.36e+00, 2.91e-01 / 1.84e-01 | 20260507 23:04:24.787 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.51e-01 / 3.85e-01, 5.73e-02 / 3.15e-02 | 20260507 23:04:24.787 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:04:24.787 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:04:24.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:04:24.787 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 24.56%, Upload -- 2.63e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 24.17%, Download -- 2.58e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 30.80%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 2.31%, LockUpload -- 2.47e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 2.27%, LockDownload -- 2.43e+00 / 0.00e+00, 3.69e+00 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 15.88%, LockCompute -- 1.70e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 23:04:24.787 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 23:04:24.882 | INFO | Starting a round of Baz IO | 20260507 23:04:28.337 | INFO | Populating RTMetrics report | 20260507 23:04:28.337 | INFO | RTMetrics report populated | 20260507 23:04:28.337 | INFO | Opening tmp RTMetrics json file | 20260507 23:04:28.447 | INFO | Serializing RTMetrics to json | 20260507 23:04:28.476 | INFO | Serialization done, swapping tmp file now... | 20260507 23:04:28.480 | INFO | RTMetrics done | 20260507 23:04:28.504 | INFO | Finished a round of Baz IO: Wrote 1.723 GiB in 3.622 seconds | 20260507 23:04:28.504 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.79% / 0.01% / 57.20% | 20260507 23:04:28.504 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27414% / 7.94729e-06% / 5.16574e-05% | 20260507 23:04:28.504 | INFO | Baz IO overall: | 20260507 23:04:28.504 | INFO | BytesWritten : 1503663515834 | 20260507 23:04:28.504 | INFO | NumEvents : 728308172140 | 20260507 23:04:29.921 | INFO | Analyzing chunk frames = [3301376,3301888) | 20260507 23:04:29.921 | INFO | MemInfo: { | 20260507 23:04:29.921 | INFO | "gpu" : " 17 GB", | 20260507 23:04:29.921 | INFO | "huge" : "", | 20260507 23:04:29.921 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:29.921 | INFO | "malloc" : " 14 GB", | 20260507 23:04:29.921 | INFO | "pinned" : " 29 GB" | 20260507 23:04:29.921 | INFO | } | 20260507 23:04:29.921 | INFO | Perf report for the last 5.13588 seconds: | 20260507 23:04:29.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:29.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:29.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:29.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:29.921 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 23:04:29.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:29.921 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 23:04:29.921 | INFO | PRE_HQ : 65.5%, 1.7, 56.9ms, 0 | 20260507 23:04:29.921 | INFO | BAZWRITER : 70.1%, 3.7, 133.7ms, 0 | 20260507 23:04:29.921 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 23:04:29.921 | INFO | Uploaded 36GiB to GPU in 2.55331s (14.0993GiB/s) | 20260507 23:04:29.921 | INFO | Downloaded 38GiB to GPU in 2.68264s (14.1651GiB/s) | 20260507 23:04:35.059 | INFO | Analyzing chunk frames = [3301888,3302400) | 20260507 23:04:35.059 | INFO | MemInfo: { | 20260507 23:04:35.059 | INFO | "gpu" : " 17 GB", | 20260507 23:04:35.059 | INFO | "huge" : "", | 20260507 23:04:35.059 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:35.059 | INFO | "malloc" : " 14 GB", | 20260507 23:04:35.059 | INFO | "pinned" : " 29 GB" | 20260507 23:04:35.059 | INFO | } | 20260507 23:04:35.059 | INFO | Perf report for the last 5.13822 seconds: | 20260507 23:04:35.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:35.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260507 23:04:35.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:04:35.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:35.059 | INFO | Uploaded 36GiB to GPU in 2.59038s (13.8976GiB/s) | 20260507 23:04:35.059 | INFO | Downloaded 34GiB to GPU in 2.47074s (13.7611GiB/s) | 20260507 23:04:40.197 | INFO | Analyzing chunk frames = [3302400,3302912) | 20260507 23:04:40.197 | INFO | MemInfo: { | 20260507 23:04:40.197 | INFO | "gpu" : " 17 GB", | 20260507 23:04:40.197 | INFO | "huge" : "", | 20260507 23:04:40.197 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:40.197 | INFO | "malloc" : " 14 GB", | 20260507 23:04:40.197 | INFO | "pinned" : " 29 GB" | 20260507 23:04:40.197 | INFO | } | 20260507 23:04:40.198 | INFO | Perf report for the last 5.13851 seconds: | 20260507 23:04:40.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:40.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:04:40.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:04:40.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:40.198 | INFO | Uploaded 36GiB to GPU in 2.55s (14.1176GiB/s) | 20260507 23:04:40.198 | INFO | Downloaded 34GiB to GPU in 2.4504s (13.8753GiB/s) | 20260507 23:04:45.344 | INFO | Analyzing chunk frames = [3302912,3303424) | 20260507 23:04:45.344 | INFO | MemInfo: { | 20260507 23:04:45.344 | INFO | "gpu" : " 17 GB", | 20260507 23:04:45.344 | INFO | "huge" : "", | 20260507 23:04:45.344 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:45.344 | INFO | "malloc" : " 14 GB", | 20260507 23:04:45.344 | INFO | "pinned" : " 29 GB" | 20260507 23:04:45.344 | INFO | } | 20260507 23:04:45.344 | INFO | Perf report for the last 5.14613 seconds: | 20260507 23:04:45.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:45.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260507 23:04:45.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:04:45.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:45.344 | INFO | Uploaded 36GiB to GPU in 2.49095s (14.4523GiB/s) | 20260507 23:04:45.344 | INFO | Downloaded 34GiB to GPU in 2.3864s (14.2474GiB/s) | 20260507 23:04:50.484 | INFO | Analyzing chunk frames = [3303424,3303936) | 20260507 23:04:50.484 | INFO | MemInfo: { | 20260507 23:04:50.484 | INFO | "gpu" : " 17 GB", | 20260507 23:04:50.484 | INFO | "huge" : "", | 20260507 23:04:50.484 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:50.484 | INFO | "malloc" : " 14 GB", | 20260507 23:04:50.484 | INFO | "pinned" : " 29 GB" | 20260507 23:04:50.484 | INFO | } | 20260507 23:04:50.484 | INFO | Perf report for the last 5.14075 seconds: | 20260507 23:04:50.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:50.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.485 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:04:50.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.485 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:04:50.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:50.485 | INFO | Uploaded 36GiB to GPU in 2.46448s (14.6075GiB/s) | 20260507 23:04:50.485 | INFO | Downloaded 34GiB to GPU in 2.49564s (13.6237GiB/s) | 20260507 23:04:55.622 | INFO | Analyzing chunk frames = [3303936,3304448) | 20260507 23:04:55.622 | INFO | MemInfo: { | 20260507 23:04:55.622 | INFO | "gpu" : " 17 GB", | 20260507 23:04:55.622 | INFO | "huge" : "", | 20260507 23:04:55.622 | INFO | "huge_shared" : " 29 GB", | 20260507 23:04:55.622 | INFO | "malloc" : " 14 GB", | 20260507 23:04:55.622 | INFO | "pinned" : " 29 GB" | 20260507 23:04:55.622 | INFO | } | 20260507 23:04:55.622 | INFO | Perf report for the last 5.13739 seconds: | 20260507 23:04:55.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:04:55.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:04:55.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:04:55.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:04:55.622 | INFO | Uploaded 36GiB to GPU in 2.40269s (14.9832GiB/s) | 20260507 23:04:55.622 | INFO | Downloaded 34GiB to GPU in 2.40226s (14.1533GiB/s) | 20260507 23:05:00.758 | INFO | Analyzing chunk frames = [3304448,3304960) | 20260507 23:05:00.758 | INFO | MemInfo: { | 20260507 23:05:00.758 | INFO | "gpu" : " 17 GB", | 20260507 23:05:00.758 | INFO | "huge" : "", | 20260507 23:05:00.758 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:00.758 | INFO | "malloc" : " 14 GB", | 20260507 23:05:00.758 | INFO | "pinned" : " 29 GB" | 20260507 23:05:00.758 | INFO | } | 20260507 23:05:00.758 | INFO | Perf report for the last 5.13602 seconds: | 20260507 23:05:00.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:00.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:05:00.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:05:00.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:00.758 | INFO | Uploaded 36GiB to GPU in 2.5312s (14.2225GiB/s) | 20260507 23:05:00.758 | INFO | Downloaded 34GiB to GPU in 2.45019s (13.8765GiB/s) | 20260507 23:05:05.895 | INFO | Analyzing chunk frames = [3304960,3305472) | 20260507 23:05:05.895 | INFO | MemInfo: { | 20260507 23:05:05.895 | INFO | "gpu" : " 17 GB", | 20260507 23:05:05.895 | INFO | "huge" : "", | 20260507 23:05:05.895 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:05.895 | INFO | "malloc" : " 14 GB", | 20260507 23:05:05.895 | INFO | "pinned" : " 29 GB" | 20260507 23:05:05.895 | INFO | } | 20260507 23:05:05.895 | INFO | Perf report for the last 5.13727 seconds: | 20260507 23:05:05.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:05.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:05:05.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:05:05.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:05.895 | INFO | Uploaded 36GiB to GPU in 2.51183s (14.3322GiB/s) | 20260507 23:05:05.896 | INFO | Downloaded 34GiB to GPU in 2.44178s (13.9242GiB/s) | 20260507 23:05:05.993 | INFO | Starting a round of Baz IO | 20260507 23:05:09.463 | INFO | Populating RTMetrics report | 20260507 23:05:09.463 | INFO | RTMetrics report populated | 20260507 23:05:09.463 | INFO | Opening tmp RTMetrics json file | 20260507 23:05:09.468 | INFO | Serializing RTMetrics to json | 20260507 23:05:09.494 | INFO | Serialization done, swapping tmp file now... | 20260507 23:05:09.504 | INFO | RTMetrics done | 20260507 23:05:09.568 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.574 seconds | 20260507 23:05:09.568 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.78% / 0.01% / 57.21% | 20260507 23:05:09.568 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26962% / 7.94729e-06% / 8.74201e-05% | 20260507 23:05:09.568 | INFO | Baz IO overall: | 20260507 23:05:09.568 | INFO | BytesWritten : 1505512346784 | 20260507 23:05:09.568 | INFO | NumEvents : 729199755947 | 20260507 23:05:11.034 | INFO | Analyzing chunk frames = [3305472,3305984) | 20260507 23:05:11.034 | INFO | MemInfo: { | 20260507 23:05:11.034 | INFO | "gpu" : " 17 GB", | 20260507 23:05:11.034 | INFO | "huge" : "", | 20260507 23:05:11.034 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:11.034 | INFO | "malloc" : " 14 GB", | 20260507 23:05:11.034 | INFO | "pinned" : " 29 GB" | 20260507 23:05:11.034 | INFO | } | 20260507 23:05:11.034 | INFO | Perf report for the last 5.1388 seconds: | 20260507 23:05:11.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:11.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:11.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:11.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:11.034 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 23:05:11.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:11.035 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 23:05:11.035 | INFO | PRE_HQ : 65.3%, 1.7, 56.8ms, 0 | 20260507 23:05:11.035 | INFO | BAZWRITER : 69.0%, 3.7, 131.7ms, 0 | 20260507 23:05:11.035 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 23:05:11.035 | INFO | Uploaded 36GiB to GPU in 2.56605s (14.0293GiB/s) | 20260507 23:05:11.035 | INFO | Downloaded 38GiB to GPU in 2.69476s (14.1014GiB/s) | 20260507 23:05:16.176 | INFO | Analyzing chunk frames = [3305984,3306496) | 20260507 23:05:16.176 | INFO | MemInfo: { | 20260507 23:05:16.176 | INFO | "gpu" : " 17 GB", | 20260507 23:05:16.176 | INFO | "huge" : "", | 20260507 23:05:16.176 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:16.176 | INFO | "malloc" : " 14 GB", | 20260507 23:05:16.176 | INFO | "pinned" : " 29 GB" | 20260507 23:05:16.176 | INFO | } | 20260507 23:05:16.176 | INFO | Perf report for the last 5.14195 seconds: | 20260507 23:05:16.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:16.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.176 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:05:16.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.176 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 23:05:16.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:16.177 | INFO | Uploaded 36GiB to GPU in 2.51471s (14.3158GiB/s) | 20260507 23:05:16.177 | INFO | Downloaded 34GiB to GPU in 2.48187s (13.6993GiB/s) | 20260507 23:05:21.318 | INFO | Analyzing chunk frames = [3306496,3307008) | 20260507 23:05:21.318 | INFO | MemInfo: { | 20260507 23:05:21.318 | INFO | "gpu" : " 17 GB", | 20260507 23:05:21.318 | INFO | "huge" : "", | 20260507 23:05:21.318 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:21.318 | INFO | "malloc" : " 14 GB", | 20260507 23:05:21.318 | INFO | "pinned" : " 29 GB" | 20260507 23:05:21.318 | INFO | } | 20260507 23:05:21.318 | INFO | Perf report for the last 5.14249 seconds: | 20260507 23:05:21.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:21.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.319 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260507 23:05:21.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.319 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:05:21.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:21.319 | INFO | Uploaded 36GiB to GPU in 2.55431s (14.0939GiB/s) | 20260507 23:05:21.319 | INFO | Downloaded 34GiB to GPU in 2.47492s (13.7378GiB/s) | 20260507 23:05:26.457 | INFO | Analyzing chunk frames = [3307008,3307520) | 20260507 23:05:26.457 | INFO | MemInfo: { | 20260507 23:05:26.457 | INFO | "gpu" : " 17 GB", | 20260507 23:05:26.458 | INFO | "huge" : "", | 20260507 23:05:26.458 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:26.458 | INFO | "malloc" : " 14 GB", | 20260507 23:05:26.458 | INFO | "pinned" : " 29 GB" | 20260507 23:05:26.458 | INFO | } | 20260507 23:05:26.458 | INFO | Perf report for the last 5.13918 seconds: | 20260507 23:05:26.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:26.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:05:26.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:05:26.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:26.458 | INFO | Uploaded 36GiB to GPU in 2.48905s (14.4634GiB/s) | 20260507 23:05:26.458 | INFO | Downloaded 34GiB to GPU in 2.49677s (13.6176GiB/s) | 20260507 23:05:31.595 | INFO | Analyzing chunk frames = [3307520,3308032) | 20260507 23:05:31.595 | INFO | MemInfo: { | 20260507 23:05:31.595 | INFO | "gpu" : " 17 GB", | 20260507 23:05:31.595 | INFO | "huge" : "", | 20260507 23:05:31.595 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:31.595 | INFO | "malloc" : " 14 GB", | 20260507 23:05:31.595 | INFO | "pinned" : " 29 GB" | 20260507 23:05:31.595 | INFO | } | 20260507 23:05:31.595 | INFO | Perf report for the last 5.13722 seconds: | 20260507 23:05:31.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:31.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 23:05:31.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:05:31.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:31.595 | INFO | Uploaded 36GiB to GPU in 2.31059s (15.5804GiB/s) | 20260507 23:05:31.596 | INFO | Downloaded 34GiB to GPU in 2.47022s (13.764GiB/s) | 20260507 23:05:36.738 | INFO | Analyzing chunk frames = [3308032,3308544) | 20260507 23:05:36.738 | INFO | MemInfo: { | 20260507 23:05:36.738 | INFO | "gpu" : " 17 GB", | 20260507 23:05:36.738 | INFO | "huge" : "", | 20260507 23:05:36.738 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:36.738 | INFO | "malloc" : " 14 GB", | 20260507 23:05:36.738 | INFO | "pinned" : " 29 GB" | 20260507 23:05:36.738 | INFO | } | 20260507 23:05:36.738 | INFO | Perf report for the last 5.14297 seconds: | 20260507 23:05:36.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:36.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.738 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:05:36.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.738 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:05:36.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:36.739 | INFO | Uploaded 36GiB to GPU in 2.51582s (14.3095GiB/s) | 20260507 23:05:36.739 | INFO | Downloaded 34GiB to GPU in 2.42117s (14.0428GiB/s) | 20260507 23:05:41.873 | INFO | Analyzing chunk frames = [3308544,3309056) | 20260507 23:05:41.873 | INFO | MemInfo: { | 20260507 23:05:41.873 | INFO | "gpu" : " 17 GB", | 20260507 23:05:41.873 | INFO | "huge" : "", | 20260507 23:05:41.873 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:41.873 | INFO | "malloc" : " 14 GB", | 20260507 23:05:41.873 | INFO | "pinned" : " 29 GB" | 20260507 23:05:41.873 | INFO | } | 20260507 23:05:41.873 | INFO | Perf report for the last 5.13559 seconds: | 20260507 23:05:41.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:41.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.874 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:05:41.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.874 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:05:41.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:41.874 | INFO | Uploaded 36GiB to GPU in 2.5596s (14.0647GiB/s) | 20260507 23:05:41.874 | INFO | Downloaded 34GiB to GPU in 2.46098s (13.8156GiB/s) | 20260507 23:05:47.008 | INFO | Analyzing chunk frames = [3309056,3309568) | 20260507 23:05:47.008 | INFO | MemInfo: { | 20260507 23:05:47.008 | INFO | "gpu" : " 17 GB", | 20260507 23:05:47.008 | INFO | "huge" : "", | 20260507 23:05:47.009 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:47.009 | INFO | "malloc" : " 14 GB", | 20260507 23:05:47.009 | INFO | "pinned" : " 29 GB" | 20260507 23:05:47.009 | INFO | } | 20260507 23:05:47.009 | INFO | Perf report for the last 5.13534 seconds: | 20260507 23:05:47.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:47.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 23:05:47.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:05:47.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:47.009 | INFO | Uploaded 36GiB to GPU in 2.50751s (14.3569GiB/s) | 20260507 23:05:47.009 | INFO | Downloaded 34GiB to GPU in 2.43548s (13.9603GiB/s) | 20260507 23:05:47.010 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:05:47.010 | INFO | Total mem usage: 16.7 GiB | 20260507 23:05:47.010 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:05:47.010 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260507 23:05:47.010 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:05:47.010 | INFO | High Water Breakdown: | 20260507 23:05:47.010 | INFO | 59.9% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:05:47.010 | INFO | 39.9% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:05:47.010 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:05:47.010 | INFO | ------------------------------------------ | 20260507 23:05:47.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:05:47.010 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:05:47.010 | INFO | Total mem usage: 26.5 GiB | 20260507 23:05:47.010 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:05:47.010 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:05:47.010 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:05:47.010 | INFO | High Water Breakdown: | 20260507 23:05:47.010 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:05:47.010 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:05:47.010 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:05:47.010 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:05:47.010 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:05:47.010 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:05:47.010 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:05:47.010 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:05:47.010 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:05:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:05:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:05:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:05:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:05:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:05:47.010 | INFO | ------------------------------------------ | 20260507 23:05:47.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:05:47.010 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:05:47.010 | INFO | Total mem usage: 27.8 GiB | 20260507 23:05:47.010 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:05:47.010 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:05:47.010 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:05:47.010 | INFO | High Water Breakdown: | 20260507 23:05:47.010 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:05:47.010 | INFO | ------------------------------------------ | 20260507 23:05:47.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:05:47.010 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:05:47.010 | INFO | Total mem usage: 15.7 GiB | 20260507 23:05:47.010 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:05:47.010 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:05:47.010 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:05:47.010 | INFO | High Water Breakdown: | 20260507 23:05:47.010 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:05:47.010 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:05:47.010 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:05:47.010 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:05:47.010 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:05:47.010 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:05:47.010 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:05:47.010 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:05:47.010 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:05:47.010 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:05:47.010 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:05:47.010 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:05:47.010 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:05:47.011 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:05:47.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:05:47.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:05:47.011 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:05:47.011 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:05:47.011 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:05:47.011 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:05:47.011 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:05:47.011 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:05:47.011 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:05:47.011 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:05:47.011 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:05:47.011 | INFO | ------------------------------------------ | 20260507 23:05:47.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:05:47.011 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:05:47.011 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:05:47.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:05:47.011 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.61e+01, 1.41e+01 / 1.64e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 0.29%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.13e-02 / 6.20e-03 | 20260507 23:05:47.011 | INFO | PerfReport: 12.12%, Baseline -- 3.99e+00 / 4.50e+00, 5.95e-01 / 2.09e-01 | 20260507 23:05:47.011 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.09e-01 / 1.09e-01 | 20260507 23:05:47.011 | INFO | PerfReport: 15.87%, DME -- 4.18e+01 / 4.53e+01, 3.99e+00 / 1.21e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.79e+00 / 1.57e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.42e+00 / 2.77e+00, 4.30e-01 / 1.98e-01 | 20260507 23:05:47.011 | INFO | PerfReport: 3.43%, Metrics -- 1.13e+00 / 1.34e+00, 2.89e-01 / 1.61e-01 | 20260507 23:05:47.011 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.80e-01, 5.87e-02 / 3.09e-02 | 20260507 23:05:47.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:05:47.011 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:05:47.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:05:47.011 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 24.20%, Download -- 2.59e+01 / 0.00e+00, 8.49e+00 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 30.80%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.29e+00 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 2.49%, LockDownload -- 2.66e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 15.92%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 23:05:47.011 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 23:05:47.122 | INFO | Starting a round of Baz IO | 20260507 23:05:50.581 | INFO | Populating RTMetrics report | 20260507 23:05:50.581 | INFO | RTMetrics report populated | 20260507 23:05:50.581 | INFO | Opening tmp RTMetrics json file | 20260507 23:05:50.605 | INFO | Serializing RTMetrics to json | 20260507 23:05:50.607 | INFO | Serialization done, swapping tmp file now... | 20260507 23:05:50.608 | INFO | RTMetrics done | 20260507 23:05:50.728 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.605 seconds | 20260507 23:05:50.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.78% / 0.01% / 57.22% | 20260507 23:05:50.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26678% / 1.98682e-05% / 7.15256e-05% | 20260507 23:05:50.728 | INFO | Baz IO overall: | 20260507 23:05:50.728 | INFO | BytesWritten : 1507360451412 | 20260507 23:05:50.728 | INFO | NumEvents : 730090871661 | 20260507 23:05:52.147 | INFO | Analyzing chunk frames = [3309568,3310080) | 20260507 23:05:52.147 | INFO | MemInfo: { | 20260507 23:05:52.147 | INFO | "gpu" : " 17 GB", | 20260507 23:05:52.147 | INFO | "huge" : "", | 20260507 23:05:52.147 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:52.147 | INFO | "malloc" : " 14 GB", | 20260507 23:05:52.147 | INFO | "pinned" : " 29 GB" | 20260507 23:05:52.147 | INFO | } | 20260507 23:05:52.147 | INFO | Perf report for the last 5.1378 seconds: | 20260507 23:05:52.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:52.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:52.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:52.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:52.147 | INFO | ANALYSIS : 68.7%, 2.9, 105.4ms, 0 | 20260507 23:05:52.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:52.147 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 23:05:52.147 | INFO | PRE_HQ : 66.3%, 1.8, 62.0ms, 0 | 20260507 23:05:52.147 | INFO | BAZWRITER : 69.8%, 3.7, 131.0ms, 0 | 20260507 23:05:52.147 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 23:05:52.147 | INFO | Uploaded 36GiB to GPU in 2.58594s (13.9214GiB/s) | 20260507 23:05:52.147 | INFO | Downloaded 38GiB to GPU in 2.68726s (14.1408GiB/s) | 20260507 23:05:57.292 | INFO | Analyzing chunk frames = [3310080,3310592) | 20260507 23:05:57.292 | INFO | MemInfo: { | 20260507 23:05:57.292 | INFO | "gpu" : " 17 GB", | 20260507 23:05:57.292 | INFO | "huge" : "", | 20260507 23:05:57.292 | INFO | "huge_shared" : " 29 GB", | 20260507 23:05:57.292 | INFO | "malloc" : " 14 GB", | 20260507 23:05:57.292 | INFO | "pinned" : " 29 GB" | 20260507 23:05:57.292 | INFO | } | 20260507 23:05:57.292 | INFO | Perf report for the last 5.14568 seconds: | 20260507 23:05:57.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:05:57.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.292 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:05:57.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.292 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:05:57.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:05:57.293 | INFO | Uploaded 36GiB to GPU in 2.55529s (14.0884GiB/s) | 20260507 23:05:57.293 | INFO | Downloaded 34GiB to GPU in 2.45446s (13.8523GiB/s) | 20260507 23:06:02.433 | INFO | Analyzing chunk frames = [3310592,3311104) | 20260507 23:06:02.433 | INFO | MemInfo: { | 20260507 23:06:02.433 | INFO | "gpu" : " 17 GB", | 20260507 23:06:02.433 | INFO | "huge" : "", | 20260507 23:06:02.433 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:02.433 | INFO | "malloc" : " 14 GB", | 20260507 23:06:02.433 | INFO | "pinned" : " 29 GB" | 20260507 23:06:02.433 | INFO | } | 20260507 23:06:02.433 | INFO | Perf report for the last 5.14105 seconds: | 20260507 23:06:02.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:02.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.433 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 23:06:02.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.434 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:06:02.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:02.434 | INFO | Uploaded 36GiB to GPU in 2.5398s (14.1743GiB/s) | 20260507 23:06:02.434 | INFO | Downloaded 34GiB to GPU in 2.48183s (13.6996GiB/s) | 20260507 23:06:07.577 | INFO | Analyzing chunk frames = [3311104,3311616) | 20260507 23:06:07.577 | INFO | MemInfo: { | 20260507 23:06:07.577 | INFO | "gpu" : " 17 GB", | 20260507 23:06:07.577 | INFO | "huge" : "", | 20260507 23:06:07.577 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:07.577 | INFO | "malloc" : " 14 GB", | 20260507 23:06:07.577 | INFO | "pinned" : " 29 GB" | 20260507 23:06:07.577 | INFO | } | 20260507 23:06:07.577 | INFO | Perf report for the last 5.14336 seconds: | 20260507 23:06:07.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:07.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 23:06:07.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:06:07.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:07.577 | INFO | Uploaded 36GiB to GPU in 2.55616s (14.0836GiB/s) | 20260507 23:06:07.577 | INFO | Downloaded 34GiB to GPU in 2.48979s (13.6558GiB/s) | 20260507 23:06:12.706 | INFO | Analyzing chunk frames = [3311616,3312128) | 20260507 23:06:12.706 | INFO | MemInfo: { | 20260507 23:06:12.706 | INFO | "gpu" : " 17 GB", | 20260507 23:06:12.706 | INFO | "huge" : "", | 20260507 23:06:12.706 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:12.706 | INFO | "malloc" : " 14 GB", | 20260507 23:06:12.706 | INFO | "pinned" : " 29 GB" | 20260507 23:06:12.706 | INFO | } | 20260507 23:06:12.706 | INFO | Perf report for the last 5.12945 seconds: | 20260507 23:06:12.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:12.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260507 23:06:12.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:06:12.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:12.706 | INFO | Uploaded 36GiB to GPU in 2.40234s (14.9854GiB/s) | 20260507 23:06:12.707 | INFO | Downloaded 34GiB to GPU in 2.51386s (13.525GiB/s) | 20260507 23:06:17.845 | INFO | Analyzing chunk frames = [3312128,3312640) | 20260507 23:06:17.845 | INFO | MemInfo: { | 20260507 23:06:17.845 | INFO | "gpu" : " 17 GB", | 20260507 23:06:17.845 | INFO | "huge" : "", | 20260507 23:06:17.845 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:17.845 | INFO | "malloc" : " 14 GB", | 20260507 23:06:17.845 | INFO | "pinned" : " 29 GB" | 20260507 23:06:17.845 | INFO | } | 20260507 23:06:17.845 | INFO | Perf report for the last 5.13911 seconds: | 20260507 23:06:17.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:17.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.845 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260507 23:06:17.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.845 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:06:17.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:17.846 | INFO | Uploaded 36GiB to GPU in 2.54535s (14.1434GiB/s) | 20260507 23:06:17.846 | INFO | Downloaded 34GiB to GPU in 2.42833s (14.0014GiB/s) | 20260507 23:06:22.988 | INFO | Analyzing chunk frames = [3312640,3313152) | 20260507 23:06:22.988 | INFO | MemInfo: { | 20260507 23:06:22.988 | INFO | "gpu" : " 17 GB", | 20260507 23:06:22.988 | INFO | "huge" : "", | 20260507 23:06:22.988 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:22.988 | INFO | "malloc" : " 14 GB", | 20260507 23:06:22.988 | INFO | "pinned" : " 29 GB" | 20260507 23:06:22.988 | INFO | } | 20260507 23:06:22.988 | INFO | Perf report for the last 5.14239 seconds: | 20260507 23:06:22.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:22.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | ANALYSIS : 67.2%, 3.0, 105.1ms, 0 | 20260507 23:06:22.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 23:06:22.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:22.988 | INFO | Uploaded 36GiB to GPU in 2.52984s (14.2302GiB/s) | 20260507 23:06:22.989 | INFO | Downloaded 34GiB to GPU in 2.43588s (13.958GiB/s) | 20260507 23:06:28.130 | INFO | Analyzing chunk frames = [3313152,3313664) | 20260507 23:06:28.131 | INFO | MemInfo: { | 20260507 23:06:28.131 | INFO | "gpu" : " 17 GB", | 20260507 23:06:28.131 | INFO | "huge" : "", | 20260507 23:06:28.131 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:28.131 | INFO | "malloc" : " 14 GB", | 20260507 23:06:28.131 | INFO | "pinned" : " 29 GB" | 20260507 23:06:28.131 | INFO | } | 20260507 23:06:28.131 | INFO | Perf report for the last 5.14318 seconds: | 20260507 23:06:28.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:28.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 23:06:28.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 23:06:28.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:28.131 | INFO | Uploaded 36GiB to GPU in 2.52931s (14.2331GiB/s) | 20260507 23:06:28.131 | INFO | Downloaded 34GiB to GPU in 2.43628s (13.9557GiB/s) | 20260507 23:06:28.242 | INFO | Starting a round of Baz IO | 20260507 23:06:31.715 | INFO | Populating RTMetrics report | 20260507 23:06:31.715 | INFO | RTMetrics report populated | 20260507 23:06:31.715 | INFO | Opening tmp RTMetrics json file | 20260507 23:06:31.766 | INFO | Serializing RTMetrics to json | 20260507 23:06:31.797 | INFO | Serialization done, swapping tmp file now... | 20260507 23:06:31.806 | INFO | RTMetrics done | 20260507 23:06:31.939 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.697 seconds | 20260507 23:06:31.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.77% / 0.01% / 57.22% | 20260507 23:06:31.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26431% / 7.94729e-06% / 5.16574e-05% | 20260507 23:06:31.939 | INFO | Baz IO overall: | 20260507 23:06:31.939 | INFO | BytesWritten : 1509209204669 | 20260507 23:06:31.939 | INFO | NumEvents : 730982443781 | 20260507 23:06:33.264 | INFO | Analyzing chunk frames = [3313664,3314176) | 20260507 23:06:33.264 | INFO | MemInfo: { | 20260507 23:06:33.264 | INFO | "gpu" : " 17 GB", | 20260507 23:06:33.264 | INFO | "huge" : "", | 20260507 23:06:33.264 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:33.264 | INFO | "malloc" : " 14 GB", | 20260507 23:06:33.264 | INFO | "pinned" : " 29 GB" | 20260507 23:06:33.264 | INFO | } | 20260507 23:06:33.264 | INFO | Perf report for the last 5.13342 seconds: | 20260507 23:06:33.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:33.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:33.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:33.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:33.264 | INFO | ANALYSIS : 68.8%, 2.9, 104.6ms, 0 | 20260507 23:06:33.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:33.264 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 23:06:33.264 | INFO | PRE_HQ : 67.5%, 1.8, 63.6ms, 0 | 20260507 23:06:33.264 | INFO | BAZWRITER : 71.6%, 4.3, 157.5ms, 0 | 20260507 23:06:33.265 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 23:06:33.265 | INFO | Uploaded 36GiB to GPU in 2.60124s (13.8395GiB/s) | 20260507 23:06:33.265 | INFO | Downloaded 38GiB to GPU in 2.69899s (14.0793GiB/s) | 20260507 23:06:38.406 | INFO | Analyzing chunk frames = [3314176,3314688) | 20260507 23:06:38.406 | INFO | MemInfo: { | 20260507 23:06:38.406 | INFO | "gpu" : " 17 GB", | 20260507 23:06:38.406 | INFO | "huge" : "", | 20260507 23:06:38.406 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:38.406 | INFO | "malloc" : " 14 GB", | 20260507 23:06:38.406 | INFO | "pinned" : " 29 GB" | 20260507 23:06:38.406 | INFO | } | 20260507 23:06:38.407 | INFO | Perf report for the last 5.14216 seconds: | 20260507 23:06:38.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:38.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:06:38.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:06:38.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:38.407 | INFO | Uploaded 36GiB to GPU in 2.56467s (14.0369GiB/s) | 20260507 23:06:38.407 | INFO | Downloaded 34GiB to GPU in 2.48634s (13.6747GiB/s) | 20260507 23:06:43.546 | INFO | Analyzing chunk frames = [3314688,3315200) | 20260507 23:06:43.546 | INFO | MemInfo: { | 20260507 23:06:43.546 | INFO | "gpu" : " 17 GB", | 20260507 23:06:43.546 | INFO | "huge" : "", | 20260507 23:06:43.546 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:43.546 | INFO | "malloc" : " 14 GB", | 20260507 23:06:43.546 | INFO | "pinned" : " 29 GB" | 20260507 23:06:43.546 | INFO | } | 20260507 23:06:43.546 | INFO | Perf report for the last 5.13962 seconds: | 20260507 23:06:43.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:43.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 23:06:43.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:06:43.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:43.546 | INFO | Uploaded 36GiB to GPU in 2.55934s (14.0661GiB/s) | 20260507 23:06:43.547 | INFO | Downloaded 34GiB to GPU in 2.47699s (13.7263GiB/s) | 20260507 23:06:48.690 | INFO | Analyzing chunk frames = [3315200,3315712) | 20260507 23:06:48.690 | INFO | MemInfo: { | 20260507 23:06:48.690 | INFO | "gpu" : " 17 GB", | 20260507 23:06:48.690 | INFO | "huge" : "", | 20260507 23:06:48.690 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:48.690 | INFO | "malloc" : " 14 GB", | 20260507 23:06:48.690 | INFO | "pinned" : " 29 GB" | 20260507 23:06:48.690 | INFO | } | 20260507 23:06:48.691 | INFO | Perf report for the last 5.14436 seconds: | 20260507 23:06:48.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:48.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 23:06:48.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:06:48.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:48.691 | INFO | Uploaded 36GiB to GPU in 2.48883s (14.4647GiB/s) | 20260507 23:06:48.691 | INFO | Downloaded 34GiB to GPU in 2.40915s (14.1129GiB/s) | 20260507 23:06:53.825 | INFO | Analyzing chunk frames = [3315712,3316224) | 20260507 23:06:53.825 | INFO | MemInfo: { | 20260507 23:06:53.825 | INFO | "gpu" : " 17 GB", | 20260507 23:06:53.825 | INFO | "huge" : "", | 20260507 23:06:53.825 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:53.825 | INFO | "malloc" : " 14 GB", | 20260507 23:06:53.825 | INFO | "pinned" : " 29 GB" | 20260507 23:06:53.825 | INFO | } | 20260507 23:06:53.825 | INFO | Perf report for the last 5.1349 seconds: | 20260507 23:06:53.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:53.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260507 23:06:53.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:06:53.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:53.826 | INFO | Uploaded 36GiB to GPU in 2.33535s (15.4153GiB/s) | 20260507 23:06:53.826 | INFO | Downloaded 34GiB to GPU in 2.51185s (13.5358GiB/s) | 20260507 23:06:58.961 | INFO | Analyzing chunk frames = [3316224,3316736) | 20260507 23:06:58.961 | INFO | MemInfo: { | 20260507 23:06:58.961 | INFO | "gpu" : " 17 GB", | 20260507 23:06:58.961 | INFO | "huge" : "", | 20260507 23:06:58.961 | INFO | "huge_shared" : " 29 GB", | 20260507 23:06:58.961 | INFO | "malloc" : " 14 GB", | 20260507 23:06:58.961 | INFO | "pinned" : " 29 GB" | 20260507 23:06:58.961 | INFO | } | 20260507 23:06:58.961 | INFO | Perf report for the last 5.13597 seconds: | 20260507 23:06:58.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:06:58.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.961 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 23:06:58.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.962 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:06:58.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:06:58.962 | INFO | Uploaded 36GiB to GPU in 2.4643s (14.6086GiB/s) | 20260507 23:06:58.962 | INFO | Downloaded 34GiB to GPU in 2.44371s (13.9133GiB/s) | 20260507 23:07:04.102 | INFO | Analyzing chunk frames = [3316736,3317248) | 20260507 23:07:04.102 | INFO | MemInfo: { | 20260507 23:07:04.102 | INFO | "gpu" : " 17 GB", | 20260507 23:07:04.102 | INFO | "huge" : "", | 20260507 23:07:04.102 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:04.102 | INFO | "malloc" : " 14 GB", | 20260507 23:07:04.102 | INFO | "pinned" : " 29 GB" | 20260507 23:07:04.102 | INFO | } | 20260507 23:07:04.102 | INFO | Perf report for the last 5.14055 seconds: | 20260507 23:07:04.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:04.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:07:04.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 23:07:04.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:04.102 | INFO | Uploaded 36GiB to GPU in 2.51492s (14.3146GiB/s) | 20260507 23:07:04.102 | INFO | Downloaded 34GiB to GPU in 2.47451s (13.7401GiB/s) | 20260507 23:07:09.243 | INFO | Analyzing chunk frames = [3317248,3317760) | 20260507 23:07:09.243 | INFO | MemInfo: { | 20260507 23:07:09.243 | INFO | "gpu" : " 17 GB", | 20260507 23:07:09.243 | INFO | "huge" : "", | 20260507 23:07:09.243 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:09.243 | INFO | "malloc" : " 14 GB", | 20260507 23:07:09.243 | INFO | "pinned" : " 29 GB" | 20260507 23:07:09.243 | INFO | } | 20260507 23:07:09.243 | INFO | Perf report for the last 5.14108 seconds: | 20260507 23:07:09.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:09.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:07:09.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:07:09.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:09.243 | INFO | Uploaded 36GiB to GPU in 2.48287s (14.4994GiB/s) | 20260507 23:07:09.243 | INFO | Downloaded 34GiB to GPU in 2.46073s (13.817GiB/s) | 20260507 23:07:09.243 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:07:09.243 | INFO | Total mem usage: 16.7 GiB | 20260507 23:07:09.243 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:07:09.243 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.2%) | 20260507 23:07:09.243 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:07:09.243 | INFO | High Water Breakdown: | 20260507 23:07:09.243 | INFO | 60.1% : 9.02 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:07:09.243 | INFO | 39.8% : 5.97 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:07:09.243 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:07:09.243 | INFO | ------------------------------------------ | 20260507 23:07:09.243 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:07:09.243 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:07:09.243 | INFO | Total mem usage: 26.5 GiB | 20260507 23:07:09.243 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:07:09.244 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 23:07:09.244 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:07:09.244 | INFO | High Water Breakdown: | 20260507 23:07:09.244 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:07:09.244 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:07:09.244 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:07:09.244 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:07:09.244 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:07:09.244 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:07:09.244 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:07:09.244 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:07:09.244 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:07:09.244 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:07:09.244 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:07:09.244 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:07:09.244 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:07:09.244 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:07:09.244 | INFO | ------------------------------------------ | 20260507 23:07:09.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:07:09.244 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:07:09.244 | INFO | Total mem usage: 27.8 GiB | 20260507 23:07:09.244 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:07:09.244 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:07:09.244 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:07:09.244 | INFO | High Water Breakdown: | 20260507 23:07:09.244 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:07:09.244 | INFO | ------------------------------------------ | 20260507 23:07:09.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:07:09.244 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:07:09.244 | INFO | Total mem usage: 15.7 GiB | 20260507 23:07:09.244 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:07:09.244 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:07:09.244 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:07:09.244 | INFO | High Water Breakdown: | 20260507 23:07:09.244 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:07:09.244 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:07:09.244 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:07:09.244 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:07:09.244 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:07:09.244 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:07:09.244 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:07:09.244 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:07:09.244 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:07:09.244 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:07:09.244 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:07:09.244 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:07:09.244 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:07:09.244 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:07:09.244 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:07:09.244 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:07:09.244 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:07:09.244 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:07:09.244 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:07:09.244 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:07:09.244 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:07:09.244 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:07:09.244 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:07:09.244 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:07:09.244 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:07:09.244 | INFO | ------------------------------------------ | 20260507 23:07:09.244 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:07:09.244 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:07:09.244 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:07:09.244 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:07:09.244 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.66e+01, 1.41e+01 / 1.67e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.13e-02 / 5.37e-03 | 20260507 23:07:09.244 | INFO | PerfReport: 12.12%, Baseline -- 3.98e+00 / 4.45e+00, 5.86e-01 / 2.17e-01 | 20260507 23:07:09.244 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.06e-01 / 1.17e-01 | 20260507 23:07:09.244 | INFO | PerfReport: 15.93%, DME -- 4.19e+01 / 4.57e+01, 4.07e+00 / 1.58e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.85e+01 / 2.04e+01, 1.70e+00 / 1.51e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.68e+00, 4.32e-01 / 2.49e-01 | 20260507 23:07:09.244 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.31e+00, 2.84e-01 / 1.30e-01 | 20260507 23:07:09.244 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.50e-01 / 3.78e-01, 5.61e-02 / 2.42e-02 | 20260507 23:07:09.244 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:07:09.244 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:07:09.244 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:07:09.244 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 24.55%, Upload -- 2.62e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 24.33%, Download -- 2.60e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 30.78%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 2.11%, LockUpload -- 2.26e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 15.67%, LockCompute -- 1.67e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 23:07:09.244 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260507 23:07:09.340 | INFO | Starting a round of Baz IO | 20260507 23:07:12.804 | INFO | Populating RTMetrics report | 20260507 23:07:12.804 | INFO | RTMetrics report populated | 20260507 23:07:12.804 | INFO | Opening tmp RTMetrics json file | 20260507 23:07:12.917 | INFO | Serializing RTMetrics to json | 20260507 23:07:13.018 | INFO | Serialization done, swapping tmp file now... | 20260507 23:07:13.023 | INFO | RTMetrics done | 20260507 23:07:13.036 | INFO | Finished a round of Baz IO: Wrote 1.723 GiB in 3.695 seconds | 20260507 23:07:13.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.77% / 0.01% / 57.23% | 20260507 23:07:13.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26686% / 0% / 5.96046e-05% | 20260507 23:07:13.036 | INFO | Baz IO overall: | 20260507 23:07:13.036 | INFO | BytesWritten : 1511059308113 | 20260507 23:07:13.036 | INFO | NumEvents : 731875298753 | 20260507 23:07:14.388 | INFO | Analyzing chunk frames = [3317760,3318272) | 20260507 23:07:14.388 | INFO | MemInfo: { | 20260507 23:07:14.388 | INFO | "gpu" : " 17 GB", | 20260507 23:07:14.388 | INFO | "huge" : "", | 20260507 23:07:14.388 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:14.388 | INFO | "malloc" : " 14 GB", | 20260507 23:07:14.388 | INFO | "pinned" : " 29 GB" | 20260507 23:07:14.388 | INFO | } | 20260507 23:07:14.388 | INFO | Perf report for the last 5.14527 seconds: | 20260507 23:07:14.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:14.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:14.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:14.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:14.389 | INFO | ANALYSIS : 68.5%, 3.0, 106.9ms, 0 | 20260507 23:07:14.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:14.389 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260507 23:07:14.389 | INFO | PRE_HQ : 65.7%, 1.7, 57.2ms, 0 | 20260507 23:07:14.389 | INFO | BAZWRITER : 71.1%, 4.0, 144.7ms, 0 | 20260507 23:07:14.389 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 2.0ms, 0 | 20260507 23:07:14.389 | INFO | Uploaded 36GiB to GPU in 2.46833s (14.5848GiB/s) | 20260507 23:07:14.389 | INFO | Downloaded 38GiB to GPU in 2.70418s (14.0523GiB/s) | 20260507 23:07:19.522 | INFO | Analyzing chunk frames = [3318272,3318784) | 20260507 23:07:19.522 | INFO | MemInfo: { | 20260507 23:07:19.522 | INFO | "gpu" : " 17 GB", | 20260507 23:07:19.522 | INFO | "huge" : "", | 20260507 23:07:19.522 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:19.522 | INFO | "malloc" : " 14 GB", | 20260507 23:07:19.522 | INFO | "pinned" : " 29 GB" | 20260507 23:07:19.522 | INFO | } | 20260507 23:07:19.522 | INFO | Perf report for the last 5.13392 seconds: | 20260507 23:07:19.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:19.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.522 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 23:07:19.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.522 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:07:19.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:19.523 | INFO | Uploaded 36GiB to GPU in 2.57046s (14.0053GiB/s) | 20260507 23:07:19.523 | INFO | Downloaded 34GiB to GPU in 2.48608s (13.6762GiB/s) | 20260507 23:07:24.658 | INFO | Analyzing chunk frames = [3318784,3319296) | 20260507 23:07:24.658 | INFO | MemInfo: { | 20260507 23:07:24.658 | INFO | "gpu" : " 17 GB", | 20260507 23:07:24.658 | INFO | "huge" : "", | 20260507 23:07:24.658 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:24.658 | INFO | "malloc" : " 14 GB", | 20260507 23:07:24.658 | INFO | "pinned" : " 29 GB" | 20260507 23:07:24.658 | INFO | } | 20260507 23:07:24.658 | INFO | Perf report for the last 5.13542 seconds: | 20260507 23:07:24.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:24.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:07:24.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:07:24.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:24.658 | INFO | Uploaded 36GiB to GPU in 2.50084s (14.3951GiB/s) | 20260507 23:07:24.658 | INFO | Downloaded 34GiB to GPU in 2.48721s (13.6699GiB/s) | 20260507 23:07:29.800 | INFO | Analyzing chunk frames = [3319296,3319808) | 20260507 23:07:29.800 | INFO | MemInfo: { | 20260507 23:07:29.800 | INFO | "gpu" : " 17 GB", | 20260507 23:07:29.800 | INFO | "huge" : "", | 20260507 23:07:29.800 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:29.800 | INFO | "malloc" : " 14 GB", | 20260507 23:07:29.800 | INFO | "pinned" : " 29 GB" | 20260507 23:07:29.800 | INFO | } | 20260507 23:07:29.800 | INFO | Perf report for the last 5.14212 seconds: | 20260507 23:07:29.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:29.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 23:07:29.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:07:29.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:29.800 | INFO | Uploaded 36GiB to GPU in 2.45134s (14.6858GiB/s) | 20260507 23:07:29.800 | INFO | Downloaded 34GiB to GPU in 2.38812s (14.2372GiB/s) | 20260507 23:07:34.937 | INFO | Analyzing chunk frames = [3319808,3320320) | 20260507 23:07:34.937 | INFO | MemInfo: { | 20260507 23:07:34.937 | INFO | "gpu" : " 17 GB", | 20260507 23:07:34.937 | INFO | "huge" : "", | 20260507 23:07:34.937 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:34.937 | INFO | "malloc" : " 14 GB", | 20260507 23:07:34.937 | INFO | "pinned" : " 29 GB" | 20260507 23:07:34.937 | INFO | } | 20260507 23:07:34.937 | INFO | Perf report for the last 5.13694 seconds: | 20260507 23:07:34.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:34.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260507 23:07:34.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:07:34.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:34.937 | INFO | Uploaded 36GiB to GPU in 2.41451s (14.9099GiB/s) | 20260507 23:07:34.937 | INFO | Downloaded 34GiB to GPU in 2.49587s (13.6225GiB/s) | 20260507 23:07:40.079 | INFO | Analyzing chunk frames = [3320320,3320832) | 20260507 23:07:40.079 | INFO | MemInfo: { | 20260507 23:07:40.079 | INFO | "gpu" : " 17 GB", | 20260507 23:07:40.079 | INFO | "huge" : "", | 20260507 23:07:40.079 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:40.079 | INFO | "malloc" : " 14 GB", | 20260507 23:07:40.079 | INFO | "pinned" : " 29 GB" | 20260507 23:07:40.079 | INFO | } | 20260507 23:07:40.079 | INFO | Perf report for the last 5.14255 seconds: | 20260507 23:07:40.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:40.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.079 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:07:40.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.080 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 23:07:40.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:40.080 | INFO | Uploaded 36GiB to GPU in 2.50661s (14.3621GiB/s) | 20260507 23:07:40.080 | INFO | Downloaded 34GiB to GPU in 2.43438s (13.9666GiB/s) | 20260507 23:07:45.216 | INFO | Analyzing chunk frames = [3320832,3321344) | 20260507 23:07:45.216 | INFO | MemInfo: { | 20260507 23:07:45.216 | INFO | "gpu" : " 17 GB", | 20260507 23:07:45.216 | INFO | "huge" : "", | 20260507 23:07:45.216 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:45.216 | INFO | "malloc" : " 14 GB", | 20260507 23:07:45.216 | INFO | "pinned" : " 29 GB" | 20260507 23:07:45.216 | INFO | } | 20260507 23:07:45.216 | INFO | Perf report for the last 5.13651 seconds: | 20260507 23:07:45.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:45.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.216 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:07:45.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.216 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:07:45.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:45.217 | INFO | Uploaded 36GiB to GPU in 2.52919s (14.2338GiB/s) | 20260507 23:07:45.217 | INFO | Downloaded 34GiB to GPU in 2.44132s (13.9269GiB/s) | 20260507 23:07:50.357 | INFO | Analyzing chunk frames = [3321344,3321856) | 20260507 23:07:50.357 | INFO | MemInfo: { | 20260507 23:07:50.357 | INFO | "gpu" : " 17 GB", | 20260507 23:07:50.357 | INFO | "huge" : "", | 20260507 23:07:50.357 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:50.357 | INFO | "malloc" : " 14 GB", | 20260507 23:07:50.357 | INFO | "pinned" : " 29 GB" | 20260507 23:07:50.357 | INFO | } | 20260507 23:07:50.357 | INFO | Perf report for the last 5.14147 seconds: | 20260507 23:07:50.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:50.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.357 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 23:07:50.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.358 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:07:50.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:50.358 | INFO | Uploaded 36GiB to GPU in 2.52389s (14.2637GiB/s) | 20260507 23:07:50.358 | INFO | Downloaded 34GiB to GPU in 2.44071s (13.9304GiB/s) | 20260507 23:07:50.454 | INFO | Starting a round of Baz IO | 20260507 23:07:53.930 | INFO | Populating RTMetrics report | 20260507 23:07:53.931 | INFO | RTMetrics report populated | 20260507 23:07:53.931 | INFO | Opening tmp RTMetrics json file | 20260507 23:07:53.957 | INFO | Serializing RTMetrics to json | 20260507 23:07:53.962 | INFO | Serialization done, swapping tmp file now... | 20260507 23:07:53.965 | INFO | RTMetrics done | 20260507 23:07:54.254 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.799 seconds | 20260507 23:07:54.254 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.76% / 0.01% / 57.23% | 20260507 23:07:54.254 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25977% / 0% / 6.75519e-05% | 20260507 23:07:54.254 | INFO | Baz IO overall: | 20260507 23:07:54.254 | INFO | BytesWritten : 1512905660241 | 20260507 23:07:54.254 | INFO | NumEvents : 732765285458 | 20260507 23:07:55.495 | INFO | Analyzing chunk frames = [3321856,3322368) | 20260507 23:07:55.495 | INFO | MemInfo: { | 20260507 23:07:55.495 | INFO | "gpu" : " 17 GB", | 20260507 23:07:55.495 | INFO | "huge" : "", | 20260507 23:07:55.495 | INFO | "huge_shared" : " 29 GB", | 20260507 23:07:55.495 | INFO | "malloc" : " 14 GB", | 20260507 23:07:55.495 | INFO | "pinned" : " 29 GB" | 20260507 23:07:55.495 | INFO | } | 20260507 23:07:55.495 | INFO | Perf report for the last 5.13788 seconds: | 20260507 23:07:55.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:07:55.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:55.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:55.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:55.495 | INFO | ANALYSIS : 68.8%, 3.0, 107.2ms, 0 | 20260507 23:07:55.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:07:55.495 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 23:07:55.495 | INFO | PRE_HQ : 65.6%, 1.7, 57.2ms, 0 | 20260507 23:07:55.495 | INFO | BAZWRITER : 73.5%, 4.0, 150.7ms, 0 | 20260507 23:07:55.495 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 23:07:55.496 | INFO | Uploaded 36GiB to GPU in 2.52723s (14.2448GiB/s) | 20260507 23:07:55.496 | INFO | Downloaded 38GiB to GPU in 2.69301s (14.1106GiB/s) | 20260507 23:08:00.642 | INFO | Analyzing chunk frames = [3322368,3322880) | 20260507 23:08:00.642 | INFO | MemInfo: { | 20260507 23:08:00.642 | INFO | "gpu" : " 17 GB", | 20260507 23:08:00.642 | INFO | "huge" : "", | 20260507 23:08:00.642 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:00.642 | INFO | "malloc" : " 14 GB", | 20260507 23:08:00.642 | INFO | "pinned" : " 29 GB" | 20260507 23:08:00.642 | INFO | } | 20260507 23:08:00.642 | INFO | Perf report for the last 5.14659 seconds: | 20260507 23:08:00.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:00.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 23:08:00.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:08:00.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:00.642 | INFO | Uploaded 36GiB to GPU in 2.58781s (13.9114GiB/s) | 20260507 23:08:00.642 | INFO | Downloaded 34GiB to GPU in 2.46849s (13.7736GiB/s) | 20260507 23:08:05.776 | INFO | Analyzing chunk frames = [3322880,3323392) | 20260507 23:08:05.776 | INFO | MemInfo: { | 20260507 23:08:05.776 | INFO | "gpu" : " 17 GB", | 20260507 23:08:05.776 | INFO | "huge" : "", | 20260507 23:08:05.776 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:05.776 | INFO | "malloc" : " 14 GB", | 20260507 23:08:05.776 | INFO | "pinned" : " 29 GB" | 20260507 23:08:05.776 | INFO | } | 20260507 23:08:05.776 | INFO | Perf report for the last 5.13407 seconds: | 20260507 23:08:05.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:05.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 23:08:05.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:08:05.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:05.776 | INFO | Uploaded 36GiB to GPU in 2.53369s (14.2086GiB/s) | 20260507 23:08:05.776 | INFO | Downloaded 34GiB to GPU in 2.47652s (13.7289GiB/s) | 20260507 23:08:10.913 | INFO | Analyzing chunk frames = [3323392,3323904) | 20260507 23:08:10.913 | INFO | MemInfo: { | 20260507 23:08:10.913 | INFO | "gpu" : " 17 GB", | 20260507 23:08:10.913 | INFO | "huge" : "", | 20260507 23:08:10.913 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:10.913 | INFO | "malloc" : " 14 GB", | 20260507 23:08:10.913 | INFO | "pinned" : " 29 GB" | 20260507 23:08:10.913 | INFO | } | 20260507 23:08:10.914 | INFO | Perf report for the last 5.13751 seconds: | 20260507 23:08:10.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:10.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 23:08:10.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:08:10.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:10.914 | INFO | Uploaded 36GiB to GPU in 2.47923s (14.5207GiB/s) | 20260507 23:08:10.914 | INFO | Downloaded 34GiB to GPU in 2.38462s (14.258GiB/s) | 20260507 23:08:16.056 | INFO | Analyzing chunk frames = [3323904,3324416) | 20260507 23:08:16.056 | INFO | MemInfo: { | 20260507 23:08:16.056 | INFO | "gpu" : " 17 GB", | 20260507 23:08:16.056 | INFO | "huge" : "", | 20260507 23:08:16.056 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:16.056 | INFO | "malloc" : " 14 GB", | 20260507 23:08:16.056 | INFO | "pinned" : " 29 GB" | 20260507 23:08:16.056 | INFO | } | 20260507 23:08:16.057 | INFO | Perf report for the last 5.14301 seconds: | 20260507 23:08:16.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:16.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 23:08:16.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:08:16.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:16.057 | INFO | Uploaded 36GiB to GPU in 2.37221s (15.1757GiB/s) | 20260507 23:08:16.057 | INFO | Downloaded 34GiB to GPU in 2.49662s (13.6184GiB/s) | 20260507 23:08:21.196 | INFO | Analyzing chunk frames = [3324416,3324928) | 20260507 23:08:21.196 | INFO | MemInfo: { | 20260507 23:08:21.196 | INFO | "gpu" : " 17 GB", | 20260507 23:08:21.196 | INFO | "huge" : "", | 20260507 23:08:21.196 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:21.196 | INFO | "malloc" : " 14 GB", | 20260507 23:08:21.196 | INFO | "pinned" : " 29 GB" | 20260507 23:08:21.196 | INFO | } | 20260507 23:08:21.196 | INFO | Perf report for the last 5.13981 seconds: | 20260507 23:08:21.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:21.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.196 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 23:08:21.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.196 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:08:21.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:21.197 | INFO | Uploaded 36GiB to GPU in 2.39702s (15.0187GiB/s) | 20260507 23:08:21.197 | INFO | Downloaded 34GiB to GPU in 2.39151s (14.217GiB/s) | 20260507 23:08:26.334 | INFO | Analyzing chunk frames = [3324928,3325440) | 20260507 23:08:26.334 | INFO | MemInfo: { | 20260507 23:08:26.334 | INFO | "gpu" : " 17 GB", | 20260507 23:08:26.334 | INFO | "huge" : "", | 20260507 23:08:26.334 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:26.334 | INFO | "malloc" : " 14 GB", | 20260507 23:08:26.334 | INFO | "pinned" : " 29 GB" | 20260507 23:08:26.334 | INFO | } | 20260507 23:08:26.334 | INFO | Perf report for the last 5.13745 seconds: | 20260507 23:08:26.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:26.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 23:08:26.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:08:26.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:26.334 | INFO | Uploaded 36GiB to GPU in 2.55703s (14.0788GiB/s) | 20260507 23:08:26.334 | INFO | Downloaded 34GiB to GPU in 2.46676s (13.7833GiB/s) | 20260507 23:08:31.474 | INFO | Analyzing chunk frames = [3325440,3325952) | 20260507 23:08:31.474 | INFO | MemInfo: { | 20260507 23:08:31.474 | INFO | "gpu" : " 17 GB", | 20260507 23:08:31.474 | INFO | "huge" : "", | 20260507 23:08:31.474 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:31.474 | INFO | "malloc" : " 14 GB", | 20260507 23:08:31.474 | INFO | "pinned" : " 29 GB" | 20260507 23:08:31.474 | INFO | } | 20260507 23:08:31.474 | INFO | Perf report for the last 5.14027 seconds: | 20260507 23:08:31.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:31.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:08:31.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:08:31.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:31.474 | INFO | Uploaded 36GiB to GPU in 2.50752s (14.3568GiB/s) | 20260507 23:08:31.474 | INFO | Downloaded 34GiB to GPU in 2.44522s (13.9047GiB/s) | 20260507 23:08:31.475 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:08:31.475 | INFO | Total mem usage: 16.7 GiB | 20260507 23:08:31.475 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:08:31.475 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.4%) | 20260507 23:08:31.475 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:08:31.475 | INFO | High Water Breakdown: | 20260507 23:08:31.475 | INFO | 60.2% : 9.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:08:31.475 | INFO | 39.6% : 5.99 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:08:31.475 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:08:31.475 | INFO | ------------------------------------------ | 20260507 23:08:31.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:08:31.475 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:08:31.475 | INFO | Total mem usage: 26.5 GiB | 20260507 23:08:31.475 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:08:31.475 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:08:31.475 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:08:31.475 | INFO | High Water Breakdown: | 20260507 23:08:31.475 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:08:31.475 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:08:31.475 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:08:31.475 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:08:31.475 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:08:31.475 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:08:31.475 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:08:31.475 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:08:31.475 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:08:31.475 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:08:31.475 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:08:31.475 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:08:31.475 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:08:31.475 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:08:31.475 | INFO | ------------------------------------------ | 20260507 23:08:31.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:08:31.475 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:08:31.475 | INFO | Total mem usage: 27.8 GiB | 20260507 23:08:31.475 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:08:31.475 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:08:31.475 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:08:31.475 | INFO | High Water Breakdown: | 20260507 23:08:31.475 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:08:31.475 | INFO | ------------------------------------------ | 20260507 23:08:31.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:08:31.475 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:08:31.475 | INFO | Total mem usage: 15.7 GiB | 20260507 23:08:31.475 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:08:31.475 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:08:31.475 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 23:08:31.475 | INFO | High Water Breakdown: | 20260507 23:08:31.475 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:08:31.475 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:08:31.475 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:08:31.475 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:08:31.475 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:08:31.475 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:08:31.475 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:08:31.475 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:08:31.475 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:08:31.475 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:08:31.475 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:08:31.475 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:08:31.475 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:08:31.475 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:08:31.475 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:08:31.475 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:08:31.475 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:08:31.475 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:08:31.475 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:08:31.475 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:08:31.475 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:08:31.475 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:08:31.475 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:08:31.475 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:08:31.475 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:08:31.475 | INFO | ------------------------------------------ | 20260507 23:08:31.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:08:31.475 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:08:31.475 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:08:31.475 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:08:31.475 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.42e+01 / 1.54e+00 | 20260507 23:08:31.475 | INFO | PerfReport: 0.29%, AggregateStats -- 9.66e-02 / 1.01e-01, 1.09e-02 / 9.01e-03 | 20260507 23:08:31.475 | INFO | PerfReport: 12.14%, Baseline -- 3.99e+00 / 4.48e+00, 5.92e-01 / 1.89e-01 | 20260507 23:08:31.475 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.26e-01 | 20260507 23:08:31.475 | INFO | PerfReport: 16.01%, DME -- 4.21e+01 / 4.59e+01, 4.09e+00 / 1.48e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.71e+00 / 1.32e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.46e+00 / 2.72e+00, 4.27e-01 / 2.33e-01 | 20260507 23:08:31.476 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.38e+00, 2.82e-01 / 1.36e-01 | 20260507 23:08:31.476 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.49e-01 / 3.89e-01, 5.67e-02 / 2.86e-02 | 20260507 23:08:31.476 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:08:31.476 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:08:31.476 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:08:31.476 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 24.34%, Upload -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 24.20%, Download -- 2.58e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 30.76%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 2.16%, LockUpload -- 2.30e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 2.59%, LockDownload -- 2.76e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 15.94%, LockCompute -- 1.70e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:08:31.476 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260507 23:08:31.571 | INFO | Starting a round of Baz IO | 20260507 23:08:35.038 | INFO | Populating RTMetrics report | 20260507 23:08:35.039 | INFO | RTMetrics report populated | 20260507 23:08:35.039 | INFO | Opening tmp RTMetrics json file | 20260507 23:08:35.070 | INFO | Serializing RTMetrics to json | 20260507 23:08:35.095 | INFO | Serialization done, swapping tmp file now... | 20260507 23:08:35.102 | INFO | RTMetrics done | 20260507 23:08:35.156 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.584 seconds | 20260507 23:08:35.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.76% / 0.01% / 57.24% | 20260507 23:08:35.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25986% / 1.19209e-05% / 4.76837e-05% | 20260507 23:08:35.156 | INFO | Baz IO overall: | 20260507 23:08:35.156 | INFO | BytesWritten : 1514752974544 | 20260507 23:08:35.156 | INFO | NumEvents : 733655839191 | 20260507 23:08:36.610 | INFO | Analyzing chunk frames = [3325952,3326464) | 20260507 23:08:36.610 | INFO | MemInfo: { | 20260507 23:08:36.610 | INFO | "gpu" : " 17 GB", | 20260507 23:08:36.610 | INFO | "huge" : "", | 20260507 23:08:36.610 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:36.610 | INFO | "malloc" : " 14 GB", | 20260507 23:08:36.610 | INFO | "pinned" : " 29 GB" | 20260507 23:08:36.610 | INFO | } | 20260507 23:08:36.610 | INFO | Perf report for the last 5.13622 seconds: | 20260507 23:08:36.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:36.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:36.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:36.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:36.610 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 23:08:36.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:36.611 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 23:08:36.611 | INFO | PRE_HQ : 65.5%, 1.7, 57.2ms, 0 | 20260507 23:08:36.611 | INFO | BAZWRITER : 69.4%, 4.1, 146.0ms, 0 | 20260507 23:08:36.611 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 23:08:36.611 | INFO | Uploaded 36GiB to GPU in 2.50711s (14.3592GiB/s) | 20260507 23:08:36.611 | INFO | Downloaded 38GiB to GPU in 2.68812s (14.1363GiB/s) | 20260507 23:08:41.751 | INFO | Analyzing chunk frames = [3326464,3326976) | 20260507 23:08:41.751 | INFO | MemInfo: { | 20260507 23:08:41.751 | INFO | "gpu" : " 17 GB", | 20260507 23:08:41.751 | INFO | "huge" : "", | 20260507 23:08:41.751 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:41.751 | INFO | "malloc" : " 14 GB", | 20260507 23:08:41.751 | INFO | "pinned" : " 29 GB" | 20260507 23:08:41.751 | INFO | } | 20260507 23:08:41.752 | INFO | Perf report for the last 5.14115 seconds: | 20260507 23:08:41.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:41.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:08:41.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:08:41.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:41.752 | INFO | Uploaded 36GiB to GPU in 2.43052s (14.8116GiB/s) | 20260507 23:08:41.752 | INFO | Downloaded 34GiB to GPU in 2.47547s (13.7348GiB/s) | 20260507 23:08:46.894 | INFO | Analyzing chunk frames = [3326976,3327488) | 20260507 23:08:46.894 | INFO | MemInfo: { | 20260507 23:08:46.894 | INFO | "gpu" : " 17 GB", | 20260507 23:08:46.894 | INFO | "huge" : "", | 20260507 23:08:46.894 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:46.894 | INFO | "malloc" : " 14 GB", | 20260507 23:08:46.894 | INFO | "pinned" : " 29 GB" | 20260507 23:08:46.894 | INFO | } | 20260507 23:08:46.894 | INFO | Perf report for the last 5.14293 seconds: | 20260507 23:08:46.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:46.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.895 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:08:46.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.895 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260507 23:08:46.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:46.895 | INFO | Uploaded 36GiB to GPU in 2.49015s (14.457GiB/s) | 20260507 23:08:46.895 | INFO | Downloaded 34GiB to GPU in 2.50043s (13.5977GiB/s) | 20260507 23:08:52.027 | INFO | Analyzing chunk frames = [3327488,3328000) | 20260507 23:08:52.027 | INFO | MemInfo: { | 20260507 23:08:52.027 | INFO | "gpu" : " 17 GB", | 20260507 23:08:52.027 | INFO | "huge" : "", | 20260507 23:08:52.027 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:52.027 | INFO | "malloc" : " 14 GB", | 20260507 23:08:52.027 | INFO | "pinned" : " 29 GB" | 20260507 23:08:52.027 | INFO | } | 20260507 23:08:52.027 | INFO | Perf report for the last 5.13242 seconds: | 20260507 23:08:52.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:52.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.027 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 23:08:52.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.027 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:08:52.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:52.028 | INFO | Uploaded 36GiB to GPU in 2.45409s (14.6694GiB/s) | 20260507 23:08:52.028 | INFO | Downloaded 34GiB to GPU in 2.50863s (13.5532GiB/s) | 20260507 23:08:57.166 | INFO | Analyzing chunk frames = [3328000,3328512) | 20260507 23:08:57.166 | INFO | MemInfo: { | 20260507 23:08:57.166 | INFO | "gpu" : " 17 GB", | 20260507 23:08:57.166 | INFO | "huge" : "", | 20260507 23:08:57.166 | INFO | "huge_shared" : " 29 GB", | 20260507 23:08:57.166 | INFO | "malloc" : " 14 GB", | 20260507 23:08:57.166 | INFO | "pinned" : " 29 GB" | 20260507 23:08:57.166 | INFO | } | 20260507 23:08:57.166 | INFO | Perf report for the last 5.13929 seconds: | 20260507 23:08:57.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:08:57.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.166 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 23:08:57.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.166 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:08:57.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:08:57.167 | INFO | Uploaded 36GiB to GPU in 2.3696s (15.1924GiB/s) | 20260507 23:08:57.167 | INFO | Downloaded 34GiB to GPU in 2.5031s (13.5831GiB/s) | 20260507 23:09:02.309 | INFO | Analyzing chunk frames = [3328512,3329024) | 20260507 23:09:02.309 | INFO | MemInfo: { | 20260507 23:09:02.309 | INFO | "gpu" : " 17 GB", | 20260507 23:09:02.309 | INFO | "huge" : "", | 20260507 23:09:02.309 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:02.309 | INFO | "malloc" : " 14 GB", | 20260507 23:09:02.309 | INFO | "pinned" : " 29 GB" | 20260507 23:09:02.309 | INFO | } | 20260507 23:09:02.309 | INFO | Perf report for the last 5.14285 seconds: | 20260507 23:09:02.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:02.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 23:09:02.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:09:02.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:02.309 | INFO | Uploaded 36GiB to GPU in 2.51449s (14.317GiB/s) | 20260507 23:09:02.309 | INFO | Downloaded 34GiB to GPU in 2.43483s (13.964GiB/s) | 20260507 23:09:07.453 | INFO | Analyzing chunk frames = [3329024,3329536) | 20260507 23:09:07.453 | INFO | MemInfo: { | 20260507 23:09:07.453 | INFO | "gpu" : " 17 GB", | 20260507 23:09:07.453 | INFO | "huge" : "", | 20260507 23:09:07.453 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:07.453 | INFO | "malloc" : " 14 GB", | 20260507 23:09:07.453 | INFO | "pinned" : " 29 GB" | 20260507 23:09:07.453 | INFO | } | 20260507 23:09:07.453 | INFO | Perf report for the last 5.14374 seconds: | 20260507 23:09:07.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:07.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:09:07.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:09:07.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:07.453 | INFO | Uploaded 36GiB to GPU in 2.55082s (14.1131GiB/s) | 20260507 23:09:07.453 | INFO | Downloaded 34GiB to GPU in 2.4708s (13.7607GiB/s) | 20260507 23:09:12.591 | INFO | Analyzing chunk frames = [3329536,3330048) | 20260507 23:09:12.591 | INFO | MemInfo: { | 20260507 23:09:12.591 | INFO | "gpu" : " 17 GB", | 20260507 23:09:12.591 | INFO | "huge" : "", | 20260507 23:09:12.591 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:12.591 | INFO | "malloc" : " 14 GB", | 20260507 23:09:12.591 | INFO | "pinned" : " 29 GB" | 20260507 23:09:12.591 | INFO | } | 20260507 23:09:12.591 | INFO | Perf report for the last 5.13845 seconds: | 20260507 23:09:12.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:12.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.591 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:09:12.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.591 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 23:09:12.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:12.592 | INFO | Uploaded 36GiB to GPU in 2.52343s (14.2663GiB/s) | 20260507 23:09:12.592 | INFO | Downloaded 34GiB to GPU in 2.45216s (13.8654GiB/s) | 20260507 23:09:12.689 | INFO | Starting a round of Baz IO | 20260507 23:09:16.159 | INFO | Populating RTMetrics report | 20260507 23:09:16.160 | INFO | RTMetrics report populated | 20260507 23:09:16.160 | INFO | Opening tmp RTMetrics json file | 20260507 23:09:16.171 | INFO | Serializing RTMetrics to json | 20260507 23:09:16.188 | INFO | Serialization done, swapping tmp file now... | 20260507 23:09:16.189 | INFO | RTMetrics done | 20260507 23:09:16.255 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.565 seconds | 20260507 23:09:16.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.75% / 0.01% / 57.24% | 20260507 23:09:16.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25643% / 0% / 4.76837e-05% | 20260507 23:09:16.255 | INFO | Baz IO overall: | 20260507 23:09:16.255 | INFO | BytesWritten : 1516600137193 | 20260507 23:09:16.255 | INFO | NumEvents : 734546501908 | 20260507 23:09:17.726 | INFO | Analyzing chunk frames = [3330048,3330560) | 20260507 23:09:17.726 | INFO | MemInfo: { | 20260507 23:09:17.726 | INFO | "gpu" : " 17 GB", | 20260507 23:09:17.726 | INFO | "huge" : "", | 20260507 23:09:17.726 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:17.726 | INFO | "malloc" : " 14 GB", | 20260507 23:09:17.726 | INFO | "pinned" : " 29 GB" | 20260507 23:09:17.726 | INFO | } | 20260507 23:09:17.726 | INFO | Perf report for the last 5.13506 seconds: | 20260507 23:09:17.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:17.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:17.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:17.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:17.726 | INFO | ANALYSIS : 68.5%, 3.0, 107.0ms, 0 | 20260507 23:09:17.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:17.727 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 23:09:17.727 | INFO | PRE_HQ : 65.7%, 1.7, 57.4ms, 0 | 20260507 23:09:17.727 | INFO | BAZWRITER : 69.1%, 3.7, 131.3ms, 0 | 20260507 23:09:17.727 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 23:09:17.727 | INFO | Uploaded 36GiB to GPU in 2.53574s (14.197GiB/s) | 20260507 23:09:17.727 | INFO | Downloaded 38GiB to GPU in 2.68569s (14.149GiB/s) | 20260507 23:09:22.872 | INFO | Analyzing chunk frames = [3330560,3331072) | 20260507 23:09:22.872 | INFO | MemInfo: { | 20260507 23:09:22.872 | INFO | "gpu" : " 17 GB", | 20260507 23:09:22.872 | INFO | "huge" : "", | 20260507 23:09:22.872 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:22.872 | INFO | "malloc" : " 14 GB", | 20260507 23:09:22.872 | INFO | "pinned" : " 29 GB" | 20260507 23:09:22.872 | INFO | } | 20260507 23:09:22.872 | INFO | Perf report for the last 5.14607 seconds: | 20260507 23:09:22.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:22.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.873 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260507 23:09:22.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.873 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:09:22.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:22.873 | INFO | Uploaded 36GiB to GPU in 2.5717s (13.9985GiB/s) | 20260507 23:09:22.873 | INFO | Downloaded 34GiB to GPU in 2.49034s (13.6528GiB/s) | 20260507 23:09:28.012 | INFO | Analyzing chunk frames = [3331072,3331584) | 20260507 23:09:28.012 | INFO | MemInfo: { | 20260507 23:09:28.012 | INFO | "gpu" : " 17 GB", | 20260507 23:09:28.012 | INFO | "huge" : "", | 20260507 23:09:28.012 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:28.012 | INFO | "malloc" : " 14 GB", | 20260507 23:09:28.012 | INFO | "pinned" : " 29 GB" | 20260507 23:09:28.012 | INFO | } | 20260507 23:09:28.012 | INFO | Perf report for the last 5.13955 seconds: | 20260507 23:09:28.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:28.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:09:28.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:09:28.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:28.012 | INFO | Uploaded 36GiB to GPU in 2.5099s (14.3432GiB/s) | 20260507 23:09:28.012 | INFO | Downloaded 34GiB to GPU in 2.48045s (13.7072GiB/s) | 20260507 23:09:33.151 | INFO | Analyzing chunk frames = [3331584,3332096) | 20260507 23:09:33.151 | INFO | MemInfo: { | 20260507 23:09:33.151 | INFO | "gpu" : " 17 GB", | 20260507 23:09:33.151 | INFO | "huge" : "", | 20260507 23:09:33.151 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:33.151 | INFO | "malloc" : " 14 GB", | 20260507 23:09:33.151 | INFO | "pinned" : " 29 GB" | 20260507 23:09:33.151 | INFO | } | 20260507 23:09:33.151 | INFO | Perf report for the last 5.13927 seconds: | 20260507 23:09:33.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:33.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.151 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:09:33.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.151 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:09:33.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:33.152 | INFO | Uploaded 36GiB to GPU in 2.50893s (14.3488GiB/s) | 20260507 23:09:33.152 | INFO | Downloaded 34GiB to GPU in 2.48916s (13.6593GiB/s) | 20260507 23:09:38.279 | INFO | Analyzing chunk frames = [3332096,3332608) | 20260507 23:09:38.279 | INFO | MemInfo: { | 20260507 23:09:38.279 | INFO | "gpu" : " 17 GB", | 20260507 23:09:38.279 | INFO | "huge" : "", | 20260507 23:09:38.279 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:38.280 | INFO | "malloc" : " 14 GB", | 20260507 23:09:38.280 | INFO | "pinned" : " 29 GB" | 20260507 23:09:38.280 | INFO | } | 20260507 23:09:38.280 | INFO | Perf report for the last 5.12832 seconds: | 20260507 23:09:38.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:38.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260507 23:09:38.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:09:38.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:38.280 | INFO | Uploaded 36GiB to GPU in 2.32452s (15.4871GiB/s) | 20260507 23:09:38.280 | INFO | Downloaded 34GiB to GPU in 2.48685s (13.6719GiB/s) | 20260507 23:09:43.420 | INFO | Analyzing chunk frames = [3332608,3333120) | 20260507 23:09:43.420 | INFO | MemInfo: { | 20260507 23:09:43.420 | INFO | "gpu" : " 17 GB", | 20260507 23:09:43.420 | INFO | "huge" : "", | 20260507 23:09:43.420 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:43.420 | INFO | "malloc" : " 14 GB", | 20260507 23:09:43.420 | INFO | "pinned" : " 29 GB" | 20260507 23:09:43.420 | INFO | } | 20260507 23:09:43.420 | INFO | Perf report for the last 5.14055 seconds: | 20260507 23:09:43.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:43.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.420 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:09:43.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.420 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:09:43.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:43.421 | INFO | Uploaded 36GiB to GPU in 2.5118s (14.3323GiB/s) | 20260507 23:09:43.421 | INFO | Downloaded 34GiB to GPU in 2.43045s (13.9892GiB/s) | 20260507 23:09:48.563 | INFO | Analyzing chunk frames = [3333120,3333632) | 20260507 23:09:48.563 | INFO | MemInfo: { | 20260507 23:09:48.563 | INFO | "gpu" : " 17 GB", | 20260507 23:09:48.563 | INFO | "huge" : "", | 20260507 23:09:48.563 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:48.563 | INFO | "malloc" : " 14 GB", | 20260507 23:09:48.563 | INFO | "pinned" : " 29 GB" | 20260507 23:09:48.563 | INFO | } | 20260507 23:09:48.563 | INFO | Perf report for the last 5.14318 seconds: | 20260507 23:09:48.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:48.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.563 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:09:48.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.563 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 23:09:48.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:48.564 | INFO | Uploaded 36GiB to GPU in 2.52593s (14.2522GiB/s) | 20260507 23:09:48.564 | INFO | Downloaded 34GiB to GPU in 2.47712s (13.7256GiB/s) | 20260507 23:09:53.706 | INFO | Analyzing chunk frames = [3333632,3334144) | 20260507 23:09:53.706 | INFO | MemInfo: { | 20260507 23:09:53.706 | INFO | "gpu" : " 17 GB", | 20260507 23:09:53.706 | INFO | "huge" : "", | 20260507 23:09:53.706 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:53.706 | INFO | "malloc" : " 14 GB", | 20260507 23:09:53.706 | INFO | "pinned" : " 29 GB" | 20260507 23:09:53.706 | INFO | } | 20260507 23:09:53.706 | INFO | Perf report for the last 5.14274 seconds: | 20260507 23:09:53.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:53.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.706 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:09:53.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.706 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 23:09:53.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:53.707 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:09:53.707 | INFO | Total mem usage: 16.7 GiB | 20260507 23:09:53.707 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:09:53.707 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.2%) | 20260507 23:09:53.707 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:09:53.707 | INFO | High Water Breakdown: | 20260507 23:09:53.707 | INFO | 60.1% : 8.93 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:09:53.707 | INFO | 39.8% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:09:53.707 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:09:53.707 | INFO | ------------------------------------------ | 20260507 23:09:53.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:09:53.707 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:09:53.707 | INFO | Total mem usage: 26.5 GiB | 20260507 23:09:53.707 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:09:53.707 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:09:53.707 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:09:53.707 | INFO | High Water Breakdown: | 20260507 23:09:53.707 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:09:53.707 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:09:53.707 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:09:53.707 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:09:53.707 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:09:53.707 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:09:53.707 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:09:53.707 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:09:53.707 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:09:53.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:09:53.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:09:53.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:09:53.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:09:53.707 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:09:53.707 | INFO | ------------------------------------------ | 20260507 23:09:53.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:09:53.707 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:09:53.707 | INFO | Total mem usage: 27.8 GiB | 20260507 23:09:53.707 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:09:53.707 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:09:53.707 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:09:53.707 | INFO | High Water Breakdown: | 20260507 23:09:53.707 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:09:53.707 | INFO | ------------------------------------------ | 20260507 23:09:53.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:09:53.707 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:09:53.707 | INFO | Total mem usage: 15.7 GiB | 20260507 23:09:53.707 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:09:53.707 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:09:53.707 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:09:53.707 | INFO | High Water Breakdown: | 20260507 23:09:53.707 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:09:53.707 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:09:53.707 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:09:53.707 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:09:53.707 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:09:53.707 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:09:53.707 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:09:53.707 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:09:53.707 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:09:53.707 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:09:53.707 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:09:53.707 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:09:53.707 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:09:53.707 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:09:53.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:09:53.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:09:53.707 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:09:53.707 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:09:53.707 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:09:53.707 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:09:53.707 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:09:53.707 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:09:53.707 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:09:53.707 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:09:53.707 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:09:53.707 | INFO | ------------------------------------------ | 20260507 23:09:53.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:09:53.707 | INFO | Uploaded 36GiB to GPU in 2.49177s (14.4476GiB/s) | 20260507 23:09:53.707 | INFO | Downloaded 34GiB to GPU in 2.46228s (13.8083GiB/s) | 20260507 23:09:53.707 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:09:53.708 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:09:53.708 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:09:53.708 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.41e+01 / 1.60e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.13e-02 / 6.23e-03 | 20260507 23:09:53.708 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.49e+00, 5.86e-01 / 1.75e-01 | 20260507 23:09:53.708 | INFO | PerfReport: 3.34%, Histogram -- 1.10e+00 / 1.48e+00, 2.06e-01 / 1.18e-01 | 20260507 23:09:53.708 | INFO | PerfReport: 16.12%, DME -- 4.23e+01 / 4.62e+01, 4.04e+00 / 1.84e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 56.01%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.59e+00 / 1.58e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.48e+00 / 2.77e+00, 4.32e-01 / 1.86e-01 | 20260507 23:09:53.708 | INFO | PerfReport: 3.46%, Metrics -- 1.14e+00 / 1.39e+00, 2.84e-01 / 1.59e-01 | 20260507 23:09:53.708 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.49e-01 / 3.88e-01, 5.65e-02 / 2.41e-02 | 20260507 23:09:53.708 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:09:53.708 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:09:53.708 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:09:53.708 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 24.25%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 24.39%, Download -- 2.61e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 30.69%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 2.01%, LockUpload -- 2.15e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 2.92%, LockDownload -- 3.12e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 15.72%, LockCompute -- 1.68e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 23:09:53.708 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 23:09:53.820 | INFO | Starting a round of Baz IO | 20260507 23:09:57.300 | INFO | Populating RTMetrics report | 20260507 23:09:57.300 | INFO | RTMetrics report populated | 20260507 23:09:57.300 | INFO | Opening tmp RTMetrics json file | 20260507 23:09:57.321 | INFO | Serializing RTMetrics to json | 20260507 23:09:57.332 | INFO | Serialization done, swapping tmp file now... | 20260507 23:09:57.349 | INFO | RTMetrics done | 20260507 23:09:57.396 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.576 seconds | 20260507 23:09:57.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.75% / 0.01% / 57.25% | 20260507 23:09:57.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25608% / 1.19209e-05% / 7.94729e-05% | 20260507 23:09:57.397 | INFO | Baz IO overall: | 20260507 23:09:57.397 | INFO | BytesWritten : 1518447825182 | 20260507 23:09:57.397 | INFO | NumEvents : 735437581517 | 20260507 23:09:58.846 | INFO | Analyzing chunk frames = [3334144,3334656) | 20260507 23:09:58.846 | INFO | MemInfo: { | 20260507 23:09:58.846 | INFO | "gpu" : " 17 GB", | 20260507 23:09:58.846 | INFO | "huge" : "", | 20260507 23:09:58.846 | INFO | "huge_shared" : " 29 GB", | 20260507 23:09:58.846 | INFO | "malloc" : " 14 GB", | 20260507 23:09:58.846 | INFO | "pinned" : " 29 GB" | 20260507 23:09:58.846 | INFO | } | 20260507 23:09:58.847 | INFO | Perf report for the last 5.14036 seconds: | 20260507 23:09:58.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:09:58.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:58.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:58.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:58.847 | INFO | ANALYSIS : 69.0%, 2.9, 104.9ms, 0 | 20260507 23:09:58.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:09:58.847 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260507 23:09:58.847 | INFO | PRE_HQ : 68.1%, 1.8, 63.3ms, 0 | 20260507 23:09:58.847 | INFO | BAZWRITER : 69.2%, 3.8, 136.3ms, 0 | 20260507 23:09:58.847 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 23:09:58.847 | INFO | Uploaded 36GiB to GPU in 2.58558s (13.9234GiB/s) | 20260507 23:09:58.847 | INFO | Downloaded 38GiB to GPU in 2.71371s (14.003GiB/s) | 20260507 23:10:03.980 | INFO | Analyzing chunk frames = [3334656,3335168) | 20260507 23:10:03.980 | INFO | MemInfo: { | 20260507 23:10:03.980 | INFO | "gpu" : " 17 GB", | 20260507 23:10:03.980 | INFO | "huge" : "", | 20260507 23:10:03.980 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:03.980 | INFO | "malloc" : " 14 GB", | 20260507 23:10:03.980 | INFO | "pinned" : " 29 GB" | 20260507 23:10:03.980 | INFO | } | 20260507 23:10:03.980 | INFO | Perf report for the last 5.13335 seconds: | 20260507 23:10:03.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:03.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 23:10:03.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:10:03.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:03.980 | INFO | Uploaded 36GiB to GPU in 2.51477s (14.3154GiB/s) | 20260507 23:10:03.980 | INFO | Downloaded 34GiB to GPU in 2.51201s (13.535GiB/s) | 20260507 23:10:09.122 | INFO | Analyzing chunk frames = [3335168,3335680) | 20260507 23:10:09.122 | INFO | MemInfo: { | 20260507 23:10:09.122 | INFO | "gpu" : " 17 GB", | 20260507 23:10:09.122 | INFO | "huge" : "", | 20260507 23:10:09.122 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:09.122 | INFO | "malloc" : " 14 GB", | 20260507 23:10:09.122 | INFO | "pinned" : " 29 GB" | 20260507 23:10:09.122 | INFO | } | 20260507 23:10:09.122 | INFO | Perf report for the last 5.14201 seconds: | 20260507 23:10:09.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:09.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 23:10:09.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:10:09.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:09.122 | INFO | Uploaded 36GiB to GPU in 2.43089s (14.8094GiB/s) | 20260507 23:10:09.122 | INFO | Downloaded 34GiB to GPU in 2.50148s (13.5919GiB/s) | 20260507 23:10:14.260 | INFO | Analyzing chunk frames = [3335680,3336192) | 20260507 23:10:14.260 | INFO | MemInfo: { | 20260507 23:10:14.260 | INFO | "gpu" : " 17 GB", | 20260507 23:10:14.260 | INFO | "huge" : "", | 20260507 23:10:14.260 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:14.260 | INFO | "malloc" : " 14 GB", | 20260507 23:10:14.260 | INFO | "pinned" : " 29 GB" | 20260507 23:10:14.260 | INFO | } | 20260507 23:10:14.260 | INFO | Perf report for the last 5.13799 seconds: | 20260507 23:10:14.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:14.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:10:14.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:10:14.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:14.260 | INFO | Uploaded 36GiB to GPU in 2.49991s (14.4005GiB/s) | 20260507 23:10:14.260 | INFO | Downloaded 34GiB to GPU in 2.51167s (13.5368GiB/s) | 20260507 23:10:19.399 | INFO | Analyzing chunk frames = [3336192,3336704) | 20260507 23:10:19.399 | INFO | MemInfo: { | 20260507 23:10:19.399 | INFO | "gpu" : " 17 GB", | 20260507 23:10:19.399 | INFO | "huge" : "", | 20260507 23:10:19.399 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:19.399 | INFO | "malloc" : " 14 GB", | 20260507 23:10:19.399 | INFO | "pinned" : " 29 GB" | 20260507 23:10:19.399 | INFO | } | 20260507 23:10:19.399 | INFO | Perf report for the last 5.13939 seconds: | 20260507 23:10:19.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:19.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 23:10:19.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:10:19.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:19.399 | INFO | Uploaded 36GiB to GPU in 2.34494s (15.3522GiB/s) | 20260507 23:10:19.400 | INFO | Downloaded 34GiB to GPU in 2.50497s (13.573GiB/s) | 20260507 23:10:24.541 | INFO | Analyzing chunk frames = [3336704,3337216) | 20260507 23:10:24.541 | INFO | MemInfo: { | 20260507 23:10:24.541 | INFO | "gpu" : " 17 GB", | 20260507 23:10:24.541 | INFO | "huge" : "", | 20260507 23:10:24.541 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:24.541 | INFO | "malloc" : " 14 GB", | 20260507 23:10:24.541 | INFO | "pinned" : " 29 GB" | 20260507 23:10:24.541 | INFO | } | 20260507 23:10:24.541 | INFO | Perf report for the last 5.14206 seconds: | 20260507 23:10:24.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:24.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.541 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:10:24.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.541 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:10:24.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:24.542 | INFO | Uploaded 36GiB to GPU in 2.49993s (14.4004GiB/s) | 20260507 23:10:24.542 | INFO | Downloaded 34GiB to GPU in 2.43467s (13.9649GiB/s) | 20260507 23:10:29.682 | INFO | Analyzing chunk frames = [3337216,3337728) | 20260507 23:10:29.682 | INFO | MemInfo: { | 20260507 23:10:29.682 | INFO | "gpu" : " 17 GB", | 20260507 23:10:29.682 | INFO | "huge" : "", | 20260507 23:10:29.682 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:29.682 | INFO | "malloc" : " 14 GB", | 20260507 23:10:29.682 | INFO | "pinned" : " 29 GB" | 20260507 23:10:29.682 | INFO | } | 20260507 23:10:29.682 | INFO | Perf report for the last 5.14088 seconds: | 20260507 23:10:29.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:29.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.682 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:10:29.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.682 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 23:10:29.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:29.683 | INFO | Uploaded 36GiB to GPU in 2.53187s (14.2187GiB/s) | 20260507 23:10:29.683 | INFO | Downloaded 34GiB to GPU in 2.46417s (13.7977GiB/s) | 20260507 23:10:34.821 | INFO | Analyzing chunk frames = [3337728,3338240) | 20260507 23:10:34.821 | INFO | MemInfo: { | 20260507 23:10:34.821 | INFO | "gpu" : " 17 GB", | 20260507 23:10:34.821 | INFO | "huge" : "", | 20260507 23:10:34.821 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:34.821 | INFO | "malloc" : " 14 GB", | 20260507 23:10:34.821 | INFO | "pinned" : " 29 GB" | 20260507 23:10:34.821 | INFO | } | 20260507 23:10:34.821 | INFO | Perf report for the last 5.13862 seconds: | 20260507 23:10:34.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:34.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:10:34.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:10:34.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:34.821 | INFO | Uploaded 36GiB to GPU in 2.50994s (14.343GiB/s) | 20260507 23:10:34.821 | INFO | Downloaded 34GiB to GPU in 2.44275s (13.9187GiB/s) | 20260507 23:10:34.920 | INFO | Starting a round of Baz IO | 20260507 23:10:38.380 | INFO | Populating RTMetrics report | 20260507 23:10:38.380 | INFO | RTMetrics report populated | 20260507 23:10:38.380 | INFO | Opening tmp RTMetrics json file | 20260507 23:10:38.466 | INFO | Serializing RTMetrics to json | 20260507 23:10:38.469 | INFO | Serialization done, swapping tmp file now... | 20260507 23:10:38.473 | INFO | RTMetrics done | 20260507 23:10:38.589 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.668 seconds | 20260507 23:10:38.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.74% / 0.01% / 57.25% | 20260507 23:10:38.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24796% / 1.58946e-05% / 8.34465e-05% | 20260507 23:10:38.589 | INFO | Baz IO overall: | 20260507 23:10:38.589 | INFO | BytesWritten : 1520294157866 | 20260507 23:10:38.589 | INFO | NumEvents : 736327442829 | 20260507 23:10:39.959 | INFO | Analyzing chunk frames = [3338240,3338752) | 20260507 23:10:39.959 | INFO | MemInfo: { | 20260507 23:10:39.959 | INFO | "gpu" : " 17 GB", | 20260507 23:10:39.959 | INFO | "huge" : "", | 20260507 23:10:39.959 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:39.959 | INFO | "malloc" : " 14 GB", | 20260507 23:10:39.959 | INFO | "pinned" : " 29 GB" | 20260507 23:10:39.959 | INFO | } | 20260507 23:10:39.959 | INFO | Perf report for the last 5.13822 seconds: | 20260507 23:10:39.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:39.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:39.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:39.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:39.959 | INFO | ANALYSIS : 68.6%, 3.0, 106.3ms, 0 | 20260507 23:10:39.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:39.959 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 23:10:39.959 | INFO | PRE_HQ : 65.8%, 1.7, 57.3ms, 0 | 20260507 23:10:39.959 | INFO | BAZWRITER : 71.1%, 4.6, 167.0ms, 0 | 20260507 23:10:39.959 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 23:10:39.959 | INFO | Uploaded 36GiB to GPU in 2.5276s (14.2428GiB/s) | 20260507 23:10:39.959 | INFO | Downloaded 38GiB to GPU in 2.69985s (14.0748GiB/s) | 20260507 23:10:45.095 | INFO | Analyzing chunk frames = [3338752,3339264) | 20260507 23:10:45.095 | INFO | MemInfo: { | 20260507 23:10:45.095 | INFO | "gpu" : " 17 GB", | 20260507 23:10:45.095 | INFO | "huge" : "", | 20260507 23:10:45.095 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:45.095 | INFO | "malloc" : " 14 GB", | 20260507 23:10:45.096 | INFO | "pinned" : " 29 GB" | 20260507 23:10:45.096 | INFO | } | 20260507 23:10:45.096 | INFO | Perf report for the last 5.1365 seconds: | 20260507 23:10:45.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:45.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 23:10:45.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:10:45.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:45.096 | INFO | Uploaded 36GiB to GPU in 2.56491s (14.0356GiB/s) | 20260507 23:10:45.096 | INFO | Downloaded 34GiB to GPU in 2.49601s (13.6218GiB/s) | 20260507 23:10:50.233 | INFO | Analyzing chunk frames = [3339264,3339776) | 20260507 23:10:50.233 | INFO | MemInfo: { | 20260507 23:10:50.233 | INFO | "gpu" : " 17 GB", | 20260507 23:10:50.233 | INFO | "huge" : "", | 20260507 23:10:50.233 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:50.233 | INFO | "malloc" : " 14 GB", | 20260507 23:10:50.233 | INFO | "pinned" : " 29 GB" | 20260507 23:10:50.233 | INFO | } | 20260507 23:10:50.233 | INFO | Perf report for the last 5.13767 seconds: | 20260507 23:10:50.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:50.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.233 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:10:50.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.233 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:10:50.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:50.234 | INFO | Uploaded 36GiB to GPU in 2.51723s (14.3014GiB/s) | 20260507 23:10:50.234 | INFO | Downloaded 34GiB to GPU in 2.47875s (13.7166GiB/s) | 20260507 23:10:55.370 | INFO | Analyzing chunk frames = [3339776,3340288) | 20260507 23:10:55.370 | INFO | MemInfo: { | 20260507 23:10:55.370 | INFO | "gpu" : " 17 GB", | 20260507 23:10:55.370 | INFO | "huge" : "", | 20260507 23:10:55.370 | INFO | "huge_shared" : " 29 GB", | 20260507 23:10:55.371 | INFO | "malloc" : " 14 GB", | 20260507 23:10:55.371 | INFO | "pinned" : " 29 GB" | 20260507 23:10:55.371 | INFO | } | 20260507 23:10:55.371 | INFO | Perf report for the last 5.13729 seconds: | 20260507 23:10:55.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:10:55.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260507 23:10:55.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:10:55.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:10:55.371 | INFO | Uploaded 36GiB to GPU in 2.5679s (14.0192GiB/s) | 20260507 23:10:55.371 | INFO | Downloaded 34GiB to GPU in 2.48399s (13.6877GiB/s) | 20260507 23:11:00.513 | INFO | Analyzing chunk frames = [3340288,3340800) | 20260507 23:11:00.513 | INFO | MemInfo: { | 20260507 23:11:00.513 | INFO | "gpu" : " 17 GB", | 20260507 23:11:00.513 | INFO | "huge" : "", | 20260507 23:11:00.513 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:00.513 | INFO | "malloc" : " 14 GB", | 20260507 23:11:00.513 | INFO | "pinned" : " 29 GB" | 20260507 23:11:00.513 | INFO | } | 20260507 23:11:00.513 | INFO | Perf report for the last 5.14278 seconds: | 20260507 23:11:00.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:00.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.514 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 23:11:00.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.514 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:11:00.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:00.514 | INFO | Uploaded 36GiB to GPU in 2.33698s (15.4045GiB/s) | 20260507 23:11:00.514 | INFO | Downloaded 34GiB to GPU in 2.50948s (13.5486GiB/s) | 20260507 23:11:05.658 | INFO | Analyzing chunk frames = [3340800,3341312) | 20260507 23:11:05.658 | INFO | MemInfo: { | 20260507 23:11:05.658 | INFO | "gpu" : " 17 GB", | 20260507 23:11:05.658 | INFO | "huge" : "", | 20260507 23:11:05.658 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:05.658 | INFO | "malloc" : " 14 GB", | 20260507 23:11:05.658 | INFO | "pinned" : " 29 GB" | 20260507 23:11:05.658 | INFO | } | 20260507 23:11:05.658 | INFO | Perf report for the last 5.14454 seconds: | 20260507 23:11:05.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:05.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.658 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 23:11:05.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.658 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260507 23:11:05.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:05.659 | INFO | Uploaded 36GiB to GPU in 2.48367s (14.4947GiB/s) | 20260507 23:11:05.659 | INFO | Downloaded 34GiB to GPU in 2.44369s (13.9134GiB/s) | 20260507 23:11:10.796 | INFO | Analyzing chunk frames = [3341312,3341824) | 20260507 23:11:10.796 | INFO | MemInfo: { | 20260507 23:11:10.796 | INFO | "gpu" : " 17 GB", | 20260507 23:11:10.796 | INFO | "huge" : "", | 20260507 23:11:10.796 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:10.796 | INFO | "malloc" : " 14 GB", | 20260507 23:11:10.796 | INFO | "pinned" : " 29 GB" | 20260507 23:11:10.796 | INFO | } | 20260507 23:11:10.796 | INFO | Perf report for the last 5.1381 seconds: | 20260507 23:11:10.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:10.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.796 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:11:10.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.796 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 23:11:10.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:10.797 | INFO | Uploaded 36GiB to GPU in 2.5262s (14.2506GiB/s) | 20260507 23:11:10.797 | INFO | Downloaded 34GiB to GPU in 2.47858s (13.7175GiB/s) | 20260507 23:11:15.931 | INFO | Analyzing chunk frames = [3341824,3342336) | 20260507 23:11:15.931 | INFO | MemInfo: { | 20260507 23:11:15.931 | INFO | "gpu" : " 17 GB", | 20260507 23:11:15.931 | INFO | "huge" : "", | 20260507 23:11:15.931 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:15.931 | INFO | "malloc" : " 14 GB", | 20260507 23:11:15.931 | INFO | "pinned" : " 29 GB" | 20260507 23:11:15.931 | INFO | } | 20260507 23:11:15.931 | INFO | Perf report for the last 5.13492 seconds: | 20260507 23:11:15.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:15.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.931 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:11:15.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.931 | INFO | PRE_HQ : 36.1%, 1.0, 18.7ms, 0 | 20260507 23:11:15.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:15.932 | INFO | Uploaded 36GiB to GPU in 2.51726s (14.3013GiB/s) | 20260507 23:11:15.932 | INFO | Downloaded 34GiB to GPU in 2.46374s (13.8002GiB/s) | 20260507 23:11:15.932 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:11:15.932 | INFO | Total mem usage: 16.7 GiB | 20260507 23:11:15.932 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:11:15.932 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -8.72%) | 20260507 23:11:15.932 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:11:15.932 | INFO | High Water Breakdown: | 20260507 23:11:15.932 | INFO | 60.5% : 9.28 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:11:15.932 | INFO | 39.4% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:11:15.932 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:11:15.932 | INFO | ------------------------------------------ | 20260507 23:11:15.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:11:15.932 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:11:15.932 | INFO | Total mem usage: 26.5 GiB | 20260507 23:11:15.932 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:11:15.932 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:11:15.932 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260507 23:11:15.932 | INFO | High Water Breakdown: | 20260507 23:11:15.932 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:11:15.932 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:11:15.932 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:11:15.932 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:11:15.932 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:11:15.932 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:11:15.932 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:11:15.932 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:11:15.932 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:11:15.932 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:11:15.932 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:11:15.932 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:11:15.932 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:11:15.932 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:11:15.932 | INFO | ------------------------------------------ | 20260507 23:11:15.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:11:15.932 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:11:15.932 | INFO | Total mem usage: 27.8 GiB | 20260507 23:11:15.932 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:11:15.932 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:11:15.932 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:11:15.932 | INFO | High Water Breakdown: | 20260507 23:11:15.932 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:11:15.932 | INFO | ------------------------------------------ | 20260507 23:11:15.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:11:15.932 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:11:15.932 | INFO | Total mem usage: 15.7 GiB | 20260507 23:11:15.932 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:11:15.932 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:11:15.932 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:11:15.932 | INFO | High Water Breakdown: | 20260507 23:11:15.932 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:11:15.933 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:11:15.933 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:11:15.933 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:11:15.933 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:11:15.933 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:11:15.933 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:11:15.933 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:11:15.933 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:11:15.933 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:11:15.933 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:11:15.933 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:11:15.933 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:11:15.933 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:11:15.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:11:15.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:11:15.933 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:11:15.933 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:11:15.933 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:11:15.933 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:11:15.933 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:11:15.933 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:11:15.933 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:11:15.933 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:11:15.933 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:11:15.933 | INFO | ------------------------------------------ | 20260507 23:11:15.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:11:15.933 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:11:15.933 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:11:15.933 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:11:15.933 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.66e+01, 1.40e+01 / 1.57e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 0.30%, AggregateStats -- 9.93e-02 / 1.04e-01, 1.24e-02 / 9.95e-03 | 20260507 23:11:15.933 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.47e+00, 5.85e-01 / 2.16e-01 | 20260507 23:11:15.933 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.03e-01 / 9.59e-02 | 20260507 23:11:15.933 | INFO | PerfReport: 16.09%, DME -- 4.22e+01 / 4.62e+01, 4.05e+00 / 1.58e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.59e+00 / 1.41e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.47e+00 / 2.73e+00, 4.36e-01 / 2.13e-01 | 20260507 23:11:15.933 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.34e+00, 2.86e-01 / 1.63e-01 | 20260507 23:11:15.933 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.50e-01 / 3.84e-01, 5.72e-02 / 2.83e-02 | 20260507 23:11:15.933 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:11:15.933 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:11:15.933 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:11:15.933 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 24.37%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 24.48%, Download -- 2.61e+01 / 0.00e+00, 8.47e+00 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 30.73%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 1.95%, LockUpload -- 2.08e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 2.88%, LockDownload -- 3.08e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:11:15.933 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 4.10e-02 / 0.00e+00 | 20260507 23:11:16.061 | INFO | Starting a round of Baz IO | 20260507 23:11:19.504 | INFO | Populating RTMetrics report | 20260507 23:11:19.505 | INFO | RTMetrics report populated | 20260507 23:11:19.505 | INFO | Opening tmp RTMetrics json file | 20260507 23:11:19.628 | INFO | Serializing RTMetrics to json | 20260507 23:11:19.637 | INFO | Serialization done, swapping tmp file now... | 20260507 23:11:19.638 | INFO | RTMetrics done | 20260507 23:11:19.656 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.595 seconds | 20260507 23:11:19.656 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.74% / 0.01% / 57.26% | 20260507 23:11:19.656 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24836% / 7.94729e-06% / 6.75519e-05% | 20260507 23:11:19.656 | INFO | Baz IO overall: | 20260507 23:11:19.656 | INFO | BytesWritten : 1522141570049 | 20260507 23:11:19.656 | INFO | NumEvents : 737218169660 | 20260507 23:11:21.072 | INFO | Analyzing chunk frames = [3342336,3342848) | 20260507 23:11:21.072 | INFO | MemInfo: { | 20260507 23:11:21.072 | INFO | "gpu" : " 17 GB", | 20260507 23:11:21.072 | INFO | "huge" : "", | 20260507 23:11:21.072 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:21.072 | INFO | "malloc" : " 14 GB", | 20260507 23:11:21.072 | INFO | "pinned" : " 29 GB" | 20260507 23:11:21.072 | INFO | } | 20260507 23:11:21.072 | INFO | Perf report for the last 5.1408 seconds: | 20260507 23:11:21.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:21.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:21.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:21.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:21.072 | INFO | ANALYSIS : 68.5%, 2.9, 104.9ms, 0 | 20260507 23:11:21.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:21.072 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 23:11:21.072 | INFO | PRE_HQ : 67.1%, 1.8, 62.6ms, 0 | 20260507 23:11:21.072 | INFO | BAZWRITER : 69.7%, 3.8, 136.4ms, 0 | 20260507 23:11:21.072 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 23:11:21.072 | INFO | Uploaded 36GiB to GPU in 2.54861s (14.1253GiB/s) | 20260507 23:11:21.072 | INFO | Downloaded 38GiB to GPU in 2.68507s (14.1523GiB/s) | 20260507 23:11:26.208 | INFO | Analyzing chunk frames = [3342848,3343360) | 20260507 23:11:26.208 | INFO | MemInfo: { | 20260507 23:11:26.208 | INFO | "gpu" : " 17 GB", | 20260507 23:11:26.208 | INFO | "huge" : "", | 20260507 23:11:26.208 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:26.208 | INFO | "malloc" : " 14 GB", | 20260507 23:11:26.209 | INFO | "pinned" : " 29 GB" | 20260507 23:11:26.209 | INFO | } | 20260507 23:11:26.209 | INFO | Perf report for the last 5.1368 seconds: | 20260507 23:11:26.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:26.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 23:11:26.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:11:26.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:26.209 | INFO | Uploaded 36GiB to GPU in 2.54127s (14.1662GiB/s) | 20260507 23:11:26.209 | INFO | Downloaded 34GiB to GPU in 2.50175s (13.5905GiB/s) | 20260507 23:11:31.347 | INFO | Analyzing chunk frames = [3343360,3343872) | 20260507 23:11:31.347 | INFO | MemInfo: { | 20260507 23:11:31.347 | INFO | "gpu" : " 17 GB", | 20260507 23:11:31.347 | INFO | "huge" : "", | 20260507 23:11:31.347 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:31.347 | INFO | "malloc" : " 14 GB", | 20260507 23:11:31.347 | INFO | "pinned" : " 29 GB" | 20260507 23:11:31.347 | INFO | } | 20260507 23:11:31.348 | INFO | Perf report for the last 5.13885 seconds: | 20260507 23:11:31.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:31.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:11:31.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:11:31.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:31.348 | INFO | Uploaded 36GiB to GPU in 2.4984s (14.4092GiB/s) | 20260507 23:11:31.348 | INFO | Downloaded 34GiB to GPU in 2.48861s (13.6623GiB/s) | 20260507 23:11:36.490 | INFO | Analyzing chunk frames = [3343872,3344384) | 20260507 23:11:36.490 | INFO | MemInfo: { | 20260507 23:11:36.490 | INFO | "gpu" : " 17 GB", | 20260507 23:11:36.490 | INFO | "huge" : "", | 20260507 23:11:36.490 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:36.490 | INFO | "malloc" : " 14 GB", | 20260507 23:11:36.490 | INFO | "pinned" : " 29 GB" | 20260507 23:11:36.490 | INFO | } | 20260507 23:11:36.490 | INFO | Perf report for the last 5.14267 seconds: | 20260507 23:11:36.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:36.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.490 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 23:11:36.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.490 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:11:36.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:36.491 | INFO | Uploaded 36GiB to GPU in 2.44707s (14.7115GiB/s) | 20260507 23:11:36.491 | INFO | Downloaded 34GiB to GPU in 2.42358s (14.0288GiB/s) | 20260507 23:11:41.632 | INFO | Analyzing chunk frames = [3344384,3344896) | 20260507 23:11:41.632 | INFO | MemInfo: { | 20260507 23:11:41.632 | INFO | "gpu" : " 17 GB", | 20260507 23:11:41.632 | INFO | "huge" : "", | 20260507 23:11:41.632 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:41.632 | INFO | "malloc" : " 14 GB", | 20260507 23:11:41.632 | INFO | "pinned" : " 29 GB" | 20260507 23:11:41.633 | INFO | } | 20260507 23:11:41.633 | INFO | Perf report for the last 5.14231 seconds: | 20260507 23:11:41.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:41.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 23:11:41.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:11:41.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:41.633 | INFO | Uploaded 36GiB to GPU in 2.36272s (15.2367GiB/s) | 20260507 23:11:41.633 | INFO | Downloaded 34GiB to GPU in 2.50067s (13.5963GiB/s) | 20260507 23:11:46.772 | INFO | Analyzing chunk frames = [3344896,3345408) | 20260507 23:11:46.773 | INFO | MemInfo: { | 20260507 23:11:46.773 | INFO | "gpu" : " 17 GB", | 20260507 23:11:46.773 | INFO | "huge" : "", | 20260507 23:11:46.773 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:46.773 | INFO | "malloc" : " 14 GB", | 20260507 23:11:46.773 | INFO | "pinned" : " 29 GB" | 20260507 23:11:46.773 | INFO | } | 20260507 23:11:46.773 | INFO | Perf report for the last 5.14019 seconds: | 20260507 23:11:46.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:46.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:11:46.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:11:46.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:46.773 | INFO | Uploaded 36GiB to GPU in 2.36838s (15.2003GiB/s) | 20260507 23:11:46.773 | INFO | Downloaded 34GiB to GPU in 2.42659s (14.0114GiB/s) | 20260507 23:11:51.912 | INFO | Analyzing chunk frames = [3345408,3345920) | 20260507 23:11:51.912 | INFO | MemInfo: { | 20260507 23:11:51.912 | INFO | "gpu" : " 17 GB", | 20260507 23:11:51.912 | INFO | "huge" : "", | 20260507 23:11:51.912 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:51.912 | INFO | "malloc" : " 14 GB", | 20260507 23:11:51.912 | INFO | "pinned" : " 29 GB" | 20260507 23:11:51.912 | INFO | } | 20260507 23:11:51.912 | INFO | Perf report for the last 5.13959 seconds: | 20260507 23:11:51.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:51.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.913 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:11:51.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.913 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 23:11:51.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:51.913 | INFO | Uploaded 36GiB to GPU in 2.51632s (14.3066GiB/s) | 20260507 23:11:51.913 | INFO | Downloaded 34GiB to GPU in 2.47473s (13.7389GiB/s) | 20260507 23:11:57.046 | INFO | Analyzing chunk frames = [3345920,3346432) | 20260507 23:11:57.046 | INFO | MemInfo: { | 20260507 23:11:57.046 | INFO | "gpu" : " 17 GB", | 20260507 23:11:57.046 | INFO | "huge" : "", | 20260507 23:11:57.046 | INFO | "huge_shared" : " 29 GB", | 20260507 23:11:57.046 | INFO | "malloc" : " 14 GB", | 20260507 23:11:57.046 | INFO | "pinned" : " 29 GB" | 20260507 23:11:57.046 | INFO | } | 20260507 23:11:57.046 | INFO | Perf report for the last 5.13363 seconds: | 20260507 23:11:57.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:11:57.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260507 23:11:57.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260507 23:11:57.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:11:57.046 | INFO | Uploaded 36GiB to GPU in 2.49652s (14.4201GiB/s) | 20260507 23:11:57.046 | INFO | Downloaded 34GiB to GPU in 2.46697s (13.7821GiB/s) | 20260507 23:11:57.143 | INFO | Starting a round of Baz IO | 20260507 23:12:00.613 | INFO | Populating RTMetrics report | 20260507 23:12:00.614 | INFO | RTMetrics report populated | 20260507 23:12:00.614 | INFO | Opening tmp RTMetrics json file | 20260507 23:12:00.623 | INFO | Serializing RTMetrics to json | 20260507 23:12:00.643 | INFO | Serialization done, swapping tmp file now... | 20260507 23:12:00.653 | INFO | RTMetrics done | 20260507 23:12:00.716 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.572 seconds | 20260507 23:12:00.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.73% / 0.01% / 57.26% | 20260507 23:12:00.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.2434% / 3.97364e-06% / 6.75519e-05% | 20260507 23:12:00.716 | INFO | Baz IO overall: | 20260507 23:12:00.716 | INFO | BytesWritten : 1523987390699 | 20260507 23:12:00.716 | INFO | NumEvents : 738108061389 | 20260507 23:12:02.182 | INFO | Analyzing chunk frames = [3346432,3346944) | 20260507 23:12:02.182 | INFO | MemInfo: { | 20260507 23:12:02.182 | INFO | "gpu" : " 17 GB", | 20260507 23:12:02.182 | INFO | "huge" : "", | 20260507 23:12:02.182 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:02.182 | INFO | "malloc" : " 14 GB", | 20260507 23:12:02.182 | INFO | "pinned" : " 29 GB" | 20260507 23:12:02.182 | INFO | } | 20260507 23:12:02.183 | INFO | Perf report for the last 5.13652 seconds: | 20260507 23:12:02.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:02.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:02.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:02.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:02.183 | INFO | ANALYSIS : 68.8%, 3.0, 106.7ms, 0 | 20260507 23:12:02.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:02.183 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 23:12:02.183 | INFO | PRE_HQ : 66.2%, 1.7, 57.2ms, 0 | 20260507 23:12:02.183 | INFO | BAZWRITER : 69.1%, 4.1, 145.1ms, 0 | 20260507 23:12:02.183 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 23:12:02.183 | INFO | Uploaded 36GiB to GPU in 2.50324s (14.3814GiB/s) | 20260507 23:12:02.183 | INFO | Downloaded 38GiB to GPU in 2.69755s (14.0869GiB/s) | 20260507 23:12:07.325 | INFO | Analyzing chunk frames = [3346944,3347456) | 20260507 23:12:07.325 | INFO | MemInfo: { | 20260507 23:12:07.325 | INFO | "gpu" : " 17 GB", | 20260507 23:12:07.325 | INFO | "huge" : "", | 20260507 23:12:07.325 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:07.325 | INFO | "malloc" : " 14 GB", | 20260507 23:12:07.325 | INFO | "pinned" : " 29 GB" | 20260507 23:12:07.325 | INFO | } | 20260507 23:12:07.325 | INFO | Perf report for the last 5.14258 seconds: | 20260507 23:12:07.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:07.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:12:07.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:12:07.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:07.325 | INFO | Uploaded 36GiB to GPU in 2.51407s (14.3194GiB/s) | 20260507 23:12:07.326 | INFO | Downloaded 34GiB to GPU in 2.50487s (13.5736GiB/s) | 20260507 23:12:12.467 | INFO | Analyzing chunk frames = [3347456,3347968) | 20260507 23:12:12.467 | INFO | MemInfo: { | 20260507 23:12:12.467 | INFO | "gpu" : " 17 GB", | 20260507 23:12:12.467 | INFO | "huge" : "", | 20260507 23:12:12.467 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:12.467 | INFO | "malloc" : " 14 GB", | 20260507 23:12:12.467 | INFO | "pinned" : " 29 GB" | 20260507 23:12:12.467 | INFO | } | 20260507 23:12:12.467 | INFO | Perf report for the last 5.14172 seconds: | 20260507 23:12:12.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:12.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260507 23:12:12.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:12:12.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:12.467 | INFO | Uploaded 36GiB to GPU in 2.4832s (14.4974GiB/s) | 20260507 23:12:12.468 | INFO | Downloaded 34GiB to GPU in 2.47504s (13.7372GiB/s) | 20260507 23:12:17.606 | INFO | Analyzing chunk frames = [3347968,3348480) | 20260507 23:12:17.606 | INFO | MemInfo: { | 20260507 23:12:17.606 | INFO | "gpu" : " 17 GB", | 20260507 23:12:17.606 | INFO | "huge" : "", | 20260507 23:12:17.606 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:17.606 | INFO | "malloc" : " 14 GB", | 20260507 23:12:17.606 | INFO | "pinned" : " 29 GB" | 20260507 23:12:17.606 | INFO | } | 20260507 23:12:17.606 | INFO | Perf report for the last 5.13946 seconds: | 20260507 23:12:17.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:17.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.606 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260507 23:12:17.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.606 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:12:17.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:17.607 | INFO | Uploaded 36GiB to GPU in 2.47261s (14.5595GiB/s) | 20260507 23:12:17.607 | INFO | Downloaded 34GiB to GPU in 2.41627s (14.0713GiB/s) | 20260507 23:12:22.743 | INFO | Analyzing chunk frames = [3348480,3348992) | 20260507 23:12:22.743 | INFO | MemInfo: { | 20260507 23:12:22.743 | INFO | "gpu" : " 17 GB", | 20260507 23:12:22.743 | INFO | "huge" : "", | 20260507 23:12:22.743 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:22.743 | INFO | "malloc" : " 14 GB", | 20260507 23:12:22.743 | INFO | "pinned" : " 29 GB" | 20260507 23:12:22.743 | INFO | } | 20260507 23:12:22.743 | INFO | Perf report for the last 5.1365 seconds: | 20260507 23:12:22.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:22.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 23:12:22.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 23:12:22.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:22.743 | INFO | Uploaded 36GiB to GPU in 2.37927s (15.1307GiB/s) | 20260507 23:12:22.743 | INFO | Downloaded 34GiB to GPU in 2.50168s (13.5909GiB/s) | 20260507 23:12:27.883 | INFO | Analyzing chunk frames = [3348992,3349504) | 20260507 23:12:27.883 | INFO | MemInfo: { | 20260507 23:12:27.883 | INFO | "gpu" : " 17 GB", | 20260507 23:12:27.883 | INFO | "huge" : "", | 20260507 23:12:27.883 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:27.884 | INFO | "malloc" : " 14 GB", | 20260507 23:12:27.884 | INFO | "pinned" : " 29 GB" | 20260507 23:12:27.884 | INFO | } | 20260507 23:12:27.884 | INFO | Perf report for the last 5.14096 seconds: | 20260507 23:12:27.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:27.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:12:27.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:12:27.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:27.884 | INFO | Uploaded 36GiB to GPU in 2.50017s (14.399GiB/s) | 20260507 23:12:27.884 | INFO | Downloaded 34GiB to GPU in 2.4502s (13.8764GiB/s) | 20260507 23:12:33.023 | INFO | Analyzing chunk frames = [3349504,3350016) | 20260507 23:12:33.023 | INFO | MemInfo: { | 20260507 23:12:33.023 | INFO | "gpu" : " 17 GB", | 20260507 23:12:33.023 | INFO | "huge" : "", | 20260507 23:12:33.023 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:33.023 | INFO | "malloc" : " 14 GB", | 20260507 23:12:33.023 | INFO | "pinned" : " 29 GB" | 20260507 23:12:33.023 | INFO | } | 20260507 23:12:33.023 | INFO | Perf report for the last 5.13956 seconds: | 20260507 23:12:33.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:33.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.024 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:12:33.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.024 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260507 23:12:33.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:33.024 | INFO | Uploaded 36GiB to GPU in 2.50379s (14.3782GiB/s) | 20260507 23:12:33.024 | INFO | Downloaded 34GiB to GPU in 2.47174s (13.7555GiB/s) | 20260507 23:12:38.162 | INFO | Analyzing chunk frames = [3350016,3350528) | 20260507 23:12:38.162 | INFO | MemInfo: { | 20260507 23:12:38.162 | INFO | "gpu" : " 17 GB", | 20260507 23:12:38.162 | INFO | "huge" : "", | 20260507 23:12:38.162 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:38.162 | INFO | "malloc" : " 14 GB", | 20260507 23:12:38.162 | INFO | "pinned" : " 29 GB" | 20260507 23:12:38.162 | INFO | } | 20260507 23:12:38.162 | INFO | Perf report for the last 5.13856 seconds: | 20260507 23:12:38.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:38.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:12:38.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 23:12:38.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:38.162 | INFO | Uploaded 36GiB to GPU in 2.47227s (14.5615GiB/s) | 20260507 23:12:38.162 | INFO | Downloaded 34GiB to GPU in 2.46345s (13.8018GiB/s) | 20260507 23:12:38.163 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:12:38.163 | INFO | Total mem usage: 16.7 GiB | 20260507 23:12:38.163 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:12:38.163 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260507 23:12:38.163 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:12:38.163 | INFO | High Water Breakdown: | 20260507 23:12:38.163 | INFO | 59.7% : 8.72 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:12:38.163 | INFO | 40.1% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:12:38.163 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:12:38.163 | INFO | ------------------------------------------ | 20260507 23:12:38.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:12:38.163 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:12:38.163 | INFO | Total mem usage: 26.5 GiB | 20260507 23:12:38.163 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:12:38.163 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260507 23:12:38.163 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:12:38.163 | INFO | High Water Breakdown: | 20260507 23:12:38.163 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:12:38.163 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:12:38.163 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:12:38.163 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:12:38.163 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:12:38.163 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:12:38.163 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:12:38.163 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:12:38.163 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:12:38.163 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:12:38.163 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:12:38.163 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:12:38.163 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:12:38.163 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:12:38.163 | INFO | ------------------------------------------ | 20260507 23:12:38.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:12:38.163 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:12:38.163 | INFO | Total mem usage: 27.8 GiB | 20260507 23:12:38.163 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:12:38.163 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:12:38.163 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:12:38.163 | INFO | High Water Breakdown: | 20260507 23:12:38.163 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:12:38.163 | INFO | ------------------------------------------ | 20260507 23:12:38.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:12:38.163 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:12:38.163 | INFO | Total mem usage: 15.7 GiB | 20260507 23:12:38.163 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:12:38.163 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:12:38.163 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260507 23:12:38.163 | INFO | High Water Breakdown: | 20260507 23:12:38.163 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:12:38.163 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:12:38.163 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:12:38.163 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:12:38.163 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:12:38.163 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:12:38.163 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:12:38.163 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:12:38.163 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:12:38.163 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:12:38.163 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:12:38.164 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:12:38.164 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:12:38.164 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:12:38.164 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:12:38.164 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:12:38.164 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:12:38.164 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:12:38.164 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:12:38.164 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:12:38.164 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:12:38.164 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:12:38.164 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:12:38.164 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:12:38.164 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:12:38.164 | INFO | ------------------------------------------ | 20260507 23:12:38.164 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:12:38.164 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:12:38.164 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:12:38.164 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:12:38.164 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.60e+01, 1.40e+01 / 1.33e+00 | 20260507 23:12:38.164 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.03e-01, 1.20e-02 / 5.10e-03 | 20260507 23:12:38.164 | INFO | PerfReport: 12.12%, Baseline -- 4.00e+00 / 4.47e+00, 5.93e-01 / 2.19e-01 | 20260507 23:12:38.164 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.15e-01 | 20260507 23:12:38.164 | INFO | PerfReport: 15.74%, DME -- 4.15e+01 / 4.50e+01, 3.83e+00 / 1.14e+00 | 20260507 23:12:38.164 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.87e+00 / 1.86e+00 | 20260507 23:12:38.164 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.42e+00 / 2.67e+00, 4.28e-01 / 2.62e-01 | 20260507 23:12:38.164 | INFO | PerfReport: 3.42%, Metrics -- 1.13e+00 / 1.35e+00, 2.89e-01 / 1.81e-01 | 20260507 23:12:38.164 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.79e-01, 5.64e-02 / 2.85e-02 | 20260507 23:12:38.164 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:12:38.164 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:12:38.164 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:12:38.164 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260507 23:12:38.164 | INFO | PerfReport: 24.21%, Upload -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 23:12:38.164 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.53e+00 / 0.00e+00 | 20260507 23:12:38.165 | INFO | PerfReport: 30.98%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:12:38.165 | INFO | PerfReport: 1.67%, LockUpload -- 1.78e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 23:12:38.165 | INFO | PerfReport: 2.79%, LockDownload -- 2.97e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260507 23:12:38.165 | INFO | PerfReport: 15.91%, LockCompute -- 1.69e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:12:38.165 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 23:12:38.275 | INFO | Starting a round of Baz IO | 20260507 23:12:41.722 | INFO | Populating RTMetrics report | 20260507 23:12:41.723 | INFO | RTMetrics report populated | 20260507 23:12:41.723 | INFO | Opening tmp RTMetrics json file | 20260507 23:12:41.806 | INFO | Serializing RTMetrics to json | 20260507 23:12:41.864 | INFO | Serialization done, swapping tmp file now... | 20260507 23:12:41.866 | INFO | RTMetrics done | 20260507 23:12:41.905 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.629 seconds | 20260507 23:12:41.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.72% / 0.01% / 57.27% | 20260507 23:12:41.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24411% / 7.94729e-06% / 6.75519e-05% | 20260507 23:12:41.905 | INFO | Baz IO overall: | 20260507 23:12:41.905 | INFO | BytesWritten : 1525834784113 | 20260507 23:12:41.905 | INFO | NumEvents : 738999098287 | 20260507 23:12:43.299 | INFO | Analyzing chunk frames = [3350528,3351040) | 20260507 23:12:43.299 | INFO | MemInfo: { | 20260507 23:12:43.299 | INFO | "gpu" : " 17 GB", | 20260507 23:12:43.299 | INFO | "huge" : "", | 20260507 23:12:43.299 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:43.299 | INFO | "malloc" : " 14 GB", | 20260507 23:12:43.299 | INFO | "pinned" : " 29 GB" | 20260507 23:12:43.299 | INFO | } | 20260507 23:12:43.299 | INFO | Perf report for the last 5.13666 seconds: | 20260507 23:12:43.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:43.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:43.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:43.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:43.299 | INFO | ANALYSIS : 68.6%, 2.9, 104.8ms, 0 | 20260507 23:12:43.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:43.299 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260507 23:12:43.299 | INFO | PRE_HQ : 67.2%, 1.8, 63.1ms, 0 | 20260507 23:12:43.299 | INFO | BAZWRITER : 70.2%, 3.6, 128.1ms, 0 | 20260507 23:12:43.299 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 23:12:43.299 | INFO | Uploaded 36GiB to GPU in 2.57003s (14.0076GiB/s) | 20260507 23:12:43.299 | INFO | Downloaded 38GiB to GPU in 2.70123s (14.0677GiB/s) | 20260507 23:12:48.439 | INFO | Analyzing chunk frames = [3351040,3351552) | 20260507 23:12:48.439 | INFO | MemInfo: { | 20260507 23:12:48.439 | INFO | "gpu" : " 17 GB", | 20260507 23:12:48.439 | INFO | "huge" : "", | 20260507 23:12:48.439 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:48.439 | INFO | "malloc" : " 14 GB", | 20260507 23:12:48.439 | INFO | "pinned" : " 29 GB" | 20260507 23:12:48.439 | INFO | } | 20260507 23:12:48.439 | INFO | Perf report for the last 5.14069 seconds: | 20260507 23:12:48.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:48.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.439 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:12:48.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.439 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:12:48.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:48.440 | INFO | Uploaded 36GiB to GPU in 2.56213s (14.0508GiB/s) | 20260507 23:12:48.440 | INFO | Downloaded 34GiB to GPU in 2.49382s (13.6337GiB/s) | 20260507 23:12:53.584 | INFO | Analyzing chunk frames = [3351552,3352064) | 20260507 23:12:53.584 | INFO | MemInfo: { | 20260507 23:12:53.584 | INFO | "gpu" : " 17 GB", | 20260507 23:12:53.584 | INFO | "huge" : "", | 20260507 23:12:53.584 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:53.584 | INFO | "malloc" : " 14 GB", | 20260507 23:12:53.584 | INFO | "pinned" : " 29 GB" | 20260507 23:12:53.584 | INFO | } | 20260507 23:12:53.584 | INFO | Perf report for the last 5.14501 seconds: | 20260507 23:12:53.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:53.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.584 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260507 23:12:53.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.585 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:12:53.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:53.585 | INFO | Uploaded 36GiB to GPU in 2.53107s (14.2233GiB/s) | 20260507 23:12:53.585 | INFO | Downloaded 34GiB to GPU in 2.46311s (13.8037GiB/s) | 20260507 23:12:58.723 | INFO | Analyzing chunk frames = [3352064,3352576) | 20260507 23:12:58.723 | INFO | MemInfo: { | 20260507 23:12:58.723 | INFO | "gpu" : " 17 GB", | 20260507 23:12:58.723 | INFO | "huge" : "", | 20260507 23:12:58.723 | INFO | "huge_shared" : " 29 GB", | 20260507 23:12:58.723 | INFO | "malloc" : " 14 GB", | 20260507 23:12:58.723 | INFO | "pinned" : " 29 GB" | 20260507 23:12:58.723 | INFO | } | 20260507 23:12:58.723 | INFO | Perf report for the last 5.13862 seconds: | 20260507 23:12:58.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:12:58.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:12:58.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:12:58.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:12:58.723 | INFO | Uploaded 36GiB to GPU in 2.50532s (14.3694GiB/s) | 20260507 23:12:58.723 | INFO | Downloaded 34GiB to GPU in 2.48381s (13.6886GiB/s) | 20260507 23:13:03.863 | INFO | Analyzing chunk frames = [3352576,3353088) | 20260507 23:13:03.863 | INFO | MemInfo: { | 20260507 23:13:03.863 | INFO | "gpu" : " 17 GB", | 20260507 23:13:03.863 | INFO | "huge" : "", | 20260507 23:13:03.863 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:03.863 | INFO | "malloc" : " 14 GB", | 20260507 23:13:03.863 | INFO | "pinned" : " 29 GB" | 20260507 23:13:03.863 | INFO | } | 20260507 23:13:03.863 | INFO | Perf report for the last 5.14026 seconds: | 20260507 23:13:03.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:03.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.863 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 23:13:03.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.863 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:13:03.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:03.864 | INFO | Uploaded 36GiB to GPU in 2.33133s (15.4418GiB/s) | 20260507 23:13:03.864 | INFO | Downloaded 34GiB to GPU in 2.47247s (13.7514GiB/s) | 20260507 23:13:08.998 | INFO | Analyzing chunk frames = [3353088,3353600) | 20260507 23:13:08.998 | INFO | MemInfo: { | 20260507 23:13:08.998 | INFO | "gpu" : " 17 GB", | 20260507 23:13:08.998 | INFO | "huge" : "", | 20260507 23:13:08.998 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:08.998 | INFO | "malloc" : " 14 GB", | 20260507 23:13:08.998 | INFO | "pinned" : " 29 GB" | 20260507 23:13:08.998 | INFO | } | 20260507 23:13:08.998 | INFO | Perf report for the last 5.13466 seconds: | 20260507 23:13:08.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:08.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260507 23:13:08.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:13:08.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:08.998 | INFO | Uploaded 36GiB to GPU in 2.42933s (14.8189GiB/s) | 20260507 23:13:08.998 | INFO | Downloaded 34GiB to GPU in 2.40032s (14.1648GiB/s) | 20260507 23:13:14.138 | INFO | Analyzing chunk frames = [3353600,3354112) | 20260507 23:13:14.138 | INFO | MemInfo: { | 20260507 23:13:14.138 | INFO | "gpu" : " 17 GB", | 20260507 23:13:14.138 | INFO | "huge" : "", | 20260507 23:13:14.138 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:14.138 | INFO | "malloc" : " 14 GB", | 20260507 23:13:14.138 | INFO | "pinned" : " 29 GB" | 20260507 23:13:14.138 | INFO | } | 20260507 23:13:14.139 | INFO | Perf report for the last 5.14051 seconds: | 20260507 23:13:14.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:14.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:13:14.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 23:13:14.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:14.139 | INFO | Uploaded 36GiB to GPU in 2.52721s (14.2449GiB/s) | 20260507 23:13:14.139 | INFO | Downloaded 34GiB to GPU in 2.44108s (13.9283GiB/s) | 20260507 23:13:19.273 | INFO | Analyzing chunk frames = [3354112,3354624) | 20260507 23:13:19.273 | INFO | MemInfo: { | 20260507 23:13:19.273 | INFO | "gpu" : " 17 GB", | 20260507 23:13:19.273 | INFO | "huge" : "", | 20260507 23:13:19.273 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:19.273 | INFO | "malloc" : " 14 GB", | 20260507 23:13:19.274 | INFO | "pinned" : " 29 GB" | 20260507 23:13:19.274 | INFO | } | 20260507 23:13:19.274 | INFO | Perf report for the last 5.13511 seconds: | 20260507 23:13:19.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:19.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 23:13:19.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | PRE_HQ : 36.5%, 1.0, 19.0ms, 0 | 20260507 23:13:19.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:19.274 | INFO | Uploaded 36GiB to GPU in 2.54218s (14.161GiB/s) | 20260507 23:13:19.274 | INFO | Downloaded 34GiB to GPU in 2.44219s (13.9219GiB/s) | 20260507 23:13:19.387 | INFO | Starting a round of Baz IO | 20260507 23:13:22.867 | INFO | Populating RTMetrics report | 20260507 23:13:22.868 | INFO | RTMetrics report populated | 20260507 23:13:22.868 | INFO | Opening tmp RTMetrics json file | 20260507 23:13:22.872 | INFO | Serializing RTMetrics to json | 20260507 23:13:22.879 | INFO | Serialization done, swapping tmp file now... | 20260507 23:13:22.922 | INFO | RTMetrics done | 20260507 23:13:23.168 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.78 seconds | 20260507 23:13:23.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.72% / 0.01% / 57.28% | 20260507 23:13:23.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.2418% / 3.97364e-06% / 4.37101e-05% | 20260507 23:13:23.168 | INFO | Baz IO overall: | 20260507 23:13:23.168 | INFO | BytesWritten : 1527681683619 | 20260507 23:13:23.168 | INFO | NumEvents : 739889895007 | 20260507 23:13:24.420 | INFO | Analyzing chunk frames = [3354624,3355136) | 20260507 23:13:24.420 | INFO | MemInfo: { | 20260507 23:13:24.420 | INFO | "gpu" : " 17 GB", | 20260507 23:13:24.420 | INFO | "huge" : "", | 20260507 23:13:24.420 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:24.420 | INFO | "malloc" : " 14 GB", | 20260507 23:13:24.420 | INFO | "pinned" : " 29 GB" | 20260507 23:13:24.420 | INFO | } | 20260507 23:13:24.420 | INFO | Perf report for the last 5.14614 seconds: | 20260507 23:13:24.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:24.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:24.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:24.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:24.420 | INFO | ANALYSIS : 69.1%, 2.9, 104.6ms, 0 | 20260507 23:13:24.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:24.420 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260507 23:13:24.420 | INFO | PRE_HQ : 68.1%, 1.8, 64.2ms, 0 | 20260507 23:13:24.420 | INFO | BAZWRITER : 73.1%, 4.8, 180.9ms, 0 | 20260507 23:13:24.420 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260507 23:13:24.420 | INFO | Uploaded 36GiB to GPU in 2.59328s (13.882GiB/s) | 20260507 23:13:24.420 | INFO | Downloaded 38GiB to GPU in 2.71378s (14.0026GiB/s) | 20260507 23:13:29.553 | INFO | Analyzing chunk frames = [3355136,3355648) | 20260507 23:13:29.553 | INFO | MemInfo: { | 20260507 23:13:29.553 | INFO | "gpu" : " 17 GB", | 20260507 23:13:29.553 | INFO | "huge" : "", | 20260507 23:13:29.553 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:29.553 | INFO | "malloc" : " 14 GB", | 20260507 23:13:29.553 | INFO | "pinned" : " 29 GB" | 20260507 23:13:29.553 | INFO | } | 20260507 23:13:29.554 | INFO | Perf report for the last 5.13371 seconds: | 20260507 23:13:29.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:29.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 23:13:29.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:13:29.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:29.554 | INFO | Uploaded 36GiB to GPU in 2.56803s (14.0185GiB/s) | 20260507 23:13:29.554 | INFO | Downloaded 34GiB to GPU in 2.49473s (13.6288GiB/s) | 20260507 23:13:34.697 | INFO | Analyzing chunk frames = [3355648,3356160) | 20260507 23:13:34.697 | INFO | MemInfo: { | 20260507 23:13:34.697 | INFO | "gpu" : " 17 GB", | 20260507 23:13:34.697 | INFO | "huge" : "", | 20260507 23:13:34.697 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:34.697 | INFO | "malloc" : " 14 GB", | 20260507 23:13:34.697 | INFO | "pinned" : " 29 GB" | 20260507 23:13:34.697 | INFO | } | 20260507 23:13:34.697 | INFO | Perf report for the last 5.14381 seconds: | 20260507 23:13:34.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:34.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.697 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 23:13:34.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.697 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:13:34.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:34.698 | INFO | Uploaded 36GiB to GPU in 2.55284s (14.1019GiB/s) | 20260507 23:13:34.698 | INFO | Downloaded 34GiB to GPU in 2.47621s (13.7306GiB/s) | 20260507 23:13:39.837 | INFO | Analyzing chunk frames = [3356160,3356672) | 20260507 23:13:39.837 | INFO | MemInfo: { | 20260507 23:13:39.837 | INFO | "gpu" : " 17 GB", | 20260507 23:13:39.837 | INFO | "huge" : "", | 20260507 23:13:39.837 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:39.837 | INFO | "malloc" : " 14 GB", | 20260507 23:13:39.837 | INFO | "pinned" : " 29 GB" | 20260507 23:13:39.837 | INFO | } | 20260507 23:13:39.837 | INFO | Perf report for the last 5.13946 seconds: | 20260507 23:13:39.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:39.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:13:39.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:13:39.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:39.837 | INFO | Uploaded 36GiB to GPU in 2.48342s (14.4961GiB/s) | 20260507 23:13:39.837 | INFO | Downloaded 34GiB to GPU in 2.41058s (14.1045GiB/s) | 20260507 23:13:44.974 | INFO | Analyzing chunk frames = [3356672,3357184) | 20260507 23:13:44.974 | INFO | MemInfo: { | 20260507 23:13:44.975 | INFO | "gpu" : " 17 GB", | 20260507 23:13:44.975 | INFO | "huge" : "", | 20260507 23:13:44.975 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:44.975 | INFO | "malloc" : " 14 GB", | 20260507 23:13:44.975 | INFO | "pinned" : " 29 GB" | 20260507 23:13:44.975 | INFO | } | 20260507 23:13:44.975 | INFO | Perf report for the last 5.13791 seconds: | 20260507 23:13:44.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:44.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260507 23:13:44.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 23:13:44.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:44.975 | INFO | Uploaded 36GiB to GPU in 2.36067s (15.2499GiB/s) | 20260507 23:13:44.975 | INFO | Downloaded 34GiB to GPU in 2.5154s (13.5167GiB/s) | 20260507 23:13:50.112 | INFO | Analyzing chunk frames = [3357184,3357696) | 20260507 23:13:50.112 | INFO | MemInfo: { | 20260507 23:13:50.112 | INFO | "gpu" : " 17 GB", | 20260507 23:13:50.112 | INFO | "huge" : "", | 20260507 23:13:50.112 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:50.112 | INFO | "malloc" : " 14 GB", | 20260507 23:13:50.112 | INFO | "pinned" : " 29 GB" | 20260507 23:13:50.112 | INFO | } | 20260507 23:13:50.112 | INFO | Perf report for the last 5.13749 seconds: | 20260507 23:13:50.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:50.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.112 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:13:50.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.112 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:13:50.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:50.113 | INFO | Uploaded 36GiB to GPU in 2.37966s (15.1282GiB/s) | 20260507 23:13:50.113 | INFO | Downloaded 34GiB to GPU in 2.42249s (14.0352GiB/s) | 20260507 23:13:55.250 | INFO | Analyzing chunk frames = [3357696,3358208) | 20260507 23:13:55.250 | INFO | MemInfo: { | 20260507 23:13:55.250 | INFO | "gpu" : " 17 GB", | 20260507 23:13:55.250 | INFO | "huge" : "", | 20260507 23:13:55.250 | INFO | "huge_shared" : " 29 GB", | 20260507 23:13:55.250 | INFO | "malloc" : " 14 GB", | 20260507 23:13:55.250 | INFO | "pinned" : " 29 GB" | 20260507 23:13:55.250 | INFO | } | 20260507 23:13:55.250 | INFO | Perf report for the last 5.13763 seconds: | 20260507 23:13:55.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:13:55.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:13:55.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 23:13:55.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:13:55.250 | INFO | Uploaded 36GiB to GPU in 2.5106s (14.3392GiB/s) | 20260507 23:13:55.250 | INFO | Downloaded 34GiB to GPU in 2.46449s (13.796GiB/s) | 20260507 23:14:00.397 | INFO | Analyzing chunk frames = [3358208,3358720) | 20260507 23:14:00.397 | INFO | MemInfo: { | 20260507 23:14:00.397 | INFO | "gpu" : " 17 GB", | 20260507 23:14:00.397 | INFO | "huge" : "", | 20260507 23:14:00.397 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:00.397 | INFO | "malloc" : " 14 GB", | 20260507 23:14:00.397 | INFO | "pinned" : " 29 GB" | 20260507 23:14:00.397 | INFO | } | 20260507 23:14:00.397 | INFO | Perf report for the last 5.14697 seconds: | 20260507 23:14:00.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:00.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260507 23:14:00.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:14:00.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:00.397 | INFO | Uploaded 36GiB to GPU in 2.47931s (14.5201GiB/s) | 20260507 23:14:00.397 | INFO | Downloaded 34GiB to GPU in 2.45271s (13.8622GiB/s) | 20260507 23:14:00.397 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:14:00.397 | INFO | Total mem usage: 16.7 GiB | 20260507 23:14:00.397 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:14:00.397 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -8.7%) | 20260507 23:14:00.397 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:14:00.397 | INFO | High Water Breakdown: | 20260507 23:14:00.398 | INFO | 60.5% : 9.29 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:14:00.398 | INFO | 39.4% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:14:00.398 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:14:00.398 | INFO | ------------------------------------------ | 20260507 23:14:00.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:14:00.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:14:00.398 | INFO | Total mem usage: 26.5 GiB | 20260507 23:14:00.398 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:14:00.398 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 23:14:00.398 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:14:00.398 | INFO | High Water Breakdown: | 20260507 23:14:00.398 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:14:00.398 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:14:00.398 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:14:00.398 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:14:00.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:14:00.398 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:14:00.398 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:14:00.398 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:14:00.398 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:14:00.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:14:00.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:14:00.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:14:00.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:14:00.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:14:00.398 | INFO | ------------------------------------------ | 20260507 23:14:00.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:14:00.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:14:00.398 | INFO | Total mem usage: 27.8 GiB | 20260507 23:14:00.398 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:14:00.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:14:00.398 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:14:00.398 | INFO | High Water Breakdown: | 20260507 23:14:00.398 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:14:00.398 | INFO | ------------------------------------------ | 20260507 23:14:00.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:14:00.398 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:14:00.398 | INFO | Total mem usage: 15.7 GiB | 20260507 23:14:00.398 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:14:00.398 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:14:00.398 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:14:00.398 | INFO | High Water Breakdown: | 20260507 23:14:00.398 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:14:00.398 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:14:00.398 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:14:00.398 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:14:00.398 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:14:00.398 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:14:00.398 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:14:00.398 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:14:00.398 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:14:00.398 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:14:00.398 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:14:00.398 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:14:00.398 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:14:00.398 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:14:00.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:14:00.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:14:00.398 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:14:00.398 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:14:00.398 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:14:00.398 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:14:00.398 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:14:00.398 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:14:00.398 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:14:00.398 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:14:00.398 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:14:00.398 | INFO | ------------------------------------------ | 20260507 23:14:00.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:14:00.398 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:14:00.398 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:14:00.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:14:00.398 | INFO | PerfReport: Full Execution -- 3.30e+01 / 7.62e+01, 1.38e+01 / 1.44e+00 | 20260507 23:14:00.398 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.15e-02 / 8.03e-03 | 20260507 23:14:00.398 | INFO | PerfReport: 12.16%, Baseline -- 4.01e+00 / 4.49e+00, 5.88e-01 / 2.21e-01 | 20260507 23:14:00.398 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.50e+00, 2.08e-01 / 1.18e-01 | 20260507 23:14:00.398 | INFO | PerfReport: 15.86%, DME -- 4.18e+01 / 4.54e+01, 4.01e+00 / 1.28e+00 | 20260507 23:14:00.398 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.71e+00 | 20260507 23:14:00.398 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.42e+00 / 2.68e+00, 4.31e-01 / 2.51e-01 | 20260507 23:14:00.398 | INFO | PerfReport: 3.42%, Metrics -- 1.13e+00 / 1.31e+00, 2.88e-01 / 1.45e-01 | 20260507 23:14:00.399 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.80e-01, 5.78e-02 / 3.51e-02 | 20260507 23:14:00.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:14:00.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:14:00.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:14:00.399 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 24.37%, Download -- 2.59e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 30.95%, Compute -- 3.30e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 1.99%, LockUpload -- 2.12e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 15.71%, LockCompute -- 1.67e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 23:14:00.399 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 23:14:00.494 | INFO | Starting a round of Baz IO | 20260507 23:14:03.967 | INFO | Populating RTMetrics report | 20260507 23:14:03.968 | INFO | RTMetrics report populated | 20260507 23:14:03.968 | INFO | Opening tmp RTMetrics json file | 20260507 23:14:03.991 | INFO | Serializing RTMetrics to json | 20260507 23:14:04.033 | INFO | Serialization done, swapping tmp file now... | 20260507 23:14:04.041 | INFO | RTMetrics done | 20260507 23:14:04.064 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.569 seconds | 20260507 23:14:04.064 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.71% / 0.01% / 57.28% | 20260507 23:14:04.064 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23936% / 7.94729e-06% / 5.5631e-05% | 20260507 23:14:04.064 | INFO | Baz IO overall: | 20260507 23:14:04.064 | INFO | BytesWritten : 1529527101912 | 20260507 23:14:04.064 | INFO | NumEvents : 740779737412 | 20260507 23:14:05.533 | INFO | Analyzing chunk frames = [3358720,3359232) | 20260507 23:14:05.533 | INFO | MemInfo: { | 20260507 23:14:05.533 | INFO | "gpu" : " 17 GB", | 20260507 23:14:05.533 | INFO | "huge" : "", | 20260507 23:14:05.533 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:05.533 | INFO | "malloc" : " 14 GB", | 20260507 23:14:05.533 | INFO | "pinned" : " 29 GB" | 20260507 23:14:05.533 | INFO | } | 20260507 23:14:05.533 | INFO | Perf report for the last 5.13649 seconds: | 20260507 23:14:05.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:05.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:05.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:05.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:05.534 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260507 23:14:05.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:05.534 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 23:14:05.534 | INFO | PRE_HQ : 66.0%, 1.7, 57.0ms, 0 | 20260507 23:14:05.534 | INFO | BAZWRITER : 69.1%, 4.6, 163.7ms, 0 | 20260507 23:14:05.534 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 23:14:05.534 | INFO | Uploaded 36GiB to GPU in 2.56268s (14.0478GiB/s) | 20260507 23:14:05.534 | INFO | Downloaded 38GiB to GPU in 2.68975s (14.1277GiB/s) | 20260507 23:14:10.674 | INFO | Analyzing chunk frames = [3359232,3359744) | 20260507 23:14:10.674 | INFO | MemInfo: { | 20260507 23:14:10.674 | INFO | "gpu" : " 17 GB", | 20260507 23:14:10.674 | INFO | "huge" : "", | 20260507 23:14:10.674 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:10.674 | INFO | "malloc" : " 14 GB", | 20260507 23:14:10.674 | INFO | "pinned" : " 29 GB" | 20260507 23:14:10.674 | INFO | } | 20260507 23:14:10.674 | INFO | Perf report for the last 5.14066 seconds: | 20260507 23:14:10.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:10.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.674 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 23:14:10.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.674 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:14:10.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:10.675 | INFO | Uploaded 36GiB to GPU in 2.5451s (14.1448GiB/s) | 20260507 23:14:10.675 | INFO | Downloaded 34GiB to GPU in 2.50147s (13.592GiB/s) | 20260507 23:14:15.813 | INFO | Analyzing chunk frames = [3359744,3360256) | 20260507 23:14:15.813 | INFO | MemInfo: { | 20260507 23:14:15.813 | INFO | "gpu" : " 17 GB", | 20260507 23:14:15.813 | INFO | "huge" : "", | 20260507 23:14:15.813 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:15.813 | INFO | "malloc" : " 14 GB", | 20260507 23:14:15.813 | INFO | "pinned" : " 29 GB" | 20260507 23:14:15.813 | INFO | } | 20260507 23:14:15.813 | INFO | Perf report for the last 5.13937 seconds: | 20260507 23:14:15.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:15.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.813 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 23:14:15.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.814 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:14:15.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:15.814 | INFO | Uploaded 36GiB to GPU in 2.44943s (14.6973GiB/s) | 20260507 23:14:15.814 | INFO | Downloaded 34GiB to GPU in 2.49337s (13.6362GiB/s) | 20260507 23:14:20.950 | INFO | Analyzing chunk frames = [3360256,3360768) | 20260507 23:14:20.950 | INFO | MemInfo: { | 20260507 23:14:20.950 | INFO | "gpu" : " 17 GB", | 20260507 23:14:20.950 | INFO | "huge" : "", | 20260507 23:14:20.950 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:20.950 | INFO | "malloc" : " 14 GB", | 20260507 23:14:20.950 | INFO | "pinned" : " 29 GB" | 20260507 23:14:20.950 | INFO | } | 20260507 23:14:20.950 | INFO | Perf report for the last 5.13632 seconds: | 20260507 23:14:20.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:20.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260507 23:14:20.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:14:20.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:20.950 | INFO | Uploaded 36GiB to GPU in 2.46002s (14.634GiB/s) | 20260507 23:14:20.950 | INFO | Downloaded 34GiB to GPU in 2.49048s (13.652GiB/s) | 20260507 23:14:26.087 | INFO | Analyzing chunk frames = [3360768,3361280) | 20260507 23:14:26.087 | INFO | MemInfo: { | 20260507 23:14:26.087 | INFO | "gpu" : " 17 GB", | 20260507 23:14:26.087 | INFO | "huge" : "", | 20260507 23:14:26.087 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:26.087 | INFO | "malloc" : " 14 GB", | 20260507 23:14:26.087 | INFO | "pinned" : " 29 GB" | 20260507 23:14:26.087 | INFO | } | 20260507 23:14:26.087 | INFO | Perf report for the last 5.1374 seconds: | 20260507 23:14:26.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:26.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.087 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 23:14:26.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.087 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:14:26.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:26.088 | INFO | Uploaded 36GiB to GPU in 2.34748s (15.3356GiB/s) | 20260507 23:14:26.088 | INFO | Downloaded 34GiB to GPU in 2.50662s (13.5641GiB/s) | 20260507 23:14:31.228 | INFO | Analyzing chunk frames = [3361280,3361792) | 20260507 23:14:31.228 | INFO | MemInfo: { | 20260507 23:14:31.228 | INFO | "gpu" : " 17 GB", | 20260507 23:14:31.228 | INFO | "huge" : "", | 20260507 23:14:31.228 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:31.228 | INFO | "malloc" : " 14 GB", | 20260507 23:14:31.228 | INFO | "pinned" : " 29 GB" | 20260507 23:14:31.228 | INFO | } | 20260507 23:14:31.228 | INFO | Perf report for the last 5.14085 seconds: | 20260507 23:14:31.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:31.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 23:14:31.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260507 23:14:31.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:31.228 | INFO | Uploaded 36GiB to GPU in 2.50066s (14.3962GiB/s) | 20260507 23:14:31.228 | INFO | Downloaded 34GiB to GPU in 2.4374s (13.9493GiB/s) | 20260507 23:14:36.362 | INFO | Analyzing chunk frames = [3361792,3362304) | 20260507 23:14:36.362 | INFO | MemInfo: { | 20260507 23:14:36.362 | INFO | "gpu" : " 17 GB", | 20260507 23:14:36.362 | INFO | "huge" : "", | 20260507 23:14:36.362 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:36.362 | INFO | "malloc" : " 14 GB", | 20260507 23:14:36.362 | INFO | "pinned" : " 29 GB" | 20260507 23:14:36.362 | INFO | } | 20260507 23:14:36.362 | INFO | Perf report for the last 5.13425 seconds: | 20260507 23:14:36.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:36.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 23:14:36.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:14:36.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:36.362 | INFO | Uploaded 36GiB to GPU in 2.54364s (14.1529GiB/s) | 20260507 23:14:36.363 | INFO | Downloaded 34GiB to GPU in 2.46754s (13.7789GiB/s) | 20260507 23:14:41.504 | INFO | Analyzing chunk frames = [3362304,3362816) | 20260507 23:14:41.504 | INFO | MemInfo: { | 20260507 23:14:41.504 | INFO | "gpu" : " 17 GB", | 20260507 23:14:41.504 | INFO | "huge" : "", | 20260507 23:14:41.504 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:41.504 | INFO | "malloc" : " 14 GB", | 20260507 23:14:41.504 | INFO | "pinned" : " 29 GB" | 20260507 23:14:41.504 | INFO | } | 20260507 23:14:41.504 | INFO | Perf report for the last 5.14154 seconds: | 20260507 23:14:41.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:41.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:14:41.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:14:41.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:41.504 | INFO | Uploaded 36GiB to GPU in 2.49071s (14.4537GiB/s) | 20260507 23:14:41.504 | INFO | Downloaded 34GiB to GPU in 2.4472s (13.8934GiB/s) | 20260507 23:14:41.601 | INFO | Starting a round of Baz IO | 20260507 23:14:45.071 | INFO | Populating RTMetrics report | 20260507 23:14:45.071 | INFO | RTMetrics report populated | 20260507 23:14:45.071 | INFO | Opening tmp RTMetrics json file | 20260507 23:14:45.078 | INFO | Serializing RTMetrics to json | 20260507 23:14:45.102 | INFO | Serialization done, swapping tmp file now... | 20260507 23:14:45.106 | INFO | RTMetrics done | 20260507 23:14:45.194 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.593 seconds | 20260507 23:14:45.194 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.71% / 0.01% / 57.29% | 20260507 23:14:45.194 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24033% / 3.97364e-06% / 4.37101e-05% | 20260507 23:14:45.194 | INFO | Baz IO overall: | 20260507 23:14:45.194 | INFO | BytesWritten : 1531374171694 | 20260507 23:14:45.194 | INFO | NumEvents : 741670984973 | 20260507 23:14:46.647 | INFO | Analyzing chunk frames = [3362816,3363328) | 20260507 23:14:46.647 | INFO | MemInfo: { | 20260507 23:14:46.647 | INFO | "gpu" : " 17 GB", | 20260507 23:14:46.648 | INFO | "huge" : "", | 20260507 23:14:46.648 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:46.648 | INFO | "malloc" : " 14 GB", | 20260507 23:14:46.648 | INFO | "pinned" : " 29 GB" | 20260507 23:14:46.648 | INFO | } | 20260507 23:14:46.648 | INFO | Perf report for the last 5.14381 seconds: | 20260507 23:14:46.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:46.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:46.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:46.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:46.648 | INFO | ANALYSIS : 68.6%, 3.0, 107.2ms, 0 | 20260507 23:14:46.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:46.648 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 23:14:46.648 | INFO | PRE_HQ : 65.4%, 1.7, 57.4ms, 0 | 20260507 23:14:46.648 | INFO | BAZWRITER : 69.5%, 3.7, 130.5ms, 0 | 20260507 23:14:46.648 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260507 23:14:46.648 | INFO | Uploaded 36GiB to GPU in 2.52203s (14.2742GiB/s) | 20260507 23:14:46.648 | INFO | Downloaded 38GiB to GPU in 2.68933s (14.1299GiB/s) | 20260507 23:14:51.790 | INFO | Analyzing chunk frames = [3363328,3363840) | 20260507 23:14:51.790 | INFO | MemInfo: { | 20260507 23:14:51.790 | INFO | "gpu" : " 17 GB", | 20260507 23:14:51.790 | INFO | "huge" : "", | 20260507 23:14:51.791 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:51.791 | INFO | "malloc" : " 14 GB", | 20260507 23:14:51.791 | INFO | "pinned" : " 29 GB" | 20260507 23:14:51.791 | INFO | } | 20260507 23:14:51.791 | INFO | Perf report for the last 5.14298 seconds: | 20260507 23:14:51.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:51.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 23:14:51.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:14:51.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:51.791 | INFO | Uploaded 36GiB to GPU in 2.58011s (13.9529GiB/s) | 20260507 23:14:51.791 | INFO | Downloaded 34GiB to GPU in 2.49328s (13.6366GiB/s) | 20260507 23:14:56.931 | INFO | Analyzing chunk frames = [3363840,3364352) | 20260507 23:14:56.931 | INFO | MemInfo: { | 20260507 23:14:56.931 | INFO | "gpu" : " 17 GB", | 20260507 23:14:56.931 | INFO | "huge" : "", | 20260507 23:14:56.931 | INFO | "huge_shared" : " 29 GB", | 20260507 23:14:56.931 | INFO | "malloc" : " 14 GB", | 20260507 23:14:56.931 | INFO | "pinned" : " 29 GB" | 20260507 23:14:56.931 | INFO | } | 20260507 23:14:56.931 | INFO | Perf report for the last 5.14034 seconds: | 20260507 23:14:56.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:14:56.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.931 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 23:14:56.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.931 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:14:56.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:14:56.932 | INFO | Uploaded 36GiB to GPU in 2.50743s (14.3574GiB/s) | 20260507 23:14:56.932 | INFO | Downloaded 34GiB to GPU in 2.49452s (13.6299GiB/s) | 20260507 23:15:02.063 | INFO | Analyzing chunk frames = [3364352,3364864) | 20260507 23:15:02.063 | INFO | MemInfo: { | 20260507 23:15:02.063 | INFO | "gpu" : " 17 GB", | 20260507 23:15:02.063 | INFO | "huge" : "", | 20260507 23:15:02.064 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:02.064 | INFO | "malloc" : " 14 GB", | 20260507 23:15:02.064 | INFO | "pinned" : " 29 GB" | 20260507 23:15:02.064 | INFO | } | 20260507 23:15:02.064 | INFO | Perf report for the last 5.13261 seconds: | 20260507 23:15:02.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:02.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260507 23:15:02.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:15:02.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:02.064 | INFO | Uploaded 36GiB to GPU in 2.50731s (14.358GiB/s) | 20260507 23:15:02.064 | INFO | Downloaded 34GiB to GPU in 2.49565s (13.6237GiB/s) | 20260507 23:15:07.204 | INFO | Analyzing chunk frames = [3364864,3365376) | 20260507 23:15:07.204 | INFO | MemInfo: { | 20260507 23:15:07.204 | INFO | "gpu" : " 17 GB", | 20260507 23:15:07.204 | INFO | "huge" : "", | 20260507 23:15:07.204 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:07.204 | INFO | "malloc" : " 14 GB", | 20260507 23:15:07.204 | INFO | "pinned" : " 29 GB" | 20260507 23:15:07.204 | INFO | } | 20260507 23:15:07.204 | INFO | Perf report for the last 5.14074 seconds: | 20260507 23:15:07.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:07.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.204 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 23:15:07.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.205 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 23:15:07.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:07.205 | INFO | Uploaded 36GiB to GPU in 2.34938s (15.3232GiB/s) | 20260507 23:15:07.205 | INFO | Downloaded 34GiB to GPU in 2.5042s (13.5772GiB/s) | 20260507 23:15:12.345 | INFO | Analyzing chunk frames = [3365376,3365888) | 20260507 23:15:12.345 | INFO | MemInfo: { | 20260507 23:15:12.345 | INFO | "gpu" : " 17 GB", | 20260507 23:15:12.345 | INFO | "huge" : "", | 20260507 23:15:12.345 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:12.345 | INFO | "malloc" : " 14 GB", | 20260507 23:15:12.345 | INFO | "pinned" : " 29 GB" | 20260507 23:15:12.345 | INFO | } | 20260507 23:15:12.345 | INFO | Perf report for the last 5.1404 seconds: | 20260507 23:15:12.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:12.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 23:15:12.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:15:12.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:12.345 | INFO | Uploaded 36GiB to GPU in 2.50805s (14.3538GiB/s) | 20260507 23:15:12.345 | INFO | Downloaded 34GiB to GPU in 2.4359s (13.9579GiB/s) | 20260507 23:15:17.487 | INFO | Analyzing chunk frames = [3365888,3366400) | 20260507 23:15:17.487 | INFO | MemInfo: { | 20260507 23:15:17.487 | INFO | "gpu" : " 17 GB", | 20260507 23:15:17.487 | INFO | "huge" : "", | 20260507 23:15:17.487 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:17.487 | INFO | "malloc" : " 14 GB", | 20260507 23:15:17.487 | INFO | "pinned" : " 29 GB" | 20260507 23:15:17.487 | INFO | } | 20260507 23:15:17.487 | INFO | Perf report for the last 5.14262 seconds: | 20260507 23:15:17.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:17.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.488 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:15:17.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.488 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:15:17.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:17.488 | INFO | Uploaded 36GiB to GPU in 2.53648s (14.1929GiB/s) | 20260507 23:15:17.488 | INFO | Downloaded 34GiB to GPU in 2.46727s (13.7804GiB/s) | 20260507 23:15:22.624 | INFO | Analyzing chunk frames = [3366400,3366912) | 20260507 23:15:22.624 | INFO | MemInfo: { | 20260507 23:15:22.624 | INFO | "gpu" : " 17 GB", | 20260507 23:15:22.624 | INFO | "huge" : "", | 20260507 23:15:22.624 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:22.624 | INFO | "malloc" : " 14 GB", | 20260507 23:15:22.624 | INFO | "pinned" : " 29 GB" | 20260507 23:15:22.624 | INFO | } | 20260507 23:15:22.624 | INFO | Perf report for the last 5.13663 seconds: | 20260507 23:15:22.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:22.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:15:22.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:15:22.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:22.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:15:22.624 | INFO | Total mem usage: 16.7 GiB | 20260507 23:15:22.624 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:15:22.624 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.05%) | 20260507 23:15:22.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:15:22.624 | INFO | High Water Breakdown: | 20260507 23:15:22.624 | INFO | 60.5% : 9.25 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:15:22.624 | INFO | 39.4% : 6.03 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:15:22.624 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:15:22.624 | INFO | ------------------------------------------ | 20260507 23:15:22.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:15:22.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:15:22.624 | INFO | Total mem usage: 26.5 GiB | 20260507 23:15:22.624 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:15:22.625 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:15:22.625 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:15:22.625 | INFO | High Water Breakdown: | 20260507 23:15:22.625 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:15:22.625 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:15:22.625 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:15:22.625 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:15:22.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:15:22.625 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:15:22.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:15:22.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:15:22.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:15:22.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:15:22.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:15:22.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:15:22.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:15:22.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:15:22.625 | INFO | ------------------------------------------ | 20260507 23:15:22.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:15:22.625 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:15:22.625 | INFO | Total mem usage: 27.8 GiB | 20260507 23:15:22.625 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:15:22.625 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:15:22.625 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:15:22.625 | INFO | High Water Breakdown: | 20260507 23:15:22.625 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:15:22.625 | INFO | ------------------------------------------ | 20260507 23:15:22.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:15:22.625 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:15:22.625 | INFO | Total mem usage: 15.7 GiB | 20260507 23:15:22.625 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:15:22.625 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:15:22.625 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:15:22.625 | INFO | High Water Breakdown: | 20260507 23:15:22.625 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:15:22.625 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:15:22.625 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:15:22.625 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:15:22.625 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:15:22.625 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:15:22.625 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:15:22.625 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:15:22.625 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:15:22.625 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:15:22.625 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:15:22.625 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:15:22.625 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:15:22.625 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:15:22.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:15:22.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:15:22.625 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:15:22.625 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:15:22.625 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:15:22.625 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:15:22.625 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:15:22.625 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:15:22.625 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:15:22.625 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:15:22.625 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:15:22.625 | INFO | ------------------------------------------ | 20260507 23:15:22.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:15:22.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:15:22.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:15:22.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:15:22.625 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.40e+01 / 1.63e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.20e-02 / 6.15e-03 | 20260507 23:15:22.625 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.50e+00, 5.89e-01 / 1.82e-01 | 20260507 23:15:22.625 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.05e-01 / 1.17e-01 | 20260507 23:15:22.625 | INFO | PerfReport: 16.17%, DME -- 4.24e+01 / 4.64e+01, 4.21e+00 / 1.93e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.65e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.47e+00 / 2.75e+00, 4.35e-01 / 2.34e-01 | 20260507 23:15:22.625 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.35e+00, 2.81e-01 / 1.58e-01 | 20260507 23:15:22.625 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.47e-01 / 3.79e-01, 5.53e-02 / 2.38e-02 | 20260507 23:15:22.625 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:15:22.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:15:22.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:15:22.625 | INFO | Uploaded 36GiB to GPU in 2.49279s (14.4416GiB/s) | 20260507 23:15:22.625 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 24.30%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 24.39%, Download -- 2.61e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 30.67%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:15:22.625 | INFO | Downloaded 34GiB to GPU in 2.44627s (13.8987GiB/s) | 20260507 23:15:22.625 | INFO | PerfReport: 2.08%, LockUpload -- 2.22e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 2.88%, LockDownload -- 3.08e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 23:15:22.625 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260507 23:15:22.722 | INFO | Starting a round of Baz IO | 20260507 23:15:26.194 | INFO | Populating RTMetrics report | 20260507 23:15:26.195 | INFO | RTMetrics report populated | 20260507 23:15:26.195 | INFO | Opening tmp RTMetrics json file | 20260507 23:15:26.232 | INFO | Serializing RTMetrics to json | 20260507 23:15:26.273 | INFO | Serialization done, swapping tmp file now... | 20260507 23:15:26.277 | INFO | RTMetrics done | 20260507 23:15:26.301 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.579 seconds | 20260507 23:15:26.301 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.70% / 0.01% / 57.29% | 20260507 23:15:26.301 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23678% / 1.19209e-05% / 5.16574e-05% | 20260507 23:15:26.301 | INFO | Baz IO overall: | 20260507 23:15:26.301 | INFO | BytesWritten : 1533220043017 | 20260507 23:15:26.301 | INFO | NumEvents : 742561327985 | 20260507 23:15:27.760 | INFO | Analyzing chunk frames = [3366912,3367424) | 20260507 23:15:27.760 | INFO | MemInfo: { | 20260507 23:15:27.760 | INFO | "gpu" : " 17 GB", | 20260507 23:15:27.760 | INFO | "huge" : "", | 20260507 23:15:27.760 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:27.760 | INFO | "malloc" : " 14 GB", | 20260507 23:15:27.760 | INFO | "pinned" : " 29 GB" | 20260507 23:15:27.760 | INFO | } | 20260507 23:15:27.760 | INFO | Perf report for the last 5.13553 seconds: | 20260507 23:15:27.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:27.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:27.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:27.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:27.760 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 23:15:27.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:27.760 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 23:15:27.760 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260507 23:15:27.760 | INFO | BAZWRITER : 69.3%, 3.9, 138.3ms, 0 | 20260507 23:15:27.760 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 23:15:27.760 | INFO | Uploaded 36GiB to GPU in 2.54886s (14.124GiB/s) | 20260507 23:15:27.760 | INFO | Downloaded 38GiB to GPU in 2.68243s (14.1663GiB/s) | 20260507 23:15:32.899 | INFO | Analyzing chunk frames = [3367424,3367936) | 20260507 23:15:32.899 | INFO | MemInfo: { | 20260507 23:15:32.899 | INFO | "gpu" : " 17 GB", | 20260507 23:15:32.899 | INFO | "huge" : "", | 20260507 23:15:32.899 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:32.899 | INFO | "malloc" : " 14 GB", | 20260507 23:15:32.899 | INFO | "pinned" : " 29 GB" | 20260507 23:15:32.899 | INFO | } | 20260507 23:15:32.899 | INFO | Perf report for the last 5.13967 seconds: | 20260507 23:15:32.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:32.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.899 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:15:32.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.899 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:15:32.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:32.900 | INFO | Uploaded 36GiB to GPU in 2.553s (14.1011GiB/s) | 20260507 23:15:32.900 | INFO | Downloaded 34GiB to GPU in 2.48417s (13.6867GiB/s) | 20260507 23:15:38.039 | INFO | Analyzing chunk frames = [3367936,3368448) | 20260507 23:15:38.039 | INFO | MemInfo: { | 20260507 23:15:38.039 | INFO | "gpu" : " 17 GB", | 20260507 23:15:38.039 | INFO | "huge" : "", | 20260507 23:15:38.039 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:38.039 | INFO | "malloc" : " 14 GB", | 20260507 23:15:38.039 | INFO | "pinned" : " 29 GB" | 20260507 23:15:38.039 | INFO | } | 20260507 23:15:38.039 | INFO | Perf report for the last 5.13949 seconds: | 20260507 23:15:38.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:38.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 23:15:38.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:15:38.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:38.039 | INFO | Uploaded 36GiB to GPU in 2.56858s (14.0155GiB/s) | 20260507 23:15:38.039 | INFO | Downloaded 34GiB to GPU in 2.47681s (13.7273GiB/s) | 20260507 23:15:43.178 | INFO | Analyzing chunk frames = [3368448,3368960) | 20260507 23:15:43.178 | INFO | MemInfo: { | 20260507 23:15:43.178 | INFO | "gpu" : " 17 GB", | 20260507 23:15:43.178 | INFO | "huge" : "", | 20260507 23:15:43.178 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:43.178 | INFO | "malloc" : " 14 GB", | 20260507 23:15:43.178 | INFO | "pinned" : " 29 GB" | 20260507 23:15:43.178 | INFO | } | 20260507 23:15:43.178 | INFO | Perf report for the last 5.13901 seconds: | 20260507 23:15:43.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:43.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260507 23:15:43.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:15:43.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:43.178 | INFO | Uploaded 36GiB to GPU in 2.46891s (14.5813GiB/s) | 20260507 23:15:43.178 | INFO | Downloaded 34GiB to GPU in 2.40698s (14.1256GiB/s) | 20260507 23:15:48.316 | INFO | Analyzing chunk frames = [3368960,3369472) | 20260507 23:15:48.316 | INFO | MemInfo: { | 20260507 23:15:48.316 | INFO | "gpu" : " 17 GB", | 20260507 23:15:48.316 | INFO | "huge" : "", | 20260507 23:15:48.316 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:48.316 | INFO | "malloc" : " 14 GB", | 20260507 23:15:48.316 | INFO | "pinned" : " 29 GB" | 20260507 23:15:48.316 | INFO | } | 20260507 23:15:48.316 | INFO | Perf report for the last 5.13852 seconds: | 20260507 23:15:48.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:48.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.316 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:15:48.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.316 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 23:15:48.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:48.317 | INFO | Uploaded 36GiB to GPU in 2.33746s (15.4013GiB/s) | 20260507 23:15:48.317 | INFO | Downloaded 34GiB to GPU in 2.50856s (13.5536GiB/s) | 20260507 23:15:53.457 | INFO | Analyzing chunk frames = [3369472,3369984) | 20260507 23:15:53.457 | INFO | MemInfo: { | 20260507 23:15:53.457 | INFO | "gpu" : " 17 GB", | 20260507 23:15:53.457 | INFO | "huge" : "", | 20260507 23:15:53.457 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:53.457 | INFO | "malloc" : " 14 GB", | 20260507 23:15:53.457 | INFO | "pinned" : " 29 GB" | 20260507 23:15:53.457 | INFO | } | 20260507 23:15:53.457 | INFO | Perf report for the last 5.14049 seconds: | 20260507 23:15:53.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:53.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:15:53.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:15:53.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:53.457 | INFO | Uploaded 36GiB to GPU in 2.39641s (15.0225GiB/s) | 20260507 23:15:53.457 | INFO | Downloaded 34GiB to GPU in 2.41225s (14.0947GiB/s) | 20260507 23:15:58.602 | INFO | Analyzing chunk frames = [3369984,3370496) | 20260507 23:15:58.602 | INFO | MemInfo: { | 20260507 23:15:58.602 | INFO | "gpu" : " 17 GB", | 20260507 23:15:58.602 | INFO | "huge" : "", | 20260507 23:15:58.602 | INFO | "huge_shared" : " 29 GB", | 20260507 23:15:58.602 | INFO | "malloc" : " 14 GB", | 20260507 23:15:58.602 | INFO | "pinned" : " 29 GB" | 20260507 23:15:58.602 | INFO | } | 20260507 23:15:58.602 | INFO | Perf report for the last 5.14499 seconds: | 20260507 23:15:58.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:15:58.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:15:58.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:15:58.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:15:58.602 | INFO | Uploaded 36GiB to GPU in 2.5252s (14.2563GiB/s) | 20260507 23:15:58.602 | INFO | Downloaded 34GiB to GPU in 2.46869s (13.7725GiB/s) | 20260507 23:16:03.742 | INFO | Analyzing chunk frames = [3370496,3371008) | 20260507 23:16:03.742 | INFO | MemInfo: { | 20260507 23:16:03.742 | INFO | "gpu" : " 17 GB", | 20260507 23:16:03.743 | INFO | "huge" : "", | 20260507 23:16:03.743 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:03.743 | INFO | "malloc" : " 14 GB", | 20260507 23:16:03.743 | INFO | "pinned" : " 29 GB" | 20260507 23:16:03.743 | INFO | } | 20260507 23:16:03.743 | INFO | Perf report for the last 5.14082 seconds: | 20260507 23:16:03.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:03.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:16:03.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:16:03.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:03.743 | INFO | Uploaded 36GiB to GPU in 2.49707s (14.4169GiB/s) | 20260507 23:16:03.743 | INFO | Downloaded 34GiB to GPU in 2.44477s (13.9073GiB/s) | 20260507 23:16:03.840 | INFO | Starting a round of Baz IO | 20260507 23:16:07.306 | INFO | Populating RTMetrics report | 20260507 23:16:07.307 | INFO | RTMetrics report populated | 20260507 23:16:07.307 | INFO | Opening tmp RTMetrics json file | 20260507 23:16:07.320 | INFO | Serializing RTMetrics to json | 20260507 23:16:07.375 | INFO | Serialization done, swapping tmp file now... | 20260507 23:16:07.416 | INFO | RTMetrics done | 20260507 23:16:07.453 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.613 seconds | 20260507 23:16:07.453 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.70% / 0.01% / 57.30% | 20260507 23:16:07.453 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23269% / 7.94729e-06% / 5.5631e-05% | 20260507 23:16:07.453 | INFO | Baz IO overall: | 20260507 23:16:07.453 | INFO | BytesWritten : 1535064446594 | 20260507 23:16:07.453 | INFO | NumEvents : 743450699647 | 20260507 23:16:08.879 | INFO | Analyzing chunk frames = [3371008,3371520) | 20260507 23:16:08.879 | INFO | MemInfo: { | 20260507 23:16:08.879 | INFO | "gpu" : " 17 GB", | 20260507 23:16:08.879 | INFO | "huge" : "", | 20260507 23:16:08.879 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:08.879 | INFO | "malloc" : " 14 GB", | 20260507 23:16:08.879 | INFO | "pinned" : " 29 GB" | 20260507 23:16:08.879 | INFO | } | 20260507 23:16:08.880 | INFO | Perf report for the last 5.1368 seconds: | 20260507 23:16:08.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:08.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:08.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:08.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:08.880 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 23:16:08.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:08.880 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 23:16:08.880 | INFO | PRE_HQ : 65.7%, 1.7, 56.8ms, 0 | 20260507 23:16:08.880 | INFO | BAZWRITER : 70.0%, 4.1, 147.0ms, 0 | 20260507 23:16:08.880 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 23:16:08.880 | INFO | Uploaded 36GiB to GPU in 2.59231s (13.8872GiB/s) | 20260507 23:16:08.880 | INFO | Downloaded 38GiB to GPU in 2.69285s (14.1115GiB/s) | 20260507 23:16:14.017 | INFO | Analyzing chunk frames = [3371520,3372032) | 20260507 23:16:14.017 | INFO | MemInfo: { | 20260507 23:16:14.017 | INFO | "gpu" : " 17 GB", | 20260507 23:16:14.017 | INFO | "huge" : "", | 20260507 23:16:14.017 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:14.017 | INFO | "malloc" : " 14 GB", | 20260507 23:16:14.017 | INFO | "pinned" : " 29 GB" | 20260507 23:16:14.017 | INFO | } | 20260507 23:16:14.017 | INFO | Perf report for the last 5.13723 seconds: | 20260507 23:16:14.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:14.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 23:16:14.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:16:14.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:14.017 | INFO | Uploaded 36GiB to GPU in 2.57565s (13.9771GiB/s) | 20260507 23:16:14.017 | INFO | Downloaded 34GiB to GPU in 2.49534s (13.6254GiB/s) | 20260507 23:16:19.157 | INFO | Analyzing chunk frames = [3372032,3372544) | 20260507 23:16:19.157 | INFO | MemInfo: { | 20260507 23:16:19.157 | INFO | "gpu" : " 17 GB", | 20260507 23:16:19.157 | INFO | "huge" : "", | 20260507 23:16:19.157 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:19.157 | INFO | "malloc" : " 14 GB", | 20260507 23:16:19.157 | INFO | "pinned" : " 29 GB" | 20260507 23:16:19.157 | INFO | } | 20260507 23:16:19.157 | INFO | Perf report for the last 5.14013 seconds: | 20260507 23:16:19.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:19.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:16:19.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:16:19.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:19.157 | INFO | Uploaded 36GiB to GPU in 2.50557s (14.368GiB/s) | 20260507 23:16:19.157 | INFO | Downloaded 34GiB to GPU in 2.48902s (13.66GiB/s) | 20260507 23:16:24.294 | INFO | Analyzing chunk frames = [3372544,3373056) | 20260507 23:16:24.294 | INFO | MemInfo: { | 20260507 23:16:24.294 | INFO | "gpu" : " 17 GB", | 20260507 23:16:24.294 | INFO | "huge" : "", | 20260507 23:16:24.295 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:24.295 | INFO | "malloc" : " 14 GB", | 20260507 23:16:24.295 | INFO | "pinned" : " 29 GB" | 20260507 23:16:24.295 | INFO | } | 20260507 23:16:24.295 | INFO | Perf report for the last 5.13777 seconds: | 20260507 23:16:24.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:24.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 23:16:24.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:16:24.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:24.295 | INFO | Uploaded 36GiB to GPU in 2.48166s (14.5064GiB/s) | 20260507 23:16:24.295 | INFO | Downloaded 34GiB to GPU in 2.40984s (14.1088GiB/s) | 20260507 23:16:29.434 | INFO | Analyzing chunk frames = [3373056,3373568) | 20260507 23:16:29.434 | INFO | MemInfo: { | 20260507 23:16:29.434 | INFO | "gpu" : " 17 GB", | 20260507 23:16:29.434 | INFO | "huge" : "", | 20260507 23:16:29.434 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:29.434 | INFO | "malloc" : " 14 GB", | 20260507 23:16:29.434 | INFO | "pinned" : " 29 GB" | 20260507 23:16:29.434 | INFO | } | 20260507 23:16:29.434 | INFO | Perf report for the last 5.13927 seconds: | 20260507 23:16:29.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:29.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:16:29.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:16:29.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:29.434 | INFO | Uploaded 36GiB to GPU in 2.32519s (15.4826GiB/s) | 20260507 23:16:29.434 | INFO | Downloaded 34GiB to GPU in 2.50994s (13.5462GiB/s) | 20260507 23:16:34.571 | INFO | Analyzing chunk frames = [3373568,3374080) | 20260507 23:16:34.571 | INFO | MemInfo: { | 20260507 23:16:34.571 | INFO | "gpu" : " 17 GB", | 20260507 23:16:34.571 | INFO | "huge" : "", | 20260507 23:16:34.571 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:34.571 | INFO | "malloc" : " 14 GB", | 20260507 23:16:34.571 | INFO | "pinned" : " 29 GB" | 20260507 23:16:34.571 | INFO | } | 20260507 23:16:34.571 | INFO | Perf report for the last 5.13707 seconds: | 20260507 23:16:34.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:34.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.571 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 23:16:34.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.571 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:16:34.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:34.572 | INFO | Uploaded 36GiB to GPU in 2.40813s (14.9494GiB/s) | 20260507 23:16:34.572 | INFO | Downloaded 34GiB to GPU in 2.41331s (14.0885GiB/s) | 20260507 23:16:39.712 | INFO | Analyzing chunk frames = [3374080,3374592) | 20260507 23:16:39.712 | INFO | MemInfo: { | 20260507 23:16:39.712 | INFO | "gpu" : " 17 GB", | 20260507 23:16:39.712 | INFO | "huge" : "", | 20260507 23:16:39.712 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:39.712 | INFO | "malloc" : " 14 GB", | 20260507 23:16:39.712 | INFO | "pinned" : " 29 GB" | 20260507 23:16:39.712 | INFO | } | 20260507 23:16:39.712 | INFO | Perf report for the last 5.14137 seconds: | 20260507 23:16:39.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:39.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.712 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:16:39.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.713 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:16:39.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:39.713 | INFO | Uploaded 36GiB to GPU in 2.52183s (14.2753GiB/s) | 20260507 23:16:39.713 | INFO | Downloaded 34GiB to GPU in 2.46383s (13.7997GiB/s) | 20260507 23:16:44.851 | INFO | Analyzing chunk frames = [3374592,3375104) | 20260507 23:16:44.851 | INFO | MemInfo: { | 20260507 23:16:44.851 | INFO | "gpu" : " 17 GB", | 20260507 23:16:44.851 | INFO | "huge" : "", | 20260507 23:16:44.851 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:44.851 | INFO | "malloc" : " 14 GB", | 20260507 23:16:44.851 | INFO | "pinned" : " 29 GB" | 20260507 23:16:44.851 | INFO | } | 20260507 23:16:44.851 | INFO | Perf report for the last 5.13885 seconds: | 20260507 23:16:44.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:44.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.851 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:16:44.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.851 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:16:44.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:44.852 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:16:44.852 | INFO | Total mem usage: 16.7 GiB | 20260507 23:16:44.852 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:16:44.852 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260507 23:16:44.852 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:16:44.852 | INFO | High Water Breakdown: | 20260507 23:16:44.852 | INFO | 59.9% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:16:44.852 | INFO | 39.9% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:16:44.852 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:16:44.852 | INFO | ------------------------------------------ | 20260507 23:16:44.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:16:44.852 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:16:44.852 | INFO | Total mem usage: 26.5 GiB | 20260507 23:16:44.852 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:16:44.852 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:16:44.852 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260507 23:16:44.852 | INFO | High Water Breakdown: | 20260507 23:16:44.852 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:16:44.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:16:44.852 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:16:44.852 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:16:44.852 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:16:44.852 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:16:44.852 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:16:44.852 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:16:44.852 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:16:44.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:16:44.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:16:44.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:16:44.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:16:44.852 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:16:44.852 | INFO | ------------------------------------------ | 20260507 23:16:44.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:16:44.852 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:16:44.852 | INFO | Total mem usage: 27.8 GiB | 20260507 23:16:44.852 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:16:44.852 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:16:44.852 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:16:44.852 | INFO | High Water Breakdown: | 20260507 23:16:44.852 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:16:44.852 | INFO | ------------------------------------------ | 20260507 23:16:44.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:16:44.852 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:16:44.852 | INFO | Total mem usage: 15.7 GiB | 20260507 23:16:44.852 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:16:44.852 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:16:44.852 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:16:44.852 | INFO | High Water Breakdown: | 20260507 23:16:44.852 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:16:44.852 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:16:44.852 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:16:44.852 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:16:44.852 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:16:44.852 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:16:44.852 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:16:44.852 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:16:44.852 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:16:44.852 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:16:44.852 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:16:44.852 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:16:44.852 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:16:44.852 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:16:44.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:16:44.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:16:44.852 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:16:44.852 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:16:44.852 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:16:44.852 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:16:44.852 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:16:44.852 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:16:44.852 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:16:44.852 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:16:44.852 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:16:44.852 | INFO | ------------------------------------------ | 20260507 23:16:44.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:16:44.852 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:16:44.852 | INFO | Uploaded 36GiB to GPU in 2.49974s (14.4015GiB/s) | 20260507 23:16:44.852 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:16:44.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:16:44.852 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.41e+01 / 1.61e+00 | 20260507 23:16:44.852 | INFO | Downloaded 34GiB to GPU in 2.44879s (13.8844GiB/s) | 20260507 23:16:44.852 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.04e-01, 1.19e-02 / 8.90e-03 | 20260507 23:16:44.853 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.51e+00, 5.97e-01 / 1.67e-01 | 20260507 23:16:44.853 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.50e+00, 2.07e-01 / 1.03e-01 | 20260507 23:16:44.853 | INFO | PerfReport: 15.93%, DME -- 4.19e+01 / 4.55e+01, 3.87e+00 / 1.47e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.73e+00 / 1.81e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.31e-01 / 2.33e-01 | 20260507 23:16:44.853 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.35e+00, 2.86e-01 / 1.67e-01 | 20260507 23:16:44.853 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.85e-01, 5.58e-02 / 2.99e-02 | 20260507 23:16:44.853 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:16:44.853 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:16:44.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:16:44.853 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 30.87%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 1.99%, LockUpload -- 2.12e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 2.58%, LockDownload -- 2.75e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 15.91%, LockCompute -- 1.70e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 23:16:44.853 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260507 23:16:44.950 | INFO | Starting a round of Baz IO | 20260507 23:16:48.414 | INFO | Populating RTMetrics report | 20260507 23:16:48.414 | INFO | RTMetrics report populated | 20260507 23:16:48.414 | INFO | Opening tmp RTMetrics json file | 20260507 23:16:48.438 | INFO | Serializing RTMetrics to json | 20260507 23:16:48.439 | INFO | Serialization done, swapping tmp file now... | 20260507 23:16:48.440 | INFO | RTMetrics done | 20260507 23:16:48.531 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.581 seconds | 20260507 23:16:48.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.69% / 0.01% / 57.30% | 20260507 23:16:48.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23066% / 7.94729e-06% / 5.96046e-05% | 20260507 23:16:48.531 | INFO | Baz IO overall: | 20260507 23:16:48.531 | INFO | BytesWritten : 1536908836140 | 20260507 23:16:48.531 | INFO | NumEvents : 744339993876 | 20260507 23:16:49.989 | INFO | Analyzing chunk frames = [3375104,3375616) | 20260507 23:16:49.989 | INFO | MemInfo: { | 20260507 23:16:49.989 | INFO | "gpu" : " 17 GB", | 20260507 23:16:49.989 | INFO | "huge" : "", | 20260507 23:16:49.989 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:49.989 | INFO | "malloc" : " 14 GB", | 20260507 23:16:49.989 | INFO | "pinned" : " 29 GB" | 20260507 23:16:49.989 | INFO | } | 20260507 23:16:49.989 | INFO | Perf report for the last 5.1379 seconds: | 20260507 23:16:49.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:49.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:49.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:49.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:49.989 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 23:16:49.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:49.989 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:16:49.989 | INFO | PRE_HQ : 65.9%, 1.7, 57.1ms, 0 | 20260507 23:16:49.989 | INFO | BAZWRITER : 69.3%, 4.0, 142.5ms, 0 | 20260507 23:16:49.989 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:16:49.990 | INFO | Uploaded 36GiB to GPU in 2.54118s (14.1666GiB/s) | 20260507 23:16:49.990 | INFO | Downloaded 38GiB to GPU in 2.68889s (14.1322GiB/s) | 20260507 23:16:55.125 | INFO | Analyzing chunk frames = [3375616,3376128) | 20260507 23:16:55.125 | INFO | MemInfo: { | 20260507 23:16:55.125 | INFO | "gpu" : " 17 GB", | 20260507 23:16:55.125 | INFO | "huge" : "", | 20260507 23:16:55.125 | INFO | "huge_shared" : " 29 GB", | 20260507 23:16:55.125 | INFO | "malloc" : " 14 GB", | 20260507 23:16:55.125 | INFO | "pinned" : " 29 GB" | 20260507 23:16:55.125 | INFO | } | 20260507 23:16:55.125 | INFO | Perf report for the last 5.13583 seconds: | 20260507 23:16:55.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:16:55.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 23:16:55.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:16:55.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:16:55.125 | INFO | Uploaded 36GiB to GPU in 2.54417s (14.15GiB/s) | 20260507 23:16:55.125 | INFO | Downloaded 34GiB to GPU in 2.47967s (13.7115GiB/s) | 20260507 23:17:00.266 | INFO | Analyzing chunk frames = [3376128,3376640) | 20260507 23:17:00.266 | INFO | MemInfo: { | 20260507 23:17:00.266 | INFO | "gpu" : " 17 GB", | 20260507 23:17:00.266 | INFO | "huge" : "", | 20260507 23:17:00.266 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:00.266 | INFO | "malloc" : " 14 GB", | 20260507 23:17:00.266 | INFO | "pinned" : " 29 GB" | 20260507 23:17:00.266 | INFO | } | 20260507 23:17:00.266 | INFO | Perf report for the last 5.14139 seconds: | 20260507 23:17:00.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:00.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.267 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:17:00.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.267 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:17:00.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:00.267 | INFO | Uploaded 36GiB to GPU in 2.51951s (14.2885GiB/s) | 20260507 23:17:00.267 | INFO | Downloaded 34GiB to GPU in 2.45686s (13.8388GiB/s) | 20260507 23:17:05.409 | INFO | Analyzing chunk frames = [3376640,3377152) | 20260507 23:17:05.409 | INFO | MemInfo: { | 20260507 23:17:05.409 | INFO | "gpu" : " 17 GB", | 20260507 23:17:05.409 | INFO | "huge" : "", | 20260507 23:17:05.409 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:05.409 | INFO | "malloc" : " 14 GB", | 20260507 23:17:05.409 | INFO | "pinned" : " 29 GB" | 20260507 23:17:05.409 | INFO | } | 20260507 23:17:05.409 | INFO | Perf report for the last 5.1429 seconds: | 20260507 23:17:05.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:05.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.409 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:17:05.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.409 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:17:05.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:05.410 | INFO | Uploaded 36GiB to GPU in 2.47329s (14.5555GiB/s) | 20260507 23:17:05.410 | INFO | Downloaded 34GiB to GPU in 2.40056s (14.1634GiB/s) | 20260507 23:17:10.550 | INFO | Analyzing chunk frames = [3377152,3377664) | 20260507 23:17:10.550 | INFO | MemInfo: { | 20260507 23:17:10.550 | INFO | "gpu" : " 17 GB", | 20260507 23:17:10.550 | INFO | "huge" : "", | 20260507 23:17:10.550 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:10.550 | INFO | "malloc" : " 14 GB", | 20260507 23:17:10.550 | INFO | "pinned" : " 29 GB" | 20260507 23:17:10.550 | INFO | } | 20260507 23:17:10.550 | INFO | Perf report for the last 5.14096 seconds: | 20260507 23:17:10.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:10.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.550 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 23:17:10.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.550 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:17:10.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:10.551 | INFO | Uploaded 36GiB to GPU in 2.34004s (15.3844GiB/s) | 20260507 23:17:10.551 | INFO | Downloaded 34GiB to GPU in 2.50782s (13.5576GiB/s) | 20260507 23:17:15.691 | INFO | Analyzing chunk frames = [3377664,3378176) | 20260507 23:17:15.691 | INFO | MemInfo: { | 20260507 23:17:15.691 | INFO | "gpu" : " 17 GB", | 20260507 23:17:15.691 | INFO | "huge" : "", | 20260507 23:17:15.691 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:15.691 | INFO | "malloc" : " 14 GB", | 20260507 23:17:15.691 | INFO | "pinned" : " 29 GB" | 20260507 23:17:15.691 | INFO | } | 20260507 23:17:15.691 | INFO | Perf report for the last 5.14054 seconds: | 20260507 23:17:15.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:15.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:17:15.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:17:15.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:15.691 | INFO | Uploaded 36GiB to GPU in 2.38358s (15.1034GiB/s) | 20260507 23:17:15.691 | INFO | Downloaded 34GiB to GPU in 2.4243s (14.0247GiB/s) | 20260507 23:17:20.830 | INFO | Analyzing chunk frames = [3378176,3378688) | 20260507 23:17:20.830 | INFO | MemInfo: { | 20260507 23:17:20.830 | INFO | "gpu" : " 17 GB", | 20260507 23:17:20.830 | INFO | "huge" : "", | 20260507 23:17:20.830 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:20.830 | INFO | "malloc" : " 14 GB", | 20260507 23:17:20.830 | INFO | "pinned" : " 29 GB" | 20260507 23:17:20.830 | INFO | } | 20260507 23:17:20.830 | INFO | Perf report for the last 5.13889 seconds: | 20260507 23:17:20.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:20.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:17:20.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 23:17:20.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:20.830 | INFO | Uploaded 36GiB to GPU in 2.54108s (14.1672GiB/s) | 20260507 23:17:20.830 | INFO | Downloaded 34GiB to GPU in 2.47722s (13.725GiB/s) | 20260507 23:17:25.969 | INFO | Analyzing chunk frames = [3378688,3379200) | 20260507 23:17:25.969 | INFO | MemInfo: { | 20260507 23:17:25.969 | INFO | "gpu" : " 17 GB", | 20260507 23:17:25.969 | INFO | "huge" : "", | 20260507 23:17:25.969 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:25.969 | INFO | "malloc" : " 14 GB", | 20260507 23:17:25.969 | INFO | "pinned" : " 29 GB" | 20260507 23:17:25.969 | INFO | } | 20260507 23:17:25.969 | INFO | Perf report for the last 5.13952 seconds: | 20260507 23:17:25.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:25.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.969 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260507 23:17:25.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.969 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:17:25.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:25.970 | INFO | Uploaded 36GiB to GPU in 2.44034s (14.752GiB/s) | 20260507 23:17:25.970 | INFO | Downloaded 34GiB to GPU in 2.4535s (13.8578GiB/s) | 20260507 23:17:26.066 | INFO | Starting a round of Baz IO | 20260507 23:17:29.539 | INFO | Populating RTMetrics report | 20260507 23:17:29.539 | INFO | RTMetrics report populated | 20260507 23:17:29.539 | INFO | Opening tmp RTMetrics json file | 20260507 23:17:29.558 | INFO | Serializing RTMetrics to json | 20260507 23:17:29.561 | INFO | Serialization done, swapping tmp file now... | 20260507 23:17:29.563 | INFO | RTMetrics done | 20260507 23:17:29.637 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.57 seconds | 20260507 23:17:29.637 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.69% / 0.01% / 57.31% | 20260507 23:17:29.637 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.227% / 1.19209e-05% / 5.96046e-05% | 20260507 23:17:29.637 | INFO | Baz IO overall: | 20260507 23:17:29.637 | INFO | BytesWritten : 1538752844583 | 20260507 23:17:29.637 | INFO | NumEvents : 745228990129 | 20260507 23:17:31.108 | INFO | Analyzing chunk frames = [3379200,3379712) | 20260507 23:17:31.108 | INFO | MemInfo: { | 20260507 23:17:31.108 | INFO | "gpu" : " 17 GB", | 20260507 23:17:31.108 | INFO | "huge" : "", | 20260507 23:17:31.108 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:31.108 | INFO | "malloc" : " 14 GB", | 20260507 23:17:31.108 | INFO | "pinned" : " 29 GB" | 20260507 23:17:31.109 | INFO | } | 20260507 23:17:31.109 | INFO | Perf report for the last 5.13923 seconds: | 20260507 23:17:31.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:31.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:31.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:31.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:31.109 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260507 23:17:31.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:31.109 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:17:31.109 | INFO | PRE_HQ : 65.4%, 1.7, 57.1ms, 0 | 20260507 23:17:31.109 | INFO | BAZWRITER : 69.1%, 3.8, 135.9ms, 0 | 20260507 23:17:31.109 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 23:17:31.109 | INFO | Uploaded 36GiB to GPU in 2.51446s (14.3172GiB/s) | 20260507 23:17:31.109 | INFO | Downloaded 38GiB to GPU in 2.69377s (14.1066GiB/s) | 20260507 23:17:36.247 | INFO | Analyzing chunk frames = [3379712,3380224) | 20260507 23:17:36.247 | INFO | MemInfo: { | 20260507 23:17:36.247 | INFO | "gpu" : " 17 GB", | 20260507 23:17:36.247 | INFO | "huge" : "", | 20260507 23:17:36.247 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:36.247 | INFO | "malloc" : " 14 GB", | 20260507 23:17:36.247 | INFO | "pinned" : " 29 GB" | 20260507 23:17:36.247 | INFO | } | 20260507 23:17:36.247 | INFO | Perf report for the last 5.13881 seconds: | 20260507 23:17:36.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:36.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.247 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 23:17:36.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.248 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:17:36.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:36.248 | INFO | Uploaded 36GiB to GPU in 2.55792s (14.0739GiB/s) | 20260507 23:17:36.248 | INFO | Downloaded 34GiB to GPU in 2.4785s (13.718GiB/s) | 20260507 23:17:41.383 | INFO | Analyzing chunk frames = [3380224,3380736) | 20260507 23:17:41.383 | INFO | MemInfo: { | 20260507 23:17:41.383 | INFO | "gpu" : " 17 GB", | 20260507 23:17:41.383 | INFO | "huge" : "", | 20260507 23:17:41.383 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:41.383 | INFO | "malloc" : " 14 GB", | 20260507 23:17:41.383 | INFO | "pinned" : " 29 GB" | 20260507 23:17:41.383 | INFO | } | 20260507 23:17:41.383 | INFO | Perf report for the last 5.13541 seconds: | 20260507 23:17:41.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:41.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:17:41.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:17:41.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:41.383 | INFO | Uploaded 36GiB to GPU in 2.51021s (14.3414GiB/s) | 20260507 23:17:41.383 | INFO | Downloaded 34GiB to GPU in 2.48507s (13.6817GiB/s) | 20260507 23:17:46.522 | INFO | Analyzing chunk frames = [3380736,3381248) | 20260507 23:17:46.522 | INFO | MemInfo: { | 20260507 23:17:46.522 | INFO | "gpu" : " 17 GB", | 20260507 23:17:46.522 | INFO | "huge" : "", | 20260507 23:17:46.522 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:46.522 | INFO | "malloc" : " 14 GB", | 20260507 23:17:46.522 | INFO | "pinned" : " 29 GB" | 20260507 23:17:46.522 | INFO | } | 20260507 23:17:46.522 | INFO | Perf report for the last 5.13903 seconds: | 20260507 23:17:46.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:46.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:17:46.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:17:46.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:46.522 | INFO | Uploaded 36GiB to GPU in 2.4799s (14.5167GiB/s) | 20260507 23:17:46.523 | INFO | Downloaded 34GiB to GPU in 2.4027s (14.1508GiB/s) | 20260507 23:17:51.661 | INFO | Analyzing chunk frames = [3381248,3381760) | 20260507 23:17:51.661 | INFO | MemInfo: { | 20260507 23:17:51.661 | INFO | "gpu" : " 17 GB", | 20260507 23:17:51.661 | INFO | "huge" : "", | 20260507 23:17:51.661 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:51.661 | INFO | "malloc" : " 14 GB", | 20260507 23:17:51.661 | INFO | "pinned" : " 29 GB" | 20260507 23:17:51.661 | INFO | } | 20260507 23:17:51.661 | INFO | Perf report for the last 5.13958 seconds: | 20260507 23:17:51.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:51.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.662 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:17:51.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.662 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:17:51.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:51.662 | INFO | Uploaded 36GiB to GPU in 2.44369s (14.7318GiB/s) | 20260507 23:17:51.662 | INFO | Downloaded 34GiB to GPU in 2.49348s (13.6356GiB/s) | 20260507 23:17:56.804 | INFO | Analyzing chunk frames = [3381760,3382272) | 20260507 23:17:56.804 | INFO | MemInfo: { | 20260507 23:17:56.804 | INFO | "gpu" : " 17 GB", | 20260507 23:17:56.804 | INFO | "huge" : "", | 20260507 23:17:56.804 | INFO | "huge_shared" : " 29 GB", | 20260507 23:17:56.804 | INFO | "malloc" : " 14 GB", | 20260507 23:17:56.804 | INFO | "pinned" : " 29 GB" | 20260507 23:17:56.804 | INFO | } | 20260507 23:17:56.804 | INFO | Perf report for the last 5.14248 seconds: | 20260507 23:17:56.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:17:56.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 23:17:56.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:17:56.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:17:56.804 | INFO | Uploaded 36GiB to GPU in 2.48861s (14.4659GiB/s) | 20260507 23:17:56.804 | INFO | Downloaded 34GiB to GPU in 2.41722s (14.0658GiB/s) | 20260507 23:18:01.942 | INFO | Analyzing chunk frames = [3382272,3382784) | 20260507 23:18:01.942 | INFO | MemInfo: { | 20260507 23:18:01.942 | INFO | "gpu" : " 17 GB", | 20260507 23:18:01.942 | INFO | "huge" : "", | 20260507 23:18:01.942 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:01.942 | INFO | "malloc" : " 14 GB", | 20260507 23:18:01.942 | INFO | "pinned" : " 29 GB" | 20260507 23:18:01.942 | INFO | } | 20260507 23:18:01.942 | INFO | Perf report for the last 5.13801 seconds: | 20260507 23:18:01.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:01.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 23:18:01.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:18:01.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:01.942 | INFO | Uploaded 36GiB to GPU in 2.5324s (14.2158GiB/s) | 20260507 23:18:01.942 | INFO | Downloaded 34GiB to GPU in 2.45867s (13.8286GiB/s) | 20260507 23:18:07.076 | INFO | Analyzing chunk frames = [3382784,3383296) | 20260507 23:18:07.076 | INFO | MemInfo: { | 20260507 23:18:07.076 | INFO | "gpu" : " 17 GB", | 20260507 23:18:07.076 | INFO | "huge" : "", | 20260507 23:18:07.076 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:07.079 | INFO | "malloc" : " 14 GB", | 20260507 23:18:07.079 | INFO | "pinned" : " 29 GB" | 20260507 23:18:07.079 | INFO | } | 20260507 23:18:07.080 | INFO | Perf report for the last 5.13757 seconds: | 20260507 23:18:07.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:07.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:18:07.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:18:07.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:07.080 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:18:07.080 | INFO | Total mem usage: 16.7 GiB | 20260507 23:18:07.080 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:18:07.080 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260507 23:18:07.080 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:18:07.080 | INFO | High Water Breakdown: | 20260507 23:18:07.080 | INFO | 59.9% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:18:07.080 | INFO | 39.9% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:18:07.080 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:18:07.080 | INFO | ------------------------------------------ | 20260507 23:18:07.080 | INFO | Uploaded 36GiB to GPU in 2.50487s (14.372GiB/s) | 20260507 23:18:07.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:18:07.080 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:18:07.080 | INFO | Total mem usage: 26.5 GiB | 20260507 23:18:07.080 | INFO | Downloaded 34GiB to GPU in 2.43594s (13.9576GiB/s) | 20260507 23:18:07.080 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:18:07.080 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:18:07.081 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:18:07.081 | INFO | High Water Breakdown: | 20260507 23:18:07.081 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:18:07.081 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:18:07.081 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:18:07.081 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:18:07.081 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:18:07.081 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:18:07.081 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:18:07.081 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:18:07.081 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:18:07.081 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:18:07.081 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:18:07.081 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:18:07.081 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:18:07.081 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:18:07.081 | INFO | ------------------------------------------ | 20260507 23:18:07.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:18:07.081 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:18:07.081 | INFO | Total mem usage: 27.8 GiB | 20260507 23:18:07.081 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:18:07.081 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:18:07.081 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:18:07.081 | INFO | High Water Breakdown: | 20260507 23:18:07.081 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:18:07.081 | INFO | ------------------------------------------ | 20260507 23:18:07.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:18:07.081 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:18:07.081 | INFO | Total mem usage: 15.7 GiB | 20260507 23:18:07.081 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:18:07.081 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:18:07.081 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:18:07.081 | INFO | High Water Breakdown: | 20260507 23:18:07.081 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:18:07.081 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:18:07.081 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:18:07.081 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:18:07.081 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:18:07.081 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:18:07.081 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:18:07.081 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:18:07.081 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:18:07.081 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:18:07.081 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:18:07.081 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:18:07.081 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:18:07.081 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:18:07.081 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:18:07.081 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:18:07.081 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:18:07.081 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:18:07.081 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:18:07.081 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:18:07.081 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:18:07.081 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:18:07.081 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:18:07.081 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:18:07.081 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:18:07.081 | INFO | ------------------------------------------ | 20260507 23:18:07.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:18:07.081 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:18:07.081 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:18:07.081 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:18:07.081 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.39e+01 / 1.49e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.02e-01, 1.14e-02 / 6.25e-03 | 20260507 23:18:07.081 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.50e+00, 5.97e-01 / 2.02e-01 | 20260507 23:18:07.081 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.13e-01 | 20260507 23:18:07.081 | INFO | PerfReport: 15.87%, DME -- 4.18e+01 / 4.54e+01, 3.84e+00 / 1.49e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.56e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.43e+00 / 2.74e+00, 4.25e-01 / 2.22e-01 | 20260507 23:18:07.081 | INFO | PerfReport: 3.42%, Metrics -- 1.13e+00 / 1.34e+00, 2.85e-01 / 1.63e-01 | 20260507 23:18:07.081 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.47e-01 / 3.86e-01, 5.54e-02 / 2.89e-02 | 20260507 23:18:07.081 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:18:07.081 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:18:07.081 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:18:07.081 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 24.30%, Upload -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.48e+00 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 30.88%, Compute -- 3.29e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 1.92%, LockUpload -- 2.05e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 2.65%, LockDownload -- 2.83e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 15.98%, LockCompute -- 1.71e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:18:07.081 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 3.99e-02 / 0.00e+00 | 20260507 23:18:07.178 | INFO | Starting a round of Baz IO | 20260507 23:18:10.648 | INFO | Populating RTMetrics report | 20260507 23:18:10.648 | INFO | RTMetrics report populated | 20260507 23:18:10.648 | INFO | Opening tmp RTMetrics json file | 20260507 23:18:10.671 | INFO | Serializing RTMetrics to json | 20260507 23:18:10.673 | INFO | Serialization done, swapping tmp file now... | 20260507 23:18:10.674 | INFO | RTMetrics done | 20260507 23:18:10.749 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.57 seconds | 20260507 23:18:10.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.68% / 0.01% / 57.31% | 20260507 23:18:10.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22295% / 7.94729e-06% / 6.35783e-05% | 20260507 23:18:10.749 | INFO | Baz IO overall: | 20260507 23:18:10.749 | INFO | BytesWritten : 1540596272141 | 20260507 23:18:10.749 | INFO | NumEvents : 746117484543 | 20260507 23:18:12.220 | INFO | Analyzing chunk frames = [3383296,3383808) | 20260507 23:18:12.220 | INFO | MemInfo: { | 20260507 23:18:12.220 | INFO | "gpu" : " 17 GB", | 20260507 23:18:12.220 | INFO | "huge" : "", | 20260507 23:18:12.220 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:12.220 | INFO | "malloc" : " 14 GB", | 20260507 23:18:12.220 | INFO | "pinned" : " 29 GB" | 20260507 23:18:12.220 | INFO | } | 20260507 23:18:12.220 | INFO | Perf report for the last 5.14062 seconds: | 20260507 23:18:12.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:12.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:12.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:12.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:12.220 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 23:18:12.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:12.220 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 23:18:12.220 | INFO | PRE_HQ : 65.6%, 1.7, 57.1ms, 0 | 20260507 23:18:12.220 | INFO | BAZWRITER : 69.1%, 4.1, 144.2ms, 0 | 20260507 23:18:12.220 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 23:18:12.221 | INFO | Uploaded 36GiB to GPU in 2.5296s (14.2315GiB/s) | 20260507 23:18:12.221 | INFO | Downloaded 38GiB to GPU in 2.70933s (14.0256GiB/s) | 20260507 23:18:17.357 | INFO | Analyzing chunk frames = [3383808,3384320) | 20260507 23:18:17.357 | INFO | MemInfo: { | 20260507 23:18:17.357 | INFO | "gpu" : " 17 GB", | 20260507 23:18:17.357 | INFO | "huge" : "", | 20260507 23:18:17.357 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:17.357 | INFO | "malloc" : " 14 GB", | 20260507 23:18:17.357 | INFO | "pinned" : " 29 GB" | 20260507 23:18:17.357 | INFO | } | 20260507 23:18:17.357 | INFO | Perf report for the last 5.13699 seconds: | 20260507 23:18:17.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:17.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 23:18:17.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:18:17.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:17.357 | INFO | Uploaded 36GiB to GPU in 2.53138s (14.2215GiB/s) | 20260507 23:18:17.357 | INFO | Downloaded 34GiB to GPU in 2.49799s (13.6109GiB/s) | 20260507 23:18:22.506 | INFO | Analyzing chunk frames = [3384320,3384832) | 20260507 23:18:22.506 | INFO | MemInfo: { | 20260507 23:18:22.506 | INFO | "gpu" : " 17 GB", | 20260507 23:18:22.506 | INFO | "huge" : "", | 20260507 23:18:22.506 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:22.506 | INFO | "malloc" : " 14 GB", | 20260507 23:18:22.506 | INFO | "pinned" : " 29 GB" | 20260507 23:18:22.506 | INFO | } | 20260507 23:18:22.506 | INFO | Perf report for the last 5.14905 seconds: | 20260507 23:18:22.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:22.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.506 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260507 23:18:22.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.507 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:18:22.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:22.507 | INFO | Uploaded 36GiB to GPU in 2.49848s (14.4088GiB/s) | 20260507 23:18:22.507 | INFO | Downloaded 34GiB to GPU in 2.47948s (13.7125GiB/s) | 20260507 23:18:27.640 | INFO | Analyzing chunk frames = [3384832,3385344) | 20260507 23:18:27.640 | INFO | MemInfo: { | 20260507 23:18:27.640 | INFO | "gpu" : " 17 GB", | 20260507 23:18:27.640 | INFO | "huge" : "", | 20260507 23:18:27.640 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:27.640 | INFO | "malloc" : " 14 GB", | 20260507 23:18:27.640 | INFO | "pinned" : " 29 GB" | 20260507 23:18:27.640 | INFO | } | 20260507 23:18:27.640 | INFO | Perf report for the last 5.13382 seconds: | 20260507 23:18:27.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:27.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260507 23:18:27.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260507 23:18:27.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:27.640 | INFO | Uploaded 36GiB to GPU in 2.44933s (14.6979GiB/s) | 20260507 23:18:27.640 | INFO | Downloaded 34GiB to GPU in 2.4955s (13.6245GiB/s) | 20260507 23:18:32.775 | INFO | Analyzing chunk frames = [3385344,3385856) | 20260507 23:18:32.775 | INFO | MemInfo: { | 20260507 23:18:32.775 | INFO | "gpu" : " 17 GB", | 20260507 23:18:32.775 | INFO | "huge" : "", | 20260507 23:18:32.775 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:32.775 | INFO | "malloc" : " 14 GB", | 20260507 23:18:32.775 | INFO | "pinned" : " 29 GB" | 20260507 23:18:32.775 | INFO | } | 20260507 23:18:32.775 | INFO | Perf report for the last 5.13537 seconds: | 20260507 23:18:32.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:32.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.776 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:18:32.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.776 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:18:32.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:32.776 | INFO | Uploaded 36GiB to GPU in 2.29633s (15.6772GiB/s) | 20260507 23:18:32.776 | INFO | Downloaded 34GiB to GPU in 2.49992s (13.6004GiB/s) | 20260507 23:18:37.922 | INFO | Analyzing chunk frames = [3385856,3386368) | 20260507 23:18:37.922 | INFO | MemInfo: { | 20260507 23:18:37.922 | INFO | "gpu" : " 17 GB", | 20260507 23:18:37.922 | INFO | "huge" : "", | 20260507 23:18:37.922 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:37.922 | INFO | "malloc" : " 14 GB", | 20260507 23:18:37.922 | INFO | "pinned" : " 29 GB" | 20260507 23:18:37.922 | INFO | } | 20260507 23:18:37.922 | INFO | Perf report for the last 5.14632 seconds: | 20260507 23:18:37.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:37.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 23:18:37.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:18:37.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:37.922 | INFO | Uploaded 36GiB to GPU in 2.51483s (14.3151GiB/s) | 20260507 23:18:37.922 | INFO | Downloaded 34GiB to GPU in 2.4353s (13.9613GiB/s) | 20260507 23:18:43.058 | INFO | Analyzing chunk frames = [3386368,3386880) | 20260507 23:18:43.058 | INFO | MemInfo: { | 20260507 23:18:43.058 | INFO | "gpu" : " 17 GB", | 20260507 23:18:43.058 | INFO | "huge" : "", | 20260507 23:18:43.058 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:43.061 | INFO | "malloc" : " 14 GB", | 20260507 23:18:43.061 | INFO | "pinned" : " 29 GB" | 20260507 23:18:43.061 | INFO | } | 20260507 23:18:43.061 | INFO | Perf report for the last 5.1392 seconds: | 20260507 23:18:43.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:43.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.061 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:18:43.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.061 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:18:43.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:43.062 | INFO | Uploaded 36GiB to GPU in 2.51779s (14.2983GiB/s) | 20260507 23:18:43.062 | INFO | Downloaded 34GiB to GPU in 2.46194s (13.8103GiB/s) | 20260507 23:18:48.200 | INFO | Analyzing chunk frames = [3386880,3387392) | 20260507 23:18:48.200 | INFO | MemInfo: { | 20260507 23:18:48.200 | INFO | "gpu" : " 17 GB", | 20260507 23:18:48.200 | INFO | "huge" : "", | 20260507 23:18:48.200 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:48.200 | INFO | "malloc" : " 14 GB", | 20260507 23:18:48.200 | INFO | "pinned" : " 29 GB" | 20260507 23:18:48.200 | INFO | } | 20260507 23:18:48.200 | INFO | Perf report for the last 5.13881 seconds: | 20260507 23:18:48.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:48.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:18:48.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:18:48.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:48.200 | INFO | Uploaded 36GiB to GPU in 2.48231s (14.5026GiB/s) | 20260507 23:18:48.200 | INFO | Downloaded 34GiB to GPU in 2.44649s (13.8975GiB/s) | 20260507 23:18:48.298 | INFO | Starting a round of Baz IO | 20260507 23:18:51.756 | INFO | Populating RTMetrics report | 20260507 23:18:51.756 | INFO | RTMetrics report populated | 20260507 23:18:51.756 | INFO | Opening tmp RTMetrics json file | 20260507 23:18:51.932 | INFO | Serializing RTMetrics to json | 20260507 23:18:51.995 | INFO | Serialization done, swapping tmp file now... | 20260507 23:18:52.001 | INFO | RTMetrics done | 20260507 23:18:52.097 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.799 seconds | 20260507 23:18:52.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.67% / 0.01% / 57.32% | 20260507 23:18:52.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22156% / 1.19209e-05% / 6.35783e-05% | 20260507 23:18:52.097 | INFO | Baz IO overall: | 20260507 23:18:52.097 | INFO | BytesWritten : 1542439522423 | 20260507 23:18:52.097 | INFO | NumEvents : 747005989725 | 20260507 23:18:53.331 | INFO | Analyzing chunk frames = [3387392,3387904) | 20260507 23:18:53.331 | INFO | MemInfo: { | 20260507 23:18:53.331 | INFO | "gpu" : " 17 GB", | 20260507 23:18:53.331 | INFO | "huge" : "", | 20260507 23:18:53.331 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:53.331 | INFO | "malloc" : " 14 GB", | 20260507 23:18:53.331 | INFO | "pinned" : " 29 GB" | 20260507 23:18:53.331 | INFO | } | 20260507 23:18:53.331 | INFO | Perf report for the last 5.13148 seconds: | 20260507 23:18:53.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:53.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:53.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:53.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:53.331 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 23:18:53.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:53.331 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 23:18:53.331 | INFO | PRE_HQ : 65.7%, 1.7, 57.3ms, 0 | 20260507 23:18:53.331 | INFO | BAZWRITER : 73.6%, 3.9, 146.9ms, 0 | 20260507 23:18:53.331 | INFO | BAZ_SAMPLING : 4.8%, 1.0, 2.5ms, 0 | 20260507 23:18:53.332 | INFO | Uploaded 36GiB to GPU in 2.56817s (14.0178GiB/s) | 20260507 23:18:53.332 | INFO | Downloaded 38GiB to GPU in 2.68482s (14.1537GiB/s) | 20260507 23:18:58.472 | INFO | Analyzing chunk frames = [3387904,3388416) | 20260507 23:18:58.472 | INFO | MemInfo: { | 20260507 23:18:58.472 | INFO | "gpu" : " 17 GB", | 20260507 23:18:58.472 | INFO | "huge" : "", | 20260507 23:18:58.472 | INFO | "huge_shared" : " 29 GB", | 20260507 23:18:58.472 | INFO | "malloc" : " 14 GB", | 20260507 23:18:58.472 | INFO | "pinned" : " 29 GB" | 20260507 23:18:58.472 | INFO | } | 20260507 23:18:58.472 | INFO | Perf report for the last 5.14089 seconds: | 20260507 23:18:58.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:18:58.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.472 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:18:58.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.472 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 23:18:58.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:18:58.473 | INFO | Uploaded 36GiB to GPU in 2.57442s (13.9837GiB/s) | 20260507 23:18:58.473 | INFO | Downloaded 34GiB to GPU in 2.49465s (13.6292GiB/s) | 20260507 23:19:03.613 | INFO | Analyzing chunk frames = [3388416,3388928) | 20260507 23:19:03.613 | INFO | MemInfo: { | 20260507 23:19:03.613 | INFO | "gpu" : " 17 GB", | 20260507 23:19:03.613 | INFO | "huge" : "", | 20260507 23:19:03.613 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:03.613 | INFO | "malloc" : " 14 GB", | 20260507 23:19:03.613 | INFO | "pinned" : " 29 GB" | 20260507 23:19:03.613 | INFO | } | 20260507 23:19:03.613 | INFO | Perf report for the last 5.14111 seconds: | 20260507 23:19:03.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:03.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.613 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 23:19:03.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.614 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:19:03.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:03.614 | INFO | Uploaded 36GiB to GPU in 2.56134s (14.0551GiB/s) | 20260507 23:19:03.614 | INFO | Downloaded 34GiB to GPU in 2.47935s (13.7133GiB/s) | 20260507 23:19:08.757 | INFO | Analyzing chunk frames = [3388928,3389440) | 20260507 23:19:08.757 | INFO | MemInfo: { | 20260507 23:19:08.757 | INFO | "gpu" : " 17 GB", | 20260507 23:19:08.757 | INFO | "huge" : "", | 20260507 23:19:08.757 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:08.757 | INFO | "malloc" : " 14 GB", | 20260507 23:19:08.757 | INFO | "pinned" : " 29 GB" | 20260507 23:19:08.757 | INFO | } | 20260507 23:19:08.757 | INFO | Perf report for the last 5.14344 seconds: | 20260507 23:19:08.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:08.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 23:19:08.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:19:08.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:08.757 | INFO | Uploaded 36GiB to GPU in 2.48077s (14.5116GiB/s) | 20260507 23:19:08.757 | INFO | Downloaded 34GiB to GPU in 2.40991s (14.1084GiB/s) | 20260507 23:19:13.894 | INFO | Analyzing chunk frames = [3389440,3389952) | 20260507 23:19:13.894 | INFO | MemInfo: { | 20260507 23:19:13.894 | INFO | "gpu" : " 17 GB", | 20260507 23:19:13.894 | INFO | "huge" : "", | 20260507 23:19:13.894 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:13.894 | INFO | "malloc" : " 14 GB", | 20260507 23:19:13.894 | INFO | "pinned" : " 29 GB" | 20260507 23:19:13.894 | INFO | } | 20260507 23:19:13.894 | INFO | Perf report for the last 5.13752 seconds: | 20260507 23:19:13.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:13.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.894 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 23:19:13.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.894 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:19:13.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:13.895 | INFO | Uploaded 36GiB to GPU in 2.36066s (15.25GiB/s) | 20260507 23:19:13.895 | INFO | Downloaded 34GiB to GPU in 2.50599s (13.5675GiB/s) | 20260507 23:19:19.029 | INFO | Analyzing chunk frames = [3389952,3390464) | 20260507 23:19:19.029 | INFO | MemInfo: { | 20260507 23:19:19.029 | INFO | "gpu" : " 17 GB", | 20260507 23:19:19.029 | INFO | "huge" : "", | 20260507 23:19:19.029 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:19.030 | INFO | "malloc" : " 14 GB", | 20260507 23:19:19.030 | INFO | "pinned" : " 29 GB" | 20260507 23:19:19.030 | INFO | } | 20260507 23:19:19.030 | INFO | Perf report for the last 5.1352 seconds: | 20260507 23:19:19.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:19.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 23:19:19.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:19:19.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:19.030 | INFO | Uploaded 36GiB to GPU in 2.35686s (15.2745GiB/s) | 20260507 23:19:19.030 | INFO | Downloaded 34GiB to GPU in 2.41791s (14.0618GiB/s) | 20260507 23:19:24.171 | INFO | Analyzing chunk frames = [3390464,3390976) | 20260507 23:19:24.171 | INFO | MemInfo: { | 20260507 23:19:24.171 | INFO | "gpu" : " 17 GB", | 20260507 23:19:24.171 | INFO | "huge" : "", | 20260507 23:19:24.171 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:24.171 | INFO | "malloc" : " 14 GB", | 20260507 23:19:24.171 | INFO | "pinned" : " 29 GB" | 20260507 23:19:24.171 | INFO | } | 20260507 23:19:24.171 | INFO | Perf report for the last 5.14174 seconds: | 20260507 23:19:24.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:24.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.171 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 23:19:24.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.171 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:19:24.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:24.172 | INFO | Uploaded 36GiB to GPU in 2.46879s (14.5821GiB/s) | 20260507 23:19:24.172 | INFO | Downloaded 34GiB to GPU in 2.45592s (13.8441GiB/s) | 20260507 23:19:29.309 | INFO | Analyzing chunk frames = [3390976,3391488) | 20260507 23:19:29.309 | INFO | MemInfo: { | 20260507 23:19:29.309 | INFO | "gpu" : " 17 GB", | 20260507 23:19:29.309 | INFO | "huge" : "", | 20260507 23:19:29.309 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:29.309 | INFO | "malloc" : " 14 GB", | 20260507 23:19:29.310 | INFO | "pinned" : " 29 GB" | 20260507 23:19:29.310 | INFO | } | 20260507 23:19:29.310 | INFO | Perf report for the last 5.13821 seconds: | 20260507 23:19:29.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:29.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:19:29.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:19:29.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:29.310 | INFO | Uploaded 36GiB to GPU in 2.46808s (14.5862GiB/s) | 20260507 23:19:29.310 | INFO | Downloaded 34GiB to GPU in 2.45495s (13.8496GiB/s) | 20260507 23:19:29.310 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:19:29.310 | INFO | Total mem usage: 16.7 GiB | 20260507 23:19:29.310 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:19:29.310 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260507 23:19:29.310 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:19:29.310 | INFO | High Water Breakdown: | 20260507 23:19:29.310 | INFO | 60% : 8.93 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:19:29.310 | INFO | 39.9% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:19:29.310 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:19:29.310 | INFO | ------------------------------------------ | 20260507 23:19:29.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:19:29.310 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:19:29.310 | INFO | Total mem usage: 26.5 GiB | 20260507 23:19:29.310 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:19:29.310 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:19:29.310 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:19:29.310 | INFO | High Water Breakdown: | 20260507 23:19:29.310 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:19:29.310 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:19:29.310 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:19:29.310 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:19:29.310 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:19:29.310 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:19:29.310 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:19:29.310 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:19:29.310 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:19:29.310 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:19:29.310 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:19:29.310 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:19:29.310 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:19:29.310 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:19:29.310 | INFO | ------------------------------------------ | 20260507 23:19:29.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:19:29.310 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:19:29.310 | INFO | Total mem usage: 27.8 GiB | 20260507 23:19:29.310 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:19:29.310 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:19:29.310 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:19:29.310 | INFO | High Water Breakdown: | 20260507 23:19:29.310 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:19:29.310 | INFO | ------------------------------------------ | 20260507 23:19:29.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:19:29.310 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:19:29.310 | INFO | Total mem usage: 15.7 GiB | 20260507 23:19:29.310 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:19:29.310 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:19:29.310 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:19:29.310 | INFO | High Water Breakdown: | 20260507 23:19:29.310 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:19:29.310 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:19:29.310 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:19:29.310 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:19:29.310 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:19:29.310 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:19:29.310 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:19:29.310 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:19:29.310 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:19:29.310 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:19:29.310 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:19:29.310 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:19:29.310 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:19:29.310 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:19:29.310 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:19:29.310 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:19:29.310 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:19:29.310 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:19:29.310 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:19:29.310 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:19:29.310 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:19:29.310 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:19:29.310 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:19:29.310 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:19:29.310 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:19:29.310 | INFO | ------------------------------------------ | 20260507 23:19:29.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:19:29.310 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:19:29.310 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:19:29.310 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:19:29.310 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.38e+01 / 1.48e+00 | 20260507 23:19:29.310 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.01e-01, 1.10e-02 / 5.01e-03 | 20260507 23:19:29.311 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.51e+00, 5.91e-01 / 1.62e-01 | 20260507 23:19:29.311 | INFO | PerfReport: 3.39%, Histogram -- 1.12e+00 / 1.50e+00, 2.07e-01 / 1.10e-01 | 20260507 23:19:29.311 | INFO | PerfReport: 15.85%, DME -- 4.17e+01 / 4.53e+01, 3.97e+00 / 1.27e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.51e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.44e+00 / 2.71e+00, 4.26e-01 / 2.04e-01 | 20260507 23:19:29.311 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.36e+00, 2.85e-01 / 1.58e-01 | 20260507 23:19:29.311 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.83e-01, 5.62e-02 / 2.98e-02 | 20260507 23:19:29.311 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:19:29.311 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:19:29.311 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:19:29.311 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 24.21%, Upload -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 30.87%, Compute -- 3.29e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 1.84%, LockUpload -- 1.97e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 2.79%, LockDownload -- 2.98e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 15.88%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 23:19:29.311 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 23:19:29.424 | INFO | Starting a round of Baz IO | 20260507 23:19:32.877 | INFO | Populating RTMetrics report | 20260507 23:19:32.877 | INFO | RTMetrics report populated | 20260507 23:19:32.877 | INFO | Opening tmp RTMetrics json file | 20260507 23:19:32.900 | INFO | Serializing RTMetrics to json | 20260507 23:19:32.907 | INFO | Serialization done, swapping tmp file now... | 20260507 23:19:32.908 | INFO | RTMetrics done | 20260507 23:19:32.969 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.544 seconds | 20260507 23:19:32.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.67% / 0.01% / 57.33% | 20260507 23:19:32.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21881% / 7.94729e-06% / 6.35783e-05% | 20260507 23:19:32.969 | INFO | Baz IO overall: | 20260507 23:19:32.969 | INFO | BytesWritten : 1544282945169 | 20260507 23:19:32.969 | INFO | NumEvents : 747894421386 | 20260507 23:19:34.450 | INFO | Analyzing chunk frames = [3391488,3392000) | 20260507 23:19:34.450 | INFO | MemInfo: { | 20260507 23:19:34.450 | INFO | "gpu" : " 17 GB", | 20260507 23:19:34.450 | INFO | "huge" : "", | 20260507 23:19:34.450 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:34.450 | INFO | "malloc" : " 14 GB", | 20260507 23:19:34.450 | INFO | "pinned" : " 29 GB" | 20260507 23:19:34.450 | INFO | } | 20260507 23:19:34.451 | INFO | Perf report for the last 5.14094 seconds: | 20260507 23:19:34.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:34.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:34.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:34.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:34.451 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 23:19:34.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:34.451 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 23:19:34.451 | INFO | PRE_HQ : 66.1%, 1.7, 59.4ms, 0 | 20260507 23:19:34.451 | INFO | BAZWRITER : 68.6%, 4.1, 144.3ms, 0 | 20260507 23:19:34.451 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 23:19:34.451 | INFO | Uploaded 36GiB to GPU in 2.50265s (14.3847GiB/s) | 20260507 23:19:34.451 | INFO | Downloaded 38GiB to GPU in 2.68841s (14.1347GiB/s) | 20260507 23:19:39.590 | INFO | Analyzing chunk frames = [3392000,3392512) | 20260507 23:19:39.590 | INFO | MemInfo: { | 20260507 23:19:39.590 | INFO | "gpu" : " 17 GB", | 20260507 23:19:39.590 | INFO | "huge" : "", | 20260507 23:19:39.590 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:39.590 | INFO | "malloc" : " 14 GB", | 20260507 23:19:39.590 | INFO | "pinned" : " 29 GB" | 20260507 23:19:39.590 | INFO | } | 20260507 23:19:39.590 | INFO | Perf report for the last 5.13946 seconds: | 20260507 23:19:39.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:39.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 23:19:39.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:19:39.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:39.590 | INFO | Uploaded 36GiB to GPU in 2.5717s (13.9985GiB/s) | 20260507 23:19:39.590 | INFO | Downloaded 34GiB to GPU in 2.48794s (13.6659GiB/s) | 20260507 23:19:44.731 | INFO | Analyzing chunk frames = [3392512,3393024) | 20260507 23:19:44.731 | INFO | MemInfo: { | 20260507 23:19:44.731 | INFO | "gpu" : " 17 GB", | 20260507 23:19:44.731 | INFO | "huge" : "", | 20260507 23:19:44.731 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:44.731 | INFO | "malloc" : " 14 GB", | 20260507 23:19:44.731 | INFO | "pinned" : " 29 GB" | 20260507 23:19:44.731 | INFO | } | 20260507 23:19:44.731 | INFO | Perf report for the last 5.1414 seconds: | 20260507 23:19:44.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:44.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.731 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 23:19:44.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.732 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:19:44.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:44.732 | INFO | Uploaded 36GiB to GPU in 2.47078s (14.5703GiB/s) | 20260507 23:19:44.732 | INFO | Downloaded 34GiB to GPU in 2.48342s (13.6908GiB/s) | 20260507 23:19:49.870 | INFO | Analyzing chunk frames = [3393024,3393536) | 20260507 23:19:49.870 | INFO | MemInfo: { | 20260507 23:19:49.870 | INFO | "gpu" : " 17 GB", | 20260507 23:19:49.870 | INFO | "huge" : "", | 20260507 23:19:49.870 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:49.870 | INFO | "malloc" : " 14 GB", | 20260507 23:19:49.870 | INFO | "pinned" : " 29 GB" | 20260507 23:19:49.870 | INFO | } | 20260507 23:19:49.870 | INFO | Perf report for the last 5.1388 seconds: | 20260507 23:19:49.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:49.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.871 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:19:49.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.871 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:19:49.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:49.872 | INFO | Uploaded 36GiB to GPU in 2.51386s (14.3206GiB/s) | 20260507 23:19:49.872 | INFO | Downloaded 34GiB to GPU in 2.47035s (13.7633GiB/s) | 20260507 23:19:55.011 | INFO | Analyzing chunk frames = [3393536,3394048) | 20260507 23:19:55.011 | INFO | MemInfo: { | 20260507 23:19:55.011 | INFO | "gpu" : " 17 GB", | 20260507 23:19:55.011 | INFO | "huge" : "", | 20260507 23:19:55.011 | INFO | "huge_shared" : " 29 GB", | 20260507 23:19:55.011 | INFO | "malloc" : " 14 GB", | 20260507 23:19:55.011 | INFO | "pinned" : " 29 GB" | 20260507 23:19:55.011 | INFO | } | 20260507 23:19:55.012 | INFO | Perf report for the last 5.14127 seconds: | 20260507 23:19:55.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:19:55.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:19:55.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:19:55.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:19:55.012 | INFO | Uploaded 36GiB to GPU in 2.34619s (15.344GiB/s) | 20260507 23:19:55.012 | INFO | Downloaded 34GiB to GPU in 2.50555s (13.5699GiB/s) | 20260507 23:20:00.144 | INFO | Analyzing chunk frames = [3394048,3394560) | 20260507 23:20:00.144 | INFO | MemInfo: { | 20260507 23:20:00.144 | INFO | "gpu" : " 17 GB", | 20260507 23:20:00.144 | INFO | "huge" : "", | 20260507 23:20:00.144 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:00.144 | INFO | "malloc" : " 14 GB", | 20260507 23:20:00.144 | INFO | "pinned" : " 29 GB" | 20260507 23:20:00.144 | INFO | } | 20260507 23:20:00.144 | INFO | Perf report for the last 5.1325 seconds: | 20260507 23:20:00.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:00.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:20:00.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:20:00.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:00.144 | INFO | Uploaded 36GiB to GPU in 2.42146s (14.8671GiB/s) | 20260507 23:20:00.144 | INFO | Downloaded 34GiB to GPU in 2.38643s (14.2472GiB/s) | 20260507 23:20:05.287 | INFO | Analyzing chunk frames = [3394560,3395072) | 20260507 23:20:05.287 | INFO | MemInfo: { | 20260507 23:20:05.287 | INFO | "gpu" : " 17 GB", | 20260507 23:20:05.287 | INFO | "huge" : "", | 20260507 23:20:05.287 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:05.287 | INFO | "malloc" : " 14 GB", | 20260507 23:20:05.287 | INFO | "pinned" : " 29 GB" | 20260507 23:20:05.287 | INFO | } | 20260507 23:20:05.287 | INFO | Perf report for the last 5.14305 seconds: | 20260507 23:20:05.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:05.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260507 23:20:05.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 23:20:05.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:05.288 | INFO | Uploaded 36GiB to GPU in 2.5401s (14.1727GiB/s) | 20260507 23:20:05.288 | INFO | Downloaded 34GiB to GPU in 2.44707s (13.8941GiB/s) | 20260507 23:20:10.425 | INFO | Analyzing chunk frames = [3395072,3395584) | 20260507 23:20:10.425 | INFO | MemInfo: { | 20260507 23:20:10.425 | INFO | "gpu" : " 17 GB", | 20260507 23:20:10.425 | INFO | "huge" : "", | 20260507 23:20:10.425 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:10.425 | INFO | "malloc" : " 14 GB", | 20260507 23:20:10.425 | INFO | "pinned" : " 29 GB" | 20260507 23:20:10.425 | INFO | } | 20260507 23:20:10.425 | INFO | Perf report for the last 5.13788 seconds: | 20260507 23:20:10.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:10.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.425 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 23:20:10.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.425 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 23:20:10.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:10.426 | INFO | Uploaded 36GiB to GPU in 2.54148s (14.165GiB/s) | 20260507 23:20:10.426 | INFO | Downloaded 34GiB to GPU in 2.45092s (13.8723GiB/s) | 20260507 23:20:10.533 | INFO | Starting a round of Baz IO | 20260507 23:20:14.021 | INFO | Populating RTMetrics report | 20260507 23:20:14.021 | INFO | RTMetrics report populated | 20260507 23:20:14.021 | INFO | Opening tmp RTMetrics json file | 20260507 23:20:14.033 | INFO | Serializing RTMetrics to json | 20260507 23:20:14.052 | INFO | Serialization done, swapping tmp file now... | 20260507 23:20:14.054 | INFO | RTMetrics done | 20260507 23:20:14.114 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.581 seconds | 20260507 23:20:14.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.66% / 0.01% / 57.33% | 20260507 23:20:14.115 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21697% / 7.94729e-06% / 6.75519e-05% | 20260507 23:20:14.115 | INFO | Baz IO overall: | 20260507 23:20:14.115 | INFO | BytesWritten : 1546126491777 | 20260507 23:20:14.115 | INFO | NumEvents : 748783094691 | 20260507 23:20:15.563 | INFO | Analyzing chunk frames = [3395584,3396096) | 20260507 23:20:15.563 | INFO | MemInfo: { | 20260507 23:20:15.563 | INFO | "gpu" : " 17 GB", | 20260507 23:20:15.563 | INFO | "huge" : "", | 20260507 23:20:15.563 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:15.564 | INFO | "malloc" : " 14 GB", | 20260507 23:20:15.564 | INFO | "pinned" : " 29 GB" | 20260507 23:20:15.564 | INFO | } | 20260507 23:20:15.564 | INFO | Perf report for the last 5.13862 seconds: | 20260507 23:20:15.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:15.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:15.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:15.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:15.564 | INFO | ANALYSIS : 69.0%, 2.9, 104.7ms, 0 | 20260507 23:20:15.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:15.564 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 23:20:15.564 | INFO | PRE_HQ : 67.6%, 1.8, 63.1ms, 0 | 20260507 23:20:15.564 | INFO | BAZWRITER : 69.1%, 4.0, 142.4ms, 0 | 20260507 23:20:15.564 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 23:20:15.564 | INFO | Uploaded 36GiB to GPU in 2.58166s (13.9445GiB/s) | 20260507 23:20:15.564 | INFO | Downloaded 38GiB to GPU in 2.69986s (14.0748GiB/s) | 20260507 23:20:20.704 | INFO | Analyzing chunk frames = [3396096,3396608) | 20260507 23:20:20.704 | INFO | MemInfo: { | 20260507 23:20:20.704 | INFO | "gpu" : " 17 GB", | 20260507 23:20:20.704 | INFO | "huge" : "", | 20260507 23:20:20.704 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:20.704 | INFO | "malloc" : " 14 GB", | 20260507 23:20:20.705 | INFO | "pinned" : " 29 GB" | 20260507 23:20:20.705 | INFO | } | 20260507 23:20:20.705 | INFO | Perf report for the last 5.14091 seconds: | 20260507 23:20:20.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:20.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 23:20:20.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:20:20.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:20.705 | INFO | Uploaded 36GiB to GPU in 2.5538s (14.0967GiB/s) | 20260507 23:20:20.705 | INFO | Downloaded 34GiB to GPU in 2.48212s (13.698GiB/s) | 20260507 23:20:25.843 | INFO | Analyzing chunk frames = [3396608,3397120) | 20260507 23:20:25.843 | INFO | MemInfo: { | 20260507 23:20:25.843 | INFO | "gpu" : " 17 GB", | 20260507 23:20:25.843 | INFO | "huge" : "", | 20260507 23:20:25.843 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:25.843 | INFO | "malloc" : " 14 GB", | 20260507 23:20:25.843 | INFO | "pinned" : " 29 GB" | 20260507 23:20:25.843 | INFO | } | 20260507 23:20:25.843 | INFO | Perf report for the last 5.13882 seconds: | 20260507 23:20:25.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:25.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.843 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:20:25.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.844 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:20:25.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:25.844 | INFO | Uploaded 36GiB to GPU in 2.54067s (14.1695GiB/s) | 20260507 23:20:25.844 | INFO | Downloaded 34GiB to GPU in 2.48634s (13.6747GiB/s) | 20260507 23:20:30.984 | INFO | Analyzing chunk frames = [3397120,3397632) | 20260507 23:20:30.984 | INFO | MemInfo: { | 20260507 23:20:30.984 | INFO | "gpu" : " 17 GB", | 20260507 23:20:30.984 | INFO | "huge" : "", | 20260507 23:20:30.984 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:30.984 | INFO | "malloc" : " 14 GB", | 20260507 23:20:30.984 | INFO | "pinned" : " 29 GB" | 20260507 23:20:30.985 | INFO | } | 20260507 23:20:30.985 | INFO | Perf report for the last 5.14126 seconds: | 20260507 23:20:30.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:30.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:20:30.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:20:30.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:30.985 | INFO | Uploaded 36GiB to GPU in 2.47424s (14.5499GiB/s) | 20260507 23:20:30.985 | INFO | Downloaded 34GiB to GPU in 2.41766s (14.0632GiB/s) | 20260507 23:20:36.124 | INFO | Analyzing chunk frames = [3397632,3398144) | 20260507 23:20:36.124 | INFO | MemInfo: { | 20260507 23:20:36.124 | INFO | "gpu" : " 17 GB", | 20260507 23:20:36.124 | INFO | "huge" : "", | 20260507 23:20:36.124 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:36.124 | INFO | "malloc" : " 14 GB", | 20260507 23:20:36.124 | INFO | "pinned" : " 29 GB" | 20260507 23:20:36.124 | INFO | } | 20260507 23:20:36.124 | INFO | Perf report for the last 5.13948 seconds: | 20260507 23:20:36.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:36.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.124 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260507 23:20:36.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.124 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:20:36.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:36.125 | INFO | Uploaded 36GiB to GPU in 2.3162s (15.5427GiB/s) | 20260507 23:20:36.125 | INFO | Downloaded 34GiB to GPU in 2.48103s (13.704GiB/s) | 20260507 23:20:41.263 | INFO | Analyzing chunk frames = [3398144,3398656) | 20260507 23:20:41.263 | INFO | MemInfo: { | 20260507 23:20:41.263 | INFO | "gpu" : " 17 GB", | 20260507 23:20:41.263 | INFO | "huge" : "", | 20260507 23:20:41.263 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:41.263 | INFO | "malloc" : " 14 GB", | 20260507 23:20:41.263 | INFO | "pinned" : " 29 GB" | 20260507 23:20:41.263 | INFO | } | 20260507 23:20:41.263 | INFO | Perf report for the last 5.13878 seconds: | 20260507 23:20:41.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:41.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:20:41.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:20:41.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:41.263 | INFO | Uploaded 36GiB to GPU in 2.37954s (15.1289GiB/s) | 20260507 23:20:41.263 | INFO | Downloaded 34GiB to GPU in 2.39757s (14.181GiB/s) | 20260507 23:20:46.399 | INFO | Analyzing chunk frames = [3398656,3399168) | 20260507 23:20:46.399 | INFO | MemInfo: { | 20260507 23:20:46.399 | INFO | "gpu" : " 17 GB", | 20260507 23:20:46.399 | INFO | "huge" : "", | 20260507 23:20:46.399 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:46.399 | INFO | "malloc" : " 14 GB", | 20260507 23:20:46.399 | INFO | "pinned" : " 29 GB" | 20260507 23:20:46.399 | INFO | } | 20260507 23:20:46.399 | INFO | Perf report for the last 5.13623 seconds: | 20260507 23:20:46.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:46.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:20:46.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 23:20:46.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:46.399 | INFO | Uploaded 36GiB to GPU in 2.53053s (14.2263GiB/s) | 20260507 23:20:46.399 | INFO | Downloaded 34GiB to GPU in 2.47049s (13.7625GiB/s) | 20260507 23:20:51.539 | INFO | Analyzing chunk frames = [3399168,3399680) | 20260507 23:20:51.539 | INFO | MemInfo: { | 20260507 23:20:51.539 | INFO | "gpu" : " 17 GB", | 20260507 23:20:51.539 | INFO | "huge" : "", | 20260507 23:20:51.539 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:51.539 | INFO | "malloc" : " 14 GB", | 20260507 23:20:51.539 | INFO | "pinned" : " 29 GB" | 20260507 23:20:51.539 | INFO | } | 20260507 23:20:51.539 | INFO | Perf report for the last 5.13963 seconds: | 20260507 23:20:51.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:51.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:20:51.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:20:51.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:51.539 | INFO | Uploaded 36GiB to GPU in 2.5057s (14.3673GiB/s) | 20260507 23:20:51.539 | INFO | Downloaded 34GiB to GPU in 2.4478s (13.89GiB/s) | 20260507 23:20:51.539 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:20:51.539 | INFO | Total mem usage: 16.7 GiB | 20260507 23:20:51.539 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:20:51.539 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -7.07%) | 20260507 23:20:51.540 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:20:51.540 | INFO | High Water Breakdown: | 20260507 23:20:51.540 | INFO | 60.8% : 9.47 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:20:51.540 | INFO | 39.1% : 6.09 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:20:51.540 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:20:51.540 | INFO | ------------------------------------------ | 20260507 23:20:51.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:20:51.540 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:20:51.540 | INFO | Total mem usage: 26.5 GiB | 20260507 23:20:51.540 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:20:51.540 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 23:20:51.540 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 23:20:51.540 | INFO | High Water Breakdown: | 20260507 23:20:51.540 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:20:51.540 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:20:51.540 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:20:51.540 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:20:51.540 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:20:51.540 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:20:51.540 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:20:51.540 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:20:51.540 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:20:51.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:20:51.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:20:51.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:20:51.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:20:51.540 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:20:51.540 | INFO | ------------------------------------------ | 20260507 23:20:51.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:20:51.540 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:20:51.540 | INFO | Total mem usage: 27.8 GiB | 20260507 23:20:51.540 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:20:51.540 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:20:51.540 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:20:51.540 | INFO | High Water Breakdown: | 20260507 23:20:51.540 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:20:51.540 | INFO | ------------------------------------------ | 20260507 23:20:51.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:20:51.540 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:20:51.540 | INFO | Total mem usage: 15.7 GiB | 20260507 23:20:51.540 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:20:51.540 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:20:51.540 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:20:51.540 | INFO | High Water Breakdown: | 20260507 23:20:51.540 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:20:51.540 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:20:51.540 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:20:51.540 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:20:51.540 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:20:51.540 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:20:51.540 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:20:51.540 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:20:51.540 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:20:51.540 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:20:51.540 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:20:51.540 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:20:51.540 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:20:51.540 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:20:51.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:20:51.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:20:51.540 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:20:51.540 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:20:51.540 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:20:51.540 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:20:51.540 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:20:51.540 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:20:51.540 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:20:51.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:20:51.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:20:51.541 | INFO | ------------------------------------------ | 20260507 23:20:51.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:20:51.541 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:20:51.541 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:20:51.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:20:51.541 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.42e+01 / 1.57e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.02e-01, 1.17e-02 / 5.17e-03 | 20260507 23:20:51.541 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.48e+00, 5.95e-01 / 2.47e-01 | 20260507 23:20:51.541 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.05e-01 | 20260507 23:20:51.541 | INFO | PerfReport: 15.97%, DME -- 4.21e+01 / 4.58e+01, 3.94e+00 / 1.51e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.72e+00 / 1.78e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.45e+00 / 2.72e+00, 4.35e-01 / 2.36e-01 | 20260507 23:20:51.541 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.35e+00, 2.84e-01 / 1.56e-01 | 20260507 23:20:51.541 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.49e-01 / 3.81e-01, 5.69e-02 / 2.91e-02 | 20260507 23:20:51.541 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:20:51.541 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:20:51.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:20:51.541 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 24.31%, Upload -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 30.90%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 2.08%, LockUpload -- 2.21e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 2.64%, LockDownload -- 2.81e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 15.73%, LockCompute -- 1.68e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:20:51.541 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260507 23:20:51.636 | INFO | Starting a round of Baz IO | 20260507 23:20:55.100 | INFO | Populating RTMetrics report | 20260507 23:20:55.101 | INFO | RTMetrics report populated | 20260507 23:20:55.101 | INFO | Opening tmp RTMetrics json file | 20260507 23:20:55.133 | INFO | Serializing RTMetrics to json | 20260507 23:20:55.242 | INFO | Serialization done, swapping tmp file now... | 20260507 23:20:55.244 | INFO | RTMetrics done | 20260507 23:20:55.273 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.637 seconds | 20260507 23:20:55.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.66% / 0.01% / 57.34% | 20260507 23:20:55.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21638% / 1.19209e-05% / 5.16574e-05% | 20260507 23:20:55.274 | INFO | Baz IO overall: | 20260507 23:20:55.274 | INFO | BytesWritten : 1547969962220 | 20260507 23:20:55.274 | INFO | NumEvents : 749671956478 | 20260507 23:20:56.682 | INFO | Analyzing chunk frames = [3399680,3400192) | 20260507 23:20:56.682 | INFO | MemInfo: { | 20260507 23:20:56.682 | INFO | "gpu" : " 17 GB", | 20260507 23:20:56.682 | INFO | "huge" : "", | 20260507 23:20:56.682 | INFO | "huge_shared" : " 29 GB", | 20260507 23:20:56.682 | INFO | "malloc" : " 14 GB", | 20260507 23:20:56.682 | INFO | "pinned" : " 29 GB" | 20260507 23:20:56.682 | INFO | } | 20260507 23:20:56.682 | INFO | Perf report for the last 5.14344 seconds: | 20260507 23:20:56.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:20:56.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:56.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:56.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:56.682 | INFO | ANALYSIS : 68.5%, 3.0, 106.9ms, 0 | 20260507 23:20:56.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:20:56.682 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 23:20:56.682 | INFO | PRE_HQ : 65.6%, 1.7, 56.9ms, 0 | 20260507 23:20:56.682 | INFO | BAZWRITER : 70.3%, 3.6, 128.9ms, 0 | 20260507 23:20:56.682 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260507 23:20:56.683 | INFO | Uploaded 36GiB to GPU in 2.55078s (14.1133GiB/s) | 20260507 23:20:56.683 | INFO | Downloaded 38GiB to GPU in 2.70002s (14.0739GiB/s) | 20260507 23:21:01.825 | INFO | Analyzing chunk frames = [3400192,3400704) | 20260507 23:21:01.825 | INFO | MemInfo: { | 20260507 23:21:01.825 | INFO | "gpu" : " 17 GB", | 20260507 23:21:01.825 | INFO | "huge" : "", | 20260507 23:21:01.825 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:01.825 | INFO | "malloc" : " 14 GB", | 20260507 23:21:01.825 | INFO | "pinned" : " 29 GB" | 20260507 23:21:01.825 | INFO | } | 20260507 23:21:01.825 | INFO | Perf report for the last 5.14273 seconds: | 20260507 23:21:01.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:01.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 23:21:01.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:21:01.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:01.825 | INFO | Uploaded 36GiB to GPU in 2.50665s (14.3618GiB/s) | 20260507 23:21:01.826 | INFO | Downloaded 34GiB to GPU in 2.48866s (13.662GiB/s) | 20260507 23:21:06.960 | INFO | Analyzing chunk frames = [3400704,3401216) | 20260507 23:21:06.960 | INFO | MemInfo: { | 20260507 23:21:06.960 | INFO | "gpu" : " 17 GB", | 20260507 23:21:06.960 | INFO | "huge" : "", | 20260507 23:21:06.960 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:06.960 | INFO | "malloc" : " 14 GB", | 20260507 23:21:06.960 | INFO | "pinned" : " 29 GB" | 20260507 23:21:06.960 | INFO | } | 20260507 23:21:06.960 | INFO | Perf report for the last 5.13518 seconds: | 20260507 23:21:06.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:06.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.960 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:21:06.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.960 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:21:06.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:06.961 | INFO | Uploaded 36GiB to GPU in 2.502s (14.3885GiB/s) | 20260507 23:21:06.961 | INFO | Downloaded 34GiB to GPU in 2.48612s (13.6759GiB/s) | 20260507 23:21:12.095 | INFO | Analyzing chunk frames = [3401216,3401728) | 20260507 23:21:12.095 | INFO | MemInfo: { | 20260507 23:21:12.095 | INFO | "gpu" : " 17 GB", | 20260507 23:21:12.095 | INFO | "huge" : "", | 20260507 23:21:12.095 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:12.095 | INFO | "malloc" : " 14 GB", | 20260507 23:21:12.095 | INFO | "pinned" : " 29 GB" | 20260507 23:21:12.095 | INFO | } | 20260507 23:21:12.095 | INFO | Perf report for the last 5.13471 seconds: | 20260507 23:21:12.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:12.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.095 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:21:12.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.095 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:21:12.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:12.096 | INFO | Uploaded 36GiB to GPU in 2.4841s (14.4922GiB/s) | 20260507 23:21:12.096 | INFO | Downloaded 34GiB to GPU in 2.49963s (13.602GiB/s) | 20260507 23:21:17.239 | INFO | Analyzing chunk frames = [3401728,3402240) | 20260507 23:21:17.239 | INFO | MemInfo: { | 20260507 23:21:17.239 | INFO | "gpu" : " 17 GB", | 20260507 23:21:17.239 | INFO | "huge" : "", | 20260507 23:21:17.239 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:17.239 | INFO | "malloc" : " 14 GB", | 20260507 23:21:17.239 | INFO | "pinned" : " 29 GB" | 20260507 23:21:17.239 | INFO | } | 20260507 23:21:17.240 | INFO | Perf report for the last 5.14456 seconds: | 20260507 23:21:17.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:17.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 23:21:17.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:21:17.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:17.240 | INFO | Uploaded 36GiB to GPU in 2.37679s (15.1465GiB/s) | 20260507 23:21:17.240 | INFO | Downloaded 34GiB to GPU in 2.50512s (13.5722GiB/s) | 20260507 23:21:22.375 | INFO | Analyzing chunk frames = [3402240,3402752) | 20260507 23:21:22.375 | INFO | MemInfo: { | 20260507 23:21:22.375 | INFO | "gpu" : " 17 GB", | 20260507 23:21:22.375 | INFO | "huge" : "", | 20260507 23:21:22.375 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:22.375 | INFO | "malloc" : " 14 GB", | 20260507 23:21:22.375 | INFO | "pinned" : " 29 GB" | 20260507 23:21:22.375 | INFO | } | 20260507 23:21:22.376 | INFO | Perf report for the last 5.13614 seconds: | 20260507 23:21:22.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:22.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 23:21:22.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 23:21:22.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:22.376 | INFO | Uploaded 36GiB to GPU in 2.34179s (15.3729GiB/s) | 20260507 23:21:22.376 | INFO | Downloaded 34GiB to GPU in 2.4265s (14.012GiB/s) | 20260507 23:21:27.519 | INFO | Analyzing chunk frames = [3402752,3403264) | 20260507 23:21:27.519 | INFO | MemInfo: { | 20260507 23:21:27.519 | INFO | "gpu" : " 17 GB", | 20260507 23:21:27.519 | INFO | "huge" : "", | 20260507 23:21:27.519 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:27.519 | INFO | "malloc" : " 14 GB", | 20260507 23:21:27.519 | INFO | "pinned" : " 29 GB" | 20260507 23:21:27.519 | INFO | } | 20260507 23:21:27.519 | INFO | Perf report for the last 5.14323 seconds: | 20260507 23:21:27.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:27.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:21:27.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:21:27.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:27.519 | INFO | Uploaded 36GiB to GPU in 2.50272s (14.3844GiB/s) | 20260507 23:21:27.519 | INFO | Downloaded 34GiB to GPU in 2.47501s (13.7373GiB/s) | 20260507 23:21:32.654 | INFO | Analyzing chunk frames = [3403264,3403776) | 20260507 23:21:32.654 | INFO | MemInfo: { | 20260507 23:21:32.654 | INFO | "gpu" : " 17 GB", | 20260507 23:21:32.654 | INFO | "huge" : "", | 20260507 23:21:32.654 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:32.654 | INFO | "malloc" : " 14 GB", | 20260507 23:21:32.654 | INFO | "pinned" : " 29 GB" | 20260507 23:21:32.654 | INFO | } | 20260507 23:21:32.654 | INFO | Perf report for the last 5.13548 seconds: | 20260507 23:21:32.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:32.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.655 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 23:21:32.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.655 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:21:32.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:32.655 | INFO | Uploaded 36GiB to GPU in 2.48772s (14.4711GiB/s) | 20260507 23:21:32.655 | INFO | Downloaded 34GiB to GPU in 2.4572s (13.8369GiB/s) | 20260507 23:21:32.752 | INFO | Starting a round of Baz IO | 20260507 23:21:36.216 | INFO | Populating RTMetrics report | 20260507 23:21:36.216 | INFO | RTMetrics report populated | 20260507 23:21:36.216 | INFO | Opening tmp RTMetrics json file | 20260507 23:21:36.231 | INFO | Serializing RTMetrics to json | 20260507 23:21:36.239 | INFO | Serialization done, swapping tmp file now... | 20260507 23:21:36.245 | INFO | RTMetrics done | 20260507 23:21:36.315 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.563 seconds | 20260507 23:21:36.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.65% / 0.01% / 57.34% | 20260507 23:21:36.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21297% / 3.97364e-06% / 6.75519e-05% | 20260507 23:21:36.316 | INFO | Baz IO overall: | 20260507 23:21:36.316 | INFO | BytesWritten : 1549812562982 | 20260507 23:21:36.316 | INFO | NumEvents : 750560171132 | 20260507 23:21:37.791 | INFO | Analyzing chunk frames = [3403776,3404288) | 20260507 23:21:37.791 | INFO | MemInfo: { | 20260507 23:21:37.791 | INFO | "gpu" : " 17 GB", | 20260507 23:21:37.791 | INFO | "huge" : "", | 20260507 23:21:37.791 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:37.791 | INFO | "malloc" : " 14 GB", | 20260507 23:21:37.791 | INFO | "pinned" : " 29 GB" | 20260507 23:21:37.792 | INFO | } | 20260507 23:21:37.792 | INFO | Perf report for the last 5.13711 seconds: | 20260507 23:21:37.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:37.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:37.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:37.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:37.792 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260507 23:21:37.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:37.792 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:21:37.792 | INFO | PRE_HQ : 65.7%, 1.7, 57.1ms, 0 | 20260507 23:21:37.792 | INFO | BAZWRITER : 68.9%, 4.1, 145.8ms, 0 | 20260507 23:21:37.792 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 23:21:37.792 | INFO | Uploaded 36GiB to GPU in 2.52021s (14.2845GiB/s) | 20260507 23:21:37.792 | INFO | Downloaded 38GiB to GPU in 2.69104s (14.1209GiB/s) | 20260507 23:21:42.936 | INFO | Analyzing chunk frames = [3404288,3404800) | 20260507 23:21:42.936 | INFO | MemInfo: { | 20260507 23:21:42.936 | INFO | "gpu" : " 17 GB", | 20260507 23:21:42.936 | INFO | "huge" : "", | 20260507 23:21:42.936 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:42.936 | INFO | "malloc" : " 14 GB", | 20260507 23:21:42.936 | INFO | "pinned" : " 29 GB" | 20260507 23:21:42.936 | INFO | } | 20260507 23:21:42.936 | INFO | Perf report for the last 5.14465 seconds: | 20260507 23:21:42.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:42.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.936 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 23:21:42.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.936 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:21:42.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:42.937 | INFO | Uploaded 36GiB to GPU in 2.48264s (14.5007GiB/s) | 20260507 23:21:42.937 | INFO | Downloaded 34GiB to GPU in 2.4954s (13.6251GiB/s) | 20260507 23:21:48.079 | INFO | Analyzing chunk frames = [3404800,3405312) | 20260507 23:21:48.079 | INFO | MemInfo: { | 20260507 23:21:48.079 | INFO | "gpu" : " 17 GB", | 20260507 23:21:48.079 | INFO | "huge" : "", | 20260507 23:21:48.079 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:48.079 | INFO | "malloc" : " 14 GB", | 20260507 23:21:48.079 | INFO | "pinned" : " 29 GB" | 20260507 23:21:48.079 | INFO | } | 20260507 23:21:48.079 | INFO | Perf report for the last 5.14295 seconds: | 20260507 23:21:48.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:48.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.079 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260507 23:21:48.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.079 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 23:21:48.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:48.080 | INFO | Uploaded 36GiB to GPU in 2.43877s (14.7616GiB/s) | 20260507 23:21:48.080 | INFO | Downloaded 34GiB to GPU in 2.49144s (13.6467GiB/s) | 20260507 23:21:53.212 | INFO | Analyzing chunk frames = [3405312,3405824) | 20260507 23:21:53.212 | INFO | MemInfo: { | 20260507 23:21:53.212 | INFO | "gpu" : " 17 GB", | 20260507 23:21:53.212 | INFO | "huge" : "", | 20260507 23:21:53.212 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:53.213 | INFO | "malloc" : " 14 GB", | 20260507 23:21:53.213 | INFO | "pinned" : " 29 GB" | 20260507 23:21:53.213 | INFO | } | 20260507 23:21:53.213 | INFO | Perf report for the last 5.13349 seconds: | 20260507 23:21:53.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:53.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260507 23:21:53.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:21:53.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:53.213 | INFO | Uploaded 36GiB to GPU in 2.44012s (14.7534GiB/s) | 20260507 23:21:53.213 | INFO | Downloaded 34GiB to GPU in 2.48672s (13.6727GiB/s) | 20260507 23:21:58.346 | INFO | Analyzing chunk frames = [3405824,3406336) | 20260507 23:21:58.346 | INFO | MemInfo: { | 20260507 23:21:58.346 | INFO | "gpu" : " 17 GB", | 20260507 23:21:58.346 | INFO | "huge" : "", | 20260507 23:21:58.346 | INFO | "huge_shared" : " 29 GB", | 20260507 23:21:58.346 | INFO | "malloc" : " 14 GB", | 20260507 23:21:58.346 | INFO | "pinned" : " 29 GB" | 20260507 23:21:58.346 | INFO | } | 20260507 23:21:58.346 | INFO | Perf report for the last 5.13377 seconds: | 20260507 23:21:58.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:21:58.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260507 23:21:58.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260507 23:21:58.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:21:58.347 | INFO | Uploaded 36GiB to GPU in 2.32178s (15.5054GiB/s) | 20260507 23:21:58.347 | INFO | Downloaded 34GiB to GPU in 2.49633s (13.62GiB/s) | 20260507 23:22:03.489 | INFO | Analyzing chunk frames = [3406336,3406848) | 20260507 23:22:03.489 | INFO | MemInfo: { | 20260507 23:22:03.489 | INFO | "gpu" : " 17 GB", | 20260507 23:22:03.489 | INFO | "huge" : "", | 20260507 23:22:03.490 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:03.490 | INFO | "malloc" : " 14 GB", | 20260507 23:22:03.490 | INFO | "pinned" : " 29 GB" | 20260507 23:22:03.490 | INFO | } | 20260507 23:22:03.490 | INFO | Perf report for the last 5.1432 seconds: | 20260507 23:22:03.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:03.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 23:22:03.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:22:03.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:03.490 | INFO | Uploaded 36GiB to GPU in 2.5009s (14.3948GiB/s) | 20260507 23:22:03.490 | INFO | Downloaded 34GiB to GPU in 2.43534s (13.9611GiB/s) | 20260507 23:22:08.633 | INFO | Analyzing chunk frames = [3406848,3407360) | 20260507 23:22:08.633 | INFO | MemInfo: { | 20260507 23:22:08.633 | INFO | "gpu" : " 17 GB", | 20260507 23:22:08.633 | INFO | "huge" : "", | 20260507 23:22:08.633 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:08.633 | INFO | "malloc" : " 14 GB", | 20260507 23:22:08.633 | INFO | "pinned" : " 29 GB" | 20260507 23:22:08.633 | INFO | } | 20260507 23:22:08.633 | INFO | Perf report for the last 5.14361 seconds: | 20260507 23:22:08.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:08.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.633 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 23:22:08.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.633 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:22:08.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:08.634 | INFO | Uploaded 36GiB to GPU in 2.48746s (14.4726GiB/s) | 20260507 23:22:08.634 | INFO | Downloaded 34GiB to GPU in 2.44859s (13.8855GiB/s) | 20260507 23:22:13.775 | INFO | Analyzing chunk frames = [3407360,3407872) | 20260507 23:22:13.775 | INFO | MemInfo: { | 20260507 23:22:13.775 | INFO | "gpu" : " 17 GB", | 20260507 23:22:13.775 | INFO | "huge" : "", | 20260507 23:22:13.776 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:13.776 | INFO | "malloc" : " 14 GB", | 20260507 23:22:13.776 | INFO | "pinned" : " 29 GB" | 20260507 23:22:13.776 | INFO | } | 20260507 23:22:13.776 | INFO | Perf report for the last 5.1424 seconds: | 20260507 23:22:13.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:13.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:22:13.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:22:13.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:13.776 | INFO | Uploaded 36GiB to GPU in 2.4827s (14.5004GiB/s) | 20260507 23:22:13.776 | INFO | Downloaded 34GiB to GPU in 2.44998s (13.8777GiB/s) | 20260507 23:22:13.776 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:22:13.776 | INFO | Total mem usage: 16.7 GiB | 20260507 23:22:13.776 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:22:13.776 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260507 23:22:13.776 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:22:13.776 | INFO | High Water Breakdown: | 20260507 23:22:13.776 | INFO | 60.1% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:22:13.776 | INFO | 39.8% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:22:13.776 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:22:13.776 | INFO | ------------------------------------------ | 20260507 23:22:13.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:22:13.776 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:22:13.776 | INFO | Total mem usage: 26.5 GiB | 20260507 23:22:13.776 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:22:13.776 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:22:13.776 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:22:13.776 | INFO | High Water Breakdown: | 20260507 23:22:13.776 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:22:13.776 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:22:13.776 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:22:13.776 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:22:13.776 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:22:13.776 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:22:13.776 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:22:13.776 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:22:13.776 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:22:13.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:22:13.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:22:13.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:22:13.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:22:13.776 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:22:13.776 | INFO | ------------------------------------------ | 20260507 23:22:13.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:22:13.776 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:22:13.776 | INFO | Total mem usage: 27.8 GiB | 20260507 23:22:13.776 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:22:13.776 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:22:13.776 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:22:13.776 | INFO | High Water Breakdown: | 20260507 23:22:13.776 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:22:13.776 | INFO | ------------------------------------------ | 20260507 23:22:13.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:22:13.776 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:22:13.776 | INFO | Total mem usage: 15.7 GiB | 20260507 23:22:13.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:22:13.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:22:13.776 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:22:13.776 | INFO | High Water Breakdown: | 20260507 23:22:13.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:22:13.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:22:13.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:22:13.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:22:13.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:22:13.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:22:13.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:22:13.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:22:13.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:22:13.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:22:13.777 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:22:13.777 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:22:13.777 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:22:13.777 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:22:13.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:22:13.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:22:13.777 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:22:13.777 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:22:13.777 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:22:13.777 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:22:13.777 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:22:13.777 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:22:13.777 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:22:13.777 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:22:13.777 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:22:13.777 | INFO | ------------------------------------------ | 20260507 23:22:13.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:22:13.777 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:22:13.777 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:22:13.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:22:13.777 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.43e+01 / 1.73e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.01e-01, 1.14e-02 / 5.79e-03 | 20260507 23:22:13.777 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.48e+00, 5.93e-01 / 2.02e-01 | 20260507 23:22:13.777 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.10e-01 | 20260507 23:22:13.777 | INFO | PerfReport: 16.14%, DME -- 4.24e+01 / 4.63e+01, 4.15e+00 / 2.01e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.55e+00 / 1.53e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.47e+00 / 2.74e+00, 4.36e-01 / 2.13e-01 | 20260507 23:22:13.777 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.37e+00, 2.81e-01 / 1.48e-01 | 20260507 23:22:13.777 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.84e-01, 5.62e-02 / 2.48e-02 | 20260507 23:22:13.777 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:22:13.777 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:22:13.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:22:13.777 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 24.06%, Upload -- 2.57e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 24.43%, Download -- 2.61e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 30.76%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 1.88%, LockUpload -- 2.01e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 3.12%, LockDownload -- 3.33e+00 / 0.00e+00, 4.51e+00 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 15.74%, LockCompute -- 1.68e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 23:22:13.777 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260507 23:22:13.882 | INFO | Starting a round of Baz IO | 20260507 23:22:17.345 | INFO | Populating RTMetrics report | 20260507 23:22:17.345 | INFO | RTMetrics report populated | 20260507 23:22:17.345 | INFO | Opening tmp RTMetrics json file | 20260507 23:22:17.376 | INFO | Serializing RTMetrics to json | 20260507 23:22:17.378 | INFO | Serialization done, swapping tmp file now... | 20260507 23:22:17.379 | INFO | RTMetrics done | 20260507 23:22:17.444 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.562 seconds | 20260507 23:22:17.444 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.65% / 0.01% / 57.35% | 20260507 23:22:17.444 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21002% / 3.97364e-06% / 5.5631e-05% | 20260507 23:22:17.444 | INFO | Baz IO overall: | 20260507 23:22:17.444 | INFO | BytesWritten : 1551655784734 | 20260507 23:22:17.444 | INFO | NumEvents : 751448763779 | 20260507 23:22:18.911 | INFO | Analyzing chunk frames = [3407872,3408384) | 20260507 23:22:18.911 | INFO | MemInfo: { | 20260507 23:22:18.911 | INFO | "gpu" : " 17 GB", | 20260507 23:22:18.911 | INFO | "huge" : "", | 20260507 23:22:18.911 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:18.911 | INFO | "malloc" : " 14 GB", | 20260507 23:22:18.911 | INFO | "pinned" : " 29 GB" | 20260507 23:22:18.911 | INFO | } | 20260507 23:22:18.911 | INFO | Perf report for the last 5.13541 seconds: | 20260507 23:22:18.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:18.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:18.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:18.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:18.911 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 23:22:18.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:18.911 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 23:22:18.911 | INFO | PRE_HQ : 65.7%, 1.7, 57.5ms, 0 | 20260507 23:22:18.911 | INFO | BAZWRITER : 69.0%, 3.9, 139.7ms, 0 | 20260507 23:22:18.911 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 23:22:18.912 | INFO | Uploaded 36GiB to GPU in 2.5372s (14.1889GiB/s) | 20260507 23:22:18.912 | INFO | Downloaded 38GiB to GPU in 2.69309s (14.1102GiB/s) | 20260507 23:22:24.047 | INFO | Analyzing chunk frames = [3408384,3408896) | 20260507 23:22:24.047 | INFO | MemInfo: { | 20260507 23:22:24.047 | INFO | "gpu" : " 17 GB", | 20260507 23:22:24.047 | INFO | "huge" : "", | 20260507 23:22:24.047 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:24.047 | INFO | "malloc" : " 14 GB", | 20260507 23:22:24.047 | INFO | "pinned" : " 29 GB" | 20260507 23:22:24.047 | INFO | } | 20260507 23:22:24.047 | INFO | Perf report for the last 5.1363 seconds: | 20260507 23:22:24.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:24.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.047 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260507 23:22:24.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.047 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:22:24.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:24.048 | INFO | Uploaded 36GiB to GPU in 2.56237s (14.0495GiB/s) | 20260507 23:22:24.048 | INFO | Downloaded 34GiB to GPU in 2.49586s (13.6225GiB/s) | 20260507 23:22:29.190 | INFO | Analyzing chunk frames = [3408896,3409408) | 20260507 23:22:29.190 | INFO | MemInfo: { | 20260507 23:22:29.190 | INFO | "gpu" : " 17 GB", | 20260507 23:22:29.190 | INFO | "huge" : "", | 20260507 23:22:29.190 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:29.190 | INFO | "malloc" : " 14 GB", | 20260507 23:22:29.190 | INFO | "pinned" : " 29 GB" | 20260507 23:22:29.190 | INFO | } | 20260507 23:22:29.190 | INFO | Perf report for the last 5.1429 seconds: | 20260507 23:22:29.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:29.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.190 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:22:29.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.191 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:22:29.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:29.191 | INFO | Uploaded 36GiB to GPU in 2.48673s (14.4769GiB/s) | 20260507 23:22:29.192 | INFO | Downloaded 34GiB to GPU in 2.48859s (13.6623GiB/s) | 20260507 23:22:34.328 | INFO | Analyzing chunk frames = [3409408,3409920) | 20260507 23:22:34.328 | INFO | MemInfo: { | 20260507 23:22:34.328 | INFO | "gpu" : " 17 GB", | 20260507 23:22:34.328 | INFO | "huge" : "", | 20260507 23:22:34.328 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:34.328 | INFO | "malloc" : " 14 GB", | 20260507 23:22:34.328 | INFO | "pinned" : " 29 GB" | 20260507 23:22:34.328 | INFO | } | 20260507 23:22:34.329 | INFO | Perf report for the last 5.13822 seconds: | 20260507 23:22:34.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:34.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:22:34.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260507 23:22:34.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:34.329 | INFO | Uploaded 36GiB to GPU in 2.50651s (14.3626GiB/s) | 20260507 23:22:34.329 | INFO | Downloaded 34GiB to GPU in 2.49583s (13.6227GiB/s) | 20260507 23:22:39.461 | INFO | Analyzing chunk frames = [3409920,3410432) | 20260507 23:22:39.461 | INFO | MemInfo: { | 20260507 23:22:39.461 | INFO | "gpu" : " 17 GB", | 20260507 23:22:39.461 | INFO | "huge" : "", | 20260507 23:22:39.461 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:39.461 | INFO | "malloc" : " 14 GB", | 20260507 23:22:39.461 | INFO | "pinned" : " 29 GB" | 20260507 23:22:39.461 | INFO | } | 20260507 23:22:39.461 | INFO | Perf report for the last 5.13249 seconds: | 20260507 23:22:39.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:39.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 23:22:39.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 23:22:39.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:39.461 | INFO | Uploaded 36GiB to GPU in 2.36958s (15.1925GiB/s) | 20260507 23:22:39.461 | INFO | Downloaded 34GiB to GPU in 2.50384s (13.5791GiB/s) | 20260507 23:22:44.602 | INFO | Analyzing chunk frames = [3410432,3410944) | 20260507 23:22:44.602 | INFO | MemInfo: { | 20260507 23:22:44.602 | INFO | "gpu" : " 17 GB", | 20260507 23:22:44.602 | INFO | "huge" : "", | 20260507 23:22:44.602 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:44.602 | INFO | "malloc" : " 14 GB", | 20260507 23:22:44.602 | INFO | "pinned" : " 29 GB" | 20260507 23:22:44.602 | INFO | } | 20260507 23:22:44.602 | INFO | Perf report for the last 5.14114 seconds: | 20260507 23:22:44.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:44.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260507 23:22:44.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 23:22:44.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:44.602 | INFO | Uploaded 36GiB to GPU in 2.46931s (14.579GiB/s) | 20260507 23:22:44.603 | INFO | Downloaded 34GiB to GPU in 2.44487s (13.9067GiB/s) | 20260507 23:22:49.744 | INFO | Analyzing chunk frames = [3410944,3411456) | 20260507 23:22:49.744 | INFO | MemInfo: { | 20260507 23:22:49.744 | INFO | "gpu" : " 17 GB", | 20260507 23:22:49.744 | INFO | "huge" : "", | 20260507 23:22:49.744 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:49.744 | INFO | "malloc" : " 14 GB", | 20260507 23:22:49.744 | INFO | "pinned" : " 29 GB" | 20260507 23:22:49.744 | INFO | } | 20260507 23:22:49.744 | INFO | Perf report for the last 5.14183 seconds: | 20260507 23:22:49.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:49.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.744 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260507 23:22:49.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.744 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:22:49.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:49.745 | INFO | Uploaded 36GiB to GPU in 2.4506s (14.6903GiB/s) | 20260507 23:22:49.745 | INFO | Downloaded 34GiB to GPU in 2.46331s (13.8026GiB/s) | 20260507 23:22:54.887 | INFO | Analyzing chunk frames = [3411456,3411968) | 20260507 23:22:54.887 | INFO | MemInfo: { | 20260507 23:22:54.887 | INFO | "gpu" : " 17 GB", | 20260507 23:22:54.887 | INFO | "huge" : "", | 20260507 23:22:54.887 | INFO | "huge_shared" : " 29 GB", | 20260507 23:22:54.887 | INFO | "malloc" : " 14 GB", | 20260507 23:22:54.887 | INFO | "pinned" : " 29 GB" | 20260507 23:22:54.887 | INFO | } | 20260507 23:22:54.887 | INFO | Perf report for the last 5.1429 seconds: | 20260507 23:22:54.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:22:54.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.887 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260507 23:22:54.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.887 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:22:54.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:22:54.888 | INFO | Uploaded 36GiB to GPU in 2.42129s (14.8681GiB/s) | 20260507 23:22:54.888 | INFO | Downloaded 34GiB to GPU in 2.44972s (13.8792GiB/s) | 20260507 23:22:54.985 | INFO | Starting a round of Baz IO | 20260507 23:22:58.439 | INFO | Populating RTMetrics report | 20260507 23:22:58.439 | INFO | RTMetrics report populated | 20260507 23:22:58.439 | INFO | Opening tmp RTMetrics json file | 20260507 23:22:58.444 | INFO | Serializing RTMetrics to json | 20260507 23:22:58.474 | INFO | Serialization done, swapping tmp file now... | 20260507 23:22:58.486 | INFO | RTMetrics done | 20260507 23:22:58.554 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.569 seconds | 20260507 23:22:58.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.64% / 0.01% / 57.35% | 20260507 23:22:58.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20354% / 7.94729e-06% / 2.78155e-05% | 20260507 23:22:58.554 | INFO | Baz IO overall: | 20260507 23:22:58.554 | INFO | BytesWritten : 1553500646590 | 20260507 23:22:58.554 | INFO | NumEvents : 752336731184 | 20260507 23:23:00.027 | INFO | Analyzing chunk frames = [3411968,3412480) | 20260507 23:23:00.027 | INFO | MemInfo: { | 20260507 23:23:00.027 | INFO | "gpu" : " 17 GB", | 20260507 23:23:00.027 | INFO | "huge" : "", | 20260507 23:23:00.027 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:00.027 | INFO | "malloc" : " 14 GB", | 20260507 23:23:00.027 | INFO | "pinned" : " 29 GB" | 20260507 23:23:00.028 | INFO | } | 20260507 23:23:00.028 | INFO | Perf report for the last 5.14097 seconds: | 20260507 23:23:00.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:00.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:00.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:00.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:00.028 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 23:23:00.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:00.028 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 23:23:00.028 | INFO | PRE_HQ : 65.5%, 1.7, 57.0ms, 0 | 20260507 23:23:00.028 | INFO | BAZWRITER : 69.0%, 3.6, 128.4ms, 0 | 20260507 23:23:00.028 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.6ms, 0 | 20260507 23:23:00.028 | INFO | Uploaded 36GiB to GPU in 2.53316s (14.2115GiB/s) | 20260507 23:23:00.028 | INFO | Downloaded 38GiB to GPU in 2.69108s (14.1207GiB/s) | 20260507 23:23:05.161 | INFO | Analyzing chunk frames = [3412480,3412992) | 20260507 23:23:05.161 | INFO | MemInfo: { | 20260507 23:23:05.161 | INFO | "gpu" : " 17 GB", | 20260507 23:23:05.161 | INFO | "huge" : "", | 20260507 23:23:05.161 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:05.161 | INFO | "malloc" : " 14 GB", | 20260507 23:23:05.161 | INFO | "pinned" : " 29 GB" | 20260507 23:23:05.161 | INFO | } | 20260507 23:23:05.161 | INFO | Perf report for the last 5.1332 seconds: | 20260507 23:23:05.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:05.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.161 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 23:23:05.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.161 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:23:05.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:05.162 | INFO | Uploaded 36GiB to GPU in 2.49143s (14.4496GiB/s) | 20260507 23:23:05.162 | INFO | Downloaded 34GiB to GPU in 2.43786s (13.9467GiB/s) | 20260507 23:23:10.299 | INFO | Analyzing chunk frames = [3412992,3413504) | 20260507 23:23:10.299 | INFO | MemInfo: { | 20260507 23:23:10.299 | INFO | "gpu" : " 17 GB", | 20260507 23:23:10.299 | INFO | "huge" : "", | 20260507 23:23:10.299 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:10.299 | INFO | "malloc" : " 14 GB", | 20260507 23:23:10.299 | INFO | "pinned" : " 29 GB" | 20260507 23:23:10.299 | INFO | } | 20260507 23:23:10.299 | INFO | Perf report for the last 5.138 seconds: | 20260507 23:23:10.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:10.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.299 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:23:10.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.299 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:23:10.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:10.300 | INFO | Uploaded 36GiB to GPU in 2.48791s (14.47GiB/s) | 20260507 23:23:10.300 | INFO | Downloaded 34GiB to GPU in 2.50465s (13.5747GiB/s) | 20260507 23:23:15.440 | INFO | Analyzing chunk frames = [3413504,3414016) | 20260507 23:23:15.440 | INFO | MemInfo: { | 20260507 23:23:15.440 | INFO | "gpu" : " 17 GB", | 20260507 23:23:15.440 | INFO | "huge" : "", | 20260507 23:23:15.440 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:15.440 | INFO | "malloc" : " 14 GB", | 20260507 23:23:15.440 | INFO | "pinned" : " 29 GB" | 20260507 23:23:15.440 | INFO | } | 20260507 23:23:15.441 | INFO | Perf report for the last 5.14134 seconds: | 20260507 23:23:15.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:15.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260507 23:23:15.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:23:15.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:15.441 | INFO | Uploaded 36GiB to GPU in 2.43308s (14.796GiB/s) | 20260507 23:23:15.441 | INFO | Downloaded 34GiB to GPU in 2.41964s (14.0517GiB/s) | 20260507 23:23:20.585 | INFO | Analyzing chunk frames = [3414016,3414528) | 20260507 23:23:20.585 | INFO | MemInfo: { | 20260507 23:23:20.585 | INFO | "gpu" : " 17 GB", | 20260507 23:23:20.585 | INFO | "huge" : "", | 20260507 23:23:20.585 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:20.585 | INFO | "malloc" : " 14 GB", | 20260507 23:23:20.585 | INFO | "pinned" : " 29 GB" | 20260507 23:23:20.585 | INFO | } | 20260507 23:23:20.585 | INFO | Perf report for the last 5.14447 seconds: | 20260507 23:23:20.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:20.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.585 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 23:23:20.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.585 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:23:20.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:20.586 | INFO | Uploaded 36GiB to GPU in 2.37666s (15.1473GiB/s) | 20260507 23:23:20.586 | INFO | Downloaded 34GiB to GPU in 2.50205s (13.5888GiB/s) | 20260507 23:23:25.727 | INFO | Analyzing chunk frames = [3414528,3415040) | 20260507 23:23:25.727 | INFO | MemInfo: { | 20260507 23:23:25.727 | INFO | "gpu" : " 17 GB", | 20260507 23:23:25.727 | INFO | "huge" : "", | 20260507 23:23:25.727 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:25.727 | INFO | "malloc" : " 14 GB", | 20260507 23:23:25.727 | INFO | "pinned" : " 29 GB" | 20260507 23:23:25.727 | INFO | } | 20260507 23:23:25.727 | INFO | Perf report for the last 5.14211 seconds: | 20260507 23:23:25.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:25.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:23:25.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:23:25.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:25.727 | INFO | Uploaded 36GiB to GPU in 2.39381s (15.0388GiB/s) | 20260507 23:23:25.727 | INFO | Downloaded 34GiB to GPU in 2.40954s (14.1106GiB/s) | 20260507 23:23:30.859 | INFO | Analyzing chunk frames = [3415040,3415552) | 20260507 23:23:30.859 | INFO | MemInfo: { | 20260507 23:23:30.859 | INFO | "gpu" : " 17 GB", | 20260507 23:23:30.859 | INFO | "huge" : "", | 20260507 23:23:30.859 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:30.859 | INFO | "malloc" : " 14 GB", | 20260507 23:23:30.859 | INFO | "pinned" : " 29 GB" | 20260507 23:23:30.859 | INFO | } | 20260507 23:23:30.859 | INFO | Perf report for the last 5.13225 seconds: | 20260507 23:23:30.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:30.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:23:30.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:23:30.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:30.860 | INFO | Uploaded 36GiB to GPU in 2.55192s (14.107GiB/s) | 20260507 23:23:30.860 | INFO | Downloaded 34GiB to GPU in 2.47434s (13.7411GiB/s) | 20260507 23:23:36.000 | INFO | Analyzing chunk frames = [3415552,3416064) | 20260507 23:23:36.000 | INFO | MemInfo: { | 20260507 23:23:36.000 | INFO | "gpu" : " 17 GB", | 20260507 23:23:36.000 | INFO | "huge" : "", | 20260507 23:23:36.000 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:36.000 | INFO | "malloc" : " 14 GB", | 20260507 23:23:36.000 | INFO | "pinned" : " 29 GB" | 20260507 23:23:36.000 | INFO | } | 20260507 23:23:36.001 | INFO | Perf report for the last 5.1412 seconds: | 20260507 23:23:36.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:36.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:23:36.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 23:23:36.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:36.001 | INFO | Uploaded 36GiB to GPU in 2.50615s (14.3647GiB/s) | 20260507 23:23:36.001 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:23:36.001 | INFO | Total mem usage: 16.7 GiB | 20260507 23:23:36.001 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:23:36.001 | INFO | Downloaded 34GiB to GPU in 2.44987s (13.8783GiB/s) | 20260507 23:23:36.001 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.5%) | 20260507 23:23:36.001 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:23:36.001 | INFO | High Water Breakdown: | 20260507 23:23:36.001 | INFO | 59.6% : 8.69 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:23:36.001 | INFO | 40.2% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:23:36.001 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:23:36.001 | INFO | ------------------------------------------ | 20260507 23:23:36.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:23:36.001 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:23:36.001 | INFO | Total mem usage: 26.5 GiB | 20260507 23:23:36.001 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:23:36.001 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:23:36.001 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:23:36.001 | INFO | High Water Breakdown: | 20260507 23:23:36.001 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:23:36.001 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:23:36.001 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:23:36.001 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:23:36.001 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:23:36.001 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:23:36.001 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:23:36.001 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:23:36.001 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:23:36.001 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:23:36.001 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:23:36.001 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:23:36.001 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:23:36.001 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:23:36.001 | INFO | ------------------------------------------ | 20260507 23:23:36.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:23:36.001 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:23:36.001 | INFO | Total mem usage: 27.8 GiB | 20260507 23:23:36.001 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:23:36.001 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:23:36.001 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:23:36.001 | INFO | High Water Breakdown: | 20260507 23:23:36.001 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:23:36.001 | INFO | ------------------------------------------ | 20260507 23:23:36.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:23:36.001 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:23:36.001 | INFO | Total mem usage: 15.7 GiB | 20260507 23:23:36.001 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:23:36.001 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:23:36.001 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:23:36.001 | INFO | High Water Breakdown: | 20260507 23:23:36.001 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:23:36.001 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:23:36.001 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:23:36.001 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:23:36.001 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:23:36.001 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:23:36.001 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:23:36.001 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:23:36.001 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:23:36.001 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:23:36.001 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:23:36.001 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:23:36.001 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:23:36.001 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:23:36.001 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:23:36.001 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:23:36.001 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:23:36.001 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:23:36.001 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:23:36.001 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:23:36.001 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:23:36.001 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:23:36.001 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:23:36.001 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:23:36.001 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:23:36.001 | INFO | ------------------------------------------ | 20260507 23:23:36.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:23:36.001 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:23:36.001 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:23:36.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:23:36.001 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.61e+01, 1.41e+01 / 1.53e+00 | 20260507 23:23:36.001 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.16e-02 / 4.90e-03 | 20260507 23:23:36.001 | INFO | PerfReport: 12.14%, Baseline -- 3.99e+00 / 4.49e+00, 5.81e-01 / 1.79e-01 | 20260507 23:23:36.001 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.09e-01 | 20260507 23:23:36.001 | INFO | PerfReport: 15.99%, DME -- 4.20e+01 / 4.59e+01, 3.95e+00 / 1.47e+00 | 20260507 23:23:36.001 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.46e+00 | 20260507 23:23:36.001 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.27e-01 / 1.95e-01 | 20260507 23:23:36.001 | INFO | PerfReport: 3.43%, Metrics -- 1.13e+00 / 1.35e+00, 2.81e-01 / 1.37e-01 | 20260507 23:23:36.001 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.90e-01, 5.63e-02 / 2.61e-02 | 20260507 23:23:36.001 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:23:36.001 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:23:36.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:23:36.002 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 24.15%, Upload -- 2.58e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 24.36%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 30.79%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 1.91%, LockUpload -- 2.03e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 2.89%, LockDownload -- 3.09e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 15.88%, LockCompute -- 1.69e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 23:23:36.002 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260507 23:23:36.098 | INFO | Starting a round of Baz IO | 20260507 23:23:39.560 | INFO | Populating RTMetrics report | 20260507 23:23:39.561 | INFO | RTMetrics report populated | 20260507 23:23:39.561 | INFO | Opening tmp RTMetrics json file | 20260507 23:23:39.562 | INFO | Serializing RTMetrics to json | 20260507 23:23:39.586 | INFO | Serialization done, swapping tmp file now... | 20260507 23:23:39.589 | INFO | RTMetrics done | 20260507 23:23:39.671 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.573 seconds | 20260507 23:23:39.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.64% / 0.01% / 57.36% | 20260507 23:23:39.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19241% / 0% / 4.37101e-05% | 20260507 23:23:39.672 | INFO | Baz IO overall: | 20260507 23:23:39.672 | INFO | BytesWritten : 1555346560559 | 20260507 23:23:39.672 | INFO | NumEvents : 753224161770 | 20260507 23:23:41.138 | INFO | Analyzing chunk frames = [3416064,3416576) | 20260507 23:23:41.138 | INFO | MemInfo: { | 20260507 23:23:41.138 | INFO | "gpu" : " 17 GB", | 20260507 23:23:41.138 | INFO | "huge" : "", | 20260507 23:23:41.138 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:41.138 | INFO | "malloc" : " 14 GB", | 20260507 23:23:41.138 | INFO | "pinned" : " 29 GB" | 20260507 23:23:41.138 | INFO | } | 20260507 23:23:41.138 | INFO | Perf report for the last 5.13714 seconds: | 20260507 23:23:41.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:41.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:41.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:41.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:41.138 | INFO | ANALYSIS : 68.6%, 3.0, 106.3ms, 0 | 20260507 23:23:41.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:41.138 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 23:23:41.138 | INFO | PRE_HQ : 65.9%, 1.7, 57.5ms, 0 | 20260507 23:23:41.138 | INFO | BAZWRITER : 69.1%, 3.9, 138.3ms, 0 | 20260507 23:23:41.138 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 23:23:41.138 | INFO | Uploaded 36GiB to GPU in 2.55302s (14.101GiB/s) | 20260507 23:23:41.138 | INFO | Downloaded 38GiB to GPU in 2.69971s (14.0756GiB/s) | 20260507 23:23:46.276 | INFO | Analyzing chunk frames = [3416576,3417088) | 20260507 23:23:46.276 | INFO | MemInfo: { | 20260507 23:23:46.276 | INFO | "gpu" : " 17 GB", | 20260507 23:23:46.276 | INFO | "huge" : "", | 20260507 23:23:46.276 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:46.276 | INFO | "malloc" : " 14 GB", | 20260507 23:23:46.276 | INFO | "pinned" : " 29 GB" | 20260507 23:23:46.276 | INFO | } | 20260507 23:23:46.276 | INFO | Perf report for the last 5.13848 seconds: | 20260507 23:23:46.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:46.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 23:23:46.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:23:46.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:46.276 | INFO | Uploaded 36GiB to GPU in 2.56711s (14.0236GiB/s) | 20260507 23:23:46.276 | INFO | Downloaded 34GiB to GPU in 2.49701s (13.6163GiB/s) | 20260507 23:23:51.412 | INFO | Analyzing chunk frames = [3417088,3417600) | 20260507 23:23:51.412 | INFO | MemInfo: { | 20260507 23:23:51.412 | INFO | "gpu" : " 17 GB", | 20260507 23:23:51.412 | INFO | "huge" : "", | 20260507 23:23:51.412 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:51.412 | INFO | "malloc" : " 14 GB", | 20260507 23:23:51.412 | INFO | "pinned" : " 29 GB" | 20260507 23:23:51.412 | INFO | } | 20260507 23:23:51.412 | INFO | Perf report for the last 5.13596 seconds: | 20260507 23:23:51.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:51.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.412 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:23:51.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.413 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:23:51.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:51.413 | INFO | Uploaded 36GiB to GPU in 2.49524s (14.4275GiB/s) | 20260507 23:23:51.413 | INFO | Downloaded 34GiB to GPU in 2.49234s (13.6418GiB/s) | 20260507 23:23:56.553 | INFO | Analyzing chunk frames = [3417600,3418112) | 20260507 23:23:56.553 | INFO | MemInfo: { | 20260507 23:23:56.553 | INFO | "gpu" : " 17 GB", | 20260507 23:23:56.553 | INFO | "huge" : "", | 20260507 23:23:56.553 | INFO | "huge_shared" : " 29 GB", | 20260507 23:23:56.553 | INFO | "malloc" : " 14 GB", | 20260507 23:23:56.553 | INFO | "pinned" : " 29 GB" | 20260507 23:23:56.553 | INFO | } | 20260507 23:23:56.553 | INFO | Perf report for the last 5.14047 seconds: | 20260507 23:23:56.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:23:56.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:23:56.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:23:56.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:23:56.553 | INFO | Uploaded 36GiB to GPU in 2.46364s (14.6125GiB/s) | 20260507 23:23:56.553 | INFO | Downloaded 34GiB to GPU in 2.40391s (14.1436GiB/s) | 20260507 23:24:01.695 | INFO | Analyzing chunk frames = [3418112,3418624) | 20260507 23:24:01.695 | INFO | MemInfo: { | 20260507 23:24:01.695 | INFO | "gpu" : " 17 GB", | 20260507 23:24:01.695 | INFO | "huge" : "", | 20260507 23:24:01.695 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:01.695 | INFO | "malloc" : " 14 GB", | 20260507 23:24:01.695 | INFO | "pinned" : " 29 GB" | 20260507 23:24:01.695 | INFO | } | 20260507 23:24:01.695 | INFO | Perf report for the last 5.14207 seconds: | 20260507 23:24:01.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:01.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:24:01.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:24:01.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:01.695 | INFO | Uploaded 36GiB to GPU in 2.32312s (15.4964GiB/s) | 20260507 23:24:01.695 | INFO | Downloaded 34GiB to GPU in 2.50615s (13.5666GiB/s) | 20260507 23:24:06.838 | INFO | Analyzing chunk frames = [3418624,3419136) | 20260507 23:24:06.838 | INFO | MemInfo: { | 20260507 23:24:06.838 | INFO | "gpu" : " 17 GB", | 20260507 23:24:06.838 | INFO | "huge" : "", | 20260507 23:24:06.838 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:06.838 | INFO | "malloc" : " 14 GB", | 20260507 23:24:06.838 | INFO | "pinned" : " 29 GB" | 20260507 23:24:06.838 | INFO | } | 20260507 23:24:06.838 | INFO | Perf report for the last 5.14344 seconds: | 20260507 23:24:06.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:06.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.838 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260507 23:24:06.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.838 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:24:06.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:06.839 | INFO | Uploaded 36GiB to GPU in 2.5025s (14.3856GiB/s) | 20260507 23:24:06.839 | INFO | Downloaded 34GiB to GPU in 2.43978s (13.9357GiB/s) | 20260507 23:24:11.977 | INFO | Analyzing chunk frames = [3419136,3419648) | 20260507 23:24:11.977 | INFO | MemInfo: { | 20260507 23:24:11.977 | INFO | "gpu" : " 17 GB", | 20260507 23:24:11.977 | INFO | "huge" : "", | 20260507 23:24:11.977 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:11.977 | INFO | "malloc" : " 14 GB", | 20260507 23:24:11.977 | INFO | "pinned" : " 29 GB" | 20260507 23:24:11.978 | INFO | } | 20260507 23:24:11.978 | INFO | Perf report for the last 5.13928 seconds: | 20260507 23:24:11.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:11.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 23:24:11.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:24:11.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:11.978 | INFO | Uploaded 36GiB to GPU in 2.52917s (14.2339GiB/s) | 20260507 23:24:11.978 | INFO | Downloaded 34GiB to GPU in 2.47087s (13.7603GiB/s) | 20260507 23:24:17.111 | INFO | Analyzing chunk frames = [3419648,3420160) | 20260507 23:24:17.111 | INFO | MemInfo: { | 20260507 23:24:17.111 | INFO | "gpu" : " 17 GB", | 20260507 23:24:17.111 | INFO | "huge" : "", | 20260507 23:24:17.111 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:17.111 | INFO | "malloc" : " 14 GB", | 20260507 23:24:17.111 | INFO | "pinned" : " 29 GB" | 20260507 23:24:17.111 | INFO | } | 20260507 23:24:17.112 | INFO | Perf report for the last 5.13395 seconds: | 20260507 23:24:17.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:17.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 23:24:17.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260507 23:24:17.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:17.112 | INFO | Uploaded 36GiB to GPU in 2.47802s (14.5278GiB/s) | 20260507 23:24:17.112 | INFO | Downloaded 34GiB to GPU in 2.44513s (13.9052GiB/s) | 20260507 23:24:17.208 | INFO | Starting a round of Baz IO | 20260507 23:24:20.674 | INFO | Populating RTMetrics report | 20260507 23:24:20.674 | INFO | RTMetrics report populated | 20260507 23:24:20.674 | INFO | Opening tmp RTMetrics json file | 20260507 23:24:20.684 | INFO | Serializing RTMetrics to json | 20260507 23:24:20.697 | INFO | Serialization done, swapping tmp file now... | 20260507 23:24:20.697 | INFO | RTMetrics done | 20260507 23:24:20.775 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.566 seconds | 20260507 23:24:20.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.63% / 0.00% / 57.36% | 20260507 23:24:20.775 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19167% / 7.94729e-06% / 4.76837e-05% | 20260507 23:24:20.775 | INFO | Baz IO overall: | 20260507 23:24:20.775 | INFO | BytesWritten : 1557190280006 | 20260507 23:24:20.776 | INFO | NumEvents : 754111027610 | 20260507 23:24:22.255 | INFO | Analyzing chunk frames = [3420160,3420672) | 20260507 23:24:22.255 | INFO | MemInfo: { | 20260507 23:24:22.255 | INFO | "gpu" : " 17 GB", | 20260507 23:24:22.255 | INFO | "huge" : "", | 20260507 23:24:22.255 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:22.255 | INFO | "malloc" : " 14 GB", | 20260507 23:24:22.255 | INFO | "pinned" : " 29 GB" | 20260507 23:24:22.255 | INFO | } | 20260507 23:24:22.255 | INFO | Perf report for the last 5.14324 seconds: | 20260507 23:24:22.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:22.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:22.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:22.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:22.255 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260507 23:24:22.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:22.255 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 23:24:22.255 | INFO | PRE_HQ : 65.8%, 1.7, 57.2ms, 0 | 20260507 23:24:22.255 | INFO | BAZWRITER : 68.8%, 3.7, 130.4ms, 0 | 20260507 23:24:22.255 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 23:24:22.255 | INFO | Uploaded 36GiB to GPU in 2.52876s (14.2362GiB/s) | 20260507 23:24:22.255 | INFO | Downloaded 38GiB to GPU in 2.69681s (14.0907GiB/s) | 20260507 23:24:27.393 | INFO | Analyzing chunk frames = [3420672,3421184) | 20260507 23:24:27.393 | INFO | MemInfo: { | 20260507 23:24:27.393 | INFO | "gpu" : " 17 GB", | 20260507 23:24:27.393 | INFO | "huge" : "", | 20260507 23:24:27.393 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:27.393 | INFO | "malloc" : " 14 GB", | 20260507 23:24:27.393 | INFO | "pinned" : " 29 GB" | 20260507 23:24:27.393 | INFO | } | 20260507 23:24:27.393 | INFO | Perf report for the last 5.13817 seconds: | 20260507 23:24:27.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:27.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:24:27.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:24:27.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:27.393 | INFO | Uploaded 36GiB to GPU in 2.59202s (13.8888GiB/s) | 20260507 23:24:27.393 | INFO | Downloaded 34GiB to GPU in 2.47748s (13.7236GiB/s) | 20260507 23:24:32.539 | INFO | Analyzing chunk frames = [3421184,3421696) | 20260507 23:24:32.539 | INFO | MemInfo: { | 20260507 23:24:32.539 | INFO | "gpu" : " 17 GB", | 20260507 23:24:32.539 | INFO | "huge" : "", | 20260507 23:24:32.539 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:32.539 | INFO | "malloc" : " 14 GB", | 20260507 23:24:32.539 | INFO | "pinned" : " 29 GB" | 20260507 23:24:32.539 | INFO | } | 20260507 23:24:32.539 | INFO | Perf report for the last 5.14594 seconds: | 20260507 23:24:32.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:32.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260507 23:24:32.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 23:24:32.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:32.539 | INFO | Uploaded 36GiB to GPU in 2.57601s (13.9751GiB/s) | 20260507 23:24:32.540 | INFO | Downloaded 34GiB to GPU in 2.46887s (13.7715GiB/s) | 20260507 23:24:37.674 | INFO | Analyzing chunk frames = [3421696,3422208) | 20260507 23:24:37.674 | INFO | MemInfo: { | 20260507 23:24:37.674 | INFO | "gpu" : " 17 GB", | 20260507 23:24:37.674 | INFO | "huge" : "", | 20260507 23:24:37.674 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:37.674 | INFO | "malloc" : " 14 GB", | 20260507 23:24:37.674 | INFO | "pinned" : " 29 GB" | 20260507 23:24:37.675 | INFO | } | 20260507 23:24:37.675 | INFO | Perf report for the last 5.13561 seconds: | 20260507 23:24:37.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:37.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:24:37.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:24:37.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:37.675 | INFO | Uploaded 36GiB to GPU in 2.49135s (14.45GiB/s) | 20260507 23:24:37.675 | INFO | Downloaded 34GiB to GPU in 2.46718s (13.7809GiB/s) | 20260507 23:24:42.810 | INFO | Analyzing chunk frames = [3422208,3422720) | 20260507 23:24:42.810 | INFO | MemInfo: { | 20260507 23:24:42.810 | INFO | "gpu" : " 17 GB", | 20260507 23:24:42.810 | INFO | "huge" : "", | 20260507 23:24:42.810 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:42.810 | INFO | "malloc" : " 14 GB", | 20260507 23:24:42.810 | INFO | "pinned" : " 29 GB" | 20260507 23:24:42.810 | INFO | } | 20260507 23:24:42.810 | INFO | Perf report for the last 5.13571 seconds: | 20260507 23:24:42.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:42.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.810 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:24:42.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.810 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:24:42.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:42.811 | INFO | Uploaded 36GiB to GPU in 2.31006s (15.584GiB/s) | 20260507 23:24:42.811 | INFO | Downloaded 34GiB to GPU in 2.46442s (13.7964GiB/s) | 20260507 23:24:47.952 | INFO | Analyzing chunk frames = [3422720,3423232) | 20260507 23:24:47.952 | INFO | MemInfo: { | 20260507 23:24:47.952 | INFO | "gpu" : " 17 GB", | 20260507 23:24:47.952 | INFO | "huge" : "", | 20260507 23:24:47.952 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:47.952 | INFO | "malloc" : " 14 GB", | 20260507 23:24:47.952 | INFO | "pinned" : " 29 GB" | 20260507 23:24:47.952 | INFO | } | 20260507 23:24:47.953 | INFO | Perf report for the last 5.14222 seconds: | 20260507 23:24:47.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:47.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 23:24:47.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:24:47.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:47.953 | INFO | Uploaded 36GiB to GPU in 2.52064s (14.2821GiB/s) | 20260507 23:24:47.953 | INFO | Downloaded 34GiB to GPU in 2.43084s (13.9869GiB/s) | 20260507 23:24:53.091 | INFO | Analyzing chunk frames = [3423232,3423744) | 20260507 23:24:53.091 | INFO | MemInfo: { | 20260507 23:24:53.091 | INFO | "gpu" : " 17 GB", | 20260507 23:24:53.091 | INFO | "huge" : "", | 20260507 23:24:53.091 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:53.091 | INFO | "malloc" : " 14 GB", | 20260507 23:24:53.091 | INFO | "pinned" : " 29 GB" | 20260507 23:24:53.091 | INFO | } | 20260507 23:24:53.091 | INFO | Perf report for the last 5.13858 seconds: | 20260507 23:24:53.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:53.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.091 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:24:53.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.091 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 23:24:53.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:53.092 | INFO | Uploaded 36GiB to GPU in 2.51541s (14.3118GiB/s) | 20260507 23:24:53.092 | INFO | Downloaded 34GiB to GPU in 2.44692s (13.895GiB/s) | 20260507 23:24:58.227 | INFO | Analyzing chunk frames = [3423744,3424256) | 20260507 23:24:58.227 | INFO | MemInfo: { | 20260507 23:24:58.227 | INFO | "gpu" : " 17 GB", | 20260507 23:24:58.227 | INFO | "huge" : "", | 20260507 23:24:58.227 | INFO | "huge_shared" : " 29 GB", | 20260507 23:24:58.227 | INFO | "malloc" : " 14 GB", | 20260507 23:24:58.227 | INFO | "pinned" : " 29 GB" | 20260507 23:24:58.227 | INFO | } | 20260507 23:24:58.227 | INFO | Perf report for the last 5.13581 seconds: | 20260507 23:24:58.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:24:58.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:24:58.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:24:58.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:24:58.227 | INFO | Uploaded 36GiB to GPU in 2.48552s (14.4839GiB/s) | 20260507 23:24:58.227 | INFO | Downloaded 34GiB to GPU in 2.43614s (13.9565GiB/s) | 20260507 23:24:58.227 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:24:58.227 | INFO | Total mem usage: 16.7 GiB | 20260507 23:24:58.227 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:24:58.227 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260507 23:24:58.227 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:24:58.227 | INFO | High Water Breakdown: | 20260507 23:24:58.227 | INFO | 60% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:24:58.227 | INFO | 39.9% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:24:58.227 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:24:58.227 | INFO | ------------------------------------------ | 20260507 23:24:58.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:24:58.227 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:24:58.227 | INFO | Total mem usage: 26.5 GiB | 20260507 23:24:58.227 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:24:58.227 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:24:58.227 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:24:58.227 | INFO | High Water Breakdown: | 20260507 23:24:58.227 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:24:58.227 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:24:58.227 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:24:58.227 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:24:58.227 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:24:58.227 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:24:58.227 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:24:58.227 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:24:58.227 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:24:58.227 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:24:58.227 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:24:58.227 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:24:58.227 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:24:58.227 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:24:58.227 | INFO | ------------------------------------------ | 20260507 23:24:58.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:24:58.227 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:24:58.227 | INFO | Total mem usage: 27.8 GiB | 20260507 23:24:58.227 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:24:58.227 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:24:58.227 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:24:58.227 | INFO | High Water Breakdown: | 20260507 23:24:58.227 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:24:58.227 | INFO | ------------------------------------------ | 20260507 23:24:58.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:24:58.227 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:24:58.227 | INFO | Total mem usage: 15.7 GiB | 20260507 23:24:58.227 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:24:58.227 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:24:58.227 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:24:58.227 | INFO | High Water Breakdown: | 20260507 23:24:58.227 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:24:58.227 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:24:58.227 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:24:58.227 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:24:58.227 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:24:58.227 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:24:58.227 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:24:58.227 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:24:58.227 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:24:58.227 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:24:58.227 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:24:58.227 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:24:58.227 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:24:58.228 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:24:58.228 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:24:58.228 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:24:58.228 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:24:58.228 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:24:58.228 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:24:58.228 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:24:58.228 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:24:58.228 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:24:58.228 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:24:58.228 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:24:58.228 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:24:58.228 | INFO | ------------------------------------------ | 20260507 23:24:58.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:24:58.228 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:24:58.228 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:24:58.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:24:58.228 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.65e+01, 1.41e+01 / 1.66e+00 | 20260507 23:24:58.228 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.15e-02 / 4.73e-03 | 20260507 23:24:58.228 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.45e+00, 5.97e-01 / 2.21e-01 | 20260507 23:24:58.228 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.50e+00, 2.08e-01 / 9.13e-02 | 20260507 23:24:58.228 | INFO | PerfReport: 15.99%, DME -- 4.20e+01 / 4.60e+01, 4.11e+00 / 1.67e+00 | 20260507 23:24:58.228 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.68e+00 / 1.70e+00 | 20260507 23:24:58.228 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.74e+00, 4.34e-01 / 2.02e-01 | 20260507 23:24:58.228 | INFO | PerfReport: 3.43%, Metrics -- 1.13e+00 / 1.36e+00, 2.85e-01 / 1.52e-01 | 20260507 23:24:58.228 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.78e-01, 5.72e-02 / 2.78e-02 | 20260507 23:24:58.228 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:24:58.228 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:24:58.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:24:58.228 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:24:58.228 | INFO | PerfReport: 24.35%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 23:24:58.229 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 23:24:58.229 | INFO | PerfReport: 30.78%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:24:58.229 | INFO | PerfReport: 2.08%, LockUpload -- 2.23e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 23:24:58.229 | INFO | PerfReport: 2.67%, LockDownload -- 2.86e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260507 23:24:58.229 | INFO | PerfReport: 15.80%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:24:58.229 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.04e-02 / 0.00e+00 | 20260507 23:24:58.324 | INFO | Starting a round of Baz IO | 20260507 23:25:01.792 | INFO | Populating RTMetrics report | 20260507 23:25:01.793 | INFO | RTMetrics report populated | 20260507 23:25:01.793 | INFO | Opening tmp RTMetrics json file | 20260507 23:25:01.892 | INFO | Serializing RTMetrics to json | 20260507 23:25:01.903 | INFO | Serialization done, swapping tmp file now... | 20260507 23:25:01.905 | INFO | RTMetrics done | 20260507 23:25:01.937 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.612 seconds | 20260507 23:25:01.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.63% / 0.01% / 57.37% | 20260507 23:25:01.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19179% / 1.19209e-05% / 5.96046e-05% | 20260507 23:25:01.937 | INFO | Baz IO overall: | 20260507 23:25:01.937 | INFO | BytesWritten : 1559032770860 | 20260507 23:25:01.937 | INFO | NumEvents : 754997801160 | 20260507 23:25:03.361 | INFO | Analyzing chunk frames = [3424256,3424768) | 20260507 23:25:03.361 | INFO | MemInfo: { | 20260507 23:25:03.361 | INFO | "gpu" : " 17 GB", | 20260507 23:25:03.361 | INFO | "huge" : "", | 20260507 23:25:03.361 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:03.361 | INFO | "malloc" : " 14 GB", | 20260507 23:25:03.361 | INFO | "pinned" : " 29 GB" | 20260507 23:25:03.361 | INFO | } | 20260507 23:25:03.361 | INFO | Perf report for the last 5.13435 seconds: | 20260507 23:25:03.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:03.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:03.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:03.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:03.362 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260507 23:25:03.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:03.362 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 23:25:03.362 | INFO | PRE_HQ : 65.7%, 1.7, 57.0ms, 0 | 20260507 23:25:03.362 | INFO | BAZWRITER : 69.9%, 3.6, 130.1ms, 0 | 20260507 23:25:03.362 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260507 23:25:03.362 | INFO | Uploaded 36GiB to GPU in 2.52351s (14.2658GiB/s) | 20260507 23:25:03.362 | INFO | Downloaded 38GiB to GPU in 2.68053s (14.1763GiB/s) | 20260507 23:25:08.503 | INFO | Analyzing chunk frames = [3424768,3425280) | 20260507 23:25:08.503 | INFO | MemInfo: { | 20260507 23:25:08.503 | INFO | "gpu" : " 17 GB", | 20260507 23:25:08.503 | INFO | "huge" : "", | 20260507 23:25:08.503 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:08.504 | INFO | "malloc" : " 14 GB", | 20260507 23:25:08.504 | INFO | "pinned" : " 29 GB" | 20260507 23:25:08.504 | INFO | } | 20260507 23:25:08.504 | INFO | Perf report for the last 5.14238 seconds: | 20260507 23:25:08.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:08.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 23:25:08.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:25:08.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:08.504 | INFO | Uploaded 36GiB to GPU in 2.53126s (14.2221GiB/s) | 20260507 23:25:08.504 | INFO | Downloaded 34GiB to GPU in 2.4862s (13.6755GiB/s) | 20260507 23:25:13.644 | INFO | Analyzing chunk frames = [3425280,3425792) | 20260507 23:25:13.644 | INFO | MemInfo: { | 20260507 23:25:13.644 | INFO | "gpu" : " 17 GB", | 20260507 23:25:13.644 | INFO | "huge" : "", | 20260507 23:25:13.644 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:13.644 | INFO | "malloc" : " 14 GB", | 20260507 23:25:13.644 | INFO | "pinned" : " 29 GB" | 20260507 23:25:13.644 | INFO | } | 20260507 23:25:13.644 | INFO | Perf report for the last 5.1403 seconds: | 20260507 23:25:13.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:13.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.644 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:25:13.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.644 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:25:13.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:13.645 | INFO | Uploaded 36GiB to GPU in 2.5228s (14.2699GiB/s) | 20260507 23:25:13.645 | INFO | Downloaded 34GiB to GPU in 2.46467s (13.795GiB/s) | 20260507 23:25:18.796 | INFO | Analyzing chunk frames = [3425792,3426304) | 20260507 23:25:18.796 | INFO | MemInfo: { | 20260507 23:25:18.796 | INFO | "gpu" : " 17 GB", | 20260507 23:25:18.796 | INFO | "huge" : "", | 20260507 23:25:18.796 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:18.796 | INFO | "malloc" : " 14 GB", | 20260507 23:25:18.796 | INFO | "pinned" : " 29 GB" | 20260507 23:25:18.796 | INFO | } | 20260507 23:25:18.796 | INFO | Perf report for the last 5.15243 seconds: | 20260507 23:25:18.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:18.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.796 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 23:25:18.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.797 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:25:18.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:18.797 | INFO | Uploaded 36GiB to GPU in 2.4704s (14.5726GiB/s) | 20260507 23:25:18.797 | INFO | Downloaded 34GiB to GPU in 2.39275s (14.2096GiB/s) | 20260507 23:25:23.926 | INFO | Analyzing chunk frames = [3426304,3426816) | 20260507 23:25:23.926 | INFO | MemInfo: { | 20260507 23:25:23.926 | INFO | "gpu" : " 17 GB", | 20260507 23:25:23.926 | INFO | "huge" : "", | 20260507 23:25:23.926 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:23.926 | INFO | "malloc" : " 14 GB", | 20260507 23:25:23.926 | INFO | "pinned" : " 29 GB" | 20260507 23:25:23.926 | INFO | } | 20260507 23:25:23.927 | INFO | Perf report for the last 5.13009 seconds: | 20260507 23:25:23.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:23.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:25:23.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:25:23.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:23.927 | INFO | Uploaded 36GiB to GPU in 2.32719s (15.4693GiB/s) | 20260507 23:25:23.927 | INFO | Downloaded 34GiB to GPU in 2.47973s (13.7112GiB/s) | 20260507 23:25:29.067 | INFO | Analyzing chunk frames = [3426816,3427328) | 20260507 23:25:29.067 | INFO | MemInfo: { | 20260507 23:25:29.067 | INFO | "gpu" : " 17 GB", | 20260507 23:25:29.067 | INFO | "huge" : "", | 20260507 23:25:29.067 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:29.067 | INFO | "malloc" : " 14 GB", | 20260507 23:25:29.067 | INFO | "pinned" : " 29 GB" | 20260507 23:25:29.067 | INFO | } | 20260507 23:25:29.067 | INFO | Perf report for the last 5.14074 seconds: | 20260507 23:25:29.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:29.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.067 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:25:29.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.068 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260507 23:25:29.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:29.068 | INFO | Uploaded 36GiB to GPU in 2.43958s (14.7566GiB/s) | 20260507 23:25:29.068 | INFO | Downloaded 34GiB to GPU in 2.38667s (14.2458GiB/s) | 20260507 23:25:34.207 | INFO | Analyzing chunk frames = [3427328,3427840) | 20260507 23:25:34.207 | INFO | MemInfo: { | 20260507 23:25:34.207 | INFO | "gpu" : " 17 GB", | 20260507 23:25:34.208 | INFO | "huge" : "", | 20260507 23:25:34.208 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:34.208 | INFO | "malloc" : " 14 GB", | 20260507 23:25:34.208 | INFO | "pinned" : " 29 GB" | 20260507 23:25:34.208 | INFO | } | 20260507 23:25:34.208 | INFO | Perf report for the last 5.14043 seconds: | 20260507 23:25:34.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:34.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.208 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 23:25:34.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.208 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 23:25:34.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:34.209 | INFO | Uploaded 36GiB to GPU in 2.47482s (14.5465GiB/s) | 20260507 23:25:34.209 | INFO | Downloaded 34GiB to GPU in 2.42059s (14.0462GiB/s) | 20260507 23:25:39.341 | INFO | Analyzing chunk frames = [3427840,3428352) | 20260507 23:25:39.341 | INFO | MemInfo: { | 20260507 23:25:39.341 | INFO | "gpu" : " 17 GB", | 20260507 23:25:39.341 | INFO | "huge" : "", | 20260507 23:25:39.341 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:39.341 | INFO | "malloc" : " 14 GB", | 20260507 23:25:39.341 | INFO | "pinned" : " 29 GB" | 20260507 23:25:39.341 | INFO | } | 20260507 23:25:39.341 | INFO | Perf report for the last 5.13364 seconds: | 20260507 23:25:39.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:39.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.342 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:25:39.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.342 | INFO | PRE_HQ : 35.4%, 1.0, 18.3ms, 0 | 20260507 23:25:39.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:39.342 | INFO | Uploaded 36GiB to GPU in 2.56486s (14.0359GiB/s) | 20260507 23:25:39.342 | INFO | Downloaded 34GiB to GPU in 2.45006s (13.8772GiB/s) | 20260507 23:25:39.453 | INFO | Starting a round of Baz IO | 20260507 23:25:42.930 | INFO | Populating RTMetrics report | 20260507 23:25:42.931 | INFO | RTMetrics report populated | 20260507 23:25:42.931 | INFO | Opening tmp RTMetrics json file | 20260507 23:25:42.944 | INFO | Serializing RTMetrics to json | 20260507 23:25:42.959 | INFO | Serialization done, swapping tmp file now... | 20260507 23:25:42.960 | INFO | RTMetrics done | 20260507 23:25:43.035 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.581 seconds | 20260507 23:25:43.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.62% / 0.01% / 57.37% | 20260507 23:25:43.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18946% / 3.97364e-06% / 6.75519e-05% | 20260507 23:25:43.035 | INFO | Baz IO overall: | 20260507 23:25:43.035 | INFO | BytesWritten : 1560874672034 | 20260507 23:25:43.035 | INFO | NumEvents : 755884841765 | 20260507 23:25:44.485 | INFO | Analyzing chunk frames = [3428352,3428864) | 20260507 23:25:44.485 | INFO | MemInfo: { | 20260507 23:25:44.485 | INFO | "gpu" : " 17 GB", | 20260507 23:25:44.485 | INFO | "huge" : "", | 20260507 23:25:44.485 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:44.485 | INFO | "malloc" : " 14 GB", | 20260507 23:25:44.485 | INFO | "pinned" : " 29 GB" | 20260507 23:25:44.485 | INFO | } | 20260507 23:25:44.485 | INFO | Perf report for the last 5.14348 seconds: | 20260507 23:25:44.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:44.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:44.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:44.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:44.485 | INFO | ANALYSIS : 69.1%, 2.9, 105.0ms, 0 | 20260507 23:25:44.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:44.485 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 23:25:44.485 | INFO | PRE_HQ : 67.8%, 1.8, 63.6ms, 0 | 20260507 23:25:44.485 | INFO | BAZWRITER : 69.2%, 4.0, 142.8ms, 0 | 20260507 23:25:44.485 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 23:25:44.486 | INFO | Uploaded 36GiB to GPU in 2.60496s (13.8198GiB/s) | 20260507 23:25:44.486 | INFO | Downloaded 38GiB to GPU in 2.71558s (13.9933GiB/s) | 20260507 23:25:49.620 | INFO | Analyzing chunk frames = [3428864,3429376) | 20260507 23:25:49.620 | INFO | MemInfo: { | 20260507 23:25:49.620 | INFO | "gpu" : " 17 GB", | 20260507 23:25:49.620 | INFO | "huge" : "", | 20260507 23:25:49.620 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:49.620 | INFO | "malloc" : " 14 GB", | 20260507 23:25:49.620 | INFO | "pinned" : " 29 GB" | 20260507 23:25:49.620 | INFO | } | 20260507 23:25:49.620 | INFO | Perf report for the last 5.13503 seconds: | 20260507 23:25:49.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:49.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260507 23:25:49.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:25:49.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:49.620 | INFO | Uploaded 36GiB to GPU in 2.58548s (13.9239GiB/s) | 20260507 23:25:49.620 | INFO | Downloaded 34GiB to GPU in 2.47573s (13.7333GiB/s) | 20260507 23:25:54.762 | INFO | Analyzing chunk frames = [3429376,3429888) | 20260507 23:25:54.762 | INFO | MemInfo: { | 20260507 23:25:54.762 | INFO | "gpu" : " 17 GB", | 20260507 23:25:54.762 | INFO | "huge" : "", | 20260507 23:25:54.762 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:54.762 | INFO | "malloc" : " 14 GB", | 20260507 23:25:54.762 | INFO | "pinned" : " 29 GB" | 20260507 23:25:54.763 | INFO | } | 20260507 23:25:54.763 | INFO | Perf report for the last 5.14258 seconds: | 20260507 23:25:54.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:54.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 23:25:54.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:25:54.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:54.763 | INFO | Uploaded 36GiB to GPU in 2.59203s (13.8887GiB/s) | 20260507 23:25:54.763 | INFO | Downloaded 34GiB to GPU in 2.46601s (13.7874GiB/s) | 20260507 23:25:59.901 | INFO | Analyzing chunk frames = [3429888,3430400) | 20260507 23:25:59.901 | INFO | MemInfo: { | 20260507 23:25:59.901 | INFO | "gpu" : " 17 GB", | 20260507 23:25:59.901 | INFO | "huge" : "", | 20260507 23:25:59.901 | INFO | "huge_shared" : " 29 GB", | 20260507 23:25:59.901 | INFO | "malloc" : " 14 GB", | 20260507 23:25:59.901 | INFO | "pinned" : " 29 GB" | 20260507 23:25:59.901 | INFO | } | 20260507 23:25:59.902 | INFO | Perf report for the last 5.13896 seconds: | 20260507 23:25:59.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:25:59.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 23:25:59.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:25:59.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:25:59.902 | INFO | Uploaded 36GiB to GPU in 2.56174s (14.053GiB/s) | 20260507 23:25:59.902 | INFO | Downloaded 34GiB to GPU in 2.49416s (13.6318GiB/s) | 20260507 23:26:05.042 | INFO | Analyzing chunk frames = [3430400,3430912) | 20260507 23:26:05.042 | INFO | MemInfo: { | 20260507 23:26:05.042 | INFO | "gpu" : " 17 GB", | 20260507 23:26:05.042 | INFO | "huge" : "", | 20260507 23:26:05.042 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:05.042 | INFO | "malloc" : " 14 GB", | 20260507 23:26:05.042 | INFO | "pinned" : " 29 GB" | 20260507 23:26:05.042 | INFO | } | 20260507 23:26:05.042 | INFO | Perf report for the last 5.1403 seconds: | 20260507 23:26:05.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:05.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 23:26:05.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:26:05.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:05.042 | INFO | Uploaded 36GiB to GPU in 2.41961s (14.8784GiB/s) | 20260507 23:26:05.042 | INFO | Downloaded 34GiB to GPU in 2.49929s (13.6039GiB/s) | 20260507 23:26:10.176 | INFO | Analyzing chunk frames = [3430912,3431424) | 20260507 23:26:10.176 | INFO | MemInfo: { | 20260507 23:26:10.176 | INFO | "gpu" : " 17 GB", | 20260507 23:26:10.176 | INFO | "huge" : "", | 20260507 23:26:10.176 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:10.176 | INFO | "malloc" : " 14 GB", | 20260507 23:26:10.176 | INFO | "pinned" : " 29 GB" | 20260507 23:26:10.176 | INFO | } | 20260507 23:26:10.176 | INFO | Perf report for the last 5.13386 seconds: | 20260507 23:26:10.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:10.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:26:10.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:26:10.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:10.176 | INFO | Uploaded 36GiB to GPU in 2.39644s (15.0223GiB/s) | 20260507 23:26:10.176 | INFO | Downloaded 34GiB to GPU in 2.39727s (14.1828GiB/s) | 20260507 23:26:15.313 | INFO | Analyzing chunk frames = [3431424,3431936) | 20260507 23:26:15.313 | INFO | MemInfo: { | 20260507 23:26:15.313 | INFO | "gpu" : " 17 GB", | 20260507 23:26:15.313 | INFO | "huge" : "", | 20260507 23:26:15.313 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:15.313 | INFO | "malloc" : " 14 GB", | 20260507 23:26:15.313 | INFO | "pinned" : " 29 GB" | 20260507 23:26:15.313 | INFO | } | 20260507 23:26:15.313 | INFO | Perf report for the last 5.13776 seconds: | 20260507 23:26:15.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:15.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:26:15.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 23:26:15.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:15.314 | INFO | Uploaded 36GiB to GPU in 2.5412s (14.1665GiB/s) | 20260507 23:26:15.314 | INFO | Downloaded 34GiB to GPU in 2.46184s (13.8108GiB/s) | 20260507 23:26:20.458 | INFO | Analyzing chunk frames = [3431936,3432448) | 20260507 23:26:20.458 | INFO | MemInfo: { | 20260507 23:26:20.458 | INFO | "gpu" : " 17 GB", | 20260507 23:26:20.458 | INFO | "huge" : "", | 20260507 23:26:20.458 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:20.458 | INFO | "malloc" : " 14 GB", | 20260507 23:26:20.458 | INFO | "pinned" : " 29 GB" | 20260507 23:26:20.458 | INFO | } | 20260507 23:26:20.458 | INFO | Perf report for the last 5.14465 seconds: | 20260507 23:26:20.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:20.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:26:20.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:26:20.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:20.458 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:26:20.458 | INFO | Total mem usage: 16.7 GiB | 20260507 23:26:20.458 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:26:20.458 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260507 23:26:20.458 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:26:20.458 | INFO | High Water Breakdown: | 20260507 23:26:20.459 | INFO | 60.2% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:26:20.459 | INFO | 39.7% : 5.96 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:26:20.459 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:26:20.459 | INFO | ------------------------------------------ | 20260507 23:26:20.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:26:20.459 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:26:20.459 | INFO | Total mem usage: 26.5 GiB | 20260507 23:26:20.459 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:26:20.459 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 23:26:20.459 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 23:26:20.459 | INFO | High Water Breakdown: | 20260507 23:26:20.459 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:26:20.459 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:26:20.459 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:26:20.459 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:26:20.459 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:26:20.459 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:26:20.459 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:26:20.459 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:26:20.459 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:26:20.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:26:20.459 | INFO | Uploaded 36GiB to GPU in 2.506s (14.3655GiB/s) | 20260507 23:26:20.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:26:20.459 | INFO | Downloaded 34GiB to GPU in 2.44986s (13.8784GiB/s) | 20260507 23:26:20.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:26:20.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:26:20.459 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:26:20.459 | INFO | ------------------------------------------ | 20260507 23:26:20.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:26:20.459 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:26:20.459 | INFO | Total mem usage: 27.8 GiB | 20260507 23:26:20.459 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:26:20.459 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:26:20.459 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:26:20.459 | INFO | High Water Breakdown: | 20260507 23:26:20.459 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:26:20.459 | INFO | ------------------------------------------ | 20260507 23:26:20.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:26:20.459 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:26:20.459 | INFO | Total mem usage: 15.7 GiB | 20260507 23:26:20.459 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:26:20.459 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:26:20.459 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:26:20.459 | INFO | High Water Breakdown: | 20260507 23:26:20.459 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:26:20.459 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:26:20.459 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:26:20.459 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:26:20.460 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:26:20.460 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:26:20.460 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:26:20.460 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:26:20.460 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:26:20.460 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:26:20.460 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:26:20.460 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:26:20.460 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:26:20.460 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:26:20.460 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:26:20.460 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:26:20.460 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:26:20.460 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:26:20.460 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:26:20.460 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:26:20.460 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:26:20.460 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:26:20.460 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:26:20.460 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:26:20.460 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:26:20.460 | INFO | ------------------------------------------ | 20260507 23:26:20.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:26:20.460 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:26:20.460 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:26:20.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:26:20.460 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.63e+01, 1.41e+01 / 1.67e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.04e-01, 1.23e-02 / 6.17e-03 | 20260507 23:26:20.460 | INFO | PerfReport: 12.18%, Baseline -- 4.01e+00 / 4.49e+00, 5.98e-01 / 2.06e-01 | 20260507 23:26:20.460 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.00e-01 | 20260507 23:26:20.460 | INFO | PerfReport: 15.87%, DME -- 4.18e+01 / 4.54e+01, 4.01e+00 / 1.53e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.55e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.32e-01 / 2.24e-01 | 20260507 23:26:20.460 | INFO | PerfReport: 3.42%, Metrics -- 1.13e+00 / 1.33e+00, 2.85e-01 / 1.48e-01 | 20260507 23:26:20.460 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.79e-01, 5.64e-02 / 2.72e-02 | 20260507 23:26:20.460 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:26:20.460 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:26:20.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:26:20.460 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 24.47%, Upload -- 2.61e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 30.91%, Compute -- 3.30e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 2.11%, LockUpload -- 2.25e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 2.40%, LockDownload -- 2.56e+00 / 0.00e+00, 3.84e+00 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 15.84%, LockCompute -- 1.69e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 23:26:20.460 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260507 23:26:20.556 | INFO | Starting a round of Baz IO | 20260507 23:26:24.019 | INFO | Populating RTMetrics report | 20260507 23:26:24.020 | INFO | RTMetrics report populated | 20260507 23:26:24.020 | INFO | Opening tmp RTMetrics json file | 20260507 23:26:24.041 | INFO | Serializing RTMetrics to json | 20260507 23:26:24.042 | INFO | Serialization done, swapping tmp file now... | 20260507 23:26:24.043 | INFO | RTMetrics done | 20260507 23:26:24.106 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.55 seconds | 20260507 23:26:24.106 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.62% / 0.01% / 57.38% | 20260507 23:26:24.106 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18887% / 1.58946e-05% / 6.35783e-05% | 20260507 23:26:24.107 | INFO | Baz IO overall: | 20260507 23:26:24.107 | INFO | BytesWritten : 1562715496241 | 20260507 23:26:24.107 | INFO | NumEvents : 756771423439 | 20260507 23:26:25.599 | INFO | Analyzing chunk frames = [3432448,3432960) | 20260507 23:26:25.599 | INFO | MemInfo: { | 20260507 23:26:25.599 | INFO | "gpu" : " 17 GB", | 20260507 23:26:25.599 | INFO | "huge" : "", | 20260507 23:26:25.599 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:25.599 | INFO | "malloc" : " 14 GB", | 20260507 23:26:25.599 | INFO | "pinned" : " 29 GB" | 20260507 23:26:25.599 | INFO | } | 20260507 23:26:25.599 | INFO | Perf report for the last 5.14127 seconds: | 20260507 23:26:25.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:25.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:25.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:25.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:25.600 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260507 23:26:25.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:25.600 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:26:25.600 | INFO | PRE_HQ : 65.4%, 1.7, 56.7ms, 0 | 20260507 23:26:25.600 | INFO | BAZWRITER : 68.6%, 3.9, 138.4ms, 0 | 20260507 23:26:25.600 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 23:26:25.600 | INFO | Uploaded 36GiB to GPU in 2.48226s (14.5029GiB/s) | 20260507 23:26:25.600 | INFO | Downloaded 38GiB to GPU in 2.68662s (14.1442GiB/s) | 20260507 23:26:30.740 | INFO | Analyzing chunk frames = [3432960,3433472) | 20260507 23:26:30.740 | INFO | MemInfo: { | 20260507 23:26:30.740 | INFO | "gpu" : " 17 GB", | 20260507 23:26:30.740 | INFO | "huge" : "", | 20260507 23:26:30.740 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:30.740 | INFO | "malloc" : " 14 GB", | 20260507 23:26:30.740 | INFO | "pinned" : " 29 GB" | 20260507 23:26:30.740 | INFO | } | 20260507 23:26:30.740 | INFO | Perf report for the last 5.14051 seconds: | 20260507 23:26:30.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:30.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:26:30.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:26:30.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:30.740 | INFO | Uploaded 36GiB to GPU in 2.55557s (14.0869GiB/s) | 20260507 23:26:30.740 | INFO | Downloaded 34GiB to GPU in 2.49447s (13.6301GiB/s) | 20260507 23:26:35.876 | INFO | Analyzing chunk frames = [3433472,3433984) | 20260507 23:26:35.876 | INFO | MemInfo: { | 20260507 23:26:35.876 | INFO | "gpu" : " 17 GB", | 20260507 23:26:35.876 | INFO | "huge" : "", | 20260507 23:26:35.876 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:35.876 | INFO | "malloc" : " 14 GB", | 20260507 23:26:35.876 | INFO | "pinned" : " 29 GB" | 20260507 23:26:35.877 | INFO | } | 20260507 23:26:35.877 | INFO | Perf report for the last 5.13657 seconds: | 20260507 23:26:35.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:35.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 23:26:35.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:26:35.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:35.877 | INFO | Uploaded 36GiB to GPU in 2.50022s (14.3987GiB/s) | 20260507 23:26:35.877 | INFO | Downloaded 34GiB to GPU in 2.47702s (13.7262GiB/s) | 20260507 23:26:41.020 | INFO | Analyzing chunk frames = [3433984,3434496) | 20260507 23:26:41.020 | INFO | MemInfo: { | 20260507 23:26:41.020 | INFO | "gpu" : " 17 GB", | 20260507 23:26:41.021 | INFO | "huge" : "", | 20260507 23:26:41.021 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:41.021 | INFO | "malloc" : " 14 GB", | 20260507 23:26:41.021 | INFO | "pinned" : " 29 GB" | 20260507 23:26:41.021 | INFO | } | 20260507 23:26:41.021 | INFO | Perf report for the last 5.14407 seconds: | 20260507 23:26:41.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:41.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260507 23:26:41.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:26:41.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:41.021 | INFO | Uploaded 36GiB to GPU in 2.50275s (14.3842GiB/s) | 20260507 23:26:41.021 | INFO | Downloaded 34GiB to GPU in 2.49539s (13.6251GiB/s) | 20260507 23:26:46.158 | INFO | Analyzing chunk frames = [3434496,3435008) | 20260507 23:26:46.158 | INFO | MemInfo: { | 20260507 23:26:46.158 | INFO | "gpu" : " 17 GB", | 20260507 23:26:46.158 | INFO | "huge" : "", | 20260507 23:26:46.158 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:46.158 | INFO | "malloc" : " 14 GB", | 20260507 23:26:46.158 | INFO | "pinned" : " 29 GB" | 20260507 23:26:46.158 | INFO | } | 20260507 23:26:46.158 | INFO | Perf report for the last 5.13741 seconds: | 20260507 23:26:46.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:46.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:26:46.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 23:26:46.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:46.158 | INFO | Uploaded 36GiB to GPU in 2.33831s (15.3958GiB/s) | 20260507 23:26:46.158 | INFO | Downloaded 34GiB to GPU in 2.47477s (13.7387GiB/s) | 20260507 23:26:51.296 | INFO | Analyzing chunk frames = [3435008,3435520) | 20260507 23:26:51.296 | INFO | MemInfo: { | 20260507 23:26:51.296 | INFO | "gpu" : " 17 GB", | 20260507 23:26:51.296 | INFO | "huge" : "", | 20260507 23:26:51.296 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:51.296 | INFO | "malloc" : " 14 GB", | 20260507 23:26:51.296 | INFO | "pinned" : " 29 GB" | 20260507 23:26:51.296 | INFO | } | 20260507 23:26:51.296 | INFO | Perf report for the last 5.13782 seconds: | 20260507 23:26:51.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:51.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.296 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 23:26:51.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.296 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260507 23:26:51.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:51.297 | INFO | Uploaded 36GiB to GPU in 2.5159s (14.309GiB/s) | 20260507 23:26:51.297 | INFO | Downloaded 34GiB to GPU in 2.4307s (13.9878GiB/s) | 20260507 23:26:56.432 | INFO | Analyzing chunk frames = [3435520,3436032) | 20260507 23:26:56.432 | INFO | MemInfo: { | 20260507 23:26:56.432 | INFO | "gpu" : " 17 GB", | 20260507 23:26:56.432 | INFO | "huge" : "", | 20260507 23:26:56.432 | INFO | "huge_shared" : " 29 GB", | 20260507 23:26:56.432 | INFO | "malloc" : " 14 GB", | 20260507 23:26:56.432 | INFO | "pinned" : " 29 GB" | 20260507 23:26:56.432 | INFO | } | 20260507 23:26:56.432 | INFO | Perf report for the last 5.13615 seconds: | 20260507 23:26:56.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:26:56.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 23:26:56.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 23:26:56.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:26:56.432 | INFO | Uploaded 36GiB to GPU in 2.55628s (14.083GiB/s) | 20260507 23:26:56.433 | INFO | Downloaded 34GiB to GPU in 2.45227s (13.8647GiB/s) | 20260507 23:27:01.571 | INFO | Analyzing chunk frames = [3436032,3436544) | 20260507 23:27:01.571 | INFO | MemInfo: { | 20260507 23:27:01.571 | INFO | "gpu" : " 17 GB", | 20260507 23:27:01.571 | INFO | "huge" : "", | 20260507 23:27:01.571 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:01.571 | INFO | "malloc" : " 14 GB", | 20260507 23:27:01.571 | INFO | "pinned" : " 29 GB" | 20260507 23:27:01.571 | INFO | } | 20260507 23:27:01.571 | INFO | Perf report for the last 5.13929 seconds: | 20260507 23:27:01.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:01.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.572 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 23:27:01.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.572 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260507 23:27:01.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:01.572 | INFO | Uploaded 36GiB to GPU in 2.51943s (14.2889GiB/s) | 20260507 23:27:01.572 | INFO | Downloaded 34GiB to GPU in 2.44677s (13.8959GiB/s) | 20260507 23:27:01.685 | INFO | Starting a round of Baz IO | 20260507 23:27:05.159 | INFO | Populating RTMetrics report | 20260507 23:27:05.160 | INFO | RTMetrics report populated | 20260507 23:27:05.160 | INFO | Opening tmp RTMetrics json file | 20260507 23:27:05.184 | INFO | Serializing RTMetrics to json | 20260507 23:27:05.256 | INFO | Serialization done, swapping tmp file now... | 20260507 23:27:05.258 | INFO | RTMetrics done | 20260507 23:27:05.277 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.591 seconds | 20260507 23:27:05.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.61% / 0.01% / 57.38% | 20260507 23:27:05.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18836% / 3.97364e-06% / 5.5631e-05% | 20260507 23:27:05.277 | INFO | Baz IO overall: | 20260507 23:27:05.277 | INFO | BytesWritten : 1564556200430 | 20260507 23:27:05.277 | INFO | NumEvents : 757658286897 | 20260507 23:27:06.714 | INFO | Analyzing chunk frames = [3436544,3437056) | 20260507 23:27:06.714 | INFO | MemInfo: { | 20260507 23:27:06.714 | INFO | "gpu" : " 17 GB", | 20260507 23:27:06.714 | INFO | "huge" : "", | 20260507 23:27:06.714 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:06.714 | INFO | "malloc" : " 14 GB", | 20260507 23:27:06.714 | INFO | "pinned" : " 29 GB" | 20260507 23:27:06.714 | INFO | } | 20260507 23:27:06.714 | INFO | Perf report for the last 5.14311 seconds: | 20260507 23:27:06.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:06.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:06.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:06.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:06.715 | INFO | ANALYSIS : 69.1%, 2.9, 104.7ms, 0 | 20260507 23:27:06.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:06.715 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260507 23:27:06.715 | INFO | PRE_HQ : 68.1%, 1.8, 63.7ms, 0 | 20260507 23:27:06.715 | INFO | BAZWRITER : 69.5%, 4.1, 145.2ms, 0 | 20260507 23:27:06.715 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 23:27:06.715 | INFO | Uploaded 36GiB to GPU in 2.59195s (13.8892GiB/s) | 20260507 23:27:06.715 | INFO | Downloaded 38GiB to GPU in 2.70438s (14.0513GiB/s) | 20260507 23:27:11.855 | INFO | Analyzing chunk frames = [3437056,3437568) | 20260507 23:27:11.855 | INFO | MemInfo: { | 20260507 23:27:11.855 | INFO | "gpu" : " 17 GB", | 20260507 23:27:11.855 | INFO | "huge" : "", | 20260507 23:27:11.855 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:11.855 | INFO | "malloc" : " 14 GB", | 20260507 23:27:11.855 | INFO | "pinned" : " 29 GB" | 20260507 23:27:11.855 | INFO | } | 20260507 23:27:11.855 | INFO | Perf report for the last 5.14038 seconds: | 20260507 23:27:11.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:11.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 23:27:11.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:27:11.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:11.855 | INFO | Uploaded 36GiB to GPU in 2.58611s (13.9205GiB/s) | 20260507 23:27:11.855 | INFO | Downloaded 34GiB to GPU in 2.47898s (13.7153GiB/s) | 20260507 23:27:16.990 | INFO | Analyzing chunk frames = [3437568,3438080) | 20260507 23:27:16.990 | INFO | MemInfo: { | 20260507 23:27:16.990 | INFO | "gpu" : " 17 GB", | 20260507 23:27:16.990 | INFO | "huge" : "", | 20260507 23:27:16.990 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:16.990 | INFO | "malloc" : " 14 GB", | 20260507 23:27:16.990 | INFO | "pinned" : " 29 GB" | 20260507 23:27:16.990 | INFO | } | 20260507 23:27:16.990 | INFO | Perf report for the last 5.13543 seconds: | 20260507 23:27:16.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:16.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.990 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 23:27:16.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.990 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:27:16.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:16.991 | INFO | Uploaded 36GiB to GPU in 2.52268s (14.2705GiB/s) | 20260507 23:27:16.991 | INFO | Downloaded 34GiB to GPU in 2.48044s (13.7073GiB/s) | 20260507 23:27:22.125 | INFO | Analyzing chunk frames = [3438080,3438592) | 20260507 23:27:22.125 | INFO | MemInfo: { | 20260507 23:27:22.125 | INFO | "gpu" : " 17 GB", | 20260507 23:27:22.125 | INFO | "huge" : "", | 20260507 23:27:22.125 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:22.125 | INFO | "malloc" : " 14 GB", | 20260507 23:27:22.125 | INFO | "pinned" : " 29 GB" | 20260507 23:27:22.125 | INFO | } | 20260507 23:27:22.125 | INFO | Perf report for the last 5.13491 seconds: | 20260507 23:27:22.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:22.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.125 | INFO | ANALYSIS : 67.7%, 3.0, 104.6ms, 0 | 20260507 23:27:22.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.125 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:27:22.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:22.126 | INFO | Uploaded 36GiB to GPU in 2.54147s (14.165GiB/s) | 20260507 23:27:22.126 | INFO | Downloaded 34GiB to GPU in 2.49391s (13.6332GiB/s) | 20260507 23:27:27.271 | INFO | Analyzing chunk frames = [3438592,3439104) | 20260507 23:27:27.271 | INFO | MemInfo: { | 20260507 23:27:27.271 | INFO | "gpu" : " 17 GB", | 20260507 23:27:27.271 | INFO | "huge" : "", | 20260507 23:27:27.271 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:27.271 | INFO | "malloc" : " 14 GB", | 20260507 23:27:27.271 | INFO | "pinned" : " 29 GB" | 20260507 23:27:27.271 | INFO | } | 20260507 23:27:27.271 | INFO | Perf report for the last 5.14617 seconds: | 20260507 23:27:27.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:27.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 23:27:27.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:27:27.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:27.272 | INFO | Uploaded 36GiB to GPU in 2.3785s (15.1356GiB/s) | 20260507 23:27:27.272 | INFO | Downloaded 34GiB to GPU in 2.49859s (13.6077GiB/s) | 20260507 23:27:32.408 | INFO | Analyzing chunk frames = [3439104,3439616) | 20260507 23:27:32.408 | INFO | MemInfo: { | 20260507 23:27:32.408 | INFO | "gpu" : " 17 GB", | 20260507 23:27:32.408 | INFO | "huge" : "", | 20260507 23:27:32.408 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:32.408 | INFO | "malloc" : " 14 GB", | 20260507 23:27:32.408 | INFO | "pinned" : " 29 GB" | 20260507 23:27:32.408 | INFO | } | 20260507 23:27:32.408 | INFO | Perf report for the last 5.13689 seconds: | 20260507 23:27:32.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:32.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.408 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 23:27:32.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.408 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:27:32.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:32.409 | INFO | Uploaded 36GiB to GPU in 2.50854s (14.351GiB/s) | 20260507 23:27:32.409 | INFO | Downloaded 34GiB to GPU in 2.43565s (13.9593GiB/s) | 20260507 23:27:37.548 | INFO | Analyzing chunk frames = [3439616,3440128) | 20260507 23:27:37.548 | INFO | MemInfo: { | 20260507 23:27:37.548 | INFO | "gpu" : " 17 GB", | 20260507 23:27:37.548 | INFO | "huge" : "", | 20260507 23:27:37.548 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:37.548 | INFO | "malloc" : " 14 GB", | 20260507 23:27:37.548 | INFO | "pinned" : " 29 GB" | 20260507 23:27:37.548 | INFO | } | 20260507 23:27:37.548 | INFO | Perf report for the last 5.14004 seconds: | 20260507 23:27:37.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:37.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.548 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:27:37.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.549 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 23:27:37.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:37.549 | INFO | Uploaded 36GiB to GPU in 2.54339s (14.1543GiB/s) | 20260507 23:27:37.549 | INFO | Downloaded 34GiB to GPU in 2.46343s (13.8019GiB/s) | 20260507 23:27:42.689 | INFO | Analyzing chunk frames = [3440128,3440640) | 20260507 23:27:42.689 | INFO | MemInfo: { | 20260507 23:27:42.689 | INFO | "gpu" : " 17 GB", | 20260507 23:27:42.689 | INFO | "huge" : "", | 20260507 23:27:42.689 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:42.689 | INFO | "malloc" : " 14 GB", | 20260507 23:27:42.689 | INFO | "pinned" : " 29 GB" | 20260507 23:27:42.689 | INFO | } | 20260507 23:27:42.689 | INFO | Perf report for the last 5.14041 seconds: | 20260507 23:27:42.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:42.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:27:42.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:27:42.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:42.689 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:27:42.689 | INFO | Total mem usage: 16.7 GiB | 20260507 23:27:42.689 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:27:42.689 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.5%) | 20260507 23:27:42.689 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:27:42.689 | INFO | High Water Breakdown: | 20260507 23:27:42.689 | INFO | 59.8% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:27:42.689 | INFO | 40% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:27:42.689 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:27:42.689 | INFO | ------------------------------------------ | 20260507 23:27:42.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:27:42.689 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:27:42.689 | INFO | Total mem usage: 26.5 GiB | 20260507 23:27:42.689 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:27:42.689 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:27:42.689 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:27:42.689 | INFO | High Water Breakdown: | 20260507 23:27:42.689 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:27:42.689 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:27:42.689 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:27:42.689 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:27:42.689 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:27:42.689 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:27:42.689 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:27:42.689 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:27:42.689 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:27:42.689 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:27:42.689 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:27:42.689 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:27:42.689 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:27:42.689 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:27:42.689 | INFO | ------------------------------------------ | 20260507 23:27:42.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:27:42.689 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:27:42.689 | INFO | Total mem usage: 27.8 GiB | 20260507 23:27:42.689 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:27:42.689 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:27:42.689 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:27:42.689 | INFO | High Water Breakdown: | 20260507 23:27:42.689 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:27:42.689 | INFO | ------------------------------------------ | 20260507 23:27:42.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:27:42.689 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:27:42.689 | INFO | Total mem usage: 15.7 GiB | 20260507 23:27:42.689 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:27:42.689 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:27:42.689 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:27:42.689 | INFO | High Water Breakdown: | 20260507 23:27:42.689 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:27:42.689 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:27:42.689 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:27:42.689 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:27:42.689 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:27:42.689 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:27:42.689 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:27:42.689 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:27:42.689 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:27:42.689 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:27:42.689 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:27:42.689 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:27:42.689 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:27:42.689 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:27:42.689 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:27:42.689 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:27:42.689 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:27:42.689 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:27:42.689 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:27:42.689 | INFO | Uploaded 36GiB to GPU in 2.50862s (14.3505GiB/s) | 20260507 23:27:42.690 | INFO | Downloaded 34GiB to GPU in 2.44173s (13.9245GiB/s) | 20260507 23:27:42.689 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:27:42.690 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:27:42.690 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:27:42.690 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:27:42.690 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:27:42.690 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:27:42.690 | INFO | ------------------------------------------ | 20260507 23:27:42.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:27:42.690 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:27:42.690 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:27:42.690 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:27:42.690 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.66e+01, 1.42e+01 / 1.87e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.13e-02 / 6.56e-03 | 20260507 23:27:42.690 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.47e+00, 5.87e-01 / 1.89e-01 | 20260507 23:27:42.690 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.12e-01 | 20260507 23:27:42.690 | INFO | PerfReport: 16.19%, DME -- 4.24e+01 / 4.65e+01, 4.22e+00 / 2.10e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 55.97%, FrameLabeling -- 1.83e+01 / 1.94e+01, 1.53e+00 / 1.38e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.47e+00 / 2.77e+00, 4.37e-01 / 1.88e-01 | 20260507 23:27:42.690 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.35e+00, 2.80e-01 / 1.50e-01 | 20260507 23:27:42.690 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.49e-01 / 3.87e-01, 5.49e-02 / 3.57e-02 | 20260507 23:27:42.690 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:27:42.690 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:27:42.690 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:27:42.690 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 24.34%, Download -- 2.60e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 30.66%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 2.29%, LockUpload -- 2.45e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260507 23:27:42.690 | INFO | PerfReport: 2.74%, LockDownload -- 2.93e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 23:27:42.691 | INFO | PerfReport: 15.50%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 23:27:42.691 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260507 23:27:42.787 | INFO | Starting a round of Baz IO | 20260507 23:27:46.264 | INFO | Populating RTMetrics report | 20260507 23:27:46.264 | INFO | RTMetrics report populated | 20260507 23:27:46.264 | INFO | Opening tmp RTMetrics json file | 20260507 23:27:46.284 | INFO | Serializing RTMetrics to json | 20260507 23:27:46.285 | INFO | Serialization done, swapping tmp file now... | 20260507 23:27:46.288 | INFO | RTMetrics done | 20260507 23:27:46.351 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.564 seconds | 20260507 23:27:46.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.61% / 0.01% / 57.39% | 20260507 23:27:46.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18508% / 3.97364e-06% / 6.75519e-05% | 20260507 23:27:46.352 | INFO | Baz IO overall: | 20260507 23:27:46.352 | INFO | BytesWritten : 1566396300849 | 20260507 23:27:46.352 | INFO | NumEvents : 758544664087 | 20260507 23:27:47.827 | INFO | Analyzing chunk frames = [3440640,3441152) | 20260507 23:27:47.827 | INFO | MemInfo: { | 20260507 23:27:47.827 | INFO | "gpu" : " 17 GB", | 20260507 23:27:47.827 | INFO | "huge" : "", | 20260507 23:27:47.827 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:47.827 | INFO | "malloc" : " 14 GB", | 20260507 23:27:47.827 | INFO | "pinned" : " 29 GB" | 20260507 23:27:47.827 | INFO | } | 20260507 23:27:47.827 | INFO | Perf report for the last 5.13843 seconds: | 20260507 23:27:47.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:47.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:47.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:47.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:47.827 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 23:27:47.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:47.827 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 23:27:47.827 | INFO | PRE_HQ : 66.0%, 1.7, 57.1ms, 0 | 20260507 23:27:47.827 | INFO | BAZWRITER : 69.0%, 3.7, 131.1ms, 0 | 20260507 23:27:47.827 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 23:27:47.828 | INFO | Uploaded 36GiB to GPU in 2.55583s (14.0855GiB/s) | 20260507 23:27:47.828 | INFO | Downloaded 38GiB to GPU in 2.69016s (14.1256GiB/s) | 20260507 23:27:52.970 | INFO | Analyzing chunk frames = [3441152,3441664) | 20260507 23:27:52.970 | INFO | MemInfo: { | 20260507 23:27:52.970 | INFO | "gpu" : " 17 GB", | 20260507 23:27:52.970 | INFO | "huge" : "", | 20260507 23:27:52.970 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:52.970 | INFO | "malloc" : " 14 GB", | 20260507 23:27:52.970 | INFO | "pinned" : " 29 GB" | 20260507 23:27:52.970 | INFO | } | 20260507 23:27:52.971 | INFO | Perf report for the last 5.14327 seconds: | 20260507 23:27:52.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:52.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:27:52.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:27:52.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:52.971 | INFO | Uploaded 36GiB to GPU in 2.54209s (14.1616GiB/s) | 20260507 23:27:52.971 | INFO | Downloaded 34GiB to GPU in 2.49197s (13.6438GiB/s) | 20260507 23:27:58.112 | INFO | Analyzing chunk frames = [3441664,3442176) | 20260507 23:27:58.112 | INFO | MemInfo: { | 20260507 23:27:58.112 | INFO | "gpu" : " 17 GB", | 20260507 23:27:58.112 | INFO | "huge" : "", | 20260507 23:27:58.112 | INFO | "huge_shared" : " 29 GB", | 20260507 23:27:58.112 | INFO | "malloc" : " 14 GB", | 20260507 23:27:58.112 | INFO | "pinned" : " 29 GB" | 20260507 23:27:58.112 | INFO | } | 20260507 23:27:58.112 | INFO | Perf report for the last 5.14128 seconds: | 20260507 23:27:58.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:27:58.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:27:58.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:27:58.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:27:58.112 | INFO | Uploaded 36GiB to GPU in 2.51927s (14.2899GiB/s) | 20260507 23:27:58.112 | INFO | Downloaded 34GiB to GPU in 2.48296s (13.6933GiB/s) | 20260507 23:28:03.244 | INFO | Analyzing chunk frames = [3442176,3442688) | 20260507 23:28:03.244 | INFO | MemInfo: { | 20260507 23:28:03.244 | INFO | "gpu" : " 17 GB", | 20260507 23:28:03.244 | INFO | "huge" : "", | 20260507 23:28:03.244 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:03.244 | INFO | "malloc" : " 14 GB", | 20260507 23:28:03.244 | INFO | "pinned" : " 29 GB" | 20260507 23:28:03.244 | INFO | } | 20260507 23:28:03.246 | INFO | Perf report for the last 5.13371 seconds: | 20260507 23:28:03.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:03.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260507 23:28:03.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 23:28:03.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:03.246 | INFO | Uploaded 36GiB to GPU in 2.52046s (14.2831GiB/s) | 20260507 23:28:03.246 | INFO | Downloaded 34GiB to GPU in 2.4823s (13.6969GiB/s) | 20260507 23:28:08.386 | INFO | Analyzing chunk frames = [3442688,3443200) | 20260507 23:28:08.386 | INFO | MemInfo: { | 20260507 23:28:08.386 | INFO | "gpu" : " 17 GB", | 20260507 23:28:08.386 | INFO | "huge" : "", | 20260507 23:28:08.386 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:08.387 | INFO | "malloc" : " 14 GB", | 20260507 23:28:08.387 | INFO | "pinned" : " 29 GB" | 20260507 23:28:08.387 | INFO | } | 20260507 23:28:08.387 | INFO | Perf report for the last 5.14113 seconds: | 20260507 23:28:08.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:08.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 23:28:08.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 23:28:08.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:08.387 | INFO | Uploaded 36GiB to GPU in 2.40286s (14.9821GiB/s) | 20260507 23:28:08.387 | INFO | Downloaded 34GiB to GPU in 2.48708s (13.6706GiB/s) | 20260507 23:28:13.522 | INFO | Analyzing chunk frames = [3443200,3443712) | 20260507 23:28:13.522 | INFO | MemInfo: { | 20260507 23:28:13.522 | INFO | "gpu" : " 17 GB", | 20260507 23:28:13.523 | INFO | "huge" : "", | 20260507 23:28:13.523 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:13.523 | INFO | "malloc" : " 14 GB", | 20260507 23:28:13.523 | INFO | "pinned" : " 29 GB" | 20260507 23:28:13.523 | INFO | } | 20260507 23:28:13.523 | INFO | Perf report for the last 5.13607 seconds: | 20260507 23:28:13.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:13.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 23:28:13.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 23:28:13.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:13.523 | INFO | Uploaded 36GiB to GPU in 2.43775s (14.7677GiB/s) | 20260507 23:28:13.523 | INFO | Downloaded 34GiB to GPU in 2.38776s (14.2393GiB/s) | 20260507 23:28:18.665 | INFO | Analyzing chunk frames = [3443712,3444224) | 20260507 23:28:18.665 | INFO | MemInfo: { | 20260507 23:28:18.665 | INFO | "gpu" : " 17 GB", | 20260507 23:28:18.665 | INFO | "huge" : "", | 20260507 23:28:18.665 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:18.665 | INFO | "malloc" : " 14 GB", | 20260507 23:28:18.665 | INFO | "pinned" : " 29 GB" | 20260507 23:28:18.665 | INFO | } | 20260507 23:28:18.666 | INFO | Perf report for the last 5.14275 seconds: | 20260507 23:28:18.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:18.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.666 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260507 23:28:18.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.666 | INFO | PRE_HQ : 35.5%, 1.0, 18.5ms, 0 | 20260507 23:28:18.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:18.667 | INFO | Uploaded 36GiB to GPU in 2.56459s (14.0373GiB/s) | 20260507 23:28:18.667 | INFO | Downloaded 34GiB to GPU in 2.45069s (13.8736GiB/s) | 20260507 23:28:23.804 | INFO | Analyzing chunk frames = [3444224,3444736) | 20260507 23:28:23.804 | INFO | MemInfo: { | 20260507 23:28:23.804 | INFO | "gpu" : " 17 GB", | 20260507 23:28:23.804 | INFO | "huge" : "", | 20260507 23:28:23.804 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:23.804 | INFO | "malloc" : " 14 GB", | 20260507 23:28:23.804 | INFO | "pinned" : " 29 GB" | 20260507 23:28:23.805 | INFO | } | 20260507 23:28:23.805 | INFO | Perf report for the last 5.13916 seconds: | 20260507 23:28:23.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:23.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:28:23.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 23:28:23.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:23.805 | INFO | Uploaded 36GiB to GPU in 2.54498s (14.1455GiB/s) | 20260507 23:28:23.806 | INFO | Downloaded 34GiB to GPU in 2.44135s (13.9267GiB/s) | 20260507 23:28:23.918 | INFO | Starting a round of Baz IO | 20260507 23:28:27.398 | INFO | Populating RTMetrics report | 20260507 23:28:27.398 | INFO | RTMetrics report populated | 20260507 23:28:27.398 | INFO | Opening tmp RTMetrics json file | 20260507 23:28:27.409 | INFO | Serializing RTMetrics to json | 20260507 23:28:27.425 | INFO | Serialization done, swapping tmp file now... | 20260507 23:28:27.425 | INFO | RTMetrics done | 20260507 23:28:27.498 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.579 seconds | 20260507 23:28:27.498 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.60% / 0.01% / 57.39% | 20260507 23:28:27.498 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18364% / 7.94729e-06% / 6.75519e-05% | 20260507 23:28:27.498 | INFO | Baz IO overall: | 20260507 23:28:27.498 | INFO | BytesWritten : 1568236355405 | 20260507 23:28:27.498 | INFO | NumEvents : 759431125530 | 20260507 23:28:28.941 | INFO | Analyzing chunk frames = [3444736,3445248) | 20260507 23:28:28.941 | INFO | MemInfo: { | 20260507 23:28:28.941 | INFO | "gpu" : " 17 GB", | 20260507 23:28:28.941 | INFO | "huge" : "", | 20260507 23:28:28.941 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:28.941 | INFO | "malloc" : " 14 GB", | 20260507 23:28:28.941 | INFO | "pinned" : " 29 GB" | 20260507 23:28:28.941 | INFO | } | 20260507 23:28:28.941 | INFO | Perf report for the last 5.13631 seconds: | 20260507 23:28:28.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:28.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:28.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:28.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:28.941 | INFO | ANALYSIS : 69.2%, 2.9, 105.3ms, 0 | 20260507 23:28:28.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:28.941 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260507 23:28:28.941 | INFO | PRE_HQ : 68.1%, 1.8, 63.7ms, 0 | 20260507 23:28:28.941 | INFO | BAZWRITER : 69.2%, 4.3, 152.9ms, 0 | 20260507 23:28:28.941 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 23:28:28.941 | INFO | Uploaded 36GiB to GPU in 2.6184s (13.7488GiB/s) | 20260507 23:28:28.941 | INFO | Downloaded 38GiB to GPU in 2.69766s (14.0863GiB/s) | 20260507 23:28:34.081 | INFO | Analyzing chunk frames = [3445248,3445760) | 20260507 23:28:34.081 | INFO | MemInfo: { | 20260507 23:28:34.081 | INFO | "gpu" : " 17 GB", | 20260507 23:28:34.081 | INFO | "huge" : "", | 20260507 23:28:34.081 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:34.081 | INFO | "malloc" : " 14 GB", | 20260507 23:28:34.081 | INFO | "pinned" : " 29 GB" | 20260507 23:28:34.081 | INFO | } | 20260507 23:28:34.081 | INFO | Perf report for the last 5.13997 seconds: | 20260507 23:28:34.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:34.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:28:34.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:28:34.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:34.081 | INFO | Uploaded 36GiB to GPU in 2.56327s (14.0446GiB/s) | 20260507 23:28:34.081 | INFO | Downloaded 34GiB to GPU in 2.4807s (13.7058GiB/s) | 20260507 23:28:39.220 | INFO | Analyzing chunk frames = [3445760,3446272) | 20260507 23:28:39.220 | INFO | MemInfo: { | 20260507 23:28:39.220 | INFO | "gpu" : " 17 GB", | 20260507 23:28:39.220 | INFO | "huge" : "", | 20260507 23:28:39.220 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:39.220 | INFO | "malloc" : " 14 GB", | 20260507 23:28:39.220 | INFO | "pinned" : " 29 GB" | 20260507 23:28:39.220 | INFO | } | 20260507 23:28:39.220 | INFO | Perf report for the last 5.13935 seconds: | 20260507 23:28:39.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:39.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.220 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:28:39.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.221 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:28:39.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:39.221 | INFO | Uploaded 36GiB to GPU in 2.51696s (14.303GiB/s) | 20260507 23:28:39.221 | INFO | Downloaded 34GiB to GPU in 2.49178s (13.6449GiB/s) | 20260507 23:28:44.362 | INFO | Analyzing chunk frames = [3446272,3446784) | 20260507 23:28:44.362 | INFO | MemInfo: { | 20260507 23:28:44.362 | INFO | "gpu" : " 17 GB", | 20260507 23:28:44.362 | INFO | "huge" : "", | 20260507 23:28:44.362 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:44.364 | INFO | "malloc" : " 14 GB", | 20260507 23:28:44.364 | INFO | "pinned" : " 29 GB" | 20260507 23:28:44.364 | INFO | } | 20260507 23:28:44.364 | INFO | Perf report for the last 5.14316 seconds: | 20260507 23:28:44.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:44.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260507 23:28:44.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:28:44.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:44.364 | INFO | Uploaded 36GiB to GPU in 2.56951s (14.0105GiB/s) | 20260507 23:28:44.364 | INFO | Downloaded 34GiB to GPU in 2.47547s (13.7348GiB/s) | 20260507 23:28:49.497 | INFO | Analyzing chunk frames = [3446784,3447296) | 20260507 23:28:49.497 | INFO | MemInfo: { | 20260507 23:28:49.497 | INFO | "gpu" : " 17 GB", | 20260507 23:28:49.497 | INFO | "huge" : "", | 20260507 23:28:49.497 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:49.497 | INFO | "malloc" : " 14 GB", | 20260507 23:28:49.497 | INFO | "pinned" : " 29 GB" | 20260507 23:28:49.497 | INFO | } | 20260507 23:28:49.497 | INFO | Perf report for the last 5.13332 seconds: | 20260507 23:28:49.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:49.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:28:49.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:28:49.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:49.497 | INFO | Uploaded 36GiB to GPU in 2.29141s (15.7108GiB/s) | 20260507 23:28:49.497 | INFO | Downloaded 34GiB to GPU in 2.47409s (13.7424GiB/s) | 20260507 23:28:54.637 | INFO | Analyzing chunk frames = [3447296,3447808) | 20260507 23:28:54.637 | INFO | MemInfo: { | 20260507 23:28:54.637 | INFO | "gpu" : " 17 GB", | 20260507 23:28:54.637 | INFO | "huge" : "", | 20260507 23:28:54.637 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:54.637 | INFO | "malloc" : " 14 GB", | 20260507 23:28:54.637 | INFO | "pinned" : " 29 GB" | 20260507 23:28:54.637 | INFO | } | 20260507 23:28:54.637 | INFO | Perf report for the last 5.14032 seconds: | 20260507 23:28:54.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:54.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:28:54.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:28:54.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:54.637 | INFO | Uploaded 36GiB to GPU in 2.40864s (14.9462GiB/s) | 20260507 23:28:54.638 | INFO | Downloaded 34GiB to GPU in 2.40435s (14.1411GiB/s) | 20260507 23:28:59.781 | INFO | Analyzing chunk frames = [3447808,3448320) | 20260507 23:28:59.781 | INFO | MemInfo: { | 20260507 23:28:59.781 | INFO | "gpu" : " 17 GB", | 20260507 23:28:59.781 | INFO | "huge" : "", | 20260507 23:28:59.781 | INFO | "huge_shared" : " 29 GB", | 20260507 23:28:59.781 | INFO | "malloc" : " 14 GB", | 20260507 23:28:59.782 | INFO | "pinned" : " 29 GB" | 20260507 23:28:59.782 | INFO | } | 20260507 23:28:59.782 | INFO | Perf report for the last 5.14443 seconds: | 20260507 23:28:59.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:28:59.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:28:59.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 23:28:59.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:28:59.782 | INFO | Uploaded 36GiB to GPU in 2.52335s (14.2668GiB/s) | 20260507 23:28:59.782 | INFO | Downloaded 34GiB to GPU in 2.46296s (13.8045GiB/s) | 20260507 23:29:04.923 | INFO | Analyzing chunk frames = [3448320,3448832) | 20260507 23:29:04.923 | INFO | MemInfo: { | 20260507 23:29:04.923 | INFO | "gpu" : " 17 GB", | 20260507 23:29:04.923 | INFO | "huge" : "", | 20260507 23:29:04.923 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:04.923 | INFO | "malloc" : " 14 GB", | 20260507 23:29:04.923 | INFO | "pinned" : " 29 GB" | 20260507 23:29:04.923 | INFO | } | 20260507 23:29:04.923 | INFO | Perf report for the last 5.14143 seconds: | 20260507 23:29:04.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:04.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.923 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:29:04.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.923 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 23:29:04.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:04.924 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:29:04.924 | INFO | Total mem usage: 16.7 GiB | 20260507 23:29:04.924 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:29:04.924 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.9%) | 20260507 23:29:04.924 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:29:04.924 | INFO | High Water Breakdown: | 20260507 23:29:04.924 | INFO | 60% : 8.95 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:29:04.924 | INFO | 39.8% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:29:04.924 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:29:04.924 | INFO | ------------------------------------------ | 20260507 23:29:04.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:29:04.924 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:29:04.924 | INFO | Total mem usage: 26.5 GiB | 20260507 23:29:04.924 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:29:04.924 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:29:04.924 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:29:04.924 | INFO | High Water Breakdown: | 20260507 23:29:04.924 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:29:04.924 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:29:04.924 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:29:04.924 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:29:04.924 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:29:04.924 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:29:04.924 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:29:04.924 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:29:04.924 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:29:04.924 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:29:04.924 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:29:04.924 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:29:04.924 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:29:04.924 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:29:04.924 | INFO | ------------------------------------------ | 20260507 23:29:04.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:29:04.924 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:29:04.924 | INFO | Total mem usage: 27.8 GiB | 20260507 23:29:04.924 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:29:04.924 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:29:04.924 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:29:04.924 | INFO | High Water Breakdown: | 20260507 23:29:04.924 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:29:04.924 | INFO | ------------------------------------------ | 20260507 23:29:04.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:29:04.924 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:29:04.924 | INFO | Total mem usage: 15.7 GiB | 20260507 23:29:04.924 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:29:04.924 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:29:04.924 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:29:04.924 | INFO | High Water Breakdown: | 20260507 23:29:04.924 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:29:04.924 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:29:04.924 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:29:04.924 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:29:04.924 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:29:04.924 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:29:04.924 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:29:04.924 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:29:04.924 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:29:04.924 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:29:04.924 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:29:04.924 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:29:04.924 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:29:04.924 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:29:04.924 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:29:04.924 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:29:04.924 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:29:04.924 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:29:04.924 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:29:04.924 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:29:04.924 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:29:04.924 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:29:04.924 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:29:04.924 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:29:04.924 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:29:04.924 | INFO | ------------------------------------------ | 20260507 23:29:04.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:29:04.924 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:29:04.924 | INFO | Uploaded 36GiB to GPU in 2.52034s (14.2838GiB/s) | 20260507 23:29:04.924 | INFO | Downloaded 34GiB to GPU in 2.45054s (13.8745GiB/s) | 20260507 23:29:04.924 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:29:04.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:29:04.925 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.64e+01, 1.41e+01 / 1.62e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.03e-01, 1.15e-02 / 6.28e-03 | 20260507 23:29:04.925 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.48e+00, 5.87e-01 / 1.92e-01 | 20260507 23:29:04.925 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.07e-01 / 1.11e-01 | 20260507 23:29:04.925 | INFO | PerfReport: 16.01%, DME -- 4.21e+01 / 4.58e+01, 4.10e+00 / 1.47e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.69e+00 / 1.59e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.32e-01 / 1.94e-01 | 20260507 23:29:04.925 | INFO | PerfReport: 3.43%, Metrics -- 1.13e+00 / 1.31e+00, 2.81e-01 / 1.49e-01 | 20260507 23:29:04.925 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.49e-01 / 3.78e-01, 5.63e-02 / 2.74e-02 | 20260507 23:29:04.925 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:29:04.925 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:29:04.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:29:04.925 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 30.81%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 2.19%, LockUpload -- 2.33e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 2.57%, LockDownload -- 2.74e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 23:29:04.925 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 3.58e-02 / 0.00e+00 | 20260507 23:29:05.021 | INFO | Starting a round of Baz IO | 20260507 23:29:08.478 | INFO | Populating RTMetrics report | 20260507 23:29:08.478 | INFO | RTMetrics report populated | 20260507 23:29:08.478 | INFO | Opening tmp RTMetrics json file | 20260507 23:29:08.506 | INFO | Serializing RTMetrics to json | 20260507 23:29:08.507 | INFO | Serialization done, swapping tmp file now... | 20260507 23:29:08.508 | INFO | RTMetrics done | 20260507 23:29:08.585 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.564 seconds | 20260507 23:29:08.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.60% / 0.01% / 57.40% | 20260507 23:29:08.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18584% / 3.97364e-06% / 6.75519e-05% | 20260507 23:29:08.586 | INFO | Baz IO overall: | 20260507 23:29:08.586 | INFO | BytesWritten : 1570077530335 | 20260507 23:29:08.586 | INFO | NumEvents : 760318744723 | 20260507 23:29:10.061 | INFO | Analyzing chunk frames = [3448832,3449344) | 20260507 23:29:10.061 | INFO | MemInfo: { | 20260507 23:29:10.061 | INFO | "gpu" : " 17 GB", | 20260507 23:29:10.061 | INFO | "huge" : "", | 20260507 23:29:10.061 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:10.061 | INFO | "malloc" : " 14 GB", | 20260507 23:29:10.061 | INFO | "pinned" : " 29 GB" | 20260507 23:29:10.061 | INFO | } | 20260507 23:29:10.061 | INFO | Perf report for the last 5.13746 seconds: | 20260507 23:29:10.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:10.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:10.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:10.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:10.061 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260507 23:29:10.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:10.061 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 23:29:10.061 | INFO | PRE_HQ : 66.2%, 1.7, 56.8ms, 0 | 20260507 23:29:10.061 | INFO | BAZWRITER : 69.0%, 3.7, 130.2ms, 0 | 20260507 23:29:10.061 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260507 23:29:10.062 | INFO | Uploaded 36GiB to GPU in 2.5577s (14.0751GiB/s) | 20260507 23:29:10.062 | INFO | Downloaded 38GiB to GPU in 2.67287s (14.2169GiB/s) | 20260507 23:29:15.194 | INFO | Analyzing chunk frames = [3449344,3449856) | 20260507 23:29:15.194 | INFO | MemInfo: { | 20260507 23:29:15.194 | INFO | "gpu" : " 17 GB", | 20260507 23:29:15.194 | INFO | "huge" : "", | 20260507 23:29:15.194 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:15.194 | INFO | "malloc" : " 14 GB", | 20260507 23:29:15.194 | INFO | "pinned" : " 29 GB" | 20260507 23:29:15.194 | INFO | } | 20260507 23:29:15.194 | INFO | Perf report for the last 5.1337 seconds: | 20260507 23:29:15.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:15.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.194 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 23:29:15.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.194 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:29:15.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:15.195 | INFO | Uploaded 36GiB to GPU in 2.5756s (13.9774GiB/s) | 20260507 23:29:15.195 | INFO | Downloaded 34GiB to GPU in 2.49624s (13.6205GiB/s) | 20260507 23:29:20.335 | INFO | Analyzing chunk frames = [3449856,3450368) | 20260507 23:29:20.335 | INFO | MemInfo: { | 20260507 23:29:20.335 | INFO | "gpu" : " 17 GB", | 20260507 23:29:20.335 | INFO | "huge" : "", | 20260507 23:29:20.335 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:20.335 | INFO | "malloc" : " 14 GB", | 20260507 23:29:20.335 | INFO | "pinned" : " 29 GB" | 20260507 23:29:20.335 | INFO | } | 20260507 23:29:20.335 | INFO | Perf report for the last 5.1411 seconds: | 20260507 23:29:20.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:20.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.335 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260507 23:29:20.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.335 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:29:20.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:20.336 | INFO | Uploaded 36GiB to GPU in 2.52156s (14.2769GiB/s) | 20260507 23:29:20.336 | INFO | Downloaded 34GiB to GPU in 2.48942s (13.6578GiB/s) | 20260507 23:29:25.479 | INFO | Analyzing chunk frames = [3450368,3450880) | 20260507 23:29:25.479 | INFO | MemInfo: { | 20260507 23:29:25.479 | INFO | "gpu" : " 17 GB", | 20260507 23:29:25.479 | INFO | "huge" : "", | 20260507 23:29:25.479 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:25.479 | INFO | "malloc" : " 14 GB", | 20260507 23:29:25.479 | INFO | "pinned" : " 29 GB" | 20260507 23:29:25.480 | INFO | } | 20260507 23:29:25.480 | INFO | Perf report for the last 5.14412 seconds: | 20260507 23:29:25.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:25.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:29:25.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:29:25.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:25.480 | INFO | Uploaded 36GiB to GPU in 2.48588s (14.4818GiB/s) | 20260507 23:29:25.480 | INFO | Downloaded 34GiB to GPU in 2.49629s (13.6202GiB/s) | 20260507 23:29:30.617 | INFO | Analyzing chunk frames = [3450880,3451392) | 20260507 23:29:30.617 | INFO | MemInfo: { | 20260507 23:29:30.617 | INFO | "gpu" : " 17 GB", | 20260507 23:29:30.617 | INFO | "huge" : "", | 20260507 23:29:30.617 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:30.617 | INFO | "malloc" : " 14 GB", | 20260507 23:29:30.617 | INFO | "pinned" : " 29 GB" | 20260507 23:29:30.618 | INFO | } | 20260507 23:29:30.618 | INFO | Perf report for the last 5.13798 seconds: | 20260507 23:29:30.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:30.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 23:29:30.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:29:30.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:30.618 | INFO | Uploaded 36GiB to GPU in 2.28867s (15.7297GiB/s) | 20260507 23:29:30.618 | INFO | Downloaded 34GiB to GPU in 2.48028s (13.7081GiB/s) | 20260507 23:29:35.752 | INFO | Analyzing chunk frames = [3451392,3451904) | 20260507 23:29:35.752 | INFO | MemInfo: { | 20260507 23:29:35.752 | INFO | "gpu" : " 17 GB", | 20260507 23:29:35.752 | INFO | "huge" : "", | 20260507 23:29:35.752 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:35.752 | INFO | "malloc" : " 14 GB", | 20260507 23:29:35.752 | INFO | "pinned" : " 29 GB" | 20260507 23:29:35.752 | INFO | } | 20260507 23:29:35.752 | INFO | Perf report for the last 5.13487 seconds: | 20260507 23:29:35.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:35.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:29:35.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 23:29:35.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:35.753 | INFO | Uploaded 36GiB to GPU in 2.36612s (15.2148GiB/s) | 20260507 23:29:35.753 | INFO | Downloaded 34GiB to GPU in 2.41435s (14.0824GiB/s) | 20260507 23:29:40.890 | INFO | Analyzing chunk frames = [3451904,3452416) | 20260507 23:29:40.891 | INFO | MemInfo: { | 20260507 23:29:40.891 | INFO | "gpu" : " 17 GB", | 20260507 23:29:40.891 | INFO | "huge" : "", | 20260507 23:29:40.891 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:40.891 | INFO | "malloc" : " 14 GB", | 20260507 23:29:40.891 | INFO | "pinned" : " 29 GB" | 20260507 23:29:40.891 | INFO | } | 20260507 23:29:40.891 | INFO | Perf report for the last 5.13855 seconds: | 20260507 23:29:40.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:40.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:29:40.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260507 23:29:40.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:40.891 | INFO | Uploaded 36GiB to GPU in 2.50558s (14.368GiB/s) | 20260507 23:29:40.891 | INFO | Downloaded 34GiB to GPU in 2.46876s (13.7721GiB/s) | 20260507 23:29:46.033 | INFO | Analyzing chunk frames = [3452416,3452928) | 20260507 23:29:46.033 | INFO | MemInfo: { | 20260507 23:29:46.033 | INFO | "gpu" : " 17 GB", | 20260507 23:29:46.033 | INFO | "huge" : "", | 20260507 23:29:46.034 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:46.034 | INFO | "malloc" : " 14 GB", | 20260507 23:29:46.034 | INFO | "pinned" : " 29 GB" | 20260507 23:29:46.034 | INFO | } | 20260507 23:29:46.034 | INFO | Perf report for the last 5.1427 seconds: | 20260507 23:29:46.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:46.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.034 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:29:46.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.034 | INFO | PRE_HQ : 34.9%, 1.0, 18.2ms, 0 | 20260507 23:29:46.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:46.035 | INFO | Uploaded 36GiB to GPU in 2.49679s (14.4185GiB/s) | 20260507 23:29:46.035 | INFO | Downloaded 34GiB to GPU in 2.4631s (13.8038GiB/s) | 20260507 23:29:46.148 | INFO | Starting a round of Baz IO | 20260507 23:29:49.615 | INFO | Populating RTMetrics report | 20260507 23:29:49.615 | INFO | RTMetrics report populated | 20260507 23:29:49.615 | INFO | Opening tmp RTMetrics json file | 20260507 23:29:49.632 | INFO | Serializing RTMetrics to json | 20260507 23:29:49.634 | INFO | Serialization done, swapping tmp file now... | 20260507 23:29:49.634 | INFO | RTMetrics done | 20260507 23:29:49.707 | INFO | Finished a round of Baz IO: Wrote 1.712 GiB in 3.559 seconds | 20260507 23:29:49.707 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.59% / 0.01% / 57.40% | 20260507 23:29:49.707 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18179% / 1.19209e-05% / 5.5631e-05% | 20260507 23:29:49.707 | INFO | Baz IO overall: | 20260507 23:29:49.707 | INFO | BytesWritten : 1571916221644 | 20260507 23:29:49.707 | INFO | NumEvents : 761204391727 | 20260507 23:29:51.173 | INFO | Analyzing chunk frames = [3452928,3453440) | 20260507 23:29:51.173 | INFO | MemInfo: { | 20260507 23:29:51.173 | INFO | "gpu" : " 17 GB", | 20260507 23:29:51.173 | INFO | "huge" : "", | 20260507 23:29:51.173 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:51.173 | INFO | "malloc" : " 14 GB", | 20260507 23:29:51.173 | INFO | "pinned" : " 29 GB" | 20260507 23:29:51.173 | INFO | } | 20260507 23:29:51.173 | INFO | Perf report for the last 5.13939 seconds: | 20260507 23:29:51.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:51.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:51.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:51.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:51.173 | INFO | ANALYSIS : 69.0%, 2.9, 104.8ms, 0 | 20260507 23:29:51.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:51.173 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260507 23:29:51.173 | INFO | PRE_HQ : 68.0%, 1.8, 63.6ms, 0 | 20260507 23:29:51.173 | INFO | BAZWRITER : 68.9%, 4.2, 150.2ms, 0 | 20260507 23:29:51.173 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 23:29:51.173 | INFO | Uploaded 36GiB to GPU in 2.53871s (14.1804GiB/s) | 20260507 23:29:51.173 | INFO | Downloaded 38GiB to GPU in 2.70798s (14.0326GiB/s) | 20260507 23:29:56.310 | INFO | Analyzing chunk frames = [3453440,3453952) | 20260507 23:29:56.310 | INFO | MemInfo: { | 20260507 23:29:56.310 | INFO | "gpu" : " 17 GB", | 20260507 23:29:56.310 | INFO | "huge" : "", | 20260507 23:29:56.310 | INFO | "huge_shared" : " 29 GB", | 20260507 23:29:56.310 | INFO | "malloc" : " 14 GB", | 20260507 23:29:56.310 | INFO | "pinned" : " 29 GB" | 20260507 23:29:56.310 | INFO | } | 20260507 23:29:56.310 | INFO | Perf report for the last 5.13649 seconds: | 20260507 23:29:56.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:29:56.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:29:56.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:29:56.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:29:56.310 | INFO | Uploaded 36GiB to GPU in 2.50291s (14.3833GiB/s) | 20260507 23:29:56.310 | INFO | Downloaded 34GiB to GPU in 2.50063s (13.5965GiB/s) | 20260507 23:30:01.446 | INFO | Analyzing chunk frames = [3453952,3454464) | 20260507 23:30:01.446 | INFO | MemInfo: { | 20260507 23:30:01.446 | INFO | "gpu" : " 17 GB", | 20260507 23:30:01.446 | INFO | "huge" : "", | 20260507 23:30:01.446 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:01.446 | INFO | "malloc" : " 14 GB", | 20260507 23:30:01.446 | INFO | "pinned" : " 29 GB" | 20260507 23:30:01.446 | INFO | } | 20260507 23:30:01.446 | INFO | Perf report for the last 5.13636 seconds: | 20260507 23:30:01.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:01.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 23:30:01.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:30:01.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:01.446 | INFO | Uploaded 36GiB to GPU in 2.45352s (14.6728GiB/s) | 20260507 23:30:01.446 | INFO | Downloaded 34GiB to GPU in 2.50046s (13.5975GiB/s) | 20260507 23:30:06.594 | INFO | Analyzing chunk frames = [3454464,3454976) | 20260507 23:30:06.594 | INFO | MemInfo: { | 20260507 23:30:06.594 | INFO | "gpu" : " 17 GB", | 20260507 23:30:06.594 | INFO | "huge" : "", | 20260507 23:30:06.594 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:06.594 | INFO | "malloc" : " 14 GB", | 20260507 23:30:06.594 | INFO | "pinned" : " 29 GB" | 20260507 23:30:06.594 | INFO | } | 20260507 23:30:06.594 | INFO | Perf report for the last 5.14823 seconds: | 20260507 23:30:06.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:06.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.594 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 23:30:06.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.594 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:30:06.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:06.595 | INFO | Uploaded 36GiB to GPU in 2.47922s (14.5207GiB/s) | 20260507 23:30:06.595 | INFO | Downloaded 34GiB to GPU in 2.41964s (14.0517GiB/s) | 20260507 23:30:11.734 | INFO | Analyzing chunk frames = [3454976,3455488) | 20260507 23:30:11.734 | INFO | MemInfo: { | 20260507 23:30:11.734 | INFO | "gpu" : " 17 GB", | 20260507 23:30:11.734 | INFO | "huge" : "", | 20260507 23:30:11.734 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:11.734 | INFO | "malloc" : " 14 GB", | 20260507 23:30:11.734 | INFO | "pinned" : " 29 GB" | 20260507 23:30:11.734 | INFO | } | 20260507 23:30:11.734 | INFO | Perf report for the last 5.13975 seconds: | 20260507 23:30:11.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:11.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.734 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 23:30:11.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.734 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:30:11.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:11.735 | INFO | Uploaded 36GiB to GPU in 2.309s (15.5911GiB/s) | 20260507 23:30:11.735 | INFO | Downloaded 34GiB to GPU in 2.50065s (13.5965GiB/s) | 20260507 23:30:16.875 | INFO | Analyzing chunk frames = [3455488,3456000) | 20260507 23:30:16.875 | INFO | MemInfo: { | 20260507 23:30:16.875 | INFO | "gpu" : " 17 GB", | 20260507 23:30:16.875 | INFO | "huge" : "", | 20260507 23:30:16.875 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:16.875 | INFO | "malloc" : " 14 GB", | 20260507 23:30:16.875 | INFO | "pinned" : " 29 GB" | 20260507 23:30:16.875 | INFO | } | 20260507 23:30:16.875 | INFO | Perf report for the last 5.1409 seconds: | 20260507 23:30:16.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:16.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260507 23:30:16.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:30:16.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:16.875 | INFO | Uploaded 36GiB to GPU in 2.40476s (14.9703GiB/s) | 20260507 23:30:16.876 | INFO | Downloaded 34GiB to GPU in 2.40257s (14.1515GiB/s) | 20260507 23:30:22.011 | INFO | Analyzing chunk frames = [3456000,3456512) | 20260507 23:30:22.011 | INFO | MemInfo: { | 20260507 23:30:22.011 | INFO | "gpu" : " 17 GB", | 20260507 23:30:22.011 | INFO | "huge" : "", | 20260507 23:30:22.011 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:22.011 | INFO | "malloc" : " 14 GB", | 20260507 23:30:22.011 | INFO | "pinned" : " 29 GB" | 20260507 23:30:22.011 | INFO | } | 20260507 23:30:22.011 | INFO | Perf report for the last 5.13604 seconds: | 20260507 23:30:22.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:22.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.011 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 23:30:22.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.011 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 23:30:22.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:22.012 | INFO | Uploaded 36GiB to GPU in 2.53323s (14.2111GiB/s) | 20260507 23:30:22.012 | INFO | Downloaded 34GiB to GPU in 2.45727s (13.8365GiB/s) | 20260507 23:30:27.144 | INFO | Analyzing chunk frames = [3456512,3457024) | 20260507 23:30:27.144 | INFO | MemInfo: { | 20260507 23:30:27.144 | INFO | "gpu" : " 17 GB", | 20260507 23:30:27.144 | INFO | "huge" : "", | 20260507 23:30:27.144 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:27.144 | INFO | "malloc" : " 14 GB", | 20260507 23:30:27.145 | INFO | "pinned" : " 29 GB" | 20260507 23:30:27.145 | INFO | } | 20260507 23:30:27.145 | INFO | Perf report for the last 5.13384 seconds: | 20260507 23:30:27.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:27.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:30:27.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 23:30:27.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:27.145 | INFO | Uploaded 36GiB to GPU in 2.51398s (14.3199GiB/s) | 20260507 23:30:27.145 | INFO | Downloaded 34GiB to GPU in 2.45515s (13.8485GiB/s) | 20260507 23:30:27.145 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:30:27.145 | INFO | Total mem usage: 16.7 GiB | 20260507 23:30:27.145 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:30:27.145 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260507 23:30:27.145 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:30:27.145 | INFO | High Water Breakdown: | 20260507 23:30:27.145 | INFO | 60% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:30:27.145 | INFO | 39.8% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:30:27.145 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:30:27.145 | INFO | ------------------------------------------ | 20260507 23:30:27.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:30:27.146 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:30:27.146 | INFO | Total mem usage: 26.5 GiB | 20260507 23:30:27.146 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:30:27.146 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 23:30:27.146 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 23:30:27.146 | INFO | High Water Breakdown: | 20260507 23:30:27.146 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:30:27.146 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:30:27.146 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:30:27.146 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:30:27.146 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:30:27.146 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:30:27.146 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:30:27.146 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:30:27.146 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:30:27.146 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:30:27.146 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:30:27.146 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:30:27.146 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:30:27.146 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:30:27.146 | INFO | ------------------------------------------ | 20260507 23:30:27.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:30:27.146 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:30:27.146 | INFO | Total mem usage: 27.8 GiB | 20260507 23:30:27.146 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:30:27.146 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:30:27.146 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:30:27.146 | INFO | High Water Breakdown: | 20260507 23:30:27.146 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:30:27.146 | INFO | ------------------------------------------ | 20260507 23:30:27.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:30:27.146 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:30:27.146 | INFO | Total mem usage: 15.7 GiB | 20260507 23:30:27.146 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:30:27.146 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:30:27.146 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:30:27.146 | INFO | High Water Breakdown: | 20260507 23:30:27.146 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:30:27.146 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:30:27.146 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:30:27.146 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:30:27.147 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:30:27.147 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:30:27.147 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:30:27.147 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:30:27.147 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:30:27.147 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:30:27.147 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:30:27.147 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:30:27.147 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:30:27.147 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:30:27.147 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:30:27.147 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:30:27.147 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:30:27.147 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:30:27.147 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:30:27.147 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:30:27.147 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:30:27.147 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:30:27.147 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:30:27.147 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:30:27.147 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:30:27.147 | INFO | ------------------------------------------ | 20260507 23:30:27.147 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:30:27.147 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:30:27.147 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:30:27.147 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:30:27.147 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.39e+01 / 1.69e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.04e-01, 1.13e-02 / 9.52e-03 | 20260507 23:30:27.147 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.47e+00, 5.89e-01 / 2.29e-01 | 20260507 23:30:27.147 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.50e+00, 2.10e-01 / 1.13e-01 | 20260507 23:30:27.147 | INFO | PerfReport: 16.04%, DME -- 4.22e+01 / 4.61e+01, 4.32e+00 / 1.83e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.85e+01 / 1.93e+01, 1.65e+00 / 1.54e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.28e-01 / 2.63e-01 | 20260507 23:30:27.147 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.34e+00, 2.83e-01 / 1.33e-01 | 20260507 23:30:27.147 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.50e-01 / 3.91e-01, 5.57e-02 / 2.86e-02 | 20260507 23:30:27.147 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:30:27.147 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:30:27.147 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:30:27.147 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 24.16%, Upload -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 30.87%, Compute -- 3.29e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.24e+00 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 2.83%, LockDownload -- 3.01e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 15.76%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:30:27.147 | INFO | PerfReport: 0.01%, Misc -- 1.59e-02 / 0.00e+00, 3.62e-02 / 0.00e+00 | 20260507 23:30:27.258 | INFO | Starting a round of Baz IO | 20260507 23:30:30.719 | INFO | Populating RTMetrics report | 20260507 23:30:30.719 | INFO | RTMetrics report populated | 20260507 23:30:30.719 | INFO | Opening tmp RTMetrics json file | 20260507 23:30:30.753 | INFO | Serializing RTMetrics to json | 20260507 23:30:30.784 | INFO | Serialization done, swapping tmp file now... | 20260507 23:30:30.810 | INFO | RTMetrics done | 20260507 23:30:30.932 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.674 seconds | 20260507 23:30:30.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.59% / 0.01% / 57.41% | 20260507 23:30:30.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18197% / 7.94729e-06% / 6.75519e-05% | 20260507 23:30:30.933 | INFO | Baz IO overall: | 20260507 23:30:30.933 | INFO | BytesWritten : 1573756827908 | 20260507 23:30:30.933 | INFO | NumEvents : 762091482624 | 20260507 23:30:32.284 | INFO | Analyzing chunk frames = [3457024,3457536) | 20260507 23:30:32.284 | INFO | MemInfo: { | 20260507 23:30:32.284 | INFO | "gpu" : " 17 GB", | 20260507 23:30:32.284 | INFO | "huge" : "", | 20260507 23:30:32.284 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:32.284 | INFO | "malloc" : " 14 GB", | 20260507 23:30:32.284 | INFO | "pinned" : " 29 GB" | 20260507 23:30:32.284 | INFO | } | 20260507 23:30:32.284 | INFO | Perf report for the last 5.13942 seconds: | 20260507 23:30:32.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:32.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:32.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:32.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:32.284 | INFO | ANALYSIS : 68.8%, 2.9, 104.6ms, 0 | 20260507 23:30:32.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:32.284 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260507 23:30:32.284 | INFO | PRE_HQ : 67.8%, 1.8, 64.7ms, 0 | 20260507 23:30:32.284 | INFO | BAZWRITER : 71.0%, 4.7, 170.9ms, 0 | 20260507 23:30:32.284 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:30:32.285 | INFO | Uploaded 36GiB to GPU in 2.59337s (13.8815GiB/s) | 20260507 23:30:32.285 | INFO | Downloaded 38GiB to GPU in 2.70273s (14.0599GiB/s) | 20260507 23:30:37.431 | INFO | Analyzing chunk frames = [3457536,3458048) | 20260507 23:30:37.431 | INFO | MemInfo: { | 20260507 23:30:37.431 | INFO | "gpu" : " 17 GB", | 20260507 23:30:37.431 | INFO | "huge" : "", | 20260507 23:30:37.431 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:37.431 | INFO | "malloc" : " 14 GB", | 20260507 23:30:37.431 | INFO | "pinned" : " 29 GB" | 20260507 23:30:37.431 | INFO | } | 20260507 23:30:37.431 | INFO | Perf report for the last 5.14718 seconds: | 20260507 23:30:37.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:37.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.432 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 23:30:37.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.432 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:30:37.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:37.432 | INFO | Uploaded 36GiB to GPU in 2.58896s (13.9052GiB/s) | 20260507 23:30:37.432 | INFO | Downloaded 34GiB to GPU in 2.47432s (13.7412GiB/s) | 20260507 23:30:42.565 | INFO | Analyzing chunk frames = [3458048,3458560) | 20260507 23:30:42.565 | INFO | MemInfo: { | 20260507 23:30:42.565 | INFO | "gpu" : " 17 GB", | 20260507 23:30:42.565 | INFO | "huge" : "", | 20260507 23:30:42.565 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:42.565 | INFO | "malloc" : " 14 GB", | 20260507 23:30:42.565 | INFO | "pinned" : " 29 GB" | 20260507 23:30:42.565 | INFO | } | 20260507 23:30:42.565 | INFO | Perf report for the last 5.13372 seconds: | 20260507 23:30:42.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:42.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.565 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 23:30:42.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.565 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:30:42.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:42.566 | INFO | Uploaded 36GiB to GPU in 2.53336s (14.2104GiB/s) | 20260507 23:30:42.566 | INFO | Downloaded 34GiB to GPU in 2.46708s (13.7815GiB/s) | 20260507 23:30:47.700 | INFO | Analyzing chunk frames = [3458560,3459072) | 20260507 23:30:47.700 | INFO | MemInfo: { | 20260507 23:30:47.700 | INFO | "gpu" : " 17 GB", | 20260507 23:30:47.700 | INFO | "huge" : "", | 20260507 23:30:47.701 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:47.701 | INFO | "malloc" : " 14 GB", | 20260507 23:30:47.701 | INFO | "pinned" : " 29 GB" | 20260507 23:30:47.701 | INFO | } | 20260507 23:30:47.701 | INFO | Perf report for the last 5.13539 seconds: | 20260507 23:30:47.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:47.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260507 23:30:47.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:30:47.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:47.701 | INFO | Uploaded 36GiB to GPU in 2.51126s (14.3354GiB/s) | 20260507 23:30:47.701 | INFO | Downloaded 34GiB to GPU in 2.39965s (14.1688GiB/s) | 20260507 23:30:52.844 | INFO | Analyzing chunk frames = [3459072,3459584) | 20260507 23:30:52.844 | INFO | MemInfo: { | 20260507 23:30:52.844 | INFO | "gpu" : " 17 GB", | 20260507 23:30:52.844 | INFO | "huge" : "", | 20260507 23:30:52.844 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:52.844 | INFO | "malloc" : " 14 GB", | 20260507 23:30:52.844 | INFO | "pinned" : " 29 GB" | 20260507 23:30:52.844 | INFO | } | 20260507 23:30:52.844 | INFO | Perf report for the last 5.14339 seconds: | 20260507 23:30:52.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:52.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 23:30:52.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:30:52.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:52.844 | INFO | Uploaded 36GiB to GPU in 2.39633s (15.023GiB/s) | 20260507 23:30:52.844 | INFO | Downloaded 34GiB to GPU in 2.48767s (13.6674GiB/s) | 20260507 23:30:57.984 | INFO | Analyzing chunk frames = [3459584,3460096) | 20260507 23:30:57.984 | INFO | MemInfo: { | 20260507 23:30:57.984 | INFO | "gpu" : " 17 GB", | 20260507 23:30:57.984 | INFO | "huge" : "", | 20260507 23:30:57.984 | INFO | "huge_shared" : " 29 GB", | 20260507 23:30:57.984 | INFO | "malloc" : " 14 GB", | 20260507 23:30:57.984 | INFO | "pinned" : " 29 GB" | 20260507 23:30:57.984 | INFO | } | 20260507 23:30:57.984 | INFO | Perf report for the last 5.1399 seconds: | 20260507 23:30:57.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:30:57.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.984 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:30:57.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.984 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:30:57.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:30:57.985 | INFO | Uploaded 36GiB to GPU in 2.51958s (14.2881GiB/s) | 20260507 23:30:57.985 | INFO | Downloaded 34GiB to GPU in 2.43246s (13.9776GiB/s) | 20260507 23:31:03.128 | INFO | Analyzing chunk frames = [3460096,3460608) | 20260507 23:31:03.128 | INFO | MemInfo: { | 20260507 23:31:03.128 | INFO | "gpu" : " 17 GB", | 20260507 23:31:03.128 | INFO | "huge" : "", | 20260507 23:31:03.128 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:03.128 | INFO | "malloc" : " 14 GB", | 20260507 23:31:03.128 | INFO | "pinned" : " 29 GB" | 20260507 23:31:03.128 | INFO | } | 20260507 23:31:03.128 | INFO | Perf report for the last 5.14402 seconds: | 20260507 23:31:03.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:03.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.128 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 23:31:03.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.128 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:31:03.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:03.129 | INFO | Uploaded 36GiB to GPU in 2.53398s (14.2069GiB/s) | 20260507 23:31:03.129 | INFO | Downloaded 34GiB to GPU in 2.45657s (13.8404GiB/s) | 20260507 23:31:08.263 | INFO | Analyzing chunk frames = [3460608,3461120) | 20260507 23:31:08.263 | INFO | MemInfo: { | 20260507 23:31:08.263 | INFO | "gpu" : " 17 GB", | 20260507 23:31:08.263 | INFO | "huge" : "", | 20260507 23:31:08.263 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:08.263 | INFO | "malloc" : " 14 GB", | 20260507 23:31:08.263 | INFO | "pinned" : " 29 GB" | 20260507 23:31:08.263 | INFO | } | 20260507 23:31:08.263 | INFO | Perf report for the last 5.13537 seconds: | 20260507 23:31:08.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:08.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.263 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:31:08.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.263 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:31:08.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:08.264 | INFO | Uploaded 36GiB to GPU in 2.52915s (14.234GiB/s) | 20260507 23:31:08.264 | INFO | Downloaded 34GiB to GPU in 2.44265s (13.9193GiB/s) | 20260507 23:31:08.361 | INFO | Starting a round of Baz IO | 20260507 23:31:11.821 | INFO | Populating RTMetrics report | 20260507 23:31:11.821 | INFO | RTMetrics report populated | 20260507 23:31:11.822 | INFO | Opening tmp RTMetrics json file | 20260507 23:31:11.846 | INFO | Serializing RTMetrics to json | 20260507 23:31:11.939 | INFO | Serialization done, swapping tmp file now... | 20260507 23:31:11.942 | INFO | RTMetrics done | 20260507 23:31:11.970 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.609 seconds | 20260507 23:31:11.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.58% / 0.00% / 57.41% | 20260507 23:31:11.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17932% / 3.97364e-06% / 6.35783e-05% | 20260507 23:31:11.970 | INFO | Baz IO overall: | 20260507 23:31:11.970 | INFO | BytesWritten : 1575597080062 | 20260507 23:31:11.970 | INFO | NumEvents : 762978347423 | 20260507 23:31:13.403 | INFO | Analyzing chunk frames = [3461120,3461632) | 20260507 23:31:13.403 | INFO | MemInfo: { | 20260507 23:31:13.403 | INFO | "gpu" : " 17 GB", | 20260507 23:31:13.403 | INFO | "huge" : "", | 20260507 23:31:13.403 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:13.403 | INFO | "malloc" : " 14 GB", | 20260507 23:31:13.403 | INFO | "pinned" : " 29 GB" | 20260507 23:31:13.403 | INFO | } | 20260507 23:31:13.403 | INFO | Perf report for the last 5.1395 seconds: | 20260507 23:31:13.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:13.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:13.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:13.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:13.403 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260507 23:31:13.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:13.403 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 23:31:13.403 | INFO | PRE_HQ : 65.4%, 1.7, 56.9ms, 0 | 20260507 23:31:13.403 | INFO | BAZWRITER : 69.8%, 3.6, 128.4ms, 0 | 20260507 23:31:13.403 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260507 23:31:13.403 | INFO | Uploaded 36GiB to GPU in 2.5321s (14.2175GiB/s) | 20260507 23:31:13.403 | INFO | Downloaded 38GiB to GPU in 2.68487s (14.1534GiB/s) | 20260507 23:31:18.543 | INFO | Analyzing chunk frames = [3461632,3462144) | 20260507 23:31:18.543 | INFO | MemInfo: { | 20260507 23:31:18.543 | INFO | "gpu" : " 17 GB", | 20260507 23:31:18.543 | INFO | "huge" : "", | 20260507 23:31:18.543 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:18.543 | INFO | "malloc" : " 14 GB", | 20260507 23:31:18.543 | INFO | "pinned" : " 29 GB" | 20260507 23:31:18.543 | INFO | } | 20260507 23:31:18.543 | INFO | Perf report for the last 5.1404 seconds: | 20260507 23:31:18.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:18.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.543 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 23:31:18.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.543 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:31:18.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:18.544 | INFO | Uploaded 36GiB to GPU in 2.49962s (14.4022GiB/s) | 20260507 23:31:18.544 | INFO | Downloaded 34GiB to GPU in 2.50111s (13.594GiB/s) | 20260507 23:31:23.685 | INFO | Analyzing chunk frames = [3462144,3462656) | 20260507 23:31:23.685 | INFO | MemInfo: { | 20260507 23:31:23.685 | INFO | "gpu" : " 17 GB", | 20260507 23:31:23.685 | INFO | "huge" : "", | 20260507 23:31:23.685 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:23.685 | INFO | "malloc" : " 14 GB", | 20260507 23:31:23.685 | INFO | "pinned" : " 29 GB" | 20260507 23:31:23.685 | INFO | } | 20260507 23:31:23.685 | INFO | Perf report for the last 5.14177 seconds: | 20260507 23:31:23.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:23.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.685 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:31:23.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.685 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:31:23.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:23.686 | INFO | Uploaded 36GiB to GPU in 2.4469s (14.7125GiB/s) | 20260507 23:31:23.686 | INFO | Downloaded 34GiB to GPU in 2.49488s (13.6279GiB/s) | 20260507 23:31:28.826 | INFO | Analyzing chunk frames = [3462656,3463168) | 20260507 23:31:28.826 | INFO | MemInfo: { | 20260507 23:31:28.826 | INFO | "gpu" : " 17 GB", | 20260507 23:31:28.826 | INFO | "huge" : "", | 20260507 23:31:28.826 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:28.826 | INFO | "malloc" : " 14 GB", | 20260507 23:31:28.826 | INFO | "pinned" : " 29 GB" | 20260507 23:31:28.826 | INFO | } | 20260507 23:31:28.826 | INFO | Perf report for the last 5.14052 seconds: | 20260507 23:31:28.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:28.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260507 23:31:28.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:31:28.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:28.826 | INFO | Uploaded 36GiB to GPU in 2.46253s (14.6191GiB/s) | 20260507 23:31:28.826 | INFO | Downloaded 34GiB to GPU in 2.48079s (13.7053GiB/s) | 20260507 23:31:33.959 | INFO | Analyzing chunk frames = [3463168,3463680) | 20260507 23:31:33.959 | INFO | MemInfo: { | 20260507 23:31:33.959 | INFO | "gpu" : " 17 GB", | 20260507 23:31:33.959 | INFO | "huge" : "", | 20260507 23:31:33.959 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:33.959 | INFO | "malloc" : " 14 GB", | 20260507 23:31:33.959 | INFO | "pinned" : " 29 GB" | 20260507 23:31:33.959 | INFO | } | 20260507 23:31:33.959 | INFO | Perf report for the last 5.13345 seconds: | 20260507 23:31:33.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:33.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.959 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260507 23:31:33.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.959 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:31:33.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:33.960 | INFO | Uploaded 36GiB to GPU in 2.27303s (15.8379GiB/s) | 20260507 23:31:33.960 | INFO | Downloaded 34GiB to GPU in 2.482s (13.6986GiB/s) | 20260507 23:31:39.097 | INFO | Analyzing chunk frames = [3463680,3464192) | 20260507 23:31:39.097 | INFO | MemInfo: { | 20260507 23:31:39.097 | INFO | "gpu" : " 17 GB", | 20260507 23:31:39.097 | INFO | "huge" : "", | 20260507 23:31:39.097 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:39.097 | INFO | "malloc" : " 14 GB", | 20260507 23:31:39.097 | INFO | "pinned" : " 29 GB" | 20260507 23:31:39.097 | INFO | } | 20260507 23:31:39.097 | INFO | Perf report for the last 5.13826 seconds: | 20260507 23:31:39.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:39.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.097 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260507 23:31:39.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.097 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260507 23:31:39.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:39.098 | INFO | Uploaded 36GiB to GPU in 2.49673s (14.4189GiB/s) | 20260507 23:31:39.098 | INFO | Downloaded 34GiB to GPU in 2.4341s (13.9682GiB/s) | 20260507 23:31:44.242 | INFO | Analyzing chunk frames = [3464192,3464704) | 20260507 23:31:44.242 | INFO | MemInfo: { | 20260507 23:31:44.242 | INFO | "gpu" : " 17 GB", | 20260507 23:31:44.242 | INFO | "huge" : "", | 20260507 23:31:44.242 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:44.242 | INFO | "malloc" : " 14 GB", | 20260507 23:31:44.242 | INFO | "pinned" : " 29 GB" | 20260507 23:31:44.242 | INFO | } | 20260507 23:31:44.242 | INFO | Perf report for the last 5.14468 seconds: | 20260507 23:31:44.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:44.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:31:44.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:31:44.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:44.242 | INFO | Uploaded 36GiB to GPU in 2.52211s (14.2737GiB/s) | 20260507 23:31:44.242 | INFO | Downloaded 34GiB to GPU in 2.46154s (13.8125GiB/s) | 20260507 23:31:49.377 | INFO | Analyzing chunk frames = [3464704,3465216) | 20260507 23:31:49.377 | INFO | MemInfo: { | 20260507 23:31:49.377 | INFO | "gpu" : " 17 GB", | 20260507 23:31:49.377 | INFO | "huge" : "", | 20260507 23:31:49.377 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:49.377 | INFO | "malloc" : " 14 GB", | 20260507 23:31:49.377 | INFO | "pinned" : " 29 GB" | 20260507 23:31:49.377 | INFO | } | 20260507 23:31:49.377 | INFO | Perf report for the last 5.135 seconds: | 20260507 23:31:49.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:49.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.377 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:31:49.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.377 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:31:49.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:49.378 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:31:49.378 | INFO | Total mem usage: 16.7 GiB | 20260507 23:31:49.378 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:31:49.378 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.82%) | 20260507 23:31:49.378 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:31:49.378 | INFO | High Water Breakdown: | 20260507 23:31:49.378 | INFO | 60.7% : 9.39 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:31:49.378 | INFO | 39.2% : 6.07 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:31:49.378 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:31:49.378 | INFO | ------------------------------------------ | 20260507 23:31:49.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:31:49.378 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:31:49.378 | INFO | Total mem usage: 26.5 GiB | 20260507 23:31:49.378 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:31:49.378 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 23:31:49.378 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 23:31:49.378 | INFO | High Water Breakdown: | 20260507 23:31:49.378 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:31:49.378 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:31:49.378 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:31:49.378 | INFO | Uploaded 36GiB to GPU in 2.52051s (14.2828GiB/s) | 20260507 23:31:49.378 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:31:49.378 | INFO | Downloaded 34GiB to GPU in 2.44882s (13.8843GiB/s) | 20260507 23:31:49.378 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:31:49.378 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:31:49.378 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:31:49.378 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:31:49.378 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:31:49.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:31:49.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:31:49.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:31:49.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:31:49.378 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:31:49.378 | INFO | ------------------------------------------ | 20260507 23:31:49.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:31:49.378 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:31:49.378 | INFO | Total mem usage: 27.8 GiB | 20260507 23:31:49.378 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:31:49.378 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:31:49.378 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:31:49.378 | INFO | High Water Breakdown: | 20260507 23:31:49.378 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:31:49.378 | INFO | ------------------------------------------ | 20260507 23:31:49.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:31:49.378 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:31:49.378 | INFO | Total mem usage: 15.7 GiB | 20260507 23:31:49.378 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:31:49.378 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:31:49.378 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:31:49.378 | INFO | High Water Breakdown: | 20260507 23:31:49.378 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:31:49.378 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:31:49.378 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:31:49.378 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:31:49.378 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:31:49.378 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:31:49.378 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:31:49.378 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:31:49.378 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:31:49.378 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:31:49.378 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:31:49.378 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:31:49.378 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:31:49.378 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:31:49.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:31:49.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:31:49.378 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:31:49.378 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:31:49.378 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:31:49.378 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:31:49.378 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:31:49.378 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:31:49.378 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:31:49.378 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:31:49.378 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:31:49.378 | INFO | ------------------------------------------ | 20260507 23:31:49.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:31:49.378 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:31:49.378 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:31:49.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:31:49.378 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.41e+01 / 1.55e+00 | 20260507 23:31:49.378 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.28e-02 / 5.67e-03 | 20260507 23:31:49.378 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.46e+00, 5.84e-01 / 1.82e-01 | 20260507 23:31:49.378 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.06e-01 / 9.52e-02 | 20260507 23:31:49.378 | INFO | PerfReport: 16.06%, DME -- 4.21e+01 / 4.60e+01, 4.09e+00 / 1.68e+00 | 20260507 23:31:49.378 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.61e+00 / 1.45e+00 | 20260507 23:31:49.378 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.47e+00 / 2.71e+00, 4.36e-01 / 2.33e-01 | 20260507 23:31:49.378 | INFO | PerfReport: 3.44%, Metrics -- 1.13e+00 / 1.34e+00, 2.80e-01 / 1.60e-01 | 20260507 23:31:49.378 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.47e-01 / 3.77e-01, 5.44e-02 / 2.27e-02 | 20260507 23:31:49.378 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:31:49.378 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:31:49.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:31:49.378 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 30.74%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 2.11%, LockUpload -- 2.25e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 2.78%, LockDownload -- 2.97e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:31:49.379 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 23:31:49.474 | INFO | Starting a round of Baz IO | 20260507 23:31:52.952 | INFO | Populating RTMetrics report | 20260507 23:31:52.952 | INFO | RTMetrics report populated | 20260507 23:31:52.952 | INFO | Opening tmp RTMetrics json file | 20260507 23:31:52.987 | INFO | Serializing RTMetrics to json | 20260507 23:31:53.035 | INFO | Serialization done, swapping tmp file now... | 20260507 23:31:53.036 | INFO | RTMetrics done | 20260507 23:31:53.056 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.581 seconds | 20260507 23:31:53.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.58% / 0.01% / 57.42% | 20260507 23:31:53.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17777% / 3.97364e-06% / 3.97364e-05% | 20260507 23:31:53.056 | INFO | Baz IO overall: | 20260507 23:31:53.056 | INFO | BytesWritten : 1577437387128 | 20260507 23:31:53.056 | INFO | NumEvents : 763865435046 | 20260507 23:31:54.514 | INFO | Analyzing chunk frames = [3465216,3465728) | 20260507 23:31:54.514 | INFO | MemInfo: { | 20260507 23:31:54.514 | INFO | "gpu" : " 17 GB", | 20260507 23:31:54.514 | INFO | "huge" : "", | 20260507 23:31:54.514 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:54.514 | INFO | "malloc" : " 14 GB", | 20260507 23:31:54.514 | INFO | "pinned" : " 29 GB" | 20260507 23:31:54.514 | INFO | } | 20260507 23:31:54.514 | INFO | Perf report for the last 5.13661 seconds: | 20260507 23:31:54.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:54.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:54.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:54.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:54.514 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 23:31:54.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:54.514 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 23:31:54.514 | INFO | PRE_HQ : 65.5%, 1.7, 56.7ms, 0 | 20260507 23:31:54.514 | INFO | BAZWRITER : 69.3%, 3.8, 135.2ms, 0 | 20260507 23:31:54.514 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 23:31:54.514 | INFO | Uploaded 36GiB to GPU in 2.54163s (14.1642GiB/s) | 20260507 23:31:54.514 | INFO | Downloaded 38GiB to GPU in 2.6795s (14.1817GiB/s) | 20260507 23:31:59.654 | INFO | Analyzing chunk frames = [3465728,3466240) | 20260507 23:31:59.654 | INFO | MemInfo: { | 20260507 23:31:59.654 | INFO | "gpu" : " 17 GB", | 20260507 23:31:59.654 | INFO | "huge" : "", | 20260507 23:31:59.654 | INFO | "huge_shared" : " 29 GB", | 20260507 23:31:59.654 | INFO | "malloc" : " 14 GB", | 20260507 23:31:59.654 | INFO | "pinned" : " 29 GB" | 20260507 23:31:59.654 | INFO | } | 20260507 23:31:59.654 | INFO | Perf report for the last 5.14035 seconds: | 20260507 23:31:59.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:31:59.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:31:59.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:31:59.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:31:59.654 | INFO | Uploaded 36GiB to GPU in 2.56839s (14.0166GiB/s) | 20260507 23:31:59.654 | INFO | Downloaded 34GiB to GPU in 2.49236s (13.6417GiB/s) | 20260507 23:32:04.796 | INFO | Analyzing chunk frames = [3466240,3466752) | 20260507 23:32:04.796 | INFO | MemInfo: { | 20260507 23:32:04.796 | INFO | "gpu" : " 17 GB", | 20260507 23:32:04.797 | INFO | "huge" : "", | 20260507 23:32:04.797 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:04.797 | INFO | "malloc" : " 14 GB", | 20260507 23:32:04.797 | INFO | "pinned" : " 29 GB" | 20260507 23:32:04.797 | INFO | } | 20260507 23:32:04.797 | INFO | Perf report for the last 5.14271 seconds: | 20260507 23:32:04.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:04.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 23:32:04.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:32:04.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:04.797 | INFO | Uploaded 36GiB to GPU in 2.56022s (14.0613GiB/s) | 20260507 23:32:04.797 | INFO | Downloaded 34GiB to GPU in 2.47907s (13.7148GiB/s) | 20260507 23:32:09.937 | INFO | Analyzing chunk frames = [3466752,3467264) | 20260507 23:32:09.937 | INFO | MemInfo: { | 20260507 23:32:09.937 | INFO | "gpu" : " 17 GB", | 20260507 23:32:09.937 | INFO | "huge" : "", | 20260507 23:32:09.937 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:09.937 | INFO | "malloc" : " 14 GB", | 20260507 23:32:09.937 | INFO | "pinned" : " 29 GB" | 20260507 23:32:09.937 | INFO | } | 20260507 23:32:09.938 | INFO | Perf report for the last 5.14074 seconds: | 20260507 23:32:09.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:09.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:32:09.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:32:09.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:09.938 | INFO | Uploaded 36GiB to GPU in 2.49152s (14.449GiB/s) | 20260507 23:32:09.938 | INFO | Downloaded 34GiB to GPU in 2.41358s (14.087GiB/s) | 20260507 23:32:15.075 | INFO | Analyzing chunk frames = [3467264,3467776) | 20260507 23:32:15.075 | INFO | MemInfo: { | 20260507 23:32:15.075 | INFO | "gpu" : " 17 GB", | 20260507 23:32:15.075 | INFO | "huge" : "", | 20260507 23:32:15.075 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:15.075 | INFO | "malloc" : " 14 GB", | 20260507 23:32:15.075 | INFO | "pinned" : " 29 GB" | 20260507 23:32:15.075 | INFO | } | 20260507 23:32:15.075 | INFO | Perf report for the last 5.13763 seconds: | 20260507 23:32:15.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:15.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:32:15.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260507 23:32:15.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:15.075 | INFO | Uploaded 36GiB to GPU in 2.31786s (15.5316GiB/s) | 20260507 23:32:15.076 | INFO | Downloaded 34GiB to GPU in 2.49729s (13.6148GiB/s) | 20260507 23:32:20.211 | INFO | Analyzing chunk frames = [3467776,3468288) | 20260507 23:32:20.211 | INFO | MemInfo: { | 20260507 23:32:20.211 | INFO | "gpu" : " 17 GB", | 20260507 23:32:20.211 | INFO | "huge" : "", | 20260507 23:32:20.211 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:20.211 | INFO | "malloc" : " 14 GB", | 20260507 23:32:20.211 | INFO | "pinned" : " 29 GB" | 20260507 23:32:20.211 | INFO | } | 20260507 23:32:20.211 | INFO | Perf report for the last 5.13566 seconds: | 20260507 23:32:20.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:20.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 23:32:20.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:32:20.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:20.211 | INFO | Uploaded 36GiB to GPU in 2.39794s (15.0129GiB/s) | 20260507 23:32:20.211 | INFO | Downloaded 34GiB to GPU in 2.40182s (14.156GiB/s) | 20260507 23:32:25.356 | INFO | Analyzing chunk frames = [3468288,3468800) | 20260507 23:32:25.356 | INFO | MemInfo: { | 20260507 23:32:25.356 | INFO | "gpu" : " 17 GB", | 20260507 23:32:25.356 | INFO | "huge" : "", | 20260507 23:32:25.356 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:25.356 | INFO | "malloc" : " 14 GB", | 20260507 23:32:25.356 | INFO | "pinned" : " 29 GB" | 20260507 23:32:25.356 | INFO | } | 20260507 23:32:25.356 | INFO | Perf report for the last 5.14556 seconds: | 20260507 23:32:25.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:25.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.357 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260507 23:32:25.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.357 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 23:32:25.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:25.357 | INFO | Uploaded 36GiB to GPU in 2.51651s (14.3055GiB/s) | 20260507 23:32:25.357 | INFO | Downloaded 34GiB to GPU in 2.46192s (13.8103GiB/s) | 20260507 23:32:30.492 | INFO | Analyzing chunk frames = [3468800,3469312) | 20260507 23:32:30.492 | INFO | MemInfo: { | 20260507 23:32:30.492 | INFO | "gpu" : " 17 GB", | 20260507 23:32:30.492 | INFO | "huge" : "", | 20260507 23:32:30.492 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:30.492 | INFO | "malloc" : " 14 GB", | 20260507 23:32:30.492 | INFO | "pinned" : " 29 GB" | 20260507 23:32:30.492 | INFO | } | 20260507 23:32:30.493 | INFO | Perf report for the last 5.13612 seconds: | 20260507 23:32:30.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:30.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:32:30.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:32:30.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:30.493 | INFO | Uploaded 36GiB to GPU in 2.53107s (14.2233GiB/s) | 20260507 23:32:30.493 | INFO | Downloaded 34GiB to GPU in 2.45111s (13.8712GiB/s) | 20260507 23:32:30.590 | INFO | Starting a round of Baz IO | 20260507 23:32:34.045 | INFO | Populating RTMetrics report | 20260507 23:32:34.046 | INFO | RTMetrics report populated | 20260507 23:32:34.046 | INFO | Opening tmp RTMetrics json file | 20260507 23:32:34.049 | INFO | Serializing RTMetrics to json | 20260507 23:32:34.074 | INFO | Serialization done, swapping tmp file now... | 20260507 23:32:34.075 | INFO | RTMetrics done | 20260507 23:32:34.156 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.565 seconds | 20260507 23:32:34.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.57% / 0.01% / 57.42% | 20260507 23:32:34.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17491% / 3.97364e-06% / 3.57628e-05% | 20260507 23:32:34.156 | INFO | Baz IO overall: | 20260507 23:32:34.156 | INFO | BytesWritten : 1579277476752 | 20260507 23:32:34.156 | INFO | NumEvents : 764752102026 | 20260507 23:32:35.633 | INFO | Analyzing chunk frames = [3469312,3469824) | 20260507 23:32:35.633 | INFO | MemInfo: { | 20260507 23:32:35.633 | INFO | "gpu" : " 17 GB", | 20260507 23:32:35.633 | INFO | "huge" : "", | 20260507 23:32:35.633 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:35.633 | INFO | "malloc" : " 14 GB", | 20260507 23:32:35.633 | INFO | "pinned" : " 29 GB" | 20260507 23:32:35.633 | INFO | } | 20260507 23:32:35.633 | INFO | Perf report for the last 5.14082 seconds: | 20260507 23:32:35.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:35.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:35.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:35.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:35.633 | INFO | ANALYSIS : 68.7%, 3.0, 106.5ms, 0 | 20260507 23:32:35.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:35.633 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 23:32:35.633 | INFO | PRE_HQ : 65.6%, 1.7, 57.1ms, 0 | 20260507 23:32:35.633 | INFO | BAZWRITER : 69.0%, 3.9, 137.9ms, 0 | 20260507 23:32:35.633 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 23:32:35.634 | INFO | Uploaded 36GiB to GPU in 2.55838s (14.0714GiB/s) | 20260507 23:32:35.634 | INFO | Downloaded 38GiB to GPU in 2.69616s (14.0941GiB/s) | 20260507 23:32:40.772 | INFO | Analyzing chunk frames = [3469824,3470336) | 20260507 23:32:40.772 | INFO | MemInfo: { | 20260507 23:32:40.772 | INFO | "gpu" : " 17 GB", | 20260507 23:32:40.772 | INFO | "huge" : "", | 20260507 23:32:40.772 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:40.772 | INFO | "malloc" : " 14 GB", | 20260507 23:32:40.772 | INFO | "pinned" : " 29 GB" | 20260507 23:32:40.772 | INFO | } | 20260507 23:32:40.773 | INFO | Perf report for the last 5.13916 seconds: | 20260507 23:32:40.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:40.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260507 23:32:40.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:32:40.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:40.773 | INFO | Uploaded 36GiB to GPU in 2.59455s (13.8753GiB/s) | 20260507 23:32:40.773 | INFO | Downloaded 34GiB to GPU in 2.47799s (13.7208GiB/s) | 20260507 23:32:45.906 | INFO | Analyzing chunk frames = [3470336,3470848) | 20260507 23:32:45.906 | INFO | MemInfo: { | 20260507 23:32:45.906 | INFO | "gpu" : " 17 GB", | 20260507 23:32:45.906 | INFO | "huge" : "", | 20260507 23:32:45.906 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:45.906 | INFO | "malloc" : " 14 GB", | 20260507 23:32:45.906 | INFO | "pinned" : " 29 GB" | 20260507 23:32:45.906 | INFO | } | 20260507 23:32:45.907 | INFO | Perf report for the last 5.13396 seconds: | 20260507 23:32:45.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:45.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 23:32:45.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:32:45.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:45.907 | INFO | Uploaded 36GiB to GPU in 2.60286s (13.8309GiB/s) | 20260507 23:32:45.907 | INFO | Downloaded 34GiB to GPU in 2.46756s (13.7788GiB/s) | 20260507 23:32:51.049 | INFO | Analyzing chunk frames = [3470848,3471360) | 20260507 23:32:51.049 | INFO | MemInfo: { | 20260507 23:32:51.049 | INFO | "gpu" : " 17 GB", | 20260507 23:32:51.049 | INFO | "huge" : "", | 20260507 23:32:51.049 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:51.049 | INFO | "malloc" : " 14 GB", | 20260507 23:32:51.049 | INFO | "pinned" : " 29 GB" | 20260507 23:32:51.049 | INFO | } | 20260507 23:32:51.050 | INFO | Perf report for the last 5.14301 seconds: | 20260507 23:32:51.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:51.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260507 23:32:51.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:32:51.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:51.050 | INFO | Uploaded 36GiB to GPU in 2.49551s (14.4259GiB/s) | 20260507 23:32:51.050 | INFO | Downloaded 34GiB to GPU in 2.37453s (14.3186GiB/s) | 20260507 23:32:56.190 | INFO | Analyzing chunk frames = [3471360,3471872) | 20260507 23:32:56.190 | INFO | MemInfo: { | 20260507 23:32:56.190 | INFO | "gpu" : " 17 GB", | 20260507 23:32:56.190 | INFO | "huge" : "", | 20260507 23:32:56.190 | INFO | "huge_shared" : " 29 GB", | 20260507 23:32:56.190 | INFO | "malloc" : " 14 GB", | 20260507 23:32:56.190 | INFO | "pinned" : " 29 GB" | 20260507 23:32:56.190 | INFO | } | 20260507 23:32:56.190 | INFO | Perf report for the last 5.14057 seconds: | 20260507 23:32:56.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:32:56.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:32:56.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:32:56.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:32:56.190 | INFO | Uploaded 36GiB to GPU in 2.44962s (14.6962GiB/s) | 20260507 23:32:56.190 | INFO | Downloaded 34GiB to GPU in 2.4982s (13.6098GiB/s) | 20260507 23:33:01.326 | INFO | Analyzing chunk frames = [3471872,3472384) | 20260507 23:33:01.326 | INFO | MemInfo: { | 20260507 23:33:01.326 | INFO | "gpu" : " 17 GB", | 20260507 23:33:01.326 | INFO | "huge" : "", | 20260507 23:33:01.326 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:01.326 | INFO | "malloc" : " 14 GB", | 20260507 23:33:01.326 | INFO | "pinned" : " 29 GB" | 20260507 23:33:01.326 | INFO | } | 20260507 23:33:01.327 | INFO | Perf report for the last 5.13636 seconds: | 20260507 23:33:01.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:01.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:33:01.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:33:01.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:01.327 | INFO | Uploaded 36GiB to GPU in 2.40344s (14.9786GiB/s) | 20260507 23:33:01.327 | INFO | Downloaded 34GiB to GPU in 2.3991s (14.172GiB/s) | 20260507 23:33:06.464 | INFO | Analyzing chunk frames = [3472384,3472896) | 20260507 23:33:06.464 | INFO | MemInfo: { | 20260507 23:33:06.464 | INFO | "gpu" : " 17 GB", | 20260507 23:33:06.464 | INFO | "huge" : "", | 20260507 23:33:06.464 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:06.464 | INFO | "malloc" : " 14 GB", | 20260507 23:33:06.464 | INFO | "pinned" : " 29 GB" | 20260507 23:33:06.464 | INFO | } | 20260507 23:33:06.464 | INFO | Perf report for the last 5.13776 seconds: | 20260507 23:33:06.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:06.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.464 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:33:06.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.464 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:33:06.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:06.465 | INFO | Uploaded 36GiB to GPU in 2.52295s (14.269GiB/s) | 20260507 23:33:06.465 | INFO | Downloaded 34GiB to GPU in 2.44769s (13.8907GiB/s) | 20260507 23:33:11.604 | INFO | Analyzing chunk frames = [3472896,3473408) | 20260507 23:33:11.604 | INFO | MemInfo: { | 20260507 23:33:11.604 | INFO | "gpu" : " 17 GB", | 20260507 23:33:11.604 | INFO | "huge" : "", | 20260507 23:33:11.604 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:11.604 | INFO | "malloc" : " 14 GB", | 20260507 23:33:11.604 | INFO | "pinned" : " 29 GB" | 20260507 23:33:11.604 | INFO | } | 20260507 23:33:11.604 | INFO | Perf report for the last 5.13968 seconds: | 20260507 23:33:11.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:11.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:33:11.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:33:11.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:11.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:33:11.604 | INFO | Total mem usage: 16.7 GiB | 20260507 23:33:11.604 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:33:11.604 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.5%) | 20260507 23:33:11.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:33:11.604 | INFO | High Water Breakdown: | 20260507 23:33:11.604 | INFO | 59.8% : 8.79 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:33:11.604 | INFO | 40% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:33:11.604 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:33:11.604 | INFO | ------------------------------------------ | 20260507 23:33:11.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:33:11.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:33:11.604 | INFO | Total mem usage: 26.5 GiB | 20260507 23:33:11.604 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:33:11.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:33:11.604 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:33:11.604 | INFO | High Water Breakdown: | 20260507 23:33:11.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:33:11.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:33:11.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:33:11.604 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:33:11.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:33:11.604 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:33:11.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:33:11.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:33:11.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:33:11.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:33:11.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:33:11.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:33:11.605 | INFO | Uploaded 36GiB to GPU in 2.48303s (14.4984GiB/s) | 20260507 23:33:11.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:33:11.605 | INFO | Downloaded 34GiB to GPU in 2.44011s (13.9338GiB/s) | 20260507 23:33:11.605 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:33:11.605 | INFO | ------------------------------------------ | 20260507 23:33:11.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:33:11.605 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:33:11.605 | INFO | Total mem usage: 27.8 GiB | 20260507 23:33:11.605 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:33:11.605 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:33:11.605 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:33:11.605 | INFO | High Water Breakdown: | 20260507 23:33:11.605 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:33:11.605 | INFO | ------------------------------------------ | 20260507 23:33:11.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:33:11.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:33:11.605 | INFO | Total mem usage: 15.7 GiB | 20260507 23:33:11.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:33:11.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:33:11.605 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:33:11.605 | INFO | High Water Breakdown: | 20260507 23:33:11.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:33:11.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:33:11.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:33:11.605 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:33:11.605 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:33:11.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:33:11.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:33:11.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:33:11.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:33:11.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:33:11.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:33:11.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:33:11.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:33:11.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:33:11.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:33:11.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:33:11.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:33:11.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:33:11.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:33:11.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:33:11.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:33:11.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:33:11.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:33:11.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:33:11.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:33:11.605 | INFO | ------------------------------------------ | 20260507 23:33:11.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:33:11.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:33:11.605 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:33:11.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:33:11.605 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.40e+01 / 1.40e+00 | 20260507 23:33:11.605 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.09e-02 / 5.43e-03 | 20260507 23:33:11.605 | INFO | PerfReport: 12.17%, Baseline -- 4.01e+00 / 4.48e+00, 5.97e-01 / 2.36e-01 | 20260507 23:33:11.605 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.07e-01 / 1.13e-01 | 20260507 23:33:11.605 | INFO | PerfReport: 15.77%, DME -- 4.16e+01 / 4.50e+01, 3.90e+00 / 1.13e+00 | 20260507 23:33:11.605 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.87e+00 / 1.87e+00 | 20260507 23:33:11.605 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.41e+00 / 2.72e+00, 4.24e-01 / 2.37e-01 | 20260507 23:33:11.605 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.36e+00, 2.82e-01 / 1.71e-01 | 20260507 23:33:11.605 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.47e-01 / 3.77e-01, 5.57e-02 / 2.58e-02 | 20260507 23:33:11.606 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:33:11.606 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:33:11.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:33:11.606 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 24.22%, Download -- 2.58e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 30.90%, Compute -- 3.30e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 2.05%, LockUpload -- 2.18e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 2.38%, LockDownload -- 2.54e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 16.00%, LockCompute -- 1.71e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260507 23:33:11.606 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 23:33:11.702 | INFO | Starting a round of Baz IO | 20260507 23:33:15.178 | INFO | Populating RTMetrics report | 20260507 23:33:15.178 | INFO | RTMetrics report populated | 20260507 23:33:15.178 | INFO | Opening tmp RTMetrics json file | 20260507 23:33:15.215 | INFO | Serializing RTMetrics to json | 20260507 23:33:15.264 | INFO | Serialization done, swapping tmp file now... | 20260507 23:33:15.266 | INFO | RTMetrics done | 20260507 23:33:15.289 | INFO | Finished a round of Baz IO: Wrote 1.713 GiB in 3.587 seconds | 20260507 23:33:15.289 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.57% / 0.00% / 57.43% | 20260507 23:33:15.289 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17188% / 7.94729e-06% / 4.76837e-05% | 20260507 23:33:15.289 | INFO | Baz IO overall: | 20260507 23:33:15.289 | INFO | BytesWritten : 1581116555587 | 20260507 23:33:15.289 | INFO | NumEvents : 765638161663 | 20260507 23:33:16.745 | INFO | Analyzing chunk frames = [3473408,3473920) | 20260507 23:33:16.745 | INFO | MemInfo: { | 20260507 23:33:16.745 | INFO | "gpu" : " 17 GB", | 20260507 23:33:16.745 | INFO | "huge" : "", | 20260507 23:33:16.745 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:16.745 | INFO | "malloc" : " 14 GB", | 20260507 23:33:16.745 | INFO | "pinned" : " 29 GB" | 20260507 23:33:16.745 | INFO | } | 20260507 23:33:16.745 | INFO | Perf report for the last 5.14129 seconds: | 20260507 23:33:16.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:16.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:16.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:16.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:16.745 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 23:33:16.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:16.745 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 23:33:16.745 | INFO | PRE_HQ : 65.8%, 1.7, 56.9ms, 0 | 20260507 23:33:16.745 | INFO | BAZWRITER : 69.4%, 3.7, 130.3ms, 0 | 20260507 23:33:16.745 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 23:33:16.746 | INFO | Uploaded 36GiB to GPU in 2.54984s (14.1185GiB/s) | 20260507 23:33:16.746 | INFO | Downloaded 38GiB to GPU in 2.70124s (14.0676GiB/s) | 20260507 23:33:21.890 | INFO | Analyzing chunk frames = [3473920,3474432) | 20260507 23:33:21.890 | INFO | MemInfo: { | 20260507 23:33:21.890 | INFO | "gpu" : " 17 GB", | 20260507 23:33:21.890 | INFO | "huge" : "", | 20260507 23:33:21.890 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:21.890 | INFO | "malloc" : " 14 GB", | 20260507 23:33:21.890 | INFO | "pinned" : " 29 GB" | 20260507 23:33:21.890 | INFO | } | 20260507 23:33:21.890 | INFO | Perf report for the last 5.14497 seconds: | 20260507 23:33:21.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:21.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:33:21.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260507 23:33:21.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:21.890 | INFO | Uploaded 36GiB to GPU in 2.52604s (14.2516GiB/s) | 20260507 23:33:21.890 | INFO | Downloaded 34GiB to GPU in 2.49675s (13.6177GiB/s) | 20260507 23:33:27.028 | INFO | Analyzing chunk frames = [3474432,3474944) | 20260507 23:33:27.028 | INFO | MemInfo: { | 20260507 23:33:27.028 | INFO | "gpu" : " 17 GB", | 20260507 23:33:27.028 | INFO | "huge" : "", | 20260507 23:33:27.028 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:27.028 | INFO | "malloc" : " 14 GB", | 20260507 23:33:27.028 | INFO | "pinned" : " 29 GB" | 20260507 23:33:27.028 | INFO | } | 20260507 23:33:27.028 | INFO | Perf report for the last 5.13809 seconds: | 20260507 23:33:27.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:27.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.028 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:33:27.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.028 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:33:27.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:27.029 | INFO | Uploaded 36GiB to GPU in 2.52921s (14.2337GiB/s) | 20260507 23:33:27.029 | INFO | Downloaded 34GiB to GPU in 2.48304s (13.6929GiB/s) | 20260507 23:33:32.162 | INFO | Analyzing chunk frames = [3474944,3475456) | 20260507 23:33:32.162 | INFO | MemInfo: { | 20260507 23:33:32.162 | INFO | "gpu" : " 17 GB", | 20260507 23:33:32.162 | INFO | "huge" : "", | 20260507 23:33:32.162 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:32.163 | INFO | "malloc" : " 14 GB", | 20260507 23:33:32.163 | INFO | "pinned" : " 29 GB" | 20260507 23:33:32.163 | INFO | } | 20260507 23:33:32.163 | INFO | Perf report for the last 5.1342 seconds: | 20260507 23:33:32.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:32.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260507 23:33:32.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:33:32.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:32.163 | INFO | Uploaded 36GiB to GPU in 2.4618s (14.6235GiB/s) | 20260507 23:33:32.163 | INFO | Downloaded 34GiB to GPU in 2.50137s (13.5925GiB/s) | 20260507 23:33:37.307 | INFO | Analyzing chunk frames = [3475456,3475968) | 20260507 23:33:37.307 | INFO | MemInfo: { | 20260507 23:33:37.307 | INFO | "gpu" : " 17 GB", | 20260507 23:33:37.307 | INFO | "huge" : "", | 20260507 23:33:37.307 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:37.307 | INFO | "malloc" : " 14 GB", | 20260507 23:33:37.307 | INFO | "pinned" : " 29 GB" | 20260507 23:33:37.307 | INFO | } | 20260507 23:33:37.307 | INFO | Perf report for the last 5.14425 seconds: | 20260507 23:33:37.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:37.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 23:33:37.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:33:37.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:37.307 | INFO | Uploaded 36GiB to GPU in 2.33138s (15.4415GiB/s) | 20260507 23:33:37.307 | INFO | Downloaded 34GiB to GPU in 2.51179s (13.5361GiB/s) | 20260507 23:33:42.444 | INFO | Analyzing chunk frames = [3475968,3476480) | 20260507 23:33:42.444 | INFO | MemInfo: { | 20260507 23:33:42.444 | INFO | "gpu" : " 17 GB", | 20260507 23:33:42.444 | INFO | "huge" : "", | 20260507 23:33:42.444 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:42.444 | INFO | "malloc" : " 14 GB", | 20260507 23:33:42.444 | INFO | "pinned" : " 29 GB" | 20260507 23:33:42.444 | INFO | } | 20260507 23:33:42.444 | INFO | Perf report for the last 5.1376 seconds: | 20260507 23:33:42.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:42.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.445 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:33:42.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.445 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:33:42.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:42.445 | INFO | Uploaded 36GiB to GPU in 2.36533s (15.2198GiB/s) | 20260507 23:33:42.445 | INFO | Downloaded 34GiB to GPU in 2.43035s (13.9897GiB/s) | 20260507 23:33:47.582 | INFO | Analyzing chunk frames = [3476480,3476992) | 20260507 23:33:47.582 | INFO | MemInfo: { | 20260507 23:33:47.582 | INFO | "gpu" : " 17 GB", | 20260507 23:33:47.582 | INFO | "huge" : "", | 20260507 23:33:47.582 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:47.582 | INFO | "malloc" : " 14 GB", | 20260507 23:33:47.582 | INFO | "pinned" : " 29 GB" | 20260507 23:33:47.582 | INFO | } | 20260507 23:33:47.582 | INFO | Perf report for the last 5.13737 seconds: | 20260507 23:33:47.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:47.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 23:33:47.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:33:47.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:47.582 | INFO | Uploaded 36GiB to GPU in 2.50661s (14.362GiB/s) | 20260507 23:33:47.583 | INFO | Downloaded 34GiB to GPU in 2.47784s (13.7216GiB/s) | 20260507 23:33:52.718 | INFO | Analyzing chunk frames = [3476992,3477504) | 20260507 23:33:52.718 | INFO | MemInfo: { | 20260507 23:33:52.718 | INFO | "gpu" : " 17 GB", | 20260507 23:33:52.718 | INFO | "huge" : "", | 20260507 23:33:52.718 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:52.718 | INFO | "malloc" : " 14 GB", | 20260507 23:33:52.718 | INFO | "pinned" : " 29 GB" | 20260507 23:33:52.718 | INFO | } | 20260507 23:33:52.718 | INFO | Perf report for the last 5.13638 seconds: | 20260507 23:33:52.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:52.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.718 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260507 23:33:52.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.719 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:33:52.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:52.719 | INFO | Uploaded 36GiB to GPU in 2.45851s (14.643GiB/s) | 20260507 23:33:52.719 | INFO | Downloaded 34GiB to GPU in 2.45406s (13.8546GiB/s) | 20260507 23:33:52.830 | INFO | Starting a round of Baz IO | 20260507 23:33:56.306 | INFO | Populating RTMetrics report | 20260507 23:33:56.306 | INFO | RTMetrics report populated | 20260507 23:33:56.306 | INFO | Opening tmp RTMetrics json file | 20260507 23:33:56.492 | INFO | Serializing RTMetrics to json | 20260507 23:33:56.565 | INFO | Serialization done, swapping tmp file now... | 20260507 23:33:56.566 | INFO | RTMetrics done | 20260507 23:33:56.585 | INFO | Finished a round of Baz IO: Wrote 1.713 GiB in 3.755 seconds | 20260507 23:33:56.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.56% / 0.00% / 57.43% | 20260507 23:33:56.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17256% / 7.94729e-06% / 5.5631e-05% | 20260507 23:33:56.585 | INFO | Baz IO overall: | 20260507 23:33:56.585 | INFO | BytesWritten : 1582956213566 | 20260507 23:33:56.585 | INFO | NumEvents : 766525110353 | 20260507 23:33:57.861 | INFO | Analyzing chunk frames = [3477504,3478016) | 20260507 23:33:57.861 | INFO | MemInfo: { | 20260507 23:33:57.861 | INFO | "gpu" : " 17 GB", | 20260507 23:33:57.861 | INFO | "huge" : "", | 20260507 23:33:57.861 | INFO | "huge_shared" : " 29 GB", | 20260507 23:33:57.862 | INFO | "malloc" : " 14 GB", | 20260507 23:33:57.862 | INFO | "pinned" : " 29 GB" | 20260507 23:33:57.862 | INFO | } | 20260507 23:33:57.862 | INFO | Perf report for the last 5.14343 seconds: | 20260507 23:33:57.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:33:57.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:57.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:57.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:57.862 | INFO | ANALYSIS : 69.0%, 2.9, 104.8ms, 0 | 20260507 23:33:57.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:33:57.862 | INFO | RT_METRICS : 49.1%, 1.0, 25.7ms, 0 | 20260507 23:33:57.862 | INFO | PRE_HQ : 67.8%, 1.8, 62.8ms, 0 | 20260507 23:33:57.862 | INFO | BAZWRITER : 72.6%, 4.0, 150.0ms, 0 | 20260507 23:33:57.862 | INFO | BAZ_SAMPLING : 5.0%, 1.0, 2.6ms, 0 | 20260507 23:33:57.862 | INFO | Uploaded 36GiB to GPU in 2.5618s (14.0526GiB/s) | 20260507 23:33:57.862 | INFO | Downloaded 38GiB to GPU in 2.69537s (14.0982GiB/s) | 20260507 23:34:03.003 | INFO | Analyzing chunk frames = [3478016,3478528) | 20260507 23:34:03.003 | INFO | MemInfo: { | 20260507 23:34:03.003 | INFO | "gpu" : " 17 GB", | 20260507 23:34:03.003 | INFO | "huge" : "", | 20260507 23:34:03.003 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:03.003 | INFO | "malloc" : " 14 GB", | 20260507 23:34:03.003 | INFO | "pinned" : " 29 GB" | 20260507 23:34:03.003 | INFO | } | 20260507 23:34:03.003 | INFO | Perf report for the last 5.14156 seconds: | 20260507 23:34:03.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:03.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.003 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:34:03.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.003 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:34:03.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:03.004 | INFO | Uploaded 36GiB to GPU in 2.55035s (14.1157GiB/s) | 20260507 23:34:03.004 | INFO | Downloaded 34GiB to GPU in 2.48435s (13.6857GiB/s) | 20260507 23:34:08.144 | INFO | Analyzing chunk frames = [3478528,3479040) | 20260507 23:34:08.144 | INFO | MemInfo: { | 20260507 23:34:08.144 | INFO | "gpu" : " 17 GB", | 20260507 23:34:08.144 | INFO | "huge" : "", | 20260507 23:34:08.144 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:08.144 | INFO | "malloc" : " 14 GB", | 20260507 23:34:08.144 | INFO | "pinned" : " 29 GB" | 20260507 23:34:08.144 | INFO | } | 20260507 23:34:08.144 | INFO | Perf report for the last 5.14115 seconds: | 20260507 23:34:08.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:08.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.144 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 23:34:08.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.145 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:34:08.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:08.145 | INFO | Uploaded 36GiB to GPU in 2.47043s (14.5723GiB/s) | 20260507 23:34:08.145 | INFO | Downloaded 34GiB to GPU in 2.48067s (13.706GiB/s) | 20260507 23:34:13.276 | INFO | Analyzing chunk frames = [3479040,3479552) | 20260507 23:34:13.276 | INFO | MemInfo: { | 20260507 23:34:13.276 | INFO | "gpu" : " 17 GB", | 20260507 23:34:13.276 | INFO | "huge" : "", | 20260507 23:34:13.276 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:13.276 | INFO | "malloc" : " 14 GB", | 20260507 23:34:13.276 | INFO | "pinned" : " 29 GB" | 20260507 23:34:13.276 | INFO | } | 20260507 23:34:13.276 | INFO | Perf report for the last 5.1318 seconds: | 20260507 23:34:13.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:13.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 23:34:13.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:34:13.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:13.276 | INFO | Uploaded 36GiB to GPU in 2.48405s (14.4925GiB/s) | 20260507 23:34:13.276 | INFO | Downloaded 34GiB to GPU in 2.49688s (13.617GiB/s) | 20260507 23:34:18.417 | INFO | Analyzing chunk frames = [3479552,3480064) | 20260507 23:34:18.417 | INFO | MemInfo: { | 20260507 23:34:18.417 | INFO | "gpu" : " 17 GB", | 20260507 23:34:18.417 | INFO | "huge" : "", | 20260507 23:34:18.417 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:18.417 | INFO | "malloc" : " 14 GB", | 20260507 23:34:18.417 | INFO | "pinned" : " 29 GB" | 20260507 23:34:18.417 | INFO | } | 20260507 23:34:18.417 | INFO | Perf report for the last 5.14075 seconds: | 20260507 23:34:18.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:18.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 23:34:18.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:34:18.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:18.417 | INFO | Uploaded 36GiB to GPU in 2.30343s (15.6289GiB/s) | 20260507 23:34:18.417 | INFO | Downloaded 34GiB to GPU in 2.46831s (13.7746GiB/s) | 20260507 23:34:23.559 | INFO | Analyzing chunk frames = [3480064,3480576) | 20260507 23:34:23.559 | INFO | MemInfo: { | 20260507 23:34:23.559 | INFO | "gpu" : " 17 GB", | 20260507 23:34:23.559 | INFO | "huge" : "", | 20260507 23:34:23.559 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:23.559 | INFO | "malloc" : " 14 GB", | 20260507 23:34:23.559 | INFO | "pinned" : " 29 GB" | 20260507 23:34:23.559 | INFO | } | 20260507 23:34:23.560 | INFO | Perf report for the last 5.14255 seconds: | 20260507 23:34:23.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:23.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 23:34:23.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:34:23.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:23.560 | INFO | Uploaded 36GiB to GPU in 2.53947s (14.1762GiB/s) | 20260507 23:34:23.560 | INFO | Downloaded 34GiB to GPU in 2.43029s (13.9901GiB/s) | 20260507 23:34:28.699 | INFO | Analyzing chunk frames = [3480576,3481088) | 20260507 23:34:28.699 | INFO | MemInfo: { | 20260507 23:34:28.699 | INFO | "gpu" : " 17 GB", | 20260507 23:34:28.699 | INFO | "huge" : "", | 20260507 23:34:28.700 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:28.700 | INFO | "malloc" : " 14 GB", | 20260507 23:34:28.700 | INFO | "pinned" : " 29 GB" | 20260507 23:34:28.700 | INFO | } | 20260507 23:34:28.700 | INFO | Perf report for the last 5.14021 seconds: | 20260507 23:34:28.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:28.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 23:34:28.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:34:28.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:28.700 | INFO | Uploaded 36GiB to GPU in 2.54243s (14.1597GiB/s) | 20260507 23:34:28.700 | INFO | Downloaded 34GiB to GPU in 2.4476s (13.8912GiB/s) | 20260507 23:34:33.837 | INFO | Analyzing chunk frames = [3481088,3481600) | 20260507 23:34:33.837 | INFO | MemInfo: { | 20260507 23:34:33.837 | INFO | "gpu" : " 17 GB", | 20260507 23:34:33.837 | INFO | "huge" : "", | 20260507 23:34:33.837 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:33.837 | INFO | "malloc" : " 14 GB", | 20260507 23:34:33.838 | INFO | "pinned" : " 29 GB" | 20260507 23:34:33.838 | INFO | } | 20260507 23:34:33.838 | INFO | Perf report for the last 5.13794 seconds: | 20260507 23:34:33.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:33.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 23:34:33.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:34:33.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:33.838 | INFO | Uploaded 36GiB to GPU in 2.5245s (14.2603GiB/s) | 20260507 23:34:33.838 | INFO | Downloaded 34GiB to GPU in 2.44219s (13.9219GiB/s) | 20260507 23:34:33.838 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:34:33.838 | INFO | Total mem usage: 16.7 GiB | 20260507 23:34:33.838 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:34:33.838 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.2%) | 20260507 23:34:33.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:34:33.838 | INFO | High Water Breakdown: | 20260507 23:34:33.838 | INFO | 60.3% : 9.13 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:34:33.838 | INFO | 39.6% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:34:33.838 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:34:33.838 | INFO | ------------------------------------------ | 20260507 23:34:33.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:34:33.838 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:34:33.838 | INFO | Total mem usage: 26.5 GiB | 20260507 23:34:33.838 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:34:33.838 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:34:33.838 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:34:33.838 | INFO | High Water Breakdown: | 20260507 23:34:33.838 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:34:33.838 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:34:33.838 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:34:33.838 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:34:33.839 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:34:33.839 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:34:33.839 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:34:33.839 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:34:33.839 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:34:33.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:34:33.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:34:33.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:34:33.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:34:33.839 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:34:33.839 | INFO | ------------------------------------------ | 20260507 23:34:33.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:34:33.839 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:34:33.839 | INFO | Total mem usage: 27.8 GiB | 20260507 23:34:33.839 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:34:33.839 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:34:33.839 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:34:33.839 | INFO | High Water Breakdown: | 20260507 23:34:33.839 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:34:33.839 | INFO | ------------------------------------------ | 20260507 23:34:33.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:34:33.839 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:34:33.839 | INFO | Total mem usage: 15.7 GiB | 20260507 23:34:33.839 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:34:33.839 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:34:33.839 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:34:33.839 | INFO | High Water Breakdown: | 20260507 23:34:33.839 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:34:33.839 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:34:33.839 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:34:33.839 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:34:33.839 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:34:33.839 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:34:33.839 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:34:33.839 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:34:33.839 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:34:33.839 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:34:33.839 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:34:33.839 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:34:33.839 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:34:33.839 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:34:33.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:34:33.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:34:33.839 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:34:33.839 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:34:33.839 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:34:33.839 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:34:33.839 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:34:33.839 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:34:33.839 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:34:33.839 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:34:33.839 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:34:33.839 | INFO | ------------------------------------------ | 20260507 23:34:33.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:34:33.839 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:34:33.839 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:34:33.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:34:33.839 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.66e+01, 1.42e+01 / 1.68e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.24e-02 / 8.15e-03 | 20260507 23:34:33.839 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.47e+00, 5.91e-01 / 2.20e-01 | 20260507 23:34:33.839 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.05e-01 | 20260507 23:34:33.839 | INFO | PerfReport: 16.08%, DME -- 4.22e+01 / 4.63e+01, 4.06e+00 / 1.81e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.62e+00 / 1.80e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 2.69e+00, 4.32e-01 / 2.65e-01 | 20260507 23:34:33.839 | INFO | PerfReport: 3.43%, Metrics -- 1.12e+00 / 1.32e+00, 2.80e-01 / 1.60e-01 | 20260507 23:34:33.839 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.46e-01 / 3.78e-01, 5.52e-02 / 2.82e-02 | 20260507 23:34:33.839 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:34:33.839 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:34:33.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:34:33.839 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 24.25%, Upload -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 30.79%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 1.99%, LockUpload -- 2.12e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 2.93%, LockDownload -- 3.12e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:34:33.839 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260507 23:34:33.950 | INFO | Starting a round of Baz IO | 20260507 23:34:37.413 | INFO | Populating RTMetrics report | 20260507 23:34:37.413 | INFO | RTMetrics report populated | 20260507 23:34:37.413 | INFO | Opening tmp RTMetrics json file | 20260507 23:34:37.442 | INFO | Serializing RTMetrics to json | 20260507 23:34:37.445 | INFO | Serialization done, swapping tmp file now... | 20260507 23:34:37.446 | INFO | RTMetrics done | 20260507 23:34:37.510 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.559 seconds | 20260507 23:34:37.510 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.56% / 0.01% / 57.44% | 20260507 23:34:37.510 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.16656% / 3.97364e-06% / 5.16574e-05% | 20260507 23:34:37.510 | INFO | Baz IO overall: | 20260507 23:34:37.510 | INFO | BytesWritten : 1584793570291 | 20260507 23:34:37.510 | INFO | NumEvents : 767410183434 | 20260507 23:34:38.977 | INFO | Analyzing chunk frames = [3481600,3482112) | 20260507 23:34:38.977 | INFO | MemInfo: { | 20260507 23:34:38.977 | INFO | "gpu" : " 17 GB", | 20260507 23:34:38.977 | INFO | "huge" : "", | 20260507 23:34:38.977 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:38.977 | INFO | "malloc" : " 14 GB", | 20260507 23:34:38.977 | INFO | "pinned" : " 29 GB" | 20260507 23:34:38.977 | INFO | } | 20260507 23:34:38.977 | INFO | Perf report for the last 5.1397 seconds: | 20260507 23:34:38.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:38.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:38.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:38.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:38.978 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260507 23:34:38.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:38.978 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 23:34:38.978 | INFO | PRE_HQ : 65.9%, 1.7, 57.4ms, 0 | 20260507 23:34:38.978 | INFO | BAZWRITER : 69.1%, 3.9, 139.8ms, 0 | 20260507 23:34:38.978 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260507 23:34:38.978 | INFO | Uploaded 36GiB to GPU in 2.56204s (14.0513GiB/s) | 20260507 23:34:38.978 | INFO | Downloaded 38GiB to GPU in 2.69558s (14.0971GiB/s) | 20260507 23:34:44.117 | INFO | Analyzing chunk frames = [3482112,3482624) | 20260507 23:34:44.117 | INFO | MemInfo: { | 20260507 23:34:44.117 | INFO | "gpu" : " 17 GB", | 20260507 23:34:44.117 | INFO | "huge" : "", | 20260507 23:34:44.117 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:44.117 | INFO | "malloc" : " 14 GB", | 20260507 23:34:44.117 | INFO | "pinned" : " 29 GB" | 20260507 23:34:44.117 | INFO | } | 20260507 23:34:44.117 | INFO | Perf report for the last 5.13998 seconds: | 20260507 23:34:44.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:44.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.117 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260507 23:34:44.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.118 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:34:44.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:44.118 | INFO | Uploaded 36GiB to GPU in 2.59266s (13.8853GiB/s) | 20260507 23:34:44.118 | INFO | Downloaded 34GiB to GPU in 2.48166s (13.7005GiB/s) | 20260507 23:34:49.258 | INFO | Analyzing chunk frames = [3482624,3483136) | 20260507 23:34:49.258 | INFO | MemInfo: { | 20260507 23:34:49.258 | INFO | "gpu" : " 17 GB", | 20260507 23:34:49.258 | INFO | "huge" : "", | 20260507 23:34:49.258 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:49.258 | INFO | "malloc" : " 14 GB", | 20260507 23:34:49.258 | INFO | "pinned" : " 29 GB" | 20260507 23:34:49.258 | INFO | } | 20260507 23:34:49.258 | INFO | Perf report for the last 5.14055 seconds: | 20260507 23:34:49.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:49.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:34:49.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:34:49.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:49.258 | INFO | Uploaded 36GiB to GPU in 2.5373s (14.1883GiB/s) | 20260507 23:34:49.259 | INFO | Downloaded 34GiB to GPU in 2.48352s (13.6903GiB/s) | 20260507 23:34:54.394 | INFO | Analyzing chunk frames = [3483136,3483648) | 20260507 23:34:54.394 | INFO | MemInfo: { | 20260507 23:34:54.394 | INFO | "gpu" : " 17 GB", | 20260507 23:34:54.394 | INFO | "huge" : "", | 20260507 23:34:54.394 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:54.394 | INFO | "malloc" : " 14 GB", | 20260507 23:34:54.394 | INFO | "pinned" : " 29 GB" | 20260507 23:34:54.394 | INFO | } | 20260507 23:34:54.394 | INFO | Perf report for the last 5.13609 seconds: | 20260507 23:34:54.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:54.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260507 23:34:54.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:34:54.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:54.394 | INFO | Uploaded 36GiB to GPU in 2.49657s (14.4198GiB/s) | 20260507 23:34:54.394 | INFO | Downloaded 34GiB to GPU in 2.39371s (14.2039GiB/s) | 20260507 23:34:59.529 | INFO | Analyzing chunk frames = [3483648,3484160) | 20260507 23:34:59.529 | INFO | MemInfo: { | 20260507 23:34:59.529 | INFO | "gpu" : " 17 GB", | 20260507 23:34:59.529 | INFO | "huge" : "", | 20260507 23:34:59.529 | INFO | "huge_shared" : " 29 GB", | 20260507 23:34:59.529 | INFO | "malloc" : " 14 GB", | 20260507 23:34:59.529 | INFO | "pinned" : " 29 GB" | 20260507 23:34:59.529 | INFO | } | 20260507 23:34:59.529 | INFO | Perf report for the last 5.1351 seconds: | 20260507 23:34:59.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:34:59.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.529 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:34:59.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.530 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:34:59.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:34:59.530 | INFO | Uploaded 36GiB to GPU in 2.37221s (15.1757GiB/s) | 20260507 23:34:59.530 | INFO | Downloaded 34GiB to GPU in 2.48445s (13.6851GiB/s) | 20260507 23:35:04.672 | INFO | Analyzing chunk frames = [3484160,3484672) | 20260507 23:35:04.672 | INFO | MemInfo: { | 20260507 23:35:04.672 | INFO | "gpu" : " 17 GB", | 20260507 23:35:04.672 | INFO | "huge" : "", | 20260507 23:35:04.672 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:04.672 | INFO | "malloc" : " 14 GB", | 20260507 23:35:04.672 | INFO | "pinned" : " 29 GB" | 20260507 23:35:04.672 | INFO | } | 20260507 23:35:04.672 | INFO | Perf report for the last 5.14269 seconds: | 20260507 23:35:04.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:04.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.672 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260507 23:35:04.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.672 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260507 23:35:04.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:04.673 | INFO | Uploaded 36GiB to GPU in 2.43302s (14.7964GiB/s) | 20260507 23:35:04.673 | INFO | Downloaded 34GiB to GPU in 2.4031s (14.1484GiB/s) | 20260507 23:35:09.814 | INFO | Analyzing chunk frames = [3484672,3485184) | 20260507 23:35:09.814 | INFO | MemInfo: { | 20260507 23:35:09.814 | INFO | "gpu" : " 17 GB", | 20260507 23:35:09.814 | INFO | "huge" : "", | 20260507 23:35:09.815 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:09.815 | INFO | "malloc" : " 14 GB", | 20260507 23:35:09.815 | INFO | "pinned" : " 29 GB" | 20260507 23:35:09.815 | INFO | } | 20260507 23:35:09.815 | INFO | Perf report for the last 5.14269 seconds: | 20260507 23:35:09.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:09.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260507 23:35:09.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:35:09.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:09.815 | INFO | Uploaded 36GiB to GPU in 2.52886s (14.2357GiB/s) | 20260507 23:35:09.815 | INFO | Downloaded 34GiB to GPU in 2.44507s (13.9055GiB/s) | 20260507 23:35:14.957 | INFO | Analyzing chunk frames = [3485184,3485696) | 20260507 23:35:14.957 | INFO | MemInfo: { | 20260507 23:35:14.957 | INFO | "gpu" : " 17 GB", | 20260507 23:35:14.957 | INFO | "huge" : "", | 20260507 23:35:14.957 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:14.957 | INFO | "malloc" : " 14 GB", | 20260507 23:35:14.957 | INFO | "pinned" : " 29 GB" | 20260507 23:35:14.957 | INFO | } | 20260507 23:35:14.957 | INFO | Perf report for the last 5.1423 seconds: | 20260507 23:35:14.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:14.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:35:14.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:35:14.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:14.957 | INFO | Uploaded 36GiB to GPU in 2.51076s (14.3383GiB/s) | 20260507 23:35:14.958 | INFO | Downloaded 34GiB to GPU in 2.44083s (13.9297GiB/s) | 20260507 23:35:15.054 | INFO | Starting a round of Baz IO | 20260507 23:35:18.511 | INFO | Populating RTMetrics report | 20260507 23:35:18.511 | INFO | RTMetrics report populated | 20260507 23:35:18.511 | INFO | Opening tmp RTMetrics json file | 20260507 23:35:18.523 | INFO | Serializing RTMetrics to json | 20260507 23:35:18.540 | INFO | Serialization done, swapping tmp file now... | 20260507 23:35:18.541 | INFO | RTMetrics done | 20260507 23:35:18.616 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.561 seconds | 20260507 23:35:18.616 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.55% / 0.01% / 57.44% | 20260507 23:35:18.616 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.16667% / 0% / 6.35783e-05% | 20260507 23:35:18.616 | INFO | Baz IO overall: | 20260507 23:35:18.616 | INFO | BytesWritten : 1586631199098 | 20260507 23:35:18.616 | INFO | NumEvents : 768295488113 | 20260507 23:35:20.096 | INFO | Analyzing chunk frames = [3485696,3486208) | 20260507 23:35:20.096 | INFO | MemInfo: { | 20260507 23:35:20.096 | INFO | "gpu" : " 17 GB", | 20260507 23:35:20.096 | INFO | "huge" : "", | 20260507 23:35:20.096 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:20.096 | INFO | "malloc" : " 14 GB", | 20260507 23:35:20.096 | INFO | "pinned" : " 29 GB" | 20260507 23:35:20.096 | INFO | } | 20260507 23:35:20.096 | INFO | Perf report for the last 5.13923 seconds: | 20260507 23:35:20.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:20.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:20.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:20.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:20.096 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260507 23:35:20.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:20.096 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 23:35:20.096 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260507 23:35:20.096 | INFO | BAZWRITER : 68.9%, 3.7, 130.6ms, 0 | 20260507 23:35:20.096 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 23:35:20.096 | INFO | Uploaded 36GiB to GPU in 2.57012s (14.0071GiB/s) | 20260507 23:35:20.096 | INFO | Downloaded 38GiB to GPU in 2.68558s (14.1496GiB/s) | 20260507 23:35:25.230 | INFO | Analyzing chunk frames = [3486208,3486720) | 20260507 23:35:25.230 | INFO | MemInfo: { | 20260507 23:35:25.230 | INFO | "gpu" : " 17 GB", | 20260507 23:35:25.230 | INFO | "huge" : "", | 20260507 23:35:25.230 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:25.230 | INFO | "malloc" : " 14 GB", | 20260507 23:35:25.231 | INFO | "pinned" : " 29 GB" | 20260507 23:35:25.231 | INFO | } | 20260507 23:35:25.231 | INFO | Perf report for the last 5.13444 seconds: | 20260507 23:35:25.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:25.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:35:25.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:35:25.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:25.231 | INFO | Uploaded 36GiB to GPU in 2.54555s (14.1423GiB/s) | 20260507 23:35:25.231 | INFO | Downloaded 34GiB to GPU in 2.42085s (14.0447GiB/s) | 20260507 23:35:30.368 | INFO | Analyzing chunk frames = [3486720,3487232) | 20260507 23:35:30.368 | INFO | MemInfo: { | 20260507 23:35:30.368 | INFO | "gpu" : " 17 GB", | 20260507 23:35:30.368 | INFO | "huge" : "", | 20260507 23:35:30.368 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:30.368 | INFO | "malloc" : " 14 GB", | 20260507 23:35:30.368 | INFO | "pinned" : " 29 GB" | 20260507 23:35:30.368 | INFO | } | 20260507 23:35:30.368 | INFO | Perf report for the last 5.13732 seconds: | 20260507 23:35:30.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:30.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:35:30.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 23:35:30.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:30.368 | INFO | Uploaded 36GiB to GPU in 2.51221s (14.33GiB/s) | 20260507 23:35:30.368 | INFO | Downloaded 34GiB to GPU in 2.48616s (13.6757GiB/s) | 20260507 23:35:35.508 | INFO | Analyzing chunk frames = [3487232,3487744) | 20260507 23:35:35.508 | INFO | MemInfo: { | 20260507 23:35:35.508 | INFO | "gpu" : " 17 GB", | 20260507 23:35:35.508 | INFO | "huge" : "", | 20260507 23:35:35.508 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:35.508 | INFO | "malloc" : " 14 GB", | 20260507 23:35:35.508 | INFO | "pinned" : " 29 GB" | 20260507 23:35:35.508 | INFO | } | 20260507 23:35:35.508 | INFO | Perf report for the last 5.14012 seconds: | 20260507 23:35:35.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:35.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 23:35:35.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:35:35.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:35.508 | INFO | Uploaded 36GiB to GPU in 2.53431s (14.2051GiB/s) | 20260507 23:35:35.509 | INFO | Downloaded 34GiB to GPU in 2.48762s (13.6677GiB/s) | 20260507 23:35:40.652 | INFO | Analyzing chunk frames = [3487744,3488256) | 20260507 23:35:40.652 | INFO | MemInfo: { | 20260507 23:35:40.652 | INFO | "gpu" : " 17 GB", | 20260507 23:35:40.652 | INFO | "huge" : "", | 20260507 23:35:40.652 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:40.652 | INFO | "malloc" : " 14 GB", | 20260507 23:35:40.652 | INFO | "pinned" : " 29 GB" | 20260507 23:35:40.652 | INFO | } | 20260507 23:35:40.652 | INFO | Perf report for the last 5.144 seconds: | 20260507 23:35:40.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:40.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 23:35:40.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260507 23:35:40.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:40.652 | INFO | Uploaded 36GiB to GPU in 2.39875s (15.0078GiB/s) | 20260507 23:35:40.652 | INFO | Downloaded 34GiB to GPU in 2.48913s (13.6594GiB/s) | 20260507 23:35:45.789 | INFO | Analyzing chunk frames = [3488256,3488768) | 20260507 23:35:45.789 | INFO | MemInfo: { | 20260507 23:35:45.789 | INFO | "gpu" : " 17 GB", | 20260507 23:35:45.789 | INFO | "huge" : "", | 20260507 23:35:45.789 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:45.789 | INFO | "malloc" : " 14 GB", | 20260507 23:35:45.789 | INFO | "pinned" : " 29 GB" | 20260507 23:35:45.789 | INFO | } | 20260507 23:35:45.789 | INFO | Perf report for the last 5.13697 seconds: | 20260507 23:35:45.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:45.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.789 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:35:45.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.789 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:35:45.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:45.790 | INFO | Uploaded 36GiB to GPU in 2.40371s (14.9769GiB/s) | 20260507 23:35:45.790 | INFO | Downloaded 34GiB to GPU in 2.39955s (14.1693GiB/s) | 20260507 23:35:50.928 | INFO | Analyzing chunk frames = [3488768,3489280) | 20260507 23:35:50.928 | INFO | MemInfo: { | 20260507 23:35:50.928 | INFO | "gpu" : " 17 GB", | 20260507 23:35:50.928 | INFO | "huge" : "", | 20260507 23:35:50.928 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:50.928 | INFO | "malloc" : " 14 GB", | 20260507 23:35:50.928 | INFO | "pinned" : " 29 GB" | 20260507 23:35:50.928 | INFO | } | 20260507 23:35:50.928 | INFO | Perf report for the last 5.13912 seconds: | 20260507 23:35:50.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:50.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.928 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260507 23:35:50.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.928 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:35:50.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:50.929 | INFO | Uploaded 36GiB to GPU in 2.54355s (14.1535GiB/s) | 20260507 23:35:50.929 | INFO | Downloaded 34GiB to GPU in 2.46356s (13.8012GiB/s) | 20260507 23:35:56.070 | INFO | Analyzing chunk frames = [3489280,3489792) | 20260507 23:35:56.070 | INFO | MemInfo: { | 20260507 23:35:56.070 | INFO | "gpu" : " 17 GB", | 20260507 23:35:56.070 | INFO | "huge" : "", | 20260507 23:35:56.070 | INFO | "huge_shared" : " 29 GB", | 20260507 23:35:56.070 | INFO | "malloc" : " 14 GB", | 20260507 23:35:56.070 | INFO | "pinned" : " 29 GB" | 20260507 23:35:56.070 | INFO | } | 20260507 23:35:56.070 | INFO | Perf report for the last 5.14221 seconds: | 20260507 23:35:56.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:35:56.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.071 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:35:56.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.071 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 23:35:56.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:35:56.071 | INFO | Uploaded 36GiB to GPU in 2.47804s (14.5276GiB/s) | 20260507 23:35:56.071 | INFO | Downloaded 34GiB to GPU in 2.44115s (13.9279GiB/s) | 20260507 23:35:56.071 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:35:56.071 | INFO | Total mem usage: 16.7 GiB | 20260507 23:35:56.071 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:35:56.071 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260507 23:35:56.071 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:35:56.071 | INFO | High Water Breakdown: | 20260507 23:35:56.071 | INFO | 59.9% : 8.87 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:35:56.071 | INFO | 39.9% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:35:56.071 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:35:56.071 | INFO | ------------------------------------------ | 20260507 23:35:56.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:35:56.071 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:35:56.071 | INFO | Total mem usage: 26.5 GiB | 20260507 23:35:56.071 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:35:56.071 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:35:56.071 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:35:56.071 | INFO | High Water Breakdown: | 20260507 23:35:56.071 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:35:56.071 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:35:56.071 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:35:56.071 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:35:56.071 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:35:56.071 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:35:56.071 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:35:56.071 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:35:56.071 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:35:56.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:35:56.071 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:35:56.071 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:35:56.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:35:56.071 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:35:56.071 | INFO | ------------------------------------------ | 20260507 23:35:56.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:35:56.071 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:35:56.071 | INFO | Total mem usage: 27.8 GiB | 20260507 23:35:56.071 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:35:56.071 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:35:56.071 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:35:56.071 | INFO | High Water Breakdown: | 20260507 23:35:56.071 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:35:56.071 | INFO | ------------------------------------------ | 20260507 23:35:56.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:35:56.071 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:35:56.071 | INFO | Total mem usage: 15.7 GiB | 20260507 23:35:56.071 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:35:56.071 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:35:56.071 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:35:56.071 | INFO | High Water Breakdown: | 20260507 23:35:56.071 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:35:56.071 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:35:56.071 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:35:56.071 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:35:56.071 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:35:56.071 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:35:56.071 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:35:56.071 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:35:56.071 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:35:56.071 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:35:56.071 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:35:56.071 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:35:56.071 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:35:56.071 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:35:56.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:35:56.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:35:56.071 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:35:56.071 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:35:56.071 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:35:56.071 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:35:56.071 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:35:56.071 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:35:56.071 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:35:56.071 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:35:56.071 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:35:56.071 | INFO | ------------------------------------------ | 20260507 23:35:56.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:35:56.071 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:35:56.071 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:35:56.071 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:35:56.071 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.63e+01, 1.43e+01 / 1.82e+00 | 20260507 23:35:56.071 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.02e-01, 1.10e-02 / 4.81e-03 | 20260507 23:35:56.071 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.48e+00, 5.87e-01 / 1.96e-01 | 20260507 23:35:56.071 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.21e-01 | 20260507 23:35:56.071 | INFO | PerfReport: 16.03%, DME -- 4.21e+01 / 4.61e+01, 4.30e+00 / 2.03e+00 | 20260507 23:35:56.071 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.42e+00 | 20260507 23:35:56.071 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.29e-01 / 2.14e-01 | 20260507 23:35:56.071 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.36e+00, 2.82e-01 / 1.35e-01 | 20260507 23:35:56.071 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.47e-01 / 3.87e-01, 5.62e-02 / 2.72e-02 | 20260507 23:35:56.071 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:35:56.072 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:35:56.072 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:35:56.072 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 24.39%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 24.20%, Download -- 2.58e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 30.72%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 2.31%, LockUpload -- 2.47e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 15.81%, LockCompute -- 1.69e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 23:35:56.072 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 23:35:56.170 | INFO | Starting a round of Baz IO | 20260507 23:35:59.626 | INFO | Populating RTMetrics report | 20260507 23:35:59.627 | INFO | RTMetrics report populated | 20260507 23:35:59.627 | INFO | Opening tmp RTMetrics json file | 20260507 23:35:59.659 | INFO | Serializing RTMetrics to json | 20260507 23:35:59.703 | INFO | Serialization done, swapping tmp file now... | 20260507 23:35:59.706 | INFO | RTMetrics done | 20260507 23:35:59.731 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 3.561 seconds | 20260507 23:35:59.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.54% / 0.01% / 57.45% | 20260507 23:35:59.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.16157% / 1.58946e-05% / 4.76837e-05% | 20260507 23:35:59.731 | INFO | Baz IO overall: | 20260507 23:35:59.731 | INFO | BytesWritten : 1588467753725 | 20260507 23:35:59.731 | INFO | NumEvents : 769180031307 | 20260507 23:36:01.208 | INFO | Analyzing chunk frames = [3489792,3490304) | 20260507 23:36:01.208 | INFO | MemInfo: { | 20260507 23:36:01.208 | INFO | "gpu" : " 17 GB", | 20260507 23:36:01.208 | INFO | "huge" : "", | 20260507 23:36:01.208 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:01.208 | INFO | "malloc" : " 14 GB", | 20260507 23:36:01.208 | INFO | "pinned" : " 29 GB" | 20260507 23:36:01.208 | INFO | } | 20260507 23:36:01.208 | INFO | Perf report for the last 5.13756 seconds: | 20260507 23:36:01.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:01.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:01.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:01.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:01.208 | INFO | ANALYSIS : 68.5%, 3.0, 106.1ms, 0 | 20260507 23:36:01.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:01.208 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 23:36:01.208 | INFO | PRE_HQ : 66.1%, 1.7, 57.3ms, 0 | 20260507 23:36:01.208 | INFO | BAZWRITER : 68.9%, 3.7, 131.1ms, 0 | 20260507 23:36:01.208 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 23:36:01.208 | INFO | Uploaded 36GiB to GPU in 2.52081s (14.2811GiB/s) | 20260507 23:36:01.208 | INFO | Downloaded 38GiB to GPU in 2.70792s (14.0329GiB/s) | 20260507 23:36:06.348 | INFO | Analyzing chunk frames = [3490304,3490816) | 20260507 23:36:06.348 | INFO | MemInfo: { | 20260507 23:36:06.348 | INFO | "gpu" : " 17 GB", | 20260507 23:36:06.348 | INFO | "huge" : "", | 20260507 23:36:06.348 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:06.348 | INFO | "malloc" : " 14 GB", | 20260507 23:36:06.348 | INFO | "pinned" : " 29 GB" | 20260507 23:36:06.348 | INFO | } | 20260507 23:36:06.348 | INFO | Perf report for the last 5.13999 seconds: | 20260507 23:36:06.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:06.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 23:36:06.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:36:06.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:06.348 | INFO | Uploaded 36GiB to GPU in 2.5688s (14.0143GiB/s) | 20260507 23:36:06.348 | INFO | Downloaded 34GiB to GPU in 2.48381s (13.6886GiB/s) | 20260507 23:36:11.481 | INFO | Analyzing chunk frames = [3490816,3491328) | 20260507 23:36:11.481 | INFO | MemInfo: { | 20260507 23:36:11.481 | INFO | "gpu" : " 17 GB", | 20260507 23:36:11.481 | INFO | "huge" : "", | 20260507 23:36:11.481 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:11.481 | INFO | "malloc" : " 14 GB", | 20260507 23:36:11.481 | INFO | "pinned" : " 29 GB" | 20260507 23:36:11.481 | INFO | } | 20260507 23:36:11.481 | INFO | Perf report for the last 5.13319 seconds: | 20260507 23:36:11.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:11.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 23:36:11.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 23:36:11.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:11.481 | INFO | Uploaded 36GiB to GPU in 2.52847s (14.2378GiB/s) | 20260507 23:36:11.482 | INFO | Downloaded 34GiB to GPU in 2.4866s (13.6733GiB/s) | 20260507 23:36:16.625 | INFO | Analyzing chunk frames = [3491328,3491840) | 20260507 23:36:16.625 | INFO | MemInfo: { | 20260507 23:36:16.625 | INFO | "gpu" : " 17 GB", | 20260507 23:36:16.625 | INFO | "huge" : "", | 20260507 23:36:16.625 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:16.625 | INFO | "malloc" : " 14 GB", | 20260507 23:36:16.625 | INFO | "pinned" : " 29 GB" | 20260507 23:36:16.625 | INFO | } | 20260507 23:36:16.625 | INFO | Perf report for the last 5.1438 seconds: | 20260507 23:36:16.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:16.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | ANALYSIS : 66.1%, 3.0, 102.0ms, 0 | 20260507 23:36:16.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | PRE_HQ : 25.2%, 1.0, 12.9ms, 0 | 20260507 23:36:16.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:16.625 | INFO | Uploaded 36GiB to GPU in 2.46265s (14.6184GiB/s) | 20260507 23:36:16.625 | INFO | Downloaded 34GiB to GPU in 2.39262s (14.2104GiB/s) | 20260507 23:36:21.769 | INFO | Analyzing chunk frames = [3491840,3492352) | 20260507 23:36:21.769 | INFO | MemInfo: { | 20260507 23:36:21.769 | INFO | "gpu" : " 17 GB", | 20260507 23:36:21.769 | INFO | "huge" : "", | 20260507 23:36:21.769 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:21.769 | INFO | "malloc" : " 14 GB", | 20260507 23:36:21.769 | INFO | "pinned" : " 29 GB" | 20260507 23:36:21.769 | INFO | } | 20260507 23:36:21.769 | INFO | Perf report for the last 5.14422 seconds: | 20260507 23:36:21.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:21.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.769 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 23:36:21.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.769 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:36:21.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:21.770 | INFO | Uploaded 36GiB to GPU in 2.376s (15.1515GiB/s) | 20260507 23:36:21.770 | INFO | Downloaded 34GiB to GPU in 2.49097s (13.6493GiB/s) | 20260507 23:36:26.907 | INFO | Analyzing chunk frames = [3492352,3492864) | 20260507 23:36:26.907 | INFO | MemInfo: { | 20260507 23:36:26.907 | INFO | "gpu" : " 17 GB", | 20260507 23:36:26.907 | INFO | "huge" : "", | 20260507 23:36:26.907 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:26.907 | INFO | "malloc" : " 14 GB", | 20260507 23:36:26.907 | INFO | "pinned" : " 29 GB" | 20260507 23:36:26.908 | INFO | } | 20260507 23:36:26.908 | INFO | Perf report for the last 5.1383 seconds: | 20260507 23:36:26.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:26.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:36:26.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:36:26.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:26.908 | INFO | Uploaded 36GiB to GPU in 2.4254s (14.8429GiB/s) | 20260507 23:36:26.908 | INFO | Downloaded 34GiB to GPU in 2.39038s (14.2237GiB/s) | 20260507 23:36:32.041 | INFO | Analyzing chunk frames = [3492864,3493376) | 20260507 23:36:32.041 | INFO | MemInfo: { | 20260507 23:36:32.041 | INFO | "gpu" : " 17 GB", | 20260507 23:36:32.041 | INFO | "huge" : "", | 20260507 23:36:32.041 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:32.041 | INFO | "malloc" : " 14 GB", | 20260507 23:36:32.042 | INFO | "pinned" : " 29 GB" | 20260507 23:36:32.042 | INFO | } | 20260507 23:36:32.042 | INFO | Perf report for the last 5.13398 seconds: | 20260507 23:36:32.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:32.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 23:36:32.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:36:32.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:32.042 | INFO | Uploaded 36GiB to GPU in 2.57272s (13.993GiB/s) | 20260507 23:36:32.042 | INFO | Downloaded 34GiB to GPU in 2.46374s (13.8002GiB/s) | 20260507 23:36:37.180 | INFO | Analyzing chunk frames = [3493376,3493888) | 20260507 23:36:37.180 | INFO | MemInfo: { | 20260507 23:36:37.180 | INFO | "gpu" : " 17 GB", | 20260507 23:36:37.180 | INFO | "huge" : "", | 20260507 23:36:37.180 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:37.180 | INFO | "malloc" : " 14 GB", | 20260507 23:36:37.181 | INFO | "pinned" : " 29 GB" | 20260507 23:36:37.181 | INFO | } | 20260507 23:36:37.181 | INFO | Perf report for the last 5.13911 seconds: | 20260507 23:36:37.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:37.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:36:37.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:36:37.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:37.181 | INFO | Uploaded 36GiB to GPU in 2.54239s (14.1599GiB/s) | 20260507 23:36:37.181 | INFO | Downloaded 34GiB to GPU in 2.44245s (13.9204GiB/s) | 20260507 23:36:37.278 | INFO | Starting a round of Baz IO | 20260507 23:36:40.743 | INFO | Populating RTMetrics report | 20260507 23:36:40.743 | INFO | RTMetrics report populated | 20260507 23:36:40.743 | INFO | Opening tmp RTMetrics json file | 20260507 23:36:40.748 | INFO | Serializing RTMetrics to json | 20260507 23:36:40.770 | INFO | Serialization done, swapping tmp file now... | 20260507 23:36:40.770 | INFO | RTMetrics done | 20260507 23:36:40.877 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.599 seconds | 20260507 23:36:40.877 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.54% / 0.01% / 57.46% | 20260507 23:36:40.877 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15843% / 7.94729e-06% / 4.76837e-05% | 20260507 23:36:40.877 | INFO | Baz IO overall: | 20260507 23:36:40.877 | INFO | BytesWritten : 1590302658382 | 20260507 23:36:40.877 | INFO | NumEvents : 770063343133 | 20260507 23:36:42.320 | INFO | Analyzing chunk frames = [3493888,3494400) | 20260507 23:36:42.320 | INFO | MemInfo: { | 20260507 23:36:42.320 | INFO | "gpu" : " 17 GB", | 20260507 23:36:42.320 | INFO | "huge" : "", | 20260507 23:36:42.320 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:42.320 | INFO | "malloc" : " 14 GB", | 20260507 23:36:42.320 | INFO | "pinned" : " 29 GB" | 20260507 23:36:42.320 | INFO | } | 20260507 23:36:42.320 | INFO | Perf report for the last 5.13936 seconds: | 20260507 23:36:42.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:42.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:42.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:42.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:42.320 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 23:36:42.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:42.320 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:36:42.320 | INFO | PRE_HQ : 65.7%, 1.7, 57.0ms, 0 | 20260507 23:36:42.320 | INFO | BAZWRITER : 69.6%, 3.7, 133.1ms, 0 | 20260507 23:36:42.320 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 23:36:42.320 | INFO | Uploaded 36GiB to GPU in 2.58148s (13.9455GiB/s) | 20260507 23:36:42.320 | INFO | Downloaded 38GiB to GPU in 2.69707s (14.0894GiB/s) | 20260507 23:36:47.463 | INFO | Analyzing chunk frames = [3494400,3494912) | 20260507 23:36:47.463 | INFO | MemInfo: { | 20260507 23:36:47.463 | INFO | "gpu" : " 17 GB", | 20260507 23:36:47.463 | INFO | "huge" : "", | 20260507 23:36:47.463 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:47.463 | INFO | "malloc" : " 14 GB", | 20260507 23:36:47.463 | INFO | "pinned" : " 29 GB" | 20260507 23:36:47.463 | INFO | } | 20260507 23:36:47.463 | INFO | Perf report for the last 5.14327 seconds: | 20260507 23:36:47.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:47.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.463 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:36:47.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.463 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 23:36:47.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:47.464 | INFO | Uploaded 36GiB to GPU in 2.53288s (14.2131GiB/s) | 20260507 23:36:47.464 | INFO | Downloaded 34GiB to GPU in 2.46479s (13.7943GiB/s) | 20260507 23:36:52.597 | INFO | Analyzing chunk frames = [3494912,3495424) | 20260507 23:36:52.597 | INFO | MemInfo: { | 20260507 23:36:52.597 | INFO | "gpu" : " 17 GB", | 20260507 23:36:52.597 | INFO | "huge" : "", | 20260507 23:36:52.597 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:52.597 | INFO | "malloc" : " 14 GB", | 20260507 23:36:52.597 | INFO | "pinned" : " 29 GB" | 20260507 23:36:52.597 | INFO | } | 20260507 23:36:52.598 | INFO | Perf report for the last 5.13419 seconds: | 20260507 23:36:52.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:52.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 23:36:52.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:36:52.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:52.598 | INFO | Uploaded 36GiB to GPU in 2.53851s (14.1815GiB/s) | 20260507 23:36:52.598 | INFO | Downloaded 34GiB to GPU in 2.46793s (13.7767GiB/s) | 20260507 23:36:57.738 | INFO | Analyzing chunk frames = [3495424,3495936) | 20260507 23:36:57.738 | INFO | MemInfo: { | 20260507 23:36:57.738 | INFO | "gpu" : " 17 GB", | 20260507 23:36:57.738 | INFO | "huge" : "", | 20260507 23:36:57.738 | INFO | "huge_shared" : " 29 GB", | 20260507 23:36:57.738 | INFO | "malloc" : " 14 GB", | 20260507 23:36:57.738 | INFO | "pinned" : " 29 GB" | 20260507 23:36:57.738 | INFO | } | 20260507 23:36:57.738 | INFO | Perf report for the last 5.1403 seconds: | 20260507 23:36:57.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:36:57.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260507 23:36:57.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:36:57.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:36:57.738 | INFO | Uploaded 36GiB to GPU in 2.5944s (13.876GiB/s) | 20260507 23:36:57.738 | INFO | Downloaded 34GiB to GPU in 2.47056s (13.762GiB/s) | 20260507 23:37:02.881 | INFO | Analyzing chunk frames = [3495936,3496448) | 20260507 23:37:02.881 | INFO | MemInfo: { | 20260507 23:37:02.881 | INFO | "gpu" : " 17 GB", | 20260507 23:37:02.881 | INFO | "huge" : "", | 20260507 23:37:02.881 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:02.881 | INFO | "malloc" : " 14 GB", | 20260507 23:37:02.881 | INFO | "pinned" : " 29 GB" | 20260507 23:37:02.881 | INFO | } | 20260507 23:37:02.881 | INFO | Perf report for the last 5.14294 seconds: | 20260507 23:37:02.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:02.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260507 23:37:02.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:37:02.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:02.881 | INFO | Uploaded 36GiB to GPU in 2.37477s (15.1594GiB/s) | 20260507 23:37:02.881 | INFO | Downloaded 34GiB to GPU in 2.49845s (13.6084GiB/s) | 20260507 23:37:08.023 | INFO | Analyzing chunk frames = [3496448,3496960) | 20260507 23:37:08.023 | INFO | MemInfo: { | 20260507 23:37:08.023 | INFO | "gpu" : " 17 GB", | 20260507 23:37:08.023 | INFO | "huge" : "", | 20260507 23:37:08.023 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:08.023 | INFO | "malloc" : " 14 GB", | 20260507 23:37:08.023 | INFO | "pinned" : " 29 GB" | 20260507 23:37:08.023 | INFO | } | 20260507 23:37:08.023 | INFO | Perf report for the last 5.1426 seconds: | 20260507 23:37:08.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:08.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.023 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:37:08.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.024 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:37:08.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:08.024 | INFO | Uploaded 36GiB to GPU in 2.39669s (15.0207GiB/s) | 20260507 23:37:08.024 | INFO | Downloaded 34GiB to GPU in 2.40415s (14.1422GiB/s) | 20260507 23:37:13.160 | INFO | Analyzing chunk frames = [3496960,3497472) | 20260507 23:37:13.160 | INFO | MemInfo: { | 20260507 23:37:13.160 | INFO | "gpu" : " 17 GB", | 20260507 23:37:13.160 | INFO | "huge" : "", | 20260507 23:37:13.161 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:13.161 | INFO | "malloc" : " 14 GB", | 20260507 23:37:13.161 | INFO | "pinned" : " 29 GB" | 20260507 23:37:13.161 | INFO | } | 20260507 23:37:13.161 | INFO | Perf report for the last 5.13727 seconds: | 20260507 23:37:13.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:13.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260507 23:37:13.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:37:13.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:13.161 | INFO | Uploaded 36GiB to GPU in 2.51411s (14.3192GiB/s) | 20260507 23:37:13.161 | INFO | Downloaded 34GiB to GPU in 2.44898s (13.8834GiB/s) | 20260507 23:37:18.291 | INFO | Analyzing chunk frames = [3497472,3497984) | 20260507 23:37:18.291 | INFO | MemInfo: { | 20260507 23:37:18.291 | INFO | "gpu" : " 17 GB", | 20260507 23:37:18.291 | INFO | "huge" : "", | 20260507 23:37:18.291 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:18.291 | INFO | "malloc" : " 14 GB", | 20260507 23:37:18.291 | INFO | "pinned" : " 29 GB" | 20260507 23:37:18.291 | INFO | } | 20260507 23:37:18.291 | INFO | Perf report for the last 5.13027 seconds: | 20260507 23:37:18.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:18.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260507 23:37:18.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:37:18.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:18.291 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:37:18.291 | INFO | Total mem usage: 16.7 GiB | 20260507 23:37:18.291 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:37:18.291 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.5%) | 20260507 23:37:18.291 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:37:18.291 | INFO | High Water Breakdown: | 20260507 23:37:18.291 | INFO | 59.8% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:37:18.291 | INFO | 40% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:37:18.291 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:37:18.291 | INFO | ------------------------------------------ | 20260507 23:37:18.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:37:18.291 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:37:18.291 | INFO | Total mem usage: 26.5 GiB | 20260507 23:37:18.291 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:37:18.291 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:37:18.292 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:37:18.292 | INFO | High Water Breakdown: | 20260507 23:37:18.292 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:37:18.292 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:37:18.292 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:37:18.292 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:37:18.292 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:37:18.292 | INFO | Uploaded 36GiB to GPU in 2.52722s (14.2449GiB/s) | 20260507 23:37:18.292 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:37:18.292 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:37:18.292 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:37:18.292 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:37:18.292 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:37:18.292 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:37:18.292 | INFO | Downloaded 34GiB to GPU in 2.44803s (13.8887GiB/s) | 20260507 23:37:18.292 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:37:18.292 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:37:18.292 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:37:18.292 | INFO | ------------------------------------------ | 20260507 23:37:18.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:37:18.292 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:37:18.292 | INFO | Total mem usage: 27.8 GiB | 20260507 23:37:18.292 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:37:18.292 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:37:18.292 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:37:18.292 | INFO | High Water Breakdown: | 20260507 23:37:18.292 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:37:18.292 | INFO | ------------------------------------------ | 20260507 23:37:18.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:37:18.292 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:37:18.292 | INFO | Total mem usage: 15.7 GiB | 20260507 23:37:18.292 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:37:18.292 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:37:18.292 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:37:18.292 | INFO | High Water Breakdown: | 20260507 23:37:18.292 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:37:18.292 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:37:18.292 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:37:18.292 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:37:18.292 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:37:18.292 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:37:18.292 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:37:18.292 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:37:18.292 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:37:18.292 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:37:18.292 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:37:18.292 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:37:18.293 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:37:18.293 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:37:18.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:37:18.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:37:18.293 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:37:18.293 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:37:18.293 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:37:18.293 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:37:18.293 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:37:18.293 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:37:18.293 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:37:18.293 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:37:18.293 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:37:18.293 | INFO | ------------------------------------------ | 20260507 23:37:18.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:37:18.293 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:37:18.293 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:37:18.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:37:18.293 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.33e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.23e-02 / 4.55e-03 | 20260507 23:37:18.293 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.51e+00, 5.92e-01 / 1.82e-01 | 20260507 23:37:18.293 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.05e-01 / 1.00e-01 | 20260507 23:37:18.293 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.55e+01, 3.92e+00 / 1.34e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.70e+00 / 1.62e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.31e-01 / 2.17e-01 | 20260507 23:37:18.293 | INFO | PerfReport: 3.42%, Metrics -- 1.12e+00 / 1.35e+00, 2.83e-01 / 1.60e-01 | 20260507 23:37:18.293 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.45e-01 / 3.85e-01, 5.54e-02 / 3.12e-02 | 20260507 23:37:18.293 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:37:18.293 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:37:18.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:37:18.293 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 30.76%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 2.20%, LockUpload -- 2.34e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:37:18.293 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 9.80e-03 / 0.00e+00 | 20260507 23:37:18.404 | INFO | Starting a round of Baz IO | 20260507 23:37:21.869 | INFO | Populating RTMetrics report | 20260507 23:37:21.869 | INFO | RTMetrics report populated | 20260507 23:37:21.869 | INFO | Opening tmp RTMetrics json file | 20260507 23:37:21.911 | INFO | Serializing RTMetrics to json | 20260507 23:37:21.960 | INFO | Serialization done, swapping tmp file now... | 20260507 23:37:21.961 | INFO | RTMetrics done | 20260507 23:37:21.982 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.577 seconds | 20260507 23:37:21.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.53% / 0.01% / 57.46% | 20260507 23:37:21.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15579% / 1.19209e-05% / 3.57628e-05% | 20260507 23:37:21.982 | INFO | Baz IO overall: | 20260507 23:37:21.982 | INFO | BytesWritten : 1592137780909 | 20260507 23:37:21.982 | INFO | NumEvents : 770947147416 | 20260507 23:37:23.435 | INFO | Analyzing chunk frames = [3497984,3498496) | 20260507 23:37:23.435 | INFO | MemInfo: { | 20260507 23:37:23.435 | INFO | "gpu" : " 17 GB", | 20260507 23:37:23.435 | INFO | "huge" : "", | 20260507 23:37:23.435 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:23.435 | INFO | "malloc" : " 14 GB", | 20260507 23:37:23.435 | INFO | "pinned" : " 29 GB" | 20260507 23:37:23.435 | INFO | } | 20260507 23:37:23.435 | INFO | Perf report for the last 5.1439 seconds: | 20260507 23:37:23.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:23.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:23.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:23.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:23.435 | INFO | ANALYSIS : 68.6%, 2.9, 105.2ms, 0 | 20260507 23:37:23.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:23.435 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 23:37:23.435 | INFO | PRE_HQ : 67.6%, 1.8, 62.3ms, 0 | 20260507 23:37:23.435 | INFO | BAZWRITER : 69.1%, 4.5, 159.3ms, 0 | 20260507 23:37:23.435 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 23:37:23.435 | INFO | Uploaded 36GiB to GPU in 2.54998s (14.1178GiB/s) | 20260507 23:37:23.435 | INFO | Downloaded 38GiB to GPU in 2.68667s (14.1439GiB/s) | 20260507 23:37:28.575 | INFO | Analyzing chunk frames = [3498496,3499008) | 20260507 23:37:28.575 | INFO | MemInfo: { | 20260507 23:37:28.575 | INFO | "gpu" : " 17 GB", | 20260507 23:37:28.575 | INFO | "huge" : "", | 20260507 23:37:28.575 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:28.575 | INFO | "malloc" : " 14 GB", | 20260507 23:37:28.575 | INFO | "pinned" : " 29 GB" | 20260507 23:37:28.575 | INFO | } | 20260507 23:37:28.575 | INFO | Perf report for the last 5.14047 seconds: | 20260507 23:37:28.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:28.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.575 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 23:37:28.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.576 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 23:37:28.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:28.576 | INFO | Uploaded 36GiB to GPU in 2.52929s (14.2333GiB/s) | 20260507 23:37:28.576 | INFO | Downloaded 34GiB to GPU in 2.49011s (13.654GiB/s) | 20260507 23:37:33.718 | INFO | Analyzing chunk frames = [3499008,3499520) | 20260507 23:37:33.718 | INFO | MemInfo: { | 20260507 23:37:33.718 | INFO | "gpu" : " 17 GB", | 20260507 23:37:33.718 | INFO | "huge" : "", | 20260507 23:37:33.718 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:33.718 | INFO | "malloc" : " 14 GB", | 20260507 23:37:33.718 | INFO | "pinned" : " 29 GB" | 20260507 23:37:33.718 | INFO | } | 20260507 23:37:33.718 | INFO | Perf report for the last 5.14289 seconds: | 20260507 23:37:33.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:33.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.718 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260507 23:37:33.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.718 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:37:33.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:33.719 | INFO | Uploaded 36GiB to GPU in 2.51355s (14.3224GiB/s) | 20260507 23:37:33.719 | INFO | Downloaded 34GiB to GPU in 2.48885s (13.6609GiB/s) | 20260507 23:37:38.857 | INFO | Analyzing chunk frames = [3499520,3500032) | 20260507 23:37:38.857 | INFO | MemInfo: { | 20260507 23:37:38.857 | INFO | "gpu" : " 17 GB", | 20260507 23:37:38.857 | INFO | "huge" : "", | 20260507 23:37:38.857 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:38.857 | INFO | "malloc" : " 14 GB", | 20260507 23:37:38.857 | INFO | "pinned" : " 29 GB" | 20260507 23:37:38.857 | INFO | } | 20260507 23:37:38.858 | INFO | Perf report for the last 5.13921 seconds: | 20260507 23:37:38.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:38.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260507 23:37:38.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:37:38.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:38.858 | INFO | Uploaded 36GiB to GPU in 2.46618s (14.5975GiB/s) | 20260507 23:37:38.858 | INFO | Downloaded 34GiB to GPU in 2.48894s (13.6605GiB/s) | 20260507 23:37:43.992 | INFO | Analyzing chunk frames = [3500032,3500544) | 20260507 23:37:43.992 | INFO | MemInfo: { | 20260507 23:37:43.992 | INFO | "gpu" : " 17 GB", | 20260507 23:37:43.992 | INFO | "huge" : "", | 20260507 23:37:43.992 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:43.993 | INFO | "malloc" : " 14 GB", | 20260507 23:37:43.993 | INFO | "pinned" : " 29 GB" | 20260507 23:37:43.993 | INFO | } | 20260507 23:37:43.993 | INFO | Perf report for the last 5.13518 seconds: | 20260507 23:37:43.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:43.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260507 23:37:43.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:37:43.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:43.993 | INFO | Uploaded 36GiB to GPU in 2.30507s (15.6178GiB/s) | 20260507 23:37:43.993 | INFO | Downloaded 34GiB to GPU in 2.49335s (13.6363GiB/s) | 20260507 23:37:49.132 | INFO | Analyzing chunk frames = [3500544,3501056) | 20260507 23:37:49.132 | INFO | MemInfo: { | 20260507 23:37:49.132 | INFO | "gpu" : " 17 GB", | 20260507 23:37:49.132 | INFO | "huge" : "", | 20260507 23:37:49.132 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:49.133 | INFO | "malloc" : " 14 GB", | 20260507 23:37:49.133 | INFO | "pinned" : " 29 GB" | 20260507 23:37:49.133 | INFO | } | 20260507 23:37:49.133 | INFO | Perf report for the last 5.13999 seconds: | 20260507 23:37:49.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:49.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 23:37:49.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:37:49.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:49.133 | INFO | Uploaded 36GiB to GPU in 2.48411s (14.4921GiB/s) | 20260507 23:37:49.133 | INFO | Downloaded 34GiB to GPU in 2.44589s (13.9009GiB/s) | 20260507 23:37:54.272 | INFO | Analyzing chunk frames = [3501056,3501568) | 20260507 23:37:54.272 | INFO | MemInfo: { | 20260507 23:37:54.272 | INFO | "gpu" : " 17 GB", | 20260507 23:37:54.272 | INFO | "huge" : "", | 20260507 23:37:54.272 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:54.272 | INFO | "malloc" : " 14 GB", | 20260507 23:37:54.272 | INFO | "pinned" : " 29 GB" | 20260507 23:37:54.272 | INFO | } | 20260507 23:37:54.272 | INFO | Perf report for the last 5.13912 seconds: | 20260507 23:37:54.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:54.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:37:54.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:37:54.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:54.272 | INFO | Uploaded 36GiB to GPU in 2.50523s (14.3699GiB/s) | 20260507 23:37:54.273 | INFO | Downloaded 34GiB to GPU in 2.46519s (13.792GiB/s) | 20260507 23:37:59.413 | INFO | Analyzing chunk frames = [3501568,3502080) | 20260507 23:37:59.413 | INFO | MemInfo: { | 20260507 23:37:59.413 | INFO | "gpu" : " 17 GB", | 20260507 23:37:59.413 | INFO | "huge" : "", | 20260507 23:37:59.413 | INFO | "huge_shared" : " 29 GB", | 20260507 23:37:59.413 | INFO | "malloc" : " 14 GB", | 20260507 23:37:59.413 | INFO | "pinned" : " 29 GB" | 20260507 23:37:59.413 | INFO | } | 20260507 23:37:59.413 | INFO | Perf report for the last 5.14113 seconds: | 20260507 23:37:59.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:37:59.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.413 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:37:59.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.413 | INFO | PRE_HQ : 35.3%, 1.0, 18.4ms, 0 | 20260507 23:37:59.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:37:59.414 | INFO | Uploaded 36GiB to GPU in 2.47262s (14.5595GiB/s) | 20260507 23:37:59.414 | INFO | Downloaded 34GiB to GPU in 2.45803s (13.8322GiB/s) | 20260507 23:37:59.528 | INFO | Starting a round of Baz IO | 20260507 23:38:03.000 | INFO | Populating RTMetrics report | 20260507 23:38:03.000 | INFO | RTMetrics report populated | 20260507 23:38:03.000 | INFO | Opening tmp RTMetrics json file | 20260507 23:38:03.008 | INFO | Serializing RTMetrics to json | 20260507 23:38:03.027 | INFO | Serialization done, swapping tmp file now... | 20260507 23:38:03.027 | INFO | RTMetrics done | 20260507 23:38:03.113 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 3.585 seconds | 20260507 23:38:03.113 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.53% / 0.00% / 57.47% | 20260507 23:38:03.113 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15551% / 7.94729e-06% / 5.5631e-05% | 20260507 23:38:03.113 | INFO | Baz IO overall: | 20260507 23:38:03.113 | INFO | BytesWritten : 1593973738365 | 20260507 23:38:03.113 | INFO | NumEvents : 771831668595 | 20260507 23:38:04.546 | INFO | Analyzing chunk frames = [3502080,3502592) | 20260507 23:38:04.546 | INFO | MemInfo: { | 20260507 23:38:04.546 | INFO | "gpu" : " 17 GB", | 20260507 23:38:04.546 | INFO | "huge" : "", | 20260507 23:38:04.546 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:04.546 | INFO | "malloc" : " 14 GB", | 20260507 23:38:04.546 | INFO | "pinned" : " 29 GB" | 20260507 23:38:04.546 | INFO | } | 20260507 23:38:04.546 | INFO | Perf report for the last 5.13294 seconds: | 20260507 23:38:04.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:04.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:04.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:04.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:04.546 | INFO | ANALYSIS : 69.2%, 2.9, 104.7ms, 0 | 20260507 23:38:04.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:04.546 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 23:38:04.546 | INFO | PRE_HQ : 68.3%, 1.8, 64.2ms, 0 | 20260507 23:38:04.546 | INFO | BAZWRITER : 69.3%, 4.4, 157.1ms, 0 | 20260507 23:38:04.546 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 23:38:04.547 | INFO | Uploaded 36GiB to GPU in 2.56965s (14.0097GiB/s) | 20260507 23:38:04.547 | INFO | Downloaded 38GiB to GPU in 2.69864s (14.0812GiB/s) | 20260507 23:38:09.689 | INFO | Analyzing chunk frames = [3502592,3503104) | 20260507 23:38:09.689 | INFO | MemInfo: { | 20260507 23:38:09.689 | INFO | "gpu" : " 17 GB", | 20260507 23:38:09.689 | INFO | "huge" : "", | 20260507 23:38:09.689 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:09.689 | INFO | "malloc" : " 14 GB", | 20260507 23:38:09.689 | INFO | "pinned" : " 29 GB" | 20260507 23:38:09.689 | INFO | } | 20260507 23:38:09.689 | INFO | Perf report for the last 5.1431 seconds: | 20260507 23:38:09.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:09.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.689 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 23:38:09.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.689 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:38:09.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:09.690 | INFO | Uploaded 36GiB to GPU in 2.5627s (14.0477GiB/s) | 20260507 23:38:09.690 | INFO | Downloaded 34GiB to GPU in 2.47906s (13.7149GiB/s) | 20260507 23:38:14.830 | INFO | Analyzing chunk frames = [3503104,3503616) | 20260507 23:38:14.830 | INFO | MemInfo: { | 20260507 23:38:14.830 | INFO | "gpu" : " 17 GB", | 20260507 23:38:14.830 | INFO | "huge" : "", | 20260507 23:38:14.830 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:14.830 | INFO | "malloc" : " 14 GB", | 20260507 23:38:14.830 | INFO | "pinned" : " 29 GB" | 20260507 23:38:14.830 | INFO | } | 20260507 23:38:14.830 | INFO | Perf report for the last 5.14103 seconds: | 20260507 23:38:14.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:14.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 23:38:14.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:38:14.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:14.830 | INFO | Uploaded 36GiB to GPU in 2.58362s (13.9339GiB/s) | 20260507 23:38:14.830 | INFO | Downloaded 34GiB to GPU in 2.47598s (13.732GiB/s) | 20260507 23:38:19.972 | INFO | Analyzing chunk frames = [3503616,3504128) | 20260507 23:38:19.972 | INFO | MemInfo: { | 20260507 23:38:19.972 | INFO | "gpu" : " 17 GB", | 20260507 23:38:19.972 | INFO | "huge" : "", | 20260507 23:38:19.973 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:19.973 | INFO | "malloc" : " 14 GB", | 20260507 23:38:19.973 | INFO | "pinned" : " 29 GB" | 20260507 23:38:19.973 | INFO | } | 20260507 23:38:19.973 | INFO | Perf report for the last 5.14259 seconds: | 20260507 23:38:19.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:19.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:38:19.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:38:19.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:19.973 | INFO | Uploaded 36GiB to GPU in 2.50962s (14.3448GiB/s) | 20260507 23:38:19.973 | INFO | Downloaded 34GiB to GPU in 2.49928s (13.6039GiB/s) | 20260507 23:38:25.110 | INFO | Analyzing chunk frames = [3504128,3504640) | 20260507 23:38:25.110 | INFO | MemInfo: { | 20260507 23:38:25.110 | INFO | "gpu" : " 17 GB", | 20260507 23:38:25.110 | INFO | "huge" : "", | 20260507 23:38:25.110 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:25.110 | INFO | "malloc" : " 14 GB", | 20260507 23:38:25.110 | INFO | "pinned" : " 29 GB" | 20260507 23:38:25.110 | INFO | } | 20260507 23:38:25.110 | INFO | Perf report for the last 5.13762 seconds: | 20260507 23:38:25.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:25.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.110 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:38:25.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.110 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 23:38:25.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:25.111 | INFO | Uploaded 36GiB to GPU in 2.32581s (15.4785GiB/s) | 20260507 23:38:25.111 | INFO | Downloaded 34GiB to GPU in 2.46855s (13.7733GiB/s) | 20260507 23:38:30.246 | INFO | Analyzing chunk frames = [3504640,3505152) | 20260507 23:38:30.246 | INFO | MemInfo: { | 20260507 23:38:30.246 | INFO | "gpu" : " 17 GB", | 20260507 23:38:30.246 | INFO | "huge" : "", | 20260507 23:38:30.246 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:30.246 | INFO | "malloc" : " 14 GB", | 20260507 23:38:30.246 | INFO | "pinned" : " 29 GB" | 20260507 23:38:30.246 | INFO | } | 20260507 23:38:30.246 | INFO | Perf report for the last 5.13554 seconds: | 20260507 23:38:30.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:30.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:38:30.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:38:30.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:30.246 | INFO | Uploaded 36GiB to GPU in 2.43075s (14.8102GiB/s) | 20260507 23:38:30.246 | INFO | Downloaded 34GiB to GPU in 2.39327s (14.2065GiB/s) | 20260507 23:38:35.389 | INFO | Analyzing chunk frames = [3505152,3505664) | 20260507 23:38:35.389 | INFO | MemInfo: { | 20260507 23:38:35.389 | INFO | "gpu" : " 17 GB", | 20260507 23:38:35.389 | INFO | "huge" : "", | 20260507 23:38:35.389 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:35.389 | INFO | "malloc" : " 14 GB", | 20260507 23:38:35.389 | INFO | "pinned" : " 29 GB" | 20260507 23:38:35.389 | INFO | } | 20260507 23:38:35.389 | INFO | Perf report for the last 5.14324 seconds: | 20260507 23:38:35.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:35.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.389 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:38:35.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.389 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 23:38:35.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:35.390 | INFO | Uploaded 36GiB to GPU in 2.53509s (14.2007GiB/s) | 20260507 23:38:35.390 | INFO | Downloaded 34GiB to GPU in 2.43924s (13.9388GiB/s) | 20260507 23:38:40.526 | INFO | Analyzing chunk frames = [3505664,3506176) | 20260507 23:38:40.526 | INFO | MemInfo: { | 20260507 23:38:40.526 | INFO | "gpu" : " 17 GB", | 20260507 23:38:40.526 | INFO | "huge" : "", | 20260507 23:38:40.527 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:40.527 | INFO | "malloc" : " 14 GB", | 20260507 23:38:40.527 | INFO | "pinned" : " 29 GB" | 20260507 23:38:40.527 | INFO | } | 20260507 23:38:40.527 | INFO | Perf report for the last 5.13764 seconds: | 20260507 23:38:40.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:40.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 23:38:40.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | PRE_HQ : 35.5%, 1.0, 18.4ms, 0 | 20260507 23:38:40.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:40.527 | INFO | Uploaded 36GiB to GPU in 2.56067s (14.0588GiB/s) | 20260507 23:38:40.527 | INFO | Downloaded 34GiB to GPU in 2.44736s (13.8925GiB/s) | 20260507 23:38:40.527 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:38:40.527 | INFO | Total mem usage: 16.7 GiB | 20260507 23:38:40.527 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:38:40.527 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.6%) | 20260507 23:38:40.527 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:38:40.527 | INFO | High Water Breakdown: | 20260507 23:38:40.527 | INFO | 60.6% : 9.3 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:38:40.527 | INFO | 39.3% : 6.04 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:38:40.527 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:38:40.527 | INFO | ------------------------------------------ | 20260507 23:38:40.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:38:40.527 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:38:40.528 | INFO | Total mem usage: 26.5 GiB | 20260507 23:38:40.528 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:38:40.528 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:38:40.528 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:38:40.528 | INFO | High Water Breakdown: | 20260507 23:38:40.528 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:38:40.528 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:38:40.528 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:38:40.528 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:38:40.528 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:38:40.528 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:38:40.528 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:38:40.528 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:38:40.528 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:38:40.528 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:38:40.528 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:38:40.528 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:38:40.528 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:38:40.528 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:38:40.528 | INFO | ------------------------------------------ | 20260507 23:38:40.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:38:40.528 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:38:40.528 | INFO | Total mem usage: 27.8 GiB | 20260507 23:38:40.528 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:38:40.528 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:38:40.528 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:38:40.528 | INFO | High Water Breakdown: | 20260507 23:38:40.528 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:38:40.528 | INFO | ------------------------------------------ | 20260507 23:38:40.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:38:40.528 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:38:40.528 | INFO | Total mem usage: 15.7 GiB | 20260507 23:38:40.528 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:38:40.528 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:38:40.528 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:38:40.528 | INFO | High Water Breakdown: | 20260507 23:38:40.528 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:38:40.528 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:38:40.528 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:38:40.528 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:38:40.528 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:38:40.528 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:38:40.528 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:38:40.528 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:38:40.528 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:38:40.528 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:38:40.528 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:38:40.528 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:38:40.528 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:38:40.528 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:38:40.528 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:38:40.528 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:38:40.528 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:38:40.528 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:38:40.528 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:38:40.528 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:38:40.528 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:38:40.528 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:38:40.528 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:38:40.528 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:38:40.528 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:38:40.528 | INFO | ------------------------------------------ | 20260507 23:38:40.528 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:38:40.528 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:38:40.528 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:38:40.528 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:38:40.528 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.66e+01, 1.43e+01 / 1.65e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.21e-02 / 7.96e-03 | 20260507 23:38:40.529 | INFO | PerfReport: 12.16%, Baseline -- 4.00e+00 / 4.47e+00, 5.84e-01 / 2.04e-01 | 20260507 23:38:40.529 | INFO | PerfReport: 3.36%, Histogram -- 1.11e+00 / 1.48e+00, 2.08e-01 / 1.10e-01 | 20260507 23:38:40.529 | INFO | PerfReport: 15.94%, DME -- 4.20e+01 / 4.59e+01, 4.14e+00 / 1.87e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.72e+00 / 1.94e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.43e+00 / 2.67e+00, 4.34e-01 / 2.56e-01 | 20260507 23:38:40.529 | INFO | PerfReport: 3.43%, Metrics -- 1.13e+00 / 1.32e+00, 2.82e-01 / 1.61e-01 | 20260507 23:38:40.529 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.76e-01, 5.59e-02 / 3.04e-02 | 20260507 23:38:40.529 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:38:40.529 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:38:40.529 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:38:40.529 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 30.88%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 2.00%, LockUpload -- 2.13e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 2.72%, LockDownload -- 2.89e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:38:40.529 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.05e-02 / 0.00e+00 | 20260507 23:38:40.640 | INFO | Starting a round of Baz IO | 20260507 23:38:44.114 | INFO | Populating RTMetrics report | 20260507 23:38:44.115 | INFO | RTMetrics report populated | 20260507 23:38:44.115 | INFO | Opening tmp RTMetrics json file | 20260507 23:38:44.116 | INFO | Serializing RTMetrics to json | 20260507 23:38:44.117 | INFO | Serialization done, swapping tmp file now... | 20260507 23:38:44.118 | INFO | RTMetrics done | 20260507 23:38:44.217 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.577 seconds | 20260507 23:38:44.217 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.52% / 0.01% / 57.47% | 20260507 23:38:44.217 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15318% / 0% / 6.35783e-05% | 20260507 23:38:44.217 | INFO | Baz IO overall: | 20260507 23:38:44.217 | INFO | BytesWritten : 1595808382020 | 20260507 23:38:44.217 | INFO | NumEvents : 772715131721 | 20260507 23:38:45.666 | INFO | Analyzing chunk frames = [3506176,3506688) | 20260507 23:38:45.667 | INFO | MemInfo: { | 20260507 23:38:45.667 | INFO | "gpu" : " 17 GB", | 20260507 23:38:45.667 | INFO | "huge" : "", | 20260507 23:38:45.667 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:45.667 | INFO | "malloc" : " 14 GB", | 20260507 23:38:45.667 | INFO | "pinned" : " 29 GB" | 20260507 23:38:45.667 | INFO | } | 20260507 23:38:45.667 | INFO | Perf report for the last 5.13981 seconds: | 20260507 23:38:45.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:45.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:45.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:45.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:45.667 | INFO | ANALYSIS : 69.4%, 2.9, 105.0ms, 0 | 20260507 23:38:45.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:45.667 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 23:38:45.667 | INFO | PRE_HQ : 67.9%, 1.8, 63.3ms, 0 | 20260507 23:38:45.667 | INFO | BAZWRITER : 69.2%, 4.2, 150.4ms, 0 | 20260507 23:38:45.667 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:38:45.667 | INFO | Uploaded 36GiB to GPU in 2.54361s (14.1531GiB/s) | 20260507 23:38:45.667 | INFO | Downloaded 38GiB to GPU in 2.7096s (14.0242GiB/s) | 20260507 23:38:50.807 | INFO | Analyzing chunk frames = [3506688,3507200) | 20260507 23:38:50.807 | INFO | MemInfo: { | 20260507 23:38:50.807 | INFO | "gpu" : " 17 GB", | 20260507 23:38:50.807 | INFO | "huge" : "", | 20260507 23:38:50.807 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:50.807 | INFO | "malloc" : " 14 GB", | 20260507 23:38:50.807 | INFO | "pinned" : " 29 GB" | 20260507 23:38:50.807 | INFO | } | 20260507 23:38:50.807 | INFO | Perf report for the last 5.1402 seconds: | 20260507 23:38:50.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:50.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 23:38:50.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:38:50.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:50.807 | INFO | Uploaded 36GiB to GPU in 2.56153s (14.0541GiB/s) | 20260507 23:38:50.807 | INFO | Downloaded 34GiB to GPU in 2.49307s (13.6378GiB/s) | 20260507 23:38:55.942 | INFO | Analyzing chunk frames = [3507200,3507712) | 20260507 23:38:55.942 | INFO | MemInfo: { | 20260507 23:38:55.942 | INFO | "gpu" : " 17 GB", | 20260507 23:38:55.942 | INFO | "huge" : "", | 20260507 23:38:55.942 | INFO | "huge_shared" : " 29 GB", | 20260507 23:38:55.942 | INFO | "malloc" : " 14 GB", | 20260507 23:38:55.942 | INFO | "pinned" : " 29 GB" | 20260507 23:38:55.942 | INFO | } | 20260507 23:38:55.942 | INFO | Perf report for the last 5.13493 seconds: | 20260507 23:38:55.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:38:55.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:38:55.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:38:55.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:38:55.942 | INFO | Uploaded 36GiB to GPU in 2.51563s (14.3105GiB/s) | 20260507 23:38:55.942 | INFO | Downloaded 34GiB to GPU in 2.48993s (13.655GiB/s) | 20260507 23:39:01.085 | INFO | Analyzing chunk frames = [3507712,3508224) | 20260507 23:39:01.085 | INFO | MemInfo: { | 20260507 23:39:01.085 | INFO | "gpu" : " 17 GB", | 20260507 23:39:01.085 | INFO | "huge" : "", | 20260507 23:39:01.085 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:01.085 | INFO | "malloc" : " 14 GB", | 20260507 23:39:01.085 | INFO | "pinned" : " 29 GB" | 20260507 23:39:01.085 | INFO | } | 20260507 23:39:01.085 | INFO | Perf report for the last 5.14307 seconds: | 20260507 23:39:01.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:01.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260507 23:39:01.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:39:01.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:01.085 | INFO | Uploaded 36GiB to GPU in 2.47862s (14.5242GiB/s) | 20260507 23:39:01.086 | INFO | Downloaded 34GiB to GPU in 2.41297s (14.0905GiB/s) | 20260507 23:39:06.224 | INFO | Analyzing chunk frames = [3508224,3508736) | 20260507 23:39:06.224 | INFO | MemInfo: { | 20260507 23:39:06.224 | INFO | "gpu" : " 17 GB", | 20260507 23:39:06.224 | INFO | "huge" : "", | 20260507 23:39:06.225 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:06.225 | INFO | "malloc" : " 14 GB", | 20260507 23:39:06.225 | INFO | "pinned" : " 29 GB" | 20260507 23:39:06.225 | INFO | } | 20260507 23:39:06.225 | INFO | Perf report for the last 5.1398 seconds: | 20260507 23:39:06.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:06.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:39:06.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:39:06.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:06.225 | INFO | Uploaded 36GiB to GPU in 2.31388s (15.5583GiB/s) | 20260507 23:39:06.225 | INFO | Downloaded 34GiB to GPU in 2.4797s (13.7113GiB/s) | 20260507 23:39:11.358 | INFO | Analyzing chunk frames = [3508736,3509248) | 20260507 23:39:11.358 | INFO | MemInfo: { | 20260507 23:39:11.358 | INFO | "gpu" : " 17 GB", | 20260507 23:39:11.358 | INFO | "huge" : "", | 20260507 23:39:11.358 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:11.358 | INFO | "malloc" : " 14 GB", | 20260507 23:39:11.358 | INFO | "pinned" : " 29 GB" | 20260507 23:39:11.358 | INFO | } | 20260507 23:39:11.358 | INFO | Perf report for the last 5.1337 seconds: | 20260507 23:39:11.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:11.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.358 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:39:11.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.359 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:39:11.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:11.359 | INFO | Uploaded 36GiB to GPU in 2.37363s (15.1666GiB/s) | 20260507 23:39:11.359 | INFO | Downloaded 34GiB to GPU in 2.42337s (14.0301GiB/s) | 20260507 23:39:16.499 | INFO | Analyzing chunk frames = [3509248,3509760) | 20260507 23:39:16.499 | INFO | MemInfo: { | 20260507 23:39:16.499 | INFO | "gpu" : " 17 GB", | 20260507 23:39:16.499 | INFO | "huge" : "", | 20260507 23:39:16.499 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:16.499 | INFO | "malloc" : " 14 GB", | 20260507 23:39:16.499 | INFO | "pinned" : " 29 GB" | 20260507 23:39:16.499 | INFO | } | 20260507 23:39:16.499 | INFO | Perf report for the last 5.14076 seconds: | 20260507 23:39:16.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:16.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.499 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260507 23:39:16.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.499 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 23:39:16.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:16.500 | INFO | Uploaded 36GiB to GPU in 2.49228s (14.4446GiB/s) | 20260507 23:39:16.500 | INFO | Downloaded 34GiB to GPU in 2.46449s (13.7959GiB/s) | 20260507 23:39:21.640 | INFO | Analyzing chunk frames = [3509760,3510272) | 20260507 23:39:21.640 | INFO | MemInfo: { | 20260507 23:39:21.640 | INFO | "gpu" : " 17 GB", | 20260507 23:39:21.640 | INFO | "huge" : "", | 20260507 23:39:21.640 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:21.640 | INFO | "malloc" : " 14 GB", | 20260507 23:39:21.640 | INFO | "pinned" : " 29 GB" | 20260507 23:39:21.641 | INFO | } | 20260507 23:39:21.641 | INFO | Perf report for the last 5.14145 seconds: | 20260507 23:39:21.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:21.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:39:21.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 23:39:21.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:21.641 | INFO | Uploaded 36GiB to GPU in 2.45474s (14.6655GiB/s) | 20260507 23:39:21.641 | INFO | Downloaded 34GiB to GPU in 2.45456s (13.8518GiB/s) | 20260507 23:39:21.738 | INFO | Starting a round of Baz IO | 20260507 23:39:25.206 | INFO | Populating RTMetrics report | 20260507 23:39:25.206 | INFO | RTMetrics report populated | 20260507 23:39:25.206 | INFO | Opening tmp RTMetrics json file | 20260507 23:39:25.213 | INFO | Serializing RTMetrics to json | 20260507 23:39:25.244 | INFO | Serialization done, swapping tmp file now... | 20260507 23:39:25.303 | INFO | RTMetrics done | 20260507 23:39:25.326 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 3.587 seconds | 20260507 23:39:25.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.52% / 0.01% / 57.48% | 20260507 23:39:25.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15251% / 1.58946e-05% / 6.75519e-05% | 20260507 23:39:25.326 | INFO | Baz IO overall: | 20260507 23:39:25.326 | INFO | BytesWritten : 1597644602232 | 20260507 23:39:25.326 | INFO | NumEvents : 773599951044 | 20260507 23:39:26.786 | INFO | Analyzing chunk frames = [3510272,3510784) | 20260507 23:39:26.786 | INFO | MemInfo: { | 20260507 23:39:26.786 | INFO | "gpu" : " 17 GB", | 20260507 23:39:26.786 | INFO | "huge" : "", | 20260507 23:39:26.786 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:26.786 | INFO | "malloc" : " 14 GB", | 20260507 23:39:26.786 | INFO | "pinned" : " 29 GB" | 20260507 23:39:26.786 | INFO | } | 20260507 23:39:26.786 | INFO | Perf report for the last 5.14536 seconds: | 20260507 23:39:26.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:26.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:26.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:26.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:26.786 | INFO | ANALYSIS : 68.4%, 3.0, 106.8ms, 0 | 20260507 23:39:26.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:26.786 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 23:39:26.786 | INFO | PRE_HQ : 65.5%, 1.7, 57.2ms, 0 | 20260507 23:39:26.786 | INFO | BAZWRITER : 69.3%, 3.6, 128.9ms, 0 | 20260507 23:39:26.786 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 23:39:26.786 | INFO | Uploaded 36GiB to GPU in 2.51819s (14.296GiB/s) | 20260507 23:39:26.786 | INFO | Downloaded 38GiB to GPU in 2.68606s (14.1471GiB/s) | 20260507 23:39:31.923 | INFO | Analyzing chunk frames = [3510784,3511296) | 20260507 23:39:31.923 | INFO | MemInfo: { | 20260507 23:39:31.923 | INFO | "gpu" : " 17 GB", | 20260507 23:39:31.923 | INFO | "huge" : "", | 20260507 23:39:31.923 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:31.923 | INFO | "malloc" : " 14 GB", | 20260507 23:39:31.923 | INFO | "pinned" : " 29 GB" | 20260507 23:39:31.923 | INFO | } | 20260507 23:39:31.924 | INFO | Perf report for the last 5.13748 seconds: | 20260507 23:39:31.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:31.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:39:31.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:39:31.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:31.924 | INFO | Uploaded 36GiB to GPU in 2.57665s (13.9716GiB/s) | 20260507 23:39:31.924 | INFO | Downloaded 34GiB to GPU in 2.48319s (13.6921GiB/s) | 20260507 23:39:37.059 | INFO | Analyzing chunk frames = [3511296,3511808) | 20260507 23:39:37.059 | INFO | MemInfo: { | 20260507 23:39:37.059 | INFO | "gpu" : " 17 GB", | 20260507 23:39:37.059 | INFO | "huge" : "", | 20260507 23:39:37.059 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:37.059 | INFO | "malloc" : " 14 GB", | 20260507 23:39:37.059 | INFO | "pinned" : " 29 GB" | 20260507 23:39:37.059 | INFO | } | 20260507 23:39:37.059 | INFO | Perf report for the last 5.13563 seconds: | 20260507 23:39:37.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:37.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.059 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:39:37.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.059 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:39:37.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:37.060 | INFO | Uploaded 36GiB to GPU in 2.50542s (14.3688GiB/s) | 20260507 23:39:37.060 | INFO | Downloaded 34GiB to GPU in 2.47772s (13.7223GiB/s) | 20260507 23:39:42.199 | INFO | Analyzing chunk frames = [3511808,3512320) | 20260507 23:39:42.199 | INFO | MemInfo: { | 20260507 23:39:42.199 | INFO | "gpu" : " 17 GB", | 20260507 23:39:42.199 | INFO | "huge" : "", | 20260507 23:39:42.199 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:42.199 | INFO | "malloc" : " 14 GB", | 20260507 23:39:42.199 | INFO | "pinned" : " 29 GB" | 20260507 23:39:42.199 | INFO | } | 20260507 23:39:42.199 | INFO | Perf report for the last 5.13973 seconds: | 20260507 23:39:42.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:42.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:39:42.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:39:42.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:42.199 | INFO | Uploaded 36GiB to GPU in 2.50082s (14.3953GiB/s) | 20260507 23:39:42.199 | INFO | Downloaded 34GiB to GPU in 2.48531s (13.6804GiB/s) | 20260507 23:39:47.340 | INFO | Analyzing chunk frames = [3512320,3512832) | 20260507 23:39:47.340 | INFO | MemInfo: { | 20260507 23:39:47.340 | INFO | "gpu" : " 17 GB", | 20260507 23:39:47.340 | INFO | "huge" : "", | 20260507 23:39:47.340 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:47.340 | INFO | "malloc" : " 14 GB", | 20260507 23:39:47.340 | INFO | "pinned" : " 29 GB" | 20260507 23:39:47.340 | INFO | } | 20260507 23:39:47.340 | INFO | Perf report for the last 5.1412 seconds: | 20260507 23:39:47.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:47.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.340 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 23:39:47.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.340 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:39:47.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:47.341 | INFO | Uploaded 36GiB to GPU in 2.37318s (15.1695GiB/s) | 20260507 23:39:47.341 | INFO | Downloaded 34GiB to GPU in 2.5048s (13.5739GiB/s) | 20260507 23:39:52.479 | INFO | Analyzing chunk frames = [3512832,3513344) | 20260507 23:39:52.480 | INFO | MemInfo: { | 20260507 23:39:52.480 | INFO | "gpu" : " 17 GB", | 20260507 23:39:52.480 | INFO | "huge" : "", | 20260507 23:39:52.480 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:52.480 | INFO | "malloc" : " 14 GB", | 20260507 23:39:52.480 | INFO | "pinned" : " 29 GB" | 20260507 23:39:52.480 | INFO | } | 20260507 23:39:52.480 | INFO | Perf report for the last 5.13983 seconds: | 20260507 23:39:52.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:52.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.480 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 23:39:52.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.480 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260507 23:39:52.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:52.481 | INFO | Uploaded 36GiB to GPU in 2.43834s (14.7641GiB/s) | 20260507 23:39:52.481 | INFO | Downloaded 34GiB to GPU in 2.38927s (14.2303GiB/s) | 20260507 23:39:57.615 | INFO | Analyzing chunk frames = [3513344,3513856) | 20260507 23:39:57.616 | INFO | MemInfo: { | 20260507 23:39:57.616 | INFO | "gpu" : " 17 GB", | 20260507 23:39:57.616 | INFO | "huge" : "", | 20260507 23:39:57.616 | INFO | "huge_shared" : " 29 GB", | 20260507 23:39:57.616 | INFO | "malloc" : " 14 GB", | 20260507 23:39:57.616 | INFO | "pinned" : " 29 GB" | 20260507 23:39:57.616 | INFO | } | 20260507 23:39:57.616 | INFO | Perf report for the last 5.13599 seconds: | 20260507 23:39:57.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:39:57.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.616 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:39:57.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.616 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 23:39:57.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:39:57.617 | INFO | Uploaded 36GiB to GPU in 2.56547s (14.0325GiB/s) | 20260507 23:39:57.617 | INFO | Downloaded 34GiB to GPU in 2.45284s (13.8615GiB/s) | 20260507 23:40:02.750 | INFO | Analyzing chunk frames = [3513856,3514368) | 20260507 23:40:02.750 | INFO | MemInfo: { | 20260507 23:40:02.750 | INFO | "gpu" : " 17 GB", | 20260507 23:40:02.750 | INFO | "huge" : "", | 20260507 23:40:02.751 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:02.751 | INFO | "malloc" : " 14 GB", | 20260507 23:40:02.751 | INFO | "pinned" : " 29 GB" | 20260507 23:40:02.751 | INFO | } | 20260507 23:40:02.751 | INFO | Perf report for the last 5.13471 seconds: | 20260507 23:40:02.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:02.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:40:02.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 23:40:02.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:02.751 | INFO | Uploaded 36GiB to GPU in 2.56208s (14.0511GiB/s) | 20260507 23:40:02.751 | INFO | Downloaded 34GiB to GPU in 2.4496s (13.8798GiB/s) | 20260507 23:40:02.752 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:40:02.752 | INFO | Total mem usage: 16.7 GiB | 20260507 23:40:02.752 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:40:02.752 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260507 23:40:02.752 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:40:02.752 | INFO | High Water Breakdown: | 20260507 23:40:02.752 | INFO | 60% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:40:02.752 | INFO | 39.9% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:40:02.752 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:40:02.752 | INFO | ------------------------------------------ | 20260507 23:40:02.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:40:02.752 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:40:02.752 | INFO | Total mem usage: 26.5 GiB | 20260507 23:40:02.752 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:40:02.752 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 23:40:02.752 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260507 23:40:02.752 | INFO | High Water Breakdown: | 20260507 23:40:02.752 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:40:02.752 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:40:02.752 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:40:02.752 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:40:02.752 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:40:02.752 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:40:02.752 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:40:02.752 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:40:02.752 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:40:02.752 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:40:02.752 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:40:02.752 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:40:02.752 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:40:02.752 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:40:02.752 | INFO | ------------------------------------------ | 20260507 23:40:02.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:40:02.752 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:40:02.752 | INFO | Total mem usage: 27.8 GiB | 20260507 23:40:02.752 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:40:02.752 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:40:02.752 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:40:02.752 | INFO | High Water Breakdown: | 20260507 23:40:02.752 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:40:02.752 | INFO | ------------------------------------------ | 20260507 23:40:02.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:40:02.752 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:40:02.752 | INFO | Total mem usage: 15.7 GiB | 20260507 23:40:02.752 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:40:02.752 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:40:02.752 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:40:02.752 | INFO | High Water Breakdown: | 20260507 23:40:02.752 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:40:02.752 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:40:02.752 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:40:02.752 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:40:02.753 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:40:02.753 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:40:02.753 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:40:02.753 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:40:02.753 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:40:02.753 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:40:02.753 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:40:02.753 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:40:02.753 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:40:02.753 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:40:02.753 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:40:02.753 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:40:02.753 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:40:02.753 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:40:02.753 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:40:02.753 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:40:02.753 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:40:02.753 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:40:02.753 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:40:02.753 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:40:02.753 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:40:02.753 | INFO | ------------------------------------------ | 20260507 23:40:02.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:40:02.753 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:40:02.753 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:40:02.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:40:02.753 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.59e+01, 1.41e+01 / 1.42e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.11e-02 / 6.57e-03 | 20260507 23:40:02.753 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.45e+00, 5.88e-01 / 2.68e-01 | 20260507 23:40:02.753 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.09e-01 | 20260507 23:40:02.753 | INFO | PerfReport: 16.04%, DME -- 4.22e+01 / 4.58e+01, 4.00e+00 / 1.58e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.64e+00 / 1.49e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.46e+00 / 2.74e+00, 4.30e-01 / 2.58e-01 | 20260507 23:40:02.753 | INFO | PerfReport: 3.43%, Metrics -- 1.13e+00 / 1.35e+00, 2.82e-01 / 1.82e-01 | 20260507 23:40:02.753 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.87e-01, 5.53e-02 / 2.85e-02 | 20260507 23:40:02.753 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:40:02.753 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:40:02.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:40:02.753 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 24.28%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 30.82%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 2.14%, LockUpload -- 2.28e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 15.74%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:40:02.753 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.18e-02 / 0.00e+00 | 20260507 23:40:02.862 | INFO | Starting a round of Baz IO | 20260507 23:40:06.337 | INFO | Populating RTMetrics report | 20260507 23:40:06.338 | INFO | RTMetrics report populated | 20260507 23:40:06.338 | INFO | Opening tmp RTMetrics json file | 20260507 23:40:06.339 | INFO | Serializing RTMetrics to json | 20260507 23:40:06.341 | INFO | Serialization done, swapping tmp file now... | 20260507 23:40:06.342 | INFO | RTMetrics done | 20260507 23:40:06.424 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.561 seconds | 20260507 23:40:06.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.51% / 0.01% / 57.48% | 20260507 23:40:06.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15364% / 0% / 5.96046e-05% | 20260507 23:40:06.424 | INFO | Baz IO overall: | 20260507 23:40:06.424 | INFO | BytesWritten : 1599481590557 | 20260507 23:40:06.424 | INFO | NumEvents : 774485080217 | 20260507 23:40:07.892 | INFO | Analyzing chunk frames = [3514368,3514880) | 20260507 23:40:07.892 | INFO | MemInfo: { | 20260507 23:40:07.892 | INFO | "gpu" : " 17 GB", | 20260507 23:40:07.892 | INFO | "huge" : "", | 20260507 23:40:07.892 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:07.892 | INFO | "malloc" : " 14 GB", | 20260507 23:40:07.892 | INFO | "pinned" : " 29 GB" | 20260507 23:40:07.892 | INFO | } | 20260507 23:40:07.892 | INFO | Perf report for the last 5.14095 seconds: | 20260507 23:40:07.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:07.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:07.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:07.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:07.892 | INFO | ANALYSIS : 69.3%, 2.9, 105.0ms, 0 | 20260507 23:40:07.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:07.892 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 23:40:07.892 | INFO | PRE_HQ : 68.1%, 1.8, 62.7ms, 0 | 20260507 23:40:07.892 | INFO | BAZWRITER : 68.8%, 4.1, 146.7ms, 0 | 20260507 23:40:07.892 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260507 23:40:07.892 | INFO | Uploaded 36GiB to GPU in 2.53046s (14.2267GiB/s) | 20260507 23:40:07.892 | INFO | Downloaded 38GiB to GPU in 2.70293s (14.0588GiB/s) | 20260507 23:40:13.036 | INFO | Analyzing chunk frames = [3514880,3515392) | 20260507 23:40:13.036 | INFO | MemInfo: { | 20260507 23:40:13.036 | INFO | "gpu" : " 17 GB", | 20260507 23:40:13.036 | INFO | "huge" : "", | 20260507 23:40:13.037 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:13.037 | INFO | "malloc" : " 14 GB", | 20260507 23:40:13.037 | INFO | "pinned" : " 29 GB" | 20260507 23:40:13.037 | INFO | } | 20260507 23:40:13.037 | INFO | Perf report for the last 5.14499 seconds: | 20260507 23:40:13.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:13.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260507 23:40:13.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:40:13.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:13.037 | INFO | Uploaded 36GiB to GPU in 2.58739s (13.9136GiB/s) | 20260507 23:40:13.037 | INFO | Downloaded 34GiB to GPU in 2.47537s (13.7353GiB/s) | 20260507 23:40:18.175 | INFO | Analyzing chunk frames = [3515392,3515904) | 20260507 23:40:18.175 | INFO | MemInfo: { | 20260507 23:40:18.175 | INFO | "gpu" : " 17 GB", | 20260507 23:40:18.175 | INFO | "huge" : "", | 20260507 23:40:18.175 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:18.175 | INFO | "malloc" : " 14 GB", | 20260507 23:40:18.175 | INFO | "pinned" : " 29 GB" | 20260507 23:40:18.175 | INFO | } | 20260507 23:40:18.175 | INFO | Perf report for the last 5.1386 seconds: | 20260507 23:40:18.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:18.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.175 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260507 23:40:18.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.175 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:40:18.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:18.176 | INFO | Uploaded 36GiB to GPU in 2.53441s (14.2045GiB/s) | 20260507 23:40:18.176 | INFO | Downloaded 34GiB to GPU in 2.47605s (13.7315GiB/s) | 20260507 23:40:23.311 | INFO | Analyzing chunk frames = [3515904,3516416) | 20260507 23:40:23.311 | INFO | MemInfo: { | 20260507 23:40:23.311 | INFO | "gpu" : " 17 GB", | 20260507 23:40:23.311 | INFO | "huge" : "", | 20260507 23:40:23.311 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:23.311 | INFO | "malloc" : " 14 GB", | 20260507 23:40:23.311 | INFO | "pinned" : " 29 GB" | 20260507 23:40:23.311 | INFO | } | 20260507 23:40:23.311 | INFO | Perf report for the last 5.13566 seconds: | 20260507 23:40:23.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:23.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | ANALYSIS : 67.6%, 3.0, 104.5ms, 0 | 20260507 23:40:23.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:40:23.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:23.311 | INFO | Uploaded 36GiB to GPU in 2.55676s (14.0803GiB/s) | 20260507 23:40:23.311 | INFO | Downloaded 34GiB to GPU in 2.46919s (13.7697GiB/s) | 20260507 23:40:28.453 | INFO | Analyzing chunk frames = [3516416,3516928) | 20260507 23:40:28.453 | INFO | MemInfo: { | 20260507 23:40:28.453 | INFO | "gpu" : " 17 GB", | 20260507 23:40:28.453 | INFO | "huge" : "", | 20260507 23:40:28.453 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:28.453 | INFO | "malloc" : " 14 GB", | 20260507 23:40:28.454 | INFO | "pinned" : " 29 GB" | 20260507 23:40:28.454 | INFO | } | 20260507 23:40:28.454 | INFO | Perf report for the last 5.1426 seconds: | 20260507 23:40:28.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:28.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:40:28.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:40:28.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:28.454 | INFO | Uploaded 36GiB to GPU in 2.45377s (14.6713GiB/s) | 20260507 23:40:28.454 | INFO | Downloaded 34GiB to GPU in 2.49473s (13.6287GiB/s) | 20260507 23:40:33.591 | INFO | Analyzing chunk frames = [3516928,3517440) | 20260507 23:40:33.591 | INFO | MemInfo: { | 20260507 23:40:33.591 | INFO | "gpu" : " 17 GB", | 20260507 23:40:33.591 | INFO | "huge" : "", | 20260507 23:40:33.591 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:33.591 | INFO | "malloc" : " 14 GB", | 20260507 23:40:33.591 | INFO | "pinned" : " 29 GB" | 20260507 23:40:33.591 | INFO | } | 20260507 23:40:33.591 | INFO | Perf report for the last 5.13785 seconds: | 20260507 23:40:33.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:33.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.592 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:40:33.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.592 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:40:33.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:33.592 | INFO | Uploaded 36GiB to GPU in 2.40872s (14.9457GiB/s) | 20260507 23:40:33.592 | INFO | Downloaded 34GiB to GPU in 2.39668s (14.1863GiB/s) | 20260507 23:40:38.734 | INFO | Analyzing chunk frames = [3517440,3517952) | 20260507 23:40:38.734 | INFO | MemInfo: { | 20260507 23:40:38.734 | INFO | "gpu" : " 17 GB", | 20260507 23:40:38.734 | INFO | "huge" : "", | 20260507 23:40:38.734 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:38.734 | INFO | "malloc" : " 14 GB", | 20260507 23:40:38.734 | INFO | "pinned" : " 29 GB" | 20260507 23:40:38.734 | INFO | } | 20260507 23:40:38.734 | INFO | Perf report for the last 5.14272 seconds: | 20260507 23:40:38.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:38.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.734 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260507 23:40:38.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.734 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 23:40:38.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:38.735 | INFO | Uploaded 36GiB to GPU in 2.50664s (14.3619GiB/s) | 20260507 23:40:38.735 | INFO | Downloaded 34GiB to GPU in 2.44218s (13.922GiB/s) | 20260507 23:40:43.872 | INFO | Analyzing chunk frames = [3517952,3518464) | 20260507 23:40:43.872 | INFO | MemInfo: { | 20260507 23:40:43.872 | INFO | "gpu" : " 17 GB", | 20260507 23:40:43.872 | INFO | "huge" : "", | 20260507 23:40:43.872 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:43.872 | INFO | "malloc" : " 14 GB", | 20260507 23:40:43.872 | INFO | "pinned" : " 29 GB" | 20260507 23:40:43.872 | INFO | } | 20260507 23:40:43.872 | INFO | Perf report for the last 5.13763 seconds: | 20260507 23:40:43.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:43.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 23:40:43.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260507 23:40:43.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:43.872 | INFO | Uploaded 36GiB to GPU in 2.5596s (14.0647GiB/s) | 20260507 23:40:43.872 | INFO | Downloaded 34GiB to GPU in 2.45962s (13.8233GiB/s) | 20260507 23:40:43.988 | INFO | Starting a round of Baz IO | 20260507 23:40:47.445 | INFO | Populating RTMetrics report | 20260507 23:40:47.445 | INFO | RTMetrics report populated | 20260507 23:40:47.445 | INFO | Opening tmp RTMetrics json file | 20260507 23:40:47.475 | INFO | Serializing RTMetrics to json | 20260507 23:40:47.501 | INFO | Serialization done, swapping tmp file now... | 20260507 23:40:47.502 | INFO | RTMetrics done | 20260507 23:40:47.630 | INFO | Finished a round of Baz IO: Wrote 1.707 GiB in 3.642 seconds | 20260507 23:40:47.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.51% / 0.00% / 57.49% | 20260507 23:40:47.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.14475% / 1.58946e-05% / 6.35783e-05% | 20260507 23:40:47.630 | INFO | Baz IO overall: | 20260507 23:40:47.630 | INFO | BytesWritten : 1601314042670 | 20260507 23:40:47.630 | INFO | NumEvents : 775366907626 | 20260507 23:40:49.011 | INFO | Analyzing chunk frames = [3518464,3518976) | 20260507 23:40:49.011 | INFO | MemInfo: { | 20260507 23:40:49.011 | INFO | "gpu" : " 17 GB", | 20260507 23:40:49.011 | INFO | "huge" : "", | 20260507 23:40:49.011 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:49.011 | INFO | "malloc" : " 14 GB", | 20260507 23:40:49.011 | INFO | "pinned" : " 29 GB" | 20260507 23:40:49.011 | INFO | } | 20260507 23:40:49.011 | INFO | Perf report for the last 5.13936 seconds: | 20260507 23:40:49.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:49.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:49.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:49.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:49.011 | INFO | ANALYSIS : 68.7%, 2.9, 104.2ms, 0 | 20260507 23:40:49.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:49.011 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 23:40:49.011 | INFO | PRE_HQ : 67.5%, 1.8, 63.4ms, 0 | 20260507 23:40:49.011 | INFO | BAZWRITER : 70.4%, 4.6, 165.0ms, 0 | 20260507 23:40:49.011 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:40:49.012 | INFO | Uploaded 36GiB to GPU in 2.51991s (14.2862GiB/s) | 20260507 23:40:49.012 | INFO | Downloaded 38GiB to GPU in 2.71373s (14.0029GiB/s) | 20260507 23:40:54.151 | INFO | Analyzing chunk frames = [3518976,3519488) | 20260507 23:40:54.151 | INFO | MemInfo: { | 20260507 23:40:54.151 | INFO | "gpu" : " 17 GB", | 20260507 23:40:54.151 | INFO | "huge" : "", | 20260507 23:40:54.151 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:54.151 | INFO | "malloc" : " 14 GB", | 20260507 23:40:54.151 | INFO | "pinned" : " 29 GB" | 20260507 23:40:54.151 | INFO | } | 20260507 23:40:54.151 | INFO | Perf report for the last 5.13959 seconds: | 20260507 23:40:54.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:54.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 23:40:54.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:40:54.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:54.151 | INFO | Uploaded 36GiB to GPU in 2.45306s (14.6756GiB/s) | 20260507 23:40:54.151 | INFO | Downloaded 34GiB to GPU in 2.49071s (13.6507GiB/s) | 20260507 23:40:59.288 | INFO | Analyzing chunk frames = [3519488,3520000) | 20260507 23:40:59.288 | INFO | MemInfo: { | 20260507 23:40:59.288 | INFO | "gpu" : " 17 GB", | 20260507 23:40:59.288 | INFO | "huge" : "", | 20260507 23:40:59.288 | INFO | "huge_shared" : " 29 GB", | 20260507 23:40:59.288 | INFO | "malloc" : " 14 GB", | 20260507 23:40:59.288 | INFO | "pinned" : " 29 GB" | 20260507 23:40:59.288 | INFO | } | 20260507 23:40:59.288 | INFO | Perf report for the last 5.13731 seconds: | 20260507 23:40:59.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:40:59.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.288 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:40:59.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.288 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:40:59.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:40:59.289 | INFO | Uploaded 36GiB to GPU in 2.45969s (14.636GiB/s) | 20260507 23:40:59.289 | INFO | Downloaded 34GiB to GPU in 2.48514s (13.6813GiB/s) | 20260507 23:41:04.429 | INFO | Analyzing chunk frames = [3520000,3520512) | 20260507 23:41:04.429 | INFO | MemInfo: { | 20260507 23:41:04.429 | INFO | "gpu" : " 17 GB", | 20260507 23:41:04.429 | INFO | "huge" : "", | 20260507 23:41:04.429 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:04.429 | INFO | "malloc" : " 14 GB", | 20260507 23:41:04.429 | INFO | "pinned" : " 29 GB" | 20260507 23:41:04.429 | INFO | } | 20260507 23:41:04.430 | INFO | Perf report for the last 5.14139 seconds: | 20260507 23:41:04.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:04.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260507 23:41:04.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:41:04.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:04.430 | INFO | Uploaded 36GiB to GPU in 2.44891s (14.7004GiB/s) | 20260507 23:41:04.430 | INFO | Downloaded 34GiB to GPU in 2.41201s (14.0961GiB/s) | 20260507 23:41:09.565 | INFO | Analyzing chunk frames = [3520512,3521024) | 20260507 23:41:09.565 | INFO | MemInfo: { | 20260507 23:41:09.565 | INFO | "gpu" : " 17 GB", | 20260507 23:41:09.565 | INFO | "huge" : "", | 20260507 23:41:09.565 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:09.565 | INFO | "malloc" : " 14 GB", | 20260507 23:41:09.565 | INFO | "pinned" : " 29 GB" | 20260507 23:41:09.565 | INFO | } | 20260507 23:41:09.565 | INFO | Perf report for the last 5.13593 seconds: | 20260507 23:41:09.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:09.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.566 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260507 23:41:09.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.566 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:41:09.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:09.566 | INFO | Uploaded 36GiB to GPU in 2.30865s (15.5936GiB/s) | 20260507 23:41:09.566 | INFO | Downloaded 34GiB to GPU in 2.49738s (13.6143GiB/s) | 20260507 23:41:14.704 | INFO | Analyzing chunk frames = [3521024,3521536) | 20260507 23:41:14.704 | INFO | MemInfo: { | 20260507 23:41:14.704 | INFO | "gpu" : " 17 GB", | 20260507 23:41:14.704 | INFO | "huge" : "", | 20260507 23:41:14.704 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:14.704 | INFO | "malloc" : " 14 GB", | 20260507 23:41:14.704 | INFO | "pinned" : " 29 GB" | 20260507 23:41:14.704 | INFO | } | 20260507 23:41:14.704 | INFO | Perf report for the last 5.13833 seconds: | 20260507 23:41:14.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:14.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 23:41:14.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:41:14.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:14.704 | INFO | Uploaded 36GiB to GPU in 2.40998s (14.9379GiB/s) | 20260507 23:41:14.704 | INFO | Downloaded 34GiB to GPU in 2.40114s (14.1599GiB/s) | 20260507 23:41:19.844 | INFO | Analyzing chunk frames = [3521536,3522048) | 20260507 23:41:19.844 | INFO | MemInfo: { | 20260507 23:41:19.844 | INFO | "gpu" : " 17 GB", | 20260507 23:41:19.844 | INFO | "huge" : "", | 20260507 23:41:19.844 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:19.844 | INFO | "malloc" : " 14 GB", | 20260507 23:41:19.845 | INFO | "pinned" : " 29 GB" | 20260507 23:41:19.845 | INFO | } | 20260507 23:41:19.845 | INFO | Perf report for the last 5.14082 seconds: | 20260507 23:41:19.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:19.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:41:19.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 23:41:19.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:19.845 | INFO | Uploaded 36GiB to GPU in 2.54475s (14.1468GiB/s) | 20260507 23:41:19.845 | INFO | Downloaded 34GiB to GPU in 2.46923s (13.7695GiB/s) | 20260507 23:41:24.985 | INFO | Analyzing chunk frames = [3522048,3522560) | 20260507 23:41:24.985 | INFO | MemInfo: { | 20260507 23:41:24.985 | INFO | "gpu" : " 17 GB", | 20260507 23:41:24.985 | INFO | "huge" : "", | 20260507 23:41:24.985 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:24.985 | INFO | "malloc" : " 14 GB", | 20260507 23:41:24.985 | INFO | "pinned" : " 29 GB" | 20260507 23:41:24.985 | INFO | } | 20260507 23:41:24.985 | INFO | Perf report for the last 5.14031 seconds: | 20260507 23:41:24.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:24.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.985 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:41:24.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.985 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260507 23:41:24.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:24.986 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:41:24.986 | INFO | Total mem usage: 16.7 GiB | 20260507 23:41:24.986 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:41:24.986 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260507 23:41:24.986 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:41:24.986 | INFO | High Water Breakdown: | 20260507 23:41:24.986 | INFO | 60.1% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:41:24.986 | INFO | 39.8% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:41:24.986 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:41:24.986 | INFO | ------------------------------------------ | 20260507 23:41:24.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:41:24.986 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:41:24.986 | INFO | Total mem usage: 26.5 GiB | 20260507 23:41:24.986 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:41:24.986 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:41:24.986 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:41:24.986 | INFO | Uploaded 36GiB to GPU in 2.50547s (14.3685GiB/s) | 20260507 23:41:24.986 | INFO | Downloaded 34GiB to GPU in 2.45057s (13.8743GiB/s) | 20260507 23:41:24.986 | INFO | High Water Breakdown: | 20260507 23:41:24.986 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:41:24.986 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:41:24.986 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:41:24.986 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:41:24.986 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:41:24.986 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:41:24.986 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:41:24.986 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:41:24.986 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:41:24.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:41:24.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:41:24.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:41:24.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:41:24.986 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:41:24.986 | INFO | ------------------------------------------ | 20260507 23:41:24.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:41:24.986 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:41:24.986 | INFO | Total mem usage: 27.8 GiB | 20260507 23:41:24.986 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:41:24.986 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:41:24.986 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:41:24.986 | INFO | High Water Breakdown: | 20260507 23:41:24.986 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:41:24.986 | INFO | ------------------------------------------ | 20260507 23:41:24.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:41:24.986 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:41:24.986 | INFO | Total mem usage: 15.7 GiB | 20260507 23:41:24.986 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:41:24.986 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:41:24.986 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:41:24.986 | INFO | High Water Breakdown: | 20260507 23:41:24.986 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:41:24.986 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:41:24.986 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:41:24.986 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:41:24.986 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:41:24.986 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:41:24.986 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:41:24.986 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:41:24.986 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:41:24.986 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:41:24.986 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:41:24.986 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:41:24.986 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:41:24.986 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:41:24.986 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:41:24.986 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:41:24.986 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:41:24.986 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:41:24.986 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:41:24.986 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:41:24.986 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:41:24.986 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:41:24.986 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:41:24.986 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:41:24.986 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:41:24.987 | INFO | ------------------------------------------ | 20260507 23:41:24.987 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:41:24.987 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:41:24.987 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:41:24.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:41:24.987 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.41e+01 / 1.44e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.17e-02 / 5.36e-03 | 20260507 23:41:24.987 | INFO | PerfReport: 12.15%, Baseline -- 4.00e+00 / 4.47e+00, 5.87e-01 / 2.33e-01 | 20260507 23:41:24.987 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.50e+00, 2.07e-01 / 9.85e-02 | 20260507 23:41:24.987 | INFO | PerfReport: 16.04%, DME -- 4.22e+01 / 4.60e+01, 4.23e+00 / 1.48e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.74e+00 / 1.49e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.72e+00, 4.30e-01 / 2.27e-01 | 20260507 23:41:24.987 | INFO | PerfReport: 3.39%, Metrics -- 1.12e+00 / 1.32e+00, 2.76e-01 / 1.33e-01 | 20260507 23:41:24.987 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.47e-01 / 3.80e-01, 5.47e-02 / 2.13e-02 | 20260507 23:41:24.987 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:41:24.987 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:41:24.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:41:24.987 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 24.36%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 30.90%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 2.68%, LockDownload -- 2.85e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 15.63%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:41:24.987 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260507 23:41:25.097 | INFO | Starting a round of Baz IO | 20260507 23:41:28.551 | INFO | Populating RTMetrics report | 20260507 23:41:28.551 | INFO | RTMetrics report populated | 20260507 23:41:28.551 | INFO | Opening tmp RTMetrics json file | 20260507 23:41:28.575 | INFO | Serializing RTMetrics to json | 20260507 23:41:28.577 | INFO | Serialization done, swapping tmp file now... | 20260507 23:41:28.577 | INFO | RTMetrics done | 20260507 23:41:28.655 | INFO | Finished a round of Baz IO: Wrote 1.708 GiB in 3.557 seconds | 20260507 23:41:28.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.50% / 0.01% / 57.49% | 20260507 23:41:28.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.14456% / 3.97364e-06% / 8.74201e-05% | 20260507 23:41:28.655 | INFO | Baz IO overall: | 20260507 23:41:28.655 | INFO | BytesWritten : 1603148033430 | 20260507 23:41:28.655 | INFO | NumEvents : 776249983514 | 20260507 23:41:30.124 | INFO | Analyzing chunk frames = [3522560,3523072) | 20260507 23:41:30.124 | INFO | MemInfo: { | 20260507 23:41:30.124 | INFO | "gpu" : " 17 GB", | 20260507 23:41:30.124 | INFO | "huge" : "", | 20260507 23:41:30.124 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:30.124 | INFO | "malloc" : " 14 GB", | 20260507 23:41:30.124 | INFO | "pinned" : " 29 GB" | 20260507 23:41:30.124 | INFO | } | 20260507 23:41:30.124 | INFO | Perf report for the last 5.13932 seconds: | 20260507 23:41:30.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:30.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:30.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:30.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:30.124 | INFO | ANALYSIS : 68.7%, 2.9, 105.1ms, 0 | 20260507 23:41:30.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:30.125 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 23:41:30.125 | INFO | PRE_HQ : 67.2%, 1.8, 61.5ms, 0 | 20260507 23:41:30.125 | INFO | BAZWRITER : 68.8%, 4.3, 152.8ms, 0 | 20260507 23:41:30.125 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:41:30.125 | INFO | Uploaded 36GiB to GPU in 2.5897s (13.9012GiB/s) | 20260507 23:41:30.125 | INFO | Downloaded 38GiB to GPU in 2.7029s (14.059GiB/s) | 20260507 23:41:35.262 | INFO | Analyzing chunk frames = [3523072,3523584) | 20260507 23:41:35.262 | INFO | MemInfo: { | 20260507 23:41:35.262 | INFO | "gpu" : " 17 GB", | 20260507 23:41:35.262 | INFO | "huge" : "", | 20260507 23:41:35.262 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:35.262 | INFO | "malloc" : " 14 GB", | 20260507 23:41:35.262 | INFO | "pinned" : " 29 GB" | 20260507 23:41:35.262 | INFO | } | 20260507 23:41:35.262 | INFO | Perf report for the last 5.13761 seconds: | 20260507 23:41:35.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:35.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260507 23:41:35.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 23:41:35.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:35.262 | INFO | Uploaded 36GiB to GPU in 2.57264s (13.9934GiB/s) | 20260507 23:41:35.262 | INFO | Downloaded 34GiB to GPU in 2.47422s (13.7417GiB/s) | 20260507 23:41:40.406 | INFO | Analyzing chunk frames = [3523584,3524096) | 20260507 23:41:40.406 | INFO | MemInfo: { | 20260507 23:41:40.406 | INFO | "gpu" : " 17 GB", | 20260507 23:41:40.406 | INFO | "huge" : "", | 20260507 23:41:40.406 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:40.406 | INFO | "malloc" : " 14 GB", | 20260507 23:41:40.406 | INFO | "pinned" : " 29 GB" | 20260507 23:41:40.406 | INFO | } | 20260507 23:41:40.406 | INFO | Perf report for the last 5.14413 seconds: | 20260507 23:41:40.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:40.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 23:41:40.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:41:40.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:40.406 | INFO | Uploaded 36GiB to GPU in 2.58876s (13.9063GiB/s) | 20260507 23:41:40.406 | INFO | Downloaded 34GiB to GPU in 2.46902s (13.7706GiB/s) | 20260507 23:41:45.544 | INFO | Analyzing chunk frames = [3524096,3524608) | 20260507 23:41:45.544 | INFO | MemInfo: { | 20260507 23:41:45.544 | INFO | "gpu" : " 17 GB", | 20260507 23:41:45.544 | INFO | "huge" : "", | 20260507 23:41:45.544 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:45.544 | INFO | "malloc" : " 14 GB", | 20260507 23:41:45.544 | INFO | "pinned" : " 29 GB" | 20260507 23:41:45.544 | INFO | } | 20260507 23:41:45.545 | INFO | Perf report for the last 5.13846 seconds: | 20260507 23:41:45.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:45.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 23:41:45.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:41:45.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:45.545 | INFO | Uploaded 36GiB to GPU in 2.54388s (14.1516GiB/s) | 20260507 23:41:45.545 | INFO | Downloaded 34GiB to GPU in 2.46707s (13.7815GiB/s) | 20260507 23:41:50.681 | INFO | Analyzing chunk frames = [3524608,3525120) | 20260507 23:41:50.681 | INFO | MemInfo: { | 20260507 23:41:50.681 | INFO | "gpu" : " 17 GB", | 20260507 23:41:50.681 | INFO | "huge" : "", | 20260507 23:41:50.681 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:50.681 | INFO | "malloc" : " 14 GB", | 20260507 23:41:50.681 | INFO | "pinned" : " 29 GB" | 20260507 23:41:50.681 | INFO | } | 20260507 23:41:50.681 | INFO | Perf report for the last 5.1368 seconds: | 20260507 23:41:50.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:50.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.681 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:41:50.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.681 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:41:50.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:50.682 | INFO | Uploaded 36GiB to GPU in 2.33581s (15.4122GiB/s) | 20260507 23:41:50.682 | INFO | Downloaded 34GiB to GPU in 2.46317s (13.8033GiB/s) | 20260507 23:41:55.818 | INFO | Analyzing chunk frames = [3525120,3525632) | 20260507 23:41:55.818 | INFO | MemInfo: { | 20260507 23:41:55.818 | INFO | "gpu" : " 17 GB", | 20260507 23:41:55.818 | INFO | "huge" : "", | 20260507 23:41:55.818 | INFO | "huge_shared" : " 29 GB", | 20260507 23:41:55.818 | INFO | "malloc" : " 14 GB", | 20260507 23:41:55.819 | INFO | "pinned" : " 29 GB" | 20260507 23:41:55.819 | INFO | } | 20260507 23:41:55.819 | INFO | Perf report for the last 5.13721 seconds: | 20260507 23:41:55.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:41:55.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:41:55.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:41:55.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:41:55.819 | INFO | Uploaded 36GiB to GPU in 2.39745s (15.0159GiB/s) | 20260507 23:41:55.819 | INFO | Downloaded 34GiB to GPU in 2.4046s (14.1395GiB/s) | 20260507 23:42:00.963 | INFO | Analyzing chunk frames = [3525632,3526144) | 20260507 23:42:00.963 | INFO | MemInfo: { | 20260507 23:42:00.963 | INFO | "gpu" : " 17 GB", | 20260507 23:42:00.963 | INFO | "huge" : "", | 20260507 23:42:00.963 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:00.963 | INFO | "malloc" : " 14 GB", | 20260507 23:42:00.963 | INFO | "pinned" : " 29 GB" | 20260507 23:42:00.963 | INFO | } | 20260507 23:42:00.963 | INFO | Perf report for the last 5.14444 seconds: | 20260507 23:42:00.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:00.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.963 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260507 23:42:00.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.963 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 23:42:00.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:00.964 | INFO | Uploaded 36GiB to GPU in 2.50646s (14.3629GiB/s) | 20260507 23:42:00.964 | INFO | Downloaded 34GiB to GPU in 2.44635s (13.8983GiB/s) | 20260507 23:42:06.103 | INFO | Analyzing chunk frames = [3526144,3526656) | 20260507 23:42:06.103 | INFO | MemInfo: { | 20260507 23:42:06.103 | INFO | "gpu" : " 17 GB", | 20260507 23:42:06.103 | INFO | "huge" : "", | 20260507 23:42:06.103 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:06.103 | INFO | "malloc" : " 14 GB", | 20260507 23:42:06.103 | INFO | "pinned" : " 29 GB" | 20260507 23:42:06.103 | INFO | } | 20260507 23:42:06.104 | INFO | Perf report for the last 5.14038 seconds: | 20260507 23:42:06.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:06.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260507 23:42:06.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260507 23:42:06.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:06.104 | INFO | Uploaded 36GiB to GPU in 2.52706s (14.2458GiB/s) | 20260507 23:42:06.104 | INFO | Downloaded 34GiB to GPU in 2.4508s (13.873GiB/s) | 20260507 23:42:06.216 | INFO | Starting a round of Baz IO | 20260507 23:42:09.698 | INFO | Populating RTMetrics report | 20260507 23:42:09.699 | INFO | RTMetrics report populated | 20260507 23:42:09.699 | INFO | Opening tmp RTMetrics json file | 20260507 23:42:09.720 | INFO | Serializing RTMetrics to json | 20260507 23:42:09.853 | INFO | Serialization done, swapping tmp file now... | 20260507 23:42:09.856 | INFO | RTMetrics done | 20260507 23:42:09.885 | INFO | Finished a round of Baz IO: Wrote 1.708 GiB in 3.669 seconds | 20260507 23:42:09.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.50% / 0.01% / 57.50% | 20260507 23:42:09.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.14139% / 3.97364e-06% / 7.15256e-05% | 20260507 23:42:09.885 | INFO | Baz IO overall: | 20260507 23:42:09.885 | INFO | BytesWritten : 1604982053915 | 20260507 23:42:09.885 | INFO | NumEvents : 777132870137 | 20260507 23:42:11.244 | INFO | Analyzing chunk frames = [3526656,3527168) | 20260507 23:42:11.244 | INFO | MemInfo: { | 20260507 23:42:11.244 | INFO | "gpu" : " 17 GB", | 20260507 23:42:11.244 | INFO | "huge" : "", | 20260507 23:42:11.244 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:11.244 | INFO | "malloc" : " 14 GB", | 20260507 23:42:11.244 | INFO | "pinned" : " 29 GB" | 20260507 23:42:11.244 | INFO | } | 20260507 23:42:11.244 | INFO | Perf report for the last 5.14092 seconds: | 20260507 23:42:11.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:11.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:11.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:11.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:11.244 | INFO | ANALYSIS : 69.3%, 2.9, 104.5ms, 0 | 20260507 23:42:11.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:11.245 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260507 23:42:11.245 | INFO | PRE_HQ : 68.3%, 1.8, 63.7ms, 0 | 20260507 23:42:11.245 | INFO | BAZWRITER : 70.9%, 3.9, 142.6ms, 0 | 20260507 23:42:11.245 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.6ms, 0 | 20260507 23:42:11.245 | INFO | Uploaded 36GiB to GPU in 2.58634s (13.9193GiB/s) | 20260507 23:42:11.245 | INFO | Downloaded 38GiB to GPU in 2.72794s (13.9299GiB/s) | 20260507 23:42:16.377 | INFO | Analyzing chunk frames = [3527168,3527680) | 20260507 23:42:16.377 | INFO | MemInfo: { | 20260507 23:42:16.377 | INFO | "gpu" : " 17 GB", | 20260507 23:42:16.377 | INFO | "huge" : "", | 20260507 23:42:16.377 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:16.377 | INFO | "malloc" : " 14 GB", | 20260507 23:42:16.377 | INFO | "pinned" : " 29 GB" | 20260507 23:42:16.377 | INFO | } | 20260507 23:42:16.377 | INFO | Perf report for the last 5.13305 seconds: | 20260507 23:42:16.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:16.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.378 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260507 23:42:16.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.378 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:42:16.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:16.378 | INFO | Uploaded 36GiB to GPU in 2.53779s (14.1856GiB/s) | 20260507 23:42:16.378 | INFO | Downloaded 34GiB to GPU in 2.49767s (13.6127GiB/s) | 20260507 23:42:21.515 | INFO | Analyzing chunk frames = [3527680,3528192) | 20260507 23:42:21.515 | INFO | MemInfo: { | 20260507 23:42:21.515 | INFO | "gpu" : " 17 GB", | 20260507 23:42:21.515 | INFO | "huge" : "", | 20260507 23:42:21.516 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:21.516 | INFO | "malloc" : " 14 GB", | 20260507 23:42:21.516 | INFO | "pinned" : " 29 GB" | 20260507 23:42:21.516 | INFO | } | 20260507 23:42:21.516 | INFO | Perf report for the last 5.13828 seconds: | 20260507 23:42:21.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:21.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:42:21.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:42:21.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:21.516 | INFO | Uploaded 36GiB to GPU in 2.43506s (14.784GiB/s) | 20260507 23:42:21.516 | INFO | Downloaded 34GiB to GPU in 2.49649s (13.6191GiB/s) | 20260507 23:42:26.655 | INFO | Analyzing chunk frames = [3528192,3528704) | 20260507 23:42:26.655 | INFO | MemInfo: { | 20260507 23:42:26.655 | INFO | "gpu" : " 17 GB", | 20260507 23:42:26.655 | INFO | "huge" : "", | 20260507 23:42:26.655 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:26.655 | INFO | "malloc" : " 14 GB", | 20260507 23:42:26.655 | INFO | "pinned" : " 29 GB" | 20260507 23:42:26.655 | INFO | } | 20260507 23:42:26.656 | INFO | Perf report for the last 5.13972 seconds: | 20260507 23:42:26.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:26.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260507 23:42:26.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:42:26.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:26.656 | INFO | Uploaded 36GiB to GPU in 2.5345s (14.204GiB/s) | 20260507 23:42:26.656 | INFO | Downloaded 34GiB to GPU in 2.49156s (13.6461GiB/s) | 20260507 23:42:31.794 | INFO | Analyzing chunk frames = [3528704,3529216) | 20260507 23:42:31.794 | INFO | MemInfo: { | 20260507 23:42:31.794 | INFO | "gpu" : " 17 GB", | 20260507 23:42:31.794 | INFO | "huge" : "", | 20260507 23:42:31.794 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:31.794 | INFO | "malloc" : " 14 GB", | 20260507 23:42:31.794 | INFO | "pinned" : " 29 GB" | 20260507 23:42:31.794 | INFO | } | 20260507 23:42:31.795 | INFO | Perf report for the last 5.13897 seconds: | 20260507 23:42:31.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:31.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260507 23:42:31.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 23:42:31.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:31.795 | INFO | Uploaded 36GiB to GPU in 2.36871s (15.1982GiB/s) | 20260507 23:42:31.795 | INFO | Downloaded 34GiB to GPU in 2.50092s (13.595GiB/s) | 20260507 23:42:36.938 | INFO | Analyzing chunk frames = [3529216,3529728) | 20260507 23:42:36.938 | INFO | MemInfo: { | 20260507 23:42:36.938 | INFO | "gpu" : " 17 GB", | 20260507 23:42:36.938 | INFO | "huge" : "", | 20260507 23:42:36.938 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:36.938 | INFO | "malloc" : " 14 GB", | 20260507 23:42:36.938 | INFO | "pinned" : " 29 GB" | 20260507 23:42:36.938 | INFO | } | 20260507 23:42:36.938 | INFO | Perf report for the last 5.14376 seconds: | 20260507 23:42:36.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:36.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.938 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:42:36.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.938 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:42:36.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:36.939 | INFO | Uploaded 36GiB to GPU in 2.5163s (14.3067GiB/s) | 20260507 23:42:36.939 | INFO | Downloaded 34GiB to GPU in 2.43536s (13.961GiB/s) | 20260507 23:42:42.072 | INFO | Analyzing chunk frames = [3529728,3530240) | 20260507 23:42:42.072 | INFO | MemInfo: { | 20260507 23:42:42.072 | INFO | "gpu" : " 17 GB", | 20260507 23:42:42.072 | INFO | "huge" : "", | 20260507 23:42:42.072 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:42.072 | INFO | "malloc" : " 14 GB", | 20260507 23:42:42.072 | INFO | "pinned" : " 29 GB" | 20260507 23:42:42.072 | INFO | } | 20260507 23:42:42.072 | INFO | Perf report for the last 5.13402 seconds: | 20260507 23:42:42.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:42.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.072 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:42:42.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.073 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 23:42:42.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:42.073 | INFO | Uploaded 36GiB to GPU in 2.55564s (14.0865GiB/s) | 20260507 23:42:42.073 | INFO | Downloaded 34GiB to GPU in 2.47109s (13.7591GiB/s) | 20260507 23:42:47.216 | INFO | Analyzing chunk frames = [3530240,3530752) | 20260507 23:42:47.216 | INFO | MemInfo: { | 20260507 23:42:47.216 | INFO | "gpu" : " 17 GB", | 20260507 23:42:47.216 | INFO | "huge" : "", | 20260507 23:42:47.216 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:47.216 | INFO | "malloc" : " 14 GB", | 20260507 23:42:47.216 | INFO | "pinned" : " 29 GB" | 20260507 23:42:47.216 | INFO | } | 20260507 23:42:47.216 | INFO | Perf report for the last 5.14398 seconds: | 20260507 23:42:47.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:47.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.216 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:42:47.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.217 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 23:42:47.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:47.217 | INFO | Uploaded 36GiB to GPU in 2.52193s (14.2748GiB/s) | 20260507 23:42:47.217 | INFO | Downloaded 34GiB to GPU in 2.45057s (13.8743GiB/s) | 20260507 23:42:47.217 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:42:47.217 | INFO | Total mem usage: 16.7 GiB | 20260507 23:42:47.217 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:42:47.217 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13.1%) | 20260507 23:42:47.217 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:42:47.217 | INFO | High Water Breakdown: | 20260507 23:42:47.217 | INFO | 59.9% : 8.85 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:42:47.217 | INFO | 39.9% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:42:47.217 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:42:47.217 | INFO | ------------------------------------------ | 20260507 23:42:47.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:42:47.217 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:42:47.217 | INFO | Total mem usage: 26.5 GiB | 20260507 23:42:47.217 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:42:47.217 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:42:47.217 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:42:47.217 | INFO | High Water Breakdown: | 20260507 23:42:47.217 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:42:47.217 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:42:47.217 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:42:47.217 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:42:47.217 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:42:47.217 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:42:47.217 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:42:47.217 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:42:47.217 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:42:47.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:42:47.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:42:47.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:42:47.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:42:47.217 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:42:47.217 | INFO | ------------------------------------------ | 20260507 23:42:47.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:42:47.217 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:42:47.217 | INFO | Total mem usage: 27.8 GiB | 20260507 23:42:47.217 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:42:47.217 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:42:47.217 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:42:47.217 | INFO | High Water Breakdown: | 20260507 23:42:47.217 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:42:47.217 | INFO | ------------------------------------------ | 20260507 23:42:47.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:42:47.217 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:42:47.217 | INFO | Total mem usage: 15.7 GiB | 20260507 23:42:47.217 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:42:47.217 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:42:47.217 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:42:47.217 | INFO | High Water Breakdown: | 20260507 23:42:47.217 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:42:47.217 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:42:47.217 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:42:47.217 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:42:47.217 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:42:47.217 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:42:47.217 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:42:47.217 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:42:47.217 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:42:47.217 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:42:47.217 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:42:47.217 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:42:47.217 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:42:47.217 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:42:47.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:42:47.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:42:47.217 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:42:47.217 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:42:47.217 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:42:47.217 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:42:47.217 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:42:47.217 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:42:47.217 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:42:47.217 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:42:47.217 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:42:47.217 | INFO | ------------------------------------------ | 20260507 23:42:47.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:42:47.217 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:42:47.218 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:42:47.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:42:47.218 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.63e+01, 1.41e+01 / 1.80e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.02e-01, 1.21e-02 / 4.43e-03 | 20260507 23:42:47.218 | INFO | PerfReport: 12.14%, Baseline -- 3.99e+00 / 4.46e+00, 5.83e-01 / 2.14e-01 | 20260507 23:42:47.218 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.00e-01 | 20260507 23:42:47.218 | INFO | PerfReport: 15.93%, DME -- 4.19e+01 / 4.56e+01, 4.07e+00 / 1.53e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.75e+00 / 1.92e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.44e+00 / 2.67e+00, 4.30e-01 / 2.61e-01 | 20260507 23:42:47.218 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.28e+00, 2.80e-01 / 1.68e-01 | 20260507 23:42:47.218 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.72e-01, 5.66e-02 / 3.08e-02 | 20260507 23:42:47.218 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:42:47.218 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:42:47.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:42:47.218 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 30.84%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 2.06%, LockUpload -- 2.20e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 2.65%, LockDownload -- 2.83e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260507 23:42:47.218 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 23:42:47.219 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 23:42:47.333 | INFO | Starting a round of Baz IO | 20260507 23:42:50.781 | INFO | Populating RTMetrics report | 20260507 23:42:50.781 | INFO | RTMetrics report populated | 20260507 23:42:50.781 | INFO | Opening tmp RTMetrics json file | 20260507 23:42:50.783 | INFO | Serializing RTMetrics to json | 20260507 23:42:50.784 | INFO | Serialization done, swapping tmp file now... | 20260507 23:42:50.785 | INFO | RTMetrics done | 20260507 23:42:50.881 | INFO | Finished a round of Baz IO: Wrote 1.707 GiB in 3.548 seconds | 20260507 23:42:50.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.49% / 0.01% / 57.50% | 20260507 23:42:50.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13858% / 7.94729e-06% / 7.54992e-05% | 20260507 23:42:50.881 | INFO | Baz IO overall: | 20260507 23:42:50.881 | INFO | BytesWritten : 1606815298647 | 20260507 23:42:50.881 | INFO | NumEvents : 778015342024 | 20260507 23:42:52.358 | INFO | Analyzing chunk frames = [3530752,3531264) | 20260507 23:42:52.358 | INFO | MemInfo: { | 20260507 23:42:52.358 | INFO | "gpu" : " 17 GB", | 20260507 23:42:52.358 | INFO | "huge" : "", | 20260507 23:42:52.358 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:52.358 | INFO | "malloc" : " 14 GB", | 20260507 23:42:52.358 | INFO | "pinned" : " 29 GB" | 20260507 23:42:52.358 | INFO | } | 20260507 23:42:52.358 | INFO | Perf report for the last 5.1416 seconds: | 20260507 23:42:52.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:52.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:52.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:52.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:52.358 | INFO | ANALYSIS : 68.8%, 2.9, 104.0ms, 0 | 20260507 23:42:52.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:52.358 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 23:42:52.358 | INFO | PRE_HQ : 68.0%, 1.8, 63.7ms, 0 | 20260507 23:42:52.358 | INFO | BAZWRITER : 68.6%, 4.3, 150.4ms, 0 | 20260507 23:42:52.358 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 23:42:52.358 | INFO | Uploaded 36GiB to GPU in 2.51743s (14.3003GiB/s) | 20260507 23:42:52.358 | INFO | Downloaded 38GiB to GPU in 2.66651s (14.2508GiB/s) | 20260507 23:42:57.494 | INFO | Analyzing chunk frames = [3531264,3531776) | 20260507 23:42:57.494 | INFO | MemInfo: { | 20260507 23:42:57.494 | INFO | "gpu" : " 17 GB", | 20260507 23:42:57.494 | INFO | "huge" : "", | 20260507 23:42:57.494 | INFO | "huge_shared" : " 29 GB", | 20260507 23:42:57.494 | INFO | "malloc" : " 14 GB", | 20260507 23:42:57.494 | INFO | "pinned" : " 29 GB" | 20260507 23:42:57.494 | INFO | } | 20260507 23:42:57.494 | INFO | Perf report for the last 5.13642 seconds: | 20260507 23:42:57.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:42:57.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.494 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260507 23:42:57.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.494 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:42:57.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:42:57.495 | INFO | Uploaded 36GiB to GPU in 2.60039s (13.8441GiB/s) | 20260507 23:42:57.495 | INFO | Downloaded 34GiB to GPU in 2.47784s (13.7216GiB/s) | 20260507 23:43:02.630 | INFO | Analyzing chunk frames = [3531776,3532288) | 20260507 23:43:02.630 | INFO | MemInfo: { | 20260507 23:43:02.630 | INFO | "gpu" : " 17 GB", | 20260507 23:43:02.630 | INFO | "huge" : "", | 20260507 23:43:02.630 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:02.630 | INFO | "malloc" : " 14 GB", | 20260507 23:43:02.630 | INFO | "pinned" : " 29 GB" | 20260507 23:43:02.630 | INFO | } | 20260507 23:43:02.630 | INFO | Perf report for the last 5.13604 seconds: | 20260507 23:43:02.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:02.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.631 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:43:02.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.631 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:43:02.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:02.631 | INFO | Uploaded 36GiB to GPU in 2.54642s (14.1375GiB/s) | 20260507 23:43:02.631 | INFO | Downloaded 34GiB to GPU in 2.47728s (13.7247GiB/s) | 20260507 23:43:07.766 | INFO | Analyzing chunk frames = [3532288,3532800) | 20260507 23:43:07.766 | INFO | MemInfo: { | 20260507 23:43:07.766 | INFO | "gpu" : " 17 GB", | 20260507 23:43:07.766 | INFO | "huge" : "", | 20260507 23:43:07.766 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:07.766 | INFO | "malloc" : " 14 GB", | 20260507 23:43:07.766 | INFO | "pinned" : " 29 GB" | 20260507 23:43:07.766 | INFO | } | 20260507 23:43:07.766 | INFO | Perf report for the last 5.1357 seconds: | 20260507 23:43:07.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:07.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.766 | INFO | ANALYSIS : 67.6%, 3.0, 104.5ms, 0 | 20260507 23:43:07.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.766 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:43:07.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:07.767 | INFO | Uploaded 36GiB to GPU in 2.55554s (14.087GiB/s) | 20260507 23:43:07.767 | INFO | Downloaded 34GiB to GPU in 2.47724s (13.725GiB/s) | 20260507 23:43:12.912 | INFO | Analyzing chunk frames = [3532800,3533312) | 20260507 23:43:12.912 | INFO | MemInfo: { | 20260507 23:43:12.912 | INFO | "gpu" : " 17 GB", | 20260507 23:43:12.912 | INFO | "huge" : "", | 20260507 23:43:12.912 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:12.912 | INFO | "malloc" : " 14 GB", | 20260507 23:43:12.912 | INFO | "pinned" : " 29 GB" | 20260507 23:43:12.912 | INFO | } | 20260507 23:43:12.912 | INFO | Perf report for the last 5.14629 seconds: | 20260507 23:43:12.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:12.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.912 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260507 23:43:12.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.913 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:43:12.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:12.913 | INFO | Uploaded 36GiB to GPU in 2.3862s (15.0868GiB/s) | 20260507 23:43:12.913 | INFO | Downloaded 34GiB to GPU in 2.50073s (13.596GiB/s) | 20260507 23:43:18.056 | INFO | Analyzing chunk frames = [3533312,3533824) | 20260507 23:43:18.057 | INFO | MemInfo: { | 20260507 23:43:18.057 | INFO | "gpu" : " 17 GB", | 20260507 23:43:18.057 | INFO | "huge" : "", | 20260507 23:43:18.057 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:18.057 | INFO | "malloc" : " 14 GB", | 20260507 23:43:18.057 | INFO | "pinned" : " 29 GB" | 20260507 23:43:18.057 | INFO | } | 20260507 23:43:18.057 | INFO | Perf report for the last 5.14439 seconds: | 20260507 23:43:18.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:18.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | ANALYSIS : 66.9%, 3.0, 104.6ms, 0 | 20260507 23:43:18.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:43:18.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:18.057 | INFO | Uploaded 36GiB to GPU in 2.54323s (14.1552GiB/s) | 20260507 23:43:18.057 | INFO | Downloaded 34GiB to GPU in 2.42462s (14.0228GiB/s) | 20260507 23:43:23.194 | INFO | Analyzing chunk frames = [3533824,3534336) | 20260507 23:43:23.194 | INFO | MemInfo: { | 20260507 23:43:23.194 | INFO | "gpu" : " 17 GB", | 20260507 23:43:23.194 | INFO | "huge" : "", | 20260507 23:43:23.194 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:23.194 | INFO | "malloc" : " 14 GB", | 20260507 23:43:23.194 | INFO | "pinned" : " 29 GB" | 20260507 23:43:23.195 | INFO | } | 20260507 23:43:23.195 | INFO | Perf report for the last 5.13775 seconds: | 20260507 23:43:23.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:23.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:43:23.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:43:23.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:23.195 | INFO | Uploaded 36GiB to GPU in 2.54195s (14.1624GiB/s) | 20260507 23:43:23.195 | INFO | Downloaded 34GiB to GPU in 2.44327s (13.9158GiB/s) | 20260507 23:43:28.327 | INFO | Analyzing chunk frames = [3534336,3534848) | 20260507 23:43:28.327 | INFO | MemInfo: { | 20260507 23:43:28.327 | INFO | "gpu" : " 17 GB", | 20260507 23:43:28.327 | INFO | "huge" : "", | 20260507 23:43:28.327 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:28.327 | INFO | "malloc" : " 14 GB", | 20260507 23:43:28.327 | INFO | "pinned" : " 29 GB" | 20260507 23:43:28.327 | INFO | } | 20260507 23:43:28.327 | INFO | Perf report for the last 5.13236 seconds: | 20260507 23:43:28.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:28.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:43:28.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260507 23:43:28.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:28.327 | INFO | Uploaded 36GiB to GPU in 2.57999s (13.9536GiB/s) | 20260507 23:43:28.327 | INFO | Downloaded 34GiB to GPU in 2.45245s (13.8637GiB/s) | 20260507 23:43:28.424 | INFO | Starting a round of Baz IO | 20260507 23:43:31.888 | INFO | Populating RTMetrics report | 20260507 23:43:31.889 | INFO | RTMetrics report populated | 20260507 23:43:31.889 | INFO | Opening tmp RTMetrics json file | 20260507 23:43:31.903 | INFO | Serializing RTMetrics to json | 20260507 23:43:31.928 | INFO | Serialization done, swapping tmp file now... | 20260507 23:43:31.952 | INFO | RTMetrics done | 20260507 23:43:31.992 | INFO | Finished a round of Baz IO: Wrote 1.708 GiB in 3.568 seconds | 20260507 23:43:31.993 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.49% / 0.01% / 57.51% | 20260507 23:43:31.993 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13479% / 7.94729e-06% / 8.74201e-05% | 20260507 23:43:31.993 | INFO | Baz IO overall: | 20260507 23:43:31.993 | INFO | BytesWritten : 1608649269307 | 20260507 23:43:31.993 | INFO | NumEvents : 778898390538 | 20260507 23:43:33.468 | INFO | Analyzing chunk frames = [3534848,3535360) | 20260507 23:43:33.468 | INFO | MemInfo: { | 20260507 23:43:33.468 | INFO | "gpu" : " 17 GB", | 20260507 23:43:33.468 | INFO | "huge" : "", | 20260507 23:43:33.468 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:33.468 | INFO | "malloc" : " 14 GB", | 20260507 23:43:33.468 | INFO | "pinned" : " 29 GB" | 20260507 23:43:33.468 | INFO | } | 20260507 23:43:33.468 | INFO | Perf report for the last 5.14136 seconds: | 20260507 23:43:33.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:33.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:33.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:33.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:33.468 | INFO | ANALYSIS : 68.6%, 3.0, 106.7ms, 0 | 20260507 23:43:33.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:33.468 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 23:43:33.468 | INFO | PRE_HQ : 65.5%, 1.7, 56.9ms, 0 | 20260507 23:43:33.468 | INFO | BAZWRITER : 69.0%, 3.8, 136.2ms, 0 | 20260507 23:43:33.469 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 23:43:33.469 | INFO | Uploaded 36GiB to GPU in 2.56565s (14.0315GiB/s) | 20260507 23:43:33.469 | INFO | Downloaded 38GiB to GPU in 2.69674s (14.0911GiB/s) | 20260507 23:43:38.605 | INFO | Analyzing chunk frames = [3535360,3535872) | 20260507 23:43:38.605 | INFO | MemInfo: { | 20260507 23:43:38.605 | INFO | "gpu" : " 17 GB", | 20260507 23:43:38.605 | INFO | "huge" : "", | 20260507 23:43:38.605 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:38.605 | INFO | "malloc" : " 14 GB", | 20260507 23:43:38.605 | INFO | "pinned" : " 29 GB" | 20260507 23:43:38.605 | INFO | } | 20260507 23:43:38.605 | INFO | Perf report for the last 5.13687 seconds: | 20260507 23:43:38.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:38.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:43:38.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:43:38.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:38.605 | INFO | Uploaded 36GiB to GPU in 2.5873s (13.9141GiB/s) | 20260507 23:43:38.606 | INFO | Downloaded 34GiB to GPU in 2.47059s (13.7619GiB/s) | 20260507 23:43:43.745 | INFO | Analyzing chunk frames = [3535872,3536384) | 20260507 23:43:43.745 | INFO | MemInfo: { | 20260507 23:43:43.745 | INFO | "gpu" : " 17 GB", | 20260507 23:43:43.745 | INFO | "huge" : "", | 20260507 23:43:43.746 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:43.746 | INFO | "malloc" : " 14 GB", | 20260507 23:43:43.746 | INFO | "pinned" : " 29 GB" | 20260507 23:43:43.746 | INFO | } | 20260507 23:43:43.746 | INFO | Perf report for the last 5.14058 seconds: | 20260507 23:43:43.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:43.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 23:43:43.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:43:43.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:43.746 | INFO | Uploaded 36GiB to GPU in 2.57575s (13.9765GiB/s) | 20260507 23:43:43.746 | INFO | Downloaded 34GiB to GPU in 2.46096s (13.8158GiB/s) | 20260507 23:43:48.885 | INFO | Analyzing chunk frames = [3536384,3536896) | 20260507 23:43:48.885 | INFO | MemInfo: { | 20260507 23:43:48.885 | INFO | "gpu" : " 17 GB", | 20260507 23:43:48.885 | INFO | "huge" : "", | 20260507 23:43:48.886 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:48.886 | INFO | "malloc" : " 14 GB", | 20260507 23:43:48.886 | INFO | "pinned" : " 29 GB" | 20260507 23:43:48.886 | INFO | } | 20260507 23:43:48.886 | INFO | Perf report for the last 5.13993 seconds: | 20260507 23:43:48.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:48.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 23:43:48.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:43:48.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:48.886 | INFO | Uploaded 36GiB to GPU in 2.55198s (14.1067GiB/s) | 20260507 23:43:48.886 | INFO | Downloaded 34GiB to GPU in 2.49029s (13.653GiB/s) | 20260507 23:43:54.025 | INFO | Analyzing chunk frames = [3536896,3537408) | 20260507 23:43:54.025 | INFO | MemInfo: { | 20260507 23:43:54.025 | INFO | "gpu" : " 17 GB", | 20260507 23:43:54.025 | INFO | "huge" : "", | 20260507 23:43:54.025 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:54.025 | INFO | "malloc" : " 14 GB", | 20260507 23:43:54.025 | INFO | "pinned" : " 29 GB" | 20260507 23:43:54.026 | INFO | } | 20260507 23:43:54.026 | INFO | Perf report for the last 5.13984 seconds: | 20260507 23:43:54.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:54.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260507 23:43:54.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:43:54.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:54.026 | INFO | Uploaded 36GiB to GPU in 2.47663s (14.5359GiB/s) | 20260507 23:43:54.026 | INFO | Downloaded 34GiB to GPU in 2.498s (13.6109GiB/s) | 20260507 23:43:59.169 | INFO | Analyzing chunk frames = [3537408,3537920) | 20260507 23:43:59.169 | INFO | MemInfo: { | 20260507 23:43:59.169 | INFO | "gpu" : " 17 GB", | 20260507 23:43:59.169 | INFO | "huge" : "", | 20260507 23:43:59.170 | INFO | "huge_shared" : " 29 GB", | 20260507 23:43:59.170 | INFO | "malloc" : " 14 GB", | 20260507 23:43:59.170 | INFO | "pinned" : " 29 GB" | 20260507 23:43:59.170 | INFO | } | 20260507 23:43:59.170 | INFO | Perf report for the last 5.14407 seconds: | 20260507 23:43:59.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:43:59.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260507 23:43:59.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:43:59.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:43:59.170 | INFO | Uploaded 36GiB to GPU in 2.4099s (14.9384GiB/s) | 20260507 23:43:59.170 | INFO | Downloaded 34GiB to GPU in 2.40923s (14.1124GiB/s) | 20260507 23:44:04.306 | INFO | Analyzing chunk frames = [3537920,3538432) | 20260507 23:44:04.306 | INFO | MemInfo: { | 20260507 23:44:04.306 | INFO | "gpu" : " 17 GB", | 20260507 23:44:04.306 | INFO | "huge" : "", | 20260507 23:44:04.306 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:04.306 | INFO | "malloc" : " 14 GB", | 20260507 23:44:04.306 | INFO | "pinned" : " 29 GB" | 20260507 23:44:04.306 | INFO | } | 20260507 23:44:04.306 | INFO | Perf report for the last 5.13606 seconds: | 20260507 23:44:04.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:04.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 23:44:04.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 23:44:04.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:04.306 | INFO | Uploaded 36GiB to GPU in 2.54825s (14.1273GiB/s) | 20260507 23:44:04.307 | INFO | Downloaded 34GiB to GPU in 2.45931s (13.825GiB/s) | 20260507 23:44:09.446 | INFO | Analyzing chunk frames = [3538432,3538944) | 20260507 23:44:09.446 | INFO | MemInfo: { | 20260507 23:44:09.446 | INFO | "gpu" : " 17 GB", | 20260507 23:44:09.446 | INFO | "huge" : "", | 20260507 23:44:09.446 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:09.446 | INFO | "malloc" : " 14 GB", | 20260507 23:44:09.446 | INFO | "pinned" : " 29 GB" | 20260507 23:44:09.446 | INFO | } | 20260507 23:44:09.446 | INFO | Perf report for the last 5.14022 seconds: | 20260507 23:44:09.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:09.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.446 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:44:09.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.446 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 23:44:09.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:09.447 | INFO | Uploaded 36GiB to GPU in 2.53068s (14.2255GiB/s) | 20260507 23:44:09.447 | INFO | Downloaded 34GiB to GPU in 2.45485s (13.8501GiB/s) | 20260507 23:44:09.447 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:44:09.447 | INFO | Total mem usage: 16.7 GiB | 20260507 23:44:09.447 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:44:09.447 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.54%) | 20260507 23:44:09.447 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:44:09.447 | INFO | High Water Breakdown: | 20260507 23:44:09.447 | INFO | 60.5% : 9.21 GiB ( 89.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:44:09.447 | INFO | 39.4% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:44:09.447 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:44:09.447 | INFO | ------------------------------------------ | 20260507 23:44:09.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:44:09.447 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:44:09.447 | INFO | Total mem usage: 26.5 GiB | 20260507 23:44:09.447 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:44:09.447 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:44:09.447 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:44:09.447 | INFO | High Water Breakdown: | 20260507 23:44:09.447 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:44:09.447 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:44:09.447 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:44:09.447 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:44:09.447 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:44:09.447 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:44:09.447 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:44:09.447 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:44:09.447 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:44:09.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:44:09.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:44:09.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:44:09.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:44:09.447 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:44:09.447 | INFO | ------------------------------------------ | 20260507 23:44:09.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:44:09.447 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:44:09.447 | INFO | Total mem usage: 27.8 GiB | 20260507 23:44:09.447 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:44:09.447 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:44:09.447 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:44:09.447 | INFO | High Water Breakdown: | 20260507 23:44:09.447 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:44:09.447 | INFO | ------------------------------------------ | 20260507 23:44:09.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:44:09.447 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:44:09.447 | INFO | Total mem usage: 15.7 GiB | 20260507 23:44:09.447 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:44:09.447 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:44:09.447 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:44:09.448 | INFO | High Water Breakdown: | 20260507 23:44:09.448 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:44:09.448 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:44:09.448 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:44:09.448 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:44:09.448 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:44:09.448 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:44:09.448 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:44:09.448 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:44:09.448 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:44:09.448 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:44:09.448 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:44:09.448 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:44:09.448 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:44:09.448 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:44:09.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:44:09.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:44:09.448 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:44:09.448 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:44:09.448 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:44:09.448 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:44:09.448 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:44:09.448 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:44:09.448 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:44:09.448 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:44:09.448 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:44:09.448 | INFO | ------------------------------------------ | 20260507 23:44:09.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:44:09.448 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:44:09.448 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:44:09.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:44:09.448 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.67e+01, 1.42e+01 / 1.78e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.02e-01, 1.19e-02 / 6.08e-03 | 20260507 23:44:09.448 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.46e+00, 5.84e-01 / 2.49e-01 | 20260507 23:44:09.448 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.06e-01 | 20260507 23:44:09.448 | INFO | PerfReport: 16.06%, DME -- 4.21e+01 / 4.62e+01, 3.92e+00 / 1.72e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.84e+01 / 2.00e+01, 1.63e+00 / 1.85e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.32e-01 / 2.17e-01 | 20260507 23:44:09.448 | INFO | PerfReport: 3.42%, Metrics -- 1.12e+00 / 1.33e+00, 2.82e-01 / 1.88e-01 | 20260507 23:44:09.448 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.82e-01, 5.53e-02 / 3.18e-02 | 20260507 23:44:09.448 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:44:09.448 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:44:09.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:44:09.448 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 24.67%, Upload -- 2.64e+01 / 0.00e+00, 8.42e+00 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 24.27%, Download -- 2.60e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 30.67%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 2.45%, LockUpload -- 2.62e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 2.39%, LockDownload -- 2.55e+00 / 0.00e+00, 3.72e+00 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 15.54%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:44:09.448 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260507 23:44:09.560 | INFO | Starting a round of Baz IO | 20260507 23:44:13.024 | INFO | Populating RTMetrics report | 20260507 23:44:13.025 | INFO | RTMetrics report populated | 20260507 23:44:13.025 | INFO | Opening tmp RTMetrics json file | 20260507 23:44:13.052 | INFO | Serializing RTMetrics to json | 20260507 23:44:13.054 | INFO | Serialization done, swapping tmp file now... | 20260507 23:44:13.058 | INFO | RTMetrics done | 20260507 23:44:13.115 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.555 seconds | 20260507 23:44:13.115 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.48% / 0.00% / 57.51% | 20260507 23:44:13.115 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13147% / 3.97364e-06% / 6.75519e-05% | 20260507 23:44:13.115 | INFO | Baz IO overall: | 20260507 23:44:13.115 | INFO | BytesWritten : 1610480127121 | 20260507 23:44:13.116 | INFO | NumEvents : 779779428878 | 20260507 23:44:14.582 | INFO | Analyzing chunk frames = [3538944,3539456) | 20260507 23:44:14.582 | INFO | MemInfo: { | 20260507 23:44:14.582 | INFO | "gpu" : " 17 GB", | 20260507 23:44:14.582 | INFO | "huge" : "", | 20260507 23:44:14.582 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:14.582 | INFO | "malloc" : " 14 GB", | 20260507 23:44:14.582 | INFO | "pinned" : " 29 GB" | 20260507 23:44:14.582 | INFO | } | 20260507 23:44:14.582 | INFO | Perf report for the last 5.13584 seconds: | 20260507 23:44:14.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:14.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:14.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:14.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:14.582 | INFO | ANALYSIS : 68.7%, 2.9, 104.0ms, 0 | 20260507 23:44:14.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:14.582 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 23:44:14.582 | INFO | PRE_HQ : 67.8%, 1.8, 63.9ms, 0 | 20260507 23:44:14.582 | INFO | BAZWRITER : 68.8%, 4.2, 148.0ms, 0 | 20260507 23:44:14.582 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 23:44:14.582 | INFO | Uploaded 36GiB to GPU in 2.59774s (13.8582GiB/s) | 20260507 23:44:14.582 | INFO | Downloaded 38GiB to GPU in 2.70381s (14.0543GiB/s) | 20260507 23:44:19.721 | INFO | Analyzing chunk frames = [3539456,3539968) | 20260507 23:44:19.721 | INFO | MemInfo: { | 20260507 23:44:19.721 | INFO | "gpu" : " 17 GB", | 20260507 23:44:19.721 | INFO | "huge" : "", | 20260507 23:44:19.721 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:19.722 | INFO | "malloc" : " 14 GB", | 20260507 23:44:19.722 | INFO | "pinned" : " 29 GB" | 20260507 23:44:19.722 | INFO | } | 20260507 23:44:19.722 | INFO | Perf report for the last 5.1398 seconds: | 20260507 23:44:19.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:19.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.722 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 23:44:19.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.722 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:44:19.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:19.723 | INFO | Uploaded 36GiB to GPU in 2.51707s (14.3023GiB/s) | 20260507 23:44:19.723 | INFO | Downloaded 34GiB to GPU in 2.4764s (13.7296GiB/s) | 20260507 23:44:24.865 | INFO | Analyzing chunk frames = [3539968,3540480) | 20260507 23:44:24.865 | INFO | MemInfo: { | 20260507 23:44:24.865 | INFO | "gpu" : " 17 GB", | 20260507 23:44:24.865 | INFO | "huge" : "", | 20260507 23:44:24.866 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:24.866 | INFO | "malloc" : " 14 GB", | 20260507 23:44:24.866 | INFO | "pinned" : " 29 GB" | 20260507 23:44:24.866 | INFO | } | 20260507 23:44:24.866 | INFO | Perf report for the last 5.14396 seconds: | 20260507 23:44:24.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:24.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260507 23:44:24.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:44:24.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:24.866 | INFO | Uploaded 36GiB to GPU in 2.47949s (14.5191GiB/s) | 20260507 23:44:24.866 | INFO | Downloaded 34GiB to GPU in 2.48313s (13.6924GiB/s) | 20260507 23:44:30.006 | INFO | Analyzing chunk frames = [3540480,3540992) | 20260507 23:44:30.006 | INFO | MemInfo: { | 20260507 23:44:30.006 | INFO | "gpu" : " 17 GB", | 20260507 23:44:30.006 | INFO | "huge" : "", | 20260507 23:44:30.006 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:30.006 | INFO | "malloc" : " 14 GB", | 20260507 23:44:30.006 | INFO | "pinned" : " 29 GB" | 20260507 23:44:30.006 | INFO | } | 20260507 23:44:30.006 | INFO | Perf report for the last 5.14055 seconds: | 20260507 23:44:30.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:30.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.006 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:44:30.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.006 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:44:30.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:30.007 | INFO | Uploaded 36GiB to GPU in 2.51342s (14.3231GiB/s) | 20260507 23:44:30.007 | INFO | Downloaded 34GiB to GPU in 2.48859s (13.6623GiB/s) | 20260507 23:44:35.141 | INFO | Analyzing chunk frames = [3540992,3541504) | 20260507 23:44:35.141 | INFO | MemInfo: { | 20260507 23:44:35.141 | INFO | "gpu" : " 17 GB", | 20260507 23:44:35.141 | INFO | "huge" : "", | 20260507 23:44:35.141 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:35.141 | INFO | "malloc" : " 14 GB", | 20260507 23:44:35.141 | INFO | "pinned" : " 29 GB" | 20260507 23:44:35.141 | INFO | } | 20260507 23:44:35.141 | INFO | Perf report for the last 5.13517 seconds: | 20260507 23:44:35.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:35.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.141 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260507 23:44:35.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.142 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 23:44:35.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:35.142 | INFO | Uploaded 36GiB to GPU in 2.4612s (14.627GiB/s) | 20260507 23:44:35.142 | INFO | Downloaded 34GiB to GPU in 2.49336s (13.6362GiB/s) | 20260507 23:44:40.278 | INFO | Analyzing chunk frames = [3541504,3542016) | 20260507 23:44:40.278 | INFO | MemInfo: { | 20260507 23:44:40.278 | INFO | "gpu" : " 17 GB", | 20260507 23:44:40.278 | INFO | "huge" : "", | 20260507 23:44:40.278 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:40.278 | INFO | "malloc" : " 14 GB", | 20260507 23:44:40.278 | INFO | "pinned" : " 29 GB" | 20260507 23:44:40.278 | INFO | } | 20260507 23:44:40.278 | INFO | Perf report for the last 5.13679 seconds: | 20260507 23:44:40.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:40.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:44:40.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:44:40.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:40.278 | INFO | Uploaded 36GiB to GPU in 2.41658s (14.8971GiB/s) | 20260507 23:44:40.279 | INFO | Downloaded 34GiB to GPU in 2.39413s (14.2014GiB/s) | 20260507 23:44:45.419 | INFO | Analyzing chunk frames = [3542016,3542528) | 20260507 23:44:45.419 | INFO | MemInfo: { | 20260507 23:44:45.419 | INFO | "gpu" : " 17 GB", | 20260507 23:44:45.419 | INFO | "huge" : "", | 20260507 23:44:45.419 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:45.419 | INFO | "malloc" : " 14 GB", | 20260507 23:44:45.419 | INFO | "pinned" : " 29 GB" | 20260507 23:44:45.419 | INFO | } | 20260507 23:44:45.419 | INFO | Perf report for the last 5.14095 seconds: | 20260507 23:44:45.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:45.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 23:44:45.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:44:45.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:45.419 | INFO | Uploaded 36GiB to GPU in 2.57454s (13.9831GiB/s) | 20260507 23:44:45.420 | INFO | Downloaded 34GiB to GPU in 2.46115s (13.8147GiB/s) | 20260507 23:44:50.555 | INFO | Analyzing chunk frames = [3542528,3543040) | 20260507 23:44:50.555 | INFO | MemInfo: { | 20260507 23:44:50.555 | INFO | "gpu" : " 17 GB", | 20260507 23:44:50.555 | INFO | "huge" : "", | 20260507 23:44:50.555 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:50.555 | INFO | "malloc" : " 14 GB", | 20260507 23:44:50.555 | INFO | "pinned" : " 29 GB" | 20260507 23:44:50.555 | INFO | } | 20260507 23:44:50.555 | INFO | Perf report for the last 5.1363 seconds: | 20260507 23:44:50.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:50.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:44:50.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:44:50.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:50.556 | INFO | Uploaded 36GiB to GPU in 2.5202s (14.2846GiB/s) | 20260507 23:44:50.556 | INFO | Downloaded 34GiB to GPU in 2.43526s (13.9615GiB/s) | 20260507 23:44:50.653 | INFO | Starting a round of Baz IO | 20260507 23:44:54.122 | INFO | Populating RTMetrics report | 20260507 23:44:54.122 | INFO | RTMetrics report populated | 20260507 23:44:54.122 | INFO | Opening tmp RTMetrics json file | 20260507 23:44:54.139 | INFO | Serializing RTMetrics to json | 20260507 23:44:54.147 | INFO | Serialization done, swapping tmp file now... | 20260507 23:44:54.148 | INFO | RTMetrics done | 20260507 23:44:54.218 | INFO | Finished a round of Baz IO: Wrote 1.708 GiB in 3.565 seconds | 20260507 23:44:54.218 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.48% / 0.01% / 57.52% | 20260507 23:44:54.218 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13268% / 7.94729e-06% / 8.34465e-05% | 20260507 23:44:54.218 | INFO | Baz IO overall: | 20260507 23:44:54.218 | INFO | BytesWritten : 1612314165122 | 20260507 23:44:54.218 | INFO | NumEvents : 780662606079 | 20260507 23:44:55.698 | INFO | Analyzing chunk frames = [3543040,3543552) | 20260507 23:44:55.698 | INFO | MemInfo: { | 20260507 23:44:55.698 | INFO | "gpu" : " 17 GB", | 20260507 23:44:55.698 | INFO | "huge" : "", | 20260507 23:44:55.698 | INFO | "huge_shared" : " 29 GB", | 20260507 23:44:55.698 | INFO | "malloc" : " 14 GB", | 20260507 23:44:55.699 | INFO | "pinned" : " 29 GB" | 20260507 23:44:55.699 | INFO | } | 20260507 23:44:55.699 | INFO | Perf report for the last 5.1431 seconds: | 20260507 23:44:55.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:44:55.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:55.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:55.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:55.699 | INFO | ANALYSIS : 68.7%, 3.0, 106.8ms, 0 | 20260507 23:44:55.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:44:55.699 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 23:44:55.699 | INFO | PRE_HQ : 65.8%, 1.7, 56.7ms, 0 | 20260507 23:44:55.699 | INFO | BAZWRITER : 68.9%, 3.9, 137.1ms, 0 | 20260507 23:44:55.699 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 23:44:55.699 | INFO | Uploaded 36GiB to GPU in 2.51716s (14.3018GiB/s) | 20260507 23:44:55.699 | INFO | Downloaded 38GiB to GPU in 2.68737s (14.1402GiB/s) | 20260507 23:45:00.838 | INFO | Analyzing chunk frames = [3543552,3544064) | 20260507 23:45:00.838 | INFO | MemInfo: { | 20260507 23:45:00.838 | INFO | "gpu" : " 17 GB", | 20260507 23:45:00.838 | INFO | "huge" : "", | 20260507 23:45:00.838 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:00.838 | INFO | "malloc" : " 14 GB", | 20260507 23:45:00.838 | INFO | "pinned" : " 29 GB" | 20260507 23:45:00.838 | INFO | } | 20260507 23:45:00.838 | INFO | Perf report for the last 5.13955 seconds: | 20260507 23:45:00.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:00.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.838 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 23:45:00.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.838 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:45:00.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:00.839 | INFO | Uploaded 36GiB to GPU in 2.58601s (13.9211GiB/s) | 20260507 23:45:00.839 | INFO | Downloaded 34GiB to GPU in 2.47755s (13.7232GiB/s) | 20260507 23:45:05.978 | INFO | Analyzing chunk frames = [3544064,3544576) | 20260507 23:45:05.978 | INFO | MemInfo: { | 20260507 23:45:05.978 | INFO | "gpu" : " 17 GB", | 20260507 23:45:05.978 | INFO | "huge" : "", | 20260507 23:45:05.978 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:05.978 | INFO | "malloc" : " 14 GB", | 20260507 23:45:05.978 | INFO | "pinned" : " 29 GB" | 20260507 23:45:05.978 | INFO | } | 20260507 23:45:05.978 | INFO | Perf report for the last 5.14018 seconds: | 20260507 23:45:05.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:05.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260507 23:45:05.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:45:05.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:05.979 | INFO | Uploaded 36GiB to GPU in 2.56434s (14.0387GiB/s) | 20260507 23:45:05.979 | INFO | Downloaded 34GiB to GPU in 2.47365s (13.7449GiB/s) | 20260507 23:45:11.119 | INFO | Analyzing chunk frames = [3544576,3545088) | 20260507 23:45:11.119 | INFO | MemInfo: { | 20260507 23:45:11.119 | INFO | "gpu" : " 17 GB", | 20260507 23:45:11.120 | INFO | "huge" : "", | 20260507 23:45:11.120 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:11.120 | INFO | "malloc" : " 14 GB", | 20260507 23:45:11.120 | INFO | "pinned" : " 29 GB" | 20260507 23:45:11.120 | INFO | } | 20260507 23:45:11.120 | INFO | Perf report for the last 5.14133 seconds: | 20260507 23:45:11.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:11.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260507 23:45:11.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:45:11.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:11.120 | INFO | Uploaded 36GiB to GPU in 2.54257s (14.1589GiB/s) | 20260507 23:45:11.120 | INFO | Downloaded 34GiB to GPU in 2.47878s (13.7164GiB/s) | 20260507 23:45:16.259 | INFO | Analyzing chunk frames = [3545088,3545600) | 20260507 23:45:16.259 | INFO | MemInfo: { | 20260507 23:45:16.259 | INFO | "gpu" : " 17 GB", | 20260507 23:45:16.259 | INFO | "huge" : "", | 20260507 23:45:16.259 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:16.260 | INFO | "malloc" : " 14 GB", | 20260507 23:45:16.260 | INFO | "pinned" : " 29 GB" | 20260507 23:45:16.260 | INFO | } | 20260507 23:45:16.260 | INFO | Perf report for the last 5.13994 seconds: | 20260507 23:45:16.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:16.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:45:16.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 23:45:16.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:16.260 | INFO | Uploaded 36GiB to GPU in 2.44962s (14.6962GiB/s) | 20260507 23:45:16.260 | INFO | Downloaded 34GiB to GPU in 2.49436s (13.6307GiB/s) | 20260507 23:45:21.392 | INFO | Analyzing chunk frames = [3545600,3546112) | 20260507 23:45:21.392 | INFO | MemInfo: { | 20260507 23:45:21.392 | INFO | "gpu" : " 17 GB", | 20260507 23:45:21.392 | INFO | "huge" : "", | 20260507 23:45:21.392 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:21.392 | INFO | "malloc" : " 14 GB", | 20260507 23:45:21.392 | INFO | "pinned" : " 29 GB" | 20260507 23:45:21.392 | INFO | } | 20260507 23:45:21.392 | INFO | Perf report for the last 5.13243 seconds: | 20260507 23:45:21.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:21.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.392 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:45:21.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.392 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260507 23:45:21.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:21.393 | INFO | Uploaded 36GiB to GPU in 2.40763s (14.9525GiB/s) | 20260507 23:45:21.393 | INFO | Downloaded 34GiB to GPU in 2.38558s (14.2523GiB/s) | 20260507 23:45:26.531 | INFO | Analyzing chunk frames = [3546112,3546624) | 20260507 23:45:26.531 | INFO | MemInfo: { | 20260507 23:45:26.531 | INFO | "gpu" : " 17 GB", | 20260507 23:45:26.531 | INFO | "huge" : "", | 20260507 23:45:26.531 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:26.531 | INFO | "malloc" : " 14 GB", | 20260507 23:45:26.531 | INFO | "pinned" : " 29 GB" | 20260507 23:45:26.531 | INFO | } | 20260507 23:45:26.531 | INFO | Perf report for the last 5.13921 seconds: | 20260507 23:45:26.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:26.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.531 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260507 23:45:26.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.532 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 23:45:26.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:26.532 | INFO | Uploaded 36GiB to GPU in 2.45105s (14.6876GiB/s) | 20260507 23:45:26.532 | INFO | Downloaded 34GiB to GPU in 2.40938s (14.1115GiB/s) | 20260507 23:45:31.671 | INFO | Analyzing chunk frames = [3546624,3547136) | 20260507 23:45:31.671 | INFO | MemInfo: { | 20260507 23:45:31.671 | INFO | "gpu" : " 17 GB", | 20260507 23:45:31.671 | INFO | "huge" : "", | 20260507 23:45:31.671 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:31.671 | INFO | "malloc" : " 14 GB", | 20260507 23:45:31.672 | INFO | "pinned" : " 29 GB" | 20260507 23:45:31.672 | INFO | } | 20260507 23:45:31.672 | INFO | Perf report for the last 5.14032 seconds: | 20260507 23:45:31.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:31.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 23:45:31.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 23:45:31.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:31.672 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:45:31.672 | INFO | Total mem usage: 16.7 GiB | 20260507 23:45:31.672 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:45:31.672 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.2%) | 20260507 23:45:31.672 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:45:31.672 | INFO | High Water Breakdown: | 20260507 23:45:31.672 | INFO | 60.3% : 9.13 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:45:31.672 | INFO | 39.6% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:45:31.672 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:45:31.672 | INFO | ------------------------------------------ | 20260507 23:45:31.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:45:31.672 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:45:31.672 | INFO | Total mem usage: 26.5 GiB | 20260507 23:45:31.672 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:45:31.672 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260507 23:45:31.672 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260507 23:45:31.672 | INFO | High Water Breakdown: | 20260507 23:45:31.672 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:45:31.672 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:45:31.672 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:45:31.672 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:45:31.672 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:45:31.672 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:45:31.672 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:45:31.672 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:45:31.672 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:45:31.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:45:31.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:45:31.672 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:45:31.672 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:45:31.672 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:45:31.672 | INFO | ------------------------------------------ | 20260507 23:45:31.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:45:31.672 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:45:31.672 | INFO | Total mem usage: 27.8 GiB | 20260507 23:45:31.672 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:45:31.672 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:45:31.672 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:45:31.672 | INFO | High Water Breakdown: | 20260507 23:45:31.672 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:45:31.672 | INFO | ------------------------------------------ | 20260507 23:45:31.672 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:45:31.672 | INFO | Uploaded 36GiB to GPU in 2.55134s (14.1102GiB/s) | 20260507 23:45:31.672 | INFO | Downloaded 34GiB to GPU in 2.4446s (13.9082GiB/s) | 20260507 23:45:31.672 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:45:31.672 | INFO | Total mem usage: 15.7 GiB | 20260507 23:45:31.672 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:45:31.672 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:45:31.672 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:45:31.672 | INFO | High Water Breakdown: | 20260507 23:45:31.673 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:45:31.673 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:45:31.673 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:45:31.673 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:45:31.673 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:45:31.673 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:45:31.673 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:45:31.673 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:45:31.673 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:45:31.673 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:45:31.673 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:45:31.673 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:45:31.673 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:45:31.673 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:45:31.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:45:31.673 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:45:31.673 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:45:31.673 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:45:31.673 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:45:31.673 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:45:31.673 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:45:31.673 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:45:31.673 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:45:31.673 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:45:31.673 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:45:31.673 | INFO | ------------------------------------------ | 20260507 23:45:31.673 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:45:31.673 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:45:31.673 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:45:31.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:45:31.673 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.63e+01, 1.42e+01 / 1.86e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.22e-02 / 1.25e-02 | 20260507 23:45:31.673 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.50e+00, 5.89e-01 / 1.79e-01 | 20260507 23:45:31.673 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.51e+00, 2.08e-01 / 1.02e-01 | 20260507 23:45:31.673 | INFO | PerfReport: 16.11%, DME -- 4.23e+01 / 4.62e+01, 4.10e+00 / 2.05e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.67e+00 / 1.54e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.31e-01 / 2.46e-01 | 20260507 23:45:31.673 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.34e+00, 2.79e-01 / 1.57e-01 | 20260507 23:45:31.673 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.46e-01 / 3.82e-01, 5.48e-02 / 2.88e-02 | 20260507 23:45:31.673 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:45:31.673 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:45:31.673 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:45:31.673 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 24.27%, Download -- 2.59e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 30.78%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 2.28%, LockUpload -- 2.43e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 2.53%, LockDownload -- 2.70e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 15.64%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 23:45:31.673 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 23:45:31.785 | INFO | Starting a round of Baz IO | 20260507 23:45:35.234 | INFO | Populating RTMetrics report | 20260507 23:45:35.234 | INFO | RTMetrics report populated | 20260507 23:45:35.234 | INFO | Opening tmp RTMetrics json file | 20260507 23:45:35.243 | INFO | Serializing RTMetrics to json | 20260507 23:45:35.272 | INFO | Serialization done, swapping tmp file now... | 20260507 23:45:35.298 | INFO | RTMetrics done | 20260507 23:45:35.508 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.723 seconds | 20260507 23:45:35.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.47% / 0.00% / 57.52% | 20260507 23:45:35.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12731% / 1.58946e-05% / 9.13938e-05% | 20260507 23:45:35.508 | INFO | Baz IO overall: | 20260507 23:45:35.508 | INFO | BytesWritten : 1614145111376 | 20260507 23:45:35.508 | INFO | NumEvents : 781543557813 | 20260507 23:45:36.821 | INFO | Analyzing chunk frames = [3547136,3547648) | 20260507 23:45:36.821 | INFO | MemInfo: { | 20260507 23:45:36.821 | INFO | "gpu" : " 17 GB", | 20260507 23:45:36.821 | INFO | "huge" : "", | 20260507 23:45:36.821 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:36.821 | INFO | "malloc" : " 14 GB", | 20260507 23:45:36.821 | INFO | "pinned" : " 29 GB" | 20260507 23:45:36.821 | INFO | } | 20260507 23:45:36.821 | INFO | Perf report for the last 5.14927 seconds: | 20260507 23:45:36.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:36.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:36.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:36.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:36.821 | INFO | ANALYSIS : 68.5%, 2.9, 104.5ms, 0 | 20260507 23:45:36.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:36.821 | INFO | RT_METRICS : 45.4%, 1.0, 23.9ms, 0 | 20260507 23:45:36.821 | INFO | PRE_HQ : 67.1%, 1.8, 61.7ms, 0 | 20260507 23:45:36.821 | INFO | BAZWRITER : 71.9%, 3.9, 144.6ms, 0 | 20260507 23:45:36.821 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:45:36.821 | INFO | Uploaded 36GiB to GPU in 2.57247s (13.9943GiB/s) | 20260507 23:45:36.821 | INFO | Downloaded 38GiB to GPU in 2.70513s (14.0474GiB/s) | 20260507 23:45:41.957 | INFO | Analyzing chunk frames = [3547648,3548160) | 20260507 23:45:41.957 | INFO | MemInfo: { | 20260507 23:45:41.957 | INFO | "gpu" : " 17 GB", | 20260507 23:45:41.957 | INFO | "huge" : "", | 20260507 23:45:41.957 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:41.957 | INFO | "malloc" : " 14 GB", | 20260507 23:45:41.957 | INFO | "pinned" : " 29 GB" | 20260507 23:45:41.957 | INFO | } | 20260507 23:45:41.957 | INFO | Perf report for the last 5.13629 seconds: | 20260507 23:45:41.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:41.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.957 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:45:41.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.957 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 23:45:41.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:41.958 | INFO | Uploaded 36GiB to GPU in 2.59182s (13.8899GiB/s) | 20260507 23:45:41.958 | INFO | Downloaded 34GiB to GPU in 2.48065s (13.7061GiB/s) | 20260507 23:45:47.088 | INFO | Analyzing chunk frames = [3548160,3548672) | 20260507 23:45:47.088 | INFO | MemInfo: { | 20260507 23:45:47.088 | INFO | "gpu" : " 17 GB", | 20260507 23:45:47.088 | INFO | "huge" : "", | 20260507 23:45:47.088 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:47.088 | INFO | "malloc" : " 14 GB", | 20260507 23:45:47.088 | INFO | "pinned" : " 29 GB" | 20260507 23:45:47.089 | INFO | } | 20260507 23:45:47.089 | INFO | Perf report for the last 5.13142 seconds: | 20260507 23:45:47.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:47.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 23:45:47.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:45:47.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:47.089 | INFO | Uploaded 36GiB to GPU in 2.53898s (14.1789GiB/s) | 20260507 23:45:47.089 | INFO | Downloaded 34GiB to GPU in 2.47588s (13.7325GiB/s) | 20260507 23:45:52.230 | INFO | Analyzing chunk frames = [3548672,3549184) | 20260507 23:45:52.230 | INFO | MemInfo: { | 20260507 23:45:52.230 | INFO | "gpu" : " 17 GB", | 20260507 23:45:52.230 | INFO | "huge" : "", | 20260507 23:45:52.230 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:52.231 | INFO | "malloc" : " 14 GB", | 20260507 23:45:52.231 | INFO | "pinned" : " 29 GB" | 20260507 23:45:52.231 | INFO | } | 20260507 23:45:52.231 | INFO | Perf report for the last 5.14199 seconds: | 20260507 23:45:52.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:52.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 23:45:52.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:45:52.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:52.231 | INFO | Uploaded 36GiB to GPU in 2.51602s (14.3083GiB/s) | 20260507 23:45:52.231 | INFO | Downloaded 34GiB to GPU in 2.40553s (14.1341GiB/s) | 20260507 23:45:57.373 | INFO | Analyzing chunk frames = [3549184,3549696) | 20260507 23:45:57.373 | INFO | MemInfo: { | 20260507 23:45:57.373 | INFO | "gpu" : " 17 GB", | 20260507 23:45:57.373 | INFO | "huge" : "", | 20260507 23:45:57.373 | INFO | "huge_shared" : " 29 GB", | 20260507 23:45:57.373 | INFO | "malloc" : " 14 GB", | 20260507 23:45:57.373 | INFO | "pinned" : " 29 GB" | 20260507 23:45:57.373 | INFO | } | 20260507 23:45:57.373 | INFO | Perf report for the last 5.14236 seconds: | 20260507 23:45:57.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:45:57.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260507 23:45:57.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:45:57.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:45:57.373 | INFO | Uploaded 36GiB to GPU in 2.36033s (15.2521GiB/s) | 20260507 23:45:57.373 | INFO | Downloaded 34GiB to GPU in 2.47612s (13.7312GiB/s) | 20260507 23:46:02.509 | INFO | Analyzing chunk frames = [3549696,3550208) | 20260507 23:46:02.509 | INFO | MemInfo: { | 20260507 23:46:02.509 | INFO | "gpu" : " 17 GB", | 20260507 23:46:02.509 | INFO | "huge" : "", | 20260507 23:46:02.509 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:02.509 | INFO | "malloc" : " 14 GB", | 20260507 23:46:02.509 | INFO | "pinned" : " 29 GB" | 20260507 23:46:02.509 | INFO | } | 20260507 23:46:02.509 | INFO | Perf report for the last 5.13639 seconds: | 20260507 23:46:02.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:02.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.509 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 23:46:02.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.510 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:46:02.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:02.510 | INFO | Uploaded 36GiB to GPU in 2.54244s (14.1596GiB/s) | 20260507 23:46:02.510 | INFO | Downloaded 34GiB to GPU in 2.42244s (14.0355GiB/s) | 20260507 23:46:07.646 | INFO | Analyzing chunk frames = [3550208,3550720) | 20260507 23:46:07.646 | INFO | MemInfo: { | 20260507 23:46:07.646 | INFO | "gpu" : " 17 GB", | 20260507 23:46:07.646 | INFO | "huge" : "", | 20260507 23:46:07.646 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:07.646 | INFO | "malloc" : " 14 GB", | 20260507 23:46:07.646 | INFO | "pinned" : " 29 GB" | 20260507 23:46:07.646 | INFO | } | 20260507 23:46:07.646 | INFO | Perf report for the last 5.13637 seconds: | 20260507 23:46:07.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:07.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 23:46:07.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 23:46:07.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:07.646 | INFO | Uploaded 36GiB to GPU in 2.53934s (14.1769GiB/s) | 20260507 23:46:07.646 | INFO | Downloaded 34GiB to GPU in 2.44927s (13.8817GiB/s) | 20260507 23:46:12.787 | INFO | Analyzing chunk frames = [3550720,3551232) | 20260507 23:46:12.787 | INFO | MemInfo: { | 20260507 23:46:12.787 | INFO | "gpu" : " 17 GB", | 20260507 23:46:12.787 | INFO | "huge" : "", | 20260507 23:46:12.787 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:12.787 | INFO | "malloc" : " 14 GB", | 20260507 23:46:12.787 | INFO | "pinned" : " 29 GB" | 20260507 23:46:12.787 | INFO | } | 20260507 23:46:12.787 | INFO | Perf report for the last 5.14132 seconds: | 20260507 23:46:12.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:12.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:46:12.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:46:12.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:12.787 | INFO | Uploaded 36GiB to GPU in 2.52384s (14.264GiB/s) | 20260507 23:46:12.788 | INFO | Downloaded 34GiB to GPU in 2.43838s (13.9437GiB/s) | 20260507 23:46:12.884 | INFO | Starting a round of Baz IO | 20260507 23:46:16.346 | INFO | Populating RTMetrics report | 20260507 23:46:16.347 | INFO | RTMetrics report populated | 20260507 23:46:16.347 | INFO | Opening tmp RTMetrics json file | 20260507 23:46:16.367 | INFO | Serializing RTMetrics to json | 20260507 23:46:16.425 | INFO | Serialization done, swapping tmp file now... | 20260507 23:46:16.449 | INFO | RTMetrics done | 20260507 23:46:16.483 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.599 seconds | 20260507 23:46:16.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.47% / 0.00% / 57.53% | 20260507 23:46:16.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1261% / 7.94729e-06% / 8.34465e-05% | 20260507 23:46:16.483 | INFO | Baz IO overall: | 20260507 23:46:16.483 | INFO | BytesWritten : 1615975502078 | 20260507 23:46:16.483 | INFO | NumEvents : 782424376506 | 20260507 23:46:17.930 | INFO | Analyzing chunk frames = [3551232,3551744) | 20260507 23:46:17.930 | INFO | MemInfo: { | 20260507 23:46:17.930 | INFO | "gpu" : " 17 GB", | 20260507 23:46:17.930 | INFO | "huge" : "", | 20260507 23:46:17.930 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:17.930 | INFO | "malloc" : " 14 GB", | 20260507 23:46:17.930 | INFO | "pinned" : " 29 GB" | 20260507 23:46:17.930 | INFO | } | 20260507 23:46:17.931 | INFO | Perf report for the last 5.1433 seconds: | 20260507 23:46:17.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:17.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:17.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:17.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:17.931 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260507 23:46:17.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:17.931 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 23:46:17.931 | INFO | PRE_HQ : 65.2%, 1.7, 57.0ms, 0 | 20260507 23:46:17.931 | INFO | BAZWRITER : 69.6%, 3.6, 129.6ms, 0 | 20260507 23:46:17.931 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260507 23:46:17.931 | INFO | Uploaded 36GiB to GPU in 2.53359s (14.2091GiB/s) | 20260507 23:46:17.931 | INFO | Downloaded 38GiB to GPU in 2.68863s (14.1336GiB/s) | 20260507 23:46:23.071 | INFO | Analyzing chunk frames = [3551744,3552256) | 20260507 23:46:23.071 | INFO | MemInfo: { | 20260507 23:46:23.071 | INFO | "gpu" : " 17 GB", | 20260507 23:46:23.071 | INFO | "huge" : "", | 20260507 23:46:23.072 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:23.072 | INFO | "malloc" : " 14 GB", | 20260507 23:46:23.072 | INFO | "pinned" : " 29 GB" | 20260507 23:46:23.072 | INFO | } | 20260507 23:46:23.072 | INFO | Perf report for the last 5.14119 seconds: | 20260507 23:46:23.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:23.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 23:46:23.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:46:23.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:23.072 | INFO | Uploaded 36GiB to GPU in 2.58396s (13.9321GiB/s) | 20260507 23:46:23.072 | INFO | Downloaded 34GiB to GPU in 2.47603s (13.7317GiB/s) | 20260507 23:46:28.206 | INFO | Analyzing chunk frames = [3552256,3552768) | 20260507 23:46:28.206 | INFO | MemInfo: { | 20260507 23:46:28.206 | INFO | "gpu" : " 17 GB", | 20260507 23:46:28.206 | INFO | "huge" : "", | 20260507 23:46:28.206 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:28.206 | INFO | "malloc" : " 14 GB", | 20260507 23:46:28.206 | INFO | "pinned" : " 29 GB" | 20260507 23:46:28.206 | INFO | } | 20260507 23:46:28.206 | INFO | Perf report for the last 5.13451 seconds: | 20260507 23:46:28.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:28.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.206 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:46:28.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.206 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:46:28.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:28.207 | INFO | Uploaded 36GiB to GPU in 2.54254s (14.1591GiB/s) | 20260507 23:46:28.207 | INFO | Downloaded 34GiB to GPU in 2.47633s (13.73GiB/s) | 20260507 23:46:33.346 | INFO | Analyzing chunk frames = [3552768,3553280) | 20260507 23:46:33.346 | INFO | MemInfo: { | 20260507 23:46:33.347 | INFO | "gpu" : " 17 GB", | 20260507 23:46:33.347 | INFO | "huge" : "", | 20260507 23:46:33.347 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:33.347 | INFO | "malloc" : " 14 GB", | 20260507 23:46:33.347 | INFO | "pinned" : " 29 GB" | 20260507 23:46:33.347 | INFO | } | 20260507 23:46:33.347 | INFO | Perf report for the last 5.14055 seconds: | 20260507 23:46:33.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:33.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.347 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 23:46:33.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.347 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:46:33.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:33.348 | INFO | Uploaded 36GiB to GPU in 2.54716s (14.1334GiB/s) | 20260507 23:46:33.348 | INFO | Downloaded 34GiB to GPU in 2.48926s (13.6587GiB/s) | 20260507 23:46:38.485 | INFO | Analyzing chunk frames = [3553280,3553792) | 20260507 23:46:38.485 | INFO | MemInfo: { | 20260507 23:46:38.485 | INFO | "gpu" : " 17 GB", | 20260507 23:46:38.485 | INFO | "huge" : "", | 20260507 23:46:38.485 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:38.485 | INFO | "malloc" : " 14 GB", | 20260507 23:46:38.485 | INFO | "pinned" : " 29 GB" | 20260507 23:46:38.485 | INFO | } | 20260507 23:46:38.485 | INFO | Perf report for the last 5.13853 seconds: | 20260507 23:46:38.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:38.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260507 23:46:38.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:46:38.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:38.486 | INFO | Uploaded 36GiB to GPU in 2.37937s (15.1301GiB/s) | 20260507 23:46:38.486 | INFO | Downloaded 34GiB to GPU in 2.47777s (13.722GiB/s) | 20260507 23:46:43.626 | INFO | Analyzing chunk frames = [3553792,3554304) | 20260507 23:46:43.626 | INFO | MemInfo: { | 20260507 23:46:43.626 | INFO | "gpu" : " 17 GB", | 20260507 23:46:43.626 | INFO | "huge" : "", | 20260507 23:46:43.627 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:43.627 | INFO | "malloc" : " 14 GB", | 20260507 23:46:43.627 | INFO | "pinned" : " 29 GB" | 20260507 23:46:43.627 | INFO | } | 20260507 23:46:43.627 | INFO | Perf report for the last 5.14137 seconds: | 20260507 23:46:43.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:43.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.627 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260507 23:46:43.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.627 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 23:46:43.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:43.628 | INFO | Uploaded 36GiB to GPU in 2.39177s (15.0516GiB/s) | 20260507 23:46:43.628 | INFO | Downloaded 34GiB to GPU in 2.39626s (14.1888GiB/s) | 20260507 23:46:48.770 | INFO | Analyzing chunk frames = [3554304,3554816) | 20260507 23:46:48.770 | INFO | MemInfo: { | 20260507 23:46:48.770 | INFO | "gpu" : " 17 GB", | 20260507 23:46:48.770 | INFO | "huge" : "", | 20260507 23:46:48.770 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:48.770 | INFO | "malloc" : " 14 GB", | 20260507 23:46:48.770 | INFO | "pinned" : " 29 GB" | 20260507 23:46:48.770 | INFO | } | 20260507 23:46:48.770 | INFO | Perf report for the last 5.14331 seconds: | 20260507 23:46:48.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:48.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.770 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260507 23:46:48.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.770 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 23:46:48.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:48.771 | INFO | Uploaded 36GiB to GPU in 2.54492s (14.1458GiB/s) | 20260507 23:46:48.771 | INFO | Downloaded 34GiB to GPU in 2.46082s (13.8165GiB/s) | 20260507 23:46:53.904 | INFO | Analyzing chunk frames = [3554816,3555328) | 20260507 23:46:53.904 | INFO | MemInfo: { | 20260507 23:46:53.904 | INFO | "gpu" : " 17 GB", | 20260507 23:46:53.904 | INFO | "huge" : "", | 20260507 23:46:53.904 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:53.904 | INFO | "malloc" : " 14 GB", | 20260507 23:46:53.904 | INFO | "pinned" : " 29 GB" | 20260507 23:46:53.904 | INFO | } | 20260507 23:46:53.905 | INFO | Perf report for the last 5.13441 seconds: | 20260507 23:46:53.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:53.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:46:53.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260507 23:46:53.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:53.905 | INFO | Uploaded 36GiB to GPU in 2.53708s (14.1895GiB/s) | 20260507 23:46:53.905 | INFO | Downloaded 34GiB to GPU in 2.4589s (13.8273GiB/s) | 20260507 23:46:53.905 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:46:53.905 | INFO | Total mem usage: 16.7 GiB | 20260507 23:46:53.905 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:46:53.905 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.9%) | 20260507 23:46:53.905 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:46:53.905 | INFO | High Water Breakdown: | 20260507 23:46:53.905 | INFO | 59.9% : 8.85 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:46:53.905 | INFO | 40% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:46:53.905 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:46:53.905 | INFO | ------------------------------------------ | 20260507 23:46:53.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:46:53.905 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:46:53.905 | INFO | Total mem usage: 26.5 GiB | 20260507 23:46:53.905 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:46:53.905 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:46:53.905 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:46:53.905 | INFO | High Water Breakdown: | 20260507 23:46:53.905 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:46:53.905 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:46:53.905 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:46:53.905 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:46:53.905 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:46:53.905 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:46:53.905 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:46:53.905 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:46:53.905 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:46:53.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:46:53.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:46:53.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:46:53.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:46:53.905 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:46:53.905 | INFO | ------------------------------------------ | 20260507 23:46:53.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:46:53.905 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:46:53.905 | INFO | Total mem usage: 27.8 GiB | 20260507 23:46:53.905 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:46:53.905 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:46:53.905 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:46:53.905 | INFO | High Water Breakdown: | 20260507 23:46:53.905 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:46:53.905 | INFO | ------------------------------------------ | 20260507 23:46:53.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:46:53.905 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:46:53.905 | INFO | Total mem usage: 15.7 GiB | 20260507 23:46:53.905 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:46:53.905 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:46:53.905 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:46:53.905 | INFO | High Water Breakdown: | 20260507 23:46:53.905 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:46:53.905 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:46:53.905 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:46:53.905 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:46:53.905 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:46:53.905 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:46:53.905 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:46:53.905 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:46:53.905 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:46:53.905 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:46:53.905 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:46:53.905 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:46:53.905 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:46:53.906 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:46:53.906 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:46:53.906 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:46:53.906 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:46:53.906 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:46:53.906 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:46:53.906 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:46:53.906 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:46:53.906 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:46:53.906 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:46:53.906 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:46:53.906 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:46:53.906 | INFO | ------------------------------------------ | 20260507 23:46:53.906 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:46:53.906 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:46:53.906 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:46:53.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:46:53.906 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.40e+01 / 1.52e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.15e-02 / 6.56e-03 | 20260507 23:46:53.906 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.46e+00, 5.82e-01 / 2.18e-01 | 20260507 23:46:53.906 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.06e-01 / 1.02e-01 | 20260507 23:46:53.906 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.62e+01, 4.08e+00 / 1.74e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.63e+00 / 1.51e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.32e-01 / 2.07e-01 | 20260507 23:46:53.906 | INFO | PerfReport: 3.43%, Metrics -- 1.12e+00 / 1.34e+00, 2.81e-01 / 1.52e-01 | 20260507 23:46:53.906 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.47e-01 / 3.83e-01, 5.56e-02 / 2.63e-02 | 20260507 23:46:53.906 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:46:53.906 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:46:53.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:46:53.906 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 24.54%, Upload -- 2.62e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 30.68%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 2.40%, LockUpload -- 2.56e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 2.50%, LockDownload -- 2.66e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260507 23:46:53.906 | INFO | PerfReport: 15.61%, LockCompute -- 1.67e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260507 23:46:53.907 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 23:46:54.019 | INFO | Starting a round of Baz IO | 20260507 23:46:57.479 | INFO | Populating RTMetrics report | 20260507 23:46:57.479 | INFO | RTMetrics report populated | 20260507 23:46:57.479 | INFO | Opening tmp RTMetrics json file | 20260507 23:46:57.657 | INFO | Serializing RTMetrics to json | 20260507 23:46:57.659 | INFO | Serialization done, swapping tmp file now... | 20260507 23:46:57.659 | INFO | RTMetrics done | 20260507 23:46:57.680 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.66 seconds | 20260507 23:46:57.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.46% / 0.01% / 57.53% | 20260507 23:46:57.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12423% / 3.97364e-06% / 8.34465e-05% | 20260507 23:46:57.680 | INFO | Baz IO overall: | 20260507 23:46:57.680 | INFO | BytesWritten : 1617806391144 | 20260507 23:46:57.680 | INFO | NumEvents : 783305743187 | 20260507 23:46:59.041 | INFO | Analyzing chunk frames = [3555328,3555840) | 20260507 23:46:59.041 | INFO | MemInfo: { | 20260507 23:46:59.041 | INFO | "gpu" : " 17 GB", | 20260507 23:46:59.041 | INFO | "huge" : "", | 20260507 23:46:59.041 | INFO | "huge_shared" : " 29 GB", | 20260507 23:46:59.041 | INFO | "malloc" : " 14 GB", | 20260507 23:46:59.041 | INFO | "pinned" : " 29 GB" | 20260507 23:46:59.041 | INFO | } | 20260507 23:46:59.041 | INFO | Perf report for the last 5.13645 seconds: | 20260507 23:46:59.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:46:59.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:59.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:59.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:59.041 | INFO | ANALYSIS : 68.8%, 2.9, 105.2ms, 0 | 20260507 23:46:59.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:46:59.042 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260507 23:46:59.042 | INFO | PRE_HQ : 67.4%, 1.8, 61.3ms, 0 | 20260507 23:46:59.042 | INFO | BAZWRITER : 70.7%, 3.9, 140.9ms, 0 | 20260507 23:46:59.042 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260507 23:46:59.043 | INFO | Uploaded 36GiB to GPU in 2.531s (14.2236GiB/s) | 20260507 23:46:59.043 | INFO | Downloaded 38GiB to GPU in 2.70786s (14.0332GiB/s) | 20260507 23:47:04.184 | INFO | Analyzing chunk frames = [3555840,3556352) | 20260507 23:47:04.184 | INFO | MemInfo: { | 20260507 23:47:04.184 | INFO | "gpu" : " 17 GB", | 20260507 23:47:04.184 | INFO | "huge" : "", | 20260507 23:47:04.184 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:04.184 | INFO | "malloc" : " 14 GB", | 20260507 23:47:04.184 | INFO | "pinned" : " 29 GB" | 20260507 23:47:04.184 | INFO | } | 20260507 23:47:04.184 | INFO | Perf report for the last 5.14318 seconds: | 20260507 23:47:04.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:04.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.184 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260507 23:47:04.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.184 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:47:04.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:04.185 | INFO | Uploaded 36GiB to GPU in 2.55432s (14.0938GiB/s) | 20260507 23:47:04.185 | INFO | Downloaded 34GiB to GPU in 2.49848s (13.6083GiB/s) | 20260507 23:47:09.322 | INFO | Analyzing chunk frames = [3556352,3556864) | 20260507 23:47:09.322 | INFO | MemInfo: { | 20260507 23:47:09.322 | INFO | "gpu" : " 17 GB", | 20260507 23:47:09.322 | INFO | "huge" : "", | 20260507 23:47:09.322 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:09.322 | INFO | "malloc" : " 14 GB", | 20260507 23:47:09.322 | INFO | "pinned" : " 29 GB" | 20260507 23:47:09.322 | INFO | } | 20260507 23:47:09.322 | INFO | Perf report for the last 5.13781 seconds: | 20260507 23:47:09.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:09.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.322 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260507 23:47:09.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.322 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:47:09.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:09.323 | INFO | Uploaded 36GiB to GPU in 2.42584s (14.8402GiB/s) | 20260507 23:47:09.323 | INFO | Downloaded 34GiB to GPU in 2.49308s (13.6377GiB/s) | 20260507 23:47:14.456 | INFO | Analyzing chunk frames = [3556864,3557376) | 20260507 23:47:14.456 | INFO | MemInfo: { | 20260507 23:47:14.456 | INFO | "gpu" : " 17 GB", | 20260507 23:47:14.456 | INFO | "huge" : "", | 20260507 23:47:14.456 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:14.456 | INFO | "malloc" : " 14 GB", | 20260507 23:47:14.456 | INFO | "pinned" : " 29 GB" | 20260507 23:47:14.456 | INFO | } | 20260507 23:47:14.456 | INFO | Perf report for the last 5.13434 seconds: | 20260507 23:47:14.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:14.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.456 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260507 23:47:14.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.456 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:47:14.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:14.457 | INFO | Uploaded 36GiB to GPU in 2.48486s (14.4878GiB/s) | 20260507 23:47:14.457 | INFO | Downloaded 34GiB to GPU in 2.49102s (13.649GiB/s) | 20260507 23:47:19.596 | INFO | Analyzing chunk frames = [3557376,3557888) | 20260507 23:47:19.596 | INFO | MemInfo: { | 20260507 23:47:19.596 | INFO | "gpu" : " 17 GB", | 20260507 23:47:19.596 | INFO | "huge" : "", | 20260507 23:47:19.596 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:19.596 | INFO | "malloc" : " 14 GB", | 20260507 23:47:19.596 | INFO | "pinned" : " 29 GB" | 20260507 23:47:19.596 | INFO | } | 20260507 23:47:19.596 | INFO | Perf report for the last 5.13945 seconds: | 20260507 23:47:19.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:19.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:47:19.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 23:47:19.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:19.596 | INFO | Uploaded 36GiB to GPU in 2.31772s (15.5325GiB/s) | 20260507 23:47:19.596 | INFO | Downloaded 34GiB to GPU in 2.5153s (13.5173GiB/s) | 20260507 23:47:24.740 | INFO | Analyzing chunk frames = [3557888,3558400) | 20260507 23:47:24.740 | INFO | MemInfo: { | 20260507 23:47:24.740 | INFO | "gpu" : " 17 GB", | 20260507 23:47:24.740 | INFO | "huge" : "", | 20260507 23:47:24.740 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:24.740 | INFO | "malloc" : " 14 GB", | 20260507 23:47:24.740 | INFO | "pinned" : " 29 GB" | 20260507 23:47:24.740 | INFO | } | 20260507 23:47:24.740 | INFO | Perf report for the last 5.14406 seconds: | 20260507 23:47:24.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:24.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260507 23:47:24.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:47:24.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:24.740 | INFO | Uploaded 36GiB to GPU in 2.50436s (14.3749GiB/s) | 20260507 23:47:24.740 | INFO | Downloaded 34GiB to GPU in 2.43586s (13.9581GiB/s) | 20260507 23:47:29.881 | INFO | Analyzing chunk frames = [3558400,3558912) | 20260507 23:47:29.881 | INFO | MemInfo: { | 20260507 23:47:29.881 | INFO | "gpu" : " 17 GB", | 20260507 23:47:29.881 | INFO | "huge" : "", | 20260507 23:47:29.881 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:29.881 | INFO | "malloc" : " 14 GB", | 20260507 23:47:29.881 | INFO | "pinned" : " 29 GB" | 20260507 23:47:29.881 | INFO | } | 20260507 23:47:29.881 | INFO | Perf report for the last 5.14128 seconds: | 20260507 23:47:29.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:29.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.881 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:47:29.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.881 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 23:47:29.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:29.882 | INFO | Uploaded 36GiB to GPU in 2.52858s (14.2372GiB/s) | 20260507 23:47:29.882 | INFO | Downloaded 34GiB to GPU in 2.46739s (13.7797GiB/s) | 20260507 23:47:35.025 | INFO | Analyzing chunk frames = [3558912,3559424) | 20260507 23:47:35.025 | INFO | MemInfo: { | 20260507 23:47:35.025 | INFO | "gpu" : " 17 GB", | 20260507 23:47:35.025 | INFO | "huge" : "", | 20260507 23:47:35.025 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:35.025 | INFO | "malloc" : " 14 GB", | 20260507 23:47:35.026 | INFO | "pinned" : " 29 GB" | 20260507 23:47:35.026 | INFO | } | 20260507 23:47:35.026 | INFO | Perf report for the last 5.14446 seconds: | 20260507 23:47:35.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:35.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:47:35.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:47:35.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:35.026 | INFO | Uploaded 36GiB to GPU in 2.51402s (14.3197GiB/s) | 20260507 23:47:35.026 | INFO | Downloaded 34GiB to GPU in 2.45231s (13.8645GiB/s) | 20260507 23:47:35.123 | INFO | Starting a round of Baz IO | 20260507 23:47:38.587 | INFO | Populating RTMetrics report | 20260507 23:47:38.588 | INFO | RTMetrics report populated | 20260507 23:47:38.588 | INFO | Opening tmp RTMetrics json file | 20260507 23:47:38.608 | INFO | Serializing RTMetrics to json | 20260507 23:47:38.627 | INFO | Serialization done, swapping tmp file now... | 20260507 23:47:38.654 | INFO | RTMetrics done | 20260507 23:47:38.682 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.558 seconds | 20260507 23:47:38.682 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.46% / 0.00% / 57.54% | 20260507 23:47:38.682 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11918% / 1.19209e-05% / 7.54992e-05% | 20260507 23:47:38.682 | INFO | Baz IO overall: | 20260507 23:47:38.682 | INFO | BytesWritten : 1619634568984 | 20260507 23:47:38.682 | INFO | NumEvents : 784185536681 | 20260507 23:47:40.160 | INFO | Analyzing chunk frames = [3559424,3559936) | 20260507 23:47:40.160 | INFO | MemInfo: { | 20260507 23:47:40.160 | INFO | "gpu" : " 17 GB", | 20260507 23:47:40.160 | INFO | "huge" : "", | 20260507 23:47:40.160 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:40.160 | INFO | "malloc" : " 14 GB", | 20260507 23:47:40.160 | INFO | "pinned" : " 29 GB" | 20260507 23:47:40.160 | INFO | } | 20260507 23:47:40.160 | INFO | Perf report for the last 5.13475 seconds: | 20260507 23:47:40.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:40.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:40.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:40.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:40.161 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260507 23:47:40.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:40.161 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 23:47:40.161 | INFO | PRE_HQ : 66.0%, 1.7, 56.8ms, 0 | 20260507 23:47:40.161 | INFO | BAZWRITER : 68.8%, 4.4, 154.1ms, 0 | 20260507 23:47:40.161 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 23:47:40.161 | INFO | Uploaded 36GiB to GPU in 2.58717s (13.9148GiB/s) | 20260507 23:47:40.161 | INFO | Downloaded 38GiB to GPU in 2.69591s (14.0954GiB/s) | 20260507 23:47:45.297 | INFO | Analyzing chunk frames = [3559936,3560448) | 20260507 23:47:45.297 | INFO | MemInfo: { | 20260507 23:47:45.297 | INFO | "gpu" : " 17 GB", | 20260507 23:47:45.297 | INFO | "huge" : "", | 20260507 23:47:45.297 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:45.297 | INFO | "malloc" : " 14 GB", | 20260507 23:47:45.297 | INFO | "pinned" : " 29 GB" | 20260507 23:47:45.297 | INFO | } | 20260507 23:47:45.297 | INFO | Perf report for the last 5.13682 seconds: | 20260507 23:47:45.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:45.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:47:45.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:47:45.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:45.297 | INFO | Uploaded 36GiB to GPU in 2.52816s (14.2396GiB/s) | 20260507 23:47:45.298 | INFO | Downloaded 34GiB to GPU in 2.43737s (13.9495GiB/s) | 20260507 23:47:50.438 | INFO | Analyzing chunk frames = [3560448,3560960) | 20260507 23:47:50.438 | INFO | MemInfo: { | 20260507 23:47:50.438 | INFO | "gpu" : " 17 GB", | 20260507 23:47:50.438 | INFO | "huge" : "", | 20260507 23:47:50.438 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:50.438 | INFO | "malloc" : " 14 GB", | 20260507 23:47:50.438 | INFO | "pinned" : " 29 GB" | 20260507 23:47:50.438 | INFO | } | 20260507 23:47:50.438 | INFO | Perf report for the last 5.14074 seconds: | 20260507 23:47:50.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:50.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:47:50.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:47:50.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:50.438 | INFO | Uploaded 36GiB to GPU in 2.56064s (14.059GiB/s) | 20260507 23:47:50.439 | INFO | Downloaded 34GiB to GPU in 2.48593s (13.677GiB/s) | 20260507 23:47:55.575 | INFO | Analyzing chunk frames = [3560960,3561472) | 20260507 23:47:55.575 | INFO | MemInfo: { | 20260507 23:47:55.575 | INFO | "gpu" : " 17 GB", | 20260507 23:47:55.575 | INFO | "huge" : "", | 20260507 23:47:55.575 | INFO | "huge_shared" : " 29 GB", | 20260507 23:47:55.575 | INFO | "malloc" : " 14 GB", | 20260507 23:47:55.575 | INFO | "pinned" : " 29 GB" | 20260507 23:47:55.575 | INFO | } | 20260507 23:47:55.575 | INFO | Perf report for the last 5.13703 seconds: | 20260507 23:47:55.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:47:55.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.575 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:47:55.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.575 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:47:55.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:47:55.576 | INFO | Uploaded 36GiB to GPU in 2.50325s (14.3813GiB/s) | 20260507 23:47:55.576 | INFO | Downloaded 34GiB to GPU in 2.49191s (13.6441GiB/s) | 20260507 23:48:00.715 | INFO | Analyzing chunk frames = [3561472,3561984) | 20260507 23:48:00.715 | INFO | MemInfo: { | 20260507 23:48:00.715 | INFO | "gpu" : " 17 GB", | 20260507 23:48:00.715 | INFO | "huge" : "", | 20260507 23:48:00.715 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:00.715 | INFO | "malloc" : " 14 GB", | 20260507 23:48:00.715 | INFO | "pinned" : " 29 GB" | 20260507 23:48:00.715 | INFO | } | 20260507 23:48:00.716 | INFO | Perf report for the last 5.14048 seconds: | 20260507 23:48:00.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:00.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260507 23:48:00.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:48:00.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:00.716 | INFO | Uploaded 36GiB to GPU in 2.35903s (15.2605GiB/s) | 20260507 23:48:00.716 | INFO | Downloaded 34GiB to GPU in 2.50051s (13.5972GiB/s) | 20260507 23:48:05.852 | INFO | Analyzing chunk frames = [3561984,3562496) | 20260507 23:48:05.852 | INFO | MemInfo: { | 20260507 23:48:05.852 | INFO | "gpu" : " 17 GB", | 20260507 23:48:05.852 | INFO | "huge" : "", | 20260507 23:48:05.852 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:05.852 | INFO | "malloc" : " 14 GB", | 20260507 23:48:05.852 | INFO | "pinned" : " 29 GB" | 20260507 23:48:05.852 | INFO | } | 20260507 23:48:05.852 | INFO | Perf report for the last 5.13691 seconds: | 20260507 23:48:05.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:05.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 23:48:05.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260507 23:48:05.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:05.853 | INFO | Uploaded 36GiB to GPU in 2.4682s (14.5855GiB/s) | 20260507 23:48:05.853 | INFO | Downloaded 34GiB to GPU in 2.44905s (13.883GiB/s) | 20260507 23:48:10.997 | INFO | Analyzing chunk frames = [3562496,3563008) | 20260507 23:48:10.997 | INFO | MemInfo: { | 20260507 23:48:10.997 | INFO | "gpu" : " 17 GB", | 20260507 23:48:10.997 | INFO | "huge" : "", | 20260507 23:48:10.997 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:10.997 | INFO | "malloc" : " 14 GB", | 20260507 23:48:10.997 | INFO | "pinned" : " 29 GB" | 20260507 23:48:10.997 | INFO | } | 20260507 23:48:10.997 | INFO | Perf report for the last 5.14498 seconds: | 20260507 23:48:10.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:10.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:48:10.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 23:48:10.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:10.998 | INFO | Uploaded 36GiB to GPU in 2.51241s (14.3289GiB/s) | 20260507 23:48:10.998 | INFO | Downloaded 34GiB to GPU in 2.47822s (13.7195GiB/s) | 20260507 23:48:16.136 | INFO | Analyzing chunk frames = [3563008,3563520) | 20260507 23:48:16.136 | INFO | MemInfo: { | 20260507 23:48:16.136 | INFO | "gpu" : " 17 GB", | 20260507 23:48:16.136 | INFO | "huge" : "", | 20260507 23:48:16.136 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:16.136 | INFO | "malloc" : " 14 GB", | 20260507 23:48:16.136 | INFO | "pinned" : " 29 GB" | 20260507 23:48:16.136 | INFO | } | 20260507 23:48:16.136 | INFO | Perf report for the last 5.13868 seconds: | 20260507 23:48:16.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:16.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.136 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:48:16.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.136 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260507 23:48:16.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:16.137 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:48:16.137 | INFO | Total mem usage: 16.7 GiB | 20260507 23:48:16.137 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:48:16.137 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260507 23:48:16.137 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:48:16.137 | INFO | High Water Breakdown: | 20260507 23:48:16.137 | INFO | 60.2% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:48:16.137 | INFO | 39.7% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:48:16.137 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:48:16.137 | INFO | ------------------------------------------ | 20260507 23:48:16.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:48:16.137 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:48:16.137 | INFO | Total mem usage: 26.5 GiB | 20260507 23:48:16.137 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:48:16.137 | INFO | Uploaded 36GiB to GPU in 2.51267s (14.3274GiB/s) | 20260507 23:48:16.137 | INFO | Downloaded 34GiB to GPU in 2.46529s (13.7915GiB/s) | 20260507 23:48:16.137 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:48:16.137 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:48:16.137 | INFO | High Water Breakdown: | 20260507 23:48:16.137 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:48:16.137 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:48:16.137 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:48:16.137 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:48:16.137 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:48:16.137 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:48:16.137 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:48:16.137 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:48:16.137 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:48:16.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:48:16.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:48:16.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:48:16.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:48:16.137 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:48:16.137 | INFO | ------------------------------------------ | 20260507 23:48:16.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:48:16.137 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:48:16.137 | INFO | Total mem usage: 27.8 GiB | 20260507 23:48:16.137 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:48:16.137 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:48:16.137 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:48:16.137 | INFO | High Water Breakdown: | 20260507 23:48:16.137 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:48:16.137 | INFO | ------------------------------------------ | 20260507 23:48:16.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:48:16.138 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:48:16.138 | INFO | Total mem usage: 15.7 GiB | 20260507 23:48:16.138 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:48:16.138 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:48:16.138 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:48:16.138 | INFO | High Water Breakdown: | 20260507 23:48:16.138 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:48:16.138 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:48:16.138 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:48:16.138 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:48:16.138 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:48:16.138 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:48:16.138 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:48:16.138 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:48:16.138 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:48:16.138 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:48:16.138 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:48:16.138 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:48:16.138 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:48:16.138 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:48:16.138 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:48:16.138 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:48:16.138 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:48:16.138 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:48:16.138 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:48:16.138 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:48:16.138 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:48:16.138 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:48:16.138 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:48:16.138 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:48:16.138 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:48:16.138 | INFO | ------------------------------------------ | 20260507 23:48:16.138 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:48:16.138 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:48:16.138 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:48:16.138 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:48:16.138 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.43e+01 / 1.60e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.18e-02 / 5.54e-03 | 20260507 23:48:16.138 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.48e+00, 5.81e-01 / 1.91e-01 | 20260507 23:48:16.138 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.05e-01 | 20260507 23:48:16.138 | INFO | PerfReport: 16.02%, DME -- 4.20e+01 / 4.59e+01, 3.88e+00 / 1.58e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.84e+01 / 1.99e+01, 1.66e+00 / 1.58e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.46e+00 / 2.71e+00, 4.31e-01 / 1.80e-01 | 20260507 23:48:16.138 | INFO | PerfReport: 3.42%, Metrics -- 1.12e+00 / 1.33e+00, 2.80e-01 / 1.73e-01 | 20260507 23:48:16.138 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.46e-01 / 3.82e-01, 5.54e-02 / 3.24e-02 | 20260507 23:48:16.138 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:48:16.138 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:48:16.138 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:48:16.138 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 24.35%, Upload -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 24.45%, Download -- 2.61e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 30.75%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 2.91%, LockDownload -- 3.10e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 23:48:16.138 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 2.94e-02 / 0.00e+00 | 20260507 23:48:16.253 | INFO | Starting a round of Baz IO | 20260507 23:48:19.719 | INFO | Populating RTMetrics report | 20260507 23:48:19.720 | INFO | RTMetrics report populated | 20260507 23:48:19.720 | INFO | Opening tmp RTMetrics json file | 20260507 23:48:19.721 | INFO | Serializing RTMetrics to json | 20260507 23:48:19.723 | INFO | Serialization done, swapping tmp file now... | 20260507 23:48:19.724 | INFO | RTMetrics done | 20260507 23:48:19.825 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.572 seconds | 20260507 23:48:19.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.45% / 0.01% / 57.54% | 20260507 23:48:19.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12005% / 7.94729e-06% / 8.34465e-05% | 20260507 23:48:19.826 | INFO | Baz IO overall: | 20260507 23:48:19.826 | INFO | BytesWritten : 1621464798241 | 20260507 23:48:19.826 | INFO | NumEvents : 785067100370 | 20260507 23:48:21.274 | INFO | Analyzing chunk frames = [3563520,3564032) | 20260507 23:48:21.274 | INFO | MemInfo: { | 20260507 23:48:21.274 | INFO | "gpu" : " 17 GB", | 20260507 23:48:21.274 | INFO | "huge" : "", | 20260507 23:48:21.274 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:21.274 | INFO | "malloc" : " 14 GB", | 20260507 23:48:21.274 | INFO | "pinned" : " 29 GB" | 20260507 23:48:21.274 | INFO | } | 20260507 23:48:21.275 | INFO | Perf report for the last 5.13833 seconds: | 20260507 23:48:21.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:21.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:21.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:21.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:21.275 | INFO | ANALYSIS : 69.3%, 2.9, 104.9ms, 0 | 20260507 23:48:21.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:21.275 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 23:48:21.275 | INFO | PRE_HQ : 68.1%, 1.8, 63.4ms, 0 | 20260507 23:48:21.275 | INFO | BAZWRITER : 69.0%, 3.9, 136.5ms, 0 | 20260507 23:48:21.275 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260507 23:48:21.275 | INFO | Uploaded 36GiB to GPU in 2.54344s (14.154GiB/s) | 20260507 23:48:21.275 | INFO | Downloaded 38GiB to GPU in 2.71516s (13.9955GiB/s) | 20260507 23:48:26.408 | INFO | Analyzing chunk frames = [3564032,3564544) | 20260507 23:48:26.408 | INFO | MemInfo: { | 20260507 23:48:26.408 | INFO | "gpu" : " 17 GB", | 20260507 23:48:26.408 | INFO | "huge" : "", | 20260507 23:48:26.408 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:26.408 | INFO | "malloc" : " 14 GB", | 20260507 23:48:26.408 | INFO | "pinned" : " 29 GB" | 20260507 23:48:26.408 | INFO | } | 20260507 23:48:26.408 | INFO | Perf report for the last 5.1339 seconds: | 20260507 23:48:26.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:26.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.408 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260507 23:48:26.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.408 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:48:26.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:26.409 | INFO | Uploaded 36GiB to GPU in 2.56987s (14.0085GiB/s) | 20260507 23:48:26.409 | INFO | Downloaded 34GiB to GPU in 2.4863s (13.6749GiB/s) | 20260507 23:48:31.548 | INFO | Analyzing chunk frames = [3564544,3565056) | 20260507 23:48:31.548 | INFO | MemInfo: { | 20260507 23:48:31.548 | INFO | "gpu" : " 17 GB", | 20260507 23:48:31.548 | INFO | "huge" : "", | 20260507 23:48:31.548 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:31.548 | INFO | "malloc" : " 14 GB", | 20260507 23:48:31.548 | INFO | "pinned" : " 29 GB" | 20260507 23:48:31.548 | INFO | } | 20260507 23:48:31.548 | INFO | Perf report for the last 5.13965 seconds: | 20260507 23:48:31.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:31.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:48:31.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:48:31.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:31.548 | INFO | Uploaded 36GiB to GPU in 2.50149s (14.3914GiB/s) | 20260507 23:48:31.549 | INFO | Downloaded 34GiB to GPU in 2.48153s (13.7012GiB/s) | 20260507 23:48:36.689 | INFO | Analyzing chunk frames = [3565056,3565568) | 20260507 23:48:36.689 | INFO | MemInfo: { | 20260507 23:48:36.689 | INFO | "gpu" : " 17 GB", | 20260507 23:48:36.689 | INFO | "huge" : "", | 20260507 23:48:36.689 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:36.689 | INFO | "malloc" : " 14 GB", | 20260507 23:48:36.689 | INFO | "pinned" : " 29 GB" | 20260507 23:48:36.689 | INFO | } | 20260507 23:48:36.689 | INFO | Perf report for the last 5.14136 seconds: | 20260507 23:48:36.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:36.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.689 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260507 23:48:36.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.689 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:48:36.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:36.690 | INFO | Uploaded 36GiB to GPU in 2.47504s (14.5452GiB/s) | 20260507 23:48:36.690 | INFO | Downloaded 34GiB to GPU in 2.40819s (14.1185GiB/s) | 20260507 23:48:41.837 | INFO | Analyzing chunk frames = [3565568,3566080) | 20260507 23:48:41.837 | INFO | MemInfo: { | 20260507 23:48:41.837 | INFO | "gpu" : " 17 GB", | 20260507 23:48:41.837 | INFO | "huge" : "", | 20260507 23:48:41.837 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:41.837 | INFO | "malloc" : " 14 GB", | 20260507 23:48:41.837 | INFO | "pinned" : " 29 GB" | 20260507 23:48:41.837 | INFO | } | 20260507 23:48:41.837 | INFO | Perf report for the last 5.14787 seconds: | 20260507 23:48:41.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:41.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.837 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260507 23:48:41.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.837 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:48:41.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:41.838 | INFO | Uploaded 36GiB to GPU in 2.36902s (15.1962GiB/s) | 20260507 23:48:41.838 | INFO | Downloaded 34GiB to GPU in 2.50797s (13.5568GiB/s) | 20260507 23:48:46.976 | INFO | Analyzing chunk frames = [3566080,3566592) | 20260507 23:48:46.976 | INFO | MemInfo: { | 20260507 23:48:46.976 | INFO | "gpu" : " 17 GB", | 20260507 23:48:46.976 | INFO | "huge" : "", | 20260507 23:48:46.976 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:46.976 | INFO | "malloc" : " 14 GB", | 20260507 23:48:46.976 | INFO | "pinned" : " 29 GB" | 20260507 23:48:46.976 | INFO | } | 20260507 23:48:46.976 | INFO | Perf report for the last 5.13845 seconds: | 20260507 23:48:46.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:46.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260507 23:48:46.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:48:46.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:46.976 | INFO | Uploaded 36GiB to GPU in 2.40345s (14.9785GiB/s) | 20260507 23:48:46.976 | INFO | Downloaded 34GiB to GPU in 2.39613s (14.1896GiB/s) | 20260507 23:48:52.108 | INFO | Analyzing chunk frames = [3566592,3567104) | 20260507 23:48:52.108 | INFO | MemInfo: { | 20260507 23:48:52.108 | INFO | "gpu" : " 17 GB", | 20260507 23:48:52.108 | INFO | "huge" : "", | 20260507 23:48:52.108 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:52.108 | INFO | "malloc" : " 14 GB", | 20260507 23:48:52.108 | INFO | "pinned" : " 29 GB" | 20260507 23:48:52.108 | INFO | } | 20260507 23:48:52.108 | INFO | Perf report for the last 5.13225 seconds: | 20260507 23:48:52.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:52.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 23:48:52.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 23:48:52.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:52.108 | INFO | Uploaded 36GiB to GPU in 2.54871s (14.1248GiB/s) | 20260507 23:48:52.108 | INFO | Downloaded 34GiB to GPU in 2.44832s (13.8871GiB/s) | 20260507 23:48:57.250 | INFO | Analyzing chunk frames = [3567104,3567616) | 20260507 23:48:57.250 | INFO | MemInfo: { | 20260507 23:48:57.250 | INFO | "gpu" : " 17 GB", | 20260507 23:48:57.250 | INFO | "huge" : "", | 20260507 23:48:57.250 | INFO | "huge_shared" : " 29 GB", | 20260507 23:48:57.250 | INFO | "malloc" : " 14 GB", | 20260507 23:48:57.250 | INFO | "pinned" : " 29 GB" | 20260507 23:48:57.250 | INFO | } | 20260507 23:48:57.251 | INFO | Perf report for the last 5.14243 seconds: | 20260507 23:48:57.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:48:57.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 23:48:57.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260507 23:48:57.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:48:57.251 | INFO | Uploaded 36GiB to GPU in 2.53605s (14.1953GiB/s) | 20260507 23:48:57.251 | INFO | Downloaded 34GiB to GPU in 2.44249s (13.9202GiB/s) | 20260507 23:48:57.348 | INFO | Starting a round of Baz IO | 20260507 23:49:00.802 | INFO | Populating RTMetrics report | 20260507 23:49:00.802 | INFO | RTMetrics report populated | 20260507 23:49:00.802 | INFO | Opening tmp RTMetrics json file | 20260507 23:49:00.813 | INFO | Serializing RTMetrics to json | 20260507 23:49:00.835 | INFO | Serialization done, swapping tmp file now... | 20260507 23:49:00.843 | INFO | RTMetrics done | 20260507 23:49:00.929 | INFO | Finished a round of Baz IO: Wrote 1.704 GiB in 3.58 seconds | 20260507 23:49:00.929 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.45% / 0.01% / 57.55% | 20260507 23:49:00.929 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11945% / 7.94729e-06% / 5.5631e-05% | 20260507 23:49:00.929 | INFO | Baz IO overall: | 20260507 23:49:00.929 | INFO | BytesWritten : 1623294213461 | 20260507 23:49:00.929 | INFO | NumEvents : 785948147787 | 20260507 23:49:02.388 | INFO | Analyzing chunk frames = [3567616,3568128) | 20260507 23:49:02.388 | INFO | MemInfo: { | 20260507 23:49:02.388 | INFO | "gpu" : " 17 GB", | 20260507 23:49:02.388 | INFO | "huge" : "", | 20260507 23:49:02.388 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:02.388 | INFO | "malloc" : " 14 GB", | 20260507 23:49:02.388 | INFO | "pinned" : " 29 GB" | 20260507 23:49:02.388 | INFO | } | 20260507 23:49:02.389 | INFO | Perf report for the last 5.13801 seconds: | 20260507 23:49:02.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:02.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:02.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:02.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:02.389 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260507 23:49:02.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:02.389 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 23:49:02.389 | INFO | PRE_HQ : 65.7%, 1.7, 57.1ms, 0 | 20260507 23:49:02.389 | INFO | BAZWRITER : 69.3%, 3.5, 126.4ms, 0 | 20260507 23:49:02.389 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260507 23:49:02.389 | INFO | Uploaded 36GiB to GPU in 2.56528s (14.0336GiB/s) | 20260507 23:49:02.389 | INFO | Downloaded 38GiB to GPU in 2.68733s (14.1404GiB/s) | 20260507 23:49:07.527 | INFO | Analyzing chunk frames = [3568128,3568640) | 20260507 23:49:07.527 | INFO | MemInfo: { | 20260507 23:49:07.527 | INFO | "gpu" : " 17 GB", | 20260507 23:49:07.527 | INFO | "huge" : "", | 20260507 23:49:07.527 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:07.527 | INFO | "malloc" : " 14 GB", | 20260507 23:49:07.527 | INFO | "pinned" : " 29 GB" | 20260507 23:49:07.527 | INFO | } | 20260507 23:49:07.527 | INFO | Perf report for the last 5.13828 seconds: | 20260507 23:49:07.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:07.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.527 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 23:49:07.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.527 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 23:49:07.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:07.528 | INFO | Uploaded 36GiB to GPU in 2.51624s (14.3071GiB/s) | 20260507 23:49:07.528 | INFO | Downloaded 34GiB to GPU in 2.46104s (13.8153GiB/s) | 20260507 23:49:12.666 | INFO | Analyzing chunk frames = [3568640,3569152) | 20260507 23:49:12.666 | INFO | MemInfo: { | 20260507 23:49:12.666 | INFO | "gpu" : " 17 GB", | 20260507 23:49:12.666 | INFO | "huge" : "", | 20260507 23:49:12.666 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:12.666 | INFO | "malloc" : " 14 GB", | 20260507 23:49:12.666 | INFO | "pinned" : " 29 GB" | 20260507 23:49:12.666 | INFO | } | 20260507 23:49:12.666 | INFO | Perf report for the last 5.13962 seconds: | 20260507 23:49:12.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:12.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.667 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 23:49:12.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.667 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:49:12.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:12.667 | INFO | Uploaded 36GiB to GPU in 2.5602s (14.0614GiB/s) | 20260507 23:49:12.667 | INFO | Downloaded 34GiB to GPU in 2.47495s (13.7377GiB/s) | 20260507 23:49:17.805 | INFO | Analyzing chunk frames = [3569152,3569664) | 20260507 23:49:17.805 | INFO | MemInfo: { | 20260507 23:49:17.805 | INFO | "gpu" : " 17 GB", | 20260507 23:49:17.805 | INFO | "huge" : "", | 20260507 23:49:17.805 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:17.805 | INFO | "malloc" : " 14 GB", | 20260507 23:49:17.805 | INFO | "pinned" : " 29 GB" | 20260507 23:49:17.805 | INFO | } | 20260507 23:49:17.805 | INFO | Perf report for the last 5.139 seconds: | 20260507 23:49:17.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:17.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 23:49:17.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:49:17.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:17.806 | INFO | Uploaded 36GiB to GPU in 2.55011s (14.117GiB/s) | 20260507 23:49:17.806 | INFO | Downloaded 34GiB to GPU in 2.48771s (13.6672GiB/s) | 20260507 23:49:22.946 | INFO | Analyzing chunk frames = [3569664,3570176) | 20260507 23:49:22.946 | INFO | MemInfo: { | 20260507 23:49:22.946 | INFO | "gpu" : " 17 GB", | 20260507 23:49:22.946 | INFO | "huge" : "", | 20260507 23:49:22.946 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:22.946 | INFO | "malloc" : " 14 GB", | 20260507 23:49:22.946 | INFO | "pinned" : " 29 GB" | 20260507 23:49:22.947 | INFO | } | 20260507 23:49:22.947 | INFO | Perf report for the last 5.14113 seconds: | 20260507 23:49:22.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:22.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:49:22.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:49:22.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:22.947 | INFO | Uploaded 36GiB to GPU in 2.44999s (14.6939GiB/s) | 20260507 23:49:22.947 | INFO | Downloaded 34GiB to GPU in 2.50224s (13.5878GiB/s) | 20260507 23:49:28.090 | INFO | Analyzing chunk frames = [3570176,3570688) | 20260507 23:49:28.090 | INFO | MemInfo: { | 20260507 23:49:28.090 | INFO | "gpu" : " 17 GB", | 20260507 23:49:28.090 | INFO | "huge" : "", | 20260507 23:49:28.090 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:28.090 | INFO | "malloc" : " 14 GB", | 20260507 23:49:28.090 | INFO | "pinned" : " 29 GB" | 20260507 23:49:28.090 | INFO | } | 20260507 23:49:28.090 | INFO | Perf report for the last 5.14315 seconds: | 20260507 23:49:28.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:28.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260507 23:49:28.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:49:28.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:28.090 | INFO | Uploaded 36GiB to GPU in 2.3886s (15.0716GiB/s) | 20260507 23:49:28.090 | INFO | Downloaded 34GiB to GPU in 2.40543s (14.1347GiB/s) | 20260507 23:49:33.223 | INFO | Analyzing chunk frames = [3570688,3571200) | 20260507 23:49:33.223 | INFO | MemInfo: { | 20260507 23:49:33.223 | INFO | "gpu" : " 17 GB", | 20260507 23:49:33.224 | INFO | "huge" : "", | 20260507 23:49:33.224 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:33.224 | INFO | "malloc" : " 14 GB", | 20260507 23:49:33.224 | INFO | "pinned" : " 29 GB" | 20260507 23:49:33.224 | INFO | } | 20260507 23:49:33.224 | INFO | Perf report for the last 5.13404 seconds: | 20260507 23:49:33.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:33.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260507 23:49:33.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | PRE_HQ : 36.0%, 1.0, 18.6ms, 0 | 20260507 23:49:33.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:33.224 | INFO | Uploaded 36GiB to GPU in 2.43993s (14.7545GiB/s) | 20260507 23:49:33.224 | INFO | Downloaded 34GiB to GPU in 2.42963s (13.9939GiB/s) | 20260507 23:49:38.361 | INFO | Analyzing chunk frames = [3571200,3571712) | 20260507 23:49:38.361 | INFO | MemInfo: { | 20260507 23:49:38.361 | INFO | "gpu" : " 17 GB", | 20260507 23:49:38.361 | INFO | "huge" : "", | 20260507 23:49:38.361 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:38.361 | INFO | "malloc" : " 14 GB", | 20260507 23:49:38.361 | INFO | "pinned" : " 29 GB" | 20260507 23:49:38.361 | INFO | } | 20260507 23:49:38.361 | INFO | Perf report for the last 5.13699 seconds: | 20260507 23:49:38.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:38.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:49:38.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260507 23:49:38.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:38.361 | INFO | Uploaded 36GiB to GPU in 2.53572s (14.1972GiB/s) | 20260507 23:49:38.361 | INFO | Downloaded 34GiB to GPU in 2.45315s (13.8597GiB/s) | 20260507 23:49:38.361 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:49:38.361 | INFO | Total mem usage: 16.7 GiB | 20260507 23:49:38.361 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:49:38.361 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260507 23:49:38.361 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:49:38.361 | INFO | High Water Breakdown: | 20260507 23:49:38.361 | INFO | 59.7% : 8.71 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:49:38.361 | INFO | 40.2% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:49:38.361 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:49:38.361 | INFO | ------------------------------------------ | 20260507 23:49:38.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:49:38.361 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:49:38.361 | INFO | Total mem usage: 26.5 GiB | 20260507 23:49:38.361 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:49:38.361 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:49:38.361 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:49:38.361 | INFO | High Water Breakdown: | 20260507 23:49:38.361 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:49:38.361 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:49:38.361 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:49:38.361 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:49:38.361 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:49:38.361 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:49:38.361 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:49:38.361 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:49:38.361 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:49:38.361 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:49:38.361 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:49:38.361 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:49:38.361 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:49:38.361 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:49:38.361 | INFO | ------------------------------------------ | 20260507 23:49:38.361 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:49:38.361 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:49:38.361 | INFO | Total mem usage: 27.8 GiB | 20260507 23:49:38.361 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:49:38.361 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:49:38.361 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:49:38.361 | INFO | High Water Breakdown: | 20260507 23:49:38.362 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:49:38.362 | INFO | ------------------------------------------ | 20260507 23:49:38.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:49:38.362 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:49:38.362 | INFO | Total mem usage: 15.7 GiB | 20260507 23:49:38.362 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:49:38.362 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:49:38.362 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:49:38.362 | INFO | High Water Breakdown: | 20260507 23:49:38.362 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:49:38.362 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:49:38.362 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:49:38.362 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:49:38.362 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:49:38.362 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:49:38.362 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:49:38.362 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:49:38.362 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:49:38.362 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:49:38.362 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:49:38.362 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:49:38.362 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:49:38.362 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:49:38.362 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:49:38.362 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:49:38.362 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:49:38.362 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:49:38.362 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:49:38.362 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:49:38.362 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:49:38.362 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:49:38.362 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:49:38.362 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:49:38.362 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:49:38.362 | INFO | ------------------------------------------ | 20260507 23:49:38.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:49:38.362 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:49:38.363 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:49:38.363 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:49:38.363 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.60e+01, 1.43e+01 / 1.49e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.02e-01, 1.12e-02 / 6.06e-03 | 20260507 23:49:38.363 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.45e+00, 5.88e-01 / 2.47e-01 | 20260507 23:49:38.363 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.33e-01 | 20260507 23:49:38.363 | INFO | PerfReport: 16.06%, DME -- 4.22e+01 / 4.59e+01, 3.95e+00 / 1.52e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.68e+00 / 1.44e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.25e-01 / 2.27e-01 | 20260507 23:49:38.363 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.34e+00, 2.81e-01 / 1.70e-01 | 20260507 23:49:38.363 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.46e-01 / 3.83e-01, 5.47e-02 / 2.69e-02 | 20260507 23:49:38.363 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:49:38.363 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:49:38.363 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:49:38.363 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 30.83%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 2.18%, LockUpload -- 2.33e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 15.71%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260507 23:49:38.363 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260507 23:49:38.458 | INFO | Starting a round of Baz IO | 20260507 23:49:41.918 | INFO | Populating RTMetrics report | 20260507 23:49:41.919 | INFO | RTMetrics report populated | 20260507 23:49:41.919 | INFO | Opening tmp RTMetrics json file | 20260507 23:49:41.943 | INFO | Serializing RTMetrics to json | 20260507 23:49:41.945 | INFO | Serialization done, swapping tmp file now... | 20260507 23:49:41.946 | INFO | RTMetrics done | 20260507 23:49:42.020 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 3.561 seconds | 20260507 23:49:42.020 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.44% / 0.00% / 57.55% | 20260507 23:49:42.020 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11674% / 7.94729e-06% / 5.96046e-05% | 20260507 23:49:42.020 | INFO | Baz IO overall: | 20260507 23:49:42.020 | INFO | BytesWritten : 1625122158519 | 20260507 23:49:42.020 | INFO | NumEvents : 786828227031 | 20260507 23:49:43.502 | INFO | Analyzing chunk frames = [3571712,3572224) | 20260507 23:49:43.502 | INFO | MemInfo: { | 20260507 23:49:43.502 | INFO | "gpu" : " 17 GB", | 20260507 23:49:43.502 | INFO | "huge" : "", | 20260507 23:49:43.502 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:43.502 | INFO | "malloc" : " 14 GB", | 20260507 23:49:43.502 | INFO | "pinned" : " 29 GB" | 20260507 23:49:43.502 | INFO | } | 20260507 23:49:43.502 | INFO | Perf report for the last 5.14123 seconds: | 20260507 23:49:43.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:43.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:43.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:43.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:43.502 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260507 23:49:43.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:43.502 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 23:49:43.502 | INFO | PRE_HQ : 65.3%, 1.7, 56.5ms, 0 | 20260507 23:49:43.502 | INFO | BAZWRITER : 68.9%, 3.7, 130.3ms, 0 | 20260507 23:49:43.502 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:49:43.503 | INFO | Uploaded 36GiB to GPU in 2.54352s (14.1536GiB/s) | 20260507 23:49:43.503 | INFO | Downloaded 38GiB to GPU in 2.71058s (14.0191GiB/s) | 20260507 23:49:48.642 | INFO | Analyzing chunk frames = [3572224,3572736) | 20260507 23:49:48.642 | INFO | MemInfo: { | 20260507 23:49:48.642 | INFO | "gpu" : " 17 GB", | 20260507 23:49:48.642 | INFO | "huge" : "", | 20260507 23:49:48.642 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:48.642 | INFO | "malloc" : " 14 GB", | 20260507 23:49:48.642 | INFO | "pinned" : " 29 GB" | 20260507 23:49:48.642 | INFO | } | 20260507 23:49:48.642 | INFO | Perf report for the last 5.13966 seconds: | 20260507 23:49:48.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:48.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260507 23:49:48.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:49:48.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:48.642 | INFO | Uploaded 36GiB to GPU in 2.50317s (14.3818GiB/s) | 20260507 23:49:48.642 | INFO | Downloaded 34GiB to GPU in 2.48056s (13.7066GiB/s) | 20260507 23:49:53.786 | INFO | Analyzing chunk frames = [3572736,3573248) | 20260507 23:49:53.786 | INFO | MemInfo: { | 20260507 23:49:53.786 | INFO | "gpu" : " 17 GB", | 20260507 23:49:53.786 | INFO | "huge" : "", | 20260507 23:49:53.786 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:53.786 | INFO | "malloc" : " 14 GB", | 20260507 23:49:53.786 | INFO | "pinned" : " 29 GB" | 20260507 23:49:53.786 | INFO | } | 20260507 23:49:53.786 | INFO | Perf report for the last 5.14409 seconds: | 20260507 23:49:53.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:53.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260507 23:49:53.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:49:53.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:53.786 | INFO | Uploaded 36GiB to GPU in 2.47563s (14.5418GiB/s) | 20260507 23:49:53.786 | INFO | Downloaded 34GiB to GPU in 2.4842s (13.6865GiB/s) | 20260507 23:49:58.921 | INFO | Analyzing chunk frames = [3573248,3573760) | 20260507 23:49:58.921 | INFO | MemInfo: { | 20260507 23:49:58.921 | INFO | "gpu" : " 17 GB", | 20260507 23:49:58.921 | INFO | "huge" : "", | 20260507 23:49:58.921 | INFO | "huge_shared" : " 29 GB", | 20260507 23:49:58.922 | INFO | "malloc" : " 14 GB", | 20260507 23:49:58.922 | INFO | "pinned" : " 29 GB" | 20260507 23:49:58.922 | INFO | } | 20260507 23:49:58.922 | INFO | Perf report for the last 5.13574 seconds: | 20260507 23:49:58.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:49:58.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 23:49:58.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:49:58.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:49:58.922 | INFO | Uploaded 36GiB to GPU in 2.51181s (14.3323GiB/s) | 20260507 23:49:58.922 | INFO | Downloaded 34GiB to GPU in 2.49102s (13.649GiB/s) | 20260507 23:50:04.061 | INFO | Analyzing chunk frames = [3573760,3574272) | 20260507 23:50:04.061 | INFO | MemInfo: { | 20260507 23:50:04.061 | INFO | "gpu" : " 17 GB", | 20260507 23:50:04.061 | INFO | "huge" : "", | 20260507 23:50:04.061 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:04.061 | INFO | "malloc" : " 14 GB", | 20260507 23:50:04.061 | INFO | "pinned" : " 29 GB" | 20260507 23:50:04.061 | INFO | } | 20260507 23:50:04.061 | INFO | Perf report for the last 5.13924 seconds: | 20260507 23:50:04.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:04.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260507 23:50:04.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:50:04.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:04.061 | INFO | Uploaded 36GiB to GPU in 2.385s (15.0944GiB/s) | 20260507 23:50:04.061 | INFO | Downloaded 34GiB to GPU in 2.49352s (13.6354GiB/s) | 20260507 23:50:09.203 | INFO | Analyzing chunk frames = [3574272,3574784) | 20260507 23:50:09.203 | INFO | MemInfo: { | 20260507 23:50:09.203 | INFO | "gpu" : " 17 GB", | 20260507 23:50:09.203 | INFO | "huge" : "", | 20260507 23:50:09.203 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:09.204 | INFO | "malloc" : " 14 GB", | 20260507 23:50:09.204 | INFO | "pinned" : " 29 GB" | 20260507 23:50:09.204 | INFO | } | 20260507 23:50:09.204 | INFO | Perf report for the last 5.1428 seconds: | 20260507 23:50:09.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:09.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260507 23:50:09.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:50:09.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:09.204 | INFO | Uploaded 36GiB to GPU in 2.42883s (14.8219GiB/s) | 20260507 23:50:09.204 | INFO | Downloaded 34GiB to GPU in 2.38979s (14.2272GiB/s) | 20260507 23:50:14.340 | INFO | Analyzing chunk frames = [3574784,3575296) | 20260507 23:50:14.340 | INFO | MemInfo: { | 20260507 23:50:14.340 | INFO | "gpu" : " 17 GB", | 20260507 23:50:14.340 | INFO | "huge" : "", | 20260507 23:50:14.340 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:14.340 | INFO | "malloc" : " 14 GB", | 20260507 23:50:14.340 | INFO | "pinned" : " 29 GB" | 20260507 23:50:14.340 | INFO | } | 20260507 23:50:14.340 | INFO | Perf report for the last 5.13611 seconds: | 20260507 23:50:14.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:14.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260507 23:50:14.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260507 23:50:14.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:14.340 | INFO | Uploaded 36GiB to GPU in 2.44144s (14.7454GiB/s) | 20260507 23:50:14.340 | INFO | Downloaded 34GiB to GPU in 2.41553s (14.0756GiB/s) | 20260507 23:50:19.478 | INFO | Analyzing chunk frames = [3575296,3575808) | 20260507 23:50:19.478 | INFO | MemInfo: { | 20260507 23:50:19.478 | INFO | "gpu" : " 17 GB", | 20260507 23:50:19.478 | INFO | "huge" : "", | 20260507 23:50:19.478 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:19.479 | INFO | "malloc" : " 14 GB", | 20260507 23:50:19.479 | INFO | "pinned" : " 29 GB" | 20260507 23:50:19.479 | INFO | } | 20260507 23:50:19.479 | INFO | Perf report for the last 5.13892 seconds: | 20260507 23:50:19.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:19.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260507 23:50:19.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 23:50:19.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:19.479 | INFO | Uploaded 36GiB to GPU in 2.55734s (14.0771GiB/s) | 20260507 23:50:19.479 | INFO | Downloaded 34GiB to GPU in 2.45204s (13.866GiB/s) | 20260507 23:50:19.577 | INFO | Starting a round of Baz IO | 20260507 23:50:23.042 | INFO | Populating RTMetrics report | 20260507 23:50:23.042 | INFO | RTMetrics report populated | 20260507 23:50:23.042 | INFO | Opening tmp RTMetrics json file | 20260507 23:50:23.109 | INFO | Serializing RTMetrics to json | 20260507 23:50:23.115 | INFO | Serialization done, swapping tmp file now... | 20260507 23:50:23.119 | INFO | RTMetrics done | 20260507 23:50:23.265 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.687 seconds | 20260507 23:50:23.265 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.44% / 0.00% / 57.56% | 20260507 23:50:23.265 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12087% / 1.19209e-05% / 5.96046e-05% | 20260507 23:50:23.265 | INFO | Baz IO overall: | 20260507 23:50:23.265 | INFO | BytesWritten : 1626953994653 | 20260507 23:50:23.265 | INFO | NumEvents : 787710841711 | 20260507 23:50:24.616 | INFO | Analyzing chunk frames = [3575808,3576320) | 20260507 23:50:24.616 | INFO | MemInfo: { | 20260507 23:50:24.616 | INFO | "gpu" : " 17 GB", | 20260507 23:50:24.616 | INFO | "huge" : "", | 20260507 23:50:24.616 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:24.616 | INFO | "malloc" : " 14 GB", | 20260507 23:50:24.616 | INFO | "pinned" : " 29 GB" | 20260507 23:50:24.616 | INFO | } | 20260507 23:50:24.616 | INFO | Perf report for the last 5.13746 seconds: | 20260507 23:50:24.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:24.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:24.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:24.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:24.616 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 23:50:24.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:24.616 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 23:50:24.616 | INFO | PRE_HQ : 65.6%, 1.7, 56.8ms, 0 | 20260507 23:50:24.616 | INFO | BAZWRITER : 71.4%, 4.0, 147.9ms, 0 | 20260507 23:50:24.616 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260507 23:50:24.617 | INFO | Uploaded 36GiB to GPU in 2.57362s (13.9881GiB/s) | 20260507 23:50:24.617 | INFO | Downloaded 38GiB to GPU in 2.67856s (14.1867GiB/s) | 20260507 23:50:29.757 | INFO | Analyzing chunk frames = [3576320,3576832) | 20260507 23:50:29.757 | INFO | MemInfo: { | 20260507 23:50:29.757 | INFO | "gpu" : " 17 GB", | 20260507 23:50:29.757 | INFO | "huge" : "", | 20260507 23:50:29.757 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:29.757 | INFO | "malloc" : " 14 GB", | 20260507 23:50:29.757 | INFO | "pinned" : " 29 GB" | 20260507 23:50:29.757 | INFO | } | 20260507 23:50:29.757 | INFO | Perf report for the last 5.14114 seconds: | 20260507 23:50:29.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:29.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.757 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 23:50:29.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.757 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:50:29.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:29.758 | INFO | Uploaded 36GiB to GPU in 2.59198s (13.889GiB/s) | 20260507 23:50:29.758 | INFO | Downloaded 34GiB to GPU in 2.47538s (13.7353GiB/s) | 20260507 23:50:34.900 | INFO | Analyzing chunk frames = [3576832,3577344) | 20260507 23:50:34.900 | INFO | MemInfo: { | 20260507 23:50:34.900 | INFO | "gpu" : " 17 GB", | 20260507 23:50:34.900 | INFO | "huge" : "", | 20260507 23:50:34.901 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:34.901 | INFO | "malloc" : " 14 GB", | 20260507 23:50:34.901 | INFO | "pinned" : " 29 GB" | 20260507 23:50:34.901 | INFO | } | 20260507 23:50:34.901 | INFO | Perf report for the last 5.14318 seconds: | 20260507 23:50:34.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:34.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:50:34.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:50:34.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:34.901 | INFO | Uploaded 36GiB to GPU in 2.56056s (14.0594GiB/s) | 20260507 23:50:34.901 | INFO | Downloaded 34GiB to GPU in 2.47255s (13.751GiB/s) | 20260507 23:50:40.039 | INFO | Analyzing chunk frames = [3577344,3577856) | 20260507 23:50:40.039 | INFO | MemInfo: { | 20260507 23:50:40.039 | INFO | "gpu" : " 17 GB", | 20260507 23:50:40.039 | INFO | "huge" : "", | 20260507 23:50:40.039 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:40.039 | INFO | "malloc" : " 14 GB", | 20260507 23:50:40.039 | INFO | "pinned" : " 29 GB" | 20260507 23:50:40.039 | INFO | } | 20260507 23:50:40.039 | INFO | Perf report for the last 5.13885 seconds: | 20260507 23:50:40.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:40.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 23:50:40.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:50:40.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:40.040 | INFO | Uploaded 36GiB to GPU in 2.53975s (14.1746GiB/s) | 20260507 23:50:40.040 | INFO | Downloaded 34GiB to GPU in 2.46149s (13.8128GiB/s) | 20260507 23:50:45.174 | INFO | Analyzing chunk frames = [3577856,3578368) | 20260507 23:50:45.174 | INFO | MemInfo: { | 20260507 23:50:45.174 | INFO | "gpu" : " 17 GB", | 20260507 23:50:45.174 | INFO | "huge" : "", | 20260507 23:50:45.174 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:45.174 | INFO | "malloc" : " 14 GB", | 20260507 23:50:45.174 | INFO | "pinned" : " 29 GB" | 20260507 23:50:45.174 | INFO | } | 20260507 23:50:45.174 | INFO | Perf report for the last 5.1347 seconds: | 20260507 23:50:45.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:45.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.174 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260507 23:50:45.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.174 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:50:45.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:45.175 | INFO | Uploaded 36GiB to GPU in 2.38897s (15.0693GiB/s) | 20260507 23:50:45.175 | INFO | Downloaded 34GiB to GPU in 2.537s (13.4017GiB/s) | 20260507 23:50:50.316 | INFO | Analyzing chunk frames = [3578368,3578880) | 20260507 23:50:50.316 | INFO | MemInfo: { | 20260507 23:50:50.316 | INFO | "gpu" : " 17 GB", | 20260507 23:50:50.316 | INFO | "huge" : "", | 20260507 23:50:50.316 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:50.316 | INFO | "malloc" : " 14 GB", | 20260507 23:50:50.316 | INFO | "pinned" : " 29 GB" | 20260507 23:50:50.316 | INFO | } | 20260507 23:50:50.316 | INFO | Perf report for the last 5.14208 seconds: | 20260507 23:50:50.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:50.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.316 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260507 23:50:50.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.317 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 23:50:50.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:50.317 | INFO | Uploaded 36GiB to GPU in 2.43504s (14.7841GiB/s) | 20260507 23:50:50.317 | INFO | Downloaded 34GiB to GPU in 2.3925s (14.2111GiB/s) | 20260507 23:50:55.454 | INFO | Analyzing chunk frames = [3578880,3579392) | 20260507 23:50:55.454 | INFO | MemInfo: { | 20260507 23:50:55.454 | INFO | "gpu" : " 17 GB", | 20260507 23:50:55.454 | INFO | "huge" : "", | 20260507 23:50:55.454 | INFO | "huge_shared" : " 29 GB", | 20260507 23:50:55.454 | INFO | "malloc" : " 14 GB", | 20260507 23:50:55.454 | INFO | "pinned" : " 29 GB" | 20260507 23:50:55.454 | INFO | } | 20260507 23:50:55.454 | INFO | Perf report for the last 5.13815 seconds: | 20260507 23:50:55.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:50:55.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.454 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260507 23:50:55.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.454 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 23:50:55.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:50:55.455 | INFO | Uploaded 36GiB to GPU in 2.55803s (14.0734GiB/s) | 20260507 23:50:55.455 | INFO | Downloaded 34GiB to GPU in 2.44506s (13.9056GiB/s) | 20260507 23:51:00.594 | INFO | Analyzing chunk frames = [3579392,3579904) | 20260507 23:51:00.594 | INFO | MemInfo: { | 20260507 23:51:00.594 | INFO | "gpu" : " 17 GB", | 20260507 23:51:00.594 | INFO | "huge" : "", | 20260507 23:51:00.594 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:00.594 | INFO | "malloc" : " 14 GB", | 20260507 23:51:00.594 | INFO | "pinned" : " 29 GB" | 20260507 23:51:00.594 | INFO | } | 20260507 23:51:00.594 | INFO | Perf report for the last 5.13971 seconds: | 20260507 23:51:00.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:00.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:51:00.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:51:00.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:00.594 | INFO | Uploaded 36GiB to GPU in 2.53968s (14.175GiB/s) | 20260507 23:51:00.594 | INFO | Downloaded 34GiB to GPU in 2.44095s (13.929GiB/s) | 20260507 23:51:00.594 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:51:00.594 | INFO | Total mem usage: 16.7 GiB | 20260507 23:51:00.594 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:51:00.594 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260507 23:51:00.594 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:51:00.595 | INFO | High Water Breakdown: | 20260507 23:51:00.595 | INFO | 60% : 8.93 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:51:00.595 | INFO | 39.9% : 5.94 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:51:00.595 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:51:00.595 | INFO | ------------------------------------------ | 20260507 23:51:00.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:51:00.595 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:51:00.595 | INFO | Total mem usage: 26.5 GiB | 20260507 23:51:00.595 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:51:00.595 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:51:00.595 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:51:00.595 | INFO | High Water Breakdown: | 20260507 23:51:00.595 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:51:00.595 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:51:00.595 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:51:00.595 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:51:00.595 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:51:00.595 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:51:00.595 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:51:00.595 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:51:00.595 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:51:00.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:51:00.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:51:00.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:51:00.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:51:00.595 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:51:00.595 | INFO | ------------------------------------------ | 20260507 23:51:00.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:51:00.595 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:51:00.595 | INFO | Total mem usage: 27.8 GiB | 20260507 23:51:00.595 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:51:00.595 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:51:00.595 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:51:00.595 | INFO | High Water Breakdown: | 20260507 23:51:00.595 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:51:00.595 | INFO | ------------------------------------------ | 20260507 23:51:00.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:51:00.595 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:51:00.595 | INFO | Total mem usage: 15.7 GiB | 20260507 23:51:00.595 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:51:00.595 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:51:00.595 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:51:00.595 | INFO | High Water Breakdown: | 20260507 23:51:00.595 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:51:00.595 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:51:00.595 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:51:00.595 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:51:00.595 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:51:00.595 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:51:00.595 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:51:00.595 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:51:00.595 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:51:00.595 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:51:00.595 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:51:00.595 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:51:00.595 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:51:00.595 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:51:00.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:51:00.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:51:00.595 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:51:00.595 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:51:00.595 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:51:00.595 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:51:00.595 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:51:00.595 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:51:00.595 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:51:00.595 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:51:00.595 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:51:00.595 | INFO | ------------------------------------------ | 20260507 23:51:00.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:51:00.595 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:51:00.595 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:51:00.595 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:51:00.595 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.43e+01 / 1.64e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.26e-02 / 5.49e-03 | 20260507 23:51:00.595 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.50e+00, 5.85e-01 / 1.84e-01 | 20260507 23:51:00.595 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.10e-01 | 20260507 23:51:00.595 | INFO | PerfReport: 16.15%, DME -- 4.24e+01 / 4.63e+01, 4.33e+00 / 2.09e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.60e+00 / 1.51e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.28e-01 / 2.14e-01 | 20260507 23:51:00.595 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.35e+00, 2.79e-01 / 1.53e-01 | 20260507 23:51:00.595 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.45e-01 / 3.82e-01, 5.49e-02 / 2.65e-02 | 20260507 23:51:00.595 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:51:00.595 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:51:00.595 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:51:00.595 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 24.38%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 30.69%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 2.33%, LockUpload -- 2.49e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 2.63%, LockDownload -- 2.81e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 15.70%, LockCompute -- 1.68e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 23:51:00.595 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 23:51:00.693 | INFO | Starting a round of Baz IO | 20260507 23:51:04.157 | INFO | Populating RTMetrics report | 20260507 23:51:04.158 | INFO | RTMetrics report populated | 20260507 23:51:04.158 | INFO | Opening tmp RTMetrics json file | 20260507 23:51:04.159 | INFO | Serializing RTMetrics to json | 20260507 23:51:04.184 | INFO | Serialization done, swapping tmp file now... | 20260507 23:51:04.187 | INFO | RTMetrics done | 20260507 23:51:04.264 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.57 seconds | 20260507 23:51:04.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.43% / 0.01% / 57.56% | 20260507 23:51:04.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11516% / 3.97364e-06% / 8.34465e-05% | 20260507 23:51:04.264 | INFO | Baz IO overall: | 20260507 23:51:04.264 | INFO | BytesWritten : 1628784433222 | 20260507 23:51:04.264 | INFO | NumEvents : 788591947080 | 20260507 23:51:05.733 | INFO | Analyzing chunk frames = [3579904,3580416) | 20260507 23:51:05.733 | INFO | MemInfo: { | 20260507 23:51:05.733 | INFO | "gpu" : " 17 GB", | 20260507 23:51:05.733 | INFO | "huge" : "", | 20260507 23:51:05.733 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:05.733 | INFO | "malloc" : " 14 GB", | 20260507 23:51:05.733 | INFO | "pinned" : " 29 GB" | 20260507 23:51:05.733 | INFO | } | 20260507 23:51:05.733 | INFO | Perf report for the last 5.13878 seconds: | 20260507 23:51:05.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:05.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:05.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:05.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:05.733 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 23:51:05.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:05.733 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 23:51:05.733 | INFO | PRE_HQ : 65.7%, 1.7, 56.9ms, 0 | 20260507 23:51:05.733 | INFO | BAZWRITER : 69.1%, 4.1, 145.0ms, 0 | 20260507 23:51:05.733 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260507 23:51:05.733 | INFO | Uploaded 36GiB to GPU in 2.60625s (13.813GiB/s) | 20260507 23:51:05.733 | INFO | Downloaded 38GiB to GPU in 2.67643s (14.198GiB/s) | 20260507 23:51:10.873 | INFO | Analyzing chunk frames = [3580416,3580928) | 20260507 23:51:10.873 | INFO | MemInfo: { | 20260507 23:51:10.873 | INFO | "gpu" : " 17 GB", | 20260507 23:51:10.873 | INFO | "huge" : "", | 20260507 23:51:10.873 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:10.873 | INFO | "malloc" : " 14 GB", | 20260507 23:51:10.873 | INFO | "pinned" : " 29 GB" | 20260507 23:51:10.873 | INFO | } | 20260507 23:51:10.873 | INFO | Perf report for the last 5.14052 seconds: | 20260507 23:51:10.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:10.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:51:10.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:51:10.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:10.874 | INFO | Uploaded 36GiB to GPU in 2.59237s (13.8869GiB/s) | 20260507 23:51:10.874 | INFO | Downloaded 34GiB to GPU in 2.47439s (13.7407GiB/s) | 20260507 23:51:16.013 | INFO | Analyzing chunk frames = [3580928,3581440) | 20260507 23:51:16.013 | INFO | MemInfo: { | 20260507 23:51:16.013 | INFO | "gpu" : " 17 GB", | 20260507 23:51:16.013 | INFO | "huge" : "", | 20260507 23:51:16.013 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:16.013 | INFO | "malloc" : " 14 GB", | 20260507 23:51:16.013 | INFO | "pinned" : " 29 GB" | 20260507 23:51:16.013 | INFO | } | 20260507 23:51:16.013 | INFO | Perf report for the last 5.13991 seconds: | 20260507 23:51:16.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:16.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.013 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:51:16.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.013 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:51:16.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:16.014 | INFO | Uploaded 36GiB to GPU in 2.54102s (14.1675GiB/s) | 20260507 23:51:16.014 | INFO | Downloaded 34GiB to GPU in 2.47446s (13.7404GiB/s) | 20260507 23:51:21.154 | INFO | Analyzing chunk frames = [3581440,3581952) | 20260507 23:51:21.154 | INFO | MemInfo: { | 20260507 23:51:21.154 | INFO | "gpu" : " 17 GB", | 20260507 23:51:21.154 | INFO | "huge" : "", | 20260507 23:51:21.154 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:21.154 | INFO | "malloc" : " 14 GB", | 20260507 23:51:21.154 | INFO | "pinned" : " 29 GB" | 20260507 23:51:21.154 | INFO | } | 20260507 23:51:21.154 | INFO | Perf report for the last 5.14109 seconds: | 20260507 23:51:21.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:21.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.154 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:51:21.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.155 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:51:21.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:21.155 | INFO | Uploaded 36GiB to GPU in 2.53713s (14.1893GiB/s) | 20260507 23:51:21.155 | INFO | Downloaded 34GiB to GPU in 2.46472s (13.7947GiB/s) | 20260507 23:51:26.293 | INFO | Analyzing chunk frames = [3581952,3582464) | 20260507 23:51:26.293 | INFO | MemInfo: { | 20260507 23:51:26.293 | INFO | "gpu" : " 17 GB", | 20260507 23:51:26.293 | INFO | "huge" : "", | 20260507 23:51:26.293 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:26.293 | INFO | "malloc" : " 14 GB", | 20260507 23:51:26.293 | INFO | "pinned" : " 29 GB" | 20260507 23:51:26.293 | INFO | } | 20260507 23:51:26.293 | INFO | Perf report for the last 5.13817 seconds: | 20260507 23:51:26.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:26.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 23:51:26.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 23:51:26.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:26.293 | INFO | Uploaded 36GiB to GPU in 2.3878s (15.0767GiB/s) | 20260507 23:51:26.293 | INFO | Downloaded 34GiB to GPU in 2.46918s (13.7697GiB/s) | 20260507 23:51:31.428 | INFO | Analyzing chunk frames = [3582464,3582976) | 20260507 23:51:31.428 | INFO | MemInfo: { | 20260507 23:51:31.428 | INFO | "gpu" : " 17 GB", | 20260507 23:51:31.428 | INFO | "huge" : "", | 20260507 23:51:31.429 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:31.429 | INFO | "malloc" : " 14 GB", | 20260507 23:51:31.429 | INFO | "pinned" : " 29 GB" | 20260507 23:51:31.429 | INFO | } | 20260507 23:51:31.429 | INFO | Perf report for the last 5.13597 seconds: | 20260507 23:51:31.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:31.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:51:31.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:51:31.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:31.429 | INFO | Uploaded 36GiB to GPU in 2.41993s (14.8765GiB/s) | 20260507 23:51:31.429 | INFO | Downloaded 34GiB to GPU in 2.39006s (14.2256GiB/s) | 20260507 23:51:36.569 | INFO | Analyzing chunk frames = [3582976,3583488) | 20260507 23:51:36.569 | INFO | MemInfo: { | 20260507 23:51:36.569 | INFO | "gpu" : " 17 GB", | 20260507 23:51:36.569 | INFO | "huge" : "", | 20260507 23:51:36.569 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:36.569 | INFO | "malloc" : " 14 GB", | 20260507 23:51:36.569 | INFO | "pinned" : " 29 GB" | 20260507 23:51:36.569 | INFO | } | 20260507 23:51:36.569 | INFO | Perf report for the last 5.14022 seconds: | 20260507 23:51:36.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:36.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:51:36.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:51:36.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:36.569 | INFO | Uploaded 36GiB to GPU in 2.51239s (14.329GiB/s) | 20260507 23:51:36.569 | INFO | Downloaded 34GiB to GPU in 2.42981s (13.9928GiB/s) | 20260507 23:51:41.705 | INFO | Analyzing chunk frames = [3583488,3584000) | 20260507 23:51:41.705 | INFO | MemInfo: { | 20260507 23:51:41.705 | INFO | "gpu" : " 17 GB", | 20260507 23:51:41.705 | INFO | "huge" : "", | 20260507 23:51:41.705 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:41.705 | INFO | "malloc" : " 14 GB", | 20260507 23:51:41.705 | INFO | "pinned" : " 29 GB" | 20260507 23:51:41.705 | INFO | } | 20260507 23:51:41.705 | INFO | Perf report for the last 5.13642 seconds: | 20260507 23:51:41.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:41.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 23:51:41.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:51:41.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:41.706 | INFO | Uploaded 36GiB to GPU in 2.5387s (14.1805GiB/s) | 20260507 23:51:41.706 | INFO | Downloaded 34GiB to GPU in 2.44471s (13.9076GiB/s) | 20260507 23:51:41.803 | INFO | Starting a round of Baz IO | 20260507 23:51:45.268 | INFO | Populating RTMetrics report | 20260507 23:51:45.269 | INFO | RTMetrics report populated | 20260507 23:51:45.269 | INFO | Opening tmp RTMetrics json file | 20260507 23:51:45.274 | INFO | Serializing RTMetrics to json | 20260507 23:51:45.341 | INFO | Serialization done, swapping tmp file now... | 20260507 23:51:45.361 | INFO | RTMetrics done | 20260507 23:51:45.409 | INFO | Finished a round of Baz IO: Wrote 1.707 GiB in 3.605 seconds | 20260507 23:51:45.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.43% / 0.01% / 57.57% | 20260507 23:51:45.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11986% / 1.19209e-05% / 5.96046e-05% | 20260507 23:51:45.409 | INFO | Baz IO overall: | 20260507 23:51:45.409 | INFO | BytesWritten : 1630617321398 | 20260507 23:51:45.409 | INFO | NumEvents : 789475491661 | 20260507 23:51:46.849 | INFO | Analyzing chunk frames = [3584000,3584512) | 20260507 23:51:46.849 | INFO | MemInfo: { | 20260507 23:51:46.849 | INFO | "gpu" : " 17 GB", | 20260507 23:51:46.849 | INFO | "huge" : "", | 20260507 23:51:46.849 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:46.849 | INFO | "malloc" : " 14 GB", | 20260507 23:51:46.849 | INFO | "pinned" : " 29 GB" | 20260507 23:51:46.849 | INFO | } | 20260507 23:51:46.849 | INFO | Perf report for the last 5.14391 seconds: | 20260507 23:51:46.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:46.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:46.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:46.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:46.849 | INFO | ANALYSIS : 68.6%, 3.0, 106.9ms, 0 | 20260507 23:51:46.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:46.849 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 23:51:46.849 | INFO | PRE_HQ : 65.6%, 1.7, 57.0ms, 0 | 20260507 23:51:46.849 | INFO | BAZWRITER : 69.3%, 3.8, 135.4ms, 0 | 20260507 23:51:46.849 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260507 23:51:46.850 | INFO | Uploaded 36GiB to GPU in 2.5624s (14.0493GiB/s) | 20260507 23:51:46.850 | INFO | Downloaded 38GiB to GPU in 2.69886s (14.08GiB/s) | 20260507 23:51:51.991 | INFO | Analyzing chunk frames = [3584512,3585024) | 20260507 23:51:51.991 | INFO | MemInfo: { | 20260507 23:51:51.991 | INFO | "gpu" : " 17 GB", | 20260507 23:51:51.991 | INFO | "huge" : "", | 20260507 23:51:51.991 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:51.991 | INFO | "malloc" : " 14 GB", | 20260507 23:51:51.991 | INFO | "pinned" : " 29 GB" | 20260507 23:51:51.991 | INFO | } | 20260507 23:51:51.991 | INFO | Perf report for the last 5.14175 seconds: | 20260507 23:51:51.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:51.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260507 23:51:51.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:51:51.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:51.991 | INFO | Uploaded 36GiB to GPU in 2.5469s (14.1348GiB/s) | 20260507 23:51:51.991 | INFO | Downloaded 34GiB to GPU in 2.41628s (14.0712GiB/s) | 20260507 23:51:57.125 | INFO | Analyzing chunk frames = [3585024,3585536) | 20260507 23:51:57.125 | INFO | MemInfo: { | 20260507 23:51:57.125 | INFO | "gpu" : " 17 GB", | 20260507 23:51:57.125 | INFO | "huge" : "", | 20260507 23:51:57.125 | INFO | "huge_shared" : " 29 GB", | 20260507 23:51:57.125 | INFO | "malloc" : " 14 GB", | 20260507 23:51:57.125 | INFO | "pinned" : " 29 GB" | 20260507 23:51:57.125 | INFO | } | 20260507 23:51:57.125 | INFO | Perf report for the last 5.13433 seconds: | 20260507 23:51:57.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:51:57.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.125 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 23:51:57.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.125 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:51:57.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:51:57.126 | INFO | Uploaded 36GiB to GPU in 2.54994s (14.118GiB/s) | 20260507 23:51:57.126 | INFO | Downloaded 34GiB to GPU in 2.47498s (13.7375GiB/s) | 20260507 23:52:02.265 | INFO | Analyzing chunk frames = [3585536,3586048) | 20260507 23:52:02.265 | INFO | MemInfo: { | 20260507 23:52:02.265 | INFO | "gpu" : " 17 GB", | 20260507 23:52:02.265 | INFO | "huge" : "", | 20260507 23:52:02.265 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:02.265 | INFO | "malloc" : " 14 GB", | 20260507 23:52:02.265 | INFO | "pinned" : " 29 GB" | 20260507 23:52:02.265 | INFO | } | 20260507 23:52:02.265 | INFO | Perf report for the last 5.14006 seconds: | 20260507 23:52:02.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:02.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.265 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260507 23:52:02.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.266 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260507 23:52:02.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:02.266 | INFO | Uploaded 36GiB to GPU in 2.5458s (14.141GiB/s) | 20260507 23:52:02.266 | INFO | Downloaded 34GiB to GPU in 2.4816s (13.7008GiB/s) | 20260507 23:52:07.405 | INFO | Analyzing chunk frames = [3586048,3586560) | 20260507 23:52:07.405 | INFO | MemInfo: { | 20260507 23:52:07.405 | INFO | "gpu" : " 17 GB", | 20260507 23:52:07.405 | INFO | "huge" : "", | 20260507 23:52:07.405 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:07.405 | INFO | "malloc" : " 14 GB", | 20260507 23:52:07.405 | INFO | "pinned" : " 29 GB" | 20260507 23:52:07.405 | INFO | } | 20260507 23:52:07.405 | INFO | Perf report for the last 5.13979 seconds: | 20260507 23:52:07.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:07.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.405 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260507 23:52:07.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.405 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260507 23:52:07.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:07.406 | INFO | Uploaded 36GiB to GPU in 2.4106s (14.9341GiB/s) | 20260507 23:52:07.406 | INFO | Downloaded 34GiB to GPU in 2.47505s (13.7371GiB/s) | 20260507 23:52:12.544 | INFO | Analyzing chunk frames = [3586560,3587072) | 20260507 23:52:12.544 | INFO | MemInfo: { | 20260507 23:52:12.544 | INFO | "gpu" : " 17 GB", | 20260507 23:52:12.544 | INFO | "huge" : "", | 20260507 23:52:12.544 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:12.544 | INFO | "malloc" : " 14 GB", | 20260507 23:52:12.544 | INFO | "pinned" : " 29 GB" | 20260507 23:52:12.544 | INFO | } | 20260507 23:52:12.545 | INFO | Perf report for the last 5.13934 seconds: | 20260507 23:52:12.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:12.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260507 23:52:12.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:52:12.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:12.545 | INFO | Uploaded 36GiB to GPU in 2.52187s (14.2751GiB/s) | 20260507 23:52:12.545 | INFO | Downloaded 34GiB to GPU in 2.42996s (13.992GiB/s) | 20260507 23:52:17.683 | INFO | Analyzing chunk frames = [3587072,3587584) | 20260507 23:52:17.683 | INFO | MemInfo: { | 20260507 23:52:17.683 | INFO | "gpu" : " 17 GB", | 20260507 23:52:17.683 | INFO | "huge" : "", | 20260507 23:52:17.683 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:17.684 | INFO | "malloc" : " 14 GB", | 20260507 23:52:17.684 | INFO | "pinned" : " 29 GB" | 20260507 23:52:17.684 | INFO | } | 20260507 23:52:17.684 | INFO | Perf report for the last 5.13916 seconds: | 20260507 23:52:17.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:17.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:52:17.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:52:17.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:17.684 | INFO | Uploaded 36GiB to GPU in 2.53126s (14.2222GiB/s) | 20260507 23:52:17.684 | INFO | Downloaded 34GiB to GPU in 2.44931s (13.8815GiB/s) | 20260507 23:52:22.824 | INFO | Analyzing chunk frames = [3587584,3588096) | 20260507 23:52:22.824 | INFO | MemInfo: { | 20260507 23:52:22.824 | INFO | "gpu" : " 17 GB", | 20260507 23:52:22.824 | INFO | "huge" : "", | 20260507 23:52:22.825 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:22.825 | INFO | "malloc" : " 14 GB", | 20260507 23:52:22.825 | INFO | "pinned" : " 29 GB" | 20260507 23:52:22.825 | INFO | } | 20260507 23:52:22.825 | INFO | Perf report for the last 5.14098 seconds: | 20260507 23:52:22.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:22.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:52:22.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 23:52:22.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:22.825 | INFO | Uploaded 36GiB to GPU in 2.53807s (14.184GiB/s) | 20260507 23:52:22.825 | INFO | Downloaded 34GiB to GPU in 2.4468s (13.8957GiB/s) | 20260507 23:52:22.825 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:52:22.825 | INFO | Total mem usage: 16.7 GiB | 20260507 23:52:22.825 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:52:22.825 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260507 23:52:22.825 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:52:22.825 | INFO | High Water Breakdown: | 20260507 23:52:22.825 | INFO | 59.9% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:52:22.825 | INFO | 39.9% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:52:22.825 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:52:22.825 | INFO | ------------------------------------------ | 20260507 23:52:22.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:52:22.825 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:52:22.825 | INFO | Total mem usage: 26.5 GiB | 20260507 23:52:22.825 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:52:22.825 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:52:22.825 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:52:22.825 | INFO | High Water Breakdown: | 20260507 23:52:22.825 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:52:22.825 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:52:22.825 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:52:22.825 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:52:22.825 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:52:22.825 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:52:22.825 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:52:22.825 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:52:22.825 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:52:22.825 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:52:22.826 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:52:22.826 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:52:22.826 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:52:22.826 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:52:22.826 | INFO | ------------------------------------------ | 20260507 23:52:22.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:52:22.826 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:52:22.826 | INFO | Total mem usage: 27.8 GiB | 20260507 23:52:22.826 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:52:22.826 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:52:22.826 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:52:22.826 | INFO | High Water Breakdown: | 20260507 23:52:22.826 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:52:22.826 | INFO | ------------------------------------------ | 20260507 23:52:22.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:52:22.826 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:52:22.826 | INFO | Total mem usage: 15.7 GiB | 20260507 23:52:22.826 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:52:22.826 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:52:22.826 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:52:22.826 | INFO | High Water Breakdown: | 20260507 23:52:22.826 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:52:22.826 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:52:22.826 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:52:22.826 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:52:22.826 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:52:22.826 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:52:22.826 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:52:22.826 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:52:22.826 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:52:22.826 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:52:22.826 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:52:22.826 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:52:22.826 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:52:22.826 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:52:22.826 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:52:22.826 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:52:22.826 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:52:22.826 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:52:22.826 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:52:22.826 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:52:22.826 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:52:22.826 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:52:22.826 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:52:22.826 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:52:22.826 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:52:22.826 | INFO | ------------------------------------------ | 20260507 23:52:22.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:52:22.826 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:52:22.826 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:52:22.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:52:22.826 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.63e+01, 1.42e+01 / 1.51e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.17e-02 / 8.89e-03 | 20260507 23:52:22.826 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.49e+00, 5.78e-01 / 1.67e-01 | 20260507 23:52:22.826 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.05e-01 / 1.18e-01 | 20260507 23:52:22.826 | INFO | PerfReport: 16.11%, DME -- 4.22e+01 / 4.64e+01, 4.19e+00 / 1.77e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.58e+00 / 1.43e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.46e+00 / 2.79e+00, 4.27e-01 / 1.75e-01 | 20260507 23:52:22.826 | INFO | PerfReport: 3.42%, Metrics -- 1.12e+00 / 1.35e+00, 2.79e-01 / 1.49e-01 | 20260507 23:52:22.826 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.46e-01 / 3.85e-01, 5.60e-02 / 2.30e-02 | 20260507 23:52:22.826 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:52:22.826 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:52:22.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:52:22.826 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 24.54%, Upload -- 2.63e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 24.15%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 30.62%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 2.46%, LockUpload -- 2.64e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 2.54%, LockDownload -- 2.72e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 15.67%, LockCompute -- 1.68e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260507 23:52:22.826 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 23:52:22.922 | INFO | Starting a round of Baz IO | 20260507 23:52:26.403 | INFO | Populating RTMetrics report | 20260507 23:52:26.403 | INFO | RTMetrics report populated | 20260507 23:52:26.403 | INFO | Opening tmp RTMetrics json file | 20260507 23:52:26.434 | INFO | Serializing RTMetrics to json | 20260507 23:52:26.458 | INFO | Serialization done, swapping tmp file now... | 20260507 23:52:26.488 | INFO | RTMetrics done | 20260507 23:52:26.516 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.594 seconds | 20260507 23:52:26.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.42% / 0.00% / 57.57% | 20260507 23:52:26.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11454% / 1.19209e-05% / 5.96046e-05% | 20260507 23:52:26.516 | INFO | Baz IO overall: | 20260507 23:52:26.516 | INFO | BytesWritten : 1632448354837 | 20260507 23:52:26.516 | INFO | NumEvents : 790357560126 | 20260507 23:52:27.964 | INFO | Analyzing chunk frames = [3588096,3588608) | 20260507 23:52:27.964 | INFO | MemInfo: { | 20260507 23:52:27.964 | INFO | "gpu" : " 17 GB", | 20260507 23:52:27.964 | INFO | "huge" : "", | 20260507 23:52:27.964 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:27.964 | INFO | "malloc" : " 14 GB", | 20260507 23:52:27.964 | INFO | "pinned" : " 29 GB" | 20260507 23:52:27.964 | INFO | } | 20260507 23:52:27.964 | INFO | Perf report for the last 5.13903 seconds: | 20260507 23:52:27.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:27.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:27.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:27.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:27.964 | INFO | ANALYSIS : 68.7%, 3.0, 107.0ms, 0 | 20260507 23:52:27.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:27.964 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 23:52:27.964 | INFO | PRE_HQ : 65.9%, 1.7, 57.1ms, 0 | 20260507 23:52:27.964 | INFO | BAZWRITER : 69.4%, 4.0, 141.9ms, 0 | 20260507 23:52:27.964 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:52:27.964 | INFO | Uploaded 36GiB to GPU in 2.5915s (13.8915GiB/s) | 20260507 23:52:27.964 | INFO | Downloaded 38GiB to GPU in 2.70309s (14.058GiB/s) | 20260507 23:52:33.107 | INFO | Analyzing chunk frames = [3588608,3589120) | 20260507 23:52:33.107 | INFO | MemInfo: { | 20260507 23:52:33.107 | INFO | "gpu" : " 17 GB", | 20260507 23:52:33.107 | INFO | "huge" : "", | 20260507 23:52:33.107 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:33.107 | INFO | "malloc" : " 14 GB", | 20260507 23:52:33.107 | INFO | "pinned" : " 29 GB" | 20260507 23:52:33.107 | INFO | } | 20260507 23:52:33.107 | INFO | Perf report for the last 5.14319 seconds: | 20260507 23:52:33.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:33.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260507 23:52:33.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:52:33.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:33.107 | INFO | Uploaded 36GiB to GPU in 2.56113s (14.0563GiB/s) | 20260507 23:52:33.107 | INFO | Downloaded 34GiB to GPU in 2.49442s (13.6304GiB/s) | 20260507 23:52:38.243 | INFO | Analyzing chunk frames = [3589120,3589632) | 20260507 23:52:38.243 | INFO | MemInfo: { | 20260507 23:52:38.243 | INFO | "gpu" : " 17 GB", | 20260507 23:52:38.243 | INFO | "huge" : "", | 20260507 23:52:38.243 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:38.243 | INFO | "malloc" : " 14 GB", | 20260507 23:52:38.243 | INFO | "pinned" : " 29 GB" | 20260507 23:52:38.243 | INFO | } | 20260507 23:52:38.243 | INFO | Perf report for the last 5.13601 seconds: | 20260507 23:52:38.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:38.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260507 23:52:38.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:52:38.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:38.243 | INFO | Uploaded 36GiB to GPU in 2.46434s (14.6084GiB/s) | 20260507 23:52:38.243 | INFO | Downloaded 34GiB to GPU in 2.48288s (13.6937GiB/s) | 20260507 23:52:43.379 | INFO | Analyzing chunk frames = [3589632,3590144) | 20260507 23:52:43.379 | INFO | MemInfo: { | 20260507 23:52:43.379 | INFO | "gpu" : " 17 GB", | 20260507 23:52:43.379 | INFO | "huge" : "", | 20260507 23:52:43.379 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:43.379 | INFO | "malloc" : " 14 GB", | 20260507 23:52:43.380 | INFO | "pinned" : " 29 GB" | 20260507 23:52:43.380 | INFO | } | 20260507 23:52:43.380 | INFO | Perf report for the last 5.13665 seconds: | 20260507 23:52:43.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:43.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 23:52:43.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:52:43.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:43.380 | INFO | Uploaded 36GiB to GPU in 2.47763s (14.53GiB/s) | 20260507 23:52:43.380 | INFO | Downloaded 34GiB to GPU in 2.4866s (13.6733GiB/s) | 20260507 23:52:48.519 | INFO | Analyzing chunk frames = [3590144,3590656) | 20260507 23:52:48.519 | INFO | MemInfo: { | 20260507 23:52:48.519 | INFO | "gpu" : " 17 GB", | 20260507 23:52:48.519 | INFO | "huge" : "", | 20260507 23:52:48.519 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:48.519 | INFO | "malloc" : " 14 GB", | 20260507 23:52:48.519 | INFO | "pinned" : " 29 GB" | 20260507 23:52:48.519 | INFO | } | 20260507 23:52:48.519 | INFO | Perf report for the last 5.1398 seconds: | 20260507 23:52:48.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:48.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260507 23:52:48.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:52:48.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:48.520 | INFO | Uploaded 36GiB to GPU in 2.35431s (15.2911GiB/s) | 20260507 23:52:48.520 | INFO | Downloaded 34GiB to GPU in 2.50034s (13.5981GiB/s) | 20260507 23:52:53.657 | INFO | Analyzing chunk frames = [3590656,3591168) | 20260507 23:52:53.657 | INFO | MemInfo: { | 20260507 23:52:53.657 | INFO | "gpu" : " 17 GB", | 20260507 23:52:53.657 | INFO | "huge" : "", | 20260507 23:52:53.657 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:53.657 | INFO | "malloc" : " 14 GB", | 20260507 23:52:53.657 | INFO | "pinned" : " 29 GB" | 20260507 23:52:53.657 | INFO | } | 20260507 23:52:53.657 | INFO | Perf report for the last 5.13777 seconds: | 20260507 23:52:53.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:53.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.657 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260507 23:52:53.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.657 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:52:53.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:53.658 | INFO | Uploaded 36GiB to GPU in 2.54025s (14.1718GiB/s) | 20260507 23:52:53.658 | INFO | Downloaded 34GiB to GPU in 2.42143s (14.0413GiB/s) | 20260507 23:52:58.800 | INFO | Analyzing chunk frames = [3591168,3591680) | 20260507 23:52:58.800 | INFO | MemInfo: { | 20260507 23:52:58.800 | INFO | "gpu" : " 17 GB", | 20260507 23:52:58.800 | INFO | "huge" : "", | 20260507 23:52:58.800 | INFO | "huge_shared" : " 29 GB", | 20260507 23:52:58.800 | INFO | "malloc" : " 14 GB", | 20260507 23:52:58.800 | INFO | "pinned" : " 29 GB" | 20260507 23:52:58.800 | INFO | } | 20260507 23:52:58.800 | INFO | Perf report for the last 5.14257 seconds: | 20260507 23:52:58.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:52:58.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260507 23:52:58.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:52:58.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:52:58.800 | INFO | Uploaded 36GiB to GPU in 2.53162s (14.2201GiB/s) | 20260507 23:52:58.800 | INFO | Downloaded 34GiB to GPU in 2.44751s (13.8917GiB/s) | 20260507 23:53:03.935 | INFO | Analyzing chunk frames = [3591680,3592192) | 20260507 23:53:03.935 | INFO | MemInfo: { | 20260507 23:53:03.935 | INFO | "gpu" : " 17 GB", | 20260507 23:53:03.935 | INFO | "huge" : "", | 20260507 23:53:03.935 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:03.936 | INFO | "malloc" : " 14 GB", | 20260507 23:53:03.936 | INFO | "pinned" : " 29 GB" | 20260507 23:53:03.936 | INFO | } | 20260507 23:53:03.936 | INFO | Perf report for the last 5.1358 seconds: | 20260507 23:53:03.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:03.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:53:03.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:53:03.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:03.936 | INFO | Uploaded 36GiB to GPU in 2.54876s (14.1245GiB/s) | 20260507 23:53:03.936 | INFO | Downloaded 34GiB to GPU in 2.4479s (13.8895GiB/s) | 20260507 23:53:04.033 | INFO | Starting a round of Baz IO | 20260507 23:53:07.502 | INFO | Populating RTMetrics report | 20260507 23:53:07.502 | INFO | RTMetrics report populated | 20260507 23:53:07.502 | INFO | Opening tmp RTMetrics json file | 20260507 23:53:07.512 | INFO | Serializing RTMetrics to json | 20260507 23:53:07.525 | INFO | Serialization done, swapping tmp file now... | 20260507 23:53:07.526 | INFO | RTMetrics done | 20260507 23:53:07.588 | INFO | Finished a round of Baz IO: Wrote 1.704 GiB in 3.554 seconds | 20260507 23:53:07.588 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.42% / 0.00% / 57.58% | 20260507 23:53:07.588 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11085% / 1.19209e-05% / 6.75519e-05% | 20260507 23:53:07.588 | INFO | Baz IO overall: | 20260507 23:53:07.588 | INFO | BytesWritten : 1634278384206 | 20260507 23:53:07.588 | INFO | NumEvents : 791238759752 | 20260507 23:53:09.074 | INFO | Analyzing chunk frames = [3592192,3592704) | 20260507 23:53:09.074 | INFO | MemInfo: { | 20260507 23:53:09.074 | INFO | "gpu" : " 17 GB", | 20260507 23:53:09.074 | INFO | "huge" : "", | 20260507 23:53:09.074 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:09.074 | INFO | "malloc" : " 14 GB", | 20260507 23:53:09.074 | INFO | "pinned" : " 29 GB" | 20260507 23:53:09.074 | INFO | } | 20260507 23:53:09.075 | INFO | Perf report for the last 5.13887 seconds: | 20260507 23:53:09.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:09.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:09.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:09.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:09.075 | INFO | ANALYSIS : 68.8%, 3.0, 106.5ms, 0 | 20260507 23:53:09.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:09.075 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 23:53:09.075 | INFO | PRE_HQ : 66.1%, 1.7, 57.0ms, 0 | 20260507 23:53:09.075 | INFO | BAZWRITER : 68.8%, 3.8, 133.5ms, 0 | 20260507 23:53:09.075 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:53:09.075 | INFO | Uploaded 36GiB to GPU in 2.53459s (14.2035GiB/s) | 20260507 23:53:09.075 | INFO | Downloaded 38GiB to GPU in 2.68135s (14.172GiB/s) | 20260507 23:53:14.218 | INFO | Analyzing chunk frames = [3592704,3593216) | 20260507 23:53:14.218 | INFO | MemInfo: { | 20260507 23:53:14.218 | INFO | "gpu" : " 17 GB", | 20260507 23:53:14.218 | INFO | "huge" : "", | 20260507 23:53:14.218 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:14.218 | INFO | "malloc" : " 14 GB", | 20260507 23:53:14.218 | INFO | "pinned" : " 29 GB" | 20260507 23:53:14.218 | INFO | } | 20260507 23:53:14.218 | INFO | Perf report for the last 5.14337 seconds: | 20260507 23:53:14.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:14.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260507 23:53:14.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 23:53:14.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:14.218 | INFO | Uploaded 36GiB to GPU in 2.5812s (13.947GiB/s) | 20260507 23:53:14.218 | INFO | Downloaded 34GiB to GPU in 2.46868s (13.7726GiB/s) | 20260507 23:53:19.356 | INFO | Analyzing chunk frames = [3593216,3593728) | 20260507 23:53:19.356 | INFO | MemInfo: { | 20260507 23:53:19.356 | INFO | "gpu" : " 17 GB", | 20260507 23:53:19.356 | INFO | "huge" : "", | 20260507 23:53:19.356 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:19.356 | INFO | "malloc" : " 14 GB", | 20260507 23:53:19.356 | INFO | "pinned" : " 29 GB" | 20260507 23:53:19.356 | INFO | } | 20260507 23:53:19.356 | INFO | Perf report for the last 5.13798 seconds: | 20260507 23:53:19.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:19.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:53:19.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 23:53:19.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:19.356 | INFO | Uploaded 36GiB to GPU in 2.53998s (14.1734GiB/s) | 20260507 23:53:19.357 | INFO | Downloaded 34GiB to GPU in 2.45921s (13.8256GiB/s) | 20260507 23:53:24.492 | INFO | Analyzing chunk frames = [3593728,3594240) | 20260507 23:53:24.492 | INFO | MemInfo: { | 20260507 23:53:24.492 | INFO | "gpu" : " 17 GB", | 20260507 23:53:24.492 | INFO | "huge" : "", | 20260507 23:53:24.492 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:24.492 | INFO | "malloc" : " 14 GB", | 20260507 23:53:24.492 | INFO | "pinned" : " 29 GB" | 20260507 23:53:24.492 | INFO | } | 20260507 23:53:24.492 | INFO | Perf report for the last 5.13606 seconds: | 20260507 23:53:24.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:24.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.492 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260507 23:53:24.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.492 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:53:24.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:24.493 | INFO | Uploaded 36GiB to GPU in 2.55106s (14.1118GiB/s) | 20260507 23:53:24.493 | INFO | Downloaded 34GiB to GPU in 2.49362s (13.6348GiB/s) | 20260507 23:53:29.630 | INFO | Analyzing chunk frames = [3594240,3594752) | 20260507 23:53:29.630 | INFO | MemInfo: { | 20260507 23:53:29.630 | INFO | "gpu" : " 17 GB", | 20260507 23:53:29.630 | INFO | "huge" : "", | 20260507 23:53:29.630 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:29.630 | INFO | "malloc" : " 14 GB", | 20260507 23:53:29.630 | INFO | "pinned" : " 29 GB" | 20260507 23:53:29.630 | INFO | } | 20260507 23:53:29.630 | INFO | Perf report for the last 5.13844 seconds: | 20260507 23:53:29.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:29.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.631 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260507 23:53:29.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.631 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 23:53:29.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:29.631 | INFO | Uploaded 36GiB to GPU in 2.41554s (14.9035GiB/s) | 20260507 23:53:29.631 | INFO | Downloaded 34GiB to GPU in 2.4994s (13.6033GiB/s) | 20260507 23:53:34.773 | INFO | Analyzing chunk frames = [3594752,3595264) | 20260507 23:53:34.773 | INFO | MemInfo: { | 20260507 23:53:34.773 | INFO | "gpu" : " 17 GB", | 20260507 23:53:34.773 | INFO | "huge" : "", | 20260507 23:53:34.773 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:34.773 | INFO | "malloc" : " 14 GB", | 20260507 23:53:34.773 | INFO | "pinned" : " 29 GB" | 20260507 23:53:34.774 | INFO | } | 20260507 23:53:34.774 | INFO | Perf report for the last 5.1431 seconds: | 20260507 23:53:34.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:34.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260507 23:53:34.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:53:34.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:34.774 | INFO | Uploaded 36GiB to GPU in 2.54346s (14.154GiB/s) | 20260507 23:53:34.774 | INFO | Downloaded 34GiB to GPU in 2.42713s (14.0083GiB/s) | 20260507 23:53:39.916 | INFO | Analyzing chunk frames = [3595264,3595776) | 20260507 23:53:39.917 | INFO | MemInfo: { | 20260507 23:53:39.917 | INFO | "gpu" : " 17 GB", | 20260507 23:53:39.917 | INFO | "huge" : "", | 20260507 23:53:39.917 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:39.917 | INFO | "malloc" : " 14 GB", | 20260507 23:53:39.917 | INFO | "pinned" : " 29 GB" | 20260507 23:53:39.917 | INFO | } | 20260507 23:53:39.917 | INFO | Perf report for the last 5.14318 seconds: | 20260507 23:53:39.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:39.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:53:39.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:53:39.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:39.917 | INFO | Uploaded 36GiB to GPU in 2.56044s (14.0601GiB/s) | 20260507 23:53:39.917 | INFO | Downloaded 34GiB to GPU in 2.45392s (13.8554GiB/s) | 20260507 23:53:45.054 | INFO | Analyzing chunk frames = [3595776,3596288) | 20260507 23:53:45.054 | INFO | MemInfo: { | 20260507 23:53:45.054 | INFO | "gpu" : " 17 GB", | 20260507 23:53:45.054 | INFO | "huge" : "", | 20260507 23:53:45.054 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:45.054 | INFO | "malloc" : " 14 GB", | 20260507 23:53:45.054 | INFO | "pinned" : " 29 GB" | 20260507 23:53:45.054 | INFO | } | 20260507 23:53:45.054 | INFO | Perf report for the last 5.13749 seconds: | 20260507 23:53:45.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:45.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.054 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260507 23:53:45.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.054 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:53:45.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:45.055 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:53:45.055 | INFO | Total mem usage: 16.7 GiB | 20260507 23:53:45.055 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:53:45.055 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.5%) | 20260507 23:53:45.055 | INFO | Uploaded 36GiB to GPU in 2.56191s (14.052GiB/s) | 20260507 23:53:45.055 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:53:45.055 | INFO | High Water Breakdown: | 20260507 23:53:45.055 | INFO | Downloaded 34GiB to GPU in 2.44744s (13.8921GiB/s) | 20260507 23:53:45.055 | INFO | 59.8% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:53:45.055 | INFO | 40% : 5.88 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:53:45.055 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:53:45.055 | INFO | ------------------------------------------ | 20260507 23:53:45.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:53:45.055 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:53:45.055 | INFO | Total mem usage: 26.5 GiB | 20260507 23:53:45.055 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:53:45.055 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:53:45.055 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:53:45.055 | INFO | High Water Breakdown: | 20260507 23:53:45.055 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:53:45.055 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:53:45.055 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:53:45.055 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:53:45.055 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:53:45.055 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:53:45.055 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:53:45.055 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:53:45.055 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:53:45.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:53:45.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:53:45.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:53:45.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:53:45.055 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:53:45.055 | INFO | ------------------------------------------ | 20260507 23:53:45.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:53:45.055 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:53:45.055 | INFO | Total mem usage: 27.8 GiB | 20260507 23:53:45.055 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:53:45.055 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:53:45.055 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:53:45.055 | INFO | High Water Breakdown: | 20260507 23:53:45.055 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:53:45.055 | INFO | ------------------------------------------ | 20260507 23:53:45.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:53:45.055 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:53:45.055 | INFO | Total mem usage: 15.7 GiB | 20260507 23:53:45.055 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:53:45.055 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:53:45.055 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:53:45.055 | INFO | High Water Breakdown: | 20260507 23:53:45.055 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:53:45.056 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:53:45.056 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:53:45.056 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:53:45.056 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:53:45.056 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:53:45.056 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:53:45.056 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:53:45.056 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:53:45.056 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:53:45.056 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:53:45.056 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:53:45.056 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:53:45.056 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:53:45.056 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:53:45.056 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:53:45.056 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:53:45.056 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:53:45.056 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:53:45.056 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:53:45.056 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:53:45.056 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:53:45.056 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:53:45.056 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:53:45.056 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:53:45.056 | INFO | ------------------------------------------ | 20260507 23:53:45.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:53:45.056 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:53:45.056 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:53:45.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:53:45.056 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.67e+01, 1.42e+01 / 2.02e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.03e-01, 1.18e-02 / 6.04e-03 | 20260507 23:53:45.056 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.47e+00, 5.82e-01 / 2.25e-01 | 20260507 23:53:45.056 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.04e-01 / 1.03e-01 | 20260507 23:53:45.056 | INFO | PerfReport: 16.20%, DME -- 4.24e+01 / 4.68e+01, 4.23e+00 / 2.14e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.83e+01 / 1.93e+01, 1.52e+00 / 1.42e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.46e+00 / 2.77e+00, 4.33e-01 / 1.95e-01 | 20260507 23:53:45.056 | INFO | PerfReport: 3.42%, Metrics -- 1.12e+00 / 1.37e+00, 2.79e-01 / 1.84e-01 | 20260507 23:53:45.056 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.86e-01, 5.45e-02 / 2.40e-02 | 20260507 23:53:45.056 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:53:45.056 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:53:45.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:53:45.056 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 24.54%, Upload -- 2.63e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 24.27%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 30.55%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 2.44%, LockUpload -- 2.61e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 2.65%, LockDownload -- 2.84e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 15.54%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 23:53:45.056 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 23:53:45.152 | INFO | Starting a round of Baz IO | 20260507 23:53:48.611 | INFO | Populating RTMetrics report | 20260507 23:53:48.612 | INFO | RTMetrics report populated | 20260507 23:53:48.612 | INFO | Opening tmp RTMetrics json file | 20260507 23:53:48.678 | INFO | Serializing RTMetrics to json | 20260507 23:53:48.715 | INFO | Serialization done, swapping tmp file now... | 20260507 23:53:48.716 | INFO | RTMetrics done | 20260507 23:53:48.732 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.58 seconds | 20260507 23:53:48.732 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.41% / 0.00% / 57.58% | 20260507 23:53:48.732 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11072% / 0% / 5.5631e-05% | 20260507 23:53:48.732 | INFO | Baz IO overall: | 20260507 23:53:48.732 | INFO | BytesWritten : 1636108857365 | 20260507 23:53:48.732 | INFO | NumEvents : 792120468625 | 20260507 23:53:50.188 | INFO | Analyzing chunk frames = [3596288,3596800) | 20260507 23:53:50.188 | INFO | MemInfo: { | 20260507 23:53:50.188 | INFO | "gpu" : " 17 GB", | 20260507 23:53:50.188 | INFO | "huge" : "", | 20260507 23:53:50.188 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:50.188 | INFO | "malloc" : " 14 GB", | 20260507 23:53:50.188 | INFO | "pinned" : " 29 GB" | 20260507 23:53:50.188 | INFO | } | 20260507 23:53:50.188 | INFO | Perf report for the last 5.13392 seconds: | 20260507 23:53:50.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:50.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:50.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:50.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:50.188 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 23:53:50.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:50.188 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 23:53:50.189 | INFO | PRE_HQ : 65.6%, 1.7, 57.0ms, 0 | 20260507 23:53:50.189 | INFO | BAZWRITER : 69.3%, 4.2, 148.8ms, 0 | 20260507 23:53:50.189 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260507 23:53:50.189 | INFO | Uploaded 36GiB to GPU in 2.56169s (14.0532GiB/s) | 20260507 23:53:50.189 | INFO | Downloaded 38GiB to GPU in 2.69201s (14.1159GiB/s) | 20260507 23:53:55.331 | INFO | Analyzing chunk frames = [3596800,3597312) | 20260507 23:53:55.332 | INFO | MemInfo: { | 20260507 23:53:55.332 | INFO | "gpu" : " 17 GB", | 20260507 23:53:55.332 | INFO | "huge" : "", | 20260507 23:53:55.332 | INFO | "huge_shared" : " 29 GB", | 20260507 23:53:55.332 | INFO | "malloc" : " 14 GB", | 20260507 23:53:55.332 | INFO | "pinned" : " 29 GB" | 20260507 23:53:55.332 | INFO | } | 20260507 23:53:55.332 | INFO | Perf report for the last 5.14358 seconds: | 20260507 23:53:55.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:53:55.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260507 23:53:55.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 23:53:55.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:53:55.332 | INFO | Uploaded 36GiB to GPU in 2.58594s (13.9215GiB/s) | 20260507 23:53:55.332 | INFO | Downloaded 34GiB to GPU in 2.47564s (13.7338GiB/s) | 20260507 23:54:00.473 | INFO | Analyzing chunk frames = [3597312,3597824) | 20260507 23:54:00.473 | INFO | MemInfo: { | 20260507 23:54:00.473 | INFO | "gpu" : " 17 GB", | 20260507 23:54:00.473 | INFO | "huge" : "", | 20260507 23:54:00.473 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:00.473 | INFO | "malloc" : " 14 GB", | 20260507 23:54:00.473 | INFO | "pinned" : " 29 GB" | 20260507 23:54:00.473 | INFO | } | 20260507 23:54:00.473 | INFO | Perf report for the last 5.14134 seconds: | 20260507 23:54:00.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:00.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260507 23:54:00.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:54:00.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:00.473 | INFO | Uploaded 36GiB to GPU in 2.59641s (13.8653GiB/s) | 20260507 23:54:00.474 | INFO | Downloaded 34GiB to GPU in 2.46604s (13.7873GiB/s) | 20260507 23:54:05.616 | INFO | Analyzing chunk frames = [3597824,3598336) | 20260507 23:54:05.616 | INFO | MemInfo: { | 20260507 23:54:05.616 | INFO | "gpu" : " 17 GB", | 20260507 23:54:05.616 | INFO | "huge" : "", | 20260507 23:54:05.616 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:05.616 | INFO | "malloc" : " 14 GB", | 20260507 23:54:05.616 | INFO | "pinned" : " 29 GB" | 20260507 23:54:05.616 | INFO | } | 20260507 23:54:05.616 | INFO | Perf report for the last 5.1432 seconds: | 20260507 23:54:05.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:05.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.616 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:54:05.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.616 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:54:05.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:05.617 | INFO | Uploaded 36GiB to GPU in 2.5558s (14.0856GiB/s) | 20260507 23:54:05.617 | INFO | Downloaded 34GiB to GPU in 2.46657s (13.7843GiB/s) | 20260507 23:54:10.745 | INFO | Analyzing chunk frames = [3598336,3598848) | 20260507 23:54:10.745 | INFO | MemInfo: { | 20260507 23:54:10.745 | INFO | "gpu" : " 17 GB", | 20260507 23:54:10.745 | INFO | "huge" : "", | 20260507 23:54:10.745 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:10.745 | INFO | "malloc" : " 14 GB", | 20260507 23:54:10.746 | INFO | "pinned" : " 29 GB" | 20260507 23:54:10.746 | INFO | } | 20260507 23:54:10.746 | INFO | Perf report for the last 5.12936 seconds: | 20260507 23:54:10.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:10.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260507 23:54:10.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 23:54:10.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:10.746 | INFO | Uploaded 36GiB to GPU in 2.41706s (14.8941GiB/s) | 20260507 23:54:10.746 | INFO | Downloaded 34GiB to GPU in 2.49713s (13.6156GiB/s) | 20260507 23:54:15.884 | INFO | Analyzing chunk frames = [3598848,3599360) | 20260507 23:54:15.884 | INFO | MemInfo: { | 20260507 23:54:15.884 | INFO | "gpu" : " 17 GB", | 20260507 23:54:15.884 | INFO | "huge" : "", | 20260507 23:54:15.884 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:15.884 | INFO | "malloc" : " 14 GB", | 20260507 23:54:15.884 | INFO | "pinned" : " 29 GB" | 20260507 23:54:15.885 | INFO | } | 20260507 23:54:15.885 | INFO | Perf report for the last 5.13905 seconds: | 20260507 23:54:15.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:15.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260507 23:54:15.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:54:15.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:15.885 | INFO | Uploaded 36GiB to GPU in 2.41468s (14.9088GiB/s) | 20260507 23:54:15.885 | INFO | Downloaded 34GiB to GPU in 2.40845s (14.1169GiB/s) | 20260507 23:54:21.028 | INFO | Analyzing chunk frames = [3599360,3599872) | 20260507 23:54:21.028 | INFO | MemInfo: { | 20260507 23:54:21.028 | INFO | "gpu" : " 17 GB", | 20260507 23:54:21.028 | INFO | "huge" : "", | 20260507 23:54:21.028 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:21.028 | INFO | "malloc" : " 14 GB", | 20260507 23:54:21.028 | INFO | "pinned" : " 29 GB" | 20260507 23:54:21.028 | INFO | } | 20260507 23:54:21.028 | INFO | Perf report for the last 5.14349 seconds: | 20260507 23:54:21.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:21.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.028 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260507 23:54:21.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.028 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:54:21.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:21.029 | INFO | Uploaded 36GiB to GPU in 2.52679s (14.2473GiB/s) | 20260507 23:54:21.029 | INFO | Downloaded 34GiB to GPU in 2.44742s (13.8922GiB/s) | 20260507 23:54:26.167 | INFO | Analyzing chunk frames = [3599872,3600384) | 20260507 23:54:26.167 | INFO | MemInfo: { | 20260507 23:54:26.167 | INFO | "gpu" : " 17 GB", | 20260507 23:54:26.167 | INFO | "huge" : "", | 20260507 23:54:26.167 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:26.167 | INFO | "malloc" : " 14 GB", | 20260507 23:54:26.167 | INFO | "pinned" : " 29 GB" | 20260507 23:54:26.167 | INFO | } | 20260507 23:54:26.167 | INFO | Perf report for the last 5.13905 seconds: | 20260507 23:54:26.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:26.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.167 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260507 23:54:26.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.168 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:54:26.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:26.168 | INFO | Uploaded 36GiB to GPU in 2.49413s (14.4339GiB/s) | 20260507 23:54:26.168 | INFO | Downloaded 34GiB to GPU in 2.43622s (13.956GiB/s) | 20260507 23:54:26.282 | INFO | Starting a round of Baz IO | 20260507 23:54:29.731 | INFO | Populating RTMetrics report | 20260507 23:54:29.732 | INFO | RTMetrics report populated | 20260507 23:54:29.732 | INFO | Opening tmp RTMetrics json file | 20260507 23:54:29.748 | INFO | Serializing RTMetrics to json | 20260507 23:54:29.756 | INFO | Serialization done, swapping tmp file now... | 20260507 23:54:29.757 | INFO | RTMetrics done | 20260507 23:54:29.856 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.573 seconds | 20260507 23:54:29.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.41% / 0.00% / 57.59% | 20260507 23:54:29.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11041% / 1.19209e-05% / 6.35783e-05% | 20260507 23:54:29.856 | INFO | Baz IO overall: | 20260507 23:54:29.856 | INFO | BytesWritten : 1637939534351 | 20260507 23:54:29.856 | INFO | NumEvents : 793002255844 | 20260507 23:54:31.308 | INFO | Analyzing chunk frames = [3600384,3600896) | 20260507 23:54:31.308 | INFO | MemInfo: { | 20260507 23:54:31.308 | INFO | "gpu" : " 17 GB", | 20260507 23:54:31.308 | INFO | "huge" : "", | 20260507 23:54:31.308 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:31.308 | INFO | "malloc" : " 14 GB", | 20260507 23:54:31.308 | INFO | "pinned" : " 29 GB" | 20260507 23:54:31.308 | INFO | } | 20260507 23:54:31.308 | INFO | Perf report for the last 5.14106 seconds: | 20260507 23:54:31.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:31.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:31.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:31.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:31.309 | INFO | ANALYSIS : 68.6%, 2.9, 104.1ms, 0 | 20260507 23:54:31.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:31.309 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260507 23:54:31.309 | INFO | PRE_HQ : 67.5%, 1.8, 63.1ms, 0 | 20260507 23:54:31.309 | INFO | BAZWRITER : 69.2%, 3.8, 135.0ms, 0 | 20260507 23:54:31.309 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 23:54:31.309 | INFO | Uploaded 36GiB to GPU in 2.59255s (13.8859GiB/s) | 20260507 23:54:31.309 | INFO | Downloaded 38GiB to GPU in 2.70477s (14.0493GiB/s) | 20260507 23:54:36.444 | INFO | Analyzing chunk frames = [3600896,3601408) | 20260507 23:54:36.444 | INFO | MemInfo: { | 20260507 23:54:36.444 | INFO | "gpu" : " 17 GB", | 20260507 23:54:36.444 | INFO | "huge" : "", | 20260507 23:54:36.444 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:36.444 | INFO | "malloc" : " 14 GB", | 20260507 23:54:36.444 | INFO | "pinned" : " 29 GB" | 20260507 23:54:36.444 | INFO | } | 20260507 23:54:36.444 | INFO | Perf report for the last 5.13521 seconds: | 20260507 23:54:36.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:36.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260507 23:54:36.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:54:36.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:36.444 | INFO | Uploaded 36GiB to GPU in 2.53987s (14.174GiB/s) | 20260507 23:54:36.444 | INFO | Downloaded 34GiB to GPU in 2.42075s (14.0452GiB/s) | 20260507 23:54:41.584 | INFO | Analyzing chunk frames = [3601408,3601920) | 20260507 23:54:41.584 | INFO | MemInfo: { | 20260507 23:54:41.584 | INFO | "gpu" : " 17 GB", | 20260507 23:54:41.584 | INFO | "huge" : "", | 20260507 23:54:41.584 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:41.584 | INFO | "malloc" : " 14 GB", | 20260507 23:54:41.584 | INFO | "pinned" : " 29 GB" | 20260507 23:54:41.584 | INFO | } | 20260507 23:54:41.584 | INFO | Perf report for the last 5.14037 seconds: | 20260507 23:54:41.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:41.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:54:41.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:54:41.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:41.584 | INFO | Uploaded 36GiB to GPU in 2.48668s (14.4771GiB/s) | 20260507 23:54:41.584 | INFO | Downloaded 34GiB to GPU in 2.48955s (13.6571GiB/s) | 20260507 23:54:46.725 | INFO | Analyzing chunk frames = [3601920,3602432) | 20260507 23:54:46.725 | INFO | MemInfo: { | 20260507 23:54:46.725 | INFO | "gpu" : " 17 GB", | 20260507 23:54:46.725 | INFO | "huge" : "", | 20260507 23:54:46.725 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:46.725 | INFO | "malloc" : " 14 GB", | 20260507 23:54:46.725 | INFO | "pinned" : " 29 GB" | 20260507 23:54:46.725 | INFO | } | 20260507 23:54:46.725 | INFO | Perf report for the last 5.14112 seconds: | 20260507 23:54:46.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:46.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:54:46.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:54:46.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:46.725 | INFO | Uploaded 36GiB to GPU in 2.51121s (14.3357GiB/s) | 20260507 23:54:46.725 | INFO | Downloaded 34GiB to GPU in 2.49918s (13.6045GiB/s) | 20260507 23:54:51.868 | INFO | Analyzing chunk frames = [3602432,3602944) | 20260507 23:54:51.868 | INFO | MemInfo: { | 20260507 23:54:51.868 | INFO | "gpu" : " 17 GB", | 20260507 23:54:51.868 | INFO | "huge" : "", | 20260507 23:54:51.868 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:51.868 | INFO | "malloc" : " 14 GB", | 20260507 23:54:51.868 | INFO | "pinned" : " 29 GB" | 20260507 23:54:51.868 | INFO | } | 20260507 23:54:51.868 | INFO | Perf report for the last 5.14262 seconds: | 20260507 23:54:51.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:51.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260507 23:54:51.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:54:51.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:51.868 | INFO | Uploaded 36GiB to GPU in 2.37503s (15.1577GiB/s) | 20260507 23:54:51.868 | INFO | Downloaded 34GiB to GPU in 2.49894s (13.6058GiB/s) | 20260507 23:54:57.003 | INFO | Analyzing chunk frames = [3602944,3603456) | 20260507 23:54:57.003 | INFO | MemInfo: { | 20260507 23:54:57.003 | INFO | "gpu" : " 17 GB", | 20260507 23:54:57.003 | INFO | "huge" : "", | 20260507 23:54:57.003 | INFO | "huge_shared" : " 29 GB", | 20260507 23:54:57.003 | INFO | "malloc" : " 14 GB", | 20260507 23:54:57.004 | INFO | "pinned" : " 29 GB" | 20260507 23:54:57.004 | INFO | } | 20260507 23:54:57.004 | INFO | Perf report for the last 5.13576 seconds: | 20260507 23:54:57.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:54:57.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:54:57.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:54:57.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:54:57.004 | INFO | Uploaded 36GiB to GPU in 2.44065s (14.7502GiB/s) | 20260507 23:54:57.004 | INFO | Downloaded 34GiB to GPU in 2.37693s (14.3042GiB/s) | 20260507 23:55:02.142 | INFO | Analyzing chunk frames = [3603456,3603968) | 20260507 23:55:02.142 | INFO | MemInfo: { | 20260507 23:55:02.142 | INFO | "gpu" : " 17 GB", | 20260507 23:55:02.142 | INFO | "huge" : "", | 20260507 23:55:02.142 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:02.142 | INFO | "malloc" : " 14 GB", | 20260507 23:55:02.142 | INFO | "pinned" : " 29 GB" | 20260507 23:55:02.142 | INFO | } | 20260507 23:55:02.143 | INFO | Perf report for the last 5.1389 seconds: | 20260507 23:55:02.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:02.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 23:55:02.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 23:55:02.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:02.143 | INFO | Uploaded 36GiB to GPU in 2.54456s (14.1479GiB/s) | 20260507 23:55:02.143 | INFO | Downloaded 34GiB to GPU in 2.4413s (13.927GiB/s) | 20260507 23:55:07.285 | INFO | Analyzing chunk frames = [3603968,3604480) | 20260507 23:55:07.285 | INFO | MemInfo: { | 20260507 23:55:07.285 | INFO | "gpu" : " 17 GB", | 20260507 23:55:07.285 | INFO | "huge" : "", | 20260507 23:55:07.285 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:07.285 | INFO | "malloc" : " 14 GB", | 20260507 23:55:07.286 | INFO | "pinned" : " 29 GB" | 20260507 23:55:07.286 | INFO | } | 20260507 23:55:07.286 | INFO | Perf report for the last 5.1433 seconds: | 20260507 23:55:07.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:07.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260507 23:55:07.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | PRE_HQ : 34.9%, 1.0, 18.2ms, 0 | 20260507 23:55:07.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:07.286 | INFO | Uploaded 36GiB to GPU in 2.54056s (14.1701GiB/s) | 20260507 23:55:07.286 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:55:07.286 | INFO | Total mem usage: 16.7 GiB | 20260507 23:55:07.286 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:55:07.286 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13%) | 20260507 23:55:07.286 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:55:07.286 | INFO | High Water Breakdown: | 20260507 23:55:07.286 | INFO | 60% : 8.85 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:55:07.286 | INFO | 39.9% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:55:07.286 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:55:07.286 | INFO | ------------------------------------------ | 20260507 23:55:07.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:55:07.286 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:55:07.286 | INFO | Downloaded 34GiB to GPU in 2.44144s (13.9262GiB/s) | 20260507 23:55:07.286 | INFO | Total mem usage: 26.5 GiB | 20260507 23:55:07.286 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:55:07.286 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:55:07.287 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:55:07.287 | INFO | High Water Breakdown: | 20260507 23:55:07.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:55:07.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:55:07.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:55:07.287 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:55:07.287 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:55:07.287 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:55:07.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:55:07.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:55:07.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:55:07.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:55:07.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:55:07.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:55:07.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:55:07.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:55:07.287 | INFO | ------------------------------------------ | 20260507 23:55:07.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:55:07.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:55:07.287 | INFO | Total mem usage: 27.8 GiB | 20260507 23:55:07.287 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:55:07.287 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:55:07.287 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:55:07.287 | INFO | High Water Breakdown: | 20260507 23:55:07.287 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:55:07.287 | INFO | ------------------------------------------ | 20260507 23:55:07.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:55:07.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:55:07.287 | INFO | Total mem usage: 15.7 GiB | 20260507 23:55:07.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:55:07.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:55:07.287 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:55:07.287 | INFO | High Water Breakdown: | 20260507 23:55:07.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:55:07.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:55:07.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:55:07.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:55:07.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:55:07.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:55:07.287 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:55:07.287 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:55:07.287 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:55:07.288 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:55:07.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:55:07.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:55:07.288 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:55:07.288 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:55:07.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:55:07.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:55:07.288 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:55:07.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:55:07.288 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:55:07.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:55:07.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:55:07.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:55:07.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:55:07.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:55:07.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:55:07.288 | INFO | ------------------------------------------ | 20260507 23:55:07.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:55:07.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:55:07.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:55:07.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:55:07.288 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.66e+01, 1.40e+01 / 1.73e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.07e-02 / 3.93e-03 | 20260507 23:55:07.288 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.46e+00, 5.82e-01 / 2.16e-01 | 20260507 23:55:07.288 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.49e+00, 2.06e-01 / 8.93e-02 | 20260507 23:55:07.288 | INFO | PerfReport: 15.99%, DME -- 4.20e+01 / 4.60e+01, 4.00e+00 / 1.85e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.69e+00 / 1.61e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.30e-01 / 2.18e-01 | 20260507 23:55:07.288 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.31e+00, 2.79e-01 / 1.55e-01 | 20260507 23:55:07.288 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.46e-01 / 3.80e-01, 5.49e-02 / 2.61e-02 | 20260507 23:55:07.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:55:07.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:55:07.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:55:07.288 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 24.51%, Upload -- 2.62e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 30.76%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 2.24%, LockUpload -- 2.40e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 2.57%, LockDownload -- 2.74e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260507 23:55:07.288 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260507 23:55:07.399 | INFO | Starting a round of Baz IO | 20260507 23:55:10.875 | INFO | Populating RTMetrics report | 20260507 23:55:10.876 | INFO | RTMetrics report populated | 20260507 23:55:10.876 | INFO | Opening tmp RTMetrics json file | 20260507 23:55:10.895 | INFO | Serializing RTMetrics to json | 20260507 23:55:10.903 | INFO | Serialization done, swapping tmp file now... | 20260507 23:55:10.905 | INFO | RTMetrics done | 20260507 23:55:10.979 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.58 seconds | 20260507 23:55:10.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.41% / 0.00% / 57.59% | 20260507 23:55:10.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.10394% / 2.38419e-05% / 5.96046e-05% | 20260507 23:55:10.979 | INFO | Baz IO overall: | 20260507 23:55:10.980 | INFO | BytesWritten : 1639767672580 | 20260507 23:55:10.980 | INFO | NumEvents : 793882343672 | 20260507 23:55:12.428 | INFO | Analyzing chunk frames = [3604480,3604992) | 20260507 23:55:12.428 | INFO | MemInfo: { | 20260507 23:55:12.428 | INFO | "gpu" : " 17 GB", | 20260507 23:55:12.428 | INFO | "huge" : "", | 20260507 23:55:12.428 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:12.428 | INFO | "malloc" : " 14 GB", | 20260507 23:55:12.428 | INFO | "pinned" : " 29 GB" | 20260507 23:55:12.428 | INFO | } | 20260507 23:55:12.428 | INFO | Perf report for the last 5.14187 seconds: | 20260507 23:55:12.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:12.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:12.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:12.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:12.428 | INFO | ANALYSIS : 69.1%, 2.9, 104.6ms, 0 | 20260507 23:55:12.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:12.428 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260507 23:55:12.428 | INFO | PRE_HQ : 68.1%, 1.8, 65.3ms, 0 | 20260507 23:55:12.428 | INFO | BAZWRITER : 69.2%, 4.5, 161.7ms, 0 | 20260507 23:55:12.428 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260507 23:55:12.428 | INFO | Uploaded 36GiB to GPU in 2.63101s (13.6829GiB/s) | 20260507 23:55:12.428 | INFO | Downloaded 38GiB to GPU in 2.71293s (14.007GiB/s) | 20260507 23:55:17.562 | INFO | Analyzing chunk frames = [3604992,3605504) | 20260507 23:55:17.562 | INFO | MemInfo: { | 20260507 23:55:17.562 | INFO | "gpu" : " 17 GB", | 20260507 23:55:17.562 | INFO | "huge" : "", | 20260507 23:55:17.562 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:17.562 | INFO | "malloc" : " 14 GB", | 20260507 23:55:17.562 | INFO | "pinned" : " 29 GB" | 20260507 23:55:17.562 | INFO | } | 20260507 23:55:17.562 | INFO | Perf report for the last 5.13451 seconds: | 20260507 23:55:17.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:17.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:55:17.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:55:17.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:17.562 | INFO | Uploaded 36GiB to GPU in 2.52228s (14.2728GiB/s) | 20260507 23:55:17.563 | INFO | Downloaded 34GiB to GPU in 2.43311s (13.9739GiB/s) | 20260507 23:55:22.699 | INFO | Analyzing chunk frames = [3605504,3606016) | 20260507 23:55:22.699 | INFO | MemInfo: { | 20260507 23:55:22.699 | INFO | "gpu" : " 17 GB", | 20260507 23:55:22.699 | INFO | "huge" : "", | 20260507 23:55:22.699 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:22.699 | INFO | "malloc" : " 14 GB", | 20260507 23:55:22.700 | INFO | "pinned" : " 29 GB" | 20260507 23:55:22.700 | INFO | } | 20260507 23:55:22.700 | INFO | Perf report for the last 5.13742 seconds: | 20260507 23:55:22.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:22.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260507 23:55:22.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 23:55:22.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:22.700 | INFO | Uploaded 36GiB to GPU in 2.51495s (14.3144GiB/s) | 20260507 23:55:22.700 | INFO | Downloaded 34GiB to GPU in 2.48836s (13.6636GiB/s) | 20260507 23:55:27.837 | INFO | Analyzing chunk frames = [3606016,3606528) | 20260507 23:55:27.837 | INFO | MemInfo: { | 20260507 23:55:27.837 | INFO | "gpu" : " 17 GB", | 20260507 23:55:27.837 | INFO | "huge" : "", | 20260507 23:55:27.837 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:27.837 | INFO | "malloc" : " 14 GB", | 20260507 23:55:27.838 | INFO | "pinned" : " 29 GB" | 20260507 23:55:27.838 | INFO | } | 20260507 23:55:27.838 | INFO | Perf report for the last 5.13787 seconds: | 20260507 23:55:27.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:27.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260507 23:55:27.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:55:27.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:27.838 | INFO | Uploaded 36GiB to GPU in 2.50021s (14.3988GiB/s) | 20260507 23:55:27.838 | INFO | Downloaded 34GiB to GPU in 2.501s (13.5945GiB/s) | 20260507 23:55:32.979 | INFO | Analyzing chunk frames = [3606528,3607040) | 20260507 23:55:32.979 | INFO | MemInfo: { | 20260507 23:55:32.979 | INFO | "gpu" : " 17 GB", | 20260507 23:55:32.979 | INFO | "huge" : "", | 20260507 23:55:32.979 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:32.979 | INFO | "malloc" : " 14 GB", | 20260507 23:55:32.979 | INFO | "pinned" : " 29 GB" | 20260507 23:55:32.979 | INFO | } | 20260507 23:55:32.979 | INFO | Perf report for the last 5.14172 seconds: | 20260507 23:55:32.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:32.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.979 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260507 23:55:32.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.979 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:55:32.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:32.980 | INFO | Uploaded 36GiB to GPU in 2.29798s (15.6659GiB/s) | 20260507 23:55:32.980 | INFO | Downloaded 34GiB to GPU in 2.48834s (13.6638GiB/s) | 20260507 23:55:38.123 | INFO | Analyzing chunk frames = [3607040,3607552) | 20260507 23:55:38.123 | INFO | MemInfo: { | 20260507 23:55:38.123 | INFO | "gpu" : " 17 GB", | 20260507 23:55:38.123 | INFO | "huge" : "", | 20260507 23:55:38.123 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:38.123 | INFO | "malloc" : " 14 GB", | 20260507 23:55:38.123 | INFO | "pinned" : " 29 GB" | 20260507 23:55:38.123 | INFO | } | 20260507 23:55:38.123 | INFO | Perf report for the last 5.1438 seconds: | 20260507 23:55:38.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:38.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260507 23:55:38.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 23:55:38.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:38.123 | INFO | Uploaded 36GiB to GPU in 2.48748s (14.4725GiB/s) | 20260507 23:55:38.124 | INFO | Downloaded 34GiB to GPU in 2.44342s (13.9149GiB/s) | 20260507 23:55:43.256 | INFO | Analyzing chunk frames = [3607552,3608064) | 20260507 23:55:43.256 | INFO | MemInfo: { | 20260507 23:55:43.256 | INFO | "gpu" : " 17 GB", | 20260507 23:55:43.256 | INFO | "huge" : "", | 20260507 23:55:43.256 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:43.256 | INFO | "malloc" : " 14 GB", | 20260507 23:55:43.256 | INFO | "pinned" : " 29 GB" | 20260507 23:55:43.256 | INFO | } | 20260507 23:55:43.256 | INFO | Perf report for the last 5.13281 seconds: | 20260507 23:55:43.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:43.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260507 23:55:43.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 23:55:43.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:43.256 | INFO | Uploaded 36GiB to GPU in 2.54744s (14.1318GiB/s) | 20260507 23:55:43.256 | INFO | Downloaded 34GiB to GPU in 2.48539s (13.68GiB/s) | 20260507 23:55:48.395 | INFO | Analyzing chunk frames = [3608064,3608576) | 20260507 23:55:48.395 | INFO | MemInfo: { | 20260507 23:55:48.395 | INFO | "gpu" : " 17 GB", | 20260507 23:55:48.395 | INFO | "huge" : "", | 20260507 23:55:48.395 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:48.395 | INFO | "malloc" : " 14 GB", | 20260507 23:55:48.395 | INFO | "pinned" : " 29 GB" | 20260507 23:55:48.395 | INFO | } | 20260507 23:55:48.395 | INFO | Perf report for the last 5.1388 seconds: | 20260507 23:55:48.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:48.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:55:48.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 23:55:48.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:48.395 | INFO | Uploaded 36GiB to GPU in 2.49506s (14.4285GiB/s) | 20260507 23:55:48.395 | INFO | Downloaded 34GiB to GPU in 2.46158s (13.8123GiB/s) | 20260507 23:55:48.493 | INFO | Starting a round of Baz IO | 20260507 23:55:51.936 | INFO | Populating RTMetrics report | 20260507 23:55:51.936 | INFO | RTMetrics report populated | 20260507 23:55:51.936 | INFO | Opening tmp RTMetrics json file | 20260507 23:55:51.942 | INFO | Serializing RTMetrics to json | 20260507 23:55:51.970 | INFO | Serialization done, swapping tmp file now... | 20260507 23:55:51.975 | INFO | RTMetrics done | 20260507 23:55:52.056 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.563 seconds | 20260507 23:55:52.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.40% / 0.00% / 57.59% | 20260507 23:55:52.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.10011% / 7.94729e-06% / 6.35783e-05% | 20260507 23:55:52.057 | INFO | Baz IO overall: | 20260507 23:55:52.057 | INFO | BytesWritten : 1641594517056 | 20260507 23:55:52.057 | INFO | NumEvents : 794761416235 | 20260507 23:55:53.536 | INFO | Analyzing chunk frames = [3608576,3609088) | 20260507 23:55:53.536 | INFO | MemInfo: { | 20260507 23:55:53.536 | INFO | "gpu" : " 17 GB", | 20260507 23:55:53.536 | INFO | "huge" : "", | 20260507 23:55:53.536 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:53.536 | INFO | "malloc" : " 14 GB", | 20260507 23:55:53.536 | INFO | "pinned" : " 29 GB" | 20260507 23:55:53.536 | INFO | } | 20260507 23:55:53.536 | INFO | Perf report for the last 5.14162 seconds: | 20260507 23:55:53.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:53.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:53.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:53.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:53.536 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260507 23:55:53.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:53.536 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:55:53.536 | INFO | PRE_HQ : 65.8%, 1.7, 56.8ms, 0 | 20260507 23:55:53.537 | INFO | BAZWRITER : 68.9%, 3.8, 135.2ms, 0 | 20260507 23:55:53.537 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260507 23:55:53.537 | INFO | Uploaded 36GiB to GPU in 2.4953s (14.4271GiB/s) | 20260507 23:55:53.537 | INFO | Downloaded 38GiB to GPU in 2.6931s (14.1102GiB/s) | 20260507 23:55:58.675 | INFO | Analyzing chunk frames = [3609088,3609600) | 20260507 23:55:58.675 | INFO | MemInfo: { | 20260507 23:55:58.675 | INFO | "gpu" : " 17 GB", | 20260507 23:55:58.675 | INFO | "huge" : "", | 20260507 23:55:58.675 | INFO | "huge_shared" : " 29 GB", | 20260507 23:55:58.675 | INFO | "malloc" : " 14 GB", | 20260507 23:55:58.675 | INFO | "pinned" : " 29 GB" | 20260507 23:55:58.675 | INFO | } | 20260507 23:55:58.675 | INFO | Perf report for the last 5.13898 seconds: | 20260507 23:55:58.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:55:58.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.675 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260507 23:55:58.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.676 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:55:58.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:55:58.676 | INFO | Uploaded 36GiB to GPU in 2.57019s (14.0067GiB/s) | 20260507 23:55:58.676 | INFO | Downloaded 34GiB to GPU in 2.48562s (13.6787GiB/s) | 20260507 23:56:03.817 | INFO | Analyzing chunk frames = [3609600,3610112) | 20260507 23:56:03.817 | INFO | MemInfo: { | 20260507 23:56:03.817 | INFO | "gpu" : " 17 GB", | 20260507 23:56:03.817 | INFO | "huge" : "", | 20260507 23:56:03.817 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:03.817 | INFO | "malloc" : " 14 GB", | 20260507 23:56:03.817 | INFO | "pinned" : " 29 GB" | 20260507 23:56:03.817 | INFO | } | 20260507 23:56:03.817 | INFO | Perf report for the last 5.14186 seconds: | 20260507 23:56:03.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:03.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.817 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260507 23:56:03.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.817 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:56:03.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:03.818 | INFO | Uploaded 36GiB to GPU in 2.5305s (14.2264GiB/s) | 20260507 23:56:03.818 | INFO | Downloaded 34GiB to GPU in 2.4823s (13.697GiB/s) | 20260507 23:56:08.956 | INFO | Analyzing chunk frames = [3610112,3610624) | 20260507 23:56:08.956 | INFO | MemInfo: { | 20260507 23:56:08.956 | INFO | "gpu" : " 17 GB", | 20260507 23:56:08.956 | INFO | "huge" : "", | 20260507 23:56:08.956 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:08.956 | INFO | "malloc" : " 14 GB", | 20260507 23:56:08.956 | INFO | "pinned" : " 29 GB" | 20260507 23:56:08.956 | INFO | } | 20260507 23:56:08.956 | INFO | Perf report for the last 5.13905 seconds: | 20260507 23:56:08.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:08.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.956 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260507 23:56:08.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.956 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:56:08.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:08.957 | INFO | Uploaded 36GiB to GPU in 2.52319s (14.2676GiB/s) | 20260507 23:56:08.957 | INFO | Downloaded 34GiB to GPU in 2.48211s (13.698GiB/s) | 20260507 23:56:14.095 | INFO | Analyzing chunk frames = [3610624,3611136) | 20260507 23:56:14.095 | INFO | MemInfo: { | 20260507 23:56:14.095 | INFO | "gpu" : " 17 GB", | 20260507 23:56:14.095 | INFO | "huge" : "", | 20260507 23:56:14.095 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:14.095 | INFO | "malloc" : " 14 GB", | 20260507 23:56:14.095 | INFO | "pinned" : " 29 GB" | 20260507 23:56:14.095 | INFO | } | 20260507 23:56:14.095 | INFO | Perf report for the last 5.13897 seconds: | 20260507 23:56:14.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:14.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.095 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260507 23:56:14.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.095 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 23:56:14.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:14.096 | INFO | Uploaded 36GiB to GPU in 2.31767s (15.5328GiB/s) | 20260507 23:56:14.096 | INFO | Downloaded 34GiB to GPU in 2.49221s (13.6425GiB/s) | 20260507 23:56:19.237 | INFO | Analyzing chunk frames = [3611136,3611648) | 20260507 23:56:19.237 | INFO | MemInfo: { | 20260507 23:56:19.237 | INFO | "gpu" : " 17 GB", | 20260507 23:56:19.237 | INFO | "huge" : "", | 20260507 23:56:19.237 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:19.237 | INFO | "malloc" : " 14 GB", | 20260507 23:56:19.237 | INFO | "pinned" : " 29 GB" | 20260507 23:56:19.237 | INFO | } | 20260507 23:56:19.237 | INFO | Perf report for the last 5.142 seconds: | 20260507 23:56:19.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:19.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.237 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:56:19.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.237 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 23:56:19.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:19.238 | INFO | Uploaded 36GiB to GPU in 2.42086s (14.8707GiB/s) | 20260507 23:56:19.238 | INFO | Downloaded 34GiB to GPU in 2.39129s (14.2183GiB/s) | 20260507 23:56:24.375 | INFO | Analyzing chunk frames = [3611648,3612160) | 20260507 23:56:24.375 | INFO | MemInfo: { | 20260507 23:56:24.375 | INFO | "gpu" : " 17 GB", | 20260507 23:56:24.375 | INFO | "huge" : "", | 20260507 23:56:24.375 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:24.375 | INFO | "malloc" : " 14 GB", | 20260507 23:56:24.375 | INFO | "pinned" : " 29 GB" | 20260507 23:56:24.375 | INFO | } | 20260507 23:56:24.375 | INFO | Perf report for the last 5.13758 seconds: | 20260507 23:56:24.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:24.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.375 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260507 23:56:24.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.375 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:56:24.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:24.376 | INFO | Uploaded 36GiB to GPU in 2.55185s (14.1074GiB/s) | 20260507 23:56:24.376 | INFO | Downloaded 34GiB to GPU in 2.44513s (13.9052GiB/s) | 20260507 23:56:29.512 | INFO | Analyzing chunk frames = [3612160,3612672) | 20260507 23:56:29.512 | INFO | MemInfo: { | 20260507 23:56:29.512 | INFO | "gpu" : " 17 GB", | 20260507 23:56:29.512 | INFO | "huge" : "", | 20260507 23:56:29.512 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:29.512 | INFO | "malloc" : " 14 GB", | 20260507 23:56:29.512 | INFO | "pinned" : " 29 GB" | 20260507 23:56:29.513 | INFO | } | 20260507 23:56:29.513 | INFO | Perf report for the last 5.13767 seconds: | 20260507 23:56:29.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:29.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:56:29.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:56:29.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:29.513 | INFO | Uploaded 36GiB to GPU in 2.54518s (14.1444GiB/s) | 20260507 23:56:29.513 | INFO | Downloaded 34GiB to GPU in 2.44228s (13.9214GiB/s) | 20260507 23:56:29.513 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:56:29.513 | INFO | Total mem usage: 16.7 GiB | 20260507 23:56:29.513 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:56:29.513 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -7.07%) | 20260507 23:56:29.513 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:56:29.513 | INFO | High Water Breakdown: | 20260507 23:56:29.513 | INFO | 60.8% : 9.47 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:56:29.513 | INFO | 39.1% : 6.09 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:56:29.513 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:56:29.513 | INFO | ------------------------------------------ | 20260507 23:56:29.513 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:56:29.513 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:56:29.513 | INFO | Total mem usage: 26.5 GiB | 20260507 23:56:29.513 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:56:29.513 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 23:56:29.513 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 23:56:29.513 | INFO | High Water Breakdown: | 20260507 23:56:29.513 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:56:29.513 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:56:29.513 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:56:29.513 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:56:29.513 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:56:29.513 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:56:29.514 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:56:29.514 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:56:29.514 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:56:29.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:56:29.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:56:29.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:56:29.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:56:29.514 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:56:29.514 | INFO | ------------------------------------------ | 20260507 23:56:29.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:56:29.514 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:56:29.514 | INFO | Total mem usage: 27.8 GiB | 20260507 23:56:29.514 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:56:29.514 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:56:29.514 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:56:29.514 | INFO | High Water Breakdown: | 20260507 23:56:29.514 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:56:29.514 | INFO | ------------------------------------------ | 20260507 23:56:29.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:56:29.514 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:56:29.514 | INFO | Total mem usage: 15.7 GiB | 20260507 23:56:29.514 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:56:29.514 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:56:29.514 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:56:29.514 | INFO | High Water Breakdown: | 20260507 23:56:29.514 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:56:29.514 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:56:29.514 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:56:29.514 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:56:29.514 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:56:29.514 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:56:29.514 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:56:29.514 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:56:29.514 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:56:29.514 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:56:29.514 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:56:29.514 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:56:29.514 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:56:29.514 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:56:29.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:56:29.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:56:29.514 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:56:29.514 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:56:29.514 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:56:29.514 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:56:29.514 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:56:29.514 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:56:29.514 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:56:29.514 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:56:29.514 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:56:29.514 | INFO | ------------------------------------------ | 20260507 23:56:29.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:56:29.514 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:56:29.514 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:56:29.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:56:29.514 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.63e+01, 1.41e+01 / 1.53e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.17e-02 / 7.18e-03 | 20260507 23:56:29.514 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.45e+00, 5.79e-01 / 2.05e-01 | 20260507 23:56:29.514 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.05e-01 | 20260507 23:56:29.514 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.63e+01, 4.17e+00 / 1.82e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.55e+00 / 1.58e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.46e+00 / 2.71e+00, 4.28e-01 / 2.49e-01 | 20260507 23:56:29.514 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.30e+00, 2.77e-01 / 1.43e-01 | 20260507 23:56:29.514 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.73e-01, 5.56e-02 / 2.89e-02 | 20260507 23:56:29.514 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:56:29.514 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:56:29.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:56:29.514 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 2.18%, LockUpload -- 2.32e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 15.53%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 23:56:29.514 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 23:56:29.610 | INFO | Starting a round of Baz IO | 20260507 23:56:33.074 | INFO | Populating RTMetrics report | 20260507 23:56:33.075 | INFO | RTMetrics report populated | 20260507 23:56:33.075 | INFO | Opening tmp RTMetrics json file | 20260507 23:56:33.233 | INFO | Serializing RTMetrics to json | 20260507 23:56:33.283 | INFO | Serialization done, swapping tmp file now... | 20260507 23:56:33.310 | INFO | RTMetrics done | 20260507 23:56:33.337 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.727 seconds | 20260507 23:56:33.337 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.40% / 0.00% / 57.60% | 20260507 23:56:33.337 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.10024% / 0% / 6.75519e-05% | 20260507 23:56:33.337 | INFO | Baz IO overall: | 20260507 23:56:33.337 | INFO | BytesWritten : 1643422968572 | 20260507 23:56:33.337 | INFO | NumEvents : 795641841170 | 20260507 23:56:34.650 | INFO | Analyzing chunk frames = [3612672,3613184) | 20260507 23:56:34.650 | INFO | MemInfo: { | 20260507 23:56:34.650 | INFO | "gpu" : " 17 GB", | 20260507 23:56:34.650 | INFO | "huge" : "", | 20260507 23:56:34.650 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:34.650 | INFO | "malloc" : " 14 GB", | 20260507 23:56:34.650 | INFO | "pinned" : " 29 GB" | 20260507 23:56:34.650 | INFO | } | 20260507 23:56:34.650 | INFO | Perf report for the last 5.13758 seconds: | 20260507 23:56:34.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:34.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:34.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:34.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:34.650 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260507 23:56:34.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:34.650 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 23:56:34.650 | INFO | PRE_HQ : 65.7%, 1.7, 56.8ms, 0 | 20260507 23:56:34.650 | INFO | BAZWRITER : 72.0%, 4.1, 153.4ms, 0 | 20260507 23:56:34.650 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260507 23:56:34.651 | INFO | Uploaded 36GiB to GPU in 2.59791s (13.8573GiB/s) | 20260507 23:56:34.651 | INFO | Downloaded 38GiB to GPU in 2.68828s (14.1355GiB/s) | 20260507 23:56:39.793 | INFO | Analyzing chunk frames = [3613184,3613696) | 20260507 23:56:39.793 | INFO | MemInfo: { | 20260507 23:56:39.793 | INFO | "gpu" : " 17 GB", | 20260507 23:56:39.793 | INFO | "huge" : "", | 20260507 23:56:39.793 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:39.793 | INFO | "malloc" : " 14 GB", | 20260507 23:56:39.793 | INFO | "pinned" : " 29 GB" | 20260507 23:56:39.793 | INFO | } | 20260507 23:56:39.793 | INFO | Perf report for the last 5.1426 seconds: | 20260507 23:56:39.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:39.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260507 23:56:39.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 23:56:39.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:39.793 | INFO | Uploaded 36GiB to GPU in 2.58839s (13.9083GiB/s) | 20260507 23:56:39.794 | INFO | Downloaded 34GiB to GPU in 2.47635s (13.7299GiB/s) | 20260507 23:56:44.934 | INFO | Analyzing chunk frames = [3613696,3614208) | 20260507 23:56:44.934 | INFO | MemInfo: { | 20260507 23:56:44.934 | INFO | "gpu" : " 17 GB", | 20260507 23:56:44.934 | INFO | "huge" : "", | 20260507 23:56:44.934 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:44.934 | INFO | "malloc" : " 14 GB", | 20260507 23:56:44.934 | INFO | "pinned" : " 29 GB" | 20260507 23:56:44.934 | INFO | } | 20260507 23:56:44.934 | INFO | Perf report for the last 5.14147 seconds: | 20260507 23:56:44.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:44.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.934 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260507 23:56:44.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.934 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:56:44.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:44.935 | INFO | Uploaded 36GiB to GPU in 2.53385s (14.2076GiB/s) | 20260507 23:56:44.935 | INFO | Downloaded 34GiB to GPU in 2.48383s (13.6885GiB/s) | 20260507 23:56:50.070 | INFO | Analyzing chunk frames = [3614208,3614720) | 20260507 23:56:50.070 | INFO | MemInfo: { | 20260507 23:56:50.070 | INFO | "gpu" : " 17 GB", | 20260507 23:56:50.070 | INFO | "huge" : "", | 20260507 23:56:50.070 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:50.071 | INFO | "malloc" : " 14 GB", | 20260507 23:56:50.071 | INFO | "pinned" : " 29 GB" | 20260507 23:56:50.071 | INFO | } | 20260507 23:56:50.071 | INFO | Perf report for the last 5.13673 seconds: | 20260507 23:56:50.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:50.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.071 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 23:56:50.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.071 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 23:56:50.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:50.072 | INFO | Uploaded 36GiB to GPU in 2.52766s (14.2424GiB/s) | 20260507 23:56:50.072 | INFO | Downloaded 34GiB to GPU in 2.4722s (13.7529GiB/s) | 20260507 23:56:55.203 | INFO | Analyzing chunk frames = [3614720,3615232) | 20260507 23:56:55.203 | INFO | MemInfo: { | 20260507 23:56:55.203 | INFO | "gpu" : " 17 GB", | 20260507 23:56:55.203 | INFO | "huge" : "", | 20260507 23:56:55.203 | INFO | "huge_shared" : " 29 GB", | 20260507 23:56:55.203 | INFO | "malloc" : " 14 GB", | 20260507 23:56:55.203 | INFO | "pinned" : " 29 GB" | 20260507 23:56:55.203 | INFO | } | 20260507 23:56:55.203 | INFO | Perf report for the last 5.13241 seconds: | 20260507 23:56:55.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:56:55.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.203 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260507 23:56:55.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.204 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260507 23:56:55.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:56:55.204 | INFO | Uploaded 36GiB to GPU in 2.38908s (15.0685GiB/s) | 20260507 23:56:55.204 | INFO | Downloaded 34GiB to GPU in 2.51639s (13.5114GiB/s) | 20260507 23:57:00.347 | INFO | Analyzing chunk frames = [3615232,3615744) | 20260507 23:57:00.347 | INFO | MemInfo: { | 20260507 23:57:00.347 | INFO | "gpu" : " 17 GB", | 20260507 23:57:00.347 | INFO | "huge" : "", | 20260507 23:57:00.347 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:00.347 | INFO | "malloc" : " 14 GB", | 20260507 23:57:00.347 | INFO | "pinned" : " 29 GB" | 20260507 23:57:00.347 | INFO | } | 20260507 23:57:00.348 | INFO | Perf report for the last 5.14411 seconds: | 20260507 23:57:00.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:00.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | ANALYSIS : 66.7%, 3.0, 104.4ms, 0 | 20260507 23:57:00.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:57:00.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:00.348 | INFO | Uploaded 36GiB to GPU in 2.50429s (14.3753GiB/s) | 20260507 23:57:00.348 | INFO | Downloaded 34GiB to GPU in 2.41922s (14.0541GiB/s) | 20260507 23:57:05.491 | INFO | Analyzing chunk frames = [3615744,3616256) | 20260507 23:57:05.491 | INFO | MemInfo: { | 20260507 23:57:05.491 | INFO | "gpu" : " 17 GB", | 20260507 23:57:05.491 | INFO | "huge" : "", | 20260507 23:57:05.491 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:05.491 | INFO | "malloc" : " 14 GB", | 20260507 23:57:05.491 | INFO | "pinned" : " 29 GB" | 20260507 23:57:05.491 | INFO | } | 20260507 23:57:05.491 | INFO | Perf report for the last 5.14309 seconds: | 20260507 23:57:05.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:05.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260507 23:57:05.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:57:05.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:05.491 | INFO | Uploaded 36GiB to GPU in 2.53707s (14.1896GiB/s) | 20260507 23:57:05.491 | INFO | Downloaded 34GiB to GPU in 2.44309s (13.9168GiB/s) | 20260507 23:57:10.633 | INFO | Analyzing chunk frames = [3616256,3616768) | 20260507 23:57:10.633 | INFO | MemInfo: { | 20260507 23:57:10.633 | INFO | "gpu" : " 17 GB", | 20260507 23:57:10.633 | INFO | "huge" : "", | 20260507 23:57:10.633 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:10.633 | INFO | "malloc" : " 14 GB", | 20260507 23:57:10.633 | INFO | "pinned" : " 29 GB" | 20260507 23:57:10.633 | INFO | } | 20260507 23:57:10.633 | INFO | Perf report for the last 5.14213 seconds: | 20260507 23:57:10.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:10.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260507 23:57:10.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 23:57:10.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:10.633 | INFO | Uploaded 36GiB to GPU in 2.54236s (14.1601GiB/s) | 20260507 23:57:10.633 | INFO | Downloaded 34GiB to GPU in 2.44439s (13.9094GiB/s) | 20260507 23:57:10.738 | INFO | Starting a round of Baz IO | 20260507 23:57:14.180 | INFO | Populating RTMetrics report | 20260507 23:57:14.181 | INFO | RTMetrics report populated | 20260507 23:57:14.181 | INFO | Opening tmp RTMetrics json file | 20260507 23:57:14.392 | INFO | Serializing RTMetrics to json | 20260507 23:57:14.497 | INFO | Serialization done, swapping tmp file now... | 20260507 23:57:14.500 | INFO | RTMetrics done | 20260507 23:57:14.527 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 3.789 seconds | 20260507 23:57:14.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.39% / 0.00% / 57.60% | 20260507 23:57:14.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09881% / 0% / 5.5631e-05% | 20260507 23:57:14.527 | INFO | Baz IO overall: | 20260507 23:57:14.528 | INFO | BytesWritten : 1645250506681 | 20260507 23:57:14.528 | INFO | NumEvents : 796521451177 | 20260507 23:57:15.767 | INFO | Analyzing chunk frames = [3616768,3617280) | 20260507 23:57:15.767 | INFO | MemInfo: { | 20260507 23:57:15.767 | INFO | "gpu" : " 17 GB", | 20260507 23:57:15.767 | INFO | "huge" : "", | 20260507 23:57:15.767 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:15.767 | INFO | "malloc" : " 14 GB", | 20260507 23:57:15.767 | INFO | "pinned" : " 29 GB" | 20260507 23:57:15.767 | INFO | } | 20260507 23:57:15.767 | INFO | Perf report for the last 5.13429 seconds: | 20260507 23:57:15.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:15.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:15.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:15.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:15.767 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260507 23:57:15.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:15.767 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260507 23:57:15.767 | INFO | PRE_HQ : 65.8%, 1.7, 57.0ms, 0 | 20260507 23:57:15.767 | INFO | BAZWRITER : 73.8%, 4.1, 156.8ms, 0 | 20260507 23:57:15.767 | INFO | BAZ_SAMPLING : 6.4%, 1.0, 3.3ms, 0 | 20260507 23:57:15.767 | INFO | Uploaded 36GiB to GPU in 2.58626s (13.9197GiB/s) | 20260507 23:57:15.768 | INFO | Downloaded 38GiB to GPU in 2.69594s (14.0953GiB/s) | 20260507 23:57:20.906 | INFO | Analyzing chunk frames = [3617280,3617792) | 20260507 23:57:20.906 | INFO | MemInfo: { | 20260507 23:57:20.906 | INFO | "gpu" : " 17 GB", | 20260507 23:57:20.906 | INFO | "huge" : "", | 20260507 23:57:20.906 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:20.906 | INFO | "malloc" : " 14 GB", | 20260507 23:57:20.906 | INFO | "pinned" : " 29 GB" | 20260507 23:57:20.906 | INFO | } | 20260507 23:57:20.907 | INFO | Perf report for the last 5.13941 seconds: | 20260507 23:57:20.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:20.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260507 23:57:20.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 23:57:20.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:20.907 | INFO | Uploaded 36GiB to GPU in 2.51071s (14.3386GiB/s) | 20260507 23:57:20.907 | INFO | Downloaded 34GiB to GPU in 2.42863s (13.9997GiB/s) | 20260507 23:57:26.049 | INFO | Analyzing chunk frames = [3617792,3618304) | 20260507 23:57:26.049 | INFO | MemInfo: { | 20260507 23:57:26.049 | INFO | "gpu" : " 17 GB", | 20260507 23:57:26.049 | INFO | "huge" : "", | 20260507 23:57:26.049 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:26.049 | INFO | "malloc" : " 14 GB", | 20260507 23:57:26.049 | INFO | "pinned" : " 29 GB" | 20260507 23:57:26.049 | INFO | } | 20260507 23:57:26.049 | INFO | Perf report for the last 5.14268 seconds: | 20260507 23:57:26.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:26.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.049 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260507 23:57:26.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.049 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:57:26.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:26.050 | INFO | Uploaded 36GiB to GPU in 2.51237s (14.3291GiB/s) | 20260507 23:57:26.050 | INFO | Downloaded 34GiB to GPU in 2.48368s (13.6894GiB/s) | 20260507 23:57:31.187 | INFO | Analyzing chunk frames = [3618304,3618816) | 20260507 23:57:31.187 | INFO | MemInfo: { | 20260507 23:57:31.187 | INFO | "gpu" : " 17 GB", | 20260507 23:57:31.187 | INFO | "huge" : "", | 20260507 23:57:31.187 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:31.187 | INFO | "malloc" : " 14 GB", | 20260507 23:57:31.187 | INFO | "pinned" : " 29 GB" | 20260507 23:57:31.187 | INFO | } | 20260507 23:57:31.187 | INFO | Perf report for the last 5.13816 seconds: | 20260507 23:57:31.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:31.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.187 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260507 23:57:31.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.188 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260507 23:57:31.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:31.188 | INFO | Uploaded 36GiB to GPU in 2.50812s (14.3534GiB/s) | 20260507 23:57:31.188 | INFO | Downloaded 34GiB to GPU in 2.49863s (13.6075GiB/s) | 20260507 23:57:36.329 | INFO | Analyzing chunk frames = [3618816,3619328) | 20260507 23:57:36.329 | INFO | MemInfo: { | 20260507 23:57:36.329 | INFO | "gpu" : " 17 GB", | 20260507 23:57:36.329 | INFO | "huge" : "", | 20260507 23:57:36.329 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:36.329 | INFO | "malloc" : " 14 GB", | 20260507 23:57:36.329 | INFO | "pinned" : " 29 GB" | 20260507 23:57:36.329 | INFO | } | 20260507 23:57:36.329 | INFO | Perf report for the last 5.14163 seconds: | 20260507 23:57:36.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:36.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260507 23:57:36.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 23:57:36.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:36.329 | INFO | Uploaded 36GiB to GPU in 2.36249s (15.2382GiB/s) | 20260507 23:57:36.329 | INFO | Downloaded 34GiB to GPU in 2.50154s (13.5916GiB/s) | 20260507 23:57:41.462 | INFO | Analyzing chunk frames = [3619328,3619840) | 20260507 23:57:41.462 | INFO | MemInfo: { | 20260507 23:57:41.462 | INFO | "gpu" : " 17 GB", | 20260507 23:57:41.462 | INFO | "huge" : "", | 20260507 23:57:41.462 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:41.462 | INFO | "malloc" : " 14 GB", | 20260507 23:57:41.465 | INFO | "pinned" : " 29 GB" | 20260507 23:57:41.465 | INFO | } | 20260507 23:57:41.465 | INFO | Perf report for the last 5.13605 seconds: | 20260507 23:57:41.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:41.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:57:41.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:57:41.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:41.465 | INFO | Uploaded 36GiB to GPU in 2.39118s (15.0553GiB/s) | 20260507 23:57:41.465 | INFO | Downloaded 34GiB to GPU in 2.41077s (14.1034GiB/s) | 20260507 23:57:46.602 | INFO | Analyzing chunk frames = [3619840,3620352) | 20260507 23:57:46.602 | INFO | MemInfo: { | 20260507 23:57:46.602 | INFO | "gpu" : " 17 GB", | 20260507 23:57:46.602 | INFO | "huge" : "", | 20260507 23:57:46.602 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:46.602 | INFO | "malloc" : " 14 GB", | 20260507 23:57:46.602 | INFO | "pinned" : " 29 GB" | 20260507 23:57:46.602 | INFO | } | 20260507 23:57:46.602 | INFO | Perf report for the last 5.1369 seconds: | 20260507 23:57:46.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:46.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260507 23:57:46.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 23:57:46.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:46.602 | INFO | Uploaded 36GiB to GPU in 2.52176s (14.2758GiB/s) | 20260507 23:57:46.602 | INFO | Downloaded 34GiB to GPU in 2.46058s (13.8179GiB/s) | 20260507 23:57:51.744 | INFO | Analyzing chunk frames = [3620352,3620864) | 20260507 23:57:51.744 | INFO | MemInfo: { | 20260507 23:57:51.744 | INFO | "gpu" : " 17 GB", | 20260507 23:57:51.744 | INFO | "huge" : "", | 20260507 23:57:51.744 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:51.744 | INFO | "malloc" : " 14 GB", | 20260507 23:57:51.744 | INFO | "pinned" : " 29 GB" | 20260507 23:57:51.744 | INFO | } | 20260507 23:57:51.745 | INFO | Perf report for the last 5.14253 seconds: | 20260507 23:57:51.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:51.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260507 23:57:51.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:57:51.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:51.745 | INFO | Uploaded 36GiB to GPU in 2.49655s (14.4199GiB/s) | 20260507 23:57:51.745 | INFO | Downloaded 34GiB to GPU in 2.45153s (13.8689GiB/s) | 20260507 23:57:51.745 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:57:51.745 | INFO | Total mem usage: 16.7 GiB | 20260507 23:57:51.745 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:57:51.745 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.56%) | 20260507 23:57:51.745 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:57:51.745 | INFO | High Water Breakdown: | 20260507 23:57:51.745 | INFO | 60.6% : 9.31 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:57:51.745 | INFO | 39.3% : 6.04 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:57:51.745 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:57:51.745 | INFO | ------------------------------------------ | 20260507 23:57:51.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:57:51.745 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:57:51.745 | INFO | Total mem usage: 26.5 GiB | 20260507 23:57:51.745 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:57:51.745 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:57:51.745 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:57:51.746 | INFO | High Water Breakdown: | 20260507 23:57:51.746 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:57:51.746 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:57:51.746 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:57:51.746 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:57:51.746 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:57:51.746 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:57:51.746 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:57:51.746 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:57:51.746 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:57:51.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:57:51.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:57:51.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:57:51.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:57:51.746 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:57:51.746 | INFO | ------------------------------------------ | 20260507 23:57:51.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:57:51.746 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:57:51.746 | INFO | Total mem usage: 27.8 GiB | 20260507 23:57:51.746 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:57:51.746 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:57:51.746 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:57:51.746 | INFO | High Water Breakdown: | 20260507 23:57:51.746 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:57:51.746 | INFO | ------------------------------------------ | 20260507 23:57:51.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:57:51.746 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:57:51.746 | INFO | Total mem usage: 15.7 GiB | 20260507 23:57:51.746 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:57:51.746 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:57:51.746 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:57:51.746 | INFO | High Water Breakdown: | 20260507 23:57:51.746 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:57:51.746 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:57:51.746 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:57:51.746 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:57:51.746 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:57:51.746 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:57:51.746 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:57:51.746 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:57:51.746 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:57:51.746 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:57:51.746 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:57:51.746 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:57:51.746 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:57:51.746 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:57:51.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:57:51.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:57:51.746 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:57:51.746 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:57:51.746 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:57:51.746 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:57:51.746 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:57:51.746 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:57:51.746 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:57:51.746 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:57:51.746 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:57:51.746 | INFO | ------------------------------------------ | 20260507 23:57:51.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:57:51.746 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:57:51.746 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:57:51.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:57:51.746 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.40e+01 / 1.81e+00 | 20260507 23:57:51.746 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.15e-02 / 5.46e-03 | 20260507 23:57:51.746 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.47e+00, 5.78e-01 / 1.97e-01 | 20260507 23:57:51.746 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.06e-01 / 1.23e-01 | 20260507 23:57:51.746 | INFO | PerfReport: 16.11%, DME -- 4.22e+01 / 4.64e+01, 4.11e+00 / 2.09e+00 | 20260507 23:57:51.746 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.61e+00 / 1.56e+00 | 20260507 23:57:51.746 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.27e-01 / 2.11e-01 | 20260507 23:57:51.746 | INFO | PerfReport: 3.40%, Metrics -- 1.11e+00 / 1.34e+00, 2.76e-01 / 1.63e-01 | 20260507 23:57:51.746 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.84e-01, 5.49e-02 / 2.74e-02 | 20260507 23:57:51.746 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:57:51.746 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:57:51.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:57:51.746 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 24.43%, Upload -- 2.61e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 24.29%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 30.67%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 2.31%, LockUpload -- 2.46e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 2.65%, LockDownload -- 2.84e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260507 23:57:51.747 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 23:57:51.842 | INFO | Starting a round of Baz IO | 20260507 23:57:55.301 | INFO | Populating RTMetrics report | 20260507 23:57:55.302 | INFO | RTMetrics report populated | 20260507 23:57:55.302 | INFO | Opening tmp RTMetrics json file | 20260507 23:57:55.325 | INFO | Serializing RTMetrics to json | 20260507 23:57:55.326 | INFO | Serialization done, swapping tmp file now... | 20260507 23:57:55.327 | INFO | RTMetrics done | 20260507 23:57:55.398 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.556 seconds | 20260507 23:57:55.398 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.39% / 0.01% / 57.61% | 20260507 23:57:55.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09411% / 7.94729e-06% / 6.75519e-05% | 20260507 23:57:55.398 | INFO | Baz IO overall: | 20260507 23:57:55.398 | INFO | BytesWritten : 1647077189501 | 20260507 23:57:55.398 | INFO | NumEvents : 797400784421 | 20260507 23:57:56.883 | INFO | Analyzing chunk frames = [3620864,3621376) | 20260507 23:57:56.883 | INFO | MemInfo: { | 20260507 23:57:56.883 | INFO | "gpu" : " 17 GB", | 20260507 23:57:56.883 | INFO | "huge" : "", | 20260507 23:57:56.883 | INFO | "huge_shared" : " 29 GB", | 20260507 23:57:56.883 | INFO | "malloc" : " 14 GB", | 20260507 23:57:56.883 | INFO | "pinned" : " 29 GB" | 20260507 23:57:56.883 | INFO | } | 20260507 23:57:56.883 | INFO | Perf report for the last 5.13881 seconds: | 20260507 23:57:56.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:57:56.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:56.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:56.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:56.884 | INFO | ANALYSIS : 68.5%, 3.0, 106.3ms, 0 | 20260507 23:57:56.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:57:56.884 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 23:57:56.884 | INFO | PRE_HQ : 66.0%, 1.7, 56.9ms, 0 | 20260507 23:57:56.884 | INFO | BAZWRITER : 68.8%, 3.8, 133.1ms, 0 | 20260507 23:57:56.884 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 23:57:56.884 | INFO | Uploaded 36GiB to GPU in 2.55456s (14.0924GiB/s) | 20260507 23:57:56.884 | INFO | Downloaded 38GiB to GPU in 2.69093s (14.1215GiB/s) | 20260507 23:58:02.018 | INFO | Analyzing chunk frames = [3621376,3621888) | 20260507 23:58:02.018 | INFO | MemInfo: { | 20260507 23:58:02.018 | INFO | "gpu" : " 17 GB", | 20260507 23:58:02.018 | INFO | "huge" : "", | 20260507 23:58:02.018 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:02.018 | INFO | "malloc" : " 14 GB", | 20260507 23:58:02.018 | INFO | "pinned" : " 29 GB" | 20260507 23:58:02.019 | INFO | } | 20260507 23:58:02.019 | INFO | Perf report for the last 5.13543 seconds: | 20260507 23:58:02.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:02.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260507 23:58:02.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:58:02.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:02.019 | INFO | Uploaded 36GiB to GPU in 2.54839s (14.1266GiB/s) | 20260507 23:58:02.020 | INFO | Downloaded 34GiB to GPU in 2.41349s (14.0875GiB/s) | 20260507 23:58:07.158 | INFO | Analyzing chunk frames = [3621888,3622400) | 20260507 23:58:07.158 | INFO | MemInfo: { | 20260507 23:58:07.158 | INFO | "gpu" : " 17 GB", | 20260507 23:58:07.158 | INFO | "huge" : "", | 20260507 23:58:07.158 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:07.158 | INFO | "malloc" : " 14 GB", | 20260507 23:58:07.158 | INFO | "pinned" : " 29 GB" | 20260507 23:58:07.158 | INFO | } | 20260507 23:58:07.158 | INFO | Perf report for the last 5.13904 seconds: | 20260507 23:58:07.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:07.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260507 23:58:07.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 23:58:07.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:07.159 | INFO | Uploaded 36GiB to GPU in 2.53054s (14.2262GiB/s) | 20260507 23:58:07.159 | INFO | Downloaded 34GiB to GPU in 2.48245s (13.6962GiB/s) | 20260507 23:58:12.299 | INFO | Analyzing chunk frames = [3622400,3622912) | 20260507 23:58:12.299 | INFO | MemInfo: { | 20260507 23:58:12.299 | INFO | "gpu" : " 17 GB", | 20260507 23:58:12.299 | INFO | "huge" : "", | 20260507 23:58:12.299 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:12.299 | INFO | "malloc" : " 14 GB", | 20260507 23:58:12.299 | INFO | "pinned" : " 29 GB" | 20260507 23:58:12.299 | INFO | } | 20260507 23:58:12.299 | INFO | Perf report for the last 5.14119 seconds: | 20260507 23:58:12.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:12.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260507 23:58:12.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:58:12.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:12.299 | INFO | Uploaded 36GiB to GPU in 2.51132s (14.3351GiB/s) | 20260507 23:58:12.299 | INFO | Downloaded 34GiB to GPU in 2.40221s (14.1537GiB/s) | 20260507 23:58:17.435 | INFO | Analyzing chunk frames = [3622912,3623424) | 20260507 23:58:17.435 | INFO | MemInfo: { | 20260507 23:58:17.435 | INFO | "gpu" : " 17 GB", | 20260507 23:58:17.435 | INFO | "huge" : "", | 20260507 23:58:17.435 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:17.435 | INFO | "malloc" : " 14 GB", | 20260507 23:58:17.436 | INFO | "pinned" : " 29 GB" | 20260507 23:58:17.436 | INFO | } | 20260507 23:58:17.436 | INFO | Perf report for the last 5.13662 seconds: | 20260507 23:58:17.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:17.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260507 23:58:17.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 23:58:17.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:17.436 | INFO | Uploaded 36GiB to GPU in 2.42078s (14.8712GiB/s) | 20260507 23:58:17.436 | INFO | Downloaded 34GiB to GPU in 2.49282s (13.6392GiB/s) | 20260507 23:58:22.579 | INFO | Analyzing chunk frames = [3623424,3623936) | 20260507 23:58:22.579 | INFO | MemInfo: { | 20260507 23:58:22.579 | INFO | "gpu" : " 17 GB", | 20260507 23:58:22.579 | INFO | "huge" : "", | 20260507 23:58:22.579 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:22.579 | INFO | "malloc" : " 14 GB", | 20260507 23:58:22.579 | INFO | "pinned" : " 29 GB" | 20260507 23:58:22.579 | INFO | } | 20260507 23:58:22.579 | INFO | Perf report for the last 5.14345 seconds: | 20260507 23:58:22.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:22.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.579 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260507 23:58:22.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.579 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:58:22.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:22.580 | INFO | Uploaded 36GiB to GPU in 2.40152s (14.9905GiB/s) | 20260507 23:58:22.580 | INFO | Downloaded 34GiB to GPU in 2.41138s (14.0998GiB/s) | 20260507 23:58:27.718 | INFO | Analyzing chunk frames = [3623936,3624448) | 20260507 23:58:27.718 | INFO | MemInfo: { | 20260507 23:58:27.718 | INFO | "gpu" : " 17 GB", | 20260507 23:58:27.718 | INFO | "huge" : "", | 20260507 23:58:27.718 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:27.718 | INFO | "malloc" : " 14 GB", | 20260507 23:58:27.718 | INFO | "pinned" : " 29 GB" | 20260507 23:58:27.719 | INFO | } | 20260507 23:58:27.719 | INFO | Perf report for the last 5.13955 seconds: | 20260507 23:58:27.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:27.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:58:27.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 23:58:27.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:27.719 | INFO | Uploaded 36GiB to GPU in 2.55136s (14.1101GiB/s) | 20260507 23:58:27.719 | INFO | Downloaded 34GiB to GPU in 2.4701s (13.7646GiB/s) | 20260507 23:58:32.862 | INFO | Analyzing chunk frames = [3624448,3624960) | 20260507 23:58:32.862 | INFO | MemInfo: { | 20260507 23:58:32.862 | INFO | "gpu" : " 17 GB", | 20260507 23:58:32.862 | INFO | "huge" : "", | 20260507 23:58:32.862 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:32.862 | INFO | "malloc" : " 14 GB", | 20260507 23:58:32.862 | INFO | "pinned" : " 29 GB" | 20260507 23:58:32.862 | INFO | } | 20260507 23:58:32.862 | INFO | Perf report for the last 5.14359 seconds: | 20260507 23:58:32.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:32.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.862 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:58:32.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.862 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:58:32.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:32.863 | INFO | Uploaded 36GiB to GPU in 2.52516s (14.2565GiB/s) | 20260507 23:58:32.863 | INFO | Downloaded 34GiB to GPU in 2.45001s (13.8775GiB/s) | 20260507 23:58:32.960 | INFO | Starting a round of Baz IO | 20260507 23:58:36.424 | INFO | Populating RTMetrics report | 20260507 23:58:36.424 | INFO | RTMetrics report populated | 20260507 23:58:36.424 | INFO | Opening tmp RTMetrics json file | 20260507 23:58:36.426 | INFO | Serializing RTMetrics to json | 20260507 23:58:36.449 | INFO | Serialization done, swapping tmp file now... | 20260507 23:58:36.452 | INFO | RTMetrics done | 20260507 23:58:36.532 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 3.571 seconds | 20260507 23:58:36.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.38% / 0.00% / 57.61% | 20260507 23:58:36.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09549% / 1.19209e-05% / 5.16574e-05% | 20260507 23:58:36.532 | INFO | Baz IO overall: | 20260507 23:58:36.532 | INFO | BytesWritten : 1648904638715 | 20260507 23:58:36.532 | INFO | NumEvents : 798280541722 | 20260507 23:58:37.998 | INFO | Analyzing chunk frames = [3624960,3625472) | 20260507 23:58:37.998 | INFO | MemInfo: { | 20260507 23:58:37.998 | INFO | "gpu" : " 17 GB", | 20260507 23:58:37.998 | INFO | "huge" : "", | 20260507 23:58:37.998 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:37.998 | INFO | "malloc" : " 14 GB", | 20260507 23:58:37.998 | INFO | "pinned" : " 29 GB" | 20260507 23:58:37.998 | INFO | } | 20260507 23:58:37.998 | INFO | Perf report for the last 5.13594 seconds: | 20260507 23:58:37.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:37.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:37.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:37.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:37.998 | INFO | ANALYSIS : 68.7%, 3.0, 106.9ms, 0 | 20260507 23:58:37.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:37.998 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 23:58:37.998 | INFO | PRE_HQ : 66.3%, 1.7, 58.7ms, 0 | 20260507 23:58:37.998 | INFO | BAZWRITER : 69.1%, 3.8, 136.5ms, 0 | 20260507 23:58:37.998 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 23:58:37.999 | INFO | Uploaded 36GiB to GPU in 2.50899s (14.3484GiB/s) | 20260507 23:58:37.999 | INFO | Downloaded 38GiB to GPU in 2.68724s (14.1409GiB/s) | 20260507 23:58:43.135 | INFO | Analyzing chunk frames = [3625472,3625984) | 20260507 23:58:43.135 | INFO | MemInfo: { | 20260507 23:58:43.135 | INFO | "gpu" : " 17 GB", | 20260507 23:58:43.135 | INFO | "huge" : "", | 20260507 23:58:43.135 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:43.135 | INFO | "malloc" : " 14 GB", | 20260507 23:58:43.135 | INFO | "pinned" : " 29 GB" | 20260507 23:58:43.135 | INFO | } | 20260507 23:58:43.135 | INFO | Perf report for the last 5.13664 seconds: | 20260507 23:58:43.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:43.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260507 23:58:43.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:58:43.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:43.135 | INFO | Uploaded 36GiB to GPU in 2.49181s (14.4473GiB/s) | 20260507 23:58:43.135 | INFO | Downloaded 34GiB to GPU in 2.47842s (13.7184GiB/s) | 20260507 23:58:48.270 | INFO | Analyzing chunk frames = [3625984,3626496) | 20260507 23:58:48.270 | INFO | MemInfo: { | 20260507 23:58:48.270 | INFO | "gpu" : " 17 GB", | 20260507 23:58:48.270 | INFO | "huge" : "", | 20260507 23:58:48.270 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:48.270 | INFO | "malloc" : " 14 GB", | 20260507 23:58:48.270 | INFO | "pinned" : " 29 GB" | 20260507 23:58:48.270 | INFO | } | 20260507 23:58:48.271 | INFO | Perf report for the last 5.1358 seconds: | 20260507 23:58:48.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:48.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260507 23:58:48.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 23:58:48.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:48.271 | INFO | Uploaded 36GiB to GPU in 2.52835s (14.2385GiB/s) | 20260507 23:58:48.271 | INFO | Downloaded 34GiB to GPU in 2.47497s (13.7375GiB/s) | 20260507 23:58:53.417 | INFO | Analyzing chunk frames = [3626496,3627008) | 20260507 23:58:53.417 | INFO | MemInfo: { | 20260507 23:58:53.417 | INFO | "gpu" : " 17 GB", | 20260507 23:58:53.417 | INFO | "huge" : "", | 20260507 23:58:53.417 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:53.417 | INFO | "malloc" : " 14 GB", | 20260507 23:58:53.417 | INFO | "pinned" : " 29 GB" | 20260507 23:58:53.417 | INFO | } | 20260507 23:58:53.417 | INFO | Perf report for the last 5.14615 seconds: | 20260507 23:58:53.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:53.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260507 23:58:53.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 23:58:53.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:53.417 | INFO | Uploaded 36GiB to GPU in 2.49232s (14.4444GiB/s) | 20260507 23:58:53.418 | INFO | Downloaded 34GiB to GPU in 2.49659s (13.6186GiB/s) | 20260507 23:58:58.553 | INFO | Analyzing chunk frames = [3627008,3627520) | 20260507 23:58:58.553 | INFO | MemInfo: { | 20260507 23:58:58.553 | INFO | "gpu" : " 17 GB", | 20260507 23:58:58.553 | INFO | "huge" : "", | 20260507 23:58:58.553 | INFO | "huge_shared" : " 29 GB", | 20260507 23:58:58.553 | INFO | "malloc" : " 14 GB", | 20260507 23:58:58.553 | INFO | "pinned" : " 29 GB" | 20260507 23:58:58.553 | INFO | } | 20260507 23:58:58.553 | INFO | Perf report for the last 5.1363 seconds: | 20260507 23:58:58.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:58:58.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.553 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260507 23:58:58.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.553 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:58:58.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:58:58.554 | INFO | Uploaded 36GiB to GPU in 2.3555s (15.2834GiB/s) | 20260507 23:58:58.554 | INFO | Downloaded 34GiB to GPU in 2.49545s (13.6248GiB/s) | 20260507 23:59:03.691 | INFO | Analyzing chunk frames = [3627520,3628032) | 20260507 23:59:03.691 | INFO | MemInfo: { | 20260507 23:59:03.692 | INFO | "gpu" : " 17 GB", | 20260507 23:59:03.692 | INFO | "huge" : "", | 20260507 23:59:03.692 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:03.692 | INFO | "malloc" : " 14 GB", | 20260507 23:59:03.692 | INFO | "pinned" : " 29 GB" | 20260507 23:59:03.692 | INFO | } | 20260507 23:59:03.692 | INFO | Perf report for the last 5.13855 seconds: | 20260507 23:59:03.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:03.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260507 23:59:03.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 23:59:03.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:03.692 | INFO | Uploaded 36GiB to GPU in 2.42726s (14.8315GiB/s) | 20260507 23:59:03.692 | INFO | Downloaded 34GiB to GPU in 2.39215s (14.2132GiB/s) | 20260507 23:59:08.831 | INFO | Analyzing chunk frames = [3628032,3628544) | 20260507 23:59:08.831 | INFO | MemInfo: { | 20260507 23:59:08.831 | INFO | "gpu" : " 17 GB", | 20260507 23:59:08.831 | INFO | "huge" : "", | 20260507 23:59:08.831 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:08.831 | INFO | "malloc" : " 14 GB", | 20260507 23:59:08.831 | INFO | "pinned" : " 29 GB" | 20260507 23:59:08.831 | INFO | } | 20260507 23:59:08.831 | INFO | Perf report for the last 5.13917 seconds: | 20260507 23:59:08.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:08.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260507 23:59:08.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:59:08.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:08.831 | INFO | Uploaded 36GiB to GPU in 2.56408s (14.0401GiB/s) | 20260507 23:59:08.831 | INFO | Downloaded 34GiB to GPU in 2.45808s (13.8319GiB/s) | 20260507 23:59:13.970 | INFO | Analyzing chunk frames = [3628544,3629056) | 20260507 23:59:13.970 | INFO | MemInfo: { | 20260507 23:59:13.970 | INFO | "gpu" : " 17 GB", | 20260507 23:59:13.970 | INFO | "huge" : "", | 20260507 23:59:13.970 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:13.970 | INFO | "malloc" : " 14 GB", | 20260507 23:59:13.970 | INFO | "pinned" : " 29 GB" | 20260507 23:59:13.970 | INFO | } | 20260507 23:59:13.970 | INFO | Perf report for the last 5.13914 seconds: | 20260507 23:59:13.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:13.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260507 23:59:13.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 23:59:13.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:13.970 | INFO | Uploaded 36GiB to GPU in 2.51719s (14.3017GiB/s) | 20260507 23:59:13.970 | INFO | Downloaded 34GiB to GPU in 2.43892s (13.9406GiB/s) | 20260507 23:59:13.971 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 23:59:13.971 | INFO | Total mem usage: 16.7 GiB | 20260507 23:59:13.971 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260507 23:59:13.971 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.1%) | 20260507 23:59:13.971 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 23:59:13.971 | INFO | High Water Breakdown: | 20260507 23:59:13.971 | INFO | 59.6% : 8.65 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 23:59:13.971 | INFO | 40.2% : 5.83 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 23:59:13.971 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 23:59:13.971 | INFO | ------------------------------------------ | 20260507 23:59:13.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:59:13.971 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 23:59:13.971 | INFO | Total mem usage: 26.5 GiB | 20260507 23:59:13.971 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260507 23:59:13.971 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 23:59:13.971 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260507 23:59:13.971 | INFO | High Water Breakdown: | 20260507 23:59:13.971 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:59:13.971 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:59:13.971 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:59:13.971 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:59:13.971 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:59:13.971 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:59:13.971 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 23:59:13.971 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 23:59:13.971 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:59:13.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 23:59:13.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 23:59:13.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 23:59:13.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 23:59:13.971 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 23:59:13.971 | INFO | ------------------------------------------ | 20260507 23:59:13.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:59:13.971 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 23:59:13.971 | INFO | Total mem usage: 27.8 GiB | 20260507 23:59:13.971 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260507 23:59:13.971 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260507 23:59:13.971 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260507 23:59:13.971 | INFO | High Water Breakdown: | 20260507 23:59:13.971 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260507 23:59:13.971 | INFO | ------------------------------------------ | 20260507 23:59:13.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:59:13.971 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 23:59:13.971 | INFO | Total mem usage: 15.7 GiB | 20260507 23:59:13.971 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 23:59:13.971 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260507 23:59:13.971 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 23:59:13.971 | INFO | High Water Breakdown: | 20260507 23:59:13.971 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 23:59:13.971 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 23:59:13.971 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 23:59:13.971 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 23:59:13.971 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 23:59:13.971 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 23:59:13.971 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 23:59:13.971 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 23:59:13.971 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 23:59:13.971 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 23:59:13.971 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 23:59:13.971 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 23:59:13.971 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 23:59:13.971 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 23:59:13.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 23:59:13.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 23:59:13.971 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 23:59:13.971 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 23:59:13.971 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 23:59:13.971 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 23:59:13.972 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 23:59:13.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 23:59:13.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 23:59:13.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 23:59:13.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 23:59:13.972 | INFO | ------------------------------------------ | 20260507 23:59:13.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 23:59:13.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 23:59:13.972 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:59:13.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:59:13.972 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.42e+01 / 1.77e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.13e-02 / 9.27e-03 | 20260507 23:59:13.972 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.47e+00, 5.93e-01 / 2.20e-01 | 20260507 23:59:13.972 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.06e-01 / 1.11e-01 | 20260507 23:59:13.972 | INFO | PerfReport: 16.01%, DME -- 4.20e+01 / 4.61e+01, 4.12e+00 / 1.77e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.51e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.45e+00 / 2.74e+00, 4.29e-01 / 2.00e-01 | 20260507 23:59:13.972 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.34e+00, 2.80e-01 / 1.64e-01 | 20260507 23:59:13.972 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.80e-01, 5.53e-02 / 2.22e-02 | 20260507 23:59:13.972 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 23:59:13.972 | INFO | PerfReport: Gathered 1536 samples. | 20260507 23:59:13.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 23:59:13.972 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 24.36%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 30.76%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 2.20%, LockUpload -- 2.35e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 2.64%, LockDownload -- 2.82e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260507 23:59:13.972 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.15e-02 / 0.00e+00 | 20260507 23:59:14.068 | INFO | Starting a round of Baz IO | 20260507 23:59:17.532 | INFO | Populating RTMetrics report | 20260507 23:59:17.532 | INFO | RTMetrics report populated | 20260507 23:59:17.532 | INFO | Opening tmp RTMetrics json file | 20260507 23:59:17.534 | INFO | Serializing RTMetrics to json | 20260507 23:59:17.538 | INFO | Serialization done, swapping tmp file now... | 20260507 23:59:17.558 | INFO | RTMetrics done | 20260507 23:59:17.654 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.585 seconds | 20260507 23:59:17.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.38% / 0.01% / 57.62% | 20260507 23:59:17.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09207% / 0% / 8.34465e-05% | 20260507 23:59:17.654 | INFO | Baz IO overall: | 20260507 23:59:17.654 | INFO | BytesWritten : 1650731224528 | 20260507 23:59:17.654 | INFO | NumEvents : 799159756294 | 20260507 23:59:19.110 | INFO | Analyzing chunk frames = [3629056,3629568) | 20260507 23:59:19.110 | INFO | MemInfo: { | 20260507 23:59:19.110 | INFO | "gpu" : " 17 GB", | 20260507 23:59:19.110 | INFO | "huge" : "", | 20260507 23:59:19.110 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:19.110 | INFO | "malloc" : " 14 GB", | 20260507 23:59:19.110 | INFO | "pinned" : " 29 GB" | 20260507 23:59:19.110 | INFO | } | 20260507 23:59:19.110 | INFO | Perf report for the last 5.13984 seconds: | 20260507 23:59:19.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:19.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:19.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:19.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:19.110 | INFO | ANALYSIS : 68.6%, 3.0, 106.8ms, 0 | 20260507 23:59:19.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:19.110 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 23:59:19.110 | INFO | PRE_HQ : 65.9%, 1.7, 57.1ms, 0 | 20260507 23:59:19.110 | INFO | BAZWRITER : 69.4%, 3.8, 135.7ms, 0 | 20260507 23:59:19.110 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260507 23:59:19.110 | INFO | Uploaded 36GiB to GPU in 2.57085s (14.0031GiB/s) | 20260507 23:59:19.111 | INFO | Downloaded 38GiB to GPU in 2.68984s (14.1272GiB/s) | 20260507 23:59:24.250 | INFO | Analyzing chunk frames = [3629568,3630080) | 20260507 23:59:24.250 | INFO | MemInfo: { | 20260507 23:59:24.250 | INFO | "gpu" : " 17 GB", | 20260507 23:59:24.250 | INFO | "huge" : "", | 20260507 23:59:24.250 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:24.250 | INFO | "malloc" : " 14 GB", | 20260507 23:59:24.250 | INFO | "pinned" : " 29 GB" | 20260507 23:59:24.251 | INFO | } | 20260507 23:59:24.251 | INFO | Perf report for the last 5.14057 seconds: | 20260507 23:59:24.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:24.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260507 23:59:24.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 23:59:24.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:24.251 | INFO | Uploaded 36GiB to GPU in 2.5862s (13.92GiB/s) | 20260507 23:59:24.251 | INFO | Downloaded 34GiB to GPU in 2.47844s (13.7183GiB/s) | 20260507 23:59:29.387 | INFO | Analyzing chunk frames = [3630080,3630592) | 20260507 23:59:29.387 | INFO | MemInfo: { | 20260507 23:59:29.387 | INFO | "gpu" : " 17 GB", | 20260507 23:59:29.387 | INFO | "huge" : "", | 20260507 23:59:29.387 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:29.387 | INFO | "malloc" : " 14 GB", | 20260507 23:59:29.387 | INFO | "pinned" : " 29 GB" | 20260507 23:59:29.387 | INFO | } | 20260507 23:59:29.388 | INFO | Perf report for the last 5.13702 seconds: | 20260507 23:59:29.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:29.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260507 23:59:29.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 23:59:29.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:29.388 | INFO | Uploaded 36GiB to GPU in 2.52691s (14.2466GiB/s) | 20260507 23:59:29.388 | INFO | Downloaded 34GiB to GPU in 2.47366s (13.7448GiB/s) | 20260507 23:59:34.525 | INFO | Analyzing chunk frames = [3630592,3631104) | 20260507 23:59:34.525 | INFO | MemInfo: { | 20260507 23:59:34.525 | INFO | "gpu" : " 17 GB", | 20260507 23:59:34.525 | INFO | "huge" : "", | 20260507 23:59:34.525 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:34.525 | INFO | "malloc" : " 14 GB", | 20260507 23:59:34.525 | INFO | "pinned" : " 29 GB" | 20260507 23:59:34.525 | INFO | } | 20260507 23:59:34.525 | INFO | Perf report for the last 5.13784 seconds: | 20260507 23:59:34.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:34.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.526 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260507 23:59:34.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.526 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 23:59:34.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:34.526 | INFO | Uploaded 36GiB to GPU in 2.52024s (14.2844GiB/s) | 20260507 23:59:34.526 | INFO | Downloaded 34GiB to GPU in 2.48469s (13.6838GiB/s) | 20260507 23:59:39.667 | INFO | Analyzing chunk frames = [3631104,3631616) | 20260507 23:59:39.667 | INFO | MemInfo: { | 20260507 23:59:39.667 | INFO | "gpu" : " 17 GB", | 20260507 23:59:39.667 | INFO | "huge" : "", | 20260507 23:59:39.667 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:39.667 | INFO | "malloc" : " 14 GB", | 20260507 23:59:39.667 | INFO | "pinned" : " 29 GB" | 20260507 23:59:39.667 | INFO | } | 20260507 23:59:39.667 | INFO | Perf report for the last 5.14149 seconds: | 20260507 23:59:39.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:39.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.667 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260507 23:59:39.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.667 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 23:59:39.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:39.668 | INFO | Uploaded 36GiB to GPU in 2.31519s (15.5495GiB/s) | 20260507 23:59:39.668 | INFO | Downloaded 34GiB to GPU in 2.46718s (13.7809GiB/s) | 20260507 23:59:44.811 | INFO | Analyzing chunk frames = [3631616,3632128) | 20260507 23:59:44.811 | INFO | MemInfo: { | 20260507 23:59:44.811 | INFO | "gpu" : " 17 GB", | 20260507 23:59:44.811 | INFO | "huge" : "", | 20260507 23:59:44.811 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:44.811 | INFO | "malloc" : " 14 GB", | 20260507 23:59:44.811 | INFO | "pinned" : " 29 GB" | 20260507 23:59:44.811 | INFO | } | 20260507 23:59:44.812 | INFO | Perf report for the last 5.1447 seconds: | 20260507 23:59:44.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:44.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260507 23:59:44.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 23:59:44.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:44.812 | INFO | Uploaded 36GiB to GPU in 2.39119s (15.0553GiB/s) | 20260507 23:59:44.812 | INFO | Downloaded 34GiB to GPU in 2.39854s (14.1753GiB/s) | 20260507 23:59:49.951 | INFO | Analyzing chunk frames = [3632128,3632640) | 20260507 23:59:49.951 | INFO | MemInfo: { | 20260507 23:59:49.951 | INFO | "gpu" : " 17 GB", | 20260507 23:59:49.951 | INFO | "huge" : "", | 20260507 23:59:49.951 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:49.951 | INFO | "malloc" : " 14 GB", | 20260507 23:59:49.951 | INFO | "pinned" : " 29 GB" | 20260507 23:59:49.951 | INFO | } | 20260507 23:59:49.951 | INFO | Perf report for the last 5.13945 seconds: | 20260507 23:59:49.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:49.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.951 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260507 23:59:49.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.951 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 23:59:49.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:49.952 | INFO | Uploaded 36GiB to GPU in 2.53273s (14.2139GiB/s) | 20260507 23:59:49.952 | INFO | Downloaded 34GiB to GPU in 2.46175s (13.8113GiB/s) | 20260507 23:59:55.086 | INFO | Analyzing chunk frames = [3632640,3633152) | 20260507 23:59:55.086 | INFO | MemInfo: { | 20260507 23:59:55.086 | INFO | "gpu" : " 17 GB", | 20260507 23:59:55.086 | INFO | "huge" : "", | 20260507 23:59:55.086 | INFO | "huge_shared" : " 29 GB", | 20260507 23:59:55.086 | INFO | "malloc" : " 14 GB", | 20260507 23:59:55.086 | INFO | "pinned" : " 29 GB" | 20260507 23:59:55.086 | INFO | } | 20260507 23:59:55.086 | INFO | Perf report for the last 5.13492 seconds: | 20260507 23:59:55.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 23:59:55.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260507 23:59:55.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 23:59:55.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 23:59:55.086 | INFO | Uploaded 36GiB to GPU in 2.5133s (14.3238GiB/s) | 20260507 23:59:55.086 | INFO | Downloaded 34GiB to GPU in 2.44464s (13.908GiB/s) | 20260507 23:59:55.183 | INFO | Starting a round of Baz IO | 20260507 23:59:58.640 | INFO | Populating RTMetrics report | 20260507 23:59:58.641 | INFO | RTMetrics report populated | 20260507 23:59:58.641 | INFO | Opening tmp RTMetrics json file | 20260507 23:59:58.656 | INFO | Serializing RTMetrics to json | 20260507 23:59:58.666 | INFO | Serialization done, swapping tmp file now... | 20260507 23:59:58.667 | INFO | RTMetrics done | 20260507 23:59:58.749 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.566 seconds | 20260507 23:59:58.750 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.37% / 0.00% / 57.62% | 20260507 23:59:58.750 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09319% / 1.58946e-05% / 7.94729e-05% | 20260507 23:59:58.750 | INFO | Baz IO overall: | 20260507 23:59:58.750 | INFO | BytesWritten : 1652559457879 | 20260507 23:59:58.750 | INFO | NumEvents : 800040460834 | 20260508 00:00:00.224 | INFO | Analyzing chunk frames = [3633152,3633664) | 20260508 00:00:00.224 | INFO | MemInfo: { | 20260508 00:00:00.224 | INFO | "gpu" : " 17 GB", | 20260508 00:00:00.224 | INFO | "huge" : "", | 20260508 00:00:00.224 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:00.224 | INFO | "malloc" : " 14 GB", | 20260508 00:00:00.224 | INFO | "pinned" : " 29 GB" | 20260508 00:00:00.224 | INFO | } | 20260508 00:00:00.224 | INFO | Perf report for the last 5.13812 seconds: | 20260508 00:00:00.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:00.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:00.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:00.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:00.224 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260508 00:00:00.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:00.224 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260508 00:00:00.224 | INFO | PRE_HQ : 65.5%, 1.7, 56.5ms, 0 | 20260508 00:00:00.224 | INFO | BAZWRITER : 69.0%, 3.9, 139.8ms, 0 | 20260508 00:00:00.224 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 00:00:00.224 | INFO | Uploaded 36GiB to GPU in 2.51909s (14.2909GiB/s) | 20260508 00:00:00.224 | INFO | Downloaded 38GiB to GPU in 2.69121s (14.12GiB/s) | 20260508 00:00:05.364 | INFO | Analyzing chunk frames = [3633664,3634176) | 20260508 00:00:05.364 | INFO | MemInfo: { | 20260508 00:00:05.364 | INFO | "gpu" : " 17 GB", | 20260508 00:00:05.364 | INFO | "huge" : "", | 20260508 00:00:05.364 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:05.364 | INFO | "malloc" : " 14 GB", | 20260508 00:00:05.364 | INFO | "pinned" : " 29 GB" | 20260508 00:00:05.364 | INFO | } | 20260508 00:00:05.365 | INFO | Perf report for the last 5.1403 seconds: | 20260508 00:00:05.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:05.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 00:00:05.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:00:05.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:05.365 | INFO | Uploaded 36GiB to GPU in 2.56019s (14.0615GiB/s) | 20260508 00:00:05.365 | INFO | Downloaded 34GiB to GPU in 2.48814s (13.6648GiB/s) | 20260508 00:00:10.504 | INFO | Analyzing chunk frames = [3634176,3634688) | 20260508 00:00:10.504 | INFO | MemInfo: { | 20260508 00:00:10.504 | INFO | "gpu" : " 17 GB", | 20260508 00:00:10.504 | INFO | "huge" : "", | 20260508 00:00:10.504 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:10.504 | INFO | "malloc" : " 14 GB", | 20260508 00:00:10.504 | INFO | "pinned" : " 29 GB" | 20260508 00:00:10.504 | INFO | } | 20260508 00:00:10.504 | INFO | Perf report for the last 5.13949 seconds: | 20260508 00:00:10.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:10.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 00:00:10.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:00:10.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:10.504 | INFO | Uploaded 36GiB to GPU in 2.48077s (14.5116GiB/s) | 20260508 00:00:10.504 | INFO | Downloaded 34GiB to GPU in 2.48522s (13.6809GiB/s) | 20260508 00:00:15.643 | INFO | Analyzing chunk frames = [3634688,3635200) | 20260508 00:00:15.643 | INFO | MemInfo: { | 20260508 00:00:15.643 | INFO | "gpu" : " 17 GB", | 20260508 00:00:15.643 | INFO | "huge" : "", | 20260508 00:00:15.643 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:15.643 | INFO | "malloc" : " 14 GB", | 20260508 00:00:15.643 | INFO | "pinned" : " 29 GB" | 20260508 00:00:15.643 | INFO | } | 20260508 00:00:15.643 | INFO | Perf report for the last 5.13882 seconds: | 20260508 00:00:15.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:15.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 00:00:15.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:00:15.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:15.643 | INFO | Uploaded 36GiB to GPU in 2.57219s (13.9958GiB/s) | 20260508 00:00:15.643 | INFO | Downloaded 34GiB to GPU in 2.47831s (13.719GiB/s) | 20260508 00:00:20.779 | INFO | Analyzing chunk frames = [3635200,3635712) | 20260508 00:00:20.779 | INFO | MemInfo: { | 20260508 00:00:20.779 | INFO | "gpu" : " 17 GB", | 20260508 00:00:20.779 | INFO | "huge" : "", | 20260508 00:00:20.779 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:20.779 | INFO | "malloc" : " 14 GB", | 20260508 00:00:20.779 | INFO | "pinned" : " 29 GB" | 20260508 00:00:20.779 | INFO | } | 20260508 00:00:20.779 | INFO | Perf report for the last 5.13657 seconds: | 20260508 00:00:20.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:20.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.780 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:00:20.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.780 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260508 00:00:20.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:20.780 | INFO | Uploaded 36GiB to GPU in 2.38802s (15.0752GiB/s) | 20260508 00:00:20.780 | INFO | Downloaded 34GiB to GPU in 2.47544s (13.7349GiB/s) | 20260508 00:00:25.919 | INFO | Analyzing chunk frames = [3635712,3636224) | 20260508 00:00:25.919 | INFO | MemInfo: { | 20260508 00:00:25.919 | INFO | "gpu" : " 17 GB", | 20260508 00:00:25.919 | INFO | "huge" : "", | 20260508 00:00:25.919 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:25.919 | INFO | "malloc" : " 14 GB", | 20260508 00:00:25.919 | INFO | "pinned" : " 29 GB" | 20260508 00:00:25.919 | INFO | } | 20260508 00:00:25.919 | INFO | Perf report for the last 5.13941 seconds: | 20260508 00:00:25.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:25.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:00:25.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:00:25.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:25.919 | INFO | Uploaded 36GiB to GPU in 2.40175s (14.9891GiB/s) | 20260508 00:00:25.919 | INFO | Downloaded 34GiB to GPU in 2.39733s (14.1824GiB/s) | 20260508 00:00:31.063 | INFO | Analyzing chunk frames = [3636224,3636736) | 20260508 00:00:31.063 | INFO | MemInfo: { | 20260508 00:00:31.063 | INFO | "gpu" : " 17 GB", | 20260508 00:00:31.063 | INFO | "huge" : "", | 20260508 00:00:31.063 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:31.063 | INFO | "malloc" : " 14 GB", | 20260508 00:00:31.063 | INFO | "pinned" : " 29 GB" | 20260508 00:00:31.063 | INFO | } | 20260508 00:00:31.063 | INFO | Perf report for the last 5.14428 seconds: | 20260508 00:00:31.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:31.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 00:00:31.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:00:31.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:31.063 | INFO | Uploaded 36GiB to GPU in 2.50545s (14.3687GiB/s) | 20260508 00:00:31.064 | INFO | Downloaded 34GiB to GPU in 2.44908s (13.8828GiB/s) | 20260508 00:00:36.204 | INFO | Analyzing chunk frames = [3636736,3637248) | 20260508 00:00:36.204 | INFO | MemInfo: { | 20260508 00:00:36.204 | INFO | "gpu" : " 17 GB", | 20260508 00:00:36.204 | INFO | "huge" : "", | 20260508 00:00:36.204 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:36.204 | INFO | "malloc" : " 14 GB", | 20260508 00:00:36.204 | INFO | "pinned" : " 29 GB" | 20260508 00:00:36.204 | INFO | } | 20260508 00:00:36.204 | INFO | Perf report for the last 5.14109 seconds: | 20260508 00:00:36.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:36.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.204 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:00:36.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.204 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:00:36.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:36.205 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:00:36.205 | INFO | Total mem usage: 16.7 GiB | 20260508 00:00:36.205 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:00:36.205 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260508 00:00:36.205 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:00:36.205 | INFO | High Water Breakdown: | 20260508 00:00:36.205 | INFO | 59.8% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:00:36.205 | INFO | 40% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:00:36.205 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:00:36.205 | INFO | ------------------------------------------ | 20260508 00:00:36.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:00:36.205 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:00:36.205 | INFO | Total mem usage: 26.5 GiB | 20260508 00:00:36.205 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:00:36.205 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:00:36.205 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:00:36.205 | INFO | High Water Breakdown: | 20260508 00:00:36.205 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:00:36.205 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:00:36.205 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:00:36.205 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:00:36.205 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:00:36.205 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:00:36.205 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:00:36.205 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:00:36.205 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:00:36.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:00:36.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:00:36.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:00:36.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:00:36.205 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:00:36.205 | INFO | ------------------------------------------ | 20260508 00:00:36.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:00:36.205 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:00:36.205 | INFO | Total mem usage: 27.8 GiB | 20260508 00:00:36.205 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:00:36.205 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:00:36.205 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:00:36.205 | INFO | High Water Breakdown: | 20260508 00:00:36.205 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:00:36.205 | INFO | ------------------------------------------ | 20260508 00:00:36.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:00:36.205 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:00:36.205 | INFO | Total mem usage: 15.7 GiB | 20260508 00:00:36.205 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:00:36.205 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:00:36.205 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:00:36.205 | INFO | High Water Breakdown: | 20260508 00:00:36.205 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:00:36.205 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:00:36.205 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:00:36.205 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:00:36.205 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:00:36.205 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:00:36.205 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:00:36.205 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:00:36.205 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:00:36.205 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:00:36.205 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:00:36.205 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:00:36.205 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:00:36.205 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:00:36.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:00:36.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:00:36.205 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:00:36.205 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:00:36.205 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:00:36.205 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:00:36.205 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:00:36.205 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:00:36.205 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:00:36.205 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:00:36.205 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:00:36.205 | INFO | ------------------------------------------ | 20260508 00:00:36.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:00:36.205 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:00:36.205 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:00:36.205 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:00:36.205 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.65e+01, 1.41e+01 / 1.58e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.01e-01, 1.11e-02 / 3.88e-03 | 20260508 00:00:36.205 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.47e+00, 5.93e-01 / 1.90e-01 | 20260508 00:00:36.205 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.06e-01 / 9.52e-02 | 20260508 00:00:36.205 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.61e+01, 4.29e+00 / 1.79e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.65e+00 / 1.59e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.30e-01 / 2.08e-01 | 20260508 00:00:36.205 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.36e+00, 2.85e-01 / 1.44e-01 | 20260508 00:00:36.205 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.80e-01, 5.52e-02 / 2.67e-02 | 20260508 00:00:36.205 | INFO | Uploaded 36GiB to GPU in 2.51039s (14.3404GiB/s) | 20260508 00:00:36.205 | INFO | Downloaded 34GiB to GPU in 2.44701s (13.8945GiB/s) | 20260508 00:00:36.205 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:00:36.205 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:00:36.205 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:00:36.205 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 24.33%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 30.74%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 2.13%, LockUpload -- 2.28e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 2.74%, LockDownload -- 2.92e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 15.77%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:00:36.205 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260508 00:00:36.302 | INFO | Starting a round of Baz IO | 20260508 00:00:39.751 | INFO | Populating RTMetrics report | 20260508 00:00:39.751 | INFO | RTMetrics report populated | 20260508 00:00:39.751 | INFO | Opening tmp RTMetrics json file | 20260508 00:00:39.781 | INFO | Serializing RTMetrics to json | 20260508 00:00:39.822 | INFO | Serialization done, swapping tmp file now... | 20260508 00:00:39.845 | INFO | RTMetrics done | 20260508 00:00:39.868 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.566 seconds | 20260508 00:00:39.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.37% / 0.00% / 57.63% | 20260508 00:00:39.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08685% / 0% / 4.37101e-05% | 20260508 00:00:39.869 | INFO | Baz IO overall: | 20260508 00:00:39.869 | INFO | BytesWritten : 1654385826327 | 20260508 00:00:39.869 | INFO | NumEvents : 800919675121 | 20260508 00:00:41.339 | INFO | Analyzing chunk frames = [3637248,3637760) | 20260508 00:00:41.339 | INFO | MemInfo: { | 20260508 00:00:41.339 | INFO | "gpu" : " 17 GB", | 20260508 00:00:41.339 | INFO | "huge" : "", | 20260508 00:00:41.339 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:41.339 | INFO | "malloc" : " 14 GB", | 20260508 00:00:41.339 | INFO | "pinned" : " 29 GB" | 20260508 00:00:41.339 | INFO | } | 20260508 00:00:41.339 | INFO | Perf report for the last 5.13494 seconds: | 20260508 00:00:41.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:41.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:41.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:41.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:41.339 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260508 00:00:41.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:41.339 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 00:00:41.339 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260508 00:00:41.340 | INFO | BAZWRITER : 69.0%, 3.7, 129.5ms, 0 | 20260508 00:00:41.340 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:00:41.340 | INFO | Uploaded 36GiB to GPU in 2.51076s (14.3383GiB/s) | 20260508 00:00:41.340 | INFO | Downloaded 38GiB to GPU in 2.68186s (14.1693GiB/s) | 20260508 00:00:46.479 | INFO | Analyzing chunk frames = [3637760,3638272) | 20260508 00:00:46.479 | INFO | MemInfo: { | 20260508 00:00:46.479 | INFO | "gpu" : " 17 GB", | 20260508 00:00:46.479 | INFO | "huge" : "", | 20260508 00:00:46.479 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:46.479 | INFO | "malloc" : " 14 GB", | 20260508 00:00:46.479 | INFO | "pinned" : " 29 GB" | 20260508 00:00:46.479 | INFO | } | 20260508 00:00:46.479 | INFO | Perf report for the last 5.13967 seconds: | 20260508 00:00:46.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:46.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.479 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 00:00:46.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.479 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:00:46.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:46.480 | INFO | Uploaded 36GiB to GPU in 2.56091s (14.0575GiB/s) | 20260508 00:00:46.480 | INFO | Downloaded 34GiB to GPU in 2.49126s (13.6477GiB/s) | 20260508 00:00:51.617 | INFO | Analyzing chunk frames = [3638272,3638784) | 20260508 00:00:51.617 | INFO | MemInfo: { | 20260508 00:00:51.617 | INFO | "gpu" : " 17 GB", | 20260508 00:00:51.617 | INFO | "huge" : "", | 20260508 00:00:51.618 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:51.618 | INFO | "malloc" : " 14 GB", | 20260508 00:00:51.618 | INFO | "pinned" : " 29 GB" | 20260508 00:00:51.618 | INFO | } | 20260508 00:00:51.618 | INFO | Perf report for the last 5.1387 seconds: | 20260508 00:00:51.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:51.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:00:51.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:00:51.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:51.618 | INFO | Uploaded 36GiB to GPU in 2.50578s (14.3668GiB/s) | 20260508 00:00:51.618 | INFO | Downloaded 34GiB to GPU in 2.4766s (13.7285GiB/s) | 20260508 00:00:56.763 | INFO | Analyzing chunk frames = [3638784,3639296) | 20260508 00:00:56.763 | INFO | MemInfo: { | 20260508 00:00:56.763 | INFO | "gpu" : " 17 GB", | 20260508 00:00:56.763 | INFO | "huge" : "", | 20260508 00:00:56.763 | INFO | "huge_shared" : " 29 GB", | 20260508 00:00:56.763 | INFO | "malloc" : " 14 GB", | 20260508 00:00:56.763 | INFO | "pinned" : " 29 GB" | 20260508 00:00:56.763 | INFO | } | 20260508 00:00:56.763 | INFO | Perf report for the last 5.14571 seconds: | 20260508 00:00:56.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:00:56.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.763 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 00:00:56.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.763 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:00:56.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:00:56.764 | INFO | Uploaded 36GiB to GPU in 2.52047s (14.283GiB/s) | 20260508 00:00:56.764 | INFO | Downloaded 34GiB to GPU in 2.47058s (13.7619GiB/s) | 20260508 00:01:01.900 | INFO | Analyzing chunk frames = [3639296,3639808) | 20260508 00:01:01.900 | INFO | MemInfo: { | 20260508 00:01:01.900 | INFO | "gpu" : " 17 GB", | 20260508 00:01:01.900 | INFO | "huge" : "", | 20260508 00:01:01.900 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:01.900 | INFO | "malloc" : " 14 GB", | 20260508 00:01:01.900 | INFO | "pinned" : " 29 GB" | 20260508 00:01:01.900 | INFO | } | 20260508 00:01:01.900 | INFO | Perf report for the last 5.13673 seconds: | 20260508 00:01:01.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:01.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 00:01:01.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:01:01.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:01.900 | INFO | Uploaded 36GiB to GPU in 2.36733s (15.207GiB/s) | 20260508 00:01:01.900 | INFO | Downloaded 34GiB to GPU in 2.49681s (13.6174GiB/s) | 20260508 00:01:07.038 | INFO | Analyzing chunk frames = [3639808,3640320) | 20260508 00:01:07.038 | INFO | MemInfo: { | 20260508 00:01:07.038 | INFO | "gpu" : " 17 GB", | 20260508 00:01:07.038 | INFO | "huge" : "", | 20260508 00:01:07.038 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:07.038 | INFO | "malloc" : " 14 GB", | 20260508 00:01:07.038 | INFO | "pinned" : " 29 GB" | 20260508 00:01:07.038 | INFO | } | 20260508 00:01:07.038 | INFO | Perf report for the last 5.13805 seconds: | 20260508 00:01:07.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:07.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 00:01:07.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:01:07.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:07.038 | INFO | Uploaded 36GiB to GPU in 2.5131s (14.325GiB/s) | 20260508 00:01:07.038 | INFO | Downloaded 34GiB to GPU in 2.41782s (14.0622GiB/s) | 20260508 00:01:12.177 | INFO | Analyzing chunk frames = [3640320,3640832) | 20260508 00:01:12.177 | INFO | MemInfo: { | 20260508 00:01:12.177 | INFO | "gpu" : " 17 GB", | 20260508 00:01:12.177 | INFO | "huge" : "", | 20260508 00:01:12.177 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:12.177 | INFO | "malloc" : " 14 GB", | 20260508 00:01:12.177 | INFO | "pinned" : " 29 GB" | 20260508 00:01:12.177 | INFO | } | 20260508 00:01:12.178 | INFO | Perf report for the last 5.13938 seconds: | 20260508 00:01:12.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:12.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 00:01:12.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:01:12.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:12.178 | INFO | Uploaded 36GiB to GPU in 2.56133s (14.0552GiB/s) | 20260508 00:01:12.178 | INFO | Downloaded 34GiB to GPU in 2.45479s (13.8505GiB/s) | 20260508 00:01:17.317 | INFO | Analyzing chunk frames = [3640832,3641344) | 20260508 00:01:17.317 | INFO | MemInfo: { | 20260508 00:01:17.317 | INFO | "gpu" : " 17 GB", | 20260508 00:01:17.317 | INFO | "huge" : "", | 20260508 00:01:17.317 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:17.317 | INFO | "malloc" : " 14 GB", | 20260508 00:01:17.318 | INFO | "pinned" : " 29 GB" | 20260508 00:01:17.318 | INFO | } | 20260508 00:01:17.318 | INFO | Perf report for the last 5.14013 seconds: | 20260508 00:01:17.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:17.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:01:17.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 00:01:17.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:17.318 | INFO | Uploaded 36GiB to GPU in 2.52593s (14.2522GiB/s) | 20260508 00:01:17.318 | INFO | Downloaded 34GiB to GPU in 2.44165s (13.925GiB/s) | 20260508 00:01:17.415 | INFO | Starting a round of Baz IO | 20260508 00:01:20.875 | INFO | Populating RTMetrics report | 20260508 00:01:20.876 | INFO | RTMetrics report populated | 20260508 00:01:20.876 | INFO | Opening tmp RTMetrics json file | 20260508 00:01:20.892 | INFO | Serializing RTMetrics to json | 20260508 00:01:20.899 | INFO | Serialization done, swapping tmp file now... | 20260508 00:01:20.900 | INFO | RTMetrics done | 20260508 00:01:21.003 | INFO | Finished a round of Baz IO: Wrote 1.7 GiB in 3.588 seconds | 20260508 00:01:21.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.36% / 0.00% / 57.63% | 20260508 00:01:21.004 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08428% / 0% / 5.96046e-05% | 20260508 00:01:21.004 | INFO | Baz IO overall: | 20260508 00:01:21.004 | INFO | BytesWritten : 1656211710347 | 20260508 00:01:21.004 | INFO | NumEvents : 801798741286 | 20260508 00:01:22.450 | INFO | Analyzing chunk frames = [3641344,3641856) | 20260508 00:01:22.450 | INFO | MemInfo: { | 20260508 00:01:22.450 | INFO | "gpu" : " 17 GB", | 20260508 00:01:22.450 | INFO | "huge" : "", | 20260508 00:01:22.450 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:22.450 | INFO | "malloc" : " 14 GB", | 20260508 00:01:22.450 | INFO | "pinned" : " 29 GB" | 20260508 00:01:22.450 | INFO | } | 20260508 00:01:22.450 | INFO | Perf report for the last 5.1322 seconds: | 20260508 00:01:22.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:22.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:22.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:22.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:22.450 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260508 00:01:22.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:22.450 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260508 00:01:22.450 | INFO | PRE_HQ : 65.7%, 1.7, 56.7ms, 0 | 20260508 00:01:22.450 | INFO | BAZWRITER : 69.5%, 3.7, 130.9ms, 0 | 20260508 00:01:22.450 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 00:01:22.451 | INFO | Uploaded 36GiB to GPU in 2.55913s (14.0673GiB/s) | 20260508 00:01:22.451 | INFO | Downloaded 38GiB to GPU in 2.69757s (14.0867GiB/s) | 20260508 00:01:27.590 | INFO | Analyzing chunk frames = [3641856,3642368) | 20260508 00:01:27.590 | INFO | MemInfo: { | 20260508 00:01:27.590 | INFO | "gpu" : " 17 GB", | 20260508 00:01:27.590 | INFO | "huge" : "", | 20260508 00:01:27.590 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:27.590 | INFO | "malloc" : " 14 GB", | 20260508 00:01:27.590 | INFO | "pinned" : " 29 GB" | 20260508 00:01:27.590 | INFO | } | 20260508 00:01:27.590 | INFO | Perf report for the last 5.14053 seconds: | 20260508 00:01:27.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:27.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.590 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:01:27.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.591 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 00:01:27.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:27.591 | INFO | Uploaded 36GiB to GPU in 2.5654s (14.0329GiB/s) | 20260508 00:01:27.591 | INFO | Downloaded 34GiB to GPU in 2.49061s (13.6513GiB/s) | 20260508 00:01:32.733 | INFO | Analyzing chunk frames = [3642368,3642880) | 20260508 00:01:32.733 | INFO | MemInfo: { | 20260508 00:01:32.733 | INFO | "gpu" : " 17 GB", | 20260508 00:01:32.733 | INFO | "huge" : "", | 20260508 00:01:32.733 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:32.733 | INFO | "malloc" : " 14 GB", | 20260508 00:01:32.733 | INFO | "pinned" : " 29 GB" | 20260508 00:01:32.733 | INFO | } | 20260508 00:01:32.733 | INFO | Perf report for the last 5.14241 seconds: | 20260508 00:01:32.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:32.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:01:32.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:01:32.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:32.733 | INFO | Uploaded 36GiB to GPU in 2.55124s (14.1108GiB/s) | 20260508 00:01:32.734 | INFO | Downloaded 34GiB to GPU in 2.48701s (13.671GiB/s) | 20260508 00:01:37.874 | INFO | Analyzing chunk frames = [3642880,3643392) | 20260508 00:01:37.874 | INFO | MemInfo: { | 20260508 00:01:37.874 | INFO | "gpu" : " 17 GB", | 20260508 00:01:37.874 | INFO | "huge" : "", | 20260508 00:01:37.874 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:37.874 | INFO | "malloc" : " 14 GB", | 20260508 00:01:37.874 | INFO | "pinned" : " 29 GB" | 20260508 00:01:37.874 | INFO | } | 20260508 00:01:37.875 | INFO | Perf report for the last 5.1417 seconds: | 20260508 00:01:37.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:37.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 00:01:37.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:01:37.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:37.875 | INFO | Uploaded 36GiB to GPU in 2.47195s (14.5634GiB/s) | 20260508 00:01:37.875 | INFO | Downloaded 34GiB to GPU in 2.41548s (14.0759GiB/s) | 20260508 00:01:43.019 | INFO | Analyzing chunk frames = [3643392,3643904) | 20260508 00:01:43.019 | INFO | MemInfo: { | 20260508 00:01:43.019 | INFO | "gpu" : " 17 GB", | 20260508 00:01:43.019 | INFO | "huge" : "", | 20260508 00:01:43.019 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:43.019 | INFO | "malloc" : " 14 GB", | 20260508 00:01:43.020 | INFO | "pinned" : " 29 GB" | 20260508 00:01:43.020 | INFO | } | 20260508 00:01:43.020 | INFO | Perf report for the last 5.14514 seconds: | 20260508 00:01:43.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:43.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:01:43.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:01:43.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:43.020 | INFO | Uploaded 36GiB to GPU in 2.30689s (15.6054GiB/s) | 20260508 00:01:43.020 | INFO | Downloaded 34GiB to GPU in 2.47016s (13.7643GiB/s) | 20260508 00:01:48.154 | INFO | Analyzing chunk frames = [3643904,3644416) | 20260508 00:01:48.154 | INFO | MemInfo: { | 20260508 00:01:48.154 | INFO | "gpu" : " 17 GB", | 20260508 00:01:48.154 | INFO | "huge" : "", | 20260508 00:01:48.154 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:48.154 | INFO | "malloc" : " 14 GB", | 20260508 00:01:48.154 | INFO | "pinned" : " 29 GB" | 20260508 00:01:48.154 | INFO | } | 20260508 00:01:48.154 | INFO | Perf report for the last 5.13475 seconds: | 20260508 00:01:48.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:48.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:01:48.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:01:48.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:48.155 | INFO | Uploaded 36GiB to GPU in 2.38854s (15.072GiB/s) | 20260508 00:01:48.155 | INFO | Downloaded 34GiB to GPU in 2.41258s (14.0928GiB/s) | 20260508 00:01:53.293 | INFO | Analyzing chunk frames = [3644416,3644928) | 20260508 00:01:53.293 | INFO | MemInfo: { | 20260508 00:01:53.293 | INFO | "gpu" : " 17 GB", | 20260508 00:01:53.293 | INFO | "huge" : "", | 20260508 00:01:53.293 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:53.293 | INFO | "malloc" : " 14 GB", | 20260508 00:01:53.293 | INFO | "pinned" : " 29 GB" | 20260508 00:01:53.293 | INFO | } | 20260508 00:01:53.293 | INFO | Perf report for the last 5.13874 seconds: | 20260508 00:01:53.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:53.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.293 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:01:53.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.293 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:01:53.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:53.294 | INFO | Uploaded 36GiB to GPU in 2.52859s (14.2372GiB/s) | 20260508 00:01:53.294 | INFO | Downloaded 34GiB to GPU in 2.46337s (13.8022GiB/s) | 20260508 00:01:58.434 | INFO | Analyzing chunk frames = [3644928,3645440) | 20260508 00:01:58.434 | INFO | MemInfo: { | 20260508 00:01:58.434 | INFO | "gpu" : " 17 GB", | 20260508 00:01:58.434 | INFO | "huge" : "", | 20260508 00:01:58.434 | INFO | "huge_shared" : " 29 GB", | 20260508 00:01:58.434 | INFO | "malloc" : " 14 GB", | 20260508 00:01:58.434 | INFO | "pinned" : " 29 GB" | 20260508 00:01:58.434 | INFO | } | 20260508 00:01:58.434 | INFO | Perf report for the last 5.14093 seconds: | 20260508 00:01:58.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:01:58.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.434 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 00:01:58.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.434 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:01:58.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:01:58.435 | INFO | Uploaded 36GiB to GPU in 2.518s (14.2971GiB/s) | 20260508 00:01:58.435 | INFO | Downloaded 34GiB to GPU in 2.45489s (13.8499GiB/s) | 20260508 00:01:58.435 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:01:58.435 | INFO | Total mem usage: 16.7 GiB | 20260508 00:01:58.435 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:01:58.435 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.7%) | 20260508 00:01:58.435 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:01:58.435 | INFO | High Water Breakdown: | 20260508 00:01:58.435 | INFO | 59.7% : 8.69 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:01:58.435 | INFO | 40.1% : 5.84 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:01:58.435 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:01:58.435 | INFO | ------------------------------------------ | 20260508 00:01:58.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:01:58.435 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:01:58.435 | INFO | Total mem usage: 26.5 GiB | 20260508 00:01:58.435 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:01:58.435 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:01:58.435 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:01:58.435 | INFO | High Water Breakdown: | 20260508 00:01:58.435 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:01:58.435 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:01:58.435 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:01:58.435 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:01:58.435 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:01:58.435 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:01:58.435 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:01:58.435 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:01:58.435 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:01:58.435 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:01:58.435 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:01:58.435 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:01:58.435 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:01:58.435 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:01:58.435 | INFO | ------------------------------------------ | 20260508 00:01:58.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:01:58.435 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:01:58.435 | INFO | Total mem usage: 27.8 GiB | 20260508 00:01:58.435 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:01:58.435 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:01:58.435 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:01:58.435 | INFO | High Water Breakdown: | 20260508 00:01:58.435 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:01:58.435 | INFO | ------------------------------------------ | 20260508 00:01:58.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:01:58.435 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:01:58.435 | INFO | Total mem usage: 15.7 GiB | 20260508 00:01:58.435 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:01:58.435 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:01:58.436 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:01:58.436 | INFO | High Water Breakdown: | 20260508 00:01:58.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:01:58.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:01:58.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:01:58.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:01:58.436 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:01:58.436 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:01:58.436 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:01:58.436 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:01:58.436 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:01:58.436 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:01:58.436 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:01:58.436 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:01:58.436 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:01:58.436 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:01:58.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:01:58.436 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:01:58.436 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:01:58.436 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:01:58.436 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:01:58.436 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:01:58.436 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:01:58.436 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:01:58.436 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:01:58.436 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:01:58.436 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:01:58.436 | INFO | ------------------------------------------ | 20260508 00:01:58.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:01:58.436 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:01:58.436 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:01:58.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:01:58.436 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.41e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.01e-01, 1.17e-02 / 6.51e-03 | 20260508 00:01:58.436 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.48e+00, 5.89e-01 / 2.27e-01 | 20260508 00:01:58.436 | INFO | PerfReport: 3.34%, Histogram -- 1.10e+00 / 1.46e+00, 2.05e-01 / 1.19e-01 | 20260508 00:01:58.436 | INFO | PerfReport: 15.97%, DME -- 4.19e+01 / 4.57e+01, 3.94e+00 / 1.41e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.69e+00 / 1.93e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.71e+00, 4.32e-01 / 2.22e-01 | 20260508 00:01:58.436 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.33e+00, 2.82e-01 / 1.66e-01 | 20260508 00:01:58.436 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.43e-01 / 3.75e-01, 5.52e-02 / 2.18e-02 | 20260508 00:01:58.436 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:01:58.436 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:01:58.436 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:01:58.436 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 30.74%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 2.14%, LockUpload -- 2.29e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 2.64%, LockDownload -- 2.82e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 15.80%, LockCompute -- 1.69e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:01:58.436 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260508 00:01:58.532 | INFO | Starting a round of Baz IO | 20260508 00:02:02.008 | INFO | Populating RTMetrics report | 20260508 00:02:02.008 | INFO | RTMetrics report populated | 20260508 00:02:02.008 | INFO | Opening tmp RTMetrics json file | 20260508 00:02:02.053 | INFO | Serializing RTMetrics to json | 20260508 00:02:02.091 | INFO | Serialization done, swapping tmp file now... | 20260508 00:02:02.092 | INFO | RTMetrics done | 20260508 00:02:02.119 | INFO | Finished a round of Baz IO: Wrote 1.7 GiB in 3.586 seconds | 20260508 00:02:02.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.36% / 0.00% / 57.64% | 20260508 00:02:02.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08271% / 7.94729e-06% / 5.16574e-05% | 20260508 00:02:02.119 | INFO | Baz IO overall: | 20260508 00:02:02.119 | INFO | BytesWritten : 1658036740952 | 20260508 00:02:02.119 | INFO | NumEvents : 802677209382 | 20260508 00:02:03.569 | INFO | Analyzing chunk frames = [3645440,3645952) | 20260508 00:02:03.569 | INFO | MemInfo: { | 20260508 00:02:03.569 | INFO | "gpu" : " 17 GB", | 20260508 00:02:03.569 | INFO | "huge" : "", | 20260508 00:02:03.569 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:03.569 | INFO | "malloc" : " 14 GB", | 20260508 00:02:03.569 | INFO | "pinned" : " 29 GB" | 20260508 00:02:03.569 | INFO | } | 20260508 00:02:03.569 | INFO | Perf report for the last 5.13493 seconds: | 20260508 00:02:03.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:03.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:03.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:03.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:03.569 | INFO | ANALYSIS : 68.7%, 2.9, 104.8ms, 0 | 20260508 00:02:03.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:03.569 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260508 00:02:03.569 | INFO | PRE_HQ : 67.3%, 1.7, 60.8ms, 0 | 20260508 00:02:03.569 | INFO | BAZWRITER : 69.3%, 3.9, 138.4ms, 0 | 20260508 00:02:03.569 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 00:02:03.569 | INFO | Uploaded 36GiB to GPU in 2.56083s (14.058GiB/s) | 20260508 00:02:03.569 | INFO | Downloaded 38GiB to GPU in 2.69622s (14.0938GiB/s) | 20260508 00:02:08.713 | INFO | Analyzing chunk frames = [3645952,3646464) | 20260508 00:02:08.713 | INFO | MemInfo: { | 20260508 00:02:08.713 | INFO | "gpu" : " 17 GB", | 20260508 00:02:08.713 | INFO | "huge" : "", | 20260508 00:02:08.713 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:08.713 | INFO | "malloc" : " 14 GB", | 20260508 00:02:08.713 | INFO | "pinned" : " 29 GB" | 20260508 00:02:08.713 | INFO | } | 20260508 00:02:08.713 | INFO | Perf report for the last 5.1438 seconds: | 20260508 00:02:08.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:08.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:02:08.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:02:08.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:08.713 | INFO | Uploaded 36GiB to GPU in 2.56299s (14.0461GiB/s) | 20260508 00:02:08.714 | INFO | Downloaded 34GiB to GPU in 2.49497s (13.6274GiB/s) | 20260508 00:02:13.852 | INFO | Analyzing chunk frames = [3646464,3646976) | 20260508 00:02:13.852 | INFO | MemInfo: { | 20260508 00:02:13.852 | INFO | "gpu" : " 17 GB", | 20260508 00:02:13.852 | INFO | "huge" : "", | 20260508 00:02:13.852 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:13.852 | INFO | "malloc" : " 14 GB", | 20260508 00:02:13.852 | INFO | "pinned" : " 29 GB" | 20260508 00:02:13.852 | INFO | } | 20260508 00:02:13.852 | INFO | Perf report for the last 5.13886 seconds: | 20260508 00:02:13.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:13.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 00:02:13.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:02:13.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:13.852 | INFO | Uploaded 36GiB to GPU in 2.52189s (14.275GiB/s) | 20260508 00:02:13.852 | INFO | Downloaded 34GiB to GPU in 2.49391s (13.6332GiB/s) | 20260508 00:02:18.984 | INFO | Analyzing chunk frames = [3646976,3647488) | 20260508 00:02:18.984 | INFO | MemInfo: { | 20260508 00:02:18.984 | INFO | "gpu" : " 17 GB", | 20260508 00:02:18.984 | INFO | "huge" : "", | 20260508 00:02:18.984 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:18.984 | INFO | "malloc" : " 14 GB", | 20260508 00:02:18.984 | INFO | "pinned" : " 29 GB" | 20260508 00:02:18.985 | INFO | } | 20260508 00:02:18.985 | INFO | Perf report for the last 5.13279 seconds: | 20260508 00:02:18.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:18.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | ANALYSIS : 67.6%, 3.0, 104.4ms, 0 | 20260508 00:02:18.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260508 00:02:18.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:18.985 | INFO | Uploaded 36GiB to GPU in 2.5204s (14.2834GiB/s) | 20260508 00:02:18.985 | INFO | Downloaded 34GiB to GPU in 2.50063s (13.5966GiB/s) | 20260508 00:02:24.132 | INFO | Analyzing chunk frames = [3647488,3648000) | 20260508 00:02:24.132 | INFO | MemInfo: { | 20260508 00:02:24.132 | INFO | "gpu" : " 17 GB", | 20260508 00:02:24.132 | INFO | "huge" : "", | 20260508 00:02:24.132 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:24.132 | INFO | "malloc" : " 14 GB", | 20260508 00:02:24.132 | INFO | "pinned" : " 29 GB" | 20260508 00:02:24.132 | INFO | } | 20260508 00:02:24.133 | INFO | Perf report for the last 5.14793 seconds: | 20260508 00:02:24.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:24.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 00:02:24.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:02:24.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:24.133 | INFO | Uploaded 36GiB to GPU in 2.36755s (15.2056GiB/s) | 20260508 00:02:24.133 | INFO | Downloaded 34GiB to GPU in 2.49911s (13.6048GiB/s) | 20260508 00:02:29.269 | INFO | Analyzing chunk frames = [3648000,3648512) | 20260508 00:02:29.269 | INFO | MemInfo: { | 20260508 00:02:29.269 | INFO | "gpu" : " 17 GB", | 20260508 00:02:29.269 | INFO | "huge" : "", | 20260508 00:02:29.269 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:29.269 | INFO | "malloc" : " 14 GB", | 20260508 00:02:29.269 | INFO | "pinned" : " 29 GB" | 20260508 00:02:29.270 | INFO | } | 20260508 00:02:29.270 | INFO | Perf report for the last 5.13715 seconds: | 20260508 00:02:29.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:29.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:02:29.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:02:29.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:29.270 | INFO | Uploaded 36GiB to GPU in 2.36061s (15.2503GiB/s) | 20260508 00:02:29.270 | INFO | Downloaded 34GiB to GPU in 2.42155s (14.0406GiB/s) | 20260508 00:02:34.408 | INFO | Analyzing chunk frames = [3648512,3649024) | 20260508 00:02:34.408 | INFO | MemInfo: { | 20260508 00:02:34.408 | INFO | "gpu" : " 17 GB", | 20260508 00:02:34.408 | INFO | "huge" : "", | 20260508 00:02:34.408 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:34.408 | INFO | "malloc" : " 14 GB", | 20260508 00:02:34.408 | INFO | "pinned" : " 29 GB" | 20260508 00:02:34.408 | INFO | } | 20260508 00:02:34.408 | INFO | Perf report for the last 5.13831 seconds: | 20260508 00:02:34.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:34.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.408 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:02:34.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.408 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260508 00:02:34.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:34.409 | INFO | Uploaded 36GiB to GPU in 2.48975s (14.4593GiB/s) | 20260508 00:02:34.409 | INFO | Downloaded 34GiB to GPU in 2.46314s (13.8035GiB/s) | 20260508 00:02:39.541 | INFO | Analyzing chunk frames = [3649024,3649536) | 20260508 00:02:39.541 | INFO | MemInfo: { | 20260508 00:02:39.541 | INFO | "gpu" : " 17 GB", | 20260508 00:02:39.541 | INFO | "huge" : "", | 20260508 00:02:39.541 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:39.541 | INFO | "malloc" : " 14 GB", | 20260508 00:02:39.541 | INFO | "pinned" : " 29 GB" | 20260508 00:02:39.541 | INFO | } | 20260508 00:02:39.541 | INFO | Perf report for the last 5.1333 seconds: | 20260508 00:02:39.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:39.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260508 00:02:39.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 00:02:39.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:39.541 | INFO | Uploaded 36GiB to GPU in 2.54123s (14.1663GiB/s) | 20260508 00:02:39.542 | INFO | Downloaded 34GiB to GPU in 2.46596s (13.7878GiB/s) | 20260508 00:02:39.638 | INFO | Starting a round of Baz IO | 20260508 00:02:43.109 | INFO | Populating RTMetrics report | 20260508 00:02:43.109 | INFO | RTMetrics report populated | 20260508 00:02:43.109 | INFO | Opening tmp RTMetrics json file | 20260508 00:02:43.130 | INFO | Serializing RTMetrics to json | 20260508 00:02:43.131 | INFO | Serialization done, swapping tmp file now... | 20260508 00:02:43.132 | INFO | RTMetrics done | 20260508 00:02:43.203 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.564 seconds | 20260508 00:02:43.203 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.35% / 0.00% / 57.64% | 20260508 00:02:43.203 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08265% / 0% / 5.96046e-05% | 20260508 00:02:43.203 | INFO | Baz IO overall: | 20260508 00:02:43.203 | INFO | BytesWritten : 1659863062751 | 20260508 00:02:43.203 | INFO | NumEvents : 803556865668 | 20260508 00:02:44.684 | INFO | Analyzing chunk frames = [3649536,3650048) | 20260508 00:02:44.684 | INFO | MemInfo: { | 20260508 00:02:44.684 | INFO | "gpu" : " 17 GB", | 20260508 00:02:44.684 | INFO | "huge" : "", | 20260508 00:02:44.684 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:44.684 | INFO | "malloc" : " 14 GB", | 20260508 00:02:44.684 | INFO | "pinned" : " 29 GB" | 20260508 00:02:44.684 | INFO | } | 20260508 00:02:44.684 | INFO | Perf report for the last 5.14292 seconds: | 20260508 00:02:44.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:44.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:44.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:44.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:44.684 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 00:02:44.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:44.684 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 00:02:44.684 | INFO | PRE_HQ : 65.6%, 1.7, 56.4ms, 0 | 20260508 00:02:44.684 | INFO | BAZWRITER : 68.9%, 4.0, 141.5ms, 0 | 20260508 00:02:44.684 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 00:02:44.685 | INFO | Uploaded 36GiB to GPU in 2.55757s (14.0759GiB/s) | 20260508 00:02:44.685 | INFO | Downloaded 38GiB to GPU in 2.69191s (14.1164GiB/s) | 20260508 00:02:49.828 | INFO | Analyzing chunk frames = [3650048,3650560) | 20260508 00:02:49.829 | INFO | MemInfo: { | 20260508 00:02:49.829 | INFO | "gpu" : " 17 GB", | 20260508 00:02:49.829 | INFO | "huge" : "", | 20260508 00:02:49.829 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:49.829 | INFO | "malloc" : " 14 GB", | 20260508 00:02:49.829 | INFO | "pinned" : " 29 GB" | 20260508 00:02:49.829 | INFO | } | 20260508 00:02:49.829 | INFO | Perf report for the last 5.14449 seconds: | 20260508 00:02:49.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:49.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 00:02:49.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:02:49.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:49.829 | INFO | Uploaded 36GiB to GPU in 2.55212s (14.1059GiB/s) | 20260508 00:02:49.829 | INFO | Downloaded 34GiB to GPU in 2.50196s (13.5894GiB/s) | 20260508 00:02:54.973 | INFO | Analyzing chunk frames = [3650560,3651072) | 20260508 00:02:54.973 | INFO | MemInfo: { | 20260508 00:02:54.973 | INFO | "gpu" : " 17 GB", | 20260508 00:02:54.973 | INFO | "huge" : "", | 20260508 00:02:54.973 | INFO | "huge_shared" : " 29 GB", | 20260508 00:02:54.973 | INFO | "malloc" : " 14 GB", | 20260508 00:02:54.973 | INFO | "pinned" : " 29 GB" | 20260508 00:02:54.973 | INFO | } | 20260508 00:02:54.973 | INFO | Perf report for the last 5.14415 seconds: | 20260508 00:02:54.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:02:54.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 00:02:54.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 00:02:54.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:02:54.973 | INFO | Uploaded 36GiB to GPU in 2.47686s (14.5345GiB/s) | 20260508 00:02:54.973 | INFO | Downloaded 34GiB to GPU in 2.5093s (13.5496GiB/s) | 20260508 00:03:00.105 | INFO | Analyzing chunk frames = [3651072,3651584) | 20260508 00:03:00.105 | INFO | MemInfo: { | 20260508 00:03:00.105 | INFO | "gpu" : " 17 GB", | 20260508 00:03:00.105 | INFO | "huge" : "", | 20260508 00:03:00.105 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:00.105 | INFO | "malloc" : " 14 GB", | 20260508 00:03:00.105 | INFO | "pinned" : " 29 GB" | 20260508 00:03:00.105 | INFO | } | 20260508 00:03:00.106 | INFO | Perf report for the last 5.13253 seconds: | 20260508 00:03:00.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:00.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:03:00.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:03:00.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:00.106 | INFO | Uploaded 36GiB to GPU in 2.46156s (14.6249GiB/s) | 20260508 00:03:00.106 | INFO | Downloaded 34GiB to GPU in 2.50974s (13.5472GiB/s) | 20260508 00:03:05.241 | INFO | Analyzing chunk frames = [3651584,3652096) | 20260508 00:03:05.241 | INFO | MemInfo: { | 20260508 00:03:05.241 | INFO | "gpu" : " 17 GB", | 20260508 00:03:05.241 | INFO | "huge" : "", | 20260508 00:03:05.241 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:05.241 | INFO | "malloc" : " 14 GB", | 20260508 00:03:05.241 | INFO | "pinned" : " 29 GB" | 20260508 00:03:05.241 | INFO | } | 20260508 00:03:05.241 | INFO | Perf report for the last 5.13594 seconds: | 20260508 00:03:05.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:05.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:03:05.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:03:05.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:05.242 | INFO | Uploaded 36GiB to GPU in 2.29476s (15.6879GiB/s) | 20260508 00:03:05.242 | INFO | Downloaded 34GiB to GPU in 2.48468s (13.6839GiB/s) | 20260508 00:03:10.381 | INFO | Analyzing chunk frames = [3652096,3652608) | 20260508 00:03:10.381 | INFO | MemInfo: { | 20260508 00:03:10.381 | INFO | "gpu" : " 17 GB", | 20260508 00:03:10.381 | INFO | "huge" : "", | 20260508 00:03:10.381 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:10.381 | INFO | "malloc" : " 14 GB", | 20260508 00:03:10.382 | INFO | "pinned" : " 29 GB" | 20260508 00:03:10.382 | INFO | } | 20260508 00:03:10.382 | INFO | Perf report for the last 5.14015 seconds: | 20260508 00:03:10.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:10.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:03:10.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:03:10.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:10.382 | INFO | Uploaded 36GiB to GPU in 2.40935s (14.9418GiB/s) | 20260508 00:03:10.382 | INFO | Downloaded 34GiB to GPU in 2.40603s (14.1311GiB/s) | 20260508 00:03:15.521 | INFO | Analyzing chunk frames = [3652608,3653120) | 20260508 00:03:15.521 | INFO | MemInfo: { | 20260508 00:03:15.521 | INFO | "gpu" : " 17 GB", | 20260508 00:03:15.521 | INFO | "huge" : "", | 20260508 00:03:15.521 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:15.521 | INFO | "malloc" : " 14 GB", | 20260508 00:03:15.522 | INFO | "pinned" : " 29 GB" | 20260508 00:03:15.522 | INFO | } | 20260508 00:03:15.522 | INFO | Perf report for the last 5.13994 seconds: | 20260508 00:03:15.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:15.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:03:15.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:03:15.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:15.522 | INFO | Uploaded 36GiB to GPU in 2.53369s (14.2085GiB/s) | 20260508 00:03:15.522 | INFO | Downloaded 34GiB to GPU in 2.4572s (13.8369GiB/s) | 20260508 00:03:20.660 | INFO | Analyzing chunk frames = [3653120,3653632) | 20260508 00:03:20.660 | INFO | MemInfo: { | 20260508 00:03:20.660 | INFO | "gpu" : " 17 GB", | 20260508 00:03:20.660 | INFO | "huge" : "", | 20260508 00:03:20.660 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:20.660 | INFO | "malloc" : " 14 GB", | 20260508 00:03:20.660 | INFO | "pinned" : " 29 GB" | 20260508 00:03:20.660 | INFO | } | 20260508 00:03:20.660 | INFO | Perf report for the last 5.1388 seconds: | 20260508 00:03:20.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:20.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.660 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 00:03:20.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.661 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:03:20.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:20.661 | INFO | Uploaded 36GiB to GPU in 2.48563s (14.4833GiB/s) | 20260508 00:03:20.661 | INFO | Downloaded 34GiB to GPU in 2.44767s (13.8908GiB/s) | 20260508 00:03:20.661 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:03:20.661 | INFO | Total mem usage: 16.7 GiB | 20260508 00:03:20.661 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:03:20.661 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260508 00:03:20.661 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:03:20.661 | INFO | High Water Breakdown: | 20260508 00:03:20.661 | INFO | 60.2% : 9.05 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:03:20.661 | INFO | 39.7% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:03:20.661 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:03:20.661 | INFO | ------------------------------------------ | 20260508 00:03:20.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:03:20.661 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:03:20.661 | INFO | Total mem usage: 26.5 GiB | 20260508 00:03:20.661 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:03:20.661 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:03:20.661 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:03:20.661 | INFO | High Water Breakdown: | 20260508 00:03:20.661 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:03:20.661 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:03:20.661 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:03:20.661 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:03:20.661 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:03:20.661 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:03:20.661 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:03:20.661 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:03:20.661 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:03:20.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:03:20.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:03:20.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:03:20.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:03:20.661 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:03:20.661 | INFO | ------------------------------------------ | 20260508 00:03:20.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:03:20.661 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:03:20.662 | INFO | Total mem usage: 27.8 GiB | 20260508 00:03:20.662 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:03:20.662 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:03:20.662 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:03:20.662 | INFO | High Water Breakdown: | 20260508 00:03:20.662 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:03:20.662 | INFO | ------------------------------------------ | 20260508 00:03:20.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:03:20.662 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:03:20.662 | INFO | Total mem usage: 15.7 GiB | 20260508 00:03:20.662 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:03:20.662 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:03:20.662 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:03:20.662 | INFO | High Water Breakdown: | 20260508 00:03:20.662 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:03:20.662 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:03:20.662 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:03:20.662 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:03:20.662 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:03:20.662 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:03:20.662 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:03:20.662 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:03:20.662 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:03:20.662 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:03:20.662 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:03:20.662 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:03:20.662 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:03:20.662 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:03:20.662 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:03:20.662 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:03:20.662 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:03:20.662 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:03:20.662 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:03:20.662 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:03:20.662 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:03:20.662 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:03:20.662 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:03:20.662 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:03:20.662 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:03:20.662 | INFO | ------------------------------------------ | 20260508 00:03:20.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:03:20.662 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:03:20.662 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:03:20.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:03:20.662 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.44e+01 / 1.60e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 9.99e-02, 1.30e-02 / 4.95e-03 | 20260508 00:03:20.662 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.46e+00, 5.83e-01 / 2.36e-01 | 20260508 00:03:20.662 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.05e-01 / 1.23e-01 | 20260508 00:03:20.662 | INFO | PerfReport: 16.15%, DME -- 4.24e+01 / 4.64e+01, 4.21e+00 / 1.82e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.61e+00 / 1.51e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.46e+00 / 2.73e+00, 4.31e-01 / 2.40e-01 | 20260508 00:03:20.662 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.32e+00, 2.77e-01 / 1.32e-01 | 20260508 00:03:20.662 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.43e-01 / 3.81e-01, 5.48e-02 / 2.82e-02 | 20260508 00:03:20.662 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:03:20.662 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:03:20.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:03:20.662 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 24.46%, Download -- 2.61e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 2.04%, LockUpload -- 2.17e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 2.88%, LockDownload -- 3.07e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:03:20.662 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 00:03:20.757 | INFO | Starting a round of Baz IO | 20260508 00:03:24.220 | INFO | Populating RTMetrics report | 20260508 00:03:24.220 | INFO | RTMetrics report populated | 20260508 00:03:24.220 | INFO | Opening tmp RTMetrics json file | 20260508 00:03:24.251 | INFO | Serializing RTMetrics to json | 20260508 00:03:24.273 | INFO | Serialization done, swapping tmp file now... | 20260508 00:03:24.294 | INFO | RTMetrics done | 20260508 00:03:24.322 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.564 seconds | 20260508 00:03:24.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.35% / 0.00% / 57.65% | 20260508 00:03:24.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07618% / 3.97364e-06% / 6.35783e-05% | 20260508 00:03:24.322 | INFO | Baz IO overall: | 20260508 00:03:24.322 | INFO | BytesWritten : 1661686882193 | 20260508 00:03:24.322 | INFO | NumEvents : 804434644164 | 20260508 00:03:25.803 | INFO | Analyzing chunk frames = [3653632,3654144) | 20260508 00:03:25.803 | INFO | MemInfo: { | 20260508 00:03:25.803 | INFO | "gpu" : " 17 GB", | 20260508 00:03:25.803 | INFO | "huge" : "", | 20260508 00:03:25.803 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:25.803 | INFO | "malloc" : " 14 GB", | 20260508 00:03:25.803 | INFO | "pinned" : " 29 GB" | 20260508 00:03:25.803 | INFO | } | 20260508 00:03:25.803 | INFO | Perf report for the last 5.14267 seconds: | 20260508 00:03:25.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:25.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:25.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:25.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:25.803 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260508 00:03:25.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:25.803 | INFO | RT_METRICS : 41.6%, 1.0, 21.9ms, 0 | 20260508 00:03:25.803 | INFO | PRE_HQ : 65.2%, 1.7, 56.7ms, 0 | 20260508 00:03:25.803 | INFO | BAZWRITER : 68.9%, 3.7, 131.4ms, 0 | 20260508 00:03:25.803 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 00:03:25.804 | INFO | Uploaded 36GiB to GPU in 2.52777s (14.2418GiB/s) | 20260508 00:03:25.804 | INFO | Downloaded 38GiB to GPU in 2.69087s (14.1218GiB/s) | 20260508 00:03:30.941 | INFO | Analyzing chunk frames = [3654144,3654656) | 20260508 00:03:30.941 | INFO | MemInfo: { | 20260508 00:03:30.941 | INFO | "gpu" : " 17 GB", | 20260508 00:03:30.941 | INFO | "huge" : "", | 20260508 00:03:30.941 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:30.941 | INFO | "malloc" : " 14 GB", | 20260508 00:03:30.941 | INFO | "pinned" : " 29 GB" | 20260508 00:03:30.941 | INFO | } | 20260508 00:03:30.941 | INFO | Perf report for the last 5.13827 seconds: | 20260508 00:03:30.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:30.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.941 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 00:03:30.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.942 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:03:30.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:30.942 | INFO | Uploaded 36GiB to GPU in 2.59782s (13.8578GiB/s) | 20260508 00:03:30.942 | INFO | Downloaded 34GiB to GPU in 2.46947s (13.7681GiB/s) | 20260508 00:03:36.081 | INFO | Analyzing chunk frames = [3654656,3655168) | 20260508 00:03:36.081 | INFO | MemInfo: { | 20260508 00:03:36.081 | INFO | "gpu" : " 17 GB", | 20260508 00:03:36.081 | INFO | "huge" : "", | 20260508 00:03:36.081 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:36.081 | INFO | "malloc" : " 14 GB", | 20260508 00:03:36.081 | INFO | "pinned" : " 29 GB" | 20260508 00:03:36.081 | INFO | } | 20260508 00:03:36.081 | INFO | Perf report for the last 5.13969 seconds: | 20260508 00:03:36.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:36.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 00:03:36.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:03:36.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:36.081 | INFO | Uploaded 36GiB to GPU in 2.52564s (14.2538GiB/s) | 20260508 00:03:36.081 | INFO | Downloaded 34GiB to GPU in 2.47061s (13.7618GiB/s) | 20260508 00:03:41.218 | INFO | Analyzing chunk frames = [3655168,3655680) | 20260508 00:03:41.218 | INFO | MemInfo: { | 20260508 00:03:41.218 | INFO | "gpu" : " 17 GB", | 20260508 00:03:41.218 | INFO | "huge" : "", | 20260508 00:03:41.218 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:41.218 | INFO | "malloc" : " 14 GB", | 20260508 00:03:41.218 | INFO | "pinned" : " 29 GB" | 20260508 00:03:41.219 | INFO | } | 20260508 00:03:41.219 | INFO | Perf report for the last 5.13769 seconds: | 20260508 00:03:41.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:41.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:03:41.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:03:41.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:41.219 | INFO | Uploaded 36GiB to GPU in 2.5466s (14.1365GiB/s) | 20260508 00:03:41.219 | INFO | Downloaded 34GiB to GPU in 2.47048s (13.7625GiB/s) | 20260508 00:03:46.352 | INFO | Analyzing chunk frames = [3655680,3656192) | 20260508 00:03:46.352 | INFO | MemInfo: { | 20260508 00:03:46.352 | INFO | "gpu" : " 17 GB", | 20260508 00:03:46.352 | INFO | "huge" : "", | 20260508 00:03:46.352 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:46.352 | INFO | "malloc" : " 14 GB", | 20260508 00:03:46.352 | INFO | "pinned" : " 29 GB" | 20260508 00:03:46.352 | INFO | } | 20260508 00:03:46.352 | INFO | Perf report for the last 5.13347 seconds: | 20260508 00:03:46.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:46.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.352 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 00:03:46.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.352 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 00:03:46.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:46.353 | INFO | Uploaded 36GiB to GPU in 2.35942s (15.258GiB/s) | 20260508 00:03:46.353 | INFO | Downloaded 34GiB to GPU in 2.50046s (13.5975GiB/s) | 20260508 00:03:51.492 | INFO | Analyzing chunk frames = [3656192,3656704) | 20260508 00:03:51.492 | INFO | MemInfo: { | 20260508 00:03:51.492 | INFO | "gpu" : " 17 GB", | 20260508 00:03:51.492 | INFO | "huge" : "", | 20260508 00:03:51.492 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:51.492 | INFO | "malloc" : " 14 GB", | 20260508 00:03:51.493 | INFO | "pinned" : " 29 GB" | 20260508 00:03:51.493 | INFO | } | 20260508 00:03:51.493 | INFO | Perf report for the last 5.14045 seconds: | 20260508 00:03:51.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:51.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260508 00:03:51.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:03:51.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:51.493 | INFO | Uploaded 36GiB to GPU in 2.52764s (14.2425GiB/s) | 20260508 00:03:51.493 | INFO | Downloaded 34GiB to GPU in 2.42716s (14.0081GiB/s) | 20260508 00:03:56.636 | INFO | Analyzing chunk frames = [3656704,3657216) | 20260508 00:03:56.636 | INFO | MemInfo: { | 20260508 00:03:56.636 | INFO | "gpu" : " 17 GB", | 20260508 00:03:56.636 | INFO | "huge" : "", | 20260508 00:03:56.636 | INFO | "huge_shared" : " 29 GB", | 20260508 00:03:56.636 | INFO | "malloc" : " 14 GB", | 20260508 00:03:56.636 | INFO | "pinned" : " 29 GB" | 20260508 00:03:56.636 | INFO | } | 20260508 00:03:56.636 | INFO | Perf report for the last 5.14343 seconds: | 20260508 00:03:56.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:03:56.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.636 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:03:56.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.636 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:03:56.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:03:56.637 | INFO | Uploaded 36GiB to GPU in 2.55067s (14.114GiB/s) | 20260508 00:03:56.637 | INFO | Downloaded 34GiB to GPU in 2.4664s (13.7853GiB/s) | 20260508 00:04:01.780 | INFO | Analyzing chunk frames = [3657216,3657728) | 20260508 00:04:01.780 | INFO | MemInfo: { | 20260508 00:04:01.780 | INFO | "gpu" : " 17 GB", | 20260508 00:04:01.780 | INFO | "huge" : "", | 20260508 00:04:01.780 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:01.780 | INFO | "malloc" : " 14 GB", | 20260508 00:04:01.780 | INFO | "pinned" : " 29 GB" | 20260508 00:04:01.780 | INFO | } | 20260508 00:04:01.780 | INFO | Perf report for the last 5.14371 seconds: | 20260508 00:04:01.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:01.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:04:01.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:04:01.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:01.780 | INFO | Uploaded 36GiB to GPU in 2.54947s (14.1206GiB/s) | 20260508 00:04:01.781 | INFO | Downloaded 34GiB to GPU in 2.45336s (13.8585GiB/s) | 20260508 00:04:01.877 | INFO | Starting a round of Baz IO | 20260508 00:04:05.337 | INFO | Populating RTMetrics report | 20260508 00:04:05.338 | INFO | RTMetrics report populated | 20260508 00:04:05.338 | INFO | Opening tmp RTMetrics json file | 20260508 00:04:05.354 | INFO | Serializing RTMetrics to json | 20260508 00:04:05.404 | INFO | Serialization done, swapping tmp file now... | 20260508 00:04:05.428 | INFO | RTMetrics done | 20260508 00:04:05.462 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.584 seconds | 20260508 00:04:05.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.34% / 0.00% / 57.65% | 20260508 00:04:05.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07745% / 7.94729e-06% / 5.96046e-05% | 20260508 00:04:05.462 | INFO | Baz IO overall: | 20260508 00:04:05.462 | INFO | BytesWritten : 1663511260461 | 20260508 00:04:05.462 | INFO | NumEvents : 805312791509 | 20260508 00:04:06.918 | INFO | Analyzing chunk frames = [3657728,3658240) | 20260508 00:04:06.918 | INFO | MemInfo: { | 20260508 00:04:06.918 | INFO | "gpu" : " 17 GB", | 20260508 00:04:06.918 | INFO | "huge" : "", | 20260508 00:04:06.918 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:06.918 | INFO | "malloc" : " 14 GB", | 20260508 00:04:06.918 | INFO | "pinned" : " 29 GB" | 20260508 00:04:06.919 | INFO | } | 20260508 00:04:06.919 | INFO | Perf report for the last 5.13871 seconds: | 20260508 00:04:06.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:06.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:06.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:06.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:06.919 | INFO | ANALYSIS : 68.2%, 3.0, 106.6ms, 0 | 20260508 00:04:06.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:06.919 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 00:04:06.919 | INFO | PRE_HQ : 65.6%, 1.7, 56.8ms, 0 | 20260508 00:04:06.919 | INFO | BAZWRITER : 69.2%, 3.6, 128.1ms, 0 | 20260508 00:04:06.919 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 00:04:06.919 | INFO | Uploaded 36GiB to GPU in 2.53337s (14.2103GiB/s) | 20260508 00:04:06.919 | INFO | Downloaded 38GiB to GPU in 2.69005s (14.1262GiB/s) | 20260508 00:04:12.054 | INFO | Analyzing chunk frames = [3658240,3658752) | 20260508 00:04:12.054 | INFO | MemInfo: { | 20260508 00:04:12.054 | INFO | "gpu" : " 17 GB", | 20260508 00:04:12.054 | INFO | "huge" : "", | 20260508 00:04:12.054 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:12.054 | INFO | "malloc" : " 14 GB", | 20260508 00:04:12.054 | INFO | "pinned" : " 29 GB" | 20260508 00:04:12.054 | INFO | } | 20260508 00:04:12.054 | INFO | Perf report for the last 5.13552 seconds: | 20260508 00:04:12.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:12.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.054 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:04:12.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.054 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:04:12.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:12.055 | INFO | Uploaded 36GiB to GPU in 2.52808s (14.2401GiB/s) | 20260508 00:04:12.055 | INFO | Downloaded 34GiB to GPU in 2.42525s (14.0192GiB/s) | 20260508 00:04:17.196 | INFO | Analyzing chunk frames = [3658752,3659264) | 20260508 00:04:17.196 | INFO | MemInfo: { | 20260508 00:04:17.196 | INFO | "gpu" : " 17 GB", | 20260508 00:04:17.196 | INFO | "huge" : "", | 20260508 00:04:17.196 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:17.196 | INFO | "malloc" : " 14 GB", | 20260508 00:04:17.196 | INFO | "pinned" : " 29 GB" | 20260508 00:04:17.196 | INFO | } | 20260508 00:04:17.196 | INFO | Perf report for the last 5.14202 seconds: | 20260508 00:04:17.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:17.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.196 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 00:04:17.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.196 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:04:17.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:17.197 | INFO | Uploaded 36GiB to GPU in 2.5099s (14.3432GiB/s) | 20260508 00:04:17.197 | INFO | Downloaded 34GiB to GPU in 2.48914s (13.6593GiB/s) | 20260508 00:04:22.333 | INFO | Analyzing chunk frames = [3659264,3659776) | 20260508 00:04:22.333 | INFO | MemInfo: { | 20260508 00:04:22.333 | INFO | "gpu" : " 17 GB", | 20260508 00:04:22.333 | INFO | "huge" : "", | 20260508 00:04:22.333 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:22.333 | INFO | "malloc" : " 14 GB", | 20260508 00:04:22.333 | INFO | "pinned" : " 29 GB" | 20260508 00:04:22.333 | INFO | } | 20260508 00:04:22.333 | INFO | Perf report for the last 5.13645 seconds: | 20260508 00:04:22.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:22.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:04:22.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:04:22.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:22.333 | INFO | Uploaded 36GiB to GPU in 2.52631s (14.25GiB/s) | 20260508 00:04:22.333 | INFO | Downloaded 34GiB to GPU in 2.49957s (13.6024GiB/s) | 20260508 00:04:27.472 | INFO | Analyzing chunk frames = [3659776,3660288) | 20260508 00:04:27.472 | INFO | MemInfo: { | 20260508 00:04:27.472 | INFO | "gpu" : " 17 GB", | 20260508 00:04:27.472 | INFO | "huge" : "", | 20260508 00:04:27.472 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:27.472 | INFO | "malloc" : " 14 GB", | 20260508 00:04:27.472 | INFO | "pinned" : " 29 GB" | 20260508 00:04:27.472 | INFO | } | 20260508 00:04:27.472 | INFO | Perf report for the last 5.1396 seconds: | 20260508 00:04:27.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:27.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.472 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:04:27.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.472 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:04:27.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:27.473 | INFO | Uploaded 36GiB to GPU in 2.3344s (15.4215GiB/s) | 20260508 00:04:27.473 | INFO | Downloaded 34GiB to GPU in 2.47103s (13.7595GiB/s) | 20260508 00:04:32.607 | INFO | Analyzing chunk frames = [3660288,3660800) | 20260508 00:04:32.607 | INFO | MemInfo: { | 20260508 00:04:32.607 | INFO | "gpu" : " 17 GB", | 20260508 00:04:32.607 | INFO | "huge" : "", | 20260508 00:04:32.607 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:32.607 | INFO | "malloc" : " 14 GB", | 20260508 00:04:32.608 | INFO | "pinned" : " 29 GB" | 20260508 00:04:32.608 | INFO | } | 20260508 00:04:32.608 | INFO | Perf report for the last 5.13547 seconds: | 20260508 00:04:32.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:32.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 00:04:32.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:04:32.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:32.608 | INFO | Uploaded 36GiB to GPU in 2.41535s (14.9047GiB/s) | 20260508 00:04:32.608 | INFO | Downloaded 34GiB to GPU in 2.39067s (14.2219GiB/s) | 20260508 00:04:37.745 | INFO | Analyzing chunk frames = [3660800,3661312) | 20260508 00:04:37.745 | INFO | MemInfo: { | 20260508 00:04:37.745 | INFO | "gpu" : " 17 GB", | 20260508 00:04:37.745 | INFO | "huge" : "", | 20260508 00:04:37.745 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:37.745 | INFO | "malloc" : " 14 GB", | 20260508 00:04:37.745 | INFO | "pinned" : " 29 GB" | 20260508 00:04:37.745 | INFO | } | 20260508 00:04:37.745 | INFO | Perf report for the last 5.13758 seconds: | 20260508 00:04:37.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:37.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.745 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:04:37.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.745 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:04:37.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:37.746 | INFO | Uploaded 36GiB to GPU in 2.55599s (14.0846GiB/s) | 20260508 00:04:37.746 | INFO | Downloaded 34GiB to GPU in 2.45986s (13.8219GiB/s) | 20260508 00:04:42.891 | INFO | Analyzing chunk frames = [3661312,3661824) | 20260508 00:04:42.891 | INFO | MemInfo: { | 20260508 00:04:42.891 | INFO | "gpu" : " 17 GB", | 20260508 00:04:42.891 | INFO | "huge" : "", | 20260508 00:04:42.891 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:42.891 | INFO | "malloc" : " 14 GB", | 20260508 00:04:42.891 | INFO | "pinned" : " 29 GB" | 20260508 00:04:42.891 | INFO | } | 20260508 00:04:42.891 | INFO | Perf report for the last 5.14549 seconds: | 20260508 00:04:42.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:42.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:04:42.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:04:42.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:42.891 | INFO | Uploaded 36GiB to GPU in 2.51013s (14.3419GiB/s) | 20260508 00:04:42.891 | INFO | Downloaded 34GiB to GPU in 2.43615s (13.9565GiB/s) | 20260508 00:04:42.891 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:04:42.891 | INFO | Total mem usage: 16.7 GiB | 20260508 00:04:42.891 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:04:42.891 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.4%) | 20260508 00:04:42.891 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:04:42.891 | INFO | High Water Breakdown: | 20260508 00:04:42.891 | INFO | 59.5% : 8.6 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:04:42.891 | INFO | 40.3% : 5.83 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:04:42.891 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:04:42.892 | INFO | ------------------------------------------ | 20260508 00:04:42.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:04:42.892 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:04:42.892 | INFO | Total mem usage: 26.5 GiB | 20260508 00:04:42.892 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:04:42.892 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:04:42.892 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:04:42.892 | INFO | High Water Breakdown: | 20260508 00:04:42.892 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:04:42.892 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:04:42.892 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:04:42.892 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:04:42.892 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:04:42.892 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:04:42.892 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:04:42.892 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:04:42.892 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:04:42.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:04:42.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:04:42.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:04:42.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:04:42.892 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:04:42.892 | INFO | ------------------------------------------ | 20260508 00:04:42.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:04:42.892 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:04:42.892 | INFO | Total mem usage: 27.8 GiB | 20260508 00:04:42.892 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:04:42.892 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:04:42.892 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:04:42.892 | INFO | High Water Breakdown: | 20260508 00:04:42.892 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:04:42.892 | INFO | ------------------------------------------ | 20260508 00:04:42.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:04:42.892 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:04:42.892 | INFO | Total mem usage: 15.7 GiB | 20260508 00:04:42.892 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:04:42.892 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:04:42.892 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:04:42.892 | INFO | High Water Breakdown: | 20260508 00:04:42.892 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:04:42.892 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:04:42.892 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:04:42.892 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:04:42.892 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:04:42.892 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:04:42.892 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:04:42.892 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:04:42.892 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:04:42.892 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:04:42.892 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:04:42.892 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:04:42.892 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:04:42.892 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:04:42.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:04:42.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:04:42.892 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:04:42.892 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:04:42.892 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:04:42.892 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:04:42.892 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:04:42.892 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:04:42.892 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:04:42.892 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:04:42.892 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:04:42.892 | INFO | ------------------------------------------ | 20260508 00:04:42.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:04:42.893 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:04:42.893 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:04:42.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:04:42.893 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.66e+01, 1.42e+01 / 1.67e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 9.89e-02, 1.16e-02 / 5.06e-03 | 20260508 00:04:42.893 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.45e+00, 5.87e-01 / 1.74e-01 | 20260508 00:04:42.893 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.05e-01 / 1.00e-01 | 20260508 00:04:42.893 | INFO | PerfReport: 16.13%, DME -- 4.22e+01 / 4.67e+01, 4.26e+00 / 1.77e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.57e+00 / 1.47e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.46e+00 / 2.78e+00, 4.31e-01 / 1.92e-01 | 20260508 00:04:42.893 | INFO | PerfReport: 3.40%, Metrics -- 1.11e+00 / 1.35e+00, 2.74e-01 / 1.24e-01 | 20260508 00:04:42.893 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.42e-01 / 3.79e-01, 5.43e-02 / 2.48e-02 | 20260508 00:04:42.893 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:04:42.893 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:04:42.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:04:42.893 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 24.42%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 30.62%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 2.34%, LockUpload -- 2.50e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 2.71%, LockDownload -- 2.90e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 00:04:42.893 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260508 00:04:42.988 | INFO | Starting a round of Baz IO | 20260508 00:04:46.455 | INFO | Populating RTMetrics report | 20260508 00:04:46.455 | INFO | RTMetrics report populated | 20260508 00:04:46.455 | INFO | Opening tmp RTMetrics json file | 20260508 00:04:46.484 | INFO | Serializing RTMetrics to json | 20260508 00:04:46.505 | INFO | Serialization done, swapping tmp file now... | 20260508 00:04:46.530 | INFO | RTMetrics done | 20260508 00:04:46.555 | INFO | Finished a round of Baz IO: Wrote 1.698 GiB in 3.566 seconds | 20260508 00:04:46.555 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.34% / 0.00% / 57.66% | 20260508 00:04:46.555 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07299% / 3.97364e-06% / 5.96046e-05% | 20260508 00:04:46.555 | INFO | Baz IO overall: | 20260508 00:04:46.555 | INFO | BytesWritten : 1665334552693 | 20260508 00:04:46.555 | INFO | NumEvents : 806190216628 | 20260508 00:04:48.033 | INFO | Analyzing chunk frames = [3661824,3662336) | 20260508 00:04:48.033 | INFO | MemInfo: { | 20260508 00:04:48.033 | INFO | "gpu" : " 17 GB", | 20260508 00:04:48.033 | INFO | "huge" : "", | 20260508 00:04:48.033 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:48.033 | INFO | "malloc" : " 14 GB", | 20260508 00:04:48.033 | INFO | "pinned" : " 29 GB" | 20260508 00:04:48.033 | INFO | } | 20260508 00:04:48.033 | INFO | Perf report for the last 5.14235 seconds: | 20260508 00:04:48.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:48.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:48.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:48.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:48.033 | INFO | ANALYSIS : 68.4%, 3.0, 106.8ms, 0 | 20260508 00:04:48.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:48.033 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 00:04:48.033 | INFO | PRE_HQ : 65.8%, 1.7, 56.7ms, 0 | 20260508 00:04:48.034 | INFO | BAZWRITER : 69.0%, 4.4, 155.1ms, 0 | 20260508 00:04:48.034 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 00:04:48.034 | INFO | Uploaded 36GiB to GPU in 2.59102s (13.8941GiB/s) | 20260508 00:04:48.034 | INFO | Downloaded 38GiB to GPU in 2.69917s (14.0784GiB/s) | 20260508 00:04:53.173 | INFO | Analyzing chunk frames = [3662336,3662848) | 20260508 00:04:53.173 | INFO | MemInfo: { | 20260508 00:04:53.173 | INFO | "gpu" : " 17 GB", | 20260508 00:04:53.173 | INFO | "huge" : "", | 20260508 00:04:53.173 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:53.173 | INFO | "malloc" : " 14 GB", | 20260508 00:04:53.173 | INFO | "pinned" : " 29 GB" | 20260508 00:04:53.173 | INFO | } | 20260508 00:04:53.173 | INFO | Perf report for the last 5.13969 seconds: | 20260508 00:04:53.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:53.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:04:53.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 00:04:53.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:53.173 | INFO | Uploaded 36GiB to GPU in 2.52079s (14.2812GiB/s) | 20260508 00:04:53.173 | INFO | Downloaded 34GiB to GPU in 2.4263s (14.0131GiB/s) | 20260508 00:04:58.303 | INFO | Analyzing chunk frames = [3662848,3663360) | 20260508 00:04:58.303 | INFO | MemInfo: { | 20260508 00:04:58.303 | INFO | "gpu" : " 17 GB", | 20260508 00:04:58.303 | INFO | "huge" : "", | 20260508 00:04:58.303 | INFO | "huge_shared" : " 29 GB", | 20260508 00:04:58.304 | INFO | "malloc" : " 14 GB", | 20260508 00:04:58.304 | INFO | "pinned" : " 29 GB" | 20260508 00:04:58.304 | INFO | } | 20260508 00:04:58.304 | INFO | Perf report for the last 5.13099 seconds: | 20260508 00:04:58.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:04:58.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.304 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260508 00:04:58.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.304 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 00:04:58.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:04:58.305 | INFO | Uploaded 36GiB to GPU in 2.51933s (14.2895GiB/s) | 20260508 00:04:58.305 | INFO | Downloaded 34GiB to GPU in 2.49996s (13.6002GiB/s) | 20260508 00:05:03.447 | INFO | Analyzing chunk frames = [3663360,3663872) | 20260508 00:05:03.447 | INFO | MemInfo: { | 20260508 00:05:03.447 | INFO | "gpu" : " 17 GB", | 20260508 00:05:03.447 | INFO | "huge" : "", | 20260508 00:05:03.448 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:03.448 | INFO | "malloc" : " 14 GB", | 20260508 00:05:03.448 | INFO | "pinned" : " 29 GB" | 20260508 00:05:03.448 | INFO | } | 20260508 00:05:03.448 | INFO | Perf report for the last 5.14381 seconds: | 20260508 00:05:03.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:03.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:05:03.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 00:05:03.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:03.448 | INFO | Uploaded 36GiB to GPU in 2.52588s (14.2525GiB/s) | 20260508 00:05:03.449 | INFO | Downloaded 34GiB to GPU in 2.50053s (13.5971GiB/s) | 20260508 00:05:08.586 | INFO | Analyzing chunk frames = [3663872,3664384) | 20260508 00:05:08.586 | INFO | MemInfo: { | 20260508 00:05:08.586 | INFO | "gpu" : " 17 GB", | 20260508 00:05:08.586 | INFO | "huge" : "", | 20260508 00:05:08.586 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:08.586 | INFO | "malloc" : " 14 GB", | 20260508 00:05:08.586 | INFO | "pinned" : " 29 GB" | 20260508 00:05:08.586 | INFO | } | 20260508 00:05:08.587 | INFO | Perf report for the last 5.13878 seconds: | 20260508 00:05:08.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:08.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:05:08.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 00:05:08.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:08.587 | INFO | Uploaded 36GiB to GPU in 2.35635s (15.2778GiB/s) | 20260508 00:05:08.587 | INFO | Downloaded 34GiB to GPU in 2.49888s (13.6061GiB/s) | 20260508 00:05:13.726 | INFO | Analyzing chunk frames = [3664384,3664896) | 20260508 00:05:13.726 | INFO | MemInfo: { | 20260508 00:05:13.726 | INFO | "gpu" : " 17 GB", | 20260508 00:05:13.726 | INFO | "huge" : "", | 20260508 00:05:13.726 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:13.726 | INFO | "malloc" : " 14 GB", | 20260508 00:05:13.726 | INFO | "pinned" : " 29 GB" | 20260508 00:05:13.726 | INFO | } | 20260508 00:05:13.726 | INFO | Perf report for the last 5.13947 seconds: | 20260508 00:05:13.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:13.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:05:13.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:05:13.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:13.726 | INFO | Uploaded 36GiB to GPU in 2.39128s (15.0547GiB/s) | 20260508 00:05:13.726 | INFO | Downloaded 34GiB to GPU in 2.4082s (14.1185GiB/s) | 20260508 00:05:18.867 | INFO | Analyzing chunk frames = [3664896,3665408) | 20260508 00:05:18.867 | INFO | MemInfo: { | 20260508 00:05:18.867 | INFO | "gpu" : " 17 GB", | 20260508 00:05:18.867 | INFO | "huge" : "", | 20260508 00:05:18.867 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:18.867 | INFO | "malloc" : " 14 GB", | 20260508 00:05:18.867 | INFO | "pinned" : " 29 GB" | 20260508 00:05:18.867 | INFO | } | 20260508 00:05:18.867 | INFO | Perf report for the last 5.141 seconds: | 20260508 00:05:18.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:18.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.867 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:05:18.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.867 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:05:18.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:18.868 | INFO | Uploaded 36GiB to GPU in 2.54926s (14.1217GiB/s) | 20260508 00:05:18.868 | INFO | Downloaded 34GiB to GPU in 2.47456s (13.7398GiB/s) | 20260508 00:05:24.001 | INFO | Analyzing chunk frames = [3665408,3665920) | 20260508 00:05:24.001 | INFO | MemInfo: { | 20260508 00:05:24.001 | INFO | "gpu" : " 17 GB", | 20260508 00:05:24.001 | INFO | "huge" : "", | 20260508 00:05:24.001 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:24.002 | INFO | "malloc" : " 14 GB", | 20260508 00:05:24.002 | INFO | "pinned" : " 29 GB" | 20260508 00:05:24.002 | INFO | } | 20260508 00:05:24.002 | INFO | Perf report for the last 5.13464 seconds: | 20260508 00:05:24.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:24.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 00:05:24.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:05:24.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:24.002 | INFO | Uploaded 36GiB to GPU in 2.49594s (14.4234GiB/s) | 20260508 00:05:24.002 | INFO | Downloaded 34GiB to GPU in 2.44931s (13.8814GiB/s) | 20260508 00:05:24.100 | INFO | Starting a round of Baz IO | 20260508 00:05:27.569 | INFO | Populating RTMetrics report | 20260508 00:05:27.570 | INFO | RTMetrics report populated | 20260508 00:05:27.570 | INFO | Opening tmp RTMetrics json file | 20260508 00:05:27.585 | INFO | Serializing RTMetrics to json | 20260508 00:05:27.590 | INFO | Serialization done, swapping tmp file now... | 20260508 00:05:27.591 | INFO | RTMetrics done | 20260508 00:05:27.666 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.565 seconds | 20260508 00:05:27.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.33% / 0.00% / 57.66% | 20260508 00:05:27.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07222% / 0% / 5.5631e-05% | 20260508 00:05:27.666 | INFO | Baz IO overall: | 20260508 00:05:27.666 | INFO | BytesWritten : 1667158490873 | 20260508 00:05:27.666 | INFO | NumEvents : 807068082828 | 20260508 00:05:29.143 | INFO | Analyzing chunk frames = [3665920,3666432) | 20260508 00:05:29.143 | INFO | MemInfo: { | 20260508 00:05:29.143 | INFO | "gpu" : " 17 GB", | 20260508 00:05:29.143 | INFO | "huge" : "", | 20260508 00:05:29.143 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:29.143 | INFO | "malloc" : " 14 GB", | 20260508 00:05:29.143 | INFO | "pinned" : " 29 GB" | 20260508 00:05:29.143 | INFO | } | 20260508 00:05:29.143 | INFO | Perf report for the last 5.14134 seconds: | 20260508 00:05:29.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:29.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:29.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:29.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:29.143 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 00:05:29.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:29.143 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 00:05:29.143 | INFO | PRE_HQ : 65.3%, 1.7, 56.5ms, 0 | 20260508 00:05:29.143 | INFO | BAZWRITER : 69.0%, 4.2, 149.7ms, 0 | 20260508 00:05:29.143 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 00:05:29.144 | INFO | Uploaded 36GiB to GPU in 2.56904s (14.013GiB/s) | 20260508 00:05:29.144 | INFO | Downloaded 38GiB to GPU in 2.68238s (14.1665GiB/s) | 20260508 00:05:34.285 | INFO | Analyzing chunk frames = [3666432,3666944) | 20260508 00:05:34.285 | INFO | MemInfo: { | 20260508 00:05:34.285 | INFO | "gpu" : " 17 GB", | 20260508 00:05:34.285 | INFO | "huge" : "", | 20260508 00:05:34.285 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:34.285 | INFO | "malloc" : " 14 GB", | 20260508 00:05:34.285 | INFO | "pinned" : " 29 GB" | 20260508 00:05:34.285 | INFO | } | 20260508 00:05:34.285 | INFO | Perf report for the last 5.14217 seconds: | 20260508 00:05:34.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:34.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.285 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:05:34.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.285 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:05:34.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:34.286 | INFO | Uploaded 36GiB to GPU in 2.54288s (14.1572GiB/s) | 20260508 00:05:34.286 | INFO | Downloaded 34GiB to GPU in 2.40598s (14.1315GiB/s) | 20260508 00:05:39.420 | INFO | Analyzing chunk frames = [3666944,3667456) | 20260508 00:05:39.420 | INFO | MemInfo: { | 20260508 00:05:39.420 | INFO | "gpu" : " 17 GB", | 20260508 00:05:39.420 | INFO | "huge" : "", | 20260508 00:05:39.420 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:39.420 | INFO | "malloc" : " 14 GB", | 20260508 00:05:39.420 | INFO | "pinned" : " 29 GB" | 20260508 00:05:39.420 | INFO | } | 20260508 00:05:39.420 | INFO | Perf report for the last 5.13514 seconds: | 20260508 00:05:39.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:39.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.420 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:05:39.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.420 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:05:39.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:39.421 | INFO | Uploaded 36GiB to GPU in 2.51818s (14.296GiB/s) | 20260508 00:05:39.421 | INFO | Downloaded 34GiB to GPU in 2.47564s (13.7338GiB/s) | 20260508 00:05:44.560 | INFO | Analyzing chunk frames = [3667456,3667968) | 20260508 00:05:44.560 | INFO | MemInfo: { | 20260508 00:05:44.560 | INFO | "gpu" : " 17 GB", | 20260508 00:05:44.560 | INFO | "huge" : "", | 20260508 00:05:44.560 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:44.560 | INFO | "malloc" : " 14 GB", | 20260508 00:05:44.560 | INFO | "pinned" : " 29 GB" | 20260508 00:05:44.560 | INFO | } | 20260508 00:05:44.560 | INFO | Perf report for the last 5.13957 seconds: | 20260508 00:05:44.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:44.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.560 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260508 00:05:44.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.560 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:05:44.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:44.561 | INFO | Uploaded 36GiB to GPU in 2.56939s (14.0111GiB/s) | 20260508 00:05:44.561 | INFO | Downloaded 34GiB to GPU in 2.49069s (13.6508GiB/s) | 20260508 00:05:49.702 | INFO | Analyzing chunk frames = [3667968,3668480) | 20260508 00:05:49.702 | INFO | MemInfo: { | 20260508 00:05:49.702 | INFO | "gpu" : " 17 GB", | 20260508 00:05:49.702 | INFO | "huge" : "", | 20260508 00:05:49.702 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:49.702 | INFO | "malloc" : " 14 GB", | 20260508 00:05:49.702 | INFO | "pinned" : " 29 GB" | 20260508 00:05:49.702 | INFO | } | 20260508 00:05:49.702 | INFO | Perf report for the last 5.14253 seconds: | 20260508 00:05:49.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:49.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.703 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 00:05:49.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.703 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:05:49.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:49.703 | INFO | Uploaded 36GiB to GPU in 2.30069s (15.6475GiB/s) | 20260508 00:05:49.703 | INFO | Downloaded 34GiB to GPU in 2.47177s (13.7553GiB/s) | 20260508 00:05:54.840 | INFO | Analyzing chunk frames = [3668480,3668992) | 20260508 00:05:54.840 | INFO | MemInfo: { | 20260508 00:05:54.840 | INFO | "gpu" : " 17 GB", | 20260508 00:05:54.840 | INFO | "huge" : "", | 20260508 00:05:54.840 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:54.840 | INFO | "malloc" : " 14 GB", | 20260508 00:05:54.840 | INFO | "pinned" : " 29 GB" | 20260508 00:05:54.840 | INFO | } | 20260508 00:05:54.840 | INFO | Perf report for the last 5.1376 seconds: | 20260508 00:05:54.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:54.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.840 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:05:54.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.840 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:05:54.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:54.841 | INFO | Uploaded 36GiB to GPU in 2.38346s (15.1041GiB/s) | 20260508 00:05:54.841 | INFO | Downloaded 34GiB to GPU in 2.41114s (14.1012GiB/s) | 20260508 00:05:59.981 | INFO | Analyzing chunk frames = [3668992,3669504) | 20260508 00:05:59.981 | INFO | MemInfo: { | 20260508 00:05:59.981 | INFO | "gpu" : " 17 GB", | 20260508 00:05:59.981 | INFO | "huge" : "", | 20260508 00:05:59.981 | INFO | "huge_shared" : " 29 GB", | 20260508 00:05:59.981 | INFO | "malloc" : " 14 GB", | 20260508 00:05:59.981 | INFO | "pinned" : " 29 GB" | 20260508 00:05:59.981 | INFO | } | 20260508 00:05:59.981 | INFO | Perf report for the last 5.14097 seconds: | 20260508 00:05:59.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:05:59.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:05:59.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:05:59.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:05:59.981 | INFO | Uploaded 36GiB to GPU in 2.52271s (14.2703GiB/s) | 20260508 00:05:59.981 | INFO | Downloaded 34GiB to GPU in 2.45326s (13.8591GiB/s) | 20260508 00:06:05.120 | INFO | Analyzing chunk frames = [3669504,3670016) | 20260508 00:06:05.120 | INFO | MemInfo: { | 20260508 00:06:05.120 | INFO | "gpu" : " 17 GB", | 20260508 00:06:05.120 | INFO | "huge" : "", | 20260508 00:06:05.120 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:05.120 | INFO | "malloc" : " 14 GB", | 20260508 00:06:05.120 | INFO | "pinned" : " 29 GB" | 20260508 00:06:05.120 | INFO | } | 20260508 00:06:05.120 | INFO | Perf report for the last 5.13924 seconds: | 20260508 00:06:05.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:05.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:06:05.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:06:05.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:05.121 | INFO | Uploaded 36GiB to GPU in 2.51748s (14.3GiB/s) | 20260508 00:06:05.121 | INFO | Downloaded 34GiB to GPU in 2.44753s (13.8916GiB/s) | 20260508 00:06:05.121 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:06:05.121 | INFO | Total mem usage: 16.7 GiB | 20260508 00:06:05.121 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:06:05.121 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 00:06:05.121 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:06:05.121 | INFO | High Water Breakdown: | 20260508 00:06:05.121 | INFO | 60.1% : 8.97 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:06:05.121 | INFO | 39.8% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:06:05.121 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:06:05.121 | INFO | ------------------------------------------ | 20260508 00:06:05.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:06:05.121 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:06:05.121 | INFO | Total mem usage: 26.5 GiB | 20260508 00:06:05.121 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:06:05.121 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:06:05.121 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:06:05.121 | INFO | High Water Breakdown: | 20260508 00:06:05.121 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:06:05.121 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:06:05.121 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:06:05.121 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:06:05.121 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:06:05.121 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:06:05.121 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:06:05.121 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:06:05.121 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:06:05.121 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:06:05.121 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:06:05.121 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:06:05.121 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:06:05.121 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:06:05.121 | INFO | ------------------------------------------ | 20260508 00:06:05.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:06:05.121 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:06:05.121 | INFO | Total mem usage: 27.8 GiB | 20260508 00:06:05.121 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:06:05.121 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:06:05.121 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:06:05.121 | INFO | High Water Breakdown: | 20260508 00:06:05.121 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:06:05.121 | INFO | ------------------------------------------ | 20260508 00:06:05.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:06:05.122 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:06:05.122 | INFO | Total mem usage: 15.7 GiB | 20260508 00:06:05.122 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:06:05.122 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:06:05.122 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:06:05.122 | INFO | High Water Breakdown: | 20260508 00:06:05.122 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:06:05.122 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:06:05.122 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:06:05.122 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:06:05.122 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:06:05.122 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:06:05.122 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:06:05.122 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:06:05.122 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:06:05.122 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:06:05.122 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:06:05.122 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:06:05.122 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:06:05.122 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:06:05.122 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:06:05.122 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:06:05.122 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:06:05.122 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:06:05.122 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:06:05.122 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:06:05.122 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:06:05.122 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:06:05.122 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:06:05.122 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:06:05.122 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:06:05.122 | INFO | ------------------------------------------ | 20260508 00:06:05.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:06:05.122 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:06:05.122 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:06:05.122 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:06:05.122 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.43e+01 / 1.69e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.04e-01, 1.35e-02 / 5.51e-03 | 20260508 00:06:05.122 | INFO | PerfReport: 12.20%, Baseline -- 4.01e+00 / 4.46e+00, 5.93e-01 / 2.40e-01 | 20260508 00:06:05.122 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.21e-01 | 20260508 00:06:05.122 | INFO | PerfReport: 16.06%, DME -- 4.22e+01 / 4.59e+01, 4.10e+00 / 1.79e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.65e+00 / 1.48e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.46e+00 / 2.77e+00, 4.32e-01 / 2.20e-01 | 20260508 00:06:05.122 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.33e+00, 2.78e-01 / 1.52e-01 | 20260508 00:06:05.122 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.46e-01 / 3.86e-01, 5.51e-02 / 3.06e-02 | 20260508 00:06:05.122 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:06:05.122 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:06:05.122 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:06:05.122 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 24.31%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 24.27%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 30.77%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 2.18%, LockUpload -- 2.33e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 2.67%, LockDownload -- 2.85e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 15.79%, LockCompute -- 1.69e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 00:06:05.122 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260508 00:06:05.217 | INFO | Starting a round of Baz IO | 20260508 00:06:08.681 | INFO | Populating RTMetrics report | 20260508 00:06:08.681 | INFO | RTMetrics report populated | 20260508 00:06:08.681 | INFO | Opening tmp RTMetrics json file | 20260508 00:06:08.706 | INFO | Serializing RTMetrics to json | 20260508 00:06:08.714 | INFO | Serialization done, swapping tmp file now... | 20260508 00:06:08.733 | INFO | RTMetrics done | 20260508 00:06:08.780 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.563 seconds | 20260508 00:06:08.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.33% / 0.00% / 57.67% | 20260508 00:06:08.780 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07209% / 1.19209e-05% / 4.37101e-05% | 20260508 00:06:08.781 | INFO | Baz IO overall: | 20260508 00:06:08.781 | INFO | BytesWritten : 1668983094547 | 20260508 00:06:08.781 | INFO | NumEvents : 807946669868 | 20260508 00:06:10.257 | INFO | Analyzing chunk frames = [3670016,3670528) | 20260508 00:06:10.257 | INFO | MemInfo: { | 20260508 00:06:10.257 | INFO | "gpu" : " 17 GB", | 20260508 00:06:10.257 | INFO | "huge" : "", | 20260508 00:06:10.257 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:10.257 | INFO | "malloc" : " 14 GB", | 20260508 00:06:10.257 | INFO | "pinned" : " 29 GB" | 20260508 00:06:10.257 | INFO | } | 20260508 00:06:10.257 | INFO | Perf report for the last 5.13687 seconds: | 20260508 00:06:10.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:10.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:10.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:10.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:10.257 | INFO | ANALYSIS : 68.5%, 3.0, 106.8ms, 0 | 20260508 00:06:10.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:10.257 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260508 00:06:10.257 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260508 00:06:10.257 | INFO | BAZWRITER : 68.9%, 3.7, 129.7ms, 0 | 20260508 00:06:10.257 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:06:10.258 | INFO | Uploaded 36GiB to GPU in 2.52103s (14.2799GiB/s) | 20260508 00:06:10.258 | INFO | Downloaded 38GiB to GPU in 2.68183s (14.1694GiB/s) | 20260508 00:06:15.399 | INFO | Analyzing chunk frames = [3670528,3671040) | 20260508 00:06:15.399 | INFO | MemInfo: { | 20260508 00:06:15.399 | INFO | "gpu" : " 17 GB", | 20260508 00:06:15.399 | INFO | "huge" : "", | 20260508 00:06:15.399 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:15.399 | INFO | "malloc" : " 14 GB", | 20260508 00:06:15.399 | INFO | "pinned" : " 29 GB" | 20260508 00:06:15.399 | INFO | } | 20260508 00:06:15.399 | INFO | Perf report for the last 5.14211 seconds: | 20260508 00:06:15.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:15.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.399 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 00:06:15.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.399 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260508 00:06:15.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:15.400 | INFO | Uploaded 36GiB to GPU in 2.55999s (14.0626GiB/s) | 20260508 00:06:15.400 | INFO | Downloaded 34GiB to GPU in 2.48239s (13.6965GiB/s) | 20260508 00:06:20.536 | INFO | Analyzing chunk frames = [3671040,3671552) | 20260508 00:06:20.536 | INFO | MemInfo: { | 20260508 00:06:20.536 | INFO | "gpu" : " 17 GB", | 20260508 00:06:20.536 | INFO | "huge" : "", | 20260508 00:06:20.536 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:20.536 | INFO | "malloc" : " 14 GB", | 20260508 00:06:20.536 | INFO | "pinned" : " 29 GB" | 20260508 00:06:20.536 | INFO | } | 20260508 00:06:20.536 | INFO | Perf report for the last 5.13658 seconds: | 20260508 00:06:20.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:20.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:06:20.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:06:20.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:20.536 | INFO | Uploaded 36GiB to GPU in 2.514s (14.3198GiB/s) | 20260508 00:06:20.536 | INFO | Downloaded 34GiB to GPU in 2.47637s (13.7298GiB/s) | 20260508 00:06:25.671 | INFO | Analyzing chunk frames = [3671552,3672064) | 20260508 00:06:25.671 | INFO | MemInfo: { | 20260508 00:06:25.671 | INFO | "gpu" : " 17 GB", | 20260508 00:06:25.671 | INFO | "huge" : "", | 20260508 00:06:25.671 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:25.671 | INFO | "malloc" : " 14 GB", | 20260508 00:06:25.671 | INFO | "pinned" : " 29 GB" | 20260508 00:06:25.671 | INFO | } | 20260508 00:06:25.671 | INFO | Perf report for the last 5.13516 seconds: | 20260508 00:06:25.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:25.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.671 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 00:06:25.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.671 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:06:25.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:25.672 | INFO | Uploaded 36GiB to GPU in 2.49248s (14.4435GiB/s) | 20260508 00:06:25.672 | INFO | Downloaded 34GiB to GPU in 2.48546s (13.6795GiB/s) | 20260508 00:06:30.811 | INFO | Analyzing chunk frames = [3672064,3672576) | 20260508 00:06:30.811 | INFO | MemInfo: { | 20260508 00:06:30.811 | INFO | "gpu" : " 17 GB", | 20260508 00:06:30.811 | INFO | "huge" : "", | 20260508 00:06:30.811 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:30.811 | INFO | "malloc" : " 14 GB", | 20260508 00:06:30.811 | INFO | "pinned" : " 29 GB" | 20260508 00:06:30.812 | INFO | } | 20260508 00:06:30.812 | INFO | Perf report for the last 5.14046 seconds: | 20260508 00:06:30.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:30.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:06:30.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 00:06:30.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:30.812 | INFO | Uploaded 36GiB to GPU in 2.4308s (14.8099GiB/s) | 20260508 00:06:30.812 | INFO | Downloaded 34GiB to GPU in 2.49716s (13.6155GiB/s) | 20260508 00:06:35.954 | INFO | Analyzing chunk frames = [3672576,3673088) | 20260508 00:06:35.954 | INFO | MemInfo: { | 20260508 00:06:35.954 | INFO | "gpu" : " 17 GB", | 20260508 00:06:35.954 | INFO | "huge" : "", | 20260508 00:06:35.954 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:35.954 | INFO | "malloc" : " 14 GB", | 20260508 00:06:35.954 | INFO | "pinned" : " 29 GB" | 20260508 00:06:35.954 | INFO | } | 20260508 00:06:35.954 | INFO | Perf report for the last 5.14239 seconds: | 20260508 00:06:35.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:35.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 00:06:35.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:06:35.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:35.954 | INFO | Uploaded 36GiB to GPU in 2.44297s (14.7361GiB/s) | 20260508 00:06:35.954 | INFO | Downloaded 34GiB to GPU in 2.39965s (14.1687GiB/s) | 20260508 00:06:41.098 | INFO | Analyzing chunk frames = [3673088,3673600) | 20260508 00:06:41.098 | INFO | MemInfo: { | 20260508 00:06:41.098 | INFO | "gpu" : " 17 GB", | 20260508 00:06:41.098 | INFO | "huge" : "", | 20260508 00:06:41.098 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:41.098 | INFO | "malloc" : " 14 GB", | 20260508 00:06:41.098 | INFO | "pinned" : " 29 GB" | 20260508 00:06:41.098 | INFO | } | 20260508 00:06:41.099 | INFO | Perf report for the last 5.14453 seconds: | 20260508 00:06:41.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:41.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 00:06:41.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:06:41.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:41.099 | INFO | Uploaded 36GiB to GPU in 2.55729s (14.0774GiB/s) | 20260508 00:06:41.099 | INFO | Downloaded 34GiB to GPU in 2.44662s (13.8967GiB/s) | 20260508 00:06:46.237 | INFO | Analyzing chunk frames = [3673600,3674112) | 20260508 00:06:46.237 | INFO | MemInfo: { | 20260508 00:06:46.237 | INFO | "gpu" : " 17 GB", | 20260508 00:06:46.237 | INFO | "huge" : "", | 20260508 00:06:46.237 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:46.237 | INFO | "malloc" : " 14 GB", | 20260508 00:06:46.237 | INFO | "pinned" : " 29 GB" | 20260508 00:06:46.237 | INFO | } | 20260508 00:06:46.237 | INFO | Perf report for the last 5.13871 seconds: | 20260508 00:06:46.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:46.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.237 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:06:46.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.237 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:06:46.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:46.238 | INFO | Uploaded 36GiB to GPU in 2.55085s (14.113GiB/s) | 20260508 00:06:46.238 | INFO | Downloaded 34GiB to GPU in 2.44798s (13.889GiB/s) | 20260508 00:06:46.348 | INFO | Starting a round of Baz IO | 20260508 00:06:49.823 | INFO | Populating RTMetrics report | 20260508 00:06:49.823 | INFO | RTMetrics report populated | 20260508 00:06:49.823 | INFO | Opening tmp RTMetrics json file | 20260508 00:06:49.854 | INFO | Serializing RTMetrics to json | 20260508 00:06:49.912 | INFO | Serialization done, swapping tmp file now... | 20260508 00:06:49.916 | INFO | RTMetrics done | 20260508 00:06:49.939 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.591 seconds | 20260508 00:06:49.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.33% / 0.00% / 57.67% | 20260508 00:06:49.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07004% / 3.97364e-06% / 5.5631e-05% | 20260508 00:06:49.940 | INFO | Baz IO overall: | 20260508 00:06:49.940 | INFO | BytesWritten : 1670807120503 | 20260508 00:06:49.940 | INFO | NumEvents : 808824599750 | 20260508 00:06:51.370 | INFO | Analyzing chunk frames = [3674112,3674624) | 20260508 00:06:51.370 | INFO | MemInfo: { | 20260508 00:06:51.370 | INFO | "gpu" : " 17 GB", | 20260508 00:06:51.370 | INFO | "huge" : "", | 20260508 00:06:51.370 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:51.370 | INFO | "malloc" : " 14 GB", | 20260508 00:06:51.370 | INFO | "pinned" : " 29 GB" | 20260508 00:06:51.370 | INFO | } | 20260508 00:06:51.370 | INFO | Perf report for the last 5.13296 seconds: | 20260508 00:06:51.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:51.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:51.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:51.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:51.370 | INFO | ANALYSIS : 69.0%, 2.9, 105.3ms, 0 | 20260508 00:06:51.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:51.370 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260508 00:06:51.370 | INFO | PRE_HQ : 67.6%, 1.8, 62.1ms, 0 | 20260508 00:06:51.370 | INFO | BAZWRITER : 69.5%, 3.9, 139.5ms, 0 | 20260508 00:06:51.370 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 00:06:51.371 | INFO | Uploaded 36GiB to GPU in 2.5871s (13.9152GiB/s) | 20260508 00:06:51.371 | INFO | Downloaded 38GiB to GPU in 2.70759s (14.0346GiB/s) | 20260508 00:06:56.511 | INFO | Analyzing chunk frames = [3674624,3675136) | 20260508 00:06:56.511 | INFO | MemInfo: { | 20260508 00:06:56.511 | INFO | "gpu" : " 17 GB", | 20260508 00:06:56.511 | INFO | "huge" : "", | 20260508 00:06:56.511 | INFO | "huge_shared" : " 29 GB", | 20260508 00:06:56.511 | INFO | "malloc" : " 14 GB", | 20260508 00:06:56.511 | INFO | "pinned" : " 29 GB" | 20260508 00:06:56.511 | INFO | } | 20260508 00:06:56.511 | INFO | Perf report for the last 5.14086 seconds: | 20260508 00:06:56.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:06:56.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.511 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:06:56.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.511 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:06:56.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:06:56.512 | INFO | Uploaded 36GiB to GPU in 2.56425s (14.0392GiB/s) | 20260508 00:06:56.512 | INFO | Downloaded 34GiB to GPU in 2.49664s (13.6183GiB/s) | 20260508 00:07:01.652 | INFO | Analyzing chunk frames = [3675136,3675648) | 20260508 00:07:01.652 | INFO | MemInfo: { | 20260508 00:07:01.652 | INFO | "gpu" : " 17 GB", | 20260508 00:07:01.652 | INFO | "huge" : "", | 20260508 00:07:01.652 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:01.652 | INFO | "malloc" : " 14 GB", | 20260508 00:07:01.652 | INFO | "pinned" : " 29 GB" | 20260508 00:07:01.653 | INFO | } | 20260508 00:07:01.653 | INFO | Perf report for the last 5.14152 seconds: | 20260508 00:07:01.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:01.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 00:07:01.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:07:01.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:01.653 | INFO | Uploaded 36GiB to GPU in 2.51003s (14.3424GiB/s) | 20260508 00:07:01.653 | INFO | Downloaded 34GiB to GPU in 2.48717s (13.6702GiB/s) | 20260508 00:07:06.794 | INFO | Analyzing chunk frames = [3675648,3676160) | 20260508 00:07:06.794 | INFO | MemInfo: { | 20260508 00:07:06.794 | INFO | "gpu" : " 17 GB", | 20260508 00:07:06.794 | INFO | "huge" : "", | 20260508 00:07:06.795 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:06.795 | INFO | "malloc" : " 14 GB", | 20260508 00:07:06.795 | INFO | "pinned" : " 29 GB" | 20260508 00:07:06.795 | INFO | } | 20260508 00:07:06.795 | INFO | Perf report for the last 5.14201 seconds: | 20260508 00:07:06.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:06.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 00:07:06.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:07:06.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:06.795 | INFO | Uploaded 36GiB to GPU in 2.48628s (14.4795GiB/s) | 20260508 00:07:06.795 | INFO | Downloaded 34GiB to GPU in 2.4986s (13.6076GiB/s) | 20260508 00:07:11.931 | INFO | Analyzing chunk frames = [3676160,3676672) | 20260508 00:07:11.931 | INFO | MemInfo: { | 20260508 00:07:11.931 | INFO | "gpu" : " 17 GB", | 20260508 00:07:11.931 | INFO | "huge" : "", | 20260508 00:07:11.931 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:11.931 | INFO | "malloc" : " 14 GB", | 20260508 00:07:11.931 | INFO | "pinned" : " 29 GB" | 20260508 00:07:11.931 | INFO | } | 20260508 00:07:11.931 | INFO | Perf report for the last 5.1367 seconds: | 20260508 00:07:11.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:11.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.931 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 00:07:11.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.932 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:07:11.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:11.932 | INFO | Uploaded 36GiB to GPU in 2.30608s (15.6109GiB/s) | 20260508 00:07:11.932 | INFO | Downloaded 34GiB to GPU in 2.49502s (13.6271GiB/s) | 20260508 00:07:17.066 | INFO | Analyzing chunk frames = [3676672,3677184) | 20260508 00:07:17.066 | INFO | MemInfo: { | 20260508 00:07:17.066 | INFO | "gpu" : " 17 GB", | 20260508 00:07:17.066 | INFO | "huge" : "", | 20260508 00:07:17.066 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:17.066 | INFO | "malloc" : " 14 GB", | 20260508 00:07:17.066 | INFO | "pinned" : " 29 GB" | 20260508 00:07:17.066 | INFO | } | 20260508 00:07:17.067 | INFO | Perf report for the last 5.13523 seconds: | 20260508 00:07:17.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:17.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 00:07:17.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:07:17.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:17.067 | INFO | Uploaded 36GiB to GPU in 2.38222s (15.112GiB/s) | 20260508 00:07:17.067 | INFO | Downloaded 34GiB to GPU in 2.40469s (14.139GiB/s) | 20260508 00:07:22.209 | INFO | Analyzing chunk frames = [3677184,3677696) | 20260508 00:07:22.209 | INFO | MemInfo: { | 20260508 00:07:22.209 | INFO | "gpu" : " 17 GB", | 20260508 00:07:22.209 | INFO | "huge" : "", | 20260508 00:07:22.209 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:22.209 | INFO | "malloc" : " 14 GB", | 20260508 00:07:22.209 | INFO | "pinned" : " 29 GB" | 20260508 00:07:22.209 | INFO | } | 20260508 00:07:22.209 | INFO | Perf report for the last 5.14278 seconds: | 20260508 00:07:22.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:22.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.209 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260508 00:07:22.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.210 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 00:07:22.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:22.210 | INFO | Uploaded 36GiB to GPU in 2.49599s (14.4231GiB/s) | 20260508 00:07:22.210 | INFO | Downloaded 34GiB to GPU in 2.4551s (13.8487GiB/s) | 20260508 00:07:27.351 | INFO | Analyzing chunk frames = [3677696,3678208) | 20260508 00:07:27.351 | INFO | MemInfo: { | 20260508 00:07:27.351 | INFO | "gpu" : " 17 GB", | 20260508 00:07:27.351 | INFO | "huge" : "", | 20260508 00:07:27.351 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:27.351 | INFO | "malloc" : " 14 GB", | 20260508 00:07:27.351 | INFO | "pinned" : " 29 GB" | 20260508 00:07:27.351 | INFO | } | 20260508 00:07:27.351 | INFO | Perf report for the last 5.14192 seconds: | 20260508 00:07:27.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:27.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.351 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:07:27.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.352 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 00:07:27.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:27.352 | INFO | Uploaded 36GiB to GPU in 2.50089s (14.3949GiB/s) | 20260508 00:07:27.352 | INFO | Downloaded 34GiB to GPU in 2.45716s (13.8371GiB/s) | 20260508 00:07:27.352 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:07:27.352 | INFO | Total mem usage: 16.7 GiB | 20260508 00:07:27.352 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:07:27.352 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.7%) | 20260508 00:07:27.352 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:07:27.352 | INFO | High Water Breakdown: | 20260508 00:07:27.352 | INFO | 59.7% : 8.69 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:07:27.352 | INFO | 40.1% : 5.84 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:07:27.352 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:07:27.352 | INFO | ------------------------------------------ | 20260508 00:07:27.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:07:27.352 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:07:27.352 | INFO | Total mem usage: 26.5 GiB | 20260508 00:07:27.352 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:07:27.352 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:07:27.352 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:07:27.352 | INFO | High Water Breakdown: | 20260508 00:07:27.352 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:07:27.352 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:07:27.352 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:07:27.352 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:07:27.352 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:07:27.352 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:07:27.352 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:07:27.352 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:07:27.352 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:07:27.352 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:07:27.352 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:07:27.352 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:07:27.352 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:07:27.352 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:07:27.352 | INFO | ------------------------------------------ | 20260508 00:07:27.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:07:27.352 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:07:27.353 | INFO | Total mem usage: 27.8 GiB | 20260508 00:07:27.353 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:07:27.353 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:07:27.353 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:07:27.353 | INFO | High Water Breakdown: | 20260508 00:07:27.353 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:07:27.353 | INFO | ------------------------------------------ | 20260508 00:07:27.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:07:27.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:07:27.353 | INFO | Total mem usage: 15.7 GiB | 20260508 00:07:27.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:07:27.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:07:27.353 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:07:27.353 | INFO | High Water Breakdown: | 20260508 00:07:27.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:07:27.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:07:27.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:07:27.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:07:27.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:07:27.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:07:27.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:07:27.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:07:27.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:07:27.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:07:27.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:07:27.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:07:27.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:07:27.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:07:27.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:07:27.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:07:27.353 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:07:27.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:07:27.353 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:07:27.353 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:07:27.353 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:07:27.353 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:07:27.353 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:07:27.353 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:07:27.353 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:07:27.353 | INFO | ------------------------------------------ | 20260508 00:07:27.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:07:27.353 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:07:27.353 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:07:27.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:07:27.353 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.40e+01 / 1.48e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.04e-01, 1.20e-02 / 1.19e-02 | 20260508 00:07:27.353 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.47e+00, 5.88e-01 / 2.21e-01 | 20260508 00:07:27.353 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.24e-01 | 20260508 00:07:27.353 | INFO | PerfReport: 16.13%, DME -- 4.24e+01 / 4.62e+01, 4.18e+00 / 1.78e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.62e+00 / 1.55e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.46e+00 / 2.70e+00, 4.31e-01 / 2.51e-01 | 20260508 00:07:27.353 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.32e+00, 2.77e-01 / 1.53e-01 | 20260508 00:07:27.353 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.78e-01, 5.49e-02 / 2.66e-02 | 20260508 00:07:27.353 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:07:27.353 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:07:27.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:07:27.353 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 24.34%, Upload -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 24.35%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 30.76%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 2.21%, LockUpload -- 2.36e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 2.71%, LockDownload -- 2.89e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 00:07:27.353 | INFO | PerfReport: 15.62%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:07:27.354 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 00:07:27.464 | INFO | Starting a round of Baz IO | 20260508 00:07:30.929 | INFO | Populating RTMetrics report | 20260508 00:07:30.929 | INFO | RTMetrics report populated | 20260508 00:07:30.929 | INFO | Opening tmp RTMetrics json file | 20260508 00:07:30.952 | INFO | Serializing RTMetrics to json | 20260508 00:07:30.953 | INFO | Serialization done, swapping tmp file now... | 20260508 00:07:30.954 | INFO | RTMetrics done | 20260508 00:07:31.038 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.574 seconds | 20260508 00:07:31.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.32% / 0.00% / 57.67% | 20260508 00:07:31.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07113% / 1.19209e-05% / 5.96046e-05% | 20260508 00:07:31.038 | INFO | Baz IO overall: | 20260508 00:07:31.039 | INFO | BytesWritten : 1672630982537 | 20260508 00:07:31.039 | INFO | NumEvents : 809702752418 | 20260508 00:07:32.489 | INFO | Analyzing chunk frames = [3678208,3678720) | 20260508 00:07:32.489 | INFO | MemInfo: { | 20260508 00:07:32.489 | INFO | "gpu" : " 17 GB", | 20260508 00:07:32.489 | INFO | "huge" : "", | 20260508 00:07:32.489 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:32.489 | INFO | "malloc" : " 14 GB", | 20260508 00:07:32.489 | INFO | "pinned" : " 29 GB" | 20260508 00:07:32.489 | INFO | } | 20260508 00:07:32.489 | INFO | Perf report for the last 5.138 seconds: | 20260508 00:07:32.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:32.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:32.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:32.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:32.490 | INFO | ANALYSIS : 68.9%, 2.9, 104.6ms, 0 | 20260508 00:07:32.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:32.490 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 00:07:32.490 | INFO | PRE_HQ : 67.5%, 1.8, 62.7ms, 0 | 20260508 00:07:32.490 | INFO | BAZWRITER : 69.2%, 3.7, 133.1ms, 0 | 20260508 00:07:32.490 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 00:07:32.490 | INFO | Uploaded 36GiB to GPU in 2.5632s (14.0449GiB/s) | 20260508 00:07:32.490 | INFO | Downloaded 38GiB to GPU in 2.69368s (14.1071GiB/s) | 20260508 00:07:37.624 | INFO | Analyzing chunk frames = [3678720,3679232) | 20260508 00:07:37.625 | INFO | MemInfo: { | 20260508 00:07:37.625 | INFO | "gpu" : " 17 GB", | 20260508 00:07:37.625 | INFO | "huge" : "", | 20260508 00:07:37.625 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:37.625 | INFO | "malloc" : " 14 GB", | 20260508 00:07:37.625 | INFO | "pinned" : " 29 GB" | 20260508 00:07:37.625 | INFO | } | 20260508 00:07:37.625 | INFO | Perf report for the last 5.13539 seconds: | 20260508 00:07:37.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:37.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:07:37.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:07:37.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:37.625 | INFO | Uploaded 36GiB to GPU in 2.5386s (14.1811GiB/s) | 20260508 00:07:37.625 | INFO | Downloaded 34GiB to GPU in 2.43023s (13.9905GiB/s) | 20260508 00:07:42.761 | INFO | Analyzing chunk frames = [3679232,3679744) | 20260508 00:07:42.761 | INFO | MemInfo: { | 20260508 00:07:42.762 | INFO | "gpu" : " 17 GB", | 20260508 00:07:42.762 | INFO | "huge" : "", | 20260508 00:07:42.762 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:42.762 | INFO | "malloc" : " 14 GB", | 20260508 00:07:42.762 | INFO | "pinned" : " 29 GB" | 20260508 00:07:42.762 | INFO | } | 20260508 00:07:42.762 | INFO | Perf report for the last 5.13697 seconds: | 20260508 00:07:42.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:42.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:07:42.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:07:42.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:42.762 | INFO | Uploaded 36GiB to GPU in 2.48476s (14.4883GiB/s) | 20260508 00:07:42.762 | INFO | Downloaded 34GiB to GPU in 2.49036s (13.6527GiB/s) | 20260508 00:07:47.905 | INFO | Analyzing chunk frames = [3679744,3680256) | 20260508 00:07:47.905 | INFO | MemInfo: { | 20260508 00:07:47.905 | INFO | "gpu" : " 17 GB", | 20260508 00:07:47.905 | INFO | "huge" : "", | 20260508 00:07:47.905 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:47.905 | INFO | "malloc" : " 14 GB", | 20260508 00:07:47.905 | INFO | "pinned" : " 29 GB" | 20260508 00:07:47.905 | INFO | } | 20260508 00:07:47.905 | INFO | Perf report for the last 5.14313 seconds: | 20260508 00:07:47.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:47.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.905 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 00:07:47.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.905 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:07:47.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:47.906 | INFO | Uploaded 36GiB to GPU in 2.49487s (14.4296GiB/s) | 20260508 00:07:47.906 | INFO | Downloaded 34GiB to GPU in 2.41203s (14.096GiB/s) | 20260508 00:07:53.050 | INFO | Analyzing chunk frames = [3680256,3680768) | 20260508 00:07:53.050 | INFO | MemInfo: { | 20260508 00:07:53.050 | INFO | "gpu" : " 17 GB", | 20260508 00:07:53.050 | INFO | "huge" : "", | 20260508 00:07:53.050 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:53.051 | INFO | "malloc" : " 14 GB", | 20260508 00:07:53.051 | INFO | "pinned" : " 29 GB" | 20260508 00:07:53.051 | INFO | } | 20260508 00:07:53.051 | INFO | Perf report for the last 5.14575 seconds: | 20260508 00:07:53.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:53.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 00:07:53.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 00:07:53.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:53.051 | INFO | Uploaded 36GiB to GPU in 2.33846s (15.3947GiB/s) | 20260508 00:07:53.051 | INFO | Downloaded 34GiB to GPU in 2.48819s (13.6645GiB/s) | 20260508 00:07:58.190 | INFO | Analyzing chunk frames = [3680768,3681280) | 20260508 00:07:58.190 | INFO | MemInfo: { | 20260508 00:07:58.190 | INFO | "gpu" : " 17 GB", | 20260508 00:07:58.190 | INFO | "huge" : "", | 20260508 00:07:58.190 | INFO | "huge_shared" : " 29 GB", | 20260508 00:07:58.190 | INFO | "malloc" : " 14 GB", | 20260508 00:07:58.190 | INFO | "pinned" : " 29 GB" | 20260508 00:07:58.190 | INFO | } | 20260508 00:07:58.190 | INFO | Perf report for the last 5.13967 seconds: | 20260508 00:07:58.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:07:58.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.190 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:07:58.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.191 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 00:07:58.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:07:58.191 | INFO | Uploaded 36GiB to GPU in 2.3886s (15.0716GiB/s) | 20260508 00:07:58.191 | INFO | Downloaded 34GiB to GPU in 2.40729s (14.1237GiB/s) | 20260508 00:08:03.328 | INFO | Analyzing chunk frames = [3681280,3681792) | 20260508 00:08:03.328 | INFO | MemInfo: { | 20260508 00:08:03.328 | INFO | "gpu" : " 17 GB", | 20260508 00:08:03.328 | INFO | "huge" : "", | 20260508 00:08:03.328 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:03.328 | INFO | "malloc" : " 14 GB", | 20260508 00:08:03.328 | INFO | "pinned" : " 29 GB" | 20260508 00:08:03.328 | INFO | } | 20260508 00:08:03.328 | INFO | Perf report for the last 5.13748 seconds: | 20260508 00:08:03.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:03.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:08:03.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 00:08:03.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:03.328 | INFO | Uploaded 36GiB to GPU in 2.52143s (14.2776GiB/s) | 20260508 00:08:03.329 | INFO | Downloaded 34GiB to GPU in 2.45483s (13.8502GiB/s) | 20260508 00:08:08.465 | INFO | Analyzing chunk frames = [3681792,3682304) | 20260508 00:08:08.465 | INFO | MemInfo: { | 20260508 00:08:08.465 | INFO | "gpu" : " 17 GB", | 20260508 00:08:08.466 | INFO | "huge" : "", | 20260508 00:08:08.466 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:08.466 | INFO | "malloc" : " 14 GB", | 20260508 00:08:08.466 | INFO | "pinned" : " 29 GB" | 20260508 00:08:08.466 | INFO | } | 20260508 00:08:08.466 | INFO | Perf report for the last 5.13791 seconds: | 20260508 00:08:08.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:08.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 00:08:08.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260508 00:08:08.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:08.466 | INFO | Uploaded 36GiB to GPU in 2.537s (14.19GiB/s) | 20260508 00:08:08.466 | INFO | Downloaded 34GiB to GPU in 2.45972s (13.8227GiB/s) | 20260508 00:08:08.578 | INFO | Starting a round of Baz IO | 20260508 00:08:12.040 | INFO | Populating RTMetrics report | 20260508 00:08:12.040 | INFO | RTMetrics report populated | 20260508 00:08:12.040 | INFO | Opening tmp RTMetrics json file | 20260508 00:08:12.058 | INFO | Serializing RTMetrics to json | 20260508 00:08:12.092 | INFO | Serialization done, swapping tmp file now... | 20260508 00:08:12.132 | INFO | RTMetrics done | 20260508 00:08:12.153 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.574 seconds | 20260508 00:08:12.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.32% / 0.00% / 57.68% | 20260508 00:08:12.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06661% / 3.97364e-06% / 7.94729e-05% | 20260508 00:08:12.153 | INFO | Baz IO overall: | 20260508 00:08:12.154 | INFO | BytesWritten : 1674453082106 | 20260508 00:08:12.154 | INFO | NumEvents : 810579863059 | 20260508 00:08:13.606 | INFO | Analyzing chunk frames = [3682304,3682816) | 20260508 00:08:13.606 | INFO | MemInfo: { | 20260508 00:08:13.606 | INFO | "gpu" : " 17 GB", | 20260508 00:08:13.606 | INFO | "huge" : "", | 20260508 00:08:13.606 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:13.606 | INFO | "malloc" : " 14 GB", | 20260508 00:08:13.607 | INFO | "pinned" : " 29 GB" | 20260508 00:08:13.607 | INFO | } | 20260508 00:08:13.607 | INFO | Perf report for the last 5.14085 seconds: | 20260508 00:08:13.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:13.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:13.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:13.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:13.607 | INFO | ANALYSIS : 68.8%, 2.9, 104.4ms, 0 | 20260508 00:08:13.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:13.607 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260508 00:08:13.607 | INFO | PRE_HQ : 67.8%, 1.8, 63.3ms, 0 | 20260508 00:08:13.607 | INFO | BAZWRITER : 69.1%, 4.2, 147.9ms, 0 | 20260508 00:08:13.607 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:08:13.607 | INFO | Uploaded 36GiB to GPU in 2.6378s (13.6477GiB/s) | 20260508 00:08:13.607 | INFO | Downloaded 38GiB to GPU in 2.72073s (13.9668GiB/s) | 20260508 00:08:18.739 | INFO | Analyzing chunk frames = [3682816,3683328) | 20260508 00:08:18.739 | INFO | MemInfo: { | 20260508 00:08:18.739 | INFO | "gpu" : " 17 GB", | 20260508 00:08:18.739 | INFO | "huge" : "", | 20260508 00:08:18.739 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:18.739 | INFO | "malloc" : " 14 GB", | 20260508 00:08:18.739 | INFO | "pinned" : " 29 GB" | 20260508 00:08:18.739 | INFO | } | 20260508 00:08:18.739 | INFO | Perf report for the last 5.13257 seconds: | 20260508 00:08:18.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:18.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.739 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 00:08:18.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.739 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:08:18.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:18.740 | INFO | Uploaded 36GiB to GPU in 2.56963s (14.0098GiB/s) | 20260508 00:08:18.740 | INFO | Downloaded 34GiB to GPU in 2.4993s (13.6038GiB/s) | 20260508 00:08:23.879 | INFO | Analyzing chunk frames = [3683328,3683840) | 20260508 00:08:23.879 | INFO | MemInfo: { | 20260508 00:08:23.879 | INFO | "gpu" : " 17 GB", | 20260508 00:08:23.879 | INFO | "huge" : "", | 20260508 00:08:23.879 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:23.879 | INFO | "malloc" : " 14 GB", | 20260508 00:08:23.880 | INFO | "pinned" : " 29 GB" | 20260508 00:08:23.880 | INFO | } | 20260508 00:08:23.880 | INFO | Perf report for the last 5.14038 seconds: | 20260508 00:08:23.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:23.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:08:23.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:08:23.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:23.880 | INFO | Uploaded 36GiB to GPU in 2.50551s (14.3683GiB/s) | 20260508 00:08:23.880 | INFO | Downloaded 34GiB to GPU in 2.49415s (13.6319GiB/s) | 20260508 00:08:29.018 | INFO | Analyzing chunk frames = [3683840,3684352) | 20260508 00:08:29.018 | INFO | MemInfo: { | 20260508 00:08:29.018 | INFO | "gpu" : " 17 GB", | 20260508 00:08:29.018 | INFO | "huge" : "", | 20260508 00:08:29.018 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:29.019 | INFO | "malloc" : " 14 GB", | 20260508 00:08:29.019 | INFO | "pinned" : " 29 GB" | 20260508 00:08:29.019 | INFO | } | 20260508 00:08:29.019 | INFO | Perf report for the last 5.13982 seconds: | 20260508 00:08:29.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:29.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.020 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:08:29.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.020 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:08:29.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:29.020 | INFO | Uploaded 36GiB to GPU in 2.50601s (14.3655GiB/s) | 20260508 00:08:29.020 | INFO | Downloaded 34GiB to GPU in 2.50011s (13.5994GiB/s) | 20260508 00:08:34.159 | INFO | Analyzing chunk frames = [3684352,3684864) | 20260508 00:08:34.159 | INFO | MemInfo: { | 20260508 00:08:34.159 | INFO | "gpu" : " 17 GB", | 20260508 00:08:34.159 | INFO | "huge" : "", | 20260508 00:08:34.159 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:34.159 | INFO | "malloc" : " 14 GB", | 20260508 00:08:34.159 | INFO | "pinned" : " 29 GB" | 20260508 00:08:34.159 | INFO | } | 20260508 00:08:34.159 | INFO | Perf report for the last 5.13936 seconds: | 20260508 00:08:34.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:34.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 00:08:34.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 00:08:34.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:34.159 | INFO | Uploaded 36GiB to GPU in 2.36153s (15.2443GiB/s) | 20260508 00:08:34.159 | INFO | Downloaded 34GiB to GPU in 2.50969s (13.5475GiB/s) | 20260508 00:08:39.300 | INFO | Analyzing chunk frames = [3684864,3685376) | 20260508 00:08:39.300 | INFO | MemInfo: { | 20260508 00:08:39.300 | INFO | "gpu" : " 17 GB", | 20260508 00:08:39.300 | INFO | "huge" : "", | 20260508 00:08:39.300 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:39.300 | INFO | "malloc" : " 14 GB", | 20260508 00:08:39.300 | INFO | "pinned" : " 29 GB" | 20260508 00:08:39.300 | INFO | } | 20260508 00:08:39.300 | INFO | Perf report for the last 5.14156 seconds: | 20260508 00:08:39.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:39.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.300 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 00:08:39.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.301 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:08:39.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:39.301 | INFO | Uploaded 36GiB to GPU in 2.40514s (14.9679GiB/s) | 20260508 00:08:39.301 | INFO | Downloaded 34GiB to GPU in 2.40487s (14.138GiB/s) | 20260508 00:08:44.441 | INFO | Analyzing chunk frames = [3685376,3685888) | 20260508 00:08:44.441 | INFO | MemInfo: { | 20260508 00:08:44.441 | INFO | "gpu" : " 17 GB", | 20260508 00:08:44.441 | INFO | "huge" : "", | 20260508 00:08:44.441 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:44.441 | INFO | "malloc" : " 14 GB", | 20260508 00:08:44.441 | INFO | "pinned" : " 29 GB" | 20260508 00:08:44.441 | INFO | } | 20260508 00:08:44.441 | INFO | Perf report for the last 5.14068 seconds: | 20260508 00:08:44.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:44.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 00:08:44.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 00:08:44.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:44.441 | INFO | Uploaded 36GiB to GPU in 2.51797s (14.2972GiB/s) | 20260508 00:08:44.442 | INFO | Downloaded 34GiB to GPU in 2.45349s (13.8578GiB/s) | 20260508 00:08:49.575 | INFO | Analyzing chunk frames = [3685888,3686400) | 20260508 00:08:49.575 | INFO | MemInfo: { | 20260508 00:08:49.575 | INFO | "gpu" : " 17 GB", | 20260508 00:08:49.575 | INFO | "huge" : "", | 20260508 00:08:49.575 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:49.575 | INFO | "malloc" : " 14 GB", | 20260508 00:08:49.576 | INFO | "pinned" : " 29 GB" | 20260508 00:08:49.576 | INFO | } | 20260508 00:08:49.576 | INFO | Perf report for the last 5.13459 seconds: | 20260508 00:08:49.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:49.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:08:49.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:08:49.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:49.576 | INFO | Uploaded 36GiB to GPU in 2.52632s (14.25GiB/s) | 20260508 00:08:49.576 | INFO | Downloaded 34GiB to GPU in 2.45283s (13.8616GiB/s) | 20260508 00:08:49.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:08:49.576 | INFO | Total mem usage: 16.7 GiB | 20260508 00:08:49.576 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:08:49.576 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 00:08:49.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:08:49.576 | INFO | High Water Breakdown: | 20260508 00:08:49.576 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:08:49.577 | INFO | 39.9% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:08:49.577 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:08:49.577 | INFO | ------------------------------------------ | 20260508 00:08:49.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:08:49.577 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:08:49.577 | INFO | Total mem usage: 26.5 GiB | 20260508 00:08:49.577 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:08:49.577 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:08:49.577 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:08:49.577 | INFO | High Water Breakdown: | 20260508 00:08:49.577 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:08:49.577 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:08:49.577 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:08:49.577 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:08:49.577 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:08:49.577 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:08:49.577 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:08:49.577 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:08:49.577 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:08:49.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:08:49.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:08:49.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:08:49.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:08:49.577 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:08:49.577 | INFO | ------------------------------------------ | 20260508 00:08:49.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:08:49.577 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:08:49.577 | INFO | Total mem usage: 27.8 GiB | 20260508 00:08:49.577 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:08:49.577 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:08:49.577 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:08:49.577 | INFO | High Water Breakdown: | 20260508 00:08:49.577 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:08:49.577 | INFO | ------------------------------------------ | 20260508 00:08:49.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:08:49.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:08:49.577 | INFO | Total mem usage: 15.7 GiB | 20260508 00:08:49.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:08:49.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:08:49.577 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:08:49.577 | INFO | High Water Breakdown: | 20260508 00:08:49.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:08:49.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:08:49.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:08:49.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:08:49.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:08:49.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:08:49.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:08:49.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:08:49.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:08:49.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:08:49.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:08:49.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:08:49.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:08:49.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:08:49.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:08:49.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:08:49.577 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:08:49.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:08:49.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:08:49.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:08:49.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:08:49.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:08:49.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:08:49.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:08:49.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:08:49.577 | INFO | ------------------------------------------ | 20260508 00:08:49.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:08:49.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:08:49.577 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:08:49.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:08:49.577 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.38e+01 / 1.67e+00 | 20260508 00:08:49.577 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.02e-01, 1.20e-02 / 5.14e-03 | 20260508 00:08:49.577 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.46e+00, 5.87e-01 / 2.14e-01 | 20260508 00:08:49.577 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.11e-01 | 20260508 00:08:49.577 | INFO | PerfReport: 16.01%, DME -- 4.21e+01 / 4.60e+01, 4.19e+00 / 1.85e+00 | 20260508 00:08:49.577 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.71e+00 / 1.39e+00 | 20260508 00:08:49.577 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.45e+00 / 2.71e+00, 4.28e-01 / 2.48e-01 | 20260508 00:08:49.577 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.32e+00, 2.79e-01 / 1.46e-01 | 20260508 00:08:49.577 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.79e-01, 5.52e-02 / 3.02e-02 | 20260508 00:08:49.578 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:08:49.578 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:08:49.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:08:49.578 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 30.86%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 2.07%, LockUpload -- 2.20e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 2.65%, LockDownload -- 2.82e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:08:49.578 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260508 00:08:49.673 | INFO | Starting a round of Baz IO | 20260508 00:08:53.134 | INFO | Populating RTMetrics report | 20260508 00:08:53.135 | INFO | RTMetrics report populated | 20260508 00:08:53.135 | INFO | Opening tmp RTMetrics json file | 20260508 00:08:53.158 | INFO | Serializing RTMetrics to json | 20260508 00:08:53.160 | INFO | Serialization done, swapping tmp file now... | 20260508 00:08:53.160 | INFO | RTMetrics done | 20260508 00:08:53.236 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.563 seconds | 20260508 00:08:53.236 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.31% / 0.00% / 57.68% | 20260508 00:08:53.236 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0687% / 1.19209e-05% / 5.5631e-05% | 20260508 00:08:53.236 | INFO | Baz IO overall: | 20260508 00:08:53.236 | INFO | BytesWritten : 1676277230953 | 20260508 00:08:53.236 | INFO | NumEvents : 811458774711 | 20260508 00:08:54.717 | INFO | Analyzing chunk frames = [3686400,3686912) | 20260508 00:08:54.717 | INFO | MemInfo: { | 20260508 00:08:54.717 | INFO | "gpu" : " 17 GB", | 20260508 00:08:54.717 | INFO | "huge" : "", | 20260508 00:08:54.717 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:54.717 | INFO | "malloc" : " 14 GB", | 20260508 00:08:54.717 | INFO | "pinned" : " 29 GB" | 20260508 00:08:54.717 | INFO | } | 20260508 00:08:54.717 | INFO | Perf report for the last 5.14089 seconds: | 20260508 00:08:54.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:54.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:54.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:54.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:54.717 | INFO | ANALYSIS : 68.5%, 3.0, 106.7ms, 0 | 20260508 00:08:54.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:54.717 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260508 00:08:54.717 | INFO | PRE_HQ : 65.7%, 1.7, 56.7ms, 0 | 20260508 00:08:54.717 | INFO | BAZWRITER : 68.9%, 4.1, 144.0ms, 0 | 20260508 00:08:54.717 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 00:08:54.717 | INFO | Uploaded 36GiB to GPU in 2.54367s (14.1528GiB/s) | 20260508 00:08:54.717 | INFO | Downloaded 38GiB to GPU in 2.70205s (14.0634GiB/s) | 20260508 00:08:59.856 | INFO | Analyzing chunk frames = [3686912,3687424) | 20260508 00:08:59.856 | INFO | MemInfo: { | 20260508 00:08:59.856 | INFO | "gpu" : " 17 GB", | 20260508 00:08:59.856 | INFO | "huge" : "", | 20260508 00:08:59.856 | INFO | "huge_shared" : " 29 GB", | 20260508 00:08:59.856 | INFO | "malloc" : " 14 GB", | 20260508 00:08:59.856 | INFO | "pinned" : " 29 GB" | 20260508 00:08:59.856 | INFO | } | 20260508 00:08:59.856 | INFO | Perf report for the last 5.13955 seconds: | 20260508 00:08:59.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:08:59.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.856 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:08:59.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.856 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:08:59.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:08:59.857 | INFO | Uploaded 36GiB to GPU in 2.53158s (14.2204GiB/s) | 20260508 00:08:59.857 | INFO | Downloaded 34GiB to GPU in 2.49139s (13.647GiB/s) | 20260508 00:09:05.002 | INFO | Analyzing chunk frames = [3687424,3687936) | 20260508 00:09:05.002 | INFO | MemInfo: { | 20260508 00:09:05.002 | INFO | "gpu" : " 17 GB", | 20260508 00:09:05.002 | INFO | "huge" : "", | 20260508 00:09:05.002 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:05.002 | INFO | "malloc" : " 14 GB", | 20260508 00:09:05.002 | INFO | "pinned" : " 29 GB" | 20260508 00:09:05.002 | INFO | } | 20260508 00:09:05.002 | INFO | Perf report for the last 5.14599 seconds: | 20260508 00:09:05.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:05.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 00:09:05.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 00:09:05.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:05.002 | INFO | Uploaded 36GiB to GPU in 2.48421s (14.4916GiB/s) | 20260508 00:09:05.003 | INFO | Downloaded 34GiB to GPU in 2.46313s (13.8036GiB/s) | 20260508 00:09:10.139 | INFO | Analyzing chunk frames = [3687936,3688448) | 20260508 00:09:10.139 | INFO | MemInfo: { | 20260508 00:09:10.139 | INFO | "gpu" : " 17 GB", | 20260508 00:09:10.139 | INFO | "huge" : "", | 20260508 00:09:10.139 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:10.139 | INFO | "malloc" : " 14 GB", | 20260508 00:09:10.139 | INFO | "pinned" : " 29 GB" | 20260508 00:09:10.139 | INFO | } | 20260508 00:09:10.139 | INFO | Perf report for the last 5.13686 seconds: | 20260508 00:09:10.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:10.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.139 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:09:10.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.139 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:09:10.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:10.140 | INFO | Uploaded 36GiB to GPU in 2.49561s (14.4253GiB/s) | 20260508 00:09:10.140 | INFO | Downloaded 34GiB to GPU in 2.47721s (13.7251GiB/s) | 20260508 00:09:15.277 | INFO | Analyzing chunk frames = [3688448,3688960) | 20260508 00:09:15.277 | INFO | MemInfo: { | 20260508 00:09:15.277 | INFO | "gpu" : " 17 GB", | 20260508 00:09:15.277 | INFO | "huge" : "", | 20260508 00:09:15.277 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:15.277 | INFO | "malloc" : " 14 GB", | 20260508 00:09:15.277 | INFO | "pinned" : " 29 GB" | 20260508 00:09:15.277 | INFO | } | 20260508 00:09:15.277 | INFO | Perf report for the last 5.13806 seconds: | 20260508 00:09:15.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:15.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.277 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:09:15.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.277 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:09:15.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:15.278 | INFO | Uploaded 36GiB to GPU in 2.38119s (15.1185GiB/s) | 20260508 00:09:15.278 | INFO | Downloaded 34GiB to GPU in 2.57955s (13.1806GiB/s) | 20260508 00:09:20.416 | INFO | Analyzing chunk frames = [3688960,3689472) | 20260508 00:09:20.416 | INFO | MemInfo: { | 20260508 00:09:20.416 | INFO | "gpu" : " 17 GB", | 20260508 00:09:20.416 | INFO | "huge" : "", | 20260508 00:09:20.416 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:20.416 | INFO | "malloc" : " 14 GB", | 20260508 00:09:20.416 | INFO | "pinned" : " 29 GB" | 20260508 00:09:20.416 | INFO | } | 20260508 00:09:20.417 | INFO | Perf report for the last 5.13942 seconds: | 20260508 00:09:20.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:20.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:09:20.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260508 00:09:20.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:20.417 | INFO | Uploaded 36GiB to GPU in 2.38871s (15.0709GiB/s) | 20260508 00:09:20.417 | INFO | Downloaded 34GiB to GPU in 2.39961s (14.169GiB/s) | 20260508 00:09:25.555 | INFO | Analyzing chunk frames = [3689472,3689984) | 20260508 00:09:25.555 | INFO | MemInfo: { | 20260508 00:09:25.555 | INFO | "gpu" : " 17 GB", | 20260508 00:09:25.555 | INFO | "huge" : "", | 20260508 00:09:25.555 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:25.555 | INFO | "malloc" : " 14 GB", | 20260508 00:09:25.555 | INFO | "pinned" : " 29 GB" | 20260508 00:09:25.555 | INFO | } | 20260508 00:09:25.555 | INFO | Perf report for the last 5.13885 seconds: | 20260508 00:09:25.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:25.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 00:09:25.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | PRE_HQ : 34.9%, 1.0, 18.2ms, 0 | 20260508 00:09:25.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:25.556 | INFO | Uploaded 36GiB to GPU in 2.5323s (14.2163GiB/s) | 20260508 00:09:25.556 | INFO | Downloaded 34GiB to GPU in 2.46156s (13.8124GiB/s) | 20260508 00:09:30.688 | INFO | Analyzing chunk frames = [3689984,3690496) | 20260508 00:09:30.688 | INFO | MemInfo: { | 20260508 00:09:30.688 | INFO | "gpu" : " 17 GB", | 20260508 00:09:30.688 | INFO | "huge" : "", | 20260508 00:09:30.688 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:30.688 | INFO | "malloc" : " 14 GB", | 20260508 00:09:30.688 | INFO | "pinned" : " 29 GB" | 20260508 00:09:30.688 | INFO | } | 20260508 00:09:30.688 | INFO | Perf report for the last 5.13253 seconds: | 20260508 00:09:30.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:30.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260508 00:09:30.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 00:09:30.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:30.688 | INFO | Uploaded 36GiB to GPU in 2.5397s (14.1749GiB/s) | 20260508 00:09:30.688 | INFO | Downloaded 34GiB to GPU in 2.45704s (13.8378GiB/s) | 20260508 00:09:30.801 | INFO | Starting a round of Baz IO | 20260508 00:09:34.275 | INFO | Populating RTMetrics report | 20260508 00:09:34.275 | INFO | RTMetrics report populated | 20260508 00:09:34.275 | INFO | Opening tmp RTMetrics json file | 20260508 00:09:34.282 | INFO | Serializing RTMetrics to json | 20260508 00:09:34.302 | INFO | Serialization done, swapping tmp file now... | 20260508 00:09:34.303 | INFO | RTMetrics done | 20260508 00:09:34.373 | INFO | Finished a round of Baz IO: Wrote 1.7 GiB in 3.572 seconds | 20260508 00:09:34.373 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.31% / 0.00% / 57.69% | 20260508 00:09:34.373 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06835% / 1.19209e-05% / 6.35783e-05% | 20260508 00:09:34.373 | INFO | Baz IO overall: | 20260508 00:09:34.373 | INFO | BytesWritten : 1678102446213 | 20260508 00:09:34.374 | INFO | NumEvents : 812338360678 | 20260508 00:09:35.825 | INFO | Analyzing chunk frames = [3690496,3691008) | 20260508 00:09:35.825 | INFO | MemInfo: { | 20260508 00:09:35.825 | INFO | "gpu" : " 17 GB", | 20260508 00:09:35.825 | INFO | "huge" : "", | 20260508 00:09:35.825 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:35.825 | INFO | "malloc" : " 14 GB", | 20260508 00:09:35.825 | INFO | "pinned" : " 29 GB" | 20260508 00:09:35.825 | INFO | } | 20260508 00:09:35.825 | INFO | Perf report for the last 5.13694 seconds: | 20260508 00:09:35.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:35.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:35.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:35.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:35.825 | INFO | ANALYSIS : 69.2%, 2.9, 105.2ms, 0 | 20260508 00:09:35.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:35.825 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260508 00:09:35.825 | INFO | PRE_HQ : 68.0%, 1.8, 62.9ms, 0 | 20260508 00:09:35.825 | INFO | BAZWRITER : 69.1%, 3.7, 129.6ms, 0 | 20260508 00:09:35.825 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:09:35.826 | INFO | Uploaded 36GiB to GPU in 2.55066s (14.114GiB/s) | 20260508 00:09:35.826 | INFO | Downloaded 38GiB to GPU in 2.71181s (14.0128GiB/s) | 20260508 00:09:40.968 | INFO | Analyzing chunk frames = [3691008,3691520) | 20260508 00:09:40.968 | INFO | MemInfo: { | 20260508 00:09:40.968 | INFO | "gpu" : " 17 GB", | 20260508 00:09:40.969 | INFO | "huge" : "", | 20260508 00:09:40.969 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:40.969 | INFO | "malloc" : " 14 GB", | 20260508 00:09:40.969 | INFO | "pinned" : " 29 GB" | 20260508 00:09:40.969 | INFO | } | 20260508 00:09:40.969 | INFO | Perf report for the last 5.14375 seconds: | 20260508 00:09:40.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:40.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 00:09:40.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:09:40.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:40.969 | INFO | Uploaded 36GiB to GPU in 2.55198s (14.1067GiB/s) | 20260508 00:09:40.969 | INFO | Downloaded 34GiB to GPU in 2.48156s (13.701GiB/s) | 20260508 00:09:46.114 | INFO | Analyzing chunk frames = [3691520,3692032) | 20260508 00:09:46.114 | INFO | MemInfo: { | 20260508 00:09:46.114 | INFO | "gpu" : " 17 GB", | 20260508 00:09:46.114 | INFO | "huge" : "", | 20260508 00:09:46.114 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:46.114 | INFO | "malloc" : " 14 GB", | 20260508 00:09:46.114 | INFO | "pinned" : " 29 GB" | 20260508 00:09:46.114 | INFO | } | 20260508 00:09:46.114 | INFO | Perf report for the last 5.14571 seconds: | 20260508 00:09:46.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:46.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.114 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 00:09:46.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.115 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:09:46.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:46.115 | INFO | Uploaded 36GiB to GPU in 2.55434s (14.0937GiB/s) | 20260508 00:09:46.115 | INFO | Downloaded 34GiB to GPU in 2.48376s (13.6889GiB/s) | 20260508 00:09:51.255 | INFO | Analyzing chunk frames = [3692032,3692544) | 20260508 00:09:51.255 | INFO | MemInfo: { | 20260508 00:09:51.255 | INFO | "gpu" : " 17 GB", | 20260508 00:09:51.255 | INFO | "huge" : "", | 20260508 00:09:51.255 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:51.255 | INFO | "malloc" : " 14 GB", | 20260508 00:09:51.255 | INFO | "pinned" : " 29 GB" | 20260508 00:09:51.255 | INFO | } | 20260508 00:09:51.255 | INFO | Perf report for the last 5.14106 seconds: | 20260508 00:09:51.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:51.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:09:51.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:09:51.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:51.256 | INFO | Uploaded 36GiB to GPU in 2.50453s (14.374GiB/s) | 20260508 00:09:51.256 | INFO | Downloaded 34GiB to GPU in 2.49629s (13.6202GiB/s) | 20260508 00:09:56.385 | INFO | Analyzing chunk frames = [3692544,3693056) | 20260508 00:09:56.385 | INFO | MemInfo: { | 20260508 00:09:56.385 | INFO | "gpu" : " 17 GB", | 20260508 00:09:56.385 | INFO | "huge" : "", | 20260508 00:09:56.385 | INFO | "huge_shared" : " 29 GB", | 20260508 00:09:56.385 | INFO | "malloc" : " 14 GB", | 20260508 00:09:56.385 | INFO | "pinned" : " 29 GB" | 20260508 00:09:56.385 | INFO | } | 20260508 00:09:56.385 | INFO | Perf report for the last 5.12938 seconds: | 20260508 00:09:56.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:09:56.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 00:09:56.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:09:56.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:09:56.385 | INFO | Uploaded 36GiB to GPU in 2.32506s (15.4835GiB/s) | 20260508 00:09:56.385 | INFO | Downloaded 34GiB to GPU in 2.50183s (13.59GiB/s) | 20260508 00:10:01.527 | INFO | Analyzing chunk frames = [3693056,3693568) | 20260508 00:10:01.527 | INFO | MemInfo: { | 20260508 00:10:01.527 | INFO | "gpu" : " 17 GB", | 20260508 00:10:01.527 | INFO | "huge" : "", | 20260508 00:10:01.527 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:01.527 | INFO | "malloc" : " 14 GB", | 20260508 00:10:01.527 | INFO | "pinned" : " 29 GB" | 20260508 00:10:01.527 | INFO | } | 20260508 00:10:01.528 | INFO | Perf report for the last 5.14261 seconds: | 20260508 00:10:01.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:01.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 00:10:01.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 00:10:01.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:01.528 | INFO | Uploaded 36GiB to GPU in 2.48237s (14.5023GiB/s) | 20260508 00:10:01.528 | INFO | Downloaded 34GiB to GPU in 2.44875s (13.8846GiB/s) | 20260508 00:10:06.669 | INFO | Analyzing chunk frames = [3693568,3694080) | 20260508 00:10:06.669 | INFO | MemInfo: { | 20260508 00:10:06.669 | INFO | "gpu" : " 17 GB", | 20260508 00:10:06.669 | INFO | "huge" : "", | 20260508 00:10:06.669 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:06.669 | INFO | "malloc" : " 14 GB", | 20260508 00:10:06.669 | INFO | "pinned" : " 29 GB" | 20260508 00:10:06.669 | INFO | } | 20260508 00:10:06.669 | INFO | Perf report for the last 5.14164 seconds: | 20260508 00:10:06.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:06.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260508 00:10:06.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 00:10:06.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:06.669 | INFO | Uploaded 36GiB to GPU in 2.48277s (14.4999GiB/s) | 20260508 00:10:06.670 | INFO | Downloaded 34GiB to GPU in 2.45995s (13.8214GiB/s) | 20260508 00:10:11.809 | INFO | Analyzing chunk frames = [3694080,3694592) | 20260508 00:10:11.809 | INFO | MemInfo: { | 20260508 00:10:11.809 | INFO | "gpu" : " 17 GB", | 20260508 00:10:11.809 | INFO | "huge" : "", | 20260508 00:10:11.809 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:11.809 | INFO | "malloc" : " 14 GB", | 20260508 00:10:11.809 | INFO | "pinned" : " 29 GB" | 20260508 00:10:11.809 | INFO | } | 20260508 00:10:11.809 | INFO | Perf report for the last 5.14003 seconds: | 20260508 00:10:11.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:11.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 00:10:11.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:10:11.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:11.809 | INFO | Uploaded 36GiB to GPU in 2.48072s (14.5119GiB/s) | 20260508 00:10:11.809 | INFO | Downloaded 34GiB to GPU in 2.45273s (13.8621GiB/s) | 20260508 00:10:11.810 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:10:11.810 | INFO | Total mem usage: 16.7 GiB | 20260508 00:10:11.810 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:10:11.810 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260508 00:10:11.810 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:10:11.810 | INFO | High Water Breakdown: | 20260508 00:10:11.810 | INFO | 60% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:10:11.810 | INFO | 39.9% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:10:11.810 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:10:11.810 | INFO | ------------------------------------------ | 20260508 00:10:11.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:10:11.810 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:10:11.810 | INFO | Total mem usage: 26.5 GiB | 20260508 00:10:11.810 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:10:11.810 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:10:11.810 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:10:11.810 | INFO | High Water Breakdown: | 20260508 00:10:11.810 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:10:11.810 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:10:11.810 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:10:11.810 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:10:11.810 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:10:11.810 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:10:11.810 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:10:11.810 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:10:11.810 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:10:11.810 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:10:11.810 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:10:11.810 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:10:11.810 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:10:11.810 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:10:11.810 | INFO | ------------------------------------------ | 20260508 00:10:11.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:10:11.810 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:10:11.810 | INFO | Total mem usage: 27.8 GiB | 20260508 00:10:11.810 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:10:11.810 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:10:11.810 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:10:11.810 | INFO | High Water Breakdown: | 20260508 00:10:11.810 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:10:11.810 | INFO | ------------------------------------------ | 20260508 00:10:11.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:10:11.810 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:10:11.810 | INFO | Total mem usage: 15.7 GiB | 20260508 00:10:11.810 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:10:11.810 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:10:11.810 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:10:11.810 | INFO | High Water Breakdown: | 20260508 00:10:11.810 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:10:11.810 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:10:11.810 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:10:11.810 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:10:11.810 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:10:11.810 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:10:11.810 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:10:11.810 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:10:11.810 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:10:11.810 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:10:11.810 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:10:11.810 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:10:11.810 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:10:11.810 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:10:11.810 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:10:11.810 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:10:11.810 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:10:11.810 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:10:11.810 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:10:11.810 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:10:11.810 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:10:11.810 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:10:11.810 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:10:11.810 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:10:11.810 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:10:11.810 | INFO | ------------------------------------------ | 20260508 00:10:11.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:10:11.810 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:10:11.810 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:10:11.810 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:10:11.810 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.67e+01, 1.42e+01 / 2.20e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.04e-01, 1.27e-02 / 1.15e-02 | 20260508 00:10:11.810 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.49e+00, 5.92e-01 / 1.80e-01 | 20260508 00:10:11.810 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.20e-01 | 20260508 00:10:11.810 | INFO | PerfReport: 16.09%, DME -- 4.23e+01 / 4.64e+01, 4.36e+00 / 2.43e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.63e+00 / 1.85e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.35e-01 / 2.29e-01 | 20260508 00:10:11.810 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.32e+00, 2.85e-01 / 1.42e-01 | 20260508 00:10:11.810 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.74e-01, 5.53e-02 / 2.38e-02 | 20260508 00:10:11.810 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:10:11.810 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:10:11.810 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:10:11.810 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 24.27%, Upload -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 24.42%, Download -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 30.75%, Compute -- 3.29e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 2.86%, LockDownload -- 3.06e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:10:11.810 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260508 00:10:11.906 | INFO | Starting a round of Baz IO | 20260508 00:10:15.357 | INFO | Populating RTMetrics report | 20260508 00:10:15.357 | INFO | RTMetrics report populated | 20260508 00:10:15.357 | INFO | Opening tmp RTMetrics json file | 20260508 00:10:15.379 | INFO | Serializing RTMetrics to json | 20260508 00:10:15.381 | INFO | Serialization done, swapping tmp file now... | 20260508 00:10:15.381 | INFO | RTMetrics done | 20260508 00:10:15.454 | INFO | Finished a round of Baz IO: Wrote 1.7 GiB in 3.547 seconds | 20260508 00:10:15.454 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.30% / 0.00% / 57.69% | 20260508 00:10:15.454 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06471% / 1.19209e-05% / 5.5631e-05% | 20260508 00:10:15.454 | INFO | Baz IO overall: | 20260508 00:10:15.454 | INFO | BytesWritten : 1679927558692 | 20260508 00:10:15.454 | INFO | NumEvents : 813217765334 | 20260508 00:10:16.953 | INFO | Analyzing chunk frames = [3694592,3695104) | 20260508 00:10:16.953 | INFO | MemInfo: { | 20260508 00:10:16.953 | INFO | "gpu" : " 17 GB", | 20260508 00:10:16.953 | INFO | "huge" : "", | 20260508 00:10:16.953 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:16.953 | INFO | "malloc" : " 14 GB", | 20260508 00:10:16.953 | INFO | "pinned" : " 29 GB" | 20260508 00:10:16.953 | INFO | } | 20260508 00:10:16.953 | INFO | Perf report for the last 5.14361 seconds: | 20260508 00:10:16.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:16.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:16.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:16.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:16.953 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 00:10:16.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:16.953 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 00:10:16.953 | INFO | PRE_HQ : 65.5%, 1.7, 57.0ms, 0 | 20260508 00:10:16.953 | INFO | BAZWRITER : 68.5%, 3.8, 132.3ms, 0 | 20260508 00:10:16.953 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:10:16.953 | INFO | Uploaded 36GiB to GPU in 2.53259s (14.2147GiB/s) | 20260508 00:10:16.953 | INFO | Downloaded 38GiB to GPU in 2.7035s (14.0558GiB/s) | 20260508 00:10:22.086 | INFO | Analyzing chunk frames = [3695104,3695616) | 20260508 00:10:22.086 | INFO | MemInfo: { | 20260508 00:10:22.086 | INFO | "gpu" : " 17 GB", | 20260508 00:10:22.086 | INFO | "huge" : "", | 20260508 00:10:22.086 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:22.086 | INFO | "malloc" : " 14 GB", | 20260508 00:10:22.086 | INFO | "pinned" : " 29 GB" | 20260508 00:10:22.086 | INFO | } | 20260508 00:10:22.086 | INFO | Perf report for the last 5.13332 seconds: | 20260508 00:10:22.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:22.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:10:22.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 00:10:22.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:22.086 | INFO | Uploaded 36GiB to GPU in 2.50805s (14.3538GiB/s) | 20260508 00:10:22.086 | INFO | Downloaded 34GiB to GPU in 2.43757s (13.9483GiB/s) | 20260508 00:10:27.228 | INFO | Analyzing chunk frames = [3695616,3696128) | 20260508 00:10:27.228 | INFO | MemInfo: { | 20260508 00:10:27.228 | INFO | "gpu" : " 17 GB", | 20260508 00:10:27.228 | INFO | "huge" : "", | 20260508 00:10:27.228 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:27.228 | INFO | "malloc" : " 14 GB", | 20260508 00:10:27.228 | INFO | "pinned" : " 29 GB" | 20260508 00:10:27.228 | INFO | } | 20260508 00:10:27.228 | INFO | Perf report for the last 5.14211 seconds: | 20260508 00:10:27.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:27.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.229 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 00:10:27.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.229 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:10:27.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:27.229 | INFO | Uploaded 36GiB to GPU in 2.51334s (14.3236GiB/s) | 20260508 00:10:27.229 | INFO | Downloaded 34GiB to GPU in 2.48033s (13.7079GiB/s) | 20260508 00:10:32.367 | INFO | Analyzing chunk frames = [3696128,3696640) | 20260508 00:10:32.367 | INFO | MemInfo: { | 20260508 00:10:32.367 | INFO | "gpu" : " 17 GB", | 20260508 00:10:32.367 | INFO | "huge" : "", | 20260508 00:10:32.367 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:32.367 | INFO | "malloc" : " 14 GB", | 20260508 00:10:32.367 | INFO | "pinned" : " 29 GB" | 20260508 00:10:32.367 | INFO | } | 20260508 00:10:32.367 | INFO | Perf report for the last 5.13858 seconds: | 20260508 00:10:32.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:32.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.367 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 00:10:32.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.367 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:10:32.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:32.368 | INFO | Uploaded 36GiB to GPU in 2.4575s (14.649GiB/s) | 20260508 00:10:32.368 | INFO | Downloaded 34GiB to GPU in 2.499s (13.6055GiB/s) | 20260508 00:10:37.505 | INFO | Analyzing chunk frames = [3696640,3697152) | 20260508 00:10:37.505 | INFO | MemInfo: { | 20260508 00:10:37.505 | INFO | "gpu" : " 17 GB", | 20260508 00:10:37.505 | INFO | "huge" : "", | 20260508 00:10:37.505 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:37.505 | INFO | "malloc" : " 14 GB", | 20260508 00:10:37.505 | INFO | "pinned" : " 29 GB" | 20260508 00:10:37.505 | INFO | } | 20260508 00:10:37.505 | INFO | Perf report for the last 5.13827 seconds: | 20260508 00:10:37.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:37.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.505 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:10:37.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.505 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:10:37.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:37.506 | INFO | Uploaded 36GiB to GPU in 2.30201s (15.6385GiB/s) | 20260508 00:10:37.506 | INFO | Downloaded 34GiB to GPU in 2.48951s (13.6573GiB/s) | 20260508 00:10:42.639 | INFO | Analyzing chunk frames = [3697152,3697664) | 20260508 00:10:42.639 | INFO | MemInfo: { | 20260508 00:10:42.639 | INFO | "gpu" : " 17 GB", | 20260508 00:10:42.639 | INFO | "huge" : "", | 20260508 00:10:42.640 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:42.640 | INFO | "malloc" : " 14 GB", | 20260508 00:10:42.640 | INFO | "pinned" : " 29 GB" | 20260508 00:10:42.640 | INFO | } | 20260508 00:10:42.640 | INFO | Perf report for the last 5.13461 seconds: | 20260508 00:10:42.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:42.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.640 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:10:42.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.640 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260508 00:10:42.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:42.641 | INFO | Uploaded 36GiB to GPU in 2.39473s (15.033GiB/s) | 20260508 00:10:42.641 | INFO | Downloaded 34GiB to GPU in 2.40752s (14.1224GiB/s) | 20260508 00:10:47.780 | INFO | Analyzing chunk frames = [3697664,3698176) | 20260508 00:10:47.780 | INFO | MemInfo: { | 20260508 00:10:47.780 | INFO | "gpu" : " 17 GB", | 20260508 00:10:47.780 | INFO | "huge" : "", | 20260508 00:10:47.781 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:47.781 | INFO | "malloc" : " 14 GB", | 20260508 00:10:47.781 | INFO | "pinned" : " 29 GB" | 20260508 00:10:47.781 | INFO | } | 20260508 00:10:47.781 | INFO | Perf report for the last 5.141 seconds: | 20260508 00:10:47.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:47.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:10:47.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260508 00:10:47.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:47.781 | INFO | Uploaded 36GiB to GPU in 2.55858s (14.0703GiB/s) | 20260508 00:10:47.782 | INFO | Downloaded 34GiB to GPU in 2.47924s (13.7139GiB/s) | 20260508 00:10:52.922 | INFO | Analyzing chunk frames = [3698176,3698688) | 20260508 00:10:52.922 | INFO | MemInfo: { | 20260508 00:10:52.922 | INFO | "gpu" : " 17 GB", | 20260508 00:10:52.922 | INFO | "huge" : "", | 20260508 00:10:52.923 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:52.923 | INFO | "malloc" : " 14 GB", | 20260508 00:10:52.923 | INFO | "pinned" : " 29 GB" | 20260508 00:10:52.923 | INFO | } | 20260508 00:10:52.923 | INFO | Perf report for the last 5.14195 seconds: | 20260508 00:10:52.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:52.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.923 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:10:52.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.923 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260508 00:10:52.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:52.924 | INFO | Uploaded 36GiB to GPU in 2.49704s (14.4171GiB/s) | 20260508 00:10:52.924 | INFO | Downloaded 34GiB to GPU in 2.45259s (13.8629GiB/s) | 20260508 00:10:53.036 | INFO | Starting a round of Baz IO | 20260508 00:10:56.495 | INFO | Populating RTMetrics report | 20260508 00:10:56.495 | INFO | RTMetrics report populated | 20260508 00:10:56.495 | INFO | Opening tmp RTMetrics json file | 20260508 00:10:56.600 | INFO | Serializing RTMetrics to json | 20260508 00:10:56.682 | INFO | Serialization done, swapping tmp file now... | 20260508 00:10:56.684 | INFO | RTMetrics done | 20260508 00:10:56.703 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.667 seconds | 20260508 00:10:56.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.30% / 0.00% / 57.70% | 20260508 00:10:56.703 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06051% / 7.94729e-06% / 7.15256e-05% | 20260508 00:10:56.703 | INFO | Baz IO overall: | 20260508 00:10:56.703 | INFO | BytesWritten : 1681751799260 | 20260508 00:10:56.703 | INFO | NumEvents : 814096235439 | 20260508 00:10:58.066 | INFO | Analyzing chunk frames = [3698688,3699200) | 20260508 00:10:58.066 | INFO | MemInfo: { | 20260508 00:10:58.066 | INFO | "gpu" : " 17 GB", | 20260508 00:10:58.066 | INFO | "huge" : "", | 20260508 00:10:58.066 | INFO | "huge_shared" : " 29 GB", | 20260508 00:10:58.066 | INFO | "malloc" : " 14 GB", | 20260508 00:10:58.066 | INFO | "pinned" : " 29 GB" | 20260508 00:10:58.066 | INFO | } | 20260508 00:10:58.066 | INFO | Perf report for the last 5.14321 seconds: | 20260508 00:10:58.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:10:58.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:58.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:58.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:58.066 | INFO | ANALYSIS : 68.7%, 2.9, 105.6ms, 0 | 20260508 00:10:58.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:10:58.066 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260508 00:10:58.066 | INFO | PRE_HQ : 67.4%, 1.8, 61.6ms, 0 | 20260508 00:10:58.066 | INFO | BAZWRITER : 70.9%, 3.9, 142.1ms, 0 | 20260508 00:10:58.066 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260508 00:10:58.066 | INFO | Uploaded 36GiB to GPU in 2.60488s (13.8202GiB/s) | 20260508 00:10:58.066 | INFO | Downloaded 38GiB to GPU in 2.69442s (14.1032GiB/s) | 20260508 00:11:03.204 | INFO | Analyzing chunk frames = [3699200,3699712) | 20260508 00:11:03.204 | INFO | MemInfo: { | 20260508 00:11:03.204 | INFO | "gpu" : " 17 GB", | 20260508 00:11:03.204 | INFO | "huge" : "", | 20260508 00:11:03.204 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:03.204 | INFO | "malloc" : " 14 GB", | 20260508 00:11:03.205 | INFO | "pinned" : " 29 GB" | 20260508 00:11:03.205 | INFO | } | 20260508 00:11:03.205 | INFO | Perf report for the last 5.13858 seconds: | 20260508 00:11:03.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:03.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 00:11:03.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 00:11:03.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:03.205 | INFO | Uploaded 36GiB to GPU in 2.54737s (14.1322GiB/s) | 20260508 00:11:03.205 | INFO | Downloaded 34GiB to GPU in 2.42253s (14.0349GiB/s) | 20260508 00:11:08.341 | INFO | Analyzing chunk frames = [3699712,3700224) | 20260508 00:11:08.341 | INFO | MemInfo: { | 20260508 00:11:08.341 | INFO | "gpu" : " 17 GB", | 20260508 00:11:08.341 | INFO | "huge" : "", | 20260508 00:11:08.341 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:08.341 | INFO | "malloc" : " 14 GB", | 20260508 00:11:08.341 | INFO | "pinned" : " 29 GB" | 20260508 00:11:08.341 | INFO | } | 20260508 00:11:08.341 | INFO | Perf report for the last 5.13645 seconds: | 20260508 00:11:08.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:08.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:11:08.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:11:08.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:08.341 | INFO | Uploaded 36GiB to GPU in 2.52917s (14.2339GiB/s) | 20260508 00:11:08.341 | INFO | Downloaded 34GiB to GPU in 2.47181s (13.7551GiB/s) | 20260508 00:11:13.482 | INFO | Analyzing chunk frames = [3700224,3700736) | 20260508 00:11:13.482 | INFO | MemInfo: { | 20260508 00:11:13.482 | INFO | "gpu" : " 17 GB", | 20260508 00:11:13.482 | INFO | "huge" : "", | 20260508 00:11:13.482 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:13.482 | INFO | "malloc" : " 14 GB", | 20260508 00:11:13.482 | INFO | "pinned" : " 29 GB" | 20260508 00:11:13.482 | INFO | } | 20260508 00:11:13.482 | INFO | Perf report for the last 5.14127 seconds: | 20260508 00:11:13.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:13.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.482 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 00:11:13.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.482 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:11:13.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:13.483 | INFO | Uploaded 36GiB to GPU in 2.5358s (14.1967GiB/s) | 20260508 00:11:13.483 | INFO | Downloaded 34GiB to GPU in 2.48735s (13.6691GiB/s) | 20260508 00:11:18.620 | INFO | Analyzing chunk frames = [3700736,3701248) | 20260508 00:11:18.620 | INFO | MemInfo: { | 20260508 00:11:18.620 | INFO | "gpu" : " 17 GB", | 20260508 00:11:18.620 | INFO | "huge" : "", | 20260508 00:11:18.620 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:18.621 | INFO | "malloc" : " 14 GB", | 20260508 00:11:18.621 | INFO | "pinned" : " 29 GB" | 20260508 00:11:18.621 | INFO | } | 20260508 00:11:18.621 | INFO | Perf report for the last 5.13827 seconds: | 20260508 00:11:18.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:18.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:11:18.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:11:18.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:18.621 | INFO | Uploaded 36GiB to GPU in 2.37853s (15.1354GiB/s) | 20260508 00:11:18.621 | INFO | Downloaded 34GiB to GPU in 2.47751s (13.7235GiB/s) | 20260508 00:11:23.761 | INFO | Analyzing chunk frames = [3701248,3701760) | 20260508 00:11:23.761 | INFO | MemInfo: { | 20260508 00:11:23.761 | INFO | "gpu" : " 17 GB", | 20260508 00:11:23.761 | INFO | "huge" : "", | 20260508 00:11:23.762 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:23.762 | INFO | "malloc" : " 14 GB", | 20260508 00:11:23.762 | INFO | "pinned" : " 29 GB" | 20260508 00:11:23.762 | INFO | } | 20260508 00:11:23.762 | INFO | Perf report for the last 5.141 seconds: | 20260508 00:11:23.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:23.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 00:11:23.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:11:23.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:23.762 | INFO | Uploaded 36GiB to GPU in 2.39835s (15.0103GiB/s) | 20260508 00:11:23.762 | INFO | Downloaded 34GiB to GPU in 2.40302s (14.1489GiB/s) | 20260508 00:11:28.901 | INFO | Analyzing chunk frames = [3701760,3702272) | 20260508 00:11:28.901 | INFO | MemInfo: { | 20260508 00:11:28.901 | INFO | "gpu" : " 17 GB", | 20260508 00:11:28.901 | INFO | "huge" : "", | 20260508 00:11:28.901 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:28.901 | INFO | "malloc" : " 14 GB", | 20260508 00:11:28.901 | INFO | "pinned" : " 29 GB" | 20260508 00:11:28.901 | INFO | } | 20260508 00:11:28.902 | INFO | Perf report for the last 5.13988 seconds: | 20260508 00:11:28.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:28.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:11:28.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 00:11:28.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:28.902 | INFO | Uploaded 36GiB to GPU in 2.5321s (14.2174GiB/s) | 20260508 00:11:28.902 | INFO | Downloaded 34GiB to GPU in 2.45917s (13.8258GiB/s) | 20260508 00:11:34.039 | INFO | Analyzing chunk frames = [3702272,3702784) | 20260508 00:11:34.039 | INFO | MemInfo: { | 20260508 00:11:34.039 | INFO | "gpu" : " 17 GB", | 20260508 00:11:34.039 | INFO | "huge" : "", | 20260508 00:11:34.039 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:34.039 | INFO | "malloc" : " 14 GB", | 20260508 00:11:34.039 | INFO | "pinned" : " 29 GB" | 20260508 00:11:34.039 | INFO | } | 20260508 00:11:34.039 | INFO | Perf report for the last 5.1376 seconds: | 20260508 00:11:34.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:34.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.039 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 00:11:34.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.039 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260508 00:11:34.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:34.040 | INFO | Uploaded 36GiB to GPU in 2.54855s (14.1257GiB/s) | 20260508 00:11:34.040 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:11:34.040 | INFO | Total mem usage: 16.7 GiB | 20260508 00:11:34.040 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:11:34.040 | INFO | Downloaded 34GiB to GPU in 2.45944s (13.8243GiB/s) | 20260508 00:11:34.040 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 00:11:34.040 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:11:34.040 | INFO | High Water Breakdown: | 20260508 00:11:34.040 | INFO | 59.7% : 8.72 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:11:34.040 | INFO | 40.1% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:11:34.040 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:11:34.040 | INFO | ------------------------------------------ | 20260508 00:11:34.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:11:34.040 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:11:34.040 | INFO | Total mem usage: 26.5 GiB | 20260508 00:11:34.040 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:11:34.040 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 00:11:34.040 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:11:34.040 | INFO | High Water Breakdown: | 20260508 00:11:34.040 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:11:34.040 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:11:34.040 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:11:34.040 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:11:34.040 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:11:34.040 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:11:34.040 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:11:34.040 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:11:34.040 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:11:34.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:11:34.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:11:34.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:11:34.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:11:34.040 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:11:34.040 | INFO | ------------------------------------------ | 20260508 00:11:34.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:11:34.040 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:11:34.040 | INFO | Total mem usage: 27.8 GiB | 20260508 00:11:34.040 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:11:34.040 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:11:34.040 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:11:34.040 | INFO | High Water Breakdown: | 20260508 00:11:34.040 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:11:34.040 | INFO | ------------------------------------------ | 20260508 00:11:34.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:11:34.040 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:11:34.040 | INFO | Total mem usage: 15.7 GiB | 20260508 00:11:34.040 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:11:34.040 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:11:34.040 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:11:34.040 | INFO | High Water Breakdown: | 20260508 00:11:34.040 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:11:34.040 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:11:34.040 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:11:34.040 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:11:34.040 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:11:34.040 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:11:34.040 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:11:34.040 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:11:34.040 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:11:34.040 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:11:34.040 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:11:34.040 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:11:34.040 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:11:34.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:11:34.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:11:34.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:11:34.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:11:34.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:11:34.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:11:34.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:11:34.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:11:34.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:11:34.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:11:34.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:11:34.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:11:34.041 | INFO | ------------------------------------------ | 20260508 00:11:34.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:11:34.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:11:34.041 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:11:34.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:11:34.041 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.40e+01 / 1.73e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 0.30%, AggregateStats -- 9.91e-02 / 1.04e-01, 1.25e-02 / 5.36e-03 | 20260508 00:11:34.041 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.47e+00, 5.89e-01 / 2.18e-01 | 20260508 00:11:34.041 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.06e-01 / 9.99e-02 | 20260508 00:11:34.041 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.58e+01, 4.14e+00 / 1.52e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.68e+00 / 1.62e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.30e-01 / 2.08e-01 | 20260508 00:11:34.041 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.29e+00, 2.82e-01 / 1.34e-01 | 20260508 00:11:34.041 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.46e-01 / 3.81e-01, 5.66e-02 / 2.96e-02 | 20260508 00:11:34.041 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:11:34.041 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:11:34.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:11:34.041 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 2.13%, LockUpload -- 2.27e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 2.73%, LockDownload -- 2.91e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 15.71%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:11:34.041 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 00:11:34.153 | INFO | Starting a round of Baz IO | 20260508 00:11:37.616 | INFO | Populating RTMetrics report | 20260508 00:11:37.616 | INFO | RTMetrics report populated | 20260508 00:11:37.616 | INFO | Opening tmp RTMetrics json file | 20260508 00:11:37.861 | INFO | Serializing RTMetrics to json | 20260508 00:11:37.885 | INFO | Serialization done, swapping tmp file now... | 20260508 00:11:37.885 | INFO | RTMetrics done | 20260508 00:11:37.918 | INFO | Finished a round of Baz IO: Wrote 1.698 GiB in 3.764 seconds | 20260508 00:11:37.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.29% / 0.00% / 57.70% | 20260508 00:11:37.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.05817% / 7.94729e-06% / 5.96046e-05% | 20260508 00:11:37.918 | INFO | Baz IO overall: | 20260508 00:11:37.918 | INFO | BytesWritten : 1683574645606 | 20260508 00:11:37.918 | INFO | NumEvents : 814973573060 | 20260508 00:11:39.178 | INFO | Analyzing chunk frames = [3702784,3703296) | 20260508 00:11:39.178 | INFO | MemInfo: { | 20260508 00:11:39.178 | INFO | "gpu" : " 17 GB", | 20260508 00:11:39.178 | INFO | "huge" : "", | 20260508 00:11:39.178 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:39.178 | INFO | "malloc" : " 14 GB", | 20260508 00:11:39.178 | INFO | "pinned" : " 29 GB" | 20260508 00:11:39.178 | INFO | } | 20260508 00:11:39.178 | INFO | Perf report for the last 5.13871 seconds: | 20260508 00:11:39.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:39.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:39.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:39.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:39.178 | INFO | ANALYSIS : 69.2%, 2.9, 104.7ms, 0 | 20260508 00:11:39.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:39.178 | INFO | RT_METRICS : 49.3%, 1.0, 25.9ms, 0 | 20260508 00:11:39.178 | INFO | PRE_HQ : 67.9%, 1.8, 62.2ms, 0 | 20260508 00:11:39.178 | INFO | BAZWRITER : 72.9%, 4.3, 160.7ms, 0 | 20260508 00:11:39.178 | INFO | BAZ_SAMPLING : 4.5%, 1.0, 2.4ms, 0 | 20260508 00:11:39.178 | INFO | Uploaded 36GiB to GPU in 2.4964s (14.4208GiB/s) | 20260508 00:11:39.178 | INFO | Downloaded 38GiB to GPU in 2.69154s (14.1183GiB/s) | 20260508 00:11:44.315 | INFO | Analyzing chunk frames = [3703296,3703808) | 20260508 00:11:44.315 | INFO | MemInfo: { | 20260508 00:11:44.315 | INFO | "gpu" : " 17 GB", | 20260508 00:11:44.315 | INFO | "huge" : "", | 20260508 00:11:44.315 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:44.315 | INFO | "malloc" : " 14 GB", | 20260508 00:11:44.316 | INFO | "pinned" : " 29 GB" | 20260508 00:11:44.316 | INFO | } | 20260508 00:11:44.316 | INFO | Perf report for the last 5.13774 seconds: | 20260508 00:11:44.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:44.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:11:44.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:11:44.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:44.316 | INFO | Uploaded 36GiB to GPU in 2.54192s (14.1625GiB/s) | 20260508 00:11:44.316 | INFO | Downloaded 34GiB to GPU in 2.49636s (13.6198GiB/s) | 20260508 00:11:49.458 | INFO | Analyzing chunk frames = [3703808,3704320) | 20260508 00:11:49.458 | INFO | MemInfo: { | 20260508 00:11:49.458 | INFO | "gpu" : " 17 GB", | 20260508 00:11:49.458 | INFO | "huge" : "", | 20260508 00:11:49.458 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:49.458 | INFO | "malloc" : " 14 GB", | 20260508 00:11:49.458 | INFO | "pinned" : " 29 GB" | 20260508 00:11:49.458 | INFO | } | 20260508 00:11:49.458 | INFO | Perf report for the last 5.14222 seconds: | 20260508 00:11:49.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:49.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:11:49.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:11:49.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:49.458 | INFO | Uploaded 36GiB to GPU in 2.43929s (14.7584GiB/s) | 20260508 00:11:49.458 | INFO | Downloaded 34GiB to GPU in 2.49885s (13.6063GiB/s) | 20260508 00:11:54.591 | INFO | Analyzing chunk frames = [3704320,3704832) | 20260508 00:11:54.591 | INFO | MemInfo: { | 20260508 00:11:54.591 | INFO | "gpu" : " 17 GB", | 20260508 00:11:54.591 | INFO | "huge" : "", | 20260508 00:11:54.591 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:54.591 | INFO | "malloc" : " 14 GB", | 20260508 00:11:54.591 | INFO | "pinned" : " 29 GB" | 20260508 00:11:54.591 | INFO | } | 20260508 00:11:54.591 | INFO | Perf report for the last 5.13303 seconds: | 20260508 00:11:54.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:54.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.591 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 00:11:54.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.591 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:11:54.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:54.592 | INFO | Uploaded 36GiB to GPU in 2.48402s (14.4926GiB/s) | 20260508 00:11:54.592 | INFO | Downloaded 34GiB to GPU in 2.48539s (13.68GiB/s) | 20260508 00:11:59.732 | INFO | Analyzing chunk frames = [3704832,3705344) | 20260508 00:11:59.732 | INFO | MemInfo: { | 20260508 00:11:59.732 | INFO | "gpu" : " 17 GB", | 20260508 00:11:59.732 | INFO | "huge" : "", | 20260508 00:11:59.732 | INFO | "huge_shared" : " 29 GB", | 20260508 00:11:59.732 | INFO | "malloc" : " 14 GB", | 20260508 00:11:59.732 | INFO | "pinned" : " 29 GB" | 20260508 00:11:59.732 | INFO | } | 20260508 00:11:59.732 | INFO | Perf report for the last 5.14103 seconds: | 20260508 00:11:59.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:11:59.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 00:11:59.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 00:11:59.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:11:59.732 | INFO | Uploaded 36GiB to GPU in 2.38744s (15.0789GiB/s) | 20260508 00:11:59.732 | INFO | Downloaded 34GiB to GPU in 2.50189s (13.5897GiB/s) | 20260508 00:12:04.878 | INFO | Analyzing chunk frames = [3705344,3705856) | 20260508 00:12:04.878 | INFO | MemInfo: { | 20260508 00:12:04.878 | INFO | "gpu" : " 17 GB", | 20260508 00:12:04.878 | INFO | "huge" : "", | 20260508 00:12:04.878 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:04.878 | INFO | "malloc" : " 14 GB", | 20260508 00:12:04.878 | INFO | "pinned" : " 29 GB" | 20260508 00:12:04.878 | INFO | } | 20260508 00:12:04.878 | INFO | Perf report for the last 5.14575 seconds: | 20260508 00:12:04.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:04.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260508 00:12:04.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:12:04.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:04.878 | INFO | Uploaded 36GiB to GPU in 2.52031s (14.2839GiB/s) | 20260508 00:12:04.878 | INFO | Downloaded 34GiB to GPU in 2.43731s (13.9498GiB/s) | 20260508 00:12:10.015 | INFO | Analyzing chunk frames = [3705856,3706368) | 20260508 00:12:10.015 | INFO | MemInfo: { | 20260508 00:12:10.015 | INFO | "gpu" : " 17 GB", | 20260508 00:12:10.015 | INFO | "huge" : "", | 20260508 00:12:10.015 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:10.015 | INFO | "malloc" : " 14 GB", | 20260508 00:12:10.015 | INFO | "pinned" : " 29 GB" | 20260508 00:12:10.015 | INFO | } | 20260508 00:12:10.015 | INFO | Perf report for the last 5.13754 seconds: | 20260508 00:12:10.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:10.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.015 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 00:12:10.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.015 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 00:12:10.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:10.016 | INFO | Uploaded 36GiB to GPU in 2.55221s (14.1054GiB/s) | 20260508 00:12:10.016 | INFO | Downloaded 34GiB to GPU in 2.46997s (13.7653GiB/s) | 20260508 00:12:15.154 | INFO | Analyzing chunk frames = [3706368,3706880) | 20260508 00:12:15.154 | INFO | MemInfo: { | 20260508 00:12:15.154 | INFO | "gpu" : " 17 GB", | 20260508 00:12:15.154 | INFO | "huge" : "", | 20260508 00:12:15.154 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:15.154 | INFO | "malloc" : " 14 GB", | 20260508 00:12:15.154 | INFO | "pinned" : " 29 GB" | 20260508 00:12:15.154 | INFO | } | 20260508 00:12:15.154 | INFO | Perf report for the last 5.13921 seconds: | 20260508 00:12:15.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:15.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.154 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 00:12:15.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.155 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 00:12:15.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:15.155 | INFO | Uploaded 36GiB to GPU in 2.50145s (14.3917GiB/s) | 20260508 00:12:15.155 | INFO | Downloaded 34GiB to GPU in 2.4464s (13.8979GiB/s) | 20260508 00:12:15.253 | INFO | Starting a round of Baz IO | 20260508 00:12:18.723 | INFO | Populating RTMetrics report | 20260508 00:12:18.724 | INFO | RTMetrics report populated | 20260508 00:12:18.724 | INFO | Opening tmp RTMetrics json file | 20260508 00:12:18.734 | INFO | Serializing RTMetrics to json | 20260508 00:12:18.748 | INFO | Serialization done, swapping tmp file now... | 20260508 00:12:18.749 | INFO | RTMetrics done | 20260508 00:12:18.821 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.567 seconds | 20260508 00:12:18.821 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.29% / 0.00% / 57.71% | 20260508 00:12:18.821 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.05639% / 1.19209e-05% / 7.54992e-05% | 20260508 00:12:18.821 | INFO | Baz IO overall: | 20260508 00:12:18.821 | INFO | BytesWritten : 1685397058443 | 20260508 00:12:18.821 | INFO | NumEvents : 815850862500 | 20260508 00:12:20.295 | INFO | Analyzing chunk frames = [3706880,3707392) | 20260508 00:12:20.295 | INFO | MemInfo: { | 20260508 00:12:20.295 | INFO | "gpu" : " 17 GB", | 20260508 00:12:20.295 | INFO | "huge" : "", | 20260508 00:12:20.295 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:20.295 | INFO | "malloc" : " 14 GB", | 20260508 00:12:20.295 | INFO | "pinned" : " 29 GB" | 20260508 00:12:20.295 | INFO | } | 20260508 00:12:20.295 | INFO | Perf report for the last 5.14026 seconds: | 20260508 00:12:20.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:20.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:20.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:20.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:20.295 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260508 00:12:20.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:20.295 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260508 00:12:20.295 | INFO | PRE_HQ : 65.7%, 1.7, 56.9ms, 0 | 20260508 00:12:20.295 | INFO | BAZWRITER : 69.0%, 4.5, 161.3ms, 0 | 20260508 00:12:20.295 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 00:12:20.295 | INFO | Uploaded 36GiB to GPU in 2.58645s (13.9187GiB/s) | 20260508 00:12:20.295 | INFO | Downloaded 38GiB to GPU in 2.70055s (14.0712GiB/s) | 20260508 00:12:25.438 | INFO | Analyzing chunk frames = [3707392,3707904) | 20260508 00:12:25.438 | INFO | MemInfo: { | 20260508 00:12:25.438 | INFO | "gpu" : " 17 GB", | 20260508 00:12:25.438 | INFO | "huge" : "", | 20260508 00:12:25.438 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:25.438 | INFO | "malloc" : " 14 GB", | 20260508 00:12:25.438 | INFO | "pinned" : " 29 GB" | 20260508 00:12:25.438 | INFO | } | 20260508 00:12:25.438 | INFO | Perf report for the last 5.14311 seconds: | 20260508 00:12:25.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:25.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:12:25.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:12:25.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:25.438 | INFO | Uploaded 36GiB to GPU in 2.58184s (13.9435GiB/s) | 20260508 00:12:25.439 | INFO | Downloaded 34GiB to GPU in 2.49657s (13.6187GiB/s) | 20260508 00:12:30.570 | INFO | Analyzing chunk frames = [3707904,3708416) | 20260508 00:12:30.570 | INFO | MemInfo: { | 20260508 00:12:30.570 | INFO | "gpu" : " 17 GB", | 20260508 00:12:30.571 | INFO | "huge" : "", | 20260508 00:12:30.571 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:30.571 | INFO | "malloc" : " 14 GB", | 20260508 00:12:30.571 | INFO | "pinned" : " 29 GB" | 20260508 00:12:30.571 | INFO | } | 20260508 00:12:30.571 | INFO | Perf report for the last 5.13281 seconds: | 20260508 00:12:30.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:30.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 00:12:30.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:12:30.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:30.571 | INFO | Uploaded 36GiB to GPU in 2.49515s (14.428GiB/s) | 20260508 00:12:30.571 | INFO | Downloaded 34GiB to GPU in 2.48879s (13.6613GiB/s) | 20260508 00:12:35.707 | INFO | Analyzing chunk frames = [3708416,3708928) | 20260508 00:12:35.707 | INFO | MemInfo: { | 20260508 00:12:35.707 | INFO | "gpu" : " 17 GB", | 20260508 00:12:35.707 | INFO | "huge" : "", | 20260508 00:12:35.707 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:35.707 | INFO | "malloc" : " 14 GB", | 20260508 00:12:35.707 | INFO | "pinned" : " 29 GB" | 20260508 00:12:35.707 | INFO | } | 20260508 00:12:35.707 | INFO | Perf report for the last 5.13669 seconds: | 20260508 00:12:35.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:35.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.707 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:12:35.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.707 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:12:35.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:35.708 | INFO | Uploaded 36GiB to GPU in 2.50329s (14.3811GiB/s) | 20260508 00:12:35.708 | INFO | Downloaded 34GiB to GPU in 2.48734s (13.6692GiB/s) | 20260508 00:12:40.849 | INFO | Analyzing chunk frames = [3708928,3709440) | 20260508 00:12:40.849 | INFO | MemInfo: { | 20260508 00:12:40.849 | INFO | "gpu" : " 17 GB", | 20260508 00:12:40.849 | INFO | "huge" : "", | 20260508 00:12:40.849 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:40.849 | INFO | "malloc" : " 14 GB", | 20260508 00:12:40.849 | INFO | "pinned" : " 29 GB" | 20260508 00:12:40.849 | INFO | } | 20260508 00:12:40.849 | INFO | Perf report for the last 5.1416 seconds: | 20260508 00:12:40.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:40.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:12:40.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:12:40.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:40.849 | INFO | Uploaded 36GiB to GPU in 2.31567s (15.5462GiB/s) | 20260508 00:12:40.849 | INFO | Downloaded 34GiB to GPU in 2.51029s (13.5442GiB/s) | 20260508 00:12:45.994 | INFO | Analyzing chunk frames = [3709440,3709952) | 20260508 00:12:45.994 | INFO | MemInfo: { | 20260508 00:12:45.994 | INFO | "gpu" : " 17 GB", | 20260508 00:12:45.994 | INFO | "huge" : "", | 20260508 00:12:45.994 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:45.994 | INFO | "malloc" : " 14 GB", | 20260508 00:12:45.994 | INFO | "pinned" : " 29 GB" | 20260508 00:12:45.994 | INFO | } | 20260508 00:12:45.994 | INFO | Perf report for the last 5.14476 seconds: | 20260508 00:12:45.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:45.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 00:12:45.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260508 00:12:45.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:45.994 | INFO | Uploaded 36GiB to GPU in 2.37821s (15.1374GiB/s) | 20260508 00:12:45.994 | INFO | Downloaded 34GiB to GPU in 2.42286s (14.033GiB/s) | 20260508 00:12:51.135 | INFO | Analyzing chunk frames = [3709952,3710464) | 20260508 00:12:51.135 | INFO | MemInfo: { | 20260508 00:12:51.135 | INFO | "gpu" : " 17 GB", | 20260508 00:12:51.135 | INFO | "huge" : "", | 20260508 00:12:51.135 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:51.135 | INFO | "malloc" : " 14 GB", | 20260508 00:12:51.135 | INFO | "pinned" : " 29 GB" | 20260508 00:12:51.135 | INFO | } | 20260508 00:12:51.135 | INFO | Perf report for the last 5.14128 seconds: | 20260508 00:12:51.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:51.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 00:12:51.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:12:51.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:51.135 | INFO | Uploaded 36GiB to GPU in 2.49519s (14.4277GiB/s) | 20260508 00:12:51.135 | INFO | Downloaded 34GiB to GPU in 2.46701s (13.7819GiB/s) | 20260508 00:12:56.273 | INFO | Analyzing chunk frames = [3710464,3710976) | 20260508 00:12:56.273 | INFO | MemInfo: { | 20260508 00:12:56.273 | INFO | "gpu" : " 17 GB", | 20260508 00:12:56.273 | INFO | "huge" : "", | 20260508 00:12:56.273 | INFO | "huge_shared" : " 29 GB", | 20260508 00:12:56.273 | INFO | "malloc" : " 14 GB", | 20260508 00:12:56.273 | INFO | "pinned" : " 29 GB" | 20260508 00:12:56.273 | INFO | } | 20260508 00:12:56.273 | INFO | Perf report for the last 5.13773 seconds: | 20260508 00:12:56.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:12:56.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:12:56.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:12:56.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:12:56.273 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:12:56.273 | INFO | Total mem usage: 16.7 GiB | 20260508 00:12:56.273 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:12:56.273 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -5.69%) | 20260508 00:12:56.273 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:12:56.273 | INFO | High Water Breakdown: | 20260508 00:12:56.273 | INFO | 61% : 9.63 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:12:56.273 | INFO | 38.9% : 6.14 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:12:56.273 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:12:56.273 | INFO | ------------------------------------------ | 20260508 00:12:56.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:12:56.273 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:12:56.273 | INFO | Total mem usage: 26.5 GiB | 20260508 00:12:56.273 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:12:56.273 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:12:56.273 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:12:56.273 | INFO | High Water Breakdown: | 20260508 00:12:56.273 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:12:56.273 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:12:56.273 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:12:56.273 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:12:56.273 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:12:56.273 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:12:56.273 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:12:56.273 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:12:56.273 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:12:56.273 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:12:56.273 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:12:56.273 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:12:56.273 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:12:56.273 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:12:56.273 | INFO | ------------------------------------------ | 20260508 00:12:56.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:12:56.273 | INFO | Uploaded 36GiB to GPU in 2.51757s (14.2995GiB/s) | 20260508 00:12:56.273 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:12:56.273 | INFO | Total mem usage: 27.8 GiB | 20260508 00:12:56.273 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:12:56.273 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:12:56.273 | INFO | Downloaded 34GiB to GPU in 2.4707s (13.7613GiB/s) | 20260508 00:12:56.273 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:12:56.273 | INFO | High Water Breakdown: | 20260508 00:12:56.273 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:12:56.273 | INFO | ------------------------------------------ | 20260508 00:12:56.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:12:56.273 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:12:56.273 | INFO | Total mem usage: 15.7 GiB | 20260508 00:12:56.273 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:12:56.273 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:12:56.273 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 00:12:56.273 | INFO | High Water Breakdown: | 20260508 00:12:56.273 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:12:56.273 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:12:56.273 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:12:56.273 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:12:56.273 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:12:56.273 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:12:56.273 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:12:56.273 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:12:56.273 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:12:56.273 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:12:56.273 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:12:56.273 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:12:56.273 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:12:56.273 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:12:56.273 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:12:56.273 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:12:56.273 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:12:56.273 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:12:56.273 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:12:56.273 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:12:56.273 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:12:56.274 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:12:56.274 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:12:56.274 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:12:56.274 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:12:56.274 | INFO | ------------------------------------------ | 20260508 00:12:56.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:12:56.274 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:12:56.274 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:12:56.274 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:12:56.274 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.65e+01, 1.43e+01 / 1.85e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.14e-02 / 5.16e-03 | 20260508 00:12:56.274 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.48e+00, 5.85e-01 / 1.88e-01 | 20260508 00:12:56.274 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.50e+00, 2.07e-01 / 1.11e-01 | 20260508 00:12:56.274 | INFO | PerfReport: 16.19%, DME -- 4.25e+01 / 4.66e+01, 4.39e+00 / 2.03e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.59e+00 / 1.44e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.32e-01 / 2.25e-01 | 20260508 00:12:56.274 | INFO | PerfReport: 3.40%, Metrics -- 1.12e+00 / 1.34e+00, 2.80e-01 / 1.47e-01 | 20260508 00:12:56.274 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.43e-01 / 3.82e-01, 5.43e-02 / 2.61e-02 | 20260508 00:12:56.274 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:12:56.274 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:12:56.274 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:12:56.274 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 24.29%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 24.46%, Download -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 30.74%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 2.92%, LockDownload -- 3.11e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 15.55%, LockCompute -- 1.66e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 00:12:56.274 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260508 00:12:56.370 | INFO | Starting a round of Baz IO | 20260508 00:12:59.830 | INFO | Populating RTMetrics report | 20260508 00:12:59.830 | INFO | RTMetrics report populated | 20260508 00:12:59.830 | INFO | Opening tmp RTMetrics json file | 20260508 00:12:59.964 | INFO | Serializing RTMetrics to json | 20260508 00:13:00.072 | INFO | Serialization done, swapping tmp file now... | 20260508 00:13:00.077 | INFO | RTMetrics done | 20260508 00:13:00.095 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.724 seconds | 20260508 00:13:00.095 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.28% / 0.00% / 57.71% | 20260508 00:13:00.095 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.05234% / 0% / 4.76837e-05% | 20260508 00:13:00.095 | INFO | Baz IO overall: | 20260508 00:13:00.095 | INFO | BytesWritten : 1687217271156 | 20260508 00:13:00.095 | INFO | NumEvents : 816726652291 | 20260508 00:13:01.403 | INFO | Analyzing chunk frames = [3710976,3711488) | 20260508 00:13:01.403 | INFO | MemInfo: { | 20260508 00:13:01.403 | INFO | "gpu" : " 17 GB", | 20260508 00:13:01.403 | INFO | "huge" : "", | 20260508 00:13:01.403 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:01.404 | INFO | "malloc" : " 14 GB", | 20260508 00:13:01.404 | INFO | "pinned" : " 29 GB" | 20260508 00:13:01.404 | INFO | } | 20260508 00:13:01.404 | INFO | Perf report for the last 5.13078 seconds: | 20260508 00:13:01.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:01.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:01.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:01.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:01.404 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 00:13:01.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:01.404 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 00:13:01.404 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260508 00:13:01.404 | INFO | BAZWRITER : 71.9%, 3.9, 144.0ms, 0 | 20260508 00:13:01.404 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260508 00:13:01.404 | INFO | Uploaded 36GiB to GPU in 2.50854s (14.351GiB/s) | 20260508 00:13:01.404 | INFO | Downloaded 38GiB to GPU in 2.67971s (14.1806GiB/s) | 20260508 00:13:06.544 | INFO | Analyzing chunk frames = [3711488,3712000) | 20260508 00:13:06.544 | INFO | MemInfo: { | 20260508 00:13:06.544 | INFO | "gpu" : " 17 GB", | 20260508 00:13:06.544 | INFO | "huge" : "", | 20260508 00:13:06.544 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:06.544 | INFO | "malloc" : " 14 GB", | 20260508 00:13:06.544 | INFO | "pinned" : " 29 GB" | 20260508 00:13:06.544 | INFO | } | 20260508 00:13:06.544 | INFO | Perf report for the last 5.14079 seconds: | 20260508 00:13:06.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:06.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.544 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 00:13:06.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.545 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:13:06.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:06.545 | INFO | Uploaded 36GiB to GPU in 2.52692s (14.2466GiB/s) | 20260508 00:13:06.545 | INFO | Downloaded 34GiB to GPU in 2.49806s (13.6106GiB/s) | 20260508 00:13:11.685 | INFO | Analyzing chunk frames = [3712000,3712512) | 20260508 00:13:11.685 | INFO | MemInfo: { | 20260508 00:13:11.685 | INFO | "gpu" : " 17 GB", | 20260508 00:13:11.685 | INFO | "huge" : "", | 20260508 00:13:11.685 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:11.685 | INFO | "malloc" : " 14 GB", | 20260508 00:13:11.685 | INFO | "pinned" : " 29 GB" | 20260508 00:13:11.685 | INFO | } | 20260508 00:13:11.685 | INFO | Perf report for the last 5.14063 seconds: | 20260508 00:13:11.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:11.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 00:13:11.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:13:11.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:11.685 | INFO | Uploaded 36GiB to GPU in 2.46089s (14.6288GiB/s) | 20260508 00:13:11.685 | INFO | Downloaded 34GiB to GPU in 2.49904s (13.6052GiB/s) | 20260508 00:13:16.827 | INFO | Analyzing chunk frames = [3712512,3713024) | 20260508 00:13:16.827 | INFO | MemInfo: { | 20260508 00:13:16.827 | INFO | "gpu" : " 17 GB", | 20260508 00:13:16.827 | INFO | "huge" : "", | 20260508 00:13:16.827 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:16.827 | INFO | "malloc" : " 14 GB", | 20260508 00:13:16.827 | INFO | "pinned" : " 29 GB" | 20260508 00:13:16.827 | INFO | } | 20260508 00:13:16.827 | INFO | Perf report for the last 5.14227 seconds: | 20260508 00:13:16.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:16.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.828 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 00:13:16.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.828 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:13:16.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:16.828 | INFO | Uploaded 36GiB to GPU in 2.45077s (14.6892GiB/s) | 20260508 00:13:16.828 | INFO | Downloaded 34GiB to GPU in 2.41858s (14.0578GiB/s) | 20260508 00:13:21.966 | INFO | Analyzing chunk frames = [3713024,3713536) | 20260508 00:13:21.966 | INFO | MemInfo: { | 20260508 00:13:21.966 | INFO | "gpu" : " 17 GB", | 20260508 00:13:21.966 | INFO | "huge" : "", | 20260508 00:13:21.966 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:21.966 | INFO | "malloc" : " 14 GB", | 20260508 00:13:21.966 | INFO | "pinned" : " 29 GB" | 20260508 00:13:21.966 | INFO | } | 20260508 00:13:21.966 | INFO | Perf report for the last 5.13839 seconds: | 20260508 00:13:21.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:21.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:13:21.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260508 00:13:21.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:21.966 | INFO | Uploaded 36GiB to GPU in 2.29725s (15.6709GiB/s) | 20260508 00:13:21.966 | INFO | Downloaded 34GiB to GPU in 2.49869s (13.6071GiB/s) | 20260508 00:13:27.104 | INFO | Analyzing chunk frames = [3713536,3714048) | 20260508 00:13:27.104 | INFO | MemInfo: { | 20260508 00:13:27.104 | INFO | "gpu" : " 17 GB", | 20260508 00:13:27.104 | INFO | "huge" : "", | 20260508 00:13:27.104 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:27.104 | INFO | "malloc" : " 14 GB", | 20260508 00:13:27.104 | INFO | "pinned" : " 29 GB" | 20260508 00:13:27.104 | INFO | } | 20260508 00:13:27.104 | INFO | Perf report for the last 5.13854 seconds: | 20260508 00:13:27.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:27.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.104 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:13:27.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.105 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:13:27.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:27.105 | INFO | Uploaded 36GiB to GPU in 2.39241s (15.0476GiB/s) | 20260508 00:13:27.105 | INFO | Downloaded 34GiB to GPU in 2.39989s (14.1673GiB/s) | 20260508 00:13:32.246 | INFO | Analyzing chunk frames = [3714048,3714560) | 20260508 00:13:32.246 | INFO | MemInfo: { | 20260508 00:13:32.246 | INFO | "gpu" : " 17 GB", | 20260508 00:13:32.246 | INFO | "huge" : "", | 20260508 00:13:32.246 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:32.246 | INFO | "malloc" : " 14 GB", | 20260508 00:13:32.246 | INFO | "pinned" : " 29 GB" | 20260508 00:13:32.246 | INFO | } | 20260508 00:13:32.246 | INFO | Perf report for the last 5.14219 seconds: | 20260508 00:13:32.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:32.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:13:32.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:13:32.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:32.247 | INFO | Uploaded 36GiB to GPU in 2.55121s (14.111GiB/s) | 20260508 00:13:32.247 | INFO | Downloaded 34GiB to GPU in 2.47499s (13.7374GiB/s) | 20260508 00:13:37.384 | INFO | Analyzing chunk frames = [3714560,3715072) | 20260508 00:13:37.384 | INFO | MemInfo: { | 20260508 00:13:37.384 | INFO | "gpu" : " 17 GB", | 20260508 00:13:37.384 | INFO | "huge" : "", | 20260508 00:13:37.384 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:37.384 | INFO | "malloc" : " 14 GB", | 20260508 00:13:37.384 | INFO | "pinned" : " 29 GB" | 20260508 00:13:37.384 | INFO | } | 20260508 00:13:37.384 | INFO | Perf report for the last 5.13737 seconds: | 20260508 00:13:37.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:37.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:13:37.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:13:37.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:37.384 | INFO | Uploaded 36GiB to GPU in 2.53222s (14.2168GiB/s) | 20260508 00:13:37.384 | INFO | Downloaded 34GiB to GPU in 2.45227s (13.8647GiB/s) | 20260508 00:13:37.481 | INFO | Starting a round of Baz IO | 20260508 00:13:40.950 | INFO | Populating RTMetrics report | 20260508 00:13:40.950 | INFO | RTMetrics report populated | 20260508 00:13:40.951 | INFO | Opening tmp RTMetrics json file | 20260508 00:13:40.976 | INFO | Serializing RTMetrics to json | 20260508 00:13:41.024 | INFO | Serialization done, swapping tmp file now... | 20260508 00:13:41.053 | INFO | RTMetrics done | 20260508 00:13:41.087 | INFO | Finished a round of Baz IO: Wrote 1.694 GiB in 3.605 seconds | 20260508 00:13:41.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.28% / 0.00% / 57.72% | 20260508 00:13:41.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04884% / 0% / 7.15256e-05% | 20260508 00:13:41.087 | INFO | Baz IO overall: | 20260508 00:13:41.087 | INFO | BytesWritten : 1689036261186 | 20260508 00:13:41.087 | INFO | NumEvents : 817601780612 | 20260508 00:13:42.521 | INFO | Analyzing chunk frames = [3715072,3715584) | 20260508 00:13:42.521 | INFO | MemInfo: { | 20260508 00:13:42.521 | INFO | "gpu" : " 17 GB", | 20260508 00:13:42.521 | INFO | "huge" : "", | 20260508 00:13:42.521 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:42.522 | INFO | "malloc" : " 14 GB", | 20260508 00:13:42.522 | INFO | "pinned" : " 29 GB" | 20260508 00:13:42.522 | INFO | } | 20260508 00:13:42.522 | INFO | Perf report for the last 5.13786 seconds: | 20260508 00:13:42.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:42.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:42.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:42.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:42.522 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 00:13:42.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:42.522 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 00:13:42.522 | INFO | PRE_HQ : 66.0%, 1.8, 60.9ms, 0 | 20260508 00:13:42.522 | INFO | BAZWRITER : 69.8%, 3.9, 141.5ms, 0 | 20260508 00:13:42.522 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260508 00:13:42.522 | INFO | Uploaded 36GiB to GPU in 2.54146s (14.1651GiB/s) | 20260508 00:13:42.522 | INFO | Downloaded 38GiB to GPU in 2.69121s (14.12GiB/s) | 20260508 00:13:47.657 | INFO | Analyzing chunk frames = [3715584,3716096) | 20260508 00:13:47.657 | INFO | MemInfo: { | 20260508 00:13:47.657 | INFO | "gpu" : " 17 GB", | 20260508 00:13:47.657 | INFO | "huge" : "", | 20260508 00:13:47.657 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:47.657 | INFO | "malloc" : " 14 GB", | 20260508 00:13:47.657 | INFO | "pinned" : " 29 GB" | 20260508 00:13:47.657 | INFO | } | 20260508 00:13:47.657 | INFO | Perf report for the last 5.13569 seconds: | 20260508 00:13:47.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:47.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:13:47.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 00:13:47.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:47.658 | INFO | Uploaded 36GiB to GPU in 2.54188s (14.1628GiB/s) | 20260508 00:13:47.658 | INFO | Downloaded 34GiB to GPU in 2.43653s (13.9543GiB/s) | 20260508 00:13:52.800 | INFO | Analyzing chunk frames = [3716096,3716608) | 20260508 00:13:52.800 | INFO | MemInfo: { | 20260508 00:13:52.800 | INFO | "gpu" : " 17 GB", | 20260508 00:13:52.800 | INFO | "huge" : "", | 20260508 00:13:52.800 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:52.800 | INFO | "malloc" : " 14 GB", | 20260508 00:13:52.800 | INFO | "pinned" : " 29 GB" | 20260508 00:13:52.800 | INFO | } | 20260508 00:13:52.800 | INFO | Perf report for the last 5.14272 seconds: | 20260508 00:13:52.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:52.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.800 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 00:13:52.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.800 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 00:13:52.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:52.801 | INFO | Uploaded 36GiB to GPU in 2.51779s (14.2982GiB/s) | 20260508 00:13:52.801 | INFO | Downloaded 34GiB to GPU in 2.48214s (13.6978GiB/s) | 20260508 00:13:57.943 | INFO | Analyzing chunk frames = [3716608,3717120) | 20260508 00:13:57.943 | INFO | MemInfo: { | 20260508 00:13:57.943 | INFO | "gpu" : " 17 GB", | 20260508 00:13:57.943 | INFO | "huge" : "", | 20260508 00:13:57.943 | INFO | "huge_shared" : " 29 GB", | 20260508 00:13:57.943 | INFO | "malloc" : " 14 GB", | 20260508 00:13:57.943 | INFO | "pinned" : " 29 GB" | 20260508 00:13:57.943 | INFO | } | 20260508 00:13:57.943 | INFO | Perf report for the last 5.14299 seconds: | 20260508 00:13:57.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:13:57.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.943 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:13:57.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.943 | INFO | PRE_HQ : 34.2%, 1.0, 17.7ms, 0 | 20260508 00:13:57.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:13:57.944 | INFO | Uploaded 36GiB to GPU in 2.51642s (14.306GiB/s) | 20260508 00:13:57.944 | INFO | Downloaded 34GiB to GPU in 2.49995s (13.6003GiB/s) | 20260508 00:14:03.087 | INFO | Analyzing chunk frames = [3717120,3717632) | 20260508 00:14:03.087 | INFO | MemInfo: { | 20260508 00:14:03.087 | INFO | "gpu" : " 17 GB", | 20260508 00:14:03.087 | INFO | "huge" : "", | 20260508 00:14:03.087 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:03.087 | INFO | "malloc" : " 14 GB", | 20260508 00:14:03.087 | INFO | "pinned" : " 29 GB" | 20260508 00:14:03.087 | INFO | } | 20260508 00:14:03.087 | INFO | Perf report for the last 5.14369 seconds: | 20260508 00:14:03.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:03.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 00:14:03.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | PRE_HQ : 34.6%, 1.0, 18.1ms, 0 | 20260508 00:14:03.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:03.087 | INFO | Uploaded 36GiB to GPU in 2.29991s (15.6528GiB/s) | 20260508 00:14:03.087 | INFO | Downloaded 34GiB to GPU in 2.48016s (13.7088GiB/s) | 20260508 00:14:08.219 | INFO | Analyzing chunk frames = [3717632,3718144) | 20260508 00:14:08.219 | INFO | MemInfo: { | 20260508 00:14:08.219 | INFO | "gpu" : " 17 GB", | 20260508 00:14:08.219 | INFO | "huge" : "", | 20260508 00:14:08.219 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:08.219 | INFO | "malloc" : " 14 GB", | 20260508 00:14:08.219 | INFO | "pinned" : " 29 GB" | 20260508 00:14:08.219 | INFO | } | 20260508 00:14:08.219 | INFO | Perf report for the last 5.13237 seconds: | 20260508 00:14:08.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:08.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.219 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 00:14:08.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.219 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 00:14:08.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:08.220 | INFO | Uploaded 36GiB to GPU in 2.39465s (15.0335GiB/s) | 20260508 00:14:08.220 | INFO | Downloaded 34GiB to GPU in 2.40808s (14.1191GiB/s) | 20260508 00:14:13.358 | INFO | Analyzing chunk frames = [3718144,3718656) | 20260508 00:14:13.358 | INFO | MemInfo: { | 20260508 00:14:13.358 | INFO | "gpu" : " 17 GB", | 20260508 00:14:13.358 | INFO | "huge" : "", | 20260508 00:14:13.358 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:13.358 | INFO | "malloc" : " 14 GB", | 20260508 00:14:13.358 | INFO | "pinned" : " 29 GB" | 20260508 00:14:13.358 | INFO | } | 20260508 00:14:13.358 | INFO | Perf report for the last 5.13871 seconds: | 20260508 00:14:13.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:13.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.358 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 00:14:13.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.358 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 00:14:13.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:13.359 | INFO | Uploaded 36GiB to GPU in 2.4218s (14.865GiB/s) | 20260508 00:14:13.359 | INFO | Downloaded 34GiB to GPU in 2.42334s (14.0302GiB/s) | 20260508 00:14:18.495 | INFO | Analyzing chunk frames = [3718656,3719168) | 20260508 00:14:18.495 | INFO | MemInfo: { | 20260508 00:14:18.495 | INFO | "gpu" : " 17 GB", | 20260508 00:14:18.495 | INFO | "huge" : "", | 20260508 00:14:18.495 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:18.496 | INFO | "malloc" : " 14 GB", | 20260508 00:14:18.496 | INFO | "pinned" : " 29 GB" | 20260508 00:14:18.496 | INFO | } | 20260508 00:14:18.496 | INFO | Perf report for the last 5.13808 seconds: | 20260508 00:14:18.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:18.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.496 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:14:18.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.496 | INFO | PRE_HQ : 36.4%, 1.0, 18.9ms, 0 | 20260508 00:14:18.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:18.497 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:14:18.497 | INFO | Total mem usage: 16.7 GiB | 20260508 00:14:18.497 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:14:18.497 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -10.1%) | 20260508 00:14:18.497 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:14:18.497 | INFO | High Water Breakdown: | 20260508 00:14:18.497 | INFO | 60.4% : 9.15 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:14:18.497 | INFO | 39.5% : 5.99 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:14:18.497 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:14:18.497 | INFO | ------------------------------------------ | 20260508 00:14:18.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:14:18.497 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:14:18.497 | INFO | Total mem usage: 26.5 GiB | 20260508 00:14:18.497 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:14:18.497 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:14:18.497 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:14:18.497 | INFO | High Water Breakdown: | 20260508 00:14:18.497 | INFO | Uploaded 36GiB to GPU in 2.5187s (14.2931GiB/s) | 20260508 00:14:18.497 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:14:18.497 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:14:18.497 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:14:18.497 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:14:18.497 | INFO | Downloaded 34GiB to GPU in 2.45729s (13.8364GiB/s) | 20260508 00:14:18.497 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:14:18.497 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:14:18.497 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:14:18.497 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:14:18.497 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:14:18.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:14:18.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:14:18.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:14:18.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:14:18.497 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:14:18.497 | INFO | ------------------------------------------ | 20260508 00:14:18.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:14:18.497 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:14:18.497 | INFO | Total mem usage: 27.8 GiB | 20260508 00:14:18.497 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:14:18.497 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:14:18.497 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:14:18.497 | INFO | High Water Breakdown: | 20260508 00:14:18.497 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:14:18.497 | INFO | ------------------------------------------ | 20260508 00:14:18.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:14:18.497 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:14:18.497 | INFO | Total mem usage: 15.7 GiB | 20260508 00:14:18.497 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:14:18.497 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:14:18.497 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:14:18.497 | INFO | High Water Breakdown: | 20260508 00:14:18.497 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:14:18.497 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:14:18.497 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:14:18.497 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:14:18.497 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:14:18.497 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:14:18.497 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:14:18.497 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:14:18.497 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:14:18.498 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:14:18.498 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:14:18.498 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:14:18.498 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:14:18.498 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:14:18.498 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:14:18.498 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:14:18.498 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:14:18.498 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:14:18.498 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:14:18.498 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:14:18.498 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:14:18.498 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:14:18.498 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:14:18.498 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:14:18.498 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:14:18.498 | INFO | ------------------------------------------ | 20260508 00:14:18.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:14:18.498 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:14:18.498 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:14:18.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:14:18.498 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.59e+01, 1.42e+01 / 1.65e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 0.31%, AggregateStats -- 1.01e-01 / 1.06e-01, 1.29e-02 / 5.30e-03 | 20260508 00:14:18.498 | INFO | PerfReport: 12.19%, Baseline -- 4.01e+00 / 4.48e+00, 5.93e-01 / 2.41e-01 | 20260508 00:14:18.498 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.52e+00, 2.11e-01 / 8.34e-02 | 20260508 00:14:18.498 | INFO | PerfReport: 16.00%, DME -- 4.21e+01 / 4.59e+01, 4.28e+00 / 1.80e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.85e+01 / 1.93e+01, 1.68e+00 / 1.47e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.45e+00 / 2.76e+00, 4.30e-01 / 2.40e-01 | 20260508 00:14:18.498 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.33e+00, 2.84e-01 / 1.73e-01 | 20260508 00:14:18.498 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.48e-01 / 3.88e-01, 5.52e-02 / 2.37e-02 | 20260508 00:14:18.498 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:14:18.498 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:14:18.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:14:18.498 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 24.12%, Upload -- 2.57e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 30.86%, Compute -- 3.29e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 2.03%, LockUpload -- 2.16e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 2.75%, LockDownload -- 2.93e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 15.90%, LockCompute -- 1.69e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:14:18.498 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260508 00:14:18.610 | INFO | Starting a round of Baz IO | 20260508 00:14:22.111 | INFO | Populating RTMetrics report | 20260508 00:14:22.112 | INFO | RTMetrics report populated | 20260508 00:14:22.112 | INFO | Opening tmp RTMetrics json file | 20260508 00:14:22.118 | INFO | Serializing RTMetrics to json | 20260508 00:14:22.143 | INFO | Serialization done, swapping tmp file now... | 20260508 00:14:22.151 | INFO | RTMetrics done | 20260508 00:14:22.208 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.598 seconds | 20260508 00:14:22.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.28% / 0.00% / 57.72% | 20260508 00:14:22.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04564% / 1.58946e-05% / 8.34465e-05% | 20260508 00:14:22.209 | INFO | Baz IO overall: | 20260508 00:14:22.209 | INFO | BytesWritten : 1690856713391 | 20260508 00:14:22.209 | INFO | NumEvents : 818477839054 | 20260508 00:14:23.634 | INFO | Analyzing chunk frames = [3719168,3719680) | 20260508 00:14:23.634 | INFO | MemInfo: { | 20260508 00:14:23.634 | INFO | "gpu" : " 17 GB", | 20260508 00:14:23.634 | INFO | "huge" : "", | 20260508 00:14:23.634 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:23.634 | INFO | "malloc" : " 14 GB", | 20260508 00:14:23.634 | INFO | "pinned" : " 29 GB" | 20260508 00:14:23.634 | INFO | } | 20260508 00:14:23.634 | INFO | Perf report for the last 5.13831 seconds: | 20260508 00:14:23.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:23.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:23.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:23.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:23.635 | INFO | ANALYSIS : 69.0%, 2.9, 104.4ms, 0 | 20260508 00:14:23.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:23.635 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260508 00:14:23.635 | INFO | PRE_HQ : 68.6%, 1.9, 68.0ms, 0 | 20260508 00:14:23.635 | INFO | BAZWRITER : 69.7%, 4.1, 147.9ms, 0 | 20260508 00:14:23.635 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 00:14:23.635 | INFO | Uploaded 36GiB to GPU in 2.59909s (13.851GiB/s) | 20260508 00:14:23.635 | INFO | Downloaded 38GiB to GPU in 2.71784s (13.9817GiB/s) | 20260508 00:14:28.779 | INFO | Analyzing chunk frames = [3719680,3720192) | 20260508 00:14:28.779 | INFO | MemInfo: { | 20260508 00:14:28.779 | INFO | "gpu" : " 17 GB", | 20260508 00:14:28.779 | INFO | "huge" : "", | 20260508 00:14:28.779 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:28.779 | INFO | "malloc" : " 14 GB", | 20260508 00:14:28.779 | INFO | "pinned" : " 29 GB" | 20260508 00:14:28.779 | INFO | } | 20260508 00:14:28.779 | INFO | Perf report for the last 5.14471 seconds: | 20260508 00:14:28.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:28.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.779 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 00:14:28.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.779 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:14:28.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:28.780 | INFO | Uploaded 36GiB to GPU in 2.553s (14.101GiB/s) | 20260508 00:14:28.780 | INFO | Downloaded 34GiB to GPU in 2.47851s (13.7179GiB/s) | 20260508 00:14:33.917 | INFO | Analyzing chunk frames = [3720192,3720704) | 20260508 00:14:33.917 | INFO | MemInfo: { | 20260508 00:14:33.917 | INFO | "gpu" : " 17 GB", | 20260508 00:14:33.917 | INFO | "huge" : "", | 20260508 00:14:33.917 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:33.917 | INFO | "malloc" : " 14 GB", | 20260508 00:14:33.917 | INFO | "pinned" : " 29 GB" | 20260508 00:14:33.917 | INFO | } | 20260508 00:14:33.917 | INFO | Perf report for the last 5.13764 seconds: | 20260508 00:14:33.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:33.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 00:14:33.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:14:33.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:33.917 | INFO | Uploaded 36GiB to GPU in 2.51515s (14.3132GiB/s) | 20260508 00:14:33.917 | INFO | Downloaded 34GiB to GPU in 2.48262s (13.6952GiB/s) | 20260508 00:14:39.056 | INFO | Analyzing chunk frames = [3720704,3721216) | 20260508 00:14:39.056 | INFO | MemInfo: { | 20260508 00:14:39.056 | INFO | "gpu" : " 17 GB", | 20260508 00:14:39.056 | INFO | "huge" : "", | 20260508 00:14:39.056 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:39.056 | INFO | "malloc" : " 14 GB", | 20260508 00:14:39.056 | INFO | "pinned" : " 29 GB" | 20260508 00:14:39.057 | INFO | } | 20260508 00:14:39.057 | INFO | Perf report for the last 5.1398 seconds: | 20260508 00:14:39.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:39.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 00:14:39.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:14:39.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:39.057 | INFO | Uploaded 36GiB to GPU in 2.51846s (14.2944GiB/s) | 20260508 00:14:39.057 | INFO | Downloaded 34GiB to GPU in 2.49363s (13.6347GiB/s) | 20260508 00:14:44.199 | INFO | Analyzing chunk frames = [3721216,3721728) | 20260508 00:14:44.199 | INFO | MemInfo: { | 20260508 00:14:44.199 | INFO | "gpu" : " 17 GB", | 20260508 00:14:44.199 | INFO | "huge" : "", | 20260508 00:14:44.199 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:44.199 | INFO | "malloc" : " 14 GB", | 20260508 00:14:44.199 | INFO | "pinned" : " 29 GB" | 20260508 00:14:44.199 | INFO | } | 20260508 00:14:44.199 | INFO | Perf report for the last 5.14255 seconds: | 20260508 00:14:44.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:44.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.199 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 00:14:44.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.199 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 00:14:44.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:44.200 | INFO | Uploaded 36GiB to GPU in 2.30651s (15.608GiB/s) | 20260508 00:14:44.200 | INFO | Downloaded 34GiB to GPU in 2.48069s (13.7059GiB/s) | 20260508 00:14:49.334 | INFO | Analyzing chunk frames = [3721728,3722240) | 20260508 00:14:49.334 | INFO | MemInfo: { | 20260508 00:14:49.334 | INFO | "gpu" : " 17 GB", | 20260508 00:14:49.334 | INFO | "huge" : "", | 20260508 00:14:49.334 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:49.334 | INFO | "malloc" : " 14 GB", | 20260508 00:14:49.334 | INFO | "pinned" : " 29 GB" | 20260508 00:14:49.334 | INFO | } | 20260508 00:14:49.334 | INFO | Perf report for the last 5.13521 seconds: | 20260508 00:14:49.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:49.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.335 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:14:49.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.335 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260508 00:14:49.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:49.335 | INFO | Uploaded 36GiB to GPU in 2.37236s (15.1748GiB/s) | 20260508 00:14:49.335 | INFO | Downloaded 34GiB to GPU in 2.42867s (13.9994GiB/s) | 20260508 00:14:54.472 | INFO | Analyzing chunk frames = [3722240,3722752) | 20260508 00:14:54.472 | INFO | MemInfo: { | 20260508 00:14:54.472 | INFO | "gpu" : " 17 GB", | 20260508 00:14:54.472 | INFO | "huge" : "", | 20260508 00:14:54.472 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:54.472 | INFO | "malloc" : " 14 GB", | 20260508 00:14:54.472 | INFO | "pinned" : " 29 GB" | 20260508 00:14:54.472 | INFO | } | 20260508 00:14:54.472 | INFO | Perf report for the last 5.13794 seconds: | 20260508 00:14:54.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:54.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 00:14:54.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:14:54.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:54.473 | INFO | Uploaded 36GiB to GPU in 2.50135s (14.3922GiB/s) | 20260508 00:14:54.473 | INFO | Downloaded 34GiB to GPU in 2.47229s (13.7525GiB/s) | 20260508 00:14:59.609 | INFO | Analyzing chunk frames = [3722752,3723264) | 20260508 00:14:59.609 | INFO | MemInfo: { | 20260508 00:14:59.609 | INFO | "gpu" : " 17 GB", | 20260508 00:14:59.609 | INFO | "huge" : "", | 20260508 00:14:59.609 | INFO | "huge_shared" : " 29 GB", | 20260508 00:14:59.609 | INFO | "malloc" : " 14 GB", | 20260508 00:14:59.609 | INFO | "pinned" : " 29 GB" | 20260508 00:14:59.609 | INFO | } | 20260508 00:14:59.609 | INFO | Perf report for the last 5.13703 seconds: | 20260508 00:14:59.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:14:59.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.609 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 00:14:59.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.609 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:14:59.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:14:59.610 | INFO | Uploaded 36GiB to GPU in 2.49139s (14.4498GiB/s) | 20260508 00:14:59.610 | INFO | Downloaded 34GiB to GPU in 2.45586s (13.8445GiB/s) | 20260508 00:14:59.708 | INFO | Starting a round of Baz IO | 20260508 00:15:03.171 | INFO | Populating RTMetrics report | 20260508 00:15:03.171 | INFO | RTMetrics report populated | 20260508 00:15:03.171 | INFO | Opening tmp RTMetrics json file | 20260508 00:15:03.186 | INFO | Serializing RTMetrics to json | 20260508 00:15:03.238 | INFO | Serialization done, swapping tmp file now... | 20260508 00:15:03.260 | INFO | RTMetrics done | 20260508 00:15:03.280 | INFO | Finished a round of Baz IO: Wrote 1.694 GiB in 3.572 seconds | 20260508 00:15:03.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.27% / 0.00% / 57.72% | 20260508 00:15:03.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04239% / 1.19209e-05% / 7.94729e-05% | 20260508 00:15:03.281 | INFO | Baz IO overall: | 20260508 00:15:03.281 | INFO | BytesWritten : 1692676020756 | 20260508 00:15:03.281 | INFO | NumEvents : 819353258933 | 20260508 00:15:04.752 | INFO | Analyzing chunk frames = [3723264,3723776) | 20260508 00:15:04.752 | INFO | MemInfo: { | 20260508 00:15:04.752 | INFO | "gpu" : " 17 GB", | 20260508 00:15:04.752 | INFO | "huge" : "", | 20260508 00:15:04.752 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:04.752 | INFO | "malloc" : " 14 GB", | 20260508 00:15:04.752 | INFO | "pinned" : " 29 GB" | 20260508 00:15:04.752 | INFO | } | 20260508 00:15:04.752 | INFO | Perf report for the last 5.14262 seconds: | 20260508 00:15:04.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:04.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:04.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:04.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:04.752 | INFO | ANALYSIS : 68.5%, 3.0, 106.1ms, 0 | 20260508 00:15:04.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:04.752 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260508 00:15:04.752 | INFO | PRE_HQ : 66.2%, 1.8, 62.2ms, 0 | 20260508 00:15:04.752 | INFO | BAZWRITER : 68.9%, 4.1, 144.5ms, 0 | 20260508 00:15:04.752 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 00:15:04.753 | INFO | Uploaded 36GiB to GPU in 2.46237s (14.62GiB/s) | 20260508 00:15:04.753 | INFO | Downloaded 38GiB to GPU in 2.70628s (14.0414GiB/s) | 20260508 00:15:09.898 | INFO | Analyzing chunk frames = [3723776,3724288) | 20260508 00:15:09.898 | INFO | MemInfo: { | 20260508 00:15:09.898 | INFO | "gpu" : " 17 GB", | 20260508 00:15:09.898 | INFO | "huge" : "", | 20260508 00:15:09.898 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:09.898 | INFO | "malloc" : " 14 GB", | 20260508 00:15:09.898 | INFO | "pinned" : " 29 GB" | 20260508 00:15:09.898 | INFO | } | 20260508 00:15:09.898 | INFO | Perf report for the last 5.14629 seconds: | 20260508 00:15:09.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:09.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.898 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:15:09.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.899 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 00:15:09.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:09.899 | INFO | Uploaded 36GiB to GPU in 2.56532s (14.0334GiB/s) | 20260508 00:15:09.899 | INFO | Downloaded 34GiB to GPU in 2.49441s (13.6305GiB/s) | 20260508 00:15:15.034 | INFO | Analyzing chunk frames = [3724288,3724800) | 20260508 00:15:15.034 | INFO | MemInfo: { | 20260508 00:15:15.034 | INFO | "gpu" : " 17 GB", | 20260508 00:15:15.034 | INFO | "huge" : "", | 20260508 00:15:15.034 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:15.034 | INFO | "malloc" : " 14 GB", | 20260508 00:15:15.034 | INFO | "pinned" : " 29 GB" | 20260508 00:15:15.035 | INFO | } | 20260508 00:15:15.035 | INFO | Perf report for the last 5.13624 seconds: | 20260508 00:15:15.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:15.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.035 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:15:15.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.035 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 00:15:15.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:15.036 | INFO | Uploaded 36GiB to GPU in 2.5231s (14.2682GiB/s) | 20260508 00:15:15.036 | INFO | Downloaded 34GiB to GPU in 2.48464s (13.6841GiB/s) | 20260508 00:15:20.168 | INFO | Analyzing chunk frames = [3724800,3725312) | 20260508 00:15:20.168 | INFO | MemInfo: { | 20260508 00:15:20.168 | INFO | "gpu" : " 17 GB", | 20260508 00:15:20.168 | INFO | "huge" : "", | 20260508 00:15:20.168 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:20.169 | INFO | "malloc" : " 14 GB", | 20260508 00:15:20.169 | INFO | "pinned" : " 29 GB" | 20260508 00:15:20.169 | INFO | } | 20260508 00:15:20.169 | INFO | Perf report for the last 5.13419 seconds: | 20260508 00:15:20.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:20.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.169 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:15:20.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.169 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260508 00:15:20.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:20.170 | INFO | Uploaded 36GiB to GPU in 2.53202s (14.2179GiB/s) | 20260508 00:15:20.170 | INFO | Downloaded 34GiB to GPU in 2.49198s (13.6438GiB/s) | 20260508 00:15:25.310 | INFO | Analyzing chunk frames = [3725312,3725824) | 20260508 00:15:25.310 | INFO | MemInfo: { | 20260508 00:15:25.310 | INFO | "gpu" : " 17 GB", | 20260508 00:15:25.310 | INFO | "huge" : "", | 20260508 00:15:25.310 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:25.311 | INFO | "malloc" : " 14 GB", | 20260508 00:15:25.311 | INFO | "pinned" : " 29 GB" | 20260508 00:15:25.311 | INFO | } | 20260508 00:15:25.311 | INFO | Perf report for the last 5.14197 seconds: | 20260508 00:15:25.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:25.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:15:25.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 00:15:25.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:25.311 | INFO | Uploaded 36GiB to GPU in 2.35703s (15.2735GiB/s) | 20260508 00:15:25.311 | INFO | Downloaded 34GiB to GPU in 2.5122s (13.534GiB/s) | 20260508 00:15:30.450 | INFO | Analyzing chunk frames = [3725824,3726336) | 20260508 00:15:30.450 | INFO | MemInfo: { | 20260508 00:15:30.450 | INFO | "gpu" : " 17 GB", | 20260508 00:15:30.450 | INFO | "huge" : "", | 20260508 00:15:30.450 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:30.450 | INFO | "malloc" : " 14 GB", | 20260508 00:15:30.450 | INFO | "pinned" : " 29 GB" | 20260508 00:15:30.450 | INFO | } | 20260508 00:15:30.450 | INFO | Perf report for the last 5.13946 seconds: | 20260508 00:15:30.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:30.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 00:15:30.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 00:15:30.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:30.450 | INFO | Uploaded 36GiB to GPU in 2.43193s (14.8031GiB/s) | 20260508 00:15:30.450 | INFO | Downloaded 34GiB to GPU in 2.38696s (14.2441GiB/s) | 20260508 00:15:35.591 | INFO | Analyzing chunk frames = [3726336,3726848) | 20260508 00:15:35.591 | INFO | MemInfo: { | 20260508 00:15:35.591 | INFO | "gpu" : " 17 GB", | 20260508 00:15:35.591 | INFO | "huge" : "", | 20260508 00:15:35.591 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:35.591 | INFO | "malloc" : " 14 GB", | 20260508 00:15:35.591 | INFO | "pinned" : " 29 GB" | 20260508 00:15:35.591 | INFO | } | 20260508 00:15:35.591 | INFO | Perf report for the last 5.14046 seconds: | 20260508 00:15:35.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:35.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 00:15:35.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:15:35.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:35.591 | INFO | Uploaded 36GiB to GPU in 2.55035s (14.1157GiB/s) | 20260508 00:15:35.591 | INFO | Downloaded 34GiB to GPU in 2.45055s (13.8745GiB/s) | 20260508 00:15:40.723 | INFO | Analyzing chunk frames = [3726848,3727360) | 20260508 00:15:40.723 | INFO | MemInfo: { | 20260508 00:15:40.723 | INFO | "gpu" : " 17 GB", | 20260508 00:15:40.723 | INFO | "huge" : "", | 20260508 00:15:40.723 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:40.723 | INFO | "malloc" : " 14 GB", | 20260508 00:15:40.723 | INFO | "pinned" : " 29 GB" | 20260508 00:15:40.723 | INFO | } | 20260508 00:15:40.723 | INFO | Perf report for the last 5.13231 seconds: | 20260508 00:15:40.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:40.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:15:40.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:15:40.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:40.723 | INFO | Uploaded 36GiB to GPU in 2.53567s (14.1974GiB/s) | 20260508 00:15:40.723 | INFO | Downloaded 34GiB to GPU in 2.44204s (13.9228GiB/s) | 20260508 00:15:40.724 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:15:40.724 | INFO | Total mem usage: 16.7 GiB | 20260508 00:15:40.724 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:15:40.724 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 00:15:40.724 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:15:40.724 | INFO | High Water Breakdown: | 20260508 00:15:40.724 | INFO | 60.2% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:15:40.724 | INFO | 39.6% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:15:40.724 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:15:40.724 | INFO | ------------------------------------------ | 20260508 00:15:40.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:15:40.724 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:15:40.724 | INFO | Total mem usage: 26.5 GiB | 20260508 00:15:40.724 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:15:40.724 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:15:40.724 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:15:40.724 | INFO | High Water Breakdown: | 20260508 00:15:40.724 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:15:40.724 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:15:40.724 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:15:40.724 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:15:40.724 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:15:40.724 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:15:40.724 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:15:40.724 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:15:40.724 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:15:40.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:15:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:15:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:15:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:15:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:15:40.731 | INFO | ------------------------------------------ | 20260508 00:15:40.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:15:40.731 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:15:40.731 | INFO | Total mem usage: 27.8 GiB | 20260508 00:15:40.731 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:15:40.731 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:15:40.731 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:15:40.731 | INFO | High Water Breakdown: | 20260508 00:15:40.731 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:15:40.731 | INFO | ------------------------------------------ | 20260508 00:15:40.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:15:40.731 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:15:40.731 | INFO | Total mem usage: 15.7 GiB | 20260508 00:15:40.731 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:15:40.731 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:15:40.731 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 00:15:40.731 | INFO | High Water Breakdown: | 20260508 00:15:40.731 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:15:40.731 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:15:40.731 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:15:40.731 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:15:40.731 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:15:40.731 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:15:40.731 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:15:40.731 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:15:40.731 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:15:40.731 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:15:40.732 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:15:40.732 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:15:40.732 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:15:40.732 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:15:40.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:15:40.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:15:40.732 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:15:40.732 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:15:40.732 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:15:40.732 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:15:40.732 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:15:40.732 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:15:40.732 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:15:40.732 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:15:40.732 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:15:40.732 | INFO | ------------------------------------------ | 20260508 00:15:40.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:15:40.732 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:15:40.732 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:15:40.732 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:15:40.732 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.41e+01 / 1.75e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 0.30%, AggregateStats -- 9.98e-02 / 1.05e-01, 1.28e-02 / 5.82e-03 | 20260508 00:15:40.732 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.47e+00, 5.89e-01 / 2.18e-01 | 20260508 00:15:40.732 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.50e+00, 2.06e-01 / 1.04e-01 | 20260508 00:15:40.732 | INFO | PerfReport: 16.16%, DME -- 4.24e+01 / 4.63e+01, 4.33e+00 / 1.91e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 55.98%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.56e+00 / 1.45e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.47e+00 / 2.73e+00, 4.27e-01 / 2.37e-01 | 20260508 00:15:40.732 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.32e+00, 2.79e-01 / 1.48e-01 | 20260508 00:15:40.732 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.46e-01 / 3.82e-01, 5.46e-02 / 2.72e-02 | 20260508 00:15:40.732 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:15:40.732 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:15:40.732 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:15:40.732 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 24.31%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 30.76%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 2.15%, LockUpload -- 2.30e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 2.82%, LockDownload -- 3.01e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:15:40.732 | INFO | PerfReport: 0.01%, Misc -- 1.59e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260508 00:15:40.821 | INFO | Starting a round of Baz IO | 20260508 00:15:44.285 | INFO | Populating RTMetrics report | 20260508 00:15:44.285 | INFO | RTMetrics report populated | 20260508 00:15:44.285 | INFO | Opening tmp RTMetrics json file | 20260508 00:15:44.306 | INFO | Serializing RTMetrics to json | 20260508 00:15:44.308 | INFO | Serialization done, swapping tmp file now... | 20260508 00:15:44.308 | INFO | RTMetrics done | 20260508 00:15:44.385 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.564 seconds | 20260508 00:15:44.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.27% / 0.00% / 57.73% | 20260508 00:15:44.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04336% / 1.19209e-05% / 5.96046e-05% | 20260508 00:15:44.385 | INFO | Baz IO overall: | 20260508 00:15:44.385 | INFO | BytesWritten : 1694495593497 | 20260508 00:15:44.385 | INFO | NumEvents : 820228989877 | 20260508 00:15:45.864 | INFO | Analyzing chunk frames = [3727360,3727872) | 20260508 00:15:45.864 | INFO | MemInfo: { | 20260508 00:15:45.864 | INFO | "gpu" : " 17 GB", | 20260508 00:15:45.864 | INFO | "huge" : "", | 20260508 00:15:45.864 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:45.864 | INFO | "malloc" : " 14 GB", | 20260508 00:15:45.864 | INFO | "pinned" : " 29 GB" | 20260508 00:15:45.864 | INFO | } | 20260508 00:15:45.864 | INFO | Perf report for the last 5.1413 seconds: | 20260508 00:15:45.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:45.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:45.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:45.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:45.864 | INFO | ANALYSIS : 68.6%, 3.0, 106.6ms, 0 | 20260508 00:15:45.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:45.864 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260508 00:15:45.864 | INFO | PRE_HQ : 65.5%, 1.7, 57.3ms, 0 | 20260508 00:15:45.864 | INFO | BAZWRITER : 68.9%, 3.6, 129.3ms, 0 | 20260508 00:15:45.865 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 00:15:45.865 | INFO | Uploaded 36GiB to GPU in 2.57125s (14.001GiB/s) | 20260508 00:15:45.865 | INFO | Downloaded 38GiB to GPU in 2.68257s (14.1655GiB/s) | 20260508 00:15:51.009 | INFO | Analyzing chunk frames = [3727872,3728384) | 20260508 00:15:51.010 | INFO | MemInfo: { | 20260508 00:15:51.010 | INFO | "gpu" : " 17 GB", | 20260508 00:15:51.010 | INFO | "huge" : "", | 20260508 00:15:51.010 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:51.010 | INFO | "malloc" : " 14 GB", | 20260508 00:15:51.010 | INFO | "pinned" : " 29 GB" | 20260508 00:15:51.010 | INFO | } | 20260508 00:15:51.010 | INFO | Perf report for the last 5.14532 seconds: | 20260508 00:15:51.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:51.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 00:15:51.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:15:51.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:51.010 | INFO | Uploaded 36GiB to GPU in 2.59196s (13.8891GiB/s) | 20260508 00:15:51.010 | INFO | Downloaded 34GiB to GPU in 2.48063s (13.7062GiB/s) | 20260508 00:15:56.150 | INFO | Analyzing chunk frames = [3728384,3728896) | 20260508 00:15:56.150 | INFO | MemInfo: { | 20260508 00:15:56.150 | INFO | "gpu" : " 17 GB", | 20260508 00:15:56.150 | INFO | "huge" : "", | 20260508 00:15:56.150 | INFO | "huge_shared" : " 29 GB", | 20260508 00:15:56.150 | INFO | "malloc" : " 14 GB", | 20260508 00:15:56.150 | INFO | "pinned" : " 29 GB" | 20260508 00:15:56.150 | INFO | } | 20260508 00:15:56.150 | INFO | Perf report for the last 5.14036 seconds: | 20260508 00:15:56.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:15:56.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:15:56.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:15:56.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:15:56.150 | INFO | Uploaded 36GiB to GPU in 2.53805s (14.1841GiB/s) | 20260508 00:15:56.150 | INFO | Downloaded 34GiB to GPU in 2.47105s (13.7593GiB/s) | 20260508 00:16:01.286 | INFO | Analyzing chunk frames = [3728896,3729408) | 20260508 00:16:01.287 | INFO | MemInfo: { | 20260508 00:16:01.287 | INFO | "gpu" : " 17 GB", | 20260508 00:16:01.287 | INFO | "huge" : "", | 20260508 00:16:01.287 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:01.287 | INFO | "malloc" : " 14 GB", | 20260508 00:16:01.287 | INFO | "pinned" : " 29 GB" | 20260508 00:16:01.287 | INFO | } | 20260508 00:16:01.287 | INFO | Perf report for the last 5.13648 seconds: | 20260508 00:16:01.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:01.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:16:01.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:16:01.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:01.287 | INFO | Uploaded 36GiB to GPU in 2.53673s (14.1915GiB/s) | 20260508 00:16:01.287 | INFO | Downloaded 34GiB to GPU in 2.48404s (13.6874GiB/s) | 20260508 00:16:06.422 | INFO | Analyzing chunk frames = [3729408,3729920) | 20260508 00:16:06.422 | INFO | MemInfo: { | 20260508 00:16:06.422 | INFO | "gpu" : " 17 GB", | 20260508 00:16:06.422 | INFO | "huge" : "", | 20260508 00:16:06.422 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:06.422 | INFO | "malloc" : " 14 GB", | 20260508 00:16:06.422 | INFO | "pinned" : " 29 GB" | 20260508 00:16:06.422 | INFO | } | 20260508 00:16:06.422 | INFO | Perf report for the last 5.13551 seconds: | 20260508 00:16:06.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:06.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 00:16:06.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 00:16:06.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:06.422 | INFO | Uploaded 36GiB to GPU in 2.36072s (15.2496GiB/s) | 20260508 00:16:06.422 | INFO | Downloaded 34GiB to GPU in 2.47067s (13.7615GiB/s) | 20260508 00:16:11.562 | INFO | Analyzing chunk frames = [3729920,3730432) | 20260508 00:16:11.562 | INFO | MemInfo: { | 20260508 00:16:11.562 | INFO | "gpu" : " 17 GB", | 20260508 00:16:11.562 | INFO | "huge" : "", | 20260508 00:16:11.562 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:11.562 | INFO | "malloc" : " 14 GB", | 20260508 00:16:11.562 | INFO | "pinned" : " 29 GB" | 20260508 00:16:11.562 | INFO | } | 20260508 00:16:11.562 | INFO | Perf report for the last 5.14034 seconds: | 20260508 00:16:11.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:11.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.563 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 00:16:11.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.563 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 00:16:11.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:11.563 | INFO | Uploaded 36GiB to GPU in 2.40776s (14.9516GiB/s) | 20260508 00:16:11.563 | INFO | Downloaded 34GiB to GPU in 2.39852s (14.1754GiB/s) | 20260508 00:16:16.703 | INFO | Analyzing chunk frames = [3730432,3730944) | 20260508 00:16:16.703 | INFO | MemInfo: { | 20260508 00:16:16.703 | INFO | "gpu" : " 17 GB", | 20260508 00:16:16.703 | INFO | "huge" : "", | 20260508 00:16:16.703 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:16.703 | INFO | "malloc" : " 14 GB", | 20260508 00:16:16.703 | INFO | "pinned" : " 29 GB" | 20260508 00:16:16.703 | INFO | } | 20260508 00:16:16.704 | INFO | Perf report for the last 5.14104 seconds: | 20260508 00:16:16.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:16.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:16:16.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 00:16:16.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:16.704 | INFO | Uploaded 36GiB to GPU in 2.52072s (14.2816GiB/s) | 20260508 00:16:16.705 | INFO | Downloaded 34GiB to GPU in 2.45537s (13.8472GiB/s) | 20260508 00:16:21.847 | INFO | Analyzing chunk frames = [3730944,3731456) | 20260508 00:16:21.847 | INFO | MemInfo: { | 20260508 00:16:21.847 | INFO | "gpu" : " 17 GB", | 20260508 00:16:21.847 | INFO | "huge" : "", | 20260508 00:16:21.848 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:21.848 | INFO | "malloc" : " 14 GB", | 20260508 00:16:21.848 | INFO | "pinned" : " 29 GB" | 20260508 00:16:21.848 | INFO | } | 20260508 00:16:21.848 | INFO | Perf report for the last 5.14416 seconds: | 20260508 00:16:21.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:21.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:16:21.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | PRE_HQ : 36.3%, 1.0, 18.9ms, 0 | 20260508 00:16:21.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:21.848 | INFO | Uploaded 36GiB to GPU in 2.51262s (14.3277GiB/s) | 20260508 00:16:21.848 | INFO | Downloaded 34GiB to GPU in 2.44704s (13.8944GiB/s) | 20260508 00:16:21.960 | INFO | Starting a round of Baz IO | 20260508 00:16:25.432 | INFO | Populating RTMetrics report | 20260508 00:16:25.432 | INFO | RTMetrics report populated | 20260508 00:16:25.432 | INFO | Opening tmp RTMetrics json file | 20260508 00:16:25.434 | INFO | Serializing RTMetrics to json | 20260508 00:16:25.436 | INFO | Serialization done, swapping tmp file now... | 20260508 00:16:25.437 | INFO | RTMetrics done | 20260508 00:16:25.520 | INFO | Finished a round of Baz IO: Wrote 1.694 GiB in 3.559 seconds | 20260508 00:16:25.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.26% / 0.00% / 57.73% | 20260508 00:16:25.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03972% / 3.97364e-06% / 8.34465e-05% | 20260508 00:16:25.520 | INFO | Baz IO overall: | 20260508 00:16:25.520 | INFO | BytesWritten : 1696314162852 | 20260508 00:16:25.520 | INFO | NumEvents : 821103842107 | 20260508 00:16:26.982 | INFO | Analyzing chunk frames = [3731456,3731968) | 20260508 00:16:26.982 | INFO | MemInfo: { | 20260508 00:16:26.982 | INFO | "gpu" : " 17 GB", | 20260508 00:16:26.982 | INFO | "huge" : "", | 20260508 00:16:26.982 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:26.982 | INFO | "malloc" : " 14 GB", | 20260508 00:16:26.982 | INFO | "pinned" : " 29 GB" | 20260508 00:16:26.982 | INFO | } | 20260508 00:16:26.982 | INFO | Perf report for the last 5.13468 seconds: | 20260508 00:16:26.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:26.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:26.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:26.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:26.982 | INFO | ANALYSIS : 69.3%, 2.9, 104.5ms, 0 | 20260508 00:16:26.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:26.982 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260508 00:16:26.982 | INFO | PRE_HQ : 68.5%, 1.8, 63.6ms, 0 | 20260508 00:16:26.982 | INFO | BAZWRITER : 68.9%, 3.9, 138.3ms, 0 | 20260508 00:16:26.983 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 00:16:26.983 | INFO | Uploaded 36GiB to GPU in 2.54364s (14.1529GiB/s) | 20260508 00:16:26.983 | INFO | Downloaded 38GiB to GPU in 2.67004s (14.232GiB/s) | 20260508 00:16:32.119 | INFO | Analyzing chunk frames = [3731968,3732480) | 20260508 00:16:32.119 | INFO | MemInfo: { | 20260508 00:16:32.119 | INFO | "gpu" : " 17 GB", | 20260508 00:16:32.119 | INFO | "huge" : "", | 20260508 00:16:32.119 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:32.119 | INFO | "malloc" : " 14 GB", | 20260508 00:16:32.119 | INFO | "pinned" : " 29 GB" | 20260508 00:16:32.119 | INFO | } | 20260508 00:16:32.119 | INFO | Perf report for the last 5.13686 seconds: | 20260508 00:16:32.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:32.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.119 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 00:16:32.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.119 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:16:32.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:32.120 | INFO | Uploaded 36GiB to GPU in 2.56575s (14.031GiB/s) | 20260508 00:16:32.120 | INFO | Downloaded 34GiB to GPU in 2.49494s (13.6276GiB/s) | 20260508 00:16:37.259 | INFO | Analyzing chunk frames = [3732480,3732992) | 20260508 00:16:37.259 | INFO | MemInfo: { | 20260508 00:16:37.259 | INFO | "gpu" : " 17 GB", | 20260508 00:16:37.259 | INFO | "huge" : "", | 20260508 00:16:37.259 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:37.259 | INFO | "malloc" : " 14 GB", | 20260508 00:16:37.259 | INFO | "pinned" : " 29 GB" | 20260508 00:16:37.259 | INFO | } | 20260508 00:16:37.259 | INFO | Perf report for the last 5.14004 seconds: | 20260508 00:16:37.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:37.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.259 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:16:37.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.259 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:16:37.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:37.260 | INFO | Uploaded 36GiB to GPU in 2.57257s (13.9938GiB/s) | 20260508 00:16:37.260 | INFO | Downloaded 34GiB to GPU in 2.47553s (13.7344GiB/s) | 20260508 00:16:42.396 | INFO | Analyzing chunk frames = [3732992,3733504) | 20260508 00:16:42.396 | INFO | MemInfo: { | 20260508 00:16:42.396 | INFO | "gpu" : " 17 GB", | 20260508 00:16:42.396 | INFO | "huge" : "", | 20260508 00:16:42.396 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:42.396 | INFO | "malloc" : " 14 GB", | 20260508 00:16:42.396 | INFO | "pinned" : " 29 GB" | 20260508 00:16:42.396 | INFO | } | 20260508 00:16:42.396 | INFO | Perf report for the last 5.13678 seconds: | 20260508 00:16:42.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:42.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.396 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 00:16:42.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.396 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:16:42.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:42.397 | INFO | Uploaded 36GiB to GPU in 2.47597s (14.5398GiB/s) | 20260508 00:16:42.397 | INFO | Downloaded 34GiB to GPU in 2.47717s (13.7254GiB/s) | 20260508 00:16:47.538 | INFO | Analyzing chunk frames = [3733504,3734016) | 20260508 00:16:47.538 | INFO | MemInfo: { | 20260508 00:16:47.538 | INFO | "gpu" : " 17 GB", | 20260508 00:16:47.538 | INFO | "huge" : "", | 20260508 00:16:47.538 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:47.538 | INFO | "malloc" : " 14 GB", | 20260508 00:16:47.538 | INFO | "pinned" : " 29 GB" | 20260508 00:16:47.538 | INFO | } | 20260508 00:16:47.538 | INFO | Perf report for the last 5.14229 seconds: | 20260508 00:16:47.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:47.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.538 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:16:47.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.539 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 00:16:47.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:47.539 | INFO | Uploaded 36GiB to GPU in 2.40888s (14.9447GiB/s) | 20260508 00:16:47.539 | INFO | Downloaded 34GiB to GPU in 2.49935s (13.6035GiB/s) | 20260508 00:16:52.675 | INFO | Analyzing chunk frames = [3734016,3734528) | 20260508 00:16:52.675 | INFO | MemInfo: { | 20260508 00:16:52.675 | INFO | "gpu" : " 17 GB", | 20260508 00:16:52.675 | INFO | "huge" : "", | 20260508 00:16:52.675 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:52.675 | INFO | "malloc" : " 14 GB", | 20260508 00:16:52.675 | INFO | "pinned" : " 29 GB" | 20260508 00:16:52.675 | INFO | } | 20260508 00:16:52.675 | INFO | Perf report for the last 5.13688 seconds: | 20260508 00:16:52.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:52.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.675 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260508 00:16:52.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.675 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:16:52.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:52.676 | INFO | Uploaded 36GiB to GPU in 2.5386s (14.1811GiB/s) | 20260508 00:16:52.676 | INFO | Downloaded 34GiB to GPU in 2.43137s (13.9839GiB/s) | 20260508 00:16:57.815 | INFO | Analyzing chunk frames = [3734528,3735040) | 20260508 00:16:57.815 | INFO | MemInfo: { | 20260508 00:16:57.815 | INFO | "gpu" : " 17 GB", | 20260508 00:16:57.815 | INFO | "huge" : "", | 20260508 00:16:57.815 | INFO | "huge_shared" : " 29 GB", | 20260508 00:16:57.815 | INFO | "malloc" : " 14 GB", | 20260508 00:16:57.815 | INFO | "pinned" : " 29 GB" | 20260508 00:16:57.815 | INFO | } | 20260508 00:16:57.815 | INFO | Perf report for the last 5.13974 seconds: | 20260508 00:16:57.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:16:57.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 00:16:57.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 00:16:57.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:16:57.815 | INFO | Uploaded 36GiB to GPU in 2.53062s (14.2258GiB/s) | 20260508 00:16:57.815 | INFO | Downloaded 34GiB to GPU in 2.44161s (13.9252GiB/s) | 20260508 00:17:02.957 | INFO | Analyzing chunk frames = [3735040,3735552) | 20260508 00:17:02.957 | INFO | MemInfo: { | 20260508 00:17:02.957 | INFO | "gpu" : " 17 GB", | 20260508 00:17:02.957 | INFO | "huge" : "", | 20260508 00:17:02.957 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:02.957 | INFO | "malloc" : " 14 GB", | 20260508 00:17:02.957 | INFO | "pinned" : " 29 GB" | 20260508 00:17:02.957 | INFO | } | 20260508 00:17:02.957 | INFO | Perf report for the last 5.14224 seconds: | 20260508 00:17:02.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:02.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.957 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:17:02.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.957 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:17:02.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:02.958 | INFO | Uploaded 36GiB to GPU in 2.51644s (14.3059GiB/s) | 20260508 00:17:02.958 | INFO | Downloaded 34GiB to GPU in 2.43642s (13.9549GiB/s) | 20260508 00:17:02.958 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:17:02.958 | INFO | Total mem usage: 16.7 GiB | 20260508 00:17:02.958 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:17:02.958 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 00:17:02.958 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:17:02.958 | INFO | High Water Breakdown: | 20260508 00:17:02.958 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:17:02.958 | INFO | 39.9% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:17:02.958 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:17:02.958 | INFO | ------------------------------------------ | 20260508 00:17:02.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:17:02.958 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:17:02.958 | INFO | Total mem usage: 26.5 GiB | 20260508 00:17:02.958 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:17:02.958 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 00:17:02.958 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:17:02.958 | INFO | High Water Breakdown: | 20260508 00:17:02.958 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:17:02.958 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:17:02.958 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:17:02.958 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:17:02.958 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:17:02.958 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:17:02.958 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:17:02.958 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:17:02.958 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:17:02.958 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:17:02.958 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:17:02.958 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:17:02.958 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:17:02.958 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:17:02.958 | INFO | ------------------------------------------ | 20260508 00:17:02.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:17:02.958 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:17:02.958 | INFO | Total mem usage: 27.8 GiB | 20260508 00:17:02.958 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:17:02.958 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:17:02.958 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:17:02.958 | INFO | High Water Breakdown: | 20260508 00:17:02.958 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:17:02.958 | INFO | ------------------------------------------ | 20260508 00:17:02.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:17:02.958 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:17:02.958 | INFO | Total mem usage: 15.7 GiB | 20260508 00:17:02.958 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:17:02.958 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:17:02.958 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:17:02.958 | INFO | High Water Breakdown: | 20260508 00:17:02.958 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:17:02.958 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:17:02.958 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:17:02.958 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:17:02.958 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:17:02.958 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:17:02.958 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:17:02.958 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:17:02.958 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:17:02.958 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:17:02.958 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:17:02.958 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:17:02.958 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:17:02.958 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:17:02.958 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:17:02.958 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:17:02.958 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:17:02.958 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:17:02.958 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:17:02.958 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:17:02.958 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:17:02.958 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:17:02.958 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:17:02.958 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:17:02.958 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:17:02.958 | INFO | ------------------------------------------ | 20260508 00:17:02.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:17:02.958 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:17:02.958 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:17:02.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:17:02.958 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.65e+01, 1.41e+01 / 1.50e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.09e-02 / 5.30e-03 | 20260508 00:17:02.958 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.48e+00, 5.76e-01 / 1.99e-01 | 20260508 00:17:02.958 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.09e-01 | 20260508 00:17:02.958 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.61e+01, 4.18e+00 / 1.61e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.65e+00 / 1.83e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.74e+00, 4.20e-01 / 1.90e-01 | 20260508 00:17:02.958 | INFO | PerfReport: 3.40%, Metrics -- 1.11e+00 / 1.30e+00, 2.77e-01 / 1.47e-01 | 20260508 00:17:02.958 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.43e-01 / 3.74e-01, 5.46e-02 / 2.74e-02 | 20260508 00:17:02.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:17:02.958 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:17:02.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:17:02.958 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 24.50%, Upload -- 2.62e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 30.71%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 2.29%, LockUpload -- 2.45e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 2.61%, LockDownload -- 2.79e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 15.60%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:17:02.958 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260508 00:17:03.054 | INFO | Starting a round of Baz IO | 20260508 00:17:06.520 | INFO | Populating RTMetrics report | 20260508 00:17:06.521 | INFO | RTMetrics report populated | 20260508 00:17:06.521 | INFO | Opening tmp RTMetrics json file | 20260508 00:17:06.548 | INFO | Serializing RTMetrics to json | 20260508 00:17:06.550 | INFO | Serialization done, swapping tmp file now... | 20260508 00:17:06.551 | INFO | RTMetrics done | 20260508 00:17:06.618 | INFO | Finished a round of Baz IO: Wrote 1.694 GiB in 3.563 seconds | 20260508 00:17:06.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.26% / 0.00% / 57.74% | 20260508 00:17:06.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03872% / 7.94729e-06% / 5.5631e-05% | 20260508 00:17:06.618 | INFO | Baz IO overall: | 20260508 00:17:06.618 | INFO | BytesWritten : 1698133248253 | 20260508 00:17:06.618 | INFO | NumEvents : 821979425662 | 20260508 00:17:08.100 | INFO | Analyzing chunk frames = [3735552,3736064) | 20260508 00:17:08.100 | INFO | MemInfo: { | 20260508 00:17:08.100 | INFO | "gpu" : " 17 GB", | 20260508 00:17:08.100 | INFO | "huge" : "", | 20260508 00:17:08.100 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:08.100 | INFO | "malloc" : " 14 GB", | 20260508 00:17:08.100 | INFO | "pinned" : " 29 GB" | 20260508 00:17:08.100 | INFO | } | 20260508 00:17:08.100 | INFO | Perf report for the last 5.1426 seconds: | 20260508 00:17:08.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:08.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:08.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:08.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:08.100 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260508 00:17:08.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:08.100 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260508 00:17:08.100 | INFO | PRE_HQ : 65.9%, 1.7, 56.8ms, 0 | 20260508 00:17:08.100 | INFO | BAZWRITER : 68.9%, 4.0, 142.8ms, 0 | 20260508 00:17:08.100 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 00:17:08.101 | INFO | Uploaded 36GiB to GPU in 2.52205s (14.2741GiB/s) | 20260508 00:17:08.101 | INFO | Downloaded 38GiB to GPU in 2.6874s (14.1401GiB/s) | 20260508 00:17:13.237 | INFO | Analyzing chunk frames = [3736064,3736576) | 20260508 00:17:13.237 | INFO | MemInfo: { | 20260508 00:17:13.237 | INFO | "gpu" : " 17 GB", | 20260508 00:17:13.237 | INFO | "huge" : "", | 20260508 00:17:13.237 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:13.237 | INFO | "malloc" : " 14 GB", | 20260508 00:17:13.237 | INFO | "pinned" : " 29 GB" | 20260508 00:17:13.237 | INFO | } | 20260508 00:17:13.237 | INFO | Perf report for the last 5.13692 seconds: | 20260508 00:17:13.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:13.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:17:13.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:17:13.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:13.237 | INFO | Uploaded 36GiB to GPU in 2.5444s (14.1487GiB/s) | 20260508 00:17:13.237 | INFO | Downloaded 34GiB to GPU in 2.40141s (14.1583GiB/s) | 20260508 00:17:18.374 | INFO | Analyzing chunk frames = [3736576,3737088) | 20260508 00:17:18.374 | INFO | MemInfo: { | 20260508 00:17:18.374 | INFO | "gpu" : " 17 GB", | 20260508 00:17:18.374 | INFO | "huge" : "", | 20260508 00:17:18.374 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:18.374 | INFO | "malloc" : " 14 GB", | 20260508 00:17:18.374 | INFO | "pinned" : " 29 GB" | 20260508 00:17:18.374 | INFO | } | 20260508 00:17:18.374 | INFO | Perf report for the last 5.1372 seconds: | 20260508 00:17:18.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:18.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:17:18.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:17:18.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:18.374 | INFO | Uploaded 36GiB to GPU in 2.53293s (14.2128GiB/s) | 20260508 00:17:18.375 | INFO | Downloaded 34GiB to GPU in 2.4706s (13.7619GiB/s) | 20260508 00:17:23.516 | INFO | Analyzing chunk frames = [3737088,3737600) | 20260508 00:17:23.516 | INFO | MemInfo: { | 20260508 00:17:23.516 | INFO | "gpu" : " 17 GB", | 20260508 00:17:23.516 | INFO | "huge" : "", | 20260508 00:17:23.516 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:23.516 | INFO | "malloc" : " 14 GB", | 20260508 00:17:23.516 | INFO | "pinned" : " 29 GB" | 20260508 00:17:23.516 | INFO | } | 20260508 00:17:23.517 | INFO | Perf report for the last 5.14255 seconds: | 20260508 00:17:23.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:23.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 00:17:23.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:17:23.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:23.517 | INFO | Uploaded 36GiB to GPU in 2.53568s (14.1974GiB/s) | 20260508 00:17:23.517 | INFO | Downloaded 34GiB to GPU in 2.47998s (13.7098GiB/s) | 20260508 00:17:28.654 | INFO | Analyzing chunk frames = [3737600,3738112) | 20260508 00:17:28.654 | INFO | MemInfo: { | 20260508 00:17:28.654 | INFO | "gpu" : " 17 GB", | 20260508 00:17:28.654 | INFO | "huge" : "", | 20260508 00:17:28.654 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:28.654 | INFO | "malloc" : " 14 GB", | 20260508 00:17:28.654 | INFO | "pinned" : " 29 GB" | 20260508 00:17:28.654 | INFO | } | 20260508 00:17:28.654 | INFO | Perf report for the last 5.13718 seconds: | 20260508 00:17:28.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:28.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:17:28.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 00:17:28.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:28.654 | INFO | Uploaded 36GiB to GPU in 2.39246s (15.0473GiB/s) | 20260508 00:17:28.654 | INFO | Downloaded 34GiB to GPU in 2.49226s (13.6422GiB/s) | 20260508 00:17:33.792 | INFO | Analyzing chunk frames = [3738112,3738624) | 20260508 00:17:33.792 | INFO | MemInfo: { | 20260508 00:17:33.792 | INFO | "gpu" : " 17 GB", | 20260508 00:17:33.792 | INFO | "huge" : "", | 20260508 00:17:33.792 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:33.792 | INFO | "malloc" : " 14 GB", | 20260508 00:17:33.792 | INFO | "pinned" : " 29 GB" | 20260508 00:17:33.792 | INFO | } | 20260508 00:17:33.792 | INFO | Perf report for the last 5.1383 seconds: | 20260508 00:17:33.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:33.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.792 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 00:17:33.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.792 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260508 00:17:33.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:33.793 | INFO | Uploaded 36GiB to GPU in 2.41203s (14.9252GiB/s) | 20260508 00:17:33.793 | INFO | Downloaded 34GiB to GPU in 2.4086s (14.1161GiB/s) | 20260508 00:17:38.931 | INFO | Analyzing chunk frames = [3738624,3739136) | 20260508 00:17:38.931 | INFO | MemInfo: { | 20260508 00:17:38.931 | INFO | "gpu" : " 17 GB", | 20260508 00:17:38.931 | INFO | "huge" : "", | 20260508 00:17:38.931 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:38.931 | INFO | "malloc" : " 14 GB", | 20260508 00:17:38.931 | INFO | "pinned" : " 29 GB" | 20260508 00:17:38.931 | INFO | } | 20260508 00:17:38.931 | INFO | Perf report for the last 5.13893 seconds: | 20260508 00:17:38.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:38.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.931 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 00:17:38.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.932 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 00:17:38.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:38.932 | INFO | Uploaded 36GiB to GPU in 2.53775s (14.1858GiB/s) | 20260508 00:17:38.932 | INFO | Downloaded 34GiB to GPU in 2.45331s (13.8589GiB/s) | 20260508 00:17:44.070 | INFO | Analyzing chunk frames = [3739136,3739648) | 20260508 00:17:44.070 | INFO | MemInfo: { | 20260508 00:17:44.070 | INFO | "gpu" : " 17 GB", | 20260508 00:17:44.070 | INFO | "huge" : "", | 20260508 00:17:44.070 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:44.070 | INFO | "malloc" : " 14 GB", | 20260508 00:17:44.070 | INFO | "pinned" : " 29 GB" | 20260508 00:17:44.071 | INFO | } | 20260508 00:17:44.071 | INFO | Perf report for the last 5.13963 seconds: | 20260508 00:17:44.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:44.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.071 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:17:44.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.071 | INFO | PRE_HQ : 35.1%, 1.0, 18.3ms, 0 | 20260508 00:17:44.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:44.072 | INFO | Uploaded 36GiB to GPU in 2.5317s (14.2197GiB/s) | 20260508 00:17:44.072 | INFO | Downloaded 34GiB to GPU in 2.45639s (13.8414GiB/s) | 20260508 00:17:44.184 | INFO | Starting a round of Baz IO | 20260508 00:17:47.646 | INFO | Populating RTMetrics report | 20260508 00:17:47.646 | INFO | RTMetrics report populated | 20260508 00:17:47.646 | INFO | Opening tmp RTMetrics json file | 20260508 00:17:47.648 | INFO | Serializing RTMetrics to json | 20260508 00:17:47.667 | INFO | Serialization done, swapping tmp file now... | 20260508 00:17:47.676 | INFO | RTMetrics done | 20260508 00:17:47.749 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.564 seconds | 20260508 00:17:47.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.25% / 0.00% / 57.74% | 20260508 00:17:47.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03881% / 7.94729e-06% / 5.96046e-05% | 20260508 00:17:47.749 | INFO | Baz IO overall: | 20260508 00:17:47.749 | INFO | BytesWritten : 1699953781814 | 20260508 00:17:47.749 | INFO | NumEvents : 822856060763 | 20260508 00:17:49.213 | INFO | Analyzing chunk frames = [3739648,3740160) | 20260508 00:17:49.213 | INFO | MemInfo: { | 20260508 00:17:49.213 | INFO | "gpu" : " 17 GB", | 20260508 00:17:49.213 | INFO | "huge" : "", | 20260508 00:17:49.213 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:49.213 | INFO | "malloc" : " 14 GB", | 20260508 00:17:49.213 | INFO | "pinned" : " 29 GB" | 20260508 00:17:49.213 | INFO | } | 20260508 00:17:49.213 | INFO | Perf report for the last 5.14225 seconds: | 20260508 00:17:49.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:49.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:49.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:49.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:49.213 | INFO | ANALYSIS : 68.8%, 2.9, 104.5ms, 0 | 20260508 00:17:49.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:49.213 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260508 00:17:49.213 | INFO | PRE_HQ : 67.8%, 1.8, 63.9ms, 0 | 20260508 00:17:49.213 | INFO | BAZWRITER : 68.8%, 4.5, 157.7ms, 0 | 20260508 00:17:49.213 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:17:49.214 | INFO | Uploaded 36GiB to GPU in 2.62829s (13.6971GiB/s) | 20260508 00:17:49.214 | INFO | Downloaded 38GiB to GPU in 2.70719s (14.0367GiB/s) | 20260508 00:17:54.351 | INFO | Analyzing chunk frames = [3740160,3740672) | 20260508 00:17:54.351 | INFO | MemInfo: { | 20260508 00:17:54.351 | INFO | "gpu" : " 17 GB", | 20260508 00:17:54.351 | INFO | "huge" : "", | 20260508 00:17:54.351 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:54.351 | INFO | "malloc" : " 14 GB", | 20260508 00:17:54.351 | INFO | "pinned" : " 29 GB" | 20260508 00:17:54.351 | INFO | } | 20260508 00:17:54.351 | INFO | Perf report for the last 5.13783 seconds: | 20260508 00:17:54.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:54.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 00:17:54.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:17:54.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:54.351 | INFO | Uploaded 36GiB to GPU in 2.56814s (14.0179GiB/s) | 20260508 00:17:54.351 | INFO | Downloaded 34GiB to GPU in 2.49236s (13.6417GiB/s) | 20260508 00:17:59.484 | INFO | Analyzing chunk frames = [3740672,3741184) | 20260508 00:17:59.484 | INFO | MemInfo: { | 20260508 00:17:59.484 | INFO | "gpu" : " 17 GB", | 20260508 00:17:59.484 | INFO | "huge" : "", | 20260508 00:17:59.484 | INFO | "huge_shared" : " 29 GB", | 20260508 00:17:59.484 | INFO | "malloc" : " 14 GB", | 20260508 00:17:59.484 | INFO | "pinned" : " 29 GB" | 20260508 00:17:59.484 | INFO | } | 20260508 00:17:59.484 | INFO | Perf report for the last 5.13355 seconds: | 20260508 00:17:59.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:17:59.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.484 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 00:17:59.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.484 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:17:59.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:17:59.485 | INFO | Uploaded 36GiB to GPU in 2.49849s (14.4087GiB/s) | 20260508 00:17:59.485 | INFO | Downloaded 34GiB to GPU in 2.4804s (13.7075GiB/s) | 20260508 00:18:04.626 | INFO | Analyzing chunk frames = [3741184,3741696) | 20260508 00:18:04.626 | INFO | MemInfo: { | 20260508 00:18:04.626 | INFO | "gpu" : " 17 GB", | 20260508 00:18:04.626 | INFO | "huge" : "", | 20260508 00:18:04.626 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:04.626 | INFO | "malloc" : " 14 GB", | 20260508 00:18:04.627 | INFO | "pinned" : " 29 GB" | 20260508 00:18:04.627 | INFO | } | 20260508 00:18:04.627 | INFO | Perf report for the last 5.14224 seconds: | 20260508 00:18:04.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:04.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 00:18:04.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:18:04.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:04.627 | INFO | Uploaded 36GiB to GPU in 2.47498s (14.5456GiB/s) | 20260508 00:18:04.627 | INFO | Downloaded 34GiB to GPU in 2.40701s (14.1254GiB/s) | 20260508 00:18:09.767 | INFO | Analyzing chunk frames = [3741696,3742208) | 20260508 00:18:09.767 | INFO | MemInfo: { | 20260508 00:18:09.767 | INFO | "gpu" : " 17 GB", | 20260508 00:18:09.767 | INFO | "huge" : "", | 20260508 00:18:09.767 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:09.767 | INFO | "malloc" : " 14 GB", | 20260508 00:18:09.767 | INFO | "pinned" : " 29 GB" | 20260508 00:18:09.767 | INFO | } | 20260508 00:18:09.767 | INFO | Perf report for the last 5.1405 seconds: | 20260508 00:18:09.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:09.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 00:18:09.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 00:18:09.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:09.767 | INFO | Uploaded 36GiB to GPU in 2.36791s (15.2033GiB/s) | 20260508 00:18:09.768 | INFO | Downloaded 34GiB to GPU in 2.49106s (13.6488GiB/s) | 20260508 00:18:14.910 | INFO | Analyzing chunk frames = [3742208,3742720) | 20260508 00:18:14.910 | INFO | MemInfo: { | 20260508 00:18:14.910 | INFO | "gpu" : " 17 GB", | 20260508 00:18:14.910 | INFO | "huge" : "", | 20260508 00:18:14.910 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:14.910 | INFO | "malloc" : " 14 GB", | 20260508 00:18:14.910 | INFO | "pinned" : " 29 GB" | 20260508 00:18:14.910 | INFO | } | 20260508 00:18:14.910 | INFO | Perf report for the last 5.14272 seconds: | 20260508 00:18:14.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:14.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 00:18:14.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:18:14.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:14.910 | INFO | Uploaded 36GiB to GPU in 2.43414s (14.7896GiB/s) | 20260508 00:18:14.910 | INFO | Downloaded 34GiB to GPU in 2.38797s (14.238GiB/s) | 20260508 00:18:20.049 | INFO | Analyzing chunk frames = [3742720,3743232) | 20260508 00:18:20.049 | INFO | MemInfo: { | 20260508 00:18:20.049 | INFO | "gpu" : " 17 GB", | 20260508 00:18:20.049 | INFO | "huge" : "", | 20260508 00:18:20.049 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:20.049 | INFO | "malloc" : " 14 GB", | 20260508 00:18:20.049 | INFO | "pinned" : " 29 GB" | 20260508 00:18:20.049 | INFO | } | 20260508 00:18:20.049 | INFO | Perf report for the last 5.13952 seconds: | 20260508 00:18:20.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:20.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.050 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 00:18:20.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.050 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 00:18:20.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:20.050 | INFO | Uploaded 36GiB to GPU in 2.56055s (14.0595GiB/s) | 20260508 00:18:20.050 | INFO | Downloaded 34GiB to GPU in 2.4503s (13.8758GiB/s) | 20260508 00:18:25.185 | INFO | Analyzing chunk frames = [3743232,3743744) | 20260508 00:18:25.185 | INFO | MemInfo: { | 20260508 00:18:25.185 | INFO | "gpu" : " 17 GB", | 20260508 00:18:25.185 | INFO | "huge" : "", | 20260508 00:18:25.185 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:25.185 | INFO | "malloc" : " 14 GB", | 20260508 00:18:25.185 | INFO | "pinned" : " 29 GB" | 20260508 00:18:25.185 | INFO | } | 20260508 00:18:25.185 | INFO | Perf report for the last 5.13537 seconds: | 20260508 00:18:25.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:25.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:18:25.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:18:25.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:25.185 | INFO | Uploaded 36GiB to GPU in 2.5604s (14.0603GiB/s) | 20260508 00:18:25.185 | INFO | Downloaded 34GiB to GPU in 2.44668s (13.8964GiB/s) | 20260508 00:18:25.185 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:18:25.185 | INFO | Total mem usage: 16.7 GiB | 20260508 00:18:25.185 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:18:25.185 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.43%) | 20260508 00:18:25.185 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:18:25.185 | INFO | High Water Breakdown: | 20260508 00:18:25.185 | INFO | 60.4% : 9.22 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:18:25.185 | INFO | 39.4% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:18:25.185 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:18:25.185 | INFO | ------------------------------------------ | 20260508 00:18:25.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:18:25.185 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:18:25.185 | INFO | Total mem usage: 26.5 GiB | 20260508 00:18:25.185 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:18:25.185 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 00:18:25.185 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:18:25.185 | INFO | High Water Breakdown: | 20260508 00:18:25.185 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:18:25.185 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:18:25.185 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:18:25.185 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:18:25.185 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:18:25.185 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:18:25.185 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:18:25.185 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:18:25.185 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:18:25.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:18:25.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:18:25.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:18:25.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:18:25.185 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:18:25.185 | INFO | ------------------------------------------ | 20260508 00:18:25.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:18:25.185 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:18:25.185 | INFO | Total mem usage: 27.8 GiB | 20260508 00:18:25.185 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:18:25.185 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:18:25.185 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:18:25.185 | INFO | High Water Breakdown: | 20260508 00:18:25.185 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:18:25.185 | INFO | ------------------------------------------ | 20260508 00:18:25.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:18:25.185 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:18:25.185 | INFO | Total mem usage: 15.7 GiB | 20260508 00:18:25.185 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:18:25.185 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:18:25.185 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:18:25.185 | INFO | High Water Breakdown: | 20260508 00:18:25.185 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:18:25.185 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:18:25.185 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:18:25.185 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:18:25.185 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:18:25.185 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:18:25.185 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:18:25.185 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:18:25.185 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:18:25.185 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:18:25.185 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:18:25.185 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:18:25.185 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:18:25.185 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:18:25.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:18:25.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:18:25.185 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:18:25.185 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:18:25.185 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:18:25.185 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:18:25.185 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:18:25.185 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:18:25.185 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:18:25.185 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:18:25.185 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:18:25.185 | INFO | ------------------------------------------ | 20260508 00:18:25.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:18:25.185 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:18:25.186 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:18:25.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:18:25.186 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.41e+01 / 1.35e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.04e-01, 1.17e-02 / 6.10e-03 | 20260508 00:18:25.186 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.45e+00, 5.83e-01 / 2.50e-01 | 20260508 00:18:25.186 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.15e-01 | 20260508 00:18:25.186 | INFO | PerfReport: 16.00%, DME -- 4.20e+01 / 4.58e+01, 4.08e+00 / 1.50e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.69e+00 / 1.66e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.27e-01 / 2.65e-01 | 20260508 00:18:25.186 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.30e+00, 2.78e-01 / 1.25e-01 | 20260508 00:18:25.186 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.43e-01 / 3.76e-01, 5.51e-02 / 2.62e-02 | 20260508 00:18:25.186 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:18:25.186 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:18:25.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:18:25.186 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 2.26%, LockUpload -- 2.40e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 15.64%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:18:25.186 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260508 00:18:25.282 | INFO | Starting a round of Baz IO | 20260508 00:18:28.736 | INFO | Populating RTMetrics report | 20260508 00:18:28.736 | INFO | RTMetrics report populated | 20260508 00:18:28.736 | INFO | Opening tmp RTMetrics json file | 20260508 00:18:28.792 | INFO | Serializing RTMetrics to json | 20260508 00:18:28.820 | INFO | Serialization done, swapping tmp file now... | 20260508 00:18:28.824 | INFO | RTMetrics done | 20260508 00:18:28.846 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.564 seconds | 20260508 00:18:28.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.25% / 0.00% / 57.75% | 20260508 00:18:28.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03412% / 3.97364e-06% / 8.34465e-05% | 20260508 00:18:28.846 | INFO | Baz IO overall: | 20260508 00:18:28.846 | INFO | BytesWritten : 1701771540618 | 20260508 00:18:28.846 | INFO | NumEvents : 823730632132 | 20260508 00:18:30.327 | INFO | Analyzing chunk frames = [3743744,3744256) | 20260508 00:18:30.327 | INFO | MemInfo: { | 20260508 00:18:30.327 | INFO | "gpu" : " 17 GB", | 20260508 00:18:30.327 | INFO | "huge" : "", | 20260508 00:18:30.327 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:30.327 | INFO | "malloc" : " 14 GB", | 20260508 00:18:30.327 | INFO | "pinned" : " 29 GB" | 20260508 00:18:30.327 | INFO | } | 20260508 00:18:30.327 | INFO | Perf report for the last 5.1425 seconds: | 20260508 00:18:30.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:30.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:30.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:30.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:30.327 | INFO | ANALYSIS : 68.4%, 3.0, 106.3ms, 0 | 20260508 00:18:30.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:30.327 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260508 00:18:30.328 | INFO | PRE_HQ : 65.5%, 1.7, 56.6ms, 0 | 20260508 00:18:30.328 | INFO | BAZWRITER : 69.0%, 3.6, 128.0ms, 0 | 20260508 00:18:30.328 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:18:30.328 | INFO | Uploaded 36GiB to GPU in 2.54602s (14.1397GiB/s) | 20260508 00:18:30.328 | INFO | Downloaded 38GiB to GPU in 2.70022s (14.0729GiB/s) | 20260508 00:18:35.466 | INFO | Analyzing chunk frames = [3744256,3744768) | 20260508 00:18:35.466 | INFO | MemInfo: { | 20260508 00:18:35.466 | INFO | "gpu" : " 17 GB", | 20260508 00:18:35.466 | INFO | "huge" : "", | 20260508 00:18:35.466 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:35.466 | INFO | "malloc" : " 14 GB", | 20260508 00:18:35.466 | INFO | "pinned" : " 29 GB" | 20260508 00:18:35.466 | INFO | } | 20260508 00:18:35.466 | INFO | Perf report for the last 5.13914 seconds: | 20260508 00:18:35.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:35.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 00:18:35.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:18:35.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:35.467 | INFO | Uploaded 36GiB to GPU in 2.59472s (13.8743GiB/s) | 20260508 00:18:35.467 | INFO | Downloaded 34GiB to GPU in 2.48465s (13.684GiB/s) | 20260508 00:18:40.607 | INFO | Analyzing chunk frames = [3744768,3745280) | 20260508 00:18:40.607 | INFO | MemInfo: { | 20260508 00:18:40.607 | INFO | "gpu" : " 17 GB", | 20260508 00:18:40.607 | INFO | "huge" : "", | 20260508 00:18:40.607 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:40.607 | INFO | "malloc" : " 14 GB", | 20260508 00:18:40.607 | INFO | "pinned" : " 29 GB" | 20260508 00:18:40.607 | INFO | } | 20260508 00:18:40.607 | INFO | Perf report for the last 5.14081 seconds: | 20260508 00:18:40.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:40.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.607 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:18:40.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.607 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:18:40.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:40.608 | INFO | Uploaded 36GiB to GPU in 2.53232s (14.2162GiB/s) | 20260508 00:18:40.608 | INFO | Downloaded 34GiB to GPU in 2.47834s (13.7189GiB/s) | 20260508 00:18:45.742 | INFO | Analyzing chunk frames = [3745280,3745792) | 20260508 00:18:45.742 | INFO | MemInfo: { | 20260508 00:18:45.742 | INFO | "gpu" : " 17 GB", | 20260508 00:18:45.742 | INFO | "huge" : "", | 20260508 00:18:45.742 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:45.743 | INFO | "malloc" : " 14 GB", | 20260508 00:18:45.743 | INFO | "pinned" : " 29 GB" | 20260508 00:18:45.743 | INFO | } | 20260508 00:18:45.743 | INFO | Perf report for the last 5.13535 seconds: | 20260508 00:18:45.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:45.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:18:45.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:18:45.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:45.743 | INFO | Uploaded 36GiB to GPU in 2.53859s (14.1811GiB/s) | 20260508 00:18:45.743 | INFO | Downloaded 34GiB to GPU in 2.47658s (13.7286GiB/s) | 20260508 00:18:50.877 | INFO | Analyzing chunk frames = [3745792,3746304) | 20260508 00:18:50.877 | INFO | MemInfo: { | 20260508 00:18:50.877 | INFO | "gpu" : " 17 GB", | 20260508 00:18:50.877 | INFO | "huge" : "", | 20260508 00:18:50.877 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:50.877 | INFO | "malloc" : " 14 GB", | 20260508 00:18:50.877 | INFO | "pinned" : " 29 GB" | 20260508 00:18:50.877 | INFO | } | 20260508 00:18:50.878 | INFO | Perf report for the last 5.13489 seconds: | 20260508 00:18:50.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:50.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:18:50.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:18:50.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:50.878 | INFO | Uploaded 36GiB to GPU in 2.33714s (15.4035GiB/s) | 20260508 00:18:50.878 | INFO | Downloaded 34GiB to GPU in 2.4752s (13.7363GiB/s) | 20260508 00:18:56.019 | INFO | Analyzing chunk frames = [3746304,3746816) | 20260508 00:18:56.019 | INFO | MemInfo: { | 20260508 00:18:56.019 | INFO | "gpu" : " 17 GB", | 20260508 00:18:56.019 | INFO | "huge" : "", | 20260508 00:18:56.019 | INFO | "huge_shared" : " 29 GB", | 20260508 00:18:56.019 | INFO | "malloc" : " 14 GB", | 20260508 00:18:56.020 | INFO | "pinned" : " 29 GB" | 20260508 00:18:56.020 | INFO | } | 20260508 00:18:56.020 | INFO | Perf report for the last 5.14205 seconds: | 20260508 00:18:56.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:18:56.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260508 00:18:56.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260508 00:18:56.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:18:56.020 | INFO | Uploaded 36GiB to GPU in 2.5432s (14.1554GiB/s) | 20260508 00:18:56.020 | INFO | Downloaded 34GiB to GPU in 2.43115s (13.9852GiB/s) | 20260508 00:19:01.161 | INFO | Analyzing chunk frames = [3746816,3747328) | 20260508 00:19:01.161 | INFO | MemInfo: { | 20260508 00:19:01.161 | INFO | "gpu" : " 17 GB", | 20260508 00:19:01.161 | INFO | "huge" : "", | 20260508 00:19:01.161 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:01.161 | INFO | "malloc" : " 14 GB", | 20260508 00:19:01.161 | INFO | "pinned" : " 29 GB" | 20260508 00:19:01.161 | INFO | } | 20260508 00:19:01.161 | INFO | Perf report for the last 5.14184 seconds: | 20260508 00:19:01.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:01.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 00:19:01.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260508 00:19:01.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:01.162 | INFO | Uploaded 36GiB to GPU in 2.56144s (14.0546GiB/s) | 20260508 00:19:01.162 | INFO | Downloaded 34GiB to GPU in 2.45496s (13.8495GiB/s) | 20260508 00:19:06.303 | INFO | Analyzing chunk frames = [3747328,3747840) | 20260508 00:19:06.303 | INFO | MemInfo: { | 20260508 00:19:06.303 | INFO | "gpu" : " 17 GB", | 20260508 00:19:06.303 | INFO | "huge" : "", | 20260508 00:19:06.303 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:06.303 | INFO | "malloc" : " 14 GB", | 20260508 00:19:06.303 | INFO | "pinned" : " 29 GB" | 20260508 00:19:06.303 | INFO | } | 20260508 00:19:06.303 | INFO | Perf report for the last 5.14149 seconds: | 20260508 00:19:06.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:06.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.303 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:19:06.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.303 | INFO | PRE_HQ : 35.9%, 1.0, 18.7ms, 0 | 20260508 00:19:06.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:06.304 | INFO | Uploaded 36GiB to GPU in 2.5583s (14.0718GiB/s) | 20260508 00:19:06.304 | INFO | Downloaded 34GiB to GPU in 2.44855s (13.8858GiB/s) | 20260508 00:19:06.417 | INFO | Starting a round of Baz IO | 20260508 00:19:09.897 | INFO | Populating RTMetrics report | 20260508 00:19:09.898 | INFO | RTMetrics report populated | 20260508 00:19:09.898 | INFO | Opening tmp RTMetrics json file | 20260508 00:19:09.903 | INFO | Serializing RTMetrics to json | 20260508 00:19:09.907 | INFO | Serialization done, swapping tmp file now... | 20260508 00:19:09.907 | INFO | RTMetrics done | 20260508 00:19:10.038 | INFO | Finished a round of Baz IO: Wrote 1.691 GiB in 3.621 seconds | 20260508 00:19:10.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.24% / 0.00% / 57.75% | 20260508 00:19:10.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02956% / 0% / 7.15256e-05% | 20260508 00:19:10.038 | INFO | Baz IO overall: | 20260508 00:19:10.038 | INFO | BytesWritten : 1703587767532 | 20260508 00:19:10.038 | INFO | NumEvents : 824604191557 | 20260508 00:19:11.437 | INFO | Analyzing chunk frames = [3747840,3748352) | 20260508 00:19:11.437 | INFO | MemInfo: { | 20260508 00:19:11.437 | INFO | "gpu" : " 17 GB", | 20260508 00:19:11.437 | INFO | "huge" : "", | 20260508 00:19:11.437 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:11.437 | INFO | "malloc" : " 14 GB", | 20260508 00:19:11.437 | INFO | "pinned" : " 29 GB" | 20260508 00:19:11.437 | INFO | } | 20260508 00:19:11.437 | INFO | Perf report for the last 5.1341 seconds: | 20260508 00:19:11.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:11.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:11.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:11.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:11.437 | INFO | ANALYSIS : 69.1%, 2.9, 104.8ms, 0 | 20260508 00:19:11.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:11.437 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 00:19:11.437 | INFO | PRE_HQ : 68.1%, 1.8, 62.7ms, 0 | 20260508 00:19:11.437 | INFO | BAZWRITER : 70.1%, 4.3, 153.1ms, 0 | 20260508 00:19:11.437 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 00:19:11.437 | INFO | Uploaded 36GiB to GPU in 2.63343s (13.6704GiB/s) | 20260508 00:19:11.437 | INFO | Downloaded 38GiB to GPU in 2.71453s (13.9988GiB/s) | 20260508 00:19:16.582 | INFO | Analyzing chunk frames = [3748352,3748864) | 20260508 00:19:16.582 | INFO | MemInfo: { | 20260508 00:19:16.582 | INFO | "gpu" : " 17 GB", | 20260508 00:19:16.582 | INFO | "huge" : "", | 20260508 00:19:16.582 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:16.582 | INFO | "malloc" : " 14 GB", | 20260508 00:19:16.582 | INFO | "pinned" : " 29 GB" | 20260508 00:19:16.582 | INFO | } | 20260508 00:19:16.582 | INFO | Perf report for the last 5.14501 seconds: | 20260508 00:19:16.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:16.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.582 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 00:19:16.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.582 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:19:16.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:16.583 | INFO | Uploaded 36GiB to GPU in 2.56184s (14.0524GiB/s) | 20260508 00:19:16.583 | INFO | Downloaded 34GiB to GPU in 2.49095s (13.6494GiB/s) | 20260508 00:19:21.723 | INFO | Analyzing chunk frames = [3748864,3749376) | 20260508 00:19:21.723 | INFO | MemInfo: { | 20260508 00:19:21.723 | INFO | "gpu" : " 17 GB", | 20260508 00:19:21.723 | INFO | "huge" : "", | 20260508 00:19:21.723 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:21.723 | INFO | "malloc" : " 14 GB", | 20260508 00:19:21.723 | INFO | "pinned" : " 29 GB" | 20260508 00:19:21.723 | INFO | } | 20260508 00:19:21.723 | INFO | Perf report for the last 5.14108 seconds: | 20260508 00:19:21.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:21.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.723 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 00:19:21.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.723 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:19:21.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:21.724 | INFO | Uploaded 36GiB to GPU in 2.50593s (14.3659GiB/s) | 20260508 00:19:21.724 | INFO | Downloaded 34GiB to GPU in 2.48436s (13.6856GiB/s) | 20260508 00:19:26.860 | INFO | Analyzing chunk frames = [3749376,3749888) | 20260508 00:19:26.860 | INFO | MemInfo: { | 20260508 00:19:26.860 | INFO | "gpu" : " 17 GB", | 20260508 00:19:26.860 | INFO | "huge" : "", | 20260508 00:19:26.860 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:26.860 | INFO | "malloc" : " 14 GB", | 20260508 00:19:26.860 | INFO | "pinned" : " 29 GB" | 20260508 00:19:26.860 | INFO | } | 20260508 00:19:26.860 | INFO | Perf report for the last 5.13682 seconds: | 20260508 00:19:26.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:26.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 00:19:26.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 00:19:26.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:26.860 | INFO | Uploaded 36GiB to GPU in 2.51647s (14.3058GiB/s) | 20260508 00:19:26.860 | INFO | Downloaded 34GiB to GPU in 2.48099s (13.7042GiB/s) | 20260508 00:19:31.998 | INFO | Analyzing chunk frames = [3749888,3750400) | 20260508 00:19:31.998 | INFO | MemInfo: { | 20260508 00:19:31.998 | INFO | "gpu" : " 17 GB", | 20260508 00:19:31.998 | INFO | "huge" : "", | 20260508 00:19:31.999 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:31.999 | INFO | "malloc" : " 14 GB", | 20260508 00:19:31.999 | INFO | "pinned" : " 29 GB" | 20260508 00:19:31.999 | INFO | } | 20260508 00:19:31.999 | INFO | Perf report for the last 5.13866 seconds: | 20260508 00:19:31.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:31.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 00:19:31.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 00:19:31.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:31.999 | INFO | Uploaded 36GiB to GPU in 2.37142s (15.1808GiB/s) | 20260508 00:19:31.999 | INFO | Downloaded 34GiB to GPU in 2.57017s (13.2287GiB/s) | 20260508 00:19:37.137 | INFO | Analyzing chunk frames = [3750400,3750912) | 20260508 00:19:37.137 | INFO | MemInfo: { | 20260508 00:19:37.137 | INFO | "gpu" : " 17 GB", | 20260508 00:19:37.137 | INFO | "huge" : "", | 20260508 00:19:37.137 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:37.137 | INFO | "malloc" : " 14 GB", | 20260508 00:19:37.137 | INFO | "pinned" : " 29 GB" | 20260508 00:19:37.137 | INFO | } | 20260508 00:19:37.137 | INFO | Perf report for the last 5.13804 seconds: | 20260508 00:19:37.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:37.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 00:19:37.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:19:37.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:37.137 | INFO | Uploaded 36GiB to GPU in 2.34542s (15.3491GiB/s) | 20260508 00:19:37.137 | INFO | Downloaded 34GiB to GPU in 2.41814s (14.0604GiB/s) | 20260508 00:19:42.277 | INFO | Analyzing chunk frames = [3750912,3751424) | 20260508 00:19:42.277 | INFO | MemInfo: { | 20260508 00:19:42.277 | INFO | "gpu" : " 17 GB", | 20260508 00:19:42.277 | INFO | "huge" : "", | 20260508 00:19:42.277 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:42.277 | INFO | "malloc" : " 14 GB", | 20260508 00:19:42.278 | INFO | "pinned" : " 29 GB" | 20260508 00:19:42.278 | INFO | } | 20260508 00:19:42.278 | INFO | Perf report for the last 5.1408 seconds: | 20260508 00:19:42.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:42.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260508 00:19:42.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 00:19:42.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:42.278 | INFO | Uploaded 36GiB to GPU in 2.50126s (14.3928GiB/s) | 20260508 00:19:42.278 | INFO | Downloaded 34GiB to GPU in 2.46264s (13.8063GiB/s) | 20260508 00:19:47.417 | INFO | Analyzing chunk frames = [3751424,3751936) | 20260508 00:19:47.417 | INFO | MemInfo: { | 20260508 00:19:47.417 | INFO | "gpu" : " 17 GB", | 20260508 00:19:47.417 | INFO | "huge" : "", | 20260508 00:19:47.417 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:47.417 | INFO | "malloc" : " 14 GB", | 20260508 00:19:47.417 | INFO | "pinned" : " 29 GB" | 20260508 00:19:47.417 | INFO | } | 20260508 00:19:47.417 | INFO | Perf report for the last 5.13925 seconds: | 20260508 00:19:47.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:47.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:19:47.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | PRE_HQ : 36.2%, 1.0, 18.8ms, 0 | 20260508 00:19:47.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:47.417 | INFO | Uploaded 36GiB to GPU in 2.49337s (14.4383GiB/s) | 20260508 00:19:47.417 | INFO | Downloaded 34GiB to GPU in 2.46435s (13.7967GiB/s) | 20260508 00:19:47.417 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:19:47.417 | INFO | Total mem usage: 16.7 GiB | 20260508 00:19:47.417 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:19:47.417 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.4%) | 20260508 00:19:47.417 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:19:47.417 | INFO | High Water Breakdown: | 20260508 00:19:47.417 | INFO | 60% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:19:47.417 | INFO | 39.9% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:19:47.417 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:19:47.417 | INFO | ------------------------------------------ | 20260508 00:19:47.417 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:19:47.418 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:19:47.418 | INFO | Total mem usage: 26.5 GiB | 20260508 00:19:47.418 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:19:47.418 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:19:47.418 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:19:47.418 | INFO | High Water Breakdown: | 20260508 00:19:47.418 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:19:47.418 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:19:47.418 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:19:47.418 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:19:47.418 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:19:47.418 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:19:47.418 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:19:47.418 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:19:47.418 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:19:47.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:19:47.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:19:47.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:19:47.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:19:47.418 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:19:47.418 | INFO | ------------------------------------------ | 20260508 00:19:47.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:19:47.418 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:19:47.418 | INFO | Total mem usage: 27.8 GiB | 20260508 00:19:47.418 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:19:47.418 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:19:47.418 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:19:47.418 | INFO | High Water Breakdown: | 20260508 00:19:47.419 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:19:47.419 | INFO | ------------------------------------------ | 20260508 00:19:47.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:19:47.419 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:19:47.419 | INFO | Total mem usage: 15.7 GiB | 20260508 00:19:47.419 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:19:47.419 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:19:47.419 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:19:47.419 | INFO | High Water Breakdown: | 20260508 00:19:47.419 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:19:47.419 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:19:47.419 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:19:47.419 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:19:47.419 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:19:47.419 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:19:47.419 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:19:47.419 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:19:47.419 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:19:47.419 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:19:47.419 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:19:47.419 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:19:47.419 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:19:47.419 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:19:47.419 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:19:47.419 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:19:47.419 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:19:47.419 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:19:47.419 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:19:47.419 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:19:47.419 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:19:47.419 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:19:47.419 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:19:47.419 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:19:47.419 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:19:47.419 | INFO | ------------------------------------------ | 20260508 00:19:47.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:19:47.419 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:19:47.419 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:19:47.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:19:47.419 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.65e+01, 1.43e+01 / 1.81e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.02e-01, 1.20e-02 / 7.93e-03 | 20260508 00:19:47.419 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.49e+00, 5.82e-01 / 1.63e-01 | 20260508 00:19:47.419 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.16e-01 | 20260508 00:19:47.419 | INFO | PerfReport: 16.21%, DME -- 4.24e+01 / 4.67e+01, 4.25e+00 / 2.13e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 55.99%, FrameLabeling -- 1.83e+01 / 1.93e+01, 1.51e+00 / 1.49e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.46e+00 / 2.78e+00, 4.31e-01 / 2.03e-01 | 20260508 00:19:47.419 | INFO | PerfReport: 3.41%, Metrics -- 1.11e+00 / 1.32e+00, 2.74e-01 / 1.17e-01 | 20260508 00:19:47.419 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.76e-01, 5.46e-02 / 2.57e-02 | 20260508 00:19:47.419 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:19:47.419 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:19:47.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:19:47.419 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 24.38%, Download -- 2.61e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 30.60%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 2.33%, LockUpload -- 2.49e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 2.73%, LockDownload -- 2.92e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 15.50%, LockCompute -- 1.66e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 00:19:47.419 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 00:19:47.514 | INFO | Starting a round of Baz IO | 20260508 00:19:50.964 | INFO | Populating RTMetrics report | 20260508 00:19:50.964 | INFO | RTMetrics report populated | 20260508 00:19:50.964 | INFO | Opening tmp RTMetrics json file | 20260508 00:19:51.238 | INFO | Serializing RTMetrics to json | 20260508 00:19:51.310 | INFO | Serialization done, swapping tmp file now... | 20260508 00:19:51.313 | INFO | RTMetrics done | 20260508 00:19:51.319 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.804 seconds | 20260508 00:19:51.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.24% / 0.00% / 57.76% | 20260508 00:19:51.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02934% / 7.94729e-06% / 7.94729e-05% | 20260508 00:19:51.319 | INFO | Baz IO overall: | 20260508 00:19:51.319 | INFO | BytesWritten : 1705405541131 | 20260508 00:19:51.319 | INFO | NumEvents : 825478769072 | 20260508 00:19:52.557 | INFO | Analyzing chunk frames = [3751936,3752448) | 20260508 00:19:52.557 | INFO | MemInfo: { | 20260508 00:19:52.557 | INFO | "gpu" : " 17 GB", | 20260508 00:19:52.557 | INFO | "huge" : "", | 20260508 00:19:52.557 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:52.557 | INFO | "malloc" : " 14 GB", | 20260508 00:19:52.557 | INFO | "pinned" : " 29 GB" | 20260508 00:19:52.557 | INFO | } | 20260508 00:19:52.557 | INFO | Perf report for the last 5.14006 seconds: | 20260508 00:19:52.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:52.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:52.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:52.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:52.557 | INFO | ANALYSIS : 68.3%, 3.0, 106.6ms, 0 | 20260508 00:19:52.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:52.557 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260508 00:19:52.557 | INFO | PRE_HQ : 65.5%, 1.7, 56.6ms, 0 | 20260508 00:19:52.557 | INFO | BAZWRITER : 73.6%, 4.1, 155.1ms, 0 | 20260508 00:19:52.557 | INFO | BAZ_SAMPLING : 4.8%, 1.0, 2.5ms, 0 | 20260508 00:19:52.558 | INFO | Uploaded 36GiB to GPU in 2.51572s (14.31GiB/s) | 20260508 00:19:52.558 | INFO | Downloaded 38GiB to GPU in 2.69426s (14.104GiB/s) | 20260508 00:19:57.689 | INFO | Analyzing chunk frames = [3752448,3752960) | 20260508 00:19:57.690 | INFO | MemInfo: { | 20260508 00:19:57.690 | INFO | "gpu" : " 17 GB", | 20260508 00:19:57.690 | INFO | "huge" : "", | 20260508 00:19:57.690 | INFO | "huge_shared" : " 29 GB", | 20260508 00:19:57.690 | INFO | "malloc" : " 14 GB", | 20260508 00:19:57.690 | INFO | "pinned" : " 29 GB" | 20260508 00:19:57.690 | INFO | } | 20260508 00:19:57.690 | INFO | Perf report for the last 5.13264 seconds: | 20260508 00:19:57.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:19:57.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:19:57.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:19:57.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:19:57.690 | INFO | Uploaded 36GiB to GPU in 2.50896s (14.3486GiB/s) | 20260508 00:19:57.690 | INFO | Downloaded 34GiB to GPU in 2.43778s (13.9471GiB/s) | 20260508 00:20:02.831 | INFO | Analyzing chunk frames = [3752960,3753472) | 20260508 00:20:02.831 | INFO | MemInfo: { | 20260508 00:20:02.831 | INFO | "gpu" : " 17 GB", | 20260508 00:20:02.831 | INFO | "huge" : "", | 20260508 00:20:02.831 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:02.831 | INFO | "malloc" : " 14 GB", | 20260508 00:20:02.831 | INFO | "pinned" : " 29 GB" | 20260508 00:20:02.831 | INFO | } | 20260508 00:20:02.831 | INFO | Perf report for the last 5.14159 seconds: | 20260508 00:20:02.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:02.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.831 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 00:20:02.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.831 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:20:02.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:02.832 | INFO | Uploaded 36GiB to GPU in 2.51283s (14.3265GiB/s) | 20260508 00:20:02.832 | INFO | Downloaded 34GiB to GPU in 2.4759s (13.7324GiB/s) | 20260508 00:20:07.970 | INFO | Analyzing chunk frames = [3753472,3753984) | 20260508 00:20:07.970 | INFO | MemInfo: { | 20260508 00:20:07.970 | INFO | "gpu" : " 17 GB", | 20260508 00:20:07.970 | INFO | "huge" : "", | 20260508 00:20:07.970 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:07.970 | INFO | "malloc" : " 14 GB", | 20260508 00:20:07.970 | INFO | "pinned" : " 29 GB" | 20260508 00:20:07.970 | INFO | } | 20260508 00:20:07.970 | INFO | Perf report for the last 5.13897 seconds: | 20260508 00:20:07.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:07.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.970 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:20:07.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.970 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:20:07.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:07.971 | INFO | Uploaded 36GiB to GPU in 2.47084s (14.5699GiB/s) | 20260508 00:20:07.971 | INFO | Downloaded 34GiB to GPU in 2.51607s (13.5131GiB/s) | 20260508 00:20:13.113 | INFO | Analyzing chunk frames = [3753984,3754496) | 20260508 00:20:13.113 | INFO | MemInfo: { | 20260508 00:20:13.113 | INFO | "gpu" : " 17 GB", | 20260508 00:20:13.113 | INFO | "huge" : "", | 20260508 00:20:13.113 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:13.113 | INFO | "malloc" : " 14 GB", | 20260508 00:20:13.113 | INFO | "pinned" : " 29 GB" | 20260508 00:20:13.113 | INFO | } | 20260508 00:20:13.113 | INFO | Perf report for the last 5.14311 seconds: | 20260508 00:20:13.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:13.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.113 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 00:20:13.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.113 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:20:13.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:13.114 | INFO | Uploaded 36GiB to GPU in 2.33608s (15.4105GiB/s) | 20260508 00:20:13.114 | INFO | Downloaded 34GiB to GPU in 2.51296s (13.5299GiB/s) | 20260508 00:20:18.251 | INFO | Analyzing chunk frames = [3754496,3755008) | 20260508 00:20:18.251 | INFO | MemInfo: { | 20260508 00:20:18.251 | INFO | "gpu" : " 17 GB", | 20260508 00:20:18.251 | INFO | "huge" : "", | 20260508 00:20:18.251 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:18.251 | INFO | "malloc" : " 14 GB", | 20260508 00:20:18.252 | INFO | "pinned" : " 29 GB" | 20260508 00:20:18.252 | INFO | } | 20260508 00:20:18.252 | INFO | Perf report for the last 5.13826 seconds: | 20260508 00:20:18.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:18.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:20:18.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:20:18.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:18.252 | INFO | Uploaded 36GiB to GPU in 2.40765s (14.9523GiB/s) | 20260508 00:20:18.252 | INFO | Downloaded 34GiB to GPU in 2.40356s (14.1457GiB/s) | 20260508 00:20:23.392 | INFO | Analyzing chunk frames = [3755008,3755520) | 20260508 00:20:23.392 | INFO | MemInfo: { | 20260508 00:20:23.392 | INFO | "gpu" : " 17 GB", | 20260508 00:20:23.392 | INFO | "huge" : "", | 20260508 00:20:23.392 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:23.392 | INFO | "malloc" : " 14 GB", | 20260508 00:20:23.392 | INFO | "pinned" : " 29 GB" | 20260508 00:20:23.392 | INFO | } | 20260508 00:20:23.392 | INFO | Perf report for the last 5.14068 seconds: | 20260508 00:20:23.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:23.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.392 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:20:23.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.392 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 00:20:23.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:23.393 | INFO | Uploaded 36GiB to GPU in 2.54209s (14.1616GiB/s) | 20260508 00:20:23.393 | INFO | Downloaded 34GiB to GPU in 2.47029s (13.7636GiB/s) | 20260508 00:20:28.534 | INFO | Analyzing chunk frames = [3755520,3756032) | 20260508 00:20:28.534 | INFO | MemInfo: { | 20260508 00:20:28.534 | INFO | "gpu" : " 17 GB", | 20260508 00:20:28.534 | INFO | "huge" : "", | 20260508 00:20:28.534 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:28.534 | INFO | "malloc" : " 14 GB", | 20260508 00:20:28.534 | INFO | "pinned" : " 29 GB" | 20260508 00:20:28.534 | INFO | } | 20260508 00:20:28.534 | INFO | Perf report for the last 5.14166 seconds: | 20260508 00:20:28.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:28.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:20:28.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:20:28.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:28.534 | INFO | Uploaded 36GiB to GPU in 2.52114s (14.2792GiB/s) | 20260508 00:20:28.535 | INFO | Downloaded 34GiB to GPU in 2.45222s (13.865GiB/s) | 20260508 00:20:28.631 | INFO | Starting a round of Baz IO | 20260508 00:20:32.088 | INFO | Populating RTMetrics report | 20260508 00:20:32.088 | INFO | RTMetrics report populated | 20260508 00:20:32.088 | INFO | Opening tmp RTMetrics json file | 20260508 00:20:32.097 | INFO | Serializing RTMetrics to json | 20260508 00:20:32.112 | INFO | Serialization done, swapping tmp file now... | 20260508 00:20:32.112 | INFO | RTMetrics done | 20260508 00:20:32.194 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.563 seconds | 20260508 00:20:32.194 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.23% / 0.00% / 57.76% | 20260508 00:20:32.194 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03041% / 7.94729e-06% / 5.96046e-05% | 20260508 00:20:32.195 | INFO | Baz IO overall: | 20260508 00:20:32.195 | INFO | BytesWritten : 1707223303289 | 20260508 00:20:32.195 | INFO | NumEvents : 826353263081 | 20260508 00:20:33.668 | INFO | Analyzing chunk frames = [3756032,3756544) | 20260508 00:20:33.668 | INFO | MemInfo: { | 20260508 00:20:33.668 | INFO | "gpu" : " 17 GB", | 20260508 00:20:33.668 | INFO | "huge" : "", | 20260508 00:20:33.668 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:33.668 | INFO | "malloc" : " 14 GB", | 20260508 00:20:33.668 | INFO | "pinned" : " 29 GB" | 20260508 00:20:33.668 | INFO | } | 20260508 00:20:33.668 | INFO | Perf report for the last 5.13401 seconds: | 20260508 00:20:33.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:33.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:33.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:33.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:33.668 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260508 00:20:33.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:33.668 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260508 00:20:33.668 | INFO | PRE_HQ : 65.6%, 1.7, 56.8ms, 0 | 20260508 00:20:33.668 | INFO | BAZWRITER : 69.0%, 3.8, 135.5ms, 0 | 20260508 00:20:33.668 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 00:20:33.669 | INFO | Uploaded 36GiB to GPU in 2.56186s (14.0523GiB/s) | 20260508 00:20:33.669 | INFO | Downloaded 38GiB to GPU in 2.70674s (14.039GiB/s) | 20260508 00:20:38.806 | INFO | Analyzing chunk frames = [3756544,3757056) | 20260508 00:20:38.806 | INFO | MemInfo: { | 20260508 00:20:38.806 | INFO | "gpu" : " 17 GB", | 20260508 00:20:38.807 | INFO | "huge" : "", | 20260508 00:20:38.807 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:38.807 | INFO | "malloc" : " 14 GB", | 20260508 00:20:38.807 | INFO | "pinned" : " 29 GB" | 20260508 00:20:38.807 | INFO | } | 20260508 00:20:38.807 | INFO | Perf report for the last 5.13877 seconds: | 20260508 00:20:38.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:38.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:20:38.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:20:38.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:38.807 | INFO | Uploaded 36GiB to GPU in 2.58227s (13.9412GiB/s) | 20260508 00:20:38.807 | INFO | Downloaded 34GiB to GPU in 2.47424s (13.7416GiB/s) | 20260508 00:20:43.944 | INFO | Analyzing chunk frames = [3757056,3757568) | 20260508 00:20:43.944 | INFO | MemInfo: { | 20260508 00:20:43.944 | INFO | "gpu" : " 17 GB", | 20260508 00:20:43.944 | INFO | "huge" : "", | 20260508 00:20:43.944 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:43.944 | INFO | "malloc" : " 14 GB", | 20260508 00:20:43.944 | INFO | "pinned" : " 29 GB" | 20260508 00:20:43.944 | INFO | } | 20260508 00:20:43.944 | INFO | Perf report for the last 5.13757 seconds: | 20260508 00:20:43.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:43.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.944 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:20:43.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.944 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:20:43.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:43.945 | INFO | Uploaded 36GiB to GPU in 2.56467s (14.0369GiB/s) | 20260508 00:20:43.945 | INFO | Downloaded 34GiB to GPU in 2.46816s (13.7754GiB/s) | 20260508 00:20:49.090 | INFO | Analyzing chunk frames = [3757568,3758080) | 20260508 00:20:49.090 | INFO | MemInfo: { | 20260508 00:20:49.090 | INFO | "gpu" : " 17 GB", | 20260508 00:20:49.090 | INFO | "huge" : "", | 20260508 00:20:49.090 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:49.090 | INFO | "malloc" : " 14 GB", | 20260508 00:20:49.090 | INFO | "pinned" : " 29 GB" | 20260508 00:20:49.090 | INFO | } | 20260508 00:20:49.090 | INFO | Perf report for the last 5.14597 seconds: | 20260508 00:20:49.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:49.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.090 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 00:20:49.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.090 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 00:20:49.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:49.091 | INFO | Uploaded 36GiB to GPU in 2.48565s (14.4831GiB/s) | 20260508 00:20:49.091 | INFO | Downloaded 34GiB to GPU in 2.3849s (14.2563GiB/s) | 20260508 00:20:54.233 | INFO | Analyzing chunk frames = [3758080,3758592) | 20260508 00:20:54.233 | INFO | MemInfo: { | 20260508 00:20:54.233 | INFO | "gpu" : " 17 GB", | 20260508 00:20:54.233 | INFO | "huge" : "", | 20260508 00:20:54.233 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:54.233 | INFO | "malloc" : " 14 GB", | 20260508 00:20:54.233 | INFO | "pinned" : " 29 GB" | 20260508 00:20:54.233 | INFO | } | 20260508 00:20:54.234 | INFO | Perf report for the last 5.14333 seconds: | 20260508 00:20:54.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:54.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:20:54.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:20:54.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:54.234 | INFO | Uploaded 36GiB to GPU in 2.42601s (14.8392GiB/s) | 20260508 00:20:54.234 | INFO | Downloaded 34GiB to GPU in 2.4961s (13.6213GiB/s) | 20260508 00:20:59.371 | INFO | Analyzing chunk frames = [3758592,3759104) | 20260508 00:20:59.371 | INFO | MemInfo: { | 20260508 00:20:59.371 | INFO | "gpu" : " 17 GB", | 20260508 00:20:59.371 | INFO | "huge" : "", | 20260508 00:20:59.371 | INFO | "huge_shared" : " 29 GB", | 20260508 00:20:59.371 | INFO | "malloc" : " 14 GB", | 20260508 00:20:59.371 | INFO | "pinned" : " 29 GB" | 20260508 00:20:59.371 | INFO | } | 20260508 00:20:59.371 | INFO | Perf report for the last 5.13739 seconds: | 20260508 00:20:59.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:20:59.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:20:59.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260508 00:20:59.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:20:59.371 | INFO | Uploaded 36GiB to GPU in 2.39649s (15.022GiB/s) | 20260508 00:20:59.372 | INFO | Downloaded 34GiB to GPU in 2.39717s (14.1834GiB/s) | 20260508 00:21:04.504 | INFO | Analyzing chunk frames = [3759104,3759616) | 20260508 00:21:04.504 | INFO | MemInfo: { | 20260508 00:21:04.504 | INFO | "gpu" : " 17 GB", | 20260508 00:21:04.504 | INFO | "huge" : "", | 20260508 00:21:04.504 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:04.504 | INFO | "malloc" : " 14 GB", | 20260508 00:21:04.504 | INFO | "pinned" : " 29 GB" | 20260508 00:21:04.504 | INFO | } | 20260508 00:21:04.505 | INFO | Perf report for the last 5.13351 seconds: | 20260508 00:21:04.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:04.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.505 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 00:21:04.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.505 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 00:21:04.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:04.506 | INFO | Uploaded 36GiB to GPU in 2.44249s (14.7391GiB/s) | 20260508 00:21:04.506 | INFO | Downloaded 34GiB to GPU in 2.42171s (14.0397GiB/s) | 20260508 00:21:09.645 | INFO | Analyzing chunk frames = [3759616,3760128) | 20260508 00:21:09.645 | INFO | MemInfo: { | 20260508 00:21:09.645 | INFO | "gpu" : " 17 GB", | 20260508 00:21:09.645 | INFO | "huge" : "", | 20260508 00:21:09.645 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:09.645 | INFO | "malloc" : " 14 GB", | 20260508 00:21:09.645 | INFO | "pinned" : " 29 GB" | 20260508 00:21:09.645 | INFO | } | 20260508 00:21:09.645 | INFO | Perf report for the last 5.14083 seconds: | 20260508 00:21:09.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:09.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.646 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:21:09.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.646 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260508 00:21:09.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:09.646 | INFO | Uploaded 36GiB to GPU in 2.55913s (14.0673GiB/s) | 20260508 00:21:09.646 | INFO | Downloaded 34GiB to GPU in 2.4514s (13.8696GiB/s) | 20260508 00:21:09.646 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:21:09.646 | INFO | Total mem usage: 16.7 GiB | 20260508 00:21:09.646 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:21:09.646 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 00:21:09.646 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:21:09.646 | INFO | High Water Breakdown: | 20260508 00:21:09.646 | INFO | 60.1% : 8.97 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:21:09.646 | INFO | 39.7% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:21:09.646 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:21:09.646 | INFO | ------------------------------------------ | 20260508 00:21:09.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:21:09.646 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:21:09.646 | INFO | Total mem usage: 26.5 GiB | 20260508 00:21:09.646 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:21:09.646 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:21:09.646 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:21:09.646 | INFO | High Water Breakdown: | 20260508 00:21:09.646 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:21:09.646 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:21:09.646 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:21:09.646 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:21:09.646 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:21:09.646 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:21:09.646 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:21:09.646 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:21:09.646 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:21:09.646 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:21:09.646 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:21:09.646 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:21:09.646 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:21:09.646 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:21:09.646 | INFO | ------------------------------------------ | 20260508 00:21:09.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:21:09.646 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:21:09.646 | INFO | Total mem usage: 27.8 GiB | 20260508 00:21:09.646 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:21:09.646 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:21:09.646 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:21:09.646 | INFO | High Water Breakdown: | 20260508 00:21:09.646 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:21:09.646 | INFO | ------------------------------------------ | 20260508 00:21:09.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:21:09.646 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:21:09.646 | INFO | Total mem usage: 15.7 GiB | 20260508 00:21:09.646 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:21:09.646 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:21:09.646 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:21:09.646 | INFO | High Water Breakdown: | 20260508 00:21:09.647 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:21:09.647 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:21:09.647 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:21:09.647 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:21:09.647 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:21:09.647 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:21:09.647 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:21:09.647 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:21:09.647 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:21:09.647 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:21:09.647 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:21:09.647 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:21:09.647 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:21:09.647 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:21:09.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:21:09.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:21:09.647 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:21:09.647 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:21:09.647 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:21:09.647 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:21:09.647 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:21:09.647 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:21:09.647 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:21:09.647 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:21:09.647 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:21:09.647 | INFO | ------------------------------------------ | 20260508 00:21:09.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:21:09.647 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:21:09.647 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:21:09.647 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:21:09.647 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.38e+01 / 1.53e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.16e-02 / 4.79e-03 | 20260508 00:21:09.647 | INFO | PerfReport: 12.17%, Baseline -- 4.00e+00 / 4.49e+00, 5.95e-01 / 1.86e-01 | 20260508 00:21:09.647 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.15e-01 | 20260508 00:21:09.647 | INFO | PerfReport: 15.89%, DME -- 4.18e+01 / 4.55e+01, 3.91e+00 / 1.64e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.76e+00 / 1.84e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.42e+00 / 2.71e+00, 4.27e-01 / 2.12e-01 | 20260508 00:21:09.647 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.33e+00, 2.81e-01 / 1.80e-01 | 20260508 00:21:09.647 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.79e-01, 5.58e-02 / 2.52e-02 | 20260508 00:21:09.647 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:21:09.647 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:21:09.647 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:21:09.647 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.44e+00 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 30.84%, Compute -- 3.29e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 2.03%, LockUpload -- 2.16e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 2.62%, LockDownload -- 2.80e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 15.90%, LockCompute -- 1.70e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 00:21:09.647 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260508 00:21:09.758 | INFO | Starting a round of Baz IO | 20260508 00:21:13.229 | INFO | Populating RTMetrics report | 20260508 00:21:13.229 | INFO | RTMetrics report populated | 20260508 00:21:13.229 | INFO | Opening tmp RTMetrics json file | 20260508 00:21:13.260 | INFO | Serializing RTMetrics to json | 20260508 00:21:13.262 | INFO | Serialization done, swapping tmp file now... | 20260508 00:21:13.263 | INFO | RTMetrics done | 20260508 00:21:13.322 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.564 seconds | 20260508 00:21:13.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.23% / 0.00% / 57.77% | 20260508 00:21:13.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03276% / 7.94729e-06% / 6.75519e-05% | 20260508 00:21:13.322 | INFO | Baz IO overall: | 20260508 00:21:13.322 | INFO | BytesWritten : 1709043085955 | 20260508 00:21:13.322 | INFO | NumEvents : 827229560251 | 20260508 00:21:14.785 | INFO | Analyzing chunk frames = [3760128,3760640) | 20260508 00:21:14.785 | INFO | MemInfo: { | 20260508 00:21:14.785 | INFO | "gpu" : " 17 GB", | 20260508 00:21:14.785 | INFO | "huge" : "", | 20260508 00:21:14.785 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:14.785 | INFO | "malloc" : " 14 GB", | 20260508 00:21:14.785 | INFO | "pinned" : " 29 GB" | 20260508 00:21:14.785 | INFO | } | 20260508 00:21:14.785 | INFO | Perf report for the last 5.13957 seconds: | 20260508 00:21:14.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:14.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:14.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:14.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:14.785 | INFO | ANALYSIS : 68.8%, 2.9, 104.8ms, 0 | 20260508 00:21:14.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:14.785 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260508 00:21:14.785 | INFO | PRE_HQ : 67.5%, 1.8, 62.8ms, 0 | 20260508 00:21:14.785 | INFO | BAZWRITER : 68.9%, 4.0, 140.7ms, 0 | 20260508 00:21:14.785 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 00:21:14.785 | INFO | Uploaded 36GiB to GPU in 2.58645s (13.9187GiB/s) | 20260508 00:21:14.785 | INFO | Downloaded 38GiB to GPU in 2.69904s (14.0791GiB/s) | 20260508 00:21:19.920 | INFO | Analyzing chunk frames = [3760640,3761152) | 20260508 00:21:19.920 | INFO | MemInfo: { | 20260508 00:21:19.920 | INFO | "gpu" : " 17 GB", | 20260508 00:21:19.920 | INFO | "huge" : "", | 20260508 00:21:19.920 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:19.920 | INFO | "malloc" : " 14 GB", | 20260508 00:21:19.920 | INFO | "pinned" : " 29 GB" | 20260508 00:21:19.920 | INFO | } | 20260508 00:21:19.920 | INFO | Perf report for the last 5.13533 seconds: | 20260508 00:21:19.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:19.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.920 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:21:19.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.920 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:21:19.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:19.921 | INFO | Uploaded 36GiB to GPU in 2.52744s (14.2436GiB/s) | 20260508 00:21:19.921 | INFO | Downloaded 34GiB to GPU in 2.50074s (13.596GiB/s) | 20260508 00:21:25.066 | INFO | Analyzing chunk frames = [3761152,3761664) | 20260508 00:21:25.066 | INFO | MemInfo: { | 20260508 00:21:25.066 | INFO | "gpu" : " 17 GB", | 20260508 00:21:25.066 | INFO | "huge" : "", | 20260508 00:21:25.066 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:25.066 | INFO | "malloc" : " 14 GB", | 20260508 00:21:25.066 | INFO | "pinned" : " 29 GB" | 20260508 00:21:25.066 | INFO | } | 20260508 00:21:25.066 | INFO | Perf report for the last 5.14573 seconds: | 20260508 00:21:25.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:25.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.066 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 00:21:25.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.066 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:21:25.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:25.067 | INFO | Uploaded 36GiB to GPU in 2.50836s (14.352GiB/s) | 20260508 00:21:25.067 | INFO | Downloaded 34GiB to GPU in 2.48306s (13.6928GiB/s) | 20260508 00:21:30.201 | INFO | Analyzing chunk frames = [3761664,3762176) | 20260508 00:21:30.201 | INFO | MemInfo: { | 20260508 00:21:30.201 | INFO | "gpu" : " 17 GB", | 20260508 00:21:30.201 | INFO | "huge" : "", | 20260508 00:21:30.201 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:30.201 | INFO | "malloc" : " 14 GB", | 20260508 00:21:30.201 | INFO | "pinned" : " 29 GB" | 20260508 00:21:30.201 | INFO | } | 20260508 00:21:30.201 | INFO | Perf report for the last 5.13493 seconds: | 20260508 00:21:30.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:30.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.201 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:21:30.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.202 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:21:30.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:30.202 | INFO | Uploaded 36GiB to GPU in 2.47142s (14.5665GiB/s) | 20260508 00:21:30.202 | INFO | Downloaded 34GiB to GPU in 2.50478s (13.574GiB/s) | 20260508 00:21:35.345 | INFO | Analyzing chunk frames = [3762176,3762688) | 20260508 00:21:35.345 | INFO | MemInfo: { | 20260508 00:21:35.345 | INFO | "gpu" : " 17 GB", | 20260508 00:21:35.345 | INFO | "huge" : "", | 20260508 00:21:35.345 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:35.345 | INFO | "malloc" : " 14 GB", | 20260508 00:21:35.345 | INFO | "pinned" : " 29 GB" | 20260508 00:21:35.345 | INFO | } | 20260508 00:21:35.345 | INFO | Perf report for the last 5.1443 seconds: | 20260508 00:21:35.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:35.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.345 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:21:35.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.345 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 00:21:35.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:35.346 | INFO | Uploaded 36GiB to GPU in 2.27687s (15.8111GiB/s) | 20260508 00:21:35.346 | INFO | Downloaded 34GiB to GPU in 2.48411s (13.687GiB/s) | 20260508 00:21:40.484 | INFO | Analyzing chunk frames = [3762688,3763200) | 20260508 00:21:40.484 | INFO | MemInfo: { | 20260508 00:21:40.484 | INFO | "gpu" : " 17 GB", | 20260508 00:21:40.484 | INFO | "huge" : "", | 20260508 00:21:40.484 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:40.484 | INFO | "malloc" : " 14 GB", | 20260508 00:21:40.484 | INFO | "pinned" : " 29 GB" | 20260508 00:21:40.484 | INFO | } | 20260508 00:21:40.484 | INFO | Perf report for the last 5.13888 seconds: | 20260508 00:21:40.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:40.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 00:21:40.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:21:40.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:40.484 | INFO | Uploaded 36GiB to GPU in 2.39595s (15.0254GiB/s) | 20260508 00:21:40.485 | INFO | Downloaded 34GiB to GPU in 2.40153s (14.1576GiB/s) | 20260508 00:21:45.624 | INFO | Analyzing chunk frames = [3763200,3763712) | 20260508 00:21:45.624 | INFO | MemInfo: { | 20260508 00:21:45.624 | INFO | "gpu" : " 17 GB", | 20260508 00:21:45.624 | INFO | "huge" : "", | 20260508 00:21:45.624 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:45.624 | INFO | "malloc" : " 14 GB", | 20260508 00:21:45.624 | INFO | "pinned" : " 29 GB" | 20260508 00:21:45.624 | INFO | } | 20260508 00:21:45.624 | INFO | Perf report for the last 5.1401 seconds: | 20260508 00:21:45.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:45.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.624 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:21:45.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.625 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 00:21:45.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:45.625 | INFO | Uploaded 36GiB to GPU in 2.54614s (14.1391GiB/s) | 20260508 00:21:45.625 | INFO | Downloaded 34GiB to GPU in 2.47132s (13.7578GiB/s) | 20260508 00:21:50.756 | INFO | Analyzing chunk frames = [3763712,3764224) | 20260508 00:21:50.756 | INFO | MemInfo: { | 20260508 00:21:50.756 | INFO | "gpu" : " 17 GB", | 20260508 00:21:50.757 | INFO | "huge" : "", | 20260508 00:21:50.757 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:50.757 | INFO | "malloc" : " 14 GB", | 20260508 00:21:50.757 | INFO | "pinned" : " 29 GB" | 20260508 00:21:50.757 | INFO | } | 20260508 00:21:50.757 | INFO | Perf report for the last 5.13241 seconds: | 20260508 00:21:50.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:50.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.757 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:21:50.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.757 | INFO | PRE_HQ : 36.5%, 1.0, 18.9ms, 0 | 20260508 00:21:50.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:50.758 | INFO | Uploaded 36GiB to GPU in 2.55477s (14.0913GiB/s) | 20260508 00:21:50.758 | INFO | Downloaded 34GiB to GPU in 2.46482s (13.7941GiB/s) | 20260508 00:21:50.870 | INFO | Starting a round of Baz IO | 20260508 00:21:54.333 | INFO | Populating RTMetrics report | 20260508 00:21:54.333 | INFO | RTMetrics report populated | 20260508 00:21:54.333 | INFO | Opening tmp RTMetrics json file | 20260508 00:21:54.349 | INFO | Serializing RTMetrics to json | 20260508 00:21:54.428 | INFO | Serialization done, swapping tmp file now... | 20260508 00:21:54.450 | INFO | RTMetrics done | 20260508 00:21:54.478 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.607 seconds | 20260508 00:21:54.478 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.23% / 0.00% / 57.77% | 20260508 00:21:54.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02859% / 1.19209e-05% / 6.75519e-05% | 20260508 00:21:54.478 | INFO | Baz IO overall: | 20260508 00:21:54.478 | INFO | BytesWritten : 1710861311437 | 20260508 00:21:54.478 | INFO | NumEvents : 828104646973 | 20260508 00:21:55.897 | INFO | Analyzing chunk frames = [3764224,3764736) | 20260508 00:21:55.897 | INFO | MemInfo: { | 20260508 00:21:55.897 | INFO | "gpu" : " 17 GB", | 20260508 00:21:55.897 | INFO | "huge" : "", | 20260508 00:21:55.898 | INFO | "huge_shared" : " 29 GB", | 20260508 00:21:55.898 | INFO | "malloc" : " 14 GB", | 20260508 00:21:55.898 | INFO | "pinned" : " 29 GB" | 20260508 00:21:55.898 | INFO | } | 20260508 00:21:55.898 | INFO | Perf report for the last 5.14081 seconds: | 20260508 00:21:55.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:21:55.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:55.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:55.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:55.898 | INFO | ANALYSIS : 68.9%, 2.9, 104.6ms, 0 | 20260508 00:21:55.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:21:55.898 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260508 00:21:55.898 | INFO | PRE_HQ : 67.7%, 1.8, 62.2ms, 0 | 20260508 00:21:55.898 | INFO | BAZWRITER : 70.0%, 4.1, 149.0ms, 0 | 20260508 00:21:55.898 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260508 00:21:55.898 | INFO | Uploaded 36GiB to GPU in 2.58011s (13.9529GiB/s) | 20260508 00:21:55.898 | INFO | Downloaded 38GiB to GPU in 2.69806s (14.0842GiB/s) | 20260508 00:22:01.043 | INFO | Analyzing chunk frames = [3764736,3765248) | 20260508 00:22:01.043 | INFO | MemInfo: { | 20260508 00:22:01.043 | INFO | "gpu" : " 17 GB", | 20260508 00:22:01.043 | INFO | "huge" : "", | 20260508 00:22:01.043 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:01.043 | INFO | "malloc" : " 14 GB", | 20260508 00:22:01.043 | INFO | "pinned" : " 29 GB" | 20260508 00:22:01.043 | INFO | } | 20260508 00:22:01.043 | INFO | Perf report for the last 5.14522 seconds: | 20260508 00:22:01.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:01.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 00:22:01.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:22:01.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:01.043 | INFO | Uploaded 36GiB to GPU in 2.59436s (13.8763GiB/s) | 20260508 00:22:01.043 | INFO | Downloaded 34GiB to GPU in 2.47944s (13.7128GiB/s) | 20260508 00:22:06.183 | INFO | Analyzing chunk frames = [3765248,3765760) | 20260508 00:22:06.183 | INFO | MemInfo: { | 20260508 00:22:06.183 | INFO | "gpu" : " 17 GB", | 20260508 00:22:06.183 | INFO | "huge" : "", | 20260508 00:22:06.183 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:06.183 | INFO | "malloc" : " 14 GB", | 20260508 00:22:06.183 | INFO | "pinned" : " 29 GB" | 20260508 00:22:06.183 | INFO | } | 20260508 00:22:06.183 | INFO | Perf report for the last 5.14023 seconds: | 20260508 00:22:06.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:06.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:22:06.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:22:06.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:06.183 | INFO | Uploaded 36GiB to GPU in 2.52848s (14.2378GiB/s) | 20260508 00:22:06.183 | INFO | Downloaded 34GiB to GPU in 2.48234s (13.6968GiB/s) | 20260508 00:22:11.318 | INFO | Analyzing chunk frames = [3765760,3766272) | 20260508 00:22:11.318 | INFO | MemInfo: { | 20260508 00:22:11.318 | INFO | "gpu" : " 17 GB", | 20260508 00:22:11.318 | INFO | "huge" : "", | 20260508 00:22:11.318 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:11.318 | INFO | "malloc" : " 14 GB", | 20260508 00:22:11.318 | INFO | "pinned" : " 29 GB" | 20260508 00:22:11.318 | INFO | } | 20260508 00:22:11.318 | INFO | Perf report for the last 5.13524 seconds: | 20260508 00:22:11.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:11.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.318 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:22:11.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.318 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:22:11.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:11.319 | INFO | Uploaded 36GiB to GPU in 2.54561s (14.142GiB/s) | 20260508 00:22:11.319 | INFO | Downloaded 34GiB to GPU in 2.46775s (13.7777GiB/s) | 20260508 00:22:16.456 | INFO | Analyzing chunk frames = [3766272,3766784) | 20260508 00:22:16.456 | INFO | MemInfo: { | 20260508 00:22:16.456 | INFO | "gpu" : " 17 GB", | 20260508 00:22:16.456 | INFO | "huge" : "", | 20260508 00:22:16.456 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:16.456 | INFO | "malloc" : " 14 GB", | 20260508 00:22:16.456 | INFO | "pinned" : " 29 GB" | 20260508 00:22:16.456 | INFO | } | 20260508 00:22:16.456 | INFO | Perf report for the last 5.13794 seconds: | 20260508 00:22:16.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:16.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.456 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:22:16.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.456 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 00:22:16.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:16.457 | INFO | Uploaded 36GiB to GPU in 2.40469s (14.9707GiB/s) | 20260508 00:22:16.457 | INFO | Downloaded 34GiB to GPU in 2.48376s (13.6889GiB/s) | 20260508 00:22:21.600 | INFO | Analyzing chunk frames = [3766784,3767296) | 20260508 00:22:21.600 | INFO | MemInfo: { | 20260508 00:22:21.600 | INFO | "gpu" : " 17 GB", | 20260508 00:22:21.600 | INFO | "huge" : "", | 20260508 00:22:21.600 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:21.600 | INFO | "malloc" : " 14 GB", | 20260508 00:22:21.600 | INFO | "pinned" : " 29 GB" | 20260508 00:22:21.600 | INFO | } | 20260508 00:22:21.600 | INFO | Perf report for the last 5.14346 seconds: | 20260508 00:22:21.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:21.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260508 00:22:21.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:22:21.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:21.600 | INFO | Uploaded 36GiB to GPU in 2.51431s (14.318GiB/s) | 20260508 00:22:21.600 | INFO | Downloaded 34GiB to GPU in 2.41957s (14.0521GiB/s) | 20260508 00:22:26.737 | INFO | Analyzing chunk frames = [3767296,3767808) | 20260508 00:22:26.737 | INFO | MemInfo: { | 20260508 00:22:26.737 | INFO | "gpu" : " 17 GB", | 20260508 00:22:26.738 | INFO | "huge" : "", | 20260508 00:22:26.738 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:26.738 | INFO | "malloc" : " 14 GB", | 20260508 00:22:26.738 | INFO | "pinned" : " 29 GB" | 20260508 00:22:26.738 | INFO | } | 20260508 00:22:26.738 | INFO | Perf report for the last 5.13813 seconds: | 20260508 00:22:26.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:26.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:22:26.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 00:22:26.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:26.738 | INFO | Uploaded 36GiB to GPU in 2.53915s (14.1779GiB/s) | 20260508 00:22:26.739 | INFO | Downloaded 34GiB to GPU in 2.44322s (13.9161GiB/s) | 20260508 00:22:31.875 | INFO | Analyzing chunk frames = [3767808,3768320) | 20260508 00:22:31.875 | INFO | MemInfo: { | 20260508 00:22:31.875 | INFO | "gpu" : " 17 GB", | 20260508 00:22:31.875 | INFO | "huge" : "", | 20260508 00:22:31.875 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:31.876 | INFO | "malloc" : " 14 GB", | 20260508 00:22:31.876 | INFO | "pinned" : " 29 GB" | 20260508 00:22:31.876 | INFO | } | 20260508 00:22:31.876 | INFO | Perf report for the last 5.13793 seconds: | 20260508 00:22:31.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:31.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.876 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:22:31.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.876 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260508 00:22:31.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:31.877 | INFO | Uploaded 36GiB to GPU in 2.56459s (14.0373GiB/s) | 20260508 00:22:31.877 | INFO | Downloaded 34GiB to GPU in 2.45051s (13.8747GiB/s) | 20260508 00:22:31.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:22:31.877 | INFO | Total mem usage: 16.7 GiB | 20260508 00:22:31.877 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:22:31.877 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.6%) | 20260508 00:22:31.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:22:31.877 | INFO | High Water Breakdown: | 20260508 00:22:31.877 | INFO | 60.3% : 9.1 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:22:31.877 | INFO | 39.5% : 5.96 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:22:31.877 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:22:31.877 | INFO | ------------------------------------------ | 20260508 00:22:31.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:22:31.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:22:31.877 | INFO | Total mem usage: 26.5 GiB | 20260508 00:22:31.877 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:22:31.877 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:22:31.877 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:22:31.877 | INFO | High Water Breakdown: | 20260508 00:22:31.877 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:22:31.877 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:22:31.877 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:22:31.877 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:22:31.877 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:22:31.877 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:22:31.877 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:22:31.877 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:22:31.877 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:22:31.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:22:31.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:22:31.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:22:31.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:22:31.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:22:31.877 | INFO | ------------------------------------------ | 20260508 00:22:31.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:22:31.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:22:31.877 | INFO | Total mem usage: 27.8 GiB | 20260508 00:22:31.877 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:22:31.877 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:22:31.877 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:22:31.877 | INFO | High Water Breakdown: | 20260508 00:22:31.877 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:22:31.877 | INFO | ------------------------------------------ | 20260508 00:22:31.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:22:31.877 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:22:31.877 | INFO | Total mem usage: 15.7 GiB | 20260508 00:22:31.877 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:22:31.877 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:22:31.877 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:22:31.877 | INFO | High Water Breakdown: | 20260508 00:22:31.877 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:22:31.877 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:22:31.877 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:22:31.877 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:22:31.877 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:22:31.878 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:22:31.878 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:22:31.878 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:22:31.878 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:22:31.878 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:22:31.878 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:22:31.878 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:22:31.878 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:22:31.878 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:22:31.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:22:31.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:22:31.878 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:22:31.878 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:22:31.878 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:22:31.878 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:22:31.878 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:22:31.878 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:22:31.878 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:22:31.878 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:22:31.878 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:22:31.878 | INFO | ------------------------------------------ | 20260508 00:22:31.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:22:31.878 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:22:31.878 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:22:31.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:22:31.878 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.41e+01 / 1.55e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.22e-02 / 6.06e-03 | 20260508 00:22:31.878 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.43e+00, 5.81e-01 / 2.44e-01 | 20260508 00:22:31.878 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 1.02e-01 | 20260508 00:22:31.878 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.61e+01, 4.13e+00 / 1.57e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 56.30%, FrameLabeling -- 1.84e+01 / 2.00e+01, 1.65e+00 / 1.51e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.65e+00, 4.29e-01 / 2.26e-01 | 20260508 00:22:31.878 | INFO | PerfReport: 3.40%, Metrics -- 1.11e+00 / 1.28e+00, 2.79e-01 / 1.63e-01 | 20260508 00:22:31.878 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.44e-01 / 3.69e-01, 5.51e-02 / 2.72e-02 | 20260508 00:22:31.878 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:22:31.878 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:22:31.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:22:31.878 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 30.74%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 2.23%, LockUpload -- 2.38e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 2.65%, LockDownload -- 2.82e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 15.47%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:22:31.878 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260508 00:22:32.001 | INFO | Starting a round of Baz IO | 20260508 00:22:35.453 | INFO | Populating RTMetrics report | 20260508 00:22:35.454 | INFO | RTMetrics report populated | 20260508 00:22:35.454 | INFO | Opening tmp RTMetrics json file | 20260508 00:22:35.477 | INFO | Serializing RTMetrics to json | 20260508 00:22:35.479 | INFO | Serialization done, swapping tmp file now... | 20260508 00:22:35.479 | INFO | RTMetrics done | 20260508 00:22:35.547 | INFO | Finished a round of Baz IO: Wrote 1.692 GiB in 3.545 seconds | 20260508 00:22:35.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.22% / 0.00% / 57.77% | 20260508 00:22:35.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02377% / 3.97364e-06% / 5.96046e-05% | 20260508 00:22:35.547 | INFO | Baz IO overall: | 20260508 00:22:35.547 | INFO | BytesWritten : 1712677642544 | 20260508 00:22:35.547 | INFO | NumEvents : 828978363052 | 20260508 00:22:37.011 | INFO | Analyzing chunk frames = [3768320,3768832) | 20260508 00:22:37.011 | INFO | MemInfo: { | 20260508 00:22:37.011 | INFO | "gpu" : " 17 GB", | 20260508 00:22:37.011 | INFO | "huge" : "", | 20260508 00:22:37.011 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:37.011 | INFO | "malloc" : " 14 GB", | 20260508 00:22:37.011 | INFO | "pinned" : " 29 GB" | 20260508 00:22:37.011 | INFO | } | 20260508 00:22:37.011 | INFO | Perf report for the last 5.13507 seconds: | 20260508 00:22:37.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:37.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:37.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:37.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:37.011 | INFO | ANALYSIS : 69.0%, 2.9, 104.7ms, 0 | 20260508 00:22:37.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:37.011 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260508 00:22:37.011 | INFO | PRE_HQ : 67.8%, 1.8, 62.8ms, 0 | 20260508 00:22:37.011 | INFO | BAZWRITER : 69.1%, 3.7, 130.6ms, 0 | 20260508 00:22:37.011 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:22:37.011 | INFO | Uploaded 36GiB to GPU in 2.63143s (13.6808GiB/s) | 20260508 00:22:37.011 | INFO | Downloaded 38GiB to GPU in 2.707s (14.0377GiB/s) | 20260508 00:22:42.153 | INFO | Analyzing chunk frames = [3768832,3769344) | 20260508 00:22:42.153 | INFO | MemInfo: { | 20260508 00:22:42.153 | INFO | "gpu" : " 17 GB", | 20260508 00:22:42.153 | INFO | "huge" : "", | 20260508 00:22:42.153 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:42.153 | INFO | "malloc" : " 14 GB", | 20260508 00:22:42.153 | INFO | "pinned" : " 29 GB" | 20260508 00:22:42.153 | INFO | } | 20260508 00:22:42.153 | INFO | Perf report for the last 5.14201 seconds: | 20260508 00:22:42.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:42.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 00:22:42.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:22:42.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:42.153 | INFO | Uploaded 36GiB to GPU in 2.58506s (13.9262GiB/s) | 20260508 00:22:42.153 | INFO | Downloaded 34GiB to GPU in 2.47743s (13.7239GiB/s) | 20260508 00:22:47.296 | INFO | Analyzing chunk frames = [3769344,3769856) | 20260508 00:22:47.296 | INFO | MemInfo: { | 20260508 00:22:47.296 | INFO | "gpu" : " 17 GB", | 20260508 00:22:47.296 | INFO | "huge" : "", | 20260508 00:22:47.297 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:47.297 | INFO | "malloc" : " 14 GB", | 20260508 00:22:47.297 | INFO | "pinned" : " 29 GB" | 20260508 00:22:47.297 | INFO | } | 20260508 00:22:47.297 | INFO | Perf report for the last 5.14418 seconds: | 20260508 00:22:47.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:47.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.297 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 00:22:47.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.297 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:22:47.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:47.298 | INFO | Uploaded 36GiB to GPU in 2.52833s (14.2386GiB/s) | 20260508 00:22:47.298 | INFO | Downloaded 34GiB to GPU in 2.47813s (13.72GiB/s) | 20260508 00:22:52.434 | INFO | Analyzing chunk frames = [3769856,3770368) | 20260508 00:22:52.434 | INFO | MemInfo: { | 20260508 00:22:52.434 | INFO | "gpu" : " 17 GB", | 20260508 00:22:52.434 | INFO | "huge" : "", | 20260508 00:22:52.434 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:52.434 | INFO | "malloc" : " 14 GB", | 20260508 00:22:52.434 | INFO | "pinned" : " 29 GB" | 20260508 00:22:52.434 | INFO | } | 20260508 00:22:52.434 | INFO | Perf report for the last 5.13727 seconds: | 20260508 00:22:52.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:52.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.434 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 00:22:52.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.435 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:22:52.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:52.435 | INFO | Uploaded 36GiB to GPU in 2.56732s (14.0224GiB/s) | 20260508 00:22:52.435 | INFO | Downloaded 34GiB to GPU in 2.47679s (13.7275GiB/s) | 20260508 00:22:57.574 | INFO | Analyzing chunk frames = [3770368,3770880) | 20260508 00:22:57.574 | INFO | MemInfo: { | 20260508 00:22:57.574 | INFO | "gpu" : " 17 GB", | 20260508 00:22:57.574 | INFO | "huge" : "", | 20260508 00:22:57.574 | INFO | "huge_shared" : " 29 GB", | 20260508 00:22:57.574 | INFO | "malloc" : " 14 GB", | 20260508 00:22:57.574 | INFO | "pinned" : " 29 GB" | 20260508 00:22:57.574 | INFO | } | 20260508 00:22:57.574 | INFO | Perf report for the last 5.13969 seconds: | 20260508 00:22:57.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:22:57.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 00:22:57.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:22:57.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:22:57.574 | INFO | Uploaded 36GiB to GPU in 2.40051s (14.9968GiB/s) | 20260508 00:22:57.574 | INFO | Downloaded 34GiB to GPU in 2.51542s (13.5166GiB/s) | 20260508 00:23:02.712 | INFO | Analyzing chunk frames = [3770880,3771392) | 20260508 00:23:02.712 | INFO | MemInfo: { | 20260508 00:23:02.712 | INFO | "gpu" : " 17 GB", | 20260508 00:23:02.712 | INFO | "huge" : "", | 20260508 00:23:02.712 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:02.712 | INFO | "malloc" : " 14 GB", | 20260508 00:23:02.712 | INFO | "pinned" : " 29 GB" | 20260508 00:23:02.712 | INFO | } | 20260508 00:23:02.712 | INFO | Perf report for the last 5.13819 seconds: | 20260508 00:23:02.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:02.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.712 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:23:02.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.712 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:23:02.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:02.713 | INFO | Uploaded 36GiB to GPU in 2.42494s (14.8458GiB/s) | 20260508 00:23:02.713 | INFO | Downloaded 34GiB to GPU in 2.38287s (14.2685GiB/s) | 20260508 00:23:07.854 | INFO | Analyzing chunk frames = [3771392,3771904) | 20260508 00:23:07.854 | INFO | MemInfo: { | 20260508 00:23:07.854 | INFO | "gpu" : " 17 GB", | 20260508 00:23:07.854 | INFO | "huge" : "", | 20260508 00:23:07.854 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:07.854 | INFO | "malloc" : " 14 GB", | 20260508 00:23:07.854 | INFO | "pinned" : " 29 GB" | 20260508 00:23:07.854 | INFO | } | 20260508 00:23:07.854 | INFO | Perf report for the last 5.14188 seconds: | 20260508 00:23:07.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:07.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.854 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 00:23:07.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.854 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 00:23:07.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:07.855 | INFO | Uploaded 36GiB to GPU in 2.56154s (14.0541GiB/s) | 20260508 00:23:07.855 | INFO | Downloaded 34GiB to GPU in 2.44777s (13.8902GiB/s) | 20260508 00:23:12.991 | INFO | Analyzing chunk frames = [3771904,3772416) | 20260508 00:23:12.992 | INFO | MemInfo: { | 20260508 00:23:12.992 | INFO | "gpu" : " 17 GB", | 20260508 00:23:12.992 | INFO | "huge" : "", | 20260508 00:23:12.992 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:12.992 | INFO | "malloc" : " 14 GB", | 20260508 00:23:12.992 | INFO | "pinned" : " 29 GB" | 20260508 00:23:12.992 | INFO | } | 20260508 00:23:12.992 | INFO | Perf report for the last 5.1376 seconds: | 20260508 00:23:12.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:12.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 00:23:12.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | PRE_HQ : 35.3%, 1.0, 18.4ms, 0 | 20260508 00:23:12.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:12.992 | INFO | Uploaded 36GiB to GPU in 2.56148s (14.0544GiB/s) | 20260508 00:23:12.992 | INFO | Downloaded 34GiB to GPU in 2.45222s (13.865GiB/s) | 20260508 00:23:13.105 | INFO | Starting a round of Baz IO | 20260508 00:23:16.571 | INFO | Populating RTMetrics report | 20260508 00:23:16.572 | INFO | RTMetrics report populated | 20260508 00:23:16.572 | INFO | Opening tmp RTMetrics json file | 20260508 00:23:16.574 | INFO | Serializing RTMetrics to json | 20260508 00:23:16.600 | INFO | Serialization done, swapping tmp file now... | 20260508 00:23:16.601 | INFO | RTMetrics done | 20260508 00:23:16.674 | INFO | Finished a round of Baz IO: Wrote 1.694 GiB in 3.568 seconds | 20260508 00:23:16.674 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.22% / 0.00% / 57.78% | 20260508 00:23:16.674 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01113% / 1.19209e-05% / 5.16574e-05% | 20260508 00:23:16.674 | INFO | Baz IO overall: | 20260508 00:23:16.674 | INFO | BytesWritten : 1714496576959 | 20260508 00:23:16.674 | INFO | NumEvents : 829851816475 | 20260508 00:23:18.135 | INFO | Analyzing chunk frames = [3772416,3772928) | 20260508 00:23:18.135 | INFO | MemInfo: { | 20260508 00:23:18.135 | INFO | "gpu" : " 17 GB", | 20260508 00:23:18.135 | INFO | "huge" : "", | 20260508 00:23:18.135 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:18.135 | INFO | "malloc" : " 14 GB", | 20260508 00:23:18.135 | INFO | "pinned" : " 29 GB" | 20260508 00:23:18.135 | INFO | } | 20260508 00:23:18.135 | INFO | Perf report for the last 5.14331 seconds: | 20260508 00:23:18.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:18.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:18.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:18.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:18.135 | INFO | ANALYSIS : 68.9%, 2.9, 105.0ms, 0 | 20260508 00:23:18.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:18.135 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 00:23:18.135 | INFO | PRE_HQ : 67.7%, 1.8, 62.8ms, 0 | 20260508 00:23:18.135 | INFO | BAZWRITER : 69.0%, 3.8, 134.8ms, 0 | 20260508 00:23:18.135 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:23:18.136 | INFO | Uploaded 36GiB to GPU in 2.57865s (13.9608GiB/s) | 20260508 00:23:18.136 | INFO | Downloaded 38GiB to GPU in 2.68997s (14.1265GiB/s) | 20260508 00:23:23.271 | INFO | Analyzing chunk frames = [3772928,3773440) | 20260508 00:23:23.271 | INFO | MemInfo: { | 20260508 00:23:23.271 | INFO | "gpu" : " 17 GB", | 20260508 00:23:23.271 | INFO | "huge" : "", | 20260508 00:23:23.271 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:23.271 | INFO | "malloc" : " 14 GB", | 20260508 00:23:23.271 | INFO | "pinned" : " 29 GB" | 20260508 00:23:23.271 | INFO | } | 20260508 00:23:23.271 | INFO | Perf report for the last 5.13576 seconds: | 20260508 00:23:23.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:23.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 00:23:23.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:23:23.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:23.271 | INFO | Uploaded 36GiB to GPU in 2.60415s (13.8241GiB/s) | 20260508 00:23:23.271 | INFO | Downloaded 34GiB to GPU in 2.49032s (13.6528GiB/s) | 20260508 00:23:28.410 | INFO | Analyzing chunk frames = [3773440,3773952) | 20260508 00:23:28.410 | INFO | MemInfo: { | 20260508 00:23:28.410 | INFO | "gpu" : " 17 GB", | 20260508 00:23:28.410 | INFO | "huge" : "", | 20260508 00:23:28.411 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:28.411 | INFO | "malloc" : " 14 GB", | 20260508 00:23:28.411 | INFO | "pinned" : " 29 GB" | 20260508 00:23:28.411 | INFO | } | 20260508 00:23:28.411 | INFO | Perf report for the last 5.13969 seconds: | 20260508 00:23:28.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:28.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 00:23:28.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:23:28.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:28.411 | INFO | Uploaded 36GiB to GPU in 2.55544s (14.0876GiB/s) | 20260508 00:23:28.411 | INFO | Downloaded 34GiB to GPU in 2.48171s (13.7002GiB/s) | 20260508 00:23:33.552 | INFO | Analyzing chunk frames = [3773952,3774464) | 20260508 00:23:33.552 | INFO | MemInfo: { | 20260508 00:23:33.552 | INFO | "gpu" : " 17 GB", | 20260508 00:23:33.552 | INFO | "huge" : "", | 20260508 00:23:33.552 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:33.552 | INFO | "malloc" : " 14 GB", | 20260508 00:23:33.552 | INFO | "pinned" : " 29 GB" | 20260508 00:23:33.552 | INFO | } | 20260508 00:23:33.552 | INFO | Perf report for the last 5.14135 seconds: | 20260508 00:23:33.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:33.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.552 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:23:33.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.552 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:23:33.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:33.553 | INFO | Uploaded 36GiB to GPU in 2.53357s (14.2092GiB/s) | 20260508 00:23:33.553 | INFO | Downloaded 34GiB to GPU in 2.46584s (13.7884GiB/s) | 20260508 00:23:38.686 | INFO | Analyzing chunk frames = [3774464,3774976) | 20260508 00:23:38.686 | INFO | MemInfo: { | 20260508 00:23:38.686 | INFO | "gpu" : " 17 GB", | 20260508 00:23:38.686 | INFO | "huge" : "", | 20260508 00:23:38.686 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:38.686 | INFO | "malloc" : " 14 GB", | 20260508 00:23:38.686 | INFO | "pinned" : " 29 GB" | 20260508 00:23:38.686 | INFO | } | 20260508 00:23:38.686 | INFO | Perf report for the last 5.13402 seconds: | 20260508 00:23:38.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:38.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.686 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:23:38.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.686 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 00:23:38.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:38.687 | INFO | Uploaded 36GiB to GPU in 2.3792s (15.1311GiB/s) | 20260508 00:23:38.687 | INFO | Downloaded 34GiB to GPU in 2.49392s (13.6331GiB/s) | 20260508 00:23:43.828 | INFO | Analyzing chunk frames = [3774976,3775488) | 20260508 00:23:43.828 | INFO | MemInfo: { | 20260508 00:23:43.828 | INFO | "gpu" : " 17 GB", | 20260508 00:23:43.828 | INFO | "huge" : "", | 20260508 00:23:43.828 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:43.828 | INFO | "malloc" : " 14 GB", | 20260508 00:23:43.828 | INFO | "pinned" : " 29 GB" | 20260508 00:23:43.828 | INFO | } | 20260508 00:23:43.828 | INFO | Perf report for the last 5.14157 seconds: | 20260508 00:23:43.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:43.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 00:23:43.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:23:43.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:43.828 | INFO | Uploaded 36GiB to GPU in 2.50753s (14.3567GiB/s) | 20260508 00:23:43.828 | INFO | Downloaded 34GiB to GPU in 2.43696s (13.9518GiB/s) | 20260508 00:23:48.961 | INFO | Analyzing chunk frames = [3775488,3776000) | 20260508 00:23:48.962 | INFO | MemInfo: { | 20260508 00:23:48.962 | INFO | "gpu" : " 17 GB", | 20260508 00:23:48.962 | INFO | "huge" : "", | 20260508 00:23:48.962 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:48.962 | INFO | "malloc" : " 14 GB", | 20260508 00:23:48.962 | INFO | "pinned" : " 29 GB" | 20260508 00:23:48.962 | INFO | } | 20260508 00:23:48.962 | INFO | Perf report for the last 5.13425 seconds: | 20260508 00:23:48.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:48.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 00:23:48.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 00:23:48.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:48.962 | INFO | Uploaded 36GiB to GPU in 2.53459s (14.2035GiB/s) | 20260508 00:23:48.962 | INFO | Downloaded 34GiB to GPU in 2.45619s (13.8426GiB/s) | 20260508 00:23:54.103 | INFO | Analyzing chunk frames = [3776000,3776512) | 20260508 00:23:54.103 | INFO | MemInfo: { | 20260508 00:23:54.103 | INFO | "gpu" : " 17 GB", | 20260508 00:23:54.103 | INFO | "huge" : "", | 20260508 00:23:54.103 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:54.103 | INFO | "malloc" : " 14 GB", | 20260508 00:23:54.103 | INFO | "pinned" : " 29 GB" | 20260508 00:23:54.103 | INFO | } | 20260508 00:23:54.104 | INFO | Perf report for the last 5.14162 seconds: | 20260508 00:23:54.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:54.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:23:54.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | PRE_HQ : 36.0%, 1.0, 18.7ms, 0 | 20260508 00:23:54.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:54.104 | INFO | Uploaded 36GiB to GPU in 2.55785s (14.0743GiB/s) | 20260508 00:23:54.104 | INFO | Downloaded 34GiB to GPU in 2.45828s (13.8308GiB/s) | 20260508 00:23:54.104 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:23:54.104 | INFO | Total mem usage: 16.7 GiB | 20260508 00:23:54.104 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:23:54.104 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 00:23:54.104 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:23:54.104 | INFO | High Water Breakdown: | 20260508 00:23:54.104 | INFO | 59.8% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:23:54.104 | INFO | 40.1% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:23:54.104 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:23:54.104 | INFO | ------------------------------------------ | 20260508 00:23:54.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:23:54.104 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:23:54.104 | INFO | Total mem usage: 26.5 GiB | 20260508 00:23:54.104 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:23:54.104 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:23:54.104 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:23:54.104 | INFO | High Water Breakdown: | 20260508 00:23:54.104 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:23:54.104 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:23:54.104 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:23:54.104 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:23:54.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:23:54.104 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:23:54.104 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:23:54.105 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:23:54.105 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:23:54.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:23:54.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:23:54.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:23:54.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:23:54.105 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:23:54.105 | INFO | ------------------------------------------ | 20260508 00:23:54.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:23:54.105 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:23:54.105 | INFO | Total mem usage: 27.8 GiB | 20260508 00:23:54.105 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:23:54.105 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:23:54.105 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:23:54.105 | INFO | High Water Breakdown: | 20260508 00:23:54.105 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:23:54.105 | INFO | ------------------------------------------ | 20260508 00:23:54.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:23:54.105 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:23:54.105 | INFO | Total mem usage: 15.7 GiB | 20260508 00:23:54.105 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:23:54.105 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:23:54.105 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:23:54.105 | INFO | High Water Breakdown: | 20260508 00:23:54.105 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:23:54.105 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:23:54.105 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:23:54.105 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:23:54.105 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:23:54.105 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:23:54.105 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:23:54.105 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:23:54.105 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:23:54.105 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:23:54.105 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:23:54.105 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:23:54.105 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:23:54.105 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:23:54.105 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:23:54.105 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:23:54.105 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:23:54.105 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:23:54.105 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:23:54.105 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:23:54.105 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:23:54.105 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:23:54.105 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:23:54.105 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:23:54.105 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:23:54.105 | INFO | ------------------------------------------ | 20260508 00:23:54.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:23:54.105 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:23:54.105 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:23:54.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:23:54.105 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.67e+01, 1.43e+01 / 1.80e+00 | 20260508 00:23:54.105 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.24e-02 / 6.76e-03 | 20260508 00:23:54.105 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 5.79e-01 / 2.24e-01 | 20260508 00:23:54.105 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.23e-01 | 20260508 00:23:54.105 | INFO | PerfReport: 16.21%, DME -- 4.24e+01 / 4.68e+01, 4.32e+00 / 2.06e+00 | 20260508 00:23:54.105 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.83e+01 / 1.95e+01, 1.54e+00 / 1.48e+00 | 20260508 00:23:54.105 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.45e+00 / 2.72e+00, 4.31e-01 / 2.33e-01 | 20260508 00:23:54.105 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.30e+00, 2.75e-01 / 1.39e-01 | 20260508 00:23:54.105 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.43e-01 / 3.71e-01, 5.45e-02 / 2.54e-02 | 20260508 00:23:54.106 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:23:54.106 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:23:54.106 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:23:54.106 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 24.67%, Upload -- 2.64e+01 / 0.00e+00, 8.45e+00 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 24.31%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 30.59%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 2.56%, LockUpload -- 2.74e+00 / 0.00e+00, 5.80e+00 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 2.49%, LockDownload -- 2.66e+00 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 15.36%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:23:54.106 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260508 00:23:54.219 | INFO | Starting a round of Baz IO | 20260508 00:23:57.681 | INFO | Populating RTMetrics report | 20260508 00:23:57.681 | INFO | RTMetrics report populated | 20260508 00:23:57.681 | INFO | Opening tmp RTMetrics json file | 20260508 00:23:57.707 | INFO | Serializing RTMetrics to json | 20260508 00:23:57.714 | INFO | Serialization done, swapping tmp file now... | 20260508 00:23:57.716 | INFO | RTMetrics done | 20260508 00:23:57.849 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.63 seconds | 20260508 00:23:57.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.21% / 0.00% / 57.78% | 20260508 00:23:57.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00928% / 7.94729e-06% / 5.96046e-05% | 20260508 00:23:57.849 | INFO | Baz IO overall: | 20260508 00:23:57.849 | INFO | BytesWritten : 1716314238374 | 20260508 00:23:57.849 | INFO | NumEvents : 830724403940 | 20260508 00:23:59.249 | INFO | Analyzing chunk frames = [3776512,3777024) | 20260508 00:23:59.249 | INFO | MemInfo: { | 20260508 00:23:59.249 | INFO | "gpu" : " 17 GB", | 20260508 00:23:59.249 | INFO | "huge" : "", | 20260508 00:23:59.249 | INFO | "huge_shared" : " 29 GB", | 20260508 00:23:59.249 | INFO | "malloc" : " 14 GB", | 20260508 00:23:59.249 | INFO | "pinned" : " 29 GB" | 20260508 00:23:59.249 | INFO | } | 20260508 00:23:59.249 | INFO | Perf report for the last 5.14579 seconds: | 20260508 00:23:59.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:23:59.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:59.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:59.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:59.250 | INFO | ANALYSIS : 68.8%, 2.9, 104.5ms, 0 | 20260508 00:23:59.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:23:59.250 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260508 00:23:59.250 | INFO | PRE_HQ : 67.7%, 1.8, 63.4ms, 0 | 20260508 00:23:59.250 | INFO | BAZWRITER : 70.2%, 3.9, 141.4ms, 0 | 20260508 00:23:59.250 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 00:23:59.250 | INFO | Uploaded 36GiB to GPU in 2.56993s (14.0082GiB/s) | 20260508 00:23:59.250 | INFO | Downloaded 38GiB to GPU in 2.70036s (14.0722GiB/s) | 20260508 00:24:04.392 | INFO | Analyzing chunk frames = [3777024,3777536) | 20260508 00:24:04.392 | INFO | MemInfo: { | 20260508 00:24:04.392 | INFO | "gpu" : " 17 GB", | 20260508 00:24:04.392 | INFO | "huge" : "", | 20260508 00:24:04.392 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:04.392 | INFO | "malloc" : " 14 GB", | 20260508 00:24:04.392 | INFO | "pinned" : " 29 GB" | 20260508 00:24:04.392 | INFO | } | 20260508 00:24:04.392 | INFO | Perf report for the last 5.14308 seconds: | 20260508 00:24:04.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:04.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 00:24:04.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:24:04.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:04.393 | INFO | Uploaded 36GiB to GPU in 2.53265s (14.2144GiB/s) | 20260508 00:24:04.393 | INFO | Downloaded 34GiB to GPU in 2.42558s (14.0173GiB/s) | 20260508 00:24:09.526 | INFO | Analyzing chunk frames = [3777536,3778048) | 20260508 00:24:09.526 | INFO | MemInfo: { | 20260508 00:24:09.526 | INFO | "gpu" : " 17 GB", | 20260508 00:24:09.526 | INFO | "huge" : "", | 20260508 00:24:09.526 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:09.526 | INFO | "malloc" : " 14 GB", | 20260508 00:24:09.526 | INFO | "pinned" : " 29 GB" | 20260508 00:24:09.526 | INFO | } | 20260508 00:24:09.526 | INFO | Perf report for the last 5.1337 seconds: | 20260508 00:24:09.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:09.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.526 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 00:24:09.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.526 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 00:24:09.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:09.527 | INFO | Uploaded 36GiB to GPU in 2.54009s (14.1727GiB/s) | 20260508 00:24:09.527 | INFO | Downloaded 34GiB to GPU in 2.48604s (13.6764GiB/s) | 20260508 00:24:14.667 | INFO | Analyzing chunk frames = [3778048,3778560) | 20260508 00:24:14.667 | INFO | MemInfo: { | 20260508 00:24:14.667 | INFO | "gpu" : " 17 GB", | 20260508 00:24:14.667 | INFO | "huge" : "", | 20260508 00:24:14.667 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:14.667 | INFO | "malloc" : " 14 GB", | 20260508 00:24:14.667 | INFO | "pinned" : " 29 GB" | 20260508 00:24:14.667 | INFO | } | 20260508 00:24:14.667 | INFO | Perf report for the last 5.14066 seconds: | 20260508 00:24:14.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:14.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 00:24:14.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:24:14.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:14.667 | INFO | Uploaded 36GiB to GPU in 2.49043s (14.4553GiB/s) | 20260508 00:24:14.667 | INFO | Downloaded 34GiB to GPU in 2.46772s (13.7779GiB/s) | 20260508 00:24:19.805 | INFO | Analyzing chunk frames = [3778560,3779072) | 20260508 00:24:19.805 | INFO | MemInfo: { | 20260508 00:24:19.805 | INFO | "gpu" : " 17 GB", | 20260508 00:24:19.805 | INFO | "huge" : "", | 20260508 00:24:19.805 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:19.805 | INFO | "malloc" : " 14 GB", | 20260508 00:24:19.805 | INFO | "pinned" : " 29 GB" | 20260508 00:24:19.805 | INFO | } | 20260508 00:24:19.805 | INFO | Perf report for the last 5.13853 seconds: | 20260508 00:24:19.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:19.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:24:19.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 00:24:19.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:19.806 | INFO | Uploaded 36GiB to GPU in 2.32528s (15.482GiB/s) | 20260508 00:24:19.806 | INFO | Downloaded 34GiB to GPU in 2.48688s (13.6717GiB/s) | 20260508 00:24:24.944 | INFO | Analyzing chunk frames = [3779072,3779584) | 20260508 00:24:24.944 | INFO | MemInfo: { | 20260508 00:24:24.944 | INFO | "gpu" : " 17 GB", | 20260508 00:24:24.944 | INFO | "huge" : "", | 20260508 00:24:24.944 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:24.944 | INFO | "malloc" : " 14 GB", | 20260508 00:24:24.944 | INFO | "pinned" : " 29 GB" | 20260508 00:24:24.944 | INFO | } | 20260508 00:24:24.944 | INFO | Perf report for the last 5.13908 seconds: | 20260508 00:24:24.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:24.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 00:24:24.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:24:24.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:24.945 | INFO | Uploaded 36GiB to GPU in 2.41918s (14.881GiB/s) | 20260508 00:24:24.945 | INFO | Downloaded 34GiB to GPU in 2.39437s (14.2GiB/s) | 20260508 00:24:30.078 | INFO | Analyzing chunk frames = [3779584,3780096) | 20260508 00:24:30.078 | INFO | MemInfo: { | 20260508 00:24:30.078 | INFO | "gpu" : " 17 GB", | 20260508 00:24:30.078 | INFO | "huge" : "", | 20260508 00:24:30.078 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:30.078 | INFO | "malloc" : " 14 GB", | 20260508 00:24:30.079 | INFO | "pinned" : " 29 GB" | 20260508 00:24:30.079 | INFO | } | 20260508 00:24:30.079 | INFO | Perf report for the last 5.13424 seconds: | 20260508 00:24:30.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:30.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 00:24:30.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 00:24:30.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:30.079 | INFO | Uploaded 36GiB to GPU in 2.56006s (14.0622GiB/s) | 20260508 00:24:30.079 | INFO | Downloaded 34GiB to GPU in 2.45096s (13.8721GiB/s) | 20260508 00:24:35.219 | INFO | Analyzing chunk frames = [3780096,3780608) | 20260508 00:24:35.219 | INFO | MemInfo: { | 20260508 00:24:35.219 | INFO | "gpu" : " 17 GB", | 20260508 00:24:35.219 | INFO | "huge" : "", | 20260508 00:24:35.219 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:35.219 | INFO | "malloc" : " 14 GB", | 20260508 00:24:35.219 | INFO | "pinned" : " 29 GB" | 20260508 00:24:35.219 | INFO | } | 20260508 00:24:35.219 | INFO | Perf report for the last 5.1406 seconds: | 20260508 00:24:35.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:35.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.220 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:24:35.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.220 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:24:35.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:35.220 | INFO | Uploaded 36GiB to GPU in 2.54795s (14.129GiB/s) | 20260508 00:24:35.220 | INFO | Downloaded 34GiB to GPU in 2.44179s (13.9242GiB/s) | 20260508 00:24:35.317 | INFO | Starting a round of Baz IO | 20260508 00:24:38.775 | INFO | Populating RTMetrics report | 20260508 00:24:38.775 | INFO | RTMetrics report populated | 20260508 00:24:38.775 | INFO | Opening tmp RTMetrics json file | 20260508 00:24:38.777 | INFO | Serializing RTMetrics to json | 20260508 00:24:38.797 | INFO | Serialization done, swapping tmp file now... | 20260508 00:24:38.800 | INFO | RTMetrics done | 20260508 00:24:38.875 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.558 seconds | 20260508 00:24:38.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.21% / 0.00% / 57.79% | 20260508 00:24:38.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00778% / 3.97364e-06% / 5.16574e-05% | 20260508 00:24:38.875 | INFO | Baz IO overall: | 20260508 00:24:38.875 | INFO | BytesWritten : 1718128555007 | 20260508 00:24:38.875 | INFO | NumEvents : 831595802376 | 20260508 00:24:40.359 | INFO | Analyzing chunk frames = [3780608,3781120) | 20260508 00:24:40.359 | INFO | MemInfo: { | 20260508 00:24:40.359 | INFO | "gpu" : " 17 GB", | 20260508 00:24:40.359 | INFO | "huge" : "", | 20260508 00:24:40.359 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:40.359 | INFO | "malloc" : " 14 GB", | 20260508 00:24:40.359 | INFO | "pinned" : " 29 GB" | 20260508 00:24:40.359 | INFO | } | 20260508 00:24:40.359 | INFO | Perf report for the last 5.13994 seconds: | 20260508 00:24:40.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:40.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:40.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:40.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:40.359 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 00:24:40.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:40.359 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 00:24:40.359 | INFO | PRE_HQ : 65.5%, 1.7, 56.7ms, 0 | 20260508 00:24:40.359 | INFO | BAZWRITER : 68.8%, 3.8, 134.6ms, 0 | 20260508 00:24:40.359 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 00:24:40.360 | INFO | Uploaded 36GiB to GPU in 2.56569s (14.0313GiB/s) | 20260508 00:24:40.360 | INFO | Downloaded 38GiB to GPU in 2.69258s (14.1129GiB/s) | 20260508 00:24:45.501 | INFO | Analyzing chunk frames = [3781120,3781632) | 20260508 00:24:45.501 | INFO | MemInfo: { | 20260508 00:24:45.501 | INFO | "gpu" : " 17 GB", | 20260508 00:24:45.501 | INFO | "huge" : "", | 20260508 00:24:45.501 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:45.501 | INFO | "malloc" : " 14 GB", | 20260508 00:24:45.501 | INFO | "pinned" : " 29 GB" | 20260508 00:24:45.501 | INFO | } | 20260508 00:24:45.501 | INFO | Perf report for the last 5.14211 seconds: | 20260508 00:24:45.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:45.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.501 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:24:45.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.501 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:24:45.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:45.502 | INFO | Uploaded 36GiB to GPU in 2.58983s (13.9005GiB/s) | 20260508 00:24:45.502 | INFO | Downloaded 34GiB to GPU in 2.47391s (13.7434GiB/s) | 20260508 00:24:50.640 | INFO | Analyzing chunk frames = [3781632,3782144) | 20260508 00:24:50.640 | INFO | MemInfo: { | 20260508 00:24:50.640 | INFO | "gpu" : " 17 GB", | 20260508 00:24:50.640 | INFO | "huge" : "", | 20260508 00:24:50.640 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:50.640 | INFO | "malloc" : " 14 GB", | 20260508 00:24:50.640 | INFO | "pinned" : " 29 GB" | 20260508 00:24:50.640 | INFO | } | 20260508 00:24:50.640 | INFO | Perf report for the last 5.13873 seconds: | 20260508 00:24:50.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:50.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:24:50.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:24:50.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:50.640 | INFO | Uploaded 36GiB to GPU in 2.56091s (14.0575GiB/s) | 20260508 00:24:50.640 | INFO | Downloaded 34GiB to GPU in 2.47398s (13.743GiB/s) | 20260508 00:24:55.774 | INFO | Analyzing chunk frames = [3782144,3782656) | 20260508 00:24:55.774 | INFO | MemInfo: { | 20260508 00:24:55.774 | INFO | "gpu" : " 17 GB", | 20260508 00:24:55.774 | INFO | "huge" : "", | 20260508 00:24:55.774 | INFO | "huge_shared" : " 29 GB", | 20260508 00:24:55.774 | INFO | "malloc" : " 14 GB", | 20260508 00:24:55.774 | INFO | "pinned" : " 29 GB" | 20260508 00:24:55.774 | INFO | } | 20260508 00:24:55.774 | INFO | Perf report for the last 5.13417 seconds: | 20260508 00:24:55.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:24:55.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:24:55.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 00:24:55.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:24:55.775 | INFO | Uploaded 36GiB to GPU in 2.5502s (14.1166GiB/s) | 20260508 00:24:55.775 | INFO | Downloaded 34GiB to GPU in 2.46569s (13.7893GiB/s) | 20260508 00:25:00.920 | INFO | Analyzing chunk frames = [3782656,3783168) | 20260508 00:25:00.920 | INFO | MemInfo: { | 20260508 00:25:00.920 | INFO | "gpu" : " 17 GB", | 20260508 00:25:00.920 | INFO | "huge" : "", | 20260508 00:25:00.920 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:00.920 | INFO | "malloc" : " 14 GB", | 20260508 00:25:00.920 | INFO | "pinned" : " 29 GB" | 20260508 00:25:00.920 | INFO | } | 20260508 00:25:00.920 | INFO | Perf report for the last 5.14563 seconds: | 20260508 00:25:00.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:00.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.920 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 00:25:00.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.920 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:25:00.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:00.921 | INFO | Uploaded 36GiB to GPU in 2.40125s (14.9922GiB/s) | 20260508 00:25:00.921 | INFO | Downloaded 34GiB to GPU in 2.4953s (13.6256GiB/s) | 20260508 00:25:06.061 | INFO | Analyzing chunk frames = [3783168,3783680) | 20260508 00:25:06.061 | INFO | MemInfo: { | 20260508 00:25:06.061 | INFO | "gpu" : " 17 GB", | 20260508 00:25:06.061 | INFO | "huge" : "", | 20260508 00:25:06.061 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:06.061 | INFO | "malloc" : " 14 GB", | 20260508 00:25:06.061 | INFO | "pinned" : " 29 GB" | 20260508 00:25:06.061 | INFO | } | 20260508 00:25:06.061 | INFO | Perf report for the last 5.1414 seconds: | 20260508 00:25:06.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:06.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.061 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 00:25:06.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.062 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260508 00:25:06.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:06.062 | INFO | Uploaded 36GiB to GPU in 2.39768s (15.0145GiB/s) | 20260508 00:25:06.062 | INFO | Downloaded 34GiB to GPU in 2.39169s (14.2159GiB/s) | 20260508 00:25:11.202 | INFO | Analyzing chunk frames = [3783680,3784192) | 20260508 00:25:11.202 | INFO | MemInfo: { | 20260508 00:25:11.202 | INFO | "gpu" : " 17 GB", | 20260508 00:25:11.202 | INFO | "huge" : "", | 20260508 00:25:11.202 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:11.202 | INFO | "malloc" : " 14 GB", | 20260508 00:25:11.202 | INFO | "pinned" : " 29 GB" | 20260508 00:25:11.202 | INFO | } | 20260508 00:25:11.202 | INFO | Perf report for the last 5.14092 seconds: | 20260508 00:25:11.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:11.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.202 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 00:25:11.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.202 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:25:11.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:11.203 | INFO | Uploaded 36GiB to GPU in 2.4687s (14.5826GiB/s) | 20260508 00:25:11.203 | INFO | Downloaded 34GiB to GPU in 2.43163s (13.9824GiB/s) | 20260508 00:25:16.340 | INFO | Analyzing chunk frames = [3784192,3784704) | 20260508 00:25:16.340 | INFO | MemInfo: { | 20260508 00:25:16.340 | INFO | "gpu" : " 17 GB", | 20260508 00:25:16.340 | INFO | "huge" : "", | 20260508 00:25:16.340 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:16.340 | INFO | "malloc" : " 14 GB", | 20260508 00:25:16.340 | INFO | "pinned" : " 29 GB" | 20260508 00:25:16.340 | INFO | } | 20260508 00:25:16.340 | INFO | Perf report for the last 5.1374 seconds: | 20260508 00:25:16.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:16.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 00:25:16.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 00:25:16.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:16.340 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:25:16.340 | INFO | Total mem usage: 16.7 GiB | 20260508 00:25:16.340 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:25:16.340 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 00:25:16.340 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:25:16.340 | INFO | High Water Breakdown: | 20260508 00:25:16.340 | INFO | 59.7% : 8.72 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:25:16.340 | INFO | 40.2% : 5.87 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:25:16.340 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:25:16.340 | INFO | ------------------------------------------ | 20260508 00:25:16.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:25:16.340 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:25:16.340 | INFO | Total mem usage: 26.5 GiB | 20260508 00:25:16.340 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:25:16.340 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260508 00:25:16.340 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260508 00:25:16.340 | INFO | High Water Breakdown: | 20260508 00:25:16.340 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:25:16.340 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:25:16.340 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:25:16.340 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:25:16.340 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:25:16.340 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:25:16.340 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:25:16.340 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:25:16.340 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:25:16.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:25:16.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:25:16.340 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:25:16.340 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:25:16.340 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:25:16.340 | INFO | ------------------------------------------ | 20260508 00:25:16.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:25:16.340 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:25:16.340 | INFO | Total mem usage: 27.8 GiB | 20260508 00:25:16.340 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:25:16.340 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:25:16.340 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:25:16.340 | INFO | High Water Breakdown: | 20260508 00:25:16.340 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:25:16.340 | INFO | ------------------------------------------ | 20260508 00:25:16.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:25:16.340 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:25:16.340 | INFO | Total mem usage: 15.7 GiB | 20260508 00:25:16.340 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:25:16.340 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:25:16.340 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:25:16.340 | INFO | High Water Breakdown: | 20260508 00:25:16.340 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:25:16.340 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:25:16.340 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:25:16.340 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:25:16.340 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:25:16.340 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:25:16.340 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:25:16.340 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:25:16.340 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:25:16.340 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:25:16.340 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:25:16.340 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:25:16.340 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:25:16.340 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:25:16.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:25:16.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:25:16.340 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:25:16.340 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:25:16.340 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:25:16.340 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:25:16.340 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:25:16.340 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:25:16.340 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:25:16.340 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:25:16.340 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:25:16.340 | INFO | ------------------------------------------ | 20260508 00:25:16.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:25:16.340 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:25:16.340 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:25:16.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:25:16.340 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.40e+00 | 20260508 00:25:16.340 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.00e-01, 1.17e-02 / 5.11e-03 | 20260508 00:25:16.340 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.50e+00, 5.88e-01 / 1.53e-01 | 20260508 00:25:16.340 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.51e+00, 2.08e-01 / 9.54e-02 | 20260508 00:25:16.341 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.60e+01, 3.83e+00 / 1.56e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.61e+00 / 1.43e+00 | 20260508 00:25:16.341 | INFO | Uploaded 36GiB to GPU in 2.55455s (14.0925GiB/s) | 20260508 00:25:16.341 | INFO | Downloaded 34GiB to GPU in 2.45045s (13.875GiB/s) | 20260508 00:25:16.341 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.27e-01 / 2.11e-01 | 20260508 00:25:16.341 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.32e+00, 2.78e-01 / 1.13e-01 | 20260508 00:25:16.341 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.81e-01, 5.46e-02 / 2.68e-02 | 20260508 00:25:16.341 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:25:16.341 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:25:16.341 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:25:16.341 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 30.74%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 2.36%, LockUpload -- 2.52e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 2.60%, LockDownload -- 2.77e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:25:16.341 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260508 00:25:16.438 | INFO | Starting a round of Baz IO | 20260508 00:25:19.887 | INFO | Populating RTMetrics report | 20260508 00:25:19.887 | INFO | RTMetrics report populated | 20260508 00:25:19.887 | INFO | Opening tmp RTMetrics json file | 20260508 00:25:19.910 | INFO | Serializing RTMetrics to json | 20260508 00:25:19.912 | INFO | Serialization done, swapping tmp file now... | 20260508 00:25:19.912 | INFO | RTMetrics done | 20260508 00:25:19.976 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.538 seconds | 20260508 00:25:19.976 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.20% / 0.00% / 57.79% | 20260508 00:25:19.976 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00926% / 0% / 6.75519e-05% | 20260508 00:25:19.976 | INFO | Baz IO overall: | 20260508 00:25:19.976 | INFO | BytesWritten : 1719942890878 | 20260508 00:25:19.976 | INFO | NumEvents : 832468012398 | 20260508 00:25:21.473 | INFO | Analyzing chunk frames = [3784704,3785216) | 20260508 00:25:21.473 | INFO | MemInfo: { | 20260508 00:25:21.473 | INFO | "gpu" : " 17 GB", | 20260508 00:25:21.473 | INFO | "huge" : "", | 20260508 00:25:21.473 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:21.473 | INFO | "malloc" : " 14 GB", | 20260508 00:25:21.473 | INFO | "pinned" : " 29 GB" | 20260508 00:25:21.473 | INFO | } | 20260508 00:25:21.474 | INFO | Perf report for the last 5.13379 seconds: | 20260508 00:25:21.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:21.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:21.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:21.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:21.474 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260508 00:25:21.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:21.474 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 00:25:21.474 | INFO | PRE_HQ : 66.0%, 1.7, 56.6ms, 0 | 20260508 00:25:21.474 | INFO | BAZWRITER : 68.5%, 3.8, 133.7ms, 0 | 20260508 00:25:21.474 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:25:21.474 | INFO | Uploaded 36GiB to GPU in 2.52434s (14.2611GiB/s) | 20260508 00:25:21.474 | INFO | Downloaded 38GiB to GPU in 2.70453s (14.0505GiB/s) | 20260508 00:25:26.617 | INFO | Analyzing chunk frames = [3785216,3785728) | 20260508 00:25:26.617 | INFO | MemInfo: { | 20260508 00:25:26.617 | INFO | "gpu" : " 17 GB", | 20260508 00:25:26.617 | INFO | "huge" : "", | 20260508 00:25:26.617 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:26.617 | INFO | "malloc" : " 14 GB", | 20260508 00:25:26.617 | INFO | "pinned" : " 29 GB" | 20260508 00:25:26.617 | INFO | } | 20260508 00:25:26.617 | INFO | Perf report for the last 5.14332 seconds: | 20260508 00:25:26.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:26.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 00:25:26.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:25:26.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:26.617 | INFO | Uploaded 36GiB to GPU in 2.56399s (14.0406GiB/s) | 20260508 00:25:26.617 | INFO | Downloaded 34GiB to GPU in 2.487s (13.6711GiB/s) | 20260508 00:25:31.753 | INFO | Analyzing chunk frames = [3785728,3786240) | 20260508 00:25:31.753 | INFO | MemInfo: { | 20260508 00:25:31.753 | INFO | "gpu" : " 17 GB", | 20260508 00:25:31.753 | INFO | "huge" : "", | 20260508 00:25:31.753 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:31.753 | INFO | "malloc" : " 14 GB", | 20260508 00:25:31.753 | INFO | "pinned" : " 29 GB" | 20260508 00:25:31.753 | INFO | } | 20260508 00:25:31.753 | INFO | Perf report for the last 5.13636 seconds: | 20260508 00:25:31.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:31.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.753 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:25:31.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.753 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:25:31.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:31.754 | INFO | Uploaded 36GiB to GPU in 2.50502s (14.3711GiB/s) | 20260508 00:25:31.754 | INFO | Downloaded 34GiB to GPU in 2.48829s (13.664GiB/s) | 20260508 00:25:36.891 | INFO | Analyzing chunk frames = [3786240,3786752) | 20260508 00:25:36.891 | INFO | MemInfo: { | 20260508 00:25:36.891 | INFO | "gpu" : " 17 GB", | 20260508 00:25:36.891 | INFO | "huge" : "", | 20260508 00:25:36.891 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:36.891 | INFO | "malloc" : " 14 GB", | 20260508 00:25:36.891 | INFO | "pinned" : " 29 GB" | 20260508 00:25:36.891 | INFO | } | 20260508 00:25:36.891 | INFO | Perf report for the last 5.13782 seconds: | 20260508 00:25:36.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:36.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 00:25:36.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:25:36.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:36.891 | INFO | Uploaded 36GiB to GPU in 2.52357s (14.2655GiB/s) | 20260508 00:25:36.892 | INFO | Downloaded 34GiB to GPU in 2.47149s (13.7569GiB/s) | 20260508 00:25:42.031 | INFO | Analyzing chunk frames = [3786752,3787264) | 20260508 00:25:42.031 | INFO | MemInfo: { | 20260508 00:25:42.031 | INFO | "gpu" : " 17 GB", | 20260508 00:25:42.031 | INFO | "huge" : "", | 20260508 00:25:42.031 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:42.031 | INFO | "malloc" : " 14 GB", | 20260508 00:25:42.031 | INFO | "pinned" : " 29 GB" | 20260508 00:25:42.032 | INFO | } | 20260508 00:25:42.032 | INFO | Perf report for the last 5.14051 seconds: | 20260508 00:25:42.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:42.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 00:25:42.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260508 00:25:42.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:42.032 | INFO | Uploaded 36GiB to GPU in 2.39915s (15.0053GiB/s) | 20260508 00:25:42.032 | INFO | Downloaded 34GiB to GPU in 2.49205s (13.6434GiB/s) | 20260508 00:25:47.169 | INFO | Analyzing chunk frames = [3787264,3787776) | 20260508 00:25:47.169 | INFO | MemInfo: { | 20260508 00:25:47.169 | INFO | "gpu" : " 17 GB", | 20260508 00:25:47.169 | INFO | "huge" : "", | 20260508 00:25:47.169 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:47.169 | INFO | "malloc" : " 14 GB", | 20260508 00:25:47.169 | INFO | "pinned" : " 29 GB" | 20260508 00:25:47.169 | INFO | } | 20260508 00:25:47.169 | INFO | Perf report for the last 5.13743 seconds: | 20260508 00:25:47.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:47.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 00:25:47.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:25:47.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:47.169 | INFO | Uploaded 36GiB to GPU in 2.43347s (14.7937GiB/s) | 20260508 00:25:47.169 | INFO | Downloaded 34GiB to GPU in 2.41116s (14.1011GiB/s) | 20260508 00:25:52.309 | INFO | Analyzing chunk frames = [3787776,3788288) | 20260508 00:25:52.309 | INFO | MemInfo: { | 20260508 00:25:52.309 | INFO | "gpu" : " 17 GB", | 20260508 00:25:52.309 | INFO | "huge" : "", | 20260508 00:25:52.309 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:52.309 | INFO | "malloc" : " 14 GB", | 20260508 00:25:52.309 | INFO | "pinned" : " 29 GB" | 20260508 00:25:52.309 | INFO | } | 20260508 00:25:52.309 | INFO | Perf report for the last 5.14016 seconds: | 20260508 00:25:52.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:52.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.309 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:25:52.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.309 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:25:52.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:52.310 | INFO | Uploaded 36GiB to GPU in 2.52729s (14.2445GiB/s) | 20260508 00:25:52.310 | INFO | Downloaded 34GiB to GPU in 2.43664s (13.9536GiB/s) | 20260508 00:25:57.454 | INFO | Analyzing chunk frames = [3788288,3788800) | 20260508 00:25:57.454 | INFO | MemInfo: { | 20260508 00:25:57.454 | INFO | "gpu" : " 17 GB", | 20260508 00:25:57.454 | INFO | "huge" : "", | 20260508 00:25:57.454 | INFO | "huge_shared" : " 29 GB", | 20260508 00:25:57.454 | INFO | "malloc" : " 14 GB", | 20260508 00:25:57.454 | INFO | "pinned" : " 29 GB" | 20260508 00:25:57.454 | INFO | } | 20260508 00:25:57.454 | INFO | Perf report for the last 5.14517 seconds: | 20260508 00:25:57.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:25:57.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.455 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 00:25:57.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.455 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:25:57.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:25:57.455 | INFO | Uploaded 36GiB to GPU in 2.54998s (14.1178GiB/s) | 20260508 00:25:57.455 | INFO | Downloaded 34GiB to GPU in 2.4445s (13.9088GiB/s) | 20260508 00:25:57.552 | INFO | Starting a round of Baz IO | 20260508 00:26:01.009 | INFO | Populating RTMetrics report | 20260508 00:26:01.009 | INFO | RTMetrics report populated | 20260508 00:26:01.009 | INFO | Opening tmp RTMetrics json file | 20260508 00:26:01.105 | INFO | Serializing RTMetrics to json | 20260508 00:26:01.169 | INFO | Serialization done, swapping tmp file now... | 20260508 00:26:01.177 | INFO | RTMetrics done | 20260508 00:26:01.354 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.802 seconds | 20260508 00:26:01.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.20% / 0.00% / 57.80% | 20260508 00:26:01.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00679% / 7.94729e-06% / 5.96046e-05% | 20260508 00:26:01.354 | INFO | Baz IO overall: | 20260508 00:26:01.354 | INFO | BytesWritten : 1721757463097 | 20260508 00:26:01.354 | INFO | NumEvents : 833340580500 | 20260508 00:26:02.591 | INFO | Analyzing chunk frames = [3788800,3789312) | 20260508 00:26:02.591 | INFO | MemInfo: { | 20260508 00:26:02.591 | INFO | "gpu" : " 17 GB", | 20260508 00:26:02.591 | INFO | "huge" : "", | 20260508 00:26:02.591 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:02.591 | INFO | "malloc" : " 14 GB", | 20260508 00:26:02.591 | INFO | "pinned" : " 29 GB" | 20260508 00:26:02.591 | INFO | } | 20260508 00:26:02.591 | INFO | Perf report for the last 5.137 seconds: | 20260508 00:26:02.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:02.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:02.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:02.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:02.591 | INFO | ANALYSIS : 68.5%, 3.0, 106.9ms, 0 | 20260508 00:26:02.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:02.592 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 00:26:02.592 | INFO | PRE_HQ : 65.6%, 1.7, 56.6ms, 0 | 20260508 00:26:02.592 | INFO | BAZWRITER : 73.5%, 4.7, 176.6ms, 0 | 20260508 00:26:02.592 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260508 00:26:02.592 | INFO | Uploaded 36GiB to GPU in 2.55794s (14.0738GiB/s) | 20260508 00:26:02.592 | INFO | Downloaded 38GiB to GPU in 2.69072s (14.1226GiB/s) | 20260508 00:26:07.729 | INFO | Analyzing chunk frames = [3789312,3789824) | 20260508 00:26:07.729 | INFO | MemInfo: { | 20260508 00:26:07.729 | INFO | "gpu" : " 17 GB", | 20260508 00:26:07.729 | INFO | "huge" : "", | 20260508 00:26:07.729 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:07.729 | INFO | "malloc" : " 14 GB", | 20260508 00:26:07.729 | INFO | "pinned" : " 29 GB" | 20260508 00:26:07.729 | INFO | } | 20260508 00:26:07.730 | INFO | Perf report for the last 5.13811 seconds: | 20260508 00:26:07.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:07.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:26:07.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:26:07.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:07.730 | INFO | Uploaded 36GiB to GPU in 2.55022s (14.1164GiB/s) | 20260508 00:26:07.730 | INFO | Downloaded 34GiB to GPU in 2.41522s (14.0774GiB/s) | 20260508 00:26:12.870 | INFO | Analyzing chunk frames = [3789824,3790336) | 20260508 00:26:12.870 | INFO | MemInfo: { | 20260508 00:26:12.870 | INFO | "gpu" : " 17 GB", | 20260508 00:26:12.870 | INFO | "huge" : "", | 20260508 00:26:12.870 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:12.870 | INFO | "malloc" : " 14 GB", | 20260508 00:26:12.870 | INFO | "pinned" : " 29 GB" | 20260508 00:26:12.870 | INFO | } | 20260508 00:26:12.870 | INFO | Perf report for the last 5.14079 seconds: | 20260508 00:26:12.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:12.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.871 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 00:26:12.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.871 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 00:26:12.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:12.871 | INFO | Uploaded 36GiB to GPU in 2.55284s (14.1019GiB/s) | 20260508 00:26:12.871 | INFO | Downloaded 34GiB to GPU in 2.47161s (13.7562GiB/s) | 20260508 00:26:18.007 | INFO | Analyzing chunk frames = [3790336,3790848) | 20260508 00:26:18.007 | INFO | MemInfo: { | 20260508 00:26:18.007 | INFO | "gpu" : " 17 GB", | 20260508 00:26:18.007 | INFO | "huge" : "", | 20260508 00:26:18.007 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:18.007 | INFO | "malloc" : " 14 GB", | 20260508 00:26:18.007 | INFO | "pinned" : " 29 GB" | 20260508 00:26:18.007 | INFO | } | 20260508 00:26:18.007 | INFO | Perf report for the last 5.13669 seconds: | 20260508 00:26:18.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:18.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.007 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:26:18.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.007 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:26:18.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:18.008 | INFO | Uploaded 36GiB to GPU in 2.5522s (14.1055GiB/s) | 20260508 00:26:18.008 | INFO | Downloaded 34GiB to GPU in 2.47198s (13.7541GiB/s) | 20260508 00:26:23.148 | INFO | Analyzing chunk frames = [3790848,3791360) | 20260508 00:26:23.148 | INFO | MemInfo: { | 20260508 00:26:23.148 | INFO | "gpu" : " 17 GB", | 20260508 00:26:23.148 | INFO | "huge" : "", | 20260508 00:26:23.148 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:23.148 | INFO | "malloc" : " 14 GB", | 20260508 00:26:23.148 | INFO | "pinned" : " 29 GB" | 20260508 00:26:23.148 | INFO | } | 20260508 00:26:23.148 | INFO | Perf report for the last 5.14089 seconds: | 20260508 00:26:23.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:23.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 00:26:23.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:26:23.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:23.148 | INFO | Uploaded 36GiB to GPU in 2.40284s (14.9823GiB/s) | 20260508 00:26:23.148 | INFO | Downloaded 34GiB to GPU in 2.47018s (13.7642GiB/s) | 20260508 00:26:28.286 | INFO | Analyzing chunk frames = [3791360,3791872) | 20260508 00:26:28.286 | INFO | MemInfo: { | 20260508 00:26:28.286 | INFO | "gpu" : " 17 GB", | 20260508 00:26:28.286 | INFO | "huge" : "", | 20260508 00:26:28.286 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:28.286 | INFO | "malloc" : " 14 GB", | 20260508 00:26:28.286 | INFO | "pinned" : " 29 GB" | 20260508 00:26:28.286 | INFO | } | 20260508 00:26:28.286 | INFO | Perf report for the last 5.13767 seconds: | 20260508 00:26:28.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:28.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:26:28.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:26:28.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:28.286 | INFO | Uploaded 36GiB to GPU in 2.4201s (14.8754GiB/s) | 20260508 00:26:28.287 | INFO | Downloaded 34GiB to GPU in 2.39189s (14.2147GiB/s) | 20260508 00:26:33.425 | INFO | Analyzing chunk frames = [3791872,3792384) | 20260508 00:26:33.425 | INFO | MemInfo: { | 20260508 00:26:33.425 | INFO | "gpu" : " 17 GB", | 20260508 00:26:33.425 | INFO | "huge" : "", | 20260508 00:26:33.425 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:33.425 | INFO | "malloc" : " 14 GB", | 20260508 00:26:33.425 | INFO | "pinned" : " 29 GB" | 20260508 00:26:33.425 | INFO | } | 20260508 00:26:33.426 | INFO | Perf report for the last 5.13988 seconds: | 20260508 00:26:33.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:33.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 00:26:33.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:26:33.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:33.426 | INFO | Uploaded 36GiB to GPU in 2.5607s (14.0587GiB/s) | 20260508 00:26:33.426 | INFO | Downloaded 34GiB to GPU in 2.44837s (13.8868GiB/s) | 20260508 00:26:38.567 | INFO | Analyzing chunk frames = [3792384,3792896) | 20260508 00:26:38.567 | INFO | MemInfo: { | 20260508 00:26:38.567 | INFO | "gpu" : " 17 GB", | 20260508 00:26:38.567 | INFO | "huge" : "", | 20260508 00:26:38.567 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:38.567 | INFO | "malloc" : " 14 GB", | 20260508 00:26:38.567 | INFO | "pinned" : " 29 GB" | 20260508 00:26:38.568 | INFO | } | 20260508 00:26:38.568 | INFO | Perf report for the last 5.14205 seconds: | 20260508 00:26:38.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:38.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:26:38.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:26:38.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:38.568 | INFO | Uploaded 36GiB to GPU in 2.5197s (14.2874GiB/s) | 20260508 00:26:38.568 | INFO | Downloaded 34GiB to GPU in 2.43958s (13.9368GiB/s) | 20260508 00:26:38.568 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:26:38.568 | INFO | Total mem usage: 16.7 GiB | 20260508 00:26:38.568 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:26:38.568 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.6%) | 20260508 00:26:38.568 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:26:38.568 | INFO | High Water Breakdown: | 20260508 00:26:38.568 | INFO | 60.4% : 9.19 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:26:38.568 | INFO | 39.5% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:26:38.568 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:26:38.568 | INFO | ------------------------------------------ | 20260508 00:26:38.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:26:38.568 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:26:38.568 | INFO | Total mem usage: 26.5 GiB | 20260508 00:26:38.568 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:26:38.568 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:26:38.568 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:26:38.568 | INFO | High Water Breakdown: | 20260508 00:26:38.568 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:26:38.568 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:26:38.568 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:26:38.568 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:26:38.568 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:26:38.568 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:26:38.568 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:26:38.568 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:26:38.568 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:26:38.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:26:38.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:26:38.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:26:38.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:26:38.568 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:26:38.568 | INFO | ------------------------------------------ | 20260508 00:26:38.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:26:38.568 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:26:38.568 | INFO | Total mem usage: 27.8 GiB | 20260508 00:26:38.568 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:26:38.568 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:26:38.568 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:26:38.568 | INFO | High Water Breakdown: | 20260508 00:26:38.568 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:26:38.568 | INFO | ------------------------------------------ | 20260508 00:26:38.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:26:38.568 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:26:38.568 | INFO | Total mem usage: 15.7 GiB | 20260508 00:26:38.568 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:26:38.568 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:26:38.568 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:26:38.568 | INFO | High Water Breakdown: | 20260508 00:26:38.568 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:26:38.569 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:26:38.569 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:26:38.569 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:26:38.569 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:26:38.569 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:26:38.569 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:26:38.569 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:26:38.569 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:26:38.569 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:26:38.569 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:26:38.569 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:26:38.569 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:26:38.569 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:26:38.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:26:38.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:26:38.569 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:26:38.569 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:26:38.569 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:26:38.569 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:26:38.569 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:26:38.569 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:26:38.569 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:26:38.569 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:26:38.569 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:26:38.569 | INFO | ------------------------------------------ | 20260508 00:26:38.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:26:38.569 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:26:38.569 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:26:38.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:26:38.569 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.42e+01 / 1.46e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.20e-02 / 6.95e-03 | 20260508 00:26:38.569 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.44e+00, 5.83e-01 / 2.04e-01 | 20260508 00:26:38.569 | INFO | PerfReport: 3.34%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.15e-01 | 20260508 00:26:38.569 | INFO | PerfReport: 16.11%, DME -- 4.22e+01 / 4.62e+01, 4.05e+00 / 1.77e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.61e+00 / 1.46e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.24e-01 / 1.60e-01 | 20260508 00:26:38.569 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.32e+00, 2.75e-01 / 1.28e-01 | 20260508 00:26:38.569 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.80e-01, 5.46e-02 / 2.52e-02 | 20260508 00:26:38.569 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:26:38.569 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:26:38.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:26:38.569 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 24.21%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 30.65%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 2.41%, LockUpload -- 2.58e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 2.58%, LockDownload -- 2.76e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 15.68%, LockCompute -- 1.68e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 00:26:38.569 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 4.02e-02 / 0.00e+00 | 20260508 00:26:38.665 | INFO | Starting a round of Baz IO | 20260508 00:26:42.120 | INFO | Populating RTMetrics report | 20260508 00:26:42.120 | INFO | RTMetrics report populated | 20260508 00:26:42.120 | INFO | Opening tmp RTMetrics json file | 20260508 00:26:42.151 | INFO | Serializing RTMetrics to json | 20260508 00:26:42.222 | INFO | Serialization done, swapping tmp file now... | 20260508 00:26:42.228 | INFO | RTMetrics done | 20260508 00:26:42.262 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.597 seconds | 20260508 00:26:42.262 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.19% / 0.00% / 57.80% | 20260508 00:26:42.262 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00826% / 0% / 7.15256e-05% | 20260508 00:26:42.262 | INFO | Baz IO overall: | 20260508 00:26:42.262 | INFO | BytesWritten : 1723572557589 | 20260508 00:26:42.262 | INFO | NumEvents : 834213763560 | 20260508 00:26:43.708 | INFO | Analyzing chunk frames = [3792896,3793408) | 20260508 00:26:43.708 | INFO | MemInfo: { | 20260508 00:26:43.708 | INFO | "gpu" : " 17 GB", | 20260508 00:26:43.708 | INFO | "huge" : "", | 20260508 00:26:43.708 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:43.708 | INFO | "malloc" : " 14 GB", | 20260508 00:26:43.708 | INFO | "pinned" : " 29 GB" | 20260508 00:26:43.708 | INFO | } | 20260508 00:26:43.708 | INFO | Perf report for the last 5.14032 seconds: | 20260508 00:26:43.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:43.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:43.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:43.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:43.708 | INFO | ANALYSIS : 68.4%, 3.0, 106.8ms, 0 | 20260508 00:26:43.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:43.708 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260508 00:26:43.708 | INFO | PRE_HQ : 65.4%, 1.7, 56.6ms, 0 | 20260508 00:26:43.708 | INFO | BAZWRITER : 69.6%, 3.4, 122.8ms, 0 | 20260508 00:26:43.708 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260508 00:26:43.709 | INFO | Uploaded 36GiB to GPU in 2.55658s (14.0813GiB/s) | 20260508 00:26:43.709 | INFO | Downloaded 38GiB to GPU in 2.69313s (14.11GiB/s) | 20260508 00:26:48.842 | INFO | Analyzing chunk frames = [3793408,3793920) | 20260508 00:26:48.842 | INFO | MemInfo: { | 20260508 00:26:48.842 | INFO | "gpu" : " 17 GB", | 20260508 00:26:48.842 | INFO | "huge" : "", | 20260508 00:26:48.842 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:48.842 | INFO | "malloc" : " 14 GB", | 20260508 00:26:48.842 | INFO | "pinned" : " 29 GB" | 20260508 00:26:48.842 | INFO | } | 20260508 00:26:48.842 | INFO | Perf report for the last 5.13392 seconds: | 20260508 00:26:48.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:48.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 00:26:48.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260508 00:26:48.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:48.842 | INFO | Uploaded 36GiB to GPU in 2.55034s (14.1157GiB/s) | 20260508 00:26:48.842 | INFO | Downloaded 34GiB to GPU in 2.4176s (14.0635GiB/s) | 20260508 00:26:53.982 | INFO | Analyzing chunk frames = [3793920,3794432) | 20260508 00:26:53.982 | INFO | MemInfo: { | 20260508 00:26:53.982 | INFO | "gpu" : " 17 GB", | 20260508 00:26:53.982 | INFO | "huge" : "", | 20260508 00:26:53.982 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:53.982 | INFO | "malloc" : " 14 GB", | 20260508 00:26:53.982 | INFO | "pinned" : " 29 GB" | 20260508 00:26:53.982 | INFO | } | 20260508 00:26:53.982 | INFO | Perf report for the last 5.14011 seconds: | 20260508 00:26:53.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:53.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 00:26:53.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:26:53.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:53.982 | INFO | Uploaded 36GiB to GPU in 2.55036s (14.1156GiB/s) | 20260508 00:26:53.982 | INFO | Downloaded 34GiB to GPU in 2.46977s (13.7664GiB/s) | 20260508 00:26:59.122 | INFO | Analyzing chunk frames = [3794432,3794944) | 20260508 00:26:59.122 | INFO | MemInfo: { | 20260508 00:26:59.122 | INFO | "gpu" : " 17 GB", | 20260508 00:26:59.122 | INFO | "huge" : "", | 20260508 00:26:59.122 | INFO | "huge_shared" : " 29 GB", | 20260508 00:26:59.122 | INFO | "malloc" : " 14 GB", | 20260508 00:26:59.122 | INFO | "pinned" : " 29 GB" | 20260508 00:26:59.122 | INFO | } | 20260508 00:26:59.122 | INFO | Perf report for the last 5.14041 seconds: | 20260508 00:26:59.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:26:59.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.122 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 00:26:59.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.122 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:26:59.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:26:59.123 | INFO | Uploaded 36GiB to GPU in 2.53755s (14.1869GiB/s) | 20260508 00:26:59.123 | INFO | Downloaded 34GiB to GPU in 2.47689s (13.7269GiB/s) | 20260508 00:27:04.262 | INFO | Analyzing chunk frames = [3794944,3795456) | 20260508 00:27:04.262 | INFO | MemInfo: { | 20260508 00:27:04.262 | INFO | "gpu" : " 17 GB", | 20260508 00:27:04.262 | INFO | "huge" : "", | 20260508 00:27:04.262 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:04.262 | INFO | "malloc" : " 14 GB", | 20260508 00:27:04.262 | INFO | "pinned" : " 29 GB" | 20260508 00:27:04.262 | INFO | } | 20260508 00:27:04.262 | INFO | Perf report for the last 5.13994 seconds: | 20260508 00:27:04.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:04.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.262 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:27:04.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.262 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 00:27:04.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:04.263 | INFO | Uploaded 36GiB to GPU in 2.41058s (14.9342GiB/s) | 20260508 00:27:04.263 | INFO | Downloaded 34GiB to GPU in 2.47455s (13.7399GiB/s) | 20260508 00:27:09.404 | INFO | Analyzing chunk frames = [3795456,3795968) | 20260508 00:27:09.404 | INFO | MemInfo: { | 20260508 00:27:09.404 | INFO | "gpu" : " 17 GB", | 20260508 00:27:09.404 | INFO | "huge" : "", | 20260508 00:27:09.404 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:09.404 | INFO | "malloc" : " 14 GB", | 20260508 00:27:09.404 | INFO | "pinned" : " 29 GB" | 20260508 00:27:09.404 | INFO | } | 20260508 00:27:09.404 | INFO | Perf report for the last 5.14174 seconds: | 20260508 00:27:09.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:09.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 00:27:09.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:27:09.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:09.404 | INFO | Uploaded 36GiB to GPU in 2.4362s (14.7771GiB/s) | 20260508 00:27:09.404 | INFO | Downloaded 34GiB to GPU in 2.39307s (14.2077GiB/s) | 20260508 00:27:14.537 | INFO | Analyzing chunk frames = [3795968,3796480) | 20260508 00:27:14.537 | INFO | MemInfo: { | 20260508 00:27:14.537 | INFO | "gpu" : " 17 GB", | 20260508 00:27:14.537 | INFO | "huge" : "", | 20260508 00:27:14.537 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:14.537 | INFO | "malloc" : " 14 GB", | 20260508 00:27:14.537 | INFO | "pinned" : " 29 GB" | 20260508 00:27:14.537 | INFO | } | 20260508 00:27:14.537 | INFO | Perf report for the last 5.13296 seconds: | 20260508 00:27:14.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:14.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.537 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 00:27:14.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.537 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:27:14.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:14.538 | INFO | Uploaded 36GiB to GPU in 2.44865s (14.702GiB/s) | 20260508 00:27:14.538 | INFO | Downloaded 34GiB to GPU in 2.41414s (14.0837GiB/s) | 20260508 00:27:19.680 | INFO | Analyzing chunk frames = [3796480,3796992) | 20260508 00:27:19.680 | INFO | MemInfo: { | 20260508 00:27:19.680 | INFO | "gpu" : " 17 GB", | 20260508 00:27:19.680 | INFO | "huge" : "", | 20260508 00:27:19.680 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:19.680 | INFO | "malloc" : " 14 GB", | 20260508 00:27:19.680 | INFO | "pinned" : " 29 GB" | 20260508 00:27:19.680 | INFO | } | 20260508 00:27:19.680 | INFO | Perf report for the last 5.14334 seconds: | 20260508 00:27:19.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:19.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.681 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:27:19.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.681 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 00:27:19.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:19.681 | INFO | Uploaded 36GiB to GPU in 2.5219s (14.275GiB/s) | 20260508 00:27:19.681 | INFO | Downloaded 34GiB to GPU in 2.43716s (13.9507GiB/s) | 20260508 00:27:19.795 | INFO | Starting a round of Baz IO | 20260508 00:27:23.266 | INFO | Populating RTMetrics report | 20260508 00:27:23.266 | INFO | RTMetrics report populated | 20260508 00:27:23.266 | INFO | Opening tmp RTMetrics json file | 20260508 00:27:23.271 | INFO | Serializing RTMetrics to json | 20260508 00:27:23.293 | INFO | Serialization done, swapping tmp file now... | 20260508 00:27:23.294 | INFO | RTMetrics done | 20260508 00:27:23.383 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.588 seconds | 20260508 00:27:23.383 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.19% / 0.00% / 57.81% | 20260508 00:27:23.383 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00791% / 3.97364e-06% / 4.37101e-05% | 20260508 00:27:23.383 | INFO | Baz IO overall: | 20260508 00:27:23.383 | INFO | BytesWritten : 1725387102478 | 20260508 00:27:23.383 | INFO | NumEvents : 835086818679 | 20260508 00:27:24.824 | INFO | Analyzing chunk frames = [3796992,3797504) | 20260508 00:27:24.824 | INFO | MemInfo: { | 20260508 00:27:24.824 | INFO | "gpu" : " 17 GB", | 20260508 00:27:24.824 | INFO | "huge" : "", | 20260508 00:27:24.824 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:24.824 | INFO | "malloc" : " 14 GB", | 20260508 00:27:24.824 | INFO | "pinned" : " 29 GB" | 20260508 00:27:24.824 | INFO | } | 20260508 00:27:24.824 | INFO | Perf report for the last 5.14334 seconds: | 20260508 00:27:24.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:24.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:24.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:24.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:24.824 | INFO | ANALYSIS : 69.0%, 2.9, 104.2ms, 0 | 20260508 00:27:24.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:24.824 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260508 00:27:24.824 | INFO | PRE_HQ : 68.0%, 1.8, 64.2ms, 0 | 20260508 00:27:24.824 | INFO | BAZWRITER : 69.6%, 4.2, 151.4ms, 0 | 20260508 00:27:24.824 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 00:27:24.825 | INFO | Uploaded 36GiB to GPU in 2.60323s (13.829GiB/s) | 20260508 00:27:24.825 | INFO | Downloaded 38GiB to GPU in 2.71103s (14.0168GiB/s) | 20260508 00:27:29.962 | INFO | Analyzing chunk frames = [3797504,3798016) | 20260508 00:27:29.962 | INFO | MemInfo: { | 20260508 00:27:29.962 | INFO | "gpu" : " 17 GB", | 20260508 00:27:29.962 | INFO | "huge" : "", | 20260508 00:27:29.962 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:29.962 | INFO | "malloc" : " 14 GB", | 20260508 00:27:29.962 | INFO | "pinned" : " 29 GB" | 20260508 00:27:29.962 | INFO | } | 20260508 00:27:29.962 | INFO | Perf report for the last 5.13814 seconds: | 20260508 00:27:29.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:29.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:27:29.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:27:29.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:29.962 | INFO | Uploaded 36GiB to GPU in 2.53013s (14.2285GiB/s) | 20260508 00:27:29.962 | INFO | Downloaded 34GiB to GPU in 2.42365s (14.0284GiB/s) | 20260508 00:27:35.096 | INFO | Analyzing chunk frames = [3798016,3798528) | 20260508 00:27:35.096 | INFO | MemInfo: { | 20260508 00:27:35.096 | INFO | "gpu" : " 17 GB", | 20260508 00:27:35.096 | INFO | "huge" : "", | 20260508 00:27:35.096 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:35.096 | INFO | "malloc" : " 14 GB", | 20260508 00:27:35.096 | INFO | "pinned" : " 29 GB" | 20260508 00:27:35.096 | INFO | } | 20260508 00:27:35.096 | INFO | Perf report for the last 5.13417 seconds: | 20260508 00:27:35.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:35.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:27:35.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:27:35.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:35.096 | INFO | Uploaded 36GiB to GPU in 2.51166s (14.3332GiB/s) | 20260508 00:27:35.096 | INFO | Downloaded 34GiB to GPU in 2.48371s (13.6892GiB/s) | 20260508 00:27:40.231 | INFO | Analyzing chunk frames = [3798528,3799040) | 20260508 00:27:40.231 | INFO | MemInfo: { | 20260508 00:27:40.231 | INFO | "gpu" : " 17 GB", | 20260508 00:27:40.231 | INFO | "huge" : "", | 20260508 00:27:40.231 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:40.231 | INFO | "malloc" : " 14 GB", | 20260508 00:27:40.231 | INFO | "pinned" : " 29 GB" | 20260508 00:27:40.231 | INFO | } | 20260508 00:27:40.231 | INFO | Perf report for the last 5.13508 seconds: | 20260508 00:27:40.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:40.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.231 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 00:27:40.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.231 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:27:40.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:40.232 | INFO | Uploaded 36GiB to GPU in 2.52107s (14.2796GiB/s) | 20260508 00:27:40.232 | INFO | Downloaded 34GiB to GPU in 2.48941s (13.6579GiB/s) | 20260508 00:27:45.375 | INFO | Analyzing chunk frames = [3799040,3799552) | 20260508 00:27:45.375 | INFO | MemInfo: { | 20260508 00:27:45.375 | INFO | "gpu" : " 17 GB", | 20260508 00:27:45.375 | INFO | "huge" : "", | 20260508 00:27:45.375 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:45.375 | INFO | "malloc" : " 14 GB", | 20260508 00:27:45.375 | INFO | "pinned" : " 29 GB" | 20260508 00:27:45.375 | INFO | } | 20260508 00:27:45.375 | INFO | Perf report for the last 5.14355 seconds: | 20260508 00:27:45.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:45.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 00:27:45.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:27:45.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:45.375 | INFO | Uploaded 36GiB to GPU in 2.4095s (14.9408GiB/s) | 20260508 00:27:45.375 | INFO | Downloaded 34GiB to GPU in 2.49285s (13.639GiB/s) | 20260508 00:27:50.518 | INFO | Analyzing chunk frames = [3799552,3800064) | 20260508 00:27:50.518 | INFO | MemInfo: { | 20260508 00:27:50.518 | INFO | "gpu" : " 17 GB", | 20260508 00:27:50.518 | INFO | "huge" : "", | 20260508 00:27:50.518 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:50.518 | INFO | "malloc" : " 14 GB", | 20260508 00:27:50.518 | INFO | "pinned" : " 29 GB" | 20260508 00:27:50.518 | INFO | } | 20260508 00:27:50.518 | INFO | Perf report for the last 5.14348 seconds: | 20260508 00:27:50.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:50.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.518 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 00:27:50.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.518 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:27:50.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:50.519 | INFO | Uploaded 36GiB to GPU in 2.52151s (14.2772GiB/s) | 20260508 00:27:50.519 | INFO | Downloaded 34GiB to GPU in 2.43803s (13.9457GiB/s) | 20260508 00:27:55.658 | INFO | Analyzing chunk frames = [3800064,3800576) | 20260508 00:27:55.658 | INFO | MemInfo: { | 20260508 00:27:55.658 | INFO | "gpu" : " 17 GB", | 20260508 00:27:55.658 | INFO | "huge" : "", | 20260508 00:27:55.658 | INFO | "huge_shared" : " 29 GB", | 20260508 00:27:55.658 | INFO | "malloc" : " 14 GB", | 20260508 00:27:55.658 | INFO | "pinned" : " 29 GB" | 20260508 00:27:55.658 | INFO | } | 20260508 00:27:55.658 | INFO | Perf report for the last 5.14018 seconds: | 20260508 00:27:55.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:27:55.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.658 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:27:55.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.659 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 00:27:55.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:27:55.659 | INFO | Uploaded 36GiB to GPU in 2.5173s (14.301GiB/s) | 20260508 00:27:55.659 | INFO | Downloaded 34GiB to GPU in 2.45302s (13.8605GiB/s) | 20260508 00:28:00.793 | INFO | Analyzing chunk frames = [3800576,3801088) | 20260508 00:28:00.793 | INFO | MemInfo: { | 20260508 00:28:00.793 | INFO | "gpu" : " 17 GB", | 20260508 00:28:00.793 | INFO | "huge" : "", | 20260508 00:28:00.793 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:00.793 | INFO | "malloc" : " 14 GB", | 20260508 00:28:00.793 | INFO | "pinned" : " 29 GB" | 20260508 00:28:00.793 | INFO | } | 20260508 00:28:00.794 | INFO | Perf report for the last 5.13499 seconds: | 20260508 00:28:00.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:00.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:28:00.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260508 00:28:00.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:00.794 | INFO | Uploaded 36GiB to GPU in 2.55698s (14.0791GiB/s) | 20260508 00:28:00.794 | INFO | Downloaded 34GiB to GPU in 2.45826s (13.8309GiB/s) | 20260508 00:28:00.794 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:28:00.794 | INFO | Total mem usage: 16.7 GiB | 20260508 00:28:00.794 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:28:00.794 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 00:28:00.794 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:28:00.794 | INFO | High Water Breakdown: | 20260508 00:28:00.794 | INFO | 60.3% : 9.07 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:28:00.794 | INFO | 39.6% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:28:00.794 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:28:00.794 | INFO | ------------------------------------------ | 20260508 00:28:00.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:28:00.794 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:28:00.794 | INFO | Total mem usage: 26.5 GiB | 20260508 00:28:00.794 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:28:00.794 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 00:28:00.794 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:28:00.794 | INFO | High Water Breakdown: | 20260508 00:28:00.794 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:28:00.794 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:28:00.794 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:28:00.794 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:28:00.794 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:28:00.794 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:28:00.794 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:28:00.794 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:28:00.794 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:28:00.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:28:00.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:28:00.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:28:00.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:28:00.794 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:28:00.794 | INFO | ------------------------------------------ | 20260508 00:28:00.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:28:00.794 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:28:00.794 | INFO | Total mem usage: 27.8 GiB | 20260508 00:28:00.795 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:28:00.795 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:28:00.795 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:28:00.795 | INFO | High Water Breakdown: | 20260508 00:28:00.795 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:28:00.795 | INFO | ------------------------------------------ | 20260508 00:28:00.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:28:00.795 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:28:00.795 | INFO | Total mem usage: 15.7 GiB | 20260508 00:28:00.795 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:28:00.795 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:28:00.795 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:28:00.795 | INFO | High Water Breakdown: | 20260508 00:28:00.795 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:28:00.795 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:28:00.795 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:28:00.795 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:28:00.795 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:28:00.795 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:28:00.795 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:28:00.795 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:28:00.795 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:28:00.795 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:28:00.795 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:28:00.795 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:28:00.795 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:28:00.795 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:28:00.795 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:28:00.795 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:28:00.795 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:28:00.795 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:28:00.795 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:28:00.795 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:28:00.795 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:28:00.795 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:28:00.795 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:28:00.795 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:28:00.795 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:28:00.795 | INFO | ------------------------------------------ | 20260508 00:28:00.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:28:00.795 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:28:00.795 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:28:00.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:28:00.795 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.41e+01 / 1.54e+00 | 20260508 00:28:00.795 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.22e-02 / 5.94e-03 | 20260508 00:28:00.795 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.47e+00, 5.80e-01 / 2.47e-01 | 20260508 00:28:00.795 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.06e-01 / 1.14e-01 | 20260508 00:28:00.795 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.59e+01, 4.00e+00 / 1.61e+00 | 20260508 00:28:00.795 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.66e+00 / 1.46e+00 | 20260508 00:28:00.795 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.71e+00, 4.26e-01 / 2.32e-01 | 20260508 00:28:00.796 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.28e+00, 2.79e-01 / 1.33e-01 | 20260508 00:28:00.796 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.43e-01 / 3.83e-01, 5.55e-02 / 2.91e-02 | 20260508 00:28:00.796 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:28:00.796 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:28:00.796 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:28:00.796 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 24.52%, Upload -- 2.62e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 24.24%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 30.73%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 2.38%, LockUpload -- 2.54e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 2.54%, LockDownload -- 2.71e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:28:00.796 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260508 00:28:00.911 | INFO | Starting a round of Baz IO | 20260508 00:28:04.362 | INFO | Populating RTMetrics report | 20260508 00:28:04.363 | INFO | RTMetrics report populated | 20260508 00:28:04.363 | INFO | Opening tmp RTMetrics json file | 20260508 00:28:04.399 | INFO | Serializing RTMetrics to json | 20260508 00:28:04.444 | INFO | Serialization done, swapping tmp file now... | 20260508 00:28:04.464 | INFO | RTMetrics done | 20260508 00:28:04.488 | INFO | Finished a round of Baz IO: Wrote 1.691 GiB in 3.577 seconds | 20260508 00:28:04.488 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.19% / 0.00% / 57.81% | 20260508 00:28:04.488 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00645% / 7.94729e-06% / 6.35783e-05% | 20260508 00:28:04.488 | INFO | Baz IO overall: | 20260508 00:28:04.488 | INFO | BytesWritten : 1727203024303 | 20260508 00:28:04.488 | INFO | NumEvents : 835960849505 | 20260508 00:28:05.936 | INFO | Analyzing chunk frames = [3801088,3801600) | 20260508 00:28:05.936 | INFO | MemInfo: { | 20260508 00:28:05.936 | INFO | "gpu" : " 17 GB", | 20260508 00:28:05.936 | INFO | "huge" : "", | 20260508 00:28:05.936 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:05.936 | INFO | "malloc" : " 14 GB", | 20260508 00:28:05.936 | INFO | "pinned" : " 29 GB" | 20260508 00:28:05.936 | INFO | } | 20260508 00:28:05.936 | INFO | Perf report for the last 5.14218 seconds: | 20260508 00:28:05.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:05.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:05.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:05.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:05.936 | INFO | ANALYSIS : 68.7%, 2.9, 104.7ms, 0 | 20260508 00:28:05.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:05.936 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260508 00:28:05.936 | INFO | PRE_HQ : 67.1%, 1.8, 62.3ms, 0 | 20260508 00:28:05.936 | INFO | BAZWRITER : 68.9%, 4.7, 167.8ms, 0 | 20260508 00:28:05.936 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:28:05.936 | INFO | Uploaded 36GiB to GPU in 2.61567s (13.7632GiB/s) | 20260508 00:28:05.936 | INFO | Downloaded 38GiB to GPU in 2.70439s (14.0512GiB/s) | 20260508 00:28:11.071 | INFO | Analyzing chunk frames = [3801600,3802112) | 20260508 00:28:11.071 | INFO | MemInfo: { | 20260508 00:28:11.071 | INFO | "gpu" : " 17 GB", | 20260508 00:28:11.071 | INFO | "huge" : "", | 20260508 00:28:11.071 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:11.071 | INFO | "malloc" : " 14 GB", | 20260508 00:28:11.071 | INFO | "pinned" : " 29 GB" | 20260508 00:28:11.071 | INFO | } | 20260508 00:28:11.071 | INFO | Perf report for the last 5.13568 seconds: | 20260508 00:28:11.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:11.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.071 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:28:11.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.071 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:28:11.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:11.072 | INFO | Uploaded 36GiB to GPU in 2.51236s (14.3291GiB/s) | 20260508 00:28:11.072 | INFO | Downloaded 34GiB to GPU in 2.42431s (14.0246GiB/s) | 20260508 00:28:16.212 | INFO | Analyzing chunk frames = [3802112,3802624) | 20260508 00:28:16.212 | INFO | MemInfo: { | 20260508 00:28:16.212 | INFO | "gpu" : " 17 GB", | 20260508 00:28:16.212 | INFO | "huge" : "", | 20260508 00:28:16.212 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:16.212 | INFO | "malloc" : " 14 GB", | 20260508 00:28:16.212 | INFO | "pinned" : " 29 GB" | 20260508 00:28:16.212 | INFO | } | 20260508 00:28:16.212 | INFO | Perf report for the last 5.14066 seconds: | 20260508 00:28:16.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:16.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.212 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:28:16.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.212 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:28:16.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:16.213 | INFO | Uploaded 36GiB to GPU in 2.51092s (14.3374GiB/s) | 20260508 00:28:16.213 | INFO | Downloaded 34GiB to GPU in 2.4848s (13.6832GiB/s) | 20260508 00:28:21.351 | INFO | Analyzing chunk frames = [3802624,3803136) | 20260508 00:28:21.351 | INFO | MemInfo: { | 20260508 00:28:21.351 | INFO | "gpu" : " 17 GB", | 20260508 00:28:21.351 | INFO | "huge" : "", | 20260508 00:28:21.351 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:21.351 | INFO | "malloc" : " 14 GB", | 20260508 00:28:21.351 | INFO | "pinned" : " 29 GB" | 20260508 00:28:21.351 | INFO | } | 20260508 00:28:21.351 | INFO | Perf report for the last 5.13933 seconds: | 20260508 00:28:21.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:21.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.351 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:28:21.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.351 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:28:21.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:21.352 | INFO | Uploaded 36GiB to GPU in 2.51445s (14.3172GiB/s) | 20260508 00:28:21.352 | INFO | Downloaded 34GiB to GPU in 2.47791s (13.7212GiB/s) | 20260508 00:28:26.490 | INFO | Analyzing chunk frames = [3803136,3803648) | 20260508 00:28:26.490 | INFO | MemInfo: { | 20260508 00:28:26.490 | INFO | "gpu" : " 17 GB", | 20260508 00:28:26.490 | INFO | "huge" : "", | 20260508 00:28:26.490 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:26.490 | INFO | "malloc" : " 14 GB", | 20260508 00:28:26.490 | INFO | "pinned" : " 29 GB" | 20260508 00:28:26.490 | INFO | } | 20260508 00:28:26.490 | INFO | Perf report for the last 5.13872 seconds: | 20260508 00:28:26.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:26.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.490 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:28:26.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.490 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:28:26.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:26.491 | INFO | Uploaded 36GiB to GPU in 2.2724s (15.8423GiB/s) | 20260508 00:28:26.491 | INFO | Downloaded 34GiB to GPU in 2.4741s (13.7424GiB/s) | 20260508 00:28:31.632 | INFO | Analyzing chunk frames = [3803648,3804160) | 20260508 00:28:31.632 | INFO | MemInfo: { | 20260508 00:28:31.632 | INFO | "gpu" : " 17 GB", | 20260508 00:28:31.633 | INFO | "huge" : "", | 20260508 00:28:31.633 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:31.633 | INFO | "malloc" : " 14 GB", | 20260508 00:28:31.633 | INFO | "pinned" : " 29 GB" | 20260508 00:28:31.633 | INFO | } | 20260508 00:28:31.633 | INFO | Perf report for the last 5.14245 seconds: | 20260508 00:28:31.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:31.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:28:31.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:28:31.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:31.633 | INFO | Uploaded 36GiB to GPU in 2.34742s (15.336GiB/s) | 20260508 00:28:31.633 | INFO | Downloaded 34GiB to GPU in 2.42306s (14.0318GiB/s) | 20260508 00:28:36.773 | INFO | Analyzing chunk frames = [3804160,3804672) | 20260508 00:28:36.773 | INFO | MemInfo: { | 20260508 00:28:36.773 | INFO | "gpu" : " 17 GB", | 20260508 00:28:36.773 | INFO | "huge" : "", | 20260508 00:28:36.773 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:36.773 | INFO | "malloc" : " 14 GB", | 20260508 00:28:36.773 | INFO | "pinned" : " 29 GB" | 20260508 00:28:36.773 | INFO | } | 20260508 00:28:36.773 | INFO | Perf report for the last 5.14035 seconds: | 20260508 00:28:36.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:36.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 00:28:36.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 00:28:36.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:36.773 | INFO | Uploaded 36GiB to GPU in 2.48749s (14.4724GiB/s) | 20260508 00:28:36.774 | INFO | Downloaded 34GiB to GPU in 2.46827s (13.7748GiB/s) | 20260508 00:28:41.909 | INFO | Analyzing chunk frames = [3804672,3805184) | 20260508 00:28:41.909 | INFO | MemInfo: { | 20260508 00:28:41.909 | INFO | "gpu" : " 17 GB", | 20260508 00:28:41.909 | INFO | "huge" : "", | 20260508 00:28:41.909 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:41.909 | INFO | "malloc" : " 14 GB", | 20260508 00:28:41.909 | INFO | "pinned" : " 29 GB" | 20260508 00:28:41.909 | INFO | } | 20260508 00:28:41.909 | INFO | Perf report for the last 5.13621 seconds: | 20260508 00:28:41.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:41.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.909 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 00:28:41.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.909 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 00:28:41.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:41.910 | INFO | Uploaded 36GiB to GPU in 2.51818s (14.296GiB/s) | 20260508 00:28:41.910 | INFO | Downloaded 34GiB to GPU in 2.47337s (13.7464GiB/s) | 20260508 00:28:42.018 | INFO | Starting a round of Baz IO | 20260508 00:28:45.481 | INFO | Populating RTMetrics report | 20260508 00:28:45.481 | INFO | RTMetrics report populated | 20260508 00:28:45.481 | INFO | Opening tmp RTMetrics json file | 20260508 00:28:45.641 | INFO | Serializing RTMetrics to json | 20260508 00:28:45.827 | INFO | Serialization done, swapping tmp file now... | 20260508 00:28:45.831 | INFO | RTMetrics done | 20260508 00:28:45.856 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.837 seconds | 20260508 00:28:45.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.18% / 0.00% / 57.81% | 20260508 00:28:45.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00478% / 1.19209e-05% / 7.54992e-05% | 20260508 00:28:45.856 | INFO | Baz IO overall: | 20260508 00:28:45.856 | INFO | BytesWritten : 1729017419551 | 20260508 00:28:45.856 | INFO | NumEvents : 836833608119 | 20260508 00:28:47.044 | INFO | Analyzing chunk frames = [3805184,3805696) | 20260508 00:28:47.044 | INFO | MemInfo: { | 20260508 00:28:47.044 | INFO | "gpu" : " 17 GB", | 20260508 00:28:47.044 | INFO | "huge" : "", | 20260508 00:28:47.044 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:47.044 | INFO | "malloc" : " 14 GB", | 20260508 00:28:47.044 | INFO | "pinned" : " 29 GB" | 20260508 00:28:47.044 | INFO | } | 20260508 00:28:47.045 | INFO | Perf report for the last 5.13542 seconds: | 20260508 00:28:47.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:47.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:47.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:47.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:47.045 | INFO | ANALYSIS : 69.1%, 2.9, 103.8ms, 0 | 20260508 00:28:47.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:47.045 | INFO | RT_METRICS : 52.4%, 1.0, 27.5ms, 0 | 20260508 00:28:47.045 | INFO | PRE_HQ : 67.6%, 1.8, 64.5ms, 0 | 20260508 00:28:47.045 | INFO | BAZWRITER : 74.3%, 4.5, 171.0ms, 0 | 20260508 00:28:47.045 | INFO | BAZ_SAMPLING : 5.8%, 1.0, 3.0ms, 0 | 20260508 00:28:47.045 | INFO | Uploaded 36GiB to GPU in 2.45581s (14.6591GiB/s) | 20260508 00:28:47.045 | INFO | Downloaded 38GiB to GPU in 2.69005s (14.1261GiB/s) | 20260508 00:28:52.186 | INFO | Analyzing chunk frames = [3805696,3806208) | 20260508 00:28:52.186 | INFO | MemInfo: { | 20260508 00:28:52.186 | INFO | "gpu" : " 17 GB", | 20260508 00:28:52.186 | INFO | "huge" : "", | 20260508 00:28:52.186 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:52.186 | INFO | "malloc" : " 14 GB", | 20260508 00:28:52.186 | INFO | "pinned" : " 29 GB" | 20260508 00:28:52.186 | INFO | } | 20260508 00:28:52.186 | INFO | Perf report for the last 5.14187 seconds: | 20260508 00:28:52.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:52.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.187 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 00:28:52.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.187 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:28:52.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:52.188 | INFO | Uploaded 36GiB to GPU in 2.53931s (14.1771GiB/s) | 20260508 00:28:52.188 | INFO | Downloaded 34GiB to GPU in 2.49592s (13.6222GiB/s) | 20260508 00:28:57.329 | INFO | Analyzing chunk frames = [3806208,3806720) | 20260508 00:28:57.329 | INFO | MemInfo: { | 20260508 00:28:57.329 | INFO | "gpu" : " 17 GB", | 20260508 00:28:57.329 | INFO | "huge" : "", | 20260508 00:28:57.329 | INFO | "huge_shared" : " 29 GB", | 20260508 00:28:57.329 | INFO | "malloc" : " 14 GB", | 20260508 00:28:57.329 | INFO | "pinned" : " 29 GB" | 20260508 00:28:57.329 | INFO | } | 20260508 00:28:57.329 | INFO | Perf report for the last 5.14243 seconds: | 20260508 00:28:57.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:28:57.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:28:57.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:28:57.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:28:57.329 | INFO | Uploaded 36GiB to GPU in 2.45027s (14.6923GiB/s) | 20260508 00:28:57.329 | INFO | Downloaded 34GiB to GPU in 2.49264s (13.6402GiB/s) | 20260508 00:29:02.470 | INFO | Analyzing chunk frames = [3806720,3807232) | 20260508 00:29:02.470 | INFO | MemInfo: { | 20260508 00:29:02.470 | INFO | "gpu" : " 17 GB", | 20260508 00:29:02.470 | INFO | "huge" : "", | 20260508 00:29:02.470 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:02.470 | INFO | "malloc" : " 14 GB", | 20260508 00:29:02.470 | INFO | "pinned" : " 29 GB" | 20260508 00:29:02.470 | INFO | } | 20260508 00:29:02.470 | INFO | Perf report for the last 5.14143 seconds: | 20260508 00:29:02.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:02.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.470 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 00:29:02.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.470 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:29:02.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:02.471 | INFO | Uploaded 36GiB to GPU in 2.46762s (14.5889GiB/s) | 20260508 00:29:02.471 | INFO | Downloaded 34GiB to GPU in 2.4924s (13.6415GiB/s) | 20260508 00:29:07.608 | INFO | Analyzing chunk frames = [3807232,3807744) | 20260508 00:29:07.608 | INFO | MemInfo: { | 20260508 00:29:07.608 | INFO | "gpu" : " 17 GB", | 20260508 00:29:07.608 | INFO | "huge" : "", | 20260508 00:29:07.608 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:07.608 | INFO | "malloc" : " 14 GB", | 20260508 00:29:07.609 | INFO | "pinned" : " 29 GB" | 20260508 00:29:07.609 | INFO | } | 20260508 00:29:07.609 | INFO | Perf report for the last 5.13821 seconds: | 20260508 00:29:07.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:07.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 00:29:07.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:29:07.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:07.609 | INFO | Uploaded 36GiB to GPU in 2.29796s (15.6661GiB/s) | 20260508 00:29:07.609 | INFO | Downloaded 34GiB to GPU in 2.47936s (13.7132GiB/s) | 20260508 00:29:12.747 | INFO | Analyzing chunk frames = [3807744,3808256) | 20260508 00:29:12.747 | INFO | MemInfo: { | 20260508 00:29:12.747 | INFO | "gpu" : " 17 GB", | 20260508 00:29:12.747 | INFO | "huge" : "", | 20260508 00:29:12.747 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:12.747 | INFO | "malloc" : " 14 GB", | 20260508 00:29:12.747 | INFO | "pinned" : " 29 GB" | 20260508 00:29:12.747 | INFO | } | 20260508 00:29:12.747 | INFO | Perf report for the last 5.13858 seconds: | 20260508 00:29:12.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:12.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:29:12.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:29:12.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:12.747 | INFO | Uploaded 36GiB to GPU in 2.38095s (15.12GiB/s) | 20260508 00:29:12.747 | INFO | Downloaded 34GiB to GPU in 2.41287s (14.0911GiB/s) | 20260508 00:29:17.887 | INFO | Analyzing chunk frames = [3808256,3808768) | 20260508 00:29:17.887 | INFO | MemInfo: { | 20260508 00:29:17.887 | INFO | "gpu" : " 17 GB", | 20260508 00:29:17.887 | INFO | "huge" : "", | 20260508 00:29:17.887 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:17.887 | INFO | "malloc" : " 14 GB", | 20260508 00:29:17.887 | INFO | "pinned" : " 29 GB" | 20260508 00:29:17.887 | INFO | } | 20260508 00:29:17.887 | INFO | Perf report for the last 5.13995 seconds: | 20260508 00:29:17.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:17.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 00:29:17.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:29:17.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:17.887 | INFO | Uploaded 36GiB to GPU in 2.408s (14.9501GiB/s) | 20260508 00:29:17.887 | INFO | Downloaded 34GiB to GPU in 2.42048s (14.0468GiB/s) | 20260508 00:29:23.022 | INFO | Analyzing chunk frames = [3808768,3809280) | 20260508 00:29:23.022 | INFO | MemInfo: { | 20260508 00:29:23.022 | INFO | "gpu" : " 17 GB", | 20260508 00:29:23.022 | INFO | "huge" : "", | 20260508 00:29:23.022 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:23.022 | INFO | "malloc" : " 14 GB", | 20260508 00:29:23.022 | INFO | "pinned" : " 29 GB" | 20260508 00:29:23.022 | INFO | } | 20260508 00:29:23.022 | INFO | Perf report for the last 5.13448 seconds: | 20260508 00:29:23.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:23.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 00:29:23.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:29:23.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:23.022 | INFO | Uploaded 36GiB to GPU in 2.51108s (14.3365GiB/s) | 20260508 00:29:23.022 | INFO | Downloaded 34GiB to GPU in 2.4522s (13.8651GiB/s) | 20260508 00:29:23.022 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:29:23.022 | INFO | Total mem usage: 16.7 GiB | 20260508 00:29:23.022 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:29:23.022 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.24%) | 20260508 00:29:23.022 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:29:23.022 | INFO | High Water Breakdown: | 20260508 00:29:23.022 | INFO | 61% : 9.57 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:29:23.022 | INFO | 38.9% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:29:23.022 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:29:23.022 | INFO | ------------------------------------------ | 20260508 00:29:23.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:29:23.022 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:29:23.022 | INFO | Total mem usage: 26.5 GiB | 20260508 00:29:23.022 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:29:23.022 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:29:23.022 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 00:29:23.022 | INFO | High Water Breakdown: | 20260508 00:29:23.022 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:29:23.022 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:29:23.022 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:29:23.022 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:29:23.022 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:29:23.022 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:29:23.022 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:29:23.022 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:29:23.022 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:29:23.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:29:23.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:29:23.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:29:23.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:29:23.022 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:29:23.022 | INFO | ------------------------------------------ | 20260508 00:29:23.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:29:23.022 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:29:23.022 | INFO | Total mem usage: 27.8 GiB | 20260508 00:29:23.022 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:29:23.022 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:29:23.022 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:29:23.022 | INFO | High Water Breakdown: | 20260508 00:29:23.022 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:29:23.022 | INFO | ------------------------------------------ | 20260508 00:29:23.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:29:23.022 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:29:23.022 | INFO | Total mem usage: 15.7 GiB | 20260508 00:29:23.022 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:29:23.022 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:29:23.022 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:29:23.022 | INFO | High Water Breakdown: | 20260508 00:29:23.022 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:29:23.022 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:29:23.022 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:29:23.022 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:29:23.022 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:29:23.022 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:29:23.022 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:29:23.022 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:29:23.022 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:29:23.022 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:29:23.022 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:29:23.022 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:29:23.022 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:29:23.022 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:29:23.022 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:29:23.022 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:29:23.022 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:29:23.022 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:29:23.022 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:29:23.022 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:29:23.022 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:29:23.022 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:29:23.022 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:29:23.022 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:29:23.022 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:29:23.022 | INFO | ------------------------------------------ | 20260508 00:29:23.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:29:23.023 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:29:23.023 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:29:23.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:29:23.023 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.43e+01 / 1.20e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.19e-02 / 5.96e-03 | 20260508 00:29:23.023 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.49e+00, 5.88e-01 / 2.13e-01 | 20260508 00:29:23.023 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.09e-01 / 1.00e-01 | 20260508 00:29:23.023 | INFO | PerfReport: 16.07%, DME -- 4.22e+01 / 4.61e+01, 4.14e+00 / 1.44e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.64e+00 / 1.47e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.28e-01 / 2.48e-01 | 20260508 00:29:23.023 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.30e+00, 2.76e-01 / 1.29e-01 | 20260508 00:29:23.023 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.42e-01 / 3.80e-01, 5.45e-02 / 2.74e-02 | 20260508 00:29:23.023 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:29:23.023 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:29:23.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:29:23.023 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 24.08%, Upload -- 2.56e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 30.91%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 1.97%, LockUpload -- 2.09e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 2.96%, LockDownload -- 3.14e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:29:23.023 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.04e-02 / 0.00e+00 | 20260508 00:29:23.119 | INFO | Starting a round of Baz IO | 20260508 00:29:26.578 | INFO | Populating RTMetrics report | 20260508 00:29:26.578 | INFO | RTMetrics report populated | 20260508 00:29:26.578 | INFO | Opening tmp RTMetrics json file | 20260508 00:29:26.601 | INFO | Serializing RTMetrics to json | 20260508 00:29:26.602 | INFO | Serialization done, swapping tmp file now... | 20260508 00:29:26.603 | INFO | RTMetrics done | 20260508 00:29:26.674 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.555 seconds | 20260508 00:29:26.674 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.18% / 0.00% / 57.82% | 20260508 00:29:26.674 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00242% / 3.97364e-06% / 5.16574e-05% | 20260508 00:29:26.674 | INFO | Baz IO overall: | 20260508 00:29:26.674 | INFO | BytesWritten : 1730830338573 | 20260508 00:29:26.674 | INFO | NumEvents : 837705552513 | 20260508 00:29:28.165 | INFO | Analyzing chunk frames = [3809280,3809792) | 20260508 00:29:28.165 | INFO | MemInfo: { | 20260508 00:29:28.165 | INFO | "gpu" : " 17 GB", | 20260508 00:29:28.165 | INFO | "huge" : "", | 20260508 00:29:28.165 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:28.165 | INFO | "malloc" : " 14 GB", | 20260508 00:29:28.165 | INFO | "pinned" : " 29 GB" | 20260508 00:29:28.165 | INFO | } | 20260508 00:29:28.165 | INFO | Perf report for the last 5.14331 seconds: | 20260508 00:29:28.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:28.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:28.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:28.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:28.165 | INFO | ANALYSIS : 68.4%, 3.0, 106.9ms, 0 | 20260508 00:29:28.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:28.165 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 00:29:28.165 | INFO | PRE_HQ : 65.4%, 1.7, 56.5ms, 0 | 20260508 00:29:28.165 | INFO | BAZWRITER : 68.7%, 3.8, 133.2ms, 0 | 20260508 00:29:28.165 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:29:28.166 | INFO | Uploaded 36GiB to GPU in 2.54317s (14.1556GiB/s) | 20260508 00:29:28.166 | INFO | Downloaded 38GiB to GPU in 2.69628s (14.0935GiB/s) | 20260508 00:29:33.296 | INFO | Analyzing chunk frames = [3809792,3810304) | 20260508 00:29:33.296 | INFO | MemInfo: { | 20260508 00:29:33.296 | INFO | "gpu" : " 17 GB", | 20260508 00:29:33.296 | INFO | "huge" : "", | 20260508 00:29:33.296 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:33.296 | INFO | "malloc" : " 14 GB", | 20260508 00:29:33.296 | INFO | "pinned" : " 29 GB" | 20260508 00:29:33.296 | INFO | } | 20260508 00:29:33.296 | INFO | Perf report for the last 5.13086 seconds: | 20260508 00:29:33.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:33.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:29:33.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:29:33.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:33.296 | INFO | Uploaded 36GiB to GPU in 2.5263s (14.2501GiB/s) | 20260508 00:29:33.297 | INFO | Downloaded 34GiB to GPU in 2.43106s (13.9857GiB/s) | 20260508 00:29:38.442 | INFO | Analyzing chunk frames = [3810304,3810816) | 20260508 00:29:38.442 | INFO | MemInfo: { | 20260508 00:29:38.442 | INFO | "gpu" : " 17 GB", | 20260508 00:29:38.443 | INFO | "huge" : "", | 20260508 00:29:38.443 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:38.443 | INFO | "malloc" : " 14 GB", | 20260508 00:29:38.443 | INFO | "pinned" : " 29 GB" | 20260508 00:29:38.443 | INFO | } | 20260508 00:29:38.443 | INFO | Perf report for the last 5.1469 seconds: | 20260508 00:29:38.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:38.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 00:29:38.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:29:38.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:38.443 | INFO | Uploaded 36GiB to GPU in 2.53734s (14.1881GiB/s) | 20260508 00:29:38.443 | INFO | Downloaded 34GiB to GPU in 2.47638s (13.7297GiB/s) | 20260508 00:29:43.585 | INFO | Analyzing chunk frames = [3810816,3811328) | 20260508 00:29:43.585 | INFO | MemInfo: { | 20260508 00:29:43.585 | INFO | "gpu" : " 17 GB", | 20260508 00:29:43.585 | INFO | "huge" : "", | 20260508 00:29:43.585 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:43.585 | INFO | "malloc" : " 14 GB", | 20260508 00:29:43.585 | INFO | "pinned" : " 29 GB" | 20260508 00:29:43.585 | INFO | } | 20260508 00:29:43.585 | INFO | Perf report for the last 5.1425 seconds: | 20260508 00:29:43.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:43.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.585 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:29:43.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.585 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:29:43.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:43.586 | INFO | Uploaded 36GiB to GPU in 2.50603s (14.3653GiB/s) | 20260508 00:29:43.586 | INFO | Downloaded 34GiB to GPU in 2.4818s (13.6997GiB/s) | 20260508 00:29:48.726 | INFO | Analyzing chunk frames = [3811328,3811840) | 20260508 00:29:48.726 | INFO | MemInfo: { | 20260508 00:29:48.726 | INFO | "gpu" : " 17 GB", | 20260508 00:29:48.726 | INFO | "huge" : "", | 20260508 00:29:48.726 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:48.726 | INFO | "malloc" : " 14 GB", | 20260508 00:29:48.726 | INFO | "pinned" : " 29 GB" | 20260508 00:29:48.726 | INFO | } | 20260508 00:29:48.726 | INFO | Perf report for the last 5.14105 seconds: | 20260508 00:29:48.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:48.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.726 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:29:48.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.726 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:29:48.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:48.727 | INFO | Uploaded 36GiB to GPU in 2.29734s (15.6703GiB/s) | 20260508 00:29:48.727 | INFO | Downloaded 34GiB to GPU in 2.49883s (13.6063GiB/s) | 20260508 00:29:53.858 | INFO | Analyzing chunk frames = [3811840,3812352) | 20260508 00:29:53.858 | INFO | MemInfo: { | 20260508 00:29:53.858 | INFO | "gpu" : " 17 GB", | 20260508 00:29:53.858 | INFO | "huge" : "", | 20260508 00:29:53.858 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:53.858 | INFO | "malloc" : " 14 GB", | 20260508 00:29:53.858 | INFO | "pinned" : " 29 GB" | 20260508 00:29:53.858 | INFO | } | 20260508 00:29:53.859 | INFO | Perf report for the last 5.13217 seconds: | 20260508 00:29:53.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:53.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:29:53.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260508 00:29:53.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:53.859 | INFO | Uploaded 36GiB to GPU in 2.35051s (15.3158GiB/s) | 20260508 00:29:53.859 | INFO | Downloaded 34GiB to GPU in 2.42135s (14.0418GiB/s) | 20260508 00:29:58.997 | INFO | Analyzing chunk frames = [3812352,3812864) | 20260508 00:29:58.997 | INFO | MemInfo: { | 20260508 00:29:58.997 | INFO | "gpu" : " 17 GB", | 20260508 00:29:58.997 | INFO | "huge" : "", | 20260508 00:29:58.997 | INFO | "huge_shared" : " 29 GB", | 20260508 00:29:58.997 | INFO | "malloc" : " 14 GB", | 20260508 00:29:58.997 | INFO | "pinned" : " 29 GB" | 20260508 00:29:58.997 | INFO | } | 20260508 00:29:58.997 | INFO | Perf report for the last 5.13882 seconds: | 20260508 00:29:58.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:29:58.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.997 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:29:58.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.997 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:29:58.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:29:58.998 | INFO | Uploaded 36GiB to GPU in 2.51797s (14.2972GiB/s) | 20260508 00:29:58.998 | INFO | Downloaded 34GiB to GPU in 2.47905s (13.715GiB/s) | 20260508 00:30:04.139 | INFO | Analyzing chunk frames = [3812864,3813376) | 20260508 00:30:04.139 | INFO | MemInfo: { | 20260508 00:30:04.139 | INFO | "gpu" : " 17 GB", | 20260508 00:30:04.139 | INFO | "huge" : "", | 20260508 00:30:04.139 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:04.139 | INFO | "malloc" : " 14 GB", | 20260508 00:30:04.139 | INFO | "pinned" : " 29 GB" | 20260508 00:30:04.139 | INFO | } | 20260508 00:30:04.139 | INFO | Perf report for the last 5.14184 seconds: | 20260508 00:30:04.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:04.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.139 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 00:30:04.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.139 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:30:04.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:04.140 | INFO | Uploaded 36GiB to GPU in 2.49477s (14.4302GiB/s) | 20260508 00:30:04.140 | INFO | Downloaded 34GiB to GPU in 2.46356s (13.8011GiB/s) | 20260508 00:30:04.236 | INFO | Starting a round of Baz IO | 20260508 00:30:07.691 | INFO | Populating RTMetrics report | 20260508 00:30:07.691 | INFO | RTMetrics report populated | 20260508 00:30:07.691 | INFO | Opening tmp RTMetrics json file | 20260508 00:30:07.714 | INFO | Serializing RTMetrics to json | 20260508 00:30:07.750 | INFO | Serialization done, swapping tmp file now... | 20260508 00:30:07.766 | INFO | RTMetrics done | 20260508 00:30:07.793 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.556 seconds | 20260508 00:30:07.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.17% / 0.00% / 57.82% | 20260508 00:30:07.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00249% / 3.97364e-06% / 5.5631e-05% | 20260508 00:30:07.793 | INFO | Baz IO overall: | 20260508 00:30:07.793 | INFO | BytesWritten : 1732642565275 | 20260508 00:30:07.793 | INFO | NumEvents : 838577451216 | 20260508 00:30:09.280 | INFO | Analyzing chunk frames = [3813376,3813888) | 20260508 00:30:09.280 | INFO | MemInfo: { | 20260508 00:30:09.280 | INFO | "gpu" : " 17 GB", | 20260508 00:30:09.280 | INFO | "huge" : "", | 20260508 00:30:09.280 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:09.280 | INFO | "malloc" : " 14 GB", | 20260508 00:30:09.280 | INFO | "pinned" : " 29 GB" | 20260508 00:30:09.280 | INFO | } | 20260508 00:30:09.280 | INFO | Perf report for the last 5.14099 seconds: | 20260508 00:30:09.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:09.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:09.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:09.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:09.280 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 00:30:09.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:09.280 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 00:30:09.280 | INFO | PRE_HQ : 65.5%, 1.7, 56.8ms, 0 | 20260508 00:30:09.280 | INFO | BAZWRITER : 68.8%, 4.1, 144.6ms, 0 | 20260508 00:30:09.280 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 00:30:09.281 | INFO | Uploaded 36GiB to GPU in 2.51815s (14.2962GiB/s) | 20260508 00:30:09.281 | INFO | Downloaded 38GiB to GPU in 2.69145s (14.1188GiB/s) | 20260508 00:30:14.426 | INFO | Analyzing chunk frames = [3813888,3814400) | 20260508 00:30:14.426 | INFO | MemInfo: { | 20260508 00:30:14.426 | INFO | "gpu" : " 17 GB", | 20260508 00:30:14.426 | INFO | "huge" : "", | 20260508 00:30:14.426 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:14.426 | INFO | "malloc" : " 14 GB", | 20260508 00:30:14.426 | INFO | "pinned" : " 29 GB" | 20260508 00:30:14.426 | INFO | } | 20260508 00:30:14.426 | INFO | Perf report for the last 5.14557 seconds: | 20260508 00:30:14.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:14.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 00:30:14.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:30:14.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:14.426 | INFO | Uploaded 36GiB to GPU in 2.55196s (14.1068GiB/s) | 20260508 00:30:14.426 | INFO | Downloaded 34GiB to GPU in 2.50398s (13.5784GiB/s) | 20260508 00:30:19.554 | INFO | Analyzing chunk frames = [3814400,3814912) | 20260508 00:30:19.554 | INFO | MemInfo: { | 20260508 00:30:19.554 | INFO | "gpu" : " 17 GB", | 20260508 00:30:19.554 | INFO | "huge" : "", | 20260508 00:30:19.554 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:19.554 | INFO | "malloc" : " 14 GB", | 20260508 00:30:19.554 | INFO | "pinned" : " 29 GB" | 20260508 00:30:19.554 | INFO | } | 20260508 00:30:19.554 | INFO | Perf report for the last 5.12841 seconds: | 20260508 00:30:19.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:19.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.554 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 00:30:19.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.554 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 00:30:19.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:19.555 | INFO | Uploaded 36GiB to GPU in 2.47234s (14.5611GiB/s) | 20260508 00:30:19.555 | INFO | Downloaded 34GiB to GPU in 2.50307s (13.5833GiB/s) | 20260508 00:30:24.695 | INFO | Analyzing chunk frames = [3814912,3815424) | 20260508 00:30:24.696 | INFO | MemInfo: { | 20260508 00:30:24.696 | INFO | "gpu" : " 17 GB", | 20260508 00:30:24.696 | INFO | "huge" : "", | 20260508 00:30:24.696 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:24.696 | INFO | "malloc" : " 14 GB", | 20260508 00:30:24.696 | INFO | "pinned" : " 29 GB" | 20260508 00:30:24.696 | INFO | } | 20260508 00:30:24.696 | INFO | Perf report for the last 5.14176 seconds: | 20260508 00:30:24.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:24.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 00:30:24.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:30:24.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:24.696 | INFO | Uploaded 36GiB to GPU in 2.45325s (14.6744GiB/s) | 20260508 00:30:24.696 | INFO | Downloaded 34GiB to GPU in 2.49007s (13.6542GiB/s) | 20260508 00:30:29.839 | INFO | Analyzing chunk frames = [3815424,3815936) | 20260508 00:30:29.839 | INFO | MemInfo: { | 20260508 00:30:29.839 | INFO | "gpu" : " 17 GB", | 20260508 00:30:29.839 | INFO | "huge" : "", | 20260508 00:30:29.839 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:29.839 | INFO | "malloc" : " 14 GB", | 20260508 00:30:29.839 | INFO | "pinned" : " 29 GB" | 20260508 00:30:29.839 | INFO | } | 20260508 00:30:29.839 | INFO | Perf report for the last 5.14305 seconds: | 20260508 00:30:29.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:29.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.839 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:30:29.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.839 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 00:30:29.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:29.840 | INFO | Uploaded 36GiB to GPU in 2.32794s (15.4643GiB/s) | 20260508 00:30:29.840 | INFO | Downloaded 34GiB to GPU in 2.50103s (13.5944GiB/s) | 20260508 00:30:34.974 | INFO | Analyzing chunk frames = [3815936,3816448) | 20260508 00:30:34.974 | INFO | MemInfo: { | 20260508 00:30:34.974 | INFO | "gpu" : " 17 GB", | 20260508 00:30:34.974 | INFO | "huge" : "", | 20260508 00:30:34.974 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:34.974 | INFO | "malloc" : " 14 GB", | 20260508 00:30:34.974 | INFO | "pinned" : " 29 GB" | 20260508 00:30:34.974 | INFO | } | 20260508 00:30:34.974 | INFO | Perf report for the last 5.13502 seconds: | 20260508 00:30:34.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:34.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:30:34.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:30:34.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:34.974 | INFO | Uploaded 36GiB to GPU in 2.39674s (15.0204GiB/s) | 20260508 00:30:34.974 | INFO | Downloaded 34GiB to GPU in 2.40883s (14.1148GiB/s) | 20260508 00:30:40.111 | INFO | Analyzing chunk frames = [3816448,3816960) | 20260508 00:30:40.111 | INFO | MemInfo: { | 20260508 00:30:40.111 | INFO | "gpu" : " 17 GB", | 20260508 00:30:40.111 | INFO | "huge" : "", | 20260508 00:30:40.111 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:40.111 | INFO | "malloc" : " 14 GB", | 20260508 00:30:40.111 | INFO | "pinned" : " 29 GB" | 20260508 00:30:40.111 | INFO | } | 20260508 00:30:40.111 | INFO | Perf report for the last 5.13738 seconds: | 20260508 00:30:40.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:40.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.111 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:30:40.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.112 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:30:40.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:40.112 | INFO | Uploaded 36GiB to GPU in 2.52893s (14.2352GiB/s) | 20260508 00:30:40.112 | INFO | Downloaded 34GiB to GPU in 2.45766s (13.8343GiB/s) | 20260508 00:30:45.250 | INFO | Analyzing chunk frames = [3816960,3817472) | 20260508 00:30:45.250 | INFO | MemInfo: { | 20260508 00:30:45.250 | INFO | "gpu" : " 17 GB", | 20260508 00:30:45.250 | INFO | "huge" : "", | 20260508 00:30:45.250 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:45.250 | INFO | "malloc" : " 14 GB", | 20260508 00:30:45.250 | INFO | "pinned" : " 29 GB" | 20260508 00:30:45.251 | INFO | } | 20260508 00:30:45.251 | INFO | Perf report for the last 5.1392 seconds: | 20260508 00:30:45.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:45.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:30:45.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 00:30:45.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:45.251 | INFO | Uploaded 36GiB to GPU in 2.48977s (14.4592GiB/s) | 20260508 00:30:45.251 | INFO | Downloaded 34GiB to GPU in 2.44731s (13.8928GiB/s) | 20260508 00:30:45.251 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:30:45.251 | INFO | Total mem usage: 16.7 GiB | 20260508 00:30:45.251 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:30:45.251 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 00:30:45.251 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:30:45.251 | INFO | High Water Breakdown: | 20260508 00:30:45.251 | INFO | 60.2% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:30:45.251 | INFO | 39.7% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:30:45.251 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:30:45.251 | INFO | ------------------------------------------ | 20260508 00:30:45.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:30:45.251 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:30:45.251 | INFO | Total mem usage: 26.5 GiB | 20260508 00:30:45.251 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:30:45.251 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:30:45.251 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:30:45.251 | INFO | High Water Breakdown: | 20260508 00:30:45.251 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:30:45.251 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:30:45.251 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:30:45.251 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:30:45.251 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:30:45.251 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:30:45.251 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:30:45.251 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:30:45.251 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:30:45.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:30:45.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:30:45.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:30:45.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:30:45.251 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:30:45.251 | INFO | ------------------------------------------ | 20260508 00:30:45.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:30:45.251 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:30:45.251 | INFO | Total mem usage: 27.8 GiB | 20260508 00:30:45.251 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:30:45.251 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:30:45.251 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:30:45.251 | INFO | High Water Breakdown: | 20260508 00:30:45.251 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:30:45.251 | INFO | ------------------------------------------ | 20260508 00:30:45.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:30:45.251 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:30:45.251 | INFO | Total mem usage: 15.7 GiB | 20260508 00:30:45.251 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:30:45.251 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:30:45.251 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:30:45.251 | INFO | High Water Breakdown: | 20260508 00:30:45.251 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:30:45.251 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:30:45.251 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:30:45.251 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:30:45.251 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:30:45.251 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:30:45.251 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:30:45.251 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:30:45.251 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:30:45.251 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:30:45.251 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:30:45.251 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:30:45.251 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:30:45.251 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:30:45.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:30:45.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:30:45.251 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:30:45.251 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:30:45.251 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:30:45.251 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:30:45.251 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:30:45.251 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:30:45.251 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:30:45.251 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:30:45.251 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:30:45.251 | INFO | ------------------------------------------ | 20260508 00:30:45.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:30:45.251 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:30:45.251 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:30:45.251 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:30:45.251 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.64e+01, 1.42e+01 / 1.73e+00 | 20260508 00:30:45.251 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.10e-02 / 4.04e-03 | 20260508 00:30:45.251 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.50e+00, 5.85e-01 / 1.73e-01 | 20260508 00:30:45.251 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.00e-01 | 20260508 00:30:45.251 | INFO | PerfReport: 16.02%, DME -- 4.20e+01 / 4.60e+01, 4.16e+00 / 1.83e+00 | 20260508 00:30:45.251 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.64e+00 / 1.93e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.44e+00 / 2.71e+00, 4.24e-01 / 2.27e-01 | 20260508 00:30:45.252 | INFO | PerfReport: 3.40%, Metrics -- 1.11e+00 / 1.33e+00, 2.77e-01 / 1.74e-01 | 20260508 00:30:45.252 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.75e-01, 5.38e-02 / 2.64e-02 | 20260508 00:30:45.252 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:30:45.252 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:30:45.252 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:30:45.252 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 24.13%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 1.94%, LockUpload -- 2.07e+00 / 0.00e+00, 5.24e+00 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 2.96%, LockDownload -- 3.16e+00 / 0.00e+00, 4.38e+00 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 15.78%, LockCompute -- 1.68e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 00:30:45.252 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.12e-02 / 0.00e+00 | 20260508 00:30:45.350 | INFO | Starting a round of Baz IO | 20260508 00:30:48.812 | INFO | Populating RTMetrics report | 20260508 00:30:48.812 | INFO | RTMetrics report populated | 20260508 00:30:48.812 | INFO | Opening tmp RTMetrics json file | 20260508 00:30:48.832 | INFO | Serializing RTMetrics to json | 20260508 00:30:48.839 | INFO | Serialization done, swapping tmp file now... | 20260508 00:30:48.840 | INFO | RTMetrics done | 20260508 00:30:48.910 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.559 seconds | 20260508 00:30:48.910 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.17% / 0.00% / 57.83% | 20260508 00:30:48.910 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.999415% / 1.58946e-05% / 7.15256e-05% | 20260508 00:30:48.910 | INFO | Baz IO overall: | 20260508 00:30:48.910 | INFO | BytesWritten : 1734455008209 | 20260508 00:30:48.910 | INFO | NumEvents : 839449588126 | 20260508 00:30:50.394 | INFO | Analyzing chunk frames = [3817472,3817984) | 20260508 00:30:50.394 | INFO | MemInfo: { | 20260508 00:30:50.394 | INFO | "gpu" : " 17 GB", | 20260508 00:30:50.394 | INFO | "huge" : "", | 20260508 00:30:50.394 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:50.394 | INFO | "malloc" : " 14 GB", | 20260508 00:30:50.394 | INFO | "pinned" : " 29 GB" | 20260508 00:30:50.394 | INFO | } | 20260508 00:30:50.394 | INFO | Perf report for the last 5.1433 seconds: | 20260508 00:30:50.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:50.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:50.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:50.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:50.394 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260508 00:30:50.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:50.394 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 00:30:50.394 | INFO | PRE_HQ : 65.4%, 1.7, 56.8ms, 0 | 20260508 00:30:50.394 | INFO | BAZWRITER : 68.8%, 4.3, 151.6ms, 0 | 20260508 00:30:50.394 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 00:30:50.394 | INFO | Uploaded 36GiB to GPU in 2.54484s (14.1463GiB/s) | 20260508 00:30:50.394 | INFO | Downloaded 38GiB to GPU in 2.6855s (14.1501GiB/s) | 20260508 00:30:55.535 | INFO | Analyzing chunk frames = [3817984,3818496) | 20260508 00:30:55.535 | INFO | MemInfo: { | 20260508 00:30:55.535 | INFO | "gpu" : " 17 GB", | 20260508 00:30:55.535 | INFO | "huge" : "", | 20260508 00:30:55.535 | INFO | "huge_shared" : " 29 GB", | 20260508 00:30:55.535 | INFO | "malloc" : " 14 GB", | 20260508 00:30:55.535 | INFO | "pinned" : " 29 GB" | 20260508 00:30:55.535 | INFO | } | 20260508 00:30:55.536 | INFO | Perf report for the last 5.14166 seconds: | 20260508 00:30:55.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:30:55.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 00:30:55.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:30:55.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:30:55.536 | INFO | Uploaded 36GiB to GPU in 2.57591s (13.9756GiB/s) | 20260508 00:30:55.536 | INFO | Downloaded 34GiB to GPU in 2.49266s (13.6401GiB/s) | 20260508 00:31:00.677 | INFO | Analyzing chunk frames = [3818496,3819008) | 20260508 00:31:00.677 | INFO | MemInfo: { | 20260508 00:31:00.677 | INFO | "gpu" : " 17 GB", | 20260508 00:31:00.677 | INFO | "huge" : "", | 20260508 00:31:00.677 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:00.677 | INFO | "malloc" : " 14 GB", | 20260508 00:31:00.677 | INFO | "pinned" : " 29 GB" | 20260508 00:31:00.677 | INFO | } | 20260508 00:31:00.677 | INFO | Perf report for the last 5.14129 seconds: | 20260508 00:31:00.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:00.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.677 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:31:00.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.677 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 00:31:00.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:00.678 | INFO | Uploaded 36GiB to GPU in 2.51521s (14.3129GiB/s) | 20260508 00:31:00.678 | INFO | Downloaded 34GiB to GPU in 2.49418s (13.6317GiB/s) | 20260508 00:31:05.809 | INFO | Analyzing chunk frames = [3819008,3819520) | 20260508 00:31:05.809 | INFO | MemInfo: { | 20260508 00:31:05.809 | INFO | "gpu" : " 17 GB", | 20260508 00:31:05.809 | INFO | "huge" : "", | 20260508 00:31:05.809 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:05.809 | INFO | "malloc" : " 14 GB", | 20260508 00:31:05.809 | INFO | "pinned" : " 29 GB" | 20260508 00:31:05.809 | INFO | } | 20260508 00:31:05.809 | INFO | Perf report for the last 5.13178 seconds: | 20260508 00:31:05.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:05.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | ANALYSIS : 67.4%, 3.0, 104.1ms, 0 | 20260508 00:31:05.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:31:05.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:05.809 | INFO | Uploaded 36GiB to GPU in 2.51564s (14.3105GiB/s) | 20260508 00:31:05.809 | INFO | Downloaded 34GiB to GPU in 2.49493s (13.6276GiB/s) | 20260508 00:31:10.951 | INFO | Analyzing chunk frames = [3819520,3820032) | 20260508 00:31:10.951 | INFO | MemInfo: { | 20260508 00:31:10.951 | INFO | "gpu" : " 17 GB", | 20260508 00:31:10.951 | INFO | "huge" : "", | 20260508 00:31:10.951 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:10.951 | INFO | "malloc" : " 14 GB", | 20260508 00:31:10.951 | INFO | "pinned" : " 29 GB" | 20260508 00:31:10.951 | INFO | } | 20260508 00:31:10.951 | INFO | Perf report for the last 5.14268 seconds: | 20260508 00:31:10.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:10.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.951 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 00:31:10.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.952 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 00:31:10.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:10.952 | INFO | Uploaded 36GiB to GPU in 2.38438s (15.0983GiB/s) | 20260508 00:31:10.952 | INFO | Downloaded 34GiB to GPU in 2.50507s (13.5725GiB/s) | 20260508 00:31:16.091 | INFO | Analyzing chunk frames = [3820032,3820544) | 20260508 00:31:16.091 | INFO | MemInfo: { | 20260508 00:31:16.091 | INFO | "gpu" : " 17 GB", | 20260508 00:31:16.091 | INFO | "huge" : "", | 20260508 00:31:16.091 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:16.091 | INFO | "malloc" : " 14 GB", | 20260508 00:31:16.091 | INFO | "pinned" : " 29 GB" | 20260508 00:31:16.091 | INFO | } | 20260508 00:31:16.091 | INFO | Perf report for the last 5.13954 seconds: | 20260508 00:31:16.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:16.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.091 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 00:31:16.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.091 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:31:16.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:16.092 | INFO | Uploaded 36GiB to GPU in 2.3949s (15.032GiB/s) | 20260508 00:31:16.092 | INFO | Downloaded 34GiB to GPU in 2.40519s (14.1361GiB/s) | 20260508 00:31:21.234 | INFO | Analyzing chunk frames = [3820544,3821056) | 20260508 00:31:21.234 | INFO | MemInfo: { | 20260508 00:31:21.234 | INFO | "gpu" : " 17 GB", | 20260508 00:31:21.234 | INFO | "huge" : "", | 20260508 00:31:21.234 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:21.234 | INFO | "malloc" : " 14 GB", | 20260508 00:31:21.234 | INFO | "pinned" : " 29 GB" | 20260508 00:31:21.234 | INFO | } | 20260508 00:31:21.234 | INFO | Perf report for the last 5.14284 seconds: | 20260508 00:31:21.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:21.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 00:31:21.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:31:21.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:21.234 | INFO | Uploaded 36GiB to GPU in 2.4119s (14.926GiB/s) | 20260508 00:31:21.234 | INFO | Downloaded 34GiB to GPU in 2.41816s (14.0603GiB/s) | 20260508 00:31:26.372 | INFO | Analyzing chunk frames = [3821056,3821568) | 20260508 00:31:26.372 | INFO | MemInfo: { | 20260508 00:31:26.372 | INFO | "gpu" : " 17 GB", | 20260508 00:31:26.372 | INFO | "huge" : "", | 20260508 00:31:26.372 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:26.372 | INFO | "malloc" : " 14 GB", | 20260508 00:31:26.372 | INFO | "pinned" : " 29 GB" | 20260508 00:31:26.372 | INFO | } | 20260508 00:31:26.372 | INFO | Perf report for the last 5.1382 seconds: | 20260508 00:31:26.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:26.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.372 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:31:26.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.372 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 00:31:26.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:26.373 | INFO | Uploaded 36GiB to GPU in 2.53734s (14.1881GiB/s) | 20260508 00:31:26.373 | INFO | Downloaded 34GiB to GPU in 2.45283s (13.8615GiB/s) | 20260508 00:31:26.469 | INFO | Starting a round of Baz IO | 20260508 00:31:29.922 | INFO | Populating RTMetrics report | 20260508 00:31:29.922 | INFO | RTMetrics report populated | 20260508 00:31:29.922 | INFO | Opening tmp RTMetrics json file | 20260508 00:31:30.012 | INFO | Serializing RTMetrics to json | 20260508 00:31:30.067 | INFO | Serialization done, swapping tmp file now... | 20260508 00:31:30.087 | INFO | RTMetrics done | 20260508 00:31:30.237 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.768 seconds | 20260508 00:31:30.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.16% / 0.00% / 57.83% | 20260508 00:31:30.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.996037% / 2.38419e-05% / 4.37101e-05% | 20260508 00:31:30.238 | INFO | Baz IO overall: | 20260508 00:31:30.238 | INFO | BytesWritten : 1736265675767 | 20260508 00:31:30.238 | INFO | NumEvents : 840320431639 | 20260508 00:31:31.509 | INFO | Analyzing chunk frames = [3821568,3822080) | 20260508 00:31:31.509 | INFO | MemInfo: { | 20260508 00:31:31.509 | INFO | "gpu" : " 17 GB", | 20260508 00:31:31.509 | INFO | "huge" : "", | 20260508 00:31:31.509 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:31.509 | INFO | "malloc" : " 14 GB", | 20260508 00:31:31.509 | INFO | "pinned" : " 29 GB" | 20260508 00:31:31.509 | INFO | } | 20260508 00:31:31.509 | INFO | Perf report for the last 5.13681 seconds: | 20260508 00:31:31.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:31.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:31.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:31.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:31.509 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 00:31:31.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:31.509 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 00:31:31.509 | INFO | PRE_HQ : 65.5%, 1.7, 56.7ms, 0 | 20260508 00:31:31.509 | INFO | BAZWRITER : 73.0%, 3.7, 139.7ms, 0 | 20260508 00:31:31.509 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260508 00:31:31.509 | INFO | Uploaded 36GiB to GPU in 2.56797s (14.0189GiB/s) | 20260508 00:31:31.509 | INFO | Downloaded 38GiB to GPU in 2.67712s (14.1944GiB/s) | 20260508 00:31:36.649 | INFO | Analyzing chunk frames = [3822080,3822592) | 20260508 00:31:36.649 | INFO | MemInfo: { | 20260508 00:31:36.649 | INFO | "gpu" : " 17 GB", | 20260508 00:31:36.649 | INFO | "huge" : "", | 20260508 00:31:36.649 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:36.649 | INFO | "malloc" : " 14 GB", | 20260508 00:31:36.649 | INFO | "pinned" : " 29 GB" | 20260508 00:31:36.649 | INFO | } | 20260508 00:31:36.649 | INFO | Perf report for the last 5.13971 seconds: | 20260508 00:31:36.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:36.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:31:36.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 00:31:36.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:36.649 | INFO | Uploaded 36GiB to GPU in 2.52536s (14.2554GiB/s) | 20260508 00:31:36.649 | INFO | Downloaded 34GiB to GPU in 2.42878s (13.9988GiB/s) | 20260508 00:31:41.789 | INFO | Analyzing chunk frames = [3822592,3823104) | 20260508 00:31:41.789 | INFO | MemInfo: { | 20260508 00:31:41.789 | INFO | "gpu" : " 17 GB", | 20260508 00:31:41.789 | INFO | "huge" : "", | 20260508 00:31:41.789 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:41.789 | INFO | "malloc" : " 14 GB", | 20260508 00:31:41.789 | INFO | "pinned" : " 29 GB" | 20260508 00:31:41.789 | INFO | } | 20260508 00:31:41.789 | INFO | Perf report for the last 5.14053 seconds: | 20260508 00:31:41.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:41.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.789 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:31:41.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.789 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:31:41.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:41.790 | INFO | Uploaded 36GiB to GPU in 2.50958s (14.345GiB/s) | 20260508 00:31:41.790 | INFO | Downloaded 34GiB to GPU in 2.48227s (13.6971GiB/s) | 20260508 00:31:46.924 | INFO | Analyzing chunk frames = [3823104,3823616) | 20260508 00:31:46.924 | INFO | MemInfo: { | 20260508 00:31:46.924 | INFO | "gpu" : " 17 GB", | 20260508 00:31:46.924 | INFO | "huge" : "", | 20260508 00:31:46.924 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:46.924 | INFO | "malloc" : " 14 GB", | 20260508 00:31:46.924 | INFO | "pinned" : " 29 GB" | 20260508 00:31:46.924 | INFO | } | 20260508 00:31:46.924 | INFO | Perf report for the last 5.13498 seconds: | 20260508 00:31:46.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:46.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 00:31:46.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:31:46.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:46.924 | INFO | Uploaded 36GiB to GPU in 2.507s (14.3598GiB/s) | 20260508 00:31:46.925 | INFO | Downloaded 34GiB to GPU in 2.47928s (13.7136GiB/s) | 20260508 00:31:52.059 | INFO | Analyzing chunk frames = [3823616,3824128) | 20260508 00:31:52.059 | INFO | MemInfo: { | 20260508 00:31:52.059 | INFO | "gpu" : " 17 GB", | 20260508 00:31:52.059 | INFO | "huge" : "", | 20260508 00:31:52.059 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:52.059 | INFO | "malloc" : " 14 GB", | 20260508 00:31:52.059 | INFO | "pinned" : " 29 GB" | 20260508 00:31:52.059 | INFO | } | 20260508 00:31:52.059 | INFO | Perf report for the last 5.13494 seconds: | 20260508 00:31:52.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:52.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:31:52.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:31:52.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:52.059 | INFO | Uploaded 36GiB to GPU in 2.34952s (15.3223GiB/s) | 20260508 00:31:52.059 | INFO | Downloaded 34GiB to GPU in 2.50348s (13.5811GiB/s) | 20260508 00:31:57.205 | INFO | Analyzing chunk frames = [3824128,3824640) | 20260508 00:31:57.205 | INFO | MemInfo: { | 20260508 00:31:57.205 | INFO | "gpu" : " 17 GB", | 20260508 00:31:57.205 | INFO | "huge" : "", | 20260508 00:31:57.205 | INFO | "huge_shared" : " 29 GB", | 20260508 00:31:57.205 | INFO | "malloc" : " 14 GB", | 20260508 00:31:57.205 | INFO | "pinned" : " 29 GB" | 20260508 00:31:57.205 | INFO | } | 20260508 00:31:57.205 | INFO | Perf report for the last 5.14595 seconds: | 20260508 00:31:57.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:31:57.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260508 00:31:57.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:31:57.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:31:57.205 | INFO | Uploaded 36GiB to GPU in 2.48801s (14.4694GiB/s) | 20260508 00:31:57.205 | INFO | Downloaded 34GiB to GPU in 2.43762s (13.948GiB/s) | 20260508 00:32:02.346 | INFO | Analyzing chunk frames = [3824640,3825152) | 20260508 00:32:02.346 | INFO | MemInfo: { | 20260508 00:32:02.346 | INFO | "gpu" : " 17 GB", | 20260508 00:32:02.346 | INFO | "huge" : "", | 20260508 00:32:02.346 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:02.346 | INFO | "malloc" : " 14 GB", | 20260508 00:32:02.346 | INFO | "pinned" : " 29 GB" | 20260508 00:32:02.346 | INFO | } | 20260508 00:32:02.346 | INFO | Perf report for the last 5.14144 seconds: | 20260508 00:32:02.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:02.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 00:32:02.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:32:02.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:02.347 | INFO | Uploaded 36GiB to GPU in 2.48915s (14.4627GiB/s) | 20260508 00:32:02.347 | INFO | Downloaded 34GiB to GPU in 2.46322s (13.8031GiB/s) | 20260508 00:32:07.491 | INFO | Analyzing chunk frames = [3825152,3825664) | 20260508 00:32:07.491 | INFO | MemInfo: { | 20260508 00:32:07.491 | INFO | "gpu" : " 17 GB", | 20260508 00:32:07.491 | INFO | "huge" : "", | 20260508 00:32:07.491 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:07.491 | INFO | "malloc" : " 14 GB", | 20260508 00:32:07.491 | INFO | "pinned" : " 29 GB" | 20260508 00:32:07.491 | INFO | } | 20260508 00:32:07.491 | INFO | Perf report for the last 5.14458 seconds: | 20260508 00:32:07.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:07.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 00:32:07.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 00:32:07.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:07.491 | INFO | Uploaded 36GiB to GPU in 2.50031s (14.3982GiB/s) | 20260508 00:32:07.491 | INFO | Downloaded 34GiB to GPU in 2.45938s (13.8246GiB/s) | 20260508 00:32:07.491 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:32:07.491 | INFO | Total mem usage: 16.7 GiB | 20260508 00:32:07.491 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:32:07.491 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.44%) | 20260508 00:32:07.491 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:32:07.491 | INFO | High Water Breakdown: | 20260508 00:32:07.491 | INFO | 60.5% : 9.22 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:32:07.491 | INFO | 39.4% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:32:07.491 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:32:07.491 | INFO | ------------------------------------------ | 20260508 00:32:07.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:32:07.491 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:32:07.491 | INFO | Total mem usage: 26.5 GiB | 20260508 00:32:07.491 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:32:07.491 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:32:07.491 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:32:07.491 | INFO | High Water Breakdown: | 20260508 00:32:07.491 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:32:07.491 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:32:07.491 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:32:07.491 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:32:07.491 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:32:07.491 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:32:07.491 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:32:07.491 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:32:07.491 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:32:07.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:32:07.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:32:07.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:32:07.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:32:07.491 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:32:07.491 | INFO | ------------------------------------------ | 20260508 00:32:07.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:32:07.491 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:32:07.491 | INFO | Total mem usage: 27.8 GiB | 20260508 00:32:07.491 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:32:07.491 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:32:07.491 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:32:07.491 | INFO | High Water Breakdown: | 20260508 00:32:07.491 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:32:07.491 | INFO | ------------------------------------------ | 20260508 00:32:07.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:32:07.491 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:32:07.492 | INFO | Total mem usage: 15.7 GiB | 20260508 00:32:07.492 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:32:07.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:32:07.492 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:32:07.492 | INFO | High Water Breakdown: | 20260508 00:32:07.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:32:07.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:32:07.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:32:07.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:32:07.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:32:07.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:32:07.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:32:07.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:32:07.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:32:07.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:32:07.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:32:07.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:32:07.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:32:07.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:32:07.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:32:07.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:32:07.492 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:32:07.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:32:07.492 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:32:07.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:32:07.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:32:07.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:32:07.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:32:07.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:32:07.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:32:07.492 | INFO | ------------------------------------------ | 20260508 00:32:07.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:32:07.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:32:07.492 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:32:07.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:32:07.492 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.40e+01 / 1.32e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.19e-02 / 5.53e-03 | 20260508 00:32:07.492 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.47e+00, 5.86e-01 / 2.15e-01 | 20260508 00:32:07.492 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.06e-01 | 20260508 00:32:07.492 | INFO | PerfReport: 16.11%, DME -- 4.22e+01 / 4.61e+01, 4.00e+00 / 1.64e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.62e+00 / 1.60e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.31e-01 / 1.88e-01 | 20260508 00:32:07.492 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.33e+00, 2.76e-01 / 1.52e-01 | 20260508 00:32:07.492 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.82e-01, 5.43e-02 / 3.02e-02 | 20260508 00:32:07.492 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:32:07.492 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:32:07.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:32:07.492 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 24.30%, Upload -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 24.34%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 30.72%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 2.10%, LockUpload -- 2.24e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 15.71%, LockCompute -- 1.68e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:32:07.492 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 00:32:07.588 | INFO | Starting a round of Baz IO | 20260508 00:32:11.047 | INFO | Populating RTMetrics report | 20260508 00:32:11.047 | INFO | RTMetrics report populated | 20260508 00:32:11.047 | INFO | Opening tmp RTMetrics json file | 20260508 00:32:11.068 | INFO | Serializing RTMetrics to json | 20260508 00:32:11.074 | INFO | Serialization done, swapping tmp file now... | 20260508 00:32:11.075 | INFO | RTMetrics done | 20260508 00:32:11.144 | INFO | Finished a round of Baz IO: Wrote 1.687 GiB in 3.555 seconds | 20260508 00:32:11.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.16% / 0.00% / 57.84% | 20260508 00:32:11.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.995751% / 1.58946e-05% / 4.76837e-05% | 20260508 00:32:11.144 | INFO | Baz IO overall: | 20260508 00:32:11.144 | INFO | BytesWritten : 1738076646959 | 20260508 00:32:11.144 | INFO | NumEvents : 841191572985 | 20260508 00:32:12.627 | INFO | Analyzing chunk frames = [3825664,3826176) | 20260508 00:32:12.627 | INFO | MemInfo: { | 20260508 00:32:12.627 | INFO | "gpu" : " 17 GB", | 20260508 00:32:12.627 | INFO | "huge" : "", | 20260508 00:32:12.627 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:12.627 | INFO | "malloc" : " 14 GB", | 20260508 00:32:12.627 | INFO | "pinned" : " 29 GB" | 20260508 00:32:12.627 | INFO | } | 20260508 00:32:12.627 | INFO | Perf report for the last 5.13558 seconds: | 20260508 00:32:12.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:12.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:12.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:12.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:12.627 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 00:32:12.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:12.627 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 00:32:12.627 | INFO | PRE_HQ : 65.7%, 1.7, 56.7ms, 0 | 20260508 00:32:12.627 | INFO | BAZWRITER : 68.8%, 3.7, 129.5ms, 0 | 20260508 00:32:12.627 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 00:32:12.627 | INFO | Uploaded 36GiB to GPU in 2.57547s (13.978GiB/s) | 20260508 00:32:12.627 | INFO | Downloaded 38GiB to GPU in 2.68231s (14.1669GiB/s) | 20260508 00:32:17.764 | INFO | Analyzing chunk frames = [3826176,3826688) | 20260508 00:32:17.764 | INFO | MemInfo: { | 20260508 00:32:17.764 | INFO | "gpu" : " 17 GB", | 20260508 00:32:17.764 | INFO | "huge" : "", | 20260508 00:32:17.764 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:17.764 | INFO | "malloc" : " 14 GB", | 20260508 00:32:17.764 | INFO | "pinned" : " 29 GB" | 20260508 00:32:17.764 | INFO | } | 20260508 00:32:17.765 | INFO | Perf report for the last 5.13781 seconds: | 20260508 00:32:17.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:17.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:32:17.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:32:17.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:17.765 | INFO | Uploaded 36GiB to GPU in 2.51031s (14.3408GiB/s) | 20260508 00:32:17.765 | INFO | Downloaded 34GiB to GPU in 2.43849s (13.9431GiB/s) | 20260508 00:32:22.904 | INFO | Analyzing chunk frames = [3826688,3827200) | 20260508 00:32:22.904 | INFO | MemInfo: { | 20260508 00:32:22.904 | INFO | "gpu" : " 17 GB", | 20260508 00:32:22.904 | INFO | "huge" : "", | 20260508 00:32:22.905 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:22.905 | INFO | "malloc" : " 14 GB", | 20260508 00:32:22.905 | INFO | "pinned" : " 29 GB" | 20260508 00:32:22.905 | INFO | } | 20260508 00:32:22.905 | INFO | Perf report for the last 5.14019 seconds: | 20260508 00:32:22.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:22.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:32:22.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:32:22.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:22.905 | INFO | Uploaded 36GiB to GPU in 2.46746s (14.5899GiB/s) | 20260508 00:32:22.905 | INFO | Downloaded 34GiB to GPU in 2.50869s (13.5529GiB/s) | 20260508 00:32:28.043 | INFO | Analyzing chunk frames = [3827200,3827712) | 20260508 00:32:28.043 | INFO | MemInfo: { | 20260508 00:32:28.043 | INFO | "gpu" : " 17 GB", | 20260508 00:32:28.043 | INFO | "huge" : "", | 20260508 00:32:28.043 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:28.043 | INFO | "malloc" : " 14 GB", | 20260508 00:32:28.043 | INFO | "pinned" : " 29 GB" | 20260508 00:32:28.043 | INFO | } | 20260508 00:32:28.043 | INFO | Perf report for the last 5.13867 seconds: | 20260508 00:32:28.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:28.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.043 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 00:32:28.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.043 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:32:28.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:28.044 | INFO | Uploaded 36GiB to GPU in 2.45251s (14.6788GiB/s) | 20260508 00:32:28.044 | INFO | Downloaded 34GiB to GPU in 2.50622s (13.5663GiB/s) | 20260508 00:32:33.184 | INFO | Analyzing chunk frames = [3827712,3828224) | 20260508 00:32:33.184 | INFO | MemInfo: { | 20260508 00:32:33.184 | INFO | "gpu" : " 17 GB", | 20260508 00:32:33.184 | INFO | "huge" : "", | 20260508 00:32:33.184 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:33.184 | INFO | "malloc" : " 14 GB", | 20260508 00:32:33.184 | INFO | "pinned" : " 29 GB" | 20260508 00:32:33.184 | INFO | } | 20260508 00:32:33.184 | INFO | Perf report for the last 5.14027 seconds: | 20260508 00:32:33.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:33.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 00:32:33.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 00:32:33.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:33.184 | INFO | Uploaded 36GiB to GPU in 2.30834s (15.5956GiB/s) | 20260508 00:32:33.184 | INFO | Downloaded 34GiB to GPU in 2.48271s (13.6947GiB/s) | 20260508 00:32:38.315 | INFO | Analyzing chunk frames = [3828224,3828736) | 20260508 00:32:38.315 | INFO | MemInfo: { | 20260508 00:32:38.315 | INFO | "gpu" : " 17 GB", | 20260508 00:32:38.316 | INFO | "huge" : "", | 20260508 00:32:38.316 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:38.316 | INFO | "malloc" : " 14 GB", | 20260508 00:32:38.316 | INFO | "pinned" : " 29 GB" | 20260508 00:32:38.316 | INFO | } | 20260508 00:32:38.316 | INFO | Perf report for the last 5.13195 seconds: | 20260508 00:32:38.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:38.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 00:32:38.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:32:38.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:38.316 | INFO | Uploaded 36GiB to GPU in 2.37977s (15.1275GiB/s) | 20260508 00:32:38.316 | INFO | Downloaded 34GiB to GPU in 2.40479s (14.1384GiB/s) | 20260508 00:32:43.459 | INFO | Analyzing chunk frames = [3828736,3829248) | 20260508 00:32:43.459 | INFO | MemInfo: { | 20260508 00:32:43.459 | INFO | "gpu" : " 17 GB", | 20260508 00:32:43.459 | INFO | "huge" : "", | 20260508 00:32:43.459 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:43.459 | INFO | "malloc" : " 14 GB", | 20260508 00:32:43.459 | INFO | "pinned" : " 29 GB" | 20260508 00:32:43.459 | INFO | } | 20260508 00:32:43.459 | INFO | Perf report for the last 5.14301 seconds: | 20260508 00:32:43.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:43.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:32:43.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260508 00:32:43.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:43.459 | INFO | Uploaded 36GiB to GPU in 2.55285s (14.1019GiB/s) | 20260508 00:32:43.459 | INFO | Downloaded 34GiB to GPU in 2.4722s (13.7529GiB/s) | 20260508 00:32:48.601 | INFO | Analyzing chunk frames = [3829248,3829760) | 20260508 00:32:48.601 | INFO | MemInfo: { | 20260508 00:32:48.601 | INFO | "gpu" : " 17 GB", | 20260508 00:32:48.601 | INFO | "huge" : "", | 20260508 00:32:48.601 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:48.601 | INFO | "malloc" : " 14 GB", | 20260508 00:32:48.601 | INFO | "pinned" : " 29 GB" | 20260508 00:32:48.601 | INFO | } | 20260508 00:32:48.601 | INFO | Perf report for the last 5.14242 seconds: | 20260508 00:32:48.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:48.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.601 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 00:32:48.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.601 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 00:32:48.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:48.602 | INFO | Uploaded 36GiB to GPU in 2.49819s (14.4104GiB/s) | 20260508 00:32:48.602 | INFO | Downloaded 34GiB to GPU in 2.44667s (13.8964GiB/s) | 20260508 00:32:48.698 | INFO | Starting a round of Baz IO | 20260508 00:32:52.158 | INFO | Populating RTMetrics report | 20260508 00:32:52.158 | INFO | RTMetrics report populated | 20260508 00:32:52.158 | INFO | Opening tmp RTMetrics json file | 20260508 00:32:52.160 | INFO | Serializing RTMetrics to json | 20260508 00:32:52.183 | INFO | Serialization done, swapping tmp file now... | 20260508 00:32:52.185 | INFO | RTMetrics done | 20260508 00:32:52.271 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.573 seconds | 20260508 00:32:52.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.15% / 0.00% / 57.84% | 20260508 00:32:52.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.992382% / 3.97364e-06% / 7.54992e-05% | 20260508 00:32:52.271 | INFO | Baz IO overall: | 20260508 00:32:52.271 | INFO | BytesWritten : 1739886522196 | 20260508 00:32:52.271 | INFO | NumEvents : 842061948865 | 20260508 00:32:53.745 | INFO | Analyzing chunk frames = [3829760,3830272) | 20260508 00:32:53.745 | INFO | MemInfo: { | 20260508 00:32:53.745 | INFO | "gpu" : " 17 GB", | 20260508 00:32:53.745 | INFO | "huge" : "", | 20260508 00:32:53.745 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:53.745 | INFO | "malloc" : " 14 GB", | 20260508 00:32:53.745 | INFO | "pinned" : " 29 GB" | 20260508 00:32:53.745 | INFO | } | 20260508 00:32:53.745 | INFO | Perf report for the last 5.14384 seconds: | 20260508 00:32:53.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:53.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:53.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:53.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:53.745 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260508 00:32:53.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:53.745 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 00:32:53.745 | INFO | PRE_HQ : 65.5%, 1.7, 56.5ms, 0 | 20260508 00:32:53.745 | INFO | BAZWRITER : 69.1%, 4.1, 146.6ms, 0 | 20260508 00:32:53.745 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 00:32:53.746 | INFO | Uploaded 36GiB to GPU in 2.54265s (14.1584GiB/s) | 20260508 00:32:53.746 | INFO | Downloaded 38GiB to GPU in 2.68646s (14.145GiB/s) | 20260508 00:32:58.873 | INFO | Analyzing chunk frames = [3830272,3830784) | 20260508 00:32:58.873 | INFO | MemInfo: { | 20260508 00:32:58.873 | INFO | "gpu" : " 17 GB", | 20260508 00:32:58.873 | INFO | "huge" : "", | 20260508 00:32:58.873 | INFO | "huge_shared" : " 29 GB", | 20260508 00:32:58.873 | INFO | "malloc" : " 14 GB", | 20260508 00:32:58.873 | INFO | "pinned" : " 29 GB" | 20260508 00:32:58.873 | INFO | } | 20260508 00:32:58.873 | INFO | Perf report for the last 5.1282 seconds: | 20260508 00:32:58.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:32:58.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260508 00:32:58.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 00:32:58.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:32:58.873 | INFO | Uploaded 36GiB to GPU in 2.53242s (14.2157GiB/s) | 20260508 00:32:58.873 | INFO | Downloaded 34GiB to GPU in 2.43224s (13.9789GiB/s) | 20260508 00:33:04.015 | INFO | Analyzing chunk frames = [3830784,3831296) | 20260508 00:33:04.015 | INFO | MemInfo: { | 20260508 00:33:04.015 | INFO | "gpu" : " 17 GB", | 20260508 00:33:04.015 | INFO | "huge" : "", | 20260508 00:33:04.015 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:04.015 | INFO | "malloc" : " 14 GB", | 20260508 00:33:04.015 | INFO | "pinned" : " 29 GB" | 20260508 00:33:04.015 | INFO | } | 20260508 00:33:04.015 | INFO | Perf report for the last 5.14209 seconds: | 20260508 00:33:04.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:04.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 00:33:04.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:33:04.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:04.015 | INFO | Uploaded 36GiB to GPU in 2.49983s (14.401GiB/s) | 20260508 00:33:04.015 | INFO | Downloaded 34GiB to GPU in 2.48765s (13.6675GiB/s) | 20260508 00:33:09.156 | INFO | Analyzing chunk frames = [3831296,3831808) | 20260508 00:33:09.156 | INFO | MemInfo: { | 20260508 00:33:09.156 | INFO | "gpu" : " 17 GB", | 20260508 00:33:09.156 | INFO | "huge" : "", | 20260508 00:33:09.156 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:09.156 | INFO | "malloc" : " 14 GB", | 20260508 00:33:09.156 | INFO | "pinned" : " 29 GB" | 20260508 00:33:09.156 | INFO | } | 20260508 00:33:09.156 | INFO | Perf report for the last 5.14098 seconds: | 20260508 00:33:09.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:09.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.156 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:33:09.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.156 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:33:09.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:09.157 | INFO | Uploaded 36GiB to GPU in 2.50882s (14.3494GiB/s) | 20260508 00:33:09.157 | INFO | Downloaded 34GiB to GPU in 2.48868s (13.6618GiB/s) | 20260508 00:33:14.299 | INFO | Analyzing chunk frames = [3831808,3832320) | 20260508 00:33:14.299 | INFO | MemInfo: { | 20260508 00:33:14.299 | INFO | "gpu" : " 17 GB", | 20260508 00:33:14.299 | INFO | "huge" : "", | 20260508 00:33:14.299 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:14.299 | INFO | "malloc" : " 14 GB", | 20260508 00:33:14.299 | INFO | "pinned" : " 29 GB" | 20260508 00:33:14.299 | INFO | } | 20260508 00:33:14.299 | INFO | Perf report for the last 5.14307 seconds: | 20260508 00:33:14.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:14.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.299 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 00:33:14.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.299 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 00:33:14.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:14.300 | INFO | Uploaded 36GiB to GPU in 2.3665s (15.2124GiB/s) | 20260508 00:33:14.300 | INFO | Downloaded 34GiB to GPU in 2.50387s (13.579GiB/s) | 20260508 00:33:19.440 | INFO | Analyzing chunk frames = [3832320,3832832) | 20260508 00:33:19.440 | INFO | MemInfo: { | 20260508 00:33:19.440 | INFO | "gpu" : " 17 GB", | 20260508 00:33:19.440 | INFO | "huge" : "", | 20260508 00:33:19.440 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:19.440 | INFO | "malloc" : " 14 GB", | 20260508 00:33:19.440 | INFO | "pinned" : " 29 GB" | 20260508 00:33:19.440 | INFO | } | 20260508 00:33:19.440 | INFO | Perf report for the last 5.14079 seconds: | 20260508 00:33:19.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:19.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:33:19.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:33:19.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:19.440 | INFO | Uploaded 36GiB to GPU in 2.4003s (14.9981GiB/s) | 20260508 00:33:19.440 | INFO | Downloaded 34GiB to GPU in 2.41225s (14.0947GiB/s) | 20260508 00:33:24.573 | INFO | Analyzing chunk frames = [3832832,3833344) | 20260508 00:33:24.573 | INFO | MemInfo: { | 20260508 00:33:24.573 | INFO | "gpu" : " 17 GB", | 20260508 00:33:24.573 | INFO | "huge" : "", | 20260508 00:33:24.574 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:24.574 | INFO | "malloc" : " 14 GB", | 20260508 00:33:24.574 | INFO | "pinned" : " 29 GB" | 20260508 00:33:24.574 | INFO | } | 20260508 00:33:24.574 | INFO | Perf report for the last 5.13346 seconds: | 20260508 00:33:24.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:24.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 00:33:24.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:33:24.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:24.574 | INFO | Uploaded 36GiB to GPU in 2.45475s (14.6654GiB/s) | 20260508 00:33:24.574 | INFO | Downloaded 34GiB to GPU in 2.43292s (13.975GiB/s) | 20260508 00:33:29.717 | INFO | Analyzing chunk frames = [3833344,3833856) | 20260508 00:33:29.717 | INFO | MemInfo: { | 20260508 00:33:29.717 | INFO | "gpu" : " 17 GB", | 20260508 00:33:29.717 | INFO | "huge" : "", | 20260508 00:33:29.717 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:29.717 | INFO | "malloc" : " 14 GB", | 20260508 00:33:29.717 | INFO | "pinned" : " 29 GB" | 20260508 00:33:29.717 | INFO | } | 20260508 00:33:29.717 | INFO | Perf report for the last 5.14312 seconds: | 20260508 00:33:29.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:29.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 00:33:29.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:33:29.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:29.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:33:29.717 | INFO | Total mem usage: 16.7 GiB | 20260508 00:33:29.717 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:33:29.717 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 00:33:29.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:33:29.717 | INFO | High Water Breakdown: | 20260508 00:33:29.717 | INFO | 60.3% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:33:29.717 | INFO | 39.6% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:33:29.717 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:33:29.717 | INFO | ------------------------------------------ | 20260508 00:33:29.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:33:29.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:33:29.717 | INFO | Total mem usage: 26.5 GiB | 20260508 00:33:29.717 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:33:29.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:33:29.717 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:33:29.717 | INFO | High Water Breakdown: | 20260508 00:33:29.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:33:29.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:33:29.717 | INFO | Uploaded 36GiB to GPU in 2.48975s (14.4593GiB/s) | 20260508 00:33:29.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:33:29.718 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:33:29.718 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:33:29.718 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:33:29.718 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:33:29.718 | INFO | Downloaded 34GiB to GPU in 2.44983s (13.8785GiB/s) | 20260508 00:33:29.718 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:33:29.718 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:33:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:33:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:33:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:33:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:33:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:33:29.718 | INFO | ------------------------------------------ | 20260508 00:33:29.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:33:29.718 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:33:29.718 | INFO | Total mem usage: 27.8 GiB | 20260508 00:33:29.718 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:33:29.718 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:33:29.718 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:33:29.718 | INFO | High Water Breakdown: | 20260508 00:33:29.718 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:33:29.718 | INFO | ------------------------------------------ | 20260508 00:33:29.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:33:29.718 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:33:29.718 | INFO | Total mem usage: 15.7 GiB | 20260508 00:33:29.718 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:33:29.718 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:33:29.718 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:33:29.718 | INFO | High Water Breakdown: | 20260508 00:33:29.718 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:33:29.718 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:33:29.718 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:33:29.718 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:33:29.718 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:33:29.718 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:33:29.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:33:29.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:33:29.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:33:29.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:33:29.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:33:29.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:33:29.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:33:29.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:33:29.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:33:29.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:33:29.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:33:29.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:33:29.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:33:29.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:33:29.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:33:29.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:33:29.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:33:29.718 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:33:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:33:29.718 | INFO | ------------------------------------------ | 20260508 00:33:29.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:33:29.718 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:33:29.718 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:33:29.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:33:29.718 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.43e+01 / 1.50e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.16e-02 / 7.10e-03 | 20260508 00:33:29.718 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.47e+00, 5.90e-01 / 2.00e-01 | 20260508 00:33:29.718 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.10e-01 / 1.07e-01 | 20260508 00:33:29.718 | INFO | PerfReport: 16.09%, DME -- 4.22e+01 / 4.63e+01, 4.08e+00 / 1.54e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.60e+00 / 1.38e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.27e-01 / 1.75e-01 | 20260508 00:33:29.718 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.32e+00, 2.74e-01 / 1.38e-01 | 20260508 00:33:29.718 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.76e-01, 5.37e-02 / 2.07e-02 | 20260508 00:33:29.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:33:29.718 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:33:29.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:33:29.718 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 24.17%, Upload -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 24.35%, Download -- 2.59e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 2.89%, LockDownload -- 3.08e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 15.75%, LockCompute -- 1.68e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:33:29.718 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260508 00:33:29.815 | INFO | Starting a round of Baz IO | 20260508 00:33:33.282 | INFO | Populating RTMetrics report | 20260508 00:33:33.283 | INFO | RTMetrics report populated | 20260508 00:33:33.283 | INFO | Opening tmp RTMetrics json file | 20260508 00:33:33.310 | INFO | Serializing RTMetrics to json | 20260508 00:33:33.327 | INFO | Serialization done, swapping tmp file now... | 20260508 00:33:33.359 | INFO | RTMetrics done | 20260508 00:33:33.378 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.562 seconds | 20260508 00:33:33.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.15% / 0.00% / 57.85% | 20260508 00:33:33.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.993037% / 1.58946e-05% / 4.76837e-05% | 20260508 00:33:33.378 | INFO | Baz IO overall: | 20260508 00:33:33.378 | INFO | BytesWritten : 1741697151064 | 20260508 00:33:33.378 | INFO | NumEvents : 842932985174 | 20260508 00:33:34.859 | INFO | Analyzing chunk frames = [3833856,3834368) | 20260508 00:33:34.859 | INFO | MemInfo: { | 20260508 00:33:34.859 | INFO | "gpu" : " 17 GB", | 20260508 00:33:34.859 | INFO | "huge" : "", | 20260508 00:33:34.859 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:34.859 | INFO | "malloc" : " 14 GB", | 20260508 00:33:34.859 | INFO | "pinned" : " 29 GB" | 20260508 00:33:34.859 | INFO | } | 20260508 00:33:34.859 | INFO | Perf report for the last 5.14216 seconds: | 20260508 00:33:34.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:34.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:34.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:34.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:34.859 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260508 00:33:34.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:34.859 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 00:33:34.859 | INFO | PRE_HQ : 65.7%, 1.7, 56.3ms, 0 | 20260508 00:33:34.859 | INFO | BAZWRITER : 68.9%, 3.8, 133.6ms, 0 | 20260508 00:33:34.859 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 00:33:34.860 | INFO | Uploaded 36GiB to GPU in 2.54386s (14.1517GiB/s) | 20260508 00:33:34.860 | INFO | Downloaded 38GiB to GPU in 2.69544s (14.0979GiB/s) | 20260508 00:33:39.992 | INFO | Analyzing chunk frames = [3834368,3834880) | 20260508 00:33:39.992 | INFO | MemInfo: { | 20260508 00:33:39.992 | INFO | "gpu" : " 17 GB", | 20260508 00:33:39.992 | INFO | "huge" : "", | 20260508 00:33:39.992 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:39.992 | INFO | "malloc" : " 14 GB", | 20260508 00:33:39.992 | INFO | "pinned" : " 29 GB" | 20260508 00:33:39.992 | INFO | } | 20260508 00:33:39.993 | INFO | Perf report for the last 5.1336 seconds: | 20260508 00:33:39.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:39.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260508 00:33:39.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:33:39.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:39.993 | INFO | Uploaded 36GiB to GPU in 2.52326s (14.2672GiB/s) | 20260508 00:33:39.993 | INFO | Downloaded 34GiB to GPU in 2.42703s (14.0089GiB/s) | 20260508 00:33:45.129 | INFO | Analyzing chunk frames = [3834880,3835392) | 20260508 00:33:45.129 | INFO | MemInfo: { | 20260508 00:33:45.129 | INFO | "gpu" : " 17 GB", | 20260508 00:33:45.129 | INFO | "huge" : "", | 20260508 00:33:45.129 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:45.129 | INFO | "malloc" : " 14 GB", | 20260508 00:33:45.129 | INFO | "pinned" : " 29 GB" | 20260508 00:33:45.129 | INFO | } | 20260508 00:33:45.129 | INFO | Perf report for the last 5.13693 seconds: | 20260508 00:33:45.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:45.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.130 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:33:45.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.130 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:33:45.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:45.130 | INFO | Uploaded 36GiB to GPU in 2.49371s (14.4363GiB/s) | 20260508 00:33:45.130 | INFO | Downloaded 34GiB to GPU in 2.4848s (13.6832GiB/s) | 20260508 00:33:50.268 | INFO | Analyzing chunk frames = [3835392,3835904) | 20260508 00:33:50.268 | INFO | MemInfo: { | 20260508 00:33:50.268 | INFO | "gpu" : " 17 GB", | 20260508 00:33:50.268 | INFO | "huge" : "", | 20260508 00:33:50.268 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:50.268 | INFO | "malloc" : " 14 GB", | 20260508 00:33:50.268 | INFO | "pinned" : " 29 GB" | 20260508 00:33:50.268 | INFO | } | 20260508 00:33:50.268 | INFO | Perf report for the last 5.13884 seconds: | 20260508 00:33:50.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:50.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.268 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:33:50.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.268 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 00:33:50.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:50.269 | INFO | Uploaded 36GiB to GPU in 2.50049s (14.3972GiB/s) | 20260508 00:33:50.269 | INFO | Downloaded 34GiB to GPU in 2.49961s (13.6021GiB/s) | 20260508 00:33:55.412 | INFO | Analyzing chunk frames = [3835904,3836416) | 20260508 00:33:55.412 | INFO | MemInfo: { | 20260508 00:33:55.412 | INFO | "gpu" : " 17 GB", | 20260508 00:33:55.412 | INFO | "huge" : "", | 20260508 00:33:55.412 | INFO | "huge_shared" : " 29 GB", | 20260508 00:33:55.412 | INFO | "malloc" : " 14 GB", | 20260508 00:33:55.412 | INFO | "pinned" : " 29 GB" | 20260508 00:33:55.412 | INFO | } | 20260508 00:33:55.412 | INFO | Perf report for the last 5.14409 seconds: | 20260508 00:33:55.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:33:55.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.413 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 00:33:55.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.413 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:33:55.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:33:55.413 | INFO | Uploaded 36GiB to GPU in 2.39914s (15.0054GiB/s) | 20260508 00:33:55.413 | INFO | Downloaded 34GiB to GPU in 2.52907s (13.4437GiB/s) | 20260508 00:34:00.553 | INFO | Analyzing chunk frames = [3836416,3836928) | 20260508 00:34:00.553 | INFO | MemInfo: { | 20260508 00:34:00.553 | INFO | "gpu" : " 17 GB", | 20260508 00:34:00.553 | INFO | "huge" : "", | 20260508 00:34:00.554 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:00.554 | INFO | "malloc" : " 14 GB", | 20260508 00:34:00.554 | INFO | "pinned" : " 29 GB" | 20260508 00:34:00.554 | INFO | } | 20260508 00:34:00.554 | INFO | Perf report for the last 5.14118 seconds: | 20260508 00:34:00.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:00.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:34:00.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:34:00.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:00.554 | INFO | Uploaded 36GiB to GPU in 2.39552s (15.0281GiB/s) | 20260508 00:34:00.554 | INFO | Downloaded 34GiB to GPU in 2.40549s (14.1343GiB/s) | 20260508 00:34:05.688 | INFO | Analyzing chunk frames = [3836928,3837440) | 20260508 00:34:05.688 | INFO | MemInfo: { | 20260508 00:34:05.688 | INFO | "gpu" : " 17 GB", | 20260508 00:34:05.688 | INFO | "huge" : "", | 20260508 00:34:05.688 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:05.688 | INFO | "malloc" : " 14 GB", | 20260508 00:34:05.688 | INFO | "pinned" : " 29 GB" | 20260508 00:34:05.688 | INFO | } | 20260508 00:34:05.688 | INFO | Perf report for the last 5.13412 seconds: | 20260508 00:34:05.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:05.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 00:34:05.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:34:05.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:05.688 | INFO | Uploaded 36GiB to GPU in 2.4336s (14.7929GiB/s) | 20260508 00:34:05.688 | INFO | Downloaded 34GiB to GPU in 2.42782s (14.0043GiB/s) | 20260508 00:34:10.825 | INFO | Analyzing chunk frames = [3837440,3837952) | 20260508 00:34:10.825 | INFO | MemInfo: { | 20260508 00:34:10.825 | INFO | "gpu" : " 17 GB", | 20260508 00:34:10.825 | INFO | "huge" : "", | 20260508 00:34:10.825 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:10.825 | INFO | "malloc" : " 14 GB", | 20260508 00:34:10.825 | INFO | "pinned" : " 29 GB" | 20260508 00:34:10.825 | INFO | } | 20260508 00:34:10.825 | INFO | Perf report for the last 5.13745 seconds: | 20260508 00:34:10.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:10.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.825 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:34:10.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.825 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:34:10.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:10.826 | INFO | Uploaded 36GiB to GPU in 2.51297s (14.3257GiB/s) | 20260508 00:34:10.826 | INFO | Downloaded 34GiB to GPU in 2.45377s (13.8562GiB/s) | 20260508 00:34:10.922 | INFO | Starting a round of Baz IO | 20260508 00:34:14.394 | INFO | Populating RTMetrics report | 20260508 00:34:14.394 | INFO | RTMetrics report populated | 20260508 00:34:14.394 | INFO | Opening tmp RTMetrics json file | 20260508 00:34:14.407 | INFO | Serializing RTMetrics to json | 20260508 00:34:14.424 | INFO | Serialization done, swapping tmp file now... | 20260508 00:34:14.430 | INFO | RTMetrics done | 20260508 00:34:14.499 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.576 seconds | 20260508 00:34:14.499 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.15% / 0.00% / 57.85% | 20260508 00:34:14.499 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.992783% / 1.98682e-05% / 3.97364e-05% | 20260508 00:34:14.499 | INFO | Baz IO overall: | 20260508 00:34:14.499 | INFO | BytesWritten : 1743509205555 | 20260508 00:34:14.499 | INFO | NumEvents : 843805134230 | 20260508 00:34:15.970 | INFO | Analyzing chunk frames = [3837952,3838464) | 20260508 00:34:15.970 | INFO | MemInfo: { | 20260508 00:34:15.970 | INFO | "gpu" : " 17 GB", | 20260508 00:34:15.970 | INFO | "huge" : "", | 20260508 00:34:15.970 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:15.970 | INFO | "malloc" : " 14 GB", | 20260508 00:34:15.970 | INFO | "pinned" : " 29 GB" | 20260508 00:34:15.970 | INFO | } | 20260508 00:34:15.970 | INFO | Perf report for the last 5.14496 seconds: | 20260508 00:34:15.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:15.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:15.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:15.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:15.970 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260508 00:34:15.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:15.970 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260508 00:34:15.970 | INFO | PRE_HQ : 65.4%, 1.7, 56.5ms, 0 | 20260508 00:34:15.970 | INFO | BAZWRITER : 69.0%, 3.8, 134.0ms, 0 | 20260508 00:34:15.970 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:34:15.970 | INFO | Uploaded 36GiB to GPU in 2.54761s (14.1309GiB/s) | 20260508 00:34:15.971 | INFO | Downloaded 38GiB to GPU in 2.69811s (14.0839GiB/s) | 20260508 00:34:21.110 | INFO | Analyzing chunk frames = [3838464,3838976) | 20260508 00:34:21.110 | INFO | MemInfo: { | 20260508 00:34:21.110 | INFO | "gpu" : " 17 GB", | 20260508 00:34:21.110 | INFO | "huge" : "", | 20260508 00:34:21.110 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:21.110 | INFO | "malloc" : " 14 GB", | 20260508 00:34:21.110 | INFO | "pinned" : " 29 GB" | 20260508 00:34:21.110 | INFO | } | 20260508 00:34:21.110 | INFO | Perf report for the last 5.14022 seconds: | 20260508 00:34:21.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:21.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.110 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:34:21.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.111 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:34:21.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:21.111 | INFO | Uploaded 36GiB to GPU in 2.57927s (13.9574GiB/s) | 20260508 00:34:21.111 | INFO | Downloaded 34GiB to GPU in 2.49595s (13.6221GiB/s) | 20260508 00:34:26.248 | INFO | Analyzing chunk frames = [3838976,3839488) | 20260508 00:34:26.248 | INFO | MemInfo: { | 20260508 00:34:26.248 | INFO | "gpu" : " 17 GB", | 20260508 00:34:26.248 | INFO | "huge" : "", | 20260508 00:34:26.248 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:26.248 | INFO | "malloc" : " 14 GB", | 20260508 00:34:26.248 | INFO | "pinned" : " 29 GB" | 20260508 00:34:26.248 | INFO | } | 20260508 00:34:26.248 | INFO | Perf report for the last 5.13784 seconds: | 20260508 00:34:26.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:26.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:34:26.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:34:26.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:26.248 | INFO | Uploaded 36GiB to GPU in 2.54424s (14.1496GiB/s) | 20260508 00:34:26.249 | INFO | Downloaded 34GiB to GPU in 2.50039s (13.5979GiB/s) | 20260508 00:34:31.389 | INFO | Analyzing chunk frames = [3839488,3840000) | 20260508 00:34:31.389 | INFO | MemInfo: { | 20260508 00:34:31.389 | INFO | "gpu" : " 17 GB", | 20260508 00:34:31.389 | INFO | "huge" : "", | 20260508 00:34:31.389 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:31.389 | INFO | "malloc" : " 14 GB", | 20260508 00:34:31.389 | INFO | "pinned" : " 29 GB" | 20260508 00:34:31.389 | INFO | } | 20260508 00:34:31.389 | INFO | Perf report for the last 5.14067 seconds: | 20260508 00:34:31.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:31.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 00:34:31.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:34:31.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:31.389 | INFO | Uploaded 36GiB to GPU in 2.49496s (14.4291GiB/s) | 20260508 00:34:31.390 | INFO | Downloaded 34GiB to GPU in 2.49389s (13.6333GiB/s) | 20260508 00:34:36.525 | INFO | Analyzing chunk frames = [3840000,3840512) | 20260508 00:34:36.525 | INFO | MemInfo: { | 20260508 00:34:36.525 | INFO | "gpu" : " 17 GB", | 20260508 00:34:36.525 | INFO | "huge" : "", | 20260508 00:34:36.525 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:36.525 | INFO | "malloc" : " 14 GB", | 20260508 00:34:36.525 | INFO | "pinned" : " 29 GB" | 20260508 00:34:36.525 | INFO | } | 20260508 00:34:36.525 | INFO | Perf report for the last 5.13575 seconds: | 20260508 00:34:36.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:36.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 00:34:36.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:34:36.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:36.525 | INFO | Uploaded 36GiB to GPU in 2.31224s (15.5693GiB/s) | 20260508 00:34:36.525 | INFO | Downloaded 34GiB to GPU in 2.49415s (13.6319GiB/s) | 20260508 00:34:41.668 | INFO | Analyzing chunk frames = [3840512,3841024) | 20260508 00:34:41.668 | INFO | MemInfo: { | 20260508 00:34:41.668 | INFO | "gpu" : " 17 GB", | 20260508 00:34:41.668 | INFO | "huge" : "", | 20260508 00:34:41.668 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:41.668 | INFO | "malloc" : " 14 GB", | 20260508 00:34:41.668 | INFO | "pinned" : " 29 GB" | 20260508 00:34:41.668 | INFO | } | 20260508 00:34:41.668 | INFO | Perf report for the last 5.14358 seconds: | 20260508 00:34:41.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:41.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.668 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:34:41.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.668 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:34:41.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:41.669 | INFO | Uploaded 36GiB to GPU in 2.3502s (15.3179GiB/s) | 20260508 00:34:41.669 | INFO | Downloaded 34GiB to GPU in 2.44779s (13.8901GiB/s) | 20260508 00:34:46.810 | INFO | Analyzing chunk frames = [3841024,3841536) | 20260508 00:34:46.810 | INFO | MemInfo: { | 20260508 00:34:46.810 | INFO | "gpu" : " 17 GB", | 20260508 00:34:46.810 | INFO | "huge" : "", | 20260508 00:34:46.810 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:46.811 | INFO | "malloc" : " 14 GB", | 20260508 00:34:46.811 | INFO | "pinned" : " 29 GB" | 20260508 00:34:46.811 | INFO | } | 20260508 00:34:46.811 | INFO | Perf report for the last 5.14223 seconds: | 20260508 00:34:46.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:46.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 00:34:46.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:34:46.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:46.811 | INFO | Uploaded 36GiB to GPU in 2.39632s (15.023GiB/s) | 20260508 00:34:46.811 | INFO | Downloaded 34GiB to GPU in 2.43846s (13.9432GiB/s) | 20260508 00:34:51.944 | INFO | Analyzing chunk frames = [3841536,3842048) | 20260508 00:34:51.944 | INFO | MemInfo: { | 20260508 00:34:51.944 | INFO | "gpu" : " 17 GB", | 20260508 00:34:51.944 | INFO | "huge" : "", | 20260508 00:34:51.944 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:51.944 | INFO | "malloc" : " 14 GB", | 20260508 00:34:51.944 | INFO | "pinned" : " 29 GB" | 20260508 00:34:51.944 | INFO | } | 20260508 00:34:51.944 | INFO | Perf report for the last 5.13317 seconds: | 20260508 00:34:51.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:51.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 00:34:51.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 00:34:51.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:51.944 | INFO | Uploaded 36GiB to GPU in 2.5192s (14.2902GiB/s) | 20260508 00:34:51.944 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:34:51.944 | INFO | Total mem usage: 16.7 GiB | 20260508 00:34:51.944 | INFO | Downloaded 34GiB to GPU in 2.46453s (13.7957GiB/s) | 20260508 00:34:51.944 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:34:51.944 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 00:34:51.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:34:51.944 | INFO | High Water Breakdown: | 20260508 00:34:51.944 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:34:51.944 | INFO | 39.9% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:34:51.944 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:34:51.944 | INFO | ------------------------------------------ | 20260508 00:34:51.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:34:51.944 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:34:51.944 | INFO | Total mem usage: 26.5 GiB | 20260508 00:34:51.944 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:34:51.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:34:51.944 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 00:34:51.944 | INFO | High Water Breakdown: | 20260508 00:34:51.944 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:34:51.944 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:34:51.944 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:34:51.944 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:34:51.944 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:34:51.944 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:34:51.944 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:34:51.944 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:34:51.944 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:34:51.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:34:51.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:34:51.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:34:51.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:34:51.944 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:34:51.944 | INFO | ------------------------------------------ | 20260508 00:34:51.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:34:51.944 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:34:51.944 | INFO | Total mem usage: 27.8 GiB | 20260508 00:34:51.944 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:34:51.944 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:34:51.944 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:34:51.944 | INFO | High Water Breakdown: | 20260508 00:34:51.944 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:34:51.944 | INFO | ------------------------------------------ | 20260508 00:34:51.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:34:51.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:34:51.944 | INFO | Total mem usage: 15.7 GiB | 20260508 00:34:51.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:34:51.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:34:51.944 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:34:51.945 | INFO | High Water Breakdown: | 20260508 00:34:51.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:34:51.945 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:34:51.945 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:34:51.945 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:34:51.945 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:34:51.945 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:34:51.945 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:34:51.945 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:34:51.945 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:34:51.945 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:34:51.945 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:34:51.945 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:34:51.945 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:34:51.945 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:34:51.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:34:51.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:34:51.945 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:34:51.945 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:34:51.945 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:34:51.945 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:34:51.945 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:34:51.945 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:34:51.945 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:34:51.945 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:34:51.945 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:34:51.945 | INFO | ------------------------------------------ | 20260508 00:34:51.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:34:51.945 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:34:51.945 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:34:51.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:34:51.945 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.43e+01 / 1.76e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.21e-02 / 4.16e-03 | 20260508 00:34:51.945 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.46e+00, 5.88e-01 / 2.11e-01 | 20260508 00:34:51.945 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 1.09e-01 | 20260508 00:34:51.945 | INFO | PerfReport: 16.12%, DME -- 4.23e+01 / 4.62e+01, 4.31e+00 / 1.95e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.66e+00 / 1.53e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.23e-01 / 1.92e-01 | 20260508 00:34:51.945 | INFO | PerfReport: 3.37%, Metrics -- 1.11e+00 / 1.32e+00, 2.74e-01 / 1.54e-01 | 20260508 00:34:51.945 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.39e-01 / 3.78e-01, 5.21e-02 / 2.51e-02 | 20260508 00:34:51.945 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:34:51.945 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:34:51.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:34:51.945 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 24.16%, Upload -- 2.57e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 30.80%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 2.05%, LockUpload -- 2.18e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 2.83%, LockDownload -- 3.01e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 15.75%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:34:51.945 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260508 00:34:52.040 | INFO | Starting a round of Baz IO | 20260508 00:34:55.492 | INFO | Populating RTMetrics report | 20260508 00:34:55.493 | INFO | RTMetrics report populated | 20260508 00:34:55.493 | INFO | Opening tmp RTMetrics json file | 20260508 00:34:55.528 | INFO | Serializing RTMetrics to json | 20260508 00:34:55.571 | INFO | Serialization done, swapping tmp file now... | 20260508 00:34:55.574 | INFO | RTMetrics done | 20260508 00:34:55.603 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.562 seconds | 20260508 00:34:55.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.14% / 0.00% / 57.85% | 20260508 00:34:55.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.992874% / 0% / 5.5631e-05% | 20260508 00:34:55.603 | INFO | Baz IO overall: | 20260508 00:34:55.603 | INFO | BytesWritten : 1745321499356 | 20260508 00:34:55.603 | INFO | NumEvents : 844677508503 | 20260508 00:34:57.079 | INFO | Analyzing chunk frames = [3842048,3842560) | 20260508 00:34:57.079 | INFO | MemInfo: { | 20260508 00:34:57.079 | INFO | "gpu" : " 17 GB", | 20260508 00:34:57.079 | INFO | "huge" : "", | 20260508 00:34:57.080 | INFO | "huge_shared" : " 29 GB", | 20260508 00:34:57.080 | INFO | "malloc" : " 14 GB", | 20260508 00:34:57.080 | INFO | "pinned" : " 29 GB" | 20260508 00:34:57.080 | INFO | } | 20260508 00:34:57.080 | INFO | Perf report for the last 5.1358 seconds: | 20260508 00:34:57.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:34:57.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:57.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:57.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:57.080 | INFO | ANALYSIS : 68.4%, 2.9, 105.1ms, 0 | 20260508 00:34:57.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:34:57.080 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 00:34:57.080 | INFO | PRE_HQ : 66.5%, 1.7, 59.3ms, 0 | 20260508 00:34:57.080 | INFO | BAZWRITER : 68.9%, 4.1, 146.1ms, 0 | 20260508 00:34:57.080 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 00:34:57.080 | INFO | Uploaded 36GiB to GPU in 2.53708s (14.1895GiB/s) | 20260508 00:34:57.080 | INFO | Downloaded 38GiB to GPU in 2.69413s (14.1048GiB/s) | 20260508 00:35:02.221 | INFO | Analyzing chunk frames = [3842560,3843072) | 20260508 00:35:02.221 | INFO | MemInfo: { | 20260508 00:35:02.221 | INFO | "gpu" : " 17 GB", | 20260508 00:35:02.221 | INFO | "huge" : "", | 20260508 00:35:02.221 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:02.221 | INFO | "malloc" : " 14 GB", | 20260508 00:35:02.221 | INFO | "pinned" : " 29 GB" | 20260508 00:35:02.221 | INFO | } | 20260508 00:35:02.221 | INFO | Perf report for the last 5.14166 seconds: | 20260508 00:35:02.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:02.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 00:35:02.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:35:02.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:02.221 | INFO | Uploaded 36GiB to GPU in 2.52909s (14.2344GiB/s) | 20260508 00:35:02.221 | INFO | Downloaded 34GiB to GPU in 2.50212s (13.5885GiB/s) | 20260508 00:35:07.363 | INFO | Analyzing chunk frames = [3843072,3843584) | 20260508 00:35:07.363 | INFO | MemInfo: { | 20260508 00:35:07.363 | INFO | "gpu" : " 17 GB", | 20260508 00:35:07.363 | INFO | "huge" : "", | 20260508 00:35:07.363 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:07.363 | INFO | "malloc" : " 14 GB", | 20260508 00:35:07.363 | INFO | "pinned" : " 29 GB" | 20260508 00:35:07.363 | INFO | } | 20260508 00:35:07.363 | INFO | Perf report for the last 5.14167 seconds: | 20260508 00:35:07.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:07.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 00:35:07.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:35:07.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:07.363 | INFO | Uploaded 36GiB to GPU in 2.44731s (14.71GiB/s) | 20260508 00:35:07.363 | INFO | Downloaded 34GiB to GPU in 2.48645s (13.6741GiB/s) | 20260508 00:35:12.506 | INFO | Analyzing chunk frames = [3843584,3844096) | 20260508 00:35:12.506 | INFO | MemInfo: { | 20260508 00:35:12.506 | INFO | "gpu" : " 17 GB", | 20260508 00:35:12.506 | INFO | "huge" : "", | 20260508 00:35:12.506 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:12.506 | INFO | "malloc" : " 14 GB", | 20260508 00:35:12.506 | INFO | "pinned" : " 29 GB" | 20260508 00:35:12.506 | INFO | } | 20260508 00:35:12.506 | INFO | Perf report for the last 5.14276 seconds: | 20260508 00:35:12.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:12.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 00:35:12.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:35:12.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:12.506 | INFO | Uploaded 36GiB to GPU in 2.45756s (14.6487GiB/s) | 20260508 00:35:12.506 | INFO | Downloaded 34GiB to GPU in 2.50127s (13.5931GiB/s) | 20260508 00:35:17.641 | INFO | Analyzing chunk frames = [3844096,3844608) | 20260508 00:35:17.641 | INFO | MemInfo: { | 20260508 00:35:17.641 | INFO | "gpu" : " 17 GB", | 20260508 00:35:17.641 | INFO | "huge" : "", | 20260508 00:35:17.641 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:17.641 | INFO | "malloc" : " 14 GB", | 20260508 00:35:17.641 | INFO | "pinned" : " 29 GB" | 20260508 00:35:17.641 | INFO | } | 20260508 00:35:17.641 | INFO | Perf report for the last 5.13577 seconds: | 20260508 00:35:17.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:17.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 00:35:17.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 00:35:17.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:17.642 | INFO | Uploaded 36GiB to GPU in 2.37147s (15.1804GiB/s) | 20260508 00:35:17.642 | INFO | Downloaded 34GiB to GPU in 2.5025s (13.5864GiB/s) | 20260508 00:35:22.780 | INFO | Analyzing chunk frames = [3844608,3845120) | 20260508 00:35:22.780 | INFO | MemInfo: { | 20260508 00:35:22.780 | INFO | "gpu" : " 17 GB", | 20260508 00:35:22.780 | INFO | "huge" : "", | 20260508 00:35:22.780 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:22.780 | INFO | "malloc" : " 14 GB", | 20260508 00:35:22.780 | INFO | "pinned" : " 29 GB" | 20260508 00:35:22.780 | INFO | } | 20260508 00:35:22.780 | INFO | Perf report for the last 5.13869 seconds: | 20260508 00:35:22.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:22.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:35:22.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:35:22.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:22.780 | INFO | Uploaded 36GiB to GPU in 2.36333s (15.2327GiB/s) | 20260508 00:35:22.780 | INFO | Downloaded 34GiB to GPU in 2.42347s (14.0295GiB/s) | 20260508 00:35:27.920 | INFO | Analyzing chunk frames = [3845120,3845632) | 20260508 00:35:27.920 | INFO | MemInfo: { | 20260508 00:35:27.920 | INFO | "gpu" : " 17 GB", | 20260508 00:35:27.920 | INFO | "huge" : "", | 20260508 00:35:27.920 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:27.921 | INFO | "malloc" : " 14 GB", | 20260508 00:35:27.921 | INFO | "pinned" : " 29 GB" | 20260508 00:35:27.921 | INFO | } | 20260508 00:35:27.921 | INFO | Perf report for the last 5.14036 seconds: | 20260508 00:35:27.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:27.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 00:35:27.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 00:35:27.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:27.921 | INFO | Uploaded 36GiB to GPU in 2.49886s (14.4066GiB/s) | 20260508 00:35:27.921 | INFO | Downloaded 34GiB to GPU in 2.46569s (13.7892GiB/s) | 20260508 00:35:33.062 | INFO | Analyzing chunk frames = [3845632,3846144) | 20260508 00:35:33.062 | INFO | MemInfo: { | 20260508 00:35:33.062 | INFO | "gpu" : " 17 GB", | 20260508 00:35:33.062 | INFO | "huge" : "", | 20260508 00:35:33.062 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:33.062 | INFO | "malloc" : " 14 GB", | 20260508 00:35:33.062 | INFO | "pinned" : " 29 GB" | 20260508 00:35:33.062 | INFO | } | 20260508 00:35:33.062 | INFO | Perf report for the last 5.14122 seconds: | 20260508 00:35:33.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:33.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 00:35:33.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 00:35:33.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:33.062 | INFO | Uploaded 36GiB to GPU in 2.49934s (14.4038GiB/s) | 20260508 00:35:33.062 | INFO | Downloaded 34GiB to GPU in 2.46216s (13.809GiB/s) | 20260508 00:35:33.159 | INFO | Starting a round of Baz IO | 20260508 00:35:36.608 | INFO | Populating RTMetrics report | 20260508 00:35:36.609 | INFO | RTMetrics report populated | 20260508 00:35:36.609 | INFO | Opening tmp RTMetrics json file | 20260508 00:35:36.627 | INFO | Serializing RTMetrics to json | 20260508 00:35:36.645 | INFO | Serialization done, swapping tmp file now... | 20260508 00:35:36.699 | INFO | RTMetrics done | 20260508 00:35:36.726 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.566 seconds | 20260508 00:35:36.726 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.14% / 0.00% / 57.86% | 20260508 00:35:36.726 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.986588% / 0% / 3.97364e-05% | 20260508 00:35:36.726 | INFO | Baz IO overall: | 20260508 00:35:36.726 | INFO | BytesWritten : 1747130427744 | 20260508 00:35:36.726 | INFO | NumEvents : 845547157776 | 20260508 00:35:38.200 | INFO | Analyzing chunk frames = [3846144,3846656) | 20260508 00:35:38.200 | INFO | MemInfo: { | 20260508 00:35:38.200 | INFO | "gpu" : " 17 GB", | 20260508 00:35:38.200 | INFO | "huge" : "", | 20260508 00:35:38.200 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:38.200 | INFO | "malloc" : " 14 GB", | 20260508 00:35:38.200 | INFO | "pinned" : " 29 GB" | 20260508 00:35:38.200 | INFO | } | 20260508 00:35:38.200 | INFO | Perf report for the last 5.13867 seconds: | 20260508 00:35:38.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:38.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:38.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:38.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:38.200 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 00:35:38.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:38.201 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260508 00:35:38.201 | INFO | PRE_HQ : 65.9%, 1.7, 56.6ms, 0 | 20260508 00:35:38.201 | INFO | BAZWRITER : 69.0%, 4.3, 150.8ms, 0 | 20260508 00:35:38.201 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 00:35:38.201 | INFO | Uploaded 36GiB to GPU in 2.50649s (14.3627GiB/s) | 20260508 00:35:38.201 | INFO | Downloaded 38GiB to GPU in 2.69904s (14.0791GiB/s) | 20260508 00:35:43.334 | INFO | Analyzing chunk frames = [3846656,3847168) | 20260508 00:35:43.334 | INFO | MemInfo: { | 20260508 00:35:43.334 | INFO | "gpu" : " 17 GB", | 20260508 00:35:43.334 | INFO | "huge" : "", | 20260508 00:35:43.334 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:43.334 | INFO | "malloc" : " 14 GB", | 20260508 00:35:43.334 | INFO | "pinned" : " 29 GB" | 20260508 00:35:43.334 | INFO | } | 20260508 00:35:43.334 | INFO | Perf report for the last 5.13362 seconds: | 20260508 00:35:43.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:43.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:35:43.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:35:43.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:43.334 | INFO | Uploaded 36GiB to GPU in 2.53978s (14.1745GiB/s) | 20260508 00:35:43.334 | INFO | Downloaded 34GiB to GPU in 2.42362s (14.0286GiB/s) | 20260508 00:35:48.477 | INFO | Analyzing chunk frames = [3847168,3847680) | 20260508 00:35:48.477 | INFO | MemInfo: { | 20260508 00:35:48.477 | INFO | "gpu" : " 17 GB", | 20260508 00:35:48.477 | INFO | "huge" : "", | 20260508 00:35:48.477 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:48.477 | INFO | "malloc" : " 14 GB", | 20260508 00:35:48.477 | INFO | "pinned" : " 29 GB" | 20260508 00:35:48.477 | INFO | } | 20260508 00:35:48.477 | INFO | Perf report for the last 5.1429 seconds: | 20260508 00:35:48.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:48.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 00:35:48.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:35:48.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:48.477 | INFO | Uploaded 36GiB to GPU in 2.56878s (14.0144GiB/s) | 20260508 00:35:48.477 | INFO | Downloaded 34GiB to GPU in 2.4803s (13.708GiB/s) | 20260508 00:35:53.621 | INFO | Analyzing chunk frames = [3847680,3848192) | 20260508 00:35:53.621 | INFO | MemInfo: { | 20260508 00:35:53.621 | INFO | "gpu" : " 17 GB", | 20260508 00:35:53.621 | INFO | "huge" : "", | 20260508 00:35:53.621 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:53.621 | INFO | "malloc" : " 14 GB", | 20260508 00:35:53.621 | INFO | "pinned" : " 29 GB" | 20260508 00:35:53.621 | INFO | } | 20260508 00:35:53.621 | INFO | Perf report for the last 5.1438 seconds: | 20260508 00:35:53.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:53.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 00:35:53.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:35:53.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:53.621 | INFO | Uploaded 36GiB to GPU in 2.48629s (14.4794GiB/s) | 20260508 00:35:53.621 | INFO | Downloaded 34GiB to GPU in 2.48304s (13.6929GiB/s) | 20260508 00:35:58.760 | INFO | Analyzing chunk frames = [3848192,3848704) | 20260508 00:35:58.760 | INFO | MemInfo: { | 20260508 00:35:58.760 | INFO | "gpu" : " 17 GB", | 20260508 00:35:58.760 | INFO | "huge" : "", | 20260508 00:35:58.760 | INFO | "huge_shared" : " 29 GB", | 20260508 00:35:58.760 | INFO | "malloc" : " 14 GB", | 20260508 00:35:58.760 | INFO | "pinned" : " 29 GB" | 20260508 00:35:58.760 | INFO | } | 20260508 00:35:58.760 | INFO | Perf report for the last 5.13882 seconds: | 20260508 00:35:58.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:35:58.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 00:35:58.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:35:58.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:35:58.760 | INFO | Uploaded 36GiB to GPU in 2.35766s (15.2694GiB/s) | 20260508 00:35:58.760 | INFO | Downloaded 34GiB to GPU in 2.48607s (13.6762GiB/s) | 20260508 00:36:03.893 | INFO | Analyzing chunk frames = [3848704,3849216) | 20260508 00:36:03.893 | INFO | MemInfo: { | 20260508 00:36:03.893 | INFO | "gpu" : " 17 GB", | 20260508 00:36:03.893 | INFO | "huge" : "", | 20260508 00:36:03.893 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:03.893 | INFO | "malloc" : " 14 GB", | 20260508 00:36:03.893 | INFO | "pinned" : " 29 GB" | 20260508 00:36:03.893 | INFO | } | 20260508 00:36:03.893 | INFO | Perf report for the last 5.13313 seconds: | 20260508 00:36:03.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:03.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:36:03.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260508 00:36:03.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:03.893 | INFO | Uploaded 36GiB to GPU in 2.40175s (14.9891GiB/s) | 20260508 00:36:03.893 | INFO | Downloaded 34GiB to GPU in 2.41202s (14.0961GiB/s) | 20260508 00:36:09.034 | INFO | Analyzing chunk frames = [3849216,3849728) | 20260508 00:36:09.034 | INFO | MemInfo: { | 20260508 00:36:09.034 | INFO | "gpu" : " 17 GB", | 20260508 00:36:09.034 | INFO | "huge" : "", | 20260508 00:36:09.034 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:09.034 | INFO | "malloc" : " 14 GB", | 20260508 00:36:09.034 | INFO | "pinned" : " 29 GB" | 20260508 00:36:09.034 | INFO | } | 20260508 00:36:09.034 | INFO | Perf report for the last 5.14099 seconds: | 20260508 00:36:09.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:09.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:36:09.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 00:36:09.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:09.034 | INFO | Uploaded 36GiB to GPU in 2.55607s (14.0841GiB/s) | 20260508 00:36:09.034 | INFO | Downloaded 34GiB to GPU in 2.46737s (13.7798GiB/s) | 20260508 00:36:14.176 | INFO | Analyzing chunk frames = [3849728,3850240) | 20260508 00:36:14.176 | INFO | MemInfo: { | 20260508 00:36:14.176 | INFO | "gpu" : " 17 GB", | 20260508 00:36:14.176 | INFO | "huge" : "", | 20260508 00:36:14.176 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:14.176 | INFO | "malloc" : " 14 GB", | 20260508 00:36:14.176 | INFO | "pinned" : " 29 GB" | 20260508 00:36:14.176 | INFO | } | 20260508 00:36:14.176 | INFO | Perf report for the last 5.14188 seconds: | 20260508 00:36:14.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:14.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:36:14.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:36:14.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:14.176 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:36:14.176 | INFO | Total mem usage: 16.7 GiB | 20260508 00:36:14.176 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:36:14.176 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.8%) | 20260508 00:36:14.176 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:36:14.176 | INFO | High Water Breakdown: | 20260508 00:36:14.176 | INFO | 59.9% : 8.87 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:36:14.176 | INFO | 39.9% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:36:14.176 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:36:14.176 | INFO | ------------------------------------------ | 20260508 00:36:14.176 | INFO | Uploaded 36GiB to GPU in 2.53963s (14.1753GiB/s) | 20260508 00:36:14.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:36:14.176 | INFO | Downloaded 34GiB to GPU in 2.45041s (13.8752GiB/s) | 20260508 00:36:14.176 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:36:14.176 | INFO | Total mem usage: 26.5 GiB | 20260508 00:36:14.176 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:36:14.176 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:36:14.176 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:36:14.176 | INFO | High Water Breakdown: | 20260508 00:36:14.176 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:36:14.176 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:36:14.176 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:36:14.176 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:36:14.176 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:36:14.176 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:36:14.176 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:36:14.176 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:36:14.176 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:36:14.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:36:14.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:36:14.176 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:36:14.176 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:36:14.176 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:36:14.176 | INFO | ------------------------------------------ | 20260508 00:36:14.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:36:14.176 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:36:14.176 | INFO | Total mem usage: 27.8 GiB | 20260508 00:36:14.176 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:36:14.176 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:36:14.176 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:36:14.176 | INFO | High Water Breakdown: | 20260508 00:36:14.176 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:36:14.176 | INFO | ------------------------------------------ | 20260508 00:36:14.176 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:36:14.176 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:36:14.177 | INFO | Total mem usage: 15.7 GiB | 20260508 00:36:14.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:36:14.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:36:14.177 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:36:14.177 | INFO | High Water Breakdown: | 20260508 00:36:14.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:36:14.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:36:14.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:36:14.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:36:14.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:36:14.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:36:14.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:36:14.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:36:14.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:36:14.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:36:14.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:36:14.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:36:14.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:36:14.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:36:14.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:36:14.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:36:14.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:36:14.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:36:14.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:36:14.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:36:14.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:36:14.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:36:14.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:36:14.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:36:14.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:36:14.177 | INFO | ------------------------------------------ | 20260508 00:36:14.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:36:14.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:36:14.177 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:36:14.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:36:14.177 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.42e+01 / 1.46e+00 | 20260508 00:36:14.177 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.15e-02 / 4.69e-03 | 20260508 00:36:14.178 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.47e+00, 5.88e-01 / 2.04e-01 | 20260508 00:36:14.178 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.06e-01 / 9.93e-02 | 20260508 00:36:14.178 | INFO | PerfReport: 15.98%, DME -- 4.19e+01 / 4.58e+01, 4.02e+00 / 1.73e+00 | 20260508 00:36:14.178 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.72e+00 / 1.92e+00 | 20260508 00:36:14.178 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.67e+00, 4.26e-01 / 2.34e-01 | 20260508 00:36:14.178 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.32e+00, 2.78e-01 / 1.86e-01 | 20260508 00:36:14.178 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.77e-01, 5.41e-02 / 3.08e-02 | 20260508 00:36:14.178 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:36:14.178 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:36:14.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:36:14.179 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 1.92%, LockUpload -- 2.05e+00 / 0.00e+00, 5.22e+00 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 2.91%, LockDownload -- 3.10e+00 / 0.00e+00, 4.30e+00 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 15.71%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:36:14.179 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260508 00:36:14.275 | INFO | Starting a round of Baz IO | 20260508 00:36:17.726 | INFO | Populating RTMetrics report | 20260508 00:36:17.726 | INFO | RTMetrics report populated | 20260508 00:36:17.726 | INFO | Opening tmp RTMetrics json file | 20260508 00:36:17.756 | INFO | Serializing RTMetrics to json | 20260508 00:36:17.767 | INFO | Serialization done, swapping tmp file now... | 20260508 00:36:17.800 | INFO | RTMetrics done | 20260508 00:36:17.864 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.588 seconds | 20260508 00:36:17.864 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.13% / 0.00% / 57.86% | 20260508 00:36:17.864 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.98714% / 0% / 5.5631e-05% | 20260508 00:36:17.864 | INFO | Baz IO overall: | 20260508 00:36:17.864 | INFO | BytesWritten : 1748940188272 | 20260508 00:36:17.864 | INFO | NumEvents : 846417662497 | 20260508 00:36:19.312 | INFO | Analyzing chunk frames = [3850240,3850752) | 20260508 00:36:19.312 | INFO | MemInfo: { | 20260508 00:36:19.312 | INFO | "gpu" : " 17 GB", | 20260508 00:36:19.312 | INFO | "huge" : "", | 20260508 00:36:19.312 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:19.312 | INFO | "malloc" : " 14 GB", | 20260508 00:36:19.312 | INFO | "pinned" : " 29 GB" | 20260508 00:36:19.312 | INFO | } | 20260508 00:36:19.312 | INFO | Perf report for the last 5.13656 seconds: | 20260508 00:36:19.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:19.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:19.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:19.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:19.312 | INFO | ANALYSIS : 68.4%, 3.0, 106.3ms, 0 | 20260508 00:36:19.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:19.312 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 00:36:19.312 | INFO | PRE_HQ : 65.7%, 1.7, 56.6ms, 0 | 20260508 00:36:19.312 | INFO | BAZWRITER : 69.5%, 4.1, 146.0ms, 0 | 20260508 00:36:19.312 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:36:19.313 | INFO | Uploaded 36GiB to GPU in 2.56712s (14.0235GiB/s) | 20260508 00:36:19.313 | INFO | Downloaded 38GiB to GPU in 2.69593s (14.0953GiB/s) | 20260508 00:36:24.450 | INFO | Analyzing chunk frames = [3850752,3851264) | 20260508 00:36:24.450 | INFO | MemInfo: { | 20260508 00:36:24.450 | INFO | "gpu" : " 17 GB", | 20260508 00:36:24.451 | INFO | "huge" : "", | 20260508 00:36:24.451 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:24.451 | INFO | "malloc" : " 14 GB", | 20260508 00:36:24.451 | INFO | "pinned" : " 29 GB" | 20260508 00:36:24.451 | INFO | } | 20260508 00:36:24.451 | INFO | Perf report for the last 5.13832 seconds: | 20260508 00:36:24.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:24.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:36:24.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:36:24.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:24.451 | INFO | Uploaded 36GiB to GPU in 2.52145s (14.2775GiB/s) | 20260508 00:36:24.451 | INFO | Downloaded 34GiB to GPU in 2.42823s (14.002GiB/s) | 20260508 00:36:29.588 | INFO | Analyzing chunk frames = [3851264,3851776) | 20260508 00:36:29.588 | INFO | MemInfo: { | 20260508 00:36:29.588 | INFO | "gpu" : " 17 GB", | 20260508 00:36:29.588 | INFO | "huge" : "", | 20260508 00:36:29.588 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:29.589 | INFO | "malloc" : " 14 GB", | 20260508 00:36:29.589 | INFO | "pinned" : " 29 GB" | 20260508 00:36:29.589 | INFO | } | 20260508 00:36:29.589 | INFO | Perf report for the last 5.13804 seconds: | 20260508 00:36:29.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:29.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:36:29.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:36:29.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:29.589 | INFO | Uploaded 36GiB to GPU in 2.50729s (14.3581GiB/s) | 20260508 00:36:29.589 | INFO | Downloaded 34GiB to GPU in 2.48829s (13.664GiB/s) | 20260508 00:36:34.731 | INFO | Analyzing chunk frames = [3851776,3852288) | 20260508 00:36:34.731 | INFO | MemInfo: { | 20260508 00:36:34.731 | INFO | "gpu" : " 17 GB", | 20260508 00:36:34.731 | INFO | "huge" : "", | 20260508 00:36:34.731 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:34.731 | INFO | "malloc" : " 14 GB", | 20260508 00:36:34.731 | INFO | "pinned" : " 29 GB" | 20260508 00:36:34.731 | INFO | } | 20260508 00:36:34.731 | INFO | Perf report for the last 5.1426 seconds: | 20260508 00:36:34.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:34.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.732 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:36:34.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.732 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:36:34.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:34.732 | INFO | Uploaded 36GiB to GPU in 2.53747s (14.1873GiB/s) | 20260508 00:36:34.732 | INFO | Downloaded 34GiB to GPU in 2.49951s (13.6027GiB/s) | 20260508 00:36:39.873 | INFO | Analyzing chunk frames = [3852288,3852800) | 20260508 00:36:39.873 | INFO | MemInfo: { | 20260508 00:36:39.873 | INFO | "gpu" : " 17 GB", | 20260508 00:36:39.873 | INFO | "huge" : "", | 20260508 00:36:39.873 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:39.873 | INFO | "malloc" : " 14 GB", | 20260508 00:36:39.873 | INFO | "pinned" : " 29 GB" | 20260508 00:36:39.873 | INFO | } | 20260508 00:36:39.873 | INFO | Perf report for the last 5.14191 seconds: | 20260508 00:36:39.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:39.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.873 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:36:39.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.873 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260508 00:36:39.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:39.874 | INFO | Uploaded 36GiB to GPU in 2.37293s (15.1711GiB/s) | 20260508 00:36:39.874 | INFO | Downloaded 34GiB to GPU in 2.4896s (13.6568GiB/s) | 20260508 00:36:45.011 | INFO | Analyzing chunk frames = [3852800,3853312) | 20260508 00:36:45.011 | INFO | MemInfo: { | 20260508 00:36:45.011 | INFO | "gpu" : " 17 GB", | 20260508 00:36:45.011 | INFO | "huge" : "", | 20260508 00:36:45.011 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:45.011 | INFO | "malloc" : " 14 GB", | 20260508 00:36:45.011 | INFO | "pinned" : " 29 GB" | 20260508 00:36:45.011 | INFO | } | 20260508 00:36:45.011 | INFO | Perf report for the last 5.13803 seconds: | 20260508 00:36:45.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:45.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.011 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:36:45.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.011 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:36:45.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:45.012 | INFO | Uploaded 36GiB to GPU in 2.41037s (14.9355GiB/s) | 20260508 00:36:45.012 | INFO | Downloaded 34GiB to GPU in 2.39992s (14.1671GiB/s) | 20260508 00:36:50.144 | INFO | Analyzing chunk frames = [3853312,3853824) | 20260508 00:36:50.144 | INFO | MemInfo: { | 20260508 00:36:50.144 | INFO | "gpu" : " 17 GB", | 20260508 00:36:50.144 | INFO | "huge" : "", | 20260508 00:36:50.144 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:50.144 | INFO | "malloc" : " 14 GB", | 20260508 00:36:50.144 | INFO | "pinned" : " 29 GB" | 20260508 00:36:50.144 | INFO | } | 20260508 00:36:50.144 | INFO | Perf report for the last 5.13266 seconds: | 20260508 00:36:50.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:50.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 00:36:50.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:36:50.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:50.144 | INFO | Uploaded 36GiB to GPU in 2.54911s (14.1226GiB/s) | 20260508 00:36:50.144 | INFO | Downloaded 34GiB to GPU in 2.44921s (13.882GiB/s) | 20260508 00:36:55.288 | INFO | Analyzing chunk frames = [3853824,3854336) | 20260508 00:36:55.288 | INFO | MemInfo: { | 20260508 00:36:55.288 | INFO | "gpu" : " 17 GB", | 20260508 00:36:55.288 | INFO | "huge" : "", | 20260508 00:36:55.288 | INFO | "huge_shared" : " 29 GB", | 20260508 00:36:55.288 | INFO | "malloc" : " 14 GB", | 20260508 00:36:55.288 | INFO | "pinned" : " 29 GB" | 20260508 00:36:55.288 | INFO | } | 20260508 00:36:55.288 | INFO | Perf report for the last 5.14407 seconds: | 20260508 00:36:55.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:36:55.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:36:55.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:36:55.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:36:55.288 | INFO | Uploaded 36GiB to GPU in 2.52074s (14.2815GiB/s) | 20260508 00:36:55.288 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9295GiB/s) | 20260508 00:36:55.385 | INFO | Starting a round of Baz IO | 20260508 00:36:58.845 | INFO | Populating RTMetrics report | 20260508 00:36:58.845 | INFO | RTMetrics report populated | 20260508 00:36:58.845 | INFO | Opening tmp RTMetrics json file | 20260508 00:36:58.886 | INFO | Serializing RTMetrics to json | 20260508 00:36:58.937 | INFO | Serialization done, swapping tmp file now... | 20260508 00:36:58.944 | INFO | RTMetrics done | 20260508 00:36:59.073 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.687 seconds | 20260508 00:36:59.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.13% / 0.00% / 57.87% | 20260508 00:36:59.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.987065% / 7.94729e-06% / 6.35783e-05% | 20260508 00:36:59.073 | INFO | Baz IO overall: | 20260508 00:36:59.073 | INFO | BytesWritten : 1750752361805 | 20260508 00:36:59.073 | INFO | NumEvents : 847290034922 | 20260508 00:37:00.429 | INFO | Analyzing chunk frames = [3854336,3854848) | 20260508 00:37:00.429 | INFO | MemInfo: { | 20260508 00:37:00.429 | INFO | "gpu" : " 17 GB", | 20260508 00:37:00.429 | INFO | "huge" : "", | 20260508 00:37:00.429 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:00.429 | INFO | "malloc" : " 14 GB", | 20260508 00:37:00.429 | INFO | "pinned" : " 29 GB" | 20260508 00:37:00.429 | INFO | } | 20260508 00:37:00.429 | INFO | Perf report for the last 5.14094 seconds: | 20260508 00:37:00.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:00.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:00.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:00.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:00.429 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 00:37:00.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:00.429 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 00:37:00.429 | INFO | PRE_HQ : 65.5%, 1.7, 56.3ms, 0 | 20260508 00:37:00.429 | INFO | BAZWRITER : 71.3%, 3.6, 133.2ms, 0 | 20260508 00:37:00.429 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.6ms, 0 | 20260508 00:37:00.430 | INFO | Uploaded 36GiB to GPU in 2.58374s (13.9333GiB/s) | 20260508 00:37:00.430 | INFO | Downloaded 38GiB to GPU in 2.67359s (14.2131GiB/s) | 20260508 00:37:05.572 | INFO | Analyzing chunk frames = [3854848,3855360) | 20260508 00:37:05.572 | INFO | MemInfo: { | 20260508 00:37:05.572 | INFO | "gpu" : " 17 GB", | 20260508 00:37:05.572 | INFO | "huge" : "", | 20260508 00:37:05.572 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:05.572 | INFO | "malloc" : " 14 GB", | 20260508 00:37:05.572 | INFO | "pinned" : " 29 GB" | 20260508 00:37:05.572 | INFO | } | 20260508 00:37:05.572 | INFO | Perf report for the last 5.14347 seconds: | 20260508 00:37:05.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:05.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 00:37:05.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:37:05.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:05.573 | INFO | Uploaded 36GiB to GPU in 2.59524s (13.8716GiB/s) | 20260508 00:37:05.573 | INFO | Downloaded 34GiB to GPU in 2.47928s (13.7136GiB/s) | 20260508 00:37:10.709 | INFO | Analyzing chunk frames = [3855360,3855872) | 20260508 00:37:10.709 | INFO | MemInfo: { | 20260508 00:37:10.709 | INFO | "gpu" : " 17 GB", | 20260508 00:37:10.709 | INFO | "huge" : "", | 20260508 00:37:10.709 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:10.709 | INFO | "malloc" : " 14 GB", | 20260508 00:37:10.709 | INFO | "pinned" : " 29 GB" | 20260508 00:37:10.709 | INFO | } | 20260508 00:37:10.709 | INFO | Perf report for the last 5.137 seconds: | 20260508 00:37:10.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:10.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.710 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:37:10.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.710 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:37:10.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:10.710 | INFO | Uploaded 36GiB to GPU in 2.55693s (14.0794GiB/s) | 20260508 00:37:10.710 | INFO | Downloaded 34GiB to GPU in 2.47912s (13.7145GiB/s) | 20260508 00:37:15.844 | INFO | Analyzing chunk frames = [3855872,3856384) | 20260508 00:37:15.844 | INFO | MemInfo: { | 20260508 00:37:15.844 | INFO | "gpu" : " 17 GB", | 20260508 00:37:15.844 | INFO | "huge" : "", | 20260508 00:37:15.844 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:15.844 | INFO | "malloc" : " 14 GB", | 20260508 00:37:15.844 | INFO | "pinned" : " 29 GB" | 20260508 00:37:15.844 | INFO | } | 20260508 00:37:15.845 | INFO | Perf report for the last 5.13506 seconds: | 20260508 00:37:15.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:15.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:37:15.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:37:15.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:15.845 | INFO | Uploaded 36GiB to GPU in 2.56213s (14.0508GiB/s) | 20260508 00:37:15.845 | INFO | Downloaded 34GiB to GPU in 2.47783s (13.7217GiB/s) | 20260508 00:37:20.987 | INFO | Analyzing chunk frames = [3856384,3856896) | 20260508 00:37:20.987 | INFO | MemInfo: { | 20260508 00:37:20.987 | INFO | "gpu" : " 17 GB", | 20260508 00:37:20.987 | INFO | "huge" : "", | 20260508 00:37:20.987 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:20.987 | INFO | "malloc" : " 14 GB", | 20260508 00:37:20.987 | INFO | "pinned" : " 29 GB" | 20260508 00:37:20.987 | INFO | } | 20260508 00:37:20.987 | INFO | Perf report for the last 5.14275 seconds: | 20260508 00:37:20.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:20.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.987 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:37:20.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.987 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 00:37:20.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:20.988 | INFO | Uploaded 36GiB to GPU in 2.41447s (14.9101GiB/s) | 20260508 00:37:20.988 | INFO | Downloaded 34GiB to GPU in 2.49679s (13.6175GiB/s) | 20260508 00:37:26.128 | INFO | Analyzing chunk frames = [3856896,3857408) | 20260508 00:37:26.128 | INFO | MemInfo: { | 20260508 00:37:26.128 | INFO | "gpu" : " 17 GB", | 20260508 00:37:26.128 | INFO | "huge" : "", | 20260508 00:37:26.128 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:26.128 | INFO | "malloc" : " 14 GB", | 20260508 00:37:26.128 | INFO | "pinned" : " 29 GB" | 20260508 00:37:26.128 | INFO | } | 20260508 00:37:26.128 | INFO | Perf report for the last 5.14044 seconds: | 20260508 00:37:26.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:26.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:37:26.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:37:26.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:26.128 | INFO | Uploaded 36GiB to GPU in 2.40401s (14.9749GiB/s) | 20260508 00:37:26.128 | INFO | Downloaded 34GiB to GPU in 2.3934s (14.2058GiB/s) | 20260508 00:37:31.264 | INFO | Analyzing chunk frames = [3857408,3857920) | 20260508 00:37:31.264 | INFO | MemInfo: { | 20260508 00:37:31.264 | INFO | "gpu" : " 17 GB", | 20260508 00:37:31.264 | INFO | "huge" : "", | 20260508 00:37:31.264 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:31.264 | INFO | "malloc" : " 14 GB", | 20260508 00:37:31.264 | INFO | "pinned" : " 29 GB" | 20260508 00:37:31.264 | INFO | } | 20260508 00:37:31.264 | INFO | Perf report for the last 5.13608 seconds: | 20260508 00:37:31.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:31.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 00:37:31.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 00:37:31.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:31.264 | INFO | Uploaded 36GiB to GPU in 2.45226s (14.6804GiB/s) | 20260508 00:37:31.264 | INFO | Downloaded 34GiB to GPU in 2.42589s (14.0155GiB/s) | 20260508 00:37:36.399 | INFO | Analyzing chunk frames = [3857920,3858432) | 20260508 00:37:36.399 | INFO | MemInfo: { | 20260508 00:37:36.399 | INFO | "gpu" : " 17 GB", | 20260508 00:37:36.399 | INFO | "huge" : "", | 20260508 00:37:36.399 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:36.399 | INFO | "malloc" : " 14 GB", | 20260508 00:37:36.399 | INFO | "pinned" : " 29 GB" | 20260508 00:37:36.399 | INFO | } | 20260508 00:37:36.399 | INFO | Perf report for the last 5.13547 seconds: | 20260508 00:37:36.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:36.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.399 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:37:36.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.399 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:37:36.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:36.400 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:37:36.400 | INFO | Total mem usage: 16.7 GiB | 20260508 00:37:36.400 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:37:36.400 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.2%) | 20260508 00:37:36.400 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:37:36.400 | INFO | High Water Breakdown: | 20260508 00:37:36.400 | INFO | 60.4% : 9.14 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:37:36.400 | INFO | 39.5% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:37:36.400 | INFO | Uploaded 36GiB to GPU in 2.5256s (14.254GiB/s) | 20260508 00:37:36.400 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:37:36.400 | INFO | ------------------------------------------ | 20260508 00:37:36.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:37:36.400 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:37:36.400 | INFO | Total mem usage: 26.5 GiB | 20260508 00:37:36.400 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:37:36.400 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:37:36.400 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 00:37:36.400 | INFO | High Water Breakdown: | 20260508 00:37:36.400 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:37:36.400 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:37:36.400 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:37:36.400 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:37:36.400 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:37:36.400 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:37:36.400 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:37:36.400 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:37:36.400 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:37:36.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:37:36.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:37:36.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:37:36.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:37:36.400 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:37:36.400 | INFO | ------------------------------------------ | 20260508 00:37:36.400 | INFO | Downloaded 34GiB to GPU in 2.44583s (13.9012GiB/s) | 20260508 00:37:36.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:37:36.400 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:37:36.400 | INFO | Total mem usage: 27.8 GiB | 20260508 00:37:36.400 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:37:36.400 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:37:36.400 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:37:36.400 | INFO | High Water Breakdown: | 20260508 00:37:36.400 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:37:36.400 | INFO | ------------------------------------------ | 20260508 00:37:36.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:37:36.401 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:37:36.401 | INFO | Total mem usage: 15.7 GiB | 20260508 00:37:36.401 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:37:36.401 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:37:36.401 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:37:36.401 | INFO | High Water Breakdown: | 20260508 00:37:36.401 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:37:36.401 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:37:36.401 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:37:36.401 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:37:36.401 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:37:36.401 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:37:36.401 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:37:36.401 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:37:36.401 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:37:36.401 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:37:36.401 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:37:36.401 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:37:36.401 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:37:36.401 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:37:36.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:37:36.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:37:36.401 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:37:36.401 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:37:36.401 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:37:36.401 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:37:36.401 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:37:36.401 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:37:36.401 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:37:36.401 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:37:36.401 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:37:36.401 | INFO | ------------------------------------------ | 20260508 00:37:36.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:37:36.401 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:37:36.401 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:37:36.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:37:36.401 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.42e+01 / 1.55e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.27e-02 / 8.17e-03 | 20260508 00:37:36.401 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.47e+00, 5.92e-01 / 2.32e-01 | 20260508 00:37:36.401 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.50e+00, 2.07e-01 / 9.22e-02 | 20260508 00:37:36.401 | INFO | PerfReport: 16.11%, DME -- 4.22e+01 / 4.62e+01, 4.13e+00 / 1.77e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.61e+00 / 1.47e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.30e-01 / 2.13e-01 | 20260508 00:37:36.401 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.31e+00, 2.76e-01 / 1.51e-01 | 20260508 00:37:36.401 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.78e-01, 5.38e-02 / 2.80e-02 | 20260508 00:37:36.401 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:37:36.401 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:37:36.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:37:36.401 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 24.25%, Download -- 2.59e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 30.69%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 2.61%, LockDownload -- 2.79e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 15.68%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:37:36.401 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 00:37:36.497 | INFO | Starting a round of Baz IO | 20260508 00:37:39.958 | INFO | Populating RTMetrics report | 20260508 00:37:39.958 | INFO | RTMetrics report populated | 20260508 00:37:39.958 | INFO | Opening tmp RTMetrics json file | 20260508 00:37:40.006 | INFO | Serializing RTMetrics to json | 20260508 00:37:40.034 | INFO | Serialization done, swapping tmp file now... | 20260508 00:37:40.037 | INFO | RTMetrics done | 20260508 00:37:40.065 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.567 seconds | 20260508 00:37:40.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.12% / 0.00% / 57.87% | 20260508 00:37:40.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.986795% / 0% / 5.5631e-05% | 20260508 00:37:40.065 | INFO | Baz IO overall: | 20260508 00:37:40.065 | INFO | BytesWritten : 1752564418197 | 20260508 00:37:40.065 | INFO | NumEvents : 848162611836 | 20260508 00:37:41.540 | INFO | Analyzing chunk frames = [3858432,3858944) | 20260508 00:37:41.540 | INFO | MemInfo: { | 20260508 00:37:41.540 | INFO | "gpu" : " 17 GB", | 20260508 00:37:41.540 | INFO | "huge" : "", | 20260508 00:37:41.540 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:41.540 | INFO | "malloc" : " 14 GB", | 20260508 00:37:41.540 | INFO | "pinned" : " 29 GB" | 20260508 00:37:41.540 | INFO | } | 20260508 00:37:41.540 | INFO | Perf report for the last 5.14061 seconds: | 20260508 00:37:41.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:41.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:41.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:41.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:41.540 | INFO | ANALYSIS : 68.3%, 3.0, 106.7ms, 0 | 20260508 00:37:41.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:41.540 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 00:37:41.540 | INFO | PRE_HQ : 65.4%, 1.7, 56.8ms, 0 | 20260508 00:37:41.540 | INFO | BAZWRITER : 69.0%, 3.6, 127.2ms, 0 | 20260508 00:37:41.540 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 00:37:41.540 | INFO | Uploaded 36GiB to GPU in 2.47758s (14.5303GiB/s) | 20260508 00:37:41.540 | INFO | Downloaded 38GiB to GPU in 2.68897s (14.1318GiB/s) | 20260508 00:37:46.682 | INFO | Analyzing chunk frames = [3858944,3859456) | 20260508 00:37:46.682 | INFO | MemInfo: { | 20260508 00:37:46.682 | INFO | "gpu" : " 17 GB", | 20260508 00:37:46.682 | INFO | "huge" : "", | 20260508 00:37:46.682 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:46.682 | INFO | "malloc" : " 14 GB", | 20260508 00:37:46.682 | INFO | "pinned" : " 29 GB" | 20260508 00:37:46.682 | INFO | } | 20260508 00:37:46.682 | INFO | Perf report for the last 5.14251 seconds: | 20260508 00:37:46.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:46.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.683 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:37:46.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.683 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:37:46.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:46.683 | INFO | Uploaded 36GiB to GPU in 2.53795s (14.1847GiB/s) | 20260508 00:37:46.683 | INFO | Downloaded 34GiB to GPU in 2.50492s (13.5733GiB/s) | 20260508 00:37:51.828 | INFO | Analyzing chunk frames = [3859456,3859968) | 20260508 00:37:51.828 | INFO | MemInfo: { | 20260508 00:37:51.828 | INFO | "gpu" : " 17 GB", | 20260508 00:37:51.828 | INFO | "huge" : "", | 20260508 00:37:51.828 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:51.828 | INFO | "malloc" : " 14 GB", | 20260508 00:37:51.828 | INFO | "pinned" : " 29 GB" | 20260508 00:37:51.828 | INFO | } | 20260508 00:37:51.828 | INFO | Perf report for the last 5.14552 seconds: | 20260508 00:37:51.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:51.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:37:51.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:37:51.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:51.828 | INFO | Uploaded 36GiB to GPU in 2.46702s (14.5925GiB/s) | 20260508 00:37:51.828 | INFO | Downloaded 34GiB to GPU in 2.49272s (13.6397GiB/s) | 20260508 00:37:56.962 | INFO | Analyzing chunk frames = [3859968,3860480) | 20260508 00:37:56.962 | INFO | MemInfo: { | 20260508 00:37:56.962 | INFO | "gpu" : " 17 GB", | 20260508 00:37:56.962 | INFO | "huge" : "", | 20260508 00:37:56.962 | INFO | "huge_shared" : " 29 GB", | 20260508 00:37:56.962 | INFO | "malloc" : " 14 GB", | 20260508 00:37:56.962 | INFO | "pinned" : " 29 GB" | 20260508 00:37:56.962 | INFO | } | 20260508 00:37:56.962 | INFO | Perf report for the last 5.13415 seconds: | 20260508 00:37:56.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:37:56.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:37:56.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:37:56.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:37:56.962 | INFO | Uploaded 36GiB to GPU in 2.45516s (14.663GiB/s) | 20260508 00:37:56.962 | INFO | Downloaded 34GiB to GPU in 2.50433s (13.5765GiB/s) | 20260508 00:38:02.100 | INFO | Analyzing chunk frames = [3860480,3860992) | 20260508 00:38:02.100 | INFO | MemInfo: { | 20260508 00:38:02.100 | INFO | "gpu" : " 17 GB", | 20260508 00:38:02.100 | INFO | "huge" : "", | 20260508 00:38:02.100 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:02.100 | INFO | "malloc" : " 14 GB", | 20260508 00:38:02.100 | INFO | "pinned" : " 29 GB" | 20260508 00:38:02.100 | INFO | } | 20260508 00:38:02.100 | INFO | Perf report for the last 5.13829 seconds: | 20260508 00:38:02.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:02.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.101 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 00:38:02.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.101 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:38:02.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:02.101 | INFO | Uploaded 36GiB to GPU in 2.29542s (15.6834GiB/s) | 20260508 00:38:02.101 | INFO | Downloaded 34GiB to GPU in 2.50923s (13.55GiB/s) | 20260508 00:38:07.237 | INFO | Analyzing chunk frames = [3860992,3861504) | 20260508 00:38:07.237 | INFO | MemInfo: { | 20260508 00:38:07.237 | INFO | "gpu" : " 17 GB", | 20260508 00:38:07.237 | INFO | "huge" : "", | 20260508 00:38:07.237 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:07.237 | INFO | "malloc" : " 14 GB", | 20260508 00:38:07.237 | INFO | "pinned" : " 29 GB" | 20260508 00:38:07.237 | INFO | } | 20260508 00:38:07.237 | INFO | Perf report for the last 5.13686 seconds: | 20260508 00:38:07.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:07.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.237 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 00:38:07.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.237 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:38:07.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:07.238 | INFO | Uploaded 36GiB to GPU in 2.36062s (15.2502GiB/s) | 20260508 00:38:07.238 | INFO | Downloaded 34GiB to GPU in 2.42899s (13.9976GiB/s) | 20260508 00:38:12.377 | INFO | Analyzing chunk frames = [3861504,3862016) | 20260508 00:38:12.377 | INFO | MemInfo: { | 20260508 00:38:12.377 | INFO | "gpu" : " 17 GB", | 20260508 00:38:12.377 | INFO | "huge" : "", | 20260508 00:38:12.377 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:12.377 | INFO | "malloc" : " 14 GB", | 20260508 00:38:12.377 | INFO | "pinned" : " 29 GB" | 20260508 00:38:12.377 | INFO | } | 20260508 00:38:12.377 | INFO | Perf report for the last 5.13947 seconds: | 20260508 00:38:12.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:12.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:38:12.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:38:12.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:12.377 | INFO | Uploaded 36GiB to GPU in 2.53599s (14.1956GiB/s) | 20260508 00:38:12.377 | INFO | Downloaded 34GiB to GPU in 2.48048s (13.707GiB/s) | 20260508 00:38:17.523 | INFO | Analyzing chunk frames = [3862016,3862528) | 20260508 00:38:17.523 | INFO | MemInfo: { | 20260508 00:38:17.523 | INFO | "gpu" : " 17 GB", | 20260508 00:38:17.523 | INFO | "huge" : "", | 20260508 00:38:17.523 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:17.523 | INFO | "malloc" : " 14 GB", | 20260508 00:38:17.524 | INFO | "pinned" : " 29 GB" | 20260508 00:38:17.524 | INFO | } | 20260508 00:38:17.524 | INFO | Perf report for the last 5.14693 seconds: | 20260508 00:38:17.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:17.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | ANALYSIS : 67.0%, 3.0, 104.6ms, 0 | 20260508 00:38:17.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 00:38:17.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:17.524 | INFO | Uploaded 36GiB to GPU in 2.44622s (14.7166GiB/s) | 20260508 00:38:17.524 | INFO | Downloaded 34GiB to GPU in 2.45866s (13.8287GiB/s) | 20260508 00:38:17.635 | INFO | Starting a round of Baz IO | 20260508 00:38:21.096 | INFO | Populating RTMetrics report | 20260508 00:38:21.096 | INFO | RTMetrics report populated | 20260508 00:38:21.096 | INFO | Opening tmp RTMetrics json file | 20260508 00:38:21.115 | INFO | Serializing RTMetrics to json | 20260508 00:38:21.117 | INFO | Serialization done, swapping tmp file now... | 20260508 00:38:21.118 | INFO | RTMetrics done | 20260508 00:38:21.198 | INFO | Finished a round of Baz IO: Wrote 1.687 GiB in 3.562 seconds | 20260508 00:38:21.198 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.12% / 0.00% / 57.88% | 20260508 00:38:21.198 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.983353% / 1.19209e-05% / 4.37101e-05% | 20260508 00:38:21.198 | INFO | Baz IO overall: | 20260508 00:38:21.198 | INFO | BytesWritten : 1754375403982 | 20260508 00:38:21.198 | INFO | NumEvents : 849034225688 | 20260508 00:38:22.657 | INFO | Analyzing chunk frames = [3862528,3863040) | 20260508 00:38:22.657 | INFO | MemInfo: { | 20260508 00:38:22.657 | INFO | "gpu" : " 17 GB", | 20260508 00:38:22.657 | INFO | "huge" : "", | 20260508 00:38:22.657 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:22.657 | INFO | "malloc" : " 14 GB", | 20260508 00:38:22.657 | INFO | "pinned" : " 29 GB" | 20260508 00:38:22.657 | INFO | } | 20260508 00:38:22.657 | INFO | Perf report for the last 5.1337 seconds: | 20260508 00:38:22.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:22.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:22.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:22.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:22.657 | INFO | ANALYSIS : 68.8%, 2.9, 104.5ms, 0 | 20260508 00:38:22.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:22.658 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260508 00:38:22.658 | INFO | PRE_HQ : 67.9%, 1.8, 62.5ms, 0 | 20260508 00:38:22.658 | INFO | BAZWRITER : 69.0%, 4.0, 142.5ms, 0 | 20260508 00:38:22.658 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:38:22.658 | INFO | Uploaded 36GiB to GPU in 2.61908s (13.7453GiB/s) | 20260508 00:38:22.658 | INFO | Downloaded 38GiB to GPU in 2.70435s (14.0514GiB/s) | 20260508 00:38:27.801 | INFO | Analyzing chunk frames = [3863040,3863552) | 20260508 00:38:27.801 | INFO | MemInfo: { | 20260508 00:38:27.801 | INFO | "gpu" : " 17 GB", | 20260508 00:38:27.801 | INFO | "huge" : "", | 20260508 00:38:27.801 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:27.801 | INFO | "malloc" : " 14 GB", | 20260508 00:38:27.801 | INFO | "pinned" : " 29 GB" | 20260508 00:38:27.801 | INFO | } | 20260508 00:38:27.801 | INFO | Perf report for the last 5.1437 seconds: | 20260508 00:38:27.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:27.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:38:27.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:38:27.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:27.801 | INFO | Uploaded 36GiB to GPU in 2.54758s (14.1311GiB/s) | 20260508 00:38:27.801 | INFO | Downloaded 34GiB to GPU in 2.49987s (13.6007GiB/s) | 20260508 00:38:32.938 | INFO | Analyzing chunk frames = [3863552,3864064) | 20260508 00:38:32.938 | INFO | MemInfo: { | 20260508 00:38:32.938 | INFO | "gpu" : " 17 GB", | 20260508 00:38:32.938 | INFO | "huge" : "", | 20260508 00:38:32.938 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:32.938 | INFO | "malloc" : " 14 GB", | 20260508 00:38:32.938 | INFO | "pinned" : " 29 GB" | 20260508 00:38:32.939 | INFO | } | 20260508 00:38:32.939 | INFO | Perf report for the last 5.13729 seconds: | 20260508 00:38:32.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:32.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:38:32.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:38:32.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:32.939 | INFO | Uploaded 36GiB to GPU in 2.46564s (14.6007GiB/s) | 20260508 00:38:32.939 | INFO | Downloaded 34GiB to GPU in 2.50568s (13.5692GiB/s) | 20260508 00:38:38.076 | INFO | Analyzing chunk frames = [3864064,3864576) | 20260508 00:38:38.076 | INFO | MemInfo: { | 20260508 00:38:38.076 | INFO | "gpu" : " 17 GB", | 20260508 00:38:38.076 | INFO | "huge" : "", | 20260508 00:38:38.076 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:38.076 | INFO | "malloc" : " 14 GB", | 20260508 00:38:38.076 | INFO | "pinned" : " 29 GB" | 20260508 00:38:38.076 | INFO | } | 20260508 00:38:38.076 | INFO | Perf report for the last 5.13764 seconds: | 20260508 00:38:38.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:38.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.076 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:38:38.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.076 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:38:38.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:38.077 | INFO | Uploaded 36GiB to GPU in 2.46643s (14.596GiB/s) | 20260508 00:38:38.077 | INFO | Downloaded 34GiB to GPU in 2.49269s (13.6399GiB/s) | 20260508 00:38:43.211 | INFO | Analyzing chunk frames = [3864576,3865088) | 20260508 00:38:43.211 | INFO | MemInfo: { | 20260508 00:38:43.212 | INFO | "gpu" : " 17 GB", | 20260508 00:38:43.212 | INFO | "huge" : "", | 20260508 00:38:43.212 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:43.212 | INFO | "malloc" : " 14 GB", | 20260508 00:38:43.212 | INFO | "pinned" : " 29 GB" | 20260508 00:38:43.212 | INFO | } | 20260508 00:38:43.212 | INFO | Perf report for the last 5.13537 seconds: | 20260508 00:38:43.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:43.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 00:38:43.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 00:38:43.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:43.212 | INFO | Uploaded 36GiB to GPU in 2.33197s (15.4376GiB/s) | 20260508 00:38:43.212 | INFO | Downloaded 34GiB to GPU in 2.47393s (13.7433GiB/s) | 20260508 00:38:48.347 | INFO | Analyzing chunk frames = [3865088,3865600) | 20260508 00:38:48.347 | INFO | MemInfo: { | 20260508 00:38:48.347 | INFO | "gpu" : " 17 GB", | 20260508 00:38:48.347 | INFO | "huge" : "", | 20260508 00:38:48.347 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:48.347 | INFO | "malloc" : " 14 GB", | 20260508 00:38:48.347 | INFO | "pinned" : " 29 GB" | 20260508 00:38:48.347 | INFO | } | 20260508 00:38:48.347 | INFO | Perf report for the last 5.13543 seconds: | 20260508 00:38:48.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:48.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:38:48.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:38:48.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:48.347 | INFO | Uploaded 36GiB to GPU in 2.39998s (15.0001GiB/s) | 20260508 00:38:48.348 | INFO | Downloaded 34GiB to GPU in 2.40635s (14.1293GiB/s) | 20260508 00:38:53.491 | INFO | Analyzing chunk frames = [3865600,3866112) | 20260508 00:38:53.491 | INFO | MemInfo: { | 20260508 00:38:53.491 | INFO | "gpu" : " 17 GB", | 20260508 00:38:53.491 | INFO | "huge" : "", | 20260508 00:38:53.491 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:53.491 | INFO | "malloc" : " 14 GB", | 20260508 00:38:53.492 | INFO | "pinned" : " 29 GB" | 20260508 00:38:53.492 | INFO | } | 20260508 00:38:53.492 | INFO | Perf report for the last 5.14461 seconds: | 20260508 00:38:53.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:53.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 00:38:53.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 00:38:53.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:53.492 | INFO | Uploaded 36GiB to GPU in 2.50601s (14.3655GiB/s) | 20260508 00:38:53.492 | INFO | Downloaded 34GiB to GPU in 2.45426s (13.8535GiB/s) | 20260508 00:38:58.633 | INFO | Analyzing chunk frames = [3866112,3866624) | 20260508 00:38:58.633 | INFO | MemInfo: { | 20260508 00:38:58.633 | INFO | "gpu" : " 17 GB", | 20260508 00:38:58.633 | INFO | "huge" : "", | 20260508 00:38:58.633 | INFO | "huge_shared" : " 29 GB", | 20260508 00:38:58.633 | INFO | "malloc" : " 14 GB", | 20260508 00:38:58.633 | INFO | "pinned" : " 29 GB" | 20260508 00:38:58.633 | INFO | } | 20260508 00:38:58.633 | INFO | Perf report for the last 5.14161 seconds: | 20260508 00:38:58.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:38:58.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.633 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:38:58.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.633 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 00:38:58.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:38:58.634 | INFO | Uploaded 36GiB to GPU in 2.53254s (14.215GiB/s) | 20260508 00:38:58.634 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:38:58.634 | INFO | Total mem usage: 16.7 GiB | 20260508 00:38:58.634 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:38:58.634 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260508 00:38:58.634 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:38:58.634 | INFO | High Water Breakdown: | 20260508 00:38:58.634 | INFO | 60.1% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:38:58.634 | INFO | 39.7% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:38:58.634 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:38:58.634 | INFO | ------------------------------------------ | 20260508 00:38:58.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:38:58.634 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:38:58.634 | INFO | Total mem usage: 26.5 GiB | 20260508 00:38:58.634 | INFO | Downloaded 34GiB to GPU in 2.45101s (13.8718GiB/s) | 20260508 00:38:58.634 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:38:58.634 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:38:58.634 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:38:58.634 | INFO | High Water Breakdown: | 20260508 00:38:58.634 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:38:58.634 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:38:58.634 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:38:58.634 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:38:58.634 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:38:58.634 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:38:58.634 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:38:58.634 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:38:58.634 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:38:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:38:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:38:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:38:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:38:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:38:58.634 | INFO | ------------------------------------------ | 20260508 00:38:58.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:38:58.634 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:38:58.635 | INFO | Total mem usage: 27.8 GiB | 20260508 00:38:58.635 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:38:58.635 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:38:58.635 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:38:58.635 | INFO | High Water Breakdown: | 20260508 00:38:58.635 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:38:58.635 | INFO | ------------------------------------------ | 20260508 00:38:58.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:38:58.635 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:38:58.635 | INFO | Total mem usage: 15.7 GiB | 20260508 00:38:58.635 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:38:58.635 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:38:58.635 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:38:58.635 | INFO | High Water Breakdown: | 20260508 00:38:58.635 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:38:58.635 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:38:58.635 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:38:58.635 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:38:58.635 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:38:58.635 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:38:58.635 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:38:58.635 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:38:58.635 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:38:58.635 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:38:58.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:38:58.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:38:58.635 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:38:58.635 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:38:58.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:38:58.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:38:58.635 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:38:58.635 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:38:58.635 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:38:58.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:38:58.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:38:58.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:38:58.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:38:58.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:38:58.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:38:58.635 | INFO | ------------------------------------------ | 20260508 00:38:58.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:38:58.635 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:38:58.635 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:38:58.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:38:58.635 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.42e+01 / 1.34e+00 | 20260508 00:38:58.635 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.01e-01, 1.10e-02 / 5.03e-03 | 20260508 00:38:58.635 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.47e+00, 5.82e-01 / 2.09e-01 | 20260508 00:38:58.635 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.04e-01 / 1.21e-01 | 20260508 00:38:58.635 | INFO | PerfReport: 16.13%, DME -- 4.23e+01 / 4.63e+01, 4.12e+00 / 1.79e+00 | 20260508 00:38:58.635 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.68e+00 | 20260508 00:38:58.635 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 2.70e+00, 4.31e-01 / 2.55e-01 | 20260508 00:38:58.635 | INFO | PerfReport: 3.41%, Metrics -- 1.12e+00 / 1.31e+00, 2.79e-01 / 1.52e-01 | 20260508 00:38:58.635 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.75e-01, 5.40e-02 / 3.05e-02 | 20260508 00:38:58.635 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:38:58.635 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:38:58.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:38:58.636 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 24.13%, Upload -- 2.57e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 24.51%, Download -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 1.91%, LockUpload -- 2.03e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 3.08%, LockDownload -- 3.28e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 00:38:58.636 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260508 00:38:58.746 | INFO | Starting a round of Baz IO | 20260508 00:39:02.202 | INFO | Populating RTMetrics report | 20260508 00:39:02.202 | INFO | RTMetrics report populated | 20260508 00:39:02.202 | INFO | Opening tmp RTMetrics json file | 20260508 00:39:02.208 | INFO | Serializing RTMetrics to json | 20260508 00:39:02.224 | INFO | Serialization done, swapping tmp file now... | 20260508 00:39:02.245 | INFO | RTMetrics done | 20260508 00:39:02.340 | INFO | Finished a round of Baz IO: Wrote 1.683 GiB in 3.594 seconds | 20260508 00:39:02.340 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.12% / 0.00% / 57.88% | 20260508 00:39:02.340 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.976535% / 3.97364e-06% / 4.37101e-05% | 20260508 00:39:02.340 | INFO | Baz IO overall: | 20260508 00:39:02.340 | INFO | BytesWritten : 1756182674389 | 20260508 00:39:02.340 | INFO | NumEvents : 849902972589 | 20260508 00:39:03.775 | INFO | Analyzing chunk frames = [3866624,3867136) | 20260508 00:39:03.775 | INFO | MemInfo: { | 20260508 00:39:03.775 | INFO | "gpu" : " 17 GB", | 20260508 00:39:03.775 | INFO | "huge" : "", | 20260508 00:39:03.775 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:03.775 | INFO | "malloc" : " 14 GB", | 20260508 00:39:03.775 | INFO | "pinned" : " 29 GB" | 20260508 00:39:03.775 | INFO | } | 20260508 00:39:03.775 | INFO | Perf report for the last 5.14172 seconds: | 20260508 00:39:03.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:03.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:03.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:03.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:03.775 | INFO | ANALYSIS : 68.7%, 2.9, 104.5ms, 0 | 20260508 00:39:03.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:03.775 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260508 00:39:03.775 | INFO | PRE_HQ : 67.3%, 1.8, 62.5ms, 0 | 20260508 00:39:03.775 | INFO | BAZWRITER : 69.4%, 3.7, 133.5ms, 0 | 20260508 00:39:03.775 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:39:03.775 | INFO | Uploaded 36GiB to GPU in 2.59595s (13.8678GiB/s) | 20260508 00:39:03.775 | INFO | Downloaded 38GiB to GPU in 2.71326s (14.0053GiB/s) | 20260508 00:39:08.911 | INFO | Analyzing chunk frames = [3867136,3867648) | 20260508 00:39:08.911 | INFO | MemInfo: { | 20260508 00:39:08.911 | INFO | "gpu" : " 17 GB", | 20260508 00:39:08.911 | INFO | "huge" : "", | 20260508 00:39:08.911 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:08.911 | INFO | "malloc" : " 14 GB", | 20260508 00:39:08.911 | INFO | "pinned" : " 29 GB" | 20260508 00:39:08.911 | INFO | } | 20260508 00:39:08.911 | INFO | Perf report for the last 5.13612 seconds: | 20260508 00:39:08.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:08.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.911 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:39:08.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.911 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:39:08.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:08.912 | INFO | Uploaded 36GiB to GPU in 2.54478s (14.1466GiB/s) | 20260508 00:39:08.912 | INFO | Downloaded 34GiB to GPU in 2.41538s (14.0765GiB/s) | 20260508 00:39:14.052 | INFO | Analyzing chunk frames = [3867648,3868160) | 20260508 00:39:14.052 | INFO | MemInfo: { | 20260508 00:39:14.052 | INFO | "gpu" : " 17 GB", | 20260508 00:39:14.052 | INFO | "huge" : "", | 20260508 00:39:14.052 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:14.052 | INFO | "malloc" : " 14 GB", | 20260508 00:39:14.052 | INFO | "pinned" : " 29 GB" | 20260508 00:39:14.052 | INFO | } | 20260508 00:39:14.052 | INFO | Perf report for the last 5.14098 seconds: | 20260508 00:39:14.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:14.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:39:14.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:39:14.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:14.052 | INFO | Uploaded 36GiB to GPU in 2.56933s (14.0114GiB/s) | 20260508 00:39:14.052 | INFO | Downloaded 34GiB to GPU in 2.48256s (13.6955GiB/s) | 20260508 00:39:19.190 | INFO | Analyzing chunk frames = [3868160,3868672) | 20260508 00:39:19.190 | INFO | MemInfo: { | 20260508 00:39:19.190 | INFO | "gpu" : " 17 GB", | 20260508 00:39:19.190 | INFO | "huge" : "", | 20260508 00:39:19.190 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:19.190 | INFO | "malloc" : " 14 GB", | 20260508 00:39:19.190 | INFO | "pinned" : " 29 GB" | 20260508 00:39:19.190 | INFO | } | 20260508 00:39:19.190 | INFO | Perf report for the last 5.13818 seconds: | 20260508 00:39:19.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:19.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 00:39:19.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:39:19.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:19.191 | INFO | Uploaded 36GiB to GPU in 2.53713s (14.1893GiB/s) | 20260508 00:39:19.191 | INFO | Downloaded 34GiB to GPU in 2.4684s (13.7741GiB/s) | 20260508 00:39:24.330 | INFO | Analyzing chunk frames = [3868672,3869184) | 20260508 00:39:24.330 | INFO | MemInfo: { | 20260508 00:39:24.330 | INFO | "gpu" : " 17 GB", | 20260508 00:39:24.330 | INFO | "huge" : "", | 20260508 00:39:24.330 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:24.330 | INFO | "malloc" : " 14 GB", | 20260508 00:39:24.330 | INFO | "pinned" : " 29 GB" | 20260508 00:39:24.330 | INFO | } | 20260508 00:39:24.330 | INFO | Perf report for the last 5.13998 seconds: | 20260508 00:39:24.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:24.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.330 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 00:39:24.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.330 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:39:24.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:24.331 | INFO | Uploaded 36GiB to GPU in 2.43082s (14.8098GiB/s) | 20260508 00:39:24.331 | INFO | Downloaded 34GiB to GPU in 2.49355s (13.6352GiB/s) | 20260508 00:39:29.469 | INFO | Analyzing chunk frames = [3869184,3869696) | 20260508 00:39:29.469 | INFO | MemInfo: { | 20260508 00:39:29.469 | INFO | "gpu" : " 17 GB", | 20260508 00:39:29.469 | INFO | "huge" : "", | 20260508 00:39:29.469 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:29.469 | INFO | "malloc" : " 14 GB", | 20260508 00:39:29.469 | INFO | "pinned" : " 29 GB" | 20260508 00:39:29.469 | INFO | } | 20260508 00:39:29.469 | INFO | Perf report for the last 5.13843 seconds: | 20260508 00:39:29.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:29.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:39:29.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:39:29.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:29.469 | INFO | Uploaded 36GiB to GPU in 2.39261s (15.0464GiB/s) | 20260508 00:39:29.469 | INFO | Downloaded 34GiB to GPU in 2.41843s (14.0587GiB/s) | 20260508 00:39:34.610 | INFO | Analyzing chunk frames = [3869696,3870208) | 20260508 00:39:34.610 | INFO | MemInfo: { | 20260508 00:39:34.610 | INFO | "gpu" : " 17 GB", | 20260508 00:39:34.610 | INFO | "huge" : "", | 20260508 00:39:34.610 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:34.610 | INFO | "malloc" : " 14 GB", | 20260508 00:39:34.610 | INFO | "pinned" : " 29 GB" | 20260508 00:39:34.611 | INFO | } | 20260508 00:39:34.611 | INFO | Perf report for the last 5.14186 seconds: | 20260508 00:39:34.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:34.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 00:39:34.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 00:39:34.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:34.611 | INFO | Uploaded 36GiB to GPU in 2.50778s (14.3553GiB/s) | 20260508 00:39:34.611 | INFO | Downloaded 34GiB to GPU in 2.45053s (13.8746GiB/s) | 20260508 00:39:39.752 | INFO | Analyzing chunk frames = [3870208,3870720) | 20260508 00:39:39.752 | INFO | MemInfo: { | 20260508 00:39:39.752 | INFO | "gpu" : " 17 GB", | 20260508 00:39:39.752 | INFO | "huge" : "", | 20260508 00:39:39.752 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:39.752 | INFO | "malloc" : " 14 GB", | 20260508 00:39:39.752 | INFO | "pinned" : " 29 GB" | 20260508 00:39:39.752 | INFO | } | 20260508 00:39:39.752 | INFO | Perf report for the last 5.14155 seconds: | 20260508 00:39:39.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:39.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.752 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:39:39.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.752 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 00:39:39.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:39.753 | INFO | Uploaded 36GiB to GPU in 2.53503s (14.201GiB/s) | 20260508 00:39:39.753 | INFO | Downloaded 34GiB to GPU in 2.45621s (13.8425GiB/s) | 20260508 00:39:39.865 | INFO | Starting a round of Baz IO | 20260508 00:39:43.329 | INFO | Populating RTMetrics report | 20260508 00:39:43.329 | INFO | RTMetrics report populated | 20260508 00:39:43.329 | INFO | Opening tmp RTMetrics json file | 20260508 00:39:43.339 | INFO | Serializing RTMetrics to json | 20260508 00:39:43.357 | INFO | Serialization done, swapping tmp file now... | 20260508 00:39:43.362 | INFO | RTMetrics done | 20260508 00:39:43.441 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.576 seconds | 20260508 00:39:43.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.11% / 0.00% / 57.88% | 20260508 00:39:43.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.977798% / 0% / 9.53674e-05% | 20260508 00:39:43.441 | INFO | Baz IO overall: | 20260508 00:39:43.441 | INFO | BytesWritten : 1757991415401 | 20260508 00:39:43.441 | INFO | NumEvents : 850772800427 | 20260508 00:39:44.888 | INFO | Analyzing chunk frames = [3870720,3871232) | 20260508 00:39:44.888 | INFO | MemInfo: { | 20260508 00:39:44.888 | INFO | "gpu" : " 17 GB", | 20260508 00:39:44.888 | INFO | "huge" : "", | 20260508 00:39:44.888 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:44.888 | INFO | "malloc" : " 14 GB", | 20260508 00:39:44.888 | INFO | "pinned" : " 29 GB" | 20260508 00:39:44.888 | INFO | } | 20260508 00:39:44.889 | INFO | Perf report for the last 5.13639 seconds: | 20260508 00:39:44.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:44.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:44.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:44.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:44.889 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260508 00:39:44.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:44.889 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 00:39:44.889 | INFO | PRE_HQ : 67.7%, 1.8, 62.7ms, 0 | 20260508 00:39:44.889 | INFO | BAZWRITER : 69.2%, 4.1, 146.2ms, 0 | 20260508 00:39:44.889 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 00:39:44.889 | INFO | Uploaded 36GiB to GPU in 2.53527s (14.1997GiB/s) | 20260508 00:39:44.889 | INFO | Downloaded 38GiB to GPU in 2.72112s (13.9648GiB/s) | 20260508 00:39:50.025 | INFO | Analyzing chunk frames = [3871232,3871744) | 20260508 00:39:50.025 | INFO | MemInfo: { | 20260508 00:39:50.025 | INFO | "gpu" : " 17 GB", | 20260508 00:39:50.025 | INFO | "huge" : "", | 20260508 00:39:50.025 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:50.026 | INFO | "malloc" : " 14 GB", | 20260508 00:39:50.026 | INFO | "pinned" : " 29 GB" | 20260508 00:39:50.026 | INFO | } | 20260508 00:39:50.026 | INFO | Perf report for the last 5.13697 seconds: | 20260508 00:39:50.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:50.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:39:50.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:39:50.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:50.026 | INFO | Uploaded 36GiB to GPU in 2.50548s (14.3685GiB/s) | 20260508 00:39:50.026 | INFO | Downloaded 34GiB to GPU in 2.50972s (13.5473GiB/s) | 20260508 00:39:55.159 | INFO | Analyzing chunk frames = [3871744,3872256) | 20260508 00:39:55.159 | INFO | MemInfo: { | 20260508 00:39:55.159 | INFO | "gpu" : " 17 GB", | 20260508 00:39:55.159 | INFO | "huge" : "", | 20260508 00:39:55.159 | INFO | "huge_shared" : " 29 GB", | 20260508 00:39:55.159 | INFO | "malloc" : " 14 GB", | 20260508 00:39:55.159 | INFO | "pinned" : " 29 GB" | 20260508 00:39:55.159 | INFO | } | 20260508 00:39:55.159 | INFO | Perf report for the last 5.13378 seconds: | 20260508 00:39:55.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:39:55.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.159 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 00:39:55.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.160 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:39:55.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:39:55.160 | INFO | Uploaded 36GiB to GPU in 2.45081s (14.689GiB/s) | 20260508 00:39:55.160 | INFO | Downloaded 34GiB to GPU in 2.50962s (13.5479GiB/s) | 20260508 00:40:00.302 | INFO | Analyzing chunk frames = [3872256,3872768) | 20260508 00:40:00.302 | INFO | MemInfo: { | 20260508 00:40:00.302 | INFO | "gpu" : " 17 GB", | 20260508 00:40:00.302 | INFO | "huge" : "", | 20260508 00:40:00.302 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:00.302 | INFO | "malloc" : " 14 GB", | 20260508 00:40:00.302 | INFO | "pinned" : " 29 GB" | 20260508 00:40:00.302 | INFO | } | 20260508 00:40:00.302 | INFO | Perf report for the last 5.14272 seconds: | 20260508 00:40:00.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:00.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.302 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 00:40:00.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.302 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:40:00.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:00.303 | INFO | Uploaded 36GiB to GPU in 2.44678s (14.7132GiB/s) | 20260508 00:40:00.303 | INFO | Downloaded 34GiB to GPU in 2.5017s (13.5907GiB/s) | 20260508 00:40:05.444 | INFO | Analyzing chunk frames = [3872768,3873280) | 20260508 00:40:05.444 | INFO | MemInfo: { | 20260508 00:40:05.444 | INFO | "gpu" : " 17 GB", | 20260508 00:40:05.444 | INFO | "huge" : "", | 20260508 00:40:05.444 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:05.444 | INFO | "malloc" : " 14 GB", | 20260508 00:40:05.444 | INFO | "pinned" : " 29 GB" | 20260508 00:40:05.444 | INFO | } | 20260508 00:40:05.444 | INFO | Perf report for the last 5.14213 seconds: | 20260508 00:40:05.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:05.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 00:40:05.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:40:05.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:05.445 | INFO | Uploaded 36GiB to GPU in 2.30794s (15.5983GiB/s) | 20260508 00:40:05.445 | INFO | Downloaded 34GiB to GPU in 2.50505s (13.5726GiB/s) | 20260508 00:40:10.590 | INFO | Analyzing chunk frames = [3873280,3873792) | 20260508 00:40:10.590 | INFO | MemInfo: { | 20260508 00:40:10.590 | INFO | "gpu" : " 17 GB", | 20260508 00:40:10.590 | INFO | "huge" : "", | 20260508 00:40:10.590 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:10.590 | INFO | "malloc" : " 14 GB", | 20260508 00:40:10.590 | INFO | "pinned" : " 29 GB" | 20260508 00:40:10.590 | INFO | } | 20260508 00:40:10.590 | INFO | Perf report for the last 5.14565 seconds: | 20260508 00:40:10.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:10.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 00:40:10.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:40:10.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:10.590 | INFO | Uploaded 36GiB to GPU in 2.39904s (15.006GiB/s) | 20260508 00:40:10.591 | INFO | Downloaded 34GiB to GPU in 2.40509s (14.1367GiB/s) | 20260508 00:40:15.725 | INFO | Analyzing chunk frames = [3873792,3874304) | 20260508 00:40:15.725 | INFO | MemInfo: { | 20260508 00:40:15.725 | INFO | "gpu" : " 17 GB", | 20260508 00:40:15.726 | INFO | "huge" : "", | 20260508 00:40:15.726 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:15.726 | INFO | "malloc" : " 14 GB", | 20260508 00:40:15.726 | INFO | "pinned" : " 29 GB" | 20260508 00:40:15.726 | INFO | } | 20260508 00:40:15.726 | INFO | Perf report for the last 5.13595 seconds: | 20260508 00:40:15.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:15.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.726 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 00:40:15.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.726 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 00:40:15.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:15.727 | INFO | Uploaded 36GiB to GPU in 2.40689s (14.9571GiB/s) | 20260508 00:40:15.727 | INFO | Downloaded 34GiB to GPU in 2.42656s (14.0116GiB/s) | 20260508 00:40:20.859 | INFO | Analyzing chunk frames = [3874304,3874816) | 20260508 00:40:20.859 | INFO | MemInfo: { | 20260508 00:40:20.859 | INFO | "gpu" : " 17 GB", | 20260508 00:40:20.859 | INFO | "huge" : "", | 20260508 00:40:20.859 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:20.860 | INFO | "malloc" : " 14 GB", | 20260508 00:40:20.860 | INFO | "pinned" : " 29 GB" | 20260508 00:40:20.860 | INFO | } | 20260508 00:40:20.860 | INFO | Perf report for the last 5.13366 seconds: | 20260508 00:40:20.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:20.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260508 00:40:20.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260508 00:40:20.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:20.860 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:40:20.860 | INFO | Total mem usage: 16.7 GiB | 20260508 00:40:20.860 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:40:20.860 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 00:40:20.860 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:40:20.860 | INFO | High Water Breakdown: | 20260508 00:40:20.860 | INFO | 59.8% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:40:20.860 | INFO | 40.1% : 5.86 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:40:20.860 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:40:20.860 | INFO | ------------------------------------------ | 20260508 00:40:20.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:40:20.860 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:40:20.860 | INFO | Total mem usage: 26.5 GiB | 20260508 00:40:20.860 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:40:20.860 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:40:20.860 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:40:20.860 | INFO | High Water Breakdown: | 20260508 00:40:20.860 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:40:20.860 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:40:20.860 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:40:20.860 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:40:20.860 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:40:20.860 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:40:20.860 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:40:20.860 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:40:20.860 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:40:20.860 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:40:20.860 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:40:20.860 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:40:20.860 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:40:20.860 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:40:20.860 | INFO | ------------------------------------------ | 20260508 00:40:20.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:40:20.860 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:40:20.860 | INFO | Total mem usage: 27.8 GiB | 20260508 00:40:20.860 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:40:20.860 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:40:20.860 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:40:20.860 | INFO | High Water Breakdown: | 20260508 00:40:20.860 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:40:20.860 | INFO | ------------------------------------------ | 20260508 00:40:20.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:40:20.860 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:40:20.860 | INFO | Total mem usage: 15.7 GiB | 20260508 00:40:20.860 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:40:20.860 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:40:20.860 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:40:20.860 | INFO | High Water Breakdown: | 20260508 00:40:20.860 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:40:20.860 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:40:20.860 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:40:20.860 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:40:20.860 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:40:20.860 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:40:20.860 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:40:20.860 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:40:20.860 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:40:20.860 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:40:20.860 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:40:20.860 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:40:20.860 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:40:20.860 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:40:20.860 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:40:20.860 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:40:20.860 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:40:20.860 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:40:20.860 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:40:20.860 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:40:20.860 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:40:20.860 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:40:20.860 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:40:20.860 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:40:20.860 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:40:20.860 | INFO | ------------------------------------------ | 20260508 00:40:20.860 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:40:20.860 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:40:20.860 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:40:20.860 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:40:20.860 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.42e+01 / 1.42e+00 | 20260508 00:40:20.860 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.11e-02 / 5.03e-03 | 20260508 00:40:20.860 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.48e+00, 5.86e-01 / 2.12e-01 | 20260508 00:40:20.860 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.50e+00, 2.05e-01 / 9.02e-02 | 20260508 00:40:20.860 | INFO | PerfReport: 16.11%, DME -- 4.23e+01 / 4.62e+01, 3.87e+00 / 1.47e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.62e+00 / 1.55e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 2.69e+00, 4.27e-01 / 2.60e-01 | 20260508 00:40:20.861 | INFO | PerfReport: 3.37%, Metrics -- 1.11e+00 / 1.27e+00, 2.70e-01 / 1.23e-01 | 20260508 00:40:20.861 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.76e-01, 5.38e-02 / 2.80e-02 | 20260508 00:40:20.861 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:40:20.861 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:40:20.861 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:40:20.861 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:40:20.861 | INFO | Uploaded 36GiB to GPU in 2.55296s (14.1013GiB/s) | 20260508 00:40:20.861 | INFO | Downloaded 34GiB to GPU in 2.46219s (13.8088GiB/s) | 20260508 00:40:20.861 | INFO | PerfReport: 24.31%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 24.45%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 2.11%, LockUpload -- 2.24e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 2.81%, LockDownload -- 2.99e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 15.47%, LockCompute -- 1.64e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 00:40:20.861 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 00:40:20.972 | INFO | Starting a round of Baz IO | 20260508 00:40:24.440 | INFO | Populating RTMetrics report | 20260508 00:40:24.441 | INFO | RTMetrics report populated | 20260508 00:40:24.441 | INFO | Opening tmp RTMetrics json file | 20260508 00:40:24.459 | INFO | Serializing RTMetrics to json | 20260508 00:40:24.461 | INFO | Serialization done, swapping tmp file now... | 20260508 00:40:24.461 | INFO | RTMetrics done | 20260508 00:40:24.532 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.56 seconds | 20260508 00:40:24.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.11% / 0.00% / 57.89% | 20260508 00:40:24.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.975847% / 3.97364e-06% / 5.16574e-05% | 20260508 00:40:24.533 | INFO | Baz IO overall: | 20260508 00:40:24.533 | INFO | BytesWritten : 1759800400626 | 20260508 00:40:24.533 | INFO | NumEvents : 851643120876 | 20260508 00:40:26.004 | INFO | Analyzing chunk frames = [3874816,3875328) | 20260508 00:40:26.004 | INFO | MemInfo: { | 20260508 00:40:26.004 | INFO | "gpu" : " 17 GB", | 20260508 00:40:26.004 | INFO | "huge" : "", | 20260508 00:40:26.004 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:26.004 | INFO | "malloc" : " 14 GB", | 20260508 00:40:26.004 | INFO | "pinned" : " 29 GB" | 20260508 00:40:26.004 | INFO | } | 20260508 00:40:26.004 | INFO | Perf report for the last 5.14401 seconds: | 20260508 00:40:26.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:26.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:26.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:26.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:26.004 | INFO | ANALYSIS : 68.9%, 2.9, 104.7ms, 0 | 20260508 00:40:26.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:26.004 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 00:40:26.004 | INFO | PRE_HQ : 67.8%, 1.8, 63.0ms, 0 | 20260508 00:40:26.004 | INFO | BAZWRITER : 68.8%, 3.9, 138.5ms, 0 | 20260508 00:40:26.004 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 00:40:26.004 | INFO | Uploaded 36GiB to GPU in 2.61189s (13.7831GiB/s) | 20260508 00:40:26.004 | INFO | Downloaded 38GiB to GPU in 2.70335s (14.0566GiB/s) | 20260508 00:40:31.139 | INFO | Analyzing chunk frames = [3875328,3875840) | 20260508 00:40:31.139 | INFO | MemInfo: { | 20260508 00:40:31.139 | INFO | "gpu" : " 17 GB", | 20260508 00:40:31.139 | INFO | "huge" : "", | 20260508 00:40:31.139 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:31.139 | INFO | "malloc" : " 14 GB", | 20260508 00:40:31.139 | INFO | "pinned" : " 29 GB" | 20260508 00:40:31.139 | INFO | } | 20260508 00:40:31.139 | INFO | Perf report for the last 5.13562 seconds: | 20260508 00:40:31.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:31.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:40:31.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:40:31.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:31.139 | INFO | Uploaded 36GiB to GPU in 2.57363s (13.988GiB/s) | 20260508 00:40:31.139 | INFO | Downloaded 34GiB to GPU in 2.48122s (13.7029GiB/s) | 20260508 00:40:36.278 | INFO | Analyzing chunk frames = [3875840,3876352) | 20260508 00:40:36.278 | INFO | MemInfo: { | 20260508 00:40:36.278 | INFO | "gpu" : " 17 GB", | 20260508 00:40:36.278 | INFO | "huge" : "", | 20260508 00:40:36.278 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:36.278 | INFO | "malloc" : " 14 GB", | 20260508 00:40:36.278 | INFO | "pinned" : " 29 GB" | 20260508 00:40:36.278 | INFO | } | 20260508 00:40:36.278 | INFO | Perf report for the last 5.13894 seconds: | 20260508 00:40:36.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:36.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 00:40:36.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:40:36.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:36.278 | INFO | Uploaded 36GiB to GPU in 2.52056s (14.2826GiB/s) | 20260508 00:40:36.279 | INFO | Downloaded 34GiB to GPU in 2.45897s (13.8269GiB/s) | 20260508 00:40:41.420 | INFO | Analyzing chunk frames = [3876352,3876864) | 20260508 00:40:41.420 | INFO | MemInfo: { | 20260508 00:40:41.420 | INFO | "gpu" : " 17 GB", | 20260508 00:40:41.420 | INFO | "huge" : "", | 20260508 00:40:41.420 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:41.420 | INFO | "malloc" : " 14 GB", | 20260508 00:40:41.420 | INFO | "pinned" : " 29 GB" | 20260508 00:40:41.420 | INFO | } | 20260508 00:40:41.420 | INFO | Perf report for the last 5.14164 seconds: | 20260508 00:40:41.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:41.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260508 00:40:41.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:40:41.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:41.420 | INFO | Uploaded 36GiB to GPU in 2.56707s (14.0238GiB/s) | 20260508 00:40:41.420 | INFO | Downloaded 34GiB to GPU in 2.49113s (13.6484GiB/s) | 20260508 00:40:46.561 | INFO | Analyzing chunk frames = [3876864,3877376) | 20260508 00:40:46.561 | INFO | MemInfo: { | 20260508 00:40:46.561 | INFO | "gpu" : " 17 GB", | 20260508 00:40:46.561 | INFO | "huge" : "", | 20260508 00:40:46.561 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:46.561 | INFO | "malloc" : " 14 GB", | 20260508 00:40:46.561 | INFO | "pinned" : " 29 GB" | 20260508 00:40:46.561 | INFO | } | 20260508 00:40:46.561 | INFO | Perf report for the last 5.14098 seconds: | 20260508 00:40:46.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:46.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:40:46.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 00:40:46.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:46.561 | INFO | Uploaded 36GiB to GPU in 2.36715s (15.2082GiB/s) | 20260508 00:40:46.561 | INFO | Downloaded 34GiB to GPU in 2.50943s (13.5489GiB/s) | 20260508 00:40:51.699 | INFO | Analyzing chunk frames = [3877376,3877888) | 20260508 00:40:51.699 | INFO | MemInfo: { | 20260508 00:40:51.699 | INFO | "gpu" : " 17 GB", | 20260508 00:40:51.699 | INFO | "huge" : "", | 20260508 00:40:51.699 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:51.699 | INFO | "malloc" : " 14 GB", | 20260508 00:40:51.699 | INFO | "pinned" : " 29 GB" | 20260508 00:40:51.699 | INFO | } | 20260508 00:40:51.699 | INFO | Perf report for the last 5.13857 seconds: | 20260508 00:40:51.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:51.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 00:40:51.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:40:51.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:51.700 | INFO | Uploaded 36GiB to GPU in 2.44315s (14.7351GiB/s) | 20260508 00:40:51.700 | INFO | Downloaded 34GiB to GPU in 2.39342s (14.2056GiB/s) | 20260508 00:40:56.845 | INFO | Analyzing chunk frames = [3877888,3878400) | 20260508 00:40:56.845 | INFO | MemInfo: { | 20260508 00:40:56.845 | INFO | "gpu" : " 17 GB", | 20260508 00:40:56.845 | INFO | "huge" : "", | 20260508 00:40:56.845 | INFO | "huge_shared" : " 29 GB", | 20260508 00:40:56.845 | INFO | "malloc" : " 14 GB", | 20260508 00:40:56.845 | INFO | "pinned" : " 29 GB" | 20260508 00:40:56.845 | INFO | } | 20260508 00:40:56.845 | INFO | Perf report for the last 5.14591 seconds: | 20260508 00:40:56.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:40:56.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.845 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 00:40:56.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.845 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 00:40:56.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:40:56.846 | INFO | Uploaded 36GiB to GPU in 2.55874s (14.0694GiB/s) | 20260508 00:40:56.846 | INFO | Downloaded 34GiB to GPU in 2.44912s (13.8826GiB/s) | 20260508 00:41:01.976 | INFO | Analyzing chunk frames = [3878400,3878912) | 20260508 00:41:01.976 | INFO | MemInfo: { | 20260508 00:41:01.976 | INFO | "gpu" : " 17 GB", | 20260508 00:41:01.976 | INFO | "huge" : "", | 20260508 00:41:01.976 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:01.976 | INFO | "malloc" : " 14 GB", | 20260508 00:41:01.977 | INFO | "pinned" : " 29 GB" | 20260508 00:41:01.977 | INFO | } | 20260508 00:41:01.977 | INFO | Perf report for the last 5.13133 seconds: | 20260508 00:41:01.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:01.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 00:41:01.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | PRE_HQ : 35.9%, 1.0, 18.6ms, 0 | 20260508 00:41:01.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:01.977 | INFO | Uploaded 36GiB to GPU in 2.48889s (14.4643GiB/s) | 20260508 00:41:01.977 | INFO | Downloaded 34GiB to GPU in 2.42755s (14.0059GiB/s) | 20260508 00:41:02.088 | INFO | Starting a round of Baz IO | 20260508 00:41:05.560 | INFO | Populating RTMetrics report | 20260508 00:41:05.560 | INFO | RTMetrics report populated | 20260508 00:41:05.560 | INFO | Opening tmp RTMetrics json file | 20260508 00:41:05.575 | INFO | Serializing RTMetrics to json | 20260508 00:41:05.578 | INFO | Serialization done, swapping tmp file now... | 20260508 00:41:05.578 | INFO | RTMetrics done | 20260508 00:41:05.655 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.566 seconds | 20260508 00:41:05.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.10% / 0.00% / 57.89% | 20260508 00:41:05.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.975482% / 1.19209e-05% / 7.54992e-05% | 20260508 00:41:05.655 | INFO | Baz IO overall: | 20260508 00:41:05.655 | INFO | BytesWritten : 1761609092402 | 20260508 00:41:05.655 | INFO | NumEvents : 852513296114 | 20260508 00:41:07.113 | INFO | Analyzing chunk frames = [3878912,3879424) | 20260508 00:41:07.113 | INFO | MemInfo: { | 20260508 00:41:07.113 | INFO | "gpu" : " 17 GB", | 20260508 00:41:07.113 | INFO | "huge" : "", | 20260508 00:41:07.113 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:07.113 | INFO | "malloc" : " 14 GB", | 20260508 00:41:07.113 | INFO | "pinned" : " 29 GB" | 20260508 00:41:07.113 | INFO | } | 20260508 00:41:07.113 | INFO | Perf report for the last 5.13603 seconds: | 20260508 00:41:07.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:07.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:07.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:07.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:07.113 | INFO | ANALYSIS : 68.8%, 2.9, 104.0ms, 0 | 20260508 00:41:07.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:07.113 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260508 00:41:07.113 | INFO | PRE_HQ : 67.8%, 1.9, 65.4ms, 0 | 20260508 00:41:07.113 | INFO | BAZWRITER : 69.1%, 4.3, 152.9ms, 0 | 20260508 00:41:07.113 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 00:41:07.113 | INFO | Uploaded 36GiB to GPU in 2.60951s (13.7957GiB/s) | 20260508 00:41:07.113 | INFO | Downloaded 38GiB to GPU in 2.69662s (14.0917GiB/s) | 20260508 00:41:12.254 | INFO | Analyzing chunk frames = [3879424,3879936) | 20260508 00:41:12.254 | INFO | MemInfo: { | 20260508 00:41:12.254 | INFO | "gpu" : " 17 GB", | 20260508 00:41:12.254 | INFO | "huge" : "", | 20260508 00:41:12.254 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:12.254 | INFO | "malloc" : " 14 GB", | 20260508 00:41:12.254 | INFO | "pinned" : " 29 GB" | 20260508 00:41:12.254 | INFO | } | 20260508 00:41:12.254 | INFO | Perf report for the last 5.1417 seconds: | 20260508 00:41:12.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:12.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.255 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:41:12.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.255 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 00:41:12.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:12.255 | INFO | Uploaded 36GiB to GPU in 2.57944s (13.9565GiB/s) | 20260508 00:41:12.255 | INFO | Downloaded 34GiB to GPU in 2.48142s (13.7018GiB/s) | 20260508 00:41:17.393 | INFO | Analyzing chunk frames = [3879936,3880448) | 20260508 00:41:17.393 | INFO | MemInfo: { | 20260508 00:41:17.393 | INFO | "gpu" : " 17 GB", | 20260508 00:41:17.393 | INFO | "huge" : "", | 20260508 00:41:17.393 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:17.393 | INFO | "malloc" : " 14 GB", | 20260508 00:41:17.393 | INFO | "pinned" : " 29 GB" | 20260508 00:41:17.393 | INFO | } | 20260508 00:41:17.393 | INFO | Perf report for the last 5.1384 seconds: | 20260508 00:41:17.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:17.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 00:41:17.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:41:17.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:17.393 | INFO | Uploaded 36GiB to GPU in 2.54336s (14.1545GiB/s) | 20260508 00:41:17.393 | INFO | Downloaded 34GiB to GPU in 2.47991s (13.7102GiB/s) | 20260508 00:41:22.536 | INFO | Analyzing chunk frames = [3880448,3880960) | 20260508 00:41:22.536 | INFO | MemInfo: { | 20260508 00:41:22.536 | INFO | "gpu" : " 17 GB", | 20260508 00:41:22.536 | INFO | "huge" : "", | 20260508 00:41:22.536 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:22.536 | INFO | "malloc" : " 14 GB", | 20260508 00:41:22.536 | INFO | "pinned" : " 29 GB" | 20260508 00:41:22.536 | INFO | } | 20260508 00:41:22.536 | INFO | Perf report for the last 5.14349 seconds: | 20260508 00:41:22.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:22.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 00:41:22.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:41:22.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:22.537 | INFO | Uploaded 36GiB to GPU in 2.49497s (14.429GiB/s) | 20260508 00:41:22.537 | INFO | Downloaded 34GiB to GPU in 2.39478s (14.1975GiB/s) | 20260508 00:41:27.675 | INFO | Analyzing chunk frames = [3880960,3881472) | 20260508 00:41:27.675 | INFO | MemInfo: { | 20260508 00:41:27.675 | INFO | "gpu" : " 17 GB", | 20260508 00:41:27.675 | INFO | "huge" : "", | 20260508 00:41:27.675 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:27.675 | INFO | "malloc" : " 14 GB", | 20260508 00:41:27.675 | INFO | "pinned" : " 29 GB" | 20260508 00:41:27.675 | INFO | } | 20260508 00:41:27.675 | INFO | Perf report for the last 5.13858 seconds: | 20260508 00:41:27.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:27.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 00:41:27.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:41:27.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:27.675 | INFO | Uploaded 36GiB to GPU in 2.3392s (15.3899GiB/s) | 20260508 00:41:27.675 | INFO | Downloaded 34GiB to GPU in 2.51557s (13.5158GiB/s) | 20260508 00:41:32.814 | INFO | Analyzing chunk frames = [3881472,3881984) | 20260508 00:41:32.814 | INFO | MemInfo: { | 20260508 00:41:32.814 | INFO | "gpu" : " 17 GB", | 20260508 00:41:32.814 | INFO | "huge" : "", | 20260508 00:41:32.814 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:32.814 | INFO | "malloc" : " 14 GB", | 20260508 00:41:32.814 | INFO | "pinned" : " 29 GB" | 20260508 00:41:32.814 | INFO | } | 20260508 00:41:32.814 | INFO | Perf report for the last 5.13871 seconds: | 20260508 00:41:32.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:32.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:41:32.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:41:32.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:32.814 | INFO | Uploaded 36GiB to GPU in 2.43502s (14.7843GiB/s) | 20260508 00:41:32.814 | INFO | Downloaded 34GiB to GPU in 2.4025s (14.1519GiB/s) | 20260508 00:41:37.953 | INFO | Analyzing chunk frames = [3881984,3882496) | 20260508 00:41:37.953 | INFO | MemInfo: { | 20260508 00:41:37.953 | INFO | "gpu" : " 17 GB", | 20260508 00:41:37.953 | INFO | "huge" : "", | 20260508 00:41:37.953 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:37.953 | INFO | "malloc" : " 14 GB", | 20260508 00:41:37.953 | INFO | "pinned" : " 29 GB" | 20260508 00:41:37.954 | INFO | } | 20260508 00:41:37.954 | INFO | Perf report for the last 5.13998 seconds: | 20260508 00:41:37.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:37.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 00:41:37.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 00:41:37.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:37.954 | INFO | Uploaded 36GiB to GPU in 2.55349s (14.0984GiB/s) | 20260508 00:41:37.954 | INFO | Downloaded 34GiB to GPU in 2.45029s (13.8759GiB/s) | 20260508 00:41:43.096 | INFO | Analyzing chunk frames = [3882496,3883008) | 20260508 00:41:43.096 | INFO | MemInfo: { | 20260508 00:41:43.096 | INFO | "gpu" : " 17 GB", | 20260508 00:41:43.096 | INFO | "huge" : "", | 20260508 00:41:43.096 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:43.096 | INFO | "malloc" : " 14 GB", | 20260508 00:41:43.096 | INFO | "pinned" : " 29 GB" | 20260508 00:41:43.096 | INFO | } | 20260508 00:41:43.096 | INFO | Perf report for the last 5.14232 seconds: | 20260508 00:41:43.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:43.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:41:43.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:41:43.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:43.096 | INFO | Uploaded 36GiB to GPU in 2.53965s (14.1752GiB/s) | 20260508 00:41:43.096 | INFO | Downloaded 34GiB to GPU in 2.44352s (13.9144GiB/s) | 20260508 00:41:43.097 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:41:43.097 | INFO | Total mem usage: 16.7 GiB | 20260508 00:41:43.097 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:41:43.097 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.7%) | 20260508 00:41:43.097 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:41:43.097 | INFO | High Water Breakdown: | 20260508 00:41:43.097 | INFO | 60.8% : 9.41 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:41:43.097 | INFO | 39.1% : 6.06 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:41:43.097 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:41:43.097 | INFO | ------------------------------------------ | 20260508 00:41:43.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:41:43.097 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:41:43.097 | INFO | Total mem usage: 26.5 GiB | 20260508 00:41:43.097 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:41:43.097 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:41:43.097 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:41:43.097 | INFO | High Water Breakdown: | 20260508 00:41:43.097 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:41:43.097 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:41:43.097 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:41:43.097 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:41:43.097 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:41:43.097 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:41:43.097 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:41:43.097 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:41:43.097 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:41:43.097 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:41:43.097 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:41:43.097 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:41:43.097 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:41:43.097 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:41:43.097 | INFO | ------------------------------------------ | 20260508 00:41:43.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:41:43.097 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:41:43.097 | INFO | Total mem usage: 27.8 GiB | 20260508 00:41:43.097 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:41:43.097 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:41:43.097 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:41:43.097 | INFO | High Water Breakdown: | 20260508 00:41:43.097 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:41:43.097 | INFO | ------------------------------------------ | 20260508 00:41:43.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:41:43.097 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:41:43.097 | INFO | Total mem usage: 15.7 GiB | 20260508 00:41:43.097 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:41:43.097 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:41:43.097 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:41:43.097 | INFO | High Water Breakdown: | 20260508 00:41:43.097 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:41:43.097 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:41:43.097 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:41:43.097 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:41:43.097 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:41:43.097 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:41:43.097 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:41:43.097 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:41:43.097 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:41:43.097 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:41:43.097 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:41:43.097 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:41:43.097 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:41:43.097 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:41:43.097 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:41:43.097 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:41:43.097 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:41:43.097 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:41:43.097 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:41:43.097 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:41:43.097 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:41:43.098 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:41:43.098 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:41:43.098 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:41:43.098 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:41:43.098 | INFO | ------------------------------------------ | 20260508 00:41:43.098 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:41:43.098 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:41:43.098 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:41:43.098 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:41:43.098 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.93e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.16e-02 / 1.16e-02 | 20260508 00:41:43.098 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.50e+00, 5.90e-01 / 1.68e-01 | 20260508 00:41:43.098 | INFO | PerfReport: 3.34%, Histogram -- 1.10e+00 / 1.48e+00, 2.05e-01 / 1.24e-01 | 20260508 00:41:43.098 | INFO | PerfReport: 16.08%, DME -- 4.22e+01 / 4.62e+01, 4.19e+00 / 2.12e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.65e+00 / 1.64e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.27e-01 / 2.72e-01 | 20260508 00:41:43.098 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.27e+00, 2.75e-01 / 1.40e-01 | 20260508 00:41:43.098 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.71e-01, 5.40e-02 / 2.97e-02 | 20260508 00:41:43.098 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:41:43.098 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:41:43.098 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:41:43.098 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 24.60%, Upload -- 2.62e+01 / 0.00e+00, 8.46e+00 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 30.79%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 2.39%, LockUpload -- 2.54e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 2.45%, LockDownload -- 2.61e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 15.44%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 00:41:43.098 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 00:41:43.193 | INFO | Starting a round of Baz IO | 20260508 00:41:46.646 | INFO | Populating RTMetrics report | 20260508 00:41:46.646 | INFO | RTMetrics report populated | 20260508 00:41:46.646 | INFO | Opening tmp RTMetrics json file | 20260508 00:41:46.688 | INFO | Serializing RTMetrics to json | 20260508 00:41:46.712 | INFO | Serialization done, swapping tmp file now... | 20260508 00:41:46.723 | INFO | RTMetrics done | 20260508 00:41:46.780 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.586 seconds | 20260508 00:41:46.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.10% / 0.00% / 57.90% | 20260508 00:41:46.780 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.973531% / 0% / 7.15256e-05% | 20260508 00:41:46.780 | INFO | Baz IO overall: | 20260508 00:41:46.780 | INFO | BytesWritten : 1763418393833 | 20260508 00:41:46.780 | INFO | NumEvents : 853384045585 | 20260508 00:41:48.228 | INFO | Analyzing chunk frames = [3883008,3883520) | 20260508 00:41:48.228 | INFO | MemInfo: { | 20260508 00:41:48.228 | INFO | "gpu" : " 17 GB", | 20260508 00:41:48.228 | INFO | "huge" : "", | 20260508 00:41:48.228 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:48.228 | INFO | "malloc" : " 14 GB", | 20260508 00:41:48.228 | INFO | "pinned" : " 29 GB" | 20260508 00:41:48.228 | INFO | } | 20260508 00:41:48.228 | INFO | Perf report for the last 5.13209 seconds: | 20260508 00:41:48.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:48.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:48.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:48.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:48.228 | INFO | ANALYSIS : 68.5%, 3.0, 106.7ms, 0 | 20260508 00:41:48.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:48.228 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 00:41:48.228 | INFO | PRE_HQ : 65.5%, 1.7, 56.9ms, 0 | 20260508 00:41:48.228 | INFO | BAZWRITER : 69.5%, 3.7, 131.7ms, 0 | 20260508 00:41:48.228 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:41:48.229 | INFO | Uploaded 36GiB to GPU in 2.57323s (13.9902GiB/s) | 20260508 00:41:48.229 | INFO | Downloaded 38GiB to GPU in 2.68652s (14.1447GiB/s) | 20260508 00:41:53.370 | INFO | Analyzing chunk frames = [3883520,3884032) | 20260508 00:41:53.370 | INFO | MemInfo: { | 20260508 00:41:53.370 | INFO | "gpu" : " 17 GB", | 20260508 00:41:53.370 | INFO | "huge" : "", | 20260508 00:41:53.370 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:53.370 | INFO | "malloc" : " 14 GB", | 20260508 00:41:53.370 | INFO | "pinned" : " 29 GB" | 20260508 00:41:53.370 | INFO | } | 20260508 00:41:53.370 | INFO | Perf report for the last 5.14211 seconds: | 20260508 00:41:53.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:53.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.370 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 00:41:53.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.371 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:41:53.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:53.371 | INFO | Uploaded 36GiB to GPU in 2.59172s (13.8904GiB/s) | 20260508 00:41:53.371 | INFO | Downloaded 34GiB to GPU in 2.48072s (13.7057GiB/s) | 20260508 00:41:58.512 | INFO | Analyzing chunk frames = [3884032,3884544) | 20260508 00:41:58.512 | INFO | MemInfo: { | 20260508 00:41:58.512 | INFO | "gpu" : " 17 GB", | 20260508 00:41:58.512 | INFO | "huge" : "", | 20260508 00:41:58.512 | INFO | "huge_shared" : " 29 GB", | 20260508 00:41:58.512 | INFO | "malloc" : " 14 GB", | 20260508 00:41:58.512 | INFO | "pinned" : " 29 GB" | 20260508 00:41:58.512 | INFO | } | 20260508 00:41:58.512 | INFO | Perf report for the last 5.14187 seconds: | 20260508 00:41:58.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:41:58.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:41:58.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:41:58.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:41:58.512 | INFO | Uploaded 36GiB to GPU in 2.52201s (14.2743GiB/s) | 20260508 00:41:58.513 | INFO | Downloaded 34GiB to GPU in 2.467s (13.7819GiB/s) | 20260508 00:42:03.652 | INFO | Analyzing chunk frames = [3884544,3885056) | 20260508 00:42:03.652 | INFO | MemInfo: { | 20260508 00:42:03.652 | INFO | "gpu" : " 17 GB", | 20260508 00:42:03.652 | INFO | "huge" : "", | 20260508 00:42:03.652 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:03.652 | INFO | "malloc" : " 14 GB", | 20260508 00:42:03.652 | INFO | "pinned" : " 29 GB" | 20260508 00:42:03.652 | INFO | } | 20260508 00:42:03.652 | INFO | Perf report for the last 5.14022 seconds: | 20260508 00:42:03.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:03.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.652 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 00:42:03.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.652 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:42:03.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:03.653 | INFO | Uploaded 36GiB to GPU in 2.5459s (14.1404GiB/s) | 20260508 00:42:03.653 | INFO | Downloaded 34GiB to GPU in 2.47114s (13.7588GiB/s) | 20260508 00:42:08.789 | INFO | Analyzing chunk frames = [3885056,3885568) | 20260508 00:42:08.789 | INFO | MemInfo: { | 20260508 00:42:08.789 | INFO | "gpu" : " 17 GB", | 20260508 00:42:08.789 | INFO | "huge" : "", | 20260508 00:42:08.789 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:08.790 | INFO | "malloc" : " 14 GB", | 20260508 00:42:08.790 | INFO | "pinned" : " 29 GB" | 20260508 00:42:08.790 | INFO | } | 20260508 00:42:08.790 | INFO | Perf report for the last 5.13723 seconds: | 20260508 00:42:08.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:08.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 00:42:08.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:42:08.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:08.790 | INFO | Uploaded 36GiB to GPU in 2.34835s (15.3299GiB/s) | 20260508 00:42:08.791 | INFO | Downloaded 34GiB to GPU in 2.51244s (13.5326GiB/s) | 20260508 00:42:13.925 | INFO | Analyzing chunk frames = [3885568,3886080) | 20260508 00:42:13.925 | INFO | MemInfo: { | 20260508 00:42:13.925 | INFO | "gpu" : " 17 GB", | 20260508 00:42:13.925 | INFO | "huge" : "", | 20260508 00:42:13.925 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:13.925 | INFO | "malloc" : " 14 GB", | 20260508 00:42:13.925 | INFO | "pinned" : " 29 GB" | 20260508 00:42:13.925 | INFO | } | 20260508 00:42:13.925 | INFO | Perf report for the last 5.13549 seconds: | 20260508 00:42:13.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:13.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:42:13.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260508 00:42:13.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:13.925 | INFO | Uploaded 36GiB to GPU in 2.40558s (14.9652GiB/s) | 20260508 00:42:13.925 | INFO | Downloaded 34GiB to GPU in 2.40167s (14.1568GiB/s) | 20260508 00:42:19.069 | INFO | Analyzing chunk frames = [3886080,3886592) | 20260508 00:42:19.069 | INFO | MemInfo: { | 20260508 00:42:19.069 | INFO | "gpu" : " 17 GB", | 20260508 00:42:19.069 | INFO | "huge" : "", | 20260508 00:42:19.069 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:19.069 | INFO | "malloc" : " 14 GB", | 20260508 00:42:19.069 | INFO | "pinned" : " 29 GB" | 20260508 00:42:19.069 | INFO | } | 20260508 00:42:19.069 | INFO | Perf report for the last 5.14384 seconds: | 20260508 00:42:19.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:19.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 00:42:19.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:42:19.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:19.069 | INFO | Uploaded 36GiB to GPU in 2.5388s (14.1799GiB/s) | 20260508 00:42:19.069 | INFO | Downloaded 34GiB to GPU in 2.46363s (13.8008GiB/s) | 20260508 00:42:24.208 | INFO | Analyzing chunk frames = [3886592,3887104) | 20260508 00:42:24.208 | INFO | MemInfo: { | 20260508 00:42:24.208 | INFO | "gpu" : " 17 GB", | 20260508 00:42:24.208 | INFO | "huge" : "", | 20260508 00:42:24.208 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:24.208 | INFO | "malloc" : " 14 GB", | 20260508 00:42:24.208 | INFO | "pinned" : " 29 GB" | 20260508 00:42:24.208 | INFO | } | 20260508 00:42:24.208 | INFO | Perf report for the last 5.13942 seconds: | 20260508 00:42:24.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:24.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.208 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 00:42:24.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.209 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:42:24.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:24.209 | INFO | Uploaded 36GiB to GPU in 2.51646s (14.3058GiB/s) | 20260508 00:42:24.209 | INFO | Downloaded 34GiB to GPU in 2.44823s (13.8876GiB/s) | 20260508 00:42:24.306 | INFO | Starting a round of Baz IO | 20260508 00:42:27.774 | INFO | Populating RTMetrics report | 20260508 00:42:27.775 | INFO | RTMetrics report populated | 20260508 00:42:27.775 | INFO | Opening tmp RTMetrics json file | 20260508 00:42:27.790 | INFO | Serializing RTMetrics to json | 20260508 00:42:27.792 | INFO | Serialization done, swapping tmp file now... | 20260508 00:42:27.792 | INFO | RTMetrics done | 20260508 00:42:27.860 | INFO | Finished a round of Baz IO: Wrote 1.683 GiB in 3.554 seconds | 20260508 00:42:27.860 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.09% / 0.00% / 57.90% | 20260508 00:42:27.860 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.968122% / 7.94729e-06% / 6.35783e-05% | 20260508 00:42:27.860 | INFO | Baz IO overall: | 20260508 00:42:27.860 | INFO | BytesWritten : 1765225253246 | 20260508 00:42:27.860 | INFO | NumEvents : 854253082329 | 20260508 00:42:29.347 | INFO | Analyzing chunk frames = [3887104,3887616) | 20260508 00:42:29.347 | INFO | MemInfo: { | 20260508 00:42:29.347 | INFO | "gpu" : " 17 GB", | 20260508 00:42:29.347 | INFO | "huge" : "", | 20260508 00:42:29.347 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:29.347 | INFO | "malloc" : " 14 GB", | 20260508 00:42:29.347 | INFO | "pinned" : " 29 GB" | 20260508 00:42:29.347 | INFO | } | 20260508 00:42:29.347 | INFO | Perf report for the last 5.13895 seconds: | 20260508 00:42:29.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:29.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:29.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:29.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:29.347 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 00:42:29.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:29.347 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 00:42:29.347 | INFO | PRE_HQ : 65.5%, 1.7, 56.2ms, 0 | 20260508 00:42:29.347 | INFO | BAZWRITER : 68.8%, 3.7, 130.5ms, 0 | 20260508 00:42:29.347 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 00:42:29.348 | INFO | Uploaded 36GiB to GPU in 2.56024s (14.0612GiB/s) | 20260508 00:42:29.348 | INFO | Downloaded 38GiB to GPU in 2.69886s (14.08GiB/s) | 20260508 00:42:34.482 | INFO | Analyzing chunk frames = [3887616,3888128) | 20260508 00:42:34.482 | INFO | MemInfo: { | 20260508 00:42:34.482 | INFO | "gpu" : " 17 GB", | 20260508 00:42:34.482 | INFO | "huge" : "", | 20260508 00:42:34.482 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:34.482 | INFO | "malloc" : " 14 GB", | 20260508 00:42:34.482 | INFO | "pinned" : " 29 GB" | 20260508 00:42:34.482 | INFO | } | 20260508 00:42:34.482 | INFO | Perf report for the last 5.13451 seconds: | 20260508 00:42:34.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:34.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.482 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:42:34.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.482 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:42:34.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:34.483 | INFO | Uploaded 36GiB to GPU in 2.52132s (14.2782GiB/s) | 20260508 00:42:34.483 | INFO | Downloaded 34GiB to GPU in 2.4458s (13.9014GiB/s) | 20260508 00:42:39.624 | INFO | Analyzing chunk frames = [3888128,3888640) | 20260508 00:42:39.624 | INFO | MemInfo: { | 20260508 00:42:39.624 | INFO | "gpu" : " 17 GB", | 20260508 00:42:39.624 | INFO | "huge" : "", | 20260508 00:42:39.624 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:39.624 | INFO | "malloc" : " 14 GB", | 20260508 00:42:39.624 | INFO | "pinned" : " 29 GB" | 20260508 00:42:39.624 | INFO | } | 20260508 00:42:39.624 | INFO | Perf report for the last 5.1423 seconds: | 20260508 00:42:39.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:39.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.624 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 00:42:39.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.624 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:42:39.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:39.625 | INFO | Uploaded 36GiB to GPU in 2.48065s (14.5123GiB/s) | 20260508 00:42:39.625 | INFO | Downloaded 34GiB to GPU in 2.49094s (13.6495GiB/s) | 20260508 00:42:44.769 | INFO | Analyzing chunk frames = [3888640,3889152) | 20260508 00:42:44.769 | INFO | MemInfo: { | 20260508 00:42:44.769 | INFO | "gpu" : " 17 GB", | 20260508 00:42:44.769 | INFO | "huge" : "", | 20260508 00:42:44.769 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:44.769 | INFO | "malloc" : " 14 GB", | 20260508 00:42:44.769 | INFO | "pinned" : " 29 GB" | 20260508 00:42:44.769 | INFO | } | 20260508 00:42:44.769 | INFO | Perf report for the last 5.14499 seconds: | 20260508 00:42:44.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:44.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 00:42:44.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:42:44.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:44.769 | INFO | Uploaded 36GiB to GPU in 2.47742s (14.5312GiB/s) | 20260508 00:42:44.770 | INFO | Downloaded 34GiB to GPU in 2.49752s (13.6135GiB/s) | 20260508 00:42:49.911 | INFO | Analyzing chunk frames = [3889152,3889664) | 20260508 00:42:49.911 | INFO | MemInfo: { | 20260508 00:42:49.911 | INFO | "gpu" : " 17 GB", | 20260508 00:42:49.911 | INFO | "huge" : "", | 20260508 00:42:49.911 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:49.911 | INFO | "malloc" : " 14 GB", | 20260508 00:42:49.911 | INFO | "pinned" : " 29 GB" | 20260508 00:42:49.911 | INFO | } | 20260508 00:42:49.911 | INFO | Perf report for the last 5.14206 seconds: | 20260508 00:42:49.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:49.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.911 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 00:42:49.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.911 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260508 00:42:49.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:49.912 | INFO | Uploaded 36GiB to GPU in 2.2757s (15.8193GiB/s) | 20260508 00:42:49.912 | INFO | Downloaded 34GiB to GPU in 2.51004s (13.5456GiB/s) | 20260508 00:42:55.045 | INFO | Analyzing chunk frames = [3889664,3890176) | 20260508 00:42:55.045 | INFO | MemInfo: { | 20260508 00:42:55.045 | INFO | "gpu" : " 17 GB", | 20260508 00:42:55.045 | INFO | "huge" : "", | 20260508 00:42:55.045 | INFO | "huge_shared" : " 29 GB", | 20260508 00:42:55.045 | INFO | "malloc" : " 14 GB", | 20260508 00:42:55.045 | INFO | "pinned" : " 29 GB" | 20260508 00:42:55.045 | INFO | } | 20260508 00:42:55.045 | INFO | Perf report for the last 5.13406 seconds: | 20260508 00:42:55.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:42:55.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:42:55.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:42:55.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:42:55.045 | INFO | Uploaded 36GiB to GPU in 2.39471s (15.0331GiB/s) | 20260508 00:42:55.046 | INFO | Downloaded 34GiB to GPU in 2.41454s (14.0814GiB/s) | 20260508 00:43:00.182 | INFO | Analyzing chunk frames = [3890176,3890688) | 20260508 00:43:00.182 | INFO | MemInfo: { | 20260508 00:43:00.182 | INFO | "gpu" : " 17 GB", | 20260508 00:43:00.182 | INFO | "huge" : "", | 20260508 00:43:00.182 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:00.182 | INFO | "malloc" : " 14 GB", | 20260508 00:43:00.182 | INFO | "pinned" : " 29 GB" | 20260508 00:43:00.182 | INFO | } | 20260508 00:43:00.182 | INFO | Perf report for the last 5.13703 seconds: | 20260508 00:43:00.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:00.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.182 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:43:00.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.182 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:43:00.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:00.183 | INFO | Uploaded 36GiB to GPU in 2.52556s (14.2543GiB/s) | 20260508 00:43:00.183 | INFO | Downloaded 34GiB to GPU in 2.46043s (13.8187GiB/s) | 20260508 00:43:05.324 | INFO | Analyzing chunk frames = [3890688,3891200) | 20260508 00:43:05.324 | INFO | MemInfo: { | 20260508 00:43:05.324 | INFO | "gpu" : " 17 GB", | 20260508 00:43:05.324 | INFO | "huge" : "", | 20260508 00:43:05.324 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:05.324 | INFO | "malloc" : " 14 GB", | 20260508 00:43:05.324 | INFO | "pinned" : " 29 GB" | 20260508 00:43:05.324 | INFO | } | 20260508 00:43:05.324 | INFO | Perf report for the last 5.14202 seconds: | 20260508 00:43:05.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:05.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.324 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 00:43:05.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.325 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:43:05.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:05.325 | INFO | Uploaded 36GiB to GPU in 2.50965s (14.3446GiB/s) | 20260508 00:43:05.325 | INFO | Downloaded 34GiB to GPU in 2.45097s (13.8721GiB/s) | 20260508 00:43:05.325 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:43:05.325 | INFO | Total mem usage: 16.7 GiB | 20260508 00:43:05.325 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:43:05.325 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260508 00:43:05.325 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:43:05.325 | INFO | High Water Breakdown: | 20260508 00:43:05.325 | INFO | 60.1% : 8.95 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:43:05.325 | INFO | 39.7% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:43:05.325 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:43:05.325 | INFO | ------------------------------------------ | 20260508 00:43:05.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:43:05.325 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:43:05.325 | INFO | Total mem usage: 26.5 GiB | 20260508 00:43:05.325 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:43:05.325 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:43:05.325 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:43:05.325 | INFO | High Water Breakdown: | 20260508 00:43:05.325 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:43:05.325 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:43:05.325 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:43:05.325 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:43:05.325 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:43:05.325 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:43:05.325 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:43:05.325 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:43:05.325 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:43:05.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:43:05.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:43:05.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:43:05.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:43:05.325 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:43:05.325 | INFO | ------------------------------------------ | 20260508 00:43:05.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:43:05.325 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:43:05.325 | INFO | Total mem usage: 27.8 GiB | 20260508 00:43:05.325 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:43:05.325 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:43:05.325 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:43:05.325 | INFO | High Water Breakdown: | 20260508 00:43:05.325 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:43:05.325 | INFO | ------------------------------------------ | 20260508 00:43:05.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:43:05.326 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:43:05.326 | INFO | Total mem usage: 15.7 GiB | 20260508 00:43:05.326 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:43:05.326 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:43:05.326 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:43:05.326 | INFO | High Water Breakdown: | 20260508 00:43:05.326 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:43:05.326 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:43:05.326 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:43:05.326 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:43:05.326 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:43:05.326 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:43:05.326 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:43:05.326 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:43:05.326 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:43:05.326 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:43:05.326 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:43:05.326 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:43:05.326 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:43:05.326 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:43:05.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:43:05.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:43:05.326 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:43:05.326 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:43:05.326 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:43:05.326 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:43:05.326 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:43:05.326 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:43:05.326 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:43:05.326 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:43:05.326 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:43:05.326 | INFO | ------------------------------------------ | 20260508 00:43:05.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:43:05.326 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:43:05.326 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:43:05.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:43:05.326 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.42e+01 / 1.61e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.01e-01, 1.20e-02 / 5.76e-03 | 20260508 00:43:05.326 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.49e+00, 5.91e-01 / 2.05e-01 | 20260508 00:43:05.326 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.14e-01 | 20260508 00:43:05.326 | INFO | PerfReport: 16.12%, DME -- 4.22e+01 / 4.61e+01, 4.17e+00 / 1.76e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.55e+00 / 1.49e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.46e+00 / 2.78e+00, 4.33e-01 / 2.01e-01 | 20260508 00:43:05.326 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.33e+00, 2.74e-01 / 1.35e-01 | 20260508 00:43:05.326 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.77e-01, 5.39e-02 / 2.77e-02 | 20260508 00:43:05.326 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:43:05.326 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:43:05.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:43:05.326 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 24.28%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 24.35%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 30.66%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 2.88%, LockDownload -- 3.08e+00 / 0.00e+00, 4.30e+00 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 00:43:05.326 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260508 00:43:05.421 | INFO | Starting a round of Baz IO | 20260508 00:43:08.880 | INFO | Populating RTMetrics report | 20260508 00:43:08.880 | INFO | RTMetrics report populated | 20260508 00:43:08.880 | INFO | Opening tmp RTMetrics json file | 20260508 00:43:08.909 | INFO | Serializing RTMetrics to json | 20260508 00:43:08.922 | INFO | Serialization done, swapping tmp file now... | 20260508 00:43:08.948 | INFO | RTMetrics done | 20260508 00:43:08.988 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.566 seconds | 20260508 00:43:08.988 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.09% / 0.00% / 57.91% | 20260508 00:43:08.988 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.971337% / 1.19209e-05% / 3.57628e-05% | 20260508 00:43:08.988 | INFO | Baz IO overall: | 20260508 00:43:08.988 | INFO | BytesWritten : 1767034201219 | 20260508 00:43:08.988 | INFO | NumEvents : 855123853173 | 20260508 00:43:10.464 | INFO | Analyzing chunk frames = [3891200,3891712) | 20260508 00:43:10.464 | INFO | MemInfo: { | 20260508 00:43:10.464 | INFO | "gpu" : " 17 GB", | 20260508 00:43:10.464 | INFO | "huge" : "", | 20260508 00:43:10.464 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:10.464 | INFO | "malloc" : " 14 GB", | 20260508 00:43:10.464 | INFO | "pinned" : " 29 GB" | 20260508 00:43:10.464 | INFO | } | 20260508 00:43:10.464 | INFO | Perf report for the last 5.13944 seconds: | 20260508 00:43:10.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:10.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:10.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:10.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:10.464 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260508 00:43:10.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:10.464 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260508 00:43:10.464 | INFO | PRE_HQ : 65.8%, 1.7, 56.5ms, 0 | 20260508 00:43:10.464 | INFO | BAZWRITER : 69.0%, 3.8, 133.5ms, 0 | 20260508 00:43:10.464 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:43:10.464 | INFO | Uploaded 36GiB to GPU in 2.593s (13.8836GiB/s) | 20260508 00:43:10.464 | INFO | Downloaded 38GiB to GPU in 2.68605s (14.1472GiB/s) | 20260508 00:43:15.605 | INFO | Analyzing chunk frames = [3891712,3892224) | 20260508 00:43:15.605 | INFO | MemInfo: { | 20260508 00:43:15.605 | INFO | "gpu" : " 17 GB", | 20260508 00:43:15.605 | INFO | "huge" : "", | 20260508 00:43:15.605 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:15.605 | INFO | "malloc" : " 14 GB", | 20260508 00:43:15.605 | INFO | "pinned" : " 29 GB" | 20260508 00:43:15.605 | INFO | } | 20260508 00:43:15.605 | INFO | Perf report for the last 5.14109 seconds: | 20260508 00:43:15.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:15.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.605 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:43:15.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.605 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:43:15.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:15.606 | INFO | Uploaded 36GiB to GPU in 2.53736s (14.188GiB/s) | 20260508 00:43:15.606 | INFO | Downloaded 34GiB to GPU in 2.43991s (13.9349GiB/s) | 20260508 00:43:20.740 | INFO | Analyzing chunk frames = [3892224,3892736) | 20260508 00:43:20.740 | INFO | MemInfo: { | 20260508 00:43:20.740 | INFO | "gpu" : " 17 GB", | 20260508 00:43:20.740 | INFO | "huge" : "", | 20260508 00:43:20.740 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:20.741 | INFO | "malloc" : " 14 GB", | 20260508 00:43:20.741 | INFO | "pinned" : " 29 GB" | 20260508 00:43:20.741 | INFO | } | 20260508 00:43:20.741 | INFO | Perf report for the last 5.13568 seconds: | 20260508 00:43:20.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:20.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:43:20.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:43:20.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:20.741 | INFO | Uploaded 36GiB to GPU in 2.51673s (14.3043GiB/s) | 20260508 00:43:20.741 | INFO | Downloaded 34GiB to GPU in 2.48786s (13.6663GiB/s) | 20260508 00:43:25.877 | INFO | Analyzing chunk frames = [3892736,3893248) | 20260508 00:43:25.877 | INFO | MemInfo: { | 20260508 00:43:25.877 | INFO | "gpu" : " 17 GB", | 20260508 00:43:25.877 | INFO | "huge" : "", | 20260508 00:43:25.878 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:25.878 | INFO | "malloc" : " 14 GB", | 20260508 00:43:25.878 | INFO | "pinned" : " 29 GB" | 20260508 00:43:25.878 | INFO | } | 20260508 00:43:25.878 | INFO | Perf report for the last 5.13709 seconds: | 20260508 00:43:25.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:25.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:43:25.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 00:43:25.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:25.878 | INFO | Uploaded 36GiB to GPU in 2.5372s (14.1889GiB/s) | 20260508 00:43:25.878 | INFO | Downloaded 34GiB to GPU in 2.49338s (13.6361GiB/s) | 20260508 00:43:31.017 | INFO | Analyzing chunk frames = [3893248,3893760) | 20260508 00:43:31.017 | INFO | MemInfo: { | 20260508 00:43:31.017 | INFO | "gpu" : " 17 GB", | 20260508 00:43:31.017 | INFO | "huge" : "", | 20260508 00:43:31.017 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:31.017 | INFO | "malloc" : " 14 GB", | 20260508 00:43:31.017 | INFO | "pinned" : " 29 GB" | 20260508 00:43:31.017 | INFO | } | 20260508 00:43:31.017 | INFO | Perf report for the last 5.13911 seconds: | 20260508 00:43:31.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:31.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 00:43:31.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:43:31.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:31.017 | INFO | Uploaded 36GiB to GPU in 2.361s (15.2477GiB/s) | 20260508 00:43:31.017 | INFO | Downloaded 34GiB to GPU in 2.50759s (13.5588GiB/s) | 20260508 00:43:36.160 | INFO | Analyzing chunk frames = [3893760,3894272) | 20260508 00:43:36.160 | INFO | MemInfo: { | 20260508 00:43:36.160 | INFO | "gpu" : " 17 GB", | 20260508 00:43:36.160 | INFO | "huge" : "", | 20260508 00:43:36.160 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:36.160 | INFO | "malloc" : " 14 GB", | 20260508 00:43:36.160 | INFO | "pinned" : " 29 GB" | 20260508 00:43:36.160 | INFO | } | 20260508 00:43:36.160 | INFO | Perf report for the last 5.14355 seconds: | 20260508 00:43:36.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:36.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.161 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 00:43:36.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.161 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:43:36.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:36.161 | INFO | Uploaded 36GiB to GPU in 2.38554s (15.0909GiB/s) | 20260508 00:43:36.161 | INFO | Downloaded 34GiB to GPU in 2.43132s (13.9842GiB/s) | 20260508 00:43:41.293 | INFO | Analyzing chunk frames = [3894272,3894784) | 20260508 00:43:41.293 | INFO | MemInfo: { | 20260508 00:43:41.293 | INFO | "gpu" : " 17 GB", | 20260508 00:43:41.293 | INFO | "huge" : "", | 20260508 00:43:41.293 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:41.293 | INFO | "malloc" : " 14 GB", | 20260508 00:43:41.293 | INFO | "pinned" : " 29 GB" | 20260508 00:43:41.293 | INFO | } | 20260508 00:43:41.293 | INFO | Perf report for the last 5.13295 seconds: | 20260508 00:43:41.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:41.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.293 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 00:43:41.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.293 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:43:41.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:41.294 | INFO | Uploaded 36GiB to GPU in 2.53785s (14.1852GiB/s) | 20260508 00:43:41.294 | INFO | Downloaded 34GiB to GPU in 2.48314s (13.6923GiB/s) | 20260508 00:43:46.434 | INFO | Analyzing chunk frames = [3894784,3895296) | 20260508 00:43:46.434 | INFO | MemInfo: { | 20260508 00:43:46.434 | INFO | "gpu" : " 17 GB", | 20260508 00:43:46.434 | INFO | "huge" : "", | 20260508 00:43:46.434 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:46.434 | INFO | "malloc" : " 14 GB", | 20260508 00:43:46.434 | INFO | "pinned" : " 29 GB" | 20260508 00:43:46.434 | INFO | } | 20260508 00:43:46.435 | INFO | Perf report for the last 5.14109 seconds: | 20260508 00:43:46.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:46.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 00:43:46.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:43:46.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:46.435 | INFO | Uploaded 36GiB to GPU in 2.4402s (14.7529GiB/s) | 20260508 00:43:46.435 | INFO | Downloaded 34GiB to GPU in 2.45709s (13.8375GiB/s) | 20260508 00:43:46.532 | INFO | Starting a round of Baz IO | 20260508 00:43:49.994 | INFO | Populating RTMetrics report | 20260508 00:43:49.994 | INFO | RTMetrics report populated | 20260508 00:43:49.994 | INFO | Opening tmp RTMetrics json file | 20260508 00:43:50.022 | INFO | Serializing RTMetrics to json | 20260508 00:43:50.084 | INFO | Serialization done, swapping tmp file now... | 20260508 00:43:50.087 | INFO | RTMetrics done | 20260508 00:43:50.115 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.583 seconds | 20260508 00:43:50.115 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.09% / 0.00% / 57.91% | 20260508 00:43:50.115 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.967065% / 1.58946e-05% / 5.16574e-05% | 20260508 00:43:50.115 | INFO | Baz IO overall: | 20260508 00:43:50.115 | INFO | BytesWritten : 1768841939021 | 20260508 00:43:50.115 | INFO | NumEvents : 855993491914 | 20260508 00:43:51.578 | INFO | Analyzing chunk frames = [3895296,3895808) | 20260508 00:43:51.578 | INFO | MemInfo: { | 20260508 00:43:51.578 | INFO | "gpu" : " 17 GB", | 20260508 00:43:51.578 | INFO | "huge" : "", | 20260508 00:43:51.578 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:51.578 | INFO | "malloc" : " 14 GB", | 20260508 00:43:51.578 | INFO | "pinned" : " 29 GB" | 20260508 00:43:51.578 | INFO | } | 20260508 00:43:51.578 | INFO | Perf report for the last 5.14398 seconds: | 20260508 00:43:51.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:51.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:51.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:51.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:51.579 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 00:43:51.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:51.579 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 00:43:51.579 | INFO | PRE_HQ : 65.3%, 1.7, 56.6ms, 0 | 20260508 00:43:51.579 | INFO | BAZWRITER : 69.3%, 3.8, 136.6ms, 0 | 20260508 00:43:51.579 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 00:43:51.579 | INFO | Uploaded 36GiB to GPU in 2.51426s (14.3184GiB/s) | 20260508 00:43:51.579 | INFO | Downloaded 38GiB to GPU in 2.68214s (14.1678GiB/s) | 20260508 00:43:56.716 | INFO | Analyzing chunk frames = [3895808,3896320) | 20260508 00:43:56.716 | INFO | MemInfo: { | 20260508 00:43:56.716 | INFO | "gpu" : " 17 GB", | 20260508 00:43:56.716 | INFO | "huge" : "", | 20260508 00:43:56.716 | INFO | "huge_shared" : " 29 GB", | 20260508 00:43:56.717 | INFO | "malloc" : " 14 GB", | 20260508 00:43:56.717 | INFO | "pinned" : " 29 GB" | 20260508 00:43:56.717 | INFO | } | 20260508 00:43:56.717 | INFO | Perf report for the last 5.13828 seconds: | 20260508 00:43:56.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:43:56.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:43:56.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 00:43:56.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:43:56.717 | INFO | Uploaded 36GiB to GPU in 2.509s (14.3484GiB/s) | 20260508 00:43:56.717 | INFO | Downloaded 34GiB to GPU in 2.44162s (13.9252GiB/s) | 20260508 00:44:01.861 | INFO | Analyzing chunk frames = [3896320,3896832) | 20260508 00:44:01.861 | INFO | MemInfo: { | 20260508 00:44:01.861 | INFO | "gpu" : " 17 GB", | 20260508 00:44:01.861 | INFO | "huge" : "", | 20260508 00:44:01.861 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:01.861 | INFO | "malloc" : " 14 GB", | 20260508 00:44:01.861 | INFO | "pinned" : " 29 GB" | 20260508 00:44:01.861 | INFO | } | 20260508 00:44:01.861 | INFO | Perf report for the last 5.1445 seconds: | 20260508 00:44:01.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:01.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.861 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:44:01.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.861 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:44:01.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:01.862 | INFO | Uploaded 36GiB to GPU in 2.45981s (14.6353GiB/s) | 20260508 00:44:01.862 | INFO | Downloaded 34GiB to GPU in 2.50494s (13.5732GiB/s) | 20260508 00:44:06.996 | INFO | Analyzing chunk frames = [3896832,3897344) | 20260508 00:44:06.996 | INFO | MemInfo: { | 20260508 00:44:06.996 | INFO | "gpu" : " 17 GB", | 20260508 00:44:06.996 | INFO | "huge" : "", | 20260508 00:44:06.996 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:06.996 | INFO | "malloc" : " 14 GB", | 20260508 00:44:06.996 | INFO | "pinned" : " 29 GB" | 20260508 00:44:06.996 | INFO | } | 20260508 00:44:06.996 | INFO | Perf report for the last 5.13513 seconds: | 20260508 00:44:06.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:06.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.996 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260508 00:44:06.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.997 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 00:44:06.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:06.997 | INFO | Uploaded 36GiB to GPU in 2.49772s (14.4131GiB/s) | 20260508 00:44:06.997 | INFO | Downloaded 34GiB to GPU in 2.51049s (13.5432GiB/s) | 20260508 00:44:12.137 | INFO | Analyzing chunk frames = [3897344,3897856) | 20260508 00:44:12.137 | INFO | MemInfo: { | 20260508 00:44:12.137 | INFO | "gpu" : " 17 GB", | 20260508 00:44:12.137 | INFO | "huge" : "", | 20260508 00:44:12.137 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:12.137 | INFO | "malloc" : " 14 GB", | 20260508 00:44:12.137 | INFO | "pinned" : " 29 GB" | 20260508 00:44:12.137 | INFO | } | 20260508 00:44:12.137 | INFO | Perf report for the last 5.14082 seconds: | 20260508 00:44:12.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:12.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:44:12.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:44:12.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:12.137 | INFO | Uploaded 36GiB to GPU in 2.30255s (15.6348GiB/s) | 20260508 00:44:12.137 | INFO | Downloaded 34GiB to GPU in 2.50841s (13.5544GiB/s) | 20260508 00:44:17.275 | INFO | Analyzing chunk frames = [3897856,3898368) | 20260508 00:44:17.275 | INFO | MemInfo: { | 20260508 00:44:17.275 | INFO | "gpu" : " 17 GB", | 20260508 00:44:17.275 | INFO | "huge" : "", | 20260508 00:44:17.275 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:17.275 | INFO | "malloc" : " 14 GB", | 20260508 00:44:17.275 | INFO | "pinned" : " 29 GB" | 20260508 00:44:17.275 | INFO | } | 20260508 00:44:17.275 | INFO | Perf report for the last 5.1378 seconds: | 20260508 00:44:17.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:17.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.275 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:44:17.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.275 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:44:17.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:17.276 | INFO | Uploaded 36GiB to GPU in 2.40092s (14.9942GiB/s) | 20260508 00:44:17.276 | INFO | Downloaded 34GiB to GPU in 2.40951s (14.1108GiB/s) | 20260508 00:44:22.417 | INFO | Analyzing chunk frames = [3898368,3898880) | 20260508 00:44:22.417 | INFO | MemInfo: { | 20260508 00:44:22.417 | INFO | "gpu" : " 17 GB", | 20260508 00:44:22.417 | INFO | "huge" : "", | 20260508 00:44:22.417 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:22.417 | INFO | "malloc" : " 14 GB", | 20260508 00:44:22.417 | INFO | "pinned" : " 29 GB" | 20260508 00:44:22.418 | INFO | } | 20260508 00:44:22.418 | INFO | Perf report for the last 5.14258 seconds: | 20260508 00:44:22.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:22.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 00:44:22.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:44:22.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:22.418 | INFO | Uploaded 36GiB to GPU in 2.45502s (14.6638GiB/s) | 20260508 00:44:22.418 | INFO | Downloaded 34GiB to GPU in 2.43863s (13.9423GiB/s) | 20260508 00:44:27.550 | INFO | Analyzing chunk frames = [3898880,3899392) | 20260508 00:44:27.550 | INFO | MemInfo: { | 20260508 00:44:27.550 | INFO | "gpu" : " 17 GB", | 20260508 00:44:27.550 | INFO | "huge" : "", | 20260508 00:44:27.550 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:27.550 | INFO | "malloc" : " 14 GB", | 20260508 00:44:27.550 | INFO | "pinned" : " 29 GB" | 20260508 00:44:27.550 | INFO | } | 20260508 00:44:27.550 | INFO | Perf report for the last 5.13256 seconds: | 20260508 00:44:27.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:27.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.550 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 00:44:27.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.550 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:44:27.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:27.551 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:44:27.551 | INFO | Total mem usage: 16.7 GiB | 20260508 00:44:27.551 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:44:27.551 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260508 00:44:27.551 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:44:27.551 | INFO | High Water Breakdown: | 20260508 00:44:27.551 | INFO | 60.2% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:44:27.551 | INFO | 39.7% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:44:27.551 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:44:27.551 | INFO | ------------------------------------------ | 20260508 00:44:27.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:44:27.551 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:44:27.551 | INFO | Total mem usage: 26.5 GiB | 20260508 00:44:27.551 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:44:27.551 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:44:27.551 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 00:44:27.551 | INFO | High Water Breakdown: | 20260508 00:44:27.551 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:44:27.551 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:44:27.551 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:44:27.551 | INFO | Uploaded 36GiB to GPU in 2.5334s (14.2101GiB/s) | 20260508 00:44:27.551 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:44:27.551 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:44:27.551 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:44:27.551 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:44:27.551 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:44:27.551 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:44:27.551 | INFO | Downloaded 34GiB to GPU in 2.45119s (13.8708GiB/s) | 20260508 00:44:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:44:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:44:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:44:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:44:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:44:27.551 | INFO | ------------------------------------------ | 20260508 00:44:27.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:44:27.551 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:44:27.551 | INFO | Total mem usage: 27.8 GiB | 20260508 00:44:27.551 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:44:27.551 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:44:27.551 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:44:27.551 | INFO | High Water Breakdown: | 20260508 00:44:27.551 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:44:27.551 | INFO | ------------------------------------------ | 20260508 00:44:27.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:44:27.551 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:44:27.551 | INFO | Total mem usage: 15.7 GiB | 20260508 00:44:27.551 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:44:27.551 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:44:27.551 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:44:27.551 | INFO | High Water Breakdown: | 20260508 00:44:27.551 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:44:27.551 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:44:27.551 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:44:27.551 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:44:27.551 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:44:27.551 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:44:27.552 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:44:27.552 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:44:27.552 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:44:27.552 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:44:27.552 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:44:27.552 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:44:27.552 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:44:27.552 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:44:27.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:44:27.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:44:27.552 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:44:27.552 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:44:27.552 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:44:27.552 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:44:27.552 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:44:27.552 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:44:27.552 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:44:27.552 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:44:27.552 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:44:27.552 | INFO | ------------------------------------------ | 20260508 00:44:27.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:44:27.552 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:44:27.552 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:44:27.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:44:27.552 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.39e+01 / 1.19e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.09e-02 / 4.48e-03 | 20260508 00:44:27.552 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.51e+00, 5.89e-01 / 1.85e-01 | 20260508 00:44:27.552 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.06e-01 | 20260508 00:44:27.552 | INFO | PerfReport: 16.03%, DME -- 4.20e+01 / 4.59e+01, 3.84e+00 / 1.54e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.63e+00 / 1.56e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.28e-01 / 2.56e-01 | 20260508 00:44:27.552 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.33e+00, 2.77e-01 / 1.58e-01 | 20260508 00:44:27.552 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.85e-01, 5.45e-02 / 3.38e-02 | 20260508 00:44:27.552 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:44:27.552 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:44:27.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:44:27.552 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 24.19%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 30.77%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 2.04%, LockUpload -- 2.18e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 2.91%, LockDownload -- 3.10e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:44:27.552 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260508 00:44:27.648 | INFO | Starting a round of Baz IO | 20260508 00:44:31.119 | INFO | Populating RTMetrics report | 20260508 00:44:31.119 | INFO | RTMetrics report populated | 20260508 00:44:31.119 | INFO | Opening tmp RTMetrics json file | 20260508 00:44:31.174 | INFO | Serializing RTMetrics to json | 20260508 00:44:31.194 | INFO | Serialization done, swapping tmp file now... | 20260508 00:44:31.194 | INFO | RTMetrics done | 20260508 00:44:31.220 | INFO | Finished a round of Baz IO: Wrote 1.683 GiB in 3.572 seconds | 20260508 00:44:31.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.08% / 0.00% / 57.91% | 20260508 00:44:31.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.965142% / 0% / 5.5631e-05% | 20260508 00:44:31.220 | INFO | Baz IO overall: | 20260508 00:44:31.220 | INFO | BytesWritten : 1770648707411 | 20260508 00:44:31.220 | INFO | NumEvents : 856862617628 | 20260508 00:44:32.686 | INFO | Analyzing chunk frames = [3899392,3899904) | 20260508 00:44:32.686 | INFO | MemInfo: { | 20260508 00:44:32.686 | INFO | "gpu" : " 17 GB", | 20260508 00:44:32.686 | INFO | "huge" : "", | 20260508 00:44:32.686 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:32.686 | INFO | "malloc" : " 14 GB", | 20260508 00:44:32.686 | INFO | "pinned" : " 29 GB" | 20260508 00:44:32.686 | INFO | } | 20260508 00:44:32.686 | INFO | Perf report for the last 5.13561 seconds: | 20260508 00:44:32.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:32.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:32.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:32.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:32.686 | INFO | ANALYSIS : 68.5%, 2.9, 104.8ms, 0 | 20260508 00:44:32.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:32.686 | INFO | RT_METRICS : 44.6%, 1.0, 23.3ms, 0 | 20260508 00:44:32.686 | INFO | PRE_HQ : 67.2%, 1.8, 61.2ms, 0 | 20260508 00:44:32.686 | INFO | BAZWRITER : 69.2%, 3.7, 132.4ms, 0 | 20260508 00:44:32.686 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 00:44:32.686 | INFO | Uploaded 36GiB to GPU in 2.59353s (13.8807GiB/s) | 20260508 00:44:32.686 | INFO | Downloaded 38GiB to GPU in 2.68237s (14.1666GiB/s) | 20260508 00:44:37.830 | INFO | Analyzing chunk frames = [3899904,3900416) | 20260508 00:44:37.830 | INFO | MemInfo: { | 20260508 00:44:37.830 | INFO | "gpu" : " 17 GB", | 20260508 00:44:37.830 | INFO | "huge" : "", | 20260508 00:44:37.830 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:37.830 | INFO | "malloc" : " 14 GB", | 20260508 00:44:37.830 | INFO | "pinned" : " 29 GB" | 20260508 00:44:37.830 | INFO | } | 20260508 00:44:37.830 | INFO | Perf report for the last 5.14462 seconds: | 20260508 00:44:37.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:37.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 00:44:37.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:44:37.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:37.831 | INFO | Uploaded 36GiB to GPU in 2.54982s (14.1187GiB/s) | 20260508 00:44:37.831 | INFO | Downloaded 34GiB to GPU in 2.48692s (13.6715GiB/s) | 20260508 00:44:42.970 | INFO | Analyzing chunk frames = [3900416,3900928) | 20260508 00:44:42.970 | INFO | MemInfo: { | 20260508 00:44:42.970 | INFO | "gpu" : " 17 GB", | 20260508 00:44:42.970 | INFO | "huge" : "", | 20260508 00:44:42.970 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:42.970 | INFO | "malloc" : " 14 GB", | 20260508 00:44:42.970 | INFO | "pinned" : " 29 GB" | 20260508 00:44:42.970 | INFO | } | 20260508 00:44:42.970 | INFO | Perf report for the last 5.13975 seconds: | 20260508 00:44:42.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:42.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.970 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:44:42.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.971 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:44:42.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:42.971 | INFO | Uploaded 36GiB to GPU in 2.50478s (14.3725GiB/s) | 20260508 00:44:42.971 | INFO | Downloaded 34GiB to GPU in 2.49099s (13.6492GiB/s) | 20260508 00:44:48.112 | INFO | Analyzing chunk frames = [3900928,3901440) | 20260508 00:44:48.112 | INFO | MemInfo: { | 20260508 00:44:48.112 | INFO | "gpu" : " 17 GB", | 20260508 00:44:48.112 | INFO | "huge" : "", | 20260508 00:44:48.112 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:48.112 | INFO | "malloc" : " 14 GB", | 20260508 00:44:48.112 | INFO | "pinned" : " 29 GB" | 20260508 00:44:48.112 | INFO | } | 20260508 00:44:48.112 | INFO | Perf report for the last 5.14156 seconds: | 20260508 00:44:48.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:48.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:44:48.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:44:48.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:48.112 | INFO | Uploaded 36GiB to GPU in 2.4929s (14.441GiB/s) | 20260508 00:44:48.112 | INFO | Downloaded 34GiB to GPU in 2.48744s (13.6687GiB/s) | 20260508 00:44:53.246 | INFO | Analyzing chunk frames = [3901440,3901952) | 20260508 00:44:53.246 | INFO | MemInfo: { | 20260508 00:44:53.246 | INFO | "gpu" : " 17 GB", | 20260508 00:44:53.246 | INFO | "huge" : "", | 20260508 00:44:53.246 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:53.246 | INFO | "malloc" : " 14 GB", | 20260508 00:44:53.246 | INFO | "pinned" : " 29 GB" | 20260508 00:44:53.246 | INFO | } | 20260508 00:44:53.246 | INFO | Perf report for the last 5.13397 seconds: | 20260508 00:44:53.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:53.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 00:44:53.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 00:44:53.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:53.246 | INFO | Uploaded 36GiB to GPU in 2.35097s (15.3128GiB/s) | 20260508 00:44:53.247 | INFO | Downloaded 34GiB to GPU in 2.49447s (13.6302GiB/s) | 20260508 00:44:58.389 | INFO | Analyzing chunk frames = [3901952,3902464) | 20260508 00:44:58.389 | INFO | MemInfo: { | 20260508 00:44:58.389 | INFO | "gpu" : " 17 GB", | 20260508 00:44:58.389 | INFO | "huge" : "", | 20260508 00:44:58.389 | INFO | "huge_shared" : " 29 GB", | 20260508 00:44:58.389 | INFO | "malloc" : " 14 GB", | 20260508 00:44:58.389 | INFO | "pinned" : " 29 GB" | 20260508 00:44:58.389 | INFO | } | 20260508 00:44:58.389 | INFO | Perf report for the last 5.14346 seconds: | 20260508 00:44:58.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:44:58.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.389 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 00:44:58.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.389 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:44:58.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:44:58.390 | INFO | Uploaded 36GiB to GPU in 2.33402s (15.424GiB/s) | 20260508 00:44:58.390 | INFO | Downloaded 34GiB to GPU in 2.41799s (14.0613GiB/s) | 20260508 00:45:03.527 | INFO | Analyzing chunk frames = [3902464,3902976) | 20260508 00:45:03.527 | INFO | MemInfo: { | 20260508 00:45:03.527 | INFO | "gpu" : " 17 GB", | 20260508 00:45:03.527 | INFO | "huge" : "", | 20260508 00:45:03.527 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:03.527 | INFO | "malloc" : " 14 GB", | 20260508 00:45:03.527 | INFO | "pinned" : " 29 GB" | 20260508 00:45:03.527 | INFO | } | 20260508 00:45:03.528 | INFO | Perf report for the last 5.13847 seconds: | 20260508 00:45:03.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:03.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 00:45:03.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 00:45:03.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:03.528 | INFO | Uploaded 36GiB to GPU in 2.39877s (15.0077GiB/s) | 20260508 00:45:03.528 | INFO | Downloaded 34GiB to GPU in 2.43616s (13.9564GiB/s) | 20260508 00:45:08.672 | INFO | Analyzing chunk frames = [3902976,3903488) | 20260508 00:45:08.672 | INFO | MemInfo: { | 20260508 00:45:08.672 | INFO | "gpu" : " 17 GB", | 20260508 00:45:08.672 | INFO | "huge" : "", | 20260508 00:45:08.672 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:08.672 | INFO | "malloc" : " 14 GB", | 20260508 00:45:08.672 | INFO | "pinned" : " 29 GB" | 20260508 00:45:08.672 | INFO | } | 20260508 00:45:08.672 | INFO | Perf report for the last 5.14438 seconds: | 20260508 00:45:08.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:08.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.672 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 00:45:08.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.672 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260508 00:45:08.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:08.673 | INFO | Uploaded 36GiB to GPU in 2.41597s (14.9008GiB/s) | 20260508 00:45:08.673 | INFO | Downloaded 34GiB to GPU in 2.43599s (13.9574GiB/s) | 20260508 00:45:08.770 | INFO | Starting a round of Baz IO | 20260508 00:45:12.226 | INFO | Populating RTMetrics report | 20260508 00:45:12.226 | INFO | RTMetrics report populated | 20260508 00:45:12.226 | INFO | Opening tmp RTMetrics json file | 20260508 00:45:12.467 | INFO | Serializing RTMetrics to json | 20260508 00:45:12.737 | INFO | Serialization done, swapping tmp file now... | 20260508 00:45:12.743 | INFO | RTMetrics done | 20260508 00:45:12.762 | INFO | Finished a round of Baz IO: Wrote 1.682 GiB in 3.992 seconds | 20260508 00:45:12.762 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.08% / 0.00% / 57.92% | 20260508 00:45:12.762 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.96217% / 7.94729e-06% / 5.5631e-05% | 20260508 00:45:12.762 | INFO | Baz IO overall: | 20260508 00:45:12.762 | INFO | BytesWritten : 1772455076765 | 20260508 00:45:12.763 | INFO | NumEvents : 857731482022 | 20260508 00:45:13.809 | INFO | Analyzing chunk frames = [3903488,3904000) | 20260508 00:45:13.809 | INFO | MemInfo: { | 20260508 00:45:13.809 | INFO | "gpu" : " 17 GB", | 20260508 00:45:13.809 | INFO | "huge" : "", | 20260508 00:45:13.809 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:13.809 | INFO | "malloc" : " 14 GB", | 20260508 00:45:13.809 | INFO | "pinned" : " 29 GB" | 20260508 00:45:13.809 | INFO | } | 20260508 00:45:13.809 | INFO | Perf report for the last 5.1368 seconds: | 20260508 00:45:13.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:13.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:13.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:13.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:13.809 | INFO | ANALYSIS : 68.2%, 3.0, 106.7ms, 0 | 20260508 00:45:13.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:13.809 | INFO | RT_METRICS : 50.2%, 1.0, 26.3ms, 0 | 20260508 00:45:13.809 | INFO | PRE_HQ : 65.6%, 1.7, 56.5ms, 0 | 20260508 00:45:13.809 | INFO | BAZWRITER : 77.3%, 3.5, 140.8ms, 0 | 20260508 00:45:13.809 | INFO | BAZ_SAMPLING : 10.3%, 1.0, 5.3ms, 0 | 20260508 00:45:13.810 | INFO | Uploaded 36GiB to GPU in 2.56398s (14.0407GiB/s) | 20260508 00:45:13.810 | INFO | Downloaded 38GiB to GPU in 2.69632s (14.0933GiB/s) | 20260508 00:45:18.942 | INFO | Analyzing chunk frames = [3904000,3904512) | 20260508 00:45:18.942 | INFO | MemInfo: { | 20260508 00:45:18.942 | INFO | "gpu" : " 17 GB", | 20260508 00:45:18.942 | INFO | "huge" : "", | 20260508 00:45:18.942 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:18.942 | INFO | "malloc" : " 14 GB", | 20260508 00:45:18.942 | INFO | "pinned" : " 29 GB" | 20260508 00:45:18.942 | INFO | } | 20260508 00:45:18.942 | INFO | Perf report for the last 5.13327 seconds: | 20260508 00:45:18.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:18.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.942 | INFO | ANALYSIS : 67.6%, 3.0, 105.1ms, 0 | 20260508 00:45:18.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.942 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:45:18.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:18.943 | INFO | Uploaded 36GiB to GPU in 2.49541s (14.4265GiB/s) | 20260508 00:45:18.943 | INFO | Downloaded 34GiB to GPU in 2.43278s (13.9758GiB/s) | 20260508 00:45:24.086 | INFO | Analyzing chunk frames = [3904512,3905024) | 20260508 00:45:24.086 | INFO | MemInfo: { | 20260508 00:45:24.086 | INFO | "gpu" : " 17 GB", | 20260508 00:45:24.086 | INFO | "huge" : "", | 20260508 00:45:24.086 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:24.086 | INFO | "malloc" : " 14 GB", | 20260508 00:45:24.086 | INFO | "pinned" : " 29 GB" | 20260508 00:45:24.086 | INFO | } | 20260508 00:45:24.086 | INFO | Perf report for the last 5.144 seconds: | 20260508 00:45:24.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:24.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.086 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 00:45:24.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.086 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:45:24.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:24.087 | INFO | Uploaded 36GiB to GPU in 2.46799s (14.5868GiB/s) | 20260508 00:45:24.087 | INFO | Downloaded 34GiB to GPU in 2.50538s (13.5708GiB/s) | 20260508 00:45:29.228 | INFO | Analyzing chunk frames = [3905024,3905536) | 20260508 00:45:29.228 | INFO | MemInfo: { | 20260508 00:45:29.228 | INFO | "gpu" : " 17 GB", | 20260508 00:45:29.228 | INFO | "huge" : "", | 20260508 00:45:29.228 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:29.228 | INFO | "malloc" : " 14 GB", | 20260508 00:45:29.228 | INFO | "pinned" : " 29 GB" | 20260508 00:45:29.228 | INFO | } | 20260508 00:45:29.228 | INFO | Perf report for the last 5.14213 seconds: | 20260508 00:45:29.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:29.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.228 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 00:45:29.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.229 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:45:29.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:29.229 | INFO | Uploaded 36GiB to GPU in 2.46876s (14.5822GiB/s) | 20260508 00:45:29.229 | INFO | Downloaded 34GiB to GPU in 2.50312s (13.5831GiB/s) | 20260508 00:45:34.369 | INFO | Analyzing chunk frames = [3905536,3906048) | 20260508 00:45:34.369 | INFO | MemInfo: { | 20260508 00:45:34.369 | INFO | "gpu" : " 17 GB", | 20260508 00:45:34.369 | INFO | "huge" : "", | 20260508 00:45:34.369 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:34.369 | INFO | "malloc" : " 14 GB", | 20260508 00:45:34.369 | INFO | "pinned" : " 29 GB" | 20260508 00:45:34.369 | INFO | } | 20260508 00:45:34.369 | INFO | Perf report for the last 5.14093 seconds: | 20260508 00:45:34.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:34.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.369 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:45:34.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.370 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 00:45:34.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:34.370 | INFO | Uploaded 36GiB to GPU in 2.30711s (15.604GiB/s) | 20260508 00:45:34.370 | INFO | Downloaded 34GiB to GPU in 2.49311s (13.6376GiB/s) | 20260508 00:45:39.498 | INFO | Analyzing chunk frames = [3906048,3906560) | 20260508 00:45:39.498 | INFO | MemInfo: { | 20260508 00:45:39.498 | INFO | "gpu" : " 17 GB", | 20260508 00:45:39.498 | INFO | "huge" : "", | 20260508 00:45:39.498 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:39.498 | INFO | "malloc" : " 14 GB", | 20260508 00:45:39.498 | INFO | "pinned" : " 29 GB" | 20260508 00:45:39.498 | INFO | } | 20260508 00:45:39.498 | INFO | Perf report for the last 5.12881 seconds: | 20260508 00:45:39.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:39.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.498 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:45:39.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.499 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 00:45:39.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:39.499 | INFO | Uploaded 36GiB to GPU in 2.37296s (15.1709GiB/s) | 20260508 00:45:39.499 | INFO | Downloaded 34GiB to GPU in 2.4305s (13.9889GiB/s) | 20260508 00:45:44.635 | INFO | Analyzing chunk frames = [3906560,3907072) | 20260508 00:45:44.635 | INFO | MemInfo: { | 20260508 00:45:44.635 | INFO | "gpu" : " 17 GB", | 20260508 00:45:44.635 | INFO | "huge" : "", | 20260508 00:45:44.635 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:44.635 | INFO | "malloc" : " 14 GB", | 20260508 00:45:44.635 | INFO | "pinned" : " 29 GB" | 20260508 00:45:44.635 | INFO | } | 20260508 00:45:44.635 | INFO | Perf report for the last 5.13682 seconds: | 20260508 00:45:44.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:44.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 00:45:44.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 00:45:44.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:44.635 | INFO | Uploaded 36GiB to GPU in 2.40422s (14.9737GiB/s) | 20260508 00:45:44.635 | INFO | Downloaded 34GiB to GPU in 2.44088s (13.9294GiB/s) | 20260508 00:45:49.780 | INFO | Analyzing chunk frames = [3907072,3907584) | 20260508 00:45:49.780 | INFO | MemInfo: { | 20260508 00:45:49.780 | INFO | "gpu" : " 17 GB", | 20260508 00:45:49.780 | INFO | "huge" : "", | 20260508 00:45:49.780 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:49.781 | INFO | "malloc" : " 14 GB", | 20260508 00:45:49.781 | INFO | "pinned" : " 29 GB" | 20260508 00:45:49.781 | INFO | } | 20260508 00:45:49.781 | INFO | Perf report for the last 5.14559 seconds: | 20260508 00:45:49.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:49.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 00:45:49.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | PRE_HQ : 35.7%, 1.0, 18.6ms, 0 | 20260508 00:45:49.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:49.781 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:45:49.781 | INFO | Total mem usage: 16.7 GiB | 20260508 00:45:49.781 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:45:49.781 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 00:45:49.781 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:45:49.781 | INFO | High Water Breakdown: | 20260508 00:45:49.781 | INFO | 59.8% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:45:49.781 | INFO | 40.1% : 5.86 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:45:49.781 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:45:49.781 | INFO | ------------------------------------------ | 20260508 00:45:49.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:45:49.781 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:45:49.781 | INFO | Total mem usage: 26.5 GiB | 20260508 00:45:49.781 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:45:49.781 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:45:49.781 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:45:49.781 | INFO | Uploaded 36GiB to GPU in 2.50147s (14.3915GiB/s) | 20260508 00:45:49.782 | INFO | Downloaded 34GiB to GPU in 2.46272s (13.8059GiB/s) | 20260508 00:45:49.781 | INFO | High Water Breakdown: | 20260508 00:45:49.782 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:45:49.782 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:45:49.782 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:45:49.782 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:45:49.782 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:45:49.782 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:45:49.782 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:45:49.782 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:45:49.782 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:45:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:45:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:45:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:45:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:45:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:45:49.782 | INFO | ------------------------------------------ | 20260508 00:45:49.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:45:49.782 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:45:49.782 | INFO | Total mem usage: 27.8 GiB | 20260508 00:45:49.782 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:45:49.782 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:45:49.782 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:45:49.782 | INFO | High Water Breakdown: | 20260508 00:45:49.782 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:45:49.782 | INFO | ------------------------------------------ | 20260508 00:45:49.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:45:49.782 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:45:49.782 | INFO | Total mem usage: 15.7 GiB | 20260508 00:45:49.782 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:45:49.782 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:45:49.782 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:45:49.782 | INFO | High Water Breakdown: | 20260508 00:45:49.782 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:45:49.782 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:45:49.782 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:45:49.782 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:45:49.782 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:45:49.782 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:45:49.782 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:45:49.782 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:45:49.782 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:45:49.782 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:45:49.782 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:45:49.782 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:45:49.782 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:45:49.782 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:45:49.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:45:49.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:45:49.782 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:45:49.782 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:45:49.782 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:45:49.782 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:45:49.782 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:45:49.782 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:45:49.782 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:45:49.782 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:45:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:45:49.782 | INFO | ------------------------------------------ | 20260508 00:45:49.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:45:49.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:45:49.782 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:45:49.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:45:49.782 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.61e+01, 1.44e+01 / 1.32e+00 | 20260508 00:45:49.782 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.02e-01, 1.20e-02 / 5.73e-03 | 20260508 00:45:49.782 | INFO | PerfReport: 12.20%, Baseline -- 4.01e+00 / 4.49e+00, 5.89e-01 / 1.86e-01 | 20260508 00:45:49.782 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.08e-01 / 9.55e-02 | 20260508 00:45:49.782 | INFO | PerfReport: 16.09%, DME -- 4.23e+01 / 4.61e+01, 4.21e+00 / 1.56e+00 | 20260508 00:45:49.782 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.69e+00 / 1.51e+00 | 20260508 00:45:49.782 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.31e-01 / 2.17e-01 | 20260508 00:45:49.782 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.29e+00, 2.71e-01 / 1.35e-01 | 20260508 00:45:49.782 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.75e-01, 5.34e-02 / 2.33e-02 | 20260508 00:45:49.783 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:45:49.783 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:45:49.783 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:45:49.783 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 24.03%, Upload -- 2.55e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 30.95%, Compute -- 3.29e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 1.90%, LockUpload -- 2.02e+00 / 0.00e+00, 5.23e+00 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 2.91%, LockDownload -- 3.10e+00 / 0.00e+00, 4.40e+00 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 15.76%, LockCompute -- 1.68e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:45:49.783 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.79e-02 / 0.00e+00 | 20260508 00:45:49.897 | INFO | Starting a round of Baz IO | 20260508 00:45:53.361 | INFO | Populating RTMetrics report | 20260508 00:45:53.361 | INFO | RTMetrics report populated | 20260508 00:45:53.361 | INFO | Opening tmp RTMetrics json file | 20260508 00:45:53.383 | INFO | Serializing RTMetrics to json | 20260508 00:45:53.386 | INFO | Serialization done, swapping tmp file now... | 20260508 00:45:53.386 | INFO | RTMetrics done | 20260508 00:45:53.463 | INFO | Finished a round of Baz IO: Wrote 1.682 GiB in 3.565 seconds | 20260508 00:45:53.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.07% / 0.00% / 57.92% | 20260508 00:45:53.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.96157% / 0% / 5.5631e-05% | 20260508 00:45:53.463 | INFO | Baz IO overall: | 20260508 00:45:53.463 | INFO | BytesWritten : 1774261301045 | 20260508 00:45:53.463 | INFO | NumEvents : 858600191287 | 20260508 00:45:54.927 | INFO | Analyzing chunk frames = [3907584,3908096) | 20260508 00:45:54.927 | INFO | MemInfo: { | 20260508 00:45:54.927 | INFO | "gpu" : " 17 GB", | 20260508 00:45:54.927 | INFO | "huge" : "", | 20260508 00:45:54.927 | INFO | "huge_shared" : " 29 GB", | 20260508 00:45:54.927 | INFO | "malloc" : " 14 GB", | 20260508 00:45:54.927 | INFO | "pinned" : " 29 GB" | 20260508 00:45:54.927 | INFO | } | 20260508 00:45:54.927 | INFO | Perf report for the last 5.14602 seconds: | 20260508 00:45:54.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:45:54.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:54.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:54.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:54.927 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260508 00:45:54.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:45:54.927 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260508 00:45:54.927 | INFO | PRE_HQ : 67.7%, 1.8, 63.0ms, 0 | 20260508 00:45:54.927 | INFO | BAZWRITER : 68.9%, 4.4, 154.8ms, 0 | 20260508 00:45:54.927 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:45:54.927 | INFO | Uploaded 36GiB to GPU in 2.60795s (13.8039GiB/s) | 20260508 00:45:54.927 | INFO | Downloaded 38GiB to GPU in 2.71796s (13.9811GiB/s) | 20260508 00:46:00.062 | INFO | Analyzing chunk frames = [3908096,3908608) | 20260508 00:46:00.062 | INFO | MemInfo: { | 20260508 00:46:00.062 | INFO | "gpu" : " 17 GB", | 20260508 00:46:00.062 | INFO | "huge" : "", | 20260508 00:46:00.062 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:00.062 | INFO | "malloc" : " 14 GB", | 20260508 00:46:00.062 | INFO | "pinned" : " 29 GB" | 20260508 00:46:00.062 | INFO | } | 20260508 00:46:00.062 | INFO | Perf report for the last 5.13562 seconds: | 20260508 00:46:00.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:00.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.062 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:46:00.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.062 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:46:00.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:00.063 | INFO | Uploaded 36GiB to GPU in 2.5309s (14.2242GiB/s) | 20260508 00:46:00.063 | INFO | Downloaded 34GiB to GPU in 2.42326s (14.0307GiB/s) | 20260508 00:46:05.197 | INFO | Analyzing chunk frames = [3908608,3909120) | 20260508 00:46:05.197 | INFO | MemInfo: { | 20260508 00:46:05.197 | INFO | "gpu" : " 17 GB", | 20260508 00:46:05.197 | INFO | "huge" : "", | 20260508 00:46:05.197 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:05.197 | INFO | "malloc" : " 14 GB", | 20260508 00:46:05.197 | INFO | "pinned" : " 29 GB" | 20260508 00:46:05.197 | INFO | } | 20260508 00:46:05.197 | INFO | Perf report for the last 5.13473 seconds: | 20260508 00:46:05.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:05.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.197 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 00:46:05.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.197 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:46:05.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:05.198 | INFO | Uploaded 36GiB to GPU in 2.52299s (14.2688GiB/s) | 20260508 00:46:05.198 | INFO | Downloaded 34GiB to GPU in 2.49417s (13.6318GiB/s) | 20260508 00:46:10.341 | INFO | Analyzing chunk frames = [3909120,3909632) | 20260508 00:46:10.341 | INFO | MemInfo: { | 20260508 00:46:10.341 | INFO | "gpu" : " 17 GB", | 20260508 00:46:10.342 | INFO | "huge" : "", | 20260508 00:46:10.342 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:10.342 | INFO | "malloc" : " 14 GB", | 20260508 00:46:10.342 | INFO | "pinned" : " 29 GB" | 20260508 00:46:10.342 | INFO | } | 20260508 00:46:10.342 | INFO | Perf report for the last 5.14454 seconds: | 20260508 00:46:10.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:10.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 00:46:10.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:46:10.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:10.342 | INFO | Uploaded 36GiB to GPU in 2.50903s (14.3482GiB/s) | 20260508 00:46:10.342 | INFO | Downloaded 34GiB to GPU in 2.48924s (13.6588GiB/s) | 20260508 00:46:15.481 | INFO | Analyzing chunk frames = [3909632,3910144) | 20260508 00:46:15.481 | INFO | MemInfo: { | 20260508 00:46:15.481 | INFO | "gpu" : " 17 GB", | 20260508 00:46:15.481 | INFO | "huge" : "", | 20260508 00:46:15.481 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:15.481 | INFO | "malloc" : " 14 GB", | 20260508 00:46:15.481 | INFO | "pinned" : " 29 GB" | 20260508 00:46:15.481 | INFO | } | 20260508 00:46:15.481 | INFO | Perf report for the last 5.13924 seconds: | 20260508 00:46:15.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:15.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 00:46:15.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:46:15.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:15.481 | INFO | Uploaded 36GiB to GPU in 2.37103s (15.1833GiB/s) | 20260508 00:46:15.482 | INFO | Downloaded 34GiB to GPU in 2.55085s (13.3289GiB/s) | 20260508 00:46:20.620 | INFO | Analyzing chunk frames = [3910144,3910656) | 20260508 00:46:20.620 | INFO | MemInfo: { | 20260508 00:46:20.620 | INFO | "gpu" : " 17 GB", | 20260508 00:46:20.620 | INFO | "huge" : "", | 20260508 00:46:20.620 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:20.620 | INFO | "malloc" : " 14 GB", | 20260508 00:46:20.620 | INFO | "pinned" : " 29 GB" | 20260508 00:46:20.620 | INFO | } | 20260508 00:46:20.620 | INFO | Perf report for the last 5.13919 seconds: | 20260508 00:46:20.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:20.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:46:20.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:46:20.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:20.620 | INFO | Uploaded 36GiB to GPU in 2.40959s (14.9403GiB/s) | 20260508 00:46:20.620 | INFO | Downloaded 34GiB to GPU in 2.40795s (14.1199GiB/s) | 20260508 00:46:25.760 | INFO | Analyzing chunk frames = [3910656,3911168) | 20260508 00:46:25.760 | INFO | MemInfo: { | 20260508 00:46:25.760 | INFO | "gpu" : " 17 GB", | 20260508 00:46:25.760 | INFO | "huge" : "", | 20260508 00:46:25.760 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:25.760 | INFO | "malloc" : " 14 GB", | 20260508 00:46:25.760 | INFO | "pinned" : " 29 GB" | 20260508 00:46:25.760 | INFO | } | 20260508 00:46:25.760 | INFO | Perf report for the last 5.14004 seconds: | 20260508 00:46:25.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:25.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.760 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 00:46:25.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.760 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:46:25.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:25.761 | INFO | Uploaded 36GiB to GPU in 2.4237s (14.8533GiB/s) | 20260508 00:46:25.761 | INFO | Downloaded 34GiB to GPU in 2.42143s (14.0413GiB/s) | 20260508 00:46:30.896 | INFO | Analyzing chunk frames = [3911168,3911680) | 20260508 00:46:30.896 | INFO | MemInfo: { | 20260508 00:46:30.896 | INFO | "gpu" : " 17 GB", | 20260508 00:46:30.896 | INFO | "huge" : "", | 20260508 00:46:30.896 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:30.896 | INFO | "malloc" : " 14 GB", | 20260508 00:46:30.896 | INFO | "pinned" : " 29 GB" | 20260508 00:46:30.896 | INFO | } | 20260508 00:46:30.896 | INFO | Perf report for the last 5.13628 seconds: | 20260508 00:46:30.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:30.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.896 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:46:30.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.897 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 00:46:30.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:30.897 | INFO | Uploaded 36GiB to GPU in 2.55978s (14.0637GiB/s) | 20260508 00:46:30.897 | INFO | Downloaded 34GiB to GPU in 2.46623s (13.7862GiB/s) | 20260508 00:46:31.010 | INFO | Starting a round of Baz IO | 20260508 00:46:34.474 | INFO | Populating RTMetrics report | 20260508 00:46:34.475 | INFO | RTMetrics report populated | 20260508 00:46:34.475 | INFO | Opening tmp RTMetrics json file | 20260508 00:46:34.500 | INFO | Serializing RTMetrics to json | 20260508 00:46:34.611 | INFO | Serialization done, swapping tmp file now... | 20260508 00:46:34.612 | INFO | RTMetrics done | 20260508 00:46:34.630 | INFO | Finished a round of Baz IO: Wrote 1.683 GiB in 3.62 seconds | 20260508 00:46:34.631 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.07% / 0.00% / 57.93% | 20260508 00:46:34.631 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.960191% / 0% / 6.75519e-05% | 20260508 00:46:34.631 | INFO | Baz IO overall: | 20260508 00:46:34.631 | INFO | BytesWritten : 1776068003894 | 20260508 00:46:34.631 | INFO | NumEvents : 859469446731 | 20260508 00:46:36.035 | INFO | Analyzing chunk frames = [3911680,3912192) | 20260508 00:46:36.035 | INFO | MemInfo: { | 20260508 00:46:36.035 | INFO | "gpu" : " 17 GB", | 20260508 00:46:36.035 | INFO | "huge" : "", | 20260508 00:46:36.035 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:36.035 | INFO | "malloc" : " 14 GB", | 20260508 00:46:36.035 | INFO | "pinned" : " 29 GB" | 20260508 00:46:36.035 | INFO | } | 20260508 00:46:36.035 | INFO | Perf report for the last 5.1389 seconds: | 20260508 00:46:36.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:36.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:36.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:36.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:36.035 | INFO | ANALYSIS : 68.7%, 2.9, 104.7ms, 0 | 20260508 00:46:36.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:36.035 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260508 00:46:36.035 | INFO | PRE_HQ : 67.4%, 1.8, 62.5ms, 0 | 20260508 00:46:36.035 | INFO | BAZWRITER : 70.0%, 3.9, 140.0ms, 0 | 20260508 00:46:36.035 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260508 00:46:36.036 | INFO | Uploaded 36GiB to GPU in 2.58919s (13.904GiB/s) | 20260508 00:46:36.036 | INFO | Downloaded 38GiB to GPU in 2.69672s (14.0912GiB/s) | 20260508 00:46:41.175 | INFO | Analyzing chunk frames = [3912192,3912704) | 20260508 00:46:41.175 | INFO | MemInfo: { | 20260508 00:46:41.175 | INFO | "gpu" : " 17 GB", | 20260508 00:46:41.175 | INFO | "huge" : "", | 20260508 00:46:41.175 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:41.175 | INFO | "malloc" : " 14 GB", | 20260508 00:46:41.175 | INFO | "pinned" : " 29 GB" | 20260508 00:46:41.175 | INFO | } | 20260508 00:46:41.175 | INFO | Perf report for the last 5.13982 seconds: | 20260508 00:46:41.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:41.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.175 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:46:41.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.175 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 00:46:41.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:41.176 | INFO | Uploaded 36GiB to GPU in 2.52466s (14.2594GiB/s) | 20260508 00:46:41.176 | INFO | Downloaded 34GiB to GPU in 2.50159s (13.5914GiB/s) | 20260508 00:46:46.315 | INFO | Analyzing chunk frames = [3912704,3913216) | 20260508 00:46:46.315 | INFO | MemInfo: { | 20260508 00:46:46.315 | INFO | "gpu" : " 17 GB", | 20260508 00:46:46.315 | INFO | "huge" : "", | 20260508 00:46:46.315 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:46.315 | INFO | "malloc" : " 14 GB", | 20260508 00:46:46.315 | INFO | "pinned" : " 29 GB" | 20260508 00:46:46.315 | INFO | } | 20260508 00:46:46.315 | INFO | Perf report for the last 5.14018 seconds: | 20260508 00:46:46.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:46.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 00:46:46.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:46:46.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:46.316 | INFO | Uploaded 36GiB to GPU in 2.45611s (14.6573GiB/s) | 20260508 00:46:46.316 | INFO | Downloaded 34GiB to GPU in 2.51164s (13.537GiB/s) | 20260508 00:46:51.448 | INFO | Analyzing chunk frames = [3913216,3913728) | 20260508 00:46:51.448 | INFO | MemInfo: { | 20260508 00:46:51.448 | INFO | "gpu" : " 17 GB", | 20260508 00:46:51.448 | INFO | "huge" : "", | 20260508 00:46:51.448 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:51.448 | INFO | "malloc" : " 14 GB", | 20260508 00:46:51.448 | INFO | "pinned" : " 29 GB" | 20260508 00:46:51.448 | INFO | } | 20260508 00:46:51.448 | INFO | Perf report for the last 5.13233 seconds: | 20260508 00:46:51.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:51.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:46:51.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:46:51.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:51.448 | INFO | Uploaded 36GiB to GPU in 2.46581s (14.5997GiB/s) | 20260508 00:46:51.448 | INFO | Downloaded 34GiB to GPU in 2.51181s (13.536GiB/s) | 20260508 00:46:56.591 | INFO | Analyzing chunk frames = [3913728,3914240) | 20260508 00:46:56.591 | INFO | MemInfo: { | 20260508 00:46:56.591 | INFO | "gpu" : " 17 GB", | 20260508 00:46:56.591 | INFO | "huge" : "", | 20260508 00:46:56.591 | INFO | "huge_shared" : " 29 GB", | 20260508 00:46:56.591 | INFO | "malloc" : " 14 GB", | 20260508 00:46:56.591 | INFO | "pinned" : " 29 GB" | 20260508 00:46:56.591 | INFO | } | 20260508 00:46:56.591 | INFO | Perf report for the last 5.1432 seconds: | 20260508 00:46:56.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:46:56.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.591 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 00:46:56.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.591 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:46:56.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:46:56.592 | INFO | Uploaded 36GiB to GPU in 2.32043s (15.5144GiB/s) | 20260508 00:46:56.592 | INFO | Downloaded 34GiB to GPU in 2.50631s (13.5657GiB/s) | 20260508 00:47:01.734 | INFO | Analyzing chunk frames = [3914240,3914752) | 20260508 00:47:01.734 | INFO | MemInfo: { | 20260508 00:47:01.734 | INFO | "gpu" : " 17 GB", | 20260508 00:47:01.734 | INFO | "huge" : "", | 20260508 00:47:01.734 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:01.734 | INFO | "malloc" : " 14 GB", | 20260508 00:47:01.735 | INFO | "pinned" : " 29 GB" | 20260508 00:47:01.735 | INFO | } | 20260508 00:47:01.735 | INFO | Perf report for the last 5.14373 seconds: | 20260508 00:47:01.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:01.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:47:01.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:47:01.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:01.735 | INFO | Uploaded 36GiB to GPU in 2.40853s (14.9469GiB/s) | 20260508 00:47:01.735 | INFO | Downloaded 34GiB to GPU in 2.41929s (14.0537GiB/s) | 20260508 00:47:06.874 | INFO | Analyzing chunk frames = [3914752,3915264) | 20260508 00:47:06.874 | INFO | MemInfo: { | 20260508 00:47:06.874 | INFO | "gpu" : " 17 GB", | 20260508 00:47:06.874 | INFO | "huge" : "", | 20260508 00:47:06.874 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:06.874 | INFO | "malloc" : " 14 GB", | 20260508 00:47:06.875 | INFO | "pinned" : " 29 GB" | 20260508 00:47:06.875 | INFO | } | 20260508 00:47:06.875 | INFO | Perf report for the last 5.13999 seconds: | 20260508 00:47:06.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:06.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 00:47:06.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 00:47:06.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:06.875 | INFO | Uploaded 36GiB to GPU in 2.45778s (14.6474GiB/s) | 20260508 00:47:06.875 | INFO | Downloaded 34GiB to GPU in 2.43677s (13.9529GiB/s) | 20260508 00:47:12.013 | INFO | Analyzing chunk frames = [3915264,3915776) | 20260508 00:47:12.013 | INFO | MemInfo: { | 20260508 00:47:12.013 | INFO | "gpu" : " 17 GB", | 20260508 00:47:12.013 | INFO | "huge" : "", | 20260508 00:47:12.013 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:12.013 | INFO | "malloc" : " 14 GB", | 20260508 00:47:12.013 | INFO | "pinned" : " 29 GB" | 20260508 00:47:12.013 | INFO | } | 20260508 00:47:12.013 | INFO | Perf report for the last 5.13804 seconds: | 20260508 00:47:12.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:12.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:47:12.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 00:47:12.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:12.013 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:47:12.013 | INFO | Total mem usage: 16.7 GiB | 20260508 00:47:12.013 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:47:12.013 | INFO | Uploaded 36GiB to GPU in 2.54696s (14.1345GiB/s) | 20260508 00:47:12.013 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.32%) | 20260508 00:47:12.013 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:47:12.013 | INFO | High Water Breakdown: | 20260508 00:47:12.013 | INFO | Downloaded 34GiB to GPU in 2.46321s (13.8032GiB/s) | 20260508 00:47:12.013 | INFO | 60.5% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:47:12.014 | INFO | 39.4% : 6.01 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:47:12.014 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:47:12.014 | INFO | ------------------------------------------ | 20260508 00:47:12.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:47:12.014 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:47:12.014 | INFO | Total mem usage: 26.5 GiB | 20260508 00:47:12.014 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:47:12.014 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:47:12.014 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:47:12.014 | INFO | High Water Breakdown: | 20260508 00:47:12.014 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:47:12.014 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:47:12.014 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:47:12.014 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:47:12.014 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:47:12.014 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:47:12.014 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:47:12.014 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:47:12.014 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:47:12.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:47:12.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:47:12.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:47:12.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:47:12.014 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:47:12.014 | INFO | ------------------------------------------ | 20260508 00:47:12.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:47:12.014 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:47:12.014 | INFO | Total mem usage: 27.8 GiB | 20260508 00:47:12.014 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:47:12.014 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:47:12.014 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:47:12.014 | INFO | High Water Breakdown: | 20260508 00:47:12.014 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:47:12.014 | INFO | ------------------------------------------ | 20260508 00:47:12.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:47:12.014 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:47:12.014 | INFO | Total mem usage: 15.7 GiB | 20260508 00:47:12.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:47:12.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:47:12.014 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:47:12.014 | INFO | High Water Breakdown: | 20260508 00:47:12.014 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:47:12.014 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:47:12.014 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:47:12.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:47:12.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:47:12.014 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:47:12.014 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:47:12.014 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:47:12.014 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:47:12.014 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:47:12.014 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:47:12.014 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:47:12.014 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:47:12.014 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:47:12.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:47:12.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:47:12.014 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:47:12.014 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:47:12.014 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:47:12.014 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:47:12.014 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:47:12.014 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:47:12.014 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:47:12.014 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:47:12.015 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:47:12.015 | INFO | ------------------------------------------ | 20260508 00:47:12.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:47:12.015 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:47:12.015 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:47:12.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:47:12.015 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.43e+01 / 1.77e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.14e-02 / 6.76e-03 | 20260508 00:47:12.015 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.46e+00, 5.87e-01 / 2.12e-01 | 20260508 00:47:12.015 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 9.98e-02 | 20260508 00:47:12.015 | INFO | PerfReport: 16.16%, DME -- 4.24e+01 / 4.63e+01, 4.21e+00 / 1.95e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 56.11%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.57e+00 / 1.46e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.28e-01 / 2.23e-01 | 20260508 00:47:12.015 | INFO | PerfReport: 3.37%, Metrics -- 1.11e+00 / 1.27e+00, 2.70e-01 / 1.19e-01 | 20260508 00:47:12.015 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.41e-01 / 3.75e-01, 5.34e-02 / 2.77e-02 | 20260508 00:47:12.015 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:47:12.015 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:47:12.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:47:12.015 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 24.31%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 24.50%, Download -- 2.61e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 30.83%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 2.90%, LockDownload -- 3.09e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 15.46%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:47:12.015 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260508 00:47:12.125 | INFO | Starting a round of Baz IO | 20260508 00:47:15.578 | INFO | Populating RTMetrics report | 20260508 00:47:15.579 | INFO | RTMetrics report populated | 20260508 00:47:15.579 | INFO | Opening tmp RTMetrics json file | 20260508 00:47:15.583 | INFO | Serializing RTMetrics to json | 20260508 00:47:15.606 | INFO | Serialization done, swapping tmp file now... | 20260508 00:47:15.607 | INFO | RTMetrics done | 20260508 00:47:15.678 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.552 seconds | 20260508 00:47:15.678 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.07% / 0.00% / 57.93% | 20260508 00:47:15.678 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.95636% / 3.97364e-06% / 4.76837e-05% | 20260508 00:47:15.678 | INFO | Baz IO overall: | 20260508 00:47:15.678 | INFO | BytesWritten : 1777872868000 | 20260508 00:47:15.678 | INFO | NumEvents : 860337385776 | 20260508 00:47:17.157 | INFO | Analyzing chunk frames = [3915776,3916288) | 20260508 00:47:17.157 | INFO | MemInfo: { | 20260508 00:47:17.157 | INFO | "gpu" : " 17 GB", | 20260508 00:47:17.157 | INFO | "huge" : "", | 20260508 00:47:17.157 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:17.157 | INFO | "malloc" : " 14 GB", | 20260508 00:47:17.157 | INFO | "pinned" : " 29 GB" | 20260508 00:47:17.157 | INFO | } | 20260508 00:47:17.157 | INFO | Perf report for the last 5.14432 seconds: | 20260508 00:47:17.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:17.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:17.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:17.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:17.157 | INFO | ANALYSIS : 68.6%, 2.9, 104.2ms, 0 | 20260508 00:47:17.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:17.157 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260508 00:47:17.157 | INFO | PRE_HQ : 67.7%, 1.8, 63.2ms, 0 | 20260508 00:47:17.157 | INFO | BAZWRITER : 68.7%, 4.3, 150.9ms, 0 | 20260508 00:47:17.157 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 00:47:17.157 | INFO | Uploaded 36GiB to GPU in 2.61739s (13.7542GiB/s) | 20260508 00:47:17.158 | INFO | Downloaded 38GiB to GPU in 2.70127s (14.0674GiB/s) | 20260508 00:47:22.292 | INFO | Analyzing chunk frames = [3916288,3916800) | 20260508 00:47:22.292 | INFO | MemInfo: { | 20260508 00:47:22.292 | INFO | "gpu" : " 17 GB", | 20260508 00:47:22.292 | INFO | "huge" : "", | 20260508 00:47:22.292 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:22.292 | INFO | "malloc" : " 14 GB", | 20260508 00:47:22.292 | INFO | "pinned" : " 29 GB" | 20260508 00:47:22.292 | INFO | } | 20260508 00:47:22.292 | INFO | Perf report for the last 5.13517 seconds: | 20260508 00:47:22.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:22.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.292 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 00:47:22.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.292 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:47:22.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:22.293 | INFO | Uploaded 36GiB to GPU in 2.55826s (14.0721GiB/s) | 20260508 00:47:22.293 | INFO | Downloaded 34GiB to GPU in 2.41263s (14.0925GiB/s) | 20260508 00:47:27.431 | INFO | Analyzing chunk frames = [3916800,3917312) | 20260508 00:47:27.431 | INFO | MemInfo: { | 20260508 00:47:27.431 | INFO | "gpu" : " 17 GB", | 20260508 00:47:27.431 | INFO | "huge" : "", | 20260508 00:47:27.431 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:27.431 | INFO | "malloc" : " 14 GB", | 20260508 00:47:27.431 | INFO | "pinned" : " 29 GB" | 20260508 00:47:27.431 | INFO | } | 20260508 00:47:27.432 | INFO | Perf report for the last 5.13934 seconds: | 20260508 00:47:27.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:27.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:47:27.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:47:27.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:27.432 | INFO | Uploaded 36GiB to GPU in 2.52414s (14.2623GiB/s) | 20260508 00:47:27.432 | INFO | Downloaded 34GiB to GPU in 2.4704s (13.763GiB/s) | 20260508 00:47:32.568 | INFO | Analyzing chunk frames = [3917312,3917824) | 20260508 00:47:32.568 | INFO | MemInfo: { | 20260508 00:47:32.568 | INFO | "gpu" : " 17 GB", | 20260508 00:47:32.568 | INFO | "huge" : "", | 20260508 00:47:32.568 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:32.568 | INFO | "malloc" : " 14 GB", | 20260508 00:47:32.568 | INFO | "pinned" : " 29 GB" | 20260508 00:47:32.568 | INFO | } | 20260508 00:47:32.568 | INFO | Perf report for the last 5.13685 seconds: | 20260508 00:47:32.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:32.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:47:32.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 00:47:32.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:32.569 | INFO | Uploaded 36GiB to GPU in 2.55472s (14.0915GiB/s) | 20260508 00:47:32.569 | INFO | Downloaded 34GiB to GPU in 2.488s (13.6656GiB/s) | 20260508 00:47:37.707 | INFO | Analyzing chunk frames = [3917824,3918336) | 20260508 00:47:37.707 | INFO | MemInfo: { | 20260508 00:47:37.707 | INFO | "gpu" : " 17 GB", | 20260508 00:47:37.707 | INFO | "huge" : "", | 20260508 00:47:37.707 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:37.707 | INFO | "malloc" : " 14 GB", | 20260508 00:47:37.707 | INFO | "pinned" : " 29 GB" | 20260508 00:47:37.707 | INFO | } | 20260508 00:47:37.707 | INFO | Perf report for the last 5.13838 seconds: | 20260508 00:47:37.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:37.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.707 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 00:47:37.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.707 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 00:47:37.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:37.708 | INFO | Uploaded 36GiB to GPU in 2.46559s (14.6009GiB/s) | 20260508 00:47:37.708 | INFO | Downloaded 34GiB to GPU in 2.49573s (13.6233GiB/s) | 20260508 00:47:42.846 | INFO | Analyzing chunk frames = [3918336,3918848) | 20260508 00:47:42.846 | INFO | MemInfo: { | 20260508 00:47:42.846 | INFO | "gpu" : " 17 GB", | 20260508 00:47:42.846 | INFO | "huge" : "", | 20260508 00:47:42.846 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:42.846 | INFO | "malloc" : " 14 GB", | 20260508 00:47:42.846 | INFO | "pinned" : " 29 GB" | 20260508 00:47:42.846 | INFO | } | 20260508 00:47:42.846 | INFO | Perf report for the last 5.13955 seconds: | 20260508 00:47:42.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:42.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260508 00:47:42.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:47:42.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:42.847 | INFO | Uploaded 36GiB to GPU in 2.53921s (14.1776GiB/s) | 20260508 00:47:42.847 | INFO | Downloaded 34GiB to GPU in 2.4269s (14.0096GiB/s) | 20260508 00:47:47.988 | INFO | Analyzing chunk frames = [3918848,3919360) | 20260508 00:47:47.988 | INFO | MemInfo: { | 20260508 00:47:47.988 | INFO | "gpu" : " 17 GB", | 20260508 00:47:47.988 | INFO | "huge" : "", | 20260508 00:47:47.988 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:47.988 | INFO | "malloc" : " 14 GB", | 20260508 00:47:47.988 | INFO | "pinned" : " 29 GB" | 20260508 00:47:47.988 | INFO | } | 20260508 00:47:47.988 | INFO | Perf report for the last 5.14202 seconds: | 20260508 00:47:47.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:47.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:47:47.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 00:47:47.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:47.989 | INFO | Uploaded 36GiB to GPU in 2.52398s (14.2632GiB/s) | 20260508 00:47:47.989 | INFO | Downloaded 34GiB to GPU in 2.44285s (13.9181GiB/s) | 20260508 00:47:53.130 | INFO | Analyzing chunk frames = [3919360,3919872) | 20260508 00:47:53.131 | INFO | MemInfo: { | 20260508 00:47:53.131 | INFO | "gpu" : " 17 GB", | 20260508 00:47:53.131 | INFO | "huge" : "", | 20260508 00:47:53.131 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:53.131 | INFO | "malloc" : " 14 GB", | 20260508 00:47:53.131 | INFO | "pinned" : " 29 GB" | 20260508 00:47:53.131 | INFO | } | 20260508 00:47:53.131 | INFO | Perf report for the last 5.14246 seconds: | 20260508 00:47:53.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:53.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:47:53.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 00:47:53.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:53.131 | INFO | Uploaded 36GiB to GPU in 2.55809s (14.073GiB/s) | 20260508 00:47:53.131 | INFO | Downloaded 34GiB to GPU in 2.45434s (13.853GiB/s) | 20260508 00:47:53.245 | INFO | Starting a round of Baz IO | 20260508 00:47:56.703 | INFO | Populating RTMetrics report | 20260508 00:47:56.704 | INFO | RTMetrics report populated | 20260508 00:47:56.704 | INFO | Opening tmp RTMetrics json file | 20260508 00:47:56.715 | INFO | Serializing RTMetrics to json | 20260508 00:47:56.722 | INFO | Serialization done, swapping tmp file now... | 20260508 00:47:56.723 | INFO | RTMetrics done | 20260508 00:47:56.798 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.552 seconds | 20260508 00:47:56.798 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.06% / 0.00% / 57.93% | 20260508 00:47:56.798 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.954684% / 0% / 4.76837e-05% | 20260508 00:47:56.798 | INFO | Baz IO overall: | 20260508 00:47:56.798 | INFO | BytesWritten : 1779677449055 | 20260508 00:47:56.798 | INFO | NumEvents : 861204925948 | 20260508 00:47:58.262 | INFO | Analyzing chunk frames = [3919872,3920384) | 20260508 00:47:58.262 | INFO | MemInfo: { | 20260508 00:47:58.262 | INFO | "gpu" : " 17 GB", | 20260508 00:47:58.262 | INFO | "huge" : "", | 20260508 00:47:58.262 | INFO | "huge_shared" : " 29 GB", | 20260508 00:47:58.262 | INFO | "malloc" : " 14 GB", | 20260508 00:47:58.262 | INFO | "pinned" : " 29 GB" | 20260508 00:47:58.262 | INFO | } | 20260508 00:47:58.262 | INFO | Perf report for the last 5.13088 seconds: | 20260508 00:47:58.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:47:58.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:58.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:58.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:58.262 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260508 00:47:58.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:47:58.262 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 00:47:58.262 | INFO | PRE_HQ : 67.9%, 1.8, 63.2ms, 0 | 20260508 00:47:58.262 | INFO | BAZWRITER : 68.9%, 4.0, 140.5ms, 0 | 20260508 00:47:58.262 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 00:47:58.262 | INFO | Uploaded 36GiB to GPU in 2.62067s (13.737GiB/s) | 20260508 00:47:58.262 | INFO | Downloaded 38GiB to GPU in 2.70681s (14.0387GiB/s) | 20260508 00:48:03.405 | INFO | Analyzing chunk frames = [3920384,3920896) | 20260508 00:48:03.405 | INFO | MemInfo: { | 20260508 00:48:03.405 | INFO | "gpu" : " 17 GB", | 20260508 00:48:03.405 | INFO | "huge" : "", | 20260508 00:48:03.405 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:03.405 | INFO | "malloc" : " 14 GB", | 20260508 00:48:03.405 | INFO | "pinned" : " 29 GB" | 20260508 00:48:03.405 | INFO | } | 20260508 00:48:03.405 | INFO | Perf report for the last 5.14296 seconds: | 20260508 00:48:03.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:03.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 00:48:03.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 00:48:03.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:03.405 | INFO | Uploaded 36GiB to GPU in 2.57663s (13.9717GiB/s) | 20260508 00:48:03.406 | INFO | Downloaded 34GiB to GPU in 2.49629s (13.6202GiB/s) | 20260508 00:48:08.544 | INFO | Analyzing chunk frames = [3920896,3921408) | 20260508 00:48:08.544 | INFO | MemInfo: { | 20260508 00:48:08.544 | INFO | "gpu" : " 17 GB", | 20260508 00:48:08.544 | INFO | "huge" : "", | 20260508 00:48:08.544 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:08.544 | INFO | "malloc" : " 14 GB", | 20260508 00:48:08.545 | INFO | "pinned" : " 29 GB" | 20260508 00:48:08.545 | INFO | } | 20260508 00:48:08.545 | INFO | Perf report for the last 5.13986 seconds: | 20260508 00:48:08.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:08.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:48:08.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:48:08.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:08.545 | INFO | Uploaded 36GiB to GPU in 2.52571s (14.2534GiB/s) | 20260508 00:48:08.545 | INFO | Downloaded 34GiB to GPU in 2.49569s (13.6235GiB/s) | 20260508 00:48:13.684 | INFO | Analyzing chunk frames = [3921408,3921920) | 20260508 00:48:13.684 | INFO | MemInfo: { | 20260508 00:48:13.684 | INFO | "gpu" : " 17 GB", | 20260508 00:48:13.684 | INFO | "huge" : "", | 20260508 00:48:13.684 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:13.684 | INFO | "malloc" : " 14 GB", | 20260508 00:48:13.684 | INFO | "pinned" : " 29 GB" | 20260508 00:48:13.684 | INFO | } | 20260508 00:48:13.684 | INFO | Perf report for the last 5.13955 seconds: | 20260508 00:48:13.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:13.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.684 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 00:48:13.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.684 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:48:13.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:13.685 | INFO | Uploaded 36GiB to GPU in 2.50837s (14.3519GiB/s) | 20260508 00:48:13.685 | INFO | Downloaded 34GiB to GPU in 2.48885s (13.6609GiB/s) | 20260508 00:48:18.827 | INFO | Analyzing chunk frames = [3921920,3922432) | 20260508 00:48:18.827 | INFO | MemInfo: { | 20260508 00:48:18.827 | INFO | "gpu" : " 17 GB", | 20260508 00:48:18.827 | INFO | "huge" : "", | 20260508 00:48:18.827 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:18.827 | INFO | "malloc" : " 14 GB", | 20260508 00:48:18.827 | INFO | "pinned" : " 29 GB" | 20260508 00:48:18.827 | INFO | } | 20260508 00:48:18.827 | INFO | Perf report for the last 5.14304 seconds: | 20260508 00:48:18.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:18.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.827 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 00:48:18.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.827 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:48:18.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:18.828 | INFO | Uploaded 36GiB to GPU in 2.30678s (15.6062GiB/s) | 20260508 00:48:18.828 | INFO | Downloaded 34GiB to GPU in 2.49706s (13.616GiB/s) | 20260508 00:48:23.962 | INFO | Analyzing chunk frames = [3922432,3922944) | 20260508 00:48:23.962 | INFO | MemInfo: { | 20260508 00:48:23.962 | INFO | "gpu" : " 17 GB", | 20260508 00:48:23.962 | INFO | "huge" : "", | 20260508 00:48:23.962 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:23.962 | INFO | "malloc" : " 14 GB", | 20260508 00:48:23.962 | INFO | "pinned" : " 29 GB" | 20260508 00:48:23.962 | INFO | } | 20260508 00:48:23.962 | INFO | Perf report for the last 5.13475 seconds: | 20260508 00:48:23.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:23.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.962 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:48:23.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.962 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:48:23.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:23.963 | INFO | Uploaded 36GiB to GPU in 2.37769s (15.1407GiB/s) | 20260508 00:48:23.963 | INFO | Downloaded 34GiB to GPU in 2.43713s (13.9508GiB/s) | 20260508 00:48:29.104 | INFO | Analyzing chunk frames = [3922944,3923456) | 20260508 00:48:29.104 | INFO | MemInfo: { | 20260508 00:48:29.104 | INFO | "gpu" : " 17 GB", | 20260508 00:48:29.104 | INFO | "huge" : "", | 20260508 00:48:29.104 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:29.104 | INFO | "malloc" : " 14 GB", | 20260508 00:48:29.104 | INFO | "pinned" : " 29 GB" | 20260508 00:48:29.104 | INFO | } | 20260508 00:48:29.104 | INFO | Perf report for the last 5.14172 seconds: | 20260508 00:48:29.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:29.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 00:48:29.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:48:29.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:29.104 | INFO | Uploaded 36GiB to GPU in 2.39712s (15.018GiB/s) | 20260508 00:48:29.104 | INFO | Downloaded 34GiB to GPU in 2.42888s (13.9982GiB/s) | 20260508 00:48:34.244 | INFO | Analyzing chunk frames = [3923456,3923968) | 20260508 00:48:34.244 | INFO | MemInfo: { | 20260508 00:48:34.244 | INFO | "gpu" : " 17 GB", | 20260508 00:48:34.244 | INFO | "huge" : "", | 20260508 00:48:34.244 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:34.244 | INFO | "malloc" : " 14 GB", | 20260508 00:48:34.244 | INFO | "pinned" : " 29 GB" | 20260508 00:48:34.244 | INFO | } | 20260508 00:48:34.244 | INFO | Perf report for the last 5.14065 seconds: | 20260508 00:48:34.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:34.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.244 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 00:48:34.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.245 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:48:34.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:34.245 | INFO | Uploaded 36GiB to GPU in 2.51588s (14.3091GiB/s) | 20260508 00:48:34.245 | INFO | Downloaded 34GiB to GPU in 2.46086s (13.8163GiB/s) | 20260508 00:48:34.245 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:48:34.245 | INFO | Total mem usage: 16.7 GiB | 20260508 00:48:34.245 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:48:34.245 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -10.1%) | 20260508 00:48:34.245 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:48:34.245 | INFO | High Water Breakdown: | 20260508 00:48:34.245 | INFO | 60.4% : 9.16 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:48:34.245 | INFO | 39.5% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:48:34.245 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:48:34.245 | INFO | ------------------------------------------ | 20260508 00:48:34.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:48:34.245 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:48:34.245 | INFO | Total mem usage: 26.5 GiB | 20260508 00:48:34.245 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:48:34.245 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:48:34.245 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:48:34.245 | INFO | High Water Breakdown: | 20260508 00:48:34.245 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:48:34.245 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:48:34.245 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:48:34.245 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:48:34.245 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:48:34.245 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:48:34.245 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:48:34.245 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:48:34.245 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:48:34.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:48:34.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:48:34.245 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:48:34.245 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:48:34.245 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:48:34.245 | INFO | ------------------------------------------ | 20260508 00:48:34.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:48:34.245 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:48:34.245 | INFO | Total mem usage: 27.8 GiB | 20260508 00:48:34.245 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:48:34.245 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:48:34.245 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:48:34.245 | INFO | High Water Breakdown: | 20260508 00:48:34.245 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:48:34.245 | INFO | ------------------------------------------ | 20260508 00:48:34.245 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:48:34.245 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:48:34.245 | INFO | Total mem usage: 15.7 GiB | 20260508 00:48:34.245 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:48:34.246 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:48:34.246 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:48:34.246 | INFO | High Water Breakdown: | 20260508 00:48:34.246 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:48:34.246 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:48:34.246 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:48:34.246 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:48:34.246 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:48:34.246 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:48:34.246 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:48:34.246 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:48:34.246 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:48:34.246 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:48:34.246 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:48:34.246 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:48:34.246 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:48:34.246 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:48:34.246 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:48:34.246 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:48:34.246 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:48:34.246 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:48:34.246 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:48:34.246 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:48:34.246 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:48:34.246 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:48:34.246 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:48:34.246 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:48:34.246 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:48:34.246 | INFO | ------------------------------------------ | 20260508 00:48:34.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:48:34.246 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:48:34.246 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:48:34.246 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:48:34.246 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.29e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.19e-02 / 5.05e-03 | 20260508 00:48:34.246 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.50e+00, 5.80e-01 / 1.68e-01 | 20260508 00:48:34.246 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.02e-01 / 1.06e-01 | 20260508 00:48:34.246 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.62e+01, 3.93e+00 / 1.45e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.56e+00 / 1.48e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.46e+00 / 2.66e+00, 4.29e-01 / 2.25e-01 | 20260508 00:48:34.246 | INFO | PerfReport: 3.38%, Metrics -- 1.10e+00 / 1.26e+00, 2.73e-01 / 1.23e-01 | 20260508 00:48:34.246 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.70e-01, 5.47e-02 / 2.81e-02 | 20260508 00:48:34.246 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:48:34.246 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:48:34.246 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:48:34.246 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 24.57%, Upload -- 2.62e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 30.70%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 2.29%, LockUpload -- 2.43e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 15.40%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 00:48:34.246 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.70e-02 / 0.00e+00 | 20260508 00:48:34.343 | INFO | Starting a round of Baz IO | 20260508 00:48:37.808 | INFO | Populating RTMetrics report | 20260508 00:48:37.808 | INFO | RTMetrics report populated | 20260508 00:48:37.808 | INFO | Opening tmp RTMetrics json file | 20260508 00:48:37.835 | INFO | Serializing RTMetrics to json | 20260508 00:48:37.837 | INFO | Serialization done, swapping tmp file now... | 20260508 00:48:37.841 | INFO | RTMetrics done | 20260508 00:48:37.919 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.576 seconds | 20260508 00:48:37.919 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.06% / 0.00% / 57.94% | 20260508 00:48:37.919 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.953265% / 3.97364e-06% / 3.97364e-05% | 20260508 00:48:37.919 | INFO | Baz IO overall: | 20260508 00:48:37.920 | INFO | BytesWritten : 1781482345697 | 20260508 00:48:37.920 | INFO | NumEvents : 862072802794 | 20260508 00:48:39.384 | INFO | Analyzing chunk frames = [3923968,3924480) | 20260508 00:48:39.384 | INFO | MemInfo: { | 20260508 00:48:39.384 | INFO | "gpu" : " 17 GB", | 20260508 00:48:39.384 | INFO | "huge" : "", | 20260508 00:48:39.384 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:39.384 | INFO | "malloc" : " 14 GB", | 20260508 00:48:39.384 | INFO | "pinned" : " 29 GB" | 20260508 00:48:39.384 | INFO | } | 20260508 00:48:39.384 | INFO | Perf report for the last 5.13995 seconds: | 20260508 00:48:39.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:39.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:39.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:39.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:39.385 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260508 00:48:39.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:39.385 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260508 00:48:39.385 | INFO | PRE_HQ : 65.5%, 1.7, 56.5ms, 0 | 20260508 00:48:39.385 | INFO | BAZWRITER : 69.3%, 4.0, 142.4ms, 0 | 20260508 00:48:39.385 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 00:48:39.385 | INFO | Uploaded 36GiB to GPU in 2.50841s (14.3517GiB/s) | 20260508 00:48:39.385 | INFO | Downloaded 38GiB to GPU in 2.69132s (14.1195GiB/s) | 20260508 00:48:44.515 | INFO | Analyzing chunk frames = [3924480,3924992) | 20260508 00:48:44.515 | INFO | MemInfo: { | 20260508 00:48:44.515 | INFO | "gpu" : " 17 GB", | 20260508 00:48:44.515 | INFO | "huge" : "", | 20260508 00:48:44.515 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:44.515 | INFO | "malloc" : " 14 GB", | 20260508 00:48:44.515 | INFO | "pinned" : " 29 GB" | 20260508 00:48:44.515 | INFO | } | 20260508 00:48:44.515 | INFO | Perf report for the last 5.13101 seconds: | 20260508 00:48:44.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:44.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.515 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 00:48:44.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.516 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:48:44.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:44.516 | INFO | Uploaded 36GiB to GPU in 2.50751s (14.3569GiB/s) | 20260508 00:48:44.516 | INFO | Downloaded 34GiB to GPU in 2.43728s (13.95GiB/s) | 20260508 00:48:49.658 | INFO | Analyzing chunk frames = [3924992,3925504) | 20260508 00:48:49.658 | INFO | MemInfo: { | 20260508 00:48:49.658 | INFO | "gpu" : " 17 GB", | 20260508 00:48:49.658 | INFO | "huge" : "", | 20260508 00:48:49.658 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:49.659 | INFO | "malloc" : " 14 GB", | 20260508 00:48:49.659 | INFO | "pinned" : " 29 GB" | 20260508 00:48:49.659 | INFO | } | 20260508 00:48:49.659 | INFO | Perf report for the last 5.14316 seconds: | 20260508 00:48:49.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:49.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:48:49.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:48:49.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:49.659 | INFO | Uploaded 36GiB to GPU in 2.46776s (14.5881GiB/s) | 20260508 00:48:49.659 | INFO | Downloaded 34GiB to GPU in 2.49245s (13.6412GiB/s) | 20260508 00:48:54.796 | INFO | Analyzing chunk frames = [3925504,3926016) | 20260508 00:48:54.796 | INFO | MemInfo: { | 20260508 00:48:54.796 | INFO | "gpu" : " 17 GB", | 20260508 00:48:54.796 | INFO | "huge" : "", | 20260508 00:48:54.796 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:54.796 | INFO | "malloc" : " 14 GB", | 20260508 00:48:54.796 | INFO | "pinned" : " 29 GB" | 20260508 00:48:54.796 | INFO | } | 20260508 00:48:54.796 | INFO | Perf report for the last 5.13726 seconds: | 20260508 00:48:54.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:54.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:48:54.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:48:54.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:54.796 | INFO | Uploaded 36GiB to GPU in 2.44943s (14.6973GiB/s) | 20260508 00:48:54.796 | INFO | Downloaded 34GiB to GPU in 2.491s (13.6491GiB/s) | 20260508 00:48:59.942 | INFO | Analyzing chunk frames = [3926016,3926528) | 20260508 00:48:59.942 | INFO | MemInfo: { | 20260508 00:48:59.942 | INFO | "gpu" : " 17 GB", | 20260508 00:48:59.942 | INFO | "huge" : "", | 20260508 00:48:59.942 | INFO | "huge_shared" : " 29 GB", | 20260508 00:48:59.943 | INFO | "malloc" : " 14 GB", | 20260508 00:48:59.943 | INFO | "pinned" : " 29 GB" | 20260508 00:48:59.943 | INFO | } | 20260508 00:48:59.943 | INFO | Perf report for the last 5.14684 seconds: | 20260508 00:48:59.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:48:59.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 00:48:59.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 00:48:59.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:48:59.943 | INFO | Uploaded 36GiB to GPU in 2.34615s (15.3443GiB/s) | 20260508 00:48:59.943 | INFO | Downloaded 34GiB to GPU in 2.50983s (13.5467GiB/s) | 20260508 00:49:05.078 | INFO | Analyzing chunk frames = [3926528,3927040) | 20260508 00:49:05.078 | INFO | MemInfo: { | 20260508 00:49:05.078 | INFO | "gpu" : " 17 GB", | 20260508 00:49:05.078 | INFO | "huge" : "", | 20260508 00:49:05.079 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:05.079 | INFO | "malloc" : " 14 GB", | 20260508 00:49:05.079 | INFO | "pinned" : " 29 GB" | 20260508 00:49:05.079 | INFO | } | 20260508 00:49:05.079 | INFO | Perf report for the last 5.13602 seconds: | 20260508 00:49:05.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:05.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:49:05.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 00:49:05.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:05.079 | INFO | Uploaded 36GiB to GPU in 2.37799s (15.1388GiB/s) | 20260508 00:49:05.079 | INFO | Downloaded 34GiB to GPU in 2.44168s (13.9249GiB/s) | 20260508 00:49:10.215 | INFO | Analyzing chunk frames = [3927040,3927552) | 20260508 00:49:10.215 | INFO | MemInfo: { | 20260508 00:49:10.215 | INFO | "gpu" : " 17 GB", | 20260508 00:49:10.215 | INFO | "huge" : "", | 20260508 00:49:10.215 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:10.216 | INFO | "malloc" : " 14 GB", | 20260508 00:49:10.216 | INFO | "pinned" : " 29 GB" | 20260508 00:49:10.216 | INFO | } | 20260508 00:49:10.216 | INFO | Perf report for the last 5.13713 seconds: | 20260508 00:49:10.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:10.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 00:49:10.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | PRE_HQ : 35.2%, 1.0, 18.2ms, 0 | 20260508 00:49:10.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:10.216 | INFO | Uploaded 36GiB to GPU in 2.39364s (15.0399GiB/s) | 20260508 00:49:10.216 | INFO | Downloaded 34GiB to GPU in 2.43566s (13.9593GiB/s) | 20260508 00:49:15.358 | INFO | Analyzing chunk frames = [3927552,3928064) | 20260508 00:49:15.358 | INFO | MemInfo: { | 20260508 00:49:15.358 | INFO | "gpu" : " 17 GB", | 20260508 00:49:15.358 | INFO | "huge" : "", | 20260508 00:49:15.358 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:15.358 | INFO | "malloc" : " 14 GB", | 20260508 00:49:15.358 | INFO | "pinned" : " 29 GB" | 20260508 00:49:15.358 | INFO | } | 20260508 00:49:15.358 | INFO | Perf report for the last 5.14221 seconds: | 20260508 00:49:15.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:15.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.358 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:49:15.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.358 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 00:49:15.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:15.359 | INFO | Uploaded 36GiB to GPU in 2.54849s (14.126GiB/s) | 20260508 00:49:15.359 | INFO | Downloaded 34GiB to GPU in 2.47256s (13.7509GiB/s) | 20260508 00:49:15.472 | INFO | Starting a round of Baz IO | 20260508 00:49:18.944 | INFO | Populating RTMetrics report | 20260508 00:49:18.945 | INFO | RTMetrics report populated | 20260508 00:49:18.945 | INFO | Opening tmp RTMetrics json file | 20260508 00:49:18.965 | INFO | Serializing RTMetrics to json | 20260508 00:49:18.974 | INFO | Serialization done, swapping tmp file now... | 20260508 00:49:18.988 | INFO | RTMetrics done | 20260508 00:49:19.035 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.562 seconds | 20260508 00:49:19.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.05% / 0.00% / 57.94% | 20260508 00:49:19.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.950106% / 0% / 5.5631e-05% | 20260508 00:49:19.035 | INFO | Baz IO overall: | 20260508 00:49:19.035 | INFO | BytesWritten : 1783286706969 | 20260508 00:49:19.035 | INFO | NumEvents : 862940386261 | 20260508 00:49:20.494 | INFO | Analyzing chunk frames = [3928064,3928576) | 20260508 00:49:20.494 | INFO | MemInfo: { | 20260508 00:49:20.494 | INFO | "gpu" : " 17 GB", | 20260508 00:49:20.494 | INFO | "huge" : "", | 20260508 00:49:20.494 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:20.494 | INFO | "malloc" : " 14 GB", | 20260508 00:49:20.494 | INFO | "pinned" : " 29 GB" | 20260508 00:49:20.494 | INFO | } | 20260508 00:49:20.494 | INFO | Perf report for the last 5.13584 seconds: | 20260508 00:49:20.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:20.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:20.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:20.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:20.494 | INFO | ANALYSIS : 68.9%, 2.9, 104.6ms, 0 | 20260508 00:49:20.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:20.494 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260508 00:49:20.494 | INFO | PRE_HQ : 67.8%, 1.8, 62.7ms, 0 | 20260508 00:49:20.494 | INFO | BAZWRITER : 68.8%, 4.3, 152.0ms, 0 | 20260508 00:49:20.494 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 00:49:20.494 | INFO | Uploaded 36GiB to GPU in 2.60614s (13.8135GiB/s) | 20260508 00:49:20.494 | INFO | Downloaded 38GiB to GPU in 2.71156s (14.0141GiB/s) | 20260508 00:49:25.633 | INFO | Analyzing chunk frames = [3928576,3929088) | 20260508 00:49:25.633 | INFO | MemInfo: { | 20260508 00:49:25.633 | INFO | "gpu" : " 17 GB", | 20260508 00:49:25.634 | INFO | "huge" : "", | 20260508 00:49:25.634 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:25.634 | INFO | "malloc" : " 14 GB", | 20260508 00:49:25.634 | INFO | "pinned" : " 29 GB" | 20260508 00:49:25.634 | INFO | } | 20260508 00:49:25.634 | INFO | Perf report for the last 5.13972 seconds: | 20260508 00:49:25.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:25.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 00:49:25.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:49:25.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:25.634 | INFO | Uploaded 36GiB to GPU in 2.52508s (14.257GiB/s) | 20260508 00:49:25.634 | INFO | Downloaded 34GiB to GPU in 2.41969s (14.0514GiB/s) | 20260508 00:49:30.772 | INFO | Analyzing chunk frames = [3929088,3929600) | 20260508 00:49:30.772 | INFO | MemInfo: { | 20260508 00:49:30.772 | INFO | "gpu" : " 17 GB", | 20260508 00:49:30.772 | INFO | "huge" : "", | 20260508 00:49:30.772 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:30.772 | INFO | "malloc" : " 14 GB", | 20260508 00:49:30.772 | INFO | "pinned" : " 29 GB" | 20260508 00:49:30.772 | INFO | } | 20260508 00:49:30.772 | INFO | Perf report for the last 5.13874 seconds: | 20260508 00:49:30.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:30.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.773 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 00:49:30.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.773 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:49:30.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:30.773 | INFO | Uploaded 36GiB to GPU in 2.47501s (14.5454GiB/s) | 20260508 00:49:30.773 | INFO | Downloaded 34GiB to GPU in 2.49052s (13.6517GiB/s) | 20260508 00:49:35.916 | INFO | Analyzing chunk frames = [3929600,3930112) | 20260508 00:49:35.916 | INFO | MemInfo: { | 20260508 00:49:35.916 | INFO | "gpu" : " 17 GB", | 20260508 00:49:35.916 | INFO | "huge" : "", | 20260508 00:49:35.916 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:35.916 | INFO | "malloc" : " 14 GB", | 20260508 00:49:35.916 | INFO | "pinned" : " 29 GB" | 20260508 00:49:35.916 | INFO | } | 20260508 00:49:35.916 | INFO | Perf report for the last 5.14339 seconds: | 20260508 00:49:35.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:35.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 00:49:35.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:49:35.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:35.916 | INFO | Uploaded 36GiB to GPU in 2.52335s (14.2667GiB/s) | 20260508 00:49:35.916 | INFO | Downloaded 34GiB to GPU in 2.48586s (13.6773GiB/s) | 20260508 00:49:41.058 | INFO | Analyzing chunk frames = [3930112,3930624) | 20260508 00:49:41.058 | INFO | MemInfo: { | 20260508 00:49:41.058 | INFO | "gpu" : " 17 GB", | 20260508 00:49:41.058 | INFO | "huge" : "", | 20260508 00:49:41.058 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:41.058 | INFO | "malloc" : " 14 GB", | 20260508 00:49:41.058 | INFO | "pinned" : " 29 GB" | 20260508 00:49:41.058 | INFO | } | 20260508 00:49:41.058 | INFO | Perf report for the last 5.14227 seconds: | 20260508 00:49:41.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:41.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:49:41.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:49:41.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:41.058 | INFO | Uploaded 36GiB to GPU in 2.38388s (15.1015GiB/s) | 20260508 00:49:41.059 | INFO | Downloaded 34GiB to GPU in 2.49045s (13.6522GiB/s) | 20260508 00:49:46.195 | INFO | Analyzing chunk frames = [3930624,3931136) | 20260508 00:49:46.195 | INFO | MemInfo: { | 20260508 00:49:46.195 | INFO | "gpu" : " 17 GB", | 20260508 00:49:46.195 | INFO | "huge" : "", | 20260508 00:49:46.195 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:46.195 | INFO | "malloc" : " 14 GB", | 20260508 00:49:46.196 | INFO | "pinned" : " 29 GB" | 20260508 00:49:46.196 | INFO | } | 20260508 00:49:46.196 | INFO | Perf report for the last 5.13752 seconds: | 20260508 00:49:46.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:46.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 00:49:46.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:49:46.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:46.196 | INFO | Uploaded 36GiB to GPU in 2.41683s (14.8956GiB/s) | 20260508 00:49:46.196 | INFO | Downloaded 34GiB to GPU in 2.40656s (14.128GiB/s) | 20260508 00:49:51.328 | INFO | Analyzing chunk frames = [3931136,3931648) | 20260508 00:49:51.328 | INFO | MemInfo: { | 20260508 00:49:51.328 | INFO | "gpu" : " 17 GB", | 20260508 00:49:51.328 | INFO | "huge" : "", | 20260508 00:49:51.328 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:51.328 | INFO | "malloc" : " 14 GB", | 20260508 00:49:51.328 | INFO | "pinned" : " 29 GB" | 20260508 00:49:51.328 | INFO | } | 20260508 00:49:51.329 | INFO | Perf report for the last 5.13295 seconds: | 20260508 00:49:51.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:51.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 00:49:51.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260508 00:49:51.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:51.329 | INFO | Uploaded 36GiB to GPU in 2.45414s (14.6691GiB/s) | 20260508 00:49:51.329 | INFO | Downloaded 34GiB to GPU in 2.41629s (14.0712GiB/s) | 20260508 00:49:56.468 | INFO | Analyzing chunk frames = [3931648,3932160) | 20260508 00:49:56.468 | INFO | MemInfo: { | 20260508 00:49:56.468 | INFO | "gpu" : " 17 GB", | 20260508 00:49:56.468 | INFO | "huge" : "", | 20260508 00:49:56.468 | INFO | "huge_shared" : " 29 GB", | 20260508 00:49:56.468 | INFO | "malloc" : " 14 GB", | 20260508 00:49:56.468 | INFO | "pinned" : " 29 GB" | 20260508 00:49:56.468 | INFO | } | 20260508 00:49:56.469 | INFO | Perf report for the last 5.13985 seconds: | 20260508 00:49:56.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:49:56.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 00:49:56.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 00:49:56.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:49:56.469 | INFO | Uploaded 36GiB to GPU in 2.56918s (14.0123GiB/s) | 20260508 00:49:56.469 | INFO | Downloaded 34GiB to GPU in 2.45953s (13.8238GiB/s) | 20260508 00:49:56.469 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:49:56.469 | INFO | Total mem usage: 16.7 GiB | 20260508 00:49:56.469 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:49:56.469 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.9%) | 20260508 00:49:56.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:49:56.469 | INFO | High Water Breakdown: | 20260508 00:49:56.469 | INFO | 60.1% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:49:56.469 | INFO | 39.8% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:49:56.469 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:49:56.469 | INFO | ------------------------------------------ | 20260508 00:49:56.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:49:56.469 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:49:56.469 | INFO | Total mem usage: 26.5 GiB | 20260508 00:49:56.469 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:49:56.469 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:49:56.469 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:49:56.469 | INFO | High Water Breakdown: | 20260508 00:49:56.469 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:49:56.469 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:49:56.469 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:49:56.469 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:49:56.470 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:49:56.470 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:49:56.470 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:49:56.470 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:49:56.470 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:49:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:49:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:49:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:49:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:49:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:49:56.470 | INFO | ------------------------------------------ | 20260508 00:49:56.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:49:56.470 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:49:56.470 | INFO | Total mem usage: 27.8 GiB | 20260508 00:49:56.470 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:49:56.470 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:49:56.470 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:49:56.470 | INFO | High Water Breakdown: | 20260508 00:49:56.470 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:49:56.470 | INFO | ------------------------------------------ | 20260508 00:49:56.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:49:56.470 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:49:56.470 | INFO | Total mem usage: 15.7 GiB | 20260508 00:49:56.470 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:49:56.470 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:49:56.470 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:49:56.470 | INFO | High Water Breakdown: | 20260508 00:49:56.470 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:49:56.470 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:49:56.470 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:49:56.470 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:49:56.470 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:49:56.470 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:49:56.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:49:56.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:49:56.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:49:56.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:49:56.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:49:56.470 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:49:56.470 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:49:56.470 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:49:56.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:49:56.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:49:56.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:49:56.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:49:56.470 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:49:56.470 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:49:56.470 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:49:56.470 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:49:56.470 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:49:56.470 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:49:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:49:56.470 | INFO | ------------------------------------------ | 20260508 00:49:56.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:49:56.470 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:49:56.470 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:49:56.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:49:56.470 | INFO | PerfReport: Full Execution -- 3.29e+01 / 7.62e+01, 1.43e+01 / 1.60e+00 | 20260508 00:49:56.470 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.03e-01, 1.20e-02 / 7.48e-03 | 20260508 00:49:56.470 | INFO | PerfReport: 12.18%, Baseline -- 4.01e+00 / 4.45e+00, 5.92e-01 / 2.23e-01 | 20260508 00:49:56.470 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.11e-01 / 9.73e-02 | 20260508 00:49:56.471 | INFO | PerfReport: 16.03%, DME -- 4.22e+01 / 4.60e+01, 4.05e+00 / 1.63e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.70e+00 / 1.53e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.27e-01 / 2.45e-01 | 20260508 00:49:56.471 | INFO | PerfReport: 3.35%, Metrics -- 1.10e+00 / 1.28e+00, 2.71e-01 / 1.39e-01 | 20260508 00:49:56.471 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.42e-01 / 3.79e-01, 5.48e-02 / 2.86e-02 | 20260508 00:49:56.471 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:49:56.471 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:49:56.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:49:56.471 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 24.20%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 24.38%, Download -- 2.59e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 30.90%, Compute -- 3.29e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 2.02%, LockUpload -- 2.15e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 2.80%, LockDownload -- 2.98e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 15.68%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 00:49:56.471 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260508 00:49:56.579 | INFO | Starting a round of Baz IO | 20260508 00:50:00.034 | INFO | Populating RTMetrics report | 20260508 00:50:00.034 | INFO | RTMetrics report populated | 20260508 00:50:00.034 | INFO | Opening tmp RTMetrics json file | 20260508 00:50:00.057 | INFO | Serializing RTMetrics to json | 20260508 00:50:00.059 | INFO | Serialization done, swapping tmp file now... | 20260508 00:50:00.060 | INFO | RTMetrics done | 20260508 00:50:00.144 | INFO | Finished a round of Baz IO: Wrote 1.682 GiB in 3.564 seconds | 20260508 00:50:00.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.05% / 0.00% / 57.95% | 20260508 00:50:00.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.953074% / 7.94729e-06% / 6.75519e-05% | 20260508 00:50:00.144 | INFO | Baz IO overall: | 20260508 00:50:00.144 | INFO | BytesWritten : 1785092918280 | 20260508 00:50:00.144 | INFO | NumEvents : 863809306241 | 20260508 00:50:01.606 | INFO | Analyzing chunk frames = [3932160,3932672) | 20260508 00:50:01.606 | INFO | MemInfo: { | 20260508 00:50:01.606 | INFO | "gpu" : " 17 GB", | 20260508 00:50:01.606 | INFO | "huge" : "", | 20260508 00:50:01.606 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:01.606 | INFO | "malloc" : " 14 GB", | 20260508 00:50:01.606 | INFO | "pinned" : " 29 GB" | 20260508 00:50:01.606 | INFO | } | 20260508 00:50:01.607 | INFO | Perf report for the last 5.13799 seconds: | 20260508 00:50:01.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:01.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:01.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:01.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:01.607 | INFO | ANALYSIS : 68.7%, 2.9, 104.2ms, 0 | 20260508 00:50:01.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:01.607 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260508 00:50:01.607 | INFO | PRE_HQ : 67.5%, 1.8, 62.5ms, 0 | 20260508 00:50:01.607 | INFO | BAZWRITER : 68.9%, 3.7, 130.6ms, 0 | 20260508 00:50:01.607 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 00:50:01.607 | INFO | Uploaded 36GiB to GPU in 2.60222s (13.8343GiB/s) | 20260508 00:50:01.607 | INFO | Downloaded 38GiB to GPU in 2.70403s (14.0531GiB/s) | 20260508 00:50:06.750 | INFO | Analyzing chunk frames = [3932672,3933184) | 20260508 00:50:06.750 | INFO | MemInfo: { | 20260508 00:50:06.750 | INFO | "gpu" : " 17 GB", | 20260508 00:50:06.750 | INFO | "huge" : "", | 20260508 00:50:06.750 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:06.750 | INFO | "malloc" : " 14 GB", | 20260508 00:50:06.750 | INFO | "pinned" : " 29 GB" | 20260508 00:50:06.750 | INFO | } | 20260508 00:50:06.750 | INFO | Perf report for the last 5.14355 seconds: | 20260508 00:50:06.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:06.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 00:50:06.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:50:06.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:06.750 | INFO | Uploaded 36GiB to GPU in 2.59161s (13.891GiB/s) | 20260508 00:50:06.751 | INFO | Downloaded 34GiB to GPU in 2.48129s (13.7025GiB/s) | 20260508 00:50:11.888 | INFO | Analyzing chunk frames = [3933184,3933696) | 20260508 00:50:11.888 | INFO | MemInfo: { | 20260508 00:50:11.888 | INFO | "gpu" : " 17 GB", | 20260508 00:50:11.888 | INFO | "huge" : "", | 20260508 00:50:11.888 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:11.888 | INFO | "malloc" : " 14 GB", | 20260508 00:50:11.888 | INFO | "pinned" : " 29 GB" | 20260508 00:50:11.888 | INFO | } | 20260508 00:50:11.889 | INFO | Perf report for the last 5.13846 seconds: | 20260508 00:50:11.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:11.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 00:50:11.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:50:11.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:11.889 | INFO | Uploaded 36GiB to GPU in 2.55529s (14.0884GiB/s) | 20260508 00:50:11.889 | INFO | Downloaded 34GiB to GPU in 2.4708s (13.7607GiB/s) | 20260508 00:50:17.027 | INFO | Analyzing chunk frames = [3933696,3934208) | 20260508 00:50:17.027 | INFO | MemInfo: { | 20260508 00:50:17.027 | INFO | "gpu" : " 17 GB", | 20260508 00:50:17.027 | INFO | "huge" : "", | 20260508 00:50:17.027 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:17.027 | INFO | "malloc" : " 14 GB", | 20260508 00:50:17.027 | INFO | "pinned" : " 29 GB" | 20260508 00:50:17.027 | INFO | } | 20260508 00:50:17.027 | INFO | Perf report for the last 5.13865 seconds: | 20260508 00:50:17.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:17.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.027 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260508 00:50:17.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.027 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:50:17.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:17.028 | INFO | Uploaded 36GiB to GPU in 2.57035s (14.0059GiB/s) | 20260508 00:50:17.028 | INFO | Downloaded 34GiB to GPU in 2.48798s (13.6657GiB/s) | 20260508 00:50:22.168 | INFO | Analyzing chunk frames = [3934208,3934720) | 20260508 00:50:22.168 | INFO | MemInfo: { | 20260508 00:50:22.168 | INFO | "gpu" : " 17 GB", | 20260508 00:50:22.168 | INFO | "huge" : "", | 20260508 00:50:22.168 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:22.168 | INFO | "malloc" : " 14 GB", | 20260508 00:50:22.168 | INFO | "pinned" : " 29 GB" | 20260508 00:50:22.168 | INFO | } | 20260508 00:50:22.168 | INFO | Perf report for the last 5.1408 seconds: | 20260508 00:50:22.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:22.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:50:22.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:50:22.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:22.168 | INFO | Uploaded 36GiB to GPU in 2.36949s (15.1931GiB/s) | 20260508 00:50:22.169 | INFO | Downloaded 34GiB to GPU in 2.47721s (13.7251GiB/s) | 20260508 00:50:27.307 | INFO | Analyzing chunk frames = [3934720,3935232) | 20260508 00:50:27.307 | INFO | MemInfo: { | 20260508 00:50:27.307 | INFO | "gpu" : " 17 GB", | 20260508 00:50:27.307 | INFO | "huge" : "", | 20260508 00:50:27.307 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:27.307 | INFO | "malloc" : " 14 GB", | 20260508 00:50:27.307 | INFO | "pinned" : " 29 GB" | 20260508 00:50:27.307 | INFO | } | 20260508 00:50:27.307 | INFO | Perf report for the last 5.13913 seconds: | 20260508 00:50:27.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:27.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:50:27.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:50:27.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:27.307 | INFO | Uploaded 36GiB to GPU in 2.39626s (15.0234GiB/s) | 20260508 00:50:27.308 | INFO | Downloaded 34GiB to GPU in 2.4011s (14.1602GiB/s) | 20260508 00:50:32.445 | INFO | Analyzing chunk frames = [3935232,3935744) | 20260508 00:50:32.445 | INFO | MemInfo: { | 20260508 00:50:32.445 | INFO | "gpu" : " 17 GB", | 20260508 00:50:32.445 | INFO | "huge" : "", | 20260508 00:50:32.445 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:32.445 | INFO | "malloc" : " 14 GB", | 20260508 00:50:32.445 | INFO | "pinned" : " 29 GB" | 20260508 00:50:32.445 | INFO | } | 20260508 00:50:32.445 | INFO | Perf report for the last 5.13819 seconds: | 20260508 00:50:32.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:32.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.445 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 00:50:32.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.445 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 00:50:32.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:32.446 | INFO | Uploaded 36GiB to GPU in 2.56499s (14.0351GiB/s) | 20260508 00:50:32.446 | INFO | Downloaded 34GiB to GPU in 2.4758s (13.7329GiB/s) | 20260508 00:50:37.583 | INFO | Analyzing chunk frames = [3935744,3936256) | 20260508 00:50:37.583 | INFO | MemInfo: { | 20260508 00:50:37.583 | INFO | "gpu" : " 17 GB", | 20260508 00:50:37.583 | INFO | "huge" : "", | 20260508 00:50:37.583 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:37.583 | INFO | "malloc" : " 14 GB", | 20260508 00:50:37.583 | INFO | "pinned" : " 29 GB" | 20260508 00:50:37.583 | INFO | } | 20260508 00:50:37.584 | INFO | Perf report for the last 5.13814 seconds: | 20260508 00:50:37.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:37.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 00:50:37.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:50:37.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:37.584 | INFO | Uploaded 36GiB to GPU in 2.51906s (14.2911GiB/s) | 20260508 00:50:37.584 | INFO | Downloaded 34GiB to GPU in 2.44428s (13.91GiB/s) | 20260508 00:50:37.681 | INFO | Starting a round of Baz IO | 20260508 00:50:41.136 | INFO | Populating RTMetrics report | 20260508 00:50:41.136 | INFO | RTMetrics report populated | 20260508 00:50:41.136 | INFO | Opening tmp RTMetrics json file | 20260508 00:50:41.148 | INFO | Serializing RTMetrics to json | 20260508 00:50:41.154 | INFO | Serialization done, swapping tmp file now... | 20260508 00:50:41.154 | INFO | RTMetrics done | 20260508 00:50:41.229 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.548 seconds | 20260508 00:50:41.229 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.04% / 0.00% / 57.95% | 20260508 00:50:41.229 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.951807% / 0% / 7.15256e-05% | 20260508 00:50:41.229 | INFO | Baz IO overall: | 20260508 00:50:41.229 | INFO | BytesWritten : 1786898169719 | 20260508 00:50:41.229 | INFO | NumEvents : 864677594788 | 20260508 00:50:42.719 | INFO | Analyzing chunk frames = [3936256,3936768) | 20260508 00:50:42.719 | INFO | MemInfo: { | 20260508 00:50:42.719 | INFO | "gpu" : " 17 GB", | 20260508 00:50:42.719 | INFO | "huge" : "", | 20260508 00:50:42.719 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:42.719 | INFO | "malloc" : " 14 GB", | 20260508 00:50:42.719 | INFO | "pinned" : " 29 GB" | 20260508 00:50:42.719 | INFO | } | 20260508 00:50:42.719 | INFO | Perf report for the last 5.13581 seconds: | 20260508 00:50:42.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:42.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:42.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:42.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:42.719 | INFO | ANALYSIS : 68.4%, 3.0, 106.8ms, 0 | 20260508 00:50:42.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:42.719 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 00:50:42.719 | INFO | PRE_HQ : 65.8%, 1.7, 56.5ms, 0 | 20260508 00:50:42.720 | INFO | BAZWRITER : 68.7%, 3.5, 125.2ms, 0 | 20260508 00:50:42.720 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 00:50:42.720 | INFO | Uploaded 36GiB to GPU in 2.54035s (14.1713GiB/s) | 20260508 00:50:42.720 | INFO | Downloaded 38GiB to GPU in 2.68674s (14.1436GiB/s) | 20260508 00:50:47.864 | INFO | Analyzing chunk frames = [3936768,3937280) | 20260508 00:50:47.864 | INFO | MemInfo: { | 20260508 00:50:47.864 | INFO | "gpu" : " 17 GB", | 20260508 00:50:47.864 | INFO | "huge" : "", | 20260508 00:50:47.864 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:47.864 | INFO | "malloc" : " 14 GB", | 20260508 00:50:47.864 | INFO | "pinned" : " 29 GB" | 20260508 00:50:47.865 | INFO | } | 20260508 00:50:47.865 | INFO | Perf report for the last 5.14559 seconds: | 20260508 00:50:47.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:47.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 00:50:47.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:50:47.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:47.865 | INFO | Uploaded 36GiB to GPU in 2.56752s (14.0213GiB/s) | 20260508 00:50:47.865 | INFO | Downloaded 34GiB to GPU in 2.49489s (13.6279GiB/s) | 20260508 00:50:53.007 | INFO | Analyzing chunk frames = [3937280,3937792) | 20260508 00:50:53.007 | INFO | MemInfo: { | 20260508 00:50:53.007 | INFO | "gpu" : " 17 GB", | 20260508 00:50:53.007 | INFO | "huge" : "", | 20260508 00:50:53.007 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:53.007 | INFO | "malloc" : " 14 GB", | 20260508 00:50:53.007 | INFO | "pinned" : " 29 GB" | 20260508 00:50:53.007 | INFO | } | 20260508 00:50:53.008 | INFO | Perf report for the last 5.14257 seconds: | 20260508 00:50:53.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:53.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 00:50:53.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:50:53.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:53.008 | INFO | Uploaded 36GiB to GPU in 2.48526s (14.4854GiB/s) | 20260508 00:50:53.008 | INFO | Downloaded 34GiB to GPU in 2.48477s (13.6833GiB/s) | 20260508 00:50:58.144 | INFO | Analyzing chunk frames = [3937792,3938304) | 20260508 00:50:58.144 | INFO | MemInfo: { | 20260508 00:50:58.144 | INFO | "gpu" : " 17 GB", | 20260508 00:50:58.144 | INFO | "huge" : "", | 20260508 00:50:58.144 | INFO | "huge_shared" : " 29 GB", | 20260508 00:50:58.145 | INFO | "malloc" : " 14 GB", | 20260508 00:50:58.145 | INFO | "pinned" : " 29 GB" | 20260508 00:50:58.145 | INFO | } | 20260508 00:50:58.145 | INFO | Perf report for the last 5.13719 seconds: | 20260508 00:50:58.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:50:58.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 00:50:58.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:50:58.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:50:58.145 | INFO | Uploaded 36GiB to GPU in 2.51739s (14.3005GiB/s) | 20260508 00:50:58.145 | INFO | Downloaded 34GiB to GPU in 2.4858s (13.6777GiB/s) | 20260508 00:51:03.279 | INFO | Analyzing chunk frames = [3938304,3938816) | 20260508 00:51:03.279 | INFO | MemInfo: { | 20260508 00:51:03.279 | INFO | "gpu" : " 17 GB", | 20260508 00:51:03.279 | INFO | "huge" : "", | 20260508 00:51:03.279 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:03.279 | INFO | "malloc" : " 14 GB", | 20260508 00:51:03.279 | INFO | "pinned" : " 29 GB" | 20260508 00:51:03.279 | INFO | } | 20260508 00:51:03.279 | INFO | Perf report for the last 5.13454 seconds: | 20260508 00:51:03.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:03.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.279 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 00:51:03.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.279 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:51:03.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:03.280 | INFO | Uploaded 36GiB to GPU in 2.32316s (15.4961GiB/s) | 20260508 00:51:03.280 | INFO | Downloaded 34GiB to GPU in 2.5012s (13.5935GiB/s) | 20260508 00:51:08.420 | INFO | Analyzing chunk frames = [3938816,3939328) | 20260508 00:51:08.420 | INFO | MemInfo: { | 20260508 00:51:08.420 | INFO | "gpu" : " 17 GB", | 20260508 00:51:08.420 | INFO | "huge" : "", | 20260508 00:51:08.420 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:08.420 | INFO | "malloc" : " 14 GB", | 20260508 00:51:08.420 | INFO | "pinned" : " 29 GB" | 20260508 00:51:08.420 | INFO | } | 20260508 00:51:08.420 | INFO | Perf report for the last 5.1407 seconds: | 20260508 00:51:08.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:08.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 00:51:08.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260508 00:51:08.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:08.420 | INFO | Uploaded 36GiB to GPU in 2.42447s (14.8486GiB/s) | 20260508 00:51:08.420 | INFO | Downloaded 34GiB to GPU in 2.39928s (14.1709GiB/s) | 20260508 00:51:13.561 | INFO | Analyzing chunk frames = [3939328,3939840) | 20260508 00:51:13.561 | INFO | MemInfo: { | 20260508 00:51:13.561 | INFO | "gpu" : " 17 GB", | 20260508 00:51:13.561 | INFO | "huge" : "", | 20260508 00:51:13.561 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:13.562 | INFO | "malloc" : " 14 GB", | 20260508 00:51:13.562 | INFO | "pinned" : " 29 GB" | 20260508 00:51:13.562 | INFO | } | 20260508 00:51:13.562 | INFO | Perf report for the last 5.14159 seconds: | 20260508 00:51:13.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:13.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:51:13.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:51:13.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:13.562 | INFO | Uploaded 36GiB to GPU in 2.5375s (14.1872GiB/s) | 20260508 00:51:13.562 | INFO | Downloaded 34GiB to GPU in 2.44625s (13.8988GiB/s) | 20260508 00:51:18.701 | INFO | Analyzing chunk frames = [3939840,3940352) | 20260508 00:51:18.701 | INFO | MemInfo: { | 20260508 00:51:18.701 | INFO | "gpu" : " 17 GB", | 20260508 00:51:18.701 | INFO | "huge" : "", | 20260508 00:51:18.701 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:18.701 | INFO | "malloc" : " 14 GB", | 20260508 00:51:18.701 | INFO | "pinned" : " 29 GB" | 20260508 00:51:18.701 | INFO | } | 20260508 00:51:18.701 | INFO | Perf report for the last 5.1398 seconds: | 20260508 00:51:18.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:18.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.702 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 00:51:18.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.702 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:51:18.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:18.702 | INFO | Uploaded 36GiB to GPU in 2.56105s (14.0567GiB/s) | 20260508 00:51:18.702 | INFO | Downloaded 34GiB to GPU in 2.4517s (13.8679GiB/s) | 20260508 00:51:18.702 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:51:18.702 | INFO | Total mem usage: 16.7 GiB | 20260508 00:51:18.702 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:51:18.702 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.9%) | 20260508 00:51:18.702 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:51:18.702 | INFO | High Water Breakdown: | 20260508 00:51:18.702 | INFO | 59.7% : 8.66 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:51:18.702 | INFO | 40.2% : 5.84 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:51:18.702 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:51:18.702 | INFO | ------------------------------------------ | 20260508 00:51:18.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:51:18.702 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:51:18.702 | INFO | Total mem usage: 26.5 GiB | 20260508 00:51:18.702 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:51:18.702 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 00:51:18.702 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:51:18.702 | INFO | High Water Breakdown: | 20260508 00:51:18.702 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:51:18.702 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:51:18.702 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:51:18.702 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:51:18.702 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:51:18.702 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:51:18.702 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:51:18.702 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:51:18.702 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:51:18.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:51:18.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:51:18.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:51:18.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:51:18.702 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:51:18.702 | INFO | ------------------------------------------ | 20260508 00:51:18.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:51:18.702 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:51:18.702 | INFO | Total mem usage: 27.8 GiB | 20260508 00:51:18.702 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:51:18.702 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:51:18.702 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:51:18.702 | INFO | High Water Breakdown: | 20260508 00:51:18.702 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:51:18.702 | INFO | ------------------------------------------ | 20260508 00:51:18.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:51:18.702 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:51:18.702 | INFO | Total mem usage: 15.7 GiB | 20260508 00:51:18.702 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:51:18.702 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:51:18.702 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:51:18.702 | INFO | High Water Breakdown: | 20260508 00:51:18.702 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:51:18.702 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:51:18.702 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:51:18.702 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:51:18.703 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:51:18.703 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:51:18.703 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:51:18.703 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:51:18.703 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:51:18.703 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:51:18.703 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:51:18.703 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:51:18.703 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:51:18.703 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:51:18.703 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:51:18.703 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:51:18.703 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:51:18.703 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:51:18.703 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:51:18.703 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:51:18.703 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:51:18.703 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:51:18.703 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:51:18.703 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:51:18.703 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:51:18.703 | INFO | ------------------------------------------ | 20260508 00:51:18.703 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:51:18.703 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:51:18.703 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:51:18.703 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:51:18.703 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.62e+01, 1.43e+01 / 1.31e+00 | 20260508 00:51:18.703 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.01e-01, 1.26e-02 / 4.80e-03 | 20260508 00:51:18.703 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.49e+00, 5.77e-01 / 1.82e-01 | 20260508 00:51:18.703 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 9.56e-02 | 20260508 00:51:18.703 | INFO | PerfReport: 16.16%, DME -- 4.22e+01 / 4.63e+01, 4.28e+00 / 1.45e+00 | 20260508 00:51:18.703 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.83e+01 / 1.94e+01, 1.50e+00 / 1.53e+00 | 20260508 00:51:18.703 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.46e+00 / 2.72e+00, 4.29e-01 / 2.37e-01 | 20260508 00:51:18.703 | INFO | PerfReport: 3.39%, Metrics -- 1.11e+00 / 1.30e+00, 2.76e-01 / 1.41e-01 | 20260508 00:51:18.703 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.39e-01 / 3.72e-01, 5.39e-02 / 2.46e-02 | 20260508 00:51:18.703 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:51:18.704 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:51:18.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:51:18.704 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 24.37%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 30.62%, Compute -- 3.26e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 2.31%, LockUpload -- 2.46e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 2.69%, LockDownload -- 2.87e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 15.49%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:51:18.704 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260508 00:51:18.799 | INFO | Starting a round of Baz IO | 20260508 00:51:22.260 | INFO | Populating RTMetrics report | 20260508 00:51:22.260 | INFO | RTMetrics report populated | 20260508 00:51:22.260 | INFO | Opening tmp RTMetrics json file | 20260508 00:51:22.289 | INFO | Serializing RTMetrics to json | 20260508 00:51:22.291 | INFO | Serialization done, swapping tmp file now... | 20260508 00:51:22.292 | INFO | RTMetrics done | 20260508 00:51:22.362 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.563 seconds | 20260508 00:51:22.362 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.04% / 0.00% / 57.96% | 20260508 00:51:22.362 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.947519% / 7.94729e-06% / 8.74201e-05% | 20260508 00:51:22.362 | INFO | Baz IO overall: | 20260508 00:51:22.362 | INFO | BytesWritten : 1788702373352 | 20260508 00:51:22.362 | INFO | NumEvents : 865545264594 | 20260508 00:51:23.841 | INFO | Analyzing chunk frames = [3940352,3940864) | 20260508 00:51:23.841 | INFO | MemInfo: { | 20260508 00:51:23.841 | INFO | "gpu" : " 17 GB", | 20260508 00:51:23.841 | INFO | "huge" : "", | 20260508 00:51:23.841 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:23.841 | INFO | "malloc" : " 14 GB", | 20260508 00:51:23.841 | INFO | "pinned" : " 29 GB" | 20260508 00:51:23.841 | INFO | } | 20260508 00:51:23.841 | INFO | Perf report for the last 5.13963 seconds: | 20260508 00:51:23.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:23.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:23.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:23.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:23.841 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 00:51:23.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:23.841 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260508 00:51:23.841 | INFO | PRE_HQ : 65.8%, 1.7, 56.4ms, 0 | 20260508 00:51:23.841 | INFO | BAZWRITER : 69.0%, 4.3, 152.7ms, 0 | 20260508 00:51:23.841 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 00:51:23.842 | INFO | Uploaded 36GiB to GPU in 2.56753s (14.0213GiB/s) | 20260508 00:51:23.842 | INFO | Downloaded 38GiB to GPU in 2.68469s (14.1543GiB/s) | 20260508 00:51:28.976 | INFO | Analyzing chunk frames = [3940864,3941376) | 20260508 00:51:28.976 | INFO | MemInfo: { | 20260508 00:51:28.976 | INFO | "gpu" : " 17 GB", | 20260508 00:51:28.976 | INFO | "huge" : "", | 20260508 00:51:28.976 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:28.976 | INFO | "malloc" : " 14 GB", | 20260508 00:51:28.976 | INFO | "pinned" : " 29 GB" | 20260508 00:51:28.976 | INFO | } | 20260508 00:51:28.976 | INFO | Perf report for the last 5.13472 seconds: | 20260508 00:51:28.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:28.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:51:28.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:51:28.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:28.976 | INFO | Uploaded 36GiB to GPU in 2.56193s (14.0519GiB/s) | 20260508 00:51:28.976 | INFO | Downloaded 34GiB to GPU in 2.41752s (14.064GiB/s) | 20260508 00:51:34.118 | INFO | Analyzing chunk frames = [3941376,3941888) | 20260508 00:51:34.118 | INFO | MemInfo: { | 20260508 00:51:34.118 | INFO | "gpu" : " 17 GB", | 20260508 00:51:34.118 | INFO | "huge" : "", | 20260508 00:51:34.118 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:34.118 | INFO | "malloc" : " 14 GB", | 20260508 00:51:34.118 | INFO | "pinned" : " 29 GB" | 20260508 00:51:34.118 | INFO | } | 20260508 00:51:34.118 | INFO | Perf report for the last 5.14242 seconds: | 20260508 00:51:34.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:34.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.118 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 00:51:34.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.118 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:51:34.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:34.119 | INFO | Uploaded 36GiB to GPU in 2.55062s (14.1142GiB/s) | 20260508 00:51:34.119 | INFO | Downloaded 34GiB to GPU in 2.46584s (13.7884GiB/s) | 20260508 00:51:39.258 | INFO | Analyzing chunk frames = [3941888,3942400) | 20260508 00:51:39.258 | INFO | MemInfo: { | 20260508 00:51:39.258 | INFO | "gpu" : " 17 GB", | 20260508 00:51:39.258 | INFO | "huge" : "", | 20260508 00:51:39.258 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:39.258 | INFO | "malloc" : " 14 GB", | 20260508 00:51:39.258 | INFO | "pinned" : " 29 GB" | 20260508 00:51:39.258 | INFO | } | 20260508 00:51:39.258 | INFO | Perf report for the last 5.13973 seconds: | 20260508 00:51:39.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:39.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 00:51:39.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:51:39.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:39.258 | INFO | Uploaded 36GiB to GPU in 2.54499s (14.1454GiB/s) | 20260508 00:51:39.258 | INFO | Downloaded 34GiB to GPU in 2.46188s (13.8106GiB/s) | 20260508 00:51:44.394 | INFO | Analyzing chunk frames = [3942400,3942912) | 20260508 00:51:44.394 | INFO | MemInfo: { | 20260508 00:51:44.394 | INFO | "gpu" : " 17 GB", | 20260508 00:51:44.394 | INFO | "huge" : "", | 20260508 00:51:44.394 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:44.394 | INFO | "malloc" : " 14 GB", | 20260508 00:51:44.394 | INFO | "pinned" : " 29 GB" | 20260508 00:51:44.394 | INFO | } | 20260508 00:51:44.395 | INFO | Perf report for the last 5.13655 seconds: | 20260508 00:51:44.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:44.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 00:51:44.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260508 00:51:44.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:44.395 | INFO | Uploaded 36GiB to GPU in 2.32887s (15.4581GiB/s) | 20260508 00:51:44.395 | INFO | Downloaded 34GiB to GPU in 2.46778s (13.7776GiB/s) | 20260508 00:51:49.532 | INFO | Analyzing chunk frames = [3942912,3943424) | 20260508 00:51:49.532 | INFO | MemInfo: { | 20260508 00:51:49.532 | INFO | "gpu" : " 17 GB", | 20260508 00:51:49.532 | INFO | "huge" : "", | 20260508 00:51:49.532 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:49.532 | INFO | "malloc" : " 14 GB", | 20260508 00:51:49.532 | INFO | "pinned" : " 29 GB" | 20260508 00:51:49.532 | INFO | } | 20260508 00:51:49.532 | INFO | Perf report for the last 5.13756 seconds: | 20260508 00:51:49.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:49.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.532 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:51:49.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.532 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:51:49.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:49.533 | INFO | Uploaded 36GiB to GPU in 2.42035s (14.8739GiB/s) | 20260508 00:51:49.533 | INFO | Downloaded 34GiB to GPU in 2.38279s (14.269GiB/s) | 20260508 00:51:54.674 | INFO | Analyzing chunk frames = [3943424,3943936) | 20260508 00:51:54.674 | INFO | MemInfo: { | 20260508 00:51:54.674 | INFO | "gpu" : " 17 GB", | 20260508 00:51:54.674 | INFO | "huge" : "", | 20260508 00:51:54.674 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:54.675 | INFO | "malloc" : " 14 GB", | 20260508 00:51:54.675 | INFO | "pinned" : " 29 GB" | 20260508 00:51:54.675 | INFO | } | 20260508 00:51:54.675 | INFO | Perf report for the last 5.14254 seconds: | 20260508 00:51:54.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:54.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.675 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 00:51:54.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.675 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:51:54.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:54.676 | INFO | Uploaded 36GiB to GPU in 2.44997s (14.6941GiB/s) | 20260508 00:51:54.676 | INFO | Downloaded 34GiB to GPU in 2.40457s (14.1397GiB/s) | 20260508 00:51:59.814 | INFO | Analyzing chunk frames = [3943936,3944448) | 20260508 00:51:59.814 | INFO | MemInfo: { | 20260508 00:51:59.814 | INFO | "gpu" : " 17 GB", | 20260508 00:51:59.814 | INFO | "huge" : "", | 20260508 00:51:59.814 | INFO | "huge_shared" : " 29 GB", | 20260508 00:51:59.814 | INFO | "malloc" : " 14 GB", | 20260508 00:51:59.815 | INFO | "pinned" : " 29 GB" | 20260508 00:51:59.815 | INFO | } | 20260508 00:51:59.815 | INFO | Perf report for the last 5.13985 seconds: | 20260508 00:51:59.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:51:59.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:51:59.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:51:59.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:51:59.815 | INFO | Uploaded 36GiB to GPU in 2.55071s (14.1137GiB/s) | 20260508 00:51:59.815 | INFO | Downloaded 34GiB to GPU in 2.44662s (13.8967GiB/s) | 20260508 00:51:59.912 | INFO | Starting a round of Baz IO | 20260508 00:52:03.382 | INFO | Populating RTMetrics report | 20260508 00:52:03.382 | INFO | RTMetrics report populated | 20260508 00:52:03.382 | INFO | Opening tmp RTMetrics json file | 20260508 00:52:03.580 | INFO | Serializing RTMetrics to json | 20260508 00:52:03.638 | INFO | Serialization done, swapping tmp file now... | 20260508 00:52:03.639 | INFO | RTMetrics done | 20260508 00:52:03.653 | INFO | Finished a round of Baz IO: Wrote 1.679 GiB in 3.741 seconds | 20260508 00:52:03.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.04% / 0.00% / 57.96% | 20260508 00:52:03.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.9462% / 7.94729e-06% / 5.5631e-05% | 20260508 00:52:03.654 | INFO | Baz IO overall: | 20260508 00:52:03.654 | INFO | BytesWritten : 1790505670753 | 20260508 00:52:03.654 | INFO | NumEvents : 866412420425 | 20260508 00:52:04.954 | INFO | Analyzing chunk frames = [3944448,3944960) | 20260508 00:52:04.954 | INFO | MemInfo: { | 20260508 00:52:04.954 | INFO | "gpu" : " 17 GB", | 20260508 00:52:04.954 | INFO | "huge" : "", | 20260508 00:52:04.954 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:04.954 | INFO | "malloc" : " 14 GB", | 20260508 00:52:04.954 | INFO | "pinned" : " 29 GB" | 20260508 00:52:04.954 | INFO | } | 20260508 00:52:04.954 | INFO | Perf report for the last 5.1398 seconds: | 20260508 00:52:04.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:04.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:04.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:04.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:04.954 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260508 00:52:04.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:04.954 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260508 00:52:04.954 | INFO | PRE_HQ : 65.9%, 1.6, 56.3ms, 0 | 20260508 00:52:04.954 | INFO | BAZWRITER : 72.4%, 3.9, 144.4ms, 0 | 20260508 00:52:04.955 | INFO | BAZ_SAMPLING : 5.2%, 1.0, 2.7ms, 0 | 20260508 00:52:04.955 | INFO | Uploaded 36GiB to GPU in 2.54823s (14.1275GiB/s) | 20260508 00:52:04.955 | INFO | Downloaded 38GiB to GPU in 2.6964s (14.0929GiB/s) | 20260508 00:52:10.094 | INFO | Analyzing chunk frames = [3944960,3945472) | 20260508 00:52:10.094 | INFO | MemInfo: { | 20260508 00:52:10.094 | INFO | "gpu" : " 17 GB", | 20260508 00:52:10.094 | INFO | "huge" : "", | 20260508 00:52:10.094 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:10.094 | INFO | "malloc" : " 14 GB", | 20260508 00:52:10.094 | INFO | "pinned" : " 29 GB" | 20260508 00:52:10.094 | INFO | } | 20260508 00:52:10.094 | INFO | Perf report for the last 5.1398 seconds: | 20260508 00:52:10.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:10.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.094 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 00:52:10.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.094 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:52:10.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:10.095 | INFO | Uploaded 36GiB to GPU in 2.54498s (14.1455GiB/s) | 20260508 00:52:10.095 | INFO | Downloaded 34GiB to GPU in 2.41491s (14.0792GiB/s) | 20260508 00:52:15.227 | INFO | Analyzing chunk frames = [3945472,3945984) | 20260508 00:52:15.227 | INFO | MemInfo: { | 20260508 00:52:15.227 | INFO | "gpu" : " 17 GB", | 20260508 00:52:15.227 | INFO | "huge" : "", | 20260508 00:52:15.227 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:15.227 | INFO | "malloc" : " 14 GB", | 20260508 00:52:15.227 | INFO | "pinned" : " 29 GB" | 20260508 00:52:15.227 | INFO | } | 20260508 00:52:15.227 | INFO | Perf report for the last 5.13321 seconds: | 20260508 00:52:15.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:15.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.227 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:52:15.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.228 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:52:15.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:15.228 | INFO | Uploaded 36GiB to GPU in 2.55034s (14.1157GiB/s) | 20260508 00:52:15.228 | INFO | Downloaded 34GiB to GPU in 2.48573s (13.6781GiB/s) | 20260508 00:52:20.375 | INFO | Analyzing chunk frames = [3945984,3946496) | 20260508 00:52:20.375 | INFO | MemInfo: { | 20260508 00:52:20.375 | INFO | "gpu" : " 17 GB", | 20260508 00:52:20.375 | INFO | "huge" : "", | 20260508 00:52:20.375 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:20.375 | INFO | "malloc" : " 14 GB", | 20260508 00:52:20.375 | INFO | "pinned" : " 29 GB" | 20260508 00:52:20.375 | INFO | } | 20260508 00:52:20.375 | INFO | Perf report for the last 5.14735 seconds: | 20260508 00:52:20.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:20.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 00:52:20.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 00:52:20.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:20.375 | INFO | Uploaded 36GiB to GPU in 2.59728s (13.8606GiB/s) | 20260508 00:52:20.375 | INFO | Downloaded 34GiB to GPU in 2.47017s (13.7642GiB/s) | 20260508 00:52:25.510 | INFO | Analyzing chunk frames = [3946496,3947008) | 20260508 00:52:25.510 | INFO | MemInfo: { | 20260508 00:52:25.510 | INFO | "gpu" : " 17 GB", | 20260508 00:52:25.510 | INFO | "huge" : "", | 20260508 00:52:25.510 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:25.510 | INFO | "malloc" : " 14 GB", | 20260508 00:52:25.510 | INFO | "pinned" : " 29 GB" | 20260508 00:52:25.510 | INFO | } | 20260508 00:52:25.510 | INFO | Perf report for the last 5.13508 seconds: | 20260508 00:52:25.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:25.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:52:25.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:52:25.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:25.510 | INFO | Uploaded 36GiB to GPU in 2.39344s (15.0411GiB/s) | 20260508 00:52:25.510 | INFO | Downloaded 34GiB to GPU in 2.48877s (13.6614GiB/s) | 20260508 00:52:30.651 | INFO | Analyzing chunk frames = [3947008,3947520) | 20260508 00:52:30.651 | INFO | MemInfo: { | 20260508 00:52:30.651 | INFO | "gpu" : " 17 GB", | 20260508 00:52:30.651 | INFO | "huge" : "", | 20260508 00:52:30.651 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:30.651 | INFO | "malloc" : " 14 GB", | 20260508 00:52:30.651 | INFO | "pinned" : " 29 GB" | 20260508 00:52:30.651 | INFO | } | 20260508 00:52:30.651 | INFO | Perf report for the last 5.14146 seconds: | 20260508 00:52:30.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:30.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.651 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 00:52:30.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.651 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:52:30.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:30.652 | INFO | Uploaded 36GiB to GPU in 2.4403s (14.7523GiB/s) | 20260508 00:52:30.652 | INFO | Downloaded 34GiB to GPU in 2.39915s (14.1717GiB/s) | 20260508 00:52:35.788 | INFO | Analyzing chunk frames = [3947520,3948032) | 20260508 00:52:35.788 | INFO | MemInfo: { | 20260508 00:52:35.788 | INFO | "gpu" : " 17 GB", | 20260508 00:52:35.788 | INFO | "huge" : "", | 20260508 00:52:35.788 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:35.788 | INFO | "malloc" : " 14 GB", | 20260508 00:52:35.788 | INFO | "pinned" : " 29 GB" | 20260508 00:52:35.788 | INFO | } | 20260508 00:52:35.788 | INFO | Perf report for the last 5.13664 seconds: | 20260508 00:52:35.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:35.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 00:52:35.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:52:35.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:35.788 | INFO | Uploaded 36GiB to GPU in 2.54429s (14.1494GiB/s) | 20260508 00:52:35.788 | INFO | Downloaded 34GiB to GPU in 2.44884s (13.8841GiB/s) | 20260508 00:52:40.929 | INFO | Analyzing chunk frames = [3948032,3948544) | 20260508 00:52:40.929 | INFO | MemInfo: { | 20260508 00:52:40.929 | INFO | "gpu" : " 17 GB", | 20260508 00:52:40.929 | INFO | "huge" : "", | 20260508 00:52:40.929 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:40.929 | INFO | "malloc" : " 14 GB", | 20260508 00:52:40.929 | INFO | "pinned" : " 29 GB" | 20260508 00:52:40.929 | INFO | } | 20260508 00:52:40.929 | INFO | Perf report for the last 5.14111 seconds: | 20260508 00:52:40.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:40.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:52:40.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:52:40.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:40.929 | INFO | Uploaded 36GiB to GPU in 2.55838s (14.0714GiB/s) | 20260508 00:52:40.929 | INFO | Downloaded 34GiB to GPU in 2.44964s (13.8796GiB/s) | 20260508 00:52:40.930 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:52:40.930 | INFO | Total mem usage: 16.7 GiB | 20260508 00:52:40.930 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:52:40.930 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -10%) | 20260508 00:52:40.930 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:52:40.930 | INFO | High Water Breakdown: | 20260508 00:52:40.930 | INFO | 60.4% : 9.16 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:52:40.930 | INFO | 39.4% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:52:40.930 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:52:40.930 | INFO | ------------------------------------------ | 20260508 00:52:40.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:52:40.930 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:52:40.930 | INFO | Total mem usage: 26.5 GiB | 20260508 00:52:40.930 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:52:40.930 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:52:40.930 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:52:40.930 | INFO | High Water Breakdown: | 20260508 00:52:40.930 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:52:40.930 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:52:40.930 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:52:40.930 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:52:40.930 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:52:40.930 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:52:40.930 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:52:40.930 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:52:40.930 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:52:40.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:52:40.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:52:40.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:52:40.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:52:40.930 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:52:40.930 | INFO | ------------------------------------------ | 20260508 00:52:40.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:52:40.930 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:52:40.930 | INFO | Total mem usage: 27.8 GiB | 20260508 00:52:40.930 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:52:40.930 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:52:40.930 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:52:40.930 | INFO | High Water Breakdown: | 20260508 00:52:40.930 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:52:40.930 | INFO | ------------------------------------------ | 20260508 00:52:40.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:52:40.930 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:52:40.930 | INFO | Total mem usage: 15.7 GiB | 20260508 00:52:40.930 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:52:40.930 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:52:40.930 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:52:40.930 | INFO | High Water Breakdown: | 20260508 00:52:40.930 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:52:40.930 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:52:40.930 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:52:40.930 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:52:40.930 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:52:40.930 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:52:40.930 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:52:40.930 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:52:40.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:52:40.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:52:40.931 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:52:40.931 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:52:40.931 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:52:40.931 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:52:40.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:52:40.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:52:40.931 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:52:40.931 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:52:40.931 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:52:40.931 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:52:40.931 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:52:40.931 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:52:40.931 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:52:40.931 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:52:40.931 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:52:40.931 | INFO | ------------------------------------------ | 20260508 00:52:40.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:52:40.931 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:52:40.931 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:52:40.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:52:40.931 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.64e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 0.29%, AggregateStats -- 9.66e-02 / 1.01e-01, 1.07e-02 / 4.77e-03 | 20260508 00:52:40.931 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.48e+00, 5.88e-01 / 2.01e-01 | 20260508 00:52:40.931 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.18e-01 | 20260508 00:52:40.931 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.57e+01, 4.11e+00 / 1.35e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.74e+00 / 1.84e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.42e+00 / 2.71e+00, 4.19e-01 / 2.32e-01 | 20260508 00:52:40.931 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.29e+00, 2.76e-01 / 1.27e-01 | 20260508 00:52:40.931 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.73e-01, 5.45e-02 / 2.62e-02 | 20260508 00:52:40.931 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:52:40.931 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:52:40.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:52:40.931 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 24.48%, Upload -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 24.14%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 30.73%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 2.49%, LockUpload -- 2.66e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 2.40%, LockDownload -- 2.56e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 15.74%, LockCompute -- 1.68e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 00:52:40.931 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260508 00:52:41.026 | INFO | Starting a round of Baz IO | 20260508 00:52:44.481 | INFO | Populating RTMetrics report | 20260508 00:52:44.481 | INFO | RTMetrics report populated | 20260508 00:52:44.481 | INFO | Opening tmp RTMetrics json file | 20260508 00:52:44.509 | INFO | Serializing RTMetrics to json | 20260508 00:52:44.515 | INFO | Serialization done, swapping tmp file now... | 20260508 00:52:44.557 | INFO | RTMetrics done | 20260508 00:52:44.604 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.578 seconds | 20260508 00:52:44.605 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.03% / 0.00% / 57.96% | 20260508 00:52:44.605 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.945163% / 1.58946e-05% / 6.75519e-05% | 20260508 00:52:44.605 | INFO | Baz IO overall: | 20260508 00:52:44.605 | INFO | BytesWritten : 1792309633410 | 20260508 00:52:44.605 | INFO | NumEvents : 867280240655 | 20260508 00:52:46.066 | INFO | Analyzing chunk frames = [3948544,3949056) | 20260508 00:52:46.066 | INFO | MemInfo: { | 20260508 00:52:46.066 | INFO | "gpu" : " 17 GB", | 20260508 00:52:46.066 | INFO | "huge" : "", | 20260508 00:52:46.066 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:46.066 | INFO | "malloc" : " 14 GB", | 20260508 00:52:46.066 | INFO | "pinned" : " 29 GB" | 20260508 00:52:46.066 | INFO | } | 20260508 00:52:46.067 | INFO | Perf report for the last 5.13742 seconds: | 20260508 00:52:46.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:46.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:46.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:46.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:46.067 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260508 00:52:46.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:46.067 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 00:52:46.067 | INFO | PRE_HQ : 65.6%, 1.7, 56.5ms, 0 | 20260508 00:52:46.067 | INFO | BAZWRITER : 69.3%, 3.7, 132.7ms, 0 | 20260508 00:52:46.067 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 00:52:46.067 | INFO | Uploaded 36GiB to GPU in 2.54024s (14.1719GiB/s) | 20260508 00:52:46.067 | INFO | Downloaded 38GiB to GPU in 2.70244s (14.0613GiB/s) | 20260508 00:52:51.211 | INFO | Analyzing chunk frames = [3949056,3949568) | 20260508 00:52:51.211 | INFO | MemInfo: { | 20260508 00:52:51.211 | INFO | "gpu" : " 17 GB", | 20260508 00:52:51.211 | INFO | "huge" : "", | 20260508 00:52:51.211 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:51.211 | INFO | "malloc" : " 14 GB", | 20260508 00:52:51.211 | INFO | "pinned" : " 29 GB" | 20260508 00:52:51.211 | INFO | } | 20260508 00:52:51.211 | INFO | Perf report for the last 5.14471 seconds: | 20260508 00:52:51.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:51.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.211 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 00:52:51.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.211 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 00:52:51.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:51.212 | INFO | Uploaded 36GiB to GPU in 2.55101s (14.112GiB/s) | 20260508 00:52:51.212 | INFO | Downloaded 34GiB to GPU in 2.42347s (14.0295GiB/s) | 20260508 00:52:56.346 | INFO | Analyzing chunk frames = [3949568,3950080) | 20260508 00:52:56.346 | INFO | MemInfo: { | 20260508 00:52:56.346 | INFO | "gpu" : " 17 GB", | 20260508 00:52:56.346 | INFO | "huge" : "", | 20260508 00:52:56.346 | INFO | "huge_shared" : " 29 GB", | 20260508 00:52:56.346 | INFO | "malloc" : " 14 GB", | 20260508 00:52:56.346 | INFO | "pinned" : " 29 GB" | 20260508 00:52:56.346 | INFO | } | 20260508 00:52:56.346 | INFO | Perf report for the last 5.13502 seconds: | 20260508 00:52:56.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:52:56.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.346 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:52:56.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.346 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:52:56.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:52:56.347 | INFO | Uploaded 36GiB to GPU in 2.55412s (14.0949GiB/s) | 20260508 00:52:56.347 | INFO | Downloaded 34GiB to GPU in 2.48469s (13.6838GiB/s) | 20260508 00:53:01.485 | INFO | Analyzing chunk frames = [3950080,3950592) | 20260508 00:53:01.485 | INFO | MemInfo: { | 20260508 00:53:01.485 | INFO | "gpu" : " 17 GB", | 20260508 00:53:01.485 | INFO | "huge" : "", | 20260508 00:53:01.486 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:01.486 | INFO | "malloc" : " 14 GB", | 20260508 00:53:01.486 | INFO | "pinned" : " 29 GB" | 20260508 00:53:01.486 | INFO | } | 20260508 00:53:01.486 | INFO | Perf report for the last 5.13936 seconds: | 20260508 00:53:01.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:01.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 00:53:01.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:53:01.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:01.486 | INFO | Uploaded 36GiB to GPU in 2.49904s (14.4055GiB/s) | 20260508 00:53:01.486 | INFO | Downloaded 34GiB to GPU in 2.46628s (13.786GiB/s) | 20260508 00:53:06.627 | INFO | Analyzing chunk frames = [3950592,3951104) | 20260508 00:53:06.627 | INFO | MemInfo: { | 20260508 00:53:06.627 | INFO | "gpu" : " 17 GB", | 20260508 00:53:06.627 | INFO | "huge" : "", | 20260508 00:53:06.627 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:06.627 | INFO | "malloc" : " 14 GB", | 20260508 00:53:06.627 | INFO | "pinned" : " 29 GB" | 20260508 00:53:06.627 | INFO | } | 20260508 00:53:06.627 | INFO | Perf report for the last 5.14116 seconds: | 20260508 00:53:06.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:06.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 00:53:06.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:53:06.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:06.627 | INFO | Uploaded 36GiB to GPU in 2.32601s (15.4772GiB/s) | 20260508 00:53:06.627 | INFO | Downloaded 34GiB to GPU in 2.49401s (13.6327GiB/s) | 20260508 00:53:11.765 | INFO | Analyzing chunk frames = [3951104,3951616) | 20260508 00:53:11.765 | INFO | MemInfo: { | 20260508 00:53:11.765 | INFO | "gpu" : " 17 GB", | 20260508 00:53:11.765 | INFO | "huge" : "", | 20260508 00:53:11.765 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:11.765 | INFO | "malloc" : " 14 GB", | 20260508 00:53:11.765 | INFO | "pinned" : " 29 GB" | 20260508 00:53:11.765 | INFO | } | 20260508 00:53:11.765 | INFO | Perf report for the last 5.13845 seconds: | 20260508 00:53:11.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:11.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.765 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 00:53:11.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.765 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:53:11.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:11.766 | INFO | Uploaded 36GiB to GPU in 2.42608s (14.8388GiB/s) | 20260508 00:53:11.766 | INFO | Downloaded 34GiB to GPU in 2.39293s (14.2085GiB/s) | 20260508 00:53:16.904 | INFO | Analyzing chunk frames = [3951616,3952128) | 20260508 00:53:16.904 | INFO | MemInfo: { | 20260508 00:53:16.904 | INFO | "gpu" : " 17 GB", | 20260508 00:53:16.904 | INFO | "huge" : "", | 20260508 00:53:16.904 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:16.904 | INFO | "malloc" : " 14 GB", | 20260508 00:53:16.904 | INFO | "pinned" : " 29 GB" | 20260508 00:53:16.904 | INFO | } | 20260508 00:53:16.904 | INFO | Perf report for the last 5.13856 seconds: | 20260508 00:53:16.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:16.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 00:53:16.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:53:16.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:16.904 | INFO | Uploaded 36GiB to GPU in 2.53336s (14.2104GiB/s) | 20260508 00:53:16.904 | INFO | Downloaded 34GiB to GPU in 2.44868s (13.885GiB/s) | 20260508 00:53:22.041 | INFO | Analyzing chunk frames = [3952128,3952640) | 20260508 00:53:22.041 | INFO | MemInfo: { | 20260508 00:53:22.041 | INFO | "gpu" : " 17 GB", | 20260508 00:53:22.041 | INFO | "huge" : "", | 20260508 00:53:22.041 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:22.041 | INFO | "malloc" : " 14 GB", | 20260508 00:53:22.041 | INFO | "pinned" : " 29 GB" | 20260508 00:53:22.041 | INFO | } | 20260508 00:53:22.041 | INFO | Perf report for the last 5.13715 seconds: | 20260508 00:53:22.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:22.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 00:53:22.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:53:22.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:22.041 | INFO | Uploaded 36GiB to GPU in 2.53322s (14.2111GiB/s) | 20260508 00:53:22.041 | INFO | Downloaded 34GiB to GPU in 2.44157s (13.9255GiB/s) | 20260508 00:53:22.149 | INFO | Starting a round of Baz IO | 20260508 00:53:25.592 | INFO | Populating RTMetrics report | 20260508 00:53:25.592 | INFO | RTMetrics report populated | 20260508 00:53:25.592 | INFO | Opening tmp RTMetrics json file | 20260508 00:53:25.609 | INFO | Serializing RTMetrics to json | 20260508 00:53:25.617 | INFO | Serialization done, swapping tmp file now... | 20260508 00:53:25.617 | INFO | RTMetrics done | 20260508 00:53:25.700 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.55 seconds | 20260508 00:53:25.700 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.03% / 0.00% / 57.97% | 20260508 00:53:25.700 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.939866% / 3.97364e-06% / 7.94729e-05% | 20260508 00:53:25.700 | INFO | Baz IO overall: | 20260508 00:53:25.700 | INFO | BytesWritten : 1794110768312 | 20260508 00:53:25.700 | INFO | NumEvents : 868146018756 | 20260508 00:53:27.184 | INFO | Analyzing chunk frames = [3952640,3953152) | 20260508 00:53:27.184 | INFO | MemInfo: { | 20260508 00:53:27.184 | INFO | "gpu" : " 17 GB", | 20260508 00:53:27.184 | INFO | "huge" : "", | 20260508 00:53:27.184 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:27.184 | INFO | "malloc" : " 14 GB", | 20260508 00:53:27.184 | INFO | "pinned" : " 29 GB" | 20260508 00:53:27.184 | INFO | } | 20260508 00:53:27.184 | INFO | Perf report for the last 5.14316 seconds: | 20260508 00:53:27.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:27.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:27.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:27.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:27.184 | INFO | ANALYSIS : 68.3%, 3.0, 106.8ms, 0 | 20260508 00:53:27.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:27.184 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260508 00:53:27.184 | INFO | PRE_HQ : 65.8%, 1.7, 57.6ms, 0 | 20260508 00:53:27.184 | INFO | BAZWRITER : 68.4%, 3.8, 135.1ms, 0 | 20260508 00:53:27.184 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 00:53:27.185 | INFO | Uploaded 36GiB to GPU in 2.56139s (14.0549GiB/s) | 20260508 00:53:27.185 | INFO | Downloaded 38GiB to GPU in 2.69364s (14.1073GiB/s) | 20260508 00:53:32.329 | INFO | Analyzing chunk frames = [3953152,3953664) | 20260508 00:53:32.329 | INFO | MemInfo: { | 20260508 00:53:32.329 | INFO | "gpu" : " 17 GB", | 20260508 00:53:32.329 | INFO | "huge" : "", | 20260508 00:53:32.329 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:32.329 | INFO | "malloc" : " 14 GB", | 20260508 00:53:32.329 | INFO | "pinned" : " 29 GB" | 20260508 00:53:32.329 | INFO | } | 20260508 00:53:32.329 | INFO | Perf report for the last 5.14477 seconds: | 20260508 00:53:32.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:32.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.329 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 00:53:32.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.329 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:53:32.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:32.330 | INFO | Uploaded 36GiB to GPU in 2.58673s (13.9172GiB/s) | 20260508 00:53:32.330 | INFO | Downloaded 34GiB to GPU in 2.48148s (13.7015GiB/s) | 20260508 00:53:37.464 | INFO | Analyzing chunk frames = [3953664,3954176) | 20260508 00:53:37.465 | INFO | MemInfo: { | 20260508 00:53:37.465 | INFO | "gpu" : " 17 GB", | 20260508 00:53:37.465 | INFO | "huge" : "", | 20260508 00:53:37.465 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:37.465 | INFO | "malloc" : " 14 GB", | 20260508 00:53:37.465 | INFO | "pinned" : " 29 GB" | 20260508 00:53:37.465 | INFO | } | 20260508 00:53:37.465 | INFO | Perf report for the last 5.13569 seconds: | 20260508 00:53:37.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:37.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 00:53:37.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 00:53:37.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:37.465 | INFO | Uploaded 36GiB to GPU in 2.55649s (14.0818GiB/s) | 20260508 00:53:37.465 | INFO | Downloaded 34GiB to GPU in 2.48361s (13.6898GiB/s) | 20260508 00:53:42.601 | INFO | Analyzing chunk frames = [3954176,3954688) | 20260508 00:53:42.601 | INFO | MemInfo: { | 20260508 00:53:42.601 | INFO | "gpu" : " 17 GB", | 20260508 00:53:42.601 | INFO | "huge" : "", | 20260508 00:53:42.601 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:42.601 | INFO | "malloc" : " 14 GB", | 20260508 00:53:42.601 | INFO | "pinned" : " 29 GB" | 20260508 00:53:42.601 | INFO | } | 20260508 00:53:42.601 | INFO | Perf report for the last 5.13669 seconds: | 20260508 00:53:42.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:42.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.601 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 00:53:42.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.602 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:53:42.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:42.602 | INFO | Uploaded 36GiB to GPU in 2.54869s (14.1249GiB/s) | 20260508 00:53:42.602 | INFO | Downloaded 34GiB to GPU in 2.47593s (13.7322GiB/s) | 20260508 00:53:47.738 | INFO | Analyzing chunk frames = [3954688,3955200) | 20260508 00:53:47.738 | INFO | MemInfo: { | 20260508 00:53:47.738 | INFO | "gpu" : " 17 GB", | 20260508 00:53:47.738 | INFO | "huge" : "", | 20260508 00:53:47.738 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:47.738 | INFO | "malloc" : " 14 GB", | 20260508 00:53:47.738 | INFO | "pinned" : " 29 GB" | 20260508 00:53:47.738 | INFO | } | 20260508 00:53:47.738 | INFO | Perf report for the last 5.1367 seconds: | 20260508 00:53:47.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:47.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.738 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 00:53:47.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.738 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:53:47.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:47.739 | INFO | Uploaded 36GiB to GPU in 2.38923s (15.0676GiB/s) | 20260508 00:53:47.739 | INFO | Downloaded 34GiB to GPU in 2.55581s (13.303GiB/s) | 20260508 00:53:52.878 | INFO | Analyzing chunk frames = [3955200,3955712) | 20260508 00:53:52.878 | INFO | MemInfo: { | 20260508 00:53:52.878 | INFO | "gpu" : " 17 GB", | 20260508 00:53:52.878 | INFO | "huge" : "", | 20260508 00:53:52.879 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:52.879 | INFO | "malloc" : " 14 GB", | 20260508 00:53:52.879 | INFO | "pinned" : " 29 GB" | 20260508 00:53:52.879 | INFO | } | 20260508 00:53:52.879 | INFO | Perf report for the last 5.14054 seconds: | 20260508 00:53:52.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:52.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:53:52.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:53:52.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:52.879 | INFO | Uploaded 36GiB to GPU in 2.42168s (14.8657GiB/s) | 20260508 00:53:52.879 | INFO | Downloaded 34GiB to GPU in 2.38939s (14.2296GiB/s) | 20260508 00:53:58.022 | INFO | Analyzing chunk frames = [3955712,3956224) | 20260508 00:53:58.022 | INFO | MemInfo: { | 20260508 00:53:58.022 | INFO | "gpu" : " 17 GB", | 20260508 00:53:58.022 | INFO | "huge" : "", | 20260508 00:53:58.022 | INFO | "huge_shared" : " 29 GB", | 20260508 00:53:58.022 | INFO | "malloc" : " 14 GB", | 20260508 00:53:58.022 | INFO | "pinned" : " 29 GB" | 20260508 00:53:58.022 | INFO | } | 20260508 00:53:58.022 | INFO | Perf report for the last 5.14333 seconds: | 20260508 00:53:58.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:53:58.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.022 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:53:58.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.022 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 00:53:58.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:53:58.023 | INFO | Uploaded 36GiB to GPU in 2.53991s (14.1737GiB/s) | 20260508 00:53:58.023 | INFO | Downloaded 34GiB to GPU in 2.44564s (13.9023GiB/s) | 20260508 00:54:03.161 | INFO | Analyzing chunk frames = [3956224,3956736) | 20260508 00:54:03.161 | INFO | MemInfo: { | 20260508 00:54:03.161 | INFO | "gpu" : " 17 GB", | 20260508 00:54:03.161 | INFO | "huge" : "", | 20260508 00:54:03.162 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:03.162 | INFO | "malloc" : " 14 GB", | 20260508 00:54:03.162 | INFO | "pinned" : " 29 GB" | 20260508 00:54:03.162 | INFO | } | 20260508 00:54:03.162 | INFO | Perf report for the last 5.1399 seconds: | 20260508 00:54:03.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:03.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:54:03.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 00:54:03.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:03.162 | INFO | Uploaded 36GiB to GPU in 2.56301s (14.046GiB/s) | 20260508 00:54:03.163 | INFO | Downloaded 34GiB to GPU in 2.45623s (13.8424GiB/s) | 20260508 00:54:03.163 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:54:03.163 | INFO | Total mem usage: 16.7 GiB | 20260508 00:54:03.163 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:54:03.163 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260508 00:54:03.163 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:54:03.163 | INFO | High Water Breakdown: | 20260508 00:54:03.163 | INFO | 59.8% : 8.74 GiB ( 84.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:54:03.163 | INFO | 40.1% : 5.86 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:54:03.163 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:54:03.163 | INFO | ------------------------------------------ | 20260508 00:54:03.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:54:03.163 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:54:03.163 | INFO | Total mem usage: 26.5 GiB | 20260508 00:54:03.163 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:54:03.163 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:54:03.163 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:54:03.163 | INFO | High Water Breakdown: | 20260508 00:54:03.163 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:54:03.163 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:54:03.163 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:54:03.163 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:54:03.163 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:54:03.163 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:54:03.163 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:54:03.163 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:54:03.163 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:54:03.163 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:54:03.163 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:54:03.163 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:54:03.163 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:54:03.163 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:54:03.163 | INFO | ------------------------------------------ | 20260508 00:54:03.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:54:03.163 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:54:03.163 | INFO | Total mem usage: 27.8 GiB | 20260508 00:54:03.163 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:54:03.163 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:54:03.163 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:54:03.163 | INFO | High Water Breakdown: | 20260508 00:54:03.163 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:54:03.163 | INFO | ------------------------------------------ | 20260508 00:54:03.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:54:03.163 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:54:03.163 | INFO | Total mem usage: 15.7 GiB | 20260508 00:54:03.163 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:54:03.163 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:54:03.163 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:54:03.163 | INFO | High Water Breakdown: | 20260508 00:54:03.163 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:54:03.163 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:54:03.163 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:54:03.163 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:54:03.163 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:54:03.163 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:54:03.163 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:54:03.163 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:54:03.163 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:54:03.163 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:54:03.163 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:54:03.163 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:54:03.163 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:54:03.163 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:54:03.163 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:54:03.163 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:54:03.163 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:54:03.163 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:54:03.163 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:54:03.163 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:54:03.163 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:54:03.163 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:54:03.163 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:54:03.163 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:54:03.163 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:54:03.163 | INFO | ------------------------------------------ | 20260508 00:54:03.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:54:03.164 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:54:03.164 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:54:03.164 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:54:03.164 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.64e+01, 1.43e+01 / 1.81e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.01e-01, 1.10e-02 / 5.88e-03 | 20260508 00:54:03.164 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.76e-01 / 2.03e-01 | 20260508 00:54:03.164 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.09e-01 / 1.06e-01 | 20260508 00:54:03.164 | INFO | PerfReport: 16.13%, DME -- 4.22e+01 / 4.63e+01, 4.23e+00 / 2.22e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.83e+01 / 1.96e+01, 1.55e+00 / 1.57e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 2.73e+00, 4.28e-01 / 2.31e-01 | 20260508 00:54:03.164 | INFO | PerfReport: 3.38%, Metrics -- 1.10e+00 / 1.35e+00, 2.76e-01 / 1.57e-01 | 20260508 00:54:03.164 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.39e-01 / 3.81e-01, 5.38e-02 / 3.21e-02 | 20260508 00:54:03.164 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:54:03.164 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:54:03.164 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:54:03.164 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 30.59%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 2.48%, LockUpload -- 2.65e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 2.59%, LockDownload -- 2.77e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 15.60%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 00:54:03.164 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260508 00:54:03.276 | INFO | Starting a round of Baz IO | 20260508 00:54:06.731 | INFO | Populating RTMetrics report | 20260508 00:54:06.732 | INFO | RTMetrics report populated | 20260508 00:54:06.732 | INFO | Opening tmp RTMetrics json file | 20260508 00:54:06.754 | INFO | Serializing RTMetrics to json | 20260508 00:54:06.756 | INFO | Serialization done, swapping tmp file now... | 20260508 00:54:06.757 | INFO | RTMetrics done | 20260508 00:54:06.833 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.556 seconds | 20260508 00:54:06.833 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.02% / 0.00% / 57.97% | 20260508 00:54:06.833 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.939306% / 0% / 7.54992e-05% | 20260508 00:54:06.833 | INFO | Baz IO overall: | 20260508 00:54:06.833 | INFO | BytesWritten : 1795912561336 | 20260508 00:54:06.833 | INFO | NumEvents : 869012571610 | 20260508 00:54:08.296 | INFO | Analyzing chunk frames = [3956736,3957248) | 20260508 00:54:08.297 | INFO | MemInfo: { | 20260508 00:54:08.297 | INFO | "gpu" : " 17 GB", | 20260508 00:54:08.297 | INFO | "huge" : "", | 20260508 00:54:08.297 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:08.297 | INFO | "malloc" : " 14 GB", | 20260508 00:54:08.297 | INFO | "pinned" : " 29 GB" | 20260508 00:54:08.297 | INFO | } | 20260508 00:54:08.297 | INFO | Perf report for the last 5.1348 seconds: | 20260508 00:54:08.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:08.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:08.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:08.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:08.297 | INFO | ANALYSIS : 68.8%, 2.9, 104.7ms, 0 | 20260508 00:54:08.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:08.297 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260508 00:54:08.297 | INFO | PRE_HQ : 67.7%, 1.8, 62.7ms, 0 | 20260508 00:54:08.297 | INFO | BAZWRITER : 68.8%, 3.9, 138.8ms, 0 | 20260508 00:54:08.297 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 00:54:08.297 | INFO | Uploaded 36GiB to GPU in 2.62869s (13.6951GiB/s) | 20260508 00:54:08.297 | INFO | Downloaded 38GiB to GPU in 2.70798s (14.0326GiB/s) | 20260508 00:54:13.441 | INFO | Analyzing chunk frames = [3957248,3957760) | 20260508 00:54:13.441 | INFO | MemInfo: { | 20260508 00:54:13.441 | INFO | "gpu" : " 17 GB", | 20260508 00:54:13.441 | INFO | "huge" : "", | 20260508 00:54:13.441 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:13.441 | INFO | "malloc" : " 14 GB", | 20260508 00:54:13.441 | INFO | "pinned" : " 29 GB" | 20260508 00:54:13.441 | INFO | } | 20260508 00:54:13.441 | INFO | Perf report for the last 5.14445 seconds: | 20260508 00:54:13.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:13.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.441 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 00:54:13.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.441 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:54:13.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:13.442 | INFO | Uploaded 36GiB to GPU in 2.57578s (13.9764GiB/s) | 20260508 00:54:13.442 | INFO | Downloaded 34GiB to GPU in 2.49365s (13.6346GiB/s) | 20260508 00:54:18.578 | INFO | Analyzing chunk frames = [3957760,3958272) | 20260508 00:54:18.578 | INFO | MemInfo: { | 20260508 00:54:18.578 | INFO | "gpu" : " 17 GB", | 20260508 00:54:18.578 | INFO | "huge" : "", | 20260508 00:54:18.578 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:18.578 | INFO | "malloc" : " 14 GB", | 20260508 00:54:18.578 | INFO | "pinned" : " 29 GB" | 20260508 00:54:18.578 | INFO | } | 20260508 00:54:18.578 | INFO | Perf report for the last 5.13706 seconds: | 20260508 00:54:18.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:18.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.578 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:54:18.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.578 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:54:18.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:18.579 | INFO | Uploaded 36GiB to GPU in 2.52101s (14.28GiB/s) | 20260508 00:54:18.579 | INFO | Downloaded 34GiB to GPU in 2.49965s (13.6019GiB/s) | 20260508 00:54:23.717 | INFO | Analyzing chunk frames = [3958272,3958784) | 20260508 00:54:23.717 | INFO | MemInfo: { | 20260508 00:54:23.717 | INFO | "gpu" : " 17 GB", | 20260508 00:54:23.717 | INFO | "huge" : "", | 20260508 00:54:23.717 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:23.717 | INFO | "malloc" : " 14 GB", | 20260508 00:54:23.717 | INFO | "pinned" : " 29 GB" | 20260508 00:54:23.718 | INFO | } | 20260508 00:54:23.718 | INFO | Perf report for the last 5.13928 seconds: | 20260508 00:54:23.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:23.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 00:54:23.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:54:23.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:23.718 | INFO | Uploaded 36GiB to GPU in 2.506s (14.3655GiB/s) | 20260508 00:54:23.718 | INFO | Downloaded 34GiB to GPU in 2.48898s (13.6602GiB/s) | 20260508 00:54:28.851 | INFO | Analyzing chunk frames = [3958784,3959296) | 20260508 00:54:28.851 | INFO | MemInfo: { | 20260508 00:54:28.851 | INFO | "gpu" : " 17 GB", | 20260508 00:54:28.851 | INFO | "huge" : "", | 20260508 00:54:28.851 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:28.851 | INFO | "malloc" : " 14 GB", | 20260508 00:54:28.851 | INFO | "pinned" : " 29 GB" | 20260508 00:54:28.851 | INFO | } | 20260508 00:54:28.851 | INFO | Perf report for the last 5.13359 seconds: | 20260508 00:54:28.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:28.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 00:54:28.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 00:54:28.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:28.851 | INFO | Uploaded 36GiB to GPU in 2.28836s (15.7318GiB/s) | 20260508 00:54:28.851 | INFO | Downloaded 34GiB to GPU in 2.5091s (13.5507GiB/s) | 20260508 00:54:33.992 | INFO | Analyzing chunk frames = [3959296,3959808) | 20260508 00:54:33.992 | INFO | MemInfo: { | 20260508 00:54:33.992 | INFO | "gpu" : " 17 GB", | 20260508 00:54:33.992 | INFO | "huge" : "", | 20260508 00:54:33.993 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:33.993 | INFO | "malloc" : " 14 GB", | 20260508 00:54:33.993 | INFO | "pinned" : " 29 GB" | 20260508 00:54:33.993 | INFO | } | 20260508 00:54:33.993 | INFO | Perf report for the last 5.14152 seconds: | 20260508 00:54:33.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:33.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 00:54:33.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:54:33.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:33.993 | INFO | Uploaded 36GiB to GPU in 2.34692s (15.3392GiB/s) | 20260508 00:54:33.993 | INFO | Downloaded 34GiB to GPU in 2.42613s (14.0141GiB/s) | 20260508 00:54:39.137 | INFO | Analyzing chunk frames = [3959808,3960320) | 20260508 00:54:39.137 | INFO | MemInfo: { | 20260508 00:54:39.137 | INFO | "gpu" : " 17 GB", | 20260508 00:54:39.137 | INFO | "huge" : "", | 20260508 00:54:39.137 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:39.137 | INFO | "malloc" : " 14 GB", | 20260508 00:54:39.137 | INFO | "pinned" : " 29 GB" | 20260508 00:54:39.137 | INFO | } | 20260508 00:54:39.138 | INFO | Perf report for the last 5.14475 seconds: | 20260508 00:54:39.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:39.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260508 00:54:39.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 00:54:39.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:39.138 | INFO | Uploaded 36GiB to GPU in 2.51016s (14.3417GiB/s) | 20260508 00:54:39.138 | INFO | Downloaded 34GiB to GPU in 2.46899s (13.7708GiB/s) | 20260508 00:54:44.279 | INFO | Analyzing chunk frames = [3960320,3960832) | 20260508 00:54:44.279 | INFO | MemInfo: { | 20260508 00:54:44.279 | INFO | "gpu" : " 17 GB", | 20260508 00:54:44.279 | INFO | "huge" : "", | 20260508 00:54:44.279 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:44.279 | INFO | "malloc" : " 14 GB", | 20260508 00:54:44.279 | INFO | "pinned" : " 29 GB" | 20260508 00:54:44.279 | INFO | } | 20260508 00:54:44.279 | INFO | Perf report for the last 5.14174 seconds: | 20260508 00:54:44.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:44.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.279 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 00:54:44.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.280 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 00:54:44.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:44.280 | INFO | Uploaded 36GiB to GPU in 2.53978s (14.1744GiB/s) | 20260508 00:54:44.280 | INFO | Downloaded 34GiB to GPU in 2.47094s (13.7599GiB/s) | 20260508 00:54:44.395 | INFO | Starting a round of Baz IO | 20260508 00:54:47.867 | INFO | Populating RTMetrics report | 20260508 00:54:47.867 | INFO | RTMetrics report populated | 20260508 00:54:47.867 | INFO | Opening tmp RTMetrics json file | 20260508 00:54:47.886 | INFO | Serializing RTMetrics to json | 20260508 00:54:47.888 | INFO | Serialization done, swapping tmp file now... | 20260508 00:54:47.889 | INFO | RTMetrics done | 20260508 00:54:47.962 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.567 seconds | 20260508 00:54:47.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.02% / 0.00% / 57.98% | 20260508 00:54:47.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.939778% / 0% / 5.96046e-05% | 20260508 00:54:47.962 | INFO | Baz IO overall: | 20260508 00:54:47.962 | INFO | BytesWritten : 1797717480870 | 20260508 00:54:47.962 | INFO | NumEvents : 869881427967 | 20260508 00:54:49.417 | INFO | Analyzing chunk frames = [3960832,3961344) | 20260508 00:54:49.417 | INFO | MemInfo: { | 20260508 00:54:49.417 | INFO | "gpu" : " 17 GB", | 20260508 00:54:49.417 | INFO | "huge" : "", | 20260508 00:54:49.417 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:49.417 | INFO | "malloc" : " 14 GB", | 20260508 00:54:49.417 | INFO | "pinned" : " 29 GB" | 20260508 00:54:49.417 | INFO | } | 20260508 00:54:49.417 | INFO | Perf report for the last 5.13796 seconds: | 20260508 00:54:49.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:49.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:49.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:49.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:49.417 | INFO | ANALYSIS : 68.9%, 2.9, 104.8ms, 0 | 20260508 00:54:49.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:49.417 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 00:54:49.417 | INFO | PRE_HQ : 67.7%, 1.8, 63.2ms, 0 | 20260508 00:54:49.417 | INFO | BAZWRITER : 69.1%, 3.9, 140.2ms, 0 | 20260508 00:54:49.417 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 00:54:49.418 | INFO | Uploaded 36GiB to GPU in 2.59094s (13.8946GiB/s) | 20260508 00:54:49.418 | INFO | Downloaded 38GiB to GPU in 2.72546s (13.9426GiB/s) | 20260508 00:54:54.553 | INFO | Analyzing chunk frames = [3961344,3961856) | 20260508 00:54:54.553 | INFO | MemInfo: { | 20260508 00:54:54.553 | INFO | "gpu" : " 17 GB", | 20260508 00:54:54.553 | INFO | "huge" : "", | 20260508 00:54:54.553 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:54.553 | INFO | "malloc" : " 14 GB", | 20260508 00:54:54.553 | INFO | "pinned" : " 29 GB" | 20260508 00:54:54.553 | INFO | } | 20260508 00:54:54.553 | INFO | Perf report for the last 5.13585 seconds: | 20260508 00:54:54.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:54.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 00:54:54.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 00:54:54.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:54.553 | INFO | Uploaded 36GiB to GPU in 2.50496s (14.3715GiB/s) | 20260508 00:54:54.553 | INFO | Downloaded 34GiB to GPU in 2.44334s (13.9154GiB/s) | 20260508 00:54:59.693 | INFO | Analyzing chunk frames = [3961856,3962368) | 20260508 00:54:59.693 | INFO | MemInfo: { | 20260508 00:54:59.693 | INFO | "gpu" : " 17 GB", | 20260508 00:54:59.693 | INFO | "huge" : "", | 20260508 00:54:59.693 | INFO | "huge_shared" : " 29 GB", | 20260508 00:54:59.693 | INFO | "malloc" : " 14 GB", | 20260508 00:54:59.693 | INFO | "pinned" : " 29 GB" | 20260508 00:54:59.693 | INFO | } | 20260508 00:54:59.693 | INFO | Perf report for the last 5.13971 seconds: | 20260508 00:54:59.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:54:59.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 00:54:59.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:54:59.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:54:59.693 | INFO | Uploaded 36GiB to GPU in 2.45908s (14.6396GiB/s) | 20260508 00:54:59.693 | INFO | Downloaded 34GiB to GPU in 2.50799s (13.5567GiB/s) | 20260508 00:55:04.828 | INFO | Analyzing chunk frames = [3962368,3962880) | 20260508 00:55:04.828 | INFO | MemInfo: { | 20260508 00:55:04.828 | INFO | "gpu" : " 17 GB", | 20260508 00:55:04.828 | INFO | "huge" : "", | 20260508 00:55:04.828 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:04.828 | INFO | "malloc" : " 14 GB", | 20260508 00:55:04.828 | INFO | "pinned" : " 29 GB" | 20260508 00:55:04.828 | INFO | } | 20260508 00:55:04.828 | INFO | Perf report for the last 5.13563 seconds: | 20260508 00:55:04.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:04.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.829 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:55:04.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.829 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:55:04.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:04.829 | INFO | Uploaded 36GiB to GPU in 2.45044s (14.6912GiB/s) | 20260508 00:55:04.829 | INFO | Downloaded 34GiB to GPU in 2.50846s (13.5541GiB/s) | 20260508 00:55:09.971 | INFO | Analyzing chunk frames = [3962880,3963392) | 20260508 00:55:09.971 | INFO | MemInfo: { | 20260508 00:55:09.971 | INFO | "gpu" : " 17 GB", | 20260508 00:55:09.971 | INFO | "huge" : "", | 20260508 00:55:09.971 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:09.972 | INFO | "malloc" : " 14 GB", | 20260508 00:55:09.972 | INFO | "pinned" : " 29 GB" | 20260508 00:55:09.972 | INFO | } | 20260508 00:55:09.972 | INFO | Perf report for the last 5.14324 seconds: | 20260508 00:55:09.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:09.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:55:09.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:55:09.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:09.972 | INFO | Uploaded 36GiB to GPU in 2.3006s (15.6481GiB/s) | 20260508 00:55:09.972 | INFO | Downloaded 34GiB to GPU in 2.48856s (13.6625GiB/s) | 20260508 00:55:15.110 | INFO | Analyzing chunk frames = [3963392,3963904) | 20260508 00:55:15.110 | INFO | MemInfo: { | 20260508 00:55:15.110 | INFO | "gpu" : " 17 GB", | 20260508 00:55:15.110 | INFO | "huge" : "", | 20260508 00:55:15.110 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:15.110 | INFO | "malloc" : " 14 GB", | 20260508 00:55:15.110 | INFO | "pinned" : " 29 GB" | 20260508 00:55:15.110 | INFO | } | 20260508 00:55:15.110 | INFO | Perf report for the last 5.13868 seconds: | 20260508 00:55:15.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:15.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.110 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 00:55:15.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.111 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:55:15.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:15.111 | INFO | Uploaded 36GiB to GPU in 2.36535s (15.2197GiB/s) | 20260508 00:55:15.111 | INFO | Downloaded 34GiB to GPU in 2.42261s (14.0345GiB/s) | 20260508 00:55:20.250 | INFO | Analyzing chunk frames = [3963904,3964416) | 20260508 00:55:20.250 | INFO | MemInfo: { | 20260508 00:55:20.250 | INFO | "gpu" : " 17 GB", | 20260508 00:55:20.251 | INFO | "huge" : "", | 20260508 00:55:20.251 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:20.251 | INFO | "malloc" : " 14 GB", | 20260508 00:55:20.251 | INFO | "pinned" : " 29 GB" | 20260508 00:55:20.251 | INFO | } | 20260508 00:55:20.251 | INFO | Perf report for the last 5.14038 seconds: | 20260508 00:55:20.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:20.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 00:55:20.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 00:55:20.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:20.251 | INFO | Uploaded 36GiB to GPU in 2.39265s (15.0461GiB/s) | 20260508 00:55:20.251 | INFO | Downloaded 34GiB to GPU in 2.4367s (13.9533GiB/s) | 20260508 00:55:25.391 | INFO | Analyzing chunk frames = [3964416,3964928) | 20260508 00:55:25.391 | INFO | MemInfo: { | 20260508 00:55:25.391 | INFO | "gpu" : " 17 GB", | 20260508 00:55:25.391 | INFO | "huge" : "", | 20260508 00:55:25.391 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:25.391 | INFO | "malloc" : " 14 GB", | 20260508 00:55:25.391 | INFO | "pinned" : " 29 GB" | 20260508 00:55:25.391 | INFO | } | 20260508 00:55:25.391 | INFO | Perf report for the last 5.14041 seconds: | 20260508 00:55:25.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:25.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.391 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 00:55:25.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.391 | INFO | PRE_HQ : 35.3%, 1.0, 18.4ms, 0 | 20260508 00:55:25.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:25.392 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:55:25.392 | INFO | Total mem usage: 16.7 GiB | 20260508 00:55:25.392 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:55:25.392 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260508 00:55:25.392 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:55:25.392 | INFO | High Water Breakdown: | 20260508 00:55:25.392 | INFO | 60.2% : 8.99 GiB ( 87.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:55:25.392 | INFO | 39.7% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:55:25.392 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:55:25.392 | INFO | ------------------------------------------ | 20260508 00:55:25.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:55:25.392 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:55:25.392 | INFO | Uploaded 36GiB to GPU in 2.41663s (14.8968GiB/s) | 20260508 00:55:25.392 | INFO | Total mem usage: 26.5 GiB | 20260508 00:55:25.392 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:55:25.392 | INFO | Downloaded 34GiB to GPU in 2.44302s (13.9172GiB/s) | 20260508 00:55:25.392 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 00:55:25.392 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 00:55:25.392 | INFO | High Water Breakdown: | 20260508 00:55:25.392 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:55:25.392 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:55:25.392 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:55:25.392 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:55:25.392 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:55:25.392 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:55:25.392 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:55:25.392 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:55:25.392 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:55:25.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:55:25.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:55:25.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:55:25.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:55:25.393 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:55:25.393 | INFO | ------------------------------------------ | 20260508 00:55:25.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:55:25.393 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:55:25.393 | INFO | Total mem usage: 27.8 GiB | 20260508 00:55:25.393 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:55:25.393 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:55:25.393 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:55:25.393 | INFO | High Water Breakdown: | 20260508 00:55:25.393 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:55:25.393 | INFO | ------------------------------------------ | 20260508 00:55:25.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:55:25.393 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:55:25.393 | INFO | Total mem usage: 15.7 GiB | 20260508 00:55:25.393 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:55:25.393 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:55:25.393 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:55:25.393 | INFO | High Water Breakdown: | 20260508 00:55:25.393 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:55:25.393 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:55:25.393 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:55:25.393 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:55:25.393 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:55:25.393 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:55:25.393 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:55:25.393 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:55:25.393 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:55:25.393 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:55:25.393 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:55:25.393 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:55:25.393 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:55:25.393 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:55:25.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:55:25.393 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:55:25.393 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:55:25.393 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:55:25.393 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:55:25.393 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:55:25.393 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:55:25.393 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:55:25.393 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:55:25.393 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:55:25.393 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:55:25.393 | INFO | ------------------------------------------ | 20260508 00:55:25.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:55:25.393 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:55:25.393 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:55:25.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:55:25.393 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.41e+01 / 1.40e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.01e-01, 1.13e-02 / 5.62e-03 | 20260508 00:55:25.393 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.48e+00, 5.85e-01 / 2.00e-01 | 20260508 00:55:25.393 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.07e-01 / 1.10e-01 | 20260508 00:55:25.393 | INFO | PerfReport: 16.07%, DME -- 4.22e+01 / 4.62e+01, 4.08e+00 / 1.67e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.61e+00 / 1.60e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.45e+00 / 2.71e+00, 4.28e-01 / 2.60e-01 | 20260508 00:55:25.393 | INFO | PerfReport: 3.37%, Metrics -- 1.11e+00 / 1.27e+00, 2.76e-01 / 1.31e-01 | 20260508 00:55:25.393 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.72e-01, 5.53e-02 / 2.51e-02 | 20260508 00:55:25.393 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:55:25.393 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:55:25.393 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:55:25.393 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 24.16%, Upload -- 2.56e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 24.56%, Download -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 30.90%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 1.93%, LockUpload -- 2.05e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 2.97%, LockDownload -- 3.15e+00 / 0.00e+00, 4.36e+00 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 15.46%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 00:55:25.393 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260508 00:55:25.506 | INFO | Starting a round of Baz IO | 20260508 00:55:28.961 | INFO | Populating RTMetrics report | 20260508 00:55:28.962 | INFO | RTMetrics report populated | 20260508 00:55:28.962 | INFO | Opening tmp RTMetrics json file | 20260508 00:55:28.963 | INFO | Serializing RTMetrics to json | 20260508 00:55:28.968 | INFO | Serialization done, swapping tmp file now... | 20260508 00:55:28.975 | INFO | RTMetrics done | 20260508 00:55:29.116 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.61 seconds | 20260508 00:55:29.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.02% / 0.00% / 57.98% | 20260508 00:55:29.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.933965% / 0% / 4.76837e-05% | 20260508 00:55:29.117 | INFO | Baz IO overall: | 20260508 00:55:29.117 | INFO | BytesWritten : 1799517636728 | 20260508 00:55:29.117 | INFO | NumEvents : 870746726709 | 20260508 00:55:30.527 | INFO | Analyzing chunk frames = [3964928,3965440) | 20260508 00:55:30.527 | INFO | MemInfo: { | 20260508 00:55:30.527 | INFO | "gpu" : " 17 GB", | 20260508 00:55:30.527 | INFO | "huge" : "", | 20260508 00:55:30.527 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:30.527 | INFO | "malloc" : " 14 GB", | 20260508 00:55:30.527 | INFO | "pinned" : " 29 GB" | 20260508 00:55:30.527 | INFO | } | 20260508 00:55:30.527 | INFO | Perf report for the last 5.13618 seconds: | 20260508 00:55:30.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:30.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:30.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:30.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:30.528 | INFO | ANALYSIS : 68.8%, 2.9, 104.4ms, 0 | 20260508 00:55:30.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:30.528 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260508 00:55:30.528 | INFO | PRE_HQ : 67.9%, 1.8, 63.1ms, 0 | 20260508 00:55:30.528 | INFO | BAZWRITER : 69.9%, 4.1, 147.0ms, 0 | 20260508 00:55:30.528 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260508 00:55:30.528 | INFO | Uploaded 36GiB to GPU in 2.60451s (13.8222GiB/s) | 20260508 00:55:30.528 | INFO | Downloaded 38GiB to GPU in 2.73262s (13.9061GiB/s) | 20260508 00:55:35.665 | INFO | Analyzing chunk frames = [3965440,3965952) | 20260508 00:55:35.665 | INFO | MemInfo: { | 20260508 00:55:35.665 | INFO | "gpu" : " 17 GB", | 20260508 00:55:35.665 | INFO | "huge" : "", | 20260508 00:55:35.665 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:35.665 | INFO | "malloc" : " 14 GB", | 20260508 00:55:35.665 | INFO | "pinned" : " 29 GB" | 20260508 00:55:35.665 | INFO | } | 20260508 00:55:35.665 | INFO | Perf report for the last 5.13778 seconds: | 20260508 00:55:35.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:35.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.665 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 00:55:35.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.665 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:55:35.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:35.666 | INFO | Uploaded 36GiB to GPU in 2.54552s (14.1425GiB/s) | 20260508 00:55:35.666 | INFO | Downloaded 34GiB to GPU in 2.49931s (13.6038GiB/s) | 20260508 00:55:40.802 | INFO | Analyzing chunk frames = [3965952,3966464) | 20260508 00:55:40.802 | INFO | MemInfo: { | 20260508 00:55:40.802 | INFO | "gpu" : " 17 GB", | 20260508 00:55:40.802 | INFO | "huge" : "", | 20260508 00:55:40.802 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:40.802 | INFO | "malloc" : " 14 GB", | 20260508 00:55:40.802 | INFO | "pinned" : " 29 GB" | 20260508 00:55:40.802 | INFO | } | 20260508 00:55:40.802 | INFO | Perf report for the last 5.13721 seconds: | 20260508 00:55:40.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:40.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.802 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 00:55:40.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.803 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:55:40.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:40.803 | INFO | Uploaded 36GiB to GPU in 2.46838s (14.5844GiB/s) | 20260508 00:55:40.803 | INFO | Downloaded 34GiB to GPU in 2.50053s (13.5971GiB/s) | 20260508 00:55:45.947 | INFO | Analyzing chunk frames = [3966464,3966976) | 20260508 00:55:45.947 | INFO | MemInfo: { | 20260508 00:55:45.947 | INFO | "gpu" : " 17 GB", | 20260508 00:55:45.947 | INFO | "huge" : "", | 20260508 00:55:45.947 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:45.947 | INFO | "malloc" : " 14 GB", | 20260508 00:55:45.948 | INFO | "pinned" : " 29 GB" | 20260508 00:55:45.948 | INFO | } | 20260508 00:55:45.948 | INFO | Perf report for the last 5.14527 seconds: | 20260508 00:55:45.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:45.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 00:55:45.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 00:55:45.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:45.948 | INFO | Uploaded 36GiB to GPU in 2.45021s (14.6926GiB/s) | 20260508 00:55:45.948 | INFO | Downloaded 34GiB to GPU in 2.49823s (13.6096GiB/s) | 20260508 00:55:51.093 | INFO | Analyzing chunk frames = [3966976,3967488) | 20260508 00:55:51.093 | INFO | MemInfo: { | 20260508 00:55:51.093 | INFO | "gpu" : " 17 GB", | 20260508 00:55:51.093 | INFO | "huge" : "", | 20260508 00:55:51.093 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:51.093 | INFO | "malloc" : " 14 GB", | 20260508 00:55:51.093 | INFO | "pinned" : " 29 GB" | 20260508 00:55:51.093 | INFO | } | 20260508 00:55:51.093 | INFO | Perf report for the last 5.14541 seconds: | 20260508 00:55:51.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:51.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.093 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 00:55:51.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.093 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:55:51.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:51.094 | INFO | Uploaded 36GiB to GPU in 2.28861s (15.73GiB/s) | 20260508 00:55:51.094 | INFO | Downloaded 34GiB to GPU in 2.50114s (13.5938GiB/s) | 20260508 00:55:56.229 | INFO | Analyzing chunk frames = [3967488,3968000) | 20260508 00:55:56.229 | INFO | MemInfo: { | 20260508 00:55:56.229 | INFO | "gpu" : " 17 GB", | 20260508 00:55:56.229 | INFO | "huge" : "", | 20260508 00:55:56.229 | INFO | "huge_shared" : " 29 GB", | 20260508 00:55:56.229 | INFO | "malloc" : " 14 GB", | 20260508 00:55:56.229 | INFO | "pinned" : " 29 GB" | 20260508 00:55:56.229 | INFO | } | 20260508 00:55:56.229 | INFO | Perf report for the last 5.13598 seconds: | 20260508 00:55:56.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:55:56.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.229 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:55:56.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.229 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:55:56.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:55:56.230 | INFO | Uploaded 36GiB to GPU in 2.36654s (15.2121GiB/s) | 20260508 00:55:56.230 | INFO | Downloaded 34GiB to GPU in 2.43519s (13.962GiB/s) | 20260508 00:56:01.361 | INFO | Analyzing chunk frames = [3968000,3968512) | 20260508 00:56:01.361 | INFO | MemInfo: { | 20260508 00:56:01.361 | INFO | "gpu" : " 17 GB", | 20260508 00:56:01.361 | INFO | "huge" : "", | 20260508 00:56:01.361 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:01.361 | INFO | "malloc" : " 14 GB", | 20260508 00:56:01.362 | INFO | "pinned" : " 29 GB" | 20260508 00:56:01.362 | INFO | } | 20260508 00:56:01.362 | INFO | Perf report for the last 5.13244 seconds: | 20260508 00:56:01.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:01.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 00:56:01.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 00:56:01.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:01.362 | INFO | Uploaded 36GiB to GPU in 2.39781s (15.0137GiB/s) | 20260508 00:56:01.362 | INFO | Downloaded 34GiB to GPU in 2.42932s (13.9957GiB/s) | 20260508 00:56:06.502 | INFO | Analyzing chunk frames = [3968512,3969024) | 20260508 00:56:06.502 | INFO | MemInfo: { | 20260508 00:56:06.502 | INFO | "gpu" : " 17 GB", | 20260508 00:56:06.502 | INFO | "huge" : "", | 20260508 00:56:06.502 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:06.503 | INFO | "malloc" : " 14 GB", | 20260508 00:56:06.503 | INFO | "pinned" : " 29 GB" | 20260508 00:56:06.503 | INFO | } | 20260508 00:56:06.503 | INFO | Perf report for the last 5.14106 seconds: | 20260508 00:56:06.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:06.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:56:06.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 00:56:06.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:06.503 | INFO | Uploaded 36GiB to GPU in 2.53746s (14.1874GiB/s) | 20260508 00:56:06.503 | INFO | Downloaded 34GiB to GPU in 2.46786s (13.7771GiB/s) | 20260508 00:56:06.617 | INFO | Starting a round of Baz IO | 20260508 00:56:10.069 | INFO | Populating RTMetrics report | 20260508 00:56:10.069 | INFO | RTMetrics report populated | 20260508 00:56:10.069 | INFO | Opening tmp RTMetrics json file | 20260508 00:56:10.079 | INFO | Serializing RTMetrics to json | 20260508 00:56:10.095 | INFO | Serialization done, swapping tmp file now... | 20260508 00:56:10.096 | INFO | RTMetrics done | 20260508 00:56:10.167 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.549 seconds | 20260508 00:56:10.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.01% / 0.00% / 57.98% | 20260508 00:56:10.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.93058% / 7.94729e-06% / 7.54992e-05% | 20260508 00:56:10.167 | INFO | Baz IO overall: | 20260508 00:56:10.167 | INFO | BytesWritten : 1801316932473 | 20260508 00:56:10.167 | INFO | NumEvents : 871611515828 | 20260508 00:56:11.643 | INFO | Analyzing chunk frames = [3969024,3969536) | 20260508 00:56:11.643 | INFO | MemInfo: { | 20260508 00:56:11.643 | INFO | "gpu" : " 17 GB", | 20260508 00:56:11.643 | INFO | "huge" : "", | 20260508 00:56:11.643 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:11.643 | INFO | "malloc" : " 14 GB", | 20260508 00:56:11.643 | INFO | "pinned" : " 29 GB" | 20260508 00:56:11.643 | INFO | } | 20260508 00:56:11.643 | INFO | Perf report for the last 5.14063 seconds: | 20260508 00:56:11.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:11.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:11.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:11.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:11.643 | INFO | ANALYSIS : 68.7%, 2.9, 104.3ms, 0 | 20260508 00:56:11.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:11.643 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260508 00:56:11.643 | INFO | PRE_HQ : 67.6%, 1.8, 63.1ms, 0 | 20260508 00:56:11.643 | INFO | BAZWRITER : 68.7%, 4.0, 141.1ms, 0 | 20260508 00:56:11.643 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 00:56:11.644 | INFO | Uploaded 36GiB to GPU in 2.60548s (13.817GiB/s) | 20260508 00:56:11.644 | INFO | Downloaded 38GiB to GPU in 2.70554s (14.0452GiB/s) | 20260508 00:56:16.782 | INFO | Analyzing chunk frames = [3969536,3970048) | 20260508 00:56:16.782 | INFO | MemInfo: { | 20260508 00:56:16.782 | INFO | "gpu" : " 17 GB", | 20260508 00:56:16.782 | INFO | "huge" : "", | 20260508 00:56:16.782 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:16.782 | INFO | "malloc" : " 14 GB", | 20260508 00:56:16.782 | INFO | "pinned" : " 29 GB" | 20260508 00:56:16.782 | INFO | } | 20260508 00:56:16.782 | INFO | Perf report for the last 5.13886 seconds: | 20260508 00:56:16.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:16.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 00:56:16.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 00:56:16.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:16.782 | INFO | Uploaded 36GiB to GPU in 2.49775s (14.413GiB/s) | 20260508 00:56:16.783 | INFO | Downloaded 34GiB to GPU in 2.44241s (13.9207GiB/s) | 20260508 00:56:21.923 | INFO | Analyzing chunk frames = [3970048,3970560) | 20260508 00:56:21.923 | INFO | MemInfo: { | 20260508 00:56:21.923 | INFO | "gpu" : " 17 GB", | 20260508 00:56:21.923 | INFO | "huge" : "", | 20260508 00:56:21.923 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:21.923 | INFO | "malloc" : " 14 GB", | 20260508 00:56:21.923 | INFO | "pinned" : " 29 GB" | 20260508 00:56:21.923 | INFO | } | 20260508 00:56:21.924 | INFO | Perf report for the last 5.14131 seconds: | 20260508 00:56:21.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:21.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 00:56:21.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 00:56:21.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:21.924 | INFO | Uploaded 36GiB to GPU in 2.46198s (14.6224GiB/s) | 20260508 00:56:21.924 | INFO | Downloaded 34GiB to GPU in 2.49119s (13.6481GiB/s) | 20260508 00:56:27.057 | INFO | Analyzing chunk frames = [3970560,3971072) | 20260508 00:56:27.057 | INFO | MemInfo: { | 20260508 00:56:27.058 | INFO | "gpu" : " 17 GB", | 20260508 00:56:27.058 | INFO | "huge" : "", | 20260508 00:56:27.058 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:27.058 | INFO | "malloc" : " 14 GB", | 20260508 00:56:27.058 | INFO | "pinned" : " 29 GB" | 20260508 00:56:27.058 | INFO | } | 20260508 00:56:27.058 | INFO | Perf report for the last 5.13424 seconds: | 20260508 00:56:27.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:27.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 00:56:27.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 00:56:27.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:27.058 | INFO | Uploaded 36GiB to GPU in 2.45675s (14.6535GiB/s) | 20260508 00:56:27.058 | INFO | Downloaded 34GiB to GPU in 2.50404s (13.5781GiB/s) | 20260508 00:56:32.203 | INFO | Analyzing chunk frames = [3971072,3971584) | 20260508 00:56:32.203 | INFO | MemInfo: { | 20260508 00:56:32.203 | INFO | "gpu" : " 17 GB", | 20260508 00:56:32.203 | INFO | "huge" : "", | 20260508 00:56:32.203 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:32.203 | INFO | "malloc" : " 14 GB", | 20260508 00:56:32.203 | INFO | "pinned" : " 29 GB" | 20260508 00:56:32.203 | INFO | } | 20260508 00:56:32.203 | INFO | Perf report for the last 5.14524 seconds: | 20260508 00:56:32.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:32.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 00:56:32.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:56:32.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:32.203 | INFO | Uploaded 36GiB to GPU in 2.29121s (15.7122GiB/s) | 20260508 00:56:32.203 | INFO | Downloaded 34GiB to GPU in 2.48559s (13.6788GiB/s) | 20260508 00:56:37.340 | INFO | Analyzing chunk frames = [3971584,3972096) | 20260508 00:56:37.340 | INFO | MemInfo: { | 20260508 00:56:37.340 | INFO | "gpu" : " 17 GB", | 20260508 00:56:37.340 | INFO | "huge" : "", | 20260508 00:56:37.340 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:37.340 | INFO | "malloc" : " 14 GB", | 20260508 00:56:37.340 | INFO | "pinned" : " 29 GB" | 20260508 00:56:37.340 | INFO | } | 20260508 00:56:37.340 | INFO | Perf report for the last 5.13694 seconds: | 20260508 00:56:37.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:37.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:56:37.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 00:56:37.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:37.340 | INFO | Uploaded 36GiB to GPU in 2.38981s (15.064GiB/s) | 20260508 00:56:37.340 | INFO | Downloaded 34GiB to GPU in 2.41764s (14.0633GiB/s) | 20260508 00:56:42.482 | INFO | Analyzing chunk frames = [3972096,3972608) | 20260508 00:56:42.482 | INFO | MemInfo: { | 20260508 00:56:42.482 | INFO | "gpu" : " 17 GB", | 20260508 00:56:42.482 | INFO | "huge" : "", | 20260508 00:56:42.482 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:42.483 | INFO | "malloc" : " 14 GB", | 20260508 00:56:42.483 | INFO | "pinned" : " 29 GB" | 20260508 00:56:42.483 | INFO | } | 20260508 00:56:42.483 | INFO | Perf report for the last 5.1426 seconds: | 20260508 00:56:42.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:42.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 00:56:42.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 00:56:42.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:42.483 | INFO | Uploaded 36GiB to GPU in 2.51879s (14.2926GiB/s) | 20260508 00:56:42.483 | INFO | Downloaded 34GiB to GPU in 2.45978s (13.8223GiB/s) | 20260508 00:56:47.614 | INFO | Analyzing chunk frames = [3972608,3973120) | 20260508 00:56:47.614 | INFO | MemInfo: { | 20260508 00:56:47.614 | INFO | "gpu" : " 17 GB", | 20260508 00:56:47.614 | INFO | "huge" : "", | 20260508 00:56:47.614 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:47.614 | INFO | "malloc" : " 14 GB", | 20260508 00:56:47.614 | INFO | "pinned" : " 29 GB" | 20260508 00:56:47.614 | INFO | } | 20260508 00:56:47.614 | INFO | Perf report for the last 5.13147 seconds: | 20260508 00:56:47.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:47.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.614 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 00:56:47.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.614 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 00:56:47.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:47.615 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:56:47.615 | INFO | Total mem usage: 16.7 GiB | 20260508 00:56:47.615 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:56:47.615 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.47%) | 20260508 00:56:47.615 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:56:47.615 | INFO | High Water Breakdown: | 20260508 00:56:47.615 | INFO | 60.6% : 9.33 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:56:47.615 | INFO | 39.2% : 6.03 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:56:47.615 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:56:47.615 | INFO | ------------------------------------------ | 20260508 00:56:47.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:56:47.615 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:56:47.615 | INFO | Uploaded 36GiB to GPU in 2.51929s (14.2897GiB/s) | 20260508 00:56:47.615 | INFO | Downloaded 34GiB to GPU in 2.45627s (13.8421GiB/s) | 20260508 00:56:47.615 | INFO | Total mem usage: 26.5 GiB | 20260508 00:56:47.615 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:56:47.615 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 00:56:47.615 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 00:56:47.615 | INFO | High Water Breakdown: | 20260508 00:56:47.615 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:56:47.615 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:56:47.615 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:56:47.615 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:56:47.615 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:56:47.615 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:56:47.615 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:56:47.615 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:56:47.615 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:56:47.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:56:47.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:56:47.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:56:47.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:56:47.615 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:56:47.615 | INFO | ------------------------------------------ | 20260508 00:56:47.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:56:47.615 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:56:47.615 | INFO | Total mem usage: 27.8 GiB | 20260508 00:56:47.615 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:56:47.615 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:56:47.615 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:56:47.615 | INFO | High Water Breakdown: | 20260508 00:56:47.615 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:56:47.615 | INFO | ------------------------------------------ | 20260508 00:56:47.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:56:47.615 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:56:47.615 | INFO | Total mem usage: 15.7 GiB | 20260508 00:56:47.615 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:56:47.615 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:56:47.615 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:56:47.615 | INFO | High Water Breakdown: | 20260508 00:56:47.615 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:56:47.615 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:56:47.615 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:56:47.615 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:56:47.615 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:56:47.615 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:56:47.616 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:56:47.616 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:56:47.616 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:56:47.616 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:56:47.616 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:56:47.616 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:56:47.616 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:56:47.616 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:56:47.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:56:47.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:56:47.616 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:56:47.616 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:56:47.616 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:56:47.616 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:56:47.616 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:56:47.616 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:56:47.616 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:56:47.616 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:56:47.616 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:56:47.616 | INFO | ------------------------------------------ | 20260508 00:56:47.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:56:47.616 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:56:47.616 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:56:47.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:56:47.616 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.39e+01 / 1.34e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.02e-01, 1.14e-02 / 6.20e-03 | 20260508 00:56:47.616 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.47e+00, 5.85e-01 / 1.93e-01 | 20260508 00:56:47.616 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.50e+00, 2.07e-01 / 1.04e-01 | 20260508 00:56:47.616 | INFO | PerfReport: 16.10%, DME -- 4.22e+01 / 4.62e+01, 4.12e+00 / 1.49e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.58e+00 / 1.38e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 2.68e+00, 4.32e-01 / 2.33e-01 | 20260508 00:56:47.616 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.27e+00, 2.72e-01 / 1.08e-01 | 20260508 00:56:47.616 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.69e-01, 5.38e-02 / 2.59e-02 | 20260508 00:56:47.616 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:56:47.616 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:56:47.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:56:47.616 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 24.16%, Upload -- 2.57e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 24.55%, Download -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 30.86%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 1.91%, LockUpload -- 2.02e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 3.02%, LockDownload -- 3.20e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 15.49%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:56:47.616 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.54e-02 / 0.00e+00 | 20260508 00:56:47.712 | INFO | Starting a round of Baz IO | 20260508 00:56:51.171 | INFO | Populating RTMetrics report | 20260508 00:56:51.171 | INFO | RTMetrics report populated | 20260508 00:56:51.171 | INFO | Opening tmp RTMetrics json file | 20260508 00:56:51.196 | INFO | Serializing RTMetrics to json | 20260508 00:56:51.197 | INFO | Serialization done, swapping tmp file now... | 20260508 00:56:51.199 | INFO | RTMetrics done | 20260508 00:56:51.268 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.556 seconds | 20260508 00:56:51.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.01% / 0.00% / 57.99% | 20260508 00:56:51.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.929972% / 3.97364e-06% / 6.75519e-05% | 20260508 00:56:51.269 | INFO | Baz IO overall: | 20260508 00:56:51.269 | INFO | BytesWritten : 1803117108014 | 20260508 00:56:51.269 | INFO | NumEvents : 872476927230 | 20260508 00:56:52.758 | INFO | Analyzing chunk frames = [3973120,3973632) | 20260508 00:56:52.758 | INFO | MemInfo: { | 20260508 00:56:52.758 | INFO | "gpu" : " 17 GB", | 20260508 00:56:52.758 | INFO | "huge" : "", | 20260508 00:56:52.758 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:52.758 | INFO | "malloc" : " 14 GB", | 20260508 00:56:52.758 | INFO | "pinned" : " 29 GB" | 20260508 00:56:52.758 | INFO | } | 20260508 00:56:52.758 | INFO | Perf report for the last 5.14431 seconds: | 20260508 00:56:52.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:52.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:52.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:52.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:52.758 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 00:56:52.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:52.759 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 00:56:52.759 | INFO | PRE_HQ : 65.4%, 1.7, 56.1ms, 0 | 20260508 00:56:52.759 | INFO | BAZWRITER : 68.7%, 4.5, 158.3ms, 0 | 20260508 00:56:52.759 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 00:56:52.759 | INFO | Uploaded 36GiB to GPU in 2.60671s (13.8105GiB/s) | 20260508 00:56:52.759 | INFO | Downloaded 38GiB to GPU in 2.68647s (14.1449GiB/s) | 20260508 00:56:57.899 | INFO | Analyzing chunk frames = [3973632,3974144) | 20260508 00:56:57.899 | INFO | MemInfo: { | 20260508 00:56:57.899 | INFO | "gpu" : " 17 GB", | 20260508 00:56:57.899 | INFO | "huge" : "", | 20260508 00:56:57.900 | INFO | "huge_shared" : " 29 GB", | 20260508 00:56:57.900 | INFO | "malloc" : " 14 GB", | 20260508 00:56:57.900 | INFO | "pinned" : " 29 GB" | 20260508 00:56:57.900 | INFO | } | 20260508 00:56:57.900 | INFO | Perf report for the last 5.14124 seconds: | 20260508 00:56:57.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:56:57.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 00:56:57.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 00:56:57.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:56:57.900 | INFO | Uploaded 36GiB to GPU in 2.55103s (14.1119GiB/s) | 20260508 00:56:57.900 | INFO | Downloaded 34GiB to GPU in 2.42065s (14.0458GiB/s) | 20260508 00:57:03.041 | INFO | Analyzing chunk frames = [3974144,3974656) | 20260508 00:57:03.041 | INFO | MemInfo: { | 20260508 00:57:03.041 | INFO | "gpu" : " 17 GB", | 20260508 00:57:03.041 | INFO | "huge" : "", | 20260508 00:57:03.041 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:03.041 | INFO | "malloc" : " 14 GB", | 20260508 00:57:03.041 | INFO | "pinned" : " 29 GB" | 20260508 00:57:03.041 | INFO | } | 20260508 00:57:03.041 | INFO | Perf report for the last 5.14138 seconds: | 20260508 00:57:03.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:03.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 00:57:03.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:57:03.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:03.041 | INFO | Uploaded 36GiB to GPU in 2.54024s (14.1719GiB/s) | 20260508 00:57:03.041 | INFO | Downloaded 34GiB to GPU in 2.47998s (13.7098GiB/s) | 20260508 00:57:08.176 | INFO | Analyzing chunk frames = [3974656,3975168) | 20260508 00:57:08.176 | INFO | MemInfo: { | 20260508 00:57:08.176 | INFO | "gpu" : " 17 GB", | 20260508 00:57:08.176 | INFO | "huge" : "", | 20260508 00:57:08.177 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:08.177 | INFO | "malloc" : " 14 GB", | 20260508 00:57:08.177 | INFO | "pinned" : " 29 GB" | 20260508 00:57:08.177 | INFO | } | 20260508 00:57:08.177 | INFO | Perf report for the last 5.13556 seconds: | 20260508 00:57:08.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:08.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 00:57:08.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:57:08.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:08.177 | INFO | Uploaded 36GiB to GPU in 2.54638s (14.1377GiB/s) | 20260508 00:57:08.177 | INFO | Downloaded 34GiB to GPU in 2.47694s (13.7266GiB/s) | 20260508 00:57:13.314 | INFO | Analyzing chunk frames = [3975168,3975680) | 20260508 00:57:13.314 | INFO | MemInfo: { | 20260508 00:57:13.314 | INFO | "gpu" : " 17 GB", | 20260508 00:57:13.315 | INFO | "huge" : "", | 20260508 00:57:13.315 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:13.315 | INFO | "malloc" : " 14 GB", | 20260508 00:57:13.315 | INFO | "pinned" : " 29 GB" | 20260508 00:57:13.315 | INFO | } | 20260508 00:57:13.315 | INFO | Perf report for the last 5.13815 seconds: | 20260508 00:57:13.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:13.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 00:57:13.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:57:13.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:13.315 | INFO | Uploaded 36GiB to GPU in 2.39222s (15.0488GiB/s) | 20260508 00:57:13.315 | INFO | Downloaded 34GiB to GPU in 2.52401s (13.4706GiB/s) | 20260508 00:57:18.455 | INFO | Analyzing chunk frames = [3975680,3976192) | 20260508 00:57:18.455 | INFO | MemInfo: { | 20260508 00:57:18.455 | INFO | "gpu" : " 17 GB", | 20260508 00:57:18.455 | INFO | "huge" : "", | 20260508 00:57:18.455 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:18.455 | INFO | "malloc" : " 14 GB", | 20260508 00:57:18.455 | INFO | "pinned" : " 29 GB" | 20260508 00:57:18.455 | INFO | } | 20260508 00:57:18.455 | INFO | Perf report for the last 5.14041 seconds: | 20260508 00:57:18.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:18.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.455 | INFO | ANALYSIS : 66.4%, 3.0, 102.4ms, 0 | 20260508 00:57:18.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.456 | INFO | PRE_HQ : 32.7%, 1.0, 16.8ms, 0 | 20260508 00:57:18.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:18.456 | INFO | Uploaded 36GiB to GPU in 2.42669s (14.835GiB/s) | 20260508 00:57:18.456 | INFO | Downloaded 34GiB to GPU in 2.41263s (14.0925GiB/s) | 20260508 00:57:23.595 | INFO | Analyzing chunk frames = [3976192,3976704) | 20260508 00:57:23.595 | INFO | MemInfo: { | 20260508 00:57:23.595 | INFO | "gpu" : " 17 GB", | 20260508 00:57:23.595 | INFO | "huge" : "", | 20260508 00:57:23.595 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:23.595 | INFO | "malloc" : " 14 GB", | 20260508 00:57:23.595 | INFO | "pinned" : " 29 GB" | 20260508 00:57:23.595 | INFO | } | 20260508 00:57:23.595 | INFO | Perf report for the last 5.14001 seconds: | 20260508 00:57:23.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:23.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.595 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 00:57:23.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.595 | INFO | PRE_HQ : 35.0%, 1.0, 18.1ms, 0 | 20260508 00:57:23.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:23.596 | INFO | Uploaded 36GiB to GPU in 2.45045s (14.6912GiB/s) | 20260508 00:57:23.596 | INFO | Downloaded 34GiB to GPU in 2.41847s (14.0585GiB/s) | 20260508 00:57:28.730 | INFO | Analyzing chunk frames = [3976704,3977216) | 20260508 00:57:28.730 | INFO | MemInfo: { | 20260508 00:57:28.730 | INFO | "gpu" : " 17 GB", | 20260508 00:57:28.730 | INFO | "huge" : "", | 20260508 00:57:28.730 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:28.730 | INFO | "malloc" : " 14 GB", | 20260508 00:57:28.730 | INFO | "pinned" : " 29 GB" | 20260508 00:57:28.730 | INFO | } | 20260508 00:57:28.730 | INFO | Perf report for the last 5.13464 seconds: | 20260508 00:57:28.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:28.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.730 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:57:28.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.730 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 00:57:28.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:28.731 | INFO | Uploaded 36GiB to GPU in 2.57371s (13.9876GiB/s) | 20260508 00:57:28.731 | INFO | Downloaded 34GiB to GPU in 2.45957s (13.8235GiB/s) | 20260508 00:57:28.844 | INFO | Starting a round of Baz IO | 20260508 00:57:32.295 | INFO | Populating RTMetrics report | 20260508 00:57:32.296 | INFO | RTMetrics report populated | 20260508 00:57:32.296 | INFO | Opening tmp RTMetrics json file | 20260508 00:57:32.308 | INFO | Serializing RTMetrics to json | 20260508 00:57:32.322 | INFO | Serialization done, swapping tmp file now... | 20260508 00:57:32.323 | INFO | RTMetrics done | 20260508 00:57:32.399 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.554 seconds | 20260508 00:57:32.399 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.00% / 0.00% / 57.99% | 20260508 00:57:32.399 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.926514% / 3.97364e-06% / 7.15256e-05% | 20260508 00:57:32.399 | INFO | Baz IO overall: | 20260508 00:57:32.399 | INFO | BytesWritten : 1804916389653 | 20260508 00:57:32.399 | INFO | NumEvents : 873341720099 | 20260508 00:57:33.868 | INFO | Analyzing chunk frames = [3977216,3977728) | 20260508 00:57:33.868 | INFO | MemInfo: { | 20260508 00:57:33.868 | INFO | "gpu" : " 17 GB", | 20260508 00:57:33.868 | INFO | "huge" : "", | 20260508 00:57:33.868 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:33.868 | INFO | "malloc" : " 14 GB", | 20260508 00:57:33.868 | INFO | "pinned" : " 29 GB" | 20260508 00:57:33.868 | INFO | } | 20260508 00:57:33.868 | INFO | Perf report for the last 5.13812 seconds: | 20260508 00:57:33.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:33.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:33.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:33.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:33.868 | INFO | ANALYSIS : 68.7%, 2.9, 104.1ms, 0 | 20260508 00:57:33.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:33.868 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 00:57:33.868 | INFO | PRE_HQ : 67.5%, 1.8, 62.9ms, 0 | 20260508 00:57:33.868 | INFO | BAZWRITER : 68.7%, 3.8, 134.4ms, 0 | 20260508 00:57:33.868 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 00:57:33.868 | INFO | Uploaded 36GiB to GPU in 2.58598s (13.9212GiB/s) | 20260508 00:57:33.868 | INFO | Downloaded 38GiB to GPU in 2.70271s (14.06GiB/s) | 20260508 00:57:39.008 | INFO | Analyzing chunk frames = [3977728,3978240) | 20260508 00:57:39.008 | INFO | MemInfo: { | 20260508 00:57:39.008 | INFO | "gpu" : " 17 GB", | 20260508 00:57:39.008 | INFO | "huge" : "", | 20260508 00:57:39.008 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:39.008 | INFO | "malloc" : " 14 GB", | 20260508 00:57:39.008 | INFO | "pinned" : " 29 GB" | 20260508 00:57:39.008 | INFO | } | 20260508 00:57:39.008 | INFO | Perf report for the last 5.14036 seconds: | 20260508 00:57:39.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:39.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.008 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 00:57:39.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.009 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 00:57:39.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:39.009 | INFO | Uploaded 36GiB to GPU in 2.56619s (14.0286GiB/s) | 20260508 00:57:39.009 | INFO | Downloaded 34GiB to GPU in 2.49089s (13.6498GiB/s) | 20260508 00:57:44.154 | INFO | Analyzing chunk frames = [3978240,3978752) | 20260508 00:57:44.154 | INFO | MemInfo: { | 20260508 00:57:44.154 | INFO | "gpu" : " 17 GB", | 20260508 00:57:44.154 | INFO | "huge" : "", | 20260508 00:57:44.154 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:44.154 | INFO | "malloc" : " 14 GB", | 20260508 00:57:44.154 | INFO | "pinned" : " 29 GB" | 20260508 00:57:44.155 | INFO | } | 20260508 00:57:44.155 | INFO | Perf report for the last 5.14625 seconds: | 20260508 00:57:44.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:44.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 00:57:44.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:57:44.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:44.155 | INFO | Uploaded 36GiB to GPU in 2.49492s (14.4293GiB/s) | 20260508 00:57:44.155 | INFO | Downloaded 34GiB to GPU in 2.49176s (13.645GiB/s) | 20260508 00:57:49.296 | INFO | Analyzing chunk frames = [3978752,3979264) | 20260508 00:57:49.296 | INFO | MemInfo: { | 20260508 00:57:49.296 | INFO | "gpu" : " 17 GB", | 20260508 00:57:49.296 | INFO | "huge" : "", | 20260508 00:57:49.296 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:49.296 | INFO | "malloc" : " 14 GB", | 20260508 00:57:49.296 | INFO | "pinned" : " 29 GB" | 20260508 00:57:49.296 | INFO | } | 20260508 00:57:49.296 | INFO | Perf report for the last 5.14151 seconds: | 20260508 00:57:49.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:49.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 00:57:49.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 00:57:49.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:49.296 | INFO | Uploaded 36GiB to GPU in 2.51146s (14.3343GiB/s) | 20260508 00:57:49.297 | INFO | Downloaded 34GiB to GPU in 2.49827s (13.6094GiB/s) | 20260508 00:57:54.433 | INFO | Analyzing chunk frames = [3979264,3979776) | 20260508 00:57:54.433 | INFO | MemInfo: { | 20260508 00:57:54.433 | INFO | "gpu" : " 17 GB", | 20260508 00:57:54.433 | INFO | "huge" : "", | 20260508 00:57:54.433 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:54.433 | INFO | "malloc" : " 14 GB", | 20260508 00:57:54.433 | INFO | "pinned" : " 29 GB" | 20260508 00:57:54.433 | INFO | } | 20260508 00:57:54.433 | INFO | Perf report for the last 5.13663 seconds: | 20260508 00:57:54.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:54.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 00:57:54.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 00:57:54.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:54.433 | INFO | Uploaded 36GiB to GPU in 2.35083s (15.3138GiB/s) | 20260508 00:57:54.433 | INFO | Downloaded 34GiB to GPU in 2.53252s (13.4254GiB/s) | 20260508 00:57:59.570 | INFO | Analyzing chunk frames = [3979776,3980288) | 20260508 00:57:59.570 | INFO | MemInfo: { | 20260508 00:57:59.570 | INFO | "gpu" : " 17 GB", | 20260508 00:57:59.570 | INFO | "huge" : "", | 20260508 00:57:59.570 | INFO | "huge_shared" : " 29 GB", | 20260508 00:57:59.570 | INFO | "malloc" : " 14 GB", | 20260508 00:57:59.570 | INFO | "pinned" : " 29 GB" | 20260508 00:57:59.570 | INFO | } | 20260508 00:57:59.572 | INFO | Perf report for the last 5.13874 seconds: | 20260508 00:57:59.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:57:59.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:57:59.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 00:57:59.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:57:59.572 | INFO | Uploaded 36GiB to GPU in 2.35365s (15.2954GiB/s) | 20260508 00:57:59.572 | INFO | Downloaded 34GiB to GPU in 2.44306s (13.917GiB/s) | 20260508 00:58:04.711 | INFO | Analyzing chunk frames = [3980288,3980800) | 20260508 00:58:04.711 | INFO | MemInfo: { | 20260508 00:58:04.711 | INFO | "gpu" : " 17 GB", | 20260508 00:58:04.711 | INFO | "huge" : "", | 20260508 00:58:04.711 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:04.711 | INFO | "malloc" : " 14 GB", | 20260508 00:58:04.711 | INFO | "pinned" : " 29 GB" | 20260508 00:58:04.711 | INFO | } | 20260508 00:58:04.711 | INFO | Perf report for the last 5.1398 seconds: | 20260508 00:58:04.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:04.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.711 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 00:58:04.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.712 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260508 00:58:04.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:04.712 | INFO | Uploaded 36GiB to GPU in 2.39154s (15.0531GiB/s) | 20260508 00:58:04.712 | INFO | Downloaded 34GiB to GPU in 2.43628s (13.9557GiB/s) | 20260508 00:58:09.849 | INFO | Analyzing chunk frames = [3980800,3981312) | 20260508 00:58:09.849 | INFO | MemInfo: { | 20260508 00:58:09.849 | INFO | "gpu" : " 17 GB", | 20260508 00:58:09.849 | INFO | "huge" : "", | 20260508 00:58:09.849 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:09.849 | INFO | "malloc" : " 14 GB", | 20260508 00:58:09.849 | INFO | "pinned" : " 29 GB" | 20260508 00:58:09.849 | INFO | } | 20260508 00:58:09.849 | INFO | Perf report for the last 5.13773 seconds: | 20260508 00:58:09.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:09.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 00:58:09.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:58:09.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:09.849 | INFO | Uploaded 36GiB to GPU in 2.48274s (14.5001GiB/s) | 20260508 00:58:09.850 | INFO | Downloaded 34GiB to GPU in 2.46535s (13.7912GiB/s) | 20260508 00:58:09.850 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:58:09.850 | INFO | Total mem usage: 16.7 GiB | 20260508 00:58:09.850 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:58:09.850 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.1%) | 20260508 00:58:09.850 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:58:09.850 | INFO | High Water Breakdown: | 20260508 00:58:09.850 | INFO | 60.4% : 9.15 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:58:09.850 | INFO | 39.5% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:58:09.850 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:58:09.850 | INFO | ------------------------------------------ | 20260508 00:58:09.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:58:09.850 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:58:09.850 | INFO | Total mem usage: 26.5 GiB | 20260508 00:58:09.850 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:58:09.850 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260508 00:58:09.850 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260508 00:58:09.850 | INFO | High Water Breakdown: | 20260508 00:58:09.850 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:58:09.850 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:58:09.850 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:58:09.850 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:58:09.850 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:58:09.850 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:58:09.850 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:58:09.850 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:58:09.850 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:58:09.850 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:58:09.850 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:58:09.850 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:58:09.850 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:58:09.850 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:58:09.850 | INFO | ------------------------------------------ | 20260508 00:58:09.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:58:09.850 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:58:09.850 | INFO | Total mem usage: 27.8 GiB | 20260508 00:58:09.850 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:58:09.850 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:58:09.850 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:58:09.850 | INFO | High Water Breakdown: | 20260508 00:58:09.850 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:58:09.850 | INFO | ------------------------------------------ | 20260508 00:58:09.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:58:09.850 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:58:09.850 | INFO | Total mem usage: 15.7 GiB | 20260508 00:58:09.850 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:58:09.850 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:58:09.850 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:58:09.850 | INFO | High Water Breakdown: | 20260508 00:58:09.850 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:58:09.850 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:58:09.850 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:58:09.850 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:58:09.850 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:58:09.850 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:58:09.850 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:58:09.850 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:58:09.850 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:58:09.850 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:58:09.850 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:58:09.850 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:58:09.850 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:58:09.850 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:58:09.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:58:09.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:58:09.850 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:58:09.850 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:58:09.850 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:58:09.850 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:58:09.850 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:58:09.850 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:58:09.850 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:58:09.850 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:58:09.850 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:58:09.850 | INFO | ------------------------------------------ | 20260508 00:58:09.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:58:09.850 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:58:09.850 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:58:09.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:58:09.850 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.71e+00 | 20260508 00:58:09.850 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.14e-02 / 5.49e-03 | 20260508 00:58:09.850 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.47e+00, 5.87e-01 / 2.11e-01 | 20260508 00:58:09.850 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.13e-01 | 20260508 00:58:09.850 | INFO | PerfReport: 16.13%, DME -- 4.23e+01 / 4.62e+01, 4.40e+00 / 2.03e+00 | 20260508 00:58:09.850 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.59e+00 / 1.63e+00 | 20260508 00:58:09.850 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.45e+00 / 2.74e+00, 4.26e-01 / 2.30e-01 | 20260508 00:58:09.850 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.28e+00, 2.73e-01 / 1.31e-01 | 20260508 00:58:09.850 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.40e-01 / 3.75e-01, 5.46e-02 / 2.92e-02 | 20260508 00:58:09.850 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:58:09.851 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:58:09.851 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:58:09.851 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 2.16%, LockUpload -- 2.30e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 2.74%, LockDownload -- 2.91e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 15.54%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 00:58:09.851 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260508 00:58:09.946 | INFO | Starting a round of Baz IO | 20260508 00:58:13.405 | INFO | Populating RTMetrics report | 20260508 00:58:13.405 | INFO | RTMetrics report populated | 20260508 00:58:13.405 | INFO | Opening tmp RTMetrics json file | 20260508 00:58:13.436 | INFO | Serializing RTMetrics to json | 20260508 00:58:13.477 | INFO | Serialization done, swapping tmp file now... | 20260508 00:58:13.478 | INFO | RTMetrics done | 20260508 00:58:13.499 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.552 seconds | 20260508 00:58:13.499 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.00% / 0.00% / 58.00% | 20260508 00:58:13.499 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.926681% / 3.97364e-06% / 7.54992e-05% | 20260508 00:58:13.499 | INFO | Baz IO overall: | 20260508 00:58:13.499 | INFO | BytesWritten : 1806716050706 | 20260508 00:58:13.499 | INFO | NumEvents : 874206825177 | 20260508 00:58:14.992 | INFO | Analyzing chunk frames = [3981312,3981824) | 20260508 00:58:14.992 | INFO | MemInfo: { | 20260508 00:58:14.992 | INFO | "gpu" : " 17 GB", | 20260508 00:58:14.992 | INFO | "huge" : "", | 20260508 00:58:14.992 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:14.992 | INFO | "malloc" : " 14 GB", | 20260508 00:58:14.992 | INFO | "pinned" : " 29 GB" | 20260508 00:58:14.992 | INFO | } | 20260508 00:58:14.992 | INFO | Perf report for the last 5.14261 seconds: | 20260508 00:58:14.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:14.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:14.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:14.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:14.992 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 00:58:14.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:14.992 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260508 00:58:14.992 | INFO | PRE_HQ : 65.3%, 1.7, 56.7ms, 0 | 20260508 00:58:14.992 | INFO | BAZWRITER : 68.7%, 3.8, 132.9ms, 0 | 20260508 00:58:14.992 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 00:58:14.992 | INFO | Uploaded 36GiB to GPU in 2.51081s (14.338GiB/s) | 20260508 00:58:14.992 | INFO | Downloaded 38GiB to GPU in 2.69013s (14.1257GiB/s) | 20260508 00:58:20.124 | INFO | Analyzing chunk frames = [3981824,3982336) | 20260508 00:58:20.125 | INFO | MemInfo: { | 20260508 00:58:20.125 | INFO | "gpu" : " 17 GB", | 20260508 00:58:20.125 | INFO | "huge" : "", | 20260508 00:58:20.125 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:20.125 | INFO | "malloc" : " 14 GB", | 20260508 00:58:20.125 | INFO | "pinned" : " 29 GB" | 20260508 00:58:20.125 | INFO | } | 20260508 00:58:20.125 | INFO | Perf report for the last 5.1329 seconds: | 20260508 00:58:20.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:20.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260508 00:58:20.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 00:58:20.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:20.125 | INFO | Uploaded 36GiB to GPU in 2.51929s (14.2897GiB/s) | 20260508 00:58:20.125 | INFO | Downloaded 34GiB to GPU in 2.52245s (13.479GiB/s) | 20260508 00:58:25.264 | INFO | Analyzing chunk frames = [3982336,3982848) | 20260508 00:58:25.264 | INFO | MemInfo: { | 20260508 00:58:25.264 | INFO | "gpu" : " 17 GB", | 20260508 00:58:25.264 | INFO | "huge" : "", | 20260508 00:58:25.264 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:25.264 | INFO | "malloc" : " 14 GB", | 20260508 00:58:25.264 | INFO | "pinned" : " 29 GB" | 20260508 00:58:25.264 | INFO | } | 20260508 00:58:25.264 | INFO | Perf report for the last 5.13917 seconds: | 20260508 00:58:25.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:25.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 00:58:25.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:58:25.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:25.264 | INFO | Uploaded 36GiB to GPU in 2.48006s (14.5158GiB/s) | 20260508 00:58:25.264 | INFO | Downloaded 34GiB to GPU in 2.50466s (13.5747GiB/s) | 20260508 00:58:30.407 | INFO | Analyzing chunk frames = [3982848,3983360) | 20260508 00:58:30.407 | INFO | MemInfo: { | 20260508 00:58:30.407 | INFO | "gpu" : " 17 GB", | 20260508 00:58:30.407 | INFO | "huge" : "", | 20260508 00:58:30.407 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:30.407 | INFO | "malloc" : " 14 GB", | 20260508 00:58:30.407 | INFO | "pinned" : " 29 GB" | 20260508 00:58:30.407 | INFO | } | 20260508 00:58:30.407 | INFO | Perf report for the last 5.14295 seconds: | 20260508 00:58:30.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:30.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 00:58:30.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:58:30.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:30.407 | INFO | Uploaded 36GiB to GPU in 2.46339s (14.614GiB/s) | 20260508 00:58:30.407 | INFO | Downloaded 34GiB to GPU in 2.48683s (13.672GiB/s) | 20260508 00:58:35.547 | INFO | Analyzing chunk frames = [3983360,3983872) | 20260508 00:58:35.547 | INFO | MemInfo: { | 20260508 00:58:35.547 | INFO | "gpu" : " 17 GB", | 20260508 00:58:35.547 | INFO | "huge" : "", | 20260508 00:58:35.547 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:35.547 | INFO | "malloc" : " 14 GB", | 20260508 00:58:35.547 | INFO | "pinned" : " 29 GB" | 20260508 00:58:35.547 | INFO | } | 20260508 00:58:35.548 | INFO | Perf report for the last 5.14061 seconds: | 20260508 00:58:35.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:35.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 00:58:35.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 00:58:35.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:35.548 | INFO | Uploaded 36GiB to GPU in 2.31415s (15.5565GiB/s) | 20260508 00:58:35.548 | INFO | Downloaded 34GiB to GPU in 2.49377s (13.634GiB/s) | 20260508 00:58:40.681 | INFO | Analyzing chunk frames = [3983872,3984384) | 20260508 00:58:40.681 | INFO | MemInfo: { | 20260508 00:58:40.681 | INFO | "gpu" : " 17 GB", | 20260508 00:58:40.681 | INFO | "huge" : "", | 20260508 00:58:40.681 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:40.681 | INFO | "malloc" : " 14 GB", | 20260508 00:58:40.681 | INFO | "pinned" : " 29 GB" | 20260508 00:58:40.681 | INFO | } | 20260508 00:58:40.681 | INFO | Perf report for the last 5.13382 seconds: | 20260508 00:58:40.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:40.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.681 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 00:58:40.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.681 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260508 00:58:40.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:40.682 | INFO | Uploaded 36GiB to GPU in 2.39407s (15.0371GiB/s) | 20260508 00:58:40.682 | INFO | Downloaded 34GiB to GPU in 2.41078s (14.1033GiB/s) | 20260508 00:58:45.822 | INFO | Analyzing chunk frames = [3984384,3984896) | 20260508 00:58:45.822 | INFO | MemInfo: { | 20260508 00:58:45.822 | INFO | "gpu" : " 17 GB", | 20260508 00:58:45.822 | INFO | "huge" : "", | 20260508 00:58:45.822 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:45.822 | INFO | "malloc" : " 14 GB", | 20260508 00:58:45.822 | INFO | "pinned" : " 29 GB" | 20260508 00:58:45.822 | INFO | } | 20260508 00:58:45.823 | INFO | Perf report for the last 5.14133 seconds: | 20260508 00:58:45.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:45.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 00:58:45.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:58:45.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:45.823 | INFO | Uploaded 36GiB to GPU in 2.55545s (14.0875GiB/s) | 20260508 00:58:45.823 | INFO | Downloaded 34GiB to GPU in 2.47639s (13.7297GiB/s) | 20260508 00:58:50.964 | INFO | Analyzing chunk frames = [3984896,3985408) | 20260508 00:58:50.964 | INFO | MemInfo: { | 20260508 00:58:50.964 | INFO | "gpu" : " 17 GB", | 20260508 00:58:50.964 | INFO | "huge" : "", | 20260508 00:58:50.964 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:50.964 | INFO | "malloc" : " 14 GB", | 20260508 00:58:50.964 | INFO | "pinned" : " 29 GB" | 20260508 00:58:50.964 | INFO | } | 20260508 00:58:50.964 | INFO | Perf report for the last 5.14149 seconds: | 20260508 00:58:50.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:50.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.964 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 00:58:50.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.964 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 00:58:50.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:50.965 | INFO | Uploaded 36GiB to GPU in 2.51321s (14.3243GiB/s) | 20260508 00:58:50.965 | INFO | Downloaded 34GiB to GPU in 2.44793s (13.8893GiB/s) | 20260508 00:58:51.073 | INFO | Starting a round of Baz IO | 20260508 00:58:54.511 | INFO | Populating RTMetrics report | 20260508 00:58:54.512 | INFO | RTMetrics report populated | 20260508 00:58:54.512 | INFO | Opening tmp RTMetrics json file | 20260508 00:58:54.514 | INFO | Serializing RTMetrics to json | 20260508 00:58:54.539 | INFO | Serialization done, swapping tmp file now... | 20260508 00:58:54.541 | INFO | RTMetrics done | 20260508 00:58:54.628 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.554 seconds | 20260508 00:58:54.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.00% / 0.00% / 58.00% | 20260508 00:58:54.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.926165% / 7.94729e-06% / 6.35783e-05% | 20260508 00:58:54.628 | INFO | Baz IO overall: | 20260508 00:58:54.628 | INFO | BytesWritten : 1808515971778 | 20260508 00:58:54.628 | INFO | NumEvents : 875072223991 | 20260508 00:58:56.106 | INFO | Analyzing chunk frames = [3985408,3985920) | 20260508 00:58:56.106 | INFO | MemInfo: { | 20260508 00:58:56.106 | INFO | "gpu" : " 17 GB", | 20260508 00:58:56.106 | INFO | "huge" : "", | 20260508 00:58:56.106 | INFO | "huge_shared" : " 29 GB", | 20260508 00:58:56.106 | INFO | "malloc" : " 14 GB", | 20260508 00:58:56.106 | INFO | "pinned" : " 29 GB" | 20260508 00:58:56.106 | INFO | } | 20260508 00:58:56.106 | INFO | Perf report for the last 5.14165 seconds: | 20260508 00:58:56.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:58:56.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:56.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:56.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:56.106 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260508 00:58:56.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:58:56.106 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 00:58:56.106 | INFO | PRE_HQ : 65.8%, 1.7, 56.9ms, 0 | 20260508 00:58:56.106 | INFO | BAZWRITER : 68.9%, 3.9, 138.7ms, 0 | 20260508 00:58:56.106 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 00:58:56.106 | INFO | Uploaded 36GiB to GPU in 2.56866s (14.0151GiB/s) | 20260508 00:58:56.106 | INFO | Downloaded 38GiB to GPU in 2.701s (14.0689GiB/s) | 20260508 00:59:01.250 | INFO | Analyzing chunk frames = [3985920,3986432) | 20260508 00:59:01.250 | INFO | MemInfo: { | 20260508 00:59:01.250 | INFO | "gpu" : " 17 GB", | 20260508 00:59:01.250 | INFO | "huge" : "", | 20260508 00:59:01.250 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:01.250 | INFO | "malloc" : " 14 GB", | 20260508 00:59:01.250 | INFO | "pinned" : " 29 GB" | 20260508 00:59:01.250 | INFO | } | 20260508 00:59:01.250 | INFO | Perf report for the last 5.14461 seconds: | 20260508 00:59:01.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:01.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.251 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 00:59:01.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.251 | INFO | PRE_HQ : 24.3%, 1.0, 12.7ms, 0 | 20260508 00:59:01.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:01.251 | INFO | Uploaded 36GiB to GPU in 2.58395s (13.9322GiB/s) | 20260508 00:59:01.251 | INFO | Downloaded 34GiB to GPU in 2.49067s (13.6509GiB/s) | 20260508 00:59:06.382 | INFO | Analyzing chunk frames = [3986432,3986944) | 20260508 00:59:06.382 | INFO | MemInfo: { | 20260508 00:59:06.382 | INFO | "gpu" : " 17 GB", | 20260508 00:59:06.382 | INFO | "huge" : "", | 20260508 00:59:06.382 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:06.382 | INFO | "malloc" : " 14 GB", | 20260508 00:59:06.382 | INFO | "pinned" : " 29 GB" | 20260508 00:59:06.382 | INFO | } | 20260508 00:59:06.382 | INFO | Perf report for the last 5.13143 seconds: | 20260508 00:59:06.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:06.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | ANALYSIS : 67.7%, 3.0, 104.8ms, 0 | 20260508 00:59:06.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 00:59:06.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:06.382 | INFO | Uploaded 36GiB to GPU in 2.57825s (13.963GiB/s) | 20260508 00:59:06.383 | INFO | Downloaded 34GiB to GPU in 2.4903s (13.653GiB/s) | 20260508 00:59:11.522 | INFO | Analyzing chunk frames = [3986944,3987456) | 20260508 00:59:11.522 | INFO | MemInfo: { | 20260508 00:59:11.522 | INFO | "gpu" : " 17 GB", | 20260508 00:59:11.522 | INFO | "huge" : "", | 20260508 00:59:11.522 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:11.522 | INFO | "malloc" : " 14 GB", | 20260508 00:59:11.522 | INFO | "pinned" : " 29 GB" | 20260508 00:59:11.522 | INFO | } | 20260508 00:59:11.522 | INFO | Perf report for the last 5.14024 seconds: | 20260508 00:59:11.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:11.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 00:59:11.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 00:59:11.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:11.522 | INFO | Uploaded 36GiB to GPU in 2.54852s (14.1259GiB/s) | 20260508 00:59:11.523 | INFO | Downloaded 34GiB to GPU in 2.47943s (13.7128GiB/s) | 20260508 00:59:16.663 | INFO | Analyzing chunk frames = [3987456,3987968) | 20260508 00:59:16.663 | INFO | MemInfo: { | 20260508 00:59:16.663 | INFO | "gpu" : " 17 GB", | 20260508 00:59:16.663 | INFO | "huge" : "", | 20260508 00:59:16.663 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:16.663 | INFO | "malloc" : " 14 GB", | 20260508 00:59:16.663 | INFO | "pinned" : " 29 GB" | 20260508 00:59:16.663 | INFO | } | 20260508 00:59:16.663 | INFO | Perf report for the last 5.14099 seconds: | 20260508 00:59:16.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:16.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.663 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 00:59:16.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.663 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260508 00:59:16.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:16.664 | INFO | Uploaded 36GiB to GPU in 2.38206s (15.113GiB/s) | 20260508 00:59:16.664 | INFO | Downloaded 34GiB to GPU in 2.53128s (13.432GiB/s) | 20260508 00:59:21.801 | INFO | Analyzing chunk frames = [3987968,3988480) | 20260508 00:59:21.801 | INFO | MemInfo: { | 20260508 00:59:21.801 | INFO | "gpu" : " 17 GB", | 20260508 00:59:21.801 | INFO | "huge" : "", | 20260508 00:59:21.801 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:21.801 | INFO | "malloc" : " 14 GB", | 20260508 00:59:21.801 | INFO | "pinned" : " 29 GB" | 20260508 00:59:21.801 | INFO | } | 20260508 00:59:21.801 | INFO | Perf report for the last 5.13811 seconds: | 20260508 00:59:21.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:21.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.801 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 00:59:21.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.801 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 00:59:21.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:21.802 | INFO | Uploaded 36GiB to GPU in 2.42366s (14.8536GiB/s) | 20260508 00:59:21.802 | INFO | Downloaded 34GiB to GPU in 2.38548s (14.2529GiB/s) | 20260508 00:59:26.939 | INFO | Analyzing chunk frames = [3988480,3988992) | 20260508 00:59:26.939 | INFO | MemInfo: { | 20260508 00:59:26.939 | INFO | "gpu" : " 17 GB", | 20260508 00:59:26.939 | INFO | "huge" : "", | 20260508 00:59:26.939 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:26.939 | INFO | "malloc" : " 14 GB", | 20260508 00:59:26.939 | INFO | "pinned" : " 29 GB" | 20260508 00:59:26.940 | INFO | } | 20260508 00:59:26.940 | INFO | Perf report for the last 5.13839 seconds: | 20260508 00:59:26.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:26.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 00:59:26.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 00:59:26.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:26.940 | INFO | Uploaded 36GiB to GPU in 2.55201s (14.1065GiB/s) | 20260508 00:59:26.940 | INFO | Downloaded 34GiB to GPU in 2.45024s (13.8762GiB/s) | 20260508 00:59:32.076 | INFO | Analyzing chunk frames = [3988992,3989504) | 20260508 00:59:32.076 | INFO | MemInfo: { | 20260508 00:59:32.076 | INFO | "gpu" : " 17 GB", | 20260508 00:59:32.076 | INFO | "huge" : "", | 20260508 00:59:32.076 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:32.076 | INFO | "malloc" : " 14 GB", | 20260508 00:59:32.076 | INFO | "pinned" : " 29 GB" | 20260508 00:59:32.076 | INFO | } | 20260508 00:59:32.076 | INFO | Perf report for the last 5.13652 seconds: | 20260508 00:59:32.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:32.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.076 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 00:59:32.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.076 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 00:59:32.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:32.077 | INFO | Uploaded 36GiB to GPU in 2.57343s (13.9891GiB/s) | 20260508 00:59:32.077 | INFO | Downloaded 34GiB to GPU in 2.45188s (13.8669GiB/s) | 20260508 00:59:32.077 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 00:59:32.077 | INFO | Total mem usage: 16.7 GiB | 20260508 00:59:32.077 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 00:59:32.077 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.5%) | 20260508 00:59:32.077 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 00:59:32.077 | INFO | High Water Breakdown: | 20260508 00:59:32.077 | INFO | 60% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 00:59:32.077 | INFO | 39.8% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 00:59:32.077 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 00:59:32.077 | INFO | ------------------------------------------ | 20260508 00:59:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:59:32.077 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 00:59:32.077 | INFO | Total mem usage: 26.5 GiB | 20260508 00:59:32.077 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 00:59:32.077 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 00:59:32.077 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 00:59:32.077 | INFO | High Water Breakdown: | 20260508 00:59:32.077 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:59:32.077 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:59:32.077 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:59:32.077 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:59:32.077 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:59:32.077 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:59:32.077 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 00:59:32.077 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 00:59:32.077 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:59:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 00:59:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 00:59:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 00:59:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 00:59:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 00:59:32.077 | INFO | ------------------------------------------ | 20260508 00:59:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:59:32.077 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 00:59:32.077 | INFO | Total mem usage: 27.8 GiB | 20260508 00:59:32.077 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 00:59:32.077 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 00:59:32.077 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 00:59:32.077 | INFO | High Water Breakdown: | 20260508 00:59:32.077 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 00:59:32.077 | INFO | ------------------------------------------ | 20260508 00:59:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:59:32.077 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 00:59:32.077 | INFO | Total mem usage: 15.7 GiB | 20260508 00:59:32.077 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 00:59:32.077 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 00:59:32.077 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 00:59:32.077 | INFO | High Water Breakdown: | 20260508 00:59:32.077 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 00:59:32.077 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 00:59:32.077 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 00:59:32.077 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 00:59:32.077 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 00:59:32.077 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 00:59:32.077 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 00:59:32.077 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 00:59:32.077 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 00:59:32.077 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 00:59:32.077 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 00:59:32.077 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 00:59:32.077 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 00:59:32.077 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 00:59:32.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 00:59:32.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 00:59:32.077 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 00:59:32.077 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 00:59:32.077 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 00:59:32.077 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 00:59:32.077 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 00:59:32.077 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 00:59:32.077 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 00:59:32.077 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 00:59:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 00:59:32.077 | INFO | ------------------------------------------ | 20260508 00:59:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 00:59:32.077 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 00:59:32.077 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:59:32.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:59:32.077 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.41e+01 / 1.50e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.01e-01, 1.11e-02 / 4.64e-03 | 20260508 00:59:32.077 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.49e+00, 5.81e-01 / 1.74e-01 | 20260508 00:59:32.077 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.50e+00, 2.04e-01 / 8.98e-02 | 20260508 00:59:32.077 | INFO | PerfReport: 16.16%, DME -- 4.23e+01 / 4.64e+01, 4.28e+00 / 1.79e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.83e+01 / 1.93e+01, 1.53e+00 / 1.47e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.45e+00 / 2.75e+00, 4.28e-01 / 1.82e-01 | 20260508 00:59:32.077 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.34e+00, 2.77e-01 / 1.52e-01 | 20260508 00:59:32.077 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.75e-01, 5.31e-02 / 2.62e-02 | 20260508 00:59:32.077 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 00:59:32.077 | INFO | PerfReport: Gathered 1536 samples. | 20260508 00:59:32.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 00:59:32.077 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 24.41%, Download -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 30.62%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 2.26%, LockUpload -- 2.42e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 2.77%, LockDownload -- 2.96e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 15.54%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 00:59:32.077 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.29e-02 / 0.00e+00 | 20260508 00:59:32.174 | INFO | Starting a round of Baz IO | 20260508 00:59:35.645 | INFO | Populating RTMetrics report | 20260508 00:59:35.645 | INFO | RTMetrics report populated | 20260508 00:59:35.645 | INFO | Opening tmp RTMetrics json file | 20260508 00:59:35.750 | INFO | Serializing RTMetrics to json | 20260508 00:59:35.757 | INFO | Serialization done, swapping tmp file now... | 20260508 00:59:35.763 | INFO | RTMetrics done | 20260508 00:59:35.789 | INFO | Finished a round of Baz IO: Wrote 1.674 GiB in 3.615 seconds | 20260508 00:59:35.789 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.99% / 0.00% / 58.00% | 20260508 00:59:35.789 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.922279% / 3.97364e-06% / 3.57628e-05% | 20260508 00:59:35.789 | INFO | Baz IO overall: | 20260508 00:59:35.789 | INFO | BytesWritten : 1810313667026 | 20260508 00:59:35.789 | INFO | NumEvents : 875936103508 | 20260508 00:59:37.217 | INFO | Analyzing chunk frames = [3989504,3990016) | 20260508 00:59:37.217 | INFO | MemInfo: { | 20260508 00:59:37.217 | INFO | "gpu" : " 17 GB", | 20260508 00:59:37.217 | INFO | "huge" : "", | 20260508 00:59:37.217 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:37.217 | INFO | "malloc" : " 14 GB", | 20260508 00:59:37.217 | INFO | "pinned" : " 29 GB" | 20260508 00:59:37.217 | INFO | } | 20260508 00:59:37.217 | INFO | Perf report for the last 5.14051 seconds: | 20260508 00:59:37.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:37.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:37.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:37.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:37.217 | INFO | ANALYSIS : 68.5%, 3.0, 106.7ms, 0 | 20260508 00:59:37.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:37.217 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 00:59:37.217 | INFO | PRE_HQ : 65.5%, 1.7, 56.5ms, 0 | 20260508 00:59:37.217 | INFO | BAZWRITER : 69.9%, 3.8, 137.8ms, 0 | 20260508 00:59:37.217 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260508 00:59:37.217 | INFO | Uploaded 36GiB to GPU in 2.5592s (14.0669GiB/s) | 20260508 00:59:37.217 | INFO | Downloaded 38GiB to GPU in 2.68255s (14.1656GiB/s) | 20260508 00:59:42.358 | INFO | Analyzing chunk frames = [3990016,3990528) | 20260508 00:59:42.358 | INFO | MemInfo: { | 20260508 00:59:42.358 | INFO | "gpu" : " 17 GB", | 20260508 00:59:42.358 | INFO | "huge" : "", | 20260508 00:59:42.358 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:42.358 | INFO | "malloc" : " 14 GB", | 20260508 00:59:42.358 | INFO | "pinned" : " 29 GB" | 20260508 00:59:42.358 | INFO | } | 20260508 00:59:42.358 | INFO | Perf report for the last 5.1417 seconds: | 20260508 00:59:42.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:42.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.358 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 00:59:42.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.359 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 00:59:42.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:42.359 | INFO | Uploaded 36GiB to GPU in 2.56641s (14.0274GiB/s) | 20260508 00:59:42.359 | INFO | Downloaded 34GiB to GPU in 2.49291s (13.6387GiB/s) | 20260508 00:59:47.499 | INFO | Analyzing chunk frames = [3990528,3991040) | 20260508 00:59:47.499 | INFO | MemInfo: { | 20260508 00:59:47.499 | INFO | "gpu" : " 17 GB", | 20260508 00:59:47.499 | INFO | "huge" : "", | 20260508 00:59:47.499 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:47.499 | INFO | "malloc" : " 14 GB", | 20260508 00:59:47.499 | INFO | "pinned" : " 29 GB" | 20260508 00:59:47.499 | INFO | } | 20260508 00:59:47.500 | INFO | Perf report for the last 5.14104 seconds: | 20260508 00:59:47.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:47.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 00:59:47.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 00:59:47.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:47.500 | INFO | Uploaded 36GiB to GPU in 2.52049s (14.2829GiB/s) | 20260508 00:59:47.500 | INFO | Downloaded 34GiB to GPU in 2.494s (13.6327GiB/s) | 20260508 00:59:52.634 | INFO | Analyzing chunk frames = [3991040,3991552) | 20260508 00:59:52.634 | INFO | MemInfo: { | 20260508 00:59:52.634 | INFO | "gpu" : " 17 GB", | 20260508 00:59:52.634 | INFO | "huge" : "", | 20260508 00:59:52.634 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:52.634 | INFO | "malloc" : " 14 GB", | 20260508 00:59:52.634 | INFO | "pinned" : " 29 GB" | 20260508 00:59:52.634 | INFO | } | 20260508 00:59:52.634 | INFO | Perf report for the last 5.13468 seconds: | 20260508 00:59:52.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:52.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.634 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 00:59:52.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.634 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 00:59:52.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:52.635 | INFO | Uploaded 36GiB to GPU in 2.50428s (14.3754GiB/s) | 20260508 00:59:52.635 | INFO | Downloaded 34GiB to GPU in 2.4928s (13.6393GiB/s) | 20260508 00:59:57.773 | INFO | Analyzing chunk frames = [3991552,3992064) | 20260508 00:59:57.773 | INFO | MemInfo: { | 20260508 00:59:57.773 | INFO | "gpu" : " 17 GB", | 20260508 00:59:57.773 | INFO | "huge" : "", | 20260508 00:59:57.773 | INFO | "huge_shared" : " 29 GB", | 20260508 00:59:57.773 | INFO | "malloc" : " 14 GB", | 20260508 00:59:57.774 | INFO | "pinned" : " 29 GB" | 20260508 00:59:57.774 | INFO | } | 20260508 00:59:57.774 | INFO | Perf report for the last 5.13941 seconds: | 20260508 00:59:57.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 00:59:57.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 00:59:57.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 00:59:57.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 00:59:57.774 | INFO | Uploaded 36GiB to GPU in 2.34606s (15.3448GiB/s) | 20260508 00:59:57.774 | INFO | Downloaded 34GiB to GPU in 2.51853s (13.4999GiB/s) | 20260508 01:00:02.915 | INFO | Analyzing chunk frames = [3992064,3992576) | 20260508 01:00:02.915 | INFO | MemInfo: { | 20260508 01:00:02.915 | INFO | "gpu" : " 17 GB", | 20260508 01:00:02.915 | INFO | "huge" : "", | 20260508 01:00:02.915 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:02.915 | INFO | "malloc" : " 14 GB", | 20260508 01:00:02.916 | INFO | "pinned" : " 29 GB" | 20260508 01:00:02.916 | INFO | } | 20260508 01:00:02.916 | INFO | Perf report for the last 5.142 seconds: | 20260508 01:00:02.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:02.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:00:02.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 01:00:02.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:02.916 | INFO | Uploaded 36GiB to GPU in 2.38634s (15.0859GiB/s) | 20260508 01:00:02.916 | INFO | Downloaded 34GiB to GPU in 2.41337s (14.0882GiB/s) | 20260508 01:00:08.057 | INFO | Analyzing chunk frames = [3992576,3993088) | 20260508 01:00:08.057 | INFO | MemInfo: { | 20260508 01:00:08.057 | INFO | "gpu" : " 17 GB", | 20260508 01:00:08.057 | INFO | "huge" : "", | 20260508 01:00:08.057 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:08.057 | INFO | "malloc" : " 14 GB", | 20260508 01:00:08.057 | INFO | "pinned" : " 29 GB" | 20260508 01:00:08.057 | INFO | } | 20260508 01:00:08.057 | INFO | Perf report for the last 5.14172 seconds: | 20260508 01:00:08.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:08.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260508 01:00:08.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:00:08.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:08.058 | INFO | Uploaded 36GiB to GPU in 2.5087s (14.3501GiB/s) | 20260508 01:00:08.058 | INFO | Downloaded 34GiB to GPU in 2.45779s (13.8336GiB/s) | 20260508 01:00:13.192 | INFO | Analyzing chunk frames = [3993088,3993600) | 20260508 01:00:13.192 | INFO | MemInfo: { | 20260508 01:00:13.192 | INFO | "gpu" : " 17 GB", | 20260508 01:00:13.192 | INFO | "huge" : "", | 20260508 01:00:13.192 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:13.192 | INFO | "malloc" : " 14 GB", | 20260508 01:00:13.192 | INFO | "pinned" : " 29 GB" | 20260508 01:00:13.192 | INFO | } | 20260508 01:00:13.192 | INFO | Perf report for the last 5.13495 seconds: | 20260508 01:00:13.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:13.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.192 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:00:13.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.192 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 01:00:13.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:13.193 | INFO | Uploaded 36GiB to GPU in 2.54853s (14.1258GiB/s) | 20260508 01:00:13.193 | INFO | Downloaded 34GiB to GPU in 2.47404s (13.7427GiB/s) | 20260508 01:00:13.292 | INFO | Starting a round of Baz IO | 20260508 01:00:16.741 | INFO | Populating RTMetrics report | 20260508 01:00:16.741 | INFO | RTMetrics report populated | 20260508 01:00:16.741 | INFO | Opening tmp RTMetrics json file | 20260508 01:00:16.773 | INFO | Serializing RTMetrics to json | 20260508 01:00:16.926 | INFO | Serialization done, swapping tmp file now... | 20260508 01:00:16.929 | INFO | RTMetrics done | 20260508 01:00:16.946 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.653 seconds | 20260508 01:00:16.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.99% / 0.00% / 58.01% | 20260508 01:00:16.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.92055% / 7.94729e-06% / 7.54992e-05% | 20260508 01:00:16.946 | INFO | Baz IO overall: | 20260508 01:00:16.946 | INFO | BytesWritten : 1812112745432 | 20260508 01:00:16.946 | INFO | NumEvents : 876800806721 | 20260508 01:00:18.330 | INFO | Analyzing chunk frames = [3993600,3994112) | 20260508 01:00:18.330 | INFO | MemInfo: { | 20260508 01:00:18.330 | INFO | "gpu" : " 17 GB", | 20260508 01:00:18.330 | INFO | "huge" : "", | 20260508 01:00:18.330 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:18.330 | INFO | "malloc" : " 14 GB", | 20260508 01:00:18.330 | INFO | "pinned" : " 29 GB" | 20260508 01:00:18.330 | INFO | } | 20260508 01:00:18.330 | INFO | Perf report for the last 5.13799 seconds: | 20260508 01:00:18.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:18.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:18.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:18.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:18.331 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260508 01:00:18.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:18.331 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 01:00:18.331 | INFO | PRE_HQ : 66.1%, 1.7, 56.9ms, 0 | 20260508 01:00:18.331 | INFO | BAZWRITER : 70.8%, 3.6, 131.8ms, 0 | 20260508 01:00:18.331 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.9ms, 0 | 20260508 01:00:18.331 | INFO | Uploaded 36GiB to GPU in 2.60421s (13.8238GiB/s) | 20260508 01:00:18.331 | INFO | Downloaded 38GiB to GPU in 2.69879s (14.0804GiB/s) | 20260508 01:00:23.474 | INFO | Analyzing chunk frames = [3994112,3994624) | 20260508 01:00:23.474 | INFO | MemInfo: { | 20260508 01:00:23.474 | INFO | "gpu" : " 17 GB", | 20260508 01:00:23.474 | INFO | "huge" : "", | 20260508 01:00:23.474 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:23.474 | INFO | "malloc" : " 14 GB", | 20260508 01:00:23.474 | INFO | "pinned" : " 29 GB" | 20260508 01:00:23.474 | INFO | } | 20260508 01:00:23.474 | INFO | Perf report for the last 5.14407 seconds: | 20260508 01:00:23.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:23.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.475 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 01:00:23.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.475 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 01:00:23.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:23.475 | INFO | Uploaded 36GiB to GPU in 2.54645s (14.1373GiB/s) | 20260508 01:00:23.475 | INFO | Downloaded 34GiB to GPU in 2.42s (14.0496GiB/s) | 20260508 01:00:28.618 | INFO | Analyzing chunk frames = [3994624,3995136) | 20260508 01:00:28.618 | INFO | MemInfo: { | 20260508 01:00:28.618 | INFO | "gpu" : " 17 GB", | 20260508 01:00:28.618 | INFO | "huge" : "", | 20260508 01:00:28.618 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:28.619 | INFO | "malloc" : " 14 GB", | 20260508 01:00:28.619 | INFO | "pinned" : " 29 GB" | 20260508 01:00:28.619 | INFO | } | 20260508 01:00:28.619 | INFO | Perf report for the last 5.14419 seconds: | 20260508 01:00:28.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:28.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 01:00:28.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:00:28.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:28.619 | INFO | Uploaded 36GiB to GPU in 2.53653s (14.1926GiB/s) | 20260508 01:00:28.619 | INFO | Downloaded 34GiB to GPU in 2.4673s (13.7802GiB/s) | 20260508 01:00:33.753 | INFO | Analyzing chunk frames = [3995136,3995648) | 20260508 01:00:33.753 | INFO | MemInfo: { | 20260508 01:00:33.753 | INFO | "gpu" : " 17 GB", | 20260508 01:00:33.753 | INFO | "huge" : "", | 20260508 01:00:33.753 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:33.753 | INFO | "malloc" : " 14 GB", | 20260508 01:00:33.753 | INFO | "pinned" : " 29 GB" | 20260508 01:00:33.753 | INFO | } | 20260508 01:00:33.753 | INFO | Perf report for the last 5.13411 seconds: | 20260508 01:00:33.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:33.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:00:33.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 01:00:33.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:33.753 | INFO | Uploaded 36GiB to GPU in 2.55577s (14.0858GiB/s) | 20260508 01:00:33.753 | INFO | Downloaded 34GiB to GPU in 2.486s (13.6766GiB/s) | 20260508 01:00:38.888 | INFO | Analyzing chunk frames = [3995648,3996160) | 20260508 01:00:38.888 | INFO | MemInfo: { | 20260508 01:00:38.888 | INFO | "gpu" : " 17 GB", | 20260508 01:00:38.888 | INFO | "huge" : "", | 20260508 01:00:38.888 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:38.888 | INFO | "malloc" : " 14 GB", | 20260508 01:00:38.888 | INFO | "pinned" : " 29 GB" | 20260508 01:00:38.888 | INFO | } | 20260508 01:00:38.888 | INFO | Perf report for the last 5.13532 seconds: | 20260508 01:00:38.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:38.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.888 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 01:00:38.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.888 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:00:38.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:38.889 | INFO | Uploaded 36GiB to GPU in 2.42483s (14.8464GiB/s) | 20260508 01:00:38.889 | INFO | Downloaded 34GiB to GPU in 2.53738s (13.3996GiB/s) | 20260508 01:00:44.024 | INFO | Analyzing chunk frames = [3996160,3996672) | 20260508 01:00:44.024 | INFO | MemInfo: { | 20260508 01:00:44.025 | INFO | "gpu" : " 17 GB", | 20260508 01:00:44.025 | INFO | "huge" : "", | 20260508 01:00:44.025 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:44.025 | INFO | "malloc" : " 14 GB", | 20260508 01:00:44.025 | INFO | "pinned" : " 29 GB" | 20260508 01:00:44.025 | INFO | } | 20260508 01:00:44.025 | INFO | Perf report for the last 5.13663 seconds: | 20260508 01:00:44.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:44.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:00:44.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:00:44.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:44.025 | INFO | Uploaded 36GiB to GPU in 2.39075s (15.058GiB/s) | 20260508 01:00:44.025 | INFO | Downloaded 34GiB to GPU in 2.40724s (14.124GiB/s) | 20260508 01:00:49.167 | INFO | Analyzing chunk frames = [3996672,3997184) | 20260508 01:00:49.167 | INFO | MemInfo: { | 20260508 01:00:49.167 | INFO | "gpu" : " 17 GB", | 20260508 01:00:49.167 | INFO | "huge" : "", | 20260508 01:00:49.167 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:49.167 | INFO | "malloc" : " 14 GB", | 20260508 01:00:49.168 | INFO | "pinned" : " 29 GB" | 20260508 01:00:49.168 | INFO | } | 20260508 01:00:49.168 | INFO | Perf report for the last 5.14299 seconds: | 20260508 01:00:49.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:49.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260508 01:00:49.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:00:49.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:49.168 | INFO | Uploaded 36GiB to GPU in 2.51419s (14.3187GiB/s) | 20260508 01:00:49.168 | INFO | Downloaded 34GiB to GPU in 2.45839s (13.8302GiB/s) | 20260508 01:00:54.312 | INFO | Analyzing chunk frames = [3997184,3997696) | 20260508 01:00:54.312 | INFO | MemInfo: { | 20260508 01:00:54.312 | INFO | "gpu" : " 17 GB", | 20260508 01:00:54.312 | INFO | "huge" : "", | 20260508 01:00:54.312 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:54.312 | INFO | "malloc" : " 14 GB", | 20260508 01:00:54.312 | INFO | "pinned" : " 29 GB" | 20260508 01:00:54.312 | INFO | } | 20260508 01:00:54.312 | INFO | Perf report for the last 5.14446 seconds: | 20260508 01:00:54.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:54.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.312 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:00:54.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.312 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 01:00:54.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:54.313 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:00:54.313 | INFO | Total mem usage: 16.7 GiB | 20260508 01:00:54.313 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:00:54.313 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260508 01:00:54.313 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:00:54.313 | INFO | High Water Breakdown: | 20260508 01:00:54.313 | INFO | 60.2% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:00:54.313 | INFO | 39.7% : 5.93 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:00:54.313 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:00:54.313 | INFO | ------------------------------------------ | 20260508 01:00:54.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:00:54.313 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:00:54.313 | INFO | Total mem usage: 26.5 GiB | 20260508 01:00:54.313 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:00:54.313 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:00:54.313 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:00:54.313 | INFO | High Water Breakdown: | 20260508 01:00:54.313 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:00:54.313 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:00:54.313 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:00:54.313 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:00:54.313 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:00:54.313 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:00:54.313 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:00:54.313 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:00:54.313 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:00:54.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:00:54.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:00:54.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:00:54.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:00:54.313 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:00:54.313 | INFO | ------------------------------------------ | 20260508 01:00:54.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:00:54.313 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:00:54.313 | INFO | Total mem usage: 27.8 GiB | 20260508 01:00:54.313 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:00:54.313 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:00:54.313 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:00:54.313 | INFO | High Water Breakdown: | 20260508 01:00:54.313 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:00:54.313 | INFO | ------------------------------------------ | 20260508 01:00:54.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:00:54.313 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:00:54.313 | INFO | Total mem usage: 15.7 GiB | 20260508 01:00:54.313 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:00:54.313 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:00:54.313 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:00:54.313 | INFO | High Water Breakdown: | 20260508 01:00:54.313 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:00:54.313 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:00:54.313 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:00:54.313 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:00:54.313 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:00:54.313 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:00:54.313 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:00:54.313 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:00:54.313 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:00:54.313 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:00:54.313 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:00:54.313 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:00:54.313 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:00:54.313 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:00:54.313 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:00:54.313 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:00:54.313 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:00:54.313 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:00:54.313 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:00:54.313 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:00:54.313 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:00:54.313 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:00:54.313 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:00:54.313 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:00:54.313 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:00:54.313 | INFO | ------------------------------------------ | 20260508 01:00:54.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:00:54.313 | INFO | Uploaded 36GiB to GPU in 2.52758s (14.2428GiB/s) | 20260508 01:00:54.313 | INFO | Downloaded 34GiB to GPU in 2.45502s (13.8492GiB/s) | 20260508 01:00:54.313 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:00:54.313 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:00:54.313 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:00:54.314 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.43e+01 / 1.56e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.01e-01, 1.17e-02 / 5.36e-03 | 20260508 01:00:54.314 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.45e+00, 5.89e-01 / 2.20e-01 | 20260508 01:00:54.314 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.06e-01 | 20260508 01:00:54.314 | INFO | PerfReport: 16.17%, DME -- 4.23e+01 / 4.63e+01, 4.22e+00 / 1.75e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.83e+01 / 1.94e+01, 1.54e+00 / 1.31e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.45e+00 / 2.77e+00, 4.26e-01 / 1.50e-01 | 20260508 01:00:54.314 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.30e+00, 2.73e-01 / 1.43e-01 | 20260508 01:00:54.314 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.75e-01, 5.35e-02 / 2.73e-02 | 20260508 01:00:54.314 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:00:54.314 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:00:54.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:00:54.314 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 24.41%, Upload -- 2.61e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 24.36%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 30.61%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 2.20%, LockUpload -- 2.35e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 2.80%, LockDownload -- 2.99e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 15.60%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 01:00:54.314 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260508 01:00:54.410 | INFO | Starting a round of Baz IO | 20260508 01:00:57.852 | INFO | Populating RTMetrics report | 20260508 01:00:57.852 | INFO | RTMetrics report populated | 20260508 01:00:57.852 | INFO | Opening tmp RTMetrics json file | 20260508 01:00:57.872 | INFO | Serializing RTMetrics to json | 20260508 01:00:57.875 | INFO | Serialization done, swapping tmp file now... | 20260508 01:00:57.875 | INFO | RTMetrics done | 20260508 01:00:58.172 | INFO | Finished a round of Baz IO: Wrote 1.674 GiB in 3.761 seconds | 20260508 01:00:58.172 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.98% / 0.00% / 58.01% | 20260508 01:00:58.172 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.919056% / 3.97364e-06% / 6.75519e-05% | 20260508 01:00:58.172 | INFO | Baz IO overall: | 20260508 01:00:58.172 | INFO | BytesWritten : 1813910561372 | 20260508 01:00:58.172 | INFO | NumEvents : 877664894417 | 20260508 01:00:59.446 | INFO | Analyzing chunk frames = [3997696,3998208) | 20260508 01:00:59.446 | INFO | MemInfo: { | 20260508 01:00:59.446 | INFO | "gpu" : " 17 GB", | 20260508 01:00:59.446 | INFO | "huge" : "", | 20260508 01:00:59.446 | INFO | "huge_shared" : " 29 GB", | 20260508 01:00:59.446 | INFO | "malloc" : " 14 GB", | 20260508 01:00:59.446 | INFO | "pinned" : " 29 GB" | 20260508 01:00:59.446 | INFO | } | 20260508 01:00:59.446 | INFO | Perf report for the last 5.13356 seconds: | 20260508 01:00:59.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:00:59.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:59.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:59.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:59.446 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 01:00:59.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:00:59.446 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 01:00:59.446 | INFO | PRE_HQ : 65.7%, 1.7, 56.8ms, 0 | 20260508 01:00:59.446 | INFO | BAZWRITER : 72.8%, 3.6, 136.0ms, 0 | 20260508 01:00:59.446 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 01:00:59.446 | INFO | Uploaded 36GiB to GPU in 2.54449s (14.1482GiB/s) | 20260508 01:00:59.446 | INFO | Downloaded 38GiB to GPU in 2.69579s (14.0961GiB/s) | 20260508 01:01:04.585 | INFO | Analyzing chunk frames = [3998208,3998720) | 20260508 01:01:04.585 | INFO | MemInfo: { | 20260508 01:01:04.585 | INFO | "gpu" : " 17 GB", | 20260508 01:01:04.585 | INFO | "huge" : "", | 20260508 01:01:04.585 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:04.587 | INFO | "malloc" : " 14 GB", | 20260508 01:01:04.587 | INFO | "pinned" : " 29 GB" | 20260508 01:01:04.587 | INFO | } | 20260508 01:01:04.587 | INFO | Perf report for the last 5.14105 seconds: | 20260508 01:01:04.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:04.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 01:01:04.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:01:04.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:04.587 | INFO | Uploaded 36GiB to GPU in 2.54695s (14.1345GiB/s) | 20260508 01:01:04.587 | INFO | Downloaded 34GiB to GPU in 2.42956s (13.9943GiB/s) | 20260508 01:01:09.727 | INFO | Analyzing chunk frames = [3998720,3999232) | 20260508 01:01:09.727 | INFO | MemInfo: { | 20260508 01:01:09.727 | INFO | "gpu" : " 17 GB", | 20260508 01:01:09.727 | INFO | "huge" : "", | 20260508 01:01:09.727 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:09.727 | INFO | "malloc" : " 14 GB", | 20260508 01:01:09.727 | INFO | "pinned" : " 29 GB" | 20260508 01:01:09.727 | INFO | } | 20260508 01:01:09.727 | INFO | Perf report for the last 5.14049 seconds: | 20260508 01:01:09.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:09.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.727 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 01:01:09.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.727 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:01:09.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:09.728 | INFO | Uploaded 36GiB to GPU in 2.45548s (14.6611GiB/s) | 20260508 01:01:09.728 | INFO | Downloaded 34GiB to GPU in 2.4795s (13.7124GiB/s) | 20260508 01:01:14.866 | INFO | Analyzing chunk frames = [3999232,3999744) | 20260508 01:01:14.866 | INFO | MemInfo: { | 20260508 01:01:14.866 | INFO | "gpu" : " 17 GB", | 20260508 01:01:14.866 | INFO | "huge" : "", | 20260508 01:01:14.866 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:14.866 | INFO | "malloc" : " 14 GB", | 20260508 01:01:14.866 | INFO | "pinned" : " 29 GB" | 20260508 01:01:14.866 | INFO | } | 20260508 01:01:14.866 | INFO | Perf report for the last 5.13909 seconds: | 20260508 01:01:14.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:14.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:01:14.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:01:14.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:14.867 | INFO | Uploaded 36GiB to GPU in 2.51198s (14.3313GiB/s) | 20260508 01:01:14.867 | INFO | Downloaded 34GiB to GPU in 2.48141s (13.7019GiB/s) | 20260508 01:01:20.004 | INFO | Analyzing chunk frames = [3999744,4000256) | 20260508 01:01:20.004 | INFO | MemInfo: { | 20260508 01:01:20.004 | INFO | "gpu" : " 17 GB", | 20260508 01:01:20.004 | INFO | "huge" : "", | 20260508 01:01:20.004 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:20.004 | INFO | "malloc" : " 14 GB", | 20260508 01:01:20.004 | INFO | "pinned" : " 29 GB" | 20260508 01:01:20.004 | INFO | } | 20260508 01:01:20.004 | INFO | Perf report for the last 5.13778 seconds: | 20260508 01:01:20.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:20.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.004 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:01:20.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.005 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:01:20.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:20.005 | INFO | Uploaded 36GiB to GPU in 2.35485s (15.2876GiB/s) | 20260508 01:01:20.005 | INFO | Downloaded 34GiB to GPU in 2.51196s (13.5352GiB/s) | 20260508 01:01:25.141 | INFO | Analyzing chunk frames = [4000256,4000768) | 20260508 01:01:25.141 | INFO | MemInfo: { | 20260508 01:01:25.141 | INFO | "gpu" : " 17 GB", | 20260508 01:01:25.141 | INFO | "huge" : "", | 20260508 01:01:25.141 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:25.141 | INFO | "malloc" : " 14 GB", | 20260508 01:01:25.141 | INFO | "pinned" : " 29 GB" | 20260508 01:01:25.141 | INFO | } | 20260508 01:01:25.141 | INFO | Perf report for the last 5.13635 seconds: | 20260508 01:01:25.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:25.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 01:01:25.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:01:25.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:25.141 | INFO | Uploaded 36GiB to GPU in 2.43002s (14.8147GiB/s) | 20260508 01:01:25.141 | INFO | Downloaded 34GiB to GPU in 2.4031s (14.1484GiB/s) | 20260508 01:01:30.283 | INFO | Analyzing chunk frames = [4000768,4001280) | 20260508 01:01:30.283 | INFO | MemInfo: { | 20260508 01:01:30.283 | INFO | "gpu" : " 17 GB", | 20260508 01:01:30.283 | INFO | "huge" : "", | 20260508 01:01:30.283 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:30.283 | INFO | "malloc" : " 14 GB", | 20260508 01:01:30.283 | INFO | "pinned" : " 29 GB" | 20260508 01:01:30.283 | INFO | } | 20260508 01:01:30.283 | INFO | Perf report for the last 5.14228 seconds: | 20260508 01:01:30.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:30.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:01:30.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260508 01:01:30.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:30.283 | INFO | Uploaded 36GiB to GPU in 2.571s (14.0023GiB/s) | 20260508 01:01:30.283 | INFO | Downloaded 34GiB to GPU in 2.46633s (13.7857GiB/s) | 20260508 01:01:35.425 | INFO | Analyzing chunk frames = [4001280,4001792) | 20260508 01:01:35.425 | INFO | MemInfo: { | 20260508 01:01:35.425 | INFO | "gpu" : " 17 GB", | 20260508 01:01:35.425 | INFO | "huge" : "", | 20260508 01:01:35.425 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:35.425 | INFO | "malloc" : " 14 GB", | 20260508 01:01:35.425 | INFO | "pinned" : " 29 GB" | 20260508 01:01:35.425 | INFO | } | 20260508 01:01:35.425 | INFO | Perf report for the last 5.14224 seconds: | 20260508 01:01:35.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:35.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:01:35.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:01:35.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:35.425 | INFO | Uploaded 36GiB to GPU in 2.55011s (14.117GiB/s) | 20260508 01:01:35.425 | INFO | Downloaded 34GiB to GPU in 2.44762s (13.891GiB/s) | 20260508 01:01:35.522 | INFO | Starting a round of Baz IO | 20260508 01:01:38.976 | INFO | Populating RTMetrics report | 20260508 01:01:38.977 | INFO | RTMetrics report populated | 20260508 01:01:38.977 | INFO | Opening tmp RTMetrics json file | 20260508 01:01:38.988 | INFO | Serializing RTMetrics to json | 20260508 01:01:39.003 | INFO | Serialization done, swapping tmp file now... | 20260508 01:01:39.003 | INFO | RTMetrics done | 20260508 01:01:39.078 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.555 seconds | 20260508 01:01:39.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.98% / 0.00% / 58.02% | 20260508 01:01:39.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.914999% / 1.19209e-05% / 8.74201e-05% | 20260508 01:01:39.078 | INFO | Baz IO overall: | 20260508 01:01:39.078 | INFO | BytesWritten : 1815707166097 | 20260508 01:01:39.078 | INFO | NumEvents : 878527952513 | 20260508 01:01:40.565 | INFO | Analyzing chunk frames = [4001792,4002304) | 20260508 01:01:40.565 | INFO | MemInfo: { | 20260508 01:01:40.565 | INFO | "gpu" : " 17 GB", | 20260508 01:01:40.566 | INFO | "huge" : "", | 20260508 01:01:40.566 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:40.566 | INFO | "malloc" : " 14 GB", | 20260508 01:01:40.566 | INFO | "pinned" : " 29 GB" | 20260508 01:01:40.566 | INFO | } | 20260508 01:01:40.566 | INFO | Perf report for the last 5.14047 seconds: | 20260508 01:01:40.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:40.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:40.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:40.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:40.566 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 01:01:40.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:40.566 | INFO | RT_METRICS : 40.5%, 1.0, 21.2ms, 0 | 20260508 01:01:40.566 | INFO | PRE_HQ : 65.4%, 1.6, 56.0ms, 0 | 20260508 01:01:40.566 | INFO | BAZWRITER : 68.8%, 3.8, 134.9ms, 0 | 20260508 01:01:40.566 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 01:01:40.566 | INFO | Uploaded 36GiB to GPU in 2.56952s (14.0104GiB/s) | 20260508 01:01:40.566 | INFO | Downloaded 38GiB to GPU in 2.68411s (14.1574GiB/s) | 20260508 01:01:45.702 | INFO | Analyzing chunk frames = [4002304,4002816) | 20260508 01:01:45.702 | INFO | MemInfo: { | 20260508 01:01:45.702 | INFO | "gpu" : " 17 GB", | 20260508 01:01:45.702 | INFO | "huge" : "", | 20260508 01:01:45.702 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:45.702 | INFO | "malloc" : " 14 GB", | 20260508 01:01:45.702 | INFO | "pinned" : " 29 GB" | 20260508 01:01:45.702 | INFO | } | 20260508 01:01:45.702 | INFO | Perf report for the last 5.13656 seconds: | 20260508 01:01:45.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:45.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.702 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 01:01:45.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.702 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:01:45.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:45.703 | INFO | Uploaded 36GiB to GPU in 2.55278s (14.1023GiB/s) | 20260508 01:01:45.703 | INFO | Downloaded 34GiB to GPU in 2.41301s (14.0903GiB/s) | 20260508 01:01:50.839 | INFO | Analyzing chunk frames = [4002816,4003328) | 20260508 01:01:50.839 | INFO | MemInfo: { | 20260508 01:01:50.839 | INFO | "gpu" : " 17 GB", | 20260508 01:01:50.839 | INFO | "huge" : "", | 20260508 01:01:50.839 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:50.839 | INFO | "malloc" : " 14 GB", | 20260508 01:01:50.839 | INFO | "pinned" : " 29 GB" | 20260508 01:01:50.839 | INFO | } | 20260508 01:01:50.839 | INFO | Perf report for the last 5.13675 seconds: | 20260508 01:01:50.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:50.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.839 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260508 01:01:50.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.839 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:01:50.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:50.840 | INFO | Uploaded 36GiB to GPU in 2.57496s (13.9808GiB/s) | 20260508 01:01:50.840 | INFO | Downloaded 34GiB to GPU in 2.48461s (13.6843GiB/s) | 20260508 01:01:55.980 | INFO | Analyzing chunk frames = [4003328,4003840) | 20260508 01:01:55.980 | INFO | MemInfo: { | 20260508 01:01:55.980 | INFO | "gpu" : " 17 GB", | 20260508 01:01:55.980 | INFO | "huge" : "", | 20260508 01:01:55.980 | INFO | "huge_shared" : " 29 GB", | 20260508 01:01:55.980 | INFO | "malloc" : " 14 GB", | 20260508 01:01:55.980 | INFO | "pinned" : " 29 GB" | 20260508 01:01:55.980 | INFO | } | 20260508 01:01:55.980 | INFO | Perf report for the last 5.14107 seconds: | 20260508 01:01:55.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:01:55.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.980 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 01:01:55.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.980 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:01:55.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:01:55.981 | INFO | Uploaded 36GiB to GPU in 2.54494s (14.1457GiB/s) | 20260508 01:01:55.981 | INFO | Downloaded 34GiB to GPU in 2.47042s (13.7628GiB/s) | 20260508 01:02:01.117 | INFO | Analyzing chunk frames = [4003840,4004352) | 20260508 01:02:01.117 | INFO | MemInfo: { | 20260508 01:02:01.117 | INFO | "gpu" : " 17 GB", | 20260508 01:02:01.117 | INFO | "huge" : "", | 20260508 01:02:01.117 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:01.117 | INFO | "malloc" : " 14 GB", | 20260508 01:02:01.117 | INFO | "pinned" : " 29 GB" | 20260508 01:02:01.117 | INFO | } | 20260508 01:02:01.117 | INFO | Perf report for the last 5.13688 seconds: | 20260508 01:02:01.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:01.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.117 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:02:01.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.117 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:02:01.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:01.118 | INFO | Uploaded 36GiB to GPU in 2.38528s (15.0926GiB/s) | 20260508 01:02:01.118 | INFO | Downloaded 34GiB to GPU in 2.4989s (13.606GiB/s) | 20260508 01:02:06.261 | INFO | Analyzing chunk frames = [4004352,4004864) | 20260508 01:02:06.261 | INFO | MemInfo: { | 20260508 01:02:06.261 | INFO | "gpu" : " 17 GB", | 20260508 01:02:06.261 | INFO | "huge" : "", | 20260508 01:02:06.261 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:06.261 | INFO | "malloc" : " 14 GB", | 20260508 01:02:06.261 | INFO | "pinned" : " 29 GB" | 20260508 01:02:06.261 | INFO | } | 20260508 01:02:06.261 | INFO | Perf report for the last 5.14433 seconds: | 20260508 01:02:06.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:06.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.261 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:02:06.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.261 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 01:02:06.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:06.262 | INFO | Uploaded 36GiB to GPU in 2.40302s (14.9811GiB/s) | 20260508 01:02:06.262 | INFO | Downloaded 34GiB to GPU in 2.4023s (14.1531GiB/s) | 20260508 01:02:11.397 | INFO | Analyzing chunk frames = [4004864,4005376) | 20260508 01:02:11.397 | INFO | MemInfo: { | 20260508 01:02:11.397 | INFO | "gpu" : " 17 GB", | 20260508 01:02:11.397 | INFO | "huge" : "", | 20260508 01:02:11.397 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:11.397 | INFO | "malloc" : " 14 GB", | 20260508 01:02:11.397 | INFO | "pinned" : " 29 GB" | 20260508 01:02:11.397 | INFO | } | 20260508 01:02:11.397 | INFO | Perf report for the last 5.13544 seconds: | 20260508 01:02:11.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:11.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 01:02:11.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:02:11.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:11.397 | INFO | Uploaded 36GiB to GPU in 2.43087s (14.8095GiB/s) | 20260508 01:02:11.397 | INFO | Downloaded 34GiB to GPU in 2.42752s (14.0061GiB/s) | 20260508 01:02:16.539 | INFO | Analyzing chunk frames = [4005376,4005888) | 20260508 01:02:16.539 | INFO | MemInfo: { | 20260508 01:02:16.539 | INFO | "gpu" : " 17 GB", | 20260508 01:02:16.539 | INFO | "huge" : "", | 20260508 01:02:16.539 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:16.539 | INFO | "malloc" : " 14 GB", | 20260508 01:02:16.539 | INFO | "pinned" : " 29 GB" | 20260508 01:02:16.539 | INFO | } | 20260508 01:02:16.539 | INFO | Perf report for the last 5.14214 seconds: | 20260508 01:02:16.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:16.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:02:16.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:02:16.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:16.539 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:02:16.539 | INFO | Total mem usage: 16.7 GiB | 20260508 01:02:16.539 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:02:16.539 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260508 01:02:16.539 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:02:16.539 | INFO | High Water Breakdown: | 20260508 01:02:16.539 | INFO | 59.9% : 8.82 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:02:16.539 | INFO | 40% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:02:16.539 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:02:16.539 | INFO | ------------------------------------------ | 20260508 01:02:16.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:02:16.539 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:02:16.540 | INFO | Total mem usage: 26.5 GiB | 20260508 01:02:16.540 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:02:16.540 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:02:16.540 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:02:16.540 | INFO | High Water Breakdown: | 20260508 01:02:16.540 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:02:16.540 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:02:16.540 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:02:16.540 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:02:16.540 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:02:16.540 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:02:16.540 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:02:16.540 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:02:16.540 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:02:16.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:02:16.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:02:16.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:02:16.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:02:16.540 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:02:16.540 | INFO | ------------------------------------------ | 20260508 01:02:16.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:02:16.540 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:02:16.540 | INFO | Total mem usage: 27.8 GiB | 20260508 01:02:16.540 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:02:16.540 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:02:16.540 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:02:16.540 | INFO | Uploaded 36GiB to GPU in 2.53021s (14.2281GiB/s) | 20260508 01:02:16.540 | INFO | High Water Breakdown: | 20260508 01:02:16.540 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:02:16.540 | INFO | ------------------------------------------ | 20260508 01:02:16.540 | INFO | Downloaded 34GiB to GPU in 2.44578s (13.9015GiB/s) | 20260508 01:02:16.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:02:16.540 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:02:16.540 | INFO | Total mem usage: 15.7 GiB | 20260508 01:02:16.540 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:02:16.540 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:02:16.540 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:02:16.540 | INFO | High Water Breakdown: | 20260508 01:02:16.540 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:02:16.540 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:02:16.540 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:02:16.540 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:02:16.540 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:02:16.540 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:02:16.540 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:02:16.540 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:02:16.540 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:02:16.540 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:02:16.540 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:02:16.540 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:02:16.540 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:02:16.540 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:02:16.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:02:16.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:02:16.540 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:02:16.540 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:02:16.540 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:02:16.540 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:02:16.540 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:02:16.540 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:02:16.540 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:02:16.540 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:02:16.540 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:02:16.540 | INFO | ------------------------------------------ | 20260508 01:02:16.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:02:16.540 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:02:16.540 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:02:16.540 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:02:16.540 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.43e+01 / 1.49e+00 | 20260508 01:02:16.540 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.18e-02 / 6.18e-03 | 20260508 01:02:16.540 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.46e+00, 5.88e-01 / 1.98e-01 | 20260508 01:02:16.540 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.06e-01 / 1.14e-01 | 20260508 01:02:16.540 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.60e+01, 4.07e+00 / 1.86e+00 | 20260508 01:02:16.540 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.57e+00 / 1.53e+00 | 20260508 01:02:16.540 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.27e-01 / 2.23e-01 | 20260508 01:02:16.540 | INFO | PerfReport: 3.38%, Metrics -- 1.11e+00 / 1.31e+00, 2.75e-01 / 1.54e-01 | 20260508 01:02:16.540 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.77e-01, 5.37e-02 / 2.90e-02 | 20260508 01:02:16.540 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:02:16.540 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:02:16.540 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:02:16.540 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 01:02:16.540 | INFO | PerfReport: 24.39%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 01:02:16.540 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:02:16.540 | INFO | PerfReport: 30.67%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 01:02:16.541 | INFO | PerfReport: 2.35%, LockUpload -- 2.50e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 01:02:16.541 | INFO | PerfReport: 2.69%, LockDownload -- 2.87e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 01:02:16.541 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 01:02:16.541 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260508 01:02:16.637 | INFO | Starting a round of Baz IO | 20260508 01:02:20.102 | INFO | Populating RTMetrics report | 20260508 01:02:20.102 | INFO | RTMetrics report populated | 20260508 01:02:20.102 | INFO | Opening tmp RTMetrics json file | 20260508 01:02:20.123 | INFO | Serializing RTMetrics to json | 20260508 01:02:20.133 | INFO | Serialization done, swapping tmp file now... | 20260508 01:02:20.143 | INFO | RTMetrics done | 20260508 01:02:20.206 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.569 seconds | 20260508 01:02:20.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.98% / 0.00% / 58.02% | 20260508 01:02:20.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.918388% / 1.19209e-05% / 5.5631e-05% | 20260508 01:02:20.206 | INFO | Baz IO overall: | 20260508 01:02:20.206 | INFO | BytesWritten : 1817506880367 | 20260508 01:02:20.206 | INFO | NumEvents : 879393389383 | 20260508 01:02:21.677 | INFO | Analyzing chunk frames = [4005888,4006400) | 20260508 01:02:21.677 | INFO | MemInfo: { | 20260508 01:02:21.677 | INFO | "gpu" : " 17 GB", | 20260508 01:02:21.677 | INFO | "huge" : "", | 20260508 01:02:21.677 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:21.677 | INFO | "malloc" : " 14 GB", | 20260508 01:02:21.677 | INFO | "pinned" : " 29 GB" | 20260508 01:02:21.677 | INFO | } | 20260508 01:02:21.677 | INFO | Perf report for the last 5.13836 seconds: | 20260508 01:02:21.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:21.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:21.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:21.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:21.678 | INFO | ANALYSIS : 68.4%, 3.0, 106.8ms, 0 | 20260508 01:02:21.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:21.678 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 01:02:21.678 | INFO | PRE_HQ : 65.5%, 1.7, 56.3ms, 0 | 20260508 01:02:21.678 | INFO | BAZWRITER : 69.0%, 3.6, 128.0ms, 0 | 20260508 01:02:21.678 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 01:02:21.678 | INFO | Uploaded 36GiB to GPU in 2.5683s (14.0171GiB/s) | 20260508 01:02:21.678 | INFO | Downloaded 38GiB to GPU in 2.66386s (14.265GiB/s) | 20260508 01:02:26.818 | INFO | Analyzing chunk frames = [4006400,4006912) | 20260508 01:02:26.818 | INFO | MemInfo: { | 20260508 01:02:26.818 | INFO | "gpu" : " 17 GB", | 20260508 01:02:26.818 | INFO | "huge" : "", | 20260508 01:02:26.818 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:26.818 | INFO | "malloc" : " 14 GB", | 20260508 01:02:26.818 | INFO | "pinned" : " 29 GB" | 20260508 01:02:26.818 | INFO | } | 20260508 01:02:26.818 | INFO | Perf report for the last 5.14109 seconds: | 20260508 01:02:26.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:26.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.819 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 01:02:26.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.819 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:02:26.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:26.819 | INFO | Uploaded 36GiB to GPU in 2.57216s (13.996GiB/s) | 20260508 01:02:26.819 | INFO | Downloaded 34GiB to GPU in 2.48768s (13.6674GiB/s) | 20260508 01:02:31.951 | INFO | Analyzing chunk frames = [4006912,4007424) | 20260508 01:02:31.951 | INFO | MemInfo: { | 20260508 01:02:31.951 | INFO | "gpu" : " 17 GB", | 20260508 01:02:31.951 | INFO | "huge" : "", | 20260508 01:02:31.951 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:31.951 | INFO | "malloc" : " 14 GB", | 20260508 01:02:31.951 | INFO | "pinned" : " 29 GB" | 20260508 01:02:31.951 | INFO | } | 20260508 01:02:31.951 | INFO | Perf report for the last 5.13268 seconds: | 20260508 01:02:31.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:31.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 01:02:31.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:02:31.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:31.951 | INFO | Uploaded 36GiB to GPU in 2.51697s (14.3029GiB/s) | 20260508 01:02:31.951 | INFO | Downloaded 34GiB to GPU in 2.48778s (13.6668GiB/s) | 20260508 01:02:37.090 | INFO | Analyzing chunk frames = [4007424,4007936) | 20260508 01:02:37.090 | INFO | MemInfo: { | 20260508 01:02:37.091 | INFO | "gpu" : " 17 GB", | 20260508 01:02:37.091 | INFO | "huge" : "", | 20260508 01:02:37.091 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:37.091 | INFO | "malloc" : " 14 GB", | 20260508 01:02:37.091 | INFO | "pinned" : " 29 GB" | 20260508 01:02:37.091 | INFO | } | 20260508 01:02:37.091 | INFO | Perf report for the last 5.13979 seconds: | 20260508 01:02:37.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:37.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:02:37.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:02:37.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:37.091 | INFO | Uploaded 36GiB to GPU in 2.53745s (14.1875GiB/s) | 20260508 01:02:37.091 | INFO | Downloaded 34GiB to GPU in 2.4882s (13.6645GiB/s) | 20260508 01:02:42.235 | INFO | Analyzing chunk frames = [4007936,4008448) | 20260508 01:02:42.235 | INFO | MemInfo: { | 20260508 01:02:42.235 | INFO | "gpu" : " 17 GB", | 20260508 01:02:42.235 | INFO | "huge" : "", | 20260508 01:02:42.235 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:42.235 | INFO | "malloc" : " 14 GB", | 20260508 01:02:42.235 | INFO | "pinned" : " 29 GB" | 20260508 01:02:42.235 | INFO | } | 20260508 01:02:42.235 | INFO | Perf report for the last 5.14374 seconds: | 20260508 01:02:42.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:42.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 01:02:42.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:02:42.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:42.235 | INFO | Uploaded 36GiB to GPU in 2.3727s (15.1726GiB/s) | 20260508 01:02:42.235 | INFO | Downloaded 34GiB to GPU in 2.48313s (13.6924GiB/s) | 20260508 01:02:47.380 | INFO | Analyzing chunk frames = [4008448,4008960) | 20260508 01:02:47.380 | INFO | MemInfo: { | 20260508 01:02:47.380 | INFO | "gpu" : " 17 GB", | 20260508 01:02:47.380 | INFO | "huge" : "", | 20260508 01:02:47.380 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:47.380 | INFO | "malloc" : " 14 GB", | 20260508 01:02:47.380 | INFO | "pinned" : " 29 GB" | 20260508 01:02:47.380 | INFO | } | 20260508 01:02:47.380 | INFO | Perf report for the last 5.1453 seconds: | 20260508 01:02:47.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:47.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:02:47.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:02:47.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:47.380 | INFO | Uploaded 36GiB to GPU in 2.41292s (14.9197GiB/s) | 20260508 01:02:47.380 | INFO | Downloaded 34GiB to GPU in 2.41478s (14.08GiB/s) | 20260508 01:02:52.515 | INFO | Analyzing chunk frames = [4008960,4009472) | 20260508 01:02:52.515 | INFO | MemInfo: { | 20260508 01:02:52.515 | INFO | "gpu" : " 17 GB", | 20260508 01:02:52.515 | INFO | "huge" : "", | 20260508 01:02:52.515 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:52.515 | INFO | "malloc" : " 14 GB", | 20260508 01:02:52.515 | INFO | "pinned" : " 29 GB" | 20260508 01:02:52.515 | INFO | } | 20260508 01:02:52.516 | INFO | Perf report for the last 5.13551 seconds: | 20260508 01:02:52.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:52.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 01:02:52.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:02:52.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:52.516 | INFO | Uploaded 36GiB to GPU in 2.41953s (14.8789GiB/s) | 20260508 01:02:52.516 | INFO | Downloaded 34GiB to GPU in 2.41353s (14.0873GiB/s) | 20260508 01:02:57.652 | INFO | Analyzing chunk frames = [4009472,4009984) | 20260508 01:02:57.652 | INFO | MemInfo: { | 20260508 01:02:57.652 | INFO | "gpu" : " 17 GB", | 20260508 01:02:57.652 | INFO | "huge" : "", | 20260508 01:02:57.652 | INFO | "huge_shared" : " 29 GB", | 20260508 01:02:57.652 | INFO | "malloc" : " 14 GB", | 20260508 01:02:57.652 | INFO | "pinned" : " 29 GB" | 20260508 01:02:57.652 | INFO | } | 20260508 01:02:57.653 | INFO | Perf report for the last 5.13699 seconds: | 20260508 01:02:57.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:02:57.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:02:57.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:02:57.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:02:57.653 | INFO | Uploaded 36GiB to GPU in 2.56129s (14.0554GiB/s) | 20260508 01:02:57.653 | INFO | Downloaded 34GiB to GPU in 2.45135s (13.8699GiB/s) | 20260508 01:02:57.750 | INFO | Starting a round of Baz IO | 20260508 01:03:01.196 | INFO | Populating RTMetrics report | 20260508 01:03:01.197 | INFO | RTMetrics report populated | 20260508 01:03:01.197 | INFO | Opening tmp RTMetrics json file | 20260508 01:03:01.218 | INFO | Serializing RTMetrics to json | 20260508 01:03:01.224 | INFO | Serialization done, swapping tmp file now... | 20260508 01:03:01.225 | INFO | RTMetrics done | 20260508 01:03:01.297 | INFO | Finished a round of Baz IO: Wrote 1.674 GiB in 3.546 seconds | 20260508 01:03:01.297 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.97% / 0.00% / 58.02% | 20260508 01:03:01.297 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.914001% / 7.94729e-06% / 5.16574e-05% | 20260508 01:03:01.297 | INFO | Baz IO overall: | 20260508 01:03:01.297 | INFO | BytesWritten : 1819304702731 | 20260508 01:03:01.297 | INFO | NumEvents : 880257525558 | 20260508 01:03:02.793 | INFO | Analyzing chunk frames = [4009984,4010496) | 20260508 01:03:02.793 | INFO | MemInfo: { | 20260508 01:03:02.793 | INFO | "gpu" : " 17 GB", | 20260508 01:03:02.793 | INFO | "huge" : "", | 20260508 01:03:02.793 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:02.793 | INFO | "malloc" : " 14 GB", | 20260508 01:03:02.793 | INFO | "pinned" : " 29 GB" | 20260508 01:03:02.793 | INFO | } | 20260508 01:03:02.793 | INFO | Perf report for the last 5.14051 seconds: | 20260508 01:03:02.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:02.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:02.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:02.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:02.793 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 01:03:02.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:02.793 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260508 01:03:02.793 | INFO | PRE_HQ : 65.4%, 1.7, 56.5ms, 0 | 20260508 01:03:02.793 | INFO | BAZWRITER : 68.6%, 3.7, 129.5ms, 0 | 20260508 01:03:02.793 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 01:03:02.793 | INFO | Uploaded 36GiB to GPU in 2.53833s (14.1826GiB/s) | 20260508 01:03:02.793 | INFO | Downloaded 38GiB to GPU in 2.68819s (14.1359GiB/s) | 20260508 01:03:07.932 | INFO | Analyzing chunk frames = [4010496,4011008) | 20260508 01:03:07.932 | INFO | MemInfo: { | 20260508 01:03:07.932 | INFO | "gpu" : " 17 GB", | 20260508 01:03:07.932 | INFO | "huge" : "", | 20260508 01:03:07.932 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:07.932 | INFO | "malloc" : " 14 GB", | 20260508 01:03:07.932 | INFO | "pinned" : " 29 GB" | 20260508 01:03:07.932 | INFO | } | 20260508 01:03:07.932 | INFO | Perf report for the last 5.13886 seconds: | 20260508 01:03:07.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:07.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:03:07.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:03:07.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:07.932 | INFO | Uploaded 36GiB to GPU in 2.53161s (14.2202GiB/s) | 20260508 01:03:07.932 | INFO | Downloaded 34GiB to GPU in 2.42457s (14.0231GiB/s) | 20260508 01:03:13.070 | INFO | Analyzing chunk frames = [4011008,4011520) | 20260508 01:03:13.070 | INFO | MemInfo: { | 20260508 01:03:13.070 | INFO | "gpu" : " 17 GB", | 20260508 01:03:13.070 | INFO | "huge" : "", | 20260508 01:03:13.070 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:13.070 | INFO | "malloc" : " 14 GB", | 20260508 01:03:13.070 | INFO | "pinned" : " 29 GB" | 20260508 01:03:13.070 | INFO | } | 20260508 01:03:13.070 | INFO | Perf report for the last 5.1383 seconds: | 20260508 01:03:13.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:13.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.070 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:03:13.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.070 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 01:03:13.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:13.071 | INFO | Uploaded 36GiB to GPU in 2.51766s (14.299GiB/s) | 20260508 01:03:13.071 | INFO | Downloaded 34GiB to GPU in 2.49289s (13.6388GiB/s) | 20260508 01:03:18.207 | INFO | Analyzing chunk frames = [4011520,4012032) | 20260508 01:03:18.207 | INFO | MemInfo: { | 20260508 01:03:18.207 | INFO | "gpu" : " 17 GB", | 20260508 01:03:18.207 | INFO | "huge" : "", | 20260508 01:03:18.207 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:18.207 | INFO | "malloc" : " 14 GB", | 20260508 01:03:18.207 | INFO | "pinned" : " 29 GB" | 20260508 01:03:18.207 | INFO | } | 20260508 01:03:18.207 | INFO | Perf report for the last 5.13702 seconds: | 20260508 01:03:18.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:18.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.207 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 01:03:18.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.207 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:03:18.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:18.208 | INFO | Uploaded 36GiB to GPU in 2.51523s (14.3128GiB/s) | 20260508 01:03:18.208 | INFO | Downloaded 34GiB to GPU in 2.46826s (13.7749GiB/s) | 20260508 01:03:23.344 | INFO | Analyzing chunk frames = [4012032,4012544) | 20260508 01:03:23.344 | INFO | MemInfo: { | 20260508 01:03:23.344 | INFO | "gpu" : " 17 GB", | 20260508 01:03:23.344 | INFO | "huge" : "", | 20260508 01:03:23.345 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:23.345 | INFO | "malloc" : " 14 GB", | 20260508 01:03:23.345 | INFO | "pinned" : " 29 GB" | 20260508 01:03:23.345 | INFO | } | 20260508 01:03:23.345 | INFO | Perf report for the last 5.13743 seconds: | 20260508 01:03:23.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:23.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 01:03:23.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:03:23.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:23.345 | INFO | Uploaded 36GiB to GPU in 2.3236s (15.4932GiB/s) | 20260508 01:03:23.345 | INFO | Downloaded 34GiB to GPU in 2.49343s (13.6358GiB/s) | 20260508 01:03:28.489 | INFO | Analyzing chunk frames = [4012544,4013056) | 20260508 01:03:28.489 | INFO | MemInfo: { | 20260508 01:03:28.489 | INFO | "gpu" : " 17 GB", | 20260508 01:03:28.489 | INFO | "huge" : "", | 20260508 01:03:28.489 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:28.489 | INFO | "malloc" : " 14 GB", | 20260508 01:03:28.489 | INFO | "pinned" : " 29 GB" | 20260508 01:03:28.489 | INFO | } | 20260508 01:03:28.489 | INFO | Perf report for the last 5.14446 seconds: | 20260508 01:03:28.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:28.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 01:03:28.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:03:28.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:28.489 | INFO | Uploaded 36GiB to GPU in 2.51156s (14.3337GiB/s) | 20260508 01:03:28.490 | INFO | Downloaded 34GiB to GPU in 2.43332s (13.9727GiB/s) | 20260508 01:03:33.631 | INFO | Analyzing chunk frames = [4013056,4013568) | 20260508 01:03:33.631 | INFO | MemInfo: { | 20260508 01:03:33.631 | INFO | "gpu" : " 17 GB", | 20260508 01:03:33.631 | INFO | "huge" : "", | 20260508 01:03:33.632 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:33.632 | INFO | "malloc" : " 14 GB", | 20260508 01:03:33.632 | INFO | "pinned" : " 29 GB" | 20260508 01:03:33.632 | INFO | } | 20260508 01:03:33.632 | INFO | Perf report for the last 5.14252 seconds: | 20260508 01:03:33.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:33.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 01:03:33.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:03:33.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:33.632 | INFO | Uploaded 36GiB to GPU in 2.52503s (14.2573GiB/s) | 20260508 01:03:33.632 | INFO | Downloaded 34GiB to GPU in 2.4565s (13.8408GiB/s) | 20260508 01:03:38.770 | INFO | Analyzing chunk frames = [4013568,4014080) | 20260508 01:03:38.770 | INFO | MemInfo: { | 20260508 01:03:38.770 | INFO | "gpu" : " 17 GB", | 20260508 01:03:38.770 | INFO | "huge" : "", | 20260508 01:03:38.770 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:38.770 | INFO | "malloc" : " 14 GB", | 20260508 01:03:38.770 | INFO | "pinned" : " 29 GB" | 20260508 01:03:38.770 | INFO | } | 20260508 01:03:38.770 | INFO | Perf report for the last 5.13853 seconds: | 20260508 01:03:38.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:38.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.770 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 01:03:38.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.770 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:03:38.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:38.771 | INFO | Uploaded 36GiB to GPU in 2.5472s (14.1332GiB/s) | 20260508 01:03:38.771 | INFO | Downloaded 34GiB to GPU in 2.45792s (13.8328GiB/s) | 20260508 01:03:38.771 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:03:38.771 | INFO | Total mem usage: 16.7 GiB | 20260508 01:03:38.771 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:03:38.771 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260508 01:03:38.771 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:03:38.771 | INFO | High Water Breakdown: | 20260508 01:03:38.771 | INFO | 59.8% : 8.74 GiB ( 84.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:03:38.771 | INFO | 40.1% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:03:38.771 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:03:38.771 | INFO | ------------------------------------------ | 20260508 01:03:38.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:03:38.771 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:03:38.771 | INFO | Total mem usage: 26.5 GiB | 20260508 01:03:38.771 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:03:38.771 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:03:38.771 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:03:38.771 | INFO | High Water Breakdown: | 20260508 01:03:38.771 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:03:38.771 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:03:38.771 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:03:38.771 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:03:38.771 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:03:38.771 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:03:38.771 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:03:38.771 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:03:38.771 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:03:38.771 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:03:38.771 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:03:38.771 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:03:38.771 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:03:38.771 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:03:38.771 | INFO | ------------------------------------------ | 20260508 01:03:38.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:03:38.771 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:03:38.771 | INFO | Total mem usage: 27.8 GiB | 20260508 01:03:38.771 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:03:38.771 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:03:38.771 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:03:38.771 | INFO | High Water Breakdown: | 20260508 01:03:38.771 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:03:38.771 | INFO | ------------------------------------------ | 20260508 01:03:38.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:03:38.771 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:03:38.771 | INFO | Total mem usage: 15.7 GiB | 20260508 01:03:38.771 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:03:38.771 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:03:38.771 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:03:38.771 | INFO | High Water Breakdown: | 20260508 01:03:38.771 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:03:38.772 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:03:38.772 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:03:38.772 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:03:38.772 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:03:38.772 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:03:38.772 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:03:38.772 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:03:38.772 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:03:38.772 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:03:38.772 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:03:38.772 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:03:38.772 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:03:38.772 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:03:38.772 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:03:38.772 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:03:38.772 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:03:38.772 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:03:38.772 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:03:38.772 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:03:38.772 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:03:38.772 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:03:38.772 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:03:38.772 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:03:38.772 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:03:38.772 | INFO | ------------------------------------------ | 20260508 01:03:38.772 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:03:38.772 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:03:38.772 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:03:38.772 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:03:38.772 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.34e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.00e-01, 1.20e-02 / 5.35e-03 | 20260508 01:03:38.772 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.46e+00, 5.87e-01 / 2.32e-01 | 20260508 01:03:38.772 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.06e-01 / 1.03e-01 | 20260508 01:03:38.772 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.62e+01, 4.04e+00 / 1.63e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 56.16%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.59e+00 / 1.59e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.75e+00, 4.28e-01 / 1.93e-01 | 20260508 01:03:38.772 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.31e+00, 2.77e-01 / 1.65e-01 | 20260508 01:03:38.772 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.70e-01, 5.34e-02 / 2.21e-02 | 20260508 01:03:38.772 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:03:38.772 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:03:38.772 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:03:38.772 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 30.66%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 2.20%, LockUpload -- 2.35e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 2.83%, LockDownload -- 3.02e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:03:38.772 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260508 01:03:38.867 | INFO | Starting a round of Baz IO | 20260508 01:03:42.312 | INFO | Populating RTMetrics report | 20260508 01:03:42.313 | INFO | RTMetrics report populated | 20260508 01:03:42.313 | INFO | Opening tmp RTMetrics json file | 20260508 01:03:42.333 | INFO | Serializing RTMetrics to json | 20260508 01:03:42.335 | INFO | Serialization done, swapping tmp file now... | 20260508 01:03:42.336 | INFO | RTMetrics done | 20260508 01:03:42.415 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.547 seconds | 20260508 01:03:42.415 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.97% / 0.00% / 58.03% | 20260508 01:03:42.415 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.910393% / 0% / 6.75519e-05% | 20260508 01:03:42.415 | INFO | Baz IO overall: | 20260508 01:03:42.415 | INFO | BytesWritten : 1821101378234 | 20260508 01:03:42.415 | INFO | NumEvents : 881120783723 | 20260508 01:03:43.906 | INFO | Analyzing chunk frames = [4014080,4014592) | 20260508 01:03:43.906 | INFO | MemInfo: { | 20260508 01:03:43.906 | INFO | "gpu" : " 17 GB", | 20260508 01:03:43.906 | INFO | "huge" : "", | 20260508 01:03:43.906 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:43.906 | INFO | "malloc" : " 14 GB", | 20260508 01:03:43.906 | INFO | "pinned" : " 29 GB" | 20260508 01:03:43.906 | INFO | } | 20260508 01:03:43.906 | INFO | Perf report for the last 5.13577 seconds: | 20260508 01:03:43.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:43.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:43.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:43.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:43.906 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 01:03:43.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:43.906 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260508 01:03:43.906 | INFO | PRE_HQ : 65.8%, 1.7, 56.4ms, 0 | 20260508 01:03:43.906 | INFO | BAZWRITER : 68.7%, 3.8, 133.0ms, 0 | 20260508 01:03:43.906 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:03:43.906 | INFO | Uploaded 36GiB to GPU in 2.5453s (14.1437GiB/s) | 20260508 01:03:43.906 | INFO | Downloaded 38GiB to GPU in 2.69723s (14.0885GiB/s) | 20260508 01:03:49.049 | INFO | Analyzing chunk frames = [4014592,4015104) | 20260508 01:03:49.049 | INFO | MemInfo: { | 20260508 01:03:49.049 | INFO | "gpu" : " 17 GB", | 20260508 01:03:49.049 | INFO | "huge" : "", | 20260508 01:03:49.049 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:49.049 | INFO | "malloc" : " 14 GB", | 20260508 01:03:49.049 | INFO | "pinned" : " 29 GB" | 20260508 01:03:49.049 | INFO | } | 20260508 01:03:49.049 | INFO | Perf report for the last 5.14304 seconds: | 20260508 01:03:49.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:49.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:03:49.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:03:49.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:49.049 | INFO | Uploaded 36GiB to GPU in 2.54746s (14.1317GiB/s) | 20260508 01:03:49.049 | INFO | Downloaded 34GiB to GPU in 2.42851s (14.0003GiB/s) | 20260508 01:03:54.187 | INFO | Analyzing chunk frames = [4015104,4015616) | 20260508 01:03:54.187 | INFO | MemInfo: { | 20260508 01:03:54.187 | INFO | "gpu" : " 17 GB", | 20260508 01:03:54.187 | INFO | "huge" : "", | 20260508 01:03:54.187 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:54.187 | INFO | "malloc" : " 14 GB", | 20260508 01:03:54.187 | INFO | "pinned" : " 29 GB" | 20260508 01:03:54.187 | INFO | } | 20260508 01:03:54.187 | INFO | Perf report for the last 5.13802 seconds: | 20260508 01:03:54.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:54.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:03:54.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:03:54.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:54.187 | INFO | Uploaded 36GiB to GPU in 2.52704s (14.2459GiB/s) | 20260508 01:03:54.187 | INFO | Downloaded 34GiB to GPU in 2.48849s (13.6629GiB/s) | 20260508 01:03:59.327 | INFO | Analyzing chunk frames = [4015616,4016128) | 20260508 01:03:59.327 | INFO | MemInfo: { | 20260508 01:03:59.327 | INFO | "gpu" : " 17 GB", | 20260508 01:03:59.327 | INFO | "huge" : "", | 20260508 01:03:59.327 | INFO | "huge_shared" : " 29 GB", | 20260508 01:03:59.327 | INFO | "malloc" : " 14 GB", | 20260508 01:03:59.327 | INFO | "pinned" : " 29 GB" | 20260508 01:03:59.327 | INFO | } | 20260508 01:03:59.327 | INFO | Perf report for the last 5.13988 seconds: | 20260508 01:03:59.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:03:59.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.327 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 01:03:59.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.327 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:03:59.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:03:59.328 | INFO | Uploaded 36GiB to GPU in 2.5211s (14.2795GiB/s) | 20260508 01:03:59.328 | INFO | Downloaded 34GiB to GPU in 2.46995s (13.7655GiB/s) | 20260508 01:04:04.466 | INFO | Analyzing chunk frames = [4016128,4016640) | 20260508 01:04:04.466 | INFO | MemInfo: { | 20260508 01:04:04.466 | INFO | "gpu" : " 17 GB", | 20260508 01:04:04.466 | INFO | "huge" : "", | 20260508 01:04:04.466 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:04.466 | INFO | "malloc" : " 14 GB", | 20260508 01:04:04.466 | INFO | "pinned" : " 29 GB" | 20260508 01:04:04.466 | INFO | } | 20260508 01:04:04.466 | INFO | Perf report for the last 5.139 seconds: | 20260508 01:04:04.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:04.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.466 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 01:04:04.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.466 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:04:04.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:04.467 | INFO | Uploaded 36GiB to GPU in 2.34237s (15.3691GiB/s) | 20260508 01:04:04.467 | INFO | Downloaded 34GiB to GPU in 2.47775s (13.7221GiB/s) | 20260508 01:04:09.603 | INFO | Analyzing chunk frames = [4016640,4017152) | 20260508 01:04:09.603 | INFO | MemInfo: { | 20260508 01:04:09.603 | INFO | "gpu" : " 17 GB", | 20260508 01:04:09.603 | INFO | "huge" : "", | 20260508 01:04:09.603 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:09.603 | INFO | "malloc" : " 14 GB", | 20260508 01:04:09.603 | INFO | "pinned" : " 29 GB" | 20260508 01:04:09.603 | INFO | } | 20260508 01:04:09.603 | INFO | Perf report for the last 5.13683 seconds: | 20260508 01:04:09.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:09.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 01:04:09.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:04:09.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:09.603 | INFO | Uploaded 36GiB to GPU in 2.42683s (14.8342GiB/s) | 20260508 01:04:09.603 | INFO | Downloaded 34GiB to GPU in 2.39885s (14.1734GiB/s) | 20260508 01:04:14.745 | INFO | Analyzing chunk frames = [4017152,4017664) | 20260508 01:04:14.745 | INFO | MemInfo: { | 20260508 01:04:14.745 | INFO | "gpu" : " 17 GB", | 20260508 01:04:14.745 | INFO | "huge" : "", | 20260508 01:04:14.745 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:14.745 | INFO | "malloc" : " 14 GB", | 20260508 01:04:14.745 | INFO | "pinned" : " 29 GB" | 20260508 01:04:14.745 | INFO | } | 20260508 01:04:14.745 | INFO | Perf report for the last 5.14258 seconds: | 20260508 01:04:14.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:14.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.746 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 01:04:14.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.746 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:04:14.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:14.746 | INFO | Uploaded 36GiB to GPU in 2.55284s (14.1019GiB/s) | 20260508 01:04:14.746 | INFO | Downloaded 34GiB to GPU in 2.44563s (13.9023GiB/s) | 20260508 01:04:19.879 | INFO | Analyzing chunk frames = [4017664,4018176) | 20260508 01:04:19.879 | INFO | MemInfo: { | 20260508 01:04:19.879 | INFO | "gpu" : " 17 GB", | 20260508 01:04:19.879 | INFO | "huge" : "", | 20260508 01:04:19.879 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:19.880 | INFO | "malloc" : " 14 GB", | 20260508 01:04:19.880 | INFO | "pinned" : " 29 GB" | 20260508 01:04:19.880 | INFO | } | 20260508 01:04:19.880 | INFO | Perf report for the last 5.13422 seconds: | 20260508 01:04:19.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:19.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 01:04:19.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 01:04:19.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:19.880 | INFO | Uploaded 36GiB to GPU in 2.56268s (14.0478GiB/s) | 20260508 01:04:19.880 | INFO | Downloaded 34GiB to GPU in 2.44646s (13.8977GiB/s) | 20260508 01:04:19.992 | INFO | Starting a round of Baz IO | 20260508 01:04:23.449 | INFO | Populating RTMetrics report | 20260508 01:04:23.449 | INFO | RTMetrics report populated | 20260508 01:04:23.449 | INFO | Opening tmp RTMetrics json file | 20260508 01:04:23.459 | INFO | Serializing RTMetrics to json | 20260508 01:04:23.471 | INFO | Serialization done, swapping tmp file now... | 20260508 01:04:23.471 | INFO | RTMetrics done | 20260508 01:04:23.549 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.556 seconds | 20260508 01:04:23.549 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.96% / 0.00% / 58.03% | 20260508 01:04:23.549 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.911478% / 7.94729e-06% / 6.75519e-05% | 20260508 01:04:23.549 | INFO | Baz IO overall: | 20260508 01:04:23.549 | INFO | BytesWritten : 1822898176682 | 20260508 01:04:23.549 | INFO | NumEvents : 881984262546 | 20260508 01:04:25.018 | INFO | Analyzing chunk frames = [4018176,4018688) | 20260508 01:04:25.018 | INFO | MemInfo: { | 20260508 01:04:25.018 | INFO | "gpu" : " 17 GB", | 20260508 01:04:25.018 | INFO | "huge" : "", | 20260508 01:04:25.018 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:25.018 | INFO | "malloc" : " 14 GB", | 20260508 01:04:25.018 | INFO | "pinned" : " 29 GB" | 20260508 01:04:25.018 | INFO | } | 20260508 01:04:25.018 | INFO | Perf report for the last 5.13855 seconds: | 20260508 01:04:25.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:25.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:25.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:25.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:25.018 | INFO | ANALYSIS : 68.8%, 2.9, 104.6ms, 0 | 20260508 01:04:25.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:25.018 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260508 01:04:25.018 | INFO | PRE_HQ : 67.5%, 1.8, 62.6ms, 0 | 20260508 01:04:25.018 | INFO | BAZWRITER : 68.8%, 3.9, 136.3ms, 0 | 20260508 01:04:25.018 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:04:25.019 | INFO | Uploaded 36GiB to GPU in 2.59536s (13.8709GiB/s) | 20260508 01:04:25.019 | INFO | Downloaded 38GiB to GPU in 2.69914s (14.0785GiB/s) | 20260508 01:04:30.154 | INFO | Analyzing chunk frames = [4018688,4019200) | 20260508 01:04:30.154 | INFO | MemInfo: { | 20260508 01:04:30.154 | INFO | "gpu" : " 17 GB", | 20260508 01:04:30.154 | INFO | "huge" : "", | 20260508 01:04:30.154 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:30.154 | INFO | "malloc" : " 14 GB", | 20260508 01:04:30.154 | INFO | "pinned" : " 29 GB" | 20260508 01:04:30.154 | INFO | } | 20260508 01:04:30.154 | INFO | Perf report for the last 5.13625 seconds: | 20260508 01:04:30.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:30.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 01:04:30.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:04:30.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:30.155 | INFO | Uploaded 36GiB to GPU in 2.55986s (14.0633GiB/s) | 20260508 01:04:30.155 | INFO | Downloaded 34GiB to GPU in 2.4228s (14.0333GiB/s) | 20260508 01:04:35.297 | INFO | Analyzing chunk frames = [4019200,4019712) | 20260508 01:04:35.297 | INFO | MemInfo: { | 20260508 01:04:35.297 | INFO | "gpu" : " 17 GB", | 20260508 01:04:35.297 | INFO | "huge" : "", | 20260508 01:04:35.297 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:35.297 | INFO | "malloc" : " 14 GB", | 20260508 01:04:35.297 | INFO | "pinned" : " 29 GB" | 20260508 01:04:35.297 | INFO | } | 20260508 01:04:35.297 | INFO | Perf report for the last 5.14228 seconds: | 20260508 01:04:35.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:35.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:04:35.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:04:35.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:35.297 | INFO | Uploaded 36GiB to GPU in 2.53464s (14.2032GiB/s) | 20260508 01:04:35.297 | INFO | Downloaded 34GiB to GPU in 2.47139s (13.7574GiB/s) | 20260508 01:04:40.445 | INFO | Analyzing chunk frames = [4019712,4020224) | 20260508 01:04:40.445 | INFO | MemInfo: { | 20260508 01:04:40.445 | INFO | "gpu" : " 17 GB", | 20260508 01:04:40.445 | INFO | "huge" : "", | 20260508 01:04:40.445 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:40.446 | INFO | "malloc" : " 14 GB", | 20260508 01:04:40.446 | INFO | "pinned" : " 29 GB" | 20260508 01:04:40.446 | INFO | } | 20260508 01:04:40.446 | INFO | Perf report for the last 5.14881 seconds: | 20260508 01:04:40.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:40.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:04:40.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:04:40.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:40.446 | INFO | Uploaded 36GiB to GPU in 2.53551s (14.1983GiB/s) | 20260508 01:04:40.446 | INFO | Downloaded 34GiB to GPU in 2.47246s (13.7515GiB/s) | 20260508 01:04:45.581 | INFO | Analyzing chunk frames = [4020224,4020736) | 20260508 01:04:45.581 | INFO | MemInfo: { | 20260508 01:04:45.581 | INFO | "gpu" : " 17 GB", | 20260508 01:04:45.581 | INFO | "huge" : "", | 20260508 01:04:45.581 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:45.581 | INFO | "malloc" : " 14 GB", | 20260508 01:04:45.581 | INFO | "pinned" : " 29 GB" | 20260508 01:04:45.581 | INFO | } | 20260508 01:04:45.581 | INFO | Perf report for the last 5.1352 seconds: | 20260508 01:04:45.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:45.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 01:04:45.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:04:45.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:45.581 | INFO | Uploaded 36GiB to GPU in 2.37792s (15.1393GiB/s) | 20260508 01:04:45.581 | INFO | Downloaded 34GiB to GPU in 2.52285s (13.4768GiB/s) | 20260508 01:04:50.719 | INFO | Analyzing chunk frames = [4020736,4021248) | 20260508 01:04:50.719 | INFO | MemInfo: { | 20260508 01:04:50.719 | INFO | "gpu" : " 17 GB", | 20260508 01:04:50.719 | INFO | "huge" : "", | 20260508 01:04:50.719 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:50.719 | INFO | "malloc" : " 14 GB", | 20260508 01:04:50.719 | INFO | "pinned" : " 29 GB" | 20260508 01:04:50.719 | INFO | } | 20260508 01:04:50.719 | INFO | Perf report for the last 5.1386 seconds: | 20260508 01:04:50.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:50.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:04:50.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:04:50.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:50.720 | INFO | Uploaded 36GiB to GPU in 2.3947s (15.0332GiB/s) | 20260508 01:04:50.720 | INFO | Downloaded 34GiB to GPU in 2.40452s (14.14GiB/s) | 20260508 01:04:55.858 | INFO | Analyzing chunk frames = [4021248,4021760) | 20260508 01:04:55.858 | INFO | MemInfo: { | 20260508 01:04:55.858 | INFO | "gpu" : " 17 GB", | 20260508 01:04:55.858 | INFO | "huge" : "", | 20260508 01:04:55.858 | INFO | "huge_shared" : " 29 GB", | 20260508 01:04:55.858 | INFO | "malloc" : " 14 GB", | 20260508 01:04:55.858 | INFO | "pinned" : " 29 GB" | 20260508 01:04:55.858 | INFO | } | 20260508 01:04:55.858 | INFO | Perf report for the last 5.13887 seconds: | 20260508 01:04:55.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:04:55.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.858 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:04:55.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.858 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:04:55.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:04:55.859 | INFO | Uploaded 36GiB to GPU in 2.44341s (14.7335GiB/s) | 20260508 01:04:55.859 | INFO | Downloaded 34GiB to GPU in 2.42527s (14.0191GiB/s) | 20260508 01:05:00.999 | INFO | Analyzing chunk frames = [4021760,4022272) | 20260508 01:05:00.999 | INFO | MemInfo: { | 20260508 01:05:00.999 | INFO | "gpu" : " 17 GB", | 20260508 01:05:00.999 | INFO | "huge" : "", | 20260508 01:05:00.999 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:00.999 | INFO | "malloc" : " 14 GB", | 20260508 01:05:00.999 | INFO | "pinned" : " 29 GB" | 20260508 01:05:00.999 | INFO | } | 20260508 01:05:00.999 | INFO | Perf report for the last 5.14085 seconds: | 20260508 01:05:00.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:00.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:00.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:00.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:00.999 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:05:00.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:00.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:00.999 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:05:00.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:00.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:01.000 | INFO | Uploaded 36GiB to GPU in 2.52398s (14.2632GiB/s) | 20260508 01:05:01.000 | INFO | Downloaded 34GiB to GPU in 2.44971s (13.8792GiB/s) | 20260508 01:05:01.000 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:05:01.000 | INFO | Total mem usage: 16.7 GiB | 20260508 01:05:01.000 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:05:01.000 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260508 01:05:01.000 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:05:01.000 | INFO | High Water Breakdown: | 20260508 01:05:01.000 | INFO | 59.8% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:05:01.000 | INFO | 40.1% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:05:01.000 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:05:01.000 | INFO | ------------------------------------------ | 20260508 01:05:01.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:05:01.000 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:05:01.000 | INFO | Total mem usage: 26.5 GiB | 20260508 01:05:01.000 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:05:01.000 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:05:01.000 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:05:01.000 | INFO | High Water Breakdown: | 20260508 01:05:01.000 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:05:01.000 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:05:01.000 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:05:01.000 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:05:01.000 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:05:01.000 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:05:01.000 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:05:01.000 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:05:01.000 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:05:01.000 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:05:01.000 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:05:01.000 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:05:01.000 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:05:01.000 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:05:01.000 | INFO | ------------------------------------------ | 20260508 01:05:01.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:05:01.000 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:05:01.000 | INFO | Total mem usage: 27.8 GiB | 20260508 01:05:01.000 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:05:01.000 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:05:01.000 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:05:01.000 | INFO | High Water Breakdown: | 20260508 01:05:01.000 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:05:01.000 | INFO | ------------------------------------------ | 20260508 01:05:01.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:05:01.001 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:05:01.001 | INFO | Total mem usage: 15.7 GiB | 20260508 01:05:01.001 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:05:01.001 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:05:01.001 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:05:01.001 | INFO | High Water Breakdown: | 20260508 01:05:01.001 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:05:01.001 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:05:01.001 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:05:01.001 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:05:01.001 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:05:01.001 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:05:01.001 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:05:01.001 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:05:01.001 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:05:01.001 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:05:01.001 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:05:01.001 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:05:01.001 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:05:01.001 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:05:01.001 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:05:01.001 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:05:01.001 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:05:01.001 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:05:01.001 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:05:01.001 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:05:01.001 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:05:01.001 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:05:01.001 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:05:01.001 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:05:01.001 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:05:01.001 | INFO | ------------------------------------------ | 20260508 01:05:01.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:05:01.001 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:05:01.001 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:05:01.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:05:01.001 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.39e+01 / 1.66e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.12e-02 / 4.91e-03 | 20260508 01:05:01.001 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.45e+00, 5.81e-01 / 2.17e-01 | 20260508 01:05:01.001 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.04e-01 / 1.17e-01 | 20260508 01:05:01.001 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.61e+01, 4.11e+00 / 1.83e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.60e+00 / 1.46e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.44e+00 / 2.69e+00, 4.25e-01 / 2.64e-01 | 20260508 01:05:01.001 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.27e+00, 2.71e-01 / 1.34e-01 | 20260508 01:05:01.001 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.73e-01, 5.29e-02 / 2.67e-02 | 20260508 01:05:01.001 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:05:01.001 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:05:01.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:05:01.001 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 2.31%, LockUpload -- 2.46e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 2.64%, LockDownload -- 2.81e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:05:01.001 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 01:05:01.096 | INFO | Starting a round of Baz IO | 20260508 01:05:04.549 | INFO | Populating RTMetrics report | 20260508 01:05:04.549 | INFO | RTMetrics report populated | 20260508 01:05:04.549 | INFO | Opening tmp RTMetrics json file | 20260508 01:05:04.569 | INFO | Serializing RTMetrics to json | 20260508 01:05:04.571 | INFO | Serialization done, swapping tmp file now... | 20260508 01:05:04.571 | INFO | RTMetrics done | 20260508 01:05:04.650 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.553 seconds | 20260508 01:05:04.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.96% / 0.00% / 58.04% | 20260508 01:05:04.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.908113% / 7.94729e-06% / 6.35783e-05% | 20260508 01:05:04.650 | INFO | Baz IO overall: | 20260508 01:05:04.650 | INFO | BytesWritten : 1824694439146 | 20260508 01:05:04.650 | INFO | NumEvents : 882847400265 | 20260508 01:05:06.136 | INFO | Analyzing chunk frames = [4022272,4022784) | 20260508 01:05:06.136 | INFO | MemInfo: { | 20260508 01:05:06.136 | INFO | "gpu" : " 17 GB", | 20260508 01:05:06.136 | INFO | "huge" : "", | 20260508 01:05:06.136 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:06.136 | INFO | "malloc" : " 14 GB", | 20260508 01:05:06.136 | INFO | "pinned" : " 29 GB" | 20260508 01:05:06.136 | INFO | } | 20260508 01:05:06.137 | INFO | Perf report for the last 5.1373 seconds: | 20260508 01:05:06.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:06.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:06.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:06.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:06.137 | INFO | ANALYSIS : 68.4%, 3.0, 106.3ms, 0 | 20260508 01:05:06.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:06.137 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:05:06.137 | INFO | PRE_HQ : 65.6%, 1.6, 56.1ms, 0 | 20260508 01:05:06.137 | INFO | BAZWRITER : 68.8%, 3.8, 134.9ms, 0 | 20260508 01:05:06.137 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 01:05:06.137 | INFO | Uploaded 36GiB to GPU in 2.5715s (13.9996GiB/s) | 20260508 01:05:06.137 | INFO | Downloaded 38GiB to GPU in 2.69641s (14.0928GiB/s) | 20260508 01:05:11.273 | INFO | Analyzing chunk frames = [4022784,4023296) | 20260508 01:05:11.273 | INFO | MemInfo: { | 20260508 01:05:11.273 | INFO | "gpu" : " 17 GB", | 20260508 01:05:11.273 | INFO | "huge" : "", | 20260508 01:05:11.273 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:11.273 | INFO | "malloc" : " 14 GB", | 20260508 01:05:11.273 | INFO | "pinned" : " 29 GB" | 20260508 01:05:11.273 | INFO | } | 20260508 01:05:11.273 | INFO | Perf report for the last 5.13632 seconds: | 20260508 01:05:11.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:11.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 01:05:11.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:05:11.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:11.273 | INFO | Uploaded 36GiB to GPU in 2.50094s (14.3946GiB/s) | 20260508 01:05:11.273 | INFO | Downloaded 34GiB to GPU in 2.43622s (13.9561GiB/s) | 20260508 01:05:16.411 | INFO | Analyzing chunk frames = [4023296,4023808) | 20260508 01:05:16.412 | INFO | MemInfo: { | 20260508 01:05:16.412 | INFO | "gpu" : " 17 GB", | 20260508 01:05:16.412 | INFO | "huge" : "", | 20260508 01:05:16.412 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:16.412 | INFO | "malloc" : " 14 GB", | 20260508 01:05:16.412 | INFO | "pinned" : " 29 GB" | 20260508 01:05:16.412 | INFO | } | 20260508 01:05:16.412 | INFO | Perf report for the last 5.13875 seconds: | 20260508 01:05:16.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:16.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:05:16.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:05:16.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:16.412 | INFO | Uploaded 36GiB to GPU in 2.4617s (14.6241GiB/s) | 20260508 01:05:16.412 | INFO | Downloaded 34GiB to GPU in 2.50622s (13.5662GiB/s) | 20260508 01:05:21.553 | INFO | Analyzing chunk frames = [4023808,4024320) | 20260508 01:05:21.553 | INFO | MemInfo: { | 20260508 01:05:21.553 | INFO | "gpu" : " 17 GB", | 20260508 01:05:21.553 | INFO | "huge" : "", | 20260508 01:05:21.553 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:21.553 | INFO | "malloc" : " 14 GB", | 20260508 01:05:21.553 | INFO | "pinned" : " 29 GB" | 20260508 01:05:21.553 | INFO | } | 20260508 01:05:21.553 | INFO | Perf report for the last 5.14175 seconds: | 20260508 01:05:21.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:21.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.553 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:05:21.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.554 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:05:21.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:21.554 | INFO | Uploaded 36GiB to GPU in 2.46307s (14.6159GiB/s) | 20260508 01:05:21.554 | INFO | Downloaded 34GiB to GPU in 2.50888s (13.5518GiB/s) | 20260508 01:05:26.695 | INFO | Analyzing chunk frames = [4024320,4024832) | 20260508 01:05:26.695 | INFO | MemInfo: { | 20260508 01:05:26.695 | INFO | "gpu" : " 17 GB", | 20260508 01:05:26.695 | INFO | "huge" : "", | 20260508 01:05:26.695 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:26.696 | INFO | "malloc" : " 14 GB", | 20260508 01:05:26.696 | INFO | "pinned" : " 29 GB" | 20260508 01:05:26.696 | INFO | } | 20260508 01:05:26.696 | INFO | Perf report for the last 5.14219 seconds: | 20260508 01:05:26.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:26.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 01:05:26.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:05:26.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:26.696 | INFO | Uploaded 36GiB to GPU in 2.30279s (15.6332GiB/s) | 20260508 01:05:26.696 | INFO | Downloaded 34GiB to GPU in 2.50248s (13.5865GiB/s) | 20260508 01:05:31.831 | INFO | Analyzing chunk frames = [4024832,4025344) | 20260508 01:05:31.831 | INFO | MemInfo: { | 20260508 01:05:31.831 | INFO | "gpu" : " 17 GB", | 20260508 01:05:31.831 | INFO | "huge" : "", | 20260508 01:05:31.831 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:31.831 | INFO | "malloc" : " 14 GB", | 20260508 01:05:31.831 | INFO | "pinned" : " 29 GB" | 20260508 01:05:31.831 | INFO | } | 20260508 01:05:31.831 | INFO | Perf report for the last 5.13572 seconds: | 20260508 01:05:31.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:31.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.831 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 01:05:31.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.831 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 01:05:31.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:31.832 | INFO | Uploaded 36GiB to GPU in 2.40803s (14.95GiB/s) | 20260508 01:05:31.832 | INFO | Downloaded 34GiB to GPU in 2.4097s (14.1097GiB/s) | 20260508 01:05:36.965 | INFO | Analyzing chunk frames = [4025344,4025856) | 20260508 01:05:36.965 | INFO | MemInfo: { | 20260508 01:05:36.965 | INFO | "gpu" : " 17 GB", | 20260508 01:05:36.965 | INFO | "huge" : "", | 20260508 01:05:36.965 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:36.965 | INFO | "malloc" : " 14 GB", | 20260508 01:05:36.965 | INFO | "pinned" : " 29 GB" | 20260508 01:05:36.965 | INFO | } | 20260508 01:05:36.965 | INFO | Perf report for the last 5.13365 seconds: | 20260508 01:05:36.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:36.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 01:05:36.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:05:36.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:36.965 | INFO | Uploaded 36GiB to GPU in 2.53223s (14.2167GiB/s) | 20260508 01:05:36.965 | INFO | Downloaded 34GiB to GPU in 2.463s (13.8043GiB/s) | 20260508 01:05:42.107 | INFO | Analyzing chunk frames = [4025856,4026368) | 20260508 01:05:42.107 | INFO | MemInfo: { | 20260508 01:05:42.107 | INFO | "gpu" : " 17 GB", | 20260508 01:05:42.107 | INFO | "huge" : "", | 20260508 01:05:42.107 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:42.107 | INFO | "malloc" : " 14 GB", | 20260508 01:05:42.107 | INFO | "pinned" : " 29 GB" | 20260508 01:05:42.107 | INFO | } | 20260508 01:05:42.107 | INFO | Perf report for the last 5.14206 seconds: | 20260508 01:05:42.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:42.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.107 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:05:42.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.107 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:05:42.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:42.108 | INFO | Uploaded 36GiB to GPU in 2.5307s (14.2253GiB/s) | 20260508 01:05:42.108 | INFO | Downloaded 34GiB to GPU in 2.45074s (13.8734GiB/s) | 20260508 01:05:42.205 | INFO | Starting a round of Baz IO | 20260508 01:05:45.667 | INFO | Populating RTMetrics report | 20260508 01:05:45.667 | INFO | RTMetrics report populated | 20260508 01:05:45.667 | INFO | Opening tmp RTMetrics json file | 20260508 01:05:45.683 | INFO | Serializing RTMetrics to json | 20260508 01:05:45.695 | INFO | Serialization done, swapping tmp file now... | 20260508 01:05:45.697 | INFO | RTMetrics done | 20260508 01:05:45.786 | INFO | Finished a round of Baz IO: Wrote 1.674 GiB in 3.581 seconds | 20260508 01:05:45.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.96% / 0.00% / 58.04% | 20260508 01:05:45.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.909774% / 0% / 8.34465e-05% | 20260508 01:05:45.786 | INFO | Baz IO overall: | 20260508 01:05:45.786 | INFO | BytesWritten : 1826491464253 | 20260508 01:05:45.786 | INFO | NumEvents : 883711173601 | 20260508 01:05:47.250 | INFO | Analyzing chunk frames = [4026368,4026880) | 20260508 01:05:47.250 | INFO | MemInfo: { | 20260508 01:05:47.250 | INFO | "gpu" : " 17 GB", | 20260508 01:05:47.250 | INFO | "huge" : "", | 20260508 01:05:47.250 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:47.250 | INFO | "malloc" : " 14 GB", | 20260508 01:05:47.250 | INFO | "pinned" : " 29 GB" | 20260508 01:05:47.250 | INFO | } | 20260508 01:05:47.250 | INFO | Perf report for the last 5.14307 seconds: | 20260508 01:05:47.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:47.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:47.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:47.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:47.250 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260508 01:05:47.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:47.250 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260508 01:05:47.250 | INFO | PRE_HQ : 65.5%, 1.7, 56.4ms, 0 | 20260508 01:05:47.250 | INFO | BAZWRITER : 68.9%, 3.6, 126.8ms, 0 | 20260508 01:05:47.250 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 01:05:47.250 | INFO | Uploaded 36GiB to GPU in 2.58188s (13.9433GiB/s) | 20260508 01:05:47.250 | INFO | Downloaded 38GiB to GPU in 2.70348s (14.056GiB/s) | 20260508 01:05:52.392 | INFO | Analyzing chunk frames = [4026880,4027392) | 20260508 01:05:52.392 | INFO | MemInfo: { | 20260508 01:05:52.392 | INFO | "gpu" : " 17 GB", | 20260508 01:05:52.392 | INFO | "huge" : "", | 20260508 01:05:52.392 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:52.392 | INFO | "malloc" : " 14 GB", | 20260508 01:05:52.392 | INFO | "pinned" : " 29 GB" | 20260508 01:05:52.392 | INFO | } | 20260508 01:05:52.392 | INFO | Perf report for the last 5.14208 seconds: | 20260508 01:05:52.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:52.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.392 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:05:52.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.392 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:05:52.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:52.393 | INFO | Uploaded 36GiB to GPU in 2.52243s (14.2719GiB/s) | 20260508 01:05:52.393 | INFO | Downloaded 34GiB to GPU in 2.4322s (13.9791GiB/s) | 20260508 01:05:57.532 | INFO | Analyzing chunk frames = [4027392,4027904) | 20260508 01:05:57.532 | INFO | MemInfo: { | 20260508 01:05:57.532 | INFO | "gpu" : " 17 GB", | 20260508 01:05:57.532 | INFO | "huge" : "", | 20260508 01:05:57.532 | INFO | "huge_shared" : " 29 GB", | 20260508 01:05:57.532 | INFO | "malloc" : " 14 GB", | 20260508 01:05:57.532 | INFO | "pinned" : " 29 GB" | 20260508 01:05:57.532 | INFO | } | 20260508 01:05:57.532 | INFO | Perf report for the last 5.14002 seconds: | 20260508 01:05:57.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:05:57.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.532 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 01:05:57.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.532 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:05:57.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:05:57.533 | INFO | Uploaded 36GiB to GPU in 2.52585s (14.2526GiB/s) | 20260508 01:05:57.533 | INFO | Downloaded 34GiB to GPU in 2.49449s (13.6301GiB/s) | 20260508 01:06:02.667 | INFO | Analyzing chunk frames = [4027904,4028416) | 20260508 01:06:02.667 | INFO | MemInfo: { | 20260508 01:06:02.667 | INFO | "gpu" : " 17 GB", | 20260508 01:06:02.667 | INFO | "huge" : "", | 20260508 01:06:02.667 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:02.667 | INFO | "malloc" : " 14 GB", | 20260508 01:06:02.667 | INFO | "pinned" : " 29 GB" | 20260508 01:06:02.667 | INFO | } | 20260508 01:06:02.667 | INFO | Perf report for the last 5.1351 seconds: | 20260508 01:06:02.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:02.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.667 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 01:06:02.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.668 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:06:02.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:02.668 | INFO | Uploaded 36GiB to GPU in 2.53498s (14.2013GiB/s) | 20260508 01:06:02.668 | INFO | Downloaded 34GiB to GPU in 2.49585s (13.6226GiB/s) | 20260508 01:06:07.812 | INFO | Analyzing chunk frames = [4028416,4028928) | 20260508 01:06:07.812 | INFO | MemInfo: { | 20260508 01:06:07.812 | INFO | "gpu" : " 17 GB", | 20260508 01:06:07.812 | INFO | "huge" : "", | 20260508 01:06:07.812 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:07.812 | INFO | "malloc" : " 14 GB", | 20260508 01:06:07.812 | INFO | "pinned" : " 29 GB" | 20260508 01:06:07.812 | INFO | } | 20260508 01:06:07.812 | INFO | Perf report for the last 5.14488 seconds: | 20260508 01:06:07.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:07.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.812 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:06:07.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.812 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 01:06:07.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:07.813 | INFO | Uploaded 36GiB to GPU in 2.3452s (15.3505GiB/s) | 20260508 01:06:07.813 | INFO | Downloaded 34GiB to GPU in 2.51339s (13.5275GiB/s) | 20260508 01:06:12.948 | INFO | Analyzing chunk frames = [4028928,4029440) | 20260508 01:06:12.948 | INFO | MemInfo: { | 20260508 01:06:12.948 | INFO | "gpu" : " 17 GB", | 20260508 01:06:12.948 | INFO | "huge" : "", | 20260508 01:06:12.949 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:12.949 | INFO | "malloc" : " 14 GB", | 20260508 01:06:12.949 | INFO | "pinned" : " 29 GB" | 20260508 01:06:12.949 | INFO | } | 20260508 01:06:12.949 | INFO | Perf report for the last 5.13642 seconds: | 20260508 01:06:12.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:12.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:06:12.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:06:12.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:12.949 | INFO | Uploaded 36GiB to GPU in 2.39982s (15.0011GiB/s) | 20260508 01:06:12.949 | INFO | Downloaded 34GiB to GPU in 2.40446s (14.1404GiB/s) | 20260508 01:06:18.086 | INFO | Analyzing chunk frames = [4029440,4029952) | 20260508 01:06:18.086 | INFO | MemInfo: { | 20260508 01:06:18.086 | INFO | "gpu" : " 17 GB", | 20260508 01:06:18.086 | INFO | "huge" : "", | 20260508 01:06:18.086 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:18.086 | INFO | "malloc" : " 14 GB", | 20260508 01:06:18.086 | INFO | "pinned" : " 29 GB" | 20260508 01:06:18.086 | INFO | } | 20260508 01:06:18.086 | INFO | Perf report for the last 5.13746 seconds: | 20260508 01:06:18.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:18.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.086 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 01:06:18.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.086 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:06:18.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:18.087 | INFO | Uploaded 36GiB to GPU in 2.53982s (14.1742GiB/s) | 20260508 01:06:18.087 | INFO | Downloaded 34GiB to GPU in 2.46238s (13.8078GiB/s) | 20260508 01:06:23.222 | INFO | Analyzing chunk frames = [4029952,4030464) | 20260508 01:06:23.222 | INFO | MemInfo: { | 20260508 01:06:23.222 | INFO | "gpu" : " 17 GB", | 20260508 01:06:23.222 | INFO | "huge" : "", | 20260508 01:06:23.222 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:23.223 | INFO | "malloc" : " 14 GB", | 20260508 01:06:23.223 | INFO | "pinned" : " 29 GB" | 20260508 01:06:23.223 | INFO | } | 20260508 01:06:23.223 | INFO | Perf report for the last 5.13649 seconds: | 20260508 01:06:23.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:23.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 01:06:23.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:06:23.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:23.223 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:06:23.223 | INFO | Total mem usage: 16.7 GiB | 20260508 01:06:23.223 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:06:23.223 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.7%) | 20260508 01:06:23.223 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:06:23.223 | INFO | High Water Breakdown: | 20260508 01:06:23.223 | INFO | 60.2% : 8.99 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:06:23.223 | INFO | 39.7% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:06:23.223 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:06:23.223 | INFO | ------------------------------------------ | 20260508 01:06:23.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:06:23.223 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:06:23.223 | INFO | Total mem usage: 26.5 GiB | 20260508 01:06:23.223 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:06:23.223 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:06:23.223 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:06:23.223 | INFO | High Water Breakdown: | 20260508 01:06:23.223 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:06:23.223 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:06:23.223 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:06:23.223 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:06:23.223 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:06:23.223 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:06:23.223 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:06:23.223 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:06:23.223 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:06:23.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:06:23.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:06:23.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:06:23.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:06:23.223 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:06:23.223 | INFO | ------------------------------------------ | 20260508 01:06:23.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:06:23.223 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:06:23.223 | INFO | Total mem usage: 27.8 GiB | 20260508 01:06:23.223 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:06:23.223 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:06:23.223 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:06:23.223 | INFO | High Water Breakdown: | 20260508 01:06:23.223 | INFO | Uploaded 36GiB to GPU in 2.51979s (14.2869GiB/s) | 20260508 01:06:23.224 | INFO | Downloaded 34GiB to GPU in 2.45058s (13.8743GiB/s) | 20260508 01:06:23.224 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:06:23.224 | INFO | ------------------------------------------ | 20260508 01:06:23.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:06:23.224 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:06:23.224 | INFO | Total mem usage: 15.7 GiB | 20260508 01:06:23.224 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:06:23.224 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:06:23.224 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:06:23.224 | INFO | High Water Breakdown: | 20260508 01:06:23.224 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:06:23.224 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:06:23.224 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:06:23.224 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:06:23.224 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:06:23.224 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:06:23.224 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:06:23.224 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:06:23.224 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:06:23.224 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:06:23.224 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:06:23.224 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:06:23.224 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:06:23.224 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:06:23.224 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:06:23.224 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:06:23.224 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:06:23.224 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:06:23.224 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:06:23.224 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:06:23.224 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:06:23.224 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:06:23.224 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:06:23.224 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:06:23.224 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:06:23.224 | INFO | ------------------------------------------ | 20260508 01:06:23.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:06:23.224 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:06:23.224 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:06:23.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:06:23.224 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.41e+01 / 1.54e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.12e-02 / 7.38e-03 | 20260508 01:06:23.224 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.48e+00, 5.90e-01 / 1.85e-01 | 20260508 01:06:23.224 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 1.13e-01 | 20260508 01:06:23.224 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.62e+01, 4.13e+00 / 1.72e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.50e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.76e+00, 4.23e-01 / 1.76e-01 | 20260508 01:06:23.224 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.29e+00, 2.69e-01 / 1.46e-01 | 20260508 01:06:23.224 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.75e-01, 5.39e-02 / 2.83e-02 | 20260508 01:06:23.224 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:06:23.224 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:06:23.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:06:23.224 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 24.27%, Upload -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 2.12%, LockUpload -- 2.26e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 2.87%, LockDownload -- 3.06e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:06:23.224 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 01:06:23.320 | INFO | Starting a round of Baz IO | 20260508 01:06:26.773 | INFO | Populating RTMetrics report | 20260508 01:06:26.773 | INFO | RTMetrics report populated | 20260508 01:06:26.773 | INFO | Opening tmp RTMetrics json file | 20260508 01:06:26.819 | INFO | Serializing RTMetrics to json | 20260508 01:06:26.866 | INFO | Serialization done, swapping tmp file now... | 20260508 01:06:26.867 | INFO | RTMetrics done | 20260508 01:06:26.890 | INFO | Finished a round of Baz IO: Wrote 1.674 GiB in 3.569 seconds | 20260508 01:06:26.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.95% / 0.00% / 58.04% | 20260508 01:06:26.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.90818% / 3.97364e-06% / 6.75519e-05% | 20260508 01:06:26.890 | INFO | Baz IO overall: | 20260508 01:06:26.890 | INFO | BytesWritten : 1828289129860 | 20260508 01:06:26.890 | INFO | NumEvents : 884575444911 | 20260508 01:06:28.365 | INFO | Analyzing chunk frames = [4030464,4030976) | 20260508 01:06:28.365 | INFO | MemInfo: { | 20260508 01:06:28.365 | INFO | "gpu" : " 17 GB", | 20260508 01:06:28.365 | INFO | "huge" : "", | 20260508 01:06:28.365 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:28.365 | INFO | "malloc" : " 14 GB", | 20260508 01:06:28.365 | INFO | "pinned" : " 29 GB" | 20260508 01:06:28.365 | INFO | } | 20260508 01:06:28.365 | INFO | Perf report for the last 5.14205 seconds: | 20260508 01:06:28.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:28.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:28.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:28.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:28.365 | INFO | ANALYSIS : 68.3%, 3.0, 106.7ms, 0 | 20260508 01:06:28.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:28.365 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 01:06:28.365 | INFO | PRE_HQ : 65.3%, 1.7, 56.5ms, 0 | 20260508 01:06:28.365 | INFO | BAZWRITER : 69.0%, 3.6, 128.8ms, 0 | 20260508 01:06:28.365 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 01:06:28.366 | INFO | Uploaded 36GiB to GPU in 2.53491s (14.2017GiB/s) | 20260508 01:06:28.366 | INFO | Downloaded 38GiB to GPU in 2.69228s (14.1144GiB/s) | 20260508 01:06:33.502 | INFO | Analyzing chunk frames = [4030976,4031488) | 20260508 01:06:33.502 | INFO | MemInfo: { | 20260508 01:06:33.502 | INFO | "gpu" : " 17 GB", | 20260508 01:06:33.502 | INFO | "huge" : "", | 20260508 01:06:33.502 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:33.502 | INFO | "malloc" : " 14 GB", | 20260508 01:06:33.503 | INFO | "pinned" : " 29 GB" | 20260508 01:06:33.503 | INFO | } | 20260508 01:06:33.503 | INFO | Perf report for the last 5.13787 seconds: | 20260508 01:06:33.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:33.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 01:06:33.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:06:33.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:33.503 | INFO | Uploaded 36GiB to GPU in 2.57028s (14.0062GiB/s) | 20260508 01:06:33.503 | INFO | Downloaded 34GiB to GPU in 2.49034s (13.6528GiB/s) | 20260508 01:06:38.645 | INFO | Analyzing chunk frames = [4031488,4032000) | 20260508 01:06:38.645 | INFO | MemInfo: { | 20260508 01:06:38.645 | INFO | "gpu" : " 17 GB", | 20260508 01:06:38.645 | INFO | "huge" : "", | 20260508 01:06:38.645 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:38.645 | INFO | "malloc" : " 14 GB", | 20260508 01:06:38.645 | INFO | "pinned" : " 29 GB" | 20260508 01:06:38.645 | INFO | } | 20260508 01:06:38.645 | INFO | Perf report for the last 5.14269 seconds: | 20260508 01:06:38.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:38.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.645 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 01:06:38.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.645 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:06:38.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:38.646 | INFO | Uploaded 36GiB to GPU in 2.52114s (14.2793GiB/s) | 20260508 01:06:38.646 | INFO | Downloaded 34GiB to GPU in 2.48649s (13.6739GiB/s) | 20260508 01:06:43.780 | INFO | Analyzing chunk frames = [4032000,4032512) | 20260508 01:06:43.780 | INFO | MemInfo: { | 20260508 01:06:43.780 | INFO | "gpu" : " 17 GB", | 20260508 01:06:43.780 | INFO | "huge" : "", | 20260508 01:06:43.780 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:43.780 | INFO | "malloc" : " 14 GB", | 20260508 01:06:43.780 | INFO | "pinned" : " 29 GB" | 20260508 01:06:43.780 | INFO | } | 20260508 01:06:43.780 | INFO | Perf report for the last 5.13494 seconds: | 20260508 01:06:43.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:43.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.780 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:06:43.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.780 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:06:43.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:43.781 | INFO | Uploaded 36GiB to GPU in 2.50004s (14.3998GiB/s) | 20260508 01:06:43.781 | INFO | Downloaded 34GiB to GPU in 2.49358s (13.635GiB/s) | 20260508 01:06:48.918 | INFO | Analyzing chunk frames = [4032512,4033024) | 20260508 01:06:48.918 | INFO | MemInfo: { | 20260508 01:06:48.918 | INFO | "gpu" : " 17 GB", | 20260508 01:06:48.918 | INFO | "huge" : "", | 20260508 01:06:48.918 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:48.918 | INFO | "malloc" : " 14 GB", | 20260508 01:06:48.918 | INFO | "pinned" : " 29 GB" | 20260508 01:06:48.918 | INFO | } | 20260508 01:06:48.918 | INFO | Perf report for the last 5.1376 seconds: | 20260508 01:06:48.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:48.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 01:06:48.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:06:48.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:48.918 | INFO | Uploaded 36GiB to GPU in 2.33503s (15.4174GiB/s) | 20260508 01:06:48.918 | INFO | Downloaded 34GiB to GPU in 2.50208s (13.5887GiB/s) | 20260508 01:06:54.061 | INFO | Analyzing chunk frames = [4033024,4033536) | 20260508 01:06:54.061 | INFO | MemInfo: { | 20260508 01:06:54.061 | INFO | "gpu" : " 17 GB", | 20260508 01:06:54.061 | INFO | "huge" : "", | 20260508 01:06:54.061 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:54.061 | INFO | "malloc" : " 14 GB", | 20260508 01:06:54.061 | INFO | "pinned" : " 29 GB" | 20260508 01:06:54.061 | INFO | } | 20260508 01:06:54.061 | INFO | Perf report for the last 5.14289 seconds: | 20260508 01:06:54.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:54.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:06:54.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:06:54.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:54.061 | INFO | Uploaded 36GiB to GPU in 2.36243s (15.2385GiB/s) | 20260508 01:06:54.061 | INFO | Downloaded 34GiB to GPU in 2.4341s (13.9682GiB/s) | 20260508 01:06:59.203 | INFO | Analyzing chunk frames = [4033536,4034048) | 20260508 01:06:59.203 | INFO | MemInfo: { | 20260508 01:06:59.203 | INFO | "gpu" : " 17 GB", | 20260508 01:06:59.203 | INFO | "huge" : "", | 20260508 01:06:59.203 | INFO | "huge_shared" : " 29 GB", | 20260508 01:06:59.203 | INFO | "malloc" : " 14 GB", | 20260508 01:06:59.203 | INFO | "pinned" : " 29 GB" | 20260508 01:06:59.203 | INFO | } | 20260508 01:06:59.203 | INFO | Perf report for the last 5.14222 seconds: | 20260508 01:06:59.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:06:59.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.203 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:06:59.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.203 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:06:59.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:06:59.204 | INFO | Uploaded 36GiB to GPU in 2.52979s (14.2304GiB/s) | 20260508 01:06:59.204 | INFO | Downloaded 34GiB to GPU in 2.47898s (13.7153GiB/s) | 20260508 01:07:04.346 | INFO | Analyzing chunk frames = [4034048,4034560) | 20260508 01:07:04.346 | INFO | MemInfo: { | 20260508 01:07:04.346 | INFO | "gpu" : " 17 GB", | 20260508 01:07:04.346 | INFO | "huge" : "", | 20260508 01:07:04.346 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:04.346 | INFO | "malloc" : " 14 GB", | 20260508 01:07:04.346 | INFO | "pinned" : " 29 GB" | 20260508 01:07:04.346 | INFO | } | 20260508 01:07:04.346 | INFO | Perf report for the last 5.14271 seconds: | 20260508 01:07:04.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:04.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:07:04.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:07:04.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:04.346 | INFO | Uploaded 36GiB to GPU in 2.5158s (14.3096GiB/s) | 20260508 01:07:04.346 | INFO | Downloaded 34GiB to GPU in 2.4673s (13.7803GiB/s) | 20260508 01:07:04.443 | INFO | Starting a round of Baz IO | 20260508 01:07:07.894 | INFO | Populating RTMetrics report | 20260508 01:07:07.894 | INFO | RTMetrics report populated | 20260508 01:07:07.894 | INFO | Opening tmp RTMetrics json file | 20260508 01:07:07.918 | INFO | Serializing RTMetrics to json | 20260508 01:07:07.997 | INFO | Serialization done, swapping tmp file now... | 20260508 01:07:08.001 | INFO | RTMetrics done | 20260508 01:07:08.038 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.594 seconds | 20260508 01:07:08.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.95% / 0.00% / 58.05% | 20260508 01:07:08.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.907564% / 3.97364e-06% / 5.5631e-05% | 20260508 01:07:08.038 | INFO | Baz IO overall: | 20260508 01:07:08.038 | INFO | BytesWritten : 1830085956529 | 20260508 01:07:08.038 | INFO | NumEvents : 885438950460 | 20260508 01:07:09.484 | INFO | Analyzing chunk frames = [4034560,4035072) | 20260508 01:07:09.484 | INFO | MemInfo: { | 20260508 01:07:09.484 | INFO | "gpu" : " 17 GB", | 20260508 01:07:09.484 | INFO | "huge" : "", | 20260508 01:07:09.484 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:09.484 | INFO | "malloc" : " 14 GB", | 20260508 01:07:09.484 | INFO | "pinned" : " 29 GB" | 20260508 01:07:09.484 | INFO | } | 20260508 01:07:09.484 | INFO | Perf report for the last 5.13806 seconds: | 20260508 01:07:09.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:09.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:09.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:09.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:09.484 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 01:07:09.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:09.484 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 01:07:09.484 | INFO | PRE_HQ : 65.7%, 1.7, 56.4ms, 0 | 20260508 01:07:09.484 | INFO | BAZWRITER : 69.6%, 3.6, 127.5ms, 0 | 20260508 01:07:09.484 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260508 01:07:09.484 | INFO | Uploaded 36GiB to GPU in 2.54587s (14.1406GiB/s) | 20260508 01:07:09.484 | INFO | Downloaded 38GiB to GPU in 2.68579s (14.1485GiB/s) | 20260508 01:07:14.623 | INFO | Analyzing chunk frames = [4035072,4035584) | 20260508 01:07:14.623 | INFO | MemInfo: { | 20260508 01:07:14.623 | INFO | "gpu" : " 17 GB", | 20260508 01:07:14.623 | INFO | "huge" : "", | 20260508 01:07:14.623 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:14.623 | INFO | "malloc" : " 14 GB", | 20260508 01:07:14.623 | INFO | "pinned" : " 29 GB" | 20260508 01:07:14.623 | INFO | } | 20260508 01:07:14.623 | INFO | Perf report for the last 5.13896 seconds: | 20260508 01:07:14.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:14.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:07:14.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:07:14.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:14.623 | INFO | Uploaded 36GiB to GPU in 2.53772s (14.186GiB/s) | 20260508 01:07:14.623 | INFO | Downloaded 34GiB to GPU in 2.4349s (13.9636GiB/s) | 20260508 01:07:19.765 | INFO | Analyzing chunk frames = [4035584,4036096) | 20260508 01:07:19.765 | INFO | MemInfo: { | 20260508 01:07:19.765 | INFO | "gpu" : " 17 GB", | 20260508 01:07:19.765 | INFO | "huge" : "", | 20260508 01:07:19.765 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:19.765 | INFO | "malloc" : " 14 GB", | 20260508 01:07:19.765 | INFO | "pinned" : " 29 GB" | 20260508 01:07:19.765 | INFO | } | 20260508 01:07:19.765 | INFO | Perf report for the last 5.1421 seconds: | 20260508 01:07:19.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:19.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 01:07:19.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:07:19.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:19.765 | INFO | Uploaded 36GiB to GPU in 2.48341s (14.4962GiB/s) | 20260508 01:07:19.765 | INFO | Downloaded 34GiB to GPU in 2.50292s (13.5842GiB/s) | 20260508 01:07:24.899 | INFO | Analyzing chunk frames = [4036096,4036608) | 20260508 01:07:24.899 | INFO | MemInfo: { | 20260508 01:07:24.899 | INFO | "gpu" : " 17 GB", | 20260508 01:07:24.899 | INFO | "huge" : "", | 20260508 01:07:24.899 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:24.899 | INFO | "malloc" : " 14 GB", | 20260508 01:07:24.899 | INFO | "pinned" : " 29 GB" | 20260508 01:07:24.899 | INFO | } | 20260508 01:07:24.899 | INFO | Perf report for the last 5.13429 seconds: | 20260508 01:07:24.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:24.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.899 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260508 01:07:24.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.899 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:07:24.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:24.900 | INFO | Uploaded 36GiB to GPU in 2.49381s (14.4357GiB/s) | 20260508 01:07:24.900 | INFO | Downloaded 34GiB to GPU in 2.51416s (13.5234GiB/s) | 20260508 01:07:30.038 | INFO | Analyzing chunk frames = [4036608,4037120) | 20260508 01:07:30.038 | INFO | MemInfo: { | 20260508 01:07:30.038 | INFO | "gpu" : " 17 GB", | 20260508 01:07:30.038 | INFO | "huge" : "", | 20260508 01:07:30.038 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:30.039 | INFO | "malloc" : " 14 GB", | 20260508 01:07:30.039 | INFO | "pinned" : " 29 GB" | 20260508 01:07:30.039 | INFO | } | 20260508 01:07:30.039 | INFO | Perf report for the last 5.13942 seconds: | 20260508 01:07:30.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:30.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 01:07:30.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:07:30.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:30.039 | INFO | Uploaded 36GiB to GPU in 2.29953s (15.6554GiB/s) | 20260508 01:07:30.039 | INFO | Downloaded 34GiB to GPU in 2.49299s (13.6383GiB/s) | 20260508 01:07:35.170 | INFO | Analyzing chunk frames = [4037120,4037632) | 20260508 01:07:35.170 | INFO | MemInfo: { | 20260508 01:07:35.170 | INFO | "gpu" : " 17 GB", | 20260508 01:07:35.170 | INFO | "huge" : "", | 20260508 01:07:35.170 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:35.170 | INFO | "malloc" : " 14 GB", | 20260508 01:07:35.170 | INFO | "pinned" : " 29 GB" | 20260508 01:07:35.170 | INFO | } | 20260508 01:07:35.170 | INFO | Perf report for the last 5.13143 seconds: | 20260508 01:07:35.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:35.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:07:35.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:07:35.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:35.170 | INFO | Uploaded 36GiB to GPU in 2.38525s (15.0928GiB/s) | 20260508 01:07:35.170 | INFO | Downloaded 34GiB to GPU in 2.40105s (14.1605GiB/s) | 20260508 01:07:40.317 | INFO | Analyzing chunk frames = [4037632,4038144) | 20260508 01:07:40.317 | INFO | MemInfo: { | 20260508 01:07:40.317 | INFO | "gpu" : " 17 GB", | 20260508 01:07:40.317 | INFO | "huge" : "", | 20260508 01:07:40.317 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:40.317 | INFO | "malloc" : " 14 GB", | 20260508 01:07:40.317 | INFO | "pinned" : " 29 GB" | 20260508 01:07:40.317 | INFO | } | 20260508 01:07:40.317 | INFO | Perf report for the last 5.14714 seconds: | 20260508 01:07:40.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:40.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.317 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 01:07:40.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.317 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:07:40.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:40.318 | INFO | Uploaded 36GiB to GPU in 2.54593s (14.1402GiB/s) | 20260508 01:07:40.318 | INFO | Downloaded 34GiB to GPU in 2.466s (13.7875GiB/s) | 20260508 01:07:45.458 | INFO | Analyzing chunk frames = [4038144,4038656) | 20260508 01:07:45.458 | INFO | MemInfo: { | 20260508 01:07:45.458 | INFO | "gpu" : " 17 GB", | 20260508 01:07:45.458 | INFO | "huge" : "", | 20260508 01:07:45.458 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:45.458 | INFO | "malloc" : " 14 GB", | 20260508 01:07:45.458 | INFO | "pinned" : " 29 GB" | 20260508 01:07:45.458 | INFO | } | 20260508 01:07:45.458 | INFO | Perf report for the last 5.14057 seconds: | 20260508 01:07:45.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:45.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:07:45.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:07:45.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:45.458 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:07:45.458 | INFO | Total mem usage: 16.7 GiB | 20260508 01:07:45.458 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:07:45.458 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15%) | 20260508 01:07:45.458 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:07:45.458 | INFO | High Water Breakdown: | 20260508 01:07:45.458 | INFO | 59.7% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:07:45.458 | INFO | 40.2% : 5.83 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:07:45.458 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:07:45.458 | INFO | ------------------------------------------ | 20260508 01:07:45.458 | INFO | Uploaded 36GiB to GPU in 2.5468s (14.1354GiB/s) | 20260508 01:07:45.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:07:45.459 | INFO | Downloaded 34GiB to GPU in 2.45562s (13.8458GiB/s) | 20260508 01:07:45.459 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:07:45.459 | INFO | Total mem usage: 26.5 GiB | 20260508 01:07:45.459 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:07:45.459 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:07:45.459 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:07:45.459 | INFO | High Water Breakdown: | 20260508 01:07:45.459 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:07:45.459 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:07:45.459 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:07:45.459 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:07:45.459 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:07:45.459 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:07:45.459 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:07:45.459 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:07:45.459 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:07:45.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:07:45.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:07:45.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:07:45.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:07:45.459 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:07:45.459 | INFO | ------------------------------------------ | 20260508 01:07:45.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:07:45.459 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:07:45.459 | INFO | Total mem usage: 27.8 GiB | 20260508 01:07:45.459 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:07:45.459 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:07:45.459 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:07:45.459 | INFO | High Water Breakdown: | 20260508 01:07:45.459 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:07:45.459 | INFO | ------------------------------------------ | 20260508 01:07:45.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:07:45.459 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:07:45.459 | INFO | Total mem usage: 15.7 GiB | 20260508 01:07:45.459 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:07:45.459 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:07:45.459 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:07:45.459 | INFO | High Water Breakdown: | 20260508 01:07:45.459 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:07:45.459 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:07:45.459 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:07:45.459 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:07:45.460 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:07:45.460 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:07:45.460 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:07:45.460 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:07:45.460 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:07:45.460 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:07:45.460 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:07:45.460 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:07:45.460 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:07:45.460 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:07:45.460 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:07:45.460 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:07:45.460 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:07:45.460 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:07:45.460 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:07:45.460 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:07:45.460 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:07:45.460 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:07:45.460 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:07:45.460 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:07:45.460 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:07:45.460 | INFO | ------------------------------------------ | 20260508 01:07:45.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:07:45.460 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:07:45.460 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:07:45.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:07:45.460 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.26e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.00e-01, 1.11e-02 / 3.53e-03 | 20260508 01:07:45.460 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.48e+00, 5.83e-01 / 2.11e-01 | 20260508 01:07:45.460 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.50e+00, 2.05e-01 / 9.27e-02 | 20260508 01:07:45.460 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.62e+01, 4.05e+00 / 1.57e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 56.22%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.46e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.44e+00 / 2.72e+00, 4.28e-01 / 2.19e-01 | 20260508 01:07:45.460 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.32e+00, 2.75e-01 / 1.49e-01 | 20260508 01:07:45.460 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.73e-01, 5.36e-02 / 2.34e-02 | 20260508 01:07:45.460 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:07:45.460 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:07:45.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:07:45.460 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 24.26%, Upload -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 2.05%, LockUpload -- 2.18e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 2.97%, LockDownload -- 3.16e+00 / 0.00e+00, 4.37e+00 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:07:45.460 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 01:07:45.555 | INFO | Starting a round of Baz IO | 20260508 01:07:49.012 | INFO | Populating RTMetrics report | 20260508 01:07:49.012 | INFO | RTMetrics report populated | 20260508 01:07:49.012 | INFO | Opening tmp RTMetrics json file | 20260508 01:07:49.038 | INFO | Serializing RTMetrics to json | 20260508 01:07:49.040 | INFO | Serialization done, swapping tmp file now... | 20260508 01:07:49.041 | INFO | RTMetrics done | 20260508 01:07:49.114 | INFO | Finished a round of Baz IO: Wrote 1.672 GiB in 3.558 seconds | 20260508 01:07:49.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.94% / 0.00% / 58.05% | 20260508 01:07:49.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.902867% / 0% / 8.34465e-05% | 20260508 01:07:49.114 | INFO | Baz IO overall: | 20260508 01:07:49.114 | INFO | BytesWritten : 1831880786315 | 20260508 01:07:49.114 | INFO | NumEvents : 886301108959 | 20260508 01:07:50.594 | INFO | Analyzing chunk frames = [4038656,4039168) | 20260508 01:07:50.594 | INFO | MemInfo: { | 20260508 01:07:50.594 | INFO | "gpu" : " 17 GB", | 20260508 01:07:50.594 | INFO | "huge" : "", | 20260508 01:07:50.594 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:50.594 | INFO | "malloc" : " 14 GB", | 20260508 01:07:50.594 | INFO | "pinned" : " 29 GB" | 20260508 01:07:50.594 | INFO | } | 20260508 01:07:50.594 | INFO | Perf report for the last 5.13637 seconds: | 20260508 01:07:50.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:50.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:50.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:50.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:50.594 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260508 01:07:50.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:50.594 | INFO | RT_METRICS : 40.5%, 1.0, 21.3ms, 0 | 20260508 01:07:50.594 | INFO | PRE_HQ : 65.5%, 1.7, 56.2ms, 0 | 20260508 01:07:50.594 | INFO | BAZWRITER : 68.9%, 3.7, 132.5ms, 0 | 20260508 01:07:50.594 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 01:07:50.595 | INFO | Uploaded 36GiB to GPU in 2.58269s (13.939GiB/s) | 20260508 01:07:50.595 | INFO | Downloaded 38GiB to GPU in 2.67819s (14.1887GiB/s) | 20260508 01:07:55.730 | INFO | Analyzing chunk frames = [4039168,4039680) | 20260508 01:07:55.730 | INFO | MemInfo: { | 20260508 01:07:55.730 | INFO | "gpu" : " 17 GB", | 20260508 01:07:55.730 | INFO | "huge" : "", | 20260508 01:07:55.730 | INFO | "huge_shared" : " 29 GB", | 20260508 01:07:55.730 | INFO | "malloc" : " 14 GB", | 20260508 01:07:55.731 | INFO | "pinned" : " 29 GB" | 20260508 01:07:55.731 | INFO | } | 20260508 01:07:55.731 | INFO | Perf report for the last 5.13642 seconds: | 20260508 01:07:55.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:07:55.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:07:55.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:07:55.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:07:55.731 | INFO | Uploaded 36GiB to GPU in 2.55221s (14.1054GiB/s) | 20260508 01:07:55.731 | INFO | Downloaded 34GiB to GPU in 2.42344s (14.0296GiB/s) | 20260508 01:08:00.872 | INFO | Analyzing chunk frames = [4039680,4040192) | 20260508 01:08:00.872 | INFO | MemInfo: { | 20260508 01:08:00.873 | INFO | "gpu" : " 17 GB", | 20260508 01:08:00.873 | INFO | "huge" : "", | 20260508 01:08:00.873 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:00.873 | INFO | "malloc" : " 14 GB", | 20260508 01:08:00.873 | INFO | "pinned" : " 29 GB" | 20260508 01:08:00.873 | INFO | } | 20260508 01:08:00.873 | INFO | Perf report for the last 5.14203 seconds: | 20260508 01:08:00.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:00.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 01:08:00.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:08:00.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:00.873 | INFO | Uploaded 36GiB to GPU in 2.55393s (14.0959GiB/s) | 20260508 01:08:00.873 | INFO | Downloaded 34GiB to GPU in 2.48574s (13.678GiB/s) | 20260508 01:08:06.016 | INFO | Analyzing chunk frames = [4040192,4040704) | 20260508 01:08:06.016 | INFO | MemInfo: { | 20260508 01:08:06.016 | INFO | "gpu" : " 17 GB", | 20260508 01:08:06.016 | INFO | "huge" : "", | 20260508 01:08:06.016 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:06.016 | INFO | "malloc" : " 14 GB", | 20260508 01:08:06.016 | INFO | "pinned" : " 29 GB" | 20260508 01:08:06.016 | INFO | } | 20260508 01:08:06.016 | INFO | Perf report for the last 5.14324 seconds: | 20260508 01:08:06.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:06.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 01:08:06.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:08:06.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:06.016 | INFO | Uploaded 36GiB to GPU in 2.54295s (14.1568GiB/s) | 20260508 01:08:06.016 | INFO | Downloaded 34GiB to GPU in 2.4742s (13.7418GiB/s) | 20260508 01:08:11.154 | INFO | Analyzing chunk frames = [4040704,4041216) | 20260508 01:08:11.154 | INFO | MemInfo: { | 20260508 01:08:11.154 | INFO | "gpu" : " 17 GB", | 20260508 01:08:11.154 | INFO | "huge" : "", | 20260508 01:08:11.154 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:11.154 | INFO | "malloc" : " 14 GB", | 20260508 01:08:11.154 | INFO | "pinned" : " 29 GB" | 20260508 01:08:11.154 | INFO | } | 20260508 01:08:11.154 | INFO | Perf report for the last 5.13847 seconds: | 20260508 01:08:11.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:11.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.155 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 01:08:11.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.155 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 01:08:11.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:11.155 | INFO | Uploaded 36GiB to GPU in 2.34267s (15.3671GiB/s) | 20260508 01:08:11.155 | INFO | Downloaded 34GiB to GPU in 2.48629s (13.675GiB/s) | 20260508 01:08:16.295 | INFO | Analyzing chunk frames = [4041216,4041728) | 20260508 01:08:16.295 | INFO | MemInfo: { | 20260508 01:08:16.295 | INFO | "gpu" : " 17 GB", | 20260508 01:08:16.295 | INFO | "huge" : "", | 20260508 01:08:16.295 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:16.295 | INFO | "malloc" : " 14 GB", | 20260508 01:08:16.295 | INFO | "pinned" : " 29 GB" | 20260508 01:08:16.295 | INFO | } | 20260508 01:08:16.295 | INFO | Perf report for the last 5.1409 seconds: | 20260508 01:08:16.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:16.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.295 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:08:16.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.295 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:08:16.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:16.296 | INFO | Uploaded 36GiB to GPU in 2.4069s (14.957GiB/s) | 20260508 01:08:16.296 | INFO | Downloaded 34GiB to GPU in 2.41761s (14.0635GiB/s) | 20260508 01:08:21.431 | INFO | Analyzing chunk frames = [4041728,4042240) | 20260508 01:08:21.431 | INFO | MemInfo: { | 20260508 01:08:21.431 | INFO | "gpu" : " 17 GB", | 20260508 01:08:21.431 | INFO | "huge" : "", | 20260508 01:08:21.432 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:21.432 | INFO | "malloc" : " 14 GB", | 20260508 01:08:21.432 | INFO | "pinned" : " 29 GB" | 20260508 01:08:21.432 | INFO | } | 20260508 01:08:21.432 | INFO | Perf report for the last 5.13648 seconds: | 20260508 01:08:21.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:21.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 01:08:21.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 01:08:21.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:21.432 | INFO | Uploaded 36GiB to GPU in 2.44221s (14.7407GiB/s) | 20260508 01:08:21.432 | INFO | Downloaded 34GiB to GPU in 2.41756s (14.0638GiB/s) | 20260508 01:08:26.576 | INFO | Analyzing chunk frames = [4042240,4042752) | 20260508 01:08:26.576 | INFO | MemInfo: { | 20260508 01:08:26.576 | INFO | "gpu" : " 17 GB", | 20260508 01:08:26.576 | INFO | "huge" : "", | 20260508 01:08:26.576 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:26.576 | INFO | "malloc" : " 14 GB", | 20260508 01:08:26.576 | INFO | "pinned" : " 29 GB" | 20260508 01:08:26.576 | INFO | } | 20260508 01:08:26.576 | INFO | Perf report for the last 5.14441 seconds: | 20260508 01:08:26.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:26.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.576 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:08:26.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.577 | INFO | PRE_HQ : 34.9%, 1.0, 18.2ms, 0 | 20260508 01:08:26.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:26.577 | INFO | Uploaded 36GiB to GPU in 2.5731s (13.9909GiB/s) | 20260508 01:08:26.577 | INFO | Downloaded 34GiB to GPU in 2.46372s (13.8003GiB/s) | 20260508 01:08:26.689 | INFO | Starting a round of Baz IO | 20260508 01:08:30.144 | INFO | Populating RTMetrics report | 20260508 01:08:30.144 | INFO | RTMetrics report populated | 20260508 01:08:30.144 | INFO | Opening tmp RTMetrics json file | 20260508 01:08:30.153 | INFO | Serializing RTMetrics to json | 20260508 01:08:30.168 | INFO | Serialization done, swapping tmp file now... | 20260508 01:08:30.169 | INFO | RTMetrics done | 20260508 01:08:30.238 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.549 seconds | 20260508 01:08:30.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.94% / 0.00% / 58.06% | 20260508 01:08:30.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.903078% / 3.97364e-06% / 6.35783e-05% | 20260508 01:08:30.238 | INFO | Baz IO overall: | 20260508 01:08:30.238 | INFO | BytesWritten : 1833677332816 | 20260508 01:08:30.238 | INFO | NumEvents : 887164685791 | 20260508 01:08:31.715 | INFO | Analyzing chunk frames = [4042752,4043264) | 20260508 01:08:31.715 | INFO | MemInfo: { | 20260508 01:08:31.715 | INFO | "gpu" : " 17 GB", | 20260508 01:08:31.715 | INFO | "huge" : "", | 20260508 01:08:31.715 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:31.715 | INFO | "malloc" : " 14 GB", | 20260508 01:08:31.715 | INFO | "pinned" : " 29 GB" | 20260508 01:08:31.715 | INFO | } | 20260508 01:08:31.716 | INFO | Perf report for the last 5.13933 seconds: | 20260508 01:08:31.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:31.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:31.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:31.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:31.716 | INFO | ANALYSIS : 68.7%, 2.9, 104.4ms, 0 | 20260508 01:08:31.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:31.716 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260508 01:08:31.716 | INFO | PRE_HQ : 67.8%, 1.8, 62.5ms, 0 | 20260508 01:08:31.716 | INFO | BAZWRITER : 68.6%, 3.8, 135.4ms, 0 | 20260508 01:08:31.716 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:08:31.716 | INFO | Uploaded 36GiB to GPU in 2.55152s (14.1092GiB/s) | 20260508 01:08:31.716 | INFO | Downloaded 38GiB to GPU in 2.72028s (13.9692GiB/s) | 20260508 01:08:36.850 | INFO | Analyzing chunk frames = [4043264,4043776) | 20260508 01:08:36.850 | INFO | MemInfo: { | 20260508 01:08:36.850 | INFO | "gpu" : " 17 GB", | 20260508 01:08:36.850 | INFO | "huge" : "", | 20260508 01:08:36.850 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:36.850 | INFO | "malloc" : " 14 GB", | 20260508 01:08:36.850 | INFO | "pinned" : " 29 GB" | 20260508 01:08:36.850 | INFO | } | 20260508 01:08:36.850 | INFO | Perf report for the last 5.13452 seconds: | 20260508 01:08:36.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:36.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.850 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:08:36.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.850 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:08:36.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:36.851 | INFO | Uploaded 36GiB to GPU in 2.50534s (14.3693GiB/s) | 20260508 01:08:36.851 | INFO | Downloaded 34GiB to GPU in 2.43312s (13.9738GiB/s) | 20260508 01:08:41.984 | INFO | Analyzing chunk frames = [4043776,4044288) | 20260508 01:08:41.984 | INFO | MemInfo: { | 20260508 01:08:41.984 | INFO | "gpu" : " 17 GB", | 20260508 01:08:41.984 | INFO | "huge" : "", | 20260508 01:08:41.984 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:41.984 | INFO | "malloc" : " 14 GB", | 20260508 01:08:41.984 | INFO | "pinned" : " 29 GB" | 20260508 01:08:41.984 | INFO | } | 20260508 01:08:41.984 | INFO | Perf report for the last 5.13363 seconds: | 20260508 01:08:41.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:41.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:08:41.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:08:41.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:41.984 | INFO | Uploaded 36GiB to GPU in 2.47314s (14.5564GiB/s) | 20260508 01:08:41.984 | INFO | Downloaded 34GiB to GPU in 2.50794s (13.557GiB/s) | 20260508 01:08:47.127 | INFO | Analyzing chunk frames = [4044288,4044800) | 20260508 01:08:47.127 | INFO | MemInfo: { | 20260508 01:08:47.127 | INFO | "gpu" : " 17 GB", | 20260508 01:08:47.127 | INFO | "huge" : "", | 20260508 01:08:47.127 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:47.127 | INFO | "malloc" : " 14 GB", | 20260508 01:08:47.127 | INFO | "pinned" : " 29 GB" | 20260508 01:08:47.127 | INFO | } | 20260508 01:08:47.127 | INFO | Perf report for the last 5.14306 seconds: | 20260508 01:08:47.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:47.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 01:08:47.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:08:47.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:47.127 | INFO | Uploaded 36GiB to GPU in 2.4661s (14.598GiB/s) | 20260508 01:08:47.127 | INFO | Downloaded 34GiB to GPU in 2.49429s (13.6311GiB/s) | 20260508 01:08:52.266 | INFO | Analyzing chunk frames = [4044800,4045312) | 20260508 01:08:52.266 | INFO | MemInfo: { | 20260508 01:08:52.266 | INFO | "gpu" : " 17 GB", | 20260508 01:08:52.266 | INFO | "huge" : "", | 20260508 01:08:52.266 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:52.266 | INFO | "malloc" : " 14 GB", | 20260508 01:08:52.266 | INFO | "pinned" : " 29 GB" | 20260508 01:08:52.266 | INFO | } | 20260508 01:08:52.266 | INFO | Perf report for the last 5.1392 seconds: | 20260508 01:08:52.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:52.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.266 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 01:08:52.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.266 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:08:52.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:52.267 | INFO | Uploaded 36GiB to GPU in 2.27695s (15.8106GiB/s) | 20260508 01:08:52.267 | INFO | Downloaded 34GiB to GPU in 2.50524s (13.5716GiB/s) | 20260508 01:08:57.407 | INFO | Analyzing chunk frames = [4045312,4045824) | 20260508 01:08:57.407 | INFO | MemInfo: { | 20260508 01:08:57.407 | INFO | "gpu" : " 17 GB", | 20260508 01:08:57.407 | INFO | "huge" : "", | 20260508 01:08:57.407 | INFO | "huge_shared" : " 29 GB", | 20260508 01:08:57.407 | INFO | "malloc" : " 14 GB", | 20260508 01:08:57.407 | INFO | "pinned" : " 29 GB" | 20260508 01:08:57.407 | INFO | } | 20260508 01:08:57.407 | INFO | Perf report for the last 5.14138 seconds: | 20260508 01:08:57.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:08:57.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.408 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 01:08:57.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.408 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:08:57.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:08:57.408 | INFO | Uploaded 36GiB to GPU in 2.35811s (15.2665GiB/s) | 20260508 01:08:57.408 | INFO | Downloaded 34GiB to GPU in 2.41855s (14.058GiB/s) | 20260508 01:09:02.547 | INFO | Analyzing chunk frames = [4045824,4046336) | 20260508 01:09:02.547 | INFO | MemInfo: { | 20260508 01:09:02.547 | INFO | "gpu" : " 17 GB", | 20260508 01:09:02.547 | INFO | "huge" : "", | 20260508 01:09:02.547 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:02.547 | INFO | "malloc" : " 14 GB", | 20260508 01:09:02.547 | INFO | "pinned" : " 29 GB" | 20260508 01:09:02.547 | INFO | } | 20260508 01:09:02.547 | INFO | Perf report for the last 5.13951 seconds: | 20260508 01:09:02.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:02.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:09:02.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 01:09:02.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:02.547 | INFO | Uploaded 36GiB to GPU in 2.41175s (14.9269GiB/s) | 20260508 01:09:02.547 | INFO | Downloaded 34GiB to GPU in 2.43548s (13.9603GiB/s) | 20260508 01:09:07.685 | INFO | Analyzing chunk frames = [4046336,4046848) | 20260508 01:09:07.685 | INFO | MemInfo: { | 20260508 01:09:07.685 | INFO | "gpu" : " 17 GB", | 20260508 01:09:07.685 | INFO | "huge" : "", | 20260508 01:09:07.685 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:07.685 | INFO | "malloc" : " 14 GB", | 20260508 01:09:07.685 | INFO | "pinned" : " 29 GB" | 20260508 01:09:07.685 | INFO | } | 20260508 01:09:07.685 | INFO | Perf report for the last 5.13787 seconds: | 20260508 01:09:07.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:07.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 01:09:07.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:09:07.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:07.685 | INFO | Uploaded 36GiB to GPU in 2.49719s (14.4162GiB/s) | 20260508 01:09:07.685 | INFO | Downloaded 34GiB to GPU in 2.45899s (13.8268GiB/s) | 20260508 01:09:07.685 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:09:07.685 | INFO | Total mem usage: 16.7 GiB | 20260508 01:09:07.685 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:09:07.686 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 01:09:07.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:09:07.686 | INFO | High Water Breakdown: | 20260508 01:09:07.686 | INFO | 59.8% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:09:07.686 | INFO | 40.1% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:09:07.686 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:09:07.686 | INFO | ------------------------------------------ | 20260508 01:09:07.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:09:07.686 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:09:07.686 | INFO | Total mem usage: 26.5 GiB | 20260508 01:09:07.686 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:09:07.686 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:09:07.686 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:09:07.686 | INFO | High Water Breakdown: | 20260508 01:09:07.686 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:09:07.686 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:09:07.686 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:09:07.686 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:09:07.686 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:09:07.686 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:09:07.686 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:09:07.686 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:09:07.686 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:09:07.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:09:07.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:09:07.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:09:07.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:09:07.686 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:09:07.686 | INFO | ------------------------------------------ | 20260508 01:09:07.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:09:07.686 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:09:07.686 | INFO | Total mem usage: 27.8 GiB | 20260508 01:09:07.686 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:09:07.686 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:09:07.686 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:09:07.686 | INFO | High Water Breakdown: | 20260508 01:09:07.686 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:09:07.686 | INFO | ------------------------------------------ | 20260508 01:09:07.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:09:07.686 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:09:07.686 | INFO | Total mem usage: 15.7 GiB | 20260508 01:09:07.686 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:09:07.686 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:09:07.686 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:09:07.686 | INFO | High Water Breakdown: | 20260508 01:09:07.686 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:09:07.686 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:09:07.686 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:09:07.686 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:09:07.686 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:09:07.686 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:09:07.686 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:09:07.686 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:09:07.686 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:09:07.686 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:09:07.686 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:09:07.686 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:09:07.686 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:09:07.686 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:09:07.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:09:07.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:09:07.686 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:09:07.686 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:09:07.686 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:09:07.686 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:09:07.686 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:09:07.686 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:09:07.686 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:09:07.686 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:09:07.686 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:09:07.686 | INFO | ------------------------------------------ | 20260508 01:09:07.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:09:07.686 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:09:07.686 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:09:07.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:09:07.686 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.41e+01 / 1.43e+00 | 20260508 01:09:07.686 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.15e-02 / 8.11e-03 | 20260508 01:09:07.686 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.45e+00, 5.87e-01 / 2.21e-01 | 20260508 01:09:07.686 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.50e+00, 2.09e-01 / 1.01e-01 | 20260508 01:09:07.686 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 4.08e+00 / 1.84e+00 | 20260508 01:09:07.686 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.63e+00 / 1.60e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.28e-01 / 2.50e-01 | 20260508 01:09:07.687 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.27e+00, 2.75e-01 / 1.61e-01 | 20260508 01:09:07.687 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.77e-01, 5.40e-02 / 3.19e-02 | 20260508 01:09:07.687 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:09:07.687 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:09:07.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:09:07.687 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 24.22%, Upload -- 2.57e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 24.40%, Download -- 2.59e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 2.12%, LockUpload -- 2.25e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 2.83%, LockDownload -- 3.01e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:09:07.687 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 01:09:07.782 | INFO | Starting a round of Baz IO | 20260508 01:09:11.239 | INFO | Populating RTMetrics report | 20260508 01:09:11.239 | INFO | RTMetrics report populated | 20260508 01:09:11.239 | INFO | Opening tmp RTMetrics json file | 20260508 01:09:11.258 | INFO | Serializing RTMetrics to json | 20260508 01:09:11.259 | INFO | Serialization done, swapping tmp file now... | 20260508 01:09:11.260 | INFO | RTMetrics done | 20260508 01:09:11.337 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.554 seconds | 20260508 01:09:11.337 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.94% / 0.00% / 58.06% | 20260508 01:09:11.337 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.90187% / 3.97364e-06% / 6.35783e-05% | 20260508 01:09:11.337 | INFO | Baz IO overall: | 20260508 01:09:11.337 | INFO | BytesWritten : 1835474073979 | 20260508 01:09:11.337 | INFO | NumEvents : 888028386599 | 20260508 01:09:12.827 | INFO | Analyzing chunk frames = [4046848,4047360) | 20260508 01:09:12.828 | INFO | MemInfo: { | 20260508 01:09:12.828 | INFO | "gpu" : " 17 GB", | 20260508 01:09:12.828 | INFO | "huge" : "", | 20260508 01:09:12.828 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:12.828 | INFO | "malloc" : " 14 GB", | 20260508 01:09:12.828 | INFO | "pinned" : " 29 GB" | 20260508 01:09:12.828 | INFO | } | 20260508 01:09:12.828 | INFO | Perf report for the last 5.1427 seconds: | 20260508 01:09:12.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:12.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:12.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:12.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:12.828 | INFO | ANALYSIS : 68.5%, 3.0, 106.3ms, 0 | 20260508 01:09:12.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:12.828 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:09:12.828 | INFO | PRE_HQ : 66.0%, 1.6, 56.2ms, 0 | 20260508 01:09:12.828 | INFO | BAZWRITER : 68.7%, 3.7, 132.3ms, 0 | 20260508 01:09:12.828 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:09:12.828 | INFO | Uploaded 36GiB to GPU in 2.54308s (14.1561GiB/s) | 20260508 01:09:12.828 | INFO | Downloaded 38GiB to GPU in 2.69289s (14.1113GiB/s) | 20260508 01:09:17.965 | INFO | Analyzing chunk frames = [4047360,4047872) | 20260508 01:09:17.965 | INFO | MemInfo: { | 20260508 01:09:17.965 | INFO | "gpu" : " 17 GB", | 20260508 01:09:17.965 | INFO | "huge" : "", | 20260508 01:09:17.965 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:17.965 | INFO | "malloc" : " 14 GB", | 20260508 01:09:17.965 | INFO | "pinned" : " 29 GB" | 20260508 01:09:17.965 | INFO | } | 20260508 01:09:17.965 | INFO | Perf report for the last 5.13724 seconds: | 20260508 01:09:17.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:17.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 01:09:17.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:09:17.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:17.965 | INFO | Uploaded 36GiB to GPU in 2.60245s (13.8331GiB/s) | 20260508 01:09:17.965 | INFO | Downloaded 34GiB to GPU in 2.50395s (13.5786GiB/s) | 20260508 01:09:23.103 | INFO | Analyzing chunk frames = [4047872,4048384) | 20260508 01:09:23.103 | INFO | MemInfo: { | 20260508 01:09:23.103 | INFO | "gpu" : " 17 GB", | 20260508 01:09:23.104 | INFO | "huge" : "", | 20260508 01:09:23.104 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:23.104 | INFO | "malloc" : " 14 GB", | 20260508 01:09:23.104 | INFO | "pinned" : " 29 GB" | 20260508 01:09:23.104 | INFO | } | 20260508 01:09:23.104 | INFO | Perf report for the last 5.13891 seconds: | 20260508 01:09:23.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:23.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:09:23.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:09:23.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:23.104 | INFO | Uploaded 36GiB to GPU in 2.52629s (14.2501GiB/s) | 20260508 01:09:23.104 | INFO | Downloaded 34GiB to GPU in 2.48229s (13.697GiB/s) | 20260508 01:09:28.242 | INFO | Analyzing chunk frames = [4048384,4048896) | 20260508 01:09:28.242 | INFO | MemInfo: { | 20260508 01:09:28.242 | INFO | "gpu" : " 17 GB", | 20260508 01:09:28.242 | INFO | "huge" : "", | 20260508 01:09:28.242 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:28.242 | INFO | "malloc" : " 14 GB", | 20260508 01:09:28.242 | INFO | "pinned" : " 29 GB" | 20260508 01:09:28.242 | INFO | } | 20260508 01:09:28.242 | INFO | Perf report for the last 5.13865 seconds: | 20260508 01:09:28.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:28.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.242 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:09:28.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.243 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:09:28.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:28.243 | INFO | Uploaded 36GiB to GPU in 2.54053s (14.1703GiB/s) | 20260508 01:09:28.243 | INFO | Downloaded 34GiB to GPU in 2.4901s (13.654GiB/s) | 20260508 01:09:33.381 | INFO | Analyzing chunk frames = [4048896,4049408) | 20260508 01:09:33.381 | INFO | MemInfo: { | 20260508 01:09:33.381 | INFO | "gpu" : " 17 GB", | 20260508 01:09:33.381 | INFO | "huge" : "", | 20260508 01:09:33.381 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:33.381 | INFO | "malloc" : " 14 GB", | 20260508 01:09:33.381 | INFO | "pinned" : " 29 GB" | 20260508 01:09:33.382 | INFO | } | 20260508 01:09:33.382 | INFO | Perf report for the last 5.13912 seconds: | 20260508 01:09:33.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:33.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 01:09:33.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:09:33.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:33.382 | INFO | Uploaded 36GiB to GPU in 2.39305s (15.0436GiB/s) | 20260508 01:09:33.382 | INFO | Downloaded 34GiB to GPU in 2.52235s (13.4795GiB/s) | 20260508 01:09:38.526 | INFO | Analyzing chunk frames = [4049408,4049920) | 20260508 01:09:38.526 | INFO | MemInfo: { | 20260508 01:09:38.526 | INFO | "gpu" : " 17 GB", | 20260508 01:09:38.526 | INFO | "huge" : "", | 20260508 01:09:38.526 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:38.526 | INFO | "malloc" : " 14 GB", | 20260508 01:09:38.526 | INFO | "pinned" : " 29 GB" | 20260508 01:09:38.526 | INFO | } | 20260508 01:09:38.527 | INFO | Perf report for the last 5.14502 seconds: | 20260508 01:09:38.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:38.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 01:09:38.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 01:09:38.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:38.527 | INFO | Uploaded 36GiB to GPU in 2.40499s (14.9689GiB/s) | 20260508 01:09:38.527 | INFO | Downloaded 34GiB to GPU in 2.3978s (14.1796GiB/s) | 20260508 01:09:43.662 | INFO | Analyzing chunk frames = [4049920,4050432) | 20260508 01:09:43.662 | INFO | MemInfo: { | 20260508 01:09:43.662 | INFO | "gpu" : " 17 GB", | 20260508 01:09:43.662 | INFO | "huge" : "", | 20260508 01:09:43.662 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:43.662 | INFO | "malloc" : " 14 GB", | 20260508 01:09:43.662 | INFO | "pinned" : " 29 GB" | 20260508 01:09:43.662 | INFO | } | 20260508 01:09:43.662 | INFO | Perf report for the last 5.13536 seconds: | 20260508 01:09:43.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:43.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 01:09:43.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 01:09:43.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:43.662 | INFO | Uploaded 36GiB to GPU in 2.43386s (14.7913GiB/s) | 20260508 01:09:43.662 | INFO | Downloaded 34GiB to GPU in 2.4149s (14.0793GiB/s) | 20260508 01:09:48.799 | INFO | Analyzing chunk frames = [4050432,4050944) | 20260508 01:09:48.799 | INFO | MemInfo: { | 20260508 01:09:48.799 | INFO | "gpu" : " 17 GB", | 20260508 01:09:48.799 | INFO | "huge" : "", | 20260508 01:09:48.799 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:48.800 | INFO | "malloc" : " 14 GB", | 20260508 01:09:48.800 | INFO | "pinned" : " 29 GB" | 20260508 01:09:48.800 | INFO | } | 20260508 01:09:48.800 | INFO | Perf report for the last 5.13765 seconds: | 20260508 01:09:48.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:48.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:09:48.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:09:48.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:48.800 | INFO | Uploaded 36GiB to GPU in 2.51994s (14.286GiB/s) | 20260508 01:09:48.800 | INFO | Downloaded 34GiB to GPU in 2.44055s (13.9313GiB/s) | 20260508 01:09:48.898 | INFO | Starting a round of Baz IO | 20260508 01:09:52.353 | INFO | Populating RTMetrics report | 20260508 01:09:52.353 | INFO | RTMetrics report populated | 20260508 01:09:52.353 | INFO | Opening tmp RTMetrics json file | 20260508 01:09:52.374 | INFO | Serializing RTMetrics to json | 20260508 01:09:52.407 | INFO | Serialization done, swapping tmp file now... | 20260508 01:09:52.428 | INFO | RTMetrics done | 20260508 01:09:52.461 | INFO | Finished a round of Baz IO: Wrote 1.672 GiB in 3.563 seconds | 20260508 01:09:52.461 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.93% / 0.00% / 58.06% | 20260508 01:09:52.461 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.897948% / 1.19209e-05% / 7.54992e-05% | 20260508 01:09:52.462 | INFO | Baz IO overall: | 20260508 01:09:52.462 | INFO | BytesWritten : 1837268884688 | 20260508 01:09:52.462 | INFO | NumEvents : 888890543406 | 20260508 01:09:53.936 | INFO | Analyzing chunk frames = [4050944,4051456) | 20260508 01:09:53.936 | INFO | MemInfo: { | 20260508 01:09:53.936 | INFO | "gpu" : " 17 GB", | 20260508 01:09:53.936 | INFO | "huge" : "", | 20260508 01:09:53.936 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:53.936 | INFO | "malloc" : " 14 GB", | 20260508 01:09:53.936 | INFO | "pinned" : " 29 GB" | 20260508 01:09:53.936 | INFO | } | 20260508 01:09:53.936 | INFO | Perf report for the last 5.13663 seconds: | 20260508 01:09:53.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:53.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:53.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:53.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:53.936 | INFO | ANALYSIS : 68.2%, 3.0, 106.6ms, 0 | 20260508 01:09:53.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:53.936 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260508 01:09:53.936 | INFO | PRE_HQ : 65.5%, 1.7, 56.2ms, 0 | 20260508 01:09:53.936 | INFO | BAZWRITER : 69.0%, 3.5, 125.1ms, 0 | 20260508 01:09:53.936 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 01:09:53.937 | INFO | Uploaded 36GiB to GPU in 2.51244s (14.3287GiB/s) | 20260508 01:09:53.937 | INFO | Downloaded 38GiB to GPU in 2.69098s (14.1213GiB/s) | 20260508 01:09:59.080 | INFO | Analyzing chunk frames = [4051456,4051968) | 20260508 01:09:59.080 | INFO | MemInfo: { | 20260508 01:09:59.080 | INFO | "gpu" : " 17 GB", | 20260508 01:09:59.080 | INFO | "huge" : "", | 20260508 01:09:59.080 | INFO | "huge_shared" : " 29 GB", | 20260508 01:09:59.080 | INFO | "malloc" : " 14 GB", | 20260508 01:09:59.080 | INFO | "pinned" : " 29 GB" | 20260508 01:09:59.080 | INFO | } | 20260508 01:09:59.080 | INFO | Perf report for the last 5.1435 seconds: | 20260508 01:09:59.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:09:59.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:09:59.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:09:59.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:09:59.080 | INFO | Uploaded 36GiB to GPU in 2.53778s (14.1856GiB/s) | 20260508 01:09:59.080 | INFO | Downloaded 34GiB to GPU in 2.42563s (14.017GiB/s) | 20260508 01:10:04.216 | INFO | Analyzing chunk frames = [4051968,4052480) | 20260508 01:10:04.216 | INFO | MemInfo: { | 20260508 01:10:04.216 | INFO | "gpu" : " 17 GB", | 20260508 01:10:04.216 | INFO | "huge" : "", | 20260508 01:10:04.216 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:04.216 | INFO | "malloc" : " 14 GB", | 20260508 01:10:04.216 | INFO | "pinned" : " 29 GB" | 20260508 01:10:04.216 | INFO | } | 20260508 01:10:04.216 | INFO | Perf report for the last 5.13656 seconds: | 20260508 01:10:04.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:04.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.216 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:10:04.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.217 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:10:04.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:04.217 | INFO | Uploaded 36GiB to GPU in 2.54715s (14.1334GiB/s) | 20260508 01:10:04.217 | INFO | Downloaded 34GiB to GPU in 2.48951s (13.6573GiB/s) | 20260508 01:10:09.358 | INFO | Analyzing chunk frames = [4052480,4052992) | 20260508 01:10:09.358 | INFO | MemInfo: { | 20260508 01:10:09.358 | INFO | "gpu" : " 17 GB", | 20260508 01:10:09.358 | INFO | "huge" : "", | 20260508 01:10:09.358 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:09.358 | INFO | "malloc" : " 14 GB", | 20260508 01:10:09.358 | INFO | "pinned" : " 29 GB" | 20260508 01:10:09.358 | INFO | } | 20260508 01:10:09.358 | INFO | Perf report for the last 5.14182 seconds: | 20260508 01:10:09.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:09.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.358 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:10:09.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.358 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:10:09.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:09.359 | INFO | Uploaded 36GiB to GPU in 2.4861s (14.4805GiB/s) | 20260508 01:10:09.359 | INFO | Downloaded 34GiB to GPU in 2.47478s (13.7386GiB/s) | 20260508 01:10:14.496 | INFO | Analyzing chunk frames = [4052992,4053504) | 20260508 01:10:14.496 | INFO | MemInfo: { | 20260508 01:10:14.496 | INFO | "gpu" : " 17 GB", | 20260508 01:10:14.496 | INFO | "huge" : "", | 20260508 01:10:14.496 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:14.496 | INFO | "malloc" : " 14 GB", | 20260508 01:10:14.496 | INFO | "pinned" : " 29 GB" | 20260508 01:10:14.496 | INFO | } | 20260508 01:10:14.496 | INFO | Perf report for the last 5.13808 seconds: | 20260508 01:10:14.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:14.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.496 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 01:10:14.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.496 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:10:14.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:14.497 | INFO | Uploaded 36GiB to GPU in 2.3343s (15.4222GiB/s) | 20260508 01:10:14.497 | INFO | Downloaded 34GiB to GPU in 2.48659s (13.6734GiB/s) | 20260508 01:10:19.638 | INFO | Analyzing chunk frames = [4053504,4054016) | 20260508 01:10:19.638 | INFO | MemInfo: { | 20260508 01:10:19.638 | INFO | "gpu" : " 17 GB", | 20260508 01:10:19.638 | INFO | "huge" : "", | 20260508 01:10:19.638 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:19.638 | INFO | "malloc" : " 14 GB", | 20260508 01:10:19.638 | INFO | "pinned" : " 29 GB" | 20260508 01:10:19.638 | INFO | } | 20260508 01:10:19.638 | INFO | Perf report for the last 5.14163 seconds: | 20260508 01:10:19.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:19.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:10:19.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:10:19.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:19.638 | INFO | Uploaded 36GiB to GPU in 2.41395s (14.9133GiB/s) | 20260508 01:10:19.638 | INFO | Downloaded 34GiB to GPU in 2.39165s (14.2161GiB/s) | 20260508 01:10:24.780 | INFO | Analyzing chunk frames = [4054016,4054528) | 20260508 01:10:24.780 | INFO | MemInfo: { | 20260508 01:10:24.780 | INFO | "gpu" : " 17 GB", | 20260508 01:10:24.780 | INFO | "huge" : "", | 20260508 01:10:24.780 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:24.780 | INFO | "malloc" : " 14 GB", | 20260508 01:10:24.780 | INFO | "pinned" : " 29 GB" | 20260508 01:10:24.780 | INFO | } | 20260508 01:10:24.780 | INFO | Perf report for the last 5.14202 seconds: | 20260508 01:10:24.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:24.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:10:24.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:10:24.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:24.780 | INFO | Uploaded 36GiB to GPU in 2.44494s (14.7243GiB/s) | 20260508 01:10:24.780 | INFO | Downloaded 34GiB to GPU in 2.41174s (14.0977GiB/s) | 20260508 01:10:29.919 | INFO | Analyzing chunk frames = [4054528,4055040) | 20260508 01:10:29.919 | INFO | MemInfo: { | 20260508 01:10:29.919 | INFO | "gpu" : " 17 GB", | 20260508 01:10:29.919 | INFO | "huge" : "", | 20260508 01:10:29.919 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:29.919 | INFO | "malloc" : " 14 GB", | 20260508 01:10:29.919 | INFO | "pinned" : " 29 GB" | 20260508 01:10:29.919 | INFO | } | 20260508 01:10:29.919 | INFO | Perf report for the last 5.13942 seconds: | 20260508 01:10:29.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:29.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.919 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:10:29.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.920 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:10:29.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:29.920 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:10:29.920 | INFO | Total mem usage: 16.7 GiB | 20260508 01:10:29.920 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:10:29.920 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.6%) | 20260508 01:10:29.920 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:10:29.920 | INFO | High Water Breakdown: | 20260508 01:10:29.920 | INFO | 59.8% : 8.7 GiB ( 84.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:10:29.920 | INFO | 40.1% : 5.83 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:10:29.920 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:10:29.920 | INFO | ------------------------------------------ | 20260508 01:10:29.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:10:29.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:10:29.920 | INFO | Total mem usage: 26.5 GiB | 20260508 01:10:29.920 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:10:29.920 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:10:29.920 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:10:29.920 | INFO | High Water Breakdown: | 20260508 01:10:29.920 | INFO | Uploaded 36GiB to GPU in 2.55803s (14.0733GiB/s) | 20260508 01:10:29.920 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:10:29.920 | INFO | Downloaded 34GiB to GPU in 2.44958s (13.8799GiB/s) | 20260508 01:10:29.920 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:10:29.920 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:10:29.920 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:10:29.920 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:10:29.920 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:10:29.920 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:10:29.920 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:10:29.920 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:10:29.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:10:29.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:10:29.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:10:29.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:10:29.920 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:10:29.920 | INFO | ------------------------------------------ | 20260508 01:10:29.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:10:29.920 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:10:29.920 | INFO | Total mem usage: 27.8 GiB | 20260508 01:10:29.920 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:10:29.920 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:10:29.920 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:10:29.920 | INFO | High Water Breakdown: | 20260508 01:10:29.920 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:10:29.921 | INFO | ------------------------------------------ | 20260508 01:10:29.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:10:29.921 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:10:29.921 | INFO | Total mem usage: 15.7 GiB | 20260508 01:10:29.921 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:10:29.921 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:10:29.921 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:10:29.921 | INFO | High Water Breakdown: | 20260508 01:10:29.921 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:10:29.921 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:10:29.921 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:10:29.921 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:10:29.921 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:10:29.921 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:10:29.921 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:10:29.921 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:10:29.921 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:10:29.921 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:10:29.921 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:10:29.921 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:10:29.921 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:10:29.921 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:10:29.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:10:29.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:10:29.921 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:10:29.921 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:10:29.921 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:10:29.921 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:10:29.921 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:10:29.921 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:10:29.921 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:10:29.921 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:10:29.921 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:10:29.921 | INFO | ------------------------------------------ | 20260508 01:10:29.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:10:29.921 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:10:29.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:10:29.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:10:29.921 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.42e+01 / 1.42e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.15e-02 / 5.78e-03 | 20260508 01:10:29.921 | INFO | PerfReport: 12.21%, Baseline -- 4.00e+00 / 4.52e+00, 5.88e-01 / 1.47e-01 | 20260508 01:10:29.921 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.11e-01 | 20260508 01:10:29.921 | INFO | PerfReport: 16.09%, DME -- 4.22e+01 / 4.60e+01, 4.07e+00 / 1.70e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.63e+00 / 1.53e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.44e+00 / 2.74e+00, 4.26e-01 / 2.14e-01 | 20260508 01:10:29.921 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.28e+00, 2.71e-01 / 1.31e-01 | 20260508 01:10:29.921 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.71e-01, 5.20e-02 / 2.45e-02 | 20260508 01:10:29.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:10:29.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:10:29.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:10:29.921 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 30.77%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 2.16%, LockUpload -- 2.30e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 2.78%, LockDownload -- 2.96e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:10:29.921 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260508 01:10:30.017 | INFO | Starting a round of Baz IO | 20260508 01:10:33.463 | INFO | Populating RTMetrics report | 20260508 01:10:33.464 | INFO | RTMetrics report populated | 20260508 01:10:33.464 | INFO | Opening tmp RTMetrics json file | 20260508 01:10:33.484 | INFO | Serializing RTMetrics to json | 20260508 01:10:33.485 | INFO | Serialization done, swapping tmp file now... | 20260508 01:10:33.486 | INFO | RTMetrics done | 20260508 01:10:33.558 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.54 seconds | 20260508 01:10:33.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.93% / 0.00% / 58.07% | 20260508 01:10:33.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.895985% / 7.94729e-06% / 7.94729e-05% | 20260508 01:10:33.558 | INFO | Baz IO overall: | 20260508 01:10:33.558 | INFO | BytesWritten : 1839062956989 | 20260508 01:10:33.558 | INFO | NumEvents : 889752230637 | 20260508 01:10:35.054 | INFO | Analyzing chunk frames = [4055040,4055552) | 20260508 01:10:35.054 | INFO | MemInfo: { | 20260508 01:10:35.054 | INFO | "gpu" : " 17 GB", | 20260508 01:10:35.054 | INFO | "huge" : "", | 20260508 01:10:35.054 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:35.054 | INFO | "malloc" : " 14 GB", | 20260508 01:10:35.054 | INFO | "pinned" : " 29 GB" | 20260508 01:10:35.054 | INFO | } | 20260508 01:10:35.054 | INFO | Perf report for the last 5.1346 seconds: | 20260508 01:10:35.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:35.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:35.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:35.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:35.054 | INFO | ANALYSIS : 68.3%, 3.0, 106.6ms, 0 | 20260508 01:10:35.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:35.054 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:10:35.054 | INFO | PRE_HQ : 65.7%, 1.6, 56.0ms, 0 | 20260508 01:10:35.054 | INFO | BAZWRITER : 68.6%, 3.9, 138.4ms, 0 | 20260508 01:10:35.054 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:10:35.054 | INFO | Uploaded 36GiB to GPU in 2.56519s (14.0341GiB/s) | 20260508 01:10:35.055 | INFO | Downloaded 38GiB to GPU in 2.69563s (14.0969GiB/s) | 20260508 01:10:40.188 | INFO | Analyzing chunk frames = [4055552,4056064) | 20260508 01:10:40.188 | INFO | MemInfo: { | 20260508 01:10:40.188 | INFO | "gpu" : " 17 GB", | 20260508 01:10:40.188 | INFO | "huge" : "", | 20260508 01:10:40.188 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:40.188 | INFO | "malloc" : " 14 GB", | 20260508 01:10:40.188 | INFO | "pinned" : " 29 GB" | 20260508 01:10:40.188 | INFO | } | 20260508 01:10:40.189 | INFO | Perf report for the last 5.13454 seconds: | 20260508 01:10:40.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:40.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 01:10:40.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:10:40.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:40.189 | INFO | Uploaded 36GiB to GPU in 2.56029s (14.0609GiB/s) | 20260508 01:10:40.189 | INFO | Downloaded 34GiB to GPU in 2.42119s (14.0427GiB/s) | 20260508 01:10:45.332 | INFO | Analyzing chunk frames = [4056064,4056576) | 20260508 01:10:45.332 | INFO | MemInfo: { | 20260508 01:10:45.332 | INFO | "gpu" : " 17 GB", | 20260508 01:10:45.332 | INFO | "huge" : "", | 20260508 01:10:45.332 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:45.332 | INFO | "malloc" : " 14 GB", | 20260508 01:10:45.332 | INFO | "pinned" : " 29 GB" | 20260508 01:10:45.332 | INFO | } | 20260508 01:10:45.333 | INFO | Perf report for the last 5.14398 seconds: | 20260508 01:10:45.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:45.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:10:45.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:10:45.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:45.333 | INFO | Uploaded 36GiB to GPU in 2.53478s (14.2024GiB/s) | 20260508 01:10:45.333 | INFO | Downloaded 34GiB to GPU in 2.47058s (13.7619GiB/s) | 20260508 01:10:50.477 | INFO | Analyzing chunk frames = [4056576,4057088) | 20260508 01:10:50.477 | INFO | MemInfo: { | 20260508 01:10:50.477 | INFO | "gpu" : " 17 GB", | 20260508 01:10:50.477 | INFO | "huge" : "", | 20260508 01:10:50.477 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:50.477 | INFO | "malloc" : " 14 GB", | 20260508 01:10:50.477 | INFO | "pinned" : " 29 GB" | 20260508 01:10:50.477 | INFO | } | 20260508 01:10:50.477 | INFO | Perf report for the last 5.14409 seconds: | 20260508 01:10:50.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:50.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 01:10:50.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260508 01:10:50.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:50.477 | INFO | Uploaded 36GiB to GPU in 2.54877s (14.1245GiB/s) | 20260508 01:10:50.477 | INFO | Downloaded 34GiB to GPU in 2.47153s (13.7566GiB/s) | 20260508 01:10:55.615 | INFO | Analyzing chunk frames = [4057088,4057600) | 20260508 01:10:55.615 | INFO | MemInfo: { | 20260508 01:10:55.615 | INFO | "gpu" : " 17 GB", | 20260508 01:10:55.616 | INFO | "huge" : "", | 20260508 01:10:55.616 | INFO | "huge_shared" : " 29 GB", | 20260508 01:10:55.616 | INFO | "malloc" : " 14 GB", | 20260508 01:10:55.616 | INFO | "pinned" : " 29 GB" | 20260508 01:10:55.616 | INFO | } | 20260508 01:10:55.616 | INFO | Perf report for the last 5.13908 seconds: | 20260508 01:10:55.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:10:55.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 01:10:55.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:10:55.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:10:55.616 | INFO | Uploaded 36GiB to GPU in 2.39225s (15.0486GiB/s) | 20260508 01:10:55.616 | INFO | Downloaded 34GiB to GPU in 2.49301s (13.6382GiB/s) | 20260508 01:11:00.749 | INFO | Analyzing chunk frames = [4057600,4058112) | 20260508 01:11:00.749 | INFO | MemInfo: { | 20260508 01:11:00.749 | INFO | "gpu" : " 17 GB", | 20260508 01:11:00.749 | INFO | "huge" : "", | 20260508 01:11:00.749 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:00.749 | INFO | "malloc" : " 14 GB", | 20260508 01:11:00.749 | INFO | "pinned" : " 29 GB" | 20260508 01:11:00.749 | INFO | } | 20260508 01:11:00.749 | INFO | Perf report for the last 5.13338 seconds: | 20260508 01:11:00.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:00.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 01:11:00.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:11:00.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:00.749 | INFO | Uploaded 36GiB to GPU in 2.427s (14.8331GiB/s) | 20260508 01:11:00.750 | INFO | Downloaded 34GiB to GPU in 2.41642s (14.0704GiB/s) | 20260508 01:11:05.889 | INFO | Analyzing chunk frames = [4058112,4058624) | 20260508 01:11:05.889 | INFO | MemInfo: { | 20260508 01:11:05.889 | INFO | "gpu" : " 17 GB", | 20260508 01:11:05.889 | INFO | "huge" : "", | 20260508 01:11:05.889 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:05.889 | INFO | "malloc" : " 14 GB", | 20260508 01:11:05.889 | INFO | "pinned" : " 29 GB" | 20260508 01:11:05.889 | INFO | } | 20260508 01:11:05.889 | INFO | Perf report for the last 5.13991 seconds: | 20260508 01:11:05.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:05.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.889 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 01:11:05.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.889 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:11:05.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:05.890 | INFO | Uploaded 36GiB to GPU in 2.53973s (14.1748GiB/s) | 20260508 01:11:05.890 | INFO | Downloaded 34GiB to GPU in 2.44949s (13.8804GiB/s) | 20260508 01:11:11.032 | INFO | Analyzing chunk frames = [4058624,4059136) | 20260508 01:11:11.032 | INFO | MemInfo: { | 20260508 01:11:11.032 | INFO | "gpu" : " 17 GB", | 20260508 01:11:11.032 | INFO | "huge" : "", | 20260508 01:11:11.032 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:11.032 | INFO | "malloc" : " 14 GB", | 20260508 01:11:11.032 | INFO | "pinned" : " 29 GB" | 20260508 01:11:11.032 | INFO | } | 20260508 01:11:11.032 | INFO | Perf report for the last 5.14315 seconds: | 20260508 01:11:11.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:11.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:11:11.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:11:11.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:11.032 | INFO | Uploaded 36GiB to GPU in 2.55053s (14.1147GiB/s) | 20260508 01:11:11.033 | INFO | Downloaded 34GiB to GPU in 2.44785s (13.8897GiB/s) | 20260508 01:11:11.129 | INFO | Starting a round of Baz IO | 20260508 01:11:14.589 | INFO | Populating RTMetrics report | 20260508 01:11:14.589 | INFO | RTMetrics report populated | 20260508 01:11:14.589 | INFO | Opening tmp RTMetrics json file | 20260508 01:11:14.594 | INFO | Serializing RTMetrics to json | 20260508 01:11:14.614 | INFO | Serialization done, swapping tmp file now... | 20260508 01:11:14.615 | INFO | RTMetrics done | 20260508 01:11:14.706 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.576 seconds | 20260508 01:11:14.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.92% / 0.00% / 58.07% | 20260508 01:11:14.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.900618% / 0% / 4.76837e-05% | 20260508 01:11:14.706 | INFO | Baz IO overall: | 20260508 01:11:14.706 | INFO | BytesWritten : 1840859829502 | 20260508 01:11:14.706 | INFO | NumEvents : 890616254904 | 20260508 01:11:16.171 | INFO | Analyzing chunk frames = [4059136,4059648) | 20260508 01:11:16.171 | INFO | MemInfo: { | 20260508 01:11:16.171 | INFO | "gpu" : " 17 GB", | 20260508 01:11:16.171 | INFO | "huge" : "", | 20260508 01:11:16.171 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:16.171 | INFO | "malloc" : " 14 GB", | 20260508 01:11:16.171 | INFO | "pinned" : " 29 GB" | 20260508 01:11:16.171 | INFO | } | 20260508 01:11:16.171 | INFO | Perf report for the last 5.1389 seconds: | 20260508 01:11:16.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:16.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:16.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:16.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:16.171 | INFO | ANALYSIS : 68.5%, 3.0, 106.5ms, 0 | 20260508 01:11:16.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:16.171 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 01:11:16.171 | INFO | PRE_HQ : 65.7%, 1.7, 56.5ms, 0 | 20260508 01:11:16.171 | INFO | BAZWRITER : 69.3%, 3.7, 130.1ms, 0 | 20260508 01:11:16.171 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:11:16.171 | INFO | Uploaded 36GiB to GPU in 2.60529s (13.818GiB/s) | 20260508 01:11:16.171 | INFO | Downloaded 38GiB to GPU in 2.6975s (14.0871GiB/s) | 20260508 01:11:21.305 | INFO | Analyzing chunk frames = [4059648,4060160) | 20260508 01:11:21.305 | INFO | MemInfo: { | 20260508 01:11:21.305 | INFO | "gpu" : " 17 GB", | 20260508 01:11:21.305 | INFO | "huge" : "", | 20260508 01:11:21.305 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:21.305 | INFO | "malloc" : " 14 GB", | 20260508 01:11:21.305 | INFO | "pinned" : " 29 GB" | 20260508 01:11:21.305 | INFO | } | 20260508 01:11:21.305 | INFO | Perf report for the last 5.13384 seconds: | 20260508 01:11:21.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:21.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.305 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 01:11:21.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.305 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:11:21.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:21.306 | INFO | Uploaded 36GiB to GPU in 2.53086s (14.2244GiB/s) | 20260508 01:11:21.306 | INFO | Downloaded 34GiB to GPU in 2.43113s (13.9853GiB/s) | 20260508 01:11:26.447 | INFO | Analyzing chunk frames = [4060160,4060672) | 20260508 01:11:26.447 | INFO | MemInfo: { | 20260508 01:11:26.447 | INFO | "gpu" : " 17 GB", | 20260508 01:11:26.447 | INFO | "huge" : "", | 20260508 01:11:26.447 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:26.447 | INFO | "malloc" : " 14 GB", | 20260508 01:11:26.447 | INFO | "pinned" : " 29 GB" | 20260508 01:11:26.447 | INFO | } | 20260508 01:11:26.447 | INFO | Perf report for the last 5.14225 seconds: | 20260508 01:11:26.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:26.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.447 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:11:26.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.447 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 01:11:26.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:26.448 | INFO | Uploaded 36GiB to GPU in 2.49944s (14.4032GiB/s) | 20260508 01:11:26.448 | INFO | Downloaded 34GiB to GPU in 2.48123s (13.7029GiB/s) | 20260508 01:11:31.588 | INFO | Analyzing chunk frames = [4060672,4061184) | 20260508 01:11:31.588 | INFO | MemInfo: { | 20260508 01:11:31.588 | INFO | "gpu" : " 17 GB", | 20260508 01:11:31.588 | INFO | "huge" : "", | 20260508 01:11:31.588 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:31.588 | INFO | "malloc" : " 14 GB", | 20260508 01:11:31.588 | INFO | "pinned" : " 29 GB" | 20260508 01:11:31.588 | INFO | } | 20260508 01:11:31.588 | INFO | Perf report for the last 5.1408 seconds: | 20260508 01:11:31.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:31.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.588 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:11:31.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.588 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:11:31.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:31.589 | INFO | Uploaded 36GiB to GPU in 2.52205s (14.2741GiB/s) | 20260508 01:11:31.589 | INFO | Downloaded 34GiB to GPU in 2.4813s (13.7025GiB/s) | 20260508 01:11:36.728 | INFO | Analyzing chunk frames = [4061184,4061696) | 20260508 01:11:36.728 | INFO | MemInfo: { | 20260508 01:11:36.728 | INFO | "gpu" : " 17 GB", | 20260508 01:11:36.729 | INFO | "huge" : "", | 20260508 01:11:36.729 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:36.729 | INFO | "malloc" : " 14 GB", | 20260508 01:11:36.729 | INFO | "pinned" : " 29 GB" | 20260508 01:11:36.729 | INFO | } | 20260508 01:11:36.729 | INFO | Perf report for the last 5.14064 seconds: | 20260508 01:11:36.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:36.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 01:11:36.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:11:36.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:36.729 | INFO | Uploaded 36GiB to GPU in 2.31028s (15.5825GiB/s) | 20260508 01:11:36.729 | INFO | Downloaded 34GiB to GPU in 2.49575s (13.6232GiB/s) | 20260508 01:11:41.867 | INFO | Analyzing chunk frames = [4061696,4062208) | 20260508 01:11:41.868 | INFO | MemInfo: { | 20260508 01:11:41.868 | INFO | "gpu" : " 17 GB", | 20260508 01:11:41.868 | INFO | "huge" : "", | 20260508 01:11:41.868 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:41.868 | INFO | "malloc" : " 14 GB", | 20260508 01:11:41.868 | INFO | "pinned" : " 29 GB" | 20260508 01:11:41.868 | INFO | } | 20260508 01:11:41.868 | INFO | Perf report for the last 5.13898 seconds: | 20260508 01:11:41.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:41.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:11:41.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:11:41.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:41.868 | INFO | Uploaded 36GiB to GPU in 2.36436s (15.2261GiB/s) | 20260508 01:11:41.868 | INFO | Downloaded 34GiB to GPU in 2.43312s (13.9738GiB/s) | 20260508 01:11:47.003 | INFO | Analyzing chunk frames = [4062208,4062720) | 20260508 01:11:47.003 | INFO | MemInfo: { | 20260508 01:11:47.003 | INFO | "gpu" : " 17 GB", | 20260508 01:11:47.003 | INFO | "huge" : "", | 20260508 01:11:47.003 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:47.003 | INFO | "malloc" : " 14 GB", | 20260508 01:11:47.003 | INFO | "pinned" : " 29 GB" | 20260508 01:11:47.003 | INFO | } | 20260508 01:11:47.003 | INFO | Perf report for the last 5.13569 seconds: | 20260508 01:11:47.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:47.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.004 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:11:47.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.004 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 01:11:47.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:47.004 | INFO | Uploaded 36GiB to GPU in 2.39347s (15.0409GiB/s) | 20260508 01:11:47.004 | INFO | Downloaded 34GiB to GPU in 2.42322s (14.0309GiB/s) | 20260508 01:11:52.147 | INFO | Analyzing chunk frames = [4062720,4063232) | 20260508 01:11:52.148 | INFO | MemInfo: { | 20260508 01:11:52.148 | INFO | "gpu" : " 17 GB", | 20260508 01:11:52.148 | INFO | "huge" : "", | 20260508 01:11:52.148 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:52.148 | INFO | "malloc" : " 14 GB", | 20260508 01:11:52.148 | INFO | "pinned" : " 29 GB" | 20260508 01:11:52.148 | INFO | } | 20260508 01:11:52.148 | INFO | Perf report for the last 5.14434 seconds: | 20260508 01:11:52.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:52.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:11:52.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:11:52.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:52.148 | INFO | Uploaded 36GiB to GPU in 2.51728s (14.3012GiB/s) | 20260508 01:11:52.148 | INFO | Downloaded 34GiB to GPU in 2.46373s (13.8002GiB/s) | 20260508 01:11:52.148 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:11:52.148 | INFO | Total mem usage: 16.7 GiB | 20260508 01:11:52.148 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:11:52.148 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 01:11:52.148 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:11:52.148 | INFO | High Water Breakdown: | 20260508 01:11:52.148 | INFO | 60.3% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:11:52.148 | INFO | 39.6% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:11:52.148 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:11:52.148 | INFO | ------------------------------------------ | 20260508 01:11:52.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:11:52.148 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:11:52.148 | INFO | Total mem usage: 26.5 GiB | 20260508 01:11:52.148 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:11:52.148 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:11:52.148 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:11:52.148 | INFO | High Water Breakdown: | 20260508 01:11:52.148 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:11:52.148 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:11:52.148 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:11:52.148 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:11:52.148 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:11:52.148 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:11:52.148 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:11:52.148 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:11:52.148 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:11:52.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:11:52.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:11:52.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:11:52.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:11:52.148 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:11:52.148 | INFO | ------------------------------------------ | 20260508 01:11:52.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:11:52.148 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:11:52.148 | INFO | Total mem usage: 27.8 GiB | 20260508 01:11:52.148 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:11:52.148 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:11:52.148 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:11:52.148 | INFO | High Water Breakdown: | 20260508 01:11:52.148 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:11:52.148 | INFO | ------------------------------------------ | 20260508 01:11:52.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:11:52.148 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:11:52.148 | INFO | Total mem usage: 15.7 GiB | 20260508 01:11:52.148 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:11:52.148 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:11:52.148 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:11:52.148 | INFO | High Water Breakdown: | 20260508 01:11:52.148 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:11:52.148 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:11:52.148 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:11:52.148 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:11:52.148 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:11:52.148 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:11:52.148 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:11:52.148 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:11:52.148 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:11:52.148 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:11:52.148 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:11:52.148 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:11:52.148 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:11:52.148 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:11:52.148 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:11:52.148 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:11:52.148 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:11:52.149 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:11:52.149 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:11:52.149 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:11:52.149 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:11:52.149 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:11:52.149 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:11:52.149 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:11:52.149 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:11:52.149 | INFO | ------------------------------------------ | 20260508 01:11:52.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:11:52.149 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:11:52.149 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:11:52.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:11:52.149 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.17e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.22e-02 / 4.02e-03 | 20260508 01:11:52.149 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.48e+00, 5.83e-01 / 1.62e-01 | 20260508 01:11:52.149 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.34e-01 / 1.12e-01 | 20260508 01:11:52.149 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.61e+01, 3.97e+00 / 1.61e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.60e+00 / 1.59e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.70e+00, 4.26e-01 / 2.32e-01 | 20260508 01:11:52.149 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.29e+00, 2.68e-01 / 1.63e-01 | 20260508 01:11:52.149 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.78e-01, 5.30e-02 / 2.97e-02 | 20260508 01:11:52.149 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:11:52.149 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:11:52.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:11:52.149 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 24.35%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 30.70%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 2.27%, LockUpload -- 2.42e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 2.69%, LockDownload -- 2.86e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 15.69%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:11:52.149 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260508 01:11:52.248 | INFO | Starting a round of Baz IO | 20260508 01:11:55.687 | INFO | Populating RTMetrics report | 20260508 01:11:55.687 | INFO | RTMetrics report populated | 20260508 01:11:55.687 | INFO | Opening tmp RTMetrics json file | 20260508 01:11:55.761 | INFO | Serializing RTMetrics to json | 20260508 01:11:55.775 | INFO | Serialization done, swapping tmp file now... | 20260508 01:11:55.776 | INFO | RTMetrics done | 20260508 01:11:55.799 | INFO | Finished a round of Baz IO: Wrote 1.672 GiB in 3.55 seconds | 20260508 01:11:55.799 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.92% / 0.00% / 58.08% | 20260508 01:11:55.799 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.897054% / 2.78155e-05% / 7.15256e-05% | 20260508 01:11:55.799 | INFO | Baz IO overall: | 20260508 01:11:55.799 | INFO | BytesWritten : 1842655527874 | 20260508 01:11:55.799 | INFO | NumEvents : 891479243798 | 20260508 01:11:57.283 | INFO | Analyzing chunk frames = [4063232,4063744) | 20260508 01:11:57.283 | INFO | MemInfo: { | 20260508 01:11:57.283 | INFO | "gpu" : " 17 GB", | 20260508 01:11:57.283 | INFO | "huge" : "", | 20260508 01:11:57.283 | INFO | "huge_shared" : " 29 GB", | 20260508 01:11:57.283 | INFO | "malloc" : " 14 GB", | 20260508 01:11:57.283 | INFO | "pinned" : " 29 GB" | 20260508 01:11:57.283 | INFO | } | 20260508 01:11:57.283 | INFO | Perf report for the last 5.13515 seconds: | 20260508 01:11:57.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:11:57.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:57.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:57.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:57.283 | INFO | ANALYSIS : 68.2%, 3.0, 106.5ms, 0 | 20260508 01:11:57.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:11:57.283 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 01:11:57.283 | INFO | PRE_HQ : 65.5%, 1.7, 56.5ms, 0 | 20260508 01:11:57.283 | INFO | BAZWRITER : 68.8%, 3.6, 125.6ms, 0 | 20260508 01:11:57.283 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 01:11:57.283 | INFO | Uploaded 36GiB to GPU in 2.52514s (14.2566GiB/s) | 20260508 01:11:57.283 | INFO | Downloaded 38GiB to GPU in 2.67939s (14.1824GiB/s) | 20260508 01:12:02.427 | INFO | Analyzing chunk frames = [4063744,4064256) | 20260508 01:12:02.427 | INFO | MemInfo: { | 20260508 01:12:02.427 | INFO | "gpu" : " 17 GB", | 20260508 01:12:02.427 | INFO | "huge" : "", | 20260508 01:12:02.427 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:02.428 | INFO | "malloc" : " 14 GB", | 20260508 01:12:02.428 | INFO | "pinned" : " 29 GB" | 20260508 01:12:02.428 | INFO | } | 20260508 01:12:02.428 | INFO | Perf report for the last 5.14475 seconds: | 20260508 01:12:02.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:02.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:12:02.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 01:12:02.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:02.428 | INFO | Uploaded 36GiB to GPU in 2.50529s (14.3696GiB/s) | 20260508 01:12:02.428 | INFO | Downloaded 34GiB to GPU in 2.44441s (13.9093GiB/s) | 20260508 01:12:07.564 | INFO | Analyzing chunk frames = [4064256,4064768) | 20260508 01:12:07.564 | INFO | MemInfo: { | 20260508 01:12:07.564 | INFO | "gpu" : " 17 GB", | 20260508 01:12:07.564 | INFO | "huge" : "", | 20260508 01:12:07.564 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:07.564 | INFO | "malloc" : " 14 GB", | 20260508 01:12:07.564 | INFO | "pinned" : " 29 GB" | 20260508 01:12:07.564 | INFO | } | 20260508 01:12:07.564 | INFO | Perf report for the last 5.1366 seconds: | 20260508 01:12:07.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:07.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.564 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:12:07.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.564 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:12:07.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:07.565 | INFO | Uploaded 36GiB to GPU in 2.45864s (14.6422GiB/s) | 20260508 01:12:07.565 | INFO | Downloaded 34GiB to GPU in 2.50564s (13.5694GiB/s) | 20260508 01:12:12.702 | INFO | Analyzing chunk frames = [4064768,4065280) | 20260508 01:12:12.702 | INFO | MemInfo: { | 20260508 01:12:12.702 | INFO | "gpu" : " 17 GB", | 20260508 01:12:12.702 | INFO | "huge" : "", | 20260508 01:12:12.702 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:12.702 | INFO | "malloc" : " 14 GB", | 20260508 01:12:12.702 | INFO | "pinned" : " 29 GB" | 20260508 01:12:12.702 | INFO | } | 20260508 01:12:12.702 | INFO | Perf report for the last 5.1379 seconds: | 20260508 01:12:12.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:12.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 01:12:12.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:12:12.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:12.702 | INFO | Uploaded 36GiB to GPU in 2.46781s (14.5879GiB/s) | 20260508 01:12:12.703 | INFO | Downloaded 34GiB to GPU in 2.4972s (13.6152GiB/s) | 20260508 01:12:17.845 | INFO | Analyzing chunk frames = [4065280,4065792) | 20260508 01:12:17.845 | INFO | MemInfo: { | 20260508 01:12:17.845 | INFO | "gpu" : " 17 GB", | 20260508 01:12:17.845 | INFO | "huge" : "", | 20260508 01:12:17.845 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:17.845 | INFO | "malloc" : " 14 GB", | 20260508 01:12:17.845 | INFO | "pinned" : " 29 GB" | 20260508 01:12:17.845 | INFO | } | 20260508 01:12:17.845 | INFO | Perf report for the last 5.14304 seconds: | 20260508 01:12:17.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:17.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.845 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:12:17.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.845 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 01:12:17.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:17.846 | INFO | Uploaded 36GiB to GPU in 2.26114s (15.9212GiB/s) | 20260508 01:12:17.846 | INFO | Downloaded 34GiB to GPU in 2.50567s (13.5692GiB/s) | 20260508 01:12:22.983 | INFO | Analyzing chunk frames = [4065792,4066304) | 20260508 01:12:22.983 | INFO | MemInfo: { | 20260508 01:12:22.983 | INFO | "gpu" : " 17 GB", | 20260508 01:12:22.983 | INFO | "huge" : "", | 20260508 01:12:22.983 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:22.983 | INFO | "malloc" : " 14 GB", | 20260508 01:12:22.983 | INFO | "pinned" : " 29 GB" | 20260508 01:12:22.983 | INFO | } | 20260508 01:12:22.983 | INFO | Perf report for the last 5.13791 seconds: | 20260508 01:12:22.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:22.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 01:12:22.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:12:22.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:22.983 | INFO | Uploaded 36GiB to GPU in 2.34489s (15.3526GiB/s) | 20260508 01:12:22.984 | INFO | Downloaded 34GiB to GPU in 2.43138s (13.9838GiB/s) | 20260508 01:12:28.117 | INFO | Analyzing chunk frames = [4066304,4066816) | 20260508 01:12:28.117 | INFO | MemInfo: { | 20260508 01:12:28.117 | INFO | "gpu" : " 17 GB", | 20260508 01:12:28.117 | INFO | "huge" : "", | 20260508 01:12:28.117 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:28.117 | INFO | "malloc" : " 14 GB", | 20260508 01:12:28.117 | INFO | "pinned" : " 29 GB" | 20260508 01:12:28.117 | INFO | } | 20260508 01:12:28.117 | INFO | Perf report for the last 5.13408 seconds: | 20260508 01:12:28.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:28.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.117 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:12:28.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.117 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:12:28.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:28.118 | INFO | Uploaded 36GiB to GPU in 2.49928s (14.4042GiB/s) | 20260508 01:12:28.118 | INFO | Downloaded 34GiB to GPU in 2.47143s (13.7572GiB/s) | 20260508 01:12:33.253 | INFO | Analyzing chunk frames = [4066816,4067328) | 20260508 01:12:33.253 | INFO | MemInfo: { | 20260508 01:12:33.253 | INFO | "gpu" : " 17 GB", | 20260508 01:12:33.253 | INFO | "huge" : "", | 20260508 01:12:33.253 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:33.253 | INFO | "malloc" : " 14 GB", | 20260508 01:12:33.253 | INFO | "pinned" : " 29 GB" | 20260508 01:12:33.253 | INFO | } | 20260508 01:12:33.253 | INFO | Perf report for the last 5.13578 seconds: | 20260508 01:12:33.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:33.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.253 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:12:33.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.253 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 01:12:33.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:33.254 | INFO | Uploaded 36GiB to GPU in 2.50368s (14.3789GiB/s) | 20260508 01:12:33.254 | INFO | Downloaded 34GiB to GPU in 2.46742s (13.7796GiB/s) | 20260508 01:12:33.370 | INFO | Starting a round of Baz IO | 20260508 01:12:36.850 | INFO | Populating RTMetrics report | 20260508 01:12:36.850 | INFO | RTMetrics report populated | 20260508 01:12:36.850 | INFO | Opening tmp RTMetrics json file | 20260508 01:12:36.851 | INFO | Serializing RTMetrics to json | 20260508 01:12:36.852 | INFO | Serialization done, swapping tmp file now... | 20260508 01:12:36.853 | INFO | RTMetrics done | 20260508 01:12:36.955 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.584 seconds | 20260508 01:12:36.955 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.92% / 0.00% / 58.08% | 20260508 01:12:36.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.892794% / 7.94729e-06% / 5.5631e-05% | 20260508 01:12:36.955 | INFO | Baz IO overall: | 20260508 01:12:36.955 | INFO | BytesWritten : 1844448861375 | 20260508 01:12:36.955 | INFO | NumEvents : 892340530822 | 20260508 01:12:38.397 | INFO | Analyzing chunk frames = [4067328,4067840) | 20260508 01:12:38.397 | INFO | MemInfo: { | 20260508 01:12:38.397 | INFO | "gpu" : " 17 GB", | 20260508 01:12:38.397 | INFO | "huge" : "", | 20260508 01:12:38.397 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:38.397 | INFO | "malloc" : " 14 GB", | 20260508 01:12:38.397 | INFO | "pinned" : " 29 GB" | 20260508 01:12:38.397 | INFO | } | 20260508 01:12:38.398 | INFO | Perf report for the last 5.14441 seconds: | 20260508 01:12:38.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:38.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:38.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:38.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:38.398 | INFO | ANALYSIS : 69.4%, 2.9, 104.9ms, 0 | 20260508 01:12:38.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:38.398 | INFO | RT_METRICS : 45.4%, 1.0, 23.9ms, 0 | 20260508 01:12:38.398 | INFO | PRE_HQ : 68.3%, 1.8, 65.3ms, 0 | 20260508 01:12:38.398 | INFO | BAZWRITER : 69.2%, 4.5, 160.3ms, 0 | 20260508 01:12:38.398 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 01:12:38.398 | INFO | Uploaded 36GiB to GPU in 2.51598s (14.3085GiB/s) | 20260508 01:12:38.398 | INFO | Downloaded 38GiB to GPU in 2.68732s (14.1405GiB/s) | 20260508 01:12:43.544 | INFO | Analyzing chunk frames = [4067840,4068352) | 20260508 01:12:43.544 | INFO | MemInfo: { | 20260508 01:12:43.544 | INFO | "gpu" : " 17 GB", | 20260508 01:12:43.545 | INFO | "huge" : "", | 20260508 01:12:43.545 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:43.545 | INFO | "malloc" : " 14 GB", | 20260508 01:12:43.545 | INFO | "pinned" : " 29 GB" | 20260508 01:12:43.545 | INFO | } | 20260508 01:12:43.545 | INFO | Perf report for the last 5.14712 seconds: | 20260508 01:12:43.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:43.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 01:12:43.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:12:43.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:43.545 | INFO | Uploaded 36GiB to GPU in 2.57127s (14.0009GiB/s) | 20260508 01:12:43.545 | INFO | Downloaded 34GiB to GPU in 2.48304s (13.6929GiB/s) | 20260508 01:12:48.684 | INFO | Analyzing chunk frames = [4068352,4068864) | 20260508 01:12:48.684 | INFO | MemInfo: { | 20260508 01:12:48.684 | INFO | "gpu" : " 17 GB", | 20260508 01:12:48.684 | INFO | "huge" : "", | 20260508 01:12:48.684 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:48.684 | INFO | "malloc" : " 14 GB", | 20260508 01:12:48.684 | INFO | "pinned" : " 29 GB" | 20260508 01:12:48.684 | INFO | } | 20260508 01:12:48.684 | INFO | Perf report for the last 5.1395 seconds: | 20260508 01:12:48.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:48.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 01:12:48.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:12:48.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:48.684 | INFO | Uploaded 36GiB to GPU in 2.54414s (14.1502GiB/s) | 20260508 01:12:48.684 | INFO | Downloaded 34GiB to GPU in 2.49016s (13.6538GiB/s) | 20260508 01:12:53.819 | INFO | Analyzing chunk frames = [4068864,4069376) | 20260508 01:12:53.819 | INFO | MemInfo: { | 20260508 01:12:53.819 | INFO | "gpu" : " 17 GB", | 20260508 01:12:53.819 | INFO | "huge" : "", | 20260508 01:12:53.819 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:53.819 | INFO | "malloc" : " 14 GB", | 20260508 01:12:53.819 | INFO | "pinned" : " 29 GB" | 20260508 01:12:53.819 | INFO | } | 20260508 01:12:53.819 | INFO | Perf report for the last 5.13521 seconds: | 20260508 01:12:53.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:53.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.819 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:12:53.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.820 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:12:53.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:53.820 | INFO | Uploaded 36GiB to GPU in 2.51423s (14.3185GiB/s) | 20260508 01:12:53.820 | INFO | Downloaded 34GiB to GPU in 2.48203s (13.6985GiB/s) | 20260508 01:12:58.955 | INFO | Analyzing chunk frames = [4069376,4069888) | 20260508 01:12:58.955 | INFO | MemInfo: { | 20260508 01:12:58.955 | INFO | "gpu" : " 17 GB", | 20260508 01:12:58.955 | INFO | "huge" : "", | 20260508 01:12:58.955 | INFO | "huge_shared" : " 29 GB", | 20260508 01:12:58.955 | INFO | "malloc" : " 14 GB", | 20260508 01:12:58.955 | INFO | "pinned" : " 29 GB" | 20260508 01:12:58.955 | INFO | } | 20260508 01:12:58.955 | INFO | Perf report for the last 5.13534 seconds: | 20260508 01:12:58.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:12:58.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 01:12:58.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:12:58.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:12:58.955 | INFO | Uploaded 36GiB to GPU in 2.34448s (15.3552GiB/s) | 20260508 01:12:58.955 | INFO | Downloaded 34GiB to GPU in 2.49744s (13.6139GiB/s) | 20260508 01:13:04.096 | INFO | Analyzing chunk frames = [4069888,4070400) | 20260508 01:13:04.096 | INFO | MemInfo: { | 20260508 01:13:04.096 | INFO | "gpu" : " 17 GB", | 20260508 01:13:04.096 | INFO | "huge" : "", | 20260508 01:13:04.096 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:04.096 | INFO | "malloc" : " 14 GB", | 20260508 01:13:04.096 | INFO | "pinned" : " 29 GB" | 20260508 01:13:04.096 | INFO | } | 20260508 01:13:04.097 | INFO | Perf report for the last 5.1418 seconds: | 20260508 01:13:04.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:04.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 01:13:04.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:13:04.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:04.097 | INFO | Uploaded 36GiB to GPU in 2.44737s (14.7097GiB/s) | 20260508 01:13:04.097 | INFO | Downloaded 34GiB to GPU in 2.40011s (14.166GiB/s) | 20260508 01:13:09.233 | INFO | Analyzing chunk frames = [4070400,4070912) | 20260508 01:13:09.233 | INFO | MemInfo: { | 20260508 01:13:09.233 | INFO | "gpu" : " 17 GB", | 20260508 01:13:09.233 | INFO | "huge" : "", | 20260508 01:13:09.233 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:09.233 | INFO | "malloc" : " 14 GB", | 20260508 01:13:09.233 | INFO | "pinned" : " 29 GB" | 20260508 01:13:09.233 | INFO | } | 20260508 01:13:09.234 | INFO | Perf report for the last 5.13692 seconds: | 20260508 01:13:09.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:09.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.234 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 01:13:09.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.234 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:13:09.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:09.235 | INFO | Uploaded 36GiB to GPU in 2.43539s (14.782GiB/s) | 20260508 01:13:09.235 | INFO | Downloaded 34GiB to GPU in 2.41297s (14.0905GiB/s) | 20260508 01:13:14.376 | INFO | Analyzing chunk frames = [4070912,4071424) | 20260508 01:13:14.376 | INFO | MemInfo: { | 20260508 01:13:14.376 | INFO | "gpu" : " 17 GB", | 20260508 01:13:14.376 | INFO | "huge" : "", | 20260508 01:13:14.376 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:14.376 | INFO | "malloc" : " 14 GB", | 20260508 01:13:14.376 | INFO | "pinned" : " 29 GB" | 20260508 01:13:14.376 | INFO | } | 20260508 01:13:14.376 | INFO | Perf report for the last 5.14268 seconds: | 20260508 01:13:14.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:14.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.377 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 01:13:14.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.377 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 01:13:14.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:14.377 | INFO | Uploaded 36GiB to GPU in 2.56748s (14.0215GiB/s) | 20260508 01:13:14.377 | INFO | Downloaded 34GiB to GPU in 2.45206s (13.8659GiB/s) | 20260508 01:13:14.377 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:13:14.377 | INFO | Total mem usage: 16.7 GiB | 20260508 01:13:14.377 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:13:14.377 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.33%) | 20260508 01:13:14.377 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:13:14.377 | INFO | High Water Breakdown: | 20260508 01:13:14.377 | INFO | 61% : 9.56 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:13:14.377 | INFO | 38.9% : 6.11 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:13:14.377 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:13:14.377 | INFO | ------------------------------------------ | 20260508 01:13:14.377 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:13:14.377 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:13:14.377 | INFO | Total mem usage: 26.5 GiB | 20260508 01:13:14.377 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:13:14.377 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 01:13:14.377 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 01:13:14.377 | INFO | High Water Breakdown: | 20260508 01:13:14.377 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:13:14.377 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:13:14.377 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:13:14.377 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:13:14.377 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:13:14.377 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:13:14.377 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:13:14.378 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:13:14.378 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:13:14.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:13:14.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:13:14.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:13:14.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:13:14.378 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:13:14.378 | INFO | ------------------------------------------ | 20260508 01:13:14.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:13:14.378 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:13:14.378 | INFO | Total mem usage: 27.8 GiB | 20260508 01:13:14.378 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:13:14.378 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:13:14.378 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:13:14.378 | INFO | High Water Breakdown: | 20260508 01:13:14.378 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:13:14.378 | INFO | ------------------------------------------ | 20260508 01:13:14.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:13:14.378 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:13:14.378 | INFO | Total mem usage: 15.7 GiB | 20260508 01:13:14.378 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:13:14.378 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:13:14.378 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:13:14.378 | INFO | High Water Breakdown: | 20260508 01:13:14.378 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:13:14.378 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:13:14.378 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:13:14.378 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:13:14.378 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:13:14.378 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:13:14.378 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:13:14.378 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:13:14.378 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:13:14.378 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:13:14.378 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:13:14.378 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:13:14.378 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:13:14.378 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:13:14.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:13:14.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:13:14.378 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:13:14.378 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:13:14.378 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:13:14.378 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:13:14.378 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:13:14.378 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:13:14.378 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:13:14.378 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:13:14.378 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:13:14.378 | INFO | ------------------------------------------ | 20260508 01:13:14.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:13:14.378 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:13:14.378 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:13:14.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:13:14.378 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.39e+00 | 20260508 01:13:14.378 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.11e-02 / 5.06e-03 | 20260508 01:13:14.378 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.46e+00, 5.86e-01 / 2.16e-01 | 20260508 01:13:14.378 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.09e-01 / 9.24e-02 | 20260508 01:13:14.378 | INFO | PerfReport: 16.11%, DME -- 4.22e+01 / 4.63e+01, 4.19e+00 / 1.49e+00 | 20260508 01:13:14.378 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.59e+00 / 1.44e+00 | 20260508 01:13:14.378 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.45e+00 / 2.74e+00, 4.31e-01 / 1.88e-01 | 20260508 01:13:14.378 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.30e+00, 2.73e-01 / 1.35e-01 | 20260508 01:13:14.378 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.72e-01, 5.27e-02 / 2.63e-02 | 20260508 01:13:14.379 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:13:14.379 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:13:14.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:13:14.379 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 24.17%, Upload -- 2.57e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 2.07%, LockUpload -- 2.20e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 2.97%, LockDownload -- 3.16e+00 / 0.00e+00, 4.48e+00 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:13:14.379 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260508 01:13:14.496 | INFO | Starting a round of Baz IO | 20260508 01:13:17.965 | INFO | Populating RTMetrics report | 20260508 01:13:17.965 | INFO | RTMetrics report populated | 20260508 01:13:17.965 | INFO | Opening tmp RTMetrics json file | 20260508 01:13:18.059 | INFO | Serializing RTMetrics to json | 20260508 01:13:18.065 | INFO | Serialization done, swapping tmp file now... | 20260508 01:13:18.072 | INFO | RTMetrics done | 20260508 01:13:18.098 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.601 seconds | 20260508 01:13:18.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.91% / 0.00% / 58.08% | 20260508 01:13:18.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.889818% / 0% / 5.5631e-05% | 20260508 01:13:18.098 | INFO | Baz IO overall: | 20260508 01:13:18.098 | INFO | BytesWritten : 1846242283205 | 20260508 01:13:18.098 | INFO | NumEvents : 893202060028 | 20260508 01:13:19.517 | INFO | Analyzing chunk frames = [4071424,4071936) | 20260508 01:13:19.517 | INFO | MemInfo: { | 20260508 01:13:19.517 | INFO | "gpu" : " 17 GB", | 20260508 01:13:19.517 | INFO | "huge" : "", | 20260508 01:13:19.517 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:19.517 | INFO | "malloc" : " 14 GB", | 20260508 01:13:19.517 | INFO | "pinned" : " 29 GB" | 20260508 01:13:19.517 | INFO | } | 20260508 01:13:19.517 | INFO | Perf report for the last 5.14078 seconds: | 20260508 01:13:19.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:19.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:19.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:19.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:19.517 | INFO | ANALYSIS : 68.9%, 2.9, 104.7ms, 0 | 20260508 01:13:19.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:19.517 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260508 01:13:19.517 | INFO | PRE_HQ : 67.5%, 1.8, 62.5ms, 0 | 20260508 01:13:19.517 | INFO | BAZWRITER : 69.7%, 3.7, 132.4ms, 0 | 20260508 01:13:19.517 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260508 01:13:19.517 | INFO | Uploaded 36GiB to GPU in 2.59816s (13.8559GiB/s) | 20260508 01:13:19.517 | INFO | Downloaded 38GiB to GPU in 2.68805s (14.1366GiB/s) | 20260508 01:13:24.654 | INFO | Analyzing chunk frames = [4071936,4072448) | 20260508 01:13:24.654 | INFO | MemInfo: { | 20260508 01:13:24.654 | INFO | "gpu" : " 17 GB", | 20260508 01:13:24.654 | INFO | "huge" : "", | 20260508 01:13:24.654 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:24.654 | INFO | "malloc" : " 14 GB", | 20260508 01:13:24.654 | INFO | "pinned" : " 29 GB" | 20260508 01:13:24.655 | INFO | } | 20260508 01:13:24.655 | INFO | Perf report for the last 5.13762 seconds: | 20260508 01:13:24.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:24.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:13:24.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:13:24.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:24.655 | INFO | Uploaded 36GiB to GPU in 2.54704s (14.134GiB/s) | 20260508 01:13:24.655 | INFO | Downloaded 34GiB to GPU in 2.43167s (13.9822GiB/s) | 20260508 01:13:29.795 | INFO | Analyzing chunk frames = [4072448,4072960) | 20260508 01:13:29.795 | INFO | MemInfo: { | 20260508 01:13:29.795 | INFO | "gpu" : " 17 GB", | 20260508 01:13:29.795 | INFO | "huge" : "", | 20260508 01:13:29.795 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:29.795 | INFO | "malloc" : " 14 GB", | 20260508 01:13:29.796 | INFO | "pinned" : " 29 GB" | 20260508 01:13:29.796 | INFO | } | 20260508 01:13:29.796 | INFO | Perf report for the last 5.14107 seconds: | 20260508 01:13:29.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:29.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:13:29.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:13:29.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:29.796 | INFO | Uploaded 36GiB to GPU in 2.52631s (14.25GiB/s) | 20260508 01:13:29.796 | INFO | Downloaded 34GiB to GPU in 2.48568s (13.6784GiB/s) | 20260508 01:13:34.935 | INFO | Analyzing chunk frames = [4072960,4073472) | 20260508 01:13:34.935 | INFO | MemInfo: { | 20260508 01:13:34.935 | INFO | "gpu" : " 17 GB", | 20260508 01:13:34.935 | INFO | "huge" : "", | 20260508 01:13:34.935 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:34.935 | INFO | "malloc" : " 14 GB", | 20260508 01:13:34.935 | INFO | "pinned" : " 29 GB" | 20260508 01:13:34.935 | INFO | } | 20260508 01:13:34.935 | INFO | Perf report for the last 5.13947 seconds: | 20260508 01:13:34.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:34.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 01:13:34.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:13:34.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:34.935 | INFO | Uploaded 36GiB to GPU in 2.52398s (14.2632GiB/s) | 20260508 01:13:34.936 | INFO | Downloaded 34GiB to GPU in 2.49232s (13.6419GiB/s) | 20260508 01:13:40.068 | INFO | Analyzing chunk frames = [4073472,4073984) | 20260508 01:13:40.068 | INFO | MemInfo: { | 20260508 01:13:40.068 | INFO | "gpu" : " 17 GB", | 20260508 01:13:40.068 | INFO | "huge" : "", | 20260508 01:13:40.068 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:40.068 | INFO | "malloc" : " 14 GB", | 20260508 01:13:40.069 | INFO | "pinned" : " 29 GB" | 20260508 01:13:40.069 | INFO | } | 20260508 01:13:40.069 | INFO | Perf report for the last 5.13353 seconds: | 20260508 01:13:40.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:40.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 01:13:40.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:13:40.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:40.069 | INFO | Uploaded 36GiB to GPU in 2.31205s (15.5706GiB/s) | 20260508 01:13:40.070 | INFO | Downloaded 34GiB to GPU in 2.51182s (13.536GiB/s) | 20260508 01:13:45.206 | INFO | Analyzing chunk frames = [4073984,4074496) | 20260508 01:13:45.206 | INFO | MemInfo: { | 20260508 01:13:45.206 | INFO | "gpu" : " 17 GB", | 20260508 01:13:45.206 | INFO | "huge" : "", | 20260508 01:13:45.206 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:45.206 | INFO | "malloc" : " 14 GB", | 20260508 01:13:45.206 | INFO | "pinned" : " 29 GB" | 20260508 01:13:45.206 | INFO | } | 20260508 01:13:45.206 | INFO | Perf report for the last 5.13745 seconds: | 20260508 01:13:45.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:45.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.206 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:13:45.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.206 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:13:45.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:45.207 | INFO | Uploaded 36GiB to GPU in 2.35252s (15.3027GiB/s) | 20260508 01:13:45.207 | INFO | Downloaded 34GiB to GPU in 2.43055s (13.9886GiB/s) | 20260508 01:13:50.349 | INFO | Analyzing chunk frames = [4074496,4075008) | 20260508 01:13:50.349 | INFO | MemInfo: { | 20260508 01:13:50.349 | INFO | "gpu" : " 17 GB", | 20260508 01:13:50.349 | INFO | "huge" : "", | 20260508 01:13:50.349 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:50.349 | INFO | "malloc" : " 14 GB", | 20260508 01:13:50.350 | INFO | "pinned" : " 29 GB" | 20260508 01:13:50.350 | INFO | } | 20260508 01:13:50.350 | INFO | Perf report for the last 5.14356 seconds: | 20260508 01:13:50.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:50.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 01:13:50.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:13:50.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:50.350 | INFO | Uploaded 36GiB to GPU in 2.50955s (14.3452GiB/s) | 20260508 01:13:50.351 | INFO | Downloaded 34GiB to GPU in 2.46381s (13.7998GiB/s) | 20260508 01:13:55.491 | INFO | Analyzing chunk frames = [4075008,4075520) | 20260508 01:13:55.491 | INFO | MemInfo: { | 20260508 01:13:55.491 | INFO | "gpu" : " 17 GB", | 20260508 01:13:55.491 | INFO | "huge" : "", | 20260508 01:13:55.491 | INFO | "huge_shared" : " 29 GB", | 20260508 01:13:55.491 | INFO | "malloc" : " 14 GB", | 20260508 01:13:55.491 | INFO | "pinned" : " 29 GB" | 20260508 01:13:55.491 | INFO | } | 20260508 01:13:55.491 | INFO | Perf report for the last 5.14134 seconds: | 20260508 01:13:55.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:13:55.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.491 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:13:55.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.491 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 01:13:55.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:13:55.492 | INFO | Uploaded 36GiB to GPU in 2.51932s (14.2896GiB/s) | 20260508 01:13:55.492 | INFO | Downloaded 34GiB to GPU in 2.46822s (13.7751GiB/s) | 20260508 01:13:55.605 | INFO | Starting a round of Baz IO | 20260508 01:13:59.060 | INFO | Populating RTMetrics report | 20260508 01:13:59.061 | INFO | RTMetrics report populated | 20260508 01:13:59.061 | INFO | Opening tmp RTMetrics json file | 20260508 01:13:59.063 | INFO | Serializing RTMetrics to json | 20260508 01:13:59.088 | INFO | Serialization done, swapping tmp file now... | 20260508 01:13:59.090 | INFO | RTMetrics done | 20260508 01:13:59.158 | INFO | Finished a round of Baz IO: Wrote 1.673 GiB in 3.552 seconds | 20260508 01:13:59.158 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.91% / 0.00% / 58.09% | 20260508 01:13:59.158 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.89378% / 0% / 6.75519e-05% | 20260508 01:13:59.158 | INFO | Baz IO overall: | 20260508 01:13:59.158 | INFO | BytesWritten : 1848038636011 | 20260508 01:13:59.158 | INFO | NumEvents : 894066098059 | 20260508 01:14:00.632 | INFO | Analyzing chunk frames = [4075520,4076032) | 20260508 01:14:00.632 | INFO | MemInfo: { | 20260508 01:14:00.632 | INFO | "gpu" : " 17 GB", | 20260508 01:14:00.632 | INFO | "huge" : "", | 20260508 01:14:00.632 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:00.632 | INFO | "malloc" : " 14 GB", | 20260508 01:14:00.632 | INFO | "pinned" : " 29 GB" | 20260508 01:14:00.632 | INFO | } | 20260508 01:14:00.632 | INFO | Perf report for the last 5.14076 seconds: | 20260508 01:14:00.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:00.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:00.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:00.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:00.632 | INFO | ANALYSIS : 68.8%, 2.9, 104.7ms, 0 | 20260508 01:14:00.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:00.632 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260508 01:14:00.632 | INFO | PRE_HQ : 67.4%, 1.8, 62.0ms, 0 | 20260508 01:14:00.632 | INFO | BAZWRITER : 68.7%, 4.1, 146.0ms, 0 | 20260508 01:14:00.632 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 01:14:00.632 | INFO | Uploaded 36GiB to GPU in 2.59214s (13.8881GiB/s) | 20260508 01:14:00.632 | INFO | Downloaded 38GiB to GPU in 2.7049s (14.0486GiB/s) | 20260508 01:14:05.768 | INFO | Analyzing chunk frames = [4076032,4076544) | 20260508 01:14:05.768 | INFO | MemInfo: { | 20260508 01:14:05.768 | INFO | "gpu" : " 17 GB", | 20260508 01:14:05.768 | INFO | "huge" : "", | 20260508 01:14:05.768 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:05.768 | INFO | "malloc" : " 14 GB", | 20260508 01:14:05.768 | INFO | "pinned" : " 29 GB" | 20260508 01:14:05.768 | INFO | } | 20260508 01:14:05.768 | INFO | Perf report for the last 5.13589 seconds: | 20260508 01:14:05.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:05.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:14:05.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:14:05.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:05.768 | INFO | Uploaded 36GiB to GPU in 2.51299s (14.3256GiB/s) | 20260508 01:14:05.769 | INFO | Downloaded 34GiB to GPU in 2.44009s (13.9339GiB/s) | 20260508 01:14:10.908 | INFO | Analyzing chunk frames = [4076544,4077056) | 20260508 01:14:10.908 | INFO | MemInfo: { | 20260508 01:14:10.908 | INFO | "gpu" : " 17 GB", | 20260508 01:14:10.908 | INFO | "huge" : "", | 20260508 01:14:10.908 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:10.908 | INFO | "malloc" : " 14 GB", | 20260508 01:14:10.908 | INFO | "pinned" : " 29 GB" | 20260508 01:14:10.908 | INFO | } | 20260508 01:14:10.908 | INFO | Perf report for the last 5.1401 seconds: | 20260508 01:14:10.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:10.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 01:14:10.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:14:10.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:10.908 | INFO | Uploaded 36GiB to GPU in 2.44917s (14.6989GiB/s) | 20260508 01:14:10.908 | INFO | Downloaded 34GiB to GPU in 2.49831s (13.6092GiB/s) | 20260508 01:14:16.049 | INFO | Analyzing chunk frames = [4077056,4077568) | 20260508 01:14:16.049 | INFO | MemInfo: { | 20260508 01:14:16.049 | INFO | "gpu" : " 17 GB", | 20260508 01:14:16.049 | INFO | "huge" : "", | 20260508 01:14:16.049 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:16.049 | INFO | "malloc" : " 14 GB", | 20260508 01:14:16.049 | INFO | "pinned" : " 29 GB" | 20260508 01:14:16.049 | INFO | } | 20260508 01:14:16.050 | INFO | Perf report for the last 5.1416 seconds: | 20260508 01:14:16.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:16.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:14:16.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:14:16.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:16.050 | INFO | Uploaded 36GiB to GPU in 2.44859s (14.7023GiB/s) | 20260508 01:14:16.050 | INFO | Downloaded 34GiB to GPU in 2.48682s (13.6721GiB/s) | 20260508 01:14:21.187 | INFO | Analyzing chunk frames = [4077568,4078080) | 20260508 01:14:21.187 | INFO | MemInfo: { | 20260508 01:14:21.187 | INFO | "gpu" : " 17 GB", | 20260508 01:14:21.187 | INFO | "huge" : "", | 20260508 01:14:21.187 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:21.187 | INFO | "malloc" : " 14 GB", | 20260508 01:14:21.187 | INFO | "pinned" : " 29 GB" | 20260508 01:14:21.187 | INFO | } | 20260508 01:14:21.187 | INFO | Perf report for the last 5.13748 seconds: | 20260508 01:14:21.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:21.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.187 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:14:21.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.187 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 01:14:21.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:21.188 | INFO | Uploaded 36GiB to GPU in 2.32675s (15.4722GiB/s) | 20260508 01:14:21.188 | INFO | Downloaded 34GiB to GPU in 2.52706s (13.4544GiB/s) | 20260508 01:14:26.327 | INFO | Analyzing chunk frames = [4078080,4078592) | 20260508 01:14:26.327 | INFO | MemInfo: { | 20260508 01:14:26.327 | INFO | "gpu" : " 17 GB", | 20260508 01:14:26.327 | INFO | "huge" : "", | 20260508 01:14:26.327 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:26.327 | INFO | "malloc" : " 14 GB", | 20260508 01:14:26.327 | INFO | "pinned" : " 29 GB" | 20260508 01:14:26.327 | INFO | } | 20260508 01:14:26.327 | INFO | Perf report for the last 5.13987 seconds: | 20260508 01:14:26.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:26.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 01:14:26.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:14:26.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:26.327 | INFO | Uploaded 36GiB to GPU in 2.37629s (15.1497GiB/s) | 20260508 01:14:26.327 | INFO | Downloaded 34GiB to GPU in 2.43412s (13.9681GiB/s) | 20260508 01:14:31.463 | INFO | Analyzing chunk frames = [4078592,4079104) | 20260508 01:14:31.463 | INFO | MemInfo: { | 20260508 01:14:31.463 | INFO | "gpu" : " 17 GB", | 20260508 01:14:31.463 | INFO | "huge" : "", | 20260508 01:14:31.463 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:31.464 | INFO | "malloc" : " 14 GB", | 20260508 01:14:31.464 | INFO | "pinned" : " 29 GB" | 20260508 01:14:31.464 | INFO | } | 20260508 01:14:31.464 | INFO | Perf report for the last 5.13673 seconds: | 20260508 01:14:31.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:31.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:14:31.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 01:14:31.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:31.464 | INFO | Uploaded 36GiB to GPU in 2.38634s (15.0859GiB/s) | 20260508 01:14:31.464 | INFO | Downloaded 34GiB to GPU in 2.43558s (13.9597GiB/s) | 20260508 01:14:36.606 | INFO | Analyzing chunk frames = [4079104,4079616) | 20260508 01:14:36.606 | INFO | MemInfo: { | 20260508 01:14:36.606 | INFO | "gpu" : " 17 GB", | 20260508 01:14:36.606 | INFO | "huge" : "", | 20260508 01:14:36.606 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:36.606 | INFO | "malloc" : " 14 GB", | 20260508 01:14:36.606 | INFO | "pinned" : " 29 GB" | 20260508 01:14:36.606 | INFO | } | 20260508 01:14:36.606 | INFO | Perf report for the last 5.14263 seconds: | 20260508 01:14:36.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:36.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.606 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:14:36.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.606 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260508 01:14:36.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:36.607 | INFO | Uploaded 36GiB to GPU in 2.51996s (14.2859GiB/s) | 20260508 01:14:36.607 | INFO | Downloaded 34GiB to GPU in 2.46895s (13.7711GiB/s) | 20260508 01:14:36.607 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:14:36.607 | INFO | Total mem usage: 16.7 GiB | 20260508 01:14:36.607 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:14:36.607 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 01:14:36.607 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:14:36.607 | INFO | High Water Breakdown: | 20260508 01:14:36.607 | INFO | 60% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:14:36.607 | INFO | 39.8% : 5.9 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:14:36.607 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:14:36.607 | INFO | ------------------------------------------ | 20260508 01:14:36.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:14:36.607 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:14:36.607 | INFO | Total mem usage: 26.5 GiB | 20260508 01:14:36.607 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:14:36.607 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 01:14:36.607 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 01:14:36.607 | INFO | High Water Breakdown: | 20260508 01:14:36.607 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:14:36.607 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:14:36.607 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:14:36.607 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:14:36.607 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:14:36.607 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:14:36.607 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:14:36.607 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:14:36.607 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:14:36.607 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:14:36.607 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:14:36.607 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:14:36.607 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:14:36.607 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:14:36.607 | INFO | ------------------------------------------ | 20260508 01:14:36.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:14:36.607 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:14:36.607 | INFO | Total mem usage: 27.8 GiB | 20260508 01:14:36.607 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:14:36.607 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:14:36.607 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:14:36.607 | INFO | High Water Breakdown: | 20260508 01:14:36.607 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:14:36.607 | INFO | ------------------------------------------ | 20260508 01:14:36.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:14:36.607 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:14:36.607 | INFO | Total mem usage: 15.7 GiB | 20260508 01:14:36.607 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:14:36.607 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:14:36.607 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:14:36.607 | INFO | High Water Breakdown: | 20260508 01:14:36.607 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:14:36.607 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:14:36.607 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:14:36.607 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:14:36.607 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:14:36.607 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:14:36.607 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:14:36.607 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:14:36.607 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:14:36.607 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:14:36.607 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:14:36.607 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:14:36.607 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:14:36.608 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:14:36.608 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:14:36.608 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:14:36.608 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:14:36.608 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:14:36.608 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:14:36.608 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:14:36.608 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:14:36.608 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:14:36.608 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:14:36.608 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:14:36.608 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:14:36.608 | INFO | ------------------------------------------ | 20260508 01:14:36.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:14:36.608 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:14:36.608 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:14:36.608 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:14:36.608 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 1.43e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.05e-01, 1.15e-02 / 9.99e-03 | 20260508 01:14:36.608 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.49e+00, 5.89e-01 / 2.00e-01 | 20260508 01:14:36.608 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.12e-01 | 20260508 01:14:36.608 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.60e+01, 3.99e+00 / 1.71e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.61e+00 / 1.37e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.65e+00, 4.30e-01 / 2.79e-01 | 20260508 01:14:36.608 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.24e+00, 2.71e-01 / 1.22e-01 | 20260508 01:14:36.608 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.67e-01, 5.30e-02 / 2.41e-02 | 20260508 01:14:36.608 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:14:36.608 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:14:36.608 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:14:36.608 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 24.21%, Upload -- 2.57e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 24.50%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 1.98%, LockUpload -- 2.10e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 2.99%, LockDownload -- 3.18e+00 / 0.00e+00, 4.40e+00 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:14:36.608 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 01:14:36.704 | INFO | Starting a round of Baz IO | 20260508 01:14:40.157 | INFO | Populating RTMetrics report | 20260508 01:14:40.158 | INFO | RTMetrics report populated | 20260508 01:14:40.158 | INFO | Opening tmp RTMetrics json file | 20260508 01:14:40.188 | INFO | Serializing RTMetrics to json | 20260508 01:14:40.226 | INFO | Serialization done, swapping tmp file now... | 20260508 01:14:40.229 | INFO | RTMetrics done | 20260508 01:14:40.253 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.549 seconds | 20260508 01:14:40.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.91% / 0.00% / 58.09% | 20260508 01:14:40.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.888518% / 7.94729e-06% / 6.35783e-05% | 20260508 01:14:40.253 | INFO | Baz IO overall: | 20260508 01:14:40.253 | INFO | BytesWritten : 1849831635565 | 20260508 01:14:40.253 | INFO | NumEvents : 894927777939 | 20260508 01:14:41.746 | INFO | Analyzing chunk frames = [4079616,4080128) | 20260508 01:14:41.746 | INFO | MemInfo: { | 20260508 01:14:41.746 | INFO | "gpu" : " 17 GB", | 20260508 01:14:41.746 | INFO | "huge" : "", | 20260508 01:14:41.746 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:41.746 | INFO | "malloc" : " 14 GB", | 20260508 01:14:41.746 | INFO | "pinned" : " 29 GB" | 20260508 01:14:41.746 | INFO | } | 20260508 01:14:41.746 | INFO | Perf report for the last 5.13995 seconds: | 20260508 01:14:41.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:41.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:41.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:41.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:41.746 | INFO | ANALYSIS : 68.1%, 3.0, 106.4ms, 0 | 20260508 01:14:41.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:41.746 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 01:14:41.746 | INFO | PRE_HQ : 65.2%, 1.7, 56.3ms, 0 | 20260508 01:14:41.746 | INFO | BAZWRITER : 68.6%, 4.2, 146.5ms, 0 | 20260508 01:14:41.746 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 01:14:41.747 | INFO | Uploaded 36GiB to GPU in 2.53029s (14.2276GiB/s) | 20260508 01:14:41.747 | INFO | Downloaded 38GiB to GPU in 2.67067s (14.2286GiB/s) | 20260508 01:14:46.884 | INFO | Analyzing chunk frames = [4080128,4080640) | 20260508 01:14:46.884 | INFO | MemInfo: { | 20260508 01:14:46.884 | INFO | "gpu" : " 17 GB", | 20260508 01:14:46.884 | INFO | "huge" : "", | 20260508 01:14:46.884 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:46.884 | INFO | "malloc" : " 14 GB", | 20260508 01:14:46.884 | INFO | "pinned" : " 29 GB" | 20260508 01:14:46.884 | INFO | } | 20260508 01:14:46.884 | INFO | Perf report for the last 5.13812 seconds: | 20260508 01:14:46.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:46.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.884 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:14:46.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.884 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:14:46.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:46.885 | INFO | Uploaded 36GiB to GPU in 2.5167s (14.3045GiB/s) | 20260508 01:14:46.885 | INFO | Downloaded 34GiB to GPU in 2.44363s (13.9137GiB/s) | 20260508 01:14:52.014 | INFO | Analyzing chunk frames = [4080640,4081152) | 20260508 01:14:52.014 | INFO | MemInfo: { | 20260508 01:14:52.014 | INFO | "gpu" : " 17 GB", | 20260508 01:14:52.015 | INFO | "huge" : "", | 20260508 01:14:52.015 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:52.015 | INFO | "malloc" : " 14 GB", | 20260508 01:14:52.015 | INFO | "pinned" : " 29 GB" | 20260508 01:14:52.015 | INFO | } | 20260508 01:14:52.015 | INFO | Perf report for the last 5.13039 seconds: | 20260508 01:14:52.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:52.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:14:52.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:14:52.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:52.015 | INFO | Uploaded 36GiB to GPU in 2.47433s (14.5494GiB/s) | 20260508 01:14:52.015 | INFO | Downloaded 34GiB to GPU in 2.50315s (13.5829GiB/s) | 20260508 01:14:57.163 | INFO | Analyzing chunk frames = [4081152,4081664) | 20260508 01:14:57.163 | INFO | MemInfo: { | 20260508 01:14:57.163 | INFO | "gpu" : " 17 GB", | 20260508 01:14:57.163 | INFO | "huge" : "", | 20260508 01:14:57.163 | INFO | "huge_shared" : " 29 GB", | 20260508 01:14:57.163 | INFO | "malloc" : " 14 GB", | 20260508 01:14:57.163 | INFO | "pinned" : " 29 GB" | 20260508 01:14:57.163 | INFO | } | 20260508 01:14:57.163 | INFO | Perf report for the last 5.1486 seconds: | 20260508 01:14:57.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:14:57.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.163 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 01:14:57.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.164 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 01:14:57.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:14:57.164 | INFO | Uploaded 36GiB to GPU in 2.47758s (14.5303GiB/s) | 20260508 01:14:57.164 | INFO | Downloaded 34GiB to GPU in 2.5103s (13.5442GiB/s) | 20260508 01:15:02.300 | INFO | Analyzing chunk frames = [4081664,4082176) | 20260508 01:15:02.300 | INFO | MemInfo: { | 20260508 01:15:02.300 | INFO | "gpu" : " 17 GB", | 20260508 01:15:02.300 | INFO | "huge" : "", | 20260508 01:15:02.300 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:02.300 | INFO | "malloc" : " 14 GB", | 20260508 01:15:02.300 | INFO | "pinned" : " 29 GB" | 20260508 01:15:02.300 | INFO | } | 20260508 01:15:02.300 | INFO | Perf report for the last 5.13651 seconds: | 20260508 01:15:02.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:02.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:15:02.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:15:02.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:02.300 | INFO | Uploaded 36GiB to GPU in 2.32367s (15.4927GiB/s) | 20260508 01:15:02.300 | INFO | Downloaded 34GiB to GPU in 2.50012s (13.5993GiB/s) | 20260508 01:15:07.441 | INFO | Analyzing chunk frames = [4082176,4082688) | 20260508 01:15:07.441 | INFO | MemInfo: { | 20260508 01:15:07.441 | INFO | "gpu" : " 17 GB", | 20260508 01:15:07.441 | INFO | "huge" : "", | 20260508 01:15:07.441 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:07.441 | INFO | "malloc" : " 14 GB", | 20260508 01:15:07.441 | INFO | "pinned" : " 29 GB" | 20260508 01:15:07.441 | INFO | } | 20260508 01:15:07.441 | INFO | Perf report for the last 5.14125 seconds: | 20260508 01:15:07.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:07.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.441 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:15:07.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.441 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:15:07.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:07.442 | INFO | Uploaded 36GiB to GPU in 2.38519s (15.0932GiB/s) | 20260508 01:15:07.442 | INFO | Downloaded 34GiB to GPU in 2.411s (14.102GiB/s) | 20260508 01:15:12.580 | INFO | Analyzing chunk frames = [4082688,4083200) | 20260508 01:15:12.580 | INFO | MemInfo: { | 20260508 01:15:12.580 | INFO | "gpu" : " 17 GB", | 20260508 01:15:12.580 | INFO | "huge" : "", | 20260508 01:15:12.580 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:12.580 | INFO | "malloc" : " 14 GB", | 20260508 01:15:12.580 | INFO | "pinned" : " 29 GB" | 20260508 01:15:12.580 | INFO | } | 20260508 01:15:12.580 | INFO | Perf report for the last 5.1391 seconds: | 20260508 01:15:12.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:12.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.580 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 01:15:12.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.580 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 01:15:12.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:12.581 | INFO | Uploaded 36GiB to GPU in 2.42384s (14.8525GiB/s) | 20260508 01:15:12.581 | INFO | Downloaded 34GiB to GPU in 2.43015s (13.9909GiB/s) | 20260508 01:15:17.717 | INFO | Analyzing chunk frames = [4083200,4083712) | 20260508 01:15:17.717 | INFO | MemInfo: { | 20260508 01:15:17.717 | INFO | "gpu" : " 17 GB", | 20260508 01:15:17.717 | INFO | "huge" : "", | 20260508 01:15:17.717 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:17.717 | INFO | "malloc" : " 14 GB", | 20260508 01:15:17.718 | INFO | "pinned" : " 29 GB" | 20260508 01:15:17.718 | INFO | } | 20260508 01:15:17.718 | INFO | Perf report for the last 5.13731 seconds: | 20260508 01:15:17.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:17.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:15:17.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260508 01:15:17.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:17.718 | INFO | Uploaded 36GiB to GPU in 2.55267s (14.1029GiB/s) | 20260508 01:15:17.718 | INFO | Downloaded 34GiB to GPU in 2.46305s (13.804GiB/s) | 20260508 01:15:17.828 | INFO | Starting a round of Baz IO | 20260508 01:15:21.285 | INFO | Populating RTMetrics report | 20260508 01:15:21.286 | INFO | RTMetrics report populated | 20260508 01:15:21.286 | INFO | Opening tmp RTMetrics json file | 20260508 01:15:21.303 | INFO | Serializing RTMetrics to json | 20260508 01:15:21.304 | INFO | Serialization done, swapping tmp file now... | 20260508 01:15:21.305 | INFO | RTMetrics done | 20260508 01:15:21.383 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.554 seconds | 20260508 01:15:21.383 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.90% / 0.00% / 58.09% | 20260508 01:15:21.383 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.889134% / 7.94729e-06% / 5.16574e-05% | 20260508 01:15:21.383 | INFO | Baz IO overall: | 20260508 01:15:21.383 | INFO | BytesWritten : 1851626073972 | 20260508 01:15:21.383 | INFO | NumEvents : 895790502692 | 20260508 01:15:22.860 | INFO | Analyzing chunk frames = [4083712,4084224) | 20260508 01:15:22.860 | INFO | MemInfo: { | 20260508 01:15:22.860 | INFO | "gpu" : " 17 GB", | 20260508 01:15:22.860 | INFO | "huge" : "", | 20260508 01:15:22.860 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:22.861 | INFO | "malloc" : " 14 GB", | 20260508 01:15:22.861 | INFO | "pinned" : " 29 GB" | 20260508 01:15:22.861 | INFO | } | 20260508 01:15:22.861 | INFO | Perf report for the last 5.14293 seconds: | 20260508 01:15:22.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:22.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:22.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:22.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:22.861 | INFO | ANALYSIS : 68.6%, 2.9, 104.2ms, 0 | 20260508 01:15:22.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:22.861 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 01:15:22.861 | INFO | PRE_HQ : 67.3%, 1.8, 62.2ms, 0 | 20260508 01:15:22.861 | INFO | BAZWRITER : 68.7%, 3.8, 132.7ms, 0 | 20260508 01:15:22.861 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:15:22.861 | INFO | Uploaded 36GiB to GPU in 2.55083s (14.113GiB/s) | 20260508 01:15:22.861 | INFO | Downloaded 38GiB to GPU in 2.69765s (14.0863GiB/s) | 20260508 01:15:28.001 | INFO | Analyzing chunk frames = [4084224,4084736) | 20260508 01:15:28.002 | INFO | MemInfo: { | 20260508 01:15:28.002 | INFO | "gpu" : " 17 GB", | 20260508 01:15:28.002 | INFO | "huge" : "", | 20260508 01:15:28.002 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:28.002 | INFO | "malloc" : " 14 GB", | 20260508 01:15:28.002 | INFO | "pinned" : " 29 GB" | 20260508 01:15:28.002 | INFO | } | 20260508 01:15:28.002 | INFO | Perf report for the last 5.14115 seconds: | 20260508 01:15:28.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:28.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:15:28.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:15:28.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:28.002 | INFO | Uploaded 36GiB to GPU in 2.5506s (14.1143GiB/s) | 20260508 01:15:28.002 | INFO | Downloaded 34GiB to GPU in 2.41053s (14.1048GiB/s) | 20260508 01:15:33.137 | INFO | Analyzing chunk frames = [4084736,4085248) | 20260508 01:15:33.137 | INFO | MemInfo: { | 20260508 01:15:33.137 | INFO | "gpu" : " 17 GB", | 20260508 01:15:33.137 | INFO | "huge" : "", | 20260508 01:15:33.137 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:33.137 | INFO | "malloc" : " 14 GB", | 20260508 01:15:33.137 | INFO | "pinned" : " 29 GB" | 20260508 01:15:33.137 | INFO | } | 20260508 01:15:33.137 | INFO | Perf report for the last 5.13523 seconds: | 20260508 01:15:33.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:33.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.137 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 01:15:33.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.137 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:15:33.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:33.138 | INFO | Uploaded 36GiB to GPU in 2.56214s (14.0507GiB/s) | 20260508 01:15:33.138 | INFO | Downloaded 34GiB to GPU in 2.48254s (13.6957GiB/s) | 20260508 01:15:38.274 | INFO | Analyzing chunk frames = [4085248,4085760) | 20260508 01:15:38.275 | INFO | MemInfo: { | 20260508 01:15:38.275 | INFO | "gpu" : " 17 GB", | 20260508 01:15:38.275 | INFO | "huge" : "", | 20260508 01:15:38.275 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:38.275 | INFO | "malloc" : " 14 GB", | 20260508 01:15:38.275 | INFO | "pinned" : " 29 GB" | 20260508 01:15:38.275 | INFO | } | 20260508 01:15:38.275 | INFO | Perf report for the last 5.13775 seconds: | 20260508 01:15:38.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:38.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:15:38.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:15:38.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:38.275 | INFO | Uploaded 36GiB to GPU in 2.55506s (14.0897GiB/s) | 20260508 01:15:38.275 | INFO | Downloaded 34GiB to GPU in 2.46755s (13.7788GiB/s) | 20260508 01:15:43.414 | INFO | Analyzing chunk frames = [4085760,4086272) | 20260508 01:15:43.414 | INFO | MemInfo: { | 20260508 01:15:43.414 | INFO | "gpu" : " 17 GB", | 20260508 01:15:43.414 | INFO | "huge" : "", | 20260508 01:15:43.414 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:43.414 | INFO | "malloc" : " 14 GB", | 20260508 01:15:43.414 | INFO | "pinned" : " 29 GB" | 20260508 01:15:43.414 | INFO | } | 20260508 01:15:43.414 | INFO | Perf report for the last 5.13968 seconds: | 20260508 01:15:43.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:43.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.415 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:15:43.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.415 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 01:15:43.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:43.415 | INFO | Uploaded 36GiB to GPU in 2.40297s (14.9815GiB/s) | 20260508 01:15:43.415 | INFO | Downloaded 34GiB to GPU in 2.48585s (13.6774GiB/s) | 20260508 01:15:48.555 | INFO | Analyzing chunk frames = [4086272,4086784) | 20260508 01:15:48.555 | INFO | MemInfo: { | 20260508 01:15:48.555 | INFO | "gpu" : " 17 GB", | 20260508 01:15:48.555 | INFO | "huge" : "", | 20260508 01:15:48.555 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:48.555 | INFO | "malloc" : " 14 GB", | 20260508 01:15:48.555 | INFO | "pinned" : " 29 GB" | 20260508 01:15:48.555 | INFO | } | 20260508 01:15:48.555 | INFO | Perf report for the last 5.1404 seconds: | 20260508 01:15:48.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:48.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:15:48.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:15:48.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:48.555 | INFO | Uploaded 36GiB to GPU in 2.39813s (15.0117GiB/s) | 20260508 01:15:48.555 | INFO | Downloaded 34GiB to GPU in 2.39972s (14.1683GiB/s) | 20260508 01:15:53.698 | INFO | Analyzing chunk frames = [4086784,4087296) | 20260508 01:15:53.698 | INFO | MemInfo: { | 20260508 01:15:53.698 | INFO | "gpu" : " 17 GB", | 20260508 01:15:53.698 | INFO | "huge" : "", | 20260508 01:15:53.698 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:53.698 | INFO | "malloc" : " 14 GB", | 20260508 01:15:53.698 | INFO | "pinned" : " 29 GB" | 20260508 01:15:53.698 | INFO | } | 20260508 01:15:53.698 | INFO | Perf report for the last 5.14298 seconds: | 20260508 01:15:53.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:53.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 01:15:53.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 01:15:53.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:53.698 | INFO | Uploaded 36GiB to GPU in 2.43068s (14.8106GiB/s) | 20260508 01:15:53.698 | INFO | Downloaded 34GiB to GPU in 2.42458s (14.023GiB/s) | 20260508 01:15:58.830 | INFO | Analyzing chunk frames = [4087296,4087808) | 20260508 01:15:58.830 | INFO | MemInfo: { | 20260508 01:15:58.830 | INFO | "gpu" : " 17 GB", | 20260508 01:15:58.830 | INFO | "huge" : "", | 20260508 01:15:58.830 | INFO | "huge_shared" : " 29 GB", | 20260508 01:15:58.830 | INFO | "malloc" : " 14 GB", | 20260508 01:15:58.831 | INFO | "pinned" : " 29 GB" | 20260508 01:15:58.831 | INFO | } | 20260508 01:15:58.831 | INFO | Perf report for the last 5.1328 seconds: | 20260508 01:15:58.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:15:58.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260508 01:15:58.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:15:58.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:15:58.831 | INFO | Uploaded 36GiB to GPU in 2.55221s (14.1054GiB/s) | 20260508 01:15:58.831 | INFO | Downloaded 34GiB to GPU in 2.45762s (13.8345GiB/s) | 20260508 01:15:58.831 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:15:58.831 | INFO | Total mem usage: 16.7 GiB | 20260508 01:15:58.831 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:15:58.831 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 01:15:58.831 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:15:58.831 | INFO | High Water Breakdown: | 20260508 01:15:58.831 | INFO | 60.3% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:15:58.831 | INFO | 39.6% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:15:58.831 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:15:58.831 | INFO | ------------------------------------------ | 20260508 01:15:58.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:15:58.831 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:15:58.831 | INFO | Total mem usage: 26.5 GiB | 20260508 01:15:58.831 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:15:58.831 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 01:15:58.831 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:15:58.831 | INFO | High Water Breakdown: | 20260508 01:15:58.831 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:15:58.831 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:15:58.831 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:15:58.831 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:15:58.831 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:15:58.831 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:15:58.831 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:15:58.831 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:15:58.831 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:15:58.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:15:58.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:15:58.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:15:58.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:15:58.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:15:58.832 | INFO | ------------------------------------------ | 20260508 01:15:58.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:15:58.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:15:58.832 | INFO | Total mem usage: 27.8 GiB | 20260508 01:15:58.832 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:15:58.832 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:15:58.832 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:15:58.832 | INFO | High Water Breakdown: | 20260508 01:15:58.832 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:15:58.832 | INFO | ------------------------------------------ | 20260508 01:15:58.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:15:58.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:15:58.832 | INFO | Total mem usage: 15.7 GiB | 20260508 01:15:58.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:15:58.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:15:58.832 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:15:58.832 | INFO | High Water Breakdown: | 20260508 01:15:58.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:15:58.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:15:58.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:15:58.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:15:58.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:15:58.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:15:58.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:15:58.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:15:58.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:15:58.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:15:58.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:15:58.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:15:58.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:15:58.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:15:58.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:15:58.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:15:58.832 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:15:58.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:15:58.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:15:58.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:15:58.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:15:58.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:15:58.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:15:58.833 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:15:58.833 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:15:58.833 | INFO | ------------------------------------------ | 20260508 01:15:58.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:15:58.833 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:15:58.833 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:15:58.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:15:58.833 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.37e+01 / 1.55e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.19e-02 / 6.35e-03 | 20260508 01:15:58.833 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.44e+00, 5.85e-01 / 2.35e-01 | 20260508 01:15:58.833 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.03e-01 | 20260508 01:15:58.833 | INFO | PerfReport: 16.04%, DME -- 4.21e+01 / 4.59e+01, 3.98e+00 / 1.91e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.70e+00 / 1.58e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.44e+00 / 2.69e+00, 4.25e-01 / 2.59e-01 | 20260508 01:15:58.833 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.29e+00, 2.74e-01 / 1.45e-01 | 20260508 01:15:58.833 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.73e-01, 5.41e-02 / 3.02e-02 | 20260508 01:15:58.833 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:15:58.833 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:15:58.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:15:58.833 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 24.29%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 2.16%, LockUpload -- 2.30e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 2.71%, LockDownload -- 2.88e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:15:58.833 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260508 01:15:58.929 | INFO | Starting a round of Baz IO | 20260508 01:16:02.379 | INFO | Populating RTMetrics report | 20260508 01:16:02.380 | INFO | RTMetrics report populated | 20260508 01:16:02.380 | INFO | Opening tmp RTMetrics json file | 20260508 01:16:02.439 | INFO | Serializing RTMetrics to json | 20260508 01:16:02.477 | INFO | Serialization done, swapping tmp file now... | 20260508 01:16:02.477 | INFO | RTMetrics done | 20260508 01:16:02.496 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.567 seconds | 20260508 01:16:02.496 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.90% / 0.00% / 58.10% | 20260508 01:16:02.496 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.889031% / 0% / 5.16574e-05% | 20260508 01:16:02.496 | INFO | Baz IO overall: | 20260508 01:16:02.496 | INFO | BytesWritten : 1853419470078 | 20260508 01:16:02.496 | INFO | NumEvents : 896652128334 | 20260508 01:16:03.970 | INFO | Analyzing chunk frames = [4087808,4088320) | 20260508 01:16:03.970 | INFO | MemInfo: { | 20260508 01:16:03.971 | INFO | "gpu" : " 17 GB", | 20260508 01:16:03.971 | INFO | "huge" : "", | 20260508 01:16:03.971 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:03.971 | INFO | "malloc" : " 14 GB", | 20260508 01:16:03.971 | INFO | "pinned" : " 29 GB" | 20260508 01:16:03.971 | INFO | } | 20260508 01:16:03.971 | INFO | Perf report for the last 5.13987 seconds: | 20260508 01:16:03.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:03.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:03.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:03.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:03.971 | INFO | ANALYSIS : 68.3%, 3.0, 106.7ms, 0 | 20260508 01:16:03.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:03.971 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 01:16:03.971 | INFO | PRE_HQ : 65.5%, 1.7, 56.6ms, 0 | 20260508 01:16:03.971 | INFO | BAZWRITER : 69.0%, 3.8, 136.4ms, 0 | 20260508 01:16:03.971 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 01:16:03.971 | INFO | Uploaded 36GiB to GPU in 2.57745s (13.9673GiB/s) | 20260508 01:16:03.971 | INFO | Downloaded 38GiB to GPU in 2.67979s (14.1802GiB/s) | 20260508 01:16:09.112 | INFO | Analyzing chunk frames = [4088320,4088832) | 20260508 01:16:09.112 | INFO | MemInfo: { | 20260508 01:16:09.112 | INFO | "gpu" : " 17 GB", | 20260508 01:16:09.112 | INFO | "huge" : "", | 20260508 01:16:09.112 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:09.112 | INFO | "malloc" : " 14 GB", | 20260508 01:16:09.112 | INFO | "pinned" : " 29 GB" | 20260508 01:16:09.113 | INFO | } | 20260508 01:16:09.113 | INFO | Perf report for the last 5.14202 seconds: | 20260508 01:16:09.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:09.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:16:09.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:16:09.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:09.113 | INFO | Uploaded 36GiB to GPU in 2.53932s (14.177GiB/s) | 20260508 01:16:09.113 | INFO | Downloaded 34GiB to GPU in 2.42454s (14.0233GiB/s) | 20260508 01:16:14.253 | INFO | Analyzing chunk frames = [4088832,4089344) | 20260508 01:16:14.253 | INFO | MemInfo: { | 20260508 01:16:14.253 | INFO | "gpu" : " 17 GB", | 20260508 01:16:14.253 | INFO | "huge" : "", | 20260508 01:16:14.253 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:14.253 | INFO | "malloc" : " 14 GB", | 20260508 01:16:14.253 | INFO | "pinned" : " 29 GB" | 20260508 01:16:14.253 | INFO | } | 20260508 01:16:14.253 | INFO | Perf report for the last 5.14036 seconds: | 20260508 01:16:14.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:14.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:16:14.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:16:14.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:14.253 | INFO | Uploaded 36GiB to GPU in 2.51223s (14.3299GiB/s) | 20260508 01:16:14.253 | INFO | Downloaded 34GiB to GPU in 2.47979s (13.7109GiB/s) | 20260508 01:16:19.388 | INFO | Analyzing chunk frames = [4089344,4089856) | 20260508 01:16:19.388 | INFO | MemInfo: { | 20260508 01:16:19.388 | INFO | "gpu" : " 17 GB", | 20260508 01:16:19.388 | INFO | "huge" : "", | 20260508 01:16:19.388 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:19.388 | INFO | "malloc" : " 14 GB", | 20260508 01:16:19.389 | INFO | "pinned" : " 29 GB" | 20260508 01:16:19.389 | INFO | } | 20260508 01:16:19.389 | INFO | Perf report for the last 5.13574 seconds: | 20260508 01:16:19.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:19.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:16:19.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 01:16:19.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:19.389 | INFO | Uploaded 36GiB to GPU in 2.51076s (14.3383GiB/s) | 20260508 01:16:19.389 | INFO | Downloaded 34GiB to GPU in 2.48023s (13.7084GiB/s) | 20260508 01:16:24.529 | INFO | Analyzing chunk frames = [4089856,4090368) | 20260508 01:16:24.529 | INFO | MemInfo: { | 20260508 01:16:24.529 | INFO | "gpu" : " 17 GB", | 20260508 01:16:24.529 | INFO | "huge" : "", | 20260508 01:16:24.529 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:24.529 | INFO | "malloc" : " 14 GB", | 20260508 01:16:24.529 | INFO | "pinned" : " 29 GB" | 20260508 01:16:24.529 | INFO | } | 20260508 01:16:24.529 | INFO | Perf report for the last 5.14052 seconds: | 20260508 01:16:24.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:24.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.529 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:16:24.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.529 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:16:24.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:24.530 | INFO | Uploaded 36GiB to GPU in 2.38123s (15.1182GiB/s) | 20260508 01:16:24.530 | INFO | Downloaded 34GiB to GPU in 2.50973s (13.5472GiB/s) | 20260508 01:16:29.671 | INFO | Analyzing chunk frames = [4090368,4090880) | 20260508 01:16:29.671 | INFO | MemInfo: { | 20260508 01:16:29.671 | INFO | "gpu" : " 17 GB", | 20260508 01:16:29.671 | INFO | "huge" : "", | 20260508 01:16:29.671 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:29.671 | INFO | "malloc" : " 14 GB", | 20260508 01:16:29.671 | INFO | "pinned" : " 29 GB" | 20260508 01:16:29.671 | INFO | } | 20260508 01:16:29.672 | INFO | Perf report for the last 5.14227 seconds: | 20260508 01:16:29.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:29.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:16:29.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260508 01:16:29.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:29.672 | INFO | Uploaded 36GiB to GPU in 2.40107s (14.9933GiB/s) | 20260508 01:16:29.672 | INFO | Downloaded 34GiB to GPU in 2.40388s (14.1438GiB/s) | 20260508 01:16:34.812 | INFO | Analyzing chunk frames = [4090880,4091392) | 20260508 01:16:34.812 | INFO | MemInfo: { | 20260508 01:16:34.812 | INFO | "gpu" : " 17 GB", | 20260508 01:16:34.812 | INFO | "huge" : "", | 20260508 01:16:34.812 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:34.813 | INFO | "malloc" : " 14 GB", | 20260508 01:16:34.813 | INFO | "pinned" : " 29 GB" | 20260508 01:16:34.813 | INFO | } | 20260508 01:16:34.813 | INFO | Perf report for the last 5.14136 seconds: | 20260508 01:16:34.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:34.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.813 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:16:34.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.813 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 01:16:34.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:34.814 | INFO | Uploaded 36GiB to GPU in 2.45964s (14.6363GiB/s) | 20260508 01:16:34.814 | INFO | Downloaded 34GiB to GPU in 2.4213s (14.042GiB/s) | 20260508 01:16:39.949 | INFO | Analyzing chunk frames = [4091392,4091904) | 20260508 01:16:39.949 | INFO | MemInfo: { | 20260508 01:16:39.949 | INFO | "gpu" : " 17 GB", | 20260508 01:16:39.949 | INFO | "huge" : "", | 20260508 01:16:39.949 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:39.949 | INFO | "malloc" : " 14 GB", | 20260508 01:16:39.950 | INFO | "pinned" : " 29 GB" | 20260508 01:16:39.950 | INFO | } | 20260508 01:16:39.950 | INFO | Perf report for the last 5.1367 seconds: | 20260508 01:16:39.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:39.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:16:39.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 01:16:39.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:39.950 | INFO | Uploaded 36GiB to GPU in 2.56735s (14.0222GiB/s) | 20260508 01:16:39.951 | INFO | Downloaded 34GiB to GPU in 2.45534s (13.8474GiB/s) | 20260508 01:16:40.062 | INFO | Starting a round of Baz IO | 20260508 01:16:43.523 | INFO | Populating RTMetrics report | 20260508 01:16:43.523 | INFO | RTMetrics report populated | 20260508 01:16:43.523 | INFO | Opening tmp RTMetrics json file | 20260508 01:16:43.545 | INFO | Serializing RTMetrics to json | 20260508 01:16:43.563 | INFO | Serialization done, swapping tmp file now... | 20260508 01:16:43.588 | INFO | RTMetrics done | 20260508 01:16:43.614 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.551 seconds | 20260508 01:16:43.614 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.89% / 0.00% / 58.10% | 20260508 01:16:43.614 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.884676% / 1.58946e-05% / 6.35783e-05% | 20260508 01:16:43.614 | INFO | Baz IO overall: | 20260508 01:16:43.614 | INFO | BytesWritten : 1855212420815 | 20260508 01:16:43.614 | INFO | NumEvents : 897513671334 | 20260508 01:16:45.086 | INFO | Analyzing chunk frames = [4091904,4092416) | 20260508 01:16:45.086 | INFO | MemInfo: { | 20260508 01:16:45.086 | INFO | "gpu" : " 17 GB", | 20260508 01:16:45.086 | INFO | "huge" : "", | 20260508 01:16:45.086 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:45.086 | INFO | "malloc" : " 14 GB", | 20260508 01:16:45.086 | INFO | "pinned" : " 29 GB" | 20260508 01:16:45.086 | INFO | } | 20260508 01:16:45.086 | INFO | Perf report for the last 5.13631 seconds: | 20260508 01:16:45.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:45.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:45.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:45.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:45.086 | INFO | ANALYSIS : 68.6%, 2.9, 104.8ms, 0 | 20260508 01:16:45.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:45.086 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 01:16:45.086 | INFO | PRE_HQ : 67.2%, 1.8, 61.2ms, 0 | 20260508 01:16:45.086 | INFO | BAZWRITER : 68.7%, 4.0, 142.3ms, 0 | 20260508 01:16:45.086 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 01:16:45.086 | INFO | Uploaded 36GiB to GPU in 2.58296s (13.9375GiB/s) | 20260508 01:16:45.086 | INFO | Downloaded 38GiB to GPU in 2.70415s (14.0525GiB/s) | 20260508 01:16:50.229 | INFO | Analyzing chunk frames = [4092416,4092928) | 20260508 01:16:50.229 | INFO | MemInfo: { | 20260508 01:16:50.229 | INFO | "gpu" : " 17 GB", | 20260508 01:16:50.229 | INFO | "huge" : "", | 20260508 01:16:50.229 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:50.229 | INFO | "malloc" : " 14 GB", | 20260508 01:16:50.229 | INFO | "pinned" : " 29 GB" | 20260508 01:16:50.230 | INFO | } | 20260508 01:16:50.230 | INFO | Perf report for the last 5.1436 seconds: | 20260508 01:16:50.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:50.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 01:16:50.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:16:50.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:50.230 | INFO | Uploaded 36GiB to GPU in 2.54748s (14.1316GiB/s) | 20260508 01:16:50.230 | INFO | Downloaded 34GiB to GPU in 2.50607s (13.5671GiB/s) | 20260508 01:16:55.368 | INFO | Analyzing chunk frames = [4092928,4093440) | 20260508 01:16:55.368 | INFO | MemInfo: { | 20260508 01:16:55.368 | INFO | "gpu" : " 17 GB", | 20260508 01:16:55.368 | INFO | "huge" : "", | 20260508 01:16:55.368 | INFO | "huge_shared" : " 29 GB", | 20260508 01:16:55.368 | INFO | "malloc" : " 14 GB", | 20260508 01:16:55.368 | INFO | "pinned" : " 29 GB" | 20260508 01:16:55.368 | INFO | } | 20260508 01:16:55.368 | INFO | Perf report for the last 5.13839 seconds: | 20260508 01:16:55.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:16:55.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 01:16:55.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:16:55.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:16:55.368 | INFO | Uploaded 36GiB to GPU in 2.50212s (14.3878GiB/s) | 20260508 01:16:55.368 | INFO | Downloaded 34GiB to GPU in 2.50898s (13.5513GiB/s) | 20260508 01:17:00.509 | INFO | Analyzing chunk frames = [4093440,4093952) | 20260508 01:17:00.509 | INFO | MemInfo: { | 20260508 01:17:00.509 | INFO | "gpu" : " 17 GB", | 20260508 01:17:00.509 | INFO | "huge" : "", | 20260508 01:17:00.509 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:00.509 | INFO | "malloc" : " 14 GB", | 20260508 01:17:00.509 | INFO | "pinned" : " 29 GB" | 20260508 01:17:00.509 | INFO | } | 20260508 01:17:00.509 | INFO | Perf report for the last 5.14102 seconds: | 20260508 01:17:00.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:00.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:17:00.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:17:00.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:00.509 | INFO | Uploaded 36GiB to GPU in 2.47942s (14.5195GiB/s) | 20260508 01:17:00.509 | INFO | Downloaded 34GiB to GPU in 2.50301s (13.5836GiB/s) | 20260508 01:17:05.646 | INFO | Analyzing chunk frames = [4093952,4094464) | 20260508 01:17:05.646 | INFO | MemInfo: { | 20260508 01:17:05.646 | INFO | "gpu" : " 17 GB", | 20260508 01:17:05.646 | INFO | "huge" : "", | 20260508 01:17:05.646 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:05.646 | INFO | "malloc" : " 14 GB", | 20260508 01:17:05.646 | INFO | "pinned" : " 29 GB" | 20260508 01:17:05.646 | INFO | } | 20260508 01:17:05.646 | INFO | Perf report for the last 5.13729 seconds: | 20260508 01:17:05.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:05.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.646 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 01:17:05.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.646 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:17:05.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:05.647 | INFO | Uploaded 36GiB to GPU in 2.29261s (15.7027GiB/s) | 20260508 01:17:05.647 | INFO | Downloaded 34GiB to GPU in 2.50452s (13.5755GiB/s) | 20260508 01:17:10.783 | INFO | Analyzing chunk frames = [4094464,4094976) | 20260508 01:17:10.783 | INFO | MemInfo: { | 20260508 01:17:10.783 | INFO | "gpu" : " 17 GB", | 20260508 01:17:10.783 | INFO | "huge" : "", | 20260508 01:17:10.783 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:10.783 | INFO | "malloc" : " 14 GB", | 20260508 01:17:10.783 | INFO | "pinned" : " 29 GB" | 20260508 01:17:10.783 | INFO | } | 20260508 01:17:10.783 | INFO | Perf report for the last 5.13713 seconds: | 20260508 01:17:10.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:10.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.783 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:17:10.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.784 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:17:10.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:10.784 | INFO | Uploaded 36GiB to GPU in 2.36469s (15.224GiB/s) | 20260508 01:17:10.784 | INFO | Downloaded 34GiB to GPU in 2.43391s (13.9693GiB/s) | 20260508 01:17:15.926 | INFO | Analyzing chunk frames = [4094976,4095488) | 20260508 01:17:15.926 | INFO | MemInfo: { | 20260508 01:17:15.926 | INFO | "gpu" : " 17 GB", | 20260508 01:17:15.926 | INFO | "huge" : "", | 20260508 01:17:15.926 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:15.926 | INFO | "malloc" : " 14 GB", | 20260508 01:17:15.927 | INFO | "pinned" : " 29 GB" | 20260508 01:17:15.927 | INFO | } | 20260508 01:17:15.927 | INFO | Perf report for the last 5.14316 seconds: | 20260508 01:17:15.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:15.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:17:15.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 01:17:15.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:15.927 | INFO | Uploaded 36GiB to GPU in 2.53279s (14.2136GiB/s) | 20260508 01:17:15.927 | INFO | Downloaded 34GiB to GPU in 2.48243s (13.6963GiB/s) | 20260508 01:17:21.063 | INFO | Analyzing chunk frames = [4095488,4096000) | 20260508 01:17:21.063 | INFO | MemInfo: { | 20260508 01:17:21.063 | INFO | "gpu" : " 17 GB", | 20260508 01:17:21.063 | INFO | "huge" : "", | 20260508 01:17:21.063 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:21.063 | INFO | "malloc" : " 14 GB", | 20260508 01:17:21.063 | INFO | "pinned" : " 29 GB" | 20260508 01:17:21.063 | INFO | } | 20260508 01:17:21.063 | INFO | Perf report for the last 5.13681 seconds: | 20260508 01:17:21.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:21.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.063 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 01:17:21.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.064 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:17:21.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:21.064 | INFO | Uploaded 36GiB to GPU in 2.51912s (14.2907GiB/s) | 20260508 01:17:21.064 | INFO | Downloaded 34GiB to GPU in 2.46625s (13.7861GiB/s) | 20260508 01:17:21.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:17:21.064 | INFO | Total mem usage: 16.7 GiB | 20260508 01:17:21.064 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:17:21.064 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260508 01:17:21.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:17:21.064 | INFO | High Water Breakdown: | 20260508 01:17:21.064 | INFO | 59.9% : 8.81 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:17:21.064 | INFO | 40% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:17:21.064 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:17:21.064 | INFO | ------------------------------------------ | 20260508 01:17:21.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:17:21.064 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:17:21.064 | INFO | Total mem usage: 26.5 GiB | 20260508 01:17:21.064 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:17:21.064 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 01:17:21.064 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:17:21.064 | INFO | High Water Breakdown: | 20260508 01:17:21.064 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:17:21.064 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:17:21.064 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:17:21.064 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:17:21.064 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:17:21.064 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:17:21.064 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:17:21.064 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:17:21.064 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:17:21.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:17:21.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:17:21.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:17:21.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:17:21.064 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:17:21.064 | INFO | ------------------------------------------ | 20260508 01:17:21.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:17:21.064 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:17:21.064 | INFO | Total mem usage: 27.8 GiB | 20260508 01:17:21.064 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:17:21.064 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:17:21.064 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:17:21.064 | INFO | High Water Breakdown: | 20260508 01:17:21.064 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:17:21.064 | INFO | ------------------------------------------ | 20260508 01:17:21.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:17:21.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:17:21.064 | INFO | Total mem usage: 15.7 GiB | 20260508 01:17:21.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:17:21.064 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:17:21.064 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:17:21.064 | INFO | High Water Breakdown: | 20260508 01:17:21.064 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:17:21.064 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:17:21.064 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:17:21.064 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:17:21.064 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:17:21.064 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:17:21.064 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:17:21.064 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:17:21.064 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:17:21.064 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:17:21.064 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:17:21.064 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:17:21.064 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:17:21.064 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:17:21.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:17:21.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:17:21.064 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:17:21.064 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:17:21.064 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:17:21.064 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:17:21.064 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:17:21.064 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:17:21.064 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:17:21.064 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:17:21.064 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:17:21.064 | INFO | ------------------------------------------ | 20260508 01:17:21.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:17:21.064 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:17:21.064 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:17:21.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:17:21.064 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.19e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.01e-01, 1.19e-02 / 6.22e-03 | 20260508 01:17:21.064 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.49e+00, 5.87e-01 / 1.44e-01 | 20260508 01:17:21.064 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.51e+00, 2.07e-01 / 7.86e-02 | 20260508 01:17:21.064 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.62e+01, 3.83e+00 / 1.39e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.84e+01 / 1.93e+01, 1.55e+00 / 1.47e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.45e+00 / 2.76e+00, 4.29e-01 / 2.20e-01 | 20260508 01:17:21.064 | INFO | PerfReport: 3.37%, Metrics -- 1.10e+00 / 1.29e+00, 2.74e-01 / 1.15e-01 | 20260508 01:17:21.064 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.39e-01 / 3.79e-01, 5.40e-02 / 2.81e-02 | 20260508 01:17:21.064 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:17:21.064 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:17:21.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:17:21.064 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.41e+00 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 2.06%, LockUpload -- 2.20e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 2.92%, LockDownload -- 3.11e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:17:21.064 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260508 01:17:21.161 | INFO | Starting a round of Baz IO | 20260508 01:17:24.617 | INFO | Populating RTMetrics report | 20260508 01:17:24.617 | INFO | RTMetrics report populated | 20260508 01:17:24.617 | INFO | Opening tmp RTMetrics json file | 20260508 01:17:24.709 | INFO | Serializing RTMetrics to json | 20260508 01:17:24.717 | INFO | Serialization done, swapping tmp file now... | 20260508 01:17:24.719 | INFO | RTMetrics done | 20260508 01:17:24.734 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.572 seconds | 20260508 01:17:24.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.89% / 0.00% / 58.11% | 20260508 01:17:24.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.884712% / 0% / 5.16574e-05% | 20260508 01:17:24.734 | INFO | Baz IO overall: | 20260508 01:17:24.734 | INFO | BytesWritten : 1857006354355 | 20260508 01:17:24.734 | INFO | NumEvents : 898376107991 | 20260508 01:17:26.202 | INFO | Analyzing chunk frames = [4096000,4096512) | 20260508 01:17:26.202 | INFO | MemInfo: { | 20260508 01:17:26.202 | INFO | "gpu" : " 17 GB", | 20260508 01:17:26.202 | INFO | "huge" : "", | 20260508 01:17:26.202 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:26.202 | INFO | "malloc" : " 14 GB", | 20260508 01:17:26.202 | INFO | "pinned" : " 29 GB" | 20260508 01:17:26.202 | INFO | } | 20260508 01:17:26.202 | INFO | Perf report for the last 5.13891 seconds: | 20260508 01:17:26.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:26.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:26.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:26.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:26.203 | INFO | ANALYSIS : 68.2%, 3.0, 106.5ms, 0 | 20260508 01:17:26.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:26.203 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260508 01:17:26.203 | INFO | PRE_HQ : 65.2%, 1.7, 56.2ms, 0 | 20260508 01:17:26.203 | INFO | BAZWRITER : 69.1%, 3.6, 129.4ms, 0 | 20260508 01:17:26.203 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 01:17:26.203 | INFO | Uploaded 36GiB to GPU in 2.49479s (14.43GiB/s) | 20260508 01:17:26.203 | INFO | Downloaded 38GiB to GPU in 2.6848s (14.1537GiB/s) | 20260508 01:17:31.337 | INFO | Analyzing chunk frames = [4096512,4097024) | 20260508 01:17:31.337 | INFO | MemInfo: { | 20260508 01:17:31.337 | INFO | "gpu" : " 17 GB", | 20260508 01:17:31.337 | INFO | "huge" : "", | 20260508 01:17:31.337 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:31.337 | INFO | "malloc" : " 14 GB", | 20260508 01:17:31.337 | INFO | "pinned" : " 29 GB" | 20260508 01:17:31.337 | INFO | } | 20260508 01:17:31.337 | INFO | Perf report for the last 5.1349 seconds: | 20260508 01:17:31.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:31.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.337 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 01:17:31.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.337 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:17:31.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:31.338 | INFO | Uploaded 36GiB to GPU in 2.49047s (14.4551GiB/s) | 20260508 01:17:31.338 | INFO | Downloaded 34GiB to GPU in 2.44281s (13.9184GiB/s) | 20260508 01:17:36.481 | INFO | Analyzing chunk frames = [4097024,4097536) | 20260508 01:17:36.481 | INFO | MemInfo: { | 20260508 01:17:36.481 | INFO | "gpu" : " 17 GB", | 20260508 01:17:36.481 | INFO | "huge" : "", | 20260508 01:17:36.481 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:36.481 | INFO | "malloc" : " 14 GB", | 20260508 01:17:36.482 | INFO | "pinned" : " 29 GB" | 20260508 01:17:36.482 | INFO | } | 20260508 01:17:36.482 | INFO | Perf report for the last 5.1443 seconds: | 20260508 01:17:36.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:36.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 01:17:36.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:17:36.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:36.482 | INFO | Uploaded 36GiB to GPU in 2.46049s (14.6312GiB/s) | 20260508 01:17:36.482 | INFO | Downloaded 34GiB to GPU in 2.50537s (13.5708GiB/s) | 20260508 01:17:41.624 | INFO | Analyzing chunk frames = [4097536,4098048) | 20260508 01:17:41.624 | INFO | MemInfo: { | 20260508 01:17:41.624 | INFO | "gpu" : " 17 GB", | 20260508 01:17:41.624 | INFO | "huge" : "", | 20260508 01:17:41.624 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:41.624 | INFO | "malloc" : " 14 GB", | 20260508 01:17:41.624 | INFO | "pinned" : " 29 GB" | 20260508 01:17:41.624 | INFO | } | 20260508 01:17:41.624 | INFO | Perf report for the last 5.14266 seconds: | 20260508 01:17:41.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:41.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.624 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:17:41.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.624 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:17:41.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:41.625 | INFO | Uploaded 36GiB to GPU in 2.47443s (14.5488GiB/s) | 20260508 01:17:41.625 | INFO | Downloaded 34GiB to GPU in 2.5027s (13.5853GiB/s) | 20260508 01:17:46.767 | INFO | Analyzing chunk frames = [4098048,4098560) | 20260508 01:17:46.767 | INFO | MemInfo: { | 20260508 01:17:46.767 | INFO | "gpu" : " 17 GB", | 20260508 01:17:46.767 | INFO | "huge" : "", | 20260508 01:17:46.767 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:46.768 | INFO | "malloc" : " 14 GB", | 20260508 01:17:46.768 | INFO | "pinned" : " 29 GB" | 20260508 01:17:46.768 | INFO | } | 20260508 01:17:46.768 | INFO | Perf report for the last 5.14337 seconds: | 20260508 01:17:46.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:46.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 01:17:46.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:17:46.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:46.768 | INFO | Uploaded 36GiB to GPU in 2.31973s (15.5191GiB/s) | 20260508 01:17:46.768 | INFO | Downloaded 34GiB to GPU in 2.51043s (13.5435GiB/s) | 20260508 01:17:51.904 | INFO | Analyzing chunk frames = [4098560,4099072) | 20260508 01:17:51.904 | INFO | MemInfo: { | 20260508 01:17:51.904 | INFO | "gpu" : " 17 GB", | 20260508 01:17:51.904 | INFO | "huge" : "", | 20260508 01:17:51.904 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:51.904 | INFO | "malloc" : " 14 GB", | 20260508 01:17:51.904 | INFO | "pinned" : " 29 GB" | 20260508 01:17:51.904 | INFO | } | 20260508 01:17:51.904 | INFO | Perf report for the last 5.13638 seconds: | 20260508 01:17:51.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:51.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:17:51.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260508 01:17:51.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:51.904 | INFO | Uploaded 36GiB to GPU in 2.37042s (15.1872GiB/s) | 20260508 01:17:51.905 | INFO | Downloaded 34GiB to GPU in 2.44095s (13.929GiB/s) | 20260508 01:17:57.037 | INFO | Analyzing chunk frames = [4099072,4099584) | 20260508 01:17:57.037 | INFO | MemInfo: { | 20260508 01:17:57.037 | INFO | "gpu" : " 17 GB", | 20260508 01:17:57.037 | INFO | "huge" : "", | 20260508 01:17:57.037 | INFO | "huge_shared" : " 29 GB", | 20260508 01:17:57.037 | INFO | "malloc" : " 14 GB", | 20260508 01:17:57.037 | INFO | "pinned" : " 29 GB" | 20260508 01:17:57.037 | INFO | } | 20260508 01:17:57.037 | INFO | Perf report for the last 5.13284 seconds: | 20260508 01:17:57.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:17:57.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 01:17:57.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:17:57.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:17:57.037 | INFO | Uploaded 36GiB to GPU in 2.41609s (14.9001GiB/s) | 20260508 01:17:57.037 | INFO | Downloaded 34GiB to GPU in 2.43949s (13.9373GiB/s) | 20260508 01:18:02.179 | INFO | Analyzing chunk frames = [4099584,4100096) | 20260508 01:18:02.179 | INFO | MemInfo: { | 20260508 01:18:02.179 | INFO | "gpu" : " 17 GB", | 20260508 01:18:02.179 | INFO | "huge" : "", | 20260508 01:18:02.179 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:02.179 | INFO | "malloc" : " 14 GB", | 20260508 01:18:02.179 | INFO | "pinned" : " 29 GB" | 20260508 01:18:02.179 | INFO | } | 20260508 01:18:02.179 | INFO | Perf report for the last 5.14203 seconds: | 20260508 01:18:02.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:02.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.179 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:18:02.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.179 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:18:02.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:02.180 | INFO | Uploaded 36GiB to GPU in 2.4912s (14.4508GiB/s) | 20260508 01:18:02.180 | INFO | Downloaded 34GiB to GPU in 2.4618s (13.811GiB/s) | 20260508 01:18:02.276 | INFO | Starting a round of Baz IO | 20260508 01:18:05.728 | INFO | Populating RTMetrics report | 20260508 01:18:05.728 | INFO | RTMetrics report populated | 20260508 01:18:05.728 | INFO | Opening tmp RTMetrics json file | 20260508 01:18:05.730 | INFO | Serializing RTMetrics to json | 20260508 01:18:05.737 | INFO | Serialization done, swapping tmp file now... | 20260508 01:18:05.738 | INFO | RTMetrics done | 20260508 01:18:05.854 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.578 seconds | 20260508 01:18:05.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.89% / 0.00% / 58.11% | 20260508 01:18:05.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.883679% / 1.98682e-05% / 5.5631e-05% | 20260508 01:18:05.854 | INFO | Baz IO overall: | 20260508 01:18:05.854 | INFO | BytesWritten : 1858799645182 | 20260508 01:18:05.854 | INFO | NumEvents : 899237934545 | 20260508 01:18:07.319 | INFO | Analyzing chunk frames = [4100096,4100608) | 20260508 01:18:07.319 | INFO | MemInfo: { | 20260508 01:18:07.319 | INFO | "gpu" : " 17 GB", | 20260508 01:18:07.319 | INFO | "huge" : "", | 20260508 01:18:07.319 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:07.319 | INFO | "malloc" : " 14 GB", | 20260508 01:18:07.319 | INFO | "pinned" : " 29 GB" | 20260508 01:18:07.319 | INFO | } | 20260508 01:18:07.319 | INFO | Perf report for the last 5.13989 seconds: | 20260508 01:18:07.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:07.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:07.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:07.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:07.319 | INFO | ANALYSIS : 68.6%, 3.0, 106.2ms, 0 | 20260508 01:18:07.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:07.319 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 01:18:07.319 | INFO | PRE_HQ : 66.2%, 1.6, 56.4ms, 0 | 20260508 01:18:07.319 | INFO | BAZWRITER : 69.2%, 3.8, 135.5ms, 0 | 20260508 01:18:07.319 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 01:18:07.319 | INFO | Uploaded 36GiB to GPU in 2.5043s (14.3753GiB/s) | 20260508 01:18:07.319 | INFO | Downloaded 38GiB to GPU in 2.69243s (14.1136GiB/s) | 20260508 01:18:12.458 | INFO | Analyzing chunk frames = [4100608,4101120) | 20260508 01:18:12.458 | INFO | MemInfo: { | 20260508 01:18:12.458 | INFO | "gpu" : " 17 GB", | 20260508 01:18:12.458 | INFO | "huge" : "", | 20260508 01:18:12.458 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:12.458 | INFO | "malloc" : " 14 GB", | 20260508 01:18:12.459 | INFO | "pinned" : " 29 GB" | 20260508 01:18:12.459 | INFO | } | 20260508 01:18:12.459 | INFO | Perf report for the last 5.13976 seconds: | 20260508 01:18:12.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:12.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:18:12.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:18:12.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:12.459 | INFO | Uploaded 36GiB to GPU in 2.49834s (14.4095GiB/s) | 20260508 01:18:12.459 | INFO | Downloaded 34GiB to GPU in 2.43295s (13.9748GiB/s) | 20260508 01:18:17.596 | INFO | Analyzing chunk frames = [4101120,4101632) | 20260508 01:18:17.596 | INFO | MemInfo: { | 20260508 01:18:17.596 | INFO | "gpu" : " 17 GB", | 20260508 01:18:17.596 | INFO | "huge" : "", | 20260508 01:18:17.596 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:17.596 | INFO | "malloc" : " 14 GB", | 20260508 01:18:17.596 | INFO | "pinned" : " 29 GB" | 20260508 01:18:17.596 | INFO | } | 20260508 01:18:17.596 | INFO | Perf report for the last 5.13759 seconds: | 20260508 01:18:17.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:17.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 01:18:17.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:18:17.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:17.596 | INFO | Uploaded 36GiB to GPU in 2.47167s (14.565GiB/s) | 20260508 01:18:17.596 | INFO | Downloaded 34GiB to GPU in 2.50238s (13.587GiB/s) | 20260508 01:18:22.735 | INFO | Analyzing chunk frames = [4101632,4102144) | 20260508 01:18:22.735 | INFO | MemInfo: { | 20260508 01:18:22.735 | INFO | "gpu" : " 17 GB", | 20260508 01:18:22.735 | INFO | "huge" : "", | 20260508 01:18:22.735 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:22.735 | INFO | "malloc" : " 14 GB", | 20260508 01:18:22.735 | INFO | "pinned" : " 29 GB" | 20260508 01:18:22.735 | INFO | } | 20260508 01:18:22.735 | INFO | Perf report for the last 5.13858 seconds: | 20260508 01:18:22.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:22.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:18:22.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:18:22.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:22.735 | INFO | Uploaded 36GiB to GPU in 2.47406s (14.551GiB/s) | 20260508 01:18:22.735 | INFO | Downloaded 34GiB to GPU in 2.50471s (13.5744GiB/s) | 20260508 01:18:27.874 | INFO | Analyzing chunk frames = [4102144,4102656) | 20260508 01:18:27.874 | INFO | MemInfo: { | 20260508 01:18:27.874 | INFO | "gpu" : " 17 GB", | 20260508 01:18:27.874 | INFO | "huge" : "", | 20260508 01:18:27.874 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:27.874 | INFO | "malloc" : " 14 GB", | 20260508 01:18:27.874 | INFO | "pinned" : " 29 GB" | 20260508 01:18:27.874 | INFO | } | 20260508 01:18:27.874 | INFO | Perf report for the last 5.13951 seconds: | 20260508 01:18:27.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:27.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.874 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:18:27.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.875 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 01:18:27.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:27.875 | INFO | Uploaded 36GiB to GPU in 2.27536s (15.8217GiB/s) | 20260508 01:18:27.875 | INFO | Downloaded 34GiB to GPU in 2.50444s (13.5759GiB/s) | 20260508 01:18:33.014 | INFO | Analyzing chunk frames = [4102656,4103168) | 20260508 01:18:33.014 | INFO | MemInfo: { | 20260508 01:18:33.014 | INFO | "gpu" : " 17 GB", | 20260508 01:18:33.014 | INFO | "huge" : "", | 20260508 01:18:33.014 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:33.014 | INFO | "malloc" : " 14 GB", | 20260508 01:18:33.014 | INFO | "pinned" : " 29 GB" | 20260508 01:18:33.014 | INFO | } | 20260508 01:18:33.014 | INFO | Perf report for the last 5.14014 seconds: | 20260508 01:18:33.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:33.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.015 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 01:18:33.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.015 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:18:33.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:33.015 | INFO | Uploaded 36GiB to GPU in 2.40859s (14.9465GiB/s) | 20260508 01:18:33.015 | INFO | Downloaded 34GiB to GPU in 2.40954s (14.1106GiB/s) | 20260508 01:18:38.153 | INFO | Analyzing chunk frames = [4103168,4103680) | 20260508 01:18:38.153 | INFO | MemInfo: { | 20260508 01:18:38.153 | INFO | "gpu" : " 17 GB", | 20260508 01:18:38.153 | INFO | "huge" : "", | 20260508 01:18:38.153 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:38.153 | INFO | "malloc" : " 14 GB", | 20260508 01:18:38.153 | INFO | "pinned" : " 29 GB" | 20260508 01:18:38.153 | INFO | } | 20260508 01:18:38.153 | INFO | Perf report for the last 5.1389 seconds: | 20260508 01:18:38.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:38.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.153 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 01:18:38.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.153 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:18:38.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:38.154 | INFO | Uploaded 36GiB to GPU in 2.52414s (14.2623GiB/s) | 20260508 01:18:38.154 | INFO | Downloaded 34GiB to GPU in 2.45522s (13.848GiB/s) | 20260508 01:18:43.290 | INFO | Analyzing chunk frames = [4103680,4104192) | 20260508 01:18:43.290 | INFO | MemInfo: { | 20260508 01:18:43.290 | INFO | "gpu" : " 17 GB", | 20260508 01:18:43.290 | INFO | "huge" : "", | 20260508 01:18:43.290 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:43.290 | INFO | "malloc" : " 14 GB", | 20260508 01:18:43.290 | INFO | "pinned" : " 29 GB" | 20260508 01:18:43.290 | INFO | } | 20260508 01:18:43.290 | INFO | Perf report for the last 5.13633 seconds: | 20260508 01:18:43.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:43.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:18:43.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:18:43.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:43.290 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:18:43.290 | INFO | Total mem usage: 16.7 GiB | 20260508 01:18:43.290 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:18:43.290 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.9%) | 20260508 01:18:43.290 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:18:43.290 | INFO | High Water Breakdown: | 20260508 01:18:43.290 | INFO | 60.3% : 9.07 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:18:43.290 | INFO | 39.6% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:18:43.290 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:18:43.290 | INFO | ------------------------------------------ | 20260508 01:18:43.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:18:43.290 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:18:43.290 | INFO | Uploaded 36GiB to GPU in 2.53607s (14.1952GiB/s) | 20260508 01:18:43.290 | INFO | Total mem usage: 26.5 GiB | 20260508 01:18:43.290 | INFO | Downloaded 34GiB to GPU in 2.45475s (13.8507GiB/s) | 20260508 01:18:43.290 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:18:43.290 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:18:43.290 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260508 01:18:43.291 | INFO | High Water Breakdown: | 20260508 01:18:43.291 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:18:43.291 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:18:43.291 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:18:43.291 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:18:43.291 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:18:43.291 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:18:43.291 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:18:43.291 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:18:43.291 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:18:43.291 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:18:43.291 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:18:43.291 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:18:43.291 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:18:43.291 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:18:43.291 | INFO | ------------------------------------------ | 20260508 01:18:43.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:18:43.291 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:18:43.291 | INFO | Total mem usage: 27.8 GiB | 20260508 01:18:43.291 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:18:43.291 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:18:43.291 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:18:43.291 | INFO | High Water Breakdown: | 20260508 01:18:43.291 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:18:43.291 | INFO | ------------------------------------------ | 20260508 01:18:43.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:18:43.291 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:18:43.291 | INFO | Total mem usage: 15.7 GiB | 20260508 01:18:43.291 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:18:43.291 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:18:43.291 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:18:43.291 | INFO | High Water Breakdown: | 20260508 01:18:43.291 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:18:43.291 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:18:43.291 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:18:43.291 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:18:43.291 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:18:43.291 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:18:43.291 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:18:43.291 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:18:43.291 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:18:43.291 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:18:43.291 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:18:43.291 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:18:43.291 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:18:43.291 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:18:43.291 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:18:43.291 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:18:43.291 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:18:43.291 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:18:43.291 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:18:43.291 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:18:43.291 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:18:43.291 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:18:43.291 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:18:43.291 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:18:43.291 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:18:43.291 | INFO | ------------------------------------------ | 20260508 01:18:43.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:18:43.291 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:18:43.291 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:18:43.291 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:18:43.291 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.40e+01 / 1.34e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.25e-02 / 6.40e-03 | 20260508 01:18:43.291 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.45e+00, 5.90e-01 / 2.02e-01 | 20260508 01:18:43.291 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.05e-01 | 20260508 01:18:43.291 | INFO | PerfReport: 16.04%, DME -- 4.21e+01 / 4.60e+01, 4.00e+00 / 1.69e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.68e+00 / 1.61e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.44e+00 / 2.68e+00, 4.27e-01 / 2.65e-01 | 20260508 01:18:43.291 | INFO | PerfReport: 3.35%, Metrics -- 1.10e+00 / 1.32e+00, 2.71e-01 / 1.43e-01 | 20260508 01:18:43.291 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.37e-01 / 3.76e-01, 5.32e-02 / 2.59e-02 | 20260508 01:18:43.291 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:18:43.291 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:18:43.291 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:18:43.291 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 24.00%, Upload -- 2.55e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 24.45%, Download -- 2.60e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 30.81%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 1.89%, LockUpload -- 2.01e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 3.08%, LockDownload -- 3.28e+00 / 0.00e+00, 4.57e+00 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 15.76%, LockCompute -- 1.68e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:18:43.291 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 01:18:43.389 | INFO | Starting a round of Baz IO | 20260508 01:18:46.840 | INFO | Populating RTMetrics report | 20260508 01:18:46.841 | INFO | RTMetrics report populated | 20260508 01:18:46.841 | INFO | Opening tmp RTMetrics json file | 20260508 01:18:46.916 | INFO | Serializing RTMetrics to json | 20260508 01:18:46.947 | INFO | Serialization done, swapping tmp file now... | 20260508 01:18:46.952 | INFO | RTMetrics done | 20260508 01:18:46.978 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.588 seconds | 20260508 01:18:46.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.88% / 0.00% / 58.11% | 20260508 01:18:46.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.879316% / 0% / 4.76837e-05% | 20260508 01:18:46.978 | INFO | Baz IO overall: | 20260508 01:18:46.978 | INFO | BytesWritten : 1860591902076 | 20260508 01:18:46.978 | INFO | NumEvents : 900099074517 | 20260508 01:18:48.432 | INFO | Analyzing chunk frames = [4104192,4104704) | 20260508 01:18:48.432 | INFO | MemInfo: { | 20260508 01:18:48.432 | INFO | "gpu" : " 17 GB", | 20260508 01:18:48.432 | INFO | "huge" : "", | 20260508 01:18:48.432 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:48.432 | INFO | "malloc" : " 14 GB", | 20260508 01:18:48.432 | INFO | "pinned" : " 29 GB" | 20260508 01:18:48.432 | INFO | } | 20260508 01:18:48.432 | INFO | Perf report for the last 5.14199 seconds: | 20260508 01:18:48.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:48.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:48.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:48.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:48.432 | INFO | ANALYSIS : 68.4%, 3.0, 106.7ms, 0 | 20260508 01:18:48.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:48.432 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260508 01:18:48.432 | INFO | PRE_HQ : 65.4%, 1.7, 56.5ms, 0 | 20260508 01:18:48.432 | INFO | BAZWRITER : 69.4%, 3.7, 133.0ms, 0 | 20260508 01:18:48.432 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260508 01:18:48.432 | INFO | Uploaded 36GiB to GPU in 2.58827s (13.9089GiB/s) | 20260508 01:18:48.432 | INFO | Downloaded 38GiB to GPU in 2.68245s (14.1661GiB/s) | 20260508 01:18:53.576 | INFO | Analyzing chunk frames = [4104704,4105216) | 20260508 01:18:53.576 | INFO | MemInfo: { | 20260508 01:18:53.576 | INFO | "gpu" : " 17 GB", | 20260508 01:18:53.576 | INFO | "huge" : "", | 20260508 01:18:53.576 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:53.576 | INFO | "malloc" : " 14 GB", | 20260508 01:18:53.576 | INFO | "pinned" : " 29 GB" | 20260508 01:18:53.576 | INFO | } | 20260508 01:18:53.576 | INFO | Perf report for the last 5.1443 seconds: | 20260508 01:18:53.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:53.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.576 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:18:53.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.576 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:18:53.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:53.577 | INFO | Uploaded 36GiB to GPU in 2.54771s (14.1304GiB/s) | 20260508 01:18:53.577 | INFO | Downloaded 34GiB to GPU in 2.41417s (14.0835GiB/s) | 20260508 01:18:58.714 | INFO | Analyzing chunk frames = [4105216,4105728) | 20260508 01:18:58.714 | INFO | MemInfo: { | 20260508 01:18:58.714 | INFO | "gpu" : " 17 GB", | 20260508 01:18:58.714 | INFO | "huge" : "", | 20260508 01:18:58.715 | INFO | "huge_shared" : " 29 GB", | 20260508 01:18:58.715 | INFO | "malloc" : " 14 GB", | 20260508 01:18:58.715 | INFO | "pinned" : " 29 GB" | 20260508 01:18:58.715 | INFO | } | 20260508 01:18:58.715 | INFO | Perf report for the last 5.13869 seconds: | 20260508 01:18:58.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:18:58.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 01:18:58.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 01:18:58.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:18:58.715 | INFO | Uploaded 36GiB to GPU in 2.5823s (13.9411GiB/s) | 20260508 01:18:58.715 | INFO | Downloaded 34GiB to GPU in 2.48497s (13.6822GiB/s) | 20260508 01:19:03.851 | INFO | Analyzing chunk frames = [4105728,4106240) | 20260508 01:19:03.851 | INFO | MemInfo: { | 20260508 01:19:03.851 | INFO | "gpu" : " 17 GB", | 20260508 01:19:03.851 | INFO | "huge" : "", | 20260508 01:19:03.851 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:03.851 | INFO | "malloc" : " 14 GB", | 20260508 01:19:03.851 | INFO | "pinned" : " 29 GB" | 20260508 01:19:03.851 | INFO | } | 20260508 01:19:03.851 | INFO | Perf report for the last 5.13661 seconds: | 20260508 01:19:03.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:03.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.851 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:19:03.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.851 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:19:03.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:03.852 | INFO | Uploaded 36GiB to GPU in 2.56684s (14.025GiB/s) | 20260508 01:19:03.852 | INFO | Downloaded 34GiB to GPU in 2.48464s (13.6841GiB/s) | 20260508 01:19:08.992 | INFO | Analyzing chunk frames = [4106240,4106752) | 20260508 01:19:08.992 | INFO | MemInfo: { | 20260508 01:19:08.992 | INFO | "gpu" : " 17 GB", | 20260508 01:19:08.992 | INFO | "huge" : "", | 20260508 01:19:08.992 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:08.992 | INFO | "malloc" : " 14 GB", | 20260508 01:19:08.992 | INFO | "pinned" : " 29 GB" | 20260508 01:19:08.992 | INFO | } | 20260508 01:19:08.992 | INFO | Perf report for the last 5.14049 seconds: | 20260508 01:19:08.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:08.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.992 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 01:19:08.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.992 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 01:19:08.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:08.993 | INFO | Uploaded 36GiB to GPU in 2.39847s (15.0096GiB/s) | 20260508 01:19:08.993 | INFO | Downloaded 34GiB to GPU in 2.49815s (13.6101GiB/s) | 20260508 01:19:14.132 | INFO | Analyzing chunk frames = [4106752,4107264) | 20260508 01:19:14.132 | INFO | MemInfo: { | 20260508 01:19:14.132 | INFO | "gpu" : " 17 GB", | 20260508 01:19:14.132 | INFO | "huge" : "", | 20260508 01:19:14.132 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:14.132 | INFO | "malloc" : " 14 GB", | 20260508 01:19:14.132 | INFO | "pinned" : " 29 GB" | 20260508 01:19:14.132 | INFO | } | 20260508 01:19:14.132 | INFO | Perf report for the last 5.14014 seconds: | 20260508 01:19:14.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:14.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.132 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:19:14.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.132 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:19:14.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:14.133 | INFO | Uploaded 36GiB to GPU in 2.40062s (14.9961GiB/s) | 20260508 01:19:14.133 | INFO | Downloaded 34GiB to GPU in 2.40699s (14.1256GiB/s) | 20260508 01:19:19.267 | INFO | Analyzing chunk frames = [4107264,4107776) | 20260508 01:19:19.267 | INFO | MemInfo: { | 20260508 01:19:19.267 | INFO | "gpu" : " 17 GB", | 20260508 01:19:19.267 | INFO | "huge" : "", | 20260508 01:19:19.267 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:19.267 | INFO | "malloc" : " 14 GB", | 20260508 01:19:19.267 | INFO | "pinned" : " 29 GB" | 20260508 01:19:19.267 | INFO | } | 20260508 01:19:19.268 | INFO | Perf report for the last 5.1355 seconds: | 20260508 01:19:19.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:19.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:19:19.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:19:19.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:19.268 | INFO | Uploaded 36GiB to GPU in 2.53165s (14.22GiB/s) | 20260508 01:19:19.268 | INFO | Downloaded 34GiB to GPU in 2.45359s (13.8572GiB/s) | 20260508 01:19:24.407 | INFO | Analyzing chunk frames = [4107776,4108288) | 20260508 01:19:24.407 | INFO | MemInfo: { | 20260508 01:19:24.407 | INFO | "gpu" : " 17 GB", | 20260508 01:19:24.407 | INFO | "huge" : "", | 20260508 01:19:24.407 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:24.407 | INFO | "malloc" : " 14 GB", | 20260508 01:19:24.407 | INFO | "pinned" : " 29 GB" | 20260508 01:19:24.407 | INFO | } | 20260508 01:19:24.407 | INFO | Perf report for the last 5.13927 seconds: | 20260508 01:19:24.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:24.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:19:24.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:19:24.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:24.407 | INFO | Uploaded 36GiB to GPU in 2.54798s (14.1288GiB/s) | 20260508 01:19:24.407 | INFO | Downloaded 34GiB to GPU in 2.45007s (13.8772GiB/s) | 20260508 01:19:24.504 | INFO | Starting a round of Baz IO | 20260508 01:19:27.943 | INFO | Populating RTMetrics report | 20260508 01:19:27.944 | INFO | RTMetrics report populated | 20260508 01:19:27.944 | INFO | Opening tmp RTMetrics json file | 20260508 01:19:27.953 | INFO | Serializing RTMetrics to json | 20260508 01:19:27.970 | INFO | Serialization done, swapping tmp file now... | 20260508 01:19:27.971 | INFO | RTMetrics done | 20260508 01:19:28.043 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.538 seconds | 20260508 01:19:28.043 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.88% / 0.00% / 58.12% | 20260508 01:19:28.043 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.877762% / 1.98682e-05% / 7.15256e-05% | 20260508 01:19:28.043 | INFO | Baz IO overall: | 20260508 01:19:28.043 | INFO | BytesWritten : 1862383743225 | 20260508 01:19:28.043 | INFO | NumEvents : 900960184858 | 20260508 01:19:29.544 | INFO | Analyzing chunk frames = [4108288,4108800) | 20260508 01:19:29.544 | INFO | MemInfo: { | 20260508 01:19:29.544 | INFO | "gpu" : " 17 GB", | 20260508 01:19:29.544 | INFO | "huge" : "", | 20260508 01:19:29.544 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:29.544 | INFO | "malloc" : " 14 GB", | 20260508 01:19:29.544 | INFO | "pinned" : " 29 GB" | 20260508 01:19:29.544 | INFO | } | 20260508 01:19:29.544 | INFO | Perf report for the last 5.13723 seconds: | 20260508 01:19:29.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:29.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:29.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:29.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:29.544 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260508 01:19:29.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:29.544 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 01:19:29.544 | INFO | PRE_HQ : 66.0%, 1.7, 56.6ms, 0 | 20260508 01:19:29.544 | INFO | BAZWRITER : 68.5%, 4.4, 155.3ms, 0 | 20260508 01:19:29.544 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:19:29.544 | INFO | Uploaded 36GiB to GPU in 2.53221s (14.2168GiB/s) | 20260508 01:19:29.544 | INFO | Downloaded 38GiB to GPU in 2.69439s (14.1034GiB/s) | 20260508 01:19:34.686 | INFO | Analyzing chunk frames = [4108800,4109312) | 20260508 01:19:34.686 | INFO | MemInfo: { | 20260508 01:19:34.686 | INFO | "gpu" : " 17 GB", | 20260508 01:19:34.686 | INFO | "huge" : "", | 20260508 01:19:34.686 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:34.686 | INFO | "malloc" : " 14 GB", | 20260508 01:19:34.686 | INFO | "pinned" : " 29 GB" | 20260508 01:19:34.686 | INFO | } | 20260508 01:19:34.686 | INFO | Perf report for the last 5.14201 seconds: | 20260508 01:19:34.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:34.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:19:34.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 01:19:34.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:34.686 | INFO | Uploaded 36GiB to GPU in 2.53958s (14.1756GiB/s) | 20260508 01:19:34.686 | INFO | Downloaded 34GiB to GPU in 2.4232s (14.031GiB/s) | 20260508 01:19:39.828 | INFO | Analyzing chunk frames = [4109312,4109824) | 20260508 01:19:39.828 | INFO | MemInfo: { | 20260508 01:19:39.828 | INFO | "gpu" : " 17 GB", | 20260508 01:19:39.828 | INFO | "huge" : "", | 20260508 01:19:39.828 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:39.828 | INFO | "malloc" : " 14 GB", | 20260508 01:19:39.828 | INFO | "pinned" : " 29 GB" | 20260508 01:19:39.828 | INFO | } | 20260508 01:19:39.829 | INFO | Perf report for the last 5.1424 seconds: | 20260508 01:19:39.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:39.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 01:19:39.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:19:39.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:39.829 | INFO | Uploaded 36GiB to GPU in 2.4911s (14.4514GiB/s) | 20260508 01:19:39.829 | INFO | Downloaded 34GiB to GPU in 2.48224s (13.6973GiB/s) | 20260508 01:19:44.965 | INFO | Analyzing chunk frames = [4109824,4110336) | 20260508 01:19:44.965 | INFO | MemInfo: { | 20260508 01:19:44.965 | INFO | "gpu" : " 17 GB", | 20260508 01:19:44.965 | INFO | "huge" : "", | 20260508 01:19:44.965 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:44.965 | INFO | "malloc" : " 14 GB", | 20260508 01:19:44.966 | INFO | "pinned" : " 29 GB" | 20260508 01:19:44.966 | INFO | } | 20260508 01:19:44.966 | INFO | Perf report for the last 5.13712 seconds: | 20260508 01:19:44.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:44.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260508 01:19:44.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:19:44.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:44.966 | INFO | Uploaded 36GiB to GPU in 2.50894s (14.3487GiB/s) | 20260508 01:19:44.966 | INFO | Downloaded 34GiB to GPU in 2.49088s (13.6498GiB/s) | 20260508 01:19:50.100 | INFO | Analyzing chunk frames = [4110336,4110848) | 20260508 01:19:50.100 | INFO | MemInfo: { | 20260508 01:19:50.100 | INFO | "gpu" : " 17 GB", | 20260508 01:19:50.100 | INFO | "huge" : "", | 20260508 01:19:50.100 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:50.100 | INFO | "malloc" : " 14 GB", | 20260508 01:19:50.100 | INFO | "pinned" : " 29 GB" | 20260508 01:19:50.100 | INFO | } | 20260508 01:19:50.100 | INFO | Perf report for the last 5.13414 seconds: | 20260508 01:19:50.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:50.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 01:19:50.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 01:19:50.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:50.100 | INFO | Uploaded 36GiB to GPU in 2.35767s (15.2693GiB/s) | 20260508 01:19:50.100 | INFO | Downloaded 34GiB to GPU in 2.50141s (13.5923GiB/s) | 20260508 01:19:55.244 | INFO | Analyzing chunk frames = [4110848,4111360) | 20260508 01:19:55.244 | INFO | MemInfo: { | 20260508 01:19:55.244 | INFO | "gpu" : " 17 GB", | 20260508 01:19:55.244 | INFO | "huge" : "", | 20260508 01:19:55.244 | INFO | "huge_shared" : " 29 GB", | 20260508 01:19:55.244 | INFO | "malloc" : " 14 GB", | 20260508 01:19:55.244 | INFO | "pinned" : " 29 GB" | 20260508 01:19:55.244 | INFO | } | 20260508 01:19:55.244 | INFO | Perf report for the last 5.14411 seconds: | 20260508 01:19:55.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:19:55.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:19:55.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:19:55.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:19:55.244 | INFO | Uploaded 36GiB to GPU in 2.42416s (14.8505GiB/s) | 20260508 01:19:55.244 | INFO | Downloaded 34GiB to GPU in 2.39677s (14.1857GiB/s) | 20260508 01:20:00.382 | INFO | Analyzing chunk frames = [4111360,4111872) | 20260508 01:20:00.382 | INFO | MemInfo: { | 20260508 01:20:00.382 | INFO | "gpu" : " 17 GB", | 20260508 01:20:00.382 | INFO | "huge" : "", | 20260508 01:20:00.382 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:00.382 | INFO | "malloc" : " 14 GB", | 20260508 01:20:00.382 | INFO | "pinned" : " 29 GB" | 20260508 01:20:00.382 | INFO | } | 20260508 01:20:00.383 | INFO | Perf report for the last 5.13858 seconds: | 20260508 01:20:00.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:00.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 01:20:00.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 01:20:00.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:00.383 | INFO | Uploaded 36GiB to GPU in 2.46868s (14.5827GiB/s) | 20260508 01:20:00.383 | INFO | Downloaded 34GiB to GPU in 2.41468s (14.0806GiB/s) | 20260508 01:20:05.526 | INFO | Analyzing chunk frames = [4111872,4112384) | 20260508 01:20:05.526 | INFO | MemInfo: { | 20260508 01:20:05.526 | INFO | "gpu" : " 17 GB", | 20260508 01:20:05.526 | INFO | "huge" : "", | 20260508 01:20:05.527 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:05.527 | INFO | "malloc" : " 14 GB", | 20260508 01:20:05.527 | INFO | "pinned" : " 29 GB" | 20260508 01:20:05.527 | INFO | } | 20260508 01:20:05.527 | INFO | Perf report for the last 5.14421 seconds: | 20260508 01:20:05.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:05.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 01:20:05.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:20:05.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:05.527 | INFO | Uploaded 36GiB to GPU in 2.56323s (14.0448GiB/s) | 20260508 01:20:05.527 | INFO | Downloaded 34GiB to GPU in 2.45379s (13.8561GiB/s) | 20260508 01:20:05.527 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:20:05.527 | INFO | Total mem usage: 16.7 GiB | 20260508 01:20:05.527 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:20:05.527 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -5.53%) | 20260508 01:20:05.527 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:20:05.527 | INFO | High Water Breakdown: | 20260508 01:20:05.527 | INFO | 61.1% : 9.66 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:20:05.527 | INFO | 38.8% : 6.13 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:20:05.527 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:20:05.527 | INFO | ------------------------------------------ | 20260508 01:20:05.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:20:05.527 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:20:05.527 | INFO | Total mem usage: 26.5 GiB | 20260508 01:20:05.527 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:20:05.527 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:20:05.527 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:20:05.527 | INFO | High Water Breakdown: | 20260508 01:20:05.527 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:20:05.527 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:20:05.527 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:20:05.527 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:20:05.527 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:20:05.527 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:20:05.527 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:20:05.527 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:20:05.527 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:20:05.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:20:05.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:20:05.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:20:05.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:20:05.527 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:20:05.527 | INFO | ------------------------------------------ | 20260508 01:20:05.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:20:05.527 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:20:05.527 | INFO | Total mem usage: 27.8 GiB | 20260508 01:20:05.527 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:20:05.527 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:20:05.527 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:20:05.527 | INFO | High Water Breakdown: | 20260508 01:20:05.527 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:20:05.527 | INFO | ------------------------------------------ | 20260508 01:20:05.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:20:05.527 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:20:05.527 | INFO | Total mem usage: 15.7 GiB | 20260508 01:20:05.527 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:20:05.527 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:20:05.527 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:20:05.527 | INFO | High Water Breakdown: | 20260508 01:20:05.527 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:20:05.527 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:20:05.527 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:20:05.527 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:20:05.527 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:20:05.527 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:20:05.527 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:20:05.527 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:20:05.527 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:20:05.527 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:20:05.527 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:20:05.527 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:20:05.527 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:20:05.527 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:20:05.527 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:20:05.527 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:20:05.527 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:20:05.527 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:20:05.527 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:20:05.527 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:20:05.527 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:20:05.527 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:20:05.527 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:20:05.527 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:20:05.527 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:20:05.527 | INFO | ------------------------------------------ | 20260508 01:20:05.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:20:05.528 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:20:05.528 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:20:05.528 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:20:05.528 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.42e+01 / 1.18e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.02e-01, 1.20e-02 / 6.98e-03 | 20260508 01:20:05.528 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.88e-01 / 1.73e-01 | 20260508 01:20:05.528 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.07e-01 / 1.12e-01 | 20260508 01:20:05.528 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.59e+01, 3.93e+00 / 1.56e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.62e+00 / 1.52e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.72e+00, 4.24e-01 / 1.88e-01 | 20260508 01:20:05.528 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.30e+00, 2.73e-01 / 1.47e-01 | 20260508 01:20:05.528 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.36e-01 / 3.73e-01, 5.37e-02 / 2.61e-02 | 20260508 01:20:05.528 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:20:05.528 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:20:05.528 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:20:05.528 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 24.26%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 30.66%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 2.36%, LockUpload -- 2.51e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 2.65%, LockDownload -- 2.83e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 01:20:05.528 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 01:20:05.629 | INFO | Starting a round of Baz IO | 20260508 01:20:09.078 | INFO | Populating RTMetrics report | 20260508 01:20:09.078 | INFO | RTMetrics report populated | 20260508 01:20:09.078 | INFO | Opening tmp RTMetrics json file | 20260508 01:20:09.198 | INFO | Serializing RTMetrics to json | 20260508 01:20:09.230 | INFO | Serialization done, swapping tmp file now... | 20260508 01:20:09.269 | INFO | RTMetrics done | 20260508 01:20:09.311 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.682 seconds | 20260508 01:20:09.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.88% / 0.00% / 58.12% | 20260508 01:20:09.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.876637% / 3.97364e-06% / 5.5631e-05% | 20260508 01:20:09.311 | INFO | Baz IO overall: | 20260508 01:20:09.312 | INFO | BytesWritten : 1864175830712 | 20260508 01:20:09.312 | INFO | NumEvents : 901821238495 | 20260508 01:20:10.667 | INFO | Analyzing chunk frames = [4112384,4112896) | 20260508 01:20:10.667 | INFO | MemInfo: { | 20260508 01:20:10.667 | INFO | "gpu" : " 17 GB", | 20260508 01:20:10.667 | INFO | "huge" : "", | 20260508 01:20:10.667 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:10.667 | INFO | "malloc" : " 14 GB", | 20260508 01:20:10.667 | INFO | "pinned" : " 29 GB" | 20260508 01:20:10.667 | INFO | } | 20260508 01:20:10.667 | INFO | Perf report for the last 5.14034 seconds: | 20260508 01:20:10.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:10.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:10.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:10.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:10.667 | INFO | ANALYSIS : 68.4%, 2.9, 105.2ms, 0 | 20260508 01:20:10.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:10.667 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 01:20:10.667 | INFO | PRE_HQ : 66.7%, 1.7, 58.4ms, 0 | 20260508 01:20:10.667 | INFO | BAZWRITER : 71.3%, 4.3, 158.2ms, 0 | 20260508 01:20:10.667 | INFO | BAZ_SAMPLING : 3.7%, 1.0, 1.9ms, 0 | 20260508 01:20:10.667 | INFO | Uploaded 36GiB to GPU in 2.59161s (13.891GiB/s) | 20260508 01:20:10.668 | INFO | Downloaded 38GiB to GPU in 2.68312s (14.1626GiB/s) | 20260508 01:20:15.800 | INFO | Analyzing chunk frames = [4112896,4113408) | 20260508 01:20:15.800 | INFO | MemInfo: { | 20260508 01:20:15.800 | INFO | "gpu" : " 17 GB", | 20260508 01:20:15.800 | INFO | "huge" : "", | 20260508 01:20:15.800 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:15.800 | INFO | "malloc" : " 14 GB", | 20260508 01:20:15.800 | INFO | "pinned" : " 29 GB" | 20260508 01:20:15.800 | INFO | } | 20260508 01:20:15.800 | INFO | Perf report for the last 5.13325 seconds: | 20260508 01:20:15.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:15.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.800 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 01:20:15.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.800 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:20:15.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:15.801 | INFO | Uploaded 36GiB to GPU in 2.60694s (13.8093GiB/s) | 20260508 01:20:15.801 | INFO | Downloaded 34GiB to GPU in 2.49716s (13.6155GiB/s) | 20260508 01:20:20.944 | INFO | Analyzing chunk frames = [4113408,4113920) | 20260508 01:20:20.944 | INFO | MemInfo: { | 20260508 01:20:20.944 | INFO | "gpu" : " 17 GB", | 20260508 01:20:20.944 | INFO | "huge" : "", | 20260508 01:20:20.944 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:20.944 | INFO | "malloc" : " 14 GB", | 20260508 01:20:20.944 | INFO | "pinned" : " 29 GB" | 20260508 01:20:20.944 | INFO | } | 20260508 01:20:20.944 | INFO | Perf report for the last 5.14411 seconds: | 20260508 01:20:20.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:20.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.945 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:20:20.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.945 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:20:20.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:20.945 | INFO | Uploaded 36GiB to GPU in 2.54183s (14.163GiB/s) | 20260508 01:20:20.945 | INFO | Downloaded 34GiB to GPU in 2.4712s (13.7585GiB/s) | 20260508 01:20:26.082 | INFO | Analyzing chunk frames = [4113920,4114432) | 20260508 01:20:26.082 | INFO | MemInfo: { | 20260508 01:20:26.082 | INFO | "gpu" : " 17 GB", | 20260508 01:20:26.082 | INFO | "huge" : "", | 20260508 01:20:26.082 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:26.082 | INFO | "malloc" : " 14 GB", | 20260508 01:20:26.082 | INFO | "pinned" : " 29 GB" | 20260508 01:20:26.082 | INFO | } | 20260508 01:20:26.082 | INFO | Perf report for the last 5.13769 seconds: | 20260508 01:20:26.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:26.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.082 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:20:26.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.082 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:20:26.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:26.083 | INFO | Uploaded 36GiB to GPU in 2.55682s (14.08GiB/s) | 20260508 01:20:26.083 | INFO | Downloaded 34GiB to GPU in 2.48318s (13.6921GiB/s) | 20260508 01:20:31.220 | INFO | Analyzing chunk frames = [4114432,4114944) | 20260508 01:20:31.220 | INFO | MemInfo: { | 20260508 01:20:31.220 | INFO | "gpu" : " 17 GB", | 20260508 01:20:31.220 | INFO | "huge" : "", | 20260508 01:20:31.220 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:31.220 | INFO | "malloc" : " 14 GB", | 20260508 01:20:31.220 | INFO | "pinned" : " 29 GB" | 20260508 01:20:31.220 | INFO | } | 20260508 01:20:31.220 | INFO | Perf report for the last 5.13797 seconds: | 20260508 01:20:31.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:31.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 01:20:31.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 01:20:31.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:31.220 | INFO | Uploaded 36GiB to GPU in 2.35201s (15.3061GiB/s) | 20260508 01:20:31.220 | INFO | Downloaded 34GiB to GPU in 2.50113s (13.5938GiB/s) | 20260508 01:20:36.353 | INFO | Analyzing chunk frames = [4114944,4115456) | 20260508 01:20:36.353 | INFO | MemInfo: { | 20260508 01:20:36.353 | INFO | "gpu" : " 17 GB", | 20260508 01:20:36.353 | INFO | "huge" : "", | 20260508 01:20:36.353 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:36.354 | INFO | "malloc" : " 14 GB", | 20260508 01:20:36.354 | INFO | "pinned" : " 29 GB" | 20260508 01:20:36.354 | INFO | } | 20260508 01:20:36.354 | INFO | Perf report for the last 5.13354 seconds: | 20260508 01:20:36.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:36.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 01:20:36.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 01:20:36.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:36.354 | INFO | Uploaded 36GiB to GPU in 2.41425s (14.9115GiB/s) | 20260508 01:20:36.355 | INFO | Downloaded 34GiB to GPU in 2.40099s (14.1608GiB/s) | 20260508 01:20:41.494 | INFO | Analyzing chunk frames = [4115456,4115968) | 20260508 01:20:41.494 | INFO | MemInfo: { | 20260508 01:20:41.494 | INFO | "gpu" : " 17 GB", | 20260508 01:20:41.494 | INFO | "huge" : "", | 20260508 01:20:41.494 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:41.494 | INFO | "malloc" : " 14 GB", | 20260508 01:20:41.494 | INFO | "pinned" : " 29 GB" | 20260508 01:20:41.494 | INFO | } | 20260508 01:20:41.495 | INFO | Perf report for the last 5.14091 seconds: | 20260508 01:20:41.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:41.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 01:20:41.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 01:20:41.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:41.495 | INFO | Uploaded 36GiB to GPU in 2.55375s (14.0969GiB/s) | 20260508 01:20:41.495 | INFO | Downloaded 34GiB to GPU in 2.45234s (13.8643GiB/s) | 20260508 01:20:46.636 | INFO | Analyzing chunk frames = [4115968,4116480) | 20260508 01:20:46.636 | INFO | MemInfo: { | 20260508 01:20:46.636 | INFO | "gpu" : " 17 GB", | 20260508 01:20:46.636 | INFO | "huge" : "", | 20260508 01:20:46.636 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:46.636 | INFO | "malloc" : " 14 GB", | 20260508 01:20:46.636 | INFO | "pinned" : " 29 GB" | 20260508 01:20:46.636 | INFO | } | 20260508 01:20:46.636 | INFO | Perf report for the last 5.14145 seconds: | 20260508 01:20:46.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:46.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 01:20:46.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | PRE_HQ : 35.8%, 1.0, 18.6ms, 0 | 20260508 01:20:46.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:46.636 | INFO | Uploaded 36GiB to GPU in 2.56609s (14.0291GiB/s) | 20260508 01:20:46.637 | INFO | Downloaded 34GiB to GPU in 2.45722s (13.8368GiB/s) | 20260508 01:20:46.753 | INFO | Starting a round of Baz IO | 20260508 01:20:50.204 | INFO | Populating RTMetrics report | 20260508 01:20:50.204 | INFO | RTMetrics report populated | 20260508 01:20:50.204 | INFO | Opening tmp RTMetrics json file | 20260508 01:20:50.208 | INFO | Serializing RTMetrics to json | 20260508 01:20:50.210 | INFO | Serialization done, swapping tmp file now... | 20260508 01:20:50.211 | INFO | RTMetrics done | 20260508 01:20:50.303 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.549 seconds | 20260508 01:20:50.303 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.87% / 0.00% / 58.12% | 20260508 01:20:50.303 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.877206% / 7.94729e-06% / 6.75519e-05% | 20260508 01:20:50.303 | INFO | Baz IO overall: | 20260508 01:20:50.303 | INFO | BytesWritten : 1865967906194 | 20260508 01:20:50.303 | INFO | NumEvents : 902682481117 | 20260508 01:20:51.779 | INFO | Analyzing chunk frames = [4116480,4116992) | 20260508 01:20:51.779 | INFO | MemInfo: { | 20260508 01:20:51.779 | INFO | "gpu" : " 17 GB", | 20260508 01:20:51.780 | INFO | "huge" : "", | 20260508 01:20:51.780 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:51.780 | INFO | "malloc" : " 14 GB", | 20260508 01:20:51.780 | INFO | "pinned" : " 29 GB" | 20260508 01:20:51.780 | INFO | } | 20260508 01:20:51.780 | INFO | Perf report for the last 5.14355 seconds: | 20260508 01:20:51.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:51.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:51.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:51.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:51.780 | INFO | ANALYSIS : 68.9%, 2.9, 104.1ms, 0 | 20260508 01:20:51.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:51.780 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260508 01:20:51.780 | INFO | PRE_HQ : 67.7%, 1.8, 62.8ms, 0 | 20260508 01:20:51.780 | INFO | BAZWRITER : 68.7%, 4.1, 143.1ms, 0 | 20260508 01:20:51.780 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 01:20:51.780 | INFO | Uploaded 36GiB to GPU in 2.54852s (14.1259GiB/s) | 20260508 01:20:51.780 | INFO | Downloaded 38GiB to GPU in 2.66564s (14.2555GiB/s) | 20260508 01:20:56.918 | INFO | Analyzing chunk frames = [4116992,4117504) | 20260508 01:20:56.918 | INFO | MemInfo: { | 20260508 01:20:56.918 | INFO | "gpu" : " 17 GB", | 20260508 01:20:56.918 | INFO | "huge" : "", | 20260508 01:20:56.918 | INFO | "huge_shared" : " 29 GB", | 20260508 01:20:56.918 | INFO | "malloc" : " 14 GB", | 20260508 01:20:56.918 | INFO | "pinned" : " 29 GB" | 20260508 01:20:56.918 | INFO | } | 20260508 01:20:56.918 | INFO | Perf report for the last 5.13881 seconds: | 20260508 01:20:56.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:20:56.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.919 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 01:20:56.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.919 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:20:56.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:20:56.919 | INFO | Uploaded 36GiB to GPU in 2.56483s (14.036GiB/s) | 20260508 01:20:56.919 | INFO | Downloaded 34GiB to GPU in 2.41688s (14.0677GiB/s) | 20260508 01:21:02.052 | INFO | Analyzing chunk frames = [4117504,4118016) | 20260508 01:21:02.052 | INFO | MemInfo: { | 20260508 01:21:02.052 | INFO | "gpu" : " 17 GB", | 20260508 01:21:02.052 | INFO | "huge" : "", | 20260508 01:21:02.052 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:02.052 | INFO | "malloc" : " 14 GB", | 20260508 01:21:02.052 | INFO | "pinned" : " 29 GB" | 20260508 01:21:02.052 | INFO | } | 20260508 01:21:02.052 | INFO | Perf report for the last 5.1335 seconds: | 20260508 01:21:02.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:02.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 01:21:02.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:21:02.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:02.052 | INFO | Uploaded 36GiB to GPU in 2.55796s (14.0737GiB/s) | 20260508 01:21:02.052 | INFO | Downloaded 34GiB to GPU in 2.48446s (13.6851GiB/s) | 20260508 01:21:07.196 | INFO | Analyzing chunk frames = [4118016,4118528) | 20260508 01:21:07.196 | INFO | MemInfo: { | 20260508 01:21:07.196 | INFO | "gpu" : " 17 GB", | 20260508 01:21:07.196 | INFO | "huge" : "", | 20260508 01:21:07.196 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:07.196 | INFO | "malloc" : " 14 GB", | 20260508 01:21:07.196 | INFO | "pinned" : " 29 GB" | 20260508 01:21:07.196 | INFO | } | 20260508 01:21:07.196 | INFO | Perf report for the last 5.14446 seconds: | 20260508 01:21:07.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:07.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.197 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:21:07.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.197 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:21:07.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:07.197 | INFO | Uploaded 36GiB to GPU in 2.53993s (14.1736GiB/s) | 20260508 01:21:07.197 | INFO | Downloaded 34GiB to GPU in 2.47281s (13.7495GiB/s) | 20260508 01:21:12.334 | INFO | Analyzing chunk frames = [4118528,4119040) | 20260508 01:21:12.334 | INFO | MemInfo: { | 20260508 01:21:12.334 | INFO | "gpu" : " 17 GB", | 20260508 01:21:12.334 | INFO | "huge" : "", | 20260508 01:21:12.334 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:12.334 | INFO | "malloc" : " 14 GB", | 20260508 01:21:12.334 | INFO | "pinned" : " 29 GB" | 20260508 01:21:12.334 | INFO | } | 20260508 01:21:12.334 | INFO | Perf report for the last 5.13766 seconds: | 20260508 01:21:12.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:12.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:21:12.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 01:21:12.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:12.334 | INFO | Uploaded 36GiB to GPU in 2.31789s (15.5314GiB/s) | 20260508 01:21:12.334 | INFO | Downloaded 34GiB to GPU in 2.49178s (13.6448GiB/s) | 20260508 01:21:17.473 | INFO | Analyzing chunk frames = [4119040,4119552) | 20260508 01:21:17.473 | INFO | MemInfo: { | 20260508 01:21:17.473 | INFO | "gpu" : " 17 GB", | 20260508 01:21:17.473 | INFO | "huge" : "", | 20260508 01:21:17.473 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:17.473 | INFO | "malloc" : " 14 GB", | 20260508 01:21:17.473 | INFO | "pinned" : " 29 GB" | 20260508 01:21:17.473 | INFO | } | 20260508 01:21:17.473 | INFO | Perf report for the last 5.1389 seconds: | 20260508 01:21:17.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:17.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.473 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 01:21:17.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.473 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 01:21:17.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:17.474 | INFO | Uploaded 36GiB to GPU in 2.42635s (14.8371GiB/s) | 20260508 01:21:17.474 | INFO | Downloaded 34GiB to GPU in 2.40033s (14.1647GiB/s) | 20260508 01:21:22.613 | INFO | Analyzing chunk frames = [4119552,4120064) | 20260508 01:21:22.613 | INFO | MemInfo: { | 20260508 01:21:22.613 | INFO | "gpu" : " 17 GB", | 20260508 01:21:22.613 | INFO | "huge" : "", | 20260508 01:21:22.613 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:22.613 | INFO | "malloc" : " 14 GB", | 20260508 01:21:22.613 | INFO | "pinned" : " 29 GB" | 20260508 01:21:22.613 | INFO | } | 20260508 01:21:22.613 | INFO | Perf report for the last 5.1402 seconds: | 20260508 01:21:22.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:22.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.613 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 01:21:22.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.614 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 01:21:22.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:22.614 | INFO | Uploaded 36GiB to GPU in 2.5474s (14.1321GiB/s) | 20260508 01:21:22.614 | INFO | Downloaded 34GiB to GPU in 2.44762s (13.891GiB/s) | 20260508 01:21:27.753 | INFO | Analyzing chunk frames = [4120064,4120576) | 20260508 01:21:27.753 | INFO | MemInfo: { | 20260508 01:21:27.753 | INFO | "gpu" : " 17 GB", | 20260508 01:21:27.753 | INFO | "huge" : "", | 20260508 01:21:27.753 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:27.753 | INFO | "malloc" : " 14 GB", | 20260508 01:21:27.753 | INFO | "pinned" : " 29 GB" | 20260508 01:21:27.753 | INFO | } | 20260508 01:21:27.753 | INFO | Perf report for the last 5.14015 seconds: | 20260508 01:21:27.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:27.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:21:27.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 01:21:27.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:27.754 | INFO | Uploaded 36GiB to GPU in 2.5809s (13.9486GiB/s) | 20260508 01:21:27.754 | INFO | Downloaded 34GiB to GPU in 2.45783s (13.8333GiB/s) | 20260508 01:21:27.754 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:21:27.754 | INFO | Total mem usage: 16.7 GiB | 20260508 01:21:27.754 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:21:27.754 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.31%) | 20260508 01:21:27.754 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:21:27.754 | INFO | High Water Breakdown: | 20260508 01:21:27.754 | INFO | 60.5% : 9.24 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:21:27.754 | INFO | 39.3% : 6 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:21:27.754 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:21:27.754 | INFO | ------------------------------------------ | 20260508 01:21:27.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:21:27.754 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:21:27.754 | INFO | Total mem usage: 26.5 GiB | 20260508 01:21:27.754 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:21:27.754 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 01:21:27.754 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:21:27.754 | INFO | High Water Breakdown: | 20260508 01:21:27.754 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:21:27.754 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:21:27.754 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:21:27.754 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:21:27.754 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:21:27.754 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:21:27.754 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:21:27.755 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:21:27.755 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:21:27.755 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:21:27.755 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:21:27.755 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:21:27.755 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:21:27.755 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:21:27.755 | INFO | ------------------------------------------ | 20260508 01:21:27.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:21:27.755 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:21:27.755 | INFO | Total mem usage: 27.8 GiB | 20260508 01:21:27.755 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:21:27.755 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:21:27.755 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:21:27.755 | INFO | High Water Breakdown: | 20260508 01:21:27.755 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:21:27.755 | INFO | ------------------------------------------ | 20260508 01:21:27.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:21:27.755 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:21:27.755 | INFO | Total mem usage: 15.7 GiB | 20260508 01:21:27.755 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:21:27.755 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:21:27.755 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:21:27.755 | INFO | High Water Breakdown: | 20260508 01:21:27.755 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:21:27.755 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:21:27.755 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:21:27.755 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:21:27.755 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:21:27.755 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:21:27.755 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:21:27.755 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:21:27.755 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:21:27.755 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:21:27.755 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:21:27.755 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:21:27.755 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:21:27.755 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:21:27.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:21:27.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:21:27.755 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:21:27.755 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:21:27.755 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:21:27.755 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:21:27.755 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:21:27.755 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:21:27.755 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:21:27.755 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:21:27.755 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:21:27.755 | INFO | ------------------------------------------ | 20260508 01:21:27.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:21:27.755 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:21:27.755 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:21:27.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:21:27.755 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.63e+01, 1.42e+01 / 1.16e+00 | 20260508 01:21:27.755 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.10e-02 / 5.64e-03 | 20260508 01:21:27.755 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.46e+00, 5.79e-01 / 1.77e-01 | 20260508 01:21:27.755 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 9.23e-02 | 20260508 01:21:27.755 | INFO | PerfReport: 16.09%, DME -- 4.20e+01 / 4.63e+01, 4.08e+00 / 1.64e+00 | 20260508 01:21:27.755 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.83e+01 / 1.96e+01, 1.56e+00 / 1.57e+00 | 20260508 01:21:27.755 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.45e+00 / 2.70e+00, 4.24e-01 / 2.39e-01 | 20260508 01:21:27.755 | INFO | PerfReport: 3.36%, Metrics -- 1.10e+00 / 1.28e+00, 2.70e-01 / 1.53e-01 | 20260508 01:21:27.755 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.38e-01 / 3.69e-01, 5.33e-02 / 2.09e-02 | 20260508 01:21:27.755 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:21:27.755 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:21:27.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:21:27.755 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:21:27.755 | INFO | PerfReport: 24.60%, Upload -- 2.62e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 01:21:27.755 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 01:21:27.755 | INFO | PerfReport: 30.63%, Compute -- 3.26e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:21:27.756 | INFO | PerfReport: 2.55%, LockUpload -- 2.72e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 01:21:27.756 | INFO | PerfReport: 2.51%, LockDownload -- 2.67e+00 / 0.00e+00, 3.84e+00 / 0.00e+00 | 20260508 01:21:27.756 | INFO | PerfReport: 15.39%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:21:27.756 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260508 01:21:27.865 | INFO | Starting a round of Baz IO | 20260508 01:21:31.335 | INFO | Populating RTMetrics report | 20260508 01:21:31.336 | INFO | RTMetrics report populated | 20260508 01:21:31.336 | INFO | Opening tmp RTMetrics json file | 20260508 01:21:31.401 | INFO | Serializing RTMetrics to json | 20260508 01:21:31.429 | INFO | Serialization done, swapping tmp file now... | 20260508 01:21:31.443 | INFO | RTMetrics done | 20260508 01:21:31.470 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.604 seconds | 20260508 01:21:31.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.87% / 0.00% / 58.13% | 20260508 01:21:31.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.87587% / 0% / 6.35783e-05% | 20260508 01:21:31.470 | INFO | Baz IO overall: | 20260508 01:21:31.470 | INFO | BytesWritten : 1867760260952 | 20260508 01:21:31.470 | INFO | NumEvents : 903543945329 | 20260508 01:21:32.895 | INFO | Analyzing chunk frames = [4120576,4121088) | 20260508 01:21:32.895 | INFO | MemInfo: { | 20260508 01:21:32.895 | INFO | "gpu" : " 17 GB", | 20260508 01:21:32.895 | INFO | "huge" : "", | 20260508 01:21:32.895 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:32.895 | INFO | "malloc" : " 14 GB", | 20260508 01:21:32.895 | INFO | "pinned" : " 29 GB" | 20260508 01:21:32.895 | INFO | } | 20260508 01:21:32.895 | INFO | Perf report for the last 5.1419 seconds: | 20260508 01:21:32.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:32.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:32.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:32.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:32.896 | INFO | ANALYSIS : 68.8%, 2.9, 104.5ms, 0 | 20260508 01:21:32.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:32.896 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260508 01:21:32.896 | INFO | PRE_HQ : 67.7%, 1.8, 63.0ms, 0 | 20260508 01:21:32.896 | INFO | BAZWRITER : 69.6%, 4.3, 154.7ms, 0 | 20260508 01:21:32.896 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 01:21:32.896 | INFO | Uploaded 36GiB to GPU in 2.59963s (13.8481GiB/s) | 20260508 01:21:32.896 | INFO | Downloaded 38GiB to GPU in 2.71491s (13.9968GiB/s) | 20260508 01:21:38.035 | INFO | Analyzing chunk frames = [4121088,4121600) | 20260508 01:21:38.035 | INFO | MemInfo: { | 20260508 01:21:38.035 | INFO | "gpu" : " 17 GB", | 20260508 01:21:38.035 | INFO | "huge" : "", | 20260508 01:21:38.035 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:38.035 | INFO | "malloc" : " 14 GB", | 20260508 01:21:38.035 | INFO | "pinned" : " 29 GB" | 20260508 01:21:38.035 | INFO | } | 20260508 01:21:38.035 | INFO | Perf report for the last 5.13966 seconds: | 20260508 01:21:38.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:38.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.035 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:21:38.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.035 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:21:38.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:38.036 | INFO | Uploaded 36GiB to GPU in 2.54881s (14.1242GiB/s) | 20260508 01:21:38.036 | INFO | Downloaded 34GiB to GPU in 2.42082s (14.0448GiB/s) | 20260508 01:21:43.168 | INFO | Analyzing chunk frames = [4121600,4122112) | 20260508 01:21:43.168 | INFO | MemInfo: { | 20260508 01:21:43.168 | INFO | "gpu" : " 17 GB", | 20260508 01:21:43.168 | INFO | "huge" : "", | 20260508 01:21:43.168 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:43.168 | INFO | "malloc" : " 14 GB", | 20260508 01:21:43.168 | INFO | "pinned" : " 29 GB" | 20260508 01:21:43.168 | INFO | } | 20260508 01:21:43.168 | INFO | Perf report for the last 5.13289 seconds: | 20260508 01:21:43.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:43.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 01:21:43.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:21:43.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:43.168 | INFO | Uploaded 36GiB to GPU in 2.55754s (14.076GiB/s) | 20260508 01:21:43.168 | INFO | Downloaded 34GiB to GPU in 2.48358s (13.6899GiB/s) | 20260508 01:21:48.305 | INFO | Analyzing chunk frames = [4122112,4122624) | 20260508 01:21:48.305 | INFO | MemInfo: { | 20260508 01:21:48.305 | INFO | "gpu" : " 17 GB", | 20260508 01:21:48.305 | INFO | "huge" : "", | 20260508 01:21:48.305 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:48.305 | INFO | "malloc" : " 14 GB", | 20260508 01:21:48.305 | INFO | "pinned" : " 29 GB" | 20260508 01:21:48.305 | INFO | } | 20260508 01:21:48.305 | INFO | Perf report for the last 5.13705 seconds: | 20260508 01:21:48.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:48.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 01:21:48.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:21:48.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:48.305 | INFO | Uploaded 36GiB to GPU in 2.54528s (14.1438GiB/s) | 20260508 01:21:48.305 | INFO | Downloaded 34GiB to GPU in 2.47465s (13.7393GiB/s) | 20260508 01:21:53.449 | INFO | Analyzing chunk frames = [4122624,4123136) | 20260508 01:21:53.449 | INFO | MemInfo: { | 20260508 01:21:53.449 | INFO | "gpu" : " 17 GB", | 20260508 01:21:53.450 | INFO | "huge" : "", | 20260508 01:21:53.450 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:53.450 | INFO | "malloc" : " 14 GB", | 20260508 01:21:53.450 | INFO | "pinned" : " 29 GB" | 20260508 01:21:53.450 | INFO | } | 20260508 01:21:53.450 | INFO | Perf report for the last 5.14469 seconds: | 20260508 01:21:53.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:53.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 01:21:53.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 01:21:53.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:53.450 | INFO | Uploaded 36GiB to GPU in 2.30331s (15.6297GiB/s) | 20260508 01:21:53.450 | INFO | Downloaded 34GiB to GPU in 2.46842s (13.774GiB/s) | 20260508 01:21:58.593 | INFO | Analyzing chunk frames = [4123136,4123648) | 20260508 01:21:58.593 | INFO | MemInfo: { | 20260508 01:21:58.593 | INFO | "gpu" : " 17 GB", | 20260508 01:21:58.593 | INFO | "huge" : "", | 20260508 01:21:58.593 | INFO | "huge_shared" : " 29 GB", | 20260508 01:21:58.593 | INFO | "malloc" : " 14 GB", | 20260508 01:21:58.593 | INFO | "pinned" : " 29 GB" | 20260508 01:21:58.593 | INFO | } | 20260508 01:21:58.593 | INFO | Perf report for the last 5.14349 seconds: | 20260508 01:21:58.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:21:58.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:21:58.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 01:21:58.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:21:58.593 | INFO | Uploaded 36GiB to GPU in 2.38684s (15.0827GiB/s) | 20260508 01:21:58.593 | INFO | Downloaded 34GiB to GPU in 2.40602s (14.1312GiB/s) | 20260508 01:22:03.729 | INFO | Analyzing chunk frames = [4123648,4124160) | 20260508 01:22:03.729 | INFO | MemInfo: { | 20260508 01:22:03.729 | INFO | "gpu" : " 17 GB", | 20260508 01:22:03.729 | INFO | "huge" : "", | 20260508 01:22:03.729 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:03.729 | INFO | "malloc" : " 14 GB", | 20260508 01:22:03.729 | INFO | "pinned" : " 29 GB" | 20260508 01:22:03.729 | INFO | } | 20260508 01:22:03.730 | INFO | Perf report for the last 5.13626 seconds: | 20260508 01:22:03.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:03.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:22:03.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 01:22:03.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:03.730 | INFO | Uploaded 36GiB to GPU in 2.42428s (14.8498GiB/s) | 20260508 01:22:03.730 | INFO | Downloaded 34GiB to GPU in 2.41093s (14.1024GiB/s) | 20260508 01:22:08.865 | INFO | Analyzing chunk frames = [4124160,4124672) | 20260508 01:22:08.865 | INFO | MemInfo: { | 20260508 01:22:08.865 | INFO | "gpu" : " 17 GB", | 20260508 01:22:08.865 | INFO | "huge" : "", | 20260508 01:22:08.865 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:08.865 | INFO | "malloc" : " 14 GB", | 20260508 01:22:08.865 | INFO | "pinned" : " 29 GB" | 20260508 01:22:08.865 | INFO | } | 20260508 01:22:08.865 | INFO | Perf report for the last 5.13587 seconds: | 20260508 01:22:08.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:08.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.865 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:22:08.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.866 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 01:22:08.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:08.866 | INFO | Uploaded 36GiB to GPU in 2.55638s (14.0824GiB/s) | 20260508 01:22:08.866 | INFO | Downloaded 34GiB to GPU in 2.46303s (13.8042GiB/s) | 20260508 01:22:08.979 | INFO | Starting a round of Baz IO | 20260508 01:22:12.434 | INFO | Populating RTMetrics report | 20260508 01:22:12.434 | INFO | RTMetrics report populated | 20260508 01:22:12.434 | INFO | Opening tmp RTMetrics json file | 20260508 01:22:12.447 | INFO | Serializing RTMetrics to json | 20260508 01:22:12.451 | INFO | Serialization done, swapping tmp file now... | 20260508 01:22:12.452 | INFO | RTMetrics done | 20260508 01:22:12.525 | INFO | Finished a round of Baz IO: Wrote 1.667 GiB in 3.545 seconds | 20260508 01:22:12.525 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.86% / 0.00% / 58.13% | 20260508 01:22:12.525 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.868924% / 7.94729e-06% / 7.54992e-05% | 20260508 01:22:12.525 | INFO | Baz IO overall: | 20260508 01:22:12.525 | INFO | BytesWritten : 1869549963497 | 20260508 01:22:12.525 | INFO | NumEvents : 904403454659 | 20260508 01:22:14.007 | INFO | Analyzing chunk frames = [4124672,4125184) | 20260508 01:22:14.007 | INFO | MemInfo: { | 20260508 01:22:14.007 | INFO | "gpu" : " 17 GB", | 20260508 01:22:14.007 | INFO | "huge" : "", | 20260508 01:22:14.008 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:14.008 | INFO | "malloc" : " 14 GB", | 20260508 01:22:14.008 | INFO | "pinned" : " 29 GB" | 20260508 01:22:14.008 | INFO | } | 20260508 01:22:14.008 | INFO | Perf report for the last 5.1423 seconds: | 20260508 01:22:14.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:14.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:14.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:14.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:14.008 | INFO | ANALYSIS : 68.7%, 2.9, 104.6ms, 0 | 20260508 01:22:14.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:14.008 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 01:22:14.008 | INFO | PRE_HQ : 67.4%, 1.8, 62.5ms, 0 | 20260508 01:22:14.008 | INFO | BAZWRITER : 68.6%, 3.7, 131.9ms, 0 | 20260508 01:22:14.008 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:22:14.008 | INFO | Uploaded 36GiB to GPU in 2.54853s (14.1258GiB/s) | 20260508 01:22:14.008 | INFO | Downloaded 38GiB to GPU in 2.71128s (14.0155GiB/s) | 20260508 01:22:19.148 | INFO | Analyzing chunk frames = [4125184,4125696) | 20260508 01:22:19.148 | INFO | MemInfo: { | 20260508 01:22:19.148 | INFO | "gpu" : " 17 GB", | 20260508 01:22:19.148 | INFO | "huge" : "", | 20260508 01:22:19.148 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:19.148 | INFO | "malloc" : " 14 GB", | 20260508 01:22:19.148 | INFO | "pinned" : " 29 GB" | 20260508 01:22:19.148 | INFO | } | 20260508 01:22:19.148 | INFO | Perf report for the last 5.14033 seconds: | 20260508 01:22:19.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:19.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:22:19.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:22:19.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:19.148 | INFO | Uploaded 36GiB to GPU in 2.53998s (14.1733GiB/s) | 20260508 01:22:19.148 | INFO | Downloaded 34GiB to GPU in 2.42089s (14.0444GiB/s) | 20260508 01:22:24.285 | INFO | Analyzing chunk frames = [4125696,4126208) | 20260508 01:22:24.285 | INFO | MemInfo: { | 20260508 01:22:24.285 | INFO | "gpu" : " 17 GB", | 20260508 01:22:24.285 | INFO | "huge" : "", | 20260508 01:22:24.285 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:24.285 | INFO | "malloc" : " 14 GB", | 20260508 01:22:24.285 | INFO | "pinned" : " 29 GB" | 20260508 01:22:24.285 | INFO | } | 20260508 01:22:24.285 | INFO | Perf report for the last 5.13736 seconds: | 20260508 01:22:24.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:24.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.285 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:22:24.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.286 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:22:24.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:24.286 | INFO | Uploaded 36GiB to GPU in 2.49962s (14.4022GiB/s) | 20260508 01:22:24.286 | INFO | Downloaded 34GiB to GPU in 2.49612s (13.6211GiB/s) | 20260508 01:22:29.421 | INFO | Analyzing chunk frames = [4126208,4126720) | 20260508 01:22:29.421 | INFO | MemInfo: { | 20260508 01:22:29.421 | INFO | "gpu" : " 17 GB", | 20260508 01:22:29.421 | INFO | "huge" : "", | 20260508 01:22:29.421 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:29.421 | INFO | "malloc" : " 14 GB", | 20260508 01:22:29.421 | INFO | "pinned" : " 29 GB" | 20260508 01:22:29.421 | INFO | } | 20260508 01:22:29.421 | INFO | Perf report for the last 5.13605 seconds: | 20260508 01:22:29.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:29.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.422 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 01:22:29.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.422 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 01:22:29.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:29.422 | INFO | Uploaded 36GiB to GPU in 2.52453s (14.2601GiB/s) | 20260508 01:22:29.422 | INFO | Downloaded 34GiB to GPU in 2.49019s (13.6536GiB/s) | 20260508 01:22:34.563 | INFO | Analyzing chunk frames = [4126720,4127232) | 20260508 01:22:34.563 | INFO | MemInfo: { | 20260508 01:22:34.563 | INFO | "gpu" : " 17 GB", | 20260508 01:22:34.563 | INFO | "huge" : "", | 20260508 01:22:34.563 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:34.563 | INFO | "malloc" : " 14 GB", | 20260508 01:22:34.563 | INFO | "pinned" : " 29 GB" | 20260508 01:22:34.563 | INFO | } | 20260508 01:22:34.563 | INFO | Perf report for the last 5.1413 seconds: | 20260508 01:22:34.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:34.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:22:34.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:22:34.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:34.563 | INFO | Uploaded 36GiB to GPU in 2.34027s (15.3829GiB/s) | 20260508 01:22:34.563 | INFO | Downloaded 34GiB to GPU in 2.50517s (13.5719GiB/s) | 20260508 01:22:39.708 | INFO | Analyzing chunk frames = [4127232,4127744) | 20260508 01:22:39.708 | INFO | MemInfo: { | 20260508 01:22:39.708 | INFO | "gpu" : " 17 GB", | 20260508 01:22:39.708 | INFO | "huge" : "", | 20260508 01:22:39.708 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:39.708 | INFO | "malloc" : " 14 GB", | 20260508 01:22:39.708 | INFO | "pinned" : " 29 GB" | 20260508 01:22:39.708 | INFO | } | 20260508 01:22:39.709 | INFO | Perf report for the last 5.14574 seconds: | 20260508 01:22:39.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:39.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:22:39.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:22:39.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:39.709 | INFO | Uploaded 36GiB to GPU in 2.41036s (14.9355GiB/s) | 20260508 01:22:39.709 | INFO | Downloaded 34GiB to GPU in 2.41299s (14.0904GiB/s) | 20260508 01:22:44.847 | INFO | Analyzing chunk frames = [4127744,4128256) | 20260508 01:22:44.847 | INFO | MemInfo: { | 20260508 01:22:44.847 | INFO | "gpu" : " 17 GB", | 20260508 01:22:44.847 | INFO | "huge" : "", | 20260508 01:22:44.847 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:44.847 | INFO | "malloc" : " 14 GB", | 20260508 01:22:44.847 | INFO | "pinned" : " 29 GB" | 20260508 01:22:44.847 | INFO | } | 20260508 01:22:44.847 | INFO | Perf report for the last 5.1389 seconds: | 20260508 01:22:44.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:44.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.848 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 01:22:44.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.848 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 01:22:44.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:44.848 | INFO | Uploaded 36GiB to GPU in 2.4266s (14.8356GiB/s) | 20260508 01:22:44.848 | INFO | Downloaded 34GiB to GPU in 2.42123s (14.0425GiB/s) | 20260508 01:22:49.982 | INFO | Analyzing chunk frames = [4128256,4128768) | 20260508 01:22:49.982 | INFO | MemInfo: { | 20260508 01:22:49.982 | INFO | "gpu" : " 17 GB", | 20260508 01:22:49.982 | INFO | "huge" : "", | 20260508 01:22:49.982 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:49.982 | INFO | "malloc" : " 14 GB", | 20260508 01:22:49.982 | INFO | "pinned" : " 29 GB" | 20260508 01:22:49.982 | INFO | } | 20260508 01:22:49.982 | INFO | Perf report for the last 5.1345 seconds: | 20260508 01:22:49.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:49.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.982 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 01:22:49.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.982 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 01:22:49.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:49.983 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:22:49.983 | INFO | Total mem usage: 16.7 GiB | 20260508 01:22:49.983 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:22:49.983 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.2%) | 20260508 01:22:49.983 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:22:49.983 | INFO | High Water Breakdown: | 20260508 01:22:49.983 | INFO | 60.4% : 9.14 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:22:49.983 | INFO | 39.5% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:22:49.983 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:22:49.983 | INFO | ------------------------------------------ | 20260508 01:22:49.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:22:49.983 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:22:49.983 | INFO | Total mem usage: 26.5 GiB | 20260508 01:22:49.983 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:22:49.983 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 01:22:49.983 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:22:49.983 | INFO | High Water Breakdown: | 20260508 01:22:49.983 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:22:49.983 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:22:49.983 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:22:49.983 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:22:49.983 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:22:49.983 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:22:49.983 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:22:49.983 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:22:49.983 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:22:49.983 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:22:49.983 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:22:49.983 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:22:49.983 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:22:49.983 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:22:49.983 | INFO | ------------------------------------------ | 20260508 01:22:49.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:22:49.983 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:22:49.983 | INFO | Total mem usage: 27.8 GiB | 20260508 01:22:49.983 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:22:49.983 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:22:49.983 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:22:49.983 | INFO | High Water Breakdown: | 20260508 01:22:49.983 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:22:49.983 | INFO | ------------------------------------------ | 20260508 01:22:49.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:22:49.983 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:22:49.983 | INFO | Total mem usage: 15.7 GiB | 20260508 01:22:49.983 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:22:49.983 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:22:49.983 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 01:22:49.983 | INFO | High Water Breakdown: | 20260508 01:22:49.983 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:22:49.983 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:22:49.983 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:22:49.983 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:22:49.983 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:22:49.983 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:22:49.983 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:22:49.983 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:22:49.983 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:22:49.983 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:22:49.983 | INFO | Uploaded 36GiB to GPU in 2.45127s (14.6862GiB/s) | 20260508 01:22:49.983 | INFO | Downloaded 34GiB to GPU in 2.42336s (14.0301GiB/s) | 20260508 01:22:49.983 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:22:49.983 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:22:49.983 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:22:49.983 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:22:49.983 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:22:49.984 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:22:49.984 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:22:49.984 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:22:49.984 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:22:49.984 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:22:49.984 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:22:49.984 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:22:49.984 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:22:49.984 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:22:49.984 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:22:49.984 | INFO | ------------------------------------------ | 20260508 01:22:49.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:22:49.984 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:22:49.984 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:22:49.984 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:22:49.984 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.40e+01 / 1.16e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.04e-01, 1.20e-02 / 7.08e-03 | 20260508 01:22:49.984 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.49e+00, 5.94e-01 / 2.13e-01 | 20260508 01:22:49.984 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.11e-01 / 1.05e-01 | 20260508 01:22:49.984 | INFO | PerfReport: 16.03%, DME -- 4.21e+01 / 4.60e+01, 4.07e+00 / 1.49e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.67e+00 / 1.47e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.24e-01 / 2.55e-01 | 20260508 01:22:49.984 | INFO | PerfReport: 3.34%, Metrics -- 1.10e+00 / 1.24e+00, 2.74e-01 / 1.31e-01 | 20260508 01:22:49.984 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.38e-01 / 3.76e-01, 5.33e-02 / 2.82e-02 | 20260508 01:22:49.984 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:22:49.984 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:22:49.984 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:22:49.984 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 24.32%, Upload -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 24.36%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 30.92%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 2.18%, LockUpload -- 2.31e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 2.70%, LockDownload -- 2.86e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:22:49.984 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260508 01:22:50.104 | INFO | Starting a round of Baz IO | 20260508 01:22:53.570 | INFO | Populating RTMetrics report | 20260508 01:22:53.570 | INFO | RTMetrics report populated | 20260508 01:22:53.570 | INFO | Opening tmp RTMetrics json file | 20260508 01:22:53.620 | INFO | Serializing RTMetrics to json | 20260508 01:22:53.654 | INFO | Serialization done, swapping tmp file now... | 20260508 01:22:53.659 | INFO | RTMetrics done | 20260508 01:22:53.693 | INFO | Finished a round of Baz IO: Wrote 1.667 GiB in 3.588 seconds | 20260508 01:22:53.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.86% / 0.00% / 58.14% | 20260508 01:22:53.694 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.862777% / 7.94729e-06% / 5.16574e-05% | 20260508 01:22:53.694 | INFO | Baz IO overall: | 20260508 01:22:53.694 | INFO | BytesWritten : 1871340143128 | 20260508 01:22:53.694 | INFO | NumEvents : 905262743600 | 20260508 01:22:55.117 | INFO | Analyzing chunk frames = [4128768,4129280) | 20260508 01:22:55.117 | INFO | MemInfo: { | 20260508 01:22:55.117 | INFO | "gpu" : " 17 GB", | 20260508 01:22:55.117 | INFO | "huge" : "", | 20260508 01:22:55.117 | INFO | "huge_shared" : " 29 GB", | 20260508 01:22:55.117 | INFO | "malloc" : " 14 GB", | 20260508 01:22:55.117 | INFO | "pinned" : " 29 GB" | 20260508 01:22:55.117 | INFO | } | 20260508 01:22:55.117 | INFO | Perf report for the last 5.13544 seconds: | 20260508 01:22:55.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:22:55.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:55.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:55.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:55.118 | INFO | ANALYSIS : 68.9%, 2.9, 104.4ms, 0 | 20260508 01:22:55.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:22:55.118 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260508 01:22:55.118 | INFO | PRE_HQ : 67.8%, 1.8, 63.3ms, 0 | 20260508 01:22:55.118 | INFO | BAZWRITER : 69.6%, 3.9, 140.0ms, 0 | 20260508 01:22:55.118 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 01:22:55.118 | INFO | Uploaded 36GiB to GPU in 2.608s (13.8037GiB/s) | 20260508 01:22:55.118 | INFO | Downloaded 38GiB to GPU in 2.71923s (13.9745GiB/s) | 20260508 01:23:00.260 | INFO | Analyzing chunk frames = [4129280,4129792) | 20260508 01:23:00.260 | INFO | MemInfo: { | 20260508 01:23:00.260 | INFO | "gpu" : " 17 GB", | 20260508 01:23:00.260 | INFO | "huge" : "", | 20260508 01:23:00.260 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:00.260 | INFO | "malloc" : " 14 GB", | 20260508 01:23:00.260 | INFO | "pinned" : " 29 GB" | 20260508 01:23:00.260 | INFO | } | 20260508 01:23:00.260 | INFO | Perf report for the last 5.1429 seconds: | 20260508 01:23:00.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:00.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.260 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 01:23:00.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.260 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:23:00.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:00.261 | INFO | Uploaded 36GiB to GPU in 2.58093s (13.9484GiB/s) | 20260508 01:23:00.261 | INFO | Downloaded 34GiB to GPU in 2.49365s (13.6346GiB/s) | 20260508 01:23:05.396 | INFO | Analyzing chunk frames = [4129792,4130304) | 20260508 01:23:05.396 | INFO | MemInfo: { | 20260508 01:23:05.396 | INFO | "gpu" : " 17 GB", | 20260508 01:23:05.396 | INFO | "huge" : "", | 20260508 01:23:05.396 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:05.396 | INFO | "malloc" : " 14 GB", | 20260508 01:23:05.396 | INFO | "pinned" : " 29 GB" | 20260508 01:23:05.396 | INFO | } | 20260508 01:23:05.396 | INFO | Perf report for the last 5.13565 seconds: | 20260508 01:23:05.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:05.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.396 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:23:05.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.396 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:23:05.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:05.397 | INFO | Uploaded 36GiB to GPU in 2.53154s (14.2206GiB/s) | 20260508 01:23:05.397 | INFO | Downloaded 34GiB to GPU in 2.4854s (13.6799GiB/s) | 20260508 01:23:10.544 | INFO | Analyzing chunk frames = [4130304,4130816) | 20260508 01:23:10.544 | INFO | MemInfo: { | 20260508 01:23:10.544 | INFO | "gpu" : " 17 GB", | 20260508 01:23:10.544 | INFO | "huge" : "", | 20260508 01:23:10.544 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:10.544 | INFO | "malloc" : " 14 GB", | 20260508 01:23:10.544 | INFO | "pinned" : " 29 GB" | 20260508 01:23:10.544 | INFO | } | 20260508 01:23:10.544 | INFO | Perf report for the last 5.14795 seconds: | 20260508 01:23:10.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:10.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 01:23:10.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:23:10.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:10.544 | INFO | Uploaded 36GiB to GPU in 2.49829s (14.4099GiB/s) | 20260508 01:23:10.544 | INFO | Downloaded 34GiB to GPU in 2.48427s (13.6861GiB/s) | 20260508 01:23:15.680 | INFO | Analyzing chunk frames = [4130816,4131328) | 20260508 01:23:15.680 | INFO | MemInfo: { | 20260508 01:23:15.680 | INFO | "gpu" : " 17 GB", | 20260508 01:23:15.680 | INFO | "huge" : "", | 20260508 01:23:15.680 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:15.680 | INFO | "malloc" : " 14 GB", | 20260508 01:23:15.680 | INFO | "pinned" : " 29 GB" | 20260508 01:23:15.680 | INFO | } | 20260508 01:23:15.680 | INFO | Perf report for the last 5.13621 seconds: | 20260508 01:23:15.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:15.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 01:23:15.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:23:15.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:15.680 | INFO | Uploaded 36GiB to GPU in 2.3222s (15.5026GiB/s) | 20260508 01:23:15.680 | INFO | Downloaded 34GiB to GPU in 2.4999s (13.6006GiB/s) | 20260508 01:23:20.817 | INFO | Analyzing chunk frames = [4131328,4131840) | 20260508 01:23:20.817 | INFO | MemInfo: { | 20260508 01:23:20.817 | INFO | "gpu" : " 17 GB", | 20260508 01:23:20.817 | INFO | "huge" : "", | 20260508 01:23:20.817 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:20.817 | INFO | "malloc" : " 14 GB", | 20260508 01:23:20.817 | INFO | "pinned" : " 29 GB" | 20260508 01:23:20.817 | INFO | } | 20260508 01:23:20.817 | INFO | Perf report for the last 5.13714 seconds: | 20260508 01:23:20.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:20.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.817 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:23:20.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.817 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:23:20.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:20.818 | INFO | Uploaded 36GiB to GPU in 2.39505s (15.031GiB/s) | 20260508 01:23:20.818 | INFO | Downloaded 34GiB to GPU in 2.40631s (14.1295GiB/s) | 20260508 01:23:25.957 | INFO | Analyzing chunk frames = [4131840,4132352) | 20260508 01:23:25.957 | INFO | MemInfo: { | 20260508 01:23:25.957 | INFO | "gpu" : " 17 GB", | 20260508 01:23:25.957 | INFO | "huge" : "", | 20260508 01:23:25.957 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:25.957 | INFO | "malloc" : " 14 GB", | 20260508 01:23:25.957 | INFO | "pinned" : " 29 GB" | 20260508 01:23:25.957 | INFO | } | 20260508 01:23:25.958 | INFO | Perf report for the last 5.14021 seconds: | 20260508 01:23:25.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:25.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:23:25.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 01:23:25.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:25.958 | INFO | Uploaded 36GiB to GPU in 2.44531s (14.7221GiB/s) | 20260508 01:23:25.958 | INFO | Downloaded 34GiB to GPU in 2.42494s (14.021GiB/s) | 20260508 01:23:31.096 | INFO | Analyzing chunk frames = [4132352,4132864) | 20260508 01:23:31.096 | INFO | MemInfo: { | 20260508 01:23:31.096 | INFO | "gpu" : " 17 GB", | 20260508 01:23:31.096 | INFO | "huge" : "", | 20260508 01:23:31.097 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:31.097 | INFO | "malloc" : " 14 GB", | 20260508 01:23:31.097 | INFO | "pinned" : " 29 GB" | 20260508 01:23:31.097 | INFO | } | 20260508 01:23:31.097 | INFO | Perf report for the last 5.13924 seconds: | 20260508 01:23:31.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:31.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.097 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:23:31.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.097 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 01:23:31.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:31.098 | INFO | Uploaded 36GiB to GPU in 2.53983s (14.1742GiB/s) | 20260508 01:23:31.098 | INFO | Downloaded 34GiB to GPU in 2.46015s (13.8203GiB/s) | 20260508 01:23:31.210 | INFO | Starting a round of Baz IO | 20260508 01:23:34.649 | INFO | Populating RTMetrics report | 20260508 01:23:34.649 | INFO | RTMetrics report populated | 20260508 01:23:34.649 | INFO | Opening tmp RTMetrics json file | 20260508 01:23:34.651 | INFO | Serializing RTMetrics to json | 20260508 01:23:34.674 | INFO | Serialization done, swapping tmp file now... | 20260508 01:23:34.676 | INFO | RTMetrics done | 20260508 01:23:34.757 | INFO | Finished a round of Baz IO: Wrote 1.665 GiB in 3.546 seconds | 20260508 01:23:34.757 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.86% / 0.00% / 58.14% | 20260508 01:23:34.757 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.838701% / 0% / 6.35783e-05% | 20260508 01:23:34.757 | INFO | Baz IO overall: | 20260508 01:23:34.757 | INFO | BytesWritten : 1873127783998 | 20260508 01:23:34.757 | INFO | NumEvents : 906118824492 | 20260508 01:23:36.235 | INFO | Analyzing chunk frames = [4132864,4133376) | 20260508 01:23:36.235 | INFO | MemInfo: { | 20260508 01:23:36.235 | INFO | "gpu" : " 17 GB", | 20260508 01:23:36.235 | INFO | "huge" : "", | 20260508 01:23:36.235 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:36.235 | INFO | "malloc" : " 14 GB", | 20260508 01:23:36.235 | INFO | "pinned" : " 29 GB" | 20260508 01:23:36.235 | INFO | } | 20260508 01:23:36.235 | INFO | Perf report for the last 5.13866 seconds: | 20260508 01:23:36.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:36.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:36.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:36.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:36.236 | INFO | ANALYSIS : 68.4%, 2.9, 104.1ms, 0 | 20260508 01:23:36.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:36.236 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260508 01:23:36.236 | INFO | PRE_HQ : 67.2%, 1.8, 62.9ms, 0 | 20260508 01:23:36.236 | INFO | BAZWRITER : 68.6%, 3.8, 133.1ms, 0 | 20260508 01:23:36.236 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:23:36.236 | INFO | Uploaded 36GiB to GPU in 2.58892s (13.9054GiB/s) | 20260508 01:23:36.236 | INFO | Downloaded 38GiB to GPU in 2.69658s (14.0919GiB/s) | 20260508 01:23:41.373 | INFO | Analyzing chunk frames = [4133376,4133888) | 20260508 01:23:41.373 | INFO | MemInfo: { | 20260508 01:23:41.373 | INFO | "gpu" : " 17 GB", | 20260508 01:23:41.374 | INFO | "huge" : "", | 20260508 01:23:41.374 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:41.374 | INFO | "malloc" : " 14 GB", | 20260508 01:23:41.374 | INFO | "pinned" : " 29 GB" | 20260508 01:23:41.374 | INFO | } | 20260508 01:23:41.374 | INFO | Perf report for the last 5.13861 seconds: | 20260508 01:23:41.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:41.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.374 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:23:41.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.374 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:23:41.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:41.375 | INFO | Uploaded 36GiB to GPU in 2.52909s (14.2344GiB/s) | 20260508 01:23:41.375 | INFO | Downloaded 34GiB to GPU in 2.42699s (14.0091GiB/s) | 20260508 01:23:46.513 | INFO | Analyzing chunk frames = [4133888,4134400) | 20260508 01:23:46.513 | INFO | MemInfo: { | 20260508 01:23:46.513 | INFO | "gpu" : " 17 GB", | 20260508 01:23:46.513 | INFO | "huge" : "", | 20260508 01:23:46.513 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:46.513 | INFO | "malloc" : " 14 GB", | 20260508 01:23:46.513 | INFO | "pinned" : " 29 GB" | 20260508 01:23:46.513 | INFO | } | 20260508 01:23:46.513 | INFO | Perf report for the last 5.139 seconds: | 20260508 01:23:46.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:46.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.513 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:23:46.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.513 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:23:46.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:46.514 | INFO | Uploaded 36GiB to GPU in 2.54008s (14.1728GiB/s) | 20260508 01:23:46.514 | INFO | Downloaded 34GiB to GPU in 2.49648s (13.6192GiB/s) | 20260508 01:23:51.659 | INFO | Analyzing chunk frames = [4134400,4134912) | 20260508 01:23:51.660 | INFO | MemInfo: { | 20260508 01:23:51.660 | INFO | "gpu" : " 17 GB", | 20260508 01:23:51.660 | INFO | "huge" : "", | 20260508 01:23:51.660 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:51.660 | INFO | "malloc" : " 14 GB", | 20260508 01:23:51.660 | INFO | "pinned" : " 29 GB" | 20260508 01:23:51.660 | INFO | } | 20260508 01:23:51.660 | INFO | Perf report for the last 5.14658 seconds: | 20260508 01:23:51.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:51.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 01:23:51.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:23:51.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:51.660 | INFO | Uploaded 36GiB to GPU in 2.50385s (14.3778GiB/s) | 20260508 01:23:51.660 | INFO | Downloaded 34GiB to GPU in 2.49533s (13.6255GiB/s) | 20260508 01:23:56.797 | INFO | Analyzing chunk frames = [4134912,4135424) | 20260508 01:23:56.797 | INFO | MemInfo: { | 20260508 01:23:56.797 | INFO | "gpu" : " 17 GB", | 20260508 01:23:56.797 | INFO | "huge" : "", | 20260508 01:23:56.797 | INFO | "huge_shared" : " 29 GB", | 20260508 01:23:56.797 | INFO | "malloc" : " 14 GB", | 20260508 01:23:56.797 | INFO | "pinned" : " 29 GB" | 20260508 01:23:56.797 | INFO | } | 20260508 01:23:56.797 | INFO | Perf report for the last 5.13758 seconds: | 20260508 01:23:56.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:23:56.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.797 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:23:56.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.797 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:23:56.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:23:56.798 | INFO | Uploaded 36GiB to GPU in 2.34361s (15.3609GiB/s) | 20260508 01:23:56.798 | INFO | Downloaded 34GiB to GPU in 2.50809s (13.5561GiB/s) | 20260508 01:24:01.933 | INFO | Analyzing chunk frames = [4135424,4135936) | 20260508 01:24:01.933 | INFO | MemInfo: { | 20260508 01:24:01.933 | INFO | "gpu" : " 17 GB", | 20260508 01:24:01.933 | INFO | "huge" : "", | 20260508 01:24:01.933 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:01.933 | INFO | "malloc" : " 14 GB", | 20260508 01:24:01.933 | INFO | "pinned" : " 29 GB" | 20260508 01:24:01.933 | INFO | } | 20260508 01:24:01.933 | INFO | Perf report for the last 5.13564 seconds: | 20260508 01:24:01.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:01.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:24:01.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:24:01.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:01.933 | INFO | Uploaded 36GiB to GPU in 2.39417s (15.0365GiB/s) | 20260508 01:24:01.933 | INFO | Downloaded 34GiB to GPU in 2.41395s (14.0848GiB/s) | 20260508 01:24:07.070 | INFO | Analyzing chunk frames = [4135936,4136448) | 20260508 01:24:07.070 | INFO | MemInfo: { | 20260508 01:24:07.070 | INFO | "gpu" : " 17 GB", | 20260508 01:24:07.070 | INFO | "huge" : "", | 20260508 01:24:07.070 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:07.070 | INFO | "malloc" : " 14 GB", | 20260508 01:24:07.070 | INFO | "pinned" : " 29 GB" | 20260508 01:24:07.070 | INFO | } | 20260508 01:24:07.071 | INFO | Perf report for the last 5.13757 seconds: | 20260508 01:24:07.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:07.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.071 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:24:07.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.071 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 01:24:07.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:07.072 | INFO | Uploaded 36GiB to GPU in 2.44137s (14.7458GiB/s) | 20260508 01:24:07.072 | INFO | Downloaded 34GiB to GPU in 2.41902s (14.0553GiB/s) | 20260508 01:24:12.211 | INFO | Analyzing chunk frames = [4136448,4136960) | 20260508 01:24:12.211 | INFO | MemInfo: { | 20260508 01:24:12.211 | INFO | "gpu" : " 17 GB", | 20260508 01:24:12.211 | INFO | "huge" : "", | 20260508 01:24:12.211 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:12.211 | INFO | "malloc" : " 14 GB", | 20260508 01:24:12.211 | INFO | "pinned" : " 29 GB" | 20260508 01:24:12.211 | INFO | } | 20260508 01:24:12.211 | INFO | Perf report for the last 5.14062 seconds: | 20260508 01:24:12.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:12.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.211 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:24:12.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.211 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 01:24:12.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:12.212 | INFO | Uploaded 36GiB to GPU in 2.55523s (14.0888GiB/s) | 20260508 01:24:12.212 | INFO | Downloaded 34GiB to GPU in 2.46033s (13.8193GiB/s) | 20260508 01:24:12.212 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:24:12.212 | INFO | Total mem usage: 16.7 GiB | 20260508 01:24:12.212 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:24:12.212 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 01:24:12.212 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:24:12.212 | INFO | High Water Breakdown: | 20260508 01:24:12.212 | INFO | 60% : 8.9 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:24:12.212 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:24:12.212 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:24:12.212 | INFO | ------------------------------------------ | 20260508 01:24:12.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:24:12.212 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:24:12.212 | INFO | Total mem usage: 26.5 GiB | 20260508 01:24:12.212 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:24:12.212 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 01:24:12.212 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:24:12.212 | INFO | High Water Breakdown: | 20260508 01:24:12.212 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:24:12.212 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:24:12.212 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:24:12.212 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:24:12.212 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:24:12.212 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:24:12.212 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:24:12.212 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:24:12.212 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:24:12.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:24:12.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:24:12.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:24:12.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:24:12.212 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:24:12.212 | INFO | ------------------------------------------ | 20260508 01:24:12.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:24:12.212 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:24:12.212 | INFO | Total mem usage: 27.8 GiB | 20260508 01:24:12.212 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:24:12.212 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:24:12.212 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:24:12.212 | INFO | High Water Breakdown: | 20260508 01:24:12.212 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:24:12.212 | INFO | ------------------------------------------ | 20260508 01:24:12.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:24:12.212 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:24:12.212 | INFO | Total mem usage: 15.7 GiB | 20260508 01:24:12.212 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:24:12.212 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:24:12.212 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:24:12.212 | INFO | High Water Breakdown: | 20260508 01:24:12.212 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:24:12.212 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:24:12.212 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:24:12.212 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:24:12.212 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:24:12.212 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:24:12.212 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:24:12.212 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:24:12.212 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:24:12.212 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:24:12.212 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:24:12.212 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:24:12.212 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:24:12.212 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:24:12.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:24:12.212 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:24:12.212 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:24:12.212 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:24:12.212 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:24:12.212 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:24:12.212 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:24:12.212 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:24:12.212 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:24:12.212 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:24:12.212 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:24:12.212 | INFO | ------------------------------------------ | 20260508 01:24:12.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:24:12.212 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:24:12.212 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:24:12.212 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:24:12.212 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.43e+01 / 1.58e+00 | 20260508 01:24:12.212 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.16e-02 / 5.61e-03 | 20260508 01:24:12.212 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.47e+00, 5.84e-01 / 1.99e-01 | 20260508 01:24:12.212 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 9.83e-02 | 20260508 01:24:12.212 | INFO | PerfReport: 16.13%, DME -- 4.22e+01 / 4.59e+01, 4.17e+00 / 1.72e+00 | 20260508 01:24:12.212 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.60e+00 / 1.44e+00 | 20260508 01:24:12.212 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.44e+00 / 2.68e+00, 4.27e-01 / 2.77e-01 | 20260508 01:24:12.212 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.25e+00, 2.69e-01 / 1.18e-01 | 20260508 01:24:12.212 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.66e-01, 5.35e-02 / 2.86e-02 | 20260508 01:24:12.212 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:24:12.212 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:24:12.212 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:24:12.212 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:24:12.212 | INFO | PerfReport: 24.40%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 01:24:12.212 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 01:24:12.212 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260508 01:24:12.213 | INFO | PerfReport: 2.08%, LockUpload -- 2.21e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260508 01:24:12.213 | INFO | PerfReport: 2.83%, LockDownload -- 3.01e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 01:24:12.213 | INFO | PerfReport: 15.43%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:24:12.213 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 01:24:12.308 | INFO | Starting a round of Baz IO | 20260508 01:24:15.771 | INFO | Populating RTMetrics report | 20260508 01:24:15.772 | INFO | RTMetrics report populated | 20260508 01:24:15.772 | INFO | Opening tmp RTMetrics json file | 20260508 01:24:15.798 | INFO | Serializing RTMetrics to json | 20260508 01:24:15.800 | INFO | Serialization done, swapping tmp file now... | 20260508 01:24:15.801 | INFO | RTMetrics done | 20260508 01:24:15.892 | INFO | Finished a round of Baz IO: Wrote 1.665 GiB in 3.583 seconds | 20260508 01:24:15.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.85% / 0.00% / 58.14% | 20260508 01:24:15.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.842535% / 0% / 8.74201e-05% | 20260508 01:24:15.892 | INFO | Baz IO overall: | 20260508 01:24:15.892 | INFO | BytesWritten : 1874915103959 | 20260508 01:24:15.892 | INFO | NumEvents : 906975538938 | 20260508 01:24:17.351 | INFO | Analyzing chunk frames = [4136960,4137472) | 20260508 01:24:17.351 | INFO | MemInfo: { | 20260508 01:24:17.351 | INFO | "gpu" : " 17 GB", | 20260508 01:24:17.351 | INFO | "huge" : "", | 20260508 01:24:17.351 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:17.351 | INFO | "malloc" : " 14 GB", | 20260508 01:24:17.351 | INFO | "pinned" : " 29 GB" | 20260508 01:24:17.351 | INFO | } | 20260508 01:24:17.351 | INFO | Perf report for the last 5.14002 seconds: | 20260508 01:24:17.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:17.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:17.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:17.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:17.351 | INFO | ANALYSIS : 68.5%, 3.0, 106.3ms, 0 | 20260508 01:24:17.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:17.351 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260508 01:24:17.351 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260508 01:24:17.351 | INFO | BAZWRITER : 69.3%, 4.1, 146.1ms, 0 | 20260508 01:24:17.351 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 01:24:17.352 | INFO | Uploaded 36GiB to GPU in 2.55042s (14.1153GiB/s) | 20260508 01:24:17.352 | INFO | Downloaded 38GiB to GPU in 2.69881s (14.0803GiB/s) | 20260508 01:24:22.490 | INFO | Analyzing chunk frames = [4137472,4137984) | 20260508 01:24:22.490 | INFO | MemInfo: { | 20260508 01:24:22.490 | INFO | "gpu" : " 17 GB", | 20260508 01:24:22.490 | INFO | "huge" : "", | 20260508 01:24:22.490 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:22.490 | INFO | "malloc" : " 14 GB", | 20260508 01:24:22.490 | INFO | "pinned" : " 29 GB" | 20260508 01:24:22.490 | INFO | } | 20260508 01:24:22.490 | INFO | Perf report for the last 5.1391 seconds: | 20260508 01:24:22.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:22.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.490 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 01:24:22.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.490 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:24:22.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:22.491 | INFO | Uploaded 36GiB to GPU in 2.55593s (14.0849GiB/s) | 20260508 01:24:22.491 | INFO | Downloaded 34GiB to GPU in 2.49497s (13.6274GiB/s) | 20260508 01:24:27.626 | INFO | Analyzing chunk frames = [4137984,4138496) | 20260508 01:24:27.626 | INFO | MemInfo: { | 20260508 01:24:27.626 | INFO | "gpu" : " 17 GB", | 20260508 01:24:27.626 | INFO | "huge" : "", | 20260508 01:24:27.626 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:27.626 | INFO | "malloc" : " 14 GB", | 20260508 01:24:27.626 | INFO | "pinned" : " 29 GB" | 20260508 01:24:27.626 | INFO | } | 20260508 01:24:27.627 | INFO | Perf report for the last 5.13619 seconds: | 20260508 01:24:27.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:27.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:24:27.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:24:27.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:27.627 | INFO | Uploaded 36GiB to GPU in 2.52323s (14.2674GiB/s) | 20260508 01:24:27.627 | INFO | Downloaded 34GiB to GPU in 2.48689s (13.6717GiB/s) | 20260508 01:24:32.772 | INFO | Analyzing chunk frames = [4138496,4139008) | 20260508 01:24:32.772 | INFO | MemInfo: { | 20260508 01:24:32.772 | INFO | "gpu" : " 17 GB", | 20260508 01:24:32.772 | INFO | "huge" : "", | 20260508 01:24:32.772 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:32.772 | INFO | "malloc" : " 14 GB", | 20260508 01:24:32.772 | INFO | "pinned" : " 29 GB" | 20260508 01:24:32.772 | INFO | } | 20260508 01:24:32.772 | INFO | Perf report for the last 5.14564 seconds: | 20260508 01:24:32.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:32.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:24:32.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:24:32.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:32.772 | INFO | Uploaded 36GiB to GPU in 2.50889s (14.349GiB/s) | 20260508 01:24:32.772 | INFO | Downloaded 34GiB to GPU in 2.47588s (13.7325GiB/s) | 20260508 01:24:37.914 | INFO | Analyzing chunk frames = [4139008,4139520) | 20260508 01:24:37.914 | INFO | MemInfo: { | 20260508 01:24:37.914 | INFO | "gpu" : " 17 GB", | 20260508 01:24:37.914 | INFO | "huge" : "", | 20260508 01:24:37.914 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:37.914 | INFO | "malloc" : " 14 GB", | 20260508 01:24:37.914 | INFO | "pinned" : " 29 GB" | 20260508 01:24:37.914 | INFO | } | 20260508 01:24:37.914 | INFO | Perf report for the last 5.14154 seconds: | 20260508 01:24:37.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:37.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 01:24:37.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:24:37.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:37.914 | INFO | Uploaded 36GiB to GPU in 2.31172s (15.5728GiB/s) | 20260508 01:24:37.914 | INFO | Downloaded 34GiB to GPU in 2.49808s (13.6104GiB/s) | 20260508 01:24:43.049 | INFO | Analyzing chunk frames = [4139520,4140032) | 20260508 01:24:43.049 | INFO | MemInfo: { | 20260508 01:24:43.049 | INFO | "gpu" : " 17 GB", | 20260508 01:24:43.049 | INFO | "huge" : "", | 20260508 01:24:43.050 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:43.050 | INFO | "malloc" : " 14 GB", | 20260508 01:24:43.050 | INFO | "pinned" : " 29 GB" | 20260508 01:24:43.050 | INFO | } | 20260508 01:24:43.050 | INFO | Perf report for the last 5.13605 seconds: | 20260508 01:24:43.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:43.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 01:24:43.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:24:43.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:43.050 | INFO | Uploaded 36GiB to GPU in 2.43293s (14.797GiB/s) | 20260508 01:24:43.051 | INFO | Downloaded 34GiB to GPU in 2.41514s (14.0778GiB/s) | 20260508 01:24:48.184 | INFO | Analyzing chunk frames = [4140032,4140544) | 20260508 01:24:48.184 | INFO | MemInfo: { | 20260508 01:24:48.184 | INFO | "gpu" : " 17 GB", | 20260508 01:24:48.184 | INFO | "huge" : "", | 20260508 01:24:48.184 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:48.184 | INFO | "malloc" : " 14 GB", | 20260508 01:24:48.184 | INFO | "pinned" : " 29 GB" | 20260508 01:24:48.184 | INFO | } | 20260508 01:24:48.184 | INFO | Perf report for the last 5.13453 seconds: | 20260508 01:24:48.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:48.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.185 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 01:24:48.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.185 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 01:24:48.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:48.185 | INFO | Uploaded 36GiB to GPU in 2.47387s (14.5521GiB/s) | 20260508 01:24:48.185 | INFO | Downloaded 34GiB to GPU in 2.41617s (14.0718GiB/s) | 20260508 01:24:53.325 | INFO | Analyzing chunk frames = [4140544,4141056) | 20260508 01:24:53.325 | INFO | MemInfo: { | 20260508 01:24:53.325 | INFO | "gpu" : " 17 GB", | 20260508 01:24:53.325 | INFO | "huge" : "", | 20260508 01:24:53.325 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:53.325 | INFO | "malloc" : " 14 GB", | 20260508 01:24:53.325 | INFO | "pinned" : " 29 GB" | 20260508 01:24:53.325 | INFO | } | 20260508 01:24:53.325 | INFO | Perf report for the last 5.14096 seconds: | 20260508 01:24:53.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:53.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.325 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 01:24:53.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.325 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 01:24:53.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:53.326 | INFO | Uploaded 36GiB to GPU in 2.49509s (14.4284GiB/s) | 20260508 01:24:53.326 | INFO | Downloaded 34GiB to GPU in 2.42685s (14.0099GiB/s) | 20260508 01:24:53.439 | INFO | Starting a round of Baz IO | 20260508 01:24:56.902 | INFO | Populating RTMetrics report | 20260508 01:24:56.902 | INFO | RTMetrics report populated | 20260508 01:24:56.902 | INFO | Opening tmp RTMetrics json file | 20260508 01:24:56.915 | INFO | Serializing RTMetrics to json | 20260508 01:24:56.925 | INFO | Serialization done, swapping tmp file now... | 20260508 01:24:56.926 | INFO | RTMetrics done | 20260508 01:24:57.011 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.572 seconds | 20260508 01:24:57.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.85% / 0.00% / 58.15% | 20260508 01:24:57.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.841399% / 2.38419e-05% / 5.96046e-05% | 20260508 01:24:57.011 | INFO | Baz IO overall: | 20260508 01:24:57.011 | INFO | BytesWritten : 1876699670330 | 20260508 01:24:57.011 | INFO | NumEvents : 907831363532 | 20260508 01:24:58.470 | INFO | Analyzing chunk frames = [4141056,4141568) | 20260508 01:24:58.470 | INFO | MemInfo: { | 20260508 01:24:58.470 | INFO | "gpu" : " 17 GB", | 20260508 01:24:58.470 | INFO | "huge" : "", | 20260508 01:24:58.470 | INFO | "huge_shared" : " 29 GB", | 20260508 01:24:58.470 | INFO | "malloc" : " 14 GB", | 20260508 01:24:58.470 | INFO | "pinned" : " 29 GB" | 20260508 01:24:58.470 | INFO | } | 20260508 01:24:58.470 | INFO | Perf report for the last 5.14442 seconds: | 20260508 01:24:58.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:24:58.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:58.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:58.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:58.470 | INFO | ANALYSIS : 68.8%, 2.9, 104.6ms, 0 | 20260508 01:24:58.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:24:58.470 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260508 01:24:58.470 | INFO | PRE_HQ : 67.8%, 1.8, 63.0ms, 0 | 20260508 01:24:58.470 | INFO | BAZWRITER : 69.1%, 3.9, 137.1ms, 0 | 20260508 01:24:58.470 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:24:58.470 | INFO | Uploaded 36GiB to GPU in 2.6038s (13.8259GiB/s) | 20260508 01:24:58.470 | INFO | Downloaded 38GiB to GPU in 2.69555s (14.0973GiB/s) | 20260508 01:25:03.611 | INFO | Analyzing chunk frames = [4141568,4142080) | 20260508 01:25:03.612 | INFO | MemInfo: { | 20260508 01:25:03.612 | INFO | "gpu" : " 17 GB", | 20260508 01:25:03.612 | INFO | "huge" : "", | 20260508 01:25:03.612 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:03.612 | INFO | "malloc" : " 14 GB", | 20260508 01:25:03.612 | INFO | "pinned" : " 29 GB" | 20260508 01:25:03.612 | INFO | } | 20260508 01:25:03.612 | INFO | Perf report for the last 5.14201 seconds: | 20260508 01:25:03.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:03.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 01:25:03.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:25:03.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:03.612 | INFO | Uploaded 36GiB to GPU in 2.55946s (14.0655GiB/s) | 20260508 01:25:03.612 | INFO | Downloaded 34GiB to GPU in 2.42867s (13.9994GiB/s) | 20260508 01:25:08.747 | INFO | Analyzing chunk frames = [4142080,4142592) | 20260508 01:25:08.747 | INFO | MemInfo: { | 20260508 01:25:08.747 | INFO | "gpu" : " 17 GB", | 20260508 01:25:08.747 | INFO | "huge" : "", | 20260508 01:25:08.747 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:08.748 | INFO | "malloc" : " 14 GB", | 20260508 01:25:08.748 | INFO | "pinned" : " 29 GB" | 20260508 01:25:08.748 | INFO | } | 20260508 01:25:08.748 | INFO | Perf report for the last 5.1358 seconds: | 20260508 01:25:08.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:08.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 01:25:08.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:25:08.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:08.748 | INFO | Uploaded 36GiB to GPU in 2.5599s (14.0631GiB/s) | 20260508 01:25:08.748 | INFO | Downloaded 34GiB to GPU in 2.48469s (13.6838GiB/s) | 20260508 01:25:13.880 | INFO | Analyzing chunk frames = [4142592,4143104) | 20260508 01:25:13.880 | INFO | MemInfo: { | 20260508 01:25:13.880 | INFO | "gpu" : " 17 GB", | 20260508 01:25:13.880 | INFO | "huge" : "", | 20260508 01:25:13.880 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:13.880 | INFO | "malloc" : " 14 GB", | 20260508 01:25:13.880 | INFO | "pinned" : " 29 GB" | 20260508 01:25:13.880 | INFO | } | 20260508 01:25:13.881 | INFO | Perf report for the last 5.13294 seconds: | 20260508 01:25:13.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:13.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 01:25:13.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:25:13.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:13.881 | INFO | Uploaded 36GiB to GPU in 2.56012s (14.0619GiB/s) | 20260508 01:25:13.881 | INFO | Downloaded 34GiB to GPU in 2.48701s (13.671GiB/s) | 20260508 01:25:19.026 | INFO | Analyzing chunk frames = [4143104,4143616) | 20260508 01:25:19.027 | INFO | MemInfo: { | 20260508 01:25:19.027 | INFO | "gpu" : " 17 GB", | 20260508 01:25:19.027 | INFO | "huge" : "", | 20260508 01:25:19.027 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:19.027 | INFO | "malloc" : " 14 GB", | 20260508 01:25:19.027 | INFO | "pinned" : " 29 GB" | 20260508 01:25:19.027 | INFO | } | 20260508 01:25:19.027 | INFO | Perf report for the last 5.14622 seconds: | 20260508 01:25:19.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:19.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 01:25:19.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:25:19.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:19.027 | INFO | Uploaded 36GiB to GPU in 2.39719s (15.0176GiB/s) | 20260508 01:25:19.027 | INFO | Downloaded 34GiB to GPU in 2.46997s (13.7653GiB/s) | 20260508 01:25:24.162 | INFO | Analyzing chunk frames = [4143616,4144128) | 20260508 01:25:24.162 | INFO | MemInfo: { | 20260508 01:25:24.162 | INFO | "gpu" : " 17 GB", | 20260508 01:25:24.162 | INFO | "huge" : "", | 20260508 01:25:24.162 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:24.162 | INFO | "malloc" : " 14 GB", | 20260508 01:25:24.162 | INFO | "pinned" : " 29 GB" | 20260508 01:25:24.162 | INFO | } | 20260508 01:25:24.162 | INFO | Perf report for the last 5.13542 seconds: | 20260508 01:25:24.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:24.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:25:24.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:25:24.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:24.162 | INFO | Uploaded 36GiB to GPU in 2.39011s (15.0621GiB/s) | 20260508 01:25:24.162 | INFO | Downloaded 34GiB to GPU in 2.41918s (14.0543GiB/s) | 20260508 01:25:29.304 | INFO | Analyzing chunk frames = [4144128,4144640) | 20260508 01:25:29.304 | INFO | MemInfo: { | 20260508 01:25:29.304 | INFO | "gpu" : " 17 GB", | 20260508 01:25:29.304 | INFO | "huge" : "", | 20260508 01:25:29.305 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:29.305 | INFO | "malloc" : " 14 GB", | 20260508 01:25:29.305 | INFO | "pinned" : " 29 GB" | 20260508 01:25:29.305 | INFO | } | 20260508 01:25:29.305 | INFO | Perf report for the last 5.14241 seconds: | 20260508 01:25:29.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:29.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:25:29.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 01:25:29.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:29.305 | INFO | Uploaded 36GiB to GPU in 2.42376s (14.853GiB/s) | 20260508 01:25:29.305 | INFO | Downloaded 34GiB to GPU in 2.42357s (14.0289GiB/s) | 20260508 01:25:34.437 | INFO | Analyzing chunk frames = [4144640,4145152) | 20260508 01:25:34.437 | INFO | MemInfo: { | 20260508 01:25:34.437 | INFO | "gpu" : " 17 GB", | 20260508 01:25:34.437 | INFO | "huge" : "", | 20260508 01:25:34.437 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:34.437 | INFO | "malloc" : " 14 GB", | 20260508 01:25:34.437 | INFO | "pinned" : " 29 GB" | 20260508 01:25:34.437 | INFO | } | 20260508 01:25:34.437 | INFO | Perf report for the last 5.13244 seconds: | 20260508 01:25:34.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:34.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.437 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 01:25:34.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.437 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:25:34.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:34.438 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:25:34.438 | INFO | Total mem usage: 16.7 GiB | 20260508 01:25:34.438 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:25:34.438 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.76%) | 20260508 01:25:34.438 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:25:34.438 | INFO | High Water Breakdown: | 20260508 01:25:34.438 | INFO | 60.8% : 9.41 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:25:34.438 | INFO | 39.1% : 6.06 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:25:34.438 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:25:34.438 | INFO | ------------------------------------------ | 20260508 01:25:34.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:25:34.438 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:25:34.438 | INFO | Total mem usage: 26.5 GiB | 20260508 01:25:34.438 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:25:34.438 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 01:25:34.438 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 01:25:34.438 | INFO | High Water Breakdown: | 20260508 01:25:34.438 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:25:34.438 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:25:34.438 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:25:34.438 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:25:34.438 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:25:34.438 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:25:34.438 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:25:34.438 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:25:34.438 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:25:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:25:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:25:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:25:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:25:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:25:34.438 | INFO | ------------------------------------------ | 20260508 01:25:34.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:25:34.438 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:25:34.438 | INFO | Total mem usage: 27.8 GiB | 20260508 01:25:34.438 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:25:34.438 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 01:25:34.438 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 01:25:34.438 | INFO | High Water Breakdown: | 20260508 01:25:34.438 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 01:25:34.438 | INFO | ------------------------------------------ | 20260508 01:25:34.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:25:34.438 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:25:34.438 | INFO | Total mem usage: 15.7 GiB | 20260508 01:25:34.438 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:25:34.438 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:25:34.438 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:25:34.438 | INFO | High Water Breakdown: | 20260508 01:25:34.438 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:25:34.438 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:25:34.438 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:25:34.438 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:25:34.438 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:25:34.438 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:25:34.438 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:25:34.438 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:25:34.438 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:25:34.438 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:25:34.438 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:25:34.438 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:25:34.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:25:34.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:25:34.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:25:34.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:25:34.438 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:25:34.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:25:34.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:25:34.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:25:34.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:25:34.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:25:34.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:25:34.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:25:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:25:34.438 | INFO | ------------------------------------------ | 20260508 01:25:34.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:25:34.438 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:25:34.438 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:25:34.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:25:34.438 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.40e+01 / 1.25e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.04e-01, 1.14e-02 / 8.54e-03 | 20260508 01:25:34.438 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.46e+00, 5.87e-01 / 2.10e-01 | 20260508 01:25:34.438 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.06e-01 / 9.34e-02 | 20260508 01:25:34.438 | INFO | PerfReport: 16.10%, DME -- 4.22e+01 / 4.60e+01, 4.11e+00 / 1.46e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.63e+00 / 1.46e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.25e-01 / 2.42e-01 | 20260508 01:25:34.438 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.27e+00, 2.73e-01 / 1.43e-01 | 20260508 01:25:34.438 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.71e-01, 5.39e-02 / 3.17e-02 | 20260508 01:25:34.438 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:25:34.438 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:25:34.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:25:34.438 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 24.42%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 01:25:34.438 | INFO | Uploaded 36GiB to GPU in 2.54522s (14.1442GiB/s) | 20260508 01:25:34.438 | INFO | PerfReport: 24.35%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 30.79%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 2.27%, LockUpload -- 2.41e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 2.66%, LockDownload -- 2.83e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 01:25:34.438 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:25:34.438 | INFO | Downloaded 34GiB to GPU in 2.45584s (13.8445GiB/s) | 20260508 01:25:34.438 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260508 01:25:34.534 | INFO | Starting a round of Baz IO | 20260508 01:25:37.977 | INFO | Populating RTMetrics report | 20260508 01:25:37.978 | INFO | RTMetrics report populated | 20260508 01:25:37.978 | INFO | Opening tmp RTMetrics json file | 20260508 01:25:38.001 | INFO | Serializing RTMetrics to json | 20260508 01:25:38.002 | INFO | Serialization done, swapping tmp file now... | 20260508 01:25:38.003 | INFO | RTMetrics done | 20260508 01:25:38.073 | INFO | Finished a round of Baz IO: Wrote 1.66 GiB in 3.538 seconds | 20260508 01:25:38.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.84% / 0.00% / 58.15% | 20260508 01:25:38.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.840588% / 0% / 4.76837e-05% | 20260508 01:25:38.073 | INFO | Baz IO overall: | 20260508 01:25:38.073 | INFO | BytesWritten : 1878482162900 | 20260508 01:25:38.073 | INFO | NumEvents : 908686208081 | 20260508 01:25:39.578 | INFO | Analyzing chunk frames = [4145152,4145664) | 20260508 01:25:39.578 | INFO | MemInfo: { | 20260508 01:25:39.578 | INFO | "gpu" : " 17 GB", | 20260508 01:25:39.578 | INFO | "huge" : "", | 20260508 01:25:39.578 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:39.578 | INFO | "malloc" : " 14 GB", | 20260508 01:25:39.578 | INFO | "pinned" : " 29 GB" | 20260508 01:25:39.578 | INFO | } | 20260508 01:25:39.578 | INFO | Perf report for the last 5.14097 seconds: | 20260508 01:25:39.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:39.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:39.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:39.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:39.578 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 01:25:39.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:39.578 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 01:25:39.578 | INFO | PRE_HQ : 65.3%, 1.7, 56.3ms, 0 | 20260508 01:25:39.578 | INFO | BAZWRITER : 68.4%, 3.6, 127.1ms, 0 | 20260508 01:25:39.578 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:25:39.579 | INFO | Uploaded 36GiB to GPU in 2.53295s (14.2127GiB/s) | 20260508 01:25:39.579 | INFO | Downloaded 38GiB to GPU in 2.68771s (14.1384GiB/s) | 20260508 01:25:44.721 | INFO | Analyzing chunk frames = [4145664,4146176) | 20260508 01:25:44.721 | INFO | MemInfo: { | 20260508 01:25:44.721 | INFO | "gpu" : " 17 GB", | 20260508 01:25:44.721 | INFO | "huge" : "", | 20260508 01:25:44.721 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:44.721 | INFO | "malloc" : " 14 GB", | 20260508 01:25:44.721 | INFO | "pinned" : " 29 GB" | 20260508 01:25:44.721 | INFO | } | 20260508 01:25:44.721 | INFO | Perf report for the last 5.14298 seconds: | 20260508 01:25:44.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:44.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 01:25:44.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:25:44.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:44.721 | INFO | Uploaded 36GiB to GPU in 2.57569s (13.9768GiB/s) | 20260508 01:25:44.721 | INFO | Downloaded 34GiB to GPU in 2.49435s (13.6308GiB/s) | 20260508 01:25:49.864 | INFO | Analyzing chunk frames = [4146176,4146688) | 20260508 01:25:49.864 | INFO | MemInfo: { | 20260508 01:25:49.864 | INFO | "gpu" : " 17 GB", | 20260508 01:25:49.864 | INFO | "huge" : "", | 20260508 01:25:49.864 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:49.864 | INFO | "malloc" : " 14 GB", | 20260508 01:25:49.864 | INFO | "pinned" : " 29 GB" | 20260508 01:25:49.864 | INFO | } | 20260508 01:25:49.865 | INFO | Perf report for the last 5.14347 seconds: | 20260508 01:25:49.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:49.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:25:49.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:25:49.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:49.865 | INFO | Uploaded 36GiB to GPU in 2.50163s (14.3906GiB/s) | 20260508 01:25:49.865 | INFO | Downloaded 34GiB to GPU in 2.48447s (13.685GiB/s) | 20260508 01:25:55.004 | INFO | Analyzing chunk frames = [4146688,4147200) | 20260508 01:25:55.004 | INFO | MemInfo: { | 20260508 01:25:55.004 | INFO | "gpu" : " 17 GB", | 20260508 01:25:55.004 | INFO | "huge" : "", | 20260508 01:25:55.004 | INFO | "huge_shared" : " 29 GB", | 20260508 01:25:55.004 | INFO | "malloc" : " 14 GB", | 20260508 01:25:55.004 | INFO | "pinned" : " 29 GB" | 20260508 01:25:55.004 | INFO | } | 20260508 01:25:55.004 | INFO | Perf report for the last 5.13958 seconds: | 20260508 01:25:55.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:25:55.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:25:55.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:25:55.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:25:55.004 | INFO | Uploaded 36GiB to GPU in 2.51929s (14.2897GiB/s) | 20260508 01:25:55.004 | INFO | Downloaded 34GiB to GPU in 2.49124s (13.6478GiB/s) | 20260508 01:26:00.138 | INFO | Analyzing chunk frames = [4147200,4147712) | 20260508 01:26:00.138 | INFO | MemInfo: { | 20260508 01:26:00.138 | INFO | "gpu" : " 17 GB", | 20260508 01:26:00.138 | INFO | "huge" : "", | 20260508 01:26:00.138 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:00.138 | INFO | "malloc" : " 14 GB", | 20260508 01:26:00.138 | INFO | "pinned" : " 29 GB" | 20260508 01:26:00.138 | INFO | } | 20260508 01:26:00.138 | INFO | Perf report for the last 5.13393 seconds: | 20260508 01:26:00.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:00.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 01:26:00.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 01:26:00.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:00.138 | INFO | Uploaded 36GiB to GPU in 2.3408s (15.3794GiB/s) | 20260508 01:26:00.138 | INFO | Downloaded 34GiB to GPU in 2.49402s (13.6326GiB/s) | 20260508 01:26:05.277 | INFO | Analyzing chunk frames = [4147712,4148224) | 20260508 01:26:05.277 | INFO | MemInfo: { | 20260508 01:26:05.277 | INFO | "gpu" : " 17 GB", | 20260508 01:26:05.277 | INFO | "huge" : "", | 20260508 01:26:05.277 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:05.277 | INFO | "malloc" : " 14 GB", | 20260508 01:26:05.277 | INFO | "pinned" : " 29 GB" | 20260508 01:26:05.277 | INFO | } | 20260508 01:26:05.277 | INFO | Perf report for the last 5.13916 seconds: | 20260508 01:26:05.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:05.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 01:26:05.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260508 01:26:05.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:05.277 | INFO | Uploaded 36GiB to GPU in 2.433s (14.7965GiB/s) | 20260508 01:26:05.278 | INFO | Downloaded 34GiB to GPU in 2.41502s (14.0785GiB/s) | 20260508 01:26:10.419 | INFO | Analyzing chunk frames = [4148224,4148736) | 20260508 01:26:10.419 | INFO | MemInfo: { | 20260508 01:26:10.419 | INFO | "gpu" : " 17 GB", | 20260508 01:26:10.419 | INFO | "huge" : "", | 20260508 01:26:10.419 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:10.419 | INFO | "malloc" : " 14 GB", | 20260508 01:26:10.419 | INFO | "pinned" : " 29 GB" | 20260508 01:26:10.419 | INFO | } | 20260508 01:26:10.419 | INFO | Perf report for the last 5.14167 seconds: | 20260508 01:26:10.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:10.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:26:10.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:26:10.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:10.419 | INFO | Uploaded 36GiB to GPU in 2.45168s (14.6838GiB/s) | 20260508 01:26:10.419 | INFO | Downloaded 34GiB to GPU in 2.41584s (14.0738GiB/s) | 20260508 01:26:15.562 | INFO | Analyzing chunk frames = [4148736,4149248) | 20260508 01:26:15.562 | INFO | MemInfo: { | 20260508 01:26:15.562 | INFO | "gpu" : " 17 GB", | 20260508 01:26:15.562 | INFO | "huge" : "", | 20260508 01:26:15.563 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:15.563 | INFO | "malloc" : " 14 GB", | 20260508 01:26:15.563 | INFO | "pinned" : " 29 GB" | 20260508 01:26:15.563 | INFO | } | 20260508 01:26:15.563 | INFO | Perf report for the last 5.14385 seconds: | 20260508 01:26:15.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:15.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:26:15.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:26:15.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:15.563 | INFO | Uploaded 36GiB to GPU in 2.54641s (14.1375GiB/s) | 20260508 01:26:15.563 | INFO | Downloaded 34GiB to GPU in 2.44781s (13.89GiB/s) | 20260508 01:26:15.659 | INFO | Starting a round of Baz IO | 20260508 01:26:19.112 | INFO | Populating RTMetrics report | 20260508 01:26:19.112 | INFO | RTMetrics report populated | 20260508 01:26:19.112 | INFO | Opening tmp RTMetrics json file | 20260508 01:26:19.118 | INFO | Serializing RTMetrics to json | 20260508 01:26:19.135 | INFO | Serialization done, swapping tmp file now... | 20260508 01:26:19.136 | INFO | RTMetrics done | 20260508 01:26:19.205 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.546 seconds | 20260508 01:26:19.205 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.84% / 0.00% / 58.16% | 20260508 01:26:19.205 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.845476% / 3.97364e-06% / 2.78155e-05% | 20260508 01:26:19.205 | INFO | Baz IO overall: | 20260508 01:26:19.205 | INFO | BytesWritten : 1880266641113 | 20260508 01:26:19.205 | INFO | NumEvents : 909542855138 | 20260508 01:26:20.695 | INFO | Analyzing chunk frames = [4149248,4149760) | 20260508 01:26:20.695 | INFO | MemInfo: { | 20260508 01:26:20.695 | INFO | "gpu" : " 17 GB", | 20260508 01:26:20.695 | INFO | "huge" : "", | 20260508 01:26:20.695 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:20.695 | INFO | "malloc" : " 14 GB", | 20260508 01:26:20.695 | INFO | "pinned" : " 29 GB" | 20260508 01:26:20.695 | INFO | } | 20260508 01:26:20.695 | INFO | Perf report for the last 5.13265 seconds: | 20260508 01:26:20.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:20.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:20.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:20.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:20.696 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 01:26:20.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:20.696 | INFO | RT_METRICS : 40.4%, 1.0, 21.1ms, 0 | 20260508 01:26:20.696 | INFO | PRE_HQ : 65.8%, 1.6, 56.1ms, 0 | 20260508 01:26:20.696 | INFO | BAZWRITER : 68.6%, 3.6, 127.5ms, 0 | 20260508 01:26:20.696 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:26:20.696 | INFO | Uploaded 36GiB to GPU in 2.59722s (13.861GiB/s) | 20260508 01:26:20.696 | INFO | Downloaded 38GiB to GPU in 2.68697s (14.1423GiB/s) | 20260508 01:26:25.835 | INFO | Analyzing chunk frames = [4149760,4150272) | 20260508 01:26:25.835 | INFO | MemInfo: { | 20260508 01:26:25.835 | INFO | "gpu" : " 17 GB", | 20260508 01:26:25.835 | INFO | "huge" : "", | 20260508 01:26:25.835 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:25.835 | INFO | "malloc" : " 14 GB", | 20260508 01:26:25.835 | INFO | "pinned" : " 29 GB" | 20260508 01:26:25.835 | INFO | } | 20260508 01:26:25.835 | INFO | Perf report for the last 5.13979 seconds: | 20260508 01:26:25.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:25.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.835 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:26:25.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.835 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:26:25.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:25.836 | INFO | Uploaded 36GiB to GPU in 2.56061s (14.0592GiB/s) | 20260508 01:26:25.836 | INFO | Downloaded 34GiB to GPU in 2.42492s (14.0211GiB/s) | 20260508 01:26:30.976 | INFO | Analyzing chunk frames = [4150272,4150784) | 20260508 01:26:30.976 | INFO | MemInfo: { | 20260508 01:26:30.976 | INFO | "gpu" : " 17 GB", | 20260508 01:26:30.976 | INFO | "huge" : "", | 20260508 01:26:30.976 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:30.976 | INFO | "malloc" : " 14 GB", | 20260508 01:26:30.976 | INFO | "pinned" : " 29 GB" | 20260508 01:26:30.976 | INFO | } | 20260508 01:26:30.976 | INFO | Perf report for the last 5.14057 seconds: | 20260508 01:26:30.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:30.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:26:30.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:26:30.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:30.976 | INFO | Uploaded 36GiB to GPU in 2.56544s (14.0327GiB/s) | 20260508 01:26:30.976 | INFO | Downloaded 34GiB to GPU in 2.48524s (13.6807GiB/s) | 20260508 01:26:36.114 | INFO | Analyzing chunk frames = [4150784,4151296) | 20260508 01:26:36.114 | INFO | MemInfo: { | 20260508 01:26:36.114 | INFO | "gpu" : " 17 GB", | 20260508 01:26:36.114 | INFO | "huge" : "", | 20260508 01:26:36.114 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:36.114 | INFO | "malloc" : " 14 GB", | 20260508 01:26:36.114 | INFO | "pinned" : " 29 GB" | 20260508 01:26:36.114 | INFO | } | 20260508 01:26:36.114 | INFO | Perf report for the last 5.13806 seconds: | 20260508 01:26:36.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:36.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 01:26:36.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:26:36.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:36.114 | INFO | Uploaded 36GiB to GPU in 2.53789s (14.185GiB/s) | 20260508 01:26:36.114 | INFO | Downloaded 34GiB to GPU in 2.47527s (13.7359GiB/s) | 20260508 01:26:41.254 | INFO | Analyzing chunk frames = [4151296,4151808) | 20260508 01:26:41.254 | INFO | MemInfo: { | 20260508 01:26:41.254 | INFO | "gpu" : " 17 GB", | 20260508 01:26:41.254 | INFO | "huge" : "", | 20260508 01:26:41.254 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:41.254 | INFO | "malloc" : " 14 GB", | 20260508 01:26:41.254 | INFO | "pinned" : " 29 GB" | 20260508 01:26:41.254 | INFO | } | 20260508 01:26:41.254 | INFO | Perf report for the last 5.14 seconds: | 20260508 01:26:41.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:41.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:26:41.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 01:26:41.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:41.254 | INFO | Uploaded 36GiB to GPU in 2.39421s (15.0363GiB/s) | 20260508 01:26:41.254 | INFO | Downloaded 34GiB to GPU in 2.4999s (13.6005GiB/s) | 20260508 01:26:46.388 | INFO | Analyzing chunk frames = [4151808,4152320) | 20260508 01:26:46.388 | INFO | MemInfo: { | 20260508 01:26:46.388 | INFO | "gpu" : " 17 GB", | 20260508 01:26:46.388 | INFO | "huge" : "", | 20260508 01:26:46.388 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:46.388 | INFO | "malloc" : " 14 GB", | 20260508 01:26:46.388 | INFO | "pinned" : " 29 GB" | 20260508 01:26:46.388 | INFO | } | 20260508 01:26:46.388 | INFO | Perf report for the last 5.13419 seconds: | 20260508 01:26:46.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:46.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.388 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 01:26:46.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.388 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:26:46.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:46.389 | INFO | Uploaded 36GiB to GPU in 2.42789s (14.8277GiB/s) | 20260508 01:26:46.389 | INFO | Downloaded 34GiB to GPU in 2.39701s (14.1844GiB/s) | 20260508 01:26:51.530 | INFO | Analyzing chunk frames = [4152320,4152832) | 20260508 01:26:51.530 | INFO | MemInfo: { | 20260508 01:26:51.530 | INFO | "gpu" : " 17 GB", | 20260508 01:26:51.530 | INFO | "huge" : "", | 20260508 01:26:51.530 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:51.530 | INFO | "malloc" : " 14 GB", | 20260508 01:26:51.530 | INFO | "pinned" : " 29 GB" | 20260508 01:26:51.530 | INFO | } | 20260508 01:26:51.530 | INFO | Perf report for the last 5.14233 seconds: | 20260508 01:26:51.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:51.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:26:51.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:26:51.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:51.531 | INFO | Uploaded 36GiB to GPU in 2.44854s (14.7026GiB/s) | 20260508 01:26:51.531 | INFO | Downloaded 34GiB to GPU in 2.41485s (14.0796GiB/s) | 20260508 01:26:56.675 | INFO | Analyzing chunk frames = [4152832,4153344) | 20260508 01:26:56.675 | INFO | MemInfo: { | 20260508 01:26:56.675 | INFO | "gpu" : " 17 GB", | 20260508 01:26:56.675 | INFO | "huge" : "", | 20260508 01:26:56.676 | INFO | "huge_shared" : " 29 GB", | 20260508 01:26:56.676 | INFO | "malloc" : " 14 GB", | 20260508 01:26:56.676 | INFO | "pinned" : " 29 GB" | 20260508 01:26:56.676 | INFO | } | 20260508 01:26:56.676 | INFO | Perf report for the last 5.14512 seconds: | 20260508 01:26:56.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:26:56.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:26:56.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:26:56.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:26:56.676 | INFO | Uploaded 36GiB to GPU in 2.55814s (14.0727GiB/s) | 20260508 01:26:56.676 | INFO | Downloaded 34GiB to GPU in 2.44627s (13.8987GiB/s) | 20260508 01:26:56.676 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:26:56.676 | INFO | Total mem usage: 16.7 GiB | 20260508 01:26:56.676 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:26:56.676 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 01:26:56.676 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:26:56.676 | INFO | High Water Breakdown: | 20260508 01:26:56.676 | INFO | 59.8% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:26:56.676 | INFO | 40.1% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:26:56.676 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:26:56.676 | INFO | ------------------------------------------ | 20260508 01:26:56.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:26:56.676 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:26:56.676 | INFO | Total mem usage: 26.5 GiB | 20260508 01:26:56.676 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:26:56.676 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:26:56.676 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:26:56.676 | INFO | High Water Breakdown: | 20260508 01:26:56.677 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:26:56.677 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:26:56.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:26:56.677 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:26:56.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:26:56.677 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:26:56.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:26:56.677 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:26:56.677 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:26:56.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:26:56.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:26:56.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:26:56.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:26:56.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:26:56.677 | INFO | ------------------------------------------ | 20260508 01:26:56.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:26:56.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:26:56.677 | INFO | Total mem usage: 27.8 GiB | 20260508 01:26:56.677 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:26:56.677 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:26:56.677 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:26:56.677 | INFO | High Water Breakdown: | 20260508 01:26:56.677 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:26:56.677 | INFO | ------------------------------------------ | 20260508 01:26:56.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:26:56.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:26:56.677 | INFO | Total mem usage: 15.7 GiB | 20260508 01:26:56.677 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:26:56.677 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:26:56.677 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:26:56.677 | INFO | High Water Breakdown: | 20260508 01:26:56.677 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:26:56.677 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:26:56.677 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:26:56.677 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:26:56.677 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:26:56.677 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:26:56.677 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:26:56.677 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:26:56.677 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:26:56.677 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:26:56.677 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:26:56.677 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:26:56.677 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:26:56.677 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:26:56.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:26:56.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:26:56.677 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:26:56.677 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:26:56.677 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:26:56.677 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:26:56.677 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:26:56.677 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:26:56.677 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:26:56.677 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:26:56.677 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:26:56.677 | INFO | ------------------------------------------ | 20260508 01:26:56.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:26:56.677 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:26:56.677 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:26:56.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:26:56.677 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.11e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.28e-02 / 5.43e-03 | 20260508 01:26:56.677 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.50e+00, 5.77e-01 / 1.57e-01 | 20260508 01:26:56.677 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 1.00e-01 | 20260508 01:26:56.677 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.60e+01, 3.95e+00 / 1.57e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.61e+00 / 1.36e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.21e-01 / 1.95e-01 | 20260508 01:26:56.677 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.28e+00, 2.69e-01 / 1.46e-01 | 20260508 01:26:56.677 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.66e-01, 5.26e-02 / 2.19e-02 | 20260508 01:26:56.677 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:26:56.677 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:26:56.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:26:56.677 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 2.33%, LockUpload -- 2.48e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 2.60%, LockDownload -- 2.77e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:26:56.677 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260508 01:26:56.786 | INFO | Starting a round of Baz IO | 20260508 01:27:00.227 | INFO | Populating RTMetrics report | 20260508 01:27:00.228 | INFO | RTMetrics report populated | 20260508 01:27:00.228 | INFO | Opening tmp RTMetrics json file | 20260508 01:27:00.252 | INFO | Serializing RTMetrics to json | 20260508 01:27:00.254 | INFO | Serialization done, swapping tmp file now... | 20260508 01:27:00.255 | INFO | RTMetrics done | 20260508 01:27:00.338 | INFO | Finished a round of Baz IO: Wrote 1.661 GiB in 3.551 seconds | 20260508 01:27:00.338 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.84% / 0.00% / 58.16% | 20260508 01:27:00.338 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.842404% / 3.97364e-06% / 3.97364e-05% | 20260508 01:27:00.338 | INFO | Baz IO overall: | 20260508 01:27:00.338 | INFO | BytesWritten : 1882049698488 | 20260508 01:27:00.338 | INFO | NumEvents : 910398599084 | 20260508 01:27:01.817 | INFO | Analyzing chunk frames = [4153344,4153856) | 20260508 01:27:01.817 | INFO | MemInfo: { | 20260508 01:27:01.817 | INFO | "gpu" : " 17 GB", | 20260508 01:27:01.817 | INFO | "huge" : "", | 20260508 01:27:01.817 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:01.817 | INFO | "malloc" : " 14 GB", | 20260508 01:27:01.817 | INFO | "pinned" : " 29 GB" | 20260508 01:27:01.817 | INFO | } | 20260508 01:27:01.817 | INFO | Perf report for the last 5.14129 seconds: | 20260508 01:27:01.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:01.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:01.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:01.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:01.817 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260508 01:27:01.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:01.817 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260508 01:27:01.817 | INFO | PRE_HQ : 65.6%, 1.7, 56.7ms, 0 | 20260508 01:27:01.817 | INFO | BAZWRITER : 68.9%, 3.5, 124.9ms, 0 | 20260508 01:27:01.817 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 01:27:01.817 | INFO | Uploaded 36GiB to GPU in 2.60006s (13.8458GiB/s) | 20260508 01:27:01.817 | INFO | Downloaded 38GiB to GPU in 2.67975s (14.1804GiB/s) | 20260508 01:27:06.947 | INFO | Analyzing chunk frames = [4153856,4154368) | 20260508 01:27:06.947 | INFO | MemInfo: { | 20260508 01:27:06.947 | INFO | "gpu" : " 17 GB", | 20260508 01:27:06.947 | INFO | "huge" : "", | 20260508 01:27:06.947 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:06.947 | INFO | "malloc" : " 14 GB", | 20260508 01:27:06.948 | INFO | "pinned" : " 29 GB" | 20260508 01:27:06.948 | INFO | } | 20260508 01:27:06.948 | INFO | Perf report for the last 5.13074 seconds: | 20260508 01:27:06.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:06.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260508 01:27:06.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:27:06.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:06.948 | INFO | Uploaded 36GiB to GPU in 2.56563s (14.0317GiB/s) | 20260508 01:27:06.948 | INFO | Downloaded 34GiB to GPU in 2.41638s (14.0706GiB/s) | 20260508 01:27:12.088 | INFO | Analyzing chunk frames = [4154368,4154880) | 20260508 01:27:12.088 | INFO | MemInfo: { | 20260508 01:27:12.088 | INFO | "gpu" : " 17 GB", | 20260508 01:27:12.088 | INFO | "huge" : "", | 20260508 01:27:12.089 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:12.089 | INFO | "malloc" : " 14 GB", | 20260508 01:27:12.089 | INFO | "pinned" : " 29 GB" | 20260508 01:27:12.089 | INFO | } | 20260508 01:27:12.089 | INFO | Perf report for the last 5.14102 seconds: | 20260508 01:27:12.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:12.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:27:12.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:27:12.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:12.089 | INFO | Uploaded 36GiB to GPU in 2.54219s (14.161GiB/s) | 20260508 01:27:12.089 | INFO | Downloaded 34GiB to GPU in 2.47673s (13.7278GiB/s) | 20260508 01:27:17.228 | INFO | Analyzing chunk frames = [4154880,4155392) | 20260508 01:27:17.228 | INFO | MemInfo: { | 20260508 01:27:17.228 | INFO | "gpu" : " 17 GB", | 20260508 01:27:17.228 | INFO | "huge" : "", | 20260508 01:27:17.228 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:17.228 | INFO | "malloc" : " 14 GB", | 20260508 01:27:17.228 | INFO | "pinned" : " 29 GB" | 20260508 01:27:17.228 | INFO | } | 20260508 01:27:17.228 | INFO | Perf report for the last 5.13937 seconds: | 20260508 01:27:17.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:17.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:27:17.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:27:17.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:17.228 | INFO | Uploaded 36GiB to GPU in 2.56282s (14.047GiB/s) | 20260508 01:27:17.228 | INFO | Downloaded 34GiB to GPU in 2.48057s (13.7065GiB/s) | 20260508 01:27:22.371 | INFO | Analyzing chunk frames = [4155392,4155904) | 20260508 01:27:22.371 | INFO | MemInfo: { | 20260508 01:27:22.371 | INFO | "gpu" : " 17 GB", | 20260508 01:27:22.371 | INFO | "huge" : "", | 20260508 01:27:22.371 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:22.371 | INFO | "malloc" : " 14 GB", | 20260508 01:27:22.371 | INFO | "pinned" : " 29 GB" | 20260508 01:27:22.371 | INFO | } | 20260508 01:27:22.371 | INFO | Perf report for the last 5.14329 seconds: | 20260508 01:27:22.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:22.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.371 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:27:22.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.372 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:27:22.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:22.372 | INFO | Uploaded 36GiB to GPU in 2.4111s (14.931GiB/s) | 20260508 01:27:22.372 | INFO | Downloaded 34GiB to GPU in 2.48895s (13.6604GiB/s) | 20260508 01:27:27.503 | INFO | Analyzing chunk frames = [4155904,4156416) | 20260508 01:27:27.503 | INFO | MemInfo: { | 20260508 01:27:27.503 | INFO | "gpu" : " 17 GB", | 20260508 01:27:27.503 | INFO | "huge" : "", | 20260508 01:27:27.503 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:27.503 | INFO | "malloc" : " 14 GB", | 20260508 01:27:27.503 | INFO | "pinned" : " 29 GB" | 20260508 01:27:27.503 | INFO | } | 20260508 01:27:27.504 | INFO | Perf report for the last 5.13211 seconds: | 20260508 01:27:27.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:27.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:27:27.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 01:27:27.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:27.504 | INFO | Uploaded 36GiB to GPU in 2.41096s (14.9318GiB/s) | 20260508 01:27:27.504 | INFO | Downloaded 34GiB to GPU in 2.39656s (14.187GiB/s) | 20260508 01:27:32.644 | INFO | Analyzing chunk frames = [4156416,4156928) | 20260508 01:27:32.644 | INFO | MemInfo: { | 20260508 01:27:32.644 | INFO | "gpu" : " 17 GB", | 20260508 01:27:32.644 | INFO | "huge" : "", | 20260508 01:27:32.644 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:32.644 | INFO | "malloc" : " 14 GB", | 20260508 01:27:32.644 | INFO | "pinned" : " 29 GB" | 20260508 01:27:32.644 | INFO | } | 20260508 01:27:32.644 | INFO | Perf report for the last 5.14075 seconds: | 20260508 01:27:32.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:32.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.644 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 01:27:32.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.644 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:27:32.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:32.645 | INFO | Uploaded 36GiB to GPU in 2.53075s (14.225GiB/s) | 20260508 01:27:32.645 | INFO | Downloaded 34GiB to GPU in 2.44929s (13.8816GiB/s) | 20260508 01:27:37.791 | INFO | Analyzing chunk frames = [4156928,4157440) | 20260508 01:27:37.791 | INFO | MemInfo: { | 20260508 01:27:37.791 | INFO | "gpu" : " 17 GB", | 20260508 01:27:37.791 | INFO | "huge" : "", | 20260508 01:27:37.791 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:37.791 | INFO | "malloc" : " 14 GB", | 20260508 01:27:37.791 | INFO | "pinned" : " 29 GB" | 20260508 01:27:37.791 | INFO | } | 20260508 01:27:37.791 | INFO | Perf report for the last 5.14687 seconds: | 20260508 01:27:37.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:37.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.791 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 01:27:37.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.791 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:27:37.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:37.792 | INFO | Uploaded 36GiB to GPU in 2.54553s (14.1425GiB/s) | 20260508 01:27:37.792 | INFO | Downloaded 34GiB to GPU in 2.45376s (13.8563GiB/s) | 20260508 01:27:37.888 | INFO | Starting a round of Baz IO | 20260508 01:27:41.331 | INFO | Populating RTMetrics report | 20260508 01:27:41.331 | INFO | RTMetrics report populated | 20260508 01:27:41.331 | INFO | Opening tmp RTMetrics json file | 20260508 01:27:41.414 | INFO | Serializing RTMetrics to json | 20260508 01:27:41.471 | INFO | Serialization done, swapping tmp file now... | 20260508 01:27:41.506 | INFO | RTMetrics done | 20260508 01:27:41.530 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.642 seconds | 20260508 01:27:41.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.83% / 0.00% / 58.16% | 20260508 01:27:41.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.841888% / 3.97364e-06% / 5.96046e-05% | 20260508 01:27:41.530 | INFO | Baz IO overall: | 20260508 01:27:41.530 | INFO | BytesWritten : 1883831488742 | 20260508 01:27:41.530 | INFO | NumEvents : 911253456017 | 20260508 01:27:42.929 | INFO | Analyzing chunk frames = [4157440,4157952) | 20260508 01:27:42.929 | INFO | MemInfo: { | 20260508 01:27:42.929 | INFO | "gpu" : " 17 GB", | 20260508 01:27:42.929 | INFO | "huge" : "", | 20260508 01:27:42.929 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:42.929 | INFO | "malloc" : " 14 GB", | 20260508 01:27:42.929 | INFO | "pinned" : " 29 GB" | 20260508 01:27:42.929 | INFO | } | 20260508 01:27:42.929 | INFO | Perf report for the last 5.13824 seconds: | 20260508 01:27:42.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:42.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:42.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:42.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:42.929 | INFO | ANALYSIS : 68.5%, 3.0, 106.3ms, 0 | 20260508 01:27:42.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:42.930 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 01:27:42.930 | INFO | PRE_HQ : 65.6%, 1.6, 56.2ms, 0 | 20260508 01:27:42.930 | INFO | BAZWRITER : 70.4%, 3.8, 136.7ms, 0 | 20260508 01:27:42.930 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260508 01:27:42.930 | INFO | Uploaded 36GiB to GPU in 2.56946s (14.0107GiB/s) | 20260508 01:27:42.930 | INFO | Downloaded 38GiB to GPU in 2.69456s (14.1025GiB/s) | 20260508 01:27:48.063 | INFO | Analyzing chunk frames = [4157952,4158464) | 20260508 01:27:48.063 | INFO | MemInfo: { | 20260508 01:27:48.063 | INFO | "gpu" : " 17 GB", | 20260508 01:27:48.063 | INFO | "huge" : "", | 20260508 01:27:48.063 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:48.063 | INFO | "malloc" : " 14 GB", | 20260508 01:27:48.063 | INFO | "pinned" : " 29 GB" | 20260508 01:27:48.063 | INFO | } | 20260508 01:27:48.063 | INFO | Perf report for the last 5.13368 seconds: | 20260508 01:27:48.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:48.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.063 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:27:48.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.063 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 01:27:48.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:48.064 | INFO | Uploaded 36GiB to GPU in 2.53303s (14.2122GiB/s) | 20260508 01:27:48.064 | INFO | Downloaded 34GiB to GPU in 2.42626s (14.0133GiB/s) | 20260508 01:27:53.201 | INFO | Analyzing chunk frames = [4158464,4158976) | 20260508 01:27:53.201 | INFO | MemInfo: { | 20260508 01:27:53.201 | INFO | "gpu" : " 17 GB", | 20260508 01:27:53.201 | INFO | "huge" : "", | 20260508 01:27:53.201 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:53.201 | INFO | "malloc" : " 14 GB", | 20260508 01:27:53.201 | INFO | "pinned" : " 29 GB" | 20260508 01:27:53.201 | INFO | } | 20260508 01:27:53.201 | INFO | Perf report for the last 5.138 seconds: | 20260508 01:27:53.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:53.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.201 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:27:53.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.201 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:27:53.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:53.202 | INFO | Uploaded 36GiB to GPU in 2.51222s (14.33GiB/s) | 20260508 01:27:53.202 | INFO | Downloaded 34GiB to GPU in 2.4842s (13.6865GiB/s) | 20260508 01:27:58.340 | INFO | Analyzing chunk frames = [4158976,4159488) | 20260508 01:27:58.340 | INFO | MemInfo: { | 20260508 01:27:58.340 | INFO | "gpu" : " 17 GB", | 20260508 01:27:58.340 | INFO | "huge" : "", | 20260508 01:27:58.340 | INFO | "huge_shared" : " 29 GB", | 20260508 01:27:58.340 | INFO | "malloc" : " 14 GB", | 20260508 01:27:58.341 | INFO | "pinned" : " 29 GB" | 20260508 01:27:58.341 | INFO | } | 20260508 01:27:58.341 | INFO | Perf report for the last 5.13945 seconds: | 20260508 01:27:58.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:27:58.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:27:58.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:27:58.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:27:58.341 | INFO | Uploaded 36GiB to GPU in 2.52041s (14.2834GiB/s) | 20260508 01:27:58.341 | INFO | Downloaded 34GiB to GPU in 2.48464s (13.6841GiB/s) | 20260508 01:28:03.482 | INFO | Analyzing chunk frames = [4159488,4160000) | 20260508 01:28:03.482 | INFO | MemInfo: { | 20260508 01:28:03.482 | INFO | "gpu" : " 17 GB", | 20260508 01:28:03.482 | INFO | "huge" : "", | 20260508 01:28:03.482 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:03.482 | INFO | "malloc" : " 14 GB", | 20260508 01:28:03.482 | INFO | "pinned" : " 29 GB" | 20260508 01:28:03.482 | INFO | } | 20260508 01:28:03.482 | INFO | Perf report for the last 5.14181 seconds: | 20260508 01:28:03.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:03.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.482 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 01:28:03.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.483 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 01:28:03.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:03.483 | INFO | Uploaded 36GiB to GPU in 2.33814s (15.3969GiB/s) | 20260508 01:28:03.483 | INFO | Downloaded 34GiB to GPU in 2.47695s (13.7265GiB/s) | 20260508 01:28:08.628 | INFO | Analyzing chunk frames = [4160000,4160512) | 20260508 01:28:08.628 | INFO | MemInfo: { | 20260508 01:28:08.628 | INFO | "gpu" : " 17 GB", | 20260508 01:28:08.628 | INFO | "huge" : "", | 20260508 01:28:08.628 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:08.628 | INFO | "malloc" : " 14 GB", | 20260508 01:28:08.628 | INFO | "pinned" : " 29 GB" | 20260508 01:28:08.628 | INFO | } | 20260508 01:28:08.628 | INFO | Perf report for the last 5.14596 seconds: | 20260508 01:28:08.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:08.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.628 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:28:08.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.629 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:28:08.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:08.629 | INFO | Uploaded 36GiB to GPU in 2.42039s (14.8736GiB/s) | 20260508 01:28:08.629 | INFO | Downloaded 34GiB to GPU in 2.39901s (14.1725GiB/s) | 20260508 01:28:13.764 | INFO | Analyzing chunk frames = [4160512,4161024) | 20260508 01:28:13.764 | INFO | MemInfo: { | 20260508 01:28:13.764 | INFO | "gpu" : " 17 GB", | 20260508 01:28:13.764 | INFO | "huge" : "", | 20260508 01:28:13.764 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:13.764 | INFO | "malloc" : " 14 GB", | 20260508 01:28:13.764 | INFO | "pinned" : " 29 GB" | 20260508 01:28:13.764 | INFO | } | 20260508 01:28:13.764 | INFO | Perf report for the last 5.13552 seconds: | 20260508 01:28:13.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:13.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.764 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:28:13.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.764 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:28:13.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:13.765 | INFO | Uploaded 36GiB to GPU in 2.46065s (14.6303GiB/s) | 20260508 01:28:13.765 | INFO | Downloaded 34GiB to GPU in 2.40701s (14.1254GiB/s) | 20260508 01:28:18.897 | INFO | Analyzing chunk frames = [4161024,4161536) | 20260508 01:28:18.897 | INFO | MemInfo: { | 20260508 01:28:18.897 | INFO | "gpu" : " 17 GB", | 20260508 01:28:18.897 | INFO | "huge" : "", | 20260508 01:28:18.897 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:18.897 | INFO | "malloc" : " 14 GB", | 20260508 01:28:18.898 | INFO | "pinned" : " 29 GB" | 20260508 01:28:18.898 | INFO | } | 20260508 01:28:18.898 | INFO | Perf report for the last 5.1337 seconds: | 20260508 01:28:18.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:18.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260508 01:28:18.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:28:18.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:18.898 | INFO | Uploaded 36GiB to GPU in 2.55822s (14.0723GiB/s) | 20260508 01:28:18.898 | INFO | Downloaded 34GiB to GPU in 2.44821s (13.8877GiB/s) | 20260508 01:28:18.898 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:28:18.899 | INFO | Total mem usage: 16.7 GiB | 20260508 01:28:18.899 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:28:18.899 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.7%) | 20260508 01:28:18.899 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:28:18.899 | INFO | High Water Breakdown: | 20260508 01:28:18.899 | INFO | 59.6% : 8.67 GiB ( 84.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:28:18.899 | INFO | 40.2% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:28:18.899 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:28:18.899 | INFO | ------------------------------------------ | 20260508 01:28:18.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:28:18.899 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:28:18.899 | INFO | Total mem usage: 26.5 GiB | 20260508 01:28:18.899 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:28:18.899 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:28:18.899 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:28:18.899 | INFO | High Water Breakdown: | 20260508 01:28:18.899 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:28:18.899 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:28:18.899 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:28:18.899 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:28:18.899 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:28:18.899 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:28:18.899 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:28:18.899 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:28:18.899 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:28:18.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:28:18.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:28:18.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:28:18.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:28:18.899 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:28:18.899 | INFO | ------------------------------------------ | 20260508 01:28:18.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:28:18.899 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:28:18.899 | INFO | Total mem usage: 27.8 GiB | 20260508 01:28:18.899 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:28:18.899 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:28:18.899 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:28:18.899 | INFO | High Water Breakdown: | 20260508 01:28:18.899 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:28:18.899 | INFO | ------------------------------------------ | 20260508 01:28:18.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:28:18.899 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:28:18.899 | INFO | Total mem usage: 15.7 GiB | 20260508 01:28:18.899 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:28:18.899 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:28:18.899 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:28:18.899 | INFO | High Water Breakdown: | 20260508 01:28:18.899 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:28:18.900 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:28:18.900 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:28:18.900 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:28:18.900 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:28:18.900 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:28:18.900 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:28:18.900 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:28:18.900 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:28:18.900 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:28:18.900 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:28:18.900 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:28:18.900 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:28:18.900 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:28:18.900 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:28:18.900 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:28:18.900 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:28:18.900 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:28:18.900 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:28:18.900 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:28:18.900 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:28:18.900 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:28:18.900 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:28:18.900 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:28:18.900 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:28:18.900 | INFO | ------------------------------------------ | 20260508 01:28:18.900 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:28:18.900 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:28:18.900 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:28:18.900 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:28:18.900 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.42e+01 / 1.20e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.02e-01, 1.16e-02 / 3.80e-03 | 20260508 01:28:18.900 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.53e+00, 5.88e-01 / 1.05e-01 | 20260508 01:28:18.900 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.49e+00, 2.07e-01 / 8.93e-02 | 20260508 01:28:18.900 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.59e+01, 3.99e+00 / 1.55e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.57e+00 / 1.42e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.76e+00, 4.25e-01 / 1.59e-01 | 20260508 01:28:18.900 | INFO | PerfReport: 3.34%, Metrics -- 1.09e+00 / 1.29e+00, 2.72e-01 / 1.61e-01 | 20260508 01:28:18.900 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.71e-01, 5.30e-02 / 2.87e-02 | 20260508 01:28:18.900 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:28:18.900 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:28:18.900 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:28:18.900 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 24.48%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 24.22%, Download -- 2.58e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 30.64%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 2.37%, LockUpload -- 2.53e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 2.65%, LockDownload -- 2.83e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 15.63%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 01:28:18.900 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 01:28:19.009 | INFO | Starting a round of Baz IO | 20260508 01:28:22.464 | INFO | Populating RTMetrics report | 20260508 01:28:22.464 | INFO | RTMetrics report populated | 20260508 01:28:22.464 | INFO | Opening tmp RTMetrics json file | 20260508 01:28:22.529 | INFO | Serializing RTMetrics to json | 20260508 01:28:22.560 | INFO | Serialization done, swapping tmp file now... | 20260508 01:28:22.568 | INFO | RTMetrics done | 20260508 01:28:22.583 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.574 seconds | 20260508 01:28:22.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.83% / 0.00% / 58.17% | 20260508 01:28:22.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.841935% / 0% / 3.97364e-05% | 20260508 01:28:22.583 | INFO | Baz IO overall: | 20260508 01:28:22.583 | INFO | BytesWritten : 1885613256542 | 20260508 01:28:22.583 | INFO | NumEvents : 912108546567 | 20260508 01:28:24.042 | INFO | Analyzing chunk frames = [4161536,4162048) | 20260508 01:28:24.042 | INFO | MemInfo: { | 20260508 01:28:24.042 | INFO | "gpu" : " 17 GB", | 20260508 01:28:24.042 | INFO | "huge" : "", | 20260508 01:28:24.042 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:24.042 | INFO | "malloc" : " 14 GB", | 20260508 01:28:24.042 | INFO | "pinned" : " 29 GB" | 20260508 01:28:24.042 | INFO | } | 20260508 01:28:24.042 | INFO | Perf report for the last 5.14414 seconds: | 20260508 01:28:24.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:24.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:24.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:24.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:24.042 | INFO | ANALYSIS : 68.4%, 2.9, 104.4ms, 0 | 20260508 01:28:24.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:24.042 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260508 01:28:24.042 | INFO | PRE_HQ : 67.0%, 1.8, 62.1ms, 0 | 20260508 01:28:24.042 | INFO | BAZWRITER : 68.8%, 3.9, 138.7ms, 0 | 20260508 01:28:24.042 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260508 01:28:24.042 | INFO | Uploaded 36GiB to GPU in 2.58915s (13.9042GiB/s) | 20260508 01:28:24.042 | INFO | Downloaded 38GiB to GPU in 2.70631s (14.0412GiB/s) | 20260508 01:28:29.179 | INFO | Analyzing chunk frames = [4162048,4162560) | 20260508 01:28:29.179 | INFO | MemInfo: { | 20260508 01:28:29.179 | INFO | "gpu" : " 17 GB", | 20260508 01:28:29.179 | INFO | "huge" : "", | 20260508 01:28:29.179 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:29.179 | INFO | "malloc" : " 14 GB", | 20260508 01:28:29.179 | INFO | "pinned" : " 29 GB" | 20260508 01:28:29.179 | INFO | } | 20260508 01:28:29.179 | INFO | Perf report for the last 5.13713 seconds: | 20260508 01:28:29.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:29.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.179 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 01:28:29.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.179 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:28:29.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:29.180 | INFO | Uploaded 36GiB to GPU in 2.56971s (14.0093GiB/s) | 20260508 01:28:29.180 | INFO | Downloaded 34GiB to GPU in 2.42304s (14.032GiB/s) | 20260508 01:28:34.323 | INFO | Analyzing chunk frames = [4162560,4163072) | 20260508 01:28:34.323 | INFO | MemInfo: { | 20260508 01:28:34.323 | INFO | "gpu" : " 17 GB", | 20260508 01:28:34.323 | INFO | "huge" : "", | 20260508 01:28:34.323 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:34.323 | INFO | "malloc" : " 14 GB", | 20260508 01:28:34.323 | INFO | "pinned" : " 29 GB" | 20260508 01:28:34.323 | INFO | } | 20260508 01:28:34.323 | INFO | Perf report for the last 5.14423 seconds: | 20260508 01:28:34.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:34.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 01:28:34.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:28:34.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:34.323 | INFO | Uploaded 36GiB to GPU in 2.5609s (14.0575GiB/s) | 20260508 01:28:34.324 | INFO | Downloaded 34GiB to GPU in 2.47619s (13.7308GiB/s) | 20260508 01:28:39.455 | INFO | Analyzing chunk frames = [4163072,4163584) | 20260508 01:28:39.455 | INFO | MemInfo: { | 20260508 01:28:39.455 | INFO | "gpu" : " 17 GB", | 20260508 01:28:39.455 | INFO | "huge" : "", | 20260508 01:28:39.455 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:39.455 | INFO | "malloc" : " 14 GB", | 20260508 01:28:39.455 | INFO | "pinned" : " 29 GB" | 20260508 01:28:39.455 | INFO | } | 20260508 01:28:39.456 | INFO | Perf report for the last 5.13235 seconds: | 20260508 01:28:39.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:39.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:28:39.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:28:39.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:39.456 | INFO | Uploaded 36GiB to GPU in 2.54789s (14.1293GiB/s) | 20260508 01:28:39.456 | INFO | Downloaded 34GiB to GPU in 2.47327s (13.747GiB/s) | 20260508 01:28:44.597 | INFO | Analyzing chunk frames = [4163584,4164096) | 20260508 01:28:44.597 | INFO | MemInfo: { | 20260508 01:28:44.597 | INFO | "gpu" : " 17 GB", | 20260508 01:28:44.597 | INFO | "huge" : "", | 20260508 01:28:44.597 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:44.597 | INFO | "malloc" : " 14 GB", | 20260508 01:28:44.597 | INFO | "pinned" : " 29 GB" | 20260508 01:28:44.597 | INFO | } | 20260508 01:28:44.598 | INFO | Perf report for the last 5.14201 seconds: | 20260508 01:28:44.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:44.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 01:28:44.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 01:28:44.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:44.598 | INFO | Uploaded 36GiB to GPU in 2.42037s (14.8737GiB/s) | 20260508 01:28:44.598 | INFO | Downloaded 34GiB to GPU in 2.55728s (13.2954GiB/s) | 20260508 01:28:49.739 | INFO | Analyzing chunk frames = [4164096,4164608) | 20260508 01:28:49.739 | INFO | MemInfo: { | 20260508 01:28:49.739 | INFO | "gpu" : " 17 GB", | 20260508 01:28:49.739 | INFO | "huge" : "", | 20260508 01:28:49.739 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:49.739 | INFO | "malloc" : " 14 GB", | 20260508 01:28:49.739 | INFO | "pinned" : " 29 GB" | 20260508 01:28:49.739 | INFO | } | 20260508 01:28:49.739 | INFO | Perf report for the last 5.14118 seconds: | 20260508 01:28:49.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:49.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 01:28:49.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:28:49.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:49.739 | INFO | Uploaded 36GiB to GPU in 2.39803s (15.0123GiB/s) | 20260508 01:28:49.739 | INFO | Downloaded 34GiB to GPU in 2.40156s (14.1574GiB/s) | 20260508 01:28:54.878 | INFO | Analyzing chunk frames = [4164608,4165120) | 20260508 01:28:54.878 | INFO | MemInfo: { | 20260508 01:28:54.878 | INFO | "gpu" : " 17 GB", | 20260508 01:28:54.878 | INFO | "huge" : "", | 20260508 01:28:54.878 | INFO | "huge_shared" : " 29 GB", | 20260508 01:28:54.878 | INFO | "malloc" : " 14 GB", | 20260508 01:28:54.878 | INFO | "pinned" : " 29 GB" | 20260508 01:28:54.878 | INFO | } | 20260508 01:28:54.878 | INFO | Perf report for the last 5.13962 seconds: | 20260508 01:28:54.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:28:54.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:28:54.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 01:28:54.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:28:54.879 | INFO | Uploaded 36GiB to GPU in 2.43739s (14.7699GiB/s) | 20260508 01:28:54.879 | INFO | Downloaded 34GiB to GPU in 2.41763s (14.0633GiB/s) | 20260508 01:29:00.013 | INFO | Analyzing chunk frames = [4165120,4165632) | 20260508 01:29:00.013 | INFO | MemInfo: { | 20260508 01:29:00.013 | INFO | "gpu" : " 17 GB", | 20260508 01:29:00.013 | INFO | "huge" : "", | 20260508 01:29:00.013 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:00.013 | INFO | "malloc" : " 14 GB", | 20260508 01:29:00.013 | INFO | "pinned" : " 29 GB" | 20260508 01:29:00.013 | INFO | } | 20260508 01:29:00.013 | INFO | Perf report for the last 5.13502 seconds: | 20260508 01:29:00.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:00.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.013 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:29:00.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.014 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:29:00.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:00.014 | INFO | Uploaded 36GiB to GPU in 2.54455s (14.1479GiB/s) | 20260508 01:29:00.014 | INFO | Downloaded 34GiB to GPU in 2.45558s (13.846GiB/s) | 20260508 01:29:00.111 | INFO | Starting a round of Baz IO | 20260508 01:29:03.571 | INFO | Populating RTMetrics report | 20260508 01:29:03.572 | INFO | RTMetrics report populated | 20260508 01:29:03.572 | INFO | Opening tmp RTMetrics json file | 20260508 01:29:03.589 | INFO | Serializing RTMetrics to json | 20260508 01:29:03.592 | INFO | Serialization done, swapping tmp file now... | 20260508 01:29:03.592 | INFO | RTMetrics done | 20260508 01:29:03.665 | INFO | Finished a round of Baz IO: Wrote 1.66 GiB in 3.553 seconds | 20260508 01:29:03.665 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.83% / 0.00% / 58.17% | 20260508 01:29:03.665 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.841975% / 0% / 7.94729e-05% | 20260508 01:29:03.665 | INFO | Baz IO overall: | 20260508 01:29:03.665 | INFO | BytesWritten : 1887395611180 | 20260508 01:29:03.665 | INFO | NumEvents : 912964014149 | 20260508 01:29:05.152 | INFO | Analyzing chunk frames = [4165632,4166144) | 20260508 01:29:05.152 | INFO | MemInfo: { | 20260508 01:29:05.152 | INFO | "gpu" : " 17 GB", | 20260508 01:29:05.152 | INFO | "huge" : "", | 20260508 01:29:05.152 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:05.152 | INFO | "malloc" : " 14 GB", | 20260508 01:29:05.152 | INFO | "pinned" : " 29 GB" | 20260508 01:29:05.152 | INFO | } | 20260508 01:29:05.152 | INFO | Perf report for the last 5.13884 seconds: | 20260508 01:29:05.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:05.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:05.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:05.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:05.152 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 01:29:05.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:05.152 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 01:29:05.152 | INFO | PRE_HQ : 65.4%, 1.7, 56.0ms, 0 | 20260508 01:29:05.152 | INFO | BAZWRITER : 68.8%, 3.8, 135.9ms, 0 | 20260508 01:29:05.152 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:29:05.153 | INFO | Uploaded 36GiB to GPU in 2.5824s (13.9405GiB/s) | 20260508 01:29:05.153 | INFO | Downloaded 38GiB to GPU in 2.69863s (14.0812GiB/s) | 20260508 01:29:10.295 | INFO | Analyzing chunk frames = [4166144,4166656) | 20260508 01:29:10.295 | INFO | MemInfo: { | 20260508 01:29:10.295 | INFO | "gpu" : " 17 GB", | 20260508 01:29:10.295 | INFO | "huge" : "", | 20260508 01:29:10.295 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:10.295 | INFO | "malloc" : " 14 GB", | 20260508 01:29:10.295 | INFO | "pinned" : " 29 GB" | 20260508 01:29:10.295 | INFO | } | 20260508 01:29:10.295 | INFO | Perf report for the last 5.14316 seconds: | 20260508 01:29:10.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:10.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.296 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 01:29:10.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.296 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:29:10.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:10.296 | INFO | Uploaded 36GiB to GPU in 2.52909s (14.2344GiB/s) | 20260508 01:29:10.296 | INFO | Downloaded 34GiB to GPU in 2.42619s (14.0137GiB/s) | 20260508 01:29:15.439 | INFO | Analyzing chunk frames = [4166656,4167168) | 20260508 01:29:15.439 | INFO | MemInfo: { | 20260508 01:29:15.439 | INFO | "gpu" : " 17 GB", | 20260508 01:29:15.439 | INFO | "huge" : "", | 20260508 01:29:15.439 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:15.439 | INFO | "malloc" : " 14 GB", | 20260508 01:29:15.439 | INFO | "pinned" : " 29 GB" | 20260508 01:29:15.439 | INFO | } | 20260508 01:29:15.439 | INFO | Perf report for the last 5.1434 seconds: | 20260508 01:29:15.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:15.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 01:29:15.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:29:15.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:15.439 | INFO | Uploaded 36GiB to GPU in 2.55706s (14.0787GiB/s) | 20260508 01:29:15.439 | INFO | Downloaded 34GiB to GPU in 2.5034s (13.5815GiB/s) | 20260508 01:29:20.579 | INFO | Analyzing chunk frames = [4167168,4167680) | 20260508 01:29:20.579 | INFO | MemInfo: { | 20260508 01:29:20.579 | INFO | "gpu" : " 17 GB", | 20260508 01:29:20.579 | INFO | "huge" : "", | 20260508 01:29:20.579 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:20.579 | INFO | "malloc" : " 14 GB", | 20260508 01:29:20.579 | INFO | "pinned" : " 29 GB" | 20260508 01:29:20.579 | INFO | } | 20260508 01:29:20.579 | INFO | Perf report for the last 5.14024 seconds: | 20260508 01:29:20.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:20.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:29:20.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:29:20.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:20.579 | INFO | Uploaded 36GiB to GPU in 2.51051s (14.3397GiB/s) | 20260508 01:29:20.579 | INFO | Downloaded 34GiB to GPU in 2.50437s (13.5763GiB/s) | 20260508 01:29:25.711 | INFO | Analyzing chunk frames = [4167680,4168192) | 20260508 01:29:25.711 | INFO | MemInfo: { | 20260508 01:29:25.711 | INFO | "gpu" : " 17 GB", | 20260508 01:29:25.711 | INFO | "huge" : "", | 20260508 01:29:25.711 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:25.711 | INFO | "malloc" : " 14 GB", | 20260508 01:29:25.712 | INFO | "pinned" : " 29 GB" | 20260508 01:29:25.712 | INFO | } | 20260508 01:29:25.712 | INFO | Perf report for the last 5.13252 seconds: | 20260508 01:29:25.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:25.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 01:29:25.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:29:25.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:25.712 | INFO | Uploaded 36GiB to GPU in 2.43434s (14.7884GiB/s) | 20260508 01:29:25.712 | INFO | Downloaded 34GiB to GPU in 2.55864s (13.2883GiB/s) | 20260508 01:29:30.852 | INFO | Analyzing chunk frames = [4168192,4168704) | 20260508 01:29:30.852 | INFO | MemInfo: { | 20260508 01:29:30.852 | INFO | "gpu" : " 17 GB", | 20260508 01:29:30.852 | INFO | "huge" : "", | 20260508 01:29:30.852 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:30.852 | INFO | "malloc" : " 14 GB", | 20260508 01:29:30.852 | INFO | "pinned" : " 29 GB" | 20260508 01:29:30.852 | INFO | } | 20260508 01:29:30.853 | INFO | Perf report for the last 5.14087 seconds: | 20260508 01:29:30.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:30.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 01:29:30.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260508 01:29:30.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:30.853 | INFO | Uploaded 36GiB to GPU in 2.39561s (15.0275GiB/s) | 20260508 01:29:30.853 | INFO | Downloaded 34GiB to GPU in 2.41148s (14.0992GiB/s) | 20260508 01:29:35.990 | INFO | Analyzing chunk frames = [4168704,4169216) | 20260508 01:29:35.990 | INFO | MemInfo: { | 20260508 01:29:35.990 | INFO | "gpu" : " 17 GB", | 20260508 01:29:35.990 | INFO | "huge" : "", | 20260508 01:29:35.990 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:35.990 | INFO | "malloc" : " 14 GB", | 20260508 01:29:35.990 | INFO | "pinned" : " 29 GB" | 20260508 01:29:35.990 | INFO | } | 20260508 01:29:35.990 | INFO | Perf report for the last 5.13788 seconds: | 20260508 01:29:35.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:35.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 01:29:35.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 01:29:35.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:35.991 | INFO | Uploaded 36GiB to GPU in 2.45721s (14.6507GiB/s) | 20260508 01:29:35.991 | INFO | Downloaded 34GiB to GPU in 2.42356s (14.029GiB/s) | 20260508 01:29:41.135 | INFO | Analyzing chunk frames = [4169216,4169728) | 20260508 01:29:41.135 | INFO | MemInfo: { | 20260508 01:29:41.135 | INFO | "gpu" : " 17 GB", | 20260508 01:29:41.135 | INFO | "huge" : "", | 20260508 01:29:41.136 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:41.136 | INFO | "malloc" : " 14 GB", | 20260508 01:29:41.136 | INFO | "pinned" : " 29 GB" | 20260508 01:29:41.136 | INFO | } | 20260508 01:29:41.136 | INFO | Perf report for the last 5.14527 seconds: | 20260508 01:29:41.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:41.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 01:29:41.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 01:29:41.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:41.136 | INFO | Uploaded 36GiB to GPU in 2.54371s (14.1526GiB/s) | 20260508 01:29:41.136 | INFO | Downloaded 34GiB to GPU in 2.45157s (13.8687GiB/s) | 20260508 01:29:41.137 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:29:41.137 | INFO | Total mem usage: 16.7 GiB | 20260508 01:29:41.137 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:29:41.137 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 01:29:41.137 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:29:41.137 | INFO | High Water Breakdown: | 20260508 01:29:41.137 | INFO | 60% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:29:41.137 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:29:41.137 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:29:41.137 | INFO | ------------------------------------------ | 20260508 01:29:41.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:29:41.137 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:29:41.137 | INFO | Total mem usage: 26.5 GiB | 20260508 01:29:41.137 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:29:41.137 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 01:29:41.137 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 01:29:41.137 | INFO | High Water Breakdown: | 20260508 01:29:41.137 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:29:41.137 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:29:41.137 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:29:41.137 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:29:41.137 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:29:41.137 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:29:41.137 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:29:41.137 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:29:41.137 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:29:41.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:29:41.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:29:41.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:29:41.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:29:41.137 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:29:41.137 | INFO | ------------------------------------------ | 20260508 01:29:41.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:29:41.137 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:29:41.137 | INFO | Total mem usage: 27.8 GiB | 20260508 01:29:41.137 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:29:41.137 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:29:41.137 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:29:41.137 | INFO | High Water Breakdown: | 20260508 01:29:41.137 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:29:41.137 | INFO | ------------------------------------------ | 20260508 01:29:41.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:29:41.137 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:29:41.137 | INFO | Total mem usage: 15.7 GiB | 20260508 01:29:41.137 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:29:41.137 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:29:41.137 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:29:41.137 | INFO | High Water Breakdown: | 20260508 01:29:41.137 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:29:41.137 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:29:41.137 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:29:41.137 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:29:41.137 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:29:41.137 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:29:41.137 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:29:41.137 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:29:41.138 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:29:41.138 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:29:41.138 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:29:41.138 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:29:41.138 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:29:41.138 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:29:41.138 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:29:41.138 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:29:41.138 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:29:41.138 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:29:41.138 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:29:41.138 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:29:41.138 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:29:41.138 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:29:41.138 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:29:41.138 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:29:41.138 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:29:41.138 | INFO | ------------------------------------------ | 20260508 01:29:41.138 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:29:41.138 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:29:41.138 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:29:41.138 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:29:41.138 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.42e+01 / 1.69e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.21e-02 / 6.02e-03 | 20260508 01:29:41.138 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.43e+00, 5.88e-01 / 2.39e-01 | 20260508 01:29:41.138 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.07e-01 / 1.11e-01 | 20260508 01:29:41.138 | INFO | PerfReport: 16.10%, DME -- 4.22e+01 / 4.62e+01, 4.31e+00 / 2.04e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.66e+00 / 1.47e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.24e-01 / 2.46e-01 | 20260508 01:29:41.138 | INFO | PerfReport: 3.32%, Metrics -- 1.09e+00 / 1.25e+00, 2.72e-01 / 1.28e-01 | 20260508 01:29:41.138 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.35e-01 / 3.74e-01, 5.39e-02 / 2.82e-02 | 20260508 01:29:41.138 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:29:41.138 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:29:41.138 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:29:41.138 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 24.47%, Upload -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 24.35%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 30.75%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 2.22%, LockUpload -- 2.36e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 2.69%, LockDownload -- 2.87e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:29:41.138 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260508 01:29:41.250 | INFO | Starting a round of Baz IO | 20260508 01:29:44.715 | INFO | Populating RTMetrics report | 20260508 01:29:44.715 | INFO | RTMetrics report populated | 20260508 01:29:44.715 | INFO | Opening tmp RTMetrics json file | 20260508 01:29:44.796 | INFO | Serializing RTMetrics to json | 20260508 01:29:44.803 | INFO | Serialization done, swapping tmp file now... | 20260508 01:29:44.806 | INFO | RTMetrics done | 20260508 01:29:44.831 | INFO | Finished a round of Baz IO: Wrote 1.661 GiB in 3.58 seconds | 20260508 01:29:44.831 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.82% / 0.00% / 58.17% | 20260508 01:29:44.831 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.84155% / 3.97364e-06% / 8.34465e-05% | 20260508 01:29:44.831 | INFO | Baz IO overall: | 20260508 01:29:44.831 | INFO | BytesWritten : 1889178840336 | 20260508 01:29:44.831 | INFO | NumEvents : 913820148467 | 20260508 01:29:46.267 | INFO | Analyzing chunk frames = [4169728,4170240) | 20260508 01:29:46.267 | INFO | MemInfo: { | 20260508 01:29:46.267 | INFO | "gpu" : " 17 GB", | 20260508 01:29:46.267 | INFO | "huge" : "", | 20260508 01:29:46.267 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:46.267 | INFO | "malloc" : " 14 GB", | 20260508 01:29:46.267 | INFO | "pinned" : " 29 GB" | 20260508 01:29:46.267 | INFO | } | 20260508 01:29:46.267 | INFO | Perf report for the last 5.13134 seconds: | 20260508 01:29:46.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:46.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:46.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:46.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:46.267 | INFO | ANALYSIS : 68.8%, 2.9, 104.8ms, 0 | 20260508 01:29:46.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:46.267 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260508 01:29:46.267 | INFO | PRE_HQ : 67.5%, 1.8, 62.0ms, 0 | 20260508 01:29:46.267 | INFO | BAZWRITER : 69.4%, 4.5, 159.7ms, 0 | 20260508 01:29:46.267 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 01:29:46.268 | INFO | Uploaded 36GiB to GPU in 2.58886s (13.9058GiB/s) | 20260508 01:29:46.268 | INFO | Downloaded 38GiB to GPU in 2.69529s (14.0986GiB/s) | 20260508 01:29:51.410 | INFO | Analyzing chunk frames = [4170240,4170752) | 20260508 01:29:51.410 | INFO | MemInfo: { | 20260508 01:29:51.410 | INFO | "gpu" : " 17 GB", | 20260508 01:29:51.410 | INFO | "huge" : "", | 20260508 01:29:51.410 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:51.410 | INFO | "malloc" : " 14 GB", | 20260508 01:29:51.410 | INFO | "pinned" : " 29 GB" | 20260508 01:29:51.410 | INFO | } | 20260508 01:29:51.410 | INFO | Perf report for the last 5.14283 seconds: | 20260508 01:29:51.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:51.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 01:29:51.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:29:51.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:51.410 | INFO | Uploaded 36GiB to GPU in 2.49106s (14.4517GiB/s) | 20260508 01:29:51.410 | INFO | Downloaded 34GiB to GPU in 2.48371s (13.6892GiB/s) | 20260508 01:29:56.550 | INFO | Analyzing chunk frames = [4170752,4171264) | 20260508 01:29:56.550 | INFO | MemInfo: { | 20260508 01:29:56.550 | INFO | "gpu" : " 17 GB", | 20260508 01:29:56.550 | INFO | "huge" : "", | 20260508 01:29:56.551 | INFO | "huge_shared" : " 29 GB", | 20260508 01:29:56.551 | INFO | "malloc" : " 14 GB", | 20260508 01:29:56.551 | INFO | "pinned" : " 29 GB" | 20260508 01:29:56.551 | INFO | } | 20260508 01:29:56.551 | INFO | Perf report for the last 5.14081 seconds: | 20260508 01:29:56.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:29:56.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:29:56.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:29:56.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:29:56.551 | INFO | Uploaded 36GiB to GPU in 2.521s (14.28GiB/s) | 20260508 01:29:56.551 | INFO | Downloaded 34GiB to GPU in 2.49205s (13.6434GiB/s) | 20260508 01:30:01.693 | INFO | Analyzing chunk frames = [4171264,4171776) | 20260508 01:30:01.693 | INFO | MemInfo: { | 20260508 01:30:01.693 | INFO | "gpu" : " 17 GB", | 20260508 01:30:01.693 | INFO | "huge" : "", | 20260508 01:30:01.693 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:01.693 | INFO | "malloc" : " 14 GB", | 20260508 01:30:01.693 | INFO | "pinned" : " 29 GB" | 20260508 01:30:01.693 | INFO | } | 20260508 01:30:01.693 | INFO | Perf report for the last 5.14244 seconds: | 20260508 01:30:01.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:01.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:30:01.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:30:01.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:01.693 | INFO | Uploaded 36GiB to GPU in 2.52925s (14.2335GiB/s) | 20260508 01:30:01.694 | INFO | Downloaded 34GiB to GPU in 2.48371s (13.6892GiB/s) | 20260508 01:30:06.831 | INFO | Analyzing chunk frames = [4171776,4172288) | 20260508 01:30:06.831 | INFO | MemInfo: { | 20260508 01:30:06.831 | INFO | "gpu" : " 17 GB", | 20260508 01:30:06.831 | INFO | "huge" : "", | 20260508 01:30:06.832 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:06.832 | INFO | "malloc" : " 14 GB", | 20260508 01:30:06.832 | INFO | "pinned" : " 29 GB" | 20260508 01:30:06.832 | INFO | } | 20260508 01:30:06.832 | INFO | Perf report for the last 5.13868 seconds: | 20260508 01:30:06.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:06.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:30:06.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:30:06.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:06.832 | INFO | Uploaded 36GiB to GPU in 2.33901s (15.3911GiB/s) | 20260508 01:30:06.832 | INFO | Downloaded 34GiB to GPU in 2.49433s (13.6309GiB/s) | 20260508 01:30:11.962 | INFO | Analyzing chunk frames = [4172288,4172800) | 20260508 01:30:11.962 | INFO | MemInfo: { | 20260508 01:30:11.962 | INFO | "gpu" : " 17 GB", | 20260508 01:30:11.962 | INFO | "huge" : "", | 20260508 01:30:11.963 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:11.963 | INFO | "malloc" : " 14 GB", | 20260508 01:30:11.963 | INFO | "pinned" : " 29 GB" | 20260508 01:30:11.963 | INFO | } | 20260508 01:30:11.963 | INFO | Perf report for the last 5.13073 seconds: | 20260508 01:30:11.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:11.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 01:30:11.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 01:30:11.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:11.963 | INFO | Uploaded 36GiB to GPU in 2.4211s (14.8693GiB/s) | 20260508 01:30:11.963 | INFO | Downloaded 34GiB to GPU in 2.39884s (14.1735GiB/s) | 20260508 01:30:17.106 | INFO | Analyzing chunk frames = [4172800,4173312) | 20260508 01:30:17.106 | INFO | MemInfo: { | 20260508 01:30:17.106 | INFO | "gpu" : " 17 GB", | 20260508 01:30:17.106 | INFO | "huge" : "", | 20260508 01:30:17.106 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:17.106 | INFO | "malloc" : " 14 GB", | 20260508 01:30:17.106 | INFO | "pinned" : " 29 GB" | 20260508 01:30:17.106 | INFO | } | 20260508 01:30:17.106 | INFO | Perf report for the last 5.14372 seconds: | 20260508 01:30:17.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:17.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 01:30:17.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 01:30:17.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:17.107 | INFO | Uploaded 36GiB to GPU in 2.46269s (14.6181GiB/s) | 20260508 01:30:17.107 | INFO | Downloaded 34GiB to GPU in 2.42176s (14.0394GiB/s) | 20260508 01:30:22.250 | INFO | Analyzing chunk frames = [4173312,4173824) | 20260508 01:30:22.250 | INFO | MemInfo: { | 20260508 01:30:22.250 | INFO | "gpu" : " 17 GB", | 20260508 01:30:22.250 | INFO | "huge" : "", | 20260508 01:30:22.250 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:22.250 | INFO | "malloc" : " 14 GB", | 20260508 01:30:22.250 | INFO | "pinned" : " 29 GB" | 20260508 01:30:22.250 | INFO | } | 20260508 01:30:22.250 | INFO | Perf report for the last 5.14384 seconds: | 20260508 01:30:22.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:22.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.250 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 01:30:22.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.251 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 01:30:22.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:22.251 | INFO | Uploaded 36GiB to GPU in 2.57413s (13.9853GiB/s) | 20260508 01:30:22.251 | INFO | Downloaded 34GiB to GPU in 2.45463s (13.8514GiB/s) | 20260508 01:30:22.364 | INFO | Starting a round of Baz IO | 20260508 01:30:25.828 | INFO | Populating RTMetrics report | 20260508 01:30:25.829 | INFO | RTMetrics report populated | 20260508 01:30:25.829 | INFO | Opening tmp RTMetrics json file | 20260508 01:30:25.846 | INFO | Serializing RTMetrics to json | 20260508 01:30:25.847 | INFO | Serialization done, swapping tmp file now... | 20260508 01:30:25.848 | INFO | RTMetrics done | 20260508 01:30:25.921 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.556 seconds | 20260508 01:30:25.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.82% / 0.00% / 58.18% | 20260508 01:30:25.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.844228% / 7.94729e-06% / 7.54992e-05% | 20260508 01:30:25.921 | INFO | Baz IO overall: | 20260508 01:30:25.921 | INFO | BytesWritten : 1890963467024 | 20260508 01:30:25.921 | INFO | NumEvents : 914677472779 | 20260508 01:30:27.388 | INFO | Analyzing chunk frames = [4173824,4174336) | 20260508 01:30:27.388 | INFO | MemInfo: { | 20260508 01:30:27.388 | INFO | "gpu" : " 17 GB", | 20260508 01:30:27.388 | INFO | "huge" : "", | 20260508 01:30:27.388 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:27.388 | INFO | "malloc" : " 14 GB", | 20260508 01:30:27.388 | INFO | "pinned" : " 29 GB" | 20260508 01:30:27.388 | INFO | } | 20260508 01:30:27.388 | INFO | Perf report for the last 5.13786 seconds: | 20260508 01:30:27.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:27.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:27.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:27.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:27.388 | INFO | ANALYSIS : 69.0%, 2.9, 104.9ms, 0 | 20260508 01:30:27.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:27.388 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 01:30:27.388 | INFO | PRE_HQ : 67.6%, 1.8, 62.1ms, 0 | 20260508 01:30:27.388 | INFO | BAZWRITER : 68.9%, 3.7, 131.5ms, 0 | 20260508 01:30:27.388 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:30:27.388 | INFO | Uploaded 36GiB to GPU in 2.62132s (13.7335GiB/s) | 20260508 01:30:27.388 | INFO | Downloaded 38GiB to GPU in 2.70994s (14.0224GiB/s) | 20260508 01:30:32.527 | INFO | Analyzing chunk frames = [4174336,4174848) | 20260508 01:30:32.527 | INFO | MemInfo: { | 20260508 01:30:32.527 | INFO | "gpu" : " 17 GB", | 20260508 01:30:32.527 | INFO | "huge" : "", | 20260508 01:30:32.527 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:32.527 | INFO | "malloc" : " 14 GB", | 20260508 01:30:32.527 | INFO | "pinned" : " 29 GB" | 20260508 01:30:32.527 | INFO | } | 20260508 01:30:32.527 | INFO | Perf report for the last 5.13909 seconds: | 20260508 01:30:32.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:32.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.527 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:30:32.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.527 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:30:32.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:32.528 | INFO | Uploaded 36GiB to GPU in 2.55998s (14.0626GiB/s) | 20260508 01:30:32.528 | INFO | Downloaded 34GiB to GPU in 2.41945s (14.0528GiB/s) | 20260508 01:30:37.660 | INFO | Analyzing chunk frames = [4174848,4175360) | 20260508 01:30:37.660 | INFO | MemInfo: { | 20260508 01:30:37.660 | INFO | "gpu" : " 17 GB", | 20260508 01:30:37.660 | INFO | "huge" : "", | 20260508 01:30:37.660 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:37.660 | INFO | "malloc" : " 14 GB", | 20260508 01:30:37.660 | INFO | "pinned" : " 29 GB" | 20260508 01:30:37.660 | INFO | } | 20260508 01:30:37.660 | INFO | Perf report for the last 5.13313 seconds: | 20260508 01:30:37.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:37.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.660 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 01:30:37.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.661 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:30:37.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:37.661 | INFO | Uploaded 36GiB to GPU in 2.57931s (13.9572GiB/s) | 20260508 01:30:37.661 | INFO | Downloaded 34GiB to GPU in 2.49215s (13.6428GiB/s) | 20260508 01:30:42.805 | INFO | Analyzing chunk frames = [4175360,4175872) | 20260508 01:30:42.805 | INFO | MemInfo: { | 20260508 01:30:42.805 | INFO | "gpu" : " 17 GB", | 20260508 01:30:42.805 | INFO | "huge" : "", | 20260508 01:30:42.805 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:42.805 | INFO | "malloc" : " 14 GB", | 20260508 01:30:42.805 | INFO | "pinned" : " 29 GB" | 20260508 01:30:42.805 | INFO | } | 20260508 01:30:42.805 | INFO | Perf report for the last 5.14479 seconds: | 20260508 01:30:42.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:42.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.805 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:30:42.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.805 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:30:42.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:42.806 | INFO | Uploaded 36GiB to GPU in 2.53815s (14.1836GiB/s) | 20260508 01:30:42.806 | INFO | Downloaded 34GiB to GPU in 2.47116s (13.7587GiB/s) | 20260508 01:30:47.946 | INFO | Analyzing chunk frames = [4175872,4176384) | 20260508 01:30:47.946 | INFO | MemInfo: { | 20260508 01:30:47.946 | INFO | "gpu" : " 17 GB", | 20260508 01:30:47.946 | INFO | "huge" : "", | 20260508 01:30:47.946 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:47.946 | INFO | "malloc" : " 14 GB", | 20260508 01:30:47.946 | INFO | "pinned" : " 29 GB" | 20260508 01:30:47.946 | INFO | } | 20260508 01:30:47.946 | INFO | Perf report for the last 5.14106 seconds: | 20260508 01:30:47.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:47.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.946 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 01:30:47.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.947 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:30:47.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:47.947 | INFO | Uploaded 36GiB to GPU in 2.37711s (15.1445GiB/s) | 20260508 01:30:47.947 | INFO | Downloaded 34GiB to GPU in 2.53861s (13.3932GiB/s) | 20260508 01:30:53.083 | INFO | Analyzing chunk frames = [4176384,4176896) | 20260508 01:30:53.083 | INFO | MemInfo: { | 20260508 01:30:53.083 | INFO | "gpu" : " 17 GB", | 20260508 01:30:53.083 | INFO | "huge" : "", | 20260508 01:30:53.083 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:53.083 | INFO | "malloc" : " 14 GB", | 20260508 01:30:53.083 | INFO | "pinned" : " 29 GB" | 20260508 01:30:53.083 | INFO | } | 20260508 01:30:53.083 | INFO | Perf report for the last 5.13697 seconds: | 20260508 01:30:53.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:53.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.083 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:30:53.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.083 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:30:53.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:53.084 | INFO | Uploaded 36GiB to GPU in 2.39664s (15.021GiB/s) | 20260508 01:30:53.084 | INFO | Downloaded 34GiB to GPU in 2.4117s (14.0979GiB/s) | 20260508 01:30:58.220 | INFO | Analyzing chunk frames = [4176896,4177408) | 20260508 01:30:58.220 | INFO | MemInfo: { | 20260508 01:30:58.220 | INFO | "gpu" : " 17 GB", | 20260508 01:30:58.220 | INFO | "huge" : "", | 20260508 01:30:58.220 | INFO | "huge_shared" : " 29 GB", | 20260508 01:30:58.220 | INFO | "malloc" : " 14 GB", | 20260508 01:30:58.220 | INFO | "pinned" : " 29 GB" | 20260508 01:30:58.220 | INFO | } | 20260508 01:30:58.220 | INFO | Perf report for the last 5.13712 seconds: | 20260508 01:30:58.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:30:58.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.220 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:30:58.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.220 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 01:30:58.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:30:58.221 | INFO | Uploaded 36GiB to GPU in 2.56662s (14.0263GiB/s) | 20260508 01:30:58.221 | INFO | Downloaded 34GiB to GPU in 2.47533s (13.7355GiB/s) | 20260508 01:31:03.361 | INFO | Analyzing chunk frames = [4177408,4177920) | 20260508 01:31:03.361 | INFO | MemInfo: { | 20260508 01:31:03.361 | INFO | "gpu" : " 17 GB", | 20260508 01:31:03.361 | INFO | "huge" : "", | 20260508 01:31:03.361 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:03.361 | INFO | "malloc" : " 14 GB", | 20260508 01:31:03.361 | INFO | "pinned" : " 29 GB" | 20260508 01:31:03.362 | INFO | } | 20260508 01:31:03.362 | INFO | Perf report for the last 5.14127 seconds: | 20260508 01:31:03.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:03.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:31:03.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | PRE_HQ : 35.1%, 1.0, 18.3ms, 0 | 20260508 01:31:03.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:03.362 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:31:03.362 | INFO | Total mem usage: 16.7 GiB | 20260508 01:31:03.362 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:31:03.362 | INFO | Uploaded 36GiB to GPU in 2.53739s (14.1878GiB/s) | 20260508 01:31:03.363 | INFO | Downloaded 34GiB to GPU in 2.46086s (13.8163GiB/s) | 20260508 01:31:03.363 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.8%) | 20260508 01:31:03.363 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:31:03.363 | INFO | High Water Breakdown: | 20260508 01:31:03.363 | INFO | 60% : 8.87 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:31:03.363 | INFO | 39.9% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:31:03.363 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:31:03.363 | INFO | ------------------------------------------ | 20260508 01:31:03.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:31:03.363 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:31:03.363 | INFO | Total mem usage: 26.5 GiB | 20260508 01:31:03.363 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:31:03.363 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 01:31:03.363 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 01:31:03.363 | INFO | High Water Breakdown: | 20260508 01:31:03.363 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:31:03.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:31:03.363 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:31:03.363 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:31:03.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:31:03.363 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:31:03.363 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:31:03.363 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:31:03.363 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:31:03.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:31:03.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:31:03.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:31:03.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:31:03.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:31:03.363 | INFO | ------------------------------------------ | 20260508 01:31:03.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:31:03.363 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:31:03.363 | INFO | Total mem usage: 27.8 GiB | 20260508 01:31:03.363 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:31:03.363 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:31:03.363 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:31:03.363 | INFO | High Water Breakdown: | 20260508 01:31:03.363 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:31:03.363 | INFO | ------------------------------------------ | 20260508 01:31:03.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:31:03.363 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:31:03.363 | INFO | Total mem usage: 15.7 GiB | 20260508 01:31:03.363 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:31:03.363 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:31:03.363 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 01:31:03.363 | INFO | High Water Breakdown: | 20260508 01:31:03.363 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:31:03.363 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:31:03.363 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:31:03.363 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:31:03.363 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:31:03.363 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:31:03.363 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:31:03.364 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:31:03.364 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:31:03.364 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:31:03.364 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:31:03.364 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:31:03.364 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:31:03.364 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:31:03.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:31:03.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:31:03.364 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:31:03.364 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:31:03.364 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:31:03.364 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:31:03.364 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:31:03.364 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:31:03.364 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:31:03.364 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:31:03.364 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:31:03.364 | INFO | ------------------------------------------ | 20260508 01:31:03.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:31:03.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:31:03.364 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:31:03.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:31:03.364 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.40e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.18e-02 / 8.00e-03 | 20260508 01:31:03.364 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.46e+00, 5.79e-01 / 2.10e-01 | 20260508 01:31:03.364 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 1.06e-01 | 20260508 01:31:03.364 | INFO | PerfReport: 16.10%, DME -- 4.21e+01 / 4.61e+01, 3.97e+00 / 1.73e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.57e+00 / 1.46e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.64e+00, 4.25e-01 / 2.41e-01 | 20260508 01:31:03.364 | INFO | PerfReport: 3.34%, Metrics -- 1.09e+00 / 1.24e+00, 2.70e-01 / 1.33e-01 | 20260508 01:31:03.364 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.36e-01 / 3.66e-01, 5.47e-02 / 2.96e-02 | 20260508 01:31:03.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:31:03.364 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:31:03.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:31:03.364 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 24.52%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 2.72%, LockDownload -- 2.90e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 15.35%, LockCompute -- 1.63e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:31:03.364 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260508 01:31:03.477 | INFO | Starting a round of Baz IO | 20260508 01:31:06.927 | INFO | Populating RTMetrics report | 20260508 01:31:06.928 | INFO | RTMetrics report populated | 20260508 01:31:06.928 | INFO | Opening tmp RTMetrics json file | 20260508 01:31:06.929 | INFO | Serializing RTMetrics to json | 20260508 01:31:06.933 | INFO | Serialization done, swapping tmp file now... | 20260508 01:31:06.934 | INFO | RTMetrics done | 20260508 01:31:07.026 | INFO | Finished a round of Baz IO: Wrote 1.661 GiB in 3.548 seconds | 20260508 01:31:07.026 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.82% / 0.00% / 58.18% | 20260508 01:31:07.026 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.842182% / 3.97364e-06% / 9.53674e-05% | 20260508 01:31:07.026 | INFO | Baz IO overall: | 20260508 01:31:07.026 | INFO | BytesWritten : 1892746940164 | 20260508 01:31:07.026 | INFO | NumEvents : 915533879238 | 20260508 01:31:08.500 | INFO | Analyzing chunk frames = [4177920,4178432) | 20260508 01:31:08.500 | INFO | MemInfo: { | 20260508 01:31:08.500 | INFO | "gpu" : " 17 GB", | 20260508 01:31:08.500 | INFO | "huge" : "", | 20260508 01:31:08.500 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:08.500 | INFO | "malloc" : " 14 GB", | 20260508 01:31:08.500 | INFO | "pinned" : " 29 GB" | 20260508 01:31:08.500 | INFO | } | 20260508 01:31:08.500 | INFO | Perf report for the last 5.13884 seconds: | 20260508 01:31:08.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:08.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:08.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:08.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:08.501 | INFO | ANALYSIS : 69.0%, 2.9, 103.8ms, 0 | 20260508 01:31:08.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:08.501 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260508 01:31:08.501 | INFO | PRE_HQ : 68.0%, 1.8, 63.2ms, 0 | 20260508 01:31:08.501 | INFO | BAZWRITER : 68.8%, 4.3, 152.8ms, 0 | 20260508 01:31:08.501 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.8ms, 0 | 20260508 01:31:08.501 | INFO | Uploaded 36GiB to GPU in 2.5221s (14.2738GiB/s) | 20260508 01:31:08.501 | INFO | Downloaded 38GiB to GPU in 2.6968s (14.0908GiB/s) | 20260508 01:31:13.644 | INFO | Analyzing chunk frames = [4178432,4178944) | 20260508 01:31:13.644 | INFO | MemInfo: { | 20260508 01:31:13.645 | INFO | "gpu" : " 17 GB", | 20260508 01:31:13.645 | INFO | "huge" : "", | 20260508 01:31:13.645 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:13.645 | INFO | "malloc" : " 14 GB", | 20260508 01:31:13.645 | INFO | "pinned" : " 29 GB" | 20260508 01:31:13.645 | INFO | } | 20260508 01:31:13.645 | INFO | Perf report for the last 5.14423 seconds: | 20260508 01:31:13.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:13.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:31:13.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:31:13.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:13.645 | INFO | Uploaded 36GiB to GPU in 2.49284s (14.4414GiB/s) | 20260508 01:31:13.645 | INFO | Downloaded 34GiB to GPU in 2.43261s (13.9767GiB/s) | 20260508 01:31:18.780 | INFO | Analyzing chunk frames = [4178944,4179456) | 20260508 01:31:18.780 | INFO | MemInfo: { | 20260508 01:31:18.780 | INFO | "gpu" : " 17 GB", | 20260508 01:31:18.780 | INFO | "huge" : "", | 20260508 01:31:18.780 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:18.780 | INFO | "malloc" : " 14 GB", | 20260508 01:31:18.780 | INFO | "pinned" : " 29 GB" | 20260508 01:31:18.780 | INFO | } | 20260508 01:31:18.781 | INFO | Perf report for the last 5.13579 seconds: | 20260508 01:31:18.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:18.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:31:18.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:31:18.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:18.781 | INFO | Uploaded 36GiB to GPU in 2.5053s (14.3695GiB/s) | 20260508 01:31:18.781 | INFO | Downloaded 34GiB to GPU in 2.50696s (13.5623GiB/s) | 20260508 01:31:23.913 | INFO | Analyzing chunk frames = [4179456,4179968) | 20260508 01:31:23.914 | INFO | MemInfo: { | 20260508 01:31:23.914 | INFO | "gpu" : " 17 GB", | 20260508 01:31:23.914 | INFO | "huge" : "", | 20260508 01:31:23.914 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:23.914 | INFO | "malloc" : " 14 GB", | 20260508 01:31:23.914 | INFO | "pinned" : " 29 GB" | 20260508 01:31:23.914 | INFO | } | 20260508 01:31:23.914 | INFO | Perf report for the last 5.13323 seconds: | 20260508 01:31:23.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:23.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260508 01:31:23.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:31:23.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:23.914 | INFO | Uploaded 36GiB to GPU in 2.45836s (14.6439GiB/s) | 20260508 01:31:23.914 | INFO | Downloaded 34GiB to GPU in 2.50612s (13.5668GiB/s) | 20260508 01:31:29.059 | INFO | Analyzing chunk frames = [4179968,4180480) | 20260508 01:31:29.059 | INFO | MemInfo: { | 20260508 01:31:29.059 | INFO | "gpu" : " 17 GB", | 20260508 01:31:29.059 | INFO | "huge" : "", | 20260508 01:31:29.059 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:29.059 | INFO | "malloc" : " 14 GB", | 20260508 01:31:29.059 | INFO | "pinned" : " 29 GB" | 20260508 01:31:29.059 | INFO | } | 20260508 01:31:29.059 | INFO | Perf report for the last 5.14526 seconds: | 20260508 01:31:29.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:29.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 01:31:29.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:31:29.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:29.059 | INFO | Uploaded 36GiB to GPU in 2.27815s (15.8023GiB/s) | 20260508 01:31:29.060 | INFO | Downloaded 34GiB to GPU in 2.49916s (13.6046GiB/s) | 20260508 01:31:34.196 | INFO | Analyzing chunk frames = [4180480,4180992) | 20260508 01:31:34.196 | INFO | MemInfo: { | 20260508 01:31:34.196 | INFO | "gpu" : " 17 GB", | 20260508 01:31:34.197 | INFO | "huge" : "", | 20260508 01:31:34.197 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:34.197 | INFO | "malloc" : " 14 GB", | 20260508 01:31:34.197 | INFO | "pinned" : " 29 GB" | 20260508 01:31:34.197 | INFO | } | 20260508 01:31:34.197 | INFO | Perf report for the last 5.13765 seconds: | 20260508 01:31:34.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:34.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:31:34.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:31:34.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:34.197 | INFO | Uploaded 36GiB to GPU in 2.36022s (15.2528GiB/s) | 20260508 01:31:34.197 | INFO | Downloaded 34GiB to GPU in 2.43187s (13.981GiB/s) | 20260508 01:31:39.336 | INFO | Analyzing chunk frames = [4180992,4181504) | 20260508 01:31:39.336 | INFO | MemInfo: { | 20260508 01:31:39.336 | INFO | "gpu" : " 17 GB", | 20260508 01:31:39.336 | INFO | "huge" : "", | 20260508 01:31:39.336 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:39.336 | INFO | "malloc" : " 14 GB", | 20260508 01:31:39.336 | INFO | "pinned" : " 29 GB" | 20260508 01:31:39.337 | INFO | } | 20260508 01:31:39.337 | INFO | Perf report for the last 5.13984 seconds: | 20260508 01:31:39.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:39.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:31:39.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:31:39.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:39.337 | INFO | Uploaded 36GiB to GPU in 2.53422s (14.2056GiB/s) | 20260508 01:31:39.337 | INFO | Downloaded 34GiB to GPU in 2.48195s (13.6989GiB/s) | 20260508 01:31:44.473 | INFO | Analyzing chunk frames = [4181504,4182016) | 20260508 01:31:44.473 | INFO | MemInfo: { | 20260508 01:31:44.473 | INFO | "gpu" : " 17 GB", | 20260508 01:31:44.473 | INFO | "huge" : "", | 20260508 01:31:44.473 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:44.474 | INFO | "malloc" : " 14 GB", | 20260508 01:31:44.474 | INFO | "pinned" : " 29 GB" | 20260508 01:31:44.474 | INFO | } | 20260508 01:31:44.474 | INFO | Perf report for the last 5.13703 seconds: | 20260508 01:31:44.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:44.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 01:31:44.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:31:44.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:44.474 | INFO | Uploaded 36GiB to GPU in 2.5081s (14.3535GiB/s) | 20260508 01:31:44.474 | INFO | Downloaded 34GiB to GPU in 2.46026s (13.8197GiB/s) | 20260508 01:31:44.578 | INFO | Starting a round of Baz IO | 20260508 01:31:48.023 | INFO | Populating RTMetrics report | 20260508 01:31:48.023 | INFO | RTMetrics report populated | 20260508 01:31:48.023 | INFO | Opening tmp RTMetrics json file | 20260508 01:31:48.040 | INFO | Serializing RTMetrics to json | 20260508 01:31:48.048 | INFO | Serialization done, swapping tmp file now... | 20260508 01:31:48.048 | INFO | RTMetrics done | 20260508 01:31:48.116 | INFO | Finished a round of Baz IO: Wrote 1.66 GiB in 3.538 seconds | 20260508 01:31:48.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.81% / 0.00% / 58.18% | 20260508 01:31:48.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.839849% / 1.58946e-05% / 6.35783e-05% | 20260508 01:31:48.116 | INFO | Baz IO overall: | 20260508 01:31:48.116 | INFO | BytesWritten : 1894529406023 | 20260508 01:31:48.116 | INFO | NumEvents : 916389667105 | 20260508 01:31:49.614 | INFO | Analyzing chunk frames = [4182016,4182528) | 20260508 01:31:49.614 | INFO | MemInfo: { | 20260508 01:31:49.614 | INFO | "gpu" : " 17 GB", | 20260508 01:31:49.614 | INFO | "huge" : "", | 20260508 01:31:49.614 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:49.614 | INFO | "malloc" : " 14 GB", | 20260508 01:31:49.614 | INFO | "pinned" : " 29 GB" | 20260508 01:31:49.614 | INFO | } | 20260508 01:31:49.614 | INFO | Perf report for the last 5.14055 seconds: | 20260508 01:31:49.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:49.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:49.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:49.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:49.614 | INFO | ANALYSIS : 68.3%, 3.0, 106.1ms, 0 | 20260508 01:31:49.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:49.614 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:31:49.614 | INFO | PRE_HQ : 65.5%, 1.6, 56.1ms, 0 | 20260508 01:31:49.614 | INFO | BAZWRITER : 68.8%, 4.3, 152.6ms, 0 | 20260508 01:31:49.614 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:31:49.615 | INFO | Uploaded 36GiB to GPU in 2.55344s (14.0986GiB/s) | 20260508 01:31:49.615 | INFO | Downloaded 38GiB to GPU in 2.68976s (14.1276GiB/s) | 20260508 01:31:54.754 | INFO | Analyzing chunk frames = [4182528,4183040) | 20260508 01:31:54.754 | INFO | MemInfo: { | 20260508 01:31:54.754 | INFO | "gpu" : " 17 GB", | 20260508 01:31:54.754 | INFO | "huge" : "", | 20260508 01:31:54.754 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:54.754 | INFO | "malloc" : " 14 GB", | 20260508 01:31:54.754 | INFO | "pinned" : " 29 GB" | 20260508 01:31:54.754 | INFO | } | 20260508 01:31:54.755 | INFO | Perf report for the last 5.14039 seconds: | 20260508 01:31:54.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:54.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:31:54.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:31:54.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:54.755 | INFO | Uploaded 36GiB to GPU in 2.55465s (14.092GiB/s) | 20260508 01:31:54.755 | INFO | Downloaded 34GiB to GPU in 2.42776s (14.0047GiB/s) | 20260508 01:31:59.894 | INFO | Analyzing chunk frames = [4183040,4183552) | 20260508 01:31:59.894 | INFO | MemInfo: { | 20260508 01:31:59.894 | INFO | "gpu" : " 17 GB", | 20260508 01:31:59.894 | INFO | "huge" : "", | 20260508 01:31:59.894 | INFO | "huge_shared" : " 29 GB", | 20260508 01:31:59.894 | INFO | "malloc" : " 14 GB", | 20260508 01:31:59.894 | INFO | "pinned" : " 29 GB" | 20260508 01:31:59.894 | INFO | } | 20260508 01:31:59.894 | INFO | Perf report for the last 5.1393 seconds: | 20260508 01:31:59.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:31:59.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:31:59.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:31:59.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:31:59.894 | INFO | Uploaded 36GiB to GPU in 2.49157s (14.4487GiB/s) | 20260508 01:31:59.894 | INFO | Downloaded 34GiB to GPU in 2.48597s (13.6767GiB/s) | 20260508 01:32:05.028 | INFO | Analyzing chunk frames = [4183552,4184064) | 20260508 01:32:05.028 | INFO | MemInfo: { | 20260508 01:32:05.028 | INFO | "gpu" : " 17 GB", | 20260508 01:32:05.028 | INFO | "huge" : "", | 20260508 01:32:05.028 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:05.029 | INFO | "malloc" : " 14 GB", | 20260508 01:32:05.029 | INFO | "pinned" : " 29 GB" | 20260508 01:32:05.029 | INFO | } | 20260508 01:32:05.029 | INFO | Perf report for the last 5.13474 seconds: | 20260508 01:32:05.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:05.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260508 01:32:05.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:32:05.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:05.029 | INFO | Uploaded 36GiB to GPU in 2.51526s (14.3126GiB/s) | 20260508 01:32:05.029 | INFO | Downloaded 34GiB to GPU in 2.48697s (13.6713GiB/s) | 20260508 01:32:10.169 | INFO | Analyzing chunk frames = [4184064,4184576) | 20260508 01:32:10.169 | INFO | MemInfo: { | 20260508 01:32:10.169 | INFO | "gpu" : " 17 GB", | 20260508 01:32:10.169 | INFO | "huge" : "", | 20260508 01:32:10.169 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:10.169 | INFO | "malloc" : " 14 GB", | 20260508 01:32:10.169 | INFO | "pinned" : " 29 GB" | 20260508 01:32:10.169 | INFO | } | 20260508 01:32:10.170 | INFO | Perf report for the last 5.14094 seconds: | 20260508 01:32:10.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:10.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:32:10.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 01:32:10.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:10.170 | INFO | Uploaded 36GiB to GPU in 2.42915s (14.82GiB/s) | 20260508 01:32:10.170 | INFO | Downloaded 34GiB to GPU in 2.50098s (13.5947GiB/s) | 20260508 01:32:15.313 | INFO | Analyzing chunk frames = [4184576,4185088) | 20260508 01:32:15.313 | INFO | MemInfo: { | 20260508 01:32:15.313 | INFO | "gpu" : " 17 GB", | 20260508 01:32:15.313 | INFO | "huge" : "", | 20260508 01:32:15.313 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:15.313 | INFO | "malloc" : " 14 GB", | 20260508 01:32:15.313 | INFO | "pinned" : " 29 GB" | 20260508 01:32:15.313 | INFO | } | 20260508 01:32:15.313 | INFO | Perf report for the last 5.14378 seconds: | 20260508 01:32:15.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:15.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 01:32:15.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260508 01:32:15.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:15.314 | INFO | Uploaded 36GiB to GPU in 2.44193s (14.7424GiB/s) | 20260508 01:32:15.314 | INFO | Downloaded 34GiB to GPU in 2.3985s (14.1755GiB/s) | 20260508 01:32:20.457 | INFO | Analyzing chunk frames = [4185088,4185600) | 20260508 01:32:20.457 | INFO | MemInfo: { | 20260508 01:32:20.457 | INFO | "gpu" : " 17 GB", | 20260508 01:32:20.457 | INFO | "huge" : "", | 20260508 01:32:20.457 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:20.457 | INFO | "malloc" : " 14 GB", | 20260508 01:32:20.458 | INFO | "pinned" : " 29 GB" | 20260508 01:32:20.458 | INFO | } | 20260508 01:32:20.458 | INFO | Perf report for the last 5.14421 seconds: | 20260508 01:32:20.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:20.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 01:32:20.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260508 01:32:20.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:20.458 | INFO | Uploaded 36GiB to GPU in 2.501s (14.3942GiB/s) | 20260508 01:32:20.458 | INFO | Downloaded 34GiB to GPU in 2.4236s (14.0287GiB/s) | 20260508 01:32:25.593 | INFO | Analyzing chunk frames = [4185600,4186112) | 20260508 01:32:25.593 | INFO | MemInfo: { | 20260508 01:32:25.593 | INFO | "gpu" : " 17 GB", | 20260508 01:32:25.593 | INFO | "huge" : "", | 20260508 01:32:25.593 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:25.593 | INFO | "malloc" : " 14 GB", | 20260508 01:32:25.593 | INFO | "pinned" : " 29 GB" | 20260508 01:32:25.593 | INFO | } | 20260508 01:32:25.593 | INFO | Perf report for the last 5.13579 seconds: | 20260508 01:32:25.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:25.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.593 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:32:25.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.594 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:32:25.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:25.594 | INFO | Uploaded 36GiB to GPU in 2.56865s (14.0151GiB/s) | 20260508 01:32:25.594 | INFO | Downloaded 34GiB to GPU in 2.4489s (13.8838GiB/s) | 20260508 01:32:25.594 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:32:25.594 | INFO | Total mem usage: 16.7 GiB | 20260508 01:32:25.594 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:32:25.594 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -10.1%) | 20260508 01:32:25.594 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:32:25.594 | INFO | High Water Breakdown: | 20260508 01:32:25.594 | INFO | 60.4% : 9.16 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:32:25.594 | INFO | 39.4% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:32:25.594 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:32:25.594 | INFO | ------------------------------------------ | 20260508 01:32:25.594 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:32:25.594 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:32:25.594 | INFO | Total mem usage: 26.5 GiB | 20260508 01:32:25.594 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:32:25.594 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 01:32:25.594 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:32:25.594 | INFO | High Water Breakdown: | 20260508 01:32:25.594 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:32:25.594 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:32:25.594 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:32:25.594 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:32:25.594 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:32:25.594 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:32:25.594 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:32:25.594 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:32:25.594 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:32:25.594 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:32:25.594 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:32:25.594 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:32:25.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:32:25.595 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:32:25.595 | INFO | ------------------------------------------ | 20260508 01:32:25.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:32:25.595 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:32:25.595 | INFO | Total mem usage: 27.8 GiB | 20260508 01:32:25.595 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:32:25.595 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:32:25.595 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:32:25.595 | INFO | High Water Breakdown: | 20260508 01:32:25.595 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:32:25.595 | INFO | ------------------------------------------ | 20260508 01:32:25.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:32:25.595 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:32:25.595 | INFO | Total mem usage: 15.7 GiB | 20260508 01:32:25.595 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:32:25.595 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:32:25.595 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:32:25.595 | INFO | High Water Breakdown: | 20260508 01:32:25.595 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:32:25.595 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:32:25.595 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:32:25.595 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:32:25.595 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:32:25.595 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:32:25.595 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:32:25.595 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:32:25.595 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:32:25.595 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:32:25.595 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:32:25.595 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:32:25.595 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:32:25.595 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:32:25.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:32:25.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:32:25.595 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:32:25.595 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:32:25.595 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:32:25.595 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:32:25.595 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:32:25.595 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:32:25.595 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:32:25.595 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:32:25.595 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:32:25.595 | INFO | ------------------------------------------ | 20260508 01:32:25.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:32:25.595 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:32:25.595 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:32:25.595 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:32:25.595 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.23e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.14e-02 / 4.59e-03 | 20260508 01:32:25.595 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.48e+00, 5.88e-01 / 1.71e-01 | 20260508 01:32:25.595 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 1.01e-01 | 20260508 01:32:25.595 | INFO | PerfReport: 16.03%, DME -- 4.20e+01 / 4.59e+01, 4.01e+00 / 1.61e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.66e+00 / 1.45e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.72e+00, 4.23e-01 / 2.00e-01 | 20260508 01:32:25.595 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.28e+00, 2.72e-01 / 1.23e-01 | 20260508 01:32:25.595 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.72e-01, 5.27e-02 / 2.26e-02 | 20260508 01:32:25.595 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:32:25.595 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:32:25.595 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:32:25.595 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 2.08%, LockUpload -- 2.21e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 2.87%, LockDownload -- 3.06e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 15.49%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:32:25.595 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260508 01:32:25.691 | INFO | Starting a round of Baz IO | 20260508 01:32:29.136 | INFO | Populating RTMetrics report | 20260508 01:32:29.136 | INFO | RTMetrics report populated | 20260508 01:32:29.136 | INFO | Opening tmp RTMetrics json file | 20260508 01:32:29.161 | INFO | Serializing RTMetrics to json | 20260508 01:32:29.163 | INFO | Serialization done, swapping tmp file now... | 20260508 01:32:29.164 | INFO | RTMetrics done | 20260508 01:32:29.249 | INFO | Finished a round of Baz IO: Wrote 1.661 GiB in 3.558 seconds | 20260508 01:32:29.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.81% / 0.00% / 58.19% | 20260508 01:32:29.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.837926% / 3.97364e-06% / 4.37101e-05% | 20260508 01:32:29.249 | INFO | Baz IO overall: | 20260508 01:32:29.249 | INFO | BytesWritten : 1896312462675 | 20260508 01:32:29.249 | INFO | NumEvents : 917245997219 | 20260508 01:32:30.728 | INFO | Analyzing chunk frames = [4186112,4186624) | 20260508 01:32:30.728 | INFO | MemInfo: { | 20260508 01:32:30.728 | INFO | "gpu" : " 17 GB", | 20260508 01:32:30.728 | INFO | "huge" : "", | 20260508 01:32:30.728 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:30.728 | INFO | "malloc" : " 14 GB", | 20260508 01:32:30.728 | INFO | "pinned" : " 29 GB" | 20260508 01:32:30.728 | INFO | } | 20260508 01:32:30.728 | INFO | Perf report for the last 5.13496 seconds: | 20260508 01:32:30.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:30.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:30.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:30.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:30.728 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260508 01:32:30.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:30.729 | INFO | RT_METRICS : 40.5%, 1.0, 21.2ms, 0 | 20260508 01:32:30.729 | INFO | PRE_HQ : 65.9%, 1.6, 56.4ms, 0 | 20260508 01:32:30.729 | INFO | BAZWRITER : 68.9%, 3.8, 135.1ms, 0 | 20260508 01:32:30.729 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:32:30.729 | INFO | Uploaded 36GiB to GPU in 2.58732s (13.914GiB/s) | 20260508 01:32:30.729 | INFO | Downloaded 38GiB to GPU in 2.68791s (14.1374GiB/s) | 20260508 01:32:35.869 | INFO | Analyzing chunk frames = [4186624,4187136) | 20260508 01:32:35.869 | INFO | MemInfo: { | 20260508 01:32:35.869 | INFO | "gpu" : " 17 GB", | 20260508 01:32:35.869 | INFO | "huge" : "", | 20260508 01:32:35.869 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:35.869 | INFO | "malloc" : " 14 GB", | 20260508 01:32:35.869 | INFO | "pinned" : " 29 GB" | 20260508 01:32:35.869 | INFO | } | 20260508 01:32:35.869 | INFO | Perf report for the last 5.14047 seconds: | 20260508 01:32:35.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:35.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:32:35.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:32:35.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:35.869 | INFO | Uploaded 36GiB to GPU in 2.52156s (14.2769GiB/s) | 20260508 01:32:35.869 | INFO | Downloaded 34GiB to GPU in 2.42579s (14.0161GiB/s) | 20260508 01:32:41.010 | INFO | Analyzing chunk frames = [4187136,4187648) | 20260508 01:32:41.010 | INFO | MemInfo: { | 20260508 01:32:41.010 | INFO | "gpu" : " 17 GB", | 20260508 01:32:41.010 | INFO | "huge" : "", | 20260508 01:32:41.010 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:41.010 | INFO | "malloc" : " 14 GB", | 20260508 01:32:41.010 | INFO | "pinned" : " 29 GB" | 20260508 01:32:41.010 | INFO | } | 20260508 01:32:41.010 | INFO | Perf report for the last 5.14133 seconds: | 20260508 01:32:41.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:41.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.010 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:32:41.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.010 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:32:41.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:41.011 | INFO | Uploaded 36GiB to GPU in 2.51102s (14.3368GiB/s) | 20260508 01:32:41.011 | INFO | Downloaded 34GiB to GPU in 2.49806s (13.6105GiB/s) | 20260508 01:32:46.148 | INFO | Analyzing chunk frames = [4187648,4188160) | 20260508 01:32:46.148 | INFO | MemInfo: { | 20260508 01:32:46.148 | INFO | "gpu" : " 17 GB", | 20260508 01:32:46.148 | INFO | "huge" : "", | 20260508 01:32:46.148 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:46.148 | INFO | "malloc" : " 14 GB", | 20260508 01:32:46.148 | INFO | "pinned" : " 29 GB" | 20260508 01:32:46.148 | INFO | } | 20260508 01:32:46.149 | INFO | Perf report for the last 5.13831 seconds: | 20260508 01:32:46.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:46.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:32:46.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:32:46.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:46.149 | INFO | Uploaded 36GiB to GPU in 2.52258s (14.2711GiB/s) | 20260508 01:32:46.149 | INFO | Downloaded 34GiB to GPU in 2.50031s (13.5983GiB/s) | 20260508 01:32:51.286 | INFO | Analyzing chunk frames = [4188160,4188672) | 20260508 01:32:51.286 | INFO | MemInfo: { | 20260508 01:32:51.286 | INFO | "gpu" : " 17 GB", | 20260508 01:32:51.286 | INFO | "huge" : "", | 20260508 01:32:51.287 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:51.287 | INFO | "malloc" : " 14 GB", | 20260508 01:32:51.287 | INFO | "pinned" : " 29 GB" | 20260508 01:32:51.287 | INFO | } | 20260508 01:32:51.287 | INFO | Perf report for the last 5.13816 seconds: | 20260508 01:32:51.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:51.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:32:51.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 01:32:51.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:51.287 | INFO | Uploaded 36GiB to GPU in 2.32175s (15.5055GiB/s) | 20260508 01:32:51.287 | INFO | Downloaded 34GiB to GPU in 2.49502s (13.6272GiB/s) | 20260508 01:32:56.426 | INFO | Analyzing chunk frames = [4188672,4189184) | 20260508 01:32:56.426 | INFO | MemInfo: { | 20260508 01:32:56.426 | INFO | "gpu" : " 17 GB", | 20260508 01:32:56.426 | INFO | "huge" : "", | 20260508 01:32:56.426 | INFO | "huge_shared" : " 29 GB", | 20260508 01:32:56.426 | INFO | "malloc" : " 14 GB", | 20260508 01:32:56.426 | INFO | "pinned" : " 29 GB" | 20260508 01:32:56.426 | INFO | } | 20260508 01:32:56.426 | INFO | Perf report for the last 5.13925 seconds: | 20260508 01:32:56.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:32:56.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:32:56.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:32:56.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:32:56.426 | INFO | Uploaded 36GiB to GPU in 2.3495s (15.3224GiB/s) | 20260508 01:32:56.426 | INFO | Downloaded 34GiB to GPU in 2.42349s (14.0293GiB/s) | 20260508 01:33:01.565 | INFO | Analyzing chunk frames = [4189184,4189696) | 20260508 01:33:01.565 | INFO | MemInfo: { | 20260508 01:33:01.565 | INFO | "gpu" : " 17 GB", | 20260508 01:33:01.565 | INFO | "huge" : "", | 20260508 01:33:01.565 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:01.565 | INFO | "malloc" : " 14 GB", | 20260508 01:33:01.565 | INFO | "pinned" : " 29 GB" | 20260508 01:33:01.565 | INFO | } | 20260508 01:33:01.565 | INFO | Perf report for the last 5.13939 seconds: | 20260508 01:33:01.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:01.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.565 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 01:33:01.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.565 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:33:01.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:01.566 | INFO | Uploaded 36GiB to GPU in 2.40076s (14.9952GiB/s) | 20260508 01:33:01.566 | INFO | Downloaded 34GiB to GPU in 2.43984s (13.9353GiB/s) | 20260508 01:33:06.709 | INFO | Analyzing chunk frames = [4189696,4190208) | 20260508 01:33:06.709 | INFO | MemInfo: { | 20260508 01:33:06.709 | INFO | "gpu" : " 17 GB", | 20260508 01:33:06.709 | INFO | "huge" : "", | 20260508 01:33:06.709 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:06.709 | INFO | "malloc" : " 14 GB", | 20260508 01:33:06.710 | INFO | "pinned" : " 29 GB" | 20260508 01:33:06.710 | INFO | } | 20260508 01:33:06.710 | INFO | Perf report for the last 5.14419 seconds: | 20260508 01:33:06.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:06.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 01:33:06.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:33:06.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:06.710 | INFO | Uploaded 36GiB to GPU in 2.51521s (14.3129GiB/s) | 20260508 01:33:06.710 | INFO | Downloaded 34GiB to GPU in 2.4651s (13.7925GiB/s) | 20260508 01:33:06.807 | INFO | Starting a round of Baz IO | 20260508 01:33:10.254 | INFO | Populating RTMetrics report | 20260508 01:33:10.254 | INFO | RTMetrics report populated | 20260508 01:33:10.254 | INFO | Opening tmp RTMetrics json file | 20260508 01:33:10.268 | INFO | Serializing RTMetrics to json | 20260508 01:33:10.276 | INFO | Serialization done, swapping tmp file now... | 20260508 01:33:10.277 | INFO | RTMetrics done | 20260508 01:33:10.344 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.537 seconds | 20260508 01:33:10.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.80% / 0.00% / 58.19% | 20260508 01:33:10.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.833412% / 3.97364e-06% / 6.75519e-05% | 20260508 01:33:10.344 | INFO | Baz IO overall: | 20260508 01:33:10.344 | INFO | BytesWritten : 1898092309844 | 20260508 01:33:10.344 | INFO | NumEvents : 918099765736 | 20260508 01:33:11.845 | INFO | Analyzing chunk frames = [4190208,4190720) | 20260508 01:33:11.845 | INFO | MemInfo: { | 20260508 01:33:11.845 | INFO | "gpu" : " 17 GB", | 20260508 01:33:11.845 | INFO | "huge" : "", | 20260508 01:33:11.845 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:11.845 | INFO | "malloc" : " 14 GB", | 20260508 01:33:11.845 | INFO | "pinned" : " 29 GB" | 20260508 01:33:11.845 | INFO | } | 20260508 01:33:11.845 | INFO | Perf report for the last 5.13572 seconds: | 20260508 01:33:11.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:11.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:11.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:11.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:11.845 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 01:33:11.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:11.845 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 01:33:11.845 | INFO | PRE_HQ : 65.8%, 1.6, 56.0ms, 0 | 20260508 01:33:11.845 | INFO | BAZWRITER : 68.5%, 3.5, 124.5ms, 0 | 20260508 01:33:11.845 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:33:11.846 | INFO | Uploaded 36GiB to GPU in 2.56508s (14.0346GiB/s) | 20260508 01:33:11.846 | INFO | Downloaded 38GiB to GPU in 2.70869s (14.0289GiB/s) | 20260508 01:33:16.981 | INFO | Analyzing chunk frames = [4190720,4191232) | 20260508 01:33:16.981 | INFO | MemInfo: { | 20260508 01:33:16.981 | INFO | "gpu" : " 17 GB", | 20260508 01:33:16.981 | INFO | "huge" : "", | 20260508 01:33:16.981 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:16.981 | INFO | "malloc" : " 14 GB", | 20260508 01:33:16.981 | INFO | "pinned" : " 29 GB" | 20260508 01:33:16.981 | INFO | } | 20260508 01:33:16.981 | INFO | Perf report for the last 5.13568 seconds: | 20260508 01:33:16.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:16.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:33:16.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:33:16.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:16.981 | INFO | Uploaded 36GiB to GPU in 2.50829s (14.3524GiB/s) | 20260508 01:33:16.981 | INFO | Downloaded 34GiB to GPU in 2.43694s (13.9519GiB/s) | 20260508 01:33:22.120 | INFO | Analyzing chunk frames = [4191232,4191744) | 20260508 01:33:22.120 | INFO | MemInfo: { | 20260508 01:33:22.120 | INFO | "gpu" : " 17 GB", | 20260508 01:33:22.120 | INFO | "huge" : "", | 20260508 01:33:22.120 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:22.120 | INFO | "malloc" : " 14 GB", | 20260508 01:33:22.120 | INFO | "pinned" : " 29 GB" | 20260508 01:33:22.121 | INFO | } | 20260508 01:33:22.121 | INFO | Perf report for the last 5.1396 seconds: | 20260508 01:33:22.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:22.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 01:33:22.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:33:22.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:22.121 | INFO | Uploaded 36GiB to GPU in 2.45108s (14.6874GiB/s) | 20260508 01:33:22.121 | INFO | Downloaded 34GiB to GPU in 2.4983s (13.6092GiB/s) | 20260508 01:33:27.263 | INFO | Analyzing chunk frames = [4191744,4192256) | 20260508 01:33:27.263 | INFO | MemInfo: { | 20260508 01:33:27.263 | INFO | "gpu" : " 17 GB", | 20260508 01:33:27.264 | INFO | "huge" : "", | 20260508 01:33:27.264 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:27.264 | INFO | "malloc" : " 14 GB", | 20260508 01:33:27.264 | INFO | "pinned" : " 29 GB" | 20260508 01:33:27.264 | INFO | } | 20260508 01:33:27.264 | INFO | Perf report for the last 5.14312 seconds: | 20260508 01:33:27.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:27.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:33:27.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 01:33:27.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:27.264 | INFO | Uploaded 36GiB to GPU in 2.48262s (14.5008GiB/s) | 20260508 01:33:27.264 | INFO | Downloaded 34GiB to GPU in 2.49999s (13.6001GiB/s) | 20260508 01:33:32.408 | INFO | Analyzing chunk frames = [4192256,4192768) | 20260508 01:33:32.408 | INFO | MemInfo: { | 20260508 01:33:32.408 | INFO | "gpu" : " 17 GB", | 20260508 01:33:32.408 | INFO | "huge" : "", | 20260508 01:33:32.408 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:32.408 | INFO | "malloc" : " 14 GB", | 20260508 01:33:32.408 | INFO | "pinned" : " 29 GB" | 20260508 01:33:32.408 | INFO | } | 20260508 01:33:32.408 | INFO | Perf report for the last 5.14456 seconds: | 20260508 01:33:32.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:32.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.408 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 01:33:32.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.408 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:33:32.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:32.409 | INFO | Uploaded 36GiB to GPU in 2.32838s (15.4614GiB/s) | 20260508 01:33:32.409 | INFO | Downloaded 34GiB to GPU in 2.52641s (13.4578GiB/s) | 20260508 01:33:37.545 | INFO | Analyzing chunk frames = [4192768,4193280) | 20260508 01:33:37.545 | INFO | MemInfo: { | 20260508 01:33:37.545 | INFO | "gpu" : " 17 GB", | 20260508 01:33:37.545 | INFO | "huge" : "", | 20260508 01:33:37.545 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:37.545 | INFO | "malloc" : " 14 GB", | 20260508 01:33:37.545 | INFO | "pinned" : " 29 GB" | 20260508 01:33:37.545 | INFO | } | 20260508 01:33:37.545 | INFO | Perf report for the last 5.13695 seconds: | 20260508 01:33:37.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:37.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.545 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:33:37.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.545 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:33:37.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:37.546 | INFO | Uploaded 36GiB to GPU in 2.36921s (15.1949GiB/s) | 20260508 01:33:37.546 | INFO | Downloaded 34GiB to GPU in 2.43689s (13.9522GiB/s) | 20260508 01:33:42.678 | INFO | Analyzing chunk frames = [4193280,4193792) | 20260508 01:33:42.678 | INFO | MemInfo: { | 20260508 01:33:42.678 | INFO | "gpu" : " 17 GB", | 20260508 01:33:42.678 | INFO | "huge" : "", | 20260508 01:33:42.678 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:42.678 | INFO | "malloc" : " 14 GB", | 20260508 01:33:42.678 | INFO | "pinned" : " 29 GB" | 20260508 01:33:42.678 | INFO | } | 20260508 01:33:42.679 | INFO | Perf report for the last 5.1332 seconds: | 20260508 01:33:42.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:42.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:33:42.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:33:42.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:42.679 | INFO | Uploaded 36GiB to GPU in 2.40886s (14.9448GiB/s) | 20260508 01:33:42.679 | INFO | Downloaded 34GiB to GPU in 2.43773s (13.9474GiB/s) | 20260508 01:33:47.819 | INFO | Analyzing chunk frames = [4193792,4194304) | 20260508 01:33:47.819 | INFO | MemInfo: { | 20260508 01:33:47.819 | INFO | "gpu" : " 17 GB", | 20260508 01:33:47.819 | INFO | "huge" : "", | 20260508 01:33:47.819 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:47.819 | INFO | "malloc" : " 14 GB", | 20260508 01:33:47.819 | INFO | "pinned" : " 29 GB" | 20260508 01:33:47.819 | INFO | } | 20260508 01:33:47.819 | INFO | Perf report for the last 5.14084 seconds: | 20260508 01:33:47.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:47.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.819 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:33:47.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.819 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:33:47.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:47.820 | INFO | Uploaded 36GiB to GPU in 2.51687s (14.3035GiB/s) | 20260508 01:33:47.820 | INFO | Downloaded 34GiB to GPU in 2.46388s (13.7994GiB/s) | 20260508 01:33:47.820 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:33:47.820 | INFO | Total mem usage: 16.7 GiB | 20260508 01:33:47.820 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:33:47.820 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.5%) | 20260508 01:33:47.820 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:33:47.820 | INFO | High Water Breakdown: | 20260508 01:33:47.820 | INFO | 59.7% : 8.69 GiB ( 84.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:33:47.820 | INFO | 40.2% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:33:47.820 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:33:47.820 | INFO | ------------------------------------------ | 20260508 01:33:47.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:33:47.820 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:33:47.820 | INFO | Total mem usage: 26.5 GiB | 20260508 01:33:47.820 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:33:47.820 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:33:47.820 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:33:47.820 | INFO | High Water Breakdown: | 20260508 01:33:47.820 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:33:47.820 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:33:47.820 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:33:47.820 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:33:47.820 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:33:47.820 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:33:47.820 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:33:47.820 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:33:47.820 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:33:47.820 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:33:47.820 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:33:47.820 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:33:47.820 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:33:47.820 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:33:47.820 | INFO | ------------------------------------------ | 20260508 01:33:47.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:33:47.820 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:33:47.820 | INFO | Total mem usage: 27.8 GiB | 20260508 01:33:47.820 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:33:47.820 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:33:47.820 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:33:47.820 | INFO | High Water Breakdown: | 20260508 01:33:47.820 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:33:47.820 | INFO | ------------------------------------------ | 20260508 01:33:47.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:33:47.820 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:33:47.820 | INFO | Total mem usage: 15.7 GiB | 20260508 01:33:47.820 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:33:47.820 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:33:47.820 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:33:47.820 | INFO | High Water Breakdown: | 20260508 01:33:47.820 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:33:47.820 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:33:47.820 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:33:47.820 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:33:47.820 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:33:47.820 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:33:47.820 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:33:47.820 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:33:47.820 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:33:47.820 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:33:47.820 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:33:47.820 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:33:47.820 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:33:47.820 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:33:47.821 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:33:47.821 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:33:47.821 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:33:47.821 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:33:47.821 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:33:47.821 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:33:47.821 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:33:47.821 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:33:47.821 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:33:47.821 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:33:47.821 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:33:47.821 | INFO | ------------------------------------------ | 20260508 01:33:47.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:33:47.821 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:33:47.821 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:33:47.821 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:33:47.821 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.56e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.17e-02 / 5.52e-03 | 20260508 01:33:47.821 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.50e+00, 5.84e-01 / 2.13e-01 | 20260508 01:33:47.821 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.05e-01 / 1.23e-01 | 20260508 01:33:47.821 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.60e+01, 4.02e+00 / 1.95e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.66e+00 / 1.73e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.26e-01 / 2.42e-01 | 20260508 01:33:47.821 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.29e+00, 2.69e-01 / 1.58e-01 | 20260508 01:33:47.821 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.69e-01, 5.33e-02 / 2.65e-02 | 20260508 01:33:47.821 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:33:47.821 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:33:47.821 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:33:47.821 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 24.11%, Upload -- 2.56e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 24.47%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 30.83%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 1.89%, LockUpload -- 2.01e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 2.99%, LockDownload -- 3.18e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:33:47.821 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.08e-02 / 0.00e+00 | 20260508 01:33:47.916 | INFO | Starting a round of Baz IO | 20260508 01:33:51.360 | INFO | Populating RTMetrics report | 20260508 01:33:51.361 | INFO | RTMetrics report populated | 20260508 01:33:51.361 | INFO | Opening tmp RTMetrics json file | 20260508 01:33:51.396 | INFO | Serializing RTMetrics to json | 20260508 01:33:51.450 | INFO | Serialization done, swapping tmp file now... | 20260508 01:33:51.452 | INFO | RTMetrics done | 20260508 01:33:51.491 | INFO | Finished a round of Baz IO: Wrote 1.661 GiB in 3.574 seconds | 20260508 01:33:51.491 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.80% / 0.00% / 58.20% | 20260508 01:33:51.491 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.835717% / 7.94729e-06% / 5.5631e-05% | 20260508 01:33:51.491 | INFO | Baz IO overall: | 20260508 01:33:51.491 | INFO | BytesWritten : 1899875421082 | 20260508 01:33:51.491 | INFO | NumEvents : 918956006924 | 20260508 01:33:52.961 | INFO | Analyzing chunk frames = [4194304,4194816) | 20260508 01:33:52.961 | INFO | MemInfo: { | 20260508 01:33:52.961 | INFO | "gpu" : " 17 GB", | 20260508 01:33:52.961 | INFO | "huge" : "", | 20260508 01:33:52.961 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:52.961 | INFO | "malloc" : " 14 GB", | 20260508 01:33:52.961 | INFO | "pinned" : " 29 GB" | 20260508 01:33:52.961 | INFO | } | 20260508 01:33:52.961 | INFO | Perf report for the last 5.14183 seconds: | 20260508 01:33:52.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:52.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:52.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:52.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:52.961 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 01:33:52.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:52.961 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 01:33:52.961 | INFO | PRE_HQ : 65.7%, 1.7, 56.3ms, 0 | 20260508 01:33:52.961 | INFO | BAZWRITER : 68.9%, 3.5, 124.5ms, 0 | 20260508 01:33:52.961 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260508 01:33:52.962 | INFO | Uploaded 36GiB to GPU in 2.55599s (14.0846GiB/s) | 20260508 01:33:52.962 | INFO | Downloaded 38GiB to GPU in 2.68899s (14.1317GiB/s) | 20260508 01:33:58.101 | INFO | Analyzing chunk frames = [4194816,4195328) | 20260508 01:33:58.101 | INFO | MemInfo: { | 20260508 01:33:58.101 | INFO | "gpu" : " 17 GB", | 20260508 01:33:58.101 | INFO | "huge" : "", | 20260508 01:33:58.101 | INFO | "huge_shared" : " 29 GB", | 20260508 01:33:58.101 | INFO | "malloc" : " 14 GB", | 20260508 01:33:58.101 | INFO | "pinned" : " 29 GB" | 20260508 01:33:58.101 | INFO | } | 20260508 01:33:58.101 | INFO | Perf report for the last 5.13982 seconds: | 20260508 01:33:58.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:33:58.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:33:58.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:33:58.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:33:58.101 | INFO | Uploaded 36GiB to GPU in 2.49508s (14.4284GiB/s) | 20260508 01:33:58.101 | INFO | Downloaded 34GiB to GPU in 2.43445s (13.9662GiB/s) | 20260508 01:34:03.233 | INFO | Analyzing chunk frames = [4195328,4195840) | 20260508 01:34:03.233 | INFO | MemInfo: { | 20260508 01:34:03.233 | INFO | "gpu" : " 17 GB", | 20260508 01:34:03.233 | INFO | "huge" : "", | 20260508 01:34:03.233 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:03.233 | INFO | "malloc" : " 14 GB", | 20260508 01:34:03.233 | INFO | "pinned" : " 29 GB" | 20260508 01:34:03.233 | INFO | } | 20260508 01:34:03.233 | INFO | Perf report for the last 5.13189 seconds: | 20260508 01:34:03.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:03.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.233 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:34:03.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.233 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 01:34:03.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:03.234 | INFO | Uploaded 36GiB to GPU in 2.50537s (14.3692GiB/s) | 20260508 01:34:03.234 | INFO | Downloaded 34GiB to GPU in 2.51736s (13.5062GiB/s) | 20260508 01:34:08.377 | INFO | Analyzing chunk frames = [4195840,4196352) | 20260508 01:34:08.377 | INFO | MemInfo: { | 20260508 01:34:08.377 | INFO | "gpu" : " 17 GB", | 20260508 01:34:08.377 | INFO | "huge" : "", | 20260508 01:34:08.377 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:08.377 | INFO | "malloc" : " 14 GB", | 20260508 01:34:08.377 | INFO | "pinned" : " 29 GB" | 20260508 01:34:08.377 | INFO | } | 20260508 01:34:08.377 | INFO | Perf report for the last 5.14409 seconds: | 20260508 01:34:08.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:08.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:34:08.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:34:08.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:08.377 | INFO | Uploaded 36GiB to GPU in 2.46603s (14.5984GiB/s) | 20260508 01:34:08.377 | INFO | Downloaded 34GiB to GPU in 2.50347s (13.5811GiB/s) | 20260508 01:34:13.517 | INFO | Analyzing chunk frames = [4196352,4196864) | 20260508 01:34:13.517 | INFO | MemInfo: { | 20260508 01:34:13.517 | INFO | "gpu" : " 17 GB", | 20260508 01:34:13.517 | INFO | "huge" : "", | 20260508 01:34:13.517 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:13.517 | INFO | "malloc" : " 14 GB", | 20260508 01:34:13.517 | INFO | "pinned" : " 29 GB" | 20260508 01:34:13.517 | INFO | } | 20260508 01:34:13.517 | INFO | Perf report for the last 5.14019 seconds: | 20260508 01:34:13.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:13.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.517 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 01:34:13.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.517 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:34:13.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:13.518 | INFO | Uploaded 36GiB to GPU in 2.27871s (15.7984GiB/s) | 20260508 01:34:13.518 | INFO | Downloaded 34GiB to GPU in 2.51088s (13.5411GiB/s) | 20260508 01:34:18.662 | INFO | Analyzing chunk frames = [4196864,4197376) | 20260508 01:34:18.662 | INFO | MemInfo: { | 20260508 01:34:18.662 | INFO | "gpu" : " 17 GB", | 20260508 01:34:18.662 | INFO | "huge" : "", | 20260508 01:34:18.662 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:18.662 | INFO | "malloc" : " 14 GB", | 20260508 01:34:18.662 | INFO | "pinned" : " 29 GB" | 20260508 01:34:18.662 | INFO | } | 20260508 01:34:18.662 | INFO | Perf report for the last 5.145 seconds: | 20260508 01:34:18.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:18.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.662 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 01:34:18.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.662 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:34:18.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:18.663 | INFO | Uploaded 36GiB to GPU in 2.34868s (15.3278GiB/s) | 20260508 01:34:18.663 | INFO | Downloaded 34GiB to GPU in 2.425s (14.0206GiB/s) | 20260508 01:34:23.796 | INFO | Analyzing chunk frames = [4197376,4197888) | 20260508 01:34:23.796 | INFO | MemInfo: { | 20260508 01:34:23.796 | INFO | "gpu" : " 17 GB", | 20260508 01:34:23.796 | INFO | "huge" : "", | 20260508 01:34:23.796 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:23.796 | INFO | "malloc" : " 14 GB", | 20260508 01:34:23.796 | INFO | "pinned" : " 29 GB" | 20260508 01:34:23.796 | INFO | } | 20260508 01:34:23.796 | INFO | Perf report for the last 5.13358 seconds: | 20260508 01:34:23.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:23.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.796 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 01:34:23.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.796 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:34:23.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:23.797 | INFO | Uploaded 36GiB to GPU in 2.40919s (14.9428GiB/s) | 20260508 01:34:23.797 | INFO | Downloaded 34GiB to GPU in 2.42359s (14.0288GiB/s) | 20260508 01:34:28.931 | INFO | Analyzing chunk frames = [4197888,4198400) | 20260508 01:34:28.931 | INFO | MemInfo: { | 20260508 01:34:28.931 | INFO | "gpu" : " 17 GB", | 20260508 01:34:28.931 | INFO | "huge" : "", | 20260508 01:34:28.931 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:28.931 | INFO | "malloc" : " 14 GB", | 20260508 01:34:28.931 | INFO | "pinned" : " 29 GB" | 20260508 01:34:28.931 | INFO | } | 20260508 01:34:28.931 | INFO | Perf report for the last 5.13562 seconds: | 20260508 01:34:28.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:28.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.931 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 01:34:28.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.932 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:34:28.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:28.932 | INFO | Uploaded 36GiB to GPU in 2.53181s (14.2191GiB/s) | 20260508 01:34:28.932 | INFO | Downloaded 34GiB to GPU in 2.47202s (13.754GiB/s) | 20260508 01:34:29.029 | INFO | Starting a round of Baz IO | 20260508 01:34:32.508 | INFO | Populating RTMetrics report | 20260508 01:34:32.508 | INFO | RTMetrics report populated | 20260508 01:34:32.508 | INFO | Opening tmp RTMetrics json file | 20260508 01:34:32.516 | INFO | Serializing RTMetrics to json | 20260508 01:34:32.536 | INFO | Serialization done, swapping tmp file now... | 20260508 01:34:32.550 | INFO | RTMetrics done | 20260508 01:34:32.626 | INFO | Finished a round of Baz IO: Wrote 1.66 GiB in 3.597 seconds | 20260508 01:34:32.626 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.80% / 0.00% / 58.20% | 20260508 01:34:32.626 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.835037% / 0% / 4.37101e-05% | 20260508 01:34:32.626 | INFO | Baz IO overall: | 20260508 01:34:32.626 | INFO | BytesWritten : 1901657835555 | 20260508 01:34:32.626 | INFO | NumEvents : 919811899390 | 20260508 01:34:34.076 | INFO | Analyzing chunk frames = [4198400,4198912) | 20260508 01:34:34.076 | INFO | MemInfo: { | 20260508 01:34:34.076 | INFO | "gpu" : " 17 GB", | 20260508 01:34:34.076 | INFO | "huge" : "", | 20260508 01:34:34.076 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:34.076 | INFO | "malloc" : " 14 GB", | 20260508 01:34:34.076 | INFO | "pinned" : " 29 GB" | 20260508 01:34:34.076 | INFO | } | 20260508 01:34:34.076 | INFO | Perf report for the last 5.14482 seconds: | 20260508 01:34:34.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:34.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:34.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:34.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:34.076 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260508 01:34:34.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:34.076 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 01:34:34.076 | INFO | PRE_HQ : 65.9%, 1.7, 58.0ms, 0 | 20260508 01:34:34.076 | INFO | BAZWRITER : 69.5%, 3.6, 127.9ms, 0 | 20260508 01:34:34.076 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 01:34:34.077 | INFO | Uploaded 36GiB to GPU in 2.46376s (14.6118GiB/s) | 20260508 01:34:34.077 | INFO | Downloaded 38GiB to GPU in 2.70264s (14.0603GiB/s) | 20260508 01:34:39.217 | INFO | Analyzing chunk frames = [4198912,4199424) | 20260508 01:34:39.217 | INFO | MemInfo: { | 20260508 01:34:39.217 | INFO | "gpu" : " 17 GB", | 20260508 01:34:39.217 | INFO | "huge" : "", | 20260508 01:34:39.217 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:39.217 | INFO | "malloc" : " 14 GB", | 20260508 01:34:39.217 | INFO | "pinned" : " 29 GB" | 20260508 01:34:39.217 | INFO | } | 20260508 01:34:39.218 | INFO | Perf report for the last 5.14121 seconds: | 20260508 01:34:39.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:39.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:34:39.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:34:39.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:39.218 | INFO | Uploaded 36GiB to GPU in 2.54688s (14.1349GiB/s) | 20260508 01:34:39.218 | INFO | Downloaded 34GiB to GPU in 2.42277s (14.0335GiB/s) | 20260508 01:34:44.353 | INFO | Analyzing chunk frames = [4199424,4199936) | 20260508 01:34:44.353 | INFO | MemInfo: { | 20260508 01:34:44.353 | INFO | "gpu" : " 17 GB", | 20260508 01:34:44.353 | INFO | "huge" : "", | 20260508 01:34:44.353 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:44.353 | INFO | "malloc" : " 14 GB", | 20260508 01:34:44.353 | INFO | "pinned" : " 29 GB" | 20260508 01:34:44.353 | INFO | } | 20260508 01:34:44.353 | INFO | Perf report for the last 5.13562 seconds: | 20260508 01:34:44.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:44.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:34:44.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:34:44.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:44.353 | INFO | Uploaded 36GiB to GPU in 2.52649s (14.249GiB/s) | 20260508 01:34:44.353 | INFO | Downloaded 34GiB to GPU in 2.48404s (13.6874GiB/s) | 20260508 01:34:49.490 | INFO | Analyzing chunk frames = [4199936,4200448) | 20260508 01:34:49.490 | INFO | MemInfo: { | 20260508 01:34:49.490 | INFO | "gpu" : " 17 GB", | 20260508 01:34:49.490 | INFO | "huge" : "", | 20260508 01:34:49.490 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:49.490 | INFO | "malloc" : " 14 GB", | 20260508 01:34:49.490 | INFO | "pinned" : " 29 GB" | 20260508 01:34:49.490 | INFO | } | 20260508 01:34:49.490 | INFO | Perf report for the last 5.13679 seconds: | 20260508 01:34:49.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:49.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:34:49.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:34:49.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:49.490 | INFO | Uploaded 36GiB to GPU in 2.51822s (14.2958GiB/s) | 20260508 01:34:49.491 | INFO | Downloaded 34GiB to GPU in 2.48635s (13.6747GiB/s) | 20260508 01:34:54.628 | INFO | Analyzing chunk frames = [4200448,4200960) | 20260508 01:34:54.628 | INFO | MemInfo: { | 20260508 01:34:54.628 | INFO | "gpu" : " 17 GB", | 20260508 01:34:54.628 | INFO | "huge" : "", | 20260508 01:34:54.628 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:54.628 | INFO | "malloc" : " 14 GB", | 20260508 01:34:54.628 | INFO | "pinned" : " 29 GB" | 20260508 01:34:54.628 | INFO | } | 20260508 01:34:54.628 | INFO | Perf report for the last 5.13832 seconds: | 20260508 01:34:54.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:54.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.628 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 01:34:54.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.628 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:34:54.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:54.629 | INFO | Uploaded 36GiB to GPU in 2.35917s (15.2596GiB/s) | 20260508 01:34:54.629 | INFO | Downloaded 34GiB to GPU in 2.47446s (13.7404GiB/s) | 20260508 01:34:59.769 | INFO | Analyzing chunk frames = [4200960,4201472) | 20260508 01:34:59.769 | INFO | MemInfo: { | 20260508 01:34:59.769 | INFO | "gpu" : " 17 GB", | 20260508 01:34:59.769 | INFO | "huge" : "", | 20260508 01:34:59.769 | INFO | "huge_shared" : " 29 GB", | 20260508 01:34:59.769 | INFO | "malloc" : " 14 GB", | 20260508 01:34:59.769 | INFO | "pinned" : " 29 GB" | 20260508 01:34:59.769 | INFO | } | 20260508 01:34:59.769 | INFO | Perf report for the last 5.14047 seconds: | 20260508 01:34:59.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:34:59.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.769 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 01:34:59.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.769 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:34:59.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:34:59.770 | INFO | Uploaded 36GiB to GPU in 2.4257s (14.8411GiB/s) | 20260508 01:34:59.770 | INFO | Downloaded 34GiB to GPU in 2.39952s (14.1695GiB/s) | 20260508 01:35:04.911 | INFO | Analyzing chunk frames = [4201472,4201984) | 20260508 01:35:04.911 | INFO | MemInfo: { | 20260508 01:35:04.911 | INFO | "gpu" : " 17 GB", | 20260508 01:35:04.911 | INFO | "huge" : "", | 20260508 01:35:04.911 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:04.911 | INFO | "malloc" : " 14 GB", | 20260508 01:35:04.911 | INFO | "pinned" : " 29 GB" | 20260508 01:35:04.911 | INFO | } | 20260508 01:35:04.911 | INFO | Perf report for the last 5.14235 seconds: | 20260508 01:35:04.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:04.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.911 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 01:35:04.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.911 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:35:04.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:04.912 | INFO | Uploaded 36GiB to GPU in 2.5562s (14.0834GiB/s) | 20260508 01:35:04.912 | INFO | Downloaded 34GiB to GPU in 2.44976s (13.8789GiB/s) | 20260508 01:35:10.048 | INFO | Analyzing chunk frames = [4201984,4202496) | 20260508 01:35:10.048 | INFO | MemInfo: { | 20260508 01:35:10.048 | INFO | "gpu" : " 17 GB", | 20260508 01:35:10.048 | INFO | "huge" : "", | 20260508 01:35:10.048 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:10.048 | INFO | "malloc" : " 14 GB", | 20260508 01:35:10.048 | INFO | "pinned" : " 29 GB" | 20260508 01:35:10.048 | INFO | } | 20260508 01:35:10.048 | INFO | Perf report for the last 5.13714 seconds: | 20260508 01:35:10.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:10.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.048 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:35:10.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.048 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 01:35:10.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:10.049 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:35:10.049 | INFO | Total mem usage: 16.7 GiB | 20260508 01:35:10.049 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:35:10.049 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15%) | 20260508 01:35:10.049 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:35:10.049 | INFO | High Water Breakdown: | 20260508 01:35:10.049 | INFO | Uploaded 36GiB to GPU in 2.56795s (14.019GiB/s) | 20260508 01:35:10.049 | INFO | Downloaded 34GiB to GPU in 2.45491s (13.8498GiB/s) | 20260508 01:35:10.049 | INFO | 59.7% : 8.66 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:35:10.049 | INFO | 40.2% : 5.83 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:35:10.049 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:35:10.049 | INFO | ------------------------------------------ | 20260508 01:35:10.049 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:35:10.049 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:35:10.049 | INFO | Total mem usage: 26.5 GiB | 20260508 01:35:10.049 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:35:10.049 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:35:10.049 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:35:10.049 | INFO | High Water Breakdown: | 20260508 01:35:10.049 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:35:10.049 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:35:10.049 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:35:10.049 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:35:10.049 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:35:10.049 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:35:10.049 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:35:10.049 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:35:10.049 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:35:10.049 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:35:10.049 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:35:10.049 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:35:10.049 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:35:10.049 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:35:10.049 | INFO | ------------------------------------------ | 20260508 01:35:10.049 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:35:10.049 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:35:10.049 | INFO | Total mem usage: 27.8 GiB | 20260508 01:35:10.049 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:35:10.049 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:35:10.050 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:35:10.050 | INFO | High Water Breakdown: | 20260508 01:35:10.050 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:35:10.050 | INFO | ------------------------------------------ | 20260508 01:35:10.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:35:10.050 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:35:10.050 | INFO | Total mem usage: 15.7 GiB | 20260508 01:35:10.050 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:35:10.050 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:35:10.050 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:35:10.050 | INFO | High Water Breakdown: | 20260508 01:35:10.050 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:35:10.050 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:35:10.050 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:35:10.050 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:35:10.050 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:35:10.050 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:35:10.050 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:35:10.050 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:35:10.050 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:35:10.050 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:35:10.050 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:35:10.050 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:35:10.050 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:35:10.050 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:35:10.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:35:10.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:35:10.050 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:35:10.050 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:35:10.050 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:35:10.050 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:35:10.050 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:35:10.050 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:35:10.050 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:35:10.050 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:35:10.050 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:35:10.050 | INFO | ------------------------------------------ | 20260508 01:35:10.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:35:10.050 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:35:10.050 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:35:10.050 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:35:10.050 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.42e+01 / 1.22e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.02e-01, 1.07e-02 / 4.57e-03 | 20260508 01:35:10.050 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.44e+00, 5.90e-01 / 2.60e-01 | 20260508 01:35:10.050 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.11e-01 | 20260508 01:35:10.050 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.58e+01, 3.96e+00 / 1.43e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.64e+00 / 1.35e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 2.69e+00, 4.26e-01 / 2.41e-01 | 20260508 01:35:10.050 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.28e+00, 2.69e-01 / 1.68e-01 | 20260508 01:35:10.050 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.72e-01, 5.32e-02 / 2.53e-02 | 20260508 01:35:10.050 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:35:10.050 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:35:10.050 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:35:10.050 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 24.19%, Upload -- 2.58e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 24.37%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 2.13%, LockUpload -- 2.27e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 2.92%, LockDownload -- 3.11e+00 / 0.00e+00, 4.45e+00 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 15.64%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:35:10.050 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260508 01:35:10.146 | INFO | Starting a round of Baz IO | 20260508 01:35:13.594 | INFO | Populating RTMetrics report | 20260508 01:35:13.595 | INFO | RTMetrics report populated | 20260508 01:35:13.595 | INFO | Opening tmp RTMetrics json file | 20260508 01:35:13.617 | INFO | Serializing RTMetrics to json | 20260508 01:35:13.619 | INFO | Serialization done, swapping tmp file now... | 20260508 01:35:13.620 | INFO | RTMetrics done | 20260508 01:35:13.696 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.55 seconds | 20260508 01:35:13.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.79% / 0.00% / 58.20% | 20260508 01:35:13.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.831819% / 0% / 5.5631e-05% | 20260508 01:35:13.697 | INFO | Baz IO overall: | 20260508 01:35:13.697 | INFO | BytesWritten : 1903437873093 | 20260508 01:35:13.697 | INFO | NumEvents : 920666029673 | 20260508 01:35:15.185 | INFO | Analyzing chunk frames = [4202496,4203008) | 20260508 01:35:15.185 | INFO | MemInfo: { | 20260508 01:35:15.185 | INFO | "gpu" : " 17 GB", | 20260508 01:35:15.185 | INFO | "huge" : "", | 20260508 01:35:15.185 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:15.185 | INFO | "malloc" : " 14 GB", | 20260508 01:35:15.185 | INFO | "pinned" : " 29 GB" | 20260508 01:35:15.185 | INFO | } | 20260508 01:35:15.185 | INFO | Perf report for the last 5.13713 seconds: | 20260508 01:35:15.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:15.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:15.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:15.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:15.186 | INFO | ANALYSIS : 68.3%, 3.0, 106.6ms, 0 | 20260508 01:35:15.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:15.186 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:35:15.186 | INFO | PRE_HQ : 65.5%, 1.7, 56.1ms, 0 | 20260508 01:35:15.186 | INFO | BAZWRITER : 68.7%, 3.8, 133.3ms, 0 | 20260508 01:35:15.186 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:35:15.186 | INFO | Uploaded 36GiB to GPU in 2.56356s (14.043GiB/s) | 20260508 01:35:15.186 | INFO | Downloaded 38GiB to GPU in 2.68238s (14.1665GiB/s) | 20260508 01:35:20.331 | INFO | Analyzing chunk frames = [4203008,4203520) | 20260508 01:35:20.331 | INFO | MemInfo: { | 20260508 01:35:20.331 | INFO | "gpu" : " 17 GB", | 20260508 01:35:20.331 | INFO | "huge" : "", | 20260508 01:35:20.331 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:20.331 | INFO | "malloc" : " 14 GB", | 20260508 01:35:20.331 | INFO | "pinned" : " 29 GB" | 20260508 01:35:20.331 | INFO | } | 20260508 01:35:20.331 | INFO | Perf report for the last 5.14527 seconds: | 20260508 01:35:20.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:20.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:35:20.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:35:20.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:20.331 | INFO | Uploaded 36GiB to GPU in 2.53025s (14.2279GiB/s) | 20260508 01:35:20.331 | INFO | Downloaded 34GiB to GPU in 2.43009s (13.9912GiB/s) | 20260508 01:35:25.475 | INFO | Analyzing chunk frames = [4203520,4204032) | 20260508 01:35:25.475 | INFO | MemInfo: { | 20260508 01:35:25.475 | INFO | "gpu" : " 17 GB", | 20260508 01:35:25.475 | INFO | "huge" : "", | 20260508 01:35:25.475 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:25.475 | INFO | "malloc" : " 14 GB", | 20260508 01:35:25.475 | INFO | "pinned" : " 29 GB" | 20260508 01:35:25.475 | INFO | } | 20260508 01:35:25.475 | INFO | Perf report for the last 5.14426 seconds: | 20260508 01:35:25.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:25.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.475 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:35:25.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.475 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:35:25.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:25.476 | INFO | Uploaded 36GiB to GPU in 2.52249s (14.2716GiB/s) | 20260508 01:35:25.476 | INFO | Downloaded 34GiB to GPU in 2.49867s (13.6072GiB/s) | 20260508 01:35:30.612 | INFO | Analyzing chunk frames = [4204032,4204544) | 20260508 01:35:30.612 | INFO | MemInfo: { | 20260508 01:35:30.612 | INFO | "gpu" : " 17 GB", | 20260508 01:35:30.612 | INFO | "huge" : "", | 20260508 01:35:30.612 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:30.612 | INFO | "malloc" : " 14 GB", | 20260508 01:35:30.612 | INFO | "pinned" : " 29 GB" | 20260508 01:35:30.612 | INFO | } | 20260508 01:35:30.612 | INFO | Perf report for the last 5.13697 seconds: | 20260508 01:35:30.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:30.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:35:30.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:35:30.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:30.612 | INFO | Uploaded 36GiB to GPU in 2.54243s (14.1597GiB/s) | 20260508 01:35:30.612 | INFO | Downloaded 34GiB to GPU in 2.50339s (13.5816GiB/s) | 20260508 01:35:35.747 | INFO | Analyzing chunk frames = [4204544,4205056) | 20260508 01:35:35.747 | INFO | MemInfo: { | 20260508 01:35:35.747 | INFO | "gpu" : " 17 GB", | 20260508 01:35:35.747 | INFO | "huge" : "", | 20260508 01:35:35.747 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:35.747 | INFO | "malloc" : " 14 GB", | 20260508 01:35:35.747 | INFO | "pinned" : " 29 GB" | 20260508 01:35:35.747 | INFO | } | 20260508 01:35:35.747 | INFO | Perf report for the last 5.13485 seconds: | 20260508 01:35:35.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:35.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 01:35:35.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:35:35.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:35.747 | INFO | Uploaded 36GiB to GPU in 2.39409s (15.037GiB/s) | 20260508 01:35:35.747 | INFO | Downloaded 34GiB to GPU in 2.57067s (13.2261GiB/s) | 20260508 01:35:40.887 | INFO | Analyzing chunk frames = [4205056,4205568) | 20260508 01:35:40.887 | INFO | MemInfo: { | 20260508 01:35:40.887 | INFO | "gpu" : " 17 GB", | 20260508 01:35:40.887 | INFO | "huge" : "", | 20260508 01:35:40.887 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:40.887 | INFO | "malloc" : " 14 GB", | 20260508 01:35:40.887 | INFO | "pinned" : " 29 GB" | 20260508 01:35:40.887 | INFO | } | 20260508 01:35:40.887 | INFO | Perf report for the last 5.14053 seconds: | 20260508 01:35:40.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:40.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.887 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:35:40.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.888 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:35:40.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:40.888 | INFO | Uploaded 36GiB to GPU in 2.40434s (14.9729GiB/s) | 20260508 01:35:40.888 | INFO | Downloaded 34GiB to GPU in 2.39608s (14.1899GiB/s) | 20260508 01:35:46.029 | INFO | Analyzing chunk frames = [4205568,4206080) | 20260508 01:35:46.029 | INFO | MemInfo: { | 20260508 01:35:46.029 | INFO | "gpu" : " 17 GB", | 20260508 01:35:46.029 | INFO | "huge" : "", | 20260508 01:35:46.029 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:46.029 | INFO | "malloc" : " 14 GB", | 20260508 01:35:46.029 | INFO | "pinned" : " 29 GB" | 20260508 01:35:46.029 | INFO | } | 20260508 01:35:46.029 | INFO | Perf report for the last 5.14149 seconds: | 20260508 01:35:46.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:46.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:35:46.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:35:46.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:46.029 | INFO | Uploaded 36GiB to GPU in 2.43265s (14.7987GiB/s) | 20260508 01:35:46.029 | INFO | Downloaded 34GiB to GPU in 2.42466s (14.0226GiB/s) | 20260508 01:35:51.173 | INFO | Analyzing chunk frames = [4206080,4206592) | 20260508 01:35:51.173 | INFO | MemInfo: { | 20260508 01:35:51.173 | INFO | "gpu" : " 17 GB", | 20260508 01:35:51.173 | INFO | "huge" : "", | 20260508 01:35:51.173 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:51.173 | INFO | "malloc" : " 14 GB", | 20260508 01:35:51.173 | INFO | "pinned" : " 29 GB" | 20260508 01:35:51.173 | INFO | } | 20260508 01:35:51.173 | INFO | Perf report for the last 5.14407 seconds: | 20260508 01:35:51.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:51.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 01:35:51.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:35:51.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:51.173 | INFO | Uploaded 36GiB to GPU in 2.54017s (14.1723GiB/s) | 20260508 01:35:51.173 | INFO | Downloaded 34GiB to GPU in 2.45672s (13.8396GiB/s) | 20260508 01:35:51.270 | INFO | Starting a round of Baz IO | 20260508 01:35:54.713 | INFO | Populating RTMetrics report | 20260508 01:35:54.713 | INFO | RTMetrics report populated | 20260508 01:35:54.713 | INFO | Opening tmp RTMetrics json file | 20260508 01:35:54.734 | INFO | Serializing RTMetrics to json | 20260508 01:35:54.736 | INFO | Serialization done, swapping tmp file now... | 20260508 01:35:54.737 | INFO | RTMetrics done | 20260508 01:35:54.830 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.56 seconds | 20260508 01:35:54.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.79% / 0.00% / 58.21% | 20260508 01:35:54.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.831485% / 3.97364e-06% / 5.16574e-05% | 20260508 01:35:54.830 | INFO | Baz IO overall: | 20260508 01:35:54.830 | INFO | BytesWritten : 1905218816504 | 20260508 01:35:54.830 | INFO | NumEvents : 921521273396 | 20260508 01:35:56.305 | INFO | Analyzing chunk frames = [4206592,4207104) | 20260508 01:35:56.305 | INFO | MemInfo: { | 20260508 01:35:56.305 | INFO | "gpu" : " 17 GB", | 20260508 01:35:56.305 | INFO | "huge" : "", | 20260508 01:35:56.305 | INFO | "huge_shared" : " 29 GB", | 20260508 01:35:56.305 | INFO | "malloc" : " 14 GB", | 20260508 01:35:56.305 | INFO | "pinned" : " 29 GB" | 20260508 01:35:56.305 | INFO | } | 20260508 01:35:56.306 | INFO | Perf report for the last 5.13257 seconds: | 20260508 01:35:56.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:35:56.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:56.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:56.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:56.306 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260508 01:35:56.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:35:56.306 | INFO | RT_METRICS : 40.4%, 1.0, 21.1ms, 0 | 20260508 01:35:56.306 | INFO | PRE_HQ : 65.8%, 1.6, 56.1ms, 0 | 20260508 01:35:56.306 | INFO | BAZWRITER : 68.9%, 3.6, 127.3ms, 0 | 20260508 01:35:56.306 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:35:56.306 | INFO | Uploaded 36GiB to GPU in 2.59228s (13.8874GiB/s) | 20260508 01:35:56.306 | INFO | Downloaded 38GiB to GPU in 2.70464s (14.0499GiB/s) | 20260508 01:36:01.437 | INFO | Analyzing chunk frames = [4207104,4207616) | 20260508 01:36:01.437 | INFO | MemInfo: { | 20260508 01:36:01.437 | INFO | "gpu" : " 17 GB", | 20260508 01:36:01.437 | INFO | "huge" : "", | 20260508 01:36:01.437 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:01.437 | INFO | "malloc" : " 14 GB", | 20260508 01:36:01.437 | INFO | "pinned" : " 29 GB" | 20260508 01:36:01.437 | INFO | } | 20260508 01:36:01.437 | INFO | Perf report for the last 5.1313 seconds: | 20260508 01:36:01.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:01.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260508 01:36:01.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:36:01.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:01.437 | INFO | Uploaded 36GiB to GPU in 2.53014s (14.2285GiB/s) | 20260508 01:36:01.437 | INFO | Downloaded 34GiB to GPU in 2.43298s (13.9746GiB/s) | 20260508 01:36:06.579 | INFO | Analyzing chunk frames = [4207616,4208128) | 20260508 01:36:06.579 | INFO | MemInfo: { | 20260508 01:36:06.579 | INFO | "gpu" : " 17 GB", | 20260508 01:36:06.579 | INFO | "huge" : "", | 20260508 01:36:06.579 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:06.579 | INFO | "malloc" : " 14 GB", | 20260508 01:36:06.579 | INFO | "pinned" : " 29 GB" | 20260508 01:36:06.579 | INFO | } | 20260508 01:36:06.579 | INFO | Perf report for the last 5.14254 seconds: | 20260508 01:36:06.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:06.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.579 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 01:36:06.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.580 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:36:06.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:06.580 | INFO | Uploaded 36GiB to GPU in 2.50593s (14.366GiB/s) | 20260508 01:36:06.580 | INFO | Downloaded 34GiB to GPU in 2.4929s (13.6387GiB/s) | 20260508 01:36:11.727 | INFO | Analyzing chunk frames = [4208128,4208640) | 20260508 01:36:11.727 | INFO | MemInfo: { | 20260508 01:36:11.727 | INFO | "gpu" : " 17 GB", | 20260508 01:36:11.727 | INFO | "huge" : "", | 20260508 01:36:11.727 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:11.727 | INFO | "malloc" : " 14 GB", | 20260508 01:36:11.727 | INFO | "pinned" : " 29 GB" | 20260508 01:36:11.727 | INFO | } | 20260508 01:36:11.727 | INFO | Perf report for the last 5.14742 seconds: | 20260508 01:36:11.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:11.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.727 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 01:36:11.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.727 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 01:36:11.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:11.728 | INFO | Uploaded 36GiB to GPU in 2.4918s (14.4474GiB/s) | 20260508 01:36:11.728 | INFO | Downloaded 34GiB to GPU in 2.41527s (14.0771GiB/s) | 20260508 01:36:16.865 | INFO | Analyzing chunk frames = [4208640,4209152) | 20260508 01:36:16.866 | INFO | MemInfo: { | 20260508 01:36:16.866 | INFO | "gpu" : " 17 GB", | 20260508 01:36:16.866 | INFO | "huge" : "", | 20260508 01:36:16.866 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:16.866 | INFO | "malloc" : " 14 GB", | 20260508 01:36:16.866 | INFO | "pinned" : " 29 GB" | 20260508 01:36:16.866 | INFO | } | 20260508 01:36:16.866 | INFO | Perf report for the last 5.13932 seconds: | 20260508 01:36:16.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:16.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.866 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 01:36:16.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.866 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 01:36:16.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:16.867 | INFO | Uploaded 36GiB to GPU in 2.27841s (15.8005GiB/s) | 20260508 01:36:16.867 | INFO | Downloaded 34GiB to GPU in 2.51456s (13.5212GiB/s) | 20260508 01:36:22.000 | INFO | Analyzing chunk frames = [4209152,4209664) | 20260508 01:36:22.000 | INFO | MemInfo: { | 20260508 01:36:22.000 | INFO | "gpu" : " 17 GB", | 20260508 01:36:22.000 | INFO | "huge" : "", | 20260508 01:36:22.000 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:22.000 | INFO | "malloc" : " 14 GB", | 20260508 01:36:22.000 | INFO | "pinned" : " 29 GB" | 20260508 01:36:22.000 | INFO | } | 20260508 01:36:22.001 | INFO | Perf report for the last 5.1343 seconds: | 20260508 01:36:22.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:22.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:36:22.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:36:22.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:22.001 | INFO | Uploaded 36GiB to GPU in 2.35531s (15.2846GiB/s) | 20260508 01:36:22.001 | INFO | Downloaded 34GiB to GPU in 2.43657s (13.9541GiB/s) | 20260508 01:36:27.141 | INFO | Analyzing chunk frames = [4209664,4210176) | 20260508 01:36:27.141 | INFO | MemInfo: { | 20260508 01:36:27.141 | INFO | "gpu" : " 17 GB", | 20260508 01:36:27.141 | INFO | "huge" : "", | 20260508 01:36:27.141 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:27.141 | INFO | "malloc" : " 14 GB", | 20260508 01:36:27.141 | INFO | "pinned" : " 29 GB" | 20260508 01:36:27.142 | INFO | } | 20260508 01:36:27.142 | INFO | Perf report for the last 5.14119 seconds: | 20260508 01:36:27.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:27.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:36:27.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 01:36:27.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:27.142 | INFO | Uploaded 36GiB to GPU in 2.40623s (14.9611GiB/s) | 20260508 01:36:27.142 | INFO | Downloaded 34GiB to GPU in 2.43798s (13.946GiB/s) | 20260508 01:36:32.281 | INFO | Analyzing chunk frames = [4210176,4210688) | 20260508 01:36:32.281 | INFO | MemInfo: { | 20260508 01:36:32.281 | INFO | "gpu" : " 17 GB", | 20260508 01:36:32.281 | INFO | "huge" : "", | 20260508 01:36:32.281 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:32.281 | INFO | "malloc" : " 14 GB", | 20260508 01:36:32.281 | INFO | "pinned" : " 29 GB" | 20260508 01:36:32.281 | INFO | } | 20260508 01:36:32.281 | INFO | Perf report for the last 5.13925 seconds: | 20260508 01:36:32.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:32.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:36:32.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 01:36:32.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:32.281 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:36:32.281 | INFO | Total mem usage: 16.7 GiB | 20260508 01:36:32.281 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:36:32.281 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 01:36:32.281 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:36:32.281 | INFO | High Water Breakdown: | 20260508 01:36:32.281 | INFO | 59.8% : 8.73 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:36:32.281 | INFO | 40.1% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:36:32.281 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:36:32.281 | INFO | ------------------------------------------ | 20260508 01:36:32.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:36:32.281 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:36:32.281 | INFO | Total mem usage: 26.5 GiB | 20260508 01:36:32.281 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:36:32.281 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:36:32.281 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:36:32.281 | INFO | High Water Breakdown: | 20260508 01:36:32.281 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:36:32.281 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:36:32.281 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:36:32.281 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:36:32.281 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:36:32.281 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:36:32.281 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:36:32.281 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:36:32.281 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:36:32.281 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:36:32.281 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:36:32.281 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:36:32.281 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:36:32.281 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:36:32.281 | INFO | ------------------------------------------ | 20260508 01:36:32.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:36:32.281 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:36:32.281 | INFO | Total mem usage: 27.8 GiB | 20260508 01:36:32.281 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:36:32.281 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:36:32.281 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:36:32.282 | INFO | High Water Breakdown: | 20260508 01:36:32.282 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:36:32.282 | INFO | ------------------------------------------ | 20260508 01:36:32.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:36:32.282 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:36:32.282 | INFO | Total mem usage: 15.7 GiB | 20260508 01:36:32.282 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:36:32.282 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:36:32.282 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:36:32.282 | INFO | High Water Breakdown: | 20260508 01:36:32.282 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:36:32.282 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:36:32.282 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:36:32.282 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:36:32.282 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:36:32.282 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:36:32.282 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:36:32.282 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:36:32.282 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:36:32.282 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:36:32.282 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:36:32.282 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:36:32.282 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:36:32.282 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:36:32.282 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:36:32.282 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:36:32.282 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:36:32.282 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:36:32.282 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:36:32.282 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:36:32.282 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:36:32.282 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:36:32.282 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:36:32.282 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:36:32.282 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:36:32.282 | INFO | ------------------------------------------ | 20260508 01:36:32.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:36:32.282 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:36:32.282 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:36:32.282 | INFO | Uploaded 36GiB to GPU in 2.52668s (14.2479GiB/s) | 20260508 01:36:32.282 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:36:32.282 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 1.55e+00 | 20260508 01:36:32.282 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.13e-02 / 5.10e-03 | 20260508 01:36:32.282 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.47e+00, 5.88e-01 / 1.88e-01 | 20260508 01:36:32.282 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.03e-01 | 20260508 01:36:32.282 | INFO | PerfReport: 15.97%, DME -- 4.19e+01 / 4.58e+01, 4.15e+00 / 1.82e+00 | 20260508 01:36:32.282 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.71e+00 / 1.56e+00 | 20260508 01:36:32.282 | INFO | Downloaded 34GiB to GPU in 2.46516s (13.7922GiB/s) | 20260508 01:36:32.282 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.42e+00 / 2.72e+00, 4.26e-01 / 2.08e-01 | 20260508 01:36:32.282 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.28e+00, 2.69e-01 / 1.36e-01 | 20260508 01:36:32.282 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.73e-01, 5.29e-02 / 2.70e-02 | 20260508 01:36:32.282 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:36:32.282 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:36:32.282 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:36:32.283 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 30.81%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 2.03%, LockUpload -- 2.16e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 2.78%, LockDownload -- 2.96e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 15.74%, LockCompute -- 1.68e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:36:32.283 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260508 01:36:32.379 | INFO | Starting a round of Baz IO | 20260508 01:36:35.826 | INFO | Populating RTMetrics report | 20260508 01:36:35.827 | INFO | RTMetrics report populated | 20260508 01:36:35.827 | INFO | Opening tmp RTMetrics json file | 20260508 01:36:35.855 | INFO | Serializing RTMetrics to json | 20260508 01:36:35.908 | INFO | Serialization done, swapping tmp file now... | 20260508 01:36:35.912 | INFO | RTMetrics done | 20260508 01:36:35.926 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.546 seconds | 20260508 01:36:35.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.79% / 0.00% / 58.21% | 20260508 01:36:35.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.829506% / 3.97364e-06% / 4.76837e-05% | 20260508 01:36:35.926 | INFO | Baz IO overall: | 20260508 01:36:35.926 | INFO | BytesWritten : 1907000235632 | 20260508 01:36:35.926 | INFO | NumEvents : 922377084219 | 20260508 01:36:37.423 | INFO | Analyzing chunk frames = [4210688,4211200) | 20260508 01:36:37.423 | INFO | MemInfo: { | 20260508 01:36:37.423 | INFO | "gpu" : " 17 GB", | 20260508 01:36:37.423 | INFO | "huge" : "", | 20260508 01:36:37.423 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:37.423 | INFO | "malloc" : " 14 GB", | 20260508 01:36:37.423 | INFO | "pinned" : " 29 GB" | 20260508 01:36:37.423 | INFO | } | 20260508 01:36:37.423 | INFO | Perf report for the last 5.14249 seconds: | 20260508 01:36:37.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:37.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:37.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:37.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:37.424 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 01:36:37.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:37.424 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 01:36:37.424 | INFO | PRE_HQ : 65.8%, 1.6, 56.3ms, 0 | 20260508 01:36:37.424 | INFO | BAZWRITER : 68.6%, 3.7, 128.8ms, 0 | 20260508 01:36:37.424 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 01:36:37.424 | INFO | Uploaded 36GiB to GPU in 2.53035s (14.2273GiB/s) | 20260508 01:36:37.424 | INFO | Downloaded 38GiB to GPU in 2.69839s (14.0825GiB/s) | 20260508 01:36:42.560 | INFO | Analyzing chunk frames = [4211200,4211712) | 20260508 01:36:42.560 | INFO | MemInfo: { | 20260508 01:36:42.560 | INFO | "gpu" : " 17 GB", | 20260508 01:36:42.560 | INFO | "huge" : "", | 20260508 01:36:42.560 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:42.560 | INFO | "malloc" : " 14 GB", | 20260508 01:36:42.560 | INFO | "pinned" : " 29 GB" | 20260508 01:36:42.560 | INFO | } | 20260508 01:36:42.560 | INFO | Perf report for the last 5.13686 seconds: | 20260508 01:36:42.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:42.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.560 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:36:42.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.560 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:36:42.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:42.561 | INFO | Uploaded 36GiB to GPU in 2.54372s (14.1525GiB/s) | 20260508 01:36:42.561 | INFO | Downloaded 34GiB to GPU in 2.43416s (13.9679GiB/s) | 20260508 01:36:47.696 | INFO | Analyzing chunk frames = [4211712,4212224) | 20260508 01:36:47.696 | INFO | MemInfo: { | 20260508 01:36:47.696 | INFO | "gpu" : " 17 GB", | 20260508 01:36:47.696 | INFO | "huge" : "", | 20260508 01:36:47.696 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:47.696 | INFO | "malloc" : " 14 GB", | 20260508 01:36:47.696 | INFO | "pinned" : " 29 GB" | 20260508 01:36:47.696 | INFO | } | 20260508 01:36:47.696 | INFO | Perf report for the last 5.13576 seconds: | 20260508 01:36:47.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:47.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.696 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:36:47.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.696 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:36:47.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:47.697 | INFO | Uploaded 36GiB to GPU in 2.49966s (14.4019GiB/s) | 20260508 01:36:47.697 | INFO | Downloaded 34GiB to GPU in 2.50783s (13.5575GiB/s) | 20260508 01:36:52.839 | INFO | Analyzing chunk frames = [4212224,4212736) | 20260508 01:36:52.839 | INFO | MemInfo: { | 20260508 01:36:52.839 | INFO | "gpu" : " 17 GB", | 20260508 01:36:52.839 | INFO | "huge" : "", | 20260508 01:36:52.839 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:52.839 | INFO | "malloc" : " 14 GB", | 20260508 01:36:52.839 | INFO | "pinned" : " 29 GB" | 20260508 01:36:52.839 | INFO | } | 20260508 01:36:52.839 | INFO | Perf report for the last 5.14311 seconds: | 20260508 01:36:52.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:52.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.839 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 01:36:52.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.839 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 01:36:52.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:52.840 | INFO | Uploaded 36GiB to GPU in 2.46846s (14.584GiB/s) | 20260508 01:36:52.840 | INFO | Downloaded 34GiB to GPU in 2.49782s (13.6118GiB/s) | 20260508 01:36:57.978 | INFO | Analyzing chunk frames = [4212736,4213248) | 20260508 01:36:57.978 | INFO | MemInfo: { | 20260508 01:36:57.978 | INFO | "gpu" : " 17 GB", | 20260508 01:36:57.978 | INFO | "huge" : "", | 20260508 01:36:57.978 | INFO | "huge_shared" : " 29 GB", | 20260508 01:36:57.978 | INFO | "malloc" : " 14 GB", | 20260508 01:36:57.978 | INFO | "pinned" : " 29 GB" | 20260508 01:36:57.978 | INFO | } | 20260508 01:36:57.978 | INFO | Perf report for the last 5.13851 seconds: | 20260508 01:36:57.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:36:57.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 01:36:57.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 01:36:57.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:36:57.978 | INFO | Uploaded 36GiB to GPU in 2.2872s (15.7397GiB/s) | 20260508 01:36:57.978 | INFO | Downloaded 34GiB to GPU in 2.50356s (13.5807GiB/s) | 20260508 01:37:03.117 | INFO | Analyzing chunk frames = [4213248,4213760) | 20260508 01:37:03.117 | INFO | MemInfo: { | 20260508 01:37:03.117 | INFO | "gpu" : " 17 GB", | 20260508 01:37:03.117 | INFO | "huge" : "", | 20260508 01:37:03.117 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:03.117 | INFO | "malloc" : " 14 GB", | 20260508 01:37:03.117 | INFO | "pinned" : " 29 GB" | 20260508 01:37:03.117 | INFO | } | 20260508 01:37:03.117 | INFO | Perf report for the last 5.13925 seconds: | 20260508 01:37:03.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:03.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.117 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:37:03.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.117 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:37:03.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:03.118 | INFO | Uploaded 36GiB to GPU in 2.39453s (15.0343GiB/s) | 20260508 01:37:03.118 | INFO | Downloaded 34GiB to GPU in 2.4129s (14.0909GiB/s) | 20260508 01:37:08.250 | INFO | Analyzing chunk frames = [4213760,4214272) | 20260508 01:37:08.250 | INFO | MemInfo: { | 20260508 01:37:08.250 | INFO | "gpu" : " 17 GB", | 20260508 01:37:08.250 | INFO | "huge" : "", | 20260508 01:37:08.250 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:08.250 | INFO | "malloc" : " 14 GB", | 20260508 01:37:08.250 | INFO | "pinned" : " 29 GB" | 20260508 01:37:08.250 | INFO | } | 20260508 01:37:08.251 | INFO | Perf report for the last 5.13344 seconds: | 20260508 01:37:08.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:08.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 01:37:08.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:37:08.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:08.251 | INFO | Uploaded 36GiB to GPU in 2.4705s (14.572GiB/s) | 20260508 01:37:08.251 | INFO | Downloaded 34GiB to GPU in 2.4286s (13.9998GiB/s) | 20260508 01:37:13.393 | INFO | Analyzing chunk frames = [4214272,4214784) | 20260508 01:37:13.393 | INFO | MemInfo: { | 20260508 01:37:13.393 | INFO | "gpu" : " 17 GB", | 20260508 01:37:13.393 | INFO | "huge" : "", | 20260508 01:37:13.393 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:13.393 | INFO | "malloc" : " 14 GB", | 20260508 01:37:13.393 | INFO | "pinned" : " 29 GB" | 20260508 01:37:13.393 | INFO | } | 20260508 01:37:13.393 | INFO | Perf report for the last 5.14295 seconds: | 20260508 01:37:13.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:13.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.394 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 01:37:13.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.394 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:37:13.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:13.394 | INFO | Uploaded 36GiB to GPU in 2.51532s (14.3123GiB/s) | 20260508 01:37:13.394 | INFO | Downloaded 34GiB to GPU in 2.45045s (13.875GiB/s) | 20260508 01:37:13.491 | INFO | Starting a round of Baz IO | 20260508 01:37:16.942 | INFO | Populating RTMetrics report | 20260508 01:37:16.943 | INFO | RTMetrics report populated | 20260508 01:37:16.943 | INFO | Opening tmp RTMetrics json file | 20260508 01:37:16.986 | INFO | Serializing RTMetrics to json | 20260508 01:37:17.071 | INFO | Serialization done, swapping tmp file now... | 20260508 01:37:17.077 | INFO | RTMetrics done | 20260508 01:37:17.095 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.604 seconds | 20260508 01:37:17.095 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.78% / 0.00% / 58.21% | 20260508 01:37:17.095 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.827738% / 3.97364e-06% / 4.76837e-05% | 20260508 01:37:17.095 | INFO | Baz IO overall: | 20260508 01:37:17.095 | INFO | BytesWritten : 1908780417347 | 20260508 01:37:17.095 | INFO | NumEvents : 923232065960 | 20260508 01:37:18.533 | INFO | Analyzing chunk frames = [4214784,4215296) | 20260508 01:37:18.533 | INFO | MemInfo: { | 20260508 01:37:18.533 | INFO | "gpu" : " 17 GB", | 20260508 01:37:18.533 | INFO | "huge" : "", | 20260508 01:37:18.533 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:18.533 | INFO | "malloc" : " 14 GB", | 20260508 01:37:18.533 | INFO | "pinned" : " 29 GB" | 20260508 01:37:18.533 | INFO | } | 20260508 01:37:18.533 | INFO | Perf report for the last 5.13946 seconds: | 20260508 01:37:18.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:18.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:18.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:18.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:18.533 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 01:37:18.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:18.533 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260508 01:37:18.533 | INFO | PRE_HQ : 65.7%, 1.7, 56.3ms, 0 | 20260508 01:37:18.533 | INFO | BAZWRITER : 69.7%, 3.7, 133.3ms, 0 | 20260508 01:37:18.533 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260508 01:37:18.533 | INFO | Uploaded 36GiB to GPU in 2.59468s (13.8745GiB/s) | 20260508 01:37:18.533 | INFO | Downloaded 38GiB to GPU in 2.70265s (14.0603GiB/s) | 20260508 01:37:23.675 | INFO | Analyzing chunk frames = [4215296,4215808) | 20260508 01:37:23.675 | INFO | MemInfo: { | 20260508 01:37:23.675 | INFO | "gpu" : " 17 GB", | 20260508 01:37:23.675 | INFO | "huge" : "", | 20260508 01:37:23.675 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:23.675 | INFO | "malloc" : " 14 GB", | 20260508 01:37:23.675 | INFO | "pinned" : " 29 GB" | 20260508 01:37:23.675 | INFO | } | 20260508 01:37:23.675 | INFO | Perf report for the last 5.14235 seconds: | 20260508 01:37:23.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:23.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.675 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:37:23.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.675 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:37:23.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:23.676 | INFO | Uploaded 36GiB to GPU in 2.52422s (14.2618GiB/s) | 20260508 01:37:23.676 | INFO | Downloaded 34GiB to GPU in 2.42713s (14.0083GiB/s) | 20260508 01:37:28.815 | INFO | Analyzing chunk frames = [4215808,4216320) | 20260508 01:37:28.815 | INFO | MemInfo: { | 20260508 01:37:28.815 | INFO | "gpu" : " 17 GB", | 20260508 01:37:28.815 | INFO | "huge" : "", | 20260508 01:37:28.815 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:28.815 | INFO | "malloc" : " 14 GB", | 20260508 01:37:28.815 | INFO | "pinned" : " 29 GB" | 20260508 01:37:28.815 | INFO | } | 20260508 01:37:28.815 | INFO | Perf report for the last 5.13972 seconds: | 20260508 01:37:28.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:28.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.815 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:37:28.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.815 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:37:28.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:28.816 | INFO | Uploaded 36GiB to GPU in 2.51648s (14.3057GiB/s) | 20260508 01:37:28.816 | INFO | Downloaded 34GiB to GPU in 2.49023s (13.6534GiB/s) | 20260508 01:37:33.953 | INFO | Analyzing chunk frames = [4216320,4216832) | 20260508 01:37:33.953 | INFO | MemInfo: { | 20260508 01:37:33.953 | INFO | "gpu" : " 17 GB", | 20260508 01:37:33.953 | INFO | "huge" : "", | 20260508 01:37:33.953 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:33.953 | INFO | "malloc" : " 14 GB", | 20260508 01:37:33.953 | INFO | "pinned" : " 29 GB" | 20260508 01:37:33.953 | INFO | } | 20260508 01:37:33.954 | INFO | Perf report for the last 5.13847 seconds: | 20260508 01:37:33.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:33.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:37:33.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:37:33.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:33.954 | INFO | Uploaded 36GiB to GPU in 2.51891s (14.2919GiB/s) | 20260508 01:37:33.954 | INFO | Downloaded 34GiB to GPU in 2.49201s (13.6436GiB/s) | 20260508 01:37:39.095 | INFO | Analyzing chunk frames = [4216832,4217344) | 20260508 01:37:39.095 | INFO | MemInfo: { | 20260508 01:37:39.095 | INFO | "gpu" : " 17 GB", | 20260508 01:37:39.095 | INFO | "huge" : "", | 20260508 01:37:39.095 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:39.095 | INFO | "malloc" : " 14 GB", | 20260508 01:37:39.095 | INFO | "pinned" : " 29 GB" | 20260508 01:37:39.095 | INFO | } | 20260508 01:37:39.095 | INFO | Perf report for the last 5.14169 seconds: | 20260508 01:37:39.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:39.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 01:37:39.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:37:39.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:39.095 | INFO | Uploaded 36GiB to GPU in 2.35154s (15.3091GiB/s) | 20260508 01:37:39.095 | INFO | Downloaded 34GiB to GPU in 2.53258s (13.4251GiB/s) | 20260508 01:37:44.231 | INFO | Analyzing chunk frames = [4217344,4217856) | 20260508 01:37:44.231 | INFO | MemInfo: { | 20260508 01:37:44.231 | INFO | "gpu" : " 17 GB", | 20260508 01:37:44.231 | INFO | "huge" : "", | 20260508 01:37:44.231 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:44.232 | INFO | "malloc" : " 14 GB", | 20260508 01:37:44.232 | INFO | "pinned" : " 29 GB" | 20260508 01:37:44.232 | INFO | } | 20260508 01:37:44.232 | INFO | Perf report for the last 5.13642 seconds: | 20260508 01:37:44.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:44.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:37:44.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:37:44.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:44.232 | INFO | Uploaded 36GiB to GPU in 2.36745s (15.2062GiB/s) | 20260508 01:37:44.232 | INFO | Downloaded 34GiB to GPU in 2.43082s (13.987GiB/s) | 20260508 01:37:49.370 | INFO | Analyzing chunk frames = [4217856,4218368) | 20260508 01:37:49.370 | INFO | MemInfo: { | 20260508 01:37:49.370 | INFO | "gpu" : " 17 GB", | 20260508 01:37:49.370 | INFO | "huge" : "", | 20260508 01:37:49.370 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:49.370 | INFO | "malloc" : " 14 GB", | 20260508 01:37:49.370 | INFO | "pinned" : " 29 GB" | 20260508 01:37:49.370 | INFO | } | 20260508 01:37:49.370 | INFO | Perf report for the last 5.13814 seconds: | 20260508 01:37:49.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:49.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:37:49.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:37:49.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:49.370 | INFO | Uploaded 36GiB to GPU in 2.42864s (14.8231GiB/s) | 20260508 01:37:49.370 | INFO | Downloaded 34GiB to GPU in 2.4402s (13.9333GiB/s) | 20260508 01:37:54.508 | INFO | Analyzing chunk frames = [4218368,4218880) | 20260508 01:37:54.508 | INFO | MemInfo: { | 20260508 01:37:54.508 | INFO | "gpu" : " 17 GB", | 20260508 01:37:54.508 | INFO | "huge" : "", | 20260508 01:37:54.508 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:54.508 | INFO | "malloc" : " 14 GB", | 20260508 01:37:54.508 | INFO | "pinned" : " 29 GB" | 20260508 01:37:54.508 | INFO | } | 20260508 01:37:54.508 | INFO | Perf report for the last 5.13849 seconds: | 20260508 01:37:54.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:54.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.508 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:37:54.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.508 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:37:54.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:54.509 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:37:54.509 | INFO | Total mem usage: 16.7 GiB | 20260508 01:37:54.509 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:37:54.509 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.5%) | 20260508 01:37:54.509 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:37:54.509 | INFO | High Water Breakdown: | 20260508 01:37:54.509 | INFO | 59.9% : 8.81 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:37:54.509 | INFO | 40% : 5.88 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:37:54.509 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:37:54.509 | INFO | Uploaded 36GiB to GPU in 2.53272s (14.214GiB/s) | 20260508 01:37:54.509 | INFO | ------------------------------------------ | 20260508 01:37:54.509 | INFO | Downloaded 34GiB to GPU in 2.46857s (13.7731GiB/s) | 20260508 01:37:54.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:37:54.509 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:37:54.509 | INFO | Total mem usage: 26.5 GiB | 20260508 01:37:54.509 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:37:54.509 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:37:54.509 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:37:54.509 | INFO | High Water Breakdown: | 20260508 01:37:54.509 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:37:54.509 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:37:54.509 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:37:54.509 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:37:54.509 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:37:54.509 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:37:54.509 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:37:54.509 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:37:54.509 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:37:54.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:37:54.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:37:54.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:37:54.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:37:54.509 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:37:54.509 | INFO | ------------------------------------------ | 20260508 01:37:54.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:37:54.509 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:37:54.509 | INFO | Total mem usage: 27.8 GiB | 20260508 01:37:54.509 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:37:54.509 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:37:54.509 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:37:54.509 | INFO | High Water Breakdown: | 20260508 01:37:54.509 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:37:54.509 | INFO | ------------------------------------------ | 20260508 01:37:54.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:37:54.509 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:37:54.509 | INFO | Total mem usage: 15.7 GiB | 20260508 01:37:54.510 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:37:54.510 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:37:54.510 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:37:54.510 | INFO | High Water Breakdown: | 20260508 01:37:54.510 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:37:54.510 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:37:54.510 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:37:54.510 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:37:54.510 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:37:54.510 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:37:54.510 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:37:54.510 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:37:54.510 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:37:54.510 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:37:54.510 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:37:54.510 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:37:54.510 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:37:54.510 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:37:54.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:37:54.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:37:54.510 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:37:54.510 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:37:54.510 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:37:54.510 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:37:54.510 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:37:54.510 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:37:54.510 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:37:54.510 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:37:54.510 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:37:54.510 | INFO | ------------------------------------------ | 20260508 01:37:54.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:37:54.510 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:37:54.510 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:37:54.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:37:54.510 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.41e+01 / 1.60e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.02e-01, 1.20e-02 / 5.91e-03 | 20260508 01:37:54.510 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.44e+00, 5.93e-01 / 1.95e-01 | 20260508 01:37:54.510 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.07e-01 / 1.23e-01 | 20260508 01:37:54.510 | INFO | PerfReport: 16.07%, DME -- 4.21e+01 / 4.62e+01, 4.08e+00 / 1.93e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.66e+00 / 1.60e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.28e-01 / 2.30e-01 | 20260508 01:37:54.510 | INFO | PerfReport: 3.32%, Metrics -- 1.09e+00 / 1.30e+00, 2.89e-01 / 1.56e-01 | 20260508 01:37:54.510 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.72e-01, 5.40e-02 / 2.70e-02 | 20260508 01:37:54.510 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:37:54.510 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:37:54.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:37:54.510 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 24.21%, Upload -- 2.57e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 30.83%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.27e+00 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 2.92%, LockDownload -- 3.10e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:37:54.510 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 3.93e-02 / 0.00e+00 | 20260508 01:37:54.617 | INFO | Starting a round of Baz IO | 20260508 01:37:58.055 | INFO | Populating RTMetrics report | 20260508 01:37:58.055 | INFO | RTMetrics report populated | 20260508 01:37:58.055 | INFO | Opening tmp RTMetrics json file | 20260508 01:37:58.076 | INFO | Serializing RTMetrics to json | 20260508 01:37:58.078 | INFO | Serialization done, swapping tmp file now... | 20260508 01:37:58.079 | INFO | RTMetrics done | 20260508 01:37:58.152 | INFO | Finished a round of Baz IO: Wrote 1.657 GiB in 3.534 seconds | 20260508 01:37:58.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.78% / 0.00% / 58.22% | 20260508 01:37:58.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.826752% / 1.19209e-05% / 4.37101e-05% | 20260508 01:37:58.152 | INFO | Baz IO overall: | 20260508 01:37:58.152 | INFO | BytesWritten : 1910559754356 | 20260508 01:37:58.152 | INFO | NumEvents : 924086405329 | 20260508 01:37:59.648 | INFO | Analyzing chunk frames = [4218880,4219392) | 20260508 01:37:59.648 | INFO | MemInfo: { | 20260508 01:37:59.648 | INFO | "gpu" : " 17 GB", | 20260508 01:37:59.648 | INFO | "huge" : "", | 20260508 01:37:59.648 | INFO | "huge_shared" : " 29 GB", | 20260508 01:37:59.648 | INFO | "malloc" : " 14 GB", | 20260508 01:37:59.648 | INFO | "pinned" : " 29 GB" | 20260508 01:37:59.648 | INFO | } | 20260508 01:37:59.648 | INFO | Perf report for the last 5.1399 seconds: | 20260508 01:37:59.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:37:59.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:59.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:59.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:59.648 | INFO | ANALYSIS : 68.3%, 2.9, 105.6ms, 0 | 20260508 01:37:59.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:37:59.648 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260508 01:37:59.648 | INFO | PRE_HQ : 65.7%, 1.7, 58.9ms, 0 | 20260508 01:37:59.648 | INFO | BAZWRITER : 68.3%, 3.6, 127.4ms, 0 | 20260508 01:37:59.648 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:37:59.649 | INFO | Uploaded 36GiB to GPU in 2.5438s (14.152GiB/s) | 20260508 01:37:59.649 | INFO | Downloaded 38GiB to GPU in 2.69371s (14.107GiB/s) | 20260508 01:38:04.790 | INFO | Analyzing chunk frames = [4219392,4219904) | 20260508 01:38:04.790 | INFO | MemInfo: { | 20260508 01:38:04.790 | INFO | "gpu" : " 17 GB", | 20260508 01:38:04.790 | INFO | "huge" : "", | 20260508 01:38:04.790 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:04.790 | INFO | "malloc" : " 14 GB", | 20260508 01:38:04.790 | INFO | "pinned" : " 29 GB" | 20260508 01:38:04.790 | INFO | } | 20260508 01:38:04.790 | INFO | Perf report for the last 5.14174 seconds: | 20260508 01:38:04.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:04.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:38:04.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 01:38:04.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:04.790 | INFO | Uploaded 36GiB to GPU in 2.50024s (14.3986GiB/s) | 20260508 01:38:04.790 | INFO | Downloaded 34GiB to GPU in 2.43871s (13.9418GiB/s) | 20260508 01:38:09.934 | INFO | Analyzing chunk frames = [4219904,4220416) | 20260508 01:38:09.934 | INFO | MemInfo: { | 20260508 01:38:09.934 | INFO | "gpu" : " 17 GB", | 20260508 01:38:09.934 | INFO | "huge" : "", | 20260508 01:38:09.934 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:09.934 | INFO | "malloc" : " 14 GB", | 20260508 01:38:09.934 | INFO | "pinned" : " 29 GB" | 20260508 01:38:09.934 | INFO | } | 20260508 01:38:09.934 | INFO | Perf report for the last 5.14389 seconds: | 20260508 01:38:09.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:09.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 01:38:09.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:38:09.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:09.934 | INFO | Uploaded 36GiB to GPU in 2.48243s (14.5019GiB/s) | 20260508 01:38:09.934 | INFO | Downloaded 34GiB to GPU in 2.49662s (13.6184GiB/s) | 20260508 01:38:15.067 | INFO | Analyzing chunk frames = [4220416,4220928) | 20260508 01:38:15.067 | INFO | MemInfo: { | 20260508 01:38:15.067 | INFO | "gpu" : " 17 GB", | 20260508 01:38:15.067 | INFO | "huge" : "", | 20260508 01:38:15.067 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:15.067 | INFO | "malloc" : " 14 GB", | 20260508 01:38:15.067 | INFO | "pinned" : " 29 GB" | 20260508 01:38:15.067 | INFO | } | 20260508 01:38:15.067 | INFO | Perf report for the last 5.13317 seconds: | 20260508 01:38:15.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:15.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | ANALYSIS : 67.3%, 3.0, 104.0ms, 0 | 20260508 01:38:15.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:38:15.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:15.067 | INFO | Uploaded 36GiB to GPU in 2.49371s (14.4363GiB/s) | 20260508 01:38:15.067 | INFO | Downloaded 34GiB to GPU in 2.52005s (13.4918GiB/s) | 20260508 01:38:20.202 | INFO | Analyzing chunk frames = [4220928,4221440) | 20260508 01:38:20.202 | INFO | MemInfo: { | 20260508 01:38:20.202 | INFO | "gpu" : " 17 GB", | 20260508 01:38:20.202 | INFO | "huge" : "", | 20260508 01:38:20.202 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:20.202 | INFO | "malloc" : " 14 GB", | 20260508 01:38:20.202 | INFO | "pinned" : " 29 GB" | 20260508 01:38:20.202 | INFO | } | 20260508 01:38:20.202 | INFO | Perf report for the last 5.13489 seconds: | 20260508 01:38:20.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:20.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 01:38:20.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:38:20.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:20.202 | INFO | Uploaded 36GiB to GPU in 2.31695s (15.5376GiB/s) | 20260508 01:38:20.202 | INFO | Downloaded 34GiB to GPU in 2.48982s (13.6556GiB/s) | 20260508 01:38:25.343 | INFO | Analyzing chunk frames = [4221440,4221952) | 20260508 01:38:25.343 | INFO | MemInfo: { | 20260508 01:38:25.343 | INFO | "gpu" : " 17 GB", | 20260508 01:38:25.343 | INFO | "huge" : "", | 20260508 01:38:25.343 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:25.343 | INFO | "malloc" : " 14 GB", | 20260508 01:38:25.343 | INFO | "pinned" : " 29 GB" | 20260508 01:38:25.343 | INFO | } | 20260508 01:38:25.344 | INFO | Perf report for the last 5.14157 seconds: | 20260508 01:38:25.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:25.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 01:38:25.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:38:25.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:25.344 | INFO | Uploaded 36GiB to GPU in 2.40207s (14.9871GiB/s) | 20260508 01:38:25.344 | INFO | Downloaded 34GiB to GPU in 2.40611s (14.1307GiB/s) | 20260508 01:38:30.487 | INFO | Analyzing chunk frames = [4221952,4222464) | 20260508 01:38:30.487 | INFO | MemInfo: { | 20260508 01:38:30.487 | INFO | "gpu" : " 17 GB", | 20260508 01:38:30.487 | INFO | "huge" : "", | 20260508 01:38:30.487 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:30.487 | INFO | "malloc" : " 14 GB", | 20260508 01:38:30.487 | INFO | "pinned" : " 29 GB" | 20260508 01:38:30.487 | INFO | } | 20260508 01:38:30.487 | INFO | Perf report for the last 5.14343 seconds: | 20260508 01:38:30.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:30.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 01:38:30.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:38:30.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:30.487 | INFO | Uploaded 36GiB to GPU in 2.53628s (14.194GiB/s) | 20260508 01:38:30.488 | INFO | Downloaded 34GiB to GPU in 2.46725s (13.7806GiB/s) | 20260508 01:38:35.629 | INFO | Analyzing chunk frames = [4222464,4222976) | 20260508 01:38:35.629 | INFO | MemInfo: { | 20260508 01:38:35.629 | INFO | "gpu" : " 17 GB", | 20260508 01:38:35.629 | INFO | "huge" : "", | 20260508 01:38:35.629 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:35.629 | INFO | "malloc" : " 14 GB", | 20260508 01:38:35.629 | INFO | "pinned" : " 29 GB" | 20260508 01:38:35.629 | INFO | } | 20260508 01:38:35.629 | INFO | Perf report for the last 5.14198 seconds: | 20260508 01:38:35.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:35.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:38:35.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 01:38:35.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:35.629 | INFO | Uploaded 36GiB to GPU in 2.56133s (14.0552GiB/s) | 20260508 01:38:35.630 | INFO | Downloaded 34GiB to GPU in 2.46674s (13.7834GiB/s) | 20260508 01:38:35.736 | INFO | Starting a round of Baz IO | 20260508 01:38:39.198 | INFO | Populating RTMetrics report | 20260508 01:38:39.198 | INFO | RTMetrics report populated | 20260508 01:38:39.198 | INFO | Opening tmp RTMetrics json file | 20260508 01:38:39.214 | INFO | Serializing RTMetrics to json | 20260508 01:38:39.223 | INFO | Serialization done, swapping tmp file now... | 20260508 01:38:39.224 | INFO | RTMetrics done | 20260508 01:38:39.300 | INFO | Finished a round of Baz IO: Wrote 1.657 GiB in 3.564 seconds | 20260508 01:38:39.301 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.78% / 0.00% / 58.22% | 20260508 01:38:39.301 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.823247% / 0% / 5.16574e-05% | 20260508 01:38:39.301 | INFO | Baz IO overall: | 20260508 01:38:39.301 | INFO | BytesWritten : 1912338523031 | 20260508 01:38:39.301 | INFO | NumEvents : 924940183807 | 20260508 01:38:40.762 | INFO | Analyzing chunk frames = [4222976,4223488) | 20260508 01:38:40.762 | INFO | MemInfo: { | 20260508 01:38:40.762 | INFO | "gpu" : " 17 GB", | 20260508 01:38:40.762 | INFO | "huge" : "", | 20260508 01:38:40.762 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:40.762 | INFO | "malloc" : " 14 GB", | 20260508 01:38:40.763 | INFO | "pinned" : " 29 GB" | 20260508 01:38:40.763 | INFO | } | 20260508 01:38:40.763 | INFO | Perf report for the last 5.1337 seconds: | 20260508 01:38:40.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:40.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:40.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:40.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:40.763 | INFO | ANALYSIS : 68.8%, 2.9, 104.9ms, 0 | 20260508 01:38:40.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:40.763 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260508 01:38:40.763 | INFO | PRE_HQ : 67.4%, 1.8, 61.3ms, 0 | 20260508 01:38:40.763 | INFO | BAZWRITER : 68.8%, 4.1, 145.0ms, 0 | 20260508 01:38:40.763 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 01:38:40.763 | INFO | Uploaded 36GiB to GPU in 2.60123s (13.8396GiB/s) | 20260508 01:38:40.763 | INFO | Downloaded 38GiB to GPU in 2.70438s (14.0513GiB/s) | 20260508 01:38:45.907 | INFO | Analyzing chunk frames = [4223488,4224000) | 20260508 01:38:45.907 | INFO | MemInfo: { | 20260508 01:38:45.907 | INFO | "gpu" : " 17 GB", | 20260508 01:38:45.907 | INFO | "huge" : "", | 20260508 01:38:45.907 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:45.907 | INFO | "malloc" : " 14 GB", | 20260508 01:38:45.907 | INFO | "pinned" : " 29 GB" | 20260508 01:38:45.908 | INFO | } | 20260508 01:38:45.908 | INFO | Perf report for the last 5.14487 seconds: | 20260508 01:38:45.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:45.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 01:38:45.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:38:45.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:45.908 | INFO | Uploaded 36GiB to GPU in 2.53126s (14.2222GiB/s) | 20260508 01:38:45.908 | INFO | Downloaded 34GiB to GPU in 2.43245s (13.9777GiB/s) | 20260508 01:38:51.046 | INFO | Analyzing chunk frames = [4224000,4224512) | 20260508 01:38:51.046 | INFO | MemInfo: { | 20260508 01:38:51.046 | INFO | "gpu" : " 17 GB", | 20260508 01:38:51.046 | INFO | "huge" : "", | 20260508 01:38:51.046 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:51.046 | INFO | "malloc" : " 14 GB", | 20260508 01:38:51.046 | INFO | "pinned" : " 29 GB" | 20260508 01:38:51.046 | INFO | } | 20260508 01:38:51.046 | INFO | Perf report for the last 5.13847 seconds: | 20260508 01:38:51.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:51.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:38:51.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:38:51.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:51.046 | INFO | Uploaded 36GiB to GPU in 2.5342s (14.2057GiB/s) | 20260508 01:38:51.046 | INFO | Downloaded 34GiB to GPU in 2.50388s (13.5789GiB/s) | 20260508 01:38:56.185 | INFO | Analyzing chunk frames = [4224512,4225024) | 20260508 01:38:56.185 | INFO | MemInfo: { | 20260508 01:38:56.185 | INFO | "gpu" : " 17 GB", | 20260508 01:38:56.185 | INFO | "huge" : "", | 20260508 01:38:56.185 | INFO | "huge_shared" : " 29 GB", | 20260508 01:38:56.185 | INFO | "malloc" : " 14 GB", | 20260508 01:38:56.185 | INFO | "pinned" : " 29 GB" | 20260508 01:38:56.185 | INFO | } | 20260508 01:38:56.185 | INFO | Perf report for the last 5.13893 seconds: | 20260508 01:38:56.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:38:56.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:38:56.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:38:56.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:38:56.185 | INFO | Uploaded 36GiB to GPU in 2.54693s (14.1347GiB/s) | 20260508 01:38:56.185 | INFO | Downloaded 34GiB to GPU in 2.50463s (13.5749GiB/s) | 20260508 01:39:01.318 | INFO | Analyzing chunk frames = [4225024,4225536) | 20260508 01:39:01.318 | INFO | MemInfo: { | 20260508 01:39:01.318 | INFO | "gpu" : " 17 GB", | 20260508 01:39:01.318 | INFO | "huge" : "", | 20260508 01:39:01.318 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:01.318 | INFO | "malloc" : " 14 GB", | 20260508 01:39:01.318 | INFO | "pinned" : " 29 GB" | 20260508 01:39:01.318 | INFO | } | 20260508 01:39:01.318 | INFO | Perf report for the last 5.13313 seconds: | 20260508 01:39:01.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:01.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 01:39:01.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:39:01.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:01.318 | INFO | Uploaded 36GiB to GPU in 2.41867s (14.8842GiB/s) | 20260508 01:39:01.318 | INFO | Downloaded 34GiB to GPU in 2.57546s (13.2015GiB/s) | 20260508 01:39:06.458 | INFO | Analyzing chunk frames = [4225536,4226048) | 20260508 01:39:06.458 | INFO | MemInfo: { | 20260508 01:39:06.458 | INFO | "gpu" : " 17 GB", | 20260508 01:39:06.458 | INFO | "huge" : "", | 20260508 01:39:06.458 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:06.458 | INFO | "malloc" : " 14 GB", | 20260508 01:39:06.458 | INFO | "pinned" : " 29 GB" | 20260508 01:39:06.458 | INFO | } | 20260508 01:39:06.458 | INFO | Perf report for the last 5.14009 seconds: | 20260508 01:39:06.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:06.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.458 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:39:06.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.458 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:39:06.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:06.459 | INFO | Uploaded 36GiB to GPU in 2.40197s (14.9877GiB/s) | 20260508 01:39:06.459 | INFO | Downloaded 34GiB to GPU in 2.39996s (14.1669GiB/s) | 20260508 01:39:11.600 | INFO | Analyzing chunk frames = [4226048,4226560) | 20260508 01:39:11.601 | INFO | MemInfo: { | 20260508 01:39:11.601 | INFO | "gpu" : " 17 GB", | 20260508 01:39:11.601 | INFO | "huge" : "", | 20260508 01:39:11.601 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:11.601 | INFO | "malloc" : " 14 GB", | 20260508 01:39:11.601 | INFO | "pinned" : " 29 GB" | 20260508 01:39:11.601 | INFO | } | 20260508 01:39:11.601 | INFO | Perf report for the last 5.14276 seconds: | 20260508 01:39:11.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:11.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 01:39:11.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 01:39:11.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:11.601 | INFO | Uploaded 36GiB to GPU in 2.51258s (14.3279GiB/s) | 20260508 01:39:11.601 | INFO | Downloaded 34GiB to GPU in 2.45328s (13.859GiB/s) | 20260508 01:39:16.740 | INFO | Analyzing chunk frames = [4226560,4227072) | 20260508 01:39:16.740 | INFO | MemInfo: { | 20260508 01:39:16.740 | INFO | "gpu" : " 17 GB", | 20260508 01:39:16.740 | INFO | "huge" : "", | 20260508 01:39:16.740 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:16.740 | INFO | "malloc" : " 14 GB", | 20260508 01:39:16.740 | INFO | "pinned" : " 29 GB" | 20260508 01:39:16.740 | INFO | } | 20260508 01:39:16.740 | INFO | Perf report for the last 5.13912 seconds: | 20260508 01:39:16.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:16.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.740 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:39:16.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.740 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 01:39:16.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:16.741 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:39:16.741 | INFO | Total mem usage: 16.7 GiB | 20260508 01:39:16.741 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:39:16.741 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260508 01:39:16.741 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:39:16.741 | INFO | High Water Breakdown: | 20260508 01:39:16.741 | INFO | 59.9% : 8.81 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:39:16.741 | INFO | Uploaded 36GiB to GPU in 2.53784s (14.1853GiB/s) | 20260508 01:39:16.741 | INFO | 39.9% : 5.88 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:39:16.741 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:39:16.741 | INFO | ------------------------------------------ | 20260508 01:39:16.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:39:16.741 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:39:16.741 | INFO | Total mem usage: 26.5 GiB | 20260508 01:39:16.741 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:39:16.741 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:39:16.741 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:39:16.741 | INFO | High Water Breakdown: | 20260508 01:39:16.741 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:39:16.741 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:39:16.741 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:39:16.741 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:39:16.741 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:39:16.741 | INFO | Downloaded 34GiB to GPU in 2.4651s (13.7925GiB/s) | 20260508 01:39:16.741 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:39:16.741 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:39:16.741 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:39:16.741 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:39:16.741 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:39:16.741 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:39:16.741 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:39:16.741 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:39:16.741 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:39:16.741 | INFO | ------------------------------------------ | 20260508 01:39:16.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:39:16.741 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:39:16.741 | INFO | Total mem usage: 27.8 GiB | 20260508 01:39:16.741 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:39:16.741 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:39:16.741 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:39:16.741 | INFO | High Water Breakdown: | 20260508 01:39:16.741 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:39:16.741 | INFO | ------------------------------------------ | 20260508 01:39:16.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:39:16.741 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:39:16.741 | INFO | Total mem usage: 15.7 GiB | 20260508 01:39:16.741 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:39:16.741 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:39:16.741 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:39:16.741 | INFO | High Water Breakdown: | 20260508 01:39:16.741 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:39:16.741 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:39:16.741 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:39:16.741 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:39:16.741 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:39:16.741 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:39:16.741 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:39:16.741 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:39:16.741 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:39:16.741 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:39:16.741 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:39:16.741 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:39:16.742 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:39:16.742 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:39:16.742 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:39:16.742 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:39:16.742 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:39:16.742 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:39:16.742 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:39:16.742 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:39:16.742 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:39:16.742 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:39:16.742 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:39:16.742 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:39:16.742 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:39:16.742 | INFO | ------------------------------------------ | 20260508 01:39:16.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:39:16.742 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:39:16.742 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:39:16.742 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:39:16.742 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.61e+01, 1.35e+01 / 1.67e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.13e-02 / 5.24e-03 | 20260508 01:39:16.742 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.44e+00, 5.75e-01 / 1.92e-01 | 20260508 01:39:16.742 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.03e-01 / 1.04e-01 | 20260508 01:39:16.742 | INFO | PerfReport: 16.07%, DME -- 4.19e+01 / 4.60e+01, 3.96e+00 / 1.85e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.57e+00 / 1.56e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.44e+00 / 2.67e+00, 4.23e-01 / 2.41e-01 | 20260508 01:39:16.742 | INFO | PerfReport: 3.32%, Metrics -- 1.08e+00 / 1.23e+00, 2.67e-01 / 1.10e-01 | 20260508 01:39:16.742 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.71e-01, 5.28e-02 / 3.06e-02 | 20260508 01:39:16.742 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:39:16.742 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:39:16.742 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:39:16.742 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 24.39%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 24.45%, Download -- 2.61e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 30.63%, Compute -- 3.26e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 2.90%, LockDownload -- 3.09e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 15.47%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:39:16.742 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260508 01:39:16.837 | INFO | Starting a round of Baz IO | 20260508 01:39:20.284 | INFO | Populating RTMetrics report | 20260508 01:39:20.284 | INFO | RTMetrics report populated | 20260508 01:39:20.284 | INFO | Opening tmp RTMetrics json file | 20260508 01:39:20.309 | INFO | Serializing RTMetrics to json | 20260508 01:39:20.311 | INFO | Serialization done, swapping tmp file now... | 20260508 01:39:20.311 | INFO | RTMetrics done | 20260508 01:39:20.391 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.553 seconds | 20260508 01:39:20.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.77% / 0.00% / 58.22% | 20260508 01:39:20.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.820537% / 7.94729e-06% / 6.75519e-05% | 20260508 01:39:20.391 | INFO | Baz IO overall: | 20260508 01:39:20.391 | INFO | BytesWritten : 1914116446912 | 20260508 01:39:20.391 | INFO | NumEvents : 925793573648 | 20260508 01:39:21.882 | INFO | Analyzing chunk frames = [4227072,4227584) | 20260508 01:39:21.882 | INFO | MemInfo: { | 20260508 01:39:21.882 | INFO | "gpu" : " 17 GB", | 20260508 01:39:21.882 | INFO | "huge" : "", | 20260508 01:39:21.882 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:21.882 | INFO | "malloc" : " 14 GB", | 20260508 01:39:21.882 | INFO | "pinned" : " 29 GB" | 20260508 01:39:21.882 | INFO | } | 20260508 01:39:21.882 | INFO | Perf report for the last 5.14213 seconds: | 20260508 01:39:21.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:21.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:21.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:21.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:21.883 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 01:39:21.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:21.883 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:39:21.883 | INFO | PRE_HQ : 65.5%, 1.6, 56.0ms, 0 | 20260508 01:39:21.883 | INFO | BAZWRITER : 68.8%, 3.8, 135.9ms, 0 | 20260508 01:39:21.883 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:39:21.883 | INFO | Uploaded 36GiB to GPU in 2.61508s (13.7663GiB/s) | 20260508 01:39:21.883 | INFO | Downloaded 38GiB to GPU in 2.67968s (14.1808GiB/s) | 20260508 01:39:27.017 | INFO | Analyzing chunk frames = [4227584,4228096) | 20260508 01:39:27.017 | INFO | MemInfo: { | 20260508 01:39:27.017 | INFO | "gpu" : " 17 GB", | 20260508 01:39:27.017 | INFO | "huge" : "", | 20260508 01:39:27.017 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:27.017 | INFO | "malloc" : " 14 GB", | 20260508 01:39:27.017 | INFO | "pinned" : " 29 GB" | 20260508 01:39:27.017 | INFO | } | 20260508 01:39:27.017 | INFO | Perf report for the last 5.13471 seconds: | 20260508 01:39:27.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:27.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.017 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 01:39:27.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.017 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:39:27.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:27.018 | INFO | Uploaded 36GiB to GPU in 2.5797s (13.9551GiB/s) | 20260508 01:39:27.018 | INFO | Downloaded 34GiB to GPU in 2.41813s (14.0605GiB/s) | 20260508 01:39:32.160 | INFO | Analyzing chunk frames = [4228096,4228608) | 20260508 01:39:32.160 | INFO | MemInfo: { | 20260508 01:39:32.160 | INFO | "gpu" : " 17 GB", | 20260508 01:39:32.160 | INFO | "huge" : "", | 20260508 01:39:32.160 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:32.160 | INFO | "malloc" : " 14 GB", | 20260508 01:39:32.160 | INFO | "pinned" : " 29 GB" | 20260508 01:39:32.160 | INFO | } | 20260508 01:39:32.160 | INFO | Perf report for the last 5.1428 seconds: | 20260508 01:39:32.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:32.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.160 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:39:32.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.160 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:39:32.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:32.161 | INFO | Uploaded 36GiB to GPU in 2.55909s (14.0675GiB/s) | 20260508 01:39:32.161 | INFO | Downloaded 34GiB to GPU in 2.47989s (13.7103GiB/s) | 20260508 01:39:37.294 | INFO | Analyzing chunk frames = [4228608,4229120) | 20260508 01:39:37.294 | INFO | MemInfo: { | 20260508 01:39:37.294 | INFO | "gpu" : " 17 GB", | 20260508 01:39:37.294 | INFO | "huge" : "", | 20260508 01:39:37.294 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:37.294 | INFO | "malloc" : " 14 GB", | 20260508 01:39:37.294 | INFO | "pinned" : " 29 GB" | 20260508 01:39:37.294 | INFO | } | 20260508 01:39:37.294 | INFO | Perf report for the last 5.13404 seconds: | 20260508 01:39:37.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:37.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 01:39:37.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:39:37.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:37.294 | INFO | Uploaded 36GiB to GPU in 2.56946s (14.0107GiB/s) | 20260508 01:39:37.294 | INFO | Downloaded 34GiB to GPU in 2.48365s (13.6895GiB/s) | 20260508 01:39:42.435 | INFO | Analyzing chunk frames = [4229120,4229632) | 20260508 01:39:42.436 | INFO | MemInfo: { | 20260508 01:39:42.436 | INFO | "gpu" : " 17 GB", | 20260508 01:39:42.436 | INFO | "huge" : "", | 20260508 01:39:42.436 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:42.436 | INFO | "malloc" : " 14 GB", | 20260508 01:39:42.436 | INFO | "pinned" : " 29 GB" | 20260508 01:39:42.436 | INFO | } | 20260508 01:39:42.436 | INFO | Perf report for the last 5.14186 seconds: | 20260508 01:39:42.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:42.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 01:39:42.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 01:39:42.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:42.436 | INFO | Uploaded 36GiB to GPU in 2.36995s (15.1902GiB/s) | 20260508 01:39:42.436 | INFO | Downloaded 34GiB to GPU in 2.49049s (13.6519GiB/s) | 20260508 01:39:47.575 | INFO | Analyzing chunk frames = [4229632,4230144) | 20260508 01:39:47.575 | INFO | MemInfo: { | 20260508 01:39:47.575 | INFO | "gpu" : " 17 GB", | 20260508 01:39:47.575 | INFO | "huge" : "", | 20260508 01:39:47.575 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:47.575 | INFO | "malloc" : " 14 GB", | 20260508 01:39:47.576 | INFO | "pinned" : " 29 GB" | 20260508 01:39:47.576 | INFO | } | 20260508 01:39:47.576 | INFO | Perf report for the last 5.13985 seconds: | 20260508 01:39:47.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:47.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 01:39:47.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:39:47.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:47.576 | INFO | Uploaded 36GiB to GPU in 2.39356s (15.0404GiB/s) | 20260508 01:39:47.576 | INFO | Downloaded 34GiB to GPU in 2.3957s (14.1921GiB/s) | 20260508 01:39:52.717 | INFO | Analyzing chunk frames = [4230144,4230656) | 20260508 01:39:52.717 | INFO | MemInfo: { | 20260508 01:39:52.717 | INFO | "gpu" : " 17 GB", | 20260508 01:39:52.717 | INFO | "huge" : "", | 20260508 01:39:52.717 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:52.717 | INFO | "malloc" : " 14 GB", | 20260508 01:39:52.717 | INFO | "pinned" : " 29 GB" | 20260508 01:39:52.717 | INFO | } | 20260508 01:39:52.718 | INFO | Perf report for the last 5.14188 seconds: | 20260508 01:39:52.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:52.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:39:52.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 01:39:52.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:52.718 | INFO | Uploaded 36GiB to GPU in 2.45128s (14.6862GiB/s) | 20260508 01:39:52.718 | INFO | Downloaded 34GiB to GPU in 2.42303s (14.032GiB/s) | 20260508 01:39:57.858 | INFO | Analyzing chunk frames = [4230656,4231168) | 20260508 01:39:57.858 | INFO | MemInfo: { | 20260508 01:39:57.858 | INFO | "gpu" : " 17 GB", | 20260508 01:39:57.858 | INFO | "huge" : "", | 20260508 01:39:57.858 | INFO | "huge_shared" : " 29 GB", | 20260508 01:39:57.858 | INFO | "malloc" : " 14 GB", | 20260508 01:39:57.858 | INFO | "pinned" : " 29 GB" | 20260508 01:39:57.858 | INFO | } | 20260508 01:39:57.858 | INFO | Perf report for the last 5.14054 seconds: | 20260508 01:39:57.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:39:57.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.858 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 01:39:57.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.858 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260508 01:39:57.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:39:57.859 | INFO | Uploaded 36GiB to GPU in 2.46514s (14.6036GiB/s) | 20260508 01:39:57.859 | INFO | Downloaded 34GiB to GPU in 2.43049s (13.989GiB/s) | 20260508 01:39:57.955 | INFO | Starting a round of Baz IO | 20260508 01:40:01.407 | INFO | Populating RTMetrics report | 20260508 01:40:01.407 | INFO | RTMetrics report populated | 20260508 01:40:01.407 | INFO | Opening tmp RTMetrics json file | 20260508 01:40:01.420 | INFO | Serializing RTMetrics to json | 20260508 01:40:01.427 | INFO | Serialization done, swapping tmp file now... | 20260508 01:40:01.428 | INFO | RTMetrics done | 20260508 01:40:01.518 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.562 seconds | 20260508 01:40:01.518 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.77% / 0.00% / 58.23% | 20260508 01:40:01.518 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.820585% / 1.19209e-05% / 6.75519e-05% | 20260508 01:40:01.518 | INFO | Baz IO overall: | 20260508 01:40:01.518 | INFO | BytesWritten : 1915894581406 | 20260508 01:40:01.518 | INFO | NumEvents : 926647222687 | 20260508 01:40:02.999 | INFO | Analyzing chunk frames = [4231168,4231680) | 20260508 01:40:02.999 | INFO | MemInfo: { | 20260508 01:40:02.999 | INFO | "gpu" : " 17 GB", | 20260508 01:40:02.999 | INFO | "huge" : "", | 20260508 01:40:02.999 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:02.999 | INFO | "malloc" : " 14 GB", | 20260508 01:40:02.999 | INFO | "pinned" : " 29 GB" | 20260508 01:40:02.999 | INFO | } | 20260508 01:40:02.999 | INFO | Perf report for the last 5.14108 seconds: | 20260508 01:40:02.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:02.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:02.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:02.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:02.999 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 01:40:02.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:02.999 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 01:40:02.999 | INFO | PRE_HQ : 66.0%, 1.6, 56.2ms, 0 | 20260508 01:40:02.999 | INFO | BAZWRITER : 68.9%, 3.6, 128.0ms, 0 | 20260508 01:40:02.999 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 01:40:02.999 | INFO | Uploaded 36GiB to GPU in 2.54901s (14.1231GiB/s) | 20260508 01:40:03.000 | INFO | Downloaded 38GiB to GPU in 2.69326s (14.1093GiB/s) | 20260508 01:40:08.134 | INFO | Analyzing chunk frames = [4231680,4232192) | 20260508 01:40:08.134 | INFO | MemInfo: { | 20260508 01:40:08.134 | INFO | "gpu" : " 17 GB", | 20260508 01:40:08.134 | INFO | "huge" : "", | 20260508 01:40:08.134 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:08.134 | INFO | "malloc" : " 14 GB", | 20260508 01:40:08.135 | INFO | "pinned" : " 29 GB" | 20260508 01:40:08.135 | INFO | } | 20260508 01:40:08.135 | INFO | Perf report for the last 5.13546 seconds: | 20260508 01:40:08.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:08.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 01:40:08.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:40:08.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:08.135 | INFO | Uploaded 36GiB to GPU in 2.48926s (14.4622GiB/s) | 20260508 01:40:08.135 | INFO | Downloaded 34GiB to GPU in 2.43373s (13.9703GiB/s) | 20260508 01:40:13.268 | INFO | Analyzing chunk frames = [4232192,4232704) | 20260508 01:40:13.268 | INFO | MemInfo: { | 20260508 01:40:13.268 | INFO | "gpu" : " 17 GB", | 20260508 01:40:13.268 | INFO | "huge" : "", | 20260508 01:40:13.268 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:13.268 | INFO | "malloc" : " 14 GB", | 20260508 01:40:13.268 | INFO | "pinned" : " 29 GB" | 20260508 01:40:13.268 | INFO | } | 20260508 01:40:13.268 | INFO | Perf report for the last 5.13357 seconds: | 20260508 01:40:13.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:13.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:40:13.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:40:13.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:13.268 | INFO | Uploaded 36GiB to GPU in 2.51022s (14.3414GiB/s) | 20260508 01:40:13.269 | INFO | Downloaded 34GiB to GPU in 2.51172s (13.5366GiB/s) | 20260508 01:40:18.410 | INFO | Analyzing chunk frames = [4232704,4233216) | 20260508 01:40:18.410 | INFO | MemInfo: { | 20260508 01:40:18.410 | INFO | "gpu" : " 17 GB", | 20260508 01:40:18.410 | INFO | "huge" : "", | 20260508 01:40:18.410 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:18.410 | INFO | "malloc" : " 14 GB", | 20260508 01:40:18.410 | INFO | "pinned" : " 29 GB" | 20260508 01:40:18.410 | INFO | } | 20260508 01:40:18.410 | INFO | Perf report for the last 5.14178 seconds: | 20260508 01:40:18.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:18.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 01:40:18.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:40:18.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:18.410 | INFO | Uploaded 36GiB to GPU in 2.47132s (14.5671GiB/s) | 20260508 01:40:18.410 | INFO | Downloaded 34GiB to GPU in 2.50827s (13.5552GiB/s) | 20260508 01:40:23.551 | INFO | Analyzing chunk frames = [4233216,4233728) | 20260508 01:40:23.551 | INFO | MemInfo: { | 20260508 01:40:23.551 | INFO | "gpu" : " 17 GB", | 20260508 01:40:23.551 | INFO | "huge" : "", | 20260508 01:40:23.551 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:23.551 | INFO | "malloc" : " 14 GB", | 20260508 01:40:23.551 | INFO | "pinned" : " 29 GB" | 20260508 01:40:23.551 | INFO | } | 20260508 01:40:23.551 | INFO | Perf report for the last 5.14114 seconds: | 20260508 01:40:23.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:23.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.551 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:40:23.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.551 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:40:23.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:23.552 | INFO | Uploaded 36GiB to GPU in 2.28785s (15.7353GiB/s) | 20260508 01:40:23.552 | INFO | Downloaded 34GiB to GPU in 2.50994s (13.5461GiB/s) | 20260508 01:40:28.693 | INFO | Analyzing chunk frames = [4233728,4234240) | 20260508 01:40:28.693 | INFO | MemInfo: { | 20260508 01:40:28.693 | INFO | "gpu" : " 17 GB", | 20260508 01:40:28.693 | INFO | "huge" : "", | 20260508 01:40:28.693 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:28.693 | INFO | "malloc" : " 14 GB", | 20260508 01:40:28.694 | INFO | "pinned" : " 29 GB" | 20260508 01:40:28.694 | INFO | } | 20260508 01:40:28.694 | INFO | Perf report for the last 5.14252 seconds: | 20260508 01:40:28.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:28.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:40:28.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:40:28.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:28.694 | INFO | Uploaded 36GiB to GPU in 2.39329s (15.042GiB/s) | 20260508 01:40:28.694 | INFO | Downloaded 34GiB to GPU in 2.40662s (14.1277GiB/s) | 20260508 01:40:33.829 | INFO | Analyzing chunk frames = [4234240,4234752) | 20260508 01:40:33.829 | INFO | MemInfo: { | 20260508 01:40:33.829 | INFO | "gpu" : " 17 GB", | 20260508 01:40:33.829 | INFO | "huge" : "", | 20260508 01:40:33.829 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:33.829 | INFO | "malloc" : " 14 GB", | 20260508 01:40:33.829 | INFO | "pinned" : " 29 GB" | 20260508 01:40:33.829 | INFO | } | 20260508 01:40:33.829 | INFO | Perf report for the last 5.13555 seconds: | 20260508 01:40:33.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:33.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.829 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 01:40:33.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.830 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 01:40:33.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:33.830 | INFO | Uploaded 36GiB to GPU in 2.44741s (14.7094GiB/s) | 20260508 01:40:33.831 | INFO | Downloaded 34GiB to GPU in 2.43369s (13.9705GiB/s) | 20260508 01:40:38.969 | INFO | Analyzing chunk frames = [4234752,4235264) | 20260508 01:40:38.969 | INFO | MemInfo: { | 20260508 01:40:38.969 | INFO | "gpu" : " 17 GB", | 20260508 01:40:38.969 | INFO | "huge" : "", | 20260508 01:40:38.969 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:38.969 | INFO | "malloc" : " 14 GB", | 20260508 01:40:38.970 | INFO | "pinned" : " 29 GB" | 20260508 01:40:38.970 | INFO | } | 20260508 01:40:38.970 | INFO | Perf report for the last 5.14056 seconds: | 20260508 01:40:38.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:38.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 01:40:38.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 01:40:38.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:38.970 | INFO | Uploaded 36GiB to GPU in 2.46344s (14.6137GiB/s) | 20260508 01:40:38.971 | INFO | Downloaded 34GiB to GPU in 2.43805s (13.9455GiB/s) | 20260508 01:40:38.971 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:40:38.971 | INFO | Total mem usage: 16.7 GiB | 20260508 01:40:38.971 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:40:38.971 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 01:40:38.971 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:40:38.971 | INFO | High Water Breakdown: | 20260508 01:40:38.971 | INFO | 60.2% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:40:38.971 | INFO | 39.7% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:40:38.971 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:40:38.971 | INFO | ------------------------------------------ | 20260508 01:40:38.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:40:38.971 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:40:38.971 | INFO | Total mem usage: 26.5 GiB | 20260508 01:40:38.971 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:40:38.971 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:40:38.971 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:40:38.971 | INFO | High Water Breakdown: | 20260508 01:40:38.971 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:40:38.971 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:40:38.971 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:40:38.971 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:40:38.971 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:40:38.971 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:40:38.971 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:40:38.971 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:40:38.971 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:40:38.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:40:38.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:40:38.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:40:38.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:40:38.971 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:40:38.971 | INFO | ------------------------------------------ | 20260508 01:40:38.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:40:38.971 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:40:38.971 | INFO | Total mem usage: 27.8 GiB | 20260508 01:40:38.971 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:40:38.971 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:40:38.971 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:40:38.971 | INFO | High Water Breakdown: | 20260508 01:40:38.971 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:40:38.971 | INFO | ------------------------------------------ | 20260508 01:40:38.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:40:38.971 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:40:38.971 | INFO | Total mem usage: 15.7 GiB | 20260508 01:40:38.972 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:40:38.972 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:40:38.972 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:40:38.972 | INFO | High Water Breakdown: | 20260508 01:40:38.972 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:40:38.972 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:40:38.972 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:40:38.972 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:40:38.972 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:40:38.972 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:40:38.972 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:40:38.972 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:40:38.972 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:40:38.972 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:40:38.972 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:40:38.972 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:40:38.972 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:40:38.972 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:40:38.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:40:38.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:40:38.972 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:40:38.972 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:40:38.972 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:40:38.972 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:40:38.972 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:40:38.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:40:38.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:40:38.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:40:38.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:40:38.972 | INFO | ------------------------------------------ | 20260508 01:40:38.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:40:38.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:40:38.972 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:40:38.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:40:38.972 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.58e+01, 1.43e+01 / 1.51e+00 | 20260508 01:40:38.972 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.14e-02 / 5.82e-03 | 20260508 01:40:38.972 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.51e+00, 5.87e-01 / 1.84e-01 | 20260508 01:40:38.972 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.09e-01 / 1.00e-01 | 20260508 01:40:38.972 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.58e+01, 3.94e+00 / 1.64e+00 | 20260508 01:40:38.972 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.70e+00 / 1.57e+00 | 20260508 01:40:38.972 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.43e+00 / 2.72e+00, 4.23e-01 / 2.37e-01 | 20260508 01:40:38.972 | INFO | PerfReport: 3.31%, Metrics -- 1.09e+00 / 1.25e+00, 2.68e-01 / 1.21e-01 | 20260508 01:40:38.973 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.35e-01 / 3.71e-01, 5.33e-02 / 2.73e-02 | 20260508 01:40:38.973 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:40:38.973 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:40:38.973 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:40:38.973 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 24.27%, Upload -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 30.87%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 2.14%, LockUpload -- 2.27e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 2.70%, LockDownload -- 2.87e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:40:38.973 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260508 01:40:39.078 | INFO | Starting a round of Baz IO | 20260508 01:40:42.536 | INFO | Populating RTMetrics report | 20260508 01:40:42.537 | INFO | RTMetrics report populated | 20260508 01:40:42.537 | INFO | Opening tmp RTMetrics json file | 20260508 01:40:42.559 | INFO | Serializing RTMetrics to json | 20260508 01:40:42.561 | INFO | Serialization done, swapping tmp file now... | 20260508 01:40:42.562 | INFO | RTMetrics done | 20260508 01:40:42.631 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.552 seconds | 20260508 01:40:42.631 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.76% / 0.00% / 58.23% | 20260508 01:40:42.631 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.821181% / 7.94729e-06% / 5.96046e-05% | 20260508 01:40:42.631 | INFO | Baz IO overall: | 20260508 01:40:42.631 | INFO | BytesWritten : 1917675218601 | 20260508 01:40:42.631 | INFO | NumEvents : 927502675873 | 20260508 01:40:44.109 | INFO | Analyzing chunk frames = [4235264,4235776) | 20260508 01:40:44.109 | INFO | MemInfo: { | 20260508 01:40:44.109 | INFO | "gpu" : " 17 GB", | 20260508 01:40:44.109 | INFO | "huge" : "", | 20260508 01:40:44.109 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:44.109 | INFO | "malloc" : " 14 GB", | 20260508 01:40:44.109 | INFO | "pinned" : " 29 GB" | 20260508 01:40:44.109 | INFO | } | 20260508 01:40:44.109 | INFO | Perf report for the last 5.13919 seconds: | 20260508 01:40:44.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:44.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:44.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:44.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:44.109 | INFO | ANALYSIS : 68.7%, 2.9, 104.6ms, 0 | 20260508 01:40:44.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:44.109 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260508 01:40:44.109 | INFO | PRE_HQ : 67.7%, 1.8, 62.3ms, 0 | 20260508 01:40:44.109 | INFO | BAZWRITER : 68.6%, 3.9, 137.1ms, 0 | 20260508 01:40:44.109 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:40:44.110 | INFO | Uploaded 36GiB to GPU in 2.54899s (14.1233GiB/s) | 20260508 01:40:44.110 | INFO | Downloaded 38GiB to GPU in 2.69029s (14.1249GiB/s) | 20260508 01:40:49.246 | INFO | Analyzing chunk frames = [4235776,4236288) | 20260508 01:40:49.247 | INFO | MemInfo: { | 20260508 01:40:49.247 | INFO | "gpu" : " 17 GB", | 20260508 01:40:49.247 | INFO | "huge" : "", | 20260508 01:40:49.247 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:49.247 | INFO | "malloc" : " 14 GB", | 20260508 01:40:49.247 | INFO | "pinned" : " 29 GB" | 20260508 01:40:49.247 | INFO | } | 20260508 01:40:49.247 | INFO | Perf report for the last 5.13753 seconds: | 20260508 01:40:49.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:49.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 01:40:49.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:40:49.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:49.247 | INFO | Uploaded 36GiB to GPU in 2.54846s (14.1262GiB/s) | 20260508 01:40:49.248 | INFO | Downloaded 34GiB to GPU in 2.41285s (14.0912GiB/s) | 20260508 01:40:54.388 | INFO | Analyzing chunk frames = [4236288,4236800) | 20260508 01:40:54.388 | INFO | MemInfo: { | 20260508 01:40:54.388 | INFO | "gpu" : " 17 GB", | 20260508 01:40:54.388 | INFO | "huge" : "", | 20260508 01:40:54.388 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:54.388 | INFO | "malloc" : " 14 GB", | 20260508 01:40:54.388 | INFO | "pinned" : " 29 GB" | 20260508 01:40:54.388 | INFO | } | 20260508 01:40:54.388 | INFO | Perf report for the last 5.14187 seconds: | 20260508 01:40:54.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:54.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.388 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:40:54.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.389 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:40:54.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:54.389 | INFO | Uploaded 36GiB to GPU in 2.55641s (14.0823GiB/s) | 20260508 01:40:54.389 | INFO | Downloaded 34GiB to GPU in 2.47696s (13.7265GiB/s) | 20260508 01:40:59.524 | INFO | Analyzing chunk frames = [4236800,4237312) | 20260508 01:40:59.524 | INFO | MemInfo: { | 20260508 01:40:59.524 | INFO | "gpu" : " 17 GB", | 20260508 01:40:59.524 | INFO | "huge" : "", | 20260508 01:40:59.524 | INFO | "huge_shared" : " 29 GB", | 20260508 01:40:59.524 | INFO | "malloc" : " 14 GB", | 20260508 01:40:59.524 | INFO | "pinned" : " 29 GB" | 20260508 01:40:59.524 | INFO | } | 20260508 01:40:59.524 | INFO | Perf report for the last 5.13528 seconds: | 20260508 01:40:59.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:40:59.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 01:40:59.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:40:59.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:40:59.524 | INFO | Uploaded 36GiB to GPU in 2.56017s (14.0616GiB/s) | 20260508 01:40:59.524 | INFO | Downloaded 34GiB to GPU in 2.48988s (13.6553GiB/s) | 20260508 01:41:04.666 | INFO | Analyzing chunk frames = [4237312,4237824) | 20260508 01:41:04.666 | INFO | MemInfo: { | 20260508 01:41:04.666 | INFO | "gpu" : " 17 GB", | 20260508 01:41:04.666 | INFO | "huge" : "", | 20260508 01:41:04.666 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:04.666 | INFO | "malloc" : " 14 GB", | 20260508 01:41:04.667 | INFO | "pinned" : " 29 GB" | 20260508 01:41:04.667 | INFO | } | 20260508 01:41:04.667 | INFO | Perf report for the last 5.14291 seconds: | 20260508 01:41:04.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:04.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 01:41:04.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:41:04.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:04.667 | INFO | Uploaded 36GiB to GPU in 2.4274s (14.8307GiB/s) | 20260508 01:41:04.667 | INFO | Downloaded 34GiB to GPU in 2.56493s (13.2557GiB/s) | 20260508 01:41:09.809 | INFO | Analyzing chunk frames = [4237824,4238336) | 20260508 01:41:09.809 | INFO | MemInfo: { | 20260508 01:41:09.809 | INFO | "gpu" : " 17 GB", | 20260508 01:41:09.809 | INFO | "huge" : "", | 20260508 01:41:09.809 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:09.809 | INFO | "malloc" : " 14 GB", | 20260508 01:41:09.809 | INFO | "pinned" : " 29 GB" | 20260508 01:41:09.809 | INFO | } | 20260508 01:41:09.809 | INFO | Perf report for the last 5.14232 seconds: | 20260508 01:41:09.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:09.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.809 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:41:09.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.809 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:41:09.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:09.810 | INFO | Uploaded 36GiB to GPU in 2.42638s (14.8369GiB/s) | 20260508 01:41:09.810 | INFO | Downloaded 34GiB to GPU in 2.40132s (14.1589GiB/s) | 20260508 01:41:14.948 | INFO | Analyzing chunk frames = [4238336,4238848) | 20260508 01:41:14.948 | INFO | MemInfo: { | 20260508 01:41:14.948 | INFO | "gpu" : " 17 GB", | 20260508 01:41:14.948 | INFO | "huge" : "", | 20260508 01:41:14.948 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:14.948 | INFO | "malloc" : " 14 GB", | 20260508 01:41:14.949 | INFO | "pinned" : " 29 GB" | 20260508 01:41:14.949 | INFO | } | 20260508 01:41:14.949 | INFO | Perf report for the last 5.13983 seconds: | 20260508 01:41:14.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:14.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:41:14.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 01:41:14.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:14.949 | INFO | Uploaded 36GiB to GPU in 2.46375s (14.6119GiB/s) | 20260508 01:41:14.950 | INFO | Downloaded 34GiB to GPU in 2.4102s (14.1067GiB/s) | 20260508 01:41:20.082 | INFO | Analyzing chunk frames = [4238848,4239360) | 20260508 01:41:20.082 | INFO | MemInfo: { | 20260508 01:41:20.082 | INFO | "gpu" : " 17 GB", | 20260508 01:41:20.082 | INFO | "huge" : "", | 20260508 01:41:20.082 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:20.082 | INFO | "malloc" : " 14 GB", | 20260508 01:41:20.082 | INFO | "pinned" : " 29 GB" | 20260508 01:41:20.082 | INFO | } | 20260508 01:41:20.083 | INFO | Perf report for the last 5.13367 seconds: | 20260508 01:41:20.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:20.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 01:41:20.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 01:41:20.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:20.083 | INFO | Uploaded 36GiB to GPU in 2.49003s (14.4577GiB/s) | 20260508 01:41:20.083 | INFO | Downloaded 34GiB to GPU in 2.42015s (14.0487GiB/s) | 20260508 01:41:20.194 | INFO | Starting a round of Baz IO | 20260508 01:41:23.670 | INFO | Populating RTMetrics report | 20260508 01:41:23.671 | INFO | RTMetrics report populated | 20260508 01:41:23.671 | INFO | Opening tmp RTMetrics json file | 20260508 01:41:23.689 | INFO | Serializing RTMetrics to json | 20260508 01:41:23.744 | INFO | Serialization done, swapping tmp file now... | 20260508 01:41:23.767 | INFO | RTMetrics done | 20260508 01:41:23.793 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.598 seconds | 20260508 01:41:23.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.76% / 0.00% / 58.24% | 20260508 01:41:23.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.815789% / 7.94729e-06% / 5.96046e-05% | 20260508 01:41:23.793 | INFO | Baz IO overall: | 20260508 01:41:23.793 | INFO | BytesWritten : 1919452537365 | 20260508 01:41:23.793 | INFO | NumEvents : 928355736080 | 20260508 01:41:25.220 | INFO | Analyzing chunk frames = [4239360,4239872) | 20260508 01:41:25.220 | INFO | MemInfo: { | 20260508 01:41:25.220 | INFO | "gpu" : " 17 GB", | 20260508 01:41:25.220 | INFO | "huge" : "", | 20260508 01:41:25.220 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:25.220 | INFO | "malloc" : " 14 GB", | 20260508 01:41:25.220 | INFO | "pinned" : " 29 GB" | 20260508 01:41:25.220 | INFO | } | 20260508 01:41:25.221 | INFO | Perf report for the last 5.13802 seconds: | 20260508 01:41:25.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:25.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:25.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:25.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:25.221 | INFO | ANALYSIS : 69.5%, 2.9, 104.9ms, 0 | 20260508 01:41:25.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:25.221 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260508 01:41:25.221 | INFO | PRE_HQ : 68.1%, 1.8, 62.1ms, 0 | 20260508 01:41:25.221 | INFO | BAZWRITER : 69.6%, 4.4, 159.1ms, 0 | 20260508 01:41:25.221 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 01:41:25.222 | INFO | Uploaded 36GiB to GPU in 2.58288s (13.9379GiB/s) | 20260508 01:41:25.222 | INFO | Downloaded 38GiB to GPU in 2.72462s (13.9469GiB/s) | 20260508 01:41:30.360 | INFO | Analyzing chunk frames = [4239872,4240384) | 20260508 01:41:30.360 | INFO | MemInfo: { | 20260508 01:41:30.360 | INFO | "gpu" : " 17 GB", | 20260508 01:41:30.360 | INFO | "huge" : "", | 20260508 01:41:30.360 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:30.360 | INFO | "malloc" : " 14 GB", | 20260508 01:41:30.360 | INFO | "pinned" : " 29 GB" | 20260508 01:41:30.360 | INFO | } | 20260508 01:41:30.360 | INFO | Perf report for the last 5.13949 seconds: | 20260508 01:41:30.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:30.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 01:41:30.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:41:30.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:30.360 | INFO | Uploaded 36GiB to GPU in 2.57328s (13.9899GiB/s) | 20260508 01:41:30.360 | INFO | Downloaded 34GiB to GPU in 2.4955s (13.6245GiB/s) | 20260508 01:41:35.500 | INFO | Analyzing chunk frames = [4240384,4240896) | 20260508 01:41:35.500 | INFO | MemInfo: { | 20260508 01:41:35.500 | INFO | "gpu" : " 17 GB", | 20260508 01:41:35.500 | INFO | "huge" : "", | 20260508 01:41:35.500 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:35.500 | INFO | "malloc" : " 14 GB", | 20260508 01:41:35.500 | INFO | "pinned" : " 29 GB" | 20260508 01:41:35.500 | INFO | } | 20260508 01:41:35.500 | INFO | Perf report for the last 5.14 seconds: | 20260508 01:41:35.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:35.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.500 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:41:35.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.500 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:41:35.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:35.501 | INFO | Uploaded 36GiB to GPU in 2.52224s (14.273GiB/s) | 20260508 01:41:35.501 | INFO | Downloaded 34GiB to GPU in 2.48245s (13.6962GiB/s) | 20260508 01:41:40.641 | INFO | Analyzing chunk frames = [4240896,4241408) | 20260508 01:41:40.641 | INFO | MemInfo: { | 20260508 01:41:40.641 | INFO | "gpu" : " 17 GB", | 20260508 01:41:40.641 | INFO | "huge" : "", | 20260508 01:41:40.641 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:40.641 | INFO | "malloc" : " 14 GB", | 20260508 01:41:40.641 | INFO | "pinned" : " 29 GB" | 20260508 01:41:40.641 | INFO | } | 20260508 01:41:40.641 | INFO | Perf report for the last 5.14129 seconds: | 20260508 01:41:40.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:40.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.641 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:41:40.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.642 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:41:40.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:40.642 | INFO | Uploaded 36GiB to GPU in 2.51804s (14.2968GiB/s) | 20260508 01:41:40.642 | INFO | Downloaded 34GiB to GPU in 2.49309s (13.6377GiB/s) | 20260508 01:41:45.784 | INFO | Analyzing chunk frames = [4241408,4241920) | 20260508 01:41:45.784 | INFO | MemInfo: { | 20260508 01:41:45.784 | INFO | "gpu" : " 17 GB", | 20260508 01:41:45.784 | INFO | "huge" : "", | 20260508 01:41:45.784 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:45.784 | INFO | "malloc" : " 14 GB", | 20260508 01:41:45.784 | INFO | "pinned" : " 29 GB" | 20260508 01:41:45.784 | INFO | } | 20260508 01:41:45.784 | INFO | Perf report for the last 5.14265 seconds: | 20260508 01:41:45.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:45.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 01:41:45.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:41:45.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:45.784 | INFO | Uploaded 36GiB to GPU in 2.38515s (15.0934GiB/s) | 20260508 01:41:45.784 | INFO | Downloaded 34GiB to GPU in 2.54184s (13.3761GiB/s) | 20260508 01:41:50.925 | INFO | Analyzing chunk frames = [4241920,4242432) | 20260508 01:41:50.925 | INFO | MemInfo: { | 20260508 01:41:50.925 | INFO | "gpu" : " 17 GB", | 20260508 01:41:50.925 | INFO | "huge" : "", | 20260508 01:41:50.925 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:50.925 | INFO | "malloc" : " 14 GB", | 20260508 01:41:50.925 | INFO | "pinned" : " 29 GB" | 20260508 01:41:50.925 | INFO | } | 20260508 01:41:50.925 | INFO | Perf report for the last 5.14111 seconds: | 20260508 01:41:50.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:50.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.925 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:41:50.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.925 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:41:50.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:50.926 | INFO | Uploaded 36GiB to GPU in 2.39612s (15.0243GiB/s) | 20260508 01:41:50.926 | INFO | Downloaded 34GiB to GPU in 2.42052s (14.0466GiB/s) | 20260508 01:41:56.059 | INFO | Analyzing chunk frames = [4242432,4242944) | 20260508 01:41:56.059 | INFO | MemInfo: { | 20260508 01:41:56.059 | INFO | "gpu" : " 17 GB", | 20260508 01:41:56.059 | INFO | "huge" : "", | 20260508 01:41:56.059 | INFO | "huge_shared" : " 29 GB", | 20260508 01:41:56.059 | INFO | "malloc" : " 14 GB", | 20260508 01:41:56.059 | INFO | "pinned" : " 29 GB" | 20260508 01:41:56.059 | INFO | } | 20260508 01:41:56.059 | INFO | Perf report for the last 5.13385 seconds: | 20260508 01:41:56.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:41:56.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.059 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 01:41:56.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.059 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 01:41:56.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:41:56.060 | INFO | Uploaded 36GiB to GPU in 2.42448s (14.8485GiB/s) | 20260508 01:41:56.060 | INFO | Downloaded 34GiB to GPU in 2.41734s (14.0651GiB/s) | 20260508 01:42:01.197 | INFO | Analyzing chunk frames = [4242944,4243456) | 20260508 01:42:01.197 | INFO | MemInfo: { | 20260508 01:42:01.197 | INFO | "gpu" : " 17 GB", | 20260508 01:42:01.197 | INFO | "huge" : "", | 20260508 01:42:01.197 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:01.197 | INFO | "malloc" : " 14 GB", | 20260508 01:42:01.197 | INFO | "pinned" : " 29 GB" | 20260508 01:42:01.198 | INFO | } | 20260508 01:42:01.198 | INFO | Perf report for the last 5.13849 seconds: | 20260508 01:42:01.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:01.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:42:01.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:42:01.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:01.198 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:42:01.198 | INFO | Total mem usage: 16.7 GiB | 20260508 01:42:01.198 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:42:01.198 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260508 01:42:01.198 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:42:01.198 | INFO | High Water Breakdown: | 20260508 01:42:01.198 | INFO | 60.3% : 9.06 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:42:01.198 | INFO | 39.6% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:42:01.198 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:42:01.198 | INFO | ------------------------------------------ | 20260508 01:42:01.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:42:01.198 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:42:01.198 | INFO | Total mem usage: 26.5 GiB | 20260508 01:42:01.198 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:42:01.198 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 01:42:01.198 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 01:42:01.198 | INFO | High Water Breakdown: | 20260508 01:42:01.198 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:42:01.198 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:42:01.198 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:42:01.198 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:42:01.198 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:42:01.198 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:42:01.198 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:42:01.198 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:42:01.198 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:42:01.198 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:42:01.198 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:42:01.198 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:42:01.198 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:42:01.198 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:42:01.198 | INFO | ------------------------------------------ | 20260508 01:42:01.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:42:01.198 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:42:01.198 | INFO | Total mem usage: 27.8 GiB | 20260508 01:42:01.198 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:42:01.198 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:42:01.198 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:42:01.198 | INFO | High Water Breakdown: | 20260508 01:42:01.198 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:42:01.198 | INFO | ------------------------------------------ | 20260508 01:42:01.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:42:01.198 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:42:01.198 | INFO | Total mem usage: 15.7 GiB | 20260508 01:42:01.198 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:42:01.198 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:42:01.198 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:42:01.198 | INFO | High Water Breakdown: | 20260508 01:42:01.198 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:42:01.198 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:42:01.198 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:42:01.198 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:42:01.198 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:42:01.198 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:42:01.198 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:42:01.198 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:42:01.198 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:42:01.198 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:42:01.198 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:42:01.198 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:42:01.198 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:42:01.198 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:42:01.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:42:01.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:42:01.198 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:42:01.198 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:42:01.198 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:42:01.198 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:42:01.198 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:42:01.198 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:42:01.198 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:42:01.198 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:42:01.198 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:42:01.198 | INFO | ------------------------------------------ | 20260508 01:42:01.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:42:01.198 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:42:01.198 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:42:01.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:42:01.198 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.41e+01 / 1.72e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.02e-01, 1.17e-02 / 9.90e-03 | 20260508 01:42:01.198 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.43e+00, 5.78e-01 / 2.47e-01 | 20260508 01:42:01.198 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.08e-01 | 20260508 01:42:01.198 | INFO | PerfReport: 16.14%, DME -- 4.23e+01 / 4.63e+01, 4.02e+00 / 2.11e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.64e+00 / 1.62e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.23e-01 / 2.53e-01 | 20260508 01:42:01.198 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.25e+00, 2.70e-01 / 1.37e-01 | 20260508 01:42:01.198 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.67e-01, 5.36e-02 / 2.76e-02 | 20260508 01:42:01.198 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:42:01.198 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:42:01.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:42:01.198 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:42:01.198 | INFO | PerfReport: 2.29%, LockUpload -- 2.44e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 01:42:01.198 | INFO | Uploaded 36GiB to GPU in 2.54664s (14.1363GiB/s) | 20260508 01:42:01.198 | INFO | Downloaded 34GiB to GPU in 2.46216s (13.809GiB/s) | 20260508 01:42:01.198 | INFO | PerfReport: 2.65%, LockDownload -- 2.82e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 01:42:01.199 | INFO | PerfReport: 15.43%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:42:01.199 | INFO | PerfReport: 0.02%, Misc -- 1.64e-02 / 0.00e+00, 5.37e-02 / 0.00e+00 | 20260508 01:42:01.296 | INFO | Starting a round of Baz IO | 20260508 01:42:04.752 | INFO | Populating RTMetrics report | 20260508 01:42:04.753 | INFO | RTMetrics report populated | 20260508 01:42:04.753 | INFO | Opening tmp RTMetrics json file | 20260508 01:42:04.779 | INFO | Serializing RTMetrics to json | 20260508 01:42:04.781 | INFO | Serialization done, swapping tmp file now... | 20260508 01:42:04.781 | INFO | RTMetrics done | 20260508 01:42:04.858 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.562 seconds | 20260508 01:42:04.858 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.76% / 0.00% / 58.24% | 20260508 01:42:04.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.817104% / 7.94729e-06% / 5.96046e-05% | 20260508 01:42:04.859 | INFO | Baz IO overall: | 20260508 01:42:04.859 | INFO | BytesWritten : 1921230828242 | 20260508 01:42:04.859 | INFO | NumEvents : 929209348166 | 20260508 01:42:06.336 | INFO | Analyzing chunk frames = [4243456,4243968) | 20260508 01:42:06.336 | INFO | MemInfo: { | 20260508 01:42:06.336 | INFO | "gpu" : " 17 GB", | 20260508 01:42:06.336 | INFO | "huge" : "", | 20260508 01:42:06.336 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:06.336 | INFO | "malloc" : " 14 GB", | 20260508 01:42:06.336 | INFO | "pinned" : " 29 GB" | 20260508 01:42:06.336 | INFO | } | 20260508 01:42:06.337 | INFO | Perf report for the last 5.13894 seconds: | 20260508 01:42:06.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:06.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:06.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:06.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:06.337 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260508 01:42:06.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:06.337 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:42:06.337 | INFO | PRE_HQ : 65.5%, 1.6, 55.9ms, 0 | 20260508 01:42:06.337 | INFO | BAZWRITER : 68.9%, 4.1, 144.0ms, 0 | 20260508 01:42:06.337 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:42:06.337 | INFO | Uploaded 36GiB to GPU in 2.59441s (13.876GiB/s) | 20260508 01:42:06.337 | INFO | Downloaded 38GiB to GPU in 2.69949s (14.0767GiB/s) | 20260508 01:42:11.478 | INFO | Analyzing chunk frames = [4243968,4244480) | 20260508 01:42:11.478 | INFO | MemInfo: { | 20260508 01:42:11.478 | INFO | "gpu" : " 17 GB", | 20260508 01:42:11.478 | INFO | "huge" : "", | 20260508 01:42:11.478 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:11.478 | INFO | "malloc" : " 14 GB", | 20260508 01:42:11.478 | INFO | "pinned" : " 29 GB" | 20260508 01:42:11.478 | INFO | } | 20260508 01:42:11.478 | INFO | Perf report for the last 5.14149 seconds: | 20260508 01:42:11.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:11.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 01:42:11.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:42:11.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:11.478 | INFO | Uploaded 36GiB to GPU in 2.58829s (13.9088GiB/s) | 20260508 01:42:11.478 | INFO | Downloaded 34GiB to GPU in 2.51117s (13.5395GiB/s) | 20260508 01:42:16.616 | INFO | Analyzing chunk frames = [4244480,4244992) | 20260508 01:42:16.616 | INFO | MemInfo: { | 20260508 01:42:16.616 | INFO | "gpu" : " 17 GB", | 20260508 01:42:16.616 | INFO | "huge" : "", | 20260508 01:42:16.616 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:16.616 | INFO | "malloc" : " 14 GB", | 20260508 01:42:16.616 | INFO | "pinned" : " 29 GB" | 20260508 01:42:16.616 | INFO | } | 20260508 01:42:16.616 | INFO | Perf report for the last 5.13831 seconds: | 20260508 01:42:16.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:16.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.616 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:42:16.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.616 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:42:16.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:16.617 | INFO | Uploaded 36GiB to GPU in 2.50611s (14.3649GiB/s) | 20260508 01:42:16.617 | INFO | Downloaded 34GiB to GPU in 2.49865s (13.6073GiB/s) | 20260508 01:42:21.758 | INFO | Analyzing chunk frames = [4244992,4245504) | 20260508 01:42:21.758 | INFO | MemInfo: { | 20260508 01:42:21.758 | INFO | "gpu" : " 17 GB", | 20260508 01:42:21.758 | INFO | "huge" : "", | 20260508 01:42:21.758 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:21.758 | INFO | "malloc" : " 14 GB", | 20260508 01:42:21.758 | INFO | "pinned" : " 29 GB" | 20260508 01:42:21.758 | INFO | } | 20260508 01:42:21.758 | INFO | Perf report for the last 5.14184 seconds: | 20260508 01:42:21.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:21.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:42:21.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:42:21.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:21.758 | INFO | Uploaded 36GiB to GPU in 2.50413s (14.3763GiB/s) | 20260508 01:42:21.759 | INFO | Downloaded 34GiB to GPU in 2.48626s (13.6751GiB/s) | 20260508 01:42:26.897 | INFO | Analyzing chunk frames = [4245504,4246016) | 20260508 01:42:26.897 | INFO | MemInfo: { | 20260508 01:42:26.897 | INFO | "gpu" : " 17 GB", | 20260508 01:42:26.897 | INFO | "huge" : "", | 20260508 01:42:26.897 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:26.897 | INFO | "malloc" : " 14 GB", | 20260508 01:42:26.897 | INFO | "pinned" : " 29 GB" | 20260508 01:42:26.897 | INFO | } | 20260508 01:42:26.897 | INFO | Perf report for the last 5.13899 seconds: | 20260508 01:42:26.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:26.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.897 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 01:42:26.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.897 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 01:42:26.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:26.898 | INFO | Uploaded 36GiB to GPU in 2.41183s (14.9264GiB/s) | 20260508 01:42:26.898 | INFO | Downloaded 34GiB to GPU in 2.58062s (13.1751GiB/s) | 20260508 01:42:32.032 | INFO | Analyzing chunk frames = [4246016,4246528) | 20260508 01:42:32.032 | INFO | MemInfo: { | 20260508 01:42:32.032 | INFO | "gpu" : " 17 GB", | 20260508 01:42:32.032 | INFO | "huge" : "", | 20260508 01:42:32.032 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:32.033 | INFO | "malloc" : " 14 GB", | 20260508 01:42:32.033 | INFO | "pinned" : " 29 GB" | 20260508 01:42:32.033 | INFO | } | 20260508 01:42:32.033 | INFO | Perf report for the last 5.13555 seconds: | 20260508 01:42:32.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:32.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:42:32.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260508 01:42:32.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:32.033 | INFO | Uploaded 36GiB to GPU in 2.4011s (14.9931GiB/s) | 20260508 01:42:32.033 | INFO | Downloaded 34GiB to GPU in 2.4235s (14.0293GiB/s) | 20260508 01:42:37.177 | INFO | Analyzing chunk frames = [4246528,4247040) | 20260508 01:42:37.177 | INFO | MemInfo: { | 20260508 01:42:37.177 | INFO | "gpu" : " 17 GB", | 20260508 01:42:37.177 | INFO | "huge" : "", | 20260508 01:42:37.177 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:37.177 | INFO | "malloc" : " 14 GB", | 20260508 01:42:37.177 | INFO | "pinned" : " 29 GB" | 20260508 01:42:37.177 | INFO | } | 20260508 01:42:37.177 | INFO | Perf report for the last 5.14434 seconds: | 20260508 01:42:37.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:37.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:42:37.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 01:42:37.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:37.177 | INFO | Uploaded 36GiB to GPU in 2.54412s (14.1502GiB/s) | 20260508 01:42:37.177 | INFO | Downloaded 34GiB to GPU in 2.47108s (13.7592GiB/s) | 20260508 01:42:42.314 | INFO | Analyzing chunk frames = [4247040,4247552) | 20260508 01:42:42.314 | INFO | MemInfo: { | 20260508 01:42:42.314 | INFO | "gpu" : " 17 GB", | 20260508 01:42:42.314 | INFO | "huge" : "", | 20260508 01:42:42.314 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:42.314 | INFO | "malloc" : " 14 GB", | 20260508 01:42:42.314 | INFO | "pinned" : " 29 GB" | 20260508 01:42:42.315 | INFO | } | 20260508 01:42:42.315 | INFO | Perf report for the last 5.13761 seconds: | 20260508 01:42:42.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:42.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:42:42.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:42:42.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:42.315 | INFO | Uploaded 36GiB to GPU in 2.54795s (14.129GiB/s) | 20260508 01:42:42.315 | INFO | Downloaded 34GiB to GPU in 2.46141s (13.8132GiB/s) | 20260508 01:42:42.412 | INFO | Starting a round of Baz IO | 20260508 01:42:45.860 | INFO | Populating RTMetrics report | 20260508 01:42:45.861 | INFO | RTMetrics report populated | 20260508 01:42:45.861 | INFO | Opening tmp RTMetrics json file | 20260508 01:42:45.882 | INFO | Serializing RTMetrics to json | 20260508 01:42:45.888 | INFO | Serialization done, swapping tmp file now... | 20260508 01:42:45.889 | INFO | RTMetrics done | 20260508 01:42:45.958 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.546 seconds | 20260508 01:42:45.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.75% / 0.00% / 58.24% | 20260508 01:42:45.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.814148% / 7.94729e-06% / 3.97364e-05% | 20260508 01:42:45.958 | INFO | Baz IO overall: | 20260508 01:42:45.958 | INFO | BytesWritten : 1923008076782 | 20260508 01:42:45.958 | INFO | NumEvents : 930062203776 | 20260508 01:42:47.455 | INFO | Analyzing chunk frames = [4247552,4248064) | 20260508 01:42:47.455 | INFO | MemInfo: { | 20260508 01:42:47.455 | INFO | "gpu" : " 17 GB", | 20260508 01:42:47.455 | INFO | "huge" : "", | 20260508 01:42:47.455 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:47.455 | INFO | "malloc" : " 14 GB", | 20260508 01:42:47.455 | INFO | "pinned" : " 29 GB" | 20260508 01:42:47.455 | INFO | } | 20260508 01:42:47.455 | INFO | Perf report for the last 5.14039 seconds: | 20260508 01:42:47.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:47.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:47.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:47.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:47.455 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260508 01:42:47.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:47.455 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:42:47.455 | INFO | PRE_HQ : 65.2%, 1.7, 56.2ms, 0 | 20260508 01:42:47.455 | INFO | BAZWRITER : 68.6%, 3.7, 132.1ms, 0 | 20260508 01:42:47.455 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 01:42:47.456 | INFO | Uploaded 36GiB to GPU in 2.49798s (14.4116GiB/s) | 20260508 01:42:47.456 | INFO | Downloaded 38GiB to GPU in 2.67627s (14.1989GiB/s) | 20260508 01:42:52.586 | INFO | Analyzing chunk frames = [4248064,4248576) | 20260508 01:42:52.586 | INFO | MemInfo: { | 20260508 01:42:52.586 | INFO | "gpu" : " 17 GB", | 20260508 01:42:52.586 | INFO | "huge" : "", | 20260508 01:42:52.586 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:52.586 | INFO | "malloc" : " 14 GB", | 20260508 01:42:52.586 | INFO | "pinned" : " 29 GB" | 20260508 01:42:52.587 | INFO | } | 20260508 01:42:52.587 | INFO | Perf report for the last 5.13151 seconds: | 20260508 01:42:52.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:52.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260508 01:42:52.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:42:52.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:52.587 | INFO | Uploaded 36GiB to GPU in 2.52391s (14.2636GiB/s) | 20260508 01:42:52.587 | INFO | Downloaded 34GiB to GPU in 2.43383s (13.9697GiB/s) | 20260508 01:42:57.727 | INFO | Analyzing chunk frames = [4248576,4249088) | 20260508 01:42:57.728 | INFO | MemInfo: { | 20260508 01:42:57.728 | INFO | "gpu" : " 17 GB", | 20260508 01:42:57.728 | INFO | "huge" : "", | 20260508 01:42:57.728 | INFO | "huge_shared" : " 29 GB", | 20260508 01:42:57.728 | INFO | "malloc" : " 14 GB", | 20260508 01:42:57.728 | INFO | "pinned" : " 29 GB" | 20260508 01:42:57.728 | INFO | } | 20260508 01:42:57.728 | INFO | Perf report for the last 5.14093 seconds: | 20260508 01:42:57.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:42:57.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:42:57.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:42:57.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:42:57.728 | INFO | Uploaded 36GiB to GPU in 2.52572s (14.2534GiB/s) | 20260508 01:42:57.728 | INFO | Downloaded 34GiB to GPU in 2.48641s (13.6743GiB/s) | 20260508 01:43:02.869 | INFO | Analyzing chunk frames = [4249088,4249600) | 20260508 01:43:02.869 | INFO | MemInfo: { | 20260508 01:43:02.869 | INFO | "gpu" : " 17 GB", | 20260508 01:43:02.869 | INFO | "huge" : "", | 20260508 01:43:02.869 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:02.869 | INFO | "malloc" : " 14 GB", | 20260508 01:43:02.869 | INFO | "pinned" : " 29 GB" | 20260508 01:43:02.869 | INFO | } | 20260508 01:43:02.869 | INFO | Perf report for the last 5.1416 seconds: | 20260508 01:43:02.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:02.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:43:02.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:43:02.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:02.869 | INFO | Uploaded 36GiB to GPU in 2.52661s (14.2484GiB/s) | 20260508 01:43:02.869 | INFO | Downloaded 34GiB to GPU in 2.48186s (13.6994GiB/s) | 20260508 01:43:08.014 | INFO | Analyzing chunk frames = [4249600,4250112) | 20260508 01:43:08.014 | INFO | MemInfo: { | 20260508 01:43:08.014 | INFO | "gpu" : " 17 GB", | 20260508 01:43:08.014 | INFO | "huge" : "", | 20260508 01:43:08.014 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:08.014 | INFO | "malloc" : " 14 GB", | 20260508 01:43:08.014 | INFO | "pinned" : " 29 GB" | 20260508 01:43:08.014 | INFO | } | 20260508 01:43:08.014 | INFO | Perf report for the last 5.14482 seconds: | 20260508 01:43:08.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:08.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.014 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 01:43:08.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.014 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 01:43:08.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:08.015 | INFO | Uploaded 36GiB to GPU in 2.29129s (15.7117GiB/s) | 20260508 01:43:08.015 | INFO | Downloaded 34GiB to GPU in 2.49465s (13.6291GiB/s) | 20260508 01:43:13.153 | INFO | Analyzing chunk frames = [4250112,4250624) | 20260508 01:43:13.153 | INFO | MemInfo: { | 20260508 01:43:13.153 | INFO | "gpu" : " 17 GB", | 20260508 01:43:13.153 | INFO | "huge" : "", | 20260508 01:43:13.153 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:13.153 | INFO | "malloc" : " 14 GB", | 20260508 01:43:13.153 | INFO | "pinned" : " 29 GB" | 20260508 01:43:13.153 | INFO | } | 20260508 01:43:13.153 | INFO | Perf report for the last 5.13911 seconds: | 20260508 01:43:13.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:13.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.153 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:43:13.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.153 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:43:13.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:13.154 | INFO | Uploaded 36GiB to GPU in 2.35337s (15.2972GiB/s) | 20260508 01:43:13.154 | INFO | Downloaded 34GiB to GPU in 2.4229s (14.0328GiB/s) | 20260508 01:43:18.287 | INFO | Analyzing chunk frames = [4250624,4251136) | 20260508 01:43:18.287 | INFO | MemInfo: { | 20260508 01:43:18.287 | INFO | "gpu" : " 17 GB", | 20260508 01:43:18.287 | INFO | "huge" : "", | 20260508 01:43:18.287 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:18.287 | INFO | "malloc" : " 14 GB", | 20260508 01:43:18.287 | INFO | "pinned" : " 29 GB" | 20260508 01:43:18.287 | INFO | } | 20260508 01:43:18.287 | INFO | Perf report for the last 5.13409 seconds: | 20260508 01:43:18.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:18.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.287 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 01:43:18.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.287 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:43:18.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:18.288 | INFO | Uploaded 36GiB to GPU in 2.42865s (14.823GiB/s) | 20260508 01:43:18.288 | INFO | Downloaded 34GiB to GPU in 2.44816s (13.888GiB/s) | 20260508 01:43:23.426 | INFO | Analyzing chunk frames = [4251136,4251648) | 20260508 01:43:23.426 | INFO | MemInfo: { | 20260508 01:43:23.426 | INFO | "gpu" : " 17 GB", | 20260508 01:43:23.426 | INFO | "huge" : "", | 20260508 01:43:23.426 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:23.426 | INFO | "malloc" : " 14 GB", | 20260508 01:43:23.426 | INFO | "pinned" : " 29 GB" | 20260508 01:43:23.426 | INFO | } | 20260508 01:43:23.426 | INFO | Perf report for the last 5.13885 seconds: | 20260508 01:43:23.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:23.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.426 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 01:43:23.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.426 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:43:23.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:23.427 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:43:23.427 | INFO | Total mem usage: 16.7 GiB | 20260508 01:43:23.427 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:43:23.427 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.37%) | 20260508 01:43:23.427 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:43:23.427 | INFO | High Water Breakdown: | 20260508 01:43:23.427 | INFO | 60.5% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:43:23.427 | INFO | 39.4% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:43:23.427 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:43:23.427 | INFO | ------------------------------------------ | 20260508 01:43:23.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:43:23.427 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:43:23.427 | INFO | Total mem usage: 26.5 GiB | 20260508 01:43:23.427 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:43:23.427 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:43:23.427 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:43:23.427 | INFO | High Water Breakdown: | 20260508 01:43:23.427 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:43:23.427 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:43:23.427 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:43:23.427 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:43:23.427 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:43:23.427 | INFO | Uploaded 36GiB to GPU in 2.50867s (14.3502GiB/s) | 20260508 01:43:23.427 | INFO | Downloaded 34GiB to GPU in 2.46578s (13.7887GiB/s) | 20260508 01:43:23.427 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:43:23.427 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:43:23.427 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:43:23.427 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:43:23.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:43:23.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:43:23.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:43:23.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:43:23.427 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:43:23.427 | INFO | ------------------------------------------ | 20260508 01:43:23.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:43:23.427 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:43:23.427 | INFO | Total mem usage: 27.8 GiB | 20260508 01:43:23.427 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:43:23.427 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:43:23.427 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:43:23.427 | INFO | High Water Breakdown: | 20260508 01:43:23.427 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:43:23.427 | INFO | ------------------------------------------ | 20260508 01:43:23.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:43:23.427 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:43:23.427 | INFO | Total mem usage: 15.7 GiB | 20260508 01:43:23.427 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:43:23.428 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:43:23.428 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:43:23.428 | INFO | High Water Breakdown: | 20260508 01:43:23.428 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:43:23.428 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:43:23.428 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:43:23.428 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:43:23.428 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:43:23.428 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:43:23.428 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:43:23.428 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:43:23.428 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:43:23.428 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:43:23.428 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:43:23.428 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:43:23.428 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:43:23.428 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:43:23.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:43:23.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:43:23.428 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:43:23.428 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:43:23.428 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:43:23.428 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:43:23.428 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:43:23.428 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:43:23.428 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:43:23.428 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:43:23.428 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:43:23.428 | INFO | ------------------------------------------ | 20260508 01:43:23.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:43:23.428 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:43:23.428 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:43:23.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:43:23.428 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.64e+01, 1.42e+01 / 1.40e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.16e-02 / 7.95e-03 | 20260508 01:43:23.428 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.44e+00, 5.87e-01 / 1.85e-01 | 20260508 01:43:23.428 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.16e-01 | 20260508 01:43:23.428 | INFO | PerfReport: 16.14%, DME -- 4.22e+01 / 4.65e+01, 4.20e+00 / 1.94e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.56e+00 / 1.55e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 2.71e+00, 4.28e-01 / 1.96e-01 | 20260508 01:43:23.428 | INFO | PerfReport: 3.32%, Metrics -- 1.09e+00 / 1.31e+00, 2.69e-01 / 1.42e-01 | 20260508 01:43:23.428 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.66e-01, 5.28e-02 / 2.05e-02 | 20260508 01:43:23.428 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:43:23.428 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:43:23.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:43:23.428 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 24.27%, Upload -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 24.45%, Download -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 30.64%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 2.11%, LockUpload -- 2.24e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 2.95%, LockDownload -- 3.14e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:43:23.428 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.53e-02 / 0.00e+00 | 20260508 01:43:23.524 | INFO | Starting a round of Baz IO | 20260508 01:43:26.975 | INFO | Populating RTMetrics report | 20260508 01:43:26.975 | INFO | RTMetrics report populated | 20260508 01:43:26.975 | INFO | Opening tmp RTMetrics json file | 20260508 01:43:27.041 | INFO | Serializing RTMetrics to json | 20260508 01:43:27.066 | INFO | Serialization done, swapping tmp file now... | 20260508 01:43:27.071 | INFO | RTMetrics done | 20260508 01:43:27.088 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.564 seconds | 20260508 01:43:27.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.75% / 0.00% / 58.25% | 20260508 01:43:27.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.813293% / 7.94729e-06% / 7.94729e-05% | 20260508 01:43:27.088 | INFO | Baz IO overall: | 20260508 01:43:27.088 | INFO | BytesWritten : 1924784875443 | 20260508 01:43:27.088 | INFO | NumEvents : 930914910263 | 20260508 01:43:28.566 | INFO | Analyzing chunk frames = [4251648,4252160) | 20260508 01:43:28.566 | INFO | MemInfo: { | 20260508 01:43:28.566 | INFO | "gpu" : " 17 GB", | 20260508 01:43:28.566 | INFO | "huge" : "", | 20260508 01:43:28.567 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:28.567 | INFO | "malloc" : " 14 GB", | 20260508 01:43:28.567 | INFO | "pinned" : " 29 GB" | 20260508 01:43:28.567 | INFO | } | 20260508 01:43:28.567 | INFO | Perf report for the last 5.14055 seconds: | 20260508 01:43:28.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:28.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:28.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:28.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:28.567 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 01:43:28.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:28.567 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 01:43:28.567 | INFO | PRE_HQ : 66.0%, 1.6, 56.4ms, 0 | 20260508 01:43:28.567 | INFO | BAZWRITER : 68.9%, 4.0, 142.0ms, 0 | 20260508 01:43:28.567 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 01:43:28.568 | INFO | Uploaded 36GiB to GPU in 2.54608s (14.1394GiB/s) | 20260508 01:43:28.568 | INFO | Downloaded 38GiB to GPU in 2.69634s (14.0932GiB/s) | 20260508 01:43:33.707 | INFO | Analyzing chunk frames = [4252160,4252672) | 20260508 01:43:33.707 | INFO | MemInfo: { | 20260508 01:43:33.707 | INFO | "gpu" : " 17 GB", | 20260508 01:43:33.707 | INFO | "huge" : "", | 20260508 01:43:33.707 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:33.707 | INFO | "malloc" : " 14 GB", | 20260508 01:43:33.707 | INFO | "pinned" : " 29 GB" | 20260508 01:43:33.707 | INFO | } | 20260508 01:43:33.707 | INFO | Perf report for the last 5.14072 seconds: | 20260508 01:43:33.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:33.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.707 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:43:33.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.708 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:43:33.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:33.708 | INFO | Uploaded 36GiB to GPU in 2.51074s (14.3384GiB/s) | 20260508 01:43:33.708 | INFO | Downloaded 34GiB to GPU in 2.44553s (13.9029GiB/s) | 20260508 01:43:38.847 | INFO | Analyzing chunk frames = [4252672,4253184) | 20260508 01:43:38.847 | INFO | MemInfo: { | 20260508 01:43:38.847 | INFO | "gpu" : " 17 GB", | 20260508 01:43:38.847 | INFO | "huge" : "", | 20260508 01:43:38.847 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:38.847 | INFO | "malloc" : " 14 GB", | 20260508 01:43:38.847 | INFO | "pinned" : " 29 GB" | 20260508 01:43:38.847 | INFO | } | 20260508 01:43:38.847 | INFO | Perf report for the last 5.1398 seconds: | 20260508 01:43:38.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:38.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.847 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:43:38.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.847 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:43:38.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:38.848 | INFO | Uploaded 36GiB to GPU in 2.50283s (14.3837GiB/s) | 20260508 01:43:38.848 | INFO | Downloaded 34GiB to GPU in 2.513s (13.5296GiB/s) | 20260508 01:43:43.986 | INFO | Analyzing chunk frames = [4253184,4253696) | 20260508 01:43:43.986 | INFO | MemInfo: { | 20260508 01:43:43.986 | INFO | "gpu" : " 17 GB", | 20260508 01:43:43.986 | INFO | "huge" : "", | 20260508 01:43:43.986 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:43.986 | INFO | "malloc" : " 14 GB", | 20260508 01:43:43.986 | INFO | "pinned" : " 29 GB" | 20260508 01:43:43.986 | INFO | } | 20260508 01:43:43.987 | INFO | Perf report for the last 5.13928 seconds: | 20260508 01:43:43.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:43.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:43:43.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:43:43.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:43.987 | INFO | Uploaded 36GiB to GPU in 2.4739s (14.5519GiB/s) | 20260508 01:43:43.987 | INFO | Downloaded 34GiB to GPU in 2.50594s (13.5678GiB/s) | 20260508 01:43:49.126 | INFO | Analyzing chunk frames = [4253696,4254208) | 20260508 01:43:49.126 | INFO | MemInfo: { | 20260508 01:43:49.126 | INFO | "gpu" : " 17 GB", | 20260508 01:43:49.126 | INFO | "huge" : "", | 20260508 01:43:49.126 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:49.126 | INFO | "malloc" : " 14 GB", | 20260508 01:43:49.126 | INFO | "pinned" : " 29 GB" | 20260508 01:43:49.126 | INFO | } | 20260508 01:43:49.126 | INFO | Perf report for the last 5.13972 seconds: | 20260508 01:43:49.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:49.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.126 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 01:43:49.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.126 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:43:49.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:49.127 | INFO | Uploaded 36GiB to GPU in 2.30845s (15.5949GiB/s) | 20260508 01:43:49.127 | INFO | Downloaded 34GiB to GPU in 2.50091s (13.5951GiB/s) | 20260508 01:43:54.264 | INFO | Analyzing chunk frames = [4254208,4254720) | 20260508 01:43:54.264 | INFO | MemInfo: { | 20260508 01:43:54.264 | INFO | "gpu" : " 17 GB", | 20260508 01:43:54.264 | INFO | "huge" : "", | 20260508 01:43:54.264 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:54.264 | INFO | "malloc" : " 14 GB", | 20260508 01:43:54.264 | INFO | "pinned" : " 29 GB" | 20260508 01:43:54.264 | INFO | } | 20260508 01:43:54.264 | INFO | Perf report for the last 5.1378 seconds: | 20260508 01:43:54.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:54.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:43:54.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:43:54.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:54.264 | INFO | Uploaded 36GiB to GPU in 2.39881s (15.0074GiB/s) | 20260508 01:43:54.264 | INFO | Downloaded 34GiB to GPU in 2.40687s (14.1263GiB/s) | 20260508 01:43:59.402 | INFO | Analyzing chunk frames = [4254720,4255232) | 20260508 01:43:59.402 | INFO | MemInfo: { | 20260508 01:43:59.402 | INFO | "gpu" : " 17 GB", | 20260508 01:43:59.403 | INFO | "huge" : "", | 20260508 01:43:59.403 | INFO | "huge_shared" : " 29 GB", | 20260508 01:43:59.403 | INFO | "malloc" : " 14 GB", | 20260508 01:43:59.403 | INFO | "pinned" : " 29 GB" | 20260508 01:43:59.403 | INFO | } | 20260508 01:43:59.403 | INFO | Perf report for the last 5.13868 seconds: | 20260508 01:43:59.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:43:59.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:43:59.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:43:59.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:43:59.403 | INFO | Uploaded 36GiB to GPU in 2.43182s (14.8037GiB/s) | 20260508 01:43:59.403 | INFO | Downloaded 34GiB to GPU in 2.42904s (13.9973GiB/s) | 20260508 01:44:04.541 | INFO | Analyzing chunk frames = [4255232,4255744) | 20260508 01:44:04.541 | INFO | MemInfo: { | 20260508 01:44:04.541 | INFO | "gpu" : " 17 GB", | 20260508 01:44:04.541 | INFO | "huge" : "", | 20260508 01:44:04.541 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:04.541 | INFO | "malloc" : " 14 GB", | 20260508 01:44:04.541 | INFO | "pinned" : " 29 GB" | 20260508 01:44:04.541 | INFO | } | 20260508 01:44:04.541 | INFO | Perf report for the last 5.13865 seconds: | 20260508 01:44:04.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:04.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.542 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:44:04.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.542 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:44:04.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:04.542 | INFO | Uploaded 36GiB to GPU in 2.55602s (14.0844GiB/s) | 20260508 01:44:04.542 | INFO | Downloaded 34GiB to GPU in 2.46045s (13.8186GiB/s) | 20260508 01:44:04.638 | INFO | Starting a round of Baz IO | 20260508 01:44:08.127 | INFO | Populating RTMetrics report | 20260508 01:44:08.127 | INFO | RTMetrics report populated | 20260508 01:44:08.127 | INFO | Opening tmp RTMetrics json file | 20260508 01:44:08.128 | INFO | Serializing RTMetrics to json | 20260508 01:44:08.130 | INFO | Serialization done, swapping tmp file now... | 20260508 01:44:08.130 | INFO | RTMetrics done | 20260508 01:44:08.221 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.582 seconds | 20260508 01:44:08.221 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.75% / 0.00% / 58.25% | 20260508 01:44:08.221 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.813508% / 7.94729e-06% / 7.15256e-05% | 20260508 01:44:08.221 | INFO | Baz IO overall: | 20260508 01:44:08.221 | INFO | BytesWritten : 1926561487446 | 20260508 01:44:08.221 | INFO | NumEvents : 931767552183 | 20260508 01:44:09.679 | INFO | Analyzing chunk frames = [4255744,4256256) | 20260508 01:44:09.679 | INFO | MemInfo: { | 20260508 01:44:09.679 | INFO | "gpu" : " 17 GB", | 20260508 01:44:09.679 | INFO | "huge" : "", | 20260508 01:44:09.679 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:09.679 | INFO | "malloc" : " 14 GB", | 20260508 01:44:09.679 | INFO | "pinned" : " 29 GB" | 20260508 01:44:09.679 | INFO | } | 20260508 01:44:09.679 | INFO | Perf report for the last 5.1379 seconds: | 20260508 01:44:09.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:09.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:09.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:09.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:09.680 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260508 01:44:09.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:09.680 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260508 01:44:09.680 | INFO | PRE_HQ : 66.5%, 1.7, 58.2ms, 0 | 20260508 01:44:09.680 | INFO | BAZWRITER : 69.3%, 3.7, 132.2ms, 0 | 20260508 01:44:09.680 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 01:44:09.680 | INFO | Uploaded 36GiB to GPU in 2.57632s (13.9734GiB/s) | 20260508 01:44:09.680 | INFO | Downloaded 38GiB to GPU in 2.69908s (14.0788GiB/s) | 20260508 01:44:14.822 | INFO | Analyzing chunk frames = [4256256,4256768) | 20260508 01:44:14.822 | INFO | MemInfo: { | 20260508 01:44:14.822 | INFO | "gpu" : " 17 GB", | 20260508 01:44:14.822 | INFO | "huge" : "", | 20260508 01:44:14.822 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:14.822 | INFO | "malloc" : " 14 GB", | 20260508 01:44:14.822 | INFO | "pinned" : " 29 GB" | 20260508 01:44:14.822 | INFO | } | 20260508 01:44:14.822 | INFO | Perf report for the last 5.14237 seconds: | 20260508 01:44:14.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:14.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:44:14.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 01:44:14.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:14.822 | INFO | Uploaded 36GiB to GPU in 2.54928s (14.1217GiB/s) | 20260508 01:44:14.822 | INFO | Downloaded 34GiB to GPU in 2.41174s (14.0977GiB/s) | 20260508 01:44:19.961 | INFO | Analyzing chunk frames = [4256768,4257280) | 20260508 01:44:19.962 | INFO | MemInfo: { | 20260508 01:44:19.962 | INFO | "gpu" : " 17 GB", | 20260508 01:44:19.962 | INFO | "huge" : "", | 20260508 01:44:19.962 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:19.962 | INFO | "malloc" : " 14 GB", | 20260508 01:44:19.962 | INFO | "pinned" : " 29 GB" | 20260508 01:44:19.962 | INFO | } | 20260508 01:44:19.962 | INFO | Perf report for the last 5.14002 seconds: | 20260508 01:44:19.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:19.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:44:19.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:44:19.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:19.962 | INFO | Uploaded 36GiB to GPU in 2.56072s (14.0586GiB/s) | 20260508 01:44:19.962 | INFO | Downloaded 34GiB to GPU in 2.48048s (13.707GiB/s) | 20260508 01:44:25.100 | INFO | Analyzing chunk frames = [4257280,4257792) | 20260508 01:44:25.100 | INFO | MemInfo: { | 20260508 01:44:25.100 | INFO | "gpu" : " 17 GB", | 20260508 01:44:25.100 | INFO | "huge" : "", | 20260508 01:44:25.100 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:25.100 | INFO | "malloc" : " 14 GB", | 20260508 01:44:25.100 | INFO | "pinned" : " 29 GB" | 20260508 01:44:25.100 | INFO | } | 20260508 01:44:25.100 | INFO | Perf report for the last 5.13837 seconds: | 20260508 01:44:25.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:25.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 01:44:25.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:44:25.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:25.100 | INFO | Uploaded 36GiB to GPU in 2.56332s (14.0443GiB/s) | 20260508 01:44:25.101 | INFO | Downloaded 34GiB to GPU in 2.47726s (13.7248GiB/s) | 20260508 01:44:30.243 | INFO | Analyzing chunk frames = [4257792,4258304) | 20260508 01:44:30.243 | INFO | MemInfo: { | 20260508 01:44:30.243 | INFO | "gpu" : " 17 GB", | 20260508 01:44:30.243 | INFO | "huge" : "", | 20260508 01:44:30.243 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:30.243 | INFO | "malloc" : " 14 GB", | 20260508 01:44:30.243 | INFO | "pinned" : " 29 GB" | 20260508 01:44:30.243 | INFO | } | 20260508 01:44:30.243 | INFO | Perf report for the last 5.14318 seconds: | 20260508 01:44:30.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:30.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.243 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 01:44:30.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.243 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 01:44:30.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:30.244 | INFO | Uploaded 36GiB to GPU in 2.42438s (14.8492GiB/s) | 20260508 01:44:30.244 | INFO | Downloaded 34GiB to GPU in 2.54276s (13.3713GiB/s) | 20260508 01:44:35.379 | INFO | Analyzing chunk frames = [4258304,4258816) | 20260508 01:44:35.379 | INFO | MemInfo: { | 20260508 01:44:35.379 | INFO | "gpu" : " 17 GB", | 20260508 01:44:35.379 | INFO | "huge" : "", | 20260508 01:44:35.379 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:35.379 | INFO | "malloc" : " 14 GB", | 20260508 01:44:35.379 | INFO | "pinned" : " 29 GB" | 20260508 01:44:35.379 | INFO | } | 20260508 01:44:35.379 | INFO | Perf report for the last 5.13603 seconds: | 20260508 01:44:35.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:35.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 01:44:35.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:44:35.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:35.380 | INFO | Uploaded 36GiB to GPU in 2.43565s (14.7804GiB/s) | 20260508 01:44:35.380 | INFO | Downloaded 34GiB to GPU in 2.4037s (14.1448GiB/s) | 20260508 01:44:40.518 | INFO | Analyzing chunk frames = [4258816,4259328) | 20260508 01:44:40.518 | INFO | MemInfo: { | 20260508 01:44:40.518 | INFO | "gpu" : " 17 GB", | 20260508 01:44:40.518 | INFO | "huge" : "", | 20260508 01:44:40.518 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:40.518 | INFO | "malloc" : " 14 GB", | 20260508 01:44:40.519 | INFO | "pinned" : " 29 GB" | 20260508 01:44:40.519 | INFO | } | 20260508 01:44:40.519 | INFO | Perf report for the last 5.13915 seconds: | 20260508 01:44:40.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:40.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:44:40.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:44:40.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:40.519 | INFO | Uploaded 36GiB to GPU in 2.45538s (14.6617GiB/s) | 20260508 01:44:40.519 | INFO | Downloaded 34GiB to GPU in 2.40857s (14.1162GiB/s) | 20260508 01:44:45.656 | INFO | Analyzing chunk frames = [4259328,4259840) | 20260508 01:44:45.656 | INFO | MemInfo: { | 20260508 01:44:45.656 | INFO | "gpu" : " 17 GB", | 20260508 01:44:45.656 | INFO | "huge" : "", | 20260508 01:44:45.656 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:45.656 | INFO | "malloc" : " 14 GB", | 20260508 01:44:45.656 | INFO | "pinned" : " 29 GB" | 20260508 01:44:45.656 | INFO | } | 20260508 01:44:45.656 | INFO | Perf report for the last 5.13779 seconds: | 20260508 01:44:45.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:45.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.656 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 01:44:45.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.656 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260508 01:44:45.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:45.657 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:44:45.657 | INFO | Total mem usage: 16.7 GiB | 20260508 01:44:45.657 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:44:45.657 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -10.1%) | 20260508 01:44:45.657 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:44:45.657 | INFO | High Water Breakdown: | 20260508 01:44:45.657 | INFO | 60.4% : 9.16 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:44:45.657 | INFO | 39.4% : 5.98 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:44:45.657 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:44:45.657 | INFO | ------------------------------------------ | 20260508 01:44:45.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:44:45.657 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:44:45.657 | INFO | Total mem usage: 26.5 GiB | 20260508 01:44:45.657 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:44:45.657 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:44:45.657 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:44:45.657 | INFO | High Water Breakdown: | 20260508 01:44:45.657 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:44:45.657 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:44:45.657 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:44:45.657 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:44:45.657 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:44:45.657 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:44:45.657 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:44:45.657 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:44:45.657 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:44:45.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:44:45.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:44:45.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:44:45.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:44:45.657 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:44:45.657 | INFO | ------------------------------------------ | 20260508 01:44:45.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:44:45.657 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:44:45.657 | INFO | Total mem usage: 27.8 GiB | 20260508 01:44:45.657 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:44:45.657 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:44:45.657 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:44:45.657 | INFO | High Water Breakdown: | 20260508 01:44:45.657 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:44:45.657 | INFO | ------------------------------------------ | 20260508 01:44:45.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:44:45.657 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:44:45.657 | INFO | Total mem usage: 15.7 GiB | 20260508 01:44:45.657 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:44:45.657 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:44:45.657 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:44:45.657 | INFO | High Water Breakdown: | 20260508 01:44:45.657 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:44:45.657 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:44:45.657 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:44:45.657 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:44:45.657 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:44:45.657 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:44:45.657 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:44:45.657 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:44:45.657 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:44:45.657 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:44:45.657 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:44:45.657 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:44:45.657 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:44:45.657 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:44:45.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:44:45.657 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:44:45.657 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:44:45.657 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:44:45.657 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:44:45.657 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:44:45.657 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:44:45.657 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:44:45.657 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:44:45.657 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:44:45.657 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:44:45.657 | INFO | ------------------------------------------ | 20260508 01:44:45.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:44:45.657 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:44:45.657 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:44:45.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:44:45.657 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.36e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.24e-02 / 1.97e-02 | 20260508 01:44:45.657 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.47e+00, 5.90e-01 / 1.71e-01 | 20260508 01:44:45.657 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.07e-01 / 1.05e-01 | 20260508 01:44:45.657 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.60e+01, 4.11e+00 / 1.91e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.63e+00 / 1.67e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.70e+00, 4.23e-01 / 2.23e-01 | 20260508 01:44:45.657 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.28e+00, 2.70e-01 / 1.58e-01 | 20260508 01:44:45.657 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.32e-01 / 3.70e-01, 5.22e-02 / 2.19e-02 | 20260508 01:44:45.657 | INFO | Uploaded 36GiB to GPU in 2.5765s (13.9724GiB/s) | 20260508 01:44:45.657 | INFO | Downloaded 34GiB to GPU in 2.45836s (13.8304GiB/s) | 20260508 01:44:45.657 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:44:45.657 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:44:45.657 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:44:45.657 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 24.36%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 2.28%, LockUpload -- 2.43e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 2.70%, LockDownload -- 2.88e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:44:45.657 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260508 01:44:45.754 | INFO | Starting a round of Baz IO | 20260508 01:44:49.202 | INFO | Populating RTMetrics report | 20260508 01:44:49.203 | INFO | RTMetrics report populated | 20260508 01:44:49.203 | INFO | Opening tmp RTMetrics json file | 20260508 01:44:49.229 | INFO | Serializing RTMetrics to json | 20260508 01:44:49.231 | INFO | Serialization done, swapping tmp file now... | 20260508 01:44:49.231 | INFO | RTMetrics done | 20260508 01:44:49.312 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.558 seconds | 20260508 01:44:49.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.74% / 0.00% / 58.25% | 20260508 01:44:49.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.810341% / 1.19209e-05% / 7.54992e-05% | 20260508 01:44:49.312 | INFO | Baz IO overall: | 20260508 01:44:49.312 | INFO | BytesWritten : 1928337615589 | 20260508 01:44:49.312 | INFO | NumEvents : 932619667825 | 20260508 01:44:50.796 | INFO | Analyzing chunk frames = [4259840,4260352) | 20260508 01:44:50.796 | INFO | MemInfo: { | 20260508 01:44:50.796 | INFO | "gpu" : " 17 GB", | 20260508 01:44:50.796 | INFO | "huge" : "", | 20260508 01:44:50.796 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:50.796 | INFO | "malloc" : " 14 GB", | 20260508 01:44:50.796 | INFO | "pinned" : " 29 GB" | 20260508 01:44:50.796 | INFO | } | 20260508 01:44:50.796 | INFO | Perf report for the last 5.13947 seconds: | 20260508 01:44:50.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:50.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:50.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:50.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:50.796 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260508 01:44:50.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:50.796 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:44:50.796 | INFO | PRE_HQ : 65.7%, 1.6, 56.0ms, 0 | 20260508 01:44:50.796 | INFO | BAZWRITER : 68.7%, 3.7, 132.0ms, 0 | 20260508 01:44:50.796 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 01:44:50.796 | INFO | Uploaded 36GiB to GPU in 2.58219s (13.9417GiB/s) | 20260508 01:44:50.796 | INFO | Downloaded 38GiB to GPU in 2.69507s (14.0998GiB/s) | 20260508 01:44:55.940 | INFO | Analyzing chunk frames = [4260352,4260864) | 20260508 01:44:55.940 | INFO | MemInfo: { | 20260508 01:44:55.940 | INFO | "gpu" : " 17 GB", | 20260508 01:44:55.940 | INFO | "huge" : "", | 20260508 01:44:55.940 | INFO | "huge_shared" : " 29 GB", | 20260508 01:44:55.940 | INFO | "malloc" : " 14 GB", | 20260508 01:44:55.940 | INFO | "pinned" : " 29 GB" | 20260508 01:44:55.940 | INFO | } | 20260508 01:44:55.940 | INFO | Perf report for the last 5.14431 seconds: | 20260508 01:44:55.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:44:55.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.940 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 01:44:55.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.940 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 01:44:55.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:44:55.941 | INFO | Uploaded 36GiB to GPU in 2.52958s (14.2316GiB/s) | 20260508 01:44:55.941 | INFO | Downloaded 34GiB to GPU in 2.43285s (13.9754GiB/s) | 20260508 01:45:01.078 | INFO | Analyzing chunk frames = [4260864,4261376) | 20260508 01:45:01.078 | INFO | MemInfo: { | 20260508 01:45:01.078 | INFO | "gpu" : " 17 GB", | 20260508 01:45:01.078 | INFO | "huge" : "", | 20260508 01:45:01.078 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:01.078 | INFO | "malloc" : " 14 GB", | 20260508 01:45:01.078 | INFO | "pinned" : " 29 GB" | 20260508 01:45:01.078 | INFO | } | 20260508 01:45:01.078 | INFO | Perf report for the last 5.13766 seconds: | 20260508 01:45:01.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:01.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 01:45:01.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:45:01.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:01.078 | INFO | Uploaded 36GiB to GPU in 2.54583s (14.1408GiB/s) | 20260508 01:45:01.078 | INFO | Downloaded 34GiB to GPU in 2.50558s (13.5697GiB/s) | 20260508 01:45:06.215 | INFO | Analyzing chunk frames = [4261376,4261888) | 20260508 01:45:06.215 | INFO | MemInfo: { | 20260508 01:45:06.215 | INFO | "gpu" : " 17 GB", | 20260508 01:45:06.215 | INFO | "huge" : "", | 20260508 01:45:06.215 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:06.215 | INFO | "malloc" : " 14 GB", | 20260508 01:45:06.215 | INFO | "pinned" : " 29 GB" | 20260508 01:45:06.215 | INFO | } | 20260508 01:45:06.215 | INFO | Perf report for the last 5.1375 seconds: | 20260508 01:45:06.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:06.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.215 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 01:45:06.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.216 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:45:06.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:06.216 | INFO | Uploaded 36GiB to GPU in 2.52686s (14.2469GiB/s) | 20260508 01:45:06.216 | INFO | Downloaded 34GiB to GPU in 2.49915s (13.6046GiB/s) | 20260508 01:45:11.348 | INFO | Analyzing chunk frames = [4261888,4262400) | 20260508 01:45:11.348 | INFO | MemInfo: { | 20260508 01:45:11.348 | INFO | "gpu" : " 17 GB", | 20260508 01:45:11.348 | INFO | "huge" : "", | 20260508 01:45:11.348 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:11.348 | INFO | "malloc" : " 14 GB", | 20260508 01:45:11.348 | INFO | "pinned" : " 29 GB" | 20260508 01:45:11.348 | INFO | } | 20260508 01:45:11.348 | INFO | Perf report for the last 5.13258 seconds: | 20260508 01:45:11.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:11.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.348 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 01:45:11.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.348 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 01:45:11.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:11.349 | INFO | Uploaded 36GiB to GPU in 2.33819s (15.3965GiB/s) | 20260508 01:45:11.349 | INFO | Downloaded 34GiB to GPU in 2.51544s (13.5165GiB/s) | 20260508 01:45:16.490 | INFO | Analyzing chunk frames = [4262400,4262912) | 20260508 01:45:16.490 | INFO | MemInfo: { | 20260508 01:45:16.490 | INFO | "gpu" : " 17 GB", | 20260508 01:45:16.490 | INFO | "huge" : "", | 20260508 01:45:16.490 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:16.490 | INFO | "malloc" : " 14 GB", | 20260508 01:45:16.490 | INFO | "pinned" : " 29 GB" | 20260508 01:45:16.490 | INFO | } | 20260508 01:45:16.490 | INFO | Perf report for the last 5.14186 seconds: | 20260508 01:45:16.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:16.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:45:16.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:45:16.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:16.490 | INFO | Uploaded 36GiB to GPU in 2.4086s (14.9464GiB/s) | 20260508 01:45:16.490 | INFO | Downloaded 34GiB to GPU in 2.40749s (14.1226GiB/s) | 20260508 01:45:21.629 | INFO | Analyzing chunk frames = [4262912,4263424) | 20260508 01:45:21.629 | INFO | MemInfo: { | 20260508 01:45:21.629 | INFO | "gpu" : " 17 GB", | 20260508 01:45:21.629 | INFO | "huge" : "", | 20260508 01:45:21.630 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:21.630 | INFO | "malloc" : " 14 GB", | 20260508 01:45:21.630 | INFO | "pinned" : " 29 GB" | 20260508 01:45:21.630 | INFO | } | 20260508 01:45:21.630 | INFO | Perf report for the last 5.13989 seconds: | 20260508 01:45:21.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:21.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 01:45:21.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 01:45:21.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:21.630 | INFO | Uploaded 36GiB to GPU in 2.47454s (14.5482GiB/s) | 20260508 01:45:21.630 | INFO | Downloaded 34GiB to GPU in 2.4599s (13.8217GiB/s) | 20260508 01:45:26.778 | INFO | Analyzing chunk frames = [4263424,4263936) | 20260508 01:45:26.778 | INFO | MemInfo: { | 20260508 01:45:26.778 | INFO | "gpu" : " 17 GB", | 20260508 01:45:26.778 | INFO | "huge" : "", | 20260508 01:45:26.778 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:26.778 | INFO | "malloc" : " 14 GB", | 20260508 01:45:26.778 | INFO | "pinned" : " 29 GB" | 20260508 01:45:26.778 | INFO | } | 20260508 01:45:26.778 | INFO | Perf report for the last 5.14833 seconds: | 20260508 01:45:26.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:26.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.778 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 01:45:26.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.778 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 01:45:26.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:26.779 | INFO | Uploaded 36GiB to GPU in 2.51956s (14.2882GiB/s) | 20260508 01:45:26.779 | INFO | Downloaded 34GiB to GPU in 2.45765s (13.8344GiB/s) | 20260508 01:45:26.891 | INFO | Starting a round of Baz IO | 20260508 01:45:30.344 | INFO | Populating RTMetrics report | 20260508 01:45:30.344 | INFO | RTMetrics report populated | 20260508 01:45:30.344 | INFO | Opening tmp RTMetrics json file | 20260508 01:45:30.359 | INFO | Serializing RTMetrics to json | 20260508 01:45:30.366 | INFO | Serialization done, swapping tmp file now... | 20260508 01:45:30.366 | INFO | RTMetrics done | 20260508 01:45:30.437 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.545 seconds | 20260508 01:45:30.437 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.74% / 0.00% / 58.26% | 20260508 01:45:30.437 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.810083% / 7.94729e-06% / 6.35783e-05% | 20260508 01:45:30.437 | INFO | Baz IO overall: | 20260508 01:45:30.437 | INFO | BytesWritten : 1930115253311 | 20260508 01:45:30.437 | INFO | NumEvents : 933473073109 | 20260508 01:45:31.913 | INFO | Analyzing chunk frames = [4263936,4264448) | 20260508 01:45:31.913 | INFO | MemInfo: { | 20260508 01:45:31.913 | INFO | "gpu" : " 17 GB", | 20260508 01:45:31.913 | INFO | "huge" : "", | 20260508 01:45:31.913 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:31.913 | INFO | "malloc" : " 14 GB", | 20260508 01:45:31.913 | INFO | "pinned" : " 29 GB" | 20260508 01:45:31.913 | INFO | } | 20260508 01:45:31.913 | INFO | Perf report for the last 5.13508 seconds: | 20260508 01:45:31.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:31.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:31.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:31.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:31.913 | INFO | ANALYSIS : 68.7%, 2.9, 103.9ms, 0 | 20260508 01:45:31.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:31.913 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 01:45:31.913 | INFO | PRE_HQ : 67.7%, 1.8, 62.6ms, 0 | 20260508 01:45:31.913 | INFO | BAZWRITER : 68.6%, 4.1, 145.2ms, 0 | 20260508 01:45:31.913 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:45:31.913 | INFO | Uploaded 36GiB to GPU in 2.63449s (13.6649GiB/s) | 20260508 01:45:31.914 | INFO | Downloaded 38GiB to GPU in 2.72257s (13.9574GiB/s) | 20260508 01:45:37.050 | INFO | Analyzing chunk frames = [4264448,4264960) | 20260508 01:45:37.050 | INFO | MemInfo: { | 20260508 01:45:37.050 | INFO | "gpu" : " 17 GB", | 20260508 01:45:37.050 | INFO | "huge" : "", | 20260508 01:45:37.050 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:37.050 | INFO | "malloc" : " 14 GB", | 20260508 01:45:37.050 | INFO | "pinned" : " 29 GB" | 20260508 01:45:37.050 | INFO | } | 20260508 01:45:37.050 | INFO | Perf report for the last 5.137 seconds: | 20260508 01:45:37.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:37.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.050 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 01:45:37.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.050 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:45:37.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:37.051 | INFO | Uploaded 36GiB to GPU in 2.55014s (14.1169GiB/s) | 20260508 01:45:37.051 | INFO | Downloaded 34GiB to GPU in 2.42732s (14.0072GiB/s) | 20260508 01:45:42.194 | INFO | Analyzing chunk frames = [4264960,4265472) | 20260508 01:45:42.194 | INFO | MemInfo: { | 20260508 01:45:42.194 | INFO | "gpu" : " 17 GB", | 20260508 01:45:42.194 | INFO | "huge" : "", | 20260508 01:45:42.194 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:42.194 | INFO | "malloc" : " 14 GB", | 20260508 01:45:42.194 | INFO | "pinned" : " 29 GB" | 20260508 01:45:42.194 | INFO | } | 20260508 01:45:42.194 | INFO | Perf report for the last 5.14354 seconds: | 20260508 01:45:42.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:42.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:45:42.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:45:42.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:42.194 | INFO | Uploaded 36GiB to GPU in 2.52882s (14.2359GiB/s) | 20260508 01:45:42.194 | INFO | Downloaded 34GiB to GPU in 2.49955s (13.6025GiB/s) | 20260508 01:45:47.332 | INFO | Analyzing chunk frames = [4265472,4265984) | 20260508 01:45:47.332 | INFO | MemInfo: { | 20260508 01:45:47.332 | INFO | "gpu" : " 17 GB", | 20260508 01:45:47.332 | INFO | "huge" : "", | 20260508 01:45:47.332 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:47.332 | INFO | "malloc" : " 14 GB", | 20260508 01:45:47.332 | INFO | "pinned" : " 29 GB" | 20260508 01:45:47.333 | INFO | } | 20260508 01:45:47.333 | INFO | Perf report for the last 5.13883 seconds: | 20260508 01:45:47.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:47.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:45:47.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:45:47.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:47.333 | INFO | Uploaded 36GiB to GPU in 2.53318s (14.2114GiB/s) | 20260508 01:45:47.333 | INFO | Downloaded 34GiB to GPU in 2.5055s (13.5702GiB/s) | 20260508 01:45:52.468 | INFO | Analyzing chunk frames = [4265984,4266496) | 20260508 01:45:52.468 | INFO | MemInfo: { | 20260508 01:45:52.468 | INFO | "gpu" : " 17 GB", | 20260508 01:45:52.469 | INFO | "huge" : "", | 20260508 01:45:52.469 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:52.469 | INFO | "malloc" : " 14 GB", | 20260508 01:45:52.469 | INFO | "pinned" : " 29 GB" | 20260508 01:45:52.469 | INFO | } | 20260508 01:45:52.469 | INFO | Perf report for the last 5.1361 seconds: | 20260508 01:45:52.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:52.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 01:45:52.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:45:52.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:52.469 | INFO | Uploaded 36GiB to GPU in 2.30897s (15.5914GiB/s) | 20260508 01:45:52.469 | INFO | Downloaded 34GiB to GPU in 2.53463s (13.4142GiB/s) | 20260508 01:45:57.604 | INFO | Analyzing chunk frames = [4266496,4267008) | 20260508 01:45:57.604 | INFO | MemInfo: { | 20260508 01:45:57.604 | INFO | "gpu" : " 17 GB", | 20260508 01:45:57.604 | INFO | "huge" : "", | 20260508 01:45:57.604 | INFO | "huge_shared" : " 29 GB", | 20260508 01:45:57.604 | INFO | "malloc" : " 14 GB", | 20260508 01:45:57.604 | INFO | "pinned" : " 29 GB" | 20260508 01:45:57.604 | INFO | } | 20260508 01:45:57.604 | INFO | Perf report for the last 5.13572 seconds: | 20260508 01:45:57.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:45:57.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.605 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:45:57.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.605 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:45:57.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:45:57.605 | INFO | Uploaded 36GiB to GPU in 2.3942s (15.0363GiB/s) | 20260508 01:45:57.605 | INFO | Downloaded 34GiB to GPU in 2.41494s (14.079GiB/s) | 20260508 01:46:02.752 | INFO | Analyzing chunk frames = [4267008,4267520) | 20260508 01:46:02.752 | INFO | MemInfo: { | 20260508 01:46:02.752 | INFO | "gpu" : " 17 GB", | 20260508 01:46:02.752 | INFO | "huge" : "", | 20260508 01:46:02.752 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:02.752 | INFO | "malloc" : " 14 GB", | 20260508 01:46:02.753 | INFO | "pinned" : " 29 GB" | 20260508 01:46:02.753 | INFO | } | 20260508 01:46:02.753 | INFO | Perf report for the last 5.14812 seconds: | 20260508 01:46:02.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:02.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | ANALYSIS : 66.1%, 3.0, 103.6ms, 0 | 20260508 01:46:02.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 01:46:02.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:02.753 | INFO | Uploaded 36GiB to GPU in 2.40426s (14.9734GiB/s) | 20260508 01:46:02.753 | INFO | Downloaded 34GiB to GPU in 2.42483s (14.0216GiB/s) | 20260508 01:46:07.890 | INFO | Analyzing chunk frames = [4267520,4268032) | 20260508 01:46:07.890 | INFO | MemInfo: { | 20260508 01:46:07.890 | INFO | "gpu" : " 17 GB", | 20260508 01:46:07.890 | INFO | "huge" : "", | 20260508 01:46:07.890 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:07.890 | INFO | "malloc" : " 14 GB", | 20260508 01:46:07.890 | INFO | "pinned" : " 29 GB" | 20260508 01:46:07.890 | INFO | } | 20260508 01:46:07.890 | INFO | Perf report for the last 5.13786 seconds: | 20260508 01:46:07.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:07.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.891 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:46:07.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.891 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:46:07.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:07.891 | INFO | Uploaded 36GiB to GPU in 2.55518s (14.089GiB/s) | 20260508 01:46:07.891 | INFO | Downloaded 34GiB to GPU in 2.46479s (13.7943GiB/s) | 20260508 01:46:07.891 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:46:07.891 | INFO | Total mem usage: 16.7 GiB | 20260508 01:46:07.891 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:46:07.891 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.1%) | 20260508 01:46:07.891 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:46:07.891 | INFO | High Water Breakdown: | 20260508 01:46:07.891 | INFO | 60.2% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:46:07.891 | INFO | 39.6% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:46:07.891 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:46:07.891 | INFO | ------------------------------------------ | 20260508 01:46:07.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:46:07.891 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:46:07.891 | INFO | Total mem usage: 26.5 GiB | 20260508 01:46:07.891 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:46:07.891 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 01:46:07.891 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 01:46:07.891 | INFO | High Water Breakdown: | 20260508 01:46:07.891 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:46:07.891 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:46:07.891 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:46:07.891 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:46:07.891 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:46:07.891 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:46:07.891 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:46:07.891 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:46:07.891 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:46:07.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:46:07.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:46:07.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:46:07.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:46:07.891 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:46:07.891 | INFO | ------------------------------------------ | 20260508 01:46:07.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:46:07.891 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:46:07.891 | INFO | Total mem usage: 27.8 GiB | 20260508 01:46:07.891 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:46:07.891 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:46:07.891 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:46:07.891 | INFO | High Water Breakdown: | 20260508 01:46:07.891 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:46:07.891 | INFO | ------------------------------------------ | 20260508 01:46:07.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:46:07.891 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:46:07.891 | INFO | Total mem usage: 15.7 GiB | 20260508 01:46:07.891 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:46:07.891 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:46:07.891 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:46:07.891 | INFO | High Water Breakdown: | 20260508 01:46:07.891 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:46:07.891 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:46:07.891 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:46:07.891 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:46:07.891 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:46:07.891 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:46:07.891 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:46:07.891 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:46:07.891 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:46:07.891 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:46:07.891 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:46:07.891 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:46:07.891 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:46:07.891 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:46:07.891 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:46:07.891 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:46:07.891 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:46:07.891 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:46:07.891 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:46:07.891 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:46:07.891 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:46:07.891 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:46:07.891 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:46:07.891 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:46:07.891 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:46:07.891 | INFO | ------------------------------------------ | 20260508 01:46:07.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:46:07.891 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:46:07.891 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:46:07.891 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:46:07.891 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.42e+01 / 1.52e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.26e-02 / 5.47e-03 | 20260508 01:46:07.891 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.49e+00, 5.80e-01 / 1.56e-01 | 20260508 01:46:07.891 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 9.70e-02 | 20260508 01:46:07.891 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.64e+01, 4.09e+00 / 1.73e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 56.27%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.58e+00 / 1.40e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.73e+00, 4.24e-01 / 1.90e-01 | 20260508 01:46:07.891 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.26e+00, 2.66e-01 / 1.17e-01 | 20260508 01:46:07.891 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.71e-01, 5.37e-02 / 3.11e-02 | 20260508 01:46:07.891 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:46:07.891 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:46:07.891 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:46:07.891 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 24.47%, Download -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 2.79%, LockDownload -- 2.97e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 15.48%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:46:07.891 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260508 01:46:07.988 | INFO | Starting a round of Baz IO | 20260508 01:46:11.439 | INFO | Populating RTMetrics report | 20260508 01:46:11.439 | INFO | RTMetrics report populated | 20260508 01:46:11.439 | INFO | Opening tmp RTMetrics json file | 20260508 01:46:11.521 | INFO | Serializing RTMetrics to json | 20260508 01:46:11.538 | INFO | Serialization done, swapping tmp file now... | 20260508 01:46:11.550 | INFO | RTMetrics done | 20260508 01:46:11.578 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.59 seconds | 20260508 01:46:11.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.74% / 0.00% / 58.26% | 20260508 01:46:11.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.806256% / 3.97364e-06% / 5.96046e-05% | 20260508 01:46:11.578 | INFO | Baz IO overall: | 20260508 01:46:11.578 | INFO | BytesWritten : 1931890599886 | 20260508 01:46:11.578 | INFO | NumEvents : 934324894625 | 20260508 01:46:13.029 | INFO | Analyzing chunk frames = [4268032,4268544) | 20260508 01:46:13.029 | INFO | MemInfo: { | 20260508 01:46:13.029 | INFO | "gpu" : " 17 GB", | 20260508 01:46:13.029 | INFO | "huge" : "", | 20260508 01:46:13.029 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:13.029 | INFO | "malloc" : " 14 GB", | 20260508 01:46:13.029 | INFO | "pinned" : " 29 GB" | 20260508 01:46:13.029 | INFO | } | 20260508 01:46:13.029 | INFO | Perf report for the last 5.13886 seconds: | 20260508 01:46:13.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:13.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:13.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:13.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:13.030 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 01:46:13.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:13.030 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 01:46:13.030 | INFO | PRE_HQ : 65.6%, 1.7, 56.3ms, 0 | 20260508 01:46:13.030 | INFO | BAZWRITER : 69.5%, 3.6, 128.5ms, 0 | 20260508 01:46:13.030 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260508 01:46:13.030 | INFO | Uploaded 36GiB to GPU in 2.58316s (13.9364GiB/s) | 20260508 01:46:13.030 | INFO | Downloaded 38GiB to GPU in 2.70282s (14.0594GiB/s) | 20260508 01:46:18.166 | INFO | Analyzing chunk frames = [4268544,4269056) | 20260508 01:46:18.166 | INFO | MemInfo: { | 20260508 01:46:18.166 | INFO | "gpu" : " 17 GB", | 20260508 01:46:18.166 | INFO | "huge" : "", | 20260508 01:46:18.166 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:18.166 | INFO | "malloc" : " 14 GB", | 20260508 01:46:18.166 | INFO | "pinned" : " 29 GB" | 20260508 01:46:18.166 | INFO | } | 20260508 01:46:18.166 | INFO | Perf report for the last 5.13693 seconds: | 20260508 01:46:18.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:18.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.166 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:46:18.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.166 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:46:18.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:18.167 | INFO | Uploaded 36GiB to GPU in 2.53719s (14.1889GiB/s) | 20260508 01:46:18.167 | INFO | Downloaded 34GiB to GPU in 2.43713s (13.9508GiB/s) | 20260508 01:46:23.303 | INFO | Analyzing chunk frames = [4269056,4269568) | 20260508 01:46:23.303 | INFO | MemInfo: { | 20260508 01:46:23.303 | INFO | "gpu" : " 17 GB", | 20260508 01:46:23.303 | INFO | "huge" : "", | 20260508 01:46:23.303 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:23.303 | INFO | "malloc" : " 14 GB", | 20260508 01:46:23.303 | INFO | "pinned" : " 29 GB" | 20260508 01:46:23.303 | INFO | } | 20260508 01:46:23.304 | INFO | Perf report for the last 5.13724 seconds: | 20260508 01:46:23.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:23.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 01:46:23.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:46:23.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:23.304 | INFO | Uploaded 36GiB to GPU in 2.55019s (14.1166GiB/s) | 20260508 01:46:23.304 | INFO | Downloaded 34GiB to GPU in 2.50569s (13.5691GiB/s) | 20260508 01:46:28.443 | INFO | Analyzing chunk frames = [4269568,4270080) | 20260508 01:46:28.443 | INFO | MemInfo: { | 20260508 01:46:28.443 | INFO | "gpu" : " 17 GB", | 20260508 01:46:28.443 | INFO | "huge" : "", | 20260508 01:46:28.443 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:28.443 | INFO | "malloc" : " 14 GB", | 20260508 01:46:28.443 | INFO | "pinned" : " 29 GB" | 20260508 01:46:28.443 | INFO | } | 20260508 01:46:28.443 | INFO | Perf report for the last 5.13941 seconds: | 20260508 01:46:28.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:28.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:46:28.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:46:28.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:28.443 | INFO | Uploaded 36GiB to GPU in 2.50714s (14.359GiB/s) | 20260508 01:46:28.443 | INFO | Downloaded 34GiB to GPU in 2.49652s (13.619GiB/s) | 20260508 01:46:33.579 | INFO | Analyzing chunk frames = [4270080,4270592) | 20260508 01:46:33.579 | INFO | MemInfo: { | 20260508 01:46:33.580 | INFO | "gpu" : " 17 GB", | 20260508 01:46:33.580 | INFO | "huge" : "", | 20260508 01:46:33.580 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:33.580 | INFO | "malloc" : " 14 GB", | 20260508 01:46:33.580 | INFO | "pinned" : " 29 GB" | 20260508 01:46:33.580 | INFO | } | 20260508 01:46:33.580 | INFO | Perf report for the last 5.13695 seconds: | 20260508 01:46:33.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:33.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.580 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:46:33.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.580 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 01:46:33.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:33.581 | INFO | Uploaded 36GiB to GPU in 2.30071s (15.6474GiB/s) | 20260508 01:46:33.581 | INFO | Downloaded 34GiB to GPU in 2.51378s (13.5255GiB/s) | 20260508 01:46:38.722 | INFO | Analyzing chunk frames = [4270592,4271104) | 20260508 01:46:38.722 | INFO | MemInfo: { | 20260508 01:46:38.722 | INFO | "gpu" : " 17 GB", | 20260508 01:46:38.722 | INFO | "huge" : "", | 20260508 01:46:38.722 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:38.723 | INFO | "malloc" : " 14 GB", | 20260508 01:46:38.723 | INFO | "pinned" : " 29 GB" | 20260508 01:46:38.723 | INFO | } | 20260508 01:46:38.723 | INFO | Perf report for the last 5.14279 seconds: | 20260508 01:46:38.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:38.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:46:38.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 01:46:38.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:38.723 | INFO | Uploaded 36GiB to GPU in 2.38558s (15.0907GiB/s) | 20260508 01:46:38.724 | INFO | Downloaded 34GiB to GPU in 2.42331s (14.0304GiB/s) | 20260508 01:46:43.862 | INFO | Analyzing chunk frames = [4271104,4271616) | 20260508 01:46:43.862 | INFO | MemInfo: { | 20260508 01:46:43.862 | INFO | "gpu" : " 17 GB", | 20260508 01:46:43.862 | INFO | "huge" : "", | 20260508 01:46:43.862 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:43.862 | INFO | "malloc" : " 14 GB", | 20260508 01:46:43.862 | INFO | "pinned" : " 29 GB" | 20260508 01:46:43.862 | INFO | } | 20260508 01:46:43.862 | INFO | Perf report for the last 5.13973 seconds: | 20260508 01:46:43.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:43.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.862 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 01:46:43.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.862 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 01:46:43.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:43.863 | INFO | Uploaded 36GiB to GPU in 2.43955s (14.7568GiB/s) | 20260508 01:46:43.863 | INFO | Downloaded 34GiB to GPU in 2.42779s (14.0045GiB/s) | 20260508 01:46:49.006 | INFO | Analyzing chunk frames = [4271616,4272128) | 20260508 01:46:49.006 | INFO | MemInfo: { | 20260508 01:46:49.006 | INFO | "gpu" : " 17 GB", | 20260508 01:46:49.006 | INFO | "huge" : "", | 20260508 01:46:49.006 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:49.006 | INFO | "malloc" : " 14 GB", | 20260508 01:46:49.006 | INFO | "pinned" : " 29 GB" | 20260508 01:46:49.006 | INFO | } | 20260508 01:46:49.006 | INFO | Perf report for the last 5.14356 seconds: | 20260508 01:46:49.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:49.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:46:49.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:46:49.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:49.006 | INFO | Uploaded 36GiB to GPU in 2.53906s (14.1785GiB/s) | 20260508 01:46:49.006 | INFO | Downloaded 34GiB to GPU in 2.45653s (13.8406GiB/s) | 20260508 01:46:49.103 | INFO | Starting a round of Baz IO | 20260508 01:46:52.545 | INFO | Populating RTMetrics report | 20260508 01:46:52.545 | INFO | RTMetrics report populated | 20260508 01:46:52.545 | INFO | Opening tmp RTMetrics json file | 20260508 01:46:52.566 | INFO | Serializing RTMetrics to json | 20260508 01:46:52.620 | INFO | Serialization done, swapping tmp file now... | 20260508 01:46:52.642 | INFO | RTMetrics done | 20260508 01:46:52.675 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.571 seconds | 20260508 01:46:52.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.73% / 0.00% / 58.26% | 20260508 01:46:52.675 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.807615% / 0% / 5.5631e-05% | 20260508 01:46:52.675 | INFO | Baz IO overall: | 20260508 01:46:52.675 | INFO | BytesWritten : 1933668103969 | 20260508 01:46:52.675 | INFO | NumEvents : 935178514836 | 20260508 01:46:54.142 | INFO | Analyzing chunk frames = [4272128,4272640) | 20260508 01:46:54.142 | INFO | MemInfo: { | 20260508 01:46:54.142 | INFO | "gpu" : " 17 GB", | 20260508 01:46:54.142 | INFO | "huge" : "", | 20260508 01:46:54.142 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:54.142 | INFO | "malloc" : " 14 GB", | 20260508 01:46:54.142 | INFO | "pinned" : " 29 GB" | 20260508 01:46:54.142 | INFO | } | 20260508 01:46:54.142 | INFO | Perf report for the last 5.13622 seconds: | 20260508 01:46:54.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:54.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:54.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:54.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:54.142 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 01:46:54.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:54.142 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260508 01:46:54.142 | INFO | PRE_HQ : 65.6%, 1.6, 56.0ms, 0 | 20260508 01:46:54.142 | INFO | BAZWRITER : 69.1%, 3.9, 139.7ms, 0 | 20260508 01:46:54.142 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 01:46:54.143 | INFO | Uploaded 36GiB to GPU in 2.59918s (13.8505GiB/s) | 20260508 01:46:54.143 | INFO | Downloaded 38GiB to GPU in 2.70293s (14.0588GiB/s) | 20260508 01:46:59.280 | INFO | Analyzing chunk frames = [4272640,4273152) | 20260508 01:46:59.280 | INFO | MemInfo: { | 20260508 01:46:59.280 | INFO | "gpu" : " 17 GB", | 20260508 01:46:59.280 | INFO | "huge" : "", | 20260508 01:46:59.280 | INFO | "huge_shared" : " 29 GB", | 20260508 01:46:59.280 | INFO | "malloc" : " 14 GB", | 20260508 01:46:59.280 | INFO | "pinned" : " 29 GB" | 20260508 01:46:59.280 | INFO | } | 20260508 01:46:59.280 | INFO | Perf report for the last 5.13796 seconds: | 20260508 01:46:59.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:46:59.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 01:46:59.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:46:59.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:46:59.280 | INFO | Uploaded 36GiB to GPU in 2.52344s (14.2662GiB/s) | 20260508 01:46:59.280 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.9751GiB/s) | 20260508 01:47:04.419 | INFO | Analyzing chunk frames = [4273152,4273664) | 20260508 01:47:04.419 | INFO | MemInfo: { | 20260508 01:47:04.419 | INFO | "gpu" : " 17 GB", | 20260508 01:47:04.419 | INFO | "huge" : "", | 20260508 01:47:04.419 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:04.419 | INFO | "malloc" : " 14 GB", | 20260508 01:47:04.419 | INFO | "pinned" : " 29 GB" | 20260508 01:47:04.419 | INFO | } | 20260508 01:47:04.419 | INFO | Perf report for the last 5.13916 seconds: | 20260508 01:47:04.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:04.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.419 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 01:47:04.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.420 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:47:04.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:04.420 | INFO | Uploaded 36GiB to GPU in 2.5452s (14.1443GiB/s) | 20260508 01:47:04.420 | INFO | Downloaded 34GiB to GPU in 2.4866s (13.6733GiB/s) | 20260508 01:47:09.557 | INFO | Analyzing chunk frames = [4273664,4274176) | 20260508 01:47:09.557 | INFO | MemInfo: { | 20260508 01:47:09.557 | INFO | "gpu" : " 17 GB", | 20260508 01:47:09.558 | INFO | "huge" : "", | 20260508 01:47:09.558 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:09.558 | INFO | "malloc" : " 14 GB", | 20260508 01:47:09.558 | INFO | "pinned" : " 29 GB" | 20260508 01:47:09.558 | INFO | } | 20260508 01:47:09.558 | INFO | Perf report for the last 5.13818 seconds: | 20260508 01:47:09.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:09.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:47:09.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 01:47:09.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:09.558 | INFO | Uploaded 36GiB to GPU in 2.53358s (14.2091GiB/s) | 20260508 01:47:09.558 | INFO | Downloaded 34GiB to GPU in 2.50778s (13.5578GiB/s) | 20260508 01:47:14.702 | INFO | Analyzing chunk frames = [4274176,4274688) | 20260508 01:47:14.702 | INFO | MemInfo: { | 20260508 01:47:14.702 | INFO | "gpu" : " 17 GB", | 20260508 01:47:14.702 | INFO | "huge" : "", | 20260508 01:47:14.702 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:14.702 | INFO | "malloc" : " 14 GB", | 20260508 01:47:14.702 | INFO | "pinned" : " 29 GB" | 20260508 01:47:14.702 | INFO | } | 20260508 01:47:14.702 | INFO | Perf report for the last 5.14419 seconds: | 20260508 01:47:14.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:14.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 01:47:14.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 01:47:14.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:14.702 | INFO | Uploaded 36GiB to GPU in 2.30345s (15.6287GiB/s) | 20260508 01:47:14.702 | INFO | Downloaded 34GiB to GPU in 2.50489s (13.5735GiB/s) | 20260508 01:47:19.844 | INFO | Analyzing chunk frames = [4274688,4275200) | 20260508 01:47:19.844 | INFO | MemInfo: { | 20260508 01:47:19.844 | INFO | "gpu" : " 17 GB", | 20260508 01:47:19.844 | INFO | "huge" : "", | 20260508 01:47:19.844 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:19.844 | INFO | "malloc" : " 14 GB", | 20260508 01:47:19.844 | INFO | "pinned" : " 29 GB" | 20260508 01:47:19.844 | INFO | } | 20260508 01:47:19.844 | INFO | Perf report for the last 5.14269 seconds: | 20260508 01:47:19.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:19.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:47:19.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260508 01:47:19.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:19.845 | INFO | Uploaded 36GiB to GPU in 2.40907s (14.9435GiB/s) | 20260508 01:47:19.845 | INFO | Downloaded 34GiB to GPU in 2.41714s (14.0662GiB/s) | 20260508 01:47:24.980 | INFO | Analyzing chunk frames = [4275200,4275712) | 20260508 01:47:24.980 | INFO | MemInfo: { | 20260508 01:47:24.980 | INFO | "gpu" : " 17 GB", | 20260508 01:47:24.980 | INFO | "huge" : "", | 20260508 01:47:24.980 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:24.980 | INFO | "malloc" : " 14 GB", | 20260508 01:47:24.980 | INFO | "pinned" : " 29 GB" | 20260508 01:47:24.980 | INFO | } | 20260508 01:47:24.980 | INFO | Perf report for the last 5.1353 seconds: | 20260508 01:47:24.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:24.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:47:24.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:47:24.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:24.980 | INFO | Uploaded 36GiB to GPU in 2.52196s (14.2746GiB/s) | 20260508 01:47:24.980 | INFO | Downloaded 34GiB to GPU in 2.45662s (13.8402GiB/s) | 20260508 01:47:30.113 | INFO | Analyzing chunk frames = [4275712,4276224) | 20260508 01:47:30.113 | INFO | MemInfo: { | 20260508 01:47:30.113 | INFO | "gpu" : " 17 GB", | 20260508 01:47:30.113 | INFO | "huge" : "", | 20260508 01:47:30.113 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:30.113 | INFO | "malloc" : " 14 GB", | 20260508 01:47:30.113 | INFO | "pinned" : " 29 GB" | 20260508 01:47:30.113 | INFO | } | 20260508 01:47:30.113 | INFO | Perf report for the last 5.13297 seconds: | 20260508 01:47:30.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:30.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260508 01:47:30.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:47:30.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:30.113 | INFO | Uploaded 36GiB to GPU in 2.54049s (14.1705GiB/s) | 20260508 01:47:30.113 | INFO | Downloaded 34GiB to GPU in 2.46288s (13.805GiB/s) | 20260508 01:47:30.113 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:47:30.113 | INFO | Total mem usage: 16.7 GiB | 20260508 01:47:30.113 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:47:30.113 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 01:47:30.113 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:47:30.113 | INFO | High Water Breakdown: | 20260508 01:47:30.113 | INFO | 59.8% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:47:30.113 | INFO | 40% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:47:30.113 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:47:30.113 | INFO | ------------------------------------------ | 20260508 01:47:30.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:47:30.113 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:47:30.113 | INFO | Total mem usage: 26.5 GiB | 20260508 01:47:30.113 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:47:30.113 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:47:30.113 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:47:30.113 | INFO | High Water Breakdown: | 20260508 01:47:30.113 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:47:30.113 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:47:30.113 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:47:30.113 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:47:30.113 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:47:30.113 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:47:30.113 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:47:30.113 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:47:30.113 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:47:30.113 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:47:30.113 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:47:30.113 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:47:30.113 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:47:30.113 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:47:30.113 | INFO | ------------------------------------------ | 20260508 01:47:30.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:47:30.113 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:47:30.113 | INFO | Total mem usage: 27.8 GiB | 20260508 01:47:30.113 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:47:30.113 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:47:30.113 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:47:30.113 | INFO | High Water Breakdown: | 20260508 01:47:30.113 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:47:30.113 | INFO | ------------------------------------------ | 20260508 01:47:30.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:47:30.113 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:47:30.113 | INFO | Total mem usage: 15.7 GiB | 20260508 01:47:30.113 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:47:30.113 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:47:30.113 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:47:30.113 | INFO | High Water Breakdown: | 20260508 01:47:30.113 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:47:30.113 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:47:30.113 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:47:30.113 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:47:30.113 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:47:30.113 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:47:30.113 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:47:30.113 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:47:30.113 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:47:30.113 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:47:30.113 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:47:30.113 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:47:30.113 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:47:30.113 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:47:30.113 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:47:30.113 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:47:30.114 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:47:30.114 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:47:30.114 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:47:30.114 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:47:30.114 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:47:30.114 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:47:30.114 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:47:30.114 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:47:30.114 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:47:30.114 | INFO | ------------------------------------------ | 20260508 01:47:30.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:47:30.114 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:47:30.114 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:47:30.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:47:30.114 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.42e+01 / 1.13e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.37e-02 / 5.99e-03 | 20260508 01:47:30.114 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.51e+00, 5.82e-01 / 1.76e-01 | 20260508 01:47:30.114 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.50e+00, 2.07e-01 / 8.25e-02 | 20260508 01:47:30.114 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.59e+01, 3.93e+00 / 1.31e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.61e+00 / 1.57e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.71e+00, 4.25e-01 / 2.18e-01 | 20260508 01:47:30.114 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.29e+00, 2.66e-01 / 1.54e-01 | 20260508 01:47:30.114 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.73e-01, 5.29e-02 / 3.21e-02 | 20260508 01:47:30.114 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:47:30.114 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:47:30.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:47:30.114 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 30.67%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:47:30.114 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260508 01:47:30.210 | INFO | Starting a round of Baz IO | 20260508 01:47:33.661 | INFO | Populating RTMetrics report | 20260508 01:47:33.661 | INFO | RTMetrics report populated | 20260508 01:47:33.661 | INFO | Opening tmp RTMetrics json file | 20260508 01:47:33.685 | INFO | Serializing RTMetrics to json | 20260508 01:47:33.687 | INFO | Serialization done, swapping tmp file now... | 20260508 01:47:33.688 | INFO | RTMetrics done | 20260508 01:47:33.769 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.558 seconds | 20260508 01:47:33.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.73% / 0.00% / 58.27% | 20260508 01:47:33.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.804909% / 0% / 6.75519e-05% | 20260508 01:47:33.769 | INFO | Baz IO overall: | 20260508 01:47:33.769 | INFO | BytesWritten : 1935444214343 | 20260508 01:47:33.769 | INFO | NumEvents : 936031109093 | 20260508 01:47:35.254 | INFO | Analyzing chunk frames = [4276224,4276736) | 20260508 01:47:35.254 | INFO | MemInfo: { | 20260508 01:47:35.254 | INFO | "gpu" : " 17 GB", | 20260508 01:47:35.254 | INFO | "huge" : "", | 20260508 01:47:35.254 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:35.254 | INFO | "malloc" : " 14 GB", | 20260508 01:47:35.254 | INFO | "pinned" : " 29 GB" | 20260508 01:47:35.254 | INFO | } | 20260508 01:47:35.254 | INFO | Perf report for the last 5.14091 seconds: | 20260508 01:47:35.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:35.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:35.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:35.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:35.254 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 01:47:35.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:35.254 | INFO | RT_METRICS : 40.5%, 1.0, 21.3ms, 0 | 20260508 01:47:35.254 | INFO | PRE_HQ : 65.3%, 1.7, 56.3ms, 0 | 20260508 01:47:35.254 | INFO | BAZWRITER : 68.8%, 4.0, 140.5ms, 0 | 20260508 01:47:35.254 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 01:47:35.254 | INFO | Uploaded 36GiB to GPU in 2.60717s (13.8081GiB/s) | 20260508 01:47:35.254 | INFO | Downloaded 38GiB to GPU in 2.67823s (14.1885GiB/s) | 20260508 01:47:40.397 | INFO | Analyzing chunk frames = [4276736,4277248) | 20260508 01:47:40.397 | INFO | MemInfo: { | 20260508 01:47:40.397 | INFO | "gpu" : " 17 GB", | 20260508 01:47:40.397 | INFO | "huge" : "", | 20260508 01:47:40.397 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:40.397 | INFO | "malloc" : " 14 GB", | 20260508 01:47:40.397 | INFO | "pinned" : " 29 GB" | 20260508 01:47:40.397 | INFO | } | 20260508 01:47:40.397 | INFO | Perf report for the last 5.14359 seconds: | 20260508 01:47:40.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:40.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.397 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:47:40.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.397 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:47:40.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:40.398 | INFO | Uploaded 36GiB to GPU in 2.53089s (14.2242GiB/s) | 20260508 01:47:40.398 | INFO | Downloaded 34GiB to GPU in 2.4331s (13.9739GiB/s) | 20260508 01:47:45.533 | INFO | Analyzing chunk frames = [4277248,4277760) | 20260508 01:47:45.533 | INFO | MemInfo: { | 20260508 01:47:45.533 | INFO | "gpu" : " 17 GB", | 20260508 01:47:45.533 | INFO | "huge" : "", | 20260508 01:47:45.533 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:45.533 | INFO | "malloc" : " 14 GB", | 20260508 01:47:45.533 | INFO | "pinned" : " 29 GB" | 20260508 01:47:45.533 | INFO | } | 20260508 01:47:45.533 | INFO | Perf report for the last 5.13553 seconds: | 20260508 01:47:45.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:45.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:47:45.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:47:45.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:45.533 | INFO | Uploaded 36GiB to GPU in 2.51103s (14.3368GiB/s) | 20260508 01:47:45.533 | INFO | Downloaded 34GiB to GPU in 2.49209s (13.6432GiB/s) | 20260508 01:47:50.670 | INFO | Analyzing chunk frames = [4277760,4278272) | 20260508 01:47:50.670 | INFO | MemInfo: { | 20260508 01:47:50.670 | INFO | "gpu" : " 17 GB", | 20260508 01:47:50.670 | INFO | "huge" : "", | 20260508 01:47:50.670 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:50.670 | INFO | "malloc" : " 14 GB", | 20260508 01:47:50.670 | INFO | "pinned" : " 29 GB" | 20260508 01:47:50.670 | INFO | } | 20260508 01:47:50.670 | INFO | Perf report for the last 5.13725 seconds: | 20260508 01:47:50.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:50.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.670 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 01:47:50.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.670 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:47:50.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:50.671 | INFO | Uploaded 36GiB to GPU in 2.5224s (14.2721GiB/s) | 20260508 01:47:50.671 | INFO | Downloaded 34GiB to GPU in 2.49251s (13.6409GiB/s) | 20260508 01:47:55.811 | INFO | Analyzing chunk frames = [4278272,4278784) | 20260508 01:47:55.811 | INFO | MemInfo: { | 20260508 01:47:55.811 | INFO | "gpu" : " 17 GB", | 20260508 01:47:55.814 | INFO | "huge" : "", | 20260508 01:47:55.814 | INFO | "huge_shared" : " 29 GB", | 20260508 01:47:55.814 | INFO | "malloc" : " 14 GB", | 20260508 01:47:55.814 | INFO | "pinned" : " 29 GB" | 20260508 01:47:55.814 | INFO | } | 20260508 01:47:55.814 | INFO | Perf report for the last 5.14366 seconds: | 20260508 01:47:55.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:47:55.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 01:47:55.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260508 01:47:55.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:47:55.814 | INFO | Uploaded 36GiB to GPU in 2.32919s (15.456GiB/s) | 20260508 01:47:55.814 | INFO | Downloaded 34GiB to GPU in 2.51145s (13.538GiB/s) | 20260508 01:48:00.956 | INFO | Analyzing chunk frames = [4278784,4279296) | 20260508 01:48:00.956 | INFO | MemInfo: { | 20260508 01:48:00.956 | INFO | "gpu" : " 17 GB", | 20260508 01:48:00.956 | INFO | "huge" : "", | 20260508 01:48:00.956 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:00.956 | INFO | "malloc" : " 14 GB", | 20260508 01:48:00.956 | INFO | "pinned" : " 29 GB" | 20260508 01:48:00.956 | INFO | } | 20260508 01:48:00.956 | INFO | Perf report for the last 5.1425 seconds: | 20260508 01:48:00.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:00.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.956 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 01:48:00.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.956 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:48:00.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:00.957 | INFO | Uploaded 36GiB to GPU in 2.3641s (15.2278GiB/s) | 20260508 01:48:00.957 | INFO | Downloaded 34GiB to GPU in 2.43023s (13.9905GiB/s) | 20260508 01:48:06.097 | INFO | Analyzing chunk frames = [4279296,4279808) | 20260508 01:48:06.097 | INFO | MemInfo: { | 20260508 01:48:06.097 | INFO | "gpu" : " 17 GB", | 20260508 01:48:06.097 | INFO | "huge" : "", | 20260508 01:48:06.097 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:06.097 | INFO | "malloc" : " 14 GB", | 20260508 01:48:06.097 | INFO | "pinned" : " 29 GB" | 20260508 01:48:06.097 | INFO | } | 20260508 01:48:06.097 | INFO | Perf report for the last 5.14048 seconds: | 20260508 01:48:06.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:06.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:48:06.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:48:06.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:06.097 | INFO | Uploaded 36GiB to GPU in 2.4394s (14.7577GiB/s) | 20260508 01:48:06.097 | INFO | Downloaded 34GiB to GPU in 2.44728s (13.893GiB/s) | 20260508 01:48:11.235 | INFO | Analyzing chunk frames = [4279808,4280320) | 20260508 01:48:11.235 | INFO | MemInfo: { | 20260508 01:48:11.235 | INFO | "gpu" : " 17 GB", | 20260508 01:48:11.235 | INFO | "huge" : "", | 20260508 01:48:11.235 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:11.235 | INFO | "malloc" : " 14 GB", | 20260508 01:48:11.235 | INFO | "pinned" : " 29 GB" | 20260508 01:48:11.235 | INFO | } | 20260508 01:48:11.235 | INFO | Perf report for the last 5.13836 seconds: | 20260508 01:48:11.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:11.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:48:11.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:48:11.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:11.235 | INFO | Uploaded 36GiB to GPU in 2.53959s (14.1755GiB/s) | 20260508 01:48:11.236 | INFO | Downloaded 34GiB to GPU in 2.46948s (13.7681GiB/s) | 20260508 01:48:11.333 | INFO | Starting a round of Baz IO | 20260508 01:48:14.771 | INFO | Populating RTMetrics report | 20260508 01:48:14.772 | INFO | RTMetrics report populated | 20260508 01:48:14.772 | INFO | Opening tmp RTMetrics json file | 20260508 01:48:14.789 | INFO | Serializing RTMetrics to json | 20260508 01:48:14.800 | INFO | Serialization done, swapping tmp file now... | 20260508 01:48:14.800 | INFO | RTMetrics done | 20260508 01:48:14.895 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.561 seconds | 20260508 01:48:14.895 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.73% / 0.00% / 58.27% | 20260508 01:48:14.895 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.80454% / 1.58946e-05% / 4.37101e-05% | 20260508 01:48:14.895 | INFO | Baz IO overall: | 20260508 01:48:14.895 | INFO | BytesWritten : 1937219809237 | 20260508 01:48:14.895 | INFO | NumEvents : 936883636189 | 20260508 01:48:16.369 | INFO | Analyzing chunk frames = [4280320,4280832) | 20260508 01:48:16.369 | INFO | MemInfo: { | 20260508 01:48:16.369 | INFO | "gpu" : " 17 GB", | 20260508 01:48:16.369 | INFO | "huge" : "", | 20260508 01:48:16.369 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:16.369 | INFO | "malloc" : " 14 GB", | 20260508 01:48:16.369 | INFO | "pinned" : " 29 GB" | 20260508 01:48:16.369 | INFO | } | 20260508 01:48:16.369 | INFO | Perf report for the last 5.13419 seconds: | 20260508 01:48:16.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:16.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:16.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:16.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:16.369 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 01:48:16.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:16.369 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 01:48:16.369 | INFO | PRE_HQ : 65.7%, 1.7, 56.2ms, 0 | 20260508 01:48:16.370 | INFO | BAZWRITER : 68.9%, 3.7, 130.3ms, 0 | 20260508 01:48:16.370 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 01:48:16.370 | INFO | Uploaded 36GiB to GPU in 2.58358s (13.9342GiB/s) | 20260508 01:48:16.370 | INFO | Downloaded 38GiB to GPU in 2.69547s (14.0977GiB/s) | 20260508 01:48:21.509 | INFO | Analyzing chunk frames = [4280832,4281344) | 20260508 01:48:21.509 | INFO | MemInfo: { | 20260508 01:48:21.509 | INFO | "gpu" : " 17 GB", | 20260508 01:48:21.509 | INFO | "huge" : "", | 20260508 01:48:21.509 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:21.509 | INFO | "malloc" : " 14 GB", | 20260508 01:48:21.509 | INFO | "pinned" : " 29 GB" | 20260508 01:48:21.509 | INFO | } | 20260508 01:48:21.509 | INFO | Perf report for the last 5.13989 seconds: | 20260508 01:48:21.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:21.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.509 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:48:21.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.509 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:48:21.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:21.510 | INFO | Uploaded 36GiB to GPU in 2.55194s (14.1069GiB/s) | 20260508 01:48:21.510 | INFO | Downloaded 34GiB to GPU in 2.43816s (13.9449GiB/s) | 20260508 01:48:26.651 | INFO | Analyzing chunk frames = [4281344,4281856) | 20260508 01:48:26.651 | INFO | MemInfo: { | 20260508 01:48:26.651 | INFO | "gpu" : " 17 GB", | 20260508 01:48:26.651 | INFO | "huge" : "", | 20260508 01:48:26.651 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:26.651 | INFO | "malloc" : " 14 GB", | 20260508 01:48:26.651 | INFO | "pinned" : " 29 GB" | 20260508 01:48:26.651 | INFO | } | 20260508 01:48:26.651 | INFO | Perf report for the last 5.14171 seconds: | 20260508 01:48:26.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:26.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 01:48:26.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:48:26.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:26.651 | INFO | Uploaded 36GiB to GPU in 2.44496s (14.7242GiB/s) | 20260508 01:48:26.651 | INFO | Downloaded 34GiB to GPU in 2.49429s (13.6311GiB/s) | 20260508 01:48:31.792 | INFO | Analyzing chunk frames = [4281856,4282368) | 20260508 01:48:31.792 | INFO | MemInfo: { | 20260508 01:48:31.792 | INFO | "gpu" : " 17 GB", | 20260508 01:48:31.792 | INFO | "huge" : "", | 20260508 01:48:31.792 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:31.792 | INFO | "malloc" : " 14 GB", | 20260508 01:48:31.792 | INFO | "pinned" : " 29 GB" | 20260508 01:48:31.792 | INFO | } | 20260508 01:48:31.792 | INFO | Perf report for the last 5.14092 seconds: | 20260508 01:48:31.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:31.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 01:48:31.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:48:31.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:31.792 | INFO | Uploaded 36GiB to GPU in 2.48563s (14.4833GiB/s) | 20260508 01:48:31.792 | INFO | Downloaded 34GiB to GPU in 2.4919s (13.6442GiB/s) | 20260508 01:48:36.927 | INFO | Analyzing chunk frames = [4282368,4282880) | 20260508 01:48:36.927 | INFO | MemInfo: { | 20260508 01:48:36.927 | INFO | "gpu" : " 17 GB", | 20260508 01:48:36.927 | INFO | "huge" : "", | 20260508 01:48:36.927 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:36.927 | INFO | "malloc" : " 14 GB", | 20260508 01:48:36.927 | INFO | "pinned" : " 29 GB" | 20260508 01:48:36.927 | INFO | } | 20260508 01:48:36.927 | INFO | Perf report for the last 5.13535 seconds: | 20260508 01:48:36.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:36.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.927 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:48:36.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.927 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:48:36.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:36.928 | INFO | Uploaded 36GiB to GPU in 2.31141s (15.5749GiB/s) | 20260508 01:48:36.928 | INFO | Downloaded 34GiB to GPU in 2.50351s (13.581GiB/s) | 20260508 01:48:42.066 | INFO | Analyzing chunk frames = [4282880,4283392) | 20260508 01:48:42.066 | INFO | MemInfo: { | 20260508 01:48:42.066 | INFO | "gpu" : " 17 GB", | 20260508 01:48:42.066 | INFO | "huge" : "", | 20260508 01:48:42.066 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:42.066 | INFO | "malloc" : " 14 GB", | 20260508 01:48:42.066 | INFO | "pinned" : " 29 GB" | 20260508 01:48:42.066 | INFO | } | 20260508 01:48:42.066 | INFO | Perf report for the last 5.1389 seconds: | 20260508 01:48:42.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:42.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.066 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:48:42.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.066 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:48:42.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:42.067 | INFO | Uploaded 36GiB to GPU in 2.39134s (15.0543GiB/s) | 20260508 01:48:42.067 | INFO | Downloaded 34GiB to GPU in 2.41291s (14.0909GiB/s) | 20260508 01:48:47.207 | INFO | Analyzing chunk frames = [4283392,4283904) | 20260508 01:48:47.207 | INFO | MemInfo: { | 20260508 01:48:47.207 | INFO | "gpu" : " 17 GB", | 20260508 01:48:47.207 | INFO | "huge" : "", | 20260508 01:48:47.207 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:47.207 | INFO | "malloc" : " 14 GB", | 20260508 01:48:47.207 | INFO | "pinned" : " 29 GB" | 20260508 01:48:47.207 | INFO | } | 20260508 01:48:47.207 | INFO | Perf report for the last 5.14122 seconds: | 20260508 01:48:47.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:47.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.207 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 01:48:47.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.208 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:48:47.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:47.208 | INFO | Uploaded 36GiB to GPU in 2.45663s (14.6542GiB/s) | 20260508 01:48:47.208 | INFO | Downloaded 34GiB to GPU in 2.43458s (13.9655GiB/s) | 20260508 01:48:52.346 | INFO | Analyzing chunk frames = [4283904,4284416) | 20260508 01:48:52.346 | INFO | MemInfo: { | 20260508 01:48:52.346 | INFO | "gpu" : " 17 GB", | 20260508 01:48:52.346 | INFO | "huge" : "", | 20260508 01:48:52.346 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:52.346 | INFO | "malloc" : " 14 GB", | 20260508 01:48:52.346 | INFO | "pinned" : " 29 GB" | 20260508 01:48:52.346 | INFO | } | 20260508 01:48:52.346 | INFO | Perf report for the last 5.13836 seconds: | 20260508 01:48:52.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:52.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:48:52.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:48:52.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:52.346 | INFO | Uploaded 36GiB to GPU in 2.54485s (14.1462GiB/s) | 20260508 01:48:52.346 | INFO | Downloaded 34GiB to GPU in 2.4578s (13.8335GiB/s) | 20260508 01:48:52.346 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:48:52.346 | INFO | Total mem usage: 16.7 GiB | 20260508 01:48:52.346 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:48:52.346 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 01:48:52.346 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:48:52.346 | INFO | High Water Breakdown: | 20260508 01:48:52.346 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:48:52.346 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:48:52.346 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:48:52.346 | INFO | ------------------------------------------ | 20260508 01:48:52.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:48:52.347 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:48:52.347 | INFO | Total mem usage: 26.5 GiB | 20260508 01:48:52.347 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:48:52.347 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:48:52.347 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:48:52.347 | INFO | High Water Breakdown: | 20260508 01:48:52.347 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:48:52.347 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:48:52.347 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:48:52.347 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:48:52.347 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:48:52.347 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:48:52.347 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:48:52.347 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:48:52.347 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:48:52.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:48:52.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:48:52.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:48:52.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:48:52.347 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:48:52.347 | INFO | ------------------------------------------ | 20260508 01:48:52.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:48:52.347 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:48:52.347 | INFO | Total mem usage: 27.8 GiB | 20260508 01:48:52.347 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:48:52.347 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:48:52.347 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:48:52.347 | INFO | High Water Breakdown: | 20260508 01:48:52.347 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:48:52.347 | INFO | ------------------------------------------ | 20260508 01:48:52.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:48:52.347 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:48:52.347 | INFO | Total mem usage: 15.7 GiB | 20260508 01:48:52.347 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:48:52.347 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:48:52.347 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:48:52.347 | INFO | High Water Breakdown: | 20260508 01:48:52.347 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:48:52.347 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:48:52.347 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:48:52.347 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:48:52.347 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:48:52.347 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:48:52.347 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:48:52.347 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:48:52.347 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:48:52.347 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:48:52.347 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:48:52.347 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:48:52.347 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:48:52.347 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:48:52.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:48:52.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:48:52.347 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:48:52.347 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:48:52.347 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:48:52.347 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:48:52.347 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:48:52.347 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:48:52.347 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:48:52.347 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:48:52.347 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:48:52.347 | INFO | ------------------------------------------ | 20260508 01:48:52.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:48:52.347 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:48:52.347 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:48:52.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:48:52.347 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.38e+01 / 1.24e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.02e-01, 1.14e-02 / 5.62e-03 | 20260508 01:48:52.347 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.43e+00, 5.84e-01 / 1.98e-01 | 20260508 01:48:52.347 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.06e-01 / 8.65e-02 | 20260508 01:48:52.347 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.61e+01, 4.05e+00 / 1.50e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.64e+00 / 1.55e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.25e-01 / 2.34e-01 | 20260508 01:48:52.347 | INFO | PerfReport: 3.32%, Metrics -- 1.08e+00 / 1.26e+00, 2.67e-01 / 1.26e-01 | 20260508 01:48:52.347 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.73e-01, 5.24e-02 / 2.77e-02 | 20260508 01:48:52.347 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:48:52.347 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:48:52.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:48:52.347 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 24.27%, Upload -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 2.06%, LockUpload -- 2.19e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 2.87%, LockDownload -- 3.05e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:48:52.347 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260508 01:48:52.443 | INFO | Starting a round of Baz IO | 20260508 01:48:55.897 | INFO | Populating RTMetrics report | 20260508 01:48:55.898 | INFO | RTMetrics report populated | 20260508 01:48:55.898 | INFO | Opening tmp RTMetrics json file | 20260508 01:48:55.927 | INFO | Serializing RTMetrics to json | 20260508 01:48:55.957 | INFO | Serialization done, swapping tmp file now... | 20260508 01:48:55.978 | INFO | RTMetrics done | 20260508 01:48:56.007 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.564 seconds | 20260508 01:48:56.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.72% / 0.00% / 58.27% | 20260508 01:48:56.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.802966% / 0% / 5.5631e-05% | 20260508 01:48:56.007 | INFO | Baz IO overall: | 20260508 01:48:56.007 | INFO | BytesWritten : 1938994902236 | 20260508 01:48:56.007 | INFO | NumEvents : 937735697740 | 20260508 01:48:57.482 | INFO | Analyzing chunk frames = [4284416,4284928) | 20260508 01:48:57.482 | INFO | MemInfo: { | 20260508 01:48:57.482 | INFO | "gpu" : " 17 GB", | 20260508 01:48:57.482 | INFO | "huge" : "", | 20260508 01:48:57.482 | INFO | "huge_shared" : " 29 GB", | 20260508 01:48:57.482 | INFO | "malloc" : " 14 GB", | 20260508 01:48:57.482 | INFO | "pinned" : " 29 GB" | 20260508 01:48:57.482 | INFO | } | 20260508 01:48:57.483 | INFO | Perf report for the last 5.13668 seconds: | 20260508 01:48:57.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:48:57.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:57.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:57.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:57.483 | INFO | ANALYSIS : 68.5%, 3.0, 106.7ms, 0 | 20260508 01:48:57.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:48:57.483 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260508 01:48:57.483 | INFO | PRE_HQ : 65.5%, 1.7, 56.2ms, 0 | 20260508 01:48:57.483 | INFO | BAZWRITER : 69.0%, 3.6, 128.9ms, 0 | 20260508 01:48:57.483 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 01:48:57.483 | INFO | Uploaded 36GiB to GPU in 2.57353s (13.9886GiB/s) | 20260508 01:48:57.483 | INFO | Downloaded 38GiB to GPU in 2.68114s (14.1731GiB/s) | 20260508 01:49:02.621 | INFO | Analyzing chunk frames = [4284928,4285440) | 20260508 01:49:02.621 | INFO | MemInfo: { | 20260508 01:49:02.621 | INFO | "gpu" : " 17 GB", | 20260508 01:49:02.621 | INFO | "huge" : "", | 20260508 01:49:02.621 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:02.621 | INFO | "malloc" : " 14 GB", | 20260508 01:49:02.621 | INFO | "pinned" : " 29 GB" | 20260508 01:49:02.621 | INFO | } | 20260508 01:49:02.621 | INFO | Perf report for the last 5.13841 seconds: | 20260508 01:49:02.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:02.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 01:49:02.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260508 01:49:02.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:02.621 | INFO | Uploaded 36GiB to GPU in 2.604s (13.8249GiB/s) | 20260508 01:49:02.621 | INFO | Downloaded 34GiB to GPU in 2.48213s (13.6979GiB/s) | 20260508 01:49:07.765 | INFO | Analyzing chunk frames = [4285440,4285952) | 20260508 01:49:07.765 | INFO | MemInfo: { | 20260508 01:49:07.765 | INFO | "gpu" : " 17 GB", | 20260508 01:49:07.765 | INFO | "huge" : "", | 20260508 01:49:07.765 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:07.765 | INFO | "malloc" : " 14 GB", | 20260508 01:49:07.765 | INFO | "pinned" : " 29 GB" | 20260508 01:49:07.765 | INFO | } | 20260508 01:49:07.765 | INFO | Perf report for the last 5.14419 seconds: | 20260508 01:49:07.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:07.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 01:49:07.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:49:07.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:07.765 | INFO | Uploaded 36GiB to GPU in 2.54948s (14.1205GiB/s) | 20260508 01:49:07.765 | INFO | Downloaded 34GiB to GPU in 2.47163s (13.7561GiB/s) | 20260508 01:49:12.909 | INFO | Analyzing chunk frames = [4285952,4286464) | 20260508 01:49:12.909 | INFO | MemInfo: { | 20260508 01:49:12.909 | INFO | "gpu" : " 17 GB", | 20260508 01:49:12.909 | INFO | "huge" : "", | 20260508 01:49:12.909 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:12.909 | INFO | "malloc" : " 14 GB", | 20260508 01:49:12.909 | INFO | "pinned" : " 29 GB" | 20260508 01:49:12.909 | INFO | } | 20260508 01:49:12.909 | INFO | Perf report for the last 5.14383 seconds: | 20260508 01:49:12.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:12.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.909 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:49:12.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.909 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:49:12.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:12.910 | INFO | Uploaded 36GiB to GPU in 2.56391s (14.0411GiB/s) | 20260508 01:49:12.910 | INFO | Downloaded 34GiB to GPU in 2.48493s (13.6825GiB/s) | 20260508 01:49:18.049 | INFO | Analyzing chunk frames = [4286464,4286976) | 20260508 01:49:18.049 | INFO | MemInfo: { | 20260508 01:49:18.049 | INFO | "gpu" : " 17 GB", | 20260508 01:49:18.049 | INFO | "huge" : "", | 20260508 01:49:18.049 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:18.049 | INFO | "malloc" : " 14 GB", | 20260508 01:49:18.049 | INFO | "pinned" : " 29 GB" | 20260508 01:49:18.049 | INFO | } | 20260508 01:49:18.049 | INFO | Perf report for the last 5.13978 seconds: | 20260508 01:49:18.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:18.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 01:49:18.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:49:18.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:18.049 | INFO | Uploaded 36GiB to GPU in 2.40375s (14.9766GiB/s) | 20260508 01:49:18.049 | INFO | Downloaded 34GiB to GPU in 2.51183s (13.536GiB/s) | 20260508 01:49:23.184 | INFO | Analyzing chunk frames = [4286976,4287488) | 20260508 01:49:23.184 | INFO | MemInfo: { | 20260508 01:49:23.184 | INFO | "gpu" : " 17 GB", | 20260508 01:49:23.184 | INFO | "huge" : "", | 20260508 01:49:23.184 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:23.184 | INFO | "malloc" : " 14 GB", | 20260508 01:49:23.184 | INFO | "pinned" : " 29 GB" | 20260508 01:49:23.184 | INFO | } | 20260508 01:49:23.184 | INFO | Perf report for the last 5.13517 seconds: | 20260508 01:49:23.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:23.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.184 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:49:23.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.184 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 01:49:23.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:23.185 | INFO | Uploaded 36GiB to GPU in 2.40298s (14.9814GiB/s) | 20260508 01:49:23.185 | INFO | Downloaded 34GiB to GPU in 2.41379s (14.0857GiB/s) | 20260508 01:49:28.321 | INFO | Analyzing chunk frames = [4287488,4288000) | 20260508 01:49:28.321 | INFO | MemInfo: { | 20260508 01:49:28.321 | INFO | "gpu" : " 17 GB", | 20260508 01:49:28.321 | INFO | "huge" : "", | 20260508 01:49:28.321 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:28.321 | INFO | "malloc" : " 14 GB", | 20260508 01:49:28.321 | INFO | "pinned" : " 29 GB" | 20260508 01:49:28.321 | INFO | } | 20260508 01:49:28.321 | INFO | Perf report for the last 5.13727 seconds: | 20260508 01:49:28.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:28.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:49:28.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:49:28.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:28.321 | INFO | Uploaded 36GiB to GPU in 2.43956s (14.7568GiB/s) | 20260508 01:49:28.321 | INFO | Downloaded 34GiB to GPU in 2.41529s (14.077GiB/s) | 20260508 01:49:33.464 | INFO | Analyzing chunk frames = [4288000,4288512) | 20260508 01:49:33.464 | INFO | MemInfo: { | 20260508 01:49:33.464 | INFO | "gpu" : " 17 GB", | 20260508 01:49:33.464 | INFO | "huge" : "", | 20260508 01:49:33.464 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:33.464 | INFO | "malloc" : " 14 GB", | 20260508 01:49:33.464 | INFO | "pinned" : " 29 GB" | 20260508 01:49:33.464 | INFO | } | 20260508 01:49:33.464 | INFO | Perf report for the last 5.14271 seconds: | 20260508 01:49:33.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:33.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:49:33.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 01:49:33.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:33.464 | INFO | Uploaded 36GiB to GPU in 2.55254s (14.1036GiB/s) | 20260508 01:49:33.464 | INFO | Downloaded 34GiB to GPU in 2.4528s (13.8617GiB/s) | 20260508 01:49:33.561 | INFO | Starting a round of Baz IO | 20260508 01:49:37.004 | INFO | Populating RTMetrics report | 20260508 01:49:37.004 | INFO | RTMetrics report populated | 20260508 01:49:37.004 | INFO | Opening tmp RTMetrics json file | 20260508 01:49:37.014 | INFO | Serializing RTMetrics to json | 20260508 01:49:37.026 | INFO | Serialization done, swapping tmp file now... | 20260508 01:49:37.027 | INFO | RTMetrics done | 20260508 01:49:37.118 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.557 seconds | 20260508 01:49:37.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.72% / 0.00% / 58.28% | 20260508 01:49:37.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.803904% / 3.97364e-06% / 7.15256e-05% | 20260508 01:49:37.118 | INFO | Baz IO overall: | 20260508 01:49:37.118 | INFO | BytesWritten : 1940771353371 | 20260508 01:49:37.118 | INFO | NumEvents : 938588798330 | 20260508 01:49:38.602 | INFO | Analyzing chunk frames = [4288512,4289024) | 20260508 01:49:38.602 | INFO | MemInfo: { | 20260508 01:49:38.602 | INFO | "gpu" : " 17 GB", | 20260508 01:49:38.602 | INFO | "huge" : "", | 20260508 01:49:38.602 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:38.602 | INFO | "malloc" : " 14 GB", | 20260508 01:49:38.602 | INFO | "pinned" : " 29 GB" | 20260508 01:49:38.602 | INFO | } | 20260508 01:49:38.602 | INFO | Perf report for the last 5.13817 seconds: | 20260508 01:49:38.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:38.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:38.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:38.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:38.602 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 01:49:38.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:38.602 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260508 01:49:38.602 | INFO | PRE_HQ : 66.5%, 1.7, 59.9ms, 0 | 20260508 01:49:38.602 | INFO | BAZWRITER : 68.7%, 3.6, 127.9ms, 0 | 20260508 01:49:38.602 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 01:49:38.602 | INFO | Uploaded 36GiB to GPU in 2.5849s (13.927GiB/s) | 20260508 01:49:38.602 | INFO | Downloaded 38GiB to GPU in 2.69777s (14.0857GiB/s) | 20260508 01:49:43.738 | INFO | Analyzing chunk frames = [4289024,4289536) | 20260508 01:49:43.738 | INFO | MemInfo: { | 20260508 01:49:43.738 | INFO | "gpu" : " 17 GB", | 20260508 01:49:43.738 | INFO | "huge" : "", | 20260508 01:49:43.738 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:43.738 | INFO | "malloc" : " 14 GB", | 20260508 01:49:43.738 | INFO | "pinned" : " 29 GB" | 20260508 01:49:43.738 | INFO | } | 20260508 01:49:43.738 | INFO | Perf report for the last 5.13625 seconds: | 20260508 01:49:43.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:43.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.738 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 01:49:43.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.739 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:49:43.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:43.739 | INFO | Uploaded 36GiB to GPU in 2.58037s (13.9515GiB/s) | 20260508 01:49:43.739 | INFO | Downloaded 34GiB to GPU in 2.48988s (13.6553GiB/s) | 20260508 01:49:48.875 | INFO | Analyzing chunk frames = [4289536,4290048) | 20260508 01:49:48.875 | INFO | MemInfo: { | 20260508 01:49:48.875 | INFO | "gpu" : " 17 GB", | 20260508 01:49:48.875 | INFO | "huge" : "", | 20260508 01:49:48.877 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:48.877 | INFO | "malloc" : " 14 GB", | 20260508 01:49:48.877 | INFO | "pinned" : " 29 GB" | 20260508 01:49:48.877 | INFO | } | 20260508 01:49:48.878 | INFO | Perf report for the last 5.1391 seconds: | 20260508 01:49:48.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:48.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:49:48.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:49:48.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:48.878 | INFO | Uploaded 36GiB to GPU in 2.51622s (14.3072GiB/s) | 20260508 01:49:48.878 | INFO | Downloaded 34GiB to GPU in 2.49002s (13.6545GiB/s) | 20260508 01:49:54.019 | INFO | Analyzing chunk frames = [4290048,4290560) | 20260508 01:49:54.019 | INFO | MemInfo: { | 20260508 01:49:54.019 | INFO | "gpu" : " 17 GB", | 20260508 01:49:54.019 | INFO | "huge" : "", | 20260508 01:49:54.019 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:54.019 | INFO | "malloc" : " 14 GB", | 20260508 01:49:54.019 | INFO | "pinned" : " 29 GB" | 20260508 01:49:54.019 | INFO | } | 20260508 01:49:54.019 | INFO | Perf report for the last 5.14139 seconds: | 20260508 01:49:54.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:54.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:49:54.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:49:54.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:54.019 | INFO | Uploaded 36GiB to GPU in 2.53752s (14.1871GiB/s) | 20260508 01:49:54.019 | INFO | Downloaded 34GiB to GPU in 2.48996s (13.6548GiB/s) | 20260508 01:49:59.161 | INFO | Analyzing chunk frames = [4290560,4291072) | 20260508 01:49:59.161 | INFO | MemInfo: { | 20260508 01:49:59.161 | INFO | "gpu" : " 17 GB", | 20260508 01:49:59.161 | INFO | "huge" : "", | 20260508 01:49:59.162 | INFO | "huge_shared" : " 29 GB", | 20260508 01:49:59.162 | INFO | "malloc" : " 14 GB", | 20260508 01:49:59.162 | INFO | "pinned" : " 29 GB" | 20260508 01:49:59.162 | INFO | } | 20260508 01:49:59.162 | INFO | Perf report for the last 5.1427 seconds: | 20260508 01:49:59.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:49:59.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:49:59.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:49:59.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:49:59.162 | INFO | Uploaded 36GiB to GPU in 2.33483s (15.4187GiB/s) | 20260508 01:49:59.162 | INFO | Downloaded 34GiB to GPU in 2.49453s (13.6298GiB/s) | 20260508 01:50:04.297 | INFO | Analyzing chunk frames = [4291072,4291584) | 20260508 01:50:04.297 | INFO | MemInfo: { | 20260508 01:50:04.297 | INFO | "gpu" : " 17 GB", | 20260508 01:50:04.298 | INFO | "huge" : "", | 20260508 01:50:04.298 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:04.298 | INFO | "malloc" : " 14 GB", | 20260508 01:50:04.298 | INFO | "pinned" : " 29 GB" | 20260508 01:50:04.298 | INFO | } | 20260508 01:50:04.298 | INFO | Perf report for the last 5.13606 seconds: | 20260508 01:50:04.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:04.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 01:50:04.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:50:04.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:04.298 | INFO | Uploaded 36GiB to GPU in 2.41297s (14.9194GiB/s) | 20260508 01:50:04.298 | INFO | Downloaded 34GiB to GPU in 2.4068s (14.1267GiB/s) | 20260508 01:50:09.436 | INFO | Analyzing chunk frames = [4291584,4292096) | 20260508 01:50:09.436 | INFO | MemInfo: { | 20260508 01:50:09.436 | INFO | "gpu" : " 17 GB", | 20260508 01:50:09.436 | INFO | "huge" : "", | 20260508 01:50:09.436 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:09.436 | INFO | "malloc" : " 14 GB", | 20260508 01:50:09.436 | INFO | "pinned" : " 29 GB" | 20260508 01:50:09.436 | INFO | } | 20260508 01:50:09.436 | INFO | Perf report for the last 5.13879 seconds: | 20260508 01:50:09.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:09.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 01:50:09.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:50:09.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:09.437 | INFO | Uploaded 36GiB to GPU in 2.46779s (14.588GiB/s) | 20260508 01:50:09.437 | INFO | Downloaded 34GiB to GPU in 2.41626s (14.0713GiB/s) | 20260508 01:50:14.576 | INFO | Analyzing chunk frames = [4292096,4292608) | 20260508 01:50:14.576 | INFO | MemInfo: { | 20260508 01:50:14.576 | INFO | "gpu" : " 17 GB", | 20260508 01:50:14.576 | INFO | "huge" : "", | 20260508 01:50:14.576 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:14.576 | INFO | "malloc" : " 14 GB", | 20260508 01:50:14.576 | INFO | "pinned" : " 29 GB" | 20260508 01:50:14.576 | INFO | } | 20260508 01:50:14.576 | INFO | Perf report for the last 5.13922 seconds: | 20260508 01:50:14.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:14.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:50:14.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:50:14.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:14.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:50:14.576 | INFO | Total mem usage: 16.7 GiB | 20260508 01:50:14.576 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:50:14.576 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.1%) | 20260508 01:50:14.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:50:14.576 | INFO | High Water Breakdown: | 20260508 01:50:14.576 | INFO | 59.7% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:50:14.576 | INFO | 40.2% : 5.82 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:50:14.576 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:50:14.576 | INFO | ------------------------------------------ | 20260508 01:50:14.576 | INFO | Uploaded 36GiB to GPU in 2.5676s (14.0209GiB/s) | 20260508 01:50:14.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:50:14.576 | INFO | Downloaded 34GiB to GPU in 2.44867s (13.8851GiB/s) | 20260508 01:50:14.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:50:14.576 | INFO | Total mem usage: 26.5 GiB | 20260508 01:50:14.576 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:50:14.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:50:14.577 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:50:14.577 | INFO | High Water Breakdown: | 20260508 01:50:14.577 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:50:14.577 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:50:14.577 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:50:14.577 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:50:14.577 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:50:14.577 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:50:14.577 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:50:14.577 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:50:14.577 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:50:14.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:50:14.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:50:14.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:50:14.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:50:14.577 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:50:14.577 | INFO | ------------------------------------------ | 20260508 01:50:14.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:50:14.577 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:50:14.577 | INFO | Total mem usage: 27.8 GiB | 20260508 01:50:14.577 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:50:14.577 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:50:14.577 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:50:14.577 | INFO | High Water Breakdown: | 20260508 01:50:14.577 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:50:14.577 | INFO | ------------------------------------------ | 20260508 01:50:14.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:50:14.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:50:14.577 | INFO | Total mem usage: 15.7 GiB | 20260508 01:50:14.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:50:14.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:50:14.577 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:50:14.577 | INFO | High Water Breakdown: | 20260508 01:50:14.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:50:14.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:50:14.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:50:14.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:50:14.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:50:14.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:50:14.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:50:14.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:50:14.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:50:14.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:50:14.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:50:14.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:50:14.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:50:14.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:50:14.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:50:14.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:50:14.577 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:50:14.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:50:14.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:50:14.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:50:14.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:50:14.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:50:14.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:50:14.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:50:14.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:50:14.577 | INFO | ------------------------------------------ | 20260508 01:50:14.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:50:14.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:50:14.577 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:50:14.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:50:14.577 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.42e+01 / 1.18e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.01e-01, 1.26e-02 / 5.72e-03 | 20260508 01:50:14.577 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.52e+00, 5.84e-01 / 1.34e-01 | 20260508 01:50:14.577 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.03e-01 / 1.05e-01 | 20260508 01:50:14.577 | INFO | PerfReport: 16.12%, DME -- 4.21e+01 / 4.60e+01, 4.05e+00 / 1.44e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 56.25%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.56e+00 / 1.52e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 2.73e+00, 4.26e-01 / 2.04e-01 | 20260508 01:50:14.577 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.29e+00, 2.67e-01 / 1.42e-01 | 20260508 01:50:14.577 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.69e-01, 5.27e-02 / 2.66e-02 | 20260508 01:50:14.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:50:14.577 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:50:14.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:50:14.577 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 30.63%, Compute -- 3.26e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 2.37%, LockUpload -- 2.52e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 2.64%, LockDownload -- 2.82e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 01:50:14.577 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260508 01:50:14.672 | INFO | Starting a round of Baz IO | 20260508 01:50:18.123 | INFO | Populating RTMetrics report | 20260508 01:50:18.123 | INFO | RTMetrics report populated | 20260508 01:50:18.123 | INFO | Opening tmp RTMetrics json file | 20260508 01:50:18.145 | INFO | Serializing RTMetrics to json | 20260508 01:50:18.147 | INFO | Serialization done, swapping tmp file now... | 20260508 01:50:18.148 | INFO | RTMetrics done | 20260508 01:50:18.216 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.543 seconds | 20260508 01:50:18.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.72% / 0.00% / 58.28% | 20260508 01:50:18.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.800331% / 3.97364e-06% / 5.16574e-05% | 20260508 01:50:18.216 | INFO | Baz IO overall: | 20260508 01:50:18.216 | INFO | BytesWritten : 1942546814144 | 20260508 01:50:18.216 | INFO | NumEvents : 939441009865 | 20260508 01:50:19.716 | INFO | Analyzing chunk frames = [4292608,4293120) | 20260508 01:50:19.716 | INFO | MemInfo: { | 20260508 01:50:19.716 | INFO | "gpu" : " 17 GB", | 20260508 01:50:19.716 | INFO | "huge" : "", | 20260508 01:50:19.716 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:19.716 | INFO | "malloc" : " 14 GB", | 20260508 01:50:19.716 | INFO | "pinned" : " 29 GB" | 20260508 01:50:19.716 | INFO | } | 20260508 01:50:19.716 | INFO | Perf report for the last 5.14009 seconds: | 20260508 01:50:19.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:19.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:19.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:19.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:19.716 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260508 01:50:19.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:19.716 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 01:50:19.716 | INFO | PRE_HQ : 65.3%, 1.6, 55.9ms, 0 | 20260508 01:50:19.716 | INFO | BAZWRITER : 68.5%, 3.6, 126.9ms, 0 | 20260508 01:50:19.716 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 01:50:19.716 | INFO | Uploaded 36GiB to GPU in 2.57067s (14.0041GiB/s) | 20260508 01:50:19.716 | INFO | Downloaded 38GiB to GPU in 2.68607s (14.1471GiB/s) | 20260508 01:50:24.859 | INFO | Analyzing chunk frames = [4293120,4293632) | 20260508 01:50:24.859 | INFO | MemInfo: { | 20260508 01:50:24.859 | INFO | "gpu" : " 17 GB", | 20260508 01:50:24.859 | INFO | "huge" : "", | 20260508 01:50:24.859 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:24.859 | INFO | "malloc" : " 14 GB", | 20260508 01:50:24.859 | INFO | "pinned" : " 29 GB" | 20260508 01:50:24.859 | INFO | } | 20260508 01:50:24.860 | INFO | Perf report for the last 5.14369 seconds: | 20260508 01:50:24.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:24.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 01:50:24.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 01:50:24.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:24.860 | INFO | Uploaded 36GiB to GPU in 2.55879s (14.0692GiB/s) | 20260508 01:50:24.860 | INFO | Downloaded 34GiB to GPU in 2.4209s (14.0443GiB/s) | 20260508 01:50:30.001 | INFO | Analyzing chunk frames = [4293632,4294144) | 20260508 01:50:30.001 | INFO | MemInfo: { | 20260508 01:50:30.001 | INFO | "gpu" : " 17 GB", | 20260508 01:50:30.001 | INFO | "huge" : "", | 20260508 01:50:30.001 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:30.001 | INFO | "malloc" : " 14 GB", | 20260508 01:50:30.001 | INFO | "pinned" : " 29 GB" | 20260508 01:50:30.001 | INFO | } | 20260508 01:50:30.001 | INFO | Perf report for the last 5.14172 seconds: | 20260508 01:50:30.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:30.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.001 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 01:50:30.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.001 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:50:30.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:30.002 | INFO | Uploaded 36GiB to GPU in 2.58109s (13.9476GiB/s) | 20260508 01:50:30.002 | INFO | Downloaded 34GiB to GPU in 2.4914s (13.647GiB/s) | 20260508 01:50:35.131 | INFO | Analyzing chunk frames = [4294144,4294656) | 20260508 01:50:35.131 | INFO | MemInfo: { | 20260508 01:50:35.131 | INFO | "gpu" : " 17 GB", | 20260508 01:50:35.131 | INFO | "huge" : "", | 20260508 01:50:35.131 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:35.131 | INFO | "malloc" : " 14 GB", | 20260508 01:50:35.131 | INFO | "pinned" : " 29 GB" | 20260508 01:50:35.131 | INFO | } | 20260508 01:50:35.131 | INFO | Perf report for the last 5.12954 seconds: | 20260508 01:50:35.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:35.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 01:50:35.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:50:35.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:35.131 | INFO | Uploaded 36GiB to GPU in 2.56192s (14.052GiB/s) | 20260508 01:50:35.131 | INFO | Downloaded 34GiB to GPU in 2.48786s (13.6664GiB/s) | 20260508 01:50:40.274 | INFO | Analyzing chunk frames = [4294656,4295168) | 20260508 01:50:40.274 | INFO | MemInfo: { | 20260508 01:50:40.274 | INFO | "gpu" : " 17 GB", | 20260508 01:50:40.274 | INFO | "huge" : "", | 20260508 01:50:40.274 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:40.274 | INFO | "malloc" : " 14 GB", | 20260508 01:50:40.274 | INFO | "pinned" : " 29 GB" | 20260508 01:50:40.274 | INFO | } | 20260508 01:50:40.274 | INFO | Perf report for the last 5.14299 seconds: | 20260508 01:50:40.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:40.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 01:50:40.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 01:50:40.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:40.274 | INFO | Uploaded 36GiB to GPU in 2.37537s (15.1555GiB/s) | 20260508 01:50:40.274 | INFO | Downloaded 34GiB to GPU in 2.52284s (13.4769GiB/s) | 20260508 01:50:45.414 | INFO | Analyzing chunk frames = [4295168,4295680) | 20260508 01:50:45.414 | INFO | MemInfo: { | 20260508 01:50:45.414 | INFO | "gpu" : " 17 GB", | 20260508 01:50:45.415 | INFO | "huge" : "", | 20260508 01:50:45.415 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:45.415 | INFO | "malloc" : " 14 GB", | 20260508 01:50:45.415 | INFO | "pinned" : " 29 GB" | 20260508 01:50:45.415 | INFO | } | 20260508 01:50:45.415 | INFO | Perf report for the last 5.14093 seconds: | 20260508 01:50:45.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:45.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.415 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 01:50:45.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.415 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 01:50:45.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:45.416 | INFO | Uploaded 36GiB to GPU in 2.4392s (14.7589GiB/s) | 20260508 01:50:45.416 | INFO | Downloaded 34GiB to GPU in 2.40939s (14.1115GiB/s) | 20260508 01:50:50.554 | INFO | Analyzing chunk frames = [4295680,4296192) | 20260508 01:50:50.554 | INFO | MemInfo: { | 20260508 01:50:50.554 | INFO | "gpu" : " 17 GB", | 20260508 01:50:50.554 | INFO | "huge" : "", | 20260508 01:50:50.555 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:50.555 | INFO | "malloc" : " 14 GB", | 20260508 01:50:50.555 | INFO | "pinned" : " 29 GB" | 20260508 01:50:50.555 | INFO | } | 20260508 01:50:50.555 | INFO | Perf report for the last 5.13993 seconds: | 20260508 01:50:50.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:50.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 01:50:50.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 01:50:50.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:50.555 | INFO | Uploaded 36GiB to GPU in 2.47326s (14.5557GiB/s) | 20260508 01:50:50.555 | INFO | Downloaded 34GiB to GPU in 2.42171s (14.0397GiB/s) | 20260508 01:50:55.689 | INFO | Analyzing chunk frames = [4296192,4296704) | 20260508 01:50:55.689 | INFO | MemInfo: { | 20260508 01:50:55.689 | INFO | "gpu" : " 17 GB", | 20260508 01:50:55.689 | INFO | "huge" : "", | 20260508 01:50:55.689 | INFO | "huge_shared" : " 29 GB", | 20260508 01:50:55.689 | INFO | "malloc" : " 14 GB", | 20260508 01:50:55.689 | INFO | "pinned" : " 29 GB" | 20260508 01:50:55.689 | INFO | } | 20260508 01:50:55.690 | INFO | Perf report for the last 5.1348 seconds: | 20260508 01:50:55.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:50:55.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 01:50:55.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 01:50:55.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:50:55.690 | INFO | Uploaded 36GiB to GPU in 2.49041s (14.4554GiB/s) | 20260508 01:50:55.690 | INFO | Downloaded 34GiB to GPU in 2.42763s (14.0054GiB/s) | 20260508 01:50:55.801 | INFO | Starting a round of Baz IO | 20260508 01:50:59.263 | INFO | Populating RTMetrics report | 20260508 01:50:59.263 | INFO | RTMetrics report populated | 20260508 01:50:59.263 | INFO | Opening tmp RTMetrics json file | 20260508 01:50:59.265 | INFO | Serializing RTMetrics to json | 20260508 01:50:59.269 | INFO | Serialization done, swapping tmp file now... | 20260508 01:50:59.269 | INFO | RTMetrics done | 20260508 01:50:59.367 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.565 seconds | 20260508 01:50:59.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.71% / 0.00% / 58.28% | 20260508 01:50:59.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.799779% / 1.19209e-05% / 5.5631e-05% | 20260508 01:50:59.367 | INFO | Baz IO overall: | 20260508 01:50:59.367 | INFO | BytesWritten : 1944321703316 | 20260508 01:50:59.367 | INFO | NumEvents : 940292977461 | 20260508 01:51:00.828 | INFO | Analyzing chunk frames = [4296704,4297216) | 20260508 01:51:00.828 | INFO | MemInfo: { | 20260508 01:51:00.828 | INFO | "gpu" : " 17 GB", | 20260508 01:51:00.828 | INFO | "huge" : "", | 20260508 01:51:00.828 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:00.828 | INFO | "malloc" : " 14 GB", | 20260508 01:51:00.828 | INFO | "pinned" : " 29 GB" | 20260508 01:51:00.828 | INFO | } | 20260508 01:51:00.828 | INFO | Perf report for the last 5.13859 seconds: | 20260508 01:51:00.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:00.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:00.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:00.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:00.828 | INFO | ANALYSIS : 69.1%, 2.9, 103.9ms, 0 | 20260508 01:51:00.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:00.828 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260508 01:51:00.828 | INFO | PRE_HQ : 68.1%, 1.8, 63.9ms, 0 | 20260508 01:51:00.828 | INFO | BAZWRITER : 69.0%, 4.3, 153.0ms, 0 | 20260508 01:51:00.828 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 01:51:00.829 | INFO | Uploaded 36GiB to GPU in 2.55271s (14.1027GiB/s) | 20260508 01:51:00.829 | INFO | Downloaded 38GiB to GPU in 2.66924s (14.2362GiB/s) | 20260508 01:51:05.968 | INFO | Analyzing chunk frames = [4297216,4297728) | 20260508 01:51:05.968 | INFO | MemInfo: { | 20260508 01:51:05.968 | INFO | "gpu" : " 17 GB", | 20260508 01:51:05.968 | INFO | "huge" : "", | 20260508 01:51:05.968 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:05.968 | INFO | "malloc" : " 14 GB", | 20260508 01:51:05.968 | INFO | "pinned" : " 29 GB" | 20260508 01:51:05.968 | INFO | } | 20260508 01:51:05.968 | INFO | Perf report for the last 5.14017 seconds: | 20260508 01:51:05.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:05.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.968 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 01:51:05.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.968 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:51:05.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:05.969 | INFO | Uploaded 36GiB to GPU in 2.56907s (14.0129GiB/s) | 20260508 01:51:05.969 | INFO | Downloaded 34GiB to GPU in 2.42331s (14.0304GiB/s) | 20260508 01:51:11.110 | INFO | Analyzing chunk frames = [4297728,4298240) | 20260508 01:51:11.110 | INFO | MemInfo: { | 20260508 01:51:11.110 | INFO | "gpu" : " 17 GB", | 20260508 01:51:11.110 | INFO | "huge" : "", | 20260508 01:51:11.110 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:11.110 | INFO | "malloc" : " 14 GB", | 20260508 01:51:11.110 | INFO | "pinned" : " 29 GB" | 20260508 01:51:11.110 | INFO | } | 20260508 01:51:11.110 | INFO | Perf report for the last 5.14198 seconds: | 20260508 01:51:11.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:11.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.110 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 01:51:11.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.110 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:51:11.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:11.111 | INFO | Uploaded 36GiB to GPU in 2.55653s (14.0816GiB/s) | 20260508 01:51:11.111 | INFO | Downloaded 34GiB to GPU in 2.48136s (13.7022GiB/s) | 20260508 01:51:16.248 | INFO | Analyzing chunk frames = [4298240,4298752) | 20260508 01:51:16.248 | INFO | MemInfo: { | 20260508 01:51:16.248 | INFO | "gpu" : " 17 GB", | 20260508 01:51:16.248 | INFO | "huge" : "", | 20260508 01:51:16.248 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:16.248 | INFO | "malloc" : " 14 GB", | 20260508 01:51:16.248 | INFO | "pinned" : " 29 GB" | 20260508 01:51:16.248 | INFO | } | 20260508 01:51:16.248 | INFO | Perf report for the last 5.13809 seconds: | 20260508 01:51:16.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:16.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.248 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 01:51:16.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.249 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:51:16.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:16.249 | INFO | Uploaded 36GiB to GPU in 2.55963s (14.0646GiB/s) | 20260508 01:51:16.249 | INFO | Downloaded 34GiB to GPU in 2.47798s (13.7209GiB/s) | 20260508 01:51:21.384 | INFO | Analyzing chunk frames = [4298752,4299264) | 20260508 01:51:21.384 | INFO | MemInfo: { | 20260508 01:51:21.384 | INFO | "gpu" : " 17 GB", | 20260508 01:51:21.384 | INFO | "huge" : "", | 20260508 01:51:21.384 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:21.384 | INFO | "malloc" : " 14 GB", | 20260508 01:51:21.384 | INFO | "pinned" : " 29 GB" | 20260508 01:51:21.384 | INFO | } | 20260508 01:51:21.385 | INFO | Perf report for the last 5.13607 seconds: | 20260508 01:51:21.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:21.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 01:51:21.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:51:21.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:21.385 | INFO | Uploaded 36GiB to GPU in 2.39534s (15.0292GiB/s) | 20260508 01:51:21.385 | INFO | Downloaded 34GiB to GPU in 2.51033s (13.544GiB/s) | 20260508 01:51:26.528 | INFO | Analyzing chunk frames = [4299264,4299776) | 20260508 01:51:26.528 | INFO | MemInfo: { | 20260508 01:51:26.528 | INFO | "gpu" : " 17 GB", | 20260508 01:51:26.528 | INFO | "huge" : "", | 20260508 01:51:26.528 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:26.529 | INFO | "malloc" : " 14 GB", | 20260508 01:51:26.529 | INFO | "pinned" : " 29 GB" | 20260508 01:51:26.529 | INFO | } | 20260508 01:51:26.529 | INFO | Perf report for the last 5.14408 seconds: | 20260508 01:51:26.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:26.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:51:26.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:51:26.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:26.529 | INFO | Uploaded 36GiB to GPU in 2.42304s (14.8574GiB/s) | 20260508 01:51:26.529 | INFO | Downloaded 34GiB to GPU in 2.40567s (14.1333GiB/s) | 20260508 01:51:31.666 | INFO | Analyzing chunk frames = [4299776,4300288) | 20260508 01:51:31.666 | INFO | MemInfo: { | 20260508 01:51:31.666 | INFO | "gpu" : " 17 GB", | 20260508 01:51:31.666 | INFO | "huge" : "", | 20260508 01:51:31.666 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:31.666 | INFO | "malloc" : " 14 GB", | 20260508 01:51:31.666 | INFO | "pinned" : " 29 GB" | 20260508 01:51:31.666 | INFO | } | 20260508 01:51:31.666 | INFO | Perf report for the last 5.13756 seconds: | 20260508 01:51:31.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:31.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 01:51:31.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:51:31.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:31.666 | INFO | Uploaded 36GiB to GPU in 2.42918s (14.8198GiB/s) | 20260508 01:51:31.667 | INFO | Downloaded 34GiB to GPU in 2.41927s (14.0538GiB/s) | 20260508 01:51:36.807 | INFO | Analyzing chunk frames = [4300288,4300800) | 20260508 01:51:36.807 | INFO | MemInfo: { | 20260508 01:51:36.807 | INFO | "gpu" : " 17 GB", | 20260508 01:51:36.807 | INFO | "huge" : "", | 20260508 01:51:36.807 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:36.808 | INFO | "malloc" : " 14 GB", | 20260508 01:51:36.808 | INFO | "pinned" : " 29 GB" | 20260508 01:51:36.808 | INFO | } | 20260508 01:51:36.808 | INFO | Perf report for the last 5.14138 seconds: | 20260508 01:51:36.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:36.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:51:36.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 01:51:36.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:36.808 | INFO | Uploaded 36GiB to GPU in 2.55627s (14.083GiB/s) | 20260508 01:51:36.808 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:51:36.808 | INFO | Downloaded 34GiB to GPU in 2.45898s (13.8268GiB/s) | 20260508 01:51:36.808 | INFO | Total mem usage: 16.7 GiB | 20260508 01:51:36.808 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:51:36.808 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.81%) | 20260508 01:51:36.808 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:51:36.808 | INFO | High Water Breakdown: | 20260508 01:51:36.808 | INFO | 60.8% : 9.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:51:36.808 | INFO | 39.1% : 6.05 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:51:36.809 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:51:36.809 | INFO | ------------------------------------------ | 20260508 01:51:36.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:51:36.809 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:51:36.809 | INFO | Total mem usage: 26.5 GiB | 20260508 01:51:36.809 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:51:36.809 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:51:36.809 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:51:36.809 | INFO | High Water Breakdown: | 20260508 01:51:36.809 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:51:36.809 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:51:36.809 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:51:36.809 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:51:36.809 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:51:36.809 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:51:36.809 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:51:36.809 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:51:36.809 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:51:36.809 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:51:36.809 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:51:36.809 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:51:36.809 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:51:36.809 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:51:36.809 | INFO | ------------------------------------------ | 20260508 01:51:36.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:51:36.809 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:51:36.809 | INFO | Total mem usage: 27.8 GiB | 20260508 01:51:36.809 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:51:36.809 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:51:36.809 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:51:36.809 | INFO | High Water Breakdown: | 20260508 01:51:36.809 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:51:36.809 | INFO | ------------------------------------------ | 20260508 01:51:36.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:51:36.809 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:51:36.809 | INFO | Total mem usage: 15.7 GiB | 20260508 01:51:36.809 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:51:36.809 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:51:36.809 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:51:36.809 | INFO | High Water Breakdown: | 20260508 01:51:36.809 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:51:36.809 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:51:36.809 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:51:36.809 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:51:36.809 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:51:36.809 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:51:36.809 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:51:36.809 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:51:36.809 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:51:36.809 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:51:36.809 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:51:36.809 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:51:36.809 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:51:36.809 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:51:36.809 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:51:36.809 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:51:36.809 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:51:36.809 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:51:36.809 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:51:36.809 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:51:36.809 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:51:36.809 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:51:36.809 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:51:36.809 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:51:36.809 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:51:36.810 | INFO | ------------------------------------------ | 20260508 01:51:36.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:51:36.810 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:51:36.810 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:51:36.810 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:51:36.810 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.41e+01 / 1.62e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.18e-02 / 5.36e-03 | 20260508 01:51:36.810 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.45e+00, 5.91e-01 / 2.48e-01 | 20260508 01:51:36.810 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 9.42e-02 | 20260508 01:51:36.810 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.60e+01, 4.16e+00 / 1.93e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.62e+00 / 1.31e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 2.71e+00, 4.23e-01 / 1.97e-01 | 20260508 01:51:36.810 | INFO | PerfReport: 3.32%, Metrics -- 1.09e+00 / 1.25e+00, 2.71e-01 / 1.33e-01 | 20260508 01:51:36.810 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.71e-01, 5.28e-02 / 2.54e-02 | 20260508 01:51:36.810 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:51:36.810 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:51:36.810 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:51:36.810 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 24.52%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 2.54%, LockDownload -- 2.70e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:51:36.810 | INFO | PerfReport: 0.01%, Misc -- 1.55e-02 / 0.00e+00, 2.81e-02 / 0.00e+00 | 20260508 01:51:36.921 | INFO | Starting a round of Baz IO | 20260508 01:51:40.376 | INFO | Populating RTMetrics report | 20260508 01:51:40.377 | INFO | RTMetrics report populated | 20260508 01:51:40.377 | INFO | Opening tmp RTMetrics json file | 20260508 01:51:40.409 | INFO | Serializing RTMetrics to json | 20260508 01:51:40.465 | INFO | Serialization done, swapping tmp file now... | 20260508 01:51:40.471 | INFO | RTMetrics done | 20260508 01:51:40.499 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.578 seconds | 20260508 01:51:40.499 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.71% / 0.00% / 58.29% | 20260508 01:51:40.499 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.797474% / 7.94729e-06% / 5.96046e-05% | 20260508 01:51:40.499 | INFO | Baz IO overall: | 20260508 01:51:40.499 | INFO | BytesWritten : 1946095348156 | 20260508 01:51:40.499 | INFO | NumEvents : 941143912308 | 20260508 01:51:41.945 | INFO | Analyzing chunk frames = [4300800,4301312) | 20260508 01:51:41.945 | INFO | MemInfo: { | 20260508 01:51:41.945 | INFO | "gpu" : " 17 GB", | 20260508 01:51:41.945 | INFO | "huge" : "", | 20260508 01:51:41.945 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:41.945 | INFO | "malloc" : " 14 GB", | 20260508 01:51:41.945 | INFO | "pinned" : " 29 GB" | 20260508 01:51:41.945 | INFO | } | 20260508 01:51:41.945 | INFO | Perf report for the last 5.13701 seconds: | 20260508 01:51:41.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:41.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:41.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:41.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:41.945 | INFO | ANALYSIS : 68.6%, 2.9, 104.8ms, 0 | 20260508 01:51:41.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:41.945 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260508 01:51:41.945 | INFO | PRE_HQ : 67.4%, 1.8, 62.2ms, 0 | 20260508 01:51:41.945 | INFO | BAZWRITER : 69.3%, 3.6, 127.8ms, 0 | 20260508 01:51:41.945 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 01:51:41.945 | INFO | Uploaded 36GiB to GPU in 2.58313s (13.9366GiB/s) | 20260508 01:51:41.945 | INFO | Downloaded 38GiB to GPU in 2.70781s (14.0335GiB/s) | 20260508 01:51:47.085 | INFO | Analyzing chunk frames = [4301312,4301824) | 20260508 01:51:47.085 | INFO | MemInfo: { | 20260508 01:51:47.085 | INFO | "gpu" : " 17 GB", | 20260508 01:51:47.085 | INFO | "huge" : "", | 20260508 01:51:47.085 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:47.085 | INFO | "malloc" : " 14 GB", | 20260508 01:51:47.085 | INFO | "pinned" : " 29 GB" | 20260508 01:51:47.085 | INFO | } | 20260508 01:51:47.085 | INFO | Perf report for the last 5.14061 seconds: | 20260508 01:51:47.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:47.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 01:51:47.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:51:47.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:47.085 | INFO | Uploaded 36GiB to GPU in 2.52252s (14.2714GiB/s) | 20260508 01:51:47.085 | INFO | Downloaded 34GiB to GPU in 2.43374s (13.9703GiB/s) | 20260508 01:51:52.226 | INFO | Analyzing chunk frames = [4301824,4302336) | 20260508 01:51:52.226 | INFO | MemInfo: { | 20260508 01:51:52.226 | INFO | "gpu" : " 17 GB", | 20260508 01:51:52.226 | INFO | "huge" : "", | 20260508 01:51:52.226 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:52.226 | INFO | "malloc" : " 14 GB", | 20260508 01:51:52.226 | INFO | "pinned" : " 29 GB" | 20260508 01:51:52.226 | INFO | } | 20260508 01:51:52.226 | INFO | Perf report for the last 5.14113 seconds: | 20260508 01:51:52.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:52.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.226 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:51:52.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.227 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:51:52.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:52.227 | INFO | Uploaded 36GiB to GPU in 2.53496s (14.2014GiB/s) | 20260508 01:51:52.227 | INFO | Downloaded 34GiB to GPU in 2.50616s (13.5666GiB/s) | 20260508 01:51:57.366 | INFO | Analyzing chunk frames = [4302336,4302848) | 20260508 01:51:57.366 | INFO | MemInfo: { | 20260508 01:51:57.366 | INFO | "gpu" : " 17 GB", | 20260508 01:51:57.366 | INFO | "huge" : "", | 20260508 01:51:57.366 | INFO | "huge_shared" : " 29 GB", | 20260508 01:51:57.366 | INFO | "malloc" : " 14 GB", | 20260508 01:51:57.366 | INFO | "pinned" : " 29 GB" | 20260508 01:51:57.366 | INFO | } | 20260508 01:51:57.366 | INFO | Perf report for the last 5.14011 seconds: | 20260508 01:51:57.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:51:57.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:51:57.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:51:57.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:51:57.367 | INFO | Uploaded 36GiB to GPU in 2.47479s (14.5467GiB/s) | 20260508 01:51:57.367 | INFO | Downloaded 34GiB to GPU in 2.51796s (13.503GiB/s) | 20260508 01:52:02.501 | INFO | Analyzing chunk frames = [4302848,4303360) | 20260508 01:52:02.501 | INFO | MemInfo: { | 20260508 01:52:02.501 | INFO | "gpu" : " 17 GB", | 20260508 01:52:02.501 | INFO | "huge" : "", | 20260508 01:52:02.501 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:02.501 | INFO | "malloc" : " 14 GB", | 20260508 01:52:02.501 | INFO | "pinned" : " 29 GB" | 20260508 01:52:02.501 | INFO | } | 20260508 01:52:02.501 | INFO | Perf report for the last 5.13477 seconds: | 20260508 01:52:02.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:02.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.501 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 01:52:02.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.501 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:52:02.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:02.502 | INFO | Uploaded 36GiB to GPU in 2.32411s (15.4898GiB/s) | 20260508 01:52:02.502 | INFO | Downloaded 34GiB to GPU in 2.50128s (13.593GiB/s) | 20260508 01:52:07.636 | INFO | Analyzing chunk frames = [4303360,4303872) | 20260508 01:52:07.636 | INFO | MemInfo: { | 20260508 01:52:07.636 | INFO | "gpu" : " 17 GB", | 20260508 01:52:07.636 | INFO | "huge" : "", | 20260508 01:52:07.636 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:07.637 | INFO | "malloc" : " 14 GB", | 20260508 01:52:07.637 | INFO | "pinned" : " 29 GB" | 20260508 01:52:07.637 | INFO | } | 20260508 01:52:07.637 | INFO | Perf report for the last 5.13538 seconds: | 20260508 01:52:07.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:07.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 01:52:07.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:52:07.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:07.637 | INFO | Uploaded 36GiB to GPU in 2.36631s (15.2135GiB/s) | 20260508 01:52:07.637 | INFO | Downloaded 34GiB to GPU in 2.43556s (13.9598GiB/s) | 20260508 01:52:12.781 | INFO | Analyzing chunk frames = [4303872,4304384) | 20260508 01:52:12.781 | INFO | MemInfo: { | 20260508 01:52:12.781 | INFO | "gpu" : " 17 GB", | 20260508 01:52:12.781 | INFO | "huge" : "", | 20260508 01:52:12.781 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:12.782 | INFO | "malloc" : " 14 GB", | 20260508 01:52:12.782 | INFO | "pinned" : " 29 GB" | 20260508 01:52:12.782 | INFO | } | 20260508 01:52:12.782 | INFO | Perf report for the last 5.14492 seconds: | 20260508 01:52:12.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:12.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 01:52:12.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 01:52:12.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:12.782 | INFO | Uploaded 36GiB to GPU in 2.40564s (14.9648GiB/s) | 20260508 01:52:12.782 | INFO | Downloaded 34GiB to GPU in 2.43379s (13.97GiB/s) | 20260508 01:52:17.922 | INFO | Analyzing chunk frames = [4304384,4304896) | 20260508 01:52:17.923 | INFO | MemInfo: { | 20260508 01:52:17.923 | INFO | "gpu" : " 17 GB", | 20260508 01:52:17.923 | INFO | "huge" : "", | 20260508 01:52:17.923 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:17.923 | INFO | "malloc" : " 14 GB", | 20260508 01:52:17.923 | INFO | "pinned" : " 29 GB" | 20260508 01:52:17.923 | INFO | } | 20260508 01:52:17.923 | INFO | Perf report for the last 5.14115 seconds: | 20260508 01:52:17.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:17.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 01:52:17.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:52:17.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:17.923 | INFO | Uploaded 36GiB to GPU in 2.52586s (14.2526GiB/s) | 20260508 01:52:17.923 | INFO | Downloaded 34GiB to GPU in 2.47053s (13.7623GiB/s) | 20260508 01:52:18.020 | INFO | Starting a round of Baz IO | 20260508 01:52:21.479 | INFO | Populating RTMetrics report | 20260508 01:52:21.479 | INFO | RTMetrics report populated | 20260508 01:52:21.479 | INFO | Opening tmp RTMetrics json file | 20260508 01:52:21.483 | INFO | Serializing RTMetrics to json | 20260508 01:52:21.507 | INFO | Serialization done, swapping tmp file now... | 20260508 01:52:21.509 | INFO | RTMetrics done | 20260508 01:52:21.578 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.558 seconds | 20260508 01:52:21.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.70% / 0.00% / 58.29% | 20260508 01:52:21.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.795035% / 0% / 5.5631e-05% | 20260508 01:52:21.578 | INFO | Baz IO overall: | 20260508 01:52:21.578 | INFO | BytesWritten : 1947869968613 | 20260508 01:52:21.578 | INFO | NumEvents : 941995758538 | 20260508 01:52:23.061 | INFO | Analyzing chunk frames = [4304896,4305408) | 20260508 01:52:23.061 | INFO | MemInfo: { | 20260508 01:52:23.061 | INFO | "gpu" : " 17 GB", | 20260508 01:52:23.061 | INFO | "huge" : "", | 20260508 01:52:23.061 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:23.061 | INFO | "malloc" : " 14 GB", | 20260508 01:52:23.061 | INFO | "pinned" : " 29 GB" | 20260508 01:52:23.061 | INFO | } | 20260508 01:52:23.061 | INFO | Perf report for the last 5.1387 seconds: | 20260508 01:52:23.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:23.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:23.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:23.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:23.062 | INFO | ANALYSIS : 68.5%, 3.0, 106.7ms, 0 | 20260508 01:52:23.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:23.062 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:52:23.062 | INFO | PRE_HQ : 65.7%, 1.6, 56.1ms, 0 | 20260508 01:52:23.062 | INFO | BAZWRITER : 68.8%, 3.7, 132.4ms, 0 | 20260508 01:52:23.062 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 01:52:23.062 | INFO | Uploaded 36GiB to GPU in 2.55895s (14.0683GiB/s) | 20260508 01:52:23.062 | INFO | Downloaded 38GiB to GPU in 2.67968s (14.1808GiB/s) | 20260508 01:52:28.196 | INFO | Analyzing chunk frames = [4305408,4305920) | 20260508 01:52:28.196 | INFO | MemInfo: { | 20260508 01:52:28.196 | INFO | "gpu" : " 17 GB", | 20260508 01:52:28.196 | INFO | "huge" : "", | 20260508 01:52:28.196 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:28.196 | INFO | "malloc" : " 14 GB", | 20260508 01:52:28.196 | INFO | "pinned" : " 29 GB" | 20260508 01:52:28.196 | INFO | } | 20260508 01:52:28.196 | INFO | Perf report for the last 5.1343 seconds: | 20260508 01:52:28.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:28.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 01:52:28.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:52:28.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:28.196 | INFO | Uploaded 36GiB to GPU in 2.53703s (14.1898GiB/s) | 20260508 01:52:28.196 | INFO | Downloaded 34GiB to GPU in 2.42637s (14.0127GiB/s) | 20260508 01:52:33.337 | INFO | Analyzing chunk frames = [4305920,4306432) | 20260508 01:52:33.337 | INFO | MemInfo: { | 20260508 01:52:33.337 | INFO | "gpu" : " 17 GB", | 20260508 01:52:33.337 | INFO | "huge" : "", | 20260508 01:52:33.337 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:33.337 | INFO | "malloc" : " 14 GB", | 20260508 01:52:33.337 | INFO | "pinned" : " 29 GB" | 20260508 01:52:33.337 | INFO | } | 20260508 01:52:33.337 | INFO | Perf report for the last 5.14134 seconds: | 20260508 01:52:33.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:33.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.337 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:52:33.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.337 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:52:33.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:33.338 | INFO | Uploaded 36GiB to GPU in 2.53433s (14.205GiB/s) | 20260508 01:52:33.338 | INFO | Downloaded 34GiB to GPU in 2.48522s (13.6809GiB/s) | 20260508 01:52:38.479 | INFO | Analyzing chunk frames = [4306432,4306944) | 20260508 01:52:38.479 | INFO | MemInfo: { | 20260508 01:52:38.479 | INFO | "gpu" : " 17 GB", | 20260508 01:52:38.479 | INFO | "huge" : "", | 20260508 01:52:38.479 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:38.479 | INFO | "malloc" : " 14 GB", | 20260508 01:52:38.479 | INFO | "pinned" : " 29 GB" | 20260508 01:52:38.479 | INFO | } | 20260508 01:52:38.479 | INFO | Perf report for the last 5.14157 seconds: | 20260508 01:52:38.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:38.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 01:52:38.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:52:38.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:38.479 | INFO | Uploaded 36GiB to GPU in 2.51784s (14.298GiB/s) | 20260508 01:52:38.479 | INFO | Downloaded 34GiB to GPU in 2.48965s (13.6566GiB/s) | 20260508 01:52:43.617 | INFO | Analyzing chunk frames = [4306944,4307456) | 20260508 01:52:43.617 | INFO | MemInfo: { | 20260508 01:52:43.617 | INFO | "gpu" : " 17 GB", | 20260508 01:52:43.617 | INFO | "huge" : "", | 20260508 01:52:43.617 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:43.617 | INFO | "malloc" : " 14 GB", | 20260508 01:52:43.617 | INFO | "pinned" : " 29 GB" | 20260508 01:52:43.617 | INFO | } | 20260508 01:52:43.617 | INFO | Perf report for the last 5.13847 seconds: | 20260508 01:52:43.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:43.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 01:52:43.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:52:43.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:43.617 | INFO | Uploaded 36GiB to GPU in 2.35043s (15.3164GiB/s) | 20260508 01:52:43.618 | INFO | Downloaded 34GiB to GPU in 2.50047s (13.5975GiB/s) | 20260508 01:52:48.757 | INFO | Analyzing chunk frames = [4307456,4307968) | 20260508 01:52:48.757 | INFO | MemInfo: { | 20260508 01:52:48.757 | INFO | "gpu" : " 17 GB", | 20260508 01:52:48.757 | INFO | "huge" : "", | 20260508 01:52:48.757 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:48.757 | INFO | "malloc" : " 14 GB", | 20260508 01:52:48.757 | INFO | "pinned" : " 29 GB" | 20260508 01:52:48.757 | INFO | } | 20260508 01:52:48.757 | INFO | Perf report for the last 5.13975 seconds: | 20260508 01:52:48.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:48.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 01:52:48.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | PRE_HQ : 32.2%, 1.0, 16.8ms, 0 | 20260508 01:52:48.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:48.757 | INFO | Uploaded 36GiB to GPU in 2.4497s (14.6957GiB/s) | 20260508 01:52:48.757 | INFO | Downloaded 34GiB to GPU in 2.41904s (14.0552GiB/s) | 20260508 01:52:53.892 | INFO | Analyzing chunk frames = [4307968,4308480) | 20260508 01:52:53.892 | INFO | MemInfo: { | 20260508 01:52:53.892 | INFO | "gpu" : " 17 GB", | 20260508 01:52:53.892 | INFO | "huge" : "", | 20260508 01:52:53.892 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:53.892 | INFO | "malloc" : " 14 GB", | 20260508 01:52:53.892 | INFO | "pinned" : " 29 GB" | 20260508 01:52:53.892 | INFO | } | 20260508 01:52:53.893 | INFO | Perf report for the last 5.13549 seconds: | 20260508 01:52:53.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:53.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 01:52:53.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:52:53.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:53.893 | INFO | Uploaded 36GiB to GPU in 2.46918s (14.5797GiB/s) | 20260508 01:52:53.893 | INFO | Downloaded 34GiB to GPU in 2.41846s (14.0585GiB/s) | 20260508 01:52:59.038 | INFO | Analyzing chunk frames = [4308480,4308992) | 20260508 01:52:59.038 | INFO | MemInfo: { | 20260508 01:52:59.038 | INFO | "gpu" : " 17 GB", | 20260508 01:52:59.038 | INFO | "huge" : "", | 20260508 01:52:59.038 | INFO | "huge_shared" : " 29 GB", | 20260508 01:52:59.038 | INFO | "malloc" : " 14 GB", | 20260508 01:52:59.038 | INFO | "pinned" : " 29 GB" | 20260508 01:52:59.038 | INFO | } | 20260508 01:52:59.038 | INFO | Perf report for the last 5.14554 seconds: | 20260508 01:52:59.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:52:59.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.038 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:52:59.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.038 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:52:59.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:52:59.039 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:52:59.039 | INFO | Total mem usage: 16.7 GiB | 20260508 01:52:59.039 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:52:59.039 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.1%) | 20260508 01:52:59.039 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:52:59.039 | INFO | High Water Breakdown: | 20260508 01:52:59.039 | INFO | 59.7% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:52:59.039 | INFO | 40.2% : 5.82 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:52:59.039 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:52:59.039 | INFO | ------------------------------------------ | 20260508 01:52:59.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:52:59.039 | INFO | Uploaded 36GiB to GPU in 2.56715s (14.0233GiB/s) | 20260508 01:52:59.039 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:52:59.039 | INFO | Total mem usage: 26.5 GiB | 20260508 01:52:59.039 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:52:59.039 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:52:59.039 | INFO | Downloaded 34GiB to GPU in 2.44882s (13.8842GiB/s) | 20260508 01:52:59.039 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:52:59.039 | INFO | High Water Breakdown: | 20260508 01:52:59.039 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:52:59.039 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:52:59.039 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:52:59.039 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:52:59.039 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:52:59.039 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:52:59.039 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:52:59.039 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:52:59.039 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:52:59.039 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:52:59.039 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:52:59.039 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:52:59.039 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:52:59.039 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:52:59.039 | INFO | ------------------------------------------ | 20260508 01:52:59.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:52:59.039 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:52:59.039 | INFO | Total mem usage: 27.8 GiB | 20260508 01:52:59.039 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:52:59.039 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:52:59.039 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:52:59.039 | INFO | High Water Breakdown: | 20260508 01:52:59.039 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:52:59.039 | INFO | ------------------------------------------ | 20260508 01:52:59.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:52:59.039 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:52:59.039 | INFO | Total mem usage: 15.7 GiB | 20260508 01:52:59.039 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:52:59.039 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:52:59.039 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:52:59.039 | INFO | High Water Breakdown: | 20260508 01:52:59.039 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:52:59.039 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:52:59.039 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:52:59.039 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:52:59.039 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:52:59.039 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:52:59.039 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:52:59.039 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:52:59.039 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:52:59.039 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:52:59.039 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:52:59.039 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:52:59.039 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:52:59.039 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:52:59.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:52:59.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:52:59.039 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:52:59.039 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:52:59.039 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:52:59.039 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:52:59.039 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:52:59.039 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:52:59.040 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:52:59.040 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:52:59.040 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:52:59.040 | INFO | ------------------------------------------ | 20260508 01:52:59.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:52:59.040 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:52:59.040 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:52:59.040 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:52:59.040 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.19e+00 | 20260508 01:52:59.040 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.23e-02 / 5.36e-03 | 20260508 01:52:59.040 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.46e+00, 5.76e-01 / 1.86e-01 | 20260508 01:52:59.040 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.06e-01 / 1.27e-01 | 20260508 01:52:59.040 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.58e+01, 3.93e+00 / 1.39e+00 | 20260508 01:52:59.040 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.62e+00 / 1.45e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.22e-01 / 2.54e-01 | 20260508 01:52:59.041 | INFO | PerfReport: 3.32%, Metrics -- 1.08e+00 / 1.27e+00, 2.63e-01 / 1.38e-01 | 20260508 01:52:59.041 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.66e-01, 5.30e-02 / 2.42e-02 | 20260508 01:52:59.041 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:52:59.041 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:52:59.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:52:59.041 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 24.34%, Upload -- 2.59e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 2.15%, LockUpload -- 2.28e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 2.83%, LockDownload -- 3.01e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:52:59.041 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.42e-02 / 0.00e+00 | 20260508 01:52:59.136 | INFO | Starting a round of Baz IO | 20260508 01:53:02.584 | INFO | Populating RTMetrics report | 20260508 01:53:02.585 | INFO | RTMetrics report populated | 20260508 01:53:02.585 | INFO | Opening tmp RTMetrics json file | 20260508 01:53:02.602 | INFO | Serializing RTMetrics to json | 20260508 01:53:02.603 | INFO | Serialization done, swapping tmp file now... | 20260508 01:53:02.604 | INFO | RTMetrics done | 20260508 01:53:02.667 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.531 seconds | 20260508 01:53:02.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.70% / 0.00% / 58.30% | 20260508 01:53:02.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.793902% / 0% / 5.96046e-05% | 20260508 01:53:02.667 | INFO | Baz IO overall: | 20260508 01:53:02.667 | INFO | BytesWritten : 1949643333954 | 20260508 01:53:02.667 | INFO | NumEvents : 942846560750 | 20260508 01:53:04.181 | INFO | Analyzing chunk frames = [4308992,4309504) | 20260508 01:53:04.181 | INFO | MemInfo: { | 20260508 01:53:04.181 | INFO | "gpu" : " 17 GB", | 20260508 01:53:04.181 | INFO | "huge" : "", | 20260508 01:53:04.181 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:04.181 | INFO | "malloc" : " 14 GB", | 20260508 01:53:04.181 | INFO | "pinned" : " 29 GB" | 20260508 01:53:04.181 | INFO | } | 20260508 01:53:04.181 | INFO | Perf report for the last 5.14274 seconds: | 20260508 01:53:04.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:04.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:04.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:04.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:04.181 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 01:53:04.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:04.181 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 01:53:04.181 | INFO | PRE_HQ : 65.6%, 1.6, 56.1ms, 0 | 20260508 01:53:04.181 | INFO | BAZWRITER : 68.3%, 4.1, 142.3ms, 0 | 20260508 01:53:04.181 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:53:04.181 | INFO | Uploaded 36GiB to GPU in 2.58659s (13.9179GiB/s) | 20260508 01:53:04.181 | INFO | Downloaded 38GiB to GPU in 2.69726s (14.0884GiB/s) | 20260508 01:53:09.315 | INFO | Analyzing chunk frames = [4309504,4310016) | 20260508 01:53:09.315 | INFO | MemInfo: { | 20260508 01:53:09.315 | INFO | "gpu" : " 17 GB", | 20260508 01:53:09.315 | INFO | "huge" : "", | 20260508 01:53:09.315 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:09.315 | INFO | "malloc" : " 14 GB", | 20260508 01:53:09.315 | INFO | "pinned" : " 29 GB" | 20260508 01:53:09.315 | INFO | } | 20260508 01:53:09.315 | INFO | Perf report for the last 5.13414 seconds: | 20260508 01:53:09.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:09.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 01:53:09.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:53:09.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:09.315 | INFO | Uploaded 36GiB to GPU in 2.53208s (14.2176GiB/s) | 20260508 01:53:09.315 | INFO | Downloaded 34GiB to GPU in 2.43153s (13.983GiB/s) | 20260508 01:53:14.450 | INFO | Analyzing chunk frames = [4310016,4310528) | 20260508 01:53:14.450 | INFO | MemInfo: { | 20260508 01:53:14.450 | INFO | "gpu" : " 17 GB", | 20260508 01:53:14.450 | INFO | "huge" : "", | 20260508 01:53:14.450 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:14.450 | INFO | "malloc" : " 14 GB", | 20260508 01:53:14.450 | INFO | "pinned" : " 29 GB" | 20260508 01:53:14.450 | INFO | } | 20260508 01:53:14.450 | INFO | Perf report for the last 5.135 seconds: | 20260508 01:53:14.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:14.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 01:53:14.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:53:14.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:14.450 | INFO | Uploaded 36GiB to GPU in 2.52443s (14.2606GiB/s) | 20260508 01:53:14.450 | INFO | Downloaded 34GiB to GPU in 2.493s (13.6382GiB/s) | 20260508 01:53:19.593 | INFO | Analyzing chunk frames = [4310528,4311040) | 20260508 01:53:19.593 | INFO | MemInfo: { | 20260508 01:53:19.593 | INFO | "gpu" : " 17 GB", | 20260508 01:53:19.593 | INFO | "huge" : "", | 20260508 01:53:19.593 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:19.593 | INFO | "malloc" : " 14 GB", | 20260508 01:53:19.593 | INFO | "pinned" : " 29 GB" | 20260508 01:53:19.593 | INFO | } | 20260508 01:53:19.594 | INFO | Perf report for the last 5.1436 seconds: | 20260508 01:53:19.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:19.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:53:19.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 01:53:19.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:19.594 | INFO | Uploaded 36GiB to GPU in 2.54422s (14.1497GiB/s) | 20260508 01:53:19.594 | INFO | Downloaded 34GiB to GPU in 2.50252s (13.5863GiB/s) | 20260508 01:53:24.729 | INFO | Analyzing chunk frames = [4311040,4311552) | 20260508 01:53:24.729 | INFO | MemInfo: { | 20260508 01:53:24.729 | INFO | "gpu" : " 17 GB", | 20260508 01:53:24.729 | INFO | "huge" : "", | 20260508 01:53:24.729 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:24.729 | INFO | "malloc" : " 14 GB", | 20260508 01:53:24.729 | INFO | "pinned" : " 29 GB" | 20260508 01:53:24.729 | INFO | } | 20260508 01:53:24.729 | INFO | Perf report for the last 5.1354 seconds: | 20260508 01:53:24.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:24.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 01:53:24.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 01:53:24.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:24.729 | INFO | Uploaded 36GiB to GPU in 2.41312s (14.9185GiB/s) | 20260508 01:53:24.729 | INFO | Downloaded 34GiB to GPU in 2.5665s (13.2476GiB/s) | 20260508 01:53:29.871 | INFO | Analyzing chunk frames = [4311552,4312064) | 20260508 01:53:29.871 | INFO | MemInfo: { | 20260508 01:53:29.871 | INFO | "gpu" : " 17 GB", | 20260508 01:53:29.871 | INFO | "huge" : "", | 20260508 01:53:29.871 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:29.871 | INFO | "malloc" : " 14 GB", | 20260508 01:53:29.871 | INFO | "pinned" : " 29 GB" | 20260508 01:53:29.871 | INFO | } | 20260508 01:53:29.871 | INFO | Perf report for the last 5.14206 seconds: | 20260508 01:53:29.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:29.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.871 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 01:53:29.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.871 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:53:29.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:29.872 | INFO | Uploaded 36GiB to GPU in 2.38093s (15.1202GiB/s) | 20260508 01:53:29.872 | INFO | Downloaded 34GiB to GPU in 2.43268s (13.9763GiB/s) | 20260508 01:53:35.010 | INFO | Analyzing chunk frames = [4312064,4312576) | 20260508 01:53:35.010 | INFO | MemInfo: { | 20260508 01:53:35.010 | INFO | "gpu" : " 17 GB", | 20260508 01:53:35.010 | INFO | "huge" : "", | 20260508 01:53:35.010 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:35.010 | INFO | "malloc" : " 14 GB", | 20260508 01:53:35.010 | INFO | "pinned" : " 29 GB" | 20260508 01:53:35.010 | INFO | } | 20260508 01:53:35.010 | INFO | Perf report for the last 5.13864 seconds: | 20260508 01:53:35.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:35.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:53:35.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:53:35.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:35.010 | INFO | Uploaded 36GiB to GPU in 2.42074s (14.8715GiB/s) | 20260508 01:53:35.010 | INFO | Downloaded 34GiB to GPU in 2.43592s (13.9577GiB/s) | 20260508 01:53:40.147 | INFO | Analyzing chunk frames = [4312576,4313088) | 20260508 01:53:40.147 | INFO | MemInfo: { | 20260508 01:53:40.147 | INFO | "gpu" : " 17 GB", | 20260508 01:53:40.147 | INFO | "huge" : "", | 20260508 01:53:40.147 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:40.147 | INFO | "malloc" : " 14 GB", | 20260508 01:53:40.147 | INFO | "pinned" : " 29 GB" | 20260508 01:53:40.147 | INFO | } | 20260508 01:53:40.147 | INFO | Perf report for the last 5.13764 seconds: | 20260508 01:53:40.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:40.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.147 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 01:53:40.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.147 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:53:40.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:40.148 | INFO | Uploaded 36GiB to GPU in 2.50938s (14.3462GiB/s) | 20260508 01:53:40.148 | INFO | Downloaded 34GiB to GPU in 2.46949s (13.768GiB/s) | 20260508 01:53:40.247 | INFO | Starting a round of Baz IO | 20260508 01:53:43.701 | INFO | Populating RTMetrics report | 20260508 01:53:43.701 | INFO | RTMetrics report populated | 20260508 01:53:43.701 | INFO | Opening tmp RTMetrics json file | 20260508 01:53:43.727 | INFO | Serializing RTMetrics to json | 20260508 01:53:43.788 | INFO | Serialization done, swapping tmp file now... | 20260508 01:53:43.792 | INFO | RTMetrics done | 20260508 01:53:43.822 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.574 seconds | 20260508 01:53:43.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.70% / 0.00% / 58.30% | 20260508 01:53:43.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.792968% / 0% / 7.54992e-05% | 20260508 01:53:43.822 | INFO | Baz IO overall: | 20260508 01:53:43.822 | INFO | BytesWritten : 1951418001439 | 20260508 01:53:43.822 | INFO | NumEvents : 943698453379 | 20260508 01:53:45.290 | INFO | Analyzing chunk frames = [4313088,4313600) | 20260508 01:53:45.290 | INFO | MemInfo: { | 20260508 01:53:45.290 | INFO | "gpu" : " 17 GB", | 20260508 01:53:45.290 | INFO | "huge" : "", | 20260508 01:53:45.290 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:45.290 | INFO | "malloc" : " 14 GB", | 20260508 01:53:45.290 | INFO | "pinned" : " 29 GB" | 20260508 01:53:45.290 | INFO | } | 20260508 01:53:45.290 | INFO | Perf report for the last 5.14254 seconds: | 20260508 01:53:45.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:45.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:45.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:45.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:45.290 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 01:53:45.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:45.290 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 01:53:45.290 | INFO | PRE_HQ : 65.3%, 1.7, 56.7ms, 0 | 20260508 01:53:45.290 | INFO | BAZWRITER : 69.1%, 3.6, 129.0ms, 0 | 20260508 01:53:45.290 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260508 01:53:45.290 | INFO | Uploaded 36GiB to GPU in 2.54607s (14.1394GiB/s) | 20260508 01:53:45.290 | INFO | Downloaded 38GiB to GPU in 2.68941s (14.1295GiB/s) | 20260508 01:53:50.429 | INFO | Analyzing chunk frames = [4313600,4314112) | 20260508 01:53:50.429 | INFO | MemInfo: { | 20260508 01:53:50.429 | INFO | "gpu" : " 17 GB", | 20260508 01:53:50.429 | INFO | "huge" : "", | 20260508 01:53:50.429 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:50.429 | INFO | "malloc" : " 14 GB", | 20260508 01:53:50.429 | INFO | "pinned" : " 29 GB" | 20260508 01:53:50.429 | INFO | } | 20260508 01:53:50.429 | INFO | Perf report for the last 5.13953 seconds: | 20260508 01:53:50.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:50.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.430 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 01:53:50.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.430 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:53:50.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:50.430 | INFO | Uploaded 36GiB to GPU in 2.52089s (14.2807GiB/s) | 20260508 01:53:50.430 | INFO | Downloaded 34GiB to GPU in 2.43896s (13.9404GiB/s) | 20260508 01:53:55.568 | INFO | Analyzing chunk frames = [4314112,4314624) | 20260508 01:53:55.568 | INFO | MemInfo: { | 20260508 01:53:55.568 | INFO | "gpu" : " 17 GB", | 20260508 01:53:55.568 | INFO | "huge" : "", | 20260508 01:53:55.568 | INFO | "huge_shared" : " 29 GB", | 20260508 01:53:55.569 | INFO | "malloc" : " 14 GB", | 20260508 01:53:55.569 | INFO | "pinned" : " 29 GB" | 20260508 01:53:55.569 | INFO | } | 20260508 01:53:55.569 | INFO | Perf report for the last 5.13944 seconds: | 20260508 01:53:55.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:53:55.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.569 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 01:53:55.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.569 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 01:53:55.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:53:55.570 | INFO | Uploaded 36GiB to GPU in 2.47893s (14.5224GiB/s) | 20260508 01:53:55.570 | INFO | Downloaded 34GiB to GPU in 2.4995s (13.6027GiB/s) | 20260508 01:54:00.705 | INFO | Analyzing chunk frames = [4314624,4315136) | 20260508 01:54:00.705 | INFO | MemInfo: { | 20260508 01:54:00.705 | INFO | "gpu" : " 17 GB", | 20260508 01:54:00.705 | INFO | "huge" : "", | 20260508 01:54:00.705 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:00.705 | INFO | "malloc" : " 14 GB", | 20260508 01:54:00.705 | INFO | "pinned" : " 29 GB" | 20260508 01:54:00.705 | INFO | } | 20260508 01:54:00.706 | INFO | Perf report for the last 5.13658 seconds: | 20260508 01:54:00.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:00.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:54:00.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260508 01:54:00.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:00.706 | INFO | Uploaded 36GiB to GPU in 2.4936s (14.4369GiB/s) | 20260508 01:54:00.706 | INFO | Downloaded 34GiB to GPU in 2.52252s (13.4786GiB/s) | 20260508 01:54:05.847 | INFO | Analyzing chunk frames = [4315136,4315648) | 20260508 01:54:05.847 | INFO | MemInfo: { | 20260508 01:54:05.847 | INFO | "gpu" : " 17 GB", | 20260508 01:54:05.847 | INFO | "huge" : "", | 20260508 01:54:05.847 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:05.847 | INFO | "malloc" : " 14 GB", | 20260508 01:54:05.847 | INFO | "pinned" : " 29 GB" | 20260508 01:54:05.847 | INFO | } | 20260508 01:54:05.847 | INFO | Perf report for the last 5.14124 seconds: | 20260508 01:54:05.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:05.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 01:54:05.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:54:05.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:05.847 | INFO | Uploaded 36GiB to GPU in 2.40662s (14.9587GiB/s) | 20260508 01:54:05.847 | INFO | Downloaded 34GiB to GPU in 2.56176s (13.2721GiB/s) | 20260508 01:54:10.990 | INFO | Analyzing chunk frames = [4315648,4316160) | 20260508 01:54:10.990 | INFO | MemInfo: { | 20260508 01:54:10.990 | INFO | "gpu" : " 17 GB", | 20260508 01:54:10.990 | INFO | "huge" : "", | 20260508 01:54:10.991 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:10.991 | INFO | "malloc" : " 14 GB", | 20260508 01:54:10.991 | INFO | "pinned" : " 29 GB" | 20260508 01:54:10.991 | INFO | } | 20260508 01:54:10.991 | INFO | Perf report for the last 5.14418 seconds: | 20260508 01:54:10.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:10.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:54:10.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:54:10.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:10.991 | INFO | Uploaded 36GiB to GPU in 2.38925s (15.0675GiB/s) | 20260508 01:54:10.991 | INFO | Downloaded 34GiB to GPU in 2.41774s (14.0627GiB/s) | 20260508 01:54:16.128 | INFO | Analyzing chunk frames = [4316160,4316672) | 20260508 01:54:16.128 | INFO | MemInfo: { | 20260508 01:54:16.128 | INFO | "gpu" : " 17 GB", | 20260508 01:54:16.128 | INFO | "huge" : "", | 20260508 01:54:16.128 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:16.128 | INFO | "malloc" : " 14 GB", | 20260508 01:54:16.128 | INFO | "pinned" : " 29 GB" | 20260508 01:54:16.128 | INFO | } | 20260508 01:54:16.128 | INFO | Perf report for the last 5.13737 seconds: | 20260508 01:54:16.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:16.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.129 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 01:54:16.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.129 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:54:16.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:16.129 | INFO | Uploaded 36GiB to GPU in 2.42164s (14.866GiB/s) | 20260508 01:54:16.129 | INFO | Downloaded 34GiB to GPU in 2.42793s (14.0037GiB/s) | 20260508 01:54:21.264 | INFO | Analyzing chunk frames = [4316672,4317184) | 20260508 01:54:21.264 | INFO | MemInfo: { | 20260508 01:54:21.264 | INFO | "gpu" : " 17 GB", | 20260508 01:54:21.264 | INFO | "huge" : "", | 20260508 01:54:21.264 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:21.264 | INFO | "malloc" : " 14 GB", | 20260508 01:54:21.264 | INFO | "pinned" : " 29 GB" | 20260508 01:54:21.264 | INFO | } | 20260508 01:54:21.264 | INFO | Perf report for the last 5.13592 seconds: | 20260508 01:54:21.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:21.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.264 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:54:21.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.264 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:54:21.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:21.265 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:54:21.265 | INFO | Total mem usage: 16.7 GiB | 20260508 01:54:21.265 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:54:21.265 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 01:54:21.265 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:54:21.265 | INFO | High Water Breakdown: | 20260508 01:54:21.265 | INFO | 60.1% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:54:21.265 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:54:21.265 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:54:21.265 | INFO | ------------------------------------------ | 20260508 01:54:21.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:54:21.265 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:54:21.265 | INFO | Total mem usage: 26.5 GiB | 20260508 01:54:21.265 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:54:21.265 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:54:21.265 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:54:21.265 | INFO | High Water Breakdown: | 20260508 01:54:21.265 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:54:21.265 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:54:21.265 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:54:21.265 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:54:21.265 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:54:21.265 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:54:21.265 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:54:21.265 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:54:21.265 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:54:21.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:54:21.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:54:21.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:54:21.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:54:21.265 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:54:21.265 | INFO | ------------------------------------------ | 20260508 01:54:21.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:54:21.265 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:54:21.265 | INFO | Total mem usage: 27.8 GiB | 20260508 01:54:21.265 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:54:21.265 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:54:21.265 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:54:21.265 | INFO | High Water Breakdown: | 20260508 01:54:21.265 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:54:21.265 | INFO | ------------------------------------------ | 20260508 01:54:21.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:54:21.265 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:54:21.265 | INFO | Total mem usage: 15.7 GiB | 20260508 01:54:21.265 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:54:21.265 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:54:21.265 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:54:21.265 | INFO | High Water Breakdown: | 20260508 01:54:21.265 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:54:21.265 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:54:21.265 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:54:21.265 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:54:21.265 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:54:21.265 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:54:21.265 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:54:21.265 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:54:21.265 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:54:21.265 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:54:21.265 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:54:21.265 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:54:21.265 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:54:21.265 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:54:21.265 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:54:21.265 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:54:21.265 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:54:21.265 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:54:21.265 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:54:21.265 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:54:21.265 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:54:21.265 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:54:21.265 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:54:21.265 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:54:21.265 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:54:21.265 | INFO | ------------------------------------------ | 20260508 01:54:21.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:54:21.265 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:54:21.265 | INFO | Uploaded 36GiB to GPU in 2.53335s (14.2104GiB/s) | 20260508 01:54:21.266 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:54:21.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:54:21.266 | INFO | Downloaded 34GiB to GPU in 2.45875s (13.8282GiB/s) | 20260508 01:54:21.266 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.43e+01 / 1.58e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.12e-02 / 7.61e-03 | 20260508 01:54:21.266 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.48e+00, 5.88e-01 / 1.61e-01 | 20260508 01:54:21.266 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.08e-01 / 1.07e-01 | 20260508 01:54:21.266 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.59e+01, 4.07e+00 / 2.09e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.66e+00 / 1.62e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.69e+00, 4.22e-01 / 2.40e-01 | 20260508 01:54:21.266 | INFO | PerfReport: 3.30%, Metrics -- 1.08e+00 / 1.25e+00, 2.69e-01 / 1.24e-01 | 20260508 01:54:21.266 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.67e-01, 8.28e-02 / 2.28e-02 | 20260508 01:54:21.266 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:54:21.266 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:54:21.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:54:21.266 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 24.26%, Upload -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 24.47%, Download -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 2.98%, LockDownload -- 3.18e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 01:54:21.266 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260508 01:54:21.362 | INFO | Starting a round of Baz IO | 20260508 01:54:24.806 | INFO | Populating RTMetrics report | 20260508 01:54:24.806 | INFO | RTMetrics report populated | 20260508 01:54:24.806 | INFO | Opening tmp RTMetrics json file | 20260508 01:54:24.825 | INFO | Serializing RTMetrics to json | 20260508 01:54:24.827 | INFO | Serialization done, swapping tmp file now... | 20260508 01:54:24.828 | INFO | RTMetrics done | 20260508 01:54:24.912 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.549 seconds | 20260508 01:54:24.912 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.69% / 0.00% / 58.30% | 20260508 01:54:24.912 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.791498% / 3.97364e-06% / 6.35783e-05% | 20260508 01:54:24.912 | INFO | Baz IO overall: | 20260508 01:54:24.912 | INFO | BytesWritten : 1953191945525 | 20260508 01:54:24.912 | INFO | NumEvents : 944549854993 | 20260508 01:54:26.400 | INFO | Analyzing chunk frames = [4317184,4317696) | 20260508 01:54:26.400 | INFO | MemInfo: { | 20260508 01:54:26.400 | INFO | "gpu" : " 17 GB", | 20260508 01:54:26.400 | INFO | "huge" : "", | 20260508 01:54:26.400 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:26.400 | INFO | "malloc" : " 14 GB", | 20260508 01:54:26.400 | INFO | "pinned" : " 29 GB" | 20260508 01:54:26.400 | INFO | } | 20260508 01:54:26.400 | INFO | Perf report for the last 5.13613 seconds: | 20260508 01:54:26.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:26.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:26.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:26.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:26.401 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 01:54:26.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:26.401 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 01:54:26.401 | INFO | PRE_HQ : 65.8%, 1.6, 56.0ms, 0 | 20260508 01:54:26.401 | INFO | BAZWRITER : 68.7%, 3.8, 134.7ms, 0 | 20260508 01:54:26.401 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 01:54:26.401 | INFO | Uploaded 36GiB to GPU in 2.60161s (13.8376GiB/s) | 20260508 01:54:26.401 | INFO | Downloaded 38GiB to GPU in 2.68009s (14.1786GiB/s) | 20260508 01:54:31.545 | INFO | Analyzing chunk frames = [4317696,4318208) | 20260508 01:54:31.545 | INFO | MemInfo: { | 20260508 01:54:31.545 | INFO | "gpu" : " 17 GB", | 20260508 01:54:31.545 | INFO | "huge" : "", | 20260508 01:54:31.545 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:31.545 | INFO | "malloc" : " 14 GB", | 20260508 01:54:31.545 | INFO | "pinned" : " 29 GB" | 20260508 01:54:31.545 | INFO | } | 20260508 01:54:31.545 | INFO | Perf report for the last 5.1447 seconds: | 20260508 01:54:31.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:31.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 01:54:31.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:54:31.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:31.545 | INFO | Uploaded 36GiB to GPU in 2.56826s (14.0173GiB/s) | 20260508 01:54:31.545 | INFO | Downloaded 34GiB to GPU in 2.41484s (14.0796GiB/s) | 20260508 01:54:36.681 | INFO | Analyzing chunk frames = [4318208,4318720) | 20260508 01:54:36.681 | INFO | MemInfo: { | 20260508 01:54:36.681 | INFO | "gpu" : " 17 GB", | 20260508 01:54:36.681 | INFO | "huge" : "", | 20260508 01:54:36.681 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:36.681 | INFO | "malloc" : " 14 GB", | 20260508 01:54:36.681 | INFO | "pinned" : " 29 GB" | 20260508 01:54:36.681 | INFO | } | 20260508 01:54:36.681 | INFO | Perf report for the last 5.13609 seconds: | 20260508 01:54:36.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:36.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.681 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 01:54:36.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.681 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:54:36.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:36.682 | INFO | Uploaded 36GiB to GPU in 2.5839s (13.9324GiB/s) | 20260508 01:54:36.682 | INFO | Downloaded 34GiB to GPU in 2.492s (13.6437GiB/s) | 20260508 01:54:41.820 | INFO | Analyzing chunk frames = [4318720,4319232) | 20260508 01:54:41.820 | INFO | MemInfo: { | 20260508 01:54:41.820 | INFO | "gpu" : " 17 GB", | 20260508 01:54:41.820 | INFO | "huge" : "", | 20260508 01:54:41.820 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:41.820 | INFO | "malloc" : " 14 GB", | 20260508 01:54:41.820 | INFO | "pinned" : " 29 GB" | 20260508 01:54:41.820 | INFO | } | 20260508 01:54:41.820 | INFO | Perf report for the last 5.13897 seconds: | 20260508 01:54:41.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:41.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.820 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:54:41.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.820 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:54:41.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:41.821 | INFO | Uploaded 36GiB to GPU in 2.53335s (14.2104GiB/s) | 20260508 01:54:41.821 | INFO | Downloaded 34GiB to GPU in 2.4678s (13.7774GiB/s) | 20260508 01:54:46.959 | INFO | Analyzing chunk frames = [4319232,4319744) | 20260508 01:54:46.959 | INFO | MemInfo: { | 20260508 01:54:46.959 | INFO | "gpu" : " 17 GB", | 20260508 01:54:46.959 | INFO | "huge" : "", | 20260508 01:54:46.959 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:46.959 | INFO | "malloc" : " 14 GB", | 20260508 01:54:46.959 | INFO | "pinned" : " 29 GB" | 20260508 01:54:46.959 | INFO | } | 20260508 01:54:46.959 | INFO | Perf report for the last 5.13904 seconds: | 20260508 01:54:46.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:46.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.959 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 01:54:46.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.959 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 01:54:46.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:46.960 | INFO | Uploaded 36GiB to GPU in 2.39184s (15.0512GiB/s) | 20260508 01:54:46.960 | INFO | Downloaded 34GiB to GPU in 2.49206s (13.6433GiB/s) | 20260508 01:54:52.100 | INFO | Analyzing chunk frames = [4319744,4320256) | 20260508 01:54:52.100 | INFO | MemInfo: { | 20260508 01:54:52.100 | INFO | "gpu" : " 17 GB", | 20260508 01:54:52.100 | INFO | "huge" : "", | 20260508 01:54:52.100 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:52.100 | INFO | "malloc" : " 14 GB", | 20260508 01:54:52.100 | INFO | "pinned" : " 29 GB" | 20260508 01:54:52.100 | INFO | } | 20260508 01:54:52.100 | INFO | Perf report for the last 5.14097 seconds: | 20260508 01:54:52.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:52.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.100 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:54:52.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.100 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:54:52.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:52.101 | INFO | Uploaded 36GiB to GPU in 2.4262s (14.838GiB/s) | 20260508 01:54:52.101 | INFO | Downloaded 34GiB to GPU in 2.40401s (14.1431GiB/s) | 20260508 01:54:57.245 | INFO | Analyzing chunk frames = [4320256,4320768) | 20260508 01:54:57.245 | INFO | MemInfo: { | 20260508 01:54:57.245 | INFO | "gpu" : " 17 GB", | 20260508 01:54:57.245 | INFO | "huge" : "", | 20260508 01:54:57.245 | INFO | "huge_shared" : " 29 GB", | 20260508 01:54:57.245 | INFO | "malloc" : " 14 GB", | 20260508 01:54:57.245 | INFO | "pinned" : " 29 GB" | 20260508 01:54:57.245 | INFO | } | 20260508 01:54:57.245 | INFO | Perf report for the last 5.14512 seconds: | 20260508 01:54:57.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:54:57.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.245 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 01:54:57.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.245 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:54:57.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:54:57.246 | INFO | Uploaded 36GiB to GPU in 2.45723s (14.6507GiB/s) | 20260508 01:54:57.246 | INFO | Downloaded 34GiB to GPU in 2.41209s (14.0957GiB/s) | 20260508 01:55:02.383 | INFO | Analyzing chunk frames = [4320768,4321280) | 20260508 01:55:02.384 | INFO | MemInfo: { | 20260508 01:55:02.384 | INFO | "gpu" : " 17 GB", | 20260508 01:55:02.384 | INFO | "huge" : "", | 20260508 01:55:02.384 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:02.384 | INFO | "malloc" : " 14 GB", | 20260508 01:55:02.384 | INFO | "pinned" : " 29 GB" | 20260508 01:55:02.384 | INFO | } | 20260508 01:55:02.384 | INFO | Perf report for the last 5.13839 seconds: | 20260508 01:55:02.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:02.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.384 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:55:02.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.384 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:55:02.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:02.385 | INFO | Uploaded 36GiB to GPU in 2.5778s (13.9654GiB/s) | 20260508 01:55:02.385 | INFO | Downloaded 34GiB to GPU in 2.45322s (13.8593GiB/s) | 20260508 01:55:02.492 | INFO | Starting a round of Baz IO | 20260508 01:55:05.930 | INFO | Populating RTMetrics report | 20260508 01:55:05.931 | INFO | RTMetrics report populated | 20260508 01:55:05.931 | INFO | Opening tmp RTMetrics json file | 20260508 01:55:05.932 | INFO | Serializing RTMetrics to json | 20260508 01:55:05.934 | INFO | Serialization done, swapping tmp file now... | 20260508 01:55:05.934 | INFO | RTMetrics done | 20260508 01:55:06.030 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.538 seconds | 20260508 01:55:06.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.69% / 0.00% / 58.31% | 20260508 01:55:06.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.791009% / 0% / 4.76837e-05% | 20260508 01:55:06.030 | INFO | Baz IO overall: | 20260508 01:55:06.030 | INFO | BytesWritten : 1954965764531 | 20260508 01:55:06.031 | INFO | NumEvents : 945401377747 | 20260508 01:55:07.516 | INFO | Analyzing chunk frames = [4321280,4321792) | 20260508 01:55:07.516 | INFO | MemInfo: { | 20260508 01:55:07.516 | INFO | "gpu" : " 17 GB", | 20260508 01:55:07.519 | INFO | "huge" : "", | 20260508 01:55:07.519 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:07.519 | INFO | "malloc" : " 14 GB", | 20260508 01:55:07.519 | INFO | "pinned" : " 29 GB" | 20260508 01:55:07.519 | INFO | } | 20260508 01:55:07.519 | INFO | Perf report for the last 5.13554 seconds: | 20260508 01:55:07.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:07.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:07.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:07.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:07.519 | INFO | ANALYSIS : 68.6%, 2.9, 104.1ms, 0 | 20260508 01:55:07.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:07.519 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 01:55:07.519 | INFO | PRE_HQ : 67.3%, 1.8, 61.3ms, 0 | 20260508 01:55:07.519 | INFO | BAZWRITER : 68.5%, 4.1, 144.1ms, 0 | 20260508 01:55:07.519 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 01:55:07.520 | INFO | Uploaded 36GiB to GPU in 2.50933s (14.3464GiB/s) | 20260508 01:55:07.520 | INFO | Downloaded 38GiB to GPU in 2.66663s (14.2502GiB/s) | 20260508 01:55:12.656 | INFO | Analyzing chunk frames = [4321792,4322304) | 20260508 01:55:12.656 | INFO | MemInfo: { | 20260508 01:55:12.656 | INFO | "gpu" : " 17 GB", | 20260508 01:55:12.656 | INFO | "huge" : "", | 20260508 01:55:12.656 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:12.656 | INFO | "malloc" : " 14 GB", | 20260508 01:55:12.656 | INFO | "pinned" : " 29 GB" | 20260508 01:55:12.656 | INFO | } | 20260508 01:55:12.656 | INFO | Perf report for the last 5.13692 seconds: | 20260508 01:55:12.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:12.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.656 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 01:55:12.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.656 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:55:12.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:12.657 | INFO | Uploaded 36GiB to GPU in 2.56574s (14.031GiB/s) | 20260508 01:55:12.657 | INFO | Downloaded 34GiB to GPU in 2.41544s (14.0761GiB/s) | 20260508 01:55:17.799 | INFO | Analyzing chunk frames = [4322304,4322816) | 20260508 01:55:17.799 | INFO | MemInfo: { | 20260508 01:55:17.799 | INFO | "gpu" : " 17 GB", | 20260508 01:55:17.799 | INFO | "huge" : "", | 20260508 01:55:17.799 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:17.799 | INFO | "malloc" : " 14 GB", | 20260508 01:55:17.799 | INFO | "pinned" : " 29 GB" | 20260508 01:55:17.799 | INFO | } | 20260508 01:55:17.800 | INFO | Perf report for the last 5.14326 seconds: | 20260508 01:55:17.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:17.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 01:55:17.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:55:17.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:17.800 | INFO | Uploaded 36GiB to GPU in 2.56202s (14.0514GiB/s) | 20260508 01:55:17.800 | INFO | Downloaded 34GiB to GPU in 2.48387s (13.6883GiB/s) | 20260508 01:55:22.942 | INFO | Analyzing chunk frames = [4322816,4323328) | 20260508 01:55:22.942 | INFO | MemInfo: { | 20260508 01:55:22.942 | INFO | "gpu" : " 17 GB", | 20260508 01:55:22.942 | INFO | "huge" : "", | 20260508 01:55:22.942 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:22.942 | INFO | "malloc" : " 14 GB", | 20260508 01:55:22.942 | INFO | "pinned" : " 29 GB" | 20260508 01:55:22.942 | INFO | } | 20260508 01:55:22.942 | INFO | Perf report for the last 5.14246 seconds: | 20260508 01:55:22.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:22.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 01:55:22.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 01:55:22.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:22.942 | INFO | Uploaded 36GiB to GPU in 2.54335s (14.1546GiB/s) | 20260508 01:55:22.942 | INFO | Downloaded 34GiB to GPU in 2.47339s (13.7463GiB/s) | 20260508 01:55:28.079 | INFO | Analyzing chunk frames = [4323328,4323840) | 20260508 01:55:28.079 | INFO | MemInfo: { | 20260508 01:55:28.079 | INFO | "gpu" : " 17 GB", | 20260508 01:55:28.079 | INFO | "huge" : "", | 20260508 01:55:28.079 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:28.079 | INFO | "malloc" : " 14 GB", | 20260508 01:55:28.079 | INFO | "pinned" : " 29 GB" | 20260508 01:55:28.079 | INFO | } | 20260508 01:55:28.079 | INFO | Perf report for the last 5.13729 seconds: | 20260508 01:55:28.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:28.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.079 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 01:55:28.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.079 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:55:28.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:28.080 | INFO | Uploaded 36GiB to GPU in 2.42765s (14.8291GiB/s) | 20260508 01:55:28.080 | INFO | Downloaded 34GiB to GPU in 2.56942s (13.2325GiB/s) | 20260508 01:55:33.216 | INFO | Analyzing chunk frames = [4323840,4324352) | 20260508 01:55:33.216 | INFO | MemInfo: { | 20260508 01:55:33.216 | INFO | "gpu" : " 17 GB", | 20260508 01:55:33.216 | INFO | "huge" : "", | 20260508 01:55:33.216 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:33.216 | INFO | "malloc" : " 14 GB", | 20260508 01:55:33.216 | INFO | "pinned" : " 29 GB" | 20260508 01:55:33.216 | INFO | } | 20260508 01:55:33.216 | INFO | Perf report for the last 5.13696 seconds: | 20260508 01:55:33.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:33.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.216 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:55:33.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.216 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:55:33.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:33.217 | INFO | Uploaded 36GiB to GPU in 2.38965s (15.065GiB/s) | 20260508 01:55:33.217 | INFO | Downloaded 34GiB to GPU in 2.40946s (14.111GiB/s) | 20260508 01:55:38.353 | INFO | Analyzing chunk frames = [4324352,4324864) | 20260508 01:55:38.353 | INFO | MemInfo: { | 20260508 01:55:38.353 | INFO | "gpu" : " 17 GB", | 20260508 01:55:38.353 | INFO | "huge" : "", | 20260508 01:55:38.354 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:38.354 | INFO | "malloc" : " 14 GB", | 20260508 01:55:38.354 | INFO | "pinned" : " 29 GB" | 20260508 01:55:38.354 | INFO | } | 20260508 01:55:38.354 | INFO | Perf report for the last 5.13733 seconds: | 20260508 01:55:38.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:38.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:55:38.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 01:55:38.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:38.354 | INFO | Uploaded 36GiB to GPU in 2.442s (14.742GiB/s) | 20260508 01:55:38.354 | INFO | Downloaded 34GiB to GPU in 2.42937s (13.9954GiB/s) | 20260508 01:55:43.495 | INFO | Analyzing chunk frames = [4324864,4325376) | 20260508 01:55:43.495 | INFO | MemInfo: { | 20260508 01:55:43.495 | INFO | "gpu" : " 17 GB", | 20260508 01:55:43.495 | INFO | "huge" : "", | 20260508 01:55:43.495 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:43.495 | INFO | "malloc" : " 14 GB", | 20260508 01:55:43.495 | INFO | "pinned" : " 29 GB" | 20260508 01:55:43.495 | INFO | } | 20260508 01:55:43.495 | INFO | Perf report for the last 5.14169 seconds: | 20260508 01:55:43.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:43.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.495 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:55:43.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.495 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 01:55:43.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:43.496 | INFO | Uploaded 36GiB to GPU in 2.53972s (14.1748GiB/s) | 20260508 01:55:43.496 | INFO | Downloaded 34GiB to GPU in 2.45898s (13.8269GiB/s) | 20260508 01:55:43.496 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:55:43.496 | INFO | Total mem usage: 16.7 GiB | 20260508 01:55:43.496 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:55:43.496 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.38%) | 20260508 01:55:43.496 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:55:43.496 | INFO | High Water Breakdown: | 20260508 01:55:43.496 | INFO | 60.5% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:55:43.496 | INFO | 39.4% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:55:43.496 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:55:43.496 | INFO | ------------------------------------------ | 20260508 01:55:43.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:55:43.496 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:55:43.496 | INFO | Total mem usage: 26.5 GiB | 20260508 01:55:43.496 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:55:43.496 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 01:55:43.496 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260508 01:55:43.496 | INFO | High Water Breakdown: | 20260508 01:55:43.496 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:55:43.496 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:55:43.496 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:55:43.496 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:55:43.496 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:55:43.496 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:55:43.496 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:55:43.496 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:55:43.496 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:55:43.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:55:43.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:55:43.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:55:43.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:55:43.496 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:55:43.496 | INFO | ------------------------------------------ | 20260508 01:55:43.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:55:43.496 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:55:43.496 | INFO | Total mem usage: 27.8 GiB | 20260508 01:55:43.496 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:55:43.496 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:55:43.496 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:55:43.496 | INFO | High Water Breakdown: | 20260508 01:55:43.496 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:55:43.496 | INFO | ------------------------------------------ | 20260508 01:55:43.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:55:43.496 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:55:43.497 | INFO | Total mem usage: 15.7 GiB | 20260508 01:55:43.497 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:55:43.497 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:55:43.497 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:55:43.497 | INFO | High Water Breakdown: | 20260508 01:55:43.497 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:55:43.497 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:55:43.497 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:55:43.497 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:55:43.497 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:55:43.497 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:55:43.497 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:55:43.497 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:55:43.497 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:55:43.497 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:55:43.497 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:55:43.497 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:55:43.497 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:55:43.497 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:55:43.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:55:43.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:55:43.497 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:55:43.497 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:55:43.497 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:55:43.497 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:55:43.497 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:55:43.497 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:55:43.497 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:55:43.497 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:55:43.497 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:55:43.497 | INFO | ------------------------------------------ | 20260508 01:55:43.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:55:43.497 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:55:43.497 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:55:43.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:55:43.497 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.63e+01, 1.42e+01 / 1.44e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.13e-02 / 5.73e-03 | 20260508 01:55:43.497 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.47e+00, 5.81e-01 / 2.08e-01 | 20260508 01:55:43.497 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.05e-01 / 9.90e-02 | 20260508 01:55:43.497 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.64e+01, 4.02e+00 / 1.96e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.64e+00 / 1.52e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.42e+00 / 2.71e+00, 4.16e-01 / 2.13e-01 | 20260508 01:55:43.497 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.24e+00, 2.68e-01 / 1.45e-01 | 20260508 01:55:43.497 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.31e-01 / 3.62e-01, 5.23e-02 / 2.33e-02 | 20260508 01:55:43.497 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:55:43.497 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:55:43.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:55:43.497 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 24.52%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 2.47%, LockUpload -- 2.63e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 2.58%, LockDownload -- 2.74e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 15.45%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 01:55:43.497 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 3.89e-02 / 0.00e+00 | 20260508 01:55:43.592 | INFO | Starting a round of Baz IO | 20260508 01:55:47.051 | INFO | Populating RTMetrics report | 20260508 01:55:47.051 | INFO | RTMetrics report populated | 20260508 01:55:47.051 | INFO | Opening tmp RTMetrics json file | 20260508 01:55:47.176 | INFO | Serializing RTMetrics to json | 20260508 01:55:47.185 | INFO | Serialization done, swapping tmp file now... | 20260508 01:55:47.189 | INFO | RTMetrics done | 20260508 01:55:47.209 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.616 seconds | 20260508 01:55:47.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.69% / 0.00% / 58.31% | 20260508 01:55:47.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.788391% / 3.97364e-06% / 5.96046e-05% | 20260508 01:55:47.209 | INFO | Baz IO overall: | 20260508 01:55:47.209 | INFO | BytesWritten : 1956738934801 | 20260508 01:55:47.209 | INFO | NumEvents : 946252294338 | 20260508 01:55:48.635 | INFO | Analyzing chunk frames = [4325376,4325888) | 20260508 01:55:48.635 | INFO | MemInfo: { | 20260508 01:55:48.635 | INFO | "gpu" : " 17 GB", | 20260508 01:55:48.635 | INFO | "huge" : "", | 20260508 01:55:48.635 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:48.635 | INFO | "malloc" : " 14 GB", | 20260508 01:55:48.635 | INFO | "pinned" : " 29 GB" | 20260508 01:55:48.635 | INFO | } | 20260508 01:55:48.635 | INFO | Perf report for the last 5.13986 seconds: | 20260508 01:55:48.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:48.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:48.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:48.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:48.635 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 01:55:48.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:48.635 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260508 01:55:48.635 | INFO | PRE_HQ : 65.4%, 1.7, 56.4ms, 0 | 20260508 01:55:48.635 | INFO | BAZWRITER : 70.0%, 3.7, 133.5ms, 0 | 20260508 01:55:48.635 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260508 01:55:48.636 | INFO | Uploaded 36GiB to GPU in 2.57448s (13.9834GiB/s) | 20260508 01:55:48.636 | INFO | Downloaded 38GiB to GPU in 2.68505s (14.1524GiB/s) | 20260508 01:55:53.768 | INFO | Analyzing chunk frames = [4325888,4326400) | 20260508 01:55:53.768 | INFO | MemInfo: { | 20260508 01:55:53.768 | INFO | "gpu" : " 17 GB", | 20260508 01:55:53.768 | INFO | "huge" : "", | 20260508 01:55:53.768 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:53.768 | INFO | "malloc" : " 14 GB", | 20260508 01:55:53.768 | INFO | "pinned" : " 29 GB" | 20260508 01:55:53.768 | INFO | } | 20260508 01:55:53.768 | INFO | Perf report for the last 5.13301 seconds: | 20260508 01:55:53.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:53.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.768 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 01:55:53.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.768 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 01:55:53.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:53.769 | INFO | Uploaded 36GiB to GPU in 2.54249s (14.1593GiB/s) | 20260508 01:55:53.769 | INFO | Downloaded 34GiB to GPU in 2.42597s (14.015GiB/s) | 20260508 01:55:58.908 | INFO | Analyzing chunk frames = [4326400,4326912) | 20260508 01:55:58.908 | INFO | MemInfo: { | 20260508 01:55:58.908 | INFO | "gpu" : " 17 GB", | 20260508 01:55:58.908 | INFO | "huge" : "", | 20260508 01:55:58.908 | INFO | "huge_shared" : " 29 GB", | 20260508 01:55:58.908 | INFO | "malloc" : " 14 GB", | 20260508 01:55:58.908 | INFO | "pinned" : " 29 GB" | 20260508 01:55:58.908 | INFO | } | 20260508 01:55:58.908 | INFO | Perf report for the last 5.14016 seconds: | 20260508 01:55:58.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:55:58.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.908 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 01:55:58.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.908 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 01:55:58.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:55:58.909 | INFO | Uploaded 36GiB to GPU in 2.49754s (14.4142GiB/s) | 20260508 01:55:58.909 | INFO | Downloaded 34GiB to GPU in 2.48101s (13.7041GiB/s) | 20260508 01:56:04.054 | INFO | Analyzing chunk frames = [4326912,4327424) | 20260508 01:56:04.054 | INFO | MemInfo: { | 20260508 01:56:04.054 | INFO | "gpu" : " 17 GB", | 20260508 01:56:04.054 | INFO | "huge" : "", | 20260508 01:56:04.054 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:04.054 | INFO | "malloc" : " 14 GB", | 20260508 01:56:04.054 | INFO | "pinned" : " 29 GB" | 20260508 01:56:04.054 | INFO | } | 20260508 01:56:04.055 | INFO | Perf report for the last 5.1461 seconds: | 20260508 01:56:04.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:04.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 01:56:04.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:56:04.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:04.055 | INFO | Uploaded 36GiB to GPU in 2.51642s (14.306GiB/s) | 20260508 01:56:04.055 | INFO | Downloaded 34GiB to GPU in 2.46714s (13.7811GiB/s) | 20260508 01:56:09.195 | INFO | Analyzing chunk frames = [4327424,4327936) | 20260508 01:56:09.195 | INFO | MemInfo: { | 20260508 01:56:09.195 | INFO | "gpu" : " 17 GB", | 20260508 01:56:09.195 | INFO | "huge" : "", | 20260508 01:56:09.195 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:09.195 | INFO | "malloc" : " 14 GB", | 20260508 01:56:09.195 | INFO | "pinned" : " 29 GB" | 20260508 01:56:09.195 | INFO | } | 20260508 01:56:09.195 | INFO | Perf report for the last 5.14028 seconds: | 20260508 01:56:09.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:09.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 01:56:09.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 01:56:09.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:09.195 | INFO | Uploaded 36GiB to GPU in 2.4138s (14.9142GiB/s) | 20260508 01:56:09.195 | INFO | Downloaded 34GiB to GPU in 2.5108s (13.5415GiB/s) | 20260508 01:56:14.336 | INFO | Analyzing chunk frames = [4327936,4328448) | 20260508 01:56:14.336 | INFO | MemInfo: { | 20260508 01:56:14.336 | INFO | "gpu" : " 17 GB", | 20260508 01:56:14.336 | INFO | "huge" : "", | 20260508 01:56:14.336 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:14.336 | INFO | "malloc" : " 14 GB", | 20260508 01:56:14.336 | INFO | "pinned" : " 29 GB" | 20260508 01:56:14.336 | INFO | } | 20260508 01:56:14.336 | INFO | Perf report for the last 5.14129 seconds: | 20260508 01:56:14.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:14.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 01:56:14.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 01:56:14.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:14.336 | INFO | Uploaded 36GiB to GPU in 2.40166s (14.9896GiB/s) | 20260508 01:56:14.336 | INFO | Downloaded 34GiB to GPU in 2.4122s (14.095GiB/s) | 20260508 01:56:19.469 | INFO | Analyzing chunk frames = [4328448,4328960) | 20260508 01:56:19.469 | INFO | MemInfo: { | 20260508 01:56:19.469 | INFO | "gpu" : " 17 GB", | 20260508 01:56:19.469 | INFO | "huge" : "", | 20260508 01:56:19.469 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:19.469 | INFO | "malloc" : " 14 GB", | 20260508 01:56:19.469 | INFO | "pinned" : " 29 GB" | 20260508 01:56:19.469 | INFO | } | 20260508 01:56:19.469 | INFO | Perf report for the last 5.13299 seconds: | 20260508 01:56:19.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:19.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:56:19.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:56:19.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:19.469 | INFO | Uploaded 36GiB to GPU in 2.44447s (14.7271GiB/s) | 20260508 01:56:19.469 | INFO | Downloaded 34GiB to GPU in 2.42367s (14.0283GiB/s) | 20260508 01:56:24.612 | INFO | Analyzing chunk frames = [4328960,4329472) | 20260508 01:56:24.612 | INFO | MemInfo: { | 20260508 01:56:24.612 | INFO | "gpu" : " 17 GB", | 20260508 01:56:24.612 | INFO | "huge" : "", | 20260508 01:56:24.612 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:24.612 | INFO | "malloc" : " 14 GB", | 20260508 01:56:24.612 | INFO | "pinned" : " 29 GB" | 20260508 01:56:24.612 | INFO | } | 20260508 01:56:24.612 | INFO | Perf report for the last 5.14295 seconds: | 20260508 01:56:24.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:24.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.612 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 01:56:24.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.612 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 01:56:24.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:24.613 | INFO | Uploaded 36GiB to GPU in 2.54181s (14.1631GiB/s) | 20260508 01:56:24.613 | INFO | Downloaded 34GiB to GPU in 2.46256s (13.8068GiB/s) | 20260508 01:56:24.709 | INFO | Starting a round of Baz IO | 20260508 01:56:28.148 | INFO | Populating RTMetrics report | 20260508 01:56:28.148 | INFO | RTMetrics report populated | 20260508 01:56:28.148 | INFO | Opening tmp RTMetrics json file | 20260508 01:56:28.157 | INFO | Serializing RTMetrics to json | 20260508 01:56:28.171 | INFO | Serialization done, swapping tmp file now... | 20260508 01:56:28.172 | INFO | RTMetrics done | 20260508 01:56:28.251 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.541 seconds | 20260508 01:56:28.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.68% / 0.00% / 58.31% | 20260508 01:56:28.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.7894% / 0% / 5.96046e-05% | 20260508 01:56:28.251 | INFO | Baz IO overall: | 20260508 01:56:28.251 | INFO | BytesWritten : 1958513556971 | 20260508 01:56:28.251 | INFO | NumEvents : 947104460780 | 20260508 01:56:29.750 | INFO | Analyzing chunk frames = [4329472,4329984) | 20260508 01:56:29.750 | INFO | MemInfo: { | 20260508 01:56:29.750 | INFO | "gpu" : " 17 GB", | 20260508 01:56:29.750 | INFO | "huge" : "", | 20260508 01:56:29.750 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:29.750 | INFO | "malloc" : " 14 GB", | 20260508 01:56:29.750 | INFO | "pinned" : " 29 GB" | 20260508 01:56:29.750 | INFO | } | 20260508 01:56:29.751 | INFO | Perf report for the last 5.13849 seconds: | 20260508 01:56:29.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:29.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:29.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:29.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:29.751 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 01:56:29.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:29.751 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:56:29.751 | INFO | PRE_HQ : 65.6%, 1.7, 56.3ms, 0 | 20260508 01:56:29.751 | INFO | BAZWRITER : 68.5%, 4.1, 145.4ms, 0 | 20260508 01:56:29.751 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 01:56:29.751 | INFO | Uploaded 36GiB to GPU in 2.56883s (14.0142GiB/s) | 20260508 01:56:29.751 | INFO | Downloaded 38GiB to GPU in 2.70078s (14.07GiB/s) | 20260508 01:56:34.893 | INFO | Analyzing chunk frames = [4329984,4330496) | 20260508 01:56:34.893 | INFO | MemInfo: { | 20260508 01:56:34.893 | INFO | "gpu" : " 17 GB", | 20260508 01:56:34.893 | INFO | "huge" : "", | 20260508 01:56:34.893 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:34.893 | INFO | "malloc" : " 14 GB", | 20260508 01:56:34.893 | INFO | "pinned" : " 29 GB" | 20260508 01:56:34.893 | INFO | } | 20260508 01:56:34.893 | INFO | Perf report for the last 5.14253 seconds: | 20260508 01:56:34.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:34.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 01:56:34.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 01:56:34.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:34.893 | INFO | Uploaded 36GiB to GPU in 2.50644s (14.363GiB/s) | 20260508 01:56:34.893 | INFO | Downloaded 34GiB to GPU in 2.4426s (13.9196GiB/s) | 20260508 01:56:40.029 | INFO | Analyzing chunk frames = [4330496,4331008) | 20260508 01:56:40.029 | INFO | MemInfo: { | 20260508 01:56:40.029 | INFO | "gpu" : " 17 GB", | 20260508 01:56:40.029 | INFO | "huge" : "", | 20260508 01:56:40.029 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:40.029 | INFO | "malloc" : " 14 GB", | 20260508 01:56:40.029 | INFO | "pinned" : " 29 GB" | 20260508 01:56:40.029 | INFO | } | 20260508 01:56:40.029 | INFO | Perf report for the last 5.13603 seconds: | 20260508 01:56:40.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:40.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 01:56:40.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:56:40.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:40.029 | INFO | Uploaded 36GiB to GPU in 2.48247s (14.5017GiB/s) | 20260508 01:56:40.030 | INFO | Downloaded 34GiB to GPU in 2.49289s (13.6388GiB/s) | 20260508 01:56:45.165 | INFO | Analyzing chunk frames = [4331008,4331520) | 20260508 01:56:45.165 | INFO | MemInfo: { | 20260508 01:56:45.165 | INFO | "gpu" : " 17 GB", | 20260508 01:56:45.165 | INFO | "huge" : "", | 20260508 01:56:45.165 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:45.165 | INFO | "malloc" : " 14 GB", | 20260508 01:56:45.165 | INFO | "pinned" : " 29 GB" | 20260508 01:56:45.165 | INFO | } | 20260508 01:56:45.166 | INFO | Perf report for the last 5.13634 seconds: | 20260508 01:56:45.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:45.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | ANALYSIS : 66.8%, 3.0, 103.5ms, 0 | 20260508 01:56:45.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:56:45.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:45.166 | INFO | Uploaded 36GiB to GPU in 2.40407s (14.9746GiB/s) | 20260508 01:56:45.167 | INFO | Downloaded 34GiB to GPU in 2.49407s (13.6323GiB/s) | 20260508 01:56:50.310 | INFO | Analyzing chunk frames = [4331520,4332032) | 20260508 01:56:50.310 | INFO | MemInfo: { | 20260508 01:56:50.310 | INFO | "gpu" : " 17 GB", | 20260508 01:56:50.310 | INFO | "huge" : "", | 20260508 01:56:50.310 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:50.310 | INFO | "malloc" : " 14 GB", | 20260508 01:56:50.310 | INFO | "pinned" : " 29 GB" | 20260508 01:56:50.310 | INFO | } | 20260508 01:56:50.310 | INFO | Perf report for the last 5.14441 seconds: | 20260508 01:56:50.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:50.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 01:56:50.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 01:56:50.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:50.310 | INFO | Uploaded 36GiB to GPU in 2.39113s (15.0556GiB/s) | 20260508 01:56:50.311 | INFO | Downloaded 34GiB to GPU in 2.56805s (13.2396GiB/s) | 20260508 01:56:55.452 | INFO | Analyzing chunk frames = [4332032,4332544) | 20260508 01:56:55.452 | INFO | MemInfo: { | 20260508 01:56:55.452 | INFO | "gpu" : " 17 GB", | 20260508 01:56:55.452 | INFO | "huge" : "", | 20260508 01:56:55.452 | INFO | "huge_shared" : " 29 GB", | 20260508 01:56:55.452 | INFO | "malloc" : " 14 GB", | 20260508 01:56:55.452 | INFO | "pinned" : " 29 GB" | 20260508 01:56:55.452 | INFO | } | 20260508 01:56:55.452 | INFO | Perf report for the last 5.14191 seconds: | 20260508 01:56:55.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:56:55.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:56:55.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:56:55.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:56:55.452 | INFO | Uploaded 36GiB to GPU in 2.40142s (14.9911GiB/s) | 20260508 01:56:55.452 | INFO | Downloaded 34GiB to GPU in 2.41827s (14.0596GiB/s) | 20260508 01:57:00.583 | INFO | Analyzing chunk frames = [4332544,4333056) | 20260508 01:57:00.583 | INFO | MemInfo: { | 20260508 01:57:00.583 | INFO | "gpu" : " 17 GB", | 20260508 01:57:00.583 | INFO | "huge" : "", | 20260508 01:57:00.586 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:00.586 | INFO | "malloc" : " 14 GB", | 20260508 01:57:00.586 | INFO | "pinned" : " 29 GB" | 20260508 01:57:00.586 | INFO | } | 20260508 01:57:00.586 | INFO | Perf report for the last 5.1343 seconds: | 20260508 01:57:00.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:00.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 01:57:00.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:57:00.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:00.586 | INFO | Uploaded 36GiB to GPU in 2.45465s (14.666GiB/s) | 20260508 01:57:00.586 | INFO | Downloaded 34GiB to GPU in 2.43964s (13.9365GiB/s) | 20260508 01:57:05.723 | INFO | Analyzing chunk frames = [4333056,4333568) | 20260508 01:57:05.723 | INFO | MemInfo: { | 20260508 01:57:05.723 | INFO | "gpu" : " 17 GB", | 20260508 01:57:05.723 | INFO | "huge" : "", | 20260508 01:57:05.723 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:05.723 | INFO | "malloc" : " 14 GB", | 20260508 01:57:05.723 | INFO | "pinned" : " 29 GB" | 20260508 01:57:05.723 | INFO | } | 20260508 01:57:05.723 | INFO | Perf report for the last 5.13697 seconds: | 20260508 01:57:05.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:05.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:57:05.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 01:57:05.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:05.723 | INFO | Uploaded 36GiB to GPU in 2.53792s (14.1849GiB/s) | 20260508 01:57:05.723 | INFO | Downloaded 34GiB to GPU in 2.45885s (13.8276GiB/s) | 20260508 01:57:05.723 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:57:05.723 | INFO | Total mem usage: 16.7 GiB | 20260508 01:57:05.723 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:57:05.723 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 01:57:05.724 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:57:05.724 | INFO | High Water Breakdown: | 20260508 01:57:05.724 | INFO | 60% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:57:05.724 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:57:05.724 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:57:05.724 | INFO | ------------------------------------------ | 20260508 01:57:05.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:57:05.724 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:57:05.724 | INFO | Total mem usage: 26.5 GiB | 20260508 01:57:05.724 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:57:05.724 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:57:05.724 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:57:05.724 | INFO | High Water Breakdown: | 20260508 01:57:05.724 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:57:05.724 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:57:05.724 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:57:05.724 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:57:05.724 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:57:05.724 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:57:05.724 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:57:05.724 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:57:05.724 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:57:05.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:57:05.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:57:05.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:57:05.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:57:05.724 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:57:05.724 | INFO | ------------------------------------------ | 20260508 01:57:05.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:57:05.724 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:57:05.724 | INFO | Total mem usage: 27.8 GiB | 20260508 01:57:05.724 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:57:05.724 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:57:05.724 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:57:05.724 | INFO | High Water Breakdown: | 20260508 01:57:05.724 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:57:05.724 | INFO | ------------------------------------------ | 20260508 01:57:05.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:57:05.724 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:57:05.724 | INFO | Total mem usage: 15.7 GiB | 20260508 01:57:05.724 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:57:05.724 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:57:05.724 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:57:05.724 | INFO | High Water Breakdown: | 20260508 01:57:05.724 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:57:05.724 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:57:05.724 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:57:05.724 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:57:05.724 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:57:05.724 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:57:05.724 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:57:05.724 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:57:05.724 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:57:05.724 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:57:05.724 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:57:05.724 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:57:05.724 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:57:05.724 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:57:05.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:57:05.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:57:05.724 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:57:05.724 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:57:05.724 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:57:05.724 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:57:05.724 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:57:05.724 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:57:05.724 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:57:05.724 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:57:05.724 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:57:05.724 | INFO | ------------------------------------------ | 20260508 01:57:05.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:57:05.724 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:57:05.724 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:57:05.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:57:05.724 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.39e+01 / 1.37e+00 | 20260508 01:57:05.724 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.01e-01, 1.14e-02 / 5.27e-03 | 20260508 01:57:05.724 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.44e+00, 5.88e-01 / 2.14e-01 | 20260508 01:57:05.724 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.05e-01 / 1.04e-01 | 20260508 01:57:05.724 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.59e+01, 4.05e+00 / 1.96e+00 | 20260508 01:57:05.724 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.68e+00 / 1.68e+00 | 20260508 01:57:05.724 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.42e+00 / 2.70e+00, 4.24e-01 / 2.34e-01 | 20260508 01:57:05.724 | INFO | PerfReport: 3.30%, Metrics -- 1.08e+00 / 1.26e+00, 2.65e-01 / 1.38e-01 | 20260508 01:57:05.724 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.32e-01 / 3.68e-01, 5.27e-02 / 2.74e-02 | 20260508 01:57:05.724 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:57:05.724 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:57:05.725 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:57:05.725 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 24.26%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 2.05%, LockUpload -- 2.19e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 2.92%, LockDownload -- 3.11e+00 / 0.00e+00, 4.40e+00 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 01:57:05.725 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 3.68e-02 / 0.00e+00 | 20260508 01:57:05.820 | INFO | Starting a round of Baz IO | 20260508 01:57:09.267 | INFO | Populating RTMetrics report | 20260508 01:57:09.267 | INFO | RTMetrics report populated | 20260508 01:57:09.267 | INFO | Opening tmp RTMetrics json file | 20260508 01:57:09.312 | INFO | Serializing RTMetrics to json | 20260508 01:57:09.348 | INFO | Serialization done, swapping tmp file now... | 20260508 01:57:09.352 | INFO | RTMetrics done | 20260508 01:57:09.374 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.553 seconds | 20260508 01:57:09.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.68% / 0.00% / 58.32% | 20260508 01:57:09.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.788629% / 0% / 6.35783e-05% | 20260508 01:57:09.374 | INFO | Baz IO overall: | 20260508 01:57:09.374 | INFO | BytesWritten : 1960288401160 | 20260508 01:57:09.374 | INFO | NumEvents : 947956949754 | 20260508 01:57:10.861 | INFO | Analyzing chunk frames = [4333568,4334080) | 20260508 01:57:10.861 | INFO | MemInfo: { | 20260508 01:57:10.861 | INFO | "gpu" : " 17 GB", | 20260508 01:57:10.861 | INFO | "huge" : "", | 20260508 01:57:10.861 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:10.861 | INFO | "malloc" : " 14 GB", | 20260508 01:57:10.861 | INFO | "pinned" : " 29 GB" | 20260508 01:57:10.861 | INFO | } | 20260508 01:57:10.861 | INFO | Perf report for the last 5.13804 seconds: | 20260508 01:57:10.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:10.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:10.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:10.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:10.861 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 01:57:10.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:10.861 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260508 01:57:10.861 | INFO | PRE_HQ : 65.8%, 1.7, 56.4ms, 0 | 20260508 01:57:10.861 | INFO | BAZWRITER : 68.7%, 4.2, 147.3ms, 0 | 20260508 01:57:10.861 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 01:57:10.862 | INFO | Uploaded 36GiB to GPU in 2.56233s (14.0497GiB/s) | 20260508 01:57:10.862 | INFO | Downloaded 38GiB to GPU in 2.68133s (14.1721GiB/s) | 20260508 01:57:16.007 | INFO | Analyzing chunk frames = [4334080,4334592) | 20260508 01:57:16.007 | INFO | MemInfo: { | 20260508 01:57:16.007 | INFO | "gpu" : " 17 GB", | 20260508 01:57:16.007 | INFO | "huge" : "", | 20260508 01:57:16.008 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:16.008 | INFO | "malloc" : " 14 GB", | 20260508 01:57:16.008 | INFO | "pinned" : " 29 GB" | 20260508 01:57:16.008 | INFO | } | 20260508 01:57:16.008 | INFO | Perf report for the last 5.14649 seconds: | 20260508 01:57:16.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:16.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 01:57:16.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:57:16.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:16.008 | INFO | Uploaded 36GiB to GPU in 2.52043s (14.2833GiB/s) | 20260508 01:57:16.009 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.975GiB/s) | 20260508 01:57:21.151 | INFO | Analyzing chunk frames = [4334592,4335104) | 20260508 01:57:21.152 | INFO | MemInfo: { | 20260508 01:57:21.152 | INFO | "gpu" : " 17 GB", | 20260508 01:57:21.152 | INFO | "huge" : "", | 20260508 01:57:21.152 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:21.152 | INFO | "malloc" : " 14 GB", | 20260508 01:57:21.152 | INFO | "pinned" : " 29 GB" | 20260508 01:57:21.152 | INFO | } | 20260508 01:57:21.152 | INFO | Perf report for the last 5.14424 seconds: | 20260508 01:57:21.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:21.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.152 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 01:57:21.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.152 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 01:57:21.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:21.153 | INFO | Uploaded 36GiB to GPU in 2.57579s (13.9763GiB/s) | 20260508 01:57:21.153 | INFO | Downloaded 34GiB to GPU in 2.50397s (13.5785GiB/s) | 20260508 01:57:26.287 | INFO | Analyzing chunk frames = [4335104,4335616) | 20260508 01:57:26.287 | INFO | MemInfo: { | 20260508 01:57:26.287 | INFO | "gpu" : " 17 GB", | 20260508 01:57:26.287 | INFO | "huge" : "", | 20260508 01:57:26.287 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:26.287 | INFO | "malloc" : " 14 GB", | 20260508 01:57:26.288 | INFO | "pinned" : " 29 GB" | 20260508 01:57:26.288 | INFO | } | 20260508 01:57:26.288 | INFO | Perf report for the last 5.1356 seconds: | 20260508 01:57:26.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:26.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 01:57:26.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 01:57:26.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:26.288 | INFO | Uploaded 36GiB to GPU in 2.55293s (14.1015GiB/s) | 20260508 01:57:26.288 | INFO | Downloaded 34GiB to GPU in 2.50686s (13.5628GiB/s) | 20260508 01:57:31.421 | INFO | Analyzing chunk frames = [4335616,4336128) | 20260508 01:57:31.421 | INFO | MemInfo: { | 20260508 01:57:31.421 | INFO | "gpu" : " 17 GB", | 20260508 01:57:31.421 | INFO | "huge" : "", | 20260508 01:57:31.421 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:31.421 | INFO | "malloc" : " 14 GB", | 20260508 01:57:31.422 | INFO | "pinned" : " 29 GB" | 20260508 01:57:31.422 | INFO | } | 20260508 01:57:31.422 | INFO | Perf report for the last 5.13399 seconds: | 20260508 01:57:31.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:31.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 01:57:31.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 01:57:31.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:31.422 | INFO | Uploaded 36GiB to GPU in 2.40802s (14.9501GiB/s) | 20260508 01:57:31.422 | INFO | Downloaded 34GiB to GPU in 2.56302s (13.2656GiB/s) | 20260508 01:57:36.561 | INFO | Analyzing chunk frames = [4336128,4336640) | 20260508 01:57:36.561 | INFO | MemInfo: { | 20260508 01:57:36.561 | INFO | "gpu" : " 17 GB", | 20260508 01:57:36.561 | INFO | "huge" : "", | 20260508 01:57:36.561 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:36.561 | INFO | "malloc" : " 14 GB", | 20260508 01:57:36.561 | INFO | "pinned" : " 29 GB" | 20260508 01:57:36.561 | INFO | } | 20260508 01:57:36.561 | INFO | Perf report for the last 5.13949 seconds: | 20260508 01:57:36.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:36.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:57:36.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:57:36.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:36.561 | INFO | Uploaded 36GiB to GPU in 2.38982s (15.0639GiB/s) | 20260508 01:57:36.561 | INFO | Downloaded 34GiB to GPU in 2.42171s (14.0397GiB/s) | 20260508 01:57:41.703 | INFO | Analyzing chunk frames = [4336640,4337152) | 20260508 01:57:41.703 | INFO | MemInfo: { | 20260508 01:57:41.703 | INFO | "gpu" : " 17 GB", | 20260508 01:57:41.703 | INFO | "huge" : "", | 20260508 01:57:41.703 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:41.704 | INFO | "malloc" : " 14 GB", | 20260508 01:57:41.704 | INFO | "pinned" : " 29 GB" | 20260508 01:57:41.704 | INFO | } | 20260508 01:57:41.704 | INFO | Perf report for the last 5.14253 seconds: | 20260508 01:57:41.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:41.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 01:57:41.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:57:41.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:41.704 | INFO | Uploaded 36GiB to GPU in 2.46314s (14.6155GiB/s) | 20260508 01:57:41.704 | INFO | Downloaded 34GiB to GPU in 2.44134s (13.9268GiB/s) | 20260508 01:57:46.836 | INFO | Analyzing chunk frames = [4337152,4337664) | 20260508 01:57:46.836 | INFO | MemInfo: { | 20260508 01:57:46.836 | INFO | "gpu" : " 17 GB", | 20260508 01:57:46.836 | INFO | "huge" : "", | 20260508 01:57:46.836 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:46.836 | INFO | "malloc" : " 14 GB", | 20260508 01:57:46.836 | INFO | "pinned" : " 29 GB" | 20260508 01:57:46.836 | INFO | } | 20260508 01:57:46.836 | INFO | Perf report for the last 5.13272 seconds: | 20260508 01:57:46.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:46.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.836 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:57:46.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.836 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:57:46.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:46.837 | INFO | Uploaded 36GiB to GPU in 2.54544s (14.1429GiB/s) | 20260508 01:57:46.837 | INFO | Downloaded 34GiB to GPU in 2.45912s (13.8261GiB/s) | 20260508 01:57:46.934 | INFO | Starting a round of Baz IO | 20260508 01:57:50.390 | INFO | Populating RTMetrics report | 20260508 01:57:50.391 | INFO | RTMetrics report populated | 20260508 01:57:50.391 | INFO | Opening tmp RTMetrics json file | 20260508 01:57:50.443 | INFO | Serializing RTMetrics to json | 20260508 01:57:50.489 | INFO | Serialization done, swapping tmp file now... | 20260508 01:57:50.492 | INFO | RTMetrics done | 20260508 01:57:50.636 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.702 seconds | 20260508 01:57:50.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.68% / 0.00% / 58.32% | 20260508 01:57:50.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.785863% / 0% / 5.16574e-05% | 20260508 01:57:50.636 | INFO | Baz IO overall: | 20260508 01:57:50.636 | INFO | BytesWritten : 1962061566815 | 20260508 01:57:50.636 | INFO | NumEvents : 948808268082 | 20260508 01:57:51.975 | INFO | Analyzing chunk frames = [4337664,4338176) | 20260508 01:57:51.975 | INFO | MemInfo: { | 20260508 01:57:51.975 | INFO | "gpu" : " 17 GB", | 20260508 01:57:51.975 | INFO | "huge" : "", | 20260508 01:57:51.975 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:51.976 | INFO | "malloc" : " 14 GB", | 20260508 01:57:51.976 | INFO | "pinned" : " 29 GB" | 20260508 01:57:51.976 | INFO | } | 20260508 01:57:51.976 | INFO | Perf report for the last 5.13927 seconds: | 20260508 01:57:51.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:51.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:51.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:51.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:51.976 | INFO | ANALYSIS : 68.5%, 3.0, 106.6ms, 0 | 20260508 01:57:51.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:51.976 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260508 01:57:51.976 | INFO | PRE_HQ : 65.5%, 1.7, 56.2ms, 0 | 20260508 01:57:51.976 | INFO | BAZWRITER : 71.5%, 4.1, 151.3ms, 0 | 20260508 01:57:51.976 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 01:57:51.976 | INFO | Uploaded 36GiB to GPU in 2.57187s (13.9976GiB/s) | 20260508 01:57:51.976 | INFO | Downloaded 38GiB to GPU in 2.68926s (14.1303GiB/s) | 20260508 01:57:57.117 | INFO | Analyzing chunk frames = [4338176,4338688) | 20260508 01:57:57.117 | INFO | MemInfo: { | 20260508 01:57:57.117 | INFO | "gpu" : " 17 GB", | 20260508 01:57:57.117 | INFO | "huge" : "", | 20260508 01:57:57.117 | INFO | "huge_shared" : " 29 GB", | 20260508 01:57:57.117 | INFO | "malloc" : " 14 GB", | 20260508 01:57:57.117 | INFO | "pinned" : " 29 GB" | 20260508 01:57:57.117 | INFO | } | 20260508 01:57:57.117 | INFO | Perf report for the last 5.14158 seconds: | 20260508 01:57:57.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:57:57.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.117 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 01:57:57.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.117 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 01:57:57.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:57:57.118 | INFO | Uploaded 36GiB to GPU in 2.538s (14.1844GiB/s) | 20260508 01:57:57.118 | INFO | Downloaded 34GiB to GPU in 2.43786s (13.9467GiB/s) | 20260508 01:58:02.261 | INFO | Analyzing chunk frames = [4338688,4339200) | 20260508 01:58:02.261 | INFO | MemInfo: { | 20260508 01:58:02.261 | INFO | "gpu" : " 17 GB", | 20260508 01:58:02.261 | INFO | "huge" : "", | 20260508 01:58:02.261 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:02.261 | INFO | "malloc" : " 14 GB", | 20260508 01:58:02.261 | INFO | "pinned" : " 29 GB" | 20260508 01:58:02.261 | INFO | } | 20260508 01:58:02.261 | INFO | Perf report for the last 5.14419 seconds: | 20260508 01:58:02.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:02.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.262 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 01:58:02.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.262 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 01:58:02.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:02.262 | INFO | Uploaded 36GiB to GPU in 2.53461s (14.2034GiB/s) | 20260508 01:58:02.262 | INFO | Downloaded 34GiB to GPU in 2.49924s (13.6041GiB/s) | 20260508 01:58:07.403 | INFO | Analyzing chunk frames = [4339200,4339712) | 20260508 01:58:07.403 | INFO | MemInfo: { | 20260508 01:58:07.403 | INFO | "gpu" : " 17 GB", | 20260508 01:58:07.403 | INFO | "huge" : "", | 20260508 01:58:07.403 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:07.403 | INFO | "malloc" : " 14 GB", | 20260508 01:58:07.403 | INFO | "pinned" : " 29 GB" | 20260508 01:58:07.403 | INFO | } | 20260508 01:58:07.403 | INFO | Perf report for the last 5.14134 seconds: | 20260508 01:58:07.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:07.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.403 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 01:58:07.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.403 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:58:07.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:07.404 | INFO | Uploaded 36GiB to GPU in 2.53247s (14.2154GiB/s) | 20260508 01:58:07.404 | INFO | Downloaded 34GiB to GPU in 2.49175s (13.645GiB/s) | 20260508 01:58:12.535 | INFO | Analyzing chunk frames = [4339712,4340224) | 20260508 01:58:12.535 | INFO | MemInfo: { | 20260508 01:58:12.535 | INFO | "gpu" : " 17 GB", | 20260508 01:58:12.535 | INFO | "huge" : "", | 20260508 01:58:12.535 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:12.535 | INFO | "malloc" : " 14 GB", | 20260508 01:58:12.535 | INFO | "pinned" : " 29 GB" | 20260508 01:58:12.535 | INFO | } | 20260508 01:58:12.535 | INFO | Perf report for the last 5.13265 seconds: | 20260508 01:58:12.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:12.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.536 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 01:58:12.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.536 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 01:58:12.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:12.536 | INFO | Uploaded 36GiB to GPU in 2.39355s (15.0404GiB/s) | 20260508 01:58:12.536 | INFO | Downloaded 34GiB to GPU in 2.54782s (13.3447GiB/s) | 20260508 01:58:17.674 | INFO | Analyzing chunk frames = [4340224,4340736) | 20260508 01:58:17.674 | INFO | MemInfo: { | 20260508 01:58:17.674 | INFO | "gpu" : " 17 GB", | 20260508 01:58:17.674 | INFO | "huge" : "", | 20260508 01:58:17.674 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:17.674 | INFO | "malloc" : " 14 GB", | 20260508 01:58:17.675 | INFO | "pinned" : " 29 GB" | 20260508 01:58:17.675 | INFO | } | 20260508 01:58:17.675 | INFO | Perf report for the last 5.13914 seconds: | 20260508 01:58:17.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:17.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:58:17.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:58:17.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:17.675 | INFO | Uploaded 36GiB to GPU in 2.39872s (15.008GiB/s) | 20260508 01:58:17.675 | INFO | Downloaded 34GiB to GPU in 2.42445s (14.0238GiB/s) | 20260508 01:58:22.817 | INFO | Analyzing chunk frames = [4340736,4341248) | 20260508 01:58:22.817 | INFO | MemInfo: { | 20260508 01:58:22.817 | INFO | "gpu" : " 17 GB", | 20260508 01:58:22.817 | INFO | "huge" : "", | 20260508 01:58:22.817 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:22.817 | INFO | "malloc" : " 14 GB", | 20260508 01:58:22.817 | INFO | "pinned" : " 29 GB" | 20260508 01:58:22.817 | INFO | } | 20260508 01:58:22.817 | INFO | Perf report for the last 5.14251 seconds: | 20260508 01:58:22.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:22.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 01:58:22.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:58:22.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:22.817 | INFO | Uploaded 36GiB to GPU in 2.4189s (14.8828GiB/s) | 20260508 01:58:22.817 | INFO | Downloaded 34GiB to GPU in 2.42985s (13.9926GiB/s) | 20260508 01:58:27.955 | INFO | Analyzing chunk frames = [4341248,4341760) | 20260508 01:58:27.955 | INFO | MemInfo: { | 20260508 01:58:27.955 | INFO | "gpu" : " 17 GB", | 20260508 01:58:27.955 | INFO | "huge" : "", | 20260508 01:58:27.955 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:27.955 | INFO | "malloc" : " 14 GB", | 20260508 01:58:27.955 | INFO | "pinned" : " 29 GB" | 20260508 01:58:27.955 | INFO | } | 20260508 01:58:27.955 | INFO | Perf report for the last 5.13759 seconds: | 20260508 01:58:27.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:27.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:58:27.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:58:27.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:27.955 | INFO | Uploaded 36GiB to GPU in 2.54527s (14.1439GiB/s) | 20260508 01:58:27.955 | INFO | Downloaded 34GiB to GPU in 2.45795s (13.8326GiB/s) | 20260508 01:58:27.955 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:58:27.955 | INFO | Total mem usage: 16.7 GiB | 20260508 01:58:27.955 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:58:27.955 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.81%) | 20260508 01:58:27.955 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:58:27.955 | INFO | High Water Breakdown: | 20260508 01:58:27.955 | INFO | 60.8% : 9.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:58:27.955 | INFO | 39.1% : 6.05 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:58:27.955 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:58:27.956 | INFO | ------------------------------------------ | 20260508 01:58:27.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:58:27.956 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:58:27.956 | INFO | Total mem usage: 26.5 GiB | 20260508 01:58:27.956 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:58:27.956 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 01:58:27.956 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 01:58:27.956 | INFO | High Water Breakdown: | 20260508 01:58:27.956 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:58:27.956 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:58:27.956 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:58:27.956 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:58:27.956 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:58:27.956 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:58:27.956 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:58:27.956 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:58:27.956 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:58:27.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:58:27.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:58:27.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:58:27.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:58:27.956 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:58:27.956 | INFO | ------------------------------------------ | 20260508 01:58:27.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:58:27.956 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:58:27.956 | INFO | Total mem usage: 27.8 GiB | 20260508 01:58:27.956 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:58:27.956 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:58:27.956 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:58:27.956 | INFO | High Water Breakdown: | 20260508 01:58:27.956 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:58:27.956 | INFO | ------------------------------------------ | 20260508 01:58:27.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:58:27.956 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:58:27.956 | INFO | Total mem usage: 15.7 GiB | 20260508 01:58:27.956 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:58:27.956 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:58:27.956 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:58:27.956 | INFO | High Water Breakdown: | 20260508 01:58:27.956 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:58:27.956 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:58:27.956 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:58:27.956 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:58:27.956 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:58:27.956 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:58:27.956 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:58:27.956 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:58:27.956 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:58:27.956 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:58:27.956 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:58:27.956 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:58:27.956 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:58:27.956 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:58:27.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:58:27.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:58:27.956 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:58:27.956 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:58:27.956 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:58:27.956 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:58:27.956 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:58:27.956 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:58:27.956 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:58:27.956 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:58:27.956 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:58:27.956 | INFO | ------------------------------------------ | 20260508 01:58:27.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:58:27.956 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:58:27.956 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:58:27.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:58:27.956 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.65e+00 | 20260508 01:58:27.956 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.28e-02 / 4.99e-03 | 20260508 01:58:27.957 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.49e+00, 5.84e-01 / 1.64e-01 | 20260508 01:58:27.957 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.07e-01 / 1.03e-01 | 20260508 01:58:27.957 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.59e+01, 3.96e+00 / 1.86e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.64e+00 / 1.55e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.42e+00 / 2.73e+00, 4.20e-01 / 2.09e-01 | 20260508 01:58:27.957 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.29e+00, 2.69e-01 / 1.75e-01 | 20260508 01:58:27.957 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.31e-01 / 3.74e-01, 5.23e-02 / 2.72e-02 | 20260508 01:58:27.957 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:58:27.957 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:58:27.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:58:27.957 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 24.38%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 30.64%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 2.79%, LockDownload -- 2.98e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 15.61%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 01:58:27.957 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 01:58:28.052 | INFO | Starting a round of Baz IO | 20260508 01:58:31.505 | INFO | Populating RTMetrics report | 20260508 01:58:31.505 | INFO | RTMetrics report populated | 20260508 01:58:31.505 | INFO | Opening tmp RTMetrics json file | 20260508 01:58:31.523 | INFO | Serializing RTMetrics to json | 20260508 01:58:31.525 | INFO | Serialization done, swapping tmp file now... | 20260508 01:58:31.526 | INFO | RTMetrics done | 20260508 01:58:31.606 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.554 seconds | 20260508 01:58:31.606 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.68% / 0.00% / 58.32% | 20260508 01:58:31.606 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.782255% / 0% / 6.75519e-05% | 20260508 01:58:31.606 | INFO | Baz IO overall: | 20260508 01:58:31.606 | INFO | BytesWritten : 1963832846337 | 20260508 01:58:31.606 | INFO | NumEvents : 949658300757 | 20260508 01:58:33.099 | INFO | Analyzing chunk frames = [4341760,4342272) | 20260508 01:58:33.099 | INFO | MemInfo: { | 20260508 01:58:33.099 | INFO | "gpu" : " 17 GB", | 20260508 01:58:33.099 | INFO | "huge" : "", | 20260508 01:58:33.099 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:33.099 | INFO | "malloc" : " 14 GB", | 20260508 01:58:33.099 | INFO | "pinned" : " 29 GB" | 20260508 01:58:33.099 | INFO | } | 20260508 01:58:33.099 | INFO | Perf report for the last 5.14431 seconds: | 20260508 01:58:33.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:33.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:33.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:33.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:33.099 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 01:58:33.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:33.099 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 01:58:33.099 | INFO | PRE_HQ : 65.7%, 1.7, 56.6ms, 0 | 20260508 01:58:33.099 | INFO | BAZWRITER : 68.7%, 4.4, 156.3ms, 0 | 20260508 01:58:33.099 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 01:58:33.099 | INFO | Uploaded 36GiB to GPU in 2.57928s (13.9574GiB/s) | 20260508 01:58:33.099 | INFO | Downloaded 38GiB to GPU in 2.6989s (14.0798GiB/s) | 20260508 01:58:38.230 | INFO | Analyzing chunk frames = [4342272,4342784) | 20260508 01:58:38.231 | INFO | MemInfo: { | 20260508 01:58:38.231 | INFO | "gpu" : " 17 GB", | 20260508 01:58:38.231 | INFO | "huge" : "", | 20260508 01:58:38.231 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:38.231 | INFO | "malloc" : " 14 GB", | 20260508 01:58:38.231 | INFO | "pinned" : " 29 GB" | 20260508 01:58:38.231 | INFO | } | 20260508 01:58:38.231 | INFO | Perf report for the last 5.13169 seconds: | 20260508 01:58:38.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:38.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260508 01:58:38.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 01:58:38.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:38.231 | INFO | Uploaded 36GiB to GPU in 2.56351s (14.0432GiB/s) | 20260508 01:58:38.231 | INFO | Downloaded 34GiB to GPU in 2.43301s (13.9745GiB/s) | 20260508 01:58:43.374 | INFO | Analyzing chunk frames = [4342784,4343296) | 20260508 01:58:43.374 | INFO | MemInfo: { | 20260508 01:58:43.374 | INFO | "gpu" : " 17 GB", | 20260508 01:58:43.374 | INFO | "huge" : "", | 20260508 01:58:43.374 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:43.374 | INFO | "malloc" : " 14 GB", | 20260508 01:58:43.374 | INFO | "pinned" : " 29 GB" | 20260508 01:58:43.374 | INFO | } | 20260508 01:58:43.374 | INFO | Perf report for the last 5.14315 seconds: | 20260508 01:58:43.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:43.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.374 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 01:58:43.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.374 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 01:58:43.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:43.375 | INFO | Uploaded 36GiB to GPU in 2.53717s (14.189GiB/s) | 20260508 01:58:43.375 | INFO | Downloaded 34GiB to GPU in 2.49013s (13.6539GiB/s) | 20260508 01:58:48.515 | INFO | Analyzing chunk frames = [4343296,4343808) | 20260508 01:58:48.515 | INFO | MemInfo: { | 20260508 01:58:48.515 | INFO | "gpu" : " 17 GB", | 20260508 01:58:48.515 | INFO | "huge" : "", | 20260508 01:58:48.515 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:48.515 | INFO | "malloc" : " 14 GB", | 20260508 01:58:48.515 | INFO | "pinned" : " 29 GB" | 20260508 01:58:48.515 | INFO | } | 20260508 01:58:48.515 | INFO | Perf report for the last 5.14086 seconds: | 20260508 01:58:48.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:48.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 01:58:48.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:58:48.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:48.515 | INFO | Uploaded 36GiB to GPU in 2.4993s (14.4041GiB/s) | 20260508 01:58:48.515 | INFO | Downloaded 34GiB to GPU in 2.48561s (13.6787GiB/s) | 20260508 01:58:53.651 | INFO | Analyzing chunk frames = [4343808,4344320) | 20260508 01:58:53.651 | INFO | MemInfo: { | 20260508 01:58:53.651 | INFO | "gpu" : " 17 GB", | 20260508 01:58:53.651 | INFO | "huge" : "", | 20260508 01:58:53.651 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:53.651 | INFO | "malloc" : " 14 GB", | 20260508 01:58:53.651 | INFO | "pinned" : " 29 GB" | 20260508 01:58:53.651 | INFO | } | 20260508 01:58:53.651 | INFO | Perf report for the last 5.13617 seconds: | 20260508 01:58:53.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:53.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.651 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 01:58:53.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.651 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:58:53.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:53.652 | INFO | Uploaded 36GiB to GPU in 2.41055s (14.9343GiB/s) | 20260508 01:58:53.652 | INFO | Downloaded 34GiB to GPU in 2.56649s (13.2476GiB/s) | 20260508 01:58:58.788 | INFO | Analyzing chunk frames = [4344320,4344832) | 20260508 01:58:58.788 | INFO | MemInfo: { | 20260508 01:58:58.788 | INFO | "gpu" : " 17 GB", | 20260508 01:58:58.788 | INFO | "huge" : "", | 20260508 01:58:58.788 | INFO | "huge_shared" : " 29 GB", | 20260508 01:58:58.788 | INFO | "malloc" : " 14 GB", | 20260508 01:58:58.788 | INFO | "pinned" : " 29 GB" | 20260508 01:58:58.788 | INFO | } | 20260508 01:58:58.788 | INFO | Perf report for the last 5.13693 seconds: | 20260508 01:58:58.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:58:58.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 01:58:58.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 01:58:58.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:58:58.788 | INFO | Uploaded 36GiB to GPU in 2.41285s (14.9201GiB/s) | 20260508 01:58:58.789 | INFO | Downloaded 34GiB to GPU in 2.41005s (14.1076GiB/s) | 20260508 01:59:03.926 | INFO | Analyzing chunk frames = [4344832,4345344) | 20260508 01:59:03.926 | INFO | MemInfo: { | 20260508 01:59:03.926 | INFO | "gpu" : " 17 GB", | 20260508 01:59:03.926 | INFO | "huge" : "", | 20260508 01:59:03.926 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:03.926 | INFO | "malloc" : " 14 GB", | 20260508 01:59:03.926 | INFO | "pinned" : " 29 GB" | 20260508 01:59:03.926 | INFO | } | 20260508 01:59:03.926 | INFO | Perf report for the last 5.13838 seconds: | 20260508 01:59:03.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:03.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.926 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 01:59:03.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.927 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 01:59:03.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:03.927 | INFO | Uploaded 36GiB to GPU in 2.54985s (14.1185GiB/s) | 20260508 01:59:03.927 | INFO | Downloaded 34GiB to GPU in 2.47465s (13.7393GiB/s) | 20260508 01:59:09.071 | INFO | Analyzing chunk frames = [4345344,4345856) | 20260508 01:59:09.071 | INFO | MemInfo: { | 20260508 01:59:09.071 | INFO | "gpu" : " 17 GB", | 20260508 01:59:09.071 | INFO | "huge" : "", | 20260508 01:59:09.071 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:09.071 | INFO | "malloc" : " 14 GB", | 20260508 01:59:09.071 | INFO | "pinned" : " 29 GB" | 20260508 01:59:09.071 | INFO | } | 20260508 01:59:09.071 | INFO | Perf report for the last 5.14468 seconds: | 20260508 01:59:09.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:09.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.071 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 01:59:09.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.071 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 01:59:09.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:09.072 | INFO | Uploaded 36GiB to GPU in 2.52928s (14.2333GiB/s) | 20260508 01:59:09.072 | INFO | Downloaded 34GiB to GPU in 2.46527s (13.7916GiB/s) | 20260508 01:59:09.187 | INFO | Starting a round of Baz IO | 20260508 01:59:12.638 | INFO | Populating RTMetrics report | 20260508 01:59:12.638 | INFO | RTMetrics report populated | 20260508 01:59:12.638 | INFO | Opening tmp RTMetrics json file | 20260508 01:59:12.639 | INFO | Serializing RTMetrics to json | 20260508 01:59:12.641 | INFO | Serialization done, swapping tmp file now... | 20260508 01:59:12.641 | INFO | RTMetrics done | 20260508 01:59:12.755 | INFO | Finished a round of Baz IO: Wrote 1.649 GiB in 3.568 seconds | 20260508 01:59:12.755 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.67% / 0.00% / 58.32% | 20260508 01:59:12.755 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.781004% / 0% / 7.94729e-05% | 20260508 01:59:12.755 | INFO | Baz IO overall: | 20260508 01:59:12.755 | INFO | BytesWritten : 1965603137504 | 20260508 01:59:12.755 | INFO | NumEvents : 950507719156 | 20260508 01:59:14.211 | INFO | Analyzing chunk frames = [4345856,4346368) | 20260508 01:59:14.211 | INFO | MemInfo: { | 20260508 01:59:14.211 | INFO | "gpu" : " 17 GB", | 20260508 01:59:14.211 | INFO | "huge" : "", | 20260508 01:59:14.211 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:14.211 | INFO | "malloc" : " 14 GB", | 20260508 01:59:14.211 | INFO | "pinned" : " 29 GB" | 20260508 01:59:14.211 | INFO | } | 20260508 01:59:14.211 | INFO | Perf report for the last 5.1401 seconds: | 20260508 01:59:14.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:14.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:14.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:14.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:14.211 | INFO | ANALYSIS : 68.9%, 2.9, 104.3ms, 0 | 20260508 01:59:14.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:14.211 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260508 01:59:14.211 | INFO | PRE_HQ : 68.0%, 1.8, 63.2ms, 0 | 20260508 01:59:14.211 | INFO | BAZWRITER : 69.1%, 3.8, 134.8ms, 0 | 20260508 01:59:14.211 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 01:59:14.211 | INFO | Uploaded 36GiB to GPU in 2.53465s (14.2032GiB/s) | 20260508 01:59:14.211 | INFO | Downloaded 38GiB to GPU in 2.66612s (14.2529GiB/s) | 20260508 01:59:19.348 | INFO | Analyzing chunk frames = [4346368,4346880) | 20260508 01:59:19.348 | INFO | MemInfo: { | 20260508 01:59:19.348 | INFO | "gpu" : " 17 GB", | 20260508 01:59:19.348 | INFO | "huge" : "", | 20260508 01:59:19.348 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:19.348 | INFO | "malloc" : " 14 GB", | 20260508 01:59:19.348 | INFO | "pinned" : " 29 GB" | 20260508 01:59:19.348 | INFO | } | 20260508 01:59:19.348 | INFO | Perf report for the last 5.13682 seconds: | 20260508 01:59:19.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:19.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.348 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 01:59:19.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.348 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 01:59:19.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:19.349 | INFO | Uploaded 36GiB to GPU in 2.61209s (13.7821GiB/s) | 20260508 01:59:19.349 | INFO | Downloaded 34GiB to GPU in 2.49214s (13.6429GiB/s) | 20260508 01:59:24.481 | INFO | Analyzing chunk frames = [4346880,4347392) | 20260508 01:59:24.481 | INFO | MemInfo: { | 20260508 01:59:24.481 | INFO | "gpu" : " 17 GB", | 20260508 01:59:24.481 | INFO | "huge" : "", | 20260508 01:59:24.481 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:24.481 | INFO | "malloc" : " 14 GB", | 20260508 01:59:24.481 | INFO | "pinned" : " 29 GB" | 20260508 01:59:24.481 | INFO | } | 20260508 01:59:24.481 | INFO | Perf report for the last 5.13278 seconds: | 20260508 01:59:24.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:24.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260508 01:59:24.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 01:59:24.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:24.481 | INFO | Uploaded 36GiB to GPU in 2.56501s (14.035GiB/s) | 20260508 01:59:24.481 | INFO | Downloaded 34GiB to GPU in 2.48366s (13.6895GiB/s) | 20260508 01:59:29.624 | INFO | Analyzing chunk frames = [4347392,4347904) | 20260508 01:59:29.624 | INFO | MemInfo: { | 20260508 01:59:29.624 | INFO | "gpu" : " 17 GB", | 20260508 01:59:29.624 | INFO | "huge" : "", | 20260508 01:59:29.624 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:29.624 | INFO | "malloc" : " 14 GB", | 20260508 01:59:29.624 | INFO | "pinned" : " 29 GB" | 20260508 01:59:29.624 | INFO | } | 20260508 01:59:29.624 | INFO | Perf report for the last 5.14344 seconds: | 20260508 01:59:29.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:29.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.624 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 01:59:29.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.624 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 01:59:29.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:29.625 | INFO | Uploaded 36GiB to GPU in 2.55425s (14.0942GiB/s) | 20260508 01:59:29.625 | INFO | Downloaded 34GiB to GPU in 2.47464s (13.7394GiB/s) | 20260508 01:59:34.766 | INFO | Analyzing chunk frames = [4347904,4348416) | 20260508 01:59:34.766 | INFO | MemInfo: { | 20260508 01:59:34.766 | INFO | "gpu" : " 17 GB", | 20260508 01:59:34.766 | INFO | "huge" : "", | 20260508 01:59:34.766 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:34.766 | INFO | "malloc" : " 14 GB", | 20260508 01:59:34.766 | INFO | "pinned" : " 29 GB" | 20260508 01:59:34.766 | INFO | } | 20260508 01:59:34.766 | INFO | Perf report for the last 5.14181 seconds: | 20260508 01:59:34.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:34.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.766 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 01:59:34.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.766 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 01:59:34.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:34.767 | INFO | Uploaded 36GiB to GPU in 2.35687s (15.2745GiB/s) | 20260508 01:59:34.767 | INFO | Downloaded 34GiB to GPU in 2.53667s (13.4034GiB/s) | 20260508 01:59:39.908 | INFO | Analyzing chunk frames = [4348416,4348928) | 20260508 01:59:39.908 | INFO | MemInfo: { | 20260508 01:59:39.908 | INFO | "gpu" : " 17 GB", | 20260508 01:59:39.908 | INFO | "huge" : "", | 20260508 01:59:39.908 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:39.908 | INFO | "malloc" : " 14 GB", | 20260508 01:59:39.909 | INFO | "pinned" : " 29 GB" | 20260508 01:59:39.909 | INFO | } | 20260508 01:59:39.909 | INFO | Perf report for the last 5.1426 seconds: | 20260508 01:59:39.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:39.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 01:59:39.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 01:59:39.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:39.909 | INFO | Uploaded 36GiB to GPU in 2.41729s (14.8927GiB/s) | 20260508 01:59:39.909 | INFO | Downloaded 34GiB to GPU in 2.41101s (14.102GiB/s) | 20260508 01:59:45.043 | INFO | Analyzing chunk frames = [4348928,4349440) | 20260508 01:59:45.043 | INFO | MemInfo: { | 20260508 01:59:45.043 | INFO | "gpu" : " 17 GB", | 20260508 01:59:45.044 | INFO | "huge" : "", | 20260508 01:59:45.044 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:45.044 | INFO | "malloc" : " 14 GB", | 20260508 01:59:45.044 | INFO | "pinned" : " 29 GB" | 20260508 01:59:45.044 | INFO | } | 20260508 01:59:45.044 | INFO | Perf report for the last 5.13511 seconds: | 20260508 01:59:45.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:45.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 01:59:45.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 01:59:45.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:45.044 | INFO | Uploaded 36GiB to GPU in 2.44192s (14.7425GiB/s) | 20260508 01:59:45.044 | INFO | Downloaded 34GiB to GPU in 2.42625s (14.0134GiB/s) | 20260508 01:59:50.179 | INFO | Analyzing chunk frames = [4349440,4349952) | 20260508 01:59:50.179 | INFO | MemInfo: { | 20260508 01:59:50.179 | INFO | "gpu" : " 17 GB", | 20260508 01:59:50.179 | INFO | "huge" : "", | 20260508 01:59:50.179 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:50.179 | INFO | "malloc" : " 14 GB", | 20260508 01:59:50.179 | INFO | "pinned" : " 29 GB" | 20260508 01:59:50.179 | INFO | } | 20260508 01:59:50.179 | INFO | Perf report for the last 5.13541 seconds: | 20260508 01:59:50.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:50.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 01:59:50.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260508 01:59:50.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:50.179 | INFO | Uploaded 36GiB to GPU in 2.57595s (13.9754GiB/s) | 20260508 01:59:50.180 | INFO | Downloaded 34GiB to GPU in 2.46149s (13.8128GiB/s) | 20260508 01:59:50.180 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 01:59:50.180 | INFO | Total mem usage: 16.7 GiB | 20260508 01:59:50.180 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 01:59:50.180 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.38%) | 20260508 01:59:50.180 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 01:59:50.180 | INFO | High Water Breakdown: | 20260508 01:59:50.180 | INFO | 61% : 9.56 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 01:59:50.180 | INFO | 38.9% : 6.1 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 01:59:50.180 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 01:59:50.180 | INFO | ------------------------------------------ | 20260508 01:59:50.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:59:50.180 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 01:59:50.180 | INFO | Total mem usage: 26.5 GiB | 20260508 01:59:50.180 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 01:59:50.180 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 01:59:50.180 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 01:59:50.180 | INFO | High Water Breakdown: | 20260508 01:59:50.180 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:59:50.180 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:59:50.180 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:59:50.180 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:59:50.180 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:59:50.180 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:59:50.180 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 01:59:50.180 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 01:59:50.180 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:59:50.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 01:59:50.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 01:59:50.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 01:59:50.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 01:59:50.180 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 01:59:50.180 | INFO | ------------------------------------------ | 20260508 01:59:50.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:59:50.180 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 01:59:50.180 | INFO | Total mem usage: 27.8 GiB | 20260508 01:59:50.180 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 01:59:50.180 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 01:59:50.180 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 01:59:50.180 | INFO | High Water Breakdown: | 20260508 01:59:50.180 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 01:59:50.180 | INFO | ------------------------------------------ | 20260508 01:59:50.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:59:50.180 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 01:59:50.180 | INFO | Total mem usage: 15.7 GiB | 20260508 01:59:50.180 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 01:59:50.180 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 01:59:50.180 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 01:59:50.180 | INFO | High Water Breakdown: | 20260508 01:59:50.180 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 01:59:50.180 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 01:59:50.180 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 01:59:50.180 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 01:59:50.180 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 01:59:50.180 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 01:59:50.180 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 01:59:50.180 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 01:59:50.181 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 01:59:50.181 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 01:59:50.181 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 01:59:50.181 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 01:59:50.181 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 01:59:50.181 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 01:59:50.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 01:59:50.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 01:59:50.181 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 01:59:50.181 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 01:59:50.181 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 01:59:50.181 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 01:59:50.181 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 01:59:50.181 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 01:59:50.181 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 01:59:50.181 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 01:59:50.181 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 01:59:50.181 | INFO | ------------------------------------------ | 20260508 01:59:50.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 01:59:50.181 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 01:59:50.181 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:59:50.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:59:50.181 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.62e+01, 1.42e+01 / 1.71e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.17e-02 / 4.55e-03 | 20260508 01:59:50.181 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.45e+00, 5.85e-01 / 1.72e-01 | 20260508 01:59:50.181 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 9.75e-02 | 20260508 01:59:50.181 | INFO | PerfReport: 16.11%, DME -- 4.21e+01 / 4.62e+01, 4.15e+00 / 1.96e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.57e+00 / 1.27e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.25e-01 / 1.98e-01 | 20260508 01:59:50.181 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.26e+00, 2.66e-01 / 1.47e-01 | 20260508 01:59:50.181 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.70e-01, 5.28e-02 / 2.83e-02 | 20260508 01:59:50.181 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 01:59:50.181 | INFO | PerfReport: Gathered 1536 samples. | 20260508 01:59:50.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 01:59:50.181 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 24.52%, Upload -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 30.63%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 2.28%, LockUpload -- 2.44e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 2.76%, LockDownload -- 2.94e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 15.38%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 01:59:50.181 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260508 01:59:50.276 | INFO | Starting a round of Baz IO | 20260508 01:59:53.714 | INFO | Populating RTMetrics report | 20260508 01:59:53.715 | INFO | RTMetrics report populated | 20260508 01:59:53.715 | INFO | Opening tmp RTMetrics json file | 20260508 01:59:53.798 | INFO | Serializing RTMetrics to json | 20260508 01:59:53.826 | INFO | Serialization done, swapping tmp file now... | 20260508 01:59:53.832 | INFO | RTMetrics done | 20260508 01:59:53.851 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.574 seconds | 20260508 01:59:53.851 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.67% / 0.00% / 58.33% | 20260508 01:59:53.851 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.781306% / 7.94729e-06% / 8.34465e-05% | 20260508 01:59:53.851 | INFO | Baz IO overall: | 20260508 01:59:53.851 | INFO | BytesWritten : 1967375070523 | 20260508 01:59:53.851 | INFO | NumEvents : 951358520005 | 20260508 01:59:55.325 | INFO | Analyzing chunk frames = [4349952,4350464) | 20260508 01:59:55.325 | INFO | MemInfo: { | 20260508 01:59:55.325 | INFO | "gpu" : " 17 GB", | 20260508 01:59:55.325 | INFO | "huge" : "", | 20260508 01:59:55.325 | INFO | "huge_shared" : " 29 GB", | 20260508 01:59:55.325 | INFO | "malloc" : " 14 GB", | 20260508 01:59:55.325 | INFO | "pinned" : " 29 GB" | 20260508 01:59:55.325 | INFO | } | 20260508 01:59:55.325 | INFO | Perf report for the last 5.14576 seconds: | 20260508 01:59:55.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 01:59:55.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:55.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:55.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:55.325 | INFO | ANALYSIS : 68.0%, 3.0, 106.4ms, 0 | 20260508 01:59:55.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 01:59:55.325 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 01:59:55.325 | INFO | PRE_HQ : 65.0%, 1.7, 55.9ms, 0 | 20260508 01:59:55.325 | INFO | BAZWRITER : 69.1%, 3.6, 126.2ms, 0 | 20260508 01:59:55.325 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260508 01:59:55.325 | INFO | Uploaded 36GiB to GPU in 2.53262s (14.2145GiB/s) | 20260508 01:59:55.325 | INFO | Downloaded 38GiB to GPU in 2.68574s (14.1488GiB/s) | 20260508 02:00:00.465 | INFO | Analyzing chunk frames = [4350464,4350976) | 20260508 02:00:00.465 | INFO | MemInfo: { | 20260508 02:00:00.465 | INFO | "gpu" : " 17 GB", | 20260508 02:00:00.465 | INFO | "huge" : "", | 20260508 02:00:00.465 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:00.465 | INFO | "malloc" : " 14 GB", | 20260508 02:00:00.465 | INFO | "pinned" : " 29 GB" | 20260508 02:00:00.465 | INFO | } | 20260508 02:00:00.465 | INFO | Perf report for the last 5.13982 seconds: | 20260508 02:00:00.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:00.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:00:00.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:00:00.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:00.465 | INFO | Uploaded 36GiB to GPU in 2.56112s (14.0564GiB/s) | 20260508 02:00:00.465 | INFO | Downloaded 34GiB to GPU in 2.42161s (14.0403GiB/s) | 20260508 02:00:05.602 | INFO | Analyzing chunk frames = [4350976,4351488) | 20260508 02:00:05.602 | INFO | MemInfo: { | 20260508 02:00:05.602 | INFO | "gpu" : " 17 GB", | 20260508 02:00:05.602 | INFO | "huge" : "", | 20260508 02:00:05.602 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:05.602 | INFO | "malloc" : " 14 GB", | 20260508 02:00:05.602 | INFO | "pinned" : " 29 GB" | 20260508 02:00:05.602 | INFO | } | 20260508 02:00:05.602 | INFO | Perf report for the last 5.13693 seconds: | 20260508 02:00:05.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:05.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:00:05.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:00:05.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:05.602 | INFO | Uploaded 36GiB to GPU in 2.52871s (14.2365GiB/s) | 20260508 02:00:05.602 | INFO | Downloaded 34GiB to GPU in 2.47806s (13.7204GiB/s) | 20260508 02:00:10.737 | INFO | Analyzing chunk frames = [4351488,4352000) | 20260508 02:00:10.737 | INFO | MemInfo: { | 20260508 02:00:10.737 | INFO | "gpu" : " 17 GB", | 20260508 02:00:10.737 | INFO | "huge" : "", | 20260508 02:00:10.737 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:10.737 | INFO | "malloc" : " 14 GB", | 20260508 02:00:10.737 | INFO | "pinned" : " 29 GB" | 20260508 02:00:10.737 | INFO | } | 20260508 02:00:10.737 | INFO | Perf report for the last 5.13574 seconds: | 20260508 02:00:10.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:10.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:00:10.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:00:10.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:10.738 | INFO | Uploaded 36GiB to GPU in 2.52834s (14.2386GiB/s) | 20260508 02:00:10.738 | INFO | Downloaded 34GiB to GPU in 2.48514s (13.6813GiB/s) | 20260508 02:00:15.879 | INFO | Analyzing chunk frames = [4352000,4352512) | 20260508 02:00:15.879 | INFO | MemInfo: { | 20260508 02:00:15.879 | INFO | "gpu" : " 17 GB", | 20260508 02:00:15.879 | INFO | "huge" : "", | 20260508 02:00:15.879 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:15.879 | INFO | "malloc" : " 14 GB", | 20260508 02:00:15.879 | INFO | "pinned" : " 29 GB" | 20260508 02:00:15.879 | INFO | } | 20260508 02:00:15.879 | INFO | Perf report for the last 5.14128 seconds: | 20260508 02:00:15.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:15.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 02:00:15.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:00:15.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:15.879 | INFO | Uploaded 36GiB to GPU in 2.34416s (15.3573GiB/s) | 20260508 02:00:15.879 | INFO | Downloaded 34GiB to GPU in 2.49772s (13.6124GiB/s) | 20260508 02:00:21.020 | INFO | Analyzing chunk frames = [4352512,4353024) | 20260508 02:00:21.020 | INFO | MemInfo: { | 20260508 02:00:21.020 | INFO | "gpu" : " 17 GB", | 20260508 02:00:21.020 | INFO | "huge" : "", | 20260508 02:00:21.021 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:21.021 | INFO | "malloc" : " 14 GB", | 20260508 02:00:21.021 | INFO | "pinned" : " 29 GB" | 20260508 02:00:21.021 | INFO | } | 20260508 02:00:21.021 | INFO | Perf report for the last 5.1421 seconds: | 20260508 02:00:21.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:21.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 02:00:21.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:00:21.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:21.021 | INFO | Uploaded 36GiB to GPU in 2.4287s (14.8228GiB/s) | 20260508 02:00:21.022 | INFO | Downloaded 34GiB to GPU in 2.40456s (14.1398GiB/s) | 20260508 02:00:26.162 | INFO | Analyzing chunk frames = [4353024,4353536) | 20260508 02:00:26.162 | INFO | MemInfo: { | 20260508 02:00:26.162 | INFO | "gpu" : " 17 GB", | 20260508 02:00:26.162 | INFO | "huge" : "", | 20260508 02:00:26.162 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:26.163 | INFO | "malloc" : " 14 GB", | 20260508 02:00:26.163 | INFO | "pinned" : " 29 GB" | 20260508 02:00:26.163 | INFO | } | 20260508 02:00:26.163 | INFO | Perf report for the last 5.14178 seconds: | 20260508 02:00:26.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:26.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 02:00:26.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 02:00:26.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:26.163 | INFO | Uploaded 36GiB to GPU in 2.46318s (14.6152GiB/s) | 20260508 02:00:26.163 | INFO | Downloaded 34GiB to GPU in 2.41485s (14.0796GiB/s) | 20260508 02:00:31.297 | INFO | Analyzing chunk frames = [4353536,4354048) | 20260508 02:00:31.297 | INFO | MemInfo: { | 20260508 02:00:31.297 | INFO | "gpu" : " 17 GB", | 20260508 02:00:31.297 | INFO | "huge" : "", | 20260508 02:00:31.297 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:31.297 | INFO | "malloc" : " 14 GB", | 20260508 02:00:31.297 | INFO | "pinned" : " 29 GB" | 20260508 02:00:31.297 | INFO | } | 20260508 02:00:31.297 | INFO | Perf report for the last 5.13458 seconds: | 20260508 02:00:31.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:31.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.298 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:00:31.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.298 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 02:00:31.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:31.298 | INFO | Uploaded 36GiB to GPU in 2.58024s (13.9522GiB/s) | 20260508 02:00:31.298 | INFO | Downloaded 34GiB to GPU in 2.4585s (13.8296GiB/s) | 20260508 02:00:31.411 | INFO | Starting a round of Baz IO | 20260508 02:00:34.865 | INFO | Populating RTMetrics report | 20260508 02:00:34.865 | INFO | RTMetrics report populated | 20260508 02:00:34.865 | INFO | Opening tmp RTMetrics json file | 20260508 02:00:34.880 | INFO | Serializing RTMetrics to json | 20260508 02:00:34.888 | INFO | Serialization done, swapping tmp file now... | 20260508 02:00:34.889 | INFO | RTMetrics done | 20260508 02:00:34.973 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.562 seconds | 20260508 02:00:34.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.66% / 0.00% / 58.33% | 20260508 02:00:34.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.778254% / 1.19209e-05% / 6.35783e-05% | 20260508 02:00:34.973 | INFO | Baz IO overall: | 20260508 02:00:34.973 | INFO | BytesWritten : 1969146247493 | 20260508 02:00:34.973 | INFO | NumEvents : 952208576338 | 20260508 02:00:36.435 | INFO | Analyzing chunk frames = [4354048,4354560) | 20260508 02:00:36.435 | INFO | MemInfo: { | 20260508 02:00:36.435 | INFO | "gpu" : " 17 GB", | 20260508 02:00:36.435 | INFO | "huge" : "", | 20260508 02:00:36.435 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:36.435 | INFO | "malloc" : " 14 GB", | 20260508 02:00:36.435 | INFO | "pinned" : " 29 GB" | 20260508 02:00:36.435 | INFO | } | 20260508 02:00:36.435 | INFO | Perf report for the last 5.13806 seconds: | 20260508 02:00:36.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:36.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:36.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:36.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:36.435 | INFO | ANALYSIS : 68.7%, 2.9, 104.5ms, 0 | 20260508 02:00:36.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:36.435 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260508 02:00:36.435 | INFO | PRE_HQ : 67.7%, 1.8, 63.0ms, 0 | 20260508 02:00:36.436 | INFO | BAZWRITER : 68.7%, 4.2, 146.9ms, 0 | 20260508 02:00:36.436 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 02:00:36.436 | INFO | Uploaded 36GiB to GPU in 2.6222s (13.7289GiB/s) | 20260508 02:00:36.436 | INFO | Downloaded 38GiB to GPU in 2.71753s (13.9833GiB/s) | 20260508 02:00:41.576 | INFO | Analyzing chunk frames = [4354560,4355072) | 20260508 02:00:41.576 | INFO | MemInfo: { | 20260508 02:00:41.576 | INFO | "gpu" : " 17 GB", | 20260508 02:00:41.576 | INFO | "huge" : "", | 20260508 02:00:41.576 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:41.576 | INFO | "malloc" : " 14 GB", | 20260508 02:00:41.576 | INFO | "pinned" : " 29 GB" | 20260508 02:00:41.576 | INFO | } | 20260508 02:00:41.576 | INFO | Perf report for the last 5.14063 seconds: | 20260508 02:00:41.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:41.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:00:41.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:00:41.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:41.576 | INFO | Uploaded 36GiB to GPU in 2.52302s (14.2686GiB/s) | 20260508 02:00:41.576 | INFO | Downloaded 34GiB to GPU in 2.43218s (13.9792GiB/s) | 20260508 02:00:46.719 | INFO | Analyzing chunk frames = [4355072,4355584) | 20260508 02:00:46.719 | INFO | MemInfo: { | 20260508 02:00:46.719 | INFO | "gpu" : " 17 GB", | 20260508 02:00:46.719 | INFO | "huge" : "", | 20260508 02:00:46.719 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:46.719 | INFO | "malloc" : " 14 GB", | 20260508 02:00:46.719 | INFO | "pinned" : " 29 GB" | 20260508 02:00:46.719 | INFO | } | 20260508 02:00:46.719 | INFO | Perf report for the last 5.14313 seconds: | 20260508 02:00:46.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:46.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:00:46.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:00:46.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:46.719 | INFO | Uploaded 36GiB to GPU in 2.52659s (14.2485GiB/s) | 20260508 02:00:46.720 | INFO | Downloaded 34GiB to GPU in 2.49314s (13.6374GiB/s) | 20260508 02:00:51.852 | INFO | Analyzing chunk frames = [4355584,4356096) | 20260508 02:00:51.852 | INFO | MemInfo: { | 20260508 02:00:51.852 | INFO | "gpu" : " 17 GB", | 20260508 02:00:51.852 | INFO | "huge" : "", | 20260508 02:00:51.852 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:51.852 | INFO | "malloc" : " 14 GB", | 20260508 02:00:51.852 | INFO | "pinned" : " 29 GB" | 20260508 02:00:51.852 | INFO | } | 20260508 02:00:51.852 | INFO | Perf report for the last 5.13273 seconds: | 20260508 02:00:51.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:51.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | ANALYSIS : 67.3%, 3.0, 104.0ms, 0 | 20260508 02:00:51.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:00:51.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:51.852 | INFO | Uploaded 36GiB to GPU in 2.5012s (14.3931GiB/s) | 20260508 02:00:51.852 | INFO | Downloaded 34GiB to GPU in 2.4876s (13.6678GiB/s) | 20260508 02:00:56.991 | INFO | Analyzing chunk frames = [4356096,4356608) | 20260508 02:00:56.991 | INFO | MemInfo: { | 20260508 02:00:56.991 | INFO | "gpu" : " 17 GB", | 20260508 02:00:56.991 | INFO | "huge" : "", | 20260508 02:00:56.991 | INFO | "huge_shared" : " 29 GB", | 20260508 02:00:56.991 | INFO | "malloc" : " 14 GB", | 20260508 02:00:56.991 | INFO | "pinned" : " 29 GB" | 20260508 02:00:56.991 | INFO | } | 20260508 02:00:56.991 | INFO | Perf report for the last 5.13954 seconds: | 20260508 02:00:56.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:00:56.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 02:00:56.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:00:56.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:00:56.992 | INFO | Uploaded 36GiB to GPU in 2.29523s (15.6847GiB/s) | 20260508 02:00:56.992 | INFO | Downloaded 34GiB to GPU in 2.50614s (13.5667GiB/s) | 20260508 02:01:02.134 | INFO | Analyzing chunk frames = [4356608,4357120) | 20260508 02:01:02.134 | INFO | MemInfo: { | 20260508 02:01:02.134 | INFO | "gpu" : " 17 GB", | 20260508 02:01:02.134 | INFO | "huge" : "", | 20260508 02:01:02.135 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:02.135 | INFO | "malloc" : " 14 GB", | 20260508 02:01:02.135 | INFO | "pinned" : " 29 GB" | 20260508 02:01:02.135 | INFO | } | 20260508 02:01:02.135 | INFO | Perf report for the last 5.14325 seconds: | 20260508 02:01:02.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:02.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 02:01:02.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:01:02.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:02.135 | INFO | Uploaded 36GiB to GPU in 2.37665s (15.1474GiB/s) | 20260508 02:01:02.135 | INFO | Downloaded 34GiB to GPU in 2.43725s (13.9502GiB/s) | 20260508 02:01:07.276 | INFO | Analyzing chunk frames = [4357120,4357632) | 20260508 02:01:07.276 | INFO | MemInfo: { | 20260508 02:01:07.276 | INFO | "gpu" : " 17 GB", | 20260508 02:01:07.276 | INFO | "huge" : "", | 20260508 02:01:07.276 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:07.276 | INFO | "malloc" : " 14 GB", | 20260508 02:01:07.276 | INFO | "pinned" : " 29 GB" | 20260508 02:01:07.276 | INFO | } | 20260508 02:01:07.276 | INFO | Perf report for the last 5.14159 seconds: | 20260508 02:01:07.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:07.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.276 | INFO | ANALYSIS : 67.1%, 3.0, 104.9ms, 0 | 20260508 02:01:07.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.276 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 02:01:07.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:07.277 | INFO | Uploaded 36GiB to GPU in 2.51581s (14.3095GiB/s) | 20260508 02:01:07.277 | INFO | Downloaded 34GiB to GPU in 2.46802s (13.7763GiB/s) | 20260508 02:01:12.414 | INFO | Analyzing chunk frames = [4357632,4358144) | 20260508 02:01:12.414 | INFO | MemInfo: { | 20260508 02:01:12.414 | INFO | "gpu" : " 17 GB", | 20260508 02:01:12.414 | INFO | "huge" : "", | 20260508 02:01:12.414 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:12.414 | INFO | "malloc" : " 14 GB", | 20260508 02:01:12.414 | INFO | "pinned" : " 29 GB" | 20260508 02:01:12.414 | INFO | } | 20260508 02:01:12.414 | INFO | Perf report for the last 5.13802 seconds: | 20260508 02:01:12.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:12.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.414 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:01:12.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.414 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 02:01:12.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:12.415 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:01:12.415 | INFO | Total mem usage: 16.7 GiB | 20260508 02:01:12.415 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:01:12.415 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 02:01:12.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:01:12.415 | INFO | High Water Breakdown: | 20260508 02:01:12.415 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:01:12.415 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:01:12.415 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:01:12.415 | INFO | ------------------------------------------ | 20260508 02:01:12.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:01:12.415 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:01:12.415 | INFO | Total mem usage: 26.5 GiB | 20260508 02:01:12.415 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:01:12.415 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 02:01:12.415 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 02:01:12.415 | INFO | High Water Breakdown: | 20260508 02:01:12.415 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:01:12.415 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:01:12.415 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:01:12.415 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:01:12.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:01:12.415 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:01:12.415 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:01:12.415 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:01:12.415 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:01:12.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:01:12.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:01:12.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:01:12.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:01:12.415 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:01:12.415 | INFO | ------------------------------------------ | 20260508 02:01:12.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:01:12.415 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:01:12.415 | INFO | Total mem usage: 27.8 GiB | 20260508 02:01:12.415 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:01:12.415 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:01:12.415 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:01:12.415 | INFO | High Water Breakdown: | 20260508 02:01:12.415 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:01:12.415 | INFO | ------------------------------------------ | 20260508 02:01:12.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:01:12.415 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:01:12.415 | INFO | Total mem usage: 15.7 GiB | 20260508 02:01:12.415 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:01:12.415 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:01:12.415 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:01:12.415 | INFO | High Water Breakdown: | 20260508 02:01:12.415 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:01:12.415 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:01:12.415 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:01:12.415 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:01:12.415 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:01:12.415 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:01:12.415 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:01:12.415 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:01:12.415 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:01:12.415 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:01:12.415 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:01:12.415 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:01:12.415 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:01:12.415 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:01:12.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:01:12.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:01:12.415 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:01:12.415 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:01:12.415 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:01:12.415 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:01:12.415 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:01:12.415 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:01:12.415 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:01:12.415 | INFO | Uploaded 36GiB to GPU in 2.55468s (14.0918GiB/s) | 20260508 02:01:12.415 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:01:12.415 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:01:12.415 | INFO | ------------------------------------------ | 20260508 02:01:12.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:01:12.415 | INFO | Downloaded 34GiB to GPU in 2.47672s (13.7278GiB/s) | 20260508 02:01:12.415 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:01:12.415 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:01:12.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:01:12.415 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.61e+01, 1.40e+01 / 1.21e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.05e-01, 1.14e-02 / 1.02e-02 | 20260508 02:01:12.415 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.45e+00, 5.83e-01 / 2.18e-01 | 20260508 02:01:12.415 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.06e-01 / 9.70e-02 | 20260508 02:01:12.415 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.59e+01, 3.99e+00 / 1.58e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.58e+00 / 1.61e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.44e+00 / 2.66e+00, 4.25e-01 / 2.23e-01 | 20260508 02:01:12.415 | INFO | PerfReport: 3.33%, Metrics -- 1.09e+00 / 1.25e+00, 2.70e-01 / 1.15e-01 | 20260508 02:01:12.415 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.34e-01 / 3.62e-01, 5.29e-02 / 1.93e-02 | 20260508 02:01:12.415 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:01:12.415 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:01:12.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:01:12.415 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 30.70%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 2.18%, LockUpload -- 2.32e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 2.84%, LockDownload -- 3.02e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 15.45%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:01:12.415 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 02:01:12.512 | INFO | Starting a round of Baz IO | 20260508 02:01:15.960 | INFO | Populating RTMetrics report | 20260508 02:01:15.960 | INFO | RTMetrics report populated | 20260508 02:01:15.960 | INFO | Opening tmp RTMetrics json file | 20260508 02:01:15.996 | INFO | Serializing RTMetrics to json | 20260508 02:01:16.056 | INFO | Serialization done, swapping tmp file now... | 20260508 02:01:16.067 | INFO | RTMetrics done | 20260508 02:01:16.082 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.569 seconds | 20260508 02:01:16.082 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.66% / 0.00% / 58.34% | 20260508 02:01:16.082 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.779041% / 0% / 4.37101e-05% | 20260508 02:01:16.082 | INFO | Baz IO overall: | 20260508 02:01:16.082 | INFO | BytesWritten : 1970917519601 | 20260508 02:01:16.082 | INFO | NumEvents : 953058647554 | 20260508 02:01:17.549 | INFO | Analyzing chunk frames = [4358144,4358656) | 20260508 02:01:17.549 | INFO | MemInfo: { | 20260508 02:01:17.549 | INFO | "gpu" : " 17 GB", | 20260508 02:01:17.549 | INFO | "huge" : "", | 20260508 02:01:17.549 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:17.549 | INFO | "malloc" : " 14 GB", | 20260508 02:01:17.549 | INFO | "pinned" : " 29 GB" | 20260508 02:01:17.549 | INFO | } | 20260508 02:01:17.549 | INFO | Perf report for the last 5.13487 seconds: | 20260508 02:01:17.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:17.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:17.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:17.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:17.549 | INFO | ANALYSIS : 68.4%, 3.0, 105.8ms, 0 | 20260508 02:01:17.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:17.549 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 02:01:17.549 | INFO | PRE_HQ : 66.3%, 1.6, 56.6ms, 0 | 20260508 02:01:17.549 | INFO | BAZWRITER : 69.2%, 3.6, 128.1ms, 0 | 20260508 02:01:17.549 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 02:01:17.550 | INFO | Uploaded 36GiB to GPU in 2.51095s (14.3372GiB/s) | 20260508 02:01:17.550 | INFO | Downloaded 38GiB to GPU in 2.68528s (14.1512GiB/s) | 20260508 02:01:22.689 | INFO | Analyzing chunk frames = [4358656,4359168) | 20260508 02:01:22.689 | INFO | MemInfo: { | 20260508 02:01:22.689 | INFO | "gpu" : " 17 GB", | 20260508 02:01:22.689 | INFO | "huge" : "", | 20260508 02:01:22.689 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:22.689 | INFO | "malloc" : " 14 GB", | 20260508 02:01:22.690 | INFO | "pinned" : " 29 GB" | 20260508 02:01:22.690 | INFO | } | 20260508 02:01:22.690 | INFO | Perf report for the last 5.14042 seconds: | 20260508 02:01:22.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:22.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:01:22.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:01:22.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:22.690 | INFO | Uploaded 36GiB to GPU in 2.54094s (14.168GiB/s) | 20260508 02:01:22.691 | INFO | Downloaded 34GiB to GPU in 2.42142s (14.0414GiB/s) | 20260508 02:01:27.833 | INFO | Analyzing chunk frames = [4359168,4359680) | 20260508 02:01:27.833 | INFO | MemInfo: { | 20260508 02:01:27.833 | INFO | "gpu" : " 17 GB", | 20260508 02:01:27.833 | INFO | "huge" : "", | 20260508 02:01:27.833 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:27.833 | INFO | "malloc" : " 14 GB", | 20260508 02:01:27.833 | INFO | "pinned" : " 29 GB" | 20260508 02:01:27.833 | INFO | } | 20260508 02:01:27.833 | INFO | Perf report for the last 5.14341 seconds: | 20260508 02:01:27.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:27.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:01:27.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:01:27.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:27.833 | INFO | Uploaded 36GiB to GPU in 2.51458s (14.3165GiB/s) | 20260508 02:01:27.833 | INFO | Downloaded 34GiB to GPU in 2.49404s (13.6325GiB/s) | 20260508 02:01:32.974 | INFO | Analyzing chunk frames = [4359680,4360192) | 20260508 02:01:32.974 | INFO | MemInfo: { | 20260508 02:01:32.974 | INFO | "gpu" : " 17 GB", | 20260508 02:01:32.974 | INFO | "huge" : "", | 20260508 02:01:32.974 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:32.974 | INFO | "malloc" : " 14 GB", | 20260508 02:01:32.974 | INFO | "pinned" : " 29 GB" | 20260508 02:01:32.974 | INFO | } | 20260508 02:01:32.974 | INFO | Perf report for the last 5.14104 seconds: | 20260508 02:01:32.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:32.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 02:01:32.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:01:32.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:32.974 | INFO | Uploaded 36GiB to GPU in 2.52573s (14.2533GiB/s) | 20260508 02:01:32.974 | INFO | Downloaded 34GiB to GPU in 2.48691s (13.6716GiB/s) | 20260508 02:01:38.113 | INFO | Analyzing chunk frames = [4360192,4360704) | 20260508 02:01:38.113 | INFO | MemInfo: { | 20260508 02:01:38.113 | INFO | "gpu" : " 17 GB", | 20260508 02:01:38.113 | INFO | "huge" : "", | 20260508 02:01:38.113 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:38.113 | INFO | "malloc" : " 14 GB", | 20260508 02:01:38.113 | INFO | "pinned" : " 29 GB" | 20260508 02:01:38.113 | INFO | } | 20260508 02:01:38.113 | INFO | Perf report for the last 5.13914 seconds: | 20260508 02:01:38.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:38.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.113 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 02:01:38.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.113 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 02:01:38.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:38.114 | INFO | Uploaded 36GiB to GPU in 2.38932s (15.067GiB/s) | 20260508 02:01:38.114 | INFO | Downloaded 34GiB to GPU in 2.50199s (13.5892GiB/s) | 20260508 02:01:43.245 | INFO | Analyzing chunk frames = [4360704,4361216) | 20260508 02:01:43.245 | INFO | MemInfo: { | 20260508 02:01:43.245 | INFO | "gpu" : " 17 GB", | 20260508 02:01:43.245 | INFO | "huge" : "", | 20260508 02:01:43.245 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:43.245 | INFO | "malloc" : " 14 GB", | 20260508 02:01:43.245 | INFO | "pinned" : " 29 GB" | 20260508 02:01:43.245 | INFO | } | 20260508 02:01:43.245 | INFO | Perf report for the last 5.13187 seconds: | 20260508 02:01:43.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:43.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.245 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 02:01:43.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.246 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:01:43.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:43.246 | INFO | Uploaded 36GiB to GPU in 2.40197s (14.9877GiB/s) | 20260508 02:01:43.246 | INFO | Downloaded 34GiB to GPU in 2.41708s (14.0666GiB/s) | 20260508 02:01:48.386 | INFO | Analyzing chunk frames = [4361216,4361728) | 20260508 02:01:48.386 | INFO | MemInfo: { | 20260508 02:01:48.386 | INFO | "gpu" : " 17 GB", | 20260508 02:01:48.386 | INFO | "huge" : "", | 20260508 02:01:48.386 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:48.386 | INFO | "malloc" : " 14 GB", | 20260508 02:01:48.386 | INFO | "pinned" : " 29 GB" | 20260508 02:01:48.387 | INFO | } | 20260508 02:01:48.387 | INFO | Perf report for the last 5.14139 seconds: | 20260508 02:01:48.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:48.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:01:48.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 02:01:48.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:48.387 | INFO | Uploaded 36GiB to GPU in 2.44354s (14.7327GiB/s) | 20260508 02:01:48.387 | INFO | Downloaded 34GiB to GPU in 2.42571s (14.0165GiB/s) | 20260508 02:01:53.526 | INFO | Analyzing chunk frames = [4361728,4362240) | 20260508 02:01:53.526 | INFO | MemInfo: { | 20260508 02:01:53.526 | INFO | "gpu" : " 17 GB", | 20260508 02:01:53.526 | INFO | "huge" : "", | 20260508 02:01:53.526 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:53.526 | INFO | "malloc" : " 14 GB", | 20260508 02:01:53.526 | INFO | "pinned" : " 29 GB" | 20260508 02:01:53.526 | INFO | } | 20260508 02:01:53.526 | INFO | Perf report for the last 5.13976 seconds: | 20260508 02:01:53.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:53.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:01:53.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | PRE_HQ : 34.9%, 1.0, 18.2ms, 0 | 20260508 02:01:53.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:53.527 | INFO | Uploaded 36GiB to GPU in 2.54415s (14.1501GiB/s) | 20260508 02:01:53.527 | INFO | Downloaded 34GiB to GPU in 2.45915s (13.8259GiB/s) | 20260508 02:01:53.640 | INFO | Starting a round of Baz IO | 20260508 02:01:57.111 | INFO | Populating RTMetrics report | 20260508 02:01:57.111 | INFO | RTMetrics report populated | 20260508 02:01:57.111 | INFO | Opening tmp RTMetrics json file | 20260508 02:01:57.133 | INFO | Serializing RTMetrics to json | 20260508 02:01:57.159 | INFO | Serialization done, swapping tmp file now... | 20260508 02:01:57.191 | INFO | RTMetrics done | 20260508 02:01:57.219 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.579 seconds | 20260508 02:01:57.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.66% / 0.00% / 58.34% | 20260508 02:01:57.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.7786% / 3.97364e-06% / 5.16574e-05% | 20260508 02:01:57.220 | INFO | Baz IO overall: | 20260508 02:01:57.220 | INFO | BytesWritten : 1972689730122 | 20260508 02:01:57.220 | INFO | NumEvents : 953909570426 | 20260508 02:01:58.669 | INFO | Analyzing chunk frames = [4362240,4362752) | 20260508 02:01:58.669 | INFO | MemInfo: { | 20260508 02:01:58.669 | INFO | "gpu" : " 17 GB", | 20260508 02:01:58.669 | INFO | "huge" : "", | 20260508 02:01:58.669 | INFO | "huge_shared" : " 29 GB", | 20260508 02:01:58.669 | INFO | "malloc" : " 14 GB", | 20260508 02:01:58.669 | INFO | "pinned" : " 29 GB" | 20260508 02:01:58.669 | INFO | } | 20260508 02:01:58.669 | INFO | Perf report for the last 5.14244 seconds: | 20260508 02:01:58.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:01:58.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:58.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:58.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:58.669 | INFO | ANALYSIS : 68.7%, 2.9, 104.3ms, 0 | 20260508 02:01:58.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:01:58.669 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260508 02:01:58.669 | INFO | PRE_HQ : 67.8%, 1.8, 63.8ms, 0 | 20260508 02:01:58.669 | INFO | BAZWRITER : 69.3%, 3.8, 135.5ms, 0 | 20260508 02:01:58.669 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 02:01:58.670 | INFO | Uploaded 36GiB to GPU in 2.61003s (13.7929GiB/s) | 20260508 02:01:58.670 | INFO | Downloaded 38GiB to GPU in 2.71106s (14.0166GiB/s) | 20260508 02:02:03.801 | INFO | Analyzing chunk frames = [4362752,4363264) | 20260508 02:02:03.801 | INFO | MemInfo: { | 20260508 02:02:03.801 | INFO | "gpu" : " 17 GB", | 20260508 02:02:03.801 | INFO | "huge" : "", | 20260508 02:02:03.801 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:03.801 | INFO | "malloc" : " 14 GB", | 20260508 02:02:03.801 | INFO | "pinned" : " 29 GB" | 20260508 02:02:03.801 | INFO | } | 20260508 02:02:03.801 | INFO | Perf report for the last 5.13214 seconds: | 20260508 02:02:03.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:03.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:02:03.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:02:03.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:03.801 | INFO | Uploaded 36GiB to GPU in 2.54218s (14.1611GiB/s) | 20260508 02:02:03.801 | INFO | Downloaded 34GiB to GPU in 2.43919s (13.9391GiB/s) | 20260508 02:02:08.946 | INFO | Analyzing chunk frames = [4363264,4363776) | 20260508 02:02:08.946 | INFO | MemInfo: { | 20260508 02:02:08.946 | INFO | "gpu" : " 17 GB", | 20260508 02:02:08.946 | INFO | "huge" : "", | 20260508 02:02:08.946 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:08.946 | INFO | "malloc" : " 14 GB", | 20260508 02:02:08.946 | INFO | "pinned" : " 29 GB" | 20260508 02:02:08.946 | INFO | } | 20260508 02:02:08.946 | INFO | Perf report for the last 5.14494 seconds: | 20260508 02:02:08.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:08.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.946 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 02:02:08.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.946 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:02:08.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:08.947 | INFO | Uploaded 36GiB to GPU in 2.4466s (14.7143GiB/s) | 20260508 02:02:08.947 | INFO | Downloaded 34GiB to GPU in 2.49723s (13.6151GiB/s) | 20260508 02:02:14.087 | INFO | Analyzing chunk frames = [4363776,4364288) | 20260508 02:02:14.087 | INFO | MemInfo: { | 20260508 02:02:14.087 | INFO | "gpu" : " 17 GB", | 20260508 02:02:14.087 | INFO | "huge" : "", | 20260508 02:02:14.087 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:14.087 | INFO | "malloc" : " 14 GB", | 20260508 02:02:14.087 | INFO | "pinned" : " 29 GB" | 20260508 02:02:14.087 | INFO | } | 20260508 02:02:14.087 | INFO | Perf report for the last 5.14066 seconds: | 20260508 02:02:14.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:14.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 02:02:14.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:02:14.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:14.087 | INFO | Uploaded 36GiB to GPU in 2.47833s (14.5259GiB/s) | 20260508 02:02:14.087 | INFO | Downloaded 34GiB to GPU in 2.50098s (13.5947GiB/s) | 20260508 02:02:19.231 | INFO | Analyzing chunk frames = [4364288,4364800) | 20260508 02:02:19.231 | INFO | MemInfo: { | 20260508 02:02:19.231 | INFO | "gpu" : " 17 GB", | 20260508 02:02:19.231 | INFO | "huge" : "", | 20260508 02:02:19.231 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:19.231 | INFO | "malloc" : " 14 GB", | 20260508 02:02:19.231 | INFO | "pinned" : " 29 GB" | 20260508 02:02:19.231 | INFO | } | 20260508 02:02:19.231 | INFO | Perf report for the last 5.14419 seconds: | 20260508 02:02:19.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:19.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:02:19.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 02:02:19.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:19.231 | INFO | Uploaded 36GiB to GPU in 2.35309s (15.2991GiB/s) | 20260508 02:02:19.231 | INFO | Downloaded 34GiB to GPU in 2.54512s (13.3589GiB/s) | 20260508 02:02:24.366 | INFO | Analyzing chunk frames = [4364800,4365312) | 20260508 02:02:24.366 | INFO | MemInfo: { | 20260508 02:02:24.366 | INFO | "gpu" : " 17 GB", | 20260508 02:02:24.366 | INFO | "huge" : "", | 20260508 02:02:24.366 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:24.366 | INFO | "malloc" : " 14 GB", | 20260508 02:02:24.366 | INFO | "pinned" : " 29 GB" | 20260508 02:02:24.366 | INFO | } | 20260508 02:02:24.366 | INFO | Perf report for the last 5.13497 seconds: | 20260508 02:02:24.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:24.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.366 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:02:24.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.366 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:02:24.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:24.367 | INFO | Uploaded 36GiB to GPU in 2.41362s (14.9154GiB/s) | 20260508 02:02:24.367 | INFO | Downloaded 34GiB to GPU in 2.41962s (14.0518GiB/s) | 20260508 02:02:29.504 | INFO | Analyzing chunk frames = [4365312,4365824) | 20260508 02:02:29.504 | INFO | MemInfo: { | 20260508 02:02:29.504 | INFO | "gpu" : " 17 GB", | 20260508 02:02:29.504 | INFO | "huge" : "", | 20260508 02:02:29.504 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:29.504 | INFO | "malloc" : " 14 GB", | 20260508 02:02:29.504 | INFO | "pinned" : " 29 GB" | 20260508 02:02:29.504 | INFO | } | 20260508 02:02:29.504 | INFO | Perf report for the last 5.1381 seconds: | 20260508 02:02:29.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:29.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:02:29.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:02:29.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:29.504 | INFO | Uploaded 36GiB to GPU in 2.41862s (14.8845GiB/s) | 20260508 02:02:29.504 | INFO | Downloaded 34GiB to GPU in 2.42083s (14.0448GiB/s) | 20260508 02:02:34.641 | INFO | Analyzing chunk frames = [4365824,4366336) | 20260508 02:02:34.641 | INFO | MemInfo: { | 20260508 02:02:34.641 | INFO | "gpu" : " 17 GB", | 20260508 02:02:34.641 | INFO | "huge" : "", | 20260508 02:02:34.641 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:34.641 | INFO | "malloc" : " 14 GB", | 20260508 02:02:34.641 | INFO | "pinned" : " 29 GB" | 20260508 02:02:34.641 | INFO | } | 20260508 02:02:34.642 | INFO | Perf report for the last 5.1376 seconds: | 20260508 02:02:34.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:34.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:02:34.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:02:34.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:34.642 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:02:34.642 | INFO | Total mem usage: 16.7 GiB | 20260508 02:02:34.642 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:02:34.642 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 02:02:34.642 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:02:34.642 | INFO | High Water Breakdown: | 20260508 02:02:34.642 | INFO | 60.1% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:02:34.642 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:02:34.642 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:02:34.642 | INFO | ------------------------------------------ | 20260508 02:02:34.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:02:34.642 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:02:34.642 | INFO | Total mem usage: 26.5 GiB | 20260508 02:02:34.642 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:02:34.642 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:02:34.642 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:02:34.642 | INFO | High Water Breakdown: | 20260508 02:02:34.642 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:02:34.642 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:02:34.642 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:02:34.642 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:02:34.642 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:02:34.642 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:02:34.642 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:02:34.642 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:02:34.642 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:02:34.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:02:34.642 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:02:34.642 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:02:34.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:02:34.642 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:02:34.642 | INFO | ------------------------------------------ | 20260508 02:02:34.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:02:34.642 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:02:34.642 | INFO | Total mem usage: 27.8 GiB | 20260508 02:02:34.642 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:02:34.642 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:02:34.642 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:02:34.642 | INFO | High Water Breakdown: | 20260508 02:02:34.642 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:02:34.642 | INFO | ------------------------------------------ | 20260508 02:02:34.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:02:34.642 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:02:34.642 | INFO | Total mem usage: 15.7 GiB | 20260508 02:02:34.642 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:02:34.642 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:02:34.642 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:02:34.642 | INFO | High Water Breakdown: | 20260508 02:02:34.642 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:02:34.642 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:02:34.642 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:02:34.642 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:02:34.642 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:02:34.642 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:02:34.642 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:02:34.642 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:02:34.642 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:02:34.642 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:02:34.642 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:02:34.642 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:02:34.642 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:02:34.642 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:02:34.642 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:02:34.642 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:02:34.642 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:02:34.642 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:02:34.642 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:02:34.642 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:02:34.642 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:02:34.642 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:02:34.642 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:02:34.642 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:02:34.642 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:02:34.642 | INFO | ------------------------------------------ | 20260508 02:02:34.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:02:34.642 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:02:34.642 | INFO | Uploaded 36GiB to GPU in 2.54743s (14.1319GiB/s) | 20260508 02:02:34.643 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:02:34.643 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:02:34.643 | INFO | Downloaded 34GiB to GPU in 2.4608s (13.8167GiB/s) | 20260508 02:02:34.643 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 1.27e+00 | 20260508 02:02:34.643 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.21e-02 / 6.33e-03 | 20260508 02:02:34.643 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.45e+00, 5.84e-01 / 1.74e-01 | 20260508 02:02:34.643 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 9.32e-02 | 20260508 02:02:34.643 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.61e+01, 4.02e+00 / 1.71e+00 | 20260508 02:02:34.643 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.64e+00 / 1.51e+00 | 20260508 02:02:34.643 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.22e-01 / 2.35e-01 | 20260508 02:02:34.643 | INFO | PerfReport: 3.30%, Metrics -- 1.08e+00 / 1.25e+00, 2.64e-01 / 1.09e-01 | 20260508 02:02:34.643 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.32e-01 / 3.66e-01, 5.25e-02 / 2.21e-02 | 20260508 02:02:34.643 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:02:34.643 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:02:34.643 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:02:34.643 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 24.30%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 30.79%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 2.02%, LockUpload -- 2.15e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 2.90%, LockDownload -- 3.09e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 15.54%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 02:02:34.644 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260508 02:02:34.740 | INFO | Starting a round of Baz IO | 20260508 02:02:38.185 | INFO | Populating RTMetrics report | 20260508 02:02:38.185 | INFO | RTMetrics report populated | 20260508 02:02:38.185 | INFO | Opening tmp RTMetrics json file | 20260508 02:02:38.216 | INFO | Serializing RTMetrics to json | 20260508 02:02:38.246 | INFO | Serialization done, swapping tmp file now... | 20260508 02:02:38.262 | INFO | RTMetrics done | 20260508 02:02:38.283 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.543 seconds | 20260508 02:02:38.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.65% / 0.00% / 58.34% | 20260508 02:02:38.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.777149% / 0% / 6.75519e-05% | 20260508 02:02:38.283 | INFO | Baz IO overall: | 20260508 02:02:38.283 | INFO | BytesWritten : 1974462986621 | 20260508 02:02:38.283 | INFO | NumEvents : 954761540923 | 20260508 02:02:39.781 | INFO | Analyzing chunk frames = [4366336,4366848) | 20260508 02:02:39.781 | INFO | MemInfo: { | 20260508 02:02:39.781 | INFO | "gpu" : " 17 GB", | 20260508 02:02:39.781 | INFO | "huge" : "", | 20260508 02:02:39.781 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:39.781 | INFO | "malloc" : " 14 GB", | 20260508 02:02:39.781 | INFO | "pinned" : " 29 GB" | 20260508 02:02:39.781 | INFO | } | 20260508 02:02:39.781 | INFO | Perf report for the last 5.13906 seconds: | 20260508 02:02:39.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:39.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:39.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:39.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:39.781 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 02:02:39.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:39.781 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260508 02:02:39.781 | INFO | PRE_HQ : 65.7%, 1.7, 56.3ms, 0 | 20260508 02:02:39.781 | INFO | BAZWRITER : 68.5%, 4.2, 147.4ms, 0 | 20260508 02:02:39.781 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:02:39.781 | INFO | Uploaded 36GiB to GPU in 2.59212s (13.8883GiB/s) | 20260508 02:02:39.781 | INFO | Downloaded 38GiB to GPU in 2.68598s (14.1475GiB/s) | 20260508 02:02:44.926 | INFO | Analyzing chunk frames = [4366848,4367360) | 20260508 02:02:44.926 | INFO | MemInfo: { | 20260508 02:02:44.926 | INFO | "gpu" : " 17 GB", | 20260508 02:02:44.926 | INFO | "huge" : "", | 20260508 02:02:44.926 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:44.926 | INFO | "malloc" : " 14 GB", | 20260508 02:02:44.926 | INFO | "pinned" : " 29 GB" | 20260508 02:02:44.926 | INFO | } | 20260508 02:02:44.926 | INFO | Perf report for the last 5.14571 seconds: | 20260508 02:02:44.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:44.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.926 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 02:02:44.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.927 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:02:44.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:44.927 | INFO | Uploaded 36GiB to GPU in 2.55965s (14.0644GiB/s) | 20260508 02:02:44.927 | INFO | Downloaded 34GiB to GPU in 2.41836s (14.0591GiB/s) | 20260508 02:02:50.060 | INFO | Analyzing chunk frames = [4367360,4367872) | 20260508 02:02:50.060 | INFO | MemInfo: { | 20260508 02:02:50.060 | INFO | "gpu" : " 17 GB", | 20260508 02:02:50.060 | INFO | "huge" : "", | 20260508 02:02:50.060 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:50.061 | INFO | "malloc" : " 14 GB", | 20260508 02:02:50.061 | INFO | "pinned" : " 29 GB" | 20260508 02:02:50.061 | INFO | } | 20260508 02:02:50.061 | INFO | Perf report for the last 5.13432 seconds: | 20260508 02:02:50.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:50.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 02:02:50.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:02:50.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:50.061 | INFO | Uploaded 36GiB to GPU in 2.58872s (13.9065GiB/s) | 20260508 02:02:50.061 | INFO | Downloaded 34GiB to GPU in 2.48923s (13.6588GiB/s) | 20260508 02:02:55.197 | INFO | Analyzing chunk frames = [4367872,4368384) | 20260508 02:02:55.197 | INFO | MemInfo: { | 20260508 02:02:55.197 | INFO | "gpu" : " 17 GB", | 20260508 02:02:55.197 | INFO | "huge" : "", | 20260508 02:02:55.197 | INFO | "huge_shared" : " 29 GB", | 20260508 02:02:55.197 | INFO | "malloc" : " 14 GB", | 20260508 02:02:55.197 | INFO | "pinned" : " 29 GB" | 20260508 02:02:55.197 | INFO | } | 20260508 02:02:55.197 | INFO | Perf report for the last 5.13634 seconds: | 20260508 02:02:55.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:02:55.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.197 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 02:02:55.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.197 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:02:55.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:02:55.198 | INFO | Uploaded 36GiB to GPU in 2.56008s (14.0621GiB/s) | 20260508 02:02:55.198 | INFO | Downloaded 34GiB to GPU in 2.48557s (13.679GiB/s) | 20260508 02:03:00.338 | INFO | Analyzing chunk frames = [4368384,4368896) | 20260508 02:03:00.338 | INFO | MemInfo: { | 20260508 02:03:00.338 | INFO | "gpu" : " 17 GB", | 20260508 02:03:00.338 | INFO | "huge" : "", | 20260508 02:03:00.338 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:00.338 | INFO | "malloc" : " 14 GB", | 20260508 02:03:00.338 | INFO | "pinned" : " 29 GB" | 20260508 02:03:00.338 | INFO | } | 20260508 02:03:00.338 | INFO | Perf report for the last 5.14114 seconds: | 20260508 02:03:00.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:00.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.338 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 02:03:00.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.338 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:03:00.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:00.339 | INFO | Uploaded 36GiB to GPU in 2.41604s (14.9004GiB/s) | 20260508 02:03:00.339 | INFO | Downloaded 34GiB to GPU in 2.51454s (13.5213GiB/s) | 20260508 02:03:05.478 | INFO | Analyzing chunk frames = [4368896,4369408) | 20260508 02:03:05.478 | INFO | MemInfo: { | 20260508 02:03:05.478 | INFO | "gpu" : " 17 GB", | 20260508 02:03:05.478 | INFO | "huge" : "", | 20260508 02:03:05.478 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:05.478 | INFO | "malloc" : " 14 GB", | 20260508 02:03:05.478 | INFO | "pinned" : " 29 GB" | 20260508 02:03:05.478 | INFO | } | 20260508 02:03:05.478 | INFO | Perf report for the last 5.13972 seconds: | 20260508 02:03:05.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:05.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:03:05.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260508 02:03:05.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:05.478 | INFO | Uploaded 36GiB to GPU in 2.40931s (14.9421GiB/s) | 20260508 02:03:05.478 | INFO | Downloaded 34GiB to GPU in 2.41778s (14.0625GiB/s) | 20260508 02:03:10.616 | INFO | Analyzing chunk frames = [4369408,4369920) | 20260508 02:03:10.616 | INFO | MemInfo: { | 20260508 02:03:10.616 | INFO | "gpu" : " 17 GB", | 20260508 02:03:10.616 | INFO | "huge" : "", | 20260508 02:03:10.616 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:10.617 | INFO | "malloc" : " 14 GB", | 20260508 02:03:10.617 | INFO | "pinned" : " 29 GB" | 20260508 02:03:10.617 | INFO | } | 20260508 02:03:10.617 | INFO | Perf report for the last 5.13876 seconds: | 20260508 02:03:10.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:10.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:03:10.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 02:03:10.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:10.617 | INFO | Uploaded 36GiB to GPU in 2.43408s (14.79GiB/s) | 20260508 02:03:10.617 | INFO | Downloaded 34GiB to GPU in 2.4125s (14.0933GiB/s) | 20260508 02:03:15.754 | INFO | Analyzing chunk frames = [4369920,4370432) | 20260508 02:03:15.754 | INFO | MemInfo: { | 20260508 02:03:15.754 | INFO | "gpu" : " 17 GB", | 20260508 02:03:15.754 | INFO | "huge" : "", | 20260508 02:03:15.754 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:15.755 | INFO | "malloc" : " 14 GB", | 20260508 02:03:15.755 | INFO | "pinned" : " 29 GB" | 20260508 02:03:15.755 | INFO | } | 20260508 02:03:15.755 | INFO | Perf report for the last 5.13813 seconds: | 20260508 02:03:15.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:15.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.755 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:03:15.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.755 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 02:03:15.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:15.756 | INFO | Uploaded 36GiB to GPU in 2.47064s (14.5711GiB/s) | 20260508 02:03:15.756 | INFO | Downloaded 34GiB to GPU in 2.43235s (13.9783GiB/s) | 20260508 02:03:15.869 | INFO | Starting a round of Baz IO | 20260508 02:03:19.316 | INFO | Populating RTMetrics report | 20260508 02:03:19.317 | INFO | RTMetrics report populated | 20260508 02:03:19.317 | INFO | Opening tmp RTMetrics json file | 20260508 02:03:19.332 | INFO | Serializing RTMetrics to json | 20260508 02:03:19.339 | INFO | Serialization done, swapping tmp file now... | 20260508 02:03:19.340 | INFO | RTMetrics done | 20260508 02:03:19.407 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.538 seconds | 20260508 02:03:19.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.65% / 0.00% / 58.35% | 20260508 02:03:19.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.775858% / 3.97364e-06% / 8.74201e-05% | 20260508 02:03:19.407 | INFO | Baz IO overall: | 20260508 02:03:19.407 | INFO | BytesWritten : 1976235371822 | 20260508 02:03:19.407 | INFO | NumEvents : 955612894619 | 20260508 02:03:20.899 | INFO | Analyzing chunk frames = [4370432,4370944) | 20260508 02:03:20.899 | INFO | MemInfo: { | 20260508 02:03:20.899 | INFO | "gpu" : " 17 GB", | 20260508 02:03:20.899 | INFO | "huge" : "", | 20260508 02:03:20.899 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:20.899 | INFO | "malloc" : " 14 GB", | 20260508 02:03:20.899 | INFO | "pinned" : " 29 GB" | 20260508 02:03:20.899 | INFO | } | 20260508 02:03:20.899 | INFO | Perf report for the last 5.14417 seconds: | 20260508 02:03:20.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:20.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:20.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:20.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:20.899 | INFO | ANALYSIS : 68.6%, 2.9, 104.4ms, 0 | 20260508 02:03:20.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:20.899 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260508 02:03:20.899 | INFO | PRE_HQ : 66.8%, 1.8, 62.8ms, 0 | 20260508 02:03:20.899 | INFO | BAZWRITER : 68.3%, 4.0, 142.0ms, 0 | 20260508 02:03:20.899 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:03:20.899 | INFO | Uploaded 36GiB to GPU in 2.54057s (14.1701GiB/s) | 20260508 02:03:20.899 | INFO | Downloaded 38GiB to GPU in 2.6998s (14.0751GiB/s) | 20260508 02:03:26.043 | INFO | Analyzing chunk frames = [4370944,4371456) | 20260508 02:03:26.043 | INFO | MemInfo: { | 20260508 02:03:26.043 | INFO | "gpu" : " 17 GB", | 20260508 02:03:26.043 | INFO | "huge" : "", | 20260508 02:03:26.043 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:26.043 | INFO | "malloc" : " 14 GB", | 20260508 02:03:26.043 | INFO | "pinned" : " 29 GB" | 20260508 02:03:26.043 | INFO | } | 20260508 02:03:26.043 | INFO | Perf report for the last 5.14408 seconds: | 20260508 02:03:26.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:26.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:03:26.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:03:26.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:26.043 | INFO | Uploaded 36GiB to GPU in 2.53949s (14.1761GiB/s) | 20260508 02:03:26.043 | INFO | Downloaded 34GiB to GPU in 2.42853s (14.0002GiB/s) | 20260508 02:03:31.179 | INFO | Analyzing chunk frames = [4371456,4371968) | 20260508 02:03:31.179 | INFO | MemInfo: { | 20260508 02:03:31.179 | INFO | "gpu" : " 17 GB", | 20260508 02:03:31.179 | INFO | "huge" : "", | 20260508 02:03:31.179 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:31.179 | INFO | "malloc" : " 14 GB", | 20260508 02:03:31.179 | INFO | "pinned" : " 29 GB" | 20260508 02:03:31.179 | INFO | } | 20260508 02:03:31.179 | INFO | Perf report for the last 5.13601 seconds: | 20260508 02:03:31.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:31.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.179 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 02:03:31.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.179 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:03:31.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:31.180 | INFO | Uploaded 36GiB to GPU in 2.54691s (14.1348GiB/s) | 20260508 02:03:31.180 | INFO | Downloaded 34GiB to GPU in 2.49835s (13.609GiB/s) | 20260508 02:03:36.315 | INFO | Analyzing chunk frames = [4371968,4372480) | 20260508 02:03:36.315 | INFO | MemInfo: { | 20260508 02:03:36.315 | INFO | "gpu" : " 17 GB", | 20260508 02:03:36.315 | INFO | "huge" : "", | 20260508 02:03:36.315 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:36.315 | INFO | "malloc" : " 14 GB", | 20260508 02:03:36.315 | INFO | "pinned" : " 29 GB" | 20260508 02:03:36.315 | INFO | } | 20260508 02:03:36.315 | INFO | Perf report for the last 5.13617 seconds: | 20260508 02:03:36.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:36.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.315 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 02:03:36.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.316 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:03:36.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:36.316 | INFO | Uploaded 36GiB to GPU in 2.53261s (14.2146GiB/s) | 20260508 02:03:36.316 | INFO | Downloaded 34GiB to GPU in 2.50558s (13.5697GiB/s) | 20260508 02:03:41.455 | INFO | Analyzing chunk frames = [4372480,4372992) | 20260508 02:03:41.455 | INFO | MemInfo: { | 20260508 02:03:41.455 | INFO | "gpu" : " 17 GB", | 20260508 02:03:41.455 | INFO | "huge" : "", | 20260508 02:03:41.455 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:41.455 | INFO | "malloc" : " 14 GB", | 20260508 02:03:41.455 | INFO | "pinned" : " 29 GB" | 20260508 02:03:41.455 | INFO | } | 20260508 02:03:41.456 | INFO | Perf report for the last 5.14014 seconds: | 20260508 02:03:41.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:41.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 02:03:41.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:03:41.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:41.456 | INFO | Uploaded 36GiB to GPU in 2.40808s (14.9497GiB/s) | 20260508 02:03:41.456 | INFO | Downloaded 34GiB to GPU in 2.53956s (13.3881GiB/s) | 20260508 02:03:46.595 | INFO | Analyzing chunk frames = [4372992,4373504) | 20260508 02:03:46.595 | INFO | MemInfo: { | 20260508 02:03:46.595 | INFO | "gpu" : " 17 GB", | 20260508 02:03:46.595 | INFO | "huge" : "", | 20260508 02:03:46.595 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:46.595 | INFO | "malloc" : " 14 GB", | 20260508 02:03:46.595 | INFO | "pinned" : " 29 GB" | 20260508 02:03:46.595 | INFO | } | 20260508 02:03:46.595 | INFO | Perf report for the last 5.13942 seconds: | 20260508 02:03:46.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:46.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.595 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:03:46.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.595 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:03:46.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:46.596 | INFO | Uploaded 36GiB to GPU in 2.39456s (15.0341GiB/s) | 20260508 02:03:46.596 | INFO | Downloaded 34GiB to GPU in 2.41324s (14.0889GiB/s) | 20260508 02:03:51.736 | INFO | Analyzing chunk frames = [4373504,4374016) | 20260508 02:03:51.736 | INFO | MemInfo: { | 20260508 02:03:51.736 | INFO | "gpu" : " 17 GB", | 20260508 02:03:51.736 | INFO | "huge" : "", | 20260508 02:03:51.736 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:51.736 | INFO | "malloc" : " 14 GB", | 20260508 02:03:51.736 | INFO | "pinned" : " 29 GB" | 20260508 02:03:51.737 | INFO | } | 20260508 02:03:51.737 | INFO | Perf report for the last 5.14164 seconds: | 20260508 02:03:51.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:51.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 02:03:51.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:03:51.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:51.737 | INFO | Uploaded 36GiB to GPU in 2.51934s (14.2894GiB/s) | 20260508 02:03:51.737 | INFO | Downloaded 34GiB to GPU in 2.45292s (13.861GiB/s) | 20260508 02:03:56.872 | INFO | Analyzing chunk frames = [4374016,4374528) | 20260508 02:03:56.872 | INFO | MemInfo: { | 20260508 02:03:56.872 | INFO | "gpu" : " 17 GB", | 20260508 02:03:56.872 | INFO | "huge" : "", | 20260508 02:03:56.872 | INFO | "huge_shared" : " 29 GB", | 20260508 02:03:56.872 | INFO | "malloc" : " 14 GB", | 20260508 02:03:56.872 | INFO | "pinned" : " 29 GB" | 20260508 02:03:56.872 | INFO | } | 20260508 02:03:56.872 | INFO | Perf report for the last 5.13557 seconds: | 20260508 02:03:56.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:03:56.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.872 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:03:56.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.872 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:03:56.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:03:56.873 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:03:56.873 | INFO | Total mem usage: 16.7 GiB | 20260508 02:03:56.873 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:03:56.873 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.37%) | 20260508 02:03:56.873 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:03:56.873 | INFO | High Water Breakdown: | 20260508 02:03:56.873 | INFO | Uploaded 36GiB to GPU in 2.54194s (14.1624GiB/s) | 20260508 02:03:56.873 | INFO | 60.5% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:03:56.873 | INFO | 39.3% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:03:56.873 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:03:56.873 | INFO | Downloaded 34GiB to GPU in 2.45826s (13.8309GiB/s) | 20260508 02:03:56.873 | INFO | ------------------------------------------ | 20260508 02:03:56.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:03:56.873 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:03:56.873 | INFO | Total mem usage: 26.5 GiB | 20260508 02:03:56.873 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:03:56.873 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260508 02:03:56.873 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:03:56.873 | INFO | High Water Breakdown: | 20260508 02:03:56.873 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:03:56.873 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:03:56.873 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:03:56.873 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:03:56.873 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:03:56.873 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:03:56.873 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:03:56.873 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:03:56.873 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:03:56.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:03:56.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:03:56.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:03:56.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:03:56.873 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:03:56.873 | INFO | ------------------------------------------ | 20260508 02:03:56.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:03:56.873 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:03:56.873 | INFO | Total mem usage: 27.8 GiB | 20260508 02:03:56.873 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:03:56.873 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:03:56.873 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:03:56.873 | INFO | High Water Breakdown: | 20260508 02:03:56.873 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:03:56.873 | INFO | ------------------------------------------ | 20260508 02:03:56.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:03:56.873 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:03:56.873 | INFO | Total mem usage: 15.7 GiB | 20260508 02:03:56.873 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:03:56.874 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:03:56.874 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:03:56.874 | INFO | High Water Breakdown: | 20260508 02:03:56.874 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:03:56.874 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:03:56.874 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:03:56.874 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:03:56.874 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:03:56.874 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:03:56.874 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:03:56.874 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:03:56.874 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:03:56.874 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:03:56.874 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:03:56.874 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:03:56.874 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:03:56.874 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:03:56.874 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:03:56.874 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:03:56.874 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:03:56.874 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:03:56.874 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:03:56.874 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:03:56.874 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:03:56.874 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:03:56.874 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:03:56.874 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:03:56.874 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:03:56.874 | INFO | ------------------------------------------ | 20260508 02:03:56.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:03:56.874 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:03:56.874 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:03:56.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:03:56.874 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.67e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.23e-02 / 5.19e-03 | 20260508 02:03:56.874 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.44e+00, 5.82e-01 / 2.20e-01 | 20260508 02:03:56.874 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.03e-01 / 1.18e-01 | 20260508 02:03:56.874 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.61e+01, 4.09e+00 / 1.81e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.64e+00 / 1.51e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.70e+00, 4.23e-01 / 2.39e-01 | 20260508 02:03:56.874 | INFO | PerfReport: 3.30%, Metrics -- 1.08e+00 / 1.24e+00, 2.68e-01 / 1.32e-01 | 20260508 02:03:56.874 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.68e-01, 5.31e-02 / 3.24e-02 | 20260508 02:03:56.874 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:03:56.874 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:03:56.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:03:56.874 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 24.48%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 24.35%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 2.26%, LockUpload -- 2.41e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 2.71%, LockDownload -- 2.89e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:03:56.874 | INFO | PerfReport: 0.01%, Misc -- 1.57e-02 / 0.00e+00, 3.89e-02 / 0.00e+00 | 20260508 02:03:56.970 | INFO | Starting a round of Baz IO | 20260508 02:04:00.422 | INFO | Populating RTMetrics report | 20260508 02:04:00.422 | INFO | RTMetrics report populated | 20260508 02:04:00.422 | INFO | Opening tmp RTMetrics json file | 20260508 02:04:00.443 | INFO | Serializing RTMetrics to json | 20260508 02:04:00.445 | INFO | Serialization done, swapping tmp file now... | 20260508 02:04:00.445 | INFO | RTMetrics done | 20260508 02:04:00.512 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.542 seconds | 20260508 02:04:00.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.65% / 0.00% / 58.35% | 20260508 02:04:00.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.775663% / 3.97364e-06% / 7.15256e-05% | 20260508 02:04:00.512 | INFO | Baz IO overall: | 20260508 02:04:00.512 | INFO | BytesWritten : 1978006943045 | 20260508 02:04:00.512 | INFO | NumEvents : 956463660373 | 20260508 02:04:02.006 | INFO | Analyzing chunk frames = [4374528,4375040) | 20260508 02:04:02.006 | INFO | MemInfo: { | 20260508 02:04:02.006 | INFO | "gpu" : " 17 GB", | 20260508 02:04:02.006 | INFO | "huge" : "", | 20260508 02:04:02.006 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:02.006 | INFO | "malloc" : " 14 GB", | 20260508 02:04:02.006 | INFO | "pinned" : " 29 GB" | 20260508 02:04:02.006 | INFO | } | 20260508 02:04:02.006 | INFO | Perf report for the last 5.13391 seconds: | 20260508 02:04:02.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:02.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:02.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:02.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:02.006 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:04:02.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:02.006 | INFO | RT_METRICS : 40.5%, 1.0, 21.2ms, 0 | 20260508 02:04:02.006 | INFO | PRE_HQ : 65.7%, 1.7, 56.5ms, 0 | 20260508 02:04:02.006 | INFO | BAZWRITER : 68.6%, 3.7, 129.6ms, 0 | 20260508 02:04:02.006 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 02:04:02.006 | INFO | Uploaded 36GiB to GPU in 2.56605s (14.0293GiB/s) | 20260508 02:04:02.006 | INFO | Downloaded 38GiB to GPU in 2.69156s (14.1182GiB/s) | 20260508 02:04:07.150 | INFO | Analyzing chunk frames = [4375040,4375552) | 20260508 02:04:07.150 | INFO | MemInfo: { | 20260508 02:04:07.150 | INFO | "gpu" : " 17 GB", | 20260508 02:04:07.150 | INFO | "huge" : "", | 20260508 02:04:07.150 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:07.150 | INFO | "malloc" : " 14 GB", | 20260508 02:04:07.150 | INFO | "pinned" : " 29 GB" | 20260508 02:04:07.150 | INFO | } | 20260508 02:04:07.150 | INFO | Perf report for the last 5.1436 seconds: | 20260508 02:04:07.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:07.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:04:07.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:04:07.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:07.150 | INFO | Uploaded 36GiB to GPU in 2.54203s (14.1619GiB/s) | 20260508 02:04:07.150 | INFO | Downloaded 34GiB to GPU in 2.42817s (14.0023GiB/s) | 20260508 02:04:12.292 | INFO | Analyzing chunk frames = [4375552,4376064) | 20260508 02:04:12.293 | INFO | MemInfo: { | 20260508 02:04:12.293 | INFO | "gpu" : " 17 GB", | 20260508 02:04:12.293 | INFO | "huge" : "", | 20260508 02:04:12.293 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:12.293 | INFO | "malloc" : " 14 GB", | 20260508 02:04:12.293 | INFO | "pinned" : " 29 GB" | 20260508 02:04:12.293 | INFO | } | 20260508 02:04:12.293 | INFO | Perf report for the last 5.14306 seconds: | 20260508 02:04:12.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:12.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.293 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:04:12.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.293 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:04:12.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:12.294 | INFO | Uploaded 36GiB to GPU in 2.52859s (14.2372GiB/s) | 20260508 02:04:12.294 | INFO | Downloaded 34GiB to GPU in 2.48522s (13.6809GiB/s) | 20260508 02:04:17.431 | INFO | Analyzing chunk frames = [4376064,4376576) | 20260508 02:04:17.431 | INFO | MemInfo: { | 20260508 02:04:17.431 | INFO | "gpu" : " 17 GB", | 20260508 02:04:17.431 | INFO | "huge" : "", | 20260508 02:04:17.431 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:17.431 | INFO | "malloc" : " 14 GB", | 20260508 02:04:17.431 | INFO | "pinned" : " 29 GB" | 20260508 02:04:17.431 | INFO | } | 20260508 02:04:17.431 | INFO | Perf report for the last 5.13842 seconds: | 20260508 02:04:17.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:17.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.431 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:04:17.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.431 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:04:17.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:17.432 | INFO | Uploaded 36GiB to GPU in 2.53106s (14.2233GiB/s) | 20260508 02:04:17.432 | INFO | Downloaded 34GiB to GPU in 2.49448s (13.6301GiB/s) | 20260508 02:04:22.566 | INFO | Analyzing chunk frames = [4376576,4377088) | 20260508 02:04:22.566 | INFO | MemInfo: { | 20260508 02:04:22.566 | INFO | "gpu" : " 17 GB", | 20260508 02:04:22.566 | INFO | "huge" : "", | 20260508 02:04:22.566 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:22.566 | INFO | "malloc" : " 14 GB", | 20260508 02:04:22.566 | INFO | "pinned" : " 29 GB" | 20260508 02:04:22.566 | INFO | } | 20260508 02:04:22.566 | INFO | Perf report for the last 5.13496 seconds: | 20260508 02:04:22.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:22.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.566 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 02:04:22.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.566 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 02:04:22.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:22.567 | INFO | Uploaded 36GiB to GPU in 2.42998s (14.8149GiB/s) | 20260508 02:04:22.567 | INFO | Downloaded 34GiB to GPU in 2.56031s (13.2797GiB/s) | 20260508 02:04:27.710 | INFO | Analyzing chunk frames = [4377088,4377600) | 20260508 02:04:27.710 | INFO | MemInfo: { | 20260508 02:04:27.710 | INFO | "gpu" : " 17 GB", | 20260508 02:04:27.710 | INFO | "huge" : "", | 20260508 02:04:27.710 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:27.710 | INFO | "malloc" : " 14 GB", | 20260508 02:04:27.710 | INFO | "pinned" : " 29 GB" | 20260508 02:04:27.710 | INFO | } | 20260508 02:04:27.710 | INFO | Perf report for the last 5.14384 seconds: | 20260508 02:04:27.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:27.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.710 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:04:27.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.710 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260508 02:04:27.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:27.711 | INFO | Uploaded 36GiB to GPU in 2.41154s (14.9282GiB/s) | 20260508 02:04:27.711 | INFO | Downloaded 34GiB to GPU in 2.4049s (14.1378GiB/s) | 20260508 02:04:32.851 | INFO | Analyzing chunk frames = [4377600,4378112) | 20260508 02:04:32.851 | INFO | MemInfo: { | 20260508 02:04:32.851 | INFO | "gpu" : " 17 GB", | 20260508 02:04:32.851 | INFO | "huge" : "", | 20260508 02:04:32.851 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:32.852 | INFO | "malloc" : " 14 GB", | 20260508 02:04:32.852 | INFO | "pinned" : " 29 GB" | 20260508 02:04:32.852 | INFO | } | 20260508 02:04:32.852 | INFO | Perf report for the last 5.14166 seconds: | 20260508 02:04:32.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:32.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:04:32.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 02:04:32.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:32.852 | INFO | Uploaded 36GiB to GPU in 2.44807s (14.7055GiB/s) | 20260508 02:04:32.852 | INFO | Downloaded 34GiB to GPU in 2.4253s (14.0189GiB/s) | 20260508 02:04:37.995 | INFO | Analyzing chunk frames = [4378112,4378624) | 20260508 02:04:37.995 | INFO | MemInfo: { | 20260508 02:04:37.995 | INFO | "gpu" : " 17 GB", | 20260508 02:04:37.995 | INFO | "huge" : "", | 20260508 02:04:37.995 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:37.996 | INFO | "malloc" : " 14 GB", | 20260508 02:04:37.996 | INFO | "pinned" : " 29 GB" | 20260508 02:04:37.996 | INFO | } | 20260508 02:04:37.996 | INFO | Perf report for the last 5.14402 seconds: | 20260508 02:04:37.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:37.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:04:37.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 02:04:37.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:37.996 | INFO | Uploaded 36GiB to GPU in 2.48579s (14.4823GiB/s) | 20260508 02:04:37.996 | INFO | Downloaded 34GiB to GPU in 2.42383s (14.0274GiB/s) | 20260508 02:04:38.111 | INFO | Starting a round of Baz IO | 20260508 02:04:41.558 | INFO | Populating RTMetrics report | 20260508 02:04:41.558 | INFO | RTMetrics report populated | 20260508 02:04:41.558 | INFO | Opening tmp RTMetrics json file | 20260508 02:04:41.559 | INFO | Serializing RTMetrics to json | 20260508 02:04:41.578 | INFO | Serialization done, swapping tmp file now... | 20260508 02:04:41.580 | INFO | RTMetrics done | 20260508 02:04:41.655 | INFO | Finished a round of Baz IO: Wrote 1.648 GiB in 3.544 seconds | 20260508 02:04:41.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.64% / 0.00% / 58.35% | 20260508 02:04:41.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.771761% / 7.94729e-06% / 5.16574e-05% | 20260508 02:04:41.655 | INFO | Baz IO overall: | 20260508 02:04:41.655 | INFO | BytesWritten : 1979776225576 | 20260508 02:04:41.655 | INFO | NumEvents : 957312686397 | 20260508 02:04:43.132 | INFO | Analyzing chunk frames = [4378624,4379136) | 20260508 02:04:43.132 | INFO | MemInfo: { | 20260508 02:04:43.132 | INFO | "gpu" : " 17 GB", | 20260508 02:04:43.132 | INFO | "huge" : "", | 20260508 02:04:43.132 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:43.132 | INFO | "malloc" : " 14 GB", | 20260508 02:04:43.132 | INFO | "pinned" : " 29 GB" | 20260508 02:04:43.132 | INFO | } | 20260508 02:04:43.132 | INFO | Perf report for the last 5.13638 seconds: | 20260508 02:04:43.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:43.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:43.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:43.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:43.132 | INFO | ANALYSIS : 68.7%, 2.9, 104.6ms, 0 | 20260508 02:04:43.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:43.132 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 02:04:43.132 | INFO | PRE_HQ : 67.5%, 1.8, 62.0ms, 0 | 20260508 02:04:43.132 | INFO | BAZWRITER : 68.7%, 3.6, 128.1ms, 0 | 20260508 02:04:43.132 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:04:43.132 | INFO | Uploaded 36GiB to GPU in 2.63952s (13.6388GiB/s) | 20260508 02:04:43.132 | INFO | Downloaded 38GiB to GPU in 2.71048s (14.0197GiB/s) | 20260508 02:04:48.265 | INFO | Analyzing chunk frames = [4379136,4379648) | 20260508 02:04:48.265 | INFO | MemInfo: { | 20260508 02:04:48.265 | INFO | "gpu" : " 17 GB", | 20260508 02:04:48.265 | INFO | "huge" : "", | 20260508 02:04:48.265 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:48.265 | INFO | "malloc" : " 14 GB", | 20260508 02:04:48.265 | INFO | "pinned" : " 29 GB" | 20260508 02:04:48.265 | INFO | } | 20260508 02:04:48.265 | INFO | Perf report for the last 5.13277 seconds: | 20260508 02:04:48.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:48.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 02:04:48.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:04:48.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:48.265 | INFO | Uploaded 36GiB to GPU in 2.59171s (13.8905GiB/s) | 20260508 02:04:48.265 | INFO | Downloaded 34GiB to GPU in 2.4212s (14.0426GiB/s) | 20260508 02:04:53.406 | INFO | Analyzing chunk frames = [4379648,4380160) | 20260508 02:04:53.406 | INFO | MemInfo: { | 20260508 02:04:53.406 | INFO | "gpu" : " 17 GB", | 20260508 02:04:53.406 | INFO | "huge" : "", | 20260508 02:04:53.406 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:53.406 | INFO | "malloc" : " 14 GB", | 20260508 02:04:53.406 | INFO | "pinned" : " 29 GB" | 20260508 02:04:53.406 | INFO | } | 20260508 02:04:53.406 | INFO | Perf report for the last 5.14104 seconds: | 20260508 02:04:53.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:53.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 02:04:53.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:04:53.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:53.406 | INFO | Uploaded 36GiB to GPU in 2.55054s (14.1146GiB/s) | 20260508 02:04:53.406 | INFO | Downloaded 34GiB to GPU in 2.48574s (13.678GiB/s) | 20260508 02:04:58.547 | INFO | Analyzing chunk frames = [4380160,4380672) | 20260508 02:04:58.547 | INFO | MemInfo: { | 20260508 02:04:58.547 | INFO | "gpu" : " 17 GB", | 20260508 02:04:58.547 | INFO | "huge" : "", | 20260508 02:04:58.547 | INFO | "huge_shared" : " 29 GB", | 20260508 02:04:58.547 | INFO | "malloc" : " 14 GB", | 20260508 02:04:58.547 | INFO | "pinned" : " 29 GB" | 20260508 02:04:58.548 | INFO | } | 20260508 02:04:58.548 | INFO | Perf report for the last 5.14158 seconds: | 20260508 02:04:58.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:04:58.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:04:58.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:04:58.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:04:58.548 | INFO | Uploaded 36GiB to GPU in 2.55321s (14.0999GiB/s) | 20260508 02:04:58.548 | INFO | Downloaded 34GiB to GPU in 2.48525s (13.6807GiB/s) | 20260508 02:05:03.685 | INFO | Analyzing chunk frames = [4380672,4381184) | 20260508 02:05:03.685 | INFO | MemInfo: { | 20260508 02:05:03.685 | INFO | "gpu" : " 17 GB", | 20260508 02:05:03.685 | INFO | "huge" : "", | 20260508 02:05:03.685 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:03.685 | INFO | "malloc" : " 14 GB", | 20260508 02:05:03.685 | INFO | "pinned" : " 29 GB" | 20260508 02:05:03.685 | INFO | } | 20260508 02:05:03.685 | INFO | Perf report for the last 5.13727 seconds: | 20260508 02:05:03.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:03.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:05:03.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:05:03.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:03.685 | INFO | Uploaded 36GiB to GPU in 2.49529s (14.4272GiB/s) | 20260508 02:05:03.685 | INFO | Downloaded 34GiB to GPU in 2.59036s (13.1256GiB/s) | 20260508 02:05:08.818 | INFO | Analyzing chunk frames = [4381184,4381696) | 20260508 02:05:08.818 | INFO | MemInfo: { | 20260508 02:05:08.818 | INFO | "gpu" : " 17 GB", | 20260508 02:05:08.818 | INFO | "huge" : "", | 20260508 02:05:08.818 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:08.818 | INFO | "malloc" : " 14 GB", | 20260508 02:05:08.818 | INFO | "pinned" : " 29 GB" | 20260508 02:05:08.818 | INFO | } | 20260508 02:05:08.818 | INFO | Perf report for the last 5.13314 seconds: | 20260508 02:05:08.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:08.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:05:08.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:05:08.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:08.818 | INFO | Uploaded 36GiB to GPU in 2.39097s (15.0566GiB/s) | 20260508 02:05:08.818 | INFO | Downloaded 34GiB to GPU in 2.40857s (14.1163GiB/s) | 20260508 02:05:13.959 | INFO | Analyzing chunk frames = [4381696,4382208) | 20260508 02:05:13.959 | INFO | MemInfo: { | 20260508 02:05:13.959 | INFO | "gpu" : " 17 GB", | 20260508 02:05:13.959 | INFO | "huge" : "", | 20260508 02:05:13.959 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:13.959 | INFO | "malloc" : " 14 GB", | 20260508 02:05:13.959 | INFO | "pinned" : " 29 GB" | 20260508 02:05:13.959 | INFO | } | 20260508 02:05:13.959 | INFO | Perf report for the last 5.14135 seconds: | 20260508 02:05:13.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:13.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.960 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:05:13.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.960 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 02:05:13.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:13.960 | INFO | Uploaded 36GiB to GPU in 2.42023s (14.8746GiB/s) | 20260508 02:05:13.960 | INFO | Downloaded 34GiB to GPU in 2.42766s (14.0053GiB/s) | 20260508 02:05:19.100 | INFO | Analyzing chunk frames = [4382208,4382720) | 20260508 02:05:19.100 | INFO | MemInfo: { | 20260508 02:05:19.100 | INFO | "gpu" : " 17 GB", | 20260508 02:05:19.100 | INFO | "huge" : "", | 20260508 02:05:19.101 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:19.101 | INFO | "malloc" : " 14 GB", | 20260508 02:05:19.101 | INFO | "pinned" : " 29 GB" | 20260508 02:05:19.101 | INFO | } | 20260508 02:05:19.101 | INFO | Perf report for the last 5.14141 seconds: | 20260508 02:05:19.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:19.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:05:19.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 02:05:19.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:19.101 | INFO | Uploaded 36GiB to GPU in 2.56364s (14.0425GiB/s) | 20260508 02:05:19.101 | INFO | Downloaded 34GiB to GPU in 2.47064s (13.7616GiB/s) | 20260508 02:05:19.102 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:05:19.102 | INFO | Total mem usage: 16.7 GiB | 20260508 02:05:19.102 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:05:19.102 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 02:05:19.102 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:05:19.102 | INFO | High Water Breakdown: | 20260508 02:05:19.102 | INFO | 59.8% : 8.73 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:05:19.102 | INFO | 40.1% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:05:19.102 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:05:19.102 | INFO | ------------------------------------------ | 20260508 02:05:19.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:05:19.102 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:05:19.102 | INFO | Total mem usage: 26.5 GiB | 20260508 02:05:19.102 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:05:19.102 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 02:05:19.102 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:05:19.102 | INFO | High Water Breakdown: | 20260508 02:05:19.102 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:05:19.102 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:05:19.102 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:05:19.102 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:05:19.102 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:05:19.102 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:05:19.102 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:05:19.102 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:05:19.102 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:05:19.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:05:19.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:05:19.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:05:19.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:05:19.102 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:05:19.102 | INFO | ------------------------------------------ | 20260508 02:05:19.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:05:19.102 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:05:19.102 | INFO | Total mem usage: 27.8 GiB | 20260508 02:05:19.102 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:05:19.102 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:05:19.102 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:05:19.102 | INFO | High Water Breakdown: | 20260508 02:05:19.102 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:05:19.102 | INFO | ------------------------------------------ | 20260508 02:05:19.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:05:19.102 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:05:19.102 | INFO | Total mem usage: 15.7 GiB | 20260508 02:05:19.102 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:05:19.102 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:05:19.102 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:05:19.102 | INFO | High Water Breakdown: | 20260508 02:05:19.102 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:05:19.102 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:05:19.102 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:05:19.102 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:05:19.103 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:05:19.103 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:05:19.103 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:05:19.103 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:05:19.103 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:05:19.103 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:05:19.103 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:05:19.103 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:05:19.103 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:05:19.103 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:05:19.103 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:05:19.103 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:05:19.103 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:05:19.103 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:05:19.103 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:05:19.103 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:05:19.103 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:05:19.103 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:05:19.103 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:05:19.103 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:05:19.103 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:05:19.103 | INFO | ------------------------------------------ | 20260508 02:05:19.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:05:19.103 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:05:19.103 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:05:19.103 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:05:19.103 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.64e+01, 1.40e+01 / 1.83e+00 | 20260508 02:05:19.103 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.05e-01, 1.26e-02 / 1.53e-02 | 20260508 02:05:19.103 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.49e+00, 5.83e-01 / 1.77e-01 | 20260508 02:05:19.103 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.06e-01 / 1.13e-01 | 20260508 02:05:19.103 | INFO | PerfReport: 16.09%, DME -- 4.21e+01 / 4.63e+01, 4.27e+00 / 2.34e+00 | 20260508 02:05:19.103 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.66e+00 / 1.37e+00 | 20260508 02:05:19.103 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.42e+00 / 2.68e+00, 4.24e-01 / 2.13e-01 | 20260508 02:05:19.103 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.24e+00, 2.68e-01 / 1.13e-01 | 20260508 02:05:19.103 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.70e-01, 5.30e-02 / 2.78e-02 | 20260508 02:05:19.104 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:05:19.104 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:05:19.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:05:19.104 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 24.37%, Download -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 30.70%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 2.27%, LockUpload -- 2.42e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 15.45%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:05:19.104 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 02:05:19.214 | INFO | Starting a round of Baz IO | 20260508 02:05:22.675 | INFO | Populating RTMetrics report | 20260508 02:05:22.675 | INFO | RTMetrics report populated | 20260508 02:05:22.675 | INFO | Opening tmp RTMetrics json file | 20260508 02:05:22.701 | INFO | Serializing RTMetrics to json | 20260508 02:05:22.706 | INFO | Serialization done, swapping tmp file now... | 20260508 02:05:22.727 | INFO | RTMetrics done | 20260508 02:05:22.772 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.558 seconds | 20260508 02:05:22.772 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.64% / 0.00% / 58.36% | 20260508 02:05:22.772 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.77341% / 7.94729e-06% / 4.76837e-05% | 20260508 02:05:22.772 | INFO | Baz IO overall: | 20260508 02:05:22.772 | INFO | BytesWritten : 1981548113065 | 20260508 02:05:22.772 | INFO | NumEvents : 958163646224 | 20260508 02:05:24.244 | INFO | Analyzing chunk frames = [4382720,4383232) | 20260508 02:05:24.244 | INFO | MemInfo: { | 20260508 02:05:24.244 | INFO | "gpu" : " 17 GB", | 20260508 02:05:24.244 | INFO | "huge" : "", | 20260508 02:05:24.244 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:24.244 | INFO | "malloc" : " 14 GB", | 20260508 02:05:24.244 | INFO | "pinned" : " 29 GB" | 20260508 02:05:24.244 | INFO | } | 20260508 02:05:24.244 | INFO | Perf report for the last 5.14346 seconds: | 20260508 02:05:24.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:24.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:24.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:24.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:24.244 | INFO | ANALYSIS : 68.8%, 2.9, 104.9ms, 0 | 20260508 02:05:24.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:24.244 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260508 02:05:24.244 | INFO | PRE_HQ : 67.9%, 1.8, 63.2ms, 0 | 20260508 02:05:24.245 | INFO | BAZWRITER : 68.9%, 4.0, 141.0ms, 0 | 20260508 02:05:24.245 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:05:24.245 | INFO | Uploaded 36GiB to GPU in 2.56898s (14.0133GiB/s) | 20260508 02:05:24.245 | INFO | Downloaded 38GiB to GPU in 2.70415s (14.0525GiB/s) | 20260508 02:05:29.387 | INFO | Analyzing chunk frames = [4383232,4383744) | 20260508 02:05:29.387 | INFO | MemInfo: { | 20260508 02:05:29.387 | INFO | "gpu" : " 17 GB", | 20260508 02:05:29.387 | INFO | "huge" : "", | 20260508 02:05:29.387 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:29.387 | INFO | "malloc" : " 14 GB", | 20260508 02:05:29.387 | INFO | "pinned" : " 29 GB" | 20260508 02:05:29.387 | INFO | } | 20260508 02:05:29.387 | INFO | Perf report for the last 5.14254 seconds: | 20260508 02:05:29.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:29.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 02:05:29.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:05:29.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:29.387 | INFO | Uploaded 36GiB to GPU in 2.53996s (14.1735GiB/s) | 20260508 02:05:29.387 | INFO | Downloaded 34GiB to GPU in 2.42917s (13.9966GiB/s) | 20260508 02:05:34.520 | INFO | Analyzing chunk frames = [4383744,4384256) | 20260508 02:05:34.520 | INFO | MemInfo: { | 20260508 02:05:34.520 | INFO | "gpu" : " 17 GB", | 20260508 02:05:34.520 | INFO | "huge" : "", | 20260508 02:05:34.520 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:34.520 | INFO | "malloc" : " 14 GB", | 20260508 02:05:34.520 | INFO | "pinned" : " 29 GB" | 20260508 02:05:34.520 | INFO | } | 20260508 02:05:34.520 | INFO | Perf report for the last 5.13313 seconds: | 20260508 02:05:34.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:34.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:05:34.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:05:34.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:34.520 | INFO | Uploaded 36GiB to GPU in 2.54697s (14.1344GiB/s) | 20260508 02:05:34.520 | INFO | Downloaded 34GiB to GPU in 2.49217s (13.6427GiB/s) | 20260508 02:05:39.661 | INFO | Analyzing chunk frames = [4384256,4384768) | 20260508 02:05:39.661 | INFO | MemInfo: { | 20260508 02:05:39.661 | INFO | "gpu" : " 17 GB", | 20260508 02:05:39.661 | INFO | "huge" : "", | 20260508 02:05:39.661 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:39.661 | INFO | "malloc" : " 14 GB", | 20260508 02:05:39.661 | INFO | "pinned" : " 29 GB" | 20260508 02:05:39.661 | INFO | } | 20260508 02:05:39.662 | INFO | Perf report for the last 5.14153 seconds: | 20260508 02:05:39.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:39.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:05:39.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:05:39.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:39.662 | INFO | Uploaded 36GiB to GPU in 2.55644s (14.0821GiB/s) | 20260508 02:05:39.662 | INFO | Downloaded 34GiB to GPU in 2.49792s (13.6113GiB/s) | 20260508 02:05:44.799 | INFO | Analyzing chunk frames = [4384768,4385280) | 20260508 02:05:44.799 | INFO | MemInfo: { | 20260508 02:05:44.799 | INFO | "gpu" : " 17 GB", | 20260508 02:05:44.799 | INFO | "huge" : "", | 20260508 02:05:44.799 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:44.799 | INFO | "malloc" : " 14 GB", | 20260508 02:05:44.799 | INFO | "pinned" : " 29 GB" | 20260508 02:05:44.799 | INFO | } | 20260508 02:05:44.799 | INFO | Perf report for the last 5.13795 seconds: | 20260508 02:05:44.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:44.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 02:05:44.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:05:44.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:44.800 | INFO | Uploaded 36GiB to GPU in 2.35921s (15.2593GiB/s) | 20260508 02:05:44.800 | INFO | Downloaded 34GiB to GPU in 2.54082s (13.3815GiB/s) | 20260508 02:05:49.940 | INFO | Analyzing chunk frames = [4385280,4385792) | 20260508 02:05:49.940 | INFO | MemInfo: { | 20260508 02:05:49.940 | INFO | "gpu" : " 17 GB", | 20260508 02:05:49.940 | INFO | "huge" : "", | 20260508 02:05:49.940 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:49.940 | INFO | "malloc" : " 14 GB", | 20260508 02:05:49.940 | INFO | "pinned" : " 29 GB" | 20260508 02:05:49.940 | INFO | } | 20260508 02:05:49.940 | INFO | Perf report for the last 5.14053 seconds: | 20260508 02:05:49.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:49.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:05:49.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:05:49.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:49.940 | INFO | Uploaded 36GiB to GPU in 2.43465s (14.7865GiB/s) | 20260508 02:05:49.940 | INFO | Downloaded 34GiB to GPU in 2.41698s (14.0671GiB/s) | 20260508 02:05:55.080 | INFO | Analyzing chunk frames = [4385792,4386304) | 20260508 02:05:55.080 | INFO | MemInfo: { | 20260508 02:05:55.080 | INFO | "gpu" : " 17 GB", | 20260508 02:05:55.080 | INFO | "huge" : "", | 20260508 02:05:55.080 | INFO | "huge_shared" : " 29 GB", | 20260508 02:05:55.080 | INFO | "malloc" : " 14 GB", | 20260508 02:05:55.080 | INFO | "pinned" : " 29 GB" | 20260508 02:05:55.080 | INFO | } | 20260508 02:05:55.080 | INFO | Perf report for the last 5.14022 seconds: | 20260508 02:05:55.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:05:55.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:05:55.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:05:55.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:05:55.080 | INFO | Uploaded 36GiB to GPU in 2.48763s (14.4716GiB/s) | 20260508 02:05:55.080 | INFO | Downloaded 34GiB to GPU in 2.42744s (14.0065GiB/s) | 20260508 02:06:00.217 | INFO | Analyzing chunk frames = [4386304,4386816) | 20260508 02:06:00.217 | INFO | MemInfo: { | 20260508 02:06:00.217 | INFO | "gpu" : " 17 GB", | 20260508 02:06:00.217 | INFO | "huge" : "", | 20260508 02:06:00.217 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:00.217 | INFO | "malloc" : " 14 GB", | 20260508 02:06:00.217 | INFO | "pinned" : " 29 GB" | 20260508 02:06:00.217 | INFO | } | 20260508 02:06:00.217 | INFO | Perf report for the last 5.13657 seconds: | 20260508 02:06:00.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:00.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:06:00.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:06:00.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:00.217 | INFO | Uploaded 36GiB to GPU in 2.56575s (14.031GiB/s) | 20260508 02:06:00.217 | INFO | Downloaded 34GiB to GPU in 2.44836s (13.8869GiB/s) | 20260508 02:06:00.314 | INFO | Starting a round of Baz IO | 20260508 02:06:03.775 | INFO | Populating RTMetrics report | 20260508 02:06:03.775 | INFO | RTMetrics report populated | 20260508 02:06:03.775 | INFO | Opening tmp RTMetrics json file | 20260508 02:06:03.795 | INFO | Serializing RTMetrics to json | 20260508 02:06:03.878 | INFO | Serialization done, swapping tmp file now... | 20260508 02:06:03.880 | INFO | RTMetrics done | 20260508 02:06:03.897 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.583 seconds | 20260508 02:06:03.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.64% / 0.00% / 58.36% | 20260508 02:06:03.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.768189% / 1.19209e-05% / 6.75519e-05% | 20260508 02:06:03.897 | INFO | Baz IO overall: | 20260508 02:06:03.897 | INFO | BytesWritten : 1983316713174 | 20260508 02:06:03.897 | INFO | NumEvents : 959012028278 | 20260508 02:06:05.356 | INFO | Analyzing chunk frames = [4386816,4387328) | 20260508 02:06:05.356 | INFO | MemInfo: { | 20260508 02:06:05.356 | INFO | "gpu" : " 17 GB", | 20260508 02:06:05.357 | INFO | "huge" : "", | 20260508 02:06:05.357 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:05.357 | INFO | "malloc" : " 14 GB", | 20260508 02:06:05.357 | INFO | "pinned" : " 29 GB" | 20260508 02:06:05.357 | INFO | } | 20260508 02:06:05.357 | INFO | Perf report for the last 5.13984 seconds: | 20260508 02:06:05.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:05.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:05.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:05.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:05.357 | INFO | ANALYSIS : 68.5%, 3.0, 106.7ms, 0 | 20260508 02:06:05.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:05.357 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 02:06:05.357 | INFO | PRE_HQ : 65.4%, 1.6, 56.0ms, 0 | 20260508 02:06:05.357 | INFO | BAZWRITER : 69.3%, 3.6, 126.8ms, 0 | 20260508 02:06:05.357 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260508 02:06:05.357 | INFO | Uploaded 36GiB to GPU in 2.58036s (13.9515GiB/s) | 20260508 02:06:05.357 | INFO | Downloaded 38GiB to GPU in 2.67161s (14.2236GiB/s) | 20260508 02:06:10.498 | INFO | Analyzing chunk frames = [4387328,4387840) | 20260508 02:06:10.498 | INFO | MemInfo: { | 20260508 02:06:10.498 | INFO | "gpu" : " 17 GB", | 20260508 02:06:10.498 | INFO | "huge" : "", | 20260508 02:06:10.498 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:10.498 | INFO | "malloc" : " 14 GB", | 20260508 02:06:10.498 | INFO | "pinned" : " 29 GB" | 20260508 02:06:10.498 | INFO | } | 20260508 02:06:10.498 | INFO | Perf report for the last 5.14162 seconds: | 20260508 02:06:10.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:10.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.498 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 02:06:10.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.498 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:06:10.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:10.499 | INFO | Uploaded 36GiB to GPU in 2.55558s (14.0868GiB/s) | 20260508 02:06:10.499 | INFO | Downloaded 34GiB to GPU in 2.41511s (14.078GiB/s) | 20260508 02:06:15.633 | INFO | Analyzing chunk frames = [4387840,4388352) | 20260508 02:06:15.633 | INFO | MemInfo: { | 20260508 02:06:15.633 | INFO | "gpu" : " 17 GB", | 20260508 02:06:15.633 | INFO | "huge" : "", | 20260508 02:06:15.633 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:15.633 | INFO | "malloc" : " 14 GB", | 20260508 02:06:15.633 | INFO | "pinned" : " 29 GB" | 20260508 02:06:15.633 | INFO | } | 20260508 02:06:15.633 | INFO | Perf report for the last 5.13457 seconds: | 20260508 02:06:15.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:15.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:06:15.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:06:15.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:15.633 | INFO | Uploaded 36GiB to GPU in 2.56752s (14.0213GiB/s) | 20260508 02:06:15.634 | INFO | Downloaded 34GiB to GPU in 2.4843s (13.6859GiB/s) | 20260508 02:06:20.772 | INFO | Analyzing chunk frames = [4388352,4388864) | 20260508 02:06:20.772 | INFO | MemInfo: { | 20260508 02:06:20.772 | INFO | "gpu" : " 17 GB", | 20260508 02:06:20.772 | INFO | "huge" : "", | 20260508 02:06:20.772 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:20.772 | INFO | "malloc" : " 14 GB", | 20260508 02:06:20.772 | INFO | "pinned" : " 29 GB" | 20260508 02:06:20.772 | INFO | } | 20260508 02:06:20.772 | INFO | Perf report for the last 5.13878 seconds: | 20260508 02:06:20.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:20.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:06:20.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:06:20.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:20.772 | INFO | Uploaded 36GiB to GPU in 2.56446s (14.0381GiB/s) | 20260508 02:06:20.772 | INFO | Downloaded 34GiB to GPU in 2.4831s (13.6926GiB/s) | 20260508 02:06:25.914 | INFO | Analyzing chunk frames = [4388864,4389376) | 20260508 02:06:25.914 | INFO | MemInfo: { | 20260508 02:06:25.914 | INFO | "gpu" : " 17 GB", | 20260508 02:06:25.915 | INFO | "huge" : "", | 20260508 02:06:25.915 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:25.915 | INFO | "malloc" : " 14 GB", | 20260508 02:06:25.915 | INFO | "pinned" : " 29 GB" | 20260508 02:06:25.915 | INFO | } | 20260508 02:06:25.915 | INFO | Perf report for the last 5.14298 seconds: | 20260508 02:06:25.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:25.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 02:06:25.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:06:25.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:25.915 | INFO | Uploaded 36GiB to GPU in 2.36s (15.2542GiB/s) | 20260508 02:06:25.915 | INFO | Downloaded 34GiB to GPU in 2.49555s (13.6242GiB/s) | 20260508 02:06:31.057 | INFO | Analyzing chunk frames = [4389376,4389888) | 20260508 02:06:31.057 | INFO | MemInfo: { | 20260508 02:06:31.057 | INFO | "gpu" : " 17 GB", | 20260508 02:06:31.057 | INFO | "huge" : "", | 20260508 02:06:31.057 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:31.057 | INFO | "malloc" : " 14 GB", | 20260508 02:06:31.057 | INFO | "pinned" : " 29 GB" | 20260508 02:06:31.057 | INFO | } | 20260508 02:06:31.057 | INFO | Perf report for the last 5.14218 seconds: | 20260508 02:06:31.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:31.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 02:06:31.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260508 02:06:31.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:31.057 | INFO | Uploaded 36GiB to GPU in 2.41274s (14.9208GiB/s) | 20260508 02:06:31.057 | INFO | Downloaded 34GiB to GPU in 2.42042s (14.0472GiB/s) | 20260508 02:06:36.197 | INFO | Analyzing chunk frames = [4389888,4390400) | 20260508 02:06:36.197 | INFO | MemInfo: { | 20260508 02:06:36.197 | INFO | "gpu" : " 17 GB", | 20260508 02:06:36.197 | INFO | "huge" : "", | 20260508 02:06:36.197 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:36.197 | INFO | "malloc" : " 14 GB", | 20260508 02:06:36.197 | INFO | "pinned" : " 29 GB" | 20260508 02:06:36.197 | INFO | } | 20260508 02:06:36.197 | INFO | Perf report for the last 5.14054 seconds: | 20260508 02:06:36.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:36.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.198 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:06:36.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.198 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:06:36.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:36.198 | INFO | Uploaded 36GiB to GPU in 2.43655s (14.775GiB/s) | 20260508 02:06:36.198 | INFO | Downloaded 34GiB to GPU in 2.41024s (14.1065GiB/s) | 20260508 02:06:41.330 | INFO | Analyzing chunk frames = [4390400,4390912) | 20260508 02:06:41.330 | INFO | MemInfo: { | 20260508 02:06:41.330 | INFO | "gpu" : " 17 GB", | 20260508 02:06:41.330 | INFO | "huge" : "", | 20260508 02:06:41.330 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:41.330 | INFO | "malloc" : " 14 GB", | 20260508 02:06:41.330 | INFO | "pinned" : " 29 GB" | 20260508 02:06:41.330 | INFO | } | 20260508 02:06:41.331 | INFO | Perf report for the last 5.13304 seconds: | 20260508 02:06:41.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:41.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 02:06:41.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:06:41.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:41.331 | INFO | Uploaded 36GiB to GPU in 2.46457s (14.607GiB/s) | 20260508 02:06:41.331 | INFO | Downloaded 34GiB to GPU in 2.428s (14.0033GiB/s) | 20260508 02:06:41.331 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:06:41.331 | INFO | Total mem usage: 16.7 GiB | 20260508 02:06:41.331 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:06:41.331 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.9%) | 20260508 02:06:41.331 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:06:41.331 | INFO | High Water Breakdown: | 20260508 02:06:41.331 | INFO | 60% : 8.86 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:06:41.331 | INFO | 39.9% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:06:41.331 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:06:41.331 | INFO | ------------------------------------------ | 20260508 02:06:41.331 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:06:41.331 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:06:41.331 | INFO | Total mem usage: 26.5 GiB | 20260508 02:06:41.331 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:06:41.331 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 02:06:41.331 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:06:41.331 | INFO | High Water Breakdown: | 20260508 02:06:41.331 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:06:41.331 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:06:41.331 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:06:41.331 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:06:41.331 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:06:41.331 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:06:41.332 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:06:41.332 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:06:41.332 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:06:41.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:06:41.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:06:41.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:06:41.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:06:41.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:06:41.332 | INFO | ------------------------------------------ | 20260508 02:06:41.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:06:41.332 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:06:41.332 | INFO | Total mem usage: 27.8 GiB | 20260508 02:06:41.332 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:06:41.332 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:06:41.332 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:06:41.332 | INFO | High Water Breakdown: | 20260508 02:06:41.332 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:06:41.332 | INFO | ------------------------------------------ | 20260508 02:06:41.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:06:41.332 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:06:41.332 | INFO | Total mem usage: 15.7 GiB | 20260508 02:06:41.332 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:06:41.332 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:06:41.332 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:06:41.332 | INFO | High Water Breakdown: | 20260508 02:06:41.332 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:06:41.332 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:06:41.332 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:06:41.332 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:06:41.332 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:06:41.332 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:06:41.332 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:06:41.332 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:06:41.332 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:06:41.332 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:06:41.332 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:06:41.332 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:06:41.332 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:06:41.332 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:06:41.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:06:41.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:06:41.332 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:06:41.332 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:06:41.332 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:06:41.332 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:06:41.332 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:06:41.332 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:06:41.332 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:06:41.332 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:06:41.332 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:06:41.332 | INFO | ------------------------------------------ | 20260508 02:06:41.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:06:41.332 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:06:41.332 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:06:41.332 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:06:41.332 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.41e+01 / 1.26e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.01e-01, 1.21e-02 / 5.70e-03 | 20260508 02:06:41.332 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.46e+00, 5.91e-01 / 2.12e-01 | 20260508 02:06:41.332 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.06e-01 / 1.17e-01 | 20260508 02:06:41.332 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.59e+01, 3.97e+00 / 1.73e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.62e+00 / 1.51e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.42e+00 / 2.67e+00, 4.23e-01 / 2.57e-01 | 20260508 02:06:41.332 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.25e+00, 2.70e-01 / 1.71e-01 | 20260508 02:06:41.332 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.31e-01 / 3.66e-01, 5.29e-02 / 2.57e-02 | 20260508 02:06:41.332 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:06:41.332 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:06:41.332 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:06:41.332 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 24.48%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 02:06:41.332 | INFO | PerfReport: 2.68%, LockDownload -- 2.85e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 02:06:41.333 | INFO | PerfReport: 15.45%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:06:41.333 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260508 02:06:41.428 | INFO | Starting a round of Baz IO | 20260508 02:06:44.884 | INFO | Populating RTMetrics report | 20260508 02:06:44.884 | INFO | RTMetrics report populated | 20260508 02:06:44.884 | INFO | Opening tmp RTMetrics json file | 20260508 02:06:44.902 | INFO | Serializing RTMetrics to json | 20260508 02:06:44.904 | INFO | Serialization done, swapping tmp file now... | 20260508 02:06:44.904 | INFO | RTMetrics done | 20260508 02:06:44.979 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.55 seconds | 20260508 02:06:44.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.63% / 0.00% / 58.36% | 20260508 02:06:44.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.767346% / 7.94729e-06% / 6.35783e-05% | 20260508 02:06:44.979 | INFO | Baz IO overall: | 20260508 02:06:44.979 | INFO | BytesWritten : 1985084716408 | 20260508 02:06:44.979 | INFO | NumEvents : 959860060517 | 20260508 02:06:46.473 | INFO | Analyzing chunk frames = [4390912,4391424) | 20260508 02:06:46.473 | INFO | MemInfo: { | 20260508 02:06:46.473 | INFO | "gpu" : " 17 GB", | 20260508 02:06:46.473 | INFO | "huge" : "", | 20260508 02:06:46.473 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:46.473 | INFO | "malloc" : " 14 GB", | 20260508 02:06:46.473 | INFO | "pinned" : " 29 GB" | 20260508 02:06:46.473 | INFO | } | 20260508 02:06:46.473 | INFO | Perf report for the last 5.14242 seconds: | 20260508 02:06:46.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:46.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:46.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:46.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:46.473 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 02:06:46.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:46.473 | INFO | RT_METRICS : 40.1%, 1.0, 21.1ms, 0 | 20260508 02:06:46.473 | INFO | PRE_HQ : 65.7%, 1.6, 56.1ms, 0 | 20260508 02:06:46.473 | INFO | BAZWRITER : 68.7%, 3.9, 138.1ms, 0 | 20260508 02:06:46.473 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:06:46.473 | INFO | Uploaded 36GiB to GPU in 2.57285s (13.9923GiB/s) | 20260508 02:06:46.473 | INFO | Downloaded 38GiB to GPU in 2.69469s (14.1018GiB/s) | 20260508 02:06:51.613 | INFO | Analyzing chunk frames = [4391424,4391936) | 20260508 02:06:51.614 | INFO | MemInfo: { | 20260508 02:06:51.614 | INFO | "gpu" : " 17 GB", | 20260508 02:06:51.614 | INFO | "huge" : "", | 20260508 02:06:51.614 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:51.614 | INFO | "malloc" : " 14 GB", | 20260508 02:06:51.614 | INFO | "pinned" : " 29 GB" | 20260508 02:06:51.614 | INFO | } | 20260508 02:06:51.614 | INFO | Perf report for the last 5.14081 seconds: | 20260508 02:06:51.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:51.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 02:06:51.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:06:51.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:51.614 | INFO | Uploaded 36GiB to GPU in 2.54897s (14.1234GiB/s) | 20260508 02:06:51.614 | INFO | Downloaded 34GiB to GPU in 2.42844s (14.0007GiB/s) | 20260508 02:06:56.753 | INFO | Analyzing chunk frames = [4391936,4392448) | 20260508 02:06:56.753 | INFO | MemInfo: { | 20260508 02:06:56.753 | INFO | "gpu" : " 17 GB", | 20260508 02:06:56.753 | INFO | "huge" : "", | 20260508 02:06:56.753 | INFO | "huge_shared" : " 29 GB", | 20260508 02:06:56.753 | INFO | "malloc" : " 14 GB", | 20260508 02:06:56.753 | INFO | "pinned" : " 29 GB" | 20260508 02:06:56.753 | INFO | } | 20260508 02:06:56.753 | INFO | Perf report for the last 5.13965 seconds: | 20260508 02:06:56.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:06:56.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.754 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:06:56.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.754 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:06:56.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:06:56.754 | INFO | Uploaded 36GiB to GPU in 2.51757s (14.2995GiB/s) | 20260508 02:06:56.754 | INFO | Downloaded 34GiB to GPU in 2.49849s (13.6082GiB/s) | 20260508 02:07:01.888 | INFO | Analyzing chunk frames = [4392448,4392960) | 20260508 02:07:01.888 | INFO | MemInfo: { | 20260508 02:07:01.888 | INFO | "gpu" : " 17 GB", | 20260508 02:07:01.888 | INFO | "huge" : "", | 20260508 02:07:01.888 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:01.888 | INFO | "malloc" : " 14 GB", | 20260508 02:07:01.888 | INFO | "pinned" : " 29 GB" | 20260508 02:07:01.888 | INFO | } | 20260508 02:07:01.888 | INFO | Perf report for the last 5.13498 seconds: | 20260508 02:07:01.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:01.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.888 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:07:01.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.889 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:07:01.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:01.889 | INFO | Uploaded 36GiB to GPU in 2.53504s (14.201GiB/s) | 20260508 02:07:01.889 | INFO | Downloaded 34GiB to GPU in 2.50216s (13.5883GiB/s) | 20260508 02:07:07.025 | INFO | Analyzing chunk frames = [4392960,4393472) | 20260508 02:07:07.025 | INFO | MemInfo: { | 20260508 02:07:07.025 | INFO | "gpu" : " 17 GB", | 20260508 02:07:07.025 | INFO | "huge" : "", | 20260508 02:07:07.025 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:07.025 | INFO | "malloc" : " 14 GB", | 20260508 02:07:07.025 | INFO | "pinned" : " 29 GB" | 20260508 02:07:07.025 | INFO | } | 20260508 02:07:07.025 | INFO | Perf report for the last 5.13699 seconds: | 20260508 02:07:07.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:07.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.025 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 02:07:07.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.026 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 02:07:07.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:07.026 | INFO | Uploaded 36GiB to GPU in 2.4181s (14.8877GiB/s) | 20260508 02:07:07.026 | INFO | Downloaded 34GiB to GPU in 2.53546s (13.4098GiB/s) | 20260508 02:07:12.166 | INFO | Analyzing chunk frames = [4393472,4393984) | 20260508 02:07:12.166 | INFO | MemInfo: { | 20260508 02:07:12.166 | INFO | "gpu" : " 17 GB", | 20260508 02:07:12.166 | INFO | "huge" : "", | 20260508 02:07:12.166 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:12.166 | INFO | "malloc" : " 14 GB", | 20260508 02:07:12.166 | INFO | "pinned" : " 29 GB" | 20260508 02:07:12.166 | INFO | } | 20260508 02:07:12.166 | INFO | Perf report for the last 5.14045 seconds: | 20260508 02:07:12.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:12.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 02:07:12.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:07:12.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:12.166 | INFO | Uploaded 36GiB to GPU in 2.41626s (14.899GiB/s) | 20260508 02:07:12.166 | INFO | Downloaded 34GiB to GPU in 2.41932s (14.0536GiB/s) | 20260508 02:07:17.308 | INFO | Analyzing chunk frames = [4393984,4394496) | 20260508 02:07:17.308 | INFO | MemInfo: { | 20260508 02:07:17.308 | INFO | "gpu" : " 17 GB", | 20260508 02:07:17.308 | INFO | "huge" : "", | 20260508 02:07:17.308 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:17.308 | INFO | "malloc" : " 14 GB", | 20260508 02:07:17.308 | INFO | "pinned" : " 29 GB" | 20260508 02:07:17.308 | INFO | } | 20260508 02:07:17.308 | INFO | Perf report for the last 5.14225 seconds: | 20260508 02:07:17.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:17.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.308 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:07:17.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.308 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:07:17.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:17.309 | INFO | Uploaded 36GiB to GPU in 2.41664s (14.8967GiB/s) | 20260508 02:07:17.309 | INFO | Downloaded 34GiB to GPU in 2.41688s (14.0677GiB/s) | 20260508 02:07:22.450 | INFO | Analyzing chunk frames = [4394496,4395008) | 20260508 02:07:22.450 | INFO | MemInfo: { | 20260508 02:07:22.450 | INFO | "gpu" : " 17 GB", | 20260508 02:07:22.450 | INFO | "huge" : "", | 20260508 02:07:22.450 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:22.450 | INFO | "malloc" : " 14 GB", | 20260508 02:07:22.450 | INFO | "pinned" : " 29 GB" | 20260508 02:07:22.450 | INFO | } | 20260508 02:07:22.450 | INFO | Perf report for the last 5.14188 seconds: | 20260508 02:07:22.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:22.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 02:07:22.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 02:07:22.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:22.450 | INFO | Uploaded 36GiB to GPU in 2.54685s (14.1351GiB/s) | 20260508 02:07:22.450 | INFO | Downloaded 34GiB to GPU in 2.4575s (13.8352GiB/s) | 20260508 02:07:22.547 | INFO | Starting a round of Baz IO | 20260508 02:07:25.992 | INFO | Populating RTMetrics report | 20260508 02:07:25.992 | INFO | RTMetrics report populated | 20260508 02:07:25.992 | INFO | Opening tmp RTMetrics json file | 20260508 02:07:26.004 | INFO | Serializing RTMetrics to json | 20260508 02:07:26.016 | INFO | Serialization done, swapping tmp file now... | 20260508 02:07:26.017 | INFO | RTMetrics done | 20260508 02:07:26.099 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.551 seconds | 20260508 02:07:26.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.63% / 0.00% / 58.37% | 20260508 02:07:26.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.766555% / 0% / 5.5631e-05% | 20260508 02:07:26.099 | INFO | Baz IO overall: | 20260508 02:07:26.099 | INFO | BytesWritten : 1986853285991 | 20260508 02:07:26.099 | INFO | NumEvents : 960708748702 | 20260508 02:07:27.582 | INFO | Analyzing chunk frames = [4395008,4395520) | 20260508 02:07:27.582 | INFO | MemInfo: { | 20260508 02:07:27.582 | INFO | "gpu" : " 17 GB", | 20260508 02:07:27.582 | INFO | "huge" : "", | 20260508 02:07:27.582 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:27.582 | INFO | "malloc" : " 14 GB", | 20260508 02:07:27.582 | INFO | "pinned" : " 29 GB" | 20260508 02:07:27.582 | INFO | } | 20260508 02:07:27.582 | INFO | Perf report for the last 5.13225 seconds: | 20260508 02:07:27.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:27.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:27.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:27.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:27.582 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260508 02:07:27.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:27.582 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 02:07:27.582 | INFO | PRE_HQ : 66.1%, 1.6, 56.2ms, 0 | 20260508 02:07:27.582 | INFO | BAZWRITER : 68.8%, 3.8, 134.3ms, 0 | 20260508 02:07:27.582 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 02:07:27.583 | INFO | Uploaded 36GiB to GPU in 2.59477s (13.8741GiB/s) | 20260508 02:07:27.583 | INFO | Downloaded 38GiB to GPU in 2.69925s (14.078GiB/s) | 20260508 02:07:32.727 | INFO | Analyzing chunk frames = [4395520,4396032) | 20260508 02:07:32.727 | INFO | MemInfo: { | 20260508 02:07:32.727 | INFO | "gpu" : " 17 GB", | 20260508 02:07:32.727 | INFO | "huge" : "", | 20260508 02:07:32.727 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:32.727 | INFO | "malloc" : " 14 GB", | 20260508 02:07:32.727 | INFO | "pinned" : " 29 GB" | 20260508 02:07:32.727 | INFO | } | 20260508 02:07:32.728 | INFO | Perf report for the last 5.1452 seconds: | 20260508 02:07:32.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:32.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 02:07:32.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:07:32.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:32.728 | INFO | Uploaded 36GiB to GPU in 2.55045s (14.1152GiB/s) | 20260508 02:07:32.728 | INFO | Downloaded 34GiB to GPU in 2.42789s (14.0039GiB/s) | 20260508 02:07:37.864 | INFO | Analyzing chunk frames = [4396032,4396544) | 20260508 02:07:37.864 | INFO | MemInfo: { | 20260508 02:07:37.864 | INFO | "gpu" : " 17 GB", | 20260508 02:07:37.864 | INFO | "huge" : "", | 20260508 02:07:37.864 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:37.864 | INFO | "malloc" : " 14 GB", | 20260508 02:07:37.864 | INFO | "pinned" : " 29 GB" | 20260508 02:07:37.864 | INFO | } | 20260508 02:07:37.864 | INFO | Perf report for the last 5.13678 seconds: | 20260508 02:07:37.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:37.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.864 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:07:37.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.864 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:07:37.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:37.865 | INFO | Uploaded 36GiB to GPU in 2.52487s (14.2582GiB/s) | 20260508 02:07:37.865 | INFO | Downloaded 34GiB to GPU in 2.50399s (13.5783GiB/s) | 20260508 02:07:43.005 | INFO | Analyzing chunk frames = [4396544,4397056) | 20260508 02:07:43.005 | INFO | MemInfo: { | 20260508 02:07:43.005 | INFO | "gpu" : " 17 GB", | 20260508 02:07:43.005 | INFO | "huge" : "", | 20260508 02:07:43.006 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:43.006 | INFO | "malloc" : " 14 GB", | 20260508 02:07:43.006 | INFO | "pinned" : " 29 GB" | 20260508 02:07:43.006 | INFO | } | 20260508 02:07:43.006 | INFO | Perf report for the last 5.14132 seconds: | 20260508 02:07:43.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:43.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 02:07:43.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:07:43.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:43.006 | INFO | Uploaded 36GiB to GPU in 2.51925s (14.29GiB/s) | 20260508 02:07:43.006 | INFO | Downloaded 34GiB to GPU in 2.48921s (13.6589GiB/s) | 20260508 02:07:48.143 | INFO | Analyzing chunk frames = [4397056,4397568) | 20260508 02:07:48.143 | INFO | MemInfo: { | 20260508 02:07:48.143 | INFO | "gpu" : " 17 GB", | 20260508 02:07:48.143 | INFO | "huge" : "", | 20260508 02:07:48.143 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:48.143 | INFO | "malloc" : " 14 GB", | 20260508 02:07:48.143 | INFO | "pinned" : " 29 GB" | 20260508 02:07:48.143 | INFO | } | 20260508 02:07:48.143 | INFO | Perf report for the last 5.13765 seconds: | 20260508 02:07:48.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:48.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.143 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 02:07:48.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.143 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 02:07:48.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:48.144 | INFO | Uploaded 36GiB to GPU in 2.43771s (14.768GiB/s) | 20260508 02:07:48.144 | INFO | Downloaded 34GiB to GPU in 2.54694s (13.3494GiB/s) | 20260508 02:07:53.285 | INFO | Analyzing chunk frames = [4397568,4398080) | 20260508 02:07:53.285 | INFO | MemInfo: { | 20260508 02:07:53.285 | INFO | "gpu" : " 17 GB", | 20260508 02:07:53.285 | INFO | "huge" : "", | 20260508 02:07:53.285 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:53.285 | INFO | "malloc" : " 14 GB", | 20260508 02:07:53.285 | INFO | "pinned" : " 29 GB" | 20260508 02:07:53.285 | INFO | } | 20260508 02:07:53.286 | INFO | Perf report for the last 5.14222 seconds: | 20260508 02:07:53.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:53.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:07:53.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 02:07:53.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:53.286 | INFO | Uploaded 36GiB to GPU in 2.40776s (14.9517GiB/s) | 20260508 02:07:53.286 | INFO | Downloaded 34GiB to GPU in 2.42343s (14.0297GiB/s) | 20260508 02:07:58.422 | INFO | Analyzing chunk frames = [4398080,4398592) | 20260508 02:07:58.422 | INFO | MemInfo: { | 20260508 02:07:58.422 | INFO | "gpu" : " 17 GB", | 20260508 02:07:58.422 | INFO | "huge" : "", | 20260508 02:07:58.422 | INFO | "huge_shared" : " 29 GB", | 20260508 02:07:58.422 | INFO | "malloc" : " 14 GB", | 20260508 02:07:58.422 | INFO | "pinned" : " 29 GB" | 20260508 02:07:58.422 | INFO | } | 20260508 02:07:58.422 | INFO | Perf report for the last 5.13638 seconds: | 20260508 02:07:58.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:07:58.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:07:58.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:07:58.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:07:58.422 | INFO | Uploaded 36GiB to GPU in 2.44593s (14.7183GiB/s) | 20260508 02:07:58.422 | INFO | Downloaded 34GiB to GPU in 2.41862s (14.0576GiB/s) | 20260508 02:08:03.564 | INFO | Analyzing chunk frames = [4398592,4399104) | 20260508 02:08:03.564 | INFO | MemInfo: { | 20260508 02:08:03.564 | INFO | "gpu" : " 17 GB", | 20260508 02:08:03.564 | INFO | "huge" : "", | 20260508 02:08:03.564 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:03.564 | INFO | "malloc" : " 14 GB", | 20260508 02:08:03.564 | INFO | "pinned" : " 29 GB" | 20260508 02:08:03.564 | INFO | } | 20260508 02:08:03.564 | INFO | Perf report for the last 5.14239 seconds: | 20260508 02:08:03.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:03.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.564 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 02:08:03.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.564 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:08:03.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:03.565 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:08:03.565 | INFO | Total mem usage: 16.7 GiB | 20260508 02:08:03.565 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:08:03.565 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 02:08:03.565 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:08:03.565 | INFO | High Water Breakdown: | 20260508 02:08:03.565 | INFO | 60% : 8.9 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:08:03.565 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:08:03.565 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:08:03.565 | INFO | ------------------------------------------ | 20260508 02:08:03.565 | INFO | Uploaded 36GiB to GPU in 2.55452s (14.0927GiB/s) | 20260508 02:08:03.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:08:03.565 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:08:03.565 | INFO | Downloaded 34GiB to GPU in 2.45214s (13.8654GiB/s) | 20260508 02:08:03.565 | INFO | Total mem usage: 26.5 GiB | 20260508 02:08:03.565 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:08:03.565 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:08:03.565 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:08:03.565 | INFO | High Water Breakdown: | 20260508 02:08:03.565 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:08:03.565 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:08:03.565 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:08:03.565 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:08:03.565 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:08:03.565 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:08:03.565 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:08:03.565 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:08:03.565 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:08:03.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:08:03.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:08:03.565 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:08:03.565 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:08:03.565 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:08:03.565 | INFO | ------------------------------------------ | 20260508 02:08:03.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:08:03.565 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:08:03.565 | INFO | Total mem usage: 27.8 GiB | 20260508 02:08:03.565 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:08:03.565 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:08:03.565 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:08:03.565 | INFO | High Water Breakdown: | 20260508 02:08:03.565 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:08:03.565 | INFO | ------------------------------------------ | 20260508 02:08:03.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:08:03.565 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:08:03.565 | INFO | Total mem usage: 15.7 GiB | 20260508 02:08:03.565 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:08:03.565 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:08:03.565 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:08:03.565 | INFO | High Water Breakdown: | 20260508 02:08:03.565 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:08:03.565 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:08:03.565 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:08:03.565 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:08:03.565 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:08:03.566 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:08:03.566 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:08:03.566 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:08:03.566 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:08:03.566 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:08:03.566 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:08:03.566 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:08:03.566 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:08:03.566 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:08:03.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:08:03.566 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:08:03.566 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:08:03.566 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:08:03.566 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:08:03.566 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:08:03.566 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:08:03.566 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:08:03.566 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:08:03.566 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:08:03.566 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:08:03.566 | INFO | ------------------------------------------ | 20260508 02:08:03.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:08:03.566 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:08:03.566 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:08:03.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:08:03.566 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.43e+01 / 1.54e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.12e-02 / 6.07e-03 | 20260508 02:08:03.566 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.97e-01 / 1.75e-01 | 20260508 02:08:03.566 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.13e-01 / 9.93e-02 | 20260508 02:08:03.566 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.61e+01, 4.06e+00 / 1.93e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.64e+00 / 1.39e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.71e+00, 4.28e-01 / 2.08e-01 | 20260508 02:08:03.566 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.26e+00, 2.67e-01 / 1.49e-01 | 20260508 02:08:03.566 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.30e-01 / 3.65e-01, 5.20e-02 / 2.22e-02 | 20260508 02:08:03.566 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:08:03.566 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:08:03.566 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:08:03.566 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 24.37%, Download -- 2.60e+01 / 0.00e+00, 8.43e+00 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 2.09%, LockUpload -- 2.23e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 2.86%, LockDownload -- 3.04e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 15.55%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 02:08:03.566 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260508 02:08:03.661 | INFO | Starting a round of Baz IO | 20260508 02:08:07.113 | INFO | Populating RTMetrics report | 20260508 02:08:07.114 | INFO | RTMetrics report populated | 20260508 02:08:07.114 | INFO | Opening tmp RTMetrics json file | 20260508 02:08:07.147 | INFO | Serializing RTMetrics to json | 20260508 02:08:07.184 | INFO | Serialization done, swapping tmp file now... | 20260508 02:08:07.188 | INFO | RTMetrics done | 20260508 02:08:07.213 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.551 seconds | 20260508 02:08:07.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.63% / 0.00% / 58.37% | 20260508 02:08:07.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.764795% / 3.97364e-06% / 4.76837e-05% | 20260508 02:08:07.213 | INFO | Baz IO overall: | 20260508 02:08:07.213 | INFO | BytesWritten : 1988621473784 | 20260508 02:08:07.213 | INFO | NumEvents : 961557276127 | 20260508 02:08:08.703 | INFO | Analyzing chunk frames = [4399104,4399616) | 20260508 02:08:08.703 | INFO | MemInfo: { | 20260508 02:08:08.703 | INFO | "gpu" : " 17 GB", | 20260508 02:08:08.703 | INFO | "huge" : "", | 20260508 02:08:08.703 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:08.703 | INFO | "malloc" : " 14 GB", | 20260508 02:08:08.703 | INFO | "pinned" : " 29 GB" | 20260508 02:08:08.703 | INFO | } | 20260508 02:08:08.703 | INFO | Perf report for the last 5.13842 seconds: | 20260508 02:08:08.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:08.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:08.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:08.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:08.703 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 02:08:08.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:08.703 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260508 02:08:08.703 | INFO | PRE_HQ : 65.7%, 1.6, 56.0ms, 0 | 20260508 02:08:08.703 | INFO | BAZWRITER : 68.7%, 3.7, 130.5ms, 0 | 20260508 02:08:08.703 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 02:08:08.704 | INFO | Uploaded 36GiB to GPU in 2.56733s (14.0224GiB/s) | 20260508 02:08:08.704 | INFO | Downloaded 38GiB to GPU in 2.70303s (14.0583GiB/s) | 20260508 02:08:13.839 | INFO | Analyzing chunk frames = [4399616,4400128) | 20260508 02:08:13.839 | INFO | MemInfo: { | 20260508 02:08:13.839 | INFO | "gpu" : " 17 GB", | 20260508 02:08:13.839 | INFO | "huge" : "", | 20260508 02:08:13.839 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:13.839 | INFO | "malloc" : " 14 GB", | 20260508 02:08:13.839 | INFO | "pinned" : " 29 GB" | 20260508 02:08:13.839 | INFO | } | 20260508 02:08:13.839 | INFO | Perf report for the last 5.13648 seconds: | 20260508 02:08:13.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:13.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 02:08:13.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260508 02:08:13.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:13.839 | INFO | Uploaded 36GiB to GPU in 2.55118s (14.1111GiB/s) | 20260508 02:08:13.839 | INFO | Downloaded 34GiB to GPU in 2.42148s (14.041GiB/s) | 20260508 02:08:18.979 | INFO | Analyzing chunk frames = [4400128,4400640) | 20260508 02:08:18.979 | INFO | MemInfo: { | 20260508 02:08:18.979 | INFO | "gpu" : " 17 GB", | 20260508 02:08:18.979 | INFO | "huge" : "", | 20260508 02:08:18.979 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:18.979 | INFO | "malloc" : " 14 GB", | 20260508 02:08:18.979 | INFO | "pinned" : " 29 GB" | 20260508 02:08:18.979 | INFO | } | 20260508 02:08:18.979 | INFO | Perf report for the last 5.13985 seconds: | 20260508 02:08:18.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:18.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.979 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:08:18.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.979 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:08:18.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:18.980 | INFO | Uploaded 36GiB to GPU in 2.51813s (14.2963GiB/s) | 20260508 02:08:18.980 | INFO | Downloaded 34GiB to GPU in 2.48414s (13.6868GiB/s) | 20260508 02:08:24.118 | INFO | Analyzing chunk frames = [4400640,4401152) | 20260508 02:08:24.118 | INFO | MemInfo: { | 20260508 02:08:24.118 | INFO | "gpu" : " 17 GB", | 20260508 02:08:24.119 | INFO | "huge" : "", | 20260508 02:08:24.119 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:24.119 | INFO | "malloc" : " 14 GB", | 20260508 02:08:24.119 | INFO | "pinned" : " 29 GB" | 20260508 02:08:24.119 | INFO | } | 20260508 02:08:24.119 | INFO | Perf report for the last 5.13958 seconds: | 20260508 02:08:24.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:24.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:08:24.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:08:24.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:24.119 | INFO | Uploaded 36GiB to GPU in 2.51704s (14.3025GiB/s) | 20260508 02:08:24.119 | INFO | Downloaded 34GiB to GPU in 2.47825s (13.7194GiB/s) | 20260508 02:08:29.260 | INFO | Analyzing chunk frames = [4401152,4401664) | 20260508 02:08:29.260 | INFO | MemInfo: { | 20260508 02:08:29.260 | INFO | "gpu" : " 17 GB", | 20260508 02:08:29.260 | INFO | "huge" : "", | 20260508 02:08:29.260 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:29.260 | INFO | "malloc" : " 14 GB", | 20260508 02:08:29.260 | INFO | "pinned" : " 29 GB" | 20260508 02:08:29.260 | INFO | } | 20260508 02:08:29.260 | INFO | Perf report for the last 5.14105 seconds: | 20260508 02:08:29.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:29.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 02:08:29.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:08:29.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:29.260 | INFO | Uploaded 36GiB to GPU in 2.31168s (15.5731GiB/s) | 20260508 02:08:29.260 | INFO | Downloaded 34GiB to GPU in 2.49457s (13.6296GiB/s) | 20260508 02:08:34.398 | INFO | Analyzing chunk frames = [4401664,4402176) | 20260508 02:08:34.398 | INFO | MemInfo: { | 20260508 02:08:34.398 | INFO | "gpu" : " 17 GB", | 20260508 02:08:34.398 | INFO | "huge" : "", | 20260508 02:08:34.398 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:34.398 | INFO | "malloc" : " 14 GB", | 20260508 02:08:34.398 | INFO | "pinned" : " 29 GB" | 20260508 02:08:34.398 | INFO | } | 20260508 02:08:34.398 | INFO | Perf report for the last 5.13787 seconds: | 20260508 02:08:34.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:34.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:08:34.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:08:34.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:34.398 | INFO | Uploaded 36GiB to GPU in 2.41556s (14.9034GiB/s) | 20260508 02:08:34.398 | INFO | Downloaded 34GiB to GPU in 2.38949s (14.229GiB/s) | 20260508 02:08:39.534 | INFO | Analyzing chunk frames = [4402176,4402688) | 20260508 02:08:39.534 | INFO | MemInfo: { | 20260508 02:08:39.534 | INFO | "gpu" : " 17 GB", | 20260508 02:08:39.534 | INFO | "huge" : "", | 20260508 02:08:39.534 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:39.534 | INFO | "malloc" : " 14 GB", | 20260508 02:08:39.534 | INFO | "pinned" : " 29 GB" | 20260508 02:08:39.534 | INFO | } | 20260508 02:08:39.534 | INFO | Perf report for the last 5.1366 seconds: | 20260508 02:08:39.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:39.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.534 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:08:39.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.534 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:08:39.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:39.535 | INFO | Uploaded 36GiB to GPU in 2.45117s (14.6869GiB/s) | 20260508 02:08:39.535 | INFO | Downloaded 34GiB to GPU in 2.40498s (14.1373GiB/s) | 20260508 02:08:44.679 | INFO | Analyzing chunk frames = [4402688,4403200) | 20260508 02:08:44.679 | INFO | MemInfo: { | 20260508 02:08:44.679 | INFO | "gpu" : " 17 GB", | 20260508 02:08:44.679 | INFO | "huge" : "", | 20260508 02:08:44.679 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:44.679 | INFO | "malloc" : " 14 GB", | 20260508 02:08:44.679 | INFO | "pinned" : " 29 GB" | 20260508 02:08:44.679 | INFO | } | 20260508 02:08:44.679 | INFO | Perf report for the last 5.14492 seconds: | 20260508 02:08:44.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:44.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.679 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 02:08:44.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.679 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:08:44.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:44.680 | INFO | Uploaded 36GiB to GPU in 2.54321s (14.1554GiB/s) | 20260508 02:08:44.680 | INFO | Downloaded 34GiB to GPU in 2.44232s (13.9212GiB/s) | 20260508 02:08:44.777 | INFO | Starting a round of Baz IO | 20260508 02:08:48.230 | INFO | Populating RTMetrics report | 20260508 02:08:48.230 | INFO | RTMetrics report populated | 20260508 02:08:48.230 | INFO | Opening tmp RTMetrics json file | 20260508 02:08:48.274 | INFO | Serializing RTMetrics to json | 20260508 02:08:48.298 | INFO | Serialization done, swapping tmp file now... | 20260508 02:08:48.299 | INFO | RTMetrics done | 20260508 02:08:48.421 | INFO | Finished a round of Baz IO: Wrote 1.645 GiB in 3.644 seconds | 20260508 02:08:48.421 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.62% / 0.00% / 58.37% | 20260508 02:08:48.421 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.760782% / 3.97364e-06% / 5.5631e-05% | 20260508 02:08:48.421 | INFO | Baz IO overall: | 20260508 02:08:48.421 | INFO | BytesWritten : 1990387346071 | 20260508 02:08:48.421 | INFO | NumEvents : 962404013344 | 20260508 02:08:49.818 | INFO | Analyzing chunk frames = [4403200,4403712) | 20260508 02:08:49.818 | INFO | MemInfo: { | 20260508 02:08:49.818 | INFO | "gpu" : " 17 GB", | 20260508 02:08:49.818 | INFO | "huge" : "", | 20260508 02:08:49.818 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:49.818 | INFO | "malloc" : " 14 GB", | 20260508 02:08:49.818 | INFO | "pinned" : " 29 GB" | 20260508 02:08:49.818 | INFO | } | 20260508 02:08:49.818 | INFO | Perf report for the last 5.13896 seconds: | 20260508 02:08:49.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:49.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:49.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:49.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:49.818 | INFO | ANALYSIS : 68.4%, 3.0, 106.6ms, 0 | 20260508 02:08:49.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:49.818 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 02:08:49.818 | INFO | PRE_HQ : 66.0%, 1.8, 60.2ms, 0 | 20260508 02:08:49.818 | INFO | BAZWRITER : 70.5%, 4.1, 148.1ms, 0 | 20260508 02:08:49.818 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 02:08:49.818 | INFO | Uploaded 36GiB to GPU in 2.55636s (14.0825GiB/s) | 20260508 02:08:49.819 | INFO | Downloaded 38GiB to GPU in 2.68019s (14.1781GiB/s) | 20260508 02:08:54.958 | INFO | Analyzing chunk frames = [4403712,4404224) | 20260508 02:08:54.958 | INFO | MemInfo: { | 20260508 02:08:54.958 | INFO | "gpu" : " 17 GB", | 20260508 02:08:54.958 | INFO | "huge" : "", | 20260508 02:08:54.958 | INFO | "huge_shared" : " 29 GB", | 20260508 02:08:54.958 | INFO | "malloc" : " 14 GB", | 20260508 02:08:54.958 | INFO | "pinned" : " 29 GB" | 20260508 02:08:54.958 | INFO | } | 20260508 02:08:54.958 | INFO | Perf report for the last 5.14014 seconds: | 20260508 02:08:54.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:08:54.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.958 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:08:54.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.958 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:08:54.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:08:54.959 | INFO | Uploaded 36GiB to GPU in 2.54601s (14.1398GiB/s) | 20260508 02:08:54.959 | INFO | Downloaded 34GiB to GPU in 2.42082s (14.0448GiB/s) | 20260508 02:09:00.087 | INFO | Analyzing chunk frames = [4404224,4404736) | 20260508 02:09:00.087 | INFO | MemInfo: { | 20260508 02:09:00.087 | INFO | "gpu" : " 17 GB", | 20260508 02:09:00.087 | INFO | "huge" : "", | 20260508 02:09:00.088 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:00.088 | INFO | "malloc" : " 14 GB", | 20260508 02:09:00.088 | INFO | "pinned" : " 29 GB" | 20260508 02:09:00.088 | INFO | } | 20260508 02:09:00.088 | INFO | Perf report for the last 5.12928 seconds: | 20260508 02:09:00.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:00.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | ANALYSIS : 67.6%, 3.0, 104.6ms, 0 | 20260508 02:09:00.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:09:00.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:00.088 | INFO | Uploaded 36GiB to GPU in 2.58052s (13.9507GiB/s) | 20260508 02:09:00.088 | INFO | Downloaded 34GiB to GPU in 2.4905s (13.6519GiB/s) | 20260508 02:09:05.232 | INFO | Analyzing chunk frames = [4404736,4405248) | 20260508 02:09:05.232 | INFO | MemInfo: { | 20260508 02:09:05.232 | INFO | "gpu" : " 17 GB", | 20260508 02:09:05.232 | INFO | "huge" : "", | 20260508 02:09:05.233 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:05.233 | INFO | "malloc" : " 14 GB", | 20260508 02:09:05.233 | INFO | "pinned" : " 29 GB" | 20260508 02:09:05.233 | INFO | } | 20260508 02:09:05.233 | INFO | Perf report for the last 5.14509 seconds: | 20260508 02:09:05.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:05.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:09:05.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:09:05.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:05.233 | INFO | Uploaded 36GiB to GPU in 2.54845s (14.1262GiB/s) | 20260508 02:09:05.233 | INFO | Downloaded 34GiB to GPU in 2.47134s (13.7577GiB/s) | 20260508 02:09:10.376 | INFO | Analyzing chunk frames = [4405248,4405760) | 20260508 02:09:10.376 | INFO | MemInfo: { | 20260508 02:09:10.376 | INFO | "gpu" : " 17 GB", | 20260508 02:09:10.376 | INFO | "huge" : "", | 20260508 02:09:10.376 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:10.377 | INFO | "malloc" : " 14 GB", | 20260508 02:09:10.377 | INFO | "pinned" : " 29 GB" | 20260508 02:09:10.377 | INFO | } | 20260508 02:09:10.377 | INFO | Perf report for the last 5.14386 seconds: | 20260508 02:09:10.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:10.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 02:09:10.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:09:10.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:10.377 | INFO | Uploaded 36GiB to GPU in 2.39654s (15.0216GiB/s) | 20260508 02:09:10.377 | INFO | Downloaded 34GiB to GPU in 2.52392s (13.4711GiB/s) | 20260508 02:09:15.520 | INFO | Analyzing chunk frames = [4405760,4406272) | 20260508 02:09:15.520 | INFO | MemInfo: { | 20260508 02:09:15.520 | INFO | "gpu" : " 17 GB", | 20260508 02:09:15.520 | INFO | "huge" : "", | 20260508 02:09:15.520 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:15.520 | INFO | "malloc" : " 14 GB", | 20260508 02:09:15.520 | INFO | "pinned" : " 29 GB" | 20260508 02:09:15.520 | INFO | } | 20260508 02:09:15.520 | INFO | Perf report for the last 5.14342 seconds: | 20260508 02:09:15.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:15.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 02:09:15.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:09:15.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:15.520 | INFO | Uploaded 36GiB to GPU in 2.42104s (14.8696GiB/s) | 20260508 02:09:15.520 | INFO | Downloaded 34GiB to GPU in 2.41088s (14.1027GiB/s) | 20260508 02:09:20.653 | INFO | Analyzing chunk frames = [4406272,4406784) | 20260508 02:09:20.653 | INFO | MemInfo: { | 20260508 02:09:20.653 | INFO | "gpu" : " 17 GB", | 20260508 02:09:20.653 | INFO | "huge" : "", | 20260508 02:09:20.654 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:20.654 | INFO | "malloc" : " 14 GB", | 20260508 02:09:20.654 | INFO | "pinned" : " 29 GB" | 20260508 02:09:20.654 | INFO | } | 20260508 02:09:20.654 | INFO | Perf report for the last 5.13358 seconds: | 20260508 02:09:20.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:20.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:09:20.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:09:20.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:20.654 | INFO | Uploaded 36GiB to GPU in 2.44149s (14.7451GiB/s) | 20260508 02:09:20.654 | INFO | Downloaded 34GiB to GPU in 2.41057s (14.1046GiB/s) | 20260508 02:09:25.789 | INFO | Analyzing chunk frames = [4406784,4407296) | 20260508 02:09:25.789 | INFO | MemInfo: { | 20260508 02:09:25.789 | INFO | "gpu" : " 17 GB", | 20260508 02:09:25.789 | INFO | "huge" : "", | 20260508 02:09:25.789 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:25.789 | INFO | "malloc" : " 14 GB", | 20260508 02:09:25.789 | INFO | "pinned" : " 29 GB" | 20260508 02:09:25.789 | INFO | } | 20260508 02:09:25.789 | INFO | Perf report for the last 5.13573 seconds: | 20260508 02:09:25.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:25.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.790 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 02:09:25.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.790 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 02:09:25.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:25.790 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:09:25.790 | INFO | Total mem usage: 16.7 GiB | 20260508 02:09:25.790 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:09:25.790 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.3%) | 20260508 02:09:25.790 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:09:25.790 | INFO | High Water Breakdown: | 20260508 02:09:25.790 | INFO | 59.9% : 8.82 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:09:25.790 | INFO | 39.9% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:09:25.790 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:09:25.790 | INFO | ------------------------------------------ | 20260508 02:09:25.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:09:25.790 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:09:25.790 | INFO | Total mem usage: 26.5 GiB | 20260508 02:09:25.790 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:09:25.790 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:09:25.790 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:09:25.790 | INFO | High Water Breakdown: | 20260508 02:09:25.790 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:09:25.790 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:09:25.790 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:09:25.790 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:09:25.790 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:09:25.790 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:09:25.790 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:09:25.790 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:09:25.790 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:09:25.790 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:09:25.790 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:09:25.790 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:09:25.790 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:09:25.790 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:09:25.790 | INFO | ------------------------------------------ | 20260508 02:09:25.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:09:25.790 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:09:25.790 | INFO | Total mem usage: 27.8 GiB | 20260508 02:09:25.790 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:09:25.790 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:09:25.790 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:09:25.790 | INFO | High Water Breakdown: | 20260508 02:09:25.790 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:09:25.790 | INFO | ------------------------------------------ | 20260508 02:09:25.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:09:25.790 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:09:25.790 | INFO | Total mem usage: 15.7 GiB | 20260508 02:09:25.790 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:09:25.790 | INFO | Uploaded 36GiB to GPU in 2.48567s (14.483GiB/s) | 20260508 02:09:25.790 | INFO | Downloaded 34GiB to GPU in 2.42192s (14.0385GiB/s) | 20260508 02:09:25.790 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:09:25.791 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:09:25.791 | INFO | High Water Breakdown: | 20260508 02:09:25.791 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:09:25.791 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:09:25.791 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:09:25.791 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:09:25.791 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:09:25.791 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:09:25.791 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:09:25.791 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:09:25.791 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:09:25.791 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:09:25.791 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:09:25.791 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:09:25.791 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:09:25.791 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:09:25.791 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:09:25.791 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:09:25.791 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:09:25.791 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:09:25.791 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:09:25.791 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:09:25.791 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:09:25.791 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:09:25.791 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:09:25.791 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:09:25.791 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:09:25.791 | INFO | ------------------------------------------ | 20260508 02:09:25.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:09:25.791 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:09:25.791 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:09:25.791 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:09:25.791 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.43e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.22e-02 / 7.02e-03 | 20260508 02:09:25.791 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.48e+00, 5.90e-01 / 1.78e-01 | 20260508 02:09:25.791 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 8.94e-02 | 20260508 02:09:25.791 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.61e+01, 4.18e+00 / 1.82e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.63e+00 / 1.39e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.42e+00 / 2.74e+00, 4.23e-01 / 1.66e-01 | 20260508 02:09:25.791 | INFO | PerfReport: 3.30%, Metrics -- 1.08e+00 / 1.26e+00, 2.67e-01 / 1.36e-01 | 20260508 02:09:25.791 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.31e-01 / 3.68e-01, 5.23e-02 / 2.57e-02 | 20260508 02:09:25.791 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:09:25.791 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:09:25.791 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:09:25.791 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 24.38%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 24.26%, Download -- 2.58e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 2.33%, LockUpload -- 2.48e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 2.65%, LockDownload -- 2.82e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:09:25.791 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260508 02:09:25.902 | INFO | Starting a round of Baz IO | 20260508 02:09:29.362 | INFO | Populating RTMetrics report | 20260508 02:09:29.362 | INFO | RTMetrics report populated | 20260508 02:09:29.362 | INFO | Opening tmp RTMetrics json file | 20260508 02:09:29.387 | INFO | Serializing RTMetrics to json | 20260508 02:09:29.388 | INFO | Serialization done, swapping tmp file now... | 20260508 02:09:29.389 | INFO | RTMetrics done | 20260508 02:09:29.451 | INFO | Finished a round of Baz IO: Wrote 1.646 GiB in 3.548 seconds | 20260508 02:09:29.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.62% / 0.00% / 58.38% | 20260508 02:09:29.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.7611% / 3.97364e-06% / 5.96046e-05% | 20260508 02:09:29.451 | INFO | Baz IO overall: | 20260508 02:09:29.451 | INFO | BytesWritten : 1992154904396 | 20260508 02:09:29.451 | INFO | NumEvents : 963252112231 | 20260508 02:09:30.932 | INFO | Analyzing chunk frames = [4407296,4407808) | 20260508 02:09:30.932 | INFO | MemInfo: { | 20260508 02:09:30.932 | INFO | "gpu" : " 17 GB", | 20260508 02:09:30.932 | INFO | "huge" : "", | 20260508 02:09:30.932 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:30.932 | INFO | "malloc" : " 14 GB", | 20260508 02:09:30.932 | INFO | "pinned" : " 29 GB" | 20260508 02:09:30.932 | INFO | } | 20260508 02:09:30.933 | INFO | Perf report for the last 5.14314 seconds: | 20260508 02:09:30.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:30.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:30.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:30.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:30.933 | INFO | ANALYSIS : 68.8%, 2.9, 104.8ms, 0 | 20260508 02:09:30.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:30.933 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260508 02:09:30.933 | INFO | PRE_HQ : 67.6%, 1.8, 62.6ms, 0 | 20260508 02:09:30.933 | INFO | BAZWRITER : 68.5%, 3.8, 134.8ms, 0 | 20260508 02:09:30.933 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 02:09:30.933 | INFO | Uploaded 36GiB to GPU in 2.57197s (13.997GiB/s) | 20260508 02:09:30.933 | INFO | Downloaded 38GiB to GPU in 2.70523s (14.0469GiB/s) | 20260508 02:09:36.074 | INFO | Analyzing chunk frames = [4407808,4408320) | 20260508 02:09:36.074 | INFO | MemInfo: { | 20260508 02:09:36.074 | INFO | "gpu" : " 17 GB", | 20260508 02:09:36.074 | INFO | "huge" : "", | 20260508 02:09:36.074 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:36.074 | INFO | "malloc" : " 14 GB", | 20260508 02:09:36.075 | INFO | "pinned" : " 29 GB" | 20260508 02:09:36.075 | INFO | } | 20260508 02:09:36.075 | INFO | Perf report for the last 5.14217 seconds: | 20260508 02:09:36.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:36.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 02:09:36.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:09:36.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:36.075 | INFO | Uploaded 36GiB to GPU in 2.55668s (14.0807GiB/s) | 20260508 02:09:36.075 | INFO | Downloaded 34GiB to GPU in 2.42311s (14.0316GiB/s) | 20260508 02:09:41.208 | INFO | Analyzing chunk frames = [4408320,4408832) | 20260508 02:09:41.208 | INFO | MemInfo: { | 20260508 02:09:41.208 | INFO | "gpu" : " 17 GB", | 20260508 02:09:41.208 | INFO | "huge" : "", | 20260508 02:09:41.208 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:41.208 | INFO | "malloc" : " 14 GB", | 20260508 02:09:41.208 | INFO | "pinned" : " 29 GB" | 20260508 02:09:41.208 | INFO | } | 20260508 02:09:41.209 | INFO | Perf report for the last 5.13383 seconds: | 20260508 02:09:41.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:41.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 02:09:41.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:09:41.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:41.209 | INFO | Uploaded 36GiB to GPU in 2.58353s (13.9344GiB/s) | 20260508 02:09:41.209 | INFO | Downloaded 34GiB to GPU in 2.49169s (13.6453GiB/s) | 20260508 02:09:46.344 | INFO | Analyzing chunk frames = [4408832,4409344) | 20260508 02:09:46.344 | INFO | MemInfo: { | 20260508 02:09:46.344 | INFO | "gpu" : " 17 GB", | 20260508 02:09:46.344 | INFO | "huge" : "", | 20260508 02:09:46.344 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:46.344 | INFO | "malloc" : " 14 GB", | 20260508 02:09:46.344 | INFO | "pinned" : " 29 GB" | 20260508 02:09:46.344 | INFO | } | 20260508 02:09:46.344 | INFO | Perf report for the last 5.13545 seconds: | 20260508 02:09:46.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:46.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 02:09:46.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:09:46.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:46.344 | INFO | Uploaded 36GiB to GPU in 2.57854s (13.9614GiB/s) | 20260508 02:09:46.344 | INFO | Downloaded 34GiB to GPU in 2.48637s (13.6745GiB/s) | 20260508 02:09:51.489 | INFO | Analyzing chunk frames = [4409344,4409856) | 20260508 02:09:51.489 | INFO | MemInfo: { | 20260508 02:09:51.489 | INFO | "gpu" : " 17 GB", | 20260508 02:09:51.489 | INFO | "huge" : "", | 20260508 02:09:51.489 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:51.489 | INFO | "malloc" : " 14 GB", | 20260508 02:09:51.490 | INFO | "pinned" : " 29 GB" | 20260508 02:09:51.490 | INFO | } | 20260508 02:09:51.490 | INFO | Perf report for the last 5.14556 seconds: | 20260508 02:09:51.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:51.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 02:09:51.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:09:51.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:51.490 | INFO | Uploaded 36GiB to GPU in 2.33385s (15.4251GiB/s) | 20260508 02:09:51.490 | INFO | Downloaded 34GiB to GPU in 2.48466s (13.684GiB/s) | 20260508 02:09:56.627 | INFO | Analyzing chunk frames = [4409856,4410368) | 20260508 02:09:56.627 | INFO | MemInfo: { | 20260508 02:09:56.627 | INFO | "gpu" : " 17 GB", | 20260508 02:09:56.627 | INFO | "huge" : "", | 20260508 02:09:56.627 | INFO | "huge_shared" : " 29 GB", | 20260508 02:09:56.627 | INFO | "malloc" : " 14 GB", | 20260508 02:09:56.627 | INFO | "pinned" : " 29 GB" | 20260508 02:09:56.627 | INFO | } | 20260508 02:09:56.627 | INFO | Perf report for the last 5.1378 seconds: | 20260508 02:09:56.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:09:56.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.627 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:09:56.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.628 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:09:56.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:09:56.628 | INFO | Uploaded 36GiB to GPU in 2.40594s (14.963GiB/s) | 20260508 02:09:56.628 | INFO | Downloaded 34GiB to GPU in 2.41331s (14.0885GiB/s) | 20260508 02:10:01.772 | INFO | Analyzing chunk frames = [4410368,4410880) | 20260508 02:10:01.772 | INFO | MemInfo: { | 20260508 02:10:01.772 | INFO | "gpu" : " 17 GB", | 20260508 02:10:01.772 | INFO | "huge" : "", | 20260508 02:10:01.772 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:01.772 | INFO | "malloc" : " 14 GB", | 20260508 02:10:01.772 | INFO | "pinned" : " 29 GB" | 20260508 02:10:01.772 | INFO | } | 20260508 02:10:01.773 | INFO | Perf report for the last 5.14514 seconds: | 20260508 02:10:01.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:01.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 02:10:01.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 02:10:01.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:01.773 | INFO | Uploaded 36GiB to GPU in 2.53782s (14.1854GiB/s) | 20260508 02:10:01.773 | INFO | Downloaded 34GiB to GPU in 2.46064s (13.8175GiB/s) | 20260508 02:10:06.904 | INFO | Analyzing chunk frames = [4410880,4411392) | 20260508 02:10:06.904 | INFO | MemInfo: { | 20260508 02:10:06.904 | INFO | "gpu" : " 17 GB", | 20260508 02:10:06.904 | INFO | "huge" : "", | 20260508 02:10:06.904 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:06.904 | INFO | "malloc" : " 14 GB", | 20260508 02:10:06.904 | INFO | "pinned" : " 29 GB" | 20260508 02:10:06.904 | INFO | } | 20260508 02:10:06.905 | INFO | Perf report for the last 5.13194 seconds: | 20260508 02:10:06.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:06.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:10:06.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 02:10:06.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:06.905 | INFO | Uploaded 36GiB to GPU in 2.458s (14.646GiB/s) | 20260508 02:10:06.905 | INFO | Downloaded 34GiB to GPU in 2.42905s (13.9972GiB/s) | 20260508 02:10:07.018 | INFO | Starting a round of Baz IO | 20260508 02:10:10.479 | INFO | Populating RTMetrics report | 20260508 02:10:10.480 | INFO | RTMetrics report populated | 20260508 02:10:10.480 | INFO | Opening tmp RTMetrics json file | 20260508 02:10:10.511 | INFO | Serializing RTMetrics to json | 20260508 02:10:10.593 | INFO | Serialization done, swapping tmp file now... | 20260508 02:10:10.600 | INFO | RTMetrics done | 20260508 02:10:10.631 | INFO | Finished a round of Baz IO: Wrote 1.646 GiB in 3.612 seconds | 20260508 02:10:10.631 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.62% / 0.00% / 58.38% | 20260508 02:10:10.631 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.760595% / 7.94729e-06% / 4.76837e-05% | 20260508 02:10:10.631 | INFO | Baz IO overall: | 20260508 02:10:10.631 | INFO | BytesWritten : 1993922277776 | 20260508 02:10:10.631 | INFO | NumEvents : 964100343074 | 20260508 02:10:12.044 | INFO | Analyzing chunk frames = [4411392,4411904) | 20260508 02:10:12.044 | INFO | MemInfo: { | 20260508 02:10:12.044 | INFO | "gpu" : " 17 GB", | 20260508 02:10:12.044 | INFO | "huge" : "", | 20260508 02:10:12.044 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:12.044 | INFO | "malloc" : " 14 GB", | 20260508 02:10:12.044 | INFO | "pinned" : " 29 GB" | 20260508 02:10:12.044 | INFO | } | 20260508 02:10:12.044 | INFO | Perf report for the last 5.13935 seconds: | 20260508 02:10:12.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:12.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:12.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:12.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:12.044 | INFO | ANALYSIS : 68.6%, 2.9, 104.6ms, 0 | 20260508 02:10:12.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:12.044 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260508 02:10:12.044 | INFO | PRE_HQ : 67.3%, 1.8, 62.3ms, 0 | 20260508 02:10:12.044 | INFO | BAZWRITER : 69.9%, 3.6, 129.1ms, 0 | 20260508 02:10:12.044 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260508 02:10:12.045 | INFO | Uploaded 36GiB to GPU in 2.59744s (13.8598GiB/s) | 20260508 02:10:12.045 | INFO | Downloaded 38GiB to GPU in 2.70413s (14.0526GiB/s) | 20260508 02:10:17.184 | INFO | Analyzing chunk frames = [4411904,4412416) | 20260508 02:10:17.184 | INFO | MemInfo: { | 20260508 02:10:17.184 | INFO | "gpu" : " 17 GB", | 20260508 02:10:17.184 | INFO | "huge" : "", | 20260508 02:10:17.184 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:17.184 | INFO | "malloc" : " 14 GB", | 20260508 02:10:17.184 | INFO | "pinned" : " 29 GB" | 20260508 02:10:17.184 | INFO | } | 20260508 02:10:17.184 | INFO | Perf report for the last 5.14057 seconds: | 20260508 02:10:17.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:17.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.184 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:10:17.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.185 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:10:17.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:17.185 | INFO | Uploaded 36GiB to GPU in 2.54992s (14.1181GiB/s) | 20260508 02:10:17.185 | INFO | Downloaded 34GiB to GPU in 2.42648s (14.0121GiB/s) | 20260508 02:10:22.324 | INFO | Analyzing chunk frames = [4412416,4412928) | 20260508 02:10:22.324 | INFO | MemInfo: { | 20260508 02:10:22.324 | INFO | "gpu" : " 17 GB", | 20260508 02:10:22.324 | INFO | "huge" : "", | 20260508 02:10:22.324 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:22.324 | INFO | "malloc" : " 14 GB", | 20260508 02:10:22.324 | INFO | "pinned" : " 29 GB" | 20260508 02:10:22.324 | INFO | } | 20260508 02:10:22.324 | INFO | Perf report for the last 5.13953 seconds: | 20260508 02:10:22.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:22.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:10:22.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:10:22.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:22.324 | INFO | Uploaded 36GiB to GPU in 2.52727s (14.2446GiB/s) | 20260508 02:10:22.324 | INFO | Downloaded 34GiB to GPU in 2.49229s (13.6421GiB/s) | 20260508 02:10:27.469 | INFO | Analyzing chunk frames = [4412928,4413440) | 20260508 02:10:27.469 | INFO | MemInfo: { | 20260508 02:10:27.469 | INFO | "gpu" : " 17 GB", | 20260508 02:10:27.469 | INFO | "huge" : "", | 20260508 02:10:27.469 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:27.469 | INFO | "malloc" : " 14 GB", | 20260508 02:10:27.469 | INFO | "pinned" : " 29 GB" | 20260508 02:10:27.469 | INFO | } | 20260508 02:10:27.469 | INFO | Perf report for the last 5.14491 seconds: | 20260508 02:10:27.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:27.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:10:27.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:10:27.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:27.469 | INFO | Uploaded 36GiB to GPU in 2.54644s (14.1374GiB/s) | 20260508 02:10:27.469 | INFO | Downloaded 34GiB to GPU in 2.49451s (13.63GiB/s) | 20260508 02:10:32.605 | INFO | Analyzing chunk frames = [4413440,4413952) | 20260508 02:10:32.605 | INFO | MemInfo: { | 20260508 02:10:32.605 | INFO | "gpu" : " 17 GB", | 20260508 02:10:32.605 | INFO | "huge" : "", | 20260508 02:10:32.605 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:32.606 | INFO | "malloc" : " 14 GB", | 20260508 02:10:32.606 | INFO | "pinned" : " 29 GB" | 20260508 02:10:32.606 | INFO | } | 20260508 02:10:32.606 | INFO | Perf report for the last 5.13673 seconds: | 20260508 02:10:32.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:32.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 02:10:32.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 02:10:32.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:32.606 | INFO | Uploaded 36GiB to GPU in 2.45048s (14.691GiB/s) | 20260508 02:10:32.606 | INFO | Downloaded 34GiB to GPU in 2.56365s (13.2624GiB/s) | 20260508 02:10:37.747 | INFO | Analyzing chunk frames = [4413952,4414464) | 20260508 02:10:37.747 | INFO | MemInfo: { | 20260508 02:10:37.747 | INFO | "gpu" : " 17 GB", | 20260508 02:10:37.747 | INFO | "huge" : "", | 20260508 02:10:37.747 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:37.747 | INFO | "malloc" : " 14 GB", | 20260508 02:10:37.747 | INFO | "pinned" : " 29 GB" | 20260508 02:10:37.747 | INFO | } | 20260508 02:10:37.747 | INFO | Perf report for the last 5.14141 seconds: | 20260508 02:10:37.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:37.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.747 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:10:37.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.747 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:10:37.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:37.748 | INFO | Uploaded 36GiB to GPU in 2.42768s (14.829GiB/s) | 20260508 02:10:37.748 | INFO | Downloaded 34GiB to GPU in 2.41673s (14.0686GiB/s) | 20260508 02:10:42.882 | INFO | Analyzing chunk frames = [4414464,4414976) | 20260508 02:10:42.882 | INFO | MemInfo: { | 20260508 02:10:42.882 | INFO | "gpu" : " 17 GB", | 20260508 02:10:42.882 | INFO | "huge" : "", | 20260508 02:10:42.882 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:42.882 | INFO | "malloc" : " 14 GB", | 20260508 02:10:42.882 | INFO | "pinned" : " 29 GB" | 20260508 02:10:42.882 | INFO | } | 20260508 02:10:42.882 | INFO | Perf report for the last 5.13524 seconds: | 20260508 02:10:42.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:42.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.882 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:10:42.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.882 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:10:42.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:42.883 | INFO | Uploaded 36GiB to GPU in 2.46391s (14.6109GiB/s) | 20260508 02:10:42.883 | INFO | Downloaded 34GiB to GPU in 2.41199s (14.0963GiB/s) | 20260508 02:10:48.022 | INFO | Analyzing chunk frames = [4414976,4415488) | 20260508 02:10:48.022 | INFO | MemInfo: { | 20260508 02:10:48.022 | INFO | "gpu" : " 17 GB", | 20260508 02:10:48.022 | INFO | "huge" : "", | 20260508 02:10:48.022 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:48.022 | INFO | "malloc" : " 14 GB", | 20260508 02:10:48.022 | INFO | "pinned" : " 29 GB" | 20260508 02:10:48.022 | INFO | } | 20260508 02:10:48.022 | INFO | Perf report for the last 5.13978 seconds: | 20260508 02:10:48.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:48.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.022 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 02:10:48.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.022 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 02:10:48.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:48.023 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:10:48.023 | INFO | Total mem usage: 16.7 GiB | 20260508 02:10:48.023 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:10:48.023 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15%) | 20260508 02:10:48.023 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:10:48.023 | INFO | High Water Breakdown: | 20260508 02:10:48.023 | INFO | 59.7% : 8.66 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:10:48.023 | INFO | 40.2% : 5.82 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:10:48.023 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:10:48.023 | INFO | ------------------------------------------ | 20260508 02:10:48.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:10:48.023 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:10:48.023 | INFO | Total mem usage: 26.5 GiB | 20260508 02:10:48.023 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:10:48.023 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:10:48.023 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:10:48.023 | INFO | High Water Breakdown: | 20260508 02:10:48.023 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:10:48.023 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:10:48.023 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:10:48.023 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:10:48.023 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:10:48.023 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:10:48.023 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:10:48.023 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:10:48.023 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:10:48.023 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:10:48.023 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:10:48.023 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:10:48.023 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:10:48.023 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:10:48.023 | INFO | ------------------------------------------ | 20260508 02:10:48.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:10:48.023 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:10:48.023 | INFO | Total mem usage: 27.8 GiB | 20260508 02:10:48.023 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:10:48.023 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:10:48.023 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:10:48.023 | INFO | High Water Breakdown: | 20260508 02:10:48.023 | INFO | Uploaded 36GiB to GPU in 2.49642s (14.4206GiB/s) | 20260508 02:10:48.023 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:10:48.023 | INFO | ------------------------------------------ | 20260508 02:10:48.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:10:48.023 | INFO | Downloaded 34GiB to GPU in 2.42496s (14.0208GiB/s) | 20260508 02:10:48.023 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:10:48.023 | INFO | Total mem usage: 15.7 GiB | 20260508 02:10:48.023 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:10:48.023 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:10:48.023 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:10:48.023 | INFO | High Water Breakdown: | 20260508 02:10:48.023 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:10:48.023 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:10:48.023 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:10:48.023 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:10:48.023 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:10:48.023 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:10:48.023 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:10:48.023 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:10:48.023 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:10:48.023 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:10:48.023 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:10:48.023 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:10:48.023 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:10:48.023 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:10:48.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:10:48.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:10:48.023 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:10:48.024 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:10:48.024 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:10:48.024 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:10:48.024 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:10:48.024 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:10:48.024 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:10:48.024 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:10:48.024 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:10:48.024 | INFO | ------------------------------------------ | 20260508 02:10:48.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:10:48.024 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:10:48.024 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:10:48.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:10:48.024 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.49e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.15e-02 / 6.01e-03 | 20260508 02:10:48.024 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.48e+00, 5.89e-01 / 1.72e-01 | 20260508 02:10:48.024 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.49e+00, 2.06e-01 / 1.01e-01 | 20260508 02:10:48.024 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 4.10e+00 / 1.80e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.65e+00 / 1.31e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.43e+00 / 2.67e+00, 4.32e-01 / 2.23e-01 | 20260508 02:10:48.024 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.23e+00, 2.66e-01 / 1.19e-01 | 20260508 02:10:48.024 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.31e-01 / 3.65e-01, 5.31e-02 / 2.84e-02 | 20260508 02:10:48.024 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:10:48.024 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:10:48.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:10:48.024 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 24.53%, Upload -- 2.61e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 24.38%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 2.35%, LockUpload -- 2.50e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 2.60%, LockDownload -- 2.76e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 15.37%, LockCompute -- 1.63e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:10:48.024 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 2.97e-02 / 0.00e+00 | 20260508 02:10:48.120 | INFO | Starting a round of Baz IO | 20260508 02:10:51.563 | INFO | Populating RTMetrics report | 20260508 02:10:51.563 | INFO | RTMetrics report populated | 20260508 02:10:51.563 | INFO | Opening tmp RTMetrics json file | 20260508 02:10:51.583 | INFO | Serializing RTMetrics to json | 20260508 02:10:51.585 | INFO | Serialization done, swapping tmp file now... | 20260508 02:10:51.586 | INFO | RTMetrics done | 20260508 02:10:51.659 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.538 seconds | 20260508 02:10:51.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.61% / 0.00% / 58.38% | 20260508 02:10:51.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.757198% / 0% / 7.54992e-05% | 20260508 02:10:51.659 | INFO | Baz IO overall: | 20260508 02:10:51.659 | INFO | BytesWritten : 1995687627619 | 20260508 02:10:51.659 | INFO | NumEvents : 964947018578 | 20260508 02:10:53.157 | INFO | Analyzing chunk frames = [4415488,4416000) | 20260508 02:10:53.157 | INFO | MemInfo: { | 20260508 02:10:53.157 | INFO | "gpu" : " 17 GB", | 20260508 02:10:53.157 | INFO | "huge" : "", | 20260508 02:10:53.157 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:53.157 | INFO | "malloc" : " 14 GB", | 20260508 02:10:53.157 | INFO | "pinned" : " 29 GB" | 20260508 02:10:53.157 | INFO | } | 20260508 02:10:53.157 | INFO | Perf report for the last 5.13493 seconds: | 20260508 02:10:53.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:53.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:53.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:53.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:53.157 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:10:53.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:53.157 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 02:10:53.157 | INFO | PRE_HQ : 65.8%, 1.6, 56.1ms, 0 | 20260508 02:10:53.157 | INFO | BAZWRITER : 68.5%, 3.5, 123.3ms, 0 | 20260508 02:10:53.157 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:10:53.157 | INFO | Uploaded 36GiB to GPU in 2.60748s (13.8064GiB/s) | 20260508 02:10:53.158 | INFO | Downloaded 38GiB to GPU in 2.68918s (14.1307GiB/s) | 20260508 02:10:58.293 | INFO | Analyzing chunk frames = [4416000,4416512) | 20260508 02:10:58.293 | INFO | MemInfo: { | 20260508 02:10:58.293 | INFO | "gpu" : " 17 GB", | 20260508 02:10:58.293 | INFO | "huge" : "", | 20260508 02:10:58.293 | INFO | "huge_shared" : " 29 GB", | 20260508 02:10:58.293 | INFO | "malloc" : " 14 GB", | 20260508 02:10:58.293 | INFO | "pinned" : " 29 GB" | 20260508 02:10:58.293 | INFO | } | 20260508 02:10:58.293 | INFO | Perf report for the last 5.13558 seconds: | 20260508 02:10:58.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:10:58.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 02:10:58.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:10:58.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:10:58.293 | INFO | Uploaded 36GiB to GPU in 2.5606s (14.0592GiB/s) | 20260508 02:10:58.293 | INFO | Downloaded 34GiB to GPU in 2.42118s (14.0427GiB/s) | 20260508 02:11:03.439 | INFO | Analyzing chunk frames = [4416512,4417024) | 20260508 02:11:03.439 | INFO | MemInfo: { | 20260508 02:11:03.439 | INFO | "gpu" : " 17 GB", | 20260508 02:11:03.439 | INFO | "huge" : "", | 20260508 02:11:03.439 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:03.439 | INFO | "malloc" : " 14 GB", | 20260508 02:11:03.439 | INFO | "pinned" : " 29 GB" | 20260508 02:11:03.439 | INFO | } | 20260508 02:11:03.439 | INFO | Perf report for the last 5.14651 seconds: | 20260508 02:11:03.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:03.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.439 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 02:11:03.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.439 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:11:03.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:03.440 | INFO | Uploaded 36GiB to GPU in 2.5186s (14.2936GiB/s) | 20260508 02:11:03.440 | INFO | Downloaded 34GiB to GPU in 2.47573s (13.7333GiB/s) | 20260508 02:11:08.582 | INFO | Analyzing chunk frames = [4417024,4417536) | 20260508 02:11:08.582 | INFO | MemInfo: { | 20260508 02:11:08.582 | INFO | "gpu" : " 17 GB", | 20260508 02:11:08.582 | INFO | "huge" : "", | 20260508 02:11:08.582 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:08.582 | INFO | "malloc" : " 14 GB", | 20260508 02:11:08.582 | INFO | "pinned" : " 29 GB" | 20260508 02:11:08.582 | INFO | } | 20260508 02:11:08.582 | INFO | Perf report for the last 5.14327 seconds: | 20260508 02:11:08.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:08.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:11:08.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:11:08.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:08.583 | INFO | Uploaded 36GiB to GPU in 2.54137s (14.1656GiB/s) | 20260508 02:11:08.583 | INFO | Downloaded 34GiB to GPU in 2.46907s (13.7703GiB/s) | 20260508 02:11:13.722 | INFO | Analyzing chunk frames = [4417536,4418048) | 20260508 02:11:13.722 | INFO | MemInfo: { | 20260508 02:11:13.722 | INFO | "gpu" : " 17 GB", | 20260508 02:11:13.722 | INFO | "huge" : "", | 20260508 02:11:13.722 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:13.722 | INFO | "malloc" : " 14 GB", | 20260508 02:11:13.722 | INFO | "pinned" : " 29 GB" | 20260508 02:11:13.722 | INFO | } | 20260508 02:11:13.722 | INFO | Perf report for the last 5.1395 seconds: | 20260508 02:11:13.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:13.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:11:13.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:11:13.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:13.722 | INFO | Uploaded 36GiB to GPU in 2.3632s (15.2336GiB/s) | 20260508 02:11:13.722 | INFO | Downloaded 34GiB to GPU in 2.52751s (13.452GiB/s) | 20260508 02:11:18.854 | INFO | Analyzing chunk frames = [4418048,4418560) | 20260508 02:11:18.854 | INFO | MemInfo: { | 20260508 02:11:18.854 | INFO | "gpu" : " 17 GB", | 20260508 02:11:18.855 | INFO | "huge" : "", | 20260508 02:11:18.855 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:18.855 | INFO | "malloc" : " 14 GB", | 20260508 02:11:18.855 | INFO | "pinned" : " 29 GB" | 20260508 02:11:18.855 | INFO | } | 20260508 02:11:18.855 | INFO | Perf report for the last 5.13288 seconds: | 20260508 02:11:18.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:18.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:11:18.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:11:18.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:18.855 | INFO | Uploaded 36GiB to GPU in 2.4339s (14.7911GiB/s) | 20260508 02:11:18.855 | INFO | Downloaded 34GiB to GPU in 2.40292s (14.1495GiB/s) | 20260508 02:11:24.000 | INFO | Analyzing chunk frames = [4418560,4419072) | 20260508 02:11:24.001 | INFO | MemInfo: { | 20260508 02:11:24.001 | INFO | "gpu" : " 17 GB", | 20260508 02:11:24.001 | INFO | "huge" : "", | 20260508 02:11:24.001 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:24.001 | INFO | "malloc" : " 14 GB", | 20260508 02:11:24.001 | INFO | "pinned" : " 29 GB" | 20260508 02:11:24.001 | INFO | } | 20260508 02:11:24.001 | INFO | Perf report for the last 5.14613 seconds: | 20260508 02:11:24.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:24.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | ANALYSIS : 66.4%, 3.0, 104.0ms, 0 | 20260508 02:11:24.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 02:11:24.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:24.001 | INFO | Uploaded 36GiB to GPU in 2.4764s (14.5373GiB/s) | 20260508 02:11:24.001 | INFO | Downloaded 34GiB to GPU in 2.43042s (13.9893GiB/s) | 20260508 02:11:29.140 | INFO | Analyzing chunk frames = [4419072,4419584) | 20260508 02:11:29.140 | INFO | MemInfo: { | 20260508 02:11:29.140 | INFO | "gpu" : " 17 GB", | 20260508 02:11:29.140 | INFO | "huge" : "", | 20260508 02:11:29.140 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:29.140 | INFO | "malloc" : " 14 GB", | 20260508 02:11:29.140 | INFO | "pinned" : " 29 GB" | 20260508 02:11:29.140 | INFO | } | 20260508 02:11:29.140 | INFO | Perf report for the last 5.13897 seconds: | 20260508 02:11:29.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:29.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:11:29.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 02:11:29.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:29.140 | INFO | Uploaded 36GiB to GPU in 2.49094s (14.4524GiB/s) | 20260508 02:11:29.141 | INFO | Downloaded 34GiB to GPU in 2.42654s (14.0117GiB/s) | 20260508 02:11:29.257 | INFO | Starting a round of Baz IO | 20260508 02:11:32.695 | INFO | Populating RTMetrics report | 20260508 02:11:32.695 | INFO | RTMetrics report populated | 20260508 02:11:32.695 | INFO | Opening tmp RTMetrics json file | 20260508 02:11:32.707 | INFO | Serializing RTMetrics to json | 20260508 02:11:32.718 | INFO | Serialization done, swapping tmp file now... | 20260508 02:11:32.719 | INFO | RTMetrics done | 20260508 02:11:32.808 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.55 seconds | 20260508 02:11:32.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.61% / 0.00% / 58.39% | 20260508 02:11:32.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.756069% / 3.97364e-06% / 7.15256e-05% | 20260508 02:11:32.808 | INFO | Baz IO overall: | 20260508 02:11:32.808 | INFO | BytesWritten : 1997453187325 | 20260508 02:11:32.808 | INFO | NumEvents : 965793673272 | 20260508 02:11:34.278 | INFO | Analyzing chunk frames = [4419584,4420096) | 20260508 02:11:34.278 | INFO | MemInfo: { | 20260508 02:11:34.278 | INFO | "gpu" : " 17 GB", | 20260508 02:11:34.278 | INFO | "huge" : "", | 20260508 02:11:34.278 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:34.278 | INFO | "malloc" : " 14 GB", | 20260508 02:11:34.278 | INFO | "pinned" : " 29 GB" | 20260508 02:11:34.278 | INFO | } | 20260508 02:11:34.278 | INFO | Perf report for the last 5.13783 seconds: | 20260508 02:11:34.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:34.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:34.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:34.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:34.278 | INFO | ANALYSIS : 68.5%, 2.9, 103.5ms, 0 | 20260508 02:11:34.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:34.278 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260508 02:11:34.278 | INFO | PRE_HQ : 67.5%, 1.8, 63.5ms, 0 | 20260508 02:11:34.278 | INFO | BAZWRITER : 68.8%, 4.0, 143.1ms, 0 | 20260508 02:11:34.278 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:11:34.278 | INFO | Uploaded 36GiB to GPU in 2.62507s (13.7139GiB/s) | 20260508 02:11:34.278 | INFO | Downloaded 38GiB to GPU in 2.69123s (14.1199GiB/s) | 20260508 02:11:39.416 | INFO | Analyzing chunk frames = [4420096,4420608) | 20260508 02:11:39.416 | INFO | MemInfo: { | 20260508 02:11:39.416 | INFO | "gpu" : " 17 GB", | 20260508 02:11:39.416 | INFO | "huge" : "", | 20260508 02:11:39.416 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:39.416 | INFO | "malloc" : " 14 GB", | 20260508 02:11:39.416 | INFO | "pinned" : " 29 GB" | 20260508 02:11:39.416 | INFO | } | 20260508 02:11:39.416 | INFO | Perf report for the last 5.13804 seconds: | 20260508 02:11:39.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:39.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:11:39.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:11:39.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:39.416 | INFO | Uploaded 36GiB to GPU in 2.56242s (14.0492GiB/s) | 20260508 02:11:39.416 | INFO | Downloaded 34GiB to GPU in 2.41984s (14.0505GiB/s) | 20260508 02:11:44.555 | INFO | Analyzing chunk frames = [4420608,4421120) | 20260508 02:11:44.555 | INFO | MemInfo: { | 20260508 02:11:44.555 | INFO | "gpu" : " 17 GB", | 20260508 02:11:44.555 | INFO | "huge" : "", | 20260508 02:11:44.555 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:44.555 | INFO | "malloc" : " 14 GB", | 20260508 02:11:44.555 | INFO | "pinned" : " 29 GB" | 20260508 02:11:44.555 | INFO | } | 20260508 02:11:44.555 | INFO | Perf report for the last 5.13917 seconds: | 20260508 02:11:44.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:44.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.555 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 02:11:44.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.555 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:11:44.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:44.556 | INFO | Uploaded 36GiB to GPU in 2.57637s (13.9732GiB/s) | 20260508 02:11:44.556 | INFO | Downloaded 34GiB to GPU in 2.48416s (13.6867GiB/s) | 20260508 02:11:49.695 | INFO | Analyzing chunk frames = [4421120,4421632) | 20260508 02:11:49.695 | INFO | MemInfo: { | 20260508 02:11:49.695 | INFO | "gpu" : " 17 GB", | 20260508 02:11:49.695 | INFO | "huge" : "", | 20260508 02:11:49.695 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:49.695 | INFO | "malloc" : " 14 GB", | 20260508 02:11:49.695 | INFO | "pinned" : " 29 GB" | 20260508 02:11:49.695 | INFO | } | 20260508 02:11:49.695 | INFO | Perf report for the last 5.14015 seconds: | 20260508 02:11:49.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:49.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.695 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:11:49.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.695 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:11:49.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:49.696 | INFO | Uploaded 36GiB to GPU in 2.54925s (14.1218GiB/s) | 20260508 02:11:49.696 | INFO | Downloaded 34GiB to GPU in 2.4854s (13.6799GiB/s) | 20260508 02:11:54.839 | INFO | Analyzing chunk frames = [4421632,4422144) | 20260508 02:11:54.839 | INFO | MemInfo: { | 20260508 02:11:54.839 | INFO | "gpu" : " 17 GB", | 20260508 02:11:54.839 | INFO | "huge" : "", | 20260508 02:11:54.839 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:54.839 | INFO | "malloc" : " 14 GB", | 20260508 02:11:54.839 | INFO | "pinned" : " 29 GB" | 20260508 02:11:54.839 | INFO | } | 20260508 02:11:54.839 | INFO | Perf report for the last 5.14393 seconds: | 20260508 02:11:54.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:54.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.839 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 02:11:54.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.839 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:11:54.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:54.840 | INFO | Uploaded 36GiB to GPU in 2.35723s (15.2722GiB/s) | 20260508 02:11:54.840 | INFO | Downloaded 34GiB to GPU in 2.51639s (13.5114GiB/s) | 20260508 02:11:59.972 | INFO | Analyzing chunk frames = [4422144,4422656) | 20260508 02:11:59.972 | INFO | MemInfo: { | 20260508 02:11:59.972 | INFO | "gpu" : " 17 GB", | 20260508 02:11:59.972 | INFO | "huge" : "", | 20260508 02:11:59.972 | INFO | "huge_shared" : " 29 GB", | 20260508 02:11:59.972 | INFO | "malloc" : " 14 GB", | 20260508 02:11:59.972 | INFO | "pinned" : " 29 GB" | 20260508 02:11:59.972 | INFO | } | 20260508 02:11:59.972 | INFO | Perf report for the last 5.13258 seconds: | 20260508 02:11:59.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:11:59.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 02:11:59.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:11:59.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:11:59.972 | INFO | Uploaded 36GiB to GPU in 2.4317s (14.8045GiB/s) | 20260508 02:11:59.972 | INFO | Downloaded 34GiB to GPU in 2.42134s (14.0418GiB/s) | 20260508 02:12:05.107 | INFO | Analyzing chunk frames = [4422656,4423168) | 20260508 02:12:05.107 | INFO | MemInfo: { | 20260508 02:12:05.107 | INFO | "gpu" : " 17 GB", | 20260508 02:12:05.107 | INFO | "huge" : "", | 20260508 02:12:05.107 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:05.107 | INFO | "malloc" : " 14 GB", | 20260508 02:12:05.107 | INFO | "pinned" : " 29 GB" | 20260508 02:12:05.107 | INFO | } | 20260508 02:12:05.108 | INFO | Perf report for the last 5.13567 seconds: | 20260508 02:12:05.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:05.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 02:12:05.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 02:12:05.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:05.108 | INFO | Uploaded 36GiB to GPU in 2.45114s (14.6871GiB/s) | 20260508 02:12:05.108 | INFO | Downloaded 34GiB to GPU in 2.42201s (14.0379GiB/s) | 20260508 02:12:10.251 | INFO | Analyzing chunk frames = [4423168,4423680) | 20260508 02:12:10.251 | INFO | MemInfo: { | 20260508 02:12:10.251 | INFO | "gpu" : " 17 GB", | 20260508 02:12:10.251 | INFO | "huge" : "", | 20260508 02:12:10.251 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:10.251 | INFO | "malloc" : " 14 GB", | 20260508 02:12:10.251 | INFO | "pinned" : " 29 GB" | 20260508 02:12:10.251 | INFO | } | 20260508 02:12:10.251 | INFO | Perf report for the last 5.14386 seconds: | 20260508 02:12:10.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:10.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.252 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:12:10.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.252 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 02:12:10.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:10.252 | INFO | Uploaded 36GiB to GPU in 2.58066s (13.9499GiB/s) | 20260508 02:12:10.252 | INFO | Downloaded 34GiB to GPU in 2.46365s (13.8006GiB/s) | 20260508 02:12:10.252 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:12:10.252 | INFO | Total mem usage: 16.7 GiB | 20260508 02:12:10.252 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:12:10.252 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 02:12:10.252 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:12:10.252 | INFO | High Water Breakdown: | 20260508 02:12:10.252 | INFO | 60.2% : 8.98 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:12:10.252 | INFO | 39.7% : 5.92 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:12:10.252 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:12:10.252 | INFO | ------------------------------------------ | 20260508 02:12:10.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:12:10.252 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:12:10.252 | INFO | Total mem usage: 26.5 GiB | 20260508 02:12:10.252 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:12:10.252 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:12:10.252 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:12:10.252 | INFO | High Water Breakdown: | 20260508 02:12:10.252 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:12:10.252 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:12:10.252 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:12:10.252 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:12:10.252 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:12:10.252 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:12:10.252 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:12:10.252 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:12:10.252 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:12:10.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:12:10.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:12:10.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:12:10.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:12:10.253 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:12:10.253 | INFO | ------------------------------------------ | 20260508 02:12:10.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:12:10.253 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:12:10.253 | INFO | Total mem usage: 27.8 GiB | 20260508 02:12:10.253 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:12:10.253 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:12:10.253 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:12:10.253 | INFO | High Water Breakdown: | 20260508 02:12:10.253 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:12:10.253 | INFO | ------------------------------------------ | 20260508 02:12:10.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:12:10.253 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:12:10.253 | INFO | Total mem usage: 15.7 GiB | 20260508 02:12:10.253 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:12:10.253 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:12:10.253 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:12:10.253 | INFO | High Water Breakdown: | 20260508 02:12:10.253 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:12:10.253 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:12:10.253 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:12:10.253 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:12:10.253 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:12:10.253 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:12:10.253 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:12:10.253 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:12:10.253 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:12:10.253 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:12:10.253 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:12:10.253 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:12:10.253 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:12:10.253 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:12:10.253 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:12:10.253 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:12:10.253 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:12:10.253 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:12:10.253 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:12:10.253 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:12:10.253 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:12:10.253 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:12:10.253 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:12:10.253 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:12:10.253 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:12:10.253 | INFO | ------------------------------------------ | 20260508 02:12:10.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:12:10.253 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:12:10.253 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:12:10.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:12:10.253 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.64e+01, 1.42e+01 / 1.83e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.23e-02 / 6.04e-03 | 20260508 02:12:10.253 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.48e+00, 5.89e-01 / 1.85e-01 | 20260508 02:12:10.253 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.07e-01 / 1.29e-01 | 20260508 02:12:10.253 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.62e+01, 4.17e+00 / 2.21e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.61e+00 / 1.58e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.65e+00, 4.24e-01 / 2.50e-01 | 20260508 02:12:10.253 | INFO | PerfReport: 3.30%, Metrics -- 1.08e+00 / 1.22e+00, 2.67e-01 / 1.25e-01 | 20260508 02:12:10.253 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.33e-01 / 3.70e-01, 5.27e-02 / 2.91e-02 | 20260508 02:12:10.253 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:12:10.253 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:12:10.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:12:10.253 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 24.56%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 2.52%, LockDownload -- 2.68e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 15.44%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:12:10.253 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260508 02:12:10.364 | INFO | Starting a round of Baz IO | 20260508 02:12:13.818 | INFO | Populating RTMetrics report | 20260508 02:12:13.818 | INFO | RTMetrics report populated | 20260508 02:12:13.818 | INFO | Opening tmp RTMetrics json file | 20260508 02:12:13.843 | INFO | Serializing RTMetrics to json | 20260508 02:12:13.845 | INFO | Serialization done, swapping tmp file now... | 20260508 02:12:13.846 | INFO | RTMetrics done | 20260508 02:12:13.934 | INFO | Finished a round of Baz IO: Wrote 1.645 GiB in 3.569 seconds | 20260508 02:12:13.934 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.61% / 0.00% / 58.39% | 20260508 02:12:13.934 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.755942% / 3.97364e-06% / 4.76837e-05% | 20260508 02:12:13.934 | INFO | Baz IO overall: | 20260508 02:12:13.934 | INFO | BytesWritten : 1999219757866 | 20260508 02:12:13.934 | INFO | NumEvents : 966641273976 | 20260508 02:12:15.392 | INFO | Analyzing chunk frames = [4423680,4424192) | 20260508 02:12:15.392 | INFO | MemInfo: { | 20260508 02:12:15.392 | INFO | "gpu" : " 17 GB", | 20260508 02:12:15.393 | INFO | "huge" : "", | 20260508 02:12:15.393 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:15.393 | INFO | "malloc" : " 14 GB", | 20260508 02:12:15.393 | INFO | "pinned" : " 29 GB" | 20260508 02:12:15.393 | INFO | } | 20260508 02:12:15.393 | INFO | Perf report for the last 5.14132 seconds: | 20260508 02:12:15.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:15.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:15.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:15.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:15.393 | INFO | ANALYSIS : 68.7%, 2.9, 104.6ms, 0 | 20260508 02:12:15.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:15.393 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 02:12:15.393 | INFO | PRE_HQ : 67.1%, 1.8, 62.1ms, 0 | 20260508 02:12:15.393 | INFO | BAZWRITER : 69.0%, 3.7, 130.0ms, 0 | 20260508 02:12:15.393 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 02:12:15.393 | INFO | Uploaded 36GiB to GPU in 2.55794s (14.0738GiB/s) | 20260508 02:12:15.393 | INFO | Downloaded 38GiB to GPU in 2.69711s (14.0891GiB/s) | 20260508 02:12:20.535 | INFO | Analyzing chunk frames = [4424192,4424704) | 20260508 02:12:20.535 | INFO | MemInfo: { | 20260508 02:12:20.535 | INFO | "gpu" : " 17 GB", | 20260508 02:12:20.535 | INFO | "huge" : "", | 20260508 02:12:20.535 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:20.535 | INFO | "malloc" : " 14 GB", | 20260508 02:12:20.535 | INFO | "pinned" : " 29 GB" | 20260508 02:12:20.535 | INFO | } | 20260508 02:12:20.535 | INFO | Perf report for the last 5.14202 seconds: | 20260508 02:12:20.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:20.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 02:12:20.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:12:20.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:20.535 | INFO | Uploaded 36GiB to GPU in 2.55003s (14.1175GiB/s) | 20260508 02:12:20.535 | INFO | Downloaded 34GiB to GPU in 2.41986s (14.0504GiB/s) | 20260508 02:12:25.674 | INFO | Analyzing chunk frames = [4424704,4425216) | 20260508 02:12:25.674 | INFO | MemInfo: { | 20260508 02:12:25.674 | INFO | "gpu" : " 17 GB", | 20260508 02:12:25.674 | INFO | "huge" : "", | 20260508 02:12:25.674 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:25.674 | INFO | "malloc" : " 14 GB", | 20260508 02:12:25.674 | INFO | "pinned" : " 29 GB" | 20260508 02:12:25.674 | INFO | } | 20260508 02:12:25.674 | INFO | Perf report for the last 5.13898 seconds: | 20260508 02:12:25.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:25.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:12:25.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:12:25.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:25.674 | INFO | Uploaded 36GiB to GPU in 2.58302s (13.9372GiB/s) | 20260508 02:12:25.674 | INFO | Downloaded 34GiB to GPU in 2.48946s (13.6576GiB/s) | 20260508 02:12:30.810 | INFO | Analyzing chunk frames = [4425216,4425728) | 20260508 02:12:30.810 | INFO | MemInfo: { | 20260508 02:12:30.810 | INFO | "gpu" : " 17 GB", | 20260508 02:12:30.810 | INFO | "huge" : "", | 20260508 02:12:30.810 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:30.810 | INFO | "malloc" : " 14 GB", | 20260508 02:12:30.810 | INFO | "pinned" : " 29 GB" | 20260508 02:12:30.810 | INFO | } | 20260508 02:12:30.810 | INFO | Perf report for the last 5.13652 seconds: | 20260508 02:12:30.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:30.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 02:12:30.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:12:30.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:30.810 | INFO | Uploaded 36GiB to GPU in 2.58341s (13.9351GiB/s) | 20260508 02:12:30.811 | INFO | Downloaded 34GiB to GPU in 2.49318s (13.6372GiB/s) | 20260508 02:12:35.951 | INFO | Analyzing chunk frames = [4425728,4426240) | 20260508 02:12:35.951 | INFO | MemInfo: { | 20260508 02:12:35.951 | INFO | "gpu" : " 17 GB", | 20260508 02:12:35.952 | INFO | "huge" : "", | 20260508 02:12:35.952 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:35.952 | INFO | "malloc" : " 14 GB", | 20260508 02:12:35.952 | INFO | "pinned" : " 29 GB" | 20260508 02:12:35.952 | INFO | } | 20260508 02:12:35.952 | INFO | Perf report for the last 5.14131 seconds: | 20260508 02:12:35.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:35.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 02:12:35.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:12:35.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:35.952 | INFO | Uploaded 36GiB to GPU in 2.42392s (14.852GiB/s) | 20260508 02:12:35.952 | INFO | Downloaded 34GiB to GPU in 2.58939s (13.1305GiB/s) | 20260508 02:12:41.089 | INFO | Analyzing chunk frames = [4426240,4426752) | 20260508 02:12:41.089 | INFO | MemInfo: { | 20260508 02:12:41.089 | INFO | "gpu" : " 17 GB", | 20260508 02:12:41.089 | INFO | "huge" : "", | 20260508 02:12:41.089 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:41.089 | INFO | "malloc" : " 14 GB", | 20260508 02:12:41.089 | INFO | "pinned" : " 29 GB" | 20260508 02:12:41.089 | INFO | } | 20260508 02:12:41.089 | INFO | Perf report for the last 5.13753 seconds: | 20260508 02:12:41.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:41.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:12:41.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 02:12:41.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:41.089 | INFO | Uploaded 36GiB to GPU in 2.41776s (14.8898GiB/s) | 20260508 02:12:41.089 | INFO | Downloaded 34GiB to GPU in 2.41674s (14.0685GiB/s) | 20260508 02:12:46.229 | INFO | Analyzing chunk frames = [4426752,4427264) | 20260508 02:12:46.229 | INFO | MemInfo: { | 20260508 02:12:46.229 | INFO | "gpu" : " 17 GB", | 20260508 02:12:46.229 | INFO | "huge" : "", | 20260508 02:12:46.229 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:46.229 | INFO | "malloc" : " 14 GB", | 20260508 02:12:46.230 | INFO | "pinned" : " 29 GB" | 20260508 02:12:46.230 | INFO | } | 20260508 02:12:46.230 | INFO | Perf report for the last 5.14047 seconds: | 20260508 02:12:46.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:46.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.230 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:12:46.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.230 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:12:46.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:46.231 | INFO | Uploaded 36GiB to GPU in 2.44018s (14.753GiB/s) | 20260508 02:12:46.231 | INFO | Downloaded 34GiB to GPU in 2.42666s (14.011GiB/s) | 20260508 02:12:51.365 | INFO | Analyzing chunk frames = [4427264,4427776) | 20260508 02:12:51.365 | INFO | MemInfo: { | 20260508 02:12:51.365 | INFO | "gpu" : " 17 GB", | 20260508 02:12:51.365 | INFO | "huge" : "", | 20260508 02:12:51.365 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:51.365 | INFO | "malloc" : " 14 GB", | 20260508 02:12:51.365 | INFO | "pinned" : " 29 GB" | 20260508 02:12:51.365 | INFO | } | 20260508 02:12:51.365 | INFO | Perf report for the last 5.13569 seconds: | 20260508 02:12:51.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:51.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.365 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260508 02:12:51.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.366 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 02:12:51.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:51.366 | INFO | Uploaded 36GiB to GPU in 2.56046s (14.06GiB/s) | 20260508 02:12:51.366 | INFO | Downloaded 34GiB to GPU in 2.45971s (13.8228GiB/s) | 20260508 02:12:51.478 | INFO | Starting a round of Baz IO | 20260508 02:12:54.914 | INFO | Populating RTMetrics report | 20260508 02:12:54.914 | INFO | RTMetrics report populated | 20260508 02:12:54.914 | INFO | Opening tmp RTMetrics json file | 20260508 02:12:54.930 | INFO | Serializing RTMetrics to json | 20260508 02:12:54.937 | INFO | Serialization done, swapping tmp file now... | 20260508 02:12:54.938 | INFO | RTMetrics done | 20260508 02:12:55.016 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.537 seconds | 20260508 02:12:55.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.61% / 0.00% / 58.39% | 20260508 02:12:55.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.75347% / 3.97364e-06% / 7.54992e-05% | 20260508 02:12:55.016 | INFO | Baz IO overall: | 20260508 02:12:55.016 | INFO | BytesWritten : 2000984455847 | 20260508 02:12:55.016 | INFO | NumEvents : 967487407039 | 20260508 02:12:56.505 | INFO | Analyzing chunk frames = [4427776,4428288) | 20260508 02:12:56.505 | INFO | MemInfo: { | 20260508 02:12:56.505 | INFO | "gpu" : " 17 GB", | 20260508 02:12:56.505 | INFO | "huge" : "", | 20260508 02:12:56.505 | INFO | "huge_shared" : " 29 GB", | 20260508 02:12:56.506 | INFO | "malloc" : " 14 GB", | 20260508 02:12:56.506 | INFO | "pinned" : " 29 GB" | 20260508 02:12:56.506 | INFO | } | 20260508 02:12:56.506 | INFO | Perf report for the last 5.1403 seconds: | 20260508 02:12:56.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:12:56.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:56.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:56.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:56.506 | INFO | ANALYSIS : 68.3%, 2.9, 104.3ms, 0 | 20260508 02:12:56.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:12:56.506 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260508 02:12:56.506 | INFO | PRE_HQ : 67.0%, 1.8, 62.5ms, 0 | 20260508 02:12:56.506 | INFO | BAZWRITER : 68.4%, 4.2, 148.5ms, 0 | 20260508 02:12:56.506 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 02:12:56.506 | INFO | Uploaded 36GiB to GPU in 2.58095s (13.9484GiB/s) | 20260508 02:12:56.506 | INFO | Downloaded 38GiB to GPU in 2.69597s (14.0951GiB/s) | 20260508 02:13:01.649 | INFO | Analyzing chunk frames = [4428288,4428800) | 20260508 02:13:01.649 | INFO | MemInfo: { | 20260508 02:13:01.649 | INFO | "gpu" : " 17 GB", | 20260508 02:13:01.649 | INFO | "huge" : "", | 20260508 02:13:01.649 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:01.649 | INFO | "malloc" : " 14 GB", | 20260508 02:13:01.649 | INFO | "pinned" : " 29 GB" | 20260508 02:13:01.649 | INFO | } | 20260508 02:13:01.649 | INFO | Perf report for the last 5.14334 seconds: | 20260508 02:13:01.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:01.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.649 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:13:01.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.649 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:13:01.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:01.650 | INFO | Uploaded 36GiB to GPU in 2.50513s (14.3705GiB/s) | 20260508 02:13:01.650 | INFO | Downloaded 34GiB to GPU in 2.44158s (13.9254GiB/s) | 20260508 02:13:06.786 | INFO | Analyzing chunk frames = [4428800,4429312) | 20260508 02:13:06.786 | INFO | MemInfo: { | 20260508 02:13:06.786 | INFO | "gpu" : " 17 GB", | 20260508 02:13:06.786 | INFO | "huge" : "", | 20260508 02:13:06.786 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:06.786 | INFO | "malloc" : " 14 GB", | 20260508 02:13:06.786 | INFO | "pinned" : " 29 GB" | 20260508 02:13:06.786 | INFO | } | 20260508 02:13:06.786 | INFO | Perf report for the last 5.13716 seconds: | 20260508 02:13:06.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:06.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 02:13:06.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:13:06.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:06.786 | INFO | Uploaded 36GiB to GPU in 2.49038s (14.4556GiB/s) | 20260508 02:13:06.786 | INFO | Downloaded 34GiB to GPU in 2.51232s (13.5333GiB/s) | 20260508 02:13:11.924 | INFO | Analyzing chunk frames = [4429312,4429824) | 20260508 02:13:11.924 | INFO | MemInfo: { | 20260508 02:13:11.924 | INFO | "gpu" : " 17 GB", | 20260508 02:13:11.924 | INFO | "huge" : "", | 20260508 02:13:11.924 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:11.924 | INFO | "malloc" : " 14 GB", | 20260508 02:13:11.924 | INFO | "pinned" : " 29 GB" | 20260508 02:13:11.924 | INFO | } | 20260508 02:13:11.924 | INFO | Perf report for the last 5.13766 seconds: | 20260508 02:13:11.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:11.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 02:13:11.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:13:11.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:11.924 | INFO | Uploaded 36GiB to GPU in 2.4802s (14.515GiB/s) | 20260508 02:13:11.925 | INFO | Downloaded 34GiB to GPU in 2.51108s (13.54GiB/s) | 20260508 02:13:17.064 | INFO | Analyzing chunk frames = [4429824,4430336) | 20260508 02:13:17.064 | INFO | MemInfo: { | 20260508 02:13:17.064 | INFO | "gpu" : " 17 GB", | 20260508 02:13:17.064 | INFO | "huge" : "", | 20260508 02:13:17.064 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:17.064 | INFO | "malloc" : " 14 GB", | 20260508 02:13:17.065 | INFO | "pinned" : " 29 GB" | 20260508 02:13:17.065 | INFO | } | 20260508 02:13:17.065 | INFO | Perf report for the last 5.14069 seconds: | 20260508 02:13:17.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:17.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 02:13:17.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 02:13:17.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:17.065 | INFO | Uploaded 36GiB to GPU in 2.27891s (15.797GiB/s) | 20260508 02:13:17.065 | INFO | Downloaded 34GiB to GPU in 2.51692s (13.5086GiB/s) | 20260508 02:13:22.204 | INFO | Analyzing chunk frames = [4430336,4430848) | 20260508 02:13:22.204 | INFO | MemInfo: { | 20260508 02:13:22.204 | INFO | "gpu" : " 17 GB", | 20260508 02:13:22.204 | INFO | "huge" : "", | 20260508 02:13:22.204 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:22.204 | INFO | "malloc" : " 14 GB", | 20260508 02:13:22.204 | INFO | "pinned" : " 29 GB" | 20260508 02:13:22.204 | INFO | } | 20260508 02:13:22.204 | INFO | Perf report for the last 5.13946 seconds: | 20260508 02:13:22.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:22.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:13:22.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:13:22.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:22.204 | INFO | Uploaded 36GiB to GPU in 2.4016s (14.99GiB/s) | 20260508 02:13:22.204 | INFO | Downloaded 34GiB to GPU in 2.42676s (14.0105GiB/s) | 20260508 02:13:27.341 | INFO | Analyzing chunk frames = [4430848,4431360) | 20260508 02:13:27.341 | INFO | MemInfo: { | 20260508 02:13:27.341 | INFO | "gpu" : " 17 GB", | 20260508 02:13:27.342 | INFO | "huge" : "", | 20260508 02:13:27.342 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:27.342 | INFO | "malloc" : " 14 GB", | 20260508 02:13:27.342 | INFO | "pinned" : " 29 GB" | 20260508 02:13:27.342 | INFO | } | 20260508 02:13:27.342 | INFO | Perf report for the last 5.13768 seconds: | 20260508 02:13:27.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:27.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.342 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:13:27.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.342 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:13:27.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:27.343 | INFO | Uploaded 36GiB to GPU in 2.42885s (14.8218GiB/s) | 20260508 02:13:27.343 | INFO | Downloaded 34GiB to GPU in 2.42809s (14.0028GiB/s) | 20260508 02:13:32.480 | INFO | Analyzing chunk frames = [4431360,4431872) | 20260508 02:13:32.480 | INFO | MemInfo: { | 20260508 02:13:32.480 | INFO | "gpu" : " 17 GB", | 20260508 02:13:32.480 | INFO | "huge" : "", | 20260508 02:13:32.480 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:32.480 | INFO | "malloc" : " 14 GB", | 20260508 02:13:32.480 | INFO | "pinned" : " 29 GB" | 20260508 02:13:32.480 | INFO | } | 20260508 02:13:32.480 | INFO | Perf report for the last 5.13863 seconds: | 20260508 02:13:32.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:32.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.480 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:13:32.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.481 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 02:13:32.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:32.481 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:13:32.481 | INFO | Total mem usage: 16.7 GiB | 20260508 02:13:32.481 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:13:32.481 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.1%) | 20260508 02:13:32.481 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:13:32.481 | INFO | High Water Breakdown: | 20260508 02:13:32.481 | INFO | 60.3% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:13:32.481 | INFO | 39.6% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:13:32.481 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:13:32.481 | INFO | ------------------------------------------ | 20260508 02:13:32.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:13:32.481 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:13:32.481 | INFO | Total mem usage: 26.5 GiB | 20260508 02:13:32.481 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:13:32.481 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:13:32.481 | INFO | Uploaded 36GiB to GPU in 2.55027s (14.1161GiB/s) | 20260508 02:13:32.481 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:13:32.481 | INFO | High Water Breakdown: | 20260508 02:13:32.481 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:13:32.481 | INFO | Downloaded 34GiB to GPU in 2.46619s (13.7864GiB/s) | 20260508 02:13:32.481 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:13:32.481 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:13:32.481 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:13:32.481 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:13:32.481 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:13:32.481 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:13:32.481 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:13:32.481 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:13:32.481 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:13:32.481 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:13:32.481 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:13:32.481 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:13:32.481 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:13:32.481 | INFO | ------------------------------------------ | 20260508 02:13:32.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:13:32.481 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:13:32.481 | INFO | Total mem usage: 27.8 GiB | 20260508 02:13:32.481 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:13:32.481 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:13:32.481 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:13:32.481 | INFO | High Water Breakdown: | 20260508 02:13:32.481 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:13:32.481 | INFO | ------------------------------------------ | 20260508 02:13:32.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:13:32.482 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:13:32.482 | INFO | Total mem usage: 15.7 GiB | 20260508 02:13:32.482 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:13:32.482 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:13:32.482 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:13:32.482 | INFO | High Water Breakdown: | 20260508 02:13:32.482 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:13:32.482 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:13:32.482 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:13:32.482 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:13:32.482 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:13:32.482 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:13:32.482 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:13:32.482 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:13:32.482 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:13:32.482 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:13:32.482 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:13:32.482 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:13:32.482 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:13:32.482 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:13:32.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:13:32.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:13:32.482 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:13:32.482 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:13:32.482 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:13:32.482 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:13:32.482 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:13:32.482 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:13:32.482 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:13:32.482 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:13:32.482 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:13:32.482 | INFO | ------------------------------------------ | 20260508 02:13:32.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:13:32.482 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:13:32.482 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:13:32.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:13:32.482 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.40e+01 / 1.58e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.02e-01, 1.23e-02 / 5.05e-03 | 20260508 02:13:32.482 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.89e-01 / 2.04e-01 | 20260508 02:13:32.482 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.13e-01 | 20260508 02:13:32.482 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.62e+01, 4.10e+00 / 1.87e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.64e+00 / 1.39e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.43e+00 / 2.66e+00, 4.25e-01 / 2.39e-01 | 20260508 02:13:32.482 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.23e+00, 2.65e-01 / 1.35e-01 | 20260508 02:13:32.482 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.63e-01, 5.32e-02 / 2.40e-02 | 20260508 02:13:32.482 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:13:32.482 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:13:32.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:13:32.482 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 24.49%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 2.13%, LockUpload -- 2.27e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 15.38%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:13:32.482 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 02:13:32.595 | INFO | Starting a round of Baz IO | 20260508 02:13:36.016 | INFO | Populating RTMetrics report | 20260508 02:13:36.017 | INFO | RTMetrics report populated | 20260508 02:13:36.017 | INFO | Opening tmp RTMetrics json file | 20260508 02:13:36.053 | INFO | Serializing RTMetrics to json | 20260508 02:13:36.125 | INFO | Serialization done, swapping tmp file now... | 20260508 02:13:36.129 | INFO | RTMetrics done | 20260508 02:13:36.162 | INFO | Finished a round of Baz IO: Wrote 1.646 GiB in 3.567 seconds | 20260508 02:13:36.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.60% / 0.00% / 58.39% | 20260508 02:13:36.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.755123% / 3.97364e-06% / 5.16574e-05% | 20260508 02:13:36.162 | INFO | Baz IO overall: | 20260508 02:13:36.163 | INFO | BytesWritten : 2002751606808 | 20260508 02:13:36.163 | INFO | NumEvents : 968335148953 | 20260508 02:13:37.619 | INFO | Analyzing chunk frames = [4431872,4432384) | 20260508 02:13:37.619 | INFO | MemInfo: { | 20260508 02:13:37.619 | INFO | "gpu" : " 17 GB", | 20260508 02:13:37.619 | INFO | "huge" : "", | 20260508 02:13:37.619 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:37.619 | INFO | "malloc" : " 14 GB", | 20260508 02:13:37.619 | INFO | "pinned" : " 29 GB" | 20260508 02:13:37.619 | INFO | } | 20260508 02:13:37.619 | INFO | Perf report for the last 5.13897 seconds: | 20260508 02:13:37.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:37.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:37.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:37.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:37.620 | INFO | ANALYSIS : 68.1%, 2.9, 105.1ms, 0 | 20260508 02:13:37.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:37.620 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 02:13:37.620 | INFO | PRE_HQ : 66.2%, 1.7, 59.5ms, 0 | 20260508 02:13:37.620 | INFO | BAZWRITER : 69.0%, 3.6, 126.7ms, 0 | 20260508 02:13:37.620 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260508 02:13:37.620 | INFO | Uploaded 36GiB to GPU in 2.57797s (13.9645GiB/s) | 20260508 02:13:37.620 | INFO | Downloaded 38GiB to GPU in 2.70401s (14.0532GiB/s) | 20260508 02:13:42.762 | INFO | Analyzing chunk frames = [4432384,4432896) | 20260508 02:13:42.762 | INFO | MemInfo: { | 20260508 02:13:42.762 | INFO | "gpu" : " 17 GB", | 20260508 02:13:42.762 | INFO | "huge" : "", | 20260508 02:13:42.762 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:42.762 | INFO | "malloc" : " 14 GB", | 20260508 02:13:42.762 | INFO | "pinned" : " 29 GB" | 20260508 02:13:42.762 | INFO | } | 20260508 02:13:42.763 | INFO | Perf report for the last 5.14319 seconds: | 20260508 02:13:42.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:42.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:13:42.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:13:42.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:42.763 | INFO | Uploaded 36GiB to GPU in 2.53681s (14.1911GiB/s) | 20260508 02:13:42.763 | INFO | Downloaded 34GiB to GPU in 2.42775s (14.0047GiB/s) | 20260508 02:13:47.906 | INFO | Analyzing chunk frames = [4432896,4433408) | 20260508 02:13:47.906 | INFO | MemInfo: { | 20260508 02:13:47.906 | INFO | "gpu" : " 17 GB", | 20260508 02:13:47.906 | INFO | "huge" : "", | 20260508 02:13:47.906 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:47.906 | INFO | "malloc" : " 14 GB", | 20260508 02:13:47.906 | INFO | "pinned" : " 29 GB" | 20260508 02:13:47.906 | INFO | } | 20260508 02:13:47.906 | INFO | Perf report for the last 5.14382 seconds: | 20260508 02:13:47.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:47.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.906 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:13:47.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.906 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:13:47.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:47.907 | INFO | Uploaded 36GiB to GPU in 2.56422s (14.0394GiB/s) | 20260508 02:13:47.907 | INFO | Downloaded 34GiB to GPU in 2.50559s (13.5697GiB/s) | 20260508 02:13:53.044 | INFO | Analyzing chunk frames = [4433408,4433920) | 20260508 02:13:53.044 | INFO | MemInfo: { | 20260508 02:13:53.044 | INFO | "gpu" : " 17 GB", | 20260508 02:13:53.044 | INFO | "huge" : "", | 20260508 02:13:53.044 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:53.044 | INFO | "malloc" : " 14 GB", | 20260508 02:13:53.044 | INFO | "pinned" : " 29 GB" | 20260508 02:13:53.044 | INFO | } | 20260508 02:13:53.044 | INFO | Perf report for the last 5.13752 seconds: | 20260508 02:13:53.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:53.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:13:53.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:13:53.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:53.044 | INFO | Uploaded 36GiB to GPU in 2.55416s (14.0947GiB/s) | 20260508 02:13:53.044 | INFO | Downloaded 34GiB to GPU in 2.50347s (13.5811GiB/s) | 20260508 02:13:58.174 | INFO | Analyzing chunk frames = [4433920,4434432) | 20260508 02:13:58.174 | INFO | MemInfo: { | 20260508 02:13:58.174 | INFO | "gpu" : " 17 GB", | 20260508 02:13:58.174 | INFO | "huge" : "", | 20260508 02:13:58.174 | INFO | "huge_shared" : " 29 GB", | 20260508 02:13:58.174 | INFO | "malloc" : " 14 GB", | 20260508 02:13:58.174 | INFO | "pinned" : " 29 GB" | 20260508 02:13:58.174 | INFO | } | 20260508 02:13:58.174 | INFO | Perf report for the last 5.13008 seconds: | 20260508 02:13:58.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:13:58.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 02:13:58.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:13:58.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:13:58.174 | INFO | Uploaded 36GiB to GPU in 2.38432s (15.0986GiB/s) | 20260508 02:13:58.174 | INFO | Downloaded 34GiB to GPU in 2.54836s (13.3419GiB/s) | 20260508 02:14:03.314 | INFO | Analyzing chunk frames = [4434432,4434944) | 20260508 02:14:03.314 | INFO | MemInfo: { | 20260508 02:14:03.314 | INFO | "gpu" : " 17 GB", | 20260508 02:14:03.314 | INFO | "huge" : "", | 20260508 02:14:03.315 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:03.315 | INFO | "malloc" : " 14 GB", | 20260508 02:14:03.315 | INFO | "pinned" : " 29 GB" | 20260508 02:14:03.315 | INFO | } | 20260508 02:14:03.315 | INFO | Perf report for the last 5.14058 seconds: | 20260508 02:14:03.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:03.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 02:14:03.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:14:03.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:03.315 | INFO | Uploaded 36GiB to GPU in 2.40265s (14.9834GiB/s) | 20260508 02:14:03.315 | INFO | Downloaded 34GiB to GPU in 2.42143s (14.0413GiB/s) | 20260508 02:14:08.453 | INFO | Analyzing chunk frames = [4434944,4435456) | 20260508 02:14:08.453 | INFO | MemInfo: { | 20260508 02:14:08.453 | INFO | "gpu" : " 17 GB", | 20260508 02:14:08.453 | INFO | "huge" : "", | 20260508 02:14:08.453 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:08.453 | INFO | "malloc" : " 14 GB", | 20260508 02:14:08.453 | INFO | "pinned" : " 29 GB" | 20260508 02:14:08.453 | INFO | } | 20260508 02:14:08.453 | INFO | Perf report for the last 5.1383 seconds: | 20260508 02:14:08.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:08.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.453 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 02:14:08.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.453 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 02:14:08.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:08.454 | INFO | Uploaded 36GiB to GPU in 2.51526s (14.3127GiB/s) | 20260508 02:14:08.454 | INFO | Downloaded 34GiB to GPU in 2.45511s (13.8487GiB/s) | 20260508 02:14:13.600 | INFO | Analyzing chunk frames = [4435456,4435968) | 20260508 02:14:13.600 | INFO | MemInfo: { | 20260508 02:14:13.600 | INFO | "gpu" : " 17 GB", | 20260508 02:14:13.600 | INFO | "huge" : "", | 20260508 02:14:13.600 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:13.600 | INFO | "malloc" : " 14 GB", | 20260508 02:14:13.600 | INFO | "pinned" : " 29 GB" | 20260508 02:14:13.600 | INFO | } | 20260508 02:14:13.600 | INFO | Perf report for the last 5.14693 seconds: | 20260508 02:14:13.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:13.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 02:14:13.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 02:14:13.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:13.600 | INFO | Uploaded 36GiB to GPU in 2.54228s (14.1605GiB/s) | 20260508 02:14:13.600 | INFO | Downloaded 34GiB to GPU in 2.46055s (13.818GiB/s) | 20260508 02:14:13.713 | INFO | Starting a round of Baz IO | 20260508 02:14:17.164 | INFO | Populating RTMetrics report | 20260508 02:14:17.165 | INFO | RTMetrics report populated | 20260508 02:14:17.165 | INFO | Opening tmp RTMetrics json file | 20260508 02:14:17.179 | INFO | Serializing RTMetrics to json | 20260508 02:14:17.181 | INFO | Serialization done, swapping tmp file now... | 20260508 02:14:17.181 | INFO | RTMetrics done | 20260508 02:14:17.253 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.54 seconds | 20260508 02:14:17.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.60% / 0.00% / 58.40% | 20260508 02:14:17.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.751698% / 0% / 8.34465e-05% | 20260508 02:14:17.253 | INFO | Baz IO overall: | 20260508 02:14:17.253 | INFO | BytesWritten : 2004516134714 | 20260508 02:14:17.253 | INFO | NumEvents : 969181168225 | 20260508 02:14:18.736 | INFO | Analyzing chunk frames = [4435968,4436480) | 20260508 02:14:18.736 | INFO | MemInfo: { | 20260508 02:14:18.736 | INFO | "gpu" : " 17 GB", | 20260508 02:14:18.736 | INFO | "huge" : "", | 20260508 02:14:18.736 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:18.736 | INFO | "malloc" : " 14 GB", | 20260508 02:14:18.736 | INFO | "pinned" : " 29 GB" | 20260508 02:14:18.736 | INFO | } | 20260508 02:14:18.736 | INFO | Perf report for the last 5.13658 seconds: | 20260508 02:14:18.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:18.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:18.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:18.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:18.737 | INFO | ANALYSIS : 68.5%, 2.9, 104.5ms, 0 | 20260508 02:14:18.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:18.737 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 02:14:18.737 | INFO | PRE_HQ : 67.1%, 1.8, 63.1ms, 0 | 20260508 02:14:18.737 | INFO | BAZWRITER : 68.5%, 3.9, 136.8ms, 0 | 20260508 02:14:18.737 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:14:18.737 | INFO | Uploaded 36GiB to GPU in 2.60346s (13.8278GiB/s) | 20260508 02:14:18.737 | INFO | Downloaded 38GiB to GPU in 2.70013s (14.0734GiB/s) | 20260508 02:14:23.869 | INFO | Analyzing chunk frames = [4436480,4436992) | 20260508 02:14:23.869 | INFO | MemInfo: { | 20260508 02:14:23.869 | INFO | "gpu" : " 17 GB", | 20260508 02:14:23.869 | INFO | "huge" : "", | 20260508 02:14:23.869 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:23.869 | INFO | "malloc" : " 14 GB", | 20260508 02:14:23.869 | INFO | "pinned" : " 29 GB" | 20260508 02:14:23.869 | INFO | } | 20260508 02:14:23.869 | INFO | Perf report for the last 5.13247 seconds: | 20260508 02:14:23.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:23.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:14:23.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:14:23.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:23.869 | INFO | Uploaded 36GiB to GPU in 2.53297s (14.2126GiB/s) | 20260508 02:14:23.869 | INFO | Downloaded 34GiB to GPU in 2.42842s (14.0009GiB/s) | 20260508 02:14:29.015 | INFO | Analyzing chunk frames = [4436992,4437504) | 20260508 02:14:29.015 | INFO | MemInfo: { | 20260508 02:14:29.015 | INFO | "gpu" : " 17 GB", | 20260508 02:14:29.015 | INFO | "huge" : "", | 20260508 02:14:29.015 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:29.015 | INFO | "malloc" : " 14 GB", | 20260508 02:14:29.015 | INFO | "pinned" : " 29 GB" | 20260508 02:14:29.015 | INFO | } | 20260508 02:14:29.015 | INFO | Perf report for the last 5.14612 seconds: | 20260508 02:14:29.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:29.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 02:14:29.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:14:29.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:29.015 | INFO | Uploaded 36GiB to GPU in 2.53113s (14.2229GiB/s) | 20260508 02:14:29.017 | INFO | Downloaded 34GiB to GPU in 2.49075s (13.6505GiB/s) | 20260508 02:14:34.154 | INFO | Analyzing chunk frames = [4437504,4438016) | 20260508 02:14:34.154 | INFO | MemInfo: { | 20260508 02:14:34.154 | INFO | "gpu" : " 17 GB", | 20260508 02:14:34.154 | INFO | "huge" : "", | 20260508 02:14:34.154 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:34.154 | INFO | "malloc" : " 14 GB", | 20260508 02:14:34.154 | INFO | "pinned" : " 29 GB" | 20260508 02:14:34.154 | INFO | } | 20260508 02:14:34.154 | INFO | Perf report for the last 5.13917 seconds: | 20260508 02:14:34.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:34.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.154 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:14:34.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.154 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:14:34.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:34.155 | INFO | Uploaded 36GiB to GPU in 2.53002s (14.2291GiB/s) | 20260508 02:14:34.155 | INFO | Downloaded 34GiB to GPU in 2.48913s (13.6594GiB/s) | 20260508 02:14:39.290 | INFO | Analyzing chunk frames = [4438016,4438528) | 20260508 02:14:39.290 | INFO | MemInfo: { | 20260508 02:14:39.290 | INFO | "gpu" : " 17 GB", | 20260508 02:14:39.290 | INFO | "huge" : "", | 20260508 02:14:39.290 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:39.290 | INFO | "malloc" : " 14 GB", | 20260508 02:14:39.290 | INFO | "pinned" : " 29 GB" | 20260508 02:14:39.290 | INFO | } | 20260508 02:14:39.290 | INFO | Perf report for the last 5.13589 seconds: | 20260508 02:14:39.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:39.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 02:14:39.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:14:39.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:39.290 | INFO | Uploaded 36GiB to GPU in 2.28834s (15.732GiB/s) | 20260508 02:14:39.290 | INFO | Downloaded 34GiB to GPU in 2.5064s (13.5653GiB/s) | 20260508 02:14:44.431 | INFO | Analyzing chunk frames = [4438528,4439040) | 20260508 02:14:44.431 | INFO | MemInfo: { | 20260508 02:14:44.431 | INFO | "gpu" : " 17 GB", | 20260508 02:14:44.431 | INFO | "huge" : "", | 20260508 02:14:44.431 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:44.431 | INFO | "malloc" : " 14 GB", | 20260508 02:14:44.431 | INFO | "pinned" : " 29 GB" | 20260508 02:14:44.431 | INFO | } | 20260508 02:14:44.431 | INFO | Perf report for the last 5.14051 seconds: | 20260508 02:14:44.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:44.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:14:44.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 02:14:44.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:44.431 | INFO | Uploaded 36GiB to GPU in 2.40706s (14.956GiB/s) | 20260508 02:14:44.431 | INFO | Downloaded 34GiB to GPU in 2.4217s (14.0397GiB/s) | 20260508 02:14:49.572 | INFO | Analyzing chunk frames = [4439040,4439552) | 20260508 02:14:49.572 | INFO | MemInfo: { | 20260508 02:14:49.572 | INFO | "gpu" : " 17 GB", | 20260508 02:14:49.572 | INFO | "huge" : "", | 20260508 02:14:49.572 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:49.572 | INFO | "malloc" : " 14 GB", | 20260508 02:14:49.572 | INFO | "pinned" : " 29 GB" | 20260508 02:14:49.572 | INFO | } | 20260508 02:14:49.572 | INFO | Perf report for the last 5.14177 seconds: | 20260508 02:14:49.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:49.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 02:14:49.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 02:14:49.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:49.573 | INFO | Uploaded 36GiB to GPU in 2.44042s (14.7516GiB/s) | 20260508 02:14:49.573 | INFO | Downloaded 34GiB to GPU in 2.42079s (14.045GiB/s) | 20260508 02:14:54.712 | INFO | Analyzing chunk frames = [4439552,4440064) | 20260508 02:14:54.712 | INFO | MemInfo: { | 20260508 02:14:54.712 | INFO | "gpu" : " 17 GB", | 20260508 02:14:54.712 | INFO | "huge" : "", | 20260508 02:14:54.712 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:54.712 | INFO | "malloc" : " 14 GB", | 20260508 02:14:54.713 | INFO | "pinned" : " 29 GB" | 20260508 02:14:54.713 | INFO | } | 20260508 02:14:54.713 | INFO | Perf report for the last 5.14007 seconds: | 20260508 02:14:54.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:54.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 02:14:54.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | PRE_HQ : 35.6%, 1.0, 18.5ms, 0 | 20260508 02:14:54.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:54.713 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:14:54.713 | INFO | Total mem usage: 16.7 GiB | 20260508 02:14:54.713 | INFO | Uploaded 36GiB to GPU in 2.47221s (14.5619GiB/s) | 20260508 02:14:54.713 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:14:54.713 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.2%) | 20260508 02:14:54.713 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:14:54.713 | INFO | High Water Breakdown: | 20260508 02:14:54.713 | INFO | 59.8% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:14:54.713 | INFO | Downloaded 34GiB to GPU in 2.43807s (13.9455GiB/s) | 20260508 02:14:54.713 | INFO | 40% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:14:54.713 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:14:54.713 | INFO | ------------------------------------------ | 20260508 02:14:54.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:14:54.713 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:14:54.713 | INFO | Total mem usage: 26.5 GiB | 20260508 02:14:54.713 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:14:54.713 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:14:54.713 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 02:14:54.713 | INFO | High Water Breakdown: | 20260508 02:14:54.713 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:14:54.713 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:14:54.713 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:14:54.713 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:14:54.713 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:14:54.713 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:14:54.713 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:14:54.713 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:14:54.713 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:14:54.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:14:54.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:14:54.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:14:54.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:14:54.714 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:14:54.714 | INFO | ------------------------------------------ | 20260508 02:14:54.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:14:54.714 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:14:54.714 | INFO | Total mem usage: 27.8 GiB | 20260508 02:14:54.714 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:14:54.714 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:14:54.714 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:14:54.714 | INFO | High Water Breakdown: | 20260508 02:14:54.714 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:14:54.714 | INFO | ------------------------------------------ | 20260508 02:14:54.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:14:54.714 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:14:54.714 | INFO | Total mem usage: 15.7 GiB | 20260508 02:14:54.714 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:14:54.714 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:14:54.714 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:14:54.714 | INFO | High Water Breakdown: | 20260508 02:14:54.714 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:14:54.714 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:14:54.714 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:14:54.714 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:14:54.714 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:14:54.714 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:14:54.714 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:14:54.714 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:14:54.714 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:14:54.714 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:14:54.714 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:14:54.714 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:14:54.714 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:14:54.714 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:14:54.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:14:54.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:14:54.714 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:14:54.714 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:14:54.714 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:14:54.714 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:14:54.714 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:14:54.714 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:14:54.714 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:14:54.714 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:14:54.714 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:14:54.714 | INFO | ------------------------------------------ | 20260508 02:14:54.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:14:54.714 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:14:54.714 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:14:54.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:14:54.714 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.42e+01 / 1.55e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.02e-01, 1.17e-02 / 5.68e-03 | 20260508 02:14:54.714 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.46e+00, 5.78e-01 / 1.96e-01 | 20260508 02:14:54.714 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.08e-01 / 1.08e-01 | 20260508 02:14:54.714 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.60e+01, 4.11e+00 / 1.85e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.61e+00 / 1.53e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.43e+00 / 2.67e+00, 4.23e-01 / 2.69e-01 | 20260508 02:14:54.714 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.23e+00, 2.69e-01 / 1.20e-01 | 20260508 02:14:54.714 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.68e-01, 5.21e-02 / 2.80e-02 | 20260508 02:14:54.714 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:14:54.714 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:14:54.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:14:54.714 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 24.42%, Upload -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 2.18%, LockUpload -- 2.32e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 2.76%, LockDownload -- 2.93e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 15.45%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:14:54.714 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 6.35e-02 / 0.00e+00 | 20260508 02:14:54.824 | INFO | Starting a round of Baz IO | 20260508 02:14:58.283 | INFO | Populating RTMetrics report | 20260508 02:14:58.283 | INFO | RTMetrics report populated | 20260508 02:14:58.283 | INFO | Opening tmp RTMetrics json file | 20260508 02:14:58.289 | INFO | Serializing RTMetrics to json | 20260508 02:14:58.295 | INFO | Serialization done, swapping tmp file now... | 20260508 02:14:58.297 | INFO | RTMetrics done | 20260508 02:14:58.476 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.651 seconds | 20260508 02:14:58.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.60% / 0.00% / 58.40% | 20260508 02:14:58.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.751626% / 0% / 6.35783e-05% | 20260508 02:14:58.476 | INFO | Baz IO overall: | 20260508 02:14:58.476 | INFO | BytesWritten : 2006281599090 | 20260508 02:14:58.476 | INFO | NumEvents : 970027756194 | 20260508 02:14:59.855 | INFO | Analyzing chunk frames = [4440064,4440576) | 20260508 02:14:59.855 | INFO | MemInfo: { | 20260508 02:14:59.855 | INFO | "gpu" : " 17 GB", | 20260508 02:14:59.855 | INFO | "huge" : "", | 20260508 02:14:59.855 | INFO | "huge_shared" : " 29 GB", | 20260508 02:14:59.855 | INFO | "malloc" : " 14 GB", | 20260508 02:14:59.855 | INFO | "pinned" : " 29 GB" | 20260508 02:14:59.855 | INFO | } | 20260508 02:14:59.855 | INFO | Perf report for the last 5.14266 seconds: | 20260508 02:14:59.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:14:59.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:59.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:59.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:59.855 | INFO | ANALYSIS : 69.0%, 2.9, 104.3ms, 0 | 20260508 02:14:59.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:14:59.855 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260508 02:14:59.855 | INFO | PRE_HQ : 68.0%, 1.8, 62.6ms, 0 | 20260508 02:14:59.855 | INFO | BAZWRITER : 70.6%, 4.1, 148.4ms, 0 | 20260508 02:14:59.855 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 02:14:59.856 | INFO | Uploaded 36GiB to GPU in 2.54307s (14.1561GiB/s) | 20260508 02:14:59.856 | INFO | Downloaded 38GiB to GPU in 2.69719s (14.0888GiB/s) | 20260508 02:15:04.992 | INFO | Analyzing chunk frames = [4440576,4441088) | 20260508 02:15:04.992 | INFO | MemInfo: { | 20260508 02:15:04.992 | INFO | "gpu" : " 17 GB", | 20260508 02:15:04.993 | INFO | "huge" : "", | 20260508 02:15:04.993 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:04.993 | INFO | "malloc" : " 14 GB", | 20260508 02:15:04.993 | INFO | "pinned" : " 29 GB" | 20260508 02:15:04.993 | INFO | } | 20260508 02:15:04.993 | INFO | Perf report for the last 5.13747 seconds: | 20260508 02:15:04.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:04.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 02:15:04.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:15:04.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:04.993 | INFO | Uploaded 36GiB to GPU in 2.55082s (14.1131GiB/s) | 20260508 02:15:04.993 | INFO | Downloaded 34GiB to GPU in 2.43683s (13.9526GiB/s) | 20260508 02:15:10.123 | INFO | Analyzing chunk frames = [4441088,4441600) | 20260508 02:15:10.123 | INFO | MemInfo: { | 20260508 02:15:10.123 | INFO | "gpu" : " 17 GB", | 20260508 02:15:10.123 | INFO | "huge" : "", | 20260508 02:15:10.123 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:10.123 | INFO | "malloc" : " 14 GB", | 20260508 02:15:10.123 | INFO | "pinned" : " 29 GB" | 20260508 02:15:10.123 | INFO | } | 20260508 02:15:10.123 | INFO | Perf report for the last 5.13049 seconds: | 20260508 02:15:10.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:10.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.123 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:15:10.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.123 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:15:10.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:10.124 | INFO | Uploaded 36GiB to GPU in 2.55553s (14.0871GiB/s) | 20260508 02:15:10.124 | INFO | Downloaded 34GiB to GPU in 2.5031s (13.5831GiB/s) | 20260508 02:15:15.268 | INFO | Analyzing chunk frames = [4441600,4442112) | 20260508 02:15:15.268 | INFO | MemInfo: { | 20260508 02:15:15.268 | INFO | "gpu" : " 17 GB", | 20260508 02:15:15.268 | INFO | "huge" : "", | 20260508 02:15:15.268 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:15.268 | INFO | "malloc" : " 14 GB", | 20260508 02:15:15.268 | INFO | "pinned" : " 29 GB" | 20260508 02:15:15.268 | INFO | } | 20260508 02:15:15.268 | INFO | Perf report for the last 5.14477 seconds: | 20260508 02:15:15.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:15.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 02:15:15.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:15:15.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:15.268 | INFO | Uploaded 36GiB to GPU in 2.49341s (14.4381GiB/s) | 20260508 02:15:15.268 | INFO | Downloaded 34GiB to GPU in 2.48274s (13.6945GiB/s) | 20260508 02:15:20.406 | INFO | Analyzing chunk frames = [4442112,4442624) | 20260508 02:15:20.406 | INFO | MemInfo: { | 20260508 02:15:20.406 | INFO | "gpu" : " 17 GB", | 20260508 02:15:20.406 | INFO | "huge" : "", | 20260508 02:15:20.406 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:20.406 | INFO | "malloc" : " 14 GB", | 20260508 02:15:20.406 | INFO | "pinned" : " 29 GB" | 20260508 02:15:20.406 | INFO | } | 20260508 02:15:20.406 | INFO | Perf report for the last 5.13774 seconds: | 20260508 02:15:20.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:20.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 02:15:20.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:15:20.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:20.406 | INFO | Uploaded 36GiB to GPU in 2.36101s (15.2477GiB/s) | 20260508 02:15:20.406 | INFO | Downloaded 34GiB to GPU in 2.54814s (13.3431GiB/s) | 20260508 02:15:25.545 | INFO | Analyzing chunk frames = [4442624,4443136) | 20260508 02:15:25.545 | INFO | MemInfo: { | 20260508 02:15:25.546 | INFO | "gpu" : " 17 GB", | 20260508 02:15:25.546 | INFO | "huge" : "", | 20260508 02:15:25.546 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:25.546 | INFO | "malloc" : " 14 GB", | 20260508 02:15:25.546 | INFO | "pinned" : " 29 GB" | 20260508 02:15:25.546 | INFO | } | 20260508 02:15:25.546 | INFO | Perf report for the last 5.13995 seconds: | 20260508 02:15:25.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:25.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:15:25.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:15:25.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:25.546 | INFO | Uploaded 36GiB to GPU in 2.38832s (15.0734GiB/s) | 20260508 02:15:25.546 | INFO | Downloaded 34GiB to GPU in 2.408s (14.1196GiB/s) | 20260508 02:15:30.683 | INFO | Analyzing chunk frames = [4443136,4443648) | 20260508 02:15:30.683 | INFO | MemInfo: { | 20260508 02:15:30.683 | INFO | "gpu" : " 17 GB", | 20260508 02:15:30.683 | INFO | "huge" : "", | 20260508 02:15:30.683 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:30.683 | INFO | "malloc" : " 14 GB", | 20260508 02:15:30.683 | INFO | "pinned" : " 29 GB" | 20260508 02:15:30.683 | INFO | } | 20260508 02:15:30.683 | INFO | Perf report for the last 5.13742 seconds: | 20260508 02:15:30.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:30.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.683 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:15:30.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.683 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:15:30.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:30.684 | INFO | Uploaded 36GiB to GPU in 2.43401s (14.7904GiB/s) | 20260508 02:15:30.684 | INFO | Downloaded 34GiB to GPU in 2.42506s (14.0203GiB/s) | 20260508 02:15:35.828 | INFO | Analyzing chunk frames = [4443648,4444160) | 20260508 02:15:35.828 | INFO | MemInfo: { | 20260508 02:15:35.828 | INFO | "gpu" : " 17 GB", | 20260508 02:15:35.828 | INFO | "huge" : "", | 20260508 02:15:35.828 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:35.828 | INFO | "malloc" : " 14 GB", | 20260508 02:15:35.828 | INFO | "pinned" : " 29 GB" | 20260508 02:15:35.828 | INFO | } | 20260508 02:15:35.828 | INFO | Perf report for the last 5.14484 seconds: | 20260508 02:15:35.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:35.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.828 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:15:35.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.828 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 02:15:35.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:35.829 | INFO | Uploaded 36GiB to GPU in 2.54576s (14.1412GiB/s) | 20260508 02:15:35.829 | INFO | Downloaded 34GiB to GPU in 2.45936s (13.8247GiB/s) | 20260508 02:15:35.943 | INFO | Starting a round of Baz IO | 20260508 02:15:39.375 | INFO | Populating RTMetrics report | 20260508 02:15:39.375 | INFO | RTMetrics report populated | 20260508 02:15:39.375 | INFO | Opening tmp RTMetrics json file | 20260508 02:15:39.391 | INFO | Serializing RTMetrics to json | 20260508 02:15:39.400 | INFO | Serialization done, swapping tmp file now... | 20260508 02:15:39.401 | INFO | RTMetrics done | 20260508 02:15:39.495 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.552 seconds | 20260508 02:15:39.495 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.59% / 0.00% / 58.40% | 20260508 02:15:39.495 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.748769% / 7.94729e-06% / 5.5631e-05% | 20260508 02:15:39.495 | INFO | Baz IO overall: | 20260508 02:15:39.495 | INFO | BytesWritten : 2008046990803 | 20260508 02:15:39.495 | INFO | NumEvents : 970874348644 | 20260508 02:15:40.965 | INFO | Analyzing chunk frames = [4444160,4444672) | 20260508 02:15:40.965 | INFO | MemInfo: { | 20260508 02:15:40.965 | INFO | "gpu" : " 17 GB", | 20260508 02:15:40.965 | INFO | "huge" : "", | 20260508 02:15:40.965 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:40.965 | INFO | "malloc" : " 14 GB", | 20260508 02:15:40.965 | INFO | "pinned" : " 29 GB" | 20260508 02:15:40.965 | INFO | } | 20260508 02:15:40.965 | INFO | Perf report for the last 5.13735 seconds: | 20260508 02:15:40.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:40.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:40.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:40.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:40.965 | INFO | ANALYSIS : 68.3%, 2.9, 104.4ms, 0 | 20260508 02:15:40.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:40.965 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260508 02:15:40.966 | INFO | PRE_HQ : 67.0%, 1.8, 62.1ms, 0 | 20260508 02:15:40.966 | INFO | BAZWRITER : 68.8%, 3.5, 124.6ms, 0 | 20260508 02:15:40.966 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 02:15:40.966 | INFO | Uploaded 36GiB to GPU in 2.58311s (13.9367GiB/s) | 20260508 02:15:40.966 | INFO | Downloaded 38GiB to GPU in 2.70911s (14.0268GiB/s) | 20260508 02:15:46.107 | INFO | Analyzing chunk frames = [4444672,4445184) | 20260508 02:15:46.107 | INFO | MemInfo: { | 20260508 02:15:46.107 | INFO | "gpu" : " 17 GB", | 20260508 02:15:46.107 | INFO | "huge" : "", | 20260508 02:15:46.107 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:46.107 | INFO | "malloc" : " 14 GB", | 20260508 02:15:46.107 | INFO | "pinned" : " 29 GB" | 20260508 02:15:46.107 | INFO | } | 20260508 02:15:46.107 | INFO | Perf report for the last 5.14184 seconds: | 20260508 02:15:46.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:46.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.107 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 02:15:46.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.107 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:15:46.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:46.108 | INFO | Uploaded 36GiB to GPU in 2.57553s (13.9777GiB/s) | 20260508 02:15:46.108 | INFO | Downloaded 34GiB to GPU in 2.42348s (14.0294GiB/s) | 20260508 02:15:51.244 | INFO | Analyzing chunk frames = [4445184,4445696) | 20260508 02:15:51.244 | INFO | MemInfo: { | 20260508 02:15:51.244 | INFO | "gpu" : " 17 GB", | 20260508 02:15:51.244 | INFO | "huge" : "", | 20260508 02:15:51.244 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:51.244 | INFO | "malloc" : " 14 GB", | 20260508 02:15:51.244 | INFO | "pinned" : " 29 GB" | 20260508 02:15:51.244 | INFO | } | 20260508 02:15:51.244 | INFO | Perf report for the last 5.1372 seconds: | 20260508 02:15:51.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:51.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 02:15:51.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:15:51.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:51.245 | INFO | Uploaded 36GiB to GPU in 2.56755s (14.0211GiB/s) | 20260508 02:15:51.245 | INFO | Downloaded 34GiB to GPU in 2.48814s (13.6648GiB/s) | 20260508 02:15:56.383 | INFO | Analyzing chunk frames = [4445696,4446208) | 20260508 02:15:56.383 | INFO | MemInfo: { | 20260508 02:15:56.383 | INFO | "gpu" : " 17 GB", | 20260508 02:15:56.383 | INFO | "huge" : "", | 20260508 02:15:56.383 | INFO | "huge_shared" : " 29 GB", | 20260508 02:15:56.383 | INFO | "malloc" : " 14 GB", | 20260508 02:15:56.383 | INFO | "pinned" : " 29 GB" | 20260508 02:15:56.383 | INFO | } | 20260508 02:15:56.383 | INFO | Perf report for the last 5.13902 seconds: | 20260508 02:15:56.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:15:56.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.384 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:15:56.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.384 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:15:56.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:15:56.384 | INFO | Uploaded 36GiB to GPU in 2.54846s (14.1262GiB/s) | 20260508 02:15:56.384 | INFO | Downloaded 34GiB to GPU in 2.49234s (13.6418GiB/s) | 20260508 02:16:01.522 | INFO | Analyzing chunk frames = [4446208,4446720) | 20260508 02:16:01.522 | INFO | MemInfo: { | 20260508 02:16:01.522 | INFO | "gpu" : " 17 GB", | 20260508 02:16:01.522 | INFO | "huge" : "", | 20260508 02:16:01.522 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:01.522 | INFO | "malloc" : " 14 GB", | 20260508 02:16:01.522 | INFO | "pinned" : " 29 GB" | 20260508 02:16:01.522 | INFO | } | 20260508 02:16:01.522 | INFO | Perf report for the last 5.13853 seconds: | 20260508 02:16:01.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:01.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 02:16:01.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 02:16:01.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:01.522 | INFO | Uploaded 36GiB to GPU in 2.41849s (14.8853GiB/s) | 20260508 02:16:01.523 | INFO | Downloaded 34GiB to GPU in 2.56205s (13.2706GiB/s) | 20260508 02:16:06.664 | INFO | Analyzing chunk frames = [4446720,4447232) | 20260508 02:16:06.664 | INFO | MemInfo: { | 20260508 02:16:06.664 | INFO | "gpu" : " 17 GB", | 20260508 02:16:06.664 | INFO | "huge" : "", | 20260508 02:16:06.664 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:06.664 | INFO | "malloc" : " 14 GB", | 20260508 02:16:06.664 | INFO | "pinned" : " 29 GB" | 20260508 02:16:06.664 | INFO | } | 20260508 02:16:06.664 | INFO | Perf report for the last 5.14199 seconds: | 20260508 02:16:06.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:06.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:16:06.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:16:06.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:06.664 | INFO | Uploaded 36GiB to GPU in 2.38926s (15.0674GiB/s) | 20260508 02:16:06.664 | INFO | Downloaded 34GiB to GPU in 2.41613s (14.0721GiB/s) | 20260508 02:16:11.804 | INFO | Analyzing chunk frames = [4447232,4447744) | 20260508 02:16:11.804 | INFO | MemInfo: { | 20260508 02:16:11.804 | INFO | "gpu" : " 17 GB", | 20260508 02:16:11.804 | INFO | "huge" : "", | 20260508 02:16:11.805 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:11.805 | INFO | "malloc" : " 14 GB", | 20260508 02:16:11.805 | INFO | "pinned" : " 29 GB" | 20260508 02:16:11.805 | INFO | } | 20260508 02:16:11.805 | INFO | Perf report for the last 5.14071 seconds: | 20260508 02:16:11.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:11.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:16:11.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 02:16:11.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:11.805 | INFO | Uploaded 36GiB to GPU in 2.46153s (14.625GiB/s) | 20260508 02:16:11.805 | INFO | Downloaded 34GiB to GPU in 2.42778s (14.0045GiB/s) | 20260508 02:16:16.938 | INFO | Analyzing chunk frames = [4447744,4448256) | 20260508 02:16:16.938 | INFO | MemInfo: { | 20260508 02:16:16.938 | INFO | "gpu" : " 17 GB", | 20260508 02:16:16.938 | INFO | "huge" : "", | 20260508 02:16:16.938 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:16.938 | INFO | "malloc" : " 14 GB", | 20260508 02:16:16.938 | INFO | "pinned" : " 29 GB" | 20260508 02:16:16.938 | INFO | } | 20260508 02:16:16.939 | INFO | Perf report for the last 5.13378 seconds: | 20260508 02:16:16.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:16.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:16:16.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 02:16:16.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:16.939 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:16:16.939 | INFO | Total mem usage: 16.7 GiB | 20260508 02:16:16.939 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:16:16.939 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13%) | 20260508 02:16:16.939 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:16:16.939 | INFO | High Water Breakdown: | 20260508 02:16:16.939 | INFO | 59.9% : 8.85 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:16:16.939 | INFO | 39.9% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:16:16.939 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:16:16.939 | INFO | ------------------------------------------ | 20260508 02:16:16.939 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:16:16.939 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:16:16.939 | INFO | Total mem usage: 26.5 GiB | 20260508 02:16:16.939 | INFO | Uploaded 36GiB to GPU in 2.56345s (14.0436GiB/s) | 20260508 02:16:16.939 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:16:16.940 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:16:16.940 | INFO | Downloaded 34GiB to GPU in 2.46937s (13.7687GiB/s) | 20260508 02:16:16.940 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:16:16.940 | INFO | High Water Breakdown: | 20260508 02:16:16.940 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:16:16.940 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:16:16.940 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:16:16.940 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:16:16.940 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:16:16.940 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:16:16.940 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:16:16.940 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:16:16.940 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:16:16.940 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:16:16.940 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:16:16.940 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:16:16.940 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:16:16.940 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:16:16.940 | INFO | ------------------------------------------ | 20260508 02:16:16.940 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:16:16.940 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:16:16.940 | INFO | Total mem usage: 27.8 GiB | 20260508 02:16:16.940 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:16:16.940 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:16:16.940 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:16:16.940 | INFO | High Water Breakdown: | 20260508 02:16:16.940 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:16:16.940 | INFO | ------------------------------------------ | 20260508 02:16:16.940 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:16:16.940 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:16:16.940 | INFO | Total mem usage: 15.7 GiB | 20260508 02:16:16.940 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:16:16.940 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:16:16.940 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:16:16.940 | INFO | High Water Breakdown: | 20260508 02:16:16.940 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:16:16.940 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:16:16.940 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:16:16.940 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:16:16.940 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:16:16.940 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:16:16.940 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:16:16.940 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:16:16.940 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:16:16.940 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:16:16.940 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:16:16.940 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:16:16.940 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:16:16.940 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:16:16.940 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:16:16.940 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:16:16.940 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:16:16.940 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:16:16.940 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:16:16.940 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:16:16.940 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:16:16.940 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:16:16.941 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:16:16.941 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:16:16.941 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:16:16.941 | INFO | ------------------------------------------ | 20260508 02:16:16.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:16:16.941 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:16:16.941 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:16:16.941 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:16:16.941 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.42e+01 / 1.68e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.01e-01, 1.14e-02 / 5.87e-03 | 20260508 02:16:16.941 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.47e+00, 5.87e-01 / 1.87e-01 | 20260508 02:16:16.941 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 8.79e-02 | 20260508 02:16:16.941 | INFO | PerfReport: 16.08%, DME -- 4.20e+01 / 4.63e+01, 4.00e+00 / 2.01e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.60e+00 / 1.23e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.43e+00 / 2.71e+00, 4.27e-01 / 1.76e-01 | 20260508 02:16:16.941 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.23e+00, 2.67e-01 / 1.47e-01 | 20260508 02:16:16.941 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.32e-01 / 3.65e-01, 5.22e-02 / 2.18e-02 | 20260508 02:16:16.941 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:16:16.941 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:16:16.941 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:16:16.941 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 2.24%, LockUpload -- 2.38e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 2.76%, LockDownload -- 2.94e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 15.38%, LockCompute -- 1.64e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:16:16.941 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260508 02:16:17.056 | INFO | Starting a round of Baz IO | 20260508 02:16:20.520 | INFO | Populating RTMetrics report | 20260508 02:16:20.520 | INFO | RTMetrics report populated | 20260508 02:16:20.520 | INFO | Opening tmp RTMetrics json file | 20260508 02:16:20.547 | INFO | Serializing RTMetrics to json | 20260508 02:16:20.549 | INFO | Serialization done, swapping tmp file now... | 20260508 02:16:20.555 | INFO | RTMetrics done | 20260508 02:16:20.609 | INFO | Finished a round of Baz IO: Wrote 1.645 GiB in 3.552 seconds | 20260508 02:16:20.609 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.59% / 0.00% / 58.41% | 20260508 02:16:20.609 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.74896% / 0% / 5.5631e-05% | 20260508 02:16:20.609 | INFO | Baz IO overall: | 20260508 02:16:20.609 | INFO | BytesWritten : 2009813631140 | 20260508 02:16:20.609 | INFO | NumEvents : 971721990307 | 20260508 02:16:22.079 | INFO | Analyzing chunk frames = [4448256,4448768) | 20260508 02:16:22.079 | INFO | MemInfo: { | 20260508 02:16:22.079 | INFO | "gpu" : " 17 GB", | 20260508 02:16:22.079 | INFO | "huge" : "", | 20260508 02:16:22.079 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:22.079 | INFO | "malloc" : " 14 GB", | 20260508 02:16:22.079 | INFO | "pinned" : " 29 GB" | 20260508 02:16:22.079 | INFO | } | 20260508 02:16:22.079 | INFO | Perf report for the last 5.14044 seconds: | 20260508 02:16:22.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:22.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:22.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:22.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:22.079 | INFO | ANALYSIS : 68.7%, 2.9, 104.7ms, 0 | 20260508 02:16:22.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:22.079 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260508 02:16:22.079 | INFO | PRE_HQ : 67.6%, 1.8, 62.8ms, 0 | 20260508 02:16:22.079 | INFO | BAZWRITER : 68.7%, 4.0, 141.5ms, 0 | 20260508 02:16:22.079 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 02:16:22.080 | INFO | Uploaded 36GiB to GPU in 2.60062s (13.8429GiB/s) | 20260508 02:16:22.080 | INFO | Downloaded 38GiB to GPU in 2.71975s (13.9719GiB/s) | 20260508 02:16:27.219 | INFO | Analyzing chunk frames = [4448768,4449280) | 20260508 02:16:27.219 | INFO | MemInfo: { | 20260508 02:16:27.219 | INFO | "gpu" : " 17 GB", | 20260508 02:16:27.219 | INFO | "huge" : "", | 20260508 02:16:27.219 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:27.219 | INFO | "malloc" : " 14 GB", | 20260508 02:16:27.219 | INFO | "pinned" : " 29 GB" | 20260508 02:16:27.220 | INFO | } | 20260508 02:16:27.220 | INFO | Perf report for the last 5.14052 seconds: | 20260508 02:16:27.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:27.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:16:27.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:16:27.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:27.220 | INFO | Uploaded 36GiB to GPU in 2.56043s (14.0601GiB/s) | 20260508 02:16:27.220 | INFO | Downloaded 34GiB to GPU in 2.43184s (13.9812GiB/s) | 20260508 02:16:32.361 | INFO | Analyzing chunk frames = [4449280,4449792) | 20260508 02:16:32.361 | INFO | MemInfo: { | 20260508 02:16:32.361 | INFO | "gpu" : " 17 GB", | 20260508 02:16:32.361 | INFO | "huge" : "", | 20260508 02:16:32.361 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:32.361 | INFO | "malloc" : " 14 GB", | 20260508 02:16:32.361 | INFO | "pinned" : " 29 GB" | 20260508 02:16:32.361 | INFO | } | 20260508 02:16:32.362 | INFO | Perf report for the last 5.14197 seconds: | 20260508 02:16:32.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:32.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 02:16:32.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:16:32.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:32.362 | INFO | Uploaded 36GiB to GPU in 2.51271s (14.3272GiB/s) | 20260508 02:16:32.362 | INFO | Downloaded 34GiB to GPU in 2.48196s (13.6988GiB/s) | 20260508 02:16:37.495 | INFO | Analyzing chunk frames = [4449792,4450304) | 20260508 02:16:37.495 | INFO | MemInfo: { | 20260508 02:16:37.495 | INFO | "gpu" : " 17 GB", | 20260508 02:16:37.495 | INFO | "huge" : "", | 20260508 02:16:37.495 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:37.495 | INFO | "malloc" : " 14 GB", | 20260508 02:16:37.495 | INFO | "pinned" : " 29 GB" | 20260508 02:16:37.495 | INFO | } | 20260508 02:16:37.495 | INFO | Perf report for the last 5.13326 seconds: | 20260508 02:16:37.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:37.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 02:16:37.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:16:37.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:37.495 | INFO | Uploaded 36GiB to GPU in 2.53972s (14.1748GiB/s) | 20260508 02:16:37.495 | INFO | Downloaded 34GiB to GPU in 2.50292s (13.5841GiB/s) | 20260508 02:16:42.632 | INFO | Analyzing chunk frames = [4450304,4450816) | 20260508 02:16:42.632 | INFO | MemInfo: { | 20260508 02:16:42.632 | INFO | "gpu" : " 17 GB", | 20260508 02:16:42.632 | INFO | "huge" : "", | 20260508 02:16:42.632 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:42.632 | INFO | "malloc" : " 14 GB", | 20260508 02:16:42.632 | INFO | "pinned" : " 29 GB" | 20260508 02:16:42.632 | INFO | } | 20260508 02:16:42.632 | INFO | Perf report for the last 5.13748 seconds: | 20260508 02:16:42.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:42.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.632 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 02:16:42.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.632 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:16:42.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:42.633 | INFO | Uploaded 36GiB to GPU in 2.32756s (15.4668GiB/s) | 20260508 02:16:42.633 | INFO | Downloaded 34GiB to GPU in 2.50462s (13.5749GiB/s) | 20260508 02:16:47.775 | INFO | Analyzing chunk frames = [4450816,4451328) | 20260508 02:16:47.775 | INFO | MemInfo: { | 20260508 02:16:47.775 | INFO | "gpu" : " 17 GB", | 20260508 02:16:47.775 | INFO | "huge" : "", | 20260508 02:16:47.775 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:47.775 | INFO | "malloc" : " 14 GB", | 20260508 02:16:47.775 | INFO | "pinned" : " 29 GB" | 20260508 02:16:47.775 | INFO | } | 20260508 02:16:47.775 | INFO | Perf report for the last 5.14296 seconds: | 20260508 02:16:47.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:47.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.776 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 02:16:47.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.776 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:16:47.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:47.776 | INFO | Uploaded 36GiB to GPU in 2.42743s (14.8305GiB/s) | 20260508 02:16:47.776 | INFO | Downloaded 34GiB to GPU in 2.42529s (14.019GiB/s) | 20260508 02:16:52.922 | INFO | Analyzing chunk frames = [4451328,4451840) | 20260508 02:16:52.922 | INFO | MemInfo: { | 20260508 02:16:52.922 | INFO | "gpu" : " 17 GB", | 20260508 02:16:52.922 | INFO | "huge" : "", | 20260508 02:16:52.922 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:52.923 | INFO | "malloc" : " 14 GB", | 20260508 02:16:52.923 | INFO | "pinned" : " 29 GB" | 20260508 02:16:52.923 | INFO | } | 20260508 02:16:52.923 | INFO | Perf report for the last 5.14742 seconds: | 20260508 02:16:52.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:52.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | ANALYSIS : 67.2%, 3.0, 105.2ms, 0 | 20260508 02:16:52.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 02:16:52.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:52.923 | INFO | Uploaded 36GiB to GPU in 2.54977s (14.1189GiB/s) | 20260508 02:16:52.923 | INFO | Downloaded 34GiB to GPU in 2.45313s (13.8598GiB/s) | 20260508 02:16:58.059 | INFO | Analyzing chunk frames = [4451840,4452352) | 20260508 02:16:58.059 | INFO | MemInfo: { | 20260508 02:16:58.059 | INFO | "gpu" : " 17 GB", | 20260508 02:16:58.059 | INFO | "huge" : "", | 20260508 02:16:58.060 | INFO | "huge_shared" : " 29 GB", | 20260508 02:16:58.060 | INFO | "malloc" : " 14 GB", | 20260508 02:16:58.060 | INFO | "pinned" : " 29 GB" | 20260508 02:16:58.060 | INFO | } | 20260508 02:16:58.060 | INFO | Perf report for the last 5.13703 seconds: | 20260508 02:16:58.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:16:58.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 02:16:58.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260508 02:16:58.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:16:58.060 | INFO | Uploaded 36GiB to GPU in 2.47691s (14.5343GiB/s) | 20260508 02:16:58.060 | INFO | Downloaded 34GiB to GPU in 2.42443s (14.0239GiB/s) | 20260508 02:16:58.157 | INFO | Starting a round of Baz IO | 20260508 02:17:01.597 | INFO | Populating RTMetrics report | 20260508 02:17:01.597 | INFO | RTMetrics report populated | 20260508 02:17:01.597 | INFO | Opening tmp RTMetrics json file | 20260508 02:17:01.616 | INFO | Serializing RTMetrics to json | 20260508 02:17:01.626 | INFO | Serialization done, swapping tmp file now... | 20260508 02:17:01.633 | INFO | RTMetrics done | 20260508 02:17:01.702 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.545 seconds | 20260508 02:17:01.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.59% / 0.00% / 58.41% | 20260508 02:17:01.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.746989% / 0% / 3.97364e-05% | 20260508 02:17:01.702 | INFO | Baz IO overall: | 20260508 02:17:01.702 | INFO | BytesWritten : 2011578244172 | 20260508 02:17:01.702 | INFO | NumEvents : 972568404407 | 20260508 02:17:03.190 | INFO | Analyzing chunk frames = [4452352,4452864) | 20260508 02:17:03.190 | INFO | MemInfo: { | 20260508 02:17:03.190 | INFO | "gpu" : " 17 GB", | 20260508 02:17:03.190 | INFO | "huge" : "", | 20260508 02:17:03.190 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:03.190 | INFO | "malloc" : " 14 GB", | 20260508 02:17:03.190 | INFO | "pinned" : " 29 GB" | 20260508 02:17:03.190 | INFO | } | 20260508 02:17:03.190 | INFO | Perf report for the last 5.13022 seconds: | 20260508 02:17:03.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:03.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:03.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:03.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:03.190 | INFO | ANALYSIS : 68.3%, 3.0, 106.1ms, 0 | 20260508 02:17:03.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:03.190 | INFO | RT_METRICS : 40.6%, 1.0, 21.2ms, 0 | 20260508 02:17:03.190 | INFO | PRE_HQ : 65.8%, 1.6, 56.1ms, 0 | 20260508 02:17:03.190 | INFO | BAZWRITER : 68.7%, 3.4, 121.4ms, 0 | 20260508 02:17:03.190 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 02:17:03.190 | INFO | Uploaded 36GiB to GPU in 2.50775s (14.3555GiB/s) | 20260508 02:17:03.190 | INFO | Downloaded 38GiB to GPU in 2.69791s (14.085GiB/s) | 20260508 02:17:08.333 | INFO | Analyzing chunk frames = [4452864,4453376) | 20260508 02:17:08.333 | INFO | MemInfo: { | 20260508 02:17:08.333 | INFO | "gpu" : " 17 GB", | 20260508 02:17:08.333 | INFO | "huge" : "", | 20260508 02:17:08.333 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:08.333 | INFO | "malloc" : " 14 GB", | 20260508 02:17:08.333 | INFO | "pinned" : " 29 GB" | 20260508 02:17:08.333 | INFO | } | 20260508 02:17:08.333 | INFO | Perf report for the last 5.14339 seconds: | 20260508 02:17:08.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:08.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.333 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 02:17:08.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.333 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:17:08.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:08.334 | INFO | Uploaded 36GiB to GPU in 2.56742s (14.0219GiB/s) | 20260508 02:17:08.334 | INFO | Downloaded 34GiB to GPU in 2.42147s (14.041GiB/s) | 20260508 02:17:13.473 | INFO | Analyzing chunk frames = [4453376,4453888) | 20260508 02:17:13.474 | INFO | MemInfo: { | 20260508 02:17:13.474 | INFO | "gpu" : " 17 GB", | 20260508 02:17:13.474 | INFO | "huge" : "", | 20260508 02:17:13.474 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:13.474 | INFO | "malloc" : " 14 GB", | 20260508 02:17:13.474 | INFO | "pinned" : " 29 GB" | 20260508 02:17:13.474 | INFO | } | 20260508 02:17:13.474 | INFO | Perf report for the last 5.14038 seconds: | 20260508 02:17:13.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:13.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:17:13.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:17:13.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:13.474 | INFO | Uploaded 36GiB to GPU in 2.56389s (14.0412GiB/s) | 20260508 02:17:13.474 | INFO | Downloaded 34GiB to GPU in 2.48184s (13.6995GiB/s) | 20260508 02:17:18.613 | INFO | Analyzing chunk frames = [4453888,4454400) | 20260508 02:17:18.614 | INFO | MemInfo: { | 20260508 02:17:18.614 | INFO | "gpu" : " 17 GB", | 20260508 02:17:18.614 | INFO | "huge" : "", | 20260508 02:17:18.614 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:18.614 | INFO | "malloc" : " 14 GB", | 20260508 02:17:18.614 | INFO | "pinned" : " 29 GB" | 20260508 02:17:18.614 | INFO | } | 20260508 02:17:18.614 | INFO | Perf report for the last 5.13979 seconds: | 20260508 02:17:18.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:18.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 02:17:18.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:17:18.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:18.614 | INFO | Uploaded 36GiB to GPU in 2.57537s (13.9786GiB/s) | 20260508 02:17:18.614 | INFO | Downloaded 34GiB to GPU in 2.48841s (13.6633GiB/s) | 20260508 02:17:23.756 | INFO | Analyzing chunk frames = [4454400,4454912) | 20260508 02:17:23.756 | INFO | MemInfo: { | 20260508 02:17:23.756 | INFO | "gpu" : " 17 GB", | 20260508 02:17:23.756 | INFO | "huge" : "", | 20260508 02:17:23.756 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:23.756 | INFO | "malloc" : " 14 GB", | 20260508 02:17:23.756 | INFO | "pinned" : " 29 GB" | 20260508 02:17:23.756 | INFO | } | 20260508 02:17:23.756 | INFO | Perf report for the last 5.14245 seconds: | 20260508 02:17:23.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:23.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.756 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 02:17:23.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.756 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:17:23.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:23.757 | INFO | Uploaded 36GiB to GPU in 2.42111s (14.8692GiB/s) | 20260508 02:17:23.757 | INFO | Downloaded 34GiB to GPU in 2.53553s (13.4095GiB/s) | 20260508 02:17:28.887 | INFO | Analyzing chunk frames = [4454912,4455424) | 20260508 02:17:28.887 | INFO | MemInfo: { | 20260508 02:17:28.887 | INFO | "gpu" : " 17 GB", | 20260508 02:17:28.887 | INFO | "huge" : "", | 20260508 02:17:28.887 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:28.887 | INFO | "malloc" : " 14 GB", | 20260508 02:17:28.887 | INFO | "pinned" : " 29 GB" | 20260508 02:17:28.887 | INFO | } | 20260508 02:17:28.887 | INFO | Perf report for the last 5.13083 seconds: | 20260508 02:17:28.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:28.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:17:28.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 02:17:28.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:28.887 | INFO | Uploaded 36GiB to GPU in 2.41935s (14.88GiB/s) | 20260508 02:17:28.887 | INFO | Downloaded 34GiB to GPU in 2.41305s (14.09GiB/s) | 20260508 02:17:34.030 | INFO | Analyzing chunk frames = [4455424,4455936) | 20260508 02:17:34.030 | INFO | MemInfo: { | 20260508 02:17:34.030 | INFO | "gpu" : " 17 GB", | 20260508 02:17:34.030 | INFO | "huge" : "", | 20260508 02:17:34.030 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:34.030 | INFO | "malloc" : " 14 GB", | 20260508 02:17:34.030 | INFO | "pinned" : " 29 GB" | 20260508 02:17:34.030 | INFO | } | 20260508 02:17:34.030 | INFO | Perf report for the last 5.14315 seconds: | 20260508 02:17:34.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:34.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 02:17:34.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 02:17:34.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:34.030 | INFO | Uploaded 36GiB to GPU in 2.55068s (14.1139GiB/s) | 20260508 02:17:34.030 | INFO | Downloaded 34GiB to GPU in 2.46283s (13.8053GiB/s) | 20260508 02:17:39.170 | INFO | Analyzing chunk frames = [4455936,4456448) | 20260508 02:17:39.170 | INFO | MemInfo: { | 20260508 02:17:39.170 | INFO | "gpu" : " 17 GB", | 20260508 02:17:39.170 | INFO | "huge" : "", | 20260508 02:17:39.170 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:39.171 | INFO | "malloc" : " 14 GB", | 20260508 02:17:39.171 | INFO | "pinned" : " 29 GB" | 20260508 02:17:39.171 | INFO | } | 20260508 02:17:39.171 | INFO | Perf report for the last 5.14059 seconds: | 20260508 02:17:39.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:39.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:17:39.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 02:17:39.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:39.171 | INFO | Uploaded 36GiB to GPU in 2.57157s (13.9992GiB/s) | 20260508 02:17:39.171 | INFO | Downloaded 34GiB to GPU in 2.46283s (13.8053GiB/s) | 20260508 02:17:39.172 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:17:39.172 | INFO | Total mem usage: 16.7 GiB | 20260508 02:17:39.172 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:17:39.172 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.1%) | 20260508 02:17:39.172 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:17:39.172 | INFO | High Water Breakdown: | 20260508 02:17:39.172 | INFO | 59.7% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:17:39.172 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:17:39.172 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:17:39.172 | INFO | ------------------------------------------ | 20260508 02:17:39.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:17:39.172 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:17:39.172 | INFO | Total mem usage: 26.5 GiB | 20260508 02:17:39.172 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:17:39.172 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:17:39.172 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:17:39.172 | INFO | High Water Breakdown: | 20260508 02:17:39.172 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:17:39.172 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:17:39.172 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:17:39.172 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:17:39.172 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:17:39.172 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:17:39.172 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:17:39.172 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:17:39.172 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:17:39.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:17:39.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:17:39.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:17:39.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:17:39.172 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:17:39.172 | INFO | ------------------------------------------ | 20260508 02:17:39.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:17:39.172 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:17:39.173 | INFO | Total mem usage: 27.8 GiB | 20260508 02:17:39.173 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:17:39.173 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:17:39.173 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:17:39.173 | INFO | High Water Breakdown: | 20260508 02:17:39.173 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:17:39.173 | INFO | ------------------------------------------ | 20260508 02:17:39.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:17:39.173 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:17:39.173 | INFO | Total mem usage: 15.7 GiB | 20260508 02:17:39.173 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:17:39.173 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:17:39.173 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:17:39.173 | INFO | High Water Breakdown: | 20260508 02:17:39.173 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:17:39.173 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:17:39.173 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:17:39.173 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:17:39.173 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:17:39.173 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:17:39.173 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:17:39.173 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:17:39.173 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:17:39.173 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:17:39.173 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:17:39.173 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:17:39.173 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:17:39.173 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:17:39.173 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:17:39.173 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:17:39.173 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:17:39.173 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:17:39.173 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:17:39.173 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:17:39.173 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:17:39.173 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:17:39.173 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:17:39.173 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:17:39.173 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:17:39.173 | INFO | ------------------------------------------ | 20260508 02:17:39.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:17:39.173 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:17:39.173 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:17:39.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:17:39.173 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.43e+01 / 1.23e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.17e-02 / 7.00e-03 | 20260508 02:17:39.173 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.47e+00, 5.83e-01 / 1.86e-01 | 20260508 02:17:39.173 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.48e+00, 2.05e-01 / 1.14e-01 | 20260508 02:17:39.173 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.59e+01, 3.86e+00 / 1.16e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.60e+00 / 1.40e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.43e+00 / 2.68e+00, 4.24e-01 / 2.40e-01 | 20260508 02:17:39.173 | INFO | PerfReport: 3.31%, Metrics -- 1.08e+00 / 1.24e+00, 2.65e-01 / 1.53e-01 | 20260508 02:17:39.173 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.31e-01 / 3.61e-01, 5.13e-02 / 2.23e-02 | 20260508 02:17:39.173 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:17:39.173 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:17:39.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:17:39.173 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 24.54%, Upload -- 2.62e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 30.62%, Compute -- 3.26e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 2.37%, LockUpload -- 2.53e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 2.69%, LockDownload -- 2.86e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 15.39%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:17:39.173 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260508 02:17:39.283 | INFO | Starting a round of Baz IO | 20260508 02:17:42.726 | INFO | Populating RTMetrics report | 20260508 02:17:42.726 | INFO | RTMetrics report populated | 20260508 02:17:42.726 | INFO | Opening tmp RTMetrics json file | 20260508 02:17:42.728 | INFO | Serializing RTMetrics to json | 20260508 02:17:42.734 | INFO | Serialization done, swapping tmp file now... | 20260508 02:17:42.739 | INFO | RTMetrics done | 20260508 02:17:42.851 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.568 seconds | 20260508 02:17:42.851 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.58% / 0.00% / 58.41% | 20260508 02:17:42.851 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.745722% / 3.97364e-06% / 5.5631e-05% | 20260508 02:17:42.851 | INFO | Baz IO overall: | 20260508 02:17:42.851 | INFO | BytesWritten : 2013342801590 | 20260508 02:17:42.851 | INFO | NumEvents : 973414746165 | 20260508 02:17:44.309 | INFO | Analyzing chunk frames = [4456448,4456960) | 20260508 02:17:44.309 | INFO | MemInfo: { | 20260508 02:17:44.309 | INFO | "gpu" : " 17 GB", | 20260508 02:17:44.309 | INFO | "huge" : "", | 20260508 02:17:44.309 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:44.309 | INFO | "malloc" : " 14 GB", | 20260508 02:17:44.310 | INFO | "pinned" : " 29 GB" | 20260508 02:17:44.310 | INFO | } | 20260508 02:17:44.310 | INFO | Perf report for the last 5.13889 seconds: | 20260508 02:17:44.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:44.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:44.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:44.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:44.310 | INFO | ANALYSIS : 68.7%, 2.9, 104.1ms, 0 | 20260508 02:17:44.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:44.310 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 02:17:44.310 | INFO | PRE_HQ : 67.4%, 1.8, 62.2ms, 0 | 20260508 02:17:44.310 | INFO | BAZWRITER : 69.0%, 3.6, 129.2ms, 0 | 20260508 02:17:44.310 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 02:17:44.310 | INFO | Uploaded 36GiB to GPU in 2.50809s (14.3536GiB/s) | 20260508 02:17:44.310 | INFO | Downloaded 38GiB to GPU in 2.68075s (14.1752GiB/s) | 20260508 02:17:49.445 | INFO | Analyzing chunk frames = [4456960,4457472) | 20260508 02:17:49.445 | INFO | MemInfo: { | 20260508 02:17:49.445 | INFO | "gpu" : " 17 GB", | 20260508 02:17:49.445 | INFO | "huge" : "", | 20260508 02:17:49.445 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:49.445 | INFO | "malloc" : " 14 GB", | 20260508 02:17:49.445 | INFO | "pinned" : " 29 GB" | 20260508 02:17:49.445 | INFO | } | 20260508 02:17:49.445 | INFO | Perf report for the last 5.13571 seconds: | 20260508 02:17:49.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:49.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.445 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:17:49.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.445 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:17:49.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:49.446 | INFO | Uploaded 36GiB to GPU in 2.53455s (14.2037GiB/s) | 20260508 02:17:49.446 | INFO | Downloaded 34GiB to GPU in 2.4439s (13.9122GiB/s) | 20260508 02:17:54.589 | INFO | Analyzing chunk frames = [4457472,4457984) | 20260508 02:17:54.589 | INFO | MemInfo: { | 20260508 02:17:54.589 | INFO | "gpu" : " 17 GB", | 20260508 02:17:54.589 | INFO | "huge" : "", | 20260508 02:17:54.589 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:54.589 | INFO | "malloc" : " 14 GB", | 20260508 02:17:54.589 | INFO | "pinned" : " 29 GB" | 20260508 02:17:54.589 | INFO | } | 20260508 02:17:54.589 | INFO | Perf report for the last 5.14358 seconds: | 20260508 02:17:54.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:54.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 02:17:54.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:17:54.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:54.589 | INFO | Uploaded 36GiB to GPU in 2.46591s (14.599GiB/s) | 20260508 02:17:54.589 | INFO | Downloaded 34GiB to GPU in 2.50507s (13.5725GiB/s) | 20260508 02:17:59.726 | INFO | Analyzing chunk frames = [4457984,4458496) | 20260508 02:17:59.726 | INFO | MemInfo: { | 20260508 02:17:59.726 | INFO | "gpu" : " 17 GB", | 20260508 02:17:59.726 | INFO | "huge" : "", | 20260508 02:17:59.726 | INFO | "huge_shared" : " 29 GB", | 20260508 02:17:59.726 | INFO | "malloc" : " 14 GB", | 20260508 02:17:59.726 | INFO | "pinned" : " 29 GB" | 20260508 02:17:59.726 | INFO | } | 20260508 02:17:59.726 | INFO | Perf report for the last 5.1374 seconds: | 20260508 02:17:59.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:17:59.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.726 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 02:17:59.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.726 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:17:59.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:17:59.727 | INFO | Uploaded 36GiB to GPU in 2.4726s (14.5596GiB/s) | 20260508 02:17:59.727 | INFO | Downloaded 34GiB to GPU in 2.51153s (13.5376GiB/s) | 20260508 02:18:04.870 | INFO | Analyzing chunk frames = [4458496,4459008) | 20260508 02:18:04.870 | INFO | MemInfo: { | 20260508 02:18:04.870 | INFO | "gpu" : " 17 GB", | 20260508 02:18:04.870 | INFO | "huge" : "", | 20260508 02:18:04.870 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:04.870 | INFO | "malloc" : " 14 GB", | 20260508 02:18:04.870 | INFO | "pinned" : " 29 GB" | 20260508 02:18:04.871 | INFO | } | 20260508 02:18:04.871 | INFO | Perf report for the last 5.14427 seconds: | 20260508 02:18:04.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:04.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 02:18:04.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:18:04.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:04.871 | INFO | Uploaded 36GiB to GPU in 2.39132s (15.0545GiB/s) | 20260508 02:18:04.871 | INFO | Downloaded 34GiB to GPU in 2.57804s (13.1883GiB/s) | 20260508 02:18:10.010 | INFO | Analyzing chunk frames = [4459008,4459520) | 20260508 02:18:10.010 | INFO | MemInfo: { | 20260508 02:18:10.010 | INFO | "gpu" : " 17 GB", | 20260508 02:18:10.010 | INFO | "huge" : "", | 20260508 02:18:10.010 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:10.010 | INFO | "malloc" : " 14 GB", | 20260508 02:18:10.010 | INFO | "pinned" : " 29 GB" | 20260508 02:18:10.010 | INFO | } | 20260508 02:18:10.010 | INFO | Perf report for the last 5.13979 seconds: | 20260508 02:18:10.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:10.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.010 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:18:10.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.010 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:18:10.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:10.011 | INFO | Uploaded 36GiB to GPU in 2.38271s (15.1089GiB/s) | 20260508 02:18:10.011 | INFO | Downloaded 34GiB to GPU in 2.41792s (14.0617GiB/s) | 20260508 02:18:15.146 | INFO | Analyzing chunk frames = [4459520,4460032) | 20260508 02:18:15.146 | INFO | MemInfo: { | 20260508 02:18:15.146 | INFO | "gpu" : " 17 GB", | 20260508 02:18:15.146 | INFO | "huge" : "", | 20260508 02:18:15.146 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:15.146 | INFO | "malloc" : " 14 GB", | 20260508 02:18:15.146 | INFO | "pinned" : " 29 GB" | 20260508 02:18:15.146 | INFO | } | 20260508 02:18:15.146 | INFO | Perf report for the last 5.13599 seconds: | 20260508 02:18:15.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:15.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.146 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:18:15.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.147 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 02:18:15.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:15.147 | INFO | Uploaded 36GiB to GPU in 2.43083s (14.8098GiB/s) | 20260508 02:18:15.147 | INFO | Downloaded 34GiB to GPU in 2.43015s (13.9909GiB/s) | 20260508 02:18:20.287 | INFO | Analyzing chunk frames = [4460032,4460544) | 20260508 02:18:20.287 | INFO | MemInfo: { | 20260508 02:18:20.287 | INFO | "gpu" : " 17 GB", | 20260508 02:18:20.287 | INFO | "huge" : "", | 20260508 02:18:20.287 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:20.287 | INFO | "malloc" : " 14 GB", | 20260508 02:18:20.287 | INFO | "pinned" : " 29 GB" | 20260508 02:18:20.287 | INFO | } | 20260508 02:18:20.287 | INFO | Perf report for the last 5.14069 seconds: | 20260508 02:18:20.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:20.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:18:20.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260508 02:18:20.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:20.287 | INFO | Uploaded 36GiB to GPU in 2.46621s (14.5973GiB/s) | 20260508 02:18:20.287 | INFO | Downloaded 34GiB to GPU in 2.43055s (13.9886GiB/s) | 20260508 02:18:20.384 | INFO | Starting a round of Baz IO | 20260508 02:18:23.831 | INFO | Populating RTMetrics report | 20260508 02:18:23.831 | INFO | RTMetrics report populated | 20260508 02:18:23.831 | INFO | Opening tmp RTMetrics json file | 20260508 02:18:23.836 | INFO | Serializing RTMetrics to json | 20260508 02:18:23.885 | INFO | Serialization done, swapping tmp file now... | 20260508 02:18:23.936 | INFO | RTMetrics done | 20260508 02:18:23.956 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.571 seconds | 20260508 02:18:23.956 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.58% / 0.00% / 58.42% | 20260508 02:18:23.956 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.74379% / 7.94729e-06% / 5.96046e-05% | 20260508 02:18:23.956 | INFO | Baz IO overall: | 20260508 02:18:23.956 | INFO | BytesWritten : 2015107491850 | 20260508 02:18:23.956 | INFO | NumEvents : 974261241797 | 20260508 02:18:25.427 | INFO | Analyzing chunk frames = [4460544,4461056) | 20260508 02:18:25.427 | INFO | MemInfo: { | 20260508 02:18:25.427 | INFO | "gpu" : " 17 GB", | 20260508 02:18:25.427 | INFO | "huge" : "", | 20260508 02:18:25.427 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:25.427 | INFO | "malloc" : " 14 GB", | 20260508 02:18:25.427 | INFO | "pinned" : " 29 GB" | 20260508 02:18:25.427 | INFO | } | 20260508 02:18:25.427 | INFO | Perf report for the last 5.13992 seconds: | 20260508 02:18:25.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:25.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:25.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:25.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:25.427 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 02:18:25.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:25.427 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 02:18:25.427 | INFO | PRE_HQ : 65.8%, 1.6, 56.0ms, 0 | 20260508 02:18:25.427 | INFO | BAZWRITER : 69.0%, 3.8, 134.9ms, 0 | 20260508 02:18:25.427 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 02:18:25.427 | INFO | Uploaded 36GiB to GPU in 2.53984s (14.1741GiB/s) | 20260508 02:18:25.427 | INFO | Downloaded 38GiB to GPU in 2.68162s (14.1705GiB/s) | 20260508 02:18:30.562 | INFO | Analyzing chunk frames = [4461056,4461568) | 20260508 02:18:30.562 | INFO | MemInfo: { | 20260508 02:18:30.562 | INFO | "gpu" : " 17 GB", | 20260508 02:18:30.562 | INFO | "huge" : "", | 20260508 02:18:30.562 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:30.562 | INFO | "malloc" : " 14 GB", | 20260508 02:18:30.562 | INFO | "pinned" : " 29 GB" | 20260508 02:18:30.562 | INFO | } | 20260508 02:18:30.562 | INFO | Perf report for the last 5.13524 seconds: | 20260508 02:18:30.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:30.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.562 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:18:30.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.562 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:18:30.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:30.563 | INFO | Uploaded 36GiB to GPU in 2.56013s (14.0618GiB/s) | 20260508 02:18:30.563 | INFO | Downloaded 34GiB to GPU in 2.42564s (14.0169GiB/s) | 20260508 02:18:35.700 | INFO | Analyzing chunk frames = [4461568,4462080) | 20260508 02:18:35.700 | INFO | MemInfo: { | 20260508 02:18:35.700 | INFO | "gpu" : " 17 GB", | 20260508 02:18:35.700 | INFO | "huge" : "", | 20260508 02:18:35.700 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:35.700 | INFO | "malloc" : " 14 GB", | 20260508 02:18:35.700 | INFO | "pinned" : " 29 GB" | 20260508 02:18:35.700 | INFO | } | 20260508 02:18:35.700 | INFO | Perf report for the last 5.13779 seconds: | 20260508 02:18:35.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:35.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:18:35.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:18:35.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:35.700 | INFO | Uploaded 36GiB to GPU in 2.53942s (14.1765GiB/s) | 20260508 02:18:35.700 | INFO | Downloaded 34GiB to GPU in 2.49539s (13.6251GiB/s) | 20260508 02:18:40.842 | INFO | Analyzing chunk frames = [4462080,4462592) | 20260508 02:18:40.842 | INFO | MemInfo: { | 20260508 02:18:40.842 | INFO | "gpu" : " 17 GB", | 20260508 02:18:40.842 | INFO | "huge" : "", | 20260508 02:18:40.842 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:40.842 | INFO | "malloc" : " 14 GB", | 20260508 02:18:40.842 | INFO | "pinned" : " 29 GB" | 20260508 02:18:40.842 | INFO | } | 20260508 02:18:40.842 | INFO | Perf report for the last 5.14164 seconds: | 20260508 02:18:40.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:40.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:18:40.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:18:40.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:40.842 | INFO | Uploaded 36GiB to GPU in 2.5256s (14.2541GiB/s) | 20260508 02:18:40.842 | INFO | Downloaded 34GiB to GPU in 2.49165s (13.6456GiB/s) | 20260508 02:18:45.985 | INFO | Analyzing chunk frames = [4462592,4463104) | 20260508 02:18:45.985 | INFO | MemInfo: { | 20260508 02:18:45.985 | INFO | "gpu" : " 17 GB", | 20260508 02:18:45.985 | INFO | "huge" : "", | 20260508 02:18:45.985 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:45.985 | INFO | "malloc" : " 14 GB", | 20260508 02:18:45.985 | INFO | "pinned" : " 29 GB" | 20260508 02:18:45.985 | INFO | } | 20260508 02:18:45.985 | INFO | Perf report for the last 5.14318 seconds: | 20260508 02:18:45.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:45.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 02:18:45.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 02:18:45.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:45.985 | INFO | Uploaded 36GiB to GPU in 2.33733s (15.4022GiB/s) | 20260508 02:18:45.985 | INFO | Downloaded 34GiB to GPU in 2.50655s (13.5645GiB/s) | 20260508 02:18:51.125 | INFO | Analyzing chunk frames = [4463104,4463616) | 20260508 02:18:51.125 | INFO | MemInfo: { | 20260508 02:18:51.125 | INFO | "gpu" : " 17 GB", | 20260508 02:18:51.125 | INFO | "huge" : "", | 20260508 02:18:51.125 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:51.125 | INFO | "malloc" : " 14 GB", | 20260508 02:18:51.125 | INFO | "pinned" : " 29 GB" | 20260508 02:18:51.125 | INFO | } | 20260508 02:18:51.125 | INFO | Perf report for the last 5.14008 seconds: | 20260508 02:18:51.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:51.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 02:18:51.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 02:18:51.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:51.125 | INFO | Uploaded 36GiB to GPU in 2.41635s (14.8985GiB/s) | 20260508 02:18:51.125 | INFO | Downloaded 34GiB to GPU in 2.42736s (14.007GiB/s) | 20260508 02:18:56.258 | INFO | Analyzing chunk frames = [4463616,4464128) | 20260508 02:18:56.258 | INFO | MemInfo: { | 20260508 02:18:56.258 | INFO | "gpu" : " 17 GB", | 20260508 02:18:56.258 | INFO | "huge" : "", | 20260508 02:18:56.258 | INFO | "huge_shared" : " 29 GB", | 20260508 02:18:56.258 | INFO | "malloc" : " 14 GB", | 20260508 02:18:56.258 | INFO | "pinned" : " 29 GB" | 20260508 02:18:56.258 | INFO | } | 20260508 02:18:56.258 | INFO | Perf report for the last 5.13342 seconds: | 20260508 02:18:56.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:18:56.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.258 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 02:18:56.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.259 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:18:56.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:18:56.259 | INFO | Uploaded 36GiB to GPU in 2.41571s (14.9025GiB/s) | 20260508 02:18:56.259 | INFO | Downloaded 34GiB to GPU in 2.44116s (13.9278GiB/s) | 20260508 02:19:01.395 | INFO | Analyzing chunk frames = [4464128,4464640) | 20260508 02:19:01.395 | INFO | MemInfo: { | 20260508 02:19:01.395 | INFO | "gpu" : " 17 GB", | 20260508 02:19:01.395 | INFO | "huge" : "", | 20260508 02:19:01.395 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:01.395 | INFO | "malloc" : " 14 GB", | 20260508 02:19:01.395 | INFO | "pinned" : " 29 GB" | 20260508 02:19:01.395 | INFO | } | 20260508 02:19:01.395 | INFO | Perf report for the last 5.1369 seconds: | 20260508 02:19:01.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:01.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.395 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 02:19:01.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.395 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:19:01.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:01.396 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:19:01.396 | INFO | Total mem usage: 16.7 GiB | 20260508 02:19:01.396 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:19:01.396 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 02:19:01.396 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:19:01.396 | INFO | High Water Breakdown: | 20260508 02:19:01.396 | INFO | 59.8% : 8.72 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:19:01.396 | INFO | 40.1% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:19:01.396 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:19:01.396 | INFO | ------------------------------------------ | 20260508 02:19:01.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:19:01.396 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:19:01.396 | INFO | Total mem usage: 26.5 GiB | 20260508 02:19:01.396 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:19:01.396 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:19:01.396 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:19:01.396 | INFO | High Water Breakdown: | 20260508 02:19:01.396 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:19:01.396 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:19:01.396 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:19:01.396 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:19:01.396 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:19:01.396 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:19:01.396 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:19:01.396 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:19:01.396 | INFO | Uploaded 36GiB to GPU in 2.52112s (14.2794GiB/s) | 20260508 02:19:01.396 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:19:01.396 | INFO | Downloaded 34GiB to GPU in 2.46669s (13.7837GiB/s) | 20260508 02:19:01.396 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:19:01.396 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:19:01.396 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:19:01.396 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:19:01.396 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:19:01.396 | INFO | ------------------------------------------ | 20260508 02:19:01.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:19:01.396 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:19:01.396 | INFO | Total mem usage: 27.8 GiB | 20260508 02:19:01.396 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:19:01.396 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:19:01.397 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:19:01.397 | INFO | High Water Breakdown: | 20260508 02:19:01.397 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:19:01.397 | INFO | ------------------------------------------ | 20260508 02:19:01.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:19:01.397 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:19:01.397 | INFO | Total mem usage: 15.7 GiB | 20260508 02:19:01.397 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:19:01.397 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:19:01.397 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:19:01.397 | INFO | High Water Breakdown: | 20260508 02:19:01.397 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:19:01.397 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:19:01.397 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:19:01.397 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:19:01.397 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:19:01.397 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:19:01.397 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:19:01.397 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:19:01.397 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:19:01.397 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:19:01.397 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:19:01.397 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:19:01.397 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:19:01.397 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:19:01.397 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:19:01.397 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:19:01.397 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:19:01.397 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:19:01.397 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:19:01.397 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:19:01.397 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:19:01.397 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:19:01.397 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:19:01.397 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:19:01.397 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:19:01.397 | INFO | ------------------------------------------ | 20260508 02:19:01.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:19:01.397 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:19:01.397 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:19:01.397 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:19:01.397 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.63e+01, 1.43e+01 / 1.70e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.01e-01, 1.09e-02 / 4.88e-03 | 20260508 02:19:01.397 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.40e+00, 5.87e-01 / 2.84e-01 | 20260508 02:19:01.397 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.02e-01 | 20260508 02:19:01.397 | INFO | PerfReport: 16.09%, DME -- 4.22e+01 / 4.65e+01, 4.16e+00 / 1.94e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.66e+00 / 1.37e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.72e+00, 4.25e-01 / 1.98e-01 | 20260508 02:19:01.397 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.22e+00, 2.67e-01 / 1.23e-01 | 20260508 02:19:01.397 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.30e-01 / 3.62e-01, 5.18e-02 / 2.01e-02 | 20260508 02:19:01.397 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:19:01.397 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:19:01.397 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:19:01.397 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 24.21%, Upload -- 2.57e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 24.47%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 1.96%, LockUpload -- 2.09e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 2.96%, LockDownload -- 3.14e+00 / 0.00e+00, 4.41e+00 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 15.54%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:19:01.397 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260508 02:19:01.492 | INFO | Starting a round of Baz IO | 20260508 02:19:04.946 | INFO | Populating RTMetrics report | 20260508 02:19:04.946 | INFO | RTMetrics report populated | 20260508 02:19:04.946 | INFO | Opening tmp RTMetrics json file | 20260508 02:19:05.228 | INFO | Serializing RTMetrics to json | 20260508 02:19:05.233 | INFO | Serialization done, swapping tmp file now... | 20260508 02:19:05.238 | INFO | RTMetrics done | 20260508 02:19:05.267 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.774 seconds | 20260508 02:19:05.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.58% / 0.00% / 58.42% | 20260508 02:19:05.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.740806% / 3.97364e-06% / 3.97364e-05% | 20260508 02:19:05.267 | INFO | Baz IO overall: | 20260508 02:19:05.267 | INFO | BytesWritten : 2016869904461 | 20260508 02:19:05.267 | INFO | NumEvents : 975106114268 | 20260508 02:19:06.539 | INFO | Analyzing chunk frames = [4464640,4465152) | 20260508 02:19:06.539 | INFO | MemInfo: { | 20260508 02:19:06.539 | INFO | "gpu" : " 17 GB", | 20260508 02:19:06.539 | INFO | "huge" : "", | 20260508 02:19:06.539 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:06.539 | INFO | "malloc" : " 14 GB", | 20260508 02:19:06.539 | INFO | "pinned" : " 29 GB" | 20260508 02:19:06.539 | INFO | } | 20260508 02:19:06.539 | INFO | Perf report for the last 5.14384 seconds: | 20260508 02:19:06.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:06.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:06.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:06.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:06.539 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 02:19:06.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:06.539 | INFO | RT_METRICS : 45.6%, 1.0, 24.0ms, 0 | 20260508 02:19:06.539 | INFO | PRE_HQ : 65.4%, 1.6, 56.0ms, 0 | 20260508 02:19:06.539 | INFO | BAZWRITER : 73.0%, 3.7, 138.1ms, 0 | 20260508 02:19:06.539 | INFO | BAZ_SAMPLING : 5.8%, 1.0, 3.0ms, 0 | 20260508 02:19:06.540 | INFO | Uploaded 36GiB to GPU in 2.54285s (14.1573GiB/s) | 20260508 02:19:06.540 | INFO | Downloaded 38GiB to GPU in 2.69689s (14.0903GiB/s) | 20260508 02:19:11.682 | INFO | Analyzing chunk frames = [4465152,4465664) | 20260508 02:19:11.683 | INFO | MemInfo: { | 20260508 02:19:11.683 | INFO | "gpu" : " 17 GB", | 20260508 02:19:11.683 | INFO | "huge" : "", | 20260508 02:19:11.683 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:11.683 | INFO | "malloc" : " 14 GB", | 20260508 02:19:11.683 | INFO | "pinned" : " 29 GB" | 20260508 02:19:11.683 | INFO | } | 20260508 02:19:11.683 | INFO | Perf report for the last 5.14355 seconds: | 20260508 02:19:11.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:11.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:19:11.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:19:11.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:11.683 | INFO | Uploaded 36GiB to GPU in 2.51102s (14.3368GiB/s) | 20260508 02:19:11.683 | INFO | Downloaded 34GiB to GPU in 2.4416s (13.9253GiB/s) | 20260508 02:19:16.820 | INFO | Analyzing chunk frames = [4465664,4466176) | 20260508 02:19:16.820 | INFO | MemInfo: { | 20260508 02:19:16.820 | INFO | "gpu" : " 17 GB", | 20260508 02:19:16.820 | INFO | "huge" : "", | 20260508 02:19:16.820 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:16.820 | INFO | "malloc" : " 14 GB", | 20260508 02:19:16.821 | INFO | "pinned" : " 29 GB" | 20260508 02:19:16.821 | INFO | } | 20260508 02:19:16.821 | INFO | Perf report for the last 5.13792 seconds: | 20260508 02:19:16.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:16.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.821 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:19:16.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.821 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:19:16.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:16.822 | INFO | Uploaded 36GiB to GPU in 2.51147s (14.3343GiB/s) | 20260508 02:19:16.822 | INFO | Downloaded 34GiB to GPU in 2.51273s (13.5311GiB/s) | 20260508 02:19:21.956 | INFO | Analyzing chunk frames = [4466176,4466688) | 20260508 02:19:21.956 | INFO | MemInfo: { | 20260508 02:19:21.956 | INFO | "gpu" : " 17 GB", | 20260508 02:19:21.956 | INFO | "huge" : "", | 20260508 02:19:21.956 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:21.956 | INFO | "malloc" : " 14 GB", | 20260508 02:19:21.956 | INFO | "pinned" : " 29 GB" | 20260508 02:19:21.956 | INFO | } | 20260508 02:19:21.956 | INFO | Perf report for the last 5.13558 seconds: | 20260508 02:19:21.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:21.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.956 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:19:21.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.956 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:19:21.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:21.957 | INFO | Uploaded 36GiB to GPU in 2.49749s (14.4145GiB/s) | 20260508 02:19:21.957 | INFO | Downloaded 34GiB to GPU in 2.52288s (13.4767GiB/s) | 20260508 02:19:27.097 | INFO | Analyzing chunk frames = [4466688,4467200) | 20260508 02:19:27.097 | INFO | MemInfo: { | 20260508 02:19:27.097 | INFO | "gpu" : " 17 GB", | 20260508 02:19:27.097 | INFO | "huge" : "", | 20260508 02:19:27.097 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:27.097 | INFO | "malloc" : " 14 GB", | 20260508 02:19:27.098 | INFO | "pinned" : " 29 GB" | 20260508 02:19:27.098 | INFO | } | 20260508 02:19:27.098 | INFO | Perf report for the last 5.14132 seconds: | 20260508 02:19:27.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:27.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 02:19:27.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:19:27.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:27.098 | INFO | Uploaded 36GiB to GPU in 2.28118s (15.7813GiB/s) | 20260508 02:19:27.098 | INFO | Downloaded 34GiB to GPU in 2.50769s (13.5583GiB/s) | 20260508 02:19:32.239 | INFO | Analyzing chunk frames = [4467200,4467712) | 20260508 02:19:32.239 | INFO | MemInfo: { | 20260508 02:19:32.239 | INFO | "gpu" : " 17 GB", | 20260508 02:19:32.239 | INFO | "huge" : "", | 20260508 02:19:32.239 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:32.239 | INFO | "malloc" : " 14 GB", | 20260508 02:19:32.240 | INFO | "pinned" : " 29 GB" | 20260508 02:19:32.240 | INFO | } | 20260508 02:19:32.240 | INFO | Perf report for the last 5.142 seconds: | 20260508 02:19:32.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:32.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 02:19:32.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:19:32.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:32.240 | INFO | Uploaded 36GiB to GPU in 2.41309s (14.9187GiB/s) | 20260508 02:19:32.240 | INFO | Downloaded 34GiB to GPU in 2.42671s (14.0107GiB/s) | 20260508 02:19:37.376 | INFO | Analyzing chunk frames = [4467712,4468224) | 20260508 02:19:37.376 | INFO | MemInfo: { | 20260508 02:19:37.376 | INFO | "gpu" : " 17 GB", | 20260508 02:19:37.376 | INFO | "huge" : "", | 20260508 02:19:37.376 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:37.376 | INFO | "malloc" : " 14 GB", | 20260508 02:19:37.376 | INFO | "pinned" : " 29 GB" | 20260508 02:19:37.376 | INFO | } | 20260508 02:19:37.376 | INFO | Perf report for the last 5.13644 seconds: | 20260508 02:19:37.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:37.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:19:37.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:19:37.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:37.376 | INFO | Uploaded 36GiB to GPU in 2.41509s (14.9063GiB/s) | 20260508 02:19:37.376 | INFO | Downloaded 34GiB to GPU in 2.43616s (13.9564GiB/s) | 20260508 02:19:42.511 | INFO | Analyzing chunk frames = [4468224,4468736) | 20260508 02:19:42.511 | INFO | MemInfo: { | 20260508 02:19:42.511 | INFO | "gpu" : " 17 GB", | 20260508 02:19:42.511 | INFO | "huge" : "", | 20260508 02:19:42.511 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:42.511 | INFO | "malloc" : " 14 GB", | 20260508 02:19:42.511 | INFO | "pinned" : " 29 GB" | 20260508 02:19:42.511 | INFO | } | 20260508 02:19:42.511 | INFO | Perf report for the last 5.13476 seconds: | 20260508 02:19:42.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:42.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 02:19:42.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:19:42.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:42.511 | INFO | Uploaded 36GiB to GPU in 2.5288s (14.236GiB/s) | 20260508 02:19:42.511 | INFO | Downloaded 34GiB to GPU in 2.46374s (13.8002GiB/s) | 20260508 02:19:42.608 | INFO | Starting a round of Baz IO | 20260508 02:19:46.051 | INFO | Populating RTMetrics report | 20260508 02:19:46.052 | INFO | RTMetrics report populated | 20260508 02:19:46.052 | INFO | Opening tmp RTMetrics json file | 20260508 02:19:46.056 | INFO | Serializing RTMetrics to json | 20260508 02:19:46.078 | INFO | Serialization done, swapping tmp file now... | 20260508 02:19:46.079 | INFO | RTMetrics done | 20260508 02:19:46.152 | INFO | Finished a round of Baz IO: Wrote 1.642 GiB in 3.543 seconds | 20260508 02:19:46.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.57% / 0.00% / 58.42% | 20260508 02:19:46.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.74085% / 3.97364e-06% / 6.35783e-05% | 20260508 02:19:46.152 | INFO | Baz IO overall: | 20260508 02:19:46.152 | INFO | BytesWritten : 2018633024575 | 20260508 02:19:46.152 | INFO | NumEvents : 975951856746 | 20260508 02:19:47.651 | INFO | Analyzing chunk frames = [4468736,4469248) | 20260508 02:19:47.651 | INFO | MemInfo: { | 20260508 02:19:47.651 | INFO | "gpu" : " 17 GB", | 20260508 02:19:47.651 | INFO | "huge" : "", | 20260508 02:19:47.651 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:47.651 | INFO | "malloc" : " 14 GB", | 20260508 02:19:47.651 | INFO | "pinned" : " 29 GB" | 20260508 02:19:47.651 | INFO | } | 20260508 02:19:47.651 | INFO | Perf report for the last 5.14035 seconds: | 20260508 02:19:47.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:47.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:47.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:47.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:47.651 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 02:19:47.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:47.651 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 02:19:47.651 | INFO | PRE_HQ : 65.9%, 1.7, 56.5ms, 0 | 20260508 02:19:47.651 | INFO | BAZWRITER : 68.5%, 3.7, 129.9ms, 0 | 20260508 02:19:47.651 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:19:47.652 | INFO | Uploaded 36GiB to GPU in 2.49676s (14.4187GiB/s) | 20260508 02:19:47.652 | INFO | Downloaded 38GiB to GPU in 2.68096s (14.174GiB/s) | 20260508 02:19:52.791 | INFO | Analyzing chunk frames = [4469248,4469760) | 20260508 02:19:52.791 | INFO | MemInfo: { | 20260508 02:19:52.791 | INFO | "gpu" : " 17 GB", | 20260508 02:19:52.791 | INFO | "huge" : "", | 20260508 02:19:52.791 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:52.791 | INFO | "malloc" : " 14 GB", | 20260508 02:19:52.791 | INFO | "pinned" : " 29 GB" | 20260508 02:19:52.791 | INFO | } | 20260508 02:19:52.791 | INFO | Perf report for the last 5.1402 seconds: | 20260508 02:19:52.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:52.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.792 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:19:52.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.792 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:19:52.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:52.792 | INFO | Uploaded 36GiB to GPU in 2.52354s (14.2656GiB/s) | 20260508 02:19:52.792 | INFO | Downloaded 34GiB to GPU in 2.43724s (13.9502GiB/s) | 20260508 02:19:57.932 | INFO | Analyzing chunk frames = [4469760,4470272) | 20260508 02:19:57.932 | INFO | MemInfo: { | 20260508 02:19:57.932 | INFO | "gpu" : " 17 GB", | 20260508 02:19:57.932 | INFO | "huge" : "", | 20260508 02:19:57.932 | INFO | "huge_shared" : " 29 GB", | 20260508 02:19:57.932 | INFO | "malloc" : " 14 GB", | 20260508 02:19:57.933 | INFO | "pinned" : " 29 GB" | 20260508 02:19:57.933 | INFO | } | 20260508 02:19:57.933 | INFO | Perf report for the last 5.1412 seconds: | 20260508 02:19:57.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:19:57.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 02:19:57.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:19:57.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:19:57.933 | INFO | Uploaded 36GiB to GPU in 2.47539s (14.5432GiB/s) | 20260508 02:19:57.933 | INFO | Downloaded 34GiB to GPU in 2.50161s (13.5913GiB/s) | 20260508 02:20:03.075 | INFO | Analyzing chunk frames = [4470272,4470784) | 20260508 02:20:03.075 | INFO | MemInfo: { | 20260508 02:20:03.075 | INFO | "gpu" : " 17 GB", | 20260508 02:20:03.075 | INFO | "huge" : "", | 20260508 02:20:03.075 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:03.075 | INFO | "malloc" : " 14 GB", | 20260508 02:20:03.075 | INFO | "pinned" : " 29 GB" | 20260508 02:20:03.075 | INFO | } | 20260508 02:20:03.075 | INFO | Perf report for the last 5.14275 seconds: | 20260508 02:20:03.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:03.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.075 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 02:20:03.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.076 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:20:03.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:03.076 | INFO | Uploaded 36GiB to GPU in 2.50137s (14.3921GiB/s) | 20260508 02:20:03.076 | INFO | Downloaded 34GiB to GPU in 2.51604s (13.5133GiB/s) | 20260508 02:20:08.208 | INFO | Analyzing chunk frames = [4470784,4471296) | 20260508 02:20:08.208 | INFO | MemInfo: { | 20260508 02:20:08.208 | INFO | "gpu" : " 17 GB", | 20260508 02:20:08.208 | INFO | "huge" : "", | 20260508 02:20:08.208 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:08.208 | INFO | "malloc" : " 14 GB", | 20260508 02:20:08.208 | INFO | "pinned" : " 29 GB" | 20260508 02:20:08.208 | INFO | } | 20260508 02:20:08.208 | INFO | Perf report for the last 5.13302 seconds: | 20260508 02:20:08.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:08.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.209 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 02:20:08.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.209 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:20:08.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:08.209 | INFO | Uploaded 36GiB to GPU in 2.35867s (15.2629GiB/s) | 20260508 02:20:08.209 | INFO | Downloaded 34GiB to GPU in 2.56498s (13.2555GiB/s) | 20260508 02:20:13.347 | INFO | Analyzing chunk frames = [4471296,4471808) | 20260508 02:20:13.347 | INFO | MemInfo: { | 20260508 02:20:13.347 | INFO | "gpu" : " 17 GB", | 20260508 02:20:13.347 | INFO | "huge" : "", | 20260508 02:20:13.348 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:13.348 | INFO | "malloc" : " 14 GB", | 20260508 02:20:13.348 | INFO | "pinned" : " 29 GB" | 20260508 02:20:13.348 | INFO | } | 20260508 02:20:13.348 | INFO | Perf report for the last 5.1393 seconds: | 20260508 02:20:13.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:13.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:20:13.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:20:13.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:13.348 | INFO | Uploaded 36GiB to GPU in 2.38658s (15.0843GiB/s) | 20260508 02:20:13.348 | INFO | Downloaded 34GiB to GPU in 2.41581s (14.0739GiB/s) | 20260508 02:20:18.486 | INFO | Analyzing chunk frames = [4471808,4472320) | 20260508 02:20:18.486 | INFO | MemInfo: { | 20260508 02:20:18.486 | INFO | "gpu" : " 17 GB", | 20260508 02:20:18.486 | INFO | "huge" : "", | 20260508 02:20:18.486 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:18.486 | INFO | "malloc" : " 14 GB", | 20260508 02:20:18.486 | INFO | "pinned" : " 29 GB" | 20260508 02:20:18.486 | INFO | } | 20260508 02:20:18.486 | INFO | Perf report for the last 5.1387 seconds: | 20260508 02:20:18.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:18.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.487 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:20:18.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.487 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:20:18.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:18.487 | INFO | Uploaded 36GiB to GPU in 2.42917s (14.8199GiB/s) | 20260508 02:20:18.487 | INFO | Downloaded 34GiB to GPU in 2.4284s (14.001GiB/s) | 20260508 02:20:23.629 | INFO | Analyzing chunk frames = [4472320,4472832) | 20260508 02:20:23.629 | INFO | MemInfo: { | 20260508 02:20:23.629 | INFO | "gpu" : " 17 GB", | 20260508 02:20:23.629 | INFO | "huge" : "", | 20260508 02:20:23.629 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:23.629 | INFO | "malloc" : " 14 GB", | 20260508 02:20:23.629 | INFO | "pinned" : " 29 GB" | 20260508 02:20:23.629 | INFO | } | 20260508 02:20:23.629 | INFO | Perf report for the last 5.14281 seconds: | 20260508 02:20:23.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:23.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.629 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 02:20:23.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.629 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:20:23.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:23.630 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:20:23.630 | INFO | Total mem usage: 16.7 GiB | 20260508 02:20:23.630 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:20:23.630 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260508 02:20:23.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:20:23.630 | INFO | High Water Breakdown: | 20260508 02:20:23.630 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:20:23.630 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:20:23.630 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:20:23.630 | INFO | ------------------------------------------ | 20260508 02:20:23.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:20:23.630 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:20:23.630 | INFO | Total mem usage: 26.5 GiB | 20260508 02:20:23.630 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:20:23.630 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:20:23.630 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:20:23.630 | INFO | High Water Breakdown: | 20260508 02:20:23.630 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:20:23.630 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:20:23.630 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:20:23.630 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:20:23.630 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:20:23.630 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:20:23.630 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:20:23.630 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:20:23.630 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:20:23.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:20:23.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:20:23.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:20:23.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:20:23.630 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:20:23.630 | INFO | ------------------------------------------ | 20260508 02:20:23.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:20:23.630 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:20:23.630 | INFO | Total mem usage: 27.8 GiB | 20260508 02:20:23.630 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:20:23.630 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:20:23.630 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:20:23.630 | INFO | High Water Breakdown: | 20260508 02:20:23.630 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:20:23.630 | INFO | ------------------------------------------ | 20260508 02:20:23.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:20:23.630 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:20:23.630 | INFO | Total mem usage: 15.7 GiB | 20260508 02:20:23.630 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:20:23.630 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:20:23.630 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:20:23.630 | INFO | High Water Breakdown: | 20260508 02:20:23.630 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:20:23.630 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:20:23.630 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:20:23.630 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:20:23.630 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:20:23.630 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:20:23.630 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:20:23.630 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:20:23.630 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:20:23.630 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:20:23.630 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:20:23.630 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:20:23.630 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:20:23.630 | INFO | Uploaded 36GiB to GPU in 2.53643s (14.1932GiB/s) | 20260508 02:20:23.630 | INFO | Downloaded 34GiB to GPU in 2.4619s (13.8105GiB/s) | 20260508 02:20:23.630 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:20:23.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:20:23.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:20:23.631 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:20:23.631 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:20:23.631 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:20:23.631 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:20:23.631 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:20:23.631 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:20:23.631 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:20:23.631 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:20:23.631 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:20:23.631 | INFO | ------------------------------------------ | 20260508 02:20:23.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:20:23.631 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:20:23.631 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:20:23.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:20:23.631 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.22e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.01e-01, 1.14e-02 / 3.96e-03 | 20260508 02:20:23.631 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.50e+00, 5.87e-01 / 1.77e-01 | 20260508 02:20:23.631 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.50e+00, 2.10e-01 / 8.91e-02 | 20260508 02:20:23.631 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.60e+01, 3.94e+00 / 1.56e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.94e+01, 1.67e+00 / 1.38e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.72e+00, 4.24e-01 / 2.14e-01 | 20260508 02:20:23.631 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.23e+00, 2.61e-01 / 1.11e-01 | 20260508 02:20:23.631 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.29e-01 / 3.68e-01, 5.16e-02 / 3.10e-02 | 20260508 02:20:23.631 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:20:23.631 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:20:23.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:20:23.631 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 24.11%, Upload -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 24.49%, Download -- 2.61e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 1.92%, LockUpload -- 2.05e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 3.03%, LockDownload -- 3.22e+00 / 0.00e+00, 4.48e+00 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 15.69%, LockCompute -- 1.67e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:20:23.631 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260508 02:20:23.727 | INFO | Starting a round of Baz IO | 20260508 02:20:27.187 | INFO | Populating RTMetrics report | 20260508 02:20:27.187 | INFO | RTMetrics report populated | 20260508 02:20:27.187 | INFO | Opening tmp RTMetrics json file | 20260508 02:20:27.206 | INFO | Serializing RTMetrics to json | 20260508 02:20:27.209 | INFO | Serialization done, swapping tmp file now... | 20260508 02:20:27.209 | INFO | RTMetrics done | 20260508 02:20:27.268 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.541 seconds | 20260508 02:20:27.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.57% / 0.00% / 58.43% | 20260508 02:20:27.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.738104% / 7.94729e-06% / 6.75519e-05% | 20260508 02:20:27.269 | INFO | Baz IO overall: | 20260508 02:20:27.269 | INFO | BytesWritten : 2020395337142 | 20260508 02:20:27.269 | INFO | NumEvents : 976796996430 | 20260508 02:20:28.771 | INFO | Analyzing chunk frames = [4472832,4473344) | 20260508 02:20:28.771 | INFO | MemInfo: { | 20260508 02:20:28.771 | INFO | "gpu" : " 17 GB", | 20260508 02:20:28.771 | INFO | "huge" : "", | 20260508 02:20:28.771 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:28.771 | INFO | "malloc" : " 14 GB", | 20260508 02:20:28.771 | INFO | "pinned" : " 29 GB" | 20260508 02:20:28.771 | INFO | } | 20260508 02:20:28.771 | INFO | Perf report for the last 5.1416 seconds: | 20260508 02:20:28.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:28.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:28.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:28.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:28.771 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260508 02:20:28.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:28.771 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 02:20:28.771 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 02:20:28.771 | INFO | BAZWRITER : 68.5%, 3.9, 136.4ms, 0 | 20260508 02:20:28.771 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 02:20:28.771 | INFO | Uploaded 36GiB to GPU in 2.53784s (14.1853GiB/s) | 20260508 02:20:28.771 | INFO | Downloaded 38GiB to GPU in 2.69424s (14.1041GiB/s) | 20260508 02:20:33.909 | INFO | Analyzing chunk frames = [4473344,4473856) | 20260508 02:20:33.909 | INFO | MemInfo: { | 20260508 02:20:33.909 | INFO | "gpu" : " 17 GB", | 20260508 02:20:33.909 | INFO | "huge" : "", | 20260508 02:20:33.909 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:33.909 | INFO | "malloc" : " 14 GB", | 20260508 02:20:33.909 | INFO | "pinned" : " 29 GB" | 20260508 02:20:33.909 | INFO | } | 20260508 02:20:33.909 | INFO | Perf report for the last 5.13802 seconds: | 20260508 02:20:33.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:33.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 02:20:33.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260508 02:20:33.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:33.909 | INFO | Uploaded 36GiB to GPU in 2.58679s (13.9169GiB/s) | 20260508 02:20:33.909 | INFO | Downloaded 34GiB to GPU in 2.41851s (14.0582GiB/s) | 20260508 02:20:39.050 | INFO | Analyzing chunk frames = [4473856,4474368) | 20260508 02:20:39.050 | INFO | MemInfo: { | 20260508 02:20:39.050 | INFO | "gpu" : " 17 GB", | 20260508 02:20:39.050 | INFO | "huge" : "", | 20260508 02:20:39.050 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:39.050 | INFO | "malloc" : " 14 GB", | 20260508 02:20:39.050 | INFO | "pinned" : " 29 GB" | 20260508 02:20:39.051 | INFO | } | 20260508 02:20:39.051 | INFO | Perf report for the last 5.14169 seconds: | 20260508 02:20:39.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:39.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:20:39.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:20:39.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:39.051 | INFO | Uploaded 36GiB to GPU in 2.56186s (14.0523GiB/s) | 20260508 02:20:39.051 | INFO | Downloaded 34GiB to GPU in 2.48597s (13.6768GiB/s) | 20260508 02:20:44.189 | INFO | Analyzing chunk frames = [4474368,4474880) | 20260508 02:20:44.189 | INFO | MemInfo: { | 20260508 02:20:44.189 | INFO | "gpu" : " 17 GB", | 20260508 02:20:44.189 | INFO | "huge" : "", | 20260508 02:20:44.189 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:44.189 | INFO | "malloc" : " 14 GB", | 20260508 02:20:44.189 | INFO | "pinned" : " 29 GB" | 20260508 02:20:44.189 | INFO | } | 20260508 02:20:44.189 | INFO | Perf report for the last 5.13875 seconds: | 20260508 02:20:44.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:44.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.189 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:20:44.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.190 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:20:44.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:44.190 | INFO | Uploaded 36GiB to GPU in 2.5758s (13.9763GiB/s) | 20260508 02:20:44.190 | INFO | Downloaded 34GiB to GPU in 2.48526s (13.6807GiB/s) | 20260508 02:20:49.325 | INFO | Analyzing chunk frames = [4474880,4475392) | 20260508 02:20:49.325 | INFO | MemInfo: { | 20260508 02:20:49.325 | INFO | "gpu" : " 17 GB", | 20260508 02:20:49.325 | INFO | "huge" : "", | 20260508 02:20:49.325 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:49.325 | INFO | "malloc" : " 14 GB", | 20260508 02:20:49.325 | INFO | "pinned" : " 29 GB" | 20260508 02:20:49.325 | INFO | } | 20260508 02:20:49.325 | INFO | Perf report for the last 5.13579 seconds: | 20260508 02:20:49.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:49.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.325 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 02:20:49.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.325 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:20:49.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:49.326 | INFO | Uploaded 36GiB to GPU in 2.44859s (14.7023GiB/s) | 20260508 02:20:49.326 | INFO | Downloaded 34GiB to GPU in 2.55585s (13.3028GiB/s) | 20260508 02:20:54.462 | INFO | Analyzing chunk frames = [4475392,4475904) | 20260508 02:20:54.462 | INFO | MemInfo: { | 20260508 02:20:54.462 | INFO | "gpu" : " 17 GB", | 20260508 02:20:54.462 | INFO | "huge" : "", | 20260508 02:20:54.462 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:54.463 | INFO | "malloc" : " 14 GB", | 20260508 02:20:54.463 | INFO | "pinned" : " 29 GB" | 20260508 02:20:54.463 | INFO | } | 20260508 02:20:54.463 | INFO | Perf report for the last 5.13742 seconds: | 20260508 02:20:54.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:54.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:20:54.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:20:54.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:54.463 | INFO | Uploaded 36GiB to GPU in 2.42779s (14.8283GiB/s) | 20260508 02:20:54.463 | INFO | Downloaded 34GiB to GPU in 2.42133s (14.0419GiB/s) | 20260508 02:20:59.602 | INFO | Analyzing chunk frames = [4475904,4476416) | 20260508 02:20:59.602 | INFO | MemInfo: { | 20260508 02:20:59.602 | INFO | "gpu" : " 17 GB", | 20260508 02:20:59.602 | INFO | "huge" : "", | 20260508 02:20:59.602 | INFO | "huge_shared" : " 29 GB", | 20260508 02:20:59.602 | INFO | "malloc" : " 14 GB", | 20260508 02:20:59.602 | INFO | "pinned" : " 29 GB" | 20260508 02:20:59.602 | INFO | } | 20260508 02:20:59.602 | INFO | Perf report for the last 5.13976 seconds: | 20260508 02:20:59.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:20:59.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.602 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:20:59.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.603 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:20:59.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:20:59.603 | INFO | Uploaded 36GiB to GPU in 2.42257s (14.8602GiB/s) | 20260508 02:20:59.603 | INFO | Downloaded 34GiB to GPU in 2.41717s (14.0661GiB/s) | 20260508 02:21:04.749 | INFO | Analyzing chunk frames = [4476416,4476928) | 20260508 02:21:04.749 | INFO | MemInfo: { | 20260508 02:21:04.749 | INFO | "gpu" : " 17 GB", | 20260508 02:21:04.749 | INFO | "huge" : "", | 20260508 02:21:04.749 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:04.749 | INFO | "malloc" : " 14 GB", | 20260508 02:21:04.749 | INFO | "pinned" : " 29 GB" | 20260508 02:21:04.749 | INFO | } | 20260508 02:21:04.749 | INFO | Perf report for the last 5.14678 seconds: | 20260508 02:21:04.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:04.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.749 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 02:21:04.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.749 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:21:04.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:04.750 | INFO | Uploaded 36GiB to GPU in 2.53743s (14.1876GiB/s) | 20260508 02:21:04.750 | INFO | Downloaded 34GiB to GPU in 2.45961s (13.8233GiB/s) | 20260508 02:21:04.860 | INFO | Starting a round of Baz IO | 20260508 02:21:08.296 | INFO | Populating RTMetrics report | 20260508 02:21:08.296 | INFO | RTMetrics report populated | 20260508 02:21:08.296 | INFO | Opening tmp RTMetrics json file | 20260508 02:21:08.317 | INFO | Serializing RTMetrics to json | 20260508 02:21:08.350 | INFO | Serialization done, swapping tmp file now... | 20260508 02:21:08.367 | INFO | RTMetrics done | 20260508 02:21:08.394 | INFO | Finished a round of Baz IO: Wrote 1.642 GiB in 3.533 seconds | 20260508 02:21:08.394 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.57% / 0.00% / 58.43% | 20260508 02:21:08.394 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.738533% / 1.19209e-05% / 4.76837e-05% | 20260508 02:21:08.394 | INFO | Baz IO overall: | 20260508 02:21:08.394 | INFO | BytesWritten : 2022158667554 | 20260508 02:21:08.394 | INFO | NumEvents : 977643093074 | 20260508 02:21:09.887 | INFO | Analyzing chunk frames = [4476928,4477440) | 20260508 02:21:09.887 | INFO | MemInfo: { | 20260508 02:21:09.887 | INFO | "gpu" : " 17 GB", | 20260508 02:21:09.887 | INFO | "huge" : "", | 20260508 02:21:09.888 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:09.888 | INFO | "malloc" : " 14 GB", | 20260508 02:21:09.888 | INFO | "pinned" : " 29 GB" | 20260508 02:21:09.888 | INFO | } | 20260508 02:21:09.888 | INFO | Perf report for the last 5.13839 seconds: | 20260508 02:21:09.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:09.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:09.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:09.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:09.888 | INFO | ANALYSIS : 68.1%, 2.9, 105.1ms, 0 | 20260508 02:21:09.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:09.888 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 02:21:09.888 | INFO | PRE_HQ : 66.3%, 1.7, 59.6ms, 0 | 20260508 02:21:09.888 | INFO | BAZWRITER : 68.3%, 3.9, 136.7ms, 0 | 20260508 02:21:09.888 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 02:21:09.888 | INFO | Uploaded 36GiB to GPU in 2.55788s (14.0742GiB/s) | 20260508 02:21:09.888 | INFO | Downloaded 38GiB to GPU in 2.7019s (14.0642GiB/s) | 20260508 02:21:15.023 | INFO | Analyzing chunk frames = [4477440,4477952) | 20260508 02:21:15.023 | INFO | MemInfo: { | 20260508 02:21:15.023 | INFO | "gpu" : " 17 GB", | 20260508 02:21:15.023 | INFO | "huge" : "", | 20260508 02:21:15.023 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:15.023 | INFO | "malloc" : " 14 GB", | 20260508 02:21:15.023 | INFO | "pinned" : " 29 GB" | 20260508 02:21:15.023 | INFO | } | 20260508 02:21:15.023 | INFO | Perf report for the last 5.13556 seconds: | 20260508 02:21:15.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:15.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.023 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260508 02:21:15.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.023 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:21:15.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:15.024 | INFO | Uploaded 36GiB to GPU in 2.51282s (14.3266GiB/s) | 20260508 02:21:15.024 | INFO | Downloaded 34GiB to GPU in 2.43695s (13.9519GiB/s) | 20260508 02:21:20.159 | INFO | Analyzing chunk frames = [4477952,4478464) | 20260508 02:21:20.159 | INFO | MemInfo: { | 20260508 02:21:20.159 | INFO | "gpu" : " 17 GB", | 20260508 02:21:20.159 | INFO | "huge" : "", | 20260508 02:21:20.159 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:20.159 | INFO | "malloc" : " 14 GB", | 20260508 02:21:20.159 | INFO | "pinned" : " 29 GB" | 20260508 02:21:20.159 | INFO | } | 20260508 02:21:20.159 | INFO | Perf report for the last 5.13621 seconds: | 20260508 02:21:20.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:20.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.159 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:21:20.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.160 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:21:20.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:20.160 | INFO | Uploaded 36GiB to GPU in 2.52827s (14.239GiB/s) | 20260508 02:21:20.160 | INFO | Downloaded 34GiB to GPU in 2.50878s (13.5524GiB/s) | 20260508 02:21:25.302 | INFO | Analyzing chunk frames = [4478464,4478976) | 20260508 02:21:25.302 | INFO | MemInfo: { | 20260508 02:21:25.302 | INFO | "gpu" : " 17 GB", | 20260508 02:21:25.302 | INFO | "huge" : "", | 20260508 02:21:25.302 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:25.302 | INFO | "malloc" : " 14 GB", | 20260508 02:21:25.302 | INFO | "pinned" : " 29 GB" | 20260508 02:21:25.302 | INFO | } | 20260508 02:21:25.302 | INFO | Perf report for the last 5.14304 seconds: | 20260508 02:21:25.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:25.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 02:21:25.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:21:25.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:25.303 | INFO | Uploaded 36GiB to GPU in 2.48777s (14.4708GiB/s) | 20260508 02:21:25.303 | INFO | Downloaded 34GiB to GPU in 2.50389s (13.5789GiB/s) | 20260508 02:21:30.441 | INFO | Analyzing chunk frames = [4478976,4479488) | 20260508 02:21:30.441 | INFO | MemInfo: { | 20260508 02:21:30.441 | INFO | "gpu" : " 17 GB", | 20260508 02:21:30.441 | INFO | "huge" : "", | 20260508 02:21:30.441 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:30.441 | INFO | "malloc" : " 14 GB", | 20260508 02:21:30.441 | INFO | "pinned" : " 29 GB" | 20260508 02:21:30.441 | INFO | } | 20260508 02:21:30.441 | INFO | Perf report for the last 5.13875 seconds: | 20260508 02:21:30.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:30.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 02:21:30.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 02:21:30.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:30.441 | INFO | Uploaded 36GiB to GPU in 2.39127s (15.0548GiB/s) | 20260508 02:21:30.442 | INFO | Downloaded 34GiB to GPU in 2.57525s (13.2026GiB/s) | 20260508 02:21:35.580 | INFO | Analyzing chunk frames = [4479488,4480000) | 20260508 02:21:35.580 | INFO | MemInfo: { | 20260508 02:21:35.580 | INFO | "gpu" : " 17 GB", | 20260508 02:21:35.580 | INFO | "huge" : "", | 20260508 02:21:35.580 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:35.580 | INFO | "malloc" : " 14 GB", | 20260508 02:21:35.580 | INFO | "pinned" : " 29 GB" | 20260508 02:21:35.580 | INFO | } | 20260508 02:21:35.580 | INFO | Perf report for the last 5.13891 seconds: | 20260508 02:21:35.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:35.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 02:21:35.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:21:35.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:35.580 | INFO | Uploaded 36GiB to GPU in 2.37512s (15.1571GiB/s) | 20260508 02:21:35.580 | INFO | Downloaded 34GiB to GPU in 2.42731s (14.0073GiB/s) | 20260508 02:21:40.717 | INFO | Analyzing chunk frames = [4480000,4480512) | 20260508 02:21:40.717 | INFO | MemInfo: { | 20260508 02:21:40.717 | INFO | "gpu" : " 17 GB", | 20260508 02:21:40.717 | INFO | "huge" : "", | 20260508 02:21:40.717 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:40.717 | INFO | "malloc" : " 14 GB", | 20260508 02:21:40.717 | INFO | "pinned" : " 29 GB" | 20260508 02:21:40.717 | INFO | } | 20260508 02:21:40.718 | INFO | Perf report for the last 5.13736 seconds: | 20260508 02:21:40.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:40.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:21:40.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:21:40.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:40.718 | INFO | Uploaded 36GiB to GPU in 2.44376s (14.7314GiB/s) | 20260508 02:21:40.718 | INFO | Downloaded 34GiB to GPU in 2.44165s (13.925GiB/s) | 20260508 02:21:45.861 | INFO | Analyzing chunk frames = [4480512,4481024) | 20260508 02:21:45.861 | INFO | MemInfo: { | 20260508 02:21:45.861 | INFO | "gpu" : " 17 GB", | 20260508 02:21:45.862 | INFO | "huge" : "", | 20260508 02:21:45.862 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:45.862 | INFO | "malloc" : " 14 GB", | 20260508 02:21:45.862 | INFO | "pinned" : " 29 GB" | 20260508 02:21:45.862 | INFO | } | 20260508 02:21:45.862 | INFO | Perf report for the last 5.14421 seconds: | 20260508 02:21:45.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:45.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:21:45.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | PRE_HQ : 34.9%, 1.0, 18.2ms, 0 | 20260508 02:21:45.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:45.862 | INFO | Uploaded 36GiB to GPU in 2.54804s (14.1285GiB/s) | 20260508 02:21:45.862 | INFO | Downloaded 34GiB to GPU in 2.47626s (13.7304GiB/s) | 20260508 02:21:45.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:21:45.863 | INFO | Total mem usage: 16.7 GiB | 20260508 02:21:45.863 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:21:45.863 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.5%) | 20260508 02:21:45.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:21:45.863 | INFO | High Water Breakdown: | 20260508 02:21:45.863 | INFO | 60% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:21:45.863 | INFO | 39.8% : 5.9 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:21:45.863 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:21:45.863 | INFO | ------------------------------------------ | 20260508 02:21:45.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:21:45.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:21:45.863 | INFO | Total mem usage: 26.5 GiB | 20260508 02:21:45.863 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:21:45.863 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:21:45.863 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:21:45.863 | INFO | High Water Breakdown: | 20260508 02:21:45.863 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:21:45.863 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:21:45.863 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:21:45.863 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:21:45.863 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:21:45.863 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:21:45.863 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:21:45.863 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:21:45.863 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:21:45.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:21:45.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:21:45.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:21:45.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:21:45.863 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:21:45.863 | INFO | ------------------------------------------ | 20260508 02:21:45.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:21:45.863 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:21:45.863 | INFO | Total mem usage: 27.8 GiB | 20260508 02:21:45.863 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:21:45.863 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:21:45.863 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:21:45.863 | INFO | High Water Breakdown: | 20260508 02:21:45.863 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:21:45.863 | INFO | ------------------------------------------ | 20260508 02:21:45.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:21:45.864 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:21:45.864 | INFO | Total mem usage: 15.7 GiB | 20260508 02:21:45.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:21:45.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:21:45.864 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:21:45.864 | INFO | High Water Breakdown: | 20260508 02:21:45.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:21:45.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:21:45.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:21:45.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:21:45.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:21:45.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:21:45.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:21:45.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:21:45.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:21:45.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:21:45.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:21:45.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:21:45.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:21:45.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:21:45.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:21:45.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:21:45.864 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:21:45.864 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:21:45.864 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:21:45.864 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:21:45.864 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:21:45.864 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:21:45.864 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:21:45.864 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:21:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:21:45.864 | INFO | ------------------------------------------ | 20260508 02:21:45.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:21:45.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:21:45.864 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:21:45.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:21:45.864 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.40e+01 / 1.83e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.14e-02 / 6.65e-03 | 20260508 02:21:45.864 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.46e+00, 5.84e-01 / 2.13e-01 | 20260508 02:21:45.864 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.04e-01 / 1.04e-01 | 20260508 02:21:45.864 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.60e+01, 4.23e+00 / 2.19e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.64e+00 / 1.57e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.72e+00, 4.24e-01 / 2.04e-01 | 20260508 02:21:45.864 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.27e+00, 2.69e-01 / 1.46e-01 | 20260508 02:21:45.864 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.30e-01 / 3.70e-01, 5.18e-02 / 2.71e-02 | 20260508 02:21:45.864 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:21:45.864 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:21:45.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:21:45.864 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 24.39%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 2.16%, LockUpload -- 2.30e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 2.87%, LockDownload -- 3.06e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 15.45%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:21:45.864 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 3.36e-02 / 0.00e+00 | 20260508 02:21:45.975 | INFO | Starting a round of Baz IO | 20260508 02:21:49.424 | INFO | Populating RTMetrics report | 20260508 02:21:49.425 | INFO | RTMetrics report populated | 20260508 02:21:49.425 | INFO | Opening tmp RTMetrics json file | 20260508 02:21:49.426 | INFO | Serializing RTMetrics to json | 20260508 02:21:49.428 | INFO | Serialization done, swapping tmp file now... | 20260508 02:21:49.428 | INFO | RTMetrics done | 20260508 02:21:49.527 | INFO | Finished a round of Baz IO: Wrote 1.639 GiB in 3.552 seconds | 20260508 02:21:49.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.56% / 0.00% / 58.43% | 20260508 02:21:49.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.731794% / 7.94729e-06% / 6.75519e-05% | 20260508 02:21:49.527 | INFO | Baz IO overall: | 20260508 02:21:49.527 | INFO | BytesWritten : 2023919033273 | 20260508 02:21:49.527 | INFO | NumEvents : 978486926688 | 20260508 02:21:51.005 | INFO | Analyzing chunk frames = [4481024,4481536) | 20260508 02:21:51.005 | INFO | MemInfo: { | 20260508 02:21:51.005 | INFO | "gpu" : " 17 GB", | 20260508 02:21:51.005 | INFO | "huge" : "", | 20260508 02:21:51.005 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:51.005 | INFO | "malloc" : " 14 GB", | 20260508 02:21:51.005 | INFO | "pinned" : " 29 GB" | 20260508 02:21:51.005 | INFO | } | 20260508 02:21:51.006 | INFO | Perf report for the last 5.14376 seconds: | 20260508 02:21:51.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:51.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:51.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:51.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:51.006 | INFO | ANALYSIS : 68.8%, 2.9, 104.4ms, 0 | 20260508 02:21:51.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:51.006 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 02:21:51.006 | INFO | PRE_HQ : 67.7%, 1.8, 63.1ms, 0 | 20260508 02:21:51.006 | INFO | BAZWRITER : 68.7%, 3.8, 133.2ms, 0 | 20260508 02:21:51.006 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:21:51.006 | INFO | Uploaded 36GiB to GPU in 2.54284s (14.1574GiB/s) | 20260508 02:21:51.006 | INFO | Downloaded 38GiB to GPU in 2.66981s (14.2332GiB/s) | 20260508 02:21:56.142 | INFO | Analyzing chunk frames = [4481536,4482048) | 20260508 02:21:56.142 | INFO | MemInfo: { | 20260508 02:21:56.142 | INFO | "gpu" : " 17 GB", | 20260508 02:21:56.142 | INFO | "huge" : "", | 20260508 02:21:56.142 | INFO | "huge_shared" : " 29 GB", | 20260508 02:21:56.142 | INFO | "malloc" : " 14 GB", | 20260508 02:21:56.142 | INFO | "pinned" : " 29 GB" | 20260508 02:21:56.142 | INFO | } | 20260508 02:21:56.142 | INFO | Perf report for the last 5.13635 seconds: | 20260508 02:21:56.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:21:56.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:21:56.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:21:56.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:21:56.142 | INFO | Uploaded 36GiB to GPU in 2.56907s (14.0128GiB/s) | 20260508 02:21:56.142 | INFO | Downloaded 34GiB to GPU in 2.42183s (14.039GiB/s) | 20260508 02:22:01.275 | INFO | Analyzing chunk frames = [4482048,4482560) | 20260508 02:22:01.275 | INFO | MemInfo: { | 20260508 02:22:01.275 | INFO | "gpu" : " 17 GB", | 20260508 02:22:01.275 | INFO | "huge" : "", | 20260508 02:22:01.275 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:01.275 | INFO | "malloc" : " 14 GB", | 20260508 02:22:01.275 | INFO | "pinned" : " 29 GB" | 20260508 02:22:01.275 | INFO | } | 20260508 02:22:01.275 | INFO | Perf report for the last 5.13314 seconds: | 20260508 02:22:01.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:01.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 02:22:01.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:22:01.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:01.275 | INFO | Uploaded 36GiB to GPU in 2.53772s (14.186GiB/s) | 20260508 02:22:01.275 | INFO | Downloaded 34GiB to GPU in 2.50047s (13.5974GiB/s) | 20260508 02:22:06.414 | INFO | Analyzing chunk frames = [4482560,4483072) | 20260508 02:22:06.414 | INFO | MemInfo: { | 20260508 02:22:06.414 | INFO | "gpu" : " 17 GB", | 20260508 02:22:06.414 | INFO | "huge" : "", | 20260508 02:22:06.414 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:06.414 | INFO | "malloc" : " 14 GB", | 20260508 02:22:06.415 | INFO | "pinned" : " 29 GB" | 20260508 02:22:06.415 | INFO | } | 20260508 02:22:06.415 | INFO | Perf report for the last 5.13952 seconds: | 20260508 02:22:06.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:06.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:22:06.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:22:06.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:06.415 | INFO | Uploaded 36GiB to GPU in 2.51661s (14.305GiB/s) | 20260508 02:22:06.415 | INFO | Downloaded 34GiB to GPU in 2.5034s (13.5815GiB/s) | 20260508 02:22:11.555 | INFO | Analyzing chunk frames = [4483072,4483584) | 20260508 02:22:11.555 | INFO | MemInfo: { | 20260508 02:22:11.555 | INFO | "gpu" : " 17 GB", | 20260508 02:22:11.555 | INFO | "huge" : "", | 20260508 02:22:11.555 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:11.555 | INFO | "malloc" : " 14 GB", | 20260508 02:22:11.555 | INFO | "pinned" : " 29 GB" | 20260508 02:22:11.555 | INFO | } | 20260508 02:22:11.556 | INFO | Perf report for the last 5.14089 seconds: | 20260508 02:22:11.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:11.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 02:22:11.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:22:11.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:11.556 | INFO | Uploaded 36GiB to GPU in 2.40399s (14.9751GiB/s) | 20260508 02:22:11.556 | INFO | Downloaded 34GiB to GPU in 2.54227s (13.3739GiB/s) | 20260508 02:22:16.698 | INFO | Analyzing chunk frames = [4483584,4484096) | 20260508 02:22:16.698 | INFO | MemInfo: { | 20260508 02:22:16.698 | INFO | "gpu" : " 17 GB", | 20260508 02:22:16.698 | INFO | "huge" : "", | 20260508 02:22:16.698 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:16.698 | INFO | "malloc" : " 14 GB", | 20260508 02:22:16.698 | INFO | "pinned" : " 29 GB" | 20260508 02:22:16.698 | INFO | } | 20260508 02:22:16.698 | INFO | Perf report for the last 5.1429 seconds: | 20260508 02:22:16.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:16.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.698 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 02:22:16.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.699 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:22:16.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:16.699 | INFO | Uploaded 36GiB to GPU in 2.42875s (14.8224GiB/s) | 20260508 02:22:16.699 | INFO | Downloaded 34GiB to GPU in 2.40135s (14.1587GiB/s) | 20260508 02:22:21.839 | INFO | Analyzing chunk frames = [4484096,4484608) | 20260508 02:22:21.839 | INFO | MemInfo: { | 20260508 02:22:21.839 | INFO | "gpu" : " 17 GB", | 20260508 02:22:21.839 | INFO | "huge" : "", | 20260508 02:22:21.839 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:21.839 | INFO | "malloc" : " 14 GB", | 20260508 02:22:21.840 | INFO | "pinned" : " 29 GB" | 20260508 02:22:21.840 | INFO | } | 20260508 02:22:21.840 | INFO | Perf report for the last 5.14123 seconds: | 20260508 02:22:21.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:21.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 02:22:21.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 02:22:21.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:21.840 | INFO | Uploaded 36GiB to GPU in 2.45396s (14.6701GiB/s) | 20260508 02:22:21.840 | INFO | Downloaded 34GiB to GPU in 2.42034s (14.0476GiB/s) | 20260508 02:22:26.974 | INFO | Analyzing chunk frames = [4484608,4485120) | 20260508 02:22:26.974 | INFO | MemInfo: { | 20260508 02:22:26.974 | INFO | "gpu" : " 17 GB", | 20260508 02:22:26.974 | INFO | "huge" : "", | 20260508 02:22:26.974 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:26.974 | INFO | "malloc" : " 14 GB", | 20260508 02:22:26.974 | INFO | "pinned" : " 29 GB" | 20260508 02:22:26.974 | INFO | } | 20260508 02:22:26.975 | INFO | Perf report for the last 5.13483 seconds: | 20260508 02:22:26.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:26.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 02:22:26.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | PRE_HQ : 35.5%, 1.0, 18.4ms, 0 | 20260508 02:22:26.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:26.975 | INFO | Uploaded 36GiB to GPU in 2.47321s (14.556GiB/s) | 20260508 02:22:26.975 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.975GiB/s) | 20260508 02:22:27.088 | INFO | Starting a round of Baz IO | 20260508 02:22:30.532 | INFO | Populating RTMetrics report | 20260508 02:22:30.532 | INFO | RTMetrics report populated | 20260508 02:22:30.532 | INFO | Opening tmp RTMetrics json file | 20260508 02:22:30.545 | INFO | Serializing RTMetrics to json | 20260508 02:22:30.553 | INFO | Serialization done, swapping tmp file now... | 20260508 02:22:30.554 | INFO | RTMetrics done | 20260508 02:22:30.634 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.546 seconds | 20260508 02:22:30.634 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.56% / 0.00% / 58.44% | 20260508 02:22:30.634 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.732954% / 7.94729e-06% / 7.94729e-05% | 20260508 02:22:30.634 | INFO | Baz IO overall: | 20260508 02:22:30.634 | INFO | BytesWritten : 2025680796486 | 20260508 02:22:30.634 | INFO | NumEvents : 979331907231 | 20260508 02:22:32.114 | INFO | Analyzing chunk frames = [4485120,4485632) | 20260508 02:22:32.114 | INFO | MemInfo: { | 20260508 02:22:32.114 | INFO | "gpu" : " 17 GB", | 20260508 02:22:32.114 | INFO | "huge" : "", | 20260508 02:22:32.114 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:32.114 | INFO | "malloc" : " 14 GB", | 20260508 02:22:32.114 | INFO | "pinned" : " 29 GB" | 20260508 02:22:32.114 | INFO | } | 20260508 02:22:32.114 | INFO | Perf report for the last 5.13993 seconds: | 20260508 02:22:32.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:32.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:32.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:32.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:32.115 | INFO | ANALYSIS : 68.5%, 2.9, 104.6ms, 0 | 20260508 02:22:32.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:32.115 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 02:22:32.115 | INFO | PRE_HQ : 67.3%, 1.8, 61.7ms, 0 | 20260508 02:22:32.115 | INFO | BAZWRITER : 68.5%, 3.7, 131.4ms, 0 | 20260508 02:22:32.115 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 02:22:32.115 | INFO | Uploaded 36GiB to GPU in 2.6141s (13.7715GiB/s) | 20260508 02:22:32.115 | INFO | Downloaded 38GiB to GPU in 2.70228s (14.0622GiB/s) | 20260508 02:22:37.250 | INFO | Analyzing chunk frames = [4485632,4486144) | 20260508 02:22:37.250 | INFO | MemInfo: { | 20260508 02:22:37.250 | INFO | "gpu" : " 17 GB", | 20260508 02:22:37.250 | INFO | "huge" : "", | 20260508 02:22:37.250 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:37.250 | INFO | "malloc" : " 14 GB", | 20260508 02:22:37.250 | INFO | "pinned" : " 29 GB" | 20260508 02:22:37.250 | INFO | } | 20260508 02:22:37.250 | INFO | Perf report for the last 5.13534 seconds: | 20260508 02:22:37.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:37.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 02:22:37.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:22:37.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:37.250 | INFO | Uploaded 36GiB to GPU in 2.55345s (14.0986GiB/s) | 20260508 02:22:37.250 | INFO | Downloaded 34GiB to GPU in 2.42187s (14.0387GiB/s) | 20260508 02:22:42.391 | INFO | Analyzing chunk frames = [4486144,4486656) | 20260508 02:22:42.391 | INFO | MemInfo: { | 20260508 02:22:42.391 | INFO | "gpu" : " 17 GB", | 20260508 02:22:42.391 | INFO | "huge" : "", | 20260508 02:22:42.391 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:42.391 | INFO | "malloc" : " 14 GB", | 20260508 02:22:42.391 | INFO | "pinned" : " 29 GB" | 20260508 02:22:42.391 | INFO | } | 20260508 02:22:42.391 | INFO | Perf report for the last 5.14116 seconds: | 20260508 02:22:42.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:42.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:22:42.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:22:42.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:42.391 | INFO | Uploaded 36GiB to GPU in 2.55828s (14.0719GiB/s) | 20260508 02:22:42.391 | INFO | Downloaded 34GiB to GPU in 2.4867s (13.6727GiB/s) | 20260508 02:22:47.527 | INFO | Analyzing chunk frames = [4486656,4487168) | 20260508 02:22:47.527 | INFO | MemInfo: { | 20260508 02:22:47.527 | INFO | "gpu" : " 17 GB", | 20260508 02:22:47.527 | INFO | "huge" : "", | 20260508 02:22:47.528 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:47.528 | INFO | "malloc" : " 14 GB", | 20260508 02:22:47.528 | INFO | "pinned" : " 29 GB" | 20260508 02:22:47.528 | INFO | } | 20260508 02:22:47.528 | INFO | Perf report for the last 5.13679 seconds: | 20260508 02:22:47.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:47.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:22:47.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:22:47.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:47.528 | INFO | Uploaded 36GiB to GPU in 2.55274s (14.1025GiB/s) | 20260508 02:22:47.528 | INFO | Downloaded 34GiB to GPU in 2.4827s (13.6948GiB/s) | 20260508 02:22:52.671 | INFO | Analyzing chunk frames = [4487168,4487680) | 20260508 02:22:52.671 | INFO | MemInfo: { | 20260508 02:22:52.671 | INFO | "gpu" : " 17 GB", | 20260508 02:22:52.671 | INFO | "huge" : "", | 20260508 02:22:52.671 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:52.671 | INFO | "malloc" : " 14 GB", | 20260508 02:22:52.671 | INFO | "pinned" : " 29 GB" | 20260508 02:22:52.671 | INFO | } | 20260508 02:22:52.671 | INFO | Perf report for the last 5.14349 seconds: | 20260508 02:22:52.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:52.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.671 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 02:22:52.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.671 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:22:52.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:52.672 | INFO | Uploaded 36GiB to GPU in 2.43547s (14.7815GiB/s) | 20260508 02:22:52.672 | INFO | Downloaded 34GiB to GPU in 2.57591s (13.1992GiB/s) | 20260508 02:22:57.812 | INFO | Analyzing chunk frames = [4487680,4488192) | 20260508 02:22:57.812 | INFO | MemInfo: { | 20260508 02:22:57.812 | INFO | "gpu" : " 17 GB", | 20260508 02:22:57.812 | INFO | "huge" : "", | 20260508 02:22:57.812 | INFO | "huge_shared" : " 29 GB", | 20260508 02:22:57.812 | INFO | "malloc" : " 14 GB", | 20260508 02:22:57.812 | INFO | "pinned" : " 29 GB" | 20260508 02:22:57.812 | INFO | } | 20260508 02:22:57.812 | INFO | Perf report for the last 5.14089 seconds: | 20260508 02:22:57.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:22:57.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:22:57.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:22:57.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:22:57.812 | INFO | Uploaded 36GiB to GPU in 2.41869s (14.8841GiB/s) | 20260508 02:22:57.812 | INFO | Downloaded 34GiB to GPU in 2.42471s (14.0223GiB/s) | 20260508 02:23:02.954 | INFO | Analyzing chunk frames = [4488192,4488704) | 20260508 02:23:02.954 | INFO | MemInfo: { | 20260508 02:23:02.954 | INFO | "gpu" : " 17 GB", | 20260508 02:23:02.954 | INFO | "huge" : "", | 20260508 02:23:02.954 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:02.954 | INFO | "malloc" : " 14 GB", | 20260508 02:23:02.954 | INFO | "pinned" : " 29 GB" | 20260508 02:23:02.954 | INFO | } | 20260508 02:23:02.954 | INFO | Perf report for the last 5.14203 seconds: | 20260508 02:23:02.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:02.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 02:23:02.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:23:02.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:02.955 | INFO | Uploaded 36GiB to GPU in 2.4408s (14.7493GiB/s) | 20260508 02:23:02.955 | INFO | Downloaded 34GiB to GPU in 2.42571s (14.0165GiB/s) | 20260508 02:23:08.088 | INFO | Analyzing chunk frames = [4488704,4489216) | 20260508 02:23:08.088 | INFO | MemInfo: { | 20260508 02:23:08.088 | INFO | "gpu" : " 17 GB", | 20260508 02:23:08.088 | INFO | "huge" : "", | 20260508 02:23:08.088 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:08.088 | INFO | "malloc" : " 14 GB", | 20260508 02:23:08.088 | INFO | "pinned" : " 29 GB" | 20260508 02:23:08.089 | INFO | } | 20260508 02:23:08.089 | INFO | Perf report for the last 5.13438 seconds: | 20260508 02:23:08.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:08.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.089 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 02:23:08.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.089 | INFO | PRE_HQ : 35.0%, 1.0, 18.1ms, 0 | 20260508 02:23:08.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:08.090 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:23:08.090 | INFO | Total mem usage: 16.7 GiB | 20260508 02:23:08.090 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:23:08.090 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 02:23:08.090 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:23:08.090 | INFO | High Water Breakdown: | 20260508 02:23:08.090 | INFO | 59.8% : 8.72 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:23:08.090 | INFO | 40.1% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:23:08.090 | INFO | Uploaded 36GiB to GPU in 2.47792s (14.5283GiB/s) | 20260508 02:23:08.090 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:23:08.090 | INFO | ------------------------------------------ | 20260508 02:23:08.090 | INFO | Downloaded 34GiB to GPU in 2.43463s (13.9651GiB/s) | 20260508 02:23:08.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:23:08.090 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:23:08.090 | INFO | Total mem usage: 26.5 GiB | 20260508 02:23:08.090 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:23:08.090 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:23:08.090 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:23:08.090 | INFO | High Water Breakdown: | 20260508 02:23:08.090 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:23:08.090 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:23:08.090 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:23:08.090 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:23:08.090 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:23:08.090 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:23:08.090 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:23:08.090 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:23:08.090 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:23:08.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:23:08.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:23:08.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:23:08.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:23:08.090 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:23:08.090 | INFO | ------------------------------------------ | 20260508 02:23:08.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:23:08.090 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:23:08.090 | INFO | Total mem usage: 27.8 GiB | 20260508 02:23:08.090 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:23:08.090 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:23:08.090 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:23:08.090 | INFO | High Water Breakdown: | 20260508 02:23:08.090 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:23:08.090 | INFO | ------------------------------------------ | 20260508 02:23:08.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:23:08.090 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:23:08.090 | INFO | Total mem usage: 15.7 GiB | 20260508 02:23:08.090 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:23:08.090 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:23:08.090 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:23:08.090 | INFO | High Water Breakdown: | 20260508 02:23:08.090 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:23:08.090 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:23:08.090 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:23:08.090 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:23:08.090 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:23:08.090 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:23:08.090 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:23:08.090 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:23:08.090 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:23:08.090 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:23:08.090 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:23:08.090 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:23:08.090 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:23:08.090 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:23:08.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:23:08.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:23:08.090 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:23:08.091 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:23:08.091 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:23:08.091 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:23:08.091 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:23:08.091 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:23:08.091 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:23:08.091 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:23:08.091 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:23:08.091 | INFO | ------------------------------------------ | 20260508 02:23:08.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:23:08.091 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:23:08.091 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:23:08.091 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:23:08.091 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.41e+01 / 1.61e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.15e-02 / 6.04e-03 | 20260508 02:23:08.091 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.49e+00, 5.84e-01 / 2.09e-01 | 20260508 02:23:08.091 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.23e-01 | 20260508 02:23:08.091 | INFO | PerfReport: 16.09%, DME -- 4.22e+01 / 4.63e+01, 3.93e+00 / 1.99e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.66e+00 / 1.51e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.42e+00 / 2.69e+00, 4.19e-01 / 2.34e-01 | 20260508 02:23:08.091 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.22e+00, 2.66e-01 / 1.08e-01 | 20260508 02:23:08.091 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.30e-01 / 3.68e-01, 5.24e-02 / 2.65e-02 | 20260508 02:23:08.091 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:23:08.091 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:23:08.091 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:23:08.091 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 24.48%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 24.40%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 30.81%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 2.74%, LockDownload -- 2.91e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 15.41%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:23:08.091 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 02:23:08.202 | INFO | Starting a round of Baz IO | 20260508 02:23:11.648 | INFO | Populating RTMetrics report | 20260508 02:23:11.648 | INFO | RTMetrics report populated | 20260508 02:23:11.648 | INFO | Opening tmp RTMetrics json file | 20260508 02:23:11.650 | INFO | Serializing RTMetrics to json | 20260508 02:23:11.652 | INFO | Serialization done, swapping tmp file now... | 20260508 02:23:11.653 | INFO | RTMetrics done | 20260508 02:23:11.756 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.553 seconds | 20260508 02:23:11.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.56% / 0.00% / 58.44% | 20260508 02:23:11.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.720119% / 3.97364e-06% / 6.35783e-05% | 20260508 02:23:11.756 | INFO | Baz IO overall: | 20260508 02:23:11.756 | INFO | BytesWritten : 2027442253180 | 20260508 02:23:11.756 | INFO | NumEvents : 980174828757 | 20260508 02:23:13.226 | INFO | Analyzing chunk frames = [4489216,4489728) | 20260508 02:23:13.226 | INFO | MemInfo: { | 20260508 02:23:13.226 | INFO | "gpu" : " 17 GB", | 20260508 02:23:13.226 | INFO | "huge" : "", | 20260508 02:23:13.226 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:13.226 | INFO | "malloc" : " 14 GB", | 20260508 02:23:13.226 | INFO | "pinned" : " 29 GB" | 20260508 02:23:13.226 | INFO | } | 20260508 02:23:13.227 | INFO | Perf report for the last 5.13787 seconds: | 20260508 02:23:13.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:13.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:13.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:13.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:13.227 | INFO | ANALYSIS : 68.8%, 2.9, 104.0ms, 0 | 20260508 02:23:13.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:13.227 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 02:23:13.227 | INFO | PRE_HQ : 68.0%, 1.8, 62.7ms, 0 | 20260508 02:23:13.227 | INFO | BAZWRITER : 68.7%, 4.0, 140.7ms, 0 | 20260508 02:23:13.227 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 02:23:13.227 | INFO | Uploaded 36GiB to GPU in 2.5151s (14.3136GiB/s) | 20260508 02:23:13.227 | INFO | Downloaded 38GiB to GPU in 2.66235s (14.2731GiB/s) | 20260508 02:23:18.366 | INFO | Analyzing chunk frames = [4489728,4490240) | 20260508 02:23:18.366 | INFO | MemInfo: { | 20260508 02:23:18.366 | INFO | "gpu" : " 17 GB", | 20260508 02:23:18.366 | INFO | "huge" : "", | 20260508 02:23:18.366 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:18.366 | INFO | "malloc" : " 14 GB", | 20260508 02:23:18.366 | INFO | "pinned" : " 29 GB" | 20260508 02:23:18.366 | INFO | } | 20260508 02:23:18.366 | INFO | Perf report for the last 5.13941 seconds: | 20260508 02:23:18.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:18.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 02:23:18.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:23:18.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:18.366 | INFO | Uploaded 36GiB to GPU in 2.56399s (14.0406GiB/s) | 20260508 02:23:18.366 | INFO | Downloaded 34GiB to GPU in 2.4367s (13.9533GiB/s) | 20260508 02:23:23.508 | INFO | Analyzing chunk frames = [4490240,4490752) | 20260508 02:23:23.508 | INFO | MemInfo: { | 20260508 02:23:23.508 | INFO | "gpu" : " 17 GB", | 20260508 02:23:23.508 | INFO | "huge" : "", | 20260508 02:23:23.508 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:23.508 | INFO | "malloc" : " 14 GB", | 20260508 02:23:23.508 | INFO | "pinned" : " 29 GB" | 20260508 02:23:23.508 | INFO | } | 20260508 02:23:23.508 | INFO | Perf report for the last 5.14206 seconds: | 20260508 02:23:23.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:23.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:23:23.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:23:23.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:23.508 | INFO | Uploaded 36GiB to GPU in 2.50984s (14.3436GiB/s) | 20260508 02:23:23.509 | INFO | Downloaded 34GiB to GPU in 2.49835s (13.609GiB/s) | 20260508 02:23:28.647 | INFO | Analyzing chunk frames = [4490752,4491264) | 20260508 02:23:28.647 | INFO | MemInfo: { | 20260508 02:23:28.647 | INFO | "gpu" : " 17 GB", | 20260508 02:23:28.647 | INFO | "huge" : "", | 20260508 02:23:28.647 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:28.647 | INFO | "malloc" : " 14 GB", | 20260508 02:23:28.647 | INFO | "pinned" : " 29 GB" | 20260508 02:23:28.647 | INFO | } | 20260508 02:23:28.647 | INFO | Perf report for the last 5.13897 seconds: | 20260508 02:23:28.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:28.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.647 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:23:28.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.647 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:23:28.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:28.648 | INFO | Uploaded 36GiB to GPU in 2.49647s (14.4204GiB/s) | 20260508 02:23:28.648 | INFO | Downloaded 34GiB to GPU in 2.48875s (13.6615GiB/s) | 20260508 02:23:33.783 | INFO | Analyzing chunk frames = [4491264,4491776) | 20260508 02:23:33.783 | INFO | MemInfo: { | 20260508 02:23:33.783 | INFO | "gpu" : " 17 GB", | 20260508 02:23:33.783 | INFO | "huge" : "", | 20260508 02:23:33.783 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:33.784 | INFO | "malloc" : " 14 GB", | 20260508 02:23:33.784 | INFO | "pinned" : " 29 GB" | 20260508 02:23:33.784 | INFO | } | 20260508 02:23:33.784 | INFO | Perf report for the last 5.13665 seconds: | 20260508 02:23:33.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:33.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 02:23:33.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:23:33.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:33.784 | INFO | Uploaded 36GiB to GPU in 2.38982s (15.0639GiB/s) | 20260508 02:23:33.784 | INFO | Downloaded 34GiB to GPU in 2.54392s (13.3652GiB/s) | 20260508 02:23:38.925 | INFO | Analyzing chunk frames = [4491776,4492288) | 20260508 02:23:38.925 | INFO | MemInfo: { | 20260508 02:23:38.925 | INFO | "gpu" : " 17 GB", | 20260508 02:23:38.925 | INFO | "huge" : "", | 20260508 02:23:38.925 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:38.925 | INFO | "malloc" : " 14 GB", | 20260508 02:23:38.925 | INFO | "pinned" : " 29 GB" | 20260508 02:23:38.925 | INFO | } | 20260508 02:23:38.925 | INFO | Perf report for the last 5.14141 seconds: | 20260508 02:23:38.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:38.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 02:23:38.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:23:38.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:38.925 | INFO | Uploaded 36GiB to GPU in 2.44155s (14.7447GiB/s) | 20260508 02:23:38.925 | INFO | Downloaded 34GiB to GPU in 2.41537s (14.0765GiB/s) | 20260508 02:23:44.069 | INFO | Analyzing chunk frames = [4492288,4492800) | 20260508 02:23:44.069 | INFO | MemInfo: { | 20260508 02:23:44.069 | INFO | "gpu" : " 17 GB", | 20260508 02:23:44.069 | INFO | "huge" : "", | 20260508 02:23:44.069 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:44.069 | INFO | "malloc" : " 14 GB", | 20260508 02:23:44.069 | INFO | "pinned" : " 29 GB" | 20260508 02:23:44.069 | INFO | } | 20260508 02:23:44.069 | INFO | Perf report for the last 5.14419 seconds: | 20260508 02:23:44.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:44.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.069 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 02:23:44.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.069 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:23:44.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:44.070 | INFO | Uploaded 36GiB to GPU in 2.46933s (14.5789GiB/s) | 20260508 02:23:44.070 | INFO | Downloaded 34GiB to GPU in 2.41057s (14.1045GiB/s) | 20260508 02:23:49.204 | INFO | Analyzing chunk frames = [4492800,4493312) | 20260508 02:23:49.204 | INFO | MemInfo: { | 20260508 02:23:49.204 | INFO | "gpu" : " 17 GB", | 20260508 02:23:49.204 | INFO | "huge" : "", | 20260508 02:23:49.204 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:49.204 | INFO | "malloc" : " 14 GB", | 20260508 02:23:49.204 | INFO | "pinned" : " 29 GB" | 20260508 02:23:49.204 | INFO | } | 20260508 02:23:49.204 | INFO | Perf report for the last 5.13504 seconds: | 20260508 02:23:49.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:49.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.204 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 02:23:49.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.205 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 02:23:49.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:49.205 | INFO | Uploaded 36GiB to GPU in 2.49948s (14.403GiB/s) | 20260508 02:23:49.206 | INFO | Downloaded 34GiB to GPU in 2.42801s (14.0032GiB/s) | 20260508 02:23:49.318 | INFO | Starting a round of Baz IO | 20260508 02:23:52.767 | INFO | Populating RTMetrics report | 20260508 02:23:52.768 | INFO | RTMetrics report populated | 20260508 02:23:52.768 | INFO | Opening tmp RTMetrics json file | 20260508 02:23:52.775 | INFO | Serializing RTMetrics to json | 20260508 02:23:52.787 | INFO | Serialization done, swapping tmp file now... | 20260508 02:23:52.788 | INFO | RTMetrics done | 20260508 02:23:52.878 | INFO | Finished a round of Baz IO: Wrote 1.639 GiB in 3.56 seconds | 20260508 02:23:52.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.55% / 0.00% / 58.44% | 20260508 02:23:52.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.715729% / 2.38419e-05% / 4.76837e-05% | 20260508 02:23:52.878 | INFO | Baz IO overall: | 20260508 02:23:52.878 | INFO | BytesWritten : 2029201901592 | 20260508 02:23:52.878 | INFO | NumEvents : 981016240578 | 20260508 02:23:54.343 | INFO | Analyzing chunk frames = [4493312,4493824) | 20260508 02:23:54.343 | INFO | MemInfo: { | 20260508 02:23:54.343 | INFO | "gpu" : " 17 GB", | 20260508 02:23:54.343 | INFO | "huge" : "", | 20260508 02:23:54.344 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:54.344 | INFO | "malloc" : " 14 GB", | 20260508 02:23:54.344 | INFO | "pinned" : " 29 GB" | 20260508 02:23:54.344 | INFO | } | 20260508 02:23:54.344 | INFO | Perf report for the last 5.13929 seconds: | 20260508 02:23:54.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:54.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:54.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:54.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:54.344 | INFO | ANALYSIS : 68.6%, 2.9, 104.6ms, 0 | 20260508 02:23:54.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:54.344 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260508 02:23:54.344 | INFO | PRE_HQ : 67.5%, 1.8, 63.2ms, 0 | 20260508 02:23:54.344 | INFO | BAZWRITER : 68.9%, 3.9, 136.4ms, 0 | 20260508 02:23:54.344 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:23:54.344 | INFO | Uploaded 36GiB to GPU in 2.63119s (13.682GiB/s) | 20260508 02:23:54.344 | INFO | Downloaded 38GiB to GPU in 2.71133s (14.0153GiB/s) | 20260508 02:23:59.479 | INFO | Analyzing chunk frames = [4493824,4494336) | 20260508 02:23:59.479 | INFO | MemInfo: { | 20260508 02:23:59.479 | INFO | "gpu" : " 17 GB", | 20260508 02:23:59.479 | INFO | "huge" : "", | 20260508 02:23:59.479 | INFO | "huge_shared" : " 29 GB", | 20260508 02:23:59.479 | INFO | "malloc" : " 14 GB", | 20260508 02:23:59.479 | INFO | "pinned" : " 29 GB" | 20260508 02:23:59.479 | INFO | } | 20260508 02:23:59.479 | INFO | Perf report for the last 5.13538 seconds: | 20260508 02:23:59.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:23:59.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.479 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:23:59.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.479 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:23:59.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:23:59.480 | INFO | Uploaded 36GiB to GPU in 2.58138s (13.946GiB/s) | 20260508 02:23:59.480 | INFO | Downloaded 34GiB to GPU in 2.42145s (14.0412GiB/s) | 20260508 02:24:04.624 | INFO | Analyzing chunk frames = [4494336,4494848) | 20260508 02:24:04.624 | INFO | MemInfo: { | 20260508 02:24:04.624 | INFO | "gpu" : " 17 GB", | 20260508 02:24:04.624 | INFO | "huge" : "", | 20260508 02:24:04.624 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:04.624 | INFO | "malloc" : " 14 GB", | 20260508 02:24:04.624 | INFO | "pinned" : " 29 GB" | 20260508 02:24:04.624 | INFO | } | 20260508 02:24:04.624 | INFO | Perf report for the last 5.14487 seconds: | 20260508 02:24:04.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:04.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:24:04.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:24:04.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:04.624 | INFO | Uploaded 36GiB to GPU in 2.5677s (14.0203GiB/s) | 20260508 02:24:04.624 | INFO | Downloaded 34GiB to GPU in 2.48571s (13.6782GiB/s) | 20260508 02:24:09.767 | INFO | Analyzing chunk frames = [4494848,4495360) | 20260508 02:24:09.767 | INFO | MemInfo: { | 20260508 02:24:09.767 | INFO | "gpu" : " 17 GB", | 20260508 02:24:09.767 | INFO | "huge" : "", | 20260508 02:24:09.767 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:09.767 | INFO | "malloc" : " 14 GB", | 20260508 02:24:09.767 | INFO | "pinned" : " 29 GB" | 20260508 02:24:09.767 | INFO | } | 20260508 02:24:09.767 | INFO | Perf report for the last 5.14314 seconds: | 20260508 02:24:09.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:09.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.767 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:24:09.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.767 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 02:24:09.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:09.768 | INFO | Uploaded 36GiB to GPU in 2.5599s (14.063GiB/s) | 20260508 02:24:09.768 | INFO | Downloaded 34GiB to GPU in 2.49008s (13.6542GiB/s) | 20260508 02:24:14.906 | INFO | Analyzing chunk frames = [4495360,4495872) | 20260508 02:24:14.906 | INFO | MemInfo: { | 20260508 02:24:14.906 | INFO | "gpu" : " 17 GB", | 20260508 02:24:14.906 | INFO | "huge" : "", | 20260508 02:24:14.906 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:14.906 | INFO | "malloc" : " 14 GB", | 20260508 02:24:14.906 | INFO | "pinned" : " 29 GB" | 20260508 02:24:14.906 | INFO | } | 20260508 02:24:14.906 | INFO | Perf report for the last 5.13908 seconds: | 20260508 02:24:14.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:14.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.906 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:24:14.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.906 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:24:14.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:14.907 | INFO | Uploaded 36GiB to GPU in 2.43695s (14.7726GiB/s) | 20260508 02:24:14.907 | INFO | Downloaded 34GiB to GPU in 2.58678s (13.1438GiB/s) | 20260508 02:24:20.041 | INFO | Analyzing chunk frames = [4495872,4496384) | 20260508 02:24:20.041 | INFO | MemInfo: { | 20260508 02:24:20.041 | INFO | "gpu" : " 17 GB", | 20260508 02:24:20.041 | INFO | "huge" : "", | 20260508 02:24:20.041 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:20.041 | INFO | "malloc" : " 14 GB", | 20260508 02:24:20.041 | INFO | "pinned" : " 29 GB" | 20260508 02:24:20.041 | INFO | } | 20260508 02:24:20.041 | INFO | Perf report for the last 5.13525 seconds: | 20260508 02:24:20.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:20.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.041 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 02:24:20.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.042 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:24:20.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:20.042 | INFO | Uploaded 36GiB to GPU in 2.43522s (14.7831GiB/s) | 20260508 02:24:20.042 | INFO | Downloaded 34GiB to GPU in 2.41255s (14.093GiB/s) | 20260508 02:24:25.177 | INFO | Analyzing chunk frames = [4496384,4496896) | 20260508 02:24:25.177 | INFO | MemInfo: { | 20260508 02:24:25.177 | INFO | "gpu" : " 17 GB", | 20260508 02:24:25.177 | INFO | "huge" : "", | 20260508 02:24:25.177 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:25.177 | INFO | "malloc" : " 14 GB", | 20260508 02:24:25.177 | INFO | "pinned" : " 29 GB" | 20260508 02:24:25.177 | INFO | } | 20260508 02:24:25.178 | INFO | Perf report for the last 5.13605 seconds: | 20260508 02:24:25.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:25.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:24:25.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:24:25.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:25.178 | INFO | Uploaded 36GiB to GPU in 2.47891s (14.5225GiB/s) | 20260508 02:24:25.178 | INFO | Downloaded 34GiB to GPU in 2.42298s (14.0323GiB/s) | 20260508 02:24:30.319 | INFO | Analyzing chunk frames = [4496896,4497408) | 20260508 02:24:30.319 | INFO | MemInfo: { | 20260508 02:24:30.319 | INFO | "gpu" : " 17 GB", | 20260508 02:24:30.319 | INFO | "huge" : "", | 20260508 02:24:30.319 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:30.319 | INFO | "malloc" : " 14 GB", | 20260508 02:24:30.319 | INFO | "pinned" : " 29 GB" | 20260508 02:24:30.319 | INFO | } | 20260508 02:24:30.319 | INFO | Perf report for the last 5.14179 seconds: | 20260508 02:24:30.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:30.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.319 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:24:30.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.319 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:24:30.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:30.320 | INFO | Uploaded 36GiB to GPU in 2.56715s (14.0233GiB/s) | 20260508 02:24:30.320 | INFO | Downloaded 34GiB to GPU in 2.45012s (13.8769GiB/s) | 20260508 02:24:30.320 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:24:30.320 | INFO | Total mem usage: 16.7 GiB | 20260508 02:24:30.320 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:24:30.320 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 02:24:30.320 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:24:30.320 | INFO | High Water Breakdown: | 20260508 02:24:30.320 | INFO | 60.2% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:24:30.320 | INFO | 39.7% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:24:30.320 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:24:30.320 | INFO | ------------------------------------------ | 20260508 02:24:30.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:24:30.320 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:24:30.320 | INFO | Total mem usage: 26.5 GiB | 20260508 02:24:30.320 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:24:30.320 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 02:24:30.320 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 02:24:30.320 | INFO | High Water Breakdown: | 20260508 02:24:30.320 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:24:30.320 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:24:30.320 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:24:30.320 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:24:30.320 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:24:30.320 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:24:30.320 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:24:30.320 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:24:30.320 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:24:30.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:24:30.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:24:30.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:24:30.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:24:30.320 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:24:30.320 | INFO | ------------------------------------------ | 20260508 02:24:30.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:24:30.320 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:24:30.320 | INFO | Total mem usage: 27.8 GiB | 20260508 02:24:30.320 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:24:30.320 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:24:30.320 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:24:30.320 | INFO | High Water Breakdown: | 20260508 02:24:30.320 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:24:30.320 | INFO | ------------------------------------------ | 20260508 02:24:30.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:24:30.320 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:24:30.320 | INFO | Total mem usage: 15.7 GiB | 20260508 02:24:30.320 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:24:30.320 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:24:30.320 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:24:30.320 | INFO | High Water Breakdown: | 20260508 02:24:30.320 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:24:30.320 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:24:30.320 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:24:30.320 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:24:30.320 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:24:30.320 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:24:30.320 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:24:30.320 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:24:30.320 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:24:30.320 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:24:30.320 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:24:30.320 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:24:30.320 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:24:30.320 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:24:30.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:24:30.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:24:30.320 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:24:30.320 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:24:30.320 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:24:30.320 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:24:30.320 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:24:30.320 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:24:30.320 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:24:30.320 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:24:30.320 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:24:30.320 | INFO | ------------------------------------------ | 20260508 02:24:30.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:24:30.320 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:24:30.320 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:24:30.320 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:24:30.320 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.63e+01, 1.43e+01 / 1.70e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.02e-01, 1.25e-02 / 4.43e-03 | 20260508 02:24:30.320 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.47e+00, 6.15e-01 / 2.07e-01 | 20260508 02:24:30.320 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 1.05e-01 | 20260508 02:24:30.320 | INFO | PerfReport: 16.11%, DME -- 4.21e+01 / 4.64e+01, 4.12e+00 / 2.08e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.63e+00 / 1.48e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.42e+00 / 2.70e+00, 4.24e-01 / 2.28e-01 | 20260508 02:24:30.320 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.22e+00, 2.68e-01 / 1.08e-01 | 20260508 02:24:30.320 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.28e-01 / 3.62e-01, 5.22e-02 / 2.45e-02 | 20260508 02:24:30.320 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:24:30.320 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:24:30.320 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:24:30.320 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 24.56%, Upload -- 2.61e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 2.31%, LockUpload -- 2.45e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 2.67%, LockDownload -- 2.84e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 15.33%, LockCompute -- 1.63e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:24:30.320 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260508 02:24:30.417 | INFO | Starting a round of Baz IO | 20260508 02:24:33.870 | INFO | Populating RTMetrics report | 20260508 02:24:33.870 | INFO | RTMetrics report populated | 20260508 02:24:33.870 | INFO | Opening tmp RTMetrics json file | 20260508 02:24:33.950 | INFO | Serializing RTMetrics to json | 20260508 02:24:33.952 | INFO | Serialization done, swapping tmp file now... | 20260508 02:24:33.953 | INFO | RTMetrics done | 20260508 02:24:33.979 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.562 seconds | 20260508 02:24:33.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.55% / 0.00% / 58.45% | 20260508 02:24:33.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.718749% / 0% / 3.97364e-05% | 20260508 02:24:33.979 | INFO | Baz IO overall: | 20260508 02:24:33.979 | INFO | BytesWritten : 2030962364664 | 20260508 02:24:33.979 | INFO | NumEvents : 981859307371 | 20260508 02:24:35.455 | INFO | Analyzing chunk frames = [4497408,4497920) | 20260508 02:24:35.455 | INFO | MemInfo: { | 20260508 02:24:35.455 | INFO | "gpu" : " 17 GB", | 20260508 02:24:35.455 | INFO | "huge" : "", | 20260508 02:24:35.455 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:35.455 | INFO | "malloc" : " 14 GB", | 20260508 02:24:35.455 | INFO | "pinned" : " 29 GB" | 20260508 02:24:35.455 | INFO | } | 20260508 02:24:35.455 | INFO | Perf report for the last 5.13612 seconds: | 20260508 02:24:35.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:35.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:35.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:35.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:35.455 | INFO | ANALYSIS : 68.2%, 3.0, 106.6ms, 0 | 20260508 02:24:35.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:35.455 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260508 02:24:35.455 | INFO | PRE_HQ : 65.3%, 1.7, 55.9ms, 0 | 20260508 02:24:35.455 | INFO | BAZWRITER : 69.0%, 3.9, 138.4ms, 0 | 20260508 02:24:35.455 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 02:24:35.456 | INFO | Uploaded 36GiB to GPU in 2.58666s (13.9175GiB/s) | 20260508 02:24:35.456 | INFO | Downloaded 38GiB to GPU in 2.67599s (14.2004GiB/s) | 20260508 02:24:40.602 | INFO | Analyzing chunk frames = [4497920,4498432) | 20260508 02:24:40.602 | INFO | MemInfo: { | 20260508 02:24:40.602 | INFO | "gpu" : " 17 GB", | 20260508 02:24:40.602 | INFO | "huge" : "", | 20260508 02:24:40.602 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:40.602 | INFO | "malloc" : " 14 GB", | 20260508 02:24:40.602 | INFO | "pinned" : " 29 GB" | 20260508 02:24:40.602 | INFO | } | 20260508 02:24:40.602 | INFO | Perf report for the last 5.14673 seconds: | 20260508 02:24:40.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:40.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 02:24:40.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:24:40.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:40.602 | INFO | Uploaded 36GiB to GPU in 2.59629s (13.8659GiB/s) | 20260508 02:24:40.602 | INFO | Downloaded 34GiB to GPU in 2.48557s (13.6789GiB/s) | 20260508 02:24:45.735 | INFO | Analyzing chunk frames = [4498432,4498944) | 20260508 02:24:45.735 | INFO | MemInfo: { | 20260508 02:24:45.735 | INFO | "gpu" : " 17 GB", | 20260508 02:24:45.735 | INFO | "huge" : "", | 20260508 02:24:45.735 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:45.735 | INFO | "malloc" : " 14 GB", | 20260508 02:24:45.735 | INFO | "pinned" : " 29 GB" | 20260508 02:24:45.735 | INFO | } | 20260508 02:24:45.735 | INFO | Perf report for the last 5.13292 seconds: | 20260508 02:24:45.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:45.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260508 02:24:45.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:24:45.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:45.735 | INFO | Uploaded 36GiB to GPU in 2.59314s (13.8828GiB/s) | 20260508 02:24:45.735 | INFO | Downloaded 34GiB to GPU in 2.4939s (13.6333GiB/s) | 20260508 02:24:50.880 | INFO | Analyzing chunk frames = [4498944,4499456) | 20260508 02:24:50.880 | INFO | MemInfo: { | 20260508 02:24:50.880 | INFO | "gpu" : " 17 GB", | 20260508 02:24:50.880 | INFO | "huge" : "", | 20260508 02:24:50.880 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:50.880 | INFO | "malloc" : " 14 GB", | 20260508 02:24:50.880 | INFO | "pinned" : " 29 GB" | 20260508 02:24:50.880 | INFO | } | 20260508 02:24:50.880 | INFO | Perf report for the last 5.14537 seconds: | 20260508 02:24:50.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:50.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:24:50.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:24:50.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:50.881 | INFO | Uploaded 36GiB to GPU in 2.57007s (14.0074GiB/s) | 20260508 02:24:50.881 | INFO | Downloaded 34GiB to GPU in 2.48266s (13.695GiB/s) | 20260508 02:24:56.017 | INFO | Analyzing chunk frames = [4499456,4499968) | 20260508 02:24:56.017 | INFO | MemInfo: { | 20260508 02:24:56.017 | INFO | "gpu" : " 17 GB", | 20260508 02:24:56.017 | INFO | "huge" : "", | 20260508 02:24:56.017 | INFO | "huge_shared" : " 29 GB", | 20260508 02:24:56.017 | INFO | "malloc" : " 14 GB", | 20260508 02:24:56.017 | INFO | "pinned" : " 29 GB" | 20260508 02:24:56.017 | INFO | } | 20260508 02:24:56.017 | INFO | Perf report for the last 5.13671 seconds: | 20260508 02:24:56.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:24:56.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.017 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 02:24:56.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.017 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:24:56.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:24:56.018 | INFO | Uploaded 36GiB to GPU in 2.43282s (14.7976GiB/s) | 20260508 02:24:56.018 | INFO | Downloaded 34GiB to GPU in 2.57655s (13.1959GiB/s) | 20260508 02:25:01.155 | INFO | Analyzing chunk frames = [4499968,4500480) | 20260508 02:25:01.155 | INFO | MemInfo: { | 20260508 02:25:01.155 | INFO | "gpu" : " 17 GB", | 20260508 02:25:01.155 | INFO | "huge" : "", | 20260508 02:25:01.155 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:01.155 | INFO | "malloc" : " 14 GB", | 20260508 02:25:01.155 | INFO | "pinned" : " 29 GB" | 20260508 02:25:01.155 | INFO | } | 20260508 02:25:01.155 | INFO | Perf report for the last 5.13805 seconds: | 20260508 02:25:01.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:01.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.155 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:25:01.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.156 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 02:25:01.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:01.156 | INFO | Uploaded 36GiB to GPU in 2.42594s (14.8396GiB/s) | 20260508 02:25:01.156 | INFO | Downloaded 34GiB to GPU in 2.40873s (14.1153GiB/s) | 20260508 02:25:06.293 | INFO | Analyzing chunk frames = [4500480,4500992) | 20260508 02:25:06.293 | INFO | MemInfo: { | 20260508 02:25:06.293 | INFO | "gpu" : " 17 GB", | 20260508 02:25:06.293 | INFO | "huge" : "", | 20260508 02:25:06.293 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:06.293 | INFO | "malloc" : " 14 GB", | 20260508 02:25:06.293 | INFO | "pinned" : " 29 GB" | 20260508 02:25:06.293 | INFO | } | 20260508 02:25:06.293 | INFO | Perf report for the last 5.13787 seconds: | 20260508 02:25:06.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:06.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.293 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:25:06.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.293 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 02:25:06.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:06.294 | INFO | Uploaded 36GiB to GPU in 2.47771s (14.5296GiB/s) | 20260508 02:25:06.294 | INFO | Downloaded 34GiB to GPU in 2.42613s (14.0141GiB/s) | 20260508 02:25:11.433 | INFO | Analyzing chunk frames = [4500992,4501504) | 20260508 02:25:11.433 | INFO | MemInfo: { | 20260508 02:25:11.433 | INFO | "gpu" : " 17 GB", | 20260508 02:25:11.433 | INFO | "huge" : "", | 20260508 02:25:11.433 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:11.433 | INFO | "malloc" : " 14 GB", | 20260508 02:25:11.433 | INFO | "pinned" : " 29 GB" | 20260508 02:25:11.433 | INFO | } | 20260508 02:25:11.433 | INFO | Perf report for the last 5.13998 seconds: | 20260508 02:25:11.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:11.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:25:11.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 02:25:11.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:11.433 | INFO | Uploaded 36GiB to GPU in 2.58247s (13.9401GiB/s) | 20260508 02:25:11.433 | INFO | Downloaded 34GiB to GPU in 2.4614s (13.8133GiB/s) | 20260508 02:25:11.531 | INFO | Starting a round of Baz IO | 20260508 02:25:14.975 | INFO | Populating RTMetrics report | 20260508 02:25:14.976 | INFO | RTMetrics report populated | 20260508 02:25:14.976 | INFO | Opening tmp RTMetrics json file | 20260508 02:25:14.977 | INFO | Serializing RTMetrics to json | 20260508 02:25:14.984 | INFO | Serialization done, swapping tmp file now... | 20260508 02:25:15.006 | INFO | RTMetrics done | 20260508 02:25:15.089 | INFO | Finished a round of Baz IO: Wrote 1.638 GiB in 3.558 seconds | 20260508 02:25:15.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.55% / 0.00% / 58.45% | 20260508 02:25:15.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.718709% / 7.94729e-06% / 4.37101e-05% | 20260508 02:25:15.089 | INFO | Baz IO overall: | 20260508 02:25:15.089 | INFO | BytesWritten : 2032721193196 | 20260508 02:25:15.089 | INFO | NumEvents : 982701776000 | 20260508 02:25:16.571 | INFO | Analyzing chunk frames = [4501504,4502016) | 20260508 02:25:16.571 | INFO | MemInfo: { | 20260508 02:25:16.571 | INFO | "gpu" : " 17 GB", | 20260508 02:25:16.571 | INFO | "huge" : "", | 20260508 02:25:16.571 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:16.571 | INFO | "malloc" : " 14 GB", | 20260508 02:25:16.571 | INFO | "pinned" : " 29 GB" | 20260508 02:25:16.571 | INFO | } | 20260508 02:25:16.571 | INFO | Perf report for the last 5.13781 seconds: | 20260508 02:25:16.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:16.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:16.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:16.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:16.571 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 02:25:16.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:16.571 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 02:25:16.571 | INFO | PRE_HQ : 65.9%, 1.6, 56.2ms, 0 | 20260508 02:25:16.571 | INFO | BAZWRITER : 68.8%, 3.8, 135.4ms, 0 | 20260508 02:25:16.571 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:25:16.571 | INFO | Uploaded 36GiB to GPU in 2.60238s (13.8335GiB/s) | 20260508 02:25:16.571 | INFO | Downloaded 38GiB to GPU in 2.69875s (14.0806GiB/s) | 20260508 02:25:21.716 | INFO | Analyzing chunk frames = [4502016,4502528) | 20260508 02:25:21.716 | INFO | MemInfo: { | 20260508 02:25:21.716 | INFO | "gpu" : " 17 GB", | 20260508 02:25:21.716 | INFO | "huge" : "", | 20260508 02:25:21.716 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:21.716 | INFO | "malloc" : " 14 GB", | 20260508 02:25:21.716 | INFO | "pinned" : " 29 GB" | 20260508 02:25:21.716 | INFO | } | 20260508 02:25:21.716 | INFO | Perf report for the last 5.14541 seconds: | 20260508 02:25:21.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:21.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.716 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 02:25:21.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.717 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:25:21.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:21.717 | INFO | Uploaded 36GiB to GPU in 2.56833s (14.0169GiB/s) | 20260508 02:25:21.717 | INFO | Downloaded 34GiB to GPU in 2.42345s (14.0296GiB/s) | 20260508 02:25:26.854 | INFO | Analyzing chunk frames = [4502528,4503040) | 20260508 02:25:26.854 | INFO | MemInfo: { | 20260508 02:25:26.854 | INFO | "gpu" : " 17 GB", | 20260508 02:25:26.854 | INFO | "huge" : "", | 20260508 02:25:26.854 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:26.854 | INFO | "malloc" : " 14 GB", | 20260508 02:25:26.854 | INFO | "pinned" : " 29 GB" | 20260508 02:25:26.854 | INFO | } | 20260508 02:25:26.854 | INFO | Perf report for the last 5.13739 seconds: | 20260508 02:25:26.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:26.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 02:25:26.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:25:26.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:26.854 | INFO | Uploaded 36GiB to GPU in 2.58691s (13.9162GiB/s) | 20260508 02:25:26.854 | INFO | Downloaded 34GiB to GPU in 2.49184s (13.6445GiB/s) | 20260508 02:25:31.988 | INFO | Analyzing chunk frames = [4503040,4503552) | 20260508 02:25:31.988 | INFO | MemInfo: { | 20260508 02:25:31.988 | INFO | "gpu" : " 17 GB", | 20260508 02:25:31.988 | INFO | "huge" : "", | 20260508 02:25:31.988 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:31.988 | INFO | "malloc" : " 14 GB", | 20260508 02:25:31.988 | INFO | "pinned" : " 29 GB" | 20260508 02:25:31.988 | INFO | } | 20260508 02:25:31.988 | INFO | Perf report for the last 5.13403 seconds: | 20260508 02:25:31.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:31.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 02:25:31.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:25:31.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:31.988 | INFO | Uploaded 36GiB to GPU in 2.56229s (14.0499GiB/s) | 20260508 02:25:31.989 | INFO | Downloaded 34GiB to GPU in 2.49456s (13.6297GiB/s) | 20260508 02:25:37.134 | INFO | Analyzing chunk frames = [4503552,4504064) | 20260508 02:25:37.134 | INFO | MemInfo: { | 20260508 02:25:37.134 | INFO | "gpu" : " 17 GB", | 20260508 02:25:37.134 | INFO | "huge" : "", | 20260508 02:25:37.134 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:37.134 | INFO | "malloc" : " 14 GB", | 20260508 02:25:37.134 | INFO | "pinned" : " 29 GB" | 20260508 02:25:37.134 | INFO | } | 20260508 02:25:37.134 | INFO | Perf report for the last 5.1465 seconds: | 20260508 02:25:37.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:37.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.134 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 02:25:37.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.135 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:25:37.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:37.135 | INFO | Uploaded 36GiB to GPU in 2.34154s (15.3745GiB/s) | 20260508 02:25:37.135 | INFO | Downloaded 34GiB to GPU in 2.49642s (13.6195GiB/s) | 20260508 02:25:42.270 | INFO | Analyzing chunk frames = [4504064,4504576) | 20260508 02:25:42.270 | INFO | MemInfo: { | 20260508 02:25:42.270 | INFO | "gpu" : " 17 GB", | 20260508 02:25:42.270 | INFO | "huge" : "", | 20260508 02:25:42.270 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:42.270 | INFO | "malloc" : " 14 GB", | 20260508 02:25:42.270 | INFO | "pinned" : " 29 GB" | 20260508 02:25:42.270 | INFO | } | 20260508 02:25:42.270 | INFO | Perf report for the last 5.13573 seconds: | 20260508 02:25:42.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:42.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:25:42.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:25:42.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:42.270 | INFO | Uploaded 36GiB to GPU in 2.40798s (14.9503GiB/s) | 20260508 02:25:42.270 | INFO | Downloaded 34GiB to GPU in 2.41761s (14.0635GiB/s) | 20260508 02:25:47.410 | INFO | Analyzing chunk frames = [4504576,4505088) | 20260508 02:25:47.410 | INFO | MemInfo: { | 20260508 02:25:47.410 | INFO | "gpu" : " 17 GB", | 20260508 02:25:47.410 | INFO | "huge" : "", | 20260508 02:25:47.410 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:47.410 | INFO | "malloc" : " 14 GB", | 20260508 02:25:47.410 | INFO | "pinned" : " 29 GB" | 20260508 02:25:47.410 | INFO | } | 20260508 02:25:47.410 | INFO | Perf report for the last 5.14019 seconds: | 20260508 02:25:47.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:47.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.410 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:25:47.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.411 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260508 02:25:47.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:47.411 | INFO | Uploaded 36GiB to GPU in 2.43109s (14.8082GiB/s) | 20260508 02:25:47.412 | INFO | Downloaded 34GiB to GPU in 2.42333s (14.0303GiB/s) | 20260508 02:25:52.547 | INFO | Analyzing chunk frames = [4505088,4505600) | 20260508 02:25:52.547 | INFO | MemInfo: { | 20260508 02:25:52.547 | INFO | "gpu" : " 17 GB", | 20260508 02:25:52.547 | INFO | "huge" : "", | 20260508 02:25:52.547 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:52.547 | INFO | "malloc" : " 14 GB", | 20260508 02:25:52.547 | INFO | "pinned" : " 29 GB" | 20260508 02:25:52.547 | INFO | } | 20260508 02:25:52.547 | INFO | Perf report for the last 5.13713 seconds: | 20260508 02:25:52.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:52.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:25:52.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | PRE_HQ : 34.6%, 1.0, 17.9ms, 0 | 20260508 02:25:52.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:52.548 | INFO | Uploaded 36GiB to GPU in 2.46402s (14.6103GiB/s) | 20260508 02:25:52.548 | INFO | Downloaded 34GiB to GPU in 2.43414s (13.968GiB/s) | 20260508 02:25:52.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:25:52.548 | INFO | Total mem usage: 16.7 GiB | 20260508 02:25:52.548 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:25:52.548 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 02:25:52.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:25:52.548 | INFO | High Water Breakdown: | 20260508 02:25:52.548 | INFO | 60.2% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:25:52.548 | INFO | 39.7% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:25:52.548 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:25:52.548 | INFO | ------------------------------------------ | 20260508 02:25:52.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:25:52.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:25:52.548 | INFO | Total mem usage: 26.5 GiB | 20260508 02:25:52.548 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:25:52.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:25:52.548 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:25:52.548 | INFO | High Water Breakdown: | 20260508 02:25:52.548 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:25:52.548 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:25:52.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:25:52.548 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:25:52.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:25:52.548 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:25:52.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:25:52.548 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:25:52.548 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:25:52.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:25:52.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:25:52.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:25:52.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:25:52.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:25:52.548 | INFO | ------------------------------------------ | 20260508 02:25:52.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:25:52.549 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:25:52.549 | INFO | Total mem usage: 27.8 GiB | 20260508 02:25:52.549 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:25:52.549 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:25:52.549 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:25:52.549 | INFO | High Water Breakdown: | 20260508 02:25:52.549 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:25:52.549 | INFO | ------------------------------------------ | 20260508 02:25:52.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:25:52.549 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:25:52.549 | INFO | Total mem usage: 15.7 GiB | 20260508 02:25:52.549 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:25:52.549 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:25:52.549 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:25:52.549 | INFO | High Water Breakdown: | 20260508 02:25:52.549 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:25:52.549 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:25:52.549 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:25:52.549 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:25:52.549 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:25:52.549 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:25:52.549 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:25:52.549 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:25:52.549 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:25:52.549 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:25:52.549 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:25:52.549 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:25:52.549 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:25:52.549 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:25:52.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:25:52.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:25:52.549 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:25:52.549 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:25:52.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:25:52.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:25:52.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:25:52.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:25:52.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:25:52.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:25:52.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:25:52.549 | INFO | ------------------------------------------ | 20260508 02:25:52.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:25:52.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:25:52.549 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:25:52.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:25:52.549 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.42e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.03e-01, 1.29e-02 / 7.66e-03 | 20260508 02:25:52.549 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.48e+00, 5.84e-01 / 2.13e-01 | 20260508 02:25:52.549 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 9.69e-02 | 20260508 02:25:52.549 | INFO | PerfReport: 16.08%, DME -- 4.20e+01 / 4.61e+01, 4.14e+00 / 1.78e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.60e+00 / 1.42e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.42e+00 / 2.70e+00, 4.23e-01 / 1.87e-01 | 20260508 02:25:52.549 | INFO | PerfReport: 3.29%, Metrics -- 1.07e+00 / 1.27e+00, 2.65e-01 / 1.20e-01 | 20260508 02:25:52.549 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.31e-01 / 3.67e-01, 5.27e-02 / 2.56e-02 | 20260508 02:25:52.549 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:25:52.549 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:25:52.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:25:52.549 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 24.56%, Upload -- 2.62e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 24.35%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 30.62%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 2.50%, LockUpload -- 2.67e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 2.52%, LockDownload -- 2.69e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 15.44%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:25:52.549 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260508 02:25:52.660 | INFO | Starting a round of Baz IO | 20260508 02:25:56.111 | INFO | Populating RTMetrics report | 20260508 02:25:56.112 | INFO | RTMetrics report populated | 20260508 02:25:56.112 | INFO | Opening tmp RTMetrics json file | 20260508 02:25:56.115 | INFO | Serializing RTMetrics to json | 20260508 02:25:56.119 | INFO | Serialization done, swapping tmp file now... | 20260508 02:25:56.122 | INFO | RTMetrics done | 20260508 02:25:56.211 | INFO | Finished a round of Baz IO: Wrote 1.638 GiB in 3.55 seconds | 20260508 02:25:56.211 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.54% / 0.00% / 58.45% | 20260508 02:25:56.211 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.717775% / 7.94729e-06% / 4.37101e-05% | 20260508 02:25:56.211 | INFO | Baz IO overall: | 20260508 02:25:56.211 | INFO | BytesWritten : 2034479586007 | 20260508 02:25:56.211 | INFO | NumEvents : 983543990088 | 20260508 02:25:57.688 | INFO | Analyzing chunk frames = [4505600,4506112) | 20260508 02:25:57.688 | INFO | MemInfo: { | 20260508 02:25:57.688 | INFO | "gpu" : " 17 GB", | 20260508 02:25:57.688 | INFO | "huge" : "", | 20260508 02:25:57.688 | INFO | "huge_shared" : " 29 GB", | 20260508 02:25:57.688 | INFO | "malloc" : " 14 GB", | 20260508 02:25:57.688 | INFO | "pinned" : " 29 GB" | 20260508 02:25:57.688 | INFO | } | 20260508 02:25:57.688 | INFO | Perf report for the last 5.14068 seconds: | 20260508 02:25:57.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:25:57.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:57.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:57.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:57.688 | INFO | ANALYSIS : 68.9%, 2.9, 104.4ms, 0 | 20260508 02:25:57.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:25:57.688 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 02:25:57.688 | INFO | PRE_HQ : 67.9%, 1.8, 62.7ms, 0 | 20260508 02:25:57.688 | INFO | BAZWRITER : 68.6%, 3.7, 131.8ms, 0 | 20260508 02:25:57.688 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 02:25:57.689 | INFO | Uploaded 36GiB to GPU in 2.53038s (14.2271GiB/s) | 20260508 02:25:57.689 | INFO | Downloaded 38GiB to GPU in 2.67867s (14.1861GiB/s) | 20260508 02:26:02.833 | INFO | Analyzing chunk frames = [4506112,4506624) | 20260508 02:26:02.833 | INFO | MemInfo: { | 20260508 02:26:02.833 | INFO | "gpu" : " 17 GB", | 20260508 02:26:02.833 | INFO | "huge" : "", | 20260508 02:26:02.833 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:02.833 | INFO | "malloc" : " 14 GB", | 20260508 02:26:02.833 | INFO | "pinned" : " 29 GB" | 20260508 02:26:02.833 | INFO | } | 20260508 02:26:02.833 | INFO | Perf report for the last 5.14506 seconds: | 20260508 02:26:02.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:02.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.833 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 02:26:02.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.833 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:26:02.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:02.834 | INFO | Uploaded 36GiB to GPU in 2.54418s (14.1499GiB/s) | 20260508 02:26:02.834 | INFO | Downloaded 34GiB to GPU in 2.43081s (13.9871GiB/s) | 20260508 02:26:07.970 | INFO | Analyzing chunk frames = [4506624,4507136) | 20260508 02:26:07.970 | INFO | MemInfo: { | 20260508 02:26:07.970 | INFO | "gpu" : " 17 GB", | 20260508 02:26:07.970 | INFO | "huge" : "", | 20260508 02:26:07.970 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:07.970 | INFO | "malloc" : " 14 GB", | 20260508 02:26:07.970 | INFO | "pinned" : " 29 GB" | 20260508 02:26:07.970 | INFO | } | 20260508 02:26:07.970 | INFO | Perf report for the last 5.13661 seconds: | 20260508 02:26:07.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:07.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 02:26:07.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:26:07.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:07.970 | INFO | Uploaded 36GiB to GPU in 2.55125s (14.1107GiB/s) | 20260508 02:26:07.970 | INFO | Downloaded 34GiB to GPU in 2.50118s (13.5936GiB/s) | 20260508 02:26:13.103 | INFO | Analyzing chunk frames = [4507136,4507648) | 20260508 02:26:13.103 | INFO | MemInfo: { | 20260508 02:26:13.103 | INFO | "gpu" : " 17 GB", | 20260508 02:26:13.103 | INFO | "huge" : "", | 20260508 02:26:13.103 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:13.103 | INFO | "malloc" : " 14 GB", | 20260508 02:26:13.103 | INFO | "pinned" : " 29 GB" | 20260508 02:26:13.103 | INFO | } | 20260508 02:26:13.103 | INFO | Perf report for the last 5.13311 seconds: | 20260508 02:26:13.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:13.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | ANALYSIS : 67.3%, 3.0, 104.0ms, 0 | 20260508 02:26:13.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:26:13.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:13.103 | INFO | Uploaded 36GiB to GPU in 2.54106s (14.1673GiB/s) | 20260508 02:26:13.103 | INFO | Downloaded 34GiB to GPU in 2.49366s (13.6346GiB/s) | 20260508 02:26:18.241 | INFO | Analyzing chunk frames = [4507648,4508160) | 20260508 02:26:18.241 | INFO | MemInfo: { | 20260508 02:26:18.241 | INFO | "gpu" : " 17 GB", | 20260508 02:26:18.241 | INFO | "huge" : "", | 20260508 02:26:18.241 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:18.241 | INFO | "malloc" : " 14 GB", | 20260508 02:26:18.241 | INFO | "pinned" : " 29 GB" | 20260508 02:26:18.241 | INFO | } | 20260508 02:26:18.242 | INFO | Perf report for the last 5.13855 seconds: | 20260508 02:26:18.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:18.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 02:26:18.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:26:18.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:18.242 | INFO | Uploaded 36GiB to GPU in 2.4382s (14.765GiB/s) | 20260508 02:26:18.242 | INFO | Downloaded 34GiB to GPU in 2.55276s (13.3189GiB/s) | 20260508 02:26:23.382 | INFO | Analyzing chunk frames = [4508160,4508672) | 20260508 02:26:23.382 | INFO | MemInfo: { | 20260508 02:26:23.382 | INFO | "gpu" : " 17 GB", | 20260508 02:26:23.382 | INFO | "huge" : "", | 20260508 02:26:23.382 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:23.382 | INFO | "malloc" : " 14 GB", | 20260508 02:26:23.382 | INFO | "pinned" : " 29 GB" | 20260508 02:26:23.382 | INFO | } | 20260508 02:26:23.382 | INFO | Perf report for the last 5.14076 seconds: | 20260508 02:26:23.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:23.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.382 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:26:23.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.382 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:26:23.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:23.383 | INFO | Uploaded 36GiB to GPU in 2.39971s (15.0018GiB/s) | 20260508 02:26:23.383 | INFO | Downloaded 34GiB to GPU in 2.42499s (14.0207GiB/s) | 20260508 02:26:28.525 | INFO | Analyzing chunk frames = [4508672,4509184) | 20260508 02:26:28.525 | INFO | MemInfo: { | 20260508 02:26:28.525 | INFO | "gpu" : " 17 GB", | 20260508 02:26:28.525 | INFO | "huge" : "", | 20260508 02:26:28.525 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:28.525 | INFO | "malloc" : " 14 GB", | 20260508 02:26:28.525 | INFO | "pinned" : " 29 GB" | 20260508 02:26:28.525 | INFO | } | 20260508 02:26:28.526 | INFO | Perf report for the last 5.1432 seconds: | 20260508 02:26:28.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:28.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | ANALYSIS : 67.2%, 3.0, 105.0ms, 0 | 20260508 02:26:28.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:26:28.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:28.526 | INFO | Uploaded 36GiB to GPU in 2.53712s (14.1893GiB/s) | 20260508 02:26:28.526 | INFO | Downloaded 34GiB to GPU in 2.46377s (13.8GiB/s) | 20260508 02:26:33.669 | INFO | Analyzing chunk frames = [4509184,4509696) | 20260508 02:26:33.669 | INFO | MemInfo: { | 20260508 02:26:33.669 | INFO | "gpu" : " 17 GB", | 20260508 02:26:33.669 | INFO | "huge" : "", | 20260508 02:26:33.669 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:33.669 | INFO | "malloc" : " 14 GB", | 20260508 02:26:33.669 | INFO | "pinned" : " 29 GB" | 20260508 02:26:33.669 | INFO | } | 20260508 02:26:33.669 | INFO | Perf report for the last 5.14394 seconds: | 20260508 02:26:33.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:33.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 02:26:33.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 02:26:33.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:33.670 | INFO | Uploaded 36GiB to GPU in 2.45593s (14.6584GiB/s) | 20260508 02:26:33.670 | INFO | Downloaded 34GiB to GPU in 2.43063s (13.9882GiB/s) | 20260508 02:26:33.766 | INFO | Starting a round of Baz IO | 20260508 02:26:37.206 | INFO | Populating RTMetrics report | 20260508 02:26:37.207 | INFO | RTMetrics report populated | 20260508 02:26:37.207 | INFO | Opening tmp RTMetrics json file | 20260508 02:26:37.241 | INFO | Serializing RTMetrics to json | 20260508 02:26:37.307 | INFO | Serialization done, swapping tmp file now... | 20260508 02:26:37.308 | INFO | RTMetrics done | 20260508 02:26:37.327 | INFO | Finished a round of Baz IO: Wrote 1.639 GiB in 3.56 seconds | 20260508 02:26:37.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.54% / 0.00% / 58.46% | 20260508 02:26:37.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.719261% / 3.97364e-06% / 6.35783e-05% | 20260508 02:26:37.327 | INFO | Baz IO overall: | 20260508 02:26:37.327 | INFO | BytesWritten : 2036239449498 | 20260508 02:26:37.327 | INFO | NumEvents : 984387696546 | 20260508 02:26:38.803 | INFO | Analyzing chunk frames = [4509696,4510208) | 20260508 02:26:38.803 | INFO | MemInfo: { | 20260508 02:26:38.803 | INFO | "gpu" : " 17 GB", | 20260508 02:26:38.803 | INFO | "huge" : "", | 20260508 02:26:38.803 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:38.803 | INFO | "malloc" : " 14 GB", | 20260508 02:26:38.804 | INFO | "pinned" : " 29 GB" | 20260508 02:26:38.804 | INFO | } | 20260508 02:26:38.804 | INFO | Perf report for the last 5.13412 seconds: | 20260508 02:26:38.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:38.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:38.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:38.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:38.804 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 02:26:38.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:38.804 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 02:26:38.804 | INFO | PRE_HQ : 65.8%, 1.6, 55.9ms, 0 | 20260508 02:26:38.804 | INFO | BAZWRITER : 68.9%, 3.6, 127.5ms, 0 | 20260508 02:26:38.804 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260508 02:26:38.804 | INFO | Uploaded 36GiB to GPU in 2.55889s (14.0686GiB/s) | 20260508 02:26:38.804 | INFO | Downloaded 38GiB to GPU in 2.6912s (14.1201GiB/s) | 20260508 02:26:43.945 | INFO | Analyzing chunk frames = [4510208,4510720) | 20260508 02:26:43.945 | INFO | MemInfo: { | 20260508 02:26:43.945 | INFO | "gpu" : " 17 GB", | 20260508 02:26:43.945 | INFO | "huge" : "", | 20260508 02:26:43.945 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:43.945 | INFO | "malloc" : " 14 GB", | 20260508 02:26:43.945 | INFO | "pinned" : " 29 GB" | 20260508 02:26:43.945 | INFO | } | 20260508 02:26:43.945 | INFO | Perf report for the last 5.14129 seconds: | 20260508 02:26:43.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:43.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 02:26:43.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:26:43.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:43.945 | INFO | Uploaded 36GiB to GPU in 2.56188s (14.0522GiB/s) | 20260508 02:26:43.945 | INFO | Downloaded 34GiB to GPU in 2.42542s (14.0182GiB/s) | 20260508 02:26:49.084 | INFO | Analyzing chunk frames = [4510720,4511232) | 20260508 02:26:49.084 | INFO | MemInfo: { | 20260508 02:26:49.084 | INFO | "gpu" : " 17 GB", | 20260508 02:26:49.084 | INFO | "huge" : "", | 20260508 02:26:49.084 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:49.084 | INFO | "malloc" : " 14 GB", | 20260508 02:26:49.084 | INFO | "pinned" : " 29 GB" | 20260508 02:26:49.084 | INFO | } | 20260508 02:26:49.084 | INFO | Perf report for the last 5.13903 seconds: | 20260508 02:26:49.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:49.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:26:49.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:26:49.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:49.084 | INFO | Uploaded 36GiB to GPU in 2.54956s (14.1201GiB/s) | 20260508 02:26:49.084 | INFO | Downloaded 34GiB to GPU in 2.49254s (13.6407GiB/s) | 20260508 02:26:54.220 | INFO | Analyzing chunk frames = [4511232,4511744) | 20260508 02:26:54.220 | INFO | MemInfo: { | 20260508 02:26:54.220 | INFO | "gpu" : " 17 GB", | 20260508 02:26:54.220 | INFO | "huge" : "", | 20260508 02:26:54.220 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:54.220 | INFO | "malloc" : " 14 GB", | 20260508 02:26:54.220 | INFO | "pinned" : " 29 GB" | 20260508 02:26:54.220 | INFO | } | 20260508 02:26:54.220 | INFO | Perf report for the last 5.13621 seconds: | 20260508 02:26:54.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:54.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 02:26:54.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:26:54.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:54.220 | INFO | Uploaded 36GiB to GPU in 2.5468s (14.1354GiB/s) | 20260508 02:26:54.220 | INFO | Downloaded 34GiB to GPU in 2.49589s (13.6224GiB/s) | 20260508 02:26:59.362 | INFO | Analyzing chunk frames = [4511744,4512256) | 20260508 02:26:59.362 | INFO | MemInfo: { | 20260508 02:26:59.362 | INFO | "gpu" : " 17 GB", | 20260508 02:26:59.362 | INFO | "huge" : "", | 20260508 02:26:59.362 | INFO | "huge_shared" : " 29 GB", | 20260508 02:26:59.362 | INFO | "malloc" : " 14 GB", | 20260508 02:26:59.362 | INFO | "pinned" : " 29 GB" | 20260508 02:26:59.362 | INFO | } | 20260508 02:26:59.362 | INFO | Perf report for the last 5.14186 seconds: | 20260508 02:26:59.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:26:59.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:26:59.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:26:59.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:26:59.362 | INFO | Uploaded 36GiB to GPU in 2.37781s (15.14GiB/s) | 20260508 02:26:59.363 | INFO | Downloaded 34GiB to GPU in 2.50413s (13.5776GiB/s) | 20260508 02:27:04.497 | INFO | Analyzing chunk frames = [4512256,4512768) | 20260508 02:27:04.497 | INFO | MemInfo: { | 20260508 02:27:04.497 | INFO | "gpu" : " 17 GB", | 20260508 02:27:04.497 | INFO | "huge" : "", | 20260508 02:27:04.497 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:04.497 | INFO | "malloc" : " 14 GB", | 20260508 02:27:04.497 | INFO | "pinned" : " 29 GB" | 20260508 02:27:04.497 | INFO | } | 20260508 02:27:04.497 | INFO | Perf report for the last 5.13531 seconds: | 20260508 02:27:04.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:04.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.497 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:27:04.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.498 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:27:04.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:04.498 | INFO | Uploaded 36GiB to GPU in 2.41332s (14.9172GiB/s) | 20260508 02:27:04.498 | INFO | Downloaded 34GiB to GPU in 2.41188s (14.0969GiB/s) | 20260508 02:27:09.640 | INFO | Analyzing chunk frames = [4512768,4513280) | 20260508 02:27:09.640 | INFO | MemInfo: { | 20260508 02:27:09.640 | INFO | "gpu" : " 17 GB", | 20260508 02:27:09.640 | INFO | "huge" : "", | 20260508 02:27:09.640 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:09.640 | INFO | "malloc" : " 14 GB", | 20260508 02:27:09.640 | INFO | "pinned" : " 29 GB" | 20260508 02:27:09.640 | INFO | } | 20260508 02:27:09.640 | INFO | Perf report for the last 5.14265 seconds: | 20260508 02:27:09.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:09.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:27:09.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260508 02:27:09.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:09.640 | INFO | Uploaded 36GiB to GPU in 2.43601s (14.7783GiB/s) | 20260508 02:27:09.640 | INFO | Downloaded 34GiB to GPU in 2.42592s (14.0153GiB/s) | 20260508 02:27:14.779 | INFO | Analyzing chunk frames = [4513280,4513792) | 20260508 02:27:14.779 | INFO | MemInfo: { | 20260508 02:27:14.779 | INFO | "gpu" : " 17 GB", | 20260508 02:27:14.779 | INFO | "huge" : "", | 20260508 02:27:14.779 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:14.779 | INFO | "malloc" : " 14 GB", | 20260508 02:27:14.780 | INFO | "pinned" : " 29 GB" | 20260508 02:27:14.780 | INFO | } | 20260508 02:27:14.780 | INFO | Perf report for the last 5.13961 seconds: | 20260508 02:27:14.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:14.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:27:14.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:27:14.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:14.780 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:27:14.780 | INFO | Total mem usage: 16.7 GiB | 20260508 02:27:14.780 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:27:14.780 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 02:27:14.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:27:14.780 | INFO | High Water Breakdown: | 20260508 02:27:14.780 | INFO | 59.8% : 8.73 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:27:14.780 | INFO | 40% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:27:14.780 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:27:14.780 | INFO | ------------------------------------------ | 20260508 02:27:14.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:27:14.780 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:27:14.780 | INFO | Total mem usage: 26.5 GiB | 20260508 02:27:14.780 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:27:14.780 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:27:14.780 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:27:14.780 | INFO | High Water Breakdown: | 20260508 02:27:14.780 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:27:14.780 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:27:14.780 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:27:14.780 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:27:14.780 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:27:14.780 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:27:14.780 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:27:14.780 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:27:14.780 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:27:14.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:27:14.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:27:14.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:27:14.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:27:14.780 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:27:14.780 | INFO | ------------------------------------------ | 20260508 02:27:14.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:27:14.780 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:27:14.780 | INFO | Total mem usage: 27.8 GiB | 20260508 02:27:14.780 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:27:14.780 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:27:14.780 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:27:14.780 | INFO | High Water Breakdown: | 20260508 02:27:14.780 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:27:14.780 | INFO | ------------------------------------------ | 20260508 02:27:14.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:27:14.780 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:27:14.780 | INFO | Total mem usage: 15.7 GiB | 20260508 02:27:14.780 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:27:14.780 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:27:14.780 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:27:14.780 | INFO | High Water Breakdown: | 20260508 02:27:14.780 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:27:14.780 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:27:14.780 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:27:14.780 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:27:14.780 | INFO | Uploaded 36GiB to GPU in 2.55547s (14.0874GiB/s) | 20260508 02:27:14.781 | INFO | Downloaded 34GiB to GPU in 2.4537s (13.8567GiB/s) | 20260508 02:27:14.781 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:27:14.781 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:27:14.781 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:27:14.781 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:27:14.781 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:27:14.781 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:27:14.781 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:27:14.781 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:27:14.781 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:27:14.781 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:27:14.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:27:14.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:27:14.781 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:27:14.781 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:27:14.781 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:27:14.781 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:27:14.781 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:27:14.781 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:27:14.781 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:27:14.781 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:27:14.781 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:27:14.781 | INFO | ------------------------------------------ | 20260508 02:27:14.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:27:14.781 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:27:14.781 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:27:14.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:27:14.781 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.40e+01 / 1.69e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.28e-02 / 6.03e-03 | 20260508 02:27:14.781 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.44e+00, 5.84e-01 / 2.23e-01 | 20260508 02:27:14.781 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.02e-01 | 20260508 02:27:14.781 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.59e+01, 3.99e+00 / 2.00e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.62e+00 / 1.56e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.42e+00 / 2.68e+00, 4.22e-01 / 2.23e-01 | 20260508 02:27:14.781 | INFO | PerfReport: 3.29%, Metrics -- 1.07e+00 / 1.25e+00, 2.65e-01 / 1.58e-01 | 20260508 02:27:14.781 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.29e-01 / 3.63e-01, 5.09e-02 / 2.19e-02 | 20260508 02:27:14.781 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:27:14.781 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:27:14.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:27:14.781 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 30.67%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 2.20%, LockUpload -- 2.34e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 02:27:14.781 | INFO | PerfReport: 2.83%, LockDownload -- 3.02e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 02:27:14.782 | INFO | PerfReport: 15.44%, LockCompute -- 1.64e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:27:14.782 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260508 02:27:14.877 | INFO | Starting a round of Baz IO | 20260508 02:27:18.324 | INFO | Populating RTMetrics report | 20260508 02:27:18.324 | INFO | RTMetrics report populated | 20260508 02:27:18.324 | INFO | Opening tmp RTMetrics json file | 20260508 02:27:18.379 | INFO | Serializing RTMetrics to json | 20260508 02:27:18.398 | INFO | Serialization done, swapping tmp file now... | 20260508 02:27:18.401 | INFO | RTMetrics done | 20260508 02:27:18.429 | INFO | Finished a round of Baz IO: Wrote 1.637 GiB in 3.551 seconds | 20260508 02:27:18.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.54% / 0.00% / 58.46% | 20260508 02:27:18.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.718534% / 7.94729e-06% / 6.35783e-05% | 20260508 02:27:18.429 | INFO | Baz IO overall: | 20260508 02:27:18.429 | INFO | BytesWritten : 2037996718936 | 20260508 02:27:18.429 | INFO | NumEvents : 985229547615 | 20260508 02:27:19.919 | INFO | Analyzing chunk frames = [4513792,4514304) | 20260508 02:27:19.919 | INFO | MemInfo: { | 20260508 02:27:19.919 | INFO | "gpu" : " 17 GB", | 20260508 02:27:19.919 | INFO | "huge" : "", | 20260508 02:27:19.919 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:19.919 | INFO | "malloc" : " 14 GB", | 20260508 02:27:19.919 | INFO | "pinned" : " 29 GB" | 20260508 02:27:19.919 | INFO | } | 20260508 02:27:19.919 | INFO | Perf report for the last 5.13935 seconds: | 20260508 02:27:19.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:19.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:19.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:19.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:19.919 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 02:27:19.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:19.919 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260508 02:27:19.919 | INFO | PRE_HQ : 65.6%, 1.7, 56.3ms, 0 | 20260508 02:27:19.919 | INFO | BAZWRITER : 68.5%, 3.7, 128.8ms, 0 | 20260508 02:27:19.919 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 02:27:19.919 | INFO | Uploaded 36GiB to GPU in 2.5309s (14.2242GiB/s) | 20260508 02:27:19.919 | INFO | Downloaded 38GiB to GPU in 2.68973s (14.1278GiB/s) | 20260508 02:27:25.052 | INFO | Analyzing chunk frames = [4514304,4514816) | 20260508 02:27:25.052 | INFO | MemInfo: { | 20260508 02:27:25.052 | INFO | "gpu" : " 17 GB", | 20260508 02:27:25.052 | INFO | "huge" : "", | 20260508 02:27:25.052 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:25.052 | INFO | "malloc" : " 14 GB", | 20260508 02:27:25.052 | INFO | "pinned" : " 29 GB" | 20260508 02:27:25.052 | INFO | } | 20260508 02:27:25.052 | INFO | Perf report for the last 5.13323 seconds: | 20260508 02:27:25.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:25.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.052 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:27:25.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.052 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 02:27:25.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:25.053 | INFO | Uploaded 36GiB to GPU in 2.54438s (14.1488GiB/s) | 20260508 02:27:25.053 | INFO | Downloaded 34GiB to GPU in 2.43251s (13.9773GiB/s) | 20260508 02:27:30.193 | INFO | Analyzing chunk frames = [4514816,4515328) | 20260508 02:27:30.193 | INFO | MemInfo: { | 20260508 02:27:30.193 | INFO | "gpu" : " 17 GB", | 20260508 02:27:30.193 | INFO | "huge" : "", | 20260508 02:27:30.193 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:30.193 | INFO | "malloc" : " 14 GB", | 20260508 02:27:30.193 | INFO | "pinned" : " 29 GB" | 20260508 02:27:30.193 | INFO | } | 20260508 02:27:30.194 | INFO | Perf report for the last 5.14122 seconds: | 20260508 02:27:30.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:30.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:27:30.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:27:30.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:30.194 | INFO | Uploaded 36GiB to GPU in 2.52898s (14.235GiB/s) | 20260508 02:27:30.194 | INFO | Downloaded 34GiB to GPU in 2.49794s (13.6112GiB/s) | 20260508 02:27:35.334 | INFO | Analyzing chunk frames = [4515328,4515840) | 20260508 02:27:35.334 | INFO | MemInfo: { | 20260508 02:27:35.334 | INFO | "gpu" : " 17 GB", | 20260508 02:27:35.334 | INFO | "huge" : "", | 20260508 02:27:35.334 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:35.334 | INFO | "malloc" : " 14 GB", | 20260508 02:27:35.334 | INFO | "pinned" : " 29 GB" | 20260508 02:27:35.334 | INFO | } | 20260508 02:27:35.334 | INFO | Perf report for the last 5.14071 seconds: | 20260508 02:27:35.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:35.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.334 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:27:35.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.334 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:27:35.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:35.335 | INFO | Uploaded 36GiB to GPU in 2.52833s (14.2387GiB/s) | 20260508 02:27:35.335 | INFO | Downloaded 34GiB to GPU in 2.49509s (13.6268GiB/s) | 20260508 02:27:40.475 | INFO | Analyzing chunk frames = [4515840,4516352) | 20260508 02:27:40.475 | INFO | MemInfo: { | 20260508 02:27:40.475 | INFO | "gpu" : " 17 GB", | 20260508 02:27:40.475 | INFO | "huge" : "", | 20260508 02:27:40.475 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:40.475 | INFO | "malloc" : " 14 GB", | 20260508 02:27:40.475 | INFO | "pinned" : " 29 GB" | 20260508 02:27:40.475 | INFO | } | 20260508 02:27:40.475 | INFO | Perf report for the last 5.14112 seconds: | 20260508 02:27:40.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:40.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.475 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 02:27:40.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.475 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:27:40.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:40.476 | INFO | Uploaded 36GiB to GPU in 2.34377s (15.3599GiB/s) | 20260508 02:27:40.476 | INFO | Downloaded 34GiB to GPU in 2.48956s (13.657GiB/s) | 20260508 02:27:45.618 | INFO | Analyzing chunk frames = [4516352,4516864) | 20260508 02:27:45.618 | INFO | MemInfo: { | 20260508 02:27:45.618 | INFO | "gpu" : " 17 GB", | 20260508 02:27:45.618 | INFO | "huge" : "", | 20260508 02:27:45.618 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:45.618 | INFO | "malloc" : " 14 GB", | 20260508 02:27:45.618 | INFO | "pinned" : " 29 GB" | 20260508 02:27:45.618 | INFO | } | 20260508 02:27:45.619 | INFO | Perf report for the last 5.14314 seconds: | 20260508 02:27:45.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:45.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:27:45.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:27:45.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:45.619 | INFO | Uploaded 36GiB to GPU in 2.42621s (14.838GiB/s) | 20260508 02:27:45.619 | INFO | Downloaded 34GiB to GPU in 2.41138s (14.0998GiB/s) | 20260508 02:27:50.750 | INFO | Analyzing chunk frames = [4516864,4517376) | 20260508 02:27:50.750 | INFO | MemInfo: { | 20260508 02:27:50.750 | INFO | "gpu" : " 17 GB", | 20260508 02:27:50.750 | INFO | "huge" : "", | 20260508 02:27:50.750 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:50.750 | INFO | "malloc" : " 14 GB", | 20260508 02:27:50.750 | INFO | "pinned" : " 29 GB" | 20260508 02:27:50.750 | INFO | } | 20260508 02:27:50.750 | INFO | Perf report for the last 5.13121 seconds: | 20260508 02:27:50.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:50.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:27:50.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:27:50.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:50.750 | INFO | Uploaded 36GiB to GPU in 2.44829s (14.7041GiB/s) | 20260508 02:27:50.750 | INFO | Downloaded 34GiB to GPU in 2.41052s (14.1049GiB/s) | 20260508 02:27:55.895 | INFO | Analyzing chunk frames = [4517376,4517888) | 20260508 02:27:55.895 | INFO | MemInfo: { | 20260508 02:27:55.895 | INFO | "gpu" : " 17 GB", | 20260508 02:27:55.895 | INFO | "huge" : "", | 20260508 02:27:55.895 | INFO | "huge_shared" : " 29 GB", | 20260508 02:27:55.895 | INFO | "malloc" : " 14 GB", | 20260508 02:27:55.895 | INFO | "pinned" : " 29 GB" | 20260508 02:27:55.895 | INFO | } | 20260508 02:27:55.895 | INFO | Perf report for the last 5.14548 seconds: | 20260508 02:27:55.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:27:55.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.895 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 02:27:55.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.895 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:27:55.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:27:55.896 | INFO | Uploaded 36GiB to GPU in 2.56346s (14.0435GiB/s) | 20260508 02:27:55.896 | INFO | Downloaded 34GiB to GPU in 2.45443s (13.8525GiB/s) | 20260508 02:27:55.992 | INFO | Starting a round of Baz IO | 20260508 02:27:59.430 | INFO | Populating RTMetrics report | 20260508 02:27:59.431 | INFO | RTMetrics report populated | 20260508 02:27:59.431 | INFO | Opening tmp RTMetrics json file | 20260508 02:27:59.434 | INFO | Serializing RTMetrics to json | 20260508 02:27:59.459 | INFO | Serialization done, swapping tmp file now... | 20260508 02:27:59.461 | INFO | RTMetrics done | 20260508 02:27:59.534 | INFO | Finished a round of Baz IO: Wrote 1.637 GiB in 3.541 seconds | 20260508 02:27:59.534 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.53% / 0.00% / 58.46% | 20260508 02:27:59.534 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.718212% / 3.97364e-06% / 5.5631e-05% | 20260508 02:27:59.534 | INFO | Baz IO overall: | 20260508 02:27:59.534 | INFO | BytesWritten : 2039754880944 | 20260508 02:27:59.534 | INFO | NumEvents : 986072271690 | 20260508 02:28:01.032 | INFO | Analyzing chunk frames = [4517888,4518400) | 20260508 02:28:01.032 | INFO | MemInfo: { | 20260508 02:28:01.032 | INFO | "gpu" : " 17 GB", | 20260508 02:28:01.032 | INFO | "huge" : "", | 20260508 02:28:01.033 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:01.033 | INFO | "malloc" : " 14 GB", | 20260508 02:28:01.033 | INFO | "pinned" : " 29 GB" | 20260508 02:28:01.033 | INFO | } | 20260508 02:28:01.033 | INFO | Perf report for the last 5.13739 seconds: | 20260508 02:28:01.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:01.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:01.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:01.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:01.033 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 02:28:01.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:01.033 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 02:28:01.033 | INFO | PRE_HQ : 65.9%, 1.6, 56.2ms, 0 | 20260508 02:28:01.033 | INFO | BAZWRITER : 68.5%, 4.3, 150.5ms, 0 | 20260508 02:28:01.033 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 02:28:01.033 | INFO | Uploaded 36GiB to GPU in 2.56059s (14.0592GiB/s) | 20260508 02:28:01.033 | INFO | Downloaded 38GiB to GPU in 2.68952s (14.1289GiB/s) | 20260508 02:28:06.171 | INFO | Analyzing chunk frames = [4518400,4518912) | 20260508 02:28:06.171 | INFO | MemInfo: { | 20260508 02:28:06.171 | INFO | "gpu" : " 17 GB", | 20260508 02:28:06.171 | INFO | "huge" : "", | 20260508 02:28:06.172 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:06.172 | INFO | "malloc" : " 14 GB", | 20260508 02:28:06.172 | INFO | "pinned" : " 29 GB" | 20260508 02:28:06.172 | INFO | } | 20260508 02:28:06.172 | INFO | Perf report for the last 5.13904 seconds: | 20260508 02:28:06.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:06.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 02:28:06.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:28:06.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:06.172 | INFO | Uploaded 36GiB to GPU in 2.55382s (14.0965GiB/s) | 20260508 02:28:06.172 | INFO | Downloaded 34GiB to GPU in 2.42064s (14.0459GiB/s) | 20260508 02:28:11.306 | INFO | Analyzing chunk frames = [4518912,4519424) | 20260508 02:28:11.306 | INFO | MemInfo: { | 20260508 02:28:11.306 | INFO | "gpu" : " 17 GB", | 20260508 02:28:11.307 | INFO | "huge" : "", | 20260508 02:28:11.307 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:11.307 | INFO | "malloc" : " 14 GB", | 20260508 02:28:11.307 | INFO | "pinned" : " 29 GB" | 20260508 02:28:11.307 | INFO | } | 20260508 02:28:11.307 | INFO | Perf report for the last 5.13504 seconds: | 20260508 02:28:11.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:11.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:28:11.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:28:11.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:11.307 | INFO | Uploaded 36GiB to GPU in 2.57299s (13.9915GiB/s) | 20260508 02:28:11.307 | INFO | Downloaded 34GiB to GPU in 2.49272s (13.6397GiB/s) | 20260508 02:28:16.446 | INFO | Analyzing chunk frames = [4519424,4519936) | 20260508 02:28:16.446 | INFO | MemInfo: { | 20260508 02:28:16.446 | INFO | "gpu" : " 17 GB", | 20260508 02:28:16.446 | INFO | "huge" : "", | 20260508 02:28:16.446 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:16.446 | INFO | "malloc" : " 14 GB", | 20260508 02:28:16.446 | INFO | "pinned" : " 29 GB" | 20260508 02:28:16.446 | INFO | } | 20260508 02:28:16.446 | INFO | Perf report for the last 5.13911 seconds: | 20260508 02:28:16.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:16.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 02:28:16.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260508 02:28:16.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:16.446 | INFO | Uploaded 36GiB to GPU in 2.54114s (14.1669GiB/s) | 20260508 02:28:16.446 | INFO | Downloaded 34GiB to GPU in 2.47966s (13.7115GiB/s) | 20260508 02:28:21.592 | INFO | Analyzing chunk frames = [4519936,4520448) | 20260508 02:28:21.592 | INFO | MemInfo: { | 20260508 02:28:21.592 | INFO | "gpu" : " 17 GB", | 20260508 02:28:21.592 | INFO | "huge" : "", | 20260508 02:28:21.593 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:21.593 | INFO | "malloc" : " 14 GB", | 20260508 02:28:21.593 | INFO | "pinned" : " 29 GB" | 20260508 02:28:21.593 | INFO | } | 20260508 02:28:21.593 | INFO | Perf report for the last 5.14678 seconds: | 20260508 02:28:21.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:21.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 02:28:21.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 02:28:21.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:21.593 | INFO | Uploaded 36GiB to GPU in 2.34466s (15.3541GiB/s) | 20260508 02:28:21.593 | INFO | Downloaded 34GiB to GPU in 2.48938s (13.658GiB/s) | 20260508 02:28:26.733 | INFO | Analyzing chunk frames = [4520448,4520960) | 20260508 02:28:26.734 | INFO | MemInfo: { | 20260508 02:28:26.734 | INFO | "gpu" : " 17 GB", | 20260508 02:28:26.734 | INFO | "huge" : "", | 20260508 02:28:26.734 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:26.734 | INFO | "malloc" : " 14 GB", | 20260508 02:28:26.734 | INFO | "pinned" : " 29 GB" | 20260508 02:28:26.734 | INFO | } | 20260508 02:28:26.734 | INFO | Perf report for the last 5.14112 seconds: | 20260508 02:28:26.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:26.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:28:26.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:28:26.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:26.734 | INFO | Uploaded 36GiB to GPU in 2.41474s (14.9084GiB/s) | 20260508 02:28:26.735 | INFO | Downloaded 34GiB to GPU in 2.42334s (14.0302GiB/s) | 20260508 02:28:31.867 | INFO | Analyzing chunk frames = [4520960,4521472) | 20260508 02:28:31.867 | INFO | MemInfo: { | 20260508 02:28:31.867 | INFO | "gpu" : " 17 GB", | 20260508 02:28:31.867 | INFO | "huge" : "", | 20260508 02:28:31.867 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:31.867 | INFO | "malloc" : " 14 GB", | 20260508 02:28:31.867 | INFO | "pinned" : " 29 GB" | 20260508 02:28:31.867 | INFO | } | 20260508 02:28:31.867 | INFO | Perf report for the last 5.13349 seconds: | 20260508 02:28:31.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:31.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.867 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:28:31.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.867 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:28:31.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:31.868 | INFO | Uploaded 36GiB to GPU in 2.43807s (14.7658GiB/s) | 20260508 02:28:31.868 | INFO | Downloaded 34GiB to GPU in 2.42362s (14.0286GiB/s) | 20260508 02:28:37.006 | INFO | Analyzing chunk frames = [4521472,4521984) | 20260508 02:28:37.006 | INFO | MemInfo: { | 20260508 02:28:37.006 | INFO | "gpu" : " 17 GB", | 20260508 02:28:37.006 | INFO | "huge" : "", | 20260508 02:28:37.006 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:37.006 | INFO | "malloc" : " 14 GB", | 20260508 02:28:37.006 | INFO | "pinned" : " 29 GB" | 20260508 02:28:37.006 | INFO | } | 20260508 02:28:37.006 | INFO | Perf report for the last 5.13858 seconds: | 20260508 02:28:37.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:37.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:28:37.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:28:37.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:37.006 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:28:37.006 | INFO | Total mem usage: 16.7 GiB | 20260508 02:28:37.006 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:28:37.006 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.8%) | 20260508 02:28:37.006 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:28:37.006 | INFO | High Water Breakdown: | 20260508 02:28:37.006 | INFO | 60.2% : 8.98 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:28:37.006 | INFO | 39.7% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:28:37.006 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:28:37.006 | INFO | ------------------------------------------ | 20260508 02:28:37.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:28:37.006 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:28:37.006 | INFO | Total mem usage: 26.5 GiB | 20260508 02:28:37.006 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:28:37.006 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:28:37.006 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:28:37.006 | INFO | High Water Breakdown: | 20260508 02:28:37.006 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:28:37.006 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:28:37.006 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:28:37.006 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:28:37.006 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:28:37.006 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:28:37.006 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:28:37.006 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:28:37.006 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:28:37.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:28:37.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:28:37.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:28:37.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:28:37.006 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:28:37.006 | INFO | ------------------------------------------ | 20260508 02:28:37.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:28:37.006 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:28:37.006 | INFO | Total mem usage: 27.8 GiB | 20260508 02:28:37.006 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:28:37.006 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:28:37.006 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:28:37.006 | INFO | High Water Breakdown: | 20260508 02:28:37.006 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:28:37.006 | INFO | ------------------------------------------ | 20260508 02:28:37.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:28:37.006 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:28:37.006 | INFO | Total mem usage: 15.7 GiB | 20260508 02:28:37.006 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:28:37.006 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:28:37.006 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:28:37.006 | INFO | High Water Breakdown: | 20260508 02:28:37.006 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:28:37.006 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:28:37.006 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:28:37.006 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:28:37.006 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:28:37.006 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:28:37.007 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:28:37.007 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:28:37.007 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:28:37.007 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:28:37.007 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:28:37.007 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:28:37.007 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:28:37.007 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:28:37.007 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:28:37.007 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:28:37.007 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:28:37.007 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:28:37.007 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:28:37.007 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:28:37.007 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:28:37.007 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:28:37.007 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:28:37.007 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:28:37.007 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:28:37.007 | INFO | ------------------------------------------ | 20260508 02:28:37.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:28:37.007 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:28:37.007 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:28:37.007 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:28:37.007 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.40e+01 / 1.14e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.13e-02 / 5.94e-03 | 20260508 02:28:37.007 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.42e+00, 5.85e-01 / 2.64e-01 | 20260508 02:28:37.007 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.04e-01 / 1.16e-01 | 20260508 02:28:37.007 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.60e+01, 3.79e+00 / 1.41e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.62e+00 / 1.56e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.42e+00 / 2.68e+00, 4.21e-01 / 2.46e-01 | 20260508 02:28:37.007 | INFO | PerfReport: 3.29%, Metrics -- 1.08e+00 / 1.26e+00, 2.64e-01 / 1.24e-01 | 20260508 02:28:37.007 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.29e-01 / 3.61e-01, 5.17e-02 / 2.31e-02 | 20260508 02:28:37.007 | INFO | Uploaded 36GiB to GPU in 2.57481s (13.9816GiB/s) | 20260508 02:28:37.007 | INFO | Downloaded 34GiB to GPU in 2.46586s (13.7883GiB/s) | 20260508 02:28:37.007 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:28:37.007 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:28:37.007 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:28:37.007 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 30.69%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 2.30%, LockUpload -- 2.45e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 2.67%, LockDownload -- 2.84e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 15.56%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:28:37.007 | INFO | PerfReport: 0.01%, Misc -- 1.54e-02 / 0.00e+00, 3.97e-02 / 0.00e+00 | 20260508 02:28:37.104 | INFO | Starting a round of Baz IO | 20260508 02:28:40.536 | INFO | Populating RTMetrics report | 20260508 02:28:40.536 | INFO | RTMetrics report populated | 20260508 02:28:40.536 | INFO | Opening tmp RTMetrics json file | 20260508 02:28:40.556 | INFO | Serializing RTMetrics to json | 20260508 02:28:40.558 | INFO | Serialization done, swapping tmp file now... | 20260508 02:28:40.558 | INFO | RTMetrics done | 20260508 02:28:40.627 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.522 seconds | 20260508 02:28:40.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.53% / 0.00% / 58.47% | 20260508 02:28:40.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.715001% / 3.97364e-06% / 6.75519e-05% | 20260508 02:28:40.627 | INFO | Baz IO overall: | 20260508 02:28:40.627 | INFO | BytesWritten : 2041511078597 | 20260508 02:28:40.627 | INFO | NumEvents : 986913508477 | 20260508 02:28:42.150 | INFO | Analyzing chunk frames = [4521984,4522496) | 20260508 02:28:42.150 | INFO | MemInfo: { | 20260508 02:28:42.150 | INFO | "gpu" : " 17 GB", | 20260508 02:28:42.150 | INFO | "huge" : "", | 20260508 02:28:42.150 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:42.150 | INFO | "malloc" : " 14 GB", | 20260508 02:28:42.150 | INFO | "pinned" : " 29 GB" | 20260508 02:28:42.150 | INFO | } | 20260508 02:28:42.150 | INFO | Perf report for the last 5.14377 seconds: | 20260508 02:28:42.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:42.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:42.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:42.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:42.150 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 02:28:42.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:42.150 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 02:28:42.150 | INFO | PRE_HQ : 65.7%, 1.6, 55.8ms, 0 | 20260508 02:28:42.150 | INFO | BAZWRITER : 68.1%, 3.4, 120.6ms, 0 | 20260508 02:28:42.150 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 02:28:42.150 | INFO | Uploaded 36GiB to GPU in 2.55501s (14.09GiB/s) | 20260508 02:28:42.150 | INFO | Downloaded 38GiB to GPU in 2.70054s (14.0713GiB/s) | 20260508 02:28:47.288 | INFO | Analyzing chunk frames = [4522496,4523008) | 20260508 02:28:47.288 | INFO | MemInfo: { | 20260508 02:28:47.288 | INFO | "gpu" : " 17 GB", | 20260508 02:28:47.288 | INFO | "huge" : "", | 20260508 02:28:47.288 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:47.288 | INFO | "malloc" : " 14 GB", | 20260508 02:28:47.288 | INFO | "pinned" : " 29 GB" | 20260508 02:28:47.288 | INFO | } | 20260508 02:28:47.288 | INFO | Perf report for the last 5.13875 seconds: | 20260508 02:28:47.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:47.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.289 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:28:47.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.289 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:28:47.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:47.289 | INFO | Uploaded 36GiB to GPU in 2.54708s (14.1339GiB/s) | 20260508 02:28:47.289 | INFO | Downloaded 34GiB to GPU in 2.43624s (13.9559GiB/s) | 20260508 02:28:52.426 | INFO | Analyzing chunk frames = [4523008,4523520) | 20260508 02:28:52.426 | INFO | MemInfo: { | 20260508 02:28:52.426 | INFO | "gpu" : " 17 GB", | 20260508 02:28:52.426 | INFO | "huge" : "", | 20260508 02:28:52.426 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:52.426 | INFO | "malloc" : " 14 GB", | 20260508 02:28:52.426 | INFO | "pinned" : " 29 GB" | 20260508 02:28:52.426 | INFO | } | 20260508 02:28:52.426 | INFO | Perf report for the last 5.13805 seconds: | 20260508 02:28:52.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:52.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:28:52.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 02:28:52.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:52.427 | INFO | Uploaded 36GiB to GPU in 2.53046s (14.2267GiB/s) | 20260508 02:28:52.427 | INFO | Downloaded 34GiB to GPU in 2.50538s (13.5708GiB/s) | 20260508 02:28:57.565 | INFO | Analyzing chunk frames = [4523520,4524032) | 20260508 02:28:57.565 | INFO | MemInfo: { | 20260508 02:28:57.565 | INFO | "gpu" : " 17 GB", | 20260508 02:28:57.565 | INFO | "huge" : "", | 20260508 02:28:57.565 | INFO | "huge_shared" : " 29 GB", | 20260508 02:28:57.565 | INFO | "malloc" : " 14 GB", | 20260508 02:28:57.565 | INFO | "pinned" : " 29 GB" | 20260508 02:28:57.565 | INFO | } | 20260508 02:28:57.565 | INFO | Perf report for the last 5.1383 seconds: | 20260508 02:28:57.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:28:57.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:28:57.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:28:57.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:28:57.565 | INFO | Uploaded 36GiB to GPU in 2.54046s (14.1707GiB/s) | 20260508 02:28:57.566 | INFO | Downloaded 34GiB to GPU in 2.49615s (13.621GiB/s) | 20260508 02:29:02.707 | INFO | Analyzing chunk frames = [4524032,4524544) | 20260508 02:29:02.707 | INFO | MemInfo: { | 20260508 02:29:02.707 | INFO | "gpu" : " 17 GB", | 20260508 02:29:02.707 | INFO | "huge" : "", | 20260508 02:29:02.707 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:02.707 | INFO | "malloc" : " 14 GB", | 20260508 02:29:02.707 | INFO | "pinned" : " 29 GB" | 20260508 02:29:02.707 | INFO | } | 20260508 02:29:02.707 | INFO | Perf report for the last 5.142 seconds: | 20260508 02:29:02.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:02.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:29:02.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:29:02.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:02.707 | INFO | Uploaded 36GiB to GPU in 2.40211s (14.9868GiB/s) | 20260508 02:29:02.707 | INFO | Downloaded 34GiB to GPU in 2.5333s (13.4212GiB/s) | 20260508 02:29:07.842 | INFO | Analyzing chunk frames = [4524544,4525056) | 20260508 02:29:07.842 | INFO | MemInfo: { | 20260508 02:29:07.842 | INFO | "gpu" : " 17 GB", | 20260508 02:29:07.842 | INFO | "huge" : "", | 20260508 02:29:07.842 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:07.842 | INFO | "malloc" : " 14 GB", | 20260508 02:29:07.842 | INFO | "pinned" : " 29 GB" | 20260508 02:29:07.843 | INFO | } | 20260508 02:29:07.843 | INFO | Perf report for the last 5.13584 seconds: | 20260508 02:29:07.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:07.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 02:29:07.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:29:07.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:07.843 | INFO | Uploaded 36GiB to GPU in 2.43687s (14.773GiB/s) | 20260508 02:29:07.843 | INFO | Downloaded 34GiB to GPU in 2.40908s (14.1132GiB/s) | 20260508 02:29:12.984 | INFO | Analyzing chunk frames = [4525056,4525568) | 20260508 02:29:12.984 | INFO | MemInfo: { | 20260508 02:29:12.984 | INFO | "gpu" : " 17 GB", | 20260508 02:29:12.984 | INFO | "huge" : "", | 20260508 02:29:12.984 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:12.984 | INFO | "malloc" : " 14 GB", | 20260508 02:29:12.984 | INFO | "pinned" : " 29 GB" | 20260508 02:29:12.984 | INFO | } | 20260508 02:29:12.984 | INFO | Perf report for the last 5.14149 seconds: | 20260508 02:29:12.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:12.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.984 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:29:12.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.984 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:29:12.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:12.985 | INFO | Uploaded 36GiB to GPU in 2.45155s (14.6846GiB/s) | 20260508 02:29:12.985 | INFO | Downloaded 34GiB to GPU in 2.41804s (14.061GiB/s) | 20260508 02:29:18.121 | INFO | Analyzing chunk frames = [4525568,4526080) | 20260508 02:29:18.121 | INFO | MemInfo: { | 20260508 02:29:18.121 | INFO | "gpu" : " 17 GB", | 20260508 02:29:18.121 | INFO | "huge" : "", | 20260508 02:29:18.121 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:18.121 | INFO | "malloc" : " 14 GB", | 20260508 02:29:18.121 | INFO | "pinned" : " 29 GB" | 20260508 02:29:18.121 | INFO | } | 20260508 02:29:18.121 | INFO | Perf report for the last 5.1371 seconds: | 20260508 02:29:18.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:18.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.121 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 02:29:18.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.121 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:29:18.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:18.122 | INFO | Uploaded 36GiB to GPU in 2.562s (14.0515GiB/s) | 20260508 02:29:18.122 | INFO | Downloaded 34GiB to GPU in 2.44737s (13.8924GiB/s) | 20260508 02:29:18.219 | INFO | Starting a round of Baz IO | 20260508 02:29:21.669 | INFO | Populating RTMetrics report | 20260508 02:29:21.669 | INFO | RTMetrics report populated | 20260508 02:29:21.669 | INFO | Opening tmp RTMetrics json file | 20260508 02:29:21.701 | INFO | Serializing RTMetrics to json | 20260508 02:29:21.761 | INFO | Serialization done, swapping tmp file now... | 20260508 02:29:21.765 | INFO | RTMetrics done | 20260508 02:29:21.788 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.569 seconds | 20260508 02:29:21.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.53% / 0.00% / 58.47% | 20260508 02:29:21.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.713889% / 1.19209e-05% / 6.35783e-05% | 20260508 02:29:21.788 | INFO | Baz IO overall: | 20260508 02:29:21.788 | INFO | BytesWritten : 2043267644993 | 20260508 02:29:21.788 | INFO | NumEvents : 987755130642 | 20260508 02:29:23.259 | INFO | Analyzing chunk frames = [4526080,4526592) | 20260508 02:29:23.259 | INFO | MemInfo: { | 20260508 02:29:23.259 | INFO | "gpu" : " 17 GB", | 20260508 02:29:23.259 | INFO | "huge" : "", | 20260508 02:29:23.259 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:23.259 | INFO | "malloc" : " 14 GB", | 20260508 02:29:23.259 | INFO | "pinned" : " 29 GB" | 20260508 02:29:23.259 | INFO | } | 20260508 02:29:23.260 | INFO | Perf report for the last 5.13818 seconds: | 20260508 02:29:23.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:23.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:23.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:23.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:23.260 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:29:23.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:23.260 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 02:29:23.260 | INFO | PRE_HQ : 65.5%, 1.7, 56.3ms, 0 | 20260508 02:29:23.260 | INFO | BAZWRITER : 69.1%, 3.6, 127.2ms, 0 | 20260508 02:29:23.260 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 02:29:23.260 | INFO | Uploaded 36GiB to GPU in 2.57511s (13.98GiB/s) | 20260508 02:29:23.260 | INFO | Downloaded 38GiB to GPU in 2.68085s (14.1746GiB/s) | 20260508 02:29:28.400 | INFO | Analyzing chunk frames = [4526592,4527104) | 20260508 02:29:28.400 | INFO | MemInfo: { | 20260508 02:29:28.400 | INFO | "gpu" : " 17 GB", | 20260508 02:29:28.400 | INFO | "huge" : "", | 20260508 02:29:28.400 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:28.400 | INFO | "malloc" : " 14 GB", | 20260508 02:29:28.400 | INFO | "pinned" : " 29 GB" | 20260508 02:29:28.400 | INFO | } | 20260508 02:29:28.400 | INFO | Perf report for the last 5.14059 seconds: | 20260508 02:29:28.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:28.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.400 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:29:28.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.400 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:29:28.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:28.401 | INFO | Uploaded 36GiB to GPU in 2.52192s (14.2749GiB/s) | 20260508 02:29:28.401 | INFO | Downloaded 34GiB to GPU in 2.43097s (13.9862GiB/s) | 20260508 02:29:33.540 | INFO | Analyzing chunk frames = [4527104,4527616) | 20260508 02:29:33.540 | INFO | MemInfo: { | 20260508 02:29:33.540 | INFO | "gpu" : " 17 GB", | 20260508 02:29:33.540 | INFO | "huge" : "", | 20260508 02:29:33.540 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:33.540 | INFO | "malloc" : " 14 GB", | 20260508 02:29:33.540 | INFO | "pinned" : " 29 GB" | 20260508 02:29:33.540 | INFO | } | 20260508 02:29:33.540 | INFO | Perf report for the last 5.13974 seconds: | 20260508 02:29:33.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:33.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:29:33.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:29:33.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:33.540 | INFO | Uploaded 36GiB to GPU in 2.52819s (14.2394GiB/s) | 20260508 02:29:33.540 | INFO | Downloaded 34GiB to GPU in 2.49995s (13.6003GiB/s) | 20260508 02:29:38.681 | INFO | Analyzing chunk frames = [4527616,4528128) | 20260508 02:29:38.681 | INFO | MemInfo: { | 20260508 02:29:38.681 | INFO | "gpu" : " 17 GB", | 20260508 02:29:38.681 | INFO | "huge" : "", | 20260508 02:29:38.681 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:38.681 | INFO | "malloc" : " 14 GB", | 20260508 02:29:38.681 | INFO | "pinned" : " 29 GB" | 20260508 02:29:38.682 | INFO | } | 20260508 02:29:38.682 | INFO | Perf report for the last 5.1419 seconds: | 20260508 02:29:38.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:38.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 02:29:38.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:29:38.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:38.682 | INFO | Uploaded 36GiB to GPU in 2.52815s (14.2396GiB/s) | 20260508 02:29:38.682 | INFO | Downloaded 34GiB to GPU in 2.5063s (13.5658GiB/s) | 20260508 02:29:43.818 | INFO | Analyzing chunk frames = [4528128,4528640) | 20260508 02:29:43.818 | INFO | MemInfo: { | 20260508 02:29:43.818 | INFO | "gpu" : " 17 GB", | 20260508 02:29:43.818 | INFO | "huge" : "", | 20260508 02:29:43.818 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:43.818 | INFO | "malloc" : " 14 GB", | 20260508 02:29:43.818 | INFO | "pinned" : " 29 GB" | 20260508 02:29:43.818 | INFO | } | 20260508 02:29:43.818 | INFO | Perf report for the last 5.13643 seconds: | 20260508 02:29:43.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:43.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 02:29:43.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:29:43.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:43.818 | INFO | Uploaded 36GiB to GPU in 2.3399s (15.3853GiB/s) | 20260508 02:29:43.819 | INFO | Downloaded 34GiB to GPU in 2.51212s (13.5344GiB/s) | 20260508 02:29:48.958 | INFO | Analyzing chunk frames = [4528640,4529152) | 20260508 02:29:48.958 | INFO | MemInfo: { | 20260508 02:29:48.958 | INFO | "gpu" : " 17 GB", | 20260508 02:29:48.958 | INFO | "huge" : "", | 20260508 02:29:48.958 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:48.958 | INFO | "malloc" : " 14 GB", | 20260508 02:29:48.958 | INFO | "pinned" : " 29 GB" | 20260508 02:29:48.958 | INFO | } | 20260508 02:29:48.958 | INFO | Perf report for the last 5.1397 seconds: | 20260508 02:29:48.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:48.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:29:48.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:29:48.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:48.958 | INFO | Uploaded 36GiB to GPU in 2.39186s (15.0511GiB/s) | 20260508 02:29:48.958 | INFO | Downloaded 34GiB to GPU in 2.41797s (14.0614GiB/s) | 20260508 02:29:54.100 | INFO | Analyzing chunk frames = [4529152,4529664) | 20260508 02:29:54.100 | INFO | MemInfo: { | 20260508 02:29:54.100 | INFO | "gpu" : " 17 GB", | 20260508 02:29:54.100 | INFO | "huge" : "", | 20260508 02:29:54.100 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:54.100 | INFO | "malloc" : " 14 GB", | 20260508 02:29:54.100 | INFO | "pinned" : " 29 GB" | 20260508 02:29:54.100 | INFO | } | 20260508 02:29:54.100 | INFO | Perf report for the last 5.14242 seconds: | 20260508 02:29:54.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:54.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.100 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 02:29:54.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.101 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:29:54.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:54.102 | INFO | Uploaded 36GiB to GPU in 2.4305s (14.8118GiB/s) | 20260508 02:29:54.102 | INFO | Downloaded 34GiB to GPU in 2.43154s (13.9829GiB/s) | 20260508 02:29:59.239 | INFO | Analyzing chunk frames = [4529664,4530176) | 20260508 02:29:59.239 | INFO | MemInfo: { | 20260508 02:29:59.239 | INFO | "gpu" : " 17 GB", | 20260508 02:29:59.239 | INFO | "huge" : "", | 20260508 02:29:59.239 | INFO | "huge_shared" : " 29 GB", | 20260508 02:29:59.239 | INFO | "malloc" : " 14 GB", | 20260508 02:29:59.239 | INFO | "pinned" : " 29 GB" | 20260508 02:29:59.239 | INFO | } | 20260508 02:29:59.239 | INFO | Perf report for the last 5.13905 seconds: | 20260508 02:29:59.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:29:59.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.239 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:29:59.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.240 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 02:29:59.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:29:59.240 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:29:59.240 | INFO | Total mem usage: 16.7 GiB | 20260508 02:29:59.240 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:29:59.240 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.1%) | 20260508 02:29:59.240 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:29:59.240 | INFO | High Water Breakdown: | 20260508 02:29:59.240 | INFO | 59.7% : 8.65 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:29:59.240 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:29:59.240 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:29:59.240 | INFO | ------------------------------------------ | 20260508 02:29:59.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:29:59.240 | INFO | Uploaded 36GiB to GPU in 2.44843s (14.7033GiB/s) | 20260508 02:29:59.240 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:29:59.240 | INFO | Total mem usage: 26.5 GiB | 20260508 02:29:59.240 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:29:59.240 | INFO | Downloaded 34GiB to GPU in 2.43423s (13.9675GiB/s) | 20260508 02:29:59.240 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 02:29:59.240 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 02:29:59.240 | INFO | High Water Breakdown: | 20260508 02:29:59.240 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:29:59.240 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:29:59.240 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:29:59.240 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:29:59.240 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:29:59.240 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:29:59.240 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:29:59.240 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:29:59.240 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:29:59.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:29:59.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:29:59.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:29:59.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:29:59.240 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:29:59.240 | INFO | ------------------------------------------ | 20260508 02:29:59.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:29:59.241 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:29:59.241 | INFO | Total mem usage: 27.8 GiB | 20260508 02:29:59.241 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:29:59.241 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:29:59.241 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:29:59.241 | INFO | High Water Breakdown: | 20260508 02:29:59.241 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:29:59.241 | INFO | ------------------------------------------ | 20260508 02:29:59.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:29:59.241 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:29:59.241 | INFO | Total mem usage: 15.7 GiB | 20260508 02:29:59.241 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:29:59.241 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:29:59.241 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:29:59.241 | INFO | High Water Breakdown: | 20260508 02:29:59.241 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:29:59.241 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:29:59.241 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:29:59.241 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:29:59.241 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:29:59.241 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:29:59.241 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:29:59.241 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:29:59.241 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:29:59.241 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:29:59.241 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:29:59.241 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:29:59.241 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:29:59.241 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:29:59.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:29:59.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:29:59.241 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:29:59.241 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:29:59.241 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:29:59.241 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:29:59.241 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:29:59.241 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:29:59.241 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:29:59.241 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:29:59.241 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:29:59.241 | INFO | ------------------------------------------ | 20260508 02:29:59.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:29:59.241 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:29:59.241 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:29:59.241 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:29:59.241 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.42e+01 / 1.32e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.01e-01, 1.15e-02 / 4.30e-03 | 20260508 02:29:59.241 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.50e+00, 5.88e-01 / 1.56e-01 | 20260508 02:29:59.241 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.08e-01 / 8.24e-02 | 20260508 02:29:59.241 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.58e+01, 3.96e+00 / 1.33e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.65e+00 / 1.51e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.42e+00 / 2.70e+00, 4.25e-01 / 2.17e-01 | 20260508 02:29:59.241 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.24e+00, 2.65e-01 / 1.27e-01 | 20260508 02:29:59.241 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.29e-01 / 3.69e-01, 5.21e-02 / 2.96e-02 | 20260508 02:29:59.241 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:29:59.241 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:29:59.241 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:29:59.241 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 24.34%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 2.13%, LockUpload -- 2.26e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 2.81%, LockDownload -- 2.99e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:29:59.241 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260508 02:29:59.338 | INFO | Starting a round of Baz IO | 20260508 02:30:02.779 | INFO | Populating RTMetrics report | 20260508 02:30:02.779 | INFO | RTMetrics report populated | 20260508 02:30:02.779 | INFO | Opening tmp RTMetrics json file | 20260508 02:30:02.803 | INFO | Serializing RTMetrics to json | 20260508 02:30:02.805 | INFO | Serialization done, swapping tmp file now... | 20260508 02:30:02.806 | INFO | RTMetrics done | 20260508 02:30:02.886 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.548 seconds | 20260508 02:30:02.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.53% / 0.00% / 58.47% | 20260508 02:30:02.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.714266% / 0% / 5.96046e-05% | 20260508 02:30:02.886 | INFO | Baz IO overall: | 20260508 02:30:02.886 | INFO | BytesWritten : 2045024486125 | 20260508 02:30:02.886 | INFO | NumEvents : 988597020776 | 20260508 02:30:04.375 | INFO | Analyzing chunk frames = [4530176,4530688) | 20260508 02:30:04.375 | INFO | MemInfo: { | 20260508 02:30:04.375 | INFO | "gpu" : " 17 GB", | 20260508 02:30:04.375 | INFO | "huge" : "", | 20260508 02:30:04.375 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:04.375 | INFO | "malloc" : " 14 GB", | 20260508 02:30:04.375 | INFO | "pinned" : " 29 GB" | 20260508 02:30:04.375 | INFO | } | 20260508 02:30:04.375 | INFO | Perf report for the last 5.13588 seconds: | 20260508 02:30:04.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:04.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:04.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:04.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:04.376 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 02:30:04.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:04.376 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 02:30:04.376 | INFO | PRE_HQ : 65.7%, 1.6, 55.9ms, 0 | 20260508 02:30:04.376 | INFO | BAZWRITER : 68.7%, 4.3, 151.2ms, 0 | 20260508 02:30:04.376 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:30:04.376 | INFO | Uploaded 36GiB to GPU in 2.60221s (13.8344GiB/s) | 20260508 02:30:04.376 | INFO | Downloaded 38GiB to GPU in 2.69672s (14.0912GiB/s) | 20260508 02:30:09.511 | INFO | Analyzing chunk frames = [4530688,4531200) | 20260508 02:30:09.511 | INFO | MemInfo: { | 20260508 02:30:09.511 | INFO | "gpu" : " 17 GB", | 20260508 02:30:09.511 | INFO | "huge" : "", | 20260508 02:30:09.511 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:09.511 | INFO | "malloc" : " 14 GB", | 20260508 02:30:09.511 | INFO | "pinned" : " 29 GB" | 20260508 02:30:09.511 | INFO | } | 20260508 02:30:09.511 | INFO | Perf report for the last 5.13582 seconds: | 20260508 02:30:09.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:09.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:30:09.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:30:09.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:09.511 | INFO | Uploaded 36GiB to GPU in 2.56257s (14.0484GiB/s) | 20260508 02:30:09.512 | INFO | Downloaded 34GiB to GPU in 2.41361s (14.0868GiB/s) | 20260508 02:30:14.653 | INFO | Analyzing chunk frames = [4531200,4531712) | 20260508 02:30:14.653 | INFO | MemInfo: { | 20260508 02:30:14.653 | INFO | "gpu" : " 17 GB", | 20260508 02:30:14.653 | INFO | "huge" : "", | 20260508 02:30:14.653 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:14.653 | INFO | "malloc" : " 14 GB", | 20260508 02:30:14.654 | INFO | "pinned" : " 29 GB" | 20260508 02:30:14.654 | INFO | } | 20260508 02:30:14.654 | INFO | Perf report for the last 5.1425 seconds: | 20260508 02:30:14.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:14.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 02:30:14.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:30:14.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:14.654 | INFO | Uploaded 36GiB to GPU in 2.55444s (14.0931GiB/s) | 20260508 02:30:14.654 | INFO | Downloaded 34GiB to GPU in 2.48115s (13.7033GiB/s) | 20260508 02:30:19.800 | INFO | Analyzing chunk frames = [4531712,4532224) | 20260508 02:30:19.800 | INFO | MemInfo: { | 20260508 02:30:19.800 | INFO | "gpu" : " 17 GB", | 20260508 02:30:19.800 | INFO | "huge" : "", | 20260508 02:30:19.800 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:19.800 | INFO | "malloc" : " 14 GB", | 20260508 02:30:19.800 | INFO | "pinned" : " 29 GB" | 20260508 02:30:19.800 | INFO | } | 20260508 02:30:19.800 | INFO | Perf report for the last 5.14651 seconds: | 20260508 02:30:19.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:19.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.800 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:30:19.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.800 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:30:19.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:19.801 | INFO | Uploaded 36GiB to GPU in 2.55688s (14.0797GiB/s) | 20260508 02:30:19.801 | INFO | Downloaded 34GiB to GPU in 2.4888s (13.6612GiB/s) | 20260508 02:30:24.935 | INFO | Analyzing chunk frames = [4532224,4532736) | 20260508 02:30:24.935 | INFO | MemInfo: { | 20260508 02:30:24.935 | INFO | "gpu" : " 17 GB", | 20260508 02:30:24.935 | INFO | "huge" : "", | 20260508 02:30:24.935 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:24.935 | INFO | "malloc" : " 14 GB", | 20260508 02:30:24.935 | INFO | "pinned" : " 29 GB" | 20260508 02:30:24.935 | INFO | } | 20260508 02:30:24.935 | INFO | Perf report for the last 5.13486 seconds: | 20260508 02:30:24.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:24.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 02:30:24.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:30:24.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:24.935 | INFO | Uploaded 36GiB to GPU in 2.42736s (14.8309GiB/s) | 20260508 02:30:24.935 | INFO | Downloaded 34GiB to GPU in 2.56015s (13.2805GiB/s) | 20260508 02:30:30.070 | INFO | Analyzing chunk frames = [4532736,4533248) | 20260508 02:30:30.070 | INFO | MemInfo: { | 20260508 02:30:30.070 | INFO | "gpu" : " 17 GB", | 20260508 02:30:30.070 | INFO | "huge" : "", | 20260508 02:30:30.070 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:30.070 | INFO | "malloc" : " 14 GB", | 20260508 02:30:30.070 | INFO | "pinned" : " 29 GB" | 20260508 02:30:30.070 | INFO | } | 20260508 02:30:30.070 | INFO | Perf report for the last 5.13483 seconds: | 20260508 02:30:30.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:30.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 02:30:30.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 02:30:30.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:30.070 | INFO | Uploaded 36GiB to GPU in 2.44834s (14.7038GiB/s) | 20260508 02:30:30.070 | INFO | Downloaded 34GiB to GPU in 2.41094s (14.1024GiB/s) | 20260508 02:30:35.210 | INFO | Analyzing chunk frames = [4533248,4533760) | 20260508 02:30:35.210 | INFO | MemInfo: { | 20260508 02:30:35.210 | INFO | "gpu" : " 17 GB", | 20260508 02:30:35.210 | INFO | "huge" : "", | 20260508 02:30:35.210 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:35.210 | INFO | "malloc" : " 14 GB", | 20260508 02:30:35.210 | INFO | "pinned" : " 29 GB" | 20260508 02:30:35.210 | INFO | } | 20260508 02:30:35.210 | INFO | Perf report for the last 5.14021 seconds: | 20260508 02:30:35.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:35.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 02:30:35.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 02:30:35.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:35.210 | INFO | Uploaded 36GiB to GPU in 2.46964s (14.577GiB/s) | 20260508 02:30:35.211 | INFO | Downloaded 34GiB to GPU in 2.42484s (14.0215GiB/s) | 20260508 02:30:40.352 | INFO | Analyzing chunk frames = [4533760,4534272) | 20260508 02:30:40.352 | INFO | MemInfo: { | 20260508 02:30:40.352 | INFO | "gpu" : " 17 GB", | 20260508 02:30:40.352 | INFO | "huge" : "", | 20260508 02:30:40.352 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:40.352 | INFO | "malloc" : " 14 GB", | 20260508 02:30:40.352 | INFO | "pinned" : " 29 GB" | 20260508 02:30:40.352 | INFO | } | 20260508 02:30:40.352 | INFO | Perf report for the last 5.1422 seconds: | 20260508 02:30:40.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:40.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.352 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:30:40.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.353 | INFO | PRE_HQ : 33.9%, 1.0, 17.7ms, 0 | 20260508 02:30:40.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:40.353 | INFO | Uploaded 36GiB to GPU in 2.57868s (13.9606GiB/s) | 20260508 02:30:40.353 | INFO | Downloaded 34GiB to GPU in 2.46111s (13.8149GiB/s) | 20260508 02:30:40.465 | INFO | Starting a round of Baz IO | 20260508 02:30:43.906 | INFO | Populating RTMetrics report | 20260508 02:30:43.907 | INFO | RTMetrics report populated | 20260508 02:30:43.907 | INFO | Opening tmp RTMetrics json file | 20260508 02:30:43.908 | INFO | Serializing RTMetrics to json | 20260508 02:30:43.925 | INFO | Serialization done, swapping tmp file now... | 20260508 02:30:43.939 | INFO | RTMetrics done | 20260508 02:30:44.029 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.563 seconds | 20260508 02:30:44.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.52% / 0.00% / 58.47% | 20260508 02:30:44.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.712574% / 3.97364e-06% / 6.35783e-05% | 20260508 02:30:44.029 | INFO | Baz IO overall: | 20260508 02:30:44.029 | INFO | BytesWritten : 2046781193104 | 20260508 02:30:44.029 | INFO | NumEvents : 989439193940 | 20260508 02:30:45.492 | INFO | Analyzing chunk frames = [4534272,4534784) | 20260508 02:30:45.492 | INFO | MemInfo: { | 20260508 02:30:45.492 | INFO | "gpu" : " 17 GB", | 20260508 02:30:45.492 | INFO | "huge" : "", | 20260508 02:30:45.492 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:45.492 | INFO | "malloc" : " 14 GB", | 20260508 02:30:45.492 | INFO | "pinned" : " 29 GB" | 20260508 02:30:45.492 | INFO | } | 20260508 02:30:45.492 | INFO | Perf report for the last 5.13963 seconds: | 20260508 02:30:45.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:45.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:45.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:45.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:45.492 | INFO | ANALYSIS : 68.5%, 2.9, 104.0ms, 0 | 20260508 02:30:45.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:45.492 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260508 02:30:45.492 | INFO | PRE_HQ : 67.3%, 1.8, 62.2ms, 0 | 20260508 02:30:45.492 | INFO | BAZWRITER : 68.9%, 3.7, 130.9ms, 0 | 20260508 02:30:45.492 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 02:30:45.492 | INFO | Uploaded 36GiB to GPU in 2.60862s (13.8004GiB/s) | 20260508 02:30:45.492 | INFO | Downloaded 38GiB to GPU in 2.69865s (14.0811GiB/s) | 20260508 02:30:50.630 | INFO | Analyzing chunk frames = [4534784,4535296) | 20260508 02:30:50.630 | INFO | MemInfo: { | 20260508 02:30:50.630 | INFO | "gpu" : " 17 GB", | 20260508 02:30:50.630 | INFO | "huge" : "", | 20260508 02:30:50.630 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:50.630 | INFO | "malloc" : " 14 GB", | 20260508 02:30:50.630 | INFO | "pinned" : " 29 GB" | 20260508 02:30:50.630 | INFO | } | 20260508 02:30:50.630 | INFO | Perf report for the last 5.13834 seconds: | 20260508 02:30:50.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:50.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.630 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:30:50.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.630 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:30:50.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:50.631 | INFO | Uploaded 36GiB to GPU in 2.56651s (14.0268GiB/s) | 20260508 02:30:50.631 | INFO | Downloaded 34GiB to GPU in 2.42306s (14.0319GiB/s) | 20260508 02:30:55.764 | INFO | Analyzing chunk frames = [4535296,4535808) | 20260508 02:30:55.764 | INFO | MemInfo: { | 20260508 02:30:55.764 | INFO | "gpu" : " 17 GB", | 20260508 02:30:55.764 | INFO | "huge" : "", | 20260508 02:30:55.764 | INFO | "huge_shared" : " 29 GB", | 20260508 02:30:55.764 | INFO | "malloc" : " 14 GB", | 20260508 02:30:55.764 | INFO | "pinned" : " 29 GB" | 20260508 02:30:55.764 | INFO | } | 20260508 02:30:55.764 | INFO | Perf report for the last 5.13347 seconds: | 20260508 02:30:55.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:30:55.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:30:55.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:30:55.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:30:55.764 | INFO | Uploaded 36GiB to GPU in 2.56794s (14.019GiB/s) | 20260508 02:30:55.764 | INFO | Downloaded 34GiB to GPU in 2.48845s (13.6631GiB/s) | 20260508 02:31:00.909 | INFO | Analyzing chunk frames = [4535808,4536320) | 20260508 02:31:00.909 | INFO | MemInfo: { | 20260508 02:31:00.909 | INFO | "gpu" : " 17 GB", | 20260508 02:31:00.909 | INFO | "huge" : "", | 20260508 02:31:00.909 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:00.909 | INFO | "malloc" : " 14 GB", | 20260508 02:31:00.909 | INFO | "pinned" : " 29 GB" | 20260508 02:31:00.909 | INFO | } | 20260508 02:31:00.909 | INFO | Perf report for the last 5.1454 seconds: | 20260508 02:31:00.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:00.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.909 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:31:00.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.909 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:31:00.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:00.910 | INFO | Uploaded 36GiB to GPU in 2.54821s (14.1276GiB/s) | 20260508 02:31:00.910 | INFO | Downloaded 34GiB to GPU in 2.46691s (13.7824GiB/s) | 20260508 02:31:06.050 | INFO | Analyzing chunk frames = [4536320,4536832) | 20260508 02:31:06.050 | INFO | MemInfo: { | 20260508 02:31:06.050 | INFO | "gpu" : " 17 GB", | 20260508 02:31:06.051 | INFO | "huge" : "", | 20260508 02:31:06.051 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:06.051 | INFO | "malloc" : " 14 GB", | 20260508 02:31:06.051 | INFO | "pinned" : " 29 GB" | 20260508 02:31:06.051 | INFO | } | 20260508 02:31:06.051 | INFO | Perf report for the last 5.14151 seconds: | 20260508 02:31:06.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:06.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 02:31:06.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:31:06.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:06.051 | INFO | Uploaded 36GiB to GPU in 2.3531s (15.299GiB/s) | 20260508 02:31:06.051 | INFO | Downloaded 34GiB to GPU in 2.53175s (13.4294GiB/s) | 20260508 02:31:11.191 | INFO | Analyzing chunk frames = [4536832,4537344) | 20260508 02:31:11.191 | INFO | MemInfo: { | 20260508 02:31:11.191 | INFO | "gpu" : " 17 GB", | 20260508 02:31:11.191 | INFO | "huge" : "", | 20260508 02:31:11.191 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:11.191 | INFO | "malloc" : " 14 GB", | 20260508 02:31:11.191 | INFO | "pinned" : " 29 GB" | 20260508 02:31:11.191 | INFO | } | 20260508 02:31:11.191 | INFO | Perf report for the last 5.14064 seconds: | 20260508 02:31:11.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:11.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:31:11.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:31:11.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:11.192 | INFO | Uploaded 36GiB to GPU in 2.39498s (15.0314GiB/s) | 20260508 02:31:11.192 | INFO | Downloaded 34GiB to GPU in 2.41732s (14.0652GiB/s) | 20260508 02:31:16.326 | INFO | Analyzing chunk frames = [4537344,4537856) | 20260508 02:31:16.326 | INFO | MemInfo: { | 20260508 02:31:16.326 | INFO | "gpu" : " 17 GB", | 20260508 02:31:16.326 | INFO | "huge" : "", | 20260508 02:31:16.326 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:16.326 | INFO | "malloc" : " 14 GB", | 20260508 02:31:16.326 | INFO | "pinned" : " 29 GB" | 20260508 02:31:16.327 | INFO | } | 20260508 02:31:16.327 | INFO | Perf report for the last 5.13542 seconds: | 20260508 02:31:16.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:16.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:31:16.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 02:31:16.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:16.327 | INFO | Uploaded 36GiB to GPU in 2.43583s (14.7793GiB/s) | 20260508 02:31:16.327 | INFO | Downloaded 34GiB to GPU in 2.42811s (14.0027GiB/s) | 20260508 02:31:21.468 | INFO | Analyzing chunk frames = [4537856,4538368) | 20260508 02:31:21.468 | INFO | MemInfo: { | 20260508 02:31:21.468 | INFO | "gpu" : " 17 GB", | 20260508 02:31:21.468 | INFO | "huge" : "", | 20260508 02:31:21.468 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:21.468 | INFO | "malloc" : " 14 GB", | 20260508 02:31:21.468 | INFO | "pinned" : " 29 GB" | 20260508 02:31:21.468 | INFO | } | 20260508 02:31:21.468 | INFO | Perf report for the last 5.14105 seconds: | 20260508 02:31:21.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:21.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:31:21.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260508 02:31:21.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:21.468 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:31:21.468 | INFO | Total mem usage: 16.7 GiB | 20260508 02:31:21.468 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:31:21.468 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.41%) | 20260508 02:31:21.468 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:31:21.468 | INFO | High Water Breakdown: | 20260508 02:31:21.468 | INFO | 60.5% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:31:21.468 | INFO | 39.3% : 6 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:31:21.468 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:31:21.468 | INFO | ------------------------------------------ | 20260508 02:31:21.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:31:21.468 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:31:21.468 | INFO | Total mem usage: 26.5 GiB | 20260508 02:31:21.469 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:31:21.469 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:31:21.469 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:31:21.469 | INFO | Uploaded 36GiB to GPU in 2.44775s (14.7074GiB/s) | 20260508 02:31:21.469 | INFO | High Water Breakdown: | 20260508 02:31:21.469 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:31:21.469 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:31:21.469 | INFO | Downloaded 34GiB to GPU in 2.43292s (13.975GiB/s) | 20260508 02:31:21.469 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:31:21.469 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:31:21.469 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:31:21.469 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:31:21.469 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:31:21.469 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:31:21.469 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:31:21.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:31:21.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:31:21.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:31:21.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:31:21.469 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:31:21.469 | INFO | ------------------------------------------ | 20260508 02:31:21.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:31:21.469 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:31:21.469 | INFO | Total mem usage: 27.8 GiB | 20260508 02:31:21.469 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:31:21.469 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:31:21.469 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:31:21.469 | INFO | High Water Breakdown: | 20260508 02:31:21.469 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:31:21.469 | INFO | ------------------------------------------ | 20260508 02:31:21.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:31:21.469 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:31:21.469 | INFO | Total mem usage: 15.7 GiB | 20260508 02:31:21.469 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:31:21.469 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:31:21.469 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:31:21.469 | INFO | High Water Breakdown: | 20260508 02:31:21.469 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:31:21.469 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:31:21.469 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:31:21.469 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:31:21.469 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:31:21.469 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:31:21.469 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:31:21.469 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:31:21.469 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:31:21.469 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:31:21.469 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:31:21.469 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:31:21.469 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:31:21.469 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:31:21.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:31:21.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:31:21.469 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:31:21.469 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:31:21.469 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:31:21.469 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:31:21.469 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:31:21.469 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:31:21.469 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:31:21.469 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:31:21.469 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:31:21.469 | INFO | ------------------------------------------ | 20260508 02:31:21.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:31:21.469 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:31:21.469 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:31:21.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:31:21.469 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.62e+01, 1.40e+01 / 1.70e+00 | 20260508 02:31:21.469 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.23e-02 / 5.51e-03 | 20260508 02:31:21.470 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.45e+00, 5.81e-01 / 1.95e-01 | 20260508 02:31:21.470 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.04e-01 / 1.07e-01 | 20260508 02:31:21.470 | INFO | PerfReport: 16.09%, DME -- 4.20e+01 / 4.62e+01, 4.05e+00 / 2.19e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.60e+00 / 1.53e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.42e+00 / 2.67e+00, 4.24e-01 / 2.32e-01 | 20260508 02:31:21.470 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.21e+00, 2.64e-01 / 1.20e-01 | 20260508 02:31:21.470 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.29e-01 / 3.62e-01, 5.22e-02 / 3.00e-02 | 20260508 02:31:21.470 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:31:21.470 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:31:21.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:31:21.470 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 24.55%, Upload -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 24.36%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 2.37%, LockUpload -- 2.52e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 2.63%, LockDownload -- 2.80e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 15.40%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:31:21.470 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.42e-02 / 0.00e+00 | 20260508 02:31:21.565 | INFO | Starting a round of Baz IO | 20260508 02:31:25.002 | INFO | Populating RTMetrics report | 20260508 02:31:25.002 | INFO | RTMetrics report populated | 20260508 02:31:25.002 | INFO | Opening tmp RTMetrics json file | 20260508 02:31:25.025 | INFO | Serializing RTMetrics to json | 20260508 02:31:25.027 | INFO | Serialization done, swapping tmp file now... | 20260508 02:31:25.028 | INFO | RTMetrics done | 20260508 02:31:25.105 | INFO | Finished a round of Baz IO: Wrote 1.635 GiB in 3.539 seconds | 20260508 02:31:25.105 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.52% / 0.00% / 58.48% | 20260508 02:31:25.105 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.712121% / 3.97364e-06% / 6.75519e-05% | 20260508 02:31:25.105 | INFO | Baz IO overall: | 20260508 02:31:25.105 | INFO | BytesWritten : 2048536941248 | 20260508 02:31:25.105 | INFO | NumEvents : 990280335682 | 20260508 02:31:26.604 | INFO | Analyzing chunk frames = [4538368,4538880) | 20260508 02:31:26.604 | INFO | MemInfo: { | 20260508 02:31:26.604 | INFO | "gpu" : " 17 GB", | 20260508 02:31:26.604 | INFO | "huge" : "", | 20260508 02:31:26.604 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:26.604 | INFO | "malloc" : " 14 GB", | 20260508 02:31:26.604 | INFO | "pinned" : " 29 GB" | 20260508 02:31:26.604 | INFO | } | 20260508 02:31:26.605 | INFO | Perf report for the last 5.13659 seconds: | 20260508 02:31:26.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:26.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:26.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:26.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:26.605 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 02:31:26.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:26.605 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 02:31:26.605 | INFO | PRE_HQ : 65.6%, 1.6, 55.7ms, 0 | 20260508 02:31:26.605 | INFO | BAZWRITER : 68.5%, 4.6, 162.2ms, 0 | 20260508 02:31:26.605 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:31:26.605 | INFO | Uploaded 36GiB to GPU in 2.55161s (14.1087GiB/s) | 20260508 02:31:26.605 | INFO | Downloaded 38GiB to GPU in 2.71226s (14.0105GiB/s) | 20260508 02:31:31.748 | INFO | Analyzing chunk frames = [4538880,4539392) | 20260508 02:31:31.748 | INFO | MemInfo: { | 20260508 02:31:31.748 | INFO | "gpu" : " 17 GB", | 20260508 02:31:31.748 | INFO | "huge" : "", | 20260508 02:31:31.748 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:31.748 | INFO | "malloc" : " 14 GB", | 20260508 02:31:31.748 | INFO | "pinned" : " 29 GB" | 20260508 02:31:31.749 | INFO | } | 20260508 02:31:31.749 | INFO | Perf report for the last 5.14415 seconds: | 20260508 02:31:31.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:31.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 02:31:31.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:31:31.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:31.749 | INFO | Uploaded 36GiB to GPU in 2.58973s (13.9011GiB/s) | 20260508 02:31:31.749 | INFO | Downloaded 34GiB to GPU in 2.51109s (13.5399GiB/s) | 20260508 02:31:36.886 | INFO | Analyzing chunk frames = [4539392,4539904) | 20260508 02:31:36.886 | INFO | MemInfo: { | 20260508 02:31:36.886 | INFO | "gpu" : " 17 GB", | 20260508 02:31:36.886 | INFO | "huge" : "", | 20260508 02:31:36.886 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:36.886 | INFO | "malloc" : " 14 GB", | 20260508 02:31:36.886 | INFO | "pinned" : " 29 GB" | 20260508 02:31:36.887 | INFO | } | 20260508 02:31:36.887 | INFO | Perf report for the last 5.13787 seconds: | 20260508 02:31:36.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:36.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:31:36.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:31:36.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:36.887 | INFO | Uploaded 36GiB to GPU in 2.56466s (14.037GiB/s) | 20260508 02:31:36.887 | INFO | Downloaded 34GiB to GPU in 2.49942s (13.6032GiB/s) | 20260508 02:31:42.024 | INFO | Analyzing chunk frames = [4539904,4540416) | 20260508 02:31:42.024 | INFO | MemInfo: { | 20260508 02:31:42.024 | INFO | "gpu" : " 17 GB", | 20260508 02:31:42.024 | INFO | "huge" : "", | 20260508 02:31:42.024 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:42.024 | INFO | "malloc" : " 14 GB", | 20260508 02:31:42.024 | INFO | "pinned" : " 29 GB" | 20260508 02:31:42.024 | INFO | } | 20260508 02:31:42.024 | INFO | Perf report for the last 5.1375 seconds: | 20260508 02:31:42.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:42.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.024 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:31:42.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.024 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:31:42.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:42.025 | INFO | Uploaded 36GiB to GPU in 2.53496s (14.2014GiB/s) | 20260508 02:31:42.025 | INFO | Downloaded 34GiB to GPU in 2.49373s (13.6342GiB/s) | 20260508 02:31:47.167 | INFO | Analyzing chunk frames = [4540416,4540928) | 20260508 02:31:47.167 | INFO | MemInfo: { | 20260508 02:31:47.167 | INFO | "gpu" : " 17 GB", | 20260508 02:31:47.167 | INFO | "huge" : "", | 20260508 02:31:47.167 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:47.168 | INFO | "malloc" : " 14 GB", | 20260508 02:31:47.168 | INFO | "pinned" : " 29 GB" | 20260508 02:31:47.168 | INFO | } | 20260508 02:31:47.168 | INFO | Perf report for the last 5.14345 seconds: | 20260508 02:31:47.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:47.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 02:31:47.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:31:47.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:47.168 | INFO | Uploaded 36GiB to GPU in 2.43768s (14.7681GiB/s) | 20260508 02:31:47.169 | INFO | Downloaded 34GiB to GPU in 2.56978s (13.2307GiB/s) | 20260508 02:31:52.307 | INFO | Analyzing chunk frames = [4540928,4541440) | 20260508 02:31:52.307 | INFO | MemInfo: { | 20260508 02:31:52.307 | INFO | "gpu" : " 17 GB", | 20260508 02:31:52.307 | INFO | "huge" : "", | 20260508 02:31:52.307 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:52.307 | INFO | "malloc" : " 14 GB", | 20260508 02:31:52.307 | INFO | "pinned" : " 29 GB" | 20260508 02:31:52.307 | INFO | } | 20260508 02:31:52.307 | INFO | Perf report for the last 5.13966 seconds: | 20260508 02:31:52.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:52.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.307 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 02:31:52.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.307 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:31:52.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:52.308 | INFO | Uploaded 36GiB to GPU in 2.44699s (14.712GiB/s) | 20260508 02:31:52.308 | INFO | Downloaded 34GiB to GPU in 2.419s (14.0554GiB/s) | 20260508 02:31:57.441 | INFO | Analyzing chunk frames = [4541440,4541952) | 20260508 02:31:57.441 | INFO | MemInfo: { | 20260508 02:31:57.441 | INFO | "gpu" : " 17 GB", | 20260508 02:31:57.441 | INFO | "huge" : "", | 20260508 02:31:57.441 | INFO | "huge_shared" : " 29 GB", | 20260508 02:31:57.441 | INFO | "malloc" : " 14 GB", | 20260508 02:31:57.441 | INFO | "pinned" : " 29 GB" | 20260508 02:31:57.441 | INFO | } | 20260508 02:31:57.441 | INFO | Perf report for the last 5.13373 seconds: | 20260508 02:31:57.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:31:57.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:31:57.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:31:57.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:31:57.441 | INFO | Uploaded 36GiB to GPU in 2.44853s (14.7027GiB/s) | 20260508 02:31:57.441 | INFO | Downloaded 34GiB to GPU in 2.4186s (14.0577GiB/s) | 20260508 02:32:02.577 | INFO | Analyzing chunk frames = [4541952,4542464) | 20260508 02:32:02.577 | INFO | MemInfo: { | 20260508 02:32:02.577 | INFO | "gpu" : " 17 GB", | 20260508 02:32:02.577 | INFO | "huge" : "", | 20260508 02:32:02.577 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:02.577 | INFO | "malloc" : " 14 GB", | 20260508 02:32:02.577 | INFO | "pinned" : " 29 GB" | 20260508 02:32:02.577 | INFO | } | 20260508 02:32:02.577 | INFO | Perf report for the last 5.13588 seconds: | 20260508 02:32:02.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:02.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:32:02.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:32:02.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:02.577 | INFO | Uploaded 36GiB to GPU in 2.56329s (14.0444GiB/s) | 20260508 02:32:02.577 | INFO | Downloaded 34GiB to GPU in 2.45424s (13.8536GiB/s) | 20260508 02:32:02.674 | INFO | Starting a round of Baz IO | 20260508 02:32:06.133 | INFO | Populating RTMetrics report | 20260508 02:32:06.133 | INFO | RTMetrics report populated | 20260508 02:32:06.133 | INFO | Opening tmp RTMetrics json file | 20260508 02:32:06.157 | INFO | Serializing RTMetrics to json | 20260508 02:32:06.217 | INFO | Serialization done, swapping tmp file now... | 20260508 02:32:06.222 | INFO | RTMetrics done | 20260508 02:32:06.246 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.572 seconds | 20260508 02:32:06.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.52% / 0.00% / 58.48% | 20260508 02:32:06.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.713054% / 7.94729e-06% / 6.75519e-05% | 20260508 02:32:06.246 | INFO | Baz IO overall: | 20260508 02:32:06.246 | INFO | BytesWritten : 2050294030431 | 20260508 02:32:06.246 | INFO | NumEvents : 991122471463 | 20260508 02:32:07.719 | INFO | Analyzing chunk frames = [4542464,4542976) | 20260508 02:32:07.719 | INFO | MemInfo: { | 20260508 02:32:07.719 | INFO | "gpu" : " 17 GB", | 20260508 02:32:07.719 | INFO | "huge" : "", | 20260508 02:32:07.719 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:07.719 | INFO | "malloc" : " 14 GB", | 20260508 02:32:07.719 | INFO | "pinned" : " 29 GB" | 20260508 02:32:07.719 | INFO | } | 20260508 02:32:07.719 | INFO | Perf report for the last 5.14224 seconds: | 20260508 02:32:07.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:07.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:07.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:07.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:07.719 | INFO | ANALYSIS : 68.2%, 3.0, 106.5ms, 0 | 20260508 02:32:07.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:07.719 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260508 02:32:07.719 | INFO | PRE_HQ : 65.1%, 1.6, 55.7ms, 0 | 20260508 02:32:07.719 | INFO | BAZWRITER : 69.1%, 3.4, 121.2ms, 0 | 20260508 02:32:07.719 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 02:32:07.719 | INFO | Uploaded 36GiB to GPU in 2.58085s (13.9489GiB/s) | 20260508 02:32:07.719 | INFO | Downloaded 38GiB to GPU in 2.67667s (14.1967GiB/s) | 20260508 02:32:12.860 | INFO | Analyzing chunk frames = [4542976,4543488) | 20260508 02:32:12.860 | INFO | MemInfo: { | 20260508 02:32:12.860 | INFO | "gpu" : " 17 GB", | 20260508 02:32:12.860 | INFO | "huge" : "", | 20260508 02:32:12.860 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:12.860 | INFO | "malloc" : " 14 GB", | 20260508 02:32:12.860 | INFO | "pinned" : " 29 GB" | 20260508 02:32:12.860 | INFO | } | 20260508 02:32:12.860 | INFO | Perf report for the last 5.14103 seconds: | 20260508 02:32:12.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:12.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.860 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:32:12.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.860 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:32:12.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:12.861 | INFO | Uploaded 36GiB to GPU in 2.51846s (14.2945GiB/s) | 20260508 02:32:12.861 | INFO | Downloaded 34GiB to GPU in 2.42434s (14.0244GiB/s) | 20260508 02:32:18.007 | INFO | Analyzing chunk frames = [4543488,4544000) | 20260508 02:32:18.007 | INFO | MemInfo: { | 20260508 02:32:18.007 | INFO | "gpu" : " 17 GB", | 20260508 02:32:18.007 | INFO | "huge" : "", | 20260508 02:32:18.007 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:18.007 | INFO | "malloc" : " 14 GB", | 20260508 02:32:18.007 | INFO | "pinned" : " 29 GB" | 20260508 02:32:18.007 | INFO | } | 20260508 02:32:18.007 | INFO | Perf report for the last 5.14721 seconds: | 20260508 02:32:18.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:18.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.007 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 02:32:18.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.008 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:32:18.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:18.008 | INFO | Uploaded 36GiB to GPU in 2.51655s (14.3053GiB/s) | 20260508 02:32:18.008 | INFO | Downloaded 34GiB to GPU in 2.4866s (13.6733GiB/s) | 20260508 02:32:23.137 | INFO | Analyzing chunk frames = [4544000,4544512) | 20260508 02:32:23.137 | INFO | MemInfo: { | 20260508 02:32:23.137 | INFO | "gpu" : " 17 GB", | 20260508 02:32:23.137 | INFO | "huge" : "", | 20260508 02:32:23.137 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:23.137 | INFO | "malloc" : " 14 GB", | 20260508 02:32:23.137 | INFO | "pinned" : " 29 GB" | 20260508 02:32:23.137 | INFO | } | 20260508 02:32:23.137 | INFO | Perf report for the last 5.12993 seconds: | 20260508 02:32:23.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:23.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.137 | INFO | ANALYSIS : 67.4%, 3.0, 104.1ms, 0 | 20260508 02:32:23.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.137 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:32:23.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:23.138 | INFO | Uploaded 36GiB to GPU in 2.53463s (14.2032GiB/s) | 20260508 02:32:23.138 | INFO | Downloaded 34GiB to GPU in 2.50853s (13.5538GiB/s) | 20260508 02:32:28.278 | INFO | Analyzing chunk frames = [4544512,4545024) | 20260508 02:32:28.278 | INFO | MemInfo: { | 20260508 02:32:28.278 | INFO | "gpu" : " 17 GB", | 20260508 02:32:28.278 | INFO | "huge" : "", | 20260508 02:32:28.278 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:28.278 | INFO | "malloc" : " 14 GB", | 20260508 02:32:28.278 | INFO | "pinned" : " 29 GB" | 20260508 02:32:28.278 | INFO | } | 20260508 02:32:28.278 | INFO | Perf report for the last 5.14098 seconds: | 20260508 02:32:28.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:28.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.278 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 02:32:28.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.278 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 02:32:28.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:28.279 | INFO | Uploaded 36GiB to GPU in 2.40469s (14.9708GiB/s) | 20260508 02:32:28.279 | INFO | Downloaded 34GiB to GPU in 2.55066s (13.3299GiB/s) | 20260508 02:32:33.418 | INFO | Analyzing chunk frames = [4545024,4545536) | 20260508 02:32:33.418 | INFO | MemInfo: { | 20260508 02:32:33.418 | INFO | "gpu" : " 17 GB", | 20260508 02:32:33.418 | INFO | "huge" : "", | 20260508 02:32:33.418 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:33.418 | INFO | "malloc" : " 14 GB", | 20260508 02:32:33.418 | INFO | "pinned" : " 29 GB" | 20260508 02:32:33.418 | INFO | } | 20260508 02:32:33.419 | INFO | Perf report for the last 5.14027 seconds: | 20260508 02:32:33.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:33.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 02:32:33.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:32:33.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:33.419 | INFO | Uploaded 36GiB to GPU in 2.39883s (15.0073GiB/s) | 20260508 02:32:33.419 | INFO | Downloaded 34GiB to GPU in 2.41162s (14.0984GiB/s) | 20260508 02:32:38.560 | INFO | Analyzing chunk frames = [4545536,4546048) | 20260508 02:32:38.560 | INFO | MemInfo: { | 20260508 02:32:38.560 | INFO | "gpu" : " 17 GB", | 20260508 02:32:38.560 | INFO | "huge" : "", | 20260508 02:32:38.560 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:38.560 | INFO | "malloc" : " 14 GB", | 20260508 02:32:38.560 | INFO | "pinned" : " 29 GB" | 20260508 02:32:38.560 | INFO | } | 20260508 02:32:38.560 | INFO | Perf report for the last 5.14138 seconds: | 20260508 02:32:38.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:38.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:32:38.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:32:38.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:38.560 | INFO | Uploaded 36GiB to GPU in 2.42804s (14.8268GiB/s) | 20260508 02:32:38.560 | INFO | Downloaded 34GiB to GPU in 2.42077s (14.0451GiB/s) | 20260508 02:32:43.698 | INFO | Analyzing chunk frames = [4546048,4546560) | 20260508 02:32:43.698 | INFO | MemInfo: { | 20260508 02:32:43.698 | INFO | "gpu" : " 17 GB", | 20260508 02:32:43.698 | INFO | "huge" : "", | 20260508 02:32:43.698 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:43.698 | INFO | "malloc" : " 14 GB", | 20260508 02:32:43.698 | INFO | "pinned" : " 29 GB" | 20260508 02:32:43.698 | INFO | } | 20260508 02:32:43.698 | INFO | Perf report for the last 5.13776 seconds: | 20260508 02:32:43.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:43.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:32:43.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:32:43.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:43.698 | INFO | Uploaded 36GiB to GPU in 2.55455s (14.0925GiB/s) | 20260508 02:32:43.698 | INFO | Downloaded 34GiB to GPU in 2.46045s (13.8186GiB/s) | 20260508 02:32:43.698 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:32:43.698 | INFO | Total mem usage: 16.7 GiB | 20260508 02:32:43.698 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:32:43.698 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -7.98%) | 20260508 02:32:43.698 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:32:43.698 | INFO | High Water Breakdown: | 20260508 02:32:43.698 | INFO | 60.7% : 9.38 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:32:43.698 | INFO | 39.2% : 6.05 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:32:43.698 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:32:43.698 | INFO | ------------------------------------------ | 20260508 02:32:43.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:32:43.698 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:32:43.698 | INFO | Total mem usage: 26.5 GiB | 20260508 02:32:43.698 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:32:43.698 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 02:32:43.698 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 02:32:43.698 | INFO | High Water Breakdown: | 20260508 02:32:43.698 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:32:43.698 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:32:43.698 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:32:43.698 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:32:43.698 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:32:43.698 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:32:43.698 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:32:43.698 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:32:43.698 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:32:43.698 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:32:43.698 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:32:43.698 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:32:43.698 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:32:43.698 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:32:43.698 | INFO | ------------------------------------------ | 20260508 02:32:43.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:32:43.698 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:32:43.698 | INFO | Total mem usage: 27.8 GiB | 20260508 02:32:43.698 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:32:43.698 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:32:43.698 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:32:43.698 | INFO | High Water Breakdown: | 20260508 02:32:43.698 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:32:43.698 | INFO | ------------------------------------------ | 20260508 02:32:43.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:32:43.698 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:32:43.698 | INFO | Total mem usage: 15.7 GiB | 20260508 02:32:43.698 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:32:43.698 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:32:43.698 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:32:43.698 | INFO | High Water Breakdown: | 20260508 02:32:43.698 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:32:43.698 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:32:43.698 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:32:43.698 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:32:43.698 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:32:43.698 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:32:43.698 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:32:43.698 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:32:43.698 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:32:43.698 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:32:43.698 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:32:43.698 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:32:43.698 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:32:43.698 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:32:43.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:32:43.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:32:43.698 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:32:43.698 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:32:43.698 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:32:43.698 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:32:43.698 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:32:43.698 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:32:43.698 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:32:43.698 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:32:43.698 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:32:43.698 | INFO | ------------------------------------------ | 20260508 02:32:43.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:32:43.698 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:32:43.698 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:32:43.698 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:32:43.698 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.62e+01, 1.40e+01 / 1.69e+00 | 20260508 02:32:43.698 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.02e-01, 1.10e-02 / 5.71e-03 | 20260508 02:32:43.698 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.47e+00, 5.80e-01 / 1.78e-01 | 20260508 02:32:43.698 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.04e-01 / 1.16e-01 | 20260508 02:32:43.698 | INFO | PerfReport: 16.12%, DME -- 4.21e+01 / 4.62e+01, 3.96e+00 / 2.00e+00 | 20260508 02:32:43.698 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.60e+00 / 1.34e+00 | 20260508 02:32:43.698 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.41e+00 / 2.70e+00, 4.24e-01 / 1.92e-01 | 20260508 02:32:43.698 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.27e+00, 2.68e-01 / 1.29e-01 | 20260508 02:32:43.698 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.66e-01, 5.19e-02 / 2.42e-02 | 20260508 02:32:43.699 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:32:43.699 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:32:43.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:32:43.699 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 24.45%, Upload -- 2.61e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 24.42%, Download -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 30.58%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 2.24%, LockUpload -- 2.39e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 2.81%, LockDownload -- 3.00e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 15.49%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 02:32:43.699 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260508 02:32:43.795 | INFO | Starting a round of Baz IO | 20260508 02:32:47.243 | INFO | Populating RTMetrics report | 20260508 02:32:47.243 | INFO | RTMetrics report populated | 20260508 02:32:47.243 | INFO | Opening tmp RTMetrics json file | 20260508 02:32:47.268 | INFO | Serializing RTMetrics to json | 20260508 02:32:47.270 | INFO | Serialization done, swapping tmp file now... | 20260508 02:32:47.270 | INFO | RTMetrics done | 20260508 02:32:47.345 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.55 seconds | 20260508 02:32:47.345 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.51% / 0.00% / 58.48% | 20260508 02:32:47.345 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.710801% / 1.19209e-05% / 5.5631e-05% | 20260508 02:32:47.345 | INFO | Baz IO overall: | 20260508 02:32:47.345 | INFO | BytesWritten : 2052051140707 | 20260508 02:32:47.345 | INFO | NumEvents : 991964700381 | 20260508 02:32:48.834 | INFO | Analyzing chunk frames = [4546560,4547072) | 20260508 02:32:48.834 | INFO | MemInfo: { | 20260508 02:32:48.834 | INFO | "gpu" : " 17 GB", | 20260508 02:32:48.834 | INFO | "huge" : "", | 20260508 02:32:48.834 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:48.834 | INFO | "malloc" : " 14 GB", | 20260508 02:32:48.834 | INFO | "pinned" : " 29 GB" | 20260508 02:32:48.834 | INFO | } | 20260508 02:32:48.834 | INFO | Perf report for the last 5.13634 seconds: | 20260508 02:32:48.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:48.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:48.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:48.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:48.834 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:32:48.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:48.834 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 02:32:48.834 | INFO | PRE_HQ : 65.7%, 1.6, 56.1ms, 0 | 20260508 02:32:48.834 | INFO | BAZWRITER : 68.7%, 3.5, 122.7ms, 0 | 20260508 02:32:48.834 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 02:32:48.835 | INFO | Uploaded 36GiB to GPU in 2.55015s (14.1168GiB/s) | 20260508 02:32:48.835 | INFO | Downloaded 38GiB to GPU in 2.69199s (14.116GiB/s) | 20260508 02:32:53.973 | INFO | Analyzing chunk frames = [4547072,4547584) | 20260508 02:32:53.973 | INFO | MemInfo: { | 20260508 02:32:53.973 | INFO | "gpu" : " 17 GB", | 20260508 02:32:53.973 | INFO | "huge" : "", | 20260508 02:32:53.973 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:53.973 | INFO | "malloc" : " 14 GB", | 20260508 02:32:53.973 | INFO | "pinned" : " 29 GB" | 20260508 02:32:53.973 | INFO | } | 20260508 02:32:53.974 | INFO | Perf report for the last 5.13938 seconds: | 20260508 02:32:53.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:53.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:32:53.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:32:53.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:53.974 | INFO | Uploaded 36GiB to GPU in 2.55601s (14.0845GiB/s) | 20260508 02:32:53.974 | INFO | Downloaded 34GiB to GPU in 2.41916s (14.0544GiB/s) | 20260508 02:32:59.115 | INFO | Analyzing chunk frames = [4547584,4548096) | 20260508 02:32:59.115 | INFO | MemInfo: { | 20260508 02:32:59.115 | INFO | "gpu" : " 17 GB", | 20260508 02:32:59.115 | INFO | "huge" : "", | 20260508 02:32:59.115 | INFO | "huge_shared" : " 29 GB", | 20260508 02:32:59.115 | INFO | "malloc" : " 14 GB", | 20260508 02:32:59.115 | INFO | "pinned" : " 29 GB" | 20260508 02:32:59.115 | INFO | } | 20260508 02:32:59.116 | INFO | Perf report for the last 5.14217 seconds: | 20260508 02:32:59.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:32:59.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 02:32:59.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:32:59.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:32:59.116 | INFO | Uploaded 36GiB to GPU in 2.56322s (14.0448GiB/s) | 20260508 02:32:59.116 | INFO | Downloaded 34GiB to GPU in 2.4857s (13.6782GiB/s) | 20260508 02:33:04.258 | INFO | Analyzing chunk frames = [4548096,4548608) | 20260508 02:33:04.258 | INFO | MemInfo: { | 20260508 02:33:04.258 | INFO | "gpu" : " 17 GB", | 20260508 02:33:04.258 | INFO | "huge" : "", | 20260508 02:33:04.258 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:04.259 | INFO | "malloc" : " 14 GB", | 20260508 02:33:04.259 | INFO | "pinned" : " 29 GB" | 20260508 02:33:04.259 | INFO | } | 20260508 02:33:04.259 | INFO | Perf report for the last 5.14299 seconds: | 20260508 02:33:04.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:04.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 02:33:04.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:33:04.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:04.259 | INFO | Uploaded 36GiB to GPU in 2.57985s (13.9543GiB/s) | 20260508 02:33:04.259 | INFO | Downloaded 34GiB to GPU in 2.49391s (13.6332GiB/s) | 20260508 02:33:09.391 | INFO | Analyzing chunk frames = [4548608,4549120) | 20260508 02:33:09.391 | INFO | MemInfo: { | 20260508 02:33:09.391 | INFO | "gpu" : " 17 GB", | 20260508 02:33:09.391 | INFO | "huge" : "", | 20260508 02:33:09.391 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:09.391 | INFO | "malloc" : " 14 GB", | 20260508 02:33:09.391 | INFO | "pinned" : " 29 GB" | 20260508 02:33:09.391 | INFO | } | 20260508 02:33:09.391 | INFO | Perf report for the last 5.13269 seconds: | 20260508 02:33:09.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:09.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.392 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 02:33:09.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.392 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:33:09.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:09.392 | INFO | Uploaded 36GiB to GPU in 2.4411s (14.7474GiB/s) | 20260508 02:33:09.392 | INFO | Downloaded 34GiB to GPU in 2.56415s (13.2598GiB/s) | 20260508 02:33:14.528 | INFO | Analyzing chunk frames = [4549120,4549632) | 20260508 02:33:14.528 | INFO | MemInfo: { | 20260508 02:33:14.528 | INFO | "gpu" : " 17 GB", | 20260508 02:33:14.528 | INFO | "huge" : "", | 20260508 02:33:14.528 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:14.528 | INFO | "malloc" : " 14 GB", | 20260508 02:33:14.528 | INFO | "pinned" : " 29 GB" | 20260508 02:33:14.528 | INFO | } | 20260508 02:33:14.528 | INFO | Perf report for the last 5.137 seconds: | 20260508 02:33:14.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:14.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.529 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:33:14.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.529 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:33:14.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:14.529 | INFO | Uploaded 36GiB to GPU in 2.41843s (14.8857GiB/s) | 20260508 02:33:14.529 | INFO | Downloaded 34GiB to GPU in 2.41279s (14.0916GiB/s) | 20260508 02:33:19.671 | INFO | Analyzing chunk frames = [4549632,4550144) | 20260508 02:33:19.671 | INFO | MemInfo: { | 20260508 02:33:19.671 | INFO | "gpu" : " 17 GB", | 20260508 02:33:19.671 | INFO | "huge" : "", | 20260508 02:33:19.671 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:19.671 | INFO | "malloc" : " 14 GB", | 20260508 02:33:19.671 | INFO | "pinned" : " 29 GB" | 20260508 02:33:19.671 | INFO | } | 20260508 02:33:19.671 | INFO | Perf report for the last 5.1427 seconds: | 20260508 02:33:19.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:19.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 02:33:19.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:33:19.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:19.671 | INFO | Uploaded 36GiB to GPU in 2.55946s (14.0654GiB/s) | 20260508 02:33:19.671 | INFO | Downloaded 34GiB to GPU in 2.45808s (13.8319GiB/s) | 20260508 02:33:24.815 | INFO | Analyzing chunk frames = [4550144,4550656) | 20260508 02:33:24.815 | INFO | MemInfo: { | 20260508 02:33:24.815 | INFO | "gpu" : " 17 GB", | 20260508 02:33:24.815 | INFO | "huge" : "", | 20260508 02:33:24.815 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:24.815 | INFO | "malloc" : " 14 GB", | 20260508 02:33:24.815 | INFO | "pinned" : " 29 GB" | 20260508 02:33:24.815 | INFO | } | 20260508 02:33:24.815 | INFO | Perf report for the last 5.14407 seconds: | 20260508 02:33:24.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:24.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 02:33:24.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 02:33:24.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:24.815 | INFO | Uploaded 36GiB to GPU in 2.56372s (14.0421GiB/s) | 20260508 02:33:24.816 | INFO | Downloaded 34GiB to GPU in 2.45781s (13.8335GiB/s) | 20260508 02:33:24.912 | INFO | Starting a round of Baz IO | 20260508 02:33:28.363 | INFO | Populating RTMetrics report | 20260508 02:33:28.364 | INFO | RTMetrics report populated | 20260508 02:33:28.364 | INFO | Opening tmp RTMetrics json file | 20260508 02:33:28.385 | INFO | Serializing RTMetrics to json | 20260508 02:33:28.429 | INFO | Serialization done, swapping tmp file now... | 20260508 02:33:28.444 | INFO | RTMetrics done | 20260508 02:33:28.477 | INFO | Finished a round of Baz IO: Wrote 1.635 GiB in 3.564 seconds | 20260508 02:33:28.477 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.51% / 0.00% / 58.49% | 20260508 02:33:28.477 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.708052% / 0% / 8.74201e-05% | 20260508 02:33:28.477 | INFO | Baz IO overall: | 20260508 02:33:28.477 | INFO | BytesWritten : 2053807104790 | 20260508 02:33:28.477 | INFO | NumEvents : 992806577610 | 20260508 02:33:29.956 | INFO | Analyzing chunk frames = [4550656,4551168) | 20260508 02:33:29.956 | INFO | MemInfo: { | 20260508 02:33:29.956 | INFO | "gpu" : " 17 GB", | 20260508 02:33:29.956 | INFO | "huge" : "", | 20260508 02:33:29.956 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:29.956 | INFO | "malloc" : " 14 GB", | 20260508 02:33:29.956 | INFO | "pinned" : " 29 GB" | 20260508 02:33:29.956 | INFO | } | 20260508 02:33:29.956 | INFO | Perf report for the last 5.14065 seconds: | 20260508 02:33:29.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:29.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:29.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:29.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:29.956 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 02:33:29.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:29.956 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260508 02:33:29.956 | INFO | PRE_HQ : 65.3%, 1.6, 55.8ms, 0 | 20260508 02:33:29.956 | INFO | BAZWRITER : 68.9%, 3.5, 122.3ms, 0 | 20260508 02:33:29.956 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 02:33:29.956 | INFO | Uploaded 36GiB to GPU in 2.56067s (14.0588GiB/s) | 20260508 02:33:29.956 | INFO | Downloaded 38GiB to GPU in 2.67864s (14.1863GiB/s) | 20260508 02:33:35.089 | INFO | Analyzing chunk frames = [4551168,4551680) | 20260508 02:33:35.089 | INFO | MemInfo: { | 20260508 02:33:35.089 | INFO | "gpu" : " 17 GB", | 20260508 02:33:35.089 | INFO | "huge" : "", | 20260508 02:33:35.090 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:35.090 | INFO | "malloc" : " 14 GB", | 20260508 02:33:35.090 | INFO | "pinned" : " 29 GB" | 20260508 02:33:35.090 | INFO | } | 20260508 02:33:35.090 | INFO | Perf report for the last 5.13372 seconds: | 20260508 02:33:35.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:35.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260508 02:33:35.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 02:33:35.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:35.090 | INFO | Uploaded 36GiB to GPU in 2.58397s (13.932GiB/s) | 20260508 02:33:35.090 | INFO | Downloaded 34GiB to GPU in 2.41953s (14.0523GiB/s) | 20260508 02:33:40.234 | INFO | Analyzing chunk frames = [4551680,4552192) | 20260508 02:33:40.234 | INFO | MemInfo: { | 20260508 02:33:40.234 | INFO | "gpu" : " 17 GB", | 20260508 02:33:40.234 | INFO | "huge" : "", | 20260508 02:33:40.234 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:40.234 | INFO | "malloc" : " 14 GB", | 20260508 02:33:40.234 | INFO | "pinned" : " 29 GB" | 20260508 02:33:40.234 | INFO | } | 20260508 02:33:40.234 | INFO | Perf report for the last 5.14473 seconds: | 20260508 02:33:40.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:40.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.234 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 02:33:40.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.234 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:33:40.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:40.235 | INFO | Uploaded 36GiB to GPU in 2.58977s (13.9008GiB/s) | 20260508 02:33:40.235 | INFO | Downloaded 34GiB to GPU in 2.49359s (13.635GiB/s) | 20260508 02:33:45.371 | INFO | Analyzing chunk frames = [4552192,4552704) | 20260508 02:33:45.371 | INFO | MemInfo: { | 20260508 02:33:45.371 | INFO | "gpu" : " 17 GB", | 20260508 02:33:45.371 | INFO | "huge" : "", | 20260508 02:33:45.371 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:45.371 | INFO | "malloc" : " 14 GB", | 20260508 02:33:45.371 | INFO | "pinned" : " 29 GB" | 20260508 02:33:45.371 | INFO | } | 20260508 02:33:45.372 | INFO | Perf report for the last 5.13716 seconds: | 20260508 02:33:45.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:45.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:33:45.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:33:45.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:45.372 | INFO | Uploaded 36GiB to GPU in 2.56274s (14.0475GiB/s) | 20260508 02:33:45.372 | INFO | Downloaded 34GiB to GPU in 2.49643s (13.6195GiB/s) | 20260508 02:33:50.510 | INFO | Analyzing chunk frames = [4552704,4553216) | 20260508 02:33:50.510 | INFO | MemInfo: { | 20260508 02:33:50.510 | INFO | "gpu" : " 17 GB", | 20260508 02:33:50.510 | INFO | "huge" : "", | 20260508 02:33:50.510 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:50.510 | INFO | "malloc" : " 14 GB", | 20260508 02:33:50.510 | INFO | "pinned" : " 29 GB" | 20260508 02:33:50.510 | INFO | } | 20260508 02:33:50.510 | INFO | Perf report for the last 5.13893 seconds: | 20260508 02:33:50.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:50.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.511 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 02:33:50.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.511 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:33:50.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:50.511 | INFO | Uploaded 36GiB to GPU in 2.44318s (14.7349GiB/s) | 20260508 02:33:50.511 | INFO | Downloaded 34GiB to GPU in 2.57786s (13.1892GiB/s) | 20260508 02:33:55.649 | INFO | Analyzing chunk frames = [4553216,4553728) | 20260508 02:33:55.649 | INFO | MemInfo: { | 20260508 02:33:55.649 | INFO | "gpu" : " 17 GB", | 20260508 02:33:55.649 | INFO | "huge" : "", | 20260508 02:33:55.649 | INFO | "huge_shared" : " 29 GB", | 20260508 02:33:55.649 | INFO | "malloc" : " 14 GB", | 20260508 02:33:55.649 | INFO | "pinned" : " 29 GB" | 20260508 02:33:55.649 | INFO | } | 20260508 02:33:55.649 | INFO | Perf report for the last 5.13876 seconds: | 20260508 02:33:55.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:33:55.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.649 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:33:55.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.650 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:33:55.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:33:55.650 | INFO | Uploaded 36GiB to GPU in 2.41205s (14.9251GiB/s) | 20260508 02:33:55.650 | INFO | Downloaded 34GiB to GPU in 2.42382s (14.0274GiB/s) | 20260508 02:34:00.782 | INFO | Analyzing chunk frames = [4553728,4554240) | 20260508 02:34:00.782 | INFO | MemInfo: { | 20260508 02:34:00.782 | INFO | "gpu" : " 17 GB", | 20260508 02:34:00.782 | INFO | "huge" : "", | 20260508 02:34:00.782 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:00.782 | INFO | "malloc" : " 14 GB", | 20260508 02:34:00.782 | INFO | "pinned" : " 29 GB" | 20260508 02:34:00.782 | INFO | } | 20260508 02:34:00.782 | INFO | Perf report for the last 5.13318 seconds: | 20260508 02:34:00.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:00.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.783 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:34:00.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.783 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:34:00.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:00.783 | INFO | Uploaded 36GiB to GPU in 2.43401s (14.7904GiB/s) | 20260508 02:34:00.783 | INFO | Downloaded 34GiB to GPU in 2.42026s (14.0481GiB/s) | 20260508 02:34:05.929 | INFO | Analyzing chunk frames = [4554240,4554752) | 20260508 02:34:05.929 | INFO | MemInfo: { | 20260508 02:34:05.929 | INFO | "gpu" : " 17 GB", | 20260508 02:34:05.929 | INFO | "huge" : "", | 20260508 02:34:05.929 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:05.929 | INFO | "malloc" : " 14 GB", | 20260508 02:34:05.929 | INFO | "pinned" : " 29 GB" | 20260508 02:34:05.929 | INFO | } | 20260508 02:34:05.929 | INFO | Perf report for the last 5.14685 seconds: | 20260508 02:34:05.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:05.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.929 | INFO | ANALYSIS : 67.1%, 3.0, 104.8ms, 0 | 20260508 02:34:05.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.929 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:34:05.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:05.930 | INFO | Uploaded 36GiB to GPU in 2.53453s (14.2038GiB/s) | 20260508 02:34:05.930 | INFO | Downloaded 34GiB to GPU in 2.46002s (13.821GiB/s) | 20260508 02:34:05.930 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:34:05.930 | INFO | Total mem usage: 16.7 GiB | 20260508 02:34:05.930 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:34:05.930 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.2%) | 20260508 02:34:05.930 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:34:05.930 | INFO | High Water Breakdown: | 20260508 02:34:05.930 | INFO | 59.7% : 8.64 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:34:05.930 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:34:05.930 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:34:05.930 | INFO | ------------------------------------------ | 20260508 02:34:05.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:34:05.930 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:34:05.930 | INFO | Total mem usage: 26.5 GiB | 20260508 02:34:05.930 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:34:05.930 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:34:05.930 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:34:05.930 | INFO | High Water Breakdown: | 20260508 02:34:05.930 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:34:05.930 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:34:05.930 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:34:05.930 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:34:05.930 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:34:05.930 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:34:05.930 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:34:05.930 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:34:05.930 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:34:05.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:34:05.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:34:05.930 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:34:05.930 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:34:05.930 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:34:05.930 | INFO | ------------------------------------------ | 20260508 02:34:05.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:34:05.930 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:34:05.930 | INFO | Total mem usage: 27.8 GiB | 20260508 02:34:05.930 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:34:05.930 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 02:34:05.930 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 02:34:05.930 | INFO | High Water Breakdown: | 20260508 02:34:05.930 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 02:34:05.930 | INFO | ------------------------------------------ | 20260508 02:34:05.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:34:05.930 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:34:05.930 | INFO | Total mem usage: 15.7 GiB | 20260508 02:34:05.930 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:34:05.930 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:34:05.930 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:34:05.930 | INFO | High Water Breakdown: | 20260508 02:34:05.930 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:34:05.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:34:05.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:34:05.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:34:05.931 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:34:05.931 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:34:05.931 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:34:05.931 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:34:05.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:34:05.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:34:05.931 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:34:05.931 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:34:05.931 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:34:05.931 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:34:05.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:34:05.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:34:05.931 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:34:05.931 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:34:05.931 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:34:05.931 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:34:05.931 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:34:05.931 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:34:05.931 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:34:05.931 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:34:05.931 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:34:05.931 | INFO | ------------------------------------------ | 20260508 02:34:05.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:34:05.931 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:34:05.931 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:34:05.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:34:05.931 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.61e+01, 1.41e+01 / 1.70e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 0.29%, AggregateStats -- 9.61e-02 / 1.01e-01, 1.08e-02 / 4.50e-03 | 20260508 02:34:05.931 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.47e+00, 5.78e-01 / 1.56e-01 | 20260508 02:34:05.931 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.47e+00, 2.05e-01 / 8.87e-02 | 20260508 02:34:05.931 | INFO | PerfReport: 16.10%, DME -- 4.20e+01 / 4.62e+01, 4.14e+00 / 2.08e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.61e+00 / 1.23e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.41e+00 / 2.70e+00, 4.22e-01 / 2.33e-01 | 20260508 02:34:05.931 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.24e+00, 2.67e-01 / 1.38e-01 | 20260508 02:34:05.931 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.63e-01, 5.17e-02 / 2.76e-02 | 20260508 02:34:05.931 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:34:05.931 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:34:05.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:34:05.931 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 24.58%, Upload -- 2.63e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 24.33%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 30.50%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 2.49%, LockUpload -- 2.66e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 2.61%, LockDownload -- 2.79e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 15.48%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:34:05.931 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260508 02:34:06.040 | INFO | Starting a round of Baz IO | 20260508 02:34:09.488 | INFO | Populating RTMetrics report | 20260508 02:34:09.488 | INFO | RTMetrics report populated | 20260508 02:34:09.488 | INFO | Opening tmp RTMetrics json file | 20260508 02:34:09.516 | INFO | Serializing RTMetrics to json | 20260508 02:34:09.524 | INFO | Serialization done, swapping tmp file now... | 20260508 02:34:09.564 | INFO | RTMetrics done | 20260508 02:34:09.592 | INFO | Finished a round of Baz IO: Wrote 1.637 GiB in 3.551 seconds | 20260508 02:34:09.592 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.51% / 0.00% / 58.49% | 20260508 02:34:09.592 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.710769% / 0% / 4.76837e-05% | 20260508 02:34:09.592 | INFO | Baz IO overall: | 20260508 02:34:09.592 | INFO | BytesWritten : 2055565188413 | 20260508 02:34:09.592 | INFO | NumEvents : 993649740171 | 20260508 02:34:11.070 | INFO | Analyzing chunk frames = [4554752,4555264) | 20260508 02:34:11.070 | INFO | MemInfo: { | 20260508 02:34:11.070 | INFO | "gpu" : " 17 GB", | 20260508 02:34:11.070 | INFO | "huge" : "", | 20260508 02:34:11.070 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:11.070 | INFO | "malloc" : " 14 GB", | 20260508 02:34:11.070 | INFO | "pinned" : " 29 GB" | 20260508 02:34:11.070 | INFO | } | 20260508 02:34:11.070 | INFO | Perf report for the last 5.14087 seconds: | 20260508 02:34:11.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:11.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:11.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:11.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:11.070 | INFO | ANALYSIS : 68.6%, 2.9, 104.7ms, 0 | 20260508 02:34:11.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:11.070 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 02:34:11.070 | INFO | PRE_HQ : 67.3%, 1.7, 60.9ms, 0 | 20260508 02:34:11.070 | INFO | BAZWRITER : 68.7%, 3.7, 129.1ms, 0 | 20260508 02:34:11.070 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:34:11.071 | INFO | Uploaded 36GiB to GPU in 2.5785s (13.9616GiB/s) | 20260508 02:34:11.071 | INFO | Downloaded 38GiB to GPU in 2.70851s (14.0299GiB/s) | 20260508 02:34:16.204 | INFO | Analyzing chunk frames = [4555264,4555776) | 20260508 02:34:16.204 | INFO | MemInfo: { | 20260508 02:34:16.204 | INFO | "gpu" : " 17 GB", | 20260508 02:34:16.204 | INFO | "huge" : "", | 20260508 02:34:16.204 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:16.204 | INFO | "malloc" : " 14 GB", | 20260508 02:34:16.205 | INFO | "pinned" : " 29 GB" | 20260508 02:34:16.205 | INFO | } | 20260508 02:34:16.205 | INFO | Perf report for the last 5.13442 seconds: | 20260508 02:34:16.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:16.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:34:16.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:34:16.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:16.205 | INFO | Uploaded 36GiB to GPU in 2.52947s (14.2322GiB/s) | 20260508 02:34:16.205 | INFO | Downloaded 34GiB to GPU in 2.43991s (13.9349GiB/s) | 20260508 02:34:21.338 | INFO | Analyzing chunk frames = [4555776,4556288) | 20260508 02:34:21.338 | INFO | MemInfo: { | 20260508 02:34:21.338 | INFO | "gpu" : " 17 GB", | 20260508 02:34:21.338 | INFO | "huge" : "", | 20260508 02:34:21.338 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:21.338 | INFO | "malloc" : " 14 GB", | 20260508 02:34:21.338 | INFO | "pinned" : " 29 GB" | 20260508 02:34:21.338 | INFO | } | 20260508 02:34:21.338 | INFO | Perf report for the last 5.13313 seconds: | 20260508 02:34:21.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:21.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:34:21.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:34:21.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:21.338 | INFO | Uploaded 36GiB to GPU in 2.50945s (14.3458GiB/s) | 20260508 02:34:21.338 | INFO | Downloaded 34GiB to GPU in 2.51443s (13.5219GiB/s) | 20260508 02:34:26.484 | INFO | Analyzing chunk frames = [4556288,4556800) | 20260508 02:34:26.484 | INFO | MemInfo: { | 20260508 02:34:26.484 | INFO | "gpu" : " 17 GB", | 20260508 02:34:26.484 | INFO | "huge" : "", | 20260508 02:34:26.484 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:26.484 | INFO | "malloc" : " 14 GB", | 20260508 02:34:26.484 | INFO | "pinned" : " 29 GB" | 20260508 02:34:26.484 | INFO | } | 20260508 02:34:26.484 | INFO | Perf report for the last 5.14633 seconds: | 20260508 02:34:26.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:26.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.484 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 02:34:26.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.484 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:34:26.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:26.485 | INFO | Uploaded 36GiB to GPU in 2.47121s (14.5677GiB/s) | 20260508 02:34:26.485 | INFO | Downloaded 34GiB to GPU in 2.49944s (13.603GiB/s) | 20260508 02:34:31.625 | INFO | Analyzing chunk frames = [4556800,4557312) | 20260508 02:34:31.625 | INFO | MemInfo: { | 20260508 02:34:31.625 | INFO | "gpu" : " 17 GB", | 20260508 02:34:31.625 | INFO | "huge" : "", | 20260508 02:34:31.625 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:31.625 | INFO | "malloc" : " 14 GB", | 20260508 02:34:31.625 | INFO | "pinned" : " 29 GB" | 20260508 02:34:31.625 | INFO | } | 20260508 02:34:31.626 | INFO | Perf report for the last 5.14137 seconds: | 20260508 02:34:31.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:31.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 02:34:31.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 02:34:31.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:31.626 | INFO | Uploaded 36GiB to GPU in 2.41188s (14.9261GiB/s) | 20260508 02:34:31.626 | INFO | Downloaded 34GiB to GPU in 2.58648s (13.1453GiB/s) | 20260508 02:34:36.765 | INFO | Analyzing chunk frames = [4557312,4557824) | 20260508 02:34:36.765 | INFO | MemInfo: { | 20260508 02:34:36.765 | INFO | "gpu" : " 17 GB", | 20260508 02:34:36.765 | INFO | "huge" : "", | 20260508 02:34:36.765 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:36.765 | INFO | "malloc" : " 14 GB", | 20260508 02:34:36.765 | INFO | "pinned" : " 29 GB" | 20260508 02:34:36.765 | INFO | } | 20260508 02:34:36.765 | INFO | Perf report for the last 5.13994 seconds: | 20260508 02:34:36.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:36.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.766 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:34:36.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.766 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:34:36.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:36.766 | INFO | Uploaded 36GiB to GPU in 2.39398s (15.0377GiB/s) | 20260508 02:34:36.766 | INFO | Downloaded 34GiB to GPU in 2.41097s (14.1022GiB/s) | 20260508 02:34:41.903 | INFO | Analyzing chunk frames = [4557824,4558336) | 20260508 02:34:41.903 | INFO | MemInfo: { | 20260508 02:34:41.903 | INFO | "gpu" : " 17 GB", | 20260508 02:34:41.903 | INFO | "huge" : "", | 20260508 02:34:41.903 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:41.904 | INFO | "malloc" : " 14 GB", | 20260508 02:34:41.904 | INFO | "pinned" : " 29 GB" | 20260508 02:34:41.904 | INFO | } | 20260508 02:34:41.904 | INFO | Perf report for the last 5.13836 seconds: | 20260508 02:34:41.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:41.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:34:41.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:34:41.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:41.904 | INFO | Uploaded 36GiB to GPU in 2.4307s (14.8106GiB/s) | 20260508 02:34:41.904 | INFO | Downloaded 34GiB to GPU in 2.42922s (13.9963GiB/s) | 20260508 02:34:47.043 | INFO | Analyzing chunk frames = [4558336,4558848) | 20260508 02:34:47.043 | INFO | MemInfo: { | 20260508 02:34:47.043 | INFO | "gpu" : " 17 GB", | 20260508 02:34:47.043 | INFO | "huge" : "", | 20260508 02:34:47.043 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:47.043 | INFO | "malloc" : " 14 GB", | 20260508 02:34:47.043 | INFO | "pinned" : " 29 GB" | 20260508 02:34:47.043 | INFO | } | 20260508 02:34:47.043 | INFO | Perf report for the last 5.13923 seconds: | 20260508 02:34:47.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:47.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.043 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:34:47.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.043 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 02:34:47.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:47.044 | INFO | Uploaded 36GiB to GPU in 2.4476s (14.7083GiB/s) | 20260508 02:34:47.044 | INFO | Downloaded 34GiB to GPU in 2.43491s (13.9636GiB/s) | 20260508 02:34:47.156 | INFO | Starting a round of Baz IO | 20260508 02:34:50.602 | INFO | Populating RTMetrics report | 20260508 02:34:50.602 | INFO | RTMetrics report populated | 20260508 02:34:50.602 | INFO | Opening tmp RTMetrics json file | 20260508 02:34:50.626 | INFO | Serializing RTMetrics to json | 20260508 02:34:50.651 | INFO | Serialization done, swapping tmp file now... | 20260508 02:34:50.705 | INFO | RTMetrics done | 20260508 02:34:50.716 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.559 seconds | 20260508 02:34:50.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.50% / 0.00% / 58.49% | 20260508 02:34:50.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.706462% / 0% / 3.57628e-05% | 20260508 02:34:50.716 | INFO | Baz IO overall: | 20260508 02:34:50.716 | INFO | BytesWritten : 2057319942219 | 20260508 02:34:50.716 | INFO | NumEvents : 994490323309 | 20260508 02:34:52.182 | INFO | Analyzing chunk frames = [4558848,4559360) | 20260508 02:34:52.183 | INFO | MemInfo: { | 20260508 02:34:52.183 | INFO | "gpu" : " 17 GB", | 20260508 02:34:52.183 | INFO | "huge" : "", | 20260508 02:34:52.183 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:52.183 | INFO | "malloc" : " 14 GB", | 20260508 02:34:52.183 | INFO | "pinned" : " 29 GB" | 20260508 02:34:52.183 | INFO | } | 20260508 02:34:52.183 | INFO | Perf report for the last 5.1397 seconds: | 20260508 02:34:52.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:52.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:52.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:52.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:52.183 | INFO | ANALYSIS : 68.6%, 2.9, 104.4ms, 0 | 20260508 02:34:52.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:52.183 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260508 02:34:52.183 | INFO | PRE_HQ : 67.0%, 1.8, 62.1ms, 0 | 20260508 02:34:52.183 | INFO | BAZWRITER : 68.9%, 3.7, 131.9ms, 0 | 20260508 02:34:52.183 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 02:34:52.183 | INFO | Uploaded 36GiB to GPU in 2.54749s (14.1316GiB/s) | 20260508 02:34:52.183 | INFO | Downloaded 38GiB to GPU in 2.69949s (14.0767GiB/s) | 20260508 02:34:57.325 | INFO | Analyzing chunk frames = [4559360,4559872) | 20260508 02:34:57.325 | INFO | MemInfo: { | 20260508 02:34:57.325 | INFO | "gpu" : " 17 GB", | 20260508 02:34:57.325 | INFO | "huge" : "", | 20260508 02:34:57.325 | INFO | "huge_shared" : " 29 GB", | 20260508 02:34:57.325 | INFO | "malloc" : " 14 GB", | 20260508 02:34:57.325 | INFO | "pinned" : " 29 GB" | 20260508 02:34:57.325 | INFO | } | 20260508 02:34:57.325 | INFO | Perf report for the last 5.14241 seconds: | 20260508 02:34:57.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:34:57.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:34:57.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:34:57.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:34:57.325 | INFO | Uploaded 36GiB to GPU in 2.54769s (14.1304GiB/s) | 20260508 02:34:57.326 | INFO | Downloaded 34GiB to GPU in 2.43078s (13.9873GiB/s) | 20260508 02:35:02.458 | INFO | Analyzing chunk frames = [4559872,4560384) | 20260508 02:35:02.458 | INFO | MemInfo: { | 20260508 02:35:02.458 | INFO | "gpu" : " 17 GB", | 20260508 02:35:02.458 | INFO | "huge" : "", | 20260508 02:35:02.458 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:02.458 | INFO | "malloc" : " 14 GB", | 20260508 02:35:02.459 | INFO | "pinned" : " 29 GB" | 20260508 02:35:02.459 | INFO | } | 20260508 02:35:02.459 | INFO | Perf report for the last 5.13352 seconds: | 20260508 02:35:02.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:02.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.459 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:35:02.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.459 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:35:02.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:02.460 | INFO | Uploaded 36GiB to GPU in 2.53317s (14.2115GiB/s) | 20260508 02:35:02.460 | INFO | Downloaded 34GiB to GPU in 2.49884s (13.6063GiB/s) | 20260508 02:35:07.594 | INFO | Analyzing chunk frames = [4560384,4560896) | 20260508 02:35:07.594 | INFO | MemInfo: { | 20260508 02:35:07.594 | INFO | "gpu" : " 17 GB", | 20260508 02:35:07.594 | INFO | "huge" : "", | 20260508 02:35:07.594 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:07.594 | INFO | "malloc" : " 14 GB", | 20260508 02:35:07.595 | INFO | "pinned" : " 29 GB" | 20260508 02:35:07.595 | INFO | } | 20260508 02:35:07.595 | INFO | Perf report for the last 5.13592 seconds: | 20260508 02:35:07.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:07.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 02:35:07.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:35:07.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:07.595 | INFO | Uploaded 36GiB to GPU in 2.54076s (14.169GiB/s) | 20260508 02:35:07.595 | INFO | Downloaded 34GiB to GPU in 2.50685s (13.5628GiB/s) | 20260508 02:35:12.738 | INFO | Analyzing chunk frames = [4560896,4561408) | 20260508 02:35:12.738 | INFO | MemInfo: { | 20260508 02:35:12.738 | INFO | "gpu" : " 17 GB", | 20260508 02:35:12.738 | INFO | "huge" : "", | 20260508 02:35:12.738 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:12.738 | INFO | "malloc" : " 14 GB", | 20260508 02:35:12.738 | INFO | "pinned" : " 29 GB" | 20260508 02:35:12.738 | INFO | } | 20260508 02:35:12.738 | INFO | Perf report for the last 5.14355 seconds: | 20260508 02:35:12.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:12.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 02:35:12.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:35:12.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:12.738 | INFO | Uploaded 36GiB to GPU in 2.30084s (15.6464GiB/s) | 20260508 02:35:12.738 | INFO | Downloaded 34GiB to GPU in 2.51202s (13.5349GiB/s) | 20260508 02:35:17.879 | INFO | Analyzing chunk frames = [4561408,4561920) | 20260508 02:35:17.879 | INFO | MemInfo: { | 20260508 02:35:17.879 | INFO | "gpu" : " 17 GB", | 20260508 02:35:17.879 | INFO | "huge" : "", | 20260508 02:35:17.879 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:17.879 | INFO | "malloc" : " 14 GB", | 20260508 02:35:17.879 | INFO | "pinned" : " 29 GB" | 20260508 02:35:17.879 | INFO | } | 20260508 02:35:17.879 | INFO | Perf report for the last 5.14064 seconds: | 20260508 02:35:17.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:17.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:35:17.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:35:17.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:17.879 | INFO | Uploaded 36GiB to GPU in 2.40962s (14.9401GiB/s) | 20260508 02:35:17.879 | INFO | Downloaded 34GiB to GPU in 2.42684s (14.01GiB/s) | 20260508 02:35:23.022 | INFO | Analyzing chunk frames = [4561920,4562432) | 20260508 02:35:23.023 | INFO | MemInfo: { | 20260508 02:35:23.023 | INFO | "gpu" : " 17 GB", | 20260508 02:35:23.023 | INFO | "huge" : "", | 20260508 02:35:23.023 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:23.023 | INFO | "malloc" : " 14 GB", | 20260508 02:35:23.023 | INFO | "pinned" : " 29 GB" | 20260508 02:35:23.023 | INFO | } | 20260508 02:35:23.023 | INFO | Perf report for the last 5.14402 seconds: | 20260508 02:35:23.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:23.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:35:23.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:35:23.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:23.023 | INFO | Uploaded 36GiB to GPU in 2.43469s (14.7863GiB/s) | 20260508 02:35:23.023 | INFO | Downloaded 34GiB to GPU in 2.42426s (14.0249GiB/s) | 20260508 02:35:28.155 | INFO | Analyzing chunk frames = [4562432,4562944) | 20260508 02:35:28.155 | INFO | MemInfo: { | 20260508 02:35:28.155 | INFO | "gpu" : " 17 GB", | 20260508 02:35:28.155 | INFO | "huge" : "", | 20260508 02:35:28.155 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:28.155 | INFO | "malloc" : " 14 GB", | 20260508 02:35:28.155 | INFO | "pinned" : " 29 GB" | 20260508 02:35:28.155 | INFO | } | 20260508 02:35:28.155 | INFO | Perf report for the last 5.132 seconds: | 20260508 02:35:28.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:28.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 02:35:28.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | PRE_HQ : 34.6%, 1.0, 17.9ms, 0 | 20260508 02:35:28.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:28.155 | INFO | Uploaded 36GiB to GPU in 2.44353s (14.7328GiB/s) | 20260508 02:35:28.155 | INFO | Downloaded 34GiB to GPU in 2.4282s (14.0021GiB/s) | 20260508 02:35:28.155 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:35:28.155 | INFO | Total mem usage: 16.7 GiB | 20260508 02:35:28.155 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:35:28.155 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 02:35:28.155 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:35:28.155 | INFO | High Water Breakdown: | 20260508 02:35:28.155 | INFO | 59.8% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:35:28.155 | INFO | 40.1% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:35:28.155 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:35:28.155 | INFO | ------------------------------------------ | 20260508 02:35:28.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:35:28.155 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:35:28.155 | INFO | Total mem usage: 26.5 GiB | 20260508 02:35:28.155 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:35:28.155 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:35:28.155 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:35:28.155 | INFO | High Water Breakdown: | 20260508 02:35:28.155 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:35:28.155 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:35:28.155 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:35:28.155 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:35:28.155 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:35:28.155 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:35:28.155 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:35:28.155 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:35:28.155 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:35:28.155 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:35:28.155 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:35:28.155 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:35:28.155 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:35:28.155 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:35:28.155 | INFO | ------------------------------------------ | 20260508 02:35:28.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:35:28.155 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:35:28.155 | INFO | Total mem usage: 27.8 GiB | 20260508 02:35:28.155 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:35:28.155 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:35:28.155 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:35:28.155 | INFO | High Water Breakdown: | 20260508 02:35:28.155 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:35:28.155 | INFO | ------------------------------------------ | 20260508 02:35:28.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:35:28.155 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:35:28.155 | INFO | Total mem usage: 15.7 GiB | 20260508 02:35:28.155 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:35:28.155 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:35:28.155 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:35:28.155 | INFO | High Water Breakdown: | 20260508 02:35:28.155 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:35:28.155 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:35:28.155 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:35:28.155 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:35:28.155 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:35:28.155 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:35:28.155 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:35:28.155 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:35:28.155 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:35:28.155 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:35:28.155 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:35:28.155 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:35:28.155 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:35:28.155 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:35:28.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:35:28.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:35:28.155 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:35:28.155 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:35:28.155 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:35:28.155 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:35:28.155 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:35:28.155 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:35:28.156 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:35:28.156 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:35:28.156 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:35:28.156 | INFO | ------------------------------------------ | 20260508 02:35:28.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:35:28.156 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:35:28.157 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:35:28.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:35:28.157 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 1.61e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.02e-01, 1.21e-02 / 4.82e-03 | 20260508 02:35:28.157 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.43e+00, 5.84e-01 / 2.10e-01 | 20260508 02:35:28.157 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.49e+00, 2.07e-01 / 9.66e-02 | 20260508 02:35:28.157 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.62e+01, 4.04e+00 / 1.95e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.67e+00 / 1.22e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.42e+00 / 2.68e+00, 4.22e-01 / 2.30e-01 | 20260508 02:35:28.157 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.23e+00, 2.65e-01 / 1.39e-01 | 20260508 02:35:28.157 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.29e-01 / 3.67e-01, 5.19e-02 / 2.97e-02 | 20260508 02:35:28.157 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:35:28.157 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:35:28.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:35:28.157 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 24.26%, Upload -- 2.57e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 24.52%, Download -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 2.03%, LockUpload -- 2.15e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 2.88%, LockDownload -- 3.05e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 15.44%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 02:35:28.157 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.04e-02 / 0.00e+00 | 20260508 02:35:28.252 | INFO | Starting a round of Baz IO | 20260508 02:35:31.685 | INFO | Populating RTMetrics report | 20260508 02:35:31.686 | INFO | RTMetrics report populated | 20260508 02:35:31.686 | INFO | Opening tmp RTMetrics json file | 20260508 02:35:31.715 | INFO | Serializing RTMetrics to json | 20260508 02:35:31.744 | INFO | Serialization done, swapping tmp file now... | 20260508 02:35:31.764 | INFO | RTMetrics done | 20260508 02:35:31.787 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.534 seconds | 20260508 02:35:31.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.50% / 0.00% / 58.50% | 20260508 02:35:31.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.706685% / 3.97364e-06% / 6.35783e-05% | 20260508 02:35:31.787 | INFO | Baz IO overall: | 20260508 02:35:31.787 | INFO | BytesWritten : 2059074909269 | 20260508 02:35:31.787 | INFO | NumEvents : 995331362666 | 20260508 02:35:33.292 | INFO | Analyzing chunk frames = [4562944,4563456) | 20260508 02:35:33.292 | INFO | MemInfo: { | 20260508 02:35:33.292 | INFO | "gpu" : " 17 GB", | 20260508 02:35:33.292 | INFO | "huge" : "", | 20260508 02:35:33.292 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:33.292 | INFO | "malloc" : " 14 GB", | 20260508 02:35:33.292 | INFO | "pinned" : " 29 GB" | 20260508 02:35:33.292 | INFO | } | 20260508 02:35:33.292 | INFO | Perf report for the last 5.13738 seconds: | 20260508 02:35:33.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:33.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:33.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:33.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:33.293 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260508 02:35:33.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:33.293 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260508 02:35:33.293 | INFO | PRE_HQ : 65.6%, 1.6, 56.1ms, 0 | 20260508 02:35:33.293 | INFO | BAZWRITER : 68.4%, 3.9, 135.8ms, 0 | 20260508 02:35:33.293 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:35:33.293 | INFO | Uploaded 36GiB to GPU in 2.57529s (13.979GiB/s) | 20260508 02:35:33.293 | INFO | Downloaded 38GiB to GPU in 2.70524s (14.0468GiB/s) | 20260508 02:35:38.433 | INFO | Analyzing chunk frames = [4563456,4563968) | 20260508 02:35:38.433 | INFO | MemInfo: { | 20260508 02:35:38.433 | INFO | "gpu" : " 17 GB", | 20260508 02:35:38.433 | INFO | "huge" : "", | 20260508 02:35:38.433 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:38.433 | INFO | "malloc" : " 14 GB", | 20260508 02:35:38.433 | INFO | "pinned" : " 29 GB" | 20260508 02:35:38.433 | INFO | } | 20260508 02:35:38.433 | INFO | Perf report for the last 5.14092 seconds: | 20260508 02:35:38.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:38.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.433 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:35:38.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.433 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:35:38.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:38.434 | INFO | Uploaded 36GiB to GPU in 2.5621s (14.051GiB/s) | 20260508 02:35:38.434 | INFO | Downloaded 34GiB to GPU in 2.42184s (14.0389GiB/s) | 20260508 02:35:43.573 | INFO | Analyzing chunk frames = [4563968,4564480) | 20260508 02:35:43.573 | INFO | MemInfo: { | 20260508 02:35:43.573 | INFO | "gpu" : " 17 GB", | 20260508 02:35:43.573 | INFO | "huge" : "", | 20260508 02:35:43.573 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:43.573 | INFO | "malloc" : " 14 GB", | 20260508 02:35:43.573 | INFO | "pinned" : " 29 GB" | 20260508 02:35:43.573 | INFO | } | 20260508 02:35:43.573 | INFO | Perf report for the last 5.14015 seconds: | 20260508 02:35:43.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:43.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.573 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:35:43.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.574 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 02:35:43.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:43.574 | INFO | Uploaded 36GiB to GPU in 2.56085s (14.0578GiB/s) | 20260508 02:35:43.574 | INFO | Downloaded 34GiB to GPU in 2.48832s (13.6638GiB/s) | 20260508 02:35:48.718 | INFO | Analyzing chunk frames = [4564480,4564992) | 20260508 02:35:48.718 | INFO | MemInfo: { | 20260508 02:35:48.718 | INFO | "gpu" : " 17 GB", | 20260508 02:35:48.718 | INFO | "huge" : "", | 20260508 02:35:48.718 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:48.718 | INFO | "malloc" : " 14 GB", | 20260508 02:35:48.718 | INFO | "pinned" : " 29 GB" | 20260508 02:35:48.718 | INFO | } | 20260508 02:35:48.718 | INFO | Perf report for the last 5.14505 seconds: | 20260508 02:35:48.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:48.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.718 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:35:48.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.719 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:35:48.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:48.719 | INFO | Uploaded 36GiB to GPU in 2.55806s (14.0731GiB/s) | 20260508 02:35:48.719 | INFO | Downloaded 34GiB to GPU in 2.48808s (13.6652GiB/s) | 20260508 02:35:53.853 | INFO | Analyzing chunk frames = [4564992,4565504) | 20260508 02:35:53.853 | INFO | MemInfo: { | 20260508 02:35:53.853 | INFO | "gpu" : " 17 GB", | 20260508 02:35:53.853 | INFO | "huge" : "", | 20260508 02:35:53.853 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:53.853 | INFO | "malloc" : " 14 GB", | 20260508 02:35:53.853 | INFO | "pinned" : " 29 GB" | 20260508 02:35:53.853 | INFO | } | 20260508 02:35:53.853 | INFO | Perf report for the last 5.13496 seconds: | 20260508 02:35:53.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:53.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.853 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 02:35:53.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.854 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:35:53.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:53.854 | INFO | Uploaded 36GiB to GPU in 2.43666s (14.7743GiB/s) | 20260508 02:35:53.854 | INFO | Downloaded 34GiB to GPU in 2.56017s (13.2804GiB/s) | 20260508 02:35:58.993 | INFO | Analyzing chunk frames = [4565504,4566016) | 20260508 02:35:58.993 | INFO | MemInfo: { | 20260508 02:35:58.993 | INFO | "gpu" : " 17 GB", | 20260508 02:35:58.993 | INFO | "huge" : "", | 20260508 02:35:58.993 | INFO | "huge_shared" : " 29 GB", | 20260508 02:35:58.993 | INFO | "malloc" : " 14 GB", | 20260508 02:35:58.993 | INFO | "pinned" : " 29 GB" | 20260508 02:35:58.993 | INFO | } | 20260508 02:35:58.993 | INFO | Perf report for the last 5.1397 seconds: | 20260508 02:35:58.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:35:58.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:35:58.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:35:58.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:35:58.993 | INFO | Uploaded 36GiB to GPU in 2.44299s (14.736GiB/s) | 20260508 02:35:58.993 | INFO | Downloaded 34GiB to GPU in 2.42294s (14.0325GiB/s) | 20260508 02:36:04.131 | INFO | Analyzing chunk frames = [4566016,4566528) | 20260508 02:36:04.131 | INFO | MemInfo: { | 20260508 02:36:04.131 | INFO | "gpu" : " 17 GB", | 20260508 02:36:04.131 | INFO | "huge" : "", | 20260508 02:36:04.131 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:04.131 | INFO | "malloc" : " 14 GB", | 20260508 02:36:04.131 | INFO | "pinned" : " 29 GB" | 20260508 02:36:04.131 | INFO | } | 20260508 02:36:04.131 | INFO | Perf report for the last 5.13799 seconds: | 20260508 02:36:04.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:04.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.131 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:36:04.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.131 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 02:36:04.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:04.132 | INFO | Uploaded 36GiB to GPU in 2.49209s (14.4457GiB/s) | 20260508 02:36:04.132 | INFO | Downloaded 34GiB to GPU in 2.43002s (13.9916GiB/s) | 20260508 02:36:09.269 | INFO | Analyzing chunk frames = [4566528,4567040) | 20260508 02:36:09.269 | INFO | MemInfo: { | 20260508 02:36:09.269 | INFO | "gpu" : " 17 GB", | 20260508 02:36:09.269 | INFO | "huge" : "", | 20260508 02:36:09.269 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:09.269 | INFO | "malloc" : " 14 GB", | 20260508 02:36:09.269 | INFO | "pinned" : " 29 GB" | 20260508 02:36:09.269 | INFO | } | 20260508 02:36:09.269 | INFO | Perf report for the last 5.13823 seconds: | 20260508 02:36:09.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:09.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.269 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:36:09.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.270 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:36:09.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:09.270 | INFO | Uploaded 36GiB to GPU in 2.56243s (14.0492GiB/s) | 20260508 02:36:09.270 | INFO | Downloaded 34GiB to GPU in 2.45014s (13.8768GiB/s) | 20260508 02:36:09.367 | INFO | Starting a round of Baz IO | 20260508 02:36:12.818 | INFO | Populating RTMetrics report | 20260508 02:36:12.818 | INFO | RTMetrics report populated | 20260508 02:36:12.818 | INFO | Opening tmp RTMetrics json file | 20260508 02:36:12.859 | INFO | Serializing RTMetrics to json | 20260508 02:36:12.861 | INFO | Serialization done, swapping tmp file now... | 20260508 02:36:12.861 | INFO | RTMetrics done | 20260508 02:36:13.017 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.649 seconds | 20260508 02:36:13.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.50% / 0.00% / 58.50% | 20260508 02:36:13.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.703144% / 7.94729e-06% / 5.16574e-05% | 20260508 02:36:13.017 | INFO | Baz IO overall: | 20260508 02:36:13.017 | INFO | BytesWritten : 2060829080287 | 20260508 02:36:13.017 | INFO | NumEvents : 996171841616 | 20260508 02:36:14.404 | INFO | Analyzing chunk frames = [4567040,4567552) | 20260508 02:36:14.404 | INFO | MemInfo: { | 20260508 02:36:14.404 | INFO | "gpu" : " 17 GB", | 20260508 02:36:14.404 | INFO | "huge" : "", | 20260508 02:36:14.404 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:14.404 | INFO | "malloc" : " 14 GB", | 20260508 02:36:14.404 | INFO | "pinned" : " 29 GB" | 20260508 02:36:14.404 | INFO | } | 20260508 02:36:14.405 | INFO | Perf report for the last 5.13509 seconds: | 20260508 02:36:14.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:14.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:14.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:14.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:14.405 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:36:14.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:14.405 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260508 02:36:14.406 | INFO | PRE_HQ : 65.5%, 1.7, 56.1ms, 0 | 20260508 02:36:14.406 | INFO | BAZWRITER : 70.6%, 4.2, 152.0ms, 0 | 20260508 02:36:14.406 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 02:36:14.406 | INFO | Uploaded 36GiB to GPU in 2.57773s (13.9658GiB/s) | 20260508 02:36:14.406 | INFO | Downloaded 38GiB to GPU in 2.69695s (14.09GiB/s) | 20260508 02:36:19.547 | INFO | Analyzing chunk frames = [4567552,4568064) | 20260508 02:36:19.547 | INFO | MemInfo: { | 20260508 02:36:19.547 | INFO | "gpu" : " 17 GB", | 20260508 02:36:19.547 | INFO | "huge" : "", | 20260508 02:36:19.547 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:19.547 | INFO | "malloc" : " 14 GB", | 20260508 02:36:19.547 | INFO | "pinned" : " 29 GB" | 20260508 02:36:19.547 | INFO | } | 20260508 02:36:19.547 | INFO | Perf report for the last 5.1428 seconds: | 20260508 02:36:19.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:19.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.547 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:36:19.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.547 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:36:19.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:19.548 | INFO | Uploaded 36GiB to GPU in 2.56612s (14.0289GiB/s) | 20260508 02:36:19.548 | INFO | Downloaded 34GiB to GPU in 2.41197s (14.0963GiB/s) | 20260508 02:36:24.693 | INFO | Analyzing chunk frames = [4568064,4568576) | 20260508 02:36:24.693 | INFO | MemInfo: { | 20260508 02:36:24.693 | INFO | "gpu" : " 17 GB", | 20260508 02:36:24.693 | INFO | "huge" : "", | 20260508 02:36:24.693 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:24.693 | INFO | "malloc" : " 14 GB", | 20260508 02:36:24.693 | INFO | "pinned" : " 29 GB" | 20260508 02:36:24.693 | INFO | } | 20260508 02:36:24.693 | INFO | Perf report for the last 5.14549 seconds: | 20260508 02:36:24.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:24.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 02:36:24.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:36:24.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:24.693 | INFO | Uploaded 36GiB to GPU in 2.55581s (14.0856GiB/s) | 20260508 02:36:24.693 | INFO | Downloaded 34GiB to GPU in 2.48185s (13.6994GiB/s) | 20260508 02:36:29.835 | INFO | Analyzing chunk frames = [4568576,4569088) | 20260508 02:36:29.835 | INFO | MemInfo: { | 20260508 02:36:29.835 | INFO | "gpu" : " 17 GB", | 20260508 02:36:29.835 | INFO | "huge" : "", | 20260508 02:36:29.835 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:29.835 | INFO | "malloc" : " 14 GB", | 20260508 02:36:29.835 | INFO | "pinned" : " 29 GB" | 20260508 02:36:29.835 | INFO | } | 20260508 02:36:29.835 | INFO | Perf report for the last 5.14247 seconds: | 20260508 02:36:29.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:29.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.835 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 02:36:29.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.835 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:36:29.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:29.836 | INFO | Uploaded 36GiB to GPU in 2.5788s (13.96GiB/s) | 20260508 02:36:29.836 | INFO | Downloaded 34GiB to GPU in 2.49016s (13.6537GiB/s) | 20260508 02:36:34.968 | INFO | Analyzing chunk frames = [4569088,4569600) | 20260508 02:36:34.968 | INFO | MemInfo: { | 20260508 02:36:34.968 | INFO | "gpu" : " 17 GB", | 20260508 02:36:34.968 | INFO | "huge" : "", | 20260508 02:36:34.968 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:34.968 | INFO | "malloc" : " 14 GB", | 20260508 02:36:34.968 | INFO | "pinned" : " 29 GB" | 20260508 02:36:34.968 | INFO | } | 20260508 02:36:34.968 | INFO | Perf report for the last 5.13298 seconds: | 20260508 02:36:34.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:34.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.968 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 02:36:34.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.969 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:36:34.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:34.969 | INFO | Uploaded 36GiB to GPU in 2.43762s (14.7685GiB/s) | 20260508 02:36:34.969 | INFO | Downloaded 34GiB to GPU in 2.53314s (13.4221GiB/s) | 20260508 02:36:40.104 | INFO | Analyzing chunk frames = [4569600,4570112) | 20260508 02:36:40.104 | INFO | MemInfo: { | 20260508 02:36:40.104 | INFO | "gpu" : " 17 GB", | 20260508 02:36:40.104 | INFO | "huge" : "", | 20260508 02:36:40.104 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:40.104 | INFO | "malloc" : " 14 GB", | 20260508 02:36:40.104 | INFO | "pinned" : " 29 GB" | 20260508 02:36:40.104 | INFO | } | 20260508 02:36:40.104 | INFO | Perf report for the last 5.13607 seconds: | 20260508 02:36:40.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:40.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.104 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 02:36:40.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.105 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:36:40.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:40.105 | INFO | Uploaded 36GiB to GPU in 2.45744s (14.6494GiB/s) | 20260508 02:36:40.105 | INFO | Downloaded 34GiB to GPU in 2.40833s (14.1177GiB/s) | 20260508 02:36:45.247 | INFO | Analyzing chunk frames = [4570112,4570624) | 20260508 02:36:45.247 | INFO | MemInfo: { | 20260508 02:36:45.247 | INFO | "gpu" : " 17 GB", | 20260508 02:36:45.247 | INFO | "huge" : "", | 20260508 02:36:45.247 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:45.247 | INFO | "malloc" : " 14 GB", | 20260508 02:36:45.247 | INFO | "pinned" : " 29 GB" | 20260508 02:36:45.247 | INFO | } | 20260508 02:36:45.247 | INFO | Perf report for the last 5.1424 seconds: | 20260508 02:36:45.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:45.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:36:45.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:36:45.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:45.247 | INFO | Uploaded 36GiB to GPU in 2.45273s (14.6775GiB/s) | 20260508 02:36:45.248 | INFO | Downloaded 34GiB to GPU in 2.41097s (14.1022GiB/s) | 20260508 02:36:50.387 | INFO | Analyzing chunk frames = [4570624,4571136) | 20260508 02:36:50.387 | INFO | MemInfo: { | 20260508 02:36:50.387 | INFO | "gpu" : " 17 GB", | 20260508 02:36:50.387 | INFO | "huge" : "", | 20260508 02:36:50.387 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:50.387 | INFO | "malloc" : " 14 GB", | 20260508 02:36:50.387 | INFO | "pinned" : " 29 GB" | 20260508 02:36:50.387 | INFO | } | 20260508 02:36:50.387 | INFO | Perf report for the last 5.14041 seconds: | 20260508 02:36:50.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:50.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.387 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 02:36:50.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.387 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:36:50.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:50.388 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:36:50.388 | INFO | Total mem usage: 16.7 GiB | 20260508 02:36:50.388 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:36:50.388 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.9%) | 20260508 02:36:50.388 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:36:50.388 | INFO | High Water Breakdown: | 20260508 02:36:50.388 | INFO | 59.9% : 8.86 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:36:50.388 | INFO | 39.9% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:36:50.388 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:36:50.388 | INFO | ------------------------------------------ | 20260508 02:36:50.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:36:50.388 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:36:50.388 | INFO | Total mem usage: 26.5 GiB | 20260508 02:36:50.388 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:36:50.388 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:36:50.388 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:36:50.388 | INFO | High Water Breakdown: | 20260508 02:36:50.388 | INFO | Uploaded 36GiB to GPU in 2.48598s (14.4812GiB/s) | 20260508 02:36:50.388 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:36:50.388 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:36:50.388 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:36:50.388 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:36:50.388 | INFO | Downloaded 34GiB to GPU in 2.42102s (14.0437GiB/s) | 20260508 02:36:50.388 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:36:50.388 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:36:50.388 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:36:50.388 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:36:50.388 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:36:50.388 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:36:50.388 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:36:50.388 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:36:50.388 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:36:50.388 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:36:50.388 | INFO | ------------------------------------------ | 20260508 02:36:50.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:36:50.388 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:36:50.388 | INFO | Total mem usage: 27.8 GiB | 20260508 02:36:50.388 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:36:50.388 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:36:50.388 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:36:50.388 | INFO | High Water Breakdown: | 20260508 02:36:50.388 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:36:50.388 | INFO | ------------------------------------------ | 20260508 02:36:50.388 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:36:50.388 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:36:50.388 | INFO | Total mem usage: 15.7 GiB | 20260508 02:36:50.388 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:36:50.388 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:36:50.388 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:36:50.388 | INFO | High Water Breakdown: | 20260508 02:36:50.388 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:36:50.388 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:36:50.388 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:36:50.389 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:36:50.389 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:36:50.389 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:36:50.389 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:36:50.389 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:36:50.389 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:36:50.389 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:36:50.389 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:36:50.389 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:36:50.389 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:36:50.389 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:36:50.389 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:36:50.389 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:36:50.389 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:36:50.389 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:36:50.389 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:36:50.389 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:36:50.389 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:36:50.389 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:36:50.389 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:36:50.389 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:36:50.389 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:36:50.389 | INFO | ------------------------------------------ | 20260508 02:36:50.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:36:50.389 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:36:50.389 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:36:50.389 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:36:50.389 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 1.13e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.01e-01, 1.10e-02 / 4.74e-03 | 20260508 02:36:50.389 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.48e+00, 5.83e-01 / 1.57e-01 | 20260508 02:36:50.389 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.03e-01 / 9.79e-02 | 20260508 02:36:50.389 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.61e+01, 3.85e+00 / 1.80e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.65e+00 / 1.31e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.65e+00, 4.20e-01 / 2.22e-01 | 20260508 02:36:50.389 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.23e+00, 2.66e-01 / 1.41e-01 | 20260508 02:36:50.389 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.64e-01, 5.20e-02 / 2.41e-02 | 20260508 02:36:50.389 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:36:50.389 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:36:50.389 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:36:50.389 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 24.60%, Upload -- 2.62e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 30.63%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 2.46%, LockUpload -- 2.62e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 2.54%, LockDownload -- 2.71e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 15.46%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:36:50.389 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 3.19e-02 / 0.00e+00 | 20260508 02:36:50.485 | INFO | Starting a round of Baz IO | 20260508 02:36:53.938 | INFO | Populating RTMetrics report | 20260508 02:36:53.939 | INFO | RTMetrics report populated | 20260508 02:36:53.939 | INFO | Opening tmp RTMetrics json file | 20260508 02:36:54.026 | INFO | Serializing RTMetrics to json | 20260508 02:36:54.035 | INFO | Serialization done, swapping tmp file now... | 20260508 02:36:54.040 | INFO | RTMetrics done | 20260508 02:36:54.054 | INFO | Finished a round of Baz IO: Wrote 1.635 GiB in 3.569 seconds | 20260508 02:36:54.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.49% / 0.00% / 58.50% | 20260508 02:36:54.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.705445% / 0% / 6.75519e-05% | 20260508 02:36:54.054 | INFO | Baz IO overall: | 20260508 02:36:54.054 | INFO | BytesWritten : 2062585057734 | 20260508 02:36:54.054 | INFO | NumEvents : 997013631739 | 20260508 02:36:55.527 | INFO | Analyzing chunk frames = [4571136,4571648) | 20260508 02:36:55.527 | INFO | MemInfo: { | 20260508 02:36:55.527 | INFO | "gpu" : " 17 GB", | 20260508 02:36:55.527 | INFO | "huge" : "", | 20260508 02:36:55.527 | INFO | "huge_shared" : " 29 GB", | 20260508 02:36:55.527 | INFO | "malloc" : " 14 GB", | 20260508 02:36:55.527 | INFO | "pinned" : " 29 GB" | 20260508 02:36:55.527 | INFO | } | 20260508 02:36:55.527 | INFO | Perf report for the last 5.13954 seconds: | 20260508 02:36:55.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:36:55.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:55.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:55.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:55.527 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260508 02:36:55.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:36:55.527 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 02:36:55.527 | INFO | PRE_HQ : 65.8%, 1.6, 55.9ms, 0 | 20260508 02:36:55.527 | INFO | BAZWRITER : 69.0%, 4.1, 145.6ms, 0 | 20260508 02:36:55.527 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 02:36:55.527 | INFO | Uploaded 36GiB to GPU in 2.54645s (14.1374GiB/s) | 20260508 02:36:55.527 | INFO | Downloaded 38GiB to GPU in 2.70514s (14.0473GiB/s) | 20260508 02:37:00.663 | INFO | Analyzing chunk frames = [4571648,4572160) | 20260508 02:37:00.663 | INFO | MemInfo: { | 20260508 02:37:00.663 | INFO | "gpu" : " 17 GB", | 20260508 02:37:00.663 | INFO | "huge" : "", | 20260508 02:37:00.663 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:00.663 | INFO | "malloc" : " 14 GB", | 20260508 02:37:00.663 | INFO | "pinned" : " 29 GB" | 20260508 02:37:00.663 | INFO | } | 20260508 02:37:00.663 | INFO | Perf report for the last 5.13652 seconds: | 20260508 02:37:00.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:00.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.663 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:37:00.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.663 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:37:00.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:00.664 | INFO | Uploaded 36GiB to GPU in 2.55379s (14.0967GiB/s) | 20260508 02:37:00.664 | INFO | Downloaded 34GiB to GPU in 2.43241s (13.9779GiB/s) | 20260508 02:37:05.805 | INFO | Analyzing chunk frames = [4572160,4572672) | 20260508 02:37:05.805 | INFO | MemInfo: { | 20260508 02:37:05.805 | INFO | "gpu" : " 17 GB", | 20260508 02:37:05.805 | INFO | "huge" : "", | 20260508 02:37:05.805 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:05.805 | INFO | "malloc" : " 14 GB", | 20260508 02:37:05.805 | INFO | "pinned" : " 29 GB" | 20260508 02:37:05.805 | INFO | } | 20260508 02:37:05.805 | INFO | Perf report for the last 5.14188 seconds: | 20260508 02:37:05.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:05.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.805 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:37:05.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.805 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:37:05.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:05.806 | INFO | Uploaded 36GiB to GPU in 2.52889s (14.2355GiB/s) | 20260508 02:37:05.806 | INFO | Downloaded 34GiB to GPU in 2.50195s (13.5894GiB/s) | 20260508 02:37:10.946 | INFO | Analyzing chunk frames = [4572672,4573184) | 20260508 02:37:10.946 | INFO | MemInfo: { | 20260508 02:37:10.946 | INFO | "gpu" : " 17 GB", | 20260508 02:37:10.946 | INFO | "huge" : "", | 20260508 02:37:10.946 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:10.946 | INFO | "malloc" : " 14 GB", | 20260508 02:37:10.946 | INFO | "pinned" : " 29 GB" | 20260508 02:37:10.947 | INFO | } | 20260508 02:37:10.947 | INFO | Perf report for the last 5.14138 seconds: | 20260508 02:37:10.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:10.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:37:10.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 02:37:10.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:10.947 | INFO | Uploaded 36GiB to GPU in 2.54826s (14.1273GiB/s) | 20260508 02:37:10.947 | INFO | Downloaded 34GiB to GPU in 2.50347s (13.5812GiB/s) | 20260508 02:37:16.089 | INFO | Analyzing chunk frames = [4573184,4573696) | 20260508 02:37:16.089 | INFO | MemInfo: { | 20260508 02:37:16.089 | INFO | "gpu" : " 17 GB", | 20260508 02:37:16.089 | INFO | "huge" : "", | 20260508 02:37:16.089 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:16.089 | INFO | "malloc" : " 14 GB", | 20260508 02:37:16.089 | INFO | "pinned" : " 29 GB" | 20260508 02:37:16.089 | INFO | } | 20260508 02:37:16.089 | INFO | Perf report for the last 5.14249 seconds: | 20260508 02:37:16.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:16.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.089 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 02:37:16.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.089 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:37:16.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:16.090 | INFO | Uploaded 36GiB to GPU in 2.40659s (14.9589GiB/s) | 20260508 02:37:16.090 | INFO | Downloaded 34GiB to GPU in 2.57676s (13.1949GiB/s) | 20260508 02:37:21.221 | INFO | Analyzing chunk frames = [4573696,4574208) | 20260508 02:37:21.221 | INFO | MemInfo: { | 20260508 02:37:21.221 | INFO | "gpu" : " 17 GB", | 20260508 02:37:21.221 | INFO | "huge" : "", | 20260508 02:37:21.221 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:21.221 | INFO | "malloc" : " 14 GB", | 20260508 02:37:21.221 | INFO | "pinned" : " 29 GB" | 20260508 02:37:21.221 | INFO | } | 20260508 02:37:21.221 | INFO | Perf report for the last 5.13204 seconds: | 20260508 02:37:21.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:21.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 02:37:21.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:37:21.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:21.221 | INFO | Uploaded 36GiB to GPU in 2.38154s (15.1163GiB/s) | 20260508 02:37:21.221 | INFO | Downloaded 34GiB to GPU in 2.43642s (13.9549GiB/s) | 20260508 02:37:26.359 | INFO | Analyzing chunk frames = [4574208,4574720) | 20260508 02:37:26.359 | INFO | MemInfo: { | 20260508 02:37:26.359 | INFO | "gpu" : " 17 GB", | 20260508 02:37:26.359 | INFO | "huge" : "", | 20260508 02:37:26.359 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:26.359 | INFO | "malloc" : " 14 GB", | 20260508 02:37:26.359 | INFO | "pinned" : " 29 GB" | 20260508 02:37:26.359 | INFO | } | 20260508 02:37:26.359 | INFO | Perf report for the last 5.13817 seconds: | 20260508 02:37:26.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:26.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.359 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:37:26.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.359 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:37:26.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:26.360 | INFO | Uploaded 36GiB to GPU in 2.40307s (14.9808GiB/s) | 20260508 02:37:26.360 | INFO | Downloaded 34GiB to GPU in 2.43722s (13.9503GiB/s) | 20260508 02:37:31.503 | INFO | Analyzing chunk frames = [4574720,4575232) | 20260508 02:37:31.503 | INFO | MemInfo: { | 20260508 02:37:31.503 | INFO | "gpu" : " 17 GB", | 20260508 02:37:31.503 | INFO | "huge" : "", | 20260508 02:37:31.503 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:31.503 | INFO | "malloc" : " 14 GB", | 20260508 02:37:31.503 | INFO | "pinned" : " 29 GB" | 20260508 02:37:31.503 | INFO | } | 20260508 02:37:31.503 | INFO | Perf report for the last 5.14392 seconds: | 20260508 02:37:31.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:31.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.503 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 02:37:31.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.503 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:37:31.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:31.504 | INFO | Uploaded 36GiB to GPU in 2.52646s (14.2492GiB/s) | 20260508 02:37:31.504 | INFO | Downloaded 34GiB to GPU in 2.46858s (13.7731GiB/s) | 20260508 02:37:31.601 | INFO | Starting a round of Baz IO | 20260508 02:37:35.031 | INFO | Populating RTMetrics report | 20260508 02:37:35.031 | INFO | RTMetrics report populated | 20260508 02:37:35.031 | INFO | Opening tmp RTMetrics json file | 20260508 02:37:35.043 | INFO | Serializing RTMetrics to json | 20260508 02:37:35.049 | INFO | Serialization done, swapping tmp file now... | 20260508 02:37:35.050 | INFO | RTMetrics done | 20260508 02:37:35.130 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.529 seconds | 20260508 02:37:35.130 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.49% / 0.00% / 58.51% | 20260508 02:37:35.130 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.70262% / 3.97364e-06% / 5.96046e-05% | 20260508 02:37:35.130 | INFO | Baz IO overall: | 20260508 02:37:35.130 | INFO | BytesWritten : 2064339517347 | 20260508 02:37:35.130 | INFO | NumEvents : 997854368103 | 20260508 02:37:36.648 | INFO | Analyzing chunk frames = [4575232,4575744) | 20260508 02:37:36.648 | INFO | MemInfo: { | 20260508 02:37:36.648 | INFO | "gpu" : " 17 GB", | 20260508 02:37:36.648 | INFO | "huge" : "", | 20260508 02:37:36.648 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:36.648 | INFO | "malloc" : " 14 GB", | 20260508 02:37:36.648 | INFO | "pinned" : " 29 GB" | 20260508 02:37:36.648 | INFO | } | 20260508 02:37:36.648 | INFO | Perf report for the last 5.14447 seconds: | 20260508 02:37:36.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:36.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:36.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:36.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:36.648 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 02:37:36.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:36.648 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 02:37:36.648 | INFO | PRE_HQ : 65.5%, 1.7, 56.2ms, 0 | 20260508 02:37:36.648 | INFO | BAZWRITER : 68.2%, 4.2, 147.1ms, 0 | 20260508 02:37:36.648 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 02:37:36.648 | INFO | Uploaded 36GiB to GPU in 2.54542s (14.143GiB/s) | 20260508 02:37:36.648 | INFO | Downloaded 38GiB to GPU in 2.69855s (14.0817GiB/s) | 20260508 02:37:41.783 | INFO | Analyzing chunk frames = [4575744,4576256) | 20260508 02:37:41.783 | INFO | MemInfo: { | 20260508 02:37:41.783 | INFO | "gpu" : " 17 GB", | 20260508 02:37:41.783 | INFO | "huge" : "", | 20260508 02:37:41.783 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:41.783 | INFO | "malloc" : " 14 GB", | 20260508 02:37:41.783 | INFO | "pinned" : " 29 GB" | 20260508 02:37:41.783 | INFO | } | 20260508 02:37:41.784 | INFO | Perf report for the last 5.13578 seconds: | 20260508 02:37:41.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:41.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:37:41.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:37:41.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:41.784 | INFO | Uploaded 36GiB to GPU in 2.54701s (14.1342GiB/s) | 20260508 02:37:41.784 | INFO | Downloaded 34GiB to GPU in 2.44896s (13.8834GiB/s) | 20260508 02:37:46.918 | INFO | Analyzing chunk frames = [4576256,4576768) | 20260508 02:37:46.918 | INFO | MemInfo: { | 20260508 02:37:46.918 | INFO | "gpu" : " 17 GB", | 20260508 02:37:46.918 | INFO | "huge" : "", | 20260508 02:37:46.918 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:46.918 | INFO | "malloc" : " 14 GB", | 20260508 02:37:46.918 | INFO | "pinned" : " 29 GB" | 20260508 02:37:46.918 | INFO | } | 20260508 02:37:46.918 | INFO | Perf report for the last 5.13482 seconds: | 20260508 02:37:46.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:46.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.918 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:37:46.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.919 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:37:46.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:46.919 | INFO | Uploaded 36GiB to GPU in 2.52403s (14.2629GiB/s) | 20260508 02:37:46.919 | INFO | Downloaded 34GiB to GPU in 2.5185s (13.5001GiB/s) | 20260508 02:37:52.057 | INFO | Analyzing chunk frames = [4576768,4577280) | 20260508 02:37:52.057 | INFO | MemInfo: { | 20260508 02:37:52.057 | INFO | "gpu" : " 17 GB", | 20260508 02:37:52.057 | INFO | "huge" : "", | 20260508 02:37:52.057 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:52.057 | INFO | "malloc" : " 14 GB", | 20260508 02:37:52.058 | INFO | "pinned" : " 29 GB" | 20260508 02:37:52.058 | INFO | } | 20260508 02:37:52.058 | INFO | Perf report for the last 5.13939 seconds: | 20260508 02:37:52.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:52.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:37:52.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:37:52.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:52.058 | INFO | Uploaded 36GiB to GPU in 2.49091s (14.4525GiB/s) | 20260508 02:37:52.058 | INFO | Downloaded 34GiB to GPU in 2.51537s (13.5169GiB/s) | 20260508 02:37:57.198 | INFO | Analyzing chunk frames = [4577280,4577792) | 20260508 02:37:57.198 | INFO | MemInfo: { | 20260508 02:37:57.198 | INFO | "gpu" : " 17 GB", | 20260508 02:37:57.198 | INFO | "huge" : "", | 20260508 02:37:57.198 | INFO | "huge_shared" : " 29 GB", | 20260508 02:37:57.198 | INFO | "malloc" : " 14 GB", | 20260508 02:37:57.198 | INFO | "pinned" : " 29 GB" | 20260508 02:37:57.198 | INFO | } | 20260508 02:37:57.198 | INFO | Perf report for the last 5.14032 seconds: | 20260508 02:37:57.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:37:57.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 02:37:57.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 02:37:57.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:37:57.198 | INFO | Uploaded 36GiB to GPU in 2.28985s (15.7215GiB/s) | 20260508 02:37:57.199 | INFO | Downloaded 34GiB to GPU in 2.51252s (13.5323GiB/s) | 20260508 02:38:02.336 | INFO | Analyzing chunk frames = [4577792,4578304) | 20260508 02:38:02.336 | INFO | MemInfo: { | 20260508 02:38:02.336 | INFO | "gpu" : " 17 GB", | 20260508 02:38:02.336 | INFO | "huge" : "", | 20260508 02:38:02.336 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:02.336 | INFO | "malloc" : " 14 GB", | 20260508 02:38:02.336 | INFO | "pinned" : " 29 GB" | 20260508 02:38:02.336 | INFO | } | 20260508 02:38:02.336 | INFO | Perf report for the last 5.13806 seconds: | 20260508 02:38:02.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:02.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.336 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:38:02.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.336 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 02:38:02.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:02.337 | INFO | Uploaded 36GiB to GPU in 2.38821s (15.0741GiB/s) | 20260508 02:38:02.337 | INFO | Downloaded 34GiB to GPU in 2.44286s (13.9181GiB/s) | 20260508 02:38:07.476 | INFO | Analyzing chunk frames = [4578304,4578816) | 20260508 02:38:07.476 | INFO | MemInfo: { | 20260508 02:38:07.476 | INFO | "gpu" : " 17 GB", | 20260508 02:38:07.476 | INFO | "huge" : "", | 20260508 02:38:07.476 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:07.476 | INFO | "malloc" : " 14 GB", | 20260508 02:38:07.476 | INFO | "pinned" : " 29 GB" | 20260508 02:38:07.476 | INFO | } | 20260508 02:38:07.476 | INFO | Perf report for the last 5.14012 seconds: | 20260508 02:38:07.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:07.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.476 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 02:38:07.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.476 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:38:07.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:07.477 | INFO | Uploaded 36GiB to GPU in 2.40424s (14.9736GiB/s) | 20260508 02:38:07.477 | INFO | Downloaded 34GiB to GPU in 2.44058s (13.9311GiB/s) | 20260508 02:38:12.612 | INFO | Analyzing chunk frames = [4578816,4579328) | 20260508 02:38:12.612 | INFO | MemInfo: { | 20260508 02:38:12.612 | INFO | "gpu" : " 17 GB", | 20260508 02:38:12.612 | INFO | "huge" : "", | 20260508 02:38:12.612 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:12.612 | INFO | "malloc" : " 14 GB", | 20260508 02:38:12.613 | INFO | "pinned" : " 29 GB" | 20260508 02:38:12.613 | INFO | } | 20260508 02:38:12.613 | INFO | Perf report for the last 5.13635 seconds: | 20260508 02:38:12.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:12.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 02:38:12.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:38:12.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:12.613 | INFO | Uploaded 36GiB to GPU in 2.53064s (14.2256GiB/s) | 20260508 02:38:12.613 | INFO | Downloaded 34GiB to GPU in 2.4726s (13.7507GiB/s) | 20260508 02:38:12.613 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:38:12.613 | INFO | Total mem usage: 16.7 GiB | 20260508 02:38:12.613 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:38:12.613 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.58%) | 20260508 02:38:12.613 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:38:12.613 | INFO | High Water Breakdown: | 20260508 02:38:12.613 | INFO | 60.7% : 9.32 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:38:12.613 | INFO | 39.2% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:38:12.613 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:38:12.613 | INFO | ------------------------------------------ | 20260508 02:38:12.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:38:12.613 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:38:12.613 | INFO | Total mem usage: 26.5 GiB | 20260508 02:38:12.613 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:38:12.613 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:38:12.613 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:38:12.613 | INFO | High Water Breakdown: | 20260508 02:38:12.613 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:38:12.613 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:38:12.613 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:38:12.613 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:38:12.613 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:38:12.613 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:38:12.613 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:38:12.613 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:38:12.613 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:38:12.613 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:38:12.613 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:38:12.613 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:38:12.613 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:38:12.613 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:38:12.613 | INFO | ------------------------------------------ | 20260508 02:38:12.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:38:12.613 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:38:12.613 | INFO | Total mem usage: 27.8 GiB | 20260508 02:38:12.613 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:38:12.613 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:38:12.613 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:38:12.613 | INFO | High Water Breakdown: | 20260508 02:38:12.613 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:38:12.613 | INFO | ------------------------------------------ | 20260508 02:38:12.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:38:12.613 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:38:12.613 | INFO | Total mem usage: 15.7 GiB | 20260508 02:38:12.613 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:38:12.613 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:38:12.613 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:38:12.613 | INFO | High Water Breakdown: | 20260508 02:38:12.613 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:38:12.613 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:38:12.613 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:38:12.613 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:38:12.613 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:38:12.613 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:38:12.613 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:38:12.613 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:38:12.613 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:38:12.613 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:38:12.613 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:38:12.613 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:38:12.613 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:38:12.613 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:38:12.613 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:38:12.613 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:38:12.613 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:38:12.613 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:38:12.613 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:38:12.613 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:38:12.613 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:38:12.613 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:38:12.613 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:38:12.613 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:38:12.613 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:38:12.613 | INFO | ------------------------------------------ | 20260508 02:38:12.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:38:12.613 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:38:12.613 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:38:12.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:38:12.613 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.62e+01, 1.41e+01 / 1.42e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.01e-01, 1.27e-02 / 5.78e-03 | 20260508 02:38:12.613 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.46e+00, 5.86e-01 / 1.59e-01 | 20260508 02:38:12.613 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.49e+00, 2.07e-01 / 8.05e-02 | 20260508 02:38:12.613 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.62e+01, 4.10e+00 / 1.58e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.64e+00 / 1.52e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.26e-01 / 1.84e-01 | 20260508 02:38:12.613 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.23e+00, 2.65e-01 / 1.20e-01 | 20260508 02:38:12.613 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.28e-01 / 3.67e-01, 5.18e-02 / 2.86e-02 | 20260508 02:38:12.613 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:38:12.613 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:38:12.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:38:12.613 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 24.55%, Download -- 2.61e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 30.68%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 1.97%, LockUpload -- 2.09e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 3.04%, LockDownload -- 3.23e+00 / 0.00e+00, 4.53e+00 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:38:12.613 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260508 02:38:12.711 | INFO | Starting a round of Baz IO | 20260508 02:38:16.165 | INFO | Populating RTMetrics report | 20260508 02:38:16.165 | INFO | RTMetrics report populated | 20260508 02:38:16.165 | INFO | Opening tmp RTMetrics json file | 20260508 02:38:16.168 | INFO | Serializing RTMetrics to json | 20260508 02:38:16.197 | INFO | Serialization done, swapping tmp file now... | 20260508 02:38:16.212 | INFO | RTMetrics done | 20260508 02:38:16.271 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.559 seconds | 20260508 02:38:16.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.49% / 0.00% / 58.51% | 20260508 02:38:16.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.701845% / 3.97364e-06% / 6.35783e-05% | 20260508 02:38:16.271 | INFO | Baz IO overall: | 20260508 02:38:16.271 | INFO | BytesWritten : 2066093520936 | 20260508 02:38:16.271 | INFO | NumEvents : 998694623013 | 20260508 02:38:17.757 | INFO | Analyzing chunk frames = [4579328,4579840) | 20260508 02:38:17.757 | INFO | MemInfo: { | 20260508 02:38:17.757 | INFO | "gpu" : " 17 GB", | 20260508 02:38:17.757 | INFO | "huge" : "", | 20260508 02:38:17.757 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:17.757 | INFO | "malloc" : " 14 GB", | 20260508 02:38:17.757 | INFO | "pinned" : " 29 GB" | 20260508 02:38:17.757 | INFO | } | 20260508 02:38:17.757 | INFO | Perf report for the last 5.14392 seconds: | 20260508 02:38:17.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:17.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:17.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:17.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:17.757 | INFO | ANALYSIS : 68.3%, 2.9, 104.7ms, 0 | 20260508 02:38:17.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:17.757 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260508 02:38:17.757 | INFO | PRE_HQ : 66.8%, 1.7, 59.0ms, 0 | 20260508 02:38:17.757 | INFO | BAZWRITER : 68.8%, 3.9, 136.5ms, 0 | 20260508 02:38:17.757 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 02:38:17.757 | INFO | Uploaded 36GiB to GPU in 2.55055s (14.1146GiB/s) | 20260508 02:38:17.757 | INFO | Downloaded 38GiB to GPU in 2.69449s (14.1028GiB/s) | 20260508 02:38:22.897 | INFO | Analyzing chunk frames = [4579840,4580352) | 20260508 02:38:22.897 | INFO | MemInfo: { | 20260508 02:38:22.897 | INFO | "gpu" : " 17 GB", | 20260508 02:38:22.897 | INFO | "huge" : "", | 20260508 02:38:22.897 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:22.897 | INFO | "malloc" : " 14 GB", | 20260508 02:38:22.897 | INFO | "pinned" : " 29 GB" | 20260508 02:38:22.897 | INFO | } | 20260508 02:38:22.897 | INFO | Perf report for the last 5.14078 seconds: | 20260508 02:38:22.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:22.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.897 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 02:38:22.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.897 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:38:22.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:22.898 | INFO | Uploaded 36GiB to GPU in 2.53632s (14.1938GiB/s) | 20260508 02:38:22.898 | INFO | Downloaded 34GiB to GPU in 2.42883s (13.9985GiB/s) | 20260508 02:38:28.035 | INFO | Analyzing chunk frames = [4580352,4580864) | 20260508 02:38:28.035 | INFO | MemInfo: { | 20260508 02:38:28.035 | INFO | "gpu" : " 17 GB", | 20260508 02:38:28.035 | INFO | "huge" : "", | 20260508 02:38:28.035 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:28.035 | INFO | "malloc" : " 14 GB", | 20260508 02:38:28.035 | INFO | "pinned" : " 29 GB" | 20260508 02:38:28.035 | INFO | } | 20260508 02:38:28.035 | INFO | Perf report for the last 5.13809 seconds: | 20260508 02:38:28.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:28.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:38:28.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:38:28.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:28.036 | INFO | Uploaded 36GiB to GPU in 2.51443s (14.3173GiB/s) | 20260508 02:38:28.036 | INFO | Downloaded 34GiB to GPU in 2.50173s (13.5906GiB/s) | 20260508 02:38:33.170 | INFO | Analyzing chunk frames = [4580864,4581376) | 20260508 02:38:33.170 | INFO | MemInfo: { | 20260508 02:38:33.170 | INFO | "gpu" : " 17 GB", | 20260508 02:38:33.170 | INFO | "huge" : "", | 20260508 02:38:33.170 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:33.170 | INFO | "malloc" : " 14 GB", | 20260508 02:38:33.170 | INFO | "pinned" : " 29 GB" | 20260508 02:38:33.170 | INFO | } | 20260508 02:38:33.170 | INFO | Perf report for the last 5.13492 seconds: | 20260508 02:38:33.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:33.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.170 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:38:33.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.170 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:38:33.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:33.171 | INFO | Uploaded 36GiB to GPU in 2.53306s (14.212GiB/s) | 20260508 02:38:33.171 | INFO | Downloaded 34GiB to GPU in 2.50122s (13.5934GiB/s) | 20260508 02:38:38.310 | INFO | Analyzing chunk frames = [4581376,4581888) | 20260508 02:38:38.310 | INFO | MemInfo: { | 20260508 02:38:38.310 | INFO | "gpu" : " 17 GB", | 20260508 02:38:38.310 | INFO | "huge" : "", | 20260508 02:38:38.310 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:38.310 | INFO | "malloc" : " 14 GB", | 20260508 02:38:38.310 | INFO | "pinned" : " 29 GB" | 20260508 02:38:38.310 | INFO | } | 20260508 02:38:38.310 | INFO | Perf report for the last 5.14007 seconds: | 20260508 02:38:38.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:38.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:38:38.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:38:38.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:38.311 | INFO | Uploaded 36GiB to GPU in 2.47114s (14.5682GiB/s) | 20260508 02:38:38.311 | INFO | Downloaded 34GiB to GPU in 2.57211s (13.2187GiB/s) | 20260508 02:38:43.451 | INFO | Analyzing chunk frames = [4581888,4582400) | 20260508 02:38:43.451 | INFO | MemInfo: { | 20260508 02:38:43.451 | INFO | "gpu" : " 17 GB", | 20260508 02:38:43.451 | INFO | "huge" : "", | 20260508 02:38:43.451 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:43.451 | INFO | "malloc" : " 14 GB", | 20260508 02:38:43.452 | INFO | "pinned" : " 29 GB" | 20260508 02:38:43.452 | INFO | } | 20260508 02:38:43.452 | INFO | Perf report for the last 5.14122 seconds: | 20260508 02:38:43.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:43.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:38:43.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 02:38:43.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:43.452 | INFO | Uploaded 36GiB to GPU in 2.42848s (14.8241GiB/s) | 20260508 02:38:43.452 | INFO | Downloaded 34GiB to GPU in 2.40987s (14.1087GiB/s) | 20260508 02:38:48.593 | INFO | Analyzing chunk frames = [4582400,4582912) | 20260508 02:38:48.593 | INFO | MemInfo: { | 20260508 02:38:48.593 | INFO | "gpu" : " 17 GB", | 20260508 02:38:48.593 | INFO | "huge" : "", | 20260508 02:38:48.594 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:48.594 | INFO | "malloc" : " 14 GB", | 20260508 02:38:48.594 | INFO | "pinned" : " 29 GB" | 20260508 02:38:48.594 | INFO | } | 20260508 02:38:48.594 | INFO | Perf report for the last 5.14225 seconds: | 20260508 02:38:48.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:48.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.594 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:38:48.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.594 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 02:38:48.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:48.595 | INFO | Uploaded 36GiB to GPU in 2.48928s (14.462GiB/s) | 20260508 02:38:48.595 | INFO | Downloaded 34GiB to GPU in 2.42885s (13.9984GiB/s) | 20260508 02:38:53.731 | INFO | Analyzing chunk frames = [4582912,4583424) | 20260508 02:38:53.731 | INFO | MemInfo: { | 20260508 02:38:53.731 | INFO | "gpu" : " 17 GB", | 20260508 02:38:53.731 | INFO | "huge" : "", | 20260508 02:38:53.731 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:53.731 | INFO | "malloc" : " 14 GB", | 20260508 02:38:53.731 | INFO | "pinned" : " 29 GB" | 20260508 02:38:53.731 | INFO | } | 20260508 02:38:53.731 | INFO | Perf report for the last 5.13704 seconds: | 20260508 02:38:53.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:53.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.732 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:38:53.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.732 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 02:38:53.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:53.732 | INFO | Uploaded 36GiB to GPU in 2.47987s (14.5169GiB/s) | 20260508 02:38:53.732 | INFO | Downloaded 34GiB to GPU in 2.42657s (14.0115GiB/s) | 20260508 02:38:53.846 | INFO | Starting a round of Baz IO | 20260508 02:38:57.278 | INFO | Populating RTMetrics report | 20260508 02:38:57.279 | INFO | RTMetrics report populated | 20260508 02:38:57.279 | INFO | Opening tmp RTMetrics json file | 20260508 02:38:57.280 | INFO | Serializing RTMetrics to json | 20260508 02:38:57.282 | INFO | Serialization done, swapping tmp file now... | 20260508 02:38:57.282 | INFO | RTMetrics done | 20260508 02:38:57.396 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.55 seconds | 20260508 02:38:57.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.49% / 0.00% / 58.51% | 20260508 02:38:57.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.700982% / 3.97364e-06% / 7.94729e-05% | 20260508 02:38:57.396 | INFO | Baz IO overall: | 20260508 02:38:57.396 | INFO | BytesWritten : 2067847620318 | 20260508 02:38:57.396 | INFO | NumEvents : 999535025802 | 20260508 02:38:58.863 | INFO | Analyzing chunk frames = [4583424,4583936) | 20260508 02:38:58.863 | INFO | MemInfo: { | 20260508 02:38:58.863 | INFO | "gpu" : " 17 GB", | 20260508 02:38:58.863 | INFO | "huge" : "", | 20260508 02:38:58.863 | INFO | "huge_shared" : " 29 GB", | 20260508 02:38:58.863 | INFO | "malloc" : " 14 GB", | 20260508 02:38:58.863 | INFO | "pinned" : " 29 GB" | 20260508 02:38:58.863 | INFO | } | 20260508 02:38:58.863 | INFO | Perf report for the last 5.13222 seconds: | 20260508 02:38:58.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:38:58.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:58.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:58.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:58.863 | INFO | ANALYSIS : 68.7%, 2.9, 104.0ms, 0 | 20260508 02:38:58.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:38:58.863 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260508 02:38:58.863 | INFO | PRE_HQ : 67.3%, 1.8, 62.0ms, 0 | 20260508 02:38:58.863 | INFO | BAZWRITER : 68.8%, 3.7, 132.0ms, 0 | 20260508 02:38:58.863 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 02:38:58.863 | INFO | Uploaded 36GiB to GPU in 2.55984s (14.0634GiB/s) | 20260508 02:38:58.863 | INFO | Downloaded 38GiB to GPU in 2.6813s (14.1722GiB/s) | 20260508 02:39:04.011 | INFO | Analyzing chunk frames = [4583936,4584448) | 20260508 02:39:04.011 | INFO | MemInfo: { | 20260508 02:39:04.011 | INFO | "gpu" : " 17 GB", | 20260508 02:39:04.011 | INFO | "huge" : "", | 20260508 02:39:04.011 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:04.011 | INFO | "malloc" : " 14 GB", | 20260508 02:39:04.011 | INFO | "pinned" : " 29 GB" | 20260508 02:39:04.011 | INFO | } | 20260508 02:39:04.011 | INFO | Perf report for the last 5.14741 seconds: | 20260508 02:39:04.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:04.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 02:39:04.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:39:04.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:04.011 | INFO | Uploaded 36GiB to GPU in 2.5507s (14.1138GiB/s) | 20260508 02:39:04.011 | INFO | Downloaded 34GiB to GPU in 2.42391s (14.0269GiB/s) | 20260508 02:39:09.151 | INFO | Analyzing chunk frames = [4584448,4584960) | 20260508 02:39:09.151 | INFO | MemInfo: { | 20260508 02:39:09.151 | INFO | "gpu" : " 17 GB", | 20260508 02:39:09.151 | INFO | "huge" : "", | 20260508 02:39:09.151 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:09.151 | INFO | "malloc" : " 14 GB", | 20260508 02:39:09.151 | INFO | "pinned" : " 29 GB" | 20260508 02:39:09.151 | INFO | } | 20260508 02:39:09.151 | INFO | Perf report for the last 5.14063 seconds: | 20260508 02:39:09.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:09.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.151 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:39:09.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.151 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:39:09.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:09.152 | INFO | Uploaded 36GiB to GPU in 2.57007s (14.0074GiB/s) | 20260508 02:39:09.152 | INFO | Downloaded 34GiB to GPU in 2.49082s (13.6501GiB/s) | 20260508 02:39:14.289 | INFO | Analyzing chunk frames = [4584960,4585472) | 20260508 02:39:14.289 | INFO | MemInfo: { | 20260508 02:39:14.289 | INFO | "gpu" : " 17 GB", | 20260508 02:39:14.289 | INFO | "huge" : "", | 20260508 02:39:14.289 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:14.289 | INFO | "malloc" : " 14 GB", | 20260508 02:39:14.289 | INFO | "pinned" : " 29 GB" | 20260508 02:39:14.289 | INFO | } | 20260508 02:39:14.289 | INFO | Perf report for the last 5.13746 seconds: | 20260508 02:39:14.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:14.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 02:39:14.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:39:14.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:14.289 | INFO | Uploaded 36GiB to GPU in 2.56969s (14.0095GiB/s) | 20260508 02:39:14.289 | INFO | Downloaded 34GiB to GPU in 2.4846s (13.6843GiB/s) | 20260508 02:39:19.423 | INFO | Analyzing chunk frames = [4585472,4585984) | 20260508 02:39:19.423 | INFO | MemInfo: { | 20260508 02:39:19.423 | INFO | "gpu" : " 17 GB", | 20260508 02:39:19.423 | INFO | "huge" : "", | 20260508 02:39:19.423 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:19.424 | INFO | "malloc" : " 14 GB", | 20260508 02:39:19.424 | INFO | "pinned" : " 29 GB" | 20260508 02:39:19.424 | INFO | } | 20260508 02:39:19.424 | INFO | Perf report for the last 5.13476 seconds: | 20260508 02:39:19.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:19.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 02:39:19.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:39:19.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:19.424 | INFO | Uploaded 36GiB to GPU in 2.43317s (14.7955GiB/s) | 20260508 02:39:19.424 | INFO | Downloaded 34GiB to GPU in 2.55349s (13.3151GiB/s) | 20260508 02:39:24.566 | INFO | Analyzing chunk frames = [4585984,4586496) | 20260508 02:39:24.566 | INFO | MemInfo: { | 20260508 02:39:24.566 | INFO | "gpu" : " 17 GB", | 20260508 02:39:24.567 | INFO | "huge" : "", | 20260508 02:39:24.567 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:24.567 | INFO | "malloc" : " 14 GB", | 20260508 02:39:24.567 | INFO | "pinned" : " 29 GB" | 20260508 02:39:24.567 | INFO | } | 20260508 02:39:24.567 | INFO | Perf report for the last 5.14301 seconds: | 20260508 02:39:24.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:24.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 02:39:24.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:39:24.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:24.567 | INFO | Uploaded 36GiB to GPU in 2.43644s (14.7757GiB/s) | 20260508 02:39:24.567 | INFO | Downloaded 34GiB to GPU in 2.41361s (14.0868GiB/s) | 20260508 02:39:29.706 | INFO | Analyzing chunk frames = [4586496,4587008) | 20260508 02:39:29.706 | INFO | MemInfo: { | 20260508 02:39:29.706 | INFO | "gpu" : " 17 GB", | 20260508 02:39:29.706 | INFO | "huge" : "", | 20260508 02:39:29.706 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:29.706 | INFO | "malloc" : " 14 GB", | 20260508 02:39:29.706 | INFO | "pinned" : " 29 GB" | 20260508 02:39:29.706 | INFO | } | 20260508 02:39:29.706 | INFO | Perf report for the last 5.13912 seconds: | 20260508 02:39:29.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:29.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 02:39:29.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 02:39:29.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:29.706 | INFO | Uploaded 36GiB to GPU in 2.4667s (14.5944GiB/s) | 20260508 02:39:29.706 | INFO | Downloaded 34GiB to GPU in 2.41672s (14.0686GiB/s) | 20260508 02:39:34.850 | INFO | Analyzing chunk frames = [4587008,4587520) | 20260508 02:39:34.850 | INFO | MemInfo: { | 20260508 02:39:34.850 | INFO | "gpu" : " 17 GB", | 20260508 02:39:34.850 | INFO | "huge" : "", | 20260508 02:39:34.850 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:34.850 | INFO | "malloc" : " 14 GB", | 20260508 02:39:34.850 | INFO | "pinned" : " 29 GB" | 20260508 02:39:34.850 | INFO | } | 20260508 02:39:34.850 | INFO | Perf report for the last 5.14456 seconds: | 20260508 02:39:34.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:34.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.850 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 02:39:34.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.850 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:39:34.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:34.851 | INFO | Uploaded 36GiB to GPU in 2.57153s (13.9995GiB/s) | 20260508 02:39:34.851 | INFO | Downloaded 34GiB to GPU in 2.45331s (13.8588GiB/s) | 20260508 02:39:34.851 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:39:34.851 | INFO | Total mem usage: 16.7 GiB | 20260508 02:39:34.851 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:39:34.851 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260508 02:39:34.851 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:39:34.851 | INFO | High Water Breakdown: | 20260508 02:39:34.851 | INFO | 59.9% : 8.82 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:39:34.851 | INFO | 39.9% : 5.87 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:39:34.851 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:39:34.851 | INFO | ------------------------------------------ | 20260508 02:39:34.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:39:34.851 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:39:34.851 | INFO | Total mem usage: 26.5 GiB | 20260508 02:39:34.851 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:39:34.851 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.3%) | 20260508 02:39:34.851 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260508 02:39:34.851 | INFO | High Water Breakdown: | 20260508 02:39:34.851 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:39:34.851 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:39:34.851 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:39:34.851 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:39:34.851 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:39:34.851 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:39:34.851 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:39:34.851 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:39:34.851 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:39:34.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:39:34.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:39:34.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:39:34.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:39:34.851 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:39:34.851 | INFO | ------------------------------------------ | 20260508 02:39:34.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:39:34.851 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:39:34.851 | INFO | Total mem usage: 27.8 GiB | 20260508 02:39:34.851 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:39:34.851 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:39:34.851 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:39:34.851 | INFO | High Water Breakdown: | 20260508 02:39:34.851 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:39:34.851 | INFO | ------------------------------------------ | 20260508 02:39:34.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:39:34.851 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:39:34.851 | INFO | Total mem usage: 15.7 GiB | 20260508 02:39:34.851 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:39:34.851 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:39:34.851 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:39:34.851 | INFO | High Water Breakdown: | 20260508 02:39:34.851 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:39:34.851 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:39:34.851 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:39:34.851 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:39:34.851 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:39:34.851 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:39:34.851 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:39:34.851 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:39:34.851 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:39:34.851 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:39:34.851 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:39:34.851 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:39:34.851 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:39:34.851 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:39:34.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:39:34.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:39:34.851 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:39:34.851 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:39:34.851 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:39:34.851 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:39:34.851 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:39:34.851 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:39:34.851 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:39:34.851 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:39:34.851 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:39:34.851 | INFO | ------------------------------------------ | 20260508 02:39:34.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:39:34.851 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:39:34.851 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:39:34.851 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:39:34.851 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.63e+01, 1.43e+01 / 1.90e+00 | 20260508 02:39:34.851 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.24e-02 / 7.13e-03 | 20260508 02:39:34.851 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.47e+00, 5.90e-01 / 1.86e-01 | 20260508 02:39:34.851 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.05e-01 | 20260508 02:39:34.851 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.63e+01, 4.27e+00 / 2.23e+00 | 20260508 02:39:34.851 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.65e+00 / 1.48e+00 | 20260508 02:39:34.851 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.72e+00, 4.19e-01 / 1.76e-01 | 20260508 02:39:34.851 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.23e+00, 2.66e-01 / 1.05e-01 | 20260508 02:39:34.851 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.28e-01 / 3.64e-01, 5.19e-02 / 2.45e-02 | 20260508 02:39:34.852 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:39:34.852 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:39:34.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:39:34.852 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 24.56%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 2.33%, LockUpload -- 2.48e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 15.35%, LockCompute -- 1.63e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:39:34.852 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260508 02:39:34.947 | INFO | Starting a round of Baz IO | 20260508 02:39:38.390 | INFO | Populating RTMetrics report | 20260508 02:39:38.390 | INFO | RTMetrics report populated | 20260508 02:39:38.390 | INFO | Opening tmp RTMetrics json file | 20260508 02:39:38.411 | INFO | Serializing RTMetrics to json | 20260508 02:39:38.412 | INFO | Serialization done, swapping tmp file now... | 20260508 02:39:38.412 | INFO | RTMetrics done | 20260508 02:39:38.476 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.528 seconds | 20260508 02:39:38.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.48% / 0.00% / 58.51% | 20260508 02:39:38.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.698702% / 3.97364e-06% / 5.96046e-05% | 20260508 02:39:38.476 | INFO | Baz IO overall: | 20260508 02:39:38.476 | INFO | BytesWritten : 2069601406462 | 20260508 02:39:38.476 | INFO | NumEvents : 1000375337309 | 20260508 02:39:39.986 | INFO | Analyzing chunk frames = [4587520,4588032) | 20260508 02:39:39.986 | INFO | MemInfo: { | 20260508 02:39:39.986 | INFO | "gpu" : " 17 GB", | 20260508 02:39:39.986 | INFO | "huge" : "", | 20260508 02:39:39.986 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:39.986 | INFO | "malloc" : " 14 GB", | 20260508 02:39:39.986 | INFO | "pinned" : " 29 GB" | 20260508 02:39:39.986 | INFO | } | 20260508 02:39:39.986 | INFO | Perf report for the last 5.1354 seconds: | 20260508 02:39:39.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:39.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:39.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:39.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:39.986 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260508 02:39:39.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:39.986 | INFO | RT_METRICS : 40.2%, 1.0, 21.0ms, 0 | 20260508 02:39:39.986 | INFO | PRE_HQ : 66.0%, 1.6, 55.7ms, 0 | 20260508 02:39:39.986 | INFO | BAZWRITER : 68.3%, 3.7, 129.9ms, 0 | 20260508 02:39:39.986 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:39:39.986 | INFO | Uploaded 36GiB to GPU in 2.52839s (14.2383GiB/s) | 20260508 02:39:39.986 | INFO | Downloaded 38GiB to GPU in 2.68537s (14.1508GiB/s) | 20260508 02:39:45.121 | INFO | Analyzing chunk frames = [4588032,4588544) | 20260508 02:39:45.121 | INFO | MemInfo: { | 20260508 02:39:45.121 | INFO | "gpu" : " 17 GB", | 20260508 02:39:45.121 | INFO | "huge" : "", | 20260508 02:39:45.121 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:45.121 | INFO | "malloc" : " 14 GB", | 20260508 02:39:45.121 | INFO | "pinned" : " 29 GB" | 20260508 02:39:45.121 | INFO | } | 20260508 02:39:45.121 | INFO | Perf report for the last 5.13527 seconds: | 20260508 02:39:45.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:45.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:39:45.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:39:45.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:45.121 | INFO | Uploaded 36GiB to GPU in 2.57276s (13.9927GiB/s) | 20260508 02:39:45.121 | INFO | Downloaded 34GiB to GPU in 2.41325s (14.0889GiB/s) | 20260508 02:39:50.265 | INFO | Analyzing chunk frames = [4588544,4589056) | 20260508 02:39:50.265 | INFO | MemInfo: { | 20260508 02:39:50.265 | INFO | "gpu" : " 17 GB", | 20260508 02:39:50.265 | INFO | "huge" : "", | 20260508 02:39:50.265 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:50.265 | INFO | "malloc" : " 14 GB", | 20260508 02:39:50.265 | INFO | "pinned" : " 29 GB" | 20260508 02:39:50.265 | INFO | } | 20260508 02:39:50.265 | INFO | Perf report for the last 5.1442 seconds: | 20260508 02:39:50.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:50.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 02:39:50.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:39:50.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:50.265 | INFO | Uploaded 36GiB to GPU in 2.56109s (14.0565GiB/s) | 20260508 02:39:50.265 | INFO | Downloaded 34GiB to GPU in 2.48384s (13.6885GiB/s) | 20260508 02:39:55.403 | INFO | Analyzing chunk frames = [4589056,4589568) | 20260508 02:39:55.403 | INFO | MemInfo: { | 20260508 02:39:55.403 | INFO | "gpu" : " 17 GB", | 20260508 02:39:55.403 | INFO | "huge" : "", | 20260508 02:39:55.403 | INFO | "huge_shared" : " 29 GB", | 20260508 02:39:55.403 | INFO | "malloc" : " 14 GB", | 20260508 02:39:55.403 | INFO | "pinned" : " 29 GB" | 20260508 02:39:55.403 | INFO | } | 20260508 02:39:55.403 | INFO | Perf report for the last 5.13803 seconds: | 20260508 02:39:55.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:39:55.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.403 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:39:55.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.403 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:39:55.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:39:55.404 | INFO | Uploaded 36GiB to GPU in 2.53915s (14.178GiB/s) | 20260508 02:39:55.404 | INFO | Downloaded 34GiB to GPU in 2.48665s (13.673GiB/s) | 20260508 02:40:00.543 | INFO | Analyzing chunk frames = [4589568,4590080) | 20260508 02:40:00.543 | INFO | MemInfo: { | 20260508 02:40:00.543 | INFO | "gpu" : " 17 GB", | 20260508 02:40:00.543 | INFO | "huge" : "", | 20260508 02:40:00.543 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:00.543 | INFO | "malloc" : " 14 GB", | 20260508 02:40:00.543 | INFO | "pinned" : " 29 GB" | 20260508 02:40:00.543 | INFO | } | 20260508 02:40:00.543 | INFO | Perf report for the last 5.13975 seconds: | 20260508 02:40:00.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:00.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 02:40:00.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:40:00.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:00.543 | INFO | Uploaded 36GiB to GPU in 2.32022s (15.5158GiB/s) | 20260508 02:40:00.543 | INFO | Downloaded 34GiB to GPU in 2.49102s (13.649GiB/s) | 20260508 02:40:05.677 | INFO | Analyzing chunk frames = [4590080,4590592) | 20260508 02:40:05.677 | INFO | MemInfo: { | 20260508 02:40:05.677 | INFO | "gpu" : " 17 GB", | 20260508 02:40:05.677 | INFO | "huge" : "", | 20260508 02:40:05.677 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:05.677 | INFO | "malloc" : " 14 GB", | 20260508 02:40:05.678 | INFO | "pinned" : " 29 GB" | 20260508 02:40:05.678 | INFO | } | 20260508 02:40:05.678 | INFO | Perf report for the last 5.13456 seconds: | 20260508 02:40:05.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:05.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 02:40:05.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260508 02:40:05.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:05.678 | INFO | Uploaded 36GiB to GPU in 2.41653s (14.8974GiB/s) | 20260508 02:40:05.678 | INFO | Downloaded 34GiB to GPU in 2.42162s (14.0402GiB/s) | 20260508 02:40:10.820 | INFO | Analyzing chunk frames = [4590592,4591104) | 20260508 02:40:10.820 | INFO | MemInfo: { | 20260508 02:40:10.820 | INFO | "gpu" : " 17 GB", | 20260508 02:40:10.820 | INFO | "huge" : "", | 20260508 02:40:10.820 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:10.820 | INFO | "malloc" : " 14 GB", | 20260508 02:40:10.820 | INFO | "pinned" : " 29 GB" | 20260508 02:40:10.820 | INFO | } | 20260508 02:40:10.820 | INFO | Perf report for the last 5.14285 seconds: | 20260508 02:40:10.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:10.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.821 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:40:10.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.821 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 02:40:10.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:10.821 | INFO | Uploaded 36GiB to GPU in 2.42478s (14.8467GiB/s) | 20260508 02:40:10.821 | INFO | Downloaded 34GiB to GPU in 2.42035s (14.0476GiB/s) | 20260508 02:40:15.961 | INFO | Analyzing chunk frames = [4591104,4591616) | 20260508 02:40:15.961 | INFO | MemInfo: { | 20260508 02:40:15.961 | INFO | "gpu" : " 17 GB", | 20260508 02:40:15.961 | INFO | "huge" : "", | 20260508 02:40:15.961 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:15.961 | INFO | "malloc" : " 14 GB", | 20260508 02:40:15.961 | INFO | "pinned" : " 29 GB" | 20260508 02:40:15.961 | INFO | } | 20260508 02:40:15.961 | INFO | Perf report for the last 5.14063 seconds: | 20260508 02:40:15.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:15.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.961 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:40:15.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.961 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 02:40:15.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:15.962 | INFO | Uploaded 36GiB to GPU in 2.55907s (14.0676GiB/s) | 20260508 02:40:15.962 | INFO | Downloaded 34GiB to GPU in 2.46149s (13.8128GiB/s) | 20260508 02:40:16.074 | INFO | Starting a round of Baz IO | 20260508 02:40:19.525 | INFO | Populating RTMetrics report | 20260508 02:40:19.525 | INFO | RTMetrics report populated | 20260508 02:40:19.525 | INFO | Opening tmp RTMetrics json file | 20260508 02:40:19.528 | INFO | Serializing RTMetrics to json | 20260508 02:40:19.531 | INFO | Serialization done, swapping tmp file now... | 20260508 02:40:19.535 | INFO | RTMetrics done | 20260508 02:40:19.638 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.563 seconds | 20260508 02:40:19.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.48% / 0.00% / 58.52% | 20260508 02:40:19.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.69714% / 7.94729e-06% / 7.54992e-05% | 20260508 02:40:19.638 | INFO | Baz IO overall: | 20260508 02:40:19.638 | INFO | BytesWritten : 2071354928605 | 20260508 02:40:19.638 | INFO | NumEvents : 1001215490696 | 20260508 02:40:21.103 | INFO | Analyzing chunk frames = [4591616,4592128) | 20260508 02:40:21.103 | INFO | MemInfo: { | 20260508 02:40:21.103 | INFO | "gpu" : " 17 GB", | 20260508 02:40:21.103 | INFO | "huge" : "", | 20260508 02:40:21.103 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:21.103 | INFO | "malloc" : " 14 GB", | 20260508 02:40:21.103 | INFO | "pinned" : " 29 GB" | 20260508 02:40:21.103 | INFO | } | 20260508 02:40:21.103 | INFO | Perf report for the last 5.14196 seconds: | 20260508 02:40:21.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:21.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:21.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:21.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:21.103 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260508 02:40:21.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:21.103 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 02:40:21.103 | INFO | PRE_HQ : 67.7%, 1.8, 62.6ms, 0 | 20260508 02:40:21.103 | INFO | BAZWRITER : 68.9%, 3.5, 125.7ms, 0 | 20260508 02:40:21.103 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:40:21.103 | INFO | Uploaded 36GiB to GPU in 2.52583s (14.2528GiB/s) | 20260508 02:40:21.103 | INFO | Downloaded 38GiB to GPU in 2.69159s (14.1181GiB/s) | 20260508 02:40:26.239 | INFO | Analyzing chunk frames = [4592128,4592640) | 20260508 02:40:26.239 | INFO | MemInfo: { | 20260508 02:40:26.239 | INFO | "gpu" : " 17 GB", | 20260508 02:40:26.239 | INFO | "huge" : "", | 20260508 02:40:26.239 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:26.239 | INFO | "malloc" : " 14 GB", | 20260508 02:40:26.239 | INFO | "pinned" : " 29 GB" | 20260508 02:40:26.239 | INFO | } | 20260508 02:40:26.240 | INFO | Perf report for the last 5.13645 seconds: | 20260508 02:40:26.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:26.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:40:26.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:40:26.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:26.240 | INFO | Uploaded 36GiB to GPU in 2.50628s (14.3639GiB/s) | 20260508 02:40:26.240 | INFO | Downloaded 34GiB to GPU in 2.4389s (13.9407GiB/s) | 20260508 02:40:31.379 | INFO | Analyzing chunk frames = [4592640,4593152) | 20260508 02:40:31.379 | INFO | MemInfo: { | 20260508 02:40:31.379 | INFO | "gpu" : " 17 GB", | 20260508 02:40:31.379 | INFO | "huge" : "", | 20260508 02:40:31.379 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:31.379 | INFO | "malloc" : " 14 GB", | 20260508 02:40:31.379 | INFO | "pinned" : " 29 GB" | 20260508 02:40:31.379 | INFO | } | 20260508 02:40:31.379 | INFO | Perf report for the last 5.13951 seconds: | 20260508 02:40:31.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:31.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 02:40:31.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:40:31.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:31.379 | INFO | Uploaded 36GiB to GPU in 2.50348s (14.38GiB/s) | 20260508 02:40:31.379 | INFO | Downloaded 34GiB to GPU in 2.51304s (13.5294GiB/s) | 20260508 02:40:36.520 | INFO | Analyzing chunk frames = [4593152,4593664) | 20260508 02:40:36.520 | INFO | MemInfo: { | 20260508 02:40:36.520 | INFO | "gpu" : " 17 GB", | 20260508 02:40:36.520 | INFO | "huge" : "", | 20260508 02:40:36.520 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:36.520 | INFO | "malloc" : " 14 GB", | 20260508 02:40:36.520 | INFO | "pinned" : " 29 GB" | 20260508 02:40:36.520 | INFO | } | 20260508 02:40:36.520 | INFO | Perf report for the last 5.14067 seconds: | 20260508 02:40:36.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:36.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:40:36.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:40:36.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:36.520 | INFO | Uploaded 36GiB to GPU in 2.48469s (14.4887GiB/s) | 20260508 02:40:36.520 | INFO | Downloaded 34GiB to GPU in 2.52199s (13.4814GiB/s) | 20260508 02:40:41.662 | INFO | Analyzing chunk frames = [4593664,4594176) | 20260508 02:40:41.662 | INFO | MemInfo: { | 20260508 02:40:41.662 | INFO | "gpu" : " 17 GB", | 20260508 02:40:41.662 | INFO | "huge" : "", | 20260508 02:40:41.662 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:41.662 | INFO | "malloc" : " 14 GB", | 20260508 02:40:41.662 | INFO | "pinned" : " 29 GB" | 20260508 02:40:41.662 | INFO | } | 20260508 02:40:41.662 | INFO | Perf report for the last 5.14262 seconds: | 20260508 02:40:41.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:41.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.662 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:40:41.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.662 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:40:41.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:41.663 | INFO | Uploaded 36GiB to GPU in 2.41864s (14.8844GiB/s) | 20260508 02:40:41.663 | INFO | Downloaded 34GiB to GPU in 2.59076s (13.1236GiB/s) | 20260508 02:40:46.796 | INFO | Analyzing chunk frames = [4594176,4594688) | 20260508 02:40:46.796 | INFO | MemInfo: { | 20260508 02:40:46.796 | INFO | "gpu" : " 17 GB", | 20260508 02:40:46.796 | INFO | "huge" : "", | 20260508 02:40:46.796 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:46.796 | INFO | "malloc" : " 14 GB", | 20260508 02:40:46.797 | INFO | "pinned" : " 29 GB" | 20260508 02:40:46.797 | INFO | } | 20260508 02:40:46.797 | INFO | Perf report for the last 5.13421 seconds: | 20260508 02:40:46.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:46.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:40:46.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:40:46.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:46.797 | INFO | Uploaded 36GiB to GPU in 2.39911s (15.0056GiB/s) | 20260508 02:40:46.797 | INFO | Downloaded 34GiB to GPU in 2.42951s (13.9946GiB/s) | 20260508 02:40:51.935 | INFO | Analyzing chunk frames = [4594688,4595200) | 20260508 02:40:51.935 | INFO | MemInfo: { | 20260508 02:40:51.935 | INFO | "gpu" : " 17 GB", | 20260508 02:40:51.935 | INFO | "huge" : "", | 20260508 02:40:51.935 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:51.935 | INFO | "malloc" : " 14 GB", | 20260508 02:40:51.935 | INFO | "pinned" : " 29 GB" | 20260508 02:40:51.935 | INFO | } | 20260508 02:40:51.935 | INFO | Perf report for the last 5.13864 seconds: | 20260508 02:40:51.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:51.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.935 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:40:51.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.935 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 02:40:51.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:51.936 | INFO | Uploaded 36GiB to GPU in 2.46439s (14.6081GiB/s) | 20260508 02:40:51.936 | INFO | Downloaded 34GiB to GPU in 2.43993s (13.9348GiB/s) | 20260508 02:40:57.071 | INFO | Analyzing chunk frames = [4595200,4595712) | 20260508 02:40:57.071 | INFO | MemInfo: { | 20260508 02:40:57.071 | INFO | "gpu" : " 17 GB", | 20260508 02:40:57.071 | INFO | "huge" : "", | 20260508 02:40:57.071 | INFO | "huge_shared" : " 29 GB", | 20260508 02:40:57.071 | INFO | "malloc" : " 14 GB", | 20260508 02:40:57.071 | INFO | "pinned" : " 29 GB" | 20260508 02:40:57.071 | INFO | } | 20260508 02:40:57.071 | INFO | Perf report for the last 5.13596 seconds: | 20260508 02:40:57.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:40:57.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.071 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:40:57.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.071 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 02:40:57.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:40:57.072 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:40:57.072 | INFO | Total mem usage: 16.7 GiB | 20260508 02:40:57.072 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:40:57.072 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260508 02:40:57.072 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:40:57.072 | INFO | High Water Breakdown: | 20260508 02:40:57.072 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:40:57.072 | INFO | 39.8% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:40:57.072 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:40:57.072 | INFO | ------------------------------------------ | 20260508 02:40:57.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:40:57.072 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:40:57.072 | INFO | Total mem usage: 26.5 GiB | 20260508 02:40:57.072 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:40:57.072 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:40:57.072 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:40:57.072 | INFO | High Water Breakdown: | 20260508 02:40:57.072 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:40:57.072 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:40:57.072 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:40:57.072 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:40:57.072 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:40:57.072 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:40:57.072 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:40:57.072 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:40:57.072 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:40:57.072 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:40:57.072 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:40:57.072 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:40:57.072 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:40:57.072 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:40:57.072 | INFO | ------------------------------------------ | 20260508 02:40:57.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:40:57.072 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:40:57.072 | INFO | Total mem usage: 27.8 GiB | 20260508 02:40:57.072 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:40:57.072 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:40:57.072 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:40:57.072 | INFO | High Water Breakdown: | 20260508 02:40:57.072 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:40:57.072 | INFO | ------------------------------------------ | 20260508 02:40:57.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:40:57.072 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:40:57.072 | INFO | Total mem usage: 15.7 GiB | 20260508 02:40:57.072 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:40:57.072 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:40:57.072 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 02:40:57.072 | INFO | High Water Breakdown: | 20260508 02:40:57.072 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:40:57.072 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:40:57.072 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:40:57.072 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:40:57.072 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:40:57.072 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:40:57.072 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:40:57.072 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:40:57.072 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:40:57.072 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:40:57.072 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:40:57.072 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:40:57.072 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:40:57.072 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:40:57.072 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:40:57.072 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:40:57.072 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:40:57.072 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:40:57.072 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:40:57.072 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:40:57.072 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:40:57.072 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:40:57.072 | INFO | Uploaded 36GiB to GPU in 2.43653s (14.7751GiB/s) | 20260508 02:40:57.072 | INFO | Downloaded 34GiB to GPU in 2.42915s (13.9966GiB/s) | 20260508 02:40:57.072 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:40:57.072 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:40:57.072 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:40:57.072 | INFO | ------------------------------------------ | 20260508 02:40:57.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:40:57.072 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:40:57.072 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:40:57.072 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:40:57.073 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.40e+01 / 1.31e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 0.30%, AggregateStats -- 9.89e-02 / 1.02e-01, 1.24e-02 / 6.10e-03 | 20260508 02:40:57.073 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.52e+00, 5.91e-01 / 1.54e-01 | 20260508 02:40:57.073 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.09e-01 / 1.04e-01 | 20260508 02:40:57.073 | INFO | PerfReport: 16.03%, DME -- 4.20e+01 / 4.60e+01, 4.11e+00 / 1.56e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.72e+00 / 1.42e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.42e+00 / 2.73e+00, 4.25e-01 / 2.02e-01 | 20260508 02:40:57.073 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.25e+00, 2.67e-01 / 1.38e-01 | 20260508 02:40:57.073 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.30e-01 / 3.66e-01, 5.32e-02 / 2.26e-02 | 20260508 02:40:57.073 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:40:57.073 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:40:57.073 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:40:57.073 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 24.29%, Upload -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 2.12%, LockUpload -- 2.25e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 2.81%, LockDownload -- 2.99e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:40:57.073 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260508 02:40:57.169 | INFO | Starting a round of Baz IO | 20260508 02:41:00.618 | INFO | Populating RTMetrics report | 20260508 02:41:00.619 | INFO | RTMetrics report populated | 20260508 02:41:00.619 | INFO | Opening tmp RTMetrics json file | 20260508 02:41:00.648 | INFO | Serializing RTMetrics to json | 20260508 02:41:00.678 | INFO | Serialization done, swapping tmp file now... | 20260508 02:41:00.697 | INFO | RTMetrics done | 20260508 02:41:00.719 | INFO | Finished a round of Baz IO: Wrote 1.632 GiB in 3.549 seconds | 20260508 02:41:00.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.48% / 0.00% / 58.52% | 20260508 02:41:00.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.69501% / 3.97364e-06% / 5.96046e-05% | 20260508 02:41:00.719 | INFO | Baz IO overall: | 20260508 02:41:00.719 | INFO | BytesWritten : 2073107067875 | 20260508 02:41:00.719 | INFO | NumEvents : 1002055043323 | 20260508 02:41:02.218 | INFO | Analyzing chunk frames = [4595712,4596224) | 20260508 02:41:02.218 | INFO | MemInfo: { | 20260508 02:41:02.218 | INFO | "gpu" : " 17 GB", | 20260508 02:41:02.218 | INFO | "huge" : "", | 20260508 02:41:02.218 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:02.218 | INFO | "malloc" : " 14 GB", | 20260508 02:41:02.218 | INFO | "pinned" : " 29 GB" | 20260508 02:41:02.218 | INFO | } | 20260508 02:41:02.218 | INFO | Perf report for the last 5.14657 seconds: | 20260508 02:41:02.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:02.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:02.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:02.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:02.218 | INFO | ANALYSIS : 68.2%, 3.0, 106.6ms, 0 | 20260508 02:41:02.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:02.218 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260508 02:41:02.218 | INFO | PRE_HQ : 65.3%, 1.7, 56.0ms, 0 | 20260508 02:41:02.218 | INFO | BAZWRITER : 68.4%, 3.8, 132.8ms, 0 | 20260508 02:41:02.218 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 02:41:02.218 | INFO | Uploaded 36GiB to GPU in 2.59243s (13.8866GiB/s) | 20260508 02:41:02.218 | INFO | Downloaded 38GiB to GPU in 2.67191s (14.222GiB/s) | 20260508 02:41:07.359 | INFO | Analyzing chunk frames = [4596224,4596736) | 20260508 02:41:07.359 | INFO | MemInfo: { | 20260508 02:41:07.359 | INFO | "gpu" : " 17 GB", | 20260508 02:41:07.359 | INFO | "huge" : "", | 20260508 02:41:07.359 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:07.359 | INFO | "malloc" : " 14 GB", | 20260508 02:41:07.359 | INFO | "pinned" : " 29 GB" | 20260508 02:41:07.359 | INFO | } | 20260508 02:41:07.359 | INFO | Perf report for the last 5.14128 seconds: | 20260508 02:41:07.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:07.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:41:07.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:41:07.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:07.359 | INFO | Uploaded 36GiB to GPU in 2.53856s (14.1812GiB/s) | 20260508 02:41:07.360 | INFO | Downloaded 34GiB to GPU in 2.42376s (14.0278GiB/s) | 20260508 02:41:12.491 | INFO | Analyzing chunk frames = [4596736,4597248) | 20260508 02:41:12.491 | INFO | MemInfo: { | 20260508 02:41:12.491 | INFO | "gpu" : " 17 GB", | 20260508 02:41:12.491 | INFO | "huge" : "", | 20260508 02:41:12.491 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:12.492 | INFO | "malloc" : " 14 GB", | 20260508 02:41:12.492 | INFO | "pinned" : " 29 GB" | 20260508 02:41:12.492 | INFO | } | 20260508 02:41:12.492 | INFO | Perf report for the last 5.13244 seconds: | 20260508 02:41:12.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:12.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:41:12.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:41:12.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:12.492 | INFO | Uploaded 36GiB to GPU in 2.53961s (14.1754GiB/s) | 20260508 02:41:12.492 | INFO | Downloaded 34GiB to GPU in 2.44032s (13.9326GiB/s) | 20260508 02:41:17.626 | INFO | Analyzing chunk frames = [4597248,4597760) | 20260508 02:41:17.626 | INFO | MemInfo: { | 20260508 02:41:17.626 | INFO | "gpu" : " 17 GB", | 20260508 02:41:17.626 | INFO | "huge" : "", | 20260508 02:41:17.626 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:17.626 | INFO | "malloc" : " 14 GB", | 20260508 02:41:17.626 | INFO | "pinned" : " 29 GB" | 20260508 02:41:17.626 | INFO | } | 20260508 02:41:17.626 | INFO | Perf report for the last 5.13478 seconds: | 20260508 02:41:17.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:17.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.626 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 02:41:17.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.627 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:41:17.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:17.627 | INFO | Uploaded 36GiB to GPU in 2.52402s (14.2629GiB/s) | 20260508 02:41:17.627 | INFO | Downloaded 34GiB to GPU in 2.50753s (13.5592GiB/s) | 20260508 02:41:22.774 | INFO | Analyzing chunk frames = [4597760,4598272) | 20260508 02:41:22.774 | INFO | MemInfo: { | 20260508 02:41:22.774 | INFO | "gpu" : " 17 GB", | 20260508 02:41:22.774 | INFO | "huge" : "", | 20260508 02:41:22.774 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:22.774 | INFO | "malloc" : " 14 GB", | 20260508 02:41:22.774 | INFO | "pinned" : " 29 GB" | 20260508 02:41:22.774 | INFO | } | 20260508 02:41:22.774 | INFO | Perf report for the last 5.14743 seconds: | 20260508 02:41:22.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:22.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 02:41:22.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:41:22.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:22.774 | INFO | Uploaded 36GiB to GPU in 2.38238s (15.111GiB/s) | 20260508 02:41:22.774 | INFO | Downloaded 34GiB to GPU in 2.58093s (13.1736GiB/s) | 20260508 02:41:27.917 | INFO | Analyzing chunk frames = [4598272,4598784) | 20260508 02:41:27.917 | INFO | MemInfo: { | 20260508 02:41:27.917 | INFO | "gpu" : " 17 GB", | 20260508 02:41:27.917 | INFO | "huge" : "", | 20260508 02:41:27.917 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:27.917 | INFO | "malloc" : " 14 GB", | 20260508 02:41:27.917 | INFO | "pinned" : " 29 GB" | 20260508 02:41:27.917 | INFO | } | 20260508 02:41:27.917 | INFO | Perf report for the last 5.1431 seconds: | 20260508 02:41:27.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:27.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 02:41:27.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:41:27.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:27.917 | INFO | Uploaded 36GiB to GPU in 2.39219s (15.0489GiB/s) | 20260508 02:41:27.918 | INFO | Downloaded 34GiB to GPU in 2.41228s (14.0946GiB/s) | 20260508 02:41:33.054 | INFO | Analyzing chunk frames = [4598784,4599296) | 20260508 02:41:33.054 | INFO | MemInfo: { | 20260508 02:41:33.054 | INFO | "gpu" : " 17 GB", | 20260508 02:41:33.054 | INFO | "huge" : "", | 20260508 02:41:33.054 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:33.054 | INFO | "malloc" : " 14 GB", | 20260508 02:41:33.054 | INFO | "pinned" : " 29 GB" | 20260508 02:41:33.054 | INFO | } | 20260508 02:41:33.054 | INFO | Perf report for the last 5.13735 seconds: | 20260508 02:41:33.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:33.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.054 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:41:33.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.055 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 02:41:33.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:33.055 | INFO | Uploaded 36GiB to GPU in 2.41294s (14.9196GiB/s) | 20260508 02:41:33.055 | INFO | Downloaded 34GiB to GPU in 2.42532s (14.0188GiB/s) | 20260508 02:41:38.188 | INFO | Analyzing chunk frames = [4599296,4599808) | 20260508 02:41:38.188 | INFO | MemInfo: { | 20260508 02:41:38.188 | INFO | "gpu" : " 17 GB", | 20260508 02:41:38.188 | INFO | "huge" : "", | 20260508 02:41:38.189 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:38.189 | INFO | "malloc" : " 14 GB", | 20260508 02:41:38.189 | INFO | "pinned" : " 29 GB" | 20260508 02:41:38.189 | INFO | } | 20260508 02:41:38.189 | INFO | Perf report for the last 5.13438 seconds: | 20260508 02:41:38.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:38.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 02:41:38.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 02:41:38.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:38.189 | INFO | Uploaded 36GiB to GPU in 2.44023s (14.7527GiB/s) | 20260508 02:41:38.189 | INFO | Downloaded 34GiB to GPU in 2.43649s (13.9545GiB/s) | 20260508 02:41:38.302 | INFO | Starting a round of Baz IO | 20260508 02:41:41.734 | INFO | Populating RTMetrics report | 20260508 02:41:41.734 | INFO | RTMetrics report populated | 20260508 02:41:41.734 | INFO | Opening tmp RTMetrics json file | 20260508 02:41:41.751 | INFO | Serializing RTMetrics to json | 20260508 02:41:41.759 | INFO | Serialization done, swapping tmp file now... | 20260508 02:41:41.759 | INFO | RTMetrics done | 20260508 02:41:41.826 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.523 seconds | 20260508 02:41:41.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.47% / 0.00% / 58.52% | 20260508 02:41:41.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.695868% / 3.97364e-06% / 6.75519e-05% | 20260508 02:41:41.826 | INFO | Baz IO overall: | 20260508 02:41:41.826 | INFO | BytesWritten : 2074861038210 | 20260508 02:41:41.826 | INFO | NumEvents : 1002895946846 | 20260508 02:41:43.333 | INFO | Analyzing chunk frames = [4599808,4600320) | 20260508 02:41:43.333 | INFO | MemInfo: { | 20260508 02:41:43.333 | INFO | "gpu" : " 17 GB", | 20260508 02:41:43.333 | INFO | "huge" : "", | 20260508 02:41:43.333 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:43.333 | INFO | "malloc" : " 14 GB", | 20260508 02:41:43.333 | INFO | "pinned" : " 29 GB" | 20260508 02:41:43.333 | INFO | } | 20260508 02:41:43.333 | INFO | Perf report for the last 5.14428 seconds: | 20260508 02:41:43.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:43.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:43.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:43.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:43.333 | INFO | ANALYSIS : 68.3%, 2.9, 104.3ms, 0 | 20260508 02:41:43.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:43.333 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 02:41:43.333 | INFO | PRE_HQ : 67.0%, 1.8, 61.9ms, 0 | 20260508 02:41:43.333 | INFO | BAZWRITER : 68.1%, 3.7, 131.1ms, 0 | 20260508 02:41:43.333 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260508 02:41:43.334 | INFO | Uploaded 36GiB to GPU in 2.55519s (14.0889GiB/s) | 20260508 02:41:43.334 | INFO | Downloaded 38GiB to GPU in 2.69534s (14.0984GiB/s) | 20260508 02:41:48.468 | INFO | Analyzing chunk frames = [4600320,4600832) | 20260508 02:41:48.468 | INFO | MemInfo: { | 20260508 02:41:48.468 | INFO | "gpu" : " 17 GB", | 20260508 02:41:48.468 | INFO | "huge" : "", | 20260508 02:41:48.468 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:48.468 | INFO | "malloc" : " 14 GB", | 20260508 02:41:48.468 | INFO | "pinned" : " 29 GB" | 20260508 02:41:48.468 | INFO | } | 20260508 02:41:48.468 | INFO | Perf report for the last 5.13532 seconds: | 20260508 02:41:48.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:48.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.468 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:41:48.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.468 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:41:48.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:48.469 | INFO | Uploaded 36GiB to GPU in 2.56784s (14.0196GiB/s) | 20260508 02:41:48.469 | INFO | Downloaded 34GiB to GPU in 2.42846s (14.0006GiB/s) | 20260508 02:41:53.611 | INFO | Analyzing chunk frames = [4600832,4601344) | 20260508 02:41:53.611 | INFO | MemInfo: { | 20260508 02:41:53.611 | INFO | "gpu" : " 17 GB", | 20260508 02:41:53.611 | INFO | "huge" : "", | 20260508 02:41:53.611 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:53.611 | INFO | "malloc" : " 14 GB", | 20260508 02:41:53.611 | INFO | "pinned" : " 29 GB" | 20260508 02:41:53.611 | INFO | } | 20260508 02:41:53.611 | INFO | Perf report for the last 5.14248 seconds: | 20260508 02:41:53.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:53.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:41:53.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:41:53.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:53.611 | INFO | Uploaded 36GiB to GPU in 2.55268s (14.1028GiB/s) | 20260508 02:41:53.611 | INFO | Downloaded 34GiB to GPU in 2.50679s (13.5631GiB/s) | 20260508 02:41:58.744 | INFO | Analyzing chunk frames = [4601344,4601856) | 20260508 02:41:58.744 | INFO | MemInfo: { | 20260508 02:41:58.744 | INFO | "gpu" : " 17 GB", | 20260508 02:41:58.745 | INFO | "huge" : "", | 20260508 02:41:58.745 | INFO | "huge_shared" : " 29 GB", | 20260508 02:41:58.745 | INFO | "malloc" : " 14 GB", | 20260508 02:41:58.745 | INFO | "pinned" : " 29 GB" | 20260508 02:41:58.745 | INFO | } | 20260508 02:41:58.745 | INFO | Perf report for the last 5.13389 seconds: | 20260508 02:41:58.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:41:58.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 02:41:58.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:41:58.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:41:58.745 | INFO | Uploaded 36GiB to GPU in 2.52923s (14.2336GiB/s) | 20260508 02:41:58.745 | INFO | Downloaded 34GiB to GPU in 2.5031s (13.5831GiB/s) | 20260508 02:42:03.881 | INFO | Analyzing chunk frames = [4601856,4602368) | 20260508 02:42:03.881 | INFO | MemInfo: { | 20260508 02:42:03.881 | INFO | "gpu" : " 17 GB", | 20260508 02:42:03.881 | INFO | "huge" : "", | 20260508 02:42:03.881 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:03.881 | INFO | "malloc" : " 14 GB", | 20260508 02:42:03.881 | INFO | "pinned" : " 29 GB" | 20260508 02:42:03.881 | INFO | } | 20260508 02:42:03.881 | INFO | Perf report for the last 5.13667 seconds: | 20260508 02:42:03.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:03.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.881 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 02:42:03.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.882 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:42:03.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:03.882 | INFO | Uploaded 36GiB to GPU in 2.37733s (15.143GiB/s) | 20260508 02:42:03.882 | INFO | Downloaded 34GiB to GPU in 2.55069s (13.3297GiB/s) | 20260508 02:42:09.028 | INFO | Analyzing chunk frames = [4602368,4602880) | 20260508 02:42:09.028 | INFO | MemInfo: { | 20260508 02:42:09.028 | INFO | "gpu" : " 17 GB", | 20260508 02:42:09.028 | INFO | "huge" : "", | 20260508 02:42:09.028 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:09.028 | INFO | "malloc" : " 14 GB", | 20260508 02:42:09.028 | INFO | "pinned" : " 29 GB" | 20260508 02:42:09.028 | INFO | } | 20260508 02:42:09.028 | INFO | Perf report for the last 5.14691 seconds: | 20260508 02:42:09.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:09.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.028 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:42:09.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.028 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 02:42:09.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:09.029 | INFO | Uploaded 36GiB to GPU in 2.39996s (15.0003GiB/s) | 20260508 02:42:09.029 | INFO | Downloaded 34GiB to GPU in 2.41984s (14.0505GiB/s) | 20260508 02:42:14.165 | INFO | Analyzing chunk frames = [4602880,4603392) | 20260508 02:42:14.165 | INFO | MemInfo: { | 20260508 02:42:14.165 | INFO | "gpu" : " 17 GB", | 20260508 02:42:14.165 | INFO | "huge" : "", | 20260508 02:42:14.165 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:14.165 | INFO | "malloc" : " 14 GB", | 20260508 02:42:14.165 | INFO | "pinned" : " 29 GB" | 20260508 02:42:14.165 | INFO | } | 20260508 02:42:14.166 | INFO | Perf report for the last 5.13722 seconds: | 20260508 02:42:14.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:14.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:42:14.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 02:42:14.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:14.166 | INFO | Uploaded 36GiB to GPU in 2.4161s (14.9GiB/s) | 20260508 02:42:14.167 | INFO | Downloaded 34GiB to GPU in 2.41828s (14.0596GiB/s) | 20260508 02:42:19.307 | INFO | Analyzing chunk frames = [4603392,4603904) | 20260508 02:42:19.307 | INFO | MemInfo: { | 20260508 02:42:19.307 | INFO | "gpu" : " 17 GB", | 20260508 02:42:19.307 | INFO | "huge" : "", | 20260508 02:42:19.307 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:19.307 | INFO | "malloc" : " 14 GB", | 20260508 02:42:19.307 | INFO | "pinned" : " 29 GB" | 20260508 02:42:19.307 | INFO | } | 20260508 02:42:19.307 | INFO | Perf report for the last 5.14138 seconds: | 20260508 02:42:19.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:19.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:42:19.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 02:42:19.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:19.307 | INFO | Uploaded 36GiB to GPU in 2.56398s (14.0406GiB/s) | 20260508 02:42:19.307 | INFO | Downloaded 34GiB to GPU in 2.46666s (13.7838GiB/s) | 20260508 02:42:19.308 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:42:19.308 | INFO | Total mem usage: 16.7 GiB | 20260508 02:42:19.308 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:42:19.308 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.1%) | 20260508 02:42:19.308 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:42:19.308 | INFO | High Water Breakdown: | 20260508 02:42:19.308 | INFO | 59.7% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:42:19.308 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:42:19.308 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:42:19.308 | INFO | ------------------------------------------ | 20260508 02:42:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:42:19.308 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:42:19.308 | INFO | Total mem usage: 26.5 GiB | 20260508 02:42:19.308 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:42:19.308 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 02:42:19.308 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:42:19.308 | INFO | High Water Breakdown: | 20260508 02:42:19.308 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:42:19.308 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:42:19.308 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:42:19.308 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:42:19.308 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:42:19.308 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:42:19.308 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:42:19.308 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:42:19.308 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:42:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:42:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:42:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:42:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:42:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:42:19.308 | INFO | ------------------------------------------ | 20260508 02:42:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:42:19.308 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:42:19.308 | INFO | Total mem usage: 27.8 GiB | 20260508 02:42:19.308 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:42:19.308 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:42:19.308 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:42:19.308 | INFO | High Water Breakdown: | 20260508 02:42:19.308 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:42:19.308 | INFO | ------------------------------------------ | 20260508 02:42:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:42:19.308 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:42:19.308 | INFO | Total mem usage: 15.7 GiB | 20260508 02:42:19.308 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:42:19.308 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:42:19.308 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:42:19.308 | INFO | High Water Breakdown: | 20260508 02:42:19.308 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:42:19.308 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:42:19.308 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:42:19.309 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:42:19.309 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:42:19.309 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:42:19.309 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:42:19.309 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:42:19.309 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:42:19.309 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:42:19.309 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:42:19.309 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:42:19.309 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:42:19.309 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:42:19.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:42:19.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:42:19.309 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:42:19.309 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:42:19.309 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:42:19.309 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:42:19.309 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:42:19.309 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:42:19.309 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:42:19.309 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:42:19.309 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:42:19.309 | INFO | ------------------------------------------ | 20260508 02:42:19.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:42:19.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:42:19.309 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:42:19.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:42:19.309 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.43e+01 / 1.43e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.01e-01, 1.29e-02 / 5.19e-03 | 20260508 02:42:19.309 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 5.86e-01 / 2.05e-01 | 20260508 02:42:19.309 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.18e-01 | 20260508 02:42:19.309 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.61e+01, 3.91e+00 / 1.85e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 56.49%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.69e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.42e+00 / 2.73e+00, 4.22e-01 / 2.10e-01 | 20260508 02:42:19.309 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.28e+00, 2.68e-01 / 1.39e-01 | 20260508 02:42:19.309 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.30e-01 / 3.66e-01, 5.23e-02 / 2.50e-02 | 20260508 02:42:19.309 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:42:19.309 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:42:19.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:42:19.309 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 2.12%, LockUpload -- 2.26e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 2.75%, LockDownload -- 2.93e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 02:42:19.309 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260508 02:42:19.404 | INFO | Starting a round of Baz IO | 20260508 02:42:22.841 | INFO | Populating RTMetrics report | 20260508 02:42:22.841 | INFO | RTMetrics report populated | 20260508 02:42:22.841 | INFO | Opening tmp RTMetrics json file | 20260508 02:42:22.869 | INFO | Serializing RTMetrics to json | 20260508 02:42:22.871 | INFO | Serialization done, swapping tmp file now... | 20260508 02:42:22.871 | INFO | RTMetrics done | 20260508 02:42:22.938 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.533 seconds | 20260508 02:42:22.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.47% / 0.00% / 58.53% | 20260508 02:42:22.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.696258% / 3.97364e-06% / 6.75519e-05% | 20260508 02:42:22.938 | INFO | Baz IO overall: | 20260508 02:42:22.938 | INFO | BytesWritten : 2076615264886 | 20260508 02:42:22.938 | INFO | NumEvents : 1003737326378 | 20260508 02:42:24.444 | INFO | Analyzing chunk frames = [4603904,4604416) | 20260508 02:42:24.444 | INFO | MemInfo: { | 20260508 02:42:24.444 | INFO | "gpu" : " 17 GB", | 20260508 02:42:24.444 | INFO | "huge" : "", | 20260508 02:42:24.444 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:24.444 | INFO | "malloc" : " 14 GB", | 20260508 02:42:24.444 | INFO | "pinned" : " 29 GB" | 20260508 02:42:24.444 | INFO | } | 20260508 02:42:24.444 | INFO | Perf report for the last 5.13723 seconds: | 20260508 02:42:24.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:24.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:24.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:24.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:24.444 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 02:42:24.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:24.444 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 02:42:24.444 | INFO | PRE_HQ : 65.8%, 1.6, 55.9ms, 0 | 20260508 02:42:24.444 | INFO | BAZWRITER : 68.4%, 3.6, 127.7ms, 0 | 20260508 02:42:24.444 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 02:42:24.445 | INFO | Uploaded 36GiB to GPU in 2.58597s (13.9213GiB/s) | 20260508 02:42:24.445 | INFO | Downloaded 38GiB to GPU in 2.69377s (14.1066GiB/s) | 20260508 02:42:29.584 | INFO | Analyzing chunk frames = [4604416,4604928) | 20260508 02:42:29.584 | INFO | MemInfo: { | 20260508 02:42:29.584 | INFO | "gpu" : " 17 GB", | 20260508 02:42:29.584 | INFO | "huge" : "", | 20260508 02:42:29.584 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:29.584 | INFO | "malloc" : " 14 GB", | 20260508 02:42:29.584 | INFO | "pinned" : " 29 GB" | 20260508 02:42:29.584 | INFO | } | 20260508 02:42:29.584 | INFO | Perf report for the last 5.1396 seconds: | 20260508 02:42:29.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:29.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:42:29.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:42:29.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:29.584 | INFO | Uploaded 36GiB to GPU in 2.55545s (14.0875GiB/s) | 20260508 02:42:29.585 | INFO | Downloaded 34GiB to GPU in 2.42182s (14.039GiB/s) | 20260508 02:42:34.722 | INFO | Analyzing chunk frames = [4604928,4605440) | 20260508 02:42:34.722 | INFO | MemInfo: { | 20260508 02:42:34.722 | INFO | "gpu" : " 17 GB", | 20260508 02:42:34.722 | INFO | "huge" : "", | 20260508 02:42:34.722 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:34.722 | INFO | "malloc" : " 14 GB", | 20260508 02:42:34.722 | INFO | "pinned" : " 29 GB" | 20260508 02:42:34.722 | INFO | } | 20260508 02:42:34.722 | INFO | Perf report for the last 5.13848 seconds: | 20260508 02:42:34.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:34.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.722 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 02:42:34.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.722 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:42:34.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:34.723 | INFO | Uploaded 36GiB to GPU in 2.53135s (14.2217GiB/s) | 20260508 02:42:34.723 | INFO | Downloaded 34GiB to GPU in 2.50356s (13.5807GiB/s) | 20260508 02:42:39.864 | INFO | Analyzing chunk frames = [4605440,4605952) | 20260508 02:42:39.864 | INFO | MemInfo: { | 20260508 02:42:39.864 | INFO | "gpu" : " 17 GB", | 20260508 02:42:39.864 | INFO | "huge" : "", | 20260508 02:42:39.864 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:39.864 | INFO | "malloc" : " 14 GB", | 20260508 02:42:39.864 | INFO | "pinned" : " 29 GB" | 20260508 02:42:39.864 | INFO | } | 20260508 02:42:39.864 | INFO | Perf report for the last 5.1415 seconds: | 20260508 02:42:39.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:39.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:42:39.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:42:39.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:39.864 | INFO | Uploaded 36GiB to GPU in 2.53528s (14.1996GiB/s) | 20260508 02:42:39.864 | INFO | Downloaded 34GiB to GPU in 2.50299s (13.5837GiB/s) | 20260508 02:42:45.002 | INFO | Analyzing chunk frames = [4605952,4606464) | 20260508 02:42:45.002 | INFO | MemInfo: { | 20260508 02:42:45.002 | INFO | "gpu" : " 17 GB", | 20260508 02:42:45.002 | INFO | "huge" : "", | 20260508 02:42:45.002 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:45.002 | INFO | "malloc" : " 14 GB", | 20260508 02:42:45.002 | INFO | "pinned" : " 29 GB" | 20260508 02:42:45.002 | INFO | } | 20260508 02:42:45.003 | INFO | Perf report for the last 5.13869 seconds: | 20260508 02:42:45.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:45.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 02:42:45.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:42:45.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:45.003 | INFO | Uploaded 36GiB to GPU in 2.41079s (14.9329GiB/s) | 20260508 02:42:45.003 | INFO | Downloaded 34GiB to GPU in 2.53548s (13.4097GiB/s) | 20260508 02:42:50.141 | INFO | Analyzing chunk frames = [4606464,4606976) | 20260508 02:42:50.141 | INFO | MemInfo: { | 20260508 02:42:50.141 | INFO | "gpu" : " 17 GB", | 20260508 02:42:50.141 | INFO | "huge" : "", | 20260508 02:42:50.141 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:50.141 | INFO | "malloc" : " 14 GB", | 20260508 02:42:50.141 | INFO | "pinned" : " 29 GB" | 20260508 02:42:50.141 | INFO | } | 20260508 02:42:50.141 | INFO | Perf report for the last 5.13872 seconds: | 20260508 02:42:50.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:50.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.141 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 02:42:50.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.142 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:42:50.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:50.142 | INFO | Uploaded 36GiB to GPU in 2.38197s (15.1136GiB/s) | 20260508 02:42:50.142 | INFO | Downloaded 34GiB to GPU in 2.44087s (13.9295GiB/s) | 20260508 02:42:55.281 | INFO | Analyzing chunk frames = [4606976,4607488) | 20260508 02:42:55.281 | INFO | MemInfo: { | 20260508 02:42:55.281 | INFO | "gpu" : " 17 GB", | 20260508 02:42:55.281 | INFO | "huge" : "", | 20260508 02:42:55.281 | INFO | "huge_shared" : " 29 GB", | 20260508 02:42:55.281 | INFO | "malloc" : " 14 GB", | 20260508 02:42:55.281 | INFO | "pinned" : " 29 GB" | 20260508 02:42:55.281 | INFO | } | 20260508 02:42:55.281 | INFO | Perf report for the last 5.13982 seconds: | 20260508 02:42:55.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:42:55.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 02:42:55.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:42:55.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:42:55.281 | INFO | Uploaded 36GiB to GPU in 2.3908s (15.0577GiB/s) | 20260508 02:42:55.281 | INFO | Downloaded 34GiB to GPU in 2.43969s (13.9362GiB/s) | 20260508 02:43:00.422 | INFO | Analyzing chunk frames = [4607488,4608000) | 20260508 02:43:00.422 | INFO | MemInfo: { | 20260508 02:43:00.422 | INFO | "gpu" : " 17 GB", | 20260508 02:43:00.422 | INFO | "huge" : "", | 20260508 02:43:00.422 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:00.422 | INFO | "malloc" : " 14 GB", | 20260508 02:43:00.422 | INFO | "pinned" : " 29 GB" | 20260508 02:43:00.422 | INFO | } | 20260508 02:43:00.422 | INFO | Perf report for the last 5.14091 seconds: | 20260508 02:43:00.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:00.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.422 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:43:00.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.422 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:43:00.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:00.423 | INFO | Uploaded 36GiB to GPU in 2.52522s (14.2562GiB/s) | 20260508 02:43:00.423 | INFO | Downloaded 34GiB to GPU in 2.46127s (13.814GiB/s) | 20260508 02:43:00.520 | INFO | Starting a round of Baz IO | 20260508 02:43:03.969 | INFO | Populating RTMetrics report | 20260508 02:43:03.969 | INFO | RTMetrics report populated | 20260508 02:43:03.969 | INFO | Opening tmp RTMetrics json file | 20260508 02:43:03.983 | INFO | Serializing RTMetrics to json | 20260508 02:43:03.992 | INFO | Serialization done, swapping tmp file now... | 20260508 02:43:03.993 | INFO | RTMetrics done | 20260508 02:43:04.063 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.543 seconds | 20260508 02:43:04.063 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.47% / 0.00% / 58.53% | 20260508 02:43:04.063 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.691326% / 7.94729e-06% / 5.96046e-05% | 20260508 02:43:04.063 | INFO | Baz IO overall: | 20260508 02:43:04.063 | INFO | BytesWritten : 2078365963159 | 20260508 02:43:04.063 | INFO | NumEvents : 1004576328442 | 20260508 02:43:05.557 | INFO | Analyzing chunk frames = [4608000,4608512) | 20260508 02:43:05.557 | INFO | MemInfo: { | 20260508 02:43:05.557 | INFO | "gpu" : " 17 GB", | 20260508 02:43:05.557 | INFO | "huge" : "", | 20260508 02:43:05.557 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:05.557 | INFO | "malloc" : " 14 GB", | 20260508 02:43:05.558 | INFO | "pinned" : " 29 GB" | 20260508 02:43:05.558 | INFO | } | 20260508 02:43:05.558 | INFO | Perf report for the last 5.13551 seconds: | 20260508 02:43:05.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:05.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:05.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:05.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:05.558 | INFO | ANALYSIS : 68.4%, 3.0, 106.5ms, 0 | 20260508 02:43:05.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:05.558 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 02:43:05.558 | INFO | PRE_HQ : 66.0%, 1.6, 55.9ms, 0 | 20260508 02:43:05.558 | INFO | BAZWRITER : 68.4%, 3.5, 123.9ms, 0 | 20260508 02:43:05.558 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:43:05.558 | INFO | Uploaded 36GiB to GPU in 2.52599s (14.2518GiB/s) | 20260508 02:43:05.558 | INFO | Downloaded 38GiB to GPU in 2.69077s (14.1224GiB/s) | 20260508 02:43:10.694 | INFO | Analyzing chunk frames = [4608512,4609024) | 20260508 02:43:10.694 | INFO | MemInfo: { | 20260508 02:43:10.694 | INFO | "gpu" : " 17 GB", | 20260508 02:43:10.694 | INFO | "huge" : "", | 20260508 02:43:10.694 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:10.694 | INFO | "malloc" : " 14 GB", | 20260508 02:43:10.694 | INFO | "pinned" : " 29 GB" | 20260508 02:43:10.694 | INFO | } | 20260508 02:43:10.694 | INFO | Perf report for the last 5.1367 seconds: | 20260508 02:43:10.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:10.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.694 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:43:10.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.694 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:43:10.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:10.695 | INFO | Uploaded 36GiB to GPU in 2.55151s (14.1093GiB/s) | 20260508 02:43:10.695 | INFO | Downloaded 34GiB to GPU in 2.42595s (14.0151GiB/s) | 20260508 02:43:15.838 | INFO | Analyzing chunk frames = [4609024,4609536) | 20260508 02:43:15.838 | INFO | MemInfo: { | 20260508 02:43:15.838 | INFO | "gpu" : " 17 GB", | 20260508 02:43:15.838 | INFO | "huge" : "", | 20260508 02:43:15.838 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:15.838 | INFO | "malloc" : " 14 GB", | 20260508 02:43:15.838 | INFO | "pinned" : " 29 GB" | 20260508 02:43:15.838 | INFO | } | 20260508 02:43:15.839 | INFO | Perf report for the last 5.14422 seconds: | 20260508 02:43:15.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:15.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 02:43:15.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:43:15.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:15.839 | INFO | Uploaded 36GiB to GPU in 2.53052s (14.2263GiB/s) | 20260508 02:43:15.839 | INFO | Downloaded 34GiB to GPU in 2.49941s (13.6032GiB/s) | 20260508 02:43:20.985 | INFO | Analyzing chunk frames = [4609536,4610048) | 20260508 02:43:20.985 | INFO | MemInfo: { | 20260508 02:43:20.985 | INFO | "gpu" : " 17 GB", | 20260508 02:43:20.985 | INFO | "huge" : "", | 20260508 02:43:20.985 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:20.986 | INFO | "malloc" : " 14 GB", | 20260508 02:43:20.986 | INFO | "pinned" : " 29 GB" | 20260508 02:43:20.986 | INFO | } | 20260508 02:43:20.986 | INFO | Perf report for the last 5.14707 seconds: | 20260508 02:43:20.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:20.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 02:43:20.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:43:20.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:20.986 | INFO | Uploaded 36GiB to GPU in 2.5136s (14.3221GiB/s) | 20260508 02:43:20.986 | INFO | Downloaded 34GiB to GPU in 2.50212s (13.5885GiB/s) | 20260508 02:43:26.123 | INFO | Analyzing chunk frames = [4610048,4610560) | 20260508 02:43:26.123 | INFO | MemInfo: { | 20260508 02:43:26.123 | INFO | "gpu" : " 17 GB", | 20260508 02:43:26.123 | INFO | "huge" : "", | 20260508 02:43:26.123 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:26.123 | INFO | "malloc" : " 14 GB", | 20260508 02:43:26.123 | INFO | "pinned" : " 29 GB" | 20260508 02:43:26.123 | INFO | } | 20260508 02:43:26.123 | INFO | Perf report for the last 5.13757 seconds: | 20260508 02:43:26.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:26.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 02:43:26.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 02:43:26.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:26.123 | INFO | Uploaded 36GiB to GPU in 2.43704s (14.772GiB/s) | 20260508 02:43:26.123 | INFO | Downloaded 34GiB to GPU in 2.57801s (13.1885GiB/s) | 20260508 02:43:31.256 | INFO | Analyzing chunk frames = [4610560,4611072) | 20260508 02:43:31.256 | INFO | MemInfo: { | 20260508 02:43:31.256 | INFO | "gpu" : " 17 GB", | 20260508 02:43:31.256 | INFO | "huge" : "", | 20260508 02:43:31.256 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:31.256 | INFO | "malloc" : " 14 GB", | 20260508 02:43:31.256 | INFO | "pinned" : " 29 GB" | 20260508 02:43:31.256 | INFO | } | 20260508 02:43:31.256 | INFO | Perf report for the last 5.133 seconds: | 20260508 02:43:31.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:31.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 02:43:31.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:43:31.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:31.256 | INFO | Uploaded 36GiB to GPU in 2.44457s (14.7265GiB/s) | 20260508 02:43:31.257 | INFO | Downloaded 34GiB to GPU in 2.40938s (14.1115GiB/s) | 20260508 02:43:36.393 | INFO | Analyzing chunk frames = [4611072,4611584) | 20260508 02:43:36.393 | INFO | MemInfo: { | 20260508 02:43:36.393 | INFO | "gpu" : " 17 GB", | 20260508 02:43:36.393 | INFO | "huge" : "", | 20260508 02:43:36.393 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:36.393 | INFO | "malloc" : " 14 GB", | 20260508 02:43:36.393 | INFO | "pinned" : " 29 GB" | 20260508 02:43:36.393 | INFO | } | 20260508 02:43:36.393 | INFO | Perf report for the last 5.13688 seconds: | 20260508 02:43:36.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:36.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:43:36.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:43:36.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:36.393 | INFO | Uploaded 36GiB to GPU in 2.46293s (14.6167GiB/s) | 20260508 02:43:36.394 | INFO | Downloaded 34GiB to GPU in 2.41615s (14.072GiB/s) | 20260508 02:43:41.534 | INFO | Analyzing chunk frames = [4611584,4612096) | 20260508 02:43:41.534 | INFO | MemInfo: { | 20260508 02:43:41.534 | INFO | "gpu" : " 17 GB", | 20260508 02:43:41.534 | INFO | "huge" : "", | 20260508 02:43:41.535 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:41.535 | INFO | "malloc" : " 14 GB", | 20260508 02:43:41.535 | INFO | "pinned" : " 29 GB" | 20260508 02:43:41.535 | INFO | } | 20260508 02:43:41.535 | INFO | Perf report for the last 5.14161 seconds: | 20260508 02:43:41.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:41.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:43:41.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:43:41.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:41.535 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:43:41.535 | INFO | Total mem usage: 16.7 GiB | 20260508 02:43:41.535 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:43:41.535 | INFO | Uploaded 36GiB to GPU in 2.57328s (13.9899GiB/s) | 20260508 02:43:41.535 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -13.9%) | 20260508 02:43:41.535 | INFO | Downloaded 34GiB to GPU in 2.44894s (13.8836GiB/s) | 20260508 02:43:41.535 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:43:41.535 | INFO | High Water Breakdown: | 20260508 02:43:41.535 | INFO | 59.9% : 8.78 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:43:41.535 | INFO | 39.9% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:43:41.535 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:43:41.535 | INFO | ------------------------------------------ | 20260508 02:43:41.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:43:41.535 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:43:41.536 | INFO | Total mem usage: 26.5 GiB | 20260508 02:43:41.536 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:43:41.536 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:43:41.536 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:43:41.536 | INFO | High Water Breakdown: | 20260508 02:43:41.536 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:43:41.536 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:43:41.536 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:43:41.536 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:43:41.536 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:43:41.536 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:43:41.536 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:43:41.536 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:43:41.536 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:43:41.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:43:41.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:43:41.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:43:41.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:43:41.536 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:43:41.536 | INFO | ------------------------------------------ | 20260508 02:43:41.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:43:41.536 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:43:41.536 | INFO | Total mem usage: 27.8 GiB | 20260508 02:43:41.536 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:43:41.536 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:43:41.536 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:43:41.536 | INFO | High Water Breakdown: | 20260508 02:43:41.536 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:43:41.536 | INFO | ------------------------------------------ | 20260508 02:43:41.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:43:41.536 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:43:41.536 | INFO | Total mem usage: 15.7 GiB | 20260508 02:43:41.536 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:43:41.536 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:43:41.536 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:43:41.536 | INFO | High Water Breakdown: | 20260508 02:43:41.536 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:43:41.536 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:43:41.536 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:43:41.536 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:43:41.536 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:43:41.536 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:43:41.536 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:43:41.536 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:43:41.536 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:43:41.536 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:43:41.536 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:43:41.536 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:43:41.536 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:43:41.536 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:43:41.536 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:43:41.536 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:43:41.536 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:43:41.536 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:43:41.536 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:43:41.536 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:43:41.536 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:43:41.536 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:43:41.536 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:43:41.536 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:43:41.536 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:43:41.536 | INFO | ------------------------------------------ | 20260508 02:43:41.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:43:41.536 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:43:41.536 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:43:41.536 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:43:41.536 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.63e+01, 1.40e+01 / 1.84e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.01e-01, 1.11e-02 / 5.42e-03 | 20260508 02:43:41.536 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.48e+00, 5.85e-01 / 1.62e-01 | 20260508 02:43:41.536 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.07e-01 / 1.21e-01 | 20260508 02:43:41.536 | INFO | PerfReport: 16.09%, DME -- 4.21e+01 / 4.64e+01, 4.13e+00 / 2.06e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.64e+00 / 1.51e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.70e+00, 4.26e-01 / 2.02e-01 | 20260508 02:43:41.536 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.23e+00, 2.66e-01 / 1.07e-01 | 20260508 02:43:41.536 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.60e-01, 5.17e-02 / 2.13e-02 | 20260508 02:43:41.536 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:43:41.536 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:43:41.536 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:43:41.536 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 30.68%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 2.17%, LockUpload -- 2.32e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 2.79%, LockDownload -- 2.98e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 15.53%, LockCompute -- 1.66e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 02:43:41.536 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260508 02:43:41.632 | INFO | Starting a round of Baz IO | 20260508 02:43:45.070 | INFO | Populating RTMetrics report | 20260508 02:43:45.071 | INFO | RTMetrics report populated | 20260508 02:43:45.071 | INFO | Opening tmp RTMetrics json file | 20260508 02:43:45.093 | INFO | Serializing RTMetrics to json | 20260508 02:43:45.095 | INFO | Serialization done, swapping tmp file now... | 20260508 02:43:45.096 | INFO | RTMetrics done | 20260508 02:43:45.176 | INFO | Finished a round of Baz IO: Wrote 1.631 GiB in 3.543 seconds | 20260508 02:43:45.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.46% / 0.00% / 58.53% | 20260508 02:43:45.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.691295% / 7.94729e-06% / 5.16574e-05% | 20260508 02:43:45.176 | INFO | Baz IO overall: | 20260508 02:43:45.176 | INFO | BytesWritten : 2080117425102 | 20260508 02:43:45.176 | INFO | NumEvents : 1005415716388 | 20260508 02:43:46.675 | INFO | Analyzing chunk frames = [4612096,4612608) | 20260508 02:43:46.675 | INFO | MemInfo: { | 20260508 02:43:46.675 | INFO | "gpu" : " 17 GB", | 20260508 02:43:46.675 | INFO | "huge" : "", | 20260508 02:43:46.675 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:46.675 | INFO | "malloc" : " 14 GB", | 20260508 02:43:46.675 | INFO | "pinned" : " 29 GB" | 20260508 02:43:46.675 | INFO | } | 20260508 02:43:46.675 | INFO | Perf report for the last 5.13992 seconds: | 20260508 02:43:46.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:46.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:46.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:46.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:46.675 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 02:43:46.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:46.675 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 02:43:46.675 | INFO | PRE_HQ : 65.7%, 1.6, 56.1ms, 0 | 20260508 02:43:46.675 | INFO | BAZWRITER : 68.6%, 4.3, 150.3ms, 0 | 20260508 02:43:46.675 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 02:43:46.675 | INFO | Uploaded 36GiB to GPU in 2.59994s (13.8465GiB/s) | 20260508 02:43:46.675 | INFO | Downloaded 38GiB to GPU in 2.69113s (14.1204GiB/s) | 20260508 02:43:51.814 | INFO | Analyzing chunk frames = [4612608,4613120) | 20260508 02:43:51.814 | INFO | MemInfo: { | 20260508 02:43:51.814 | INFO | "gpu" : " 17 GB", | 20260508 02:43:51.814 | INFO | "huge" : "", | 20260508 02:43:51.814 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:51.814 | INFO | "malloc" : " 14 GB", | 20260508 02:43:51.814 | INFO | "pinned" : " 29 GB" | 20260508 02:43:51.814 | INFO | } | 20260508 02:43:51.814 | INFO | Perf report for the last 5.13969 seconds: | 20260508 02:43:51.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:51.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.814 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 02:43:51.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.814 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:43:51.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:51.815 | INFO | Uploaded 36GiB to GPU in 2.61279s (13.7784GiB/s) | 20260508 02:43:51.815 | INFO | Downloaded 34GiB to GPU in 2.49329s (13.6366GiB/s) | 20260508 02:43:56.951 | INFO | Analyzing chunk frames = [4613120,4613632) | 20260508 02:43:56.951 | INFO | MemInfo: { | 20260508 02:43:56.951 | INFO | "gpu" : " 17 GB", | 20260508 02:43:56.951 | INFO | "huge" : "", | 20260508 02:43:56.951 | INFO | "huge_shared" : " 29 GB", | 20260508 02:43:56.951 | INFO | "malloc" : " 14 GB", | 20260508 02:43:56.951 | INFO | "pinned" : " 29 GB" | 20260508 02:43:56.951 | INFO | } | 20260508 02:43:56.951 | INFO | Perf report for the last 5.13637 seconds: | 20260508 02:43:56.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:43:56.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:43:56.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:43:56.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:43:56.951 | INFO | Uploaded 36GiB to GPU in 2.56454s (14.0376GiB/s) | 20260508 02:43:56.951 | INFO | Downloaded 34GiB to GPU in 2.4199s (14.0502GiB/s) | 20260508 02:44:02.092 | INFO | Analyzing chunk frames = [4613632,4614144) | 20260508 02:44:02.092 | INFO | MemInfo: { | 20260508 02:44:02.092 | INFO | "gpu" : " 17 GB", | 20260508 02:44:02.092 | INFO | "huge" : "", | 20260508 02:44:02.092 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:02.092 | INFO | "malloc" : " 14 GB", | 20260508 02:44:02.093 | INFO | "pinned" : " 29 GB" | 20260508 02:44:02.093 | INFO | } | 20260508 02:44:02.093 | INFO | Perf report for the last 5.14181 seconds: | 20260508 02:44:02.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:02.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 02:44:02.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:44:02.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:02.093 | INFO | Uploaded 36GiB to GPU in 2.57782s (13.9653GiB/s) | 20260508 02:44:02.093 | INFO | Downloaded 34GiB to GPU in 2.49084s (13.65GiB/s) | 20260508 02:44:07.234 | INFO | Analyzing chunk frames = [4614144,4614656) | 20260508 02:44:07.234 | INFO | MemInfo: { | 20260508 02:44:07.234 | INFO | "gpu" : " 17 GB", | 20260508 02:44:07.234 | INFO | "huge" : "", | 20260508 02:44:07.234 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:07.234 | INFO | "malloc" : " 14 GB", | 20260508 02:44:07.234 | INFO | "pinned" : " 29 GB" | 20260508 02:44:07.234 | INFO | } | 20260508 02:44:07.234 | INFO | Perf report for the last 5.14154 seconds: | 20260508 02:44:07.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:07.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 02:44:07.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 02:44:07.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:07.234 | INFO | Uploaded 36GiB to GPU in 2.32623s (15.4757GiB/s) | 20260508 02:44:07.234 | INFO | Downloaded 34GiB to GPU in 2.48199s (13.6987GiB/s) | 20260508 02:44:12.369 | INFO | Analyzing chunk frames = [4614656,4615168) | 20260508 02:44:12.369 | INFO | MemInfo: { | 20260508 02:44:12.369 | INFO | "gpu" : " 17 GB", | 20260508 02:44:12.369 | INFO | "huge" : "", | 20260508 02:44:12.369 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:12.369 | INFO | "malloc" : " 14 GB", | 20260508 02:44:12.369 | INFO | "pinned" : " 29 GB" | 20260508 02:44:12.369 | INFO | } | 20260508 02:44:12.369 | INFO | Perf report for the last 5.1352 seconds: | 20260508 02:44:12.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:12.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.369 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:44:12.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.369 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:44:12.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:12.370 | INFO | Uploaded 36GiB to GPU in 2.4194s (14.8797GiB/s) | 20260508 02:44:12.370 | INFO | Downloaded 34GiB to GPU in 2.40988s (14.1086GiB/s) | 20260508 02:44:17.506 | INFO | Analyzing chunk frames = [4615168,4615680) | 20260508 02:44:17.506 | INFO | MemInfo: { | 20260508 02:44:17.506 | INFO | "gpu" : " 17 GB", | 20260508 02:44:17.506 | INFO | "huge" : "", | 20260508 02:44:17.506 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:17.506 | INFO | "malloc" : " 14 GB", | 20260508 02:44:17.506 | INFO | "pinned" : " 29 GB" | 20260508 02:44:17.506 | INFO | } | 20260508 02:44:17.506 | INFO | Perf report for the last 5.13687 seconds: | 20260508 02:44:17.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:17.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.506 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 02:44:17.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.506 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 02:44:17.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:17.507 | INFO | Uploaded 36GiB to GPU in 2.42927s (14.8193GiB/s) | 20260508 02:44:17.507 | INFO | Downloaded 34GiB to GPU in 2.41809s (14.0607GiB/s) | 20260508 02:44:22.644 | INFO | Analyzing chunk frames = [4615680,4616192) | 20260508 02:44:22.644 | INFO | MemInfo: { | 20260508 02:44:22.644 | INFO | "gpu" : " 17 GB", | 20260508 02:44:22.645 | INFO | "huge" : "", | 20260508 02:44:22.645 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:22.645 | INFO | "malloc" : " 14 GB", | 20260508 02:44:22.645 | INFO | "pinned" : " 29 GB" | 20260508 02:44:22.645 | INFO | } | 20260508 02:44:22.645 | INFO | Perf report for the last 5.13856 seconds: | 20260508 02:44:22.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:22.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:44:22.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:44:22.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:22.645 | INFO | Uploaded 36GiB to GPU in 2.57379s (13.9872GiB/s) | 20260508 02:44:22.645 | INFO | Downloaded 34GiB to GPU in 2.45999s (13.8212GiB/s) | 20260508 02:44:22.741 | INFO | Starting a round of Baz IO | 20260508 02:44:26.197 | INFO | Populating RTMetrics report | 20260508 02:44:26.197 | INFO | RTMetrics report populated | 20260508 02:44:26.197 | INFO | Opening tmp RTMetrics json file | 20260508 02:44:26.216 | INFO | Serializing RTMetrics to json | 20260508 02:44:26.225 | INFO | Serialization done, swapping tmp file now... | 20260508 02:44:26.227 | INFO | RTMetrics done | 20260508 02:44:26.292 | INFO | Finished a round of Baz IO: Wrote 1.631 GiB in 3.55 seconds | 20260508 02:44:26.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.46% / 0.00% / 58.54% | 20260508 02:44:26.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.690122% / 1.19209e-05% / 5.96046e-05% | 20260508 02:44:26.292 | INFO | Baz IO overall: | 20260508 02:44:26.292 | INFO | BytesWritten : 2081869090297 | 20260508 02:44:26.292 | INFO | NumEvents : 1006255374185 | 20260508 02:44:27.787 | INFO | Analyzing chunk frames = [4616192,4616704) | 20260508 02:44:27.787 | INFO | MemInfo: { | 20260508 02:44:27.787 | INFO | "gpu" : " 17 GB", | 20260508 02:44:27.787 | INFO | "huge" : "", | 20260508 02:44:27.787 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:27.787 | INFO | "malloc" : " 14 GB", | 20260508 02:44:27.787 | INFO | "pinned" : " 29 GB" | 20260508 02:44:27.787 | INFO | } | 20260508 02:44:27.787 | INFO | Perf report for the last 5.14267 seconds: | 20260508 02:44:27.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:27.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:27.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:27.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:27.788 | INFO | ANALYSIS : 68.1%, 3.0, 106.5ms, 0 | 20260508 02:44:27.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:27.788 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 02:44:27.788 | INFO | PRE_HQ : 65.1%, 1.6, 55.8ms, 0 | 20260508 02:44:27.788 | INFO | BAZWRITER : 68.6%, 3.5, 123.6ms, 0 | 20260508 02:44:27.788 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 02:44:27.788 | INFO | Uploaded 36GiB to GPU in 2.56407s (14.0402GiB/s) | 20260508 02:44:27.788 | INFO | Downloaded 38GiB to GPU in 2.68373s (14.1594GiB/s) | 20260508 02:44:32.929 | INFO | Analyzing chunk frames = [4616704,4617216) | 20260508 02:44:32.929 | INFO | MemInfo: { | 20260508 02:44:32.929 | INFO | "gpu" : " 17 GB", | 20260508 02:44:32.929 | INFO | "huge" : "", | 20260508 02:44:32.929 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:32.929 | INFO | "malloc" : " 14 GB", | 20260508 02:44:32.929 | INFO | "pinned" : " 29 GB" | 20260508 02:44:32.929 | INFO | } | 20260508 02:44:32.929 | INFO | Perf report for the last 5.14156 seconds: | 20260508 02:44:32.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:32.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 02:44:32.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 02:44:32.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:32.929 | INFO | Uploaded 36GiB to GPU in 2.51481s (14.3152GiB/s) | 20260508 02:44:32.929 | INFO | Downloaded 34GiB to GPU in 2.44246s (13.9204GiB/s) | 20260508 02:44:38.072 | INFO | Analyzing chunk frames = [4617216,4617728) | 20260508 02:44:38.072 | INFO | MemInfo: { | 20260508 02:44:38.072 | INFO | "gpu" : " 17 GB", | 20260508 02:44:38.072 | INFO | "huge" : "", | 20260508 02:44:38.072 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:38.072 | INFO | "malloc" : " 14 GB", | 20260508 02:44:38.072 | INFO | "pinned" : " 29 GB" | 20260508 02:44:38.072 | INFO | } | 20260508 02:44:38.072 | INFO | Perf report for the last 5.14315 seconds: | 20260508 02:44:38.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:38.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.072 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 02:44:38.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.072 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:44:38.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:38.073 | INFO | Uploaded 36GiB to GPU in 2.47066s (14.571GiB/s) | 20260508 02:44:38.073 | INFO | Downloaded 34GiB to GPU in 2.50872s (13.5527GiB/s) | 20260508 02:44:43.206 | INFO | Analyzing chunk frames = [4617728,4618240) | 20260508 02:44:43.206 | INFO | MemInfo: { | 20260508 02:44:43.206 | INFO | "gpu" : " 17 GB", | 20260508 02:44:43.206 | INFO | "huge" : "", | 20260508 02:44:43.206 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:43.206 | INFO | "malloc" : " 14 GB", | 20260508 02:44:43.206 | INFO | "pinned" : " 29 GB" | 20260508 02:44:43.206 | INFO | } | 20260508 02:44:43.206 | INFO | Perf report for the last 5.13428 seconds: | 20260508 02:44:43.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:43.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.207 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:44:43.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.207 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:44:43.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:43.207 | INFO | Uploaded 36GiB to GPU in 2.50674s (14.3613GiB/s) | 20260508 02:44:43.207 | INFO | Downloaded 34GiB to GPU in 2.519s (13.4974GiB/s) | 20260508 02:44:48.345 | INFO | Analyzing chunk frames = [4618240,4618752) | 20260508 02:44:48.345 | INFO | MemInfo: { | 20260508 02:44:48.345 | INFO | "gpu" : " 17 GB", | 20260508 02:44:48.345 | INFO | "huge" : "", | 20260508 02:44:48.345 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:48.345 | INFO | "malloc" : " 14 GB", | 20260508 02:44:48.345 | INFO | "pinned" : " 29 GB" | 20260508 02:44:48.345 | INFO | } | 20260508 02:44:48.345 | INFO | Perf report for the last 5.13848 seconds: | 20260508 02:44:48.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:48.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 02:44:48.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:44:48.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:48.345 | INFO | Uploaded 36GiB to GPU in 2.3982s (15.0112GiB/s) | 20260508 02:44:48.346 | INFO | Downloaded 34GiB to GPU in 2.57252s (13.2166GiB/s) | 20260508 02:44:53.482 | INFO | Analyzing chunk frames = [4618752,4619264) | 20260508 02:44:53.482 | INFO | MemInfo: { | 20260508 02:44:53.482 | INFO | "gpu" : " 17 GB", | 20260508 02:44:53.482 | INFO | "huge" : "", | 20260508 02:44:53.483 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:53.483 | INFO | "malloc" : " 14 GB", | 20260508 02:44:53.483 | INFO | "pinned" : " 29 GB" | 20260508 02:44:53.483 | INFO | } | 20260508 02:44:53.483 | INFO | Perf report for the last 5.13769 seconds: | 20260508 02:44:53.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:53.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.483 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:44:53.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.483 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:44:53.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:53.484 | INFO | Uploaded 36GiB to GPU in 2.37352s (15.1674GiB/s) | 20260508 02:44:53.484 | INFO | Downloaded 34GiB to GPU in 2.4371s (13.951GiB/s) | 20260508 02:44:58.622 | INFO | Analyzing chunk frames = [4619264,4619776) | 20260508 02:44:58.622 | INFO | MemInfo: { | 20260508 02:44:58.622 | INFO | "gpu" : " 17 GB", | 20260508 02:44:58.622 | INFO | "huge" : "", | 20260508 02:44:58.622 | INFO | "huge_shared" : " 29 GB", | 20260508 02:44:58.622 | INFO | "malloc" : " 14 GB", | 20260508 02:44:58.622 | INFO | "pinned" : " 29 GB" | 20260508 02:44:58.622 | INFO | } | 20260508 02:44:58.622 | INFO | Perf report for the last 5.13943 seconds: | 20260508 02:44:58.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:44:58.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.622 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:44:58.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.622 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:44:58.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:44:58.623 | INFO | Uploaded 36GiB to GPU in 2.40433s (14.973GiB/s) | 20260508 02:44:58.623 | INFO | Downloaded 34GiB to GPU in 2.44303s (13.9172GiB/s) | 20260508 02:45:03.762 | INFO | Analyzing chunk frames = [4619776,4620288) | 20260508 02:45:03.762 | INFO | MemInfo: { | 20260508 02:45:03.762 | INFO | "gpu" : " 17 GB", | 20260508 02:45:03.762 | INFO | "huge" : "", | 20260508 02:45:03.762 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:03.762 | INFO | "malloc" : " 14 GB", | 20260508 02:45:03.762 | INFO | "pinned" : " 29 GB" | 20260508 02:45:03.762 | INFO | } | 20260508 02:45:03.762 | INFO | Perf report for the last 5.13997 seconds: | 20260508 02:45:03.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:03.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.762 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:45:03.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.762 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:45:03.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:03.763 | INFO | Uploaded 36GiB to GPU in 2.51539s (14.3119GiB/s) | 20260508 02:45:03.763 | INFO | Downloaded 34GiB to GPU in 2.46482s (13.7941GiB/s) | 20260508 02:45:03.763 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:45:03.763 | INFO | Total mem usage: 16.7 GiB | 20260508 02:45:03.763 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:45:03.763 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.9%) | 20260508 02:45:03.763 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:45:03.763 | INFO | High Water Breakdown: | 20260508 02:45:03.763 | INFO | 60.1% : 8.97 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:45:03.763 | INFO | 39.7% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:45:03.763 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:45:03.763 | INFO | ------------------------------------------ | 20260508 02:45:03.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:45:03.763 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:45:03.763 | INFO | Total mem usage: 26.5 GiB | 20260508 02:45:03.763 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:45:03.763 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 02:45:03.763 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 02:45:03.763 | INFO | High Water Breakdown: | 20260508 02:45:03.763 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:45:03.763 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:45:03.763 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:45:03.763 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:45:03.763 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:45:03.763 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:45:03.763 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:45:03.763 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:45:03.763 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:45:03.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:45:03.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:45:03.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:45:03.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:45:03.763 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:45:03.763 | INFO | ------------------------------------------ | 20260508 02:45:03.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:45:03.763 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:45:03.763 | INFO | Total mem usage: 27.8 GiB | 20260508 02:45:03.763 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:45:03.763 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:45:03.763 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:45:03.763 | INFO | High Water Breakdown: | 20260508 02:45:03.763 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:45:03.763 | INFO | ------------------------------------------ | 20260508 02:45:03.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:45:03.763 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:45:03.763 | INFO | Total mem usage: 15.7 GiB | 20260508 02:45:03.763 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:45:03.763 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:45:03.763 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:45:03.763 | INFO | High Water Breakdown: | 20260508 02:45:03.763 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:45:03.763 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:45:03.763 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:45:03.763 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:45:03.763 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:45:03.763 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:45:03.763 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:45:03.763 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:45:03.763 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:45:03.763 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:45:03.763 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:45:03.763 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:45:03.763 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:45:03.763 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:45:03.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:45:03.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:45:03.764 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:45:03.764 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:45:03.764 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:45:03.764 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:45:03.764 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:45:03.764 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:45:03.764 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:45:03.764 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:45:03.764 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:45:03.764 | INFO | ------------------------------------------ | 20260508 02:45:03.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:45:03.764 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:45:03.764 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:45:03.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:45:03.764 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 1.42e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.02e-01, 1.10e-02 / 5.28e-03 | 20260508 02:45:03.764 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.49e+00, 5.83e-01 / 1.69e-01 | 20260508 02:45:03.764 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.06e-01 / 1.24e-01 | 20260508 02:45:03.764 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.59e+01, 4.05e+00 / 1.68e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.66e+00 / 1.48e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.69e+00, 4.23e-01 / 2.02e-01 | 20260508 02:45:03.764 | INFO | PerfReport: 3.28%, Metrics -- 1.07e+00 / 1.28e+00, 2.68e-01 / 1.30e-01 | 20260508 02:45:03.764 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.62e-01, 5.19e-02 / 2.23e-02 | 20260508 02:45:03.764 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:45:03.764 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:45:03.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:45:03.764 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 24.36%, Upload -- 2.59e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 30.67%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 2.18%, LockUpload -- 2.33e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 2.80%, LockDownload -- 2.99e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 15.56%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 02:45:03.764 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260508 02:45:03.863 | INFO | Starting a round of Baz IO | 20260508 02:45:07.324 | INFO | Populating RTMetrics report | 20260508 02:45:07.325 | INFO | RTMetrics report populated | 20260508 02:45:07.325 | INFO | Opening tmp RTMetrics json file | 20260508 02:45:07.341 | INFO | Serializing RTMetrics to json | 20260508 02:45:07.342 | INFO | Serialization done, swapping tmp file now... | 20260508 02:45:07.343 | INFO | RTMetrics done | 20260508 02:45:07.406 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.543 seconds | 20260508 02:45:07.406 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.46% / 0.00% / 58.54% | 20260508 02:45:07.406 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.68661% / 3.97364e-06% / 5.16574e-05% | 20260508 02:45:07.406 | INFO | Baz IO overall: | 20260508 02:45:07.407 | INFO | BytesWritten : 2083618420469 | 20260508 02:45:07.407 | INFO | NumEvents : 1007093150763 | 20260508 02:45:08.904 | INFO | Analyzing chunk frames = [4620288,4620800) | 20260508 02:45:08.904 | INFO | MemInfo: { | 20260508 02:45:08.904 | INFO | "gpu" : " 17 GB", | 20260508 02:45:08.904 | INFO | "huge" : "", | 20260508 02:45:08.904 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:08.904 | INFO | "malloc" : " 14 GB", | 20260508 02:45:08.904 | INFO | "pinned" : " 29 GB" | 20260508 02:45:08.904 | INFO | } | 20260508 02:45:08.904 | INFO | Perf report for the last 5.14166 seconds: | 20260508 02:45:08.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:08.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:08.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:08.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:08.904 | INFO | ANALYSIS : 68.3%, 2.9, 105.3ms, 0 | 20260508 02:45:08.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:08.904 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260508 02:45:08.904 | INFO | PRE_HQ : 66.6%, 1.7, 60.1ms, 0 | 20260508 02:45:08.904 | INFO | BAZWRITER : 68.6%, 3.9, 136.1ms, 0 | 20260508 02:45:08.904 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 02:45:08.904 | INFO | Uploaded 36GiB to GPU in 2.57216s (13.996GiB/s) | 20260508 02:45:08.905 | INFO | Downloaded 38GiB to GPU in 2.70476s (14.0493GiB/s) | 20260508 02:45:14.046 | INFO | Analyzing chunk frames = [4620800,4621312) | 20260508 02:45:14.046 | INFO | MemInfo: { | 20260508 02:45:14.046 | INFO | "gpu" : " 17 GB", | 20260508 02:45:14.046 | INFO | "huge" : "", | 20260508 02:45:14.046 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:14.046 | INFO | "malloc" : " 14 GB", | 20260508 02:45:14.046 | INFO | "pinned" : " 29 GB" | 20260508 02:45:14.046 | INFO | } | 20260508 02:45:14.046 | INFO | Perf report for the last 5.14234 seconds: | 20260508 02:45:14.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:14.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:45:14.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:45:14.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:14.046 | INFO | Uploaded 36GiB to GPU in 2.50488s (14.372GiB/s) | 20260508 02:45:14.046 | INFO | Downloaded 34GiB to GPU in 2.44505s (13.9056GiB/s) | 20260508 02:45:19.182 | INFO | Analyzing chunk frames = [4621312,4621824) | 20260508 02:45:19.182 | INFO | MemInfo: { | 20260508 02:45:19.182 | INFO | "gpu" : " 17 GB", | 20260508 02:45:19.182 | INFO | "huge" : "", | 20260508 02:45:19.182 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:19.182 | INFO | "malloc" : " 14 GB", | 20260508 02:45:19.182 | INFO | "pinned" : " 29 GB" | 20260508 02:45:19.182 | INFO | } | 20260508 02:45:19.182 | INFO | Perf report for the last 5.13562 seconds: | 20260508 02:45:19.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:19.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:45:19.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:45:19.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:19.182 | INFO | Uploaded 36GiB to GPU in 2.49555s (14.4257GiB/s) | 20260508 02:45:19.182 | INFO | Downloaded 34GiB to GPU in 2.51851s (13.5GiB/s) | 20260508 02:45:24.320 | INFO | Analyzing chunk frames = [4621824,4622336) | 20260508 02:45:24.320 | INFO | MemInfo: { | 20260508 02:45:24.320 | INFO | "gpu" : " 17 GB", | 20260508 02:45:24.320 | INFO | "huge" : "", | 20260508 02:45:24.321 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:24.321 | INFO | "malloc" : " 14 GB", | 20260508 02:45:24.321 | INFO | "pinned" : " 29 GB" | 20260508 02:45:24.321 | INFO | } | 20260508 02:45:24.321 | INFO | Perf report for the last 5.13899 seconds: | 20260508 02:45:24.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:24.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 02:45:24.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:45:24.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:24.321 | INFO | Uploaded 36GiB to GPU in 2.494s (14.4347GiB/s) | 20260508 02:45:24.321 | INFO | Downloaded 34GiB to GPU in 2.51417s (13.5233GiB/s) | 20260508 02:45:29.457 | INFO | Analyzing chunk frames = [4622336,4622848) | 20260508 02:45:29.457 | INFO | MemInfo: { | 20260508 02:45:29.457 | INFO | "gpu" : " 17 GB", | 20260508 02:45:29.457 | INFO | "huge" : "", | 20260508 02:45:29.457 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:29.457 | INFO | "malloc" : " 14 GB", | 20260508 02:45:29.457 | INFO | "pinned" : " 29 GB" | 20260508 02:45:29.457 | INFO | } | 20260508 02:45:29.457 | INFO | Perf report for the last 5.13669 seconds: | 20260508 02:45:29.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:29.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.458 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 02:45:29.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.458 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:45:29.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:29.458 | INFO | Uploaded 36GiB to GPU in 2.41155s (14.9281GiB/s) | 20260508 02:45:29.458 | INFO | Downloaded 34GiB to GPU in 2.56535s (13.2536GiB/s) | 20260508 02:45:34.597 | INFO | Analyzing chunk frames = [4622848,4623360) | 20260508 02:45:34.597 | INFO | MemInfo: { | 20260508 02:45:34.597 | INFO | "gpu" : " 17 GB", | 20260508 02:45:34.597 | INFO | "huge" : "", | 20260508 02:45:34.597 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:34.597 | INFO | "malloc" : " 14 GB", | 20260508 02:45:34.597 | INFO | "pinned" : " 29 GB" | 20260508 02:45:34.597 | INFO | } | 20260508 02:45:34.597 | INFO | Perf report for the last 5.13954 seconds: | 20260508 02:45:34.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:34.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:45:34.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:45:34.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:34.597 | INFO | Uploaded 36GiB to GPU in 2.4032s (14.98GiB/s) | 20260508 02:45:34.597 | INFO | Downloaded 34GiB to GPU in 2.41513s (14.0779GiB/s) | 20260508 02:45:39.746 | INFO | Analyzing chunk frames = [4623360,4623872) | 20260508 02:45:39.746 | INFO | MemInfo: { | 20260508 02:45:39.746 | INFO | "gpu" : " 17 GB", | 20260508 02:45:39.746 | INFO | "huge" : "", | 20260508 02:45:39.746 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:39.746 | INFO | "malloc" : " 14 GB", | 20260508 02:45:39.746 | INFO | "pinned" : " 29 GB" | 20260508 02:45:39.746 | INFO | } | 20260508 02:45:39.746 | INFO | Perf report for the last 5.14903 seconds: | 20260508 02:45:39.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:39.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.746 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 02:45:39.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.746 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 02:45:39.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:39.747 | INFO | Uploaded 36GiB to GPU in 2.44039s (14.7518GiB/s) | 20260508 02:45:39.747 | INFO | Downloaded 34GiB to GPU in 2.4306s (13.9883GiB/s) | 20260508 02:45:44.883 | INFO | Analyzing chunk frames = [4623872,4624384) | 20260508 02:45:44.883 | INFO | MemInfo: { | 20260508 02:45:44.883 | INFO | "gpu" : " 17 GB", | 20260508 02:45:44.884 | INFO | "huge" : "", | 20260508 02:45:44.884 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:44.884 | INFO | "malloc" : " 14 GB", | 20260508 02:45:44.884 | INFO | "pinned" : " 29 GB" | 20260508 02:45:44.884 | INFO | } | 20260508 02:45:44.884 | INFO | Perf report for the last 5.13755 seconds: | 20260508 02:45:44.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:44.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:45:44.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 02:45:44.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:44.884 | INFO | Uploaded 36GiB to GPU in 2.4526s (14.6783GiB/s) | 20260508 02:45:44.884 | INFO | Downloaded 34GiB to GPU in 2.44104s (13.9285GiB/s) | 20260508 02:45:44.999 | INFO | Starting a round of Baz IO | 20260508 02:45:48.456 | INFO | Populating RTMetrics report | 20260508 02:45:48.456 | INFO | RTMetrics report populated | 20260508 02:45:48.456 | INFO | Opening tmp RTMetrics json file | 20260508 02:45:48.457 | INFO | Serializing RTMetrics to json | 20260508 02:45:48.459 | INFO | Serialization done, swapping tmp file now... | 20260508 02:45:48.459 | INFO | RTMetrics done | 20260508 02:45:48.546 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.547 seconds | 20260508 02:45:48.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.45% / 0.00% / 58.54% | 20260508 02:45:48.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.686542% / 3.97364e-06% / 5.16574e-05% | 20260508 02:45:48.546 | INFO | Baz IO overall: | 20260508 02:45:48.546 | INFO | BytesWritten : 2085368878436 | 20260508 02:45:48.546 | INFO | NumEvents : 1007931805703 | 20260508 02:45:50.019 | INFO | Analyzing chunk frames = [4624384,4624896) | 20260508 02:45:50.019 | INFO | MemInfo: { | 20260508 02:45:50.019 | INFO | "gpu" : " 17 GB", | 20260508 02:45:50.019 | INFO | "huge" : "", | 20260508 02:45:50.019 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:50.019 | INFO | "malloc" : " 14 GB", | 20260508 02:45:50.019 | INFO | "pinned" : " 29 GB" | 20260508 02:45:50.019 | INFO | } | 20260508 02:45:50.019 | INFO | Perf report for the last 5.13535 seconds: | 20260508 02:45:50.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:50.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:50.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:50.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:50.019 | INFO | ANALYSIS : 69.2%, 2.9, 104.6ms, 0 | 20260508 02:45:50.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:50.019 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260508 02:45:50.019 | INFO | PRE_HQ : 68.0%, 1.8, 62.8ms, 0 | 20260508 02:45:50.019 | INFO | BAZWRITER : 68.7%, 3.7, 132.2ms, 0 | 20260508 02:45:50.019 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260508 02:45:50.020 | INFO | Uploaded 36GiB to GPU in 2.51508s (14.3136GiB/s) | 20260508 02:45:50.020 | INFO | Downloaded 38GiB to GPU in 2.70143s (14.0666GiB/s) | 20260508 02:45:55.153 | INFO | Analyzing chunk frames = [4624896,4625408) | 20260508 02:45:55.153 | INFO | MemInfo: { | 20260508 02:45:55.153 | INFO | "gpu" : " 17 GB", | 20260508 02:45:55.153 | INFO | "huge" : "", | 20260508 02:45:55.153 | INFO | "huge_shared" : " 29 GB", | 20260508 02:45:55.153 | INFO | "malloc" : " 14 GB", | 20260508 02:45:55.153 | INFO | "pinned" : " 29 GB" | 20260508 02:45:55.153 | INFO | } | 20260508 02:45:55.154 | INFO | Perf report for the last 5.1345 seconds: | 20260508 02:45:55.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:45:55.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:45:55.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:45:55.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:45:55.154 | INFO | Uploaded 36GiB to GPU in 2.58177s (13.9439GiB/s) | 20260508 02:45:55.154 | INFO | Downloaded 34GiB to GPU in 2.42557s (14.0173GiB/s) | 20260508 02:46:00.297 | INFO | Analyzing chunk frames = [4625408,4625920) | 20260508 02:46:00.297 | INFO | MemInfo: { | 20260508 02:46:00.297 | INFO | "gpu" : " 17 GB", | 20260508 02:46:00.297 | INFO | "huge" : "", | 20260508 02:46:00.297 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:00.297 | INFO | "malloc" : " 14 GB", | 20260508 02:46:00.297 | INFO | "pinned" : " 29 GB" | 20260508 02:46:00.297 | INFO | } | 20260508 02:46:00.297 | INFO | Perf report for the last 5.14377 seconds: | 20260508 02:46:00.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:00.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.297 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 02:46:00.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.297 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:46:00.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:00.298 | INFO | Uploaded 36GiB to GPU in 2.57727s (13.9683GiB/s) | 20260508 02:46:00.298 | INFO | Downloaded 34GiB to GPU in 2.48536s (13.6801GiB/s) | 20260508 02:46:05.436 | INFO | Analyzing chunk frames = [4625920,4626432) | 20260508 02:46:05.436 | INFO | MemInfo: { | 20260508 02:46:05.436 | INFO | "gpu" : " 17 GB", | 20260508 02:46:05.436 | INFO | "huge" : "", | 20260508 02:46:05.436 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:05.436 | INFO | "malloc" : " 14 GB", | 20260508 02:46:05.436 | INFO | "pinned" : " 29 GB" | 20260508 02:46:05.436 | INFO | } | 20260508 02:46:05.436 | INFO | Perf report for the last 5.13898 seconds: | 20260508 02:46:05.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:05.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.436 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 02:46:05.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.436 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:46:05.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:05.437 | INFO | Uploaded 36GiB to GPU in 2.56163s (14.0536GiB/s) | 20260508 02:46:05.437 | INFO | Downloaded 34GiB to GPU in 2.49194s (13.644GiB/s) | 20260508 02:46:10.572 | INFO | Analyzing chunk frames = [4626432,4626944) | 20260508 02:46:10.572 | INFO | MemInfo: { | 20260508 02:46:10.572 | INFO | "gpu" : " 17 GB", | 20260508 02:46:10.572 | INFO | "huge" : "", | 20260508 02:46:10.572 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:10.572 | INFO | "malloc" : " 14 GB", | 20260508 02:46:10.572 | INFO | "pinned" : " 29 GB" | 20260508 02:46:10.572 | INFO | } | 20260508 02:46:10.572 | INFO | Perf report for the last 5.13582 seconds: | 20260508 02:46:10.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:10.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 02:46:10.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:46:10.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:10.572 | INFO | Uploaded 36GiB to GPU in 2.45103s (14.6877GiB/s) | 20260508 02:46:10.572 | INFO | Downloaded 34GiB to GPU in 2.55556s (13.3043GiB/s) | 20260508 02:46:15.712 | INFO | Analyzing chunk frames = [4626944,4627456) | 20260508 02:46:15.712 | INFO | MemInfo: { | 20260508 02:46:15.712 | INFO | "gpu" : " 17 GB", | 20260508 02:46:15.712 | INFO | "huge" : "", | 20260508 02:46:15.712 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:15.712 | INFO | "malloc" : " 14 GB", | 20260508 02:46:15.712 | INFO | "pinned" : " 29 GB" | 20260508 02:46:15.712 | INFO | } | 20260508 02:46:15.712 | INFO | Perf report for the last 5.13976 seconds: | 20260508 02:46:15.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:15.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 02:46:15.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:46:15.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:15.712 | INFO | Uploaded 36GiB to GPU in 2.43413s (14.7897GiB/s) | 20260508 02:46:15.713 | INFO | Downloaded 34GiB to GPU in 2.42433s (14.0245GiB/s) | 20260508 02:46:20.853 | INFO | Analyzing chunk frames = [4627456,4627968) | 20260508 02:46:20.853 | INFO | MemInfo: { | 20260508 02:46:20.853 | INFO | "gpu" : " 17 GB", | 20260508 02:46:20.853 | INFO | "huge" : "", | 20260508 02:46:20.853 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:20.853 | INFO | "malloc" : " 14 GB", | 20260508 02:46:20.853 | INFO | "pinned" : " 29 GB" | 20260508 02:46:20.854 | INFO | } | 20260508 02:46:20.854 | INFO | Perf report for the last 5.1417 seconds: | 20260508 02:46:20.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:20.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:46:20.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:46:20.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:20.854 | INFO | Uploaded 36GiB to GPU in 2.47815s (14.527GiB/s) | 20260508 02:46:20.854 | INFO | Downloaded 34GiB to GPU in 2.42311s (14.0316GiB/s) | 20260508 02:46:25.999 | INFO | Analyzing chunk frames = [4627968,4628480) | 20260508 02:46:25.999 | INFO | MemInfo: { | 20260508 02:46:25.999 | INFO | "gpu" : " 17 GB", | 20260508 02:46:25.999 | INFO | "huge" : "", | 20260508 02:46:25.999 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:25.999 | INFO | "malloc" : " 14 GB", | 20260508 02:46:25.999 | INFO | "pinned" : " 29 GB" | 20260508 02:46:25.999 | INFO | } | 20260508 02:46:25.999 | INFO | Perf report for the last 5.14531 seconds: | 20260508 02:46:25.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:25.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 02:46:25.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 02:46:25.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:25.999 | INFO | Uploaded 36GiB to GPU in 2.57617s (13.9742GiB/s) | 20260508 02:46:25.999 | INFO | Downloaded 34GiB to GPU in 2.45937s (13.8247GiB/s) | 20260508 02:46:26.000 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:46:26.000 | INFO | Total mem usage: 16.7 GiB | 20260508 02:46:26.000 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:46:26.000 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.6%) | 20260508 02:46:26.000 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:46:26.000 | INFO | High Water Breakdown: | 20260508 02:46:26.000 | INFO | 60.1% : 8.9 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:46:26.000 | INFO | 39.8% : 5.9 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:46:26.000 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:46:26.000 | INFO | ------------------------------------------ | 20260508 02:46:26.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:46:26.000 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:46:26.000 | INFO | Total mem usage: 26.5 GiB | 20260508 02:46:26.000 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:46:26.000 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:46:26.000 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:46:26.000 | INFO | High Water Breakdown: | 20260508 02:46:26.000 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:46:26.000 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:46:26.000 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:46:26.000 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:46:26.000 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:46:26.000 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:46:26.000 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:46:26.000 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:46:26.000 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:46:26.000 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:46:26.000 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:46:26.000 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:46:26.000 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:46:26.000 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:46:26.000 | INFO | ------------------------------------------ | 20260508 02:46:26.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:46:26.000 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:46:26.000 | INFO | Total mem usage: 27.8 GiB | 20260508 02:46:26.000 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:46:26.000 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:46:26.000 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:46:26.000 | INFO | High Water Breakdown: | 20260508 02:46:26.000 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:46:26.000 | INFO | ------------------------------------------ | 20260508 02:46:26.000 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:46:26.000 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:46:26.000 | INFO | Total mem usage: 15.7 GiB | 20260508 02:46:26.000 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:46:26.000 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:46:26.000 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:46:26.000 | INFO | High Water Breakdown: | 20260508 02:46:26.000 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:46:26.000 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:46:26.000 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:46:26.000 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:46:26.000 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:46:26.000 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:46:26.000 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:46:26.000 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:46:26.000 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:46:26.000 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:46:26.000 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:46:26.000 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:46:26.000 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:46:26.000 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:46:26.000 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:46:26.000 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:46:26.000 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:46:26.000 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:46:26.001 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:46:26.001 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:46:26.001 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:46:26.001 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:46:26.001 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:46:26.001 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:46:26.001 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:46:26.001 | INFO | ------------------------------------------ | 20260508 02:46:26.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:46:26.001 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:46:26.001 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:46:26.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:46:26.001 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.38e+01 / 1.82e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.19e-02 / 7.41e-03 | 20260508 02:46:26.001 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.49e+00, 5.86e-01 / 1.76e-01 | 20260508 02:46:26.001 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.08e-01 / 1.09e-01 | 20260508 02:46:26.001 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.60e+01, 4.28e+00 / 2.12e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.72e+00 / 1.36e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.41e+00 / 2.70e+00, 4.23e-01 / 2.15e-01 | 20260508 02:46:26.001 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.23e+00, 2.65e-01 / 1.58e-01 | 20260508 02:46:26.001 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.65e-01, 5.18e-02 / 2.74e-02 | 20260508 02:46:26.001 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:46:26.001 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:46:26.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:46:26.001 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 24.42%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 24.45%, Download -- 2.60e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 30.75%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 2.71%, LockDownload -- 2.88e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 15.44%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:46:26.001 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.66e-02 / 0.00e+00 | 20260508 02:46:26.112 | INFO | Starting a round of Baz IO | 20260508 02:46:29.548 | INFO | Populating RTMetrics report | 20260508 02:46:29.548 | INFO | RTMetrics report populated | 20260508 02:46:29.548 | INFO | Opening tmp RTMetrics json file | 20260508 02:46:29.567 | INFO | Serializing RTMetrics to json | 20260508 02:46:29.672 | INFO | Serialization done, swapping tmp file now... | 20260508 02:46:29.678 | INFO | RTMetrics done | 20260508 02:46:29.708 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.595 seconds | 20260508 02:46:29.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.45% / 0.00% / 58.55% | 20260508 02:46:29.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.685751% / 0% / 6.35783e-05% | 20260508 02:46:29.708 | INFO | Baz IO overall: | 20260508 02:46:29.708 | INFO | BytesWritten : 2087118464299 | 20260508 02:46:29.708 | INFO | NumEvents : 1008770151105 | 20260508 02:46:31.132 | INFO | Analyzing chunk frames = [4628480,4628992) | 20260508 02:46:31.132 | INFO | MemInfo: { | 20260508 02:46:31.132 | INFO | "gpu" : " 17 GB", | 20260508 02:46:31.132 | INFO | "huge" : "", | 20260508 02:46:31.132 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:31.132 | INFO | "malloc" : " 14 GB", | 20260508 02:46:31.132 | INFO | "pinned" : " 29 GB" | 20260508 02:46:31.132 | INFO | } | 20260508 02:46:31.133 | INFO | Perf report for the last 5.13358 seconds: | 20260508 02:46:31.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:31.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:31.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:31.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:31.133 | INFO | ANALYSIS : 68.4%, 2.9, 104.4ms, 0 | 20260508 02:46:31.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:31.133 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260508 02:46:31.133 | INFO | PRE_HQ : 67.4%, 1.8, 63.0ms, 0 | 20260508 02:46:31.133 | INFO | BAZWRITER : 69.7%, 3.6, 128.9ms, 0 | 20260508 02:46:31.133 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260508 02:46:31.133 | INFO | Uploaded 36GiB to GPU in 2.60826s (13.8023GiB/s) | 20260508 02:46:31.133 | INFO | Downloaded 38GiB to GPU in 2.70321s (14.0574GiB/s) | 20260508 02:46:36.268 | INFO | Analyzing chunk frames = [4628992,4629504) | 20260508 02:46:36.268 | INFO | MemInfo: { | 20260508 02:46:36.268 | INFO | "gpu" : " 17 GB", | 20260508 02:46:36.268 | INFO | "huge" : "", | 20260508 02:46:36.268 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:36.268 | INFO | "malloc" : " 14 GB", | 20260508 02:46:36.268 | INFO | "pinned" : " 29 GB" | 20260508 02:46:36.268 | INFO | } | 20260508 02:46:36.268 | INFO | Perf report for the last 5.13587 seconds: | 20260508 02:46:36.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:36.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.268 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:46:36.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.269 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 02:46:36.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:36.269 | INFO | Uploaded 36GiB to GPU in 2.58144s (13.9457GiB/s) | 20260508 02:46:36.269 | INFO | Downloaded 34GiB to GPU in 2.41804s (14.061GiB/s) | 20260508 02:46:41.408 | INFO | Analyzing chunk frames = [4629504,4630016) | 20260508 02:46:41.408 | INFO | MemInfo: { | 20260508 02:46:41.408 | INFO | "gpu" : " 17 GB", | 20260508 02:46:41.408 | INFO | "huge" : "", | 20260508 02:46:41.408 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:41.408 | INFO | "malloc" : " 14 GB", | 20260508 02:46:41.408 | INFO | "pinned" : " 29 GB" | 20260508 02:46:41.408 | INFO | } | 20260508 02:46:41.409 | INFO | Perf report for the last 5.14008 seconds: | 20260508 02:46:41.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:41.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 02:46:41.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:46:41.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:41.409 | INFO | Uploaded 36GiB to GPU in 2.55972s (14.064GiB/s) | 20260508 02:46:41.409 | INFO | Downloaded 34GiB to GPU in 2.48594s (13.6769GiB/s) | 20260508 02:46:46.547 | INFO | Analyzing chunk frames = [4630016,4630528) | 20260508 02:46:46.547 | INFO | MemInfo: { | 20260508 02:46:46.547 | INFO | "gpu" : " 17 GB", | 20260508 02:46:46.547 | INFO | "huge" : "", | 20260508 02:46:46.547 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:46.548 | INFO | "malloc" : " 14 GB", | 20260508 02:46:46.548 | INFO | "pinned" : " 29 GB" | 20260508 02:46:46.548 | INFO | } | 20260508 02:46:46.548 | INFO | Perf report for the last 5.1391 seconds: | 20260508 02:46:46.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:46.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:46:46.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:46:46.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:46.548 | INFO | Uploaded 36GiB to GPU in 2.5522s (14.1055GiB/s) | 20260508 02:46:46.548 | INFO | Downloaded 34GiB to GPU in 2.47551s (13.7345GiB/s) | 20260508 02:46:51.692 | INFO | Analyzing chunk frames = [4630528,4631040) | 20260508 02:46:51.692 | INFO | MemInfo: { | 20260508 02:46:51.692 | INFO | "gpu" : " 17 GB", | 20260508 02:46:51.692 | INFO | "huge" : "", | 20260508 02:46:51.692 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:51.692 | INFO | "malloc" : " 14 GB", | 20260508 02:46:51.692 | INFO | "pinned" : " 29 GB" | 20260508 02:46:51.692 | INFO | } | 20260508 02:46:51.692 | INFO | Perf report for the last 5.14415 seconds: | 20260508 02:46:51.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:51.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 02:46:51.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:46:51.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:51.692 | INFO | Uploaded 36GiB to GPU in 2.43624s (14.7769GiB/s) | 20260508 02:46:51.693 | INFO | Downloaded 34GiB to GPU in 2.57022s (13.2285GiB/s) | 20260508 02:46:56.834 | INFO | Analyzing chunk frames = [4631040,4631552) | 20260508 02:46:56.834 | INFO | MemInfo: { | 20260508 02:46:56.834 | INFO | "gpu" : " 17 GB", | 20260508 02:46:56.834 | INFO | "huge" : "", | 20260508 02:46:56.834 | INFO | "huge_shared" : " 29 GB", | 20260508 02:46:56.834 | INFO | "malloc" : " 14 GB", | 20260508 02:46:56.834 | INFO | "pinned" : " 29 GB" | 20260508 02:46:56.834 | INFO | } | 20260508 02:46:56.834 | INFO | Perf report for the last 5.14265 seconds: | 20260508 02:46:56.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:46:56.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.834 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 02:46:56.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.835 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:46:56.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:46:56.835 | INFO | Uploaded 36GiB to GPU in 2.38926s (15.0674GiB/s) | 20260508 02:46:56.835 | INFO | Downloaded 34GiB to GPU in 2.40597s (14.1315GiB/s) | 20260508 02:47:01.968 | INFO | Analyzing chunk frames = [4631552,4632064) | 20260508 02:47:01.968 | INFO | MemInfo: { | 20260508 02:47:01.968 | INFO | "gpu" : " 17 GB", | 20260508 02:47:01.968 | INFO | "huge" : "", | 20260508 02:47:01.968 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:01.968 | INFO | "malloc" : " 14 GB", | 20260508 02:47:01.968 | INFO | "pinned" : " 29 GB" | 20260508 02:47:01.968 | INFO | } | 20260508 02:47:01.969 | INFO | Perf report for the last 5.13402 seconds: | 20260508 02:47:01.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:01.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 02:47:01.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 02:47:01.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:01.969 | INFO | Uploaded 36GiB to GPU in 2.42634s (14.8372GiB/s) | 20260508 02:47:01.970 | INFO | Downloaded 34GiB to GPU in 2.43081s (13.9871GiB/s) | 20260508 02:47:07.110 | INFO | Analyzing chunk frames = [4632064,4632576) | 20260508 02:47:07.110 | INFO | MemInfo: { | 20260508 02:47:07.110 | INFO | "gpu" : " 17 GB", | 20260508 02:47:07.111 | INFO | "huge" : "", | 20260508 02:47:07.111 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:07.111 | INFO | "malloc" : " 14 GB", | 20260508 02:47:07.111 | INFO | "pinned" : " 29 GB" | 20260508 02:47:07.111 | INFO | } | 20260508 02:47:07.111 | INFO | Perf report for the last 5.14227 seconds: | 20260508 02:47:07.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:07.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.111 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 02:47:07.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.111 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 02:47:07.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:07.112 | INFO | Uploaded 36GiB to GPU in 2.46733s (14.5907GiB/s) | 20260508 02:47:07.112 | INFO | Downloaded 34GiB to GPU in 2.4314s (13.9837GiB/s) | 20260508 02:47:07.224 | INFO | Starting a round of Baz IO | 20260508 02:47:10.676 | INFO | Populating RTMetrics report | 20260508 02:47:10.676 | INFO | RTMetrics report populated | 20260508 02:47:10.676 | INFO | Opening tmp RTMetrics json file | 20260508 02:47:10.677 | INFO | Serializing RTMetrics to json | 20260508 02:47:10.679 | INFO | Serialization done, swapping tmp file now... | 20260508 02:47:10.684 | INFO | RTMetrics done | 20260508 02:47:10.779 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.555 seconds | 20260508 02:47:10.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.45% / 0.00% / 58.55% | 20260508 02:47:10.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.683538% / 7.94729e-06% / 4.37101e-05% | 20260508 02:47:10.779 | INFO | Baz IO overall: | 20260508 02:47:10.779 | INFO | BytesWritten : 2088867218792 | 20260508 02:47:10.779 | INFO | NumEvents : 1009607781311 | 20260508 02:47:12.248 | INFO | Analyzing chunk frames = [4632576,4633088) | 20260508 02:47:12.248 | INFO | MemInfo: { | 20260508 02:47:12.248 | INFO | "gpu" : " 17 GB", | 20260508 02:47:12.248 | INFO | "huge" : "", | 20260508 02:47:12.248 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:12.248 | INFO | "malloc" : " 14 GB", | 20260508 02:47:12.248 | INFO | "pinned" : " 29 GB" | 20260508 02:47:12.248 | INFO | } | 20260508 02:47:12.248 | INFO | Perf report for the last 5.13709 seconds: | 20260508 02:47:12.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:12.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:12.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:12.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:12.248 | INFO | ANALYSIS : 69.0%, 2.9, 104.6ms, 0 | 20260508 02:47:12.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:12.248 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 02:47:12.248 | INFO | PRE_HQ : 67.8%, 1.8, 61.9ms, 0 | 20260508 02:47:12.248 | INFO | BAZWRITER : 68.7%, 3.8, 133.8ms, 0 | 20260508 02:47:12.248 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 02:47:12.248 | INFO | Uploaded 36GiB to GPU in 2.53672s (14.1916GiB/s) | 20260508 02:47:12.248 | INFO | Downloaded 38GiB to GPU in 2.7131s (14.0061GiB/s) | 20260508 02:47:17.385 | INFO | Analyzing chunk frames = [4633088,4633600) | 20260508 02:47:17.385 | INFO | MemInfo: { | 20260508 02:47:17.385 | INFO | "gpu" : " 17 GB", | 20260508 02:47:17.385 | INFO | "huge" : "", | 20260508 02:47:17.385 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:17.385 | INFO | "malloc" : " 14 GB", | 20260508 02:47:17.385 | INFO | "pinned" : " 29 GB" | 20260508 02:47:17.385 | INFO | } | 20260508 02:47:17.385 | INFO | Perf report for the last 5.13725 seconds: | 20260508 02:47:17.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:17.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:47:17.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:47:17.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:17.385 | INFO | Uploaded 36GiB to GPU in 2.55844s (14.0711GiB/s) | 20260508 02:47:17.385 | INFO | Downloaded 34GiB to GPU in 2.42976s (13.9931GiB/s) | 20260508 02:47:22.519 | INFO | Analyzing chunk frames = [4633600,4634112) | 20260508 02:47:22.519 | INFO | MemInfo: { | 20260508 02:47:22.519 | INFO | "gpu" : " 17 GB", | 20260508 02:47:22.519 | INFO | "huge" : "", | 20260508 02:47:22.520 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:22.520 | INFO | "malloc" : " 14 GB", | 20260508 02:47:22.520 | INFO | "pinned" : " 29 GB" | 20260508 02:47:22.520 | INFO | } | 20260508 02:47:22.520 | INFO | Perf report for the last 5.13454 seconds: | 20260508 02:47:22.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:22.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 02:47:22.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:47:22.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:22.520 | INFO | Uploaded 36GiB to GPU in 2.54058s (14.17GiB/s) | 20260508 02:47:22.520 | INFO | Downloaded 34GiB to GPU in 2.49431s (13.631GiB/s) | 20260508 02:47:27.666 | INFO | Analyzing chunk frames = [4634112,4634624) | 20260508 02:47:27.666 | INFO | MemInfo: { | 20260508 02:47:27.666 | INFO | "gpu" : " 17 GB", | 20260508 02:47:27.666 | INFO | "huge" : "", | 20260508 02:47:27.666 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:27.666 | INFO | "malloc" : " 14 GB", | 20260508 02:47:27.666 | INFO | "pinned" : " 29 GB" | 20260508 02:47:27.666 | INFO | } | 20260508 02:47:27.666 | INFO | Perf report for the last 5.14625 seconds: | 20260508 02:47:27.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:27.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 02:47:27.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:47:27.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:27.666 | INFO | Uploaded 36GiB to GPU in 2.53091s (14.2241GiB/s) | 20260508 02:47:27.666 | INFO | Downloaded 34GiB to GPU in 2.49432s (13.631GiB/s) | 20260508 02:47:32.805 | INFO | Analyzing chunk frames = [4634624,4635136) | 20260508 02:47:32.805 | INFO | MemInfo: { | 20260508 02:47:32.805 | INFO | "gpu" : " 17 GB", | 20260508 02:47:32.805 | INFO | "huge" : "", | 20260508 02:47:32.805 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:32.805 | INFO | "malloc" : " 14 GB", | 20260508 02:47:32.806 | INFO | "pinned" : " 29 GB" | 20260508 02:47:32.806 | INFO | } | 20260508 02:47:32.806 | INFO | Perf report for the last 5.13964 seconds: | 20260508 02:47:32.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:32.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:47:32.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:47:32.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:32.806 | INFO | Uploaded 36GiB to GPU in 2.43062s (14.8111GiB/s) | 20260508 02:47:32.806 | INFO | Downloaded 34GiB to GPU in 2.57275s (13.2155GiB/s) | 20260508 02:47:37.947 | INFO | Analyzing chunk frames = [4635136,4635648) | 20260508 02:47:37.948 | INFO | MemInfo: { | 20260508 02:47:37.948 | INFO | "gpu" : " 17 GB", | 20260508 02:47:37.948 | INFO | "huge" : "", | 20260508 02:47:37.948 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:37.948 | INFO | "malloc" : " 14 GB", | 20260508 02:47:37.948 | INFO | "pinned" : " 29 GB" | 20260508 02:47:37.948 | INFO | } | 20260508 02:47:37.948 | INFO | Perf report for the last 5.1421 seconds: | 20260508 02:47:37.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:37.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 02:47:37.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:47:37.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:37.948 | INFO | Uploaded 36GiB to GPU in 2.42742s (14.8305GiB/s) | 20260508 02:47:37.948 | INFO | Downloaded 34GiB to GPU in 2.41861s (14.0576GiB/s) | 20260508 02:47:43.083 | INFO | Analyzing chunk frames = [4635648,4636160) | 20260508 02:47:43.083 | INFO | MemInfo: { | 20260508 02:47:43.083 | INFO | "gpu" : " 17 GB", | 20260508 02:47:43.083 | INFO | "huge" : "", | 20260508 02:47:43.083 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:43.083 | INFO | "malloc" : " 14 GB", | 20260508 02:47:43.083 | INFO | "pinned" : " 29 GB" | 20260508 02:47:43.084 | INFO | } | 20260508 02:47:43.084 | INFO | Perf report for the last 5.1359 seconds: | 20260508 02:47:43.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:43.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:47:43.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:47:43.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:43.084 | INFO | Uploaded 36GiB to GPU in 2.45522s (14.6626GiB/s) | 20260508 02:47:43.084 | INFO | Downloaded 34GiB to GPU in 2.41959s (14.052GiB/s) | 20260508 02:47:48.216 | INFO | Analyzing chunk frames = [4636160,4636672) | 20260508 02:47:48.216 | INFO | MemInfo: { | 20260508 02:47:48.216 | INFO | "gpu" : " 17 GB", | 20260508 02:47:48.216 | INFO | "huge" : "", | 20260508 02:47:48.216 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:48.216 | INFO | "malloc" : " 14 GB", | 20260508 02:47:48.216 | INFO | "pinned" : " 29 GB" | 20260508 02:47:48.216 | INFO | } | 20260508 02:47:48.217 | INFO | Perf report for the last 5.1329 seconds: | 20260508 02:47:48.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:48.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 02:47:48.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260508 02:47:48.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:48.217 | INFO | Uploaded 36GiB to GPU in 2.48149s (14.5074GiB/s) | 20260508 02:47:48.217 | INFO | Downloaded 34GiB to GPU in 2.43107s (13.9856GiB/s) | 20260508 02:47:48.217 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:47:48.217 | INFO | Total mem usage: 16.7 GiB | 20260508 02:47:48.217 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:47:48.217 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260508 02:47:48.217 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:47:48.217 | INFO | High Water Breakdown: | 20260508 02:47:48.217 | INFO | 60% : 8.82 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:47:48.217 | INFO | 39.8% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:47:48.217 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:47:48.217 | INFO | ------------------------------------------ | 20260508 02:47:48.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:47:48.217 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:47:48.217 | INFO | Total mem usage: 26.5 GiB | 20260508 02:47:48.217 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:47:48.217 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:47:48.217 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:47:48.217 | INFO | High Water Breakdown: | 20260508 02:47:48.217 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:47:48.217 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:47:48.218 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:47:48.218 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:47:48.218 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:47:48.218 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:47:48.218 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:47:48.218 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:47:48.218 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:47:48.218 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:47:48.218 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:47:48.218 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:47:48.218 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:47:48.218 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:47:48.218 | INFO | ------------------------------------------ | 20260508 02:47:48.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:47:48.218 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:47:48.218 | INFO | Total mem usage: 27.8 GiB | 20260508 02:47:48.218 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:47:48.218 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:47:48.218 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:47:48.218 | INFO | High Water Breakdown: | 20260508 02:47:48.218 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:47:48.218 | INFO | ------------------------------------------ | 20260508 02:47:48.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:47:48.218 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:47:48.218 | INFO | Total mem usage: 15.7 GiB | 20260508 02:47:48.218 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:47:48.218 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:47:48.218 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:47:48.218 | INFO | High Water Breakdown: | 20260508 02:47:48.218 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:47:48.218 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:47:48.218 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:47:48.218 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:47:48.218 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:47:48.218 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:47:48.218 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:47:48.218 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:47:48.218 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:47:48.218 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:47:48.218 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:47:48.218 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:47:48.218 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:47:48.218 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:47:48.218 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:47:48.218 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:47:48.218 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:47:48.218 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:47:48.218 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:47:48.218 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:47:48.218 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:47:48.218 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:47:48.218 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:47:48.218 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:47:48.218 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:47:48.218 | INFO | ------------------------------------------ | 20260508 02:47:48.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:47:48.218 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:47:48.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:47:48.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:47:48.218 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.44e+01 / 1.76e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.03e-01, 1.21e-02 / 5.14e-03 | 20260508 02:47:48.218 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.45e+00, 5.86e-01 / 2.37e-01 | 20260508 02:47:48.218 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.11e-01 | 20260508 02:47:48.218 | INFO | PerfReport: 16.10%, DME -- 4.22e+01 / 4.62e+01, 4.19e+00 / 2.10e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.66e+00 / 1.38e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.23e-01 / 2.59e-01 | 20260508 02:47:48.218 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.22e+00, 2.66e-01 / 1.46e-01 | 20260508 02:47:48.218 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.29e-01 / 3.64e-01, 5.28e-02 / 2.68e-02 | 20260508 02:47:48.218 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:47:48.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:47:48.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:47:48.218 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 24.48%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 30.81%, Compute -- 3.28e+01 / 0.00e+00, 1.44e+01 / 0.00e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 2.22%, LockUpload -- 2.36e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 02:47:48.218 | INFO | PerfReport: 2.70%, LockDownload -- 2.87e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 02:47:48.219 | INFO | PerfReport: 15.35%, LockCompute -- 1.63e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 02:47:48.219 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260508 02:47:48.330 | INFO | Starting a round of Baz IO | 20260508 02:47:51.795 | INFO | Populating RTMetrics report | 20260508 02:47:51.795 | INFO | RTMetrics report populated | 20260508 02:47:51.795 | INFO | Opening tmp RTMetrics json file | 20260508 02:47:51.809 | INFO | Serializing RTMetrics to json | 20260508 02:47:51.811 | INFO | Serialization done, swapping tmp file now... | 20260508 02:47:51.812 | INFO | RTMetrics done | 20260508 02:47:51.887 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.557 seconds | 20260508 02:47:51.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.45% / 0.00% / 58.55% | 20260508 02:47:51.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.684114% / 1.98682e-05% / 5.96046e-05% | 20260508 02:47:51.887 | INFO | Baz IO overall: | 20260508 02:47:51.887 | INFO | BytesWritten : 2090617282292 | 20260508 02:47:51.887 | INFO | NumEvents : 1010446403712 | 20260508 02:47:53.361 | INFO | Analyzing chunk frames = [4636672,4637184) | 20260508 02:47:53.361 | INFO | MemInfo: { | 20260508 02:47:53.361 | INFO | "gpu" : " 17 GB", | 20260508 02:47:53.361 | INFO | "huge" : "", | 20260508 02:47:53.361 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:53.361 | INFO | "malloc" : " 14 GB", | 20260508 02:47:53.361 | INFO | "pinned" : " 29 GB" | 20260508 02:47:53.361 | INFO | } | 20260508 02:47:53.361 | INFO | Perf report for the last 5.14447 seconds: | 20260508 02:47:53.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:53.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:53.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:53.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:53.361 | INFO | ANALYSIS : 68.8%, 2.9, 105.0ms, 0 | 20260508 02:47:53.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:53.361 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 02:47:53.361 | INFO | PRE_HQ : 67.5%, 1.8, 61.7ms, 0 | 20260508 02:47:53.361 | INFO | BAZWRITER : 68.7%, 4.3, 153.0ms, 0 | 20260508 02:47:53.361 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:47:53.361 | INFO | Uploaded 36GiB to GPU in 2.61952s (13.743GiB/s) | 20260508 02:47:53.361 | INFO | Downloaded 38GiB to GPU in 2.70424s (14.052GiB/s) | 20260508 02:47:58.501 | INFO | Analyzing chunk frames = [4637184,4637696) | 20260508 02:47:58.501 | INFO | MemInfo: { | 20260508 02:47:58.501 | INFO | "gpu" : " 17 GB", | 20260508 02:47:58.501 | INFO | "huge" : "", | 20260508 02:47:58.501 | INFO | "huge_shared" : " 29 GB", | 20260508 02:47:58.501 | INFO | "malloc" : " 14 GB", | 20260508 02:47:58.501 | INFO | "pinned" : " 29 GB" | 20260508 02:47:58.501 | INFO | } | 20260508 02:47:58.501 | INFO | Perf report for the last 5.13983 seconds: | 20260508 02:47:58.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:47:58.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:47:58.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 02:47:58.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:47:58.501 | INFO | Uploaded 36GiB to GPU in 2.58022s (13.9523GiB/s) | 20260508 02:47:58.501 | INFO | Downloaded 34GiB to GPU in 2.4219s (14.0386GiB/s) | 20260508 02:48:03.644 | INFO | Analyzing chunk frames = [4637696,4638208) | 20260508 02:48:03.644 | INFO | MemInfo: { | 20260508 02:48:03.644 | INFO | "gpu" : " 17 GB", | 20260508 02:48:03.644 | INFO | "huge" : "", | 20260508 02:48:03.644 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:03.644 | INFO | "malloc" : " 14 GB", | 20260508 02:48:03.644 | INFO | "pinned" : " 29 GB" | 20260508 02:48:03.644 | INFO | } | 20260508 02:48:03.644 | INFO | Perf report for the last 5.14324 seconds: | 20260508 02:48:03.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:03.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.644 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:48:03.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.644 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:48:03.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:03.645 | INFO | Uploaded 36GiB to GPU in 2.57604s (13.9749GiB/s) | 20260508 02:48:03.645 | INFO | Downloaded 34GiB to GPU in 2.48928s (13.6585GiB/s) | 20260508 02:48:08.783 | INFO | Analyzing chunk frames = [4638208,4638720) | 20260508 02:48:08.783 | INFO | MemInfo: { | 20260508 02:48:08.783 | INFO | "gpu" : " 17 GB", | 20260508 02:48:08.783 | INFO | "huge" : "", | 20260508 02:48:08.783 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:08.783 | INFO | "malloc" : " 14 GB", | 20260508 02:48:08.783 | INFO | "pinned" : " 29 GB" | 20260508 02:48:08.783 | INFO | } | 20260508 02:48:08.783 | INFO | Perf report for the last 5.13908 seconds: | 20260508 02:48:08.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:08.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.783 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:48:08.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.783 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:48:08.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:08.784 | INFO | Uploaded 36GiB to GPU in 2.57315s (13.9906GiB/s) | 20260508 02:48:08.784 | INFO | Downloaded 34GiB to GPU in 2.48813s (13.6649GiB/s) | 20260508 02:48:13.921 | INFO | Analyzing chunk frames = [4638720,4639232) | 20260508 02:48:13.921 | INFO | MemInfo: { | 20260508 02:48:13.921 | INFO | "gpu" : " 17 GB", | 20260508 02:48:13.921 | INFO | "huge" : "", | 20260508 02:48:13.921 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:13.921 | INFO | "malloc" : " 14 GB", | 20260508 02:48:13.921 | INFO | "pinned" : " 29 GB" | 20260508 02:48:13.921 | INFO | } | 20260508 02:48:13.921 | INFO | Perf report for the last 5.13764 seconds: | 20260508 02:48:13.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:13.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:48:13.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:48:13.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:13.921 | INFO | Uploaded 36GiB to GPU in 2.46266s (14.6184GiB/s) | 20260508 02:48:13.921 | INFO | Downloaded 34GiB to GPU in 2.56966s (13.2313GiB/s) | 20260508 02:48:19.062 | INFO | Analyzing chunk frames = [4639232,4639744) | 20260508 02:48:19.062 | INFO | MemInfo: { | 20260508 02:48:19.062 | INFO | "gpu" : " 17 GB", | 20260508 02:48:19.062 | INFO | "huge" : "", | 20260508 02:48:19.062 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:19.062 | INFO | "malloc" : " 14 GB", | 20260508 02:48:19.062 | INFO | "pinned" : " 29 GB" | 20260508 02:48:19.062 | INFO | } | 20260508 02:48:19.063 | INFO | Perf report for the last 5.14167 seconds: | 20260508 02:48:19.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:19.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:48:19.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 02:48:19.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:19.063 | INFO | Uploaded 36GiB to GPU in 2.41004s (14.9375GiB/s) | 20260508 02:48:19.063 | INFO | Downloaded 34GiB to GPU in 2.42059s (14.0462GiB/s) | 20260508 02:48:24.201 | INFO | Analyzing chunk frames = [4639744,4640256) | 20260508 02:48:24.201 | INFO | MemInfo: { | 20260508 02:48:24.201 | INFO | "gpu" : " 17 GB", | 20260508 02:48:24.201 | INFO | "huge" : "", | 20260508 02:48:24.201 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:24.201 | INFO | "malloc" : " 14 GB", | 20260508 02:48:24.202 | INFO | "pinned" : " 29 GB" | 20260508 02:48:24.202 | INFO | } | 20260508 02:48:24.202 | INFO | Perf report for the last 5.13909 seconds: | 20260508 02:48:24.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:24.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:48:24.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:48:24.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:24.202 | INFO | Uploaded 36GiB to GPU in 2.44192s (14.7425GiB/s) | 20260508 02:48:24.202 | INFO | Downloaded 34GiB to GPU in 2.41862s (14.0576GiB/s) | 20260508 02:48:29.338 | INFO | Analyzing chunk frames = [4640256,4640768) | 20260508 02:48:29.338 | INFO | MemInfo: { | 20260508 02:48:29.338 | INFO | "gpu" : " 17 GB", | 20260508 02:48:29.338 | INFO | "huge" : "", | 20260508 02:48:29.338 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:29.338 | INFO | "malloc" : " 14 GB", | 20260508 02:48:29.338 | INFO | "pinned" : " 29 GB" | 20260508 02:48:29.338 | INFO | } | 20260508 02:48:29.338 | INFO | Perf report for the last 5.13632 seconds: | 20260508 02:48:29.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:29.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:48:29.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 02:48:29.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:29.338 | INFO | Uploaded 36GiB to GPU in 2.47566s (14.5416GiB/s) | 20260508 02:48:29.339 | INFO | Downloaded 34GiB to GPU in 2.43516s (13.9621GiB/s) | 20260508 02:48:29.453 | INFO | Starting a round of Baz IO | 20260508 02:48:32.904 | INFO | Populating RTMetrics report | 20260508 02:48:32.904 | INFO | RTMetrics report populated | 20260508 02:48:32.904 | INFO | Opening tmp RTMetrics json file | 20260508 02:48:32.922 | INFO | Serializing RTMetrics to json | 20260508 02:48:32.924 | INFO | Serialization done, swapping tmp file now... | 20260508 02:48:32.925 | INFO | RTMetrics done | 20260508 02:48:33.007 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.553 seconds | 20260508 02:48:33.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.44% / 0.00% / 58.55% | 20260508 02:48:33.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.682946% / 7.94729e-06% / 5.5631e-05% | 20260508 02:48:33.007 | INFO | Baz IO overall: | 20260508 02:48:33.007 | INFO | BytesWritten : 2092366759333 | 20260508 02:48:33.007 | INFO | NumEvents : 1011284516874 | 20260508 02:48:34.470 | INFO | Analyzing chunk frames = [4640768,4641280) | 20260508 02:48:34.470 | INFO | MemInfo: { | 20260508 02:48:34.470 | INFO | "gpu" : " 17 GB", | 20260508 02:48:34.470 | INFO | "huge" : "", | 20260508 02:48:34.470 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:34.470 | INFO | "malloc" : " 14 GB", | 20260508 02:48:34.470 | INFO | "pinned" : " 29 GB" | 20260508 02:48:34.470 | INFO | } | 20260508 02:48:34.470 | INFO | Perf report for the last 5.1322 seconds: | 20260508 02:48:34.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:34.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:34.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:34.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:34.470 | INFO | ANALYSIS : 68.6%, 2.9, 103.9ms, 0 | 20260508 02:48:34.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:34.470 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 02:48:34.470 | INFO | PRE_HQ : 67.5%, 1.8, 62.3ms, 0 | 20260508 02:48:34.470 | INFO | BAZWRITER : 68.9%, 3.8, 135.4ms, 0 | 20260508 02:48:34.470 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 02:48:34.471 | INFO | Uploaded 36GiB to GPU in 2.60237s (13.8335GiB/s) | 20260508 02:48:34.471 | INFO | Downloaded 38GiB to GPU in 2.69714s (14.089GiB/s) | 20260508 02:48:39.617 | INFO | Analyzing chunk frames = [4641280,4641792) | 20260508 02:48:39.617 | INFO | MemInfo: { | 20260508 02:48:39.617 | INFO | "gpu" : " 17 GB", | 20260508 02:48:39.617 | INFO | "huge" : "", | 20260508 02:48:39.617 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:39.617 | INFO | "malloc" : " 14 GB", | 20260508 02:48:39.618 | INFO | "pinned" : " 29 GB" | 20260508 02:48:39.618 | INFO | } | 20260508 02:48:39.618 | INFO | Perf report for the last 5.14742 seconds: | 20260508 02:48:39.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:39.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 02:48:39.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:48:39.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:39.618 | INFO | Uploaded 36GiB to GPU in 2.49998s (14.4001GiB/s) | 20260508 02:48:39.618 | INFO | Downloaded 34GiB to GPU in 2.44003s (13.9343GiB/s) | 20260508 02:48:44.760 | INFO | Analyzing chunk frames = [4641792,4642304) | 20260508 02:48:44.760 | INFO | MemInfo: { | 20260508 02:48:44.760 | INFO | "gpu" : " 17 GB", | 20260508 02:48:44.760 | INFO | "huge" : "", | 20260508 02:48:44.760 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:44.760 | INFO | "malloc" : " 14 GB", | 20260508 02:48:44.760 | INFO | "pinned" : " 29 GB" | 20260508 02:48:44.761 | INFO | } | 20260508 02:48:44.761 | INFO | Perf report for the last 5.14291 seconds: | 20260508 02:48:44.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:44.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 02:48:44.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:48:44.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:44.761 | INFO | Uploaded 36GiB to GPU in 2.4684s (14.5843GiB/s) | 20260508 02:48:44.761 | INFO | Downloaded 34GiB to GPU in 2.50313s (13.583GiB/s) | 20260508 02:48:49.899 | INFO | Analyzing chunk frames = [4642304,4642816) | 20260508 02:48:49.900 | INFO | MemInfo: { | 20260508 02:48:49.900 | INFO | "gpu" : " 17 GB", | 20260508 02:48:49.900 | INFO | "huge" : "", | 20260508 02:48:49.900 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:49.900 | INFO | "malloc" : " 14 GB", | 20260508 02:48:49.900 | INFO | "pinned" : " 29 GB" | 20260508 02:48:49.900 | INFO | } | 20260508 02:48:49.900 | INFO | Perf report for the last 5.13922 seconds: | 20260508 02:48:49.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:49.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 02:48:49.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:48:49.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:49.900 | INFO | Uploaded 36GiB to GPU in 2.48653s (14.478GiB/s) | 20260508 02:48:49.900 | INFO | Downloaded 34GiB to GPU in 2.51578s (13.5147GiB/s) | 20260508 02:48:55.032 | INFO | Analyzing chunk frames = [4642816,4643328) | 20260508 02:48:55.032 | INFO | MemInfo: { | 20260508 02:48:55.032 | INFO | "gpu" : " 17 GB", | 20260508 02:48:55.032 | INFO | "huge" : "", | 20260508 02:48:55.032 | INFO | "huge_shared" : " 29 GB", | 20260508 02:48:55.032 | INFO | "malloc" : " 14 GB", | 20260508 02:48:55.032 | INFO | "pinned" : " 29 GB" | 20260508 02:48:55.032 | INFO | } | 20260508 02:48:55.032 | INFO | Perf report for the last 5.13263 seconds: | 20260508 02:48:55.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:48:55.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.033 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 02:48:55.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.033 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:48:55.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:48:55.033 | INFO | Uploaded 36GiB to GPU in 2.33808s (15.3973GiB/s) | 20260508 02:48:55.033 | INFO | Downloaded 34GiB to GPU in 2.56773s (13.2413GiB/s) | 20260508 02:49:00.172 | INFO | Analyzing chunk frames = [4643328,4643840) | 20260508 02:49:00.172 | INFO | MemInfo: { | 20260508 02:49:00.172 | INFO | "gpu" : " 17 GB", | 20260508 02:49:00.172 | INFO | "huge" : "", | 20260508 02:49:00.172 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:00.172 | INFO | "malloc" : " 14 GB", | 20260508 02:49:00.172 | INFO | "pinned" : " 29 GB" | 20260508 02:49:00.172 | INFO | } | 20260508 02:49:00.172 | INFO | Perf report for the last 5.13941 seconds: | 20260508 02:49:00.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:00.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.172 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 02:49:00.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.172 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:49:00.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:00.173 | INFO | Uploaded 36GiB to GPU in 2.39018s (15.0616GiB/s) | 20260508 02:49:00.173 | INFO | Downloaded 34GiB to GPU in 2.44312s (13.9166GiB/s) | 20260508 02:49:05.311 | INFO | Analyzing chunk frames = [4643840,4644352) | 20260508 02:49:05.311 | INFO | MemInfo: { | 20260508 02:49:05.311 | INFO | "gpu" : " 17 GB", | 20260508 02:49:05.311 | INFO | "huge" : "", | 20260508 02:49:05.311 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:05.311 | INFO | "malloc" : " 14 GB", | 20260508 02:49:05.311 | INFO | "pinned" : " 29 GB" | 20260508 02:49:05.311 | INFO | } | 20260508 02:49:05.311 | INFO | Perf report for the last 5.13952 seconds: | 20260508 02:49:05.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:05.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.312 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:49:05.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.312 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:49:05.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:05.312 | INFO | Uploaded 36GiB to GPU in 2.40989s (14.9384GiB/s) | 20260508 02:49:05.312 | INFO | Downloaded 34GiB to GPU in 2.43731s (13.9498GiB/s) | 20260508 02:49:10.450 | INFO | Analyzing chunk frames = [4644352,4644864) | 20260508 02:49:10.450 | INFO | MemInfo: { | 20260508 02:49:10.450 | INFO | "gpu" : " 17 GB", | 20260508 02:49:10.450 | INFO | "huge" : "", | 20260508 02:49:10.450 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:10.450 | INFO | "malloc" : " 14 GB", | 20260508 02:49:10.450 | INFO | "pinned" : " 29 GB" | 20260508 02:49:10.451 | INFO | } | 20260508 02:49:10.451 | INFO | Perf report for the last 5.13918 seconds: | 20260508 02:49:10.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:10.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 02:49:10.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 02:49:10.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:10.451 | INFO | Uploaded 36GiB to GPU in 2.43028s (14.8131GiB/s) | 20260508 02:49:10.451 | INFO | Downloaded 34GiB to GPU in 2.44884s (13.8841GiB/s) | 20260508 02:49:10.451 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:49:10.451 | INFO | Total mem usage: 16.7 GiB | 20260508 02:49:10.451 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:49:10.452 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11%) | 20260508 02:49:10.452 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:49:10.452 | INFO | High Water Breakdown: | 20260508 02:49:10.452 | INFO | 60.3% : 9.06 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:49:10.452 | INFO | 39.6% : 5.95 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:49:10.452 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:49:10.452 | INFO | ------------------------------------------ | 20260508 02:49:10.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:49:10.452 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:49:10.452 | INFO | Total mem usage: 26.5 GiB | 20260508 02:49:10.452 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:49:10.452 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 02:49:10.452 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 02:49:10.452 | INFO | High Water Breakdown: | 20260508 02:49:10.452 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:49:10.452 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:49:10.452 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:49:10.452 | INFO | 1.99% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:49:10.452 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:49:10.452 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:49:10.452 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:49:10.452 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:49:10.452 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:49:10.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:49:10.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:49:10.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:49:10.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:49:10.452 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:49:10.452 | INFO | ------------------------------------------ | 20260508 02:49:10.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:49:10.452 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:49:10.452 | INFO | Total mem usage: 27.8 GiB | 20260508 02:49:10.452 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:49:10.452 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:49:10.452 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:49:10.452 | INFO | High Water Breakdown: | 20260508 02:49:10.452 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:49:10.452 | INFO | ------------------------------------------ | 20260508 02:49:10.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:49:10.452 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:49:10.452 | INFO | Total mem usage: 15.7 GiB | 20260508 02:49:10.452 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:49:10.452 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:49:10.452 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:49:10.452 | INFO | High Water Breakdown: | 20260508 02:49:10.452 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:49:10.452 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:49:10.452 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:49:10.452 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:49:10.452 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:49:10.452 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:49:10.452 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:49:10.452 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:49:10.452 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:49:10.452 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:49:10.452 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:49:10.452 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:49:10.452 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:49:10.452 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:49:10.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:49:10.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:49:10.452 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:49:10.452 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:49:10.452 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:49:10.453 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:49:10.453 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:49:10.453 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:49:10.453 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:49:10.453 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:49:10.453 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:49:10.453 | INFO | ------------------------------------------ | 20260508 02:49:10.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:49:10.453 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:49:10.453 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:49:10.453 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:49:10.453 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.41e+01 / 1.71e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.02e-01, 1.21e-02 / 4.56e-03 | 20260508 02:49:10.453 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.45e+00, 5.88e-01 / 1.99e-01 | 20260508 02:49:10.453 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.07e-01 / 1.16e-01 | 20260508 02:49:10.453 | INFO | PerfReport: 16.05%, DME -- 4.21e+01 / 4.61e+01, 4.27e+00 / 2.03e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.43e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.41e+00 / 2.66e+00, 4.23e-01 / 2.16e-01 | 20260508 02:49:10.453 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.19e+00, 2.63e-01 / 1.11e-01 | 20260508 02:49:10.453 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.58e-01, 5.15e-02 / 2.64e-02 | 20260508 02:49:10.453 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:49:10.453 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:49:10.453 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:49:10.453 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 24.52%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 2.10%, LockUpload -- 2.23e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 2.80%, LockDownload -- 2.97e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 15.36%, LockCompute -- 1.63e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:49:10.453 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260508 02:49:10.565 | INFO | Starting a round of Baz IO | 20260508 02:49:14.016 | INFO | Populating RTMetrics report | 20260508 02:49:14.017 | INFO | RTMetrics report populated | 20260508 02:49:14.017 | INFO | Opening tmp RTMetrics json file | 20260508 02:49:14.031 | INFO | Serializing RTMetrics to json | 20260508 02:49:14.033 | INFO | Serialization done, swapping tmp file now... | 20260508 02:49:14.033 | INFO | RTMetrics done | 20260508 02:49:14.097 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.532 seconds | 20260508 02:49:14.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.44% / 0.00% / 58.56% | 20260508 02:49:14.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.679119% / 0% / 4.37101e-05% | 20260508 02:49:14.097 | INFO | Baz IO overall: | 20260508 02:49:14.097 | INFO | BytesWritten : 2094115016449 | 20260508 02:49:14.097 | INFO | NumEvents : 1012121845907 | 20260508 02:49:15.597 | INFO | Analyzing chunk frames = [4644864,4645376) | 20260508 02:49:15.597 | INFO | MemInfo: { | 20260508 02:49:15.597 | INFO | "gpu" : " 17 GB", | 20260508 02:49:15.597 | INFO | "huge" : "", | 20260508 02:49:15.597 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:15.597 | INFO | "malloc" : " 14 GB", | 20260508 02:49:15.597 | INFO | "pinned" : " 29 GB" | 20260508 02:49:15.597 | INFO | } | 20260508 02:49:15.597 | INFO | Perf report for the last 5.14663 seconds: | 20260508 02:49:15.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:15.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:15.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:15.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:15.597 | INFO | ANALYSIS : 68.3%, 2.9, 103.9ms, 0 | 20260508 02:49:15.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:15.597 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260508 02:49:15.597 | INFO | PRE_HQ : 67.3%, 1.8, 62.6ms, 0 | 20260508 02:49:15.597 | INFO | BAZWRITER : 68.2%, 3.8, 132.8ms, 0 | 20260508 02:49:15.597 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:49:15.598 | INFO | Uploaded 36GiB to GPU in 2.59355s (13.8806GiB/s) | 20260508 02:49:15.598 | INFO | Downloaded 38GiB to GPU in 2.69887s (14.08GiB/s) | 20260508 02:49:20.733 | INFO | Analyzing chunk frames = [4645376,4645888) | 20260508 02:49:20.733 | INFO | MemInfo: { | 20260508 02:49:20.733 | INFO | "gpu" : " 17 GB", | 20260508 02:49:20.733 | INFO | "huge" : "", | 20260508 02:49:20.733 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:20.733 | INFO | "malloc" : " 14 GB", | 20260508 02:49:20.733 | INFO | "pinned" : " 29 GB" | 20260508 02:49:20.733 | INFO | } | 20260508 02:49:20.734 | INFO | Perf report for the last 5.13629 seconds: | 20260508 02:49:20.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:20.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:49:20.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:49:20.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:20.734 | INFO | Uploaded 36GiB to GPU in 2.55325s (14.0997GiB/s) | 20260508 02:49:20.734 | INFO | Downloaded 34GiB to GPU in 2.4536s (13.8572GiB/s) | 20260508 02:49:25.877 | INFO | Analyzing chunk frames = [4645888,4646400) | 20260508 02:49:25.877 | INFO | MemInfo: { | 20260508 02:49:25.877 | INFO | "gpu" : " 17 GB", | 20260508 02:49:25.877 | INFO | "huge" : "", | 20260508 02:49:25.877 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:25.877 | INFO | "malloc" : " 14 GB", | 20260508 02:49:25.878 | INFO | "pinned" : " 29 GB" | 20260508 02:49:25.878 | INFO | } | 20260508 02:49:25.878 | INFO | Perf report for the last 5.14435 seconds: | 20260508 02:49:25.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:25.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 02:49:25.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:49:25.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:25.878 | INFO | Uploaded 36GiB to GPU in 2.51264s (14.3276GiB/s) | 20260508 02:49:25.878 | INFO | Downloaded 34GiB to GPU in 2.51173s (13.5365GiB/s) | 20260508 02:49:31.012 | INFO | Analyzing chunk frames = [4646400,4646912) | 20260508 02:49:31.012 | INFO | MemInfo: { | 20260508 02:49:31.012 | INFO | "gpu" : " 17 GB", | 20260508 02:49:31.012 | INFO | "huge" : "", | 20260508 02:49:31.012 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:31.012 | INFO | "malloc" : " 14 GB", | 20260508 02:49:31.012 | INFO | "pinned" : " 29 GB" | 20260508 02:49:31.012 | INFO | } | 20260508 02:49:31.012 | INFO | Perf report for the last 5.13399 seconds: | 20260508 02:49:31.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:31.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:49:31.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:49:31.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:31.012 | INFO | Uploaded 36GiB to GPU in 2.5139s (14.3204GiB/s) | 20260508 02:49:31.012 | INFO | Downloaded 34GiB to GPU in 2.52289s (13.4766GiB/s) | 20260508 02:49:36.148 | INFO | Analyzing chunk frames = [4646912,4647424) | 20260508 02:49:36.148 | INFO | MemInfo: { | 20260508 02:49:36.148 | INFO | "gpu" : " 17 GB", | 20260508 02:49:36.148 | INFO | "huge" : "", | 20260508 02:49:36.148 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:36.148 | INFO | "malloc" : " 14 GB", | 20260508 02:49:36.148 | INFO | "pinned" : " 29 GB" | 20260508 02:49:36.148 | INFO | } | 20260508 02:49:36.148 | INFO | Perf report for the last 5.13639 seconds: | 20260508 02:49:36.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:36.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.148 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 02:49:36.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.148 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:49:36.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:36.149 | INFO | Uploaded 36GiB to GPU in 2.41179s (14.9267GiB/s) | 20260508 02:49:36.149 | INFO | Downloaded 34GiB to GPU in 2.5853s (13.1513GiB/s) | 20260508 02:49:41.285 | INFO | Analyzing chunk frames = [4647424,4647936) | 20260508 02:49:41.285 | INFO | MemInfo: { | 20260508 02:49:41.285 | INFO | "gpu" : " 17 GB", | 20260508 02:49:41.285 | INFO | "huge" : "", | 20260508 02:49:41.285 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:41.285 | INFO | "malloc" : " 14 GB", | 20260508 02:49:41.285 | INFO | "pinned" : " 29 GB" | 20260508 02:49:41.285 | INFO | } | 20260508 02:49:41.285 | INFO | Perf report for the last 5.13676 seconds: | 20260508 02:49:41.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:41.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:49:41.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:49:41.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:41.285 | INFO | Uploaded 36GiB to GPU in 2.3946s (15.0338GiB/s) | 20260508 02:49:41.285 | INFO | Downloaded 34GiB to GPU in 2.42481s (14.0217GiB/s) | 20260508 02:49:46.427 | INFO | Analyzing chunk frames = [4647936,4648448) | 20260508 02:49:46.427 | INFO | MemInfo: { | 20260508 02:49:46.427 | INFO | "gpu" : " 17 GB", | 20260508 02:49:46.427 | INFO | "huge" : "", | 20260508 02:49:46.427 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:46.428 | INFO | "malloc" : " 14 GB", | 20260508 02:49:46.428 | INFO | "pinned" : " 29 GB" | 20260508 02:49:46.428 | INFO | } | 20260508 02:49:46.428 | INFO | Perf report for the last 5.14272 seconds: | 20260508 02:49:46.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:46.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:49:46.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:49:46.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:46.428 | INFO | Uploaded 36GiB to GPU in 2.43728s (14.7706GiB/s) | 20260508 02:49:46.428 | INFO | Downloaded 34GiB to GPU in 2.43351s (13.9716GiB/s) | 20260508 02:49:51.567 | INFO | Analyzing chunk frames = [4648448,4648960) | 20260508 02:49:51.567 | INFO | MemInfo: { | 20260508 02:49:51.567 | INFO | "gpu" : " 17 GB", | 20260508 02:49:51.567 | INFO | "huge" : "", | 20260508 02:49:51.567 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:51.567 | INFO | "malloc" : " 14 GB", | 20260508 02:49:51.567 | INFO | "pinned" : " 29 GB" | 20260508 02:49:51.567 | INFO | } | 20260508 02:49:51.567 | INFO | Perf report for the last 5.1394 seconds: | 20260508 02:49:51.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:51.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.567 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:49:51.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.567 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260508 02:49:51.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:51.568 | INFO | Uploaded 36GiB to GPU in 2.45632s (14.6561GiB/s) | 20260508 02:49:51.568 | INFO | Downloaded 34GiB to GPU in 2.43893s (13.9405GiB/s) | 20260508 02:49:51.664 | INFO | Starting a round of Baz IO | 20260508 02:49:55.112 | INFO | Populating RTMetrics report | 20260508 02:49:55.112 | INFO | RTMetrics report populated | 20260508 02:49:55.112 | INFO | Opening tmp RTMetrics json file | 20260508 02:49:55.126 | INFO | Serializing RTMetrics to json | 20260508 02:49:55.132 | INFO | Serialization done, swapping tmp file now... | 20260508 02:49:55.133 | INFO | RTMetrics done | 20260508 02:49:55.217 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.552 seconds | 20260508 02:49:55.217 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.44% / 0.00% / 58.56% | 20260508 02:49:55.217 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.681396% / 0% / 4.76837e-05% | 20260508 02:49:55.217 | INFO | Baz IO overall: | 20260508 02:49:55.217 | INFO | BytesWritten : 2095864063686 | 20260508 02:49:55.217 | INFO | NumEvents : 1012959793018 | 20260508 02:49:56.709 | INFO | Analyzing chunk frames = [4648960,4649472) | 20260508 02:49:56.709 | INFO | MemInfo: { | 20260508 02:49:56.709 | INFO | "gpu" : " 17 GB", | 20260508 02:49:56.709 | INFO | "huge" : "", | 20260508 02:49:56.709 | INFO | "huge_shared" : " 29 GB", | 20260508 02:49:56.709 | INFO | "malloc" : " 14 GB", | 20260508 02:49:56.709 | INFO | "pinned" : " 29 GB" | 20260508 02:49:56.709 | INFO | } | 20260508 02:49:56.709 | INFO | Perf report for the last 5.14161 seconds: | 20260508 02:49:56.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:49:56.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:56.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:56.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:56.709 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:49:56.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:49:56.709 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 02:49:56.709 | INFO | PRE_HQ : 65.4%, 1.6, 55.9ms, 0 | 20260508 02:49:56.709 | INFO | BAZWRITER : 68.7%, 3.4, 121.2ms, 0 | 20260508 02:49:56.709 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:49:56.709 | INFO | Uploaded 36GiB to GPU in 2.51642s (14.306GiB/s) | 20260508 02:49:56.709 | INFO | Downloaded 38GiB to GPU in 2.68805s (14.1366GiB/s) | 20260508 02:50:01.845 | INFO | Analyzing chunk frames = [4649472,4649984) | 20260508 02:50:01.845 | INFO | MemInfo: { | 20260508 02:50:01.845 | INFO | "gpu" : " 17 GB", | 20260508 02:50:01.845 | INFO | "huge" : "", | 20260508 02:50:01.845 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:01.845 | INFO | "malloc" : " 14 GB", | 20260508 02:50:01.845 | INFO | "pinned" : " 29 GB" | 20260508 02:50:01.845 | INFO | } | 20260508 02:50:01.845 | INFO | Perf report for the last 5.13654 seconds: | 20260508 02:50:01.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:01.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.846 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 02:50:01.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.846 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 02:50:01.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:01.846 | INFO | Uploaded 36GiB to GPU in 2.56864s (14.0152GiB/s) | 20260508 02:50:01.846 | INFO | Downloaded 34GiB to GPU in 2.42167s (14.0399GiB/s) | 20260508 02:50:06.984 | INFO | Analyzing chunk frames = [4649984,4650496) | 20260508 02:50:06.984 | INFO | MemInfo: { | 20260508 02:50:06.984 | INFO | "gpu" : " 17 GB", | 20260508 02:50:06.984 | INFO | "huge" : "", | 20260508 02:50:06.984 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:06.984 | INFO | "malloc" : " 14 GB", | 20260508 02:50:06.984 | INFO | "pinned" : " 29 GB" | 20260508 02:50:06.984 | INFO | } | 20260508 02:50:06.984 | INFO | Perf report for the last 5.13841 seconds: | 20260508 02:50:06.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:06.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 02:50:06.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:50:06.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:06.984 | INFO | Uploaded 36GiB to GPU in 2.59066s (13.8961GiB/s) | 20260508 02:50:06.984 | INFO | Downloaded 34GiB to GPU in 2.48709s (13.6706GiB/s) | 20260508 02:50:12.127 | INFO | Analyzing chunk frames = [4650496,4651008) | 20260508 02:50:12.127 | INFO | MemInfo: { | 20260508 02:50:12.127 | INFO | "gpu" : " 17 GB", | 20260508 02:50:12.127 | INFO | "huge" : "", | 20260508 02:50:12.127 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:12.127 | INFO | "malloc" : " 14 GB", | 20260508 02:50:12.127 | INFO | "pinned" : " 29 GB" | 20260508 02:50:12.127 | INFO | } | 20260508 02:50:12.127 | INFO | Perf report for the last 5.14323 seconds: | 20260508 02:50:12.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:12.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 02:50:12.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:50:12.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:12.127 | INFO | Uploaded 36GiB to GPU in 2.58154s (13.9452GiB/s) | 20260508 02:50:12.127 | INFO | Downloaded 34GiB to GPU in 2.48909s (13.6596GiB/s) | 20260508 02:50:17.265 | INFO | Analyzing chunk frames = [4651008,4651520) | 20260508 02:50:17.265 | INFO | MemInfo: { | 20260508 02:50:17.265 | INFO | "gpu" : " 17 GB", | 20260508 02:50:17.265 | INFO | "huge" : "", | 20260508 02:50:17.265 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:17.265 | INFO | "malloc" : " 14 GB", | 20260508 02:50:17.266 | INFO | "pinned" : " 29 GB" | 20260508 02:50:17.266 | INFO | } | 20260508 02:50:17.266 | INFO | Perf report for the last 5.13855 seconds: | 20260508 02:50:17.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:17.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.266 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 02:50:17.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.266 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 02:50:17.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:17.267 | INFO | Uploaded 36GiB to GPU in 2.45754s (14.6488GiB/s) | 20260508 02:50:17.267 | INFO | Downloaded 34GiB to GPU in 2.5487s (13.3401GiB/s) | 20260508 02:50:22.405 | INFO | Analyzing chunk frames = [4651520,4652032) | 20260508 02:50:22.405 | INFO | MemInfo: { | 20260508 02:50:22.405 | INFO | "gpu" : " 17 GB", | 20260508 02:50:22.405 | INFO | "huge" : "", | 20260508 02:50:22.405 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:22.405 | INFO | "malloc" : " 14 GB", | 20260508 02:50:22.405 | INFO | "pinned" : " 29 GB" | 20260508 02:50:22.405 | INFO | } | 20260508 02:50:22.405 | INFO | Perf report for the last 5.13956 seconds: | 20260508 02:50:22.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:22.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 02:50:22.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:50:22.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:22.405 | INFO | Uploaded 36GiB to GPU in 2.43369s (14.7924GiB/s) | 20260508 02:50:22.406 | INFO | Downloaded 34GiB to GPU in 2.4067s (14.1273GiB/s) | 20260508 02:50:27.541 | INFO | Analyzing chunk frames = [4652032,4652544) | 20260508 02:50:27.541 | INFO | MemInfo: { | 20260508 02:50:27.541 | INFO | "gpu" : " 17 GB", | 20260508 02:50:27.541 | INFO | "huge" : "", | 20260508 02:50:27.541 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:27.541 | INFO | "malloc" : " 14 GB", | 20260508 02:50:27.541 | INFO | "pinned" : " 29 GB" | 20260508 02:50:27.542 | INFO | } | 20260508 02:50:27.542 | INFO | Perf report for the last 5.13632 seconds: | 20260508 02:50:27.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:27.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:50:27.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:50:27.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:27.542 | INFO | Uploaded 36GiB to GPU in 2.4626s (14.6187GiB/s) | 20260508 02:50:27.542 | INFO | Downloaded 34GiB to GPU in 2.41555s (14.0755GiB/s) | 20260508 02:50:32.683 | INFO | Analyzing chunk frames = [4652544,4653056) | 20260508 02:50:32.683 | INFO | MemInfo: { | 20260508 02:50:32.683 | INFO | "gpu" : " 17 GB", | 20260508 02:50:32.683 | INFO | "huge" : "", | 20260508 02:50:32.683 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:32.683 | INFO | "malloc" : " 14 GB", | 20260508 02:50:32.683 | INFO | "pinned" : " 29 GB" | 20260508 02:50:32.683 | INFO | } | 20260508 02:50:32.683 | INFO | Perf report for the last 5.14168 seconds: | 20260508 02:50:32.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:32.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.683 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 02:50:32.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.683 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:50:32.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:32.684 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:50:32.684 | INFO | Total mem usage: 16.7 GiB | 20260508 02:50:32.684 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:50:32.684 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.7%) | 20260508 02:50:32.684 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:50:32.684 | INFO | High Water Breakdown: | 20260508 02:50:32.684 | INFO | 60% : 8.89 GiB ( 86.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:50:32.684 | INFO | 39.8% : 5.9 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:50:32.684 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:50:32.684 | INFO | ------------------------------------------ | 20260508 02:50:32.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:50:32.684 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:50:32.684 | INFO | Total mem usage: 26.5 GiB | 20260508 02:50:32.684 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:50:32.684 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 02:50:32.684 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 02:50:32.684 | INFO | High Water Breakdown: | 20260508 02:50:32.684 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:50:32.684 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:50:32.684 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:50:32.684 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:50:32.684 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:50:32.684 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:50:32.684 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:50:32.684 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:50:32.684 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:50:32.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:50:32.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:50:32.684 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:50:32.684 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:50:32.684 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:50:32.684 | INFO | ------------------------------------------ | 20260508 02:50:32.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:50:32.684 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:50:32.684 | INFO | Total mem usage: 27.8 GiB | 20260508 02:50:32.684 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:50:32.684 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:50:32.684 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:50:32.684 | INFO | High Water Breakdown: | 20260508 02:50:32.684 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:50:32.684 | INFO | ------------------------------------------ | 20260508 02:50:32.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:50:32.684 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:50:32.684 | INFO | Total mem usage: 15.7 GiB | 20260508 02:50:32.684 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:50:32.684 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:50:32.684 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:50:32.684 | INFO | High Water Breakdown: | 20260508 02:50:32.684 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:50:32.684 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:50:32.684 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:50:32.684 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:50:32.684 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:50:32.684 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:50:32.684 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:50:32.684 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:50:32.684 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:50:32.684 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:50:32.684 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:50:32.684 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:50:32.684 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:50:32.684 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:50:32.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:50:32.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:50:32.684 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:50:32.684 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:50:32.684 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:50:32.684 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:50:32.684 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:50:32.684 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:50:32.684 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:50:32.684 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:50:32.684 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:50:32.684 | INFO | ------------------------------------------ | 20260508 02:50:32.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:50:32.684 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:50:32.684 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:50:32.684 | INFO | Uploaded 36GiB to GPU in 2.56893s (14.0136GiB/s) | 20260508 02:50:32.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:50:32.684 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.41e+01 / 1.64e+00 | 20260508 02:50:32.684 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.01e-01, 1.14e-02 / 5.32e-03 | 20260508 02:50:32.684 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.46e+00, 5.82e-01 / 1.66e-01 | 20260508 02:50:32.684 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.45e+00, 2.07e-01 / 1.20e-01 | 20260508 02:50:32.684 | INFO | Downloaded 34GiB to GPU in 2.44929s (13.8815GiB/s) | 20260508 02:50:32.684 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.59e+01, 4.13e+00 / 1.98e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.70e+00 / 1.42e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.41e+00 / 2.62e+00, 4.18e-01 / 2.51e-01 | 20260508 02:50:32.685 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.21e+00, 2.63e-01 / 1.24e-01 | 20260508 02:50:32.685 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.56e-01, 5.07e-02 / 2.19e-02 | 20260508 02:50:32.685 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:50:32.685 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:50:32.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:50:32.685 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 24.49%, Upload -- 2.61e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 2.25%, LockUpload -- 2.40e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 2.70%, LockDownload -- 2.87e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 15.42%, LockCompute -- 1.64e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:50:32.685 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.34e-02 / 0.00e+00 | 20260508 02:50:32.782 | INFO | Starting a round of Baz IO | 20260508 02:50:36.231 | INFO | Populating RTMetrics report | 20260508 02:50:36.232 | INFO | RTMetrics report populated | 20260508 02:50:36.232 | INFO | Opening tmp RTMetrics json file | 20260508 02:50:36.253 | INFO | Serializing RTMetrics to json | 20260508 02:50:36.254 | INFO | Serialization done, swapping tmp file now... | 20260508 02:50:36.255 | INFO | RTMetrics done | 20260508 02:50:36.318 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.536 seconds | 20260508 02:50:36.318 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.43% / 0.00% / 58.56% | 20260508 02:50:36.318 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.679036% / 3.97364e-06% / 4.37101e-05% | 20260508 02:50:36.318 | INFO | Baz IO overall: | 20260508 02:50:36.318 | INFO | BytesWritten : 2097612309103 | 20260508 02:50:36.318 | INFO | NumEvents : 1013797190003 | 20260508 02:50:37.823 | INFO | Analyzing chunk frames = [4653056,4653568) | 20260508 02:50:37.823 | INFO | MemInfo: { | 20260508 02:50:37.823 | INFO | "gpu" : " 17 GB", | 20260508 02:50:37.823 | INFO | "huge" : "", | 20260508 02:50:37.823 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:37.823 | INFO | "malloc" : " 14 GB", | 20260508 02:50:37.823 | INFO | "pinned" : " 29 GB" | 20260508 02:50:37.823 | INFO | } | 20260508 02:50:37.823 | INFO | Perf report for the last 5.13986 seconds: | 20260508 02:50:37.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:37.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:37.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:37.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:37.823 | INFO | ANALYSIS : 68.3%, 3.0, 106.5ms, 0 | 20260508 02:50:37.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:37.823 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 02:50:37.823 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 02:50:37.823 | INFO | BAZWRITER : 68.4%, 3.5, 121.5ms, 0 | 20260508 02:50:37.823 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:50:37.824 | INFO | Uploaded 36GiB to GPU in 2.5183s (14.2954GiB/s) | 20260508 02:50:37.824 | INFO | Downloaded 38GiB to GPU in 2.69703s (14.0896GiB/s) | 20260508 02:50:42.965 | INFO | Analyzing chunk frames = [4653568,4654080) | 20260508 02:50:42.965 | INFO | MemInfo: { | 20260508 02:50:42.965 | INFO | "gpu" : " 17 GB", | 20260508 02:50:42.965 | INFO | "huge" : "", | 20260508 02:50:42.965 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:42.965 | INFO | "malloc" : " 14 GB", | 20260508 02:50:42.965 | INFO | "pinned" : " 29 GB" | 20260508 02:50:42.965 | INFO | } | 20260508 02:50:42.965 | INFO | Perf report for the last 5.14235 seconds: | 20260508 02:50:42.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:42.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 02:50:42.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:50:42.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:42.966 | INFO | Uploaded 36GiB to GPU in 2.57841s (13.9621GiB/s) | 20260508 02:50:42.966 | INFO | Downloaded 34GiB to GPU in 2.41727s (14.0655GiB/s) | 20260508 02:50:48.098 | INFO | Analyzing chunk frames = [4654080,4654592) | 20260508 02:50:48.098 | INFO | MemInfo: { | 20260508 02:50:48.098 | INFO | "gpu" : " 17 GB", | 20260508 02:50:48.098 | INFO | "huge" : "", | 20260508 02:50:48.098 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:48.098 | INFO | "malloc" : " 14 GB", | 20260508 02:50:48.098 | INFO | "pinned" : " 29 GB" | 20260508 02:50:48.098 | INFO | } | 20260508 02:50:48.098 | INFO | Perf report for the last 5.13281 seconds: | 20260508 02:50:48.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:48.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.098 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260508 02:50:48.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.098 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:50:48.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:48.099 | INFO | Uploaded 36GiB to GPU in 2.56777s (14.02GiB/s) | 20260508 02:50:48.099 | INFO | Downloaded 34GiB to GPU in 2.48945s (13.6576GiB/s) | 20260508 02:50:53.235 | INFO | Analyzing chunk frames = [4654592,4655104) | 20260508 02:50:53.235 | INFO | MemInfo: { | 20260508 02:50:53.235 | INFO | "gpu" : " 17 GB", | 20260508 02:50:53.235 | INFO | "huge" : "", | 20260508 02:50:53.235 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:53.235 | INFO | "malloc" : " 14 GB", | 20260508 02:50:53.235 | INFO | "pinned" : " 29 GB" | 20260508 02:50:53.235 | INFO | } | 20260508 02:50:53.235 | INFO | Perf report for the last 5.1366 seconds: | 20260508 02:50:53.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:53.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.235 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 02:50:53.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.235 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:50:53.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:53.236 | INFO | Uploaded 36GiB to GPU in 2.57154s (13.9994GiB/s) | 20260508 02:50:53.236 | INFO | Downloaded 34GiB to GPU in 2.49407s (13.6323GiB/s) | 20260508 02:50:58.377 | INFO | Analyzing chunk frames = [4655104,4655616) | 20260508 02:50:58.378 | INFO | MemInfo: { | 20260508 02:50:58.378 | INFO | "gpu" : " 17 GB", | 20260508 02:50:58.378 | INFO | "huge" : "", | 20260508 02:50:58.378 | INFO | "huge_shared" : " 29 GB", | 20260508 02:50:58.378 | INFO | "malloc" : " 14 GB", | 20260508 02:50:58.378 | INFO | "pinned" : " 29 GB" | 20260508 02:50:58.378 | INFO | } | 20260508 02:50:58.378 | INFO | Perf report for the last 5.14281 seconds: | 20260508 02:50:58.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:50:58.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 02:50:58.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 02:50:58.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:50:58.378 | INFO | Uploaded 36GiB to GPU in 2.34733s (15.3366GiB/s) | 20260508 02:50:58.378 | INFO | Downloaded 34GiB to GPU in 2.49665s (13.6182GiB/s) | 20260508 02:51:03.517 | INFO | Analyzing chunk frames = [4655616,4656128) | 20260508 02:51:03.517 | INFO | MemInfo: { | 20260508 02:51:03.517 | INFO | "gpu" : " 17 GB", | 20260508 02:51:03.517 | INFO | "huge" : "", | 20260508 02:51:03.517 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:03.517 | INFO | "malloc" : " 14 GB", | 20260508 02:51:03.517 | INFO | "pinned" : " 29 GB" | 20260508 02:51:03.517 | INFO | } | 20260508 02:51:03.517 | INFO | Perf report for the last 5.1391 seconds: | 20260508 02:51:03.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:03.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:51:03.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:51:03.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:03.517 | INFO | Uploaded 36GiB to GPU in 2.41803s (14.8881GiB/s) | 20260508 02:51:03.517 | INFO | Downloaded 34GiB to GPU in 2.42115s (14.0429GiB/s) | 20260508 02:51:08.658 | INFO | Analyzing chunk frames = [4656128,4656640) | 20260508 02:51:08.658 | INFO | MemInfo: { | 20260508 02:51:08.658 | INFO | "gpu" : " 17 GB", | 20260508 02:51:08.658 | INFO | "huge" : "", | 20260508 02:51:08.658 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:08.658 | INFO | "malloc" : " 14 GB", | 20260508 02:51:08.659 | INFO | "pinned" : " 29 GB" | 20260508 02:51:08.659 | INFO | } | 20260508 02:51:08.659 | INFO | Perf report for the last 5.14172 seconds: | 20260508 02:51:08.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:08.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:51:08.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:51:08.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:08.659 | INFO | Uploaded 36GiB to GPU in 2.43543s (14.7818GiB/s) | 20260508 02:51:08.659 | INFO | Downloaded 34GiB to GPU in 2.41933s (14.0535GiB/s) | 20260508 02:51:13.796 | INFO | Analyzing chunk frames = [4656640,4657152) | 20260508 02:51:13.796 | INFO | MemInfo: { | 20260508 02:51:13.796 | INFO | "gpu" : " 17 GB", | 20260508 02:51:13.796 | INFO | "huge" : "", | 20260508 02:51:13.796 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:13.796 | INFO | "malloc" : " 14 GB", | 20260508 02:51:13.796 | INFO | "pinned" : " 29 GB" | 20260508 02:51:13.796 | INFO | } | 20260508 02:51:13.796 | INFO | Perf report for the last 5.13755 seconds: | 20260508 02:51:13.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:13.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.796 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:51:13.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.796 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:51:13.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:13.797 | INFO | Uploaded 36GiB to GPU in 2.55195s (14.1069GiB/s) | 20260508 02:51:13.797 | INFO | Downloaded 34GiB to GPU in 2.45919s (13.8257GiB/s) | 20260508 02:51:13.900 | INFO | Starting a round of Baz IO | 20260508 02:51:17.319 | INFO | Populating RTMetrics report | 20260508 02:51:17.319 | INFO | RTMetrics report populated | 20260508 02:51:17.319 | INFO | Opening tmp RTMetrics json file | 20260508 02:51:17.329 | INFO | Serializing RTMetrics to json | 20260508 02:51:17.340 | INFO | Serialization done, swapping tmp file now... | 20260508 02:51:17.341 | INFO | RTMetrics done | 20260508 02:51:17.427 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.526 seconds | 20260508 02:51:17.427 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.43% / 0.00% / 58.57% | 20260508 02:51:17.427 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.676314% / 7.94729e-06% / 5.16574e-05% | 20260508 02:51:17.427 | INFO | Baz IO overall: | 20260508 02:51:17.427 | INFO | BytesWritten : 2099360553329 | 20260508 02:51:17.427 | INFO | NumEvents : 1014634572922 | 20260508 02:51:18.937 | INFO | Analyzing chunk frames = [4657152,4657664) | 20260508 02:51:18.937 | INFO | MemInfo: { | 20260508 02:51:18.937 | INFO | "gpu" : " 17 GB", | 20260508 02:51:18.937 | INFO | "huge" : "", | 20260508 02:51:18.937 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:18.937 | INFO | "malloc" : " 14 GB", | 20260508 02:51:18.937 | INFO | "pinned" : " 29 GB" | 20260508 02:51:18.937 | INFO | } | 20260508 02:51:18.937 | INFO | Perf report for the last 5.1413 seconds: | 20260508 02:51:18.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:18.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:18.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:18.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:18.938 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 02:51:18.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:18.938 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 02:51:18.938 | INFO | PRE_HQ : 65.7%, 1.6, 56.0ms, 0 | 20260508 02:51:18.938 | INFO | BAZWRITER : 68.4%, 3.5, 123.8ms, 0 | 20260508 02:51:18.938 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 02:51:18.938 | INFO | Uploaded 36GiB to GPU in 2.5589s (14.0685GiB/s) | 20260508 02:51:18.938 | INFO | Downloaded 38GiB to GPU in 2.70818s (14.0316GiB/s) | 20260508 02:51:24.076 | INFO | Analyzing chunk frames = [4657664,4658176) | 20260508 02:51:24.076 | INFO | MemInfo: { | 20260508 02:51:24.076 | INFO | "gpu" : " 17 GB", | 20260508 02:51:24.076 | INFO | "huge" : "", | 20260508 02:51:24.077 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:24.077 | INFO | "malloc" : " 14 GB", | 20260508 02:51:24.077 | INFO | "pinned" : " 29 GB" | 20260508 02:51:24.077 | INFO | } | 20260508 02:51:24.077 | INFO | Perf report for the last 5.13922 seconds: | 20260508 02:51:24.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:24.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:51:24.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:51:24.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:24.077 | INFO | Uploaded 36GiB to GPU in 2.51641s (14.3061GiB/s) | 20260508 02:51:24.077 | INFO | Downloaded 34GiB to GPU in 2.44464s (13.908GiB/s) | 20260508 02:51:29.215 | INFO | Analyzing chunk frames = [4658176,4658688) | 20260508 02:51:29.215 | INFO | MemInfo: { | 20260508 02:51:29.215 | INFO | "gpu" : " 17 GB", | 20260508 02:51:29.215 | INFO | "huge" : "", | 20260508 02:51:29.215 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:29.215 | INFO | "malloc" : " 14 GB", | 20260508 02:51:29.215 | INFO | "pinned" : " 29 GB" | 20260508 02:51:29.215 | INFO | } | 20260508 02:51:29.215 | INFO | Perf report for the last 5.13871 seconds: | 20260508 02:51:29.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:29.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.216 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:51:29.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.216 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:51:29.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:29.216 | INFO | Uploaded 36GiB to GPU in 2.52491s (14.2579GiB/s) | 20260508 02:51:29.216 | INFO | Downloaded 34GiB to GPU in 2.5129s (13.5302GiB/s) | 20260508 02:51:34.355 | INFO | Analyzing chunk frames = [4658688,4659200) | 20260508 02:51:34.355 | INFO | MemInfo: { | 20260508 02:51:34.355 | INFO | "gpu" : " 17 GB", | 20260508 02:51:34.355 | INFO | "huge" : "", | 20260508 02:51:34.355 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:34.355 | INFO | "malloc" : " 14 GB", | 20260508 02:51:34.355 | INFO | "pinned" : " 29 GB" | 20260508 02:51:34.355 | INFO | } | 20260508 02:51:34.355 | INFO | Perf report for the last 5.13959 seconds: | 20260508 02:51:34.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:34.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 02:51:34.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:51:34.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:34.355 | INFO | Uploaded 36GiB to GPU in 2.51097s (14.3371GiB/s) | 20260508 02:51:34.355 | INFO | Downloaded 34GiB to GPU in 2.51479s (13.52GiB/s) | 20260508 02:51:39.493 | INFO | Analyzing chunk frames = [4659200,4659712) | 20260508 02:51:39.493 | INFO | MemInfo: { | 20260508 02:51:39.493 | INFO | "gpu" : " 17 GB", | 20260508 02:51:39.493 | INFO | "huge" : "", | 20260508 02:51:39.493 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:39.493 | INFO | "malloc" : " 14 GB", | 20260508 02:51:39.493 | INFO | "pinned" : " 29 GB" | 20260508 02:51:39.493 | INFO | } | 20260508 02:51:39.493 | INFO | Perf report for the last 5.13822 seconds: | 20260508 02:51:39.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:39.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.493 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 02:51:39.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.493 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:51:39.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:39.494 | INFO | Uploaded 36GiB to GPU in 2.37018s (15.1887GiB/s) | 20260508 02:51:39.494 | INFO | Downloaded 34GiB to GPU in 2.54588s (13.3549GiB/s) | 20260508 02:51:44.637 | INFO | Analyzing chunk frames = [4659712,4660224) | 20260508 02:51:44.637 | INFO | MemInfo: { | 20260508 02:51:44.637 | INFO | "gpu" : " 17 GB", | 20260508 02:51:44.637 | INFO | "huge" : "", | 20260508 02:51:44.637 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:44.637 | INFO | "malloc" : " 14 GB", | 20260508 02:51:44.637 | INFO | "pinned" : " 29 GB" | 20260508 02:51:44.637 | INFO | } | 20260508 02:51:44.637 | INFO | Perf report for the last 5.14352 seconds: | 20260508 02:51:44.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:44.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 02:51:44.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:51:44.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:44.637 | INFO | Uploaded 36GiB to GPU in 2.37911s (15.1317GiB/s) | 20260508 02:51:44.637 | INFO | Downloaded 34GiB to GPU in 2.44742s (13.8922GiB/s) | 20260508 02:51:49.773 | INFO | Analyzing chunk frames = [4660224,4660736) | 20260508 02:51:49.773 | INFO | MemInfo: { | 20260508 02:51:49.773 | INFO | "gpu" : " 17 GB", | 20260508 02:51:49.773 | INFO | "huge" : "", | 20260508 02:51:49.773 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:49.773 | INFO | "malloc" : " 14 GB", | 20260508 02:51:49.773 | INFO | "pinned" : " 29 GB" | 20260508 02:51:49.773 | INFO | } | 20260508 02:51:49.773 | INFO | Perf report for the last 5.13608 seconds: | 20260508 02:51:49.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:49.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 02:51:49.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:51:49.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:49.773 | INFO | Uploaded 36GiB to GPU in 2.41138s (14.9292GiB/s) | 20260508 02:51:49.773 | INFO | Downloaded 34GiB to GPU in 2.44011s (13.9338GiB/s) | 20260508 02:51:54.914 | INFO | Analyzing chunk frames = [4660736,4661248) | 20260508 02:51:54.914 | INFO | MemInfo: { | 20260508 02:51:54.914 | INFO | "gpu" : " 17 GB", | 20260508 02:51:54.914 | INFO | "huge" : "", | 20260508 02:51:54.914 | INFO | "huge_shared" : " 29 GB", | 20260508 02:51:54.914 | INFO | "malloc" : " 14 GB", | 20260508 02:51:54.914 | INFO | "pinned" : " 29 GB" | 20260508 02:51:54.914 | INFO | } | 20260508 02:51:54.914 | INFO | Perf report for the last 5.14106 seconds: | 20260508 02:51:54.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:51:54.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 02:51:54.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 02:51:54.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:51:54.914 | INFO | Uploaded 36GiB to GPU in 2.53417s (14.2058GiB/s) | 20260508 02:51:54.914 | INFO | Downloaded 34GiB to GPU in 2.46867s (13.7726GiB/s) | 20260508 02:51:54.914 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:51:54.914 | INFO | Total mem usage: 16.7 GiB | 20260508 02:51:54.914 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:51:54.914 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 02:51:54.914 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:51:54.914 | INFO | High Water Breakdown: | 20260508 02:51:54.914 | INFO | 59.8% : 8.73 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:51:54.914 | INFO | 40.1% : 5.84 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:51:54.914 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:51:54.914 | INFO | ------------------------------------------ | 20260508 02:51:54.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:51:54.914 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:51:54.914 | INFO | Total mem usage: 26.5 GiB | 20260508 02:51:54.914 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:51:54.914 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:51:54.914 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 02:51:54.914 | INFO | High Water Breakdown: | 20260508 02:51:54.914 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:51:54.914 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:51:54.914 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:51:54.914 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:51:54.914 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:51:54.914 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:51:54.914 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:51:54.914 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:51:54.914 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:51:54.914 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:51:54.914 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:51:54.914 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:51:54.914 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:51:54.914 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:51:54.914 | INFO | ------------------------------------------ | 20260508 02:51:54.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:51:54.914 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:51:54.914 | INFO | Total mem usage: 27.8 GiB | 20260508 02:51:54.914 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:51:54.914 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:51:54.915 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:51:54.915 | INFO | High Water Breakdown: | 20260508 02:51:54.915 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:51:54.915 | INFO | ------------------------------------------ | 20260508 02:51:54.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:51:54.915 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:51:54.915 | INFO | Total mem usage: 15.7 GiB | 20260508 02:51:54.915 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:51:54.915 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:51:54.915 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:51:54.915 | INFO | High Water Breakdown: | 20260508 02:51:54.915 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:51:54.915 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:51:54.915 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:51:54.915 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:51:54.915 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:51:54.915 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:51:54.915 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:51:54.915 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:51:54.915 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:51:54.915 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:51:54.915 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:51:54.915 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:51:54.915 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:51:54.915 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:51:54.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:51:54.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:51:54.915 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:51:54.915 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:51:54.915 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:51:54.915 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:51:54.915 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:51:54.915 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:51:54.915 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:51:54.915 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:51:54.915 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:51:54.915 | INFO | ------------------------------------------ | 20260508 02:51:54.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:51:54.915 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:51:54.915 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:51:54.915 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:51:54.915 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.60e+01, 1.41e+01 / 1.38e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.22e-02 / 8.49e-03 | 20260508 02:51:54.915 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.46e+00, 5.81e-01 / 1.63e-01 | 20260508 02:51:54.915 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.45e+00, 2.03e-01 / 1.19e-01 | 20260508 02:51:54.915 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.61e+01, 3.95e+00 / 1.67e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.61e+00 / 1.51e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.42e+00 / 2.67e+00, 4.21e-01 / 2.18e-01 | 20260508 02:51:54.915 | INFO | PerfReport: 3.29%, Metrics -- 1.07e+00 / 1.25e+00, 2.64e-01 / 1.42e-01 | 20260508 02:51:54.915 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.66e-01, 5.10e-02 / 2.21e-02 | 20260508 02:51:54.915 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:51:54.915 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:51:54.915 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:51:54.915 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 24.36%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 24.47%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 30.66%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 2.86%, LockDownload -- 3.04e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 15.48%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:51:54.915 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 3.11e-02 / 0.00e+00 | 20260508 02:51:55.011 | INFO | Starting a round of Baz IO | 20260508 02:51:58.444 | INFO | Populating RTMetrics report | 20260508 02:51:58.445 | INFO | RTMetrics report populated | 20260508 02:51:58.445 | INFO | Opening tmp RTMetrics json file | 20260508 02:51:58.464 | INFO | Serializing RTMetrics to json | 20260508 02:51:58.466 | INFO | Serialization done, swapping tmp file now... | 20260508 02:51:58.466 | INFO | RTMetrics done | 20260508 02:51:58.532 | INFO | Finished a round of Baz IO: Wrote 1.627 GiB in 3.521 seconds | 20260508 02:51:58.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.43% / 0.00% / 58.57% | 20260508 02:51:58.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.675356% / 7.94729e-06% / 3.97364e-05% | 20260508 02:51:58.532 | INFO | Baz IO overall: | 20260508 02:51:58.532 | INFO | BytesWritten : 2101107754652 | 20260508 02:51:58.532 | INFO | NumEvents : 1015471247885 | 20260508 02:52:00.048 | INFO | Analyzing chunk frames = [4661248,4661760) | 20260508 02:52:00.048 | INFO | MemInfo: { | 20260508 02:52:00.048 | INFO | "gpu" : " 17 GB", | 20260508 02:52:00.048 | INFO | "huge" : "", | 20260508 02:52:00.048 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:00.048 | INFO | "malloc" : " 14 GB", | 20260508 02:52:00.048 | INFO | "pinned" : " 29 GB" | 20260508 02:52:00.048 | INFO | } | 20260508 02:52:00.048 | INFO | Perf report for the last 5.13439 seconds: | 20260508 02:52:00.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:00.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:00.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:00.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:00.048 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 02:52:00.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:00.048 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 02:52:00.048 | INFO | PRE_HQ : 65.6%, 1.6, 55.9ms, 0 | 20260508 02:52:00.049 | INFO | BAZWRITER : 68.2%, 3.9, 135.3ms, 0 | 20260508 02:52:00.049 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 02:52:00.049 | INFO | Uploaded 36GiB to GPU in 2.54455s (14.1479GiB/s) | 20260508 02:52:00.049 | INFO | Downloaded 38GiB to GPU in 2.70352s (14.0558GiB/s) | 20260508 02:52:05.190 | INFO | Analyzing chunk frames = [4661760,4662272) | 20260508 02:52:05.190 | INFO | MemInfo: { | 20260508 02:52:05.190 | INFO | "gpu" : " 17 GB", | 20260508 02:52:05.190 | INFO | "huge" : "", | 20260508 02:52:05.190 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:05.190 | INFO | "malloc" : " 14 GB", | 20260508 02:52:05.190 | INFO | "pinned" : " 29 GB" | 20260508 02:52:05.190 | INFO | } | 20260508 02:52:05.191 | INFO | Perf report for the last 5.14216 seconds: | 20260508 02:52:05.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:05.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 02:52:05.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 02:52:05.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:05.191 | INFO | Uploaded 36GiB to GPU in 2.55084s (14.113GiB/s) | 20260508 02:52:05.191 | INFO | Downloaded 34GiB to GPU in 2.43644s (13.9548GiB/s) | 20260508 02:52:10.327 | INFO | Analyzing chunk frames = [4662272,4662784) | 20260508 02:52:10.327 | INFO | MemInfo: { | 20260508 02:52:10.327 | INFO | "gpu" : " 17 GB", | 20260508 02:52:10.327 | INFO | "huge" : "", | 20260508 02:52:10.327 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:10.327 | INFO | "malloc" : " 14 GB", | 20260508 02:52:10.328 | INFO | "pinned" : " 29 GB" | 20260508 02:52:10.328 | INFO | } | 20260508 02:52:10.328 | INFO | Perf report for the last 5.13716 seconds: | 20260508 02:52:10.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:10.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 02:52:10.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:52:10.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:10.328 | INFO | Uploaded 36GiB to GPU in 2.55462s (14.0921GiB/s) | 20260508 02:52:10.328 | INFO | Downloaded 34GiB to GPU in 2.50026s (13.5986GiB/s) | 20260508 02:52:15.468 | INFO | Analyzing chunk frames = [4662784,4663296) | 20260508 02:52:15.468 | INFO | MemInfo: { | 20260508 02:52:15.468 | INFO | "gpu" : " 17 GB", | 20260508 02:52:15.468 | INFO | "huge" : "", | 20260508 02:52:15.468 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:15.468 | INFO | "malloc" : " 14 GB", | 20260508 02:52:15.469 | INFO | "pinned" : " 29 GB" | 20260508 02:52:15.469 | INFO | } | 20260508 02:52:15.469 | INFO | Perf report for the last 5.14097 seconds: | 20260508 02:52:15.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:15.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:52:15.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:52:15.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:15.469 | INFO | Uploaded 36GiB to GPU in 2.53227s (14.2165GiB/s) | 20260508 02:52:15.469 | INFO | Downloaded 34GiB to GPU in 2.48579s (13.6777GiB/s) | 20260508 02:52:20.607 | INFO | Analyzing chunk frames = [4663296,4663808) | 20260508 02:52:20.607 | INFO | MemInfo: { | 20260508 02:52:20.607 | INFO | "gpu" : " 17 GB", | 20260508 02:52:20.607 | INFO | "huge" : "", | 20260508 02:52:20.607 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:20.607 | INFO | "malloc" : " 14 GB", | 20260508 02:52:20.607 | INFO | "pinned" : " 29 GB" | 20260508 02:52:20.607 | INFO | } | 20260508 02:52:20.607 | INFO | Perf report for the last 5.13876 seconds: | 20260508 02:52:20.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:20.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.608 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 02:52:20.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.608 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 02:52:20.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:20.608 | INFO | Uploaded 36GiB to GPU in 2.44705s (14.7116GiB/s) | 20260508 02:52:20.608 | INFO | Downloaded 34GiB to GPU in 2.59586s (13.0978GiB/s) | 20260508 02:52:25.746 | INFO | Analyzing chunk frames = [4663808,4664320) | 20260508 02:52:25.746 | INFO | MemInfo: { | 20260508 02:52:25.746 | INFO | "gpu" : " 17 GB", | 20260508 02:52:25.746 | INFO | "huge" : "", | 20260508 02:52:25.746 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:25.746 | INFO | "malloc" : " 14 GB", | 20260508 02:52:25.746 | INFO | "pinned" : " 29 GB" | 20260508 02:52:25.746 | INFO | } | 20260508 02:52:25.746 | INFO | Perf report for the last 5.13875 seconds: | 20260508 02:52:25.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:25.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:52:25.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 02:52:25.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:25.746 | INFO | Uploaded 36GiB to GPU in 2.42178s (14.8651GiB/s) | 20260508 02:52:25.747 | INFO | Downloaded 34GiB to GPU in 2.40145s (14.1581GiB/s) | 20260508 02:52:30.888 | INFO | Analyzing chunk frames = [4664320,4664832) | 20260508 02:52:30.888 | INFO | MemInfo: { | 20260508 02:52:30.888 | INFO | "gpu" : " 17 GB", | 20260508 02:52:30.888 | INFO | "huge" : "", | 20260508 02:52:30.888 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:30.889 | INFO | "malloc" : " 14 GB", | 20260508 02:52:30.889 | INFO | "pinned" : " 29 GB" | 20260508 02:52:30.889 | INFO | } | 20260508 02:52:30.889 | INFO | Perf report for the last 5.14245 seconds: | 20260508 02:52:30.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:30.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 02:52:30.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:52:30.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:30.889 | INFO | Uploaded 36GiB to GPU in 2.45216s (14.681GiB/s) | 20260508 02:52:30.889 | INFO | Downloaded 34GiB to GPU in 2.41384s (14.0854GiB/s) | 20260508 02:52:36.030 | INFO | Analyzing chunk frames = [4664832,4665344) | 20260508 02:52:36.030 | INFO | MemInfo: { | 20260508 02:52:36.030 | INFO | "gpu" : " 17 GB", | 20260508 02:52:36.030 | INFO | "huge" : "", | 20260508 02:52:36.030 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:36.030 | INFO | "malloc" : " 14 GB", | 20260508 02:52:36.030 | INFO | "pinned" : " 29 GB" | 20260508 02:52:36.030 | INFO | } | 20260508 02:52:36.030 | INFO | Perf report for the last 5.14127 seconds: | 20260508 02:52:36.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:36.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.030 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:52:36.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.030 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:52:36.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:36.031 | INFO | Uploaded 36GiB to GPU in 2.57457s (13.9829GiB/s) | 20260508 02:52:36.031 | INFO | Downloaded 34GiB to GPU in 2.45713s (13.8373GiB/s) | 20260508 02:52:36.154 | INFO | Starting a round of Baz IO | 20260508 02:52:39.589 | INFO | Populating RTMetrics report | 20260508 02:52:39.589 | INFO | RTMetrics report populated | 20260508 02:52:39.589 | INFO | Opening tmp RTMetrics json file | 20260508 02:52:39.590 | INFO | Serializing RTMetrics to json | 20260508 02:52:39.592 | INFO | Serialization done, swapping tmp file now... | 20260508 02:52:39.593 | INFO | RTMetrics done | 20260508 02:52:39.681 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.526 seconds | 20260508 02:52:39.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.42% / 0.00% / 58.57% | 20260508 02:52:39.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.672722% / 7.94729e-06% / 5.5631e-05% | 20260508 02:52:39.681 | INFO | Baz IO overall: | 20260508 02:52:39.681 | INFO | BytesWritten : 2102855560949 | 20260508 02:52:39.681 | INFO | NumEvents : 1016308373106 | 20260508 02:52:41.169 | INFO | Analyzing chunk frames = [4665344,4665856) | 20260508 02:52:41.169 | INFO | MemInfo: { | 20260508 02:52:41.169 | INFO | "gpu" : " 17 GB", | 20260508 02:52:41.169 | INFO | "huge" : "", | 20260508 02:52:41.169 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:41.169 | INFO | "malloc" : " 14 GB", | 20260508 02:52:41.169 | INFO | "pinned" : " 29 GB" | 20260508 02:52:41.169 | INFO | } | 20260508 02:52:41.169 | INFO | Perf report for the last 5.13875 seconds: | 20260508 02:52:41.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:41.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:41.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:41.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:41.169 | INFO | ANALYSIS : 68.9%, 2.9, 104.4ms, 0 | 20260508 02:52:41.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:41.169 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 02:52:41.169 | INFO | PRE_HQ : 67.7%, 1.8, 62.4ms, 0 | 20260508 02:52:41.169 | INFO | BAZWRITER : 68.6%, 3.9, 136.8ms, 0 | 20260508 02:52:41.169 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 02:52:41.169 | INFO | Uploaded 36GiB to GPU in 2.57397s (13.9862GiB/s) | 20260508 02:52:41.169 | INFO | Downloaded 38GiB to GPU in 2.69921s (14.0782GiB/s) | 20260508 02:52:46.301 | INFO | Analyzing chunk frames = [4665856,4666368) | 20260508 02:52:46.301 | INFO | MemInfo: { | 20260508 02:52:46.301 | INFO | "gpu" : " 17 GB", | 20260508 02:52:46.301 | INFO | "huge" : "", | 20260508 02:52:46.301 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:46.301 | INFO | "malloc" : " 14 GB", | 20260508 02:52:46.301 | INFO | "pinned" : " 29 GB" | 20260508 02:52:46.301 | INFO | } | 20260508 02:52:46.302 | INFO | Perf report for the last 5.13275 seconds: | 20260508 02:52:46.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:46.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:52:46.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:52:46.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:46.302 | INFO | Uploaded 36GiB to GPU in 2.56724s (14.0228GiB/s) | 20260508 02:52:46.302 | INFO | Downloaded 34GiB to GPU in 2.43401s (13.9687GiB/s) | 20260508 02:52:51.448 | INFO | Analyzing chunk frames = [4666368,4666880) | 20260508 02:52:51.448 | INFO | MemInfo: { | 20260508 02:52:51.448 | INFO | "gpu" : " 17 GB", | 20260508 02:52:51.448 | INFO | "huge" : "", | 20260508 02:52:51.448 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:51.448 | INFO | "malloc" : " 14 GB", | 20260508 02:52:51.448 | INFO | "pinned" : " 29 GB" | 20260508 02:52:51.449 | INFO | } | 20260508 02:52:51.449 | INFO | Perf report for the last 5.14709 seconds: | 20260508 02:52:51.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:51.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 02:52:51.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:52:51.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:51.449 | INFO | Uploaded 36GiB to GPU in 2.53061s (14.2258GiB/s) | 20260508 02:52:51.449 | INFO | Downloaded 34GiB to GPU in 2.49694s (13.6166GiB/s) | 20260508 02:52:56.587 | INFO | Analyzing chunk frames = [4666880,4667392) | 20260508 02:52:56.587 | INFO | MemInfo: { | 20260508 02:52:56.587 | INFO | "gpu" : " 17 GB", | 20260508 02:52:56.587 | INFO | "huge" : "", | 20260508 02:52:56.587 | INFO | "huge_shared" : " 29 GB", | 20260508 02:52:56.587 | INFO | "malloc" : " 14 GB", | 20260508 02:52:56.587 | INFO | "pinned" : " 29 GB" | 20260508 02:52:56.587 | INFO | } | 20260508 02:52:56.587 | INFO | Perf report for the last 5.13877 seconds: | 20260508 02:52:56.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:52:56.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.587 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:52:56.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.588 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:52:56.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:52:56.588 | INFO | Uploaded 36GiB to GPU in 2.54298s (14.1566GiB/s) | 20260508 02:52:56.588 | INFO | Downloaded 34GiB to GPU in 2.50329s (13.5821GiB/s) | 20260508 02:53:01.726 | INFO | Analyzing chunk frames = [4667392,4667904) | 20260508 02:53:01.726 | INFO | MemInfo: { | 20260508 02:53:01.726 | INFO | "gpu" : " 17 GB", | 20260508 02:53:01.726 | INFO | "huge" : "", | 20260508 02:53:01.726 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:01.726 | INFO | "malloc" : " 14 GB", | 20260508 02:53:01.726 | INFO | "pinned" : " 29 GB" | 20260508 02:53:01.726 | INFO | } | 20260508 02:53:01.726 | INFO | Perf report for the last 5.13858 seconds: | 20260508 02:53:01.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:01.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 02:53:01.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:53:01.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:01.726 | INFO | Uploaded 36GiB to GPU in 2.39152s (15.0532GiB/s) | 20260508 02:53:01.726 | INFO | Downloaded 34GiB to GPU in 2.58166s (13.1698GiB/s) | 20260508 02:53:06.862 | INFO | Analyzing chunk frames = [4667904,4668416) | 20260508 02:53:06.862 | INFO | MemInfo: { | 20260508 02:53:06.862 | INFO | "gpu" : " 17 GB", | 20260508 02:53:06.862 | INFO | "huge" : "", | 20260508 02:53:06.862 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:06.862 | INFO | "malloc" : " 14 GB", | 20260508 02:53:06.862 | INFO | "pinned" : " 29 GB" | 20260508 02:53:06.862 | INFO | } | 20260508 02:53:06.863 | INFO | Perf report for the last 5.13655 seconds: | 20260508 02:53:06.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:06.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:53:06.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:53:06.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:06.863 | INFO | Uploaded 36GiB to GPU in 2.40089s (14.9945GiB/s) | 20260508 02:53:06.863 | INFO | Downloaded 34GiB to GPU in 2.42882s (13.9986GiB/s) | 20260508 02:53:11.999 | INFO | Analyzing chunk frames = [4668416,4668928) | 20260508 02:53:11.999 | INFO | MemInfo: { | 20260508 02:53:11.999 | INFO | "gpu" : " 17 GB", | 20260508 02:53:11.999 | INFO | "huge" : "", | 20260508 02:53:11.999 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:11.999 | INFO | "malloc" : " 14 GB", | 20260508 02:53:11.999 | INFO | "pinned" : " 29 GB" | 20260508 02:53:11.999 | INFO | } | 20260508 02:53:11.999 | INFO | Perf report for the last 5.13655 seconds: | 20260508 02:53:11.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:11.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:53:11.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:53:11.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:11.999 | INFO | Uploaded 36GiB to GPU in 2.43238s (14.8003GiB/s) | 20260508 02:53:12.000 | INFO | Downloaded 34GiB to GPU in 2.42687s (14.0098GiB/s) | 20260508 02:53:17.142 | INFO | Analyzing chunk frames = [4668928,4669440) | 20260508 02:53:17.142 | INFO | MemInfo: { | 20260508 02:53:17.142 | INFO | "gpu" : " 17 GB", | 20260508 02:53:17.142 | INFO | "huge" : "", | 20260508 02:53:17.142 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:17.142 | INFO | "malloc" : " 14 GB", | 20260508 02:53:17.142 | INFO | "pinned" : " 29 GB" | 20260508 02:53:17.143 | INFO | } | 20260508 02:53:17.143 | INFO | Perf report for the last 5.14348 seconds: | 20260508 02:53:17.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:17.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 02:53:17.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 02:53:17.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:17.143 | INFO | Uploaded 36GiB to GPU in 2.46658s (14.5951GiB/s) | 20260508 02:53:17.143 | INFO | Downloaded 34GiB to GPU in 2.43716s (13.9506GiB/s) | 20260508 02:53:17.143 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:53:17.143 | INFO | Total mem usage: 16.7 GiB | 20260508 02:53:17.143 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:53:17.143 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.3%) | 20260508 02:53:17.143 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:53:17.143 | INFO | High Water Breakdown: | 20260508 02:53:17.143 | INFO | 60.4% : 9.14 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:53:17.143 | INFO | 39.5% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:53:17.143 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:53:17.144 | INFO | ------------------------------------------ | 20260508 02:53:17.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:53:17.144 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:53:17.144 | INFO | Total mem usage: 26.5 GiB | 20260508 02:53:17.144 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:53:17.144 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 02:53:17.144 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 02:53:17.144 | INFO | High Water Breakdown: | 20260508 02:53:17.144 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:53:17.144 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:53:17.144 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:53:17.144 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:53:17.144 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:53:17.144 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:53:17.144 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:53:17.144 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:53:17.144 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:53:17.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:53:17.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:53:17.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:53:17.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:53:17.144 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:53:17.144 | INFO | ------------------------------------------ | 20260508 02:53:17.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:53:17.144 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:53:17.144 | INFO | Total mem usage: 27.8 GiB | 20260508 02:53:17.144 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:53:17.144 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:53:17.144 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:53:17.144 | INFO | High Water Breakdown: | 20260508 02:53:17.144 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:53:17.144 | INFO | ------------------------------------------ | 20260508 02:53:17.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:53:17.144 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:53:17.144 | INFO | Total mem usage: 15.7 GiB | 20260508 02:53:17.144 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:53:17.144 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:53:17.144 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:53:17.144 | INFO | High Water Breakdown: | 20260508 02:53:17.144 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:53:17.144 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:53:17.144 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:53:17.144 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:53:17.144 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:53:17.144 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:53:17.144 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:53:17.144 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:53:17.144 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:53:17.144 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:53:17.144 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:53:17.145 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:53:17.145 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:53:17.145 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:53:17.145 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:53:17.145 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:53:17.145 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:53:17.145 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:53:17.145 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:53:17.145 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:53:17.145 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:53:17.145 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:53:17.145 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:53:17.145 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:53:17.145 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:53:17.145 | INFO | ------------------------------------------ | 20260508 02:53:17.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:53:17.145 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:53:17.145 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:53:17.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:53:17.145 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.41e+01 / 1.71e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.21e-02 / 6.02e-03 | 20260508 02:53:17.145 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.91e-01 / 2.07e-01 | 20260508 02:53:17.145 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.05e-01 / 1.14e-01 | 20260508 02:53:17.145 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.63e+01, 4.16e+00 / 2.34e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.64e+00 / 1.51e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.70e+00, 4.19e-01 / 1.86e-01 | 20260508 02:53:17.145 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.21e+00, 2.69e-01 / 1.35e-01 | 20260508 02:53:17.145 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.64e-01, 5.14e-02 / 2.55e-02 | 20260508 02:53:17.145 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:53:17.145 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:53:17.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:53:17.145 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 24.45%, Upload -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 24.46%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 2.18%, LockUpload -- 2.32e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 2.79%, LockDownload -- 2.97e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 15.40%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:53:17.145 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260508 02:53:17.255 | INFO | Starting a round of Baz IO | 20260508 02:53:20.712 | INFO | Populating RTMetrics report | 20260508 02:53:20.712 | INFO | RTMetrics report populated | 20260508 02:53:20.712 | INFO | Opening tmp RTMetrics json file | 20260508 02:53:20.807 | INFO | Serializing RTMetrics to json | 20260508 02:53:20.810 | INFO | Serialization done, swapping tmp file now... | 20260508 02:53:20.815 | INFO | RTMetrics done | 20260508 02:53:20.838 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.582 seconds | 20260508 02:53:20.838 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.42% / 0.00% / 58.58% | 20260508 02:53:20.838 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.672309% / 3.97364e-06% / 3.57628e-05% | 20260508 02:53:20.838 | INFO | Baz IO overall: | 20260508 02:53:20.838 | INFO | BytesWritten : 2104603786251 | 20260508 02:53:20.838 | INFO | NumEvents : 1017145964096 | 20260508 02:53:22.283 | INFO | Analyzing chunk frames = [4669440,4669952) | 20260508 02:53:22.283 | INFO | MemInfo: { | 20260508 02:53:22.283 | INFO | "gpu" : " 17 GB", | 20260508 02:53:22.283 | INFO | "huge" : "", | 20260508 02:53:22.283 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:22.283 | INFO | "malloc" : " 14 GB", | 20260508 02:53:22.283 | INFO | "pinned" : " 29 GB" | 20260508 02:53:22.283 | INFO | } | 20260508 02:53:22.283 | INFO | Perf report for the last 5.14079 seconds: | 20260508 02:53:22.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:22.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:22.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:22.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:22.284 | INFO | ANALYSIS : 68.6%, 2.9, 104.7ms, 0 | 20260508 02:53:22.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:22.284 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260508 02:53:22.284 | INFO | PRE_HQ : 67.4%, 1.8, 62.2ms, 0 | 20260508 02:53:22.284 | INFO | BAZWRITER : 69.3%, 3.7, 131.8ms, 0 | 20260508 02:53:22.284 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260508 02:53:22.284 | INFO | Uploaded 36GiB to GPU in 2.63334s (13.6709GiB/s) | 20260508 02:53:22.284 | INFO | Downloaded 38GiB to GPU in 2.70391s (14.0537GiB/s) | 20260508 02:53:27.419 | INFO | Analyzing chunk frames = [4669952,4670464) | 20260508 02:53:27.419 | INFO | MemInfo: { | 20260508 02:53:27.419 | INFO | "gpu" : " 17 GB", | 20260508 02:53:27.419 | INFO | "huge" : "", | 20260508 02:53:27.419 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:27.419 | INFO | "malloc" : " 14 GB", | 20260508 02:53:27.419 | INFO | "pinned" : " 29 GB" | 20260508 02:53:27.419 | INFO | } | 20260508 02:53:27.419 | INFO | Perf report for the last 5.13605 seconds: | 20260508 02:53:27.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:27.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.420 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:53:27.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.420 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:53:27.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:27.420 | INFO | Uploaded 36GiB to GPU in 2.52963s (14.2313GiB/s) | 20260508 02:53:27.420 | INFO | Downloaded 34GiB to GPU in 2.42434s (14.0244GiB/s) | 20260508 02:53:32.556 | INFO | Analyzing chunk frames = [4670464,4670976) | 20260508 02:53:32.556 | INFO | MemInfo: { | 20260508 02:53:32.556 | INFO | "gpu" : " 17 GB", | 20260508 02:53:32.556 | INFO | "huge" : "", | 20260508 02:53:32.556 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:32.556 | INFO | "malloc" : " 14 GB", | 20260508 02:53:32.556 | INFO | "pinned" : " 29 GB" | 20260508 02:53:32.556 | INFO | } | 20260508 02:53:32.556 | INFO | Perf report for the last 5.13654 seconds: | 20260508 02:53:32.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:32.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:53:32.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:53:32.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:32.556 | INFO | Uploaded 36GiB to GPU in 2.56998s (14.0079GiB/s) | 20260508 02:53:32.557 | INFO | Downloaded 34GiB to GPU in 2.50807s (13.5563GiB/s) | 20260508 02:53:37.700 | INFO | Analyzing chunk frames = [4670976,4671488) | 20260508 02:53:37.700 | INFO | MemInfo: { | 20260508 02:53:37.700 | INFO | "gpu" : " 17 GB", | 20260508 02:53:37.700 | INFO | "huge" : "", | 20260508 02:53:37.700 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:37.700 | INFO | "malloc" : " 14 GB", | 20260508 02:53:37.700 | INFO | "pinned" : " 29 GB" | 20260508 02:53:37.700 | INFO | } | 20260508 02:53:37.700 | INFO | Perf report for the last 5.14431 seconds: | 20260508 02:53:37.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:37.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.700 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:53:37.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.700 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:53:37.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:37.701 | INFO | Uploaded 36GiB to GPU in 2.52906s (14.2346GiB/s) | 20260508 02:53:37.701 | INFO | Downloaded 34GiB to GPU in 2.5028s (13.5848GiB/s) | 20260508 02:53:42.842 | INFO | Analyzing chunk frames = [4671488,4672000) | 20260508 02:53:42.842 | INFO | MemInfo: { | 20260508 02:53:42.842 | INFO | "gpu" : " 17 GB", | 20260508 02:53:42.842 | INFO | "huge" : "", | 20260508 02:53:42.842 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:42.842 | INFO | "malloc" : " 14 GB", | 20260508 02:53:42.842 | INFO | "pinned" : " 29 GB" | 20260508 02:53:42.842 | INFO | } | 20260508 02:53:42.842 | INFO | Perf report for the last 5.14151 seconds: | 20260508 02:53:42.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:42.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.842 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 02:53:42.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.842 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 02:53:42.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:42.843 | INFO | Uploaded 36GiB to GPU in 2.39055s (15.0593GiB/s) | 20260508 02:53:42.843 | INFO | Downloaded 34GiB to GPU in 2.57444s (13.2068GiB/s) | 20260508 02:53:47.983 | INFO | Analyzing chunk frames = [4672000,4672512) | 20260508 02:53:47.983 | INFO | MemInfo: { | 20260508 02:53:47.983 | INFO | "gpu" : " 17 GB", | 20260508 02:53:47.983 | INFO | "huge" : "", | 20260508 02:53:47.983 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:47.984 | INFO | "malloc" : " 14 GB", | 20260508 02:53:47.984 | INFO | "pinned" : " 29 GB" | 20260508 02:53:47.984 | INFO | } | 20260508 02:53:47.984 | INFO | Perf report for the last 5.14183 seconds: | 20260508 02:53:47.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:47.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 02:53:47.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:53:47.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:47.984 | INFO | Uploaded 36GiB to GPU in 2.37599s (15.1516GiB/s) | 20260508 02:53:47.984 | INFO | Downloaded 34GiB to GPU in 2.43463s (13.9652GiB/s) | 20260508 02:53:53.117 | INFO | Analyzing chunk frames = [4672512,4673024) | 20260508 02:53:53.117 | INFO | MemInfo: { | 20260508 02:53:53.117 | INFO | "gpu" : " 17 GB", | 20260508 02:53:53.117 | INFO | "huge" : "", | 20260508 02:53:53.117 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:53.117 | INFO | "malloc" : " 14 GB", | 20260508 02:53:53.117 | INFO | "pinned" : " 29 GB" | 20260508 02:53:53.117 | INFO | } | 20260508 02:53:53.117 | INFO | Perf report for the last 5.13329 seconds: | 20260508 02:53:53.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:53.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.117 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 02:53:53.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.117 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:53:53.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:53.118 | INFO | Uploaded 36GiB to GPU in 2.39017s (15.0617GiB/s) | 20260508 02:53:53.118 | INFO | Downloaded 34GiB to GPU in 2.43923s (13.9388GiB/s) | 20260508 02:53:58.256 | INFO | Analyzing chunk frames = [4673024,4673536) | 20260508 02:53:58.256 | INFO | MemInfo: { | 20260508 02:53:58.256 | INFO | "gpu" : " 17 GB", | 20260508 02:53:58.256 | INFO | "huge" : "", | 20260508 02:53:58.256 | INFO | "huge_shared" : " 29 GB", | 20260508 02:53:58.256 | INFO | "malloc" : " 14 GB", | 20260508 02:53:58.256 | INFO | "pinned" : " 29 GB" | 20260508 02:53:58.256 | INFO | } | 20260508 02:53:58.256 | INFO | Perf report for the last 5.13905 seconds: | 20260508 02:53:58.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:53:58.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.256 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 02:53:58.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.256 | INFO | PRE_HQ : 34.6%, 1.0, 17.9ms, 0 | 20260508 02:53:58.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:53:58.257 | INFO | Uploaded 36GiB to GPU in 2.41608s (14.9002GiB/s) | 20260508 02:53:58.257 | INFO | Downloaded 34GiB to GPU in 2.44625s (13.8988GiB/s) | 20260508 02:53:58.369 | INFO | Starting a round of Baz IO | 20260508 02:54:01.820 | INFO | Populating RTMetrics report | 20260508 02:54:01.821 | INFO | RTMetrics report populated | 20260508 02:54:01.821 | INFO | Opening tmp RTMetrics json file | 20260508 02:54:01.823 | INFO | Serializing RTMetrics to json | 20260508 02:54:01.829 | INFO | Serialization done, swapping tmp file now... | 20260508 02:54:01.836 | INFO | RTMetrics done | 20260508 02:54:01.949 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.58 seconds | 20260508 02:54:01.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.42% / 0.00% / 58.58% | 20260508 02:54:01.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.671872% / 3.97364e-06% / 6.75519e-05% | 20260508 02:54:01.949 | INFO | Baz IO overall: | 20260508 02:54:01.949 | INFO | BytesWritten : 2106352455552 | 20260508 02:54:01.949 | INFO | NumEvents : 1017983994083 | 20260508 02:54:03.397 | INFO | Analyzing chunk frames = [4673536,4674048) | 20260508 02:54:03.397 | INFO | MemInfo: { | 20260508 02:54:03.397 | INFO | "gpu" : " 17 GB", | 20260508 02:54:03.397 | INFO | "huge" : "", | 20260508 02:54:03.397 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:03.397 | INFO | "malloc" : " 14 GB", | 20260508 02:54:03.397 | INFO | "pinned" : " 29 GB" | 20260508 02:54:03.397 | INFO | } | 20260508 02:54:03.398 | INFO | Perf report for the last 5.14142 seconds: | 20260508 02:54:03.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:03.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:03.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:03.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:03.398 | INFO | ANALYSIS : 68.9%, 2.9, 104.5ms, 0 | 20260508 02:54:03.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:03.398 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 02:54:03.398 | INFO | PRE_HQ : 67.9%, 1.8, 62.1ms, 0 | 20260508 02:54:03.398 | INFO | BAZWRITER : 69.2%, 3.6, 127.6ms, 0 | 20260508 02:54:03.398 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 02:54:03.398 | INFO | Uploaded 36GiB to GPU in 2.54111s (14.167GiB/s) | 20260508 02:54:03.398 | INFO | Downloaded 38GiB to GPU in 2.70716s (14.0369GiB/s) | 20260508 02:54:08.541 | INFO | Analyzing chunk frames = [4674048,4674560) | 20260508 02:54:08.541 | INFO | MemInfo: { | 20260508 02:54:08.541 | INFO | "gpu" : " 17 GB", | 20260508 02:54:08.541 | INFO | "huge" : "", | 20260508 02:54:08.541 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:08.541 | INFO | "malloc" : " 14 GB", | 20260508 02:54:08.541 | INFO | "pinned" : " 29 GB" | 20260508 02:54:08.541 | INFO | } | 20260508 02:54:08.541 | INFO | Perf report for the last 5.14325 seconds: | 20260508 02:54:08.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:08.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:54:08.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:54:08.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:08.541 | INFO | Uploaded 36GiB to GPU in 2.50933s (14.3465GiB/s) | 20260508 02:54:08.541 | INFO | Downloaded 34GiB to GPU in 2.52039s (13.49GiB/s) | 20260508 02:54:13.677 | INFO | Analyzing chunk frames = [4674560,4675072) | 20260508 02:54:13.677 | INFO | MemInfo: { | 20260508 02:54:13.677 | INFO | "gpu" : " 17 GB", | 20260508 02:54:13.677 | INFO | "huge" : "", | 20260508 02:54:13.678 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:13.678 | INFO | "malloc" : " 14 GB", | 20260508 02:54:13.678 | INFO | "pinned" : " 29 GB" | 20260508 02:54:13.678 | INFO | } | 20260508 02:54:13.678 | INFO | Perf report for the last 5.13679 seconds: | 20260508 02:54:13.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:13.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 02:54:13.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:54:13.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:13.678 | INFO | Uploaded 36GiB to GPU in 2.50797s (14.3543GiB/s) | 20260508 02:54:13.678 | INFO | Downloaded 34GiB to GPU in 2.51505s (13.5186GiB/s) | 20260508 02:54:18.812 | INFO | Analyzing chunk frames = [4675072,4675584) | 20260508 02:54:18.812 | INFO | MemInfo: { | 20260508 02:54:18.812 | INFO | "gpu" : " 17 GB", | 20260508 02:54:18.812 | INFO | "huge" : "", | 20260508 02:54:18.812 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:18.812 | INFO | "malloc" : " 14 GB", | 20260508 02:54:18.812 | INFO | "pinned" : " 29 GB" | 20260508 02:54:18.812 | INFO | } | 20260508 02:54:18.812 | INFO | Perf report for the last 5.13422 seconds: | 20260508 02:54:18.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:18.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 02:54:18.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 02:54:18.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:18.812 | INFO | Uploaded 36GiB to GPU in 2.49574s (14.4246GiB/s) | 20260508 02:54:18.812 | INFO | Downloaded 34GiB to GPU in 2.52638s (13.458GiB/s) | 20260508 02:54:23.956 | INFO | Analyzing chunk frames = [4675584,4676096) | 20260508 02:54:23.956 | INFO | MemInfo: { | 20260508 02:54:23.956 | INFO | "gpu" : " 17 GB", | 20260508 02:54:23.956 | INFO | "huge" : "", | 20260508 02:54:23.956 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:23.956 | INFO | "malloc" : " 14 GB", | 20260508 02:54:23.956 | INFO | "pinned" : " 29 GB" | 20260508 02:54:23.956 | INFO | } | 20260508 02:54:23.956 | INFO | Perf report for the last 5.14426 seconds: | 20260508 02:54:23.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:23.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 02:54:23.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:54:23.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:23.956 | INFO | Uploaded 36GiB to GPU in 2.29835s (15.6634GiB/s) | 20260508 02:54:23.956 | INFO | Downloaded 34GiB to GPU in 2.50502s (13.5727GiB/s) | 20260508 02:54:29.091 | INFO | Analyzing chunk frames = [4676096,4676608) | 20260508 02:54:29.091 | INFO | MemInfo: { | 20260508 02:54:29.091 | INFO | "gpu" : " 17 GB", | 20260508 02:54:29.091 | INFO | "huge" : "", | 20260508 02:54:29.091 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:29.091 | INFO | "malloc" : " 14 GB", | 20260508 02:54:29.091 | INFO | "pinned" : " 29 GB" | 20260508 02:54:29.091 | INFO | } | 20260508 02:54:29.091 | INFO | Perf report for the last 5.13488 seconds: | 20260508 02:54:29.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:29.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:54:29.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:54:29.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:29.091 | INFO | Uploaded 36GiB to GPU in 2.41041s (14.9352GiB/s) | 20260508 02:54:29.091 | INFO | Downloaded 34GiB to GPU in 2.43334s (13.9726GiB/s) | 20260508 02:54:34.232 | INFO | Analyzing chunk frames = [4676608,4677120) | 20260508 02:54:34.232 | INFO | MemInfo: { | 20260508 02:54:34.232 | INFO | "gpu" : " 17 GB", | 20260508 02:54:34.232 | INFO | "huge" : "", | 20260508 02:54:34.232 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:34.232 | INFO | "malloc" : " 14 GB", | 20260508 02:54:34.232 | INFO | "pinned" : " 29 GB" | 20260508 02:54:34.232 | INFO | } | 20260508 02:54:34.232 | INFO | Perf report for the last 5.14125 seconds: | 20260508 02:54:34.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:34.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.232 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:54:34.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.232 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:54:34.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:34.233 | INFO | Uploaded 36GiB to GPU in 2.44961s (14.6962GiB/s) | 20260508 02:54:34.233 | INFO | Downloaded 34GiB to GPU in 2.43529s (13.9614GiB/s) | 20260508 02:54:39.366 | INFO | Analyzing chunk frames = [4677120,4677632) | 20260508 02:54:39.366 | INFO | MemInfo: { | 20260508 02:54:39.367 | INFO | "gpu" : " 17 GB", | 20260508 02:54:39.367 | INFO | "huge" : "", | 20260508 02:54:39.367 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:39.367 | INFO | "malloc" : " 14 GB", | 20260508 02:54:39.367 | INFO | "pinned" : " 29 GB" | 20260508 02:54:39.367 | INFO | } | 20260508 02:54:39.367 | INFO | Perf report for the last 5.13444 seconds: | 20260508 02:54:39.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:39.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 02:54:39.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 02:54:39.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:39.367 | INFO | Uploaded 36GiB to GPU in 2.55913s (14.0673GiB/s) | 20260508 02:54:39.367 | INFO | Downloaded 34GiB to GPU in 2.46824s (13.775GiB/s) | 20260508 02:54:39.367 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:54:39.367 | INFO | Total mem usage: 16.7 GiB | 20260508 02:54:39.367 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:54:39.367 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.2%) | 20260508 02:54:39.367 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:54:39.367 | INFO | High Water Breakdown: | 20260508 02:54:39.367 | INFO | 59.7% : 8.65 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:54:39.367 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:54:39.367 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:54:39.367 | INFO | ------------------------------------------ | 20260508 02:54:39.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:54:39.367 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:54:39.367 | INFO | Total mem usage: 26.5 GiB | 20260508 02:54:39.367 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:54:39.367 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:54:39.367 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:54:39.367 | INFO | High Water Breakdown: | 20260508 02:54:39.367 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:54:39.367 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:54:39.367 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:54:39.367 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:54:39.367 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:54:39.367 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:54:39.367 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:54:39.367 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:54:39.367 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:54:39.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:54:39.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:54:39.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:54:39.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:54:39.368 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:54:39.368 | INFO | ------------------------------------------ | 20260508 02:54:39.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:54:39.368 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:54:39.368 | INFO | Total mem usage: 27.8 GiB | 20260508 02:54:39.368 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:54:39.368 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:54:39.368 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:54:39.368 | INFO | High Water Breakdown: | 20260508 02:54:39.368 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:54:39.368 | INFO | ------------------------------------------ | 20260508 02:54:39.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:54:39.368 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:54:39.368 | INFO | Total mem usage: 15.7 GiB | 20260508 02:54:39.368 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:54:39.368 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:54:39.368 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:54:39.368 | INFO | High Water Breakdown: | 20260508 02:54:39.368 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:54:39.368 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:54:39.368 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:54:39.368 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:54:39.368 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:54:39.368 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:54:39.368 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:54:39.368 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:54:39.368 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:54:39.368 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:54:39.368 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:54:39.368 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:54:39.368 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:54:39.368 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:54:39.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:54:39.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:54:39.368 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:54:39.368 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:54:39.368 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:54:39.368 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:54:39.368 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:54:39.368 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:54:39.368 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:54:39.368 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:54:39.368 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:54:39.368 | INFO | ------------------------------------------ | 20260508 02:54:39.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:54:39.368 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:54:39.368 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:54:39.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:54:39.368 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.40e+01 / 1.49e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.14e-02 / 4.93e-03 | 20260508 02:54:39.368 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.80e-01 / 2.11e-01 | 20260508 02:54:39.368 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 1.11e-01 | 20260508 02:54:39.368 | INFO | PerfReport: 16.08%, DME -- 4.20e+01 / 4.61e+01, 4.13e+00 / 1.62e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 56.43%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.63e+00 / 1.60e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.21e-01 / 2.20e-01 | 20260508 02:54:39.368 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.23e+00, 2.62e-01 / 1.31e-01 | 20260508 02:54:39.368 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.28e-01 / 3.63e-01, 5.11e-02 / 2.44e-02 | 20260508 02:54:39.368 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:54:39.368 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:54:39.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:54:39.368 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 24.29%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 24.63%, Download -- 2.61e+01 / 0.00e+00, 8.38e+00 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 2.05%, LockUpload -- 2.17e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 2.98%, LockDownload -- 3.16e+00 / 0.00e+00, 4.37e+00 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 15.28%, LockCompute -- 1.62e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 02:54:39.368 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260508 02:54:39.479 | INFO | Starting a round of Baz IO | 20260508 02:54:42.944 | INFO | Populating RTMetrics report | 20260508 02:54:42.944 | INFO | RTMetrics report populated | 20260508 02:54:42.944 | INFO | Opening tmp RTMetrics json file | 20260508 02:54:43.004 | INFO | Serializing RTMetrics to json | 20260508 02:54:43.009 | INFO | Serialization done, swapping tmp file now... | 20260508 02:54:43.011 | INFO | RTMetrics done | 20260508 02:54:43.074 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.595 seconds | 20260508 02:54:43.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.42% / 0.00% / 58.58% | 20260508 02:54:43.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.671784% / 0% / 5.16574e-05% | 20260508 02:54:43.074 | INFO | Baz IO overall: | 20260508 02:54:43.074 | INFO | BytesWritten : 2108100960509 | 20260508 02:54:43.074 | INFO | NumEvents : 1018822038139 | 20260508 02:54:44.509 | INFO | Analyzing chunk frames = [4677632,4678144) | 20260508 02:54:44.509 | INFO | MemInfo: { | 20260508 02:54:44.509 | INFO | "gpu" : " 17 GB", | 20260508 02:54:44.509 | INFO | "huge" : "", | 20260508 02:54:44.509 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:44.509 | INFO | "malloc" : " 14 GB", | 20260508 02:54:44.509 | INFO | "pinned" : " 29 GB" | 20260508 02:54:44.509 | INFO | } | 20260508 02:54:44.509 | INFO | Perf report for the last 5.14203 seconds: | 20260508 02:54:44.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:44.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:44.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:44.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:44.509 | INFO | ANALYSIS : 68.7%, 2.9, 104.5ms, 0 | 20260508 02:54:44.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:44.509 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260508 02:54:44.509 | INFO | PRE_HQ : 67.3%, 1.8, 61.6ms, 0 | 20260508 02:54:44.509 | INFO | BAZWRITER : 69.5%, 4.4, 156.4ms, 0 | 20260508 02:54:44.509 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 02:54:44.509 | INFO | Uploaded 36GiB to GPU in 2.64192s (13.6265GiB/s) | 20260508 02:54:44.509 | INFO | Downloaded 38GiB to GPU in 2.69928s (14.0778GiB/s) | 20260508 02:54:49.651 | INFO | Analyzing chunk frames = [4678144,4678656) | 20260508 02:54:49.651 | INFO | MemInfo: { | 20260508 02:54:49.651 | INFO | "gpu" : " 17 GB", | 20260508 02:54:49.651 | INFO | "huge" : "", | 20260508 02:54:49.651 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:49.651 | INFO | "malloc" : " 14 GB", | 20260508 02:54:49.651 | INFO | "pinned" : " 29 GB" | 20260508 02:54:49.651 | INFO | } | 20260508 02:54:49.651 | INFO | Perf report for the last 5.14211 seconds: | 20260508 02:54:49.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:49.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:54:49.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:54:49.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:49.651 | INFO | Uploaded 36GiB to GPU in 2.57469s (13.9823GiB/s) | 20260508 02:54:49.652 | INFO | Downloaded 34GiB to GPU in 2.42301s (14.0321GiB/s) | 20260508 02:54:54.795 | INFO | Analyzing chunk frames = [4678656,4679168) | 20260508 02:54:54.795 | INFO | MemInfo: { | 20260508 02:54:54.795 | INFO | "gpu" : " 17 GB", | 20260508 02:54:54.795 | INFO | "huge" : "", | 20260508 02:54:54.795 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:54.795 | INFO | "malloc" : " 14 GB", | 20260508 02:54:54.795 | INFO | "pinned" : " 29 GB" | 20260508 02:54:54.795 | INFO | } | 20260508 02:54:54.795 | INFO | Perf report for the last 5.14411 seconds: | 20260508 02:54:54.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:54.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 02:54:54.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 02:54:54.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:54.795 | INFO | Uploaded 36GiB to GPU in 2.58545s (13.9241GiB/s) | 20260508 02:54:54.795 | INFO | Downloaded 34GiB to GPU in 2.48926s (13.6587GiB/s) | 20260508 02:54:59.933 | INFO | Analyzing chunk frames = [4679168,4679680) | 20260508 02:54:59.933 | INFO | MemInfo: { | 20260508 02:54:59.933 | INFO | "gpu" : " 17 GB", | 20260508 02:54:59.933 | INFO | "huge" : "", | 20260508 02:54:59.933 | INFO | "huge_shared" : " 29 GB", | 20260508 02:54:59.933 | INFO | "malloc" : " 14 GB", | 20260508 02:54:59.933 | INFO | "pinned" : " 29 GB" | 20260508 02:54:59.933 | INFO | } | 20260508 02:54:59.933 | INFO | Perf report for the last 5.13828 seconds: | 20260508 02:54:59.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:54:59.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:54:59.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:54:59.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:54:59.933 | INFO | Uploaded 36GiB to GPU in 2.57279s (13.9926GiB/s) | 20260508 02:54:59.934 | INFO | Downloaded 34GiB to GPU in 2.48938s (13.658GiB/s) | 20260508 02:55:05.065 | INFO | Analyzing chunk frames = [4679680,4680192) | 20260508 02:55:05.065 | INFO | MemInfo: { | 20260508 02:55:05.065 | INFO | "gpu" : " 17 GB", | 20260508 02:55:05.065 | INFO | "huge" : "", | 20260508 02:55:05.065 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:05.065 | INFO | "malloc" : " 14 GB", | 20260508 02:55:05.065 | INFO | "pinned" : " 29 GB" | 20260508 02:55:05.065 | INFO | } | 20260508 02:55:05.065 | INFO | Perf report for the last 5.13181 seconds: | 20260508 02:55:05.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:05.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 02:55:05.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 02:55:05.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:05.065 | INFO | Uploaded 36GiB to GPU in 2.40521s (14.9675GiB/s) | 20260508 02:55:05.065 | INFO | Downloaded 34GiB to GPU in 2.55899s (13.2865GiB/s) | 20260508 02:55:10.210 | INFO | Analyzing chunk frames = [4680192,4680704) | 20260508 02:55:10.210 | INFO | MemInfo: { | 20260508 02:55:10.210 | INFO | "gpu" : " 17 GB", | 20260508 02:55:10.210 | INFO | "huge" : "", | 20260508 02:55:10.210 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:10.210 | INFO | "malloc" : " 14 GB", | 20260508 02:55:10.210 | INFO | "pinned" : " 29 GB" | 20260508 02:55:10.210 | INFO | } | 20260508 02:55:10.210 | INFO | Perf report for the last 5.14501 seconds: | 20260508 02:55:10.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:10.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:55:10.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:55:10.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:10.210 | INFO | Uploaded 36GiB to GPU in 2.40386s (14.9759GiB/s) | 20260508 02:55:10.210 | INFO | Downloaded 34GiB to GPU in 2.42321s (14.0309GiB/s) | 20260508 02:55:15.348 | INFO | Analyzing chunk frames = [4680704,4681216) | 20260508 02:55:15.348 | INFO | MemInfo: { | 20260508 02:55:15.348 | INFO | "gpu" : " 17 GB", | 20260508 02:55:15.348 | INFO | "huge" : "", | 20260508 02:55:15.348 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:15.348 | INFO | "malloc" : " 14 GB", | 20260508 02:55:15.348 | INFO | "pinned" : " 29 GB" | 20260508 02:55:15.348 | INFO | } | 20260508 02:55:15.348 | INFO | Perf report for the last 5.13765 seconds: | 20260508 02:55:15.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:15.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 02:55:15.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:55:15.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:15.348 | INFO | Uploaded 36GiB to GPU in 2.44608s (14.7174GiB/s) | 20260508 02:55:15.348 | INFO | Downloaded 34GiB to GPU in 2.42542s (14.0182GiB/s) | 20260508 02:55:20.483 | INFO | Analyzing chunk frames = [4681216,4681728) | 20260508 02:55:20.483 | INFO | MemInfo: { | 20260508 02:55:20.483 | INFO | "gpu" : " 17 GB", | 20260508 02:55:20.483 | INFO | "huge" : "", | 20260508 02:55:20.483 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:20.483 | INFO | "malloc" : " 14 GB", | 20260508 02:55:20.483 | INFO | "pinned" : " 29 GB" | 20260508 02:55:20.483 | INFO | } | 20260508 02:55:20.484 | INFO | Perf report for the last 5.13574 seconds: | 20260508 02:55:20.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:20.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 02:55:20.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 02:55:20.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:20.484 | INFO | Uploaded 36GiB to GPU in 2.5602s (14.0614GiB/s) | 20260508 02:55:20.484 | INFO | Downloaded 34GiB to GPU in 2.46467s (13.795GiB/s) | 20260508 02:55:20.580 | INFO | Starting a round of Baz IO | 20260508 02:55:24.029 | INFO | Populating RTMetrics report | 20260508 02:55:24.029 | INFO | RTMetrics report populated | 20260508 02:55:24.029 | INFO | Opening tmp RTMetrics json file | 20260508 02:55:24.049 | INFO | Serializing RTMetrics to json | 20260508 02:55:24.051 | INFO | Serialization done, swapping tmp file now... | 20260508 02:55:24.052 | INFO | RTMetrics done | 20260508 02:55:24.123 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.542 seconds | 20260508 02:55:24.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.41% / 0.00% / 58.58% | 20260508 02:55:24.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.669467% / 0% / 7.15256e-05% | 20260508 02:55:24.123 | INFO | Baz IO overall: | 20260508 02:55:24.123 | INFO | BytesWritten : 2109846754562 | 20260508 02:55:24.123 | INFO | NumEvents : 1019658006442 | 20260508 02:55:25.623 | INFO | Analyzing chunk frames = [4681728,4682240) | 20260508 02:55:25.623 | INFO | MemInfo: { | 20260508 02:55:25.623 | INFO | "gpu" : " 17 GB", | 20260508 02:55:25.623 | INFO | "huge" : "", | 20260508 02:55:25.623 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:25.623 | INFO | "malloc" : " 14 GB", | 20260508 02:55:25.623 | INFO | "pinned" : " 29 GB" | 20260508 02:55:25.623 | INFO | } | 20260508 02:55:25.623 | INFO | Perf report for the last 5.13974 seconds: | 20260508 02:55:25.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:25.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:25.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:25.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:25.623 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 02:55:25.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:25.623 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 02:55:25.623 | INFO | PRE_HQ : 65.5%, 1.6, 56.0ms, 0 | 20260508 02:55:25.623 | INFO | BAZWRITER : 68.5%, 3.9, 138.2ms, 0 | 20260508 02:55:25.623 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 02:55:25.624 | INFO | Uploaded 36GiB to GPU in 2.55553s (14.0871GiB/s) | 20260508 02:55:25.624 | INFO | Downloaded 38GiB to GPU in 2.69412s (14.1048GiB/s) | 20260508 02:55:30.763 | INFO | Analyzing chunk frames = [4682240,4682752) | 20260508 02:55:30.763 | INFO | MemInfo: { | 20260508 02:55:30.763 | INFO | "gpu" : " 17 GB", | 20260508 02:55:30.763 | INFO | "huge" : "", | 20260508 02:55:30.763 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:30.764 | INFO | "malloc" : " 14 GB", | 20260508 02:55:30.764 | INFO | "pinned" : " 29 GB" | 20260508 02:55:30.764 | INFO | } | 20260508 02:55:30.764 | INFO | Perf report for the last 5.14036 seconds: | 20260508 02:55:30.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:30.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:55:30.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 02:55:30.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:30.764 | INFO | Uploaded 36GiB to GPU in 2.54843s (14.1264GiB/s) | 20260508 02:55:30.764 | INFO | Downloaded 34GiB to GPU in 2.4226s (14.0345GiB/s) | 20260508 02:55:35.906 | INFO | Analyzing chunk frames = [4682752,4683264) | 20260508 02:55:35.906 | INFO | MemInfo: { | 20260508 02:55:35.906 | INFO | "gpu" : " 17 GB", | 20260508 02:55:35.906 | INFO | "huge" : "", | 20260508 02:55:35.906 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:35.906 | INFO | "malloc" : " 14 GB", | 20260508 02:55:35.906 | INFO | "pinned" : " 29 GB" | 20260508 02:55:35.906 | INFO | } | 20260508 02:55:35.906 | INFO | Perf report for the last 5.14269 seconds: | 20260508 02:55:35.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:35.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.906 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:55:35.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.907 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:55:35.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:35.907 | INFO | Uploaded 36GiB to GPU in 2.53276s (14.2137GiB/s) | 20260508 02:55:35.907 | INFO | Downloaded 34GiB to GPU in 2.49717s (13.6154GiB/s) | 20260508 02:55:41.042 | INFO | Analyzing chunk frames = [4683264,4683776) | 20260508 02:55:41.042 | INFO | MemInfo: { | 20260508 02:55:41.042 | INFO | "gpu" : " 17 GB", | 20260508 02:55:41.042 | INFO | "huge" : "", | 20260508 02:55:41.042 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:41.043 | INFO | "malloc" : " 14 GB", | 20260508 02:55:41.043 | INFO | "pinned" : " 29 GB" | 20260508 02:55:41.043 | INFO | } | 20260508 02:55:41.043 | INFO | Perf report for the last 5.13628 seconds: | 20260508 02:55:41.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:41.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 02:55:41.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 02:55:41.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:41.043 | INFO | Uploaded 36GiB to GPU in 2.53021s (14.228GiB/s) | 20260508 02:55:41.043 | INFO | Downloaded 34GiB to GPU in 2.4997s (13.6016GiB/s) | 20260508 02:55:46.180 | INFO | Analyzing chunk frames = [4683776,4684288) | 20260508 02:55:46.180 | INFO | MemInfo: { | 20260508 02:55:46.180 | INFO | "gpu" : " 17 GB", | 20260508 02:55:46.180 | INFO | "huge" : "", | 20260508 02:55:46.180 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:46.180 | INFO | "malloc" : " 14 GB", | 20260508 02:55:46.180 | INFO | "pinned" : " 29 GB" | 20260508 02:55:46.180 | INFO | } | 20260508 02:55:46.180 | INFO | Perf report for the last 5.13759 seconds: | 20260508 02:55:46.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:46.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.180 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 02:55:46.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.180 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 02:55:46.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:46.181 | INFO | Uploaded 36GiB to GPU in 2.45966s (14.6362GiB/s) | 20260508 02:55:46.181 | INFO | Downloaded 34GiB to GPU in 2.60619s (13.0459GiB/s) | 20260508 02:55:51.317 | INFO | Analyzing chunk frames = [4684288,4684800) | 20260508 02:55:51.317 | INFO | MemInfo: { | 20260508 02:55:51.317 | INFO | "gpu" : " 17 GB", | 20260508 02:55:51.317 | INFO | "huge" : "", | 20260508 02:55:51.317 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:51.317 | INFO | "malloc" : " 14 GB", | 20260508 02:55:51.317 | INFO | "pinned" : " 29 GB" | 20260508 02:55:51.317 | INFO | } | 20260508 02:55:51.317 | INFO | Perf report for the last 5.13672 seconds: | 20260508 02:55:51.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:51.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.317 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 02:55:51.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.317 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260508 02:55:51.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:51.318 | INFO | Uploaded 36GiB to GPU in 2.43404s (14.7902GiB/s) | 20260508 02:55:51.318 | INFO | Downloaded 34GiB to GPU in 2.41533s (14.0767GiB/s) | 20260508 02:55:56.458 | INFO | Analyzing chunk frames = [4684800,4685312) | 20260508 02:55:56.458 | INFO | MemInfo: { | 20260508 02:55:56.458 | INFO | "gpu" : " 17 GB", | 20260508 02:55:56.458 | INFO | "huge" : "", | 20260508 02:55:56.458 | INFO | "huge_shared" : " 29 GB", | 20260508 02:55:56.458 | INFO | "malloc" : " 14 GB", | 20260508 02:55:56.458 | INFO | "pinned" : " 29 GB" | 20260508 02:55:56.458 | INFO | } | 20260508 02:55:56.458 | INFO | Perf report for the last 5.14127 seconds: | 20260508 02:55:56.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:55:56.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.458 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:55:56.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.458 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:55:56.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:55:56.459 | INFO | Uploaded 36GiB to GPU in 2.45916s (14.6391GiB/s) | 20260508 02:55:56.459 | INFO | Downloaded 34GiB to GPU in 2.4191s (14.0548GiB/s) | 20260508 02:56:01.607 | INFO | Analyzing chunk frames = [4685312,4685824) | 20260508 02:56:01.607 | INFO | MemInfo: { | 20260508 02:56:01.607 | INFO | "gpu" : " 17 GB", | 20260508 02:56:01.607 | INFO | "huge" : "", | 20260508 02:56:01.607 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:01.607 | INFO | "malloc" : " 14 GB", | 20260508 02:56:01.607 | INFO | "pinned" : " 29 GB" | 20260508 02:56:01.607 | INFO | } | 20260508 02:56:01.607 | INFO | Perf report for the last 5.14916 seconds: | 20260508 02:56:01.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:01.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | ANALYSIS : 67.0%, 3.0, 104.8ms, 0 | 20260508 02:56:01.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 02:56:01.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:01.608 | INFO | Uploaded 36GiB to GPU in 2.56852s (14.0159GiB/s) | 20260508 02:56:01.608 | INFO | Downloaded 34GiB to GPU in 2.44172s (13.9246GiB/s) | 20260508 02:56:01.608 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:56:01.608 | INFO | Total mem usage: 16.7 GiB | 20260508 02:56:01.608 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:56:01.608 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.4%) | 20260508 02:56:01.608 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:56:01.608 | INFO | High Water Breakdown: | 20260508 02:56:01.608 | INFO | 59.9% : 8.81 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:56:01.608 | INFO | 40% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:56:01.608 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:56:01.608 | INFO | ------------------------------------------ | 20260508 02:56:01.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:56:01.608 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:56:01.608 | INFO | Total mem usage: 26.5 GiB | 20260508 02:56:01.608 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:56:01.608 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 02:56:01.608 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 02:56:01.608 | INFO | High Water Breakdown: | 20260508 02:56:01.608 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:56:01.608 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:56:01.608 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:56:01.608 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:56:01.608 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:56:01.608 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:56:01.608 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:56:01.608 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:56:01.608 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:56:01.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:56:01.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:56:01.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:56:01.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:56:01.609 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:56:01.609 | INFO | ------------------------------------------ | 20260508 02:56:01.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:56:01.609 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:56:01.609 | INFO | Total mem usage: 27.8 GiB | 20260508 02:56:01.609 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:56:01.609 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:56:01.609 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:56:01.609 | INFO | High Water Breakdown: | 20260508 02:56:01.609 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:56:01.609 | INFO | ------------------------------------------ | 20260508 02:56:01.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:56:01.609 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:56:01.609 | INFO | Total mem usage: 15.7 GiB | 20260508 02:56:01.609 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:56:01.609 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:56:01.609 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:56:01.609 | INFO | High Water Breakdown: | 20260508 02:56:01.609 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:56:01.609 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:56:01.609 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:56:01.609 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:56:01.609 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:56:01.609 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:56:01.609 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:56:01.609 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:56:01.609 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:56:01.609 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:56:01.609 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:56:01.609 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:56:01.609 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:56:01.609 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:56:01.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:56:01.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:56:01.609 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:56:01.609 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:56:01.609 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:56:01.609 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:56:01.609 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:56:01.609 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:56:01.609 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:56:01.609 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:56:01.609 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:56:01.609 | INFO | ------------------------------------------ | 20260508 02:56:01.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:56:01.609 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:56:01.609 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:56:01.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:56:01.609 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.65e+01, 1.41e+01 / 1.62e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.01e-01, 1.21e-02 / 6.95e-03 | 20260508 02:56:01.609 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.44e+00, 5.81e-01 / 1.88e-01 | 20260508 02:56:01.609 | INFO | PerfReport: 3.32%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.06e-01 | 20260508 02:56:01.609 | INFO | PerfReport: 16.15%, DME -- 4.22e+01 / 4.68e+01, 4.30e+00 / 1.89e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.84e+01 / 1.94e+01, 1.62e+00 / 1.39e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.41e+00 / 2.70e+00, 4.25e-01 / 2.26e-01 | 20260508 02:56:01.609 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.21e+00, 2.65e-01 / 1.19e-01 | 20260508 02:56:01.609 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.53e-01, 5.16e-02 / 2.33e-02 | 20260508 02:56:01.609 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:56:01.609 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:56:01.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:56:01.609 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 24.59%, Upload -- 2.62e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 24.40%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 30.62%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 2.29%, LockUpload -- 2.44e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 2.72%, LockDownload -- 2.90e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 15.36%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 02:56:01.609 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260508 02:56:01.704 | INFO | Starting a round of Baz IO | 20260508 02:56:05.135 | INFO | Populating RTMetrics report | 20260508 02:56:05.135 | INFO | RTMetrics report populated | 20260508 02:56:05.135 | INFO | Opening tmp RTMetrics json file | 20260508 02:56:05.162 | INFO | Serializing RTMetrics to json | 20260508 02:56:05.163 | INFO | Serialization done, swapping tmp file now... | 20260508 02:56:05.165 | INFO | RTMetrics done | 20260508 02:56:05.264 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.56 seconds | 20260508 02:56:05.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.41% / 0.00% / 58.59% | 20260508 02:56:05.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.667934% / 7.94729e-06% / 6.75519e-05% | 20260508 02:56:05.264 | INFO | Baz IO overall: | 20260508 02:56:05.264 | INFO | BytesWritten : 2111592140119 | 20260508 02:56:05.265 | INFO | NumEvents : 1020493652838 | 20260508 02:56:06.744 | INFO | Analyzing chunk frames = [4685824,4686336) | 20260508 02:56:06.744 | INFO | MemInfo: { | 20260508 02:56:06.744 | INFO | "gpu" : " 17 GB", | 20260508 02:56:06.744 | INFO | "huge" : "", | 20260508 02:56:06.744 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:06.744 | INFO | "malloc" : " 14 GB", | 20260508 02:56:06.744 | INFO | "pinned" : " 29 GB" | 20260508 02:56:06.744 | INFO | } | 20260508 02:56:06.744 | INFO | Perf report for the last 5.13609 seconds: | 20260508 02:56:06.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:06.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:06.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:06.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:06.744 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260508 02:56:06.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:06.744 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 02:56:06.744 | INFO | PRE_HQ : 65.6%, 1.6, 55.9ms, 0 | 20260508 02:56:06.744 | INFO | BAZWRITER : 68.9%, 3.7, 129.9ms, 0 | 20260508 02:56:06.744 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 02:56:06.744 | INFO | Uploaded 36GiB to GPU in 2.56974s (14.0092GiB/s) | 20260508 02:56:06.744 | INFO | Downloaded 38GiB to GPU in 2.71014s (14.0214GiB/s) | 20260508 02:56:11.882 | INFO | Analyzing chunk frames = [4686336,4686848) | 20260508 02:56:11.882 | INFO | MemInfo: { | 20260508 02:56:11.882 | INFO | "gpu" : " 17 GB", | 20260508 02:56:11.882 | INFO | "huge" : "", | 20260508 02:56:11.882 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:11.882 | INFO | "malloc" : " 14 GB", | 20260508 02:56:11.882 | INFO | "pinned" : " 29 GB" | 20260508 02:56:11.882 | INFO | } | 20260508 02:56:11.882 | INFO | Perf report for the last 5.13858 seconds: | 20260508 02:56:11.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:11.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.882 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 02:56:11.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.882 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 02:56:11.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:11.883 | INFO | Uploaded 36GiB to GPU in 2.57475s (13.9819GiB/s) | 20260508 02:56:11.883 | INFO | Downloaded 34GiB to GPU in 2.42364s (14.0285GiB/s) | 20260508 02:56:17.020 | INFO | Analyzing chunk frames = [4686848,4687360) | 20260508 02:56:17.020 | INFO | MemInfo: { | 20260508 02:56:17.020 | INFO | "gpu" : " 17 GB", | 20260508 02:56:17.020 | INFO | "huge" : "", | 20260508 02:56:17.020 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:17.020 | INFO | "malloc" : " 14 GB", | 20260508 02:56:17.020 | INFO | "pinned" : " 29 GB" | 20260508 02:56:17.020 | INFO | } | 20260508 02:56:17.020 | INFO | Perf report for the last 5.13808 seconds: | 20260508 02:56:17.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:17.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.020 | INFO | ANALYSIS : 67.8%, 3.0, 105.2ms, 0 | 20260508 02:56:17.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.020 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 02:56:17.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:17.021 | INFO | Uploaded 36GiB to GPU in 2.58901s (13.9049GiB/s) | 20260508 02:56:17.021 | INFO | Downloaded 34GiB to GPU in 2.49809s (13.6104GiB/s) | 20260508 02:56:22.160 | INFO | Analyzing chunk frames = [4687360,4687872) | 20260508 02:56:22.160 | INFO | MemInfo: { | 20260508 02:56:22.160 | INFO | "gpu" : " 17 GB", | 20260508 02:56:22.160 | INFO | "huge" : "", | 20260508 02:56:22.160 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:22.160 | INFO | "malloc" : " 14 GB", | 20260508 02:56:22.160 | INFO | "pinned" : " 29 GB" | 20260508 02:56:22.160 | INFO | } | 20260508 02:56:22.160 | INFO | Perf report for the last 5.1398 seconds: | 20260508 02:56:22.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:22.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:56:22.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:56:22.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:22.160 | INFO | Uploaded 36GiB to GPU in 2.57483s (13.9815GiB/s) | 20260508 02:56:22.161 | INFO | Downloaded 34GiB to GPU in 2.49259s (13.6404GiB/s) | 20260508 02:56:27.303 | INFO | Analyzing chunk frames = [4687872,4688384) | 20260508 02:56:27.303 | INFO | MemInfo: { | 20260508 02:56:27.303 | INFO | "gpu" : " 17 GB", | 20260508 02:56:27.303 | INFO | "huge" : "", | 20260508 02:56:27.303 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:27.303 | INFO | "malloc" : " 14 GB", | 20260508 02:56:27.303 | INFO | "pinned" : " 29 GB" | 20260508 02:56:27.303 | INFO | } | 20260508 02:56:27.303 | INFO | Perf report for the last 5.14274 seconds: | 20260508 02:56:27.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:27.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 02:56:27.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:56:27.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:27.303 | INFO | Uploaded 36GiB to GPU in 2.37011s (15.1892GiB/s) | 20260508 02:56:27.303 | INFO | Downloaded 34GiB to GPU in 2.51665s (13.51GiB/s) | 20260508 02:56:32.434 | INFO | Analyzing chunk frames = [4688384,4688896) | 20260508 02:56:32.434 | INFO | MemInfo: { | 20260508 02:56:32.434 | INFO | "gpu" : " 17 GB", | 20260508 02:56:32.434 | INFO | "huge" : "", | 20260508 02:56:32.434 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:32.434 | INFO | "malloc" : " 14 GB", | 20260508 02:56:32.434 | INFO | "pinned" : " 29 GB" | 20260508 02:56:32.434 | INFO | } | 20260508 02:56:32.434 | INFO | Perf report for the last 5.13125 seconds: | 20260508 02:56:32.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:32.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.434 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 02:56:32.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.434 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:56:32.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:32.435 | INFO | Uploaded 36GiB to GPU in 2.44599s (14.718GiB/s) | 20260508 02:56:32.435 | INFO | Downloaded 34GiB to GPU in 2.42164s (14.0401GiB/s) | 20260508 02:56:37.574 | INFO | Analyzing chunk frames = [4688896,4689408) | 20260508 02:56:37.574 | INFO | MemInfo: { | 20260508 02:56:37.574 | INFO | "gpu" : " 17 GB", | 20260508 02:56:37.574 | INFO | "huge" : "", | 20260508 02:56:37.574 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:37.574 | INFO | "malloc" : " 14 GB", | 20260508 02:56:37.574 | INFO | "pinned" : " 29 GB" | 20260508 02:56:37.574 | INFO | } | 20260508 02:56:37.575 | INFO | Perf report for the last 5.14044 seconds: | 20260508 02:56:37.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:37.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 02:56:37.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:56:37.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:37.575 | INFO | Uploaded 36GiB to GPU in 2.46483s (14.6055GiB/s) | 20260508 02:56:37.575 | INFO | Downloaded 34GiB to GPU in 2.41782s (14.0622GiB/s) | 20260508 02:56:42.715 | INFO | Analyzing chunk frames = [4689408,4689920) | 20260508 02:56:42.715 | INFO | MemInfo: { | 20260508 02:56:42.715 | INFO | "gpu" : " 17 GB", | 20260508 02:56:42.715 | INFO | "huge" : "", | 20260508 02:56:42.715 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:42.715 | INFO | "malloc" : " 14 GB", | 20260508 02:56:42.715 | INFO | "pinned" : " 29 GB" | 20260508 02:56:42.715 | INFO | } | 20260508 02:56:42.715 | INFO | Perf report for the last 5.1405 seconds: | 20260508 02:56:42.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:42.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.715 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 02:56:42.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.715 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 02:56:42.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:42.716 | INFO | Uploaded 36GiB to GPU in 2.57468s (13.9823GiB/s) | 20260508 02:56:42.716 | INFO | Downloaded 34GiB to GPU in 2.46109s (13.815GiB/s) | 20260508 02:56:42.813 | INFO | Starting a round of Baz IO | 20260508 02:56:46.261 | INFO | Populating RTMetrics report | 20260508 02:56:46.262 | INFO | RTMetrics report populated | 20260508 02:56:46.265 | INFO | Opening tmp RTMetrics json file | 20260508 02:56:46.283 | INFO | Serializing RTMetrics to json | 20260508 02:56:46.332 | INFO | Serialization done, swapping tmp file now... | 20260508 02:56:46.359 | INFO | RTMetrics done | 20260508 02:56:46.388 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.575 seconds | 20260508 02:56:46.388 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.41% / 0.00% / 58.59% | 20260508 02:56:46.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.667091% / 7.94729e-06% / 5.5631e-05% | 20260508 02:56:46.388 | INFO | Baz IO overall: | 20260508 02:56:46.388 | INFO | BytesWritten : 2113337425991 | 20260508 02:56:46.388 | INFO | NumEvents : 1021329400508 | 20260508 02:56:47.858 | INFO | Analyzing chunk frames = [4689920,4690432) | 20260508 02:56:47.858 | INFO | MemInfo: { | 20260508 02:56:47.858 | INFO | "gpu" : " 17 GB", | 20260508 02:56:47.858 | INFO | "huge" : "", | 20260508 02:56:47.858 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:47.858 | INFO | "malloc" : " 14 GB", | 20260508 02:56:47.858 | INFO | "pinned" : " 29 GB" | 20260508 02:56:47.858 | INFO | } | 20260508 02:56:47.858 | INFO | Perf report for the last 5.14291 seconds: | 20260508 02:56:47.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:47.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:47.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:47.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:47.858 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 02:56:47.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:47.858 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 02:56:47.858 | INFO | PRE_HQ : 65.9%, 1.6, 55.9ms, 0 | 20260508 02:56:47.858 | INFO | BAZWRITER : 69.1%, 4.0, 142.3ms, 0 | 20260508 02:56:47.858 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 02:56:47.858 | INFO | Uploaded 36GiB to GPU in 2.56226s (14.0501GiB/s) | 20260508 02:56:47.858 | INFO | Downloaded 38GiB to GPU in 2.68053s (14.1763GiB/s) | 20260508 02:56:52.995 | INFO | Analyzing chunk frames = [4690432,4690944) | 20260508 02:56:52.995 | INFO | MemInfo: { | 20260508 02:56:52.995 | INFO | "gpu" : " 17 GB", | 20260508 02:56:52.995 | INFO | "huge" : "", | 20260508 02:56:52.995 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:52.996 | INFO | "malloc" : " 14 GB", | 20260508 02:56:52.996 | INFO | "pinned" : " 29 GB" | 20260508 02:56:52.996 | INFO | } | 20260508 02:56:52.996 | INFO | Perf report for the last 5.13763 seconds: | 20260508 02:56:52.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:52.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 02:56:52.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 02:56:52.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:52.996 | INFO | Uploaded 36GiB to GPU in 2.5642s (14.0394GiB/s) | 20260508 02:56:52.996 | INFO | Downloaded 34GiB to GPU in 2.42576s (14.0162GiB/s) | 20260508 02:56:58.131 | INFO | Analyzing chunk frames = [4690944,4691456) | 20260508 02:56:58.131 | INFO | MemInfo: { | 20260508 02:56:58.131 | INFO | "gpu" : " 17 GB", | 20260508 02:56:58.132 | INFO | "huge" : "", | 20260508 02:56:58.132 | INFO | "huge_shared" : " 29 GB", | 20260508 02:56:58.132 | INFO | "malloc" : " 14 GB", | 20260508 02:56:58.132 | INFO | "pinned" : " 29 GB" | 20260508 02:56:58.132 | INFO | } | 20260508 02:56:58.132 | INFO | Perf report for the last 5.13626 seconds: | 20260508 02:56:58.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:56:58.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 02:56:58.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 02:56:58.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:56:58.132 | INFO | Uploaded 36GiB to GPU in 2.57349s (13.9888GiB/s) | 20260508 02:56:58.132 | INFO | Downloaded 34GiB to GPU in 2.48869s (13.6618GiB/s) | 20260508 02:57:03.276 | INFO | Analyzing chunk frames = [4691456,4691968) | 20260508 02:57:03.276 | INFO | MemInfo: { | 20260508 02:57:03.276 | INFO | "gpu" : " 17 GB", | 20260508 02:57:03.276 | INFO | "huge" : "", | 20260508 02:57:03.276 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:03.276 | INFO | "malloc" : " 14 GB", | 20260508 02:57:03.276 | INFO | "pinned" : " 29 GB" | 20260508 02:57:03.276 | INFO | } | 20260508 02:57:03.277 | INFO | Perf report for the last 5.14461 seconds: | 20260508 02:57:03.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:03.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 02:57:03.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 02:57:03.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:03.277 | INFO | Uploaded 36GiB to GPU in 2.5642s (14.0395GiB/s) | 20260508 02:57:03.277 | INFO | Downloaded 34GiB to GPU in 2.48963s (13.6567GiB/s) | 20260508 02:57:08.412 | INFO | Analyzing chunk frames = [4691968,4692480) | 20260508 02:57:08.412 | INFO | MemInfo: { | 20260508 02:57:08.412 | INFO | "gpu" : " 17 GB", | 20260508 02:57:08.412 | INFO | "huge" : "", | 20260508 02:57:08.413 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:08.413 | INFO | "malloc" : " 14 GB", | 20260508 02:57:08.413 | INFO | "pinned" : " 29 GB" | 20260508 02:57:08.413 | INFO | } | 20260508 02:57:08.413 | INFO | Perf report for the last 5.13614 seconds: | 20260508 02:57:08.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:08.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 02:57:08.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:57:08.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:08.413 | INFO | Uploaded 36GiB to GPU in 2.42606s (14.8389GiB/s) | 20260508 02:57:08.413 | INFO | Downloaded 34GiB to GPU in 2.55592s (13.3024GiB/s) | 20260508 02:57:13.558 | INFO | Analyzing chunk frames = [4692480,4692992) | 20260508 02:57:13.558 | INFO | MemInfo: { | 20260508 02:57:13.558 | INFO | "gpu" : " 17 GB", | 20260508 02:57:13.558 | INFO | "huge" : "", | 20260508 02:57:13.558 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:13.558 | INFO | "malloc" : " 14 GB", | 20260508 02:57:13.558 | INFO | "pinned" : " 29 GB" | 20260508 02:57:13.558 | INFO | } | 20260508 02:57:13.558 | INFO | Perf report for the last 5.14562 seconds: | 20260508 02:57:13.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:13.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.558 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 02:57:13.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.558 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 02:57:13.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:13.559 | INFO | Uploaded 36GiB to GPU in 2.4322s (14.8014GiB/s) | 20260508 02:57:13.559 | INFO | Downloaded 34GiB to GPU in 2.3997s (14.1684GiB/s) | 20260508 02:57:18.695 | INFO | Analyzing chunk frames = [4692992,4693504) | 20260508 02:57:18.695 | INFO | MemInfo: { | 20260508 02:57:18.695 | INFO | "gpu" : " 17 GB", | 20260508 02:57:18.695 | INFO | "huge" : "", | 20260508 02:57:18.695 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:18.695 | INFO | "malloc" : " 14 GB", | 20260508 02:57:18.695 | INFO | "pinned" : " 29 GB" | 20260508 02:57:18.695 | INFO | } | 20260508 02:57:18.695 | INFO | Perf report for the last 5.13665 seconds: | 20260508 02:57:18.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:18.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 02:57:18.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 02:57:18.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:18.695 | INFO | Uploaded 36GiB to GPU in 2.46052s (14.6311GiB/s) | 20260508 02:57:18.695 | INFO | Downloaded 34GiB to GPU in 2.41784s (14.0621GiB/s) | 20260508 02:57:23.829 | INFO | Analyzing chunk frames = [4693504,4694016) | 20260508 02:57:23.829 | INFO | MemInfo: { | 20260508 02:57:23.829 | INFO | "gpu" : " 17 GB", | 20260508 02:57:23.829 | INFO | "huge" : "", | 20260508 02:57:23.829 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:23.829 | INFO | "malloc" : " 14 GB", | 20260508 02:57:23.829 | INFO | "pinned" : " 29 GB" | 20260508 02:57:23.829 | INFO | } | 20260508 02:57:23.830 | INFO | Perf report for the last 5.13449 seconds: | 20260508 02:57:23.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:23.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 02:57:23.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 02:57:23.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:23.830 | INFO | Uploaded 36GiB to GPU in 2.47229s (14.5614GiB/s) | 20260508 02:57:23.830 | INFO | Downloaded 34GiB to GPU in 2.42353s (14.0291GiB/s) | 20260508 02:57:23.830 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:57:23.830 | INFO | Total mem usage: 16.7 GiB | 20260508 02:57:23.830 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:57:23.830 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260508 02:57:23.830 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:57:23.830 | INFO | High Water Breakdown: | 20260508 02:57:23.830 | INFO | 60.1% : 8.94 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:57:23.830 | INFO | 39.8% : 5.92 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:57:23.830 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:57:23.830 | INFO | ------------------------------------------ | 20260508 02:57:23.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:57:23.830 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:57:23.830 | INFO | Total mem usage: 26.5 GiB | 20260508 02:57:23.830 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:57:23.830 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:57:23.830 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:57:23.830 | INFO | High Water Breakdown: | 20260508 02:57:23.830 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:57:23.830 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:57:23.830 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:57:23.830 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:57:23.830 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:57:23.830 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:57:23.830 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:57:23.830 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:57:23.830 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:57:23.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:57:23.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:57:23.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:57:23.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:57:23.830 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:57:23.830 | INFO | ------------------------------------------ | 20260508 02:57:23.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:57:23.830 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:57:23.830 | INFO | Total mem usage: 27.8 GiB | 20260508 02:57:23.830 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:57:23.830 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:57:23.830 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:57:23.830 | INFO | High Water Breakdown: | 20260508 02:57:23.830 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:57:23.830 | INFO | ------------------------------------------ | 20260508 02:57:23.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:57:23.830 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:57:23.830 | INFO | Total mem usage: 15.7 GiB | 20260508 02:57:23.830 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:57:23.830 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:57:23.830 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:57:23.830 | INFO | High Water Breakdown: | 20260508 02:57:23.830 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:57:23.830 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:57:23.830 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:57:23.830 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:57:23.830 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:57:23.830 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:57:23.830 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:57:23.830 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:57:23.830 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:57:23.830 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:57:23.830 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:57:23.830 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:57:23.830 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:57:23.830 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:57:23.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:57:23.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:57:23.830 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:57:23.830 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:57:23.830 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:57:23.830 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:57:23.831 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:57:23.831 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:57:23.831 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:57:23.831 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:57:23.831 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:57:23.831 | INFO | ------------------------------------------ | 20260508 02:57:23.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:57:23.831 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:57:23.831 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:57:23.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:57:23.831 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.60e+01, 1.42e+01 / 1.71e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.02e-01, 1.20e-02 / 6.20e-03 | 20260508 02:57:23.831 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.51e+00, 5.88e-01 / 1.80e-01 | 20260508 02:57:23.831 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.06e-01 / 9.40e-02 | 20260508 02:57:23.831 | INFO | PerfReport: 16.09%, DME -- 4.20e+01 / 4.59e+01, 4.05e+00 / 2.05e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.66e+00 / 1.38e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.41e+00 / 2.69e+00, 4.19e-01 / 1.90e-01 | 20260508 02:57:23.831 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.25e+00, 2.65e-01 / 1.37e-01 | 20260508 02:57:23.831 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.63e-01, 5.07e-02 / 2.29e-02 | 20260508 02:57:23.831 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:57:23.831 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:57:23.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:57:23.831 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 24.56%, Upload -- 2.62e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 30.61%, Compute -- 3.26e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 2.51%, LockUpload -- 2.68e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 2.52%, LockDownload -- 2.68e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 15.46%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 02:57:23.831 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 02:57:23.927 | INFO | Starting a round of Baz IO | 20260508 02:57:27.353 | INFO | Populating RTMetrics report | 20260508 02:57:27.354 | INFO | RTMetrics report populated | 20260508 02:57:27.354 | INFO | Opening tmp RTMetrics json file | 20260508 02:57:27.437 | INFO | Serializing RTMetrics to json | 20260508 02:57:27.468 | INFO | Serialization done, swapping tmp file now... | 20260508 02:57:27.475 | INFO | RTMetrics done | 20260508 02:57:27.502 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.575 seconds | 20260508 02:57:27.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.40% / 0.00% / 58.59% | 20260508 02:57:27.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.666666% / 1.19209e-05% / 8.34465e-05% | 20260508 02:57:27.502 | INFO | Baz IO overall: | 20260508 02:57:27.502 | INFO | BytesWritten : 2115083518328 | 20260508 02:57:27.503 | INFO | NumEvents : 1022165757732 | 20260508 02:57:28.973 | INFO | Analyzing chunk frames = [4694016,4694528) | 20260508 02:57:28.973 | INFO | MemInfo: { | 20260508 02:57:28.973 | INFO | "gpu" : " 17 GB", | 20260508 02:57:28.973 | INFO | "huge" : "", | 20260508 02:57:28.973 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:28.973 | INFO | "malloc" : " 14 GB", | 20260508 02:57:28.973 | INFO | "pinned" : " 29 GB" | 20260508 02:57:28.973 | INFO | } | 20260508 02:57:28.973 | INFO | Perf report for the last 5.14364 seconds: | 20260508 02:57:28.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:28.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:28.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:28.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:28.973 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 02:57:28.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:28.973 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 02:57:28.973 | INFO | PRE_HQ : 65.7%, 1.6, 56.0ms, 0 | 20260508 02:57:28.973 | INFO | BAZWRITER : 69.1%, 3.4, 122.5ms, 0 | 20260508 02:57:28.973 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260508 02:57:28.973 | INFO | Uploaded 36GiB to GPU in 2.54727s (14.1328GiB/s) | 20260508 02:57:28.974 | INFO | Downloaded 38GiB to GPU in 2.68111s (14.1732GiB/s) | 20260508 02:57:34.112 | INFO | Analyzing chunk frames = [4694528,4695040) | 20260508 02:57:34.112 | INFO | MemInfo: { | 20260508 02:57:34.112 | INFO | "gpu" : " 17 GB", | 20260508 02:57:34.112 | INFO | "huge" : "", | 20260508 02:57:34.112 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:34.112 | INFO | "malloc" : " 14 GB", | 20260508 02:57:34.112 | INFO | "pinned" : " 29 GB" | 20260508 02:57:34.112 | INFO | } | 20260508 02:57:34.112 | INFO | Perf report for the last 5.13922 seconds: | 20260508 02:57:34.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:34.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.112 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 02:57:34.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.113 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 02:57:34.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:34.113 | INFO | Uploaded 36GiB to GPU in 2.57904s (13.9587GiB/s) | 20260508 02:57:34.113 | INFO | Downloaded 34GiB to GPU in 2.41382s (14.0856GiB/s) | 20260508 02:57:39.250 | INFO | Analyzing chunk frames = [4695040,4695552) | 20260508 02:57:39.250 | INFO | MemInfo: { | 20260508 02:57:39.250 | INFO | "gpu" : " 17 GB", | 20260508 02:57:39.250 | INFO | "huge" : "", | 20260508 02:57:39.250 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:39.250 | INFO | "malloc" : " 14 GB", | 20260508 02:57:39.250 | INFO | "pinned" : " 29 GB" | 20260508 02:57:39.250 | INFO | } | 20260508 02:57:39.250 | INFO | Perf report for the last 5.13759 seconds: | 20260508 02:57:39.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:39.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.250 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 02:57:39.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.250 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:57:39.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:39.251 | INFO | Uploaded 36GiB to GPU in 2.58443s (13.9296GiB/s) | 20260508 02:57:39.251 | INFO | Downloaded 34GiB to GPU in 2.49577s (13.623GiB/s) | 20260508 02:57:44.387 | INFO | Analyzing chunk frames = [4695552,4696064) | 20260508 02:57:44.388 | INFO | MemInfo: { | 20260508 02:57:44.388 | INFO | "gpu" : " 17 GB", | 20260508 02:57:44.388 | INFO | "huge" : "", | 20260508 02:57:44.388 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:44.388 | INFO | "malloc" : " 14 GB", | 20260508 02:57:44.388 | INFO | "pinned" : " 29 GB" | 20260508 02:57:44.388 | INFO | } | 20260508 02:57:44.388 | INFO | Perf report for the last 5.13776 seconds: | 20260508 02:57:44.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:44.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:57:44.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 02:57:44.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:44.388 | INFO | Uploaded 36GiB to GPU in 2.54561s (14.142GiB/s) | 20260508 02:57:44.388 | INFO | Downloaded 34GiB to GPU in 2.47891s (13.7157GiB/s) | 20260508 02:57:49.528 | INFO | Analyzing chunk frames = [4696064,4696576) | 20260508 02:57:49.528 | INFO | MemInfo: { | 20260508 02:57:49.528 | INFO | "gpu" : " 17 GB", | 20260508 02:57:49.528 | INFO | "huge" : "", | 20260508 02:57:49.528 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:49.528 | INFO | "malloc" : " 14 GB", | 20260508 02:57:49.529 | INFO | "pinned" : " 29 GB" | 20260508 02:57:49.529 | INFO | } | 20260508 02:57:49.529 | INFO | Perf report for the last 5.14081 seconds: | 20260508 02:57:49.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:49.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 02:57:49.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:57:49.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:49.529 | INFO | Uploaded 36GiB to GPU in 2.41635s (14.8985GiB/s) | 20260508 02:57:49.529 | INFO | Downloaded 34GiB to GPU in 2.56439s (13.2585GiB/s) | 20260508 02:57:54.670 | INFO | Analyzing chunk frames = [4696576,4697088) | 20260508 02:57:54.670 | INFO | MemInfo: { | 20260508 02:57:54.670 | INFO | "gpu" : " 17 GB", | 20260508 02:57:54.670 | INFO | "huge" : "", | 20260508 02:57:54.670 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:54.670 | INFO | "malloc" : " 14 GB", | 20260508 02:57:54.670 | INFO | "pinned" : " 29 GB" | 20260508 02:57:54.670 | INFO | } | 20260508 02:57:54.671 | INFO | Perf report for the last 5.14191 seconds: | 20260508 02:57:54.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:54.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 02:57:54.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:57:54.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:54.671 | INFO | Uploaded 36GiB to GPU in 2.41351s (14.916GiB/s) | 20260508 02:57:54.671 | INFO | Downloaded 34GiB to GPU in 2.42013s (14.0489GiB/s) | 20260508 02:57:59.810 | INFO | Analyzing chunk frames = [4697088,4697600) | 20260508 02:57:59.810 | INFO | MemInfo: { | 20260508 02:57:59.810 | INFO | "gpu" : " 17 GB", | 20260508 02:57:59.810 | INFO | "huge" : "", | 20260508 02:57:59.810 | INFO | "huge_shared" : " 29 GB", | 20260508 02:57:59.810 | INFO | "malloc" : " 14 GB", | 20260508 02:57:59.810 | INFO | "pinned" : " 29 GB" | 20260508 02:57:59.810 | INFO | } | 20260508 02:57:59.810 | INFO | Perf report for the last 5.13967 seconds: | 20260508 02:57:59.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:57:59.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 02:57:59.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 02:57:59.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:57:59.810 | INFO | Uploaded 36GiB to GPU in 2.43578s (14.7797GiB/s) | 20260508 02:57:59.811 | INFO | Downloaded 34GiB to GPU in 2.42305s (14.0319GiB/s) | 20260508 02:58:04.947 | INFO | Analyzing chunk frames = [4697600,4698112) | 20260508 02:58:04.947 | INFO | MemInfo: { | 20260508 02:58:04.947 | INFO | "gpu" : " 17 GB", | 20260508 02:58:04.947 | INFO | "huge" : "", | 20260508 02:58:04.947 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:04.947 | INFO | "malloc" : " 14 GB", | 20260508 02:58:04.947 | INFO | "pinned" : " 29 GB" | 20260508 02:58:04.947 | INFO | } | 20260508 02:58:04.947 | INFO | Perf report for the last 5.13679 seconds: | 20260508 02:58:04.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:04.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 02:58:04.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:58:04.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:04.947 | INFO | Uploaded 36GiB to GPU in 2.46187s (14.623GiB/s) | 20260508 02:58:04.947 | INFO | Downloaded 34GiB to GPU in 2.43117s (13.985GiB/s) | 20260508 02:58:05.044 | INFO | Starting a round of Baz IO | 20260508 02:58:08.504 | INFO | Populating RTMetrics report | 20260508 02:58:08.504 | INFO | RTMetrics report populated | 20260508 02:58:08.504 | INFO | Opening tmp RTMetrics json file | 20260508 02:58:08.515 | INFO | Serializing RTMetrics to json | 20260508 02:58:08.524 | INFO | Serialization done, swapping tmp file now... | 20260508 02:58:08.525 | INFO | RTMetrics done | 20260508 02:58:08.594 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.55 seconds | 20260508 02:58:08.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.40% / 0.00% / 58.60% | 20260508 02:58:08.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.668645% / 3.97364e-06% / 5.16574e-05% | 20260508 02:58:08.594 | INFO | Baz IO overall: | 20260508 02:58:08.594 | INFO | BytesWritten : 2116831807326 | 20260508 02:58:08.594 | INFO | NumEvents : 1023003954412 | 20260508 02:58:10.083 | INFO | Analyzing chunk frames = [4698112,4698624) | 20260508 02:58:10.083 | INFO | MemInfo: { | 20260508 02:58:10.083 | INFO | "gpu" : " 17 GB", | 20260508 02:58:10.083 | INFO | "huge" : "", | 20260508 02:58:10.083 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:10.083 | INFO | "malloc" : " 14 GB", | 20260508 02:58:10.083 | INFO | "pinned" : " 29 GB" | 20260508 02:58:10.084 | INFO | } | 20260508 02:58:10.084 | INFO | Perf report for the last 5.13654 seconds: | 20260508 02:58:10.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:10.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:10.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:10.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:10.084 | INFO | ANALYSIS : 68.6%, 2.9, 104.4ms, 0 | 20260508 02:58:10.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:10.084 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 02:58:10.084 | INFO | PRE_HQ : 67.1%, 1.7, 60.7ms, 0 | 20260508 02:58:10.084 | INFO | BAZWRITER : 68.7%, 3.9, 138.2ms, 0 | 20260508 02:58:10.084 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 02:58:10.084 | INFO | Uploaded 36GiB to GPU in 2.60222s (13.8344GiB/s) | 20260508 02:58:10.084 | INFO | Downloaded 38GiB to GPU in 2.71248s (14.0093GiB/s) | 20260508 02:58:15.229 | INFO | Analyzing chunk frames = [4698624,4699136) | 20260508 02:58:15.229 | INFO | MemInfo: { | 20260508 02:58:15.229 | INFO | "gpu" : " 17 GB", | 20260508 02:58:15.229 | INFO | "huge" : "", | 20260508 02:58:15.229 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:15.229 | INFO | "malloc" : " 14 GB", | 20260508 02:58:15.229 | INFO | "pinned" : " 29 GB" | 20260508 02:58:15.229 | INFO | } | 20260508 02:58:15.229 | INFO | Perf report for the last 5.14592 seconds: | 20260508 02:58:15.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:15.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 02:58:15.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 02:58:15.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:15.230 | INFO | Uploaded 36GiB to GPU in 2.54934s (14.1213GiB/s) | 20260508 02:58:15.230 | INFO | Downloaded 34GiB to GPU in 2.4296s (13.9941GiB/s) | 20260508 02:58:20.365 | INFO | Analyzing chunk frames = [4699136,4699648) | 20260508 02:58:20.365 | INFO | MemInfo: { | 20260508 02:58:20.365 | INFO | "gpu" : " 17 GB", | 20260508 02:58:20.365 | INFO | "huge" : "", | 20260508 02:58:20.365 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:20.365 | INFO | "malloc" : " 14 GB", | 20260508 02:58:20.365 | INFO | "pinned" : " 29 GB" | 20260508 02:58:20.365 | INFO | } | 20260508 02:58:20.365 | INFO | Perf report for the last 5.136 seconds: | 20260508 02:58:20.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:20.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.366 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 02:58:20.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.366 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:58:20.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:20.366 | INFO | Uploaded 36GiB to GPU in 2.53444s (14.2043GiB/s) | 20260508 02:58:20.366 | INFO | Downloaded 34GiB to GPU in 2.41821s (14.06GiB/s) | 20260508 02:58:25.507 | INFO | Analyzing chunk frames = [4699648,4700160) | 20260508 02:58:25.507 | INFO | MemInfo: { | 20260508 02:58:25.507 | INFO | "gpu" : " 17 GB", | 20260508 02:58:25.507 | INFO | "huge" : "", | 20260508 02:58:25.507 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:25.507 | INFO | "malloc" : " 14 GB", | 20260508 02:58:25.507 | INFO | "pinned" : " 29 GB" | 20260508 02:58:25.507 | INFO | } | 20260508 02:58:25.507 | INFO | Perf report for the last 5.14122 seconds: | 20260508 02:58:25.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:25.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 02:58:25.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 02:58:25.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:25.507 | INFO | Uploaded 36GiB to GPU in 2.53048s (14.2265GiB/s) | 20260508 02:58:25.507 | INFO | Downloaded 34GiB to GPU in 2.50237s (13.5871GiB/s) | 20260508 02:58:30.643 | INFO | Analyzing chunk frames = [4700160,4700672) | 20260508 02:58:30.643 | INFO | MemInfo: { | 20260508 02:58:30.643 | INFO | "gpu" : " 17 GB", | 20260508 02:58:30.643 | INFO | "huge" : "", | 20260508 02:58:30.643 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:30.643 | INFO | "malloc" : " 14 GB", | 20260508 02:58:30.643 | INFO | "pinned" : " 29 GB" | 20260508 02:58:30.644 | INFO | } | 20260508 02:58:30.644 | INFO | Perf report for the last 5.13683 seconds: | 20260508 02:58:30.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:30.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 02:58:30.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 02:58:30.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:30.644 | INFO | Uploaded 36GiB to GPU in 2.38562s (15.0904GiB/s) | 20260508 02:58:30.644 | INFO | Downloaded 34GiB to GPU in 2.55406s (13.3122GiB/s) | 20260508 02:58:35.782 | INFO | Analyzing chunk frames = [4700672,4701184) | 20260508 02:58:35.782 | INFO | MemInfo: { | 20260508 02:58:35.782 | INFO | "gpu" : " 17 GB", | 20260508 02:58:35.783 | INFO | "huge" : "", | 20260508 02:58:35.783 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:35.783 | INFO | "malloc" : " 14 GB", | 20260508 02:58:35.783 | INFO | "pinned" : " 29 GB" | 20260508 02:58:35.783 | INFO | } | 20260508 02:58:35.783 | INFO | Perf report for the last 5.13916 seconds: | 20260508 02:58:35.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:35.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 02:58:35.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:58:35.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:35.783 | INFO | Uploaded 36GiB to GPU in 2.44916s (14.6989GiB/s) | 20260508 02:58:35.783 | INFO | Downloaded 34GiB to GPU in 2.41624s (14.0715GiB/s) | 20260508 02:58:40.923 | INFO | Analyzing chunk frames = [4701184,4701696) | 20260508 02:58:40.923 | INFO | MemInfo: { | 20260508 02:58:40.923 | INFO | "gpu" : " 17 GB", | 20260508 02:58:40.923 | INFO | "huge" : "", | 20260508 02:58:40.923 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:40.923 | INFO | "malloc" : " 14 GB", | 20260508 02:58:40.923 | INFO | "pinned" : " 29 GB" | 20260508 02:58:40.923 | INFO | } | 20260508 02:58:40.923 | INFO | Perf report for the last 5.14069 seconds: | 20260508 02:58:40.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:40.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:58:40.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 02:58:40.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:40.924 | INFO | Uploaded 36GiB to GPU in 2.47119s (14.5679GiB/s) | 20260508 02:58:40.925 | INFO | Downloaded 34GiB to GPU in 2.42683s (14.01GiB/s) | 20260508 02:58:46.065 | INFO | Analyzing chunk frames = [4701696,4702208) | 20260508 02:58:46.065 | INFO | MemInfo: { | 20260508 02:58:46.065 | INFO | "gpu" : " 17 GB", | 20260508 02:58:46.065 | INFO | "huge" : "", | 20260508 02:58:46.065 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:46.065 | INFO | "malloc" : " 14 GB", | 20260508 02:58:46.065 | INFO | "pinned" : " 29 GB" | 20260508 02:58:46.065 | INFO | } | 20260508 02:58:46.065 | INFO | Perf report for the last 5.14173 seconds: | 20260508 02:58:46.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:46.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.065 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:58:46.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.065 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260508 02:58:46.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:46.066 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 02:58:46.066 | INFO | Total mem usage: 16.7 GiB | 20260508 02:58:46.066 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 02:58:46.066 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -9.43%) | 20260508 02:58:46.066 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 02:58:46.066 | INFO | High Water Breakdown: | 20260508 02:58:46.066 | INFO | 60.5% : 9.23 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 02:58:46.066 | INFO | 39.3% : 6 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 02:58:46.066 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 02:58:46.066 | INFO | ------------------------------------------ | 20260508 02:58:46.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:58:46.066 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 02:58:46.066 | INFO | Total mem usage: 26.5 GiB | 20260508 02:58:46.066 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 02:58:46.066 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 02:58:46.066 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 02:58:46.066 | INFO | High Water Breakdown: | 20260508 02:58:46.066 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:58:46.066 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:58:46.066 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:58:46.066 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:58:46.066 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:58:46.066 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:58:46.066 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 02:58:46.066 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 02:58:46.066 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:58:46.066 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 02:58:46.066 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 02:58:46.066 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 02:58:46.066 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 02:58:46.066 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 02:58:46.066 | INFO | ------------------------------------------ | 20260508 02:58:46.066 | INFO | Uploaded 36GiB to GPU in 2.47428s (14.5497GiB/s) | 20260508 02:58:46.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:58:46.066 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 02:58:46.066 | INFO | Total mem usage: 27.8 GiB | 20260508 02:58:46.066 | INFO | Downloaded 34GiB to GPU in 2.42524s (14.0192GiB/s) | 20260508 02:58:46.066 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 02:58:46.066 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 02:58:46.066 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 02:58:46.066 | INFO | High Water Breakdown: | 20260508 02:58:46.066 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 02:58:46.066 | INFO | ------------------------------------------ | 20260508 02:58:46.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:58:46.066 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 02:58:46.066 | INFO | Total mem usage: 15.7 GiB | 20260508 02:58:46.066 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 02:58:46.066 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 02:58:46.066 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 02:58:46.066 | INFO | High Water Breakdown: | 20260508 02:58:46.066 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 02:58:46.066 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 02:58:46.066 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 02:58:46.066 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 02:58:46.066 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 02:58:46.066 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 02:58:46.066 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 02:58:46.066 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 02:58:46.066 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 02:58:46.066 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 02:58:46.066 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 02:58:46.066 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 02:58:46.066 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 02:58:46.066 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 02:58:46.066 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 02:58:46.066 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 02:58:46.066 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 02:58:46.066 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 02:58:46.066 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 02:58:46.066 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 02:58:46.067 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 02:58:46.067 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 02:58:46.067 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 02:58:46.067 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 02:58:46.067 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 02:58:46.067 | INFO | ------------------------------------------ | 20260508 02:58:46.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 02:58:46.067 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 02:58:46.067 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:58:46.067 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:58:46.067 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.42e+01 / 1.51e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.24e-02 / 5.66e-03 | 20260508 02:58:46.067 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 5.91e-01 / 1.62e-01 | 20260508 02:58:46.067 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.18e-01 | 20260508 02:58:46.067 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.59e+01, 3.92e+00 / 1.99e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.71e+00 / 1.56e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.21e-01 / 2.51e-01 | 20260508 02:58:46.067 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.23e+00, 2.68e-01 / 1.25e-01 | 20260508 02:58:46.067 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.65e-01, 5.13e-02 / 2.75e-02 | 20260508 02:58:46.067 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 02:58:46.067 | INFO | PerfReport: Gathered 1536 samples. | 20260508 02:58:46.067 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 02:58:46.067 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 24.35%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 2.27%, LockUpload -- 2.42e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 2.65%, LockDownload -- 2.82e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 02:58:46.067 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260508 02:58:46.162 | INFO | Starting a round of Baz IO | 20260508 02:58:49.610 | INFO | Populating RTMetrics report | 20260508 02:58:49.610 | INFO | RTMetrics report populated | 20260508 02:58:49.610 | INFO | Opening tmp RTMetrics json file | 20260508 02:58:49.631 | INFO | Serializing RTMetrics to json | 20260508 02:58:49.632 | INFO | Serialization done, swapping tmp file now... | 20260508 02:58:49.633 | INFO | RTMetrics done | 20260508 02:58:49.708 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.545 seconds | 20260508 02:58:49.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.40% / 0.00% / 58.60% | 20260508 02:58:49.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.667667% / 3.97364e-06% / 5.5631e-05% | 20260508 02:58:49.708 | INFO | Baz IO overall: | 20260508 02:58:49.708 | INFO | BytesWritten : 2118579759803 | 20260508 02:58:49.708 | INFO | NumEvents : 1023841736623 | 20260508 02:58:51.202 | INFO | Analyzing chunk frames = [4702208,4702720) | 20260508 02:58:51.202 | INFO | MemInfo: { | 20260508 02:58:51.202 | INFO | "gpu" : " 17 GB", | 20260508 02:58:51.202 | INFO | "huge" : "", | 20260508 02:58:51.202 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:51.202 | INFO | "malloc" : " 14 GB", | 20260508 02:58:51.202 | INFO | "pinned" : " 29 GB" | 20260508 02:58:51.202 | INFO | } | 20260508 02:58:51.202 | INFO | Perf report for the last 5.13637 seconds: | 20260508 02:58:51.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:51.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:51.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:51.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:51.202 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 02:58:51.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:51.202 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 02:58:51.202 | INFO | PRE_HQ : 65.8%, 1.6, 56.1ms, 0 | 20260508 02:58:51.202 | INFO | BAZWRITER : 68.7%, 4.0, 142.1ms, 0 | 20260508 02:58:51.202 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 02:58:51.202 | INFO | Uploaded 36GiB to GPU in 2.58434s (13.93GiB/s) | 20260508 02:58:51.202 | INFO | Downloaded 38GiB to GPU in 2.70165s (14.0655GiB/s) | 20260508 02:58:56.336 | INFO | Analyzing chunk frames = [4702720,4703232) | 20260508 02:58:56.336 | INFO | MemInfo: { | 20260508 02:58:56.336 | INFO | "gpu" : " 17 GB", | 20260508 02:58:56.336 | INFO | "huge" : "", | 20260508 02:58:56.336 | INFO | "huge_shared" : " 29 GB", | 20260508 02:58:56.336 | INFO | "malloc" : " 14 GB", | 20260508 02:58:56.336 | INFO | "pinned" : " 29 GB" | 20260508 02:58:56.336 | INFO | } | 20260508 02:58:56.337 | INFO | Perf report for the last 5.13493 seconds: | 20260508 02:58:56.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:58:56.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 02:58:56.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 02:58:56.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:58:56.337 | INFO | Uploaded 36GiB to GPU in 2.5619s (14.0521GiB/s) | 20260508 02:58:56.337 | INFO | Downloaded 34GiB to GPU in 2.43854s (13.9428GiB/s) | 20260508 02:59:01.478 | INFO | Analyzing chunk frames = [4703232,4703744) | 20260508 02:59:01.478 | INFO | MemInfo: { | 20260508 02:59:01.478 | INFO | "gpu" : " 17 GB", | 20260508 02:59:01.478 | INFO | "huge" : "", | 20260508 02:59:01.478 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:01.479 | INFO | "malloc" : " 14 GB", | 20260508 02:59:01.479 | INFO | "pinned" : " 29 GB" | 20260508 02:59:01.479 | INFO | } | 20260508 02:59:01.479 | INFO | Perf report for the last 5.14204 seconds: | 20260508 02:59:01.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:01.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 02:59:01.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 02:59:01.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:01.479 | INFO | Uploaded 36GiB to GPU in 2.53449s (14.204GiB/s) | 20260508 02:59:01.479 | INFO | Downloaded 34GiB to GPU in 2.50168s (13.5909GiB/s) | 20260508 02:59:06.625 | INFO | Analyzing chunk frames = [4703744,4704256) | 20260508 02:59:06.625 | INFO | MemInfo: { | 20260508 02:59:06.625 | INFO | "gpu" : " 17 GB", | 20260508 02:59:06.625 | INFO | "huge" : "", | 20260508 02:59:06.625 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:06.625 | INFO | "malloc" : " 14 GB", | 20260508 02:59:06.625 | INFO | "pinned" : " 29 GB" | 20260508 02:59:06.625 | INFO | } | 20260508 02:59:06.625 | INFO | Perf report for the last 5.14638 seconds: | 20260508 02:59:06.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:06.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 02:59:06.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 02:59:06.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:06.625 | INFO | Uploaded 36GiB to GPU in 2.53063s (14.2257GiB/s) | 20260508 02:59:06.626 | INFO | Downloaded 34GiB to GPU in 2.50814s (13.5558GiB/s) | 20260508 02:59:11.760 | INFO | Analyzing chunk frames = [4704256,4704768) | 20260508 02:59:11.760 | INFO | MemInfo: { | 20260508 02:59:11.760 | INFO | "gpu" : " 17 GB", | 20260508 02:59:11.760 | INFO | "huge" : "", | 20260508 02:59:11.760 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:11.760 | INFO | "malloc" : " 14 GB", | 20260508 02:59:11.760 | INFO | "pinned" : " 29 GB" | 20260508 02:59:11.760 | INFO | } | 20260508 02:59:11.760 | INFO | Perf report for the last 5.13531 seconds: | 20260508 02:59:11.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:11.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.760 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 02:59:11.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.760 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 02:59:11.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:11.761 | INFO | Uploaded 36GiB to GPU in 2.31179s (15.5723GiB/s) | 20260508 02:59:11.761 | INFO | Downloaded 34GiB to GPU in 2.56352s (13.263GiB/s) | 20260508 02:59:16.894 | INFO | Analyzing chunk frames = [4704768,4705280) | 20260508 02:59:16.894 | INFO | MemInfo: { | 20260508 02:59:16.894 | INFO | "gpu" : " 17 GB", | 20260508 02:59:16.894 | INFO | "huge" : "", | 20260508 02:59:16.895 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:16.895 | INFO | "malloc" : " 14 GB", | 20260508 02:59:16.895 | INFO | "pinned" : " 29 GB" | 20260508 02:59:16.895 | INFO | } | 20260508 02:59:16.895 | INFO | Perf report for the last 5.13438 seconds: | 20260508 02:59:16.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:16.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 02:59:16.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 02:59:16.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:16.895 | INFO | Uploaded 36GiB to GPU in 2.406s (14.9626GiB/s) | 20260508 02:59:16.896 | INFO | Downloaded 34GiB to GPU in 2.42958s (13.9942GiB/s) | 20260508 02:59:22.037 | INFO | Analyzing chunk frames = [4705280,4705792) | 20260508 02:59:22.037 | INFO | MemInfo: { | 20260508 02:59:22.037 | INFO | "gpu" : " 17 GB", | 20260508 02:59:22.037 | INFO | "huge" : "", | 20260508 02:59:22.037 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:22.037 | INFO | "malloc" : " 14 GB", | 20260508 02:59:22.037 | INFO | "pinned" : " 29 GB" | 20260508 02:59:22.037 | INFO | } | 20260508 02:59:22.038 | INFO | Perf report for the last 5.14278 seconds: | 20260508 02:59:22.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:22.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 02:59:22.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 02:59:22.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:22.038 | INFO | Uploaded 36GiB to GPU in 2.46153s (14.6251GiB/s) | 20260508 02:59:22.038 | INFO | Downloaded 34GiB to GPU in 2.44771s (13.8905GiB/s) | 20260508 02:59:27.178 | INFO | Analyzing chunk frames = [4705792,4706304) | 20260508 02:59:27.178 | INFO | MemInfo: { | 20260508 02:59:27.178 | INFO | "gpu" : " 17 GB", | 20260508 02:59:27.178 | INFO | "huge" : "", | 20260508 02:59:27.178 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:27.178 | INFO | "malloc" : " 14 GB", | 20260508 02:59:27.178 | INFO | "pinned" : " 29 GB" | 20260508 02:59:27.178 | INFO | } | 20260508 02:59:27.178 | INFO | Perf report for the last 5.14068 seconds: | 20260508 02:59:27.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:27.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.178 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 02:59:27.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.178 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 02:59:27.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:27.179 | INFO | Uploaded 36GiB to GPU in 2.46405s (14.6101GiB/s) | 20260508 02:59:27.179 | INFO | Downloaded 34GiB to GPU in 2.43888s (13.9408GiB/s) | 20260508 02:59:27.291 | INFO | Starting a round of Baz IO | 20260508 02:59:30.735 | INFO | Populating RTMetrics report | 20260508 02:59:30.735 | INFO | RTMetrics report populated | 20260508 02:59:30.735 | INFO | Opening tmp RTMetrics json file | 20260508 02:59:30.758 | INFO | Serializing RTMetrics to json | 20260508 02:59:30.817 | INFO | Serialization done, swapping tmp file now... | 20260508 02:59:30.839 | INFO | RTMetrics done | 20260508 02:59:30.874 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.582 seconds | 20260508 02:59:30.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.40% / 0.00% / 58.60% | 20260508 02:59:30.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.662788% / 3.97364e-06% / 2.78155e-05% | 20260508 02:59:30.874 | INFO | Baz IO overall: | 20260508 02:59:30.874 | INFO | BytesWritten : 2120325036693 | 20260508 02:59:30.874 | INFO | NumEvents : 1024677609502 | 20260508 02:59:32.316 | INFO | Analyzing chunk frames = [4706304,4706816) | 20260508 02:59:32.316 | INFO | MemInfo: { | 20260508 02:59:32.316 | INFO | "gpu" : " 17 GB", | 20260508 02:59:32.316 | INFO | "huge" : "", | 20260508 02:59:32.316 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:32.316 | INFO | "malloc" : " 14 GB", | 20260508 02:59:32.316 | INFO | "pinned" : " 29 GB" | 20260508 02:59:32.316 | INFO | } | 20260508 02:59:32.316 | INFO | Perf report for the last 5.13763 seconds: | 20260508 02:59:32.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:32.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:32.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:32.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:32.316 | INFO | ANALYSIS : 68.5%, 2.9, 104.8ms, 0 | 20260508 02:59:32.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:32.316 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260508 02:59:32.316 | INFO | PRE_HQ : 67.4%, 1.8, 62.3ms, 0 | 20260508 02:59:32.316 | INFO | BAZWRITER : 69.3%, 3.7, 130.8ms, 0 | 20260508 02:59:32.316 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260508 02:59:32.316 | INFO | Uploaded 36GiB to GPU in 2.66021s (13.5328GiB/s) | 20260508 02:59:32.316 | INFO | Downloaded 38GiB to GPU in 2.70931s (14.0257GiB/s) | 20260508 02:59:37.458 | INFO | Analyzing chunk frames = [4706816,4707328) | 20260508 02:59:37.458 | INFO | MemInfo: { | 20260508 02:59:37.458 | INFO | "gpu" : " 17 GB", | 20260508 02:59:37.458 | INFO | "huge" : "", | 20260508 02:59:37.458 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:37.458 | INFO | "malloc" : " 14 GB", | 20260508 02:59:37.458 | INFO | "pinned" : " 29 GB" | 20260508 02:59:37.458 | INFO | } | 20260508 02:59:37.458 | INFO | Perf report for the last 5.14249 seconds: | 20260508 02:59:37.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:37.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.458 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 02:59:37.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.458 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 02:59:37.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:37.459 | INFO | Uploaded 36GiB to GPU in 2.57739s (13.9676GiB/s) | 20260508 02:59:37.459 | INFO | Downloaded 34GiB to GPU in 2.41609s (14.0723GiB/s) | 20260508 02:59:42.593 | INFO | Analyzing chunk frames = [4707328,4707840) | 20260508 02:59:42.593 | INFO | MemInfo: { | 20260508 02:59:42.593 | INFO | "gpu" : " 17 GB", | 20260508 02:59:42.593 | INFO | "huge" : "", | 20260508 02:59:42.593 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:42.593 | INFO | "malloc" : " 14 GB", | 20260508 02:59:42.593 | INFO | "pinned" : " 29 GB" | 20260508 02:59:42.593 | INFO | } | 20260508 02:59:42.593 | INFO | Perf report for the last 5.1345 seconds: | 20260508 02:59:42.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:42.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | ANALYSIS : 67.6%, 3.0, 105.0ms, 0 | 20260508 02:59:42.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 02:59:42.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:42.593 | INFO | Uploaded 36GiB to GPU in 2.55859s (14.0702GiB/s) | 20260508 02:59:42.593 | INFO | Downloaded 34GiB to GPU in 2.42515s (14.0197GiB/s) | 20260508 02:59:47.735 | INFO | Analyzing chunk frames = [4707840,4708352) | 20260508 02:59:47.735 | INFO | MemInfo: { | 20260508 02:59:47.735 | INFO | "gpu" : " 17 GB", | 20260508 02:59:47.735 | INFO | "huge" : "", | 20260508 02:59:47.735 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:47.735 | INFO | "malloc" : " 14 GB", | 20260508 02:59:47.735 | INFO | "pinned" : " 29 GB" | 20260508 02:59:47.735 | INFO | } | 20260508 02:59:47.735 | INFO | Perf report for the last 5.14198 seconds: | 20260508 02:59:47.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:47.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 02:59:47.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:59:47.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:47.735 | INFO | Uploaded 36GiB to GPU in 2.5661s (14.0291GiB/s) | 20260508 02:59:47.736 | INFO | Downloaded 34GiB to GPU in 2.48997s (13.6548GiB/s) | 20260508 02:59:52.878 | INFO | Analyzing chunk frames = [4708352,4708864) | 20260508 02:59:52.878 | INFO | MemInfo: { | 20260508 02:59:52.878 | INFO | "gpu" : " 17 GB", | 20260508 02:59:52.878 | INFO | "huge" : "", | 20260508 02:59:52.878 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:52.878 | INFO | "malloc" : " 14 GB", | 20260508 02:59:52.878 | INFO | "pinned" : " 29 GB" | 20260508 02:59:52.878 | INFO | } | 20260508 02:59:52.878 | INFO | Perf report for the last 5.14317 seconds: | 20260508 02:59:52.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:52.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.878 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 02:59:52.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.878 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 02:59:52.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:52.879 | INFO | Uploaded 36GiB to GPU in 2.41439s (14.9106GiB/s) | 20260508 02:59:52.879 | INFO | Downloaded 34GiB to GPU in 2.55769s (13.2933GiB/s) | 20260508 02:59:58.013 | INFO | Analyzing chunk frames = [4708864,4709376) | 20260508 02:59:58.013 | INFO | MemInfo: { | 20260508 02:59:58.013 | INFO | "gpu" : " 17 GB", | 20260508 02:59:58.013 | INFO | "huge" : "", | 20260508 02:59:58.013 | INFO | "huge_shared" : " 29 GB", | 20260508 02:59:58.013 | INFO | "malloc" : " 14 GB", | 20260508 02:59:58.013 | INFO | "pinned" : " 29 GB" | 20260508 02:59:58.013 | INFO | } | 20260508 02:59:58.013 | INFO | Perf report for the last 5.13495 seconds: | 20260508 02:59:58.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 02:59:58.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.013 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 02:59:58.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.013 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 02:59:58.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 02:59:58.014 | INFO | Uploaded 36GiB to GPU in 2.41853s (14.8851GiB/s) | 20260508 02:59:58.014 | INFO | Downloaded 34GiB to GPU in 2.4166s (14.0694GiB/s) | 20260508 03:00:03.149 | INFO | Analyzing chunk frames = [4709376,4709888) | 20260508 03:00:03.149 | INFO | MemInfo: { | 20260508 03:00:03.149 | INFO | "gpu" : " 17 GB", | 20260508 03:00:03.149 | INFO | "huge" : "", | 20260508 03:00:03.149 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:03.149 | INFO | "malloc" : " 14 GB", | 20260508 03:00:03.149 | INFO | "pinned" : " 29 GB" | 20260508 03:00:03.149 | INFO | } | 20260508 03:00:03.149 | INFO | Perf report for the last 5.13614 seconds: | 20260508 03:00:03.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:03.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.149 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:00:03.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.149 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:00:03.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:03.150 | INFO | Uploaded 36GiB to GPU in 2.43036s (14.8126GiB/s) | 20260508 03:00:03.150 | INFO | Downloaded 34GiB to GPU in 2.42508s (14.0201GiB/s) | 20260508 03:00:08.292 | INFO | Analyzing chunk frames = [4709888,4710400) | 20260508 03:00:08.292 | INFO | MemInfo: { | 20260508 03:00:08.292 | INFO | "gpu" : " 17 GB", | 20260508 03:00:08.292 | INFO | "huge" : "", | 20260508 03:00:08.292 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:08.292 | INFO | "malloc" : " 14 GB", | 20260508 03:00:08.292 | INFO | "pinned" : " 29 GB" | 20260508 03:00:08.292 | INFO | } | 20260508 03:00:08.292 | INFO | Perf report for the last 5.14284 seconds: | 20260508 03:00:08.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:08.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:00:08.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:00:08.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:08.292 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:00:08.292 | INFO | Total mem usage: 16.7 GiB | 20260508 03:00:08.293 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:00:08.293 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.8%) | 20260508 03:00:08.293 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:00:08.293 | INFO | High Water Breakdown: | 20260508 03:00:08.293 | INFO | 59.8% : 8.77 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:00:08.293 | INFO | 40% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:00:08.293 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:00:08.293 | INFO | ------------------------------------------ | 20260508 03:00:08.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:00:08.293 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:00:08.293 | INFO | Total mem usage: 26.5 GiB | 20260508 03:00:08.293 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:00:08.293 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:00:08.293 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:00:08.293 | INFO | High Water Breakdown: | 20260508 03:00:08.293 | INFO | Uploaded 36GiB to GPU in 2.55028s (14.1161GiB/s) | 20260508 03:00:08.293 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:00:08.293 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:00:08.293 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:00:08.293 | INFO | Downloaded 34GiB to GPU in 2.45923s (13.8255GiB/s) | 20260508 03:00:08.293 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:00:08.293 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:00:08.293 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:00:08.293 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:00:08.293 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:00:08.293 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:00:08.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:00:08.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:00:08.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:00:08.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:00:08.293 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:00:08.293 | INFO | ------------------------------------------ | 20260508 03:00:08.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:00:08.293 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:00:08.293 | INFO | Total mem usage: 27.8 GiB | 20260508 03:00:08.293 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:00:08.293 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:00:08.293 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:00:08.293 | INFO | High Water Breakdown: | 20260508 03:00:08.293 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:00:08.293 | INFO | ------------------------------------------ | 20260508 03:00:08.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:00:08.293 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:00:08.293 | INFO | Total mem usage: 15.7 GiB | 20260508 03:00:08.293 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:00:08.293 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:00:08.293 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:00:08.293 | INFO | High Water Breakdown: | 20260508 03:00:08.293 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:00:08.293 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:00:08.293 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:00:08.293 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:00:08.293 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:00:08.293 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:00:08.293 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:00:08.293 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:00:08.293 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:00:08.293 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:00:08.293 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:00:08.293 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:00:08.293 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:00:08.293 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:00:08.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:00:08.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:00:08.293 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:00:08.293 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:00:08.293 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:00:08.293 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:00:08.293 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:00:08.293 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:00:08.293 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:00:08.293 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:00:08.293 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:00:08.293 | INFO | ------------------------------------------ | 20260508 03:00:08.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:00:08.293 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:00:08.293 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:00:08.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:00:08.293 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.42e+01 / 1.84e+00 | 20260508 03:00:08.293 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.04e-01, 1.12e-02 / 5.65e-03 | 20260508 03:00:08.293 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.50e+00, 5.87e-01 / 1.84e-01 | 20260508 03:00:08.293 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 1.05e-01 | 20260508 03:00:08.293 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.58e+01, 3.99e+00 / 2.10e+00 | 20260508 03:00:08.293 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.69e+00 / 1.56e+00 | 20260508 03:00:08.293 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.41e+00 / 2.68e+00, 4.21e-01 / 2.39e-01 | 20260508 03:00:08.293 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.21e+00, 2.67e-01 / 1.19e-01 | 20260508 03:00:08.293 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.65e-01, 5.24e-02 / 2.76e-02 | 20260508 03:00:08.294 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:00:08.294 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:00:08.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:00:08.294 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 24.48%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 2.22%, LockUpload -- 2.37e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 2.70%, LockDownload -- 2.88e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 15.45%, LockCompute -- 1.64e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:00:08.294 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 03:00:08.389 | INFO | Starting a round of Baz IO | 20260508 03:00:11.833 | INFO | Populating RTMetrics report | 20260508 03:00:11.833 | INFO | RTMetrics report populated | 20260508 03:00:11.833 | INFO | Opening tmp RTMetrics json file | 20260508 03:00:11.856 | INFO | Serializing RTMetrics to json | 20260508 03:00:11.858 | INFO | Serialization done, swapping tmp file now... | 20260508 03:00:11.859 | INFO | RTMetrics done | 20260508 03:00:11.924 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.534 seconds | 20260508 03:00:11.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.39% / 0.00% / 58.60% | 20260508 03:00:11.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.662263% / 0% / 5.16574e-05% | 20260508 03:00:11.924 | INFO | Baz IO overall: | 20260508 03:00:11.924 | INFO | BytesWritten : 2122070555813 | 20260508 03:00:11.924 | INFO | NumEvents : 1025513514980 | 20260508 03:00:13.430 | INFO | Analyzing chunk frames = [4710400,4710912) | 20260508 03:00:13.430 | INFO | MemInfo: { | 20260508 03:00:13.430 | INFO | "gpu" : " 17 GB", | 20260508 03:00:13.430 | INFO | "huge" : "", | 20260508 03:00:13.430 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:13.430 | INFO | "malloc" : " 14 GB", | 20260508 03:00:13.430 | INFO | "pinned" : " 29 GB" | 20260508 03:00:13.430 | INFO | } | 20260508 03:00:13.430 | INFO | Perf report for the last 5.13794 seconds: | 20260508 03:00:13.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:13.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:13.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:13.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:13.430 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260508 03:00:13.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:13.430 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 03:00:13.430 | INFO | PRE_HQ : 66.1%, 1.6, 55.9ms, 0 | 20260508 03:00:13.430 | INFO | BAZWRITER : 68.4%, 4.2, 146.3ms, 0 | 20260508 03:00:13.430 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 03:00:13.431 | INFO | Uploaded 36GiB to GPU in 2.55333s (14.0992GiB/s) | 20260508 03:00:13.431 | INFO | Downloaded 38GiB to GPU in 2.69458s (14.1024GiB/s) | 20260508 03:00:18.568 | INFO | Analyzing chunk frames = [4710912,4711424) | 20260508 03:00:18.568 | INFO | MemInfo: { | 20260508 03:00:18.568 | INFO | "gpu" : " 17 GB", | 20260508 03:00:18.568 | INFO | "huge" : "", | 20260508 03:00:18.568 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:18.568 | INFO | "malloc" : " 14 GB", | 20260508 03:00:18.568 | INFO | "pinned" : " 29 GB" | 20260508 03:00:18.568 | INFO | } | 20260508 03:00:18.569 | INFO | Perf report for the last 5.13855 seconds: | 20260508 03:00:18.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:18.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 03:00:18.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:00:18.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:18.569 | INFO | Uploaded 36GiB to GPU in 2.52155s (14.2769GiB/s) | 20260508 03:00:18.569 | INFO | Downloaded 34GiB to GPU in 2.43618s (13.9563GiB/s) | 20260508 03:00:23.707 | INFO | Analyzing chunk frames = [4711424,4711936) | 20260508 03:00:23.707 | INFO | MemInfo: { | 20260508 03:00:23.707 | INFO | "gpu" : " 17 GB", | 20260508 03:00:23.707 | INFO | "huge" : "", | 20260508 03:00:23.707 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:23.707 | INFO | "malloc" : " 14 GB", | 20260508 03:00:23.707 | INFO | "pinned" : " 29 GB" | 20260508 03:00:23.707 | INFO | } | 20260508 03:00:23.707 | INFO | Perf report for the last 5.13879 seconds: | 20260508 03:00:23.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:23.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.707 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 03:00:23.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.707 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:00:23.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:23.708 | INFO | Uploaded 36GiB to GPU in 2.50969s (14.3444GiB/s) | 20260508 03:00:23.708 | INFO | Downloaded 34GiB to GPU in 2.50654s (13.5645GiB/s) | 20260508 03:00:28.846 | INFO | Analyzing chunk frames = [4711936,4712448) | 20260508 03:00:28.846 | INFO | MemInfo: { | 20260508 03:00:28.846 | INFO | "gpu" : " 17 GB", | 20260508 03:00:28.846 | INFO | "huge" : "", | 20260508 03:00:28.846 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:28.846 | INFO | "malloc" : " 14 GB", | 20260508 03:00:28.846 | INFO | "pinned" : " 29 GB" | 20260508 03:00:28.846 | INFO | } | 20260508 03:00:28.846 | INFO | Perf report for the last 5.13866 seconds: | 20260508 03:00:28.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:28.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 03:00:28.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:00:28.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:28.846 | INFO | Uploaded 36GiB to GPU in 2.46675s (14.5941GiB/s) | 20260508 03:00:28.846 | INFO | Downloaded 34GiB to GPU in 2.50378s (13.5795GiB/s) | 20260508 03:00:33.990 | INFO | Analyzing chunk frames = [4712448,4712960) | 20260508 03:00:33.990 | INFO | MemInfo: { | 20260508 03:00:33.990 | INFO | "gpu" : " 17 GB", | 20260508 03:00:33.991 | INFO | "huge" : "", | 20260508 03:00:33.991 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:33.991 | INFO | "malloc" : " 14 GB", | 20260508 03:00:33.991 | INFO | "pinned" : " 29 GB" | 20260508 03:00:33.991 | INFO | } | 20260508 03:00:33.991 | INFO | Perf report for the last 5.14471 seconds: | 20260508 03:00:33.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:33.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 03:00:33.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:00:33.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:33.991 | INFO | Uploaded 36GiB to GPU in 2.30092s (15.6459GiB/s) | 20260508 03:00:33.991 | INFO | Downloaded 34GiB to GPU in 2.50848s (13.554GiB/s) | 20260508 03:00:39.128 | INFO | Analyzing chunk frames = [4712960,4713472) | 20260508 03:00:39.128 | INFO | MemInfo: { | 20260508 03:00:39.128 | INFO | "gpu" : " 17 GB", | 20260508 03:00:39.128 | INFO | "huge" : "", | 20260508 03:00:39.128 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:39.128 | INFO | "malloc" : " 14 GB", | 20260508 03:00:39.128 | INFO | "pinned" : " 29 GB" | 20260508 03:00:39.128 | INFO | } | 20260508 03:00:39.128 | INFO | Perf report for the last 5.13742 seconds: | 20260508 03:00:39.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:39.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.128 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:00:39.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.128 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 03:00:39.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:39.129 | INFO | Uploaded 36GiB to GPU in 2.4057s (14.9645GiB/s) | 20260508 03:00:39.129 | INFO | Downloaded 34GiB to GPU in 2.42391s (14.0269GiB/s) | 20260508 03:00:44.268 | INFO | Analyzing chunk frames = [4713472,4713984) | 20260508 03:00:44.268 | INFO | MemInfo: { | 20260508 03:00:44.268 | INFO | "gpu" : " 17 GB", | 20260508 03:00:44.268 | INFO | "huge" : "", | 20260508 03:00:44.268 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:44.268 | INFO | "malloc" : " 14 GB", | 20260508 03:00:44.269 | INFO | "pinned" : " 29 GB" | 20260508 03:00:44.269 | INFO | } | 20260508 03:00:44.269 | INFO | Perf report for the last 5.14044 seconds: | 20260508 03:00:44.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:44.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:00:44.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 03:00:44.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:44.269 | INFO | Uploaded 36GiB to GPU in 2.42732s (14.8311GiB/s) | 20260508 03:00:44.269 | INFO | Downloaded 34GiB to GPU in 2.43258s (13.9769GiB/s) | 20260508 03:00:49.400 | INFO | Analyzing chunk frames = [4713984,4714496) | 20260508 03:00:49.400 | INFO | MemInfo: { | 20260508 03:00:49.400 | INFO | "gpu" : " 17 GB", | 20260508 03:00:49.400 | INFO | "huge" : "", | 20260508 03:00:49.400 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:49.400 | INFO | "malloc" : " 14 GB", | 20260508 03:00:49.400 | INFO | "pinned" : " 29 GB" | 20260508 03:00:49.400 | INFO | } | 20260508 03:00:49.400 | INFO | Perf report for the last 5.13188 seconds: | 20260508 03:00:49.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:49.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 03:00:49.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | PRE_HQ : 34.8%, 1.0, 18.0ms, 0 | 20260508 03:00:49.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:49.401 | INFO | Uploaded 36GiB to GPU in 2.45407s (14.6695GiB/s) | 20260508 03:00:49.401 | INFO | Downloaded 34GiB to GPU in 2.44142s (13.9263GiB/s) | 20260508 03:00:49.515 | INFO | Starting a round of Baz IO | 20260508 03:00:52.975 | INFO | Populating RTMetrics report | 20260508 03:00:52.976 | INFO | RTMetrics report populated | 20260508 03:00:52.976 | INFO | Opening tmp RTMetrics json file | 20260508 03:00:52.993 | INFO | Serializing RTMetrics to json | 20260508 03:00:52.998 | INFO | Serialization done, swapping tmp file now... | 20260508 03:00:53.002 | INFO | RTMetrics done | 20260508 03:00:53.065 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.549 seconds | 20260508 03:00:53.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.39% / 0.00% / 58.61% | 20260508 03:00:53.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.65918% / 3.97364e-06% / 4.76837e-05% | 20260508 03:00:53.065 | INFO | Baz IO overall: | 20260508 03:00:53.065 | INFO | BytesWritten : 2123814696766 | 20260508 03:00:53.065 | INFO | NumEvents : 1026348229881 | 20260508 03:00:54.543 | INFO | Analyzing chunk frames = [4714496,4715008) | 20260508 03:00:54.543 | INFO | MemInfo: { | 20260508 03:00:54.543 | INFO | "gpu" : " 17 GB", | 20260508 03:00:54.543 | INFO | "huge" : "", | 20260508 03:00:54.543 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:54.543 | INFO | "malloc" : " 14 GB", | 20260508 03:00:54.543 | INFO | "pinned" : " 29 GB" | 20260508 03:00:54.543 | INFO | } | 20260508 03:00:54.543 | INFO | Perf report for the last 5.1421 seconds: | 20260508 03:00:54.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:54.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:54.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:54.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:54.543 | INFO | ANALYSIS : 68.6%, 2.9, 104.4ms, 0 | 20260508 03:00:54.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:54.543 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260508 03:00:54.543 | INFO | PRE_HQ : 67.4%, 1.8, 62.7ms, 0 | 20260508 03:00:54.543 | INFO | BAZWRITER : 68.7%, 4.4, 153.9ms, 0 | 20260508 03:00:54.543 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 03:00:54.543 | INFO | Uploaded 36GiB to GPU in 2.62259s (13.7269GiB/s) | 20260508 03:00:54.543 | INFO | Downloaded 38GiB to GPU in 2.70238s (14.0617GiB/s) | 20260508 03:00:59.685 | INFO | Analyzing chunk frames = [4715008,4715520) | 20260508 03:00:59.686 | INFO | MemInfo: { | 20260508 03:00:59.686 | INFO | "gpu" : " 17 GB", | 20260508 03:00:59.686 | INFO | "huge" : "", | 20260508 03:00:59.686 | INFO | "huge_shared" : " 29 GB", | 20260508 03:00:59.686 | INFO | "malloc" : " 14 GB", | 20260508 03:00:59.686 | INFO | "pinned" : " 29 GB" | 20260508 03:00:59.686 | INFO | } | 20260508 03:00:59.686 | INFO | Perf report for the last 5.14317 seconds: | 20260508 03:00:59.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:00:59.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:00:59.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:00:59.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:00:59.686 | INFO | Uploaded 36GiB to GPU in 2.54295s (14.1568GiB/s) | 20260508 03:00:59.686 | INFO | Downloaded 34GiB to GPU in 2.42914s (13.9967GiB/s) | 20260508 03:01:04.829 | INFO | Analyzing chunk frames = [4715520,4716032) | 20260508 03:01:04.829 | INFO | MemInfo: { | 20260508 03:01:04.829 | INFO | "gpu" : " 17 GB", | 20260508 03:01:04.829 | INFO | "huge" : "", | 20260508 03:01:04.829 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:04.829 | INFO | "malloc" : " 14 GB", | 20260508 03:01:04.829 | INFO | "pinned" : " 29 GB" | 20260508 03:01:04.829 | INFO | } | 20260508 03:01:04.829 | INFO | Perf report for the last 5.14325 seconds: | 20260508 03:01:04.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:04.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.829 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 03:01:04.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.829 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:01:04.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:04.830 | INFO | Uploaded 36GiB to GPU in 2.55579s (14.0857GiB/s) | 20260508 03:01:04.830 | INFO | Downloaded 34GiB to GPU in 2.5056s (13.5696GiB/s) | 20260508 03:01:09.964 | INFO | Analyzing chunk frames = [4716032,4716544) | 20260508 03:01:09.964 | INFO | MemInfo: { | 20260508 03:01:09.964 | INFO | "gpu" : " 17 GB", | 20260508 03:01:09.964 | INFO | "huge" : "", | 20260508 03:01:09.964 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:09.964 | INFO | "malloc" : " 14 GB", | 20260508 03:01:09.964 | INFO | "pinned" : " 29 GB" | 20260508 03:01:09.964 | INFO | } | 20260508 03:01:09.964 | INFO | Perf report for the last 5.13529 seconds: | 20260508 03:01:09.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:09.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.964 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 03:01:09.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.964 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:01:09.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:09.965 | INFO | Uploaded 36GiB to GPU in 2.53656s (14.1925GiB/s) | 20260508 03:01:09.965 | INFO | Downloaded 34GiB to GPU in 2.50515s (13.572GiB/s) | 20260508 03:01:15.098 | INFO | Analyzing chunk frames = [4716544,4717056) | 20260508 03:01:15.098 | INFO | MemInfo: { | 20260508 03:01:15.098 | INFO | "gpu" : " 17 GB", | 20260508 03:01:15.098 | INFO | "huge" : "", | 20260508 03:01:15.098 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:15.098 | INFO | "malloc" : " 14 GB", | 20260508 03:01:15.098 | INFO | "pinned" : " 29 GB" | 20260508 03:01:15.098 | INFO | } | 20260508 03:01:15.098 | INFO | Perf report for the last 5.13408 seconds: | 20260508 03:01:15.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:15.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.099 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 03:01:15.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.099 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:01:15.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:15.099 | INFO | Uploaded 36GiB to GPU in 2.41104s (14.9313GiB/s) | 20260508 03:01:15.099 | INFO | Downloaded 34GiB to GPU in 2.57725s (13.1924GiB/s) | 20260508 03:01:20.242 | INFO | Analyzing chunk frames = [4717056,4717568) | 20260508 03:01:20.242 | INFO | MemInfo: { | 20260508 03:01:20.242 | INFO | "gpu" : " 17 GB", | 20260508 03:01:20.242 | INFO | "huge" : "", | 20260508 03:01:20.242 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:20.242 | INFO | "malloc" : " 14 GB", | 20260508 03:01:20.242 | INFO | "pinned" : " 29 GB" | 20260508 03:01:20.242 | INFO | } | 20260508 03:01:20.243 | INFO | Perf report for the last 5.1441 seconds: | 20260508 03:01:20.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:20.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:01:20.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:01:20.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:20.243 | INFO | Uploaded 36GiB to GPU in 2.4202s (14.8748GiB/s) | 20260508 03:01:20.243 | INFO | Downloaded 34GiB to GPU in 2.42851s (14.0003GiB/s) | 20260508 03:01:25.383 | INFO | Analyzing chunk frames = [4717568,4718080) | 20260508 03:01:25.383 | INFO | MemInfo: { | 20260508 03:01:25.383 | INFO | "gpu" : " 17 GB", | 20260508 03:01:25.383 | INFO | "huge" : "", | 20260508 03:01:25.383 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:25.383 | INFO | "malloc" : " 14 GB", | 20260508 03:01:25.383 | INFO | "pinned" : " 29 GB" | 20260508 03:01:25.383 | INFO | } | 20260508 03:01:25.383 | INFO | Perf report for the last 5.14087 seconds: | 20260508 03:01:25.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:25.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:01:25.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:01:25.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:25.384 | INFO | Uploaded 36GiB to GPU in 2.43071s (14.8105GiB/s) | 20260508 03:01:25.384 | INFO | Downloaded 34GiB to GPU in 2.42423s (14.025GiB/s) | 20260508 03:01:30.520 | INFO | Analyzing chunk frames = [4718080,4718592) | 20260508 03:01:30.520 | INFO | MemInfo: { | 20260508 03:01:30.520 | INFO | "gpu" : " 17 GB", | 20260508 03:01:30.520 | INFO | "huge" : "", | 20260508 03:01:30.520 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:30.520 | INFO | "malloc" : " 14 GB", | 20260508 03:01:30.520 | INFO | "pinned" : " 29 GB" | 20260508 03:01:30.520 | INFO | } | 20260508 03:01:30.520 | INFO | Perf report for the last 5.13679 seconds: | 20260508 03:01:30.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:30.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.520 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 03:01:30.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.520 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 03:01:30.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:30.521 | INFO | Uploaded 36GiB to GPU in 2.56048s (14.0598GiB/s) | 20260508 03:01:30.521 | INFO | Downloaded 34GiB to GPU in 2.46734s (13.78GiB/s) | 20260508 03:01:30.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:01:30.521 | INFO | Total mem usage: 16.7 GiB | 20260508 03:01:30.521 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:01:30.521 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.2%) | 20260508 03:01:30.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:01:30.521 | INFO | High Water Breakdown: | 20260508 03:01:30.521 | INFO | 60.4% : 9.14 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:01:30.521 | INFO | 39.5% : 5.97 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:01:30.521 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:01:30.521 | INFO | ------------------------------------------ | 20260508 03:01:30.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:01:30.521 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:01:30.521 | INFO | Total mem usage: 26.5 GiB | 20260508 03:01:30.521 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:01:30.521 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260508 03:01:30.521 | INFO | Unused Cache: 1.81 GiB (Waste: 6.81%) | 20260508 03:01:30.521 | INFO | High Water Breakdown: | 20260508 03:01:30.521 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:01:30.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:01:30.521 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:01:30.521 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:01:30.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:01:30.521 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:01:30.521 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:01:30.521 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:01:30.521 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:01:30.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:01:30.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:01:30.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:01:30.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:01:30.521 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:01:30.521 | INFO | ------------------------------------------ | 20260508 03:01:30.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:01:30.521 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:01:30.521 | INFO | Total mem usage: 27.8 GiB | 20260508 03:01:30.521 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:01:30.521 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:01:30.521 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:01:30.521 | INFO | High Water Breakdown: | 20260508 03:01:30.521 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:01:30.521 | INFO | ------------------------------------------ | 20260508 03:01:30.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:01:30.521 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:01:30.522 | INFO | Total mem usage: 15.7 GiB | 20260508 03:01:30.522 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:01:30.522 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:01:30.522 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:01:30.522 | INFO | High Water Breakdown: | 20260508 03:01:30.522 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:01:30.522 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:01:30.522 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:01:30.522 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:01:30.522 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:01:30.522 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:01:30.522 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:01:30.522 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:01:30.522 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:01:30.522 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:01:30.522 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:01:30.522 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:01:30.522 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:01:30.522 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:01:30.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:01:30.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:01:30.522 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:01:30.522 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:01:30.522 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:01:30.522 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:01:30.522 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:01:30.522 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:01:30.522 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:01:30.522 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:01:30.522 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:01:30.522 | INFO | ------------------------------------------ | 20260508 03:01:30.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:01:30.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:01:30.522 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:01:30.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:01:30.522 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.63e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.13e-02 / 4.67e-03 | 20260508 03:01:30.522 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.46e+00, 5.85e-01 / 2.47e-01 | 20260508 03:01:30.522 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.07e-01 / 1.09e-01 | 20260508 03:01:30.522 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.61e+01, 4.08e+00 / 1.91e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.64e+00 / 1.69e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.41e+00 / 2.66e+00, 4.23e-01 / 2.56e-01 | 20260508 03:01:30.522 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.23e+00, 2.66e-01 / 1.60e-01 | 20260508 03:01:30.522 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.56e-01, 5.26e-02 / 2.64e-02 | 20260508 03:01:30.522 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:01:30.522 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:01:30.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:01:30.522 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 24.52%, Download -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 2.07%, LockUpload -- 2.20e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 2.89%, LockDownload -- 3.07e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 15.40%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:01:30.522 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 03:01:30.618 | INFO | Starting a round of Baz IO | 20260508 03:01:34.057 | INFO | Populating RTMetrics report | 20260508 03:01:34.057 | INFO | RTMetrics report populated | 20260508 03:01:34.057 | INFO | Opening tmp RTMetrics json file | 20260508 03:01:34.079 | INFO | Serializing RTMetrics to json | 20260508 03:01:34.081 | INFO | Serialization done, swapping tmp file now... | 20260508 03:01:34.082 | INFO | RTMetrics done | 20260508 03:01:34.151 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.533 seconds | 20260508 03:01:34.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.39% / 0.00% / 58.61% | 20260508 03:01:34.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.65769% / 3.97364e-06% / 3.57628e-05% | 20260508 03:01:34.151 | INFO | Baz IO overall: | 20260508 03:01:34.151 | INFO | BytesWritten : 2125558737226 | 20260508 03:01:34.151 | INFO | NumEvents : 1027183191000 | 20260508 03:01:35.656 | INFO | Analyzing chunk frames = [4718592,4719104) | 20260508 03:01:35.656 | INFO | MemInfo: { | 20260508 03:01:35.656 | INFO | "gpu" : " 17 GB", | 20260508 03:01:35.656 | INFO | "huge" : "", | 20260508 03:01:35.656 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:35.656 | INFO | "malloc" : " 14 GB", | 20260508 03:01:35.656 | INFO | "pinned" : " 29 GB" | 20260508 03:01:35.656 | INFO | } | 20260508 03:01:35.656 | INFO | Perf report for the last 5.13614 seconds: | 20260508 03:01:35.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:35.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:35.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:35.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:35.657 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 03:01:35.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:35.657 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 03:01:35.657 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 03:01:35.657 | INFO | BAZWRITER : 68.4%, 4.2, 146.2ms, 0 | 20260508 03:01:35.657 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:01:35.657 | INFO | Uploaded 36GiB to GPU in 2.58468s (13.9282GiB/s) | 20260508 03:01:35.657 | INFO | Downloaded 38GiB to GPU in 2.69095s (14.1214GiB/s) | 20260508 03:01:40.796 | INFO | Analyzing chunk frames = [4719104,4719616) | 20260508 03:01:40.796 | INFO | MemInfo: { | 20260508 03:01:40.796 | INFO | "gpu" : " 17 GB", | 20260508 03:01:40.796 | INFO | "huge" : "", | 20260508 03:01:40.796 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:40.796 | INFO | "malloc" : " 14 GB", | 20260508 03:01:40.796 | INFO | "pinned" : " 29 GB" | 20260508 03:01:40.796 | INFO | } | 20260508 03:01:40.796 | INFO | Perf report for the last 5.13994 seconds: | 20260508 03:01:40.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:40.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.796 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 03:01:40.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.796 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:01:40.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:40.797 | INFO | Uploaded 36GiB to GPU in 2.57991s (13.954GiB/s) | 20260508 03:01:40.797 | INFO | Downloaded 34GiB to GPU in 2.42223s (14.0366GiB/s) | 20260508 03:01:45.940 | INFO | Analyzing chunk frames = [4719616,4720128) | 20260508 03:01:45.940 | INFO | MemInfo: { | 20260508 03:01:45.940 | INFO | "gpu" : " 17 GB", | 20260508 03:01:45.940 | INFO | "huge" : "", | 20260508 03:01:45.940 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:45.940 | INFO | "malloc" : " 14 GB", | 20260508 03:01:45.940 | INFO | "pinned" : " 29 GB" | 20260508 03:01:45.940 | INFO | } | 20260508 03:01:45.940 | INFO | Perf report for the last 5.14354 seconds: | 20260508 03:01:45.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:45.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:01:45.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:01:45.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:45.940 | INFO | Uploaded 36GiB to GPU in 2.55758s (14.0758GiB/s) | 20260508 03:01:45.940 | INFO | Downloaded 34GiB to GPU in 2.47986s (13.7105GiB/s) | 20260508 03:01:51.079 | INFO | Analyzing chunk frames = [4720128,4720640) | 20260508 03:01:51.079 | INFO | MemInfo: { | 20260508 03:01:51.079 | INFO | "gpu" : " 17 GB", | 20260508 03:01:51.079 | INFO | "huge" : "", | 20260508 03:01:51.079 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:51.079 | INFO | "malloc" : " 14 GB", | 20260508 03:01:51.079 | INFO | "pinned" : " 29 GB" | 20260508 03:01:51.079 | INFO | } | 20260508 03:01:51.079 | INFO | Perf report for the last 5.13948 seconds: | 20260508 03:01:51.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:51.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:01:51.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:01:51.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:51.080 | INFO | Uploaded 36GiB to GPU in 2.56961s (14.0099GiB/s) | 20260508 03:01:51.080 | INFO | Downloaded 34GiB to GPU in 2.48392s (13.688GiB/s) | 20260508 03:01:56.217 | INFO | Analyzing chunk frames = [4720640,4721152) | 20260508 03:01:56.217 | INFO | MemInfo: { | 20260508 03:01:56.217 | INFO | "gpu" : " 17 GB", | 20260508 03:01:56.217 | INFO | "huge" : "", | 20260508 03:01:56.217 | INFO | "huge_shared" : " 29 GB", | 20260508 03:01:56.217 | INFO | "malloc" : " 14 GB", | 20260508 03:01:56.217 | INFO | "pinned" : " 29 GB" | 20260508 03:01:56.217 | INFO | } | 20260508 03:01:56.217 | INFO | Perf report for the last 5.13788 seconds: | 20260508 03:01:56.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:01:56.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.217 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 03:01:56.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.217 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:01:56.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:01:56.218 | INFO | Uploaded 36GiB to GPU in 2.40246s (14.9846GiB/s) | 20260508 03:01:56.218 | INFO | Downloaded 34GiB to GPU in 2.53592s (13.4074GiB/s) | 20260508 03:02:01.354 | INFO | Analyzing chunk frames = [4721152,4721664) | 20260508 03:02:01.354 | INFO | MemInfo: { | 20260508 03:02:01.354 | INFO | "gpu" : " 17 GB", | 20260508 03:02:01.354 | INFO | "huge" : "", | 20260508 03:02:01.354 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:01.354 | INFO | "malloc" : " 14 GB", | 20260508 03:02:01.354 | INFO | "pinned" : " 29 GB" | 20260508 03:02:01.354 | INFO | } | 20260508 03:02:01.354 | INFO | Perf report for the last 5.1369 seconds: | 20260508 03:02:01.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:01.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.354 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:02:01.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.354 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 03:02:01.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:01.355 | INFO | Uploaded 36GiB to GPU in 2.43535s (14.7823GiB/s) | 20260508 03:02:01.355 | INFO | Downloaded 34GiB to GPU in 2.4112s (14.1009GiB/s) | 20260508 03:02:06.496 | INFO | Analyzing chunk frames = [4721664,4722176) | 20260508 03:02:06.496 | INFO | MemInfo: { | 20260508 03:02:06.496 | INFO | "gpu" : " 17 GB", | 20260508 03:02:06.496 | INFO | "huge" : "", | 20260508 03:02:06.496 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:06.496 | INFO | "malloc" : " 14 GB", | 20260508 03:02:06.496 | INFO | "pinned" : " 29 GB" | 20260508 03:02:06.496 | INFO | } | 20260508 03:02:06.497 | INFO | Perf report for the last 5.1423 seconds: | 20260508 03:02:06.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:06.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:02:06.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:02:06.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:06.497 | INFO | Uploaded 36GiB to GPU in 2.50084s (14.3952GiB/s) | 20260508 03:02:06.497 | INFO | Downloaded 34GiB to GPU in 2.4295s (13.9947GiB/s) | 20260508 03:02:11.637 | INFO | Analyzing chunk frames = [4722176,4722688) | 20260508 03:02:11.637 | INFO | MemInfo: { | 20260508 03:02:11.637 | INFO | "gpu" : " 17 GB", | 20260508 03:02:11.637 | INFO | "huge" : "", | 20260508 03:02:11.637 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:11.637 | INFO | "malloc" : " 14 GB", | 20260508 03:02:11.637 | INFO | "pinned" : " 29 GB" | 20260508 03:02:11.637 | INFO | } | 20260508 03:02:11.637 | INFO | Perf report for the last 5.14047 seconds: | 20260508 03:02:11.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:11.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:02:11.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:02:11.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:11.637 | INFO | Uploaded 36GiB to GPU in 2.47849s (14.525GiB/s) | 20260508 03:02:11.637 | INFO | Downloaded 34GiB to GPU in 2.42856s (14.0001GiB/s) | 20260508 03:02:11.734 | INFO | Starting a round of Baz IO | 20260508 03:02:15.168 | INFO | Populating RTMetrics report | 20260508 03:02:15.169 | INFO | RTMetrics report populated | 20260508 03:02:15.169 | INFO | Opening tmp RTMetrics json file | 20260508 03:02:15.182 | INFO | Serializing RTMetrics to json | 20260508 03:02:15.194 | INFO | Serialization done, swapping tmp file now... | 20260508 03:02:15.195 | INFO | RTMetrics done | 20260508 03:02:15.278 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.544 seconds | 20260508 03:02:15.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.38% / 0.00% / 58.61% | 20260508 03:02:15.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.657849% / 3.97364e-06% / 6.75519e-05% | 20260508 03:02:15.278 | INFO | Baz IO overall: | 20260508 03:02:15.278 | INFO | BytesWritten : 2127303430430 | 20260508 03:02:15.278 | INFO | NumEvents : 1028018630838 | 20260508 03:02:16.780 | INFO | Analyzing chunk frames = [4722688,4723200) | 20260508 03:02:16.780 | INFO | MemInfo: { | 20260508 03:02:16.780 | INFO | "gpu" : " 17 GB", | 20260508 03:02:16.780 | INFO | "huge" : "", | 20260508 03:02:16.780 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:16.780 | INFO | "malloc" : " 14 GB", | 20260508 03:02:16.780 | INFO | "pinned" : " 29 GB" | 20260508 03:02:16.780 | INFO | } | 20260508 03:02:16.780 | INFO | Perf report for the last 5.14281 seconds: | 20260508 03:02:16.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:16.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:16.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:16.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:16.780 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 03:02:16.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:16.780 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 03:02:16.780 | INFO | PRE_HQ : 65.7%, 1.6, 55.8ms, 0 | 20260508 03:02:16.780 | INFO | BAZWRITER : 68.5%, 3.9, 138.5ms, 0 | 20260508 03:02:16.780 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 03:02:16.780 | INFO | Uploaded 36GiB to GPU in 2.57488s (13.9813GiB/s) | 20260508 03:02:16.780 | INFO | Downloaded 38GiB to GPU in 2.69479s (14.1013GiB/s) | 20260508 03:02:21.912 | INFO | Analyzing chunk frames = [4723200,4723712) | 20260508 03:02:21.912 | INFO | MemInfo: { | 20260508 03:02:21.912 | INFO | "gpu" : " 17 GB", | 20260508 03:02:21.912 | INFO | "huge" : "", | 20260508 03:02:21.912 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:21.912 | INFO | "malloc" : " 14 GB", | 20260508 03:02:21.912 | INFO | "pinned" : " 29 GB" | 20260508 03:02:21.912 | INFO | } | 20260508 03:02:21.912 | INFO | Perf report for the last 5.1323 seconds: | 20260508 03:02:21.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:21.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.912 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260508 03:02:21.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.912 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:02:21.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:21.913 | INFO | Uploaded 36GiB to GPU in 2.56599s (14.0297GiB/s) | 20260508 03:02:21.913 | INFO | Downloaded 34GiB to GPU in 2.43222s (13.979GiB/s) | 20260508 03:02:27.053 | INFO | Analyzing chunk frames = [4723712,4724224) | 20260508 03:02:27.053 | INFO | MemInfo: { | 20260508 03:02:27.053 | INFO | "gpu" : " 17 GB", | 20260508 03:02:27.053 | INFO | "huge" : "", | 20260508 03:02:27.053 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:27.053 | INFO | "malloc" : " 14 GB", | 20260508 03:02:27.053 | INFO | "pinned" : " 29 GB" | 20260508 03:02:27.053 | INFO | } | 20260508 03:02:27.053 | INFO | Perf report for the last 5.14125 seconds: | 20260508 03:02:27.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:27.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 03:02:27.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:02:27.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:27.054 | INFO | Uploaded 36GiB to GPU in 2.54117s (14.1667GiB/s) | 20260508 03:02:27.054 | INFO | Downloaded 34GiB to GPU in 2.50279s (13.5849GiB/s) | 20260508 03:02:32.192 | INFO | Analyzing chunk frames = [4724224,4724736) | 20260508 03:02:32.192 | INFO | MemInfo: { | 20260508 03:02:32.192 | INFO | "gpu" : " 17 GB", | 20260508 03:02:32.192 | INFO | "huge" : "", | 20260508 03:02:32.192 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:32.192 | INFO | "malloc" : " 14 GB", | 20260508 03:02:32.192 | INFO | "pinned" : " 29 GB" | 20260508 03:02:32.192 | INFO | } | 20260508 03:02:32.192 | INFO | Perf report for the last 5.13879 seconds: | 20260508 03:02:32.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:32.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.192 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 03:02:32.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.192 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:02:32.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:32.193 | INFO | Uploaded 36GiB to GPU in 2.53818s (14.1834GiB/s) | 20260508 03:02:32.193 | INFO | Downloaded 34GiB to GPU in 2.50651s (13.5647GiB/s) | 20260508 03:02:37.331 | INFO | Analyzing chunk frames = [4724736,4725248) | 20260508 03:02:37.331 | INFO | MemInfo: { | 20260508 03:02:37.331 | INFO | "gpu" : " 17 GB", | 20260508 03:02:37.331 | INFO | "huge" : "", | 20260508 03:02:37.331 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:37.331 | INFO | "malloc" : " 14 GB", | 20260508 03:02:37.331 | INFO | "pinned" : " 29 GB" | 20260508 03:02:37.331 | INFO | } | 20260508 03:02:37.332 | INFO | Perf report for the last 5.13929 seconds: | 20260508 03:02:37.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:37.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 03:02:37.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 03:02:37.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:37.332 | INFO | Uploaded 36GiB to GPU in 2.39276s (15.0454GiB/s) | 20260508 03:02:37.332 | INFO | Downloaded 34GiB to GPU in 2.57289s (13.2147GiB/s) | 20260508 03:02:42.468 | INFO | Analyzing chunk frames = [4725248,4725760) | 20260508 03:02:42.468 | INFO | MemInfo: { | 20260508 03:02:42.468 | INFO | "gpu" : " 17 GB", | 20260508 03:02:42.468 | INFO | "huge" : "", | 20260508 03:02:42.468 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:42.468 | INFO | "malloc" : " 14 GB", | 20260508 03:02:42.468 | INFO | "pinned" : " 29 GB" | 20260508 03:02:42.468 | INFO | } | 20260508 03:02:42.468 | INFO | Perf report for the last 5.13665 seconds: | 20260508 03:02:42.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:42.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.468 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 03:02:42.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.468 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:02:42.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:42.469 | INFO | Uploaded 36GiB to GPU in 2.39736s (15.0165GiB/s) | 20260508 03:02:42.469 | INFO | Downloaded 34GiB to GPU in 2.45017s (13.8766GiB/s) | 20260508 03:02:47.609 | INFO | Analyzing chunk frames = [4725760,4726272) | 20260508 03:02:47.609 | INFO | MemInfo: { | 20260508 03:02:47.609 | INFO | "gpu" : " 17 GB", | 20260508 03:02:47.609 | INFO | "huge" : "", | 20260508 03:02:47.609 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:47.609 | INFO | "malloc" : " 14 GB", | 20260508 03:02:47.609 | INFO | "pinned" : " 29 GB" | 20260508 03:02:47.609 | INFO | } | 20260508 03:02:47.609 | INFO | Perf report for the last 5.14105 seconds: | 20260508 03:02:47.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:47.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.609 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 03:02:47.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.609 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:02:47.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:47.610 | INFO | Uploaded 36GiB to GPU in 2.40493s (14.9692GiB/s) | 20260508 03:02:47.610 | INFO | Downloaded 34GiB to GPU in 2.43761s (13.9481GiB/s) | 20260508 03:02:52.749 | INFO | Analyzing chunk frames = [4726272,4726784) | 20260508 03:02:52.749 | INFO | MemInfo: { | 20260508 03:02:52.749 | INFO | "gpu" : " 17 GB", | 20260508 03:02:52.749 | INFO | "huge" : "", | 20260508 03:02:52.749 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:52.749 | INFO | "malloc" : " 14 GB", | 20260508 03:02:52.749 | INFO | "pinned" : " 29 GB" | 20260508 03:02:52.750 | INFO | } | 20260508 03:02:52.750 | INFO | Perf report for the last 5.1403 seconds: | 20260508 03:02:52.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:52.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 03:02:52.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:02:52.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:52.750 | INFO | Uploaded 36GiB to GPU in 2.52514s (14.2566GiB/s) | 20260508 03:02:52.750 | INFO | Downloaded 34GiB to GPU in 2.47653s (13.7289GiB/s) | 20260508 03:02:52.750 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:02:52.750 | INFO | Total mem usage: 16.7 GiB | 20260508 03:02:52.750 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:02:52.750 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.8%) | 20260508 03:02:52.750 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:02:52.750 | INFO | High Water Breakdown: | 20260508 03:02:52.750 | INFO | 60% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:02:52.750 | INFO | 39.8% : 5.89 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:02:52.750 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:02:52.750 | INFO | ------------------------------------------ | 20260508 03:02:52.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:02:52.750 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:02:52.750 | INFO | Total mem usage: 26.5 GiB | 20260508 03:02:52.750 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:02:52.750 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.14%) | 20260508 03:02:52.750 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260508 03:02:52.750 | INFO | High Water Breakdown: | 20260508 03:02:52.750 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:02:52.750 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:02:52.750 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:02:52.750 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:02:52.750 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:02:52.750 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:02:52.750 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:02:52.750 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:02:52.750 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:02:52.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:02:52.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:02:52.750 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:02:52.750 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:02:52.750 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:02:52.750 | INFO | ------------------------------------------ | 20260508 03:02:52.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:02:52.750 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:02:52.750 | INFO | Total mem usage: 27.8 GiB | 20260508 03:02:52.750 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:02:52.750 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:02:52.750 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:02:52.750 | INFO | High Water Breakdown: | 20260508 03:02:52.750 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:02:52.751 | INFO | ------------------------------------------ | 20260508 03:02:52.751 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:02:52.751 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:02:52.751 | INFO | Total mem usage: 15.7 GiB | 20260508 03:02:52.751 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:02:52.751 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:02:52.751 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:02:52.751 | INFO | High Water Breakdown: | 20260508 03:02:52.751 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:02:52.751 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:02:52.751 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:02:52.751 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:02:52.751 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:02:52.751 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:02:52.751 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:02:52.751 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:02:52.751 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:02:52.751 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:02:52.751 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:02:52.751 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:02:52.751 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:02:52.751 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:02:52.751 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:02:52.751 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:02:52.751 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:02:52.751 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:02:52.751 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:02:52.751 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:02:52.751 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:02:52.751 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:02:52.751 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:02:52.751 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:02:52.751 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:02:52.751 | INFO | ------------------------------------------ | 20260508 03:02:52.751 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:02:52.751 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:02:52.751 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:02:52.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:02:52.751 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.41e+01 / 1.57e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.23e-02 / 5.40e-03 | 20260508 03:02:52.751 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.48e+00, 5.77e-01 / 1.78e-01 | 20260508 03:02:52.751 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.07e-01 / 1.03e-01 | 20260508 03:02:52.751 | INFO | PerfReport: 16.10%, DME -- 4.20e+01 / 4.61e+01, 3.79e+00 / 1.94e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.65e+00 / 1.37e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.71e+00, 4.20e-01 / 1.85e-01 | 20260508 03:02:52.751 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.25e+00, 2.65e-01 / 1.29e-01 | 20260508 03:02:52.751 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.64e-01, 5.14e-02 / 2.52e-02 | 20260508 03:02:52.751 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:02:52.751 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:02:52.751 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:02:52.751 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 24.47%, Upload -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 30.62%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 2.29%, LockUpload -- 2.44e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 2.73%, LockDownload -- 2.91e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 15.47%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:02:52.751 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260508 03:02:52.855 | INFO | Starting a round of Baz IO | 20260508 03:02:56.283 | INFO | Populating RTMetrics report | 20260508 03:02:56.283 | INFO | RTMetrics report populated | 20260508 03:02:56.283 | INFO | Opening tmp RTMetrics json file | 20260508 03:02:56.398 | INFO | Serializing RTMetrics to json | 20260508 03:02:56.414 | INFO | Serialization done, swapping tmp file now... | 20260508 03:02:56.416 | INFO | RTMetrics done | 20260508 03:02:56.447 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.592 seconds | 20260508 03:02:56.447 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.38% / 0.00% / 58.62% | 20260508 03:02:56.447 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.658349% / 7.94729e-06% / 7.94729e-05% | 20260508 03:02:56.447 | INFO | Baz IO overall: | 20260508 03:02:56.447 | INFO | BytesWritten : 2129047058630 | 20260508 03:02:56.447 | INFO | NumEvents : 1028853591198 | 20260508 03:02:57.893 | INFO | Analyzing chunk frames = [4726784,4727296) | 20260508 03:02:57.893 | INFO | MemInfo: { | 20260508 03:02:57.893 | INFO | "gpu" : " 17 GB", | 20260508 03:02:57.893 | INFO | "huge" : "", | 20260508 03:02:57.893 | INFO | "huge_shared" : " 29 GB", | 20260508 03:02:57.893 | INFO | "malloc" : " 14 GB", | 20260508 03:02:57.893 | INFO | "pinned" : " 29 GB" | 20260508 03:02:57.893 | INFO | } | 20260508 03:02:57.893 | INFO | Perf report for the last 5.14369 seconds: | 20260508 03:02:57.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:02:57.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:57.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:57.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:57.894 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 03:02:57.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:02:57.894 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260508 03:02:57.894 | INFO | PRE_HQ : 65.7%, 1.6, 55.8ms, 0 | 20260508 03:02:57.894 | INFO | BAZWRITER : 69.8%, 3.5, 126.3ms, 0 | 20260508 03:02:57.894 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260508 03:02:57.894 | INFO | Uploaded 36GiB to GPU in 2.55648s (14.0819GiB/s) | 20260508 03:02:57.894 | INFO | Downloaded 38GiB to GPU in 2.68151s (14.1711GiB/s) | 20260508 03:03:03.031 | INFO | Analyzing chunk frames = [4727296,4727808) | 20260508 03:03:03.031 | INFO | MemInfo: { | 20260508 03:03:03.031 | INFO | "gpu" : " 17 GB", | 20260508 03:03:03.031 | INFO | "huge" : "", | 20260508 03:03:03.031 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:03.031 | INFO | "malloc" : " 14 GB", | 20260508 03:03:03.031 | INFO | "pinned" : " 29 GB" | 20260508 03:03:03.031 | INFO | } | 20260508 03:03:03.031 | INFO | Perf report for the last 5.13769 seconds: | 20260508 03:03:03.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:03.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 03:03:03.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:03:03.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:03.032 | INFO | Uploaded 36GiB to GPU in 2.58735s (13.9139GiB/s) | 20260508 03:03:03.032 | INFO | Downloaded 34GiB to GPU in 2.50431s (13.5766GiB/s) | 20260508 03:03:08.164 | INFO | Analyzing chunk frames = [4727808,4728320) | 20260508 03:03:08.164 | INFO | MemInfo: { | 20260508 03:03:08.164 | INFO | "gpu" : " 17 GB", | 20260508 03:03:08.164 | INFO | "huge" : "", | 20260508 03:03:08.164 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:08.164 | INFO | "malloc" : " 14 GB", | 20260508 03:03:08.164 | INFO | "pinned" : " 29 GB" | 20260508 03:03:08.164 | INFO | } | 20260508 03:03:08.164 | INFO | Perf report for the last 5.13281 seconds: | 20260508 03:03:08.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:08.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.164 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 03:03:08.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.164 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 03:03:08.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:08.165 | INFO | Uploaded 36GiB to GPU in 2.55735s (14.077GiB/s) | 20260508 03:03:08.165 | INFO | Downloaded 34GiB to GPU in 2.49982s (13.601GiB/s) | 20260508 03:03:13.307 | INFO | Analyzing chunk frames = [4728320,4728832) | 20260508 03:03:13.307 | INFO | MemInfo: { | 20260508 03:03:13.307 | INFO | "gpu" : " 17 GB", | 20260508 03:03:13.307 | INFO | "huge" : "", | 20260508 03:03:13.307 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:13.307 | INFO | "malloc" : " 14 GB", | 20260508 03:03:13.307 | INFO | "pinned" : " 29 GB" | 20260508 03:03:13.307 | INFO | } | 20260508 03:03:13.307 | INFO | Perf report for the last 5.14331 seconds: | 20260508 03:03:13.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:13.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 03:03:13.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:03:13.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:13.307 | INFO | Uploaded 36GiB to GPU in 2.53531s (14.1994GiB/s) | 20260508 03:03:13.308 | INFO | Downloaded 34GiB to GPU in 2.49525s (13.6259GiB/s) | 20260508 03:03:18.449 | INFO | Analyzing chunk frames = [4728832,4729344) | 20260508 03:03:18.449 | INFO | MemInfo: { | 20260508 03:03:18.449 | INFO | "gpu" : " 17 GB", | 20260508 03:03:18.449 | INFO | "huge" : "", | 20260508 03:03:18.449 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:18.449 | INFO | "malloc" : " 14 GB", | 20260508 03:03:18.449 | INFO | "pinned" : " 29 GB" | 20260508 03:03:18.449 | INFO | } | 20260508 03:03:18.449 | INFO | Perf report for the last 5.1422 seconds: | 20260508 03:03:18.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:18.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.449 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:03:18.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.449 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:03:18.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:18.450 | INFO | Uploaded 36GiB to GPU in 2.45093s (14.6883GiB/s) | 20260508 03:03:18.450 | INFO | Downloaded 34GiB to GPU in 2.57285s (13.2149GiB/s) | 20260508 03:03:23.591 | INFO | Analyzing chunk frames = [4729344,4729856) | 20260508 03:03:23.591 | INFO | MemInfo: { | 20260508 03:03:23.591 | INFO | "gpu" : " 17 GB", | 20260508 03:03:23.591 | INFO | "huge" : "", | 20260508 03:03:23.591 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:23.591 | INFO | "malloc" : " 14 GB", | 20260508 03:03:23.591 | INFO | "pinned" : " 29 GB" | 20260508 03:03:23.591 | INFO | } | 20260508 03:03:23.591 | INFO | Perf report for the last 5.14143 seconds: | 20260508 03:03:23.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:23.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:03:23.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:03:23.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:23.591 | INFO | Uploaded 36GiB to GPU in 2.44272s (14.7376GiB/s) | 20260508 03:03:23.591 | INFO | Downloaded 34GiB to GPU in 2.40269s (14.1508GiB/s) | 20260508 03:03:28.731 | INFO | Analyzing chunk frames = [4729856,4730368) | 20260508 03:03:28.731 | INFO | MemInfo: { | 20260508 03:03:28.731 | INFO | "gpu" : " 17 GB", | 20260508 03:03:28.731 | INFO | "huge" : "", | 20260508 03:03:28.731 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:28.731 | INFO | "malloc" : " 14 GB", | 20260508 03:03:28.731 | INFO | "pinned" : " 29 GB" | 20260508 03:03:28.731 | INFO | } | 20260508 03:03:28.731 | INFO | Perf report for the last 5.14005 seconds: | 20260508 03:03:28.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:28.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:03:28.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:03:28.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:28.731 | INFO | Uploaded 36GiB to GPU in 2.44867s (14.7019GiB/s) | 20260508 03:03:28.731 | INFO | Downloaded 34GiB to GPU in 2.41553s (14.0756GiB/s) | 20260508 03:03:33.861 | INFO | Analyzing chunk frames = [4730368,4730880) | 20260508 03:03:33.861 | INFO | MemInfo: { | 20260508 03:03:33.861 | INFO | "gpu" : " 17 GB", | 20260508 03:03:33.861 | INFO | "huge" : "", | 20260508 03:03:33.861 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:33.861 | INFO | "malloc" : " 14 GB", | 20260508 03:03:33.861 | INFO | "pinned" : " 29 GB" | 20260508 03:03:33.861 | INFO | } | 20260508 03:03:33.861 | INFO | Perf report for the last 5.13036 seconds: | 20260508 03:03:33.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:33.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.861 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 03:03:33.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.861 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 03:03:33.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:33.862 | INFO | Uploaded 36GiB to GPU in 2.48855s (14.4663GiB/s) | 20260508 03:03:33.862 | INFO | Downloaded 34GiB to GPU in 2.42742s (14.0066GiB/s) | 20260508 03:03:33.958 | INFO | Starting a round of Baz IO | 20260508 03:03:37.396 | INFO | Populating RTMetrics report | 20260508 03:03:37.396 | INFO | RTMetrics report populated | 20260508 03:03:37.396 | INFO | Opening tmp RTMetrics json file | 20260508 03:03:37.413 | INFO | Serializing RTMetrics to json | 20260508 03:03:37.414 | INFO | Serialization done, swapping tmp file now... | 20260508 03:03:37.415 | INFO | RTMetrics done | 20260508 03:03:37.483 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.525 seconds | 20260508 03:03:37.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.38% / 0.00% / 58.62% | 20260508 03:03:37.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.65525% / 3.97364e-06% / 3.97364e-05% | 20260508 03:03:37.483 | INFO | Baz IO overall: | 20260508 03:03:37.483 | INFO | BytesWritten : 2130789617714 | 20260508 03:03:37.483 | INFO | NumEvents : 1029687878363 | 20260508 03:03:39.004 | INFO | Analyzing chunk frames = [4730880,4731392) | 20260508 03:03:39.004 | INFO | MemInfo: { | 20260508 03:03:39.004 | INFO | "gpu" : " 17 GB", | 20260508 03:03:39.004 | INFO | "huge" : "", | 20260508 03:03:39.004 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:39.004 | INFO | "malloc" : " 14 GB", | 20260508 03:03:39.004 | INFO | "pinned" : " 29 GB" | 20260508 03:03:39.004 | INFO | } | 20260508 03:03:39.004 | INFO | Perf report for the last 5.14297 seconds: | 20260508 03:03:39.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:39.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:39.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:39.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:39.004 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 03:03:39.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:39.004 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 03:03:39.004 | INFO | PRE_HQ : 65.5%, 1.6, 55.8ms, 0 | 20260508 03:03:39.004 | INFO | BAZWRITER : 68.2%, 3.8, 132.5ms, 0 | 20260508 03:03:39.004 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:03:39.005 | INFO | Uploaded 36GiB to GPU in 2.57728s (13.9682GiB/s) | 20260508 03:03:39.005 | INFO | Downloaded 38GiB to GPU in 2.70495s (14.0483GiB/s) | 20260508 03:03:44.145 | INFO | Analyzing chunk frames = [4731392,4731904) | 20260508 03:03:44.145 | INFO | MemInfo: { | 20260508 03:03:44.145 | INFO | "gpu" : " 17 GB", | 20260508 03:03:44.145 | INFO | "huge" : "", | 20260508 03:03:44.145 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:44.145 | INFO | "malloc" : " 14 GB", | 20260508 03:03:44.145 | INFO | "pinned" : " 29 GB" | 20260508 03:03:44.145 | INFO | } | 20260508 03:03:44.145 | INFO | Perf report for the last 5.14087 seconds: | 20260508 03:03:44.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:44.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:03:44.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 03:03:44.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:44.145 | INFO | Uploaded 36GiB to GPU in 2.57287s (13.9922GiB/s) | 20260508 03:03:44.145 | INFO | Downloaded 34GiB to GPU in 2.43246s (13.9776GiB/s) | 20260508 03:03:49.281 | INFO | Analyzing chunk frames = [4731904,4732416) | 20260508 03:03:49.281 | INFO | MemInfo: { | 20260508 03:03:49.281 | INFO | "gpu" : " 17 GB", | 20260508 03:03:49.281 | INFO | "huge" : "", | 20260508 03:03:49.281 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:49.281 | INFO | "malloc" : " 14 GB", | 20260508 03:03:49.281 | INFO | "pinned" : " 29 GB" | 20260508 03:03:49.281 | INFO | } | 20260508 03:03:49.281 | INFO | Perf report for the last 5.1357 seconds: | 20260508 03:03:49.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:49.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 03:03:49.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:03:49.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:49.281 | INFO | Uploaded 36GiB to GPU in 2.54785s (14.1296GiB/s) | 20260508 03:03:49.281 | INFO | Downloaded 34GiB to GPU in 2.5084s (13.5545GiB/s) | 20260508 03:03:54.414 | INFO | Analyzing chunk frames = [4732416,4732928) | 20260508 03:03:54.414 | INFO | MemInfo: { | 20260508 03:03:54.414 | INFO | "gpu" : " 17 GB", | 20260508 03:03:54.414 | INFO | "huge" : "", | 20260508 03:03:54.414 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:54.414 | INFO | "malloc" : " 14 GB", | 20260508 03:03:54.414 | INFO | "pinned" : " 29 GB" | 20260508 03:03:54.414 | INFO | } | 20260508 03:03:54.414 | INFO | Perf report for the last 5.13351 seconds: | 20260508 03:03:54.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:54.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | ANALYSIS : 67.4%, 3.0, 104.1ms, 0 | 20260508 03:03:54.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:03:54.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:54.415 | INFO | Uploaded 36GiB to GPU in 2.53659s (14.1923GiB/s) | 20260508 03:03:54.415 | INFO | Downloaded 34GiB to GPU in 2.50289s (13.5843GiB/s) | 20260508 03:03:59.559 | INFO | Analyzing chunk frames = [4732928,4733440) | 20260508 03:03:59.559 | INFO | MemInfo: { | 20260508 03:03:59.559 | INFO | "gpu" : " 17 GB", | 20260508 03:03:59.559 | INFO | "huge" : "", | 20260508 03:03:59.559 | INFO | "huge_shared" : " 29 GB", | 20260508 03:03:59.559 | INFO | "malloc" : " 14 GB", | 20260508 03:03:59.559 | INFO | "pinned" : " 29 GB" | 20260508 03:03:59.559 | INFO | } | 20260508 03:03:59.559 | INFO | Perf report for the last 5.14509 seconds: | 20260508 03:03:59.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:03:59.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.559 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 03:03:59.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.559 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:03:59.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:03:59.560 | INFO | Uploaded 36GiB to GPU in 2.3346s (15.4202GiB/s) | 20260508 03:03:59.560 | INFO | Downloaded 34GiB to GPU in 2.50702s (13.5619GiB/s) | 20260508 03:04:04.702 | INFO | Analyzing chunk frames = [4733440,4733952) | 20260508 03:04:04.702 | INFO | MemInfo: { | 20260508 03:04:04.702 | INFO | "gpu" : " 17 GB", | 20260508 03:04:04.702 | INFO | "huge" : "", | 20260508 03:04:04.702 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:04.702 | INFO | "malloc" : " 14 GB", | 20260508 03:04:04.702 | INFO | "pinned" : " 29 GB" | 20260508 03:04:04.702 | INFO | } | 20260508 03:04:04.702 | INFO | Perf report for the last 5.14274 seconds: | 20260508 03:04:04.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:04.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.702 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 03:04:04.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.702 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:04:04.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:04.703 | INFO | Uploaded 36GiB to GPU in 2.34628s (15.3434GiB/s) | 20260508 03:04:04.703 | INFO | Downloaded 34GiB to GPU in 2.42927s (13.996GiB/s) | 20260508 03:04:09.848 | INFO | Analyzing chunk frames = [4733952,4734464) | 20260508 03:04:09.848 | INFO | MemInfo: { | 20260508 03:04:09.848 | INFO | "gpu" : " 17 GB", | 20260508 03:04:09.848 | INFO | "huge" : "", | 20260508 03:04:09.848 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:09.848 | INFO | "malloc" : " 14 GB", | 20260508 03:04:09.848 | INFO | "pinned" : " 29 GB" | 20260508 03:04:09.848 | INFO | } | 20260508 03:04:09.848 | INFO | Perf report for the last 5.14616 seconds: | 20260508 03:04:09.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:09.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.848 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:04:09.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.848 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:04:09.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:09.849 | INFO | Uploaded 36GiB to GPU in 2.41051s (14.9346GiB/s) | 20260508 03:04:09.849 | INFO | Downloaded 34GiB to GPU in 2.43663s (13.9537GiB/s) | 20260508 03:04:14.981 | INFO | Analyzing chunk frames = [4734464,4734976) | 20260508 03:04:14.981 | INFO | MemInfo: { | 20260508 03:04:14.981 | INFO | "gpu" : " 17 GB", | 20260508 03:04:14.981 | INFO | "huge" : "", | 20260508 03:04:14.981 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:14.981 | INFO | "malloc" : " 14 GB", | 20260508 03:04:14.981 | INFO | "pinned" : " 29 GB" | 20260508 03:04:14.981 | INFO | } | 20260508 03:04:14.981 | INFO | Perf report for the last 5.13254 seconds: | 20260508 03:04:14.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:14.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.981 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 03:04:14.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.981 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:04:14.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:14.982 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:04:14.982 | INFO | Total mem usage: 16.7 GiB | 20260508 03:04:14.982 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:04:14.982 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.4%) | 20260508 03:04:14.982 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:04:14.982 | INFO | High Water Breakdown: | 20260508 03:04:14.982 | INFO | 60.5% : 9.13 GiB ( 88.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:04:14.982 | INFO | 39.4% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:04:14.982 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:04:14.982 | INFO | ------------------------------------------ | 20260508 03:04:14.982 | INFO | Uploaded 36GiB to GPU in 2.41444s (14.9103GiB/s) | 20260508 03:04:14.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:04:14.982 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:04:14.982 | INFO | Downloaded 34GiB to GPU in 2.44094s (13.929GiB/s) | 20260508 03:04:14.982 | INFO | Total mem usage: 26.5 GiB | 20260508 03:04:14.982 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:04:14.982 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:04:14.982 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:04:14.982 | INFO | High Water Breakdown: | 20260508 03:04:14.982 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:04:14.982 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:04:14.982 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:04:14.982 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:04:14.982 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:04:14.982 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:04:14.982 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:04:14.982 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:04:14.982 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:04:14.982 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:04:14.982 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:04:14.982 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:04:14.982 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:04:14.982 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:04:14.982 | INFO | ------------------------------------------ | 20260508 03:04:14.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:04:14.982 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:04:14.982 | INFO | Total mem usage: 27.8 GiB | 20260508 03:04:14.982 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:04:14.982 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:04:14.982 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:04:14.982 | INFO | High Water Breakdown: | 20260508 03:04:14.982 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:04:14.982 | INFO | ------------------------------------------ | 20260508 03:04:14.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:04:14.982 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:04:14.982 | INFO | Total mem usage: 15.7 GiB | 20260508 03:04:14.982 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:04:14.982 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:04:14.983 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:04:14.983 | INFO | High Water Breakdown: | 20260508 03:04:14.983 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:04:14.983 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:04:14.983 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:04:14.983 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:04:14.983 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:04:14.983 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:04:14.983 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:04:14.983 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:04:14.983 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:04:14.983 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:04:14.983 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:04:14.983 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:04:14.983 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:04:14.983 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:04:14.983 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:04:14.983 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:04:14.983 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:04:14.983 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:04:14.983 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:04:14.983 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:04:14.983 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:04:14.983 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:04:14.983 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:04:14.983 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:04:14.983 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:04:14.983 | INFO | ------------------------------------------ | 20260508 03:04:14.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:04:14.983 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:04:14.983 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:04:14.983 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:04:14.983 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.43e+01 / 1.43e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.01e-01, 1.23e-02 / 4.38e-03 | 20260508 03:04:14.983 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.48e+00, 5.81e-01 / 1.73e-01 | 20260508 03:04:14.983 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.09e-01 / 1.08e-01 | 20260508 03:04:14.983 | INFO | PerfReport: 16.13%, DME -- 4.21e+01 / 4.62e+01, 4.08e+00 / 1.80e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.63e+00 / 1.44e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.68e+00, 4.19e-01 / 2.27e-01 | 20260508 03:04:14.983 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.24e+00, 2.65e-01 / 1.39e-01 | 20260508 03:04:14.983 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.25e-01 / 3.59e-01, 5.13e-02 / 2.14e-02 | 20260508 03:04:14.983 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:04:14.983 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:04:14.983 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:04:14.983 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 24.47%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 2.13%, LockUpload -- 2.27e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 03:04:14.983 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 03:04:14.984 | INFO | PerfReport: 15.48%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:04:14.984 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260508 03:04:15.093 | INFO | Starting a round of Baz IO | 20260508 03:04:18.521 | INFO | Populating RTMetrics report | 20260508 03:04:18.521 | INFO | RTMetrics report populated | 20260508 03:04:18.521 | INFO | Opening tmp RTMetrics json file | 20260508 03:04:18.617 | INFO | Serializing RTMetrics to json | 20260508 03:04:18.621 | INFO | Serialization done, swapping tmp file now... | 20260508 03:04:18.625 | INFO | RTMetrics done | 20260508 03:04:18.933 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.839 seconds | 20260508 03:04:18.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.38% / 0.00% / 58.62% | 20260508 03:04:18.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.656017% / 1.19209e-05% / 5.96046e-05% | 20260508 03:04:18.933 | INFO | Baz IO overall: | 20260508 03:04:18.933 | INFO | BytesWritten : 2132532378139 | 20260508 03:04:18.933 | INFO | NumEvents : 1030522596690 | 20260508 03:04:20.118 | INFO | Analyzing chunk frames = [4734976,4735488) | 20260508 03:04:20.118 | INFO | MemInfo: { | 20260508 03:04:20.118 | INFO | "gpu" : " 17 GB", | 20260508 03:04:20.118 | INFO | "huge" : "", | 20260508 03:04:20.118 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:20.118 | INFO | "malloc" : " 14 GB", | 20260508 03:04:20.118 | INFO | "pinned" : " 29 GB" | 20260508 03:04:20.118 | INFO | } | 20260508 03:04:20.118 | INFO | Perf report for the last 5.13754 seconds: | 20260508 03:04:20.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:20.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:20.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:20.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:20.119 | INFO | ANALYSIS : 68.0%, 2.9, 104.9ms, 0 | 20260508 03:04:20.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:20.119 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 03:04:20.119 | INFO | PRE_HQ : 66.5%, 1.7, 59.8ms, 0 | 20260508 03:04:20.119 | INFO | BAZWRITER : 74.3%, 4.1, 157.8ms, 0 | 20260508 03:04:20.119 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260508 03:04:20.119 | INFO | Uploaded 36GiB to GPU in 2.53326s (14.2109GiB/s) | 20260508 03:04:20.119 | INFO | Downloaded 38GiB to GPU in 2.68302s (14.1632GiB/s) | 20260508 03:04:25.261 | INFO | Analyzing chunk frames = [4735488,4736000) | 20260508 03:04:25.261 | INFO | MemInfo: { | 20260508 03:04:25.261 | INFO | "gpu" : " 17 GB", | 20260508 03:04:25.261 | INFO | "huge" : "", | 20260508 03:04:25.261 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:25.261 | INFO | "malloc" : " 14 GB", | 20260508 03:04:25.261 | INFO | "pinned" : " 29 GB" | 20260508 03:04:25.261 | INFO | } | 20260508 03:04:25.261 | INFO | Perf report for the last 5.14267 seconds: | 20260508 03:04:25.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:25.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 03:04:25.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:04:25.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:25.261 | INFO | Uploaded 36GiB to GPU in 2.54553s (14.1425GiB/s) | 20260508 03:04:25.262 | INFO | Downloaded 34GiB to GPU in 2.42622s (14.0136GiB/s) | 20260508 03:04:30.399 | INFO | Analyzing chunk frames = [4736000,4736512) | 20260508 03:04:30.399 | INFO | MemInfo: { | 20260508 03:04:30.399 | INFO | "gpu" : " 17 GB", | 20260508 03:04:30.399 | INFO | "huge" : "", | 20260508 03:04:30.399 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:30.399 | INFO | "malloc" : " 14 GB", | 20260508 03:04:30.399 | INFO | "pinned" : " 29 GB" | 20260508 03:04:30.399 | INFO | } | 20260508 03:04:30.399 | INFO | Perf report for the last 5.138 seconds: | 20260508 03:04:30.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:30.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 03:04:30.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:04:30.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:30.399 | INFO | Uploaded 36GiB to GPU in 2.55226s (14.1051GiB/s) | 20260508 03:04:30.399 | INFO | Downloaded 34GiB to GPU in 2.42334s (14.0302GiB/s) | 20260508 03:04:35.543 | INFO | Analyzing chunk frames = [4736512,4737024) | 20260508 03:04:35.543 | INFO | MemInfo: { | 20260508 03:04:35.543 | INFO | "gpu" : " 17 GB", | 20260508 03:04:35.543 | INFO | "huge" : "", | 20260508 03:04:35.543 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:35.543 | INFO | "malloc" : " 14 GB", | 20260508 03:04:35.543 | INFO | "pinned" : " 29 GB" | 20260508 03:04:35.543 | INFO | } | 20260508 03:04:35.543 | INFO | Perf report for the last 5.14417 seconds: | 20260508 03:04:35.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:35.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.543 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 03:04:35.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.543 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:04:35.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:35.544 | INFO | Uploaded 36GiB to GPU in 2.53689s (14.1906GiB/s) | 20260508 03:04:35.544 | INFO | Downloaded 34GiB to GPU in 2.5086s (13.5534GiB/s) | 20260508 03:04:40.676 | INFO | Analyzing chunk frames = [4737024,4737536) | 20260508 03:04:40.676 | INFO | MemInfo: { | 20260508 03:04:40.676 | INFO | "gpu" : " 17 GB", | 20260508 03:04:40.676 | INFO | "huge" : "", | 20260508 03:04:40.676 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:40.676 | INFO | "malloc" : " 14 GB", | 20260508 03:04:40.676 | INFO | "pinned" : " 29 GB" | 20260508 03:04:40.676 | INFO | } | 20260508 03:04:40.676 | INFO | Perf report for the last 5.13305 seconds: | 20260508 03:04:40.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:40.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.676 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 03:04:40.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.676 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:04:40.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:40.677 | INFO | Uploaded 36GiB to GPU in 2.3762s (15.1503GiB/s) | 20260508 03:04:40.677 | INFO | Downloaded 34GiB to GPU in 2.55236s (13.321GiB/s) | 20260508 03:04:45.817 | INFO | Analyzing chunk frames = [4737536,4738048) | 20260508 03:04:45.817 | INFO | MemInfo: { | 20260508 03:04:45.817 | INFO | "gpu" : " 17 GB", | 20260508 03:04:45.817 | INFO | "huge" : "", | 20260508 03:04:45.817 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:45.817 | INFO | "malloc" : " 14 GB", | 20260508 03:04:45.817 | INFO | "pinned" : " 29 GB" | 20260508 03:04:45.817 | INFO | } | 20260508 03:04:45.817 | INFO | Perf report for the last 5.14092 seconds: | 20260508 03:04:45.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:45.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.817 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:04:45.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.817 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:04:45.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:45.818 | INFO | Uploaded 36GiB to GPU in 2.41506s (14.9065GiB/s) | 20260508 03:04:45.818 | INFO | Downloaded 34GiB to GPU in 2.41802s (14.0611GiB/s) | 20260508 03:04:50.954 | INFO | Analyzing chunk frames = [4738048,4738560) | 20260508 03:04:50.954 | INFO | MemInfo: { | 20260508 03:04:50.954 | INFO | "gpu" : " 17 GB", | 20260508 03:04:50.954 | INFO | "huge" : "", | 20260508 03:04:50.954 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:50.954 | INFO | "malloc" : " 14 GB", | 20260508 03:04:50.954 | INFO | "pinned" : " 29 GB" | 20260508 03:04:50.954 | INFO | } | 20260508 03:04:50.954 | INFO | Perf report for the last 5.13637 seconds: | 20260508 03:04:50.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:50.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:04:50.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 03:04:50.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:50.954 | INFO | Uploaded 36GiB to GPU in 2.49496s (14.4291GiB/s) | 20260508 03:04:50.955 | INFO | Downloaded 34GiB to GPU in 2.42625s (14.0134GiB/s) | 20260508 03:04:56.096 | INFO | Analyzing chunk frames = [4738560,4739072) | 20260508 03:04:56.096 | INFO | MemInfo: { | 20260508 03:04:56.096 | INFO | "gpu" : " 17 GB", | 20260508 03:04:56.096 | INFO | "huge" : "", | 20260508 03:04:56.096 | INFO | "huge_shared" : " 29 GB", | 20260508 03:04:56.096 | INFO | "malloc" : " 14 GB", | 20260508 03:04:56.096 | INFO | "pinned" : " 29 GB" | 20260508 03:04:56.096 | INFO | } | 20260508 03:04:56.096 | INFO | Perf report for the last 5.14234 seconds: | 20260508 03:04:56.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:04:56.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.097 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:04:56.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.097 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:04:56.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:04:56.097 | INFO | Uploaded 36GiB to GPU in 2.48129s (14.5086GiB/s) | 20260508 03:04:56.097 | INFO | Downloaded 34GiB to GPU in 2.42489s (14.0213GiB/s) | 20260508 03:04:56.211 | INFO | Starting a round of Baz IO | 20260508 03:04:59.643 | INFO | Populating RTMetrics report | 20260508 03:04:59.644 | INFO | RTMetrics report populated | 20260508 03:04:59.644 | INFO | Opening tmp RTMetrics json file | 20260508 03:04:59.656 | INFO | Serializing RTMetrics to json | 20260508 03:04:59.657 | INFO | Serialization done, swapping tmp file now... | 20260508 03:04:59.658 | INFO | RTMetrics done | 20260508 03:04:59.715 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.504 seconds | 20260508 03:04:59.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.37% / 0.00% / 58.62% | 20260508 03:04:59.715 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.65488% / 3.97364e-06% / 6.35783e-05% | 20260508 03:04:59.715 | INFO | Baz IO overall: | 20260508 03:04:59.716 | INFO | BytesWritten : 2134275456476 | 20260508 03:04:59.716 | INFO | NumEvents : 1031357669060 | 20260508 03:05:01.230 | INFO | Analyzing chunk frames = [4739072,4739584) | 20260508 03:05:01.230 | INFO | MemInfo: { | 20260508 03:05:01.230 | INFO | "gpu" : " 17 GB", | 20260508 03:05:01.230 | INFO | "huge" : "", | 20260508 03:05:01.230 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:01.230 | INFO | "malloc" : " 14 GB", | 20260508 03:05:01.230 | INFO | "pinned" : " 29 GB" | 20260508 03:05:01.230 | INFO | } | 20260508 03:05:01.230 | INFO | Perf report for the last 5.13404 seconds: | 20260508 03:05:01.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:01.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:01.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:01.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:01.230 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 03:05:01.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:01.230 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 03:05:01.230 | INFO | PRE_HQ : 66.5%, 1.8, 62.1ms, 0 | 20260508 03:05:01.230 | INFO | BAZWRITER : 67.6%, 2.9, 99.9ms, 0 | 20260508 03:05:01.230 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:05:01.230 | INFO | Uploaded 36GiB to GPU in 2.58838s (13.9083GiB/s) | 20260508 03:05:01.230 | INFO | Downloaded 38GiB to GPU in 2.69817s (14.0836GiB/s) | 20260508 03:05:06.370 | INFO | Analyzing chunk frames = [4739584,4740096) | 20260508 03:05:06.370 | INFO | MemInfo: { | 20260508 03:05:06.370 | INFO | "gpu" : " 17 GB", | 20260508 03:05:06.370 | INFO | "huge" : "", | 20260508 03:05:06.370 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:06.370 | INFO | "malloc" : " 14 GB", | 20260508 03:05:06.370 | INFO | "pinned" : " 29 GB" | 20260508 03:05:06.370 | INFO | } | 20260508 03:05:06.370 | INFO | Perf report for the last 5.13978 seconds: | 20260508 03:05:06.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:06.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:05:06.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:05:06.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:06.370 | INFO | Uploaded 36GiB to GPU in 2.52001s (14.2857GiB/s) | 20260508 03:05:06.370 | INFO | Downloaded 34GiB to GPU in 2.43743s (13.9491GiB/s) | 20260508 03:05:11.510 | INFO | Analyzing chunk frames = [4740096,4740608) | 20260508 03:05:11.510 | INFO | MemInfo: { | 20260508 03:05:11.510 | INFO | "gpu" : " 17 GB", | 20260508 03:05:11.510 | INFO | "huge" : "", | 20260508 03:05:11.510 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:11.510 | INFO | "malloc" : " 14 GB", | 20260508 03:05:11.510 | INFO | "pinned" : " 29 GB" | 20260508 03:05:11.510 | INFO | } | 20260508 03:05:11.510 | INFO | Perf report for the last 5.14019 seconds: | 20260508 03:05:11.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:11.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.510 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 03:05:11.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.510 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:05:11.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:11.511 | INFO | Uploaded 36GiB to GPU in 2.50792s (14.3546GiB/s) | 20260508 03:05:11.511 | INFO | Downloaded 34GiB to GPU in 2.49893s (13.6058GiB/s) | 20260508 03:05:16.656 | INFO | Analyzing chunk frames = [4740608,4741120) | 20260508 03:05:16.656 | INFO | MemInfo: { | 20260508 03:05:16.656 | INFO | "gpu" : " 17 GB", | 20260508 03:05:16.656 | INFO | "huge" : "", | 20260508 03:05:16.657 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:16.657 | INFO | "malloc" : " 14 GB", | 20260508 03:05:16.657 | INFO | "pinned" : " 29 GB" | 20260508 03:05:16.657 | INFO | } | 20260508 03:05:16.657 | INFO | Perf report for the last 5.14657 seconds: | 20260508 03:05:16.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:16.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 03:05:16.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:05:16.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:16.657 | INFO | Uploaded 36GiB to GPU in 2.54325s (14.1551GiB/s) | 20260508 03:05:16.657 | INFO | Downloaded 34GiB to GPU in 2.50524s (13.5716GiB/s) | 20260508 03:05:21.796 | INFO | Analyzing chunk frames = [4741120,4741632) | 20260508 03:05:21.796 | INFO | MemInfo: { | 20260508 03:05:21.796 | INFO | "gpu" : " 17 GB", | 20260508 03:05:21.796 | INFO | "huge" : "", | 20260508 03:05:21.796 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:21.796 | INFO | "malloc" : " 14 GB", | 20260508 03:05:21.796 | INFO | "pinned" : " 29 GB" | 20260508 03:05:21.796 | INFO | } | 20260508 03:05:21.796 | INFO | Perf report for the last 5.13912 seconds: | 20260508 03:05:21.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:21.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 03:05:21.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:05:21.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:21.796 | INFO | Uploaded 36GiB to GPU in 2.39303s (15.0437GiB/s) | 20260508 03:05:21.796 | INFO | Downloaded 34GiB to GPU in 2.57053s (13.2268GiB/s) | 20260508 03:05:26.931 | INFO | Analyzing chunk frames = [4741632,4742144) | 20260508 03:05:26.931 | INFO | MemInfo: { | 20260508 03:05:26.931 | INFO | "gpu" : " 17 GB", | 20260508 03:05:26.931 | INFO | "huge" : "", | 20260508 03:05:26.931 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:26.931 | INFO | "malloc" : " 14 GB", | 20260508 03:05:26.932 | INFO | "pinned" : " 29 GB" | 20260508 03:05:26.932 | INFO | } | 20260508 03:05:26.932 | INFO | Perf report for the last 5.13575 seconds: | 20260508 03:05:26.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:26.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:05:26.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 03:05:26.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:26.932 | INFO | Uploaded 36GiB to GPU in 2.40532s (14.9668GiB/s) | 20260508 03:05:26.932 | INFO | Downloaded 34GiB to GPU in 2.42986s (13.9926GiB/s) | 20260508 03:05:32.072 | INFO | Analyzing chunk frames = [4742144,4742656) | 20260508 03:05:32.072 | INFO | MemInfo: { | 20260508 03:05:32.072 | INFO | "gpu" : " 17 GB", | 20260508 03:05:32.072 | INFO | "huge" : "", | 20260508 03:05:32.072 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:32.073 | INFO | "malloc" : " 14 GB", | 20260508 03:05:32.073 | INFO | "pinned" : " 29 GB" | 20260508 03:05:32.073 | INFO | } | 20260508 03:05:32.073 | INFO | Perf report for the last 5.14121 seconds: | 20260508 03:05:32.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:32.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:05:32.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:05:32.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:32.073 | INFO | Uploaded 36GiB to GPU in 2.41271s (14.921GiB/s) | 20260508 03:05:32.073 | INFO | Downloaded 34GiB to GPU in 2.43428s (13.9672GiB/s) | 20260508 03:05:37.213 | INFO | Analyzing chunk frames = [4742656,4743168) | 20260508 03:05:37.213 | INFO | MemInfo: { | 20260508 03:05:37.213 | INFO | "gpu" : " 17 GB", | 20260508 03:05:37.213 | INFO | "huge" : "", | 20260508 03:05:37.213 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:37.213 | INFO | "malloc" : " 14 GB", | 20260508 03:05:37.213 | INFO | "pinned" : " 29 GB" | 20260508 03:05:37.213 | INFO | } | 20260508 03:05:37.214 | INFO | Perf report for the last 5.14072 seconds: | 20260508 03:05:37.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:37.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:05:37.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 03:05:37.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:37.214 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:05:37.214 | INFO | Total mem usage: 16.7 GiB | 20260508 03:05:37.214 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:05:37.214 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -8.96%) | 20260508 03:05:37.214 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:05:37.214 | INFO | High Water Breakdown: | 20260508 03:05:37.214 | INFO | 60.5% : 9.27 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:05:37.214 | INFO | 39.3% : 6.02 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:05:37.214 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:05:37.214 | INFO | ------------------------------------------ | 20260508 03:05:37.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:05:37.214 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:05:37.214 | INFO | Total mem usage: 26.5 GiB | 20260508 03:05:37.214 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:05:37.214 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:05:37.214 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:05:37.214 | INFO | High Water Breakdown: | 20260508 03:05:37.214 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:05:37.214 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:05:37.214 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:05:37.214 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:05:37.214 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:05:37.214 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:05:37.214 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:05:37.214 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:05:37.214 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:05:37.214 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:05:37.214 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:05:37.214 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:05:37.214 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:05:37.214 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:05:37.214 | INFO | ------------------------------------------ | 20260508 03:05:37.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:05:37.214 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:05:37.214 | INFO | Total mem usage: 27.8 GiB | 20260508 03:05:37.214 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:05:37.214 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:05:37.214 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:05:37.214 | INFO | High Water Breakdown: | 20260508 03:05:37.214 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:05:37.214 | INFO | ------------------------------------------ | 20260508 03:05:37.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:05:37.214 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:05:37.214 | INFO | Total mem usage: 15.7 GiB | 20260508 03:05:37.214 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:05:37.214 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:05:37.214 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:05:37.214 | INFO | High Water Breakdown: | 20260508 03:05:37.214 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:05:37.214 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:05:37.214 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:05:37.214 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:05:37.214 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:05:37.214 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:05:37.214 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:05:37.214 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:05:37.214 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:05:37.214 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:05:37.214 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:05:37.214 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:05:37.214 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:05:37.214 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:05:37.214 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:05:37.214 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:05:37.214 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:05:37.214 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:05:37.214 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:05:37.214 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:05:37.215 | INFO | Uploaded 36GiB to GPU in 2.45444s (14.6673GiB/s) | 20260508 03:05:37.214 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:05:37.215 | INFO | Downloaded 34GiB to GPU in 2.44248s (13.9203GiB/s) | 20260508 03:05:37.215 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:05:37.215 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:05:37.215 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:05:37.215 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:05:37.215 | INFO | ------------------------------------------ | 20260508 03:05:37.215 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:05:37.215 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:05:37.215 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:05:37.215 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:05:37.215 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.38e+01 / 1.46e+00 | 20260508 03:05:37.215 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.04e-01, 1.13e-02 / 6.05e-03 | 20260508 03:05:37.215 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.49e+00, 5.90e-01 / 1.63e-01 | 20260508 03:05:37.215 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.06e-01 / 1.13e-01 | 20260508 03:05:37.215 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.55e+01, 3.90e+00 / 1.83e+00 | 20260508 03:05:37.215 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.71e+00 / 1.53e+00 | 20260508 03:05:37.215 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.69e+00, 4.21e-01 / 2.30e-01 | 20260508 03:05:37.215 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.25e+00, 2.66e-01 / 1.54e-01 | 20260508 03:05:37.215 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.68e-01, 5.26e-02 / 2.65e-02 | 20260508 03:05:37.215 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:05:37.215 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:05:37.215 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:05:37.215 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 03:05:37.215 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 03:05:37.216 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 03:05:37.216 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 03:05:37.216 | INFO | PerfReport: 2.11%, LockUpload -- 2.25e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 03:05:37.216 | INFO | PerfReport: 2.79%, LockDownload -- 2.97e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 03:05:37.216 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:05:37.216 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260508 03:05:37.328 | INFO | Starting a round of Baz IO | 20260508 03:05:40.743 | INFO | Populating RTMetrics report | 20260508 03:05:40.743 | INFO | RTMetrics report populated | 20260508 03:05:40.743 | INFO | Opening tmp RTMetrics json file | 20260508 03:05:40.757 | INFO | Serializing RTMetrics to json | 20260508 03:05:40.758 | INFO | Serialization done, swapping tmp file now... | 20260508 03:05:40.758 | INFO | RTMetrics done | 20260508 03:05:40.819 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.49 seconds | 20260508 03:05:40.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.37% / 0.00% / 58.63% | 20260508 03:05:40.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.651248% / 3.97364e-06% / 2.78155e-05% | 20260508 03:05:40.819 | INFO | Baz IO overall: | 20260508 03:05:40.819 | INFO | BytesWritten : 2136016734251 | 20260508 03:05:40.819 | INFO | NumEvents : 1032191342362 | 20260508 03:05:42.350 | INFO | Analyzing chunk frames = [4743168,4743680) | 20260508 03:05:42.350 | INFO | MemInfo: { | 20260508 03:05:42.350 | INFO | "gpu" : " 17 GB", | 20260508 03:05:42.350 | INFO | "huge" : "", | 20260508 03:05:42.350 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:42.350 | INFO | "malloc" : " 14 GB", | 20260508 03:05:42.350 | INFO | "pinned" : " 29 GB" | 20260508 03:05:42.350 | INFO | } | 20260508 03:05:42.350 | INFO | Perf report for the last 5.13657 seconds: | 20260508 03:05:42.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:42.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:42.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:42.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:42.350 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 03:05:42.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:42.350 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 03:05:42.350 | INFO | PRE_HQ : 66.5%, 1.8, 62.5ms, 0 | 20260508 03:05:42.350 | INFO | BAZWRITER : 67.4%, 2.9, 100.1ms, 0 | 20260508 03:05:42.350 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:05:42.351 | INFO | Uploaded 36GiB to GPU in 2.58028s (13.952GiB/s) | 20260508 03:05:42.351 | INFO | Downloaded 38GiB to GPU in 2.70182s (14.0646GiB/s) | 20260508 03:05:47.486 | INFO | Analyzing chunk frames = [4743680,4744192) | 20260508 03:05:47.486 | INFO | MemInfo: { | 20260508 03:05:47.486 | INFO | "gpu" : " 17 GB", | 20260508 03:05:47.486 | INFO | "huge" : "", | 20260508 03:05:47.486 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:47.486 | INFO | "malloc" : " 14 GB", | 20260508 03:05:47.486 | INFO | "pinned" : " 29 GB" | 20260508 03:05:47.486 | INFO | } | 20260508 03:05:47.486 | INFO | Perf report for the last 5.13629 seconds: | 20260508 03:05:47.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:47.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:05:47.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:05:47.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:47.487 | INFO | Uploaded 36GiB to GPU in 2.58308s (13.9369GiB/s) | 20260508 03:05:47.487 | INFO | Downloaded 34GiB to GPU in 2.42806s (14.0029GiB/s) | 20260508 03:05:52.623 | INFO | Analyzing chunk frames = [4744192,4744704) | 20260508 03:05:52.623 | INFO | MemInfo: { | 20260508 03:05:52.623 | INFO | "gpu" : " 17 GB", | 20260508 03:05:52.623 | INFO | "huge" : "", | 20260508 03:05:52.623 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:52.623 | INFO | "malloc" : " 14 GB", | 20260508 03:05:52.623 | INFO | "pinned" : " 29 GB" | 20260508 03:05:52.623 | INFO | } | 20260508 03:05:52.623 | INFO | Perf report for the last 5.13672 seconds: | 20260508 03:05:52.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:52.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 03:05:52.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:05:52.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:52.623 | INFO | Uploaded 36GiB to GPU in 2.57736s (13.9678GiB/s) | 20260508 03:05:52.624 | INFO | Downloaded 34GiB to GPU in 2.4895s (13.6573GiB/s) | 20260508 03:05:57.768 | INFO | Analyzing chunk frames = [4744704,4745216) | 20260508 03:05:57.768 | INFO | MemInfo: { | 20260508 03:05:57.768 | INFO | "gpu" : " 17 GB", | 20260508 03:05:57.768 | INFO | "huge" : "", | 20260508 03:05:57.768 | INFO | "huge_shared" : " 29 GB", | 20260508 03:05:57.768 | INFO | "malloc" : " 14 GB", | 20260508 03:05:57.768 | INFO | "pinned" : " 29 GB" | 20260508 03:05:57.768 | INFO | } | 20260508 03:05:57.768 | INFO | Perf report for the last 5.14498 seconds: | 20260508 03:05:57.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:05:57.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.768 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 03:05:57.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.768 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:05:57.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:05:57.769 | INFO | Uploaded 36GiB to GPU in 2.53652s (14.1927GiB/s) | 20260508 03:05:57.769 | INFO | Downloaded 34GiB to GPU in 2.47704s (13.7261GiB/s) | 20260508 03:06:02.912 | INFO | Analyzing chunk frames = [4745216,4745728) | 20260508 03:06:02.912 | INFO | MemInfo: { | 20260508 03:06:02.912 | INFO | "gpu" : " 17 GB", | 20260508 03:06:02.912 | INFO | "huge" : "", | 20260508 03:06:02.912 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:02.912 | INFO | "malloc" : " 14 GB", | 20260508 03:06:02.912 | INFO | "pinned" : " 29 GB" | 20260508 03:06:02.912 | INFO | } | 20260508 03:06:02.912 | INFO | Perf report for the last 5.14414 seconds: | 20260508 03:06:02.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:02.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.912 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 03:06:02.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.912 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:06:02.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:02.913 | INFO | Uploaded 36GiB to GPU in 2.43564s (14.7805GiB/s) | 20260508 03:06:02.913 | INFO | Downloaded 34GiB to GPU in 2.57914s (13.1827GiB/s) | 20260508 03:06:08.048 | INFO | Analyzing chunk frames = [4745728,4746240) | 20260508 03:06:08.048 | INFO | MemInfo: { | 20260508 03:06:08.048 | INFO | "gpu" : " 17 GB", | 20260508 03:06:08.048 | INFO | "huge" : "", | 20260508 03:06:08.048 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:08.048 | INFO | "malloc" : " 14 GB", | 20260508 03:06:08.048 | INFO | "pinned" : " 29 GB" | 20260508 03:06:08.048 | INFO | } | 20260508 03:06:08.048 | INFO | Perf report for the last 5.13596 seconds: | 20260508 03:06:08.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:08.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.048 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:06:08.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.048 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:06:08.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:08.049 | INFO | Uploaded 36GiB to GPU in 2.44936s (14.6977GiB/s) | 20260508 03:06:08.049 | INFO | Downloaded 34GiB to GPU in 2.41602s (14.0727GiB/s) | 20260508 03:06:13.178 | INFO | Analyzing chunk frames = [4746240,4746752) | 20260508 03:06:13.178 | INFO | MemInfo: { | 20260508 03:06:13.178 | INFO | "gpu" : " 17 GB", | 20260508 03:06:13.179 | INFO | "huge" : "", | 20260508 03:06:13.179 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:13.179 | INFO | "malloc" : " 14 GB", | 20260508 03:06:13.179 | INFO | "pinned" : " 29 GB" | 20260508 03:06:13.179 | INFO | } | 20260508 03:06:13.179 | INFO | Perf report for the last 5.13054 seconds: | 20260508 03:06:13.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:13.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:06:13.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:06:13.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:13.179 | INFO | Uploaded 36GiB to GPU in 2.45631s (14.6561GiB/s) | 20260508 03:06:13.179 | INFO | Downloaded 34GiB to GPU in 2.41472s (14.0803GiB/s) | 20260508 03:06:18.324 | INFO | Analyzing chunk frames = [4746752,4747264) | 20260508 03:06:18.324 | INFO | MemInfo: { | 20260508 03:06:18.324 | INFO | "gpu" : " 17 GB", | 20260508 03:06:18.324 | INFO | "huge" : "", | 20260508 03:06:18.324 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:18.324 | INFO | "malloc" : " 14 GB", | 20260508 03:06:18.324 | INFO | "pinned" : " 29 GB" | 20260508 03:06:18.324 | INFO | } | 20260508 03:06:18.324 | INFO | Perf report for the last 5.14485 seconds: | 20260508 03:06:18.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:18.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 03:06:18.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | PRE_HQ : 35.6%, 1.0, 18.6ms, 0 | 20260508 03:06:18.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:18.324 | INFO | Uploaded 36GiB to GPU in 2.48845s (14.4668GiB/s) | 20260508 03:06:18.324 | INFO | Downloaded 34GiB to GPU in 2.43972s (13.9361GiB/s) | 20260508 03:06:18.437 | INFO | Starting a round of Baz IO | 20260508 03:06:21.847 | INFO | Populating RTMetrics report | 20260508 03:06:21.848 | INFO | RTMetrics report populated | 20260508 03:06:21.848 | INFO | Opening tmp RTMetrics json file | 20260508 03:06:21.856 | INFO | Serializing RTMetrics to json | 20260508 03:06:21.862 | INFO | Serialization done, swapping tmp file now... | 20260508 03:06:21.863 | INFO | RTMetrics done | 20260508 03:06:21.920 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.482 seconds | 20260508 03:06:21.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.37% / 0.00% / 58.63% | 20260508 03:06:21.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.649138% / 3.97364e-06% / 5.96046e-05% | 20260508 03:06:21.920 | INFO | Baz IO overall: | 20260508 03:06:21.920 | INFO | BytesWritten : 2137757509844 | 20260508 03:06:21.920 | INFO | NumEvents : 1033024707834 | 20260508 03:06:23.464 | INFO | Analyzing chunk frames = [4747264,4747776) | 20260508 03:06:23.464 | INFO | MemInfo: { | 20260508 03:06:23.464 | INFO | "gpu" : " 17 GB", | 20260508 03:06:23.464 | INFO | "huge" : "", | 20260508 03:06:23.464 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:23.464 | INFO | "malloc" : " 14 GB", | 20260508 03:06:23.464 | INFO | "pinned" : " 29 GB" | 20260508 03:06:23.464 | INFO | } | 20260508 03:06:23.464 | INFO | Perf report for the last 5.14044 seconds: | 20260508 03:06:23.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:23.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:23.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:23.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:23.464 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 03:06:23.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:23.464 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:06:23.464 | INFO | PRE_HQ : 66.6%, 1.8, 62.7ms, 0 | 20260508 03:06:23.464 | INFO | BAZWRITER : 67.0%, 2.9, 99.7ms, 0 | 20260508 03:06:23.465 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:06:23.465 | INFO | Uploaded 36GiB to GPU in 2.57439s (13.9839GiB/s) | 20260508 03:06:23.465 | INFO | Downloaded 38GiB to GPU in 2.67952s (14.1816GiB/s) | 20260508 03:06:28.609 | INFO | Analyzing chunk frames = [4747776,4748288) | 20260508 03:06:28.609 | INFO | MemInfo: { | 20260508 03:06:28.609 | INFO | "gpu" : " 17 GB", | 20260508 03:06:28.609 | INFO | "huge" : "", | 20260508 03:06:28.609 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:28.609 | INFO | "malloc" : " 14 GB", | 20260508 03:06:28.609 | INFO | "pinned" : " 29 GB" | 20260508 03:06:28.609 | INFO | } | 20260508 03:06:28.609 | INFO | Perf report for the last 5.1449 seconds: | 20260508 03:06:28.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:28.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:06:28.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:06:28.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:28.609 | INFO | Uploaded 36GiB to GPU in 2.58108s (13.9477GiB/s) | 20260508 03:06:28.609 | INFO | Downloaded 34GiB to GPU in 2.4166s (14.0694GiB/s) | 20260508 03:06:33.744 | INFO | Analyzing chunk frames = [4748288,4748800) | 20260508 03:06:33.744 | INFO | MemInfo: { | 20260508 03:06:33.744 | INFO | "gpu" : " 17 GB", | 20260508 03:06:33.744 | INFO | "huge" : "", | 20260508 03:06:33.744 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:33.744 | INFO | "malloc" : " 14 GB", | 20260508 03:06:33.744 | INFO | "pinned" : " 29 GB" | 20260508 03:06:33.744 | INFO | } | 20260508 03:06:33.744 | INFO | Perf report for the last 5.13543 seconds: | 20260508 03:06:33.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:33.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:06:33.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:06:33.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:33.745 | INFO | Uploaded 36GiB to GPU in 2.56611s (14.029GiB/s) | 20260508 03:06:33.745 | INFO | Downloaded 34GiB to GPU in 2.40853s (14.1165GiB/s) | 20260508 03:06:38.882 | INFO | Analyzing chunk frames = [4748800,4749312) | 20260508 03:06:38.882 | INFO | MemInfo: { | 20260508 03:06:38.882 | INFO | "gpu" : " 17 GB", | 20260508 03:06:38.882 | INFO | "huge" : "", | 20260508 03:06:38.882 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:38.882 | INFO | "malloc" : " 14 GB", | 20260508 03:06:38.882 | INFO | "pinned" : " 29 GB" | 20260508 03:06:38.882 | INFO | } | 20260508 03:06:38.882 | INFO | Perf report for the last 5.13766 seconds: | 20260508 03:06:38.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:38.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 03:06:38.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:06:38.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:38.882 | INFO | Uploaded 36GiB to GPU in 2.57996s (13.9537GiB/s) | 20260508 03:06:38.883 | INFO | Downloaded 34GiB to GPU in 2.49969s (13.6017GiB/s) | 20260508 03:06:44.021 | INFO | Analyzing chunk frames = [4749312,4749824) | 20260508 03:06:44.021 | INFO | MemInfo: { | 20260508 03:06:44.021 | INFO | "gpu" : " 17 GB", | 20260508 03:06:44.021 | INFO | "huge" : "", | 20260508 03:06:44.021 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:44.021 | INFO | "malloc" : " 14 GB", | 20260508 03:06:44.021 | INFO | "pinned" : " 29 GB" | 20260508 03:06:44.021 | INFO | } | 20260508 03:06:44.021 | INFO | Perf report for the last 5.13906 seconds: | 20260508 03:06:44.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:44.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.021 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:06:44.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.021 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 03:06:44.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:44.022 | INFO | Uploaded 36GiB to GPU in 2.40457s (14.9715GiB/s) | 20260508 03:06:44.022 | INFO | Downloaded 34GiB to GPU in 2.54723s (13.3478GiB/s) | 20260508 03:06:49.163 | INFO | Analyzing chunk frames = [4749824,4750336) | 20260508 03:06:49.163 | INFO | MemInfo: { | 20260508 03:06:49.163 | INFO | "gpu" : " 17 GB", | 20260508 03:06:49.163 | INFO | "huge" : "", | 20260508 03:06:49.163 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:49.163 | INFO | "malloc" : " 14 GB", | 20260508 03:06:49.163 | INFO | "pinned" : " 29 GB" | 20260508 03:06:49.163 | INFO | } | 20260508 03:06:49.163 | INFO | Perf report for the last 5.14212 seconds: | 20260508 03:06:49.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:49.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.163 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:06:49.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.164 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 03:06:49.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:49.164 | INFO | Uploaded 36GiB to GPU in 2.42657s (14.8358GiB/s) | 20260508 03:06:49.164 | INFO | Downloaded 34GiB to GPU in 2.41879s (14.0566GiB/s) | 20260508 03:06:54.298 | INFO | Analyzing chunk frames = [4750336,4750848) | 20260508 03:06:54.298 | INFO | MemInfo: { | 20260508 03:06:54.298 | INFO | "gpu" : " 17 GB", | 20260508 03:06:54.298 | INFO | "huge" : "", | 20260508 03:06:54.298 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:54.298 | INFO | "malloc" : " 14 GB", | 20260508 03:06:54.298 | INFO | "pinned" : " 29 GB" | 20260508 03:06:54.299 | INFO | } | 20260508 03:06:54.299 | INFO | Perf report for the last 5.13521 seconds: | 20260508 03:06:54.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:54.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:06:54.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 03:06:54.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:54.299 | INFO | Uploaded 36GiB to GPU in 2.43778s (14.7676GiB/s) | 20260508 03:06:54.299 | INFO | Downloaded 34GiB to GPU in 2.41824s (14.0598GiB/s) | 20260508 03:06:59.436 | INFO | Analyzing chunk frames = [4750848,4751360) | 20260508 03:06:59.436 | INFO | MemInfo: { | 20260508 03:06:59.436 | INFO | "gpu" : " 17 GB", | 20260508 03:06:59.437 | INFO | "huge" : "", | 20260508 03:06:59.437 | INFO | "huge_shared" : " 29 GB", | 20260508 03:06:59.437 | INFO | "malloc" : " 14 GB", | 20260508 03:06:59.437 | INFO | "pinned" : " 29 GB" | 20260508 03:06:59.437 | INFO | } | 20260508 03:06:59.437 | INFO | Perf report for the last 5.13828 seconds: | 20260508 03:06:59.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:06:59.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:06:59.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 03:06:59.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:06:59.437 | INFO | Uploaded 36GiB to GPU in 2.46881s (14.5819GiB/s) | 20260508 03:06:59.437 | INFO | Downloaded 34GiB to GPU in 2.42827s (14.0018GiB/s) | 20260508 03:06:59.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:06:59.437 | INFO | Total mem usage: 16.7 GiB | 20260508 03:06:59.437 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:06:59.437 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.2%) | 20260508 03:06:59.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:06:59.437 | INFO | High Water Breakdown: | 20260508 03:06:59.437 | INFO | 59.7% : 8.64 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:06:59.438 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:06:59.438 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:06:59.438 | INFO | ------------------------------------------ | 20260508 03:06:59.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:06:59.438 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:06:59.438 | INFO | Total mem usage: 26.5 GiB | 20260508 03:06:59.438 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:06:59.438 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:06:59.438 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:06:59.438 | INFO | High Water Breakdown: | 20260508 03:06:59.438 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:06:59.438 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:06:59.438 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:06:59.438 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:06:59.438 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:06:59.438 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:06:59.438 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:06:59.438 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:06:59.438 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:06:59.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:06:59.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:06:59.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:06:59.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:06:59.438 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:06:59.438 | INFO | ------------------------------------------ | 20260508 03:06:59.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:06:59.438 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:06:59.438 | INFO | Total mem usage: 27.8 GiB | 20260508 03:06:59.438 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:06:59.438 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:06:59.438 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:06:59.438 | INFO | High Water Breakdown: | 20260508 03:06:59.438 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:06:59.438 | INFO | ------------------------------------------ | 20260508 03:06:59.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:06:59.438 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:06:59.438 | INFO | Total mem usage: 15.7 GiB | 20260508 03:06:59.438 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:06:59.438 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:06:59.438 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:06:59.438 | INFO | High Water Breakdown: | 20260508 03:06:59.438 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:06:59.438 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:06:59.438 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:06:59.438 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:06:59.438 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:06:59.438 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:06:59.438 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:06:59.438 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:06:59.438 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:06:59.438 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:06:59.438 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:06:59.438 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:06:59.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:06:59.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:06:59.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:06:59.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:06:59.438 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:06:59.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:06:59.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:06:59.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:06:59.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:06:59.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:06:59.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:06:59.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:06:59.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:06:59.438 | INFO | ------------------------------------------ | 20260508 03:06:59.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:06:59.439 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:06:59.439 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:06:59.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:06:59.439 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.80e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.04e-01, 1.23e-02 / 7.41e-03 | 20260508 03:06:59.439 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.90e-01 / 1.69e-01 | 20260508 03:06:59.439 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.09e-01 / 1.07e-01 | 20260508 03:06:59.439 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.60e+01, 3.90e+00 / 2.12e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.68e+00 / 1.36e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.40e+00 / 2.65e+00, 4.18e-01 / 2.36e-01 | 20260508 03:06:59.439 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.24e+00, 2.68e-01 / 1.22e-01 | 20260508 03:06:59.439 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.65e-01, 5.23e-02 / 2.46e-02 | 20260508 03:06:59.439 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:06:59.439 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:06:59.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:06:59.439 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 24.54%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 24.30%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 30.70%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 2.39%, LockUpload -- 2.55e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 2.56%, LockDownload -- 2.73e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:06:59.439 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260508 03:06:59.550 | INFO | Starting a round of Baz IO | 20260508 03:07:02.969 | INFO | Populating RTMetrics report | 20260508 03:07:02.970 | INFO | RTMetrics report populated | 20260508 03:07:02.970 | INFO | Opening tmp RTMetrics json file | 20260508 03:07:02.984 | INFO | Serializing RTMetrics to json | 20260508 03:07:02.985 | INFO | Serialization done, swapping tmp file now... | 20260508 03:07:02.986 | INFO | RTMetrics done | 20260508 03:07:03.046 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.496 seconds | 20260508 03:07:03.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.36% / 0.00% / 58.63% | 20260508 03:07:03.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.649138% / 0% / 5.5631e-05% | 20260508 03:07:03.046 | INFO | Baz IO overall: | 20260508 03:07:03.046 | INFO | BytesWritten : 2139499255815 | 20260508 03:07:03.046 | INFO | NumEvents : 1033858828136 | 20260508 03:07:04.577 | INFO | Analyzing chunk frames = [4751360,4751872) | 20260508 03:07:04.577 | INFO | MemInfo: { | 20260508 03:07:04.577 | INFO | "gpu" : " 17 GB", | 20260508 03:07:04.577 | INFO | "huge" : "", | 20260508 03:07:04.577 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:04.577 | INFO | "malloc" : " 14 GB", | 20260508 03:07:04.577 | INFO | "pinned" : " 29 GB" | 20260508 03:07:04.577 | INFO | } | 20260508 03:07:04.577 | INFO | Perf report for the last 5.1403 seconds: | 20260508 03:07:04.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:04.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:04.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:04.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:04.577 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 03:07:04.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:04.577 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:07:04.577 | INFO | PRE_HQ : 66.4%, 1.8, 62.3ms, 0 | 20260508 03:07:04.577 | INFO | BAZWRITER : 67.4%, 2.8, 97.8ms, 0 | 20260508 03:07:04.577 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:07:04.578 | INFO | Uploaded 36GiB to GPU in 2.57416s (13.9852GiB/s) | 20260508 03:07:04.578 | INFO | Downloaded 38GiB to GPU in 2.69113s (14.1205GiB/s) | 20260508 03:07:09.720 | INFO | Analyzing chunk frames = [4751872,4752384) | 20260508 03:07:09.720 | INFO | MemInfo: { | 20260508 03:07:09.720 | INFO | "gpu" : " 17 GB", | 20260508 03:07:09.720 | INFO | "huge" : "", | 20260508 03:07:09.720 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:09.720 | INFO | "malloc" : " 14 GB", | 20260508 03:07:09.720 | INFO | "pinned" : " 29 GB" | 20260508 03:07:09.720 | INFO | } | 20260508 03:07:09.720 | INFO | Perf report for the last 5.14261 seconds: | 20260508 03:07:09.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:09.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:07:09.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:07:09.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:09.720 | INFO | Uploaded 36GiB to GPU in 2.53597s (14.1957GiB/s) | 20260508 03:07:09.720 | INFO | Downloaded 34GiB to GPU in 2.43529s (13.9614GiB/s) | 20260508 03:07:14.863 | INFO | Analyzing chunk frames = [4752384,4752896) | 20260508 03:07:14.863 | INFO | MemInfo: { | 20260508 03:07:14.863 | INFO | "gpu" : " 17 GB", | 20260508 03:07:14.863 | INFO | "huge" : "", | 20260508 03:07:14.863 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:14.863 | INFO | "malloc" : " 14 GB", | 20260508 03:07:14.863 | INFO | "pinned" : " 29 GB" | 20260508 03:07:14.863 | INFO | } | 20260508 03:07:14.863 | INFO | Perf report for the last 5.14334 seconds: | 20260508 03:07:14.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:14.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:07:14.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:07:14.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:14.863 | INFO | Uploaded 36GiB to GPU in 2.56009s (14.062GiB/s) | 20260508 03:07:14.863 | INFO | Downloaded 34GiB to GPU in 2.50017s (13.5991GiB/s) | 20260508 03:07:19.995 | INFO | Analyzing chunk frames = [4752896,4753408) | 20260508 03:07:19.995 | INFO | MemInfo: { | 20260508 03:07:19.995 | INFO | "gpu" : " 17 GB", | 20260508 03:07:19.995 | INFO | "huge" : "", | 20260508 03:07:19.995 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:19.995 | INFO | "malloc" : " 14 GB", | 20260508 03:07:19.995 | INFO | "pinned" : " 29 GB" | 20260508 03:07:19.995 | INFO | } | 20260508 03:07:19.996 | INFO | Perf report for the last 5.13232 seconds: | 20260508 03:07:19.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:19.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 03:07:19.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 03:07:19.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:19.996 | INFO | Uploaded 36GiB to GPU in 2.56469s (14.0368GiB/s) | 20260508 03:07:19.996 | INFO | Downloaded 34GiB to GPU in 2.42466s (14.0226GiB/s) | 20260508 03:07:25.131 | INFO | Analyzing chunk frames = [4753408,4753920) | 20260508 03:07:25.131 | INFO | MemInfo: { | 20260508 03:07:25.131 | INFO | "gpu" : " 17 GB", | 20260508 03:07:25.131 | INFO | "huge" : "", | 20260508 03:07:25.131 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:25.131 | INFO | "malloc" : " 14 GB", | 20260508 03:07:25.131 | INFO | "pinned" : " 29 GB" | 20260508 03:07:25.131 | INFO | } | 20260508 03:07:25.131 | INFO | Perf report for the last 5.13578 seconds: | 20260508 03:07:25.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:25.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.131 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:07:25.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.131 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:07:25.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:25.132 | INFO | Uploaded 36GiB to GPU in 2.47043s (14.5724GiB/s) | 20260508 03:07:25.132 | INFO | Downloaded 34GiB to GPU in 2.5953s (13.1006GiB/s) | 20260508 03:07:30.273 | INFO | Analyzing chunk frames = [4753920,4754432) | 20260508 03:07:30.273 | INFO | MemInfo: { | 20260508 03:07:30.273 | INFO | "gpu" : " 17 GB", | 20260508 03:07:30.273 | INFO | "huge" : "", | 20260508 03:07:30.273 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:30.273 | INFO | "malloc" : " 14 GB", | 20260508 03:07:30.273 | INFO | "pinned" : " 29 GB" | 20260508 03:07:30.273 | INFO | } | 20260508 03:07:30.273 | INFO | Perf report for the last 5.14162 seconds: | 20260508 03:07:30.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:30.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 03:07:30.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:07:30.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:30.273 | INFO | Uploaded 36GiB to GPU in 2.39179s (15.0515GiB/s) | 20260508 03:07:30.274 | INFO | Downloaded 34GiB to GPU in 2.4183s (14.0595GiB/s) | 20260508 03:07:35.415 | INFO | Analyzing chunk frames = [4754432,4754944) | 20260508 03:07:35.415 | INFO | MemInfo: { | 20260508 03:07:35.415 | INFO | "gpu" : " 17 GB", | 20260508 03:07:35.415 | INFO | "huge" : "", | 20260508 03:07:35.415 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:35.415 | INFO | "malloc" : " 14 GB", | 20260508 03:07:35.415 | INFO | "pinned" : " 29 GB" | 20260508 03:07:35.415 | INFO | } | 20260508 03:07:35.415 | INFO | Perf report for the last 5.14218 seconds: | 20260508 03:07:35.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:35.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 03:07:35.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 03:07:35.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:35.415 | INFO | Uploaded 36GiB to GPU in 2.44678s (14.7132GiB/s) | 20260508 03:07:35.416 | INFO | Downloaded 34GiB to GPU in 2.42898s (13.9977GiB/s) | 20260508 03:07:40.556 | INFO | Analyzing chunk frames = [4754944,4755456) | 20260508 03:07:40.556 | INFO | MemInfo: { | 20260508 03:07:40.556 | INFO | "gpu" : " 17 GB", | 20260508 03:07:40.556 | INFO | "huge" : "", | 20260508 03:07:40.556 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:40.556 | INFO | "malloc" : " 14 GB", | 20260508 03:07:40.556 | INFO | "pinned" : " 29 GB" | 20260508 03:07:40.556 | INFO | } | 20260508 03:07:40.556 | INFO | Perf report for the last 5.14096 seconds: | 20260508 03:07:40.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:40.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:07:40.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 03:07:40.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:40.556 | INFO | Uploaded 36GiB to GPU in 2.46295s (14.6166GiB/s) | 20260508 03:07:40.556 | INFO | Downloaded 34GiB to GPU in 2.42977s (13.9931GiB/s) | 20260508 03:07:40.654 | INFO | Starting a round of Baz IO | 20260508 03:07:44.099 | INFO | Populating RTMetrics report | 20260508 03:07:44.099 | INFO | RTMetrics report populated | 20260508 03:07:44.099 | INFO | Opening tmp RTMetrics json file | 20260508 03:07:44.108 | INFO | Serializing RTMetrics to json | 20260508 03:07:44.110 | INFO | Serialization done, swapping tmp file now... | 20260508 03:07:44.110 | INFO | RTMetrics done | 20260508 03:07:44.165 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.511 seconds | 20260508 03:07:44.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.36% / 0.00% / 58.64% | 20260508 03:07:44.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.64681% / 0% / 6.35783e-05% | 20260508 03:07:44.165 | INFO | Baz IO overall: | 20260508 03:07:44.165 | INFO | BytesWritten : 2141240011243 | 20260508 03:07:44.165 | INFO | NumEvents : 1034692016841 | 20260508 03:07:45.692 | INFO | Analyzing chunk frames = [4755456,4755968) | 20260508 03:07:45.692 | INFO | MemInfo: { | 20260508 03:07:45.692 | INFO | "gpu" : " 17 GB", | 20260508 03:07:45.692 | INFO | "huge" : "", | 20260508 03:07:45.692 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:45.692 | INFO | "malloc" : " 14 GB", | 20260508 03:07:45.692 | INFO | "pinned" : " 29 GB" | 20260508 03:07:45.692 | INFO | } | 20260508 03:07:45.693 | INFO | Perf report for the last 5.13639 seconds: | 20260508 03:07:45.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:45.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:45.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:45.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:45.693 | INFO | ANALYSIS : 68.0%, 3.0, 106.3ms, 0 | 20260508 03:07:45.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:45.693 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 03:07:45.693 | INFO | PRE_HQ : 65.9%, 1.6, 56.4ms, 0 | 20260508 03:07:45.693 | INFO | BAZWRITER : 67.9%, 2.6, 90.5ms, 0 | 20260508 03:07:45.693 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:07:45.693 | INFO | Uploaded 36GiB to GPU in 2.57025s (14.0064GiB/s) | 20260508 03:07:45.693 | INFO | Downloaded 38GiB to GPU in 2.69577s (14.0962GiB/s) | 20260508 03:07:50.834 | INFO | Analyzing chunk frames = [4755968,4756480) | 20260508 03:07:50.834 | INFO | MemInfo: { | 20260508 03:07:50.834 | INFO | "gpu" : " 17 GB", | 20260508 03:07:50.834 | INFO | "huge" : "", | 20260508 03:07:50.834 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:50.834 | INFO | "malloc" : " 14 GB", | 20260508 03:07:50.834 | INFO | "pinned" : " 29 GB" | 20260508 03:07:50.834 | INFO | } | 20260508 03:07:50.834 | INFO | Perf report for the last 5.14165 seconds: | 20260508 03:07:50.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:50.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.834 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:07:50.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.834 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:07:50.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:50.835 | INFO | Uploaded 36GiB to GPU in 2.55506s (14.0897GiB/s) | 20260508 03:07:50.835 | INFO | Downloaded 34GiB to GPU in 2.42756s (14.0058GiB/s) | 20260508 03:07:55.972 | INFO | Analyzing chunk frames = [4756480,4756992) | 20260508 03:07:55.972 | INFO | MemInfo: { | 20260508 03:07:55.972 | INFO | "gpu" : " 17 GB", | 20260508 03:07:55.972 | INFO | "huge" : "", | 20260508 03:07:55.972 | INFO | "huge_shared" : " 29 GB", | 20260508 03:07:55.972 | INFO | "malloc" : " 14 GB", | 20260508 03:07:55.972 | INFO | "pinned" : " 29 GB" | 20260508 03:07:55.972 | INFO | } | 20260508 03:07:55.972 | INFO | Perf report for the last 5.13807 seconds: | 20260508 03:07:55.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:07:55.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 03:07:55.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:07:55.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:07:55.972 | INFO | Uploaded 36GiB to GPU in 2.53478s (14.2024GiB/s) | 20260508 03:07:55.972 | INFO | Downloaded 34GiB to GPU in 2.49984s (13.6009GiB/s) | 20260508 03:08:01.107 | INFO | Analyzing chunk frames = [4756992,4757504) | 20260508 03:08:01.107 | INFO | MemInfo: { | 20260508 03:08:01.107 | INFO | "gpu" : " 17 GB", | 20260508 03:08:01.107 | INFO | "huge" : "", | 20260508 03:08:01.107 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:01.107 | INFO | "malloc" : " 14 GB", | 20260508 03:08:01.107 | INFO | "pinned" : " 29 GB" | 20260508 03:08:01.107 | INFO | } | 20260508 03:08:01.107 | INFO | Perf report for the last 5.13478 seconds: | 20260508 03:08:01.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:01.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.107 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 03:08:01.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.107 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:08:01.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:01.108 | INFO | Uploaded 36GiB to GPU in 2.5462s (14.1387GiB/s) | 20260508 03:08:01.108 | INFO | Downloaded 34GiB to GPU in 2.49718s (13.6154GiB/s) | 20260508 03:08:06.244 | INFO | Analyzing chunk frames = [4757504,4758016) | 20260508 03:08:06.244 | INFO | MemInfo: { | 20260508 03:08:06.244 | INFO | "gpu" : " 17 GB", | 20260508 03:08:06.244 | INFO | "huge" : "", | 20260508 03:08:06.244 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:06.244 | INFO | "malloc" : " 14 GB", | 20260508 03:08:06.245 | INFO | "pinned" : " 29 GB" | 20260508 03:08:06.245 | INFO | } | 20260508 03:08:06.245 | INFO | Perf report for the last 5.13764 seconds: | 20260508 03:08:06.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:06.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 03:08:06.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:08:06.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:06.245 | INFO | Uploaded 36GiB to GPU in 2.39004s (15.0625GiB/s) | 20260508 03:08:06.245 | INFO | Downloaded 34GiB to GPU in 2.54681s (13.35GiB/s) | 20260508 03:08:11.387 | INFO | Analyzing chunk frames = [4758016,4758528) | 20260508 03:08:11.387 | INFO | MemInfo: { | 20260508 03:08:11.387 | INFO | "gpu" : " 17 GB", | 20260508 03:08:11.387 | INFO | "huge" : "", | 20260508 03:08:11.387 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:11.387 | INFO | "malloc" : " 14 GB", | 20260508 03:08:11.387 | INFO | "pinned" : " 29 GB" | 20260508 03:08:11.387 | INFO | } | 20260508 03:08:11.387 | INFO | Perf report for the last 5.14222 seconds: | 20260508 03:08:11.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:11.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.387 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 03:08:11.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.387 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:08:11.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:11.388 | INFO | Uploaded 36GiB to GPU in 2.40937s (14.9417GiB/s) | 20260508 03:08:11.388 | INFO | Downloaded 34GiB to GPU in 2.41265s (14.0924GiB/s) | 20260508 03:08:16.536 | INFO | Analyzing chunk frames = [4758528,4759040) | 20260508 03:08:16.536 | INFO | MemInfo: { | 20260508 03:08:16.536 | INFO | "gpu" : " 17 GB", | 20260508 03:08:16.536 | INFO | "huge" : "", | 20260508 03:08:16.536 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:16.536 | INFO | "malloc" : " 14 GB", | 20260508 03:08:16.536 | INFO | "pinned" : " 29 GB" | 20260508 03:08:16.536 | INFO | } | 20260508 03:08:16.536 | INFO | Perf report for the last 5.14921 seconds: | 20260508 03:08:16.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:16.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | ANALYSIS : 66.3%, 3.0, 103.9ms, 0 | 20260508 03:08:16.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 03:08:16.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:16.537 | INFO | Uploaded 36GiB to GPU in 2.45639s (14.6556GiB/s) | 20260508 03:08:16.537 | INFO | Downloaded 34GiB to GPU in 2.43292s (13.975GiB/s) | 20260508 03:08:21.675 | INFO | Analyzing chunk frames = [4759040,4759552) | 20260508 03:08:21.675 | INFO | MemInfo: { | 20260508 03:08:21.675 | INFO | "gpu" : " 17 GB", | 20260508 03:08:21.675 | INFO | "huge" : "", | 20260508 03:08:21.675 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:21.675 | INFO | "malloc" : " 14 GB", | 20260508 03:08:21.675 | INFO | "pinned" : " 29 GB" | 20260508 03:08:21.675 | INFO | } | 20260508 03:08:21.675 | INFO | Perf report for the last 5.13903 seconds: | 20260508 03:08:21.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:21.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.675 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:08:21.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.676 | INFO | PRE_HQ : 35.3%, 1.0, 18.3ms, 0 | 20260508 03:08:21.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:21.676 | INFO | Uploaded 36GiB to GPU in 2.44933s (14.6979GiB/s) | 20260508 03:08:21.676 | INFO | Downloaded 34GiB to GPU in 2.42529s (14.019GiB/s) | 20260508 03:08:21.676 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:08:21.676 | INFO | Total mem usage: 16.7 GiB | 20260508 03:08:21.676 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:08:21.676 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16%) | 20260508 03:08:21.676 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:08:21.676 | INFO | High Water Breakdown: | 20260508 03:08:21.676 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:08:21.676 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:08:21.676 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:08:21.676 | INFO | ------------------------------------------ | 20260508 03:08:21.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:08:21.676 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:08:21.676 | INFO | Total mem usage: 26.5 GiB | 20260508 03:08:21.676 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:08:21.676 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:08:21.676 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:08:21.676 | INFO | High Water Breakdown: | 20260508 03:08:21.676 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:08:21.676 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:08:21.676 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:08:21.676 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:08:21.676 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:08:21.676 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:08:21.676 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:08:21.676 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:08:21.676 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:08:21.676 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:08:21.676 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:08:21.676 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:08:21.676 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:08:21.676 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:08:21.676 | INFO | ------------------------------------------ | 20260508 03:08:21.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:08:21.676 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:08:21.676 | INFO | Total mem usage: 27.8 GiB | 20260508 03:08:21.676 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:08:21.676 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:08:21.676 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:08:21.676 | INFO | High Water Breakdown: | 20260508 03:08:21.676 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:08:21.676 | INFO | ------------------------------------------ | 20260508 03:08:21.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:08:21.676 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:08:21.676 | INFO | Total mem usage: 15.7 GiB | 20260508 03:08:21.676 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:08:21.676 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:08:21.676 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:08:21.676 | INFO | High Water Breakdown: | 20260508 03:08:21.676 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:08:21.676 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:08:21.676 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:08:21.676 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:08:21.676 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:08:21.676 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:08:21.676 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:08:21.676 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:08:21.676 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:08:21.676 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:08:21.676 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:08:21.676 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:08:21.676 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:08:21.676 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:08:21.676 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:08:21.676 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:08:21.676 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:08:21.676 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:08:21.676 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:08:21.677 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:08:21.677 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:08:21.677 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:08:21.677 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:08:21.677 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:08:21.677 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:08:21.677 | INFO | ------------------------------------------ | 20260508 03:08:21.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:08:21.677 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:08:21.677 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:08:21.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:08:21.677 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.62e+01, 1.40e+01 / 1.90e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.02e-01, 1.28e-02 / 5.41e-03 | 20260508 03:08:21.677 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.40e+00, 5.87e-01 / 2.15e-01 | 20260508 03:08:21.677 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.16e-01 | 20260508 03:08:21.677 | INFO | PerfReport: 16.01%, DME -- 4.20e+01 / 4.60e+01, 4.17e+00 / 2.24e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.75e+00 / 1.42e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.24e-01 / 2.07e-01 | 20260508 03:08:21.677 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.24e+00, 2.67e-01 / 1.56e-01 | 20260508 03:08:21.677 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.60e-01, 5.27e-02 / 2.40e-02 | 20260508 03:08:21.677 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:08:21.677 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:08:21.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:08:21.677 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 2.10%, LockUpload -- 2.24e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 2.76%, LockDownload -- 2.94e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 03:08:21.677 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260508 03:08:21.788 | INFO | Starting a round of Baz IO | 20260508 03:08:25.209 | INFO | Populating RTMetrics report | 20260508 03:08:25.210 | INFO | RTMetrics report populated | 20260508 03:08:25.210 | INFO | Opening tmp RTMetrics json file | 20260508 03:08:25.224 | INFO | Serializing RTMetrics to json | 20260508 03:08:25.226 | INFO | Serialization done, swapping tmp file now... | 20260508 03:08:25.226 | INFO | RTMetrics done | 20260508 03:08:25.277 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.489 seconds | 20260508 03:08:25.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.36% / 0.00% / 58.64% | 20260508 03:08:25.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.646639% / 3.97364e-06% / 6.35783e-05% | 20260508 03:08:25.278 | INFO | Baz IO overall: | 20260508 03:08:25.278 | INFO | BytesWritten : 2142981794705 | 20260508 03:08:25.278 | INFO | NumEvents : 1035526065937 | 20260508 03:08:26.808 | INFO | Analyzing chunk frames = [4759552,4760064) | 20260508 03:08:26.808 | INFO | MemInfo: { | 20260508 03:08:26.808 | INFO | "gpu" : " 17 GB", | 20260508 03:08:26.808 | INFO | "huge" : "", | 20260508 03:08:26.808 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:26.808 | INFO | "malloc" : " 14 GB", | 20260508 03:08:26.808 | INFO | "pinned" : " 29 GB" | 20260508 03:08:26.808 | INFO | } | 20260508 03:08:26.808 | INFO | Perf report for the last 5.13296 seconds: | 20260508 03:08:26.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:26.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:26.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:26.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:26.808 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260508 03:08:26.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:26.808 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:08:26.808 | INFO | PRE_HQ : 66.7%, 1.8, 62.8ms, 0 | 20260508 03:08:26.808 | INFO | BAZWRITER : 67.5%, 2.8, 96.4ms, 0 | 20260508 03:08:26.808 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:08:26.809 | INFO | Uploaded 36GiB to GPU in 2.53046s (14.2266GiB/s) | 20260508 03:08:26.809 | INFO | Downloaded 38GiB to GPU in 2.69751s (14.0871GiB/s) | 20260508 03:08:31.943 | INFO | Analyzing chunk frames = [4760064,4760576) | 20260508 03:08:31.943 | INFO | MemInfo: { | 20260508 03:08:31.943 | INFO | "gpu" : " 17 GB", | 20260508 03:08:31.943 | INFO | "huge" : "", | 20260508 03:08:31.943 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:31.943 | INFO | "malloc" : " 14 GB", | 20260508 03:08:31.943 | INFO | "pinned" : " 29 GB" | 20260508 03:08:31.943 | INFO | } | 20260508 03:08:31.943 | INFO | Perf report for the last 5.13506 seconds: | 20260508 03:08:31.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:31.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.943 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:08:31.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.943 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:08:31.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:31.944 | INFO | Uploaded 36GiB to GPU in 2.56173s (14.053GiB/s) | 20260508 03:08:31.944 | INFO | Downloaded 34GiB to GPU in 2.44633s (13.8984GiB/s) | 20260508 03:08:37.086 | INFO | Analyzing chunk frames = [4760576,4761088) | 20260508 03:08:37.086 | INFO | MemInfo: { | 20260508 03:08:37.086 | INFO | "gpu" : " 17 GB", | 20260508 03:08:37.086 | INFO | "huge" : "", | 20260508 03:08:37.086 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:37.086 | INFO | "malloc" : " 14 GB", | 20260508 03:08:37.086 | INFO | "pinned" : " 29 GB" | 20260508 03:08:37.086 | INFO | } | 20260508 03:08:37.086 | INFO | Perf report for the last 5.14296 seconds: | 20260508 03:08:37.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:37.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.086 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:08:37.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.086 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:08:37.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:37.087 | INFO | Uploaded 36GiB to GPU in 2.51743s (14.3003GiB/s) | 20260508 03:08:37.087 | INFO | Downloaded 34GiB to GPU in 2.51332s (13.5279GiB/s) | 20260508 03:08:42.228 | INFO | Analyzing chunk frames = [4761088,4761600) | 20260508 03:08:42.228 | INFO | MemInfo: { | 20260508 03:08:42.228 | INFO | "gpu" : " 17 GB", | 20260508 03:08:42.228 | INFO | "huge" : "", | 20260508 03:08:42.228 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:42.228 | INFO | "malloc" : " 14 GB", | 20260508 03:08:42.228 | INFO | "pinned" : " 29 GB" | 20260508 03:08:42.228 | INFO | } | 20260508 03:08:42.228 | INFO | Perf report for the last 5.142 seconds: | 20260508 03:08:42.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:42.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.228 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 03:08:42.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.228 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:08:42.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:42.229 | INFO | Uploaded 36GiB to GPU in 2.51537s (14.312GiB/s) | 20260508 03:08:42.229 | INFO | Downloaded 34GiB to GPU in 2.51836s (13.5009GiB/s) | 20260508 03:08:47.369 | INFO | Analyzing chunk frames = [4761600,4762112) | 20260508 03:08:47.369 | INFO | MemInfo: { | 20260508 03:08:47.369 | INFO | "gpu" : " 17 GB", | 20260508 03:08:47.369 | INFO | "huge" : "", | 20260508 03:08:47.369 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:47.369 | INFO | "malloc" : " 14 GB", | 20260508 03:08:47.369 | INFO | "pinned" : " 29 GB" | 20260508 03:08:47.369 | INFO | } | 20260508 03:08:47.369 | INFO | Perf report for the last 5.14119 seconds: | 20260508 03:08:47.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:47.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.370 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 03:08:47.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.370 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:08:47.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:47.370 | INFO | Uploaded 36GiB to GPU in 2.36903s (15.1961GiB/s) | 20260508 03:08:47.370 | INFO | Downloaded 34GiB to GPU in 2.54615s (13.3535GiB/s) | 20260508 03:08:52.506 | INFO | Analyzing chunk frames = [4762112,4762624) | 20260508 03:08:52.506 | INFO | MemInfo: { | 20260508 03:08:52.506 | INFO | "gpu" : " 17 GB", | 20260508 03:08:52.506 | INFO | "huge" : "", | 20260508 03:08:52.506 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:52.506 | INFO | "malloc" : " 14 GB", | 20260508 03:08:52.506 | INFO | "pinned" : " 29 GB" | 20260508 03:08:52.506 | INFO | } | 20260508 03:08:52.506 | INFO | Perf report for the last 5.13644 seconds: | 20260508 03:08:52.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:52.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 03:08:52.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:08:52.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:52.506 | INFO | Uploaded 36GiB to GPU in 2.37752s (15.1419GiB/s) | 20260508 03:08:52.506 | INFO | Downloaded 34GiB to GPU in 2.4434s (13.915GiB/s) | 20260508 03:08:57.645 | INFO | Analyzing chunk frames = [4762624,4763136) | 20260508 03:08:57.645 | INFO | MemInfo: { | 20260508 03:08:57.645 | INFO | "gpu" : " 17 GB", | 20260508 03:08:57.645 | INFO | "huge" : "", | 20260508 03:08:57.645 | INFO | "huge_shared" : " 29 GB", | 20260508 03:08:57.646 | INFO | "malloc" : " 14 GB", | 20260508 03:08:57.646 | INFO | "pinned" : " 29 GB" | 20260508 03:08:57.646 | INFO | } | 20260508 03:08:57.646 | INFO | Perf report for the last 5.13969 seconds: | 20260508 03:08:57.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:08:57.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:08:57.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 03:08:57.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:08:57.646 | INFO | Uploaded 36GiB to GPU in 2.4106s (14.934GiB/s) | 20260508 03:08:57.646 | INFO | Downloaded 34GiB to GPU in 2.43635s (13.9553GiB/s) | 20260508 03:09:02.783 | INFO | Analyzing chunk frames = [4763136,4763648) | 20260508 03:09:02.783 | INFO | MemInfo: { | 20260508 03:09:02.783 | INFO | "gpu" : " 17 GB", | 20260508 03:09:02.783 | INFO | "huge" : "", | 20260508 03:09:02.783 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:02.783 | INFO | "malloc" : " 14 GB", | 20260508 03:09:02.783 | INFO | "pinned" : " 29 GB" | 20260508 03:09:02.783 | INFO | } | 20260508 03:09:02.784 | INFO | Perf report for the last 5.1378 seconds: | 20260508 03:09:02.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:02.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:09:02.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:09:02.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:02.784 | INFO | Uploaded 36GiB to GPU in 2.42691s (14.8337GiB/s) | 20260508 03:09:02.784 | INFO | Downloaded 34GiB to GPU in 2.45021s (13.8764GiB/s) | 20260508 03:09:02.899 | INFO | Starting a round of Baz IO | 20260508 03:09:06.320 | INFO | Populating RTMetrics report | 20260508 03:09:06.320 | INFO | RTMetrics report populated | 20260508 03:09:06.320 | INFO | Opening tmp RTMetrics json file | 20260508 03:09:06.330 | INFO | Serializing RTMetrics to json | 20260508 03:09:06.333 | INFO | Serialization done, swapping tmp file now... | 20260508 03:09:06.334 | INFO | RTMetrics done | 20260508 03:09:06.394 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.494 seconds | 20260508 03:09:06.394 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.36% / 0.00% / 58.64% | 20260508 03:09:06.394 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.64617% / 3.97364e-06% / 5.96046e-05% | 20260508 03:09:06.394 | INFO | Baz IO overall: | 20260508 03:09:06.394 | INFO | BytesWritten : 2144723263267 | 20260508 03:09:06.394 | INFO | NumEvents : 1036360108591 | 20260508 03:09:07.927 | INFO | Analyzing chunk frames = [4763648,4764160) | 20260508 03:09:07.927 | INFO | MemInfo: { | 20260508 03:09:07.927 | INFO | "gpu" : " 17 GB", | 20260508 03:09:07.927 | INFO | "huge" : "", | 20260508 03:09:07.927 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:07.927 | INFO | "malloc" : " 14 GB", | 20260508 03:09:07.927 | INFO | "pinned" : " 29 GB" | 20260508 03:09:07.927 | INFO | } | 20260508 03:09:07.927 | INFO | Perf report for the last 5.1438 seconds: | 20260508 03:09:07.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:07.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:07.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:07.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:07.927 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 03:09:07.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:07.927 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 03:09:07.927 | INFO | PRE_HQ : 66.5%, 1.8, 62.5ms, 0 | 20260508 03:09:07.927 | INFO | BAZWRITER : 67.5%, 2.8, 98.3ms, 0 | 20260508 03:09:07.927 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:09:07.928 | INFO | Uploaded 36GiB to GPU in 2.53501s (14.2011GiB/s) | 20260508 03:09:07.928 | INFO | Downloaded 38GiB to GPU in 2.69709s (14.0892GiB/s) | 20260508 03:09:13.060 | INFO | Analyzing chunk frames = [4764160,4764672) | 20260508 03:09:13.060 | INFO | MemInfo: { | 20260508 03:09:13.060 | INFO | "gpu" : " 17 GB", | 20260508 03:09:13.060 | INFO | "huge" : "", | 20260508 03:09:13.060 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:13.060 | INFO | "malloc" : " 14 GB", | 20260508 03:09:13.060 | INFO | "pinned" : " 29 GB" | 20260508 03:09:13.060 | INFO | } | 20260508 03:09:13.060 | INFO | Perf report for the last 5.13265 seconds: | 20260508 03:09:13.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:13.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:09:13.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:09:13.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:13.060 | INFO | Uploaded 36GiB to GPU in 2.56689s (14.0248GiB/s) | 20260508 03:09:13.060 | INFO | Downloaded 34GiB to GPU in 2.42259s (14.0346GiB/s) | 20260508 03:09:18.195 | INFO | Analyzing chunk frames = [4764672,4765184) | 20260508 03:09:18.195 | INFO | MemInfo: { | 20260508 03:09:18.195 | INFO | "gpu" : " 17 GB", | 20260508 03:09:18.195 | INFO | "huge" : "", | 20260508 03:09:18.195 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:18.196 | INFO | "malloc" : " 14 GB", | 20260508 03:09:18.196 | INFO | "pinned" : " 29 GB" | 20260508 03:09:18.196 | INFO | } | 20260508 03:09:18.196 | INFO | Perf report for the last 5.13563 seconds: | 20260508 03:09:18.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:18.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:09:18.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:09:18.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:18.196 | INFO | Uploaded 36GiB to GPU in 2.53985s (14.1741GiB/s) | 20260508 03:09:18.196 | INFO | Downloaded 34GiB to GPU in 2.50153s (13.5917GiB/s) | 20260508 03:09:23.340 | INFO | Analyzing chunk frames = [4765184,4765696) | 20260508 03:09:23.340 | INFO | MemInfo: { | 20260508 03:09:23.340 | INFO | "gpu" : " 17 GB", | 20260508 03:09:23.340 | INFO | "huge" : "", | 20260508 03:09:23.340 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:23.340 | INFO | "malloc" : " 14 GB", | 20260508 03:09:23.340 | INFO | "pinned" : " 29 GB" | 20260508 03:09:23.340 | INFO | } | 20260508 03:09:23.340 | INFO | Perf report for the last 5.14479 seconds: | 20260508 03:09:23.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:23.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.340 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:09:23.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.341 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:09:23.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:23.341 | INFO | Uploaded 36GiB to GPU in 2.54756s (14.1312GiB/s) | 20260508 03:09:23.341 | INFO | Downloaded 34GiB to GPU in 2.49943s (13.6031GiB/s) | 20260508 03:09:28.485 | INFO | Analyzing chunk frames = [4765696,4766208) | 20260508 03:09:28.485 | INFO | MemInfo: { | 20260508 03:09:28.485 | INFO | "gpu" : " 17 GB", | 20260508 03:09:28.485 | INFO | "huge" : "", | 20260508 03:09:28.485 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:28.485 | INFO | "malloc" : " 14 GB", | 20260508 03:09:28.485 | INFO | "pinned" : " 29 GB" | 20260508 03:09:28.485 | INFO | } | 20260508 03:09:28.485 | INFO | Perf report for the last 5.14457 seconds: | 20260508 03:09:28.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:28.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 03:09:28.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 03:09:28.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:28.485 | INFO | Uploaded 36GiB to GPU in 2.44272s (14.7377GiB/s) | 20260508 03:09:28.485 | INFO | Downloaded 34GiB to GPU in 2.55451s (13.3098GiB/s) | 20260508 03:09:33.625 | INFO | Analyzing chunk frames = [4766208,4766720) | 20260508 03:09:33.625 | INFO | MemInfo: { | 20260508 03:09:33.625 | INFO | "gpu" : " 17 GB", | 20260508 03:09:33.625 | INFO | "huge" : "", | 20260508 03:09:33.625 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:33.625 | INFO | "malloc" : " 14 GB", | 20260508 03:09:33.625 | INFO | "pinned" : " 29 GB" | 20260508 03:09:33.625 | INFO | } | 20260508 03:09:33.625 | INFO | Perf report for the last 5.14045 seconds: | 20260508 03:09:33.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:33.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.626 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 03:09:33.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.626 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:09:33.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:33.626 | INFO | Uploaded 36GiB to GPU in 2.42188s (14.8645GiB/s) | 20260508 03:09:33.626 | INFO | Downloaded 34GiB to GPU in 2.41976s (14.051GiB/s) | 20260508 03:09:38.760 | INFO | Analyzing chunk frames = [4766720,4767232) | 20260508 03:09:38.760 | INFO | MemInfo: { | 20260508 03:09:38.760 | INFO | "gpu" : " 17 GB", | 20260508 03:09:38.760 | INFO | "huge" : "", | 20260508 03:09:38.760 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:38.760 | INFO | "malloc" : " 14 GB", | 20260508 03:09:38.760 | INFO | "pinned" : " 29 GB" | 20260508 03:09:38.760 | INFO | } | 20260508 03:09:38.760 | INFO | Perf report for the last 5.13425 seconds: | 20260508 03:09:38.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:38.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:09:38.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:09:38.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:38.760 | INFO | Uploaded 36GiB to GPU in 2.42781s (14.8281GiB/s) | 20260508 03:09:38.760 | INFO | Downloaded 34GiB to GPU in 2.41785s (14.0621GiB/s) | 20260508 03:09:43.898 | INFO | Analyzing chunk frames = [4767232,4767744) | 20260508 03:09:43.898 | INFO | MemInfo: { | 20260508 03:09:43.898 | INFO | "gpu" : " 17 GB", | 20260508 03:09:43.898 | INFO | "huge" : "", | 20260508 03:09:43.898 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:43.898 | INFO | "malloc" : " 14 GB", | 20260508 03:09:43.899 | INFO | "pinned" : " 29 GB" | 20260508 03:09:43.899 | INFO | } | 20260508 03:09:43.899 | INFO | Perf report for the last 5.13887 seconds: | 20260508 03:09:43.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:43.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:09:43.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:09:43.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:43.899 | INFO | Uploaded 36GiB to GPU in 2.4415s (14.745GiB/s) | 20260508 03:09:43.899 | INFO | Downloaded 34GiB to GPU in 2.42749s (14.0063GiB/s) | 20260508 03:09:43.899 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:09:43.899 | INFO | Total mem usage: 16.7 GiB | 20260508 03:09:43.899 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:09:43.899 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:09:43.899 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:09:43.899 | INFO | High Water Breakdown: | 20260508 03:09:43.899 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:09:43.899 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:09:43.899 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:09:43.899 | INFO | ------------------------------------------ | 20260508 03:09:43.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:09:43.899 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:09:43.899 | INFO | Total mem usage: 26.5 GiB | 20260508 03:09:43.899 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:09:43.899 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 03:09:43.899 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 03:09:43.899 | INFO | High Water Breakdown: | 20260508 03:09:43.899 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:09:43.899 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:09:43.899 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:09:43.899 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:09:43.899 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:09:43.899 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:09:43.899 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:09:43.899 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:09:43.899 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:09:43.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:09:43.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:09:43.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:09:43.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:09:43.899 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:09:43.899 | INFO | ------------------------------------------ | 20260508 03:09:43.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:09:43.899 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:09:43.899 | INFO | Total mem usage: 27.8 GiB | 20260508 03:09:43.899 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:09:43.899 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:09:43.900 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:09:43.900 | INFO | High Water Breakdown: | 20260508 03:09:43.900 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:09:43.900 | INFO | ------------------------------------------ | 20260508 03:09:43.900 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:09:43.900 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:09:43.900 | INFO | Total mem usage: 15.7 GiB | 20260508 03:09:43.900 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:09:43.900 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:09:43.900 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:09:43.900 | INFO | High Water Breakdown: | 20260508 03:09:43.900 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:09:43.900 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:09:43.900 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:09:43.900 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:09:43.900 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:09:43.900 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:09:43.900 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:09:43.900 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:09:43.900 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:09:43.900 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:09:43.900 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:09:43.900 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:09:43.900 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:09:43.900 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:09:43.900 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:09:43.900 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:09:43.900 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:09:43.900 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:09:43.900 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:09:43.900 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:09:43.900 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:09:43.900 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:09:43.900 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:09:43.900 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:09:43.900 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:09:43.900 | INFO | ------------------------------------------ | 20260508 03:09:43.900 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:09:43.900 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:09:43.900 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:09:43.900 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:09:43.900 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.60e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.04e-01, 1.15e-02 / 7.13e-03 | 20260508 03:09:43.900 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.42e+00, 5.82e-01 / 2.39e-01 | 20260508 03:09:43.900 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.06e-01 / 1.15e-01 | 20260508 03:09:43.900 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.59e+01, 4.04e+00 / 2.01e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.65e+00 / 1.49e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.18e-01 / 2.29e-01 | 20260508 03:09:43.900 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.25e+00, 2.65e-01 / 1.54e-01 | 20260508 03:09:43.900 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.68e-01, 5.15e-02 / 2.49e-02 | 20260508 03:09:43.900 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:09:43.900 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:09:43.900 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:09:43.900 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 24.27%, Upload -- 2.58e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 24.49%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 1.99%, LockUpload -- 2.12e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 2.96%, LockDownload -- 3.14e+00 / 0.00e+00, 4.45e+00 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:09:43.900 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.60e-02 / 0.00e+00 | 20260508 03:09:44.011 | INFO | Starting a round of Baz IO | 20260508 03:09:47.427 | INFO | Populating RTMetrics report | 20260508 03:09:47.427 | INFO | RTMetrics report populated | 20260508 03:09:47.427 | INFO | Opening tmp RTMetrics json file | 20260508 03:09:47.440 | INFO | Serializing RTMetrics to json | 20260508 03:09:47.442 | INFO | Serialization done, swapping tmp file now... | 20260508 03:09:47.442 | INFO | RTMetrics done | 20260508 03:09:47.501 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.489 seconds | 20260508 03:09:47.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.35% / 0.00% / 58.64% | 20260508 03:09:47.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.646138% / 1.19209e-05% / 5.96046e-05% | 20260508 03:09:47.501 | INFO | Baz IO overall: | 20260508 03:09:47.501 | INFO | BytesWritten : 2146464921361 | 20260508 03:09:47.501 | INFO | NumEvents : 1037194306760 | 20260508 03:09:49.036 | INFO | Analyzing chunk frames = [4767744,4768256) | 20260508 03:09:49.036 | INFO | MemInfo: { | 20260508 03:09:49.036 | INFO | "gpu" : " 17 GB", | 20260508 03:09:49.036 | INFO | "huge" : "", | 20260508 03:09:49.036 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:49.036 | INFO | "malloc" : " 14 GB", | 20260508 03:09:49.036 | INFO | "pinned" : " 29 GB" | 20260508 03:09:49.036 | INFO | } | 20260508 03:09:49.036 | INFO | Perf report for the last 5.13753 seconds: | 20260508 03:09:49.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:49.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:49.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:49.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:49.036 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 03:09:49.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:49.036 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:09:49.036 | INFO | PRE_HQ : 66.6%, 1.8, 62.7ms, 0 | 20260508 03:09:49.036 | INFO | BAZWRITER : 67.2%, 2.8, 97.8ms, 0 | 20260508 03:09:49.036 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:09:49.037 | INFO | Uploaded 36GiB to GPU in 2.55119s (14.1111GiB/s) | 20260508 03:09:49.037 | INFO | Downloaded 38GiB to GPU in 2.68456s (14.155GiB/s) | 20260508 03:09:54.173 | INFO | Analyzing chunk frames = [4768256,4768768) | 20260508 03:09:54.173 | INFO | MemInfo: { | 20260508 03:09:54.173 | INFO | "gpu" : " 17 GB", | 20260508 03:09:54.173 | INFO | "huge" : "", | 20260508 03:09:54.173 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:54.173 | INFO | "malloc" : " 14 GB", | 20260508 03:09:54.173 | INFO | "pinned" : " 29 GB" | 20260508 03:09:54.173 | INFO | } | 20260508 03:09:54.173 | INFO | Perf report for the last 5.13725 seconds: | 20260508 03:09:54.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:54.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.174 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:09:54.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.174 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:09:54.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:54.174 | INFO | Uploaded 36GiB to GPU in 2.56887s (14.014GiB/s) | 20260508 03:09:54.174 | INFO | Downloaded 34GiB to GPU in 2.43204s (13.98GiB/s) | 20260508 03:09:59.315 | INFO | Analyzing chunk frames = [4768768,4769280) | 20260508 03:09:59.315 | INFO | MemInfo: { | 20260508 03:09:59.315 | INFO | "gpu" : " 17 GB", | 20260508 03:09:59.315 | INFO | "huge" : "", | 20260508 03:09:59.315 | INFO | "huge_shared" : " 29 GB", | 20260508 03:09:59.315 | INFO | "malloc" : " 14 GB", | 20260508 03:09:59.315 | INFO | "pinned" : " 29 GB" | 20260508 03:09:59.315 | INFO | } | 20260508 03:09:59.316 | INFO | Perf report for the last 5.14212 seconds: | 20260508 03:09:59.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:09:59.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:09:59.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:09:59.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:09:59.316 | INFO | Uploaded 36GiB to GPU in 2.5141s (14.3193GiB/s) | 20260508 03:09:59.316 | INFO | Downloaded 34GiB to GPU in 2.48483s (13.683GiB/s) | 20260508 03:10:04.451 | INFO | Analyzing chunk frames = [4769280,4769792) | 20260508 03:10:04.451 | INFO | MemInfo: { | 20260508 03:10:04.451 | INFO | "gpu" : " 17 GB", | 20260508 03:10:04.451 | INFO | "huge" : "", | 20260508 03:10:04.451 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:04.451 | INFO | "malloc" : " 14 GB", | 20260508 03:10:04.451 | INFO | "pinned" : " 29 GB" | 20260508 03:10:04.451 | INFO | } | 20260508 03:10:04.451 | INFO | Perf report for the last 5.13533 seconds: | 20260508 03:10:04.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:04.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 03:10:04.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:10:04.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:04.451 | INFO | Uploaded 36GiB to GPU in 2.53882s (14.1798GiB/s) | 20260508 03:10:04.451 | INFO | Downloaded 34GiB to GPU in 2.49172s (13.6452GiB/s) | 20260508 03:10:09.596 | INFO | Analyzing chunk frames = [4769792,4770304) | 20260508 03:10:09.596 | INFO | MemInfo: { | 20260508 03:10:09.596 | INFO | "gpu" : " 17 GB", | 20260508 03:10:09.596 | INFO | "huge" : "", | 20260508 03:10:09.596 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:09.596 | INFO | "malloc" : " 14 GB", | 20260508 03:10:09.596 | INFO | "pinned" : " 29 GB" | 20260508 03:10:09.596 | INFO | } | 20260508 03:10:09.596 | INFO | Perf report for the last 5.14559 seconds: | 20260508 03:10:09.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:09.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.597 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 03:10:09.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.597 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:10:09.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:09.597 | INFO | Uploaded 36GiB to GPU in 2.36176s (15.2429GiB/s) | 20260508 03:10:09.598 | INFO | Downloaded 34GiB to GPU in 2.52152s (13.4839GiB/s) | 20260508 03:10:14.736 | INFO | Analyzing chunk frames = [4770304,4770816) | 20260508 03:10:14.736 | INFO | MemInfo: { | 20260508 03:10:14.736 | INFO | "gpu" : " 17 GB", | 20260508 03:10:14.736 | INFO | "huge" : "", | 20260508 03:10:14.736 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:14.736 | INFO | "malloc" : " 14 GB", | 20260508 03:10:14.736 | INFO | "pinned" : " 29 GB" | 20260508 03:10:14.736 | INFO | } | 20260508 03:10:14.736 | INFO | Perf report for the last 5.13954 seconds: | 20260508 03:10:14.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:14.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:10:14.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:10:14.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:14.736 | INFO | Uploaded 36GiB to GPU in 2.43852s (14.763GiB/s) | 20260508 03:10:14.736 | INFO | Downloaded 34GiB to GPU in 2.41133s (14.1001GiB/s) | 20260508 03:10:19.872 | INFO | Analyzing chunk frames = [4770816,4771328) | 20260508 03:10:19.872 | INFO | MemInfo: { | 20260508 03:10:19.872 | INFO | "gpu" : " 17 GB", | 20260508 03:10:19.872 | INFO | "huge" : "", | 20260508 03:10:19.872 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:19.872 | INFO | "malloc" : " 14 GB", | 20260508 03:10:19.872 | INFO | "pinned" : " 29 GB" | 20260508 03:10:19.872 | INFO | } | 20260508 03:10:19.872 | INFO | Perf report for the last 5.13566 seconds: | 20260508 03:10:19.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:19.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:10:19.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 03:10:19.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:19.872 | INFO | Uploaded 36GiB to GPU in 2.49341s (14.4381GiB/s) | 20260508 03:10:19.872 | INFO | Downloaded 34GiB to GPU in 2.42723s (14.0077GiB/s) | 20260508 03:10:25.007 | INFO | Analyzing chunk frames = [4771328,4771840) | 20260508 03:10:25.007 | INFO | MemInfo: { | 20260508 03:10:25.007 | INFO | "gpu" : " 17 GB", | 20260508 03:10:25.007 | INFO | "huge" : "", | 20260508 03:10:25.007 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:25.007 | INFO | "malloc" : " 14 GB", | 20260508 03:10:25.008 | INFO | "pinned" : " 29 GB" | 20260508 03:10:25.008 | INFO | } | 20260508 03:10:25.008 | INFO | Perf report for the last 5.13604 seconds: | 20260508 03:10:25.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:25.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:10:25.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 03:10:25.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:25.008 | INFO | Uploaded 36GiB to GPU in 2.57514s (13.9798GiB/s) | 20260508 03:10:25.008 | INFO | Downloaded 34GiB to GPU in 2.45761s (13.8346GiB/s) | 20260508 03:10:25.121 | INFO | Starting a round of Baz IO | 20260508 03:10:28.557 | INFO | Populating RTMetrics report | 20260508 03:10:28.557 | INFO | RTMetrics report populated | 20260508 03:10:28.557 | INFO | Opening tmp RTMetrics json file | 20260508 03:10:28.567 | INFO | Serializing RTMetrics to json | 20260508 03:10:28.569 | INFO | Serialization done, swapping tmp file now... | 20260508 03:10:28.569 | INFO | RTMetrics done | 20260508 03:10:28.627 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.505 seconds | 20260508 03:10:28.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.35% / 0.00% / 58.65% | 20260508 03:10:28.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.643806% / 1.58946e-05% / 7.15256e-05% | 20260508 03:10:28.627 | INFO | Baz IO overall: | 20260508 03:10:28.627 | INFO | BytesWritten : 2148205483180 | 20260508 03:10:28.627 | INFO | NumEvents : 1038027686484 | 20260508 03:10:30.149 | INFO | Analyzing chunk frames = [4771840,4772352) | 20260508 03:10:30.149 | INFO | MemInfo: { | 20260508 03:10:30.149 | INFO | "gpu" : " 17 GB", | 20260508 03:10:30.149 | INFO | "huge" : "", | 20260508 03:10:30.149 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:30.149 | INFO | "malloc" : " 14 GB", | 20260508 03:10:30.149 | INFO | "pinned" : " 29 GB" | 20260508 03:10:30.149 | INFO | } | 20260508 03:10:30.149 | INFO | Perf report for the last 5.14168 seconds: | 20260508 03:10:30.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:30.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:30.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:30.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:30.150 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 03:10:30.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:30.150 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260508 03:10:30.150 | INFO | PRE_HQ : 66.1%, 1.8, 61.9ms, 0 | 20260508 03:10:30.150 | INFO | BAZWRITER : 67.5%, 2.8, 97.6ms, 0 | 20260508 03:10:30.150 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:10:30.150 | INFO | Uploaded 36GiB to GPU in 2.56891s (14.0137GiB/s) | 20260508 03:10:30.150 | INFO | Downloaded 38GiB to GPU in 2.69454s (14.1026GiB/s) | 20260508 03:10:35.297 | INFO | Analyzing chunk frames = [4772352,4772864) | 20260508 03:10:35.297 | INFO | MemInfo: { | 20260508 03:10:35.297 | INFO | "gpu" : " 17 GB", | 20260508 03:10:35.297 | INFO | "huge" : "", | 20260508 03:10:35.297 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:35.297 | INFO | "malloc" : " 14 GB", | 20260508 03:10:35.297 | INFO | "pinned" : " 29 GB" | 20260508 03:10:35.297 | INFO | } | 20260508 03:10:35.297 | INFO | Perf report for the last 5.14767 seconds: | 20260508 03:10:35.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:35.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 03:10:35.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:10:35.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:35.297 | INFO | Uploaded 36GiB to GPU in 2.55511s (14.0894GiB/s) | 20260508 03:10:35.298 | INFO | Downloaded 34GiB to GPU in 2.43333s (13.9726GiB/s) | 20260508 03:10:40.435 | INFO | Analyzing chunk frames = [4772864,4773376) | 20260508 03:10:40.435 | INFO | MemInfo: { | 20260508 03:10:40.435 | INFO | "gpu" : " 17 GB", | 20260508 03:10:40.435 | INFO | "huge" : "", | 20260508 03:10:40.435 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:40.435 | INFO | "malloc" : " 14 GB", | 20260508 03:10:40.435 | INFO | "pinned" : " 29 GB" | 20260508 03:10:40.436 | INFO | } | 20260508 03:10:40.436 | INFO | Perf report for the last 5.13851 seconds: | 20260508 03:10:40.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:40.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 03:10:40.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:10:40.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:40.436 | INFO | Uploaded 36GiB to GPU in 2.54964s (14.1196GiB/s) | 20260508 03:10:40.436 | INFO | Downloaded 34GiB to GPU in 2.51019s (13.5448GiB/s) | 20260508 03:10:45.573 | INFO | Analyzing chunk frames = [4773376,4773888) | 20260508 03:10:45.573 | INFO | MemInfo: { | 20260508 03:10:45.573 | INFO | "gpu" : " 17 GB", | 20260508 03:10:45.573 | INFO | "huge" : "", | 20260508 03:10:45.573 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:45.573 | INFO | "malloc" : " 14 GB", | 20260508 03:10:45.573 | INFO | "pinned" : " 29 GB" | 20260508 03:10:45.573 | INFO | } | 20260508 03:10:45.573 | INFO | Perf report for the last 5.1377 seconds: | 20260508 03:10:45.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:45.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.573 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 03:10:45.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.574 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:10:45.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:45.574 | INFO | Uploaded 36GiB to GPU in 2.53632s (14.1938GiB/s) | 20260508 03:10:45.574 | INFO | Downloaded 34GiB to GPU in 2.51318s (13.5287GiB/s) | 20260508 03:10:50.706 | INFO | Analyzing chunk frames = [4773888,4774400) | 20260508 03:10:50.706 | INFO | MemInfo: { | 20260508 03:10:50.706 | INFO | "gpu" : " 17 GB", | 20260508 03:10:50.706 | INFO | "huge" : "", | 20260508 03:10:50.706 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:50.706 | INFO | "malloc" : " 14 GB", | 20260508 03:10:50.706 | INFO | "pinned" : " 29 GB" | 20260508 03:10:50.706 | INFO | } | 20260508 03:10:50.706 | INFO | Perf report for the last 5.1327 seconds: | 20260508 03:10:50.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:50.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 03:10:50.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:10:50.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:50.706 | INFO | Uploaded 36GiB to GPU in 2.35826s (15.2655GiB/s) | 20260508 03:10:50.706 | INFO | Downloaded 34GiB to GPU in 2.55994s (13.2816GiB/s) | 20260508 03:10:55.847 | INFO | Analyzing chunk frames = [4774400,4774912) | 20260508 03:10:55.847 | INFO | MemInfo: { | 20260508 03:10:55.847 | INFO | "gpu" : " 17 GB", | 20260508 03:10:55.847 | INFO | "huge" : "", | 20260508 03:10:55.847 | INFO | "huge_shared" : " 29 GB", | 20260508 03:10:55.847 | INFO | "malloc" : " 14 GB", | 20260508 03:10:55.848 | INFO | "pinned" : " 29 GB" | 20260508 03:10:55.848 | INFO | } | 20260508 03:10:55.848 | INFO | Perf report for the last 5.14158 seconds: | 20260508 03:10:55.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:10:55.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:10:55.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:10:55.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:10:55.848 | INFO | Uploaded 36GiB to GPU in 2.41071s (14.9333GiB/s) | 20260508 03:10:55.848 | INFO | Downloaded 34GiB to GPU in 2.42483s (14.0216GiB/s) | 20260508 03:11:00.987 | INFO | Analyzing chunk frames = [4774912,4775424) | 20260508 03:11:00.987 | INFO | MemInfo: { | 20260508 03:11:00.987 | INFO | "gpu" : " 17 GB", | 20260508 03:11:00.987 | INFO | "huge" : "", | 20260508 03:11:00.987 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:00.987 | INFO | "malloc" : " 14 GB", | 20260508 03:11:00.987 | INFO | "pinned" : " 29 GB" | 20260508 03:11:00.987 | INFO | } | 20260508 03:11:00.987 | INFO | Perf report for the last 5.13957 seconds: | 20260508 03:11:00.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:00.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.987 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:11:00.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.987 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:11:00.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:00.988 | INFO | Uploaded 36GiB to GPU in 2.42222s (14.8624GiB/s) | 20260508 03:11:00.988 | INFO | Downloaded 34GiB to GPU in 2.43008s (13.9913GiB/s) | 20260508 03:11:06.128 | INFO | Analyzing chunk frames = [4775424,4775936) | 20260508 03:11:06.128 | INFO | MemInfo: { | 20260508 03:11:06.128 | INFO | "gpu" : " 17 GB", | 20260508 03:11:06.128 | INFO | "huge" : "", | 20260508 03:11:06.128 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:06.128 | INFO | "malloc" : " 14 GB", | 20260508 03:11:06.129 | INFO | "pinned" : " 29 GB" | 20260508 03:11:06.129 | INFO | } | 20260508 03:11:06.129 | INFO | Perf report for the last 5.14135 seconds: | 20260508 03:11:06.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:06.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:11:06.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 03:11:06.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:06.129 | INFO | Uploaded 36GiB to GPU in 2.44447s (14.7271GiB/s) | 20260508 03:11:06.129 | INFO | Downloaded 34GiB to GPU in 2.44202s (13.9229GiB/s) | 20260508 03:11:06.129 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:11:06.129 | INFO | Total mem usage: 16.7 GiB | 20260508 03:11:06.129 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:11:06.129 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:11:06.129 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:11:06.129 | INFO | High Water Breakdown: | 20260508 03:11:06.129 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:11:06.129 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:11:06.129 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:11:06.129 | INFO | ------------------------------------------ | 20260508 03:11:06.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:11:06.129 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:11:06.130 | INFO | Total mem usage: 26.5 GiB | 20260508 03:11:06.130 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:11:06.130 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:11:06.130 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:11:06.130 | INFO | High Water Breakdown: | 20260508 03:11:06.130 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:11:06.130 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:11:06.130 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:11:06.130 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:11:06.130 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:11:06.130 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:11:06.130 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:11:06.130 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:11:06.130 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:11:06.130 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:11:06.130 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:11:06.130 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:11:06.130 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:11:06.130 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:11:06.130 | INFO | ------------------------------------------ | 20260508 03:11:06.130 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:11:06.130 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:11:06.130 | INFO | Total mem usage: 27.8 GiB | 20260508 03:11:06.130 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:11:06.130 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:11:06.130 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:11:06.130 | INFO | High Water Breakdown: | 20260508 03:11:06.130 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:11:06.130 | INFO | ------------------------------------------ | 20260508 03:11:06.130 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:11:06.130 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:11:06.130 | INFO | Total mem usage: 15.7 GiB | 20260508 03:11:06.130 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:11:06.130 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:11:06.130 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:11:06.130 | INFO | High Water Breakdown: | 20260508 03:11:06.130 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:11:06.130 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:11:06.130 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:11:06.130 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:11:06.130 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:11:06.130 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:11:06.130 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:11:06.130 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:11:06.130 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:11:06.130 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:11:06.130 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:11:06.130 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:11:06.130 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:11:06.130 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:11:06.130 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:11:06.130 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:11:06.130 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:11:06.130 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:11:06.131 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:11:06.131 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:11:06.131 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:11:06.131 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:11:06.131 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:11:06.131 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:11:06.131 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:11:06.131 | INFO | ------------------------------------------ | 20260508 03:11:06.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:11:06.131 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:11:06.131 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:11:06.131 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:11:06.131 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.61e+01, 1.42e+01 / 1.77e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.13e-02 / 6.55e-03 | 20260508 03:11:06.131 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.47e+00, 5.83e-01 / 2.20e-01 | 20260508 03:11:06.131 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.06e-01 / 1.02e-01 | 20260508 03:11:06.131 | INFO | PerfReport: 16.06%, DME -- 4.20e+01 / 4.62e+01, 4.25e+00 / 2.02e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.65e+00 / 1.53e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.41e+00 / 2.69e+00, 4.22e-01 / 2.31e-01 | 20260508 03:11:06.131 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.25e+00, 2.62e-01 / 1.31e-01 | 20260508 03:11:06.131 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.64e-01, 5.22e-02 / 2.58e-02 | 20260508 03:11:06.131 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:11:06.131 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:11:06.131 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:11:06.131 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 2.14%, LockUpload -- 2.27e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 2.81%, LockDownload -- 2.99e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:11:06.131 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260508 03:11:06.241 | INFO | Starting a round of Baz IO | 20260508 03:11:09.656 | INFO | Populating RTMetrics report | 20260508 03:11:09.657 | INFO | RTMetrics report populated | 20260508 03:11:09.657 | INFO | Opening tmp RTMetrics json file | 20260508 03:11:09.670 | INFO | Serializing RTMetrics to json | 20260508 03:11:09.672 | INFO | Serialization done, swapping tmp file now... | 20260508 03:11:09.672 | INFO | RTMetrics done | 20260508 03:11:09.732 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.49 seconds | 20260508 03:11:09.732 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.35% / 0.00% / 58.65% | 20260508 03:11:09.732 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.644533% / 0% / 4.37101e-05% | 20260508 03:11:09.732 | INFO | Baz IO overall: | 20260508 03:11:09.732 | INFO | BytesWritten : 2149946834553 | 20260508 03:11:09.732 | INFO | NumEvents : 1038861705557 | 20260508 03:11:11.262 | INFO | Analyzing chunk frames = [4775936,4776448) | 20260508 03:11:11.262 | INFO | MemInfo: { | 20260508 03:11:11.262 | INFO | "gpu" : " 17 GB", | 20260508 03:11:11.262 | INFO | "huge" : "", | 20260508 03:11:11.262 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:11.262 | INFO | "malloc" : " 14 GB", | 20260508 03:11:11.262 | INFO | "pinned" : " 29 GB" | 20260508 03:11:11.262 | INFO | } | 20260508 03:11:11.262 | INFO | Perf report for the last 5.13354 seconds: | 20260508 03:11:11.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:11.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:11.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:11.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:11.263 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 03:11:11.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:11.263 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:11:11.263 | INFO | PRE_HQ : 66.5%, 1.8, 62.0ms, 0 | 20260508 03:11:11.263 | INFO | BAZWRITER : 67.5%, 2.8, 98.5ms, 0 | 20260508 03:11:11.263 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:11:11.263 | INFO | Uploaded 36GiB to GPU in 2.56202s (14.0514GiB/s) | 20260508 03:11:11.263 | INFO | Downloaded 38GiB to GPU in 2.70329s (14.057GiB/s) | 20260508 03:11:16.407 | INFO | Analyzing chunk frames = [4776448,4776960) | 20260508 03:11:16.407 | INFO | MemInfo: { | 20260508 03:11:16.407 | INFO | "gpu" : " 17 GB", | 20260508 03:11:16.407 | INFO | "huge" : "", | 20260508 03:11:16.407 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:16.407 | INFO | "malloc" : " 14 GB", | 20260508 03:11:16.407 | INFO | "pinned" : " 29 GB" | 20260508 03:11:16.407 | INFO | } | 20260508 03:11:16.407 | INFO | Perf report for the last 5.1448 seconds: | 20260508 03:11:16.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:16.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.407 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 03:11:16.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.407 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:11:16.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:16.408 | INFO | Uploaded 36GiB to GPU in 2.57287s (13.9921GiB/s) | 20260508 03:11:16.408 | INFO | Downloaded 34GiB to GPU in 2.42488s (14.0213GiB/s) | 20260508 03:11:21.549 | INFO | Analyzing chunk frames = [4776960,4777472) | 20260508 03:11:21.549 | INFO | MemInfo: { | 20260508 03:11:21.549 | INFO | "gpu" : " 17 GB", | 20260508 03:11:21.549 | INFO | "huge" : "", | 20260508 03:11:21.549 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:21.549 | INFO | "malloc" : " 14 GB", | 20260508 03:11:21.549 | INFO | "pinned" : " 29 GB" | 20260508 03:11:21.549 | INFO | } | 20260508 03:11:21.549 | INFO | Perf report for the last 5.14208 seconds: | 20260508 03:11:21.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:21.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 03:11:21.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:11:21.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:21.549 | INFO | Uploaded 36GiB to GPU in 2.57963s (13.9555GiB/s) | 20260508 03:11:21.549 | INFO | Downloaded 34GiB to GPU in 2.49293s (13.6386GiB/s) | 20260508 03:11:26.687 | INFO | Analyzing chunk frames = [4777472,4777984) | 20260508 03:11:26.687 | INFO | MemInfo: { | 20260508 03:11:26.687 | INFO | "gpu" : " 17 GB", | 20260508 03:11:26.687 | INFO | "huge" : "", | 20260508 03:11:26.687 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:26.687 | INFO | "malloc" : " 14 GB", | 20260508 03:11:26.687 | INFO | "pinned" : " 29 GB" | 20260508 03:11:26.687 | INFO | } | 20260508 03:11:26.687 | INFO | Perf report for the last 5.13786 seconds: | 20260508 03:11:26.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:26.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 03:11:26.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:11:26.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:26.687 | INFO | Uploaded 36GiB to GPU in 2.57653s (13.9723GiB/s) | 20260508 03:11:26.687 | INFO | Downloaded 34GiB to GPU in 2.49872s (13.607GiB/s) | 20260508 03:11:31.824 | INFO | Analyzing chunk frames = [4777984,4778496) | 20260508 03:11:31.824 | INFO | MemInfo: { | 20260508 03:11:31.824 | INFO | "gpu" : " 17 GB", | 20260508 03:11:31.824 | INFO | "huge" : "", | 20260508 03:11:31.824 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:31.824 | INFO | "malloc" : " 14 GB", | 20260508 03:11:31.824 | INFO | "pinned" : " 29 GB" | 20260508 03:11:31.824 | INFO | } | 20260508 03:11:31.824 | INFO | Perf report for the last 5.13729 seconds: | 20260508 03:11:31.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:31.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.824 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 03:11:31.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.824 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 03:11:31.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:31.825 | INFO | Uploaded 36GiB to GPU in 2.40161s (14.9899GiB/s) | 20260508 03:11:31.825 | INFO | Downloaded 34GiB to GPU in 2.53951s (13.3884GiB/s) | 20260508 03:11:36.960 | INFO | Analyzing chunk frames = [4778496,4779008) | 20260508 03:11:36.960 | INFO | MemInfo: { | 20260508 03:11:36.960 | INFO | "gpu" : " 17 GB", | 20260508 03:11:36.960 | INFO | "huge" : "", | 20260508 03:11:36.960 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:36.960 | INFO | "malloc" : " 14 GB", | 20260508 03:11:36.960 | INFO | "pinned" : " 29 GB" | 20260508 03:11:36.960 | INFO | } | 20260508 03:11:36.960 | INFO | Perf report for the last 5.13571 seconds: | 20260508 03:11:36.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:36.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.960 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:11:36.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.960 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 03:11:36.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:36.961 | INFO | Uploaded 36GiB to GPU in 2.42463s (14.8476GiB/s) | 20260508 03:11:36.961 | INFO | Downloaded 34GiB to GPU in 2.42417s (14.0254GiB/s) | 20260508 03:11:42.106 | INFO | Analyzing chunk frames = [4779008,4779520) | 20260508 03:11:42.106 | INFO | MemInfo: { | 20260508 03:11:42.106 | INFO | "gpu" : " 17 GB", | 20260508 03:11:42.106 | INFO | "huge" : "", | 20260508 03:11:42.106 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:42.106 | INFO | "malloc" : " 14 GB", | 20260508 03:11:42.106 | INFO | "pinned" : " 29 GB" | 20260508 03:11:42.106 | INFO | } | 20260508 03:11:42.106 | INFO | Perf report for the last 5.1463 seconds: | 20260508 03:11:42.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:42.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.106 | INFO | ANALYSIS : 66.2%, 3.0, 103.7ms, 0 | 20260508 03:11:42.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.106 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 03:11:42.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:42.107 | INFO | Uploaded 36GiB to GPU in 2.44912s (14.6991GiB/s) | 20260508 03:11:42.107 | INFO | Downloaded 34GiB to GPU in 2.42105s (14.0435GiB/s) | 20260508 03:11:47.243 | INFO | Analyzing chunk frames = [4779520,4780032) | 20260508 03:11:47.243 | INFO | MemInfo: { | 20260508 03:11:47.243 | INFO | "gpu" : " 17 GB", | 20260508 03:11:47.243 | INFO | "huge" : "", | 20260508 03:11:47.243 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:47.243 | INFO | "malloc" : " 14 GB", | 20260508 03:11:47.243 | INFO | "pinned" : " 29 GB" | 20260508 03:11:47.243 | INFO | } | 20260508 03:11:47.243 | INFO | Perf report for the last 5.13678 seconds: | 20260508 03:11:47.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:47.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:11:47.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:11:47.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:47.243 | INFO | Uploaded 36GiB to GPU in 2.43084s (14.8097GiB/s) | 20260508 03:11:47.243 | INFO | Downloaded 34GiB to GPU in 2.4311s (13.9855GiB/s) | 20260508 03:11:47.342 | INFO | Starting a round of Baz IO | 20260508 03:11:50.783 | INFO | Populating RTMetrics report | 20260508 03:11:50.783 | INFO | RTMetrics report populated | 20260508 03:11:50.783 | INFO | Opening tmp RTMetrics json file | 20260508 03:11:50.791 | INFO | Serializing RTMetrics to json | 20260508 03:11:50.799 | INFO | Serialization done, swapping tmp file now... | 20260508 03:11:50.800 | INFO | RTMetrics done | 20260508 03:11:50.855 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.512 seconds | 20260508 03:11:50.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.34% / 0.00% / 58.65% | 20260508 03:11:50.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.643746% / 3.97364e-06% / 4.76837e-05% | 20260508 03:11:50.855 | INFO | Baz IO overall: | 20260508 03:11:50.855 | INFO | BytesWritten : 2151687620275 | 20260508 03:11:50.855 | INFO | NumEvents : 1039695247760 | 20260508 03:11:52.388 | INFO | Analyzing chunk frames = [4780032,4780544) | 20260508 03:11:52.388 | INFO | MemInfo: { | 20260508 03:11:52.388 | INFO | "gpu" : " 17 GB", | 20260508 03:11:52.388 | INFO | "huge" : "", | 20260508 03:11:52.388 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:52.388 | INFO | "malloc" : " 14 GB", | 20260508 03:11:52.388 | INFO | "pinned" : " 29 GB" | 20260508 03:11:52.388 | INFO | } | 20260508 03:11:52.388 | INFO | Perf report for the last 5.14461 seconds: | 20260508 03:11:52.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:52.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:52.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:52.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:52.388 | INFO | ANALYSIS : 67.9%, 3.0, 106.3ms, 0 | 20260508 03:11:52.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:52.388 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:11:52.388 | INFO | PRE_HQ : 65.7%, 1.6, 56.1ms, 0 | 20260508 03:11:52.388 | INFO | BAZWRITER : 67.7%, 2.7, 95.6ms, 0 | 20260508 03:11:52.388 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:11:52.388 | INFO | Uploaded 36GiB to GPU in 2.54148s (14.165GiB/s) | 20260508 03:11:52.388 | INFO | Downloaded 38GiB to GPU in 2.68554s (14.1499GiB/s) | 20260508 03:11:57.520 | INFO | Analyzing chunk frames = [4780544,4781056) | 20260508 03:11:57.520 | INFO | MemInfo: { | 20260508 03:11:57.520 | INFO | "gpu" : " 17 GB", | 20260508 03:11:57.520 | INFO | "huge" : "", | 20260508 03:11:57.520 | INFO | "huge_shared" : " 29 GB", | 20260508 03:11:57.520 | INFO | "malloc" : " 14 GB", | 20260508 03:11:57.520 | INFO | "pinned" : " 29 GB" | 20260508 03:11:57.520 | INFO | } | 20260508 03:11:57.520 | INFO | Perf report for the last 5.13226 seconds: | 20260508 03:11:57.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:11:57.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 03:11:57.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:11:57.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:11:57.520 | INFO | Uploaded 36GiB to GPU in 2.5774s (13.9675GiB/s) | 20260508 03:11:57.520 | INFO | Downloaded 34GiB to GPU in 2.43034s (13.9898GiB/s) | 20260508 03:12:02.664 | INFO | Analyzing chunk frames = [4781056,4781568) | 20260508 03:12:02.664 | INFO | MemInfo: { | 20260508 03:12:02.664 | INFO | "gpu" : " 17 GB", | 20260508 03:12:02.664 | INFO | "huge" : "", | 20260508 03:12:02.664 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:02.664 | INFO | "malloc" : " 14 GB", | 20260508 03:12:02.664 | INFO | "pinned" : " 29 GB" | 20260508 03:12:02.664 | INFO | } | 20260508 03:12:02.664 | INFO | Perf report for the last 5.14418 seconds: | 20260508 03:12:02.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:02.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.664 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:12:02.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.664 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 03:12:02.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:02.665 | INFO | Uploaded 36GiB to GPU in 2.54002s (14.1731GiB/s) | 20260508 03:12:02.665 | INFO | Downloaded 34GiB to GPU in 2.50415s (13.5775GiB/s) | 20260508 03:12:07.801 | INFO | Analyzing chunk frames = [4781568,4782080) | 20260508 03:12:07.801 | INFO | MemInfo: { | 20260508 03:12:07.801 | INFO | "gpu" : " 17 GB", | 20260508 03:12:07.801 | INFO | "huge" : "", | 20260508 03:12:07.801 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:07.801 | INFO | "malloc" : " 14 GB", | 20260508 03:12:07.801 | INFO | "pinned" : " 29 GB" | 20260508 03:12:07.801 | INFO | } | 20260508 03:12:07.801 | INFO | Perf report for the last 5.13722 seconds: | 20260508 03:12:07.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:07.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.801 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:12:07.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.802 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:12:07.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:07.802 | INFO | Uploaded 36GiB to GPU in 2.54615s (14.139GiB/s) | 20260508 03:12:07.802 | INFO | Downloaded 34GiB to GPU in 2.50193s (13.5895GiB/s) | 20260508 03:12:12.938 | INFO | Analyzing chunk frames = [4782080,4782592) | 20260508 03:12:12.938 | INFO | MemInfo: { | 20260508 03:12:12.938 | INFO | "gpu" : " 17 GB", | 20260508 03:12:12.938 | INFO | "huge" : "", | 20260508 03:12:12.938 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:12.938 | INFO | "malloc" : " 14 GB", | 20260508 03:12:12.938 | INFO | "pinned" : " 29 GB" | 20260508 03:12:12.938 | INFO | } | 20260508 03:12:12.938 | INFO | Perf report for the last 5.13658 seconds: | 20260508 03:12:12.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:12.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 03:12:12.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:12:12.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:12.938 | INFO | Uploaded 36GiB to GPU in 2.46396s (14.6106GiB/s) | 20260508 03:12:12.938 | INFO | Downloaded 34GiB to GPU in 2.57653s (13.196GiB/s) | 20260508 03:12:18.078 | INFO | Analyzing chunk frames = [4782592,4783104) | 20260508 03:12:18.078 | INFO | MemInfo: { | 20260508 03:12:18.078 | INFO | "gpu" : " 17 GB", | 20260508 03:12:18.078 | INFO | "huge" : "", | 20260508 03:12:18.078 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:18.078 | INFO | "malloc" : " 14 GB", | 20260508 03:12:18.078 | INFO | "pinned" : " 29 GB" | 20260508 03:12:18.078 | INFO | } | 20260508 03:12:18.078 | INFO | Perf report for the last 5.14036 seconds: | 20260508 03:12:18.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:18.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.078 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:12:18.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.079 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260508 03:12:18.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:18.079 | INFO | Uploaded 36GiB to GPU in 2.40383s (14.9761GiB/s) | 20260508 03:12:18.079 | INFO | Downloaded 34GiB to GPU in 2.42863s (13.9996GiB/s) | 20260508 03:12:23.215 | INFO | Analyzing chunk frames = [4783104,4783616) | 20260508 03:12:23.215 | INFO | MemInfo: { | 20260508 03:12:23.215 | INFO | "gpu" : " 17 GB", | 20260508 03:12:23.215 | INFO | "huge" : "", | 20260508 03:12:23.215 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:23.215 | INFO | "malloc" : " 14 GB", | 20260508 03:12:23.215 | INFO | "pinned" : " 29 GB" | 20260508 03:12:23.215 | INFO | } | 20260508 03:12:23.215 | INFO | Perf report for the last 5.13666 seconds: | 20260508 03:12:23.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:23.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.215 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 03:12:23.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.215 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:12:23.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:23.216 | INFO | Uploaded 36GiB to GPU in 2.43807s (14.7658GiB/s) | 20260508 03:12:23.216 | INFO | Downloaded 34GiB to GPU in 2.4287s (13.9993GiB/s) | 20260508 03:12:28.358 | INFO | Analyzing chunk frames = [4783616,4784128) | 20260508 03:12:28.358 | INFO | MemInfo: { | 20260508 03:12:28.358 | INFO | "gpu" : " 17 GB", | 20260508 03:12:28.358 | INFO | "huge" : "", | 20260508 03:12:28.358 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:28.358 | INFO | "malloc" : " 14 GB", | 20260508 03:12:28.358 | INFO | "pinned" : " 29 GB" | 20260508 03:12:28.358 | INFO | } | 20260508 03:12:28.358 | INFO | Perf report for the last 5.1433 seconds: | 20260508 03:12:28.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:28.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.358 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 03:12:28.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.359 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:12:28.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:28.359 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:12:28.359 | INFO | Total mem usage: 16.7 GiB | 20260508 03:12:28.359 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:12:28.359 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16%) | 20260508 03:12:28.359 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:12:28.359 | INFO | High Water Breakdown: | 20260508 03:12:28.359 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:12:28.359 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:12:28.359 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:12:28.359 | INFO | ------------------------------------------ | 20260508 03:12:28.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:12:28.359 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:12:28.359 | INFO | Total mem usage: 26.5 GiB | 20260508 03:12:28.359 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:12:28.359 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:12:28.359 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:12:28.359 | INFO | High Water Breakdown: | 20260508 03:12:28.359 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:12:28.359 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:12:28.359 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:12:28.359 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:12:28.359 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:12:28.359 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:12:28.359 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:12:28.359 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:12:28.359 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:12:28.359 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:12:28.359 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:12:28.359 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:12:28.359 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:12:28.359 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:12:28.359 | INFO | ------------------------------------------ | 20260508 03:12:28.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:12:28.359 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:12:28.359 | INFO | Total mem usage: 27.8 GiB | 20260508 03:12:28.359 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:12:28.359 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:12:28.359 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:12:28.359 | INFO | High Water Breakdown: | 20260508 03:12:28.359 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:12:28.359 | INFO | ------------------------------------------ | 20260508 03:12:28.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:12:28.359 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:12:28.359 | INFO | Total mem usage: 15.7 GiB | 20260508 03:12:28.359 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:12:28.359 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:12:28.359 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:12:28.359 | INFO | High Water Breakdown: | 20260508 03:12:28.359 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:12:28.359 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:12:28.359 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:12:28.359 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:12:28.359 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:12:28.359 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:12:28.359 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:12:28.359 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:12:28.359 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:12:28.359 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:12:28.359 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:12:28.359 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:12:28.359 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:12:28.359 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:12:28.359 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:12:28.359 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:12:28.359 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:12:28.359 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:12:28.359 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:12:28.359 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:12:28.359 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:12:28.359 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:12:28.359 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:12:28.359 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:12:28.359 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:12:28.359 | INFO | ------------------------------------------ | 20260508 03:12:28.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:12:28.359 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:12:28.359 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:12:28.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:12:28.359 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.58e+01, 1.41e+01 / 1.60e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.01e-01, 1.16e-02 / 6.48e-03 | 20260508 03:12:28.359 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.47e+00, 5.82e-01 / 1.97e-01 | 20260508 03:12:28.359 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.46e+00, 2.06e-01 / 1.09e-01 | 20260508 03:12:28.359 | INFO | PerfReport: 16.09%, DME -- 4.20e+01 / 4.60e+01, 4.08e+00 / 2.06e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.65e+00 / 1.40e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.16e-01 / 2.37e-01 | 20260508 03:12:28.359 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.21e+00, 2.65e-01 / 1.40e-01 | 20260508 03:12:28.359 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.25e-01 / 3.63e-01, 5.15e-02 / 2.91e-02 | 20260508 03:12:28.359 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:12:28.359 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:12:28.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:12:28.359 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 24.47%, Upload -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 30.63%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 2.23%, LockUpload -- 2.37e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 2.76%, LockDownload -- 2.94e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 15.49%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:12:28.359 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.12e-02 / 0.00e+00 | 20260508 03:12:28.359 | INFO | Uploaded 36GiB to GPU in 2.54538s (14.1433GiB/s) | 20260508 03:12:28.359 | INFO | Downloaded 34GiB to GPU in 2.46208s (13.8095GiB/s) | 20260508 03:12:28.456 | INFO | Starting a round of Baz IO | 20260508 03:12:31.893 | INFO | Populating RTMetrics report | 20260508 03:12:31.894 | INFO | RTMetrics report populated | 20260508 03:12:31.894 | INFO | Opening tmp RTMetrics json file | 20260508 03:12:31.906 | INFO | Serializing RTMetrics to json | 20260508 03:12:31.907 | INFO | Serialization done, swapping tmp file now... | 20260508 03:12:31.907 | INFO | RTMetrics done | 20260508 03:12:31.967 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.511 seconds | 20260508 03:12:31.967 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.34% / 0.00% / 58.66% | 20260508 03:12:31.967 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.64102% / 3.97364e-06% / 5.5631e-05% | 20260508 03:12:31.967 | INFO | Baz IO overall: | 20260508 03:12:31.967 | INFO | BytesWritten : 2153428011049 | 20260508 03:12:31.967 | INFO | NumEvents : 1040528656575 | 20260508 03:12:33.499 | INFO | Analyzing chunk frames = [4784128,4784640) | 20260508 03:12:33.499 | INFO | MemInfo: { | 20260508 03:12:33.499 | INFO | "gpu" : " 17 GB", | 20260508 03:12:33.499 | INFO | "huge" : "", | 20260508 03:12:33.499 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:33.499 | INFO | "malloc" : " 14 GB", | 20260508 03:12:33.499 | INFO | "pinned" : " 29 GB" | 20260508 03:12:33.499 | INFO | } | 20260508 03:12:33.499 | INFO | Perf report for the last 5.14072 seconds: | 20260508 03:12:33.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:33.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:33.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:33.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:33.499 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 03:12:33.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:33.499 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:12:33.499 | INFO | PRE_HQ : 65.6%, 1.6, 56.2ms, 0 | 20260508 03:12:33.499 | INFO | BAZWRITER : 67.8%, 2.7, 92.7ms, 0 | 20260508 03:12:33.499 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:12:33.499 | INFO | Uploaded 36GiB to GPU in 2.55905s (14.0677GiB/s) | 20260508 03:12:33.499 | INFO | Downloaded 38GiB to GPU in 2.69194s (14.1162GiB/s) | 20260508 03:12:38.636 | INFO | Analyzing chunk frames = [4784640,4785152) | 20260508 03:12:38.636 | INFO | MemInfo: { | 20260508 03:12:38.636 | INFO | "gpu" : " 17 GB", | 20260508 03:12:38.636 | INFO | "huge" : "", | 20260508 03:12:38.636 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:38.636 | INFO | "malloc" : " 14 GB", | 20260508 03:12:38.636 | INFO | "pinned" : " 29 GB" | 20260508 03:12:38.636 | INFO | } | 20260508 03:12:38.636 | INFO | Perf report for the last 5.13668 seconds: | 20260508 03:12:38.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:38.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 03:12:38.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:12:38.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:38.636 | INFO | Uploaded 36GiB to GPU in 2.51844s (14.2946GiB/s) | 20260508 03:12:38.636 | INFO | Downloaded 34GiB to GPU in 2.43831s (13.9441GiB/s) | 20260508 03:12:43.772 | INFO | Analyzing chunk frames = [4785152,4785664) | 20260508 03:12:43.772 | INFO | MemInfo: { | 20260508 03:12:43.772 | INFO | "gpu" : " 17 GB", | 20260508 03:12:43.772 | INFO | "huge" : "", | 20260508 03:12:43.772 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:43.772 | INFO | "malloc" : " 14 GB", | 20260508 03:12:43.772 | INFO | "pinned" : " 29 GB" | 20260508 03:12:43.772 | INFO | } | 20260508 03:12:43.773 | INFO | Perf report for the last 5.13673 seconds: | 20260508 03:12:43.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:43.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 03:12:43.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:12:43.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:43.773 | INFO | Uploaded 36GiB to GPU in 2.49182s (14.4473GiB/s) | 20260508 03:12:43.773 | INFO | Downloaded 34GiB to GPU in 2.51849s (13.5002GiB/s) | 20260508 03:12:48.916 | INFO | Analyzing chunk frames = [4785664,4786176) | 20260508 03:12:48.916 | INFO | MemInfo: { | 20260508 03:12:48.916 | INFO | "gpu" : " 17 GB", | 20260508 03:12:48.916 | INFO | "huge" : "", | 20260508 03:12:48.916 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:48.916 | INFO | "malloc" : " 14 GB", | 20260508 03:12:48.917 | INFO | "pinned" : " 29 GB" | 20260508 03:12:48.917 | INFO | } | 20260508 03:12:48.917 | INFO | Perf report for the last 5.14407 seconds: | 20260508 03:12:48.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:48.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 03:12:48.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:12:48.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:48.917 | INFO | Uploaded 36GiB to GPU in 2.48543s (14.4844GiB/s) | 20260508 03:12:48.917 | INFO | Downloaded 34GiB to GPU in 2.52565s (13.4619GiB/s) | 20260508 03:12:54.061 | INFO | Analyzing chunk frames = [4786176,4786688) | 20260508 03:12:54.061 | INFO | MemInfo: { | 20260508 03:12:54.061 | INFO | "gpu" : " 17 GB", | 20260508 03:12:54.061 | INFO | "huge" : "", | 20260508 03:12:54.061 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:54.062 | INFO | "malloc" : " 14 GB", | 20260508 03:12:54.062 | INFO | "pinned" : " 29 GB" | 20260508 03:12:54.062 | INFO | } | 20260508 03:12:54.062 | INFO | Perf report for the last 5.14503 seconds: | 20260508 03:12:54.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:54.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 03:12:54.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 03:12:54.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:54.062 | INFO | Uploaded 36GiB to GPU in 2.35164s (15.3084GiB/s) | 20260508 03:12:54.062 | INFO | Downloaded 34GiB to GPU in 2.55799s (13.2917GiB/s) | 20260508 03:12:59.198 | INFO | Analyzing chunk frames = [4786688,4787200) | 20260508 03:12:59.198 | INFO | MemInfo: { | 20260508 03:12:59.198 | INFO | "gpu" : " 17 GB", | 20260508 03:12:59.198 | INFO | "huge" : "", | 20260508 03:12:59.198 | INFO | "huge_shared" : " 29 GB", | 20260508 03:12:59.198 | INFO | "malloc" : " 14 GB", | 20260508 03:12:59.198 | INFO | "pinned" : " 29 GB" | 20260508 03:12:59.198 | INFO | } | 20260508 03:12:59.198 | INFO | Perf report for the last 5.13635 seconds: | 20260508 03:12:59.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:12:59.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:12:59.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:12:59.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:12:59.198 | INFO | Uploaded 36GiB to GPU in 2.40481s (14.97GiB/s) | 20260508 03:12:59.198 | INFO | Downloaded 34GiB to GPU in 2.43195s (13.9806GiB/s) | 20260508 03:13:04.336 | INFO | Analyzing chunk frames = [4787200,4787712) | 20260508 03:13:04.336 | INFO | MemInfo: { | 20260508 03:13:04.336 | INFO | "gpu" : " 17 GB", | 20260508 03:13:04.336 | INFO | "huge" : "", | 20260508 03:13:04.336 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:04.337 | INFO | "malloc" : " 14 GB", | 20260508 03:13:04.337 | INFO | "pinned" : " 29 GB" | 20260508 03:13:04.337 | INFO | } | 20260508 03:13:04.337 | INFO | Perf report for the last 5.13859 seconds: | 20260508 03:13:04.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:04.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:13:04.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:13:04.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:04.337 | INFO | Uploaded 36GiB to GPU in 2.41908s (14.8817GiB/s) | 20260508 03:13:04.337 | INFO | Downloaded 34GiB to GPU in 2.43327s (13.973GiB/s) | 20260508 03:13:09.473 | INFO | Analyzing chunk frames = [4787712,4788224) | 20260508 03:13:09.473 | INFO | MemInfo: { | 20260508 03:13:09.473 | INFO | "gpu" : " 17 GB", | 20260508 03:13:09.473 | INFO | "huge" : "", | 20260508 03:13:09.473 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:09.473 | INFO | "malloc" : " 14 GB", | 20260508 03:13:09.473 | INFO | "pinned" : " 29 GB" | 20260508 03:13:09.473 | INFO | } | 20260508 03:13:09.473 | INFO | Perf report for the last 5.13622 seconds: | 20260508 03:13:09.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:09.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.473 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:13:09.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.473 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:13:09.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:09.474 | INFO | Uploaded 36GiB to GPU in 2.43356s (14.7931GiB/s) | 20260508 03:13:09.474 | INFO | Downloaded 34GiB to GPU in 2.43406s (13.9684GiB/s) | 20260508 03:13:09.571 | INFO | Starting a round of Baz IO | 20260508 03:13:13.000 | INFO | Populating RTMetrics report | 20260508 03:13:13.001 | INFO | RTMetrics report populated | 20260508 03:13:13.001 | INFO | Opening tmp RTMetrics json file | 20260508 03:13:13.010 | INFO | Serializing RTMetrics to json | 20260508 03:13:13.016 | INFO | Serialization done, swapping tmp file now... | 20260508 03:13:13.016 | INFO | RTMetrics done | 20260508 03:13:13.068 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.497 seconds | 20260508 03:13:13.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.34% / 0.00% / 58.66% | 20260508 03:13:13.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.638906% / 1.19209e-05% / 7.54992e-05% | 20260508 03:13:13.068 | INFO | Baz IO overall: | 20260508 03:13:13.068 | INFO | BytesWritten : 2155166643613 | 20260508 03:13:13.068 | INFO | NumEvents : 1041360811299 | 20260508 03:13:14.613 | INFO | Analyzing chunk frames = [4788224,4788736) | 20260508 03:13:14.613 | INFO | MemInfo: { | 20260508 03:13:14.613 | INFO | "gpu" : " 17 GB", | 20260508 03:13:14.613 | INFO | "huge" : "", | 20260508 03:13:14.613 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:14.613 | INFO | "malloc" : " 14 GB", | 20260508 03:13:14.613 | INFO | "pinned" : " 29 GB" | 20260508 03:13:14.613 | INFO | } | 20260508 03:13:14.613 | INFO | Perf report for the last 5.13995 seconds: | 20260508 03:13:14.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:14.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:14.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:14.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:14.613 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 03:13:14.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:14.613 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 03:13:14.613 | INFO | PRE_HQ : 65.4%, 1.6, 55.8ms, 0 | 20260508 03:13:14.613 | INFO | BAZWRITER : 67.4%, 2.7, 92.0ms, 0 | 20260508 03:13:14.613 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:13:14.614 | INFO | Uploaded 36GiB to GPU in 2.48289s (14.4993GiB/s) | 20260508 03:13:14.614 | INFO | Downloaded 38GiB to GPU in 2.66349s (14.267GiB/s) | 20260508 03:13:19.753 | INFO | Analyzing chunk frames = [4788736,4789248) | 20260508 03:13:19.753 | INFO | MemInfo: { | 20260508 03:13:19.753 | INFO | "gpu" : " 17 GB", | 20260508 03:13:19.753 | INFO | "huge" : "", | 20260508 03:13:19.753 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:19.753 | INFO | "malloc" : " 14 GB", | 20260508 03:13:19.753 | INFO | "pinned" : " 29 GB" | 20260508 03:13:19.753 | INFO | } | 20260508 03:13:19.753 | INFO | Perf report for the last 5.14032 seconds: | 20260508 03:13:19.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:19.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 03:13:19.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:13:19.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:19.753 | INFO | Uploaded 36GiB to GPU in 2.56921s (14.0121GiB/s) | 20260508 03:13:19.753 | INFO | Downloaded 34GiB to GPU in 2.43529s (13.9614GiB/s) | 20260508 03:13:24.890 | INFO | Analyzing chunk frames = [4789248,4789760) | 20260508 03:13:24.890 | INFO | MemInfo: { | 20260508 03:13:24.890 | INFO | "gpu" : " 17 GB", | 20260508 03:13:24.890 | INFO | "huge" : "", | 20260508 03:13:24.890 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:24.890 | INFO | "malloc" : " 14 GB", | 20260508 03:13:24.890 | INFO | "pinned" : " 29 GB" | 20260508 03:13:24.890 | INFO | } | 20260508 03:13:24.890 | INFO | Perf report for the last 5.13665 seconds: | 20260508 03:13:24.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:24.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | ANALYSIS : 67.7%, 3.0, 104.9ms, 0 | 20260508 03:13:24.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:13:24.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:24.890 | INFO | Uploaded 36GiB to GPU in 2.59952s (13.8487GiB/s) | 20260508 03:13:24.890 | INFO | Downloaded 34GiB to GPU in 2.49722s (13.6151GiB/s) | 20260508 03:13:30.029 | INFO | Analyzing chunk frames = [4789760,4790272) | 20260508 03:13:30.029 | INFO | MemInfo: { | 20260508 03:13:30.029 | INFO | "gpu" : " 17 GB", | 20260508 03:13:30.029 | INFO | "huge" : "", | 20260508 03:13:30.029 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:30.029 | INFO | "malloc" : " 14 GB", | 20260508 03:13:30.029 | INFO | "pinned" : " 29 GB" | 20260508 03:13:30.029 | INFO | } | 20260508 03:13:30.029 | INFO | Perf report for the last 5.13943 seconds: | 20260508 03:13:30.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:30.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.029 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:13:30.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.029 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:13:30.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:30.030 | INFO | Uploaded 36GiB to GPU in 2.56366s (14.0424GiB/s) | 20260508 03:13:30.030 | INFO | Downloaded 34GiB to GPU in 2.49443s (13.6304GiB/s) | 20260508 03:13:35.167 | INFO | Analyzing chunk frames = [4790272,4790784) | 20260508 03:13:35.167 | INFO | MemInfo: { | 20260508 03:13:35.167 | INFO | "gpu" : " 17 GB", | 20260508 03:13:35.167 | INFO | "huge" : "", | 20260508 03:13:35.167 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:35.168 | INFO | "malloc" : " 14 GB", | 20260508 03:13:35.168 | INFO | "pinned" : " 29 GB" | 20260508 03:13:35.168 | INFO | } | 20260508 03:13:35.168 | INFO | Perf report for the last 5.13833 seconds: | 20260508 03:13:35.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:35.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 03:13:35.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:13:35.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:35.168 | INFO | Uploaded 36GiB to GPU in 2.45034s (14.6918GiB/s) | 20260508 03:13:35.168 | INFO | Downloaded 34GiB to GPU in 2.57839s (13.1865GiB/s) | 20260508 03:13:40.307 | INFO | Analyzing chunk frames = [4790784,4791296) | 20260508 03:13:40.307 | INFO | MemInfo: { | 20260508 03:13:40.307 | INFO | "gpu" : " 17 GB", | 20260508 03:13:40.307 | INFO | "huge" : "", | 20260508 03:13:40.307 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:40.307 | INFO | "malloc" : " 14 GB", | 20260508 03:13:40.308 | INFO | "pinned" : " 29 GB" | 20260508 03:13:40.308 | INFO | } | 20260508 03:13:40.308 | INFO | Perf report for the last 5.14003 seconds: | 20260508 03:13:40.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:40.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 03:13:40.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:13:40.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:40.308 | INFO | Uploaded 36GiB to GPU in 2.40294s (14.9816GiB/s) | 20260508 03:13:40.308 | INFO | Downloaded 34GiB to GPU in 2.42209s (14.0375GiB/s) | 20260508 03:13:45.450 | INFO | Analyzing chunk frames = [4791296,4791808) | 20260508 03:13:45.450 | INFO | MemInfo: { | 20260508 03:13:45.450 | INFO | "gpu" : " 17 GB", | 20260508 03:13:45.450 | INFO | "huge" : "", | 20260508 03:13:45.450 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:45.450 | INFO | "malloc" : " 14 GB", | 20260508 03:13:45.450 | INFO | "pinned" : " 29 GB" | 20260508 03:13:45.450 | INFO | } | 20260508 03:13:45.450 | INFO | Perf report for the last 5.14213 seconds: | 20260508 03:13:45.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:45.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 03:13:45.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:13:45.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:45.450 | INFO | Uploaded 36GiB to GPU in 2.44151s (14.745GiB/s) | 20260508 03:13:45.451 | INFO | Downloaded 34GiB to GPU in 2.4273s (14.0074GiB/s) | 20260508 03:13:50.589 | INFO | Analyzing chunk frames = [4791808,4792320) | 20260508 03:13:50.589 | INFO | MemInfo: { | 20260508 03:13:50.589 | INFO | "gpu" : " 17 GB", | 20260508 03:13:50.589 | INFO | "huge" : "", | 20260508 03:13:50.589 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:50.589 | INFO | "malloc" : " 14 GB", | 20260508 03:13:50.589 | INFO | "pinned" : " 29 GB" | 20260508 03:13:50.589 | INFO | } | 20260508 03:13:50.589 | INFO | Perf report for the last 5.13957 seconds: | 20260508 03:13:50.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:50.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.589 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:13:50.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.589 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:13:50.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:50.590 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:13:50.590 | INFO | Total mem usage: 16.7 GiB | 20260508 03:13:50.590 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:13:50.590 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:13:50.590 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:13:50.590 | INFO | High Water Breakdown: | 20260508 03:13:50.590 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:13:50.590 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:13:50.590 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:13:50.590 | INFO | ------------------------------------------ | 20260508 03:13:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:13:50.590 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:13:50.590 | INFO | Total mem usage: 26.5 GiB | 20260508 03:13:50.590 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:13:50.590 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:13:50.590 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:13:50.590 | INFO | High Water Breakdown: | 20260508 03:13:50.590 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:13:50.590 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:13:50.590 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:13:50.590 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:13:50.590 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:13:50.590 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:13:50.590 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:13:50.590 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:13:50.590 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:13:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:13:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:13:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:13:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:13:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:13:50.590 | INFO | ------------------------------------------ | 20260508 03:13:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:13:50.590 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:13:50.590 | INFO | Total mem usage: 27.8 GiB | 20260508 03:13:50.590 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:13:50.590 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:13:50.590 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:13:50.590 | INFO | High Water Breakdown: | 20260508 03:13:50.590 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:13:50.590 | INFO | ------------------------------------------ | 20260508 03:13:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:13:50.590 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:13:50.590 | INFO | Total mem usage: 15.7 GiB | 20260508 03:13:50.590 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:13:50.590 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:13:50.590 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:13:50.590 | INFO | High Water Breakdown: | 20260508 03:13:50.590 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:13:50.590 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:13:50.590 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:13:50.590 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:13:50.590 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:13:50.590 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:13:50.590 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:13:50.590 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:13:50.590 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:13:50.590 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:13:50.590 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:13:50.590 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:13:50.590 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:13:50.590 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:13:50.590 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:13:50.590 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:13:50.590 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:13:50.590 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:13:50.590 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:13:50.590 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:13:50.590 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:13:50.590 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:13:50.590 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:13:50.590 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:13:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:13:50.590 | INFO | ------------------------------------------ | 20260508 03:13:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:13:50.590 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:13:50.590 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:13:50.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:13:50.590 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.42e+01 / 1.38e+00 | 20260508 03:13:50.590 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.20e-02 / 1.77e-02 | 20260508 03:13:50.590 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.49e+00, 5.89e-01 / 1.49e-01 | 20260508 03:13:50.590 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 9.32e-02 | 20260508 03:13:50.590 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.62e+01, 4.05e+00 / 1.72e+00 | 20260508 03:13:50.590 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.43e+00 | 20260508 03:13:50.590 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.19e-01 / 2.41e-01 | 20260508 03:13:50.590 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.23e+00, 2.67e-01 / 1.10e-01 | 20260508 03:13:50.590 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.59e-01, 5.18e-02 / 2.11e-02 | 20260508 03:13:50.590 | INFO | Uploaded 36GiB to GPU in 2.45593s (14.6584GiB/s) | 20260508 03:13:50.590 | INFO | Downloaded 34GiB to GPU in 2.43548s (13.9603GiB/s) | 20260508 03:13:50.590 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:13:50.590 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:13:50.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:13:50.590 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:13:50.590 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 03:13:50.590 | INFO | PerfReport: 24.46%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 03:13:50.591 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:13:50.591 | INFO | PerfReport: 2.11%, LockUpload -- 2.24e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 03:13:50.591 | INFO | PerfReport: 2.86%, LockDownload -- 3.04e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260508 03:13:50.591 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:13:50.591 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 03:13:50.687 | INFO | Starting a round of Baz IO | 20260508 03:13:54.109 | INFO | Populating RTMetrics report | 20260508 03:13:54.109 | INFO | RTMetrics report populated | 20260508 03:13:54.109 | INFO | Opening tmp RTMetrics json file | 20260508 03:13:54.123 | INFO | Serializing RTMetrics to json | 20260508 03:13:54.124 | INFO | Serialization done, swapping tmp file now... | 20260508 03:13:54.125 | INFO | RTMetrics done | 20260508 03:13:54.182 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.495 seconds | 20260508 03:13:54.182 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.34% / 0.00% / 58.66% | 20260508 03:13:54.182 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.639749% / 3.97364e-06% / 5.16574e-05% | 20260508 03:13:54.182 | INFO | Baz IO overall: | 20260508 03:13:54.182 | INFO | BytesWritten : 2156906599692 | 20260508 03:13:54.182 | INFO | NumEvents : 1042193778081 | 20260508 03:13:55.722 | INFO | Analyzing chunk frames = [4792320,4792832) | 20260508 03:13:55.722 | INFO | MemInfo: { | 20260508 03:13:55.722 | INFO | "gpu" : " 17 GB", | 20260508 03:13:55.722 | INFO | "huge" : "", | 20260508 03:13:55.722 | INFO | "huge_shared" : " 29 GB", | 20260508 03:13:55.722 | INFO | "malloc" : " 14 GB", | 20260508 03:13:55.722 | INFO | "pinned" : " 29 GB" | 20260508 03:13:55.722 | INFO | } | 20260508 03:13:55.722 | INFO | Perf report for the last 5.13294 seconds: | 20260508 03:13:55.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:13:55.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:55.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:55.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:55.722 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 03:13:55.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:13:55.722 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:13:55.723 | INFO | PRE_HQ : 65.6%, 1.6, 55.9ms, 0 | 20260508 03:13:55.723 | INFO | BAZWRITER : 67.6%, 2.6, 91.4ms, 0 | 20260508 03:13:55.723 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:13:55.723 | INFO | Uploaded 36GiB to GPU in 2.56125s (14.0557GiB/s) | 20260508 03:13:55.723 | INFO | Downloaded 38GiB to GPU in 2.70466s (14.0498GiB/s) | 20260508 03:14:00.865 | INFO | Analyzing chunk frames = [4792832,4793344) | 20260508 03:14:00.865 | INFO | MemInfo: { | 20260508 03:14:00.865 | INFO | "gpu" : " 17 GB", | 20260508 03:14:00.865 | INFO | "huge" : "", | 20260508 03:14:00.865 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:00.865 | INFO | "malloc" : " 14 GB", | 20260508 03:14:00.865 | INFO | "pinned" : " 29 GB" | 20260508 03:14:00.865 | INFO | } | 20260508 03:14:00.865 | INFO | Perf report for the last 5.14291 seconds: | 20260508 03:14:00.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:00.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.865 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 03:14:00.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.865 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 03:14:00.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:00.866 | INFO | Uploaded 36GiB to GPU in 2.51616s (14.3075GiB/s) | 20260508 03:14:00.866 | INFO | Downloaded 34GiB to GPU in 2.43924s (13.9388GiB/s) | 20260508 03:14:06.010 | INFO | Analyzing chunk frames = [4793344,4793856) | 20260508 03:14:06.010 | INFO | MemInfo: { | 20260508 03:14:06.010 | INFO | "gpu" : " 17 GB", | 20260508 03:14:06.010 | INFO | "huge" : "", | 20260508 03:14:06.010 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:06.010 | INFO | "malloc" : " 14 GB", | 20260508 03:14:06.011 | INFO | "pinned" : " 29 GB" | 20260508 03:14:06.011 | INFO | } | 20260508 03:14:06.011 | INFO | Perf report for the last 5.14531 seconds: | 20260508 03:14:06.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:06.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 03:14:06.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:14:06.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:06.011 | INFO | Uploaded 36GiB to GPU in 2.48432s (14.4909GiB/s) | 20260508 03:14:06.011 | INFO | Downloaded 34GiB to GPU in 2.51312s (13.529GiB/s) | 20260508 03:14:11.149 | INFO | Analyzing chunk frames = [4793856,4794368) | 20260508 03:14:11.149 | INFO | MemInfo: { | 20260508 03:14:11.149 | INFO | "gpu" : " 17 GB", | 20260508 03:14:11.149 | INFO | "huge" : "", | 20260508 03:14:11.149 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:11.149 | INFO | "malloc" : " 14 GB", | 20260508 03:14:11.149 | INFO | "pinned" : " 29 GB" | 20260508 03:14:11.149 | INFO | } | 20260508 03:14:11.150 | INFO | Perf report for the last 5.1389 seconds: | 20260508 03:14:11.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:11.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 03:14:11.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:14:11.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:11.150 | INFO | Uploaded 36GiB to GPU in 2.52621s (14.2506GiB/s) | 20260508 03:14:11.150 | INFO | Downloaded 34GiB to GPU in 2.45148s (13.8692GiB/s) | 20260508 03:14:16.283 | INFO | Analyzing chunk frames = [4794368,4794880) | 20260508 03:14:16.283 | INFO | MemInfo: { | 20260508 03:14:16.283 | INFO | "gpu" : " 17 GB", | 20260508 03:14:16.283 | INFO | "huge" : "", | 20260508 03:14:16.283 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:16.283 | INFO | "malloc" : " 14 GB", | 20260508 03:14:16.283 | INFO | "pinned" : " 29 GB" | 20260508 03:14:16.283 | INFO | } | 20260508 03:14:16.283 | INFO | Perf report for the last 5.13362 seconds: | 20260508 03:14:16.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:16.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 03:14:16.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:14:16.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:16.283 | INFO | Uploaded 36GiB to GPU in 2.3128s (15.5656GiB/s) | 20260508 03:14:16.283 | INFO | Downloaded 34GiB to GPU in 2.56424s (13.2593GiB/s) | 20260508 03:14:21.426 | INFO | Analyzing chunk frames = [4794880,4795392) | 20260508 03:14:21.426 | INFO | MemInfo: { | 20260508 03:14:21.426 | INFO | "gpu" : " 17 GB", | 20260508 03:14:21.426 | INFO | "huge" : "", | 20260508 03:14:21.426 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:21.427 | INFO | "malloc" : " 14 GB", | 20260508 03:14:21.427 | INFO | "pinned" : " 29 GB" | 20260508 03:14:21.427 | INFO | } | 20260508 03:14:21.427 | INFO | Perf report for the last 5.14361 seconds: | 20260508 03:14:21.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:21.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 03:14:21.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:14:21.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:21.427 | INFO | Uploaded 36GiB to GPU in 2.40644s (14.9599GiB/s) | 20260508 03:14:21.427 | INFO | Downloaded 34GiB to GPU in 2.43415s (13.9679GiB/s) | 20260508 03:14:26.567 | INFO | Analyzing chunk frames = [4795392,4795904) | 20260508 03:14:26.567 | INFO | MemInfo: { | 20260508 03:14:26.567 | INFO | "gpu" : " 17 GB", | 20260508 03:14:26.567 | INFO | "huge" : "", | 20260508 03:14:26.567 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:26.567 | INFO | "malloc" : " 14 GB", | 20260508 03:14:26.567 | INFO | "pinned" : " 29 GB" | 20260508 03:14:26.567 | INFO | } | 20260508 03:14:26.567 | INFO | Perf report for the last 5.1399 seconds: | 20260508 03:14:26.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:26.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:14:26.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:14:26.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:26.567 | INFO | Uploaded 36GiB to GPU in 2.4275s (14.83GiB/s) | 20260508 03:14:26.567 | INFO | Downloaded 34GiB to GPU in 2.42648s (14.0121GiB/s) | 20260508 03:14:31.705 | INFO | Analyzing chunk frames = [4795904,4796416) | 20260508 03:14:31.705 | INFO | MemInfo: { | 20260508 03:14:31.705 | INFO | "gpu" : " 17 GB", | 20260508 03:14:31.705 | INFO | "huge" : "", | 20260508 03:14:31.705 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:31.705 | INFO | "malloc" : " 14 GB", | 20260508 03:14:31.705 | INFO | "pinned" : " 29 GB" | 20260508 03:14:31.705 | INFO | } | 20260508 03:14:31.705 | INFO | Perf report for the last 5.13856 seconds: | 20260508 03:14:31.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:31.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.705 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:14:31.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.705 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 03:14:31.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:31.706 | INFO | Uploaded 36GiB to GPU in 2.43816s (14.7652GiB/s) | 20260508 03:14:31.706 | INFO | Downloaded 34GiB to GPU in 2.4278s (14.0044GiB/s) | 20260508 03:14:31.814 | INFO | Starting a round of Baz IO | 20260508 03:14:35.244 | INFO | Populating RTMetrics report | 20260508 03:14:35.244 | INFO | RTMetrics report populated | 20260508 03:14:35.244 | INFO | Opening tmp RTMetrics json file | 20260508 03:14:35.254 | INFO | Serializing RTMetrics to json | 20260508 03:14:35.255 | INFO | Serialization done, swapping tmp file now... | 20260508 03:14:35.256 | INFO | RTMetrics done | 20260508 03:14:35.313 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.499 seconds | 20260508 03:14:35.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.33% / 0.00% / 58.66% | 20260508 03:14:35.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.639987% / 0% / 6.35783e-05% | 20260508 03:14:35.314 | INFO | Baz IO overall: | 20260508 03:14:35.314 | INFO | BytesWritten : 2158647565435 | 20260508 03:14:35.314 | INFO | NumEvents : 1043027730368 | 20260508 03:14:36.841 | INFO | Analyzing chunk frames = [4796416,4796928) | 20260508 03:14:36.841 | INFO | MemInfo: { | 20260508 03:14:36.841 | INFO | "gpu" : " 17 GB", | 20260508 03:14:36.841 | INFO | "huge" : "", | 20260508 03:14:36.841 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:36.841 | INFO | "malloc" : " 14 GB", | 20260508 03:14:36.841 | INFO | "pinned" : " 29 GB" | 20260508 03:14:36.841 | INFO | } | 20260508 03:14:36.841 | INFO | Perf report for the last 5.13618 seconds: | 20260508 03:14:36.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:36.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:36.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:36.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:36.841 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 03:14:36.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:36.842 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 03:14:36.842 | INFO | PRE_HQ : 66.2%, 1.8, 61.5ms, 0 | 20260508 03:14:36.842 | INFO | BAZWRITER : 67.5%, 2.8, 97.0ms, 0 | 20260508 03:14:36.842 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:14:36.842 | INFO | Uploaded 36GiB to GPU in 2.54583s (14.1408GiB/s) | 20260508 03:14:36.842 | INFO | Downloaded 38GiB to GPU in 2.69499s (14.1002GiB/s) | 20260508 03:14:41.981 | INFO | Analyzing chunk frames = [4796928,4797440) | 20260508 03:14:41.981 | INFO | MemInfo: { | 20260508 03:14:41.981 | INFO | "gpu" : " 17 GB", | 20260508 03:14:41.981 | INFO | "huge" : "", | 20260508 03:14:41.981 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:41.981 | INFO | "malloc" : " 14 GB", | 20260508 03:14:41.981 | INFO | "pinned" : " 29 GB" | 20260508 03:14:41.981 | INFO | } | 20260508 03:14:41.981 | INFO | Perf report for the last 5.14005 seconds: | 20260508 03:14:41.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:41.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:14:41.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:14:41.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:41.982 | INFO | Uploaded 36GiB to GPU in 2.55506s (14.0897GiB/s) | 20260508 03:14:41.982 | INFO | Downloaded 34GiB to GPU in 2.42964s (13.9938GiB/s) | 20260508 03:14:47.119 | INFO | Analyzing chunk frames = [4797440,4797952) | 20260508 03:14:47.119 | INFO | MemInfo: { | 20260508 03:14:47.119 | INFO | "gpu" : " 17 GB", | 20260508 03:14:47.119 | INFO | "huge" : "", | 20260508 03:14:47.119 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:47.119 | INFO | "malloc" : " 14 GB", | 20260508 03:14:47.119 | INFO | "pinned" : " 29 GB" | 20260508 03:14:47.119 | INFO | } | 20260508 03:14:47.119 | INFO | Perf report for the last 5.13783 seconds: | 20260508 03:14:47.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:47.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.119 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:14:47.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.119 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:14:47.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:47.120 | INFO | Uploaded 36GiB to GPU in 2.57715s (13.9689GiB/s) | 20260508 03:14:47.120 | INFO | Downloaded 34GiB to GPU in 2.50969s (13.5475GiB/s) | 20260508 03:14:52.261 | INFO | Analyzing chunk frames = [4797952,4798464) | 20260508 03:14:52.261 | INFO | MemInfo: { | 20260508 03:14:52.261 | INFO | "gpu" : " 17 GB", | 20260508 03:14:52.261 | INFO | "huge" : "", | 20260508 03:14:52.261 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:52.261 | INFO | "malloc" : " 14 GB", | 20260508 03:14:52.261 | INFO | "pinned" : " 29 GB" | 20260508 03:14:52.261 | INFO | } | 20260508 03:14:52.262 | INFO | Perf report for the last 5.14217 seconds: | 20260508 03:14:52.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:52.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 03:14:52.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:14:52.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:52.262 | INFO | Uploaded 36GiB to GPU in 2.50964s (14.3447GiB/s) | 20260508 03:14:52.262 | INFO | Downloaded 34GiB to GPU in 2.49784s (13.6117GiB/s) | 20260508 03:14:57.401 | INFO | Analyzing chunk frames = [4798464,4798976) | 20260508 03:14:57.401 | INFO | MemInfo: { | 20260508 03:14:57.401 | INFO | "gpu" : " 17 GB", | 20260508 03:14:57.401 | INFO | "huge" : "", | 20260508 03:14:57.401 | INFO | "huge_shared" : " 29 GB", | 20260508 03:14:57.401 | INFO | "malloc" : " 14 GB", | 20260508 03:14:57.401 | INFO | "pinned" : " 29 GB" | 20260508 03:14:57.401 | INFO | } | 20260508 03:14:57.401 | INFO | Perf report for the last 5.13948 seconds: | 20260508 03:14:57.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:14:57.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 03:14:57.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:14:57.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:14:57.401 | INFO | Uploaded 36GiB to GPU in 2.39963s (15.0023GiB/s) | 20260508 03:14:57.401 | INFO | Downloaded 34GiB to GPU in 2.59012s (13.1268GiB/s) | 20260508 03:15:02.534 | INFO | Analyzing chunk frames = [4798976,4799488) | 20260508 03:15:02.534 | INFO | MemInfo: { | 20260508 03:15:02.534 | INFO | "gpu" : " 17 GB", | 20260508 03:15:02.534 | INFO | "huge" : "", | 20260508 03:15:02.534 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:02.534 | INFO | "malloc" : " 14 GB", | 20260508 03:15:02.534 | INFO | "pinned" : " 29 GB" | 20260508 03:15:02.534 | INFO | } | 20260508 03:15:02.534 | INFO | Perf report for the last 5.13277 seconds: | 20260508 03:15:02.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:02.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:15:02.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 03:15:02.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:02.534 | INFO | Uploaded 36GiB to GPU in 2.40577s (14.964GiB/s) | 20260508 03:15:02.534 | INFO | Downloaded 34GiB to GPU in 2.42721s (14.0079GiB/s) | 20260508 03:15:07.679 | INFO | Analyzing chunk frames = [4799488,4800000) | 20260508 03:15:07.679 | INFO | MemInfo: { | 20260508 03:15:07.679 | INFO | "gpu" : " 17 GB", | 20260508 03:15:07.679 | INFO | "huge" : "", | 20260508 03:15:07.679 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:07.679 | INFO | "malloc" : " 14 GB", | 20260508 03:15:07.679 | INFO | "pinned" : " 29 GB" | 20260508 03:15:07.679 | INFO | } | 20260508 03:15:07.679 | INFO | Perf report for the last 5.1452 seconds: | 20260508 03:15:07.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:07.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.679 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:15:07.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.679 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 03:15:07.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:07.680 | INFO | Uploaded 36GiB to GPU in 2.47255s (14.5599GiB/s) | 20260508 03:15:07.680 | INFO | Downloaded 34GiB to GPU in 2.44001s (13.9344GiB/s) | 20260508 03:15:12.820 | INFO | Analyzing chunk frames = [4800000,4800512) | 20260508 03:15:12.820 | INFO | MemInfo: { | 20260508 03:15:12.820 | INFO | "gpu" : " 17 GB", | 20260508 03:15:12.820 | INFO | "huge" : "", | 20260508 03:15:12.820 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:12.820 | INFO | "malloc" : " 14 GB", | 20260508 03:15:12.820 | INFO | "pinned" : " 29 GB" | 20260508 03:15:12.820 | INFO | } | 20260508 03:15:12.821 | INFO | Perf report for the last 5.14146 seconds: | 20260508 03:15:12.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:12.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:15:12.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | PRE_HQ : 34.2%, 1.0, 17.7ms, 0 | 20260508 03:15:12.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:12.821 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:15:12.821 | INFO | Total mem usage: 16.7 GiB | 20260508 03:15:12.821 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:15:12.821 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:15:12.821 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:15:12.821 | INFO | High Water Breakdown: | 20260508 03:15:12.821 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:15:12.821 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:15:12.821 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:15:12.821 | INFO | ------------------------------------------ | 20260508 03:15:12.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:15:12.821 | INFO | Uploaded 36GiB to GPU in 2.45966s (14.6362GiB/s) | 20260508 03:15:12.821 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:15:12.821 | INFO | Downloaded 34GiB to GPU in 2.4359s (13.9579GiB/s) | 20260508 03:15:12.821 | INFO | Total mem usage: 26.5 GiB | 20260508 03:15:12.821 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:15:12.821 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:15:12.821 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:15:12.821 | INFO | High Water Breakdown: | 20260508 03:15:12.821 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:15:12.821 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:15:12.821 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:15:12.821 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:15:12.821 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:15:12.821 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:15:12.821 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:15:12.821 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:15:12.821 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:15:12.822 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:15:12.822 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:15:12.822 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:15:12.822 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:15:12.822 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:15:12.822 | INFO | ------------------------------------------ | 20260508 03:15:12.822 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:15:12.822 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:15:12.822 | INFO | Total mem usage: 27.8 GiB | 20260508 03:15:12.822 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:15:12.822 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:15:12.822 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:15:12.822 | INFO | High Water Breakdown: | 20260508 03:15:12.822 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:15:12.822 | INFO | ------------------------------------------ | 20260508 03:15:12.822 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:15:12.822 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:15:12.822 | INFO | Total mem usage: 15.7 GiB | 20260508 03:15:12.822 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:15:12.822 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:15:12.822 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:15:12.822 | INFO | High Water Breakdown: | 20260508 03:15:12.822 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:15:12.822 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:15:12.822 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:15:12.822 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:15:12.822 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:15:12.822 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:15:12.822 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:15:12.822 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:15:12.822 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:15:12.822 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:15:12.822 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:15:12.822 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:15:12.822 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:15:12.822 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:15:12.822 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:15:12.822 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:15:12.822 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:15:12.822 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:15:12.822 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:15:12.822 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:15:12.822 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:15:12.822 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:15:12.822 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:15:12.822 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:15:12.822 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:15:12.822 | INFO | ------------------------------------------ | 20260508 03:15:12.822 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:15:12.822 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:15:12.822 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:15:12.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:15:12.822 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.41e+01 / 1.67e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.12e-02 / 6.38e-03 | 20260508 03:15:12.822 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.42e+00, 5.81e-01 / 2.23e-01 | 20260508 03:15:12.822 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.01e-01 | 20260508 03:15:12.822 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.59e+01, 3.97e+00 / 2.16e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.71e+00 / 1.46e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.64e+00, 4.20e-01 / 2.43e-01 | 20260508 03:15:12.822 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.24e+00, 2.67e-01 / 1.37e-01 | 20260508 03:15:12.822 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.58e-01, 5.14e-02 / 2.31e-02 | 20260508 03:15:12.822 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:15:12.822 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:15:12.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:15:12.822 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 24.24%, Upload -- 2.58e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 24.48%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 2.01%, LockUpload -- 2.14e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 2.90%, LockDownload -- 3.08e+00 / 0.00e+00, 4.37e+00 / 0.00e+00 | 20260508 03:15:12.822 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:15:12.823 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260508 03:15:12.933 | INFO | Starting a round of Baz IO | 20260508 03:15:16.359 | INFO | Populating RTMetrics report | 20260508 03:15:16.359 | INFO | RTMetrics report populated | 20260508 03:15:16.359 | INFO | Opening tmp RTMetrics json file | 20260508 03:15:16.374 | INFO | Serializing RTMetrics to json | 20260508 03:15:16.375 | INFO | Serialization done, swapping tmp file now... | 20260508 03:15:16.376 | INFO | RTMetrics done | 20260508 03:15:16.428 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.494 seconds | 20260508 03:15:16.428 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.33% / 0.00% / 58.67% | 20260508 03:15:16.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.636009% / 7.94729e-06% / 4.76837e-05% | 20260508 03:15:16.429 | INFO | Baz IO overall: | 20260508 03:15:16.429 | INFO | BytesWritten : 2160386035045 | 20260508 03:15:16.429 | INFO | NumEvents : 1043859967736 | 20260508 03:15:17.963 | INFO | Analyzing chunk frames = [4800512,4801024) | 20260508 03:15:17.963 | INFO | MemInfo: { | 20260508 03:15:17.963 | INFO | "gpu" : " 17 GB", | 20260508 03:15:17.963 | INFO | "huge" : "", | 20260508 03:15:17.963 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:17.963 | INFO | "malloc" : " 14 GB", | 20260508 03:15:17.963 | INFO | "pinned" : " 29 GB" | 20260508 03:15:17.963 | INFO | } | 20260508 03:15:17.963 | INFO | Perf report for the last 5.14235 seconds: | 20260508 03:15:17.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:17.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:17.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:17.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:17.963 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260508 03:15:17.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:17.963 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:15:17.963 | INFO | PRE_HQ : 66.6%, 1.8, 62.4ms, 0 | 20260508 03:15:17.963 | INFO | BAZWRITER : 67.3%, 2.9, 99.0ms, 0 | 20260508 03:15:17.964 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:15:17.964 | INFO | Uploaded 36GiB to GPU in 2.53566s (14.1975GiB/s) | 20260508 03:15:17.964 | INFO | Downloaded 38GiB to GPU in 2.69313s (14.11GiB/s) | 20260508 03:15:23.102 | INFO | Analyzing chunk frames = [4801024,4801536) | 20260508 03:15:23.102 | INFO | MemInfo: { | 20260508 03:15:23.102 | INFO | "gpu" : " 17 GB", | 20260508 03:15:23.102 | INFO | "huge" : "", | 20260508 03:15:23.102 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:23.102 | INFO | "malloc" : " 14 GB", | 20260508 03:15:23.102 | INFO | "pinned" : " 29 GB" | 20260508 03:15:23.102 | INFO | } | 20260508 03:15:23.102 | INFO | Perf report for the last 5.13944 seconds: | 20260508 03:15:23.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:23.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.102 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:15:23.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.102 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:15:23.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:23.103 | INFO | Uploaded 36GiB to GPU in 2.5696s (14.0099GiB/s) | 20260508 03:15:23.103 | INFO | Downloaded 34GiB to GPU in 2.43685s (13.9524GiB/s) | 20260508 03:15:28.236 | INFO | Analyzing chunk frames = [4801536,4802048) | 20260508 03:15:28.236 | INFO | MemInfo: { | 20260508 03:15:28.236 | INFO | "gpu" : " 17 GB", | 20260508 03:15:28.236 | INFO | "huge" : "", | 20260508 03:15:28.237 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:28.237 | INFO | "malloc" : " 14 GB", | 20260508 03:15:28.237 | INFO | "pinned" : " 29 GB" | 20260508 03:15:28.237 | INFO | } | 20260508 03:15:28.237 | INFO | Perf report for the last 5.13444 seconds: | 20260508 03:15:28.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:28.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 03:15:28.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:15:28.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:28.237 | INFO | Uploaded 36GiB to GPU in 2.5399s (14.1738GiB/s) | 20260508 03:15:28.237 | INFO | Downloaded 34GiB to GPU in 2.4283s (14.0016GiB/s) | 20260508 03:15:33.377 | INFO | Analyzing chunk frames = [4802048,4802560) | 20260508 03:15:33.377 | INFO | MemInfo: { | 20260508 03:15:33.377 | INFO | "gpu" : " 17 GB", | 20260508 03:15:33.377 | INFO | "huge" : "", | 20260508 03:15:33.377 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:33.377 | INFO | "malloc" : " 14 GB", | 20260508 03:15:33.377 | INFO | "pinned" : " 29 GB" | 20260508 03:15:33.377 | INFO | } | 20260508 03:15:33.377 | INFO | Perf report for the last 5.14053 seconds: | 20260508 03:15:33.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:33.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.377 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:15:33.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.377 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:15:33.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:33.378 | INFO | Uploaded 36GiB to GPU in 2.52773s (14.242GiB/s) | 20260508 03:15:33.378 | INFO | Downloaded 34GiB to GPU in 2.5088s (13.5523GiB/s) | 20260508 03:15:38.515 | INFO | Analyzing chunk frames = [4802560,4803072) | 20260508 03:15:38.515 | INFO | MemInfo: { | 20260508 03:15:38.515 | INFO | "gpu" : " 17 GB", | 20260508 03:15:38.515 | INFO | "huge" : "", | 20260508 03:15:38.515 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:38.515 | INFO | "malloc" : " 14 GB", | 20260508 03:15:38.515 | INFO | "pinned" : " 29 GB" | 20260508 03:15:38.515 | INFO | } | 20260508 03:15:38.515 | INFO | Perf report for the last 5.13775 seconds: | 20260508 03:15:38.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:38.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.515 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 03:15:38.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.515 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 03:15:38.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:38.516 | INFO | Uploaded 36GiB to GPU in 2.35519s (15.2854GiB/s) | 20260508 03:15:38.516 | INFO | Downloaded 34GiB to GPU in 2.54245s (13.373GiB/s) | 20260508 03:15:43.655 | INFO | Analyzing chunk frames = [4803072,4803584) | 20260508 03:15:43.655 | INFO | MemInfo: { | 20260508 03:15:43.655 | INFO | "gpu" : " 17 GB", | 20260508 03:15:43.655 | INFO | "huge" : "", | 20260508 03:15:43.655 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:43.655 | INFO | "malloc" : " 14 GB", | 20260508 03:15:43.655 | INFO | "pinned" : " 29 GB" | 20260508 03:15:43.655 | INFO | } | 20260508 03:15:43.655 | INFO | Perf report for the last 5.14016 seconds: | 20260508 03:15:43.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:43.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.655 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:15:43.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.655 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:15:43.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:43.656 | INFO | Uploaded 36GiB to GPU in 2.39369s (15.0395GiB/s) | 20260508 03:15:43.656 | INFO | Downloaded 34GiB to GPU in 2.44485s (13.9068GiB/s) | 20260508 03:15:48.790 | INFO | Analyzing chunk frames = [4803584,4804096) | 20260508 03:15:48.790 | INFO | MemInfo: { | 20260508 03:15:48.790 | INFO | "gpu" : " 17 GB", | 20260508 03:15:48.790 | INFO | "huge" : "", | 20260508 03:15:48.791 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:48.791 | INFO | "malloc" : " 14 GB", | 20260508 03:15:48.791 | INFO | "pinned" : " 29 GB" | 20260508 03:15:48.791 | INFO | } | 20260508 03:15:48.791 | INFO | Perf report for the last 5.13547 seconds: | 20260508 03:15:48.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:48.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 03:15:48.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:15:48.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:48.791 | INFO | Uploaded 36GiB to GPU in 2.42866s (14.823GiB/s) | 20260508 03:15:48.791 | INFO | Downloaded 34GiB to GPU in 2.45093s (13.8723GiB/s) | 20260508 03:15:53.930 | INFO | Analyzing chunk frames = [4804096,4804608) | 20260508 03:15:53.930 | INFO | MemInfo: { | 20260508 03:15:53.930 | INFO | "gpu" : " 17 GB", | 20260508 03:15:53.930 | INFO | "huge" : "", | 20260508 03:15:53.930 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:53.930 | INFO | "malloc" : " 14 GB", | 20260508 03:15:53.930 | INFO | "pinned" : " 29 GB" | 20260508 03:15:53.930 | INFO | } | 20260508 03:15:53.930 | INFO | Perf report for the last 5.13953 seconds: | 20260508 03:15:53.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:53.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:15:53.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:15:53.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:53.931 | INFO | Uploaded 36GiB to GPU in 2.44637s (14.7157GiB/s) | 20260508 03:15:53.931 | INFO | Downloaded 34GiB to GPU in 2.44693s (13.8949GiB/s) | 20260508 03:15:54.045 | INFO | Starting a round of Baz IO | 20260508 03:15:57.481 | INFO | Populating RTMetrics report | 20260508 03:15:57.481 | INFO | RTMetrics report populated | 20260508 03:15:57.481 | INFO | Opening tmp RTMetrics json file | 20260508 03:15:57.493 | INFO | Serializing RTMetrics to json | 20260508 03:15:57.494 | INFO | Serialization done, swapping tmp file now... | 20260508 03:15:57.495 | INFO | RTMetrics done | 20260508 03:15:57.551 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.506 seconds | 20260508 03:15:57.551 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.33% / 0.00% / 58.67% | 20260508 03:15:57.551 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.63804% / 3.97364e-06% / 3.57628e-05% | 20260508 03:15:57.551 | INFO | Baz IO overall: | 20260508 03:15:57.551 | INFO | BytesWritten : 2162125380961 | 20260508 03:15:57.551 | INFO | NumEvents : 1044692893998 | 20260508 03:15:59.073 | INFO | Analyzing chunk frames = [4804608,4805120) | 20260508 03:15:59.073 | INFO | MemInfo: { | 20260508 03:15:59.073 | INFO | "gpu" : " 17 GB", | 20260508 03:15:59.073 | INFO | "huge" : "", | 20260508 03:15:59.073 | INFO | "huge_shared" : " 29 GB", | 20260508 03:15:59.073 | INFO | "malloc" : " 14 GB", | 20260508 03:15:59.073 | INFO | "pinned" : " 29 GB" | 20260508 03:15:59.073 | INFO | } | 20260508 03:15:59.073 | INFO | Perf report for the last 5.14277 seconds: | 20260508 03:15:59.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:15:59.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:59.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:59.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:59.073 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 03:15:59.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:15:59.073 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:15:59.073 | INFO | PRE_HQ : 66.3%, 1.8, 62.5ms, 0 | 20260508 03:15:59.073 | INFO | BAZWRITER : 67.6%, 2.8, 97.9ms, 0 | 20260508 03:15:59.073 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 03:15:59.073 | INFO | Uploaded 36GiB to GPU in 2.55423s (14.0943GiB/s) | 20260508 03:15:59.073 | INFO | Downloaded 38GiB to GPU in 2.68732s (14.1405GiB/s) | 20260508 03:16:04.213 | INFO | Analyzing chunk frames = [4805120,4805632) | 20260508 03:16:04.213 | INFO | MemInfo: { | 20260508 03:16:04.213 | INFO | "gpu" : " 17 GB", | 20260508 03:16:04.213 | INFO | "huge" : "", | 20260508 03:16:04.213 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:04.213 | INFO | "malloc" : " 14 GB", | 20260508 03:16:04.213 | INFO | "pinned" : " 29 GB" | 20260508 03:16:04.213 | INFO | } | 20260508 03:16:04.213 | INFO | Perf report for the last 5.13999 seconds: | 20260508 03:16:04.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:04.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:16:04.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:16:04.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:04.213 | INFO | Uploaded 36GiB to GPU in 2.55206s (14.1062GiB/s) | 20260508 03:16:04.213 | INFO | Downloaded 34GiB to GPU in 2.42543s (14.0181GiB/s) | 20260508 03:16:09.347 | INFO | Analyzing chunk frames = [4805632,4806144) | 20260508 03:16:09.347 | INFO | MemInfo: { | 20260508 03:16:09.347 | INFO | "gpu" : " 17 GB", | 20260508 03:16:09.347 | INFO | "huge" : "", | 20260508 03:16:09.347 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:09.347 | INFO | "malloc" : " 14 GB", | 20260508 03:16:09.347 | INFO | "pinned" : " 29 GB" | 20260508 03:16:09.347 | INFO | } | 20260508 03:16:09.347 | INFO | Perf report for the last 5.13409 seconds: | 20260508 03:16:09.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:09.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:16:09.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:16:09.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:09.347 | INFO | Uploaded 36GiB to GPU in 2.55065s (14.1141GiB/s) | 20260508 03:16:09.347 | INFO | Downloaded 34GiB to GPU in 2.49632s (13.6201GiB/s) | 20260508 03:16:14.487 | INFO | Analyzing chunk frames = [4806144,4806656) | 20260508 03:16:14.487 | INFO | MemInfo: { | 20260508 03:16:14.487 | INFO | "gpu" : " 17 GB", | 20260508 03:16:14.487 | INFO | "huge" : "", | 20260508 03:16:14.487 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:14.487 | INFO | "malloc" : " 14 GB", | 20260508 03:16:14.487 | INFO | "pinned" : " 29 GB" | 20260508 03:16:14.487 | INFO | } | 20260508 03:16:14.487 | INFO | Perf report for the last 5.13989 seconds: | 20260508 03:16:14.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:14.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:16:14.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:16:14.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:14.487 | INFO | Uploaded 36GiB to GPU in 2.54228s (14.1605GiB/s) | 20260508 03:16:14.487 | INFO | Downloaded 34GiB to GPU in 2.49901s (13.6054GiB/s) | 20260508 03:16:19.629 | INFO | Analyzing chunk frames = [4806656,4807168) | 20260508 03:16:19.629 | INFO | MemInfo: { | 20260508 03:16:19.629 | INFO | "gpu" : " 17 GB", | 20260508 03:16:19.629 | INFO | "huge" : "", | 20260508 03:16:19.629 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:19.629 | INFO | "malloc" : " 14 GB", | 20260508 03:16:19.629 | INFO | "pinned" : " 29 GB" | 20260508 03:16:19.629 | INFO | } | 20260508 03:16:19.629 | INFO | Perf report for the last 5.14209 seconds: | 20260508 03:16:19.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:19.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 03:16:19.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:16:19.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:19.629 | INFO | Uploaded 36GiB to GPU in 2.42694s (14.8335GiB/s) | 20260508 03:16:19.629 | INFO | Downloaded 34GiB to GPU in 2.56259s (13.2678GiB/s) | 20260508 03:16:24.771 | INFO | Analyzing chunk frames = [4807168,4807680) | 20260508 03:16:24.771 | INFO | MemInfo: { | 20260508 03:16:24.771 | INFO | "gpu" : " 17 GB", | 20260508 03:16:24.771 | INFO | "huge" : "", | 20260508 03:16:24.771 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:24.771 | INFO | "malloc" : " 14 GB", | 20260508 03:16:24.771 | INFO | "pinned" : " 29 GB" | 20260508 03:16:24.771 | INFO | } | 20260508 03:16:24.772 | INFO | Perf report for the last 5.14234 seconds: | 20260508 03:16:24.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:24.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 03:16:24.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:16:24.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:24.772 | INFO | Uploaded 36GiB to GPU in 2.45208s (14.6814GiB/s) | 20260508 03:16:24.772 | INFO | Downloaded 34GiB to GPU in 2.40802s (14.1195GiB/s) | 20260508 03:16:29.910 | INFO | Analyzing chunk frames = [4807680,4808192) | 20260508 03:16:29.910 | INFO | MemInfo: { | 20260508 03:16:29.910 | INFO | "gpu" : " 17 GB", | 20260508 03:16:29.910 | INFO | "huge" : "", | 20260508 03:16:29.910 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:29.910 | INFO | "malloc" : " 14 GB", | 20260508 03:16:29.910 | INFO | "pinned" : " 29 GB" | 20260508 03:16:29.911 | INFO | } | 20260508 03:16:29.911 | INFO | Perf report for the last 5.13907 seconds: | 20260508 03:16:29.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:29.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:16:29.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 03:16:29.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:29.911 | INFO | Uploaded 36GiB to GPU in 2.47194s (14.5635GiB/s) | 20260508 03:16:29.911 | INFO | Downloaded 34GiB to GPU in 2.42016s (14.0486GiB/s) | 20260508 03:16:35.047 | INFO | Analyzing chunk frames = [4808192,4808704) | 20260508 03:16:35.047 | INFO | MemInfo: { | 20260508 03:16:35.047 | INFO | "gpu" : " 17 GB", | 20260508 03:16:35.047 | INFO | "huge" : "", | 20260508 03:16:35.047 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:35.047 | INFO | "malloc" : " 14 GB", | 20260508 03:16:35.047 | INFO | "pinned" : " 29 GB" | 20260508 03:16:35.047 | INFO | } | 20260508 03:16:35.047 | INFO | Perf report for the last 5.13652 seconds: | 20260508 03:16:35.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:35.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.047 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:16:35.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.047 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 03:16:35.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:35.048 | INFO | Uploaded 36GiB to GPU in 2.47102s (14.5689GiB/s) | 20260508 03:16:35.048 | INFO | Downloaded 34GiB to GPU in 2.4202s (14.0484GiB/s) | 20260508 03:16:35.048 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:16:35.048 | INFO | Total mem usage: 16.7 GiB | 20260508 03:16:35.048 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:16:35.048 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.3%) | 20260508 03:16:35.048 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:16:35.048 | INFO | High Water Breakdown: | 20260508 03:16:35.048 | INFO | 59.6% : 8.63 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:16:35.048 | INFO | 40.2% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:16:35.048 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:16:35.048 | INFO | ------------------------------------------ | 20260508 03:16:35.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:16:35.048 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:16:35.048 | INFO | Total mem usage: 26.5 GiB | 20260508 03:16:35.048 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:16:35.048 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:16:35.048 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:16:35.048 | INFO | High Water Breakdown: | 20260508 03:16:35.048 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:16:35.048 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:16:35.048 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:16:35.048 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:16:35.048 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:16:35.048 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:16:35.048 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:16:35.048 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:16:35.048 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:16:35.048 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:16:35.048 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:16:35.048 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:16:35.048 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:16:35.048 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:16:35.048 | INFO | ------------------------------------------ | 20260508 03:16:35.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:16:35.048 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:16:35.048 | INFO | Total mem usage: 27.8 GiB | 20260508 03:16:35.048 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:16:35.048 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:16:35.048 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:16:35.048 | INFO | High Water Breakdown: | 20260508 03:16:35.048 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:16:35.048 | INFO | ------------------------------------------ | 20260508 03:16:35.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:16:35.048 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:16:35.048 | INFO | Total mem usage: 15.7 GiB | 20260508 03:16:35.048 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:16:35.048 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:16:35.048 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:16:35.048 | INFO | High Water Breakdown: | 20260508 03:16:35.048 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:16:35.048 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:16:35.048 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:16:35.048 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:16:35.048 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:16:35.048 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:16:35.048 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:16:35.048 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:16:35.048 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:16:35.048 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:16:35.048 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:16:35.048 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:16:35.048 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:16:35.048 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:16:35.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:16:35.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:16:35.048 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:16:35.048 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:16:35.048 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:16:35.048 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:16:35.048 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:16:35.048 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:16:35.048 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:16:35.048 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:16:35.048 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:16:35.048 | INFO | ------------------------------------------ | 20260508 03:16:35.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:16:35.048 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:16:35.048 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:16:35.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:16:35.048 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.75e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.27e-02 / 5.26e-03 | 20260508 03:16:35.048 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.45e+00, 5.87e-01 / 2.35e-01 | 20260508 03:16:35.048 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.49e+00, 2.07e-01 / 9.44e-02 | 20260508 03:16:35.048 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.61e+01, 3.89e+00 / 1.93e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.67e+00 / 1.45e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.19e-01 / 2.38e-01 | 20260508 03:16:35.048 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.24e+00, 2.66e-01 / 1.57e-01 | 20260508 03:16:35.048 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.63e-01, 5.17e-02 / 2.10e-02 | 20260508 03:16:35.048 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:16:35.048 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:16:35.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:16:35.048 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 2.18%, LockUpload -- 2.32e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 2.80%, LockDownload -- 2.97e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 15.49%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 03:16:35.048 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260508 03:16:35.161 | INFO | Starting a round of Baz IO | 20260508 03:16:38.598 | INFO | Populating RTMetrics report | 20260508 03:16:38.598 | INFO | RTMetrics report populated | 20260508 03:16:38.598 | INFO | Opening tmp RTMetrics json file | 20260508 03:16:38.614 | INFO | Serializing RTMetrics to json | 20260508 03:16:38.615 | INFO | Serialization done, swapping tmp file now... | 20260508 03:16:38.616 | INFO | RTMetrics done | 20260508 03:16:38.669 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.508 seconds | 20260508 03:16:38.669 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.32% / 0.00% / 58.67% | 20260508 03:16:38.669 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.636498% / 7.94729e-06% / 3.17891e-05% | 20260508 03:16:38.669 | INFO | Baz IO overall: | 20260508 03:16:38.669 | INFO | BytesWritten : 2163865020761 | 20260508 03:16:38.669 | INFO | NumEvents : 1045526142877 | 20260508 03:16:40.188 | INFO | Analyzing chunk frames = [4808704,4809216) | 20260508 03:16:40.188 | INFO | MemInfo: { | 20260508 03:16:40.188 | INFO | "gpu" : " 17 GB", | 20260508 03:16:40.188 | INFO | "huge" : "", | 20260508 03:16:40.188 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:40.188 | INFO | "malloc" : " 14 GB", | 20260508 03:16:40.188 | INFO | "pinned" : " 29 GB" | 20260508 03:16:40.188 | INFO | } | 20260508 03:16:40.188 | INFO | Perf report for the last 5.14097 seconds: | 20260508 03:16:40.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:40.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:40.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:40.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:40.188 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 03:16:40.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:40.188 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 03:16:40.188 | INFO | PRE_HQ : 66.3%, 1.8, 62.1ms, 0 | 20260508 03:16:40.188 | INFO | BAZWRITER : 67.7%, 2.9, 100.8ms, 0 | 20260508 03:16:40.188 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:16:40.188 | INFO | Uploaded 36GiB to GPU in 2.58547s (13.9239GiB/s) | 20260508 03:16:40.188 | INFO | Downloaded 38GiB to GPU in 2.67871s (14.1859GiB/s) | 20260508 03:16:45.325 | INFO | Analyzing chunk frames = [4809216,4809728) | 20260508 03:16:45.325 | INFO | MemInfo: { | 20260508 03:16:45.325 | INFO | "gpu" : " 17 GB", | 20260508 03:16:45.325 | INFO | "huge" : "", | 20260508 03:16:45.325 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:45.325 | INFO | "malloc" : " 14 GB", | 20260508 03:16:45.325 | INFO | "pinned" : " 29 GB" | 20260508 03:16:45.325 | INFO | } | 20260508 03:16:45.325 | INFO | Perf report for the last 5.1368 seconds: | 20260508 03:16:45.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:45.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:16:45.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:16:45.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:45.325 | INFO | Uploaded 36GiB to GPU in 2.54545s (14.1429GiB/s) | 20260508 03:16:45.325 | INFO | Downloaded 34GiB to GPU in 2.43086s (13.9868GiB/s) | 20260508 03:16:50.467 | INFO | Analyzing chunk frames = [4809728,4810240) | 20260508 03:16:50.467 | INFO | MemInfo: { | 20260508 03:16:50.467 | INFO | "gpu" : " 17 GB", | 20260508 03:16:50.467 | INFO | "huge" : "", | 20260508 03:16:50.467 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:50.467 | INFO | "malloc" : " 14 GB", | 20260508 03:16:50.467 | INFO | "pinned" : " 29 GB" | 20260508 03:16:50.467 | INFO | } | 20260508 03:16:50.467 | INFO | Perf report for the last 5.1423 seconds: | 20260508 03:16:50.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:50.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.467 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:16:50.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.467 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:16:50.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:50.468 | INFO | Uploaded 36GiB to GPU in 2.55529s (14.0884GiB/s) | 20260508 03:16:50.468 | INFO | Downloaded 34GiB to GPU in 2.50071s (13.5961GiB/s) | 20260508 03:16:55.601 | INFO | Analyzing chunk frames = [4810240,4810752) | 20260508 03:16:55.601 | INFO | MemInfo: { | 20260508 03:16:55.601 | INFO | "gpu" : " 17 GB", | 20260508 03:16:55.601 | INFO | "huge" : "", | 20260508 03:16:55.601 | INFO | "huge_shared" : " 29 GB", | 20260508 03:16:55.601 | INFO | "malloc" : " 14 GB", | 20260508 03:16:55.601 | INFO | "pinned" : " 29 GB" | 20260508 03:16:55.601 | INFO | } | 20260508 03:16:55.601 | INFO | Perf report for the last 5.1341 seconds: | 20260508 03:16:55.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:16:55.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.601 | INFO | ANALYSIS : 67.3%, 3.0, 104.0ms, 0 | 20260508 03:16:55.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.601 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 03:16:55.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:16:55.602 | INFO | Uploaded 36GiB to GPU in 2.53811s (14.1838GiB/s) | 20260508 03:16:55.602 | INFO | Downloaded 34GiB to GPU in 2.49864s (13.6074GiB/s) | 20260508 03:17:00.740 | INFO | Analyzing chunk frames = [4810752,4811264) | 20260508 03:17:00.740 | INFO | MemInfo: { | 20260508 03:17:00.740 | INFO | "gpu" : " 17 GB", | 20260508 03:17:00.740 | INFO | "huge" : "", | 20260508 03:17:00.741 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:00.741 | INFO | "malloc" : " 14 GB", | 20260508 03:17:00.741 | INFO | "pinned" : " 29 GB" | 20260508 03:17:00.741 | INFO | } | 20260508 03:17:00.741 | INFO | Perf report for the last 5.13932 seconds: | 20260508 03:17:00.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:00.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 03:17:00.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:17:00.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:00.741 | INFO | Uploaded 36GiB to GPU in 2.40671s (14.9582GiB/s) | 20260508 03:17:00.741 | INFO | Downloaded 34GiB to GPU in 2.57037s (13.2277GiB/s) | 20260508 03:17:05.884 | INFO | Analyzing chunk frames = [4811264,4811776) | 20260508 03:17:05.884 | INFO | MemInfo: { | 20260508 03:17:05.884 | INFO | "gpu" : " 17 GB", | 20260508 03:17:05.884 | INFO | "huge" : "", | 20260508 03:17:05.884 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:05.884 | INFO | "malloc" : " 14 GB", | 20260508 03:17:05.884 | INFO | "pinned" : " 29 GB" | 20260508 03:17:05.884 | INFO | } | 20260508 03:17:05.884 | INFO | Perf report for the last 5.14318 seconds: | 20260508 03:17:05.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:05.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 03:17:05.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 03:17:05.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:05.884 | INFO | Uploaded 36GiB to GPU in 2.37846s (15.1358GiB/s) | 20260508 03:17:05.884 | INFO | Downloaded 34GiB to GPU in 2.44729s (13.8929GiB/s) | 20260508 03:17:11.025 | INFO | Analyzing chunk frames = [4811776,4812288) | 20260508 03:17:11.025 | INFO | MemInfo: { | 20260508 03:17:11.025 | INFO | "gpu" : " 17 GB", | 20260508 03:17:11.025 | INFO | "huge" : "", | 20260508 03:17:11.025 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:11.025 | INFO | "malloc" : " 14 GB", | 20260508 03:17:11.025 | INFO | "pinned" : " 29 GB" | 20260508 03:17:11.025 | INFO | } | 20260508 03:17:11.025 | INFO | Perf report for the last 5.14148 seconds: | 20260508 03:17:11.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:11.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.025 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:17:11.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.026 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 03:17:11.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:11.026 | INFO | Uploaded 36GiB to GPU in 2.40158s (14.9902GiB/s) | 20260508 03:17:11.026 | INFO | Downloaded 34GiB to GPU in 2.43662s (13.9538GiB/s) | 20260508 03:17:16.162 | INFO | Analyzing chunk frames = [4812288,4812800) | 20260508 03:17:16.162 | INFO | MemInfo: { | 20260508 03:17:16.162 | INFO | "gpu" : " 17 GB", | 20260508 03:17:16.162 | INFO | "huge" : "", | 20260508 03:17:16.162 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:16.163 | INFO | "malloc" : " 14 GB", | 20260508 03:17:16.163 | INFO | "pinned" : " 29 GB" | 20260508 03:17:16.163 | INFO | } | 20260508 03:17:16.163 | INFO | Perf report for the last 5.13723 seconds: | 20260508 03:17:16.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:16.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:17:16.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 03:17:16.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:16.163 | INFO | Uploaded 36GiB to GPU in 2.44389s (14.7306GiB/s) | 20260508 03:17:16.163 | INFO | Downloaded 34GiB to GPU in 2.44555s (13.9028GiB/s) | 20260508 03:17:16.260 | INFO | Starting a round of Baz IO | 20260508 03:17:19.707 | INFO | Populating RTMetrics report | 20260508 03:17:19.708 | INFO | RTMetrics report populated | 20260508 03:17:19.708 | INFO | Opening tmp RTMetrics json file | 20260508 03:17:19.718 | INFO | Serializing RTMetrics to json | 20260508 03:17:19.720 | INFO | Serialization done, swapping tmp file now... | 20260508 03:17:19.720 | INFO | RTMetrics done | 20260508 03:17:19.779 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.519 seconds | 20260508 03:17:19.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.32% / 0.00% / 58.68% | 20260508 03:17:19.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.634452% / 3.97364e-06% / 6.35783e-05% | 20260508 03:17:19.779 | INFO | Baz IO overall: | 20260508 03:17:19.779 | INFO | BytesWritten : 2165602879276 | 20260508 03:17:19.779 | INFO | NumEvents : 1046358121553 | 20260508 03:17:21.298 | INFO | Analyzing chunk frames = [4812800,4813312) | 20260508 03:17:21.298 | INFO | MemInfo: { | 20260508 03:17:21.298 | INFO | "gpu" : " 17 GB", | 20260508 03:17:21.298 | INFO | "huge" : "", | 20260508 03:17:21.298 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:21.298 | INFO | "malloc" : " 14 GB", | 20260508 03:17:21.298 | INFO | "pinned" : " 29 GB" | 20260508 03:17:21.298 | INFO | } | 20260508 03:17:21.298 | INFO | Perf report for the last 5.13529 seconds: | 20260508 03:17:21.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:21.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:21.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:21.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:21.298 | INFO | ANALYSIS : 68.2%, 3.0, 106.5ms, 0 | 20260508 03:17:21.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:21.298 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:17:21.298 | INFO | PRE_HQ : 65.9%, 1.6, 55.9ms, 0 | 20260508 03:17:21.298 | INFO | BAZWRITER : 68.0%, 2.6, 91.8ms, 0 | 20260508 03:17:21.298 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:17:21.298 | INFO | Uploaded 36GiB to GPU in 2.56536s (14.0331GiB/s) | 20260508 03:17:21.299 | INFO | Downloaded 38GiB to GPU in 2.69846s (14.0821GiB/s) | 20260508 03:17:26.441 | INFO | Analyzing chunk frames = [4813312,4813824) | 20260508 03:17:26.441 | INFO | MemInfo: { | 20260508 03:17:26.441 | INFO | "gpu" : " 17 GB", | 20260508 03:17:26.441 | INFO | "huge" : "", | 20260508 03:17:26.441 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:26.442 | INFO | "malloc" : " 14 GB", | 20260508 03:17:26.442 | INFO | "pinned" : " 29 GB" | 20260508 03:17:26.442 | INFO | } | 20260508 03:17:26.442 | INFO | Perf report for the last 5.14373 seconds: | 20260508 03:17:26.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:26.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 03:17:26.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:17:26.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:26.442 | INFO | Uploaded 36GiB to GPU in 2.54375s (14.1523GiB/s) | 20260508 03:17:26.442 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9295GiB/s) | 20260508 03:17:31.581 | INFO | Analyzing chunk frames = [4813824,4814336) | 20260508 03:17:31.581 | INFO | MemInfo: { | 20260508 03:17:31.581 | INFO | "gpu" : " 17 GB", | 20260508 03:17:31.581 | INFO | "huge" : "", | 20260508 03:17:31.581 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:31.581 | INFO | "malloc" : " 14 GB", | 20260508 03:17:31.581 | INFO | "pinned" : " 29 GB" | 20260508 03:17:31.581 | INFO | } | 20260508 03:17:31.581 | INFO | Perf report for the last 5.13984 seconds: | 20260508 03:17:31.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:31.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:17:31.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:17:31.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:31.582 | INFO | Uploaded 36GiB to GPU in 2.51085s (14.3378GiB/s) | 20260508 03:17:31.582 | INFO | Downloaded 34GiB to GPU in 2.43801s (13.9458GiB/s) | 20260508 03:17:36.725 | INFO | Analyzing chunk frames = [4814336,4814848) | 20260508 03:17:36.725 | INFO | MemInfo: { | 20260508 03:17:36.725 | INFO | "gpu" : " 17 GB", | 20260508 03:17:36.725 | INFO | "huge" : "", | 20260508 03:17:36.725 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:36.725 | INFO | "malloc" : " 14 GB", | 20260508 03:17:36.725 | INFO | "pinned" : " 29 GB" | 20260508 03:17:36.725 | INFO | } | 20260508 03:17:36.725 | INFO | Perf report for the last 5.14372 seconds: | 20260508 03:17:36.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:36.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.725 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 03:17:36.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.725 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:17:36.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:36.726 | INFO | Uploaded 36GiB to GPU in 2.50233s (14.3866GiB/s) | 20260508 03:17:36.726 | INFO | Downloaded 34GiB to GPU in 2.51638s (13.5115GiB/s) | 20260508 03:17:41.860 | INFO | Analyzing chunk frames = [4814848,4815360) | 20260508 03:17:41.860 | INFO | MemInfo: { | 20260508 03:17:41.860 | INFO | "gpu" : " 17 GB", | 20260508 03:17:41.860 | INFO | "huge" : "", | 20260508 03:17:41.860 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:41.860 | INFO | "malloc" : " 14 GB", | 20260508 03:17:41.860 | INFO | "pinned" : " 29 GB" | 20260508 03:17:41.860 | INFO | } | 20260508 03:17:41.860 | INFO | Perf report for the last 5.13476 seconds: | 20260508 03:17:41.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:41.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 03:17:41.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:17:41.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:41.860 | INFO | Uploaded 36GiB to GPU in 2.30355s (15.6281GiB/s) | 20260508 03:17:41.860 | INFO | Downloaded 34GiB to GPU in 2.55861s (13.2884GiB/s) | 20260508 03:17:46.996 | INFO | Analyzing chunk frames = [4815360,4815872) | 20260508 03:17:46.996 | INFO | MemInfo: { | 20260508 03:17:46.996 | INFO | "gpu" : " 17 GB", | 20260508 03:17:46.996 | INFO | "huge" : "", | 20260508 03:17:46.997 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:46.997 | INFO | "malloc" : " 14 GB", | 20260508 03:17:46.997 | INFO | "pinned" : " 29 GB" | 20260508 03:17:46.997 | INFO | } | 20260508 03:17:46.997 | INFO | Perf report for the last 5.13709 seconds: | 20260508 03:17:46.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:46.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 03:17:46.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:17:46.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:46.997 | INFO | Uploaded 36GiB to GPU in 2.39193s (15.0506GiB/s) | 20260508 03:17:46.997 | INFO | Downloaded 34GiB to GPU in 2.45033s (13.8757GiB/s) | 20260508 03:17:52.136 | INFO | Analyzing chunk frames = [4815872,4816384) | 20260508 03:17:52.136 | INFO | MemInfo: { | 20260508 03:17:52.136 | INFO | "gpu" : " 17 GB", | 20260508 03:17:52.136 | INFO | "huge" : "", | 20260508 03:17:52.136 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:52.136 | INFO | "malloc" : " 14 GB", | 20260508 03:17:52.136 | INFO | "pinned" : " 29 GB" | 20260508 03:17:52.136 | INFO | } | 20260508 03:17:52.136 | INFO | Perf report for the last 5.1388 seconds: | 20260508 03:17:52.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:52.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 03:17:52.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260508 03:17:52.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:52.136 | INFO | Uploaded 36GiB to GPU in 2.38376s (15.1022GiB/s) | 20260508 03:17:52.136 | INFO | Downloaded 34GiB to GPU in 2.43326s (13.973GiB/s) | 20260508 03:17:57.279 | INFO | Analyzing chunk frames = [4816384,4816896) | 20260508 03:17:57.279 | INFO | MemInfo: { | 20260508 03:17:57.279 | INFO | "gpu" : " 17 GB", | 20260508 03:17:57.279 | INFO | "huge" : "", | 20260508 03:17:57.279 | INFO | "huge_shared" : " 29 GB", | 20260508 03:17:57.279 | INFO | "malloc" : " 14 GB", | 20260508 03:17:57.279 | INFO | "pinned" : " 29 GB" | 20260508 03:17:57.279 | INFO | } | 20260508 03:17:57.279 | INFO | Perf report for the last 5.14307 seconds: | 20260508 03:17:57.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:17:57.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 03:17:57.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 03:17:57.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:17:57.279 | INFO | Uploaded 36GiB to GPU in 2.52284s (14.2696GiB/s) | 20260508 03:17:57.279 | INFO | Downloaded 34GiB to GPU in 2.47102s (13.7595GiB/s) | 20260508 03:17:57.280 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:17:57.280 | INFO | Total mem usage: 16.7 GiB | 20260508 03:17:57.280 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:17:57.280 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16%) | 20260508 03:17:57.280 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:17:57.280 | INFO | High Water Breakdown: | 20260508 03:17:57.280 | INFO | 59.6% : 8.57 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:17:57.280 | INFO | 40.3% : 5.8 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:17:57.280 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:17:57.280 | INFO | ------------------------------------------ | 20260508 03:17:57.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:17:57.280 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:17:57.280 | INFO | Total mem usage: 26.5 GiB | 20260508 03:17:57.280 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:17:57.280 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:17:57.280 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:17:57.280 | INFO | High Water Breakdown: | 20260508 03:17:57.280 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:17:57.280 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:17:57.280 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:17:57.280 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:17:57.280 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:17:57.280 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:17:57.280 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:17:57.280 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:17:57.280 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:17:57.280 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:17:57.280 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:17:57.280 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:17:57.280 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:17:57.280 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:17:57.280 | INFO | ------------------------------------------ | 20260508 03:17:57.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:17:57.280 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:17:57.280 | INFO | Total mem usage: 27.8 GiB | 20260508 03:17:57.280 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:17:57.280 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:17:57.280 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:17:57.280 | INFO | High Water Breakdown: | 20260508 03:17:57.280 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:17:57.280 | INFO | ------------------------------------------ | 20260508 03:17:57.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:17:57.280 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:17:57.280 | INFO | Total mem usage: 15.7 GiB | 20260508 03:17:57.280 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:17:57.280 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:17:57.280 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:17:57.280 | INFO | High Water Breakdown: | 20260508 03:17:57.280 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:17:57.280 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:17:57.280 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:17:57.280 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:17:57.280 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:17:57.280 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:17:57.280 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:17:57.280 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:17:57.280 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:17:57.280 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:17:57.280 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:17:57.280 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:17:57.280 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:17:57.280 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:17:57.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:17:57.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:17:57.280 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:17:57.280 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:17:57.280 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:17:57.280 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:17:57.280 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:17:57.280 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:17:57.280 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:17:57.280 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:17:57.280 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:17:57.280 | INFO | ------------------------------------------ | 20260508 03:17:57.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:17:57.280 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:17:57.280 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:17:57.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:17:57.280 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.41e+01 / 1.45e+00 | 20260508 03:17:57.280 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.16e-02 / 7.67e-03 | 20260508 03:17:57.280 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.46e+00, 5.84e-01 / 2.01e-01 | 20260508 03:17:57.280 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.48e+00, 2.08e-01 / 1.12e-01 | 20260508 03:17:57.280 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.59e+01, 3.78e+00 / 1.61e+00 | 20260508 03:17:57.280 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.68e+00 / 1.21e+00 | 20260508 03:17:57.280 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.69e+00, 4.18e-01 / 1.80e-01 | 20260508 03:17:57.280 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.25e+00, 2.60e-01 / 1.31e-01 | 20260508 03:17:57.280 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.25e-01 / 3.60e-01, 5.12e-02 / 2.11e-02 | 20260508 03:17:57.280 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:17:57.281 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:17:57.281 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:17:57.281 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 24.22%, Upload -- 2.58e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 24.49%, Download -- 2.61e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 1.98%, LockUpload -- 2.11e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 2.99%, LockDownload -- 3.19e+00 / 0.00e+00, 4.43e+00 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:17:57.281 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260508 03:17:57.376 | INFO | Starting a round of Baz IO | 20260508 03:18:00.823 | INFO | Populating RTMetrics report | 20260508 03:18:00.824 | INFO | RTMetrics report populated | 20260508 03:18:00.824 | INFO | Opening tmp RTMetrics json file | 20260508 03:18:00.836 | INFO | Serializing RTMetrics to json | 20260508 03:18:00.838 | INFO | Serialization done, swapping tmp file now... | 20260508 03:18:00.838 | INFO | RTMetrics done | 20260508 03:18:00.892 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.515 seconds | 20260508 03:18:00.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.32% / 0.00% / 58.68% | 20260508 03:18:00.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.634511% / 0% / 5.5631e-05% | 20260508 03:18:00.892 | INFO | Baz IO overall: | 20260508 03:18:00.892 | INFO | BytesWritten : 2167340481657 | 20260508 03:18:00.892 | INFO | NumEvents : 1047189923255 | 20260508 03:18:02.413 | INFO | Analyzing chunk frames = [4816896,4817408) | 20260508 03:18:02.413 | INFO | MemInfo: { | 20260508 03:18:02.413 | INFO | "gpu" : " 17 GB", | 20260508 03:18:02.413 | INFO | "huge" : "", | 20260508 03:18:02.413 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:02.413 | INFO | "malloc" : " 14 GB", | 20260508 03:18:02.413 | INFO | "pinned" : " 29 GB" | 20260508 03:18:02.413 | INFO | } | 20260508 03:18:02.413 | INFO | Perf report for the last 5.1342 seconds: | 20260508 03:18:02.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:02.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:02.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:02.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:02.414 | INFO | ANALYSIS : 68.1%, 3.0, 106.4ms, 0 | 20260508 03:18:02.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:02.414 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:18:02.414 | INFO | PRE_HQ : 65.4%, 1.6, 55.7ms, 0 | 20260508 03:18:02.414 | INFO | BAZWRITER : 67.9%, 2.6, 91.6ms, 0 | 20260508 03:18:02.414 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:18:02.414 | INFO | Uploaded 36GiB to GPU in 2.55404s (14.0953GiB/s) | 20260508 03:18:02.414 | INFO | Downloaded 38GiB to GPU in 2.70583s (14.0438GiB/s) | 20260508 03:18:07.549 | INFO | Analyzing chunk frames = [4817408,4817920) | 20260508 03:18:07.549 | INFO | MemInfo: { | 20260508 03:18:07.549 | INFO | "gpu" : " 17 GB", | 20260508 03:18:07.549 | INFO | "huge" : "", | 20260508 03:18:07.549 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:07.549 | INFO | "malloc" : " 14 GB", | 20260508 03:18:07.549 | INFO | "pinned" : " 29 GB" | 20260508 03:18:07.549 | INFO | } | 20260508 03:18:07.549 | INFO | Perf report for the last 5.13597 seconds: | 20260508 03:18:07.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:07.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:18:07.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:18:07.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:07.549 | INFO | Uploaded 36GiB to GPU in 2.55507s (14.0896GiB/s) | 20260508 03:18:07.550 | INFO | Downloaded 34GiB to GPU in 2.42417s (14.0254GiB/s) | 20260508 03:18:12.691 | INFO | Analyzing chunk frames = [4817920,4818432) | 20260508 03:18:12.691 | INFO | MemInfo: { | 20260508 03:18:12.691 | INFO | "gpu" : " 17 GB", | 20260508 03:18:12.691 | INFO | "huge" : "", | 20260508 03:18:12.692 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:12.692 | INFO | "malloc" : " 14 GB", | 20260508 03:18:12.692 | INFO | "pinned" : " 29 GB" | 20260508 03:18:12.692 | INFO | } | 20260508 03:18:12.692 | INFO | Perf report for the last 5.14244 seconds: | 20260508 03:18:12.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:12.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 03:18:12.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:18:12.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:12.692 | INFO | Uploaded 36GiB to GPU in 2.5599s (14.063GiB/s) | 20260508 03:18:12.692 | INFO | Downloaded 34GiB to GPU in 2.4955s (13.6245GiB/s) | 20260508 03:18:17.839 | INFO | Analyzing chunk frames = [4818432,4818944) | 20260508 03:18:17.839 | INFO | MemInfo: { | 20260508 03:18:17.839 | INFO | "gpu" : " 17 GB", | 20260508 03:18:17.839 | INFO | "huge" : "", | 20260508 03:18:17.839 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:17.839 | INFO | "malloc" : " 14 GB", | 20260508 03:18:17.839 | INFO | "pinned" : " 29 GB" | 20260508 03:18:17.839 | INFO | } | 20260508 03:18:17.839 | INFO | Perf report for the last 5.14722 seconds: | 20260508 03:18:17.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:17.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 03:18:17.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:18:17.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:17.839 | INFO | Uploaded 36GiB to GPU in 2.55532s (14.0883GiB/s) | 20260508 03:18:17.839 | INFO | Downloaded 34GiB to GPU in 2.49364s (13.6347GiB/s) | 20260508 03:18:22.977 | INFO | Analyzing chunk frames = [4818944,4819456) | 20260508 03:18:22.977 | INFO | MemInfo: { | 20260508 03:18:22.977 | INFO | "gpu" : " 17 GB", | 20260508 03:18:22.977 | INFO | "huge" : "", | 20260508 03:18:22.977 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:22.977 | INFO | "malloc" : " 14 GB", | 20260508 03:18:22.977 | INFO | "pinned" : " 29 GB" | 20260508 03:18:22.977 | INFO | } | 20260508 03:18:22.978 | INFO | Perf report for the last 5.13867 seconds: | 20260508 03:18:22.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:22.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 03:18:22.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:18:22.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:22.978 | INFO | Uploaded 36GiB to GPU in 2.39385s (15.0386GiB/s) | 20260508 03:18:22.978 | INFO | Downloaded 34GiB to GPU in 2.56329s (13.2642GiB/s) | 20260508 03:18:28.108 | INFO | Analyzing chunk frames = [4819456,4819968) | 20260508 03:18:28.108 | INFO | MemInfo: { | 20260508 03:18:28.108 | INFO | "gpu" : " 17 GB", | 20260508 03:18:28.109 | INFO | "huge" : "", | 20260508 03:18:28.109 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:28.109 | INFO | "malloc" : " 14 GB", | 20260508 03:18:28.109 | INFO | "pinned" : " 29 GB" | 20260508 03:18:28.109 | INFO | } | 20260508 03:18:28.109 | INFO | Perf report for the last 5.13125 seconds: | 20260508 03:18:28.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:28.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:18:28.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 03:18:28.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:28.109 | INFO | Uploaded 36GiB to GPU in 2.45105s (14.6876GiB/s) | 20260508 03:18:28.109 | INFO | Downloaded 34GiB to GPU in 2.41156s (14.0988GiB/s) | 20260508 03:18:33.248 | INFO | Analyzing chunk frames = [4819968,4820480) | 20260508 03:18:33.248 | INFO | MemInfo: { | 20260508 03:18:33.248 | INFO | "gpu" : " 17 GB", | 20260508 03:18:33.249 | INFO | "huge" : "", | 20260508 03:18:33.249 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:33.249 | INFO | "malloc" : " 14 GB", | 20260508 03:18:33.249 | INFO | "pinned" : " 29 GB" | 20260508 03:18:33.249 | INFO | } | 20260508 03:18:33.249 | INFO | Perf report for the last 5.13996 seconds: | 20260508 03:18:33.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:33.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 03:18:33.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:18:33.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:33.249 | INFO | Uploaded 36GiB to GPU in 2.49356s (14.4372GiB/s) | 20260508 03:18:33.249 | INFO | Downloaded 34GiB to GPU in 2.4238s (14.0276GiB/s) | 20260508 03:18:38.394 | INFO | Analyzing chunk frames = [4820480,4820992) | 20260508 03:18:38.394 | INFO | MemInfo: { | 20260508 03:18:38.394 | INFO | "gpu" : " 17 GB", | 20260508 03:18:38.394 | INFO | "huge" : "", | 20260508 03:18:38.394 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:38.394 | INFO | "malloc" : " 14 GB", | 20260508 03:18:38.394 | INFO | "pinned" : " 29 GB" | 20260508 03:18:38.394 | INFO | } | 20260508 03:18:38.394 | INFO | Perf report for the last 5.14531 seconds: | 20260508 03:18:38.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:38.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.394 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:18:38.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.394 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260508 03:18:38.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:38.395 | INFO | Uploaded 36GiB to GPU in 2.57468s (13.9823GiB/s) | 20260508 03:18:38.395 | INFO | Downloaded 34GiB to GPU in 2.45162s (13.8684GiB/s) | 20260508 03:18:38.506 | INFO | Starting a round of Baz IO | 20260508 03:18:41.926 | INFO | Populating RTMetrics report | 20260508 03:18:41.926 | INFO | RTMetrics report populated | 20260508 03:18:41.926 | INFO | Opening tmp RTMetrics json file | 20260508 03:18:41.933 | INFO | Serializing RTMetrics to json | 20260508 03:18:41.941 | INFO | Serialization done, swapping tmp file now... | 20260508 03:18:41.941 | INFO | RTMetrics done | 20260508 03:18:42.000 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.494 seconds | 20260508 03:18:42.000 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.32% / 0.00% / 58.68% | 20260508 03:18:42.000 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.63103% / 7.94729e-06% / 6.35783e-05% | 20260508 03:18:42.000 | INFO | Baz IO overall: | 20260508 03:18:42.000 | INFO | BytesWritten : 2169077338700 | 20260508 03:18:42.000 | INFO | NumEvents : 1048020997122 | 20260508 03:18:43.530 | INFO | Analyzing chunk frames = [4820992,4821504) | 20260508 03:18:43.530 | INFO | MemInfo: { | 20260508 03:18:43.530 | INFO | "gpu" : " 17 GB", | 20260508 03:18:43.530 | INFO | "huge" : "", | 20260508 03:18:43.530 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:43.530 | INFO | "malloc" : " 14 GB", | 20260508 03:18:43.530 | INFO | "pinned" : " 29 GB" | 20260508 03:18:43.530 | INFO | } | 20260508 03:18:43.530 | INFO | Perf report for the last 5.13598 seconds: | 20260508 03:18:43.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:43.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:43.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:43.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:43.530 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 03:18:43.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:43.530 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:18:43.530 | INFO | PRE_HQ : 66.3%, 1.8, 62.1ms, 0 | 20260508 03:18:43.530 | INFO | BAZWRITER : 67.4%, 2.8, 97.7ms, 0 | 20260508 03:18:43.530 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:18:43.530 | INFO | Uploaded 36GiB to GPU in 2.58522s (13.9253GiB/s) | 20260508 03:18:43.530 | INFO | Downloaded 38GiB to GPU in 2.69897s (14.0794GiB/s) | 20260508 03:18:48.674 | INFO | Analyzing chunk frames = [4821504,4822016) | 20260508 03:18:48.674 | INFO | MemInfo: { | 20260508 03:18:48.674 | INFO | "gpu" : " 17 GB", | 20260508 03:18:48.674 | INFO | "huge" : "", | 20260508 03:18:48.674 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:48.674 | INFO | "malloc" : " 14 GB", | 20260508 03:18:48.674 | INFO | "pinned" : " 29 GB" | 20260508 03:18:48.674 | INFO | } | 20260508 03:18:48.674 | INFO | Perf report for the last 5.14403 seconds: | 20260508 03:18:48.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:48.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.674 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 03:18:48.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.674 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:18:48.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:48.675 | INFO | Uploaded 36GiB to GPU in 2.53484s (14.2021GiB/s) | 20260508 03:18:48.675 | INFO | Downloaded 34GiB to GPU in 2.43199s (13.9803GiB/s) | 20260508 03:18:53.808 | INFO | Analyzing chunk frames = [4822016,4822528) | 20260508 03:18:53.808 | INFO | MemInfo: { | 20260508 03:18:53.808 | INFO | "gpu" : " 17 GB", | 20260508 03:18:53.808 | INFO | "huge" : "", | 20260508 03:18:53.808 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:53.808 | INFO | "malloc" : " 14 GB", | 20260508 03:18:53.808 | INFO | "pinned" : " 29 GB" | 20260508 03:18:53.808 | INFO | } | 20260508 03:18:53.808 | INFO | Perf report for the last 5.13387 seconds: | 20260508 03:18:53.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:53.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:18:53.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:18:53.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:53.808 | INFO | Uploaded 36GiB to GPU in 2.52763s (14.2426GiB/s) | 20260508 03:18:53.809 | INFO | Downloaded 34GiB to GPU in 2.42732s (14.0072GiB/s) | 20260508 03:18:58.956 | INFO | Analyzing chunk frames = [4822528,4823040) | 20260508 03:18:58.956 | INFO | MemInfo: { | 20260508 03:18:58.956 | INFO | "gpu" : " 17 GB", | 20260508 03:18:58.956 | INFO | "huge" : "", | 20260508 03:18:58.956 | INFO | "huge_shared" : " 29 GB", | 20260508 03:18:58.956 | INFO | "malloc" : " 14 GB", | 20260508 03:18:58.956 | INFO | "pinned" : " 29 GB" | 20260508 03:18:58.956 | INFO | } | 20260508 03:18:58.956 | INFO | Perf report for the last 5.14761 seconds: | 20260508 03:18:58.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:18:58.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 03:18:58.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:18:58.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:18:58.956 | INFO | Uploaded 36GiB to GPU in 2.5447s (14.147GiB/s) | 20260508 03:18:58.956 | INFO | Downloaded 34GiB to GPU in 2.50343s (13.5814GiB/s) | 20260508 03:19:04.090 | INFO | Analyzing chunk frames = [4823040,4823552) | 20260508 03:19:04.090 | INFO | MemInfo: { | 20260508 03:19:04.090 | INFO | "gpu" : " 17 GB", | 20260508 03:19:04.090 | INFO | "huge" : "", | 20260508 03:19:04.091 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:04.091 | INFO | "malloc" : " 14 GB", | 20260508 03:19:04.091 | INFO | "pinned" : " 29 GB" | 20260508 03:19:04.091 | INFO | } | 20260508 03:19:04.091 | INFO | Perf report for the last 5.13491 seconds: | 20260508 03:19:04.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:04.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 03:19:04.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:19:04.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:04.091 | INFO | Uploaded 36GiB to GPU in 2.35715s (15.2727GiB/s) | 20260508 03:19:04.091 | INFO | Downloaded 34GiB to GPU in 2.55855s (13.2888GiB/s) | 20260508 03:19:09.228 | INFO | Analyzing chunk frames = [4823552,4824064) | 20260508 03:19:09.228 | INFO | MemInfo: { | 20260508 03:19:09.228 | INFO | "gpu" : " 17 GB", | 20260508 03:19:09.228 | INFO | "huge" : "", | 20260508 03:19:09.228 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:09.228 | INFO | "malloc" : " 14 GB", | 20260508 03:19:09.228 | INFO | "pinned" : " 29 GB" | 20260508 03:19:09.228 | INFO | } | 20260508 03:19:09.228 | INFO | Perf report for the last 5.13722 seconds: | 20260508 03:19:09.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:09.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 03:19:09.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:19:09.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:09.228 | INFO | Uploaded 36GiB to GPU in 2.37041s (15.1872GiB/s) | 20260508 03:19:09.228 | INFO | Downloaded 34GiB to GPU in 2.43541s (13.9607GiB/s) | 20260508 03:19:14.362 | INFO | Analyzing chunk frames = [4824064,4824576) | 20260508 03:19:14.362 | INFO | MemInfo: { | 20260508 03:19:14.362 | INFO | "gpu" : " 17 GB", | 20260508 03:19:14.362 | INFO | "huge" : "", | 20260508 03:19:14.362 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:14.362 | INFO | "malloc" : " 14 GB", | 20260508 03:19:14.362 | INFO | "pinned" : " 29 GB" | 20260508 03:19:14.362 | INFO | } | 20260508 03:19:14.362 | INFO | Perf report for the last 5.13396 seconds: | 20260508 03:19:14.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:14.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:19:14.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 03:19:14.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:14.362 | INFO | Uploaded 36GiB to GPU in 2.40363s (14.9774GiB/s) | 20260508 03:19:14.362 | INFO | Downloaded 34GiB to GPU in 2.43778s (13.9471GiB/s) | 20260508 03:19:19.502 | INFO | Analyzing chunk frames = [4824576,4825088) | 20260508 03:19:19.502 | INFO | MemInfo: { | 20260508 03:19:19.502 | INFO | "gpu" : " 17 GB", | 20260508 03:19:19.502 | INFO | "huge" : "", | 20260508 03:19:19.502 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:19.502 | INFO | "malloc" : " 14 GB", | 20260508 03:19:19.502 | INFO | "pinned" : " 29 GB" | 20260508 03:19:19.502 | INFO | } | 20260508 03:19:19.502 | INFO | Perf report for the last 5.14052 seconds: | 20260508 03:19:19.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:19.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.502 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 03:19:19.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.502 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 03:19:19.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:19.503 | INFO | Uploaded 36GiB to GPU in 2.53549s (14.1985GiB/s) | 20260508 03:19:19.503 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:19:19.503 | INFO | Total mem usage: 16.7 GiB | 20260508 03:19:19.503 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:19:19.503 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:19:19.503 | INFO | Downloaded 34GiB to GPU in 2.47102s (13.7595GiB/s) | 20260508 03:19:19.503 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:19:19.503 | INFO | High Water Breakdown: | 20260508 03:19:19.503 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:19:19.503 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:19:19.503 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:19:19.503 | INFO | ------------------------------------------ | 20260508 03:19:19.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:19:19.503 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:19:19.503 | INFO | Total mem usage: 26.5 GiB | 20260508 03:19:19.503 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:19:19.503 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:19:19.503 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:19:19.503 | INFO | High Water Breakdown: | 20260508 03:19:19.503 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:19:19.503 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:19:19.503 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:19:19.503 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:19:19.503 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:19:19.503 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:19:19.503 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:19:19.503 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:19:19.503 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:19:19.503 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:19:19.503 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:19:19.503 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:19:19.503 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:19:19.503 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:19:19.503 | INFO | ------------------------------------------ | 20260508 03:19:19.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:19:19.503 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:19:19.503 | INFO | Total mem usage: 27.8 GiB | 20260508 03:19:19.503 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:19:19.503 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:19:19.503 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:19:19.503 | INFO | High Water Breakdown: | 20260508 03:19:19.503 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:19:19.503 | INFO | ------------------------------------------ | 20260508 03:19:19.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:19:19.503 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:19:19.503 | INFO | Total mem usage: 15.7 GiB | 20260508 03:19:19.503 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:19:19.503 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:19:19.503 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:19:19.503 | INFO | High Water Breakdown: | 20260508 03:19:19.503 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:19:19.503 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:19:19.503 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:19:19.503 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:19:19.503 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:19:19.503 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:19:19.503 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:19:19.503 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:19:19.503 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:19:19.503 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:19:19.503 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:19:19.503 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:19:19.503 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:19:19.503 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:19:19.503 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:19:19.503 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:19:19.503 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:19:19.503 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:19:19.503 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:19:19.503 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:19:19.503 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:19:19.503 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:19:19.503 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:19:19.503 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:19:19.503 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:19:19.503 | INFO | ------------------------------------------ | 20260508 03:19:19.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:19:19.503 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:19:19.503 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:19:19.503 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:19:19.503 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.61e+01, 1.42e+01 / 1.75e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.18e-02 / 6.63e-03 | 20260508 03:19:19.503 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.45e+00, 5.84e-01 / 2.10e-01 | 20260508 03:19:19.503 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.06e-01 / 1.02e-01 | 20260508 03:19:19.503 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.62e+01, 3.73e+00 / 2.17e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.84e+01 / 1.96e+01, 1.65e+00 / 1.28e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.18e-01 / 2.28e-01 | 20260508 03:19:19.503 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.23e+00, 2.63e-01 / 1.19e-01 | 20260508 03:19:19.503 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.63e-01, 5.17e-02 / 2.35e-02 | 20260508 03:19:19.503 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:19:19.503 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:19:19.503 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:19:19.503 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 24.39%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 30.63%, Compute -- 3.26e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 2.23%, LockUpload -- 2.38e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 2.80%, LockDownload -- 2.98e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:19:19.503 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260508 03:19:19.600 | INFO | Starting a round of Baz IO | 20260508 03:19:23.047 | INFO | Populating RTMetrics report | 20260508 03:19:23.047 | INFO | RTMetrics report populated | 20260508 03:19:23.047 | INFO | Opening tmp RTMetrics json file | 20260508 03:19:23.061 | INFO | Serializing RTMetrics to json | 20260508 03:19:23.062 | INFO | Serialization done, swapping tmp file now... | 20260508 03:19:23.063 | INFO | RTMetrics done | 20260508 03:19:23.116 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.516 seconds | 20260508 03:19:23.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.31% / 0.00% / 58.68% | 20260508 03:19:23.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.629656% / 3.97364e-06% / 5.96046e-05% | 20260508 03:19:23.116 | INFO | Baz IO overall: | 20260508 03:19:23.116 | INFO | BytesWritten : 2170814581253 | 20260508 03:19:23.117 | INFO | NumEvents : 1048852479914 | 20260508 03:19:24.649 | INFO | Analyzing chunk frames = [4825088,4825600) | 20260508 03:19:24.649 | INFO | MemInfo: { | 20260508 03:19:24.649 | INFO | "gpu" : " 17 GB", | 20260508 03:19:24.649 | INFO | "huge" : "", | 20260508 03:19:24.649 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:24.649 | INFO | "malloc" : " 14 GB", | 20260508 03:19:24.649 | INFO | "pinned" : " 29 GB" | 20260508 03:19:24.649 | INFO | } | 20260508 03:19:24.649 | INFO | Perf report for the last 5.14707 seconds: | 20260508 03:19:24.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:24.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:24.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:24.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:24.650 | INFO | ANALYSIS : 68.0%, 3.0, 106.4ms, 0 | 20260508 03:19:24.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:24.650 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:19:24.650 | INFO | PRE_HQ : 65.5%, 1.6, 56.1ms, 0 | 20260508 03:19:24.650 | INFO | BAZWRITER : 67.5%, 2.7, 92.2ms, 0 | 20260508 03:19:24.650 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:19:24.650 | INFO | Uploaded 36GiB to GPU in 2.56108s (14.0565GiB/s) | 20260508 03:19:24.650 | INFO | Downloaded 38GiB to GPU in 2.68762s (14.1389GiB/s) | 20260508 03:19:29.788 | INFO | Analyzing chunk frames = [4825600,4826112) | 20260508 03:19:29.788 | INFO | MemInfo: { | 20260508 03:19:29.788 | INFO | "gpu" : " 17 GB", | 20260508 03:19:29.788 | INFO | "huge" : "", | 20260508 03:19:29.788 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:29.788 | INFO | "malloc" : " 14 GB", | 20260508 03:19:29.788 | INFO | "pinned" : " 29 GB" | 20260508 03:19:29.788 | INFO | } | 20260508 03:19:29.788 | INFO | Perf report for the last 5.13872 seconds: | 20260508 03:19:29.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:29.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.788 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:19:29.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.788 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:19:29.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:29.789 | INFO | Uploaded 36GiB to GPU in 2.57255s (13.9939GiB/s) | 20260508 03:19:29.789 | INFO | Downloaded 34GiB to GPU in 2.43435s (13.9668GiB/s) | 20260508 03:19:34.926 | INFO | Analyzing chunk frames = [4826112,4826624) | 20260508 03:19:34.926 | INFO | MemInfo: { | 20260508 03:19:34.926 | INFO | "gpu" : " 17 GB", | 20260508 03:19:34.926 | INFO | "huge" : "", | 20260508 03:19:34.927 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:34.927 | INFO | "malloc" : " 14 GB", | 20260508 03:19:34.927 | INFO | "pinned" : " 29 GB" | 20260508 03:19:34.927 | INFO | } | 20260508 03:19:34.927 | INFO | Perf report for the last 5.13845 seconds: | 20260508 03:19:34.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:34.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:19:34.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:19:34.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:34.927 | INFO | Uploaded 36GiB to GPU in 2.5613s (14.0554GiB/s) | 20260508 03:19:34.927 | INFO | Downloaded 34GiB to GPU in 2.50075s (13.5959GiB/s) | 20260508 03:19:40.059 | INFO | Analyzing chunk frames = [4826624,4827136) | 20260508 03:19:40.059 | INFO | MemInfo: { | 20260508 03:19:40.059 | INFO | "gpu" : " 17 GB", | 20260508 03:19:40.059 | INFO | "huge" : "", | 20260508 03:19:40.060 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:40.060 | INFO | "malloc" : " 14 GB", | 20260508 03:19:40.060 | INFO | "pinned" : " 29 GB" | 20260508 03:19:40.060 | INFO | } | 20260508 03:19:40.060 | INFO | Perf report for the last 5.13299 seconds: | 20260508 03:19:40.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:40.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 03:19:40.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:19:40.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:40.060 | INFO | Uploaded 36GiB to GPU in 2.53242s (14.2156GiB/s) | 20260508 03:19:40.060 | INFO | Downloaded 34GiB to GPU in 2.49878s (13.6067GiB/s) | 20260508 03:19:45.203 | INFO | Analyzing chunk frames = [4827136,4827648) | 20260508 03:19:45.203 | INFO | MemInfo: { | 20260508 03:19:45.203 | INFO | "gpu" : " 17 GB", | 20260508 03:19:45.203 | INFO | "huge" : "", | 20260508 03:19:45.204 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:45.204 | INFO | "malloc" : " 14 GB", | 20260508 03:19:45.204 | INFO | "pinned" : " 29 GB" | 20260508 03:19:45.204 | INFO | } | 20260508 03:19:45.204 | INFO | Perf report for the last 5.1441 seconds: | 20260508 03:19:45.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:45.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 03:19:45.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:19:45.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:45.204 | INFO | Uploaded 36GiB to GPU in 2.38726s (15.08GiB/s) | 20260508 03:19:45.204 | INFO | Downloaded 34GiB to GPU in 2.55354s (13.3148GiB/s) | 20260508 03:19:50.344 | INFO | Analyzing chunk frames = [4827648,4828160) | 20260508 03:19:50.344 | INFO | MemInfo: { | 20260508 03:19:50.344 | INFO | "gpu" : " 17 GB", | 20260508 03:19:50.344 | INFO | "huge" : "", | 20260508 03:19:50.344 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:50.344 | INFO | "malloc" : " 14 GB", | 20260508 03:19:50.344 | INFO | "pinned" : " 29 GB" | 20260508 03:19:50.344 | INFO | } | 20260508 03:19:50.345 | INFO | Perf report for the last 5.14077 seconds: | 20260508 03:19:50.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:50.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:19:50.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260508 03:19:50.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:50.345 | INFO | Uploaded 36GiB to GPU in 2.45082s (14.689GiB/s) | 20260508 03:19:50.345 | INFO | Downloaded 34GiB to GPU in 2.42156s (14.0405GiB/s) | 20260508 03:19:55.482 | INFO | Analyzing chunk frames = [4828160,4828672) | 20260508 03:19:55.482 | INFO | MemInfo: { | 20260508 03:19:55.482 | INFO | "gpu" : " 17 GB", | 20260508 03:19:55.482 | INFO | "huge" : "", | 20260508 03:19:55.482 | INFO | "huge_shared" : " 29 GB", | 20260508 03:19:55.482 | INFO | "malloc" : " 14 GB", | 20260508 03:19:55.482 | INFO | "pinned" : " 29 GB" | 20260508 03:19:55.482 | INFO | } | 20260508 03:19:55.482 | INFO | Perf report for the last 5.1378 seconds: | 20260508 03:19:55.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:19:55.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.482 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:19:55.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.483 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:19:55.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:19:55.483 | INFO | Uploaded 36GiB to GPU in 2.46349s (14.6134GiB/s) | 20260508 03:19:55.483 | INFO | Downloaded 34GiB to GPU in 2.41544s (14.0761GiB/s) | 20260508 03:20:00.620 | INFO | Analyzing chunk frames = [4828672,4829184) | 20260508 03:20:00.620 | INFO | MemInfo: { | 20260508 03:20:00.620 | INFO | "gpu" : " 17 GB", | 20260508 03:20:00.620 | INFO | "huge" : "", | 20260508 03:20:00.620 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:00.620 | INFO | "malloc" : " 14 GB", | 20260508 03:20:00.620 | INFO | "pinned" : " 29 GB" | 20260508 03:20:00.620 | INFO | } | 20260508 03:20:00.620 | INFO | Perf report for the last 5.13791 seconds: | 20260508 03:20:00.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:00.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 03:20:00.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:20:00.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:00.620 | INFO | Uploaded 36GiB to GPU in 2.48313s (14.4979GiB/s) | 20260508 03:20:00.621 | INFO | Downloaded 34GiB to GPU in 2.41706s (14.0666GiB/s) | 20260508 03:20:00.735 | INFO | Starting a round of Baz IO | 20260508 03:20:04.156 | INFO | Populating RTMetrics report | 20260508 03:20:04.156 | INFO | RTMetrics report populated | 20260508 03:20:04.156 | INFO | Opening tmp RTMetrics json file | 20260508 03:20:04.165 | INFO | Serializing RTMetrics to json | 20260508 03:20:04.170 | INFO | Serialization done, swapping tmp file now... | 20260508 03:20:04.170 | INFO | RTMetrics done | 20260508 03:20:04.233 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.498 seconds | 20260508 03:20:04.233 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.31% / 0.00% / 58.69% | 20260508 03:20:04.233 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.629926% / 1.19209e-05% / 5.96046e-05% | 20260508 03:20:04.233 | INFO | Baz IO overall: | 20260508 03:20:04.233 | INFO | BytesWritten : 2172551810427 | 20260508 03:20:04.233 | INFO | NumEvents : 1049684059315 | 20260508 03:20:05.761 | INFO | Analyzing chunk frames = [4829184,4829696) | 20260508 03:20:05.761 | INFO | MemInfo: { | 20260508 03:20:05.761 | INFO | "gpu" : " 17 GB", | 20260508 03:20:05.761 | INFO | "huge" : "", | 20260508 03:20:05.761 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:05.761 | INFO | "malloc" : " 14 GB", | 20260508 03:20:05.761 | INFO | "pinned" : " 29 GB" | 20260508 03:20:05.761 | INFO | } | 20260508 03:20:05.761 | INFO | Perf report for the last 5.14059 seconds: | 20260508 03:20:05.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:05.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:05.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:05.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:05.761 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 03:20:05.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:05.761 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:20:05.761 | INFO | PRE_HQ : 66.4%, 1.8, 63.0ms, 0 | 20260508 03:20:05.761 | INFO | BAZWRITER : 67.4%, 2.8, 97.7ms, 0 | 20260508 03:20:05.761 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:20:05.761 | INFO | Uploaded 36GiB to GPU in 2.60314s (13.8295GiB/s) | 20260508 03:20:05.761 | INFO | Downloaded 38GiB to GPU in 2.70875s (14.0286GiB/s) | 20260508 03:20:10.901 | INFO | Analyzing chunk frames = [4829696,4830208) | 20260508 03:20:10.901 | INFO | MemInfo: { | 20260508 03:20:10.901 | INFO | "gpu" : " 17 GB", | 20260508 03:20:10.901 | INFO | "huge" : "", | 20260508 03:20:10.901 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:10.901 | INFO | "malloc" : " 14 GB", | 20260508 03:20:10.901 | INFO | "pinned" : " 29 GB" | 20260508 03:20:10.901 | INFO | } | 20260508 03:20:10.901 | INFO | Perf report for the last 5.14048 seconds: | 20260508 03:20:10.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:10.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.901 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:20:10.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.902 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:20:10.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:10.902 | INFO | Uploaded 36GiB to GPU in 2.57785s (13.9651GiB/s) | 20260508 03:20:10.902 | INFO | Downloaded 34GiB to GPU in 2.41994s (14.0499GiB/s) | 20260508 03:20:16.044 | INFO | Analyzing chunk frames = [4830208,4830720) | 20260508 03:20:16.044 | INFO | MemInfo: { | 20260508 03:20:16.044 | INFO | "gpu" : " 17 GB", | 20260508 03:20:16.044 | INFO | "huge" : "", | 20260508 03:20:16.044 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:16.044 | INFO | "malloc" : " 14 GB", | 20260508 03:20:16.044 | INFO | "pinned" : " 29 GB" | 20260508 03:20:16.044 | INFO | } | 20260508 03:20:16.044 | INFO | Perf report for the last 5.14271 seconds: | 20260508 03:20:16.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:16.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.044 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260508 03:20:16.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.044 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:20:16.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:16.045 | INFO | Uploaded 36GiB to GPU in 2.55046s (14.1151GiB/s) | 20260508 03:20:16.045 | INFO | Downloaded 34GiB to GPU in 2.41824s (14.0598GiB/s) | 20260508 03:20:21.176 | INFO | Analyzing chunk frames = [4830720,4831232) | 20260508 03:20:21.176 | INFO | MemInfo: { | 20260508 03:20:21.176 | INFO | "gpu" : " 17 GB", | 20260508 03:20:21.176 | INFO | "huge" : "", | 20260508 03:20:21.176 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:21.176 | INFO | "malloc" : " 14 GB", | 20260508 03:20:21.177 | INFO | "pinned" : " 29 GB" | 20260508 03:20:21.177 | INFO | } | 20260508 03:20:21.177 | INFO | Perf report for the last 5.1325 seconds: | 20260508 03:20:21.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:21.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 03:20:21.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:20:21.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:21.177 | INFO | Uploaded 36GiB to GPU in 2.56924s (14.012GiB/s) | 20260508 03:20:21.177 | INFO | Downloaded 34GiB to GPU in 2.49604s (13.6216GiB/s) | 20260508 03:20:26.316 | INFO | Analyzing chunk frames = [4831232,4831744) | 20260508 03:20:26.316 | INFO | MemInfo: { | 20260508 03:20:26.316 | INFO | "gpu" : " 17 GB", | 20260508 03:20:26.316 | INFO | "huge" : "", | 20260508 03:20:26.316 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:26.316 | INFO | "malloc" : " 14 GB", | 20260508 03:20:26.316 | INFO | "pinned" : " 29 GB" | 20260508 03:20:26.316 | INFO | } | 20260508 03:20:26.316 | INFO | Perf report for the last 5.13949 seconds: | 20260508 03:20:26.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:26.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 03:20:26.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:20:26.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:26.316 | INFO | Uploaded 36GiB to GPU in 2.43583s (14.7794GiB/s) | 20260508 03:20:26.316 | INFO | Downloaded 34GiB to GPU in 2.5545s (13.3098GiB/s) | 20260508 03:20:31.457 | INFO | Analyzing chunk frames = [4831744,4832256) | 20260508 03:20:31.457 | INFO | MemInfo: { | 20260508 03:20:31.457 | INFO | "gpu" : " 17 GB", | 20260508 03:20:31.457 | INFO | "huge" : "", | 20260508 03:20:31.458 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:31.458 | INFO | "malloc" : " 14 GB", | 20260508 03:20:31.458 | INFO | "pinned" : " 29 GB" | 20260508 03:20:31.458 | INFO | } | 20260508 03:20:31.458 | INFO | Perf report for the last 5.14146 seconds: | 20260508 03:20:31.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:31.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 03:20:31.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:20:31.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:31.458 | INFO | Uploaded 36GiB to GPU in 2.40187s (14.9883GiB/s) | 20260508 03:20:31.458 | INFO | Downloaded 34GiB to GPU in 2.42385s (14.0273GiB/s) | 20260508 03:20:36.598 | INFO | Analyzing chunk frames = [4832256,4832768) | 20260508 03:20:36.598 | INFO | MemInfo: { | 20260508 03:20:36.598 | INFO | "gpu" : " 17 GB", | 20260508 03:20:36.598 | INFO | "huge" : "", | 20260508 03:20:36.598 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:36.598 | INFO | "malloc" : " 14 GB", | 20260508 03:20:36.598 | INFO | "pinned" : " 29 GB" | 20260508 03:20:36.598 | INFO | } | 20260508 03:20:36.599 | INFO | Perf report for the last 5.1409 seconds: | 20260508 03:20:36.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:36.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:20:36.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:20:36.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:36.599 | INFO | Uploaded 36GiB to GPU in 2.44503s (14.7237GiB/s) | 20260508 03:20:36.599 | INFO | Downloaded 34GiB to GPU in 2.43135s (13.984GiB/s) | 20260508 03:20:41.736 | INFO | Analyzing chunk frames = [4832768,4833280) | 20260508 03:20:41.736 | INFO | MemInfo: { | 20260508 03:20:41.736 | INFO | "gpu" : " 17 GB", | 20260508 03:20:41.736 | INFO | "huge" : "", | 20260508 03:20:41.736 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:41.736 | INFO | "malloc" : " 14 GB", | 20260508 03:20:41.736 | INFO | "pinned" : " 29 GB" | 20260508 03:20:41.736 | INFO | } | 20260508 03:20:41.736 | INFO | Perf report for the last 5.13737 seconds: | 20260508 03:20:41.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:41.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.736 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:20:41.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.737 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 03:20:41.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:41.737 | INFO | Uploaded 36GiB to GPU in 2.43702s (14.7721GiB/s) | 20260508 03:20:41.737 | INFO | Downloaded 34GiB to GPU in 2.43152s (13.9831GiB/s) | 20260508 03:20:41.737 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:20:41.737 | INFO | Total mem usage: 16.7 GiB | 20260508 03:20:41.737 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:20:41.737 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:20:41.737 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:20:41.737 | INFO | High Water Breakdown: | 20260508 03:20:41.737 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:20:41.737 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:20:41.737 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:20:41.737 | INFO | ------------------------------------------ | 20260508 03:20:41.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:20:41.737 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:20:41.737 | INFO | Total mem usage: 26.5 GiB | 20260508 03:20:41.737 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:20:41.737 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:20:41.737 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:20:41.737 | INFO | High Water Breakdown: | 20260508 03:20:41.737 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:20:41.737 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:20:41.737 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:20:41.737 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:20:41.737 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:20:41.737 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:20:41.737 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:20:41.737 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:20:41.737 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:20:41.737 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:20:41.737 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:20:41.737 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:20:41.737 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:20:41.737 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:20:41.737 | INFO | ------------------------------------------ | 20260508 03:20:41.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:20:41.737 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:20:41.737 | INFO | Total mem usage: 27.8 GiB | 20260508 03:20:41.737 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:20:41.737 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:20:41.737 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:20:41.737 | INFO | High Water Breakdown: | 20260508 03:20:41.737 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:20:41.737 | INFO | ------------------------------------------ | 20260508 03:20:41.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:20:41.737 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:20:41.737 | INFO | Total mem usage: 15.7 GiB | 20260508 03:20:41.737 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:20:41.737 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:20:41.737 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:20:41.737 | INFO | High Water Breakdown: | 20260508 03:20:41.737 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:20:41.737 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:20:41.737 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:20:41.738 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:20:41.738 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:20:41.738 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:20:41.738 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:20:41.738 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:20:41.738 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:20:41.738 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:20:41.738 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:20:41.738 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:20:41.738 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:20:41.738 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:20:41.738 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:20:41.738 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:20:41.738 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:20:41.738 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:20:41.738 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:20:41.738 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:20:41.738 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:20:41.738 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:20:41.738 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:20:41.738 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:20:41.738 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:20:41.738 | INFO | ------------------------------------------ | 20260508 03:20:41.738 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:20:41.738 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:20:41.738 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:20:41.738 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:20:41.738 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.39e+01 / 1.63e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.15e-02 / 8.17e-03 | 20260508 03:20:41.738 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.47e+00, 5.87e-01 / 1.72e-01 | 20260508 03:20:41.738 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.09e-01 / 1.13e-01 | 20260508 03:20:41.738 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.58e+01, 3.86e+00 / 1.90e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.71e+00 / 1.48e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.41e+00 / 2.66e+00, 4.19e-01 / 2.06e-01 | 20260508 03:20:41.738 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.23e+00, 2.68e-01 / 1.26e-01 | 20260508 03:20:41.738 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.66e-01, 5.20e-02 / 2.82e-02 | 20260508 03:20:41.738 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:20:41.738 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:20:41.738 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:20:41.738 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 2.27%, LockUpload -- 2.42e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 2.67%, LockDownload -- 2.84e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 15.54%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:20:41.738 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 03:20:41.850 | INFO | Starting a round of Baz IO | 20260508 03:20:45.266 | INFO | Populating RTMetrics report | 20260508 03:20:45.266 | INFO | RTMetrics report populated | 20260508 03:20:45.266 | INFO | Opening tmp RTMetrics json file | 20260508 03:20:45.280 | INFO | Serializing RTMetrics to json | 20260508 03:20:45.281 | INFO | Serialization done, swapping tmp file now... | 20260508 03:20:45.282 | INFO | RTMetrics done | 20260508 03:20:45.339 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.489 seconds | 20260508 03:20:45.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.31% / 0.00% / 58.69% | 20260508 03:20:45.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.628312% / 7.94729e-06% / 3.57628e-05% | 20260508 03:20:45.339 | INFO | Baz IO overall: | 20260508 03:20:45.339 | INFO | BytesWritten : 2174289073682 | 20260508 03:20:45.339 | INFO | NumEvents : 1050515647950 | 20260508 03:20:46.875 | INFO | Analyzing chunk frames = [4833280,4833792) | 20260508 03:20:46.875 | INFO | MemInfo: { | 20260508 03:20:46.875 | INFO | "gpu" : " 17 GB", | 20260508 03:20:46.875 | INFO | "huge" : "", | 20260508 03:20:46.875 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:46.875 | INFO | "malloc" : " 14 GB", | 20260508 03:20:46.875 | INFO | "pinned" : " 29 GB" | 20260508 03:20:46.875 | INFO | } | 20260508 03:20:46.875 | INFO | Perf report for the last 5.13897 seconds: | 20260508 03:20:46.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:46.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:46.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:46.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:46.875 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 03:20:46.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:46.875 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:20:46.875 | INFO | PRE_HQ : 66.4%, 1.8, 62.4ms, 0 | 20260508 03:20:46.875 | INFO | BAZWRITER : 67.4%, 2.8, 97.0ms, 0 | 20260508 03:20:46.875 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:20:46.876 | INFO | Uploaded 36GiB to GPU in 2.57756s (13.9667GiB/s) | 20260508 03:20:46.876 | INFO | Downloaded 38GiB to GPU in 2.69113s (14.1205GiB/s) | 20260508 03:20:52.012 | INFO | Analyzing chunk frames = [4833792,4834304) | 20260508 03:20:52.012 | INFO | MemInfo: { | 20260508 03:20:52.012 | INFO | "gpu" : " 17 GB", | 20260508 03:20:52.012 | INFO | "huge" : "", | 20260508 03:20:52.012 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:52.012 | INFO | "malloc" : " 14 GB", | 20260508 03:20:52.012 | INFO | "pinned" : " 29 GB" | 20260508 03:20:52.012 | INFO | } | 20260508 03:20:52.012 | INFO | Perf report for the last 5.13707 seconds: | 20260508 03:20:52.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:52.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.012 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:20:52.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.012 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:20:52.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:52.013 | INFO | Uploaded 36GiB to GPU in 2.58046s (13.951GiB/s) | 20260508 03:20:52.013 | INFO | Downloaded 34GiB to GPU in 2.43387s (13.9695GiB/s) | 20260508 03:20:57.153 | INFO | Analyzing chunk frames = [4834304,4834816) | 20260508 03:20:57.153 | INFO | MemInfo: { | 20260508 03:20:57.153 | INFO | "gpu" : " 17 GB", | 20260508 03:20:57.153 | INFO | "huge" : "", | 20260508 03:20:57.153 | INFO | "huge_shared" : " 29 GB", | 20260508 03:20:57.153 | INFO | "malloc" : " 14 GB", | 20260508 03:20:57.153 | INFO | "pinned" : " 29 GB" | 20260508 03:20:57.153 | INFO | } | 20260508 03:20:57.153 | INFO | Perf report for the last 5.14136 seconds: | 20260508 03:20:57.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:20:57.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.153 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:20:57.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.153 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:20:57.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:20:57.154 | INFO | Uploaded 36GiB to GPU in 2.54589s (14.1404GiB/s) | 20260508 03:20:57.154 | INFO | Downloaded 34GiB to GPU in 2.50377s (13.5795GiB/s) | 20260508 03:21:02.292 | INFO | Analyzing chunk frames = [4834816,4835328) | 20260508 03:21:02.292 | INFO | MemInfo: { | 20260508 03:21:02.292 | INFO | "gpu" : " 17 GB", | 20260508 03:21:02.292 | INFO | "huge" : "", | 20260508 03:21:02.292 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:02.292 | INFO | "malloc" : " 14 GB", | 20260508 03:21:02.292 | INFO | "pinned" : " 29 GB" | 20260508 03:21:02.292 | INFO | } | 20260508 03:21:02.292 | INFO | Perf report for the last 5.13908 seconds: | 20260508 03:21:02.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:02.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.293 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:21:02.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.293 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:21:02.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:02.293 | INFO | Uploaded 36GiB to GPU in 2.55512s (14.0894GiB/s) | 20260508 03:21:02.293 | INFO | Downloaded 34GiB to GPU in 2.50268s (13.5854GiB/s) | 20260508 03:21:07.432 | INFO | Analyzing chunk frames = [4835328,4835840) | 20260508 03:21:07.433 | INFO | MemInfo: { | 20260508 03:21:07.433 | INFO | "gpu" : " 17 GB", | 20260508 03:21:07.433 | INFO | "huge" : "", | 20260508 03:21:07.433 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:07.433 | INFO | "malloc" : " 14 GB", | 20260508 03:21:07.433 | INFO | "pinned" : " 29 GB" | 20260508 03:21:07.433 | INFO | } | 20260508 03:21:07.433 | INFO | Perf report for the last 5.14036 seconds: | 20260508 03:21:07.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:07.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 03:21:07.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:21:07.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:07.433 | INFO | Uploaded 36GiB to GPU in 2.45633s (14.656GiB/s) | 20260508 03:21:07.433 | INFO | Downloaded 34GiB to GPU in 2.57043s (13.2273GiB/s) | 20260508 03:21:12.569 | INFO | Analyzing chunk frames = [4835840,4836352) | 20260508 03:21:12.569 | INFO | MemInfo: { | 20260508 03:21:12.569 | INFO | "gpu" : " 17 GB", | 20260508 03:21:12.569 | INFO | "huge" : "", | 20260508 03:21:12.569 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:12.569 | INFO | "malloc" : " 14 GB", | 20260508 03:21:12.569 | INFO | "pinned" : " 29 GB" | 20260508 03:21:12.569 | INFO | } | 20260508 03:21:12.569 | INFO | Perf report for the last 5.13644 seconds: | 20260508 03:21:12.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:12.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.569 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:21:12.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.569 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:21:12.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:12.570 | INFO | Uploaded 36GiB to GPU in 2.42553s (14.8421GiB/s) | 20260508 03:21:12.570 | INFO | Downloaded 34GiB to GPU in 2.42345s (14.0296GiB/s) | 20260508 03:21:17.714 | INFO | Analyzing chunk frames = [4836352,4836864) | 20260508 03:21:17.714 | INFO | MemInfo: { | 20260508 03:21:17.714 | INFO | "gpu" : " 17 GB", | 20260508 03:21:17.714 | INFO | "huge" : "", | 20260508 03:21:17.715 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:17.715 | INFO | "malloc" : " 14 GB", | 20260508 03:21:17.715 | INFO | "pinned" : " 29 GB" | 20260508 03:21:17.715 | INFO | } | 20260508 03:21:17.715 | INFO | Perf report for the last 5.14554 seconds: | 20260508 03:21:17.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:17.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:21:17.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:21:17.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:17.715 | INFO | Uploaded 36GiB to GPU in 2.463s (14.6163GiB/s) | 20260508 03:21:17.715 | INFO | Downloaded 34GiB to GPU in 2.42447s (14.0237GiB/s) | 20260508 03:21:22.852 | INFO | Analyzing chunk frames = [4836864,4837376) | 20260508 03:21:22.852 | INFO | MemInfo: { | 20260508 03:21:22.852 | INFO | "gpu" : " 17 GB", | 20260508 03:21:22.852 | INFO | "huge" : "", | 20260508 03:21:22.852 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:22.852 | INFO | "malloc" : " 14 GB", | 20260508 03:21:22.852 | INFO | "pinned" : " 29 GB" | 20260508 03:21:22.852 | INFO | } | 20260508 03:21:22.852 | INFO | Perf report for the last 5.1374 seconds: | 20260508 03:21:22.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:22.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.852 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:21:22.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.853 | INFO | PRE_HQ : 35.7%, 1.0, 18.5ms, 0 | 20260508 03:21:22.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:22.853 | INFO | Uploaded 36GiB to GPU in 2.45139s (14.6856GiB/s) | 20260508 03:21:22.853 | INFO | Downloaded 34GiB to GPU in 2.43002s (13.9917GiB/s) | 20260508 03:21:22.967 | INFO | Starting a round of Baz IO | 20260508 03:21:26.391 | INFO | Populating RTMetrics report | 20260508 03:21:26.391 | INFO | RTMetrics report populated | 20260508 03:21:26.391 | INFO | Opening tmp RTMetrics json file | 20260508 03:21:26.402 | INFO | Serializing RTMetrics to json | 20260508 03:21:26.403 | INFO | Serialization done, swapping tmp file now... | 20260508 03:21:26.404 | INFO | RTMetrics done | 20260508 03:21:26.460 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.493 seconds | 20260508 03:21:26.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.31% / 0.00% / 58.69% | 20260508 03:21:26.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.627112% / 0% / 3.17891e-05% | 20260508 03:21:26.460 | INFO | Baz IO overall: | 20260508 03:21:26.460 | INFO | BytesWritten : 2176025567764 | 20260508 03:21:26.460 | INFO | NumEvents : 1051346772575 | 20260508 03:21:27.992 | INFO | Analyzing chunk frames = [4837376,4837888) | 20260508 03:21:27.992 | INFO | MemInfo: { | 20260508 03:21:27.992 | INFO | "gpu" : " 17 GB", | 20260508 03:21:27.992 | INFO | "huge" : "", | 20260508 03:21:27.992 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:27.992 | INFO | "malloc" : " 14 GB", | 20260508 03:21:27.992 | INFO | "pinned" : " 29 GB" | 20260508 03:21:27.992 | INFO | } | 20260508 03:21:27.992 | INFO | Perf report for the last 5.13953 seconds: | 20260508 03:21:27.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:27.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:27.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:27.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:27.992 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 03:21:27.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:27.992 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 03:21:27.992 | INFO | PRE_HQ : 66.2%, 1.8, 62.4ms, 0 | 20260508 03:21:27.992 | INFO | BAZWRITER : 67.5%, 2.8, 97.6ms, 0 | 20260508 03:21:27.992 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:21:27.992 | INFO | Uploaded 36GiB to GPU in 2.58301s (13.9373GiB/s) | 20260508 03:21:27.992 | INFO | Downloaded 38GiB to GPU in 2.70251s (14.061GiB/s) | 20260508 03:21:33.130 | INFO | Analyzing chunk frames = [4837888,4838400) | 20260508 03:21:33.130 | INFO | MemInfo: { | 20260508 03:21:33.130 | INFO | "gpu" : " 17 GB", | 20260508 03:21:33.130 | INFO | "huge" : "", | 20260508 03:21:33.130 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:33.130 | INFO | "malloc" : " 14 GB", | 20260508 03:21:33.130 | INFO | "pinned" : " 29 GB" | 20260508 03:21:33.130 | INFO | } | 20260508 03:21:33.130 | INFO | Perf report for the last 5.13812 seconds: | 20260508 03:21:33.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:33.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:21:33.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:21:33.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:33.130 | INFO | Uploaded 36GiB to GPU in 2.57556s (13.9776GiB/s) | 20260508 03:21:33.130 | INFO | Downloaded 34GiB to GPU in 2.42913s (13.9968GiB/s) | 20260508 03:21:38.266 | INFO | Analyzing chunk frames = [4838400,4838912) | 20260508 03:21:38.266 | INFO | MemInfo: { | 20260508 03:21:38.266 | INFO | "gpu" : " 17 GB", | 20260508 03:21:38.266 | INFO | "huge" : "", | 20260508 03:21:38.266 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:38.266 | INFO | "malloc" : " 14 GB", | 20260508 03:21:38.266 | INFO | "pinned" : " 29 GB" | 20260508 03:21:38.266 | INFO | } | 20260508 03:21:38.266 | INFO | Perf report for the last 5.13636 seconds: | 20260508 03:21:38.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:38.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.266 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:21:38.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.266 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:21:38.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:38.267 | INFO | Uploaded 36GiB to GPU in 2.54549s (14.1427GiB/s) | 20260508 03:21:38.267 | INFO | Downloaded 34GiB to GPU in 2.50273s (13.5852GiB/s) | 20260508 03:21:43.407 | INFO | Analyzing chunk frames = [4838912,4839424) | 20260508 03:21:43.407 | INFO | MemInfo: { | 20260508 03:21:43.407 | INFO | "gpu" : " 17 GB", | 20260508 03:21:43.407 | INFO | "huge" : "", | 20260508 03:21:43.407 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:43.407 | INFO | "malloc" : " 14 GB", | 20260508 03:21:43.407 | INFO | "pinned" : " 29 GB" | 20260508 03:21:43.407 | INFO | } | 20260508 03:21:43.407 | INFO | Perf report for the last 5.14104 seconds: | 20260508 03:21:43.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:43.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.407 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 03:21:43.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.407 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:21:43.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:43.408 | INFO | Uploaded 36GiB to GPU in 2.51411s (14.3192GiB/s) | 20260508 03:21:43.408 | INFO | Downloaded 34GiB to GPU in 2.49003s (13.6544GiB/s) | 20260508 03:21:48.552 | INFO | Analyzing chunk frames = [4839424,4839936) | 20260508 03:21:48.552 | INFO | MemInfo: { | 20260508 03:21:48.552 | INFO | "gpu" : " 17 GB", | 20260508 03:21:48.552 | INFO | "huge" : "", | 20260508 03:21:48.552 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:48.552 | INFO | "malloc" : " 14 GB", | 20260508 03:21:48.552 | INFO | "pinned" : " 29 GB" | 20260508 03:21:48.552 | INFO | } | 20260508 03:21:48.552 | INFO | Perf report for the last 5.14512 seconds: | 20260508 03:21:48.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:48.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 03:21:48.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:21:48.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:48.553 | INFO | Uploaded 36GiB to GPU in 2.43623s (14.7769GiB/s) | 20260508 03:21:48.554 | INFO | Downloaded 34GiB to GPU in 2.57891s (13.1839GiB/s) | 20260508 03:21:53.690 | INFO | Analyzing chunk frames = [4839936,4840448) | 20260508 03:21:53.690 | INFO | MemInfo: { | 20260508 03:21:53.690 | INFO | "gpu" : " 17 GB", | 20260508 03:21:53.690 | INFO | "huge" : "", | 20260508 03:21:53.690 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:53.690 | INFO | "malloc" : " 14 GB", | 20260508 03:21:53.690 | INFO | "pinned" : " 29 GB" | 20260508 03:21:53.690 | INFO | } | 20260508 03:21:53.690 | INFO | Perf report for the last 5.13733 seconds: | 20260508 03:21:53.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:53.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:21:53.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:21:53.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:53.690 | INFO | Uploaded 36GiB to GPU in 2.4461s (14.7173GiB/s) | 20260508 03:21:53.690 | INFO | Downloaded 34GiB to GPU in 2.40178s (14.1561GiB/s) | 20260508 03:21:58.826 | INFO | Analyzing chunk frames = [4840448,4840960) | 20260508 03:21:58.826 | INFO | MemInfo: { | 20260508 03:21:58.826 | INFO | "gpu" : " 17 GB", | 20260508 03:21:58.826 | INFO | "huge" : "", | 20260508 03:21:58.826 | INFO | "huge_shared" : " 29 GB", | 20260508 03:21:58.826 | INFO | "malloc" : " 14 GB", | 20260508 03:21:58.826 | INFO | "pinned" : " 29 GB" | 20260508 03:21:58.826 | INFO | } | 20260508 03:21:58.826 | INFO | Perf report for the last 5.13638 seconds: | 20260508 03:21:58.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:21:58.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.826 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:21:58.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.826 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 03:21:58.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:21:58.827 | INFO | Uploaded 36GiB to GPU in 2.46188s (14.623GiB/s) | 20260508 03:21:58.827 | INFO | Downloaded 34GiB to GPU in 2.41668s (14.0689GiB/s) | 20260508 03:22:03.967 | INFO | Analyzing chunk frames = [4840960,4841472) | 20260508 03:22:03.967 | INFO | MemInfo: { | 20260508 03:22:03.967 | INFO | "gpu" : " 17 GB", | 20260508 03:22:03.967 | INFO | "huge" : "", | 20260508 03:22:03.967 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:03.967 | INFO | "malloc" : " 14 GB", | 20260508 03:22:03.968 | INFO | "pinned" : " 29 GB" | 20260508 03:22:03.968 | INFO | } | 20260508 03:22:03.968 | INFO | Perf report for the last 5.14157 seconds: | 20260508 03:22:03.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:03.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:22:03.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 03:22:03.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:03.968 | INFO | Uploaded 36GiB to GPU in 2.49438s (14.4324GiB/s) | 20260508 03:22:03.968 | INFO | Downloaded 34GiB to GPU in 2.42953s (13.9945GiB/s) | 20260508 03:22:03.968 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:22:03.968 | INFO | Total mem usage: 16.7 GiB | 20260508 03:22:03.968 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:22:03.968 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:22:03.968 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:22:03.968 | INFO | High Water Breakdown: | 20260508 03:22:03.968 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:22:03.968 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:22:03.968 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:22:03.968 | INFO | ------------------------------------------ | 20260508 03:22:03.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:22:03.968 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:22:03.968 | INFO | Total mem usage: 26.5 GiB | 20260508 03:22:03.968 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:22:03.968 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:22:03.968 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:22:03.969 | INFO | High Water Breakdown: | 20260508 03:22:03.969 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:22:03.969 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:22:03.969 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:22:03.969 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:22:03.969 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:22:03.969 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:22:03.969 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:22:03.969 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:22:03.969 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:22:03.969 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:22:03.969 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:22:03.969 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:22:03.969 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:22:03.969 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:22:03.969 | INFO | ------------------------------------------ | 20260508 03:22:03.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:22:03.969 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:22:03.969 | INFO | Total mem usage: 27.8 GiB | 20260508 03:22:03.969 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:22:03.969 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:22:03.969 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:22:03.969 | INFO | High Water Breakdown: | 20260508 03:22:03.969 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:22:03.969 | INFO | ------------------------------------------ | 20260508 03:22:03.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:22:03.969 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:22:03.969 | INFO | Total mem usage: 15.7 GiB | 20260508 03:22:03.969 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:22:03.969 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:22:03.969 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:22:03.969 | INFO | High Water Breakdown: | 20260508 03:22:03.969 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:22:03.969 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:22:03.969 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:22:03.969 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:22:03.969 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:22:03.969 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:22:03.969 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:22:03.969 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:22:03.969 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:22:03.969 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:22:03.969 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:22:03.969 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:22:03.969 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:22:03.969 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:22:03.969 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:22:03.970 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:22:03.970 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:22:03.970 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:22:03.970 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:22:03.970 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:22:03.970 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:22:03.970 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:22:03.970 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:22:03.970 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:22:03.970 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:22:03.970 | INFO | ------------------------------------------ | 20260508 03:22:03.970 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:22:03.970 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:22:03.970 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:22:03.970 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:22:03.970 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 1.75e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.18e-02 / 5.25e-03 | 20260508 03:22:03.970 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.49e+00, 5.84e-01 / 1.68e-01 | 20260508 03:22:03.970 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.06e-01 / 1.15e-01 | 20260508 03:22:03.970 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.61e+01, 4.15e+00 / 2.27e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.65e+00 / 1.35e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.19e-01 / 2.17e-01 | 20260508 03:22:03.970 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.23e+00, 2.66e-01 / 1.31e-01 | 20260508 03:22:03.970 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.63e-01, 5.13e-02 / 2.53e-02 | 20260508 03:22:03.970 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:22:03.970 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:22:03.970 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:22:03.970 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 24.52%, Upload -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 30.70%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 2.15%, LockUpload -- 2.29e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 2.76%, LockDownload -- 2.95e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 15.45%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:22:03.970 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 03:22:04.081 | INFO | Starting a round of Baz IO | 20260508 03:22:07.514 | INFO | Populating RTMetrics report | 20260508 03:22:07.515 | INFO | RTMetrics report populated | 20260508 03:22:07.515 | INFO | Opening tmp RTMetrics json file | 20260508 03:22:07.527 | INFO | Serializing RTMetrics to json | 20260508 03:22:07.528 | INFO | Serialization done, swapping tmp file now... | 20260508 03:22:07.528 | INFO | RTMetrics done | 20260508 03:22:07.584 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.503 seconds | 20260508 03:22:07.584 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.30% / 0.00% / 58.69% | 20260508 03:22:07.584 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.626282% / 1.19209e-05% / 7.54992e-05% | 20260508 03:22:07.584 | INFO | Baz IO overall: | 20260508 03:22:07.584 | INFO | BytesWritten : 2177761113555 | 20260508 03:22:07.584 | INFO | NumEvents : 1052177279589 | 20260508 03:22:09.106 | INFO | Analyzing chunk frames = [4841472,4841984) | 20260508 03:22:09.106 | INFO | MemInfo: { | 20260508 03:22:09.106 | INFO | "gpu" : " 17 GB", | 20260508 03:22:09.106 | INFO | "huge" : "", | 20260508 03:22:09.106 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:09.106 | INFO | "malloc" : " 14 GB", | 20260508 03:22:09.106 | INFO | "pinned" : " 29 GB" | 20260508 03:22:09.106 | INFO | } | 20260508 03:22:09.106 | INFO | Perf report for the last 5.13854 seconds: | 20260508 03:22:09.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:09.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:09.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:09.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:09.107 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 03:22:09.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:09.107 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 03:22:09.107 | INFO | PRE_HQ : 66.2%, 1.8, 62.4ms, 0 | 20260508 03:22:09.107 | INFO | BAZWRITER : 67.6%, 2.9, 99.5ms, 0 | 20260508 03:22:09.107 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:22:09.107 | INFO | Uploaded 36GiB to GPU in 2.59311s (13.883GiB/s) | 20260508 03:22:09.107 | INFO | Downloaded 38GiB to GPU in 2.70558s (14.045GiB/s) | 20260508 03:22:14.245 | INFO | Analyzing chunk frames = [4841984,4842496) | 20260508 03:22:14.245 | INFO | MemInfo: { | 20260508 03:22:14.245 | INFO | "gpu" : " 17 GB", | 20260508 03:22:14.245 | INFO | "huge" : "", | 20260508 03:22:14.245 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:14.245 | INFO | "malloc" : " 14 GB", | 20260508 03:22:14.245 | INFO | "pinned" : " 29 GB" | 20260508 03:22:14.245 | INFO | } | 20260508 03:22:14.245 | INFO | Perf report for the last 5.13858 seconds: | 20260508 03:22:14.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:14.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:22:14.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:22:14.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:14.245 | INFO | Uploaded 36GiB to GPU in 2.55948s (14.0654GiB/s) | 20260508 03:22:14.245 | INFO | Downloaded 34GiB to GPU in 2.41569s (14.0746GiB/s) | 20260508 03:22:19.385 | INFO | Analyzing chunk frames = [4842496,4843008) | 20260508 03:22:19.385 | INFO | MemInfo: { | 20260508 03:22:19.385 | INFO | "gpu" : " 17 GB", | 20260508 03:22:19.385 | INFO | "huge" : "", | 20260508 03:22:19.385 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:19.385 | INFO | "malloc" : " 14 GB", | 20260508 03:22:19.385 | INFO | "pinned" : " 29 GB" | 20260508 03:22:19.385 | INFO | } | 20260508 03:22:19.385 | INFO | Perf report for the last 5.14024 seconds: | 20260508 03:22:19.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:19.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:22:19.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:22:19.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:19.385 | INFO | Uploaded 36GiB to GPU in 2.57929s (13.9574GiB/s) | 20260508 03:22:19.386 | INFO | Downloaded 34GiB to GPU in 2.49393s (13.6331GiB/s) | 20260508 03:22:24.520 | INFO | Analyzing chunk frames = [4843008,4843520) | 20260508 03:22:24.520 | INFO | MemInfo: { | 20260508 03:22:24.520 | INFO | "gpu" : " 17 GB", | 20260508 03:22:24.520 | INFO | "huge" : "", | 20260508 03:22:24.520 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:24.520 | INFO | "malloc" : " 14 GB", | 20260508 03:22:24.520 | INFO | "pinned" : " 29 GB" | 20260508 03:22:24.520 | INFO | } | 20260508 03:22:24.520 | INFO | Perf report for the last 5.1351 seconds: | 20260508 03:22:24.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:24.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.520 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 03:22:24.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.520 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:22:24.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:24.521 | INFO | Uploaded 36GiB to GPU in 2.57592s (13.9756GiB/s) | 20260508 03:22:24.521 | INFO | Downloaded 34GiB to GPU in 2.50229s (13.5875GiB/s) | 20260508 03:22:29.662 | INFO | Analyzing chunk frames = [4843520,4844032) | 20260508 03:22:29.662 | INFO | MemInfo: { | 20260508 03:22:29.662 | INFO | "gpu" : " 17 GB", | 20260508 03:22:29.662 | INFO | "huge" : "", | 20260508 03:22:29.662 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:29.662 | INFO | "malloc" : " 14 GB", | 20260508 03:22:29.662 | INFO | "pinned" : " 29 GB" | 20260508 03:22:29.662 | INFO | } | 20260508 03:22:29.662 | INFO | Perf report for the last 5.1421 seconds: | 20260508 03:22:29.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:29.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.662 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 03:22:29.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.663 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:22:29.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:29.663 | INFO | Uploaded 36GiB to GPU in 2.39788s (15.0132GiB/s) | 20260508 03:22:29.663 | INFO | Downloaded 34GiB to GPU in 2.50744s (13.5597GiB/s) | 20260508 03:22:34.807 | INFO | Analyzing chunk frames = [4844032,4844544) | 20260508 03:22:34.807 | INFO | MemInfo: { | 20260508 03:22:34.807 | INFO | "gpu" : " 17 GB", | 20260508 03:22:34.807 | INFO | "huge" : "", | 20260508 03:22:34.807 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:34.807 | INFO | "malloc" : " 14 GB", | 20260508 03:22:34.807 | INFO | "pinned" : " 29 GB" | 20260508 03:22:34.807 | INFO | } | 20260508 03:22:34.807 | INFO | Perf report for the last 5.1448 seconds: | 20260508 03:22:34.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:34.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.807 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:22:34.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.807 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:22:34.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:34.808 | INFO | Uploaded 36GiB to GPU in 2.43594s (14.7787GiB/s) | 20260508 03:22:34.808 | INFO | Downloaded 34GiB to GPU in 2.41999s (14.0497GiB/s) | 20260508 03:22:39.943 | INFO | Analyzing chunk frames = [4844544,4845056) | 20260508 03:22:39.943 | INFO | MemInfo: { | 20260508 03:22:39.943 | INFO | "gpu" : " 17 GB", | 20260508 03:22:39.943 | INFO | "huge" : "", | 20260508 03:22:39.943 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:39.943 | INFO | "malloc" : " 14 GB", | 20260508 03:22:39.943 | INFO | "pinned" : " 29 GB" | 20260508 03:22:39.943 | INFO | } | 20260508 03:22:39.943 | INFO | Perf report for the last 5.13605 seconds: | 20260508 03:22:39.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:39.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.943 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 03:22:39.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.944 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 03:22:39.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:39.944 | INFO | Uploaded 36GiB to GPU in 2.44116s (14.7471GiB/s) | 20260508 03:22:39.944 | INFO | Downloaded 34GiB to GPU in 2.41829s (14.0595GiB/s) | 20260508 03:22:45.078 | INFO | Analyzing chunk frames = [4845056,4845568) | 20260508 03:22:45.078 | INFO | MemInfo: { | 20260508 03:22:45.078 | INFO | "gpu" : " 17 GB", | 20260508 03:22:45.078 | INFO | "huge" : "", | 20260508 03:22:45.078 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:45.078 | INFO | "malloc" : " 14 GB", | 20260508 03:22:45.078 | INFO | "pinned" : " 29 GB" | 20260508 03:22:45.078 | INFO | } | 20260508 03:22:45.078 | INFO | Perf report for the last 5.13518 seconds: | 20260508 03:22:45.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:45.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:22:45.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 03:22:45.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:45.079 | INFO | Uploaded 36GiB to GPU in 2.46173s (14.6239GiB/s) | 20260508 03:22:45.079 | INFO | Downloaded 34GiB to GPU in 2.4157s (14.0746GiB/s) | 20260508 03:22:45.175 | INFO | Starting a round of Baz IO | 20260508 03:22:48.626 | INFO | Populating RTMetrics report | 20260508 03:22:48.626 | INFO | RTMetrics report populated | 20260508 03:22:48.626 | INFO | Opening tmp RTMetrics json file | 20260508 03:22:48.638 | INFO | Serializing RTMetrics to json | 20260508 03:22:48.640 | INFO | Serialization done, swapping tmp file now... | 20260508 03:22:48.640 | INFO | RTMetrics done | 20260508 03:22:48.694 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.518 seconds | 20260508 03:22:48.694 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.30% / 0.00% / 58.70% | 20260508 03:22:48.694 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.629938% / 0% / 6.75519e-05% | 20260508 03:22:48.694 | INFO | Baz IO overall: | 20260508 03:22:48.694 | INFO | BytesWritten : 2179502887967 | 20260508 03:22:48.694 | INFO | NumEvents : 1053010740930 | 20260508 03:22:50.220 | INFO | Analyzing chunk frames = [4845568,4846080) | 20260508 03:22:50.220 | INFO | MemInfo: { | 20260508 03:22:50.220 | INFO | "gpu" : " 17 GB", | 20260508 03:22:50.220 | INFO | "huge" : "", | 20260508 03:22:50.220 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:50.220 | INFO | "malloc" : " 14 GB", | 20260508 03:22:50.220 | INFO | "pinned" : " 29 GB" | 20260508 03:22:50.220 | INFO | } | 20260508 03:22:50.221 | INFO | Perf report for the last 5.14201 seconds: | 20260508 03:22:50.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:50.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:50.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:50.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:50.221 | INFO | ANALYSIS : 68.1%, 3.0, 106.5ms, 0 | 20260508 03:22:50.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:50.221 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:22:50.221 | INFO | PRE_HQ : 65.6%, 1.7, 56.2ms, 0 | 20260508 03:22:50.221 | INFO | BAZWRITER : 67.8%, 2.7, 92.5ms, 0 | 20260508 03:22:50.221 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:22:50.221 | INFO | Uploaded 36GiB to GPU in 2.55099s (14.1122GiB/s) | 20260508 03:22:50.221 | INFO | Downloaded 38GiB to GPU in 2.70004s (14.0739GiB/s) | 20260508 03:22:55.358 | INFO | Analyzing chunk frames = [4846080,4846592) | 20260508 03:22:55.358 | INFO | MemInfo: { | 20260508 03:22:55.358 | INFO | "gpu" : " 17 GB", | 20260508 03:22:55.358 | INFO | "huge" : "", | 20260508 03:22:55.358 | INFO | "huge_shared" : " 29 GB", | 20260508 03:22:55.358 | INFO | "malloc" : " 14 GB", | 20260508 03:22:55.358 | INFO | "pinned" : " 29 GB" | 20260508 03:22:55.358 | INFO | } | 20260508 03:22:55.358 | INFO | Perf report for the last 5.13752 seconds: | 20260508 03:22:55.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:22:55.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:22:55.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:22:55.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:22:55.358 | INFO | Uploaded 36GiB to GPU in 2.56654s (14.0267GiB/s) | 20260508 03:22:55.358 | INFO | Downloaded 34GiB to GPU in 2.43071s (13.9877GiB/s) | 20260508 03:23:00.499 | INFO | Analyzing chunk frames = [4846592,4847104) | 20260508 03:23:00.499 | INFO | MemInfo: { | 20260508 03:23:00.499 | INFO | "gpu" : " 17 GB", | 20260508 03:23:00.499 | INFO | "huge" : "", | 20260508 03:23:00.499 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:00.499 | INFO | "malloc" : " 14 GB", | 20260508 03:23:00.499 | INFO | "pinned" : " 29 GB" | 20260508 03:23:00.499 | INFO | } | 20260508 03:23:00.499 | INFO | Perf report for the last 5.14105 seconds: | 20260508 03:23:00.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:00.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 03:23:00.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:23:00.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:00.499 | INFO | Uploaded 36GiB to GPU in 2.51882s (14.2924GiB/s) | 20260508 03:23:00.499 | INFO | Downloaded 34GiB to GPU in 2.50186s (13.5899GiB/s) | 20260508 03:23:05.636 | INFO | Analyzing chunk frames = [4847104,4847616) | 20260508 03:23:05.636 | INFO | MemInfo: { | 20260508 03:23:05.636 | INFO | "gpu" : " 17 GB", | 20260508 03:23:05.636 | INFO | "huge" : "", | 20260508 03:23:05.636 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:05.636 | INFO | "malloc" : " 14 GB", | 20260508 03:23:05.636 | INFO | "pinned" : " 29 GB" | 20260508 03:23:05.636 | INFO | } | 20260508 03:23:05.636 | INFO | Perf report for the last 5.13701 seconds: | 20260508 03:23:05.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:05.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 03:23:05.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:23:05.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:05.636 | INFO | Uploaded 36GiB to GPU in 2.5309s (14.2242GiB/s) | 20260508 03:23:05.637 | INFO | Downloaded 34GiB to GPU in 2.51123s (13.5392GiB/s) | 20260508 03:23:10.777 | INFO | Analyzing chunk frames = [4847616,4848128) | 20260508 03:23:10.777 | INFO | MemInfo: { | 20260508 03:23:10.777 | INFO | "gpu" : " 17 GB", | 20260508 03:23:10.777 | INFO | "huge" : "", | 20260508 03:23:10.777 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:10.777 | INFO | "malloc" : " 14 GB", | 20260508 03:23:10.777 | INFO | "pinned" : " 29 GB" | 20260508 03:23:10.777 | INFO | } | 20260508 03:23:10.777 | INFO | Perf report for the last 5.1411 seconds: | 20260508 03:23:10.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:10.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.777 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 03:23:10.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.777 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 03:23:10.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:10.778 | INFO | Uploaded 36GiB to GPU in 2.3764s (15.149GiB/s) | 20260508 03:23:10.778 | INFO | Downloaded 34GiB to GPU in 2.57772s (13.19GiB/s) | 20260508 03:23:15.916 | INFO | Analyzing chunk frames = [4848128,4848640) | 20260508 03:23:15.916 | INFO | MemInfo: { | 20260508 03:23:15.916 | INFO | "gpu" : " 17 GB", | 20260508 03:23:15.916 | INFO | "huge" : "", | 20260508 03:23:15.916 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:15.916 | INFO | "malloc" : " 14 GB", | 20260508 03:23:15.916 | INFO | "pinned" : " 29 GB" | 20260508 03:23:15.916 | INFO | } | 20260508 03:23:15.916 | INFO | Perf report for the last 5.1391 seconds: | 20260508 03:23:15.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:15.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.916 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:23:15.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.916 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 03:23:15.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:15.917 | INFO | Uploaded 36GiB to GPU in 2.38876s (15.0706GiB/s) | 20260508 03:23:15.917 | INFO | Downloaded 34GiB to GPU in 2.42446s (14.0238GiB/s) | 20260508 03:23:21.055 | INFO | Analyzing chunk frames = [4848640,4849152) | 20260508 03:23:21.055 | INFO | MemInfo: { | 20260508 03:23:21.055 | INFO | "gpu" : " 17 GB", | 20260508 03:23:21.055 | INFO | "huge" : "", | 20260508 03:23:21.055 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:21.055 | INFO | "malloc" : " 14 GB", | 20260508 03:23:21.055 | INFO | "pinned" : " 29 GB" | 20260508 03:23:21.055 | INFO | } | 20260508 03:23:21.055 | INFO | Perf report for the last 5.1388 seconds: | 20260508 03:23:21.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:21.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:23:21.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 03:23:21.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:21.056 | INFO | Uploaded 36GiB to GPU in 2.45455s (14.6667GiB/s) | 20260508 03:23:21.056 | INFO | Downloaded 34GiB to GPU in 2.44083s (13.9297GiB/s) | 20260508 03:23:26.193 | INFO | Analyzing chunk frames = [4849152,4849664) | 20260508 03:23:26.193 | INFO | MemInfo: { | 20260508 03:23:26.193 | INFO | "gpu" : " 17 GB", | 20260508 03:23:26.193 | INFO | "huge" : "", | 20260508 03:23:26.193 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:26.194 | INFO | "malloc" : " 14 GB", | 20260508 03:23:26.194 | INFO | "pinned" : " 29 GB" | 20260508 03:23:26.194 | INFO | } | 20260508 03:23:26.194 | INFO | Perf report for the last 5.13867 seconds: | 20260508 03:23:26.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:26.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.194 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:23:26.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.194 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 03:23:26.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:26.195 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:23:26.195 | INFO | Total mem usage: 16.7 GiB | 20260508 03:23:26.195 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:23:26.195 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:23:26.195 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:23:26.195 | INFO | High Water Breakdown: | 20260508 03:23:26.195 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:23:26.195 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:23:26.195 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:23:26.195 | INFO | ------------------------------------------ | 20260508 03:23:26.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:23:26.195 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:23:26.195 | INFO | Total mem usage: 26.5 GiB | 20260508 03:23:26.195 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:23:26.195 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:23:26.195 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:23:26.195 | INFO | High Water Breakdown: | 20260508 03:23:26.195 | INFO | Uploaded 36GiB to GPU in 2.44226s (14.7405GiB/s) | 20260508 03:23:26.195 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:23:26.195 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:23:26.195 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:23:26.195 | INFO | Downloaded 34GiB to GPU in 2.4353s (13.9613GiB/s) | 20260508 03:23:26.195 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:23:26.195 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:23:26.195 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:23:26.195 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:23:26.195 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:23:26.195 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:23:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:23:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:23:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:23:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:23:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:23:26.195 | INFO | ------------------------------------------ | 20260508 03:23:26.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:23:26.195 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:23:26.195 | INFO | Total mem usage: 27.8 GiB | 20260508 03:23:26.195 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:23:26.195 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:23:26.195 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:23:26.195 | INFO | High Water Breakdown: | 20260508 03:23:26.195 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:23:26.195 | INFO | ------------------------------------------ | 20260508 03:23:26.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:23:26.195 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:23:26.195 | INFO | Total mem usage: 15.7 GiB | 20260508 03:23:26.195 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:23:26.195 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:23:26.195 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:23:26.195 | INFO | High Water Breakdown: | 20260508 03:23:26.195 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:23:26.195 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:23:26.195 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:23:26.195 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:23:26.195 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:23:26.195 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:23:26.195 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:23:26.195 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:23:26.195 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:23:26.195 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:23:26.195 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:23:26.195 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:23:26.195 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:23:26.195 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:23:26.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:23:26.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:23:26.195 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:23:26.195 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:23:26.195 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:23:26.195 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:23:26.196 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:23:26.196 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:23:26.196 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:23:26.196 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:23:26.196 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:23:26.196 | INFO | ------------------------------------------ | 20260508 03:23:26.196 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:23:26.196 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:23:26.196 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:23:26.196 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:23:26.196 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.34e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.03e-01, 1.34e-02 / 5.76e-03 | 20260508 03:23:26.196 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.48e+00, 5.85e-01 / 2.03e-01 | 20260508 03:23:26.196 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 1.07e-01 | 20260508 03:23:26.196 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 4.00e+00 / 1.59e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.67e+00 / 1.55e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.69e+00, 4.19e-01 / 1.87e-01 | 20260508 03:23:26.196 | INFO | PerfReport: 3.26%, Metrics -- 1.07e+00 / 1.22e+00, 2.63e-01 / 9.88e-02 | 20260508 03:23:26.196 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.27e-01 / 3.61e-01, 5.11e-02 / 1.89e-02 | 20260508 03:23:26.196 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:23:26.196 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:23:26.196 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:23:26.196 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 24.39%, Upload -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 24.41%, Download -- 2.60e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 2.22%, LockUpload -- 2.37e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 2.68%, LockDownload -- 2.85e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 03:23:26.196 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260508 03:23:26.309 | INFO | Starting a round of Baz IO | 20260508 03:23:29.728 | INFO | Populating RTMetrics report | 20260508 03:23:29.728 | INFO | RTMetrics report populated | 20260508 03:23:29.728 | INFO | Opening tmp RTMetrics json file | 20260508 03:23:29.745 | INFO | Serializing RTMetrics to json | 20260508 03:23:29.746 | INFO | Serialization done, swapping tmp file now... | 20260508 03:23:29.747 | INFO | RTMetrics done | 20260508 03:23:29.806 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.497 seconds | 20260508 03:23:29.806 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.30% / 0.00% / 58.70% | 20260508 03:23:29.806 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.63012% / 0% / 7.94729e-05% | 20260508 03:23:29.806 | INFO | Baz IO overall: | 20260508 03:23:29.806 | INFO | BytesWritten : 2181251598756 | 20260508 03:23:29.806 | INFO | NumEvents : 1053845839717 | 20260508 03:23:31.327 | INFO | Analyzing chunk frames = [4849664,4850176) | 20260508 03:23:31.327 | INFO | MemInfo: { | 20260508 03:23:31.327 | INFO | "gpu" : " 17 GB", | 20260508 03:23:31.327 | INFO | "huge" : "", | 20260508 03:23:31.327 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:31.327 | INFO | "malloc" : " 14 GB", | 20260508 03:23:31.327 | INFO | "pinned" : " 29 GB" | 20260508 03:23:31.327 | INFO | } | 20260508 03:23:31.327 | INFO | Perf report for the last 5.13311 seconds: | 20260508 03:23:31.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:31.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:31.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:31.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:31.327 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 03:23:31.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:31.327 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260508 03:23:31.327 | INFO | PRE_HQ : 66.5%, 1.8, 62.4ms, 0 | 20260508 03:23:31.327 | INFO | BAZWRITER : 67.7%, 2.8, 98.6ms, 0 | 20260508 03:23:31.327 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:23:31.327 | INFO | Uploaded 36GiB to GPU in 2.59906s (13.8512GiB/s) | 20260508 03:23:31.327 | INFO | Downloaded 38GiB to GPU in 2.68425s (14.1566GiB/s) | 20260508 03:23:36.477 | INFO | Analyzing chunk frames = [4850176,4850688) | 20260508 03:23:36.477 | INFO | MemInfo: { | 20260508 03:23:36.477 | INFO | "gpu" : " 17 GB", | 20260508 03:23:36.477 | INFO | "huge" : "", | 20260508 03:23:36.477 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:36.477 | INFO | "malloc" : " 14 GB", | 20260508 03:23:36.477 | INFO | "pinned" : " 29 GB" | 20260508 03:23:36.477 | INFO | } | 20260508 03:23:36.477 | INFO | Perf report for the last 5.15003 seconds: | 20260508 03:23:36.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:36.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 03:23:36.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:23:36.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:36.477 | INFO | Uploaded 36GiB to GPU in 2.56535s (14.0332GiB/s) | 20260508 03:23:36.477 | INFO | Downloaded 34GiB to GPU in 2.42617s (14.0138GiB/s) | 20260508 03:23:41.617 | INFO | Analyzing chunk frames = [4850688,4851200) | 20260508 03:23:41.617 | INFO | MemInfo: { | 20260508 03:23:41.617 | INFO | "gpu" : " 17 GB", | 20260508 03:23:41.617 | INFO | "huge" : "", | 20260508 03:23:41.617 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:41.617 | INFO | "malloc" : " 14 GB", | 20260508 03:23:41.617 | INFO | "pinned" : " 29 GB" | 20260508 03:23:41.618 | INFO | } | 20260508 03:23:41.618 | INFO | Perf report for the last 5.14053 seconds: | 20260508 03:23:41.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:41.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260508 03:23:41.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:23:41.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:41.618 | INFO | Uploaded 36GiB to GPU in 2.59812s (13.8562GiB/s) | 20260508 03:23:41.618 | INFO | Downloaded 34GiB to GPU in 2.49803s (13.6108GiB/s) | 20260508 03:23:46.759 | INFO | Analyzing chunk frames = [4851200,4851712) | 20260508 03:23:46.759 | INFO | MemInfo: { | 20260508 03:23:46.759 | INFO | "gpu" : " 17 GB", | 20260508 03:23:46.759 | INFO | "huge" : "", | 20260508 03:23:46.759 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:46.760 | INFO | "malloc" : " 14 GB", | 20260508 03:23:46.760 | INFO | "pinned" : " 29 GB" | 20260508 03:23:46.760 | INFO | } | 20260508 03:23:46.760 | INFO | Perf report for the last 5.14206 seconds: | 20260508 03:23:46.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:46.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:23:46.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:23:46.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:46.760 | INFO | Uploaded 36GiB to GPU in 2.57392s (13.9864GiB/s) | 20260508 03:23:46.760 | INFO | Downloaded 34GiB to GPU in 2.49993s (13.6004GiB/s) | 20260508 03:23:51.888 | INFO | Analyzing chunk frames = [4851712,4852224) | 20260508 03:23:51.888 | INFO | MemInfo: { | 20260508 03:23:51.888 | INFO | "gpu" : " 17 GB", | 20260508 03:23:51.888 | INFO | "huge" : "", | 20260508 03:23:51.888 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:51.888 | INFO | "malloc" : " 14 GB", | 20260508 03:23:51.888 | INFO | "pinned" : " 29 GB" | 20260508 03:23:51.888 | INFO | } | 20260508 03:23:51.888 | INFO | Perf report for the last 5.12868 seconds: | 20260508 03:23:51.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:51.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.888 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:23:51.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.888 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 03:23:51.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:51.889 | INFO | Uploaded 36GiB to GPU in 2.38156s (15.1162GiB/s) | 20260508 03:23:51.889 | INFO | Downloaded 34GiB to GPU in 2.56185s (13.2717GiB/s) | 20260508 03:23:57.035 | INFO | Analyzing chunk frames = [4852224,4852736) | 20260508 03:23:57.035 | INFO | MemInfo: { | 20260508 03:23:57.035 | INFO | "gpu" : " 17 GB", | 20260508 03:23:57.035 | INFO | "huge" : "", | 20260508 03:23:57.035 | INFO | "huge_shared" : " 29 GB", | 20260508 03:23:57.035 | INFO | "malloc" : " 14 GB", | 20260508 03:23:57.035 | INFO | "pinned" : " 29 GB" | 20260508 03:23:57.035 | INFO | } | 20260508 03:23:57.035 | INFO | Perf report for the last 5.14676 seconds: | 20260508 03:23:57.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:23:57.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 03:23:57.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:23:57.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:23:57.035 | INFO | Uploaded 36GiB to GPU in 2.41145s (14.9288GiB/s) | 20260508 03:23:57.035 | INFO | Downloaded 34GiB to GPU in 2.42625s (14.0134GiB/s) | 20260508 03:24:02.171 | INFO | Analyzing chunk frames = [4852736,4853248) | 20260508 03:24:02.171 | INFO | MemInfo: { | 20260508 03:24:02.171 | INFO | "gpu" : " 17 GB", | 20260508 03:24:02.171 | INFO | "huge" : "", | 20260508 03:24:02.172 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:02.172 | INFO | "malloc" : " 14 GB", | 20260508 03:24:02.172 | INFO | "pinned" : " 29 GB" | 20260508 03:24:02.172 | INFO | } | 20260508 03:24:02.172 | INFO | Perf report for the last 5.13663 seconds: | 20260508 03:24:02.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:02.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:24:02.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 03:24:02.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:02.172 | INFO | Uploaded 36GiB to GPU in 2.44537s (14.7217GiB/s) | 20260508 03:24:02.172 | INFO | Downloaded 34GiB to GPU in 2.42321s (14.031GiB/s) | 20260508 03:24:07.310 | INFO | Analyzing chunk frames = [4853248,4853760) | 20260508 03:24:07.310 | INFO | MemInfo: { | 20260508 03:24:07.310 | INFO | "gpu" : " 17 GB", | 20260508 03:24:07.310 | INFO | "huge" : "", | 20260508 03:24:07.310 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:07.310 | INFO | "malloc" : " 14 GB", | 20260508 03:24:07.310 | INFO | "pinned" : " 29 GB" | 20260508 03:24:07.310 | INFO | } | 20260508 03:24:07.310 | INFO | Perf report for the last 5.13842 seconds: | 20260508 03:24:07.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:07.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.310 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:24:07.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.310 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:24:07.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:07.311 | INFO | Uploaded 36GiB to GPU in 2.44823s (14.7045GiB/s) | 20260508 03:24:07.311 | INFO | Downloaded 34GiB to GPU in 2.42791s (14.0038GiB/s) | 20260508 03:24:07.425 | INFO | Starting a round of Baz IO | 20260508 03:24:10.834 | INFO | Populating RTMetrics report | 20260508 03:24:10.834 | INFO | RTMetrics report populated | 20260508 03:24:10.834 | INFO | Opening tmp RTMetrics json file | 20260508 03:24:10.843 | INFO | Serializing RTMetrics to json | 20260508 03:24:10.848 | INFO | Serialization done, swapping tmp file now... | 20260508 03:24:10.849 | INFO | RTMetrics done | 20260508 03:24:10.909 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.483 seconds | 20260508 03:24:10.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.29% / 0.00% / 58.70% | 20260508 03:24:10.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.62896% / 0% / 4.76837e-05% | 20260508 03:24:10.909 | INFO | Baz IO overall: | 20260508 03:24:10.909 | INFO | BytesWritten : 2182996604805 | 20260508 03:24:10.909 | INFO | NumEvents : 1054679317873 | 20260508 03:24:12.450 | INFO | Analyzing chunk frames = [4853760,4854272) | 20260508 03:24:12.450 | INFO | MemInfo: { | 20260508 03:24:12.450 | INFO | "gpu" : " 17 GB", | 20260508 03:24:12.450 | INFO | "huge" : "", | 20260508 03:24:12.450 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:12.450 | INFO | "malloc" : " 14 GB", | 20260508 03:24:12.450 | INFO | "pinned" : " 29 GB" | 20260508 03:24:12.450 | INFO | } | 20260508 03:24:12.451 | INFO | Perf report for the last 5.1403 seconds: | 20260508 03:24:12.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:12.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:12.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:12.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:12.451 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 03:24:12.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:12.451 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 03:24:12.451 | INFO | PRE_HQ : 66.2%, 1.8, 61.9ms, 0 | 20260508 03:24:12.451 | INFO | BAZWRITER : 67.5%, 2.8, 97.3ms, 0 | 20260508 03:24:12.451 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:24:12.451 | INFO | Uploaded 36GiB to GPU in 2.54453s (14.148GiB/s) | 20260508 03:24:12.451 | INFO | Downloaded 38GiB to GPU in 2.69584s (14.0958GiB/s) | 20260508 03:24:17.586 | INFO | Analyzing chunk frames = [4854272,4854784) | 20260508 03:24:17.586 | INFO | MemInfo: { | 20260508 03:24:17.586 | INFO | "gpu" : " 17 GB", | 20260508 03:24:17.586 | INFO | "huge" : "", | 20260508 03:24:17.586 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:17.586 | INFO | "malloc" : " 14 GB", | 20260508 03:24:17.586 | INFO | "pinned" : " 29 GB" | 20260508 03:24:17.586 | INFO | } | 20260508 03:24:17.586 | INFO | Perf report for the last 5.13593 seconds: | 20260508 03:24:17.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:17.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.587 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:24:17.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.587 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:24:17.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:17.587 | INFO | Uploaded 36GiB to GPU in 2.57662s (13.9718GiB/s) | 20260508 03:24:17.587 | INFO | Downloaded 34GiB to GPU in 2.42392s (14.0269GiB/s) | 20260508 03:24:22.725 | INFO | Analyzing chunk frames = [4854784,4855296) | 20260508 03:24:22.725 | INFO | MemInfo: { | 20260508 03:24:22.725 | INFO | "gpu" : " 17 GB", | 20260508 03:24:22.725 | INFO | "huge" : "", | 20260508 03:24:22.725 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:22.725 | INFO | "malloc" : " 14 GB", | 20260508 03:24:22.725 | INFO | "pinned" : " 29 GB" | 20260508 03:24:22.725 | INFO | } | 20260508 03:24:22.725 | INFO | Perf report for the last 5.13894 seconds: | 20260508 03:24:22.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:22.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.726 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:24:22.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.726 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:24:22.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:22.726 | INFO | Uploaded 36GiB to GPU in 2.52787s (14.2412GiB/s) | 20260508 03:24:22.726 | INFO | Downloaded 34GiB to GPU in 2.50288s (13.5844GiB/s) | 20260508 03:24:27.867 | INFO | Analyzing chunk frames = [4855296,4855808) | 20260508 03:24:27.867 | INFO | MemInfo: { | 20260508 03:24:27.867 | INFO | "gpu" : " 17 GB", | 20260508 03:24:27.867 | INFO | "huge" : "", | 20260508 03:24:27.867 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:27.867 | INFO | "malloc" : " 14 GB", | 20260508 03:24:27.867 | INFO | "pinned" : " 29 GB" | 20260508 03:24:27.867 | INFO | } | 20260508 03:24:27.867 | INFO | Perf report for the last 5.14144 seconds: | 20260508 03:24:27.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:27.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.867 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 03:24:27.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.867 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:24:27.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:27.868 | INFO | Uploaded 36GiB to GPU in 2.51864s (14.2934GiB/s) | 20260508 03:24:27.868 | INFO | Downloaded 34GiB to GPU in 2.49408s (13.6323GiB/s) | 20260508 03:24:33.006 | INFO | Analyzing chunk frames = [4855808,4856320) | 20260508 03:24:33.006 | INFO | MemInfo: { | 20260508 03:24:33.006 | INFO | "gpu" : " 17 GB", | 20260508 03:24:33.006 | INFO | "huge" : "", | 20260508 03:24:33.006 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:33.007 | INFO | "malloc" : " 14 GB", | 20260508 03:24:33.007 | INFO | "pinned" : " 29 GB" | 20260508 03:24:33.007 | INFO | } | 20260508 03:24:33.007 | INFO | Perf report for the last 5.13982 seconds: | 20260508 03:24:33.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:33.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 03:24:33.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:24:33.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:33.007 | INFO | Uploaded 36GiB to GPU in 2.42662s (14.8354GiB/s) | 20260508 03:24:33.007 | INFO | Downloaded 34GiB to GPU in 2.58263s (13.1649GiB/s) | 20260508 03:24:38.141 | INFO | Analyzing chunk frames = [4856320,4856832) | 20260508 03:24:38.141 | INFO | MemInfo: { | 20260508 03:24:38.141 | INFO | "gpu" : " 17 GB", | 20260508 03:24:38.141 | INFO | "huge" : "", | 20260508 03:24:38.141 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:38.141 | INFO | "malloc" : " 14 GB", | 20260508 03:24:38.141 | INFO | "pinned" : " 29 GB" | 20260508 03:24:38.141 | INFO | } | 20260508 03:24:38.141 | INFO | Perf report for the last 5.1341 seconds: | 20260508 03:24:38.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:38.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:24:38.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:24:38.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:38.141 | INFO | Uploaded 36GiB to GPU in 2.427s (14.8331GiB/s) | 20260508 03:24:38.141 | INFO | Downloaded 34GiB to GPU in 2.40496s (14.1375GiB/s) | 20260508 03:24:43.282 | INFO | Analyzing chunk frames = [4856832,4857344) | 20260508 03:24:43.282 | INFO | MemInfo: { | 20260508 03:24:43.282 | INFO | "gpu" : " 17 GB", | 20260508 03:24:43.282 | INFO | "huge" : "", | 20260508 03:24:43.282 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:43.282 | INFO | "malloc" : " 14 GB", | 20260508 03:24:43.282 | INFO | "pinned" : " 29 GB" | 20260508 03:24:43.282 | INFO | } | 20260508 03:24:43.283 | INFO | Perf report for the last 5.14163 seconds: | 20260508 03:24:43.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:43.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:24:43.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:24:43.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:43.283 | INFO | Uploaded 36GiB to GPU in 2.49936s (14.4037GiB/s) | 20260508 03:24:43.283 | INFO | Downloaded 34GiB to GPU in 2.42362s (14.0286GiB/s) | 20260508 03:24:48.425 | INFO | Analyzing chunk frames = [4857344,4857856) | 20260508 03:24:48.425 | INFO | MemInfo: { | 20260508 03:24:48.425 | INFO | "gpu" : " 17 GB", | 20260508 03:24:48.425 | INFO | "huge" : "", | 20260508 03:24:48.425 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:48.425 | INFO | "malloc" : " 14 GB", | 20260508 03:24:48.425 | INFO | "pinned" : " 29 GB" | 20260508 03:24:48.425 | INFO | } | 20260508 03:24:48.425 | INFO | Perf report for the last 5.14292 seconds: | 20260508 03:24:48.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:48.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.425 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 03:24:48.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.426 | INFO | PRE_HQ : 35.5%, 1.0, 18.4ms, 0 | 20260508 03:24:48.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:48.426 | INFO | Uploaded 36GiB to GPU in 2.4936s (14.437GiB/s) | 20260508 03:24:48.426 | INFO | Downloaded 34GiB to GPU in 2.4278s (14.0045GiB/s) | 20260508 03:24:48.426 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:24:48.426 | INFO | Total mem usage: 16.7 GiB | 20260508 03:24:48.426 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:24:48.426 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:24:48.426 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:24:48.426 | INFO | High Water Breakdown: | 20260508 03:24:48.426 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:24:48.426 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:24:48.426 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:24:48.426 | INFO | ------------------------------------------ | 20260508 03:24:48.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:24:48.426 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:24:48.426 | INFO | Total mem usage: 26.5 GiB | 20260508 03:24:48.426 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:24:48.426 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:24:48.426 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:24:48.426 | INFO | High Water Breakdown: | 20260508 03:24:48.426 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:24:48.426 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:24:48.426 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:24:48.426 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:24:48.426 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:24:48.426 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:24:48.426 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:24:48.426 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:24:48.426 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:24:48.426 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:24:48.426 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:24:48.426 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:24:48.426 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:24:48.426 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:24:48.426 | INFO | ------------------------------------------ | 20260508 03:24:48.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:24:48.426 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:24:48.426 | INFO | Total mem usage: 27.8 GiB | 20260508 03:24:48.426 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:24:48.426 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:24:48.426 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:24:48.426 | INFO | High Water Breakdown: | 20260508 03:24:48.426 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:24:48.426 | INFO | ------------------------------------------ | 20260508 03:24:48.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:24:48.426 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:24:48.426 | INFO | Total mem usage: 15.7 GiB | 20260508 03:24:48.426 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:24:48.426 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:24:48.426 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:24:48.426 | INFO | High Water Breakdown: | 20260508 03:24:48.426 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:24:48.426 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:24:48.426 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:24:48.426 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:24:48.426 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:24:48.426 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:24:48.426 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:24:48.426 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:24:48.426 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:24:48.426 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:24:48.426 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:24:48.426 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:24:48.426 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:24:48.427 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:24:48.427 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:24:48.427 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:24:48.427 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:24:48.427 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:24:48.427 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:24:48.427 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:24:48.427 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:24:48.427 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:24:48.427 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:24:48.427 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:24:48.427 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:24:48.427 | INFO | ------------------------------------------ | 20260508 03:24:48.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:24:48.427 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:24:48.427 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:24:48.427 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:24:48.427 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.39e+01 / 2.04e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.03e-01, 1.24e-02 / 7.15e-03 | 20260508 03:24:48.427 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.87e-01 / 2.03e-01 | 20260508 03:24:48.427 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.07e-01 / 1.19e-01 | 20260508 03:24:48.427 | INFO | PerfReport: 16.14%, DME -- 4.22e+01 / 4.63e+01, 4.19e+00 / 2.45e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.84e+01 / 1.97e+01, 1.63e+00 / 1.29e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.40e+00 / 2.65e+00, 4.21e-01 / 2.40e-01 | 20260508 03:24:48.427 | INFO | PerfReport: 3.27%, Metrics -- 1.07e+00 / 1.23e+00, 2.68e-01 / 1.07e-01 | 20260508 03:24:48.427 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.28e-01 / 3.61e-01, 5.21e-02 / 2.18e-02 | 20260508 03:24:48.427 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:24:48.427 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:24:48.427 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:24:48.427 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 24.46%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 2.26%, LockUpload -- 2.41e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 15.51%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:24:48.427 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 03:24:48.538 | INFO | Starting a round of Baz IO | 20260508 03:24:51.964 | INFO | Populating RTMetrics report | 20260508 03:24:51.964 | INFO | RTMetrics report populated | 20260508 03:24:51.964 | INFO | Opening tmp RTMetrics json file | 20260508 03:24:51.979 | INFO | Serializing RTMetrics to json | 20260508 03:24:51.980 | INFO | Serialization done, swapping tmp file now... | 20260508 03:24:51.981 | INFO | RTMetrics done | 20260508 03:24:52.037 | INFO | Finished a round of Baz IO: Wrote 1.622 GiB in 3.498 seconds | 20260508 03:24:52.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.29% / 0.00% / 58.71% | 20260508 03:24:52.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.625861% / 1.58946e-05% / 5.5631e-05% | 20260508 03:24:52.037 | INFO | Baz IO overall: | 20260508 03:24:52.037 | INFO | BytesWritten : 2184737820278 | 20260508 03:24:52.037 | INFO | NumEvents : 1055511925279 | 20260508 03:24:53.569 | INFO | Analyzing chunk frames = [4857856,4858368) | 20260508 03:24:53.569 | INFO | MemInfo: { | 20260508 03:24:53.569 | INFO | "gpu" : " 17 GB", | 20260508 03:24:53.569 | INFO | "huge" : "", | 20260508 03:24:53.569 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:53.569 | INFO | "malloc" : " 14 GB", | 20260508 03:24:53.569 | INFO | "pinned" : " 29 GB" | 20260508 03:24:53.569 | INFO | } | 20260508 03:24:53.569 | INFO | Perf report for the last 5.14383 seconds: | 20260508 03:24:53.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:53.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:53.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:53.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:53.569 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 03:24:53.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:53.569 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:24:53.569 | INFO | PRE_HQ : 66.4%, 1.8, 63.0ms, 0 | 20260508 03:24:53.570 | INFO | BAZWRITER : 67.5%, 2.8, 98.6ms, 0 | 20260508 03:24:53.570 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:24:53.570 | INFO | Uploaded 36GiB to GPU in 2.57879s (13.96GiB/s) | 20260508 03:24:53.570 | INFO | Downloaded 38GiB to GPU in 2.69924s (14.078GiB/s) | 20260508 03:24:58.708 | INFO | Analyzing chunk frames = [4858368,4858880) | 20260508 03:24:58.708 | INFO | MemInfo: { | 20260508 03:24:58.708 | INFO | "gpu" : " 17 GB", | 20260508 03:24:58.708 | INFO | "huge" : "", | 20260508 03:24:58.708 | INFO | "huge_shared" : " 29 GB", | 20260508 03:24:58.708 | INFO | "malloc" : " 14 GB", | 20260508 03:24:58.708 | INFO | "pinned" : " 29 GB" | 20260508 03:24:58.708 | INFO | } | 20260508 03:24:58.708 | INFO | Perf report for the last 5.13914 seconds: | 20260508 03:24:58.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:24:58.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.708 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:24:58.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.709 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:24:58.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:24:58.709 | INFO | Uploaded 36GiB to GPU in 2.56584s (14.0305GiB/s) | 20260508 03:24:58.709 | INFO | Downloaded 34GiB to GPU in 2.43699s (13.9516GiB/s) | 20260508 03:25:03.843 | INFO | Analyzing chunk frames = [4858880,4859392) | 20260508 03:25:03.843 | INFO | MemInfo: { | 20260508 03:25:03.843 | INFO | "gpu" : " 17 GB", | 20260508 03:25:03.843 | INFO | "huge" : "", | 20260508 03:25:03.843 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:03.843 | INFO | "malloc" : " 14 GB", | 20260508 03:25:03.843 | INFO | "pinned" : " 29 GB" | 20260508 03:25:03.843 | INFO | } | 20260508 03:25:03.843 | INFO | Perf report for the last 5.13484 seconds: | 20260508 03:25:03.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:03.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.843 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 03:25:03.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.843 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:25:03.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:03.844 | INFO | Uploaded 36GiB to GPU in 2.5297s (14.231GiB/s) | 20260508 03:25:03.844 | INFO | Downloaded 34GiB to GPU in 2.4216s (14.0403GiB/s) | 20260508 03:25:08.984 | INFO | Analyzing chunk frames = [4859392,4859904) | 20260508 03:25:08.984 | INFO | MemInfo: { | 20260508 03:25:08.984 | INFO | "gpu" : " 17 GB", | 20260508 03:25:08.984 | INFO | "huge" : "", | 20260508 03:25:08.984 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:08.984 | INFO | "malloc" : " 14 GB", | 20260508 03:25:08.984 | INFO | "pinned" : " 29 GB" | 20260508 03:25:08.984 | INFO | } | 20260508 03:25:08.984 | INFO | Perf report for the last 5.14102 seconds: | 20260508 03:25:08.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:08.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.984 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 03:25:08.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.984 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:25:08.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:08.985 | INFO | Uploaded 36GiB to GPU in 2.55549s (14.0873GiB/s) | 20260508 03:25:08.985 | INFO | Downloaded 34GiB to GPU in 2.50802s (13.5565GiB/s) | 20260508 03:25:14.124 | INFO | Analyzing chunk frames = [4859904,4860416) | 20260508 03:25:14.124 | INFO | MemInfo: { | 20260508 03:25:14.124 | INFO | "gpu" : " 17 GB", | 20260508 03:25:14.124 | INFO | "huge" : "", | 20260508 03:25:14.124 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:14.124 | INFO | "malloc" : " 14 GB", | 20260508 03:25:14.124 | INFO | "pinned" : " 29 GB" | 20260508 03:25:14.124 | INFO | } | 20260508 03:25:14.124 | INFO | Perf report for the last 5.13967 seconds: | 20260508 03:25:14.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:14.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 03:25:14.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:25:14.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:14.124 | INFO | Uploaded 36GiB to GPU in 2.35746s (15.2706GiB/s) | 20260508 03:25:14.124 | INFO | Downloaded 34GiB to GPU in 2.55955s (13.2836GiB/s) | 20260508 03:25:19.261 | INFO | Analyzing chunk frames = [4860416,4860928) | 20260508 03:25:19.261 | INFO | MemInfo: { | 20260508 03:25:19.261 | INFO | "gpu" : " 17 GB", | 20260508 03:25:19.261 | INFO | "huge" : "", | 20260508 03:25:19.261 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:19.261 | INFO | "malloc" : " 14 GB", | 20260508 03:25:19.261 | INFO | "pinned" : " 29 GB" | 20260508 03:25:19.261 | INFO | } | 20260508 03:25:19.261 | INFO | Perf report for the last 5.13708 seconds: | 20260508 03:25:19.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:19.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.261 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:25:19.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.261 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:25:19.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:19.262 | INFO | Uploaded 36GiB to GPU in 2.39251s (15.047GiB/s) | 20260508 03:25:19.262 | INFO | Downloaded 34GiB to GPU in 2.42116s (14.0429GiB/s) | 20260508 03:25:24.396 | INFO | Analyzing chunk frames = [4860928,4861440) | 20260508 03:25:24.396 | INFO | MemInfo: { | 20260508 03:25:24.396 | INFO | "gpu" : " 17 GB", | 20260508 03:25:24.396 | INFO | "huge" : "", | 20260508 03:25:24.396 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:24.396 | INFO | "malloc" : " 14 GB", | 20260508 03:25:24.396 | INFO | "pinned" : " 29 GB" | 20260508 03:25:24.396 | INFO | } | 20260508 03:25:24.396 | INFO | Perf report for the last 5.13467 seconds: | 20260508 03:25:24.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:24.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:25:24.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 03:25:24.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:24.396 | INFO | Uploaded 36GiB to GPU in 2.43152s (14.8056GiB/s) | 20260508 03:25:24.396 | INFO | Downloaded 34GiB to GPU in 2.4284s (14.001GiB/s) | 20260508 03:25:29.535 | INFO | Analyzing chunk frames = [4861440,4861952) | 20260508 03:25:29.535 | INFO | MemInfo: { | 20260508 03:25:29.535 | INFO | "gpu" : " 17 GB", | 20260508 03:25:29.535 | INFO | "huge" : "", | 20260508 03:25:29.535 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:29.535 | INFO | "malloc" : " 14 GB", | 20260508 03:25:29.535 | INFO | "pinned" : " 29 GB" | 20260508 03:25:29.535 | INFO | } | 20260508 03:25:29.535 | INFO | Perf report for the last 5.13949 seconds: | 20260508 03:25:29.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:29.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.535 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:25:29.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.535 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:25:29.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:29.536 | INFO | Uploaded 36GiB to GPU in 2.45238s (14.6796GiB/s) | 20260508 03:25:29.536 | INFO | Downloaded 34GiB to GPU in 2.43527s (13.9615GiB/s) | 20260508 03:25:29.633 | INFO | Starting a round of Baz IO | 20260508 03:25:33.071 | INFO | Populating RTMetrics report | 20260508 03:25:33.072 | INFO | RTMetrics report populated | 20260508 03:25:33.072 | INFO | Opening tmp RTMetrics json file | 20260508 03:25:33.080 | INFO | Serializing RTMetrics to json | 20260508 03:25:33.086 | INFO | Serialization done, swapping tmp file now... | 20260508 03:25:33.086 | INFO | RTMetrics done | 20260508 03:25:33.147 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.514 seconds | 20260508 03:25:33.147 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.29% / 0.00% / 58.71% | 20260508 03:25:33.147 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.630256% / 7.94729e-06% / 3.17891e-05% | 20260508 03:25:33.147 | INFO | Baz IO overall: | 20260508 03:25:33.147 | INFO | BytesWritten : 2186481316954 | 20260508 03:25:33.147 | INFO | NumEvents : 1056347047718 | 20260508 03:25:34.680 | INFO | Analyzing chunk frames = [4861952,4862464) | 20260508 03:25:34.680 | INFO | MemInfo: { | 20260508 03:25:34.680 | INFO | "gpu" : " 17 GB", | 20260508 03:25:34.680 | INFO | "huge" : "", | 20260508 03:25:34.680 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:34.680 | INFO | "malloc" : " 14 GB", | 20260508 03:25:34.680 | INFO | "pinned" : " 29 GB" | 20260508 03:25:34.680 | INFO | } | 20260508 03:25:34.680 | INFO | Perf report for the last 5.14451 seconds: | 20260508 03:25:34.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:34.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:34.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:34.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:34.680 | INFO | ANALYSIS : 68.0%, 3.0, 106.4ms, 0 | 20260508 03:25:34.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:34.680 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:25:34.680 | INFO | PRE_HQ : 65.4%, 1.6, 55.8ms, 0 | 20260508 03:25:34.680 | INFO | BAZWRITER : 67.7%, 2.7, 93.5ms, 0 | 20260508 03:25:34.680 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:25:34.680 | INFO | Uploaded 36GiB to GPU in 2.57889s (13.9595GiB/s) | 20260508 03:25:34.680 | INFO | Downloaded 38GiB to GPU in 2.69597s (14.0951GiB/s) | 20260508 03:25:39.820 | INFO | Analyzing chunk frames = [4862464,4862976) | 20260508 03:25:39.820 | INFO | MemInfo: { | 20260508 03:25:39.820 | INFO | "gpu" : " 17 GB", | 20260508 03:25:39.820 | INFO | "huge" : "", | 20260508 03:25:39.820 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:39.820 | INFO | "malloc" : " 14 GB", | 20260508 03:25:39.820 | INFO | "pinned" : " 29 GB" | 20260508 03:25:39.820 | INFO | } | 20260508 03:25:39.820 | INFO | Perf report for the last 5.14003 seconds: | 20260508 03:25:39.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:39.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.820 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:25:39.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.820 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:25:39.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:39.821 | INFO | Uploaded 36GiB to GPU in 2.57865s (13.9608GiB/s) | 20260508 03:25:39.821 | INFO | Downloaded 34GiB to GPU in 2.41482s (14.0797GiB/s) | 20260508 03:25:44.955 | INFO | Analyzing chunk frames = [4862976,4863488) | 20260508 03:25:44.955 | INFO | MemInfo: { | 20260508 03:25:44.955 | INFO | "gpu" : " 17 GB", | 20260508 03:25:44.955 | INFO | "huge" : "", | 20260508 03:25:44.955 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:44.955 | INFO | "malloc" : " 14 GB", | 20260508 03:25:44.955 | INFO | "pinned" : " 29 GB" | 20260508 03:25:44.955 | INFO | } | 20260508 03:25:44.955 | INFO | Perf report for the last 5.13518 seconds: | 20260508 03:25:44.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:44.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.955 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:25:44.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.955 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:25:44.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:44.956 | INFO | Uploaded 36GiB to GPU in 2.56779s (14.0198GiB/s) | 20260508 03:25:44.956 | INFO | Downloaded 34GiB to GPU in 2.41944s (14.0529GiB/s) | 20260508 03:25:50.094 | INFO | Analyzing chunk frames = [4863488,4864000) | 20260508 03:25:50.094 | INFO | MemInfo: { | 20260508 03:25:50.094 | INFO | "gpu" : " 17 GB", | 20260508 03:25:50.094 | INFO | "huge" : "", | 20260508 03:25:50.095 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:50.095 | INFO | "malloc" : " 14 GB", | 20260508 03:25:50.095 | INFO | "pinned" : " 29 GB" | 20260508 03:25:50.095 | INFO | } | 20260508 03:25:50.095 | INFO | Perf report for the last 5.13971 seconds: | 20260508 03:25:50.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:50.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:25:50.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:25:50.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:50.095 | INFO | Uploaded 36GiB to GPU in 2.57301s (13.9914GiB/s) | 20260508 03:25:50.095 | INFO | Downloaded 34GiB to GPU in 2.5021s (13.5886GiB/s) | 20260508 03:25:55.236 | INFO | Analyzing chunk frames = [4864000,4864512) | 20260508 03:25:55.236 | INFO | MemInfo: { | 20260508 03:25:55.236 | INFO | "gpu" : " 17 GB", | 20260508 03:25:55.236 | INFO | "huge" : "", | 20260508 03:25:55.236 | INFO | "huge_shared" : " 29 GB", | 20260508 03:25:55.236 | INFO | "malloc" : " 14 GB", | 20260508 03:25:55.236 | INFO | "pinned" : " 29 GB" | 20260508 03:25:55.236 | INFO | } | 20260508 03:25:55.237 | INFO | Perf report for the last 5.14195 seconds: | 20260508 03:25:55.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:25:55.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 03:25:55.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 03:25:55.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:25:55.237 | INFO | Uploaded 36GiB to GPU in 2.40463s (14.9711GiB/s) | 20260508 03:25:55.237 | INFO | Downloaded 34GiB to GPU in 2.54416s (13.3639GiB/s) | 20260508 03:26:00.383 | INFO | Analyzing chunk frames = [4864512,4865024) | 20260508 03:26:00.383 | INFO | MemInfo: { | 20260508 03:26:00.383 | INFO | "gpu" : " 17 GB", | 20260508 03:26:00.383 | INFO | "huge" : "", | 20260508 03:26:00.383 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:00.384 | INFO | "malloc" : " 14 GB", | 20260508 03:26:00.384 | INFO | "pinned" : " 29 GB" | 20260508 03:26:00.384 | INFO | } | 20260508 03:26:00.384 | INFO | Perf report for the last 5.14687 seconds: | 20260508 03:26:00.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:00.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:26:00.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260508 03:26:00.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:00.384 | INFO | Uploaded 36GiB to GPU in 2.45232s (14.68GiB/s) | 20260508 03:26:00.384 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0225GiB/s) | 20260508 03:26:05.522 | INFO | Analyzing chunk frames = [4865024,4865536) | 20260508 03:26:05.522 | INFO | MemInfo: { | 20260508 03:26:05.522 | INFO | "gpu" : " 17 GB", | 20260508 03:26:05.522 | INFO | "huge" : "", | 20260508 03:26:05.522 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:05.522 | INFO | "malloc" : " 14 GB", | 20260508 03:26:05.522 | INFO | "pinned" : " 29 GB" | 20260508 03:26:05.522 | INFO | } | 20260508 03:26:05.522 | INFO | Perf report for the last 5.13824 seconds: | 20260508 03:26:05.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:05.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:26:05.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:26:05.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:05.522 | INFO | Uploaded 36GiB to GPU in 2.45504s (14.6637GiB/s) | 20260508 03:26:05.522 | INFO | Downloaded 34GiB to GPU in 2.42263s (14.0344GiB/s) | 20260508 03:26:10.654 | INFO | Analyzing chunk frames = [4865536,4866048) | 20260508 03:26:10.654 | INFO | MemInfo: { | 20260508 03:26:10.654 | INFO | "gpu" : " 17 GB", | 20260508 03:26:10.654 | INFO | "huge" : "", | 20260508 03:26:10.654 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:10.654 | INFO | "malloc" : " 14 GB", | 20260508 03:26:10.654 | INFO | "pinned" : " 29 GB" | 20260508 03:26:10.654 | INFO | } | 20260508 03:26:10.654 | INFO | Perf report for the last 5.13261 seconds: | 20260508 03:26:10.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:10.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.654 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:26:10.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.655 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:26:10.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:10.655 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:26:10.655 | INFO | Total mem usage: 16.7 GiB | 20260508 03:26:10.655 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:26:10.655 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:26:10.655 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:26:10.655 | INFO | High Water Breakdown: | 20260508 03:26:10.655 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:26:10.655 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:26:10.655 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:26:10.655 | INFO | Uploaded 36GiB to GPU in 2.46065s (14.6303GiB/s) | 20260508 03:26:10.655 | INFO | ------------------------------------------ | 20260508 03:26:10.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:26:10.655 | INFO | Downloaded 34GiB to GPU in 2.42074s (14.0453GiB/s) | 20260508 03:26:10.655 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:26:10.655 | INFO | Total mem usage: 26.5 GiB | 20260508 03:26:10.655 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:26:10.655 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:26:10.655 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:26:10.655 | INFO | High Water Breakdown: | 20260508 03:26:10.655 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:26:10.655 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:26:10.655 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:26:10.655 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:26:10.655 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:26:10.655 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:26:10.655 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:26:10.655 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:26:10.655 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:26:10.655 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:26:10.655 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:26:10.655 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:26:10.655 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:26:10.655 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:26:10.655 | INFO | ------------------------------------------ | 20260508 03:26:10.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:26:10.655 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:26:10.656 | INFO | Total mem usage: 27.8 GiB | 20260508 03:26:10.656 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:26:10.656 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:26:10.656 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:26:10.656 | INFO | High Water Breakdown: | 20260508 03:26:10.656 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:26:10.656 | INFO | ------------------------------------------ | 20260508 03:26:10.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:26:10.656 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:26:10.656 | INFO | Total mem usage: 15.7 GiB | 20260508 03:26:10.656 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:26:10.656 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:26:10.656 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:26:10.656 | INFO | High Water Breakdown: | 20260508 03:26:10.656 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:26:10.656 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:26:10.656 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:26:10.656 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:26:10.656 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:26:10.656 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:26:10.656 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:26:10.656 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:26:10.656 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:26:10.656 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:26:10.656 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:26:10.656 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:26:10.656 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:26:10.656 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:26:10.656 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:26:10.656 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:26:10.656 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:26:10.656 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:26:10.656 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:26:10.656 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:26:10.656 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:26:10.656 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:26:10.656 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:26:10.656 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:26:10.656 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:26:10.656 | INFO | ------------------------------------------ | 20260508 03:26:10.656 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:26:10.656 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:26:10.656 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:26:10.656 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:26:10.656 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.37e+01 / 1.45e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.10e-02 / 5.67e-03 | 20260508 03:26:10.656 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.49e+00, 5.83e-01 / 1.98e-01 | 20260508 03:26:10.656 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.06e-01 / 1.17e-01 | 20260508 03:26:10.656 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.57e+01, 3.87e+00 / 1.88e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.75e+00 / 1.51e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.65e+00, 4.16e-01 / 2.28e-01 | 20260508 03:26:10.656 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.23e+00, 2.64e-01 / 1.41e-01 | 20260508 03:26:10.656 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.63e-01, 5.17e-02 / 2.73e-02 | 20260508 03:26:10.656 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:26:10.656 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:26:10.656 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:26:10.656 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 2.27%, LockUpload -- 2.41e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 2.57%, LockDownload -- 2.74e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 15.63%, LockCompute -- 1.66e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 03:26:10.656 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260508 03:26:10.752 | INFO | Starting a round of Baz IO | 20260508 03:26:14.196 | INFO | Populating RTMetrics report | 20260508 03:26:14.196 | INFO | RTMetrics report populated | 20260508 03:26:14.196 | INFO | Opening tmp RTMetrics json file | 20260508 03:26:14.208 | INFO | Serializing RTMetrics to json | 20260508 03:26:14.209 | INFO | Serialization done, swapping tmp file now... | 20260508 03:26:14.209 | INFO | RTMetrics done | 20260508 03:26:14.267 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.515 seconds | 20260508 03:26:14.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.29% / 0.00% / 58.71% | 20260508 03:26:14.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.626222% / 3.97364e-06% / 4.76837e-05% | 20260508 03:26:14.267 | INFO | Baz IO overall: | 20260508 03:26:14.267 | INFO | BytesWritten : 2188219986215 | 20260508 03:26:14.267 | INFO | NumEvents : 1057179162530 | 20260508 03:26:15.797 | INFO | Analyzing chunk frames = [4866048,4866560) | 20260508 03:26:15.797 | INFO | MemInfo: { | 20260508 03:26:15.797 | INFO | "gpu" : " 17 GB", | 20260508 03:26:15.797 | INFO | "huge" : "", | 20260508 03:26:15.797 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:15.797 | INFO | "malloc" : " 14 GB", | 20260508 03:26:15.797 | INFO | "pinned" : " 29 GB" | 20260508 03:26:15.797 | INFO | } | 20260508 03:26:15.797 | INFO | Perf report for the last 5.14271 seconds: | 20260508 03:26:15.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:15.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:15.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:15.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:15.797 | INFO | ANALYSIS : 67.9%, 3.0, 106.3ms, 0 | 20260508 03:26:15.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:15.797 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:26:15.797 | INFO | PRE_HQ : 65.6%, 1.7, 56.3ms, 0 | 20260508 03:26:15.797 | INFO | BAZWRITER : 67.9%, 2.7, 93.0ms, 0 | 20260508 03:26:15.797 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:26:15.797 | INFO | Uploaded 36GiB to GPU in 2.57303s (13.9913GiB/s) | 20260508 03:26:15.797 | INFO | Downloaded 38GiB to GPU in 2.69013s (14.1257GiB/s) | 20260508 03:26:20.935 | INFO | Analyzing chunk frames = [4866560,4867072) | 20260508 03:26:20.935 | INFO | MemInfo: { | 20260508 03:26:20.935 | INFO | "gpu" : " 17 GB", | 20260508 03:26:20.935 | INFO | "huge" : "", | 20260508 03:26:20.935 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:20.935 | INFO | "malloc" : " 14 GB", | 20260508 03:26:20.935 | INFO | "pinned" : " 29 GB" | 20260508 03:26:20.935 | INFO | } | 20260508 03:26:20.935 | INFO | Perf report for the last 5.13801 seconds: | 20260508 03:26:20.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:20.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:26:20.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:26:20.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:20.935 | INFO | Uploaded 36GiB to GPU in 2.56117s (14.0561GiB/s) | 20260508 03:26:20.935 | INFO | Downloaded 34GiB to GPU in 2.43002s (13.9917GiB/s) | 20260508 03:26:26.075 | INFO | Analyzing chunk frames = [4867072,4867584) | 20260508 03:26:26.075 | INFO | MemInfo: { | 20260508 03:26:26.075 | INFO | "gpu" : " 17 GB", | 20260508 03:26:26.075 | INFO | "huge" : "", | 20260508 03:26:26.075 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:26.075 | INFO | "malloc" : " 14 GB", | 20260508 03:26:26.075 | INFO | "pinned" : " 29 GB" | 20260508 03:26:26.075 | INFO | } | 20260508 03:26:26.075 | INFO | Perf report for the last 5.13972 seconds: | 20260508 03:26:26.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:26.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 03:26:26.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:26:26.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:26.075 | INFO | Uploaded 36GiB to GPU in 2.53454s (14.2038GiB/s) | 20260508 03:26:26.075 | INFO | Downloaded 34GiB to GPU in 2.50092s (13.595GiB/s) | 20260508 03:26:31.207 | INFO | Analyzing chunk frames = [4867584,4868096) | 20260508 03:26:31.207 | INFO | MemInfo: { | 20260508 03:26:31.207 | INFO | "gpu" : " 17 GB", | 20260508 03:26:31.207 | INFO | "huge" : "", | 20260508 03:26:31.207 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:31.207 | INFO | "malloc" : " 14 GB", | 20260508 03:26:31.207 | INFO | "pinned" : " 29 GB" | 20260508 03:26:31.207 | INFO | } | 20260508 03:26:31.207 | INFO | Perf report for the last 5.13245 seconds: | 20260508 03:26:31.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:31.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.207 | INFO | ANALYSIS : 67.4%, 3.0, 104.1ms, 0 | 20260508 03:26:31.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.207 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:26:31.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:31.208 | INFO | Uploaded 36GiB to GPU in 2.54037s (14.1711GiB/s) | 20260508 03:26:31.208 | INFO | Downloaded 34GiB to GPU in 2.50591s (13.5679GiB/s) | 20260508 03:26:36.349 | INFO | Analyzing chunk frames = [4868096,4868608) | 20260508 03:26:36.349 | INFO | MemInfo: { | 20260508 03:26:36.349 | INFO | "gpu" : " 17 GB", | 20260508 03:26:36.349 | INFO | "huge" : "", | 20260508 03:26:36.349 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:36.349 | INFO | "malloc" : " 14 GB", | 20260508 03:26:36.349 | INFO | "pinned" : " 29 GB" | 20260508 03:26:36.349 | INFO | } | 20260508 03:26:36.349 | INFO | Perf report for the last 5.14187 seconds: | 20260508 03:26:36.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:36.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.349 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:26:36.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.349 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 03:26:36.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:36.350 | INFO | Uploaded 36GiB to GPU in 2.4724s (14.5608GiB/s) | 20260508 03:26:36.350 | INFO | Downloaded 34GiB to GPU in 2.60186s (13.0676GiB/s) | 20260508 03:26:41.491 | INFO | Analyzing chunk frames = [4868608,4869120) | 20260508 03:26:41.491 | INFO | MemInfo: { | 20260508 03:26:41.491 | INFO | "gpu" : " 17 GB", | 20260508 03:26:41.491 | INFO | "huge" : "", | 20260508 03:26:41.491 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:41.491 | INFO | "malloc" : " 14 GB", | 20260508 03:26:41.491 | INFO | "pinned" : " 29 GB" | 20260508 03:26:41.491 | INFO | } | 20260508 03:26:41.492 | INFO | Perf report for the last 5.14225 seconds: | 20260508 03:26:41.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:41.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 03:26:41.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:26:41.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:41.492 | INFO | Uploaded 36GiB to GPU in 2.3711s (15.1828GiB/s) | 20260508 03:26:41.492 | INFO | Downloaded 34GiB to GPU in 2.43577s (13.9586GiB/s) | 20260508 03:26:46.631 | INFO | Analyzing chunk frames = [4869120,4869632) | 20260508 03:26:46.631 | INFO | MemInfo: { | 20260508 03:26:46.631 | INFO | "gpu" : " 17 GB", | 20260508 03:26:46.631 | INFO | "huge" : "", | 20260508 03:26:46.631 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:46.631 | INFO | "malloc" : " 14 GB", | 20260508 03:26:46.631 | INFO | "pinned" : " 29 GB" | 20260508 03:26:46.631 | INFO | } | 20260508 03:26:46.631 | INFO | Perf report for the last 5.13929 seconds: | 20260508 03:26:46.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:46.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:26:46.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:26:46.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:46.631 | INFO | Uploaded 36GiB to GPU in 2.40585s (14.9635GiB/s) | 20260508 03:26:46.631 | INFO | Downloaded 34GiB to GPU in 2.44179s (13.9242GiB/s) | 20260508 03:26:51.772 | INFO | Analyzing chunk frames = [4869632,4870144) | 20260508 03:26:51.772 | INFO | MemInfo: { | 20260508 03:26:51.772 | INFO | "gpu" : " 17 GB", | 20260508 03:26:51.772 | INFO | "huge" : "", | 20260508 03:26:51.772 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:51.772 | INFO | "malloc" : " 14 GB", | 20260508 03:26:51.772 | INFO | "pinned" : " 29 GB" | 20260508 03:26:51.772 | INFO | } | 20260508 03:26:51.772 | INFO | Perf report for the last 5.14132 seconds: | 20260508 03:26:51.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:51.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.772 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:26:51.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.772 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 03:26:51.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:51.773 | INFO | Uploaded 36GiB to GPU in 2.52707s (14.2457GiB/s) | 20260508 03:26:51.773 | INFO | Downloaded 34GiB to GPU in 2.47228s (13.7525GiB/s) | 20260508 03:26:51.870 | INFO | Starting a round of Baz IO | 20260508 03:26:55.308 | INFO | Populating RTMetrics report | 20260508 03:26:55.308 | INFO | RTMetrics report populated | 20260508 03:26:55.308 | INFO | Opening tmp RTMetrics json file | 20260508 03:26:55.319 | INFO | Serializing RTMetrics to json | 20260508 03:26:55.321 | INFO | Serialization done, swapping tmp file now... | 20260508 03:26:55.321 | INFO | RTMetrics done | 20260508 03:26:55.377 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.507 seconds | 20260508 03:26:55.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.28% / 0.00% / 58.71% | 20260508 03:26:55.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.62685% / 3.97364e-06% / 4.37101e-05% | 20260508 03:26:55.377 | INFO | Baz IO overall: | 20260508 03:26:55.377 | INFO | BytesWritten : 2189959394709 | 20260508 03:26:55.377 | INFO | NumEvents : 1058012210220 | 20260508 03:26:56.908 | INFO | Analyzing chunk frames = [4870144,4870656) | 20260508 03:26:56.908 | INFO | MemInfo: { | 20260508 03:26:56.908 | INFO | "gpu" : " 17 GB", | 20260508 03:26:56.908 | INFO | "huge" : "", | 20260508 03:26:56.908 | INFO | "huge_shared" : " 29 GB", | 20260508 03:26:56.908 | INFO | "malloc" : " 14 GB", | 20260508 03:26:56.908 | INFO | "pinned" : " 29 GB" | 20260508 03:26:56.908 | INFO | } | 20260508 03:26:56.908 | INFO | Perf report for the last 5.13569 seconds: | 20260508 03:26:56.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:26:56.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:56.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:56.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:56.908 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 03:26:56.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:26:56.908 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:26:56.908 | INFO | PRE_HQ : 65.3%, 1.7, 56.2ms, 0 | 20260508 03:26:56.908 | INFO | BAZWRITER : 67.7%, 2.7, 93.6ms, 0 | 20260508 03:26:56.908 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:26:56.908 | INFO | Uploaded 36GiB to GPU in 2.49551s (14.4259GiB/s) | 20260508 03:26:56.908 | INFO | Downloaded 38GiB to GPU in 2.70022s (14.0729GiB/s) | 20260508 03:27:02.050 | INFO | Analyzing chunk frames = [4870656,4871168) | 20260508 03:27:02.050 | INFO | MemInfo: { | 20260508 03:27:02.050 | INFO | "gpu" : " 17 GB", | 20260508 03:27:02.050 | INFO | "huge" : "", | 20260508 03:27:02.050 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:02.050 | INFO | "malloc" : " 14 GB", | 20260508 03:27:02.050 | INFO | "pinned" : " 29 GB" | 20260508 03:27:02.050 | INFO | } | 20260508 03:27:02.050 | INFO | Perf report for the last 5.14192 seconds: | 20260508 03:27:02.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:02.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:27:02.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:27:02.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:02.050 | INFO | Uploaded 36GiB to GPU in 2.56211s (14.0509GiB/s) | 20260508 03:27:02.050 | INFO | Downloaded 34GiB to GPU in 2.42926s (13.996GiB/s) | 20260508 03:27:07.190 | INFO | Analyzing chunk frames = [4871168,4871680) | 20260508 03:27:07.190 | INFO | MemInfo: { | 20260508 03:27:07.190 | INFO | "gpu" : " 17 GB", | 20260508 03:27:07.190 | INFO | "huge" : "", | 20260508 03:27:07.190 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:07.190 | INFO | "malloc" : " 14 GB", | 20260508 03:27:07.190 | INFO | "pinned" : " 29 GB" | 20260508 03:27:07.190 | INFO | } | 20260508 03:27:07.190 | INFO | Perf report for the last 5.14063 seconds: | 20260508 03:27:07.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:07.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.190 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 03:27:07.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.191 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 03:27:07.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:07.191 | INFO | Uploaded 36GiB to GPU in 2.52923s (14.2336GiB/s) | 20260508 03:27:07.191 | INFO | Downloaded 34GiB to GPU in 2.41272s (14.092GiB/s) | 20260508 03:27:12.327 | INFO | Analyzing chunk frames = [4871680,4872192) | 20260508 03:27:12.327 | INFO | MemInfo: { | 20260508 03:27:12.327 | INFO | "gpu" : " 17 GB", | 20260508 03:27:12.327 | INFO | "huge" : "", | 20260508 03:27:12.327 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:12.328 | INFO | "malloc" : " 14 GB", | 20260508 03:27:12.328 | INFO | "pinned" : " 29 GB" | 20260508 03:27:12.328 | INFO | } | 20260508 03:27:12.328 | INFO | Perf report for the last 5.13725 seconds: | 20260508 03:27:12.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:12.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 03:27:12.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:27:12.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:12.328 | INFO | Uploaded 36GiB to GPU in 2.54082s (14.1687GiB/s) | 20260508 03:27:12.328 | INFO | Downloaded 34GiB to GPU in 2.51088s (13.541GiB/s) | 20260508 03:27:17.467 | INFO | Analyzing chunk frames = [4872192,4872704) | 20260508 03:27:17.467 | INFO | MemInfo: { | 20260508 03:27:17.467 | INFO | "gpu" : " 17 GB", | 20260508 03:27:17.467 | INFO | "huge" : "", | 20260508 03:27:17.467 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:17.467 | INFO | "malloc" : " 14 GB", | 20260508 03:27:17.467 | INFO | "pinned" : " 29 GB" | 20260508 03:27:17.467 | INFO | } | 20260508 03:27:17.467 | INFO | Perf report for the last 5.1394 seconds: | 20260508 03:27:17.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:17.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 03:27:17.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:27:17.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:17.467 | INFO | Uploaded 36GiB to GPU in 2.42554s (14.8421GiB/s) | 20260508 03:27:17.467 | INFO | Downloaded 34GiB to GPU in 2.55876s (13.2877GiB/s) | 20260508 03:27:22.605 | INFO | Analyzing chunk frames = [4872704,4873216) | 20260508 03:27:22.605 | INFO | MemInfo: { | 20260508 03:27:22.605 | INFO | "gpu" : " 17 GB", | 20260508 03:27:22.605 | INFO | "huge" : "", | 20260508 03:27:22.605 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:22.605 | INFO | "malloc" : " 14 GB", | 20260508 03:27:22.605 | INFO | "pinned" : " 29 GB" | 20260508 03:27:22.605 | INFO | } | 20260508 03:27:22.605 | INFO | Perf report for the last 5.13833 seconds: | 20260508 03:27:22.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:22.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.606 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:27:22.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.606 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:27:22.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:22.606 | INFO | Uploaded 36GiB to GPU in 2.41851s (14.8852GiB/s) | 20260508 03:27:22.606 | INFO | Downloaded 34GiB to GPU in 2.41687s (14.0678GiB/s) | 20260508 03:27:27.749 | INFO | Analyzing chunk frames = [4873216,4873728) | 20260508 03:27:27.749 | INFO | MemInfo: { | 20260508 03:27:27.749 | INFO | "gpu" : " 17 GB", | 20260508 03:27:27.749 | INFO | "huge" : "", | 20260508 03:27:27.749 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:27.749 | INFO | "malloc" : " 14 GB", | 20260508 03:27:27.749 | INFO | "pinned" : " 29 GB" | 20260508 03:27:27.749 | INFO | } | 20260508 03:27:27.749 | INFO | Perf report for the last 5.14373 seconds: | 20260508 03:27:27.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:27.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.749 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:27:27.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.749 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:27:27.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:27.750 | INFO | Uploaded 36GiB to GPU in 2.41802s (14.8882GiB/s) | 20260508 03:27:27.750 | INFO | Downloaded 34GiB to GPU in 2.42562s (14.017GiB/s) | 20260508 03:27:32.885 | INFO | Analyzing chunk frames = [4873728,4874240) | 20260508 03:27:32.885 | INFO | MemInfo: { | 20260508 03:27:32.885 | INFO | "gpu" : " 17 GB", | 20260508 03:27:32.885 | INFO | "huge" : "", | 20260508 03:27:32.885 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:32.885 | INFO | "malloc" : " 14 GB", | 20260508 03:27:32.885 | INFO | "pinned" : " 29 GB" | 20260508 03:27:32.885 | INFO | } | 20260508 03:27:32.885 | INFO | Perf report for the last 5.13611 seconds: | 20260508 03:27:32.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:32.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.885 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:27:32.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.885 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:27:32.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:32.886 | INFO | Uploaded 36GiB to GPU in 2.4474s (14.7095GiB/s) | 20260508 03:27:32.886 | INFO | Downloaded 34GiB to GPU in 2.426s (14.0148GiB/s) | 20260508 03:27:32.886 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:27:32.886 | INFO | Total mem usage: 16.7 GiB | 20260508 03:27:32.886 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:27:32.886 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.9%) | 20260508 03:27:32.886 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:27:32.886 | INFO | High Water Breakdown: | 20260508 03:27:32.886 | INFO | 59.4% : 8.48 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:27:32.886 | INFO | 40.4% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:27:32.886 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:27:32.886 | INFO | ------------------------------------------ | 20260508 03:27:32.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:27:32.886 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:27:32.886 | INFO | Total mem usage: 26.5 GiB | 20260508 03:27:32.886 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:27:32.886 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:27:32.886 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:27:32.886 | INFO | High Water Breakdown: | 20260508 03:27:32.886 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:27:32.886 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:27:32.886 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:27:32.886 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:27:32.886 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:27:32.886 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:27:32.886 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:27:32.886 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:27:32.886 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:27:32.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:27:32.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:27:32.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:27:32.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:27:32.886 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:27:32.886 | INFO | ------------------------------------------ | 20260508 03:27:32.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:27:32.886 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:27:32.886 | INFO | Total mem usage: 27.8 GiB | 20260508 03:27:32.886 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:27:32.886 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:27:32.886 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:27:32.886 | INFO | High Water Breakdown: | 20260508 03:27:32.886 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:27:32.886 | INFO | ------------------------------------------ | 20260508 03:27:32.887 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:27:32.887 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:27:32.887 | INFO | Total mem usage: 15.7 GiB | 20260508 03:27:32.887 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:27:32.887 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:27:32.887 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:27:32.887 | INFO | High Water Breakdown: | 20260508 03:27:32.887 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:27:32.887 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:27:32.887 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:27:32.887 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:27:32.887 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:27:32.887 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:27:32.887 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:27:32.887 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:27:32.887 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:27:32.887 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:27:32.887 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:27:32.887 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:27:32.887 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:27:32.887 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:27:32.887 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:27:32.887 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:27:32.887 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:27:32.887 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:27:32.887 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:27:32.887 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:27:32.887 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:27:32.887 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:27:32.887 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:27:32.887 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:27:32.887 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:27:32.887 | INFO | ------------------------------------------ | 20260508 03:27:32.887 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:27:32.887 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:27:32.887 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:27:32.887 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:27:32.887 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.64e+01, 1.41e+01 / 2.12e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.16e-02 / 7.84e-03 | 20260508 03:27:32.887 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.46e+00, 5.84e-01 / 1.47e-01 | 20260508 03:27:32.887 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.06e-01 / 1.33e-01 | 20260508 03:27:32.887 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.63e+01, 4.31e+00 / 2.61e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.73e+00 / 1.50e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.19e-01 / 2.01e-01 | 20260508 03:27:32.887 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.23e+00, 2.62e-01 / 1.49e-01 | 20260508 03:27:32.887 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.54e-01, 4.97e-02 / 1.99e-02 | 20260508 03:27:32.887 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:27:32.887 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:27:32.887 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:27:32.887 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 24.42%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 30.71%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 2.09%, LockUpload -- 2.23e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 2.86%, LockDownload -- 3.05e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:27:32.887 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 03:27:32.982 | INFO | Starting a round of Baz IO | 20260508 03:27:36.423 | INFO | Populating RTMetrics report | 20260508 03:27:36.423 | INFO | RTMetrics report populated | 20260508 03:27:36.423 | INFO | Opening tmp RTMetrics json file | 20260508 03:27:36.437 | INFO | Serializing RTMetrics to json | 20260508 03:27:36.439 | INFO | Serialization done, swapping tmp file now... | 20260508 03:27:36.440 | INFO | RTMetrics done | 20260508 03:27:36.502 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.519 seconds | 20260508 03:27:36.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.28% / 0.00% / 58.72% | 20260508 03:27:36.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.624839% / 0% / 5.5631e-05% | 20260508 03:27:36.502 | INFO | Baz IO overall: | 20260508 03:27:36.502 | INFO | BytesWritten : 2191698757872 | 20260508 03:27:36.502 | INFO | NumEvents : 1058845429518 | 20260508 03:27:38.025 | INFO | Analyzing chunk frames = [4874240,4874752) | 20260508 03:27:38.025 | INFO | MemInfo: { | 20260508 03:27:38.025 | INFO | "gpu" : " 17 GB", | 20260508 03:27:38.025 | INFO | "huge" : "", | 20260508 03:27:38.025 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:38.025 | INFO | "malloc" : " 14 GB", | 20260508 03:27:38.025 | INFO | "pinned" : " 29 GB" | 20260508 03:27:38.025 | INFO | } | 20260508 03:27:38.025 | INFO | Perf report for the last 5.13984 seconds: | 20260508 03:27:38.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:38.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:38.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:38.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:38.025 | INFO | ANALYSIS : 68.1%, 3.0, 106.4ms, 0 | 20260508 03:27:38.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:38.025 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:27:38.025 | INFO | PRE_HQ : 65.7%, 1.6, 56.0ms, 0 | 20260508 03:27:38.025 | INFO | BAZWRITER : 67.8%, 2.7, 94.5ms, 0 | 20260508 03:27:38.025 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:27:38.026 | INFO | Uploaded 36GiB to GPU in 2.56809s (14.0182GiB/s) | 20260508 03:27:38.026 | INFO | Downloaded 38GiB to GPU in 2.70293s (14.0588GiB/s) | 20260508 03:27:43.158 | INFO | Analyzing chunk frames = [4874752,4875264) | 20260508 03:27:43.158 | INFO | MemInfo: { | 20260508 03:27:43.158 | INFO | "gpu" : " 17 GB", | 20260508 03:27:43.158 | INFO | "huge" : "", | 20260508 03:27:43.158 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:43.158 | INFO | "malloc" : " 14 GB", | 20260508 03:27:43.158 | INFO | "pinned" : " 29 GB" | 20260508 03:27:43.158 | INFO | } | 20260508 03:27:43.158 | INFO | Perf report for the last 5.13308 seconds: | 20260508 03:27:43.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:43.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.158 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 03:27:43.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.158 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:27:43.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:43.159 | INFO | Uploaded 36GiB to GPU in 2.58012s (13.9528GiB/s) | 20260508 03:27:43.159 | INFO | Downloaded 34GiB to GPU in 2.43902s (13.94GiB/s) | 20260508 03:27:48.299 | INFO | Analyzing chunk frames = [4875264,4875776) | 20260508 03:27:48.299 | INFO | MemInfo: { | 20260508 03:27:48.299 | INFO | "gpu" : " 17 GB", | 20260508 03:27:48.299 | INFO | "huge" : "", | 20260508 03:27:48.299 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:48.299 | INFO | "malloc" : " 14 GB", | 20260508 03:27:48.299 | INFO | "pinned" : " 29 GB" | 20260508 03:27:48.299 | INFO | } | 20260508 03:27:48.299 | INFO | Perf report for the last 5.14098 seconds: | 20260508 03:27:48.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:48.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.299 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:27:48.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.299 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:27:48.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:48.300 | INFO | Uploaded 36GiB to GPU in 2.54232s (14.1603GiB/s) | 20260508 03:27:48.300 | INFO | Downloaded 34GiB to GPU in 2.51059s (13.5426GiB/s) | 20260508 03:27:53.445 | INFO | Analyzing chunk frames = [4875776,4876288) | 20260508 03:27:53.445 | INFO | MemInfo: { | 20260508 03:27:53.445 | INFO | "gpu" : " 17 GB", | 20260508 03:27:53.445 | INFO | "huge" : "", | 20260508 03:27:53.445 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:53.445 | INFO | "malloc" : " 14 GB", | 20260508 03:27:53.445 | INFO | "pinned" : " 29 GB" | 20260508 03:27:53.445 | INFO | } | 20260508 03:27:53.445 | INFO | Perf report for the last 5.14559 seconds: | 20260508 03:27:53.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:53.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 03:27:53.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:27:53.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:53.445 | INFO | Uploaded 36GiB to GPU in 2.53624s (14.1942GiB/s) | 20260508 03:27:53.445 | INFO | Downloaded 34GiB to GPU in 2.50276s (13.585GiB/s) | 20260508 03:27:58.587 | INFO | Analyzing chunk frames = [4876288,4876800) | 20260508 03:27:58.587 | INFO | MemInfo: { | 20260508 03:27:58.587 | INFO | "gpu" : " 17 GB", | 20260508 03:27:58.587 | INFO | "huge" : "", | 20260508 03:27:58.587 | INFO | "huge_shared" : " 29 GB", | 20260508 03:27:58.587 | INFO | "malloc" : " 14 GB", | 20260508 03:27:58.587 | INFO | "pinned" : " 29 GB" | 20260508 03:27:58.587 | INFO | } | 20260508 03:27:58.587 | INFO | Perf report for the last 5.14198 seconds: | 20260508 03:27:58.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:27:58.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:27:58.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 03:27:58.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:27:58.587 | INFO | Uploaded 36GiB to GPU in 2.42055s (14.8726GiB/s) | 20260508 03:27:58.587 | INFO | Downloaded 34GiB to GPU in 2.57773s (13.1899GiB/s) | 20260508 03:28:03.719 | INFO | Analyzing chunk frames = [4876800,4877312) | 20260508 03:28:03.719 | INFO | MemInfo: { | 20260508 03:28:03.719 | INFO | "gpu" : " 17 GB", | 20260508 03:28:03.719 | INFO | "huge" : "", | 20260508 03:28:03.719 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:03.719 | INFO | "malloc" : " 14 GB", | 20260508 03:28:03.719 | INFO | "pinned" : " 29 GB" | 20260508 03:28:03.719 | INFO | } | 20260508 03:28:03.719 | INFO | Perf report for the last 5.13228 seconds: | 20260508 03:28:03.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:03.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 03:28:03.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:28:03.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:03.719 | INFO | Uploaded 36GiB to GPU in 2.4557s (14.6597GiB/s) | 20260508 03:28:03.719 | INFO | Downloaded 34GiB to GPU in 2.41651s (14.0699GiB/s) | 20260508 03:28:08.860 | INFO | Analyzing chunk frames = [4877312,4877824) | 20260508 03:28:08.860 | INFO | MemInfo: { | 20260508 03:28:08.860 | INFO | "gpu" : " 17 GB", | 20260508 03:28:08.860 | INFO | "huge" : "", | 20260508 03:28:08.860 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:08.860 | INFO | "malloc" : " 14 GB", | 20260508 03:28:08.860 | INFO | "pinned" : " 29 GB" | 20260508 03:28:08.860 | INFO | } | 20260508 03:28:08.860 | INFO | Perf report for the last 5.14069 seconds: | 20260508 03:28:08.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:08.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.860 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:28:08.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.860 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:28:08.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:08.861 | INFO | Uploaded 36GiB to GPU in 2.46109s (14.6276GiB/s) | 20260508 03:28:08.861 | INFO | Downloaded 34GiB to GPU in 2.41417s (14.0835GiB/s) | 20260508 03:28:14.001 | INFO | Analyzing chunk frames = [4877824,4878336) | 20260508 03:28:14.001 | INFO | MemInfo: { | 20260508 03:28:14.001 | INFO | "gpu" : " 17 GB", | 20260508 03:28:14.001 | INFO | "huge" : "", | 20260508 03:28:14.001 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:14.001 | INFO | "malloc" : " 14 GB", | 20260508 03:28:14.001 | INFO | "pinned" : " 29 GB" | 20260508 03:28:14.001 | INFO | } | 20260508 03:28:14.001 | INFO | Perf report for the last 5.14146 seconds: | 20260508 03:28:14.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:14.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.001 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:28:14.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.001 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 03:28:14.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:14.002 | INFO | Uploaded 36GiB to GPU in 2.47151s (14.566GiB/s) | 20260508 03:28:14.002 | INFO | Downloaded 34GiB to GPU in 2.42467s (14.0225GiB/s) | 20260508 03:28:14.099 | INFO | Starting a round of Baz IO | 20260508 03:28:17.534 | INFO | Populating RTMetrics report | 20260508 03:28:17.534 | INFO | RTMetrics report populated | 20260508 03:28:17.534 | INFO | Opening tmp RTMetrics json file | 20260508 03:28:17.542 | INFO | Serializing RTMetrics to json | 20260508 03:28:17.548 | INFO | Serialization done, swapping tmp file now... | 20260508 03:28:17.549 | INFO | RTMetrics done | 20260508 03:28:17.609 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.51 seconds | 20260508 03:28:17.609 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.28% / 0.00% / 58.72% | 20260508 03:28:17.609 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.622431% / 3.97364e-06% / 5.5631e-05% | 20260508 03:28:17.609 | INFO | Baz IO overall: | 20260508 03:28:17.609 | INFO | BytesWritten : 2193435529055 | 20260508 03:28:17.609 | INFO | NumEvents : 1059676862391 | 20260508 03:28:19.139 | INFO | Analyzing chunk frames = [4878336,4878848) | 20260508 03:28:19.139 | INFO | MemInfo: { | 20260508 03:28:19.139 | INFO | "gpu" : " 17 GB", | 20260508 03:28:19.139 | INFO | "huge" : "", | 20260508 03:28:19.139 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:19.139 | INFO | "malloc" : " 14 GB", | 20260508 03:28:19.139 | INFO | "pinned" : " 29 GB" | 20260508 03:28:19.139 | INFO | } | 20260508 03:28:19.139 | INFO | Perf report for the last 5.1376 seconds: | 20260508 03:28:19.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:19.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:19.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:19.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:19.139 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 03:28:19.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:19.139 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:28:19.139 | INFO | PRE_HQ : 65.9%, 1.6, 55.8ms, 0 | 20260508 03:28:19.139 | INFO | BAZWRITER : 67.7%, 2.7, 92.5ms, 0 | 20260508 03:28:19.139 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:28:19.139 | INFO | Uploaded 36GiB to GPU in 2.57457s (13.9829GiB/s) | 20260508 03:28:19.139 | INFO | Downloaded 38GiB to GPU in 2.70467s (14.0498GiB/s) | 20260508 03:28:24.280 | INFO | Analyzing chunk frames = [4878848,4879360) | 20260508 03:28:24.280 | INFO | MemInfo: { | 20260508 03:28:24.280 | INFO | "gpu" : " 17 GB", | 20260508 03:28:24.280 | INFO | "huge" : "", | 20260508 03:28:24.280 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:24.280 | INFO | "malloc" : " 14 GB", | 20260508 03:28:24.280 | INFO | "pinned" : " 29 GB" | 20260508 03:28:24.280 | INFO | } | 20260508 03:28:24.280 | INFO | Perf report for the last 5.14113 seconds: | 20260508 03:28:24.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:24.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:28:24.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:28:24.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:24.280 | INFO | Uploaded 36GiB to GPU in 2.59031s (13.898GiB/s) | 20260508 03:28:24.280 | INFO | Downloaded 34GiB to GPU in 2.41269s (14.0922GiB/s) | 20260508 03:28:29.417 | INFO | Analyzing chunk frames = [4879360,4879872) | 20260508 03:28:29.417 | INFO | MemInfo: { | 20260508 03:28:29.417 | INFO | "gpu" : " 17 GB", | 20260508 03:28:29.417 | INFO | "huge" : "", | 20260508 03:28:29.417 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:29.417 | INFO | "malloc" : " 14 GB", | 20260508 03:28:29.417 | INFO | "pinned" : " 29 GB" | 20260508 03:28:29.417 | INFO | } | 20260508 03:28:29.417 | INFO | Perf report for the last 5.13735 seconds: | 20260508 03:28:29.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:29.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.417 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:28:29.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.418 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:28:29.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:29.418 | INFO | Uploaded 36GiB to GPU in 2.56984s (14.0086GiB/s) | 20260508 03:28:29.418 | INFO | Downloaded 34GiB to GPU in 2.42109s (14.0433GiB/s) | 20260508 03:28:34.556 | INFO | Analyzing chunk frames = [4879872,4880384) | 20260508 03:28:34.556 | INFO | MemInfo: { | 20260508 03:28:34.556 | INFO | "gpu" : " 17 GB", | 20260508 03:28:34.556 | INFO | "huge" : "", | 20260508 03:28:34.556 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:34.556 | INFO | "malloc" : " 14 GB", | 20260508 03:28:34.556 | INFO | "pinned" : " 29 GB" | 20260508 03:28:34.556 | INFO | } | 20260508 03:28:34.556 | INFO | Perf report for the last 5.13901 seconds: | 20260508 03:28:34.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:34.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.556 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:28:34.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.556 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:28:34.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:34.557 | INFO | Uploaded 36GiB to GPU in 2.56635s (14.0277GiB/s) | 20260508 03:28:34.557 | INFO | Downloaded 34GiB to GPU in 2.49392s (13.6332GiB/s) | 20260508 03:28:39.701 | INFO | Analyzing chunk frames = [4880384,4880896) | 20260508 03:28:39.701 | INFO | MemInfo: { | 20260508 03:28:39.701 | INFO | "gpu" : " 17 GB", | 20260508 03:28:39.701 | INFO | "huge" : "", | 20260508 03:28:39.701 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:39.701 | INFO | "malloc" : " 14 GB", | 20260508 03:28:39.701 | INFO | "pinned" : " 29 GB" | 20260508 03:28:39.701 | INFO | } | 20260508 03:28:39.701 | INFO | Perf report for the last 5.14465 seconds: | 20260508 03:28:39.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:39.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.701 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 03:28:39.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.701 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:28:39.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:39.702 | INFO | Uploaded 36GiB to GPU in 2.40957s (14.9404GiB/s) | 20260508 03:28:39.702 | INFO | Downloaded 34GiB to GPU in 2.55071s (13.3296GiB/s) | 20260508 03:28:44.835 | INFO | Analyzing chunk frames = [4880896,4881408) | 20260508 03:28:44.835 | INFO | MemInfo: { | 20260508 03:28:44.835 | INFO | "gpu" : " 17 GB", | 20260508 03:28:44.835 | INFO | "huge" : "", | 20260508 03:28:44.835 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:44.835 | INFO | "malloc" : " 14 GB", | 20260508 03:28:44.835 | INFO | "pinned" : " 29 GB" | 20260508 03:28:44.835 | INFO | } | 20260508 03:28:44.835 | INFO | Perf report for the last 5.13393 seconds: | 20260508 03:28:44.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:44.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.835 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:28:44.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.835 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:28:44.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:44.836 | INFO | Uploaded 36GiB to GPU in 2.41643s (14.898GiB/s) | 20260508 03:28:44.836 | INFO | Downloaded 34GiB to GPU in 2.41791s (14.0617GiB/s) | 20260508 03:28:49.977 | INFO | Analyzing chunk frames = [4881408,4881920) | 20260508 03:28:49.977 | INFO | MemInfo: { | 20260508 03:28:49.977 | INFO | "gpu" : " 17 GB", | 20260508 03:28:49.977 | INFO | "huge" : "", | 20260508 03:28:49.977 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:49.977 | INFO | "malloc" : " 14 GB", | 20260508 03:28:49.977 | INFO | "pinned" : " 29 GB" | 20260508 03:28:49.977 | INFO | } | 20260508 03:28:49.978 | INFO | Perf report for the last 5.14239 seconds: | 20260508 03:28:49.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:49.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.978 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 03:28:49.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.978 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260508 03:28:49.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:49.979 | INFO | Uploaded 36GiB to GPU in 2.4416s (14.7444GiB/s) | 20260508 03:28:49.979 | INFO | Downloaded 34GiB to GPU in 2.42196s (14.0382GiB/s) | 20260508 03:28:55.115 | INFO | Analyzing chunk frames = [4881920,4882432) | 20260508 03:28:55.115 | INFO | MemInfo: { | 20260508 03:28:55.115 | INFO | "gpu" : " 17 GB", | 20260508 03:28:55.115 | INFO | "huge" : "", | 20260508 03:28:55.115 | INFO | "huge_shared" : " 29 GB", | 20260508 03:28:55.115 | INFO | "malloc" : " 14 GB", | 20260508 03:28:55.115 | INFO | "pinned" : " 29 GB" | 20260508 03:28:55.115 | INFO | } | 20260508 03:28:55.115 | INFO | Perf report for the last 5.13764 seconds: | 20260508 03:28:55.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:28:55.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.115 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:28:55.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.115 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 03:28:55.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:28:55.116 | INFO | Uploaded 36GiB to GPU in 2.44657s (14.7145GiB/s) | 20260508 03:28:55.116 | INFO | Downloaded 34GiB to GPU in 2.43753s (13.9486GiB/s) | 20260508 03:28:55.116 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:28:55.116 | INFO | Total mem usage: 16.7 GiB | 20260508 03:28:55.116 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:28:55.116 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 03:28:55.116 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:28:55.116 | INFO | High Water Breakdown: | 20260508 03:28:55.116 | INFO | 59.4% : 8.48 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:28:55.116 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:28:55.116 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:28:55.116 | INFO | ------------------------------------------ | 20260508 03:28:55.116 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:28:55.116 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:28:55.116 | INFO | Total mem usage: 26.5 GiB | 20260508 03:28:55.116 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:28:55.116 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:28:55.116 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:28:55.116 | INFO | High Water Breakdown: | 20260508 03:28:55.116 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:28:55.116 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:28:55.117 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:28:55.117 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:28:55.117 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:28:55.117 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:28:55.117 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:28:55.117 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:28:55.117 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:28:55.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:28:55.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:28:55.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:28:55.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:28:55.117 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:28:55.117 | INFO | ------------------------------------------ | 20260508 03:28:55.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:28:55.117 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:28:55.117 | INFO | Total mem usage: 27.8 GiB | 20260508 03:28:55.117 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:28:55.117 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:28:55.117 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:28:55.117 | INFO | High Water Breakdown: | 20260508 03:28:55.117 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:28:55.117 | INFO | ------------------------------------------ | 20260508 03:28:55.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:28:55.117 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:28:55.117 | INFO | Total mem usage: 15.7 GiB | 20260508 03:28:55.117 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:28:55.117 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:28:55.117 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:28:55.117 | INFO | High Water Breakdown: | 20260508 03:28:55.117 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:28:55.117 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:28:55.117 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:28:55.117 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:28:55.117 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:28:55.117 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:28:55.117 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:28:55.117 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:28:55.117 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:28:55.117 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:28:55.117 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:28:55.117 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:28:55.117 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:28:55.117 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:28:55.117 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:28:55.117 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:28:55.117 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:28:55.117 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:28:55.117 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:28:55.117 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:28:55.117 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:28:55.117 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:28:55.117 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:28:55.117 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:28:55.117 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:28:55.117 | INFO | ------------------------------------------ | 20260508 03:28:55.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:28:55.118 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:28:55.118 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:28:55.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:28:55.118 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.68e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.02e-01, 1.20e-02 / 6.44e-03 | 20260508 03:28:55.118 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.46e+00, 5.86e-01 / 1.90e-01 | 20260508 03:28:55.118 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.09e-01 | 20260508 03:28:55.118 | INFO | PerfReport: 16.08%, DME -- 4.20e+01 / 4.62e+01, 4.09e+00 / 2.06e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.65e+00 / 1.42e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.40e+00 / 2.67e+00, 4.18e-01 / 2.13e-01 | 20260508 03:28:55.118 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.23e+00, 2.62e-01 / 1.30e-01 | 20260508 03:28:55.118 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.63e-01, 5.20e-02 / 2.50e-02 | 20260508 03:28:55.118 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:28:55.118 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:28:55.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:28:55.118 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 24.47%, Upload -- 2.61e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 24.35%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 30.66%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 2.29%, LockUpload -- 2.44e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 2.68%, LockDownload -- 2.86e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 15.54%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:28:55.118 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260508 03:28:55.227 | INFO | Starting a round of Baz IO | 20260508 03:28:58.652 | INFO | Populating RTMetrics report | 20260508 03:28:58.653 | INFO | RTMetrics report populated | 20260508 03:28:58.653 | INFO | Opening tmp RTMetrics json file | 20260508 03:28:58.654 | INFO | Serializing RTMetrics to json | 20260508 03:28:58.655 | INFO | Serialization done, swapping tmp file now... | 20260508 03:28:58.655 | INFO | RTMetrics done | 20260508 03:28:58.726 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.498 seconds | 20260508 03:28:58.726 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.28% / 0.00% / 58.72% | 20260508 03:28:58.726 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.618843% / 3.97364e-06% / 5.96046e-05% | 20260508 03:28:58.726 | INFO | Baz IO overall: | 20260508 03:28:58.726 | INFO | BytesWritten : 2195171617272 | 20260508 03:28:58.726 | INFO | NumEvents : 1060507833270 | 20260508 03:29:00.255 | INFO | Analyzing chunk frames = [4882432,4882944) | 20260508 03:29:00.255 | INFO | MemInfo: { | 20260508 03:29:00.255 | INFO | "gpu" : " 17 GB", | 20260508 03:29:00.255 | INFO | "huge" : "", | 20260508 03:29:00.255 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:00.255 | INFO | "malloc" : " 14 GB", | 20260508 03:29:00.255 | INFO | "pinned" : " 29 GB" | 20260508 03:29:00.255 | INFO | } | 20260508 03:29:00.255 | INFO | Perf report for the last 5.13972 seconds: | 20260508 03:29:00.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:00.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:00.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:00.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:00.255 | INFO | ANALYSIS : 68.4%, 3.0, 106.1ms, 0 | 20260508 03:29:00.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:00.255 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 03:29:00.255 | INFO | PRE_HQ : 66.6%, 1.8, 62.0ms, 0 | 20260508 03:29:00.255 | INFO | BAZWRITER : 67.5%, 2.8, 97.5ms, 0 | 20260508 03:29:00.255 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 03:29:00.256 | INFO | Uploaded 36GiB to GPU in 2.49883s (14.4067GiB/s) | 20260508 03:29:00.256 | INFO | Downloaded 38GiB to GPU in 2.71318s (14.0057GiB/s) | 20260508 03:29:05.393 | INFO | Analyzing chunk frames = [4882944,4883456) | 20260508 03:29:05.393 | INFO | MemInfo: { | 20260508 03:29:05.393 | INFO | "gpu" : " 17 GB", | 20260508 03:29:05.393 | INFO | "huge" : "", | 20260508 03:29:05.393 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:05.393 | INFO | "malloc" : " 14 GB", | 20260508 03:29:05.393 | INFO | "pinned" : " 29 GB" | 20260508 03:29:05.393 | INFO | } | 20260508 03:29:05.393 | INFO | Perf report for the last 5.13823 seconds: | 20260508 03:29:05.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:05.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:29:05.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:29:05.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:05.393 | INFO | Uploaded 36GiB to GPU in 2.55039s (14.1155GiB/s) | 20260508 03:29:05.393 | INFO | Downloaded 34GiB to GPU in 2.43499s (13.9631GiB/s) | 20260508 03:29:10.533 | INFO | Analyzing chunk frames = [4883456,4883968) | 20260508 03:29:10.533 | INFO | MemInfo: { | 20260508 03:29:10.533 | INFO | "gpu" : " 17 GB", | 20260508 03:29:10.533 | INFO | "huge" : "", | 20260508 03:29:10.533 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:10.533 | INFO | "malloc" : " 14 GB", | 20260508 03:29:10.533 | INFO | "pinned" : " 29 GB" | 20260508 03:29:10.533 | INFO | } | 20260508 03:29:10.533 | INFO | Perf report for the last 5.13971 seconds: | 20260508 03:29:10.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:10.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:29:10.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:29:10.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:10.533 | INFO | Uploaded 36GiB to GPU in 2.52624s (14.2504GiB/s) | 20260508 03:29:10.533 | INFO | Downloaded 34GiB to GPU in 2.5198s (13.4931GiB/s) | 20260508 03:29:15.668 | INFO | Analyzing chunk frames = [4883968,4884480) | 20260508 03:29:15.668 | INFO | MemInfo: { | 20260508 03:29:15.668 | INFO | "gpu" : " 17 GB", | 20260508 03:29:15.668 | INFO | "huge" : "", | 20260508 03:29:15.668 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:15.668 | INFO | "malloc" : " 14 GB", | 20260508 03:29:15.668 | INFO | "pinned" : " 29 GB" | 20260508 03:29:15.668 | INFO | } | 20260508 03:29:15.668 | INFO | Perf report for the last 5.13561 seconds: | 20260508 03:29:15.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:15.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.668 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 03:29:15.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.669 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:29:15.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:15.669 | INFO | Uploaded 36GiB to GPU in 2.49783s (14.4125GiB/s) | 20260508 03:29:15.669 | INFO | Downloaded 34GiB to GPU in 2.52123s (13.4855GiB/s) | 20260508 03:29:20.814 | INFO | Analyzing chunk frames = [4884480,4884992) | 20260508 03:29:20.814 | INFO | MemInfo: { | 20260508 03:29:20.814 | INFO | "gpu" : " 17 GB", | 20260508 03:29:20.814 | INFO | "huge" : "", | 20260508 03:29:20.814 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:20.814 | INFO | "malloc" : " 14 GB", | 20260508 03:29:20.814 | INFO | "pinned" : " 29 GB" | 20260508 03:29:20.814 | INFO | } | 20260508 03:29:20.814 | INFO | Perf report for the last 5.1458 seconds: | 20260508 03:29:20.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:20.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.814 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 03:29:20.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.814 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:29:20.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:20.815 | INFO | Uploaded 36GiB to GPU in 2.35186s (15.3071GiB/s) | 20260508 03:29:20.815 | INFO | Downloaded 34GiB to GPU in 2.51588s (13.5141GiB/s) | 20260508 03:29:25.952 | INFO | Analyzing chunk frames = [4884992,4885504) | 20260508 03:29:25.952 | INFO | MemInfo: { | 20260508 03:29:25.952 | INFO | "gpu" : " 17 GB", | 20260508 03:29:25.952 | INFO | "huge" : "", | 20260508 03:29:25.952 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:25.953 | INFO | "malloc" : " 14 GB", | 20260508 03:29:25.953 | INFO | "pinned" : " 29 GB" | 20260508 03:29:25.953 | INFO | } | 20260508 03:29:25.953 | INFO | Perf report for the last 5.13853 seconds: | 20260508 03:29:25.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:25.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:29:25.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:29:25.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:25.953 | INFO | Uploaded 36GiB to GPU in 2.39761s (15.015GiB/s) | 20260508 03:29:25.953 | INFO | Downloaded 34GiB to GPU in 2.43609s (13.9568GiB/s) | 20260508 03:29:31.090 | INFO | Analyzing chunk frames = [4885504,4886016) | 20260508 03:29:31.090 | INFO | MemInfo: { | 20260508 03:29:31.090 | INFO | "gpu" : " 17 GB", | 20260508 03:29:31.090 | INFO | "huge" : "", | 20260508 03:29:31.090 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:31.090 | INFO | "malloc" : " 14 GB", | 20260508 03:29:31.090 | INFO | "pinned" : " 29 GB" | 20260508 03:29:31.090 | INFO | } | 20260508 03:29:31.090 | INFO | Perf report for the last 5.1376 seconds: | 20260508 03:29:31.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:31.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.090 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:29:31.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.091 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:29:31.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:31.091 | INFO | Uploaded 36GiB to GPU in 2.42815s (14.8261GiB/s) | 20260508 03:29:31.091 | INFO | Downloaded 34GiB to GPU in 2.45325s (13.8592GiB/s) | 20260508 03:29:36.227 | INFO | Analyzing chunk frames = [4886016,4886528) | 20260508 03:29:36.227 | INFO | MemInfo: { | 20260508 03:29:36.227 | INFO | "gpu" : " 17 GB", | 20260508 03:29:36.227 | INFO | "huge" : "", | 20260508 03:29:36.227 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:36.227 | INFO | "malloc" : " 14 GB", | 20260508 03:29:36.227 | INFO | "pinned" : " 29 GB" | 20260508 03:29:36.227 | INFO | } | 20260508 03:29:36.227 | INFO | Perf report for the last 5.13704 seconds: | 20260508 03:29:36.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:36.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.228 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 03:29:36.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.228 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:29:36.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:36.228 | INFO | Uploaded 36GiB to GPU in 2.41151s (14.9284GiB/s) | 20260508 03:29:36.228 | INFO | Downloaded 34GiB to GPU in 2.44318s (13.9163GiB/s) | 20260508 03:29:36.325 | INFO | Starting a round of Baz IO | 20260508 03:29:39.748 | INFO | Populating RTMetrics report | 20260508 03:29:39.749 | INFO | RTMetrics report populated | 20260508 03:29:39.749 | INFO | Opening tmp RTMetrics json file | 20260508 03:29:39.758 | INFO | Serializing RTMetrics to json | 20260508 03:29:39.762 | INFO | Serialization done, swapping tmp file now... | 20260508 03:29:39.763 | INFO | RTMetrics done | 20260508 03:29:39.825 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.5 seconds | 20260508 03:29:39.825 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.27% / 0.00% / 58.72% | 20260508 03:29:39.825 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.617603% / 3.97364e-06% / 5.96046e-05% | 20260508 03:29:39.825 | INFO | Baz IO overall: | 20260508 03:29:39.825 | INFO | BytesWritten : 2196906537852 | 20260508 03:29:39.825 | INFO | NumEvents : 1061338093376 | 20260508 03:29:41.367 | INFO | Analyzing chunk frames = [4886528,4887040) | 20260508 03:29:41.367 | INFO | MemInfo: { | 20260508 03:29:41.367 | INFO | "gpu" : " 17 GB", | 20260508 03:29:41.367 | INFO | "huge" : "", | 20260508 03:29:41.367 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:41.367 | INFO | "malloc" : " 14 GB", | 20260508 03:29:41.367 | INFO | "pinned" : " 29 GB" | 20260508 03:29:41.367 | INFO | } | 20260508 03:29:41.367 | INFO | Perf report for the last 5.13948 seconds: | 20260508 03:29:41.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:41.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:41.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:41.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:41.367 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 03:29:41.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:41.367 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:29:41.367 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 03:29:41.367 | INFO | BAZWRITER : 67.6%, 2.7, 92.3ms, 0 | 20260508 03:29:41.367 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:29:41.367 | INFO | Uploaded 36GiB to GPU in 2.55041s (14.1154GiB/s) | 20260508 03:29:41.368 | INFO | Downloaded 38GiB to GPU in 2.69707s (14.0894GiB/s) | 20260508 03:29:46.509 | INFO | Analyzing chunk frames = [4887040,4887552) | 20260508 03:29:46.509 | INFO | MemInfo: { | 20260508 03:29:46.509 | INFO | "gpu" : " 17 GB", | 20260508 03:29:46.509 | INFO | "huge" : "", | 20260508 03:29:46.509 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:46.509 | INFO | "malloc" : " 14 GB", | 20260508 03:29:46.509 | INFO | "pinned" : " 29 GB" | 20260508 03:29:46.509 | INFO | } | 20260508 03:29:46.510 | INFO | Perf report for the last 5.14256 seconds: | 20260508 03:29:46.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:46.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 03:29:46.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:29:46.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:46.510 | INFO | Uploaded 36GiB to GPU in 2.53828s (14.1828GiB/s) | 20260508 03:29:46.510 | INFO | Downloaded 34GiB to GPU in 2.45022s (13.8763GiB/s) | 20260508 03:29:51.650 | INFO | Analyzing chunk frames = [4887552,4888064) | 20260508 03:29:51.650 | INFO | MemInfo: { | 20260508 03:29:51.650 | INFO | "gpu" : " 17 GB", | 20260508 03:29:51.650 | INFO | "huge" : "", | 20260508 03:29:51.650 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:51.650 | INFO | "malloc" : " 14 GB", | 20260508 03:29:51.650 | INFO | "pinned" : " 29 GB" | 20260508 03:29:51.650 | INFO | } | 20260508 03:29:51.650 | INFO | Perf report for the last 5.14022 seconds: | 20260508 03:29:51.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:51.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 03:29:51.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:29:51.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:51.650 | INFO | Uploaded 36GiB to GPU in 2.50126s (14.3928GiB/s) | 20260508 03:29:51.650 | INFO | Downloaded 34GiB to GPU in 2.44374s (13.9131GiB/s) | 20260508 03:29:56.788 | INFO | Analyzing chunk frames = [4888064,4888576) | 20260508 03:29:56.788 | INFO | MemInfo: { | 20260508 03:29:56.788 | INFO | "gpu" : " 17 GB", | 20260508 03:29:56.788 | INFO | "huge" : "", | 20260508 03:29:56.788 | INFO | "huge_shared" : " 29 GB", | 20260508 03:29:56.788 | INFO | "malloc" : " 14 GB", | 20260508 03:29:56.789 | INFO | "pinned" : " 29 GB" | 20260508 03:29:56.789 | INFO | } | 20260508 03:29:56.789 | INFO | Perf report for the last 5.13879 seconds: | 20260508 03:29:56.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:29:56.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 03:29:56.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:29:56.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:29:56.789 | INFO | Uploaded 36GiB to GPU in 2.49913s (14.405GiB/s) | 20260508 03:29:56.789 | INFO | Downloaded 34GiB to GPU in 2.51801s (13.5027GiB/s) | 20260508 03:30:01.921 | INFO | Analyzing chunk frames = [4888576,4889088) | 20260508 03:30:01.921 | INFO | MemInfo: { | 20260508 03:30:01.921 | INFO | "gpu" : " 17 GB", | 20260508 03:30:01.921 | INFO | "huge" : "", | 20260508 03:30:01.921 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:01.921 | INFO | "malloc" : " 14 GB", | 20260508 03:30:01.922 | INFO | "pinned" : " 29 GB" | 20260508 03:30:01.922 | INFO | } | 20260508 03:30:01.922 | INFO | Perf report for the last 5.13311 seconds: | 20260508 03:30:01.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:01.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:30:01.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:30:01.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:01.922 | INFO | Uploaded 36GiB to GPU in 2.38082s (15.1208GiB/s) | 20260508 03:30:01.922 | INFO | Downloaded 34GiB to GPU in 2.5754s (13.2018GiB/s) | 20260508 03:30:07.064 | INFO | Analyzing chunk frames = [4889088,4889600) | 20260508 03:30:07.064 | INFO | MemInfo: { | 20260508 03:30:07.064 | INFO | "gpu" : " 17 GB", | 20260508 03:30:07.064 | INFO | "huge" : "", | 20260508 03:30:07.064 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:07.064 | INFO | "malloc" : " 14 GB", | 20260508 03:30:07.064 | INFO | "pinned" : " 29 GB" | 20260508 03:30:07.064 | INFO | } | 20260508 03:30:07.064 | INFO | Perf report for the last 5.14252 seconds: | 20260508 03:30:07.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:07.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.064 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 03:30:07.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.064 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260508 03:30:07.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:07.065 | INFO | Uploaded 36GiB to GPU in 2.40866s (14.9461GiB/s) | 20260508 03:30:07.065 | INFO | Downloaded 34GiB to GPU in 2.43089s (13.9866GiB/s) | 20260508 03:30:12.206 | INFO | Analyzing chunk frames = [4889600,4890112) | 20260508 03:30:12.206 | INFO | MemInfo: { | 20260508 03:30:12.206 | INFO | "gpu" : " 17 GB", | 20260508 03:30:12.206 | INFO | "huge" : "", | 20260508 03:30:12.206 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:12.206 | INFO | "malloc" : " 14 GB", | 20260508 03:30:12.206 | INFO | "pinned" : " 29 GB" | 20260508 03:30:12.206 | INFO | } | 20260508 03:30:12.206 | INFO | Perf report for the last 5.14203 seconds: | 20260508 03:30:12.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:12.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.206 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 03:30:12.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.206 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:30:12.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:12.207 | INFO | Uploaded 36GiB to GPU in 2.43109s (14.8081GiB/s) | 20260508 03:30:12.207 | INFO | Downloaded 34GiB to GPU in 2.43439s (13.9665GiB/s) | 20260508 03:30:17.346 | INFO | Analyzing chunk frames = [4890112,4890624) | 20260508 03:30:17.346 | INFO | MemInfo: { | 20260508 03:30:17.346 | INFO | "gpu" : " 17 GB", | 20260508 03:30:17.346 | INFO | "huge" : "", | 20260508 03:30:17.346 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:17.346 | INFO | "malloc" : " 14 GB", | 20260508 03:30:17.346 | INFO | "pinned" : " 29 GB" | 20260508 03:30:17.346 | INFO | } | 20260508 03:30:17.346 | INFO | Perf report for the last 5.1395 seconds: | 20260508 03:30:17.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:17.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:30:17.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:30:17.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:17.346 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:30:17.346 | INFO | Total mem usage: 16.7 GiB | 20260508 03:30:17.346 | INFO | Uploaded 36GiB to GPU in 2.44595s (14.7182GiB/s) | 20260508 03:30:17.346 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:30:17.346 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:30:17.346 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:30:17.346 | INFO | Downloaded 34GiB to GPU in 2.43221s (13.979GiB/s) | 20260508 03:30:17.346 | INFO | High Water Breakdown: | 20260508 03:30:17.346 | INFO | 59.6% : 8.56 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:30:17.346 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:30:17.346 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:30:17.346 | INFO | ------------------------------------------ | 20260508 03:30:17.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:30:17.347 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:30:17.347 | INFO | Total mem usage: 26.5 GiB | 20260508 03:30:17.347 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:30:17.347 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:30:17.347 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:30:17.347 | INFO | High Water Breakdown: | 20260508 03:30:17.347 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:30:17.347 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:30:17.347 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:30:17.347 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:30:17.347 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:30:17.347 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:30:17.347 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:30:17.347 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:30:17.347 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:30:17.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:30:17.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:30:17.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:30:17.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:30:17.347 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:30:17.347 | INFO | ------------------------------------------ | 20260508 03:30:17.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:30:17.347 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:30:17.347 | INFO | Total mem usage: 27.8 GiB | 20260508 03:30:17.347 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:30:17.347 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:30:17.347 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:30:17.347 | INFO | High Water Breakdown: | 20260508 03:30:17.347 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:30:17.347 | INFO | ------------------------------------------ | 20260508 03:30:17.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:30:17.347 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:30:17.347 | INFO | Total mem usage: 15.7 GiB | 20260508 03:30:17.347 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:30:17.347 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:30:17.347 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:30:17.347 | INFO | High Water Breakdown: | 20260508 03:30:17.347 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:30:17.347 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:30:17.347 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:30:17.347 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:30:17.347 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:30:17.347 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:30:17.347 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:30:17.347 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:30:17.347 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:30:17.347 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:30:17.347 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:30:17.347 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:30:17.347 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:30:17.347 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:30:17.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:30:17.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:30:17.347 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:30:17.347 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:30:17.347 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:30:17.347 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:30:17.347 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:30:17.347 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:30:17.347 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:30:17.347 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:30:17.347 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:30:17.347 | INFO | ------------------------------------------ | 20260508 03:30:17.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:30:17.347 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:30:17.347 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:30:17.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:30:17.347 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.42e+01 / 1.37e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.02e-01, 1.12e-02 / 4.95e-03 | 20260508 03:30:17.347 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.43e+00, 5.81e-01 / 2.48e-01 | 20260508 03:30:17.347 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.07e-01 / 1.14e-01 | 20260508 03:30:17.347 | INFO | PerfReport: 16.02%, DME -- 4.20e+01 / 4.58e+01, 3.94e+00 / 1.97e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.74e+00 / 1.58e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.40e+00 / 2.68e+00, 4.20e-01 / 2.22e-01 | 20260508 03:30:17.347 | INFO | PerfReport: 3.25%, Metrics -- 1.07e+00 / 1.26e+00, 2.70e-01 / 1.60e-01 | 20260508 03:30:17.347 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.63e-01, 5.25e-02 / 2.22e-02 | 20260508 03:30:17.347 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:30:17.347 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:30:17.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:30:17.347 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 24.15%, Upload -- 2.57e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 24.52%, Download -- 2.61e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 1.90%, LockUpload -- 2.01e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 2.96%, LockDownload -- 3.14e+00 / 0.00e+00, 4.46e+00 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:30:17.347 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260508 03:30:17.443 | INFO | Starting a round of Baz IO | 20260508 03:30:20.877 | INFO | Populating RTMetrics report | 20260508 03:30:20.877 | INFO | RTMetrics report populated | 20260508 03:30:20.877 | INFO | Opening tmp RTMetrics json file | 20260508 03:30:20.890 | INFO | Serializing RTMetrics to json | 20260508 03:30:20.892 | INFO | Serialization done, swapping tmp file now... | 20260508 03:30:20.892 | INFO | RTMetrics done | 20260508 03:30:20.950 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.506 seconds | 20260508 03:30:20.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.27% / 0.00% / 58.73% | 20260508 03:30:20.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.619292% / 3.97364e-06% / 6.35783e-05% | 20260508 03:30:20.950 | INFO | Baz IO overall: | 20260508 03:30:20.950 | INFO | BytesWritten : 2198642543691 | 20260508 03:30:20.950 | INFO | NumEvents : 1062169147883 | 20260508 03:30:22.482 | INFO | Analyzing chunk frames = [4890624,4891136) | 20260508 03:30:22.482 | INFO | MemInfo: { | 20260508 03:30:22.482 | INFO | "gpu" : " 17 GB", | 20260508 03:30:22.482 | INFO | "huge" : "", | 20260508 03:30:22.482 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:22.482 | INFO | "malloc" : " 14 GB", | 20260508 03:30:22.482 | INFO | "pinned" : " 29 GB" | 20260508 03:30:22.482 | INFO | } | 20260508 03:30:22.482 | INFO | Perf report for the last 5.13631 seconds: | 20260508 03:30:22.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:22.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:22.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:22.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:22.482 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 03:30:22.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:22.482 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:30:22.482 | INFO | PRE_HQ : 65.9%, 1.6, 55.9ms, 0 | 20260508 03:30:22.482 | INFO | BAZWRITER : 67.6%, 2.6, 91.1ms, 0 | 20260508 03:30:22.482 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:30:22.482 | INFO | Uploaded 36GiB to GPU in 2.55251s (14.1038GiB/s) | 20260508 03:30:22.482 | INFO | Downloaded 38GiB to GPU in 2.68868s (14.1333GiB/s) | 20260508 03:30:27.621 | INFO | Analyzing chunk frames = [4891136,4891648) | 20260508 03:30:27.621 | INFO | MemInfo: { | 20260508 03:30:27.621 | INFO | "gpu" : " 17 GB", | 20260508 03:30:27.621 | INFO | "huge" : "", | 20260508 03:30:27.621 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:27.621 | INFO | "malloc" : " 14 GB", | 20260508 03:30:27.621 | INFO | "pinned" : " 29 GB" | 20260508 03:30:27.621 | INFO | } | 20260508 03:30:27.621 | INFO | Perf report for the last 5.13907 seconds: | 20260508 03:30:27.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:27.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 03:30:27.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:30:27.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:27.621 | INFO | Uploaded 36GiB to GPU in 2.5928s (13.8846GiB/s) | 20260508 03:30:27.621 | INFO | Downloaded 34GiB to GPU in 2.43036s (13.9897GiB/s) | 20260508 03:30:32.764 | INFO | Analyzing chunk frames = [4891648,4892160) | 20260508 03:30:32.764 | INFO | MemInfo: { | 20260508 03:30:32.764 | INFO | "gpu" : " 17 GB", | 20260508 03:30:32.764 | INFO | "huge" : "", | 20260508 03:30:32.764 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:32.764 | INFO | "malloc" : " 14 GB", | 20260508 03:30:32.764 | INFO | "pinned" : " 29 GB" | 20260508 03:30:32.764 | INFO | } | 20260508 03:30:32.764 | INFO | Perf report for the last 5.14325 seconds: | 20260508 03:30:32.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:32.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.764 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:30:32.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.765 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:30:32.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:32.765 | INFO | Uploaded 36GiB to GPU in 2.55844s (14.0711GiB/s) | 20260508 03:30:32.765 | INFO | Downloaded 34GiB to GPU in 2.49337s (13.6362GiB/s) | 20260508 03:30:37.904 | INFO | Analyzing chunk frames = [4892160,4892672) | 20260508 03:30:37.904 | INFO | MemInfo: { | 20260508 03:30:37.904 | INFO | "gpu" : " 17 GB", | 20260508 03:30:37.904 | INFO | "huge" : "", | 20260508 03:30:37.904 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:37.904 | INFO | "malloc" : " 14 GB", | 20260508 03:30:37.904 | INFO | "pinned" : " 29 GB" | 20260508 03:30:37.904 | INFO | } | 20260508 03:30:37.904 | INFO | Perf report for the last 5.13932 seconds: | 20260508 03:30:37.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:37.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:30:37.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:30:37.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:37.904 | INFO | Uploaded 36GiB to GPU in 2.57423s (13.9847GiB/s) | 20260508 03:30:37.905 | INFO | Downloaded 34GiB to GPU in 2.49315s (13.6374GiB/s) | 20260508 03:30:43.042 | INFO | Analyzing chunk frames = [4892672,4893184) | 20260508 03:30:43.042 | INFO | MemInfo: { | 20260508 03:30:43.042 | INFO | "gpu" : " 17 GB", | 20260508 03:30:43.043 | INFO | "huge" : "", | 20260508 03:30:43.043 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:43.043 | INFO | "malloc" : " 14 GB", | 20260508 03:30:43.043 | INFO | "pinned" : " 29 GB" | 20260508 03:30:43.043 | INFO | } | 20260508 03:30:43.043 | INFO | Perf report for the last 5.13896 seconds: | 20260508 03:30:43.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:43.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 03:30:43.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260508 03:30:43.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:43.043 | INFO | Uploaded 36GiB to GPU in 2.39623s (15.0236GiB/s) | 20260508 03:30:43.043 | INFO | Downloaded 34GiB to GPU in 2.5759s (13.1993GiB/s) | 20260508 03:30:48.179 | INFO | Analyzing chunk frames = [4893184,4893696) | 20260508 03:30:48.179 | INFO | MemInfo: { | 20260508 03:30:48.179 | INFO | "gpu" : " 17 GB", | 20260508 03:30:48.179 | INFO | "huge" : "", | 20260508 03:30:48.179 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:48.179 | INFO | "malloc" : " 14 GB", | 20260508 03:30:48.179 | INFO | "pinned" : " 29 GB" | 20260508 03:30:48.179 | INFO | } | 20260508 03:30:48.179 | INFO | Perf report for the last 5.13596 seconds: | 20260508 03:30:48.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:48.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:30:48.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:30:48.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:48.179 | INFO | Uploaded 36GiB to GPU in 2.4231s (14.857GiB/s) | 20260508 03:30:48.179 | INFO | Downloaded 34GiB to GPU in 2.41842s (14.0588GiB/s) | 20260508 03:30:53.321 | INFO | Analyzing chunk frames = [4893696,4894208) | 20260508 03:30:53.321 | INFO | MemInfo: { | 20260508 03:30:53.321 | INFO | "gpu" : " 17 GB", | 20260508 03:30:53.321 | INFO | "huge" : "", | 20260508 03:30:53.321 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:53.321 | INFO | "malloc" : " 14 GB", | 20260508 03:30:53.321 | INFO | "pinned" : " 29 GB" | 20260508 03:30:53.321 | INFO | } | 20260508 03:30:53.321 | INFO | Perf report for the last 5.14225 seconds: | 20260508 03:30:53.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:53.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:30:53.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:30:53.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:53.321 | INFO | Uploaded 36GiB to GPU in 2.42292s (14.8581GiB/s) | 20260508 03:30:53.321 | INFO | Downloaded 34GiB to GPU in 2.41552s (14.0756GiB/s) | 20260508 03:30:58.463 | INFO | Analyzing chunk frames = [4894208,4894720) | 20260508 03:30:58.463 | INFO | MemInfo: { | 20260508 03:30:58.463 | INFO | "gpu" : " 17 GB", | 20260508 03:30:58.463 | INFO | "huge" : "", | 20260508 03:30:58.463 | INFO | "huge_shared" : " 29 GB", | 20260508 03:30:58.463 | INFO | "malloc" : " 14 GB", | 20260508 03:30:58.463 | INFO | "pinned" : " 29 GB" | 20260508 03:30:58.463 | INFO | } | 20260508 03:30:58.463 | INFO | Perf report for the last 5.14223 seconds: | 20260508 03:30:58.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:30:58.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.463 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:30:58.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.463 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:30:58.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:30:58.464 | INFO | Uploaded 36GiB to GPU in 2.45092s (14.6884GiB/s) | 20260508 03:30:58.464 | INFO | Downloaded 34GiB to GPU in 2.42495s (14.0209GiB/s) | 20260508 03:30:58.560 | INFO | Starting a round of Baz IO | 20260508 03:31:01.983 | INFO | Populating RTMetrics report | 20260508 03:31:01.984 | INFO | RTMetrics report populated | 20260508 03:31:01.984 | INFO | Opening tmp RTMetrics json file | 20260508 03:31:01.990 | INFO | Serializing RTMetrics to json | 20260508 03:31:01.998 | INFO | Serialization done, swapping tmp file now... | 20260508 03:31:01.998 | INFO | RTMetrics done | 20260508 03:31:02.061 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.501 seconds | 20260508 03:31:02.061 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.27% / 0.00% / 58.73% | 20260508 03:31:02.061 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.616658% / 1.58946e-05% / 4.37101e-05% | 20260508 03:31:02.061 | INFO | Baz IO overall: | 20260508 03:31:02.062 | INFO | BytesWritten : 2200377998945 | 20260508 03:31:02.062 | INFO | NumEvents : 1062999841970 | 20260508 03:31:03.598 | INFO | Analyzing chunk frames = [4894720,4895232) | 20260508 03:31:03.598 | INFO | MemInfo: { | 20260508 03:31:03.598 | INFO | "gpu" : " 17 GB", | 20260508 03:31:03.598 | INFO | "huge" : "", | 20260508 03:31:03.599 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:03.599 | INFO | "malloc" : " 14 GB", | 20260508 03:31:03.599 | INFO | "pinned" : " 29 GB" | 20260508 03:31:03.599 | INFO | } | 20260508 03:31:03.599 | INFO | Perf report for the last 5.13547 seconds: | 20260508 03:31:03.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:03.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:03.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:03.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:03.599 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 03:31:03.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:03.599 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:31:03.599 | INFO | PRE_HQ : 65.8%, 1.6, 56.1ms, 0 | 20260508 03:31:03.599 | INFO | BAZWRITER : 67.7%, 2.6, 90.6ms, 0 | 20260508 03:31:03.599 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:31:03.599 | INFO | Uploaded 36GiB to GPU in 2.53053s (14.2263GiB/s) | 20260508 03:31:03.599 | INFO | Downloaded 38GiB to GPU in 2.70242s (14.0614GiB/s) | 20260508 03:31:08.737 | INFO | Analyzing chunk frames = [4895232,4895744) | 20260508 03:31:08.737 | INFO | MemInfo: { | 20260508 03:31:08.737 | INFO | "gpu" : " 17 GB", | 20260508 03:31:08.737 | INFO | "huge" : "", | 20260508 03:31:08.737 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:08.737 | INFO | "malloc" : " 14 GB", | 20260508 03:31:08.737 | INFO | "pinned" : " 29 GB" | 20260508 03:31:08.737 | INFO | } | 20260508 03:31:08.737 | INFO | Perf report for the last 5.13834 seconds: | 20260508 03:31:08.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:08.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:31:08.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:31:08.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:08.737 | INFO | Uploaded 36GiB to GPU in 2.57092s (14.0028GiB/s) | 20260508 03:31:08.738 | INFO | Downloaded 34GiB to GPU in 2.42454s (14.0233GiB/s) | 20260508 03:31:13.875 | INFO | Analyzing chunk frames = [4895744,4896256) | 20260508 03:31:13.875 | INFO | MemInfo: { | 20260508 03:31:13.875 | INFO | "gpu" : " 17 GB", | 20260508 03:31:13.875 | INFO | "huge" : "", | 20260508 03:31:13.875 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:13.875 | INFO | "malloc" : " 14 GB", | 20260508 03:31:13.875 | INFO | "pinned" : " 29 GB" | 20260508 03:31:13.875 | INFO | } | 20260508 03:31:13.875 | INFO | Perf report for the last 5.13789 seconds: | 20260508 03:31:13.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:13.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 03:31:13.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 03:31:13.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:13.875 | INFO | Uploaded 36GiB to GPU in 2.56215s (14.0507GiB/s) | 20260508 03:31:13.875 | INFO | Downloaded 34GiB to GPU in 2.50607s (13.567GiB/s) | 20260508 03:31:19.018 | INFO | Analyzing chunk frames = [4896256,4896768) | 20260508 03:31:19.018 | INFO | MemInfo: { | 20260508 03:31:19.018 | INFO | "gpu" : " 17 GB", | 20260508 03:31:19.018 | INFO | "huge" : "", | 20260508 03:31:19.018 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:19.018 | INFO | "malloc" : " 14 GB", | 20260508 03:31:19.018 | INFO | "pinned" : " 29 GB" | 20260508 03:31:19.018 | INFO | } | 20260508 03:31:19.018 | INFO | Perf report for the last 5.14334 seconds: | 20260508 03:31:19.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:19.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.018 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 03:31:19.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.018 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:31:19.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:19.019 | INFO | Uploaded 36GiB to GPU in 2.50284s (14.3837GiB/s) | 20260508 03:31:19.019 | INFO | Downloaded 34GiB to GPU in 2.49772s (13.6124GiB/s) | 20260508 03:31:24.155 | INFO | Analyzing chunk frames = [4896768,4897280) | 20260508 03:31:24.155 | INFO | MemInfo: { | 20260508 03:31:24.155 | INFO | "gpu" : " 17 GB", | 20260508 03:31:24.155 | INFO | "huge" : "", | 20260508 03:31:24.155 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:24.156 | INFO | "malloc" : " 14 GB", | 20260508 03:31:24.156 | INFO | "pinned" : " 29 GB" | 20260508 03:31:24.156 | INFO | } | 20260508 03:31:24.156 | INFO | Perf report for the last 5.13727 seconds: | 20260508 03:31:24.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:24.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:31:24.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:31:24.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:24.156 | INFO | Uploaded 36GiB to GPU in 2.40506s (14.9684GiB/s) | 20260508 03:31:24.156 | INFO | Downloaded 34GiB to GPU in 2.56524s (13.2541GiB/s) | 20260508 03:31:29.294 | INFO | Analyzing chunk frames = [4897280,4897792) | 20260508 03:31:29.294 | INFO | MemInfo: { | 20260508 03:31:29.294 | INFO | "gpu" : " 17 GB", | 20260508 03:31:29.294 | INFO | "huge" : "", | 20260508 03:31:29.294 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:29.294 | INFO | "malloc" : " 14 GB", | 20260508 03:31:29.294 | INFO | "pinned" : " 29 GB" | 20260508 03:31:29.294 | INFO | } | 20260508 03:31:29.294 | INFO | Perf report for the last 5.13859 seconds: | 20260508 03:31:29.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:29.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:31:29.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:31:29.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:29.294 | INFO | Uploaded 36GiB to GPU in 2.43921s (14.7589GiB/s) | 20260508 03:31:29.295 | INFO | Downloaded 34GiB to GPU in 2.4139s (14.0851GiB/s) | 20260508 03:31:34.435 | INFO | Analyzing chunk frames = [4897792,4898304) | 20260508 03:31:34.435 | INFO | MemInfo: { | 20260508 03:31:34.435 | INFO | "gpu" : " 17 GB", | 20260508 03:31:34.435 | INFO | "huge" : "", | 20260508 03:31:34.435 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:34.435 | INFO | "malloc" : " 14 GB", | 20260508 03:31:34.436 | INFO | "pinned" : " 29 GB" | 20260508 03:31:34.436 | INFO | } | 20260508 03:31:34.436 | INFO | Perf report for the last 5.14137 seconds: | 20260508 03:31:34.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:34.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:31:34.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:31:34.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:34.436 | INFO | Uploaded 36GiB to GPU in 2.45941s (14.6377GiB/s) | 20260508 03:31:34.436 | INFO | Downloaded 34GiB to GPU in 2.4152s (14.0775GiB/s) | 20260508 03:31:39.575 | INFO | Analyzing chunk frames = [4898304,4898816) | 20260508 03:31:39.575 | INFO | MemInfo: { | 20260508 03:31:39.575 | INFO | "gpu" : " 17 GB", | 20260508 03:31:39.575 | INFO | "huge" : "", | 20260508 03:31:39.575 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:39.575 | INFO | "malloc" : " 14 GB", | 20260508 03:31:39.575 | INFO | "pinned" : " 29 GB" | 20260508 03:31:39.575 | INFO | } | 20260508 03:31:39.575 | INFO | Perf report for the last 5.13984 seconds: | 20260508 03:31:39.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:39.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.575 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:31:39.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.576 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 03:31:39.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:39.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:31:39.576 | INFO | Total mem usage: 16.7 GiB | 20260508 03:31:39.576 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:31:39.576 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 03:31:39.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:31:39.576 | INFO | High Water Breakdown: | 20260508 03:31:39.576 | INFO | 59.4% : 8.47 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:31:39.576 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:31:39.576 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:31:39.576 | INFO | ------------------------------------------ | 20260508 03:31:39.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:31:39.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:31:39.576 | INFO | Total mem usage: 26.5 GiB | 20260508 03:31:39.576 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:31:39.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 03:31:39.576 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 03:31:39.576 | INFO | High Water Breakdown: | 20260508 03:31:39.576 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:31:39.576 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:31:39.576 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:31:39.576 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:31:39.576 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:31:39.576 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:31:39.576 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:31:39.576 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:31:39.576 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:31:39.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:31:39.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:31:39.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:31:39.576 | INFO | Uploaded 36GiB to GPU in 2.57586s (13.9759GiB/s) | 20260508 03:31:39.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:31:39.576 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:31:39.576 | INFO | ------------------------------------------ | 20260508 03:31:39.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:31:39.576 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:31:39.576 | INFO | Total mem usage: 27.8 GiB | 20260508 03:31:39.576 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:31:39.576 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:31:39.576 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:31:39.576 | INFO | Downloaded 34GiB to GPU in 2.45861s (13.8289GiB/s) | 20260508 03:31:39.576 | INFO | High Water Breakdown: | 20260508 03:31:39.576 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:31:39.576 | INFO | ------------------------------------------ | 20260508 03:31:39.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:31:39.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:31:39.577 | INFO | Total mem usage: 15.7 GiB | 20260508 03:31:39.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:31:39.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:31:39.577 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:31:39.577 | INFO | High Water Breakdown: | 20260508 03:31:39.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:31:39.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:31:39.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:31:39.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:31:39.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:31:39.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:31:39.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:31:39.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:31:39.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:31:39.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:31:39.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:31:39.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:31:39.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:31:39.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:31:39.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:31:39.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:31:39.577 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:31:39.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:31:39.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:31:39.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:31:39.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:31:39.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:31:39.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:31:39.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:31:39.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:31:39.577 | INFO | ------------------------------------------ | 20260508 03:31:39.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:31:39.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:31:39.577 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:31:39.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:31:39.577 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.61e+01, 1.41e+01 / 2.12e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.01e-01, 1.17e-02 / 6.11e-03 | 20260508 03:31:39.577 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.47e+00, 5.74e-01 / 1.82e-01 | 20260508 03:31:39.577 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.05e-01 / 1.07e-01 | 20260508 03:31:39.577 | INFO | PerfReport: 16.13%, DME -- 4.21e+01 / 4.63e+01, 4.21e+00 / 2.47e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.84e+01 / 1.95e+01, 1.60e+00 / 1.30e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.41e+00 / 2.72e+00, 4.16e-01 / 1.96e-01 | 20260508 03:31:39.577 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.21e+00, 2.62e-01 / 1.35e-01 | 20260508 03:31:39.577 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.56e-01, 5.09e-02 / 2.21e-02 | 20260508 03:31:39.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:31:39.577 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:31:39.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:31:39.577 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 24.44%, Upload -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 30.60%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 2.28%, LockUpload -- 2.43e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 2.75%, LockDownload -- 2.93e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:31:39.577 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260508 03:31:39.674 | INFO | Starting a round of Baz IO | 20260508 03:31:43.121 | INFO | Populating RTMetrics report | 20260508 03:31:43.121 | INFO | RTMetrics report populated | 20260508 03:31:43.121 | INFO | Opening tmp RTMetrics json file | 20260508 03:31:43.133 | INFO | Serializing RTMetrics to json | 20260508 03:31:43.134 | INFO | Serialization done, swapping tmp file now... | 20260508 03:31:43.135 | INFO | RTMetrics done | 20260508 03:31:43.187 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.513 seconds | 20260508 03:31:43.187 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.27% / 0.00% / 58.73% | 20260508 03:31:43.187 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.61636% / 0% / 7.54992e-05% | 20260508 03:31:43.187 | INFO | Baz IO overall: | 20260508 03:31:43.187 | INFO | BytesWritten : 2202114573563 | 20260508 03:31:43.187 | INFO | NumEvents : 1063831471686 | 20260508 03:31:44.717 | INFO | Analyzing chunk frames = [4898816,4899328) | 20260508 03:31:44.717 | INFO | MemInfo: { | 20260508 03:31:44.717 | INFO | "gpu" : " 17 GB", | 20260508 03:31:44.717 | INFO | "huge" : "", | 20260508 03:31:44.717 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:44.717 | INFO | "malloc" : " 14 GB", | 20260508 03:31:44.717 | INFO | "pinned" : " 29 GB" | 20260508 03:31:44.717 | INFO | } | 20260508 03:31:44.717 | INFO | Perf report for the last 5.14146 seconds: | 20260508 03:31:44.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:44.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:44.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:44.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:44.717 | INFO | ANALYSIS : 68.3%, 3.0, 106.6ms, 0 | 20260508 03:31:44.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:44.717 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:31:44.717 | INFO | PRE_HQ : 65.8%, 1.6, 55.5ms, 0 | 20260508 03:31:44.717 | INFO | BAZWRITER : 67.8%, 2.6, 91.7ms, 0 | 20260508 03:31:44.717 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:31:44.717 | INFO | Uploaded 36GiB to GPU in 2.54922s (14.122GiB/s) | 20260508 03:31:44.717 | INFO | Downloaded 38GiB to GPU in 2.69063s (14.1231GiB/s) | 20260508 03:31:49.853 | INFO | Analyzing chunk frames = [4899328,4899840) | 20260508 03:31:49.853 | INFO | MemInfo: { | 20260508 03:31:49.853 | INFO | "gpu" : " 17 GB", | 20260508 03:31:49.853 | INFO | "huge" : "", | 20260508 03:31:49.853 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:49.853 | INFO | "malloc" : " 14 GB", | 20260508 03:31:49.854 | INFO | "pinned" : " 29 GB" | 20260508 03:31:49.854 | INFO | } | 20260508 03:31:49.854 | INFO | Perf report for the last 5.1367 seconds: | 20260508 03:31:49.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:49.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 03:31:49.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:31:49.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:49.854 | INFO | Uploaded 36GiB to GPU in 2.58333s (13.9355GiB/s) | 20260508 03:31:49.854 | INFO | Downloaded 34GiB to GPU in 2.43223s (13.979GiB/s) | 20260508 03:31:54.986 | INFO | Analyzing chunk frames = [4899840,4900352) | 20260508 03:31:54.986 | INFO | MemInfo: { | 20260508 03:31:54.986 | INFO | "gpu" : " 17 GB", | 20260508 03:31:54.986 | INFO | "huge" : "", | 20260508 03:31:54.986 | INFO | "huge_shared" : " 29 GB", | 20260508 03:31:54.986 | INFO | "malloc" : " 14 GB", | 20260508 03:31:54.986 | INFO | "pinned" : " 29 GB" | 20260508 03:31:54.986 | INFO | } | 20260508 03:31:54.986 | INFO | Perf report for the last 5.13236 seconds: | 20260508 03:31:54.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:31:54.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.986 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:31:54.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.986 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:31:54.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:31:54.987 | INFO | Uploaded 36GiB to GPU in 2.58417s (13.931GiB/s) | 20260508 03:31:54.987 | INFO | Downloaded 34GiB to GPU in 2.42383s (14.0274GiB/s) | 20260508 03:32:00.127 | INFO | Analyzing chunk frames = [4900352,4900864) | 20260508 03:32:00.127 | INFO | MemInfo: { | 20260508 03:32:00.127 | INFO | "gpu" : " 17 GB", | 20260508 03:32:00.127 | INFO | "huge" : "", | 20260508 03:32:00.127 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:00.127 | INFO | "malloc" : " 14 GB", | 20260508 03:32:00.127 | INFO | "pinned" : " 29 GB" | 20260508 03:32:00.127 | INFO | } | 20260508 03:32:00.127 | INFO | Perf report for the last 5.14084 seconds: | 20260508 03:32:00.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:00.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 03:32:00.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:32:00.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:00.127 | INFO | Uploaded 36GiB to GPU in 2.57444s (13.9836GiB/s) | 20260508 03:32:00.127 | INFO | Downloaded 34GiB to GPU in 2.48904s (13.6599GiB/s) | 20260508 03:32:05.269 | INFO | Analyzing chunk frames = [4900864,4901376) | 20260508 03:32:05.269 | INFO | MemInfo: { | 20260508 03:32:05.269 | INFO | "gpu" : " 17 GB", | 20260508 03:32:05.269 | INFO | "huge" : "", | 20260508 03:32:05.269 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:05.270 | INFO | "malloc" : " 14 GB", | 20260508 03:32:05.270 | INFO | "pinned" : " 29 GB" | 20260508 03:32:05.270 | INFO | } | 20260508 03:32:05.270 | INFO | Perf report for the last 5.14284 seconds: | 20260508 03:32:05.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:05.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:32:05.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:32:05.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:05.270 | INFO | Uploaded 36GiB to GPU in 2.46982s (14.576GiB/s) | 20260508 03:32:05.270 | INFO | Downloaded 34GiB to GPU in 2.56774s (13.2412GiB/s) | 20260508 03:32:10.413 | INFO | Analyzing chunk frames = [4901376,4901888) | 20260508 03:32:10.413 | INFO | MemInfo: { | 20260508 03:32:10.413 | INFO | "gpu" : " 17 GB", | 20260508 03:32:10.413 | INFO | "huge" : "", | 20260508 03:32:10.413 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:10.413 | INFO | "malloc" : " 14 GB", | 20260508 03:32:10.413 | INFO | "pinned" : " 29 GB" | 20260508 03:32:10.413 | INFO | } | 20260508 03:32:10.413 | INFO | Perf report for the last 5.14349 seconds: | 20260508 03:32:10.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:10.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 03:32:10.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:32:10.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:10.413 | INFO | Uploaded 36GiB to GPU in 2.43451s (14.7873GiB/s) | 20260508 03:32:10.413 | INFO | Downloaded 34GiB to GPU in 2.40429s (14.1414GiB/s) | 20260508 03:32:15.551 | INFO | Analyzing chunk frames = [4901888,4902400) | 20260508 03:32:15.551 | INFO | MemInfo: { | 20260508 03:32:15.551 | INFO | "gpu" : " 17 GB", | 20260508 03:32:15.551 | INFO | "huge" : "", | 20260508 03:32:15.552 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:15.552 | INFO | "malloc" : " 14 GB", | 20260508 03:32:15.552 | INFO | "pinned" : " 29 GB" | 20260508 03:32:15.552 | INFO | } | 20260508 03:32:15.552 | INFO | Perf report for the last 5.13845 seconds: | 20260508 03:32:15.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:15.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:32:15.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:32:15.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:15.552 | INFO | Uploaded 36GiB to GPU in 2.46372s (14.612GiB/s) | 20260508 03:32:15.552 | INFO | Downloaded 34GiB to GPU in 2.4141s (14.0839GiB/s) | 20260508 03:32:20.684 | INFO | Analyzing chunk frames = [4902400,4902912) | 20260508 03:32:20.684 | INFO | MemInfo: { | 20260508 03:32:20.684 | INFO | "gpu" : " 17 GB", | 20260508 03:32:20.684 | INFO | "huge" : "", | 20260508 03:32:20.684 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:20.684 | INFO | "malloc" : " 14 GB", | 20260508 03:32:20.684 | INFO | "pinned" : " 29 GB" | 20260508 03:32:20.684 | INFO | } | 20260508 03:32:20.684 | INFO | Perf report for the last 5.13205 seconds: | 20260508 03:32:20.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:20.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 03:32:20.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260508 03:32:20.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:20.684 | INFO | Uploaded 36GiB to GPU in 2.46704s (14.5924GiB/s) | 20260508 03:32:20.684 | INFO | Downloaded 34GiB to GPU in 2.41145s (14.0994GiB/s) | 20260508 03:32:20.780 | INFO | Starting a round of Baz IO | 20260508 03:32:24.219 | INFO | Populating RTMetrics report | 20260508 03:32:24.220 | INFO | RTMetrics report populated | 20260508 03:32:24.220 | INFO | Opening tmp RTMetrics json file | 20260508 03:32:24.231 | INFO | Serializing RTMetrics to json | 20260508 03:32:24.233 | INFO | Serialization done, swapping tmp file now... | 20260508 03:32:24.233 | INFO | RTMetrics done | 20260508 03:32:24.288 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.507 seconds | 20260508 03:32:24.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.26% / 0.00% / 58.73% | 20260508 03:32:24.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.614619% / 0% / 6.75519e-05% | 20260508 03:32:24.288 | INFO | Baz IO overall: | 20260508 03:32:24.288 | INFO | BytesWritten : 2203849822077 | 20260508 03:32:24.288 | INFO | NumEvents : 1064662020315 | 20260508 03:32:25.828 | INFO | Analyzing chunk frames = [4902912,4903424) | 20260508 03:32:25.828 | INFO | MemInfo: { | 20260508 03:32:25.828 | INFO | "gpu" : " 17 GB", | 20260508 03:32:25.828 | INFO | "huge" : "", | 20260508 03:32:25.828 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:25.828 | INFO | "malloc" : " 14 GB", | 20260508 03:32:25.828 | INFO | "pinned" : " 29 GB" | 20260508 03:32:25.829 | INFO | } | 20260508 03:32:25.829 | INFO | Perf report for the last 5.14481 seconds: | 20260508 03:32:25.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:25.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:25.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:25.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:25.829 | INFO | ANALYSIS : 67.8%, 3.0, 106.2ms, 0 | 20260508 03:32:25.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:25.829 | INFO | RT_METRICS : 39.7%, 1.0, 20.9ms, 0 | 20260508 03:32:25.829 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 03:32:25.829 | INFO | BAZWRITER : 67.3%, 2.6, 91.4ms, 0 | 20260508 03:32:25.829 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:32:25.829 | INFO | Uploaded 36GiB to GPU in 2.52261s (14.2709GiB/s) | 20260508 03:32:25.829 | INFO | Downloaded 38GiB to GPU in 2.68063s (14.1758GiB/s) | 20260508 03:32:30.966 | INFO | Analyzing chunk frames = [4903424,4903936) | 20260508 03:32:30.966 | INFO | MemInfo: { | 20260508 03:32:30.966 | INFO | "gpu" : " 17 GB", | 20260508 03:32:30.966 | INFO | "huge" : "", | 20260508 03:32:30.966 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:30.966 | INFO | "malloc" : " 14 GB", | 20260508 03:32:30.966 | INFO | "pinned" : " 29 GB" | 20260508 03:32:30.966 | INFO | } | 20260508 03:32:30.966 | INFO | Perf report for the last 5.13731 seconds: | 20260508 03:32:30.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:30.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:32:30.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:32:30.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:30.966 | INFO | Uploaded 36GiB to GPU in 2.56777s (14.0199GiB/s) | 20260508 03:32:30.966 | INFO | Downloaded 34GiB to GPU in 2.42256s (14.0348GiB/s) | 20260508 03:32:36.109 | INFO | Analyzing chunk frames = [4903936,4904448) | 20260508 03:32:36.109 | INFO | MemInfo: { | 20260508 03:32:36.109 | INFO | "gpu" : " 17 GB", | 20260508 03:32:36.109 | INFO | "huge" : "", | 20260508 03:32:36.109 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:36.110 | INFO | "malloc" : " 14 GB", | 20260508 03:32:36.110 | INFO | "pinned" : " 29 GB" | 20260508 03:32:36.110 | INFO | } | 20260508 03:32:36.110 | INFO | Perf report for the last 5.14372 seconds: | 20260508 03:32:36.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:36.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 03:32:36.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:32:36.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:36.110 | INFO | Uploaded 36GiB to GPU in 2.58289s (13.9379GiB/s) | 20260508 03:32:36.110 | INFO | Downloaded 34GiB to GPU in 2.49458s (13.6295GiB/s) | 20260508 03:32:41.244 | INFO | Analyzing chunk frames = [4904448,4904960) | 20260508 03:32:41.244 | INFO | MemInfo: { | 20260508 03:32:41.244 | INFO | "gpu" : " 17 GB", | 20260508 03:32:41.244 | INFO | "huge" : "", | 20260508 03:32:41.244 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:41.244 | INFO | "malloc" : " 14 GB", | 20260508 03:32:41.244 | INFO | "pinned" : " 29 GB" | 20260508 03:32:41.244 | INFO | } | 20260508 03:32:41.244 | INFO | Perf report for the last 5.1346 seconds: | 20260508 03:32:41.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:41.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 03:32:41.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:32:41.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:41.244 | INFO | Uploaded 36GiB to GPU in 2.57609s (13.9747GiB/s) | 20260508 03:32:41.244 | INFO | Downloaded 34GiB to GPU in 2.40591s (14.1319GiB/s) | 20260508 03:32:46.387 | INFO | Analyzing chunk frames = [4904960,4905472) | 20260508 03:32:46.387 | INFO | MemInfo: { | 20260508 03:32:46.387 | INFO | "gpu" : " 17 GB", | 20260508 03:32:46.387 | INFO | "huge" : "", | 20260508 03:32:46.387 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:46.387 | INFO | "malloc" : " 14 GB", | 20260508 03:32:46.387 | INFO | "pinned" : " 29 GB" | 20260508 03:32:46.387 | INFO | } | 20260508 03:32:46.387 | INFO | Perf report for the last 5.14305 seconds: | 20260508 03:32:46.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:46.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.387 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 03:32:46.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.387 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:32:46.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:46.388 | INFO | Uploaded 36GiB to GPU in 2.3995s (15.0031GiB/s) | 20260508 03:32:46.388 | INFO | Downloaded 34GiB to GPU in 2.56444s (13.2582GiB/s) | 20260508 03:32:51.526 | INFO | Analyzing chunk frames = [4905472,4905984) | 20260508 03:32:51.526 | INFO | MemInfo: { | 20260508 03:32:51.526 | INFO | "gpu" : " 17 GB", | 20260508 03:32:51.526 | INFO | "huge" : "", | 20260508 03:32:51.526 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:51.526 | INFO | "malloc" : " 14 GB", | 20260508 03:32:51.526 | INFO | "pinned" : " 29 GB" | 20260508 03:32:51.526 | INFO | } | 20260508 03:32:51.526 | INFO | Perf report for the last 5.13881 seconds: | 20260508 03:32:51.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:51.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:32:51.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:32:51.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:51.526 | INFO | Uploaded 36GiB to GPU in 2.44152s (14.7449GiB/s) | 20260508 03:32:51.526 | INFO | Downloaded 34GiB to GPU in 2.41397s (14.0847GiB/s) | 20260508 03:32:56.662 | INFO | Analyzing chunk frames = [4905984,4906496) | 20260508 03:32:56.662 | INFO | MemInfo: { | 20260508 03:32:56.662 | INFO | "gpu" : " 17 GB", | 20260508 03:32:56.662 | INFO | "huge" : "", | 20260508 03:32:56.662 | INFO | "huge_shared" : " 29 GB", | 20260508 03:32:56.662 | INFO | "malloc" : " 14 GB", | 20260508 03:32:56.662 | INFO | "pinned" : " 29 GB" | 20260508 03:32:56.662 | INFO | } | 20260508 03:32:56.662 | INFO | Perf report for the last 5.13577 seconds: | 20260508 03:32:56.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:32:56.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.662 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:32:56.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.662 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 03:32:56.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:32:56.663 | INFO | Uploaded 36GiB to GPU in 2.45395s (14.6702GiB/s) | 20260508 03:32:56.663 | INFO | Downloaded 34GiB to GPU in 2.41364s (14.0866GiB/s) | 20260508 03:33:01.801 | INFO | Analyzing chunk frames = [4906496,4907008) | 20260508 03:33:01.801 | INFO | MemInfo: { | 20260508 03:33:01.801 | INFO | "gpu" : " 17 GB", | 20260508 03:33:01.801 | INFO | "huge" : "", | 20260508 03:33:01.801 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:01.801 | INFO | "malloc" : " 14 GB", | 20260508 03:33:01.801 | INFO | "pinned" : " 29 GB" | 20260508 03:33:01.801 | INFO | } | 20260508 03:33:01.801 | INFO | Perf report for the last 5.13916 seconds: | 20260508 03:33:01.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:01.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.801 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 03:33:01.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.801 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 03:33:01.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:01.802 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:33:01.802 | INFO | Total mem usage: 16.7 GiB | 20260508 03:33:01.802 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:33:01.802 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 03:33:01.802 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:33:01.802 | INFO | High Water Breakdown: | 20260508 03:33:01.802 | INFO | 59.4% : 8.48 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:33:01.802 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:33:01.802 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:33:01.802 | INFO | ------------------------------------------ | 20260508 03:33:01.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:33:01.802 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:33:01.802 | INFO | Total mem usage: 26.5 GiB | 20260508 03:33:01.802 | INFO | Uploaded 36GiB to GPU in 2.49757s (14.414GiB/s) | 20260508 03:33:01.802 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:33:01.802 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 03:33:01.802 | INFO | Downloaded 34GiB to GPU in 2.4366s (13.9539GiB/s) | 20260508 03:33:01.802 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 03:33:01.802 | INFO | High Water Breakdown: | 20260508 03:33:01.802 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:33:01.802 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:33:01.802 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:33:01.802 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:33:01.802 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:33:01.802 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:33:01.802 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:33:01.802 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:33:01.802 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:33:01.802 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:33:01.802 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:33:01.802 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:33:01.802 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:33:01.802 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:33:01.802 | INFO | ------------------------------------------ | 20260508 03:33:01.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:33:01.802 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:33:01.802 | INFO | Total mem usage: 27.8 GiB | 20260508 03:33:01.802 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:33:01.802 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:33:01.802 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:33:01.802 | INFO | High Water Breakdown: | 20260508 03:33:01.803 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:33:01.803 | INFO | ------------------------------------------ | 20260508 03:33:01.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:33:01.803 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:33:01.803 | INFO | Total mem usage: 15.7 GiB | 20260508 03:33:01.803 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:33:01.803 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:33:01.803 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 03:33:01.803 | INFO | High Water Breakdown: | 20260508 03:33:01.803 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:33:01.803 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:33:01.803 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:33:01.803 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:33:01.803 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:33:01.803 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:33:01.803 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:33:01.803 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:33:01.803 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:33:01.803 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:33:01.803 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:33:01.803 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:33:01.803 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:33:01.803 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:33:01.803 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:33:01.803 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:33:01.803 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:33:01.803 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:33:01.803 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:33:01.803 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:33:01.803 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:33:01.803 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:33:01.803 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:33:01.803 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:33:01.803 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:33:01.803 | INFO | ------------------------------------------ | 20260508 03:33:01.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:33:01.803 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:33:01.803 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:33:01.803 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:33:01.803 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.42e+01 / 1.85e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.18e-02 / 5.49e-03 | 20260508 03:33:01.803 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.42e+00, 5.93e-01 / 2.17e-01 | 20260508 03:33:01.803 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.10e-01 | 20260508 03:33:01.803 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.59e+01, 4.24e+00 / 2.33e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.76e+00 / 1.32e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.66e+00, 4.18e-01 / 2.02e-01 | 20260508 03:33:01.803 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.26e+00, 2.70e-01 / 1.49e-01 | 20260508 03:33:01.803 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.65e-01, 5.21e-02 / 2.57e-02 | 20260508 03:33:01.803 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:33:01.803 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:33:01.803 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:33:01.803 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 24.52%, Upload -- 2.62e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 24.21%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 30.72%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 2.49%, LockUpload -- 2.66e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 2.45%, LockDownload -- 2.61e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:33:01.803 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260508 03:33:01.915 | INFO | Starting a round of Baz IO | 20260508 03:33:05.332 | INFO | Populating RTMetrics report | 20260508 03:33:05.333 | INFO | RTMetrics report populated | 20260508 03:33:05.333 | INFO | Opening tmp RTMetrics json file | 20260508 03:33:05.347 | INFO | Serializing RTMetrics to json | 20260508 03:33:05.349 | INFO | Serialization done, swapping tmp file now... | 20260508 03:33:05.350 | INFO | RTMetrics done | 20260508 03:33:05.410 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.494 seconds | 20260508 03:33:05.410 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.26% / 0.00% / 58.74% | 20260508 03:33:05.410 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.61328% / 0% / 7.54992e-05% | 20260508 03:33:05.410 | INFO | Baz IO overall: | 20260508 03:33:05.410 | INFO | BytesWritten : 2205584801455 | 20260508 03:33:05.410 | INFO | NumEvents : 1065492580964 | 20260508 03:33:06.937 | INFO | Analyzing chunk frames = [4907008,4907520) | 20260508 03:33:06.937 | INFO | MemInfo: { | 20260508 03:33:06.937 | INFO | "gpu" : " 17 GB", | 20260508 03:33:06.937 | INFO | "huge" : "", | 20260508 03:33:06.937 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:06.937 | INFO | "malloc" : " 14 GB", | 20260508 03:33:06.937 | INFO | "pinned" : " 29 GB" | 20260508 03:33:06.937 | INFO | } | 20260508 03:33:06.937 | INFO | Perf report for the last 5.13559 seconds: | 20260508 03:33:06.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:06.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:06.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:06.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:06.937 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 03:33:06.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:06.937 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:33:06.937 | INFO | PRE_HQ : 66.4%, 1.8, 62.0ms, 0 | 20260508 03:33:06.937 | INFO | BAZWRITER : 67.4%, 2.9, 98.9ms, 0 | 20260508 03:33:06.937 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:33:06.937 | INFO | Uploaded 36GiB to GPU in 2.57157s (13.9992GiB/s) | 20260508 03:33:06.937 | INFO | Downloaded 38GiB to GPU in 2.69378s (14.1066GiB/s) | 20260508 03:33:12.081 | INFO | Analyzing chunk frames = [4907520,4908032) | 20260508 03:33:12.081 | INFO | MemInfo: { | 20260508 03:33:12.081 | INFO | "gpu" : " 17 GB", | 20260508 03:33:12.081 | INFO | "huge" : "", | 20260508 03:33:12.081 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:12.081 | INFO | "malloc" : " 14 GB", | 20260508 03:33:12.081 | INFO | "pinned" : " 29 GB" | 20260508 03:33:12.081 | INFO | } | 20260508 03:33:12.081 | INFO | Perf report for the last 5.14466 seconds: | 20260508 03:33:12.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:12.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.081 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 03:33:12.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.081 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:33:12.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:12.082 | INFO | Uploaded 36GiB to GPU in 2.5453s (14.1437GiB/s) | 20260508 03:33:12.082 | INFO | Downloaded 34GiB to GPU in 2.43231s (13.9785GiB/s) | 20260508 03:33:17.223 | INFO | Analyzing chunk frames = [4908032,4908544) | 20260508 03:33:17.223 | INFO | MemInfo: { | 20260508 03:33:17.223 | INFO | "gpu" : " 17 GB", | 20260508 03:33:17.223 | INFO | "huge" : "", | 20260508 03:33:17.224 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:17.224 | INFO | "malloc" : " 14 GB", | 20260508 03:33:17.224 | INFO | "pinned" : " 29 GB" | 20260508 03:33:17.224 | INFO | } | 20260508 03:33:17.224 | INFO | Perf report for the last 5.14235 seconds: | 20260508 03:33:17.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:17.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 03:33:17.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:33:17.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:17.224 | INFO | Uploaded 36GiB to GPU in 2.5322s (14.2169GiB/s) | 20260508 03:33:17.224 | INFO | Downloaded 34GiB to GPU in 2.50279s (13.5848GiB/s) | 20260508 03:33:22.364 | INFO | Analyzing chunk frames = [4908544,4909056) | 20260508 03:33:22.364 | INFO | MemInfo: { | 20260508 03:33:22.364 | INFO | "gpu" : " 17 GB", | 20260508 03:33:22.364 | INFO | "huge" : "", | 20260508 03:33:22.364 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:22.365 | INFO | "malloc" : " 14 GB", | 20260508 03:33:22.365 | INFO | "pinned" : " 29 GB" | 20260508 03:33:22.365 | INFO | } | 20260508 03:33:22.365 | INFO | Perf report for the last 5.14091 seconds: | 20260508 03:33:22.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:22.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:33:22.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:33:22.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:22.365 | INFO | Uploaded 36GiB to GPU in 2.53454s (14.2037GiB/s) | 20260508 03:33:22.365 | INFO | Downloaded 34GiB to GPU in 2.50888s (13.5519GiB/s) | 20260508 03:33:27.500 | INFO | Analyzing chunk frames = [4909056,4909568) | 20260508 03:33:27.500 | INFO | MemInfo: { | 20260508 03:33:27.500 | INFO | "gpu" : " 17 GB", | 20260508 03:33:27.500 | INFO | "huge" : "", | 20260508 03:33:27.500 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:27.500 | INFO | "malloc" : " 14 GB", | 20260508 03:33:27.500 | INFO | "pinned" : " 29 GB" | 20260508 03:33:27.500 | INFO | } | 20260508 03:33:27.500 | INFO | Perf report for the last 5.13509 seconds: | 20260508 03:33:27.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:27.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 03:33:27.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:33:27.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:27.500 | INFO | Uploaded 36GiB to GPU in 2.35938s (15.2583GiB/s) | 20260508 03:33:27.501 | INFO | Downloaded 34GiB to GPU in 2.56797s (13.2401GiB/s) | 20260508 03:33:32.640 | INFO | Analyzing chunk frames = [4909568,4910080) | 20260508 03:33:32.640 | INFO | MemInfo: { | 20260508 03:33:32.640 | INFO | "gpu" : " 17 GB", | 20260508 03:33:32.640 | INFO | "huge" : "", | 20260508 03:33:32.640 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:32.641 | INFO | "malloc" : " 14 GB", | 20260508 03:33:32.641 | INFO | "pinned" : " 29 GB" | 20260508 03:33:32.641 | INFO | } | 20260508 03:33:32.641 | INFO | Perf report for the last 5.14084 seconds: | 20260508 03:33:32.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:32.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:33:32.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:33:32.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:32.641 | INFO | Uploaded 36GiB to GPU in 2.39095s (15.0568GiB/s) | 20260508 03:33:32.641 | INFO | Downloaded 34GiB to GPU in 2.43727s (13.9501GiB/s) | 20260508 03:33:37.783 | INFO | Analyzing chunk frames = [4910080,4910592) | 20260508 03:33:37.783 | INFO | MemInfo: { | 20260508 03:33:37.783 | INFO | "gpu" : " 17 GB", | 20260508 03:33:37.783 | INFO | "huge" : "", | 20260508 03:33:37.784 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:37.784 | INFO | "malloc" : " 14 GB", | 20260508 03:33:37.784 | INFO | "pinned" : " 29 GB" | 20260508 03:33:37.784 | INFO | } | 20260508 03:33:37.784 | INFO | Perf report for the last 5.14312 seconds: | 20260508 03:33:37.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:37.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:33:37.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 03:33:37.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:37.784 | INFO | Uploaded 36GiB to GPU in 2.39259s (15.0464GiB/s) | 20260508 03:33:37.784 | INFO | Downloaded 34GiB to GPU in 2.44542s (13.9035GiB/s) | 20260508 03:33:42.918 | INFO | Analyzing chunk frames = [4910592,4911104) | 20260508 03:33:42.918 | INFO | MemInfo: { | 20260508 03:33:42.918 | INFO | "gpu" : " 17 GB", | 20260508 03:33:42.918 | INFO | "huge" : "", | 20260508 03:33:42.919 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:42.919 | INFO | "malloc" : " 14 GB", | 20260508 03:33:42.919 | INFO | "pinned" : " 29 GB" | 20260508 03:33:42.919 | INFO | } | 20260508 03:33:42.919 | INFO | Perf report for the last 5.13504 seconds: | 20260508 03:33:42.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:42.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.919 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:33:42.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.919 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:33:42.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:42.920 | INFO | Uploaded 36GiB to GPU in 2.43322s (14.7952GiB/s) | 20260508 03:33:42.920 | INFO | Downloaded 34GiB to GPU in 2.44457s (13.9084GiB/s) | 20260508 03:33:43.032 | INFO | Starting a round of Baz IO | 20260508 03:33:46.461 | INFO | Populating RTMetrics report | 20260508 03:33:46.461 | INFO | RTMetrics report populated | 20260508 03:33:46.461 | INFO | Opening tmp RTMetrics json file | 20260508 03:33:46.472 | INFO | Serializing RTMetrics to json | 20260508 03:33:46.474 | INFO | Serialization done, swapping tmp file now... | 20260508 03:33:46.474 | INFO | RTMetrics done | 20260508 03:33:46.531 | INFO | Finished a round of Baz IO: Wrote 1.615 GiB in 3.498 seconds | 20260508 03:33:46.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.26% / 0.00% / 58.74% | 20260508 03:33:46.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.611456% / 3.97364e-06% / 4.37101e-05% | 20260508 03:33:46.531 | INFO | Baz IO overall: | 20260508 03:33:46.531 | INFO | BytesWritten : 2207318416862 | 20260508 03:33:46.531 | INFO | NumEvents : 1066322080767 | 20260508 03:33:48.058 | INFO | Analyzing chunk frames = [4911104,4911616) | 20260508 03:33:48.058 | INFO | MemInfo: { | 20260508 03:33:48.058 | INFO | "gpu" : " 17 GB", | 20260508 03:33:48.058 | INFO | "huge" : "", | 20260508 03:33:48.058 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:48.058 | INFO | "malloc" : " 14 GB", | 20260508 03:33:48.058 | INFO | "pinned" : " 29 GB" | 20260508 03:33:48.058 | INFO | } | 20260508 03:33:48.058 | INFO | Perf report for the last 5.13908 seconds: | 20260508 03:33:48.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:48.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:48.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:48.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:48.058 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 03:33:48.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:48.058 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 03:33:48.058 | INFO | PRE_HQ : 66.0%, 1.8, 61.9ms, 0 | 20260508 03:33:48.058 | INFO | BAZWRITER : 67.6%, 2.9, 99.6ms, 0 | 20260508 03:33:48.058 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:33:48.058 | INFO | Uploaded 36GiB to GPU in 2.54066s (14.1696GiB/s) | 20260508 03:33:48.058 | INFO | Downloaded 38GiB to GPU in 2.69428s (14.104GiB/s) | 20260508 03:33:53.193 | INFO | Analyzing chunk frames = [4911616,4912128) | 20260508 03:33:53.193 | INFO | MemInfo: { | 20260508 03:33:53.193 | INFO | "gpu" : " 17 GB", | 20260508 03:33:53.193 | INFO | "huge" : "", | 20260508 03:33:53.193 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:53.193 | INFO | "malloc" : " 14 GB", | 20260508 03:33:53.193 | INFO | "pinned" : " 29 GB" | 20260508 03:33:53.193 | INFO | } | 20260508 03:33:53.193 | INFO | Perf report for the last 5.13511 seconds: | 20260508 03:33:53.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:53.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.193 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:33:53.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.193 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:33:53.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:53.194 | INFO | Uploaded 36GiB to GPU in 2.56958s (14.01GiB/s) | 20260508 03:33:53.194 | INFO | Downloaded 34GiB to GPU in 2.43053s (13.9887GiB/s) | 20260508 03:33:58.337 | INFO | Analyzing chunk frames = [4912128,4912640) | 20260508 03:33:58.337 | INFO | MemInfo: { | 20260508 03:33:58.337 | INFO | "gpu" : " 17 GB", | 20260508 03:33:58.337 | INFO | "huge" : "", | 20260508 03:33:58.337 | INFO | "huge_shared" : " 29 GB", | 20260508 03:33:58.337 | INFO | "malloc" : " 14 GB", | 20260508 03:33:58.337 | INFO | "pinned" : " 29 GB" | 20260508 03:33:58.337 | INFO | } | 20260508 03:33:58.337 | INFO | Perf report for the last 5.14399 seconds: | 20260508 03:33:58.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:33:58.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:33:58.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:33:58.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:33:58.337 | INFO | Uploaded 36GiB to GPU in 2.5557s (14.0862GiB/s) | 20260508 03:33:58.337 | INFO | Downloaded 34GiB to GPU in 2.49693s (13.6167GiB/s) | 20260508 03:34:03.477 | INFO | Analyzing chunk frames = [4912640,4913152) | 20260508 03:34:03.477 | INFO | MemInfo: { | 20260508 03:34:03.477 | INFO | "gpu" : " 17 GB", | 20260508 03:34:03.477 | INFO | "huge" : "", | 20260508 03:34:03.477 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:03.477 | INFO | "malloc" : " 14 GB", | 20260508 03:34:03.477 | INFO | "pinned" : " 29 GB" | 20260508 03:34:03.477 | INFO | } | 20260508 03:34:03.477 | INFO | Perf report for the last 5.14045 seconds: | 20260508 03:34:03.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:03.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.478 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 03:34:03.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.478 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:34:03.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:03.478 | INFO | Uploaded 36GiB to GPU in 2.53762s (14.1865GiB/s) | 20260508 03:34:03.478 | INFO | Downloaded 34GiB to GPU in 2.50612s (13.5668GiB/s) | 20260508 03:34:08.612 | INFO | Analyzing chunk frames = [4913152,4913664) | 20260508 03:34:08.612 | INFO | MemInfo: { | 20260508 03:34:08.612 | INFO | "gpu" : " 17 GB", | 20260508 03:34:08.612 | INFO | "huge" : "", | 20260508 03:34:08.612 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:08.612 | INFO | "malloc" : " 14 GB", | 20260508 03:34:08.612 | INFO | "pinned" : " 29 GB" | 20260508 03:34:08.612 | INFO | } | 20260508 03:34:08.612 | INFO | Perf report for the last 5.13466 seconds: | 20260508 03:34:08.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:08.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 03:34:08.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:34:08.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:08.612 | INFO | Uploaded 36GiB to GPU in 2.36437s (15.2261GiB/s) | 20260508 03:34:08.612 | INFO | Downloaded 34GiB to GPU in 2.55707s (13.2965GiB/s) | 20260508 03:34:13.750 | INFO | Analyzing chunk frames = [4913664,4914176) | 20260508 03:34:13.750 | INFO | MemInfo: { | 20260508 03:34:13.750 | INFO | "gpu" : " 17 GB", | 20260508 03:34:13.750 | INFO | "huge" : "", | 20260508 03:34:13.750 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:13.750 | INFO | "malloc" : " 14 GB", | 20260508 03:34:13.750 | INFO | "pinned" : " 29 GB" | 20260508 03:34:13.750 | INFO | } | 20260508 03:34:13.750 | INFO | Perf report for the last 5.13767 seconds: | 20260508 03:34:13.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:13.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:34:13.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:34:13.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:13.750 | INFO | Uploaded 36GiB to GPU in 2.42047s (14.8731GiB/s) | 20260508 03:34:13.751 | INFO | Downloaded 34GiB to GPU in 2.42539s (14.0183GiB/s) | 20260508 03:34:18.891 | INFO | Analyzing chunk frames = [4914176,4914688) | 20260508 03:34:18.891 | INFO | MemInfo: { | 20260508 03:34:18.891 | INFO | "gpu" : " 17 GB", | 20260508 03:34:18.891 | INFO | "huge" : "", | 20260508 03:34:18.892 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:18.892 | INFO | "malloc" : " 14 GB", | 20260508 03:34:18.892 | INFO | "pinned" : " 29 GB" | 20260508 03:34:18.892 | INFO | } | 20260508 03:34:18.892 | INFO | Perf report for the last 5.1419 seconds: | 20260508 03:34:18.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:18.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.892 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:34:18.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.892 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:34:18.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:18.893 | INFO | Uploaded 36GiB to GPU in 2.43743s (14.7697GiB/s) | 20260508 03:34:18.893 | INFO | Downloaded 34GiB to GPU in 2.42334s (14.0302GiB/s) | 20260508 03:34:24.037 | INFO | Analyzing chunk frames = [4914688,4915200) | 20260508 03:34:24.037 | INFO | MemInfo: { | 20260508 03:34:24.037 | INFO | "gpu" : " 17 GB", | 20260508 03:34:24.037 | INFO | "huge" : "", | 20260508 03:34:24.037 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:24.037 | INFO | "malloc" : " 14 GB", | 20260508 03:34:24.037 | INFO | "pinned" : " 29 GB" | 20260508 03:34:24.037 | INFO | } | 20260508 03:34:24.037 | INFO | Perf report for the last 5.14537 seconds: | 20260508 03:34:24.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:24.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.037 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:34:24.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.037 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 03:34:24.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:24.038 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:34:24.038 | INFO | Total mem usage: 16.7 GiB | 20260508 03:34:24.038 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:34:24.038 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.8%) | 20260508 03:34:24.038 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:34:24.038 | INFO | High Water Breakdown: | 20260508 03:34:24.038 | INFO | Uploaded 36GiB to GPU in 2.46868s (14.5827GiB/s) | 20260508 03:34:24.038 | INFO | 59.6% : 8.6 GiB ( 83.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:34:24.038 | INFO | Downloaded 34GiB to GPU in 2.42784s (14.0042GiB/s) | 20260508 03:34:24.038 | INFO | 40.2% : 5.8 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:34:24.038 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:34:24.038 | INFO | ------------------------------------------ | 20260508 03:34:24.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:34:24.038 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:34:24.038 | INFO | Total mem usage: 26.5 GiB | 20260508 03:34:24.038 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:34:24.038 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:34:24.038 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:34:24.038 | INFO | High Water Breakdown: | 20260508 03:34:24.038 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:34:24.038 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:34:24.038 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:34:24.038 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:34:24.038 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:34:24.038 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:34:24.038 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:34:24.038 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:34:24.038 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:34:24.038 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:34:24.038 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:34:24.038 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:34:24.038 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:34:24.038 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:34:24.038 | INFO | ------------------------------------------ | 20260508 03:34:24.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:34:24.038 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:34:24.038 | INFO | Total mem usage: 27.8 GiB | 20260508 03:34:24.038 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:34:24.038 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:34:24.038 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:34:24.038 | INFO | High Water Breakdown: | 20260508 03:34:24.038 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:34:24.038 | INFO | ------------------------------------------ | 20260508 03:34:24.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:34:24.038 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:34:24.038 | INFO | Total mem usage: 15.7 GiB | 20260508 03:34:24.038 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:34:24.038 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:34:24.038 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:34:24.038 | INFO | High Water Breakdown: | 20260508 03:34:24.038 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:34:24.038 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:34:24.038 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:34:24.038 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:34:24.038 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:34:24.038 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:34:24.038 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:34:24.038 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:34:24.038 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:34:24.038 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:34:24.038 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:34:24.038 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:34:24.038 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:34:24.038 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:34:24.038 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:34:24.038 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:34:24.038 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:34:24.038 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:34:24.039 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:34:24.039 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:34:24.039 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:34:24.039 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:34:24.039 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:34:24.039 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:34:24.039 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:34:24.039 | INFO | ------------------------------------------ | 20260508 03:34:24.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:34:24.039 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:34:24.039 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:34:24.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:34:24.039 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.59e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.02e-01, 1.15e-02 / 4.82e-03 | 20260508 03:34:24.039 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.44e+00, 5.81e-01 / 2.12e-01 | 20260508 03:34:24.039 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.08e-01 / 1.23e-01 | 20260508 03:34:24.039 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.61e+01, 4.21e+00 / 1.95e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.67e+00 / 1.56e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.15e-01 / 2.38e-01 | 20260508 03:34:24.039 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.21e+00, 2.64e-01 / 1.26e-01 | 20260508 03:34:24.039 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.59e-01, 5.14e-02 / 2.17e-02 | 20260508 03:34:24.039 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:34:24.039 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:34:24.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:34:24.039 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 24.28%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 24.49%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 2.00%, LockUpload -- 2.12e+00 / 0.00e+00, 5.40e+00 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 2.91%, LockDownload -- 3.09e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:34:24.039 | INFO | PerfReport: 0.01%, Misc -- 1.57e-02 / 0.00e+00, 3.61e-02 / 0.00e+00 | 20260508 03:34:24.137 | INFO | Starting a round of Baz IO | 20260508 03:34:27.569 | INFO | Populating RTMetrics report | 20260508 03:34:27.569 | INFO | RTMetrics report populated | 20260508 03:34:27.569 | INFO | Opening tmp RTMetrics json file | 20260508 03:34:27.581 | INFO | Serializing RTMetrics to json | 20260508 03:34:27.582 | INFO | Serialization done, swapping tmp file now... | 20260508 03:34:27.582 | INFO | RTMetrics done | 20260508 03:34:27.642 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.505 seconds | 20260508 03:34:27.642 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.25% / 0.00% / 58.74% | 20260508 03:34:27.642 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.611472% / 7.94729e-06% / 6.35783e-05% | 20260508 03:34:27.642 | INFO | Baz IO overall: | 20260508 03:34:27.642 | INFO | BytesWritten : 2209053428515 | 20260508 03:34:27.642 | INFO | NumEvents : 1067152785912 | 20260508 03:34:29.178 | INFO | Analyzing chunk frames = [4915200,4915712) | 20260508 03:34:29.178 | INFO | MemInfo: { | 20260508 03:34:29.178 | INFO | "gpu" : " 17 GB", | 20260508 03:34:29.178 | INFO | "huge" : "", | 20260508 03:34:29.178 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:29.178 | INFO | "malloc" : " 14 GB", | 20260508 03:34:29.178 | INFO | "pinned" : " 29 GB" | 20260508 03:34:29.178 | INFO | } | 20260508 03:34:29.178 | INFO | Perf report for the last 5.14116 seconds: | 20260508 03:34:29.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:29.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:29.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:29.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:29.178 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 03:34:29.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:29.178 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:34:29.178 | INFO | PRE_HQ : 65.6%, 1.6, 55.6ms, 0 | 20260508 03:34:29.178 | INFO | BAZWRITER : 67.7%, 2.7, 93.3ms, 0 | 20260508 03:34:29.178 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:34:29.179 | INFO | Uploaded 36GiB to GPU in 2.51857s (14.2938GiB/s) | 20260508 03:34:29.179 | INFO | Downloaded 38GiB to GPU in 2.70236s (14.0618GiB/s) | 20260508 03:34:34.316 | INFO | Analyzing chunk frames = [4915712,4916224) | 20260508 03:34:34.316 | INFO | MemInfo: { | 20260508 03:34:34.316 | INFO | "gpu" : " 17 GB", | 20260508 03:34:34.316 | INFO | "huge" : "", | 20260508 03:34:34.316 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:34.316 | INFO | "malloc" : " 14 GB", | 20260508 03:34:34.316 | INFO | "pinned" : " 29 GB" | 20260508 03:34:34.316 | INFO | } | 20260508 03:34:34.316 | INFO | Perf report for the last 5.13746 seconds: | 20260508 03:34:34.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:34.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:34:34.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:34:34.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:34.316 | INFO | Uploaded 36GiB to GPU in 2.57596s (13.9754GiB/s) | 20260508 03:34:34.317 | INFO | Downloaded 34GiB to GPU in 2.42968s (13.9936GiB/s) | 20260508 03:34:39.449 | INFO | Analyzing chunk frames = [4916224,4916736) | 20260508 03:34:39.449 | INFO | MemInfo: { | 20260508 03:34:39.449 | INFO | "gpu" : " 17 GB", | 20260508 03:34:39.449 | INFO | "huge" : "", | 20260508 03:34:39.449 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:39.449 | INFO | "malloc" : " 14 GB", | 20260508 03:34:39.449 | INFO | "pinned" : " 29 GB" | 20260508 03:34:39.449 | INFO | } | 20260508 03:34:39.449 | INFO | Perf report for the last 5.13352 seconds: | 20260508 03:34:39.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:39.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.449 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 03:34:39.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.449 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:34:39.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:39.450 | INFO | Uploaded 36GiB to GPU in 2.53991s (14.1737GiB/s) | 20260508 03:34:39.450 | INFO | Downloaded 34GiB to GPU in 2.42001s (14.0495GiB/s) | 20260508 03:34:44.591 | INFO | Analyzing chunk frames = [4916736,4917248) | 20260508 03:34:44.591 | INFO | MemInfo: { | 20260508 03:34:44.591 | INFO | "gpu" : " 17 GB", | 20260508 03:34:44.591 | INFO | "huge" : "", | 20260508 03:34:44.591 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:44.591 | INFO | "malloc" : " 14 GB", | 20260508 03:34:44.591 | INFO | "pinned" : " 29 GB" | 20260508 03:34:44.591 | INFO | } | 20260508 03:34:44.591 | INFO | Perf report for the last 5.1421 seconds: | 20260508 03:34:44.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:44.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.591 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:34:44.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.592 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:34:44.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:44.592 | INFO | Uploaded 36GiB to GPU in 2.54666s (14.1362GiB/s) | 20260508 03:34:44.592 | INFO | Downloaded 34GiB to GPU in 2.50693s (13.5624GiB/s) | 20260508 03:34:49.732 | INFO | Analyzing chunk frames = [4917248,4917760) | 20260508 03:34:49.732 | INFO | MemInfo: { | 20260508 03:34:49.732 | INFO | "gpu" : " 17 GB", | 20260508 03:34:49.732 | INFO | "huge" : "", | 20260508 03:34:49.732 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:49.732 | INFO | "malloc" : " 14 GB", | 20260508 03:34:49.732 | INFO | "pinned" : " 29 GB" | 20260508 03:34:49.732 | INFO | } | 20260508 03:34:49.732 | INFO | Perf report for the last 5.14093 seconds: | 20260508 03:34:49.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:49.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.732 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 03:34:49.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.732 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:34:49.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:49.733 | INFO | Uploaded 36GiB to GPU in 2.39735s (15.0166GiB/s) | 20260508 03:34:49.733 | INFO | Downloaded 34GiB to GPU in 2.545s (13.3596GiB/s) | 20260508 03:34:54.871 | INFO | Analyzing chunk frames = [4917760,4918272) | 20260508 03:34:54.871 | INFO | MemInfo: { | 20260508 03:34:54.871 | INFO | "gpu" : " 17 GB", | 20260508 03:34:54.871 | INFO | "huge" : "", | 20260508 03:34:54.871 | INFO | "huge_shared" : " 29 GB", | 20260508 03:34:54.871 | INFO | "malloc" : " 14 GB", | 20260508 03:34:54.871 | INFO | "pinned" : " 29 GB" | 20260508 03:34:54.871 | INFO | } | 20260508 03:34:54.872 | INFO | Perf report for the last 5.13913 seconds: | 20260508 03:34:54.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:34:54.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:34:54.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:34:54.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:34:54.872 | INFO | Uploaded 36GiB to GPU in 2.39868s (15.0082GiB/s) | 20260508 03:34:54.872 | INFO | Downloaded 34GiB to GPU in 2.41423s (14.0832GiB/s) | 20260508 03:35:00.007 | INFO | Analyzing chunk frames = [4918272,4918784) | 20260508 03:35:00.007 | INFO | MemInfo: { | 20260508 03:35:00.007 | INFO | "gpu" : " 17 GB", | 20260508 03:35:00.007 | INFO | "huge" : "", | 20260508 03:35:00.007 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:00.007 | INFO | "malloc" : " 14 GB", | 20260508 03:35:00.007 | INFO | "pinned" : " 29 GB" | 20260508 03:35:00.007 | INFO | } | 20260508 03:35:00.007 | INFO | Perf report for the last 5.13591 seconds: | 20260508 03:35:00.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:00.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.007 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:35:00.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.008 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 03:35:00.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:00.008 | INFO | Uploaded 36GiB to GPU in 2.43266s (14.7986GiB/s) | 20260508 03:35:00.008 | INFO | Downloaded 34GiB to GPU in 2.42907s (13.9971GiB/s) | 20260508 03:35:05.149 | INFO | Analyzing chunk frames = [4918784,4919296) | 20260508 03:35:05.149 | INFO | MemInfo: { | 20260508 03:35:05.149 | INFO | "gpu" : " 17 GB", | 20260508 03:35:05.149 | INFO | "huge" : "", | 20260508 03:35:05.149 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:05.150 | INFO | "malloc" : " 14 GB", | 20260508 03:35:05.150 | INFO | "pinned" : " 29 GB" | 20260508 03:35:05.150 | INFO | } | 20260508 03:35:05.150 | INFO | Perf report for the last 5.14215 seconds: | 20260508 03:35:05.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:05.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 03:35:05.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:35:05.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:05.150 | INFO | Uploaded 36GiB to GPU in 2.44416s (14.729GiB/s) | 20260508 03:35:05.150 | INFO | Downloaded 34GiB to GPU in 2.42303s (14.032GiB/s) | 20260508 03:35:05.247 | INFO | Starting a round of Baz IO | 20260508 03:35:08.680 | INFO | Populating RTMetrics report | 20260508 03:35:08.680 | INFO | RTMetrics report populated | 20260508 03:35:08.680 | INFO | Opening tmp RTMetrics json file | 20260508 03:35:08.690 | INFO | Serializing RTMetrics to json | 20260508 03:35:08.693 | INFO | Serialization done, swapping tmp file now... | 20260508 03:35:08.693 | INFO | RTMetrics done | 20260508 03:35:08.752 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.505 seconds | 20260508 03:35:08.752 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.25% / 0.00% / 58.75% | 20260508 03:35:08.752 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.612624% / 0% / 4.37101e-05% | 20260508 03:35:08.752 | INFO | Baz IO overall: | 20260508 03:35:08.752 | INFO | BytesWritten : 2210789648830 | 20260508 03:35:08.752 | INFO | NumEvents : 1067984553937 | 20260508 03:35:10.288 | INFO | Analyzing chunk frames = [4919296,4919808) | 20260508 03:35:10.288 | INFO | MemInfo: { | 20260508 03:35:10.288 | INFO | "gpu" : " 17 GB", | 20260508 03:35:10.288 | INFO | "huge" : "", | 20260508 03:35:10.288 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:10.288 | INFO | "malloc" : " 14 GB", | 20260508 03:35:10.288 | INFO | "pinned" : " 29 GB" | 20260508 03:35:10.288 | INFO | } | 20260508 03:35:10.288 | INFO | Perf report for the last 5.1387 seconds: | 20260508 03:35:10.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:10.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:10.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:10.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:10.288 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 03:35:10.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:10.288 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:35:10.288 | INFO | PRE_HQ : 65.6%, 1.6, 55.7ms, 0 | 20260508 03:35:10.288 | INFO | BAZWRITER : 67.7%, 2.6, 91.4ms, 0 | 20260508 03:35:10.288 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:35:10.289 | INFO | Uploaded 36GiB to GPU in 2.57455s (13.983GiB/s) | 20260508 03:35:10.289 | INFO | Downloaded 38GiB to GPU in 2.69481s (14.1012GiB/s) | 20260508 03:35:15.427 | INFO | Analyzing chunk frames = [4919808,4920320) | 20260508 03:35:15.427 | INFO | MemInfo: { | 20260508 03:35:15.427 | INFO | "gpu" : " 17 GB", | 20260508 03:35:15.427 | INFO | "huge" : "", | 20260508 03:35:15.427 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:15.427 | INFO | "malloc" : " 14 GB", | 20260508 03:35:15.427 | INFO | "pinned" : " 29 GB" | 20260508 03:35:15.428 | INFO | } | 20260508 03:35:15.428 | INFO | Perf report for the last 5.13928 seconds: | 20260508 03:35:15.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:15.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:35:15.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:35:15.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:15.428 | INFO | Uploaded 36GiB to GPU in 2.57809s (13.9638GiB/s) | 20260508 03:35:15.428 | INFO | Downloaded 34GiB to GPU in 2.42301s (14.0321GiB/s) | 20260508 03:35:20.562 | INFO | Analyzing chunk frames = [4920320,4920832) | 20260508 03:35:20.562 | INFO | MemInfo: { | 20260508 03:35:20.562 | INFO | "gpu" : " 17 GB", | 20260508 03:35:20.562 | INFO | "huge" : "", | 20260508 03:35:20.562 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:20.562 | INFO | "malloc" : " 14 GB", | 20260508 03:35:20.562 | INFO | "pinned" : " 29 GB" | 20260508 03:35:20.562 | INFO | } | 20260508 03:35:20.562 | INFO | Perf report for the last 5.13438 seconds: | 20260508 03:35:20.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:20.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 03:35:20.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:35:20.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:20.562 | INFO | Uploaded 36GiB to GPU in 2.55688s (14.0796GiB/s) | 20260508 03:35:20.562 | INFO | Downloaded 34GiB to GPU in 2.49628s (13.6203GiB/s) | 20260508 03:35:25.702 | INFO | Analyzing chunk frames = [4920832,4921344) | 20260508 03:35:25.702 | INFO | MemInfo: { | 20260508 03:35:25.702 | INFO | "gpu" : " 17 GB", | 20260508 03:35:25.702 | INFO | "huge" : "", | 20260508 03:35:25.702 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:25.702 | INFO | "malloc" : " 14 GB", | 20260508 03:35:25.702 | INFO | "pinned" : " 29 GB" | 20260508 03:35:25.702 | INFO | } | 20260508 03:35:25.702 | INFO | Perf report for the last 5.13994 seconds: | 20260508 03:35:25.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:25.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:35:25.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:35:25.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:25.702 | INFO | Uploaded 36GiB to GPU in 2.55874s (14.0694GiB/s) | 20260508 03:35:25.703 | INFO | Downloaded 34GiB to GPU in 2.49484s (13.6281GiB/s) | 20260508 03:35:30.845 | INFO | Analyzing chunk frames = [4921344,4921856) | 20260508 03:35:30.845 | INFO | MemInfo: { | 20260508 03:35:30.845 | INFO | "gpu" : " 17 GB", | 20260508 03:35:30.845 | INFO | "huge" : "", | 20260508 03:35:30.845 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:30.845 | INFO | "malloc" : " 14 GB", | 20260508 03:35:30.845 | INFO | "pinned" : " 29 GB" | 20260508 03:35:30.846 | INFO | } | 20260508 03:35:30.846 | INFO | Perf report for the last 5.14382 seconds: | 20260508 03:35:30.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:30.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:35:30.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 03:35:30.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:30.846 | INFO | Uploaded 36GiB to GPU in 2.3892s (15.0678GiB/s) | 20260508 03:35:30.846 | INFO | Downloaded 34GiB to GPU in 2.54715s (13.3483GiB/s) | 20260508 03:35:35.984 | INFO | Analyzing chunk frames = [4921856,4922368) | 20260508 03:35:35.984 | INFO | MemInfo: { | 20260508 03:35:35.984 | INFO | "gpu" : " 17 GB", | 20260508 03:35:35.984 | INFO | "huge" : "", | 20260508 03:35:35.984 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:35.984 | INFO | "malloc" : " 14 GB", | 20260508 03:35:35.984 | INFO | "pinned" : " 29 GB" | 20260508 03:35:35.984 | INFO | } | 20260508 03:35:35.984 | INFO | Perf report for the last 5.13801 seconds: | 20260508 03:35:35.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:35.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:35:35.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:35:35.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:35.984 | INFO | Uploaded 36GiB to GPU in 2.45263s (14.6782GiB/s) | 20260508 03:35:35.984 | INFO | Downloaded 34GiB to GPU in 2.41069s (14.1039GiB/s) | 20260508 03:35:41.126 | INFO | Analyzing chunk frames = [4922368,4922880) | 20260508 03:35:41.126 | INFO | MemInfo: { | 20260508 03:35:41.126 | INFO | "gpu" : " 17 GB", | 20260508 03:35:41.126 | INFO | "huge" : "", | 20260508 03:35:41.126 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:41.126 | INFO | "malloc" : " 14 GB", | 20260508 03:35:41.126 | INFO | "pinned" : " 29 GB" | 20260508 03:35:41.126 | INFO | } | 20260508 03:35:41.126 | INFO | Perf report for the last 5.14234 seconds: | 20260508 03:35:41.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:41.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:35:41.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:35:41.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:41.126 | INFO | Uploaded 36GiB to GPU in 2.45649s (14.6551GiB/s) | 20260508 03:35:41.126 | INFO | Downloaded 34GiB to GPU in 2.41188s (14.0969GiB/s) | 20260508 03:35:46.262 | INFO | Analyzing chunk frames = [4922880,4923392) | 20260508 03:35:46.262 | INFO | MemInfo: { | 20260508 03:35:46.262 | INFO | "gpu" : " 17 GB", | 20260508 03:35:46.262 | INFO | "huge" : "", | 20260508 03:35:46.263 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:46.263 | INFO | "malloc" : " 14 GB", | 20260508 03:35:46.263 | INFO | "pinned" : " 29 GB" | 20260508 03:35:46.263 | INFO | } | 20260508 03:35:46.263 | INFO | Perf report for the last 5.1365 seconds: | 20260508 03:35:46.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:46.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:35:46.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 03:35:46.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:46.263 | INFO | Uploaded 36GiB to GPU in 2.48675s (14.4767GiB/s) | 20260508 03:35:46.263 | INFO | Downloaded 34GiB to GPU in 2.42485s (14.0215GiB/s) | 20260508 03:35:46.263 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:35:46.263 | INFO | Total mem usage: 16.7 GiB | 20260508 03:35:46.263 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:35:46.263 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 03:35:46.263 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:35:46.263 | INFO | High Water Breakdown: | 20260508 03:35:46.263 | INFO | 59.4% : 8.48 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:35:46.263 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:35:46.263 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:35:46.263 | INFO | ------------------------------------------ | 20260508 03:35:46.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:35:46.263 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:35:46.263 | INFO | Total mem usage: 26.5 GiB | 20260508 03:35:46.263 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:35:46.263 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:35:46.263 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:35:46.263 | INFO | High Water Breakdown: | 20260508 03:35:46.263 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:35:46.263 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:35:46.263 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:35:46.263 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:35:46.263 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:35:46.263 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:35:46.263 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:35:46.263 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:35:46.263 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:35:46.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:35:46.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:35:46.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:35:46.264 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:35:46.264 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:35:46.264 | INFO | ------------------------------------------ | 20260508 03:35:46.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:35:46.264 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:35:46.264 | INFO | Total mem usage: 27.8 GiB | 20260508 03:35:46.264 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:35:46.264 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:35:46.264 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:35:46.264 | INFO | High Water Breakdown: | 20260508 03:35:46.264 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:35:46.264 | INFO | ------------------------------------------ | 20260508 03:35:46.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:35:46.264 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:35:46.264 | INFO | Total mem usage: 15.7 GiB | 20260508 03:35:46.264 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:35:46.264 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:35:46.264 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:35:46.264 | INFO | High Water Breakdown: | 20260508 03:35:46.264 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:35:46.264 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:35:46.264 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:35:46.264 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:35:46.264 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:35:46.264 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:35:46.264 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:35:46.264 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:35:46.264 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:35:46.264 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:35:46.264 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:35:46.264 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:35:46.264 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:35:46.264 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:35:46.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:35:46.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:35:46.264 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:35:46.264 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:35:46.264 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:35:46.264 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:35:46.264 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:35:46.264 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:35:46.264 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:35:46.264 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:35:46.264 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:35:46.264 | INFO | ------------------------------------------ | 20260508 03:35:46.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:35:46.264 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:35:46.264 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:35:46.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:35:46.264 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.39e+01 / 1.57e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.01e-01, 1.11e-02 / 4.27e-03 | 20260508 03:35:46.264 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.50e+00, 5.88e-01 / 1.93e-01 | 20260508 03:35:46.264 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.05e-01 | 20260508 03:35:46.264 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 3.93e+00 / 1.87e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.73e+00 / 1.54e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.68e+00, 4.23e-01 / 2.06e-01 | 20260508 03:35:46.264 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.22e+00, 2.67e-01 / 1.50e-01 | 20260508 03:35:46.264 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.64e-01, 5.25e-02 / 3.57e-02 | 20260508 03:35:46.264 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:35:46.264 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:35:46.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:35:46.264 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 24.40%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 2.22%, LockUpload -- 2.37e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 2.71%, LockDownload -- 2.89e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:35:46.264 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260508 03:35:46.360 | INFO | Starting a round of Baz IO | 20260508 03:35:49.800 | INFO | Populating RTMetrics report | 20260508 03:35:49.800 | INFO | RTMetrics report populated | 20260508 03:35:49.800 | INFO | Opening tmp RTMetrics json file | 20260508 03:35:49.814 | INFO | Serializing RTMetrics to json | 20260508 03:35:49.816 | INFO | Serialization done, swapping tmp file now... | 20260508 03:35:49.817 | INFO | RTMetrics done | 20260508 03:35:49.876 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.516 seconds | 20260508 03:35:49.876 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.25% / 0.00% / 58.75% | 20260508 03:35:49.876 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.607546% / 3.97364e-06% / 6.75519e-05% | 20260508 03:35:49.876 | INFO | Baz IO overall: | 20260508 03:35:49.876 | INFO | BytesWritten : 2212523010308 | 20260508 03:35:49.876 | INFO | NumEvents : 1068814347145 | 20260508 03:35:51.404 | INFO | Analyzing chunk frames = [4923392,4923904) | 20260508 03:35:51.404 | INFO | MemInfo: { | 20260508 03:35:51.404 | INFO | "gpu" : " 17 GB", | 20260508 03:35:51.404 | INFO | "huge" : "", | 20260508 03:35:51.404 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:51.404 | INFO | "malloc" : " 14 GB", | 20260508 03:35:51.404 | INFO | "pinned" : " 29 GB" | 20260508 03:35:51.404 | INFO | } | 20260508 03:35:51.404 | INFO | Perf report for the last 5.14129 seconds: | 20260508 03:35:51.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:51.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:51.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:51.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:51.404 | INFO | ANALYSIS : 68.1%, 3.0, 106.4ms, 0 | 20260508 03:35:51.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:51.404 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:35:51.404 | INFO | PRE_HQ : 65.7%, 1.6, 55.6ms, 0 | 20260508 03:35:51.404 | INFO | BAZWRITER : 67.9%, 2.7, 92.9ms, 0 | 20260508 03:35:51.404 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:35:51.404 | INFO | Uploaded 36GiB to GPU in 2.61324s (13.776GiB/s) | 20260508 03:35:51.404 | INFO | Downloaded 38GiB to GPU in 2.70551s (14.0454GiB/s) | 20260508 03:35:56.542 | INFO | Analyzing chunk frames = [4923904,4924416) | 20260508 03:35:56.542 | INFO | MemInfo: { | 20260508 03:35:56.542 | INFO | "gpu" : " 17 GB", | 20260508 03:35:56.542 | INFO | "huge" : "", | 20260508 03:35:56.542 | INFO | "huge_shared" : " 29 GB", | 20260508 03:35:56.542 | INFO | "malloc" : " 14 GB", | 20260508 03:35:56.542 | INFO | "pinned" : " 29 GB" | 20260508 03:35:56.542 | INFO | } | 20260508 03:35:56.543 | INFO | Perf report for the last 5.13857 seconds: | 20260508 03:35:56.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:35:56.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:35:56.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:35:56.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:35:56.543 | INFO | Uploaded 36GiB to GPU in 2.58123s (13.9468GiB/s) | 20260508 03:35:56.543 | INFO | Downloaded 34GiB to GPU in 2.42645s (14.0123GiB/s) | 20260508 03:36:01.682 | INFO | Analyzing chunk frames = [4924416,4924928) | 20260508 03:36:01.682 | INFO | MemInfo: { | 20260508 03:36:01.682 | INFO | "gpu" : " 17 GB", | 20260508 03:36:01.682 | INFO | "huge" : "", | 20260508 03:36:01.682 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:01.683 | INFO | "malloc" : " 14 GB", | 20260508 03:36:01.683 | INFO | "pinned" : " 29 GB" | 20260508 03:36:01.683 | INFO | } | 20260508 03:36:01.683 | INFO | Perf report for the last 5.14019 seconds: | 20260508 03:36:01.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:01.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260508 03:36:01.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:36:01.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:01.683 | INFO | Uploaded 36GiB to GPU in 2.56344s (14.0436GiB/s) | 20260508 03:36:01.683 | INFO | Downloaded 34GiB to GPU in 2.42547s (14.0179GiB/s) | 20260508 03:36:06.817 | INFO | Analyzing chunk frames = [4924928,4925440) | 20260508 03:36:06.817 | INFO | MemInfo: { | 20260508 03:36:06.817 | INFO | "gpu" : " 17 GB", | 20260508 03:36:06.817 | INFO | "huge" : "", | 20260508 03:36:06.817 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:06.817 | INFO | "malloc" : " 14 GB", | 20260508 03:36:06.817 | INFO | "pinned" : " 29 GB" | 20260508 03:36:06.817 | INFO | } | 20260508 03:36:06.817 | INFO | Perf report for the last 5.13404 seconds: | 20260508 03:36:06.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:06.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 03:36:06.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:36:06.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:06.817 | INFO | Uploaded 36GiB to GPU in 2.57636s (13.9732GiB/s) | 20260508 03:36:06.817 | INFO | Downloaded 34GiB to GPU in 2.49901s (13.6054GiB/s) | 20260508 03:36:11.956 | INFO | Analyzing chunk frames = [4925440,4925952) | 20260508 03:36:11.956 | INFO | MemInfo: { | 20260508 03:36:11.956 | INFO | "gpu" : " 17 GB", | 20260508 03:36:11.956 | INFO | "huge" : "", | 20260508 03:36:11.957 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:11.957 | INFO | "malloc" : " 14 GB", | 20260508 03:36:11.957 | INFO | "pinned" : " 29 GB" | 20260508 03:36:11.957 | INFO | } | 20260508 03:36:11.957 | INFO | Perf report for the last 5.13992 seconds: | 20260508 03:36:11.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:11.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 03:36:11.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:36:11.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:11.957 | INFO | Uploaded 36GiB to GPU in 2.43526s (14.7828GiB/s) | 20260508 03:36:11.957 | INFO | Downloaded 34GiB to GPU in 2.54119s (13.3796GiB/s) | 20260508 03:36:17.101 | INFO | Analyzing chunk frames = [4925952,4926464) | 20260508 03:36:17.101 | INFO | MemInfo: { | 20260508 03:36:17.101 | INFO | "gpu" : " 17 GB", | 20260508 03:36:17.101 | INFO | "huge" : "", | 20260508 03:36:17.101 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:17.101 | INFO | "malloc" : " 14 GB", | 20260508 03:36:17.101 | INFO | "pinned" : " 29 GB" | 20260508 03:36:17.101 | INFO | } | 20260508 03:36:17.101 | INFO | Perf report for the last 5.14457 seconds: | 20260508 03:36:17.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:17.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.101 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:36:17.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.101 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:36:17.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:17.102 | INFO | Uploaded 36GiB to GPU in 2.44426s (14.7284GiB/s) | 20260508 03:36:17.102 | INFO | Downloaded 34GiB to GPU in 2.42414s (14.0256GiB/s) | 20260508 03:36:22.242 | INFO | Analyzing chunk frames = [4926464,4926976) | 20260508 03:36:22.242 | INFO | MemInfo: { | 20260508 03:36:22.242 | INFO | "gpu" : " 17 GB", | 20260508 03:36:22.242 | INFO | "huge" : "", | 20260508 03:36:22.242 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:22.242 | INFO | "malloc" : " 14 GB", | 20260508 03:36:22.242 | INFO | "pinned" : " 29 GB" | 20260508 03:36:22.243 | INFO | } | 20260508 03:36:22.243 | INFO | Perf report for the last 5.14126 seconds: | 20260508 03:36:22.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:22.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:36:22.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:36:22.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:22.243 | INFO | Uploaded 36GiB to GPU in 2.45792s (14.6465GiB/s) | 20260508 03:36:22.243 | INFO | Downloaded 34GiB to GPU in 2.41361s (14.0868GiB/s) | 20260508 03:36:27.375 | INFO | Analyzing chunk frames = [4926976,4927488) | 20260508 03:36:27.375 | INFO | MemInfo: { | 20260508 03:36:27.375 | INFO | "gpu" : " 17 GB", | 20260508 03:36:27.375 | INFO | "huge" : "", | 20260508 03:36:27.375 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:27.375 | INFO | "malloc" : " 14 GB", | 20260508 03:36:27.375 | INFO | "pinned" : " 29 GB" | 20260508 03:36:27.376 | INFO | } | 20260508 03:36:27.376 | INFO | Perf report for the last 5.13302 seconds: | 20260508 03:36:27.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:27.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:36:27.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 03:36:27.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:27.376 | INFO | Uploaded 36GiB to GPU in 2.48826s (14.4679GiB/s) | 20260508 03:36:27.376 | INFO | Downloaded 34GiB to GPU in 2.43128s (13.9844GiB/s) | 20260508 03:36:27.473 | INFO | Starting a round of Baz IO | 20260508 03:36:30.905 | INFO | Populating RTMetrics report | 20260508 03:36:30.905 | INFO | RTMetrics report populated | 20260508 03:36:30.905 | INFO | Opening tmp RTMetrics json file | 20260508 03:36:30.913 | INFO | Serializing RTMetrics to json | 20260508 03:36:30.919 | INFO | Serialization done, swapping tmp file now... | 20260508 03:36:30.919 | INFO | RTMetrics done | 20260508 03:36:30.982 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.508 seconds | 20260508 03:36:30.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.25% / 0.00% / 58.75% | 20260508 03:36:30.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.6072% / 0% / 6.35783e-05% | 20260508 03:36:30.982 | INFO | Baz IO overall: | 20260508 03:36:30.983 | INFO | BytesWritten : 2214255740901 | 20260508 03:36:30.983 | INFO | NumEvents : 1069643781981 | 20260508 03:36:32.513 | INFO | Analyzing chunk frames = [4927488,4928000) | 20260508 03:36:32.513 | INFO | MemInfo: { | 20260508 03:36:32.513 | INFO | "gpu" : " 17 GB", | 20260508 03:36:32.513 | INFO | "huge" : "", | 20260508 03:36:32.513 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:32.513 | INFO | "malloc" : " 14 GB", | 20260508 03:36:32.513 | INFO | "pinned" : " 29 GB" | 20260508 03:36:32.513 | INFO | } | 20260508 03:36:32.513 | INFO | Perf report for the last 5.13784 seconds: | 20260508 03:36:32.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:32.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:32.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:32.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:32.514 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 03:36:32.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:32.514 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:36:32.514 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 03:36:32.514 | INFO | BAZWRITER : 67.7%, 2.7, 92.9ms, 0 | 20260508 03:36:32.514 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:36:32.514 | INFO | Uploaded 36GiB to GPU in 2.58035s (13.9516GiB/s) | 20260508 03:36:32.514 | INFO | Downloaded 38GiB to GPU in 2.68674s (14.1436GiB/s) | 20260508 03:36:37.655 | INFO | Analyzing chunk frames = [4928000,4928512) | 20260508 03:36:37.655 | INFO | MemInfo: { | 20260508 03:36:37.655 | INFO | "gpu" : " 17 GB", | 20260508 03:36:37.655 | INFO | "huge" : "", | 20260508 03:36:37.655 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:37.655 | INFO | "malloc" : " 14 GB", | 20260508 03:36:37.655 | INFO | "pinned" : " 29 GB" | 20260508 03:36:37.655 | INFO | } | 20260508 03:36:37.655 | INFO | Perf report for the last 5.14201 seconds: | 20260508 03:36:37.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:37.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:36:37.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 03:36:37.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:37.656 | INFO | Uploaded 36GiB to GPU in 2.58438s (13.9298GiB/s) | 20260508 03:36:37.656 | INFO | Downloaded 34GiB to GPU in 2.42037s (14.0474GiB/s) | 20260508 03:36:42.794 | INFO | Analyzing chunk frames = [4928512,4929024) | 20260508 03:36:42.794 | INFO | MemInfo: { | 20260508 03:36:42.794 | INFO | "gpu" : " 17 GB", | 20260508 03:36:42.794 | INFO | "huge" : "", | 20260508 03:36:42.794 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:42.794 | INFO | "malloc" : " 14 GB", | 20260508 03:36:42.794 | INFO | "pinned" : " 29 GB" | 20260508 03:36:42.794 | INFO | } | 20260508 03:36:42.794 | INFO | Perf report for the last 5.13898 seconds: | 20260508 03:36:42.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:42.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:36:42.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:36:42.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:42.795 | INFO | Uploaded 36GiB to GPU in 2.57818s (13.9634GiB/s) | 20260508 03:36:42.795 | INFO | Downloaded 34GiB to GPU in 2.49137s (13.6471GiB/s) | 20260508 03:36:47.941 | INFO | Analyzing chunk frames = [4929024,4929536) | 20260508 03:36:47.941 | INFO | MemInfo: { | 20260508 03:36:47.941 | INFO | "gpu" : " 17 GB", | 20260508 03:36:47.941 | INFO | "huge" : "", | 20260508 03:36:47.941 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:47.941 | INFO | "malloc" : " 14 GB", | 20260508 03:36:47.941 | INFO | "pinned" : " 29 GB" | 20260508 03:36:47.941 | INFO | } | 20260508 03:36:47.941 | INFO | Perf report for the last 5.14644 seconds: | 20260508 03:36:47.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:47.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 03:36:47.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:36:47.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:47.941 | INFO | Uploaded 36GiB to GPU in 2.57727s (13.9683GiB/s) | 20260508 03:36:47.941 | INFO | Downloaded 34GiB to GPU in 2.48823s (13.6644GiB/s) | 20260508 03:36:53.076 | INFO | Analyzing chunk frames = [4929536,4930048) | 20260508 03:36:53.076 | INFO | MemInfo: { | 20260508 03:36:53.076 | INFO | "gpu" : " 17 GB", | 20260508 03:36:53.076 | INFO | "huge" : "", | 20260508 03:36:53.076 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:53.077 | INFO | "malloc" : " 14 GB", | 20260508 03:36:53.077 | INFO | "pinned" : " 29 GB" | 20260508 03:36:53.077 | INFO | } | 20260508 03:36:53.077 | INFO | Perf report for the last 5.13581 seconds: | 20260508 03:36:53.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:53.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 03:36:53.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:36:53.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:53.077 | INFO | Uploaded 36GiB to GPU in 2.40998s (14.9379GiB/s) | 20260508 03:36:53.077 | INFO | Downloaded 34GiB to GPU in 2.55923s (13.2853GiB/s) | 20260508 03:36:58.213 | INFO | Analyzing chunk frames = [4930048,4930560) | 20260508 03:36:58.213 | INFO | MemInfo: { | 20260508 03:36:58.213 | INFO | "gpu" : " 17 GB", | 20260508 03:36:58.213 | INFO | "huge" : "", | 20260508 03:36:58.213 | INFO | "huge_shared" : " 29 GB", | 20260508 03:36:58.213 | INFO | "malloc" : " 14 GB", | 20260508 03:36:58.213 | INFO | "pinned" : " 29 GB" | 20260508 03:36:58.213 | INFO | } | 20260508 03:36:58.213 | INFO | Perf report for the last 5.13662 seconds: | 20260508 03:36:58.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:36:58.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 03:36:58.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:36:58.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:36:58.214 | INFO | Uploaded 36GiB to GPU in 2.46053s (14.631GiB/s) | 20260508 03:36:58.214 | INFO | Downloaded 34GiB to GPU in 2.42038s (14.0474GiB/s) | 20260508 03:37:03.356 | INFO | Analyzing chunk frames = [4930560,4931072) | 20260508 03:37:03.356 | INFO | MemInfo: { | 20260508 03:37:03.356 | INFO | "gpu" : " 17 GB", | 20260508 03:37:03.356 | INFO | "huge" : "", | 20260508 03:37:03.356 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:03.356 | INFO | "malloc" : " 14 GB", | 20260508 03:37:03.356 | INFO | "pinned" : " 29 GB" | 20260508 03:37:03.356 | INFO | } | 20260508 03:37:03.356 | INFO | Perf report for the last 5.14265 seconds: | 20260508 03:37:03.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:03.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:37:03.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:37:03.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:03.356 | INFO | Uploaded 36GiB to GPU in 2.47089s (14.5696GiB/s) | 20260508 03:37:03.356 | INFO | Downloaded 34GiB to GPU in 2.41893s (14.0558GiB/s) | 20260508 03:37:08.494 | INFO | Analyzing chunk frames = [4931072,4931584) | 20260508 03:37:08.494 | INFO | MemInfo: { | 20260508 03:37:08.494 | INFO | "gpu" : " 17 GB", | 20260508 03:37:08.494 | INFO | "huge" : "", | 20260508 03:37:08.494 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:08.494 | INFO | "malloc" : " 14 GB", | 20260508 03:37:08.495 | INFO | "pinned" : " 29 GB" | 20260508 03:37:08.495 | INFO | } | 20260508 03:37:08.495 | INFO | Perf report for the last 5.13855 seconds: | 20260508 03:37:08.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:08.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:37:08.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:37:08.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:08.495 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:37:08.495 | INFO | Total mem usage: 16.7 GiB | 20260508 03:37:08.495 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:37:08.495 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 03:37:08.495 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:37:08.495 | INFO | High Water Breakdown: | 20260508 03:37:08.495 | INFO | 59.4% : 8.48 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:37:08.495 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:37:08.495 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:37:08.495 | INFO | ------------------------------------------ | 20260508 03:37:08.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:37:08.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:37:08.495 | INFO | Total mem usage: 26.5 GiB | 20260508 03:37:08.495 | INFO | Uploaded 36GiB to GPU in 2.46128s (14.6266GiB/s) | 20260508 03:37:08.495 | INFO | Downloaded 34GiB to GPU in 2.41564s (14.075GiB/s) | 20260508 03:37:08.495 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:37:08.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:37:08.495 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:37:08.495 | INFO | High Water Breakdown: | 20260508 03:37:08.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:37:08.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:37:08.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:37:08.495 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:37:08.496 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:37:08.496 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:37:08.496 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:37:08.496 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:37:08.496 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:37:08.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:37:08.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:37:08.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:37:08.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:37:08.496 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:37:08.496 | INFO | ------------------------------------------ | 20260508 03:37:08.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:37:08.496 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:37:08.496 | INFO | Total mem usage: 27.8 GiB | 20260508 03:37:08.496 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:37:08.496 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:37:08.496 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:37:08.496 | INFO | High Water Breakdown: | 20260508 03:37:08.496 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:37:08.496 | INFO | ------------------------------------------ | 20260508 03:37:08.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:37:08.496 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:37:08.496 | INFO | Total mem usage: 15.7 GiB | 20260508 03:37:08.496 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:37:08.496 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:37:08.496 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:37:08.496 | INFO | High Water Breakdown: | 20260508 03:37:08.496 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:37:08.496 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:37:08.496 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:37:08.496 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:37:08.496 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:37:08.496 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:37:08.496 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:37:08.496 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:37:08.496 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:37:08.496 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:37:08.496 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:37:08.496 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:37:08.496 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:37:08.496 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:37:08.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:37:08.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:37:08.496 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:37:08.496 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:37:08.496 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:37:08.496 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:37:08.496 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:37:08.496 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:37:08.496 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:37:08.496 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:37:08.496 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:37:08.496 | INFO | ------------------------------------------ | 20260508 03:37:08.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:37:08.496 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:37:08.496 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:37:08.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:37:08.496 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.42e+01 / 1.86e+00 | 20260508 03:37:08.496 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.02e-01, 1.11e-02 / 6.54e-03 | 20260508 03:37:08.496 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.47e+00, 5.89e-01 / 2.20e-01 | 20260508 03:37:08.496 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.47e+00, 2.08e-01 / 1.17e-01 | 20260508 03:37:08.496 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.58e+01, 4.02e+00 / 2.14e+00 | 20260508 03:37:08.496 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.69e+00 / 1.42e+00 | 20260508 03:37:08.496 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.64e+00, 4.18e-01 / 2.38e-01 | 20260508 03:37:08.496 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.21e+00, 2.65e-01 / 1.40e-01 | 20260508 03:37:08.497 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.56e-01, 5.10e-02 / 2.65e-02 | 20260508 03:37:08.497 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:37:08.497 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:37:08.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:37:08.497 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 24.59%, Upload -- 2.62e+01 / 0.00e+00, 8.40e+00 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 30.64%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 2.49%, LockUpload -- 2.65e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 2.46%, LockDownload -- 2.62e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 15.52%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:37:08.497 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260508 03:37:08.607 | INFO | Starting a round of Baz IO | 20260508 03:37:12.024 | INFO | Populating RTMetrics report | 20260508 03:37:12.024 | INFO | RTMetrics report populated | 20260508 03:37:12.024 | INFO | Opening tmp RTMetrics json file | 20260508 03:37:12.037 | INFO | Serializing RTMetrics to json | 20260508 03:37:12.038 | INFO | Serialization done, swapping tmp file now... | 20260508 03:37:12.039 | INFO | RTMetrics done | 20260508 03:37:12.102 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.494 seconds | 20260508 03:37:12.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.24% / 0.00% / 58.75% | 20260508 03:37:12.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.60575% / 0% / 5.5631e-05% | 20260508 03:37:12.102 | INFO | Baz IO overall: | 20260508 03:37:12.102 | INFO | BytesWritten : 2215988456151 | 20260508 03:37:12.102 | INFO | NumEvents : 1070473199028 | 20260508 03:37:13.631 | INFO | Analyzing chunk frames = [4931584,4932096) | 20260508 03:37:13.631 | INFO | MemInfo: { | 20260508 03:37:13.631 | INFO | "gpu" : " 17 GB", | 20260508 03:37:13.631 | INFO | "huge" : "", | 20260508 03:37:13.631 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:13.631 | INFO | "malloc" : " 14 GB", | 20260508 03:37:13.631 | INFO | "pinned" : " 29 GB" | 20260508 03:37:13.631 | INFO | } | 20260508 03:37:13.631 | INFO | Perf report for the last 5.13679 seconds: | 20260508 03:37:13.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:13.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:13.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:13.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:13.632 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 03:37:13.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:13.632 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:37:13.632 | INFO | PRE_HQ : 66.4%, 1.8, 62.0ms, 0 | 20260508 03:37:13.632 | INFO | BAZWRITER : 67.5%, 2.8, 98.4ms, 0 | 20260508 03:37:13.632 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:37:13.632 | INFO | Uploaded 36GiB to GPU in 2.59085s (13.8951GiB/s) | 20260508 03:37:13.632 | INFO | Downloaded 38GiB to GPU in 2.69681s (14.0907GiB/s) | 20260508 03:37:18.767 | INFO | Analyzing chunk frames = [4932096,4932608) | 20260508 03:37:18.767 | INFO | MemInfo: { | 20260508 03:37:18.767 | INFO | "gpu" : " 17 GB", | 20260508 03:37:18.767 | INFO | "huge" : "", | 20260508 03:37:18.767 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:18.767 | INFO | "malloc" : " 14 GB", | 20260508 03:37:18.767 | INFO | "pinned" : " 29 GB" | 20260508 03:37:18.768 | INFO | } | 20260508 03:37:18.768 | INFO | Perf report for the last 5.13613 seconds: | 20260508 03:37:18.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:18.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 03:37:18.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:37:18.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:18.768 | INFO | Uploaded 36GiB to GPU in 2.59278s (13.8847GiB/s) | 20260508 03:37:18.768 | INFO | Downloaded 34GiB to GPU in 2.42234s (14.036GiB/s) | 20260508 03:37:23.910 | INFO | Analyzing chunk frames = [4932608,4933120) | 20260508 03:37:23.910 | INFO | MemInfo: { | 20260508 03:37:23.910 | INFO | "gpu" : " 17 GB", | 20260508 03:37:23.910 | INFO | "huge" : "", | 20260508 03:37:23.910 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:23.910 | INFO | "malloc" : " 14 GB", | 20260508 03:37:23.910 | INFO | "pinned" : " 29 GB" | 20260508 03:37:23.910 | INFO | } | 20260508 03:37:23.910 | INFO | Perf report for the last 5.14257 seconds: | 20260508 03:37:23.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:23.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 03:37:23.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:37:23.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:23.910 | INFO | Uploaded 36GiB to GPU in 2.59135s (13.8924GiB/s) | 20260508 03:37:23.910 | INFO | Downloaded 34GiB to GPU in 2.49397s (13.6329GiB/s) | 20260508 03:37:29.053 | INFO | Analyzing chunk frames = [4933120,4933632) | 20260508 03:37:29.053 | INFO | MemInfo: { | 20260508 03:37:29.053 | INFO | "gpu" : " 17 GB", | 20260508 03:37:29.053 | INFO | "huge" : "", | 20260508 03:37:29.053 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:29.053 | INFO | "malloc" : " 14 GB", | 20260508 03:37:29.053 | INFO | "pinned" : " 29 GB" | 20260508 03:37:29.053 | INFO | } | 20260508 03:37:29.053 | INFO | Perf report for the last 5.14275 seconds: | 20260508 03:37:29.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:29.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 03:37:29.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:37:29.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:29.053 | INFO | Uploaded 36GiB to GPU in 2.56188s (14.0522GiB/s) | 20260508 03:37:29.053 | INFO | Downloaded 34GiB to GPU in 2.49161s (13.6458GiB/s) | 20260508 03:37:34.189 | INFO | Analyzing chunk frames = [4933632,4934144) | 20260508 03:37:34.189 | INFO | MemInfo: { | 20260508 03:37:34.189 | INFO | "gpu" : " 17 GB", | 20260508 03:37:34.189 | INFO | "huge" : "", | 20260508 03:37:34.189 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:34.189 | INFO | "malloc" : " 14 GB", | 20260508 03:37:34.189 | INFO | "pinned" : " 29 GB" | 20260508 03:37:34.189 | INFO | } | 20260508 03:37:34.189 | INFO | Perf report for the last 5.13604 seconds: | 20260508 03:37:34.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:34.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.189 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:37:34.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.189 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:37:34.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:34.190 | INFO | Uploaded 36GiB to GPU in 2.40656s (14.9591GiB/s) | 20260508 03:37:34.190 | INFO | Downloaded 34GiB to GPU in 2.55435s (13.3106GiB/s) | 20260508 03:37:39.324 | INFO | Analyzing chunk frames = [4934144,4934656) | 20260508 03:37:39.324 | INFO | MemInfo: { | 20260508 03:37:39.324 | INFO | "gpu" : " 17 GB", | 20260508 03:37:39.324 | INFO | "huge" : "", | 20260508 03:37:39.324 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:39.324 | INFO | "malloc" : " 14 GB", | 20260508 03:37:39.324 | INFO | "pinned" : " 29 GB" | 20260508 03:37:39.325 | INFO | } | 20260508 03:37:39.325 | INFO | Perf report for the last 5.13574 seconds: | 20260508 03:37:39.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:39.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 03:37:39.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:37:39.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:39.325 | INFO | Uploaded 36GiB to GPU in 2.4457s (14.7197GiB/s) | 20260508 03:37:39.325 | INFO | Downloaded 34GiB to GPU in 2.40054s (14.1635GiB/s) | 20260508 03:37:44.462 | INFO | Analyzing chunk frames = [4934656,4935168) | 20260508 03:37:44.462 | INFO | MemInfo: { | 20260508 03:37:44.462 | INFO | "gpu" : " 17 GB", | 20260508 03:37:44.462 | INFO | "huge" : "", | 20260508 03:37:44.462 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:44.462 | INFO | "malloc" : " 14 GB", | 20260508 03:37:44.462 | INFO | "pinned" : " 29 GB" | 20260508 03:37:44.462 | INFO | } | 20260508 03:37:44.462 | INFO | Perf report for the last 5.13751 seconds: | 20260508 03:37:44.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:44.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.462 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:37:44.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.462 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 03:37:44.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:44.463 | INFO | Uploaded 36GiB to GPU in 2.45978s (14.6355GiB/s) | 20260508 03:37:44.463 | INFO | Downloaded 34GiB to GPU in 2.4158s (14.074GiB/s) | 20260508 03:37:49.605 | INFO | Analyzing chunk frames = [4935168,4935680) | 20260508 03:37:49.605 | INFO | MemInfo: { | 20260508 03:37:49.605 | INFO | "gpu" : " 17 GB", | 20260508 03:37:49.605 | INFO | "huge" : "", | 20260508 03:37:49.605 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:49.605 | INFO | "malloc" : " 14 GB", | 20260508 03:37:49.605 | INFO | "pinned" : " 29 GB" | 20260508 03:37:49.605 | INFO | } | 20260508 03:37:49.606 | INFO | Perf report for the last 5.14328 seconds: | 20260508 03:37:49.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:49.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 03:37:49.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:37:49.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:49.606 | INFO | Uploaded 36GiB to GPU in 2.55423s (14.0943GiB/s) | 20260508 03:37:49.606 | INFO | Downloaded 34GiB to GPU in 2.45166s (13.8682GiB/s) | 20260508 03:37:49.703 | INFO | Starting a round of Baz IO | 20260508 03:37:53.138 | INFO | Populating RTMetrics report | 20260508 03:37:53.138 | INFO | RTMetrics report populated | 20260508 03:37:53.138 | INFO | Opening tmp RTMetrics json file | 20260508 03:37:53.150 | INFO | Serializing RTMetrics to json | 20260508 03:37:53.152 | INFO | Serialization done, swapping tmp file now... | 20260508 03:37:53.152 | INFO | RTMetrics done | 20260508 03:37:53.210 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.506 seconds | 20260508 03:37:53.210 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.24% / 0.00% / 58.76% | 20260508 03:37:53.210 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.604598% / 0% / 4.76837e-05% | 20260508 03:37:53.210 | INFO | Baz IO overall: | 20260508 03:37:53.210 | INFO | BytesWritten : 2217721145590 | 20260508 03:37:53.210 | INFO | NumEvents : 1071302774888 | 20260508 03:37:54.747 | INFO | Analyzing chunk frames = [4935680,4936192) | 20260508 03:37:54.747 | INFO | MemInfo: { | 20260508 03:37:54.747 | INFO | "gpu" : " 17 GB", | 20260508 03:37:54.747 | INFO | "huge" : "", | 20260508 03:37:54.747 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:54.748 | INFO | "malloc" : " 14 GB", | 20260508 03:37:54.748 | INFO | "pinned" : " 29 GB" | 20260508 03:37:54.748 | INFO | } | 20260508 03:37:54.748 | INFO | Perf report for the last 5.14214 seconds: | 20260508 03:37:54.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:54.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:54.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:54.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:54.748 | INFO | ANALYSIS : 68.0%, 3.0, 106.3ms, 0 | 20260508 03:37:54.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:54.748 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 03:37:54.748 | INFO | PRE_HQ : 65.6%, 1.6, 55.7ms, 0 | 20260508 03:37:54.748 | INFO | BAZWRITER : 67.5%, 2.7, 92.6ms, 0 | 20260508 03:37:54.748 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:37:54.748 | INFO | Uploaded 36GiB to GPU in 2.59093s (13.8946GiB/s) | 20260508 03:37:54.749 | INFO | Downloaded 38GiB to GPU in 2.69474s (14.1016GiB/s) | 20260508 03:37:59.889 | INFO | Analyzing chunk frames = [4936192,4936704) | 20260508 03:37:59.889 | INFO | MemInfo: { | 20260508 03:37:59.889 | INFO | "gpu" : " 17 GB", | 20260508 03:37:59.889 | INFO | "huge" : "", | 20260508 03:37:59.889 | INFO | "huge_shared" : " 29 GB", | 20260508 03:37:59.889 | INFO | "malloc" : " 14 GB", | 20260508 03:37:59.889 | INFO | "pinned" : " 29 GB" | 20260508 03:37:59.889 | INFO | } | 20260508 03:37:59.889 | INFO | Perf report for the last 5.14115 seconds: | 20260508 03:37:59.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:37:59.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 03:37:59.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:37:59.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:37:59.889 | INFO | Uploaded 36GiB to GPU in 2.59174s (13.8903GiB/s) | 20260508 03:37:59.889 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0225GiB/s) | 20260508 03:38:05.027 | INFO | Analyzing chunk frames = [4936704,4937216) | 20260508 03:38:05.027 | INFO | MemInfo: { | 20260508 03:38:05.027 | INFO | "gpu" : " 17 GB", | 20260508 03:38:05.027 | INFO | "huge" : "", | 20260508 03:38:05.027 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:05.027 | INFO | "malloc" : " 14 GB", | 20260508 03:38:05.027 | INFO | "pinned" : " 29 GB" | 20260508 03:38:05.027 | INFO | } | 20260508 03:38:05.027 | INFO | Perf report for the last 5.1381 seconds: | 20260508 03:38:05.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:05.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 03:38:05.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:38:05.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:05.027 | INFO | Uploaded 36GiB to GPU in 2.53778s (14.1857GiB/s) | 20260508 03:38:05.027 | INFO | Downloaded 34GiB to GPU in 2.40086s (14.1616GiB/s) | 20260508 03:38:10.170 | INFO | Analyzing chunk frames = [4937216,4937728) | 20260508 03:38:10.170 | INFO | MemInfo: { | 20260508 03:38:10.170 | INFO | "gpu" : " 17 GB", | 20260508 03:38:10.170 | INFO | "huge" : "", | 20260508 03:38:10.170 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:10.170 | INFO | "malloc" : " 14 GB", | 20260508 03:38:10.170 | INFO | "pinned" : " 29 GB" | 20260508 03:38:10.170 | INFO | } | 20260508 03:38:10.171 | INFO | Perf report for the last 5.14353 seconds: | 20260508 03:38:10.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:10.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | ANALYSIS : 67.2%, 2.9, 104.0ms, 0 | 20260508 03:38:10.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:38:10.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:10.171 | INFO | Uploaded 36GiB to GPU in 2.59451s (13.8755GiB/s) | 20260508 03:38:10.171 | INFO | Downloaded 34GiB to GPU in 2.42731s (14.0073GiB/s) | 20260508 03:38:15.308 | INFO | Analyzing chunk frames = [4937728,4938240) | 20260508 03:38:15.308 | INFO | MemInfo: { | 20260508 03:38:15.308 | INFO | "gpu" : " 17 GB", | 20260508 03:38:15.308 | INFO | "huge" : "", | 20260508 03:38:15.308 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:15.308 | INFO | "malloc" : " 14 GB", | 20260508 03:38:15.308 | INFO | "pinned" : " 29 GB" | 20260508 03:38:15.308 | INFO | } | 20260508 03:38:15.308 | INFO | Perf report for the last 5.13794 seconds: | 20260508 03:38:15.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:15.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 03:38:15.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:38:15.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:15.309 | INFO | Uploaded 36GiB to GPU in 2.4001s (14.9994GiB/s) | 20260508 03:38:15.309 | INFO | Downloaded 34GiB to GPU in 2.55699s (13.2969GiB/s) | 20260508 03:38:20.446 | INFO | Analyzing chunk frames = [4938240,4938752) | 20260508 03:38:20.446 | INFO | MemInfo: { | 20260508 03:38:20.446 | INFO | "gpu" : " 17 GB", | 20260508 03:38:20.446 | INFO | "huge" : "", | 20260508 03:38:20.446 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:20.446 | INFO | "malloc" : " 14 GB", | 20260508 03:38:20.446 | INFO | "pinned" : " 29 GB" | 20260508 03:38:20.446 | INFO | } | 20260508 03:38:20.446 | INFO | Perf report for the last 5.13783 seconds: | 20260508 03:38:20.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:20.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.446 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 03:38:20.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.446 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 03:38:20.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:20.447 | INFO | Uploaded 36GiB to GPU in 2.44797s (14.7061GiB/s) | 20260508 03:38:20.447 | INFO | Downloaded 34GiB to GPU in 2.41026s (14.1064GiB/s) | 20260508 03:38:25.581 | INFO | Analyzing chunk frames = [4938752,4939264) | 20260508 03:38:25.581 | INFO | MemInfo: { | 20260508 03:38:25.581 | INFO | "gpu" : " 17 GB", | 20260508 03:38:25.581 | INFO | "huge" : "", | 20260508 03:38:25.581 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:25.581 | INFO | "malloc" : " 14 GB", | 20260508 03:38:25.581 | INFO | "pinned" : " 29 GB" | 20260508 03:38:25.581 | INFO | } | 20260508 03:38:25.581 | INFO | Perf report for the last 5.13513 seconds: | 20260508 03:38:25.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:25.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.582 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:38:25.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.582 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:38:25.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:25.582 | INFO | Uploaded 36GiB to GPU in 2.4544s (14.6676GiB/s) | 20260508 03:38:25.582 | INFO | Downloaded 34GiB to GPU in 2.40939s (14.1114GiB/s) | 20260508 03:38:30.716 | INFO | Analyzing chunk frames = [4939264,4939776) | 20260508 03:38:30.716 | INFO | MemInfo: { | 20260508 03:38:30.716 | INFO | "gpu" : " 17 GB", | 20260508 03:38:30.716 | INFO | "huge" : "", | 20260508 03:38:30.716 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:30.716 | INFO | "malloc" : " 14 GB", | 20260508 03:38:30.716 | INFO | "pinned" : " 29 GB" | 20260508 03:38:30.716 | INFO | } | 20260508 03:38:30.716 | INFO | Perf report for the last 5.13454 seconds: | 20260508 03:38:30.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:30.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.716 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:38:30.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.716 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 03:38:30.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:30.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:38:30.717 | INFO | Total mem usage: 16.7 GiB | 20260508 03:38:30.717 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:38:30.717 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:38:30.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:38:30.717 | INFO | High Water Breakdown: | 20260508 03:38:30.717 | INFO | 59.6% : 8.56 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:38:30.717 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:38:30.717 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:38:30.717 | INFO | ------------------------------------------ | 20260508 03:38:30.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:38:30.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:38:30.717 | INFO | Total mem usage: 26.5 GiB | 20260508 03:38:30.717 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:38:30.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:38:30.717 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:38:30.717 | INFO | High Water Breakdown: | 20260508 03:38:30.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:38:30.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:38:30.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:38:30.717 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:38:30.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:38:30.717 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:38:30.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:38:30.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:38:30.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:38:30.717 | INFO | Uploaded 36GiB to GPU in 2.56659s (14.0264GiB/s) | 20260508 03:38:30.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:38:30.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:38:30.717 | INFO | Downloaded 34GiB to GPU in 2.45541s (13.847GiB/s) | 20260508 03:38:30.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:38:30.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:38:30.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:38:30.717 | INFO | ------------------------------------------ | 20260508 03:38:30.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:38:30.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:38:30.717 | INFO | Total mem usage: 27.8 GiB | 20260508 03:38:30.717 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:38:30.717 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:38:30.717 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:38:30.717 | INFO | High Water Breakdown: | 20260508 03:38:30.717 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:38:30.717 | INFO | ------------------------------------------ | 20260508 03:38:30.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:38:30.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:38:30.717 | INFO | Total mem usage: 15.7 GiB | 20260508 03:38:30.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:38:30.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:38:30.717 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:38:30.717 | INFO | High Water Breakdown: | 20260508 03:38:30.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:38:30.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:38:30.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:38:30.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:38:30.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:38:30.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:38:30.717 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:38:30.717 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:38:30.717 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:38:30.717 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:38:30.717 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:38:30.717 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:38:30.717 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:38:30.717 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:38:30.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:38:30.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:38:30.717 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:38:30.717 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:38:30.717 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:38:30.717 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:38:30.717 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:38:30.717 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:38:30.717 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:38:30.717 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:38:30.717 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:38:30.717 | INFO | ------------------------------------------ | 20260508 03:38:30.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:38:30.718 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:38:30.718 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:38:30.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:38:30.718 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.65e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.02e-01, 1.19e-02 / 6.49e-03 | 20260508 03:38:30.718 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.54e+00, 5.87e-01 / 1.47e-01 | 20260508 03:38:30.718 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.45e+00, 2.05e-01 / 1.12e-01 | 20260508 03:38:30.718 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.57e+01, 3.88e+00 / 2.01e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.68e+00 / 1.39e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.41e+00 / 2.64e+00, 4.24e-01 / 2.37e-01 | 20260508 03:38:30.718 | INFO | PerfReport: 3.26%, Metrics -- 1.06e+00 / 1.24e+00, 2.68e-01 / 1.68e-01 | 20260508 03:38:30.718 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.58e-01, 5.13e-02 / 2.48e-02 | 20260508 03:38:30.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:38:30.718 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:38:30.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:38:30.718 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.58e+01 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 24.64%, Upload -- 2.63e+01 / 0.00e+00, 8.39e+00 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 24.23%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 30.60%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 2.56%, LockUpload -- 2.73e+00 / 0.00e+00, 5.81e+00 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 2.44%, LockDownload -- 2.60e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 15.52%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 03:38:30.718 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260508 03:38:30.813 | INFO | Starting a round of Baz IO | 20260508 03:38:34.251 | INFO | Populating RTMetrics report | 20260508 03:38:34.251 | INFO | RTMetrics report populated | 20260508 03:38:34.251 | INFO | Opening tmp RTMetrics json file | 20260508 03:38:34.264 | INFO | Serializing RTMetrics to json | 20260508 03:38:34.265 | INFO | Serialization done, swapping tmp file now... | 20260508 03:38:34.266 | INFO | RTMetrics done | 20260508 03:38:34.330 | INFO | Finished a round of Baz IO: Wrote 1.612 GiB in 3.516 seconds | 20260508 03:38:34.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.24% / 0.00% / 58.76% | 20260508 03:38:34.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.602861% / 0% / 6.35783e-05% | 20260508 03:38:34.330 | INFO | Baz IO overall: | 20260508 03:38:34.330 | INFO | BytesWritten : 2219452284926 | 20260508 03:38:34.330 | INFO | NumEvents : 1072131104215 | 20260508 03:38:35.861 | INFO | Analyzing chunk frames = [4939776,4940288) | 20260508 03:38:35.861 | INFO | MemInfo: { | 20260508 03:38:35.861 | INFO | "gpu" : " 17 GB", | 20260508 03:38:35.861 | INFO | "huge" : "", | 20260508 03:38:35.861 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:35.861 | INFO | "malloc" : " 14 GB", | 20260508 03:38:35.861 | INFO | "pinned" : " 29 GB" | 20260508 03:38:35.861 | INFO | } | 20260508 03:38:35.861 | INFO | Perf report for the last 5.145 seconds: | 20260508 03:38:35.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:35.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:35.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:35.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:35.861 | INFO | ANALYSIS : 68.0%, 3.0, 106.4ms, 0 | 20260508 03:38:35.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:35.861 | INFO | RT_METRICS : 39.7%, 1.0, 20.9ms, 0 | 20260508 03:38:35.861 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 03:38:35.861 | INFO | BAZWRITER : 67.7%, 2.6, 92.0ms, 0 | 20260508 03:38:35.861 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:38:35.861 | INFO | Uploaded 36GiB to GPU in 2.59387s (13.8789GiB/s) | 20260508 03:38:35.861 | INFO | Downloaded 38GiB to GPU in 2.68863s (14.1336GiB/s) | 20260508 03:38:41.004 | INFO | Analyzing chunk frames = [4940288,4940800) | 20260508 03:38:41.004 | INFO | MemInfo: { | 20260508 03:38:41.004 | INFO | "gpu" : " 17 GB", | 20260508 03:38:41.004 | INFO | "huge" : "", | 20260508 03:38:41.004 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:41.004 | INFO | "malloc" : " 14 GB", | 20260508 03:38:41.004 | INFO | "pinned" : " 29 GB" | 20260508 03:38:41.004 | INFO | } | 20260508 03:38:41.005 | INFO | Perf report for the last 5.1435 seconds: | 20260508 03:38:41.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:41.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 03:38:41.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:38:41.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:41.005 | INFO | Uploaded 36GiB to GPU in 2.59364s (13.8801GiB/s) | 20260508 03:38:41.005 | INFO | Downloaded 34GiB to GPU in 2.42733s (14.0071GiB/s) | 20260508 03:38:46.145 | INFO | Analyzing chunk frames = [4940800,4941312) | 20260508 03:38:46.145 | INFO | MemInfo: { | 20260508 03:38:46.145 | INFO | "gpu" : " 17 GB", | 20260508 03:38:46.145 | INFO | "huge" : "", | 20260508 03:38:46.145 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:46.145 | INFO | "malloc" : " 14 GB", | 20260508 03:38:46.145 | INFO | "pinned" : " 29 GB" | 20260508 03:38:46.146 | INFO | } | 20260508 03:38:46.146 | INFO | Perf report for the last 5.1412 seconds: | 20260508 03:38:46.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:46.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 03:38:46.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:38:46.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:46.146 | INFO | Uploaded 36GiB to GPU in 2.55842s (14.0712GiB/s) | 20260508 03:38:46.146 | INFO | Downloaded 34GiB to GPU in 2.42139s (14.0415GiB/s) | 20260508 03:38:51.273 | INFO | Analyzing chunk frames = [4941312,4941824) | 20260508 03:38:51.273 | INFO | MemInfo: { | 20260508 03:38:51.273 | INFO | "gpu" : " 17 GB", | 20260508 03:38:51.273 | INFO | "huge" : "", | 20260508 03:38:51.273 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:51.273 | INFO | "malloc" : " 14 GB", | 20260508 03:38:51.273 | INFO | "pinned" : " 29 GB" | 20260508 03:38:51.273 | INFO | } | 20260508 03:38:51.273 | INFO | Perf report for the last 5.12766 seconds: | 20260508 03:38:51.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:51.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.273 | INFO | ANALYSIS : 67.4%, 2.9, 104.0ms, 0 | 20260508 03:38:51.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.274 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:38:51.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:51.274 | INFO | Uploaded 36GiB to GPU in 2.59774s (13.8582GiB/s) | 20260508 03:38:51.274 | INFO | Downloaded 34GiB to GPU in 2.41681s (14.0681GiB/s) | 20260508 03:38:56.420 | INFO | Analyzing chunk frames = [4941824,4942336) | 20260508 03:38:56.420 | INFO | MemInfo: { | 20260508 03:38:56.420 | INFO | "gpu" : " 17 GB", | 20260508 03:38:56.420 | INFO | "huge" : "", | 20260508 03:38:56.420 | INFO | "huge_shared" : " 29 GB", | 20260508 03:38:56.420 | INFO | "malloc" : " 14 GB", | 20260508 03:38:56.420 | INFO | "pinned" : " 29 GB" | 20260508 03:38:56.420 | INFO | } | 20260508 03:38:56.420 | INFO | Perf report for the last 5.14668 seconds: | 20260508 03:38:56.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:38:56.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 03:38:56.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:38:56.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:38:56.420 | INFO | Uploaded 36GiB to GPU in 2.46973s (14.5765GiB/s) | 20260508 03:38:56.420 | INFO | Downloaded 34GiB to GPU in 2.56688s (13.2457GiB/s) | 20260508 03:39:01.558 | INFO | Analyzing chunk frames = [4942336,4942848) | 20260508 03:39:01.558 | INFO | MemInfo: { | 20260508 03:39:01.558 | INFO | "gpu" : " 17 GB", | 20260508 03:39:01.558 | INFO | "huge" : "", | 20260508 03:39:01.558 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:01.558 | INFO | "malloc" : " 14 GB", | 20260508 03:39:01.558 | INFO | "pinned" : " 29 GB" | 20260508 03:39:01.559 | INFO | } | 20260508 03:39:01.559 | INFO | Perf report for the last 5.13851 seconds: | 20260508 03:39:01.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:01.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:39:01.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:39:01.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:01.559 | INFO | Uploaded 36GiB to GPU in 2.42489s (14.8461GiB/s) | 20260508 03:39:01.559 | INFO | Downloaded 34GiB to GPU in 2.42329s (14.0305GiB/s) | 20260508 03:39:06.702 | INFO | Analyzing chunk frames = [4942848,4943360) | 20260508 03:39:06.702 | INFO | MemInfo: { | 20260508 03:39:06.702 | INFO | "gpu" : " 17 GB", | 20260508 03:39:06.702 | INFO | "huge" : "", | 20260508 03:39:06.702 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:06.703 | INFO | "malloc" : " 14 GB", | 20260508 03:39:06.703 | INFO | "pinned" : " 29 GB" | 20260508 03:39:06.703 | INFO | } | 20260508 03:39:06.703 | INFO | Perf report for the last 5.14411 seconds: | 20260508 03:39:06.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:06.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | ANALYSIS : 66.2%, 3.0, 103.7ms, 0 | 20260508 03:39:06.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:39:06.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:06.703 | INFO | Uploaded 36GiB to GPU in 2.44601s (14.7178GiB/s) | 20260508 03:39:06.703 | INFO | Downloaded 34GiB to GPU in 2.42264s (14.0343GiB/s) | 20260508 03:39:11.838 | INFO | Analyzing chunk frames = [4943360,4943872) | 20260508 03:39:11.838 | INFO | MemInfo: { | 20260508 03:39:11.838 | INFO | "gpu" : " 17 GB", | 20260508 03:39:11.838 | INFO | "huge" : "", | 20260508 03:39:11.838 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:11.838 | INFO | "malloc" : " 14 GB", | 20260508 03:39:11.838 | INFO | "pinned" : " 29 GB" | 20260508 03:39:11.838 | INFO | } | 20260508 03:39:11.838 | INFO | Perf report for the last 5.13533 seconds: | 20260508 03:39:11.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:11.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:39:11.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 03:39:11.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:11.838 | INFO | Uploaded 36GiB to GPU in 2.44917s (14.6988GiB/s) | 20260508 03:39:11.838 | INFO | Downloaded 34GiB to GPU in 2.43011s (13.9911GiB/s) | 20260508 03:39:11.936 | INFO | Starting a round of Baz IO | 20260508 03:39:15.368 | INFO | Populating RTMetrics report | 20260508 03:39:15.368 | INFO | RTMetrics report populated | 20260508 03:39:15.368 | INFO | Opening tmp RTMetrics json file | 20260508 03:39:15.380 | INFO | Serializing RTMetrics to json | 20260508 03:39:15.382 | INFO | Serialization done, swapping tmp file now... | 20260508 03:39:15.382 | INFO | RTMetrics done | 20260508 03:39:15.437 | INFO | Finished a round of Baz IO: Wrote 1.612 GiB in 3.501 seconds | 20260508 03:39:15.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.24% / 0.00% / 58.76% | 20260508 03:39:15.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.602555% / 7.94729e-06% / 5.5631e-05% | 20260508 03:39:15.438 | INFO | Baz IO overall: | 20260508 03:39:15.438 | INFO | BytesWritten : 2221183576899 | 20260508 03:39:15.438 | INFO | NumEvents : 1072959645992 | 20260508 03:39:16.977 | INFO | Analyzing chunk frames = [4943872,4944384) | 20260508 03:39:16.977 | INFO | MemInfo: { | 20260508 03:39:16.977 | INFO | "gpu" : " 17 GB", | 20260508 03:39:16.977 | INFO | "huge" : "", | 20260508 03:39:16.977 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:16.977 | INFO | "malloc" : " 14 GB", | 20260508 03:39:16.977 | INFO | "pinned" : " 29 GB" | 20260508 03:39:16.977 | INFO | } | 20260508 03:39:16.977 | INFO | Perf report for the last 5.13905 seconds: | 20260508 03:39:16.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:16.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:16.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:16.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:16.977 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 03:39:16.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:16.977 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 03:39:16.977 | INFO | PRE_HQ : 65.4%, 1.7, 56.3ms, 0 | 20260508 03:39:16.977 | INFO | BAZWRITER : 67.7%, 2.7, 93.9ms, 0 | 20260508 03:39:16.977 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:39:16.978 | INFO | Uploaded 36GiB to GPU in 2.56224s (14.0502GiB/s) | 20260508 03:39:16.978 | INFO | Downloaded 38GiB to GPU in 2.68578s (14.1486GiB/s) | 20260508 03:39:22.118 | INFO | Analyzing chunk frames = [4944384,4944896) | 20260508 03:39:22.118 | INFO | MemInfo: { | 20260508 03:39:22.118 | INFO | "gpu" : " 17 GB", | 20260508 03:39:22.118 | INFO | "huge" : "", | 20260508 03:39:22.118 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:22.118 | INFO | "malloc" : " 14 GB", | 20260508 03:39:22.118 | INFO | "pinned" : " 29 GB" | 20260508 03:39:22.118 | INFO | } | 20260508 03:39:22.119 | INFO | Perf report for the last 5.14135 seconds: | 20260508 03:39:22.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:22.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 03:39:22.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:39:22.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:22.119 | INFO | Uploaded 36GiB to GPU in 2.5704s (14.0056GiB/s) | 20260508 03:39:22.119 | INFO | Downloaded 34GiB to GPU in 2.42258s (14.0346GiB/s) | 20260508 03:39:27.259 | INFO | Analyzing chunk frames = [4944896,4945408) | 20260508 03:39:27.259 | INFO | MemInfo: { | 20260508 03:39:27.259 | INFO | "gpu" : " 17 GB", | 20260508 03:39:27.259 | INFO | "huge" : "", | 20260508 03:39:27.259 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:27.259 | INFO | "malloc" : " 14 GB", | 20260508 03:39:27.259 | INFO | "pinned" : " 29 GB" | 20260508 03:39:27.259 | INFO | } | 20260508 03:39:27.259 | INFO | Perf report for the last 5.14064 seconds: | 20260508 03:39:27.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:27.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.259 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 03:39:27.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.259 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:39:27.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:27.260 | INFO | Uploaded 36GiB to GPU in 2.56627s (14.0281GiB/s) | 20260508 03:39:27.260 | INFO | Downloaded 34GiB to GPU in 2.50191s (13.5896GiB/s) | 20260508 03:39:32.395 | INFO | Analyzing chunk frames = [4945408,4945920) | 20260508 03:39:32.395 | INFO | MemInfo: { | 20260508 03:39:32.395 | INFO | "gpu" : " 17 GB", | 20260508 03:39:32.395 | INFO | "huge" : "", | 20260508 03:39:32.395 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:32.395 | INFO | "malloc" : " 14 GB", | 20260508 03:39:32.395 | INFO | "pinned" : " 29 GB" | 20260508 03:39:32.395 | INFO | } | 20260508 03:39:32.395 | INFO | Perf report for the last 5.13573 seconds: | 20260508 03:39:32.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:32.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 03:39:32.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:39:32.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:32.395 | INFO | Uploaded 36GiB to GPU in 2.57791s (13.9648GiB/s) | 20260508 03:39:32.395 | INFO | Downloaded 34GiB to GPU in 2.43401s (13.9687GiB/s) | 20260508 03:39:37.527 | INFO | Analyzing chunk frames = [4945920,4946432) | 20260508 03:39:37.527 | INFO | MemInfo: { | 20260508 03:39:37.527 | INFO | "gpu" : " 17 GB", | 20260508 03:39:37.527 | INFO | "huge" : "", | 20260508 03:39:37.527 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:37.527 | INFO | "malloc" : " 14 GB", | 20260508 03:39:37.527 | INFO | "pinned" : " 29 GB" | 20260508 03:39:37.527 | INFO | } | 20260508 03:39:37.527 | INFO | Perf report for the last 5.13201 seconds: | 20260508 03:39:37.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:37.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 03:39:37.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | PRE_HQ : 25.4%, 1.0, 13.1ms, 0 | 20260508 03:39:37.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:37.527 | INFO | Uploaded 36GiB to GPU in 2.42351s (14.8545GiB/s) | 20260508 03:39:37.528 | INFO | Downloaded 34GiB to GPU in 2.55819s (13.2907GiB/s) | 20260508 03:39:42.673 | INFO | Analyzing chunk frames = [4946432,4946944) | 20260508 03:39:42.673 | INFO | MemInfo: { | 20260508 03:39:42.673 | INFO | "gpu" : " 17 GB", | 20260508 03:39:42.673 | INFO | "huge" : "", | 20260508 03:39:42.673 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:42.673 | INFO | "malloc" : " 14 GB", | 20260508 03:39:42.673 | INFO | "pinned" : " 29 GB" | 20260508 03:39:42.673 | INFO | } | 20260508 03:39:42.673 | INFO | Perf report for the last 5.14587 seconds: | 20260508 03:39:42.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:42.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:39:42.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:39:42.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:42.673 | INFO | Uploaded 36GiB to GPU in 2.43475s (14.7859GiB/s) | 20260508 03:39:42.674 | INFO | Downloaded 34GiB to GPU in 2.41842s (14.0588GiB/s) | 20260508 03:39:47.817 | INFO | Analyzing chunk frames = [4946944,4947456) | 20260508 03:39:47.817 | INFO | MemInfo: { | 20260508 03:39:47.817 | INFO | "gpu" : " 17 GB", | 20260508 03:39:47.817 | INFO | "huge" : "", | 20260508 03:39:47.817 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:47.817 | INFO | "malloc" : " 14 GB", | 20260508 03:39:47.817 | INFO | "pinned" : " 29 GB" | 20260508 03:39:47.817 | INFO | } | 20260508 03:39:47.817 | INFO | Perf report for the last 5.14435 seconds: | 20260508 03:39:47.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:47.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.817 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 03:39:47.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.817 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:39:47.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:47.818 | INFO | Uploaded 36GiB to GPU in 2.48272s (14.5002GiB/s) | 20260508 03:39:47.818 | INFO | Downloaded 34GiB to GPU in 2.43707s (13.9512GiB/s) | 20260508 03:39:52.954 | INFO | Analyzing chunk frames = [4947456,4947968) | 20260508 03:39:52.954 | INFO | MemInfo: { | 20260508 03:39:52.954 | INFO | "gpu" : " 17 GB", | 20260508 03:39:52.954 | INFO | "huge" : "", | 20260508 03:39:52.954 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:52.954 | INFO | "malloc" : " 14 GB", | 20260508 03:39:52.954 | INFO | "pinned" : " 29 GB" | 20260508 03:39:52.954 | INFO | } | 20260508 03:39:52.954 | INFO | Perf report for the last 5.13666 seconds: | 20260508 03:39:52.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:52.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:39:52.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:39:52.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:52.954 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:39:52.954 | INFO | Total mem usage: 16.7 GiB | 20260508 03:39:52.954 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:39:52.954 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 03:39:52.954 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:39:52.954 | INFO | High Water Breakdown: | 20260508 03:39:52.954 | INFO | 59.4% : 8.48 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:39:52.954 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:39:52.954 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:39:52.954 | INFO | Uploaded 36GiB to GPU in 2.47233s (14.5612GiB/s) | 20260508 03:39:52.955 | INFO | Downloaded 34GiB to GPU in 2.42353s (14.0291GiB/s) | 20260508 03:39:52.955 | INFO | ------------------------------------------ | 20260508 03:39:52.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:39:52.955 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:39:52.955 | INFO | Total mem usage: 26.5 GiB | 20260508 03:39:52.955 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:39:52.955 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:39:52.955 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:39:52.955 | INFO | High Water Breakdown: | 20260508 03:39:52.955 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:39:52.955 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:39:52.955 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:39:52.955 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:39:52.955 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:39:52.955 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:39:52.955 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:39:52.955 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:39:52.955 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:39:52.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:39:52.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:39:52.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:39:52.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:39:52.955 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:39:52.955 | INFO | ------------------------------------------ | 20260508 03:39:52.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:39:52.955 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:39:52.955 | INFO | Total mem usage: 27.8 GiB | 20260508 03:39:52.955 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:39:52.955 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:39:52.955 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:39:52.955 | INFO | High Water Breakdown: | 20260508 03:39:52.955 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:39:52.955 | INFO | ------------------------------------------ | 20260508 03:39:52.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:39:52.955 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:39:52.955 | INFO | Total mem usage: 15.7 GiB | 20260508 03:39:52.955 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:39:52.955 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:39:52.955 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:39:52.955 | INFO | High Water Breakdown: | 20260508 03:39:52.955 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:39:52.955 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:39:52.955 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:39:52.955 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:39:52.955 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:39:52.955 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:39:52.955 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:39:52.955 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:39:52.955 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:39:52.955 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:39:52.955 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:39:52.955 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:39:52.955 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:39:52.955 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:39:52.955 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:39:52.955 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:39:52.955 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:39:52.955 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:39:52.955 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:39:52.955 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:39:52.955 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:39:52.955 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:39:52.955 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:39:52.955 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:39:52.955 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:39:52.955 | INFO | ------------------------------------------ | 20260508 03:39:52.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:39:52.955 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:39:52.955 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:39:52.955 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:39:52.955 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.42e+01 / 1.66e+00 | 20260508 03:39:52.955 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.28e-02 / 7.09e-03 | 20260508 03:39:52.956 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.48e+00, 5.90e-01 / 1.98e-01 | 20260508 03:39:52.956 | INFO | PerfReport: 3.34%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.02e-01 | 20260508 03:39:52.956 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.54e+01, 3.79e+00 / 1.94e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.80e+00 / 1.57e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.65e+00, 4.20e-01 / 2.07e-01 | 20260508 03:39:52.956 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.25e+00, 2.65e-01 / 1.52e-01 | 20260508 03:39:52.956 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.57e-01, 5.11e-02 / 2.31e-02 | 20260508 03:39:52.956 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:39:52.956 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:39:52.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:39:52.956 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 24.58%, Upload -- 2.62e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 24.24%, Download -- 2.58e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 30.75%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 2.34%, LockUpload -- 2.49e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 2.49%, LockDownload -- 2.65e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:39:52.956 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 03:39:53.051 | INFO | Starting a round of Baz IO | 20260508 03:39:56.489 | INFO | Populating RTMetrics report | 20260508 03:39:56.490 | INFO | RTMetrics report populated | 20260508 03:39:56.490 | INFO | Opening tmp RTMetrics json file | 20260508 03:39:56.504 | INFO | Serializing RTMetrics to json | 20260508 03:39:56.505 | INFO | Serialization done, swapping tmp file now... | 20260508 03:39:56.507 | INFO | RTMetrics done | 20260508 03:39:56.566 | INFO | Finished a round of Baz IO: Wrote 1.613 GiB in 3.514 seconds | 20260508 03:39:56.566 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.23% / 0.00% / 58.76% | 20260508 03:39:56.566 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.603175% / 7.94729e-06% / 7.15256e-05% | 20260508 03:39:56.566 | INFO | Baz IO overall: | 20260508 03:39:56.566 | INFO | BytesWritten : 2222915757152 | 20260508 03:39:56.566 | INFO | NumEvents : 1073789050306 | 20260508 03:39:58.089 | INFO | Analyzing chunk frames = [4947968,4948480) | 20260508 03:39:58.089 | INFO | MemInfo: { | 20260508 03:39:58.089 | INFO | "gpu" : " 17 GB", | 20260508 03:39:58.089 | INFO | "huge" : "", | 20260508 03:39:58.089 | INFO | "huge_shared" : " 29 GB", | 20260508 03:39:58.089 | INFO | "malloc" : " 14 GB", | 20260508 03:39:58.089 | INFO | "pinned" : " 29 GB" | 20260508 03:39:58.089 | INFO | } | 20260508 03:39:58.089 | INFO | Perf report for the last 5.13473 seconds: | 20260508 03:39:58.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:39:58.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:58.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:58.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:58.089 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 03:39:58.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:39:58.089 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 03:39:58.089 | INFO | PRE_HQ : 65.7%, 1.6, 55.6ms, 0 | 20260508 03:39:58.089 | INFO | BAZWRITER : 68.0%, 2.7, 92.6ms, 0 | 20260508 03:39:58.089 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:39:58.089 | INFO | Uploaded 36GiB to GPU in 2.60659s (13.8112GiB/s) | 20260508 03:39:58.089 | INFO | Downloaded 38GiB to GPU in 2.70411s (14.0527GiB/s) | 20260508 03:40:03.228 | INFO | Analyzing chunk frames = [4948480,4948992) | 20260508 03:40:03.228 | INFO | MemInfo: { | 20260508 03:40:03.228 | INFO | "gpu" : " 17 GB", | 20260508 03:40:03.228 | INFO | "huge" : "", | 20260508 03:40:03.228 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:03.228 | INFO | "malloc" : " 14 GB", | 20260508 03:40:03.228 | INFO | "pinned" : " 29 GB" | 20260508 03:40:03.228 | INFO | } | 20260508 03:40:03.229 | INFO | Perf report for the last 5.1401 seconds: | 20260508 03:40:03.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:03.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 03:40:03.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 03:40:03.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:03.229 | INFO | Uploaded 36GiB to GPU in 2.58607s (13.9207GiB/s) | 20260508 03:40:03.229 | INFO | Downloaded 34GiB to GPU in 2.42367s (14.0283GiB/s) | 20260508 03:40:08.368 | INFO | Analyzing chunk frames = [4948992,4949504) | 20260508 03:40:08.368 | INFO | MemInfo: { | 20260508 03:40:08.368 | INFO | "gpu" : " 17 GB", | 20260508 03:40:08.368 | INFO | "huge" : "", | 20260508 03:40:08.368 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:08.368 | INFO | "malloc" : " 14 GB", | 20260508 03:40:08.368 | INFO | "pinned" : " 29 GB" | 20260508 03:40:08.368 | INFO | } | 20260508 03:40:08.369 | INFO | Perf report for the last 5.13979 seconds: | 20260508 03:40:08.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:08.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:40:08.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:40:08.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:08.369 | INFO | Uploaded 36GiB to GPU in 2.57386s (13.9868GiB/s) | 20260508 03:40:08.369 | INFO | Downloaded 34GiB to GPU in 2.48868s (13.6619GiB/s) | 20260508 03:40:13.510 | INFO | Analyzing chunk frames = [4949504,4950016) | 20260508 03:40:13.510 | INFO | MemInfo: { | 20260508 03:40:13.510 | INFO | "gpu" : " 17 GB", | 20260508 03:40:13.510 | INFO | "huge" : "", | 20260508 03:40:13.510 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:13.510 | INFO | "malloc" : " 14 GB", | 20260508 03:40:13.510 | INFO | "pinned" : " 29 GB" | 20260508 03:40:13.510 | INFO | } | 20260508 03:40:13.511 | INFO | Perf report for the last 5.14203 seconds: | 20260508 03:40:13.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:13.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 03:40:13.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:40:13.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:13.511 | INFO | Uploaded 36GiB to GPU in 2.56026s (14.0611GiB/s) | 20260508 03:40:13.511 | INFO | Downloaded 34GiB to GPU in 2.48893s (13.6605GiB/s) | 20260508 03:40:18.649 | INFO | Analyzing chunk frames = [4950016,4950528) | 20260508 03:40:18.649 | INFO | MemInfo: { | 20260508 03:40:18.649 | INFO | "gpu" : " 17 GB", | 20260508 03:40:18.649 | INFO | "huge" : "", | 20260508 03:40:18.649 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:18.649 | INFO | "malloc" : " 14 GB", | 20260508 03:40:18.649 | INFO | "pinned" : " 29 GB" | 20260508 03:40:18.649 | INFO | } | 20260508 03:40:18.650 | INFO | Perf report for the last 5.13891 seconds: | 20260508 03:40:18.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:18.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 03:40:18.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:40:18.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:18.650 | INFO | Uploaded 36GiB to GPU in 2.47517s (14.5445GiB/s) | 20260508 03:40:18.650 | INFO | Downloaded 34GiB to GPU in 2.57592s (13.1992GiB/s) | 20260508 03:40:23.786 | INFO | Analyzing chunk frames = [4950528,4951040) | 20260508 03:40:23.786 | INFO | MemInfo: { | 20260508 03:40:23.786 | INFO | "gpu" : " 17 GB", | 20260508 03:40:23.786 | INFO | "huge" : "", | 20260508 03:40:23.786 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:23.786 | INFO | "malloc" : " 14 GB", | 20260508 03:40:23.786 | INFO | "pinned" : " 29 GB" | 20260508 03:40:23.786 | INFO | } | 20260508 03:40:23.786 | INFO | Perf report for the last 5.13693 seconds: | 20260508 03:40:23.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:23.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.787 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:40:23.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.787 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:40:23.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:23.787 | INFO | Uploaded 36GiB to GPU in 2.43125s (14.8072GiB/s) | 20260508 03:40:23.787 | INFO | Downloaded 34GiB to GPU in 2.41722s (14.0657GiB/s) | 20260508 03:40:28.928 | INFO | Analyzing chunk frames = [4951040,4951552) | 20260508 03:40:28.928 | INFO | MemInfo: { | 20260508 03:40:28.928 | INFO | "gpu" : " 17 GB", | 20260508 03:40:28.928 | INFO | "huge" : "", | 20260508 03:40:28.928 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:28.928 | INFO | "malloc" : " 14 GB", | 20260508 03:40:28.928 | INFO | "pinned" : " 29 GB" | 20260508 03:40:28.928 | INFO | } | 20260508 03:40:28.928 | INFO | Perf report for the last 5.14128 seconds: | 20260508 03:40:28.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:28.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:40:28.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:40:28.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:28.928 | INFO | Uploaded 36GiB to GPU in 2.46229s (14.6205GiB/s) | 20260508 03:40:28.928 | INFO | Downloaded 34GiB to GPU in 2.42519s (14.0195GiB/s) | 20260508 03:40:34.069 | INFO | Analyzing chunk frames = [4951552,4952064) | 20260508 03:40:34.069 | INFO | MemInfo: { | 20260508 03:40:34.069 | INFO | "gpu" : " 17 GB", | 20260508 03:40:34.069 | INFO | "huge" : "", | 20260508 03:40:34.069 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:34.069 | INFO | "malloc" : " 14 GB", | 20260508 03:40:34.069 | INFO | "pinned" : " 29 GB" | 20260508 03:40:34.069 | INFO | } | 20260508 03:40:34.069 | INFO | Perf report for the last 5.14144 seconds: | 20260508 03:40:34.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:34.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.069 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:40:34.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.069 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:40:34.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:34.070 | INFO | Uploaded 36GiB to GPU in 2.45232s (14.68GiB/s) | 20260508 03:40:34.070 | INFO | Downloaded 34GiB to GPU in 2.41835s (14.0592GiB/s) | 20260508 03:40:34.180 | INFO | Starting a round of Baz IO | 20260508 03:40:37.597 | INFO | Populating RTMetrics report | 20260508 03:40:37.597 | INFO | RTMetrics report populated | 20260508 03:40:37.597 | INFO | Opening tmp RTMetrics json file | 20260508 03:40:37.609 | INFO | Serializing RTMetrics to json | 20260508 03:40:37.610 | INFO | Serialization done, swapping tmp file now... | 20260508 03:40:37.610 | INFO | RTMetrics done | 20260508 03:40:37.674 | INFO | Finished a round of Baz IO: Wrote 1.612 GiB in 3.493 seconds | 20260508 03:40:37.674 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.23% / 0.00% / 58.77% | 20260508 03:40:37.674 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.601268% / 3.97364e-06% / 9.13938e-05% | 20260508 03:40:37.674 | INFO | Baz IO overall: | 20260508 03:40:37.674 | INFO | BytesWritten : 2224647153773 | 20260508 03:40:37.674 | INFO | NumEvents : 1074618031705 | 20260508 03:40:39.207 | INFO | Analyzing chunk frames = [4952064,4952576) | 20260508 03:40:39.207 | INFO | MemInfo: { | 20260508 03:40:39.207 | INFO | "gpu" : " 17 GB", | 20260508 03:40:39.207 | INFO | "huge" : "", | 20260508 03:40:39.207 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:39.207 | INFO | "malloc" : " 14 GB", | 20260508 03:40:39.207 | INFO | "pinned" : " 29 GB" | 20260508 03:40:39.207 | INFO | } | 20260508 03:40:39.207 | INFO | Perf report for the last 5.13829 seconds: | 20260508 03:40:39.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:39.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:39.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:39.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:39.208 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 03:40:39.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:39.208 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 03:40:39.208 | INFO | PRE_HQ : 66.5%, 1.8, 61.2ms, 0 | 20260508 03:40:39.208 | INFO | BAZWRITER : 67.4%, 2.8, 96.7ms, 0 | 20260508 03:40:39.208 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:40:39.208 | INFO | Uploaded 36GiB to GPU in 2.58332s (13.9356GiB/s) | 20260508 03:40:39.208 | INFO | Downloaded 38GiB to GPU in 2.70183s (14.0646GiB/s) | 20260508 03:40:44.343 | INFO | Analyzing chunk frames = [4952576,4953088) | 20260508 03:40:44.343 | INFO | MemInfo: { | 20260508 03:40:44.343 | INFO | "gpu" : " 17 GB", | 20260508 03:40:44.343 | INFO | "huge" : "", | 20260508 03:40:44.343 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:44.344 | INFO | "malloc" : " 14 GB", | 20260508 03:40:44.344 | INFO | "pinned" : " 29 GB" | 20260508 03:40:44.344 | INFO | } | 20260508 03:40:44.344 | INFO | Perf report for the last 5.13617 seconds: | 20260508 03:40:44.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:44.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:40:44.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:40:44.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:44.344 | INFO | Uploaded 36GiB to GPU in 2.56728s (14.0226GiB/s) | 20260508 03:40:44.344 | INFO | Downloaded 34GiB to GPU in 2.42588s (14.0156GiB/s) | 20260508 03:40:49.481 | INFO | Analyzing chunk frames = [4953088,4953600) | 20260508 03:40:49.481 | INFO | MemInfo: { | 20260508 03:40:49.481 | INFO | "gpu" : " 17 GB", | 20260508 03:40:49.481 | INFO | "huge" : "", | 20260508 03:40:49.481 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:49.482 | INFO | "malloc" : " 14 GB", | 20260508 03:40:49.482 | INFO | "pinned" : " 29 GB" | 20260508 03:40:49.482 | INFO | } | 20260508 03:40:49.482 | INFO | Perf report for the last 5.138 seconds: | 20260508 03:40:49.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:49.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:40:49.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:40:49.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:49.482 | INFO | Uploaded 36GiB to GPU in 2.54464s (14.1474GiB/s) | 20260508 03:40:49.482 | INFO | Downloaded 34GiB to GPU in 2.4209s (14.0444GiB/s) | 20260508 03:40:54.622 | INFO | Analyzing chunk frames = [4953600,4954112) | 20260508 03:40:54.622 | INFO | MemInfo: { | 20260508 03:40:54.622 | INFO | "gpu" : " 17 GB", | 20260508 03:40:54.622 | INFO | "huge" : "", | 20260508 03:40:54.623 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:54.623 | INFO | "malloc" : " 14 GB", | 20260508 03:40:54.623 | INFO | "pinned" : " 29 GB" | 20260508 03:40:54.623 | INFO | } | 20260508 03:40:54.623 | INFO | Perf report for the last 5.14094 seconds: | 20260508 03:40:54.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:54.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:40:54.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:40:54.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:54.623 | INFO | Uploaded 36GiB to GPU in 2.53814s (14.1836GiB/s) | 20260508 03:40:54.623 | INFO | Downloaded 34GiB to GPU in 2.50401s (13.5782GiB/s) | 20260508 03:40:59.765 | INFO | Analyzing chunk frames = [4954112,4954624) | 20260508 03:40:59.765 | INFO | MemInfo: { | 20260508 03:40:59.765 | INFO | "gpu" : " 17 GB", | 20260508 03:40:59.765 | INFO | "huge" : "", | 20260508 03:40:59.765 | INFO | "huge_shared" : " 29 GB", | 20260508 03:40:59.765 | INFO | "malloc" : " 14 GB", | 20260508 03:40:59.765 | INFO | "pinned" : " 29 GB" | 20260508 03:40:59.766 | INFO | } | 20260508 03:40:59.766 | INFO | Perf report for the last 5.14292 seconds: | 20260508 03:40:59.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:40:59.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 03:40:59.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:40:59.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:40:59.766 | INFO | Uploaded 36GiB to GPU in 2.46279s (14.6176GiB/s) | 20260508 03:40:59.766 | INFO | Downloaded 34GiB to GPU in 2.57654s (13.196GiB/s) | 20260508 03:41:04.903 | INFO | Analyzing chunk frames = [4954624,4955136) | 20260508 03:41:04.903 | INFO | MemInfo: { | 20260508 03:41:04.903 | INFO | "gpu" : " 17 GB", | 20260508 03:41:04.903 | INFO | "huge" : "", | 20260508 03:41:04.903 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:04.903 | INFO | "malloc" : " 14 GB", | 20260508 03:41:04.903 | INFO | "pinned" : " 29 GB" | 20260508 03:41:04.903 | INFO | } | 20260508 03:41:04.903 | INFO | Perf report for the last 5.13775 seconds: | 20260508 03:41:04.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:04.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.903 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 03:41:04.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.903 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:41:04.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:04.904 | INFO | Uploaded 36GiB to GPU in 2.45188s (14.6826GiB/s) | 20260508 03:41:04.904 | INFO | Downloaded 34GiB to GPU in 2.4119s (14.0967GiB/s) | 20260508 03:41:10.039 | INFO | Analyzing chunk frames = [4955136,4955648) | 20260508 03:41:10.039 | INFO | MemInfo: { | 20260508 03:41:10.039 | INFO | "gpu" : " 17 GB", | 20260508 03:41:10.039 | INFO | "huge" : "", | 20260508 03:41:10.039 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:10.039 | INFO | "malloc" : " 14 GB", | 20260508 03:41:10.039 | INFO | "pinned" : " 29 GB" | 20260508 03:41:10.039 | INFO | } | 20260508 03:41:10.039 | INFO | Perf report for the last 5.13558 seconds: | 20260508 03:41:10.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:10.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.039 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:41:10.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.039 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:41:10.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:10.040 | INFO | Uploaded 36GiB to GPU in 2.45879s (14.6413GiB/s) | 20260508 03:41:10.040 | INFO | Downloaded 34GiB to GPU in 2.40658s (14.1279GiB/s) | 20260508 03:41:15.183 | INFO | Analyzing chunk frames = [4955648,4956160) | 20260508 03:41:15.183 | INFO | MemInfo: { | 20260508 03:41:15.183 | INFO | "gpu" : " 17 GB", | 20260508 03:41:15.183 | INFO | "huge" : "", | 20260508 03:41:15.183 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:15.183 | INFO | "malloc" : " 14 GB", | 20260508 03:41:15.183 | INFO | "pinned" : " 29 GB" | 20260508 03:41:15.183 | INFO | } | 20260508 03:41:15.183 | INFO | Perf report for the last 5.14425 seconds: | 20260508 03:41:15.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:15.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 03:41:15.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260508 03:41:15.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:15.183 | INFO | Uploaded 36GiB to GPU in 2.49533s (14.427GiB/s) | 20260508 03:41:15.184 | INFO | Downloaded 34GiB to GPU in 2.42437s (14.0243GiB/s) | 20260508 03:41:15.184 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:41:15.184 | INFO | Total mem usage: 16.7 GiB | 20260508 03:41:15.184 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:41:15.184 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:41:15.184 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:41:15.184 | INFO | High Water Breakdown: | 20260508 03:41:15.184 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:41:15.184 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:41:15.184 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:41:15.184 | INFO | ------------------------------------------ | 20260508 03:41:15.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:41:15.184 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:41:15.184 | INFO | Total mem usage: 26.5 GiB | 20260508 03:41:15.184 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:41:15.184 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:41:15.184 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:41:15.184 | INFO | High Water Breakdown: | 20260508 03:41:15.184 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:41:15.184 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:41:15.184 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:41:15.184 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:41:15.184 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:41:15.184 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:41:15.184 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:41:15.184 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:41:15.184 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:41:15.184 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:41:15.184 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:41:15.184 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:41:15.184 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:41:15.184 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:41:15.184 | INFO | ------------------------------------------ | 20260508 03:41:15.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:41:15.184 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:41:15.184 | INFO | Total mem usage: 27.8 GiB | 20260508 03:41:15.184 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:41:15.184 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:41:15.184 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:41:15.184 | INFO | High Water Breakdown: | 20260508 03:41:15.184 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:41:15.184 | INFO | ------------------------------------------ | 20260508 03:41:15.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:41:15.184 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:41:15.184 | INFO | Total mem usage: 15.7 GiB | 20260508 03:41:15.184 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:41:15.184 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:41:15.184 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:41:15.184 | INFO | High Water Breakdown: | 20260508 03:41:15.184 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:41:15.184 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:41:15.184 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:41:15.184 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:41:15.184 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:41:15.185 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:41:15.185 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:41:15.185 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:41:15.185 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:41:15.185 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:41:15.185 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:41:15.185 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:41:15.185 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:41:15.185 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:41:15.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:41:15.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:41:15.185 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:41:15.185 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:41:15.185 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:41:15.185 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:41:15.185 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:41:15.185 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:41:15.185 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:41:15.185 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:41:15.185 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:41:15.185 | INFO | ------------------------------------------ | 20260508 03:41:15.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:41:15.185 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:41:15.185 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:41:15.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:41:15.185 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 1.90e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.18e-02 / 7.12e-03 | 20260508 03:41:15.185 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.45e+00, 5.89e-01 / 1.85e-01 | 20260508 03:41:15.185 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.07e-01 / 1.10e-01 | 20260508 03:41:15.185 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.60e+01, 4.16e+00 / 2.21e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.70e+00 / 1.32e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.66e+00, 4.19e-01 / 1.98e-01 | 20260508 03:41:15.185 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.23e+00, 2.66e-01 / 1.41e-01 | 20260508 03:41:15.185 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.62e-01, 5.19e-02 / 2.51e-02 | 20260508 03:41:15.185 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:41:15.185 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:41:15.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:41:15.185 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 24.60%, Upload -- 2.62e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 2.29%, LockUpload -- 2.44e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 2.59%, LockDownload -- 2.76e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 15.48%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:41:15.185 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260508 03:41:15.280 | INFO | Starting a round of Baz IO | 20260508 03:41:18.713 | INFO | Populating RTMetrics report | 20260508 03:41:18.713 | INFO | RTMetrics report populated | 20260508 03:41:18.713 | INFO | Opening tmp RTMetrics json file | 20260508 03:41:18.725 | INFO | Serializing RTMetrics to json | 20260508 03:41:18.726 | INFO | Serialization done, swapping tmp file now... | 20260508 03:41:18.727 | INFO | RTMetrics done | 20260508 03:41:18.787 | INFO | Finished a round of Baz IO: Wrote 1.612 GiB in 3.506 seconds | 20260508 03:41:18.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.23% / 0.00% / 58.77% | 20260508 03:41:18.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.599786% / 3.97364e-06% / 6.35783e-05% | 20260508 03:41:18.787 | INFO | Baz IO overall: | 20260508 03:41:18.787 | INFO | BytesWritten : 2226377578859 | 20260508 03:41:18.787 | INFO | NumEvents : 1075446242684 | 20260508 03:41:20.321 | INFO | Analyzing chunk frames = [4956160,4956672) | 20260508 03:41:20.321 | INFO | MemInfo: { | 20260508 03:41:20.321 | INFO | "gpu" : " 17 GB", | 20260508 03:41:20.321 | INFO | "huge" : "", | 20260508 03:41:20.321 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:20.321 | INFO | "malloc" : " 14 GB", | 20260508 03:41:20.321 | INFO | "pinned" : " 29 GB" | 20260508 03:41:20.321 | INFO | } | 20260508 03:41:20.322 | INFO | Perf report for the last 5.1383 seconds: | 20260508 03:41:20.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:20.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:20.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:20.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:20.322 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 03:41:20.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:20.322 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 03:41:20.322 | INFO | PRE_HQ : 65.7%, 1.6, 55.7ms, 0 | 20260508 03:41:20.322 | INFO | BAZWRITER : 67.7%, 2.7, 92.5ms, 0 | 20260508 03:41:20.322 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:41:20.322 | INFO | Uploaded 36GiB to GPU in 2.58573s (13.9226GiB/s) | 20260508 03:41:20.322 | INFO | Downloaded 38GiB to GPU in 2.70966s (14.0239GiB/s) | 20260508 03:41:25.462 | INFO | Analyzing chunk frames = [4956672,4957184) | 20260508 03:41:25.462 | INFO | MemInfo: { | 20260508 03:41:25.462 | INFO | "gpu" : " 17 GB", | 20260508 03:41:25.462 | INFO | "huge" : "", | 20260508 03:41:25.462 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:25.462 | INFO | "malloc" : " 14 GB", | 20260508 03:41:25.462 | INFO | "pinned" : " 29 GB" | 20260508 03:41:25.462 | INFO | } | 20260508 03:41:25.462 | INFO | Perf report for the last 5.14057 seconds: | 20260508 03:41:25.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:25.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.462 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:41:25.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.462 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:41:25.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:25.463 | INFO | Uploaded 36GiB to GPU in 2.55321s (14.0999GiB/s) | 20260508 03:41:25.463 | INFO | Downloaded 34GiB to GPU in 2.43479s (13.9642GiB/s) | 20260508 03:41:30.597 | INFO | Analyzing chunk frames = [4957184,4957696) | 20260508 03:41:30.597 | INFO | MemInfo: { | 20260508 03:41:30.597 | INFO | "gpu" : " 17 GB", | 20260508 03:41:30.597 | INFO | "huge" : "", | 20260508 03:41:30.597 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:30.597 | INFO | "malloc" : " 14 GB", | 20260508 03:41:30.597 | INFO | "pinned" : " 29 GB" | 20260508 03:41:30.597 | INFO | } | 20260508 03:41:30.597 | INFO | Perf report for the last 5.13501 seconds: | 20260508 03:41:30.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:30.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:41:30.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:41:30.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:30.597 | INFO | Uploaded 36GiB to GPU in 2.52298s (14.2689GiB/s) | 20260508 03:41:30.598 | INFO | Downloaded 34GiB to GPU in 2.43346s (13.9719GiB/s) | 20260508 03:41:35.737 | INFO | Analyzing chunk frames = [4957696,4958208) | 20260508 03:41:35.737 | INFO | MemInfo: { | 20260508 03:41:35.737 | INFO | "gpu" : " 17 GB", | 20260508 03:41:35.737 | INFO | "huge" : "", | 20260508 03:41:35.737 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:35.738 | INFO | "malloc" : " 14 GB", | 20260508 03:41:35.738 | INFO | "pinned" : " 29 GB" | 20260508 03:41:35.738 | INFO | } | 20260508 03:41:35.738 | INFO | Perf report for the last 5.141 seconds: | 20260508 03:41:35.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:35.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:41:35.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:41:35.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:35.738 | INFO | Uploaded 36GiB to GPU in 2.54983s (14.1186GiB/s) | 20260508 03:41:35.739 | INFO | Downloaded 34GiB to GPU in 2.51136s (13.5385GiB/s) | 20260508 03:41:40.883 | INFO | Analyzing chunk frames = [4958208,4958720) | 20260508 03:41:40.883 | INFO | MemInfo: { | 20260508 03:41:40.883 | INFO | "gpu" : " 17 GB", | 20260508 03:41:40.883 | INFO | "huge" : "", | 20260508 03:41:40.883 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:40.883 | INFO | "malloc" : " 14 GB", | 20260508 03:41:40.883 | INFO | "pinned" : " 29 GB" | 20260508 03:41:40.883 | INFO | } | 20260508 03:41:40.883 | INFO | Perf report for the last 5.14529 seconds: | 20260508 03:41:40.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:40.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.883 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 03:41:40.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.884 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 03:41:40.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:40.884 | INFO | Uploaded 36GiB to GPU in 2.37732s (15.1431GiB/s) | 20260508 03:41:40.884 | INFO | Downloaded 34GiB to GPU in 2.58062s (13.1751GiB/s) | 20260508 03:41:46.026 | INFO | Analyzing chunk frames = [4958720,4959232) | 20260508 03:41:46.026 | INFO | MemInfo: { | 20260508 03:41:46.026 | INFO | "gpu" : " 17 GB", | 20260508 03:41:46.026 | INFO | "huge" : "", | 20260508 03:41:46.026 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:46.027 | INFO | "malloc" : " 14 GB", | 20260508 03:41:46.027 | INFO | "pinned" : " 29 GB" | 20260508 03:41:46.027 | INFO | } | 20260508 03:41:46.027 | INFO | Perf report for the last 5.14325 seconds: | 20260508 03:41:46.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:46.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:41:46.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:41:46.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:46.027 | INFO | Uploaded 36GiB to GPU in 2.41591s (14.9012GiB/s) | 20260508 03:41:46.027 | INFO | Downloaded 34GiB to GPU in 2.41655s (14.0697GiB/s) | 20260508 03:41:51.158 | INFO | Analyzing chunk frames = [4959232,4959744) | 20260508 03:41:51.158 | INFO | MemInfo: { | 20260508 03:41:51.158 | INFO | "gpu" : " 17 GB", | 20260508 03:41:51.158 | INFO | "huge" : "", | 20260508 03:41:51.158 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:51.158 | INFO | "malloc" : " 14 GB", | 20260508 03:41:51.158 | INFO | "pinned" : " 29 GB" | 20260508 03:41:51.158 | INFO | } | 20260508 03:41:51.158 | INFO | Perf report for the last 5.13151 seconds: | 20260508 03:41:51.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:51.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.158 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:41:51.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.158 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 03:41:51.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:51.159 | INFO | Uploaded 36GiB to GPU in 2.45945s (14.6374GiB/s) | 20260508 03:41:51.159 | INFO | Downloaded 34GiB to GPU in 2.43681s (13.9526GiB/s) | 20260508 03:41:56.291 | INFO | Analyzing chunk frames = [4959744,4960256) | 20260508 03:41:56.291 | INFO | MemInfo: { | 20260508 03:41:56.291 | INFO | "gpu" : " 17 GB", | 20260508 03:41:56.292 | INFO | "huge" : "", | 20260508 03:41:56.292 | INFO | "huge_shared" : " 29 GB", | 20260508 03:41:56.292 | INFO | "malloc" : " 14 GB", | 20260508 03:41:56.292 | INFO | "pinned" : " 29 GB" | 20260508 03:41:56.292 | INFO | } | 20260508 03:41:56.292 | INFO | Perf report for the last 5.13368 seconds: | 20260508 03:41:56.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:41:56.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.292 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:41:56.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.292 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 03:41:56.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:41:56.293 | INFO | Uploaded 36GiB to GPU in 2.44409s (14.7294GiB/s) | 20260508 03:41:56.293 | INFO | Downloaded 34GiB to GPU in 2.42212s (14.0373GiB/s) | 20260508 03:41:56.404 | INFO | Starting a round of Baz IO | 20260508 03:41:59.835 | INFO | Populating RTMetrics report | 20260508 03:41:59.835 | INFO | RTMetrics report populated | 20260508 03:41:59.835 | INFO | Opening tmp RTMetrics json file | 20260508 03:41:59.844 | INFO | Serializing RTMetrics to json | 20260508 03:41:59.849 | INFO | Serialization done, swapping tmp file now... | 20260508 03:41:59.850 | INFO | RTMetrics done | 20260508 03:41:59.909 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.504 seconds | 20260508 03:41:59.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.23% / 0.00% / 58.77% | 20260508 03:41:59.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.597004% / 3.97364e-06% / 5.5631e-05% | 20260508 03:41:59.909 | INFO | Baz IO overall: | 20260508 03:41:59.909 | INFO | BytesWritten : 2228107288265 | 20260508 03:41:59.909 | INFO | NumEvents : 1076273910942 | 20260508 03:42:01.434 | INFO | Analyzing chunk frames = [4960256,4960768) | 20260508 03:42:01.434 | INFO | MemInfo: { | 20260508 03:42:01.434 | INFO | "gpu" : " 17 GB", | 20260508 03:42:01.434 | INFO | "huge" : "", | 20260508 03:42:01.434 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:01.434 | INFO | "malloc" : " 14 GB", | 20260508 03:42:01.434 | INFO | "pinned" : " 29 GB" | 20260508 03:42:01.434 | INFO | } | 20260508 03:42:01.434 | INFO | Perf report for the last 5.14182 seconds: | 20260508 03:42:01.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:01.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:01.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:01.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:01.434 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 03:42:01.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:01.434 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 03:42:01.434 | INFO | PRE_HQ : 66.3%, 1.8, 61.9ms, 0 | 20260508 03:42:01.434 | INFO | BAZWRITER : 67.3%, 2.8, 97.5ms, 0 | 20260508 03:42:01.434 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:42:01.434 | INFO | Uploaded 36GiB to GPU in 2.59911s (13.8509GiB/s) | 20260508 03:42:01.434 | INFO | Downloaded 38GiB to GPU in 2.69133s (14.1194GiB/s) | 20260508 03:42:06.573 | INFO | Analyzing chunk frames = [4960768,4961280) | 20260508 03:42:06.573 | INFO | MemInfo: { | 20260508 03:42:06.573 | INFO | "gpu" : " 17 GB", | 20260508 03:42:06.573 | INFO | "huge" : "", | 20260508 03:42:06.573 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:06.573 | INFO | "malloc" : " 14 GB", | 20260508 03:42:06.573 | INFO | "pinned" : " 29 GB" | 20260508 03:42:06.573 | INFO | } | 20260508 03:42:06.573 | INFO | Perf report for the last 5.1391 seconds: | 20260508 03:42:06.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:06.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:42:06.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:42:06.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:06.573 | INFO | Uploaded 36GiB to GPU in 2.58064s (13.95GiB/s) | 20260508 03:42:06.573 | INFO | Downloaded 34GiB to GPU in 2.41944s (14.0528GiB/s) | 20260508 03:42:11.717 | INFO | Analyzing chunk frames = [4961280,4961792) | 20260508 03:42:11.717 | INFO | MemInfo: { | 20260508 03:42:11.717 | INFO | "gpu" : " 17 GB", | 20260508 03:42:11.718 | INFO | "huge" : "", | 20260508 03:42:11.718 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:11.718 | INFO | "malloc" : " 14 GB", | 20260508 03:42:11.718 | INFO | "pinned" : " 29 GB" | 20260508 03:42:11.718 | INFO | } | 20260508 03:42:11.718 | INFO | Perf report for the last 5.14481 seconds: | 20260508 03:42:11.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:11.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 03:42:11.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:42:11.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:11.718 | INFO | Uploaded 36GiB to GPU in 2.588s (13.9103GiB/s) | 20260508 03:42:11.718 | INFO | Downloaded 34GiB to GPU in 2.49513s (13.6266GiB/s) | 20260508 03:42:16.853 | INFO | Analyzing chunk frames = [4961792,4962304) | 20260508 03:42:16.853 | INFO | MemInfo: { | 20260508 03:42:16.853 | INFO | "gpu" : " 17 GB", | 20260508 03:42:16.853 | INFO | "huge" : "", | 20260508 03:42:16.853 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:16.853 | INFO | "malloc" : " 14 GB", | 20260508 03:42:16.853 | INFO | "pinned" : " 29 GB" | 20260508 03:42:16.853 | INFO | } | 20260508 03:42:16.853 | INFO | Perf report for the last 5.13549 seconds: | 20260508 03:42:16.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:16.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.853 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 03:42:16.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.853 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:42:16.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:16.854 | INFO | Uploaded 36GiB to GPU in 2.57564s (13.9771GiB/s) | 20260508 03:42:16.854 | INFO | Downloaded 34GiB to GPU in 2.40103s (14.1606GiB/s) | 20260508 03:42:21.994 | INFO | Analyzing chunk frames = [4962304,4962816) | 20260508 03:42:21.994 | INFO | MemInfo: { | 20260508 03:42:21.994 | INFO | "gpu" : " 17 GB", | 20260508 03:42:21.994 | INFO | "huge" : "", | 20260508 03:42:21.994 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:21.994 | INFO | "malloc" : " 14 GB", | 20260508 03:42:21.994 | INFO | "pinned" : " 29 GB" | 20260508 03:42:21.994 | INFO | } | 20260508 03:42:21.995 | INFO | Perf report for the last 5.14134 seconds: | 20260508 03:42:21.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:21.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:42:21.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 03:42:21.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:21.995 | INFO | Uploaded 36GiB to GPU in 2.38099s (15.1198GiB/s) | 20260508 03:42:21.995 | INFO | Downloaded 34GiB to GPU in 2.54138s (13.3786GiB/s) | 20260508 03:42:27.130 | INFO | Analyzing chunk frames = [4962816,4963328) | 20260508 03:42:27.130 | INFO | MemInfo: { | 20260508 03:42:27.130 | INFO | "gpu" : " 17 GB", | 20260508 03:42:27.130 | INFO | "huge" : "", | 20260508 03:42:27.130 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:27.130 | INFO | "malloc" : " 14 GB", | 20260508 03:42:27.130 | INFO | "pinned" : " 29 GB" | 20260508 03:42:27.130 | INFO | } | 20260508 03:42:27.130 | INFO | Perf report for the last 5.13587 seconds: | 20260508 03:42:27.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:27.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.130 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:42:27.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.131 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:42:27.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:27.131 | INFO | Uploaded 36GiB to GPU in 2.42849s (14.824GiB/s) | 20260508 03:42:27.131 | INFO | Downloaded 34GiB to GPU in 2.41819s (14.0601GiB/s) | 20260508 03:42:32.271 | INFO | Analyzing chunk frames = [4963328,4963840) | 20260508 03:42:32.271 | INFO | MemInfo: { | 20260508 03:42:32.271 | INFO | "gpu" : " 17 GB", | 20260508 03:42:32.271 | INFO | "huge" : "", | 20260508 03:42:32.271 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:32.271 | INFO | "malloc" : " 14 GB", | 20260508 03:42:32.271 | INFO | "pinned" : " 29 GB" | 20260508 03:42:32.271 | INFO | } | 20260508 03:42:32.271 | INFO | Perf report for the last 5.14062 seconds: | 20260508 03:42:32.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:32.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.271 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 03:42:32.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.271 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:42:32.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:32.272 | INFO | Uploaded 36GiB to GPU in 2.43935s (14.7581GiB/s) | 20260508 03:42:32.272 | INFO | Downloaded 34GiB to GPU in 2.42218s (14.0369GiB/s) | 20260508 03:42:37.412 | INFO | Analyzing chunk frames = [4963840,4964352) | 20260508 03:42:37.412 | INFO | MemInfo: { | 20260508 03:42:37.412 | INFO | "gpu" : " 17 GB", | 20260508 03:42:37.412 | INFO | "huge" : "", | 20260508 03:42:37.413 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:37.413 | INFO | "malloc" : " 14 GB", | 20260508 03:42:37.413 | INFO | "pinned" : " 29 GB" | 20260508 03:42:37.413 | INFO | } | 20260508 03:42:37.413 | INFO | Perf report for the last 5.14174 seconds: | 20260508 03:42:37.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:37.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:42:37.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:42:37.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:37.413 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:42:37.413 | INFO | Total mem usage: 16.7 GiB | 20260508 03:42:37.413 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:42:37.413 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:42:37.413 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:42:37.413 | INFO | High Water Breakdown: | 20260508 03:42:37.413 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:42:37.413 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:42:37.413 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:42:37.413 | INFO | ------------------------------------------ | 20260508 03:42:37.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:42:37.413 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:42:37.413 | INFO | Total mem usage: 26.5 GiB | 20260508 03:42:37.413 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:42:37.413 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:42:37.413 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:42:37.413 | INFO | High Water Breakdown: | 20260508 03:42:37.413 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:42:37.413 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:42:37.414 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:42:37.414 | INFO | Uploaded 36GiB to GPU in 2.4637s (14.6122GiB/s) | 20260508 03:42:37.414 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:42:37.414 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:42:37.414 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:42:37.414 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:42:37.414 | INFO | Downloaded 34GiB to GPU in 2.4333s (13.9728GiB/s) | 20260508 03:42:37.414 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:42:37.414 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:42:37.414 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:42:37.414 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:42:37.414 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:42:37.414 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:42:37.414 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:42:37.414 | INFO | ------------------------------------------ | 20260508 03:42:37.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:42:37.414 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:42:37.414 | INFO | Total mem usage: 27.8 GiB | 20260508 03:42:37.414 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:42:37.414 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:42:37.414 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:42:37.414 | INFO | High Water Breakdown: | 20260508 03:42:37.414 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:42:37.414 | INFO | ------------------------------------------ | 20260508 03:42:37.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:42:37.414 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:42:37.414 | INFO | Total mem usage: 15.7 GiB | 20260508 03:42:37.414 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:42:37.414 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:42:37.414 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:42:37.414 | INFO | High Water Breakdown: | 20260508 03:42:37.414 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:42:37.414 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:42:37.414 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:42:37.414 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:42:37.414 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:42:37.414 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:42:37.414 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:42:37.414 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:42:37.414 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:42:37.414 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:42:37.414 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:42:37.414 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:42:37.414 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:42:37.414 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:42:37.414 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:42:37.414 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:42:37.414 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:42:37.414 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:42:37.414 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:42:37.414 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:42:37.414 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:42:37.414 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:42:37.414 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:42:37.414 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:42:37.414 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:42:37.415 | INFO | ------------------------------------------ | 20260508 03:42:37.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:42:37.415 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:42:37.415 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:42:37.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:42:37.415 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.40e+01 / 1.67e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.03e-01, 1.22e-02 / 5.33e-03 | 20260508 03:42:37.415 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.47e+00, 5.87e-01 / 2.26e-01 | 20260508 03:42:37.415 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 9.85e-02 | 20260508 03:42:37.415 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.58e+01, 3.96e+00 / 1.95e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.74e+00 / 1.23e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.20e-01 / 2.17e-01 | 20260508 03:42:37.415 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.21e+00, 2.65e-01 / 1.44e-01 | 20260508 03:42:37.415 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.26e-01 / 3.61e-01, 5.10e-02 / 2.43e-02 | 20260508 03:42:37.415 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:42:37.415 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:42:37.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:42:37.415 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 2.28%, LockUpload -- 2.42e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 03:42:37.415 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260508 03:42:37.526 | INFO | Starting a round of Baz IO | 20260508 03:42:40.949 | INFO | Populating RTMetrics report | 20260508 03:42:40.949 | INFO | RTMetrics report populated | 20260508 03:42:40.949 | INFO | Opening tmp RTMetrics json file | 20260508 03:42:40.963 | INFO | Serializing RTMetrics to json | 20260508 03:42:40.965 | INFO | Serialization done, swapping tmp file now... | 20260508 03:42:40.965 | INFO | RTMetrics done | 20260508 03:42:41.017 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.491 seconds | 20260508 03:42:41.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.22% / 0.00% / 58.77% | 20260508 03:42:41.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.597854% / 7.94729e-06% / 6.75519e-05% | 20260508 03:42:41.017 | INFO | Baz IO overall: | 20260508 03:42:41.017 | INFO | BytesWritten : 2229837548234 | 20260508 03:42:41.017 | INFO | NumEvents : 1077101928977 | 20260508 03:42:42.547 | INFO | Analyzing chunk frames = [4964352,4964864) | 20260508 03:42:42.547 | INFO | MemInfo: { | 20260508 03:42:42.547 | INFO | "gpu" : " 17 GB", | 20260508 03:42:42.547 | INFO | "huge" : "", | 20260508 03:42:42.547 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:42.547 | INFO | "malloc" : " 14 GB", | 20260508 03:42:42.547 | INFO | "pinned" : " 29 GB" | 20260508 03:42:42.547 | INFO | } | 20260508 03:42:42.547 | INFO | Perf report for the last 5.13412 seconds: | 20260508 03:42:42.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:42.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:42.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:42.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:42.547 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 03:42:42.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:42.547 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:42:42.547 | INFO | PRE_HQ : 66.5%, 1.8, 61.6ms, 0 | 20260508 03:42:42.547 | INFO | BAZWRITER : 67.5%, 2.8, 97.2ms, 0 | 20260508 03:42:42.547 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:42:42.548 | INFO | Uploaded 36GiB to GPU in 2.5415s (14.1649GiB/s) | 20260508 03:42:42.548 | INFO | Downloaded 38GiB to GPU in 2.70188s (14.0643GiB/s) | 20260508 03:42:47.689 | INFO | Analyzing chunk frames = [4964864,4965376) | 20260508 03:42:47.689 | INFO | MemInfo: { | 20260508 03:42:47.689 | INFO | "gpu" : " 17 GB", | 20260508 03:42:47.689 | INFO | "huge" : "", | 20260508 03:42:47.689 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:47.689 | INFO | "malloc" : " 14 GB", | 20260508 03:42:47.689 | INFO | "pinned" : " 29 GB" | 20260508 03:42:47.689 | INFO | } | 20260508 03:42:47.689 | INFO | Perf report for the last 5.14224 seconds: | 20260508 03:42:47.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:47.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.689 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:42:47.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.689 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:42:47.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:47.690 | INFO | Uploaded 36GiB to GPU in 2.57413s (13.9853GiB/s) | 20260508 03:42:47.690 | INFO | Downloaded 34GiB to GPU in 2.42683s (14.01GiB/s) | 20260508 03:42:52.833 | INFO | Analyzing chunk frames = [4965376,4965888) | 20260508 03:42:52.833 | INFO | MemInfo: { | 20260508 03:42:52.833 | INFO | "gpu" : " 17 GB", | 20260508 03:42:52.833 | INFO | "huge" : "", | 20260508 03:42:52.833 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:52.833 | INFO | "malloc" : " 14 GB", | 20260508 03:42:52.833 | INFO | "pinned" : " 29 GB" | 20260508 03:42:52.833 | INFO | } | 20260508 03:42:52.834 | INFO | Perf report for the last 5.1443 seconds: | 20260508 03:42:52.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:52.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:42:52.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:42:52.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:52.834 | INFO | Uploaded 36GiB to GPU in 2.54129s (14.166GiB/s) | 20260508 03:42:52.834 | INFO | Downloaded 34GiB to GPU in 2.49003s (13.6545GiB/s) | 20260508 03:42:57.971 | INFO | Analyzing chunk frames = [4965888,4966400) | 20260508 03:42:57.971 | INFO | MemInfo: { | 20260508 03:42:57.971 | INFO | "gpu" : " 17 GB", | 20260508 03:42:57.971 | INFO | "huge" : "", | 20260508 03:42:57.971 | INFO | "huge_shared" : " 29 GB", | 20260508 03:42:57.971 | INFO | "malloc" : " 14 GB", | 20260508 03:42:57.971 | INFO | "pinned" : " 29 GB" | 20260508 03:42:57.971 | INFO | } | 20260508 03:42:57.971 | INFO | Perf report for the last 5.13768 seconds: | 20260508 03:42:57.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:42:57.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 03:42:57.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:42:57.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:42:57.971 | INFO | Uploaded 36GiB to GPU in 2.56325s (14.0447GiB/s) | 20260508 03:42:57.972 | INFO | Downloaded 34GiB to GPU in 2.42733s (14.0072GiB/s) | 20260508 03:43:03.108 | INFO | Analyzing chunk frames = [4966400,4966912) | 20260508 03:43:03.108 | INFO | MemInfo: { | 20260508 03:43:03.108 | INFO | "gpu" : " 17 GB", | 20260508 03:43:03.108 | INFO | "huge" : "", | 20260508 03:43:03.108 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:03.108 | INFO | "malloc" : " 14 GB", | 20260508 03:43:03.108 | INFO | "pinned" : " 29 GB" | 20260508 03:43:03.108 | INFO | } | 20260508 03:43:03.108 | INFO | Perf report for the last 5.1367 seconds: | 20260508 03:43:03.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:03.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.108 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 03:43:03.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.108 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:43:03.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:03.109 | INFO | Uploaded 36GiB to GPU in 2.37505s (15.1576GiB/s) | 20260508 03:43:03.109 | INFO | Downloaded 34GiB to GPU in 2.54908s (13.3382GiB/s) | 20260508 03:43:08.246 | INFO | Analyzing chunk frames = [4966912,4967424) | 20260508 03:43:08.246 | INFO | MemInfo: { | 20260508 03:43:08.246 | INFO | "gpu" : " 17 GB", | 20260508 03:43:08.246 | INFO | "huge" : "", | 20260508 03:43:08.246 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:08.247 | INFO | "malloc" : " 14 GB", | 20260508 03:43:08.247 | INFO | "pinned" : " 29 GB" | 20260508 03:43:08.247 | INFO | } | 20260508 03:43:08.247 | INFO | Perf report for the last 5.13882 seconds: | 20260508 03:43:08.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:08.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:43:08.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:43:08.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:08.247 | INFO | Uploaded 36GiB to GPU in 2.44412s (14.7292GiB/s) | 20260508 03:43:08.247 | INFO | Downloaded 34GiB to GPU in 2.40738s (14.1233GiB/s) | 20260508 03:43:13.387 | INFO | Analyzing chunk frames = [4967424,4967936) | 20260508 03:43:13.387 | INFO | MemInfo: { | 20260508 03:43:13.387 | INFO | "gpu" : " 17 GB", | 20260508 03:43:13.387 | INFO | "huge" : "", | 20260508 03:43:13.387 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:13.388 | INFO | "malloc" : " 14 GB", | 20260508 03:43:13.388 | INFO | "pinned" : " 29 GB" | 20260508 03:43:13.388 | INFO | } | 20260508 03:43:13.388 | INFO | Perf report for the last 5.14088 seconds: | 20260508 03:43:13.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:13.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:43:13.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:43:13.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:13.388 | INFO | Uploaded 36GiB to GPU in 2.46845s (14.5841GiB/s) | 20260508 03:43:13.388 | INFO | Downloaded 34GiB to GPU in 2.42201s (14.0379GiB/s) | 20260508 03:43:18.528 | INFO | Analyzing chunk frames = [4967936,4968448) | 20260508 03:43:18.528 | INFO | MemInfo: { | 20260508 03:43:18.528 | INFO | "gpu" : " 17 GB", | 20260508 03:43:18.528 | INFO | "huge" : "", | 20260508 03:43:18.528 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:18.528 | INFO | "malloc" : " 14 GB", | 20260508 03:43:18.528 | INFO | "pinned" : " 29 GB" | 20260508 03:43:18.528 | INFO | } | 20260508 03:43:18.528 | INFO | Perf report for the last 5.14032 seconds: | 20260508 03:43:18.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:18.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 03:43:18.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 03:43:18.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:18.528 | INFO | Uploaded 36GiB to GPU in 2.48322s (14.4973GiB/s) | 20260508 03:43:18.529 | INFO | Downloaded 34GiB to GPU in 2.42447s (14.0237GiB/s) | 20260508 03:43:18.644 | INFO | Starting a round of Baz IO | 20260508 03:43:22.067 | INFO | Populating RTMetrics report | 20260508 03:43:22.067 | INFO | RTMetrics report populated | 20260508 03:43:22.067 | INFO | Opening tmp RTMetrics json file | 20260508 03:43:22.076 | INFO | Serializing RTMetrics to json | 20260508 03:43:22.082 | INFO | Serialization done, swapping tmp file now... | 20260508 03:43:22.083 | INFO | RTMetrics done | 20260508 03:43:22.141 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.497 seconds | 20260508 03:43:22.141 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.22% / 0.00% / 58.78% | 20260508 03:43:22.141 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.59609% / 3.97364e-06% / 6.35783e-05% | 20260508 03:43:22.141 | INFO | Baz IO overall: | 20260508 03:43:22.141 | INFO | BytesWritten : 2231566480708 | 20260508 03:43:22.142 | INFO | NumEvents : 1077928959571 | 20260508 03:43:23.664 | INFO | Analyzing chunk frames = [4968448,4968960) | 20260508 03:43:23.664 | INFO | MemInfo: { | 20260508 03:43:23.664 | INFO | "gpu" : " 17 GB", | 20260508 03:43:23.665 | INFO | "huge" : "", | 20260508 03:43:23.665 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:23.665 | INFO | "malloc" : " 14 GB", | 20260508 03:43:23.665 | INFO | "pinned" : " 29 GB" | 20260508 03:43:23.665 | INFO | } | 20260508 03:43:23.665 | INFO | Perf report for the last 5.13667 seconds: | 20260508 03:43:23.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:23.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:23.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:23.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:23.665 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 03:43:23.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:23.665 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 03:43:23.665 | INFO | PRE_HQ : 66.9%, 1.8, 62.5ms, 0 | 20260508 03:43:23.665 | INFO | BAZWRITER : 67.5%, 2.8, 98.1ms, 0 | 20260508 03:43:23.665 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:43:23.665 | INFO | Uploaded 36GiB to GPU in 2.5782s (13.9632GiB/s) | 20260508 03:43:23.665 | INFO | Downloaded 38GiB to GPU in 2.7093s (14.0258GiB/s) | 20260508 03:43:28.798 | INFO | Analyzing chunk frames = [4968960,4969472) | 20260508 03:43:28.798 | INFO | MemInfo: { | 20260508 03:43:28.798 | INFO | "gpu" : " 17 GB", | 20260508 03:43:28.798 | INFO | "huge" : "", | 20260508 03:43:28.798 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:28.798 | INFO | "malloc" : " 14 GB", | 20260508 03:43:28.798 | INFO | "pinned" : " 29 GB" | 20260508 03:43:28.798 | INFO | } | 20260508 03:43:28.798 | INFO | Perf report for the last 5.13354 seconds: | 20260508 03:43:28.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:28.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.798 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 03:43:28.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.798 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:43:28.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:28.799 | INFO | Uploaded 36GiB to GPU in 2.57282s (13.9924GiB/s) | 20260508 03:43:28.799 | INFO | Downloaded 34GiB to GPU in 2.43252s (13.9773GiB/s) | 20260508 03:43:33.944 | INFO | Analyzing chunk frames = [4969472,4969984) | 20260508 03:43:33.944 | INFO | MemInfo: { | 20260508 03:43:33.944 | INFO | "gpu" : " 17 GB", | 20260508 03:43:33.944 | INFO | "huge" : "", | 20260508 03:43:33.944 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:33.944 | INFO | "malloc" : " 14 GB", | 20260508 03:43:33.944 | INFO | "pinned" : " 29 GB" | 20260508 03:43:33.944 | INFO | } | 20260508 03:43:33.944 | INFO | Perf report for the last 5.14577 seconds: | 20260508 03:43:33.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:33.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 03:43:33.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:43:33.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:33.944 | INFO | Uploaded 36GiB to GPU in 2.55984s (14.0634GiB/s) | 20260508 03:43:33.944 | INFO | Downloaded 34GiB to GPU in 2.50643s (13.5651GiB/s) | 20260508 03:43:39.086 | INFO | Analyzing chunk frames = [4969984,4970496) | 20260508 03:43:39.086 | INFO | MemInfo: { | 20260508 03:43:39.086 | INFO | "gpu" : " 17 GB", | 20260508 03:43:39.086 | INFO | "huge" : "", | 20260508 03:43:39.086 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:39.086 | INFO | "malloc" : " 14 GB", | 20260508 03:43:39.086 | INFO | "pinned" : " 29 GB" | 20260508 03:43:39.087 | INFO | } | 20260508 03:43:39.087 | INFO | Perf report for the last 5.14263 seconds: | 20260508 03:43:39.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:39.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 03:43:39.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:43:39.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:39.087 | INFO | Uploaded 36GiB to GPU in 2.53903s (14.1787GiB/s) | 20260508 03:43:39.087 | INFO | Downloaded 34GiB to GPU in 2.50779s (13.5578GiB/s) | 20260508 03:43:44.227 | INFO | Analyzing chunk frames = [4970496,4971008) | 20260508 03:43:44.227 | INFO | MemInfo: { | 20260508 03:43:44.227 | INFO | "gpu" : " 17 GB", | 20260508 03:43:44.227 | INFO | "huge" : "", | 20260508 03:43:44.227 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:44.227 | INFO | "malloc" : " 14 GB", | 20260508 03:43:44.227 | INFO | "pinned" : " 29 GB" | 20260508 03:43:44.227 | INFO | } | 20260508 03:43:44.227 | INFO | Perf report for the last 5.14027 seconds: | 20260508 03:43:44.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:44.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 03:43:44.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:43:44.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:44.227 | INFO | Uploaded 36GiB to GPU in 2.42587s (14.8401GiB/s) | 20260508 03:43:44.227 | INFO | Downloaded 34GiB to GPU in 2.58729s (13.1411GiB/s) | 20260508 03:43:49.361 | INFO | Analyzing chunk frames = [4971008,4971520) | 20260508 03:43:49.361 | INFO | MemInfo: { | 20260508 03:43:49.361 | INFO | "gpu" : " 17 GB", | 20260508 03:43:49.361 | INFO | "huge" : "", | 20260508 03:43:49.361 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:49.361 | INFO | "malloc" : " 14 GB", | 20260508 03:43:49.361 | INFO | "pinned" : " 29 GB" | 20260508 03:43:49.361 | INFO | } | 20260508 03:43:49.361 | INFO | Perf report for the last 5.1342 seconds: | 20260508 03:43:49.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:49.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:43:49.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 03:43:49.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:49.361 | INFO | Uploaded 36GiB to GPU in 2.41881s (14.8833GiB/s) | 20260508 03:43:49.361 | INFO | Downloaded 34GiB to GPU in 2.42352s (14.0292GiB/s) | 20260508 03:43:54.501 | INFO | Analyzing chunk frames = [4971520,4972032) | 20260508 03:43:54.501 | INFO | MemInfo: { | 20260508 03:43:54.501 | INFO | "gpu" : " 17 GB", | 20260508 03:43:54.501 | INFO | "huge" : "", | 20260508 03:43:54.501 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:54.501 | INFO | "malloc" : " 14 GB", | 20260508 03:43:54.501 | INFO | "pinned" : " 29 GB" | 20260508 03:43:54.501 | INFO | } | 20260508 03:43:54.501 | INFO | Perf report for the last 5.14024 seconds: | 20260508 03:43:54.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:54.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.501 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:43:54.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.502 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:43:54.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:54.502 | INFO | Uploaded 36GiB to GPU in 2.43108s (14.8082GiB/s) | 20260508 03:43:54.502 | INFO | Downloaded 34GiB to GPU in 2.4203s (14.0478GiB/s) | 20260508 03:43:59.641 | INFO | Analyzing chunk frames = [4972032,4972544) | 20260508 03:43:59.641 | INFO | MemInfo: { | 20260508 03:43:59.641 | INFO | "gpu" : " 17 GB", | 20260508 03:43:59.641 | INFO | "huge" : "", | 20260508 03:43:59.641 | INFO | "huge_shared" : " 29 GB", | 20260508 03:43:59.641 | INFO | "malloc" : " 14 GB", | 20260508 03:43:59.641 | INFO | "pinned" : " 29 GB" | 20260508 03:43:59.641 | INFO | } | 20260508 03:43:59.641 | INFO | Perf report for the last 5.13956 seconds: | 20260508 03:43:59.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:43:59.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.641 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:43:59.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.641 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 03:43:59.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:43:59.642 | INFO | Uploaded 36GiB to GPU in 2.45033s (14.6919GiB/s) | 20260508 03:43:59.642 | INFO | Downloaded 34GiB to GPU in 2.43595s (13.9576GiB/s) | 20260508 03:43:59.642 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:43:59.642 | INFO | Total mem usage: 16.7 GiB | 20260508 03:43:59.642 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:43:59.642 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16%) | 20260508 03:43:59.642 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:43:59.642 | INFO | High Water Breakdown: | 20260508 03:43:59.642 | INFO | 59.5% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:43:59.642 | INFO | 40.3% : 5.8 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:43:59.642 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:43:59.642 | INFO | ------------------------------------------ | 20260508 03:43:59.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:43:59.642 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:43:59.642 | INFO | Total mem usage: 26.5 GiB | 20260508 03:43:59.642 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:43:59.642 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:43:59.642 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:43:59.642 | INFO | High Water Breakdown: | 20260508 03:43:59.642 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:43:59.642 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:43:59.642 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:43:59.642 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:43:59.642 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:43:59.642 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:43:59.642 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:43:59.642 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:43:59.642 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:43:59.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:43:59.642 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:43:59.642 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:43:59.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:43:59.642 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:43:59.642 | INFO | ------------------------------------------ | 20260508 03:43:59.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:43:59.642 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:43:59.642 | INFO | Total mem usage: 27.8 GiB | 20260508 03:43:59.642 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:43:59.642 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:43:59.642 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:43:59.642 | INFO | High Water Breakdown: | 20260508 03:43:59.642 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:43:59.642 | INFO | ------------------------------------------ | 20260508 03:43:59.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:43:59.642 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:43:59.643 | INFO | Total mem usage: 15.7 GiB | 20260508 03:43:59.643 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:43:59.643 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:43:59.643 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:43:59.643 | INFO | High Water Breakdown: | 20260508 03:43:59.643 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:43:59.643 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:43:59.643 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:43:59.643 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:43:59.643 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:43:59.643 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:43:59.643 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:43:59.643 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:43:59.643 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:43:59.643 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:43:59.643 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:43:59.643 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:43:59.643 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:43:59.643 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:43:59.643 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:43:59.643 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:43:59.643 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:43:59.643 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:43:59.643 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:43:59.643 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:43:59.643 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:43:59.643 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:43:59.643 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:43:59.643 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:43:59.643 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:43:59.643 | INFO | ------------------------------------------ | 20260508 03:43:59.643 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:43:59.643 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:43:59.643 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:43:59.643 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:43:59.643 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.38e+01 / 1.62e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.06e-01, 1.16e-02 / 7.96e-03 | 20260508 03:43:59.643 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.87e-01 / 2.38e-01 | 20260508 03:43:59.643 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.07e-01 / 1.08e-01 | 20260508 03:43:59.643 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 3.86e+00 / 1.97e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.69e+00 / 1.21e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.65e+00, 4.22e-01 / 2.01e-01 | 20260508 03:43:59.643 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.22e+00, 2.63e-01 / 1.49e-01 | 20260508 03:43:59.643 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.65e-01, 5.16e-02 / 2.48e-02 | 20260508 03:43:59.643 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:43:59.643 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:43:59.643 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:43:59.643 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 24.38%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 2.71%, LockDownload -- 2.89e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 03:43:59.643 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260508 03:43:59.760 | INFO | Starting a round of Baz IO | 20260508 03:44:03.188 | INFO | Populating RTMetrics report | 20260508 03:44:03.188 | INFO | RTMetrics report populated | 20260508 03:44:03.188 | INFO | Opening tmp RTMetrics json file | 20260508 03:44:03.200 | INFO | Serializing RTMetrics to json | 20260508 03:44:03.201 | INFO | Serialization done, swapping tmp file now... | 20260508 03:44:03.202 | INFO | RTMetrics done | 20260508 03:44:03.256 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.496 seconds | 20260508 03:44:03.257 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.22% / 0.00% / 58.78% | 20260508 03:44:03.257 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.595021% / 7.94729e-06% / 5.16574e-05% | 20260508 03:44:03.257 | INFO | Baz IO overall: | 20260508 03:44:03.257 | INFO | BytesWritten : 2233295755376 | 20260508 03:44:03.257 | INFO | NumEvents : 1078756276946 | 20260508 03:44:04.783 | INFO | Analyzing chunk frames = [4972544,4973056) | 20260508 03:44:04.783 | INFO | MemInfo: { | 20260508 03:44:04.783 | INFO | "gpu" : " 17 GB", | 20260508 03:44:04.783 | INFO | "huge" : "", | 20260508 03:44:04.783 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:04.783 | INFO | "malloc" : " 14 GB", | 20260508 03:44:04.783 | INFO | "pinned" : " 29 GB" | 20260508 03:44:04.783 | INFO | } | 20260508 03:44:04.783 | INFO | Perf report for the last 5.1425 seconds: | 20260508 03:44:04.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:04.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:04.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:04.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:04.784 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 03:44:04.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:04.784 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 03:44:04.784 | INFO | PRE_HQ : 66.4%, 1.8, 61.0ms, 0 | 20260508 03:44:04.784 | INFO | BAZWRITER : 67.5%, 2.8, 96.4ms, 0 | 20260508 03:44:04.784 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:44:04.784 | INFO | Uploaded 36GiB to GPU in 2.57968s (13.9552GiB/s) | 20260508 03:44:04.784 | INFO | Downloaded 38GiB to GPU in 2.69113s (14.1205GiB/s) | 20260508 03:44:09.922 | INFO | Analyzing chunk frames = [4973056,4973568) | 20260508 03:44:09.922 | INFO | MemInfo: { | 20260508 03:44:09.922 | INFO | "gpu" : " 17 GB", | 20260508 03:44:09.922 | INFO | "huge" : "", | 20260508 03:44:09.922 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:09.922 | INFO | "malloc" : " 14 GB", | 20260508 03:44:09.922 | INFO | "pinned" : " 29 GB" | 20260508 03:44:09.922 | INFO | } | 20260508 03:44:09.922 | INFO | Perf report for the last 5.13858 seconds: | 20260508 03:44:09.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:09.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.922 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 03:44:09.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.922 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:44:09.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:09.923 | INFO | Uploaded 36GiB to GPU in 2.58512s (13.9258GiB/s) | 20260508 03:44:09.923 | INFO | Downloaded 34GiB to GPU in 2.43206s (13.9799GiB/s) | 20260508 03:44:15.056 | INFO | Analyzing chunk frames = [4973568,4974080) | 20260508 03:44:15.056 | INFO | MemInfo: { | 20260508 03:44:15.056 | INFO | "gpu" : " 17 GB", | 20260508 03:44:15.056 | INFO | "huge" : "", | 20260508 03:44:15.056 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:15.056 | INFO | "malloc" : " 14 GB", | 20260508 03:44:15.056 | INFO | "pinned" : " 29 GB" | 20260508 03:44:15.056 | INFO | } | 20260508 03:44:15.056 | INFO | Perf report for the last 5.13393 seconds: | 20260508 03:44:15.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:15.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 03:44:15.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:44:15.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:15.056 | INFO | Uploaded 36GiB to GPU in 2.5614s (14.0548GiB/s) | 20260508 03:44:15.056 | INFO | Downloaded 34GiB to GPU in 2.4079s (14.1202GiB/s) | 20260508 03:44:20.201 | INFO | Analyzing chunk frames = [4974080,4974592) | 20260508 03:44:20.201 | INFO | MemInfo: { | 20260508 03:44:20.201 | INFO | "gpu" : " 17 GB", | 20260508 03:44:20.201 | INFO | "huge" : "", | 20260508 03:44:20.201 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:20.201 | INFO | "malloc" : " 14 GB", | 20260508 03:44:20.201 | INFO | "pinned" : " 29 GB" | 20260508 03:44:20.201 | INFO | } | 20260508 03:44:20.201 | INFO | Perf report for the last 5.14516 seconds: | 20260508 03:44:20.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:20.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.201 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 03:44:20.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.201 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:44:20.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:20.202 | INFO | Uploaded 36GiB to GPU in 2.55874s (14.0694GiB/s) | 20260508 03:44:20.202 | INFO | Downloaded 34GiB to GPU in 2.49076s (13.6504GiB/s) | 20260508 03:44:25.341 | INFO | Analyzing chunk frames = [4974592,4975104) | 20260508 03:44:25.341 | INFO | MemInfo: { | 20260508 03:44:25.341 | INFO | "gpu" : " 17 GB", | 20260508 03:44:25.341 | INFO | "huge" : "", | 20260508 03:44:25.341 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:25.341 | INFO | "malloc" : " 14 GB", | 20260508 03:44:25.341 | INFO | "pinned" : " 29 GB" | 20260508 03:44:25.341 | INFO | } | 20260508 03:44:25.341 | INFO | Perf report for the last 5.13972 seconds: | 20260508 03:44:25.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:25.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 03:44:25.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:44:25.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:25.341 | INFO | Uploaded 36GiB to GPU in 2.41076s (14.933GiB/s) | 20260508 03:44:25.341 | INFO | Downloaded 34GiB to GPU in 2.55426s (13.3111GiB/s) | 20260508 03:44:30.479 | INFO | Analyzing chunk frames = [4975104,4975616) | 20260508 03:44:30.479 | INFO | MemInfo: { | 20260508 03:44:30.479 | INFO | "gpu" : " 17 GB", | 20260508 03:44:30.479 | INFO | "huge" : "", | 20260508 03:44:30.479 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:30.479 | INFO | "malloc" : " 14 GB", | 20260508 03:44:30.480 | INFO | "pinned" : " 29 GB" | 20260508 03:44:30.480 | INFO | } | 20260508 03:44:30.480 | INFO | Perf report for the last 5.13867 seconds: | 20260508 03:44:30.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:30.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:44:30.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:44:30.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:30.480 | INFO | Uploaded 36GiB to GPU in 2.43633s (14.7763GiB/s) | 20260508 03:44:30.480 | INFO | Downloaded 34GiB to GPU in 2.41984s (14.0505GiB/s) | 20260508 03:44:35.613 | INFO | Analyzing chunk frames = [4975616,4976128) | 20260508 03:44:35.613 | INFO | MemInfo: { | 20260508 03:44:35.613 | INFO | "gpu" : " 17 GB", | 20260508 03:44:35.613 | INFO | "huge" : "", | 20260508 03:44:35.613 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:35.613 | INFO | "malloc" : " 14 GB", | 20260508 03:44:35.613 | INFO | "pinned" : " 29 GB" | 20260508 03:44:35.613 | INFO | } | 20260508 03:44:35.613 | INFO | Perf report for the last 5.13366 seconds: | 20260508 03:44:35.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:35.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.613 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:44:35.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.613 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:44:35.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:35.614 | INFO | Uploaded 36GiB to GPU in 2.42136s (14.8677GiB/s) | 20260508 03:44:35.614 | INFO | Downloaded 34GiB to GPU in 2.41911s (14.0548GiB/s) | 20260508 03:44:40.751 | INFO | Analyzing chunk frames = [4976128,4976640) | 20260508 03:44:40.751 | INFO | MemInfo: { | 20260508 03:44:40.751 | INFO | "gpu" : " 17 GB", | 20260508 03:44:40.751 | INFO | "huge" : "", | 20260508 03:44:40.751 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:40.751 | INFO | "malloc" : " 14 GB", | 20260508 03:44:40.752 | INFO | "pinned" : " 29 GB" | 20260508 03:44:40.752 | INFO | } | 20260508 03:44:40.752 | INFO | Perf report for the last 5.13837 seconds: | 20260508 03:44:40.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:40.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.752 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:44:40.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.752 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 03:44:40.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:40.753 | INFO | Uploaded 36GiB to GPU in 2.44706s (14.7115GiB/s) | 20260508 03:44:40.753 | INFO | Downloaded 34GiB to GPU in 2.42276s (14.0336GiB/s) | 20260508 03:44:40.867 | INFO | Starting a round of Baz IO | 20260508 03:44:44.290 | INFO | Populating RTMetrics report | 20260508 03:44:44.290 | INFO | RTMetrics report populated | 20260508 03:44:44.290 | INFO | Opening tmp RTMetrics json file | 20260508 03:44:44.291 | INFO | Serializing RTMetrics to json | 20260508 03:44:44.292 | INFO | Serialization done, swapping tmp file now... | 20260508 03:44:44.292 | INFO | RTMetrics done | 20260508 03:44:44.363 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.496 seconds | 20260508 03:44:44.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.21% / 0.00% / 58.78% | 20260508 03:44:44.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.593019% / 0% / 5.96046e-05% | 20260508 03:44:44.363 | INFO | Baz IO overall: | 20260508 03:44:44.363 | INFO | BytesWritten : 2235024298801 | 20260508 03:44:44.363 | INFO | NumEvents : 1079583120010 | 20260508 03:44:45.893 | INFO | Analyzing chunk frames = [4976640,4977152) | 20260508 03:44:45.893 | INFO | MemInfo: { | 20260508 03:44:45.893 | INFO | "gpu" : " 17 GB", | 20260508 03:44:45.893 | INFO | "huge" : "", | 20260508 03:44:45.893 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:45.893 | INFO | "malloc" : " 14 GB", | 20260508 03:44:45.893 | INFO | "pinned" : " 29 GB" | 20260508 03:44:45.893 | INFO | } | 20260508 03:44:45.893 | INFO | Perf report for the last 5.14138 seconds: | 20260508 03:44:45.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:45.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:45.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:45.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:45.893 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260508 03:44:45.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:45.893 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 03:44:45.893 | INFO | PRE_HQ : 66.7%, 1.8, 61.9ms, 0 | 20260508 03:44:45.893 | INFO | BAZWRITER : 67.4%, 2.8, 97.9ms, 0 | 20260508 03:44:45.893 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 03:44:45.893 | INFO | Uploaded 36GiB to GPU in 2.56352s (14.0432GiB/s) | 20260508 03:44:45.893 | INFO | Downloaded 38GiB to GPU in 2.70493s (14.0484GiB/s) | 20260508 03:44:51.039 | INFO | Analyzing chunk frames = [4977152,4977664) | 20260508 03:44:51.039 | INFO | MemInfo: { | 20260508 03:44:51.039 | INFO | "gpu" : " 17 GB", | 20260508 03:44:51.039 | INFO | "huge" : "", | 20260508 03:44:51.039 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:51.039 | INFO | "malloc" : " 14 GB", | 20260508 03:44:51.039 | INFO | "pinned" : " 29 GB" | 20260508 03:44:51.039 | INFO | } | 20260508 03:44:51.039 | INFO | Perf report for the last 5.14575 seconds: | 20260508 03:44:51.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:51.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 03:44:51.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:44:51.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:51.039 | INFO | Uploaded 36GiB to GPU in 2.52888s (14.2355GiB/s) | 20260508 03:44:51.039 | INFO | Downloaded 34GiB to GPU in 2.43531s (13.9613GiB/s) | 20260508 03:44:56.178 | INFO | Analyzing chunk frames = [4977664,4978176) | 20260508 03:44:56.178 | INFO | MemInfo: { | 20260508 03:44:56.178 | INFO | "gpu" : " 17 GB", | 20260508 03:44:56.178 | INFO | "huge" : "", | 20260508 03:44:56.178 | INFO | "huge_shared" : " 29 GB", | 20260508 03:44:56.178 | INFO | "malloc" : " 14 GB", | 20260508 03:44:56.178 | INFO | "pinned" : " 29 GB" | 20260508 03:44:56.178 | INFO | } | 20260508 03:44:56.178 | INFO | Perf report for the last 5.13932 seconds: | 20260508 03:44:56.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:44:56.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 03:44:56.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:44:56.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:44:56.178 | INFO | Uploaded 36GiB to GPU in 2.51597s (14.3086GiB/s) | 20260508 03:44:56.179 | INFO | Downloaded 34GiB to GPU in 2.4436s (13.9139GiB/s) | 20260508 03:45:01.316 | INFO | Analyzing chunk frames = [4978176,4978688) | 20260508 03:45:01.316 | INFO | MemInfo: { | 20260508 03:45:01.316 | INFO | "gpu" : " 17 GB", | 20260508 03:45:01.316 | INFO | "huge" : "", | 20260508 03:45:01.316 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:01.316 | INFO | "malloc" : " 14 GB", | 20260508 03:45:01.316 | INFO | "pinned" : " 29 GB" | 20260508 03:45:01.316 | INFO | } | 20260508 03:45:01.316 | INFO | Perf report for the last 5.13827 seconds: | 20260508 03:45:01.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:01.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.316 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 03:45:01.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.317 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:45:01.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:01.317 | INFO | Uploaded 36GiB to GPU in 2.53579s (14.1967GiB/s) | 20260508 03:45:01.317 | INFO | Downloaded 34GiB to GPU in 2.43312s (13.9738GiB/s) | 20260508 03:45:06.455 | INFO | Analyzing chunk frames = [4978688,4979200) | 20260508 03:45:06.455 | INFO | MemInfo: { | 20260508 03:45:06.455 | INFO | "gpu" : " 17 GB", | 20260508 03:45:06.455 | INFO | "huge" : "", | 20260508 03:45:06.455 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:06.455 | INFO | "malloc" : " 14 GB", | 20260508 03:45:06.455 | INFO | "pinned" : " 29 GB" | 20260508 03:45:06.455 | INFO | } | 20260508 03:45:06.456 | INFO | Perf report for the last 5.1391 seconds: | 20260508 03:45:06.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:06.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 03:45:06.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 03:45:06.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:06.456 | INFO | Uploaded 36GiB to GPU in 2.28535s (15.7525GiB/s) | 20260508 03:45:06.456 | INFO | Downloaded 34GiB to GPU in 2.56029s (13.2797GiB/s) | 20260508 03:45:11.595 | INFO | Analyzing chunk frames = [4979200,4979712) | 20260508 03:45:11.595 | INFO | MemInfo: { | 20260508 03:45:11.595 | INFO | "gpu" : " 17 GB", | 20260508 03:45:11.595 | INFO | "huge" : "", | 20260508 03:45:11.595 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:11.595 | INFO | "malloc" : " 14 GB", | 20260508 03:45:11.595 | INFO | "pinned" : " 29 GB" | 20260508 03:45:11.595 | INFO | } | 20260508 03:45:11.596 | INFO | Perf report for the last 5.13996 seconds: | 20260508 03:45:11.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:11.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:45:11.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:45:11.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:11.596 | INFO | Uploaded 36GiB to GPU in 2.41991s (14.8766GiB/s) | 20260508 03:45:11.596 | INFO | Downloaded 34GiB to GPU in 2.42679s (14.0103GiB/s) | 20260508 03:45:16.731 | INFO | Analyzing chunk frames = [4979712,4980224) | 20260508 03:45:16.731 | INFO | MemInfo: { | 20260508 03:45:16.731 | INFO | "gpu" : " 17 GB", | 20260508 03:45:16.731 | INFO | "huge" : "", | 20260508 03:45:16.731 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:16.731 | INFO | "malloc" : " 14 GB", | 20260508 03:45:16.731 | INFO | "pinned" : " 29 GB" | 20260508 03:45:16.731 | INFO | } | 20260508 03:45:16.731 | INFO | Perf report for the last 5.13548 seconds: | 20260508 03:45:16.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:16.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.731 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:45:16.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.731 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 03:45:16.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:16.732 | INFO | Uploaded 36GiB to GPU in 2.41652s (14.8975GiB/s) | 20260508 03:45:16.732 | INFO | Downloaded 34GiB to GPU in 2.42962s (13.994GiB/s) | 20260508 03:45:21.868 | INFO | Analyzing chunk frames = [4980224,4980736) | 20260508 03:45:21.868 | INFO | MemInfo: { | 20260508 03:45:21.868 | INFO | "gpu" : " 17 GB", | 20260508 03:45:21.868 | INFO | "huge" : "", | 20260508 03:45:21.868 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:21.868 | INFO | "malloc" : " 14 GB", | 20260508 03:45:21.868 | INFO | "pinned" : " 29 GB" | 20260508 03:45:21.868 | INFO | } | 20260508 03:45:21.868 | INFO | Perf report for the last 5.13712 seconds: | 20260508 03:45:21.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:21.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.868 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:45:21.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.868 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 03:45:21.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:21.869 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:45:21.869 | INFO | Total mem usage: 16.7 GiB | 20260508 03:45:21.869 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:45:21.869 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:45:21.869 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:45:21.869 | INFO | High Water Breakdown: | 20260508 03:45:21.869 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:45:21.869 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:45:21.869 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:45:21.869 | INFO | ------------------------------------------ | 20260508 03:45:21.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:45:21.869 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:45:21.869 | INFO | Total mem usage: 26.5 GiB | 20260508 03:45:21.869 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:45:21.869 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:45:21.869 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:45:21.869 | INFO | Uploaded 36GiB to GPU in 2.44592s (14.7184GiB/s) | 20260508 03:45:21.869 | INFO | High Water Breakdown: | 20260508 03:45:21.869 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:45:21.869 | INFO | Downloaded 34GiB to GPU in 2.43795s (13.9462GiB/s) | 20260508 03:45:21.869 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:45:21.869 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:45:21.869 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:45:21.869 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:45:21.869 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:45:21.869 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:45:21.869 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:45:21.869 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:45:21.869 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:45:21.869 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:45:21.869 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:45:21.869 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:45:21.869 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:45:21.869 | INFO | ------------------------------------------ | 20260508 03:45:21.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:45:21.869 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:45:21.869 | INFO | Total mem usage: 27.8 GiB | 20260508 03:45:21.869 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:45:21.869 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:45:21.869 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:45:21.869 | INFO | High Water Breakdown: | 20260508 03:45:21.869 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:45:21.869 | INFO | ------------------------------------------ | 20260508 03:45:21.869 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:45:21.869 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:45:21.869 | INFO | Total mem usage: 15.7 GiB | 20260508 03:45:21.869 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:45:21.869 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:45:21.869 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:45:21.870 | INFO | High Water Breakdown: | 20260508 03:45:21.870 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:45:21.870 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:45:21.870 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:45:21.870 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:45:21.870 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:45:21.870 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:45:21.870 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:45:21.870 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:45:21.870 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:45:21.870 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:45:21.870 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:45:21.870 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:45:21.870 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:45:21.870 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:45:21.870 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:45:21.870 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:45:21.870 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:45:21.870 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:45:21.870 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:45:21.870 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:45:21.870 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:45:21.870 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:45:21.870 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:45:21.870 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:45:21.870 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:45:21.870 | INFO | ------------------------------------------ | 20260508 03:45:21.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:45:21.870 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:45:21.870 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:45:21.870 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:45:21.870 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.38e+01 / 1.79e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.04e-01, 1.14e-02 / 7.93e-03 | 20260508 03:45:21.870 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.48e+00, 5.95e-01 / 2.16e-01 | 20260508 03:45:21.870 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.33e-01 | 20260508 03:45:21.870 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.57e+01, 3.67e+00 / 2.17e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.45e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.40e+00 / 2.64e+00, 4.19e-01 / 2.07e-01 | 20260508 03:45:21.870 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.25e+00, 2.65e-01 / 1.60e-01 | 20260508 03:45:21.870 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.62e-01, 5.17e-02 / 2.47e-02 | 20260508 03:45:21.870 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:45:21.870 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:45:21.870 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:45:21.870 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 24.31%, Upload -- 2.59e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 24.31%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 30.83%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 2.19%, LockUpload -- 2.32e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 2.71%, LockDownload -- 2.88e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:45:21.870 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260508 03:45:21.982 | INFO | Starting a round of Baz IO | 20260508 03:45:25.407 | INFO | Populating RTMetrics report | 20260508 03:45:25.407 | INFO | RTMetrics report populated | 20260508 03:45:25.407 | INFO | Opening tmp RTMetrics json file | 20260508 03:45:25.421 | INFO | Serializing RTMetrics to json | 20260508 03:45:25.422 | INFO | Serialization done, swapping tmp file now... | 20260508 03:45:25.423 | INFO | RTMetrics done | 20260508 03:45:25.478 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.495 seconds | 20260508 03:45:25.478 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.21% / 0.00% / 58.79% | 20260508 03:45:25.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.590932% / 0% / 4.76837e-05% | 20260508 03:45:25.478 | INFO | Baz IO overall: | 20260508 03:45:25.478 | INFO | BytesWritten : 2236751529679 | 20260508 03:45:25.478 | INFO | NumEvents : 1080409017160 | 20260508 03:45:27.007 | INFO | Analyzing chunk frames = [4980736,4981248) | 20260508 03:45:27.007 | INFO | MemInfo: { | 20260508 03:45:27.007 | INFO | "gpu" : " 17 GB", | 20260508 03:45:27.007 | INFO | "huge" : "", | 20260508 03:45:27.007 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:27.007 | INFO | "malloc" : " 14 GB", | 20260508 03:45:27.007 | INFO | "pinned" : " 29 GB" | 20260508 03:45:27.007 | INFO | } | 20260508 03:45:27.007 | INFO | Perf report for the last 5.13909 seconds: | 20260508 03:45:27.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:27.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:27.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:27.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:27.007 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 03:45:27.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:27.007 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 03:45:27.007 | INFO | PRE_HQ : 66.4%, 1.8, 61.8ms, 0 | 20260508 03:45:27.007 | INFO | BAZWRITER : 67.7%, 2.8, 97.7ms, 0 | 20260508 03:45:27.007 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:45:27.008 | INFO | Uploaded 36GiB to GPU in 2.52486s (14.2582GiB/s) | 20260508 03:45:27.008 | INFO | Downloaded 38GiB to GPU in 2.69607s (14.0946GiB/s) | 20260508 03:45:32.154 | INFO | Analyzing chunk frames = [4981248,4981760) | 20260508 03:45:32.154 | INFO | MemInfo: { | 20260508 03:45:32.154 | INFO | "gpu" : " 17 GB", | 20260508 03:45:32.154 | INFO | "huge" : "", | 20260508 03:45:32.154 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:32.154 | INFO | "malloc" : " 14 GB", | 20260508 03:45:32.154 | INFO | "pinned" : " 29 GB" | 20260508 03:45:32.154 | INFO | } | 20260508 03:45:32.154 | INFO | Perf report for the last 5.14713 seconds: | 20260508 03:45:32.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:32.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.154 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:45:32.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.155 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:45:32.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:32.155 | INFO | Uploaded 36GiB to GPU in 2.52386s (14.2639GiB/s) | 20260508 03:45:32.155 | INFO | Downloaded 34GiB to GPU in 2.43123s (13.9847GiB/s) | 20260508 03:45:37.294 | INFO | Analyzing chunk frames = [4981760,4982272) | 20260508 03:45:37.294 | INFO | MemInfo: { | 20260508 03:45:37.294 | INFO | "gpu" : " 17 GB", | 20260508 03:45:37.294 | INFO | "huge" : "", | 20260508 03:45:37.294 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:37.295 | INFO | "malloc" : " 14 GB", | 20260508 03:45:37.295 | INFO | "pinned" : " 29 GB" | 20260508 03:45:37.295 | INFO | } | 20260508 03:45:37.295 | INFO | Perf report for the last 5.14029 seconds: | 20260508 03:45:37.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:37.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 03:45:37.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:45:37.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:37.295 | INFO | Uploaded 36GiB to GPU in 2.51796s (14.2973GiB/s) | 20260508 03:45:37.295 | INFO | Downloaded 34GiB to GPU in 2.41366s (14.0865GiB/s) | 20260508 03:45:42.428 | INFO | Analyzing chunk frames = [4982272,4982784) | 20260508 03:45:42.428 | INFO | MemInfo: { | 20260508 03:45:42.428 | INFO | "gpu" : " 17 GB", | 20260508 03:45:42.428 | INFO | "huge" : "", | 20260508 03:45:42.428 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:42.428 | INFO | "malloc" : " 14 GB", | 20260508 03:45:42.428 | INFO | "pinned" : " 29 GB" | 20260508 03:45:42.428 | INFO | } | 20260508 03:45:42.428 | INFO | Perf report for the last 5.13302 seconds: | 20260508 03:45:42.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:42.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | ANALYSIS : 67.1%, 3.0, 103.7ms, 0 | 20260508 03:45:42.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:45:42.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:42.428 | INFO | Uploaded 36GiB to GPU in 2.55559s (14.0868GiB/s) | 20260508 03:45:42.429 | INFO | Downloaded 34GiB to GPU in 2.42374s (14.0279GiB/s) | 20260508 03:45:47.562 | INFO | Analyzing chunk frames = [4982784,4983296) | 20260508 03:45:47.562 | INFO | MemInfo: { | 20260508 03:45:47.562 | INFO | "gpu" : " 17 GB", | 20260508 03:45:47.562 | INFO | "huge" : "", | 20260508 03:45:47.562 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:47.562 | INFO | "malloc" : " 14 GB", | 20260508 03:45:47.562 | INFO | "pinned" : " 29 GB" | 20260508 03:45:47.563 | INFO | } | 20260508 03:45:47.563 | INFO | Perf report for the last 5.13484 seconds: | 20260508 03:45:47.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:47.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 03:45:47.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:45:47.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:47.563 | INFO | Uploaded 36GiB to GPU in 2.39272s (15.0456GiB/s) | 20260508 03:45:47.563 | INFO | Downloaded 34GiB to GPU in 2.5436s (13.3669GiB/s) | 20260508 03:45:52.707 | INFO | Analyzing chunk frames = [4983296,4983808) | 20260508 03:45:52.707 | INFO | MemInfo: { | 20260508 03:45:52.707 | INFO | "gpu" : " 17 GB", | 20260508 03:45:52.707 | INFO | "huge" : "", | 20260508 03:45:52.707 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:52.707 | INFO | "malloc" : " 14 GB", | 20260508 03:45:52.707 | INFO | "pinned" : " 29 GB" | 20260508 03:45:52.707 | INFO | } | 20260508 03:45:52.707 | INFO | Perf report for the last 5.14482 seconds: | 20260508 03:45:52.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:52.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.707 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 03:45:52.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.708 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:45:52.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:52.708 | INFO | Uploaded 36GiB to GPU in 2.40833s (14.9481GiB/s) | 20260508 03:45:52.708 | INFO | Downloaded 34GiB to GPU in 2.41956s (14.0521GiB/s) | 20260508 03:45:57.847 | INFO | Analyzing chunk frames = [4983808,4984320) | 20260508 03:45:57.847 | INFO | MemInfo: { | 20260508 03:45:57.847 | INFO | "gpu" : " 17 GB", | 20260508 03:45:57.847 | INFO | "huge" : "", | 20260508 03:45:57.847 | INFO | "huge_shared" : " 29 GB", | 20260508 03:45:57.848 | INFO | "malloc" : " 14 GB", | 20260508 03:45:57.848 | INFO | "pinned" : " 29 GB" | 20260508 03:45:57.848 | INFO | } | 20260508 03:45:57.848 | INFO | Perf report for the last 5.14029 seconds: | 20260508 03:45:57.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:45:57.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:45:57.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 03:45:57.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:45:57.848 | INFO | Uploaded 36GiB to GPU in 2.44621s (14.7167GiB/s) | 20260508 03:45:57.848 | INFO | Downloaded 34GiB to GPU in 2.43638s (13.9551GiB/s) | 20260508 03:46:02.992 | INFO | Analyzing chunk frames = [4984320,4984832) | 20260508 03:46:02.992 | INFO | MemInfo: { | 20260508 03:46:02.992 | INFO | "gpu" : " 17 GB", | 20260508 03:46:02.992 | INFO | "huge" : "", | 20260508 03:46:02.992 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:02.992 | INFO | "malloc" : " 14 GB", | 20260508 03:46:02.992 | INFO | "pinned" : " 29 GB" | 20260508 03:46:02.992 | INFO | } | 20260508 03:46:02.992 | INFO | Perf report for the last 5.1445 seconds: | 20260508 03:46:02.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:02.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:46:02.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 03:46:02.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:02.992 | INFO | Uploaded 36GiB to GPU in 2.45683s (14.653GiB/s) | 20260508 03:46:02.993 | INFO | Downloaded 34GiB to GPU in 2.42839s (14.0011GiB/s) | 20260508 03:46:03.089 | INFO | Starting a round of Baz IO | 20260508 03:46:06.536 | INFO | Populating RTMetrics report | 20260508 03:46:06.537 | INFO | RTMetrics report populated | 20260508 03:46:06.537 | INFO | Opening tmp RTMetrics json file | 20260508 03:46:06.537 | INFO | Serializing RTMetrics to json | 20260508 03:46:06.538 | INFO | Serialization done, swapping tmp file now... | 20260508 03:46:06.539 | INFO | RTMetrics done | 20260508 03:46:06.607 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.517 seconds | 20260508 03:46:06.607 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.21% / 0.00% / 58.79% | 20260508 03:46:06.607 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.589987% / 0% / 3.57628e-05% | 20260508 03:46:06.607 | INFO | Baz IO overall: | 20260508 03:46:06.607 | INFO | BytesWritten : 2238478940220 | 20260508 03:46:06.607 | INFO | NumEvents : 1081235058229 | 20260508 03:46:08.126 | INFO | Analyzing chunk frames = [4984832,4985344) | 20260508 03:46:08.126 | INFO | MemInfo: { | 20260508 03:46:08.126 | INFO | "gpu" : " 17 GB", | 20260508 03:46:08.126 | INFO | "huge" : "", | 20260508 03:46:08.126 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:08.126 | INFO | "malloc" : " 14 GB", | 20260508 03:46:08.126 | INFO | "pinned" : " 29 GB" | 20260508 03:46:08.126 | INFO | } | 20260508 03:46:08.126 | INFO | Perf report for the last 5.13411 seconds: | 20260508 03:46:08.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:08.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:08.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:08.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:08.126 | INFO | ANALYSIS : 68.7%, 3.0, 106.7ms, 0 | 20260508 03:46:08.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:08.126 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 03:46:08.126 | INFO | PRE_HQ : 66.1%, 1.6, 55.6ms, 0 | 20260508 03:46:08.126 | INFO | BAZWRITER : 67.9%, 2.7, 92.9ms, 0 | 20260508 03:46:08.126 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 03:46:08.127 | INFO | Uploaded 36GiB to GPU in 2.54653s (14.1369GiB/s) | 20260508 03:46:08.127 | INFO | Downloaded 38GiB to GPU in 2.69958s (14.0763GiB/s) | 20260508 03:46:13.262 | INFO | Analyzing chunk frames = [4985344,4985856) | 20260508 03:46:13.262 | INFO | MemInfo: { | 20260508 03:46:13.262 | INFO | "gpu" : " 17 GB", | 20260508 03:46:13.262 | INFO | "huge" : "", | 20260508 03:46:13.262 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:13.262 | INFO | "malloc" : " 14 GB", | 20260508 03:46:13.262 | INFO | "pinned" : " 29 GB" | 20260508 03:46:13.262 | INFO | } | 20260508 03:46:13.263 | INFO | Perf report for the last 5.13618 seconds: | 20260508 03:46:13.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:13.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:46:13.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:46:13.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:13.263 | INFO | Uploaded 36GiB to GPU in 2.58637s (13.9191GiB/s) | 20260508 03:46:13.263 | INFO | Downloaded 34GiB to GPU in 2.42229s (14.0363GiB/s) | 20260508 03:46:18.403 | INFO | Analyzing chunk frames = [4985856,4986368) | 20260508 03:46:18.403 | INFO | MemInfo: { | 20260508 03:46:18.403 | INFO | "gpu" : " 17 GB", | 20260508 03:46:18.403 | INFO | "huge" : "", | 20260508 03:46:18.403 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:18.403 | INFO | "malloc" : " 14 GB", | 20260508 03:46:18.403 | INFO | "pinned" : " 29 GB" | 20260508 03:46:18.403 | INFO | } | 20260508 03:46:18.403 | INFO | Perf report for the last 5.14084 seconds: | 20260508 03:46:18.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:18.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.403 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:46:18.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.403 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:46:18.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:18.404 | INFO | Uploaded 36GiB to GPU in 2.58669s (13.9174GiB/s) | 20260508 03:46:18.404 | INFO | Downloaded 34GiB to GPU in 2.49476s (13.6285GiB/s) | 20260508 03:46:23.543 | INFO | Analyzing chunk frames = [4986368,4986880) | 20260508 03:46:23.543 | INFO | MemInfo: { | 20260508 03:46:23.543 | INFO | "gpu" : " 17 GB", | 20260508 03:46:23.543 | INFO | "huge" : "", | 20260508 03:46:23.543 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:23.543 | INFO | "malloc" : " 14 GB", | 20260508 03:46:23.543 | INFO | "pinned" : " 29 GB" | 20260508 03:46:23.543 | INFO | } | 20260508 03:46:23.543 | INFO | Perf report for the last 5.13985 seconds: | 20260508 03:46:23.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:23.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.543 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 03:46:23.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.543 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:46:23.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:23.544 | INFO | Uploaded 36GiB to GPU in 2.56056s (14.0594GiB/s) | 20260508 03:46:23.544 | INFO | Downloaded 34GiB to GPU in 2.49232s (13.6419GiB/s) | 20260508 03:46:28.679 | INFO | Analyzing chunk frames = [4986880,4987392) | 20260508 03:46:28.679 | INFO | MemInfo: { | 20260508 03:46:28.679 | INFO | "gpu" : " 17 GB", | 20260508 03:46:28.679 | INFO | "huge" : "", | 20260508 03:46:28.679 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:28.679 | INFO | "malloc" : " 14 GB", | 20260508 03:46:28.679 | INFO | "pinned" : " 29 GB" | 20260508 03:46:28.679 | INFO | } | 20260508 03:46:28.679 | INFO | Perf report for the last 5.13568 seconds: | 20260508 03:46:28.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:28.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:46:28.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:46:28.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:28.679 | INFO | Uploaded 36GiB to GPU in 2.41924s (14.8807GiB/s) | 20260508 03:46:28.680 | INFO | Downloaded 34GiB to GPU in 2.56065s (13.2779GiB/s) | 20260508 03:46:33.819 | INFO | Analyzing chunk frames = [4987392,4987904) | 20260508 03:46:33.819 | INFO | MemInfo: { | 20260508 03:46:33.819 | INFO | "gpu" : " 17 GB", | 20260508 03:46:33.819 | INFO | "huge" : "", | 20260508 03:46:33.819 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:33.819 | INFO | "malloc" : " 14 GB", | 20260508 03:46:33.819 | INFO | "pinned" : " 29 GB" | 20260508 03:46:33.819 | INFO | } | 20260508 03:46:33.820 | INFO | Perf report for the last 5.14051 seconds: | 20260508 03:46:33.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:33.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 03:46:33.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 03:46:33.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:33.820 | INFO | Uploaded 36GiB to GPU in 2.42598s (14.8394GiB/s) | 20260508 03:46:33.820 | INFO | Downloaded 34GiB to GPU in 2.42751s (14.0061GiB/s) | 20260508 03:46:38.962 | INFO | Analyzing chunk frames = [4987904,4988416) | 20260508 03:46:38.962 | INFO | MemInfo: { | 20260508 03:46:38.962 | INFO | "gpu" : " 17 GB", | 20260508 03:46:38.962 | INFO | "huge" : "", | 20260508 03:46:38.962 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:38.962 | INFO | "malloc" : " 14 GB", | 20260508 03:46:38.962 | INFO | "pinned" : " 29 GB" | 20260508 03:46:38.962 | INFO | } | 20260508 03:46:38.963 | INFO | Perf report for the last 5.14305 seconds: | 20260508 03:46:38.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:38.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:46:38.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:46:38.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:38.963 | INFO | Uploaded 36GiB to GPU in 2.44763s (14.7081GiB/s) | 20260508 03:46:38.963 | INFO | Downloaded 34GiB to GPU in 2.43678s (13.9528GiB/s) | 20260508 03:46:44.104 | INFO | Analyzing chunk frames = [4988416,4988928) | 20260508 03:46:44.104 | INFO | MemInfo: { | 20260508 03:46:44.104 | INFO | "gpu" : " 17 GB", | 20260508 03:46:44.104 | INFO | "huge" : "", | 20260508 03:46:44.104 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:44.104 | INFO | "malloc" : " 14 GB", | 20260508 03:46:44.104 | INFO | "pinned" : " 29 GB" | 20260508 03:46:44.104 | INFO | } | 20260508 03:46:44.104 | INFO | Perf report for the last 5.14172 seconds: | 20260508 03:46:44.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:44.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.104 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:46:44.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.105 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 03:46:44.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:44.105 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:46:44.105 | INFO | Total mem usage: 16.7 GiB | 20260508 03:46:44.105 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:46:44.105 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:46:44.105 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:46:44.105 | INFO | High Water Breakdown: | 20260508 03:46:44.105 | INFO | 59.6% : 8.56 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:46:44.105 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:46:44.105 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:46:44.105 | INFO | ------------------------------------------ | 20260508 03:46:44.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:46:44.105 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:46:44.105 | INFO | Total mem usage: 26.5 GiB | 20260508 03:46:44.105 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:46:44.105 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:46:44.105 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:46:44.105 | INFO | High Water Breakdown: | 20260508 03:46:44.105 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:46:44.105 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:46:44.105 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:46:44.105 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:46:44.105 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:46:44.105 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:46:44.105 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:46:44.105 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:46:44.105 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:46:44.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:46:44.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:46:44.105 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:46:44.105 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:46:44.105 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:46:44.105 | INFO | ------------------------------------------ | 20260508 03:46:44.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:46:44.105 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:46:44.105 | INFO | Total mem usage: 27.8 GiB | 20260508 03:46:44.105 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:46:44.105 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:46:44.105 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:46:44.105 | INFO | High Water Breakdown: | 20260508 03:46:44.105 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:46:44.105 | INFO | ------------------------------------------ | 20260508 03:46:44.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:46:44.105 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:46:44.105 | INFO | Total mem usage: 15.7 GiB | 20260508 03:46:44.105 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:46:44.105 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:46:44.105 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 03:46:44.105 | INFO | High Water Breakdown: | 20260508 03:46:44.105 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:46:44.105 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:46:44.105 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:46:44.105 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:46:44.105 | INFO | Uploaded 36GiB to GPU in 2.44701s (14.7118GiB/s) | 20260508 03:46:44.105 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:46:44.106 | INFO | Downloaded 34GiB to GPU in 2.42319s (14.0311GiB/s) | 20260508 03:46:44.106 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:46:44.106 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:46:44.106 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:46:44.106 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:46:44.106 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:46:44.106 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:46:44.106 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:46:44.106 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:46:44.106 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:46:44.106 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:46:44.106 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:46:44.106 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:46:44.106 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:46:44.106 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:46:44.106 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:46:44.106 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:46:44.106 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:46:44.106 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:46:44.106 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:46:44.106 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:46:44.106 | INFO | ------------------------------------------ | 20260508 03:46:44.106 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:46:44.106 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:46:44.106 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:46:44.106 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:46:44.106 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.58e+01, 1.40e+01 / 1.64e+00 | 20260508 03:46:44.106 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.10e-02 / 4.85e-03 | 20260508 03:46:44.106 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.47e+00, 5.91e-01 / 2.18e-01 | 20260508 03:46:44.106 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.26e-01 | 20260508 03:46:44.106 | INFO | PerfReport: 16.00%, DME -- 4.20e+01 / 4.56e+01, 3.95e+00 / 2.04e+00 | 20260508 03:46:44.106 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.36e+00 | 20260508 03:46:44.106 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.18e-01 / 2.42e-01 | 20260508 03:46:44.106 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.24e+00, 2.66e-01 / 1.37e-01 | 20260508 03:46:44.106 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.63e-01, 5.13e-02 / 2.22e-02 | 20260508 03:46:44.106 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:46:44.106 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:46:44.106 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:46:44.106 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 03:46:44.106 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 03:46:44.106 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 03:46:44.107 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:46:44.107 | INFO | PerfReport: 2.26%, LockUpload -- 2.41e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 03:46:44.107 | INFO | PerfReport: 2.63%, LockDownload -- 2.80e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 03:46:44.107 | INFO | PerfReport: 15.67%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:46:44.107 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 2.73e-02 / 0.00e+00 | 20260508 03:46:44.219 | INFO | Starting a round of Baz IO | 20260508 03:46:47.641 | INFO | Populating RTMetrics report | 20260508 03:46:47.642 | INFO | RTMetrics report populated | 20260508 03:46:47.642 | INFO | Opening tmp RTMetrics json file | 20260508 03:46:47.655 | INFO | Serializing RTMetrics to json | 20260508 03:46:47.657 | INFO | Serialization done, swapping tmp file now... | 20260508 03:46:47.657 | INFO | RTMetrics done | 20260508 03:46:47.712 | INFO | Finished a round of Baz IO: Wrote 1.607 GiB in 3.493 seconds | 20260508 03:46:47.712 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.21% / 0.00% / 58.79% | 20260508 03:46:47.712 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.58831% / 0% / 5.5631e-05% | 20260508 03:46:47.712 | INFO | Baz IO overall: | 20260508 03:46:47.712 | INFO | BytesWritten : 2240204831944 | 20260508 03:46:47.712 | INFO | NumEvents : 1082060134223 | 20260508 03:46:49.241 | INFO | Analyzing chunk frames = [4988928,4989440) | 20260508 03:46:49.241 | INFO | MemInfo: { | 20260508 03:46:49.241 | INFO | "gpu" : " 17 GB", | 20260508 03:46:49.241 | INFO | "huge" : "", | 20260508 03:46:49.241 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:49.241 | INFO | "malloc" : " 14 GB", | 20260508 03:46:49.241 | INFO | "pinned" : " 29 GB" | 20260508 03:46:49.241 | INFO | } | 20260508 03:46:49.241 | INFO | Perf report for the last 5.13647 seconds: | 20260508 03:46:49.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:49.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:49.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:49.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:49.241 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 03:46:49.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:49.241 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 03:46:49.241 | INFO | PRE_HQ : 66.7%, 1.8, 62.1ms, 0 | 20260508 03:46:49.241 | INFO | BAZWRITER : 67.5%, 2.9, 98.9ms, 0 | 20260508 03:46:49.241 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:46:49.241 | INFO | Uploaded 36GiB to GPU in 2.52862s (14.237GiB/s) | 20260508 03:46:49.241 | INFO | Downloaded 38GiB to GPU in 2.68085s (14.1746GiB/s) | 20260508 03:46:54.373 | INFO | Analyzing chunk frames = [4989440,4989952) | 20260508 03:46:54.373 | INFO | MemInfo: { | 20260508 03:46:54.373 | INFO | "gpu" : " 17 GB", | 20260508 03:46:54.373 | INFO | "huge" : "", | 20260508 03:46:54.373 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:54.373 | INFO | "malloc" : " 14 GB", | 20260508 03:46:54.373 | INFO | "pinned" : " 29 GB" | 20260508 03:46:54.373 | INFO | } | 20260508 03:46:54.373 | INFO | Perf report for the last 5.13215 seconds: | 20260508 03:46:54.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:54.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 03:46:54.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:46:54.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:54.373 | INFO | Uploaded 36GiB to GPU in 2.57016s (14.0069GiB/s) | 20260508 03:46:54.373 | INFO | Downloaded 34GiB to GPU in 2.44082s (13.9297GiB/s) | 20260508 03:46:59.514 | INFO | Analyzing chunk frames = [4989952,4990464) | 20260508 03:46:59.514 | INFO | MemInfo: { | 20260508 03:46:59.514 | INFO | "gpu" : " 17 GB", | 20260508 03:46:59.514 | INFO | "huge" : "", | 20260508 03:46:59.514 | INFO | "huge_shared" : " 29 GB", | 20260508 03:46:59.514 | INFO | "malloc" : " 14 GB", | 20260508 03:46:59.514 | INFO | "pinned" : " 29 GB" | 20260508 03:46:59.514 | INFO | } | 20260508 03:46:59.514 | INFO | Perf report for the last 5.14157 seconds: | 20260508 03:46:59.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:46:59.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:46:59.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:46:59.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:46:59.515 | INFO | Uploaded 36GiB to GPU in 2.51023s (14.3413GiB/s) | 20260508 03:46:59.515 | INFO | Downloaded 34GiB to GPU in 2.49554s (13.6243GiB/s) | 20260508 03:47:04.655 | INFO | Analyzing chunk frames = [4990464,4990976) | 20260508 03:47:04.655 | INFO | MemInfo: { | 20260508 03:47:04.655 | INFO | "gpu" : " 17 GB", | 20260508 03:47:04.655 | INFO | "huge" : "", | 20260508 03:47:04.655 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:04.655 | INFO | "malloc" : " 14 GB", | 20260508 03:47:04.655 | INFO | "pinned" : " 29 GB" | 20260508 03:47:04.655 | INFO | } | 20260508 03:47:04.655 | INFO | Perf report for the last 5.14085 seconds: | 20260508 03:47:04.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:04.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.655 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:47:04.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.655 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:47:04.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:04.656 | INFO | Uploaded 36GiB to GPU in 2.54597s (14.14GiB/s) | 20260508 03:47:04.656 | INFO | Downloaded 34GiB to GPU in 2.49746s (13.6138GiB/s) | 20260508 03:47:09.800 | INFO | Analyzing chunk frames = [4990976,4991488) | 20260508 03:47:09.800 | INFO | MemInfo: { | 20260508 03:47:09.800 | INFO | "gpu" : " 17 GB", | 20260508 03:47:09.800 | INFO | "huge" : "", | 20260508 03:47:09.800 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:09.800 | INFO | "malloc" : " 14 GB", | 20260508 03:47:09.800 | INFO | "pinned" : " 29 GB" | 20260508 03:47:09.800 | INFO | } | 20260508 03:47:09.800 | INFO | Perf report for the last 5.14469 seconds: | 20260508 03:47:09.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:09.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:47:09.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:47:09.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:09.800 | INFO | Uploaded 36GiB to GPU in 2.47492s (14.5459GiB/s) | 20260508 03:47:09.800 | INFO | Downloaded 34GiB to GPU in 2.58639s (13.1458GiB/s) | 20260508 03:47:14.936 | INFO | Analyzing chunk frames = [4991488,4992000) | 20260508 03:47:14.937 | INFO | MemInfo: { | 20260508 03:47:14.937 | INFO | "gpu" : " 17 GB", | 20260508 03:47:14.937 | INFO | "huge" : "", | 20260508 03:47:14.937 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:14.937 | INFO | "malloc" : " 14 GB", | 20260508 03:47:14.937 | INFO | "pinned" : " 29 GB" | 20260508 03:47:14.937 | INFO | } | 20260508 03:47:14.937 | INFO | Perf report for the last 5.13669 seconds: | 20260508 03:47:14.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:14.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:47:14.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:47:14.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:14.937 | INFO | Uploaded 36GiB to GPU in 2.44636s (14.7157GiB/s) | 20260508 03:47:14.937 | INFO | Downloaded 34GiB to GPU in 2.40364s (14.1452GiB/s) | 20260508 03:47:20.071 | INFO | Analyzing chunk frames = [4992000,4992512) | 20260508 03:47:20.071 | INFO | MemInfo: { | 20260508 03:47:20.071 | INFO | "gpu" : " 17 GB", | 20260508 03:47:20.071 | INFO | "huge" : "", | 20260508 03:47:20.071 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:20.071 | INFO | "malloc" : " 14 GB", | 20260508 03:47:20.071 | INFO | "pinned" : " 29 GB" | 20260508 03:47:20.071 | INFO | } | 20260508 03:47:20.072 | INFO | Perf report for the last 5.13472 seconds: | 20260508 03:47:20.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:20.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:47:20.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 03:47:20.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:20.072 | INFO | Uploaded 36GiB to GPU in 2.46218s (14.6212GiB/s) | 20260508 03:47:20.072 | INFO | Downloaded 34GiB to GPU in 2.41854s (14.0581GiB/s) | 20260508 03:47:25.215 | INFO | Analyzing chunk frames = [4992512,4993024) | 20260508 03:47:25.215 | INFO | MemInfo: { | 20260508 03:47:25.215 | INFO | "gpu" : " 17 GB", | 20260508 03:47:25.215 | INFO | "huge" : "", | 20260508 03:47:25.215 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:25.215 | INFO | "malloc" : " 14 GB", | 20260508 03:47:25.215 | INFO | "pinned" : " 29 GB" | 20260508 03:47:25.215 | INFO | } | 20260508 03:47:25.216 | INFO | Perf report for the last 5.14407 seconds: | 20260508 03:47:25.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:25.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 03:47:25.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | PRE_HQ : 35.0%, 1.0, 18.2ms, 0 | 20260508 03:47:25.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:25.216 | INFO | Uploaded 36GiB to GPU in 2.48376s (14.4942GiB/s) | 20260508 03:47:25.216 | INFO | Downloaded 34GiB to GPU in 2.41604s (14.0726GiB/s) | 20260508 03:47:25.327 | INFO | Starting a round of Baz IO | 20260508 03:47:28.752 | INFO | Populating RTMetrics report | 20260508 03:47:28.752 | INFO | RTMetrics report populated | 20260508 03:47:28.752 | INFO | Opening tmp RTMetrics json file | 20260508 03:47:28.763 | INFO | Serializing RTMetrics to json | 20260508 03:47:28.765 | INFO | Serialization done, swapping tmp file now... | 20260508 03:47:28.766 | INFO | RTMetrics done | 20260508 03:47:28.819 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.491 seconds | 20260508 03:47:28.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.20% / 0.00% / 58.79% | 20260508 03:47:28.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.59063% / 0% / 5.5631e-05% | 20260508 03:47:28.819 | INFO | Baz IO overall: | 20260508 03:47:28.819 | INFO | BytesWritten : 2241932442114 | 20260508 03:47:28.819 | INFO | NumEvents : 1082886835509 | 20260508 03:47:30.359 | INFO | Analyzing chunk frames = [4993024,4993536) | 20260508 03:47:30.359 | INFO | MemInfo: { | 20260508 03:47:30.359 | INFO | "gpu" : " 17 GB", | 20260508 03:47:30.359 | INFO | "huge" : "", | 20260508 03:47:30.359 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:30.359 | INFO | "malloc" : " 14 GB", | 20260508 03:47:30.359 | INFO | "pinned" : " 29 GB" | 20260508 03:47:30.359 | INFO | } | 20260508 03:47:30.359 | INFO | Perf report for the last 5.14338 seconds: | 20260508 03:47:30.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:30.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:30.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:30.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:30.359 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 03:47:30.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:30.359 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:47:30.359 | INFO | PRE_HQ : 66.1%, 1.8, 62.1ms, 0 | 20260508 03:47:30.359 | INFO | BAZWRITER : 67.3%, 2.9, 98.9ms, 0 | 20260508 03:47:30.359 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:47:30.359 | INFO | Uploaded 36GiB to GPU in 2.56626s (14.0282GiB/s) | 20260508 03:47:30.359 | INFO | Downloaded 38GiB to GPU in 2.69289s (14.1112GiB/s) | 20260508 03:47:35.494 | INFO | Analyzing chunk frames = [4993536,4994048) | 20260508 03:47:35.494 | INFO | MemInfo: { | 20260508 03:47:35.494 | INFO | "gpu" : " 17 GB", | 20260508 03:47:35.494 | INFO | "huge" : "", | 20260508 03:47:35.494 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:35.494 | INFO | "malloc" : " 14 GB", | 20260508 03:47:35.494 | INFO | "pinned" : " 29 GB" | 20260508 03:47:35.494 | INFO | } | 20260508 03:47:35.494 | INFO | Perf report for the last 5.13485 seconds: | 20260508 03:47:35.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:35.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.494 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:47:35.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.494 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:47:35.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:35.495 | INFO | Uploaded 36GiB to GPU in 2.57654s (13.9722GiB/s) | 20260508 03:47:35.495 | INFO | Downloaded 34GiB to GPU in 2.41597s (14.073GiB/s) | 20260508 03:47:40.630 | INFO | Analyzing chunk frames = [4994048,4994560) | 20260508 03:47:40.630 | INFO | MemInfo: { | 20260508 03:47:40.630 | INFO | "gpu" : " 17 GB", | 20260508 03:47:40.630 | INFO | "huge" : "", | 20260508 03:47:40.630 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:40.630 | INFO | "malloc" : " 14 GB", | 20260508 03:47:40.630 | INFO | "pinned" : " 29 GB" | 20260508 03:47:40.630 | INFO | } | 20260508 03:47:40.630 | INFO | Perf report for the last 5.13612 seconds: | 20260508 03:47:40.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:40.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.630 | INFO | ANALYSIS : 67.5%, 3.0, 104.6ms, 0 | 20260508 03:47:40.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.630 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:47:40.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:40.631 | INFO | Uploaded 36GiB to GPU in 2.57757s (13.9666GiB/s) | 20260508 03:47:40.631 | INFO | Downloaded 34GiB to GPU in 2.49276s (13.6395GiB/s) | 20260508 03:47:45.774 | INFO | Analyzing chunk frames = [4994560,4995072) | 20260508 03:47:45.775 | INFO | MemInfo: { | 20260508 03:47:45.775 | INFO | "gpu" : " 17 GB", | 20260508 03:47:45.775 | INFO | "huge" : "", | 20260508 03:47:45.775 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:45.775 | INFO | "malloc" : " 14 GB", | 20260508 03:47:45.775 | INFO | "pinned" : " 29 GB" | 20260508 03:47:45.775 | INFO | } | 20260508 03:47:45.775 | INFO | Perf report for the last 5.14476 seconds: | 20260508 03:47:45.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:45.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 03:47:45.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:47:45.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:45.775 | INFO | Uploaded 36GiB to GPU in 2.56127s (14.0555GiB/s) | 20260508 03:47:45.775 | INFO | Downloaded 34GiB to GPU in 2.49984s (13.6009GiB/s) | 20260508 03:47:50.917 | INFO | Analyzing chunk frames = [4995072,4995584) | 20260508 03:47:50.917 | INFO | MemInfo: { | 20260508 03:47:50.917 | INFO | "gpu" : " 17 GB", | 20260508 03:47:50.917 | INFO | "huge" : "", | 20260508 03:47:50.917 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:50.917 | INFO | "malloc" : " 14 GB", | 20260508 03:47:50.917 | INFO | "pinned" : " 29 GB" | 20260508 03:47:50.917 | INFO | } | 20260508 03:47:50.917 | INFO | Perf report for the last 5.14223 seconds: | 20260508 03:47:50.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:50.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.917 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:47:50.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.917 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:47:50.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:50.918 | INFO | Uploaded 36GiB to GPU in 2.4734s (14.5549GiB/s) | 20260508 03:47:50.918 | INFO | Downloaded 34GiB to GPU in 2.57744s (13.1914GiB/s) | 20260508 03:47:56.053 | INFO | Analyzing chunk frames = [4995584,4996096) | 20260508 03:47:56.053 | INFO | MemInfo: { | 20260508 03:47:56.053 | INFO | "gpu" : " 17 GB", | 20260508 03:47:56.053 | INFO | "huge" : "", | 20260508 03:47:56.053 | INFO | "huge_shared" : " 29 GB", | 20260508 03:47:56.053 | INFO | "malloc" : " 14 GB", | 20260508 03:47:56.053 | INFO | "pinned" : " 29 GB" | 20260508 03:47:56.053 | INFO | } | 20260508 03:47:56.053 | INFO | Perf report for the last 5.13641 seconds: | 20260508 03:47:56.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:47:56.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.054 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:47:56.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.054 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:47:56.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:47:56.054 | INFO | Uploaded 36GiB to GPU in 2.4398s (14.7553GiB/s) | 20260508 03:47:56.054 | INFO | Downloaded 34GiB to GPU in 2.4091s (14.1132GiB/s) | 20260508 03:48:01.195 | INFO | Analyzing chunk frames = [4996096,4996608) | 20260508 03:48:01.195 | INFO | MemInfo: { | 20260508 03:48:01.195 | INFO | "gpu" : " 17 GB", | 20260508 03:48:01.195 | INFO | "huge" : "", | 20260508 03:48:01.195 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:01.195 | INFO | "malloc" : " 14 GB", | 20260508 03:48:01.195 | INFO | "pinned" : " 29 GB" | 20260508 03:48:01.195 | INFO | } | 20260508 03:48:01.195 | INFO | Perf report for the last 5.14187 seconds: | 20260508 03:48:01.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:01.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.195 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:48:01.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.196 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 03:48:01.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:01.196 | INFO | Uploaded 36GiB to GPU in 2.46125s (14.6267GiB/s) | 20260508 03:48:01.196 | INFO | Downloaded 34GiB to GPU in 2.42234s (14.036GiB/s) | 20260508 03:48:06.326 | INFO | Analyzing chunk frames = [4996608,4997120) | 20260508 03:48:06.326 | INFO | MemInfo: { | 20260508 03:48:06.326 | INFO | "gpu" : " 17 GB", | 20260508 03:48:06.326 | INFO | "huge" : "", | 20260508 03:48:06.326 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:06.326 | INFO | "malloc" : " 14 GB", | 20260508 03:48:06.326 | INFO | "pinned" : " 29 GB" | 20260508 03:48:06.326 | INFO | } | 20260508 03:48:06.326 | INFO | Perf report for the last 5.131 seconds: | 20260508 03:48:06.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:06.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.326 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:48:06.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.327 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:48:06.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:06.327 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:48:06.327 | INFO | Total mem usage: 16.7 GiB | 20260508 03:48:06.327 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:48:06.327 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:48:06.327 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:48:06.327 | INFO | High Water Breakdown: | 20260508 03:48:06.327 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:48:06.327 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:48:06.327 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:48:06.327 | INFO | ------------------------------------------ | 20260508 03:48:06.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:48:06.327 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:48:06.327 | INFO | Total mem usage: 26.5 GiB | 20260508 03:48:06.327 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:48:06.327 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:48:06.327 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:48:06.327 | INFO | Uploaded 36GiB to GPU in 2.48922s (14.4623GiB/s) | 20260508 03:48:06.327 | INFO | High Water Breakdown: | 20260508 03:48:06.327 | INFO | Downloaded 34GiB to GPU in 2.4349s (13.9636GiB/s) | 20260508 03:48:06.327 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:48:06.327 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:48:06.327 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:48:06.327 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:48:06.327 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:48:06.327 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:48:06.327 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:48:06.327 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:48:06.327 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:48:06.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:48:06.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:48:06.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:48:06.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:48:06.327 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:48:06.327 | INFO | ------------------------------------------ | 20260508 03:48:06.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:48:06.327 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:48:06.327 | INFO | Total mem usage: 27.8 GiB | 20260508 03:48:06.327 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:48:06.327 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:48:06.328 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:48:06.328 | INFO | High Water Breakdown: | 20260508 03:48:06.328 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:48:06.328 | INFO | ------------------------------------------ | 20260508 03:48:06.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:48:06.328 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:48:06.328 | INFO | Total mem usage: 15.7 GiB | 20260508 03:48:06.328 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:48:06.328 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:48:06.328 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:48:06.328 | INFO | High Water Breakdown: | 20260508 03:48:06.328 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:48:06.328 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:48:06.328 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:48:06.328 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:48:06.328 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:48:06.328 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:48:06.328 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:48:06.328 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:48:06.328 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:48:06.328 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:48:06.328 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:48:06.328 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:48:06.328 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:48:06.328 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:48:06.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:48:06.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:48:06.328 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:48:06.328 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:48:06.328 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:48:06.328 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:48:06.328 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:48:06.328 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:48:06.328 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:48:06.328 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:48:06.328 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:48:06.328 | INFO | ------------------------------------------ | 20260508 03:48:06.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:48:06.328 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:48:06.328 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:48:06.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:48:06.328 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.64e+01, 1.42e+01 / 2.15e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.04e-01, 1.17e-02 / 7.26e-03 | 20260508 03:48:06.328 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.47e+00, 5.89e-01 / 1.74e-01 | 20260508 03:48:06.328 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.21e-01 | 20260508 03:48:06.328 | INFO | PerfReport: 16.14%, DME -- 4.22e+01 / 4.66e+01, 4.45e+00 / 2.56e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.85e+01 / 1.95e+01, 1.65e+00 / 1.42e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.67e+00, 4.18e-01 / 2.26e-01 | 20260508 03:48:06.328 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.21e+00, 2.66e-01 / 1.12e-01 | 20260508 03:48:06.328 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.60e-01, 5.11e-02 / 2.52e-02 | 20260508 03:48:06.328 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:48:06.328 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:48:06.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:48:06.328 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 24.53%, Upload -- 2.62e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 24.36%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 2.37%, LockUpload -- 2.52e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 2.63%, LockDownload -- 2.80e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 15.41%, LockCompute -- 1.64e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:48:06.328 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260508 03:48:06.439 | INFO | Starting a round of Baz IO | 20260508 03:48:09.864 | INFO | Populating RTMetrics report | 20260508 03:48:09.865 | INFO | RTMetrics report populated | 20260508 03:48:09.865 | INFO | Opening tmp RTMetrics json file | 20260508 03:48:09.878 | INFO | Serializing RTMetrics to json | 20260508 03:48:09.879 | INFO | Serialization done, swapping tmp file now... | 20260508 03:48:09.880 | INFO | RTMetrics done | 20260508 03:48:09.938 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.499 seconds | 20260508 03:48:09.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.20% / 0.00% / 58.80% | 20260508 03:48:09.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.588838% / 0% / 5.5631e-05% | 20260508 03:48:09.938 | INFO | Baz IO overall: | 20260508 03:48:09.938 | INFO | BytesWritten : 2243658917144 | 20260508 03:48:09.938 | INFO | NumEvents : 1083712573661 | 20260508 03:48:11.471 | INFO | Analyzing chunk frames = [4997120,4997632) | 20260508 03:48:11.471 | INFO | MemInfo: { | 20260508 03:48:11.471 | INFO | "gpu" : " 17 GB", | 20260508 03:48:11.471 | INFO | "huge" : "", | 20260508 03:48:11.471 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:11.471 | INFO | "malloc" : " 14 GB", | 20260508 03:48:11.471 | INFO | "pinned" : " 29 GB" | 20260508 03:48:11.471 | INFO | } | 20260508 03:48:11.471 | INFO | Perf report for the last 5.14465 seconds: | 20260508 03:48:11.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:11.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:11.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:11.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:11.471 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 03:48:11.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:11.471 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:48:11.471 | INFO | PRE_HQ : 66.5%, 1.8, 62.2ms, 0 | 20260508 03:48:11.471 | INFO | BAZWRITER : 67.5%, 2.8, 98.8ms, 0 | 20260508 03:48:11.471 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:48:11.471 | INFO | Uploaded 36GiB to GPU in 2.60068s (13.8425GiB/s) | 20260508 03:48:11.471 | INFO | Downloaded 38GiB to GPU in 2.6965s (14.0923GiB/s) | 20260508 03:48:16.607 | INFO | Analyzing chunk frames = [4997632,4998144) | 20260508 03:48:16.607 | INFO | MemInfo: { | 20260508 03:48:16.607 | INFO | "gpu" : " 17 GB", | 20260508 03:48:16.607 | INFO | "huge" : "", | 20260508 03:48:16.607 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:16.607 | INFO | "malloc" : " 14 GB", | 20260508 03:48:16.607 | INFO | "pinned" : " 29 GB" | 20260508 03:48:16.607 | INFO | } | 20260508 03:48:16.607 | INFO | Perf report for the last 5.13601 seconds: | 20260508 03:48:16.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:16.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.607 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 03:48:16.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.607 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:48:16.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:16.608 | INFO | Uploaded 36GiB to GPU in 2.59787s (13.8575GiB/s) | 20260508 03:48:16.608 | INFO | Downloaded 34GiB to GPU in 2.43252s (13.9773GiB/s) | 20260508 03:48:21.747 | INFO | Analyzing chunk frames = [4998144,4998656) | 20260508 03:48:21.747 | INFO | MemInfo: { | 20260508 03:48:21.747 | INFO | "gpu" : " 17 GB", | 20260508 03:48:21.747 | INFO | "huge" : "", | 20260508 03:48:21.747 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:21.747 | INFO | "malloc" : " 14 GB", | 20260508 03:48:21.747 | INFO | "pinned" : " 29 GB" | 20260508 03:48:21.747 | INFO | } | 20260508 03:48:21.747 | INFO | Perf report for the last 5.13983 seconds: | 20260508 03:48:21.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:21.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:48:21.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:48:21.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:21.747 | INFO | Uploaded 36GiB to GPU in 2.55733s (14.0772GiB/s) | 20260508 03:48:21.747 | INFO | Downloaded 34GiB to GPU in 2.4875s (13.6683GiB/s) | 20260508 03:48:26.885 | INFO | Analyzing chunk frames = [4998656,4999168) | 20260508 03:48:26.885 | INFO | MemInfo: { | 20260508 03:48:26.885 | INFO | "gpu" : " 17 GB", | 20260508 03:48:26.885 | INFO | "huge" : "", | 20260508 03:48:26.885 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:26.885 | INFO | "malloc" : " 14 GB", | 20260508 03:48:26.885 | INFO | "pinned" : " 29 GB" | 20260508 03:48:26.885 | INFO | } | 20260508 03:48:26.886 | INFO | Perf report for the last 5.13867 seconds: | 20260508 03:48:26.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:26.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 03:48:26.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:48:26.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:26.886 | INFO | Uploaded 36GiB to GPU in 2.56035s (14.0606GiB/s) | 20260508 03:48:26.886 | INFO | Downloaded 34GiB to GPU in 2.49756s (13.6133GiB/s) | 20260508 03:48:32.028 | INFO | Analyzing chunk frames = [4999168,4999680) | 20260508 03:48:32.028 | INFO | MemInfo: { | 20260508 03:48:32.028 | INFO | "gpu" : " 17 GB", | 20260508 03:48:32.028 | INFO | "huge" : "", | 20260508 03:48:32.028 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:32.028 | INFO | "malloc" : " 14 GB", | 20260508 03:48:32.028 | INFO | "pinned" : " 29 GB" | 20260508 03:48:32.028 | INFO | } | 20260508 03:48:32.028 | INFO | Perf report for the last 5.14225 seconds: | 20260508 03:48:32.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:32.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 03:48:32.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:48:32.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:32.028 | INFO | Uploaded 36GiB to GPU in 2.39003s (15.0626GiB/s) | 20260508 03:48:32.028 | INFO | Downloaded 34GiB to GPU in 2.54843s (13.3415GiB/s) | 20260508 03:48:37.167 | INFO | Analyzing chunk frames = [4999680,5000192) | 20260508 03:48:37.167 | INFO | MemInfo: { | 20260508 03:48:37.167 | INFO | "gpu" : " 17 GB", | 20260508 03:48:37.167 | INFO | "huge" : "", | 20260508 03:48:37.167 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:37.167 | INFO | "malloc" : " 14 GB", | 20260508 03:48:37.167 | INFO | "pinned" : " 29 GB" | 20260508 03:48:37.167 | INFO | } | 20260508 03:48:37.167 | INFO | Perf report for the last 5.13906 seconds: | 20260508 03:48:37.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:37.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:48:37.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:48:37.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:37.167 | INFO | Uploaded 36GiB to GPU in 2.4466s (14.7143GiB/s) | 20260508 03:48:37.167 | INFO | Downloaded 34GiB to GPU in 2.41494s (14.079GiB/s) | 20260508 03:48:42.307 | INFO | Analyzing chunk frames = [5000192,5000704) | 20260508 03:48:42.308 | INFO | MemInfo: { | 20260508 03:48:42.308 | INFO | "gpu" : " 17 GB", | 20260508 03:48:42.308 | INFO | "huge" : "", | 20260508 03:48:42.308 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:42.308 | INFO | "malloc" : " 14 GB", | 20260508 03:48:42.308 | INFO | "pinned" : " 29 GB" | 20260508 03:48:42.308 | INFO | } | 20260508 03:48:42.308 | INFO | Perf report for the last 5.14118 seconds: | 20260508 03:48:42.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:42.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.308 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 03:48:42.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.308 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:48:42.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:42.309 | INFO | Uploaded 36GiB to GPU in 2.48336s (14.4965GiB/s) | 20260508 03:48:42.309 | INFO | Downloaded 34GiB to GPU in 2.42296s (14.0324GiB/s) | 20260508 03:48:47.443 | INFO | Analyzing chunk frames = [5000704,5001216) | 20260508 03:48:47.443 | INFO | MemInfo: { | 20260508 03:48:47.443 | INFO | "gpu" : " 17 GB", | 20260508 03:48:47.443 | INFO | "huge" : "", | 20260508 03:48:47.443 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:47.443 | INFO | "malloc" : " 14 GB", | 20260508 03:48:47.443 | INFO | "pinned" : " 29 GB" | 20260508 03:48:47.443 | INFO | } | 20260508 03:48:47.443 | INFO | Perf report for the last 5.13491 seconds: | 20260508 03:48:47.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:47.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 03:48:47.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260508 03:48:47.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:47.443 | INFO | Uploaded 36GiB to GPU in 2.47603s (14.5394GiB/s) | 20260508 03:48:47.443 | INFO | Downloaded 34GiB to GPU in 2.42748s (14.0063GiB/s) | 20260508 03:48:47.540 | INFO | Starting a round of Baz IO | 20260508 03:48:50.975 | INFO | Populating RTMetrics report | 20260508 03:48:50.975 | INFO | RTMetrics report populated | 20260508 03:48:50.975 | INFO | Opening tmp RTMetrics json file | 20260508 03:48:50.986 | INFO | Serializing RTMetrics to json | 20260508 03:48:50.988 | INFO | Serialization done, swapping tmp file now... | 20260508 03:48:50.988 | INFO | RTMetrics done | 20260508 03:48:51.049 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.508 seconds | 20260508 03:48:51.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.20% / 0.00% / 58.80% | 20260508 03:48:51.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.589689% / 3.97364e-06% / 6.35783e-05% | 20260508 03:48:51.049 | INFO | Baz IO overall: | 20260508 03:48:51.049 | INFO | BytesWritten : 2245385803045 | 20260508 03:48:51.049 | INFO | NumEvents : 1084538769057 | 20260508 03:48:52.580 | INFO | Analyzing chunk frames = [5001216,5001728) | 20260508 03:48:52.580 | INFO | MemInfo: { | 20260508 03:48:52.580 | INFO | "gpu" : " 17 GB", | 20260508 03:48:52.580 | INFO | "huge" : "", | 20260508 03:48:52.580 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:52.580 | INFO | "malloc" : " 14 GB", | 20260508 03:48:52.580 | INFO | "pinned" : " 29 GB" | 20260508 03:48:52.581 | INFO | } | 20260508 03:48:52.581 | INFO | Perf report for the last 5.13756 seconds: | 20260508 03:48:52.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:52.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:52.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:52.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:52.581 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 03:48:52.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:52.581 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 03:48:52.581 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 03:48:52.581 | INFO | BAZWRITER : 67.7%, 2.7, 93.8ms, 0 | 20260508 03:48:52.581 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:48:52.581 | INFO | Uploaded 36GiB to GPU in 2.58811s (13.9098GiB/s) | 20260508 03:48:52.581 | INFO | Downloaded 38GiB to GPU in 2.69341s (14.1085GiB/s) | 20260508 03:48:57.724 | INFO | Analyzing chunk frames = [5001728,5002240) | 20260508 03:48:57.724 | INFO | MemInfo: { | 20260508 03:48:57.724 | INFO | "gpu" : " 17 GB", | 20260508 03:48:57.724 | INFO | "huge" : "", | 20260508 03:48:57.724 | INFO | "huge_shared" : " 29 GB", | 20260508 03:48:57.724 | INFO | "malloc" : " 14 GB", | 20260508 03:48:57.724 | INFO | "pinned" : " 29 GB" | 20260508 03:48:57.724 | INFO | } | 20260508 03:48:57.724 | INFO | Perf report for the last 5.14357 seconds: | 20260508 03:48:57.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:48:57.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.724 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 03:48:57.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.724 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:48:57.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:48:57.725 | INFO | Uploaded 36GiB to GPU in 2.57414s (13.9852GiB/s) | 20260508 03:48:57.725 | INFO | Downloaded 34GiB to GPU in 2.41972s (14.0512GiB/s) | 20260508 03:49:02.863 | INFO | Analyzing chunk frames = [5002240,5002752) | 20260508 03:49:02.863 | INFO | MemInfo: { | 20260508 03:49:02.863 | INFO | "gpu" : " 17 GB", | 20260508 03:49:02.863 | INFO | "huge" : "", | 20260508 03:49:02.863 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:02.863 | INFO | "malloc" : " 14 GB", | 20260508 03:49:02.863 | INFO | "pinned" : " 29 GB" | 20260508 03:49:02.863 | INFO | } | 20260508 03:49:02.863 | INFO | Perf report for the last 5.13895 seconds: | 20260508 03:49:02.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:02.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260508 03:49:02.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:49:02.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:02.863 | INFO | Uploaded 36GiB to GPU in 2.56677s (14.0254GiB/s) | 20260508 03:49:02.863 | INFO | Downloaded 34GiB to GPU in 2.41896s (14.0556GiB/s) | 20260508 03:49:08.004 | INFO | Analyzing chunk frames = [5002752,5003264) | 20260508 03:49:08.004 | INFO | MemInfo: { | 20260508 03:49:08.004 | INFO | "gpu" : " 17 GB", | 20260508 03:49:08.004 | INFO | "huge" : "", | 20260508 03:49:08.004 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:08.004 | INFO | "malloc" : " 14 GB", | 20260508 03:49:08.004 | INFO | "pinned" : " 29 GB" | 20260508 03:49:08.004 | INFO | } | 20260508 03:49:08.004 | INFO | Perf report for the last 5.14092 seconds: | 20260508 03:49:08.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:08.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | ANALYSIS : 67.5%, 3.0, 104.4ms, 0 | 20260508 03:49:08.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:49:08.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:08.004 | INFO | Uploaded 36GiB to GPU in 2.57139s (14.0002GiB/s) | 20260508 03:49:08.004 | INFO | Downloaded 34GiB to GPU in 2.49657s (13.6187GiB/s) | 20260508 03:49:13.138 | INFO | Analyzing chunk frames = [5003264,5003776) | 20260508 03:49:13.138 | INFO | MemInfo: { | 20260508 03:49:13.138 | INFO | "gpu" : " 17 GB", | 20260508 03:49:13.138 | INFO | "huge" : "", | 20260508 03:49:13.138 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:13.138 | INFO | "malloc" : " 14 GB", | 20260508 03:49:13.138 | INFO | "pinned" : " 29 GB" | 20260508 03:49:13.138 | INFO | } | 20260508 03:49:13.138 | INFO | Perf report for the last 5.13425 seconds: | 20260508 03:49:13.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:13.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.138 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 03:49:13.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.138 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 03:49:13.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:13.139 | INFO | Uploaded 36GiB to GPU in 2.40079s (14.995GiB/s) | 20260508 03:49:13.139 | INFO | Downloaded 34GiB to GPU in 2.55605s (13.3018GiB/s) | 20260508 03:49:18.279 | INFO | Analyzing chunk frames = [5003776,5004288) | 20260508 03:49:18.279 | INFO | MemInfo: { | 20260508 03:49:18.279 | INFO | "gpu" : " 17 GB", | 20260508 03:49:18.279 | INFO | "huge" : "", | 20260508 03:49:18.280 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:18.280 | INFO | "malloc" : " 14 GB", | 20260508 03:49:18.280 | INFO | "pinned" : " 29 GB" | 20260508 03:49:18.280 | INFO | } | 20260508 03:49:18.280 | INFO | Perf report for the last 5.14151 seconds: | 20260508 03:49:18.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:18.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.280 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 03:49:18.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.280 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:49:18.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:18.281 | INFO | Uploaded 36GiB to GPU in 2.44049s (14.7511GiB/s) | 20260508 03:49:18.281 | INFO | Downloaded 34GiB to GPU in 2.40306s (14.1486GiB/s) | 20260508 03:49:23.420 | INFO | Analyzing chunk frames = [5004288,5004800) | 20260508 03:49:23.420 | INFO | MemInfo: { | 20260508 03:49:23.420 | INFO | "gpu" : " 17 GB", | 20260508 03:49:23.420 | INFO | "huge" : "", | 20260508 03:49:23.420 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:23.420 | INFO | "malloc" : " 14 GB", | 20260508 03:49:23.420 | INFO | "pinned" : " 29 GB" | 20260508 03:49:23.420 | INFO | } | 20260508 03:49:23.420 | INFO | Perf report for the last 5.14028 seconds: | 20260508 03:49:23.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:23.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.420 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:49:23.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.420 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 03:49:23.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:23.421 | INFO | Uploaded 36GiB to GPU in 2.46212s (14.6215GiB/s) | 20260508 03:49:23.421 | INFO | Downloaded 34GiB to GPU in 2.41484s (14.0796GiB/s) | 20260508 03:49:28.559 | INFO | Analyzing chunk frames = [5004800,5005312) | 20260508 03:49:28.559 | INFO | MemInfo: { | 20260508 03:49:28.559 | INFO | "gpu" : " 17 GB", | 20260508 03:49:28.559 | INFO | "huge" : "", | 20260508 03:49:28.559 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:28.559 | INFO | "malloc" : " 14 GB", | 20260508 03:49:28.559 | INFO | "pinned" : " 29 GB" | 20260508 03:49:28.559 | INFO | } | 20260508 03:49:28.559 | INFO | Perf report for the last 5.13875 seconds: | 20260508 03:49:28.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:28.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.559 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 03:49:28.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.559 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 03:49:28.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:28.560 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:49:28.560 | INFO | Total mem usage: 16.7 GiB | 20260508 03:49:28.560 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:49:28.560 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:49:28.560 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:49:28.560 | INFO | High Water Breakdown: | 20260508 03:49:28.560 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:49:28.560 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:49:28.560 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:49:28.560 | INFO | ------------------------------------------ | 20260508 03:49:28.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:49:28.560 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:49:28.560 | INFO | Uploaded 36GiB to GPU in 2.49714s (14.4165GiB/s) | 20260508 03:49:28.560 | INFO | Total mem usage: 26.5 GiB | 20260508 03:49:28.560 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:49:28.560 | INFO | Downloaded 34GiB to GPU in 2.43021s (13.9905GiB/s) | 20260508 03:49:28.560 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:49:28.560 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:49:28.560 | INFO | High Water Breakdown: | 20260508 03:49:28.560 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:49:28.560 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:49:28.560 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:49:28.560 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:49:28.560 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:49:28.560 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:49:28.560 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:49:28.560 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:49:28.560 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:49:28.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:49:28.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:49:28.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:49:28.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:49:28.560 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:49:28.560 | INFO | ------------------------------------------ | 20260508 03:49:28.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:49:28.560 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:49:28.560 | INFO | Total mem usage: 27.8 GiB | 20260508 03:49:28.560 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:49:28.560 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:49:28.560 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:49:28.560 | INFO | High Water Breakdown: | 20260508 03:49:28.560 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:49:28.560 | INFO | ------------------------------------------ | 20260508 03:49:28.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:49:28.560 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:49:28.560 | INFO | Total mem usage: 15.7 GiB | 20260508 03:49:28.560 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:49:28.560 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:49:28.560 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:49:28.560 | INFO | High Water Breakdown: | 20260508 03:49:28.560 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:49:28.560 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:49:28.560 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:49:28.560 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:49:28.560 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:49:28.560 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:49:28.560 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:49:28.560 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:49:28.560 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:49:28.560 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:49:28.560 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:49:28.560 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:49:28.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:49:28.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:49:28.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:49:28.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:49:28.561 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:49:28.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:49:28.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:49:28.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:49:28.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:49:28.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:49:28.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:49:28.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:49:28.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:49:28.561 | INFO | ------------------------------------------ | 20260508 03:49:28.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:49:28.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:49:28.561 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:49:28.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:49:28.561 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 2.08e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.02e-01, 1.18e-02 / 7.23e-03 | 20260508 03:49:28.561 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.50e+00, 5.83e-01 / 1.66e-01 | 20260508 03:49:28.561 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.46e+00, 2.08e-01 / 1.35e-01 | 20260508 03:49:28.561 | INFO | PerfReport: 16.08%, DME -- 4.21e+01 / 4.61e+01, 3.96e+00 / 2.39e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.69e+00 / 1.44e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.40e+00 / 2.69e+00, 4.14e-01 / 2.29e-01 | 20260508 03:49:28.561 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.21e+00, 2.63e-01 / 1.02e-01 | 20260508 03:49:28.561 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.60e-01, 5.13e-02 / 2.58e-02 | 20260508 03:49:28.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:49:28.561 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:49:28.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:49:28.561 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 24.55%, Upload -- 2.62e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 24.27%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 30.67%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 2.49%, LockUpload -- 2.66e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 2.48%, LockDownload -- 2.65e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 15.52%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:49:28.561 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260508 03:49:28.672 | INFO | Starting a round of Baz IO | 20260508 03:49:32.096 | INFO | Populating RTMetrics report | 20260508 03:49:32.096 | INFO | RTMetrics report populated | 20260508 03:49:32.096 | INFO | Opening tmp RTMetrics json file | 20260508 03:49:32.097 | INFO | Serializing RTMetrics to json | 20260508 03:49:32.098 | INFO | Serialization done, swapping tmp file now... | 20260508 03:49:32.098 | INFO | RTMetrics done | 20260508 03:49:32.167 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.494 seconds | 20260508 03:49:32.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.20% / 0.00% / 58.80% | 20260508 03:49:32.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.588806% / 3.97364e-06% / 6.35783e-05% | 20260508 03:49:32.167 | INFO | Baz IO overall: | 20260508 03:49:32.167 | INFO | BytesWritten : 2247114390060 | 20260508 03:49:32.167 | INFO | NumEvents : 1085366207926 | 20260508 03:49:33.696 | INFO | Analyzing chunk frames = [5005312,5005824) | 20260508 03:49:33.696 | INFO | MemInfo: { | 20260508 03:49:33.696 | INFO | "gpu" : " 17 GB", | 20260508 03:49:33.696 | INFO | "huge" : "", | 20260508 03:49:33.696 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:33.696 | INFO | "malloc" : " 14 GB", | 20260508 03:49:33.696 | INFO | "pinned" : " 29 GB" | 20260508 03:49:33.696 | INFO | } | 20260508 03:49:33.697 | INFO | Perf report for the last 5.13755 seconds: | 20260508 03:49:33.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:33.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:33.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:33.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:33.697 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260508 03:49:33.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:33.697 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 03:49:33.697 | INFO | PRE_HQ : 66.9%, 1.8, 62.4ms, 0 | 20260508 03:49:33.697 | INFO | BAZWRITER : 67.5%, 2.9, 99.7ms, 0 | 20260508 03:49:33.697 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 03:49:33.697 | INFO | Uploaded 36GiB to GPU in 2.55246s (14.104GiB/s) | 20260508 03:49:33.697 | INFO | Downloaded 38GiB to GPU in 2.69538s (14.0982GiB/s) | 20260508 03:49:38.834 | INFO | Analyzing chunk frames = [5005824,5006336) | 20260508 03:49:38.834 | INFO | MemInfo: { | 20260508 03:49:38.834 | INFO | "gpu" : " 17 GB", | 20260508 03:49:38.834 | INFO | "huge" : "", | 20260508 03:49:38.834 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:38.834 | INFO | "malloc" : " 14 GB", | 20260508 03:49:38.834 | INFO | "pinned" : " 29 GB" | 20260508 03:49:38.834 | INFO | } | 20260508 03:49:38.834 | INFO | Perf report for the last 5.13753 seconds: | 20260508 03:49:38.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:38.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.834 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:49:38.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.834 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:49:38.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:38.835 | INFO | Uploaded 36GiB to GPU in 2.58026s (13.9521GiB/s) | 20260508 03:49:38.835 | INFO | Downloaded 34GiB to GPU in 2.43319s (13.9734GiB/s) | 20260508 03:49:43.976 | INFO | Analyzing chunk frames = [5006336,5006848) | 20260508 03:49:43.976 | INFO | MemInfo: { | 20260508 03:49:43.976 | INFO | "gpu" : " 17 GB", | 20260508 03:49:43.976 | INFO | "huge" : "", | 20260508 03:49:43.976 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:43.976 | INFO | "malloc" : " 14 GB", | 20260508 03:49:43.976 | INFO | "pinned" : " 29 GB" | 20260508 03:49:43.976 | INFO | } | 20260508 03:49:43.976 | INFO | Perf report for the last 5.14182 seconds: | 20260508 03:49:43.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:43.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | ANALYSIS : 67.6%, 3.0, 104.8ms, 0 | 20260508 03:49:43.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:49:43.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:43.976 | INFO | Uploaded 36GiB to GPU in 2.58718s (13.9147GiB/s) | 20260508 03:49:43.976 | INFO | Downloaded 34GiB to GPU in 2.49393s (13.6331GiB/s) | 20260508 03:49:49.117 | INFO | Analyzing chunk frames = [5006848,5007360) | 20260508 03:49:49.117 | INFO | MemInfo: { | 20260508 03:49:49.117 | INFO | "gpu" : " 17 GB", | 20260508 03:49:49.117 | INFO | "huge" : "", | 20260508 03:49:49.117 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:49.117 | INFO | "malloc" : " 14 GB", | 20260508 03:49:49.117 | INFO | "pinned" : " 29 GB" | 20260508 03:49:49.117 | INFO | } | 20260508 03:49:49.117 | INFO | Perf report for the last 5.14146 seconds: | 20260508 03:49:49.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:49.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.117 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 03:49:49.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.117 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:49:49.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:49.118 | INFO | Uploaded 36GiB to GPU in 2.55599s (14.0846GiB/s) | 20260508 03:49:49.118 | INFO | Downloaded 34GiB to GPU in 2.49283s (13.6391GiB/s) | 20260508 03:49:54.261 | INFO | Analyzing chunk frames = [5007360,5007872) | 20260508 03:49:54.261 | INFO | MemInfo: { | 20260508 03:49:54.261 | INFO | "gpu" : " 17 GB", | 20260508 03:49:54.261 | INFO | "huge" : "", | 20260508 03:49:54.261 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:54.261 | INFO | "malloc" : " 14 GB", | 20260508 03:49:54.261 | INFO | "pinned" : " 29 GB" | 20260508 03:49:54.261 | INFO | } | 20260508 03:49:54.262 | INFO | Perf report for the last 5.1444 seconds: | 20260508 03:49:54.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:54.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 03:49:54.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 03:49:54.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:54.262 | INFO | Uploaded 36GiB to GPU in 2.41716s (14.8935GiB/s) | 20260508 03:49:54.263 | INFO | Downloaded 34GiB to GPU in 2.55144s (13.3258GiB/s) | 20260508 03:49:59.394 | INFO | Analyzing chunk frames = [5007872,5008384) | 20260508 03:49:59.394 | INFO | MemInfo: { | 20260508 03:49:59.394 | INFO | "gpu" : " 17 GB", | 20260508 03:49:59.394 | INFO | "huge" : "", | 20260508 03:49:59.394 | INFO | "huge_shared" : " 29 GB", | 20260508 03:49:59.394 | INFO | "malloc" : " 14 GB", | 20260508 03:49:59.394 | INFO | "pinned" : " 29 GB" | 20260508 03:49:59.394 | INFO | } | 20260508 03:49:59.394 | INFO | Perf report for the last 5.13249 seconds: | 20260508 03:49:59.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:49:59.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.394 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:49:59.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.394 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:49:59.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:49:59.395 | INFO | Uploaded 36GiB to GPU in 2.41804s (14.8881GiB/s) | 20260508 03:49:59.395 | INFO | Downloaded 34GiB to GPU in 2.42682s (14.0101GiB/s) | 20260508 03:50:04.533 | INFO | Analyzing chunk frames = [5008384,5008896) | 20260508 03:50:04.533 | INFO | MemInfo: { | 20260508 03:50:04.533 | INFO | "gpu" : " 17 GB", | 20260508 03:50:04.533 | INFO | "huge" : "", | 20260508 03:50:04.534 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:04.534 | INFO | "malloc" : " 14 GB", | 20260508 03:50:04.534 | INFO | "pinned" : " 29 GB" | 20260508 03:50:04.534 | INFO | } | 20260508 03:50:04.534 | INFO | Perf report for the last 5.13961 seconds: | 20260508 03:50:04.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:04.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 03:50:04.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:50:04.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:04.534 | INFO | Uploaded 36GiB to GPU in 2.40613s (14.9618GiB/s) | 20260508 03:50:04.535 | INFO | Downloaded 34GiB to GPU in 2.41459s (14.0811GiB/s) | 20260508 03:50:09.676 | INFO | Analyzing chunk frames = [5008896,5009408) | 20260508 03:50:09.676 | INFO | MemInfo: { | 20260508 03:50:09.676 | INFO | "gpu" : " 17 GB", | 20260508 03:50:09.676 | INFO | "huge" : "", | 20260508 03:50:09.676 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:09.676 | INFO | "malloc" : " 14 GB", | 20260508 03:50:09.676 | INFO | "pinned" : " 29 GB" | 20260508 03:50:09.676 | INFO | } | 20260508 03:50:09.677 | INFO | Perf report for the last 5.14257 seconds: | 20260508 03:50:09.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:09.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:50:09.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 03:50:09.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:09.677 | INFO | Uploaded 36GiB to GPU in 2.45677s (14.6534GiB/s) | 20260508 03:50:09.677 | INFO | Downloaded 34GiB to GPU in 2.43387s (13.9695GiB/s) | 20260508 03:50:09.789 | INFO | Starting a round of Baz IO | 20260508 03:50:13.209 | INFO | Populating RTMetrics report | 20260508 03:50:13.209 | INFO | RTMetrics report populated | 20260508 03:50:13.209 | INFO | Opening tmp RTMetrics json file | 20260508 03:50:13.221 | INFO | Serializing RTMetrics to json | 20260508 03:50:13.222 | INFO | Serialization done, swapping tmp file now... | 20260508 03:50:13.223 | INFO | RTMetrics done | 20260508 03:50:13.278 | INFO | Finished a round of Baz IO: Wrote 1.607 GiB in 3.488 seconds | 20260508 03:50:13.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.19% / 0.00% / 58.80% | 20260508 03:50:13.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.585878% / 3.97364e-06% / 5.96046e-05% | 20260508 03:50:13.278 | INFO | Baz IO overall: | 20260508 03:50:13.278 | INFO | BytesWritten : 2248839581168 | 20260508 03:50:13.278 | INFO | NumEvents : 1086191375110 | 20260508 03:50:14.811 | INFO | Analyzing chunk frames = [5009408,5009920) | 20260508 03:50:14.811 | INFO | MemInfo: { | 20260508 03:50:14.811 | INFO | "gpu" : " 17 GB", | 20260508 03:50:14.811 | INFO | "huge" : "", | 20260508 03:50:14.811 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:14.811 | INFO | "malloc" : " 14 GB", | 20260508 03:50:14.811 | INFO | "pinned" : " 29 GB" | 20260508 03:50:14.811 | INFO | } | 20260508 03:50:14.811 | INFO | Perf report for the last 5.13482 seconds: | 20260508 03:50:14.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:14.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:14.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:14.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:14.811 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 03:50:14.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:14.811 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260508 03:50:14.811 | INFO | PRE_HQ : 66.2%, 1.8, 60.5ms, 0 | 20260508 03:50:14.811 | INFO | BAZWRITER : 67.2%, 2.8, 97.6ms, 0 | 20260508 03:50:14.811 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:50:14.812 | INFO | Uploaded 36GiB to GPU in 2.50872s (14.35GiB/s) | 20260508 03:50:14.812 | INFO | Downloaded 38GiB to GPU in 2.70743s (14.0354GiB/s) | 20260508 03:50:19.954 | INFO | Analyzing chunk frames = [5009920,5010432) | 20260508 03:50:19.954 | INFO | MemInfo: { | 20260508 03:50:19.954 | INFO | "gpu" : " 17 GB", | 20260508 03:50:19.954 | INFO | "huge" : "", | 20260508 03:50:19.954 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:19.954 | INFO | "malloc" : " 14 GB", | 20260508 03:50:19.954 | INFO | "pinned" : " 29 GB" | 20260508 03:50:19.954 | INFO | } | 20260508 03:50:19.954 | INFO | Perf report for the last 5.14237 seconds: | 20260508 03:50:19.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:19.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:50:19.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:50:19.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:19.954 | INFO | Uploaded 36GiB to GPU in 2.54856s (14.1256GiB/s) | 20260508 03:50:19.954 | INFO | Downloaded 34GiB to GPU in 2.44029s (13.9328GiB/s) | 20260508 03:50:25.093 | INFO | Analyzing chunk frames = [5010432,5010944) | 20260508 03:50:25.093 | INFO | MemInfo: { | 20260508 03:50:25.093 | INFO | "gpu" : " 17 GB", | 20260508 03:50:25.093 | INFO | "huge" : "", | 20260508 03:50:25.093 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:25.093 | INFO | "malloc" : " 14 GB", | 20260508 03:50:25.093 | INFO | "pinned" : " 29 GB" | 20260508 03:50:25.093 | INFO | } | 20260508 03:50:25.093 | INFO | Perf report for the last 5.13956 seconds: | 20260508 03:50:25.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:25.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.093 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 03:50:25.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.093 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:50:25.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:25.094 | INFO | Uploaded 36GiB to GPU in 2.50306s (14.3824GiB/s) | 20260508 03:50:25.094 | INFO | Downloaded 34GiB to GPU in 2.42802s (14.0032GiB/s) | 20260508 03:50:30.234 | INFO | Analyzing chunk frames = [5010944,5011456) | 20260508 03:50:30.234 | INFO | MemInfo: { | 20260508 03:50:30.234 | INFO | "gpu" : " 17 GB", | 20260508 03:50:30.234 | INFO | "huge" : "", | 20260508 03:50:30.234 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:30.234 | INFO | "malloc" : " 14 GB", | 20260508 03:50:30.234 | INFO | "pinned" : " 29 GB" | 20260508 03:50:30.234 | INFO | } | 20260508 03:50:30.234 | INFO | Perf report for the last 5.14073 seconds: | 20260508 03:50:30.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:30.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 03:50:30.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:50:30.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:30.234 | INFO | Uploaded 36GiB to GPU in 2.55598s (14.0846GiB/s) | 20260508 03:50:30.234 | INFO | Downloaded 34GiB to GPU in 2.5223s (13.4798GiB/s) | 20260508 03:50:35.371 | INFO | Analyzing chunk frames = [5011456,5011968) | 20260508 03:50:35.371 | INFO | MemInfo: { | 20260508 03:50:35.371 | INFO | "gpu" : " 17 GB", | 20260508 03:50:35.371 | INFO | "huge" : "", | 20260508 03:50:35.371 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:35.371 | INFO | "malloc" : " 14 GB", | 20260508 03:50:35.371 | INFO | "pinned" : " 29 GB" | 20260508 03:50:35.371 | INFO | } | 20260508 03:50:35.371 | INFO | Perf report for the last 5.13671 seconds: | 20260508 03:50:35.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:35.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 03:50:35.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:50:35.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:35.371 | INFO | Uploaded 36GiB to GPU in 2.34059s (15.3807GiB/s) | 20260508 03:50:35.371 | INFO | Downloaded 34GiB to GPU in 2.55596s (13.3022GiB/s) | 20260508 03:50:40.506 | INFO | Analyzing chunk frames = [5011968,5012480) | 20260508 03:50:40.506 | INFO | MemInfo: { | 20260508 03:50:40.506 | INFO | "gpu" : " 17 GB", | 20260508 03:50:40.506 | INFO | "huge" : "", | 20260508 03:50:40.506 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:40.506 | INFO | "malloc" : " 14 GB", | 20260508 03:50:40.506 | INFO | "pinned" : " 29 GB" | 20260508 03:50:40.507 | INFO | } | 20260508 03:50:40.507 | INFO | Perf report for the last 5.13581 seconds: | 20260508 03:50:40.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:40.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 03:50:40.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:50:40.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:40.507 | INFO | Uploaded 36GiB to GPU in 2.38743s (15.0789GiB/s) | 20260508 03:50:40.507 | INFO | Downloaded 34GiB to GPU in 2.44413s (13.9109GiB/s) | 20260508 03:50:45.646 | INFO | Analyzing chunk frames = [5012480,5012992) | 20260508 03:50:45.646 | INFO | MemInfo: { | 20260508 03:50:45.646 | INFO | "gpu" : " 17 GB", | 20260508 03:50:45.646 | INFO | "huge" : "", | 20260508 03:50:45.646 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:45.646 | INFO | "malloc" : " 14 GB", | 20260508 03:50:45.647 | INFO | "pinned" : " 29 GB" | 20260508 03:50:45.647 | INFO | } | 20260508 03:50:45.647 | INFO | Perf report for the last 5.14002 seconds: | 20260508 03:50:45.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:45.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:50:45.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 03:50:45.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:45.647 | INFO | Uploaded 36GiB to GPU in 2.40969s (14.9397GiB/s) | 20260508 03:50:45.647 | INFO | Downloaded 34GiB to GPU in 2.43495s (13.9634GiB/s) | 20260508 03:50:50.785 | INFO | Analyzing chunk frames = [5012992,5013504) | 20260508 03:50:50.785 | INFO | MemInfo: { | 20260508 03:50:50.785 | INFO | "gpu" : " 17 GB", | 20260508 03:50:50.785 | INFO | "huge" : "", | 20260508 03:50:50.785 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:50.785 | INFO | "malloc" : " 14 GB", | 20260508 03:50:50.785 | INFO | "pinned" : " 29 GB" | 20260508 03:50:50.785 | INFO | } | 20260508 03:50:50.785 | INFO | Perf report for the last 5.13862 seconds: | 20260508 03:50:50.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:50.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.785 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 03:50:50.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.786 | INFO | PRE_HQ : 35.4%, 1.0, 18.4ms, 0 | 20260508 03:50:50.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:50.786 | INFO | Uploaded 36GiB to GPU in 2.4201s (14.8754GiB/s) | 20260508 03:50:50.786 | INFO | Downloaded 34GiB to GPU in 2.44307s (13.9169GiB/s) | 20260508 03:50:50.786 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:50:50.786 | INFO | Total mem usage: 16.7 GiB | 20260508 03:50:50.786 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:50:50.786 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:50:50.786 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:50:50.786 | INFO | High Water Breakdown: | 20260508 03:50:50.786 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:50:50.786 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:50:50.786 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:50:50.786 | INFO | ------------------------------------------ | 20260508 03:50:50.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:50:50.786 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:50:50.786 | INFO | Total mem usage: 26.5 GiB | 20260508 03:50:50.786 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:50:50.786 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:50:50.786 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:50:50.786 | INFO | High Water Breakdown: | 20260508 03:50:50.786 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:50:50.786 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:50:50.786 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:50:50.786 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:50:50.786 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:50:50.786 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:50:50.786 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:50:50.786 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:50:50.786 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:50:50.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:50:50.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:50:50.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:50:50.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:50:50.786 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:50:50.786 | INFO | ------------------------------------------ | 20260508 03:50:50.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:50:50.786 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:50:50.786 | INFO | Total mem usage: 27.8 GiB | 20260508 03:50:50.786 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:50:50.786 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:50:50.786 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:50:50.786 | INFO | High Water Breakdown: | 20260508 03:50:50.786 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:50:50.786 | INFO | ------------------------------------------ | 20260508 03:50:50.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:50:50.786 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:50:50.786 | INFO | Total mem usage: 15.7 GiB | 20260508 03:50:50.786 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:50:50.786 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:50:50.786 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:50:50.786 | INFO | High Water Breakdown: | 20260508 03:50:50.786 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:50:50.786 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:50:50.786 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:50:50.786 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:50:50.786 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:50:50.786 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:50:50.786 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:50:50.786 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:50:50.786 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:50:50.786 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:50:50.786 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:50:50.786 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:50:50.786 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:50:50.786 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:50:50.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:50:50.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:50:50.786 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:50:50.786 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:50:50.786 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:50:50.786 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:50:50.786 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:50:50.786 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:50:50.786 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:50:50.786 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:50:50.786 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:50:50.786 | INFO | ------------------------------------------ | 20260508 03:50:50.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:50:50.786 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:50:50.786 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:50:50.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:50:50.786 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.39e+01 / 1.66e+00 | 20260508 03:50:50.786 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.05e-01, 1.32e-02 / 8.80e-03 | 20260508 03:50:50.786 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.46e+00, 5.85e-01 / 1.97e-01 | 20260508 03:50:50.786 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.14e-01 | 20260508 03:50:50.786 | INFO | PerfReport: 16.03%, DME -- 4.20e+01 / 4.59e+01, 4.00e+00 / 2.11e+00 | 20260508 03:50:50.786 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.73e+00 / 1.40e+00 | 20260508 03:50:50.786 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.68e+00, 4.17e-01 / 1.97e-01 | 20260508 03:50:50.786 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.25e+00, 2.66e-01 / 1.48e-01 | 20260508 03:50:50.786 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.66e-01, 5.13e-02 / 2.22e-02 | 20260508 03:50:50.786 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:50:50.787 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:50:50.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:50:50.787 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 24.26%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 24.43%, Download -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 2.13%, LockUpload -- 2.27e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 2.80%, LockDownload -- 2.98e+00 / 0.00e+00, 4.30e+00 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 03:50:50.787 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 03:50:50.899 | INFO | Starting a round of Baz IO | 20260508 03:50:54.320 | INFO | Populating RTMetrics report | 20260508 03:50:54.320 | INFO | RTMetrics report populated | 20260508 03:50:54.320 | INFO | Opening tmp RTMetrics json file | 20260508 03:50:54.331 | INFO | Serializing RTMetrics to json | 20260508 03:50:54.333 | INFO | Serialization done, swapping tmp file now... | 20260508 03:50:54.333 | INFO | RTMetrics done | 20260508 03:50:54.396 | INFO | Finished a round of Baz IO: Wrote 1.607 GiB in 3.496 seconds | 20260508 03:50:54.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.19% / 0.00% / 58.81% | 20260508 03:50:54.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.58558% / 3.97364e-06% / 5.96046e-05% | 20260508 03:50:54.396 | INFO | Baz IO overall: | 20260508 03:50:54.396 | INFO | BytesWritten : 2250564588837 | 20260508 03:50:54.396 | INFO | NumEvents : 1087016154882 | 20260508 03:50:55.933 | INFO | Analyzing chunk frames = [5013504,5014016) | 20260508 03:50:55.933 | INFO | MemInfo: { | 20260508 03:50:55.933 | INFO | "gpu" : " 17 GB", | 20260508 03:50:55.933 | INFO | "huge" : "", | 20260508 03:50:55.933 | INFO | "huge_shared" : " 29 GB", | 20260508 03:50:55.933 | INFO | "malloc" : " 14 GB", | 20260508 03:50:55.933 | INFO | "pinned" : " 29 GB" | 20260508 03:50:55.933 | INFO | } | 20260508 03:50:55.933 | INFO | Perf report for the last 5.14769 seconds: | 20260508 03:50:55.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:50:55.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:55.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:55.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:55.933 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 03:50:55.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:50:55.933 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260508 03:50:55.933 | INFO | PRE_HQ : 66.1%, 1.8, 61.9ms, 0 | 20260508 03:50:55.933 | INFO | BAZWRITER : 67.4%, 2.8, 97.9ms, 0 | 20260508 03:50:55.933 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:50:55.933 | INFO | Uploaded 36GiB to GPU in 2.56776s (14.02GiB/s) | 20260508 03:50:55.933 | INFO | Downloaded 38GiB to GPU in 2.70443s (14.051GiB/s) | 20260508 03:51:01.068 | INFO | Analyzing chunk frames = [5014016,5014528) | 20260508 03:51:01.068 | INFO | MemInfo: { | 20260508 03:51:01.068 | INFO | "gpu" : " 17 GB", | 20260508 03:51:01.068 | INFO | "huge" : "", | 20260508 03:51:01.068 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:01.068 | INFO | "malloc" : " 14 GB", | 20260508 03:51:01.068 | INFO | "pinned" : " 29 GB" | 20260508 03:51:01.068 | INFO | } | 20260508 03:51:01.068 | INFO | Perf report for the last 5.13525 seconds: | 20260508 03:51:01.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:01.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:51:01.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:51:01.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:01.068 | INFO | Uploaded 36GiB to GPU in 2.55662s (14.0811GiB/s) | 20260508 03:51:01.069 | INFO | Downloaded 34GiB to GPU in 2.44034s (13.9325GiB/s) | 20260508 03:51:06.209 | INFO | Analyzing chunk frames = [5014528,5015040) | 20260508 03:51:06.209 | INFO | MemInfo: { | 20260508 03:51:06.209 | INFO | "gpu" : " 17 GB", | 20260508 03:51:06.209 | INFO | "huge" : "", | 20260508 03:51:06.209 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:06.209 | INFO | "malloc" : " 14 GB", | 20260508 03:51:06.209 | INFO | "pinned" : " 29 GB" | 20260508 03:51:06.210 | INFO | } | 20260508 03:51:06.210 | INFO | Perf report for the last 5.14149 seconds: | 20260508 03:51:06.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:06.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 03:51:06.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:51:06.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:06.210 | INFO | Uploaded 36GiB to GPU in 2.5333s (14.2107GiB/s) | 20260508 03:51:06.210 | INFO | Downloaded 34GiB to GPU in 2.51619s (13.5125GiB/s) | 20260508 03:51:11.347 | INFO | Analyzing chunk frames = [5015040,5015552) | 20260508 03:51:11.347 | INFO | MemInfo: { | 20260508 03:51:11.347 | INFO | "gpu" : " 17 GB", | 20260508 03:51:11.347 | INFO | "huge" : "", | 20260508 03:51:11.347 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:11.347 | INFO | "malloc" : " 14 GB", | 20260508 03:51:11.347 | INFO | "pinned" : " 29 GB" | 20260508 03:51:11.347 | INFO | } | 20260508 03:51:11.347 | INFO | Perf report for the last 5.13769 seconds: | 20260508 03:51:11.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:11.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | ANALYSIS : 66.7%, 2.9, 103.1ms, 0 | 20260508 03:51:11.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:51:11.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:11.348 | INFO | Uploaded 36GiB to GPU in 2.49124s (14.4507GiB/s) | 20260508 03:51:11.348 | INFO | Downloaded 34GiB to GPU in 2.39651s (14.1873GiB/s) | 20260508 03:51:16.486 | INFO | Analyzing chunk frames = [5015552,5016064) | 20260508 03:51:16.486 | INFO | MemInfo: { | 20260508 03:51:16.486 | INFO | "gpu" : " 17 GB", | 20260508 03:51:16.486 | INFO | "huge" : "", | 20260508 03:51:16.486 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:16.486 | INFO | "malloc" : " 14 GB", | 20260508 03:51:16.486 | INFO | "pinned" : " 29 GB" | 20260508 03:51:16.486 | INFO | } | 20260508 03:51:16.486 | INFO | Perf report for the last 5.13879 seconds: | 20260508 03:51:16.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:16.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 03:51:16.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:51:16.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:16.486 | INFO | Uploaded 36GiB to GPU in 2.35551s (15.2833GiB/s) | 20260508 03:51:16.487 | INFO | Downloaded 34GiB to GPU in 2.55254s (13.3201GiB/s) | 20260508 03:51:21.628 | INFO | Analyzing chunk frames = [5016064,5016576) | 20260508 03:51:21.628 | INFO | MemInfo: { | 20260508 03:51:21.628 | INFO | "gpu" : " 17 GB", | 20260508 03:51:21.628 | INFO | "huge" : "", | 20260508 03:51:21.628 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:21.628 | INFO | "malloc" : " 14 GB", | 20260508 03:51:21.628 | INFO | "pinned" : " 29 GB" | 20260508 03:51:21.628 | INFO | } | 20260508 03:51:21.628 | INFO | Perf report for the last 5.14176 seconds: | 20260508 03:51:21.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:21.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:51:21.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:51:21.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:21.628 | INFO | Uploaded 36GiB to GPU in 2.42271s (14.8594GiB/s) | 20260508 03:51:21.628 | INFO | Downloaded 34GiB to GPU in 2.42077s (14.0451GiB/s) | 20260508 03:51:26.766 | INFO | Analyzing chunk frames = [5016576,5017088) | 20260508 03:51:26.766 | INFO | MemInfo: { | 20260508 03:51:26.766 | INFO | "gpu" : " 17 GB", | 20260508 03:51:26.766 | INFO | "huge" : "", | 20260508 03:51:26.766 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:26.766 | INFO | "malloc" : " 14 GB", | 20260508 03:51:26.766 | INFO | "pinned" : " 29 GB" | 20260508 03:51:26.766 | INFO | } | 20260508 03:51:26.766 | INFO | Perf report for the last 5.13841 seconds: | 20260508 03:51:26.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:26.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.766 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 03:51:26.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.766 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 03:51:26.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:26.767 | INFO | Uploaded 36GiB to GPU in 2.45917s (14.6391GiB/s) | 20260508 03:51:26.767 | INFO | Downloaded 34GiB to GPU in 2.43685s (13.9524GiB/s) | 20260508 03:51:31.902 | INFO | Analyzing chunk frames = [5017088,5017600) | 20260508 03:51:31.902 | INFO | MemInfo: { | 20260508 03:51:31.902 | INFO | "gpu" : " 17 GB", | 20260508 03:51:31.902 | INFO | "huge" : "", | 20260508 03:51:31.902 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:31.902 | INFO | "malloc" : " 14 GB", | 20260508 03:51:31.902 | INFO | "pinned" : " 29 GB" | 20260508 03:51:31.903 | INFO | } | 20260508 03:51:31.903 | INFO | Perf report for the last 5.13646 seconds: | 20260508 03:51:31.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:31.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:51:31.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 03:51:31.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:31.903 | INFO | Uploaded 36GiB to GPU in 2.44462s (14.7262GiB/s) | 20260508 03:51:31.903 | INFO | Downloaded 34GiB to GPU in 2.43228s (13.9786GiB/s) | 20260508 03:51:32.017 | INFO | Starting a round of Baz IO | 20260508 03:51:35.437 | INFO | Populating RTMetrics report | 20260508 03:51:35.437 | INFO | RTMetrics report populated | 20260508 03:51:35.437 | INFO | Opening tmp RTMetrics json file | 20260508 03:51:35.445 | INFO | Serializing RTMetrics to json | 20260508 03:51:35.451 | INFO | Serialization done, swapping tmp file now... | 20260508 03:51:35.452 | INFO | RTMetrics done | 20260508 03:51:35.512 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.494 seconds | 20260508 03:51:35.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.19% / 0.00% / 58.81% | 20260508 03:51:35.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584527% / 3.97364e-06% / 5.5631e-05% | 20260508 03:51:35.512 | INFO | Baz IO overall: | 20260508 03:51:35.512 | INFO | BytesWritten : 2252291728089 | 20260508 03:51:35.512 | INFO | NumEvents : 1087842680553 | 20260508 03:51:37.041 | INFO | Analyzing chunk frames = [5017600,5018112) | 20260508 03:51:37.041 | INFO | MemInfo: { | 20260508 03:51:37.041 | INFO | "gpu" : " 17 GB", | 20260508 03:51:37.041 | INFO | "huge" : "", | 20260508 03:51:37.041 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:37.041 | INFO | "malloc" : " 14 GB", | 20260508 03:51:37.041 | INFO | "pinned" : " 29 GB" | 20260508 03:51:37.041 | INFO | } | 20260508 03:51:37.041 | INFO | Perf report for the last 5.13805 seconds: | 20260508 03:51:37.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:37.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:37.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:37.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:37.041 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 03:51:37.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:37.041 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 03:51:37.041 | INFO | PRE_HQ : 66.3%, 1.8, 61.0ms, 0 | 20260508 03:51:37.041 | INFO | BAZWRITER : 67.4%, 2.8, 97.8ms, 0 | 20260508 03:51:37.041 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:51:37.042 | INFO | Uploaded 36GiB to GPU in 2.59662s (13.8642GiB/s) | 20260508 03:51:37.042 | INFO | Downloaded 38GiB to GPU in 2.69578s (14.0961GiB/s) | 20260508 03:51:42.184 | INFO | Analyzing chunk frames = [5018112,5018624) | 20260508 03:51:42.184 | INFO | MemInfo: { | 20260508 03:51:42.184 | INFO | "gpu" : " 17 GB", | 20260508 03:51:42.184 | INFO | "huge" : "", | 20260508 03:51:42.184 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:42.184 | INFO | "malloc" : " 14 GB", | 20260508 03:51:42.184 | INFO | "pinned" : " 29 GB" | 20260508 03:51:42.184 | INFO | } | 20260508 03:51:42.185 | INFO | Perf report for the last 5.14358 seconds: | 20260508 03:51:42.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:42.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 03:51:42.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:51:42.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:42.185 | INFO | Uploaded 36GiB to GPU in 2.57065s (14.0043GiB/s) | 20260508 03:51:42.185 | INFO | Downloaded 34GiB to GPU in 2.43688s (13.9523GiB/s) | 20260508 03:51:47.327 | INFO | Analyzing chunk frames = [5018624,5019136) | 20260508 03:51:47.327 | INFO | MemInfo: { | 20260508 03:51:47.327 | INFO | "gpu" : " 17 GB", | 20260508 03:51:47.327 | INFO | "huge" : "", | 20260508 03:51:47.327 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:47.327 | INFO | "malloc" : " 14 GB", | 20260508 03:51:47.327 | INFO | "pinned" : " 29 GB" | 20260508 03:51:47.327 | INFO | } | 20260508 03:51:47.327 | INFO | Perf report for the last 5.14282 seconds: | 20260508 03:51:47.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:47.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.327 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:51:47.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.328 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:51:47.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:47.328 | INFO | Uploaded 36GiB to GPU in 2.57849s (13.9617GiB/s) | 20260508 03:51:47.328 | INFO | Downloaded 34GiB to GPU in 2.4938s (13.6338GiB/s) | 20260508 03:51:52.459 | INFO | Analyzing chunk frames = [5019136,5019648) | 20260508 03:51:52.459 | INFO | MemInfo: { | 20260508 03:51:52.459 | INFO | "gpu" : " 17 GB", | 20260508 03:51:52.459 | INFO | "huge" : "", | 20260508 03:51:52.459 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:52.459 | INFO | "malloc" : " 14 GB", | 20260508 03:51:52.459 | INFO | "pinned" : " 29 GB" | 20260508 03:51:52.459 | INFO | } | 20260508 03:51:52.459 | INFO | Perf report for the last 5.132 seconds: | 20260508 03:51:52.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:52.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.459 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 03:51:52.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.460 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:51:52.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:52.460 | INFO | Uploaded 36GiB to GPU in 2.59366s (13.88GiB/s) | 20260508 03:51:52.460 | INFO | Downloaded 34GiB to GPU in 2.4047s (14.139GiB/s) | 20260508 03:51:57.602 | INFO | Analyzing chunk frames = [5019648,5020160) | 20260508 03:51:57.602 | INFO | MemInfo: { | 20260508 03:51:57.602 | INFO | "gpu" : " 17 GB", | 20260508 03:51:57.602 | INFO | "huge" : "", | 20260508 03:51:57.602 | INFO | "huge_shared" : " 29 GB", | 20260508 03:51:57.602 | INFO | "malloc" : " 14 GB", | 20260508 03:51:57.602 | INFO | "pinned" : " 29 GB" | 20260508 03:51:57.602 | INFO | } | 20260508 03:51:57.602 | INFO | Perf report for the last 5.14287 seconds: | 20260508 03:51:57.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:51:57.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.602 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 03:51:57.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.602 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:51:57.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:51:57.603 | INFO | Uploaded 36GiB to GPU in 2.34821s (15.3308GiB/s) | 20260508 03:51:57.603 | INFO | Downloaded 34GiB to GPU in 2.53895s (13.3914GiB/s) | 20260508 03:52:02.738 | INFO | Analyzing chunk frames = [5020160,5020672) | 20260508 03:52:02.738 | INFO | MemInfo: { | 20260508 03:52:02.738 | INFO | "gpu" : " 17 GB", | 20260508 03:52:02.738 | INFO | "huge" : "", | 20260508 03:52:02.738 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:02.738 | INFO | "malloc" : " 14 GB", | 20260508 03:52:02.738 | INFO | "pinned" : " 29 GB" | 20260508 03:52:02.738 | INFO | } | 20260508 03:52:02.738 | INFO | Perf report for the last 5.1361 seconds: | 20260508 03:52:02.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:02.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.738 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 03:52:02.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.739 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 03:52:02.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:02.739 | INFO | Uploaded 36GiB to GPU in 2.45777s (14.6474GiB/s) | 20260508 03:52:02.739 | INFO | Downloaded 34GiB to GPU in 2.4121s (14.0956GiB/s) | 20260508 03:52:07.879 | INFO | Analyzing chunk frames = [5020672,5021184) | 20260508 03:52:07.879 | INFO | MemInfo: { | 20260508 03:52:07.879 | INFO | "gpu" : " 17 GB", | 20260508 03:52:07.879 | INFO | "huge" : "", | 20260508 03:52:07.879 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:07.879 | INFO | "malloc" : " 14 GB", | 20260508 03:52:07.879 | INFO | "pinned" : " 29 GB" | 20260508 03:52:07.880 | INFO | } | 20260508 03:52:07.880 | INFO | Perf report for the last 5.14116 seconds: | 20260508 03:52:07.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:07.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.880 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:52:07.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.880 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:52:07.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:07.881 | INFO | Uploaded 36GiB to GPU in 2.49051s (14.4549GiB/s) | 20260508 03:52:07.881 | INFO | Downloaded 34GiB to GPU in 2.42206s (14.0376GiB/s) | 20260508 03:52:13.024 | INFO | Analyzing chunk frames = [5021184,5021696) | 20260508 03:52:13.024 | INFO | MemInfo: { | 20260508 03:52:13.024 | INFO | "gpu" : " 17 GB", | 20260508 03:52:13.024 | INFO | "huge" : "", | 20260508 03:52:13.024 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:13.024 | INFO | "malloc" : " 14 GB", | 20260508 03:52:13.025 | INFO | "pinned" : " 29 GB" | 20260508 03:52:13.025 | INFO | } | 20260508 03:52:13.025 | INFO | Perf report for the last 5.14504 seconds: | 20260508 03:52:13.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:13.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 03:52:13.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 03:52:13.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:13.025 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:52:13.025 | INFO | Total mem usage: 16.7 GiB | 20260508 03:52:13.025 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:52:13.025 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:52:13.025 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:52:13.025 | INFO | High Water Breakdown: | 20260508 03:52:13.025 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:52:13.025 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:52:13.025 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:52:13.025 | INFO | ------------------------------------------ | 20260508 03:52:13.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:52:13.025 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:52:13.025 | INFO | Total mem usage: 26.5 GiB | 20260508 03:52:13.025 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:52:13.025 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:52:13.025 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:52:13.025 | INFO | High Water Breakdown: | 20260508 03:52:13.025 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:52:13.025 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:52:13.025 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:52:13.025 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:52:13.025 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:52:13.025 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:52:13.025 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:52:13.025 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:52:13.025 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:52:13.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:52:13.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:52:13.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:52:13.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:52:13.025 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:52:13.025 | INFO | ------------------------------------------ | 20260508 03:52:13.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:52:13.026 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:52:13.026 | INFO | Total mem usage: 27.8 GiB | 20260508 03:52:13.026 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:52:13.026 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:52:13.026 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:52:13.026 | INFO | High Water Breakdown: | 20260508 03:52:13.026 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:52:13.026 | INFO | ------------------------------------------ | 20260508 03:52:13.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:52:13.026 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:52:13.026 | INFO | Total mem usage: 15.7 GiB | 20260508 03:52:13.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:52:13.026 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:52:13.026 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:52:13.026 | INFO | High Water Breakdown: | 20260508 03:52:13.026 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:52:13.026 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:52:13.026 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:52:13.026 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:52:13.026 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:52:13.026 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:52:13.026 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:52:13.026 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:52:13.026 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:52:13.026 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:52:13.026 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:52:13.026 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:52:13.026 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:52:13.026 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:52:13.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:52:13.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:52:13.026 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:52:13.026 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:52:13.026 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:52:13.026 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:52:13.026 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:52:13.026 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:52:13.026 | INFO | Uploaded 36GiB to GPU in 2.48122s (14.509GiB/s) | 20260508 03:52:13.026 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:52:13.026 | INFO | Downloaded 34GiB to GPU in 2.42088s (14.0445GiB/s) | 20260508 03:52:13.026 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:52:13.026 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:52:13.026 | INFO | ------------------------------------------ | 20260508 03:52:13.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:52:13.026 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:52:13.026 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:52:13.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:52:13.026 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.57e+01, 1.42e+01 / 1.54e+00 | 20260508 03:52:13.026 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.10e-02 / 5.85e-03 | 20260508 03:52:13.026 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.43e+00, 5.93e-01 / 2.46e-01 | 20260508 03:52:13.026 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.08e-01 / 1.09e-01 | 20260508 03:52:13.026 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.56e+01, 3.93e+00 / 1.98e+00 | 20260508 03:52:13.026 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.70e+00 / 1.34e+00 | 20260508 03:52:13.026 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.40e+00 / 2.60e+00, 4.21e-01 / 2.85e-01 | 20260508 03:52:13.027 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.21e+00, 2.65e-01 / 1.30e-01 | 20260508 03:52:13.027 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.61e-01, 5.14e-02 / 2.30e-02 | 20260508 03:52:13.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:52:13.027 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:52:13.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:52:13.027 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 2.23%, LockUpload -- 2.37e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 2.69%, LockDownload -- 2.86e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:52:13.027 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260508 03:52:13.140 | INFO | Starting a round of Baz IO | 20260508 03:52:16.578 | INFO | Populating RTMetrics report | 20260508 03:52:16.579 | INFO | RTMetrics report populated | 20260508 03:52:16.579 | INFO | Opening tmp RTMetrics json file | 20260508 03:52:16.591 | INFO | Serializing RTMetrics to json | 20260508 03:52:16.593 | INFO | Serialization done, swapping tmp file now... | 20260508 03:52:16.593 | INFO | RTMetrics done | 20260508 03:52:16.648 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.508 seconds | 20260508 03:52:16.648 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.19% / 0.00% / 58.81% | 20260508 03:52:16.648 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584646% / 3.97364e-06% / 5.5631e-05% | 20260508 03:52:16.648 | INFO | Baz IO overall: | 20260508 03:52:16.648 | INFO | BytesWritten : 2254017980467 | 20260508 03:52:16.648 | INFO | NumEvents : 1088668611851 | 20260508 03:52:18.158 | INFO | Analyzing chunk frames = [5021696,5022208) | 20260508 03:52:18.158 | INFO | MemInfo: { | 20260508 03:52:18.158 | INFO | "gpu" : " 17 GB", | 20260508 03:52:18.158 | INFO | "huge" : "", | 20260508 03:52:18.158 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:18.158 | INFO | "malloc" : " 14 GB", | 20260508 03:52:18.158 | INFO | "pinned" : " 29 GB" | 20260508 03:52:18.158 | INFO | } | 20260508 03:52:18.158 | INFO | Perf report for the last 5.13352 seconds: | 20260508 03:52:18.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:18.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:18.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:18.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:18.158 | INFO | ANALYSIS : 68.4%, 3.0, 106.3ms, 0 | 20260508 03:52:18.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:18.158 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 03:52:18.158 | INFO | PRE_HQ : 66.7%, 1.8, 62.0ms, 0 | 20260508 03:52:18.158 | INFO | BAZWRITER : 67.8%, 2.9, 99.2ms, 0 | 20260508 03:52:18.158 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:52:18.159 | INFO | Uploaded 36GiB to GPU in 2.61729s (13.7547GiB/s) | 20260508 03:52:18.159 | INFO | Downloaded 38GiB to GPU in 2.71119s (14.016GiB/s) | 20260508 03:52:23.302 | INFO | Analyzing chunk frames = [5022208,5022720) | 20260508 03:52:23.302 | INFO | MemInfo: { | 20260508 03:52:23.302 | INFO | "gpu" : " 17 GB", | 20260508 03:52:23.302 | INFO | "huge" : "", | 20260508 03:52:23.302 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:23.302 | INFO | "malloc" : " 14 GB", | 20260508 03:52:23.302 | INFO | "pinned" : " 29 GB" | 20260508 03:52:23.302 | INFO | } | 20260508 03:52:23.302 | INFO | Perf report for the last 5.1437 seconds: | 20260508 03:52:23.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:23.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:52:23.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:52:23.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:23.302 | INFO | Uploaded 36GiB to GPU in 2.58326s (13.9359GiB/s) | 20260508 03:52:23.303 | INFO | Downloaded 34GiB to GPU in 2.43046s (13.9891GiB/s) | 20260508 03:52:28.437 | INFO | Analyzing chunk frames = [5022720,5023232) | 20260508 03:52:28.437 | INFO | MemInfo: { | 20260508 03:52:28.437 | INFO | "gpu" : " 17 GB", | 20260508 03:52:28.437 | INFO | "huge" : "", | 20260508 03:52:28.437 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:28.437 | INFO | "malloc" : " 14 GB", | 20260508 03:52:28.437 | INFO | "pinned" : " 29 GB" | 20260508 03:52:28.437 | INFO | } | 20260508 03:52:28.437 | INFO | Perf report for the last 5.13542 seconds: | 20260508 03:52:28.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:28.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.437 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:52:28.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.437 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:52:28.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:28.438 | INFO | Uploaded 36GiB to GPU in 2.55194s (14.1069GiB/s) | 20260508 03:52:28.438 | INFO | Downloaded 34GiB to GPU in 2.41064s (14.1042GiB/s) | 20260508 03:52:33.579 | INFO | Analyzing chunk frames = [5023232,5023744) | 20260508 03:52:33.579 | INFO | MemInfo: { | 20260508 03:52:33.579 | INFO | "gpu" : " 17 GB", | 20260508 03:52:33.579 | INFO | "huge" : "", | 20260508 03:52:33.579 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:33.579 | INFO | "malloc" : " 14 GB", | 20260508 03:52:33.579 | INFO | "pinned" : " 29 GB" | 20260508 03:52:33.579 | INFO | } | 20260508 03:52:33.579 | INFO | Perf report for the last 5.14203 seconds: | 20260508 03:52:33.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:33.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.579 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 03:52:33.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.579 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:52:33.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:33.580 | INFO | Uploaded 36GiB to GPU in 2.57087s (14.0031GiB/s) | 20260508 03:52:33.580 | INFO | Downloaded 34GiB to GPU in 2.4971s (13.6158GiB/s) | 20260508 03:52:38.714 | INFO | Analyzing chunk frames = [5023744,5024256) | 20260508 03:52:38.714 | INFO | MemInfo: { | 20260508 03:52:38.714 | INFO | "gpu" : " 17 GB", | 20260508 03:52:38.714 | INFO | "huge" : "", | 20260508 03:52:38.714 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:38.714 | INFO | "malloc" : " 14 GB", | 20260508 03:52:38.714 | INFO | "pinned" : " 29 GB" | 20260508 03:52:38.714 | INFO | } | 20260508 03:52:38.714 | INFO | Perf report for the last 5.13479 seconds: | 20260508 03:52:38.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:38.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:52:38.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:52:38.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:38.714 | INFO | Uploaded 36GiB to GPU in 2.43138s (14.8064GiB/s) | 20260508 03:52:38.714 | INFO | Downloaded 34GiB to GPU in 2.58529s (13.1513GiB/s) | 20260508 03:52:43.854 | INFO | Analyzing chunk frames = [5024256,5024768) | 20260508 03:52:43.854 | INFO | MemInfo: { | 20260508 03:52:43.854 | INFO | "gpu" : " 17 GB", | 20260508 03:52:43.854 | INFO | "huge" : "", | 20260508 03:52:43.854 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:43.854 | INFO | "malloc" : " 14 GB", | 20260508 03:52:43.854 | INFO | "pinned" : " 29 GB" | 20260508 03:52:43.854 | INFO | } | 20260508 03:52:43.855 | INFO | Perf report for the last 5.14038 seconds: | 20260508 03:52:43.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:43.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 03:52:43.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:52:43.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:43.855 | INFO | Uploaded 36GiB to GPU in 2.44775s (14.7074GiB/s) | 20260508 03:52:43.855 | INFO | Downloaded 34GiB to GPU in 2.4095s (14.1108GiB/s) | 20260508 03:52:48.992 | INFO | Analyzing chunk frames = [5024768,5025280) | 20260508 03:52:48.992 | INFO | MemInfo: { | 20260508 03:52:48.992 | INFO | "gpu" : " 17 GB", | 20260508 03:52:48.992 | INFO | "huge" : "", | 20260508 03:52:48.992 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:48.992 | INFO | "malloc" : " 14 GB", | 20260508 03:52:48.993 | INFO | "pinned" : " 29 GB" | 20260508 03:52:48.993 | INFO | } | 20260508 03:52:48.993 | INFO | Perf report for the last 5.13805 seconds: | 20260508 03:52:48.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:48.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:52:48.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 03:52:48.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:48.993 | INFO | Uploaded 36GiB to GPU in 2.51387s (14.3206GiB/s) | 20260508 03:52:48.993 | INFO | Downloaded 34GiB to GPU in 2.42935s (13.9955GiB/s) | 20260508 03:52:54.137 | INFO | Analyzing chunk frames = [5025280,5025792) | 20260508 03:52:54.137 | INFO | MemInfo: { | 20260508 03:52:54.137 | INFO | "gpu" : " 17 GB", | 20260508 03:52:54.137 | INFO | "huge" : "", | 20260508 03:52:54.137 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:54.138 | INFO | "malloc" : " 14 GB", | 20260508 03:52:54.138 | INFO | "pinned" : " 29 GB" | 20260508 03:52:54.138 | INFO | } | 20260508 03:52:54.138 | INFO | Perf report for the last 5.14514 seconds: | 20260508 03:52:54.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:54.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:52:54.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | PRE_HQ : 34.9%, 1.0, 18.1ms, 0 | 20260508 03:52:54.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:54.138 | INFO | Uploaded 36GiB to GPU in 2.48136s (14.5082GiB/s) | 20260508 03:52:54.138 | INFO | Downloaded 34GiB to GPU in 2.42598s (14.015GiB/s) | 20260508 03:52:54.251 | INFO | Starting a round of Baz IO | 20260508 03:52:57.681 | INFO | Populating RTMetrics report | 20260508 03:52:57.681 | INFO | RTMetrics report populated | 20260508 03:52:57.681 | INFO | Opening tmp RTMetrics json file | 20260508 03:52:57.689 | INFO | Serializing RTMetrics to json | 20260508 03:52:57.696 | INFO | Serialization done, swapping tmp file now... | 20260508 03:52:57.697 | INFO | RTMetrics done | 20260508 03:52:57.754 | INFO | Finished a round of Baz IO: Wrote 1.607 GiB in 3.503 seconds | 20260508 03:52:57.754 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.18% / 0.00% / 58.81% | 20260508 03:52:57.754 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584698% / 7.94729e-06% / 7.15256e-05% | 20260508 03:52:57.754 | INFO | Baz IO overall: | 20260508 03:52:57.754 | INFO | BytesWritten : 2255743897290 | 20260508 03:52:57.754 | INFO | NumEvents : 1089494170040 | 20260508 03:52:59.273 | INFO | Analyzing chunk frames = [5025792,5026304) | 20260508 03:52:59.273 | INFO | MemInfo: { | 20260508 03:52:59.273 | INFO | "gpu" : " 17 GB", | 20260508 03:52:59.273 | INFO | "huge" : "", | 20260508 03:52:59.274 | INFO | "huge_shared" : " 29 GB", | 20260508 03:52:59.274 | INFO | "malloc" : " 14 GB", | 20260508 03:52:59.274 | INFO | "pinned" : " 29 GB" | 20260508 03:52:59.274 | INFO | } | 20260508 03:52:59.274 | INFO | Perf report for the last 5.13585 seconds: | 20260508 03:52:59.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:52:59.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:59.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:59.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:59.274 | INFO | ANALYSIS : 68.3%, 3.0, 106.4ms, 0 | 20260508 03:52:59.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:52:59.274 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:52:59.274 | INFO | PRE_HQ : 66.7%, 1.8, 62.1ms, 0 | 20260508 03:52:59.274 | INFO | BAZWRITER : 67.7%, 2.8, 98.1ms, 0 | 20260508 03:52:59.274 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:52:59.274 | INFO | Uploaded 36GiB to GPU in 2.53631s (14.1939GiB/s) | 20260508 03:52:59.274 | INFO | Downloaded 38GiB to GPU in 2.7023s (14.0621GiB/s) | 20260508 03:53:04.408 | INFO | Analyzing chunk frames = [5026304,5026816) | 20260508 03:53:04.408 | INFO | MemInfo: { | 20260508 03:53:04.408 | INFO | "gpu" : " 17 GB", | 20260508 03:53:04.408 | INFO | "huge" : "", | 20260508 03:53:04.408 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:04.408 | INFO | "malloc" : " 14 GB", | 20260508 03:53:04.408 | INFO | "pinned" : " 29 GB" | 20260508 03:53:04.408 | INFO | } | 20260508 03:53:04.409 | INFO | Perf report for the last 5.13489 seconds: | 20260508 03:53:04.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:04.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 03:53:04.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:53:04.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:04.409 | INFO | Uploaded 36GiB to GPU in 2.56661s (14.0263GiB/s) | 20260508 03:53:04.409 | INFO | Downloaded 34GiB to GPU in 2.4366s (13.9539GiB/s) | 20260508 03:53:09.556 | INFO | Analyzing chunk frames = [5026816,5027328) | 20260508 03:53:09.556 | INFO | MemInfo: { | 20260508 03:53:09.556 | INFO | "gpu" : " 17 GB", | 20260508 03:53:09.556 | INFO | "huge" : "", | 20260508 03:53:09.556 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:09.556 | INFO | "malloc" : " 14 GB", | 20260508 03:53:09.556 | INFO | "pinned" : " 29 GB" | 20260508 03:53:09.556 | INFO | } | 20260508 03:53:09.556 | INFO | Perf report for the last 5.14778 seconds: | 20260508 03:53:09.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:09.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:53:09.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:53:09.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:09.557 | INFO | Uploaded 36GiB to GPU in 2.57732s (13.968GiB/s) | 20260508 03:53:09.557 | INFO | Downloaded 34GiB to GPU in 2.51058s (13.5427GiB/s) | 20260508 03:53:14.698 | INFO | Analyzing chunk frames = [5027328,5027840) | 20260508 03:53:14.698 | INFO | MemInfo: { | 20260508 03:53:14.698 | INFO | "gpu" : " 17 GB", | 20260508 03:53:14.698 | INFO | "huge" : "", | 20260508 03:53:14.698 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:14.698 | INFO | "malloc" : " 14 GB", | 20260508 03:53:14.698 | INFO | "pinned" : " 29 GB" | 20260508 03:53:14.698 | INFO | } | 20260508 03:53:14.698 | INFO | Perf report for the last 5.14166 seconds: | 20260508 03:53:14.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:14.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 03:53:14.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 03:53:14.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:14.698 | INFO | Uploaded 36GiB to GPU in 2.55708s (14.0786GiB/s) | 20260508 03:53:14.698 | INFO | Downloaded 34GiB to GPU in 2.42014s (14.0488GiB/s) | 20260508 03:53:19.834 | INFO | Analyzing chunk frames = [5027840,5028352) | 20260508 03:53:19.834 | INFO | MemInfo: { | 20260508 03:53:19.834 | INFO | "gpu" : " 17 GB", | 20260508 03:53:19.834 | INFO | "huge" : "", | 20260508 03:53:19.834 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:19.834 | INFO | "malloc" : " 14 GB", | 20260508 03:53:19.834 | INFO | "pinned" : " 29 GB" | 20260508 03:53:19.834 | INFO | } | 20260508 03:53:19.834 | INFO | Perf report for the last 5.13591 seconds: | 20260508 03:53:19.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:19.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 03:53:19.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 03:53:19.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:19.834 | INFO | Uploaded 36GiB to GPU in 2.34388s (15.3592GiB/s) | 20260508 03:53:19.834 | INFO | Downloaded 34GiB to GPU in 2.56635s (13.2484GiB/s) | 20260508 03:53:24.971 | INFO | Analyzing chunk frames = [5028352,5028864) | 20260508 03:53:24.971 | INFO | MemInfo: { | 20260508 03:53:24.971 | INFO | "gpu" : " 17 GB", | 20260508 03:53:24.971 | INFO | "huge" : "", | 20260508 03:53:24.971 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:24.971 | INFO | "malloc" : " 14 GB", | 20260508 03:53:24.971 | INFO | "pinned" : " 29 GB" | 20260508 03:53:24.971 | INFO | } | 20260508 03:53:24.971 | INFO | Perf report for the last 5.13672 seconds: | 20260508 03:53:24.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:24.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 03:53:24.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:53:24.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:24.971 | INFO | Uploaded 36GiB to GPU in 2.39871s (15.0081GiB/s) | 20260508 03:53:24.971 | INFO | Downloaded 34GiB to GPU in 2.44206s (13.9227GiB/s) | 20260508 03:53:30.112 | INFO | Analyzing chunk frames = [5028864,5029376) | 20260508 03:53:30.112 | INFO | MemInfo: { | 20260508 03:53:30.112 | INFO | "gpu" : " 17 GB", | 20260508 03:53:30.112 | INFO | "huge" : "", | 20260508 03:53:30.112 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:30.112 | INFO | "malloc" : " 14 GB", | 20260508 03:53:30.112 | INFO | "pinned" : " 29 GB" | 20260508 03:53:30.112 | INFO | } | 20260508 03:53:30.112 | INFO | Perf report for the last 5.14118 seconds: | 20260508 03:53:30.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:30.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 03:53:30.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 03:53:30.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:30.112 | INFO | Uploaded 36GiB to GPU in 2.39885s (15.0072GiB/s) | 20260508 03:53:30.112 | INFO | Downloaded 34GiB to GPU in 2.43314s (13.9737GiB/s) | 20260508 03:53:35.252 | INFO | Analyzing chunk frames = [5029376,5029888) | 20260508 03:53:35.252 | INFO | MemInfo: { | 20260508 03:53:35.252 | INFO | "gpu" : " 17 GB", | 20260508 03:53:35.252 | INFO | "huge" : "", | 20260508 03:53:35.252 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:35.252 | INFO | "malloc" : " 14 GB", | 20260508 03:53:35.252 | INFO | "pinned" : " 29 GB" | 20260508 03:53:35.252 | INFO | } | 20260508 03:53:35.252 | INFO | Perf report for the last 5.14034 seconds: | 20260508 03:53:35.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:35.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 03:53:35.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:53:35.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:35.252 | INFO | Uploaded 36GiB to GPU in 2.41267s (14.9212GiB/s) | 20260508 03:53:35.252 | INFO | Downloaded 34GiB to GPU in 2.43906s (13.9398GiB/s) | 20260508 03:53:35.253 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:53:35.253 | INFO | Total mem usage: 16.7 GiB | 20260508 03:53:35.253 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:53:35.253 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.7%) | 20260508 03:53:35.253 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:53:35.253 | INFO | High Water Breakdown: | 20260508 03:53:35.253 | INFO | 59.7% : 8.6 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:53:35.253 | INFO | 40.2% : 5.8 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:53:35.253 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:53:35.253 | INFO | ------------------------------------------ | 20260508 03:53:35.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:53:35.253 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:53:35.253 | INFO | Total mem usage: 26.5 GiB | 20260508 03:53:35.253 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:53:35.253 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:53:35.253 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:53:35.253 | INFO | High Water Breakdown: | 20260508 03:53:35.253 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:53:35.253 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:53:35.253 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:53:35.253 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:53:35.253 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:53:35.253 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:53:35.253 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:53:35.253 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:53:35.253 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:53:35.253 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:53:35.253 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:53:35.253 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:53:35.253 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:53:35.253 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:53:35.253 | INFO | ------------------------------------------ | 20260508 03:53:35.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:53:35.253 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:53:35.253 | INFO | Total mem usage: 27.8 GiB | 20260508 03:53:35.253 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:53:35.253 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:53:35.253 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:53:35.253 | INFO | High Water Breakdown: | 20260508 03:53:35.253 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:53:35.253 | INFO | ------------------------------------------ | 20260508 03:53:35.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:53:35.253 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:53:35.253 | INFO | Total mem usage: 15.7 GiB | 20260508 03:53:35.253 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:53:35.253 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:53:35.253 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:53:35.253 | INFO | High Water Breakdown: | 20260508 03:53:35.253 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:53:35.253 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:53:35.253 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:53:35.254 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:53:35.254 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:53:35.254 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:53:35.254 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:53:35.254 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:53:35.254 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:53:35.254 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:53:35.254 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:53:35.254 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:53:35.254 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:53:35.254 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:53:35.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:53:35.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:53:35.254 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:53:35.254 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:53:35.254 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:53:35.254 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:53:35.254 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:53:35.254 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:53:35.254 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:53:35.254 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:53:35.254 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:53:35.254 | INFO | ------------------------------------------ | 20260508 03:53:35.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:53:35.254 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:53:35.254 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:53:35.254 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:53:35.254 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.11e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.02e-01, 1.14e-02 / 4.93e-03 | 20260508 03:53:35.254 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.48e+00, 5.91e-01 / 2.24e-01 | 20260508 03:53:35.254 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.23e-01 | 20260508 03:53:35.254 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.56e+01, 3.76e+00 / 1.36e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.31e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.24e-01 / 2.44e-01 | 20260508 03:53:35.254 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.20e+00, 2.66e-01 / 1.17e-01 | 20260508 03:53:35.254 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.57e-01, 5.15e-02 / 2.45e-02 | 20260508 03:53:35.254 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:53:35.254 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:53:35.254 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:53:35.254 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 24.37%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 2.28%, LockUpload -- 2.43e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 2.64%, LockDownload -- 2.81e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 15.48%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 03:53:35.254 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.66e-02 / 0.00e+00 | 20260508 03:53:35.349 | INFO | Starting a round of Baz IO | 20260508 03:53:38.782 | INFO | Populating RTMetrics report | 20260508 03:53:38.782 | INFO | RTMetrics report populated | 20260508 03:53:38.782 | INFO | Opening tmp RTMetrics json file | 20260508 03:53:38.794 | INFO | Serializing RTMetrics to json | 20260508 03:53:38.795 | INFO | Serialization done, swapping tmp file now... | 20260508 03:53:38.796 | INFO | RTMetrics done | 20260508 03:53:38.854 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.505 seconds | 20260508 03:53:38.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.18% / 0.00% / 58.82% | 20260508 03:53:38.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.585051% / 7.94729e-06% / 6.75519e-05% | 20260508 03:53:38.855 | INFO | Baz IO overall: | 20260508 03:53:38.855 | INFO | BytesWritten : 2257471793828 | 20260508 03:53:38.855 | INFO | NumEvents : 1090321233029 | 20260508 03:53:40.393 | INFO | Analyzing chunk frames = [5029888,5030400) | 20260508 03:53:40.393 | INFO | MemInfo: { | 20260508 03:53:40.393 | INFO | "gpu" : " 17 GB", | 20260508 03:53:40.393 | INFO | "huge" : "", | 20260508 03:53:40.393 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:40.393 | INFO | "malloc" : " 14 GB", | 20260508 03:53:40.393 | INFO | "pinned" : " 29 GB" | 20260508 03:53:40.393 | INFO | } | 20260508 03:53:40.393 | INFO | Perf report for the last 5.14116 seconds: | 20260508 03:53:40.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:40.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:40.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:40.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:40.394 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 03:53:40.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:40.394 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 03:53:40.394 | INFO | PRE_HQ : 65.4%, 1.6, 55.5ms, 0 | 20260508 03:53:40.394 | INFO | BAZWRITER : 67.6%, 2.7, 93.1ms, 0 | 20260508 03:53:40.394 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:53:40.394 | INFO | Uploaded 36GiB to GPU in 2.49092s (14.4525GiB/s) | 20260508 03:53:40.394 | INFO | Downloaded 38GiB to GPU in 2.70305s (14.0582GiB/s) | 20260508 03:53:45.532 | INFO | Analyzing chunk frames = [5030400,5030912) | 20260508 03:53:45.532 | INFO | MemInfo: { | 20260508 03:53:45.532 | INFO | "gpu" : " 17 GB", | 20260508 03:53:45.532 | INFO | "huge" : "", | 20260508 03:53:45.532 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:45.532 | INFO | "malloc" : " 14 GB", | 20260508 03:53:45.532 | INFO | "pinned" : " 29 GB" | 20260508 03:53:45.532 | INFO | } | 20260508 03:53:45.532 | INFO | Perf report for the last 5.13896 seconds: | 20260508 03:53:45.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:45.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.532 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 03:53:45.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.532 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:53:45.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:45.533 | INFO | Uploaded 36GiB to GPU in 2.56987s (14.0085GiB/s) | 20260508 03:53:45.533 | INFO | Downloaded 34GiB to GPU in 2.42994s (13.9921GiB/s) | 20260508 03:53:50.664 | INFO | Analyzing chunk frames = [5030912,5031424) | 20260508 03:53:50.664 | INFO | MemInfo: { | 20260508 03:53:50.664 | INFO | "gpu" : " 17 GB", | 20260508 03:53:50.664 | INFO | "huge" : "", | 20260508 03:53:50.664 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:50.664 | INFO | "malloc" : " 14 GB", | 20260508 03:53:50.664 | INFO | "pinned" : " 29 GB" | 20260508 03:53:50.664 | INFO | } | 20260508 03:53:50.664 | INFO | Perf report for the last 5.13215 seconds: | 20260508 03:53:50.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:50.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 03:53:50.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:53:50.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:50.665 | INFO | Uploaded 36GiB to GPU in 2.52606s (14.2514GiB/s) | 20260508 03:53:50.665 | INFO | Downloaded 34GiB to GPU in 2.41371s (14.0862GiB/s) | 20260508 03:53:55.804 | INFO | Analyzing chunk frames = [5031424,5031936) | 20260508 03:53:55.804 | INFO | MemInfo: { | 20260508 03:53:55.804 | INFO | "gpu" : " 17 GB", | 20260508 03:53:55.804 | INFO | "huge" : "", | 20260508 03:53:55.804 | INFO | "huge_shared" : " 29 GB", | 20260508 03:53:55.804 | INFO | "malloc" : " 14 GB", | 20260508 03:53:55.804 | INFO | "pinned" : " 29 GB" | 20260508 03:53:55.804 | INFO | } | 20260508 03:53:55.805 | INFO | Perf report for the last 5.14004 seconds: | 20260508 03:53:55.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:53:55.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 03:53:55.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:53:55.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:53:55.805 | INFO | Uploaded 36GiB to GPU in 2.52238s (14.2722GiB/s) | 20260508 03:53:55.805 | INFO | Downloaded 34GiB to GPU in 2.509s (13.5512GiB/s) | 20260508 03:54:00.945 | INFO | Analyzing chunk frames = [5031936,5032448) | 20260508 03:54:00.945 | INFO | MemInfo: { | 20260508 03:54:00.945 | INFO | "gpu" : " 17 GB", | 20260508 03:54:00.945 | INFO | "huge" : "", | 20260508 03:54:00.946 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:00.946 | INFO | "malloc" : " 14 GB", | 20260508 03:54:00.946 | INFO | "pinned" : " 29 GB" | 20260508 03:54:00.946 | INFO | } | 20260508 03:54:00.946 | INFO | Perf report for the last 5.14115 seconds: | 20260508 03:54:00.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:00.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 03:54:00.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:54:00.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:00.946 | INFO | Uploaded 36GiB to GPU in 2.38929s (15.0672GiB/s) | 20260508 03:54:00.946 | INFO | Downloaded 34GiB to GPU in 2.5585s (13.289GiB/s) | 20260508 03:54:06.090 | INFO | Analyzing chunk frames = [5032448,5032960) | 20260508 03:54:06.090 | INFO | MemInfo: { | 20260508 03:54:06.090 | INFO | "gpu" : " 17 GB", | 20260508 03:54:06.090 | INFO | "huge" : "", | 20260508 03:54:06.090 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:06.090 | INFO | "malloc" : " 14 GB", | 20260508 03:54:06.090 | INFO | "pinned" : " 29 GB" | 20260508 03:54:06.090 | INFO | } | 20260508 03:54:06.090 | INFO | Perf report for the last 5.14429 seconds: | 20260508 03:54:06.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:06.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.090 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 03:54:06.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.091 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260508 03:54:06.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:06.091 | INFO | Uploaded 36GiB to GPU in 2.43959s (14.7566GiB/s) | 20260508 03:54:06.091 | INFO | Downloaded 34GiB to GPU in 2.40479s (14.1385GiB/s) | 20260508 03:54:11.225 | INFO | Analyzing chunk frames = [5032960,5033472) | 20260508 03:54:11.225 | INFO | MemInfo: { | 20260508 03:54:11.225 | INFO | "gpu" : " 17 GB", | 20260508 03:54:11.225 | INFO | "huge" : "", | 20260508 03:54:11.225 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:11.225 | INFO | "malloc" : " 14 GB", | 20260508 03:54:11.225 | INFO | "pinned" : " 29 GB" | 20260508 03:54:11.225 | INFO | } | 20260508 03:54:11.226 | INFO | Perf report for the last 5.13562 seconds: | 20260508 03:54:11.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:11.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.226 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:54:11.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.226 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 03:54:11.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:11.227 | INFO | Uploaded 36GiB to GPU in 2.46254s (14.6191GiB/s) | 20260508 03:54:11.227 | INFO | Downloaded 34GiB to GPU in 2.43273s (13.9761GiB/s) | 20260508 03:54:16.363 | INFO | Analyzing chunk frames = [5033472,5033984) | 20260508 03:54:16.363 | INFO | MemInfo: { | 20260508 03:54:16.363 | INFO | "gpu" : " 17 GB", | 20260508 03:54:16.363 | INFO | "huge" : "", | 20260508 03:54:16.363 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:16.364 | INFO | "malloc" : " 14 GB", | 20260508 03:54:16.364 | INFO | "pinned" : " 29 GB" | 20260508 03:54:16.364 | INFO | } | 20260508 03:54:16.364 | INFO | Perf report for the last 5.13819 seconds: | 20260508 03:54:16.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:16.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:54:16.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 03:54:16.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:16.364 | INFO | Uploaded 36GiB to GPU in 2.4731s (14.5567GiB/s) | 20260508 03:54:16.364 | INFO | Downloaded 34GiB to GPU in 2.43845s (13.9433GiB/s) | 20260508 03:54:16.477 | INFO | Starting a round of Baz IO | 20260508 03:54:19.900 | INFO | Populating RTMetrics report | 20260508 03:54:19.900 | INFO | RTMetrics report populated | 20260508 03:54:19.900 | INFO | Opening tmp RTMetrics json file | 20260508 03:54:19.909 | INFO | Serializing RTMetrics to json | 20260508 03:54:19.915 | INFO | Serialization done, swapping tmp file now... | 20260508 03:54:19.915 | INFO | RTMetrics done | 20260508 03:54:19.969 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.492 seconds | 20260508 03:54:19.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.18% / 0.00% / 58.82% | 20260508 03:54:19.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.581431% / 0% / 6.35783e-05% | 20260508 03:54:19.969 | INFO | Baz IO overall: | 20260508 03:54:19.969 | INFO | BytesWritten : 2259196004927 | 20260508 03:54:19.969 | INFO | NumEvents : 1091145759181 | 20260508 03:54:21.504 | INFO | Analyzing chunk frames = [5033984,5034496) | 20260508 03:54:21.504 | INFO | MemInfo: { | 20260508 03:54:21.504 | INFO | "gpu" : " 17 GB", | 20260508 03:54:21.504 | INFO | "huge" : "", | 20260508 03:54:21.504 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:21.504 | INFO | "malloc" : " 14 GB", | 20260508 03:54:21.504 | INFO | "pinned" : " 29 GB" | 20260508 03:54:21.504 | INFO | } | 20260508 03:54:21.504 | INFO | Perf report for the last 5.1405 seconds: | 20260508 03:54:21.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:21.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:21.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:21.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:21.504 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 03:54:21.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:21.504 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:54:21.505 | INFO | PRE_HQ : 66.4%, 1.8, 62.4ms, 0 | 20260508 03:54:21.505 | INFO | BAZWRITER : 67.4%, 2.9, 99.0ms, 0 | 20260508 03:54:21.505 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:54:21.505 | INFO | Uploaded 36GiB to GPU in 2.54716s (14.1334GiB/s) | 20260508 03:54:21.505 | INFO | Downloaded 38GiB to GPU in 2.70021s (14.073GiB/s) | 20260508 03:54:26.641 | INFO | Analyzing chunk frames = [5034496,5035008) | 20260508 03:54:26.641 | INFO | MemInfo: { | 20260508 03:54:26.641 | INFO | "gpu" : " 17 GB", | 20260508 03:54:26.641 | INFO | "huge" : "", | 20260508 03:54:26.641 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:26.641 | INFO | "malloc" : " 14 GB", | 20260508 03:54:26.641 | INFO | "pinned" : " 29 GB" | 20260508 03:54:26.641 | INFO | } | 20260508 03:54:26.641 | INFO | Perf report for the last 5.13685 seconds: | 20260508 03:54:26.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:26.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.641 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 03:54:26.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.641 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 03:54:26.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:26.642 | INFO | Uploaded 36GiB to GPU in 2.56817s (14.0178GiB/s) | 20260508 03:54:26.642 | INFO | Downloaded 34GiB to GPU in 2.44391s (13.9121GiB/s) | 20260508 03:54:31.784 | INFO | Analyzing chunk frames = [5035008,5035520) | 20260508 03:54:31.784 | INFO | MemInfo: { | 20260508 03:54:31.784 | INFO | "gpu" : " 17 GB", | 20260508 03:54:31.784 | INFO | "huge" : "", | 20260508 03:54:31.784 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:31.784 | INFO | "malloc" : " 14 GB", | 20260508 03:54:31.784 | INFO | "pinned" : " 29 GB" | 20260508 03:54:31.784 | INFO | } | 20260508 03:54:31.784 | INFO | Perf report for the last 5.1426 seconds: | 20260508 03:54:31.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:31.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 03:54:31.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:54:31.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:31.784 | INFO | Uploaded 36GiB to GPU in 2.54538s (14.1433GiB/s) | 20260508 03:54:31.784 | INFO | Downloaded 34GiB to GPU in 2.50555s (13.5699GiB/s) | 20260508 03:54:36.916 | INFO | Analyzing chunk frames = [5035520,5036032) | 20260508 03:54:36.916 | INFO | MemInfo: { | 20260508 03:54:36.916 | INFO | "gpu" : " 17 GB", | 20260508 03:54:36.916 | INFO | "huge" : "", | 20260508 03:54:36.916 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:36.916 | INFO | "malloc" : " 14 GB", | 20260508 03:54:36.916 | INFO | "pinned" : " 29 GB" | 20260508 03:54:36.916 | INFO | } | 20260508 03:54:36.916 | INFO | Perf report for the last 5.13245 seconds: | 20260508 03:54:36.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:36.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.916 | INFO | ANALYSIS : 67.1%, 3.0, 103.7ms, 0 | 20260508 03:54:36.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.916 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:54:36.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:36.917 | INFO | Uploaded 36GiB to GPU in 2.57366s (13.9879GiB/s) | 20260508 03:54:36.917 | INFO | Downloaded 34GiB to GPU in 2.41911s (14.0547GiB/s) | 20260508 03:54:42.062 | INFO | Analyzing chunk frames = [5036032,5036544) | 20260508 03:54:42.062 | INFO | MemInfo: { | 20260508 03:54:42.062 | INFO | "gpu" : " 17 GB", | 20260508 03:54:42.062 | INFO | "huge" : "", | 20260508 03:54:42.062 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:42.062 | INFO | "malloc" : " 14 GB", | 20260508 03:54:42.062 | INFO | "pinned" : " 29 GB" | 20260508 03:54:42.062 | INFO | } | 20260508 03:54:42.062 | INFO | Perf report for the last 5.14583 seconds: | 20260508 03:54:42.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:42.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 03:54:42.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 03:54:42.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:42.062 | INFO | Uploaded 36GiB to GPU in 2.353s (15.2996GiB/s) | 20260508 03:54:42.062 | INFO | Downloaded 34GiB to GPU in 2.54391s (13.3653GiB/s) | 20260508 03:54:47.203 | INFO | Analyzing chunk frames = [5036544,5037056) | 20260508 03:54:47.203 | INFO | MemInfo: { | 20260508 03:54:47.203 | INFO | "gpu" : " 17 GB", | 20260508 03:54:47.204 | INFO | "huge" : "", | 20260508 03:54:47.204 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:47.204 | INFO | "malloc" : " 14 GB", | 20260508 03:54:47.204 | INFO | "pinned" : " 29 GB" | 20260508 03:54:47.204 | INFO | } | 20260508 03:54:47.204 | INFO | Perf report for the last 5.14159 seconds: | 20260508 03:54:47.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:47.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 03:54:47.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:54:47.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:47.204 | INFO | Uploaded 36GiB to GPU in 2.41795s (14.8887GiB/s) | 20260508 03:54:47.204 | INFO | Downloaded 34GiB to GPU in 2.43007s (13.9914GiB/s) | 20260508 03:54:52.340 | INFO | Analyzing chunk frames = [5037056,5037568) | 20260508 03:54:52.340 | INFO | MemInfo: { | 20260508 03:54:52.340 | INFO | "gpu" : " 17 GB", | 20260508 03:54:52.340 | INFO | "huge" : "", | 20260508 03:54:52.340 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:52.340 | INFO | "malloc" : " 14 GB", | 20260508 03:54:52.340 | INFO | "pinned" : " 29 GB" | 20260508 03:54:52.340 | INFO | } | 20260508 03:54:52.340 | INFO | Perf report for the last 5.13646 seconds: | 20260508 03:54:52.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:52.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.340 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 03:54:52.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.340 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 03:54:52.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:52.341 | INFO | Uploaded 36GiB to GPU in 2.43431s (14.7886GiB/s) | 20260508 03:54:52.341 | INFO | Downloaded 34GiB to GPU in 2.42659s (14.0115GiB/s) | 20260508 03:54:57.479 | INFO | Analyzing chunk frames = [5037568,5038080) | 20260508 03:54:57.479 | INFO | MemInfo: { | 20260508 03:54:57.479 | INFO | "gpu" : " 17 GB", | 20260508 03:54:57.479 | INFO | "huge" : "", | 20260508 03:54:57.479 | INFO | "huge_shared" : " 29 GB", | 20260508 03:54:57.479 | INFO | "malloc" : " 14 GB", | 20260508 03:54:57.479 | INFO | "pinned" : " 29 GB" | 20260508 03:54:57.479 | INFO | } | 20260508 03:54:57.479 | INFO | Perf report for the last 5.13898 seconds: | 20260508 03:54:57.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:54:57.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.479 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 03:54:57.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.479 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 03:54:57.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:54:57.480 | INFO | Uploaded 36GiB to GPU in 2.56297s (14.0462GiB/s) | 20260508 03:54:57.480 | INFO | Downloaded 34GiB to GPU in 2.46893s (13.7711GiB/s) | 20260508 03:54:57.480 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:54:57.480 | INFO | Total mem usage: 16.7 GiB | 20260508 03:54:57.480 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:54:57.480 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:54:57.480 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:54:57.480 | INFO | High Water Breakdown: | 20260508 03:54:57.480 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:54:57.480 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:54:57.480 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:54:57.480 | INFO | ------------------------------------------ | 20260508 03:54:57.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:54:57.480 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:54:57.480 | INFO | Total mem usage: 26.5 GiB | 20260508 03:54:57.480 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:54:57.480 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:54:57.480 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:54:57.480 | INFO | High Water Breakdown: | 20260508 03:54:57.480 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:54:57.480 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:54:57.480 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:54:57.480 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:54:57.480 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:54:57.480 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:54:57.480 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:54:57.480 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:54:57.480 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:54:57.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:54:57.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:54:57.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:54:57.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:54:57.480 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:54:57.480 | INFO | ------------------------------------------ | 20260508 03:54:57.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:54:57.480 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:54:57.480 | INFO | Total mem usage: 27.8 GiB | 20260508 03:54:57.480 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:54:57.480 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:54:57.480 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:54:57.480 | INFO | High Water Breakdown: | 20260508 03:54:57.480 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:54:57.480 | INFO | ------------------------------------------ | 20260508 03:54:57.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:54:57.481 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:54:57.481 | INFO | Total mem usage: 15.7 GiB | 20260508 03:54:57.481 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:54:57.481 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:54:57.481 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:54:57.481 | INFO | High Water Breakdown: | 20260508 03:54:57.481 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:54:57.481 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:54:57.481 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:54:57.481 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:54:57.481 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:54:57.481 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:54:57.481 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:54:57.481 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:54:57.481 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:54:57.481 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:54:57.481 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:54:57.481 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:54:57.481 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:54:57.481 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:54:57.481 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:54:57.481 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:54:57.481 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:54:57.481 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:54:57.481 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:54:57.481 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:54:57.481 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:54:57.481 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:54:57.481 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:54:57.481 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:54:57.481 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:54:57.481 | INFO | ------------------------------------------ | 20260508 03:54:57.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:54:57.481 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:54:57.481 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:54:57.481 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:54:57.481 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.40e+01 / 1.38e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.05e-01, 1.23e-02 / 5.90e-03 | 20260508 03:54:57.481 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.46e+00, 5.90e-01 / 1.75e-01 | 20260508 03:54:57.481 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.09e-01 / 1.02e-01 | 20260508 03:54:57.481 | INFO | PerfReport: 15.97%, DME -- 4.18e+01 / 4.58e+01, 3.84e+00 / 1.86e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.70e+00 / 1.38e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.41e+00 / 2.63e+00, 4.20e-01 / 2.03e-01 | 20260508 03:54:57.481 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.24e+00, 2.66e-01 / 1.44e-01 | 20260508 03:54:57.481 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.64e-01, 5.10e-02 / 2.17e-02 | 20260508 03:54:57.481 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:54:57.481 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:54:57.481 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:54:57.481 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 24.39%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 24.37%, Download -- 2.59e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 2.22%, LockUpload -- 2.36e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 2.72%, LockDownload -- 2.90e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 15.56%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 03:54:57.481 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260508 03:54:57.577 | INFO | Starting a round of Baz IO | 20260508 03:55:01.002 | INFO | Populating RTMetrics report | 20260508 03:55:01.002 | INFO | RTMetrics report populated | 20260508 03:55:01.002 | INFO | Opening tmp RTMetrics json file | 20260508 03:55:01.018 | INFO | Serializing RTMetrics to json | 20260508 03:55:01.019 | INFO | Serialization done, swapping tmp file now... | 20260508 03:55:01.020 | INFO | RTMetrics done | 20260508 03:55:01.076 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.499 seconds | 20260508 03:55:01.077 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.18% / 0.00% / 58.82% | 20260508 03:55:01.077 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.579234% / 0% / 5.5631e-05% | 20260508 03:55:01.077 | INFO | Baz IO overall: | 20260508 03:55:01.077 | INFO | BytesWritten : 2260920215147 | 20260508 03:55:01.077 | INFO | NumEvents : 1091969949425 | 20260508 03:55:02.618 | INFO | Analyzing chunk frames = [5038080,5038592) | 20260508 03:55:02.618 | INFO | MemInfo: { | 20260508 03:55:02.618 | INFO | "gpu" : " 17 GB", | 20260508 03:55:02.618 | INFO | "huge" : "", | 20260508 03:55:02.618 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:02.618 | INFO | "malloc" : " 14 GB", | 20260508 03:55:02.618 | INFO | "pinned" : " 29 GB" | 20260508 03:55:02.618 | INFO | } | 20260508 03:55:02.618 | INFO | Perf report for the last 5.13861 seconds: | 20260508 03:55:02.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:02.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:02.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:02.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:02.618 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 03:55:02.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:02.618 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 03:55:02.618 | INFO | PRE_HQ : 65.8%, 1.6, 56.2ms, 0 | 20260508 03:55:02.618 | INFO | BAZWRITER : 67.6%, 2.7, 93.8ms, 0 | 20260508 03:55:02.618 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:55:02.618 | INFO | Uploaded 36GiB to GPU in 2.56816s (14.0178GiB/s) | 20260508 03:55:02.618 | INFO | Downloaded 38GiB to GPU in 2.69905s (14.079GiB/s) | 20260508 03:55:07.755 | INFO | Analyzing chunk frames = [5038592,5039104) | 20260508 03:55:07.755 | INFO | MemInfo: { | 20260508 03:55:07.755 | INFO | "gpu" : " 17 GB", | 20260508 03:55:07.755 | INFO | "huge" : "", | 20260508 03:55:07.755 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:07.755 | INFO | "malloc" : " 14 GB", | 20260508 03:55:07.755 | INFO | "pinned" : " 29 GB" | 20260508 03:55:07.755 | INFO | } | 20260508 03:55:07.755 | INFO | Perf report for the last 5.13744 seconds: | 20260508 03:55:07.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:07.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.755 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 03:55:07.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.755 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:55:07.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:07.756 | INFO | Uploaded 36GiB to GPU in 2.59766s (13.8586GiB/s) | 20260508 03:55:07.756 | INFO | Downloaded 34GiB to GPU in 2.4347s (13.9648GiB/s) | 20260508 03:55:12.896 | INFO | Analyzing chunk frames = [5039104,5039616) | 20260508 03:55:12.896 | INFO | MemInfo: { | 20260508 03:55:12.896 | INFO | "gpu" : " 17 GB", | 20260508 03:55:12.896 | INFO | "huge" : "", | 20260508 03:55:12.896 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:12.896 | INFO | "malloc" : " 14 GB", | 20260508 03:55:12.896 | INFO | "pinned" : " 29 GB" | 20260508 03:55:12.896 | INFO | } | 20260508 03:55:12.896 | INFO | Perf report for the last 5.14105 seconds: | 20260508 03:55:12.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:12.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.896 | INFO | ANALYSIS : 67.6%, 3.0, 104.9ms, 0 | 20260508 03:55:12.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.897 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 03:55:12.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:12.897 | INFO | Uploaded 36GiB to GPU in 2.58732s (13.914GiB/s) | 20260508 03:55:12.897 | INFO | Downloaded 34GiB to GPU in 2.49078s (13.6504GiB/s) | 20260508 03:55:18.036 | INFO | Analyzing chunk frames = [5039616,5040128) | 20260508 03:55:18.036 | INFO | MemInfo: { | 20260508 03:55:18.036 | INFO | "gpu" : " 17 GB", | 20260508 03:55:18.036 | INFO | "huge" : "", | 20260508 03:55:18.036 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:18.036 | INFO | "malloc" : " 14 GB", | 20260508 03:55:18.036 | INFO | "pinned" : " 29 GB" | 20260508 03:55:18.036 | INFO | } | 20260508 03:55:18.036 | INFO | Perf report for the last 5.1399 seconds: | 20260508 03:55:18.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:18.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.036 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 03:55:18.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.036 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 03:55:18.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:18.037 | INFO | Uploaded 36GiB to GPU in 2.57898s (13.959GiB/s) | 20260508 03:55:18.037 | INFO | Downloaded 34GiB to GPU in 2.49645s (13.6193GiB/s) | 20260508 03:55:23.173 | INFO | Analyzing chunk frames = [5040128,5040640) | 20260508 03:55:23.173 | INFO | MemInfo: { | 20260508 03:55:23.173 | INFO | "gpu" : " 17 GB", | 20260508 03:55:23.173 | INFO | "huge" : "", | 20260508 03:55:23.173 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:23.173 | INFO | "malloc" : " 14 GB", | 20260508 03:55:23.173 | INFO | "pinned" : " 29 GB" | 20260508 03:55:23.173 | INFO | } | 20260508 03:55:23.173 | INFO | Perf report for the last 5.13668 seconds: | 20260508 03:55:23.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:23.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.173 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 03:55:23.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.173 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:55:23.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:23.174 | INFO | Uploaded 36GiB to GPU in 2.37455s (15.1608GiB/s) | 20260508 03:55:23.174 | INFO | Downloaded 34GiB to GPU in 2.5766s (13.1957GiB/s) | 20260508 03:55:28.316 | INFO | Analyzing chunk frames = [5040640,5041152) | 20260508 03:55:28.316 | INFO | MemInfo: { | 20260508 03:55:28.316 | INFO | "gpu" : " 17 GB", | 20260508 03:55:28.316 | INFO | "huge" : "", | 20260508 03:55:28.316 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:28.316 | INFO | "malloc" : " 14 GB", | 20260508 03:55:28.316 | INFO | "pinned" : " 29 GB" | 20260508 03:55:28.316 | INFO | } | 20260508 03:55:28.316 | INFO | Perf report for the last 5.14335 seconds: | 20260508 03:55:28.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:28.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.316 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 03:55:28.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.317 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 03:55:28.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:28.317 | INFO | Uploaded 36GiB to GPU in 2.45159s (14.6844GiB/s) | 20260508 03:55:28.317 | INFO | Downloaded 34GiB to GPU in 2.41218s (14.0951GiB/s) | 20260508 03:55:33.456 | INFO | Analyzing chunk frames = [5041152,5041664) | 20260508 03:55:33.456 | INFO | MemInfo: { | 20260508 03:55:33.456 | INFO | "gpu" : " 17 GB", | 20260508 03:55:33.456 | INFO | "huge" : "", | 20260508 03:55:33.457 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:33.457 | INFO | "malloc" : " 14 GB", | 20260508 03:55:33.457 | INFO | "pinned" : " 29 GB" | 20260508 03:55:33.457 | INFO | } | 20260508 03:55:33.457 | INFO | Perf report for the last 5.14049 seconds: | 20260508 03:55:33.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:33.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:55:33.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 03:55:33.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:33.457 | INFO | Uploaded 36GiB to GPU in 2.48674s (14.4768GiB/s) | 20260508 03:55:33.457 | INFO | Downloaded 34GiB to GPU in 2.43091s (13.9865GiB/s) | 20260508 03:55:38.595 | INFO | Analyzing chunk frames = [5041664,5042176) | 20260508 03:55:38.595 | INFO | MemInfo: { | 20260508 03:55:38.595 | INFO | "gpu" : " 17 GB", | 20260508 03:55:38.595 | INFO | "huge" : "", | 20260508 03:55:38.595 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:38.595 | INFO | "malloc" : " 14 GB", | 20260508 03:55:38.596 | INFO | "pinned" : " 29 GB" | 20260508 03:55:38.596 | INFO | } | 20260508 03:55:38.596 | INFO | Perf report for the last 5.1387 seconds: | 20260508 03:55:38.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:38.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 03:55:38.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 03:55:38.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:38.596 | INFO | Uploaded 36GiB to GPU in 2.48144s (14.5077GiB/s) | 20260508 03:55:38.596 | INFO | Downloaded 34GiB to GPU in 2.42704s (14.0088GiB/s) | 20260508 03:55:38.708 | INFO | Starting a round of Baz IO | 20260508 03:55:42.133 | INFO | Populating RTMetrics report | 20260508 03:55:42.133 | INFO | RTMetrics report populated | 20260508 03:55:42.134 | INFO | Opening tmp RTMetrics json file | 20260508 03:55:42.143 | INFO | Serializing RTMetrics to json | 20260508 03:55:42.147 | INFO | Serialization done, swapping tmp file now... | 20260508 03:55:42.148 | INFO | RTMetrics done | 20260508 03:55:42.209 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.501 seconds | 20260508 03:55:42.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.17% / 0.00% / 58.82% | 20260508 03:55:42.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.57921% / 0% / 8.34465e-05% | 20260508 03:55:42.209 | INFO | Baz IO overall: | 20260508 03:55:42.209 | INFO | BytesWritten : 2262644098963 | 20260508 03:55:42.209 | INFO | NumEvents : 1092794215997 | 20260508 03:55:43.727 | INFO | Analyzing chunk frames = [5042176,5042688) | 20260508 03:55:43.727 | INFO | MemInfo: { | 20260508 03:55:43.727 | INFO | "gpu" : " 17 GB", | 20260508 03:55:43.727 | INFO | "huge" : "", | 20260508 03:55:43.727 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:43.728 | INFO | "malloc" : " 14 GB", | 20260508 03:55:43.728 | INFO | "pinned" : " 29 GB" | 20260508 03:55:43.728 | INFO | } | 20260508 03:55:43.728 | INFO | Perf report for the last 5.13206 seconds: | 20260508 03:55:43.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:43.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:43.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:43.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:43.728 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 03:55:43.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:43.728 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 03:55:43.728 | INFO | PRE_HQ : 66.5%, 1.8, 62.1ms, 0 | 20260508 03:55:43.728 | INFO | BAZWRITER : 67.6%, 2.9, 99.3ms, 0 | 20260508 03:55:43.728 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:55:43.728 | INFO | Uploaded 36GiB to GPU in 2.59694s (13.8625GiB/s) | 20260508 03:55:43.728 | INFO | Downloaded 38GiB to GPU in 2.69996s (14.0743GiB/s) | 20260508 03:55:48.868 | INFO | Analyzing chunk frames = [5042688,5043200) | 20260508 03:55:48.868 | INFO | MemInfo: { | 20260508 03:55:48.868 | INFO | "gpu" : " 17 GB", | 20260508 03:55:48.868 | INFO | "huge" : "", | 20260508 03:55:48.868 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:48.868 | INFO | "malloc" : " 14 GB", | 20260508 03:55:48.868 | INFO | "pinned" : " 29 GB" | 20260508 03:55:48.868 | INFO | } | 20260508 03:55:48.868 | INFO | Perf report for the last 5.14072 seconds: | 20260508 03:55:48.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:48.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.868 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 03:55:48.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.869 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:55:48.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:48.869 | INFO | Uploaded 36GiB to GPU in 2.59377s (13.8794GiB/s) | 20260508 03:55:48.869 | INFO | Downloaded 34GiB to GPU in 2.41977s (14.0509GiB/s) | 20260508 03:55:54.012 | INFO | Analyzing chunk frames = [5043200,5043712) | 20260508 03:55:54.012 | INFO | MemInfo: { | 20260508 03:55:54.012 | INFO | "gpu" : " 17 GB", | 20260508 03:55:54.012 | INFO | "huge" : "", | 20260508 03:55:54.012 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:54.012 | INFO | "malloc" : " 14 GB", | 20260508 03:55:54.012 | INFO | "pinned" : " 29 GB" | 20260508 03:55:54.012 | INFO | } | 20260508 03:55:54.012 | INFO | Perf report for the last 5.14376 seconds: | 20260508 03:55:54.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:54.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.012 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 03:55:54.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.012 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:55:54.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:54.013 | INFO | Uploaded 36GiB to GPU in 2.58963s (13.9016GiB/s) | 20260508 03:55:54.013 | INFO | Downloaded 34GiB to GPU in 2.4907s (13.6508GiB/s) | 20260508 03:55:59.156 | INFO | Analyzing chunk frames = [5043712,5044224) | 20260508 03:55:59.156 | INFO | MemInfo: { | 20260508 03:55:59.156 | INFO | "gpu" : " 17 GB", | 20260508 03:55:59.156 | INFO | "huge" : "", | 20260508 03:55:59.156 | INFO | "huge_shared" : " 29 GB", | 20260508 03:55:59.156 | INFO | "malloc" : " 14 GB", | 20260508 03:55:59.156 | INFO | "pinned" : " 29 GB" | 20260508 03:55:59.156 | INFO | } | 20260508 03:55:59.156 | INFO | Perf report for the last 5.14413 seconds: | 20260508 03:55:59.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:55:59.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.156 | INFO | ANALYSIS : 67.3%, 2.9, 104.1ms, 0 | 20260508 03:55:59.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.156 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:55:59.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:55:59.157 | INFO | Uploaded 36GiB to GPU in 2.60196s (13.8357GiB/s) | 20260508 03:55:59.157 | INFO | Downloaded 34GiB to GPU in 2.42887s (13.9983GiB/s) | 20260508 03:56:04.290 | INFO | Analyzing chunk frames = [5044224,5044736) | 20260508 03:56:04.291 | INFO | MemInfo: { | 20260508 03:56:04.291 | INFO | "gpu" : " 17 GB", | 20260508 03:56:04.291 | INFO | "huge" : "", | 20260508 03:56:04.291 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:04.291 | INFO | "malloc" : " 14 GB", | 20260508 03:56:04.291 | INFO | "pinned" : " 29 GB" | 20260508 03:56:04.291 | INFO | } | 20260508 03:56:04.291 | INFO | Perf report for the last 5.13445 seconds: | 20260508 03:56:04.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:04.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 03:56:04.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:56:04.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:04.291 | INFO | Uploaded 36GiB to GPU in 2.36586s (15.2165GiB/s) | 20260508 03:56:04.291 | INFO | Downloaded 34GiB to GPU in 2.548s (13.3438GiB/s) | 20260508 03:56:09.428 | INFO | Analyzing chunk frames = [5044736,5045248) | 20260508 03:56:09.428 | INFO | MemInfo: { | 20260508 03:56:09.428 | INFO | "gpu" : " 17 GB", | 20260508 03:56:09.428 | INFO | "huge" : "", | 20260508 03:56:09.428 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:09.428 | INFO | "malloc" : " 14 GB", | 20260508 03:56:09.428 | INFO | "pinned" : " 29 GB" | 20260508 03:56:09.428 | INFO | } | 20260508 03:56:09.428 | INFO | Perf report for the last 5.13711 seconds: | 20260508 03:56:09.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:09.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 03:56:09.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:56:09.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:09.428 | INFO | Uploaded 36GiB to GPU in 2.45204s (14.6816GiB/s) | 20260508 03:56:09.428 | INFO | Downloaded 34GiB to GPU in 2.41775s (14.0626GiB/s) | 20260508 03:56:14.570 | INFO | Analyzing chunk frames = [5045248,5045760) | 20260508 03:56:14.570 | INFO | MemInfo: { | 20260508 03:56:14.570 | INFO | "gpu" : " 17 GB", | 20260508 03:56:14.570 | INFO | "huge" : "", | 20260508 03:56:14.571 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:14.571 | INFO | "malloc" : " 14 GB", | 20260508 03:56:14.571 | INFO | "pinned" : " 29 GB" | 20260508 03:56:14.571 | INFO | } | 20260508 03:56:14.571 | INFO | Perf report for the last 5.14348 seconds: | 20260508 03:56:14.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:14.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.571 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 03:56:14.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.572 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 03:56:14.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:14.572 | INFO | Uploaded 36GiB to GPU in 2.45495s (14.6643GiB/s) | 20260508 03:56:14.572 | INFO | Downloaded 34GiB to GPU in 2.41423s (14.0832GiB/s) | 20260508 03:56:19.710 | INFO | Analyzing chunk frames = [5045760,5046272) | 20260508 03:56:19.710 | INFO | MemInfo: { | 20260508 03:56:19.710 | INFO | "gpu" : " 17 GB", | 20260508 03:56:19.710 | INFO | "huge" : "", | 20260508 03:56:19.710 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:19.710 | INFO | "malloc" : " 14 GB", | 20260508 03:56:19.710 | INFO | "pinned" : " 29 GB" | 20260508 03:56:19.711 | INFO | } | 20260508 03:56:19.711 | INFO | Perf report for the last 5.13919 seconds: | 20260508 03:56:19.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:19.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 03:56:19.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | PRE_HQ : 34.8%, 1.0, 18.0ms, 0 | 20260508 03:56:19.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:19.711 | INFO | Uploaded 36GiB to GPU in 2.4682s (14.5855GiB/s) | 20260508 03:56:19.711 | INFO | Downloaded 34GiB to GPU in 2.41844s (14.0586GiB/s) | 20260508 03:56:19.711 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:56:19.711 | INFO | Total mem usage: 16.7 GiB | 20260508 03:56:19.711 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:56:19.711 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:56:19.711 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:56:19.711 | INFO | High Water Breakdown: | 20260508 03:56:19.711 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:56:19.711 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:56:19.711 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:56:19.711 | INFO | ------------------------------------------ | 20260508 03:56:19.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:56:19.711 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:56:19.711 | INFO | Total mem usage: 26.5 GiB | 20260508 03:56:19.712 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:56:19.712 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 03:56:19.712 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 03:56:19.712 | INFO | High Water Breakdown: | 20260508 03:56:19.712 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:56:19.712 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:56:19.712 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:56:19.712 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:56:19.712 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:56:19.712 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:56:19.712 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:56:19.712 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:56:19.712 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:56:19.712 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:56:19.712 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:56:19.712 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:56:19.712 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:56:19.712 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:56:19.712 | INFO | ------------------------------------------ | 20260508 03:56:19.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:56:19.712 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:56:19.712 | INFO | Total mem usage: 27.8 GiB | 20260508 03:56:19.712 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:56:19.712 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:56:19.712 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:56:19.712 | INFO | High Water Breakdown: | 20260508 03:56:19.712 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:56:19.712 | INFO | ------------------------------------------ | 20260508 03:56:19.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:56:19.712 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:56:19.712 | INFO | Total mem usage: 15.7 GiB | 20260508 03:56:19.712 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:56:19.712 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:56:19.712 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:56:19.712 | INFO | High Water Breakdown: | 20260508 03:56:19.712 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:56:19.712 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:56:19.712 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:56:19.712 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:56:19.712 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:56:19.712 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:56:19.712 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:56:19.712 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:56:19.712 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:56:19.712 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:56:19.712 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:56:19.712 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:56:19.712 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:56:19.712 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:56:19.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:56:19.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:56:19.712 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:56:19.712 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:56:19.712 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:56:19.712 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:56:19.712 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:56:19.712 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:56:19.712 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:56:19.712 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:56:19.712 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:56:19.712 | INFO | ------------------------------------------ | 20260508 03:56:19.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:56:19.712 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:56:19.712 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:56:19.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:56:19.712 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.41e+01 / 1.65e+00 | 20260508 03:56:19.712 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.04e-01, 1.28e-02 / 6.91e-03 | 20260508 03:56:19.712 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.51e+00, 5.79e-01 / 1.69e-01 | 20260508 03:56:19.712 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.48e+00, 2.08e-01 / 1.19e-01 | 20260508 03:56:19.712 | INFO | PerfReport: 16.07%, DME -- 4.19e+01 / 4.57e+01, 4.06e+00 / 1.90e+00 | 20260508 03:56:19.712 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.84e+01 / 1.98e+01, 1.63e+00 / 1.29e+00 | 20260508 03:56:19.712 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.41e+00 / 2.67e+00, 4.14e-01 / 2.36e-01 | 20260508 03:56:19.712 | INFO | PerfReport: 3.25%, Metrics -- 1.06e+00 / 1.20e+00, 2.63e-01 / 1.24e-01 | 20260508 03:56:19.712 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.25e-01 / 3.62e-01, 5.10e-02 / 2.97e-02 | 20260508 03:56:19.712 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:56:19.712 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:56:19.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:56:19.713 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 24.59%, Upload -- 2.62e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 24.32%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 30.61%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 2.54%, LockUpload -- 2.71e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 2.47%, LockDownload -- 2.63e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 15.46%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 03:56:19.713 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260508 03:56:19.808 | INFO | Starting a round of Baz IO | 20260508 03:56:23.242 | INFO | Populating RTMetrics report | 20260508 03:56:23.242 | INFO | RTMetrics report populated | 20260508 03:56:23.242 | INFO | Opening tmp RTMetrics json file | 20260508 03:56:23.259 | INFO | Serializing RTMetrics to json | 20260508 03:56:23.260 | INFO | Serialization done, swapping tmp file now... | 20260508 03:56:23.261 | INFO | RTMetrics done | 20260508 03:56:23.315 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.507 seconds | 20260508 03:56:23.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.17% / 0.00% / 58.83% | 20260508 03:56:23.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.578058% / 0% / 6.75519e-05% | 20260508 03:56:23.315 | INFO | Baz IO overall: | 20260508 03:56:23.315 | INFO | BytesWritten : 2264367490634 | 20260508 03:56:23.315 | INFO | NumEvents : 1093617938451 | 20260508 03:56:24.853 | INFO | Analyzing chunk frames = [5046272,5046784) | 20260508 03:56:24.853 | INFO | MemInfo: { | 20260508 03:56:24.853 | INFO | "gpu" : " 17 GB", | 20260508 03:56:24.853 | INFO | "huge" : "", | 20260508 03:56:24.853 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:24.853 | INFO | "malloc" : " 14 GB", | 20260508 03:56:24.853 | INFO | "pinned" : " 29 GB" | 20260508 03:56:24.853 | INFO | } | 20260508 03:56:24.853 | INFO | Perf report for the last 5.14219 seconds: | 20260508 03:56:24.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:24.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:24.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:24.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:24.853 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 03:56:24.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:24.853 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 03:56:24.853 | INFO | PRE_HQ : 65.6%, 1.6, 55.9ms, 0 | 20260508 03:56:24.853 | INFO | BAZWRITER : 67.7%, 2.7, 94.8ms, 0 | 20260508 03:56:24.853 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:56:24.854 | INFO | Uploaded 36GiB to GPU in 2.58095s (13.9484GiB/s) | 20260508 03:56:24.854 | INFO | Downloaded 38GiB to GPU in 2.70339s (14.0564GiB/s) | 20260508 03:56:29.986 | INFO | Analyzing chunk frames = [5046784,5047296) | 20260508 03:56:29.986 | INFO | MemInfo: { | 20260508 03:56:29.986 | INFO | "gpu" : " 17 GB", | 20260508 03:56:29.986 | INFO | "huge" : "", | 20260508 03:56:29.986 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:29.986 | INFO | "malloc" : " 14 GB", | 20260508 03:56:29.986 | INFO | "pinned" : " 29 GB" | 20260508 03:56:29.986 | INFO | } | 20260508 03:56:29.986 | INFO | Perf report for the last 5.13319 seconds: | 20260508 03:56:29.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:29.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 03:56:29.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:56:29.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:29.986 | INFO | Uploaded 36GiB to GPU in 2.56091s (14.0575GiB/s) | 20260508 03:56:29.986 | INFO | Downloaded 34GiB to GPU in 2.42979s (13.993GiB/s) | 20260508 03:56:35.124 | INFO | Analyzing chunk frames = [5047296,5047808) | 20260508 03:56:35.124 | INFO | MemInfo: { | 20260508 03:56:35.124 | INFO | "gpu" : " 17 GB", | 20260508 03:56:35.124 | INFO | "huge" : "", | 20260508 03:56:35.124 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:35.124 | INFO | "malloc" : " 14 GB", | 20260508 03:56:35.124 | INFO | "pinned" : " 29 GB" | 20260508 03:56:35.125 | INFO | } | 20260508 03:56:35.125 | INFO | Perf report for the last 5.13881 seconds: | 20260508 03:56:35.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:35.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 03:56:35.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 03:56:35.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:35.125 | INFO | Uploaded 36GiB to GPU in 2.57232s (13.9951GiB/s) | 20260508 03:56:35.125 | INFO | Downloaded 34GiB to GPU in 2.50627s (13.566GiB/s) | 20260508 03:56:40.265 | INFO | Analyzing chunk frames = [5047808,5048320) | 20260508 03:56:40.265 | INFO | MemInfo: { | 20260508 03:56:40.265 | INFO | "gpu" : " 17 GB", | 20260508 03:56:40.265 | INFO | "huge" : "", | 20260508 03:56:40.265 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:40.265 | INFO | "malloc" : " 14 GB", | 20260508 03:56:40.265 | INFO | "pinned" : " 29 GB" | 20260508 03:56:40.265 | INFO | } | 20260508 03:56:40.265 | INFO | Perf report for the last 5.14004 seconds: | 20260508 03:56:40.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:40.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 03:56:40.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 03:56:40.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:40.265 | INFO | Uploaded 36GiB to GPU in 2.5595s (14.0652GiB/s) | 20260508 03:56:40.265 | INFO | Downloaded 34GiB to GPU in 2.50767s (13.5584GiB/s) | 20260508 03:56:45.406 | INFO | Analyzing chunk frames = [5048320,5048832) | 20260508 03:56:45.406 | INFO | MemInfo: { | 20260508 03:56:45.406 | INFO | "gpu" : " 17 GB", | 20260508 03:56:45.406 | INFO | "huge" : "", | 20260508 03:56:45.406 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:45.406 | INFO | "malloc" : " 14 GB", | 20260508 03:56:45.406 | INFO | "pinned" : " 29 GB" | 20260508 03:56:45.406 | INFO | } | 20260508 03:56:45.406 | INFO | Perf report for the last 5.14138 seconds: | 20260508 03:56:45.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:45.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.406 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 03:56:45.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.406 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 03:56:45.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:45.407 | INFO | Uploaded 36GiB to GPU in 2.34891s (15.3263GiB/s) | 20260508 03:56:45.407 | INFO | Downloaded 34GiB to GPU in 2.58226s (13.1667GiB/s) | 20260508 03:56:50.543 | INFO | Analyzing chunk frames = [5048832,5049344) | 20260508 03:56:50.543 | INFO | MemInfo: { | 20260508 03:56:50.543 | INFO | "gpu" : " 17 GB", | 20260508 03:56:50.543 | INFO | "huge" : "", | 20260508 03:56:50.543 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:50.543 | INFO | "malloc" : " 14 GB", | 20260508 03:56:50.543 | INFO | "pinned" : " 29 GB" | 20260508 03:56:50.543 | INFO | } | 20260508 03:56:50.543 | INFO | Perf report for the last 5.13672 seconds: | 20260508 03:56:50.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:50.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:56:50.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 03:56:50.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:50.543 | INFO | Uploaded 36GiB to GPU in 2.39382s (15.0387GiB/s) | 20260508 03:56:50.544 | INFO | Downloaded 34GiB to GPU in 2.44177s (13.9243GiB/s) | 20260508 03:56:55.680 | INFO | Analyzing chunk frames = [5049344,5049856) | 20260508 03:56:55.680 | INFO | MemInfo: { | 20260508 03:56:55.680 | INFO | "gpu" : " 17 GB", | 20260508 03:56:55.680 | INFO | "huge" : "", | 20260508 03:56:55.680 | INFO | "huge_shared" : " 29 GB", | 20260508 03:56:55.680 | INFO | "malloc" : " 14 GB", | 20260508 03:56:55.680 | INFO | "pinned" : " 29 GB" | 20260508 03:56:55.680 | INFO | } | 20260508 03:56:55.680 | INFO | Perf report for the last 5.137 seconds: | 20260508 03:56:55.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:56:55.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 03:56:55.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260508 03:56:55.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:56:55.680 | INFO | Uploaded 36GiB to GPU in 2.39513s (15.0305GiB/s) | 20260508 03:56:55.681 | INFO | Downloaded 34GiB to GPU in 2.43824s (13.9445GiB/s) | 20260508 03:57:00.822 | INFO | Analyzing chunk frames = [5049856,5050368) | 20260508 03:57:00.822 | INFO | MemInfo: { | 20260508 03:57:00.822 | INFO | "gpu" : " 17 GB", | 20260508 03:57:00.822 | INFO | "huge" : "", | 20260508 03:57:00.822 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:00.822 | INFO | "malloc" : " 14 GB", | 20260508 03:57:00.822 | INFO | "pinned" : " 29 GB" | 20260508 03:57:00.823 | INFO | } | 20260508 03:57:00.823 | INFO | Perf report for the last 5.14253 seconds: | 20260508 03:57:00.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:00.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 03:57:00.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 03:57:00.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:00.823 | INFO | Uploaded 36GiB to GPU in 2.43843s (14.7636GiB/s) | 20260508 03:57:00.823 | INFO | Downloaded 34GiB to GPU in 2.43812s (13.9452GiB/s) | 20260508 03:57:00.938 | INFO | Starting a round of Baz IO | 20260508 03:57:04.360 | INFO | Populating RTMetrics report | 20260508 03:57:04.360 | INFO | RTMetrics report populated | 20260508 03:57:04.361 | INFO | Opening tmp RTMetrics json file | 20260508 03:57:04.374 | INFO | Serializing RTMetrics to json | 20260508 03:57:04.375 | INFO | Serialization done, swapping tmp file now... | 20260508 03:57:04.375 | INFO | RTMetrics done | 20260508 03:57:04.433 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.494 seconds | 20260508 03:57:04.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.17% / 0.00% / 58.83% | 20260508 03:57:04.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.577088% / 0% / 5.5631e-05% | 20260508 03:57:04.433 | INFO | Baz IO overall: | 20260508 03:57:04.433 | INFO | BytesWritten : 2266092324118 | 20260508 03:57:04.433 | INFO | NumEvents : 1094442989004 | 20260508 03:57:05.968 | INFO | Analyzing chunk frames = [5050368,5050880) | 20260508 03:57:05.968 | INFO | MemInfo: { | 20260508 03:57:05.968 | INFO | "gpu" : " 17 GB", | 20260508 03:57:05.968 | INFO | "huge" : "", | 20260508 03:57:05.968 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:05.968 | INFO | "malloc" : " 14 GB", | 20260508 03:57:05.968 | INFO | "pinned" : " 29 GB" | 20260508 03:57:05.968 | INFO | } | 20260508 03:57:05.968 | INFO | Perf report for the last 5.14576 seconds: | 20260508 03:57:05.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:05.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:05.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:05.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:05.968 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260508 03:57:05.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:05.968 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:57:05.968 | INFO | PRE_HQ : 66.7%, 1.8, 63.1ms, 0 | 20260508 03:57:05.968 | INFO | BAZWRITER : 67.3%, 2.9, 101.0ms, 0 | 20260508 03:57:05.969 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:57:05.969 | INFO | Uploaded 36GiB to GPU in 2.5494s (14.121GiB/s) | 20260508 03:57:05.969 | INFO | Downloaded 38GiB to GPU in 2.70036s (14.0722GiB/s) | 20260508 03:57:11.105 | INFO | Analyzing chunk frames = [5050880,5051392) | 20260508 03:57:11.105 | INFO | MemInfo: { | 20260508 03:57:11.105 | INFO | "gpu" : " 17 GB", | 20260508 03:57:11.105 | INFO | "huge" : "", | 20260508 03:57:11.105 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:11.105 | INFO | "malloc" : " 14 GB", | 20260508 03:57:11.105 | INFO | "pinned" : " 29 GB" | 20260508 03:57:11.106 | INFO | } | 20260508 03:57:11.106 | INFO | Perf report for the last 5.13726 seconds: | 20260508 03:57:11.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:11.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 03:57:11.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:57:11.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:11.106 | INFO | Uploaded 36GiB to GPU in 2.48259s (14.501GiB/s) | 20260508 03:57:11.106 | INFO | Downloaded 34GiB to GPU in 2.43089s (13.9867GiB/s) | 20260508 03:57:16.240 | INFO | Analyzing chunk frames = [5051392,5051904) | 20260508 03:57:16.240 | INFO | MemInfo: { | 20260508 03:57:16.240 | INFO | "gpu" : " 17 GB", | 20260508 03:57:16.240 | INFO | "huge" : "", | 20260508 03:57:16.240 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:16.240 | INFO | "malloc" : " 14 GB", | 20260508 03:57:16.240 | INFO | "pinned" : " 29 GB" | 20260508 03:57:16.240 | INFO | } | 20260508 03:57:16.240 | INFO | Perf report for the last 5.13451 seconds: | 20260508 03:57:16.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:16.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 03:57:16.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 03:57:16.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:16.240 | INFO | Uploaded 36GiB to GPU in 2.54021s (14.172GiB/s) | 20260508 03:57:16.241 | INFO | Downloaded 34GiB to GPU in 2.49912s (13.6048GiB/s) | 20260508 03:57:21.382 | INFO | Analyzing chunk frames = [5051904,5052416) | 20260508 03:57:21.382 | INFO | MemInfo: { | 20260508 03:57:21.382 | INFO | "gpu" : " 17 GB", | 20260508 03:57:21.382 | INFO | "huge" : "", | 20260508 03:57:21.382 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:21.382 | INFO | "malloc" : " 14 GB", | 20260508 03:57:21.382 | INFO | "pinned" : " 29 GB" | 20260508 03:57:21.382 | INFO | } | 20260508 03:57:21.382 | INFO | Perf report for the last 5.14209 seconds: | 20260508 03:57:21.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:21.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.382 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 03:57:21.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.382 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 03:57:21.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:21.383 | INFO | Uploaded 36GiB to GPU in 2.55828s (14.0719GiB/s) | 20260508 03:57:21.383 | INFO | Downloaded 34GiB to GPU in 2.43252s (13.9773GiB/s) | 20260508 03:57:26.517 | INFO | Analyzing chunk frames = [5052416,5052928) | 20260508 03:57:26.517 | INFO | MemInfo: { | 20260508 03:57:26.517 | INFO | "gpu" : " 17 GB", | 20260508 03:57:26.517 | INFO | "huge" : "", | 20260508 03:57:26.517 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:26.517 | INFO | "malloc" : " 14 GB", | 20260508 03:57:26.517 | INFO | "pinned" : " 29 GB" | 20260508 03:57:26.517 | INFO | } | 20260508 03:57:26.517 | INFO | Perf report for the last 5.13484 seconds: | 20260508 03:57:26.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:26.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.517 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 03:57:26.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.517 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 03:57:26.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:26.518 | INFO | Uploaded 36GiB to GPU in 2.35102s (15.3125GiB/s) | 20260508 03:57:26.518 | INFO | Downloaded 34GiB to GPU in 2.57s (13.2296GiB/s) | 20260508 03:57:31.663 | INFO | Analyzing chunk frames = [5052928,5053440) | 20260508 03:57:31.663 | INFO | MemInfo: { | 20260508 03:57:31.663 | INFO | "gpu" : " 17 GB", | 20260508 03:57:31.663 | INFO | "huge" : "", | 20260508 03:57:31.663 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:31.663 | INFO | "malloc" : " 14 GB", | 20260508 03:57:31.663 | INFO | "pinned" : " 29 GB" | 20260508 03:57:31.663 | INFO | } | 20260508 03:57:31.663 | INFO | Perf report for the last 5.14591 seconds: | 20260508 03:57:31.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:31.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 03:57:31.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 03:57:31.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:31.663 | INFO | Uploaded 36GiB to GPU in 2.47509s (14.5449GiB/s) | 20260508 03:57:31.663 | INFO | Downloaded 34GiB to GPU in 2.42496s (14.0208GiB/s) | 20260508 03:57:36.798 | INFO | Analyzing chunk frames = [5053440,5053952) | 20260508 03:57:36.798 | INFO | MemInfo: { | 20260508 03:57:36.798 | INFO | "gpu" : " 17 GB", | 20260508 03:57:36.798 | INFO | "huge" : "", | 20260508 03:57:36.798 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:36.798 | INFO | "malloc" : " 14 GB", | 20260508 03:57:36.798 | INFO | "pinned" : " 29 GB" | 20260508 03:57:36.798 | INFO | } | 20260508 03:57:36.798 | INFO | Perf report for the last 5.13508 seconds: | 20260508 03:57:36.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:36.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.798 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 03:57:36.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.798 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 03:57:36.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:36.799 | INFO | Uploaded 36GiB to GPU in 2.44209s (14.7415GiB/s) | 20260508 03:57:36.799 | INFO | Downloaded 34GiB to GPU in 2.41712s (14.0663GiB/s) | 20260508 03:57:41.933 | INFO | Analyzing chunk frames = [5053952,5054464) | 20260508 03:57:41.933 | INFO | MemInfo: { | 20260508 03:57:41.933 | INFO | "gpu" : " 17 GB", | 20260508 03:57:41.933 | INFO | "huge" : "", | 20260508 03:57:41.933 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:41.933 | INFO | "malloc" : " 14 GB", | 20260508 03:57:41.934 | INFO | "pinned" : " 29 GB" | 20260508 03:57:41.934 | INFO | } | 20260508 03:57:41.934 | INFO | Perf report for the last 5.13547 seconds: | 20260508 03:57:41.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:41.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 03:57:41.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 03:57:41.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:41.934 | INFO | Uploaded 36GiB to GPU in 2.47711s (14.5331GiB/s) | 20260508 03:57:41.934 | INFO | Downloaded 34GiB to GPU in 2.42892s (13.998GiB/s) | 20260508 03:57:41.934 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:57:41.934 | INFO | Total mem usage: 16.7 GiB | 20260508 03:57:41.934 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:57:41.934 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:57:41.934 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:57:41.934 | INFO | High Water Breakdown: | 20260508 03:57:41.934 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:57:41.934 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:57:41.934 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:57:41.934 | INFO | ------------------------------------------ | 20260508 03:57:41.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:57:41.934 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:57:41.934 | INFO | Total mem usage: 26.5 GiB | 20260508 03:57:41.935 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:57:41.935 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:57:41.935 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:57:41.935 | INFO | High Water Breakdown: | 20260508 03:57:41.935 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:57:41.935 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:57:41.935 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:57:41.935 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:57:41.935 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:57:41.935 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:57:41.935 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:57:41.935 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:57:41.935 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:57:41.935 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:57:41.935 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:57:41.935 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:57:41.935 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:57:41.935 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:57:41.935 | INFO | ------------------------------------------ | 20260508 03:57:41.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:57:41.935 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:57:41.935 | INFO | Total mem usage: 27.8 GiB | 20260508 03:57:41.935 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:57:41.935 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:57:41.935 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:57:41.935 | INFO | High Water Breakdown: | 20260508 03:57:41.935 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:57:41.935 | INFO | ------------------------------------------ | 20260508 03:57:41.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:57:41.935 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:57:41.935 | INFO | Total mem usage: 15.7 GiB | 20260508 03:57:41.935 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:57:41.935 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:57:41.935 | INFO | Unused Cache: 862 MiB (Waste: 5.37%) | 20260508 03:57:41.935 | INFO | High Water Breakdown: | 20260508 03:57:41.935 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:57:41.935 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:57:41.935 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:57:41.935 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:57:41.935 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:57:41.935 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:57:41.935 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:57:41.935 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:57:41.935 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:57:41.935 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:57:41.935 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:57:41.935 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:57:41.935 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:57:41.935 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:57:41.935 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:57:41.935 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:57:41.935 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:57:41.935 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:57:41.935 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:57:41.935 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:57:41.935 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:57:41.935 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:57:41.935 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:57:41.935 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:57:41.935 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:57:41.935 | INFO | ------------------------------------------ | 20260508 03:57:41.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:57:41.935 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:57:41.935 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:57:41.935 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:57:41.935 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.83e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.03e-01, 1.19e-02 / 5.82e-03 | 20260508 03:57:41.935 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.49e+00, 5.94e-01 / 1.83e-01 | 20260508 03:57:41.935 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.23e-01 | 20260508 03:57:41.935 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 4.15e+00 / 2.09e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.68e+00 / 1.42e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.23e-01 / 2.29e-01 | 20260508 03:57:41.935 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.24e+00, 2.66e-01 / 1.48e-01 | 20260508 03:57:41.935 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.62e-01, 5.08e-02 / 2.90e-02 | 20260508 03:57:41.935 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:57:41.935 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:57:41.935 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:57:41.935 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 24.32%, Upload -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 24.46%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 2.14%, LockUpload -- 2.27e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 2.78%, LockDownload -- 2.96e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 03:57:41.935 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260508 03:57:42.046 | INFO | Starting a round of Baz IO | 20260508 03:57:45.474 | INFO | Populating RTMetrics report | 20260508 03:57:45.474 | INFO | RTMetrics report populated | 20260508 03:57:45.474 | INFO | Opening tmp RTMetrics json file | 20260508 03:57:45.486 | INFO | Serializing RTMetrics to json | 20260508 03:57:45.488 | INFO | Serialization done, swapping tmp file now... | 20260508 03:57:45.488 | INFO | RTMetrics done | 20260508 03:57:45.547 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.5 seconds | 20260508 03:57:45.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.17% / 0.00% / 58.83% | 20260508 03:57:45.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.575189% / 0% / 5.16574e-05% | 20260508 03:57:45.547 | INFO | Baz IO overall: | 20260508 03:57:45.547 | INFO | BytesWritten : 2267816047198 | 20260508 03:57:45.547 | INFO | NumEvents : 1095267330101 | 20260508 03:57:47.075 | INFO | Analyzing chunk frames = [5054464,5054976) | 20260508 03:57:47.075 | INFO | MemInfo: { | 20260508 03:57:47.075 | INFO | "gpu" : " 17 GB", | 20260508 03:57:47.075 | INFO | "huge" : "", | 20260508 03:57:47.075 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:47.075 | INFO | "malloc" : " 14 GB", | 20260508 03:57:47.075 | INFO | "pinned" : " 29 GB" | 20260508 03:57:47.075 | INFO | } | 20260508 03:57:47.075 | INFO | Perf report for the last 5.14165 seconds: | 20260508 03:57:47.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:47.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:47.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:47.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:47.075 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 03:57:47.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:47.076 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 03:57:47.076 | INFO | PRE_HQ : 66.2%, 1.8, 62.2ms, 0 | 20260508 03:57:47.076 | INFO | BAZWRITER : 67.6%, 2.9, 99.3ms, 0 | 20260508 03:57:47.076 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 03:57:47.076 | INFO | Uploaded 36GiB to GPU in 2.57339s (13.9893GiB/s) | 20260508 03:57:47.076 | INFO | Downloaded 38GiB to GPU in 2.69357s (14.1077GiB/s) | 20260508 03:57:52.217 | INFO | Analyzing chunk frames = [5054976,5055488) | 20260508 03:57:52.217 | INFO | MemInfo: { | 20260508 03:57:52.217 | INFO | "gpu" : " 17 GB", | 20260508 03:57:52.217 | INFO | "huge" : "", | 20260508 03:57:52.217 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:52.217 | INFO | "malloc" : " 14 GB", | 20260508 03:57:52.217 | INFO | "pinned" : " 29 GB" | 20260508 03:57:52.217 | INFO | } | 20260508 03:57:52.217 | INFO | Perf report for the last 5.1422 seconds: | 20260508 03:57:52.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:52.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 03:57:52.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:57:52.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:52.218 | INFO | Uploaded 36GiB to GPU in 2.5496s (14.1199GiB/s) | 20260508 03:57:52.218 | INFO | Downloaded 34GiB to GPU in 2.43266s (13.9765GiB/s) | 20260508 03:57:57.357 | INFO | Analyzing chunk frames = [5055488,5056000) | 20260508 03:57:57.357 | INFO | MemInfo: { | 20260508 03:57:57.357 | INFO | "gpu" : " 17 GB", | 20260508 03:57:57.357 | INFO | "huge" : "", | 20260508 03:57:57.357 | INFO | "huge_shared" : " 29 GB", | 20260508 03:57:57.357 | INFO | "malloc" : " 14 GB", | 20260508 03:57:57.357 | INFO | "pinned" : " 29 GB" | 20260508 03:57:57.357 | INFO | } | 20260508 03:57:57.357 | INFO | Perf report for the last 5.13989 seconds: | 20260508 03:57:57.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:57:57.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.357 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:57:57.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.357 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:57:57.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:57:57.358 | INFO | Uploaded 36GiB to GPU in 2.53008s (14.2288GiB/s) | 20260508 03:57:57.358 | INFO | Downloaded 34GiB to GPU in 2.50539s (13.5707GiB/s) | 20260508 03:58:02.489 | INFO | Analyzing chunk frames = [5056000,5056512) | 20260508 03:58:02.489 | INFO | MemInfo: { | 20260508 03:58:02.489 | INFO | "gpu" : " 17 GB", | 20260508 03:58:02.489 | INFO | "huge" : "", | 20260508 03:58:02.489 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:02.490 | INFO | "malloc" : " 14 GB", | 20260508 03:58:02.490 | INFO | "pinned" : " 29 GB" | 20260508 03:58:02.490 | INFO | } | 20260508 03:58:02.490 | INFO | Perf report for the last 5.13231 seconds: | 20260508 03:58:02.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:02.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.490 | INFO | ANALYSIS : 67.1%, 2.9, 103.6ms, 0 | 20260508 03:58:02.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.490 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 03:58:02.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:02.491 | INFO | Uploaded 36GiB to GPU in 2.54613s (14.1391GiB/s) | 20260508 03:58:02.491 | INFO | Downloaded 34GiB to GPU in 2.41554s (14.0755GiB/s) | 20260508 03:58:07.633 | INFO | Analyzing chunk frames = [5056512,5057024) | 20260508 03:58:07.633 | INFO | MemInfo: { | 20260508 03:58:07.633 | INFO | "gpu" : " 17 GB", | 20260508 03:58:07.633 | INFO | "huge" : "", | 20260508 03:58:07.633 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:07.634 | INFO | "malloc" : " 14 GB", | 20260508 03:58:07.634 | INFO | "pinned" : " 29 GB" | 20260508 03:58:07.634 | INFO | } | 20260508 03:58:07.634 | INFO | Perf report for the last 5.14387 seconds: | 20260508 03:58:07.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:07.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 03:58:07.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 03:58:07.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:07.634 | INFO | Uploaded 36GiB to GPU in 2.40068s (14.9958GiB/s) | 20260508 03:58:07.634 | INFO | Downloaded 34GiB to GPU in 2.57541s (13.2018GiB/s) | 20260508 03:58:12.779 | INFO | Analyzing chunk frames = [5057024,5057536) | 20260508 03:58:12.779 | INFO | MemInfo: { | 20260508 03:58:12.779 | INFO | "gpu" : " 17 GB", | 20260508 03:58:12.779 | INFO | "huge" : "", | 20260508 03:58:12.779 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:12.779 | INFO | "malloc" : " 14 GB", | 20260508 03:58:12.779 | INFO | "pinned" : " 29 GB" | 20260508 03:58:12.779 | INFO | } | 20260508 03:58:12.779 | INFO | Perf report for the last 5.14517 seconds: | 20260508 03:58:12.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:12.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 03:58:12.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:58:12.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:12.779 | INFO | Uploaded 36GiB to GPU in 2.38601s (15.0879GiB/s) | 20260508 03:58:12.779 | INFO | Downloaded 34GiB to GPU in 2.44203s (13.9228GiB/s) | 20260508 03:58:17.919 | INFO | Analyzing chunk frames = [5057536,5058048) | 20260508 03:58:17.919 | INFO | MemInfo: { | 20260508 03:58:17.919 | INFO | "gpu" : " 17 GB", | 20260508 03:58:17.919 | INFO | "huge" : "", | 20260508 03:58:17.919 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:17.919 | INFO | "malloc" : " 14 GB", | 20260508 03:58:17.919 | INFO | "pinned" : " 29 GB" | 20260508 03:58:17.919 | INFO | } | 20260508 03:58:17.919 | INFO | Perf report for the last 5.14037 seconds: | 20260508 03:58:17.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:17.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.919 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 03:58:17.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.919 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 03:58:17.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:17.920 | INFO | Uploaded 36GiB to GPU in 2.3936s (15.0401GiB/s) | 20260508 03:58:17.920 | INFO | Downloaded 34GiB to GPU in 2.43104s (13.9858GiB/s) | 20260508 03:58:23.055 | INFO | Analyzing chunk frames = [5058048,5058560) | 20260508 03:58:23.055 | INFO | MemInfo: { | 20260508 03:58:23.055 | INFO | "gpu" : " 17 GB", | 20260508 03:58:23.056 | INFO | "huge" : "", | 20260508 03:58:23.056 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:23.056 | INFO | "malloc" : " 14 GB", | 20260508 03:58:23.056 | INFO | "pinned" : " 29 GB" | 20260508 03:58:23.056 | INFO | } | 20260508 03:58:23.056 | INFO | Perf report for the last 5.13653 seconds: | 20260508 03:58:23.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:23.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:58:23.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 03:58:23.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:23.056 | INFO | Uploaded 36GiB to GPU in 2.43844s (14.7636GiB/s) | 20260508 03:58:23.056 | INFO | Downloaded 34GiB to GPU in 2.44986s (13.8784GiB/s) | 20260508 03:58:23.168 | INFO | Starting a round of Baz IO | 20260508 03:58:26.592 | INFO | Populating RTMetrics report | 20260508 03:58:26.592 | INFO | RTMetrics report populated | 20260508 03:58:26.592 | INFO | Opening tmp RTMetrics json file | 20260508 03:58:26.603 | INFO | Serializing RTMetrics to json | 20260508 03:58:26.605 | INFO | Serialization done, swapping tmp file now... | 20260508 03:58:26.605 | INFO | RTMetrics done | 20260508 03:58:26.661 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.492 seconds | 20260508 03:58:26.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.16% / 0.00% / 58.83% | 20260508 03:58:26.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.573309% / 0% / 3.97364e-05% | 20260508 03:58:26.661 | INFO | Baz IO overall: | 20260508 03:58:26.661 | INFO | BytesWritten : 2269539348270 | 20260508 03:58:26.661 | INFO | NumEvents : 1096091200139 | 20260508 03:58:28.190 | INFO | Analyzing chunk frames = [5058560,5059072) | 20260508 03:58:28.190 | INFO | MemInfo: { | 20260508 03:58:28.190 | INFO | "gpu" : " 17 GB", | 20260508 03:58:28.190 | INFO | "huge" : "", | 20260508 03:58:28.190 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:28.190 | INFO | "malloc" : " 14 GB", | 20260508 03:58:28.191 | INFO | "pinned" : " 29 GB" | 20260508 03:58:28.191 | INFO | } | 20260508 03:58:28.191 | INFO | Perf report for the last 5.13481 seconds: | 20260508 03:58:28.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:28.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:28.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:28.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:28.191 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 03:58:28.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:28.191 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 03:58:28.191 | INFO | PRE_HQ : 66.6%, 1.8, 62.1ms, 0 | 20260508 03:58:28.191 | INFO | BAZWRITER : 67.5%, 2.9, 99.5ms, 0 | 20260508 03:58:28.191 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:58:28.191 | INFO | Uploaded 36GiB to GPU in 2.54366s (14.1529GiB/s) | 20260508 03:58:28.191 | INFO | Downloaded 38GiB to GPU in 2.68521s (14.1516GiB/s) | 20260508 03:58:33.332 | INFO | Analyzing chunk frames = [5059072,5059584) | 20260508 03:58:33.332 | INFO | MemInfo: { | 20260508 03:58:33.332 | INFO | "gpu" : " 17 GB", | 20260508 03:58:33.332 | INFO | "huge" : "", | 20260508 03:58:33.332 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:33.332 | INFO | "malloc" : " 14 GB", | 20260508 03:58:33.332 | INFO | "pinned" : " 29 GB" | 20260508 03:58:33.332 | INFO | } | 20260508 03:58:33.332 | INFO | Perf report for the last 5.1414 seconds: | 20260508 03:58:33.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:33.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.332 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 03:58:33.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.332 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 03:58:33.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:33.333 | INFO | Uploaded 36GiB to GPU in 2.57211s (13.9963GiB/s) | 20260508 03:58:33.333 | INFO | Downloaded 34GiB to GPU in 2.42642s (14.0124GiB/s) | 20260508 03:58:38.472 | INFO | Analyzing chunk frames = [5059584,5060096) | 20260508 03:58:38.472 | INFO | MemInfo: { | 20260508 03:58:38.472 | INFO | "gpu" : " 17 GB", | 20260508 03:58:38.472 | INFO | "huge" : "", | 20260508 03:58:38.472 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:38.472 | INFO | "malloc" : " 14 GB", | 20260508 03:58:38.472 | INFO | "pinned" : " 29 GB" | 20260508 03:58:38.472 | INFO | } | 20260508 03:58:38.472 | INFO | Perf report for the last 5.13975 seconds: | 20260508 03:58:38.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:38.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 03:58:38.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:58:38.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:38.472 | INFO | Uploaded 36GiB to GPU in 2.54569s (14.1416GiB/s) | 20260508 03:58:38.472 | INFO | Downloaded 34GiB to GPU in 2.42654s (14.0117GiB/s) | 20260508 03:58:43.613 | INFO | Analyzing chunk frames = [5060096,5060608) | 20260508 03:58:43.613 | INFO | MemInfo: { | 20260508 03:58:43.613 | INFO | "gpu" : " 17 GB", | 20260508 03:58:43.613 | INFO | "huge" : "", | 20260508 03:58:43.613 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:43.614 | INFO | "malloc" : " 14 GB", | 20260508 03:58:43.614 | INFO | "pinned" : " 29 GB" | 20260508 03:58:43.614 | INFO | } | 20260508 03:58:43.614 | INFO | Perf report for the last 5.1419 seconds: | 20260508 03:58:43.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:43.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 03:58:43.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:58:43.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:43.614 | INFO | Uploaded 36GiB to GPU in 2.53s (14.2292GiB/s) | 20260508 03:58:43.614 | INFO | Downloaded 34GiB to GPU in 2.50548s (13.5703GiB/s) | 20260508 03:58:48.746 | INFO | Analyzing chunk frames = [5060608,5061120) | 20260508 03:58:48.746 | INFO | MemInfo: { | 20260508 03:58:48.746 | INFO | "gpu" : " 17 GB", | 20260508 03:58:48.746 | INFO | "huge" : "", | 20260508 03:58:48.746 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:48.746 | INFO | "malloc" : " 14 GB", | 20260508 03:58:48.746 | INFO | "pinned" : " 29 GB" | 20260508 03:58:48.746 | INFO | } | 20260508 03:58:48.746 | INFO | Perf report for the last 5.13241 seconds: | 20260508 03:58:48.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:48.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 03:58:48.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:58:48.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:48.746 | INFO | Uploaded 36GiB to GPU in 2.38571s (15.0899GiB/s) | 20260508 03:58:48.747 | INFO | Downloaded 34GiB to GPU in 2.56211s (13.2703GiB/s) | 20260508 03:58:53.886 | INFO | Analyzing chunk frames = [5061120,5061632) | 20260508 03:58:53.886 | INFO | MemInfo: { | 20260508 03:58:53.886 | INFO | "gpu" : " 17 GB", | 20260508 03:58:53.886 | INFO | "huge" : "", | 20260508 03:58:53.886 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:53.886 | INFO | "malloc" : " 14 GB", | 20260508 03:58:53.886 | INFO | "pinned" : " 29 GB" | 20260508 03:58:53.886 | INFO | } | 20260508 03:58:53.887 | INFO | Perf report for the last 5.14044 seconds: | 20260508 03:58:53.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:53.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:58:53.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 03:58:53.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:53.887 | INFO | Uploaded 36GiB to GPU in 2.4135s (14.9161GiB/s) | 20260508 03:58:53.887 | INFO | Downloaded 34GiB to GPU in 2.42393s (14.0268GiB/s) | 20260508 03:58:59.026 | INFO | Analyzing chunk frames = [5061632,5062144) | 20260508 03:58:59.026 | INFO | MemInfo: { | 20260508 03:58:59.026 | INFO | "gpu" : " 17 GB", | 20260508 03:58:59.026 | INFO | "huge" : "", | 20260508 03:58:59.026 | INFO | "huge_shared" : " 29 GB", | 20260508 03:58:59.026 | INFO | "malloc" : " 14 GB", | 20260508 03:58:59.027 | INFO | "pinned" : " 29 GB" | 20260508 03:58:59.027 | INFO | } | 20260508 03:58:59.027 | INFO | Perf report for the last 5.14016 seconds: | 20260508 03:58:59.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:58:59.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 03:58:59.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 03:58:59.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:58:59.027 | INFO | Uploaded 36GiB to GPU in 2.42691s (14.8337GiB/s) | 20260508 03:58:59.027 | INFO | Downloaded 34GiB to GPU in 2.41856s (14.0579GiB/s) | 20260508 03:59:04.167 | INFO | Analyzing chunk frames = [5062144,5062656) | 20260508 03:59:04.167 | INFO | MemInfo: { | 20260508 03:59:04.167 | INFO | "gpu" : " 17 GB", | 20260508 03:59:04.167 | INFO | "huge" : "", | 20260508 03:59:04.167 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:04.167 | INFO | "malloc" : " 14 GB", | 20260508 03:59:04.167 | INFO | "pinned" : " 29 GB" | 20260508 03:59:04.167 | INFO | } | 20260508 03:59:04.167 | INFO | Perf report for the last 5.1403 seconds: | 20260508 03:59:04.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:04.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.167 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 03:59:04.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.167 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 03:59:04.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:04.168 | INFO | Uploaded 36GiB to GPU in 2.44799s (14.7059GiB/s) | 20260508 03:59:04.168 | INFO | Downloaded 34GiB to GPU in 2.42636s (14.0127GiB/s) | 20260508 03:59:04.168 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 03:59:04.168 | INFO | Total mem usage: 16.7 GiB | 20260508 03:59:04.168 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 03:59:04.168 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 03:59:04.168 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 03:59:04.168 | INFO | High Water Breakdown: | 20260508 03:59:04.168 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 03:59:04.168 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 03:59:04.168 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 03:59:04.168 | INFO | ------------------------------------------ | 20260508 03:59:04.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:59:04.168 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 03:59:04.168 | INFO | Total mem usage: 26.5 GiB | 20260508 03:59:04.168 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 03:59:04.168 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 03:59:04.168 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 03:59:04.168 | INFO | High Water Breakdown: | 20260508 03:59:04.168 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:59:04.168 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:59:04.168 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:59:04.168 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:59:04.168 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:59:04.168 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:59:04.168 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 03:59:04.168 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 03:59:04.168 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:59:04.168 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 03:59:04.168 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 03:59:04.168 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 03:59:04.168 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 03:59:04.168 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 03:59:04.168 | INFO | ------------------------------------------ | 20260508 03:59:04.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:59:04.168 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 03:59:04.168 | INFO | Total mem usage: 27.8 GiB | 20260508 03:59:04.168 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 03:59:04.168 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 03:59:04.168 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 03:59:04.168 | INFO | High Water Breakdown: | 20260508 03:59:04.168 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 03:59:04.168 | INFO | ------------------------------------------ | 20260508 03:59:04.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:59:04.168 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 03:59:04.168 | INFO | Total mem usage: 15.7 GiB | 20260508 03:59:04.168 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 03:59:04.168 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 03:59:04.168 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 03:59:04.168 | INFO | High Water Breakdown: | 20260508 03:59:04.168 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 03:59:04.168 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 03:59:04.168 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 03:59:04.168 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 03:59:04.168 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 03:59:04.168 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 03:59:04.168 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 03:59:04.168 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 03:59:04.168 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 03:59:04.168 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 03:59:04.168 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 03:59:04.168 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 03:59:04.168 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 03:59:04.168 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 03:59:04.168 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 03:59:04.168 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 03:59:04.168 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 03:59:04.168 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 03:59:04.168 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 03:59:04.168 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 03:59:04.168 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 03:59:04.168 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 03:59:04.168 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 03:59:04.168 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 03:59:04.168 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 03:59:04.168 | INFO | ------------------------------------------ | 20260508 03:59:04.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 03:59:04.168 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 03:59:04.169 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:59:04.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:59:04.169 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.41e+01 / 1.79e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.03e-01, 1.22e-02 / 6.07e-03 | 20260508 03:59:04.169 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 6.00e-01 / 2.39e-01 | 20260508 03:59:04.169 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.21e-01 | 20260508 03:59:04.169 | INFO | PerfReport: 16.02%, DME -- 4.20e+01 / 4.59e+01, 4.16e+00 / 2.13e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.76e+00 / 1.51e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.23e-01 / 2.74e-01 | 20260508 03:59:04.169 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.23e+00, 2.62e-01 / 1.49e-01 | 20260508 03:59:04.169 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.54e-01, 5.10e-02 / 2.27e-02 | 20260508 03:59:04.169 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 03:59:04.169 | INFO | PerfReport: Gathered 1536 samples. | 20260508 03:59:04.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 03:59:04.169 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 24.29%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 24.38%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 30.80%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 2.09%, LockUpload -- 2.23e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 2.79%, LockDownload -- 2.97e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 03:59:04.169 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260508 03:59:04.281 | INFO | Starting a round of Baz IO | 20260508 03:59:07.707 | INFO | Populating RTMetrics report | 20260508 03:59:07.707 | INFO | RTMetrics report populated | 20260508 03:59:07.707 | INFO | Opening tmp RTMetrics json file | 20260508 03:59:07.719 | INFO | Serializing RTMetrics to json | 20260508 03:59:07.720 | INFO | Serialization done, swapping tmp file now... | 20260508 03:59:07.721 | INFO | RTMetrics done | 20260508 03:59:07.777 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.496 seconds | 20260508 03:59:07.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.16% / 0.00% / 58.84% | 20260508 03:59:07.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.574311% / 0% / 3.97364e-05% | 20260508 03:59:07.777 | INFO | Baz IO overall: | 20260508 03:59:07.777 | INFO | BytesWritten : 2271262905653 | 20260508 03:59:07.777 | INFO | NumEvents : 1096915438724 | 20260508 03:59:09.305 | INFO | Analyzing chunk frames = [5062656,5063168) | 20260508 03:59:09.305 | INFO | MemInfo: { | 20260508 03:59:09.305 | INFO | "gpu" : " 17 GB", | 20260508 03:59:09.305 | INFO | "huge" : "", | 20260508 03:59:09.305 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:09.305 | INFO | "malloc" : " 14 GB", | 20260508 03:59:09.305 | INFO | "pinned" : " 29 GB" | 20260508 03:59:09.305 | INFO | } | 20260508 03:59:09.305 | INFO | Perf report for the last 5.13813 seconds: | 20260508 03:59:09.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:09.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:09.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:09.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:09.305 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 03:59:09.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:09.305 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 03:59:09.305 | INFO | PRE_HQ : 66.1%, 1.8, 61.4ms, 0 | 20260508 03:59:09.305 | INFO | BAZWRITER : 67.5%, 2.8, 97.2ms, 0 | 20260508 03:59:09.305 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 03:59:09.306 | INFO | Uploaded 36GiB to GPU in 2.55239s (14.1044GiB/s) | 20260508 03:59:09.306 | INFO | Downloaded 38GiB to GPU in 2.67788s (14.1903GiB/s) | 20260508 03:59:14.444 | INFO | Analyzing chunk frames = [5063168,5063680) | 20260508 03:59:14.444 | INFO | MemInfo: { | 20260508 03:59:14.444 | INFO | "gpu" : " 17 GB", | 20260508 03:59:14.444 | INFO | "huge" : "", | 20260508 03:59:14.444 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:14.444 | INFO | "malloc" : " 14 GB", | 20260508 03:59:14.444 | INFO | "pinned" : " 29 GB" | 20260508 03:59:14.444 | INFO | } | 20260508 03:59:14.444 | INFO | Perf report for the last 5.13917 seconds: | 20260508 03:59:14.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:14.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.444 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 03:59:14.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.445 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 03:59:14.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:14.445 | INFO | Uploaded 36GiB to GPU in 2.53704s (14.1898GiB/s) | 20260508 03:59:14.445 | INFO | Downloaded 34GiB to GPU in 2.44316s (13.9164GiB/s) | 20260508 03:59:19.587 | INFO | Analyzing chunk frames = [5063680,5064192) | 20260508 03:59:19.588 | INFO | MemInfo: { | 20260508 03:59:19.588 | INFO | "gpu" : " 17 GB", | 20260508 03:59:19.588 | INFO | "huge" : "", | 20260508 03:59:19.588 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:19.588 | INFO | "malloc" : " 14 GB", | 20260508 03:59:19.588 | INFO | "pinned" : " 29 GB" | 20260508 03:59:19.588 | INFO | } | 20260508 03:59:19.588 | INFO | Perf report for the last 5.14339 seconds: | 20260508 03:59:19.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:19.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 03:59:19.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 03:59:19.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:19.588 | INFO | Uploaded 36GiB to GPU in 2.50229s (14.3868GiB/s) | 20260508 03:59:19.588 | INFO | Downloaded 34GiB to GPU in 2.43025s (13.9903GiB/s) | 20260508 03:59:24.730 | INFO | Analyzing chunk frames = [5064192,5064704) | 20260508 03:59:24.730 | INFO | MemInfo: { | 20260508 03:59:24.730 | INFO | "gpu" : " 17 GB", | 20260508 03:59:24.730 | INFO | "huge" : "", | 20260508 03:59:24.730 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:24.730 | INFO | "malloc" : " 14 GB", | 20260508 03:59:24.730 | INFO | "pinned" : " 29 GB" | 20260508 03:59:24.730 | INFO | } | 20260508 03:59:24.730 | INFO | Perf report for the last 5.14224 seconds: | 20260508 03:59:24.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:24.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 03:59:24.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:59:24.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:24.730 | INFO | Uploaded 36GiB to GPU in 2.53035s (14.2273GiB/s) | 20260508 03:59:24.730 | INFO | Downloaded 34GiB to GPU in 2.43585s (13.9581GiB/s) | 20260508 03:59:29.871 | INFO | Analyzing chunk frames = [5064704,5065216) | 20260508 03:59:29.871 | INFO | MemInfo: { | 20260508 03:59:29.871 | INFO | "gpu" : " 17 GB", | 20260508 03:59:29.871 | INFO | "huge" : "", | 20260508 03:59:29.871 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:29.871 | INFO | "malloc" : " 14 GB", | 20260508 03:59:29.871 | INFO | "pinned" : " 29 GB" | 20260508 03:59:29.871 | INFO | } | 20260508 03:59:29.872 | INFO | Perf report for the last 5.14145 seconds: | 20260508 03:59:29.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:29.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 03:59:29.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 03:59:29.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:29.872 | INFO | Uploaded 36GiB to GPU in 2.29396s (15.6934GiB/s) | 20260508 03:59:29.872 | INFO | Downloaded 34GiB to GPU in 2.56059s (13.2782GiB/s) | 20260508 03:59:35.005 | INFO | Analyzing chunk frames = [5065216,5065728) | 20260508 03:59:35.005 | INFO | MemInfo: { | 20260508 03:59:35.005 | INFO | "gpu" : " 17 GB", | 20260508 03:59:35.005 | INFO | "huge" : "", | 20260508 03:59:35.005 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:35.005 | INFO | "malloc" : " 14 GB", | 20260508 03:59:35.005 | INFO | "pinned" : " 29 GB" | 20260508 03:59:35.005 | INFO | } | 20260508 03:59:35.005 | INFO | Perf report for the last 5.13345 seconds: | 20260508 03:59:35.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:35.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 03:59:35.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 03:59:35.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:35.005 | INFO | Uploaded 36GiB to GPU in 2.39214s (15.0493GiB/s) | 20260508 03:59:35.005 | INFO | Downloaded 34GiB to GPU in 2.44759s (13.8912GiB/s) | 20260508 03:59:40.145 | INFO | Analyzing chunk frames = [5065728,5066240) | 20260508 03:59:40.145 | INFO | MemInfo: { | 20260508 03:59:40.145 | INFO | "gpu" : " 17 GB", | 20260508 03:59:40.145 | INFO | "huge" : "", | 20260508 03:59:40.145 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:40.145 | INFO | "malloc" : " 14 GB", | 20260508 03:59:40.145 | INFO | "pinned" : " 29 GB" | 20260508 03:59:40.146 | INFO | } | 20260508 03:59:40.146 | INFO | Perf report for the last 5.1407 seconds: | 20260508 03:59:40.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:40.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 03:59:40.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 03:59:40.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:40.146 | INFO | Uploaded 36GiB to GPU in 2.37167s (15.1792GiB/s) | 20260508 03:59:40.146 | INFO | Downloaded 34GiB to GPU in 2.43804s (13.9456GiB/s) | 20260508 03:59:45.282 | INFO | Analyzing chunk frames = [5066240,5066752) | 20260508 03:59:45.282 | INFO | MemInfo: { | 20260508 03:59:45.282 | INFO | "gpu" : " 17 GB", | 20260508 03:59:45.282 | INFO | "huge" : "", | 20260508 03:59:45.282 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:45.282 | INFO | "malloc" : " 14 GB", | 20260508 03:59:45.282 | INFO | "pinned" : " 29 GB" | 20260508 03:59:45.282 | INFO | } | 20260508 03:59:45.283 | INFO | Perf report for the last 5.13684 seconds: | 20260508 03:59:45.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:45.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 03:59:45.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 03:59:45.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:45.283 | INFO | Uploaded 36GiB to GPU in 2.42006s (14.8757GiB/s) | 20260508 03:59:45.283 | INFO | Downloaded 34GiB to GPU in 2.45197s (13.8664GiB/s) | 20260508 03:59:45.398 | INFO | Starting a round of Baz IO | 20260508 03:59:48.830 | INFO | Populating RTMetrics report | 20260508 03:59:48.830 | INFO | RTMetrics report populated | 20260508 03:59:48.830 | INFO | Opening tmp RTMetrics json file | 20260508 03:59:48.842 | INFO | Serializing RTMetrics to json | 20260508 03:59:48.844 | INFO | Serialization done, swapping tmp file now... | 20260508 03:59:48.844 | INFO | RTMetrics done | 20260508 03:59:48.898 | INFO | Finished a round of Baz IO: Wrote 1.604 GiB in 3.499 seconds | 20260508 03:59:48.898 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.16% / 0.00% / 58.84% | 20260508 03:59:48.898 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.571072% / 3.97364e-06% / 5.5631e-05% | 20260508 03:59:48.898 | INFO | Baz IO overall: | 20260508 03:59:48.898 | INFO | BytesWritten : 2272984868621 | 20260508 03:59:48.899 | INFO | NumEvents : 1097738526509 | 20260508 03:59:50.420 | INFO | Analyzing chunk frames = [5066752,5067264) | 20260508 03:59:50.420 | INFO | MemInfo: { | 20260508 03:59:50.420 | INFO | "gpu" : " 17 GB", | 20260508 03:59:50.420 | INFO | "huge" : "", | 20260508 03:59:50.420 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:50.420 | INFO | "malloc" : " 14 GB", | 20260508 03:59:50.420 | INFO | "pinned" : " 29 GB" | 20260508 03:59:50.420 | INFO | } | 20260508 03:59:50.420 | INFO | Perf report for the last 5.13763 seconds: | 20260508 03:59:50.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:50.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:50.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:50.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:50.420 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 03:59:50.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:50.420 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 03:59:50.420 | INFO | PRE_HQ : 66.6%, 1.8, 62.3ms, 0 | 20260508 03:59:50.420 | INFO | BAZWRITER : 67.5%, 2.9, 99.3ms, 0 | 20260508 03:59:50.420 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 03:59:50.420 | INFO | Uploaded 36GiB to GPU in 2.57364s (13.988GiB/s) | 20260508 03:59:50.420 | INFO | Downloaded 38GiB to GPU in 2.69561s (14.097GiB/s) | 20260508 03:59:55.560 | INFO | Analyzing chunk frames = [5067264,5067776) | 20260508 03:59:55.560 | INFO | MemInfo: { | 20260508 03:59:55.560 | INFO | "gpu" : " 17 GB", | 20260508 03:59:55.560 | INFO | "huge" : "", | 20260508 03:59:55.560 | INFO | "huge_shared" : " 29 GB", | 20260508 03:59:55.560 | INFO | "malloc" : " 14 GB", | 20260508 03:59:55.560 | INFO | "pinned" : " 29 GB" | 20260508 03:59:55.560 | INFO | } | 20260508 03:59:55.560 | INFO | Perf report for the last 5.14 seconds: | 20260508 03:59:55.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 03:59:55.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.560 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 03:59:55.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.560 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 03:59:55.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 03:59:55.561 | INFO | Uploaded 36GiB to GPU in 2.48414s (14.4919GiB/s) | 20260508 03:59:55.561 | INFO | Downloaded 34GiB to GPU in 2.43685s (13.9524GiB/s) | 20260508 04:00:00.696 | INFO | Analyzing chunk frames = [5067776,5068288) | 20260508 04:00:00.696 | INFO | MemInfo: { | 20260508 04:00:00.696 | INFO | "gpu" : " 17 GB", | 20260508 04:00:00.696 | INFO | "huge" : "", | 20260508 04:00:00.696 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:00.696 | INFO | "malloc" : " 14 GB", | 20260508 04:00:00.696 | INFO | "pinned" : " 29 GB" | 20260508 04:00:00.696 | INFO | } | 20260508 04:00:00.696 | INFO | Perf report for the last 5.1358 seconds: | 20260508 04:00:00.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:00.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 04:00:00.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:00:00.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:00.696 | INFO | Uploaded 36GiB to GPU in 2.4965s (14.4202GiB/s) | 20260508 04:00:00.696 | INFO | Downloaded 34GiB to GPU in 2.44632s (13.8984GiB/s) | 20260508 04:00:05.841 | INFO | Analyzing chunk frames = [5068288,5068800) | 20260508 04:00:05.841 | INFO | MemInfo: { | 20260508 04:00:05.841 | INFO | "gpu" : " 17 GB", | 20260508 04:00:05.841 | INFO | "huge" : "", | 20260508 04:00:05.841 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:05.841 | INFO | "malloc" : " 14 GB", | 20260508 04:00:05.841 | INFO | "pinned" : " 29 GB" | 20260508 04:00:05.841 | INFO | } | 20260508 04:00:05.841 | INFO | Perf report for the last 5.14491 seconds: | 20260508 04:00:05.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:05.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 04:00:05.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:00:05.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:05.841 | INFO | Uploaded 36GiB to GPU in 2.48494s (14.4873GiB/s) | 20260508 04:00:05.841 | INFO | Downloaded 34GiB to GPU in 2.52166s (13.4832GiB/s) | 20260508 04:00:10.983 | INFO | Analyzing chunk frames = [5068800,5069312) | 20260508 04:00:10.983 | INFO | MemInfo: { | 20260508 04:00:10.983 | INFO | "gpu" : " 17 GB", | 20260508 04:00:10.983 | INFO | "huge" : "", | 20260508 04:00:10.983 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:10.983 | INFO | "malloc" : " 14 GB", | 20260508 04:00:10.983 | INFO | "pinned" : " 29 GB" | 20260508 04:00:10.984 | INFO | } | 20260508 04:00:10.984 | INFO | Perf report for the last 5.14262 seconds: | 20260508 04:00:10.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:10.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:00:10.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:00:10.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:10.984 | INFO | Uploaded 36GiB to GPU in 2.30071s (15.6473GiB/s) | 20260508 04:00:10.984 | INFO | Downloaded 34GiB to GPU in 2.5711s (13.2239GiB/s) | 20260508 04:00:16.118 | INFO | Analyzing chunk frames = [5069312,5069824) | 20260508 04:00:16.118 | INFO | MemInfo: { | 20260508 04:00:16.118 | INFO | "gpu" : " 17 GB", | 20260508 04:00:16.118 | INFO | "huge" : "", | 20260508 04:00:16.118 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:16.118 | INFO | "malloc" : " 14 GB", | 20260508 04:00:16.118 | INFO | "pinned" : " 29 GB" | 20260508 04:00:16.118 | INFO | } | 20260508 04:00:16.118 | INFO | Perf report for the last 5.13449 seconds: | 20260508 04:00:16.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:16.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 04:00:16.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:00:16.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:16.118 | INFO | Uploaded 36GiB to GPU in 2.38374s (15.1023GiB/s) | 20260508 04:00:16.118 | INFO | Downloaded 34GiB to GPU in 2.44058s (13.9311GiB/s) | 20260508 04:00:21.254 | INFO | Analyzing chunk frames = [5069824,5070336) | 20260508 04:00:21.254 | INFO | MemInfo: { | 20260508 04:00:21.254 | INFO | "gpu" : " 17 GB", | 20260508 04:00:21.254 | INFO | "huge" : "", | 20260508 04:00:21.254 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:21.254 | INFO | "malloc" : " 14 GB", | 20260508 04:00:21.254 | INFO | "pinned" : " 29 GB" | 20260508 04:00:21.254 | INFO | } | 20260508 04:00:21.254 | INFO | Perf report for the last 5.13563 seconds: | 20260508 04:00:21.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:21.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:00:21.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 04:00:21.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:21.254 | INFO | Uploaded 36GiB to GPU in 2.39367s (15.0397GiB/s) | 20260508 04:00:21.254 | INFO | Downloaded 34GiB to GPU in 2.44081s (13.9298GiB/s) | 20260508 04:00:26.400 | INFO | Analyzing chunk frames = [5070336,5070848) | 20260508 04:00:26.400 | INFO | MemInfo: { | 20260508 04:00:26.400 | INFO | "gpu" : " 17 GB", | 20260508 04:00:26.400 | INFO | "huge" : "", | 20260508 04:00:26.400 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:26.400 | INFO | "malloc" : " 14 GB", | 20260508 04:00:26.400 | INFO | "pinned" : " 29 GB" | 20260508 04:00:26.400 | INFO | } | 20260508 04:00:26.400 | INFO | Perf report for the last 5.14663 seconds: | 20260508 04:00:26.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:26.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.400 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:00:26.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.401 | INFO | PRE_HQ : 35.1%, 1.0, 18.2ms, 0 | 20260508 04:00:26.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:26.401 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:00:26.401 | INFO | Total mem usage: 16.7 GiB | 20260508 04:00:26.401 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:00:26.401 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:00:26.401 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:00:26.401 | INFO | High Water Breakdown: | 20260508 04:00:26.401 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:00:26.401 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:00:26.401 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:00:26.401 | INFO | ------------------------------------------ | 20260508 04:00:26.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:00:26.401 | INFO | Uploaded 36GiB to GPU in 2.40823s (14.9487GiB/s) | 20260508 04:00:26.401 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:00:26.401 | INFO | Total mem usage: 26.5 GiB | 20260508 04:00:26.401 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:00:26.401 | INFO | Downloaded 34GiB to GPU in 2.44721s (13.8934GiB/s) | 20260508 04:00:26.401 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:00:26.401 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:00:26.401 | INFO | High Water Breakdown: | 20260508 04:00:26.401 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:00:26.401 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:00:26.401 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:00:26.401 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:00:26.401 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:00:26.401 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:00:26.401 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:00:26.401 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:00:26.401 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:00:26.401 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:00:26.401 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:00:26.401 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:00:26.401 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:00:26.401 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:00:26.401 | INFO | ------------------------------------------ | 20260508 04:00:26.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:00:26.401 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:00:26.401 | INFO | Total mem usage: 27.8 GiB | 20260508 04:00:26.402 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:00:26.402 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:00:26.402 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:00:26.402 | INFO | High Water Breakdown: | 20260508 04:00:26.402 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:00:26.402 | INFO | ------------------------------------------ | 20260508 04:00:26.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:00:26.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:00:26.402 | INFO | Total mem usage: 15.7 GiB | 20260508 04:00:26.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:00:26.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:00:26.402 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:00:26.402 | INFO | High Water Breakdown: | 20260508 04:00:26.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:00:26.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:00:26.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:00:26.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:00:26.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:00:26.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:00:26.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:00:26.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:00:26.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:00:26.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:00:26.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:00:26.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:00:26.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:00:26.402 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:00:26.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:00:26.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:00:26.402 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:00:26.402 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:00:26.402 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:00:26.402 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:00:26.402 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:00:26.402 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:00:26.402 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:00:26.402 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:00:26.402 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:00:26.402 | INFO | ------------------------------------------ | 20260508 04:00:26.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:00:26.402 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:00:26.402 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:00:26.402 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:00:26.402 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.40e+01 / 1.60e+00 | 20260508 04:00:26.402 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.04e-01, 1.22e-02 / 6.54e-03 | 20260508 04:00:26.402 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.42e+00, 5.89e-01 / 2.66e-01 | 20260508 04:00:26.402 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.42e+00, 2.10e-01 / 1.34e-01 | 20260508 04:00:26.402 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.55e+01, 3.75e+00 / 1.85e+00 | 20260508 04:00:26.402 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.67e+00 | 20260508 04:00:26.402 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.20e-01 / 2.24e-01 | 20260508 04:00:26.402 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.23e+00, 2.60e-01 / 1.43e-01 | 20260508 04:00:26.402 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.25e-01 / 3.61e-01, 5.07e-02 / 2.55e-02 | 20260508 04:00:26.402 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:00:26.402 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:00:26.402 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:00:26.402 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 04:00:26.402 | INFO | PerfReport: 24.00%, Upload -- 2.55e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 04:00:26.402 | INFO | PerfReport: 24.47%, Download -- 2.60e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 04:00:26.403 | INFO | PerfReport: 30.90%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:00:26.403 | INFO | PerfReport: 1.88%, LockUpload -- 1.99e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260508 04:00:26.403 | INFO | PerfReport: 3.02%, LockDownload -- 3.20e+00 / 0.00e+00, 4.59e+00 / 0.00e+00 | 20260508 04:00:26.403 | INFO | PerfReport: 15.73%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 04:00:26.403 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260508 04:00:26.513 | INFO | Starting a round of Baz IO | 20260508 04:00:29.938 | INFO | Populating RTMetrics report | 20260508 04:00:29.939 | INFO | RTMetrics report populated | 20260508 04:00:29.939 | INFO | Opening tmp RTMetrics json file | 20260508 04:00:29.950 | INFO | Serializing RTMetrics to json | 20260508 04:00:29.952 | INFO | Serialization done, swapping tmp file now... | 20260508 04:00:29.952 | INFO | RTMetrics done | 20260508 04:00:30.010 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.497 seconds | 20260508 04:00:30.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.16% / 0.00% / 58.84% | 20260508 04:00:30.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.57201% / 7.94729e-06% / 5.96046e-05% | 20260508 04:00:30.010 | INFO | Baz IO overall: | 20260508 04:00:30.010 | INFO | BytesWritten : 2274708810382 | 20260508 04:00:30.010 | INFO | NumEvents : 1098563273835 | 20260508 04:00:31.542 | INFO | Analyzing chunk frames = [5070848,5071360) | 20260508 04:00:31.542 | INFO | MemInfo: { | 20260508 04:00:31.542 | INFO | "gpu" : " 17 GB", | 20260508 04:00:31.542 | INFO | "huge" : "", | 20260508 04:00:31.542 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:31.542 | INFO | "malloc" : " 14 GB", | 20260508 04:00:31.542 | INFO | "pinned" : " 29 GB" | 20260508 04:00:31.542 | INFO | } | 20260508 04:00:31.542 | INFO | Perf report for the last 5.14193 seconds: | 20260508 04:00:31.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:31.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:31.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:31.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:31.543 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 04:00:31.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:31.543 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260508 04:00:31.543 | INFO | PRE_HQ : 66.5%, 1.8, 60.6ms, 0 | 20260508 04:00:31.543 | INFO | BAZWRITER : 67.4%, 2.8, 97.2ms, 0 | 20260508 04:00:31.543 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:00:31.543 | INFO | Uploaded 36GiB to GPU in 2.53033s (14.2274GiB/s) | 20260508 04:00:31.543 | INFO | Downloaded 38GiB to GPU in 2.68233s (14.1668GiB/s) | 20260508 04:00:36.683 | INFO | Analyzing chunk frames = [5071360,5071872) | 20260508 04:00:36.683 | INFO | MemInfo: { | 20260508 04:00:36.683 | INFO | "gpu" : " 17 GB", | 20260508 04:00:36.683 | INFO | "huge" : "", | 20260508 04:00:36.683 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:36.683 | INFO | "malloc" : " 14 GB", | 20260508 04:00:36.683 | INFO | "pinned" : " 29 GB" | 20260508 04:00:36.683 | INFO | } | 20260508 04:00:36.684 | INFO | Perf report for the last 5.14119 seconds: | 20260508 04:00:36.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:36.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:00:36.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:00:36.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:36.684 | INFO | Uploaded 36GiB to GPU in 2.58579s (13.9222GiB/s) | 20260508 04:00:36.684 | INFO | Downloaded 34GiB to GPU in 2.4471s (13.894GiB/s) | 20260508 04:00:41.817 | INFO | Analyzing chunk frames = [5071872,5072384) | 20260508 04:00:41.817 | INFO | MemInfo: { | 20260508 04:00:41.817 | INFO | "gpu" : " 17 GB", | 20260508 04:00:41.817 | INFO | "huge" : "", | 20260508 04:00:41.817 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:41.818 | INFO | "malloc" : " 14 GB", | 20260508 04:00:41.818 | INFO | "pinned" : " 29 GB" | 20260508 04:00:41.818 | INFO | } | 20260508 04:00:41.818 | INFO | Perf report for the last 5.1341 seconds: | 20260508 04:00:41.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:41.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:00:41.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:00:41.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:41.818 | INFO | Uploaded 36GiB to GPU in 2.53737s (14.1879GiB/s) | 20260508 04:00:41.818 | INFO | Downloaded 34GiB to GPU in 2.4195s (14.0525GiB/s) | 20260508 04:00:46.955 | INFO | Analyzing chunk frames = [5072384,5072896) | 20260508 04:00:46.955 | INFO | MemInfo: { | 20260508 04:00:46.955 | INFO | "gpu" : " 17 GB", | 20260508 04:00:46.955 | INFO | "huge" : "", | 20260508 04:00:46.955 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:46.955 | INFO | "malloc" : " 14 GB", | 20260508 04:00:46.955 | INFO | "pinned" : " 29 GB" | 20260508 04:00:46.955 | INFO | } | 20260508 04:00:46.955 | INFO | Perf report for the last 5.13741 seconds: | 20260508 04:00:46.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:46.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:00:46.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:00:46.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:46.955 | INFO | Uploaded 36GiB to GPU in 2.56351s (14.0432GiB/s) | 20260508 04:00:46.955 | INFO | Downloaded 34GiB to GPU in 2.4084s (14.1173GiB/s) | 20260508 04:00:52.094 | INFO | Analyzing chunk frames = [5072896,5073408) | 20260508 04:00:52.094 | INFO | MemInfo: { | 20260508 04:00:52.094 | INFO | "gpu" : " 17 GB", | 20260508 04:00:52.094 | INFO | "huge" : "", | 20260508 04:00:52.095 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:52.095 | INFO | "malloc" : " 14 GB", | 20260508 04:00:52.095 | INFO | "pinned" : " 29 GB" | 20260508 04:00:52.095 | INFO | } | 20260508 04:00:52.095 | INFO | Perf report for the last 5.13963 seconds: | 20260508 04:00:52.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:52.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:00:52.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:00:52.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:52.095 | INFO | Uploaded 36GiB to GPU in 2.41884s (14.8831GiB/s) | 20260508 04:00:52.095 | INFO | Downloaded 34GiB to GPU in 2.56299s (13.2657GiB/s) | 20260508 04:00:57.236 | INFO | Analyzing chunk frames = [5073408,5073920) | 20260508 04:00:57.236 | INFO | MemInfo: { | 20260508 04:00:57.236 | INFO | "gpu" : " 17 GB", | 20260508 04:00:57.236 | INFO | "huge" : "", | 20260508 04:00:57.236 | INFO | "huge_shared" : " 29 GB", | 20260508 04:00:57.236 | INFO | "malloc" : " 14 GB", | 20260508 04:00:57.236 | INFO | "pinned" : " 29 GB" | 20260508 04:00:57.236 | INFO | } | 20260508 04:00:57.236 | INFO | Perf report for the last 5.14109 seconds: | 20260508 04:00:57.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:00:57.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:00:57.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:00:57.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:00:57.236 | INFO | Uploaded 36GiB to GPU in 2.44454s (14.7267GiB/s) | 20260508 04:00:57.236 | INFO | Downloaded 34GiB to GPU in 2.40981s (14.109GiB/s) | 20260508 04:01:02.371 | INFO | Analyzing chunk frames = [5073920,5074432) | 20260508 04:01:02.371 | INFO | MemInfo: { | 20260508 04:01:02.371 | INFO | "gpu" : " 17 GB", | 20260508 04:01:02.371 | INFO | "huge" : "", | 20260508 04:01:02.371 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:02.371 | INFO | "malloc" : " 14 GB", | 20260508 04:01:02.371 | INFO | "pinned" : " 29 GB" | 20260508 04:01:02.371 | INFO | } | 20260508 04:01:02.371 | INFO | Perf report for the last 5.13532 seconds: | 20260508 04:01:02.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:02.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:01:02.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 04:01:02.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:02.371 | INFO | Uploaded 36GiB to GPU in 2.44942s (14.6974GiB/s) | 20260508 04:01:02.371 | INFO | Downloaded 34GiB to GPU in 2.41539s (14.0764GiB/s) | 20260508 04:01:07.510 | INFO | Analyzing chunk frames = [5074432,5074944) | 20260508 04:01:07.510 | INFO | MemInfo: { | 20260508 04:01:07.510 | INFO | "gpu" : " 17 GB", | 20260508 04:01:07.510 | INFO | "huge" : "", | 20260508 04:01:07.510 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:07.510 | INFO | "malloc" : " 14 GB", | 20260508 04:01:07.510 | INFO | "pinned" : " 29 GB" | 20260508 04:01:07.510 | INFO | } | 20260508 04:01:07.510 | INFO | Perf report for the last 5.13885 seconds: | 20260508 04:01:07.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:07.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 04:01:07.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 04:01:07.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:07.510 | INFO | Uploaded 36GiB to GPU in 2.46945s (14.5781GiB/s) | 20260508 04:01:07.511 | INFO | Downloaded 34GiB to GPU in 2.42898s (13.9977GiB/s) | 20260508 04:01:07.624 | INFO | Starting a round of Baz IO | 20260508 04:01:11.048 | INFO | Populating RTMetrics report | 20260508 04:01:11.048 | INFO | RTMetrics report populated | 20260508 04:01:11.048 | INFO | Opening tmp RTMetrics json file | 20260508 04:01:11.059 | INFO | Serializing RTMetrics to json | 20260508 04:01:11.060 | INFO | Serialization done, swapping tmp file now... | 20260508 04:01:11.061 | INFO | RTMetrics done | 20260508 04:01:11.119 | INFO | Finished a round of Baz IO: Wrote 1.604 GiB in 3.495 seconds | 20260508 04:01:11.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.15% / 0.00% / 58.84% | 20260508 04:01:11.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.570103% / 3.97364e-06% / 4.37101e-05% | 20260508 04:01:11.119 | INFO | Baz IO overall: | 20260508 04:01:11.119 | INFO | BytesWritten : 2276431045406 | 20260508 04:01:11.119 | INFO | NumEvents : 1099386876744 | 20260508 04:01:12.652 | INFO | Analyzing chunk frames = [5074944,5075456) | 20260508 04:01:12.652 | INFO | MemInfo: { | 20260508 04:01:12.652 | INFO | "gpu" : " 17 GB", | 20260508 04:01:12.653 | INFO | "huge" : "", | 20260508 04:01:12.653 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:12.653 | INFO | "malloc" : " 14 GB", | 20260508 04:01:12.653 | INFO | "pinned" : " 29 GB" | 20260508 04:01:12.653 | INFO | } | 20260508 04:01:12.653 | INFO | Perf report for the last 5.14268 seconds: | 20260508 04:01:12.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:12.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:12.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:12.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:12.653 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 04:01:12.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:12.653 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 04:01:12.653 | INFO | PRE_HQ : 66.8%, 1.8, 62.5ms, 0 | 20260508 04:01:12.653 | INFO | BAZWRITER : 67.5%, 2.9, 99.7ms, 0 | 20260508 04:01:12.653 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:01:12.653 | INFO | Uploaded 36GiB to GPU in 2.56068s (14.0587GiB/s) | 20260508 04:01:12.653 | INFO | Downloaded 38GiB to GPU in 2.69736s (14.0878GiB/s) | 20260508 04:01:17.792 | INFO | Analyzing chunk frames = [5075456,5075968) | 20260508 04:01:17.792 | INFO | MemInfo: { | 20260508 04:01:17.792 | INFO | "gpu" : " 17 GB", | 20260508 04:01:17.793 | INFO | "huge" : "", | 20260508 04:01:17.793 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:17.793 | INFO | "malloc" : " 14 GB", | 20260508 04:01:17.793 | INFO | "pinned" : " 29 GB" | 20260508 04:01:17.793 | INFO | } | 20260508 04:01:17.793 | INFO | Perf report for the last 5.14005 seconds: | 20260508 04:01:17.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:17.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:01:17.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:01:17.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:17.793 | INFO | Uploaded 36GiB to GPU in 2.5726s (13.9936GiB/s) | 20260508 04:01:17.793 | INFO | Downloaded 34GiB to GPU in 2.43939s (13.9379GiB/s) | 20260508 04:01:22.932 | INFO | Analyzing chunk frames = [5075968,5076480) | 20260508 04:01:22.932 | INFO | MemInfo: { | 20260508 04:01:22.932 | INFO | "gpu" : " 17 GB", | 20260508 04:01:22.932 | INFO | "huge" : "", | 20260508 04:01:22.932 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:22.932 | INFO | "malloc" : " 14 GB", | 20260508 04:01:22.932 | INFO | "pinned" : " 29 GB" | 20260508 04:01:22.932 | INFO | } | 20260508 04:01:22.932 | INFO | Perf report for the last 5.13909 seconds: | 20260508 04:01:22.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:22.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 04:01:22.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:01:22.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:22.932 | INFO | Uploaded 36GiB to GPU in 2.52728s (14.2446GiB/s) | 20260508 04:01:22.932 | INFO | Downloaded 34GiB to GPU in 2.43228s (13.9787GiB/s) | 20260508 04:01:28.067 | INFO | Analyzing chunk frames = [5076480,5076992) | 20260508 04:01:28.067 | INFO | MemInfo: { | 20260508 04:01:28.067 | INFO | "gpu" : " 17 GB", | 20260508 04:01:28.067 | INFO | "huge" : "", | 20260508 04:01:28.067 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:28.067 | INFO | "malloc" : " 14 GB", | 20260508 04:01:28.067 | INFO | "pinned" : " 29 GB" | 20260508 04:01:28.068 | INFO | } | 20260508 04:01:28.068 | INFO | Perf report for the last 5.13566 seconds: | 20260508 04:01:28.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:28.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 04:01:28.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:01:28.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:28.068 | INFO | Uploaded 36GiB to GPU in 2.54785s (14.1296GiB/s) | 20260508 04:01:28.068 | INFO | Downloaded 34GiB to GPU in 2.50413s (13.5776GiB/s) | 20260508 04:01:33.207 | INFO | Analyzing chunk frames = [5076992,5077504) | 20260508 04:01:33.207 | INFO | MemInfo: { | 20260508 04:01:33.207 | INFO | "gpu" : " 17 GB", | 20260508 04:01:33.207 | INFO | "huge" : "", | 20260508 04:01:33.207 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:33.207 | INFO | "malloc" : " 14 GB", | 20260508 04:01:33.207 | INFO | "pinned" : " 29 GB" | 20260508 04:01:33.207 | INFO | } | 20260508 04:01:33.207 | INFO | Perf report for the last 5.13969 seconds: | 20260508 04:01:33.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:33.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.207 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 04:01:33.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.207 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:01:33.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:33.208 | INFO | Uploaded 36GiB to GPU in 2.43367s (14.7925GiB/s) | 20260508 04:01:33.208 | INFO | Downloaded 34GiB to GPU in 2.5917s (13.1188GiB/s) | 20260508 04:01:38.347 | INFO | Analyzing chunk frames = [5077504,5078016) | 20260508 04:01:38.347 | INFO | MemInfo: { | 20260508 04:01:38.347 | INFO | "gpu" : " 17 GB", | 20260508 04:01:38.347 | INFO | "huge" : "", | 20260508 04:01:38.347 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:38.348 | INFO | "malloc" : " 14 GB", | 20260508 04:01:38.348 | INFO | "pinned" : " 29 GB" | 20260508 04:01:38.348 | INFO | } | 20260508 04:01:38.348 | INFO | Perf report for the last 5.14048 seconds: | 20260508 04:01:38.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:38.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:01:38.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:01:38.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:38.348 | INFO | Uploaded 36GiB to GPU in 2.41999s (14.8761GiB/s) | 20260508 04:01:38.348 | INFO | Downloaded 34GiB to GPU in 2.42481s (14.0217GiB/s) | 20260508 04:01:43.489 | INFO | Analyzing chunk frames = [5078016,5078528) | 20260508 04:01:43.489 | INFO | MemInfo: { | 20260508 04:01:43.489 | INFO | "gpu" : " 17 GB", | 20260508 04:01:43.489 | INFO | "huge" : "", | 20260508 04:01:43.489 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:43.489 | INFO | "malloc" : " 14 GB", | 20260508 04:01:43.489 | INFO | "pinned" : " 29 GB" | 20260508 04:01:43.489 | INFO | } | 20260508 04:01:43.490 | INFO | Perf report for the last 5.14176 seconds: | 20260508 04:01:43.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:43.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 04:01:43.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:01:43.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:43.490 | INFO | Uploaded 36GiB to GPU in 2.42518s (14.8443GiB/s) | 20260508 04:01:43.490 | INFO | Downloaded 34GiB to GPU in 2.42499s (14.0207GiB/s) | 20260508 04:01:48.631 | INFO | Analyzing chunk frames = [5078528,5079040) | 20260508 04:01:48.631 | INFO | MemInfo: { | 20260508 04:01:48.631 | INFO | "gpu" : " 17 GB", | 20260508 04:01:48.631 | INFO | "huge" : "", | 20260508 04:01:48.631 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:48.631 | INFO | "malloc" : " 14 GB", | 20260508 04:01:48.631 | INFO | "pinned" : " 29 GB" | 20260508 04:01:48.631 | INFO | } | 20260508 04:01:48.631 | INFO | Perf report for the last 5.14144 seconds: | 20260508 04:01:48.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:48.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:01:48.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | PRE_HQ : 34.2%, 1.0, 17.7ms, 0 | 20260508 04:01:48.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:48.631 | INFO | Uploaded 36GiB to GPU in 2.45754s (14.6488GiB/s) | 20260508 04:01:48.631 | INFO | Downloaded 34GiB to GPU in 2.43399s (13.9688GiB/s) | 20260508 04:01:48.632 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:01:48.632 | INFO | Total mem usage: 16.7 GiB | 20260508 04:01:48.632 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:01:48.632 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.7%) | 20260508 04:01:48.632 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:01:48.632 | INFO | High Water Breakdown: | 20260508 04:01:48.632 | INFO | 59.7% : 8.6 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:01:48.632 | INFO | 40.2% : 5.8 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:01:48.632 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:01:48.632 | INFO | ------------------------------------------ | 20260508 04:01:48.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:01:48.632 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:01:48.632 | INFO | Total mem usage: 26.5 GiB | 20260508 04:01:48.632 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:01:48.632 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:01:48.632 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:01:48.632 | INFO | High Water Breakdown: | 20260508 04:01:48.632 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:01:48.632 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:01:48.632 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:01:48.632 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:01:48.632 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:01:48.632 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:01:48.632 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:01:48.632 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:01:48.632 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:01:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:01:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:01:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:01:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:01:48.632 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:01:48.632 | INFO | ------------------------------------------ | 20260508 04:01:48.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:01:48.632 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:01:48.632 | INFO | Total mem usage: 27.8 GiB | 20260508 04:01:48.632 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:01:48.632 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:01:48.632 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:01:48.632 | INFO | High Water Breakdown: | 20260508 04:01:48.632 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:01:48.633 | INFO | ------------------------------------------ | 20260508 04:01:48.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:01:48.633 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:01:48.633 | INFO | Total mem usage: 15.7 GiB | 20260508 04:01:48.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:01:48.633 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:01:48.633 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 04:01:48.633 | INFO | High Water Breakdown: | 20260508 04:01:48.633 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:01:48.633 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:01:48.633 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:01:48.633 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:01:48.633 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:01:48.633 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:01:48.633 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:01:48.633 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:01:48.633 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:01:48.633 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:01:48.633 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:01:48.633 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:01:48.633 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:01:48.633 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:01:48.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:01:48.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:01:48.633 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:01:48.633 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:01:48.633 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:01:48.633 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:01:48.633 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:01:48.633 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:01:48.633 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:01:48.633 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:01:48.633 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:01:48.633 | INFO | ------------------------------------------ | 20260508 04:01:48.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:01:48.633 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:01:48.633 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:01:48.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:01:48.633 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.41e+01 / 1.91e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.05e-01, 1.23e-02 / 6.87e-03 | 20260508 04:01:48.633 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.49e+00, 5.92e-01 / 1.68e-01 | 20260508 04:01:48.633 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.07e-01 / 1.20e-01 | 20260508 04:01:48.633 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.55e+01, 3.88e+00 / 2.14e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.79e+00 / 1.46e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.39e+00 / 2.65e+00, 4.12e-01 / 1.67e-01 | 20260508 04:01:48.633 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.26e+00, 2.65e-01 / 1.29e-01 | 20260508 04:01:48.633 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.66e-01, 5.02e-02 / 2.26e-02 | 20260508 04:01:48.633 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:01:48.633 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:01:48.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:01:48.633 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 2.25%, LockUpload -- 2.40e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 04:01:48.633 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.73e-02 / 0.00e+00 | 20260508 04:01:48.744 | INFO | Starting a round of Baz IO | 20260508 04:01:52.158 | INFO | Populating RTMetrics report | 20260508 04:01:52.159 | INFO | RTMetrics report populated | 20260508 04:01:52.159 | INFO | Opening tmp RTMetrics json file | 20260508 04:01:52.172 | INFO | Serializing RTMetrics to json | 20260508 04:01:52.173 | INFO | Serialization done, swapping tmp file now... | 20260508 04:01:52.173 | INFO | RTMetrics done | 20260508 04:01:52.231 | INFO | Finished a round of Baz IO: Wrote 1.603 GiB in 3.487 seconds | 20260508 04:01:52.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.15% / 0.00% / 58.85% | 20260508 04:01:52.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.568374% / 0% / 5.5631e-05% | 20260508 04:01:52.232 | INFO | Baz IO overall: | 20260508 04:01:52.232 | INFO | BytesWritten : 2278152549217 | 20260508 04:01:52.232 | INFO | NumEvents : 1100209818954 | 20260508 04:01:53.766 | INFO | Analyzing chunk frames = [5079040,5079552) | 20260508 04:01:53.766 | INFO | MemInfo: { | 20260508 04:01:53.766 | INFO | "gpu" : " 17 GB", | 20260508 04:01:53.766 | INFO | "huge" : "", | 20260508 04:01:53.766 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:53.766 | INFO | "malloc" : " 14 GB", | 20260508 04:01:53.766 | INFO | "pinned" : " 29 GB" | 20260508 04:01:53.766 | INFO | } | 20260508 04:01:53.767 | INFO | Perf report for the last 5.13553 seconds: | 20260508 04:01:53.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:53.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:53.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:53.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:53.767 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 04:01:53.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:53.767 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 04:01:53.767 | INFO | PRE_HQ : 66.4%, 1.8, 61.9ms, 0 | 20260508 04:01:53.767 | INFO | BAZWRITER : 67.5%, 2.8, 98.7ms, 0 | 20260508 04:01:53.767 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:01:53.767 | INFO | Uploaded 36GiB to GPU in 2.58319s (13.9363GiB/s) | 20260508 04:01:53.767 | INFO | Downloaded 38GiB to GPU in 2.69544s (14.0979GiB/s) | 20260508 04:01:58.909 | INFO | Analyzing chunk frames = [5079552,5080064) | 20260508 04:01:58.909 | INFO | MemInfo: { | 20260508 04:01:58.909 | INFO | "gpu" : " 17 GB", | 20260508 04:01:58.909 | INFO | "huge" : "", | 20260508 04:01:58.909 | INFO | "huge_shared" : " 29 GB", | 20260508 04:01:58.909 | INFO | "malloc" : " 14 GB", | 20260508 04:01:58.909 | INFO | "pinned" : " 29 GB" | 20260508 04:01:58.909 | INFO | } | 20260508 04:01:58.909 | INFO | Perf report for the last 5.14262 seconds: | 20260508 04:01:58.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:01:58.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 04:01:58.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:01:58.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:01:58.909 | INFO | Uploaded 36GiB to GPU in 2.56346s (14.0435GiB/s) | 20260508 04:01:58.909 | INFO | Downloaded 34GiB to GPU in 2.43982s (13.9355GiB/s) | 20260508 04:02:04.046 | INFO | Analyzing chunk frames = [5080064,5080576) | 20260508 04:02:04.046 | INFO | MemInfo: { | 20260508 04:02:04.046 | INFO | "gpu" : " 17 GB", | 20260508 04:02:04.046 | INFO | "huge" : "", | 20260508 04:02:04.046 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:04.046 | INFO | "malloc" : " 14 GB", | 20260508 04:02:04.046 | INFO | "pinned" : " 29 GB" | 20260508 04:02:04.046 | INFO | } | 20260508 04:02:04.046 | INFO | Perf report for the last 5.13719 seconds: | 20260508 04:02:04.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:04.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.046 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 04:02:04.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.046 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:02:04.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:04.047 | INFO | Uploaded 36GiB to GPU in 2.53033s (14.2274GiB/s) | 20260508 04:02:04.047 | INFO | Downloaded 34GiB to GPU in 2.43781s (13.947GiB/s) | 20260508 04:02:09.187 | INFO | Analyzing chunk frames = [5080576,5081088) | 20260508 04:02:09.187 | INFO | MemInfo: { | 20260508 04:02:09.187 | INFO | "gpu" : " 17 GB", | 20260508 04:02:09.187 | INFO | "huge" : "", | 20260508 04:02:09.187 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:09.187 | INFO | "malloc" : " 14 GB", | 20260508 04:02:09.187 | INFO | "pinned" : " 29 GB" | 20260508 04:02:09.187 | INFO | } | 20260508 04:02:09.187 | INFO | Perf report for the last 5.14062 seconds: | 20260508 04:02:09.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:09.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 04:02:09.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:02:09.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:09.187 | INFO | Uploaded 36GiB to GPU in 2.54303s (14.1564GiB/s) | 20260508 04:02:09.187 | INFO | Downloaded 34GiB to GPU in 2.51312s (13.529GiB/s) | 20260508 04:02:14.322 | INFO | Analyzing chunk frames = [5081088,5081600) | 20260508 04:02:14.322 | INFO | MemInfo: { | 20260508 04:02:14.322 | INFO | "gpu" : " 17 GB", | 20260508 04:02:14.322 | INFO | "huge" : "", | 20260508 04:02:14.322 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:14.322 | INFO | "malloc" : " 14 GB", | 20260508 04:02:14.322 | INFO | "pinned" : " 29 GB" | 20260508 04:02:14.322 | INFO | } | 20260508 04:02:14.322 | INFO | Perf report for the last 5.13518 seconds: | 20260508 04:02:14.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:14.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.322 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 04:02:14.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.322 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:02:14.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:14.323 | INFO | Uploaded 36GiB to GPU in 2.34252s (15.3681GiB/s) | 20260508 04:02:14.323 | INFO | Downloaded 34GiB to GPU in 2.56505s (13.2551GiB/s) | 20260508 04:02:19.460 | INFO | Analyzing chunk frames = [5081600,5082112) | 20260508 04:02:19.460 | INFO | MemInfo: { | 20260508 04:02:19.460 | INFO | "gpu" : " 17 GB", | 20260508 04:02:19.460 | INFO | "huge" : "", | 20260508 04:02:19.460 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:19.460 | INFO | "malloc" : " 14 GB", | 20260508 04:02:19.460 | INFO | "pinned" : " 29 GB" | 20260508 04:02:19.460 | INFO | } | 20260508 04:02:19.460 | INFO | Perf report for the last 5.13796 seconds: | 20260508 04:02:19.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:19.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 04:02:19.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:02:19.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:19.460 | INFO | Uploaded 36GiB to GPU in 2.38489s (15.0951GiB/s) | 20260508 04:02:19.460 | INFO | Downloaded 34GiB to GPU in 2.43965s (13.9364GiB/s) | 20260508 04:02:24.604 | INFO | Analyzing chunk frames = [5082112,5082624) | 20260508 04:02:24.604 | INFO | MemInfo: { | 20260508 04:02:24.604 | INFO | "gpu" : " 17 GB", | 20260508 04:02:24.604 | INFO | "huge" : "", | 20260508 04:02:24.604 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:24.605 | INFO | "malloc" : " 14 GB", | 20260508 04:02:24.605 | INFO | "pinned" : " 29 GB" | 20260508 04:02:24.605 | INFO | } | 20260508 04:02:24.605 | INFO | Perf report for the last 5.14449 seconds: | 20260508 04:02:24.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:24.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 04:02:24.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 04:02:24.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:24.605 | INFO | Uploaded 36GiB to GPU in 2.39519s (15.0301GiB/s) | 20260508 04:02:24.605 | INFO | Downloaded 34GiB to GPU in 2.45001s (13.8775GiB/s) | 20260508 04:02:29.746 | INFO | Analyzing chunk frames = [5082624,5083136) | 20260508 04:02:29.746 | INFO | MemInfo: { | 20260508 04:02:29.746 | INFO | "gpu" : " 17 GB", | 20260508 04:02:29.746 | INFO | "huge" : "", | 20260508 04:02:29.746 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:29.746 | INFO | "malloc" : " 14 GB", | 20260508 04:02:29.746 | INFO | "pinned" : " 29 GB" | 20260508 04:02:29.746 | INFO | } | 20260508 04:02:29.746 | INFO | Perf report for the last 5.14126 seconds: | 20260508 04:02:29.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:29.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:02:29.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:02:29.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:29.746 | INFO | Uploaded 36GiB to GPU in 2.40708s (14.9559GiB/s) | 20260508 04:02:29.746 | INFO | Downloaded 34GiB to GPU in 2.43491s (13.9636GiB/s) | 20260508 04:02:29.843 | INFO | Starting a round of Baz IO | 20260508 04:02:33.266 | INFO | Populating RTMetrics report | 20260508 04:02:33.266 | INFO | RTMetrics report populated | 20260508 04:02:33.266 | INFO | Opening tmp RTMetrics json file | 20260508 04:02:33.272 | INFO | Serializing RTMetrics to json | 20260508 04:02:33.282 | INFO | Serialization done, swapping tmp file now... | 20260508 04:02:33.283 | INFO | RTMetrics done | 20260508 04:02:33.343 | INFO | Finished a round of Baz IO: Wrote 1.604 GiB in 3.499 seconds | 20260508 04:02:33.343 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.15% / 0.00% / 58.85% | 20260508 04:02:33.343 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.569324% / 0% / 6.75519e-05% | 20260508 04:02:33.343 | INFO | Baz IO overall: | 20260508 04:02:33.343 | INFO | BytesWritten : 2279875307679 | 20260508 04:02:33.343 | INFO | NumEvents : 1101033643661 | 20260508 04:02:34.881 | INFO | Analyzing chunk frames = [5083136,5083648) | 20260508 04:02:34.881 | INFO | MemInfo: { | 20260508 04:02:34.881 | INFO | "gpu" : " 17 GB", | 20260508 04:02:34.881 | INFO | "huge" : "", | 20260508 04:02:34.881 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:34.881 | INFO | "malloc" : " 14 GB", | 20260508 04:02:34.881 | INFO | "pinned" : " 29 GB" | 20260508 04:02:34.881 | INFO | } | 20260508 04:02:34.881 | INFO | Perf report for the last 5.1355 seconds: | 20260508 04:02:34.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:34.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:34.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:34.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:34.882 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 04:02:34.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:34.882 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 04:02:34.882 | INFO | PRE_HQ : 65.6%, 1.6, 55.7ms, 0 | 20260508 04:02:34.882 | INFO | BAZWRITER : 67.6%, 2.7, 92.8ms, 0 | 20260508 04:02:34.882 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:02:34.882 | INFO | Uploaded 36GiB to GPU in 2.55981s (14.0636GiB/s) | 20260508 04:02:34.882 | INFO | Downloaded 38GiB to GPU in 2.7022s (14.0626GiB/s) | 20260508 04:02:40.015 | INFO | Analyzing chunk frames = [5083648,5084160) | 20260508 04:02:40.015 | INFO | MemInfo: { | 20260508 04:02:40.015 | INFO | "gpu" : " 17 GB", | 20260508 04:02:40.015 | INFO | "huge" : "", | 20260508 04:02:40.015 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:40.015 | INFO | "malloc" : " 14 GB", | 20260508 04:02:40.015 | INFO | "pinned" : " 29 GB" | 20260508 04:02:40.015 | INFO | } | 20260508 04:02:40.015 | INFO | Perf report for the last 5.1335 seconds: | 20260508 04:02:40.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:40.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.015 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 04:02:40.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.015 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:02:40.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:40.016 | INFO | Uploaded 36GiB to GPU in 2.55682s (14.08GiB/s) | 20260508 04:02:40.016 | INFO | Downloaded 34GiB to GPU in 2.45425s (13.8535GiB/s) | 20260508 04:02:45.160 | INFO | Analyzing chunk frames = [5084160,5084672) | 20260508 04:02:45.160 | INFO | MemInfo: { | 20260508 04:02:45.160 | INFO | "gpu" : " 17 GB", | 20260508 04:02:45.160 | INFO | "huge" : "", | 20260508 04:02:45.160 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:45.160 | INFO | "malloc" : " 14 GB", | 20260508 04:02:45.160 | INFO | "pinned" : " 29 GB" | 20260508 04:02:45.160 | INFO | } | 20260508 04:02:45.160 | INFO | Perf report for the last 5.14525 seconds: | 20260508 04:02:45.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:45.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 04:02:45.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:02:45.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:45.160 | INFO | Uploaded 36GiB to GPU in 2.47521s (14.5442GiB/s) | 20260508 04:02:45.161 | INFO | Downloaded 34GiB to GPU in 2.51901s (13.4974GiB/s) | 20260508 04:02:50.298 | INFO | Analyzing chunk frames = [5084672,5085184) | 20260508 04:02:50.298 | INFO | MemInfo: { | 20260508 04:02:50.298 | INFO | "gpu" : " 17 GB", | 20260508 04:02:50.298 | INFO | "huge" : "", | 20260508 04:02:50.298 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:50.298 | INFO | "malloc" : " 14 GB", | 20260508 04:02:50.298 | INFO | "pinned" : " 29 GB" | 20260508 04:02:50.298 | INFO | } | 20260508 04:02:50.298 | INFO | Perf report for the last 5.13791 seconds: | 20260508 04:02:50.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:50.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.298 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:02:50.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.298 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:02:50.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:50.299 | INFO | Uploaded 36GiB to GPU in 2.55118s (14.1111GiB/s) | 20260508 04:02:50.299 | INFO | Downloaded 34GiB to GPU in 2.4374s (13.9493GiB/s) | 20260508 04:02:55.443 | INFO | Analyzing chunk frames = [5085184,5085696) | 20260508 04:02:55.443 | INFO | MemInfo: { | 20260508 04:02:55.443 | INFO | "gpu" : " 17 GB", | 20260508 04:02:55.443 | INFO | "huge" : "", | 20260508 04:02:55.443 | INFO | "huge_shared" : " 29 GB", | 20260508 04:02:55.443 | INFO | "malloc" : " 14 GB", | 20260508 04:02:55.443 | INFO | "pinned" : " 29 GB" | 20260508 04:02:55.443 | INFO | } | 20260508 04:02:55.443 | INFO | Perf report for the last 5.14496 seconds: | 20260508 04:02:55.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:02:55.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.443 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:02:55.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.443 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:02:55.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:02:55.444 | INFO | Uploaded 36GiB to GPU in 2.32438s (15.488GiB/s) | 20260508 04:02:55.444 | INFO | Downloaded 34GiB to GPU in 2.55014s (13.3326GiB/s) | 20260508 04:03:00.581 | INFO | Analyzing chunk frames = [5085696,5086208) | 20260508 04:03:00.581 | INFO | MemInfo: { | 20260508 04:03:00.581 | INFO | "gpu" : " 17 GB", | 20260508 04:03:00.581 | INFO | "huge" : "", | 20260508 04:03:00.581 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:00.581 | INFO | "malloc" : " 14 GB", | 20260508 04:03:00.581 | INFO | "pinned" : " 29 GB" | 20260508 04:03:00.581 | INFO | } | 20260508 04:03:00.581 | INFO | Perf report for the last 5.13814 seconds: | 20260508 04:03:00.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:00.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.581 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:03:00.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.581 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 04:03:00.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:00.582 | INFO | Uploaded 36GiB to GPU in 2.39645s (15.0222GiB/s) | 20260508 04:03:00.582 | INFO | Downloaded 34GiB to GPU in 2.43636s (13.9552GiB/s) | 20260508 04:03:05.719 | INFO | Analyzing chunk frames = [5086208,5086720) | 20260508 04:03:05.719 | INFO | MemInfo: { | 20260508 04:03:05.719 | INFO | "gpu" : " 17 GB", | 20260508 04:03:05.719 | INFO | "huge" : "", | 20260508 04:03:05.719 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:05.719 | INFO | "malloc" : " 14 GB", | 20260508 04:03:05.719 | INFO | "pinned" : " 29 GB" | 20260508 04:03:05.719 | INFO | } | 20260508 04:03:05.719 | INFO | Perf report for the last 5.13771 seconds: | 20260508 04:03:05.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:05.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:03:05.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:03:05.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:05.719 | INFO | Uploaded 36GiB to GPU in 2.407s (14.9564GiB/s) | 20260508 04:03:05.719 | INFO | Downloaded 34GiB to GPU in 2.4458s (13.9014GiB/s) | 20260508 04:03:10.857 | INFO | Analyzing chunk frames = [5086720,5087232) | 20260508 04:03:10.857 | INFO | MemInfo: { | 20260508 04:03:10.857 | INFO | "gpu" : " 17 GB", | 20260508 04:03:10.857 | INFO | "huge" : "", | 20260508 04:03:10.857 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:10.857 | INFO | "malloc" : " 14 GB", | 20260508 04:03:10.857 | INFO | "pinned" : " 29 GB" | 20260508 04:03:10.857 | INFO | } | 20260508 04:03:10.857 | INFO | Perf report for the last 5.13836 seconds: | 20260508 04:03:10.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:10.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.857 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 04:03:10.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.858 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 04:03:10.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:10.858 | INFO | Uploaded 36GiB to GPU in 2.41608s (14.9002GiB/s) | 20260508 04:03:10.858 | INFO | Downloaded 34GiB to GPU in 2.43562s (13.9595GiB/s) | 20260508 04:03:10.858 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:03:10.858 | INFO | Total mem usage: 16.7 GiB | 20260508 04:03:10.858 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:03:10.858 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 04:03:10.858 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:03:10.858 | INFO | High Water Breakdown: | 20260508 04:03:10.858 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:03:10.858 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:03:10.858 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:03:10.858 | INFO | ------------------------------------------ | 20260508 04:03:10.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:03:10.858 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:03:10.858 | INFO | Total mem usage: 26.5 GiB | 20260508 04:03:10.858 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:03:10.858 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:03:10.858 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:03:10.858 | INFO | High Water Breakdown: | 20260508 04:03:10.858 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:03:10.858 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:03:10.858 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:03:10.858 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:03:10.858 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:03:10.858 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:03:10.858 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:03:10.858 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:03:10.858 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:03:10.858 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:03:10.858 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:03:10.858 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:03:10.858 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:03:10.858 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:03:10.858 | INFO | ------------------------------------------ | 20260508 04:03:10.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:03:10.858 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:03:10.858 | INFO | Total mem usage: 27.8 GiB | 20260508 04:03:10.858 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:03:10.858 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:03:10.858 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:03:10.858 | INFO | High Water Breakdown: | 20260508 04:03:10.858 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:03:10.858 | INFO | ------------------------------------------ | 20260508 04:03:10.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:03:10.858 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:03:10.858 | INFO | Total mem usage: 15.7 GiB | 20260508 04:03:10.858 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:03:10.858 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:03:10.858 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:03:10.858 | INFO | High Water Breakdown: | 20260508 04:03:10.858 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:03:10.858 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:03:10.858 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:03:10.858 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:03:10.858 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:03:10.859 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:03:10.859 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:03:10.859 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:03:10.859 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:03:10.859 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:03:10.859 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:03:10.859 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:03:10.859 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:03:10.859 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:03:10.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:03:10.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:03:10.859 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:03:10.859 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:03:10.859 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:03:10.859 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:03:10.859 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:03:10.859 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:03:10.859 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:03:10.859 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:03:10.859 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:03:10.859 | INFO | ------------------------------------------ | 20260508 04:03:10.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:03:10.859 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:03:10.859 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:03:10.859 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:03:10.859 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 1.70e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.14e-02 / 5.27e-03 | 20260508 04:03:10.859 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.49e+00, 5.86e-01 / 2.02e-01 | 20260508 04:03:10.859 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.15e-01 | 20260508 04:03:10.859 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.58e+01, 3.87e+00 / 2.05e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.72e+00 / 1.35e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.20e-01 / 2.51e-01 | 20260508 04:03:10.859 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.19e+00, 2.63e-01 / 1.28e-01 | 20260508 04:03:10.859 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.53e-01, 5.07e-02 / 2.53e-02 | 20260508 04:03:10.859 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:03:10.859 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:03:10.859 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:03:10.859 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 24.17%, Upload -- 2.57e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 24.49%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 30.79%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 1.97%, LockUpload -- 2.09e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 2.96%, LockDownload -- 3.14e+00 / 0.00e+00, 4.45e+00 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 04:03:10.859 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260508 04:03:10.954 | INFO | Starting a round of Baz IO | 20260508 04:03:14.388 | INFO | Populating RTMetrics report | 20260508 04:03:14.389 | INFO | RTMetrics report populated | 20260508 04:03:14.389 | INFO | Opening tmp RTMetrics json file | 20260508 04:03:14.403 | INFO | Serializing RTMetrics to json | 20260508 04:03:14.404 | INFO | Serialization done, swapping tmp file now... | 20260508 04:03:14.405 | INFO | RTMetrics done | 20260508 04:03:14.455 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.5 seconds | 20260508 04:03:14.455 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.15% / 0.00% / 58.85% | 20260508 04:03:14.455 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.566645% / 7.94729e-06% / 5.5631e-05% | 20260508 04:03:14.455 | INFO | Baz IO overall: | 20260508 04:03:14.455 | INFO | BytesWritten : 2281595847402 | 20260508 04:03:14.455 | INFO | NumEvents : 1101856021976 | 20260508 04:03:15.995 | INFO | Analyzing chunk frames = [5087232,5087744) | 20260508 04:03:15.995 | INFO | MemInfo: { | 20260508 04:03:15.995 | INFO | "gpu" : " 17 GB", | 20260508 04:03:15.995 | INFO | "huge" : "", | 20260508 04:03:15.995 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:15.995 | INFO | "malloc" : " 14 GB", | 20260508 04:03:15.995 | INFO | "pinned" : " 29 GB" | 20260508 04:03:15.995 | INFO | } | 20260508 04:03:15.995 | INFO | Perf report for the last 5.13749 seconds: | 20260508 04:03:15.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:15.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:15.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:15.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:15.995 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 04:03:15.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:15.995 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 04:03:15.995 | INFO | PRE_HQ : 65.7%, 1.6, 55.9ms, 0 | 20260508 04:03:15.995 | INFO | BAZWRITER : 67.7%, 2.7, 92.6ms, 0 | 20260508 04:03:15.995 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 04:03:15.996 | INFO | Uploaded 36GiB to GPU in 2.55954s (14.065GiB/s) | 20260508 04:03:15.996 | INFO | Downloaded 38GiB to GPU in 2.69684s (14.0906GiB/s) | 20260508 04:03:21.137 | INFO | Analyzing chunk frames = [5087744,5088256) | 20260508 04:03:21.138 | INFO | MemInfo: { | 20260508 04:03:21.138 | INFO | "gpu" : " 17 GB", | 20260508 04:03:21.138 | INFO | "huge" : "", | 20260508 04:03:21.138 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:21.138 | INFO | "malloc" : " 14 GB", | 20260508 04:03:21.138 | INFO | "pinned" : " 29 GB" | 20260508 04:03:21.138 | INFO | } | 20260508 04:03:21.138 | INFO | Perf report for the last 5.14291 seconds: | 20260508 04:03:21.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:21.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 04:03:21.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:03:21.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:21.138 | INFO | Uploaded 36GiB to GPU in 2.5115s (14.3341GiB/s) | 20260508 04:03:21.138 | INFO | Downloaded 34GiB to GPU in 2.43762s (13.948GiB/s) | 20260508 04:03:26.269 | INFO | Analyzing chunk frames = [5088256,5088768) | 20260508 04:03:26.269 | INFO | MemInfo: { | 20260508 04:03:26.269 | INFO | "gpu" : " 17 GB", | 20260508 04:03:26.269 | INFO | "huge" : "", | 20260508 04:03:26.269 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:26.269 | INFO | "malloc" : " 14 GB", | 20260508 04:03:26.269 | INFO | "pinned" : " 29 GB" | 20260508 04:03:26.269 | INFO | } | 20260508 04:03:26.269 | INFO | Perf report for the last 5.13109 seconds: | 20260508 04:03:26.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:26.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.269 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 04:03:26.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.269 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:03:26.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:26.270 | INFO | Uploaded 36GiB to GPU in 2.5107s (14.3386GiB/s) | 20260508 04:03:26.270 | INFO | Downloaded 34GiB to GPU in 2.42839s (14.0011GiB/s) | 20260508 04:03:31.414 | INFO | Analyzing chunk frames = [5088768,5089280) | 20260508 04:03:31.415 | INFO | MemInfo: { | 20260508 04:03:31.415 | INFO | "gpu" : " 17 GB", | 20260508 04:03:31.415 | INFO | "huge" : "", | 20260508 04:03:31.415 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:31.415 | INFO | "malloc" : " 14 GB", | 20260508 04:03:31.415 | INFO | "pinned" : " 29 GB" | 20260508 04:03:31.415 | INFO | } | 20260508 04:03:31.415 | INFO | Perf report for the last 5.14588 seconds: | 20260508 04:03:31.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:31.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 04:03:31.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:03:31.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:31.415 | INFO | Uploaded 36GiB to GPU in 2.50164s (14.3906GiB/s) | 20260508 04:03:31.415 | INFO | Downloaded 34GiB to GPU in 2.51611s (13.5129GiB/s) | 20260508 04:03:36.555 | INFO | Analyzing chunk frames = [5089280,5089792) | 20260508 04:03:36.555 | INFO | MemInfo: { | 20260508 04:03:36.555 | INFO | "gpu" : " 17 GB", | 20260508 04:03:36.555 | INFO | "huge" : "", | 20260508 04:03:36.555 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:36.555 | INFO | "malloc" : " 14 GB", | 20260508 04:03:36.555 | INFO | "pinned" : " 29 GB" | 20260508 04:03:36.555 | INFO | } | 20260508 04:03:36.556 | INFO | Perf report for the last 5.14075 seconds: | 20260508 04:03:36.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:36.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:03:36.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:03:36.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:36.556 | INFO | Uploaded 36GiB to GPU in 2.3348s (15.4189GiB/s) | 20260508 04:03:36.556 | INFO | Downloaded 34GiB to GPU in 2.55386s (13.3132GiB/s) | 20260508 04:03:41.695 | INFO | Analyzing chunk frames = [5089792,5090304) | 20260508 04:03:41.695 | INFO | MemInfo: { | 20260508 04:03:41.695 | INFO | "gpu" : " 17 GB", | 20260508 04:03:41.695 | INFO | "huge" : "", | 20260508 04:03:41.695 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:41.695 | INFO | "malloc" : " 14 GB", | 20260508 04:03:41.695 | INFO | "pinned" : " 29 GB" | 20260508 04:03:41.695 | INFO | } | 20260508 04:03:41.695 | INFO | Perf report for the last 5.13957 seconds: | 20260508 04:03:41.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:41.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.695 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:03:41.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.695 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:03:41.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:41.696 | INFO | Uploaded 36GiB to GPU in 2.41759s (14.8909GiB/s) | 20260508 04:03:41.696 | INFO | Downloaded 34GiB to GPU in 2.42658s (14.0115GiB/s) | 20260508 04:03:46.832 | INFO | Analyzing chunk frames = [5090304,5090816) | 20260508 04:03:46.832 | INFO | MemInfo: { | 20260508 04:03:46.832 | INFO | "gpu" : " 17 GB", | 20260508 04:03:46.832 | INFO | "huge" : "", | 20260508 04:03:46.832 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:46.832 | INFO | "malloc" : " 14 GB", | 20260508 04:03:46.832 | INFO | "pinned" : " 29 GB" | 20260508 04:03:46.832 | INFO | } | 20260508 04:03:46.832 | INFO | Perf report for the last 5.13702 seconds: | 20260508 04:03:46.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:46.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.832 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:03:46.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.832 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:03:46.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:46.833 | INFO | Uploaded 36GiB to GPU in 2.41924s (14.8807GiB/s) | 20260508 04:03:46.833 | INFO | Downloaded 34GiB to GPU in 2.43157s (13.9827GiB/s) | 20260508 04:03:51.969 | INFO | Analyzing chunk frames = [5090816,5091328) | 20260508 04:03:51.969 | INFO | MemInfo: { | 20260508 04:03:51.969 | INFO | "gpu" : " 17 GB", | 20260508 04:03:51.969 | INFO | "huge" : "", | 20260508 04:03:51.969 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:51.969 | INFO | "malloc" : " 14 GB", | 20260508 04:03:51.969 | INFO | "pinned" : " 29 GB" | 20260508 04:03:51.969 | INFO | } | 20260508 04:03:51.969 | INFO | Perf report for the last 5.13673 seconds: | 20260508 04:03:51.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:51.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.969 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:03:51.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.969 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:03:51.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:51.970 | INFO | Uploaded 36GiB to GPU in 2.45088s (14.6886GiB/s) | 20260508 04:03:51.970 | INFO | Downloaded 34GiB to GPU in 2.42817s (14.0023GiB/s) | 20260508 04:03:52.066 | INFO | Starting a round of Baz IO | 20260508 04:03:55.509 | INFO | Populating RTMetrics report | 20260508 04:03:55.509 | INFO | RTMetrics report populated | 20260508 04:03:55.509 | INFO | Opening tmp RTMetrics json file | 20260508 04:03:55.520 | INFO | Serializing RTMetrics to json | 20260508 04:03:55.521 | INFO | Serialization done, swapping tmp file now... | 20260508 04:03:55.522 | INFO | RTMetrics done | 20260508 04:03:55.579 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.512 seconds | 20260508 04:03:55.579 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.14% / 0.00% / 58.85% | 20260508 04:03:55.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.564599% / 3.97364e-06% / 5.96046e-05% | 20260508 04:03:55.579 | INFO | Baz IO overall: | 20260508 04:03:55.579 | INFO | BytesWritten : 2283315465687 | 20260508 04:03:55.579 | INFO | NumEvents : 1102677615120 | 20260508 04:03:57.112 | INFO | Analyzing chunk frames = [5091328,5091840) | 20260508 04:03:57.112 | INFO | MemInfo: { | 20260508 04:03:57.112 | INFO | "gpu" : " 17 GB", | 20260508 04:03:57.112 | INFO | "huge" : "", | 20260508 04:03:57.112 | INFO | "huge_shared" : " 29 GB", | 20260508 04:03:57.112 | INFO | "malloc" : " 14 GB", | 20260508 04:03:57.112 | INFO | "pinned" : " 29 GB" | 20260508 04:03:57.112 | INFO | } | 20260508 04:03:57.112 | INFO | Perf report for the last 5.14345 seconds: | 20260508 04:03:57.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:03:57.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:57.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:57.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:57.112 | INFO | ANALYSIS : 67.9%, 3.0, 106.3ms, 0 | 20260508 04:03:57.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:03:57.112 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:03:57.113 | INFO | PRE_HQ : 65.3%, 1.6, 55.5ms, 0 | 20260508 04:03:57.113 | INFO | BAZWRITER : 67.7%, 2.7, 92.9ms, 0 | 20260508 04:03:57.113 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 04:03:57.113 | INFO | Uploaded 36GiB to GPU in 2.57181s (13.9979GiB/s) | 20260508 04:03:57.113 | INFO | Downloaded 38GiB to GPU in 2.69563s (14.0969GiB/s) | 20260508 04:04:02.252 | INFO | Analyzing chunk frames = [5091840,5092352) | 20260508 04:04:02.252 | INFO | MemInfo: { | 20260508 04:04:02.252 | INFO | "gpu" : " 17 GB", | 20260508 04:04:02.252 | INFO | "huge" : "", | 20260508 04:04:02.252 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:02.252 | INFO | "malloc" : " 14 GB", | 20260508 04:04:02.252 | INFO | "pinned" : " 29 GB" | 20260508 04:04:02.252 | INFO | } | 20260508 04:04:02.253 | INFO | Perf report for the last 5.14016 seconds: | 20260508 04:04:02.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:02.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 04:04:02.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:04:02.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:02.253 | INFO | Uploaded 36GiB to GPU in 2.57752s (13.9669GiB/s) | 20260508 04:04:02.253 | INFO | Downloaded 34GiB to GPU in 2.4389s (13.9407GiB/s) | 20260508 04:04:07.390 | INFO | Analyzing chunk frames = [5092352,5092864) | 20260508 04:04:07.390 | INFO | MemInfo: { | 20260508 04:04:07.390 | INFO | "gpu" : " 17 GB", | 20260508 04:04:07.390 | INFO | "huge" : "", | 20260508 04:04:07.390 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:07.390 | INFO | "malloc" : " 14 GB", | 20260508 04:04:07.390 | INFO | "pinned" : " 29 GB" | 20260508 04:04:07.390 | INFO | } | 20260508 04:04:07.390 | INFO | Perf report for the last 5.13728 seconds: | 20260508 04:04:07.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:07.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 04:04:07.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 04:04:07.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:07.390 | INFO | Uploaded 36GiB to GPU in 2.53529s (14.1995GiB/s) | 20260508 04:04:07.390 | INFO | Downloaded 34GiB to GPU in 2.42097s (14.0439GiB/s) | 20260508 04:04:12.526 | INFO | Analyzing chunk frames = [5092864,5093376) | 20260508 04:04:12.526 | INFO | MemInfo: { | 20260508 04:04:12.526 | INFO | "gpu" : " 17 GB", | 20260508 04:04:12.526 | INFO | "huge" : "", | 20260508 04:04:12.526 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:12.526 | INFO | "malloc" : " 14 GB", | 20260508 04:04:12.526 | INFO | "pinned" : " 29 GB" | 20260508 04:04:12.526 | INFO | } | 20260508 04:04:12.526 | INFO | Perf report for the last 5.1361 seconds: | 20260508 04:04:12.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:12.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 04:04:12.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:04:12.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:12.526 | INFO | Uploaded 36GiB to GPU in 2.53383s (14.2077GiB/s) | 20260508 04:04:12.526 | INFO | Downloaded 34GiB to GPU in 2.50653s (13.5646GiB/s) | 20260508 04:04:17.665 | INFO | Analyzing chunk frames = [5093376,5093888) | 20260508 04:04:17.665 | INFO | MemInfo: { | 20260508 04:04:17.665 | INFO | "gpu" : " 17 GB", | 20260508 04:04:17.665 | INFO | "huge" : "", | 20260508 04:04:17.666 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:17.666 | INFO | "malloc" : " 14 GB", | 20260508 04:04:17.666 | INFO | "pinned" : " 29 GB" | 20260508 04:04:17.666 | INFO | } | 20260508 04:04:17.666 | INFO | Perf report for the last 5.13966 seconds: | 20260508 04:04:17.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:17.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 04:04:17.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:04:17.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:17.666 | INFO | Uploaded 36GiB to GPU in 2.38931s (15.0671GiB/s) | 20260508 04:04:17.666 | INFO | Downloaded 34GiB to GPU in 2.56323s (13.2645GiB/s) | 20260508 04:04:22.811 | INFO | Analyzing chunk frames = [5093888,5094400) | 20260508 04:04:22.811 | INFO | MemInfo: { | 20260508 04:04:22.811 | INFO | "gpu" : " 17 GB", | 20260508 04:04:22.811 | INFO | "huge" : "", | 20260508 04:04:22.811 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:22.811 | INFO | "malloc" : " 14 GB", | 20260508 04:04:22.811 | INFO | "pinned" : " 29 GB" | 20260508 04:04:22.811 | INFO | } | 20260508 04:04:22.812 | INFO | Perf report for the last 5.14591 seconds: | 20260508 04:04:22.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:22.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 04:04:22.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:04:22.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:22.812 | INFO | Uploaded 36GiB to GPU in 2.42013s (14.8752GiB/s) | 20260508 04:04:22.812 | INFO | Downloaded 34GiB to GPU in 2.42455s (14.0232GiB/s) | 20260508 04:04:27.944 | INFO | Analyzing chunk frames = [5094400,5094912) | 20260508 04:04:27.944 | INFO | MemInfo: { | 20260508 04:04:27.944 | INFO | "gpu" : " 17 GB", | 20260508 04:04:27.944 | INFO | "huge" : "", | 20260508 04:04:27.945 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:27.945 | INFO | "malloc" : " 14 GB", | 20260508 04:04:27.945 | INFO | "pinned" : " 29 GB" | 20260508 04:04:27.945 | INFO | } | 20260508 04:04:27.945 | INFO | Perf report for the last 5.13318 seconds: | 20260508 04:04:27.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:27.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 04:04:27.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:04:27.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:27.945 | INFO | Uploaded 36GiB to GPU in 2.46219s (14.6212GiB/s) | 20260508 04:04:27.945 | INFO | Downloaded 34GiB to GPU in 2.43926s (13.9387GiB/s) | 20260508 04:04:33.083 | INFO | Analyzing chunk frames = [5094912,5095424) | 20260508 04:04:33.083 | INFO | MemInfo: { | 20260508 04:04:33.083 | INFO | "gpu" : " 17 GB", | 20260508 04:04:33.083 | INFO | "huge" : "", | 20260508 04:04:33.083 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:33.083 | INFO | "malloc" : " 14 GB", | 20260508 04:04:33.083 | INFO | "pinned" : " 29 GB" | 20260508 04:04:33.084 | INFO | } | 20260508 04:04:33.084 | INFO | Perf report for the last 5.13879 seconds: | 20260508 04:04:33.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:33.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:04:33.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:04:33.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:33.084 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:04:33.084 | INFO | Total mem usage: 16.7 GiB | 20260508 04:04:33.084 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:04:33.084 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 04:04:33.084 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:04:33.084 | INFO | High Water Breakdown: | 20260508 04:04:33.084 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:04:33.084 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:04:33.084 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:04:33.084 | INFO | ------------------------------------------ | 20260508 04:04:33.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:04:33.084 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:04:33.084 | INFO | Total mem usage: 26.5 GiB | 20260508 04:04:33.084 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:04:33.084 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:04:33.084 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:04:33.084 | INFO | High Water Breakdown: | 20260508 04:04:33.084 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:04:33.084 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:04:33.084 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:04:33.084 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:04:33.084 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:04:33.084 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:04:33.084 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:04:33.084 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:04:33.084 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:04:33.084 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:04:33.084 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:04:33.084 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:04:33.084 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:04:33.084 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:04:33.084 | INFO | ------------------------------------------ | 20260508 04:04:33.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:04:33.084 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:04:33.084 | INFO | Total mem usage: 27.8 GiB | 20260508 04:04:33.084 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:04:33.084 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:04:33.084 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:04:33.084 | INFO | High Water Breakdown: | 20260508 04:04:33.084 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:04:33.084 | INFO | ------------------------------------------ | 20260508 04:04:33.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:04:33.084 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:04:33.084 | INFO | Total mem usage: 15.7 GiB | 20260508 04:04:33.084 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:04:33.084 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:04:33.084 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:04:33.084 | INFO | High Water Breakdown: | 20260508 04:04:33.084 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:04:33.084 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:04:33.084 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:04:33.084 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:04:33.084 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:04:33.084 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:04:33.084 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:04:33.084 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:04:33.084 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:04:33.084 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:04:33.084 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:04:33.084 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:04:33.084 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:04:33.084 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:04:33.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:04:33.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:04:33.084 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:04:33.084 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:04:33.084 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:04:33.084 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:04:33.084 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:04:33.084 | INFO | Uploaded 36GiB to GPU in 2.45198s (14.682GiB/s) | 20260508 04:04:33.084 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:04:33.084 | INFO | Downloaded 34GiB to GPU in 2.43596s (13.9576GiB/s) | 20260508 04:04:33.084 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:04:33.084 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:04:33.084 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:04:33.084 | INFO | ------------------------------------------ | 20260508 04:04:33.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:04:33.084 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:04:33.084 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:04:33.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:04:33.084 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.47e+00 | 20260508 04:04:33.084 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.02e-01, 1.11e-02 / 4.62e-03 | 20260508 04:04:33.084 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.52e+00, 5.90e-01 / 1.50e-01 | 20260508 04:04:33.084 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.09e-01 / 8.76e-02 | 20260508 04:04:33.084 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.55e+01, 3.79e+00 / 1.93e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.76e+00 / 1.20e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.15e-01 / 1.79e-01 | 20260508 04:04:33.085 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.22e+00, 2.62e-01 / 1.42e-01 | 20260508 04:04:33.085 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.62e-01, 5.13e-02 / 2.71e-02 | 20260508 04:04:33.085 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:04:33.085 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:04:33.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:04:33.085 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 24.27%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 24.40%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 2.08%, LockUpload -- 2.21e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 2.81%, LockDownload -- 2.99e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:04:33.085 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260508 04:04:33.181 | INFO | Starting a round of Baz IO | 20260508 04:04:36.610 | INFO | Populating RTMetrics report | 20260508 04:04:36.611 | INFO | RTMetrics report populated | 20260508 04:04:36.611 | INFO | Opening tmp RTMetrics json file | 20260508 04:04:36.625 | INFO | Serializing RTMetrics to json | 20260508 04:04:36.626 | INFO | Serialization done, swapping tmp file now... | 20260508 04:04:36.628 | INFO | RTMetrics done | 20260508 04:04:36.685 | INFO | Finished a round of Baz IO: Wrote 1.601 GiB in 3.504 seconds | 20260508 04:04:36.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.14% / 0.00% / 58.86% | 20260508 04:04:36.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.56469% / 0% / 7.94729e-05% | 20260508 04:04:36.685 | INFO | Baz IO overall: | 20260508 04:04:36.685 | INFO | BytesWritten : 2285035036710 | 20260508 04:04:36.685 | INFO | NumEvents : 1103499263530 | 20260508 04:04:38.224 | INFO | Analyzing chunk frames = [5095424,5095936) | 20260508 04:04:38.224 | INFO | MemInfo: { | 20260508 04:04:38.224 | INFO | "gpu" : " 17 GB", | 20260508 04:04:38.224 | INFO | "huge" : "", | 20260508 04:04:38.224 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:38.224 | INFO | "malloc" : " 14 GB", | 20260508 04:04:38.224 | INFO | "pinned" : " 29 GB" | 20260508 04:04:38.224 | INFO | } | 20260508 04:04:38.224 | INFO | Perf report for the last 5.14004 seconds: | 20260508 04:04:38.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:38.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:38.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:38.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:38.224 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 04:04:38.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:38.224 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 04:04:38.224 | INFO | PRE_HQ : 65.6%, 1.6, 55.8ms, 0 | 20260508 04:04:38.224 | INFO | BAZWRITER : 67.7%, 2.7, 93.3ms, 0 | 20260508 04:04:38.224 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:04:38.224 | INFO | Uploaded 36GiB to GPU in 2.55873s (14.0695GiB/s) | 20260508 04:04:38.224 | INFO | Downloaded 38GiB to GPU in 2.69689s (14.0903GiB/s) | 20260508 04:04:43.367 | INFO | Analyzing chunk frames = [5095936,5096448) | 20260508 04:04:43.367 | INFO | MemInfo: { | 20260508 04:04:43.367 | INFO | "gpu" : " 17 GB", | 20260508 04:04:43.367 | INFO | "huge" : "", | 20260508 04:04:43.367 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:43.367 | INFO | "malloc" : " 14 GB", | 20260508 04:04:43.367 | INFO | "pinned" : " 29 GB" | 20260508 04:04:43.367 | INFO | } | 20260508 04:04:43.367 | INFO | Perf report for the last 5.14369 seconds: | 20260508 04:04:43.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:43.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.367 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 04:04:43.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.367 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:04:43.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:43.368 | INFO | Uploaded 36GiB to GPU in 2.59407s (13.8778GiB/s) | 20260508 04:04:43.368 | INFO | Downloaded 34GiB to GPU in 2.41975s (14.0511GiB/s) | 20260508 04:04:48.507 | INFO | Analyzing chunk frames = [5096448,5096960) | 20260508 04:04:48.507 | INFO | MemInfo: { | 20260508 04:04:48.507 | INFO | "gpu" : " 17 GB", | 20260508 04:04:48.507 | INFO | "huge" : "", | 20260508 04:04:48.507 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:48.507 | INFO | "malloc" : " 14 GB", | 20260508 04:04:48.507 | INFO | "pinned" : " 29 GB" | 20260508 04:04:48.507 | INFO | } | 20260508 04:04:48.507 | INFO | Perf report for the last 5.1398 seconds: | 20260508 04:04:48.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:48.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260508 04:04:48.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:04:48.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:48.507 | INFO | Uploaded 36GiB to GPU in 2.56426s (14.0391GiB/s) | 20260508 04:04:48.507 | INFO | Downloaded 34GiB to GPU in 2.4203s (14.0478GiB/s) | 20260508 04:04:53.645 | INFO | Analyzing chunk frames = [5096960,5097472) | 20260508 04:04:53.645 | INFO | MemInfo: { | 20260508 04:04:53.645 | INFO | "gpu" : " 17 GB", | 20260508 04:04:53.645 | INFO | "huge" : "", | 20260508 04:04:53.645 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:53.645 | INFO | "malloc" : " 14 GB", | 20260508 04:04:53.645 | INFO | "pinned" : " 29 GB" | 20260508 04:04:53.645 | INFO | } | 20260508 04:04:53.645 | INFO | Perf report for the last 5.1379 seconds: | 20260508 04:04:53.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:53.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 04:04:53.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:04:53.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:53.645 | INFO | Uploaded 36GiB to GPU in 2.57441s (13.9838GiB/s) | 20260508 04:04:53.645 | INFO | Downloaded 34GiB to GPU in 2.4994s (13.6033GiB/s) | 20260508 04:04:58.780 | INFO | Analyzing chunk frames = [5097472,5097984) | 20260508 04:04:58.780 | INFO | MemInfo: { | 20260508 04:04:58.780 | INFO | "gpu" : " 17 GB", | 20260508 04:04:58.780 | INFO | "huge" : "", | 20260508 04:04:58.780 | INFO | "huge_shared" : " 29 GB", | 20260508 04:04:58.780 | INFO | "malloc" : " 14 GB", | 20260508 04:04:58.780 | INFO | "pinned" : " 29 GB" | 20260508 04:04:58.780 | INFO | } | 20260508 04:04:58.780 | INFO | Perf report for the last 5.13523 seconds: | 20260508 04:04:58.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:04:58.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.780 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:04:58.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.780 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 04:04:58.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:04:58.781 | INFO | Uploaded 36GiB to GPU in 2.40036s (14.9977GiB/s) | 20260508 04:04:58.781 | INFO | Downloaded 34GiB to GPU in 2.5476s (13.3459GiB/s) | 20260508 04:05:03.923 | INFO | Analyzing chunk frames = [5097984,5098496) | 20260508 04:05:03.923 | INFO | MemInfo: { | 20260508 04:05:03.923 | INFO | "gpu" : " 17 GB", | 20260508 04:05:03.923 | INFO | "huge" : "", | 20260508 04:05:03.923 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:03.923 | INFO | "malloc" : " 14 GB", | 20260508 04:05:03.923 | INFO | "pinned" : " 29 GB" | 20260508 04:05:03.923 | INFO | } | 20260508 04:05:03.923 | INFO | Perf report for the last 5.14315 seconds: | 20260508 04:05:03.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:03.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.924 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:05:03.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.924 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:05:03.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:03.924 | INFO | Uploaded 36GiB to GPU in 2.42152s (14.8667GiB/s) | 20260508 04:05:03.924 | INFO | Downloaded 34GiB to GPU in 2.42071s (14.0455GiB/s) | 20260508 04:05:09.063 | INFO | Analyzing chunk frames = [5098496,5099008) | 20260508 04:05:09.063 | INFO | MemInfo: { | 20260508 04:05:09.063 | INFO | "gpu" : " 17 GB", | 20260508 04:05:09.063 | INFO | "huge" : "", | 20260508 04:05:09.063 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:09.063 | INFO | "malloc" : " 14 GB", | 20260508 04:05:09.063 | INFO | "pinned" : " 29 GB" | 20260508 04:05:09.064 | INFO | } | 20260508 04:05:09.064 | INFO | Perf report for the last 5.14007 seconds: | 20260508 04:05:09.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:09.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:05:09.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:05:09.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:09.064 | INFO | Uploaded 36GiB to GPU in 2.42032s (14.8741GiB/s) | 20260508 04:05:09.064 | INFO | Downloaded 34GiB to GPU in 2.41923s (14.054GiB/s) | 20260508 04:05:14.199 | INFO | Analyzing chunk frames = [5099008,5099520) | 20260508 04:05:14.199 | INFO | MemInfo: { | 20260508 04:05:14.199 | INFO | "gpu" : " 17 GB", | 20260508 04:05:14.199 | INFO | "huge" : "", | 20260508 04:05:14.200 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:14.200 | INFO | "malloc" : " 14 GB", | 20260508 04:05:14.200 | INFO | "pinned" : " 29 GB" | 20260508 04:05:14.200 | INFO | } | 20260508 04:05:14.200 | INFO | Perf report for the last 5.13625 seconds: | 20260508 04:05:14.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:14.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:05:14.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260508 04:05:14.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:14.200 | INFO | Uploaded 36GiB to GPU in 2.4464s (14.7155GiB/s) | 20260508 04:05:14.201 | INFO | Downloaded 34GiB to GPU in 2.4312s (13.9849GiB/s) | 20260508 04:05:14.313 | INFO | Starting a round of Baz IO | 20260508 04:05:17.736 | INFO | Populating RTMetrics report | 20260508 04:05:17.737 | INFO | RTMetrics report populated | 20260508 04:05:17.737 | INFO | Opening tmp RTMetrics json file | 20260508 04:05:17.747 | INFO | Serializing RTMetrics to json | 20260508 04:05:17.748 | INFO | Serialization done, swapping tmp file now... | 20260508 04:05:17.749 | INFO | RTMetrics done | 20260508 04:05:17.806 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.492 seconds | 20260508 04:05:17.806 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.14% / 0.00% / 58.86% | 20260508 04:05:17.806 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.562874% / 3.97364e-06% / 5.96046e-05% | 20260508 04:05:17.806 | INFO | Baz IO overall: | 20260508 04:05:17.806 | INFO | BytesWritten : 2286754670698 | 20260508 04:05:17.806 | INFO | NumEvents : 1104320850454 | 20260508 04:05:19.338 | INFO | Analyzing chunk frames = [5099520,5100032) | 20260508 04:05:19.338 | INFO | MemInfo: { | 20260508 04:05:19.338 | INFO | "gpu" : " 17 GB", | 20260508 04:05:19.338 | INFO | "huge" : "", | 20260508 04:05:19.338 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:19.338 | INFO | "malloc" : " 14 GB", | 20260508 04:05:19.338 | INFO | "pinned" : " 29 GB" | 20260508 04:05:19.338 | INFO | } | 20260508 04:05:19.338 | INFO | Perf report for the last 5.13822 seconds: | 20260508 04:05:19.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:19.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:19.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:19.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:19.338 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 04:05:19.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:19.338 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 04:05:19.338 | INFO | PRE_HQ : 66.5%, 1.8, 63.1ms, 0 | 20260508 04:05:19.338 | INFO | BAZWRITER : 67.2%, 2.9, 100.2ms, 0 | 20260508 04:05:19.338 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:05:19.338 | INFO | Uploaded 36GiB to GPU in 2.5707s (14.004GiB/s) | 20260508 04:05:19.338 | INFO | Downloaded 38GiB to GPU in 2.7054s (14.046GiB/s) | 20260508 04:05:24.480 | INFO | Analyzing chunk frames = [5100032,5100544) | 20260508 04:05:24.480 | INFO | MemInfo: { | 20260508 04:05:24.480 | INFO | "gpu" : " 17 GB", | 20260508 04:05:24.480 | INFO | "huge" : "", | 20260508 04:05:24.480 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:24.480 | INFO | "malloc" : " 14 GB", | 20260508 04:05:24.480 | INFO | "pinned" : " 29 GB" | 20260508 04:05:24.480 | INFO | } | 20260508 04:05:24.480 | INFO | Perf report for the last 5.14182 seconds: | 20260508 04:05:24.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:24.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:05:24.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:05:24.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:24.480 | INFO | Uploaded 36GiB to GPU in 2.57469s (13.9823GiB/s) | 20260508 04:05:24.480 | INFO | Downloaded 34GiB to GPU in 2.44534s (13.904GiB/s) | 20260508 04:05:29.621 | INFO | Analyzing chunk frames = [5100544,5101056) | 20260508 04:05:29.621 | INFO | MemInfo: { | 20260508 04:05:29.621 | INFO | "gpu" : " 17 GB", | 20260508 04:05:29.621 | INFO | "huge" : "", | 20260508 04:05:29.621 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:29.621 | INFO | "malloc" : " 14 GB", | 20260508 04:05:29.621 | INFO | "pinned" : " 29 GB" | 20260508 04:05:29.621 | INFO | } | 20260508 04:05:29.621 | INFO | Perf report for the last 5.14141 seconds: | 20260508 04:05:29.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:29.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.621 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 04:05:29.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.621 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:05:29.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:29.622 | INFO | Uploaded 36GiB to GPU in 2.54255s (14.159GiB/s) | 20260508 04:05:29.622 | INFO | Downloaded 34GiB to GPU in 2.43065s (13.988GiB/s) | 20260508 04:05:34.761 | INFO | Analyzing chunk frames = [5101056,5101568) | 20260508 04:05:34.761 | INFO | MemInfo: { | 20260508 04:05:34.761 | INFO | "gpu" : " 17 GB", | 20260508 04:05:34.761 | INFO | "huge" : "", | 20260508 04:05:34.761 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:34.762 | INFO | "malloc" : " 14 GB", | 20260508 04:05:34.762 | INFO | "pinned" : " 29 GB" | 20260508 04:05:34.762 | INFO | } | 20260508 04:05:34.762 | INFO | Perf report for the last 5.1403 seconds: | 20260508 04:05:34.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:34.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 04:05:34.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:05:34.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:34.762 | INFO | Uploaded 36GiB to GPU in 2.58161s (13.9448GiB/s) | 20260508 04:05:34.762 | INFO | Downloaded 34GiB to GPU in 2.42115s (14.0429GiB/s) | 20260508 04:05:39.897 | INFO | Analyzing chunk frames = [5101568,5102080) | 20260508 04:05:39.897 | INFO | MemInfo: { | 20260508 04:05:39.897 | INFO | "gpu" : " 17 GB", | 20260508 04:05:39.897 | INFO | "huge" : "", | 20260508 04:05:39.897 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:39.897 | INFO | "malloc" : " 14 GB", | 20260508 04:05:39.897 | INFO | "pinned" : " 29 GB" | 20260508 04:05:39.897 | INFO | } | 20260508 04:05:39.898 | INFO | Perf report for the last 5.13591 seconds: | 20260508 04:05:39.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:39.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 04:05:39.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 04:05:39.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:39.898 | INFO | Uploaded 36GiB to GPU in 2.43004s (14.8146GiB/s) | 20260508 04:05:39.898 | INFO | Downloaded 34GiB to GPU in 2.58431s (13.1563GiB/s) | 20260508 04:05:45.032 | INFO | Analyzing chunk frames = [5102080,5102592) | 20260508 04:05:45.032 | INFO | MemInfo: { | 20260508 04:05:45.032 | INFO | "gpu" : " 17 GB", | 20260508 04:05:45.032 | INFO | "huge" : "", | 20260508 04:05:45.032 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:45.033 | INFO | "malloc" : " 14 GB", | 20260508 04:05:45.033 | INFO | "pinned" : " 29 GB" | 20260508 04:05:45.033 | INFO | } | 20260508 04:05:45.033 | INFO | Perf report for the last 5.13512 seconds: | 20260508 04:05:45.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:45.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:05:45.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 04:05:45.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:45.033 | INFO | Uploaded 36GiB to GPU in 2.44465s (14.726GiB/s) | 20260508 04:05:45.033 | INFO | Downloaded 34GiB to GPU in 2.41226s (14.0946GiB/s) | 20260508 04:05:50.176 | INFO | Analyzing chunk frames = [5102592,5103104) | 20260508 04:05:50.176 | INFO | MemInfo: { | 20260508 04:05:50.176 | INFO | "gpu" : " 17 GB", | 20260508 04:05:50.176 | INFO | "huge" : "", | 20260508 04:05:50.177 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:50.177 | INFO | "malloc" : " 14 GB", | 20260508 04:05:50.177 | INFO | "pinned" : " 29 GB" | 20260508 04:05:50.177 | INFO | } | 20260508 04:05:50.177 | INFO | Perf report for the last 5.1441 seconds: | 20260508 04:05:50.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:50.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:05:50.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:05:50.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:50.177 | INFO | Uploaded 36GiB to GPU in 2.48999s (14.4579GiB/s) | 20260508 04:05:50.177 | INFO | Downloaded 34GiB to GPU in 2.42121s (14.0425GiB/s) | 20260508 04:05:55.315 | INFO | Analyzing chunk frames = [5103104,5103616) | 20260508 04:05:55.315 | INFO | MemInfo: { | 20260508 04:05:55.315 | INFO | "gpu" : " 17 GB", | 20260508 04:05:55.315 | INFO | "huge" : "", | 20260508 04:05:55.315 | INFO | "huge_shared" : " 29 GB", | 20260508 04:05:55.316 | INFO | "malloc" : " 14 GB", | 20260508 04:05:55.316 | INFO | "pinned" : " 29 GB" | 20260508 04:05:55.316 | INFO | } | 20260508 04:05:55.316 | INFO | Perf report for the last 5.13885 seconds: | 20260508 04:05:55.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:05:55.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:05:55.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 04:05:55.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:05:55.316 | INFO | Uploaded 36GiB to GPU in 2.47009s (14.5744GiB/s) | 20260508 04:05:55.316 | INFO | Downloaded 34GiB to GPU in 2.43212s (13.9796GiB/s) | 20260508 04:05:55.316 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:05:55.316 | INFO | Total mem usage: 16.7 GiB | 20260508 04:05:55.316 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:05:55.316 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 04:05:55.316 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:05:55.316 | INFO | High Water Breakdown: | 20260508 04:05:55.316 | INFO | 59.5% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:05:55.316 | INFO | 40.3% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:05:55.316 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:05:55.316 | INFO | ------------------------------------------ | 20260508 04:05:55.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:05:55.316 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:05:55.316 | INFO | Total mem usage: 26.5 GiB | 20260508 04:05:55.317 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:05:55.317 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:05:55.317 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:05:55.317 | INFO | High Water Breakdown: | 20260508 04:05:55.317 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:05:55.317 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:05:55.317 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:05:55.317 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:05:55.317 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:05:55.317 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:05:55.317 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:05:55.317 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:05:55.317 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:05:55.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:05:55.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:05:55.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:05:55.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:05:55.317 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:05:55.317 | INFO | ------------------------------------------ | 20260508 04:05:55.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:05:55.317 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:05:55.317 | INFO | Total mem usage: 27.8 GiB | 20260508 04:05:55.317 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:05:55.317 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:05:55.317 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:05:55.317 | INFO | High Water Breakdown: | 20260508 04:05:55.317 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:05:55.317 | INFO | ------------------------------------------ | 20260508 04:05:55.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:05:55.317 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:05:55.317 | INFO | Total mem usage: 15.7 GiB | 20260508 04:05:55.317 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:05:55.317 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:05:55.317 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:05:55.317 | INFO | High Water Breakdown: | 20260508 04:05:55.317 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:05:55.317 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:05:55.317 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:05:55.317 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:05:55.317 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:05:55.317 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:05:55.317 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:05:55.317 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:05:55.317 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:05:55.317 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:05:55.317 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:05:55.317 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:05:55.317 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:05:55.317 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:05:55.317 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:05:55.317 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:05:55.317 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:05:55.317 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:05:55.317 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:05:55.317 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:05:55.317 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:05:55.317 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:05:55.317 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:05:55.317 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:05:55.317 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:05:55.317 | INFO | ------------------------------------------ | 20260508 04:05:55.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:05:55.318 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:05:55.318 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:05:55.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:05:55.318 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.83e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.04e-01, 1.12e-02 / 6.24e-03 | 20260508 04:05:55.318 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.47e+00, 5.79e-01 / 2.21e-01 | 20260508 04:05:55.318 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.06e-01 / 1.19e-01 | 20260508 04:05:55.318 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.53e+01, 3.98e+00 / 2.03e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.75e+00 / 1.55e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.61e+00, 4.17e-01 / 2.61e-01 | 20260508 04:05:55.318 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.24e+00, 2.66e-01 / 1.09e-01 | 20260508 04:05:55.318 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.59e-01, 5.13e-02 / 2.51e-02 | 20260508 04:05:55.318 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:05:55.318 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:05:55.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:05:55.318 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 24.28%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 2.52%, LockDownload -- 2.69e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 04:05:55.318 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260508 04:05:55.429 | INFO | Starting a round of Baz IO | 20260508 04:05:58.852 | INFO | Populating RTMetrics report | 20260508 04:05:58.852 | INFO | RTMetrics report populated | 20260508 04:05:58.852 | INFO | Opening tmp RTMetrics json file | 20260508 04:05:58.853 | INFO | Serializing RTMetrics to json | 20260508 04:05:58.854 | INFO | Serialization done, swapping tmp file now... | 20260508 04:05:58.854 | INFO | RTMetrics done | 20260508 04:05:58.930 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.501 seconds | 20260508 04:05:58.930 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.14% / 0.00% / 58.86% | 20260508 04:05:58.930 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.561881% / 3.97364e-06% / 5.96046e-05% | 20260508 04:05:58.930 | INFO | Baz IO overall: | 20260508 04:05:58.930 | INFO | BytesWritten : 2288474603243 | 20260508 04:05:58.930 | INFO | NumEvents : 1105142783392 | 20260508 04:06:00.457 | INFO | Analyzing chunk frames = [5103616,5104128) | 20260508 04:06:00.457 | INFO | MemInfo: { | 20260508 04:06:00.457 | INFO | "gpu" : " 17 GB", | 20260508 04:06:00.457 | INFO | "huge" : "", | 20260508 04:06:00.457 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:00.457 | INFO | "malloc" : " 14 GB", | 20260508 04:06:00.457 | INFO | "pinned" : " 29 GB" | 20260508 04:06:00.457 | INFO | } | 20260508 04:06:00.457 | INFO | Perf report for the last 5.14116 seconds: | 20260508 04:06:00.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:00.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:00.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:00.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:00.457 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 04:06:00.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:00.457 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 04:06:00.457 | INFO | PRE_HQ : 66.5%, 1.8, 62.3ms, 0 | 20260508 04:06:00.457 | INFO | BAZWRITER : 67.2%, 2.9, 99.9ms, 0 | 20260508 04:06:00.457 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:06:00.457 | INFO | Uploaded 36GiB to GPU in 2.55912s (14.0673GiB/s) | 20260508 04:06:00.457 | INFO | Downloaded 38GiB to GPU in 2.694s (14.1054GiB/s) | 20260508 04:06:05.589 | INFO | Analyzing chunk frames = [5104128,5104640) | 20260508 04:06:05.589 | INFO | MemInfo: { | 20260508 04:06:05.589 | INFO | "gpu" : " 17 GB", | 20260508 04:06:05.589 | INFO | "huge" : "", | 20260508 04:06:05.589 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:05.590 | INFO | "malloc" : " 14 GB", | 20260508 04:06:05.590 | INFO | "pinned" : " 29 GB" | 20260508 04:06:05.590 | INFO | } | 20260508 04:06:05.590 | INFO | Perf report for the last 5.13278 seconds: | 20260508 04:06:05.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:05.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 04:06:05.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:06:05.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:05.590 | INFO | Uploaded 36GiB to GPU in 2.59402s (13.8781GiB/s) | 20260508 04:06:05.590 | INFO | Downloaded 34GiB to GPU in 2.423s (14.0322GiB/s) | 20260508 04:06:10.731 | INFO | Analyzing chunk frames = [5104640,5105152) | 20260508 04:06:10.731 | INFO | MemInfo: { | 20260508 04:06:10.731 | INFO | "gpu" : " 17 GB", | 20260508 04:06:10.731 | INFO | "huge" : "", | 20260508 04:06:10.731 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:10.732 | INFO | "malloc" : " 14 GB", | 20260508 04:06:10.732 | INFO | "pinned" : " 29 GB" | 20260508 04:06:10.732 | INFO | } | 20260508 04:06:10.732 | INFO | Perf report for the last 5.14205 seconds: | 20260508 04:06:10.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:10.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 04:06:10.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:06:10.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:10.732 | INFO | Uploaded 36GiB to GPU in 2.59225s (13.8876GiB/s) | 20260508 04:06:10.732 | INFO | Downloaded 34GiB to GPU in 2.49353s (13.6353GiB/s) | 20260508 04:06:15.878 | INFO | Analyzing chunk frames = [5105152,5105664) | 20260508 04:06:15.878 | INFO | MemInfo: { | 20260508 04:06:15.878 | INFO | "gpu" : " 17 GB", | 20260508 04:06:15.878 | INFO | "huge" : "", | 20260508 04:06:15.878 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:15.878 | INFO | "malloc" : " 14 GB", | 20260508 04:06:15.878 | INFO | "pinned" : " 29 GB" | 20260508 04:06:15.878 | INFO | } | 20260508 04:06:15.879 | INFO | Perf report for the last 5.14684 seconds: | 20260508 04:06:15.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:15.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 04:06:15.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:06:15.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:15.879 | INFO | Uploaded 36GiB to GPU in 2.58042s (13.9512GiB/s) | 20260508 04:06:15.879 | INFO | Downloaded 34GiB to GPU in 2.4932s (13.6371GiB/s) | 20260508 04:06:21.016 | INFO | Analyzing chunk frames = [5105664,5106176) | 20260508 04:06:21.016 | INFO | MemInfo: { | 20260508 04:06:21.016 | INFO | "gpu" : " 17 GB", | 20260508 04:06:21.016 | INFO | "huge" : "", | 20260508 04:06:21.016 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:21.016 | INFO | "malloc" : " 14 GB", | 20260508 04:06:21.016 | INFO | "pinned" : " 29 GB" | 20260508 04:06:21.016 | INFO | } | 20260508 04:06:21.016 | INFO | Perf report for the last 5.13748 seconds: | 20260508 04:06:21.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:21.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.016 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:06:21.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.016 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:06:21.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:21.017 | INFO | Uploaded 36GiB to GPU in 2.37965s (15.1283GiB/s) | 20260508 04:06:21.017 | INFO | Downloaded 34GiB to GPU in 2.54927s (13.3372GiB/s) | 20260508 04:06:26.152 | INFO | Analyzing chunk frames = [5106176,5106688) | 20260508 04:06:26.152 | INFO | MemInfo: { | 20260508 04:06:26.152 | INFO | "gpu" : " 17 GB", | 20260508 04:06:26.152 | INFO | "huge" : "", | 20260508 04:06:26.152 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:26.152 | INFO | "malloc" : " 14 GB", | 20260508 04:06:26.152 | INFO | "pinned" : " 29 GB" | 20260508 04:06:26.152 | INFO | } | 20260508 04:06:26.152 | INFO | Perf report for the last 5.13607 seconds: | 20260508 04:06:26.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:26.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:06:26.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:06:26.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:26.152 | INFO | Uploaded 36GiB to GPU in 2.4502s (14.6927GiB/s) | 20260508 04:06:26.152 | INFO | Downloaded 34GiB to GPU in 2.40786s (14.1204GiB/s) | 20260508 04:06:31.289 | INFO | Analyzing chunk frames = [5106688,5107200) | 20260508 04:06:31.289 | INFO | MemInfo: { | 20260508 04:06:31.289 | INFO | "gpu" : " 17 GB", | 20260508 04:06:31.289 | INFO | "huge" : "", | 20260508 04:06:31.289 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:31.289 | INFO | "malloc" : " 14 GB", | 20260508 04:06:31.289 | INFO | "pinned" : " 29 GB" | 20260508 04:06:31.289 | INFO | } | 20260508 04:06:31.290 | INFO | Perf report for the last 5.13742 seconds: | 20260508 04:06:31.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:31.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:06:31.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:06:31.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:31.290 | INFO | Uploaded 36GiB to GPU in 2.4774s (14.5314GiB/s) | 20260508 04:06:31.290 | INFO | Downloaded 34GiB to GPU in 2.42359s (14.0288GiB/s) | 20260508 04:06:36.431 | INFO | Analyzing chunk frames = [5107200,5107712) | 20260508 04:06:36.431 | INFO | MemInfo: { | 20260508 04:06:36.431 | INFO | "gpu" : " 17 GB", | 20260508 04:06:36.431 | INFO | "huge" : "", | 20260508 04:06:36.431 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:36.431 | INFO | "malloc" : " 14 GB", | 20260508 04:06:36.431 | INFO | "pinned" : " 29 GB" | 20260508 04:06:36.431 | INFO | } | 20260508 04:06:36.431 | INFO | Perf report for the last 5.1419 seconds: | 20260508 04:06:36.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:36.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.432 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:06:36.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.432 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260508 04:06:36.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:36.432 | INFO | Uploaded 36GiB to GPU in 2.47351s (14.5542GiB/s) | 20260508 04:06:36.432 | INFO | Downloaded 34GiB to GPU in 2.42366s (14.0283GiB/s) | 20260508 04:06:36.528 | INFO | Starting a round of Baz IO | 20260508 04:06:39.956 | INFO | Populating RTMetrics report | 20260508 04:06:39.957 | INFO | RTMetrics report populated | 20260508 04:06:39.957 | INFO | Opening tmp RTMetrics json file | 20260508 04:06:39.968 | INFO | Serializing RTMetrics to json | 20260508 04:06:39.969 | INFO | Serialization done, swapping tmp file now... | 20260508 04:06:39.970 | INFO | RTMetrics done | 20260508 04:06:40.034 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.505 seconds | 20260508 04:06:40.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.13% / 0.00% / 58.86% | 20260508 04:06:40.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.561674% / 3.97364e-06% / 5.96046e-05% | 20260508 04:06:40.034 | INFO | Baz IO overall: | 20260508 04:06:40.034 | INFO | BytesWritten : 2290194675202 | 20260508 04:06:40.034 | INFO | NumEvents : 1105964907311 | 20260508 04:06:41.574 | INFO | Analyzing chunk frames = [5107712,5108224) | 20260508 04:06:41.574 | INFO | MemInfo: { | 20260508 04:06:41.574 | INFO | "gpu" : " 17 GB", | 20260508 04:06:41.574 | INFO | "huge" : "", | 20260508 04:06:41.574 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:41.574 | INFO | "malloc" : " 14 GB", | 20260508 04:06:41.574 | INFO | "pinned" : " 29 GB" | 20260508 04:06:41.574 | INFO | } | 20260508 04:06:41.574 | INFO | Perf report for the last 5.14271 seconds: | 20260508 04:06:41.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:41.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:41.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:41.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:41.574 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 04:06:41.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:41.574 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:06:41.574 | INFO | PRE_HQ : 65.8%, 1.6, 55.6ms, 0 | 20260508 04:06:41.574 | INFO | BAZWRITER : 67.6%, 2.7, 94.1ms, 0 | 20260508 04:06:41.574 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:06:41.574 | INFO | Uploaded 36GiB to GPU in 2.59133s (13.8925GiB/s) | 20260508 04:06:41.574 | INFO | Downloaded 38GiB to GPU in 2.68335s (14.1614GiB/s) | 20260508 04:06:46.710 | INFO | Analyzing chunk frames = [5108224,5108736) | 20260508 04:06:46.710 | INFO | MemInfo: { | 20260508 04:06:46.710 | INFO | "gpu" : " 17 GB", | 20260508 04:06:46.710 | INFO | "huge" : "", | 20260508 04:06:46.710 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:46.710 | INFO | "malloc" : " 14 GB", | 20260508 04:06:46.710 | INFO | "pinned" : " 29 GB" | 20260508 04:06:46.710 | INFO | } | 20260508 04:06:46.710 | INFO | Perf report for the last 5.13564 seconds: | 20260508 04:06:46.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:46.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 04:06:46.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:06:46.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:46.710 | INFO | Uploaded 36GiB to GPU in 2.59026s (13.8982GiB/s) | 20260508 04:06:46.710 | INFO | Downloaded 34GiB to GPU in 2.4383s (13.9441GiB/s) | 20260508 04:06:51.847 | INFO | Analyzing chunk frames = [5108736,5109248) | 20260508 04:06:51.847 | INFO | MemInfo: { | 20260508 04:06:51.847 | INFO | "gpu" : " 17 GB", | 20260508 04:06:51.847 | INFO | "huge" : "", | 20260508 04:06:51.847 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:51.847 | INFO | "malloc" : " 14 GB", | 20260508 04:06:51.847 | INFO | "pinned" : " 29 GB" | 20260508 04:06:51.847 | INFO | } | 20260508 04:06:51.847 | INFO | Perf report for the last 5.13732 seconds: | 20260508 04:06:51.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:51.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.847 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:06:51.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.847 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:06:51.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:51.848 | INFO | Uploaded 36GiB to GPU in 2.56546s (14.0326GiB/s) | 20260508 04:06:51.848 | INFO | Downloaded 34GiB to GPU in 2.42302s (14.0321GiB/s) | 20260508 04:06:56.989 | INFO | Analyzing chunk frames = [5109248,5109760) | 20260508 04:06:56.989 | INFO | MemInfo: { | 20260508 04:06:56.989 | INFO | "gpu" : " 17 GB", | 20260508 04:06:56.989 | INFO | "huge" : "", | 20260508 04:06:56.989 | INFO | "huge_shared" : " 29 GB", | 20260508 04:06:56.989 | INFO | "malloc" : " 14 GB", | 20260508 04:06:56.989 | INFO | "pinned" : " 29 GB" | 20260508 04:06:56.989 | INFO | } | 20260508 04:06:56.989 | INFO | Perf report for the last 5.14212 seconds: | 20260508 04:06:56.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:06:56.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.989 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 04:06:56.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.989 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:06:56.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:06:56.990 | INFO | Uploaded 36GiB to GPU in 2.56793s (14.0191GiB/s) | 20260508 04:06:56.990 | INFO | Downloaded 34GiB to GPU in 2.50093s (13.595GiB/s) | 20260508 04:07:02.128 | INFO | Analyzing chunk frames = [5109760,5110272) | 20260508 04:07:02.128 | INFO | MemInfo: { | 20260508 04:07:02.128 | INFO | "gpu" : " 17 GB", | 20260508 04:07:02.128 | INFO | "huge" : "", | 20260508 04:07:02.128 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:02.128 | INFO | "malloc" : " 14 GB", | 20260508 04:07:02.128 | INFO | "pinned" : " 29 GB" | 20260508 04:07:02.128 | INFO | } | 20260508 04:07:02.128 | INFO | Perf report for the last 5.13867 seconds: | 20260508 04:07:02.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:02.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 04:07:02.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:07:02.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:02.128 | INFO | Uploaded 36GiB to GPU in 2.45961s (14.6365GiB/s) | 20260508 04:07:02.128 | INFO | Downloaded 34GiB to GPU in 2.52878s (13.4452GiB/s) | 20260508 04:07:07.266 | INFO | Analyzing chunk frames = [5110272,5110784) | 20260508 04:07:07.266 | INFO | MemInfo: { | 20260508 04:07:07.266 | INFO | "gpu" : " 17 GB", | 20260508 04:07:07.266 | INFO | "huge" : "", | 20260508 04:07:07.266 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:07.266 | INFO | "malloc" : " 14 GB", | 20260508 04:07:07.266 | INFO | "pinned" : " 29 GB" | 20260508 04:07:07.266 | INFO | } | 20260508 04:07:07.266 | INFO | Perf report for the last 5.13799 seconds: | 20260508 04:07:07.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:07.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:07:07.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:07:07.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:07.266 | INFO | Uploaded 36GiB to GPU in 2.45023s (14.6925GiB/s) | 20260508 04:07:07.266 | INFO | Downloaded 34GiB to GPU in 2.41643s (14.0703GiB/s) | 20260508 04:07:12.405 | INFO | Analyzing chunk frames = [5110784,5111296) | 20260508 04:07:12.405 | INFO | MemInfo: { | 20260508 04:07:12.405 | INFO | "gpu" : " 17 GB", | 20260508 04:07:12.405 | INFO | "huge" : "", | 20260508 04:07:12.405 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:12.405 | INFO | "malloc" : " 14 GB", | 20260508 04:07:12.405 | INFO | "pinned" : " 29 GB" | 20260508 04:07:12.405 | INFO | } | 20260508 04:07:12.405 | INFO | Perf report for the last 5.13923 seconds: | 20260508 04:07:12.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:12.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.405 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:07:12.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.406 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260508 04:07:12.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:12.406 | INFO | Uploaded 36GiB to GPU in 2.45517s (14.6629GiB/s) | 20260508 04:07:12.406 | INFO | Downloaded 34GiB to GPU in 2.4233s (14.0305GiB/s) | 20260508 04:07:17.542 | INFO | Analyzing chunk frames = [5111296,5111808) | 20260508 04:07:17.542 | INFO | MemInfo: { | 20260508 04:07:17.542 | INFO | "gpu" : " 17 GB", | 20260508 04:07:17.542 | INFO | "huge" : "", | 20260508 04:07:17.542 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:17.542 | INFO | "malloc" : " 14 GB", | 20260508 04:07:17.542 | INFO | "pinned" : " 29 GB" | 20260508 04:07:17.542 | INFO | } | 20260508 04:07:17.543 | INFO | Perf report for the last 5.13733 seconds: | 20260508 04:07:17.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:17.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 04:07:17.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 04:07:17.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:17.543 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:07:17.543 | INFO | Total mem usage: 16.7 GiB | 20260508 04:07:17.543 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:07:17.543 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 04:07:17.543 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:07:17.543 | INFO | High Water Breakdown: | 20260508 04:07:17.543 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:07:17.543 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:07:17.543 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:07:17.543 | INFO | ------------------------------------------ | 20260508 04:07:17.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:07:17.543 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:07:17.543 | INFO | Total mem usage: 26.5 GiB | 20260508 04:07:17.543 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:07:17.543 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:07:17.543 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:07:17.543 | INFO | High Water Breakdown: | 20260508 04:07:17.543 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:07:17.543 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:07:17.543 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:07:17.543 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:07:17.543 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:07:17.543 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:07:17.543 | INFO | Uploaded 36GiB to GPU in 2.46139s (14.6259GiB/s) | 20260508 04:07:17.544 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:07:17.544 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:07:17.544 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:07:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:07:17.544 | INFO | Downloaded 34GiB to GPU in 2.42476s (14.022GiB/s) | 20260508 04:07:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:07:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:07:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:07:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:07:17.544 | INFO | ------------------------------------------ | 20260508 04:07:17.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:07:17.544 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:07:17.544 | INFO | Total mem usage: 27.8 GiB | 20260508 04:07:17.544 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:07:17.544 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:07:17.544 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:07:17.544 | INFO | High Water Breakdown: | 20260508 04:07:17.544 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:07:17.544 | INFO | ------------------------------------------ | 20260508 04:07:17.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:07:17.544 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:07:17.544 | INFO | Total mem usage: 15.7 GiB | 20260508 04:07:17.544 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:07:17.544 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:07:17.544 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:07:17.544 | INFO | High Water Breakdown: | 20260508 04:07:17.544 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:07:17.544 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:07:17.544 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:07:17.544 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:07:17.544 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:07:17.544 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:07:17.544 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:07:17.544 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:07:17.544 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:07:17.544 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:07:17.544 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:07:17.544 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:07:17.544 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:07:17.544 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:07:17.544 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:07:17.544 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:07:17.544 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:07:17.544 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:07:17.544 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:07:17.544 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:07:17.544 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:07:17.544 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:07:17.544 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:07:17.544 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:07:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:07:17.544 | INFO | ------------------------------------------ | 20260508 04:07:17.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:07:17.544 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:07:17.545 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:07:17.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:07:17.545 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.57e+01, 1.41e+01 / 1.11e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.03e-01, 1.19e-02 / 5.18e-03 | 20260508 04:07:17.545 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.43e+00, 5.82e-01 / 2.12e-01 | 20260508 04:07:17.545 | INFO | PerfReport: 3.32%, Histogram -- 1.08e+00 / 1.45e+00, 2.06e-01 / 1.29e-01 | 20260508 04:07:17.545 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.55e+01, 3.87e+00 / 1.44e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.85e+01 / 2.00e+01, 1.69e+00 / 1.55e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.40e+00 / 2.62e+00, 4.14e-01 / 2.29e-01 | 20260508 04:07:17.545 | INFO | PerfReport: 3.24%, Metrics -- 1.06e+00 / 1.21e+00, 2.65e-01 / 1.38e-01 | 20260508 04:07:17.545 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.65e-01, 5.03e-02 / 3.03e-02 | 20260508 04:07:17.545 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:07:17.545 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:07:17.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:07:17.545 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 24.59%, Upload -- 2.62e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 24.29%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 30.63%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 2.53%, LockUpload -- 2.69e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 2.48%, LockDownload -- 2.64e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 15.47%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:07:17.545 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260508 04:07:17.658 | INFO | Starting a round of Baz IO | 20260508 04:07:21.073 | INFO | Populating RTMetrics report | 20260508 04:07:21.073 | INFO | RTMetrics report populated | 20260508 04:07:21.073 | INFO | Opening tmp RTMetrics json file | 20260508 04:07:21.086 | INFO | Serializing RTMetrics to json | 20260508 04:07:21.087 | INFO | Serialization done, swapping tmp file now... | 20260508 04:07:21.088 | INFO | RTMetrics done | 20260508 04:07:21.148 | INFO | Finished a round of Baz IO: Wrote 1.601 GiB in 3.489 seconds | 20260508 04:07:21.148 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.13% / 0.00% / 58.87% | 20260508 04:07:21.148 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.559644% / 3.97364e-06% / 3.57628e-05% | 20260508 04:07:21.148 | INFO | Baz IO overall: | 20260508 04:07:21.148 | INFO | BytesWritten : 2291913777239 | 20260508 04:07:21.148 | INFO | NumEvents : 1106786333069 | 20260508 04:07:22.686 | INFO | Analyzing chunk frames = [5111808,5112320) | 20260508 04:07:22.686 | INFO | MemInfo: { | 20260508 04:07:22.686 | INFO | "gpu" : " 17 GB", | 20260508 04:07:22.686 | INFO | "huge" : "", | 20260508 04:07:22.686 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:22.686 | INFO | "malloc" : " 14 GB", | 20260508 04:07:22.686 | INFO | "pinned" : " 29 GB" | 20260508 04:07:22.686 | INFO | } | 20260508 04:07:22.687 | INFO | Perf report for the last 5.14394 seconds: | 20260508 04:07:22.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:22.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:22.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:22.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:22.687 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 04:07:22.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:22.687 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 04:07:22.687 | INFO | PRE_HQ : 66.1%, 1.8, 61.8ms, 0 | 20260508 04:07:22.687 | INFO | BAZWRITER : 67.4%, 2.9, 98.8ms, 0 | 20260508 04:07:22.687 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:07:22.687 | INFO | Uploaded 36GiB to GPU in 2.60222s (13.8344GiB/s) | 20260508 04:07:22.687 | INFO | Downloaded 38GiB to GPU in 2.70326s (14.0571GiB/s) | 20260508 04:07:27.829 | INFO | Analyzing chunk frames = [5112320,5112832) | 20260508 04:07:27.829 | INFO | MemInfo: { | 20260508 04:07:27.829 | INFO | "gpu" : " 17 GB", | 20260508 04:07:27.829 | INFO | "huge" : "", | 20260508 04:07:27.829 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:27.829 | INFO | "malloc" : " 14 GB", | 20260508 04:07:27.829 | INFO | "pinned" : " 29 GB" | 20260508 04:07:27.829 | INFO | } | 20260508 04:07:27.829 | INFO | Perf report for the last 5.14288 seconds: | 20260508 04:07:27.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:27.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.830 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 04:07:27.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.830 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:07:27.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:27.830 | INFO | Uploaded 36GiB to GPU in 2.59692s (13.8626GiB/s) | 20260508 04:07:27.830 | INFO | Downloaded 34GiB to GPU in 2.42976s (13.9932GiB/s) | 20260508 04:07:32.962 | INFO | Analyzing chunk frames = [5112832,5113344) | 20260508 04:07:32.962 | INFO | MemInfo: { | 20260508 04:07:32.962 | INFO | "gpu" : " 17 GB", | 20260508 04:07:32.962 | INFO | "huge" : "", | 20260508 04:07:32.962 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:32.962 | INFO | "malloc" : " 14 GB", | 20260508 04:07:32.962 | INFO | "pinned" : " 29 GB" | 20260508 04:07:32.962 | INFO | } | 20260508 04:07:32.962 | INFO | Perf report for the last 5.13252 seconds: | 20260508 04:07:32.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:32.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 04:07:32.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:07:32.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:32.962 | INFO | Uploaded 36GiB to GPU in 2.55551s (14.0872GiB/s) | 20260508 04:07:32.962 | INFO | Downloaded 34GiB to GPU in 2.39878s (14.1739GiB/s) | 20260508 04:07:38.099 | INFO | Analyzing chunk frames = [5113344,5113856) | 20260508 04:07:38.099 | INFO | MemInfo: { | 20260508 04:07:38.099 | INFO | "gpu" : " 17 GB", | 20260508 04:07:38.099 | INFO | "huge" : "", | 20260508 04:07:38.099 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:38.099 | INFO | "malloc" : " 14 GB", | 20260508 04:07:38.099 | INFO | "pinned" : " 29 GB" | 20260508 04:07:38.099 | INFO | } | 20260508 04:07:38.099 | INFO | Perf report for the last 5.13748 seconds: | 20260508 04:07:38.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:38.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.099 | INFO | ANALYSIS : 67.5%, 3.0, 104.3ms, 0 | 20260508 04:07:38.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.100 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:07:38.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:38.100 | INFO | Uploaded 36GiB to GPU in 2.57074s (14.0038GiB/s) | 20260508 04:07:38.100 | INFO | Downloaded 34GiB to GPU in 2.49959s (13.6022GiB/s) | 20260508 04:07:43.242 | INFO | Analyzing chunk frames = [5113856,5114368) | 20260508 04:07:43.242 | INFO | MemInfo: { | 20260508 04:07:43.242 | INFO | "gpu" : " 17 GB", | 20260508 04:07:43.242 | INFO | "huge" : "", | 20260508 04:07:43.242 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:43.242 | INFO | "malloc" : " 14 GB", | 20260508 04:07:43.242 | INFO | "pinned" : " 29 GB" | 20260508 04:07:43.242 | INFO | } | 20260508 04:07:43.243 | INFO | Perf report for the last 5.14303 seconds: | 20260508 04:07:43.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:43.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:07:43.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:07:43.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:43.243 | INFO | Uploaded 36GiB to GPU in 2.35069s (15.3146GiB/s) | 20260508 04:07:43.243 | INFO | Downloaded 34GiB to GPU in 2.54812s (13.3432GiB/s) | 20260508 04:07:48.384 | INFO | Analyzing chunk frames = [5114368,5114880) | 20260508 04:07:48.384 | INFO | MemInfo: { | 20260508 04:07:48.384 | INFO | "gpu" : " 17 GB", | 20260508 04:07:48.384 | INFO | "huge" : "", | 20260508 04:07:48.384 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:48.384 | INFO | "malloc" : " 14 GB", | 20260508 04:07:48.384 | INFO | "pinned" : " 29 GB" | 20260508 04:07:48.384 | INFO | } | 20260508 04:07:48.384 | INFO | Perf report for the last 5.14196 seconds: | 20260508 04:07:48.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:48.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.385 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 04:07:48.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.385 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:07:48.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:48.385 | INFO | Uploaded 36GiB to GPU in 2.47252s (14.56GiB/s) | 20260508 04:07:48.385 | INFO | Downloaded 34GiB to GPU in 2.42083s (14.0448GiB/s) | 20260508 04:07:53.524 | INFO | Analyzing chunk frames = [5114880,5115392) | 20260508 04:07:53.524 | INFO | MemInfo: { | 20260508 04:07:53.524 | INFO | "gpu" : " 17 GB", | 20260508 04:07:53.524 | INFO | "huge" : "", | 20260508 04:07:53.524 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:53.524 | INFO | "malloc" : " 14 GB", | 20260508 04:07:53.524 | INFO | "pinned" : " 29 GB" | 20260508 04:07:53.524 | INFO | } | 20260508 04:07:53.524 | INFO | Perf report for the last 5.13986 seconds: | 20260508 04:07:53.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:53.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.524 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:07:53.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.525 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 04:07:53.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:53.525 | INFO | Uploaded 36GiB to GPU in 2.4512s (14.6867GiB/s) | 20260508 04:07:53.525 | INFO | Downloaded 34GiB to GPU in 2.41237s (14.094GiB/s) | 20260508 04:07:58.661 | INFO | Analyzing chunk frames = [5115392,5115904) | 20260508 04:07:58.661 | INFO | MemInfo: { | 20260508 04:07:58.661 | INFO | "gpu" : " 17 GB", | 20260508 04:07:58.661 | INFO | "huge" : "", | 20260508 04:07:58.661 | INFO | "huge_shared" : " 29 GB", | 20260508 04:07:58.662 | INFO | "malloc" : " 14 GB", | 20260508 04:07:58.662 | INFO | "pinned" : " 29 GB" | 20260508 04:07:58.662 | INFO | } | 20260508 04:07:58.662 | INFO | Perf report for the last 5.1374 seconds: | 20260508 04:07:58.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:07:58.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 04:07:58.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 04:07:58.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:07:58.662 | INFO | Uploaded 36GiB to GPU in 2.47951s (14.519GiB/s) | 20260508 04:07:58.663 | INFO | Downloaded 34GiB to GPU in 2.42682s (14.0101GiB/s) | 20260508 04:07:58.774 | INFO | Starting a round of Baz IO | 20260508 04:08:02.189 | INFO | Populating RTMetrics report | 20260508 04:08:02.189 | INFO | RTMetrics report populated | 20260508 04:08:02.189 | INFO | Opening tmp RTMetrics json file | 20260508 04:08:02.198 | INFO | Serializing RTMetrics to json | 20260508 04:08:02.204 | INFO | Serialization done, swapping tmp file now... | 20260508 04:08:02.204 | INFO | RTMetrics done | 20260508 04:08:02.263 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.489 seconds | 20260508 04:08:02.263 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.13% / 0.00% / 58.87% | 20260508 04:08:02.263 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.560125% / 0% / 5.16574e-05% | 20260508 04:08:02.263 | INFO | Baz IO overall: | 20260508 04:08:02.263 | INFO | BytesWritten : 2293633855791 | 20260508 04:08:02.264 | INFO | NumEvents : 1107608411918 | 20260508 04:08:03.796 | INFO | Analyzing chunk frames = [5115904,5116416) | 20260508 04:08:03.796 | INFO | MemInfo: { | 20260508 04:08:03.796 | INFO | "gpu" : " 17 GB", | 20260508 04:08:03.796 | INFO | "huge" : "", | 20260508 04:08:03.796 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:03.796 | INFO | "malloc" : " 14 GB", | 20260508 04:08:03.796 | INFO | "pinned" : " 29 GB" | 20260508 04:08:03.796 | INFO | } | 20260508 04:08:03.796 | INFO | Perf report for the last 5.13445 seconds: | 20260508 04:08:03.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:03.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:03.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:03.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:03.796 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 04:08:03.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:03.796 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 04:08:03.796 | INFO | PRE_HQ : 66.5%, 1.8, 62.2ms, 0 | 20260508 04:08:03.796 | INFO | BAZWRITER : 67.4%, 2.8, 97.2ms, 0 | 20260508 04:08:03.796 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:08:03.797 | INFO | Uploaded 36GiB to GPU in 2.59072s (13.8958GiB/s) | 20260508 04:08:03.797 | INFO | Downloaded 38GiB to GPU in 2.67382s (14.2119GiB/s) | 20260508 04:08:08.944 | INFO | Analyzing chunk frames = [5116416,5116928) | 20260508 04:08:08.944 | INFO | MemInfo: { | 20260508 04:08:08.944 | INFO | "gpu" : " 17 GB", | 20260508 04:08:08.944 | INFO | "huge" : "", | 20260508 04:08:08.944 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:08.944 | INFO | "malloc" : " 14 GB", | 20260508 04:08:08.944 | INFO | "pinned" : " 29 GB" | 20260508 04:08:08.944 | INFO | } | 20260508 04:08:08.944 | INFO | Perf report for the last 5.14779 seconds: | 20260508 04:08:08.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:08.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:08:08.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:08:08.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:08.944 | INFO | Uploaded 36GiB to GPU in 2.58152s (13.9453GiB/s) | 20260508 04:08:08.944 | INFO | Downloaded 34GiB to GPU in 2.42189s (14.0386GiB/s) | 20260508 04:08:14.079 | INFO | Analyzing chunk frames = [5116928,5117440) | 20260508 04:08:14.079 | INFO | MemInfo: { | 20260508 04:08:14.079 | INFO | "gpu" : " 17 GB", | 20260508 04:08:14.079 | INFO | "huge" : "", | 20260508 04:08:14.079 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:14.079 | INFO | "malloc" : " 14 GB", | 20260508 04:08:14.079 | INFO | "pinned" : " 29 GB" | 20260508 04:08:14.079 | INFO | } | 20260508 04:08:14.079 | INFO | Perf report for the last 5.13491 seconds: | 20260508 04:08:14.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:14.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.079 | INFO | ANALYSIS : 67.5%, 3.0, 104.7ms, 0 | 20260508 04:08:14.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.079 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:08:14.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:14.080 | INFO | Uploaded 36GiB to GPU in 2.56017s (14.0616GiB/s) | 20260508 04:08:14.080 | INFO | Downloaded 34GiB to GPU in 2.40851s (14.1166GiB/s) | 20260508 04:08:19.218 | INFO | Analyzing chunk frames = [5117440,5117952) | 20260508 04:08:19.218 | INFO | MemInfo: { | 20260508 04:08:19.218 | INFO | "gpu" : " 17 GB", | 20260508 04:08:19.218 | INFO | "huge" : "", | 20260508 04:08:19.218 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:19.219 | INFO | "malloc" : " 14 GB", | 20260508 04:08:19.219 | INFO | "pinned" : " 29 GB" | 20260508 04:08:19.219 | INFO | } | 20260508 04:08:19.219 | INFO | Perf report for the last 5.13972 seconds: | 20260508 04:08:19.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:19.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | ANALYSIS : 67.3%, 2.9, 104.1ms, 0 | 20260508 04:08:19.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:08:19.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:19.219 | INFO | Uploaded 36GiB to GPU in 2.60661s (13.811GiB/s) | 20260508 04:08:19.219 | INFO | Downloaded 34GiB to GPU in 2.42484s (14.0215GiB/s) | 20260508 04:08:24.353 | INFO | Analyzing chunk frames = [5117952,5118464) | 20260508 04:08:24.353 | INFO | MemInfo: { | 20260508 04:08:24.353 | INFO | "gpu" : " 17 GB", | 20260508 04:08:24.353 | INFO | "huge" : "", | 20260508 04:08:24.353 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:24.353 | INFO | "malloc" : " 14 GB", | 20260508 04:08:24.353 | INFO | "pinned" : " 29 GB" | 20260508 04:08:24.353 | INFO | } | 20260508 04:08:24.353 | INFO | Perf report for the last 5.13442 seconds: | 20260508 04:08:24.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:24.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:08:24.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:08:24.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:24.353 | INFO | Uploaded 36GiB to GPU in 2.36363s (15.2308GiB/s) | 20260508 04:08:24.353 | INFO | Downloaded 34GiB to GPU in 2.54269s (13.3717GiB/s) | 20260508 04:08:29.496 | INFO | Analyzing chunk frames = [5118464,5118976) | 20260508 04:08:29.496 | INFO | MemInfo: { | 20260508 04:08:29.496 | INFO | "gpu" : " 17 GB", | 20260508 04:08:29.496 | INFO | "huge" : "", | 20260508 04:08:29.496 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:29.496 | INFO | "malloc" : " 14 GB", | 20260508 04:08:29.496 | INFO | "pinned" : " 29 GB" | 20260508 04:08:29.496 | INFO | } | 20260508 04:08:29.496 | INFO | Perf report for the last 5.14278 seconds: | 20260508 04:08:29.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:29.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.496 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:08:29.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.496 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:08:29.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:29.497 | INFO | Uploaded 36GiB to GPU in 2.44697s (14.712GiB/s) | 20260508 04:08:29.497 | INFO | Downloaded 34GiB to GPU in 2.41751s (14.0641GiB/s) | 20260508 04:08:34.635 | INFO | Analyzing chunk frames = [5118976,5119488) | 20260508 04:08:34.635 | INFO | MemInfo: { | 20260508 04:08:34.635 | INFO | "gpu" : " 17 GB", | 20260508 04:08:34.635 | INFO | "huge" : "", | 20260508 04:08:34.635 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:34.635 | INFO | "malloc" : " 14 GB", | 20260508 04:08:34.635 | INFO | "pinned" : " 29 GB" | 20260508 04:08:34.635 | INFO | } | 20260508 04:08:34.636 | INFO | Perf report for the last 5.13957 seconds: | 20260508 04:08:34.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:34.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:08:34.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:08:34.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:34.636 | INFO | Uploaded 36GiB to GPU in 2.46019s (14.633GiB/s) | 20260508 04:08:34.636 | INFO | Downloaded 34GiB to GPU in 2.4147s (14.0804GiB/s) | 20260508 04:08:39.775 | INFO | Analyzing chunk frames = [5119488,5120000) | 20260508 04:08:39.775 | INFO | MemInfo: { | 20260508 04:08:39.775 | INFO | "gpu" : " 17 GB", | 20260508 04:08:39.775 | INFO | "huge" : "", | 20260508 04:08:39.775 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:39.775 | INFO | "malloc" : " 14 GB", | 20260508 04:08:39.775 | INFO | "pinned" : " 29 GB" | 20260508 04:08:39.775 | INFO | } | 20260508 04:08:39.776 | INFO | Perf report for the last 5.14 seconds: | 20260508 04:08:39.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:39.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:08:39.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 04:08:39.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:39.776 | INFO | Uploaded 36GiB to GPU in 2.48486s (14.4877GiB/s) | 20260508 04:08:39.776 | INFO | Downloaded 34GiB to GPU in 2.424s (14.0264GiB/s) | 20260508 04:08:39.776 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:08:39.776 | INFO | Total mem usage: 16.7 GiB | 20260508 04:08:39.776 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:08:39.776 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:08:39.776 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:08:39.776 | INFO | High Water Breakdown: | 20260508 04:08:39.776 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:08:39.776 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:08:39.776 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:08:39.776 | INFO | ------------------------------------------ | 20260508 04:08:39.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:08:39.776 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:08:39.776 | INFO | Total mem usage: 26.5 GiB | 20260508 04:08:39.776 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:08:39.776 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:08:39.776 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:08:39.776 | INFO | High Water Breakdown: | 20260508 04:08:39.776 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:08:39.776 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:08:39.776 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:08:39.776 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:08:39.776 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:08:39.776 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:08:39.776 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:08:39.776 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:08:39.776 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:08:39.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:08:39.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:08:39.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:08:39.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:08:39.777 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:08:39.777 | INFO | ------------------------------------------ | 20260508 04:08:39.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:08:39.777 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:08:39.777 | INFO | Total mem usage: 27.8 GiB | 20260508 04:08:39.777 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:08:39.777 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:08:39.777 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:08:39.777 | INFO | High Water Breakdown: | 20260508 04:08:39.777 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:08:39.777 | INFO | ------------------------------------------ | 20260508 04:08:39.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:08:39.777 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:08:39.777 | INFO | Total mem usage: 15.7 GiB | 20260508 04:08:39.777 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:08:39.777 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:08:39.777 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:08:39.777 | INFO | High Water Breakdown: | 20260508 04:08:39.777 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:08:39.777 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:08:39.777 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:08:39.777 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:08:39.777 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:08:39.777 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:08:39.777 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:08:39.777 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:08:39.777 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:08:39.777 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:08:39.777 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:08:39.777 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:08:39.777 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:08:39.777 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:08:39.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:08:39.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:08:39.777 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:08:39.777 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:08:39.777 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:08:39.777 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:08:39.777 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:08:39.777 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:08:39.777 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:08:39.777 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:08:39.777 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:08:39.777 | INFO | ------------------------------------------ | 20260508 04:08:39.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:08:39.777 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:08:39.777 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:08:39.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:08:39.777 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.57e+01, 1.41e+01 / 1.31e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.04e-01, 1.18e-02 / 1.06e-02 | 20260508 04:08:39.777 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.49e+00, 5.85e-01 / 1.72e-01 | 20260508 04:08:39.777 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.08e-01 / 1.28e-01 | 20260508 04:08:39.777 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.53e+01, 3.57e+00 / 1.69e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.85e+01 / 2.02e+01, 1.75e+00 / 1.36e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.57e+00, 4.14e-01 / 2.38e-01 | 20260508 04:08:39.777 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.23e+00, 2.60e-01 / 1.49e-01 | 20260508 04:08:39.777 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.57e-01, 5.01e-02 / 2.27e-02 | 20260508 04:08:39.777 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:08:39.777 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:08:39.777 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:08:39.777 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 24.56%, Upload -- 2.62e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 24.19%, Download -- 2.58e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 2.53%, LockUpload -- 2.70e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 2.43%, LockDownload -- 2.58e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 15.55%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:08:39.777 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260508 04:08:39.873 | INFO | Starting a round of Baz IO | 20260508 04:08:43.306 | INFO | Populating RTMetrics report | 20260508 04:08:43.307 | INFO | RTMetrics report populated | 20260508 04:08:43.307 | INFO | Opening tmp RTMetrics json file | 20260508 04:08:43.318 | INFO | Serializing RTMetrics to json | 20260508 04:08:43.319 | INFO | Serialization done, swapping tmp file now... | 20260508 04:08:43.320 | INFO | RTMetrics done | 20260508 04:08:43.379 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.505 seconds | 20260508 04:08:43.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.13% / 0.00% / 58.87% | 20260508 04:08:43.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.559171% / 0% / 6.75519e-05% | 20260508 04:08:43.379 | INFO | Baz IO overall: | 20260508 04:08:43.379 | INFO | BytesWritten : 2295354077191 | 20260508 04:08:43.379 | INFO | NumEvents : 1108430740341 | 20260508 04:08:44.916 | INFO | Analyzing chunk frames = [5120000,5120512) | 20260508 04:08:44.916 | INFO | MemInfo: { | 20260508 04:08:44.916 | INFO | "gpu" : " 17 GB", | 20260508 04:08:44.916 | INFO | "huge" : "", | 20260508 04:08:44.916 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:44.916 | INFO | "malloc" : " 14 GB", | 20260508 04:08:44.916 | INFO | "pinned" : " 29 GB" | 20260508 04:08:44.916 | INFO | } | 20260508 04:08:44.916 | INFO | Perf report for the last 5.14093 seconds: | 20260508 04:08:44.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:44.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:44.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:44.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:44.917 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 04:08:44.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:44.917 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:08:44.917 | INFO | PRE_HQ : 65.5%, 1.6, 55.4ms, 0 | 20260508 04:08:44.917 | INFO | BAZWRITER : 67.6%, 2.6, 89.9ms, 0 | 20260508 04:08:44.917 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:08:44.917 | INFO | Uploaded 36GiB to GPU in 2.59716s (13.8613GiB/s) | 20260508 04:08:44.917 | INFO | Downloaded 38GiB to GPU in 2.69009s (14.1259GiB/s) | 20260508 04:08:50.051 | INFO | Analyzing chunk frames = [5120512,5121024) | 20260508 04:08:50.051 | INFO | MemInfo: { | 20260508 04:08:50.051 | INFO | "gpu" : " 17 GB", | 20260508 04:08:50.051 | INFO | "huge" : "", | 20260508 04:08:50.051 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:50.051 | INFO | "malloc" : " 14 GB", | 20260508 04:08:50.051 | INFO | "pinned" : " 29 GB" | 20260508 04:08:50.051 | INFO | } | 20260508 04:08:50.051 | INFO | Perf report for the last 5.13488 seconds: | 20260508 04:08:50.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:50.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.051 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 04:08:50.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.052 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:08:50.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:50.052 | INFO | Uploaded 36GiB to GPU in 2.5104s (14.3404GiB/s) | 20260508 04:08:50.052 | INFO | Downloaded 34GiB to GPU in 2.43073s (13.9876GiB/s) | 20260508 04:08:55.192 | INFO | Analyzing chunk frames = [5121024,5121536) | 20260508 04:08:55.192 | INFO | MemInfo: { | 20260508 04:08:55.192 | INFO | "gpu" : " 17 GB", | 20260508 04:08:55.192 | INFO | "huge" : "", | 20260508 04:08:55.192 | INFO | "huge_shared" : " 29 GB", | 20260508 04:08:55.192 | INFO | "malloc" : " 14 GB", | 20260508 04:08:55.192 | INFO | "pinned" : " 29 GB" | 20260508 04:08:55.192 | INFO | } | 20260508 04:08:55.192 | INFO | Perf report for the last 5.14069 seconds: | 20260508 04:08:55.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:08:55.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260508 04:08:55.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:08:55.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:08:55.192 | INFO | Uploaded 36GiB to GPU in 2.56001s (14.0624GiB/s) | 20260508 04:08:55.192 | INFO | Downloaded 34GiB to GPU in 2.42425s (14.0249GiB/s) | 20260508 04:09:00.333 | INFO | Analyzing chunk frames = [5121536,5122048) | 20260508 04:09:00.333 | INFO | MemInfo: { | 20260508 04:09:00.333 | INFO | "gpu" : " 17 GB", | 20260508 04:09:00.333 | INFO | "huge" : "", | 20260508 04:09:00.333 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:00.333 | INFO | "malloc" : " 14 GB", | 20260508 04:09:00.333 | INFO | "pinned" : " 29 GB" | 20260508 04:09:00.333 | INFO | } | 20260508 04:09:00.333 | INFO | Perf report for the last 5.14113 seconds: | 20260508 04:09:00.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:00.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.333 | INFO | ANALYSIS : 67.2%, 2.9, 104.1ms, 0 | 20260508 04:09:00.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.333 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:09:00.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:00.334 | INFO | Uploaded 36GiB to GPU in 2.59974s (13.8475GiB/s) | 20260508 04:09:00.334 | INFO | Downloaded 34GiB to GPU in 2.42913s (13.9968GiB/s) | 20260508 04:09:05.473 | INFO | Analyzing chunk frames = [5122048,5122560) | 20260508 04:09:05.473 | INFO | MemInfo: { | 20260508 04:09:05.473 | INFO | "gpu" : " 17 GB", | 20260508 04:09:05.473 | INFO | "huge" : "", | 20260508 04:09:05.473 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:05.473 | INFO | "malloc" : " 14 GB", | 20260508 04:09:05.473 | INFO | "pinned" : " 29 GB" | 20260508 04:09:05.473 | INFO | } | 20260508 04:09:05.473 | INFO | Perf report for the last 5.13987 seconds: | 20260508 04:09:05.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:05.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:09:05.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:09:05.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:05.473 | INFO | Uploaded 36GiB to GPU in 2.3789s (15.1331GiB/s) | 20260508 04:09:05.474 | INFO | Downloaded 34GiB to GPU in 2.5463s (13.3527GiB/s) | 20260508 04:09:10.609 | INFO | Analyzing chunk frames = [5122560,5123072) | 20260508 04:09:10.609 | INFO | MemInfo: { | 20260508 04:09:10.609 | INFO | "gpu" : " 17 GB", | 20260508 04:09:10.609 | INFO | "huge" : "", | 20260508 04:09:10.609 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:10.609 | INFO | "malloc" : " 14 GB", | 20260508 04:09:10.609 | INFO | "pinned" : " 29 GB" | 20260508 04:09:10.609 | INFO | } | 20260508 04:09:10.609 | INFO | Perf report for the last 5.13592 seconds: | 20260508 04:09:10.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:10.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 04:09:10.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:09:10.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:10.609 | INFO | Uploaded 36GiB to GPU in 2.45728s (14.6503GiB/s) | 20260508 04:09:10.610 | INFO | Downloaded 34GiB to GPU in 2.41754s (14.0639GiB/s) | 20260508 04:09:15.753 | INFO | Analyzing chunk frames = [5123072,5123584) | 20260508 04:09:15.753 | INFO | MemInfo: { | 20260508 04:09:15.753 | INFO | "gpu" : " 17 GB", | 20260508 04:09:15.753 | INFO | "huge" : "", | 20260508 04:09:15.753 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:15.753 | INFO | "malloc" : " 14 GB", | 20260508 04:09:15.753 | INFO | "pinned" : " 29 GB" | 20260508 04:09:15.753 | INFO | } | 20260508 04:09:15.753 | INFO | Perf report for the last 5.14421 seconds: | 20260508 04:09:15.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:15.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.753 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:09:15.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.753 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:09:15.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:15.754 | INFO | Uploaded 36GiB to GPU in 2.43896s (14.7604GiB/s) | 20260508 04:09:15.754 | INFO | Downloaded 34GiB to GPU in 2.41498s (14.0788GiB/s) | 20260508 04:09:20.892 | INFO | Analyzing chunk frames = [5123584,5124096) | 20260508 04:09:20.892 | INFO | MemInfo: { | 20260508 04:09:20.892 | INFO | "gpu" : " 17 GB", | 20260508 04:09:20.892 | INFO | "huge" : "", | 20260508 04:09:20.892 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:20.892 | INFO | "malloc" : " 14 GB", | 20260508 04:09:20.892 | INFO | "pinned" : " 29 GB" | 20260508 04:09:20.892 | INFO | } | 20260508 04:09:20.892 | INFO | Perf report for the last 5.13883 seconds: | 20260508 04:09:20.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:20.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:09:20.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:09:20.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:20.892 | INFO | Uploaded 36GiB to GPU in 2.47349s (14.5543GiB/s) | 20260508 04:09:20.892 | INFO | Downloaded 34GiB to GPU in 2.4235s (14.0293GiB/s) | 20260508 04:09:20.989 | INFO | Starting a round of Baz IO | 20260508 04:09:24.419 | INFO | Populating RTMetrics report | 20260508 04:09:24.420 | INFO | RTMetrics report populated | 20260508 04:09:24.420 | INFO | Opening tmp RTMetrics json file | 20260508 04:09:24.439 | INFO | Serializing RTMetrics to json | 20260508 04:09:24.442 | INFO | Serialization done, swapping tmp file now... | 20260508 04:09:24.443 | INFO | RTMetrics done | 20260508 04:09:24.523 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.533 seconds | 20260508 04:09:24.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.12% / 0.00% / 58.87% | 20260508 04:09:24.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.557995% / 7.94729e-06% / 5.16574e-05% | 20260508 04:09:24.523 | INFO | Baz IO overall: | 20260508 04:09:24.523 | INFO | BytesWritten : 2297071387983 | 20260508 04:09:24.523 | INFO | NumEvents : 1109251277202 | 20260508 04:09:26.031 | INFO | Analyzing chunk frames = [5124096,5124608) | 20260508 04:09:26.031 | INFO | MemInfo: { | 20260508 04:09:26.031 | INFO | "gpu" : " 17 GB", | 20260508 04:09:26.031 | INFO | "huge" : "", | 20260508 04:09:26.031 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:26.031 | INFO | "malloc" : " 14 GB", | 20260508 04:09:26.031 | INFO | "pinned" : " 29 GB" | 20260508 04:09:26.031 | INFO | } | 20260508 04:09:26.031 | INFO | Perf report for the last 5.13887 seconds: | 20260508 04:09:26.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:26.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:26.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:26.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:26.031 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 04:09:26.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:26.031 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 04:09:26.031 | INFO | PRE_HQ : 65.7%, 1.6, 55.8ms, 0 | 20260508 04:09:26.031 | INFO | BAZWRITER : 68.2%, 3.1, 108.8ms, 0 | 20260508 04:09:26.031 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 04:09:26.031 | INFO | Uploaded 36GiB to GPU in 2.61042s (13.7909GiB/s) | 20260508 04:09:26.031 | INFO | Downloaded 38GiB to GPU in 2.70386s (14.054GiB/s) | 20260508 04:09:31.167 | INFO | Analyzing chunk frames = [5124608,5125120) | 20260508 04:09:31.167 | INFO | MemInfo: { | 20260508 04:09:31.167 | INFO | "gpu" : " 17 GB", | 20260508 04:09:31.167 | INFO | "huge" : "", | 20260508 04:09:31.167 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:31.167 | INFO | "malloc" : " 14 GB", | 20260508 04:09:31.167 | INFO | "pinned" : " 29 GB" | 20260508 04:09:31.167 | INFO | } | 20260508 04:09:31.168 | INFO | Perf report for the last 5.1366 seconds: | 20260508 04:09:31.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:31.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:09:31.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:09:31.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:31.168 | INFO | Uploaded 36GiB to GPU in 2.59134s (13.8924GiB/s) | 20260508 04:09:31.168 | INFO | Downloaded 34GiB to GPU in 2.43379s (13.97GiB/s) | 20260508 04:09:36.307 | INFO | Analyzing chunk frames = [5125120,5125632) | 20260508 04:09:36.307 | INFO | MemInfo: { | 20260508 04:09:36.307 | INFO | "gpu" : " 17 GB", | 20260508 04:09:36.307 | INFO | "huge" : "", | 20260508 04:09:36.307 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:36.307 | INFO | "malloc" : " 14 GB", | 20260508 04:09:36.307 | INFO | "pinned" : " 29 GB" | 20260508 04:09:36.307 | INFO | } | 20260508 04:09:36.307 | INFO | Perf report for the last 5.13979 seconds: | 20260508 04:09:36.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:36.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.307 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 04:09:36.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.307 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:09:36.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:36.308 | INFO | Uploaded 36GiB to GPU in 2.56134s (14.0551GiB/s) | 20260508 04:09:36.308 | INFO | Downloaded 34GiB to GPU in 2.41356s (14.0871GiB/s) | 20260508 04:09:41.445 | INFO | Analyzing chunk frames = [5125632,5126144) | 20260508 04:09:41.445 | INFO | MemInfo: { | 20260508 04:09:41.445 | INFO | "gpu" : " 17 GB", | 20260508 04:09:41.445 | INFO | "huge" : "", | 20260508 04:09:41.445 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:41.445 | INFO | "malloc" : " 14 GB", | 20260508 04:09:41.445 | INFO | "pinned" : " 29 GB" | 20260508 04:09:41.445 | INFO | } | 20260508 04:09:41.445 | INFO | Perf report for the last 5.13781 seconds: | 20260508 04:09:41.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:41.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 04:09:41.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:09:41.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:41.445 | INFO | Uploaded 36GiB to GPU in 2.57505s (13.9803GiB/s) | 20260508 04:09:41.446 | INFO | Downloaded 34GiB to GPU in 2.41942s (14.053GiB/s) | 20260508 04:09:46.592 | INFO | Analyzing chunk frames = [5126144,5126656) | 20260508 04:09:46.592 | INFO | MemInfo: { | 20260508 04:09:46.592 | INFO | "gpu" : " 17 GB", | 20260508 04:09:46.592 | INFO | "huge" : "", | 20260508 04:09:46.592 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:46.592 | INFO | "malloc" : " 14 GB", | 20260508 04:09:46.592 | INFO | "pinned" : " 29 GB" | 20260508 04:09:46.592 | INFO | } | 20260508 04:09:46.592 | INFO | Perf report for the last 5.14662 seconds: | 20260508 04:09:46.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:46.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 04:09:46.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260508 04:09:46.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:46.592 | INFO | Uploaded 36GiB to GPU in 2.41819s (14.8872GiB/s) | 20260508 04:09:46.592 | INFO | Downloaded 34GiB to GPU in 2.58491s (13.1533GiB/s) | 20260508 04:09:51.726 | INFO | Analyzing chunk frames = [5126656,5127168) | 20260508 04:09:51.726 | INFO | MemInfo: { | 20260508 04:09:51.726 | INFO | "gpu" : " 17 GB", | 20260508 04:09:51.726 | INFO | "huge" : "", | 20260508 04:09:51.726 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:51.726 | INFO | "malloc" : " 14 GB", | 20260508 04:09:51.726 | INFO | "pinned" : " 29 GB" | 20260508 04:09:51.726 | INFO | } | 20260508 04:09:51.726 | INFO | Perf report for the last 5.13445 seconds: | 20260508 04:09:51.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:51.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.726 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 04:09:51.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.726 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 04:09:51.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:51.727 | INFO | Uploaded 36GiB to GPU in 2.4528s (14.6771GiB/s) | 20260508 04:09:51.727 | INFO | Downloaded 34GiB to GPU in 2.41191s (14.0967GiB/s) | 20260508 04:09:56.862 | INFO | Analyzing chunk frames = [5127168,5127680) | 20260508 04:09:56.862 | INFO | MemInfo: { | 20260508 04:09:56.862 | INFO | "gpu" : " 17 GB", | 20260508 04:09:56.862 | INFO | "huge" : "", | 20260508 04:09:56.862 | INFO | "huge_shared" : " 29 GB", | 20260508 04:09:56.862 | INFO | "malloc" : " 14 GB", | 20260508 04:09:56.862 | INFO | "pinned" : " 29 GB" | 20260508 04:09:56.862 | INFO | } | 20260508 04:09:56.862 | INFO | Perf report for the last 5.13548 seconds: | 20260508 04:09:56.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:09:56.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:09:56.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:09:56.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:09:56.862 | INFO | Uploaded 36GiB to GPU in 2.4469s (14.7125GiB/s) | 20260508 04:09:56.862 | INFO | Downloaded 34GiB to GPU in 2.41813s (14.0604GiB/s) | 20260508 04:10:02.004 | INFO | Analyzing chunk frames = [5127680,5128192) | 20260508 04:10:02.004 | INFO | MemInfo: { | 20260508 04:10:02.004 | INFO | "gpu" : " 17 GB", | 20260508 04:10:02.004 | INFO | "huge" : "", | 20260508 04:10:02.004 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:02.004 | INFO | "malloc" : " 14 GB", | 20260508 04:10:02.004 | INFO | "pinned" : " 29 GB" | 20260508 04:10:02.004 | INFO | } | 20260508 04:10:02.004 | INFO | Perf report for the last 5.14227 seconds: | 20260508 04:10:02.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:02.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.004 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:10:02.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.004 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:10:02.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:02.005 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:10:02.005 | INFO | Total mem usage: 16.7 GiB | 20260508 04:10:02.005 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:10:02.005 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.1%) | 20260508 04:10:02.005 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:10:02.005 | INFO | High Water Breakdown: | 20260508 04:10:02.005 | INFO | 59.6% : 8.56 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:10:02.005 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:10:02.005 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:10:02.005 | INFO | ------------------------------------------ | 20260508 04:10:02.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:10:02.005 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:10:02.005 | INFO | Total mem usage: 26.5 GiB | 20260508 04:10:02.005 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:10:02.005 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 04:10:02.005 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 04:10:02.005 | INFO | High Water Breakdown: | 20260508 04:10:02.005 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:10:02.005 | INFO | Uploaded 36GiB to GPU in 2.46675s (14.5941GiB/s) | 20260508 04:10:02.005 | INFO | Downloaded 34GiB to GPU in 2.4201s (14.049GiB/s) | 20260508 04:10:02.005 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:10:02.005 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:10:02.005 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:10:02.005 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:10:02.005 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:10:02.005 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:10:02.005 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:10:02.005 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:10:02.005 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:10:02.005 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:10:02.005 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:10:02.005 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:10:02.005 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:10:02.005 | INFO | ------------------------------------------ | 20260508 04:10:02.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:10:02.005 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:10:02.005 | INFO | Total mem usage: 27.8 GiB | 20260508 04:10:02.005 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:10:02.005 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:10:02.005 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:10:02.005 | INFO | High Water Breakdown: | 20260508 04:10:02.005 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:10:02.005 | INFO | ------------------------------------------ | 20260508 04:10:02.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:10:02.005 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:10:02.005 | INFO | Total mem usage: 15.7 GiB | 20260508 04:10:02.005 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:10:02.005 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:10:02.005 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:10:02.005 | INFO | High Water Breakdown: | 20260508 04:10:02.005 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:10:02.005 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:10:02.005 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:10:02.005 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:10:02.005 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:10:02.005 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:10:02.005 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:10:02.005 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:10:02.005 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:10:02.005 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:10:02.005 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:10:02.005 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:10:02.005 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:10:02.005 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:10:02.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:10:02.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:10:02.005 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:10:02.005 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:10:02.005 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:10:02.005 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:10:02.005 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:10:02.005 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:10:02.005 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:10:02.005 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:10:02.005 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:10:02.006 | INFO | ------------------------------------------ | 20260508 04:10:02.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:10:02.006 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:10:02.006 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:10:02.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:10:02.006 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 2.03e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.14e-02 / 5.01e-03 | 20260508 04:10:02.006 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.47e+00, 5.93e-01 / 1.79e-01 | 20260508 04:10:02.006 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.09e-01 / 1.23e-01 | 20260508 04:10:02.006 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.53e+01, 3.96e+00 / 2.41e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.80e+00 / 1.47e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.57e+00, 4.16e-01 / 2.27e-01 | 20260508 04:10:02.006 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.25e+00, 2.69e-01 / 1.44e-01 | 20260508 04:10:02.006 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.56e-01, 5.08e-02 / 2.25e-02 | 20260508 04:10:02.006 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:10:02.006 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:10:02.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:10:02.006 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 24.55%, Upload -- 2.61e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 24.21%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 30.77%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 2.42%, LockUpload -- 2.57e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 2.41%, LockDownload -- 2.57e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 15.63%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:10:02.006 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260508 04:10:02.101 | INFO | Starting a round of Baz IO | 20260508 04:10:05.532 | INFO | Populating RTMetrics report | 20260508 04:10:05.533 | INFO | RTMetrics report populated | 20260508 04:10:05.533 | INFO | Opening tmp RTMetrics json file | 20260508 04:10:05.604 | INFO | Serializing RTMetrics to json | 20260508 04:10:05.619 | INFO | Serialization done, swapping tmp file now... | 20260508 04:10:05.624 | INFO | RTMetrics done | 20260508 04:10:05.661 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.559 seconds | 20260508 04:10:05.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.12% / 0.00% / 58.88% | 20260508 04:10:05.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.557025% / 0% / 7.15256e-05% | 20260508 04:10:05.661 | INFO | Baz IO overall: | 20260508 04:10:05.661 | INFO | BytesWritten : 2298788231828 | 20260508 04:10:05.661 | INFO | NumEvents : 1110071637346 | 20260508 04:10:07.146 | INFO | Analyzing chunk frames = [5128192,5128704) | 20260508 04:10:07.146 | INFO | MemInfo: { | 20260508 04:10:07.146 | INFO | "gpu" : " 17 GB", | 20260508 04:10:07.146 | INFO | "huge" : "", | 20260508 04:10:07.146 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:07.146 | INFO | "malloc" : " 14 GB", | 20260508 04:10:07.146 | INFO | "pinned" : " 29 GB" | 20260508 04:10:07.146 | INFO | } | 20260508 04:10:07.146 | INFO | Perf report for the last 5.14229 seconds: | 20260508 04:10:07.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:07.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:07.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:07.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:07.147 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 04:10:07.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:07.147 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260508 04:10:07.147 | INFO | PRE_HQ : 65.6%, 1.6, 55.7ms, 0 | 20260508 04:10:07.147 | INFO | BAZWRITER : 68.8%, 3.5, 123.1ms, 0 | 20260508 04:10:07.147 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260508 04:10:07.148 | INFO | Uploaded 36GiB to GPU in 2.60136s (13.8389GiB/s) | 20260508 04:10:07.148 | INFO | Downloaded 38GiB to GPU in 2.69979s (14.0752GiB/s) | 20260508 04:10:12.287 | INFO | Analyzing chunk frames = [5128704,5129216) | 20260508 04:10:12.287 | INFO | MemInfo: { | 20260508 04:10:12.287 | INFO | "gpu" : " 17 GB", | 20260508 04:10:12.287 | INFO | "huge" : "", | 20260508 04:10:12.287 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:12.287 | INFO | "malloc" : " 14 GB", | 20260508 04:10:12.287 | INFO | "pinned" : " 29 GB" | 20260508 04:10:12.287 | INFO | } | 20260508 04:10:12.287 | INFO | Perf report for the last 5.141 seconds: | 20260508 04:10:12.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:12.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.287 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:10:12.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.288 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:10:12.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:12.288 | INFO | Uploaded 36GiB to GPU in 2.58955s (13.9021GiB/s) | 20260508 04:10:12.288 | INFO | Downloaded 34GiB to GPU in 2.44599s (13.9003GiB/s) | 20260508 04:10:17.426 | INFO | Analyzing chunk frames = [5129216,5129728) | 20260508 04:10:17.426 | INFO | MemInfo: { | 20260508 04:10:17.426 | INFO | "gpu" : " 17 GB", | 20260508 04:10:17.426 | INFO | "huge" : "", | 20260508 04:10:17.426 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:17.427 | INFO | "malloc" : " 14 GB", | 20260508 04:10:17.427 | INFO | "pinned" : " 29 GB" | 20260508 04:10:17.427 | INFO | } | 20260508 04:10:17.427 | INFO | Perf report for the last 5.1393 seconds: | 20260508 04:10:17.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:17.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 04:10:17.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:10:17.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:17.427 | INFO | Uploaded 36GiB to GPU in 2.55997s (14.0626GiB/s) | 20260508 04:10:17.427 | INFO | Downloaded 34GiB to GPU in 2.42427s (14.0248GiB/s) | 20260508 04:10:22.563 | INFO | Analyzing chunk frames = [5129728,5130240) | 20260508 04:10:22.563 | INFO | MemInfo: { | 20260508 04:10:22.563 | INFO | "gpu" : " 17 GB", | 20260508 04:10:22.563 | INFO | "huge" : "", | 20260508 04:10:22.563 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:22.563 | INFO | "malloc" : " 14 GB", | 20260508 04:10:22.563 | INFO | "pinned" : " 29 GB" | 20260508 04:10:22.563 | INFO | } | 20260508 04:10:22.563 | INFO | Perf report for the last 5.13588 seconds: | 20260508 04:10:22.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:22.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:10:22.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 04:10:22.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:22.563 | INFO | Uploaded 36GiB to GPU in 2.59229s (13.8873GiB/s) | 20260508 04:10:22.563 | INFO | Downloaded 34GiB to GPU in 2.40248s (14.152GiB/s) | 20260508 04:10:27.706 | INFO | Analyzing chunk frames = [5130240,5130752) | 20260508 04:10:27.706 | INFO | MemInfo: { | 20260508 04:10:27.706 | INFO | "gpu" : " 17 GB", | 20260508 04:10:27.706 | INFO | "huge" : "", | 20260508 04:10:27.706 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:27.706 | INFO | "malloc" : " 14 GB", | 20260508 04:10:27.706 | INFO | "pinned" : " 29 GB" | 20260508 04:10:27.706 | INFO | } | 20260508 04:10:27.706 | INFO | Perf report for the last 5.14317 seconds: | 20260508 04:10:27.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:27.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.706 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 04:10:27.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.706 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:10:27.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:27.707 | INFO | Uploaded 36GiB to GPU in 2.38444s (15.0979GiB/s) | 20260508 04:10:27.707 | INFO | Downloaded 34GiB to GPU in 2.59079s (13.1234GiB/s) | 20260508 04:10:32.842 | INFO | Analyzing chunk frames = [5130752,5131264) | 20260508 04:10:32.842 | INFO | MemInfo: { | 20260508 04:10:32.842 | INFO | "gpu" : " 17 GB", | 20260508 04:10:32.842 | INFO | "huge" : "", | 20260508 04:10:32.842 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:32.842 | INFO | "malloc" : " 14 GB", | 20260508 04:10:32.842 | INFO | "pinned" : " 29 GB" | 20260508 04:10:32.842 | INFO | } | 20260508 04:10:32.842 | INFO | Perf report for the last 5.1366 seconds: | 20260508 04:10:32.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:32.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.842 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:10:32.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.843 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:10:32.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:32.843 | INFO | Uploaded 36GiB to GPU in 2.44727s (14.7102GiB/s) | 20260508 04:10:32.843 | INFO | Downloaded 34GiB to GPU in 2.41765s (14.0632GiB/s) | 20260508 04:10:37.984 | INFO | Analyzing chunk frames = [5131264,5131776) | 20260508 04:10:37.984 | INFO | MemInfo: { | 20260508 04:10:37.984 | INFO | "gpu" : " 17 GB", | 20260508 04:10:37.984 | INFO | "huge" : "", | 20260508 04:10:37.984 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:37.984 | INFO | "malloc" : " 14 GB", | 20260508 04:10:37.984 | INFO | "pinned" : " 29 GB" | 20260508 04:10:37.984 | INFO | } | 20260508 04:10:37.984 | INFO | Perf report for the last 5.14184 seconds: | 20260508 04:10:37.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:37.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.984 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:10:37.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.984 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:10:37.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:37.985 | INFO | Uploaded 36GiB to GPU in 2.44549s (14.721GiB/s) | 20260508 04:10:37.985 | INFO | Downloaded 34GiB to GPU in 2.42085s (14.0446GiB/s) | 20260508 04:10:43.115 | INFO | Analyzing chunk frames = [5131776,5132288) | 20260508 04:10:43.115 | INFO | MemInfo: { | 20260508 04:10:43.115 | INFO | "gpu" : " 17 GB", | 20260508 04:10:43.115 | INFO | "huge" : "", | 20260508 04:10:43.116 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:43.116 | INFO | "malloc" : " 14 GB", | 20260508 04:10:43.116 | INFO | "pinned" : " 29 GB" | 20260508 04:10:43.116 | INFO | } | 20260508 04:10:43.116 | INFO | Perf report for the last 5.13139 seconds: | 20260508 04:10:43.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:43.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 04:10:43.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 04:10:43.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:43.116 | INFO | Uploaded 36GiB to GPU in 2.44993s (14.6943GiB/s) | 20260508 04:10:43.116 | INFO | Downloaded 34GiB to GPU in 2.41216s (14.0952GiB/s) | 20260508 04:10:43.213 | INFO | Starting a round of Baz IO | 20260508 04:10:46.657 | INFO | Populating RTMetrics report | 20260508 04:10:46.658 | INFO | RTMetrics report populated | 20260508 04:10:46.658 | INFO | Opening tmp RTMetrics json file | 20260508 04:10:46.716 | INFO | Serializing RTMetrics to json | 20260508 04:10:46.781 | INFO | Serialization done, swapping tmp file now... | 20260508 04:10:46.784 | INFO | RTMetrics done | 20260508 04:10:46.788 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.574 seconds | 20260508 04:10:46.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.12% / 0.00% / 58.88% | 20260508 04:10:46.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.555078% / 7.94729e-06% / 8.34465e-05% | 20260508 04:10:46.788 | INFO | Baz IO overall: | 20260508 04:10:46.788 | INFO | BytesWritten : 2300505299352 | 20260508 04:10:46.788 | INFO | NumEvents : 1110892218838 | 20260508 04:10:48.258 | INFO | Analyzing chunk frames = [5132288,5132800) | 20260508 04:10:48.258 | INFO | MemInfo: { | 20260508 04:10:48.258 | INFO | "gpu" : " 17 GB", | 20260508 04:10:48.258 | INFO | "huge" : "", | 20260508 04:10:48.258 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:48.258 | INFO | "malloc" : " 14 GB", | 20260508 04:10:48.258 | INFO | "pinned" : " 29 GB" | 20260508 04:10:48.258 | INFO | } | 20260508 04:10:48.258 | INFO | Perf report for the last 5.14269 seconds: | 20260508 04:10:48.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:48.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:48.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:48.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:48.258 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 04:10:48.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:48.258 | INFO | RT_METRICS : 41.8%, 1.0, 22.0ms, 0 | 20260508 04:10:48.258 | INFO | PRE_HQ : 65.8%, 1.6, 55.5ms, 0 | 20260508 04:10:48.259 | INFO | BAZWRITER : 69.1%, 3.9, 138.5ms, 0 | 20260508 04:10:48.259 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260508 04:10:48.259 | INFO | Uploaded 36GiB to GPU in 2.54352s (14.1536GiB/s) | 20260508 04:10:48.259 | INFO | Downloaded 38GiB to GPU in 2.70638s (14.0409GiB/s) | 20260508 04:10:53.399 | INFO | Analyzing chunk frames = [5132800,5133312) | 20260508 04:10:53.399 | INFO | MemInfo: { | 20260508 04:10:53.399 | INFO | "gpu" : " 17 GB", | 20260508 04:10:53.399 | INFO | "huge" : "", | 20260508 04:10:53.399 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:53.399 | INFO | "malloc" : " 14 GB", | 20260508 04:10:53.399 | INFO | "pinned" : " 29 GB" | 20260508 04:10:53.399 | INFO | } | 20260508 04:10:53.399 | INFO | Perf report for the last 5.141 seconds: | 20260508 04:10:53.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:53.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.399 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 04:10:53.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.399 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260508 04:10:53.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:53.400 | INFO | Uploaded 36GiB to GPU in 2.58855s (13.9074GiB/s) | 20260508 04:10:53.400 | INFO | Downloaded 34GiB to GPU in 2.422s (14.038GiB/s) | 20260508 04:10:58.538 | INFO | Analyzing chunk frames = [5133312,5133824) | 20260508 04:10:58.538 | INFO | MemInfo: { | 20260508 04:10:58.538 | INFO | "gpu" : " 17 GB", | 20260508 04:10:58.538 | INFO | "huge" : "", | 20260508 04:10:58.538 | INFO | "huge_shared" : " 29 GB", | 20260508 04:10:58.538 | INFO | "malloc" : " 14 GB", | 20260508 04:10:58.538 | INFO | "pinned" : " 29 GB" | 20260508 04:10:58.538 | INFO | } | 20260508 04:10:58.539 | INFO | Perf report for the last 5.13917 seconds: | 20260508 04:10:58.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:10:58.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 04:10:58.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:10:58.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:10:58.539 | INFO | Uploaded 36GiB to GPU in 2.56517s (14.0341GiB/s) | 20260508 04:10:58.539 | INFO | Downloaded 34GiB to GPU in 2.40465s (14.1393GiB/s) | 20260508 04:11:03.676 | INFO | Analyzing chunk frames = [5133824,5134336) | 20260508 04:11:03.676 | INFO | MemInfo: { | 20260508 04:11:03.676 | INFO | "gpu" : " 17 GB", | 20260508 04:11:03.676 | INFO | "huge" : "", | 20260508 04:11:03.676 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:03.676 | INFO | "malloc" : " 14 GB", | 20260508 04:11:03.676 | INFO | "pinned" : " 29 GB" | 20260508 04:11:03.676 | INFO | } | 20260508 04:11:03.676 | INFO | Perf report for the last 5.13758 seconds: | 20260508 04:11:03.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:03.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 04:11:03.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:11:03.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:03.676 | INFO | Uploaded 36GiB to GPU in 2.57588s (13.9758GiB/s) | 20260508 04:11:03.676 | INFO | Downloaded 34GiB to GPU in 2.4195s (14.0525GiB/s) | 20260508 04:11:08.815 | INFO | Analyzing chunk frames = [5134336,5134848) | 20260508 04:11:08.815 | INFO | MemInfo: { | 20260508 04:11:08.815 | INFO | "gpu" : " 17 GB", | 20260508 04:11:08.815 | INFO | "huge" : "", | 20260508 04:11:08.815 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:08.815 | INFO | "malloc" : " 14 GB", | 20260508 04:11:08.815 | INFO | "pinned" : " 29 GB" | 20260508 04:11:08.815 | INFO | } | 20260508 04:11:08.815 | INFO | Perf report for the last 5.13919 seconds: | 20260508 04:11:08.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:08.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.815 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:11:08.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.815 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 04:11:08.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:08.816 | INFO | Uploaded 36GiB to GPU in 2.34348s (15.3618GiB/s) | 20260508 04:11:08.816 | INFO | Downloaded 34GiB to GPU in 2.55341s (13.3155GiB/s) | 20260508 04:11:13.954 | INFO | Analyzing chunk frames = [5134848,5135360) | 20260508 04:11:13.954 | INFO | MemInfo: { | 20260508 04:11:13.954 | INFO | "gpu" : " 17 GB", | 20260508 04:11:13.954 | INFO | "huge" : "", | 20260508 04:11:13.954 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:13.954 | INFO | "malloc" : " 14 GB", | 20260508 04:11:13.954 | INFO | "pinned" : " 29 GB" | 20260508 04:11:13.954 | INFO | } | 20260508 04:11:13.954 | INFO | Perf report for the last 5.13906 seconds: | 20260508 04:11:13.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:13.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.954 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:11:13.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.954 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:11:13.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:13.955 | INFO | Uploaded 36GiB to GPU in 2.43192s (14.8031GiB/s) | 20260508 04:11:13.955 | INFO | Downloaded 34GiB to GPU in 2.40971s (14.1096GiB/s) | 20260508 04:11:19.099 | INFO | Analyzing chunk frames = [5135360,5135872) | 20260508 04:11:19.099 | INFO | MemInfo: { | 20260508 04:11:19.099 | INFO | "gpu" : " 17 GB", | 20260508 04:11:19.099 | INFO | "huge" : "", | 20260508 04:11:19.099 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:19.099 | INFO | "malloc" : " 14 GB", | 20260508 04:11:19.099 | INFO | "pinned" : " 29 GB" | 20260508 04:11:19.099 | INFO | } | 20260508 04:11:19.099 | INFO | Perf report for the last 5.14461 seconds: | 20260508 04:11:19.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:19.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:11:19.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:11:19.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:19.099 | INFO | Uploaded 36GiB to GPU in 2.42751s (14.83GiB/s) | 20260508 04:11:19.099 | INFO | Downloaded 34GiB to GPU in 2.42823s (14.002GiB/s) | 20260508 04:11:24.238 | INFO | Analyzing chunk frames = [5135872,5136384) | 20260508 04:11:24.238 | INFO | MemInfo: { | 20260508 04:11:24.238 | INFO | "gpu" : " 17 GB", | 20260508 04:11:24.238 | INFO | "huge" : "", | 20260508 04:11:24.238 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:24.238 | INFO | "malloc" : " 14 GB", | 20260508 04:11:24.238 | INFO | "pinned" : " 29 GB" | 20260508 04:11:24.238 | INFO | } | 20260508 04:11:24.239 | INFO | Perf report for the last 5.13947 seconds: | 20260508 04:11:24.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:24.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:11:24.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:11:24.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:24.239 | INFO | Uploaded 36GiB to GPU in 2.45329s (14.6742GiB/s) | 20260508 04:11:24.239 | INFO | Downloaded 34GiB to GPU in 2.42794s (14.0036GiB/s) | 20260508 04:11:24.239 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:11:24.239 | INFO | Total mem usage: 16.7 GiB | 20260508 04:11:24.239 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:11:24.239 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12%) | 20260508 04:11:24.239 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:11:24.239 | INFO | High Water Breakdown: | 20260508 04:11:24.239 | INFO | 60.2% : 8.97 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:11:24.239 | INFO | 39.7% : 5.92 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:11:24.239 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:11:24.239 | INFO | ------------------------------------------ | 20260508 04:11:24.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:11:24.239 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:11:24.239 | INFO | Total mem usage: 26.5 GiB | 20260508 04:11:24.239 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:11:24.239 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:11:24.239 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:11:24.239 | INFO | High Water Breakdown: | 20260508 04:11:24.239 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:11:24.239 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:11:24.239 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:11:24.239 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:11:24.239 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:11:24.239 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:11:24.239 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:11:24.239 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:11:24.239 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:11:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:11:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:11:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:11:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:11:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:11:24.240 | INFO | ------------------------------------------ | 20260508 04:11:24.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:11:24.240 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:11:24.240 | INFO | Total mem usage: 27.8 GiB | 20260508 04:11:24.240 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:11:24.240 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:11:24.240 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:11:24.240 | INFO | High Water Breakdown: | 20260508 04:11:24.240 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:11:24.240 | INFO | ------------------------------------------ | 20260508 04:11:24.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:11:24.240 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:11:24.240 | INFO | Total mem usage: 15.7 GiB | 20260508 04:11:24.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:11:24.240 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:11:24.240 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:11:24.240 | INFO | High Water Breakdown: | 20260508 04:11:24.240 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:11:24.240 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:11:24.240 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:11:24.240 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:11:24.240 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:11:24.240 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:11:24.240 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:11:24.240 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:11:24.240 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:11:24.240 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:11:24.240 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:11:24.240 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:11:24.240 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:11:24.240 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:11:24.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:11:24.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:11:24.240 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:11:24.240 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:11:24.240 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:11:24.240 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:11:24.240 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:11:24.240 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:11:24.240 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:11:24.240 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:11:24.240 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:11:24.240 | INFO | ------------------------------------------ | 20260508 04:11:24.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:11:24.240 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:11:24.240 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:11:24.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:11:24.240 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.60e+01, 1.38e+01 / 1.98e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.02e-01, 1.12e-02 / 7.36e-03 | 20260508 04:11:24.240 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.46e+00, 5.95e-01 / 1.71e-01 | 20260508 04:11:24.240 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.11e-01 / 1.20e-01 | 20260508 04:11:24.240 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.54e+01, 3.78e+00 / 2.29e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.80e+00 / 1.35e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.39e+00 / 2.60e+00, 4.17e-01 / 2.34e-01 | 20260508 04:11:24.240 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.24e+00, 2.64e-01 / 1.33e-01 | 20260508 04:11:24.240 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.57e-01, 5.07e-02 / 2.17e-02 | 20260508 04:11:24.240 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:11:24.240 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:11:24.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:11:24.240 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 24.48%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 30.79%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 2.39%, LockUpload -- 2.54e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 2.52%, LockDownload -- 2.68e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:11:24.240 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260508 04:11:24.336 | INFO | Starting a round of Baz IO | 20260508 04:11:27.781 | INFO | Populating RTMetrics report | 20260508 04:11:27.782 | INFO | RTMetrics report populated | 20260508 04:11:27.782 | INFO | Opening tmp RTMetrics json file | 20260508 04:11:28.429 | INFO | Serializing RTMetrics to json | 20260508 04:11:28.438 | INFO | Serialization done, swapping tmp file now... | 20260508 04:11:28.443 | INFO | RTMetrics done | 20260508 04:11:28.583 | INFO | Finished a round of Baz IO: Wrote 1.601 GiB in 4.246 seconds | 20260508 04:11:28.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.12% / 0.00% / 58.88% | 20260508 04:11:28.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.55687% / 3.97364e-06% / 8.74201e-05% | 20260508 04:11:28.583 | INFO | Baz IO overall: | 20260508 04:11:28.583 | INFO | BytesWritten : 2302224273206 | 20260508 04:11:28.583 | INFO | NumEvents : 1111714406265 | 20260508 04:11:29.377 | INFO | Analyzing chunk frames = [5136384,5136896) | 20260508 04:11:29.377 | INFO | MemInfo: { | 20260508 04:11:29.377 | INFO | "gpu" : " 17 GB", | 20260508 04:11:29.377 | INFO | "huge" : "", | 20260508 04:11:29.377 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:29.377 | INFO | "malloc" : " 14 GB", | 20260508 04:11:29.377 | INFO | "pinned" : " 29 GB" | 20260508 04:11:29.377 | INFO | } | 20260508 04:11:29.377 | INFO | Perf report for the last 5.13873 seconds: | 20260508 04:11:29.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:29.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:29.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:29.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:29.377 | INFO | ANALYSIS : 68.3%, 2.9, 104.2ms, 0 | 20260508 04:11:29.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:29.377 | INFO | RT_METRICS : 55.1%, 1.0, 28.9ms, 0 | 20260508 04:11:29.377 | INFO | PRE_HQ : 66.9%, 1.7, 59.8ms, 0 | 20260508 04:11:29.377 | INFO | BAZWRITER : 82.2%, 4.5, 188.3ms, 0 | 20260508 04:11:29.377 | INFO | BAZ_SAMPLING : 12.5%, 1.0, 6.5ms, 0 | 20260508 04:11:29.378 | INFO | Uploaded 36GiB to GPU in 2.56976s (14.0091GiB/s) | 20260508 04:11:29.378 | INFO | Downloaded 38GiB to GPU in 2.71045s (14.0198GiB/s) | 20260508 04:11:34.516 | INFO | Analyzing chunk frames = [5136896,5137408) | 20260508 04:11:34.516 | INFO | MemInfo: { | 20260508 04:11:34.516 | INFO | "gpu" : " 17 GB", | 20260508 04:11:34.516 | INFO | "huge" : "", | 20260508 04:11:34.516 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:34.516 | INFO | "malloc" : " 14 GB", | 20260508 04:11:34.516 | INFO | "pinned" : " 29 GB" | 20260508 04:11:34.517 | INFO | } | 20260508 04:11:34.517 | INFO | Perf report for the last 5.13946 seconds: | 20260508 04:11:34.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:34.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 04:11:34.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:11:34.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:34.517 | INFO | Uploaded 36GiB to GPU in 2.48375s (14.4942GiB/s) | 20260508 04:11:34.517 | INFO | Downloaded 34GiB to GPU in 2.42813s (14.0025GiB/s) | 20260508 04:11:39.654 | INFO | Analyzing chunk frames = [5137408,5137920) | 20260508 04:11:39.654 | INFO | MemInfo: { | 20260508 04:11:39.654 | INFO | "gpu" : " 17 GB", | 20260508 04:11:39.654 | INFO | "huge" : "", | 20260508 04:11:39.654 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:39.654 | INFO | "malloc" : " 14 GB", | 20260508 04:11:39.654 | INFO | "pinned" : " 29 GB" | 20260508 04:11:39.654 | INFO | } | 20260508 04:11:39.654 | INFO | Perf report for the last 5.13763 seconds: | 20260508 04:11:39.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:39.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.654 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:11:39.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.655 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:11:39.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:39.655 | INFO | Uploaded 36GiB to GPU in 2.54017s (14.1723GiB/s) | 20260508 04:11:39.655 | INFO | Downloaded 34GiB to GPU in 2.40682s (14.1265GiB/s) | 20260508 04:11:44.794 | INFO | Analyzing chunk frames = [5137920,5138432) | 20260508 04:11:44.794 | INFO | MemInfo: { | 20260508 04:11:44.794 | INFO | "gpu" : " 17 GB", | 20260508 04:11:44.794 | INFO | "huge" : "", | 20260508 04:11:44.794 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:44.794 | INFO | "malloc" : " 14 GB", | 20260508 04:11:44.794 | INFO | "pinned" : " 29 GB" | 20260508 04:11:44.794 | INFO | } | 20260508 04:11:44.794 | INFO | Perf report for the last 5.13967 seconds: | 20260508 04:11:44.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:44.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 04:11:44.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:11:44.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:44.794 | INFO | Uploaded 36GiB to GPU in 2.54618s (14.1388GiB/s) | 20260508 04:11:44.794 | INFO | Downloaded 34GiB to GPU in 2.50195s (13.5894GiB/s) | 20260508 04:11:49.927 | INFO | Analyzing chunk frames = [5138432,5138944) | 20260508 04:11:49.927 | INFO | MemInfo: { | 20260508 04:11:49.927 | INFO | "gpu" : " 17 GB", | 20260508 04:11:49.927 | INFO | "huge" : "", | 20260508 04:11:49.927 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:49.927 | INFO | "malloc" : " 14 GB", | 20260508 04:11:49.927 | INFO | "pinned" : " 29 GB" | 20260508 04:11:49.927 | INFO | } | 20260508 04:11:49.927 | INFO | Perf report for the last 5.13304 seconds: | 20260508 04:11:49.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:49.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 04:11:49.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:11:49.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:49.927 | INFO | Uploaded 36GiB to GPU in 2.37134s (15.1813GiB/s) | 20260508 04:11:49.928 | INFO | Downloaded 34GiB to GPU in 2.55834s (13.2899GiB/s) | 20260508 04:11:55.068 | INFO | Analyzing chunk frames = [5138944,5139456) | 20260508 04:11:55.068 | INFO | MemInfo: { | 20260508 04:11:55.068 | INFO | "gpu" : " 17 GB", | 20260508 04:11:55.068 | INFO | "huge" : "", | 20260508 04:11:55.068 | INFO | "huge_shared" : " 29 GB", | 20260508 04:11:55.068 | INFO | "malloc" : " 14 GB", | 20260508 04:11:55.068 | INFO | "pinned" : " 29 GB" | 20260508 04:11:55.068 | INFO | } | 20260508 04:11:55.068 | INFO | Perf report for the last 5.14063 seconds: | 20260508 04:11:55.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:11:55.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:11:55.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:11:55.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:11:55.068 | INFO | Uploaded 36GiB to GPU in 2.42671s (14.8349GiB/s) | 20260508 04:11:55.068 | INFO | Downloaded 34GiB to GPU in 2.42081s (14.0449GiB/s) | 20260508 04:12:00.211 | INFO | Analyzing chunk frames = [5139456,5139968) | 20260508 04:12:00.211 | INFO | MemInfo: { | 20260508 04:12:00.211 | INFO | "gpu" : " 17 GB", | 20260508 04:12:00.211 | INFO | "huge" : "", | 20260508 04:12:00.211 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:00.211 | INFO | "malloc" : " 14 GB", | 20260508 04:12:00.212 | INFO | "pinned" : " 29 GB" | 20260508 04:12:00.212 | INFO | } | 20260508 04:12:00.212 | INFO | Perf report for the last 5.14386 seconds: | 20260508 04:12:00.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:00.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 04:12:00.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:12:00.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:00.212 | INFO | Uploaded 36GiB to GPU in 2.44463s (14.7262GiB/s) | 20260508 04:12:00.212 | INFO | Downloaded 34GiB to GPU in 2.42329s (14.0305GiB/s) | 20260508 04:12:05.351 | INFO | Analyzing chunk frames = [5139968,5140480) | 20260508 04:12:05.351 | INFO | MemInfo: { | 20260508 04:12:05.351 | INFO | "gpu" : " 17 GB", | 20260508 04:12:05.351 | INFO | "huge" : "", | 20260508 04:12:05.351 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:05.351 | INFO | "malloc" : " 14 GB", | 20260508 04:12:05.351 | INFO | "pinned" : " 29 GB" | 20260508 04:12:05.351 | INFO | } | 20260508 04:12:05.351 | INFO | Perf report for the last 5.13897 seconds: | 20260508 04:12:05.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:05.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:12:05.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 04:12:05.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:05.351 | INFO | Uploaded 36GiB to GPU in 2.44655s (14.7146GiB/s) | 20260508 04:12:05.351 | INFO | Downloaded 34GiB to GPU in 2.43275s (13.976GiB/s) | 20260508 04:12:05.466 | INFO | Starting a round of Baz IO | 20260508 04:12:08.905 | INFO | Populating RTMetrics report | 20260508 04:12:08.905 | INFO | RTMetrics report populated | 20260508 04:12:08.905 | INFO | Opening tmp RTMetrics json file | 20260508 04:12:08.917 | INFO | Serializing RTMetrics to json | 20260508 04:12:08.919 | INFO | Serialization done, swapping tmp file now... | 20260508 04:12:08.919 | INFO | RTMetrics done | 20260508 04:12:08.984 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.518 seconds | 20260508 04:12:08.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.11% / 0.00% / 58.88% | 20260508 04:12:08.985 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.5542% / 7.94729e-06% / 6.75519e-05% | 20260508 04:12:08.985 | INFO | Baz IO overall: | 20260508 04:12:08.985 | INFO | BytesWritten : 2303940718410 | 20260508 04:12:08.985 | INFO | NumEvents : 1112534781356 | 20260508 04:12:10.488 | INFO | Analyzing chunk frames = [5140480,5140992) | 20260508 04:12:10.488 | INFO | MemInfo: { | 20260508 04:12:10.488 | INFO | "gpu" : " 17 GB", | 20260508 04:12:10.488 | INFO | "huge" : "", | 20260508 04:12:10.488 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:10.488 | INFO | "malloc" : " 14 GB", | 20260508 04:12:10.489 | INFO | "pinned" : " 29 GB" | 20260508 04:12:10.489 | INFO | } | 20260508 04:12:10.489 | INFO | Perf report for the last 5.13804 seconds: | 20260508 04:12:10.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:10.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:10.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:10.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:10.489 | INFO | ANALYSIS : 68.3%, 2.9, 104.6ms, 0 | 20260508 04:12:10.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:10.489 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 04:12:10.489 | INFO | PRE_HQ : 66.9%, 1.8, 61.4ms, 0 | 20260508 04:12:10.489 | INFO | BAZWRITER : 68.1%, 3.8, 133.6ms, 0 | 20260508 04:12:10.489 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 04:12:10.489 | INFO | Uploaded 36GiB to GPU in 2.53731s (14.1882GiB/s) | 20260508 04:12:10.489 | INFO | Downloaded 38GiB to GPU in 2.70331s (14.0568GiB/s) | 20260508 04:12:15.621 | INFO | Analyzing chunk frames = [5140992,5141504) | 20260508 04:12:15.621 | INFO | MemInfo: { | 20260508 04:12:15.621 | INFO | "gpu" : " 17 GB", | 20260508 04:12:15.621 | INFO | "huge" : "", | 20260508 04:12:15.621 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:15.621 | INFO | "malloc" : " 14 GB", | 20260508 04:12:15.621 | INFO | "pinned" : " 29 GB" | 20260508 04:12:15.621 | INFO | } | 20260508 04:12:15.622 | INFO | Perf report for the last 5.13282 seconds: | 20260508 04:12:15.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:15.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:12:15.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:12:15.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:15.622 | INFO | Uploaded 36GiB to GPU in 2.57948s (13.9563GiB/s) | 20260508 04:12:15.622 | INFO | Downloaded 34GiB to GPU in 2.44119s (13.9277GiB/s) | 20260508 04:12:20.768 | INFO | Analyzing chunk frames = [5141504,5142016) | 20260508 04:12:20.768 | INFO | MemInfo: { | 20260508 04:12:20.768 | INFO | "gpu" : " 17 GB", | 20260508 04:12:20.768 | INFO | "huge" : "", | 20260508 04:12:20.768 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:20.768 | INFO | "malloc" : " 14 GB", | 20260508 04:12:20.768 | INFO | "pinned" : " 29 GB" | 20260508 04:12:20.768 | INFO | } | 20260508 04:12:20.768 | INFO | Perf report for the last 5.14657 seconds: | 20260508 04:12:20.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:20.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 04:12:20.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:12:20.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:20.768 | INFO | Uploaded 36GiB to GPU in 2.54624s (14.1385GiB/s) | 20260508 04:12:20.769 | INFO | Downloaded 34GiB to GPU in 2.4923s (13.642GiB/s) | 20260508 04:12:25.906 | INFO | Analyzing chunk frames = [5142016,5142528) | 20260508 04:12:25.906 | INFO | MemInfo: { | 20260508 04:12:25.906 | INFO | "gpu" : " 17 GB", | 20260508 04:12:25.906 | INFO | "huge" : "", | 20260508 04:12:25.906 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:25.906 | INFO | "malloc" : " 14 GB", | 20260508 04:12:25.906 | INFO | "pinned" : " 29 GB" | 20260508 04:12:25.906 | INFO | } | 20260508 04:12:25.906 | INFO | Perf report for the last 5.13819 seconds: | 20260508 04:12:25.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:25.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.906 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:12:25.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.906 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:12:25.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:25.907 | INFO | Uploaded 36GiB to GPU in 2.56919s (14.0122GiB/s) | 20260508 04:12:25.907 | INFO | Downloaded 34GiB to GPU in 2.41553s (14.0756GiB/s) | 20260508 04:12:31.051 | INFO | Analyzing chunk frames = [5142528,5143040) | 20260508 04:12:31.051 | INFO | MemInfo: { | 20260508 04:12:31.051 | INFO | "gpu" : " 17 GB", | 20260508 04:12:31.051 | INFO | "huge" : "", | 20260508 04:12:31.051 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:31.051 | INFO | "malloc" : " 14 GB", | 20260508 04:12:31.051 | INFO | "pinned" : " 29 GB" | 20260508 04:12:31.051 | INFO | } | 20260508 04:12:31.051 | INFO | Perf report for the last 5.14489 seconds: | 20260508 04:12:31.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:31.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 04:12:31.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:12:31.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:31.051 | INFO | Uploaded 36GiB to GPU in 2.40589s (14.9633GiB/s) | 20260508 04:12:31.052 | INFO | Downloaded 34GiB to GPU in 2.56922s (13.2336GiB/s) | 20260508 04:12:36.188 | INFO | Analyzing chunk frames = [5143040,5143552) | 20260508 04:12:36.188 | INFO | MemInfo: { | 20260508 04:12:36.188 | INFO | "gpu" : " 17 GB", | 20260508 04:12:36.188 | INFO | "huge" : "", | 20260508 04:12:36.188 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:36.188 | INFO | "malloc" : " 14 GB", | 20260508 04:12:36.188 | INFO | "pinned" : " 29 GB" | 20260508 04:12:36.188 | INFO | } | 20260508 04:12:36.188 | INFO | Perf report for the last 5.13723 seconds: | 20260508 04:12:36.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:36.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.189 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:12:36.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.189 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 04:12:36.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:36.189 | INFO | Uploaded 36GiB to GPU in 2.42691s (14.8337GiB/s) | 20260508 04:12:36.189 | INFO | Downloaded 34GiB to GPU in 2.41909s (14.0549GiB/s) | 20260508 04:12:41.323 | INFO | Analyzing chunk frames = [5143552,5144064) | 20260508 04:12:41.323 | INFO | MemInfo: { | 20260508 04:12:41.323 | INFO | "gpu" : " 17 GB", | 20260508 04:12:41.323 | INFO | "huge" : "", | 20260508 04:12:41.323 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:41.323 | INFO | "malloc" : " 14 GB", | 20260508 04:12:41.323 | INFO | "pinned" : " 29 GB" | 20260508 04:12:41.323 | INFO | } | 20260508 04:12:41.324 | INFO | Perf report for the last 5.13502 seconds: | 20260508 04:12:41.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:41.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.324 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:12:41.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.324 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:12:41.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:41.325 | INFO | Uploaded 36GiB to GPU in 2.42535s (14.8432GiB/s) | 20260508 04:12:41.325 | INFO | Downloaded 34GiB to GPU in 2.41932s (14.0535GiB/s) | 20260508 04:12:46.466 | INFO | Analyzing chunk frames = [5144064,5144576) | 20260508 04:12:46.466 | INFO | MemInfo: { | 20260508 04:12:46.466 | INFO | "gpu" : " 17 GB", | 20260508 04:12:46.466 | INFO | "huge" : "", | 20260508 04:12:46.466 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:46.467 | INFO | "malloc" : " 14 GB", | 20260508 04:12:46.467 | INFO | "pinned" : " 29 GB" | 20260508 04:12:46.467 | INFO | } | 20260508 04:12:46.467 | INFO | Perf report for the last 5.14333 seconds: | 20260508 04:12:46.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:46.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:12:46.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 04:12:46.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:46.467 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:12:46.467 | INFO | Total mem usage: 16.7 GiB | 20260508 04:12:46.467 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:12:46.467 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.4%) | 20260508 04:12:46.467 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:12:46.467 | INFO | High Water Breakdown: | 20260508 04:12:46.467 | INFO | 60.2% : 9.02 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:12:46.467 | INFO | 39.7% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:12:46.467 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:12:46.467 | INFO | Uploaded 36GiB to GPU in 2.4459s (14.7185GiB/s) | 20260508 04:12:46.467 | INFO | Downloaded 34GiB to GPU in 2.41588s (14.0735GiB/s) | 20260508 04:12:46.467 | INFO | ------------------------------------------ | 20260508 04:12:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:12:46.468 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:12:46.468 | INFO | Total mem usage: 26.5 GiB | 20260508 04:12:46.468 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:12:46.468 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:12:46.468 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:12:46.468 | INFO | High Water Breakdown: | 20260508 04:12:46.468 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:12:46.468 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:12:46.468 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:12:46.468 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:12:46.468 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:12:46.468 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:12:46.468 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:12:46.468 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:12:46.468 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:12:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:12:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:12:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:12:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:12:46.468 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:12:46.468 | INFO | ------------------------------------------ | 20260508 04:12:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:12:46.468 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:12:46.468 | INFO | Total mem usage: 27.8 GiB | 20260508 04:12:46.468 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:12:46.468 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:12:46.468 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:12:46.468 | INFO | High Water Breakdown: | 20260508 04:12:46.468 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:12:46.468 | INFO | ------------------------------------------ | 20260508 04:12:46.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:12:46.468 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:12:46.468 | INFO | Total mem usage: 15.7 GiB | 20260508 04:12:46.468 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:12:46.468 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:12:46.468 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:12:46.468 | INFO | High Water Breakdown: | 20260508 04:12:46.468 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:12:46.468 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:12:46.468 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:12:46.468 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:12:46.468 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:12:46.468 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:12:46.468 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:12:46.468 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:12:46.468 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:12:46.468 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:12:46.469 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:12:46.469 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:12:46.469 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:12:46.469 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:12:46.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:12:46.469 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:12:46.469 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:12:46.469 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:12:46.469 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:12:46.469 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:12:46.469 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:12:46.469 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:12:46.469 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:12:46.469 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:12:46.469 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:12:46.469 | INFO | ------------------------------------------ | 20260508 04:12:46.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:12:46.469 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:12:46.469 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:12:46.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:12:46.469 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.61e+01, 1.42e+01 / 2.02e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.02e-01, 1.23e-02 / 5.32e-03 | 20260508 04:12:46.469 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.44e+00, 5.89e-01 / 2.06e-01 | 20260508 04:12:46.469 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.11e-01 / 1.02e-01 | 20260508 04:12:46.469 | INFO | PerfReport: 16.01%, DME -- 4.20e+01 / 4.60e+01, 3.81e+00 / 2.39e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.77e+00 / 1.18e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.18e-01 / 2.07e-01 | 20260508 04:12:46.469 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.20e+00, 2.64e-01 / 1.27e-01 | 20260508 04:12:46.469 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.55e-01, 5.09e-02 / 2.46e-02 | 20260508 04:12:46.469 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:12:46.469 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:12:46.469 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:12:46.469 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 24.39%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 30.87%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 2.14%, LockUpload -- 2.27e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 2.72%, LockDownload -- 2.89e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 15.47%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 04:12:46.469 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 2.66e-02 / 0.00e+00 | 20260508 04:12:46.580 | INFO | Starting a round of Baz IO | 20260508 04:12:50.025 | INFO | Populating RTMetrics report | 20260508 04:12:50.025 | INFO | RTMetrics report populated | 20260508 04:12:50.025 | INFO | Opening tmp RTMetrics json file | 20260508 04:12:50.041 | INFO | Serializing RTMetrics to json | 20260508 04:12:50.044 | INFO | Serialization done, swapping tmp file now... | 20260508 04:12:50.044 | INFO | RTMetrics done | 20260508 04:12:50.147 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.566 seconds | 20260508 04:12:50.147 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.11% / 0.00% / 58.89% | 20260508 04:12:50.147 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.552921% / 3.97364e-06% / 6.75519e-05% | 20260508 04:12:50.147 | INFO | Baz IO overall: | 20260508 04:12:50.147 | INFO | BytesWritten : 2305656686258 | 20260508 04:12:50.147 | INFO | NumEvents : 1113354806594 | 20260508 04:12:51.604 | INFO | Analyzing chunk frames = [5144576,5145088) | 20260508 04:12:51.604 | INFO | MemInfo: { | 20260508 04:12:51.604 | INFO | "gpu" : " 17 GB", | 20260508 04:12:51.604 | INFO | "huge" : "", | 20260508 04:12:51.604 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:51.604 | INFO | "malloc" : " 14 GB", | 20260508 04:12:51.604 | INFO | "pinned" : " 29 GB" | 20260508 04:12:51.605 | INFO | } | 20260508 04:12:51.605 | INFO | Perf report for the last 5.1377 seconds: | 20260508 04:12:51.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:51.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:51.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:51.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:51.605 | INFO | ANALYSIS : 68.6%, 2.9, 104.7ms, 0 | 20260508 04:12:51.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:51.605 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 04:12:51.605 | INFO | PRE_HQ : 67.2%, 1.8, 61.2ms, 0 | 20260508 04:12:51.605 | INFO | BAZWRITER : 69.0%, 3.7, 131.4ms, 0 | 20260508 04:12:51.605 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:12:51.605 | INFO | Uploaded 36GiB to GPU in 2.54396s (14.1511GiB/s) | 20260508 04:12:51.605 | INFO | Downloaded 38GiB to GPU in 2.71968s (13.9722GiB/s) | 20260508 04:12:56.744 | INFO | Analyzing chunk frames = [5145088,5145600) | 20260508 04:12:56.744 | INFO | MemInfo: { | 20260508 04:12:56.744 | INFO | "gpu" : " 17 GB", | 20260508 04:12:56.744 | INFO | "huge" : "", | 20260508 04:12:56.744 | INFO | "huge_shared" : " 29 GB", | 20260508 04:12:56.744 | INFO | "malloc" : " 14 GB", | 20260508 04:12:56.744 | INFO | "pinned" : " 29 GB" | 20260508 04:12:56.744 | INFO | } | 20260508 04:12:56.744 | INFO | Perf report for the last 5.1396 seconds: | 20260508 04:12:56.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:12:56.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 04:12:56.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:12:56.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:12:56.744 | INFO | Uploaded 36GiB to GPU in 2.55308s (14.1006GiB/s) | 20260508 04:12:56.744 | INFO | Downloaded 34GiB to GPU in 2.42401s (14.0263GiB/s) | 20260508 04:13:01.878 | INFO | Analyzing chunk frames = [5145600,5146112) | 20260508 04:13:01.878 | INFO | MemInfo: { | 20260508 04:13:01.878 | INFO | "gpu" : " 17 GB", | 20260508 04:13:01.878 | INFO | "huge" : "", | 20260508 04:13:01.878 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:01.878 | INFO | "malloc" : " 14 GB", | 20260508 04:13:01.878 | INFO | "pinned" : " 29 GB" | 20260508 04:13:01.878 | INFO | } | 20260508 04:13:01.878 | INFO | Perf report for the last 5.13346 seconds: | 20260508 04:13:01.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:01.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 04:13:01.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:13:01.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:01.878 | INFO | Uploaded 36GiB to GPU in 2.57031s (14.0061GiB/s) | 20260508 04:13:01.878 | INFO | Downloaded 34GiB to GPU in 2.49758s (13.6132GiB/s) | 20260508 04:13:07.019 | INFO | Analyzing chunk frames = [5146112,5146624) | 20260508 04:13:07.019 | INFO | MemInfo: { | 20260508 04:13:07.019 | INFO | "gpu" : " 17 GB", | 20260508 04:13:07.019 | INFO | "huge" : "", | 20260508 04:13:07.019 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:07.019 | INFO | "malloc" : " 14 GB", | 20260508 04:13:07.019 | INFO | "pinned" : " 29 GB" | 20260508 04:13:07.019 | INFO | } | 20260508 04:13:07.019 | INFO | Perf report for the last 5.14122 seconds: | 20260508 04:13:07.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:07.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.019 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 04:13:07.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.019 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:13:07.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:07.020 | INFO | Uploaded 36GiB to GPU in 2.53805s (14.1841GiB/s) | 20260508 04:13:07.020 | INFO | Downloaded 34GiB to GPU in 2.50828s (13.5551GiB/s) | 20260508 04:13:12.161 | INFO | Analyzing chunk frames = [5146624,5147136) | 20260508 04:13:12.161 | INFO | MemInfo: { | 20260508 04:13:12.161 | INFO | "gpu" : " 17 GB", | 20260508 04:13:12.161 | INFO | "huge" : "", | 20260508 04:13:12.161 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:12.161 | INFO | "malloc" : " 14 GB", | 20260508 04:13:12.161 | INFO | "pinned" : " 29 GB" | 20260508 04:13:12.161 | INFO | } | 20260508 04:13:12.161 | INFO | Perf report for the last 5.14203 seconds: | 20260508 04:13:12.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:12.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 04:13:12.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:13:12.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:12.162 | INFO | Uploaded 36GiB to GPU in 2.41943s (14.8795GiB/s) | 20260508 04:13:12.162 | INFO | Downloaded 34GiB to GPU in 2.56393s (13.2609GiB/s) | 20260508 04:13:17.303 | INFO | Analyzing chunk frames = [5147136,5147648) | 20260508 04:13:17.304 | INFO | MemInfo: { | 20260508 04:13:17.304 | INFO | "gpu" : " 17 GB", | 20260508 04:13:17.304 | INFO | "huge" : "", | 20260508 04:13:17.304 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:17.304 | INFO | "malloc" : " 14 GB", | 20260508 04:13:17.304 | INFO | "pinned" : " 29 GB" | 20260508 04:13:17.304 | INFO | } | 20260508 04:13:17.304 | INFO | Perf report for the last 5.14285 seconds: | 20260508 04:13:17.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:17.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 04:13:17.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:13:17.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:17.304 | INFO | Uploaded 36GiB to GPU in 2.41113s (14.9308GiB/s) | 20260508 04:13:17.304 | INFO | Downloaded 34GiB to GPU in 2.42681s (14.0102GiB/s) | 20260508 04:13:22.438 | INFO | Analyzing chunk frames = [5147648,5148160) | 20260508 04:13:22.438 | INFO | MemInfo: { | 20260508 04:13:22.438 | INFO | "gpu" : " 17 GB", | 20260508 04:13:22.438 | INFO | "huge" : "", | 20260508 04:13:22.438 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:22.438 | INFO | "malloc" : " 14 GB", | 20260508 04:13:22.438 | INFO | "pinned" : " 29 GB" | 20260508 04:13:22.438 | INFO | } | 20260508 04:13:22.438 | INFO | Perf report for the last 5.13432 seconds: | 20260508 04:13:22.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:22.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.438 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:13:22.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.438 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:13:22.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:22.439 | INFO | Uploaded 36GiB to GPU in 2.42628s (14.8375GiB/s) | 20260508 04:13:22.439 | INFO | Downloaded 34GiB to GPU in 2.4236s (14.0287GiB/s) | 20260508 04:13:27.579 | INFO | Analyzing chunk frames = [5148160,5148672) | 20260508 04:13:27.579 | INFO | MemInfo: { | 20260508 04:13:27.579 | INFO | "gpu" : " 17 GB", | 20260508 04:13:27.579 | INFO | "huge" : "", | 20260508 04:13:27.579 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:27.579 | INFO | "malloc" : " 14 GB", | 20260508 04:13:27.579 | INFO | "pinned" : " 29 GB" | 20260508 04:13:27.579 | INFO | } | 20260508 04:13:27.579 | INFO | Perf report for the last 5.14058 seconds: | 20260508 04:13:27.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:27.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:13:27.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260508 04:13:27.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:27.579 | INFO | Uploaded 36GiB to GPU in 2.45266s (14.678GiB/s) | 20260508 04:13:27.579 | INFO | Downloaded 34GiB to GPU in 2.43521s (13.9618GiB/s) | 20260508 04:13:27.676 | INFO | Starting a round of Baz IO | 20260508 04:13:31.108 | INFO | Populating RTMetrics report | 20260508 04:13:31.108 | INFO | RTMetrics report populated | 20260508 04:13:31.108 | INFO | Opening tmp RTMetrics json file | 20260508 04:13:31.235 | INFO | Serializing RTMetrics to json | 20260508 04:13:31.253 | INFO | Serialization done, swapping tmp file now... | 20260508 04:13:31.253 | INFO | RTMetrics done | 20260508 04:13:31.386 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.709 seconds | 20260508 04:13:31.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.11% / 0.00% / 58.89% | 20260508 04:13:31.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.551732% / 3.97364e-06% / 4.37101e-05% | 20260508 04:13:31.386 | INFO | Baz IO overall: | 20260508 04:13:31.386 | INFO | BytesWritten : 2307372428647 | 20260508 04:13:31.386 | INFO | NumEvents : 1114174505813 | 20260508 04:13:32.719 | INFO | Analyzing chunk frames = [5148672,5149184) | 20260508 04:13:32.719 | INFO | MemInfo: { | 20260508 04:13:32.719 | INFO | "gpu" : " 17 GB", | 20260508 04:13:32.719 | INFO | "huge" : "", | 20260508 04:13:32.719 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:32.719 | INFO | "malloc" : " 14 GB", | 20260508 04:13:32.719 | INFO | "pinned" : " 29 GB" | 20260508 04:13:32.719 | INFO | } | 20260508 04:13:32.719 | INFO | Perf report for the last 5.14027 seconds: | 20260508 04:13:32.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:32.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:32.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:32.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:32.719 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260508 04:13:32.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:32.719 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260508 04:13:32.719 | INFO | PRE_HQ : 65.7%, 1.6, 55.4ms, 0 | 20260508 04:13:32.719 | INFO | BAZWRITER : 71.7%, 3.7, 136.0ms, 0 | 20260508 04:13:32.719 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260508 04:13:32.719 | INFO | Uploaded 36GiB to GPU in 2.53884s (14.1797GiB/s) | 20260508 04:13:32.719 | INFO | Downloaded 38GiB to GPU in 2.70115s (14.0681GiB/s) | 20260508 04:13:37.857 | INFO | Analyzing chunk frames = [5149184,5149696) | 20260508 04:13:37.857 | INFO | MemInfo: { | 20260508 04:13:37.857 | INFO | "gpu" : " 17 GB", | 20260508 04:13:37.857 | INFO | "huge" : "", | 20260508 04:13:37.857 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:37.857 | INFO | "malloc" : " 14 GB", | 20260508 04:13:37.857 | INFO | "pinned" : " 29 GB" | 20260508 04:13:37.857 | INFO | } | 20260508 04:13:37.857 | INFO | Perf report for the last 5.13837 seconds: | 20260508 04:13:37.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:37.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.857 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 04:13:37.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.858 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:13:37.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:37.858 | INFO | Uploaded 36GiB to GPU in 2.55264s (14.103GiB/s) | 20260508 04:13:37.858 | INFO | Downloaded 34GiB to GPU in 2.4278s (14.0044GiB/s) | 20260508 04:13:43.001 | INFO | Analyzing chunk frames = [5149696,5150208) | 20260508 04:13:43.001 | INFO | MemInfo: { | 20260508 04:13:43.001 | INFO | "gpu" : " 17 GB", | 20260508 04:13:43.001 | INFO | "huge" : "", | 20260508 04:13:43.001 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:43.001 | INFO | "malloc" : " 14 GB", | 20260508 04:13:43.001 | INFO | "pinned" : " 29 GB" | 20260508 04:13:43.001 | INFO | } | 20260508 04:13:43.001 | INFO | Perf report for the last 5.14383 seconds: | 20260508 04:13:43.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:43.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.001 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:13:43.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.001 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:13:43.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:43.002 | INFO | Uploaded 36GiB to GPU in 2.53546s (14.1986GiB/s) | 20260508 04:13:43.002 | INFO | Downloaded 34GiB to GPU in 2.50492s (13.5733GiB/s) | 20260508 04:13:48.138 | INFO | Analyzing chunk frames = [5150208,5150720) | 20260508 04:13:48.138 | INFO | MemInfo: { | 20260508 04:13:48.138 | INFO | "gpu" : " 17 GB", | 20260508 04:13:48.138 | INFO | "huge" : "", | 20260508 04:13:48.138 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:48.138 | INFO | "malloc" : " 14 GB", | 20260508 04:13:48.138 | INFO | "pinned" : " 29 GB" | 20260508 04:13:48.138 | INFO | } | 20260508 04:13:48.138 | INFO | Perf report for the last 5.13672 seconds: | 20260508 04:13:48.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:48.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 04:13:48.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:13:48.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:48.138 | INFO | Uploaded 36GiB to GPU in 2.5167s (14.3044GiB/s) | 20260508 04:13:48.138 | INFO | Downloaded 34GiB to GPU in 2.38207s (14.2733GiB/s) | 20260508 04:13:53.279 | INFO | Analyzing chunk frames = [5150720,5151232) | 20260508 04:13:53.279 | INFO | MemInfo: { | 20260508 04:13:53.279 | INFO | "gpu" : " 17 GB", | 20260508 04:13:53.279 | INFO | "huge" : "", | 20260508 04:13:53.279 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:53.279 | INFO | "malloc" : " 14 GB", | 20260508 04:13:53.279 | INFO | "pinned" : " 29 GB" | 20260508 04:13:53.279 | INFO | } | 20260508 04:13:53.279 | INFO | Perf report for the last 5.14102 seconds: | 20260508 04:13:53.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:53.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:13:53.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:13:53.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:53.279 | INFO | Uploaded 36GiB to GPU in 2.36265s (15.2371GiB/s) | 20260508 04:13:53.279 | INFO | Downloaded 34GiB to GPU in 2.55509s (13.3068GiB/s) | 20260508 04:13:58.417 | INFO | Analyzing chunk frames = [5151232,5151744) | 20260508 04:13:58.417 | INFO | MemInfo: { | 20260508 04:13:58.417 | INFO | "gpu" : " 17 GB", | 20260508 04:13:58.417 | INFO | "huge" : "", | 20260508 04:13:58.417 | INFO | "huge_shared" : " 29 GB", | 20260508 04:13:58.417 | INFO | "malloc" : " 14 GB", | 20260508 04:13:58.417 | INFO | "pinned" : " 29 GB" | 20260508 04:13:58.417 | INFO | } | 20260508 04:13:58.417 | INFO | Perf report for the last 5.13782 seconds: | 20260508 04:13:58.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:13:58.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:13:58.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 04:13:58.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:13:58.417 | INFO | Uploaded 36GiB to GPU in 2.45551s (14.6609GiB/s) | 20260508 04:13:58.417 | INFO | Downloaded 34GiB to GPU in 2.41429s (14.0828GiB/s) | 20260508 04:14:03.553 | INFO | Analyzing chunk frames = [5151744,5152256) | 20260508 04:14:03.553 | INFO | MemInfo: { | 20260508 04:14:03.553 | INFO | "gpu" : " 17 GB", | 20260508 04:14:03.553 | INFO | "huge" : "", | 20260508 04:14:03.553 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:03.553 | INFO | "malloc" : " 14 GB", | 20260508 04:14:03.553 | INFO | "pinned" : " 29 GB" | 20260508 04:14:03.553 | INFO | } | 20260508 04:14:03.553 | INFO | Perf report for the last 5.13653 seconds: | 20260508 04:14:03.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:03.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.553 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:14:03.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.554 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:14:03.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:03.554 | INFO | Uploaded 36GiB to GPU in 2.4513s (14.6861GiB/s) | 20260508 04:14:03.554 | INFO | Downloaded 34GiB to GPU in 2.4241s (14.0258GiB/s) | 20260508 04:14:08.693 | INFO | Analyzing chunk frames = [5152256,5152768) | 20260508 04:14:08.693 | INFO | MemInfo: { | 20260508 04:14:08.693 | INFO | "gpu" : " 17 GB", | 20260508 04:14:08.693 | INFO | "huge" : "", | 20260508 04:14:08.693 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:08.693 | INFO | "malloc" : " 14 GB", | 20260508 04:14:08.694 | INFO | "pinned" : " 29 GB" | 20260508 04:14:08.694 | INFO | } | 20260508 04:14:08.694 | INFO | Perf report for the last 5.14043 seconds: | 20260508 04:14:08.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:08.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:14:08.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:14:08.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:08.694 | INFO | Uploaded 36GiB to GPU in 2.47722s (14.5324GiB/s) | 20260508 04:14:08.694 | INFO | Downloaded 34GiB to GPU in 2.42597s (14.015GiB/s) | 20260508 04:14:08.694 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:14:08.694 | INFO | Total mem usage: 16.7 GiB | 20260508 04:14:08.694 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:14:08.694 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.4%) | 20260508 04:14:08.695 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:14:08.695 | INFO | High Water Breakdown: | 20260508 04:14:08.695 | INFO | 60.2% : 9.01 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:14:08.695 | INFO | 39.7% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:14:08.695 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:14:08.695 | INFO | ------------------------------------------ | 20260508 04:14:08.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:14:08.695 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:14:08.695 | INFO | Total mem usage: 26.5 GiB | 20260508 04:14:08.695 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:14:08.695 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260508 04:14:08.695 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260508 04:14:08.695 | INFO | High Water Breakdown: | 20260508 04:14:08.695 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:14:08.695 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:14:08.695 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:14:08.695 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:14:08.695 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:14:08.695 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:14:08.695 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:14:08.695 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:14:08.695 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:14:08.695 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:14:08.695 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:14:08.695 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:14:08.695 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:14:08.695 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:14:08.695 | INFO | ------------------------------------------ | 20260508 04:14:08.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:14:08.695 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:14:08.695 | INFO | Total mem usage: 27.8 GiB | 20260508 04:14:08.695 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:14:08.695 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:14:08.695 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:14:08.695 | INFO | High Water Breakdown: | 20260508 04:14:08.695 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:14:08.695 | INFO | ------------------------------------------ | 20260508 04:14:08.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:14:08.695 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:14:08.695 | INFO | Total mem usage: 15.7 GiB | 20260508 04:14:08.695 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:14:08.695 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:14:08.695 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:14:08.695 | INFO | High Water Breakdown: | 20260508 04:14:08.695 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:14:08.695 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:14:08.695 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:14:08.695 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:14:08.695 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:14:08.695 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:14:08.695 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:14:08.695 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:14:08.695 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:14:08.695 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:14:08.695 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:14:08.695 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:14:08.695 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:14:08.695 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:14:08.695 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:14:08.695 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:14:08.695 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:14:08.695 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:14:08.695 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:14:08.695 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:14:08.695 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:14:08.695 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:14:08.695 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:14:08.695 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:14:08.695 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:14:08.695 | INFO | ------------------------------------------ | 20260508 04:14:08.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:14:08.695 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:14:08.695 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:14:08.695 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:14:08.696 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.43e+01 / 1.60e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.10e-02 / 4.10e-03 | 20260508 04:14:08.696 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.44e+00, 5.80e-01 / 2.03e-01 | 20260508 04:14:08.696 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 9.05e-02 | 20260508 04:14:08.696 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.59e+01, 4.02e+00 / 2.12e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.66e+00 / 1.36e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.40e+00 / 2.64e+00, 4.17e-01 / 2.42e-01 | 20260508 04:14:08.696 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.20e+00, 2.65e-01 / 1.19e-01 | 20260508 04:14:08.696 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.60e-01, 5.03e-02 / 2.29e-02 | 20260508 04:14:08.696 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:14:08.696 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:14:08.696 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:14:08.696 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 24.41%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 2.16%, LockUpload -- 2.30e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 2.84%, LockDownload -- 3.02e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 15.44%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 04:14:08.696 | INFO | PerfReport: 0.01%, Misc -- 1.54e-02 / 0.00e+00, 3.32e-02 / 0.00e+00 | 20260508 04:14:08.792 | INFO | Starting a round of Baz IO | 20260508 04:14:12.219 | INFO | Populating RTMetrics report | 20260508 04:14:12.220 | INFO | RTMetrics report populated | 20260508 04:14:12.220 | INFO | Opening tmp RTMetrics json file | 20260508 04:14:12.307 | INFO | Serializing RTMetrics to json | 20260508 04:14:12.311 | INFO | Serialization done, swapping tmp file now... | 20260508 04:14:12.311 | INFO | RTMetrics done | 20260508 04:14:12.327 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.535 seconds | 20260508 04:14:12.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.11% / 0.00% / 58.89% | 20260508 04:14:12.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.550107% / 0% / 5.96046e-05% | 20260508 04:14:12.327 | INFO | Baz IO overall: | 20260508 04:14:12.327 | INFO | BytesWritten : 2309088092687 | 20260508 04:14:12.327 | INFO | NumEvents : 1114994136653 | 20260508 04:14:13.828 | INFO | Analyzing chunk frames = [5152768,5153280) | 20260508 04:14:13.828 | INFO | MemInfo: { | 20260508 04:14:13.828 | INFO | "gpu" : " 17 GB", | 20260508 04:14:13.828 | INFO | "huge" : "", | 20260508 04:14:13.828 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:13.828 | INFO | "malloc" : " 14 GB", | 20260508 04:14:13.828 | INFO | "pinned" : " 29 GB" | 20260508 04:14:13.828 | INFO | } | 20260508 04:14:13.828 | INFO | Perf report for the last 5.13404 seconds: | 20260508 04:14:13.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:13.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:13.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:13.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:13.828 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 04:14:13.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:13.828 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260508 04:14:13.828 | INFO | PRE_HQ : 65.7%, 1.6, 55.9ms, 0 | 20260508 04:14:13.828 | INFO | BAZWRITER : 68.4%, 3.4, 120.2ms, 0 | 20260508 04:14:13.828 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 04:14:13.828 | INFO | Uploaded 36GiB to GPU in 2.56188s (14.0522GiB/s) | 20260508 04:14:13.828 | INFO | Downloaded 38GiB to GPU in 2.70359s (14.0554GiB/s) | 20260508 04:14:18.973 | INFO | Analyzing chunk frames = [5153280,5153792) | 20260508 04:14:18.973 | INFO | MemInfo: { | 20260508 04:14:18.973 | INFO | "gpu" : " 17 GB", | 20260508 04:14:18.973 | INFO | "huge" : "", | 20260508 04:14:18.973 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:18.973 | INFO | "malloc" : " 14 GB", | 20260508 04:14:18.973 | INFO | "pinned" : " 29 GB" | 20260508 04:14:18.973 | INFO | } | 20260508 04:14:18.973 | INFO | Perf report for the last 5.14494 seconds: | 20260508 04:14:18.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:18.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 04:14:18.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:14:18.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:18.973 | INFO | Uploaded 36GiB to GPU in 2.58648s (13.9185GiB/s) | 20260508 04:14:18.973 | INFO | Downloaded 34GiB to GPU in 2.42399s (14.0265GiB/s) | 20260508 04:14:24.110 | INFO | Analyzing chunk frames = [5153792,5154304) | 20260508 04:14:24.110 | INFO | MemInfo: { | 20260508 04:14:24.110 | INFO | "gpu" : " 17 GB", | 20260508 04:14:24.110 | INFO | "huge" : "", | 20260508 04:14:24.110 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:24.110 | INFO | "malloc" : " 14 GB", | 20260508 04:14:24.110 | INFO | "pinned" : " 29 GB" | 20260508 04:14:24.110 | INFO | } | 20260508 04:14:24.110 | INFO | Perf report for the last 5.13733 seconds: | 20260508 04:14:24.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:24.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 04:14:24.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:14:24.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:24.110 | INFO | Uploaded 36GiB to GPU in 2.55842s (14.0712GiB/s) | 20260508 04:14:24.110 | INFO | Downloaded 34GiB to GPU in 2.42016s (14.0487GiB/s) | 20260508 04:14:29.250 | INFO | Analyzing chunk frames = [5154304,5154816) | 20260508 04:14:29.250 | INFO | MemInfo: { | 20260508 04:14:29.250 | INFO | "gpu" : " 17 GB", | 20260508 04:14:29.250 | INFO | "huge" : "", | 20260508 04:14:29.250 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:29.251 | INFO | "malloc" : " 14 GB", | 20260508 04:14:29.251 | INFO | "pinned" : " 29 GB" | 20260508 04:14:29.251 | INFO | } | 20260508 04:14:29.251 | INFO | Perf report for the last 5.14047 seconds: | 20260508 04:14:29.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:29.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 04:14:29.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:14:29.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:29.251 | INFO | Uploaded 36GiB to GPU in 2.57602s (13.975GiB/s) | 20260508 04:14:29.251 | INFO | Downloaded 34GiB to GPU in 2.49536s (13.6253GiB/s) | 20260508 04:14:34.387 | INFO | Analyzing chunk frames = [5154816,5155328) | 20260508 04:14:34.387 | INFO | MemInfo: { | 20260508 04:14:34.387 | INFO | "gpu" : " 17 GB", | 20260508 04:14:34.387 | INFO | "huge" : "", | 20260508 04:14:34.387 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:34.387 | INFO | "malloc" : " 14 GB", | 20260508 04:14:34.387 | INFO | "pinned" : " 29 GB" | 20260508 04:14:34.387 | INFO | } | 20260508 04:14:34.387 | INFO | Perf report for the last 5.13666 seconds: | 20260508 04:14:34.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:34.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.387 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 04:14:34.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.387 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 04:14:34.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:34.388 | INFO | Uploaded 36GiB to GPU in 2.36658s (15.2119GiB/s) | 20260508 04:14:34.388 | INFO | Downloaded 34GiB to GPU in 2.53678s (13.4028GiB/s) | 20260508 04:14:39.533 | INFO | Analyzing chunk frames = [5155328,5155840) | 20260508 04:14:39.533 | INFO | MemInfo: { | 20260508 04:14:39.533 | INFO | "gpu" : " 17 GB", | 20260508 04:14:39.533 | INFO | "huge" : "", | 20260508 04:14:39.533 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:39.534 | INFO | "malloc" : " 14 GB", | 20260508 04:14:39.534 | INFO | "pinned" : " 29 GB" | 20260508 04:14:39.534 | INFO | } | 20260508 04:14:39.534 | INFO | Perf report for the last 5.14631 seconds: | 20260508 04:14:39.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:39.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 04:14:39.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:14:39.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:39.534 | INFO | Uploaded 36GiB to GPU in 2.44249s (14.7391GiB/s) | 20260508 04:14:39.534 | INFO | Downloaded 34GiB to GPU in 2.41593s (14.0733GiB/s) | 20260508 04:14:44.671 | INFO | Analyzing chunk frames = [5155840,5156352) | 20260508 04:14:44.671 | INFO | MemInfo: { | 20260508 04:14:44.671 | INFO | "gpu" : " 17 GB", | 20260508 04:14:44.671 | INFO | "huge" : "", | 20260508 04:14:44.671 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:44.671 | INFO | "malloc" : " 14 GB", | 20260508 04:14:44.671 | INFO | "pinned" : " 29 GB" | 20260508 04:14:44.671 | INFO | } | 20260508 04:14:44.671 | INFO | Perf report for the last 5.13782 seconds: | 20260508 04:14:44.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:44.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.672 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:14:44.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.672 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 04:14:44.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:44.672 | INFO | Uploaded 36GiB to GPU in 2.45257s (14.6785GiB/s) | 20260508 04:14:44.672 | INFO | Downloaded 34GiB to GPU in 2.41927s (14.0538GiB/s) | 20260508 04:14:49.809 | INFO | Analyzing chunk frames = [5156352,5156864) | 20260508 04:14:49.809 | INFO | MemInfo: { | 20260508 04:14:49.809 | INFO | "gpu" : " 17 GB", | 20260508 04:14:49.809 | INFO | "huge" : "", | 20260508 04:14:49.809 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:49.809 | INFO | "malloc" : " 14 GB", | 20260508 04:14:49.809 | INFO | "pinned" : " 29 GB" | 20260508 04:14:49.809 | INFO | } | 20260508 04:14:49.809 | INFO | Perf report for the last 5.13755 seconds: | 20260508 04:14:49.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:49.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:14:49.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:14:49.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:49.809 | INFO | Uploaded 36GiB to GPU in 2.46172s (14.6239GiB/s) | 20260508 04:14:49.809 | INFO | Downloaded 34GiB to GPU in 2.41134s (14.1001GiB/s) | 20260508 04:14:49.906 | INFO | Starting a round of Baz IO | 20260508 04:14:53.337 | INFO | Populating RTMetrics report | 20260508 04:14:53.338 | INFO | RTMetrics report populated | 20260508 04:14:53.338 | INFO | Opening tmp RTMetrics json file | 20260508 04:14:53.361 | INFO | Serializing RTMetrics to json | 20260508 04:14:53.380 | INFO | Serialization done, swapping tmp file now... | 20260508 04:14:53.383 | INFO | RTMetrics done | 20260508 04:14:53.521 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.615 seconds | 20260508 04:14:53.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.10% / 0.00% / 58.89% | 20260508 04:14:53.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.551569% / 3.97364e-06% / 4.37101e-05% | 20260508 04:14:53.521 | INFO | Baz IO overall: | 20260508 04:14:53.521 | INFO | BytesWritten : 2310804809447 | 20260508 04:14:53.521 | INFO | NumEvents : 1115814759336 | 20260508 04:14:54.944 | INFO | Analyzing chunk frames = [5156864,5157376) | 20260508 04:14:54.944 | INFO | MemInfo: { | 20260508 04:14:54.944 | INFO | "gpu" : " 17 GB", | 20260508 04:14:54.944 | INFO | "huge" : "", | 20260508 04:14:54.944 | INFO | "huge_shared" : " 29 GB", | 20260508 04:14:54.944 | INFO | "malloc" : " 14 GB", | 20260508 04:14:54.944 | INFO | "pinned" : " 29 GB" | 20260508 04:14:54.944 | INFO | } | 20260508 04:14:54.944 | INFO | Perf report for the last 5.13479 seconds: | 20260508 04:14:54.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:14:54.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:54.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:54.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:54.944 | INFO | ANALYSIS : 68.0%, 3.0, 105.6ms, 0 | 20260508 04:14:54.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:14:54.944 | INFO | RT_METRICS : 40.4%, 1.0, 21.1ms, 0 | 20260508 04:14:54.944 | INFO | PRE_HQ : 65.6%, 1.6, 55.4ms, 0 | 20260508 04:14:54.944 | INFO | BAZWRITER : 70.2%, 3.9, 142.4ms, 0 | 20260508 04:14:54.944 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 04:14:54.944 | INFO | Uploaded 36GiB to GPU in 2.57256s (13.9938GiB/s) | 20260508 04:14:54.944 | INFO | Downloaded 38GiB to GPU in 2.70701s (14.0376GiB/s) | 20260508 04:15:00.085 | INFO | Analyzing chunk frames = [5157376,5157888) | 20260508 04:15:00.085 | INFO | MemInfo: { | 20260508 04:15:00.085 | INFO | "gpu" : " 17 GB", | 20260508 04:15:00.085 | INFO | "huge" : "", | 20260508 04:15:00.085 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:00.085 | INFO | "malloc" : " 14 GB", | 20260508 04:15:00.085 | INFO | "pinned" : " 29 GB" | 20260508 04:15:00.085 | INFO | } | 20260508 04:15:00.085 | INFO | Perf report for the last 5.141 seconds: | 20260508 04:15:00.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:00.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.085 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:15:00.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.085 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:15:00.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:00.086 | INFO | Uploaded 36GiB to GPU in 2.58589s (13.9217GiB/s) | 20260508 04:15:00.086 | INFO | Downloaded 34GiB to GPU in 2.42576s (14.0162GiB/s) | 20260508 04:15:05.229 | INFO | Analyzing chunk frames = [5157888,5158400) | 20260508 04:15:05.229 | INFO | MemInfo: { | 20260508 04:15:05.229 | INFO | "gpu" : " 17 GB", | 20260508 04:15:05.229 | INFO | "huge" : "", | 20260508 04:15:05.229 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:05.229 | INFO | "malloc" : " 14 GB", | 20260508 04:15:05.229 | INFO | "pinned" : " 29 GB" | 20260508 04:15:05.229 | INFO | } | 20260508 04:15:05.229 | INFO | Perf report for the last 5.14398 seconds: | 20260508 04:15:05.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:05.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 04:15:05.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:15:05.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:05.229 | INFO | Uploaded 36GiB to GPU in 2.5635s (14.0433GiB/s) | 20260508 04:15:05.229 | INFO | Downloaded 34GiB to GPU in 2.41404s (14.0843GiB/s) | 20260508 04:15:10.371 | INFO | Analyzing chunk frames = [5158400,5158912) | 20260508 04:15:10.371 | INFO | MemInfo: { | 20260508 04:15:10.371 | INFO | "gpu" : " 17 GB", | 20260508 04:15:10.371 | INFO | "huge" : "", | 20260508 04:15:10.371 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:10.371 | INFO | "malloc" : " 14 GB", | 20260508 04:15:10.371 | INFO | "pinned" : " 29 GB" | 20260508 04:15:10.371 | INFO | } | 20260508 04:15:10.371 | INFO | Perf report for the last 5.1426 seconds: | 20260508 04:15:10.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:10.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.371 | INFO | ANALYSIS : 67.1%, 2.9, 103.9ms, 0 | 20260508 04:15:10.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.372 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:15:10.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:10.372 | INFO | Uploaded 36GiB to GPU in 2.59484s (13.8737GiB/s) | 20260508 04:15:10.372 | INFO | Downloaded 34GiB to GPU in 2.40954s (14.1106GiB/s) | 20260508 04:15:15.506 | INFO | Analyzing chunk frames = [5158912,5159424) | 20260508 04:15:15.506 | INFO | MemInfo: { | 20260508 04:15:15.506 | INFO | "gpu" : " 17 GB", | 20260508 04:15:15.506 | INFO | "huge" : "", | 20260508 04:15:15.506 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:15.506 | INFO | "malloc" : " 14 GB", | 20260508 04:15:15.506 | INFO | "pinned" : " 29 GB" | 20260508 04:15:15.507 | INFO | } | 20260508 04:15:15.507 | INFO | Perf report for the last 5.13522 seconds: | 20260508 04:15:15.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:15.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 04:15:15.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:15:15.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:15.507 | INFO | Uploaded 36GiB to GPU in 2.3891s (15.0684GiB/s) | 20260508 04:15:15.507 | INFO | Downloaded 34GiB to GPU in 2.57932s (13.1818GiB/s) | 20260508 04:15:20.640 | INFO | Analyzing chunk frames = [5159424,5159936) | 20260508 04:15:20.640 | INFO | MemInfo: { | 20260508 04:15:20.640 | INFO | "gpu" : " 17 GB", | 20260508 04:15:20.640 | INFO | "huge" : "", | 20260508 04:15:20.640 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:20.640 | INFO | "malloc" : " 14 GB", | 20260508 04:15:20.640 | INFO | "pinned" : " 29 GB" | 20260508 04:15:20.640 | INFO | } | 20260508 04:15:20.640 | INFO | Perf report for the last 5.13372 seconds: | 20260508 04:15:20.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:20.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.640 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:15:20.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.641 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:15:20.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:20.641 | INFO | Uploaded 36GiB to GPU in 2.45445s (14.6672GiB/s) | 20260508 04:15:20.641 | INFO | Downloaded 34GiB to GPU in 2.41563s (14.075GiB/s) | 20260508 04:15:25.782 | INFO | Analyzing chunk frames = [5159936,5160448) | 20260508 04:15:25.782 | INFO | MemInfo: { | 20260508 04:15:25.782 | INFO | "gpu" : " 17 GB", | 20260508 04:15:25.782 | INFO | "huge" : "", | 20260508 04:15:25.782 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:25.782 | INFO | "malloc" : " 14 GB", | 20260508 04:15:25.782 | INFO | "pinned" : " 29 GB" | 20260508 04:15:25.782 | INFO | } | 20260508 04:15:25.782 | INFO | Perf report for the last 5.14157 seconds: | 20260508 04:15:25.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:25.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.782 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:15:25.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.782 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:15:25.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:25.783 | INFO | Uploaded 36GiB to GPU in 2.46309s (14.6158GiB/s) | 20260508 04:15:25.783 | INFO | Downloaded 34GiB to GPU in 2.41451s (14.0815GiB/s) | 20260508 04:15:30.924 | INFO | Analyzing chunk frames = [5160448,5160960) | 20260508 04:15:30.924 | INFO | MemInfo: { | 20260508 04:15:30.924 | INFO | "gpu" : " 17 GB", | 20260508 04:15:30.924 | INFO | "huge" : "", | 20260508 04:15:30.924 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:30.924 | INFO | "malloc" : " 14 GB", | 20260508 04:15:30.924 | INFO | "pinned" : " 29 GB" | 20260508 04:15:30.924 | INFO | } | 20260508 04:15:30.924 | INFO | Perf report for the last 5.14248 seconds: | 20260508 04:15:30.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:30.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.924 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:15:30.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.924 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:15:30.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:30.925 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:15:30.925 | INFO | Total mem usage: 16.7 GiB | 20260508 04:15:30.925 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:15:30.925 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11.9%) | 20260508 04:15:30.925 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:15:30.925 | INFO | High Water Breakdown: | 20260508 04:15:30.925 | INFO | 60.2% : 8.97 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:15:30.925 | INFO | 39.7% : 5.91 GiB ( 88% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:15:30.925 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:15:30.925 | INFO | ------------------------------------------ | 20260508 04:15:30.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:15:30.925 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:15:30.925 | INFO | Total mem usage: 26.5 GiB | 20260508 04:15:30.925 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:15:30.925 | INFO | Uploaded 36GiB to GPU in 2.4908s (14.4532GiB/s) | 20260508 04:15:30.925 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:15:30.925 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:15:30.925 | INFO | High Water Breakdown: | 20260508 04:15:30.925 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:15:30.925 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:15:30.925 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:15:30.925 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:15:30.925 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:15:30.925 | INFO | Downloaded 34GiB to GPU in 2.41573s (14.0744GiB/s) | 20260508 04:15:30.925 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:15:30.925 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:15:30.925 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:15:30.925 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:15:30.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:15:30.925 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:15:30.925 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:15:30.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:15:30.925 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:15:30.925 | INFO | ------------------------------------------ | 20260508 04:15:30.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:15:30.925 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:15:30.925 | INFO | Total mem usage: 27.8 GiB | 20260508 04:15:30.925 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:15:30.925 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:15:30.926 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:15:30.926 | INFO | High Water Breakdown: | 20260508 04:15:30.926 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:15:30.926 | INFO | ------------------------------------------ | 20260508 04:15:30.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:15:30.926 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:15:30.926 | INFO | Total mem usage: 15.7 GiB | 20260508 04:15:30.926 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:15:30.926 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:15:30.926 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:15:30.926 | INFO | High Water Breakdown: | 20260508 04:15:30.926 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:15:30.926 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:15:30.926 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:15:30.926 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:15:30.926 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:15:30.926 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:15:30.926 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:15:30.926 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:15:30.926 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:15:30.926 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:15:30.926 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:15:30.926 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:15:30.926 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:15:30.926 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:15:30.926 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:15:30.926 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:15:30.926 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:15:30.926 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:15:30.926 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:15:30.926 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:15:30.926 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:15:30.926 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:15:30.926 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:15:30.926 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:15:30.926 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:15:30.926 | INFO | ------------------------------------------ | 20260508 04:15:30.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:15:30.926 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:15:30.926 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:15:30.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:15:30.926 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.40e+01 / 1.96e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.10e-02 / 7.13e-03 | 20260508 04:15:30.926 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.49e+00, 5.89e-01 / 1.70e-01 | 20260508 04:15:30.926 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.43e-01 | 20260508 04:15:30.926 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.58e+01, 3.88e+00 / 2.28e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.74e+00 / 1.31e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.21e-01 / 2.03e-01 | 20260508 04:15:30.926 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.22e+00, 2.65e-01 / 1.57e-01 | 20260508 04:15:30.926 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.52e-01, 5.10e-02 / 2.26e-02 | 20260508 04:15:30.926 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:15:30.926 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:15:30.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:15:30.926 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 24.54%, Upload -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 30.72%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 2.48%, LockUpload -- 2.64e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 2.48%, LockDownload -- 2.64e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:15:30.926 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260508 04:15:31.022 | INFO | Starting a round of Baz IO | 20260508 04:15:34.458 | INFO | Populating RTMetrics report | 20260508 04:15:34.459 | INFO | RTMetrics report populated | 20260508 04:15:34.459 | INFO | Opening tmp RTMetrics json file | 20260508 04:15:34.485 | INFO | Serializing RTMetrics to json | 20260508 04:15:34.486 | INFO | Serialization done, swapping tmp file now... | 20260508 04:15:34.490 | INFO | RTMetrics done | 20260508 04:15:34.599 | INFO | Finished a round of Baz IO: Wrote 1.6 GiB in 3.576 seconds | 20260508 04:15:34.599 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.10% / 0.00% / 58.90% | 20260508 04:15:34.599 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.551442% / 0% / 6.35783e-05% | 20260508 04:15:34.599 | INFO | Baz IO overall: | 20260508 04:15:34.599 | INFO | BytesWritten : 2312522729323 | 20260508 04:15:34.599 | INFO | NumEvents : 1116636147546 | 20260508 04:15:36.062 | INFO | Analyzing chunk frames = [5160960,5161472) | 20260508 04:15:36.062 | INFO | MemInfo: { | 20260508 04:15:36.063 | INFO | "gpu" : " 17 GB", | 20260508 04:15:36.063 | INFO | "huge" : "", | 20260508 04:15:36.063 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:36.063 | INFO | "malloc" : " 14 GB", | 20260508 04:15:36.063 | INFO | "pinned" : " 29 GB" | 20260508 04:15:36.063 | INFO | } | 20260508 04:15:36.063 | INFO | Perf report for the last 5.13821 seconds: | 20260508 04:15:36.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:36.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:36.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:36.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:36.063 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 04:15:36.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:36.063 | INFO | RT_METRICS : 40.1%, 1.0, 21.0ms, 0 | 20260508 04:15:36.063 | INFO | PRE_HQ : 65.7%, 1.6, 55.5ms, 0 | 20260508 04:15:36.063 | INFO | BAZWRITER : 69.2%, 3.6, 127.0ms, 0 | 20260508 04:15:36.063 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260508 04:15:36.063 | INFO | Uploaded 36GiB to GPU in 2.52605s (14.2515GiB/s) | 20260508 04:15:36.063 | INFO | Downloaded 38GiB to GPU in 2.69804s (14.0843GiB/s) | 20260508 04:15:41.202 | INFO | Analyzing chunk frames = [5161472,5161984) | 20260508 04:15:41.202 | INFO | MemInfo: { | 20260508 04:15:41.202 | INFO | "gpu" : " 17 GB", | 20260508 04:15:41.202 | INFO | "huge" : "", | 20260508 04:15:41.202 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:41.202 | INFO | "malloc" : " 14 GB", | 20260508 04:15:41.202 | INFO | "pinned" : " 29 GB" | 20260508 04:15:41.202 | INFO | } | 20260508 04:15:41.202 | INFO | Perf report for the last 5.13925 seconds: | 20260508 04:15:41.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:41.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:15:41.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 04:15:41.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:41.202 | INFO | Uploaded 36GiB to GPU in 2.5859s (13.9217GiB/s) | 20260508 04:15:41.203 | INFO | Downloaded 34GiB to GPU in 2.43571s (13.959GiB/s) | 20260508 04:15:46.339 | INFO | Analyzing chunk frames = [5161984,5162496) | 20260508 04:15:46.339 | INFO | MemInfo: { | 20260508 04:15:46.339 | INFO | "gpu" : " 17 GB", | 20260508 04:15:46.339 | INFO | "huge" : "", | 20260508 04:15:46.339 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:46.339 | INFO | "malloc" : " 14 GB", | 20260508 04:15:46.339 | INFO | "pinned" : " 29 GB" | 20260508 04:15:46.339 | INFO | } | 20260508 04:15:46.340 | INFO | Perf report for the last 5.1375 seconds: | 20260508 04:15:46.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:46.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 04:15:46.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 04:15:46.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:46.340 | INFO | Uploaded 36GiB to GPU in 2.55388s (14.0962GiB/s) | 20260508 04:15:46.340 | INFO | Downloaded 34GiB to GPU in 2.41339s (14.0881GiB/s) | 20260508 04:15:51.481 | INFO | Analyzing chunk frames = [5162496,5163008) | 20260508 04:15:51.481 | INFO | MemInfo: { | 20260508 04:15:51.481 | INFO | "gpu" : " 17 GB", | 20260508 04:15:51.481 | INFO | "huge" : "", | 20260508 04:15:51.481 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:51.481 | INFO | "malloc" : " 14 GB", | 20260508 04:15:51.481 | INFO | "pinned" : " 29 GB" | 20260508 04:15:51.481 | INFO | } | 20260508 04:15:51.482 | INFO | Perf report for the last 5.14203 seconds: | 20260508 04:15:51.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:51.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | ANALYSIS : 67.4%, 2.9, 104.3ms, 0 | 20260508 04:15:51.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 04:15:51.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:51.482 | INFO | Uploaded 36GiB to GPU in 2.60413s (13.8242GiB/s) | 20260508 04:15:51.483 | INFO | Downloaded 34GiB to GPU in 2.43035s (13.9898GiB/s) | 20260508 04:15:56.622 | INFO | Analyzing chunk frames = [5163008,5163520) | 20260508 04:15:56.622 | INFO | MemInfo: { | 20260508 04:15:56.622 | INFO | "gpu" : " 17 GB", | 20260508 04:15:56.622 | INFO | "huge" : "", | 20260508 04:15:56.622 | INFO | "huge_shared" : " 29 GB", | 20260508 04:15:56.623 | INFO | "malloc" : " 14 GB", | 20260508 04:15:56.623 | INFO | "pinned" : " 29 GB" | 20260508 04:15:56.623 | INFO | } | 20260508 04:15:56.623 | INFO | Perf report for the last 5.14107 seconds: | 20260508 04:15:56.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:15:56.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 04:15:56.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 04:15:56.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:15:56.623 | INFO | Uploaded 36GiB to GPU in 2.41779s (14.8896GiB/s) | 20260508 04:15:56.623 | INFO | Downloaded 34GiB to GPU in 2.56211s (13.2703GiB/s) | 20260508 04:16:01.762 | INFO | Analyzing chunk frames = [5163520,5164032) | 20260508 04:16:01.762 | INFO | MemInfo: { | 20260508 04:16:01.762 | INFO | "gpu" : " 17 GB", | 20260508 04:16:01.762 | INFO | "huge" : "", | 20260508 04:16:01.762 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:01.762 | INFO | "malloc" : " 14 GB", | 20260508 04:16:01.762 | INFO | "pinned" : " 29 GB" | 20260508 04:16:01.762 | INFO | } | 20260508 04:16:01.762 | INFO | Perf report for the last 5.1396 seconds: | 20260508 04:16:01.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:01.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.762 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:16:01.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.762 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:16:01.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:01.763 | INFO | Uploaded 36GiB to GPU in 2.41582s (14.9018GiB/s) | 20260508 04:16:01.763 | INFO | Downloaded 34GiB to GPU in 2.41908s (14.0549GiB/s) | 20260508 04:16:06.895 | INFO | Analyzing chunk frames = [5164032,5164544) | 20260508 04:16:06.895 | INFO | MemInfo: { | 20260508 04:16:06.895 | INFO | "gpu" : " 17 GB", | 20260508 04:16:06.895 | INFO | "huge" : "", | 20260508 04:16:06.895 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:06.895 | INFO | "malloc" : " 14 GB", | 20260508 04:16:06.895 | INFO | "pinned" : " 29 GB" | 20260508 04:16:06.895 | INFO | } | 20260508 04:16:06.895 | INFO | Perf report for the last 5.13317 seconds: | 20260508 04:16:06.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:06.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.895 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:16:06.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.896 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:16:06.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:06.896 | INFO | Uploaded 36GiB to GPU in 2.42052s (14.8728GiB/s) | 20260508 04:16:06.896 | INFO | Downloaded 34GiB to GPU in 2.41676s (14.0684GiB/s) | 20260508 04:16:12.037 | INFO | Analyzing chunk frames = [5164544,5165056) | 20260508 04:16:12.037 | INFO | MemInfo: { | 20260508 04:16:12.037 | INFO | "gpu" : " 17 GB", | 20260508 04:16:12.037 | INFO | "huge" : "", | 20260508 04:16:12.037 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:12.038 | INFO | "malloc" : " 14 GB", | 20260508 04:16:12.038 | INFO | "pinned" : " 29 GB" | 20260508 04:16:12.038 | INFO | } | 20260508 04:16:12.038 | INFO | Perf report for the last 5.14223 seconds: | 20260508 04:16:12.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:12.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:16:12.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 04:16:12.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:12.038 | INFO | Uploaded 36GiB to GPU in 2.44876s (14.7013GiB/s) | 20260508 04:16:12.038 | INFO | Downloaded 34GiB to GPU in 2.43571s (13.959GiB/s) | 20260508 04:16:12.135 | INFO | Starting a round of Baz IO | 20260508 04:16:15.557 | INFO | Populating RTMetrics report | 20260508 04:16:15.558 | INFO | RTMetrics report populated | 20260508 04:16:15.558 | INFO | Opening tmp RTMetrics json file | 20260508 04:16:15.575 | INFO | Serializing RTMetrics to json | 20260508 04:16:15.583 | INFO | Serialization done, swapping tmp file now... | 20260508 04:16:15.585 | INFO | RTMetrics done | 20260508 04:16:15.644 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.508 seconds | 20260508 04:16:15.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.10% / 0.00% / 58.90% | 20260508 04:16:15.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.550103% / 3.97364e-06% / 8.74201e-05% | 20260508 04:16:15.644 | INFO | Baz IO overall: | 20260508 04:16:15.644 | INFO | BytesWritten : 2314238469003 | 20260508 04:16:15.644 | INFO | NumEvents : 1117455842586 | 20260508 04:16:17.177 | INFO | Analyzing chunk frames = [5165056,5165568) | 20260508 04:16:17.177 | INFO | MemInfo: { | 20260508 04:16:17.177 | INFO | "gpu" : " 17 GB", | 20260508 04:16:17.177 | INFO | "huge" : "", | 20260508 04:16:17.177 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:17.177 | INFO | "malloc" : " 14 GB", | 20260508 04:16:17.177 | INFO | "pinned" : " 29 GB" | 20260508 04:16:17.177 | INFO | } | 20260508 04:16:17.177 | INFO | Perf report for the last 5.13976 seconds: | 20260508 04:16:17.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:17.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:17.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:17.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:17.178 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260508 04:16:17.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:17.178 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 04:16:17.178 | INFO | PRE_HQ : 65.6%, 1.6, 55.5ms, 0 | 20260508 04:16:17.178 | INFO | BAZWRITER : 67.8%, 3.6, 126.7ms, 0 | 20260508 04:16:17.178 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 04:16:17.178 | INFO | Uploaded 36GiB to GPU in 2.54108s (14.1672GiB/s) | 20260508 04:16:17.178 | INFO | Downloaded 38GiB to GPU in 2.69985s (14.0748GiB/s) | 20260508 04:16:22.320 | INFO | Analyzing chunk frames = [5165568,5166080) | 20260508 04:16:22.320 | INFO | MemInfo: { | 20260508 04:16:22.320 | INFO | "gpu" : " 17 GB", | 20260508 04:16:22.320 | INFO | "huge" : "", | 20260508 04:16:22.320 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:22.320 | INFO | "malloc" : " 14 GB", | 20260508 04:16:22.320 | INFO | "pinned" : " 29 GB" | 20260508 04:16:22.320 | INFO | } | 20260508 04:16:22.320 | INFO | Perf report for the last 5.14239 seconds: | 20260508 04:16:22.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:22.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 04:16:22.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:16:22.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:22.320 | INFO | Uploaded 36GiB to GPU in 2.53148s (14.2209GiB/s) | 20260508 04:16:22.320 | INFO | Downloaded 34GiB to GPU in 2.42412s (14.0257GiB/s) | 20260508 04:16:27.454 | INFO | Analyzing chunk frames = [5166080,5166592) | 20260508 04:16:27.454 | INFO | MemInfo: { | 20260508 04:16:27.454 | INFO | "gpu" : " 17 GB", | 20260508 04:16:27.454 | INFO | "huge" : "", | 20260508 04:16:27.454 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:27.454 | INFO | "malloc" : " 14 GB", | 20260508 04:16:27.455 | INFO | "pinned" : " 29 GB" | 20260508 04:16:27.455 | INFO | } | 20260508 04:16:27.455 | INFO | Perf report for the last 5.13486 seconds: | 20260508 04:16:27.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:27.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 04:16:27.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:16:27.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:27.455 | INFO | Uploaded 36GiB to GPU in 2.54355s (14.1535GiB/s) | 20260508 04:16:27.455 | INFO | Downloaded 34GiB to GPU in 2.42612s (14.0142GiB/s) | 20260508 04:16:32.594 | INFO | Analyzing chunk frames = [5166592,5167104) | 20260508 04:16:32.594 | INFO | MemInfo: { | 20260508 04:16:32.594 | INFO | "gpu" : " 17 GB", | 20260508 04:16:32.594 | INFO | "huge" : "", | 20260508 04:16:32.595 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:32.595 | INFO | "malloc" : " 14 GB", | 20260508 04:16:32.595 | INFO | "pinned" : " 29 GB" | 20260508 04:16:32.595 | INFO | } | 20260508 04:16:32.595 | INFO | Perf report for the last 5.13989 seconds: | 20260508 04:16:32.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:32.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | ANALYSIS : 67.1%, 3.0, 103.8ms, 0 | 20260508 04:16:32.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:16:32.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:32.595 | INFO | Uploaded 36GiB to GPU in 2.57036s (14.0058GiB/s) | 20260508 04:16:32.595 | INFO | Downloaded 34GiB to GPU in 2.42116s (14.0429GiB/s) | 20260508 04:16:37.734 | INFO | Analyzing chunk frames = [5167104,5167616) | 20260508 04:16:37.734 | INFO | MemInfo: { | 20260508 04:16:37.734 | INFO | "gpu" : " 17 GB", | 20260508 04:16:37.734 | INFO | "huge" : "", | 20260508 04:16:37.734 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:37.734 | INFO | "malloc" : " 14 GB", | 20260508 04:16:37.734 | INFO | "pinned" : " 29 GB" | 20260508 04:16:37.734 | INFO | } | 20260508 04:16:37.734 | INFO | Perf report for the last 5.13916 seconds: | 20260508 04:16:37.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:37.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 04:16:37.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:16:37.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:37.734 | INFO | Uploaded 36GiB to GPU in 2.39692s (15.0193GiB/s) | 20260508 04:16:37.734 | INFO | Downloaded 34GiB to GPU in 2.5468s (13.3501GiB/s) | 20260508 04:16:42.874 | INFO | Analyzing chunk frames = [5167616,5168128) | 20260508 04:16:42.874 | INFO | MemInfo: { | 20260508 04:16:42.874 | INFO | "gpu" : " 17 GB", | 20260508 04:16:42.874 | INFO | "huge" : "", | 20260508 04:16:42.874 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:42.875 | INFO | "malloc" : " 14 GB", | 20260508 04:16:42.875 | INFO | "pinned" : " 29 GB" | 20260508 04:16:42.875 | INFO | } | 20260508 04:16:42.875 | INFO | Perf report for the last 5.14078 seconds: | 20260508 04:16:42.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:42.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:16:42.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:16:42.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:42.875 | INFO | Uploaded 36GiB to GPU in 2.41308s (14.9187GiB/s) | 20260508 04:16:42.875 | INFO | Downloaded 34GiB to GPU in 2.42729s (14.0074GiB/s) | 20260508 04:16:48.011 | INFO | Analyzing chunk frames = [5168128,5168640) | 20260508 04:16:48.011 | INFO | MemInfo: { | 20260508 04:16:48.011 | INFO | "gpu" : " 17 GB", | 20260508 04:16:48.011 | INFO | "huge" : "", | 20260508 04:16:48.011 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:48.011 | INFO | "malloc" : " 14 GB", | 20260508 04:16:48.011 | INFO | "pinned" : " 29 GB" | 20260508 04:16:48.011 | INFO | } | 20260508 04:16:48.012 | INFO | Perf report for the last 5.1369 seconds: | 20260508 04:16:48.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:48.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:16:48.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:16:48.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:48.012 | INFO | Uploaded 36GiB to GPU in 2.42598s (14.8394GiB/s) | 20260508 04:16:48.012 | INFO | Downloaded 34GiB to GPU in 2.421s (14.0438GiB/s) | 20260508 04:16:53.150 | INFO | Analyzing chunk frames = [5168640,5169152) | 20260508 04:16:53.150 | INFO | MemInfo: { | 20260508 04:16:53.150 | INFO | "gpu" : " 17 GB", | 20260508 04:16:53.150 | INFO | "huge" : "", | 20260508 04:16:53.150 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:53.150 | INFO | "malloc" : " 14 GB", | 20260508 04:16:53.151 | INFO | "pinned" : " 29 GB" | 20260508 04:16:53.151 | INFO | } | 20260508 04:16:53.151 | INFO | Perf report for the last 5.13942 seconds: | 20260508 04:16:53.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:53.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:16:53.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:16:53.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:53.151 | INFO | Uploaded 36GiB to GPU in 2.44211s (14.7413GiB/s) | 20260508 04:16:53.151 | INFO | Downloaded 34GiB to GPU in 2.42767s (14.0052GiB/s) | 20260508 04:16:53.151 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:16:53.151 | INFO | Total mem usage: 16.7 GiB | 20260508 04:16:53.151 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:16:53.151 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260508 04:16:53.151 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:16:53.151 | INFO | High Water Breakdown: | 20260508 04:16:53.151 | INFO | 59.9% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:16:53.151 | INFO | 39.9% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:16:53.151 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:16:53.151 | INFO | ------------------------------------------ | 20260508 04:16:53.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:16:53.151 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:16:53.151 | INFO | Total mem usage: 26.5 GiB | 20260508 04:16:53.151 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:16:53.151 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:16:53.151 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:16:53.151 | INFO | High Water Breakdown: | 20260508 04:16:53.151 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:16:53.151 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:16:53.151 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:16:53.151 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:16:53.151 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:16:53.151 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:16:53.151 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:16:53.151 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:16:53.151 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:16:53.151 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:16:53.151 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:16:53.151 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:16:53.151 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:16:53.151 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:16:53.151 | INFO | ------------------------------------------ | 20260508 04:16:53.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:16:53.151 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:16:53.151 | INFO | Total mem usage: 27.8 GiB | 20260508 04:16:53.151 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:16:53.151 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:16:53.151 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:16:53.151 | INFO | High Water Breakdown: | 20260508 04:16:53.151 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:16:53.151 | INFO | ------------------------------------------ | 20260508 04:16:53.151 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:16:53.151 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:16:53.151 | INFO | Total mem usage: 15.7 GiB | 20260508 04:16:53.151 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:16:53.151 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:16:53.151 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:16:53.151 | INFO | High Water Breakdown: | 20260508 04:16:53.151 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:16:53.151 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:16:53.151 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:16:53.151 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:16:53.151 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:16:53.151 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:16:53.151 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:16:53.151 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:16:53.151 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:16:53.151 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:16:53.151 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:16:53.151 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:16:53.151 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:16:53.151 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:16:53.151 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:16:53.151 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:16:53.151 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:16:53.151 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:16:53.151 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:16:53.151 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:16:53.151 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:16:53.151 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:16:53.151 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:16:53.152 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:16:53.152 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:16:53.152 | INFO | ------------------------------------------ | 20260508 04:16:53.152 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:16:53.152 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:16:53.152 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:16:53.152 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:16:53.152 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.58e+01, 1.41e+01 / 1.88e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.04e-01, 1.25e-02 / 5.77e-03 | 20260508 04:16:53.152 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.48e+00, 5.95e-01 / 1.76e-01 | 20260508 04:16:53.152 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.43e+00, 2.10e-01 / 1.32e-01 | 20260508 04:16:53.152 | INFO | PerfReport: 15.88%, DME -- 4.17e+01 / 4.52e+01, 3.69e+00 / 2.08e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.83e+00 / 1.43e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.14e-01 / 2.51e-01 | 20260508 04:16:53.152 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.21e+00, 2.61e-01 / 1.38e-01 | 20260508 04:16:53.152 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.56e-01, 5.12e-02 / 2.50e-02 | 20260508 04:16:53.152 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:16:53.152 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:16:53.152 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:16:53.152 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 30.86%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 2.27%, LockUpload -- 2.41e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 2.58%, LockDownload -- 2.75e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 15.67%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:16:53.152 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.87e-02 / 0.00e+00 | 20260508 04:16:53.248 | INFO | Starting a round of Baz IO | 20260508 04:16:56.690 | INFO | Populating RTMetrics report | 20260508 04:16:56.690 | INFO | RTMetrics report populated | 20260508 04:16:56.690 | INFO | Opening tmp RTMetrics json file | 20260508 04:16:56.813 | INFO | Serializing RTMetrics to json | 20260508 04:16:56.816 | INFO | Serialization done, swapping tmp file now... | 20260508 04:16:56.820 | INFO | RTMetrics done | 20260508 04:16:56.829 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.581 seconds | 20260508 04:16:56.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.10% / 0.00% / 58.90% | 20260508 04:16:56.829 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.549181% / 3.97364e-06% / 6.75519e-05% | 20260508 04:16:56.829 | INFO | Baz IO overall: | 20260508 04:16:56.829 | INFO | BytesWritten : 2315955166688 | 20260508 04:16:56.829 | INFO | NumEvents : 1118276398562 | 20260508 04:16:58.293 | INFO | Analyzing chunk frames = [5169152,5169664) | 20260508 04:16:58.293 | INFO | MemInfo: { | 20260508 04:16:58.293 | INFO | "gpu" : " 17 GB", | 20260508 04:16:58.293 | INFO | "huge" : "", | 20260508 04:16:58.293 | INFO | "huge_shared" : " 29 GB", | 20260508 04:16:58.293 | INFO | "malloc" : " 14 GB", | 20260508 04:16:58.293 | INFO | "pinned" : " 29 GB" | 20260508 04:16:58.293 | INFO | } | 20260508 04:16:58.293 | INFO | Perf report for the last 5.142 seconds: | 20260508 04:16:58.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:16:58.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:58.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:58.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:58.293 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 04:16:58.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:16:58.293 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 04:16:58.293 | INFO | PRE_HQ : 65.6%, 1.6, 55.8ms, 0 | 20260508 04:16:58.293 | INFO | BAZWRITER : 69.2%, 3.4, 122.3ms, 0 | 20260508 04:16:58.293 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260508 04:16:58.293 | INFO | Uploaded 36GiB to GPU in 2.5324s (14.2158GiB/s) | 20260508 04:16:58.293 | INFO | Downloaded 38GiB to GPU in 2.70632s (14.0412GiB/s) | 20260508 04:17:03.434 | INFO | Analyzing chunk frames = [5169664,5170176) | 20260508 04:17:03.434 | INFO | MemInfo: { | 20260508 04:17:03.434 | INFO | "gpu" : " 17 GB", | 20260508 04:17:03.434 | INFO | "huge" : "", | 20260508 04:17:03.434 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:03.434 | INFO | "malloc" : " 14 GB", | 20260508 04:17:03.434 | INFO | "pinned" : " 29 GB" | 20260508 04:17:03.434 | INFO | } | 20260508 04:17:03.434 | INFO | Perf report for the last 5.14127 seconds: | 20260508 04:17:03.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:03.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.434 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 04:17:03.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.434 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:17:03.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:03.435 | INFO | Uploaded 36GiB to GPU in 2.56667s (14.026GiB/s) | 20260508 04:17:03.435 | INFO | Downloaded 34GiB to GPU in 2.42545s (14.018GiB/s) | 20260508 04:17:08.571 | INFO | Analyzing chunk frames = [5170176,5170688) | 20260508 04:17:08.571 | INFO | MemInfo: { | 20260508 04:17:08.571 | INFO | "gpu" : " 17 GB", | 20260508 04:17:08.571 | INFO | "huge" : "", | 20260508 04:17:08.571 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:08.571 | INFO | "malloc" : " 14 GB", | 20260508 04:17:08.571 | INFO | "pinned" : " 29 GB" | 20260508 04:17:08.571 | INFO | } | 20260508 04:17:08.571 | INFO | Perf report for the last 5.13707 seconds: | 20260508 04:17:08.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:08.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.571 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:17:08.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.571 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:17:08.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:08.572 | INFO | Uploaded 36GiB to GPU in 2.53554s (14.1982GiB/s) | 20260508 04:17:08.572 | INFO | Downloaded 34GiB to GPU in 2.42608s (14.0144GiB/s) | 20260508 04:17:13.708 | INFO | Analyzing chunk frames = [5170688,5171200) | 20260508 04:17:13.708 | INFO | MemInfo: { | 20260508 04:17:13.708 | INFO | "gpu" : " 17 GB", | 20260508 04:17:13.708 | INFO | "huge" : "", | 20260508 04:17:13.708 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:13.708 | INFO | "malloc" : " 14 GB", | 20260508 04:17:13.708 | INFO | "pinned" : " 29 GB" | 20260508 04:17:13.708 | INFO | } | 20260508 04:17:13.708 | INFO | Perf report for the last 5.13709 seconds: | 20260508 04:17:13.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:13.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.708 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 04:17:13.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.709 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:17:13.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:13.709 | INFO | Uploaded 36GiB to GPU in 2.5669s (14.0247GiB/s) | 20260508 04:17:13.709 | INFO | Downloaded 34GiB to GPU in 2.4174s (14.0647GiB/s) | 20260508 04:17:18.850 | INFO | Analyzing chunk frames = [5171200,5171712) | 20260508 04:17:18.850 | INFO | MemInfo: { | 20260508 04:17:18.850 | INFO | "gpu" : " 17 GB", | 20260508 04:17:18.850 | INFO | "huge" : "", | 20260508 04:17:18.850 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:18.850 | INFO | "malloc" : " 14 GB", | 20260508 04:17:18.850 | INFO | "pinned" : " 29 GB" | 20260508 04:17:18.850 | INFO | } | 20260508 04:17:18.850 | INFO | Perf report for the last 5.14149 seconds: | 20260508 04:17:18.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:18.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.850 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 04:17:18.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.850 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 04:17:18.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:18.851 | INFO | Uploaded 36GiB to GPU in 2.3898s (15.064GiB/s) | 20260508 04:17:18.851 | INFO | Downloaded 34GiB to GPU in 2.57106s (13.2241GiB/s) | 20260508 04:17:23.991 | INFO | Analyzing chunk frames = [5171712,5172224) | 20260508 04:17:23.991 | INFO | MemInfo: { | 20260508 04:17:23.991 | INFO | "gpu" : " 17 GB", | 20260508 04:17:23.991 | INFO | "huge" : "", | 20260508 04:17:23.991 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:23.991 | INFO | "malloc" : " 14 GB", | 20260508 04:17:23.991 | INFO | "pinned" : " 29 GB" | 20260508 04:17:23.991 | INFO | } | 20260508 04:17:23.991 | INFO | Perf report for the last 5.14106 seconds: | 20260508 04:17:23.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:23.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.991 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:17:23.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.991 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:17:23.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:23.992 | INFO | Uploaded 36GiB to GPU in 2.44645s (14.7152GiB/s) | 20260508 04:17:23.992 | INFO | Downloaded 34GiB to GPU in 2.41006s (14.1075GiB/s) | 20260508 04:17:29.129 | INFO | Analyzing chunk frames = [5172224,5172736) | 20260508 04:17:29.129 | INFO | MemInfo: { | 20260508 04:17:29.129 | INFO | "gpu" : " 17 GB", | 20260508 04:17:29.129 | INFO | "huge" : "", | 20260508 04:17:29.129 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:29.129 | INFO | "malloc" : " 14 GB", | 20260508 04:17:29.129 | INFO | "pinned" : " 29 GB" | 20260508 04:17:29.129 | INFO | } | 20260508 04:17:29.129 | INFO | Perf report for the last 5.1381 seconds: | 20260508 04:17:29.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:29.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.129 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:17:29.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.129 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:17:29.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:29.130 | INFO | Uploaded 36GiB to GPU in 2.43259s (14.799GiB/s) | 20260508 04:17:29.130 | INFO | Downloaded 34GiB to GPU in 2.41475s (14.0801GiB/s) | 20260508 04:17:34.268 | INFO | Analyzing chunk frames = [5172736,5173248) | 20260508 04:17:34.268 | INFO | MemInfo: { | 20260508 04:17:34.268 | INFO | "gpu" : " 17 GB", | 20260508 04:17:34.268 | INFO | "huge" : "", | 20260508 04:17:34.268 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:34.268 | INFO | "malloc" : " 14 GB", | 20260508 04:17:34.269 | INFO | "pinned" : " 29 GB" | 20260508 04:17:34.269 | INFO | } | 20260508 04:17:34.269 | INFO | Perf report for the last 5.13967 seconds: | 20260508 04:17:34.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:34.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:17:34.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:17:34.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:34.269 | INFO | Uploaded 36GiB to GPU in 2.47315s (14.5563GiB/s) | 20260508 04:17:34.269 | INFO | Downloaded 34GiB to GPU in 2.42291s (14.0327GiB/s) | 20260508 04:17:34.382 | INFO | Starting a round of Baz IO | 20260508 04:17:37.831 | INFO | Populating RTMetrics report | 20260508 04:17:37.831 | INFO | RTMetrics report populated | 20260508 04:17:37.831 | INFO | Opening tmp RTMetrics json file | 20260508 04:17:37.843 | INFO | Serializing RTMetrics to json | 20260508 04:17:37.845 | INFO | Serialization done, swapping tmp file now... | 20260508 04:17:37.851 | INFO | RTMetrics done | 20260508 04:17:37.944 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.561 seconds | 20260508 04:17:37.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.10% / 0.00% / 58.90% | 20260508 04:17:37.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.54812% / 1.19209e-05% / 8.34465e-05% | 20260508 04:17:37.944 | INFO | Baz IO overall: | 20260508 04:17:37.944 | INFO | BytesWritten : 2317670604625 | 20260508 04:17:37.944 | INFO | NumEvents : 1119096156638 | 20260508 04:17:39.405 | INFO | Analyzing chunk frames = [5173248,5173760) | 20260508 04:17:39.405 | INFO | MemInfo: { | 20260508 04:17:39.405 | INFO | "gpu" : " 17 GB", | 20260508 04:17:39.405 | INFO | "huge" : "", | 20260508 04:17:39.405 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:39.405 | INFO | "malloc" : " 14 GB", | 20260508 04:17:39.405 | INFO | "pinned" : " 29 GB" | 20260508 04:17:39.405 | INFO | } | 20260508 04:17:39.405 | INFO | Perf report for the last 5.13629 seconds: | 20260508 04:17:39.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:39.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:39.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:39.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:39.405 | INFO | ANALYSIS : 68.6%, 2.9, 104.9ms, 0 | 20260508 04:17:39.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:39.405 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260508 04:17:39.405 | INFO | PRE_HQ : 67.6%, 1.8, 61.7ms, 0 | 20260508 04:17:39.405 | INFO | BAZWRITER : 69.0%, 3.5, 122.8ms, 0 | 20260508 04:17:39.405 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 04:17:39.405 | INFO | Uploaded 36GiB to GPU in 2.58132s (13.9463GiB/s) | 20260508 04:17:39.405 | INFO | Downloaded 38GiB to GPU in 2.7053s (14.0465GiB/s) | 20260508 04:17:44.550 | INFO | Analyzing chunk frames = [5173760,5174272) | 20260508 04:17:44.550 | INFO | MemInfo: { | 20260508 04:17:44.550 | INFO | "gpu" : " 17 GB", | 20260508 04:17:44.550 | INFO | "huge" : "", | 20260508 04:17:44.550 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:44.550 | INFO | "malloc" : " 14 GB", | 20260508 04:17:44.550 | INFO | "pinned" : " 29 GB" | 20260508 04:17:44.550 | INFO | } | 20260508 04:17:44.550 | INFO | Perf report for the last 5.14533 seconds: | 20260508 04:17:44.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:44.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.551 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 04:17:44.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.551 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:17:44.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:44.551 | INFO | Uploaded 36GiB to GPU in 2.58975s (13.9009GiB/s) | 20260508 04:17:44.551 | INFO | Downloaded 34GiB to GPU in 2.42188s (14.0387GiB/s) | 20260508 04:17:49.685 | INFO | Analyzing chunk frames = [5174272,5174784) | 20260508 04:17:49.685 | INFO | MemInfo: { | 20260508 04:17:49.685 | INFO | "gpu" : " 17 GB", | 20260508 04:17:49.685 | INFO | "huge" : "", | 20260508 04:17:49.686 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:49.686 | INFO | "malloc" : " 14 GB", | 20260508 04:17:49.686 | INFO | "pinned" : " 29 GB" | 20260508 04:17:49.686 | INFO | } | 20260508 04:17:49.686 | INFO | Perf report for the last 5.13515 seconds: | 20260508 04:17:49.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:49.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:17:49.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:17:49.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:49.686 | INFO | Uploaded 36GiB to GPU in 2.54123s (14.1664GiB/s) | 20260508 04:17:49.686 | INFO | Downloaded 34GiB to GPU in 2.40495s (14.1375GiB/s) | 20260508 04:17:54.823 | INFO | Analyzing chunk frames = [5174784,5175296) | 20260508 04:17:54.823 | INFO | MemInfo: { | 20260508 04:17:54.823 | INFO | "gpu" : " 17 GB", | 20260508 04:17:54.823 | INFO | "huge" : "", | 20260508 04:17:54.823 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:54.823 | INFO | "malloc" : " 14 GB", | 20260508 04:17:54.823 | INFO | "pinned" : " 29 GB" | 20260508 04:17:54.823 | INFO | } | 20260508 04:17:54.823 | INFO | Perf report for the last 5.13763 seconds: | 20260508 04:17:54.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:54.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.823 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 04:17:54.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.823 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:17:54.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:54.824 | INFO | Uploaded 36GiB to GPU in 2.56636s (14.0277GiB/s) | 20260508 04:17:54.824 | INFO | Downloaded 34GiB to GPU in 2.49435s (13.6308GiB/s) | 20260508 04:17:59.959 | INFO | Analyzing chunk frames = [5175296,5175808) | 20260508 04:17:59.959 | INFO | MemInfo: { | 20260508 04:17:59.959 | INFO | "gpu" : " 17 GB", | 20260508 04:17:59.959 | INFO | "huge" : "", | 20260508 04:17:59.959 | INFO | "huge_shared" : " 29 GB", | 20260508 04:17:59.959 | INFO | "malloc" : " 14 GB", | 20260508 04:17:59.959 | INFO | "pinned" : " 29 GB" | 20260508 04:17:59.959 | INFO | } | 20260508 04:17:59.959 | INFO | Perf report for the last 5.13614 seconds: | 20260508 04:17:59.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:17:59.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.959 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 04:17:59.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.960 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:17:59.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:17:59.960 | INFO | Uploaded 36GiB to GPU in 2.42117s (14.8689GiB/s) | 20260508 04:17:59.960 | INFO | Downloaded 34GiB to GPU in 2.53082s (13.4344GiB/s) | 20260508 04:18:05.105 | INFO | Analyzing chunk frames = [5175808,5176320) | 20260508 04:18:05.105 | INFO | MemInfo: { | 20260508 04:18:05.105 | INFO | "gpu" : " 17 GB", | 20260508 04:18:05.105 | INFO | "huge" : "", | 20260508 04:18:05.105 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:05.105 | INFO | "malloc" : " 14 GB", | 20260508 04:18:05.105 | INFO | "pinned" : " 29 GB" | 20260508 04:18:05.105 | INFO | } | 20260508 04:18:05.105 | INFO | Perf report for the last 5.14564 seconds: | 20260508 04:18:05.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:05.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:18:05.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:18:05.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:05.105 | INFO | Uploaded 36GiB to GPU in 2.42158s (14.8663GiB/s) | 20260508 04:18:05.105 | INFO | Downloaded 34GiB to GPU in 2.4273s (14.0073GiB/s) | 20260508 04:18:10.245 | INFO | Analyzing chunk frames = [5176320,5176832) | 20260508 04:18:10.245 | INFO | MemInfo: { | 20260508 04:18:10.245 | INFO | "gpu" : " 17 GB", | 20260508 04:18:10.245 | INFO | "huge" : "", | 20260508 04:18:10.245 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:10.245 | INFO | "malloc" : " 14 GB", | 20260508 04:18:10.246 | INFO | "pinned" : " 29 GB" | 20260508 04:18:10.246 | INFO | } | 20260508 04:18:10.246 | INFO | Perf report for the last 5.14054 seconds: | 20260508 04:18:10.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:10.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:18:10.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:18:10.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:10.246 | INFO | Uploaded 36GiB to GPU in 2.43986s (14.755GiB/s) | 20260508 04:18:10.246 | INFO | Downloaded 34GiB to GPU in 2.43313s (13.9738GiB/s) | 20260508 04:18:15.388 | INFO | Analyzing chunk frames = [5176832,5177344) | 20260508 04:18:15.388 | INFO | MemInfo: { | 20260508 04:18:15.388 | INFO | "gpu" : " 17 GB", | 20260508 04:18:15.388 | INFO | "huge" : "", | 20260508 04:18:15.388 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:15.388 | INFO | "malloc" : " 14 GB", | 20260508 04:18:15.388 | INFO | "pinned" : " 29 GB" | 20260508 04:18:15.388 | INFO | } | 20260508 04:18:15.388 | INFO | Perf report for the last 5.14271 seconds: | 20260508 04:18:15.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:15.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.388 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:18:15.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.388 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 04:18:15.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:15.389 | INFO | Uploaded 36GiB to GPU in 2.47539s (14.5432GiB/s) | 20260508 04:18:15.389 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:18:15.389 | INFO | Downloaded 34GiB to GPU in 2.43693s (13.952GiB/s) | 20260508 04:18:15.389 | INFO | Total mem usage: 16.7 GiB | 20260508 04:18:15.389 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:18:15.389 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260508 04:18:15.389 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:18:15.389 | INFO | High Water Breakdown: | 20260508 04:18:15.389 | INFO | 59.9% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:18:15.389 | INFO | 39.9% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:18:15.389 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:18:15.389 | INFO | ------------------------------------------ | 20260508 04:18:15.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:18:15.389 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:18:15.389 | INFO | Total mem usage: 26.5 GiB | 20260508 04:18:15.389 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:18:15.389 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:18:15.389 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:18:15.389 | INFO | High Water Breakdown: | 20260508 04:18:15.389 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:18:15.389 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:18:15.390 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:18:15.390 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:18:15.390 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:18:15.390 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:18:15.390 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:18:15.390 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:18:15.390 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:18:15.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:18:15.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:18:15.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:18:15.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:18:15.390 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:18:15.390 | INFO | ------------------------------------------ | 20260508 04:18:15.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:18:15.390 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:18:15.390 | INFO | Total mem usage: 27.8 GiB | 20260508 04:18:15.390 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:18:15.390 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:18:15.390 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:18:15.390 | INFO | High Water Breakdown: | 20260508 04:18:15.390 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:18:15.390 | INFO | ------------------------------------------ | 20260508 04:18:15.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:18:15.390 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:18:15.390 | INFO | Total mem usage: 15.7 GiB | 20260508 04:18:15.390 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:18:15.390 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:18:15.390 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 04:18:15.390 | INFO | High Water Breakdown: | 20260508 04:18:15.390 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:18:15.390 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:18:15.390 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:18:15.390 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:18:15.390 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:18:15.390 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:18:15.390 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:18:15.390 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:18:15.390 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:18:15.390 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:18:15.390 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:18:15.390 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:18:15.390 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:18:15.390 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:18:15.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:18:15.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:18:15.390 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:18:15.390 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:18:15.390 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:18:15.390 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:18:15.390 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:18:15.390 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:18:15.390 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:18:15.390 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:18:15.390 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:18:15.390 | INFO | ------------------------------------------ | 20260508 04:18:15.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:18:15.390 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:18:15.390 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:18:15.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:18:15.390 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.40e+01 / 1.78e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.02e-01, 1.11e-02 / 4.85e-03 | 20260508 04:18:15.391 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.49e+00, 5.93e-01 / 1.93e-01 | 20260508 04:18:15.391 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.23e-01 | 20260508 04:18:15.391 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.55e+01, 3.94e+00 / 2.10e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.80e+00 / 1.42e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.61e+00, 4.20e-01 / 2.62e-01 | 20260508 04:18:15.391 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.21e+00, 2.64e-01 / 1.50e-01 | 20260508 04:18:15.391 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.57e-01, 5.07e-02 / 2.50e-02 | 20260508 04:18:15.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:18:15.391 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:18:15.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:18:15.391 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 24.27%, Download -- 2.58e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 30.82%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 2.30%, LockUpload -- 2.45e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 2.56%, LockDownload -- 2.72e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:18:15.391 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260508 04:18:15.502 | INFO | Starting a round of Baz IO | 20260508 04:18:18.937 | INFO | Populating RTMetrics report | 20260508 04:18:18.938 | INFO | RTMetrics report populated | 20260508 04:18:18.938 | INFO | Opening tmp RTMetrics json file | 20260508 04:18:18.988 | INFO | Serializing RTMetrics to json | 20260508 04:18:18.999 | INFO | Serialization done, swapping tmp file now... | 20260508 04:18:19.010 | INFO | RTMetrics done | 20260508 04:18:19.042 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.539 seconds | 20260508 04:18:19.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.09% / 0.00% / 58.90% | 20260508 04:18:19.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.545359% / 3.97364e-06% / 6.75519e-05% | 20260508 04:18:19.042 | INFO | Baz IO overall: | 20260508 04:18:19.042 | INFO | BytesWritten : 2319385422482 | 20260508 04:18:19.042 | INFO | NumEvents : 1119915074392 | 20260508 04:18:20.523 | INFO | Analyzing chunk frames = [5177344,5177856) | 20260508 04:18:20.523 | INFO | MemInfo: { | 20260508 04:18:20.523 | INFO | "gpu" : " 17 GB", | 20260508 04:18:20.523 | INFO | "huge" : "", | 20260508 04:18:20.523 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:20.523 | INFO | "malloc" : " 14 GB", | 20260508 04:18:20.523 | INFO | "pinned" : " 29 GB" | 20260508 04:18:20.523 | INFO | } | 20260508 04:18:20.523 | INFO | Perf report for the last 5.13468 seconds: | 20260508 04:18:20.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:20.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:20.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:20.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:20.523 | INFO | ANALYSIS : 68.4%, 2.9, 104.9ms, 0 | 20260508 04:18:20.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:20.523 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260508 04:18:20.523 | INFO | PRE_HQ : 67.4%, 1.8, 62.8ms, 0 | 20260508 04:18:20.523 | INFO | BAZWRITER : 68.5%, 3.9, 137.2ms, 0 | 20260508 04:18:20.523 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260508 04:18:20.523 | INFO | Uploaded 36GiB to GPU in 2.62013s (13.7398GiB/s) | 20260508 04:18:20.523 | INFO | Downloaded 38GiB to GPU in 2.71641s (13.989GiB/s) | 20260508 04:18:25.660 | INFO | Analyzing chunk frames = [5177856,5178368) | 20260508 04:18:25.660 | INFO | MemInfo: { | 20260508 04:18:25.660 | INFO | "gpu" : " 17 GB", | 20260508 04:18:25.660 | INFO | "huge" : "", | 20260508 04:18:25.660 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:25.660 | INFO | "malloc" : " 14 GB", | 20260508 04:18:25.660 | INFO | "pinned" : " 29 GB" | 20260508 04:18:25.660 | INFO | } | 20260508 04:18:25.661 | INFO | Perf report for the last 5.13742 seconds: | 20260508 04:18:25.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:25.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 04:18:25.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:18:25.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:25.661 | INFO | Uploaded 36GiB to GPU in 2.56007s (14.0621GiB/s) | 20260508 04:18:25.661 | INFO | Downloaded 34GiB to GPU in 2.44992s (13.878GiB/s) | 20260508 04:18:30.800 | INFO | Analyzing chunk frames = [5178368,5178880) | 20260508 04:18:30.800 | INFO | MemInfo: { | 20260508 04:18:30.800 | INFO | "gpu" : " 17 GB", | 20260508 04:18:30.800 | INFO | "huge" : "", | 20260508 04:18:30.800 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:30.800 | INFO | "malloc" : " 14 GB", | 20260508 04:18:30.800 | INFO | "pinned" : " 29 GB" | 20260508 04:18:30.800 | INFO | } | 20260508 04:18:30.800 | INFO | Perf report for the last 5.13942 seconds: | 20260508 04:18:30.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:30.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 04:18:30.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:18:30.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:30.800 | INFO | Uploaded 36GiB to GPU in 2.50662s (14.362GiB/s) | 20260508 04:18:30.800 | INFO | Downloaded 34GiB to GPU in 2.43963s (13.9365GiB/s) | 20260508 04:18:35.940 | INFO | Analyzing chunk frames = [5178880,5179392) | 20260508 04:18:35.940 | INFO | MemInfo: { | 20260508 04:18:35.940 | INFO | "gpu" : " 17 GB", | 20260508 04:18:35.940 | INFO | "huge" : "", | 20260508 04:18:35.940 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:35.940 | INFO | "malloc" : " 14 GB", | 20260508 04:18:35.940 | INFO | "pinned" : " 29 GB" | 20260508 04:18:35.940 | INFO | } | 20260508 04:18:35.940 | INFO | Perf report for the last 5.14028 seconds: | 20260508 04:18:35.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:35.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.940 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 04:18:35.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.940 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:18:35.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:35.941 | INFO | Uploaded 36GiB to GPU in 2.49669s (14.4191GiB/s) | 20260508 04:18:35.941 | INFO | Downloaded 34GiB to GPU in 2.51861s (13.4995GiB/s) | 20260508 04:18:41.082 | INFO | Analyzing chunk frames = [5179392,5179904) | 20260508 04:18:41.082 | INFO | MemInfo: { | 20260508 04:18:41.082 | INFO | "gpu" : " 17 GB", | 20260508 04:18:41.082 | INFO | "huge" : "", | 20260508 04:18:41.082 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:41.082 | INFO | "malloc" : " 14 GB", | 20260508 04:18:41.082 | INFO | "pinned" : " 29 GB" | 20260508 04:18:41.082 | INFO | } | 20260508 04:18:41.083 | INFO | Perf report for the last 5.14228 seconds: | 20260508 04:18:41.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:41.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 04:18:41.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:18:41.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:41.083 | INFO | Uploaded 36GiB to GPU in 2.39593s (15.0255GiB/s) | 20260508 04:18:41.083 | INFO | Downloaded 34GiB to GPU in 2.57896s (13.1836GiB/s) | 20260508 04:18:46.220 | INFO | Analyzing chunk frames = [5179904,5180416) | 20260508 04:18:46.220 | INFO | MemInfo: { | 20260508 04:18:46.220 | INFO | "gpu" : " 17 GB", | 20260508 04:18:46.221 | INFO | "huge" : "", | 20260508 04:18:46.221 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:46.221 | INFO | "malloc" : " 14 GB", | 20260508 04:18:46.221 | INFO | "pinned" : " 29 GB" | 20260508 04:18:46.221 | INFO | } | 20260508 04:18:46.221 | INFO | Perf report for the last 5.13824 seconds: | 20260508 04:18:46.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:46.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:18:46.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:18:46.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:46.221 | INFO | Uploaded 36GiB to GPU in 2.40028s (14.9983GiB/s) | 20260508 04:18:46.221 | INFO | Downloaded 34GiB to GPU in 2.4452s (13.9048GiB/s) | 20260508 04:18:51.355 | INFO | Analyzing chunk frames = [5180416,5180928) | 20260508 04:18:51.355 | INFO | MemInfo: { | 20260508 04:18:51.355 | INFO | "gpu" : " 17 GB", | 20260508 04:18:51.355 | INFO | "huge" : "", | 20260508 04:18:51.355 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:51.355 | INFO | "malloc" : " 14 GB", | 20260508 04:18:51.355 | INFO | "pinned" : " 29 GB" | 20260508 04:18:51.355 | INFO | } | 20260508 04:18:51.355 | INFO | Perf report for the last 5.13423 seconds: | 20260508 04:18:51.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:51.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.355 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 04:18:51.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.355 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:18:51.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:51.356 | INFO | Uploaded 36GiB to GPU in 2.3873s (15.0798GiB/s) | 20260508 04:18:51.356 | INFO | Downloaded 34GiB to GPU in 2.43944s (13.9376GiB/s) | 20260508 04:18:56.499 | INFO | Analyzing chunk frames = [5180928,5181440) | 20260508 04:18:56.499 | INFO | MemInfo: { | 20260508 04:18:56.499 | INFO | "gpu" : " 17 GB", | 20260508 04:18:56.499 | INFO | "huge" : "", | 20260508 04:18:56.499 | INFO | "huge_shared" : " 29 GB", | 20260508 04:18:56.499 | INFO | "malloc" : " 14 GB", | 20260508 04:18:56.499 | INFO | "pinned" : " 29 GB" | 20260508 04:18:56.499 | INFO | } | 20260508 04:18:56.499 | INFO | Perf report for the last 5.14394 seconds: | 20260508 04:18:56.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:18:56.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:18:56.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 04:18:56.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:18:56.499 | INFO | Uploaded 36GiB to GPU in 2.42746s (14.8303GiB/s) | 20260508 04:18:56.499 | INFO | Downloaded 34GiB to GPU in 2.4397s (13.9361GiB/s) | 20260508 04:18:56.611 | INFO | Starting a round of Baz IO | 20260508 04:19:00.067 | INFO | Populating RTMetrics report | 20260508 04:19:00.068 | INFO | RTMetrics report populated | 20260508 04:19:00.068 | INFO | Opening tmp RTMetrics json file | 20260508 04:19:00.072 | INFO | Serializing RTMetrics to json | 20260508 04:19:00.075 | INFO | Serialization done, swapping tmp file now... | 20260508 04:19:00.075 | INFO | RTMetrics done | 20260508 04:19:00.135 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.523 seconds | 20260508 04:19:00.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.09% / 0.00% / 58.91% | 20260508 04:19:00.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.546213% / 3.97364e-06% / 7.54992e-05% | 20260508 04:19:00.135 | INFO | Baz IO overall: | 20260508 04:19:00.135 | INFO | BytesWritten : 2321099979199 | 20260508 04:19:00.135 | INFO | NumEvents : 1120734080771 | 20260508 04:19:01.641 | INFO | Analyzing chunk frames = [5181440,5181952) | 20260508 04:19:01.641 | INFO | MemInfo: { | 20260508 04:19:01.641 | INFO | "gpu" : " 17 GB", | 20260508 04:19:01.641 | INFO | "huge" : "", | 20260508 04:19:01.641 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:01.641 | INFO | "malloc" : " 14 GB", | 20260508 04:19:01.641 | INFO | "pinned" : " 29 GB" | 20260508 04:19:01.641 | INFO | } | 20260508 04:19:01.641 | INFO | Perf report for the last 5.14205 seconds: | 20260508 04:19:01.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:01.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:01.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:01.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:01.641 | INFO | ANALYSIS : 69.0%, 2.9, 104.5ms, 0 | 20260508 04:19:01.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:01.641 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260508 04:19:01.641 | INFO | PRE_HQ : 67.9%, 1.8, 62.3ms, 0 | 20260508 04:19:01.641 | INFO | BAZWRITER : 67.7%, 3.6, 126.4ms, 0 | 20260508 04:19:01.641 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 04:19:01.641 | INFO | Uploaded 36GiB to GPU in 2.55969s (14.0642GiB/s) | 20260508 04:19:01.641 | INFO | Downloaded 38GiB to GPU in 2.69301s (14.1106GiB/s) | 20260508 04:19:06.773 | INFO | Analyzing chunk frames = [5181952,5182464) | 20260508 04:19:06.773 | INFO | MemInfo: { | 20260508 04:19:06.773 | INFO | "gpu" : " 17 GB", | 20260508 04:19:06.773 | INFO | "huge" : "", | 20260508 04:19:06.773 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:06.773 | INFO | "malloc" : " 14 GB", | 20260508 04:19:06.773 | INFO | "pinned" : " 29 GB" | 20260508 04:19:06.773 | INFO | } | 20260508 04:19:06.773 | INFO | Perf report for the last 5.13169 seconds: | 20260508 04:19:06.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:06.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:19:06.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:19:06.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:06.773 | INFO | Uploaded 36GiB to GPU in 2.5747s (13.9822GiB/s) | 20260508 04:19:06.773 | INFO | Downloaded 34GiB to GPU in 2.4267s (14.0108GiB/s) | 20260508 04:19:11.909 | INFO | Analyzing chunk frames = [5182464,5182976) | 20260508 04:19:11.909 | INFO | MemInfo: { | 20260508 04:19:11.909 | INFO | "gpu" : " 17 GB", | 20260508 04:19:11.909 | INFO | "huge" : "", | 20260508 04:19:11.910 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:11.910 | INFO | "malloc" : " 14 GB", | 20260508 04:19:11.910 | INFO | "pinned" : " 29 GB" | 20260508 04:19:11.910 | INFO | } | 20260508 04:19:11.910 | INFO | Perf report for the last 5.13695 seconds: | 20260508 04:19:11.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:11.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 04:19:11.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:19:11.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:11.910 | INFO | Uploaded 36GiB to GPU in 2.53391s (14.2073GiB/s) | 20260508 04:19:11.910 | INFO | Downloaded 34GiB to GPU in 2.50008s (13.5996GiB/s) | 20260508 04:19:17.057 | INFO | Analyzing chunk frames = [5182976,5183488) | 20260508 04:19:17.057 | INFO | MemInfo: { | 20260508 04:19:17.057 | INFO | "gpu" : " 17 GB", | 20260508 04:19:17.057 | INFO | "huge" : "", | 20260508 04:19:17.057 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:17.057 | INFO | "malloc" : " 14 GB", | 20260508 04:19:17.057 | INFO | "pinned" : " 29 GB" | 20260508 04:19:17.057 | INFO | } | 20260508 04:19:17.057 | INFO | Perf report for the last 5.1475 seconds: | 20260508 04:19:17.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:17.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 04:19:17.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:19:17.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:17.057 | INFO | Uploaded 36GiB to GPU in 2.53566s (14.1975GiB/s) | 20260508 04:19:17.057 | INFO | Downloaded 34GiB to GPU in 2.50409s (13.5778GiB/s) | 20260508 04:19:22.199 | INFO | Analyzing chunk frames = [5183488,5184000) | 20260508 04:19:22.200 | INFO | MemInfo: { | 20260508 04:19:22.200 | INFO | "gpu" : " 17 GB", | 20260508 04:19:22.200 | INFO | "huge" : "", | 20260508 04:19:22.200 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:22.200 | INFO | "malloc" : " 14 GB", | 20260508 04:19:22.200 | INFO | "pinned" : " 29 GB" | 20260508 04:19:22.200 | INFO | } | 20260508 04:19:22.200 | INFO | Perf report for the last 5.14256 seconds: | 20260508 04:19:22.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:22.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:19:22.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:19:22.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:22.200 | INFO | Uploaded 36GiB to GPU in 2.34752s (15.3353GiB/s) | 20260508 04:19:22.200 | INFO | Downloaded 34GiB to GPU in 2.54848s (13.3413GiB/s) | 20260508 04:19:27.336 | INFO | Analyzing chunk frames = [5184000,5184512) | 20260508 04:19:27.336 | INFO | MemInfo: { | 20260508 04:19:27.336 | INFO | "gpu" : " 17 GB", | 20260508 04:19:27.336 | INFO | "huge" : "", | 20260508 04:19:27.336 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:27.336 | INFO | "malloc" : " 14 GB", | 20260508 04:19:27.336 | INFO | "pinned" : " 29 GB" | 20260508 04:19:27.336 | INFO | } | 20260508 04:19:27.336 | INFO | Perf report for the last 5.13656 seconds: | 20260508 04:19:27.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:27.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.336 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:19:27.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.337 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:19:27.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:27.337 | INFO | Uploaded 36GiB to GPU in 2.45009s (14.6933GiB/s) | 20260508 04:19:27.337 | INFO | Downloaded 34GiB to GPU in 2.40888s (14.1145GiB/s) | 20260508 04:19:32.472 | INFO | Analyzing chunk frames = [5184512,5185024) | 20260508 04:19:32.472 | INFO | MemInfo: { | 20260508 04:19:32.472 | INFO | "gpu" : " 17 GB", | 20260508 04:19:32.472 | INFO | "huge" : "", | 20260508 04:19:32.472 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:32.472 | INFO | "malloc" : " 14 GB", | 20260508 04:19:32.472 | INFO | "pinned" : " 29 GB" | 20260508 04:19:32.472 | INFO | } | 20260508 04:19:32.472 | INFO | Perf report for the last 5.13605 seconds: | 20260508 04:19:32.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:32.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.473 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:19:32.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.473 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 04:19:32.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:32.474 | INFO | Uploaded 36GiB to GPU in 2.4782s (14.5267GiB/s) | 20260508 04:19:32.474 | INFO | Downloaded 34GiB to GPU in 2.43156s (13.9828GiB/s) | 20260508 04:19:37.611 | INFO | Analyzing chunk frames = [5185024,5185536) | 20260508 04:19:37.611 | INFO | MemInfo: { | 20260508 04:19:37.611 | INFO | "gpu" : " 17 GB", | 20260508 04:19:37.611 | INFO | "huge" : "", | 20260508 04:19:37.611 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:37.611 | INFO | "malloc" : " 14 GB", | 20260508 04:19:37.611 | INFO | "pinned" : " 29 GB" | 20260508 04:19:37.611 | INFO | } | 20260508 04:19:37.611 | INFO | Perf report for the last 5.1384 seconds: | 20260508 04:19:37.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:37.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:19:37.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260508 04:19:37.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:37.611 | INFO | Uploaded 36GiB to GPU in 2.45875s (14.6416GiB/s) | 20260508 04:19:37.611 | INFO | Downloaded 34GiB to GPU in 2.42912s (13.9969GiB/s) | 20260508 04:19:37.611 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:19:37.611 | INFO | Total mem usage: 16.7 GiB | 20260508 04:19:37.611 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:19:37.611 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260508 04:19:37.611 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:19:37.611 | INFO | High Water Breakdown: | 20260508 04:19:37.611 | INFO | 59.9% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:19:37.611 | INFO | 39.9% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:19:37.611 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:19:37.611 | INFO | ------------------------------------------ | 20260508 04:19:37.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:19:37.611 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:19:37.611 | INFO | Total mem usage: 26.5 GiB | 20260508 04:19:37.611 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:19:37.611 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:19:37.611 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:19:37.611 | INFO | High Water Breakdown: | 20260508 04:19:37.611 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:19:37.611 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:19:37.611 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:19:37.611 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:19:37.611 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:19:37.611 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:19:37.611 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:19:37.611 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:19:37.611 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:19:37.611 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:19:37.611 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:19:37.611 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:19:37.611 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:19:37.611 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:19:37.611 | INFO | ------------------------------------------ | 20260508 04:19:37.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:19:37.611 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:19:37.611 | INFO | Total mem usage: 27.8 GiB | 20260508 04:19:37.611 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:19:37.611 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:19:37.611 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:19:37.611 | INFO | High Water Breakdown: | 20260508 04:19:37.611 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:19:37.611 | INFO | ------------------------------------------ | 20260508 04:19:37.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:19:37.611 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:19:37.611 | INFO | Total mem usage: 15.7 GiB | 20260508 04:19:37.611 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:19:37.611 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:19:37.611 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:19:37.611 | INFO | High Water Breakdown: | 20260508 04:19:37.611 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:19:37.611 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:19:37.611 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:19:37.611 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:19:37.611 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:19:37.611 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:19:37.611 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:19:37.611 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:19:37.611 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:19:37.611 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:19:37.611 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:19:37.611 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:19:37.611 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:19:37.611 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:19:37.611 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:19:37.611 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:19:37.611 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:19:37.611 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:19:37.611 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:19:37.611 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:19:37.611 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:19:37.611 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:19:37.611 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:19:37.611 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:19:37.611 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:19:37.611 | INFO | ------------------------------------------ | 20260508 04:19:37.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:19:37.611 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:19:37.612 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:19:37.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:19:37.612 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 1.77e+00 | 20260508 04:19:37.612 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.01e-01, 1.14e-02 / 5.26e-03 | 20260508 04:19:37.612 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.50e+00, 5.90e-01 / 1.97e-01 | 20260508 04:19:37.612 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.08e-01 / 1.13e-01 | 20260508 04:19:37.613 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.63e+01, 3.95e+00 / 2.20e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.85e+01 / 1.96e+01, 1.65e+00 / 1.41e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.40e+00 / 2.66e+00, 4.23e-01 / 2.10e-01 | 20260508 04:19:37.613 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.17e+00, 2.64e-01 / 8.73e-02 | 20260508 04:19:37.613 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.51e-01, 4.99e-02 / 2.15e-02 | 20260508 04:19:37.613 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:19:37.613 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:19:37.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:19:37.613 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 24.52%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 2.90%, LockDownload -- 3.08e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 15.35%, LockCompute -- 1.63e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 04:19:37.613 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260508 04:19:37.708 | INFO | Starting a round of Baz IO | 20260508 04:19:41.150 | INFO | Populating RTMetrics report | 20260508 04:19:41.150 | INFO | RTMetrics report populated | 20260508 04:19:41.150 | INFO | Opening tmp RTMetrics json file | 20260508 04:19:41.166 | INFO | Serializing RTMetrics to json | 20260508 04:19:41.169 | INFO | Serialization done, swapping tmp file now... | 20260508 04:19:41.169 | INFO | RTMetrics done | 20260508 04:19:41.223 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.515 seconds | 20260508 04:19:41.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.09% / 0.00% / 58.91% | 20260508 04:19:41.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.544675% / 0% / 6.35783e-05% | 20260508 04:19:41.223 | INFO | Baz IO overall: | 20260508 04:19:41.223 | INFO | BytesWritten : 2322814057528 | 20260508 04:19:41.223 | INFO | NumEvents : 1121552816906 | 20260508 04:19:42.751 | INFO | Analyzing chunk frames = [5185536,5186048) | 20260508 04:19:42.751 | INFO | MemInfo: { | 20260508 04:19:42.751 | INFO | "gpu" : " 17 GB", | 20260508 04:19:42.751 | INFO | "huge" : "", | 20260508 04:19:42.751 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:42.751 | INFO | "malloc" : " 14 GB", | 20260508 04:19:42.751 | INFO | "pinned" : " 29 GB" | 20260508 04:19:42.751 | INFO | } | 20260508 04:19:42.751 | INFO | Perf report for the last 5.1399 seconds: | 20260508 04:19:42.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:42.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:42.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:42.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:42.751 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 04:19:42.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:42.751 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 04:19:42.751 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 04:19:42.751 | INFO | BAZWRITER : 67.9%, 3.6, 124.9ms, 0 | 20260508 04:19:42.751 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:19:42.751 | INFO | Uploaded 36GiB to GPU in 2.5539s (14.0961GiB/s) | 20260508 04:19:42.751 | INFO | Downloaded 38GiB to GPU in 2.69703s (14.0896GiB/s) | 20260508 04:19:47.893 | INFO | Analyzing chunk frames = [5186048,5186560) | 20260508 04:19:47.893 | INFO | MemInfo: { | 20260508 04:19:47.893 | INFO | "gpu" : " 17 GB", | 20260508 04:19:47.893 | INFO | "huge" : "", | 20260508 04:19:47.893 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:47.893 | INFO | "malloc" : " 14 GB", | 20260508 04:19:47.893 | INFO | "pinned" : " 29 GB" | 20260508 04:19:47.893 | INFO | } | 20260508 04:19:47.893 | INFO | Perf report for the last 5.14249 seconds: | 20260508 04:19:47.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:47.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:19:47.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:19:47.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:47.893 | INFO | Uploaded 36GiB to GPU in 2.59103s (13.8941GiB/s) | 20260508 04:19:47.894 | INFO | Downloaded 34GiB to GPU in 2.43126s (13.9845GiB/s) | 20260508 04:19:53.029 | INFO | Analyzing chunk frames = [5186560,5187072) | 20260508 04:19:53.029 | INFO | MemInfo: { | 20260508 04:19:53.029 | INFO | "gpu" : " 17 GB", | 20260508 04:19:53.029 | INFO | "huge" : "", | 20260508 04:19:53.029 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:53.029 | INFO | "malloc" : " 14 GB", | 20260508 04:19:53.029 | INFO | "pinned" : " 29 GB" | 20260508 04:19:53.029 | INFO | } | 20260508 04:19:53.029 | INFO | Perf report for the last 5.1356 seconds: | 20260508 04:19:53.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:53.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | ANALYSIS : 67.6%, 3.0, 104.7ms, 0 | 20260508 04:19:53.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:19:53.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:53.029 | INFO | Uploaded 36GiB to GPU in 2.58231s (13.941GiB/s) | 20260508 04:19:53.029 | INFO | Downloaded 34GiB to GPU in 2.49607s (13.6214GiB/s) | 20260508 04:19:58.172 | INFO | Analyzing chunk frames = [5187072,5187584) | 20260508 04:19:58.172 | INFO | MemInfo: { | 20260508 04:19:58.172 | INFO | "gpu" : " 17 GB", | 20260508 04:19:58.172 | INFO | "huge" : "", | 20260508 04:19:58.172 | INFO | "huge_shared" : " 29 GB", | 20260508 04:19:58.172 | INFO | "malloc" : " 14 GB", | 20260508 04:19:58.172 | INFO | "pinned" : " 29 GB" | 20260508 04:19:58.172 | INFO | } | 20260508 04:19:58.172 | INFO | Perf report for the last 5.14296 seconds: | 20260508 04:19:58.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:19:58.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 04:19:58.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:19:58.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:19:58.172 | INFO | Uploaded 36GiB to GPU in 2.56634s (14.0277GiB/s) | 20260508 04:19:58.172 | INFO | Downloaded 34GiB to GPU in 2.4979s (13.6114GiB/s) | 20260508 04:20:03.309 | INFO | Analyzing chunk frames = [5187584,5188096) | 20260508 04:20:03.309 | INFO | MemInfo: { | 20260508 04:20:03.309 | INFO | "gpu" : " 17 GB", | 20260508 04:20:03.309 | INFO | "huge" : "", | 20260508 04:20:03.309 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:03.309 | INFO | "malloc" : " 14 GB", | 20260508 04:20:03.309 | INFO | "pinned" : " 29 GB" | 20260508 04:20:03.309 | INFO | } | 20260508 04:20:03.309 | INFO | Perf report for the last 5.13697 seconds: | 20260508 04:20:03.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:03.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 04:20:03.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:20:03.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:03.309 | INFO | Uploaded 36GiB to GPU in 2.37097s (15.1837GiB/s) | 20260508 04:20:03.309 | INFO | Downloaded 34GiB to GPU in 2.55676s (13.2981GiB/s) | 20260508 04:20:08.452 | INFO | Analyzing chunk frames = [5188096,5188608) | 20260508 04:20:08.452 | INFO | MemInfo: { | 20260508 04:20:08.452 | INFO | "gpu" : " 17 GB", | 20260508 04:20:08.452 | INFO | "huge" : "", | 20260508 04:20:08.452 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:08.452 | INFO | "malloc" : " 14 GB", | 20260508 04:20:08.452 | INFO | "pinned" : " 29 GB" | 20260508 04:20:08.452 | INFO | } | 20260508 04:20:08.452 | INFO | Perf report for the last 5.1434 seconds: | 20260508 04:20:08.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:08.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.452 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:20:08.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.452 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:20:08.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:08.453 | INFO | Uploaded 36GiB to GPU in 2.39747s (15.0158GiB/s) | 20260508 04:20:08.453 | INFO | Downloaded 34GiB to GPU in 2.42706s (14.0087GiB/s) | 20260508 04:20:13.588 | INFO | Analyzing chunk frames = [5188608,5189120) | 20260508 04:20:13.588 | INFO | MemInfo: { | 20260508 04:20:13.588 | INFO | "gpu" : " 17 GB", | 20260508 04:20:13.588 | INFO | "huge" : "", | 20260508 04:20:13.588 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:13.588 | INFO | "malloc" : " 14 GB", | 20260508 04:20:13.588 | INFO | "pinned" : " 29 GB" | 20260508 04:20:13.588 | INFO | } | 20260508 04:20:13.588 | INFO | Perf report for the last 5.13563 seconds: | 20260508 04:20:13.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:13.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.588 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:20:13.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.588 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 04:20:13.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:13.589 | INFO | Uploaded 36GiB to GPU in 2.43649s (14.7753GiB/s) | 20260508 04:20:13.589 | INFO | Downloaded 34GiB to GPU in 2.43061s (13.9883GiB/s) | 20260508 04:20:18.724 | INFO | Analyzing chunk frames = [5189120,5189632) | 20260508 04:20:18.724 | INFO | MemInfo: { | 20260508 04:20:18.724 | INFO | "gpu" : " 17 GB", | 20260508 04:20:18.724 | INFO | "huge" : "", | 20260508 04:20:18.724 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:18.724 | INFO | "malloc" : " 14 GB", | 20260508 04:20:18.724 | INFO | "pinned" : " 29 GB" | 20260508 04:20:18.724 | INFO | } | 20260508 04:20:18.724 | INFO | Perf report for the last 5.13582 seconds: | 20260508 04:20:18.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:18.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:20:18.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | PRE_HQ : 35.2%, 1.0, 18.3ms, 0 | 20260508 04:20:18.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:18.724 | INFO | Uploaded 36GiB to GPU in 2.44986s (14.6947GiB/s) | 20260508 04:20:18.725 | INFO | Downloaded 34GiB to GPU in 2.43076s (13.9874GiB/s) | 20260508 04:20:18.838 | INFO | Starting a round of Baz IO | 20260508 04:20:22.273 | INFO | Populating RTMetrics report | 20260508 04:20:22.274 | INFO | RTMetrics report populated | 20260508 04:20:22.274 | INFO | Opening tmp RTMetrics json file | 20260508 04:20:22.288 | INFO | Serializing RTMetrics to json | 20260508 04:20:22.291 | INFO | Serialization done, swapping tmp file now... | 20260508 04:20:22.293 | INFO | RTMetrics done | 20260508 04:20:22.350 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.512 seconds | 20260508 04:20:22.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.09% / 0.00% / 58.91% | 20260508 04:20:22.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543813% / 7.94729e-06% / 3.17891e-05% | 20260508 04:20:22.351 | INFO | Baz IO overall: | 20260508 04:20:22.351 | INFO | BytesWritten : 2324527880667 | 20260508 04:20:22.351 | INFO | NumEvents : 1122371632923 | 20260508 04:20:23.866 | INFO | Analyzing chunk frames = [5189632,5190144) | 20260508 04:20:23.866 | INFO | MemInfo: { | 20260508 04:20:23.866 | INFO | "gpu" : " 17 GB", | 20260508 04:20:23.866 | INFO | "huge" : "", | 20260508 04:20:23.866 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:23.866 | INFO | "malloc" : " 14 GB", | 20260508 04:20:23.866 | INFO | "pinned" : " 29 GB" | 20260508 04:20:23.866 | INFO | } | 20260508 04:20:23.866 | INFO | Perf report for the last 5.14259 seconds: | 20260508 04:20:23.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:23.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:23.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:23.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:23.866 | INFO | ANALYSIS : 68.3%, 2.9, 104.3ms, 0 | 20260508 04:20:23.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:23.866 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260508 04:20:23.866 | INFO | PRE_HQ : 67.3%, 1.8, 62.8ms, 0 | 20260508 04:20:23.866 | INFO | BAZWRITER : 67.9%, 3.5, 122.2ms, 0 | 20260508 04:20:23.866 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:20:23.867 | INFO | Uploaded 36GiB to GPU in 2.55701s (14.079GiB/s) | 20260508 04:20:23.867 | INFO | Downloaded 38GiB to GPU in 2.7089s (14.0278GiB/s) | 20260508 04:20:29.006 | INFO | Analyzing chunk frames = [5190144,5190656) | 20260508 04:20:29.006 | INFO | MemInfo: { | 20260508 04:20:29.006 | INFO | "gpu" : " 17 GB", | 20260508 04:20:29.006 | INFO | "huge" : "", | 20260508 04:20:29.007 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:29.007 | INFO | "malloc" : " 14 GB", | 20260508 04:20:29.007 | INFO | "pinned" : " 29 GB" | 20260508 04:20:29.007 | INFO | } | 20260508 04:20:29.007 | INFO | Perf report for the last 5.14037 seconds: | 20260508 04:20:29.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:29.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 04:20:29.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:20:29.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:29.007 | INFO | Uploaded 36GiB to GPU in 2.57438s (13.9839GiB/s) | 20260508 04:20:29.007 | INFO | Downloaded 34GiB to GPU in 2.43997s (13.9346GiB/s) | 20260508 04:20:34.148 | INFO | Analyzing chunk frames = [5190656,5191168) | 20260508 04:20:34.148 | INFO | MemInfo: { | 20260508 04:20:34.148 | INFO | "gpu" : " 17 GB", | 20260508 04:20:34.148 | INFO | "huge" : "", | 20260508 04:20:34.148 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:34.148 | INFO | "malloc" : " 14 GB", | 20260508 04:20:34.148 | INFO | "pinned" : " 29 GB" | 20260508 04:20:34.148 | INFO | } | 20260508 04:20:34.148 | INFO | Perf report for the last 5.14168 seconds: | 20260508 04:20:34.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:34.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.148 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:20:34.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.149 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:20:34.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:34.149 | INFO | Uploaded 36GiB to GPU in 2.54998s (14.1177GiB/s) | 20260508 04:20:34.149 | INFO | Downloaded 34GiB to GPU in 2.41515s (14.0778GiB/s) | 20260508 04:20:39.285 | INFO | Analyzing chunk frames = [5191168,5191680) | 20260508 04:20:39.285 | INFO | MemInfo: { | 20260508 04:20:39.285 | INFO | "gpu" : " 17 GB", | 20260508 04:20:39.286 | INFO | "huge" : "", | 20260508 04:20:39.286 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:39.286 | INFO | "malloc" : " 14 GB", | 20260508 04:20:39.286 | INFO | "pinned" : " 29 GB" | 20260508 04:20:39.286 | INFO | } | 20260508 04:20:39.286 | INFO | Perf report for the last 5.13725 seconds: | 20260508 04:20:39.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:39.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:20:39.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:20:39.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:39.286 | INFO | Uploaded 36GiB to GPU in 2.58404s (13.9317GiB/s) | 20260508 04:20:39.286 | INFO | Downloaded 34GiB to GPU in 2.413s (14.0904GiB/s) | 20260508 04:20:44.422 | INFO | Analyzing chunk frames = [5191680,5192192) | 20260508 04:20:44.422 | INFO | MemInfo: { | 20260508 04:20:44.422 | INFO | "gpu" : " 17 GB", | 20260508 04:20:44.422 | INFO | "huge" : "", | 20260508 04:20:44.422 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:44.422 | INFO | "malloc" : " 14 GB", | 20260508 04:20:44.422 | INFO | "pinned" : " 29 GB" | 20260508 04:20:44.422 | INFO | } | 20260508 04:20:44.422 | INFO | Perf report for the last 5.13618 seconds: | 20260508 04:20:44.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:44.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:20:44.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:20:44.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:44.422 | INFO | Uploaded 36GiB to GPU in 2.36539s (15.2195GiB/s) | 20260508 04:20:44.422 | INFO | Downloaded 34GiB to GPU in 2.57459s (13.206GiB/s) | 20260508 04:20:49.567 | INFO | Analyzing chunk frames = [5192192,5192704) | 20260508 04:20:49.567 | INFO | MemInfo: { | 20260508 04:20:49.567 | INFO | "gpu" : " 17 GB", | 20260508 04:20:49.567 | INFO | "huge" : "", | 20260508 04:20:49.567 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:49.567 | INFO | "malloc" : " 14 GB", | 20260508 04:20:49.567 | INFO | "pinned" : " 29 GB" | 20260508 04:20:49.567 | INFO | } | 20260508 04:20:49.567 | INFO | Perf report for the last 5.1453 seconds: | 20260508 04:20:49.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:49.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.567 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:20:49.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.567 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:20:49.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:49.568 | INFO | Uploaded 36GiB to GPU in 2.42055s (14.8726GiB/s) | 20260508 04:20:49.568 | INFO | Downloaded 34GiB to GPU in 2.42382s (14.0274GiB/s) | 20260508 04:20:54.703 | INFO | Analyzing chunk frames = [5192704,5193216) | 20260508 04:20:54.703 | INFO | MemInfo: { | 20260508 04:20:54.703 | INFO | "gpu" : " 17 GB", | 20260508 04:20:54.703 | INFO | "huge" : "", | 20260508 04:20:54.703 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:54.703 | INFO | "malloc" : " 14 GB", | 20260508 04:20:54.703 | INFO | "pinned" : " 29 GB" | 20260508 04:20:54.703 | INFO | } | 20260508 04:20:54.703 | INFO | Perf report for the last 5.13583 seconds: | 20260508 04:20:54.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:54.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.703 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:20:54.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.703 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 04:20:54.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:54.704 | INFO | Uploaded 36GiB to GPU in 2.4307s (14.8105GiB/s) | 20260508 04:20:54.704 | INFO | Downloaded 34GiB to GPU in 2.42827s (14.0017GiB/s) | 20260508 04:20:59.843 | INFO | Analyzing chunk frames = [5193216,5193728) | 20260508 04:20:59.843 | INFO | MemInfo: { | 20260508 04:20:59.843 | INFO | "gpu" : " 17 GB", | 20260508 04:20:59.843 | INFO | "huge" : "", | 20260508 04:20:59.843 | INFO | "huge_shared" : " 29 GB", | 20260508 04:20:59.843 | INFO | "malloc" : " 14 GB", | 20260508 04:20:59.843 | INFO | "pinned" : " 29 GB" | 20260508 04:20:59.843 | INFO | } | 20260508 04:20:59.843 | INFO | Perf report for the last 5.14031 seconds: | 20260508 04:20:59.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:20:59.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.843 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:20:59.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.843 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 04:20:59.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:20:59.844 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:20:59.844 | INFO | Total mem usage: 16.7 GiB | 20260508 04:20:59.844 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:20:59.844 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.6%) | 20260508 04:20:59.844 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:20:59.844 | INFO | High Water Breakdown: | 20260508 04:20:59.844 | INFO | 59.8% : 8.7 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:20:59.844 | INFO | 40.1% : 5.84 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:20:59.844 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:20:59.844 | INFO | ------------------------------------------ | 20260508 04:20:59.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:20:59.844 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:20:59.844 | INFO | Total mem usage: 26.5 GiB | 20260508 04:20:59.844 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:20:59.844 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:20:59.844 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:20:59.844 | INFO | Uploaded 36GiB to GPU in 2.46002s (14.634GiB/s) | 20260508 04:20:59.844 | INFO | High Water Breakdown: | 20260508 04:20:59.844 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:20:59.844 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:20:59.844 | INFO | Downloaded 34GiB to GPU in 2.4361s (13.9568GiB/s) | 20260508 04:20:59.844 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:20:59.844 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:20:59.844 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:20:59.844 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:20:59.844 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:20:59.844 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:20:59.844 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:20:59.844 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:20:59.844 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:20:59.844 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:20:59.844 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:20:59.844 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:20:59.844 | INFO | ------------------------------------------ | 20260508 04:20:59.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:20:59.844 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:20:59.844 | INFO | Total mem usage: 27.8 GiB | 20260508 04:20:59.844 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:20:59.844 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:20:59.844 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:20:59.844 | INFO | High Water Breakdown: | 20260508 04:20:59.844 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:20:59.844 | INFO | ------------------------------------------ | 20260508 04:20:59.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:20:59.844 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:20:59.844 | INFO | Total mem usage: 15.7 GiB | 20260508 04:20:59.844 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:20:59.844 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:20:59.844 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:20:59.844 | INFO | High Water Breakdown: | 20260508 04:20:59.845 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:20:59.845 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:20:59.845 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:20:59.845 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:20:59.845 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:20:59.845 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:20:59.845 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:20:59.845 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:20:59.845 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:20:59.845 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:20:59.845 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:20:59.845 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:20:59.845 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:20:59.845 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:20:59.845 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:20:59.845 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:20:59.845 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:20:59.845 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:20:59.845 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:20:59.845 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:20:59.845 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:20:59.845 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:20:59.845 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:20:59.845 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:20:59.845 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:20:59.845 | INFO | ------------------------------------------ | 20260508 04:20:59.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:20:59.845 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:20:59.845 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:20:59.845 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:20:59.845 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.62e+01, 1.41e+01 / 2.07e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.03e-01, 1.14e-02 / 6.56e-03 | 20260508 04:20:59.845 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.51e+00, 5.91e-01 / 1.88e-01 | 20260508 04:20:59.845 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.12e-01 | 20260508 04:20:59.845 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.61e+01, 4.09e+00 / 2.41e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.71e+00 / 1.27e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.40e+00 / 2.64e+00, 4.18e-01 / 1.72e-01 | 20260508 04:20:59.845 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.19e+00, 2.65e-01 / 1.02e-01 | 20260508 04:20:59.845 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.58e-01, 5.06e-02 / 1.98e-02 | 20260508 04:20:59.845 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:20:59.845 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:20:59.845 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:20:59.845 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 24.38%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 2.25%, LockUpload -- 2.39e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 2.67%, LockDownload -- 2.84e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 15.50%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 04:20:59.845 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.44e-02 / 0.00e+00 | 20260508 04:20:59.958 | INFO | Starting a round of Baz IO | 20260508 04:21:03.409 | INFO | Populating RTMetrics report | 20260508 04:21:03.410 | INFO | RTMetrics report populated | 20260508 04:21:03.410 | INFO | Opening tmp RTMetrics json file | 20260508 04:21:03.486 | INFO | Serializing RTMetrics to json | 20260508 04:21:03.494 | INFO | Serialization done, swapping tmp file now... | 20260508 04:21:03.497 | INFO | RTMetrics done | 20260508 04:21:03.505 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.546 seconds | 20260508 04:21:03.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.08% / 0.00% / 58.91% | 20260508 04:21:03.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.5422% / 3.97364e-06% / 7.15256e-05% | 20260508 04:21:03.505 | INFO | Baz IO overall: | 20260508 04:21:03.505 | INFO | BytesWritten : 2326241420158 | 20260508 04:21:03.505 | INFO | NumEvents : 1123190326819 | 20260508 04:21:04.978 | INFO | Analyzing chunk frames = [5193728,5194240) | 20260508 04:21:04.978 | INFO | MemInfo: { | 20260508 04:21:04.978 | INFO | "gpu" : " 17 GB", | 20260508 04:21:04.978 | INFO | "huge" : "", | 20260508 04:21:04.978 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:04.978 | INFO | "malloc" : " 14 GB", | 20260508 04:21:04.978 | INFO | "pinned" : " 29 GB" | 20260508 04:21:04.978 | INFO | } | 20260508 04:21:04.978 | INFO | Perf report for the last 5.13468 seconds: | 20260508 04:21:04.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:04.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:04.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:04.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:04.978 | INFO | ANALYSIS : 68.9%, 2.9, 104.6ms, 0 | 20260508 04:21:04.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:04.978 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260508 04:21:04.978 | INFO | PRE_HQ : 67.4%, 1.8, 61.8ms, 0 | 20260508 04:21:04.978 | INFO | BAZWRITER : 68.7%, 4.0, 141.2ms, 0 | 20260508 04:21:04.978 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260508 04:21:04.979 | INFO | Uploaded 36GiB to GPU in 2.56592s (14.0301GiB/s) | 20260508 04:21:04.979 | INFO | Downloaded 38GiB to GPU in 2.70644s (14.0406GiB/s) | 20260508 04:21:10.120 | INFO | Analyzing chunk frames = [5194240,5194752) | 20260508 04:21:10.120 | INFO | MemInfo: { | 20260508 04:21:10.120 | INFO | "gpu" : " 17 GB", | 20260508 04:21:10.120 | INFO | "huge" : "", | 20260508 04:21:10.120 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:10.120 | INFO | "malloc" : " 14 GB", | 20260508 04:21:10.120 | INFO | "pinned" : " 29 GB" | 20260508 04:21:10.120 | INFO | } | 20260508 04:21:10.120 | INFO | Perf report for the last 5.14202 seconds: | 20260508 04:21:10.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:10.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 04:21:10.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:21:10.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:10.120 | INFO | Uploaded 36GiB to GPU in 2.55506s (14.0897GiB/s) | 20260508 04:21:10.120 | INFO | Downloaded 34GiB to GPU in 2.41757s (14.0637GiB/s) | 20260508 04:21:15.261 | INFO | Analyzing chunk frames = [5194752,5195264) | 20260508 04:21:15.261 | INFO | MemInfo: { | 20260508 04:21:15.261 | INFO | "gpu" : " 17 GB", | 20260508 04:21:15.261 | INFO | "huge" : "", | 20260508 04:21:15.261 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:15.261 | INFO | "malloc" : " 14 GB", | 20260508 04:21:15.261 | INFO | "pinned" : " 29 GB" | 20260508 04:21:15.261 | INFO | } | 20260508 04:21:15.261 | INFO | Perf report for the last 5.14111 seconds: | 20260508 04:21:15.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:15.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.261 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:21:15.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.261 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:21:15.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:15.262 | INFO | Uploaded 36GiB to GPU in 2.56692s (14.0246GiB/s) | 20260508 04:21:15.262 | INFO | Downloaded 34GiB to GPU in 2.49903s (13.6053GiB/s) | 20260508 04:21:20.403 | INFO | Analyzing chunk frames = [5195264,5195776) | 20260508 04:21:20.403 | INFO | MemInfo: { | 20260508 04:21:20.403 | INFO | "gpu" : " 17 GB", | 20260508 04:21:20.403 | INFO | "huge" : "", | 20260508 04:21:20.403 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:20.403 | INFO | "malloc" : " 14 GB", | 20260508 04:21:20.403 | INFO | "pinned" : " 29 GB" | 20260508 04:21:20.403 | INFO | } | 20260508 04:21:20.403 | INFO | Perf report for the last 5.14221 seconds: | 20260508 04:21:20.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:20.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.403 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:21:20.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.403 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:21:20.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:20.404 | INFO | Uploaded 36GiB to GPU in 2.5855s (13.9238GiB/s) | 20260508 04:21:20.404 | INFO | Downloaded 34GiB to GPU in 2.41924s (14.054GiB/s) | 20260508 04:21:25.541 | INFO | Analyzing chunk frames = [5195776,5196288) | 20260508 04:21:25.541 | INFO | MemInfo: { | 20260508 04:21:25.541 | INFO | "gpu" : " 17 GB", | 20260508 04:21:25.541 | INFO | "huge" : "", | 20260508 04:21:25.542 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:25.542 | INFO | "malloc" : " 14 GB", | 20260508 04:21:25.542 | INFO | "pinned" : " 29 GB" | 20260508 04:21:25.542 | INFO | } | 20260508 04:21:25.542 | INFO | Perf report for the last 5.13823 seconds: | 20260508 04:21:25.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:25.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:21:25.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:21:25.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:25.542 | INFO | Uploaded 36GiB to GPU in 2.37678s (15.1465GiB/s) | 20260508 04:21:25.542 | INFO | Downloaded 34GiB to GPU in 2.57774s (13.1899GiB/s) | 20260508 04:21:30.675 | INFO | Analyzing chunk frames = [5196288,5196800) | 20260508 04:21:30.675 | INFO | MemInfo: { | 20260508 04:21:30.675 | INFO | "gpu" : " 17 GB", | 20260508 04:21:30.675 | INFO | "huge" : "", | 20260508 04:21:30.675 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:30.675 | INFO | "malloc" : " 14 GB", | 20260508 04:21:30.675 | INFO | "pinned" : " 29 GB" | 20260508 04:21:30.675 | INFO | } | 20260508 04:21:30.675 | INFO | Perf report for the last 5.13353 seconds: | 20260508 04:21:30.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:30.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.675 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:21:30.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.675 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:21:30.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:30.676 | INFO | Uploaded 36GiB to GPU in 2.43344s (14.7939GiB/s) | 20260508 04:21:30.676 | INFO | Downloaded 34GiB to GPU in 2.41878s (14.0567GiB/s) | 20260508 04:21:35.820 | INFO | Analyzing chunk frames = [5196800,5197312) | 20260508 04:21:35.820 | INFO | MemInfo: { | 20260508 04:21:35.820 | INFO | "gpu" : " 17 GB", | 20260508 04:21:35.820 | INFO | "huge" : "", | 20260508 04:21:35.820 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:35.821 | INFO | "malloc" : " 14 GB", | 20260508 04:21:35.821 | INFO | "pinned" : " 29 GB" | 20260508 04:21:35.821 | INFO | } | 20260508 04:21:35.821 | INFO | Perf report for the last 5.14546 seconds: | 20260508 04:21:35.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:35.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:21:35.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 04:21:35.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:35.821 | INFO | Uploaded 36GiB to GPU in 2.43825s (14.7647GiB/s) | 20260508 04:21:35.821 | INFO | Downloaded 34GiB to GPU in 2.42366s (14.0284GiB/s) | 20260508 04:21:40.958 | INFO | Analyzing chunk frames = [5197312,5197824) | 20260508 04:21:40.958 | INFO | MemInfo: { | 20260508 04:21:40.958 | INFO | "gpu" : " 17 GB", | 20260508 04:21:40.958 | INFO | "huge" : "", | 20260508 04:21:40.958 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:40.958 | INFO | "malloc" : " 14 GB", | 20260508 04:21:40.958 | INFO | "pinned" : " 29 GB" | 20260508 04:21:40.958 | INFO | } | 20260508 04:21:40.958 | INFO | Perf report for the last 5.13738 seconds: | 20260508 04:21:40.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:40.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.958 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:21:40.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.958 | INFO | PRE_HQ : 34.8%, 1.0, 18.1ms, 0 | 20260508 04:21:40.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:40.959 | INFO | Uploaded 36GiB to GPU in 2.44808s (14.7054GiB/s) | 20260508 04:21:40.959 | INFO | Downloaded 34GiB to GPU in 2.42149s (14.0409GiB/s) | 20260508 04:21:41.072 | INFO | Starting a round of Baz IO | 20260508 04:21:44.524 | INFO | Populating RTMetrics report | 20260508 04:21:44.524 | INFO | RTMetrics report populated | 20260508 04:21:44.524 | INFO | Opening tmp RTMetrics json file | 20260508 04:21:44.525 | INFO | Serializing RTMetrics to json | 20260508 04:21:44.526 | INFO | Serialization done, swapping tmp file now... | 20260508 04:21:44.526 | INFO | RTMetrics done | 20260508 04:21:44.644 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.572 seconds | 20260508 04:21:44.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.08% / 0.00% / 58.92% | 20260508 04:21:44.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.541353% / 0% / 5.16574e-05% | 20260508 04:21:44.644 | INFO | Baz IO overall: | 20260508 04:21:44.644 | INFO | BytesWritten : 2327955329605 | 20260508 04:21:44.644 | INFO | NumEvents : 1124009402101 | 20260508 04:21:46.098 | INFO | Analyzing chunk frames = [5197824,5198336) | 20260508 04:21:46.098 | INFO | MemInfo: { | 20260508 04:21:46.098 | INFO | "gpu" : " 17 GB", | 20260508 04:21:46.098 | INFO | "huge" : "", | 20260508 04:21:46.098 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:46.098 | INFO | "malloc" : " 14 GB", | 20260508 04:21:46.098 | INFO | "pinned" : " 29 GB" | 20260508 04:21:46.098 | INFO | } | 20260508 04:21:46.098 | INFO | Perf report for the last 5.14023 seconds: | 20260508 04:21:46.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:46.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:46.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:46.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:46.098 | INFO | ANALYSIS : 69.1%, 2.9, 104.7ms, 0 | 20260508 04:21:46.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:46.098 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 04:21:46.098 | INFO | PRE_HQ : 67.8%, 1.8, 62.1ms, 0 | 20260508 04:21:46.098 | INFO | BAZWRITER : 69.2%, 3.6, 127.4ms, 0 | 20260508 04:21:46.098 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260508 04:21:46.099 | INFO | Uploaded 36GiB to GPU in 2.53628s (14.194GiB/s) | 20260508 04:21:46.099 | INFO | Downloaded 38GiB to GPU in 2.71667s (13.9877GiB/s) | 20260508 04:21:51.232 | INFO | Analyzing chunk frames = [5198336,5198848) | 20260508 04:21:51.232 | INFO | MemInfo: { | 20260508 04:21:51.232 | INFO | "gpu" : " 17 GB", | 20260508 04:21:51.232 | INFO | "huge" : "", | 20260508 04:21:51.232 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:51.232 | INFO | "malloc" : " 14 GB", | 20260508 04:21:51.232 | INFO | "pinned" : " 29 GB" | 20260508 04:21:51.232 | INFO | } | 20260508 04:21:51.232 | INFO | Perf report for the last 5.13409 seconds: | 20260508 04:21:51.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:51.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.233 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:21:51.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.233 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:21:51.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:51.233 | INFO | Uploaded 36GiB to GPU in 2.58572s (13.9226GiB/s) | 20260508 04:21:51.233 | INFO | Downloaded 34GiB to GPU in 2.43576s (13.9587GiB/s) | 20260508 04:21:56.374 | INFO | Analyzing chunk frames = [5198848,5199360) | 20260508 04:21:56.374 | INFO | MemInfo: { | 20260508 04:21:56.374 | INFO | "gpu" : " 17 GB", | 20260508 04:21:56.374 | INFO | "huge" : "", | 20260508 04:21:56.374 | INFO | "huge_shared" : " 29 GB", | 20260508 04:21:56.374 | INFO | "malloc" : " 14 GB", | 20260508 04:21:56.375 | INFO | "pinned" : " 29 GB" | 20260508 04:21:56.375 | INFO | } | 20260508 04:21:56.375 | INFO | Perf report for the last 5.14219 seconds: | 20260508 04:21:56.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:21:56.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 04:21:56.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:21:56.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:21:56.375 | INFO | Uploaded 36GiB to GPU in 2.54572s (14.1414GiB/s) | 20260508 04:21:56.375 | INFO | Downloaded 34GiB to GPU in 2.50713s (13.5613GiB/s) | 20260508 04:22:01.515 | INFO | Analyzing chunk frames = [5199360,5199872) | 20260508 04:22:01.515 | INFO | MemInfo: { | 20260508 04:22:01.515 | INFO | "gpu" : " 17 GB", | 20260508 04:22:01.515 | INFO | "huge" : "", | 20260508 04:22:01.515 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:01.515 | INFO | "malloc" : " 14 GB", | 20260508 04:22:01.515 | INFO | "pinned" : " 29 GB" | 20260508 04:22:01.515 | INFO | } | 20260508 04:22:01.515 | INFO | Perf report for the last 5.14027 seconds: | 20260508 04:22:01.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:01.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.515 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 04:22:01.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.515 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:22:01.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:01.516 | INFO | Uploaded 36GiB to GPU in 2.54597s (14.14GiB/s) | 20260508 04:22:01.516 | INFO | Downloaded 34GiB to GPU in 2.50535s (13.571GiB/s) | 20260508 04:22:06.659 | INFO | Analyzing chunk frames = [5199872,5200384) | 20260508 04:22:06.659 | INFO | MemInfo: { | 20260508 04:22:06.659 | INFO | "gpu" : " 17 GB", | 20260508 04:22:06.659 | INFO | "huge" : "", | 20260508 04:22:06.659 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:06.659 | INFO | "malloc" : " 14 GB", | 20260508 04:22:06.659 | INFO | "pinned" : " 29 GB" | 20260508 04:22:06.659 | INFO | } | 20260508 04:22:06.659 | INFO | Perf report for the last 5.14404 seconds: | 20260508 04:22:06.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:06.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:22:06.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:22:06.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:06.659 | INFO | Uploaded 36GiB to GPU in 2.36998s (15.19GiB/s) | 20260508 04:22:06.659 | INFO | Downloaded 34GiB to GPU in 2.55373s (13.3138GiB/s) | 20260508 04:22:11.790 | INFO | Analyzing chunk frames = [5200384,5200896) | 20260508 04:22:11.790 | INFO | MemInfo: { | 20260508 04:22:11.790 | INFO | "gpu" : " 17 GB", | 20260508 04:22:11.790 | INFO | "huge" : "", | 20260508 04:22:11.790 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:11.790 | INFO | "malloc" : " 14 GB", | 20260508 04:22:11.790 | INFO | "pinned" : " 29 GB" | 20260508 04:22:11.790 | INFO | } | 20260508 04:22:11.791 | INFO | Perf report for the last 5.1315 seconds: | 20260508 04:22:11.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:11.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:22:11.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:22:11.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:11.791 | INFO | Uploaded 36GiB to GPU in 2.43761s (14.7686GiB/s) | 20260508 04:22:11.791 | INFO | Downloaded 34GiB to GPU in 2.41253s (14.0931GiB/s) | 20260508 04:22:16.931 | INFO | Analyzing chunk frames = [5200896,5201408) | 20260508 04:22:16.931 | INFO | MemInfo: { | 20260508 04:22:16.932 | INFO | "gpu" : " 17 GB", | 20260508 04:22:16.932 | INFO | "huge" : "", | 20260508 04:22:16.932 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:16.932 | INFO | "malloc" : " 14 GB", | 20260508 04:22:16.932 | INFO | "pinned" : " 29 GB" | 20260508 04:22:16.932 | INFO | } | 20260508 04:22:16.932 | INFO | Perf report for the last 5.14131 seconds: | 20260508 04:22:16.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:16.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:22:16.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 04:22:16.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:16.932 | INFO | Uploaded 36GiB to GPU in 2.42793s (14.8274GiB/s) | 20260508 04:22:16.932 | INFO | Downloaded 34GiB to GPU in 2.42039s (14.0473GiB/s) | 20260508 04:22:22.069 | INFO | Analyzing chunk frames = [5201408,5201920) | 20260508 04:22:22.069 | INFO | MemInfo: { | 20260508 04:22:22.069 | INFO | "gpu" : " 17 GB", | 20260508 04:22:22.069 | INFO | "huge" : "", | 20260508 04:22:22.070 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:22.070 | INFO | "malloc" : " 14 GB", | 20260508 04:22:22.070 | INFO | "pinned" : " 29 GB" | 20260508 04:22:22.070 | INFO | } | 20260508 04:22:22.070 | INFO | Perf report for the last 5.13783 seconds: | 20260508 04:22:22.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:22.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:22:22.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260508 04:22:22.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:22.070 | INFO | Uploaded 36GiB to GPU in 2.4556s (14.6604GiB/s) | 20260508 04:22:22.070 | INFO | Downloaded 34GiB to GPU in 2.43272s (13.9761GiB/s) | 20260508 04:22:22.070 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:22:22.070 | INFO | Total mem usage: 16.7 GiB | 20260508 04:22:22.070 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:22:22.070 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.3%) | 20260508 04:22:22.070 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:22:22.070 | INFO | High Water Breakdown: | 20260508 04:22:22.070 | INFO | 60.3% : 9.04 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:22:22.070 | INFO | 39.6% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:22:22.070 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:22:22.070 | INFO | ------------------------------------------ | 20260508 04:22:22.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:22:22.070 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:22:22.070 | INFO | Total mem usage: 26.5 GiB | 20260508 04:22:22.070 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:22:22.070 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:22:22.070 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:22:22.070 | INFO | High Water Breakdown: | 20260508 04:22:22.070 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:22:22.070 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:22:22.071 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:22:22.071 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:22:22.071 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:22:22.071 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:22:22.071 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:22:22.071 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:22:22.071 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:22:22.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:22:22.071 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:22:22.071 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:22:22.071 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:22:22.071 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:22:22.071 | INFO | ------------------------------------------ | 20260508 04:22:22.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:22:22.071 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:22:22.071 | INFO | Total mem usage: 27.8 GiB | 20260508 04:22:22.071 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:22:22.071 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:22:22.071 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:22:22.071 | INFO | High Water Breakdown: | 20260508 04:22:22.071 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:22:22.071 | INFO | ------------------------------------------ | 20260508 04:22:22.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:22:22.071 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:22:22.071 | INFO | Total mem usage: 15.7 GiB | 20260508 04:22:22.071 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:22:22.071 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:22:22.071 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:22:22.071 | INFO | High Water Breakdown: | 20260508 04:22:22.071 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:22:22.071 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:22:22.071 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:22:22.071 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:22:22.071 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:22:22.071 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:22:22.071 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:22:22.071 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:22:22.071 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:22:22.071 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:22:22.071 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:22:22.071 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:22:22.071 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:22:22.071 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:22:22.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:22:22.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:22:22.071 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:22:22.071 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:22:22.071 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:22:22.071 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:22:22.071 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:22:22.071 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:22:22.071 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:22:22.071 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:22:22.071 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:22:22.071 | INFO | ------------------------------------------ | 20260508 04:22:22.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:22:22.071 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:22:22.071 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:22:22.071 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:22:22.071 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.41e+01 / 1.30e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.03e-01, 1.17e-02 / 5.08e-03 | 20260508 04:22:22.071 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.48e+00, 5.95e-01 / 2.08e-01 | 20260508 04:22:22.071 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.15e-01 | 20260508 04:22:22.071 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.59e+01, 4.19e+00 / 1.77e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.85e+01 / 1.97e+01, 1.71e+00 / 1.41e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.69e+00, 4.19e-01 / 2.44e-01 | 20260508 04:22:22.071 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.19e+00, 2.60e-01 / 1.18e-01 | 20260508 04:22:22.071 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.57e-01, 5.07e-02 / 2.52e-02 | 20260508 04:22:22.071 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:22:22.071 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:22:22.071 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:22:22.071 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 24.44%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 24.44%, Download -- 2.60e+01 / 0.00e+00, 8.37e+00 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 2.13%, LockUpload -- 2.26e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 2.78%, LockDownload -- 2.95e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 15.40%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 04:22:22.071 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260508 04:22:22.167 | INFO | Starting a round of Baz IO | 20260508 04:22:25.601 | INFO | Populating RTMetrics report | 20260508 04:22:25.601 | INFO | RTMetrics report populated | 20260508 04:22:25.601 | INFO | Opening tmp RTMetrics json file | 20260508 04:22:25.602 | INFO | Serializing RTMetrics to json | 20260508 04:22:25.603 | INFO | Serialization done, swapping tmp file now... | 20260508 04:22:25.604 | INFO | RTMetrics done | 20260508 04:22:25.702 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.534 seconds | 20260508 04:22:25.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.08% / 0.00% / 58.92% | 20260508 04:22:25.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.539708% / 0% / 6.35783e-05% | 20260508 04:22:25.702 | INFO | Baz IO overall: | 20260508 04:22:25.702 | INFO | BytesWritten : 2329668763696 | 20260508 04:22:25.702 | INFO | NumEvents : 1124828129063 | 20260508 04:22:27.212 | INFO | Analyzing chunk frames = [5201920,5202432) | 20260508 04:22:27.212 | INFO | MemInfo: { | 20260508 04:22:27.212 | INFO | "gpu" : " 17 GB", | 20260508 04:22:27.213 | INFO | "huge" : "", | 20260508 04:22:27.213 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:27.213 | INFO | "malloc" : " 14 GB", | 20260508 04:22:27.213 | INFO | "pinned" : " 29 GB" | 20260508 04:22:27.213 | INFO | } | 20260508 04:22:27.213 | INFO | Perf report for the last 5.14288 seconds: | 20260508 04:22:27.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:27.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:27.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:27.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:27.213 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 04:22:27.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:27.213 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 04:22:27.213 | INFO | PRE_HQ : 65.6%, 1.6, 55.4ms, 0 | 20260508 04:22:27.213 | INFO | BAZWRITER : 68.2%, 3.7, 128.3ms, 0 | 20260508 04:22:27.213 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260508 04:22:27.213 | INFO | Uploaded 36GiB to GPU in 2.527s (14.2462GiB/s) | 20260508 04:22:27.213 | INFO | Downloaded 38GiB to GPU in 2.70434s (14.0515GiB/s) | 20260508 04:22:32.354 | INFO | Analyzing chunk frames = [5202432,5202944) | 20260508 04:22:32.354 | INFO | MemInfo: { | 20260508 04:22:32.354 | INFO | "gpu" : " 17 GB", | 20260508 04:22:32.354 | INFO | "huge" : "", | 20260508 04:22:32.354 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:32.354 | INFO | "malloc" : " 14 GB", | 20260508 04:22:32.354 | INFO | "pinned" : " 29 GB" | 20260508 04:22:32.354 | INFO | } | 20260508 04:22:32.354 | INFO | Perf report for the last 5.1411 seconds: | 20260508 04:22:32.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:32.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:22:32.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:22:32.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:32.354 | INFO | Uploaded 36GiB to GPU in 2.57489s (13.9812GiB/s) | 20260508 04:22:32.354 | INFO | Downloaded 34GiB to GPU in 2.43126s (13.9845GiB/s) | 20260508 04:22:37.490 | INFO | Analyzing chunk frames = [5202944,5203456) | 20260508 04:22:37.490 | INFO | MemInfo: { | 20260508 04:22:37.490 | INFO | "gpu" : " 17 GB", | 20260508 04:22:37.490 | INFO | "huge" : "", | 20260508 04:22:37.490 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:37.490 | INFO | "malloc" : " 14 GB", | 20260508 04:22:37.490 | INFO | "pinned" : " 29 GB" | 20260508 04:22:37.490 | INFO | } | 20260508 04:22:37.490 | INFO | Perf report for the last 5.13639 seconds: | 20260508 04:22:37.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:37.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:22:37.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:22:37.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:37.490 | INFO | Uploaded 36GiB to GPU in 2.54865s (14.1251GiB/s) | 20260508 04:22:37.490 | INFO | Downloaded 34GiB to GPU in 2.41597s (14.073GiB/s) | 20260508 04:22:42.629 | INFO | Analyzing chunk frames = [5203456,5203968) | 20260508 04:22:42.629 | INFO | MemInfo: { | 20260508 04:22:42.629 | INFO | "gpu" : " 17 GB", | 20260508 04:22:42.629 | INFO | "huge" : "", | 20260508 04:22:42.629 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:42.629 | INFO | "malloc" : " 14 GB", | 20260508 04:22:42.629 | INFO | "pinned" : " 29 GB" | 20260508 04:22:42.629 | INFO | } | 20260508 04:22:42.629 | INFO | Perf report for the last 5.13929 seconds: | 20260508 04:22:42.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:42.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 04:22:42.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:22:42.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:42.630 | INFO | Uploaded 36GiB to GPU in 2.57182s (13.9979GiB/s) | 20260508 04:22:42.630 | INFO | Downloaded 34GiB to GPU in 2.42719s (14.008GiB/s) | 20260508 04:22:47.768 | INFO | Analyzing chunk frames = [5203968,5204480) | 20260508 04:22:47.768 | INFO | MemInfo: { | 20260508 04:22:47.768 | INFO | "gpu" : " 17 GB", | 20260508 04:22:47.768 | INFO | "huge" : "", | 20260508 04:22:47.768 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:47.768 | INFO | "malloc" : " 14 GB", | 20260508 04:22:47.768 | INFO | "pinned" : " 29 GB" | 20260508 04:22:47.768 | INFO | } | 20260508 04:22:47.768 | INFO | Perf report for the last 5.13892 seconds: | 20260508 04:22:47.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:47.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.768 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 04:22:47.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.768 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:22:47.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:47.769 | INFO | Uploaded 36GiB to GPU in 2.41985s (14.877GiB/s) | 20260508 04:22:47.769 | INFO | Downloaded 34GiB to GPU in 2.55404s (13.3122GiB/s) | 20260508 04:22:52.910 | INFO | Analyzing chunk frames = [5204480,5204992) | 20260508 04:22:52.910 | INFO | MemInfo: { | 20260508 04:22:52.910 | INFO | "gpu" : " 17 GB", | 20260508 04:22:52.910 | INFO | "huge" : "", | 20260508 04:22:52.910 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:52.910 | INFO | "malloc" : " 14 GB", | 20260508 04:22:52.910 | INFO | "pinned" : " 29 GB" | 20260508 04:22:52.910 | INFO | } | 20260508 04:22:52.910 | INFO | Perf report for the last 5.14155 seconds: | 20260508 04:22:52.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:52.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:22:52.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:22:52.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:52.910 | INFO | Uploaded 36GiB to GPU in 2.42166s (14.8658GiB/s) | 20260508 04:22:52.910 | INFO | Downloaded 34GiB to GPU in 2.41237s (14.094GiB/s) | 20260508 04:22:58.047 | INFO | Analyzing chunk frames = [5204992,5205504) | 20260508 04:22:58.047 | INFO | MemInfo: { | 20260508 04:22:58.047 | INFO | "gpu" : " 17 GB", | 20260508 04:22:58.047 | INFO | "huge" : "", | 20260508 04:22:58.047 | INFO | "huge_shared" : " 29 GB", | 20260508 04:22:58.047 | INFO | "malloc" : " 14 GB", | 20260508 04:22:58.047 | INFO | "pinned" : " 29 GB" | 20260508 04:22:58.047 | INFO | } | 20260508 04:22:58.047 | INFO | Perf report for the last 5.13757 seconds: | 20260508 04:22:58.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:22:58.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:22:58.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 04:22:58.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:22:58.048 | INFO | Uploaded 36GiB to GPU in 2.41845s (14.8856GiB/s) | 20260508 04:22:58.048 | INFO | Downloaded 34GiB to GPU in 2.4192s (14.0542GiB/s) | 20260508 04:23:03.187 | INFO | Analyzing chunk frames = [5205504,5206016) | 20260508 04:23:03.187 | INFO | MemInfo: { | 20260508 04:23:03.187 | INFO | "gpu" : " 17 GB", | 20260508 04:23:03.187 | INFO | "huge" : "", | 20260508 04:23:03.187 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:03.187 | INFO | "malloc" : " 14 GB", | 20260508 04:23:03.187 | INFO | "pinned" : " 29 GB" | 20260508 04:23:03.187 | INFO | } | 20260508 04:23:03.187 | INFO | Perf report for the last 5.13934 seconds: | 20260508 04:23:03.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:03.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.187 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:23:03.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.187 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:23:03.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:03.188 | INFO | Uploaded 36GiB to GPU in 2.44896s (14.7001GiB/s) | 20260508 04:23:03.188 | INFO | Downloaded 34GiB to GPU in 2.42037s (14.0474GiB/s) | 20260508 04:23:03.285 | INFO | Starting a round of Baz IO | 20260508 04:23:06.724 | INFO | Populating RTMetrics report | 20260508 04:23:06.724 | INFO | RTMetrics report populated | 20260508 04:23:06.724 | INFO | Opening tmp RTMetrics json file | 20260508 04:23:06.744 | INFO | Serializing RTMetrics to json | 20260508 04:23:06.745 | INFO | Serialization done, swapping tmp file now... | 20260508 04:23:06.745 | INFO | RTMetrics done | 20260508 04:23:06.818 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.532 seconds | 20260508 04:23:06.818 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.08% / 0.00% / 58.92% | 20260508 04:23:06.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.54543% / 0% / 4.76837e-05% | 20260508 04:23:06.818 | INFO | Baz IO overall: | 20260508 04:23:06.818 | INFO | BytesWritten : 2331396704591 | 20260508 04:23:06.818 | INFO | NumEvents : 1125651430226 | 20260508 04:23:08.327 | INFO | Analyzing chunk frames = [5206016,5206528) | 20260508 04:23:08.327 | INFO | MemInfo: { | 20260508 04:23:08.327 | INFO | "gpu" : " 17 GB", | 20260508 04:23:08.327 | INFO | "huge" : "", | 20260508 04:23:08.327 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:08.327 | INFO | "malloc" : " 14 GB", | 20260508 04:23:08.327 | INFO | "pinned" : " 29 GB" | 20260508 04:23:08.327 | INFO | } | 20260508 04:23:08.327 | INFO | Perf report for the last 5.14013 seconds: | 20260508 04:23:08.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:08.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:08.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:08.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:08.327 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 04:23:08.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:08.327 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:23:08.327 | INFO | PRE_HQ : 65.6%, 1.6, 55.9ms, 0 | 20260508 04:23:08.327 | INFO | BAZWRITER : 68.2%, 3.6, 127.2ms, 0 | 20260508 04:23:08.327 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 04:23:08.327 | INFO | Uploaded 36GiB to GPU in 2.57009s (14.0073GiB/s) | 20260508 04:23:08.327 | INFO | Downloaded 38GiB to GPU in 2.69354s (14.1078GiB/s) | 20260508 04:23:13.471 | INFO | Analyzing chunk frames = [5206528,5207040) | 20260508 04:23:13.471 | INFO | MemInfo: { | 20260508 04:23:13.471 | INFO | "gpu" : " 17 GB", | 20260508 04:23:13.471 | INFO | "huge" : "", | 20260508 04:23:13.471 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:13.471 | INFO | "malloc" : " 14 GB", | 20260508 04:23:13.471 | INFO | "pinned" : " 29 GB" | 20260508 04:23:13.471 | INFO | } | 20260508 04:23:13.471 | INFO | Perf report for the last 5.14417 seconds: | 20260508 04:23:13.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:13.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 04:23:13.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:23:13.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:13.471 | INFO | Uploaded 36GiB to GPU in 2.55629s (14.0829GiB/s) | 20260508 04:23:13.471 | INFO | Downloaded 34GiB to GPU in 2.43046s (13.9891GiB/s) | 20260508 04:23:18.608 | INFO | Analyzing chunk frames = [5207040,5207552) | 20260508 04:23:18.608 | INFO | MemInfo: { | 20260508 04:23:18.608 | INFO | "gpu" : " 17 GB", | 20260508 04:23:18.608 | INFO | "huge" : "", | 20260508 04:23:18.608 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:18.608 | INFO | "malloc" : " 14 GB", | 20260508 04:23:18.608 | INFO | "pinned" : " 29 GB" | 20260508 04:23:18.608 | INFO | } | 20260508 04:23:18.609 | INFO | Perf report for the last 5.13738 seconds: | 20260508 04:23:18.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:18.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 04:23:18.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:23:18.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:18.609 | INFO | Uploaded 36GiB to GPU in 2.54338s (14.1544GiB/s) | 20260508 04:23:18.609 | INFO | Downloaded 34GiB to GPU in 2.40244s (14.1523GiB/s) | 20260508 04:23:23.742 | INFO | Analyzing chunk frames = [5207552,5208064) | 20260508 04:23:23.742 | INFO | MemInfo: { | 20260508 04:23:23.742 | INFO | "gpu" : " 17 GB", | 20260508 04:23:23.742 | INFO | "huge" : "", | 20260508 04:23:23.742 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:23.742 | INFO | "malloc" : " 14 GB", | 20260508 04:23:23.742 | INFO | "pinned" : " 29 GB" | 20260508 04:23:23.742 | INFO | } | 20260508 04:23:23.742 | INFO | Perf report for the last 5.13364 seconds: | 20260508 04:23:23.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:23.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.742 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 04:23:23.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.742 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:23:23.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:23.743 | INFO | Uploaded 36GiB to GPU in 2.57974s (13.9549GiB/s) | 20260508 04:23:23.743 | INFO | Downloaded 34GiB to GPU in 2.41397s (14.0847GiB/s) | 20260508 04:23:28.886 | INFO | Analyzing chunk frames = [5208064,5208576) | 20260508 04:23:28.886 | INFO | MemInfo: { | 20260508 04:23:28.886 | INFO | "gpu" : " 17 GB", | 20260508 04:23:28.886 | INFO | "huge" : "", | 20260508 04:23:28.886 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:28.886 | INFO | "malloc" : " 14 GB", | 20260508 04:23:28.886 | INFO | "pinned" : " 29 GB" | 20260508 04:23:28.886 | INFO | } | 20260508 04:23:28.886 | INFO | Perf report for the last 5.14377 seconds: | 20260508 04:23:28.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:28.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 04:23:28.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:23:28.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:28.886 | INFO | Uploaded 36GiB to GPU in 2.42958s (14.8174GiB/s) | 20260508 04:23:28.886 | INFO | Downloaded 34GiB to GPU in 2.56269s (13.2673GiB/s) | 20260508 04:23:34.022 | INFO | Analyzing chunk frames = [5208576,5209088) | 20260508 04:23:34.022 | INFO | MemInfo: { | 20260508 04:23:34.022 | INFO | "gpu" : " 17 GB", | 20260508 04:23:34.022 | INFO | "huge" : "", | 20260508 04:23:34.022 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:34.022 | INFO | "malloc" : " 14 GB", | 20260508 04:23:34.022 | INFO | "pinned" : " 29 GB" | 20260508 04:23:34.022 | INFO | } | 20260508 04:23:34.022 | INFO | Perf report for the last 5.1365 seconds: | 20260508 04:23:34.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:34.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.022 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:23:34.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.023 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:23:34.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:34.023 | INFO | Uploaded 36GiB to GPU in 2.42235s (14.8616GiB/s) | 20260508 04:23:34.023 | INFO | Downloaded 34GiB to GPU in 2.42325s (14.0307GiB/s) | 20260508 04:23:39.162 | INFO | Analyzing chunk frames = [5209088,5209600) | 20260508 04:23:39.162 | INFO | MemInfo: { | 20260508 04:23:39.162 | INFO | "gpu" : " 17 GB", | 20260508 04:23:39.162 | INFO | "huge" : "", | 20260508 04:23:39.162 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:39.162 | INFO | "malloc" : " 14 GB", | 20260508 04:23:39.162 | INFO | "pinned" : " 29 GB" | 20260508 04:23:39.162 | INFO | } | 20260508 04:23:39.162 | INFO | Perf report for the last 5.13982 seconds: | 20260508 04:23:39.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:39.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.162 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:23:39.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.162 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:23:39.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:39.163 | INFO | Uploaded 36GiB to GPU in 2.42666s (14.8352GiB/s) | 20260508 04:23:39.163 | INFO | Downloaded 34GiB to GPU in 2.42212s (14.0373GiB/s) | 20260508 04:23:44.302 | INFO | Analyzing chunk frames = [5209600,5210112) | 20260508 04:23:44.302 | INFO | MemInfo: { | 20260508 04:23:44.302 | INFO | "gpu" : " 17 GB", | 20260508 04:23:44.302 | INFO | "huge" : "", | 20260508 04:23:44.302 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:44.302 | INFO | "malloc" : " 14 GB", | 20260508 04:23:44.302 | INFO | "pinned" : " 29 GB" | 20260508 04:23:44.302 | INFO | } | 20260508 04:23:44.302 | INFO | Perf report for the last 5.1395 seconds: | 20260508 04:23:44.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:44.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:23:44.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:23:44.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:44.302 | INFO | Uploaded 36GiB to GPU in 2.45242s (14.6794GiB/s) | 20260508 04:23:44.302 | INFO | Downloaded 34GiB to GPU in 2.41965s (14.0516GiB/s) | 20260508 04:23:44.302 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:23:44.302 | INFO | Total mem usage: 16.7 GiB | 20260508 04:23:44.302 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:23:44.302 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -8.8%) | 20260508 04:23:44.302 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:23:44.302 | INFO | High Water Breakdown: | 20260508 04:23:44.302 | INFO | 60.6% : 9.3 GiB ( 90.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:23:44.302 | INFO | 39.2% : 6.02 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:23:44.302 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:23:44.302 | INFO | ------------------------------------------ | 20260508 04:23:44.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:23:44.302 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:23:44.302 | INFO | Total mem usage: 26.5 GiB | 20260508 04:23:44.302 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:23:44.302 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:23:44.302 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:23:44.302 | INFO | High Water Breakdown: | 20260508 04:23:44.302 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:23:44.303 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:23:44.303 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:23:44.303 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:23:44.303 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:23:44.303 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:23:44.303 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:23:44.303 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:23:44.303 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:23:44.303 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:23:44.303 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:23:44.303 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:23:44.303 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:23:44.303 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:23:44.303 | INFO | ------------------------------------------ | 20260508 04:23:44.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:23:44.303 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:23:44.303 | INFO | Total mem usage: 27.8 GiB | 20260508 04:23:44.303 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:23:44.303 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:23:44.303 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:23:44.303 | INFO | High Water Breakdown: | 20260508 04:23:44.303 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:23:44.303 | INFO | ------------------------------------------ | 20260508 04:23:44.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:23:44.303 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:23:44.303 | INFO | Total mem usage: 15.7 GiB | 20260508 04:23:44.303 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:23:44.303 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:23:44.303 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:23:44.303 | INFO | High Water Breakdown: | 20260508 04:23:44.303 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:23:44.303 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:23:44.303 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:23:44.303 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:23:44.303 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:23:44.303 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:23:44.303 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:23:44.303 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:23:44.303 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:23:44.303 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:23:44.303 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:23:44.303 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:23:44.303 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:23:44.303 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:23:44.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:23:44.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:23:44.303 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:23:44.303 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:23:44.303 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:23:44.303 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:23:44.303 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:23:44.303 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:23:44.303 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:23:44.303 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:23:44.303 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:23:44.303 | INFO | ------------------------------------------ | 20260508 04:23:44.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:23:44.303 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:23:44.303 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:23:44.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:23:44.303 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.40e+01 / 1.45e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.12e-02 / 5.89e-03 | 20260508 04:23:44.303 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.43e+00, 5.93e-01 / 2.48e-01 | 20260508 04:23:44.303 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.11e-01 / 1.24e-01 | 20260508 04:23:44.303 | INFO | PerfReport: 15.88%, DME -- 4.17e+01 / 4.52e+01, 3.73e+00 / 1.78e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.85e+00 / 1.58e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.58e+00, 4.17e-01 / 2.59e-01 | 20260508 04:23:44.303 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.22e+00, 2.62e-01 / 1.70e-01 | 20260508 04:23:44.303 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.23e-01 / 3.58e-01, 5.09e-02 / 2.32e-02 | 20260508 04:23:44.303 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:23:44.303 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:23:44.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:23:44.303 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 24.43%, Upload -- 2.60e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 24.21%, Download -- 2.58e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 30.86%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 2.19%, LockUpload -- 2.33e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 15.68%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:23:44.303 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260508 04:23:44.399 | INFO | Starting a round of Baz IO | 20260508 04:23:47.837 | INFO | Populating RTMetrics report | 20260508 04:23:47.838 | INFO | RTMetrics report populated | 20260508 04:23:47.838 | INFO | Opening tmp RTMetrics json file | 20260508 04:23:47.853 | INFO | Serializing RTMetrics to json | 20260508 04:23:47.854 | INFO | Serialization done, swapping tmp file now... | 20260508 04:23:47.855 | INFO | RTMetrics done | 20260508 04:23:47.940 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.541 seconds | 20260508 04:23:47.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.07% / 0.00% / 58.92% | 20260508 04:23:47.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.547882% / 0% / 5.16574e-05% | 20260508 04:23:47.941 | INFO | Baz IO overall: | 20260508 04:23:47.941 | INFO | BytesWritten : 2333126735576 | 20260508 04:23:47.941 | INFO | NumEvents : 1126475452640 | 20260508 04:23:49.442 | INFO | Analyzing chunk frames = [5210112,5210624) | 20260508 04:23:49.442 | INFO | MemInfo: { | 20260508 04:23:49.442 | INFO | "gpu" : " 17 GB", | 20260508 04:23:49.442 | INFO | "huge" : "", | 20260508 04:23:49.442 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:49.442 | INFO | "malloc" : " 14 GB", | 20260508 04:23:49.442 | INFO | "pinned" : " 29 GB" | 20260508 04:23:49.442 | INFO | } | 20260508 04:23:49.442 | INFO | Perf report for the last 5.14011 seconds: | 20260508 04:23:49.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:49.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:49.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:49.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:49.442 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 04:23:49.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:49.442 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:23:49.442 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 04:23:49.442 | INFO | BAZWRITER : 68.7%, 3.2, 113.9ms, 0 | 20260508 04:23:49.442 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 04:23:49.443 | INFO | Uploaded 36GiB to GPU in 2.56529s (14.0335GiB/s) | 20260508 04:23:49.443 | INFO | Downloaded 38GiB to GPU in 2.68817s (14.136GiB/s) | 20260508 04:23:54.582 | INFO | Analyzing chunk frames = [5210624,5211136) | 20260508 04:23:54.582 | INFO | MemInfo: { | 20260508 04:23:54.582 | INFO | "gpu" : " 17 GB", | 20260508 04:23:54.582 | INFO | "huge" : "", | 20260508 04:23:54.582 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:54.582 | INFO | "malloc" : " 14 GB", | 20260508 04:23:54.582 | INFO | "pinned" : " 29 GB" | 20260508 04:23:54.582 | INFO | } | 20260508 04:23:54.582 | INFO | Perf report for the last 5.14046 seconds: | 20260508 04:23:54.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:54.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.583 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:23:54.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.583 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:23:54.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:54.583 | INFO | Uploaded 36GiB to GPU in 2.57499s (13.9807GiB/s) | 20260508 04:23:54.583 | INFO | Downloaded 34GiB to GPU in 2.42589s (14.0155GiB/s) | 20260508 04:23:59.722 | INFO | Analyzing chunk frames = [5211136,5211648) | 20260508 04:23:59.722 | INFO | MemInfo: { | 20260508 04:23:59.722 | INFO | "gpu" : " 17 GB", | 20260508 04:23:59.722 | INFO | "huge" : "", | 20260508 04:23:59.722 | INFO | "huge_shared" : " 29 GB", | 20260508 04:23:59.722 | INFO | "malloc" : " 14 GB", | 20260508 04:23:59.722 | INFO | "pinned" : " 29 GB" | 20260508 04:23:59.722 | INFO | } | 20260508 04:23:59.722 | INFO | Perf report for the last 5.13955 seconds: | 20260508 04:23:59.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:23:59.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260508 04:23:59.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:23:59.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:23:59.722 | INFO | Uploaded 36GiB to GPU in 2.54125s (14.1663GiB/s) | 20260508 04:23:59.722 | INFO | Downloaded 34GiB to GPU in 2.42933s (13.9956GiB/s) | 20260508 04:24:04.859 | INFO | Analyzing chunk frames = [5211648,5212160) | 20260508 04:24:04.859 | INFO | MemInfo: { | 20260508 04:24:04.859 | INFO | "gpu" : " 17 GB", | 20260508 04:24:04.859 | INFO | "huge" : "", | 20260508 04:24:04.859 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:04.859 | INFO | "malloc" : " 14 GB", | 20260508 04:24:04.859 | INFO | "pinned" : " 29 GB" | 20260508 04:24:04.859 | INFO | } | 20260508 04:24:04.859 | INFO | Perf report for the last 5.1369 seconds: | 20260508 04:24:04.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:04.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | ANALYSIS : 67.3%, 3.0, 104.1ms, 0 | 20260508 04:24:04.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:24:04.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:04.859 | INFO | Uploaded 36GiB to GPU in 2.55189s (14.1072GiB/s) | 20260508 04:24:04.859 | INFO | Downloaded 34GiB to GPU in 2.50187s (13.5898GiB/s) | 20260508 04:24:09.995 | INFO | Analyzing chunk frames = [5212160,5212672) | 20260508 04:24:09.995 | INFO | MemInfo: { | 20260508 04:24:09.995 | INFO | "gpu" : " 17 GB", | 20260508 04:24:09.995 | INFO | "huge" : "", | 20260508 04:24:09.995 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:09.995 | INFO | "malloc" : " 14 GB", | 20260508 04:24:09.995 | INFO | "pinned" : " 29 GB" | 20260508 04:24:09.995 | INFO | } | 20260508 04:24:09.995 | INFO | Perf report for the last 5.13617 seconds: | 20260508 04:24:09.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:09.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:24:09.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:24:09.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:09.995 | INFO | Uploaded 36GiB to GPU in 2.36308s (15.2343GiB/s) | 20260508 04:24:09.996 | INFO | Downloaded 34GiB to GPU in 2.57023s (13.2284GiB/s) | 20260508 04:24:15.140 | INFO | Analyzing chunk frames = [5212672,5213184) | 20260508 04:24:15.140 | INFO | MemInfo: { | 20260508 04:24:15.140 | INFO | "gpu" : " 17 GB", | 20260508 04:24:15.140 | INFO | "huge" : "", | 20260508 04:24:15.140 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:15.140 | INFO | "malloc" : " 14 GB", | 20260508 04:24:15.140 | INFO | "pinned" : " 29 GB" | 20260508 04:24:15.140 | INFO | } | 20260508 04:24:15.140 | INFO | Perf report for the last 5.14502 seconds: | 20260508 04:24:15.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:15.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:24:15.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:24:15.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:15.140 | INFO | Uploaded 36GiB to GPU in 2.44724s (14.7104GiB/s) | 20260508 04:24:15.140 | INFO | Downloaded 34GiB to GPU in 2.40472s (14.1389GiB/s) | 20260508 04:24:20.279 | INFO | Analyzing chunk frames = [5213184,5213696) | 20260508 04:24:20.279 | INFO | MemInfo: { | 20260508 04:24:20.279 | INFO | "gpu" : " 17 GB", | 20260508 04:24:20.279 | INFO | "huge" : "", | 20260508 04:24:20.279 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:20.279 | INFO | "malloc" : " 14 GB", | 20260508 04:24:20.279 | INFO | "pinned" : " 29 GB" | 20260508 04:24:20.279 | INFO | } | 20260508 04:24:20.279 | INFO | Perf report for the last 5.13869 seconds: | 20260508 04:24:20.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:20.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:24:20.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:24:20.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:20.279 | INFO | Uploaded 36GiB to GPU in 2.45312s (14.6752GiB/s) | 20260508 04:24:20.279 | INFO | Downloaded 34GiB to GPU in 2.40362s (14.1454GiB/s) | 20260508 04:24:25.423 | INFO | Analyzing chunk frames = [5213696,5214208) | 20260508 04:24:25.423 | INFO | MemInfo: { | 20260508 04:24:25.423 | INFO | "gpu" : " 17 GB", | 20260508 04:24:25.423 | INFO | "huge" : "", | 20260508 04:24:25.423 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:25.423 | INFO | "malloc" : " 14 GB", | 20260508 04:24:25.423 | INFO | "pinned" : " 29 GB" | 20260508 04:24:25.423 | INFO | } | 20260508 04:24:25.423 | INFO | Perf report for the last 5.14413 seconds: | 20260508 04:24:25.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:25.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.423 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:24:25.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.424 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:24:25.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:25.424 | INFO | Uploaded 36GiB to GPU in 2.48583s (14.4821GiB/s) | 20260508 04:24:25.424 | INFO | Downloaded 34GiB to GPU in 2.4161s (14.0723GiB/s) | 20260508 04:24:25.521 | INFO | Starting a round of Baz IO | 20260508 04:24:28.962 | INFO | Populating RTMetrics report | 20260508 04:24:28.963 | INFO | RTMetrics report populated | 20260508 04:24:28.963 | INFO | Opening tmp RTMetrics json file | 20260508 04:24:28.963 | INFO | Serializing RTMetrics to json | 20260508 04:24:28.964 | INFO | Serialization done, swapping tmp file now... | 20260508 04:24:28.964 | INFO | RTMetrics done | 20260508 04:24:29.041 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.52 seconds | 20260508 04:24:29.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.07% / 0.00% / 58.93% | 20260508 04:24:29.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.546575% / 3.97364e-06% / 7.94729e-05% | 20260508 04:24:29.041 | INFO | Baz IO overall: | 20260508 04:24:29.041 | INFO | BytesWritten : 2334850114484 | 20260508 04:24:29.042 | INFO | NumEvents : 1127297307085 | 20260508 04:24:30.559 | INFO | Analyzing chunk frames = [5214208,5214720) | 20260508 04:24:30.559 | INFO | MemInfo: { | 20260508 04:24:30.559 | INFO | "gpu" : " 17 GB", | 20260508 04:24:30.559 | INFO | "huge" : "", | 20260508 04:24:30.559 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:30.559 | INFO | "malloc" : " 14 GB", | 20260508 04:24:30.559 | INFO | "pinned" : " 29 GB" | 20260508 04:24:30.559 | INFO | } | 20260508 04:24:30.559 | INFO | Perf report for the last 5.13625 seconds: | 20260508 04:24:30.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:30.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:30.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:30.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:30.559 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 04:24:30.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:30.559 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 04:24:30.559 | INFO | PRE_HQ : 66.1%, 1.6, 55.8ms, 0 | 20260508 04:24:30.559 | INFO | BAZWRITER : 68.2%, 3.6, 125.5ms, 0 | 20260508 04:24:30.559 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:24:30.559 | INFO | Uploaded 36GiB to GPU in 2.62755s (13.701GiB/s) | 20260508 04:24:30.559 | INFO | Downloaded 38GiB to GPU in 2.69936s (14.0774GiB/s) | 20260508 04:24:35.694 | INFO | Analyzing chunk frames = [5214720,5215232) | 20260508 04:24:35.694 | INFO | MemInfo: { | 20260508 04:24:35.694 | INFO | "gpu" : " 17 GB", | 20260508 04:24:35.694 | INFO | "huge" : "", | 20260508 04:24:35.694 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:35.695 | INFO | "malloc" : " 14 GB", | 20260508 04:24:35.695 | INFO | "pinned" : " 29 GB" | 20260508 04:24:35.695 | INFO | } | 20260508 04:24:35.695 | INFO | Perf report for the last 5.13535 seconds: | 20260508 04:24:35.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:35.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 04:24:35.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 04:24:35.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:35.695 | INFO | Uploaded 36GiB to GPU in 2.50789s (14.3547GiB/s) | 20260508 04:24:35.695 | INFO | Downloaded 34GiB to GPU in 2.42763s (14.0054GiB/s) | 20260508 04:24:40.838 | INFO | Analyzing chunk frames = [5215232,5215744) | 20260508 04:24:40.838 | INFO | MemInfo: { | 20260508 04:24:40.838 | INFO | "gpu" : " 17 GB", | 20260508 04:24:40.838 | INFO | "huge" : "", | 20260508 04:24:40.838 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:40.838 | INFO | "malloc" : " 14 GB", | 20260508 04:24:40.838 | INFO | "pinned" : " 29 GB" | 20260508 04:24:40.838 | INFO | } | 20260508 04:24:40.838 | INFO | Perf report for the last 5.14325 seconds: | 20260508 04:24:40.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:40.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 04:24:40.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:24:40.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:40.838 | INFO | Uploaded 36GiB to GPU in 2.58624s (13.9198GiB/s) | 20260508 04:24:40.838 | INFO | Downloaded 34GiB to GPU in 2.491s (13.6491GiB/s) | 20260508 04:24:45.973 | INFO | Analyzing chunk frames = [5215744,5216256) | 20260508 04:24:45.973 | INFO | MemInfo: { | 20260508 04:24:45.973 | INFO | "gpu" : " 17 GB", | 20260508 04:24:45.973 | INFO | "huge" : "", | 20260508 04:24:45.973 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:45.973 | INFO | "malloc" : " 14 GB", | 20260508 04:24:45.973 | INFO | "pinned" : " 29 GB" | 20260508 04:24:45.973 | INFO | } | 20260508 04:24:45.973 | INFO | Perf report for the last 5.13514 seconds: | 20260508 04:24:45.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:45.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | ANALYSIS : 67.3%, 2.9, 104.0ms, 0 | 20260508 04:24:45.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:24:45.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:45.973 | INFO | Uploaded 36GiB to GPU in 2.60202s (13.8354GiB/s) | 20260508 04:24:45.973 | INFO | Downloaded 34GiB to GPU in 2.41339s (14.0881GiB/s) | 20260508 04:24:51.114 | INFO | Analyzing chunk frames = [5216256,5216768) | 20260508 04:24:51.114 | INFO | MemInfo: { | 20260508 04:24:51.114 | INFO | "gpu" : " 17 GB", | 20260508 04:24:51.114 | INFO | "huge" : "", | 20260508 04:24:51.114 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:51.114 | INFO | "malloc" : " 14 GB", | 20260508 04:24:51.114 | INFO | "pinned" : " 29 GB" | 20260508 04:24:51.114 | INFO | } | 20260508 04:24:51.114 | INFO | Perf report for the last 5.14091 seconds: | 20260508 04:24:51.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:51.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 04:24:51.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:24:51.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:51.114 | INFO | Uploaded 36GiB to GPU in 2.4068s (14.9576GiB/s) | 20260508 04:24:51.115 | INFO | Downloaded 34GiB to GPU in 2.54302s (13.3699GiB/s) | 20260508 04:24:56.248 | INFO | Analyzing chunk frames = [5216768,5217280) | 20260508 04:24:56.248 | INFO | MemInfo: { | 20260508 04:24:56.248 | INFO | "gpu" : " 17 GB", | 20260508 04:24:56.248 | INFO | "huge" : "", | 20260508 04:24:56.248 | INFO | "huge_shared" : " 29 GB", | 20260508 04:24:56.248 | INFO | "malloc" : " 14 GB", | 20260508 04:24:56.248 | INFO | "pinned" : " 29 GB" | 20260508 04:24:56.248 | INFO | } | 20260508 04:24:56.248 | INFO | Perf report for the last 5.1343 seconds: | 20260508 04:24:56.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:24:56.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.248 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:24:56.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.248 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:24:56.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:24:56.249 | INFO | Uploaded 36GiB to GPU in 2.4292s (14.8197GiB/s) | 20260508 04:24:56.249 | INFO | Downloaded 34GiB to GPU in 2.41599s (14.0729GiB/s) | 20260508 04:25:01.393 | INFO | Analyzing chunk frames = [5217280,5217792) | 20260508 04:25:01.393 | INFO | MemInfo: { | 20260508 04:25:01.393 | INFO | "gpu" : " 17 GB", | 20260508 04:25:01.393 | INFO | "huge" : "", | 20260508 04:25:01.393 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:01.393 | INFO | "malloc" : " 14 GB", | 20260508 04:25:01.393 | INFO | "pinned" : " 29 GB" | 20260508 04:25:01.393 | INFO | } | 20260508 04:25:01.393 | INFO | Perf report for the last 5.14509 seconds: | 20260508 04:25:01.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:01.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.393 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:25:01.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.393 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:25:01.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:01.394 | INFO | Uploaded 36GiB to GPU in 2.44426s (14.7284GiB/s) | 20260508 04:25:01.394 | INFO | Downloaded 34GiB to GPU in 2.42288s (14.0329GiB/s) | 20260508 04:25:06.533 | INFO | Analyzing chunk frames = [5217792,5218304) | 20260508 04:25:06.533 | INFO | MemInfo: { | 20260508 04:25:06.533 | INFO | "gpu" : " 17 GB", | 20260508 04:25:06.533 | INFO | "huge" : "", | 20260508 04:25:06.534 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:06.534 | INFO | "malloc" : " 14 GB", | 20260508 04:25:06.534 | INFO | "pinned" : " 29 GB" | 20260508 04:25:06.534 | INFO | } | 20260508 04:25:06.534 | INFO | Perf report for the last 5.14028 seconds: | 20260508 04:25:06.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:06.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:25:06.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:25:06.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:06.534 | INFO | Uploaded 36GiB to GPU in 2.46895s (14.5811GiB/s) | 20260508 04:25:06.534 | INFO | Downloaded 34GiB to GPU in 2.42964s (13.9938GiB/s) | 20260508 04:25:06.535 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:25:06.535 | INFO | Total mem usage: 16.7 GiB | 20260508 04:25:06.535 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:25:06.535 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.2%) | 20260508 04:25:06.535 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:25:06.535 | INFO | High Water Breakdown: | 20260508 04:25:06.535 | INFO | 60.3% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:25:06.535 | INFO | 39.6% : 5.94 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:25:06.535 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:25:06.535 | INFO | ------------------------------------------ | 20260508 04:25:06.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:25:06.535 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:25:06.535 | INFO | Total mem usage: 26.5 GiB | 20260508 04:25:06.535 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:25:06.535 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 04:25:06.535 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 04:25:06.535 | INFO | High Water Breakdown: | 20260508 04:25:06.535 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:25:06.535 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:25:06.535 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:25:06.535 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:25:06.535 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:25:06.535 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:25:06.535 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:25:06.535 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:25:06.535 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:25:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:25:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:25:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:25:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:25:06.535 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:25:06.535 | INFO | ------------------------------------------ | 20260508 04:25:06.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:25:06.535 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:25:06.535 | INFO | Total mem usage: 27.8 GiB | 20260508 04:25:06.535 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:25:06.535 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:25:06.535 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:25:06.535 | INFO | High Water Breakdown: | 20260508 04:25:06.535 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:25:06.535 | INFO | ------------------------------------------ | 20260508 04:25:06.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:25:06.535 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:25:06.535 | INFO | Total mem usage: 15.7 GiB | 20260508 04:25:06.535 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:25:06.535 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:25:06.535 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:25:06.535 | INFO | High Water Breakdown: | 20260508 04:25:06.535 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:25:06.535 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:25:06.535 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:25:06.535 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:25:06.535 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:25:06.535 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:25:06.535 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:25:06.535 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:25:06.535 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:25:06.535 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:25:06.535 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:25:06.535 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:25:06.535 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:25:06.535 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:25:06.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:25:06.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:25:06.535 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:25:06.536 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:25:06.536 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:25:06.536 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:25:06.536 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:25:06.536 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:25:06.536 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:25:06.536 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:25:06.536 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:25:06.536 | INFO | ------------------------------------------ | 20260508 04:25:06.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:25:06.536 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:25:06.536 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:25:06.536 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:25:06.536 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.42e+01 / 1.91e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.21e-02 / 5.68e-03 | 20260508 04:25:06.536 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.48e+00, 5.86e-01 / 1.86e-01 | 20260508 04:25:06.536 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.19e-01 | 20260508 04:25:06.536 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.57e+01, 3.97e+00 / 2.32e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.75e+00 / 1.47e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.65e+00, 4.22e-01 / 2.48e-01 | 20260508 04:25:06.536 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.24e+00, 2.65e-01 / 1.54e-01 | 20260508 04:25:06.536 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.57e-01, 5.10e-02 / 2.49e-02 | 20260508 04:25:06.536 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:25:06.536 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:25:06.536 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:25:06.536 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 24.50%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 24.27%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 2.34%, LockUpload -- 2.49e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 2.57%, LockDownload -- 2.73e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 15.56%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 04:25:06.536 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 04:25:06.648 | INFO | Starting a round of Baz IO | 20260508 04:25:10.078 | INFO | Populating RTMetrics report | 20260508 04:25:10.078 | INFO | RTMetrics report populated | 20260508 04:25:10.078 | INFO | Opening tmp RTMetrics json file | 20260508 04:25:10.093 | INFO | Serializing RTMetrics to json | 20260508 04:25:10.094 | INFO | Serialization done, swapping tmp file now... | 20260508 04:25:10.095 | INFO | RTMetrics done | 20260508 04:25:10.181 | INFO | Finished a round of Baz IO: Wrote 1.604 GiB in 3.532 seconds | 20260508 04:25:10.181 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.07% / 0.00% / 58.93% | 20260508 04:25:10.181 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.545796% / 3.97364e-06% / 5.16574e-05% | 20260508 04:25:10.181 | INFO | Baz IO overall: | 20260508 04:25:10.181 | INFO | BytesWritten : 2336572322856 | 20260508 04:25:10.181 | INFO | NumEvents : 1128120108159 | 20260508 04:25:11.674 | INFO | Analyzing chunk frames = [5218304,5218816) | 20260508 04:25:11.674 | INFO | MemInfo: { | 20260508 04:25:11.674 | INFO | "gpu" : " 17 GB", | 20260508 04:25:11.674 | INFO | "huge" : "", | 20260508 04:25:11.674 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:11.674 | INFO | "malloc" : " 14 GB", | 20260508 04:25:11.674 | INFO | "pinned" : " 29 GB" | 20260508 04:25:11.674 | INFO | } | 20260508 04:25:11.674 | INFO | Perf report for the last 5.14073 seconds: | 20260508 04:25:11.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:11.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:11.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:11.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:11.675 | INFO | ANALYSIS : 68.3%, 2.9, 105.1ms, 0 | 20260508 04:25:11.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:11.675 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 04:25:11.675 | INFO | PRE_HQ : 67.1%, 1.8, 62.3ms, 0 | 20260508 04:25:11.675 | INFO | BAZWRITER : 68.5%, 3.7, 129.8ms, 0 | 20260508 04:25:11.675 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:25:11.675 | INFO | Uploaded 36GiB to GPU in 2.60049s (13.8436GiB/s) | 20260508 04:25:11.675 | INFO | Downloaded 38GiB to GPU in 2.70928s (14.0259GiB/s) | 20260508 04:25:16.810 | INFO | Analyzing chunk frames = [5218816,5219328) | 20260508 04:25:16.810 | INFO | MemInfo: { | 20260508 04:25:16.810 | INFO | "gpu" : " 17 GB", | 20260508 04:25:16.810 | INFO | "huge" : "", | 20260508 04:25:16.810 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:16.810 | INFO | "malloc" : " 14 GB", | 20260508 04:25:16.811 | INFO | "pinned" : " 29 GB" | 20260508 04:25:16.811 | INFO | } | 20260508 04:25:16.811 | INFO | Perf report for the last 5.13618 seconds: | 20260508 04:25:16.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:16.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:25:16.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:25:16.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:16.811 | INFO | Uploaded 36GiB to GPU in 2.57853s (13.9614GiB/s) | 20260508 04:25:16.811 | INFO | Downloaded 34GiB to GPU in 2.43251s (13.9774GiB/s) | 20260508 04:25:21.950 | INFO | Analyzing chunk frames = [5219328,5219840) | 20260508 04:25:21.950 | INFO | MemInfo: { | 20260508 04:25:21.950 | INFO | "gpu" : " 17 GB", | 20260508 04:25:21.950 | INFO | "huge" : "", | 20260508 04:25:21.950 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:21.950 | INFO | "malloc" : " 14 GB", | 20260508 04:25:21.950 | INFO | "pinned" : " 29 GB" | 20260508 04:25:21.950 | INFO | } | 20260508 04:25:21.950 | INFO | Perf report for the last 5.1395 seconds: | 20260508 04:25:21.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:21.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 04:25:21.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:25:21.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:21.950 | INFO | Uploaded 36GiB to GPU in 2.54636s (14.1378GiB/s) | 20260508 04:25:21.950 | INFO | Downloaded 34GiB to GPU in 2.41372s (14.0861GiB/s) | 20260508 04:25:27.089 | INFO | Analyzing chunk frames = [5219840,5220352) | 20260508 04:25:27.089 | INFO | MemInfo: { | 20260508 04:25:27.089 | INFO | "gpu" : " 17 GB", | 20260508 04:25:27.089 | INFO | "huge" : "", | 20260508 04:25:27.089 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:27.089 | INFO | "malloc" : " 14 GB", | 20260508 04:25:27.089 | INFO | "pinned" : " 29 GB" | 20260508 04:25:27.089 | INFO | } | 20260508 04:25:27.089 | INFO | Perf report for the last 5.13874 seconds: | 20260508 04:25:27.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:27.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:25:27.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:25:27.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:27.089 | INFO | Uploaded 36GiB to GPU in 2.58688s (13.9164GiB/s) | 20260508 04:25:27.089 | INFO | Downloaded 34GiB to GPU in 2.4162s (14.0717GiB/s) | 20260508 04:25:32.228 | INFO | Analyzing chunk frames = [5220352,5220864) | 20260508 04:25:32.228 | INFO | MemInfo: { | 20260508 04:25:32.228 | INFO | "gpu" : " 17 GB", | 20260508 04:25:32.228 | INFO | "huge" : "", | 20260508 04:25:32.228 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:32.228 | INFO | "malloc" : " 14 GB", | 20260508 04:25:32.228 | INFO | "pinned" : " 29 GB" | 20260508 04:25:32.228 | INFO | } | 20260508 04:25:32.229 | INFO | Perf report for the last 5.13967 seconds: | 20260508 04:25:32.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:32.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:25:32.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:25:32.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:32.229 | INFO | Uploaded 36GiB to GPU in 2.32588s (15.478GiB/s) | 20260508 04:25:32.229 | INFO | Downloaded 34GiB to GPU in 2.54819s (13.3428GiB/s) | 20260508 04:25:37.370 | INFO | Analyzing chunk frames = [5220864,5221376) | 20260508 04:25:37.370 | INFO | MemInfo: { | 20260508 04:25:37.370 | INFO | "gpu" : " 17 GB", | 20260508 04:25:37.370 | INFO | "huge" : "", | 20260508 04:25:37.370 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:37.370 | INFO | "malloc" : " 14 GB", | 20260508 04:25:37.370 | INFO | "pinned" : " 29 GB" | 20260508 04:25:37.370 | INFO | } | 20260508 04:25:37.370 | INFO | Perf report for the last 5.14182 seconds: | 20260508 04:25:37.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:37.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.370 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:25:37.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.370 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:25:37.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:37.371 | INFO | Uploaded 36GiB to GPU in 2.44993s (14.6943GiB/s) | 20260508 04:25:37.371 | INFO | Downloaded 34GiB to GPU in 2.40821s (14.1184GiB/s) | 20260508 04:25:42.507 | INFO | Analyzing chunk frames = [5221376,5221888) | 20260508 04:25:42.507 | INFO | MemInfo: { | 20260508 04:25:42.507 | INFO | "gpu" : " 17 GB", | 20260508 04:25:42.507 | INFO | "huge" : "", | 20260508 04:25:42.507 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:42.508 | INFO | "malloc" : " 14 GB", | 20260508 04:25:42.508 | INFO | "pinned" : " 29 GB" | 20260508 04:25:42.508 | INFO | } | 20260508 04:25:42.508 | INFO | Perf report for the last 5.13729 seconds: | 20260508 04:25:42.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:42.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:25:42.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:25:42.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:42.508 | INFO | Uploaded 36GiB to GPU in 2.49542s (14.4264GiB/s) | 20260508 04:25:42.508 | INFO | Downloaded 34GiB to GPU in 2.43168s (13.9821GiB/s) | 20260508 04:25:47.646 | INFO | Analyzing chunk frames = [5221888,5222400) | 20260508 04:25:47.646 | INFO | MemInfo: { | 20260508 04:25:47.646 | INFO | "gpu" : " 17 GB", | 20260508 04:25:47.646 | INFO | "huge" : "", | 20260508 04:25:47.646 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:47.646 | INFO | "malloc" : " 14 GB", | 20260508 04:25:47.646 | INFO | "pinned" : " 29 GB" | 20260508 04:25:47.646 | INFO | } | 20260508 04:25:47.646 | INFO | Perf report for the last 5.13835 seconds: | 20260508 04:25:47.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:47.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.646 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:25:47.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.646 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 04:25:47.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:47.647 | INFO | Uploaded 36GiB to GPU in 2.48416s (14.4918GiB/s) | 20260508 04:25:47.647 | INFO | Downloaded 34GiB to GPU in 2.4336s (13.9711GiB/s) | 20260508 04:25:47.760 | INFO | Starting a round of Baz IO | 20260508 04:25:51.210 | INFO | Populating RTMetrics report | 20260508 04:25:51.211 | INFO | RTMetrics report populated | 20260508 04:25:51.211 | INFO | Opening tmp RTMetrics json file | 20260508 04:25:51.225 | INFO | Serializing RTMetrics to json | 20260508 04:25:51.303 | INFO | Serialization done, swapping tmp file now... | 20260508 04:25:51.308 | INFO | RTMetrics done | 20260508 04:25:51.320 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.559 seconds | 20260508 04:25:51.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.07% / 0.00% / 58.93% | 20260508 04:25:51.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543722% / 3.97364e-06% / 5.16574e-05% | 20260508 04:25:51.320 | INFO | Baz IO overall: | 20260508 04:25:51.320 | INFO | BytesWritten : 2338292044984 | 20260508 04:25:51.320 | INFO | NumEvents : 1128941745483 | 20260508 04:25:52.785 | INFO | Analyzing chunk frames = [5222400,5222912) | 20260508 04:25:52.785 | INFO | MemInfo: { | 20260508 04:25:52.785 | INFO | "gpu" : " 17 GB", | 20260508 04:25:52.785 | INFO | "huge" : "", | 20260508 04:25:52.785 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:52.785 | INFO | "malloc" : " 14 GB", | 20260508 04:25:52.785 | INFO | "pinned" : " 29 GB" | 20260508 04:25:52.785 | INFO | } | 20260508 04:25:52.785 | INFO | Perf report for the last 5.1393 seconds: | 20260508 04:25:52.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:52.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:52.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:52.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:52.786 | INFO | ANALYSIS : 68.7%, 2.9, 105.5ms, 0 | 20260508 04:25:52.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:52.786 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260508 04:25:52.786 | INFO | PRE_HQ : 67.3%, 1.8, 61.7ms, 0 | 20260508 04:25:52.786 | INFO | BAZWRITER : 68.9%, 3.4, 120.2ms, 0 | 20260508 04:25:52.786 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260508 04:25:52.786 | INFO | Uploaded 36GiB to GPU in 2.5945s (13.8755GiB/s) | 20260508 04:25:52.786 | INFO | Downloaded 38GiB to GPU in 2.71279s (14.0077GiB/s) | 20260508 04:25:57.923 | INFO | Analyzing chunk frames = [5222912,5223424) | 20260508 04:25:57.923 | INFO | MemInfo: { | 20260508 04:25:57.923 | INFO | "gpu" : " 17 GB", | 20260508 04:25:57.923 | INFO | "huge" : "", | 20260508 04:25:57.923 | INFO | "huge_shared" : " 29 GB", | 20260508 04:25:57.923 | INFO | "malloc" : " 14 GB", | 20260508 04:25:57.923 | INFO | "pinned" : " 29 GB" | 20260508 04:25:57.923 | INFO | } | 20260508 04:25:57.924 | INFO | Perf report for the last 5.13818 seconds: | 20260508 04:25:57.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:25:57.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 04:25:57.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:25:57.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:25:57.924 | INFO | Uploaded 36GiB to GPU in 2.58708s (13.9153GiB/s) | 20260508 04:25:57.924 | INFO | Downloaded 34GiB to GPU in 2.41768s (14.0631GiB/s) | 20260508 04:26:03.061 | INFO | Analyzing chunk frames = [5223424,5223936) | 20260508 04:26:03.061 | INFO | MemInfo: { | 20260508 04:26:03.061 | INFO | "gpu" : " 17 GB", | 20260508 04:26:03.061 | INFO | "huge" : "", | 20260508 04:26:03.061 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:03.061 | INFO | "malloc" : " 14 GB", | 20260508 04:26:03.061 | INFO | "pinned" : " 29 GB" | 20260508 04:26:03.061 | INFO | } | 20260508 04:26:03.061 | INFO | Perf report for the last 5.13771 seconds: | 20260508 04:26:03.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:03.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:26:03.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:26:03.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:03.061 | INFO | Uploaded 36GiB to GPU in 2.54801s (14.1287GiB/s) | 20260508 04:26:03.062 | INFO | Downloaded 34GiB to GPU in 2.42066s (14.0457GiB/s) | 20260508 04:26:08.204 | INFO | Analyzing chunk frames = [5223936,5224448) | 20260508 04:26:08.204 | INFO | MemInfo: { | 20260508 04:26:08.204 | INFO | "gpu" : " 17 GB", | 20260508 04:26:08.204 | INFO | "huge" : "", | 20260508 04:26:08.204 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:08.204 | INFO | "malloc" : " 14 GB", | 20260508 04:26:08.204 | INFO | "pinned" : " 29 GB" | 20260508 04:26:08.204 | INFO | } | 20260508 04:26:08.204 | INFO | Perf report for the last 5.14249 seconds: | 20260508 04:26:08.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:08.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 04:26:08.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:26:08.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:08.204 | INFO | Uploaded 36GiB to GPU in 2.57518s (13.9796GiB/s) | 20260508 04:26:08.204 | INFO | Downloaded 34GiB to GPU in 2.50169s (13.5908GiB/s) | 20260508 04:26:13.343 | INFO | Analyzing chunk frames = [5224448,5224960) | 20260508 04:26:13.343 | INFO | MemInfo: { | 20260508 04:26:13.343 | INFO | "gpu" : " 17 GB", | 20260508 04:26:13.343 | INFO | "huge" : "", | 20260508 04:26:13.343 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:13.343 | INFO | "malloc" : " 14 GB", | 20260508 04:26:13.343 | INFO | "pinned" : " 29 GB" | 20260508 04:26:13.343 | INFO | } | 20260508 04:26:13.343 | INFO | Perf report for the last 5.13909 seconds: | 20260508 04:26:13.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:13.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 04:26:13.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:26:13.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:13.343 | INFO | Uploaded 36GiB to GPU in 2.42895s (14.8212GiB/s) | 20260508 04:26:13.343 | INFO | Downloaded 34GiB to GPU in 2.55303s (13.3175GiB/s) | 20260508 04:26:18.483 | INFO | Analyzing chunk frames = [5224960,5225472) | 20260508 04:26:18.483 | INFO | MemInfo: { | 20260508 04:26:18.483 | INFO | "gpu" : " 17 GB", | 20260508 04:26:18.483 | INFO | "huge" : "", | 20260508 04:26:18.483 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:18.483 | INFO | "malloc" : " 14 GB", | 20260508 04:26:18.483 | INFO | "pinned" : " 29 GB" | 20260508 04:26:18.483 | INFO | } | 20260508 04:26:18.484 | INFO | Perf report for the last 5.14066 seconds: | 20260508 04:26:18.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:18.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:26:18.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:26:18.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:18.484 | INFO | Uploaded 36GiB to GPU in 2.4278s (14.8282GiB/s) | 20260508 04:26:18.484 | INFO | Downloaded 34GiB to GPU in 2.42192s (14.0384GiB/s) | 20260508 04:26:23.624 | INFO | Analyzing chunk frames = [5225472,5225984) | 20260508 04:26:23.624 | INFO | MemInfo: { | 20260508 04:26:23.624 | INFO | "gpu" : " 17 GB", | 20260508 04:26:23.624 | INFO | "huge" : "", | 20260508 04:26:23.625 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:23.625 | INFO | "malloc" : " 14 GB", | 20260508 04:26:23.625 | INFO | "pinned" : " 29 GB" | 20260508 04:26:23.625 | INFO | } | 20260508 04:26:23.625 | INFO | Perf report for the last 5.14113 seconds: | 20260508 04:26:23.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:23.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:26:23.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:26:23.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:23.625 | INFO | Uploaded 36GiB to GPU in 2.4251s (14.8448GiB/s) | 20260508 04:26:23.625 | INFO | Downloaded 34GiB to GPU in 2.41869s (14.0572GiB/s) | 20260508 04:26:28.758 | INFO | Analyzing chunk frames = [5225984,5226496) | 20260508 04:26:28.758 | INFO | MemInfo: { | 20260508 04:26:28.758 | INFO | "gpu" : " 17 GB", | 20260508 04:26:28.758 | INFO | "huge" : "", | 20260508 04:26:28.758 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:28.758 | INFO | "malloc" : " 14 GB", | 20260508 04:26:28.758 | INFO | "pinned" : " 29 GB" | 20260508 04:26:28.758 | INFO | } | 20260508 04:26:28.758 | INFO | Perf report for the last 5.13372 seconds: | 20260508 04:26:28.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:28.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.758 | INFO | ANALYSIS : 66.3%, 3.0, 102.5ms, 0 | 20260508 04:26:28.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.759 | INFO | PRE_HQ : 34.1%, 1.0, 17.5ms, 0 | 20260508 04:26:28.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:28.759 | INFO | Uploaded 36GiB to GPU in 2.44755s (14.7086GiB/s) | 20260508 04:26:28.759 | INFO | Downloaded 34GiB to GPU in 2.41149s (14.0992GiB/s) | 20260508 04:26:28.759 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:26:28.759 | INFO | Total mem usage: 16.7 GiB | 20260508 04:26:28.759 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:26:28.759 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.7%) | 20260508 04:26:28.759 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:26:28.759 | INFO | High Water Breakdown: | 20260508 04:26:28.759 | INFO | 60.4% : 9.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:26:28.759 | INFO | 39.5% : 5.96 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:26:28.759 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:26:28.759 | INFO | ------------------------------------------ | 20260508 04:26:28.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:26:28.759 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:26:28.759 | INFO | Total mem usage: 26.5 GiB | 20260508 04:26:28.759 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:26:28.759 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:26:28.759 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:26:28.759 | INFO | High Water Breakdown: | 20260508 04:26:28.759 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:26:28.759 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:26:28.759 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:26:28.759 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:26:28.759 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:26:28.759 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:26:28.759 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:26:28.759 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:26:28.759 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:26:28.759 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:26:28.759 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:26:28.759 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:26:28.759 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:26:28.759 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:26:28.759 | INFO | ------------------------------------------ | 20260508 04:26:28.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:26:28.759 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:26:28.759 | INFO | Total mem usage: 27.8 GiB | 20260508 04:26:28.759 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:26:28.759 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:26:28.759 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:26:28.759 | INFO | High Water Breakdown: | 20260508 04:26:28.759 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:26:28.759 | INFO | ------------------------------------------ | 20260508 04:26:28.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:26:28.759 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:26:28.759 | INFO | Total mem usage: 15.7 GiB | 20260508 04:26:28.759 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:26:28.759 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:26:28.759 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:26:28.759 | INFO | High Water Breakdown: | 20260508 04:26:28.759 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:26:28.760 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:26:28.760 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:26:28.760 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:26:28.760 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:26:28.760 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:26:28.760 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:26:28.760 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:26:28.760 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:26:28.760 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:26:28.760 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:26:28.760 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:26:28.760 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:26:28.760 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:26:28.760 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:26:28.760 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:26:28.760 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:26:28.760 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:26:28.760 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:26:28.760 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:26:28.760 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:26:28.760 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:26:28.760 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:26:28.760 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:26:28.760 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:26:28.760 | INFO | ------------------------------------------ | 20260508 04:26:28.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:26:28.760 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:26:28.760 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:26:28.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:26:28.760 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.65e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.12e-02 / 7.28e-03 | 20260508 04:26:28.760 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.43e+00, 5.88e-01 / 2.02e-01 | 20260508 04:26:28.760 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.11e-01 | 20260508 04:26:28.760 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.56e+01, 3.86e+00 / 1.87e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.85e+01 / 2.02e+01, 1.74e+00 / 1.07e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.57e+00, 4.21e-01 / 2.06e-01 | 20260508 04:26:28.760 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.19e+00, 2.65e-01 / 1.28e-01 | 20260508 04:26:28.760 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.60e-01, 5.02e-02 / 2.25e-02 | 20260508 04:26:28.760 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:26:28.760 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:26:28.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:26:28.760 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 24.55%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 24.28%, Download -- 2.58e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 2.38%, LockUpload -- 2.53e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 2.54%, LockDownload -- 2.70e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 15.46%, LockCompute -- 1.64e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 04:26:28.760 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 04:26:28.859 | INFO | Starting a round of Baz IO | 20260508 04:26:32.300 | INFO | Populating RTMetrics report | 20260508 04:26:32.301 | INFO | RTMetrics report populated | 20260508 04:26:32.301 | INFO | Opening tmp RTMetrics json file | 20260508 04:26:32.404 | INFO | Serializing RTMetrics to json | 20260508 04:26:32.410 | INFO | Serialization done, swapping tmp file now... | 20260508 04:26:32.413 | INFO | RTMetrics done | 20260508 04:26:32.426 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.567 seconds | 20260508 04:26:32.426 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.06% / 0.00% / 58.93% | 20260508 04:26:32.426 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543412% / 0% / 4.37101e-05% | 20260508 04:26:32.426 | INFO | Baz IO overall: | 20260508 04:26:32.426 | INFO | BytesWritten : 2340011759517 | 20260508 04:26:32.426 | INFO | NumEvents : 1129763990957 | 20260508 04:26:33.905 | INFO | Analyzing chunk frames = [5226496,5227008) | 20260508 04:26:33.905 | INFO | MemInfo: { | 20260508 04:26:33.905 | INFO | "gpu" : " 17 GB", | 20260508 04:26:33.905 | INFO | "huge" : "", | 20260508 04:26:33.905 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:33.905 | INFO | "malloc" : " 14 GB", | 20260508 04:26:33.905 | INFO | "pinned" : " 29 GB" | 20260508 04:26:33.905 | INFO | } | 20260508 04:26:33.905 | INFO | Perf report for the last 5.1466 seconds: | 20260508 04:26:33.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:33.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:33.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:33.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:33.905 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 04:26:33.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:33.905 | INFO | RT_METRICS : 41.6%, 1.0, 21.9ms, 0 | 20260508 04:26:33.905 | INFO | PRE_HQ : 65.8%, 1.6, 55.9ms, 0 | 20260508 04:26:33.905 | INFO | BAZWRITER : 68.7%, 3.2, 113.9ms, 0 | 20260508 04:26:33.905 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260508 04:26:33.905 | INFO | Uploaded 36GiB to GPU in 2.58065s (13.95GiB/s) | 20260508 04:26:33.905 | INFO | Downloaded 38GiB to GPU in 2.71488s (13.997GiB/s) | 20260508 04:26:39.040 | INFO | Analyzing chunk frames = [5227008,5227520) | 20260508 04:26:39.040 | INFO | MemInfo: { | 20260508 04:26:39.040 | INFO | "gpu" : " 17 GB", | 20260508 04:26:39.040 | INFO | "huge" : "", | 20260508 04:26:39.040 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:39.041 | INFO | "malloc" : " 14 GB", | 20260508 04:26:39.041 | INFO | "pinned" : " 29 GB" | 20260508 04:26:39.041 | INFO | } | 20260508 04:26:39.041 | INFO | Perf report for the last 5.13556 seconds: | 20260508 04:26:39.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:39.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 04:26:39.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:26:39.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:39.041 | INFO | Uploaded 36GiB to GPU in 2.56155s (14.054GiB/s) | 20260508 04:26:39.041 | INFO | Downloaded 34GiB to GPU in 2.45041s (13.8752GiB/s) | 20260508 04:26:44.182 | INFO | Analyzing chunk frames = [5227520,5228032) | 20260508 04:26:44.182 | INFO | MemInfo: { | 20260508 04:26:44.182 | INFO | "gpu" : " 17 GB", | 20260508 04:26:44.182 | INFO | "huge" : "", | 20260508 04:26:44.182 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:44.182 | INFO | "malloc" : " 14 GB", | 20260508 04:26:44.182 | INFO | "pinned" : " 29 GB" | 20260508 04:26:44.182 | INFO | } | 20260508 04:26:44.182 | INFO | Perf report for the last 5.14152 seconds: | 20260508 04:26:44.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:44.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.182 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 04:26:44.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.182 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:26:44.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:44.183 | INFO | Uploaded 36GiB to GPU in 2.51533s (14.3123GiB/s) | 20260508 04:26:44.183 | INFO | Downloaded 34GiB to GPU in 2.44109s (13.9282GiB/s) | 20260508 04:26:49.317 | INFO | Analyzing chunk frames = [5228032,5228544) | 20260508 04:26:49.317 | INFO | MemInfo: { | 20260508 04:26:49.317 | INFO | "gpu" : " 17 GB", | 20260508 04:26:49.317 | INFO | "huge" : "", | 20260508 04:26:49.317 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:49.317 | INFO | "malloc" : " 14 GB", | 20260508 04:26:49.317 | INFO | "pinned" : " 29 GB" | 20260508 04:26:49.317 | INFO | } | 20260508 04:26:49.318 | INFO | Perf report for the last 5.13538 seconds: | 20260508 04:26:49.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:49.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:26:49.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:26:49.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:49.318 | INFO | Uploaded 36GiB to GPU in 2.54763s (14.1308GiB/s) | 20260508 04:26:49.318 | INFO | Downloaded 34GiB to GPU in 2.42942s (13.9951GiB/s) | 20260508 04:26:54.456 | INFO | Analyzing chunk frames = [5228544,5229056) | 20260508 04:26:54.456 | INFO | MemInfo: { | 20260508 04:26:54.456 | INFO | "gpu" : " 17 GB", | 20260508 04:26:54.456 | INFO | "huge" : "", | 20260508 04:26:54.456 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:54.456 | INFO | "malloc" : " 14 GB", | 20260508 04:26:54.456 | INFO | "pinned" : " 29 GB" | 20260508 04:26:54.456 | INFO | } | 20260508 04:26:54.456 | INFO | Perf report for the last 5.13872 seconds: | 20260508 04:26:54.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:54.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.456 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:26:54.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.456 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:26:54.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:54.457 | INFO | Uploaded 36GiB to GPU in 2.32455s (15.4869GiB/s) | 20260508 04:26:54.457 | INFO | Downloaded 34GiB to GPU in 2.55956s (13.2835GiB/s) | 20260508 04:26:59.601 | INFO | Analyzing chunk frames = [5229056,5229568) | 20260508 04:26:59.601 | INFO | MemInfo: { | 20260508 04:26:59.601 | INFO | "gpu" : " 17 GB", | 20260508 04:26:59.601 | INFO | "huge" : "", | 20260508 04:26:59.601 | INFO | "huge_shared" : " 29 GB", | 20260508 04:26:59.601 | INFO | "malloc" : " 14 GB", | 20260508 04:26:59.601 | INFO | "pinned" : " 29 GB" | 20260508 04:26:59.601 | INFO | } | 20260508 04:26:59.601 | INFO | Perf report for the last 5.145 seconds: | 20260508 04:26:59.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:26:59.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.601 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:26:59.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.601 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:26:59.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:26:59.602 | INFO | Uploaded 36GiB to GPU in 2.42731s (14.8312GiB/s) | 20260508 04:26:59.602 | INFO | Downloaded 34GiB to GPU in 2.42875s (13.999GiB/s) | 20260508 04:27:04.740 | INFO | Analyzing chunk frames = [5229568,5230080) | 20260508 04:27:04.740 | INFO | MemInfo: { | 20260508 04:27:04.740 | INFO | "gpu" : " 17 GB", | 20260508 04:27:04.740 | INFO | "huge" : "", | 20260508 04:27:04.741 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:04.741 | INFO | "malloc" : " 14 GB", | 20260508 04:27:04.741 | INFO | "pinned" : " 29 GB" | 20260508 04:27:04.741 | INFO | } | 20260508 04:27:04.741 | INFO | Perf report for the last 5.13959 seconds: | 20260508 04:27:04.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:04.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.741 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:27:04.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.741 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260508 04:27:04.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:04.742 | INFO | Uploaded 36GiB to GPU in 2.42309s (14.857GiB/s) | 20260508 04:27:04.742 | INFO | Downloaded 34GiB to GPU in 2.42903s (13.9973GiB/s) | 20260508 04:27:09.875 | INFO | Analyzing chunk frames = [5230080,5230592) | 20260508 04:27:09.875 | INFO | MemInfo: { | 20260508 04:27:09.875 | INFO | "gpu" : " 17 GB", | 20260508 04:27:09.875 | INFO | "huge" : "", | 20260508 04:27:09.875 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:09.875 | INFO | "malloc" : " 14 GB", | 20260508 04:27:09.876 | INFO | "pinned" : " 29 GB" | 20260508 04:27:09.876 | INFO | } | 20260508 04:27:09.876 | INFO | Perf report for the last 5.13473 seconds: | 20260508 04:27:09.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:09.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:27:09.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 04:27:09.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:09.876 | INFO | Uploaded 36GiB to GPU in 2.44558s (14.7205GiB/s) | 20260508 04:27:09.876 | INFO | Downloaded 34GiB to GPU in 2.41657s (14.0695GiB/s) | 20260508 04:27:09.988 | INFO | Starting a round of Baz IO | 20260508 04:27:13.442 | INFO | Populating RTMetrics report | 20260508 04:27:13.442 | INFO | RTMetrics report populated | 20260508 04:27:13.442 | INFO | Opening tmp RTMetrics json file | 20260508 04:27:13.459 | INFO | Serializing RTMetrics to json | 20260508 04:27:13.462 | INFO | Serialization done, swapping tmp file now... | 20260508 04:27:13.462 | INFO | RTMetrics done | 20260508 04:27:13.526 | INFO | Finished a round of Baz IO: Wrote 1.598 GiB in 3.537 seconds | 20260508 04:27:13.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.06% / 0.00% / 58.94% | 20260508 04:27:13.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.539931% / 0% / 5.96046e-05% | 20260508 04:27:13.526 | INFO | Baz IO overall: | 20260508 04:27:13.526 | INFO | BytesWritten : 2341727943795 | 20260508 04:27:13.526 | INFO | NumEvents : 1130583751757 | 20260508 04:27:15.010 | INFO | Analyzing chunk frames = [5230592,5231104) | 20260508 04:27:15.010 | INFO | MemInfo: { | 20260508 04:27:15.010 | INFO | "gpu" : " 17 GB", | 20260508 04:27:15.010 | INFO | "huge" : "", | 20260508 04:27:15.010 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:15.010 | INFO | "malloc" : " 14 GB", | 20260508 04:27:15.010 | INFO | "pinned" : " 29 GB" | 20260508 04:27:15.010 | INFO | } | 20260508 04:27:15.010 | INFO | Perf report for the last 5.13483 seconds: | 20260508 04:27:15.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:15.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:15.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:15.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:15.011 | INFO | ANALYSIS : 68.5%, 2.9, 105.9ms, 0 | 20260508 04:27:15.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:15.011 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 04:27:15.011 | INFO | PRE_HQ : 67.3%, 1.8, 61.6ms, 0 | 20260508 04:27:15.011 | INFO | BAZWRITER : 68.4%, 3.5, 121.6ms, 0 | 20260508 04:27:15.011 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 04:27:15.011 | INFO | Uploaded 36GiB to GPU in 2.62795s (13.6989GiB/s) | 20260508 04:27:15.011 | INFO | Downloaded 38GiB to GPU in 2.7116s (14.0139GiB/s) | 20260508 04:27:20.159 | INFO | Analyzing chunk frames = [5231104,5231616) | 20260508 04:27:20.159 | INFO | MemInfo: { | 20260508 04:27:20.159 | INFO | "gpu" : " 17 GB", | 20260508 04:27:20.159 | INFO | "huge" : "", | 20260508 04:27:20.159 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:20.160 | INFO | "malloc" : " 14 GB", | 20260508 04:27:20.160 | INFO | "pinned" : " 29 GB" | 20260508 04:27:20.160 | INFO | } | 20260508 04:27:20.160 | INFO | Perf report for the last 5.14914 seconds: | 20260508 04:27:20.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:20.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 04:27:20.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:27:20.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:20.160 | INFO | Uploaded 36GiB to GPU in 2.58717s (13.9148GiB/s) | 20260508 04:27:20.160 | INFO | Downloaded 34GiB to GPU in 2.41972s (14.0512GiB/s) | 20260508 04:27:25.298 | INFO | Analyzing chunk frames = [5231616,5232128) | 20260508 04:27:25.298 | INFO | MemInfo: { | 20260508 04:27:25.298 | INFO | "gpu" : " 17 GB", | 20260508 04:27:25.298 | INFO | "huge" : "", | 20260508 04:27:25.298 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:25.298 | INFO | "malloc" : " 14 GB", | 20260508 04:27:25.298 | INFO | "pinned" : " 29 GB" | 20260508 04:27:25.298 | INFO | } | 20260508 04:27:25.298 | INFO | Perf report for the last 5.13865 seconds: | 20260508 04:27:25.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:25.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.298 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 04:27:25.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.298 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:27:25.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:25.299 | INFO | Uploaded 36GiB to GPU in 2.54972s (14.1192GiB/s) | 20260508 04:27:25.299 | INFO | Downloaded 34GiB to GPU in 2.40148s (14.1579GiB/s) | 20260508 04:27:30.438 | INFO | Analyzing chunk frames = [5232128,5232640) | 20260508 04:27:30.438 | INFO | MemInfo: { | 20260508 04:27:30.438 | INFO | "gpu" : " 17 GB", | 20260508 04:27:30.438 | INFO | "huge" : "", | 20260508 04:27:30.438 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:30.439 | INFO | "malloc" : " 14 GB", | 20260508 04:27:30.439 | INFO | "pinned" : " 29 GB" | 20260508 04:27:30.439 | INFO | } | 20260508 04:27:30.439 | INFO | Perf report for the last 5.14035 seconds: | 20260508 04:27:30.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:30.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | ANALYSIS : 67.4%, 3.0, 104.3ms, 0 | 20260508 04:27:30.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:27:30.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:30.439 | INFO | Uploaded 36GiB to GPU in 2.57449s (13.9834GiB/s) | 20260508 04:27:30.439 | INFO | Downloaded 34GiB to GPU in 2.5026s (13.5859GiB/s) | 20260508 04:27:35.574 | INFO | Analyzing chunk frames = [5232640,5233152) | 20260508 04:27:35.574 | INFO | MemInfo: { | 20260508 04:27:35.574 | INFO | "gpu" : " 17 GB", | 20260508 04:27:35.574 | INFO | "huge" : "", | 20260508 04:27:35.574 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:35.574 | INFO | "malloc" : " 14 GB", | 20260508 04:27:35.574 | INFO | "pinned" : " 29 GB" | 20260508 04:27:35.574 | INFO | } | 20260508 04:27:35.574 | INFO | Perf report for the last 5.1352 seconds: | 20260508 04:27:35.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:35.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 04:27:35.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 04:27:35.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:35.574 | INFO | Uploaded 36GiB to GPU in 2.40472s (14.9706GiB/s) | 20260508 04:27:35.574 | INFO | Downloaded 34GiB to GPU in 2.55901s (13.2864GiB/s) | 20260508 04:27:40.714 | INFO | Analyzing chunk frames = [5233152,5233664) | 20260508 04:27:40.714 | INFO | MemInfo: { | 20260508 04:27:40.714 | INFO | "gpu" : " 17 GB", | 20260508 04:27:40.714 | INFO | "huge" : "", | 20260508 04:27:40.714 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:40.714 | INFO | "malloc" : " 14 GB", | 20260508 04:27:40.714 | INFO | "pinned" : " 29 GB" | 20260508 04:27:40.715 | INFO | } | 20260508 04:27:40.715 | INFO | Perf report for the last 5.1407 seconds: | 20260508 04:27:40.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:40.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:27:40.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:27:40.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:40.715 | INFO | Uploaded 36GiB to GPU in 2.41992s (14.8765GiB/s) | 20260508 04:27:40.715 | INFO | Downloaded 34GiB to GPU in 2.41671s (14.0687GiB/s) | 20260508 04:27:45.852 | INFO | Analyzing chunk frames = [5233664,5234176) | 20260508 04:27:45.852 | INFO | MemInfo: { | 20260508 04:27:45.852 | INFO | "gpu" : " 17 GB", | 20260508 04:27:45.852 | INFO | "huge" : "", | 20260508 04:27:45.852 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:45.852 | INFO | "malloc" : " 14 GB", | 20260508 04:27:45.853 | INFO | "pinned" : " 29 GB" | 20260508 04:27:45.853 | INFO | } | 20260508 04:27:45.853 | INFO | Perf report for the last 5.13809 seconds: | 20260508 04:27:45.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:45.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:27:45.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:27:45.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:45.853 | INFO | Uploaded 36GiB to GPU in 2.4296s (14.8173GiB/s) | 20260508 04:27:45.853 | INFO | Downloaded 34GiB to GPU in 2.4165s (14.0699GiB/s) | 20260508 04:27:50.996 | INFO | Analyzing chunk frames = [5234176,5234688) | 20260508 04:27:50.996 | INFO | MemInfo: { | 20260508 04:27:50.996 | INFO | "gpu" : " 17 GB", | 20260508 04:27:50.996 | INFO | "huge" : "", | 20260508 04:27:50.996 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:50.996 | INFO | "malloc" : " 14 GB", | 20260508 04:27:50.996 | INFO | "pinned" : " 29 GB" | 20260508 04:27:50.996 | INFO | } | 20260508 04:27:50.996 | INFO | Perf report for the last 5.14352 seconds: | 20260508 04:27:50.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:50.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.996 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:27:50.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.996 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 04:27:50.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:50.997 | INFO | Uploaded 36GiB to GPU in 2.43381s (14.7916GiB/s) | 20260508 04:27:50.997 | INFO | Downloaded 34GiB to GPU in 2.43457s (13.9655GiB/s) | 20260508 04:27:50.997 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:27:50.997 | INFO | Total mem usage: 16.7 GiB | 20260508 04:27:50.997 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:27:50.997 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.6%) | 20260508 04:27:50.997 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:27:50.997 | INFO | High Water Breakdown: | 20260508 04:27:50.997 | INFO | 59.9% : 8.8 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:27:50.997 | INFO | 39.9% : 5.86 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:27:50.997 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:27:50.997 | INFO | ------------------------------------------ | 20260508 04:27:50.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:27:50.997 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:27:50.997 | INFO | Total mem usage: 26.5 GiB | 20260508 04:27:50.997 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:27:50.997 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:27:50.997 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:27:50.997 | INFO | High Water Breakdown: | 20260508 04:27:50.997 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:27:50.997 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:27:50.997 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:27:50.997 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:27:50.997 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:27:50.997 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:27:50.997 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:27:50.997 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:27:50.997 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:27:50.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:27:50.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:27:50.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:27:50.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:27:50.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:27:50.997 | INFO | ------------------------------------------ | 20260508 04:27:50.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:27:50.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:27:50.997 | INFO | Total mem usage: 27.8 GiB | 20260508 04:27:50.997 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:27:50.997 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:27:50.997 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:27:50.997 | INFO | High Water Breakdown: | 20260508 04:27:50.997 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:27:50.997 | INFO | ------------------------------------------ | 20260508 04:27:50.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:27:50.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:27:50.997 | INFO | Total mem usage: 15.7 GiB | 20260508 04:27:50.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:27:50.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:27:50.997 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:27:50.997 | INFO | High Water Breakdown: | 20260508 04:27:50.998 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:27:50.998 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:27:50.998 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:27:50.998 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:27:50.998 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:27:50.998 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:27:50.998 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:27:50.998 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:27:50.998 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:27:50.998 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:27:50.998 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:27:50.998 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:27:50.998 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:27:50.998 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:27:50.998 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:27:50.998 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:27:50.998 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:27:50.998 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:27:50.998 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:27:50.998 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:27:50.998 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:27:50.998 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:27:50.998 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:27:50.998 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:27:50.998 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:27:50.998 | INFO | ------------------------------------------ | 20260508 04:27:50.998 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:27:50.998 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:27:50.998 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:27:50.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:27:50.998 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.78e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.05e-01, 1.19e-02 / 7.79e-03 | 20260508 04:27:50.998 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.50e+00, 5.92e-01 / 2.06e-01 | 20260508 04:27:50.998 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.11e-01 / 1.24e-01 | 20260508 04:27:50.998 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.55e+01, 3.86e+00 / 2.09e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.76e+00 / 1.40e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.40e+00 / 2.59e+00, 4.19e-01 / 2.67e-01 | 20260508 04:27:50.998 | INFO | PerfReport: 3.23%, Metrics -- 1.06e+00 / 1.19e+00, 2.63e-01 / 1.28e-01 | 20260508 04:27:50.998 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.56e-01, 5.13e-02 / 3.18e-02 | 20260508 04:27:50.998 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:27:50.998 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:27:50.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:27:50.998 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 24.40%, Upload -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 24.33%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 2.22%, LockUpload -- 2.36e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 2.66%, LockDownload -- 2.82e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:27:50.998 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 04:27:51.111 | INFO | Starting a round of Baz IO | 20260508 04:27:54.558 | INFO | Populating RTMetrics report | 20260508 04:27:54.558 | INFO | RTMetrics report populated | 20260508 04:27:54.558 | INFO | Opening tmp RTMetrics json file | 20260508 04:27:54.579 | INFO | Serializing RTMetrics to json | 20260508 04:27:54.580 | INFO | Serialization done, swapping tmp file now... | 20260508 04:27:54.585 | INFO | RTMetrics done | 20260508 04:27:54.652 | INFO | Finished a round of Baz IO: Wrote 1.599 GiB in 3.54 seconds | 20260508 04:27:54.652 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.06% / 0.00% / 58.94% | 20260508 04:27:54.652 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.540157% / 0% / 7.94729e-05% | 20260508 04:27:54.652 | INFO | Baz IO overall: | 20260508 04:27:54.652 | INFO | BytesWritten : 2343445065887 | 20260508 04:27:54.652 | INFO | NumEvents : 1131404039567 | 20260508 04:27:56.127 | INFO | Analyzing chunk frames = [5234688,5235200) | 20260508 04:27:56.127 | INFO | MemInfo: { | 20260508 04:27:56.127 | INFO | "gpu" : " 17 GB", | 20260508 04:27:56.127 | INFO | "huge" : "", | 20260508 04:27:56.127 | INFO | "huge_shared" : " 29 GB", | 20260508 04:27:56.127 | INFO | "malloc" : " 14 GB", | 20260508 04:27:56.127 | INFO | "pinned" : " 29 GB" | 20260508 04:27:56.127 | INFO | } | 20260508 04:27:56.127 | INFO | Perf report for the last 5.13098 seconds: | 20260508 04:27:56.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:27:56.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:56.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:56.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:56.127 | INFO | ANALYSIS : 68.5%, 2.9, 104.8ms, 0 | 20260508 04:27:56.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:27:56.127 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260508 04:27:56.127 | INFO | PRE_HQ : 67.5%, 1.8, 62.4ms, 0 | 20260508 04:27:56.127 | INFO | BAZWRITER : 68.6%, 3.5, 124.1ms, 0 | 20260508 04:27:56.127 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260508 04:27:56.128 | INFO | Uploaded 36GiB to GPU in 2.5957s (13.8691GiB/s) | 20260508 04:27:56.128 | INFO | Downloaded 38GiB to GPU in 2.71951s (13.9731GiB/s) | 20260508 04:28:01.265 | INFO | Analyzing chunk frames = [5235200,5235712) | 20260508 04:28:01.265 | INFO | MemInfo: { | 20260508 04:28:01.265 | INFO | "gpu" : " 17 GB", | 20260508 04:28:01.265 | INFO | "huge" : "", | 20260508 04:28:01.265 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:01.265 | INFO | "malloc" : " 14 GB", | 20260508 04:28:01.265 | INFO | "pinned" : " 29 GB" | 20260508 04:28:01.265 | INFO | } | 20260508 04:28:01.265 | INFO | Perf report for the last 5.13782 seconds: | 20260508 04:28:01.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:01.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:28:01.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:28:01.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:01.265 | INFO | Uploaded 36GiB to GPU in 2.57102s (14.0022GiB/s) | 20260508 04:28:01.265 | INFO | Downloaded 34GiB to GPU in 2.43268s (13.9764GiB/s) | 20260508 04:28:06.412 | INFO | Analyzing chunk frames = [5235712,5236224) | 20260508 04:28:06.412 | INFO | MemInfo: { | 20260508 04:28:06.412 | INFO | "gpu" : " 17 GB", | 20260508 04:28:06.412 | INFO | "huge" : "", | 20260508 04:28:06.412 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:06.412 | INFO | "malloc" : " 14 GB", | 20260508 04:28:06.413 | INFO | "pinned" : " 29 GB" | 20260508 04:28:06.413 | INFO | } | 20260508 04:28:06.413 | INFO | Perf report for the last 5.14758 seconds: | 20260508 04:28:06.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:06.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 04:28:06.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:28:06.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:06.413 | INFO | Uploaded 36GiB to GPU in 2.53351s (14.2095GiB/s) | 20260508 04:28:06.413 | INFO | Downloaded 34GiB to GPU in 2.42884s (13.9984GiB/s) | 20260508 04:28:11.551 | INFO | Analyzing chunk frames = [5236224,5236736) | 20260508 04:28:11.551 | INFO | MemInfo: { | 20260508 04:28:11.551 | INFO | "gpu" : " 17 GB", | 20260508 04:28:11.551 | INFO | "huge" : "", | 20260508 04:28:11.551 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:11.551 | INFO | "malloc" : " 14 GB", | 20260508 04:28:11.551 | INFO | "pinned" : " 29 GB" | 20260508 04:28:11.551 | INFO | } | 20260508 04:28:11.551 | INFO | Perf report for the last 5.1386 seconds: | 20260508 04:28:11.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:11.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.551 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 04:28:11.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.551 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:28:11.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:11.552 | INFO | Uploaded 36GiB to GPU in 2.56216s (14.0506GiB/s) | 20260508 04:28:11.552 | INFO | Downloaded 34GiB to GPU in 2.41306s (14.09GiB/s) | 20260508 04:28:16.692 | INFO | Analyzing chunk frames = [5236736,5237248) | 20260508 04:28:16.692 | INFO | MemInfo: { | 20260508 04:28:16.692 | INFO | "gpu" : " 17 GB", | 20260508 04:28:16.692 | INFO | "huge" : "", | 20260508 04:28:16.692 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:16.692 | INFO | "malloc" : " 14 GB", | 20260508 04:28:16.692 | INFO | "pinned" : " 29 GB" | 20260508 04:28:16.692 | INFO | } | 20260508 04:28:16.692 | INFO | Perf report for the last 5.14035 seconds: | 20260508 04:28:16.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:16.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 04:28:16.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 04:28:16.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:16.692 | INFO | Uploaded 36GiB to GPU in 2.39852s (15.0093GiB/s) | 20260508 04:28:16.692 | INFO | Downloaded 34GiB to GPU in 2.56496s (13.2556GiB/s) | 20260508 04:28:21.826 | INFO | Analyzing chunk frames = [5237248,5237760) | 20260508 04:28:21.826 | INFO | MemInfo: { | 20260508 04:28:21.827 | INFO | "gpu" : " 17 GB", | 20260508 04:28:21.827 | INFO | "huge" : "", | 20260508 04:28:21.827 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:21.827 | INFO | "malloc" : " 14 GB", | 20260508 04:28:21.827 | INFO | "pinned" : " 29 GB" | 20260508 04:28:21.827 | INFO | } | 20260508 04:28:21.827 | INFO | Perf report for the last 5.13514 seconds: | 20260508 04:28:21.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:21.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:28:21.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:28:21.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:21.827 | INFO | Uploaded 36GiB to GPU in 2.4497s (14.6957GiB/s) | 20260508 04:28:21.827 | INFO | Downloaded 34GiB to GPU in 2.41069s (14.1039GiB/s) | 20260508 04:28:26.964 | INFO | Analyzing chunk frames = [5237760,5238272) | 20260508 04:28:26.964 | INFO | MemInfo: { | 20260508 04:28:26.964 | INFO | "gpu" : " 17 GB", | 20260508 04:28:26.964 | INFO | "huge" : "", | 20260508 04:28:26.964 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:26.964 | INFO | "malloc" : " 14 GB", | 20260508 04:28:26.964 | INFO | "pinned" : " 29 GB" | 20260508 04:28:26.964 | INFO | } | 20260508 04:28:26.964 | INFO | Perf report for the last 5.13716 seconds: | 20260508 04:28:26.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:26.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:28:26.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:28:26.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:26.964 | INFO | Uploaded 36GiB to GPU in 2.44691s (14.7124GiB/s) | 20260508 04:28:26.965 | INFO | Downloaded 34GiB to GPU in 2.41209s (14.0957GiB/s) | 20260508 04:28:32.104 | INFO | Analyzing chunk frames = [5238272,5238784) | 20260508 04:28:32.104 | INFO | MemInfo: { | 20260508 04:28:32.104 | INFO | "gpu" : " 17 GB", | 20260508 04:28:32.104 | INFO | "huge" : "", | 20260508 04:28:32.105 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:32.105 | INFO | "malloc" : " 14 GB", | 20260508 04:28:32.105 | INFO | "pinned" : " 29 GB" | 20260508 04:28:32.105 | INFO | } | 20260508 04:28:32.105 | INFO | Perf report for the last 5.14077 seconds: | 20260508 04:28:32.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:32.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 04:28:32.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 04:28:32.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:32.105 | INFO | Uploaded 36GiB to GPU in 2.47224s (14.5617GiB/s) | 20260508 04:28:32.105 | INFO | Downloaded 34GiB to GPU in 2.42793s (14.0037GiB/s) | 20260508 04:28:32.217 | INFO | Starting a round of Baz IO | 20260508 04:28:35.643 | INFO | Populating RTMetrics report | 20260508 04:28:35.643 | INFO | RTMetrics report populated | 20260508 04:28:35.643 | INFO | Opening tmp RTMetrics json file | 20260508 04:28:35.644 | INFO | Serializing RTMetrics to json | 20260508 04:28:35.645 | INFO | Serialization done, swapping tmp file now... | 20260508 04:28:35.645 | INFO | RTMetrics done | 20260508 04:28:35.771 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.553 seconds | 20260508 04:28:35.771 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.06% / 0.00% / 58.94% | 20260508 04:28:35.771 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.538107% / 0% / 7.94729e-05% | 20260508 04:28:35.771 | INFO | Baz IO overall: | 20260508 04:28:35.771 | INFO | BytesWritten : 2345160018147 | 20260508 04:28:35.771 | INFO | NumEvents : 1132223058370 | 20260508 04:28:37.248 | INFO | Analyzing chunk frames = [5238784,5239296) | 20260508 04:28:37.248 | INFO | MemInfo: { | 20260508 04:28:37.248 | INFO | "gpu" : " 17 GB", | 20260508 04:28:37.248 | INFO | "huge" : "", | 20260508 04:28:37.248 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:37.248 | INFO | "malloc" : " 14 GB", | 20260508 04:28:37.248 | INFO | "pinned" : " 29 GB" | 20260508 04:28:37.248 | INFO | } | 20260508 04:28:37.248 | INFO | Perf report for the last 5.14368 seconds: | 20260508 04:28:37.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:37.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:37.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:37.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:37.248 | INFO | ANALYSIS : 68.1%, 2.9, 103.9ms, 0 | 20260508 04:28:37.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:37.249 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260508 04:28:37.249 | INFO | PRE_HQ : 67.0%, 1.8, 63.0ms, 0 | 20260508 04:28:37.249 | INFO | BAZWRITER : 68.7%, 3.9, 139.0ms, 0 | 20260508 04:28:37.249 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 04:28:37.249 | INFO | Uploaded 36GiB to GPU in 2.59334s (13.8817GiB/s) | 20260508 04:28:37.249 | INFO | Downloaded 38GiB to GPU in 2.70177s (14.0649GiB/s) | 20260508 04:28:42.391 | INFO | Analyzing chunk frames = [5239296,5239808) | 20260508 04:28:42.391 | INFO | MemInfo: { | 20260508 04:28:42.391 | INFO | "gpu" : " 17 GB", | 20260508 04:28:42.391 | INFO | "huge" : "", | 20260508 04:28:42.391 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:42.391 | INFO | "malloc" : " 14 GB", | 20260508 04:28:42.391 | INFO | "pinned" : " 29 GB" | 20260508 04:28:42.391 | INFO | } | 20260508 04:28:42.391 | INFO | Perf report for the last 5.14294 seconds: | 20260508 04:28:42.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:42.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.391 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:28:42.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.391 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:28:42.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:42.392 | INFO | Uploaded 36GiB to GPU in 2.59108s (13.8938GiB/s) | 20260508 04:28:42.392 | INFO | Downloaded 34GiB to GPU in 2.43719s (13.9505GiB/s) | 20260508 04:28:47.522 | INFO | Analyzing chunk frames = [5239808,5240320) | 20260508 04:28:47.522 | INFO | MemInfo: { | 20260508 04:28:47.522 | INFO | "gpu" : " 17 GB", | 20260508 04:28:47.522 | INFO | "huge" : "", | 20260508 04:28:47.522 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:47.522 | INFO | "malloc" : " 14 GB", | 20260508 04:28:47.522 | INFO | "pinned" : " 29 GB" | 20260508 04:28:47.522 | INFO | } | 20260508 04:28:47.522 | INFO | Perf report for the last 5.13107 seconds: | 20260508 04:28:47.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:47.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.523 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 04:28:47.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.523 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:28:47.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:47.523 | INFO | Uploaded 36GiB to GPU in 2.55118s (14.1111GiB/s) | 20260508 04:28:47.523 | INFO | Downloaded 34GiB to GPU in 2.39943s (14.17GiB/s) | 20260508 04:28:52.666 | INFO | Analyzing chunk frames = [5240320,5240832) | 20260508 04:28:52.666 | INFO | MemInfo: { | 20260508 04:28:52.666 | INFO | "gpu" : " 17 GB", | 20260508 04:28:52.666 | INFO | "huge" : "", | 20260508 04:28:52.666 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:52.666 | INFO | "malloc" : " 14 GB", | 20260508 04:28:52.666 | INFO | "pinned" : " 29 GB" | 20260508 04:28:52.666 | INFO | } | 20260508 04:28:52.666 | INFO | Perf report for the last 5.14352 seconds: | 20260508 04:28:52.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:52.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.666 | INFO | ANALYSIS : 67.3%, 3.0, 104.2ms, 0 | 20260508 04:28:52.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.666 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:28:52.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:52.667 | INFO | Uploaded 36GiB to GPU in 2.56041s (14.0603GiB/s) | 20260508 04:28:52.667 | INFO | Downloaded 34GiB to GPU in 2.49931s (13.6037GiB/s) | 20260508 04:28:57.801 | INFO | Analyzing chunk frames = [5240832,5241344) | 20260508 04:28:57.801 | INFO | MemInfo: { | 20260508 04:28:57.801 | INFO | "gpu" : " 17 GB", | 20260508 04:28:57.801 | INFO | "huge" : "", | 20260508 04:28:57.801 | INFO | "huge_shared" : " 29 GB", | 20260508 04:28:57.801 | INFO | "malloc" : " 14 GB", | 20260508 04:28:57.801 | INFO | "pinned" : " 29 GB" | 20260508 04:28:57.801 | INFO | } | 20260508 04:28:57.801 | INFO | Perf report for the last 5.13536 seconds: | 20260508 04:28:57.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:28:57.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.801 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 04:28:57.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.801 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:28:57.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:28:57.802 | INFO | Uploaded 36GiB to GPU in 2.37972s (15.1278GiB/s) | 20260508 04:28:57.802 | INFO | Downloaded 34GiB to GPU in 2.53871s (13.3926GiB/s) | 20260508 04:29:02.942 | INFO | Analyzing chunk frames = [5241344,5241856) | 20260508 04:29:02.942 | INFO | MemInfo: { | 20260508 04:29:02.942 | INFO | "gpu" : " 17 GB", | 20260508 04:29:02.942 | INFO | "huge" : "", | 20260508 04:29:02.942 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:02.942 | INFO | "malloc" : " 14 GB", | 20260508 04:29:02.942 | INFO | "pinned" : " 29 GB" | 20260508 04:29:02.942 | INFO | } | 20260508 04:29:02.942 | INFO | Perf report for the last 5.14063 seconds: | 20260508 04:29:02.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:02.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.942 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 04:29:02.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.942 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:29:02.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:02.943 | INFO | Uploaded 36GiB to GPU in 2.45026s (14.6923GiB/s) | 20260508 04:29:02.943 | INFO | Downloaded 34GiB to GPU in 2.42701s (14.009GiB/s) | 20260508 04:29:08.077 | INFO | Analyzing chunk frames = [5241856,5242368) | 20260508 04:29:08.077 | INFO | MemInfo: { | 20260508 04:29:08.077 | INFO | "gpu" : " 17 GB", | 20260508 04:29:08.077 | INFO | "huge" : "", | 20260508 04:29:08.077 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:08.077 | INFO | "malloc" : " 14 GB", | 20260508 04:29:08.077 | INFO | "pinned" : " 29 GB" | 20260508 04:29:08.077 | INFO | } | 20260508 04:29:08.077 | INFO | Perf report for the last 5.13512 seconds: | 20260508 04:29:08.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:08.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.077 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:29:08.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.077 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 04:29:08.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:08.078 | INFO | Uploaded 36GiB to GPU in 2.44327s (14.7343GiB/s) | 20260508 04:29:08.078 | INFO | Downloaded 34GiB to GPU in 2.40761s (14.1219GiB/s) | 20260508 04:29:13.218 | INFO | Analyzing chunk frames = [5242368,5242880) | 20260508 04:29:13.218 | INFO | MemInfo: { | 20260508 04:29:13.218 | INFO | "gpu" : " 17 GB", | 20260508 04:29:13.218 | INFO | "huge" : "", | 20260508 04:29:13.219 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:13.219 | INFO | "malloc" : " 14 GB", | 20260508 04:29:13.219 | INFO | "pinned" : " 29 GB" | 20260508 04:29:13.219 | INFO | } | 20260508 04:29:13.219 | INFO | Perf report for the last 5.14219 seconds: | 20260508 04:29:13.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:13.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.219 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:29:13.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.219 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:29:13.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:13.220 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:29:13.220 | INFO | Total mem usage: 16.7 GiB | 20260508 04:29:13.220 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:29:13.220 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.8%) | 20260508 04:29:13.220 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:29:13.220 | INFO | High Water Breakdown: | 20260508 04:29:13.220 | INFO | 60.1% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:29:13.220 | INFO | 39.8% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:29:13.220 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:29:13.220 | INFO | ------------------------------------------ | 20260508 04:29:13.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:29:13.220 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:29:13.220 | INFO | Total mem usage: 26.5 GiB | 20260508 04:29:13.220 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:29:13.220 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260508 04:29:13.220 | INFO | Unused Cache: 1.77 GiB (Waste: 6.68%) | 20260508 04:29:13.220 | INFO | High Water Breakdown: | 20260508 04:29:13.220 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:29:13.220 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:29:13.220 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:29:13.220 | INFO | 1.6% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:29:13.220 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:29:13.220 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:29:13.220 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:29:13.220 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:29:13.220 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:29:13.220 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:29:13.220 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:29:13.220 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:29:13.220 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:29:13.220 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:29:13.220 | INFO | ------------------------------------------ | 20260508 04:29:13.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:29:13.220 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:29:13.220 | INFO | Total mem usage: 27.8 GiB | 20260508 04:29:13.220 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:29:13.220 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:29:13.220 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:29:13.220 | INFO | High Water Breakdown: | 20260508 04:29:13.220 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:29:13.220 | INFO | ------------------------------------------ | 20260508 04:29:13.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:29:13.220 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:29:13.220 | INFO | Total mem usage: 15.7 GiB | 20260508 04:29:13.220 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:29:13.220 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:29:13.220 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:29:13.220 | INFO | High Water Breakdown: | 20260508 04:29:13.220 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:29:13.220 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:29:13.220 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:29:13.220 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:29:13.220 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:29:13.220 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:29:13.220 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:29:13.220 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:29:13.220 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:29:13.220 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:29:13.220 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:29:13.220 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:29:13.220 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:29:13.220 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:29:13.220 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:29:13.220 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:29:13.220 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:29:13.220 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:29:13.220 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:29:13.220 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:29:13.220 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:29:13.220 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:29:13.220 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:29:13.220 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:29:13.220 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:29:13.220 | INFO | ------------------------------------------ | 20260508 04:29:13.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:29:13.220 | INFO | Uploaded 36GiB to GPU in 2.47333s (14.5552GiB/s) | 20260508 04:29:13.220 | INFO | Downloaded 34GiB to GPU in 2.42034s (14.0476GiB/s) | 20260508 04:29:13.220 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:29:13.221 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:29:13.221 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:29:13.221 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 1.61e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.21e-02 / 6.07e-03 | 20260508 04:29:13.221 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.46e+00, 5.86e-01 / 2.35e-01 | 20260508 04:29:13.221 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.08e-01 / 1.39e-01 | 20260508 04:29:13.221 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.57e+01, 3.88e+00 / 2.13e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.74e+00 / 1.43e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.40e+00 / 2.55e+00, 4.22e-01 / 2.78e-01 | 20260508 04:29:13.221 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.18e+00, 2.64e-01 / 1.25e-01 | 20260508 04:29:13.221 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.49e-01, 5.22e-02 / 2.64e-02 | 20260508 04:29:13.221 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:29:13.221 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:29:13.221 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:29:13.221 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 24.55%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 2.36%, LockUpload -- 2.51e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 2.51%, LockDownload -- 2.67e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 15.47%, LockCompute -- 1.64e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 04:29:13.221 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 04:29:13.317 | INFO | Starting a round of Baz IO | 20260508 04:29:16.759 | INFO | Populating RTMetrics report | 20260508 04:29:16.759 | INFO | RTMetrics report populated | 20260508 04:29:16.759 | INFO | Opening tmp RTMetrics json file | 20260508 04:29:16.770 | INFO | Serializing RTMetrics to json | 20260508 04:29:16.771 | INFO | Serialization done, swapping tmp file now... | 20260508 04:29:16.771 | INFO | RTMetrics done | 20260508 04:29:16.837 | INFO | Finished a round of Baz IO: Wrote 1.6 GiB in 3.519 seconds | 20260508 04:29:16.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.05% / 0.00% / 58.94% | 20260508 04:29:16.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.53823% / 1.19209e-05% / 4.76837e-05% | 20260508 04:29:16.837 | INFO | Baz IO overall: | 20260508 04:29:16.837 | INFO | BytesWritten : 2346877634023 | 20260508 04:29:16.837 | INFO | NumEvents : 1133044312505 | 20260508 04:29:18.360 | INFO | Analyzing chunk frames = [5242880,5243392) | 20260508 04:29:18.360 | INFO | MemInfo: { | 20260508 04:29:18.360 | INFO | "gpu" : " 17 GB", | 20260508 04:29:18.360 | INFO | "huge" : "", | 20260508 04:29:18.360 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:18.360 | INFO | "malloc" : " 14 GB", | 20260508 04:29:18.360 | INFO | "pinned" : " 29 GB" | 20260508 04:29:18.360 | INFO | } | 20260508 04:29:18.361 | INFO | Perf report for the last 5.14118 seconds: | 20260508 04:29:18.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:18.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:18.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:18.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:18.361 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 04:29:18.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:18.361 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:29:18.361 | INFO | PRE_HQ : 65.6%, 1.6, 55.8ms, 0 | 20260508 04:29:18.361 | INFO | BAZWRITER : 68.0%, 3.2, 113.2ms, 0 | 20260508 04:29:18.361 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:29:18.361 | INFO | Uploaded 36GiB to GPU in 2.57333s (13.9896GiB/s) | 20260508 04:29:18.361 | INFO | Downloaded 38GiB to GPU in 2.69575s (14.0963GiB/s) | 20260508 04:29:23.505 | INFO | Analyzing chunk frames = [5243392,5243904) | 20260508 04:29:23.505 | INFO | MemInfo: { | 20260508 04:29:23.505 | INFO | "gpu" : " 17 GB", | 20260508 04:29:23.505 | INFO | "huge" : "", | 20260508 04:29:23.505 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:23.505 | INFO | "malloc" : " 14 GB", | 20260508 04:29:23.505 | INFO | "pinned" : " 29 GB" | 20260508 04:29:23.505 | INFO | } | 20260508 04:29:23.505 | INFO | Perf report for the last 5.14472 seconds: | 20260508 04:29:23.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:23.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.505 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:29:23.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.505 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:29:23.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:23.506 | INFO | Uploaded 36GiB to GPU in 2.59119s (13.8932GiB/s) | 20260508 04:29:23.506 | INFO | Downloaded 34GiB to GPU in 2.42596s (14.0151GiB/s) | 20260508 04:29:28.639 | INFO | Analyzing chunk frames = [5243904,5244416) | 20260508 04:29:28.639 | INFO | MemInfo: { | 20260508 04:29:28.639 | INFO | "gpu" : " 17 GB", | 20260508 04:29:28.639 | INFO | "huge" : "", | 20260508 04:29:28.639 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:28.639 | INFO | "malloc" : " 14 GB", | 20260508 04:29:28.639 | INFO | "pinned" : " 29 GB" | 20260508 04:29:28.639 | INFO | } | 20260508 04:29:28.639 | INFO | Perf report for the last 5.13368 seconds: | 20260508 04:29:28.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:28.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.639 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 04:29:28.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.639 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:29:28.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:28.640 | INFO | Uploaded 36GiB to GPU in 2.55925s (14.0666GiB/s) | 20260508 04:29:28.640 | INFO | Downloaded 34GiB to GPU in 2.4007s (14.1625GiB/s) | 20260508 04:29:33.778 | INFO | Analyzing chunk frames = [5244416,5244928) | 20260508 04:29:33.778 | INFO | MemInfo: { | 20260508 04:29:33.778 | INFO | "gpu" : " 17 GB", | 20260508 04:29:33.778 | INFO | "huge" : "", | 20260508 04:29:33.778 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:33.778 | INFO | "malloc" : " 14 GB", | 20260508 04:29:33.778 | INFO | "pinned" : " 29 GB" | 20260508 04:29:33.778 | INFO | } | 20260508 04:29:33.778 | INFO | Perf report for the last 5.13907 seconds: | 20260508 04:29:33.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:33.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | ANALYSIS : 67.1%, 2.9, 103.9ms, 0 | 20260508 04:29:33.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:29:33.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:33.778 | INFO | Uploaded 36GiB to GPU in 2.59769s (13.8585GiB/s) | 20260508 04:29:33.778 | INFO | Downloaded 34GiB to GPU in 2.39949s (14.1697GiB/s) | 20260508 04:29:38.918 | INFO | Analyzing chunk frames = [5244928,5245440) | 20260508 04:29:38.918 | INFO | MemInfo: { | 20260508 04:29:38.918 | INFO | "gpu" : " 17 GB", | 20260508 04:29:38.918 | INFO | "huge" : "", | 20260508 04:29:38.918 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:38.918 | INFO | "malloc" : " 14 GB", | 20260508 04:29:38.918 | INFO | "pinned" : " 29 GB" | 20260508 04:29:38.918 | INFO | } | 20260508 04:29:38.918 | INFO | Perf report for the last 5.13999 seconds: | 20260508 04:29:38.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:38.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.918 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 04:29:38.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.918 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:29:38.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:38.919 | INFO | Uploaded 36GiB to GPU in 2.39978s (15.0014GiB/s) | 20260508 04:29:38.919 | INFO | Downloaded 34GiB to GPU in 2.55789s (13.2922GiB/s) | 20260508 04:29:44.056 | INFO | Analyzing chunk frames = [5245440,5245952) | 20260508 04:29:44.056 | INFO | MemInfo: { | 20260508 04:29:44.056 | INFO | "gpu" : " 17 GB", | 20260508 04:29:44.056 | INFO | "huge" : "", | 20260508 04:29:44.056 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:44.057 | INFO | "malloc" : " 14 GB", | 20260508 04:29:44.057 | INFO | "pinned" : " 29 GB" | 20260508 04:29:44.057 | INFO | } | 20260508 04:29:44.057 | INFO | Perf report for the last 5.13876 seconds: | 20260508 04:29:44.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:44.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:29:44.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 04:29:44.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:44.057 | INFO | Uploaded 36GiB to GPU in 2.41543s (14.9042GiB/s) | 20260508 04:29:44.057 | INFO | Downloaded 34GiB to GPU in 2.42078s (14.0451GiB/s) | 20260508 04:29:49.198 | INFO | Analyzing chunk frames = [5245952,5246464) | 20260508 04:29:49.198 | INFO | MemInfo: { | 20260508 04:29:49.198 | INFO | "gpu" : " 17 GB", | 20260508 04:29:49.198 | INFO | "huge" : "", | 20260508 04:29:49.198 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:49.198 | INFO | "malloc" : " 14 GB", | 20260508 04:29:49.198 | INFO | "pinned" : " 29 GB" | 20260508 04:29:49.198 | INFO | } | 20260508 04:29:49.198 | INFO | Perf report for the last 5.14148 seconds: | 20260508 04:29:49.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:49.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.198 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:29:49.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.198 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:29:49.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:49.199 | INFO | Uploaded 36GiB to GPU in 2.42162s (14.8661GiB/s) | 20260508 04:29:49.199 | INFO | Downloaded 34GiB to GPU in 2.41814s (14.0604GiB/s) | 20260508 04:29:54.332 | INFO | Analyzing chunk frames = [5246464,5246976) | 20260508 04:29:54.332 | INFO | MemInfo: { | 20260508 04:29:54.332 | INFO | "gpu" : " 17 GB", | 20260508 04:29:54.332 | INFO | "huge" : "", | 20260508 04:29:54.332 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:54.332 | INFO | "malloc" : " 14 GB", | 20260508 04:29:54.333 | INFO | "pinned" : " 29 GB" | 20260508 04:29:54.333 | INFO | } | 20260508 04:29:54.333 | INFO | Perf report for the last 5.1344 seconds: | 20260508 04:29:54.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:54.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:29:54.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:29:54.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:54.333 | INFO | Uploaded 36GiB to GPU in 2.42992s (14.8153GiB/s) | 20260508 04:29:54.333 | INFO | Downloaded 34GiB to GPU in 2.42521s (14.0194GiB/s) | 20260508 04:29:54.430 | INFO | Starting a round of Baz IO | 20260508 04:29:57.860 | INFO | Populating RTMetrics report | 20260508 04:29:57.861 | INFO | RTMetrics report populated | 20260508 04:29:57.861 | INFO | Opening tmp RTMetrics json file | 20260508 04:29:57.871 | INFO | Serializing RTMetrics to json | 20260508 04:29:57.872 | INFO | Serialization done, swapping tmp file now... | 20260508 04:29:57.874 | INFO | RTMetrics done | 20260508 04:29:57.945 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.515 seconds | 20260508 04:29:57.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.05% / 0.00% / 58.95% | 20260508 04:29:57.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.534817% / 0% / 5.96046e-05% | 20260508 04:29:57.946 | INFO | Baz IO overall: | 20260508 04:29:57.946 | INFO | BytesWritten : 2348591325470 | 20260508 04:29:57.946 | INFO | NumEvents : 1133862703564 | 20260508 04:29:59.473 | INFO | Analyzing chunk frames = [5246976,5247488) | 20260508 04:29:59.473 | INFO | MemInfo: { | 20260508 04:29:59.473 | INFO | "gpu" : " 17 GB", | 20260508 04:29:59.473 | INFO | "huge" : "", | 20260508 04:29:59.473 | INFO | "huge_shared" : " 29 GB", | 20260508 04:29:59.473 | INFO | "malloc" : " 14 GB", | 20260508 04:29:59.473 | INFO | "pinned" : " 29 GB" | 20260508 04:29:59.473 | INFO | } | 20260508 04:29:59.473 | INFO | Perf report for the last 5.14072 seconds: | 20260508 04:29:59.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:29:59.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:59.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:59.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:59.474 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 04:29:59.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:29:59.474 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:29:59.474 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 04:29:59.474 | INFO | BAZWRITER : 67.9%, 3.1, 106.6ms, 0 | 20260508 04:29:59.474 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:29:59.474 | INFO | Uploaded 36GiB to GPU in 2.54542s (14.143GiB/s) | 20260508 04:29:59.474 | INFO | Downloaded 38GiB to GPU in 2.70628s (14.0414GiB/s) | 20260508 04:30:04.617 | INFO | Analyzing chunk frames = [5247488,5248000) | 20260508 04:30:04.617 | INFO | MemInfo: { | 20260508 04:30:04.617 | INFO | "gpu" : " 17 GB", | 20260508 04:30:04.617 | INFO | "huge" : "", | 20260508 04:30:04.617 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:04.617 | INFO | "malloc" : " 14 GB", | 20260508 04:30:04.617 | INFO | "pinned" : " 29 GB" | 20260508 04:30:04.617 | INFO | } | 20260508 04:30:04.617 | INFO | Perf report for the last 5.14331 seconds: | 20260508 04:30:04.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:04.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 04:30:04.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:30:04.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:04.617 | INFO | Uploaded 36GiB to GPU in 2.57604s (13.9749GiB/s) | 20260508 04:30:04.617 | INFO | Downloaded 34GiB to GPU in 2.44055s (13.9313GiB/s) | 20260508 04:30:09.756 | INFO | Analyzing chunk frames = [5248000,5248512) | 20260508 04:30:09.756 | INFO | MemInfo: { | 20260508 04:30:09.756 | INFO | "gpu" : " 17 GB", | 20260508 04:30:09.756 | INFO | "huge" : "", | 20260508 04:30:09.756 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:09.756 | INFO | "malloc" : " 14 GB", | 20260508 04:30:09.756 | INFO | "pinned" : " 29 GB" | 20260508 04:30:09.756 | INFO | } | 20260508 04:30:09.756 | INFO | Perf report for the last 5.13949 seconds: | 20260508 04:30:09.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:09.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.756 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 04:30:09.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.756 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:30:09.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:09.757 | INFO | Uploaded 36GiB to GPU in 2.52836s (14.2385GiB/s) | 20260508 04:30:09.757 | INFO | Downloaded 34GiB to GPU in 2.41394s (14.0849GiB/s) | 20260508 04:30:14.896 | INFO | Analyzing chunk frames = [5248512,5249024) | 20260508 04:30:14.896 | INFO | MemInfo: { | 20260508 04:30:14.896 | INFO | "gpu" : " 17 GB", | 20260508 04:30:14.896 | INFO | "huge" : "", | 20260508 04:30:14.896 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:14.896 | INFO | "malloc" : " 14 GB", | 20260508 04:30:14.896 | INFO | "pinned" : " 29 GB" | 20260508 04:30:14.896 | INFO | } | 20260508 04:30:14.896 | INFO | Perf report for the last 5.13952 seconds: | 20260508 04:30:14.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:14.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 04:30:14.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:30:14.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:14.896 | INFO | Uploaded 36GiB to GPU in 2.53538s (14.1991GiB/s) | 20260508 04:30:14.896 | INFO | Downloaded 34GiB to GPU in 2.50473s (13.5743GiB/s) | 20260508 04:30:20.027 | INFO | Analyzing chunk frames = [5249024,5249536) | 20260508 04:30:20.027 | INFO | MemInfo: { | 20260508 04:30:20.027 | INFO | "gpu" : " 17 GB", | 20260508 04:30:20.027 | INFO | "huge" : "", | 20260508 04:30:20.027 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:20.027 | INFO | "malloc" : " 14 GB", | 20260508 04:30:20.027 | INFO | "pinned" : " 29 GB" | 20260508 04:30:20.027 | INFO | } | 20260508 04:30:20.027 | INFO | Perf report for the last 5.13149 seconds: | 20260508 04:30:20.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:20.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.027 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 04:30:20.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.027 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 04:30:20.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:20.028 | INFO | Uploaded 36GiB to GPU in 2.3038s (15.6264GiB/s) | 20260508 04:30:20.028 | INFO | Downloaded 34GiB to GPU in 2.4829s (13.6936GiB/s) | 20260508 04:30:25.175 | INFO | Analyzing chunk frames = [5249536,5250048) | 20260508 04:30:25.175 | INFO | MemInfo: { | 20260508 04:30:25.175 | INFO | "gpu" : " 17 GB", | 20260508 04:30:25.175 | INFO | "huge" : "", | 20260508 04:30:25.175 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:25.175 | INFO | "malloc" : " 14 GB", | 20260508 04:30:25.175 | INFO | "pinned" : " 29 GB" | 20260508 04:30:25.175 | INFO | } | 20260508 04:30:25.175 | INFO | Perf report for the last 5.14753 seconds: | 20260508 04:30:25.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:25.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 04:30:25.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:30:25.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:25.175 | INFO | Uploaded 36GiB to GPU in 2.44325s (14.7345GiB/s) | 20260508 04:30:25.175 | INFO | Downloaded 34GiB to GPU in 2.40883s (14.1147GiB/s) | 20260508 04:30:30.316 | INFO | Analyzing chunk frames = [5250048,5250560) | 20260508 04:30:30.316 | INFO | MemInfo: { | 20260508 04:30:30.316 | INFO | "gpu" : " 17 GB", | 20260508 04:30:30.316 | INFO | "huge" : "", | 20260508 04:30:30.316 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:30.316 | INFO | "malloc" : " 14 GB", | 20260508 04:30:30.316 | INFO | "pinned" : " 29 GB" | 20260508 04:30:30.316 | INFO | } | 20260508 04:30:30.316 | INFO | Perf report for the last 5.1409 seconds: | 20260508 04:30:30.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:30.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:30:30.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:30:30.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:30.316 | INFO | Uploaded 36GiB to GPU in 2.48114s (14.5095GiB/s) | 20260508 04:30:30.316 | INFO | Downloaded 34GiB to GPU in 2.42166s (14.0399GiB/s) | 20260508 04:30:35.454 | INFO | Analyzing chunk frames = [5250560,5251072) | 20260508 04:30:35.454 | INFO | MemInfo: { | 20260508 04:30:35.454 | INFO | "gpu" : " 17 GB", | 20260508 04:30:35.454 | INFO | "huge" : "", | 20260508 04:30:35.454 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:35.454 | INFO | "malloc" : " 14 GB", | 20260508 04:30:35.454 | INFO | "pinned" : " 29 GB" | 20260508 04:30:35.454 | INFO | } | 20260508 04:30:35.454 | INFO | Perf report for the last 5.13833 seconds: | 20260508 04:30:35.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:35.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.454 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:30:35.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.454 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:30:35.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:35.455 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:30:35.455 | INFO | Total mem usage: 16.7 GiB | 20260508 04:30:35.455 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:30:35.454 | INFO | Uploaded 36GiB to GPU in 2.47997s (14.5163GiB/s) | 20260508 04:30:35.455 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.3%) | 20260508 04:30:35.455 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:30:35.455 | INFO | Downloaded 34GiB to GPU in 2.42359s (14.0288GiB/s) | 20260508 04:30:35.455 | INFO | High Water Breakdown: | 20260508 04:30:35.455 | INFO | 59.7% : 8.64 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:30:35.455 | INFO | 40.2% : 5.81 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:30:35.455 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:30:35.455 | INFO | ------------------------------------------ | 20260508 04:30:35.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:30:35.455 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:30:35.455 | INFO | Total mem usage: 26.5 GiB | 20260508 04:30:35.455 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:30:35.455 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 04:30:35.455 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 04:30:35.455 | INFO | High Water Breakdown: | 20260508 04:30:35.455 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:30:35.455 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:30:35.455 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:30:35.455 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:30:35.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:30:35.455 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:30:35.455 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:30:35.455 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:30:35.455 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:30:35.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:30:35.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:30:35.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:30:35.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:30:35.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:30:35.455 | INFO | ------------------------------------------ | 20260508 04:30:35.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:30:35.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:30:35.455 | INFO | Total mem usage: 27.8 GiB | 20260508 04:30:35.455 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:30:35.455 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:30:35.455 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:30:35.455 | INFO | High Water Breakdown: | 20260508 04:30:35.455 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:30:35.455 | INFO | ------------------------------------------ | 20260508 04:30:35.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:30:35.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:30:35.455 | INFO | Total mem usage: 15.7 GiB | 20260508 04:30:35.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:30:35.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:30:35.455 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:30:35.455 | INFO | High Water Breakdown: | 20260508 04:30:35.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:30:35.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:30:35.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:30:35.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:30:35.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:30:35.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:30:35.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:30:35.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:30:35.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:30:35.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:30:35.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:30:35.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:30:35.455 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:30:35.455 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:30:35.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:30:35.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:30:35.456 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:30:35.456 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:30:35.456 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:30:35.456 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:30:35.456 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:30:35.456 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:30:35.456 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:30:35.456 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:30:35.456 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:30:35.456 | INFO | ------------------------------------------ | 20260508 04:30:35.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:30:35.456 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:30:35.456 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:30:35.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:30:35.456 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.57e+01, 1.42e+01 / 1.47e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.17e-02 / 6.31e-03 | 20260508 04:30:35.456 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.47e+00, 5.92e-01 / 1.96e-01 | 20260508 04:30:35.456 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.24e-01 | 20260508 04:30:35.456 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.55e+01, 3.83e+00 / 1.83e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.77e+00 / 1.38e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.59e+00, 4.17e-01 / 2.68e-01 | 20260508 04:30:35.456 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.22e+00, 2.67e-01 / 1.40e-01 | 20260508 04:30:35.456 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.57e-01, 5.14e-02 / 2.58e-02 | 20260508 04:30:35.456 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:30:35.456 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:30:35.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:30:35.456 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 30.80%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 2.35%, LockUpload -- 2.50e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 2.57%, LockDownload -- 2.73e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 15.67%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:30:35.456 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 3.43e-02 / 0.00e+00 | 20260508 04:30:35.551 | INFO | Starting a round of Baz IO | 20260508 04:30:38.984 | INFO | Populating RTMetrics report | 20260508 04:30:38.984 | INFO | RTMetrics report populated | 20260508 04:30:38.984 | INFO | Opening tmp RTMetrics json file | 20260508 04:30:38.998 | INFO | Serializing RTMetrics to json | 20260508 04:30:38.999 | INFO | Serialization done, swapping tmp file now... | 20260508 04:30:38.999 | INFO | RTMetrics done | 20260508 04:30:39.078 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.526 seconds | 20260508 04:30:39.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.05% / 0.00% / 58.95% | 20260508 04:30:39.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.536621% / 0% / 6.75519e-05% | 20260508 04:30:39.078 | INFO | Baz IO overall: | 20260508 04:30:39.078 | INFO | BytesWritten : 2350306379532 | 20260508 04:30:39.078 | INFO | NumEvents : 1134682343154 | 20260508 04:30:40.590 | INFO | Analyzing chunk frames = [5251072,5251584) | 20260508 04:30:40.590 | INFO | MemInfo: { | 20260508 04:30:40.590 | INFO | "gpu" : " 17 GB", | 20260508 04:30:40.590 | INFO | "huge" : "", | 20260508 04:30:40.590 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:40.590 | INFO | "malloc" : " 14 GB", | 20260508 04:30:40.590 | INFO | "pinned" : " 29 GB" | 20260508 04:30:40.590 | INFO | } | 20260508 04:30:40.590 | INFO | Perf report for the last 5.13573 seconds: | 20260508 04:30:40.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:40.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:40.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:40.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:40.590 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 04:30:40.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:40.590 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:30:40.590 | INFO | PRE_HQ : 65.9%, 1.6, 55.7ms, 0 | 20260508 04:30:40.590 | INFO | BAZWRITER : 68.3%, 3.0, 104.6ms, 0 | 20260508 04:30:40.590 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:30:40.590 | INFO | Uploaded 36GiB to GPU in 2.57329s (13.9899GiB/s) | 20260508 04:30:40.591 | INFO | Downloaded 38GiB to GPU in 2.70718s (14.0368GiB/s) | 20260508 04:30:45.731 | INFO | Analyzing chunk frames = [5251584,5252096) | 20260508 04:30:45.731 | INFO | MemInfo: { | 20260508 04:30:45.731 | INFO | "gpu" : " 17 GB", | 20260508 04:30:45.731 | INFO | "huge" : "", | 20260508 04:30:45.731 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:45.731 | INFO | "malloc" : " 14 GB", | 20260508 04:30:45.731 | INFO | "pinned" : " 29 GB" | 20260508 04:30:45.731 | INFO | } | 20260508 04:30:45.731 | INFO | Perf report for the last 5.1415 seconds: | 20260508 04:30:45.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:45.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.731 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:30:45.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.731 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:30:45.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:45.732 | INFO | Uploaded 36GiB to GPU in 2.59632s (13.8658GiB/s) | 20260508 04:30:45.732 | INFO | Downloaded 34GiB to GPU in 2.43528s (13.9615GiB/s) | 20260508 04:30:50.867 | INFO | Analyzing chunk frames = [5252096,5252608) | 20260508 04:30:50.867 | INFO | MemInfo: { | 20260508 04:30:50.867 | INFO | "gpu" : " 17 GB", | 20260508 04:30:50.867 | INFO | "huge" : "", | 20260508 04:30:50.867 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:50.867 | INFO | "malloc" : " 14 GB", | 20260508 04:30:50.868 | INFO | "pinned" : " 29 GB" | 20260508 04:30:50.868 | INFO | } | 20260508 04:30:50.868 | INFO | Perf report for the last 5.13626 seconds: | 20260508 04:30:50.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:50.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 04:30:50.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 04:30:50.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:50.868 | INFO | Uploaded 36GiB to GPU in 2.55855s (14.0705GiB/s) | 20260508 04:30:50.868 | INFO | Downloaded 34GiB to GPU in 2.40678s (14.1268GiB/s) | 20260508 04:30:56.006 | INFO | Analyzing chunk frames = [5252608,5253120) | 20260508 04:30:56.006 | INFO | MemInfo: { | 20260508 04:30:56.006 | INFO | "gpu" : " 17 GB", | 20260508 04:30:56.006 | INFO | "huge" : "", | 20260508 04:30:56.006 | INFO | "huge_shared" : " 29 GB", | 20260508 04:30:56.006 | INFO | "malloc" : " 14 GB", | 20260508 04:30:56.006 | INFO | "pinned" : " 29 GB" | 20260508 04:30:56.006 | INFO | } | 20260508 04:30:56.006 | INFO | Perf report for the last 5.13869 seconds: | 20260508 04:30:56.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:30:56.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.006 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 04:30:56.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.006 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:30:56.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:30:56.007 | INFO | Uploaded 36GiB to GPU in 2.57876s (13.9602GiB/s) | 20260508 04:30:56.007 | INFO | Downloaded 34GiB to GPU in 2.50274s (13.5851GiB/s) | 20260508 04:31:01.146 | INFO | Analyzing chunk frames = [5253120,5253632) | 20260508 04:31:01.146 | INFO | MemInfo: { | 20260508 04:31:01.146 | INFO | "gpu" : " 17 GB", | 20260508 04:31:01.146 | INFO | "huge" : "", | 20260508 04:31:01.146 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:01.146 | INFO | "malloc" : " 14 GB", | 20260508 04:31:01.146 | INFO | "pinned" : " 29 GB" | 20260508 04:31:01.146 | INFO | } | 20260508 04:31:01.146 | INFO | Perf report for the last 5.13988 seconds: | 20260508 04:31:01.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:01.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.146 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 04:31:01.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.146 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:31:01.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:01.147 | INFO | Uploaded 36GiB to GPU in 2.42418s (14.8504GiB/s) | 20260508 04:31:01.147 | INFO | Downloaded 34GiB to GPU in 2.55436s (13.3106GiB/s) | 20260508 04:31:06.281 | INFO | Analyzing chunk frames = [5253632,5254144) | 20260508 04:31:06.281 | INFO | MemInfo: { | 20260508 04:31:06.281 | INFO | "gpu" : " 17 GB", | 20260508 04:31:06.281 | INFO | "huge" : "", | 20260508 04:31:06.281 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:06.281 | INFO | "malloc" : " 14 GB", | 20260508 04:31:06.281 | INFO | "pinned" : " 29 GB" | 20260508 04:31:06.281 | INFO | } | 20260508 04:31:06.281 | INFO | Perf report for the last 5.13489 seconds: | 20260508 04:31:06.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:06.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.281 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:31:06.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.281 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:31:06.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:06.282 | INFO | Uploaded 36GiB to GPU in 2.44068s (14.75GiB/s) | 20260508 04:31:06.282 | INFO | Downloaded 34GiB to GPU in 2.40519s (14.1361GiB/s) | 20260508 04:31:11.425 | INFO | Analyzing chunk frames = [5254144,5254656) | 20260508 04:31:11.425 | INFO | MemInfo: { | 20260508 04:31:11.425 | INFO | "gpu" : " 17 GB", | 20260508 04:31:11.425 | INFO | "huge" : "", | 20260508 04:31:11.425 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:11.425 | INFO | "malloc" : " 14 GB", | 20260508 04:31:11.425 | INFO | "pinned" : " 29 GB" | 20260508 04:31:11.425 | INFO | } | 20260508 04:31:11.425 | INFO | Perf report for the last 5.14371 seconds: | 20260508 04:31:11.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:11.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:31:11.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:31:11.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:11.425 | INFO | Uploaded 36GiB to GPU in 2.455s (14.6639GiB/s) | 20260508 04:31:11.425 | INFO | Downloaded 34GiB to GPU in 2.42364s (14.0285GiB/s) | 20260508 04:31:16.566 | INFO | Analyzing chunk frames = [5254656,5255168) | 20260508 04:31:16.566 | INFO | MemInfo: { | 20260508 04:31:16.566 | INFO | "gpu" : " 17 GB", | 20260508 04:31:16.566 | INFO | "huge" : "", | 20260508 04:31:16.566 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:16.566 | INFO | "malloc" : " 14 GB", | 20260508 04:31:16.566 | INFO | "pinned" : " 29 GB" | 20260508 04:31:16.566 | INFO | } | 20260508 04:31:16.566 | INFO | Perf report for the last 5.14151 seconds: | 20260508 04:31:16.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:16.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.566 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 04:31:16.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.567 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:31:16.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:16.567 | INFO | Uploaded 36GiB to GPU in 2.57639s (13.9731GiB/s) | 20260508 04:31:16.567 | INFO | Downloaded 34GiB to GPU in 2.45898s (13.8269GiB/s) | 20260508 04:31:16.664 | INFO | Starting a round of Baz IO | 20260508 04:31:20.093 | INFO | Populating RTMetrics report | 20260508 04:31:20.093 | INFO | RTMetrics report populated | 20260508 04:31:20.093 | INFO | Opening tmp RTMetrics json file | 20260508 04:31:20.104 | INFO | Serializing RTMetrics to json | 20260508 04:31:20.106 | INFO | Serialization done, swapping tmp file now... | 20260508 04:31:20.106 | INFO | RTMetrics done | 20260508 04:31:20.209 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.545 seconds | 20260508 04:31:20.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.05% / 0.00% / 58.95% | 20260508 04:31:20.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.535564% / 7.94729e-06% / 5.96046e-05% | 20260508 04:31:20.209 | INFO | Baz IO overall: | 20260508 04:31:20.209 | INFO | BytesWritten : 2352021100526 | 20260508 04:31:20.209 | INFO | NumEvents : 1135502134881 | 20260508 04:31:21.709 | INFO | Analyzing chunk frames = [5255168,5255680) | 20260508 04:31:21.709 | INFO | MemInfo: { | 20260508 04:31:21.709 | INFO | "gpu" : " 17 GB", | 20260508 04:31:21.709 | INFO | "huge" : "", | 20260508 04:31:21.709 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:21.709 | INFO | "malloc" : " 14 GB", | 20260508 04:31:21.709 | INFO | "pinned" : " 29 GB" | 20260508 04:31:21.709 | INFO | } | 20260508 04:31:21.709 | INFO | Perf report for the last 5.14275 seconds: | 20260508 04:31:21.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:21.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:21.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:21.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:21.709 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 04:31:21.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:21.709 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:31:21.709 | INFO | PRE_HQ : 65.6%, 1.6, 55.6ms, 0 | 20260508 04:31:21.709 | INFO | BAZWRITER : 68.4%, 2.9, 102.8ms, 0 | 20260508 04:31:21.709 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 04:31:21.710 | INFO | Uploaded 36GiB to GPU in 2.57344s (13.9891GiB/s) | 20260508 04:31:21.710 | INFO | Downloaded 38GiB to GPU in 2.70406s (14.053GiB/s) | 20260508 04:31:26.847 | INFO | Analyzing chunk frames = [5255680,5256192) | 20260508 04:31:26.847 | INFO | MemInfo: { | 20260508 04:31:26.847 | INFO | "gpu" : " 17 GB", | 20260508 04:31:26.847 | INFO | "huge" : "", | 20260508 04:31:26.847 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:26.847 | INFO | "malloc" : " 14 GB", | 20260508 04:31:26.847 | INFO | "pinned" : " 29 GB" | 20260508 04:31:26.847 | INFO | } | 20260508 04:31:26.847 | INFO | Perf report for the last 5.1375 seconds: | 20260508 04:31:26.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:26.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 04:31:26.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 04:31:26.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:26.847 | INFO | Uploaded 36GiB to GPU in 2.52291s (14.2692GiB/s) | 20260508 04:31:26.847 | INFO | Downloaded 34GiB to GPU in 2.43876s (13.9415GiB/s) | 20260508 04:31:31.986 | INFO | Analyzing chunk frames = [5256192,5256704) | 20260508 04:31:31.986 | INFO | MemInfo: { | 20260508 04:31:31.986 | INFO | "gpu" : " 17 GB", | 20260508 04:31:31.986 | INFO | "huge" : "", | 20260508 04:31:31.986 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:31.986 | INFO | "malloc" : " 14 GB", | 20260508 04:31:31.986 | INFO | "pinned" : " 29 GB" | 20260508 04:31:31.986 | INFO | } | 20260508 04:31:31.986 | INFO | Perf report for the last 5.13909 seconds: | 20260508 04:31:31.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:31.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:31:31.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:31:31.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:31.986 | INFO | Uploaded 36GiB to GPU in 2.54967s (14.1195GiB/s) | 20260508 04:31:31.986 | INFO | Downloaded 34GiB to GPU in 2.40946s (14.111GiB/s) | 20260508 04:31:37.127 | INFO | Analyzing chunk frames = [5256704,5257216) | 20260508 04:31:37.127 | INFO | MemInfo: { | 20260508 04:31:37.127 | INFO | "gpu" : " 17 GB", | 20260508 04:31:37.127 | INFO | "huge" : "", | 20260508 04:31:37.127 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:37.127 | INFO | "malloc" : " 14 GB", | 20260508 04:31:37.127 | INFO | "pinned" : " 29 GB" | 20260508 04:31:37.127 | INFO | } | 20260508 04:31:37.127 | INFO | Perf report for the last 5.14127 seconds: | 20260508 04:31:37.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:37.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 04:31:37.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:31:37.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:37.127 | INFO | Uploaded 36GiB to GPU in 2.59757s (13.8591GiB/s) | 20260508 04:31:37.127 | INFO | Downloaded 34GiB to GPU in 2.41307s (14.09GiB/s) | 20260508 04:31:42.261 | INFO | Analyzing chunk frames = [5257216,5257728) | 20260508 04:31:42.261 | INFO | MemInfo: { | 20260508 04:31:42.261 | INFO | "gpu" : " 17 GB", | 20260508 04:31:42.261 | INFO | "huge" : "", | 20260508 04:31:42.262 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:42.262 | INFO | "malloc" : " 14 GB", | 20260508 04:31:42.262 | INFO | "pinned" : " 29 GB" | 20260508 04:31:42.262 | INFO | } | 20260508 04:31:42.262 | INFO | Perf report for the last 5.13463 seconds: | 20260508 04:31:42.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:42.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:31:42.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:31:42.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:42.262 | INFO | Uploaded 36GiB to GPU in 2.36315s (15.2339GiB/s) | 20260508 04:31:42.263 | INFO | Downloaded 34GiB to GPU in 2.56578s (13.2514GiB/s) | 20260508 04:31:47.402 | INFO | Analyzing chunk frames = [5257728,5258240) | 20260508 04:31:47.402 | INFO | MemInfo: { | 20260508 04:31:47.402 | INFO | "gpu" : " 17 GB", | 20260508 04:31:47.402 | INFO | "huge" : "", | 20260508 04:31:47.402 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:47.402 | INFO | "malloc" : " 14 GB", | 20260508 04:31:47.402 | INFO | "pinned" : " 29 GB" | 20260508 04:31:47.402 | INFO | } | 20260508 04:31:47.403 | INFO | Perf report for the last 5.14078 seconds: | 20260508 04:31:47.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:47.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 04:31:47.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:31:47.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:47.403 | INFO | Uploaded 36GiB to GPU in 2.45139s (14.6855GiB/s) | 20260508 04:31:47.403 | INFO | Downloaded 34GiB to GPU in 2.41702s (14.0669GiB/s) | 20260508 04:31:52.537 | INFO | Analyzing chunk frames = [5258240,5258752) | 20260508 04:31:52.537 | INFO | MemInfo: { | 20260508 04:31:52.537 | INFO | "gpu" : " 17 GB", | 20260508 04:31:52.537 | INFO | "huge" : "", | 20260508 04:31:52.537 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:52.537 | INFO | "malloc" : " 14 GB", | 20260508 04:31:52.537 | INFO | "pinned" : " 29 GB" | 20260508 04:31:52.537 | INFO | } | 20260508 04:31:52.537 | INFO | Perf report for the last 5.13451 seconds: | 20260508 04:31:52.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:52.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:31:52.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:31:52.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:52.537 | INFO | Uploaded 36GiB to GPU in 2.44016s (14.7531GiB/s) | 20260508 04:31:52.537 | INFO | Downloaded 34GiB to GPU in 2.41875s (14.0568GiB/s) | 20260508 04:31:57.681 | INFO | Analyzing chunk frames = [5258752,5259264) | 20260508 04:31:57.681 | INFO | MemInfo: { | 20260508 04:31:57.681 | INFO | "gpu" : " 17 GB", | 20260508 04:31:57.681 | INFO | "huge" : "", | 20260508 04:31:57.681 | INFO | "huge_shared" : " 29 GB", | 20260508 04:31:57.681 | INFO | "malloc" : " 14 GB", | 20260508 04:31:57.681 | INFO | "pinned" : " 29 GB" | 20260508 04:31:57.681 | INFO | } | 20260508 04:31:57.681 | INFO | Perf report for the last 5.14411 seconds: | 20260508 04:31:57.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:31:57.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.681 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:31:57.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.681 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:31:57.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:31:57.682 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:31:57.682 | INFO | Total mem usage: 16.7 GiB | 20260508 04:31:57.682 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:31:57.682 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:31:57.682 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:31:57.682 | INFO | High Water Breakdown: | 20260508 04:31:57.682 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:31:57.682 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:31:57.682 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:31:57.682 | INFO | ------------------------------------------ | 20260508 04:31:57.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:31:57.682 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:31:57.682 | INFO | Total mem usage: 26.5 GiB | 20260508 04:31:57.682 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:31:57.682 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:31:57.682 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:31:57.682 | INFO | High Water Breakdown: | 20260508 04:31:57.682 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:31:57.682 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:31:57.682 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:31:57.682 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:31:57.682 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:31:57.682 | INFO | Uploaded 36GiB to GPU in 2.47512s (14.5447GiB/s) | 20260508 04:31:57.682 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:31:57.682 | INFO | Downloaded 34GiB to GPU in 2.42538s (14.0184GiB/s) | 20260508 04:31:57.682 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:31:57.682 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:31:57.682 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:31:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:31:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:31:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:31:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:31:57.682 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:31:57.682 | INFO | ------------------------------------------ | 20260508 04:31:57.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:31:57.682 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:31:57.682 | INFO | Total mem usage: 27.8 GiB | 20260508 04:31:57.682 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:31:57.682 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:31:57.682 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:31:57.682 | INFO | High Water Breakdown: | 20260508 04:31:57.682 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:31:57.682 | INFO | ------------------------------------------ | 20260508 04:31:57.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:31:57.682 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:31:57.682 | INFO | Total mem usage: 15.7 GiB | 20260508 04:31:57.682 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:31:57.683 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:31:57.683 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:31:57.683 | INFO | High Water Breakdown: | 20260508 04:31:57.683 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:31:57.683 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:31:57.683 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:31:57.683 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:31:57.683 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:31:57.683 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:31:57.683 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:31:57.683 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:31:57.683 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:31:57.683 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:31:57.683 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:31:57.683 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:31:57.683 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:31:57.683 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:31:57.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:31:57.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:31:57.683 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:31:57.683 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:31:57.683 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:31:57.683 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:31:57.683 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:31:57.683 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:31:57.683 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:31:57.683 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:31:57.683 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:31:57.683 | INFO | ------------------------------------------ | 20260508 04:31:57.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:31:57.683 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:31:57.683 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:31:57.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:31:57.683 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.78e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.55e-02 / 5.49e-03 | 20260508 04:31:57.683 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.49e+00, 5.90e-01 / 1.95e-01 | 20260508 04:31:57.683 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.46e+00, 2.08e-01 / 1.23e-01 | 20260508 04:31:57.683 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.54e+01, 3.82e+00 / 2.04e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.85e+01 / 2.02e+01, 1.75e+00 / 1.43e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.65e+00, 4.20e-01 / 2.32e-01 | 20260508 04:31:57.683 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.24e+00, 2.66e-01 / 1.15e-01 | 20260508 04:31:57.683 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.60e-01, 5.07e-02 / 2.42e-02 | 20260508 04:31:57.683 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:31:57.683 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:31:57.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:31:57.683 | INFO | PerfReport: Full Execution -- 1.07e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 24.55%, Upload -- 2.62e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 24.25%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 30.69%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 2.49%, LockUpload -- 2.65e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 2.46%, LockDownload -- 2.62e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 15.54%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:31:57.683 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 04:31:57.779 | INFO | Starting a round of Baz IO | 20260508 04:32:01.210 | INFO | Populating RTMetrics report | 20260508 04:32:01.210 | INFO | RTMetrics report populated | 20260508 04:32:01.210 | INFO | Opening tmp RTMetrics json file | 20260508 04:32:01.230 | INFO | Serializing RTMetrics to json | 20260508 04:32:01.236 | INFO | Serialization done, swapping tmp file now... | 20260508 04:32:01.237 | INFO | RTMetrics done | 20260508 04:32:01.292 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.512 seconds | 20260508 04:32:01.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.05% / 0.00% / 58.95% | 20260508 04:32:01.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.534924% / 3.97364e-06% / 8.74201e-05% | 20260508 04:32:01.292 | INFO | Baz IO overall: | 20260508 04:32:01.292 | INFO | BytesWritten : 2353736138497 | 20260508 04:32:01.292 | INFO | NumEvents : 1136322299220 | 20260508 04:32:02.821 | INFO | Analyzing chunk frames = [5259264,5259776) | 20260508 04:32:02.821 | INFO | MemInfo: { | 20260508 04:32:02.821 | INFO | "gpu" : " 17 GB", | 20260508 04:32:02.821 | INFO | "huge" : "", | 20260508 04:32:02.821 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:02.821 | INFO | "malloc" : " 14 GB", | 20260508 04:32:02.821 | INFO | "pinned" : " 29 GB" | 20260508 04:32:02.821 | INFO | } | 20260508 04:32:02.821 | INFO | Perf report for the last 5.14023 seconds: | 20260508 04:32:02.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:02.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:02.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:02.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:02.822 | INFO | ANALYSIS : 68.0%, 3.0, 105.3ms, 0 | 20260508 04:32:02.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:02.822 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260508 04:32:02.822 | INFO | PRE_HQ : 66.5%, 1.7, 59.9ms, 0 | 20260508 04:32:02.822 | INFO | BAZWRITER : 67.6%, 3.2, 111.5ms, 0 | 20260508 04:32:02.822 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 04:32:02.822 | INFO | Uploaded 36GiB to GPU in 2.60203s (13.8353GiB/s) | 20260508 04:32:02.822 | INFO | Downloaded 38GiB to GPU in 2.69777s (14.0857GiB/s) | 20260508 04:32:07.960 | INFO | Analyzing chunk frames = [5259776,5260288) | 20260508 04:32:07.960 | INFO | MemInfo: { | 20260508 04:32:07.960 | INFO | "gpu" : " 17 GB", | 20260508 04:32:07.960 | INFO | "huge" : "", | 20260508 04:32:07.960 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:07.960 | INFO | "malloc" : " 14 GB", | 20260508 04:32:07.960 | INFO | "pinned" : " 29 GB" | 20260508 04:32:07.960 | INFO | } | 20260508 04:32:07.960 | INFO | Perf report for the last 5.13858 seconds: | 20260508 04:32:07.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:07.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:32:07.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:32:07.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:07.960 | INFO | Uploaded 36GiB to GPU in 2.56294s (14.0464GiB/s) | 20260508 04:32:07.960 | INFO | Downloaded 34GiB to GPU in 2.44499s (13.906GiB/s) | 20260508 04:32:13.093 | INFO | Analyzing chunk frames = [5260288,5260800) | 20260508 04:32:13.093 | INFO | MemInfo: { | 20260508 04:32:13.093 | INFO | "gpu" : " 17 GB", | 20260508 04:32:13.093 | INFO | "huge" : "", | 20260508 04:32:13.093 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:13.093 | INFO | "malloc" : " 14 GB", | 20260508 04:32:13.094 | INFO | "pinned" : " 29 GB" | 20260508 04:32:13.094 | INFO | } | 20260508 04:32:13.094 | INFO | Perf report for the last 5.13359 seconds: | 20260508 04:32:13.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:13.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 04:32:13.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:32:13.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:13.094 | INFO | Uploaded 36GiB to GPU in 2.52945s (14.2323GiB/s) | 20260508 04:32:13.094 | INFO | Downloaded 34GiB to GPU in 2.42574s (14.0163GiB/s) | 20260508 04:32:18.235 | INFO | Analyzing chunk frames = [5260800,5261312) | 20260508 04:32:18.235 | INFO | MemInfo: { | 20260508 04:32:18.235 | INFO | "gpu" : " 17 GB", | 20260508 04:32:18.235 | INFO | "huge" : "", | 20260508 04:32:18.235 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:18.235 | INFO | "malloc" : " 14 GB", | 20260508 04:32:18.235 | INFO | "pinned" : " 29 GB" | 20260508 04:32:18.235 | INFO | } | 20260508 04:32:18.235 | INFO | Perf report for the last 5.14184 seconds: | 20260508 04:32:18.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:18.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.236 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 04:32:18.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.236 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:32:18.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:18.236 | INFO | Uploaded 36GiB to GPU in 2.54189s (14.1627GiB/s) | 20260508 04:32:18.236 | INFO | Downloaded 34GiB to GPU in 2.51476s (13.5202GiB/s) | 20260508 04:32:23.379 | INFO | Analyzing chunk frames = [5261312,5261824) | 20260508 04:32:23.379 | INFO | MemInfo: { | 20260508 04:32:23.379 | INFO | "gpu" : " 17 GB", | 20260508 04:32:23.379 | INFO | "huge" : "", | 20260508 04:32:23.379 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:23.380 | INFO | "malloc" : " 14 GB", | 20260508 04:32:23.380 | INFO | "pinned" : " 29 GB" | 20260508 04:32:23.380 | INFO | } | 20260508 04:32:23.380 | INFO | Perf report for the last 5.14417 seconds: | 20260508 04:32:23.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:23.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:32:23.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:32:23.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:23.380 | INFO | Uploaded 36GiB to GPU in 2.35215s (15.3052GiB/s) | 20260508 04:32:23.380 | INFO | Downloaded 34GiB to GPU in 2.55056s (13.3304GiB/s) | 20260508 04:32:28.524 | INFO | Analyzing chunk frames = [5261824,5262336) | 20260508 04:32:28.524 | INFO | MemInfo: { | 20260508 04:32:28.524 | INFO | "gpu" : " 17 GB", | 20260508 04:32:28.524 | INFO | "huge" : "", | 20260508 04:32:28.524 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:28.524 | INFO | "malloc" : " 14 GB", | 20260508 04:32:28.524 | INFO | "pinned" : " 29 GB" | 20260508 04:32:28.525 | INFO | } | 20260508 04:32:28.525 | INFO | Perf report for the last 5.14497 seconds: | 20260508 04:32:28.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:28.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:32:28.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:32:28.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:28.525 | INFO | Uploaded 36GiB to GPU in 2.3864s (15.0855GiB/s) | 20260508 04:32:28.525 | INFO | Downloaded 34GiB to GPU in 2.44549s (13.9031GiB/s) | 20260508 04:32:33.659 | INFO | Analyzing chunk frames = [5262336,5262848) | 20260508 04:32:33.659 | INFO | MemInfo: { | 20260508 04:32:33.659 | INFO | "gpu" : " 17 GB", | 20260508 04:32:33.659 | INFO | "huge" : "", | 20260508 04:32:33.660 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:33.660 | INFO | "malloc" : " 14 GB", | 20260508 04:32:33.660 | INFO | "pinned" : " 29 GB" | 20260508 04:32:33.660 | INFO | } | 20260508 04:32:33.660 | INFO | Perf report for the last 5.1351 seconds: | 20260508 04:32:33.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:33.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.660 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 04:32:33.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.660 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 04:32:33.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:33.661 | INFO | Uploaded 36GiB to GPU in 2.4308s (14.81GiB/s) | 20260508 04:32:33.661 | INFO | Downloaded 34GiB to GPU in 2.45243s (13.8638GiB/s) | 20260508 04:32:38.796 | INFO | Analyzing chunk frames = [5262848,5263360) | 20260508 04:32:38.796 | INFO | MemInfo: { | 20260508 04:32:38.796 | INFO | "gpu" : " 17 GB", | 20260508 04:32:38.796 | INFO | "huge" : "", | 20260508 04:32:38.796 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:38.796 | INFO | "malloc" : " 14 GB", | 20260508 04:32:38.796 | INFO | "pinned" : " 29 GB" | 20260508 04:32:38.796 | INFO | } | 20260508 04:32:38.796 | INFO | Perf report for the last 5.13653 seconds: | 20260508 04:32:38.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:38.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.796 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:32:38.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.796 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 04:32:38.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:38.797 | INFO | Uploaded 36GiB to GPU in 2.42787s (14.8278GiB/s) | 20260508 04:32:38.797 | INFO | Downloaded 34GiB to GPU in 2.46536s (13.7911GiB/s) | 20260508 04:32:38.909 | INFO | Starting a round of Baz IO | 20260508 04:32:42.333 | INFO | Populating RTMetrics report | 20260508 04:32:42.333 | INFO | RTMetrics report populated | 20260508 04:32:42.333 | INFO | Opening tmp RTMetrics json file | 20260508 04:32:42.335 | INFO | Serializing RTMetrics to json | 20260508 04:32:42.339 | INFO | Serialization done, swapping tmp file now... | 20260508 04:32:42.341 | INFO | RTMetrics done | 20260508 04:32:42.419 | INFO | Finished a round of Baz IO: Wrote 1.596 GiB in 3.51 seconds | 20260508 04:32:42.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.04% / 0.00% / 58.95% | 20260508 04:32:42.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.532134% / 3.97364e-06% / 5.16574e-05% | 20260508 04:32:42.419 | INFO | Baz IO overall: | 20260508 04:32:42.419 | INFO | BytesWritten : 2355450242045 | 20260508 04:32:42.419 | INFO | NumEvents : 1137141672529 | 20260508 04:32:43.935 | INFO | Analyzing chunk frames = [5263360,5263872) | 20260508 04:32:43.935 | INFO | MemInfo: { | 20260508 04:32:43.935 | INFO | "gpu" : " 17 GB", | 20260508 04:32:43.935 | INFO | "huge" : "", | 20260508 04:32:43.935 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:43.935 | INFO | "malloc" : " 14 GB", | 20260508 04:32:43.935 | INFO | "pinned" : " 29 GB" | 20260508 04:32:43.935 | INFO | } | 20260508 04:32:43.935 | INFO | Perf report for the last 5.13886 seconds: | 20260508 04:32:43.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:43.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:43.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:43.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:43.935 | INFO | ANALYSIS : 68.5%, 2.9, 105.0ms, 0 | 20260508 04:32:43.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:43.935 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 04:32:43.935 | INFO | PRE_HQ : 67.0%, 1.8, 61.8ms, 0 | 20260508 04:32:43.935 | INFO | BAZWRITER : 67.9%, 3.2, 112.4ms, 0 | 20260508 04:32:43.935 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 04:32:43.935 | INFO | Uploaded 36GiB to GPU in 2.49721s (14.4161GiB/s) | 20260508 04:32:43.935 | INFO | Downloaded 38GiB to GPU in 2.71028s (14.0207GiB/s) | 20260508 04:32:49.075 | INFO | Analyzing chunk frames = [5263872,5264384) | 20260508 04:32:49.075 | INFO | MemInfo: { | 20260508 04:32:49.075 | INFO | "gpu" : " 17 GB", | 20260508 04:32:49.075 | INFO | "huge" : "", | 20260508 04:32:49.075 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:49.075 | INFO | "malloc" : " 14 GB", | 20260508 04:32:49.075 | INFO | "pinned" : " 29 GB" | 20260508 04:32:49.075 | INFO | } | 20260508 04:32:49.075 | INFO | Perf report for the last 5.14001 seconds: | 20260508 04:32:49.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:49.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.075 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:32:49.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.075 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:32:49.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:49.076 | INFO | Uploaded 36GiB to GPU in 2.56812s (14.018GiB/s) | 20260508 04:32:49.076 | INFO | Downloaded 34GiB to GPU in 2.43474s (13.9645GiB/s) | 20260508 04:32:54.212 | INFO | Analyzing chunk frames = [5264384,5264896) | 20260508 04:32:54.212 | INFO | MemInfo: { | 20260508 04:32:54.212 | INFO | "gpu" : " 17 GB", | 20260508 04:32:54.212 | INFO | "huge" : "", | 20260508 04:32:54.212 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:54.212 | INFO | "malloc" : " 14 GB", | 20260508 04:32:54.212 | INFO | "pinned" : " 29 GB" | 20260508 04:32:54.212 | INFO | } | 20260508 04:32:54.212 | INFO | Perf report for the last 5.13662 seconds: | 20260508 04:32:54.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:54.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 04:32:54.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:32:54.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:54.212 | INFO | Uploaded 36GiB to GPU in 2.5531s (14.1005GiB/s) | 20260508 04:32:54.212 | INFO | Downloaded 34GiB to GPU in 2.50342s (13.5814GiB/s) | 20260508 04:32:59.350 | INFO | Analyzing chunk frames = [5264896,5265408) | 20260508 04:32:59.350 | INFO | MemInfo: { | 20260508 04:32:59.350 | INFO | "gpu" : " 17 GB", | 20260508 04:32:59.350 | INFO | "huge" : "", | 20260508 04:32:59.350 | INFO | "huge_shared" : " 29 GB", | 20260508 04:32:59.350 | INFO | "malloc" : " 14 GB", | 20260508 04:32:59.350 | INFO | "pinned" : " 29 GB" | 20260508 04:32:59.350 | INFO | } | 20260508 04:32:59.350 | INFO | Perf report for the last 5.13849 seconds: | 20260508 04:32:59.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:32:59.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.350 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 04:32:59.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.350 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:32:59.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:32:59.351 | INFO | Uploaded 36GiB to GPU in 2.58037s (13.9515GiB/s) | 20260508 04:32:59.351 | INFO | Downloaded 34GiB to GPU in 2.41145s (14.0994GiB/s) | 20260508 04:33:04.488 | INFO | Analyzing chunk frames = [5265408,5265920) | 20260508 04:33:04.488 | INFO | MemInfo: { | 20260508 04:33:04.488 | INFO | "gpu" : " 17 GB", | 20260508 04:33:04.488 | INFO | "huge" : "", | 20260508 04:33:04.488 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:04.488 | INFO | "malloc" : " 14 GB", | 20260508 04:33:04.488 | INFO | "pinned" : " 29 GB" | 20260508 04:33:04.488 | INFO | } | 20260508 04:33:04.488 | INFO | Perf report for the last 5.1377 seconds: | 20260508 04:33:04.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:04.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 04:33:04.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 04:33:04.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:04.488 | INFO | Uploaded 36GiB to GPU in 2.40197s (14.9877GiB/s) | 20260508 04:33:04.488 | INFO | Downloaded 34GiB to GPU in 2.56967s (13.2313GiB/s) | 20260508 04:33:09.631 | INFO | Analyzing chunk frames = [5265920,5266432) | 20260508 04:33:09.631 | INFO | MemInfo: { | 20260508 04:33:09.631 | INFO | "gpu" : " 17 GB", | 20260508 04:33:09.631 | INFO | "huge" : "", | 20260508 04:33:09.631 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:09.631 | INFO | "malloc" : " 14 GB", | 20260508 04:33:09.631 | INFO | "pinned" : " 29 GB" | 20260508 04:33:09.631 | INFO | } | 20260508 04:33:09.631 | INFO | Perf report for the last 5.14312 seconds: | 20260508 04:33:09.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:09.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.631 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:33:09.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.631 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:33:09.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:09.632 | INFO | Uploaded 36GiB to GPU in 2.42961s (14.8172GiB/s) | 20260508 04:33:09.632 | INFO | Downloaded 34GiB to GPU in 2.42276s (14.0336GiB/s) | 20260508 04:33:14.773 | INFO | Analyzing chunk frames = [5266432,5266944) | 20260508 04:33:14.773 | INFO | MemInfo: { | 20260508 04:33:14.773 | INFO | "gpu" : " 17 GB", | 20260508 04:33:14.773 | INFO | "huge" : "", | 20260508 04:33:14.773 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:14.773 | INFO | "malloc" : " 14 GB", | 20260508 04:33:14.773 | INFO | "pinned" : " 29 GB" | 20260508 04:33:14.774 | INFO | } | 20260508 04:33:14.774 | INFO | Perf report for the last 5.1424 seconds: | 20260508 04:33:14.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:14.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:33:14.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:33:14.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:14.774 | INFO | Uploaded 36GiB to GPU in 2.41134s (14.9295GiB/s) | 20260508 04:33:14.774 | INFO | Downloaded 34GiB to GPU in 2.41914s (14.0546GiB/s) | 20260508 04:33:19.908 | INFO | Analyzing chunk frames = [5266944,5267456) | 20260508 04:33:19.908 | INFO | MemInfo: { | 20260508 04:33:19.908 | INFO | "gpu" : " 17 GB", | 20260508 04:33:19.908 | INFO | "huge" : "", | 20260508 04:33:19.908 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:19.908 | INFO | "malloc" : " 14 GB", | 20260508 04:33:19.908 | INFO | "pinned" : " 29 GB" | 20260508 04:33:19.908 | INFO | } | 20260508 04:33:19.909 | INFO | Perf report for the last 5.13491 seconds: | 20260508 04:33:19.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:19.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:33:19.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 04:33:19.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:19.909 | INFO | Uploaded 36GiB to GPU in 2.44263s (14.7382GiB/s) | 20260508 04:33:19.909 | INFO | Downloaded 34GiB to GPU in 2.43286s (13.9753GiB/s) | 20260508 04:33:19.909 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:33:19.909 | INFO | Total mem usage: 16.7 GiB | 20260508 04:33:19.909 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:33:19.909 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.4%) | 20260508 04:33:19.909 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:33:19.909 | INFO | High Water Breakdown: | 20260508 04:33:19.909 | INFO | 59.7% : 8.63 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:33:19.909 | INFO | 40.2% : 5.81 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:33:19.909 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:33:19.909 | INFO | ------------------------------------------ | 20260508 04:33:19.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:33:19.909 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:33:19.909 | INFO | Total mem usage: 26.5 GiB | 20260508 04:33:19.909 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:33:19.909 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:33:19.909 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:33:19.909 | INFO | High Water Breakdown: | 20260508 04:33:19.909 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:33:19.909 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:33:19.909 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:33:19.909 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:33:19.909 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:33:19.909 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:33:19.909 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:33:19.909 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:33:19.909 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:33:19.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:33:19.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:33:19.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:33:19.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:33:19.909 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:33:19.909 | INFO | ------------------------------------------ | 20260508 04:33:19.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:33:19.909 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:33:19.909 | INFO | Total mem usage: 27.8 GiB | 20260508 04:33:19.909 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:33:19.909 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:33:19.909 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:33:19.909 | INFO | High Water Breakdown: | 20260508 04:33:19.909 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:33:19.909 | INFO | ------------------------------------------ | 20260508 04:33:19.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:33:19.909 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:33:19.909 | INFO | Total mem usage: 15.7 GiB | 20260508 04:33:19.909 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:33:19.909 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:33:19.909 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:33:19.909 | INFO | High Water Breakdown: | 20260508 04:33:19.909 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:33:19.909 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:33:19.909 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:33:19.909 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:33:19.909 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:33:19.909 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:33:19.909 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:33:19.909 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:33:19.909 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:33:19.909 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:33:19.909 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:33:19.909 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:33:19.909 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:33:19.909 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:33:19.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:33:19.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:33:19.909 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:33:19.909 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:33:19.909 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:33:19.909 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:33:19.909 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:33:19.909 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:33:19.909 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:33:19.909 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:33:19.909 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:33:19.909 | INFO | ------------------------------------------ | 20260508 04:33:19.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:33:19.909 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:33:19.909 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:33:19.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:33:19.909 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.61e+01, 1.41e+01 / 1.71e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.29e-02 / 5.48e-03 | 20260508 04:33:19.909 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.89e-01 / 2.45e-01 | 20260508 04:33:19.909 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.09e-01 / 1.21e-01 | 20260508 04:33:19.909 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.60e+01, 3.87e+00 / 2.09e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.73e+00 / 1.32e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.61e+00, 4.18e-01 / 2.56e-01 | 20260508 04:33:19.909 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.22e+00, 2.61e-01 / 1.37e-01 | 20260508 04:33:19.909 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.61e-01, 5.13e-02 / 2.84e-02 | 20260508 04:33:19.909 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:33:19.909 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:33:19.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:33:19.909 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 24.46%, Download -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 04:33:19.909 | INFO | PerfReport: 2.80%, LockDownload -- 2.98e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 04:33:19.910 | INFO | PerfReport: 15.47%, LockCompute -- 1.64e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 04:33:19.910 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.44e-02 / 0.00e+00 | 20260508 04:33:20.006 | INFO | Starting a round of Baz IO | 20260508 04:33:23.426 | INFO | Populating RTMetrics report | 20260508 04:33:23.426 | INFO | RTMetrics report populated | 20260508 04:33:23.426 | INFO | Opening tmp RTMetrics json file | 20260508 04:33:23.447 | INFO | Serializing RTMetrics to json | 20260508 04:33:23.449 | INFO | Serialization done, swapping tmp file now... | 20260508 04:33:23.449 | INFO | RTMetrics done | 20260508 04:33:23.512 | INFO | Finished a round of Baz IO: Wrote 1.595 GiB in 3.506 seconds | 20260508 04:33:23.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.04% / 0.00% / 58.96% | 20260508 04:33:23.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.530493% / 0% / 6.75519e-05% | 20260508 04:33:23.512 | INFO | Baz IO overall: | 20260508 04:33:23.512 | INFO | BytesWritten : 2357162630338 | 20260508 04:33:23.512 | INFO | NumEvents : 1137960080086 | 20260508 04:33:25.047 | INFO | Analyzing chunk frames = [5267456,5267968) | 20260508 04:33:25.047 | INFO | MemInfo: { | 20260508 04:33:25.047 | INFO | "gpu" : " 17 GB", | 20260508 04:33:25.047 | INFO | "huge" : "", | 20260508 04:33:25.047 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:25.047 | INFO | "malloc" : " 14 GB", | 20260508 04:33:25.047 | INFO | "pinned" : " 29 GB" | 20260508 04:33:25.047 | INFO | } | 20260508 04:33:25.047 | INFO | Perf report for the last 5.1388 seconds: | 20260508 04:33:25.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:25.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:25.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:25.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:25.047 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 04:33:25.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:25.048 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 04:33:25.048 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 04:33:25.048 | INFO | BAZWRITER : 67.8%, 3.1, 108.7ms, 0 | 20260508 04:33:25.048 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:33:25.048 | INFO | Uploaded 36GiB to GPU in 2.52826s (14.239GiB/s) | 20260508 04:33:25.048 | INFO | Downloaded 38GiB to GPU in 2.69673s (14.0911GiB/s) | 20260508 04:33:30.190 | INFO | Analyzing chunk frames = [5267968,5268480) | 20260508 04:33:30.190 | INFO | MemInfo: { | 20260508 04:33:30.190 | INFO | "gpu" : " 17 GB", | 20260508 04:33:30.190 | INFO | "huge" : "", | 20260508 04:33:30.190 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:30.190 | INFO | "malloc" : " 14 GB", | 20260508 04:33:30.190 | INFO | "pinned" : " 29 GB" | 20260508 04:33:30.190 | INFO | } | 20260508 04:33:30.190 | INFO | Perf report for the last 5.1425 seconds: | 20260508 04:33:30.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:30.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:33:30.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 04:33:30.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:30.190 | INFO | Uploaded 36GiB to GPU in 2.58522s (13.9253GiB/s) | 20260508 04:33:30.190 | INFO | Downloaded 34GiB to GPU in 2.44019s (13.9334GiB/s) | 20260508 04:33:35.333 | INFO | Analyzing chunk frames = [5268480,5268992) | 20260508 04:33:35.333 | INFO | MemInfo: { | 20260508 04:33:35.333 | INFO | "gpu" : " 17 GB", | 20260508 04:33:35.333 | INFO | "huge" : "", | 20260508 04:33:35.333 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:35.333 | INFO | "malloc" : " 14 GB", | 20260508 04:33:35.333 | INFO | "pinned" : " 29 GB" | 20260508 04:33:35.333 | INFO | } | 20260508 04:33:35.333 | INFO | Perf report for the last 5.14351 seconds: | 20260508 04:33:35.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:35.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.333 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 04:33:35.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.334 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:33:35.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:35.334 | INFO | Uploaded 36GiB to GPU in 2.51182s (14.3323GiB/s) | 20260508 04:33:35.334 | INFO | Downloaded 34GiB to GPU in 2.40286s (14.1498GiB/s) | 20260508 04:33:40.469 | INFO | Analyzing chunk frames = [5268992,5269504) | 20260508 04:33:40.469 | INFO | MemInfo: { | 20260508 04:33:40.469 | INFO | "gpu" : " 17 GB", | 20260508 04:33:40.469 | INFO | "huge" : "", | 20260508 04:33:40.469 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:40.469 | INFO | "malloc" : " 14 GB", | 20260508 04:33:40.469 | INFO | "pinned" : " 29 GB" | 20260508 04:33:40.469 | INFO | } | 20260508 04:33:40.469 | INFO | Perf report for the last 5.13582 seconds: | 20260508 04:33:40.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:40.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.469 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 04:33:40.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.469 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:33:40.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:40.470 | INFO | Uploaded 36GiB to GPU in 2.54766s (14.1306GiB/s) | 20260508 04:33:40.470 | INFO | Downloaded 34GiB to GPU in 2.38621s (14.2486GiB/s) | 20260508 04:33:45.605 | INFO | Analyzing chunk frames = [5269504,5270016) | 20260508 04:33:45.605 | INFO | MemInfo: { | 20260508 04:33:45.605 | INFO | "gpu" : " 17 GB", | 20260508 04:33:45.605 | INFO | "huge" : "", | 20260508 04:33:45.605 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:45.605 | INFO | "malloc" : " 14 GB", | 20260508 04:33:45.605 | INFO | "pinned" : " 29 GB" | 20260508 04:33:45.605 | INFO | } | 20260508 04:33:45.605 | INFO | Perf report for the last 5.13627 seconds: | 20260508 04:33:45.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:45.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.606 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 04:33:45.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.606 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:33:45.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:45.606 | INFO | Uploaded 36GiB to GPU in 2.44448s (14.7271GiB/s) | 20260508 04:33:45.606 | INFO | Downloaded 34GiB to GPU in 2.56022s (13.2801GiB/s) | 20260508 04:33:50.745 | INFO | Analyzing chunk frames = [5270016,5270528) | 20260508 04:33:50.745 | INFO | MemInfo: { | 20260508 04:33:50.745 | INFO | "gpu" : " 17 GB", | 20260508 04:33:50.745 | INFO | "huge" : "", | 20260508 04:33:50.745 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:50.745 | INFO | "malloc" : " 14 GB", | 20260508 04:33:50.745 | INFO | "pinned" : " 29 GB" | 20260508 04:33:50.745 | INFO | } | 20260508 04:33:50.745 | INFO | Perf report for the last 5.13938 seconds: | 20260508 04:33:50.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:50.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:33:50.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:33:50.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:50.745 | INFO | Uploaded 36GiB to GPU in 2.46771s (14.5884GiB/s) | 20260508 04:33:50.745 | INFO | Downloaded 34GiB to GPU in 2.40907s (14.1133GiB/s) | 20260508 04:33:55.887 | INFO | Analyzing chunk frames = [5270528,5271040) | 20260508 04:33:55.887 | INFO | MemInfo: { | 20260508 04:33:55.887 | INFO | "gpu" : " 17 GB", | 20260508 04:33:55.887 | INFO | "huge" : "", | 20260508 04:33:55.887 | INFO | "huge_shared" : " 29 GB", | 20260508 04:33:55.887 | INFO | "malloc" : " 14 GB", | 20260508 04:33:55.887 | INFO | "pinned" : " 29 GB" | 20260508 04:33:55.887 | INFO | } | 20260508 04:33:55.887 | INFO | Perf report for the last 5.14184 seconds: | 20260508 04:33:55.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:33:55.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:33:55.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260508 04:33:55.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:33:55.887 | INFO | Uploaded 36GiB to GPU in 2.44901s (14.6998GiB/s) | 20260508 04:33:55.887 | INFO | Downloaded 34GiB to GPU in 2.40993s (14.1083GiB/s) | 20260508 04:34:01.027 | INFO | Analyzing chunk frames = [5271040,5271552) | 20260508 04:34:01.027 | INFO | MemInfo: { | 20260508 04:34:01.027 | INFO | "gpu" : " 17 GB", | 20260508 04:34:01.027 | INFO | "huge" : "", | 20260508 04:34:01.027 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:01.027 | INFO | "malloc" : " 14 GB", | 20260508 04:34:01.027 | INFO | "pinned" : " 29 GB" | 20260508 04:34:01.027 | INFO | } | 20260508 04:34:01.027 | INFO | Perf report for the last 5.14028 seconds: | 20260508 04:34:01.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:01.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 04:34:01.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:34:01.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:01.027 | INFO | Uploaded 36GiB to GPU in 2.49123s (14.4507GiB/s) | 20260508 04:34:01.027 | INFO | Downloaded 34GiB to GPU in 2.42655s (14.0117GiB/s) | 20260508 04:34:01.124 | INFO | Starting a round of Baz IO | 20260508 04:34:04.557 | INFO | Populating RTMetrics report | 20260508 04:34:04.557 | INFO | RTMetrics report populated | 20260508 04:34:04.557 | INFO | Opening tmp RTMetrics json file | 20260508 04:34:04.570 | INFO | Serializing RTMetrics to json | 20260508 04:34:04.575 | INFO | Serialization done, swapping tmp file now... | 20260508 04:34:04.575 | INFO | RTMetrics done | 20260508 04:34:04.644 | INFO | Finished a round of Baz IO: Wrote 1.593 GiB in 3.52 seconds | 20260508 04:34:04.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.04% / 0.00% / 58.96% | 20260508 04:34:04.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.528773% / 3.97364e-06% / 6.35783e-05% | 20260508 04:34:04.644 | INFO | Baz IO overall: | 20260508 04:34:04.645 | INFO | BytesWritten : 2358873431276 | 20260508 04:34:04.645 | INFO | NumEvents : 1138777226522 | 20260508 04:34:06.159 | INFO | Analyzing chunk frames = [5271552,5272064) | 20260508 04:34:06.159 | INFO | MemInfo: { | 20260508 04:34:06.159 | INFO | "gpu" : " 17 GB", | 20260508 04:34:06.159 | INFO | "huge" : "", | 20260508 04:34:06.159 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:06.159 | INFO | "malloc" : " 14 GB", | 20260508 04:34:06.159 | INFO | "pinned" : " 29 GB" | 20260508 04:34:06.159 | INFO | } | 20260508 04:34:06.159 | INFO | Perf report for the last 5.13203 seconds: | 20260508 04:34:06.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:06.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:06.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:06.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:06.159 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 04:34:06.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:06.159 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:34:06.159 | INFO | PRE_HQ : 65.8%, 1.6, 55.3ms, 0 | 20260508 04:34:06.159 | INFO | BAZWRITER : 68.2%, 3.4, 119.1ms, 0 | 20260508 04:34:06.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:34:06.160 | INFO | Uploaded 36GiB to GPU in 2.56686s (14.0249GiB/s) | 20260508 04:34:06.160 | INFO | Downloaded 38GiB to GPU in 2.70126s (14.0675GiB/s) | 20260508 04:34:11.299 | INFO | Analyzing chunk frames = [5272064,5272576) | 20260508 04:34:11.299 | INFO | MemInfo: { | 20260508 04:34:11.299 | INFO | "gpu" : " 17 GB", | 20260508 04:34:11.299 | INFO | "huge" : "", | 20260508 04:34:11.299 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:11.299 | INFO | "malloc" : " 14 GB", | 20260508 04:34:11.299 | INFO | "pinned" : " 29 GB" | 20260508 04:34:11.299 | INFO | } | 20260508 04:34:11.299 | INFO | Perf report for the last 5.14006 seconds: | 20260508 04:34:11.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:11.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:34:11.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:34:11.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:11.299 | INFO | Uploaded 36GiB to GPU in 2.58979s (13.9007GiB/s) | 20260508 04:34:11.299 | INFO | Downloaded 34GiB to GPU in 2.42471s (14.0223GiB/s) | 20260508 04:34:16.444 | INFO | Analyzing chunk frames = [5272576,5273088) | 20260508 04:34:16.444 | INFO | MemInfo: { | 20260508 04:34:16.444 | INFO | "gpu" : " 17 GB", | 20260508 04:34:16.444 | INFO | "huge" : "", | 20260508 04:34:16.444 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:16.444 | INFO | "malloc" : " 14 GB", | 20260508 04:34:16.444 | INFO | "pinned" : " 29 GB" | 20260508 04:34:16.444 | INFO | } | 20260508 04:34:16.444 | INFO | Perf report for the last 5.14512 seconds: | 20260508 04:34:16.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:16.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.444 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 04:34:16.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.444 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:34:16.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:16.445 | INFO | Uploaded 36GiB to GPU in 2.56357s (14.0429GiB/s) | 20260508 04:34:16.445 | INFO | Downloaded 34GiB to GPU in 2.42243s (14.0355GiB/s) | 20260508 04:34:21.583 | INFO | Analyzing chunk frames = [5273088,5273600) | 20260508 04:34:21.583 | INFO | MemInfo: { | 20260508 04:34:21.583 | INFO | "gpu" : " 17 GB", | 20260508 04:34:21.583 | INFO | "huge" : "", | 20260508 04:34:21.583 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:21.583 | INFO | "malloc" : " 14 GB", | 20260508 04:34:21.583 | INFO | "pinned" : " 29 GB" | 20260508 04:34:21.583 | INFO | } | 20260508 04:34:21.583 | INFO | Perf report for the last 5.1386 seconds: | 20260508 04:34:21.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:21.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 04:34:21.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:34:21.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:21.583 | INFO | Uploaded 36GiB to GPU in 2.59941s (13.8493GiB/s) | 20260508 04:34:21.583 | INFO | Downloaded 34GiB to GPU in 2.41546s (14.076GiB/s) | 20260508 04:34:26.724 | INFO | Analyzing chunk frames = [5273600,5274112) | 20260508 04:34:26.724 | INFO | MemInfo: { | 20260508 04:34:26.724 | INFO | "gpu" : " 17 GB", | 20260508 04:34:26.724 | INFO | "huge" : "", | 20260508 04:34:26.724 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:26.724 | INFO | "malloc" : " 14 GB", | 20260508 04:34:26.724 | INFO | "pinned" : " 29 GB" | 20260508 04:34:26.724 | INFO | } | 20260508 04:34:26.724 | INFO | Perf report for the last 5.14155 seconds: | 20260508 04:34:26.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:26.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 04:34:26.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:34:26.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:26.725 | INFO | Uploaded 36GiB to GPU in 2.4053s (14.967GiB/s) | 20260508 04:34:26.725 | INFO | Downloaded 34GiB to GPU in 2.55082s (13.3291GiB/s) | 20260508 04:34:31.856 | INFO | Analyzing chunk frames = [5274112,5274624) | 20260508 04:34:31.856 | INFO | MemInfo: { | 20260508 04:34:31.856 | INFO | "gpu" : " 17 GB", | 20260508 04:34:31.857 | INFO | "huge" : "", | 20260508 04:34:31.857 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:31.857 | INFO | "malloc" : " 14 GB", | 20260508 04:34:31.857 | INFO | "pinned" : " 29 GB" | 20260508 04:34:31.857 | INFO | } | 20260508 04:34:31.857 | INFO | Perf report for the last 5.1324 seconds: | 20260508 04:34:31.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:31.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 04:34:31.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 04:34:31.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:31.857 | INFO | Uploaded 36GiB to GPU in 2.46154s (14.625GiB/s) | 20260508 04:34:31.857 | INFO | Downloaded 34GiB to GPU in 2.42459s (14.023GiB/s) | 20260508 04:34:36.999 | INFO | Analyzing chunk frames = [5274624,5275136) | 20260508 04:34:37.000 | INFO | MemInfo: { | 20260508 04:34:37.000 | INFO | "gpu" : " 17 GB", | 20260508 04:34:37.000 | INFO | "huge" : "", | 20260508 04:34:37.000 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:37.000 | INFO | "malloc" : " 14 GB", | 20260508 04:34:37.000 | INFO | "pinned" : " 29 GB" | 20260508 04:34:37.000 | INFO | } | 20260508 04:34:37.000 | INFO | Perf report for the last 5.14314 seconds: | 20260508 04:34:37.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:37.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.000 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:34:37.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.000 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 04:34:37.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:37.001 | INFO | Uploaded 36GiB to GPU in 2.45028s (14.6922GiB/s) | 20260508 04:34:37.001 | INFO | Downloaded 34GiB to GPU in 2.42048s (14.0468GiB/s) | 20260508 04:34:42.140 | INFO | Analyzing chunk frames = [5275136,5275648) | 20260508 04:34:42.140 | INFO | MemInfo: { | 20260508 04:34:42.140 | INFO | "gpu" : " 17 GB", | 20260508 04:34:42.140 | INFO | "huge" : "", | 20260508 04:34:42.140 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:42.140 | INFO | "malloc" : " 14 GB", | 20260508 04:34:42.140 | INFO | "pinned" : " 29 GB" | 20260508 04:34:42.140 | INFO | } | 20260508 04:34:42.141 | INFO | Perf report for the last 5.14057 seconds: | 20260508 04:34:42.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:42.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:34:42.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 04:34:42.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:42.141 | INFO | Uploaded 36GiB to GPU in 2.45347s (14.6731GiB/s) | 20260508 04:34:42.141 | INFO | Downloaded 34GiB to GPU in 2.41612s (14.0721GiB/s) | 20260508 04:34:42.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:34:42.141 | INFO | Total mem usage: 16.7 GiB | 20260508 04:34:42.141 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:34:42.141 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13%) | 20260508 04:34:42.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:34:42.141 | INFO | High Water Breakdown: | 20260508 04:34:42.141 | INFO | 60% : 8.86 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:34:42.141 | INFO | 39.9% : 5.89 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:34:42.141 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:34:42.141 | INFO | ------------------------------------------ | 20260508 04:34:42.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:34:42.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:34:42.141 | INFO | Total mem usage: 26.5 GiB | 20260508 04:34:42.141 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:34:42.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 04:34:42.141 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 04:34:42.141 | INFO | High Water Breakdown: | 20260508 04:34:42.141 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:34:42.141 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:34:42.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:34:42.141 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:34:42.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:34:42.141 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:34:42.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:34:42.141 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:34:42.141 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:34:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:34:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:34:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:34:42.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:34:42.142 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:34:42.142 | INFO | ------------------------------------------ | 20260508 04:34:42.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:34:42.142 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:34:42.142 | INFO | Total mem usage: 27.8 GiB | 20260508 04:34:42.142 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:34:42.142 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:34:42.142 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:34:42.142 | INFO | High Water Breakdown: | 20260508 04:34:42.142 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:34:42.142 | INFO | ------------------------------------------ | 20260508 04:34:42.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:34:42.142 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:34:42.142 | INFO | Total mem usage: 15.7 GiB | 20260508 04:34:42.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:34:42.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:34:42.142 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:34:42.142 | INFO | High Water Breakdown: | 20260508 04:34:42.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:34:42.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:34:42.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:34:42.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:34:42.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:34:42.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:34:42.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:34:42.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:34:42.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:34:42.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:34:42.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:34:42.142 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:34:42.142 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:34:42.142 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:34:42.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:34:42.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:34:42.142 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:34:42.142 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:34:42.142 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:34:42.142 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:34:42.142 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:34:42.142 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:34:42.142 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:34:42.142 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:34:42.142 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:34:42.142 | INFO | ------------------------------------------ | 20260508 04:34:42.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:34:42.142 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:34:42.142 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:34:42.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:34:42.142 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.39e+01 / 1.85e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.05e-01, 1.21e-02 / 6.21e-03 | 20260508 04:34:42.142 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.49e+00, 5.89e-01 / 2.25e-01 | 20260508 04:34:42.142 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.28e-01 | 20260508 04:34:42.142 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.53e+01, 3.93e+00 / 2.19e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.76e+00 / 1.36e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.60e+00, 4.19e-01 / 2.39e-01 | 20260508 04:34:42.142 | INFO | PerfReport: 3.22%, Metrics -- 1.06e+00 / 1.23e+00, 2.66e-01 / 1.46e-01 | 20260508 04:34:42.142 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.57e-01, 5.19e-02 / 2.98e-02 | 20260508 04:34:42.142 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:34:42.142 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:34:42.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:34:42.142 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 24.53%, Upload -- 2.61e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 24.16%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 2.42%, LockUpload -- 2.57e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 2.49%, LockDownload -- 2.65e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:34:42.142 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260508 04:34:42.254 | INFO | Starting a round of Baz IO | 20260508 04:34:45.669 | INFO | Populating RTMetrics report | 20260508 04:34:45.670 | INFO | RTMetrics report populated | 20260508 04:34:45.670 | INFO | Opening tmp RTMetrics json file | 20260508 04:34:45.683 | INFO | Serializing RTMetrics to json | 20260508 04:34:45.684 | INFO | Serialization done, swapping tmp file now... | 20260508 04:34:45.685 | INFO | RTMetrics done | 20260508 04:34:45.741 | INFO | Finished a round of Baz IO: Wrote 1.593 GiB in 3.486 seconds | 20260508 04:34:45.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.04% / 0.00% / 58.96% | 20260508 04:34:45.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.528948% / 3.97364e-06% / 6.35783e-05% | 20260508 04:34:45.741 | INFO | Baz IO overall: | 20260508 04:34:45.741 | INFO | BytesWritten : 2360584294429 | 20260508 04:34:45.741 | INFO | NumEvents : 1139594463913 | 20260508 04:34:47.284 | INFO | Analyzing chunk frames = [5275648,5276160) | 20260508 04:34:47.284 | INFO | MemInfo: { | 20260508 04:34:47.284 | INFO | "gpu" : " 17 GB", | 20260508 04:34:47.284 | INFO | "huge" : "", | 20260508 04:34:47.284 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:47.284 | INFO | "malloc" : " 14 GB", | 20260508 04:34:47.284 | INFO | "pinned" : " 29 GB" | 20260508 04:34:47.284 | INFO | } | 20260508 04:34:47.284 | INFO | Perf report for the last 5.1435 seconds: | 20260508 04:34:47.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:47.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:47.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:47.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:47.284 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 04:34:47.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:47.284 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 04:34:47.284 | INFO | PRE_HQ : 66.4%, 1.8, 62.5ms, 0 | 20260508 04:34:47.284 | INFO | BAZWRITER : 67.1%, 2.9, 99.3ms, 0 | 20260508 04:34:47.284 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:34:47.285 | INFO | Uploaded 36GiB to GPU in 2.5591s (14.0674GiB/s) | 20260508 04:34:47.285 | INFO | Downloaded 38GiB to GPU in 2.69476s (14.1015GiB/s) | 20260508 04:34:52.417 | INFO | Analyzing chunk frames = [5276160,5276672) | 20260508 04:34:52.417 | INFO | MemInfo: { | 20260508 04:34:52.417 | INFO | "gpu" : " 17 GB", | 20260508 04:34:52.417 | INFO | "huge" : "", | 20260508 04:34:52.417 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:52.417 | INFO | "malloc" : " 14 GB", | 20260508 04:34:52.417 | INFO | "pinned" : " 29 GB" | 20260508 04:34:52.417 | INFO | } | 20260508 04:34:52.417 | INFO | Perf report for the last 5.1331 seconds: | 20260508 04:34:52.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:52.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.417 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 04:34:52.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.417 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:34:52.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:52.418 | INFO | Uploaded 36GiB to GPU in 2.5726s (13.9936GiB/s) | 20260508 04:34:52.418 | INFO | Downloaded 34GiB to GPU in 2.43699s (13.9516GiB/s) | 20260508 04:34:57.556 | INFO | Analyzing chunk frames = [5276672,5277184) | 20260508 04:34:57.556 | INFO | MemInfo: { | 20260508 04:34:57.556 | INFO | "gpu" : " 17 GB", | 20260508 04:34:57.556 | INFO | "huge" : "", | 20260508 04:34:57.556 | INFO | "huge_shared" : " 29 GB", | 20260508 04:34:57.556 | INFO | "malloc" : " 14 GB", | 20260508 04:34:57.557 | INFO | "pinned" : " 29 GB" | 20260508 04:34:57.557 | INFO | } | 20260508 04:34:57.557 | INFO | Perf report for the last 5.13935 seconds: | 20260508 04:34:57.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:34:57.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:34:57.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:34:57.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:34:57.557 | INFO | Uploaded 36GiB to GPU in 2.49932s (14.4039GiB/s) | 20260508 04:34:57.557 | INFO | Downloaded 34GiB to GPU in 2.41219s (14.0951GiB/s) | 20260508 04:35:02.696 | INFO | Analyzing chunk frames = [5277184,5277696) | 20260508 04:35:02.696 | INFO | MemInfo: { | 20260508 04:35:02.696 | INFO | "gpu" : " 17 GB", | 20260508 04:35:02.696 | INFO | "huge" : "", | 20260508 04:35:02.696 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:02.696 | INFO | "malloc" : " 14 GB", | 20260508 04:35:02.696 | INFO | "pinned" : " 29 GB" | 20260508 04:35:02.696 | INFO | } | 20260508 04:35:02.696 | INFO | Perf report for the last 5.13989 seconds: | 20260508 04:35:02.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:02.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.697 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:35:02.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.697 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:35:02.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:02.697 | INFO | Uploaded 36GiB to GPU in 2.55095s (14.1124GiB/s) | 20260508 04:35:02.697 | INFO | Downloaded 34GiB to GPU in 2.42231s (14.0362GiB/s) | 20260508 04:35:07.838 | INFO | Analyzing chunk frames = [5277696,5278208) | 20260508 04:35:07.838 | INFO | MemInfo: { | 20260508 04:35:07.838 | INFO | "gpu" : " 17 GB", | 20260508 04:35:07.838 | INFO | "huge" : "", | 20260508 04:35:07.838 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:07.838 | INFO | "malloc" : " 14 GB", | 20260508 04:35:07.838 | INFO | "pinned" : " 29 GB" | 20260508 04:35:07.838 | INFO | } | 20260508 04:35:07.838 | INFO | Perf report for the last 5.1415 seconds: | 20260508 04:35:07.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:07.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:35:07.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:35:07.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:07.838 | INFO | Uploaded 36GiB to GPU in 2.38284s (15.108GiB/s) | 20260508 04:35:07.838 | INFO | Downloaded 34GiB to GPU in 2.55719s (13.2958GiB/s) | 20260508 04:35:12.976 | INFO | Analyzing chunk frames = [5278208,5278720) | 20260508 04:35:12.976 | INFO | MemInfo: { | 20260508 04:35:12.976 | INFO | "gpu" : " 17 GB", | 20260508 04:35:12.977 | INFO | "huge" : "", | 20260508 04:35:12.977 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:12.977 | INFO | "malloc" : " 14 GB", | 20260508 04:35:12.977 | INFO | "pinned" : " 29 GB" | 20260508 04:35:12.977 | INFO | } | 20260508 04:35:12.977 | INFO | Perf report for the last 5.13872 seconds: | 20260508 04:35:12.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:12.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:35:12.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:35:12.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:12.977 | INFO | Uploaded 36GiB to GPU in 2.39971s (15.0018GiB/s) | 20260508 04:35:12.977 | INFO | Downloaded 34GiB to GPU in 2.41672s (14.0686GiB/s) | 20260508 04:35:18.115 | INFO | Analyzing chunk frames = [5278720,5279232) | 20260508 04:35:18.115 | INFO | MemInfo: { | 20260508 04:35:18.115 | INFO | "gpu" : " 17 GB", | 20260508 04:35:18.115 | INFO | "huge" : "", | 20260508 04:35:18.115 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:18.115 | INFO | "malloc" : " 14 GB", | 20260508 04:35:18.115 | INFO | "pinned" : " 29 GB" | 20260508 04:35:18.115 | INFO | } | 20260508 04:35:18.115 | INFO | Perf report for the last 5.138 seconds: | 20260508 04:35:18.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:18.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.115 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 04:35:18.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.115 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 04:35:18.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:18.116 | INFO | Uploaded 36GiB to GPU in 2.45612s (14.6573GiB/s) | 20260508 04:35:18.116 | INFO | Downloaded 34GiB to GPU in 2.43617s (13.9563GiB/s) | 20260508 04:35:23.254 | INFO | Analyzing chunk frames = [5279232,5279744) | 20260508 04:35:23.254 | INFO | MemInfo: { | 20260508 04:35:23.254 | INFO | "gpu" : " 17 GB", | 20260508 04:35:23.254 | INFO | "huge" : "", | 20260508 04:35:23.254 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:23.254 | INFO | "malloc" : " 14 GB", | 20260508 04:35:23.255 | INFO | "pinned" : " 29 GB" | 20260508 04:35:23.255 | INFO | } | 20260508 04:35:23.255 | INFO | Perf report for the last 5.13985 seconds: | 20260508 04:35:23.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:23.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.255 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:35:23.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.255 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 04:35:23.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:23.256 | INFO | Uploaded 36GiB to GPU in 2.44259s (14.7384GiB/s) | 20260508 04:35:23.256 | INFO | Downloaded 34GiB to GPU in 2.44098s (13.9288GiB/s) | 20260508 04:35:23.366 | INFO | Starting a round of Baz IO | 20260508 04:35:26.783 | INFO | Populating RTMetrics report | 20260508 04:35:26.784 | INFO | RTMetrics report populated | 20260508 04:35:26.784 | INFO | Opening tmp RTMetrics json file | 20260508 04:35:26.793 | INFO | Serializing RTMetrics to json | 20260508 04:35:26.797 | INFO | Serialization done, swapping tmp file now... | 20260508 04:35:26.798 | INFO | RTMetrics done | 20260508 04:35:26.857 | INFO | Finished a round of Baz IO: Wrote 1.593 GiB in 3.491 seconds | 20260508 04:35:26.857 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.03% / 0.00% / 58.96% | 20260508 04:35:26.857 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.527124% / 0% / 7.94729e-05% | 20260508 04:35:26.857 | INFO | Baz IO overall: | 20260508 04:35:26.857 | INFO | BytesWritten : 2362294898394 | 20260508 04:35:26.857 | INFO | NumEvents : 1140411611901 | 20260508 04:35:28.394 | INFO | Analyzing chunk frames = [5279744,5280256) | 20260508 04:35:28.394 | INFO | MemInfo: { | 20260508 04:35:28.394 | INFO | "gpu" : " 17 GB", | 20260508 04:35:28.394 | INFO | "huge" : "", | 20260508 04:35:28.394 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:28.394 | INFO | "malloc" : " 14 GB", | 20260508 04:35:28.394 | INFO | "pinned" : " 29 GB" | 20260508 04:35:28.394 | INFO | } | 20260508 04:35:28.394 | INFO | Perf report for the last 5.13953 seconds: | 20260508 04:35:28.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:28.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:28.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:28.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:28.394 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 04:35:28.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:28.394 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260508 04:35:28.394 | INFO | PRE_HQ : 66.2%, 1.8, 61.0ms, 0 | 20260508 04:35:28.394 | INFO | BAZWRITER : 67.4%, 2.8, 98.3ms, 0 | 20260508 04:35:28.394 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:35:28.395 | INFO | Uploaded 36GiB to GPU in 2.57679s (13.9709GiB/s) | 20260508 04:35:28.395 | INFO | Downloaded 38GiB to GPU in 2.68782s (14.1378GiB/s) | 20260508 04:35:33.534 | INFO | Analyzing chunk frames = [5280256,5280768) | 20260508 04:35:33.534 | INFO | MemInfo: { | 20260508 04:35:33.534 | INFO | "gpu" : " 17 GB", | 20260508 04:35:33.534 | INFO | "huge" : "", | 20260508 04:35:33.534 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:33.534 | INFO | "malloc" : " 14 GB", | 20260508 04:35:33.534 | INFO | "pinned" : " 29 GB" | 20260508 04:35:33.534 | INFO | } | 20260508 04:35:33.534 | INFO | Perf report for the last 5.13985 seconds: | 20260508 04:35:33.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:33.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:35:33.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:35:33.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:33.534 | INFO | Uploaded 36GiB to GPU in 2.58854s (13.9074GiB/s) | 20260508 04:35:33.534 | INFO | Downloaded 34GiB to GPU in 2.43123s (13.9847GiB/s) | 20260508 04:35:38.671 | INFO | Analyzing chunk frames = [5280768,5281280) | 20260508 04:35:38.671 | INFO | MemInfo: { | 20260508 04:35:38.671 | INFO | "gpu" : " 17 GB", | 20260508 04:35:38.671 | INFO | "huge" : "", | 20260508 04:35:38.671 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:38.671 | INFO | "malloc" : " 14 GB", | 20260508 04:35:38.671 | INFO | "pinned" : " 29 GB" | 20260508 04:35:38.671 | INFO | } | 20260508 04:35:38.671 | INFO | Perf report for the last 5.13684 seconds: | 20260508 04:35:38.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:38.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 04:35:38.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:35:38.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:38.671 | INFO | Uploaded 36GiB to GPU in 2.5874s (13.9136GiB/s) | 20260508 04:35:38.671 | INFO | Downloaded 34GiB to GPU in 2.48002s (13.7095GiB/s) | 20260508 04:35:43.812 | INFO | Analyzing chunk frames = [5281280,5281792) | 20260508 04:35:43.812 | INFO | MemInfo: { | 20260508 04:35:43.812 | INFO | "gpu" : " 17 GB", | 20260508 04:35:43.812 | INFO | "huge" : "", | 20260508 04:35:43.812 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:43.812 | INFO | "malloc" : " 14 GB", | 20260508 04:35:43.812 | INFO | "pinned" : " 29 GB" | 20260508 04:35:43.812 | INFO | } | 20260508 04:35:43.812 | INFO | Perf report for the last 5.14131 seconds: | 20260508 04:35:43.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:43.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.812 | INFO | ANALYSIS : 67.3%, 2.9, 104.2ms, 0 | 20260508 04:35:43.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.812 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:35:43.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:43.813 | INFO | Uploaded 36GiB to GPU in 2.60588s (13.8149GiB/s) | 20260508 04:35:43.813 | INFO | Downloaded 34GiB to GPU in 2.42314s (14.0314GiB/s) | 20260508 04:35:48.954 | INFO | Analyzing chunk frames = [5281792,5282304) | 20260508 04:35:48.954 | INFO | MemInfo: { | 20260508 04:35:48.954 | INFO | "gpu" : " 17 GB", | 20260508 04:35:48.954 | INFO | "huge" : "", | 20260508 04:35:48.954 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:48.954 | INFO | "malloc" : " 14 GB", | 20260508 04:35:48.954 | INFO | "pinned" : " 29 GB" | 20260508 04:35:48.954 | INFO | } | 20260508 04:35:48.954 | INFO | Perf report for the last 5.14226 seconds: | 20260508 04:35:48.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:48.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:35:48.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:35:48.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:48.955 | INFO | Uploaded 36GiB to GPU in 2.39486s (15.0322GiB/s) | 20260508 04:35:48.955 | INFO | Downloaded 34GiB to GPU in 2.53579s (13.408GiB/s) | 20260508 04:35:54.090 | INFO | Analyzing chunk frames = [5282304,5282816) | 20260508 04:35:54.090 | INFO | MemInfo: { | 20260508 04:35:54.090 | INFO | "gpu" : " 17 GB", | 20260508 04:35:54.092 | INFO | "huge" : "", | 20260508 04:35:54.092 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:54.092 | INFO | "malloc" : " 14 GB", | 20260508 04:35:54.092 | INFO | "pinned" : " 29 GB" | 20260508 04:35:54.092 | INFO | } | 20260508 04:35:54.092 | INFO | Perf report for the last 5.13791 seconds: | 20260508 04:35:54.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:54.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 04:35:54.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260508 04:35:54.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:54.093 | INFO | Uploaded 36GiB to GPU in 2.47661s (14.536GiB/s) | 20260508 04:35:54.093 | INFO | Downloaded 34GiB to GPU in 2.42167s (14.0399GiB/s) | 20260508 04:35:59.228 | INFO | Analyzing chunk frames = [5282816,5283328) | 20260508 04:35:59.228 | INFO | MemInfo: { | 20260508 04:35:59.228 | INFO | "gpu" : " 17 GB", | 20260508 04:35:59.229 | INFO | "huge" : "", | 20260508 04:35:59.229 | INFO | "huge_shared" : " 29 GB", | 20260508 04:35:59.229 | INFO | "malloc" : " 14 GB", | 20260508 04:35:59.229 | INFO | "pinned" : " 29 GB" | 20260508 04:35:59.229 | INFO | } | 20260508 04:35:59.229 | INFO | Perf report for the last 5.13633 seconds: | 20260508 04:35:59.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:35:59.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.229 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:35:59.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.229 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 04:35:59.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:35:59.230 | INFO | Uploaded 36GiB to GPU in 2.45588s (14.6587GiB/s) | 20260508 04:35:59.230 | INFO | Downloaded 34GiB to GPU in 2.40987s (14.1087GiB/s) | 20260508 04:36:04.365 | INFO | Analyzing chunk frames = [5283328,5283840) | 20260508 04:36:04.365 | INFO | MemInfo: { | 20260508 04:36:04.365 | INFO | "gpu" : " 17 GB", | 20260508 04:36:04.365 | INFO | "huge" : "", | 20260508 04:36:04.365 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:04.365 | INFO | "malloc" : " 14 GB", | 20260508 04:36:04.365 | INFO | "pinned" : " 29 GB" | 20260508 04:36:04.365 | INFO | } | 20260508 04:36:04.366 | INFO | Perf report for the last 5.13675 seconds: | 20260508 04:36:04.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:04.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 04:36:04.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | PRE_HQ : 34.6%, 1.0, 18.0ms, 0 | 20260508 04:36:04.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:04.366 | INFO | Uploaded 36GiB to GPU in 2.48458s (14.4893GiB/s) | 20260508 04:36:04.366 | INFO | Downloaded 34GiB to GPU in 2.4354s (13.9608GiB/s) | 20260508 04:36:04.366 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:36:04.366 | INFO | Total mem usage: 16.7 GiB | 20260508 04:36:04.366 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:36:04.366 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:36:04.366 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:36:04.366 | INFO | High Water Breakdown: | 20260508 04:36:04.366 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:36:04.366 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:36:04.366 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:36:04.366 | INFO | ------------------------------------------ | 20260508 04:36:04.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:36:04.366 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:36:04.366 | INFO | Total mem usage: 26.5 GiB | 20260508 04:36:04.366 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:36:04.366 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:36:04.366 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:36:04.366 | INFO | High Water Breakdown: | 20260508 04:36:04.366 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:36:04.366 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:36:04.366 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:36:04.366 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:36:04.366 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:36:04.366 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:36:04.366 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:36:04.366 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:36:04.366 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:36:04.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:36:04.366 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:36:04.366 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:36:04.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:36:04.366 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:36:04.366 | INFO | ------------------------------------------ | 20260508 04:36:04.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:36:04.366 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:36:04.366 | INFO | Total mem usage: 27.8 GiB | 20260508 04:36:04.366 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:36:04.367 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:36:04.367 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:36:04.367 | INFO | High Water Breakdown: | 20260508 04:36:04.367 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:36:04.367 | INFO | ------------------------------------------ | 20260508 04:36:04.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:36:04.367 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:36:04.367 | INFO | Total mem usage: 15.7 GiB | 20260508 04:36:04.367 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:36:04.367 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:36:04.367 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:36:04.367 | INFO | High Water Breakdown: | 20260508 04:36:04.367 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:36:04.367 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:36:04.367 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:36:04.367 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:36:04.367 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:36:04.367 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:36:04.367 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:36:04.367 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:36:04.367 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:36:04.367 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:36:04.367 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:36:04.367 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:36:04.367 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:36:04.367 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:36:04.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:36:04.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:36:04.367 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:36:04.367 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:36:04.367 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:36:04.367 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:36:04.367 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:36:04.367 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:36:04.367 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:36:04.367 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:36:04.367 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:36:04.367 | INFO | ------------------------------------------ | 20260508 04:36:04.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:36:04.367 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:36:04.367 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:36:04.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:36:04.367 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.42e+01 / 1.82e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 0.30%, AggregateStats -- 9.88e-02 / 1.06e-01, 1.19e-02 / 6.69e-03 | 20260508 04:36:04.367 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.49e+00, 5.88e-01 / 2.65e-01 | 20260508 04:36:04.367 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 1.29e-01 | 20260508 04:36:04.367 | INFO | PerfReport: 15.97%, DME -- 4.18e+01 / 4.53e+01, 3.77e+00 / 2.20e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.71e+00 / 1.50e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.60e+00, 4.23e-01 / 2.52e-01 | 20260508 04:36:04.367 | INFO | PerfReport: 3.22%, Metrics -- 1.05e+00 / 1.20e+00, 2.61e-01 / 1.45e-01 | 20260508 04:36:04.367 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.54e-01, 5.10e-02 / 2.22e-02 | 20260508 04:36:04.367 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:36:04.367 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:36:04.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:36:04.367 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 24.50%, Upload -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 24.26%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:36:04.367 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 04:36:04.479 | INFO | Starting a round of Baz IO | 20260508 04:36:07.894 | INFO | Populating RTMetrics report | 20260508 04:36:07.895 | INFO | RTMetrics report populated | 20260508 04:36:07.895 | INFO | Opening tmp RTMetrics json file | 20260508 04:36:07.908 | INFO | Serializing RTMetrics to json | 20260508 04:36:07.909 | INFO | Serialization done, swapping tmp file now... | 20260508 04:36:07.910 | INFO | RTMetrics done | 20260508 04:36:07.973 | INFO | Finished a round of Baz IO: Wrote 1.593 GiB in 3.493 seconds | 20260508 04:36:07.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.03% / 0.00% / 58.97% | 20260508 04:36:07.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.527346% / 7.94729e-06% / 5.96046e-05% | 20260508 04:36:07.973 | INFO | Baz IO overall: | 20260508 04:36:07.973 | INFO | BytesWritten : 2364005565171 | 20260508 04:36:07.973 | INFO | NumEvents : 1141228717511 | 20260508 04:36:09.508 | INFO | Analyzing chunk frames = [5283840,5284352) | 20260508 04:36:09.508 | INFO | MemInfo: { | 20260508 04:36:09.508 | INFO | "gpu" : " 17 GB", | 20260508 04:36:09.508 | INFO | "huge" : "", | 20260508 04:36:09.508 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:09.508 | INFO | "malloc" : " 14 GB", | 20260508 04:36:09.508 | INFO | "pinned" : " 29 GB" | 20260508 04:36:09.508 | INFO | } | 20260508 04:36:09.508 | INFO | Perf report for the last 5.14258 seconds: | 20260508 04:36:09.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:09.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:09.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:09.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:09.508 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 04:36:09.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:09.508 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 04:36:09.508 | INFO | PRE_HQ : 66.6%, 1.8, 62.4ms, 0 | 20260508 04:36:09.508 | INFO | BAZWRITER : 67.3%, 2.9, 100.5ms, 0 | 20260508 04:36:09.508 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:36:09.509 | INFO | Uploaded 36GiB to GPU in 2.59067s (13.896GiB/s) | 20260508 04:36:09.509 | INFO | Downloaded 38GiB to GPU in 2.68991s (14.1269GiB/s) | 20260508 04:36:14.650 | INFO | Analyzing chunk frames = [5284352,5284864) | 20260508 04:36:14.650 | INFO | MemInfo: { | 20260508 04:36:14.650 | INFO | "gpu" : " 17 GB", | 20260508 04:36:14.650 | INFO | "huge" : "", | 20260508 04:36:14.650 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:14.650 | INFO | "malloc" : " 14 GB", | 20260508 04:36:14.650 | INFO | "pinned" : " 29 GB" | 20260508 04:36:14.650 | INFO | } | 20260508 04:36:14.650 | INFO | Perf report for the last 5.14235 seconds: | 20260508 04:36:14.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:14.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.651 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:36:14.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.651 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:36:14.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:14.651 | INFO | Uploaded 36GiB to GPU in 2.59302s (13.8834GiB/s) | 20260508 04:36:14.651 | INFO | Downloaded 34GiB to GPU in 2.43102s (13.9859GiB/s) | 20260508 04:36:19.792 | INFO | Analyzing chunk frames = [5284864,5285376) | 20260508 04:36:19.792 | INFO | MemInfo: { | 20260508 04:36:19.792 | INFO | "gpu" : " 17 GB", | 20260508 04:36:19.792 | INFO | "huge" : "", | 20260508 04:36:19.792 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:19.792 | INFO | "malloc" : " 14 GB", | 20260508 04:36:19.793 | INFO | "pinned" : " 29 GB" | 20260508 04:36:19.793 | INFO | } | 20260508 04:36:19.793 | INFO | Perf report for the last 5.1426 seconds: | 20260508 04:36:19.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:19.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 04:36:19.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:36:19.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:19.793 | INFO | Uploaded 36GiB to GPU in 2.56553s (14.0322GiB/s) | 20260508 04:36:19.793 | INFO | Downloaded 34GiB to GPU in 2.42896s (13.9977GiB/s) | 20260508 04:36:24.923 | INFO | Analyzing chunk frames = [5285376,5285888) | 20260508 04:36:24.923 | INFO | MemInfo: { | 20260508 04:36:24.923 | INFO | "gpu" : " 17 GB", | 20260508 04:36:24.923 | INFO | "huge" : "", | 20260508 04:36:24.923 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:24.923 | INFO | "malloc" : " 14 GB", | 20260508 04:36:24.923 | INFO | "pinned" : " 29 GB" | 20260508 04:36:24.923 | INFO | } | 20260508 04:36:24.923 | INFO | Perf report for the last 5.13036 seconds: | 20260508 04:36:24.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:24.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | ANALYSIS : 67.2%, 3.0, 103.8ms, 0 | 20260508 04:36:24.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:36:24.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:24.924 | INFO | Uploaded 36GiB to GPU in 2.58835s (13.9085GiB/s) | 20260508 04:36:24.924 | INFO | Downloaded 34GiB to GPU in 2.40341s (14.1466GiB/s) | 20260508 04:36:30.065 | INFO | Analyzing chunk frames = [5285888,5286400) | 20260508 04:36:30.065 | INFO | MemInfo: { | 20260508 04:36:30.065 | INFO | "gpu" : " 17 GB", | 20260508 04:36:30.065 | INFO | "huge" : "", | 20260508 04:36:30.066 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:30.066 | INFO | "malloc" : " 14 GB", | 20260508 04:36:30.066 | INFO | "pinned" : " 29 GB" | 20260508 04:36:30.066 | INFO | } | 20260508 04:36:30.066 | INFO | Perf report for the last 5.14224 seconds: | 20260508 04:36:30.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:30.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:36:30.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:36:30.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:30.066 | INFO | Uploaded 36GiB to GPU in 2.3924s (15.0477GiB/s) | 20260508 04:36:30.066 | INFO | Downloaded 34GiB to GPU in 2.55521s (13.3061GiB/s) | 20260508 04:36:35.205 | INFO | Analyzing chunk frames = [5286400,5286912) | 20260508 04:36:35.205 | INFO | MemInfo: { | 20260508 04:36:35.205 | INFO | "gpu" : " 17 GB", | 20260508 04:36:35.205 | INFO | "huge" : "", | 20260508 04:36:35.205 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:35.205 | INFO | "malloc" : " 14 GB", | 20260508 04:36:35.205 | INFO | "pinned" : " 29 GB" | 20260508 04:36:35.205 | INFO | } | 20260508 04:36:35.205 | INFO | Perf report for the last 5.1396 seconds: | 20260508 04:36:35.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:35.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.205 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:36:35.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.205 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 04:36:35.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:35.206 | INFO | Uploaded 36GiB to GPU in 2.41763s (14.8906GiB/s) | 20260508 04:36:35.206 | INFO | Downloaded 34GiB to GPU in 2.41771s (14.0629GiB/s) | 20260508 04:36:40.345 | INFO | Analyzing chunk frames = [5286912,5287424) | 20260508 04:36:40.345 | INFO | MemInfo: { | 20260508 04:36:40.345 | INFO | "gpu" : " 17 GB", | 20260508 04:36:40.345 | INFO | "huge" : "", | 20260508 04:36:40.345 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:40.346 | INFO | "malloc" : " 14 GB", | 20260508 04:36:40.346 | INFO | "pinned" : " 29 GB" | 20260508 04:36:40.346 | INFO | } | 20260508 04:36:40.346 | INFO | Perf report for the last 5.14027 seconds: | 20260508 04:36:40.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:40.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:36:40.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 04:36:40.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:40.346 | INFO | Uploaded 36GiB to GPU in 2.42306s (14.8572GiB/s) | 20260508 04:36:40.346 | INFO | Downloaded 34GiB to GPU in 2.42731s (14.0073GiB/s) | 20260508 04:36:45.484 | INFO | Analyzing chunk frames = [5287424,5287936) | 20260508 04:36:45.484 | INFO | MemInfo: { | 20260508 04:36:45.484 | INFO | "gpu" : " 17 GB", | 20260508 04:36:45.484 | INFO | "huge" : "", | 20260508 04:36:45.484 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:45.484 | INFO | "malloc" : " 14 GB", | 20260508 04:36:45.484 | INFO | "pinned" : " 29 GB" | 20260508 04:36:45.484 | INFO | } | 20260508 04:36:45.484 | INFO | Perf report for the last 5.13834 seconds: | 20260508 04:36:45.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:45.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:36:45.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:36:45.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:45.484 | INFO | Uploaded 36GiB to GPU in 2.42868s (14.8229GiB/s) | 20260508 04:36:45.484 | INFO | Downloaded 34GiB to GPU in 2.42595s (14.0151GiB/s) | 20260508 04:36:45.581 | INFO | Starting a round of Baz IO | 20260508 04:36:49.023 | INFO | Populating RTMetrics report | 20260508 04:36:49.024 | INFO | RTMetrics report populated | 20260508 04:36:49.024 | INFO | Opening tmp RTMetrics json file | 20260508 04:36:49.044 | INFO | Serializing RTMetrics to json | 20260508 04:36:49.045 | INFO | Serialization done, swapping tmp file now... | 20260508 04:36:49.047 | INFO | RTMetrics done | 20260508 04:36:49.132 | INFO | Finished a round of Baz IO: Wrote 1.592 GiB in 3.55 seconds | 20260508 04:36:49.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.03% / 0.00% / 58.97% | 20260508 04:36:49.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.524942% / 7.94729e-06% / 6.75519e-05% | 20260508 04:36:49.132 | INFO | Baz IO overall: | 20260508 04:36:49.132 | INFO | BytesWritten : 2365714572795 | 20260508 04:36:49.132 | INFO | NumEvents : 1142044540221 | 20260508 04:36:50.622 | INFO | Analyzing chunk frames = [5287936,5288448) | 20260508 04:36:50.622 | INFO | MemInfo: { | 20260508 04:36:50.622 | INFO | "gpu" : " 17 GB", | 20260508 04:36:50.622 | INFO | "huge" : "", | 20260508 04:36:50.622 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:50.623 | INFO | "malloc" : " 14 GB", | 20260508 04:36:50.623 | INFO | "pinned" : " 29 GB" | 20260508 04:36:50.623 | INFO | } | 20260508 04:36:50.623 | INFO | Perf report for the last 5.13862 seconds: | 20260508 04:36:50.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:50.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:50.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:50.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:50.623 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 04:36:50.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:50.623 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 04:36:50.623 | INFO | PRE_HQ : 65.7%, 1.6, 55.3ms, 0 | 20260508 04:36:50.623 | INFO | BAZWRITER : 68.6%, 3.3, 115.8ms, 0 | 20260508 04:36:50.623 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:36:50.623 | INFO | Uploaded 36GiB to GPU in 2.53769s (14.1861GiB/s) | 20260508 04:36:50.623 | INFO | Downloaded 38GiB to GPU in 2.69996s (14.0743GiB/s) | 20260508 04:36:55.765 | INFO | Analyzing chunk frames = [5288448,5288960) | 20260508 04:36:55.765 | INFO | MemInfo: { | 20260508 04:36:55.765 | INFO | "gpu" : " 17 GB", | 20260508 04:36:55.765 | INFO | "huge" : "", | 20260508 04:36:55.765 | INFO | "huge_shared" : " 29 GB", | 20260508 04:36:55.765 | INFO | "malloc" : " 14 GB", | 20260508 04:36:55.765 | INFO | "pinned" : " 29 GB" | 20260508 04:36:55.765 | INFO | } | 20260508 04:36:55.765 | INFO | Perf report for the last 5.14253 seconds: | 20260508 04:36:55.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:36:55.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:36:55.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 04:36:55.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:36:55.765 | INFO | Uploaded 36GiB to GPU in 2.58478s (13.9277GiB/s) | 20260508 04:36:55.765 | INFO | Downloaded 34GiB to GPU in 2.43888s (13.9408GiB/s) | 20260508 04:37:00.907 | INFO | Analyzing chunk frames = [5288960,5289472) | 20260508 04:37:00.907 | INFO | MemInfo: { | 20260508 04:37:00.907 | INFO | "gpu" : " 17 GB", | 20260508 04:37:00.907 | INFO | "huge" : "", | 20260508 04:37:00.907 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:00.907 | INFO | "malloc" : " 14 GB", | 20260508 04:37:00.907 | INFO | "pinned" : " 29 GB" | 20260508 04:37:00.907 | INFO | } | 20260508 04:37:00.907 | INFO | Perf report for the last 5.14209 seconds: | 20260508 04:37:00.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:00.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 04:37:00.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:37:00.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:00.907 | INFO | Uploaded 36GiB to GPU in 2.53241s (14.2157GiB/s) | 20260508 04:37:00.907 | INFO | Downloaded 34GiB to GPU in 2.41125s (14.1006GiB/s) | 20260508 04:37:06.043 | INFO | Analyzing chunk frames = [5289472,5289984) | 20260508 04:37:06.043 | INFO | MemInfo: { | 20260508 04:37:06.043 | INFO | "gpu" : " 17 GB", | 20260508 04:37:06.043 | INFO | "huge" : "", | 20260508 04:37:06.043 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:06.044 | INFO | "malloc" : " 14 GB", | 20260508 04:37:06.044 | INFO | "pinned" : " 29 GB" | 20260508 04:37:06.044 | INFO | } | 20260508 04:37:06.044 | INFO | Perf report for the last 5.13634 seconds: | 20260508 04:37:06.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:06.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 04:37:06.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:37:06.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:06.044 | INFO | Uploaded 36GiB to GPU in 2.54221s (14.1609GiB/s) | 20260508 04:37:06.044 | INFO | Downloaded 34GiB to GPU in 2.40043s (14.1641GiB/s) | 20260508 04:37:11.179 | INFO | Analyzing chunk frames = [5289984,5290496) | 20260508 04:37:11.179 | INFO | MemInfo: { | 20260508 04:37:11.179 | INFO | "gpu" : " 17 GB", | 20260508 04:37:11.179 | INFO | "huge" : "", | 20260508 04:37:11.179 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:11.179 | INFO | "malloc" : " 14 GB", | 20260508 04:37:11.179 | INFO | "pinned" : " 29 GB" | 20260508 04:37:11.179 | INFO | } | 20260508 04:37:11.179 | INFO | Perf report for the last 5.13546 seconds: | 20260508 04:37:11.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:11.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 04:37:11.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:37:11.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:11.179 | INFO | Uploaded 36GiB to GPU in 2.38776s (15.0769GiB/s) | 20260508 04:37:11.180 | INFO | Downloaded 34GiB to GPU in 2.54715s (13.3483GiB/s) | 20260508 04:37:16.318 | INFO | Analyzing chunk frames = [5290496,5291008) | 20260508 04:37:16.318 | INFO | MemInfo: { | 20260508 04:37:16.318 | INFO | "gpu" : " 17 GB", | 20260508 04:37:16.318 | INFO | "huge" : "", | 20260508 04:37:16.318 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:16.318 | INFO | "malloc" : " 14 GB", | 20260508 04:37:16.318 | INFO | "pinned" : " 29 GB" | 20260508 04:37:16.318 | INFO | } | 20260508 04:37:16.318 | INFO | Perf report for the last 5.13904 seconds: | 20260508 04:37:16.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:16.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.318 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:37:16.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.319 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260508 04:37:16.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:16.319 | INFO | Uploaded 36GiB to GPU in 2.42356s (14.8542GiB/s) | 20260508 04:37:16.319 | INFO | Downloaded 34GiB to GPU in 2.41445s (14.0819GiB/s) | 20260508 04:37:21.460 | INFO | Analyzing chunk frames = [5291008,5291520) | 20260508 04:37:21.460 | INFO | MemInfo: { | 20260508 04:37:21.460 | INFO | "gpu" : " 17 GB", | 20260508 04:37:21.460 | INFO | "huge" : "", | 20260508 04:37:21.460 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:21.460 | INFO | "malloc" : " 14 GB", | 20260508 04:37:21.460 | INFO | "pinned" : " 29 GB" | 20260508 04:37:21.460 | INFO | } | 20260508 04:37:21.460 | INFO | Perf report for the last 5.142 seconds: | 20260508 04:37:21.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:21.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.460 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:37:21.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.460 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 04:37:21.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:21.461 | INFO | Uploaded 36GiB to GPU in 2.4331s (14.796GiB/s) | 20260508 04:37:21.461 | INFO | Downloaded 34GiB to GPU in 2.43142s (13.9836GiB/s) | 20260508 04:37:26.599 | INFO | Analyzing chunk frames = [5291520,5292032) | 20260508 04:37:26.599 | INFO | MemInfo: { | 20260508 04:37:26.599 | INFO | "gpu" : " 17 GB", | 20260508 04:37:26.599 | INFO | "huge" : "", | 20260508 04:37:26.599 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:26.599 | INFO | "malloc" : " 14 GB", | 20260508 04:37:26.599 | INFO | "pinned" : " 29 GB" | 20260508 04:37:26.599 | INFO | } | 20260508 04:37:26.600 | INFO | Perf report for the last 5.13934 seconds: | 20260508 04:37:26.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:26.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:37:26.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 04:37:26.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:26.600 | INFO | Uploaded 36GiB to GPU in 2.44451s (14.7269GiB/s) | 20260508 04:37:26.600 | INFO | Downloaded 34GiB to GPU in 2.41742s (14.0646GiB/s) | 20260508 04:37:26.600 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:37:26.600 | INFO | Total mem usage: 16.7 GiB | 20260508 04:37:26.600 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:37:26.600 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.5%) | 20260508 04:37:26.600 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:37:26.600 | INFO | High Water Breakdown: | 20260508 04:37:26.600 | INFO | 59.9% : 8.81 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:37:26.600 | INFO | 39.9% : 5.87 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:37:26.600 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:37:26.600 | INFO | ------------------------------------------ | 20260508 04:37:26.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:37:26.600 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:37:26.600 | INFO | Total mem usage: 26.5 GiB | 20260508 04:37:26.600 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:37:26.600 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:37:26.600 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:37:26.600 | INFO | High Water Breakdown: | 20260508 04:37:26.600 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:37:26.600 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:37:26.600 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:37:26.600 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:37:26.600 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:37:26.600 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:37:26.600 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:37:26.600 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:37:26.600 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:37:26.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:37:26.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:37:26.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:37:26.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:37:26.600 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:37:26.600 | INFO | ------------------------------------------ | 20260508 04:37:26.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:37:26.601 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:37:26.601 | INFO | Total mem usage: 27.8 GiB | 20260508 04:37:26.601 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:37:26.601 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:37:26.601 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:37:26.601 | INFO | High Water Breakdown: | 20260508 04:37:26.601 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:37:26.601 | INFO | ------------------------------------------ | 20260508 04:37:26.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:37:26.601 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:37:26.601 | INFO | Total mem usage: 15.7 GiB | 20260508 04:37:26.601 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:37:26.601 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:37:26.601 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:37:26.601 | INFO | High Water Breakdown: | 20260508 04:37:26.601 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:37:26.601 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:37:26.601 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:37:26.601 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:37:26.601 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:37:26.601 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:37:26.601 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:37:26.601 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:37:26.601 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:37:26.601 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:37:26.601 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:37:26.601 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:37:26.601 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:37:26.601 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:37:26.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:37:26.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:37:26.601 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:37:26.601 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:37:26.601 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:37:26.601 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:37:26.601 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:37:26.601 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:37:26.601 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:37:26.601 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:37:26.601 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:37:26.601 | INFO | ------------------------------------------ | 20260508 04:37:26.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:37:26.601 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:37:26.601 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:37:26.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:37:26.601 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.57e+01, 1.40e+01 / 1.49e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.04e-01, 1.21e-02 / 5.75e-03 | 20260508 04:37:26.601 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.50e+00, 5.94e-01 / 1.93e-01 | 20260508 04:37:26.601 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.24e-01 | 20260508 04:37:26.601 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.52e+01, 3.80e+00 / 1.73e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 1.35e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.61e+00, 4.18e-01 / 2.35e-01 | 20260508 04:37:26.601 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.22e+00, 2.62e-01 / 1.48e-01 | 20260508 04:37:26.601 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.58e-01, 5.13e-02 / 2.44e-02 | 20260508 04:37:26.601 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:37:26.601 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:37:26.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:37:26.601 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 24.41%, Upload -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 30.80%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 2.30%, LockUpload -- 2.44e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 2.59%, LockDownload -- 2.75e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 15.69%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 04:37:26.601 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260508 04:37:26.710 | INFO | Starting a round of Baz IO | 20260508 04:37:30.140 | INFO | Populating RTMetrics report | 20260508 04:37:30.140 | INFO | RTMetrics report populated | 20260508 04:37:30.140 | INFO | Opening tmp RTMetrics json file | 20260508 04:37:30.152 | INFO | Serializing RTMetrics to json | 20260508 04:37:30.153 | INFO | Serialization done, swapping tmp file now... | 20260508 04:37:30.154 | INFO | RTMetrics done | 20260508 04:37:30.208 | INFO | Finished a round of Baz IO: Wrote 1.594 GiB in 3.498 seconds | 20260508 04:37:30.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.03% / 0.00% / 58.97% | 20260508 04:37:30.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.525471% / 0% / 5.16574e-05% | 20260508 04:37:30.209 | INFO | Baz IO overall: | 20260508 04:37:30.209 | INFO | BytesWritten : 2367425589471 | 20260508 04:37:30.209 | INFO | NumEvents : 1142861943887 | 20260508 04:37:31.743 | INFO | Analyzing chunk frames = [5292032,5292544) | 20260508 04:37:31.743 | INFO | MemInfo: { | 20260508 04:37:31.743 | INFO | "gpu" : " 17 GB", | 20260508 04:37:31.743 | INFO | "huge" : "", | 20260508 04:37:31.743 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:31.743 | INFO | "malloc" : " 14 GB", | 20260508 04:37:31.743 | INFO | "pinned" : " 29 GB" | 20260508 04:37:31.743 | INFO | } | 20260508 04:37:31.743 | INFO | Perf report for the last 5.14388 seconds: | 20260508 04:37:31.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:31.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:31.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:31.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:31.744 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 04:37:31.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:31.744 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 04:37:31.744 | INFO | PRE_HQ : 67.0%, 1.8, 62.3ms, 0 | 20260508 04:37:31.744 | INFO | BAZWRITER : 67.4%, 2.9, 101.2ms, 0 | 20260508 04:37:31.744 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:37:31.744 | INFO | Uploaded 36GiB to GPU in 2.55481s (14.0911GiB/s) | 20260508 04:37:31.744 | INFO | Downloaded 38GiB to GPU in 2.69895s (14.0795GiB/s) | 20260508 04:37:36.879 | INFO | Analyzing chunk frames = [5292544,5293056) | 20260508 04:37:36.879 | INFO | MemInfo: { | 20260508 04:37:36.879 | INFO | "gpu" : " 17 GB", | 20260508 04:37:36.879 | INFO | "huge" : "", | 20260508 04:37:36.879 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:36.879 | INFO | "malloc" : " 14 GB", | 20260508 04:37:36.879 | INFO | "pinned" : " 29 GB" | 20260508 04:37:36.879 | INFO | } | 20260508 04:37:36.879 | INFO | Perf report for the last 5.13567 seconds: | 20260508 04:37:36.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:36.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 04:37:36.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:37:36.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:36.879 | INFO | Uploaded 36GiB to GPU in 2.49581s (14.4242GiB/s) | 20260508 04:37:36.879 | INFO | Downloaded 34GiB to GPU in 2.42844s (14.0007GiB/s) | 20260508 04:37:42.017 | INFO | Analyzing chunk frames = [5293056,5293568) | 20260508 04:37:42.017 | INFO | MemInfo: { | 20260508 04:37:42.017 | INFO | "gpu" : " 17 GB", | 20260508 04:37:42.017 | INFO | "huge" : "", | 20260508 04:37:42.018 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:42.018 | INFO | "malloc" : " 14 GB", | 20260508 04:37:42.018 | INFO | "pinned" : " 29 GB" | 20260508 04:37:42.018 | INFO | } | 20260508 04:37:42.019 | INFO | Perf report for the last 5.13947 seconds: | 20260508 04:37:42.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:42.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 04:37:42.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:37:42.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:42.019 | INFO | Uploaded 36GiB to GPU in 2.5373s (14.1883GiB/s) | 20260508 04:37:42.019 | INFO | Downloaded 34GiB to GPU in 2.40413s (14.1423GiB/s) | 20260508 04:37:47.155 | INFO | Analyzing chunk frames = [5293568,5294080) | 20260508 04:37:47.155 | INFO | MemInfo: { | 20260508 04:37:47.155 | INFO | "gpu" : " 17 GB", | 20260508 04:37:47.155 | INFO | "huge" : "", | 20260508 04:37:47.155 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:47.155 | INFO | "malloc" : " 14 GB", | 20260508 04:37:47.156 | INFO | "pinned" : " 29 GB" | 20260508 04:37:47.156 | INFO | } | 20260508 04:37:47.156 | INFO | Perf report for the last 5.13702 seconds: | 20260508 04:37:47.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:47.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:37:47.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:37:47.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:47.156 | INFO | Uploaded 36GiB to GPU in 2.56948s (14.0106GiB/s) | 20260508 04:37:47.156 | INFO | Downloaded 34GiB to GPU in 2.41159s (14.0986GiB/s) | 20260508 04:37:52.294 | INFO | Analyzing chunk frames = [5294080,5294592) | 20260508 04:37:52.294 | INFO | MemInfo: { | 20260508 04:37:52.294 | INFO | "gpu" : " 17 GB", | 20260508 04:37:52.294 | INFO | "huge" : "", | 20260508 04:37:52.294 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:52.294 | INFO | "malloc" : " 14 GB", | 20260508 04:37:52.294 | INFO | "pinned" : " 29 GB" | 20260508 04:37:52.294 | INFO | } | 20260508 04:37:52.294 | INFO | Perf report for the last 5.13842 seconds: | 20260508 04:37:52.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:52.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 04:37:52.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:37:52.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:52.294 | INFO | Uploaded 36GiB to GPU in 2.36187s (15.2421GiB/s) | 20260508 04:37:52.294 | INFO | Downloaded 34GiB to GPU in 2.54203s (13.3751GiB/s) | 20260508 04:37:57.433 | INFO | Analyzing chunk frames = [5294592,5295104) | 20260508 04:37:57.433 | INFO | MemInfo: { | 20260508 04:37:57.433 | INFO | "gpu" : " 17 GB", | 20260508 04:37:57.433 | INFO | "huge" : "", | 20260508 04:37:57.433 | INFO | "huge_shared" : " 29 GB", | 20260508 04:37:57.433 | INFO | "malloc" : " 14 GB", | 20260508 04:37:57.433 | INFO | "pinned" : " 29 GB" | 20260508 04:37:57.433 | INFO | } | 20260508 04:37:57.434 | INFO | Perf report for the last 5.13951 seconds: | 20260508 04:37:57.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:37:57.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:37:57.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:37:57.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:37:57.434 | INFO | Uploaded 36GiB to GPU in 2.42491s (14.8459GiB/s) | 20260508 04:37:57.434 | INFO | Downloaded 34GiB to GPU in 2.41585s (14.0737GiB/s) | 20260508 04:38:02.578 | INFO | Analyzing chunk frames = [5295104,5295616) | 20260508 04:38:02.578 | INFO | MemInfo: { | 20260508 04:38:02.578 | INFO | "gpu" : " 17 GB", | 20260508 04:38:02.578 | INFO | "huge" : "", | 20260508 04:38:02.578 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:02.578 | INFO | "malloc" : " 14 GB", | 20260508 04:38:02.578 | INFO | "pinned" : " 29 GB" | 20260508 04:38:02.578 | INFO | } | 20260508 04:38:02.578 | INFO | Perf report for the last 5.14451 seconds: | 20260508 04:38:02.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:02.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 04:38:02.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:38:02.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:02.578 | INFO | Uploaded 36GiB to GPU in 2.42165s (14.8659GiB/s) | 20260508 04:38:02.578 | INFO | Downloaded 34GiB to GPU in 2.41715s (14.0662GiB/s) | 20260508 04:38:07.717 | INFO | Analyzing chunk frames = [5295616,5296128) | 20260508 04:38:07.717 | INFO | MemInfo: { | 20260508 04:38:07.717 | INFO | "gpu" : " 17 GB", | 20260508 04:38:07.717 | INFO | "huge" : "", | 20260508 04:38:07.717 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:07.717 | INFO | "malloc" : " 14 GB", | 20260508 04:38:07.717 | INFO | "pinned" : " 29 GB" | 20260508 04:38:07.717 | INFO | } | 20260508 04:38:07.717 | INFO | Perf report for the last 5.13924 seconds: | 20260508 04:38:07.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:07.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.717 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:38:07.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.717 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:38:07.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:07.718 | INFO | Uploaded 36GiB to GPU in 2.44969s (14.6957GiB/s) | 20260508 04:38:07.718 | INFO | Downloaded 34GiB to GPU in 2.42304s (14.032GiB/s) | 20260508 04:38:07.814 | INFO | Starting a round of Baz IO | 20260508 04:38:11.238 | INFO | Populating RTMetrics report | 20260508 04:38:11.239 | INFO | RTMetrics report populated | 20260508 04:38:11.239 | INFO | Opening tmp RTMetrics json file | 20260508 04:38:11.247 | INFO | Serializing RTMetrics to json | 20260508 04:38:11.253 | INFO | Serialization done, swapping tmp file now... | 20260508 04:38:11.253 | INFO | RTMetrics done | 20260508 04:38:11.319 | INFO | Finished a round of Baz IO: Wrote 1.592 GiB in 3.504 seconds | 20260508 04:38:11.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.02% / 0.00% / 58.97% | 20260508 04:38:11.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.523678% / 0% / 5.96046e-05% | 20260508 04:38:11.319 | INFO | Baz IO overall: | 20260508 04:38:11.319 | INFO | BytesWritten : 2369134689361 | 20260508 04:38:11.319 | INFO | NumEvents : 1143678184756 | 20260508 04:38:12.854 | INFO | Analyzing chunk frames = [5296128,5296640) | 20260508 04:38:12.854 | INFO | MemInfo: { | 20260508 04:38:12.854 | INFO | "gpu" : " 17 GB", | 20260508 04:38:12.854 | INFO | "huge" : "", | 20260508 04:38:12.854 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:12.854 | INFO | "malloc" : " 14 GB", | 20260508 04:38:12.854 | INFO | "pinned" : " 29 GB" | 20260508 04:38:12.854 | INFO | } | 20260508 04:38:12.854 | INFO | Perf report for the last 5.137 seconds: | 20260508 04:38:12.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:12.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:12.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:12.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:12.854 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 04:38:12.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:12.854 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:38:12.854 | INFO | PRE_HQ : 65.6%, 1.6, 55.4ms, 0 | 20260508 04:38:12.854 | INFO | BAZWRITER : 67.7%, 2.7, 95.3ms, 0 | 20260508 04:38:12.854 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:38:12.855 | INFO | Uploaded 36GiB to GPU in 2.55851s (14.0707GiB/s) | 20260508 04:38:12.855 | INFO | Downloaded 38GiB to GPU in 2.68298s (14.1634GiB/s) | 20260508 04:38:17.991 | INFO | Analyzing chunk frames = [5296640,5297152) | 20260508 04:38:17.991 | INFO | MemInfo: { | 20260508 04:38:17.991 | INFO | "gpu" : " 17 GB", | 20260508 04:38:17.991 | INFO | "huge" : "", | 20260508 04:38:17.991 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:17.991 | INFO | "malloc" : " 14 GB", | 20260508 04:38:17.991 | INFO | "pinned" : " 29 GB" | 20260508 04:38:17.991 | INFO | } | 20260508 04:38:17.991 | INFO | Perf report for the last 5.13664 seconds: | 20260508 04:38:17.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:17.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 04:38:17.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:38:17.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:17.991 | INFO | Uploaded 36GiB to GPU in 2.5728s (13.9926GiB/s) | 20260508 04:38:17.991 | INFO | Downloaded 34GiB to GPU in 2.42945s (13.9949GiB/s) | 20260508 04:38:23.130 | INFO | Analyzing chunk frames = [5297152,5297664) | 20260508 04:38:23.130 | INFO | MemInfo: { | 20260508 04:38:23.130 | INFO | "gpu" : " 17 GB", | 20260508 04:38:23.130 | INFO | "huge" : "", | 20260508 04:38:23.130 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:23.130 | INFO | "malloc" : " 14 GB", | 20260508 04:38:23.130 | INFO | "pinned" : " 29 GB" | 20260508 04:38:23.130 | INFO | } | 20260508 04:38:23.130 | INFO | Perf report for the last 5.13881 seconds: | 20260508 04:38:23.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:23.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 04:38:23.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:38:23.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:23.130 | INFO | Uploaded 36GiB to GPU in 2.53283s (14.2133GiB/s) | 20260508 04:38:23.130 | INFO | Downloaded 34GiB to GPU in 2.43051s (13.9888GiB/s) | 20260508 04:38:28.270 | INFO | Analyzing chunk frames = [5297664,5298176) | 20260508 04:38:28.270 | INFO | MemInfo: { | 20260508 04:38:28.270 | INFO | "gpu" : " 17 GB", | 20260508 04:38:28.270 | INFO | "huge" : "", | 20260508 04:38:28.270 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:28.270 | INFO | "malloc" : " 14 GB", | 20260508 04:38:28.270 | INFO | "pinned" : " 29 GB" | 20260508 04:38:28.270 | INFO | } | 20260508 04:38:28.271 | INFO | Perf report for the last 5.14068 seconds: | 20260508 04:38:28.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:28.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:38:28.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:38:28.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:28.271 | INFO | Uploaded 36GiB to GPU in 2.57183s (13.9978GiB/s) | 20260508 04:38:28.271 | INFO | Downloaded 34GiB to GPU in 2.40883s (14.1147GiB/s) | 20260508 04:38:33.420 | INFO | Analyzing chunk frames = [5298176,5298688) | 20260508 04:38:33.420 | INFO | MemInfo: { | 20260508 04:38:33.420 | INFO | "gpu" : " 17 GB", | 20260508 04:38:33.420 | INFO | "huge" : "", | 20260508 04:38:33.420 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:33.420 | INFO | "malloc" : " 14 GB", | 20260508 04:38:33.420 | INFO | "pinned" : " 29 GB" | 20260508 04:38:33.420 | INFO | } | 20260508 04:38:33.421 | INFO | Perf report for the last 5.15003 seconds: | 20260508 04:38:33.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:33.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:38:33.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:38:33.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:33.421 | INFO | Uploaded 36GiB to GPU in 2.36652s (15.2122GiB/s) | 20260508 04:38:33.421 | INFO | Downloaded 34GiB to GPU in 2.55761s (13.2937GiB/s) | 20260508 04:38:38.555 | INFO | Analyzing chunk frames = [5298688,5299200) | 20260508 04:38:38.555 | INFO | MemInfo: { | 20260508 04:38:38.555 | INFO | "gpu" : " 17 GB", | 20260508 04:38:38.555 | INFO | "huge" : "", | 20260508 04:38:38.555 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:38.555 | INFO | "malloc" : " 14 GB", | 20260508 04:38:38.555 | INFO | "pinned" : " 29 GB" | 20260508 04:38:38.555 | INFO | } | 20260508 04:38:38.555 | INFO | Perf report for the last 5.13453 seconds: | 20260508 04:38:38.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:38.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.555 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:38:38.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.555 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 04:38:38.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:38.556 | INFO | Uploaded 36GiB to GPU in 2.43173s (14.8043GiB/s) | 20260508 04:38:38.556 | INFO | Downloaded 34GiB to GPU in 2.43452s (13.9658GiB/s) | 20260508 04:38:43.690 | INFO | Analyzing chunk frames = [5299200,5299712) | 20260508 04:38:43.690 | INFO | MemInfo: { | 20260508 04:38:43.690 | INFO | "gpu" : " 17 GB", | 20260508 04:38:43.690 | INFO | "huge" : "", | 20260508 04:38:43.690 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:43.690 | INFO | "malloc" : " 14 GB", | 20260508 04:38:43.690 | INFO | "pinned" : " 29 GB" | 20260508 04:38:43.690 | INFO | } | 20260508 04:38:43.690 | INFO | Perf report for the last 5.13508 seconds: | 20260508 04:38:43.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:43.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.691 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:38:43.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.691 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 04:38:43.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:43.691 | INFO | Uploaded 36GiB to GPU in 2.42975s (14.8163GiB/s) | 20260508 04:38:43.691 | INFO | Downloaded 34GiB to GPU in 2.43111s (13.9854GiB/s) | 20260508 04:38:48.825 | INFO | Analyzing chunk frames = [5299712,5300224) | 20260508 04:38:48.825 | INFO | MemInfo: { | 20260508 04:38:48.825 | INFO | "gpu" : " 17 GB", | 20260508 04:38:48.825 | INFO | "huge" : "", | 20260508 04:38:48.825 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:48.826 | INFO | "malloc" : " 14 GB", | 20260508 04:38:48.826 | INFO | "pinned" : " 29 GB" | 20260508 04:38:48.826 | INFO | } | 20260508 04:38:48.826 | INFO | Perf report for the last 5.13544 seconds: | 20260508 04:38:48.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:48.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:38:48.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | PRE_HQ : 34.6%, 1.0, 17.9ms, 0 | 20260508 04:38:48.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:48.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:38:48.826 | INFO | Total mem usage: 16.7 GiB | 20260508 04:38:48.826 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:38:48.826 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:38:48.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:38:48.826 | INFO | High Water Breakdown: | 20260508 04:38:48.826 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:38:48.826 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:38:48.826 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:38:48.826 | INFO | ------------------------------------------ | 20260508 04:38:48.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:38:48.826 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:38:48.826 | INFO | Total mem usage: 26.5 GiB | 20260508 04:38:48.826 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:38:48.826 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:38:48.826 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:38:48.827 | INFO | High Water Breakdown: | 20260508 04:38:48.826 | INFO | Uploaded 36GiB to GPU in 2.44289s (14.7366GiB/s) | 20260508 04:38:48.827 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:38:48.827 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:38:48.827 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:38:48.827 | INFO | Downloaded 34GiB to GPU in 2.43698s (13.9517GiB/s) | 20260508 04:38:48.827 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:38:48.827 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:38:48.827 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:38:48.827 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:38:48.827 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:38:48.827 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:38:48.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:38:48.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:38:48.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:38:48.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:38:48.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:38:48.827 | INFO | ------------------------------------------ | 20260508 04:38:48.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:38:48.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:38:48.827 | INFO | Total mem usage: 27.8 GiB | 20260508 04:38:48.827 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:38:48.827 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:38:48.827 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:38:48.827 | INFO | High Water Breakdown: | 20260508 04:38:48.827 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:38:48.827 | INFO | ------------------------------------------ | 20260508 04:38:48.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:38:48.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:38:48.827 | INFO | Total mem usage: 15.7 GiB | 20260508 04:38:48.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:38:48.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:38:48.827 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:38:48.827 | INFO | High Water Breakdown: | 20260508 04:38:48.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:38:48.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:38:48.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:38:48.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:38:48.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:38:48.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:38:48.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:38:48.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:38:48.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:38:48.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:38:48.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:38:48.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:38:48.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:38:48.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:38:48.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:38:48.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:38:48.827 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:38:48.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:38:48.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:38:48.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:38:48.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:38:48.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:38:48.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:38:48.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:38:48.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:38:48.827 | INFO | ------------------------------------------ | 20260508 04:38:48.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:38:48.827 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:38:48.827 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:38:48.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:38:48.827 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.57e+01, 1.39e+01 / 1.64e+00 | 20260508 04:38:48.827 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.04e-01, 1.16e-02 / 5.52e-03 | 20260508 04:38:48.827 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.47e+00, 5.98e-01 / 2.01e-01 | 20260508 04:38:48.828 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.23e-01 | 20260508 04:38:48.828 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.51e+01, 3.76e+00 / 2.02e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.78e+00 / 1.47e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.21e-01 / 2.30e-01 | 20260508 04:38:48.828 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.24e+00, 2.64e-01 / 1.30e-01 | 20260508 04:38:48.828 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.23e-01 / 3.65e-01, 5.09e-02 / 2.89e-02 | 20260508 04:38:48.828 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:38:48.828 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:38:48.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:38:48.828 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 30.83%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 2.19%, LockUpload -- 2.33e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 2.69%, LockDownload -- 2.86e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:38:48.828 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260508 04:38:48.940 | INFO | Starting a round of Baz IO | 20260508 04:38:52.361 | INFO | Populating RTMetrics report | 20260508 04:38:52.361 | INFO | RTMetrics report populated | 20260508 04:38:52.361 | INFO | Opening tmp RTMetrics json file | 20260508 04:38:52.371 | INFO | Serializing RTMetrics to json | 20260508 04:38:52.372 | INFO | Serialization done, swapping tmp file now... | 20260508 04:38:52.372 | INFO | RTMetrics done | 20260508 04:38:52.433 | INFO | Finished a round of Baz IO: Wrote 1.592 GiB in 3.492 seconds | 20260508 04:38:52.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.02% / 0.00% / 58.98% | 20260508 04:38:52.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.522951% / 3.97364e-06% / 6.75519e-05% | 20260508 04:38:52.433 | INFO | Baz IO overall: | 20260508 04:38:52.433 | INFO | BytesWritten : 2370844069664 | 20260508 04:38:52.433 | INFO | NumEvents : 1144494397440 | 20260508 04:38:53.969 | INFO | Analyzing chunk frames = [5300224,5300736) | 20260508 04:38:53.969 | INFO | MemInfo: { | 20260508 04:38:53.969 | INFO | "gpu" : " 17 GB", | 20260508 04:38:53.969 | INFO | "huge" : "", | 20260508 04:38:53.969 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:53.969 | INFO | "malloc" : " 14 GB", | 20260508 04:38:53.969 | INFO | "pinned" : " 29 GB" | 20260508 04:38:53.969 | INFO | } | 20260508 04:38:53.969 | INFO | Perf report for the last 5.14345 seconds: | 20260508 04:38:53.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:53.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:53.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:53.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:53.969 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 04:38:53.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:53.969 | INFO | RT_METRICS : 43.7%, 1.0, 23.0ms, 0 | 20260508 04:38:53.969 | INFO | PRE_HQ : 66.2%, 1.8, 61.9ms, 0 | 20260508 04:38:53.969 | INFO | BAZWRITER : 67.5%, 2.9, 100.2ms, 0 | 20260508 04:38:53.969 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:38:53.969 | INFO | Uploaded 36GiB to GPU in 2.54661s (14.1365GiB/s) | 20260508 04:38:53.969 | INFO | Downloaded 38GiB to GPU in 2.69787s (14.0852GiB/s) | 20260508 04:38:59.105 | INFO | Analyzing chunk frames = [5300736,5301248) | 20260508 04:38:59.105 | INFO | MemInfo: { | 20260508 04:38:59.105 | INFO | "gpu" : " 17 GB", | 20260508 04:38:59.105 | INFO | "huge" : "", | 20260508 04:38:59.105 | INFO | "huge_shared" : " 29 GB", | 20260508 04:38:59.105 | INFO | "malloc" : " 14 GB", | 20260508 04:38:59.106 | INFO | "pinned" : " 29 GB" | 20260508 04:38:59.106 | INFO | } | 20260508 04:38:59.106 | INFO | Perf report for the last 5.13651 seconds: | 20260508 04:38:59.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:38:59.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 04:38:59.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:38:59.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:38:59.106 | INFO | Uploaded 36GiB to GPU in 2.57237s (13.9949GiB/s) | 20260508 04:38:59.106 | INFO | Downloaded 34GiB to GPU in 2.45858s (13.8291GiB/s) | 20260508 04:39:04.242 | INFO | Analyzing chunk frames = [5301248,5301760) | 20260508 04:39:04.242 | INFO | MemInfo: { | 20260508 04:39:04.242 | INFO | "gpu" : " 17 GB", | 20260508 04:39:04.242 | INFO | "huge" : "", | 20260508 04:39:04.242 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:04.242 | INFO | "malloc" : " 14 GB", | 20260508 04:39:04.242 | INFO | "pinned" : " 29 GB" | 20260508 04:39:04.242 | INFO | } | 20260508 04:39:04.242 | INFO | Perf report for the last 5.1364 seconds: | 20260508 04:39:04.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:04.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:39:04.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:39:04.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:04.242 | INFO | Uploaded 36GiB to GPU in 2.52029s (14.284GiB/s) | 20260508 04:39:04.242 | INFO | Downloaded 34GiB to GPU in 2.446s (13.9002GiB/s) | 20260508 04:39:09.382 | INFO | Analyzing chunk frames = [5301760,5302272) | 20260508 04:39:09.382 | INFO | MemInfo: { | 20260508 04:39:09.382 | INFO | "gpu" : " 17 GB", | 20260508 04:39:09.382 | INFO | "huge" : "", | 20260508 04:39:09.382 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:09.382 | INFO | "malloc" : " 14 GB", | 20260508 04:39:09.383 | INFO | "pinned" : " 29 GB" | 20260508 04:39:09.383 | INFO | } | 20260508 04:39:09.383 | INFO | Perf report for the last 5.14052 seconds: | 20260508 04:39:09.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:09.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 04:39:09.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:39:09.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:09.383 | INFO | Uploaded 36GiB to GPU in 2.53534s (14.1993GiB/s) | 20260508 04:39:09.383 | INFO | Downloaded 34GiB to GPU in 2.42637s (14.0127GiB/s) | 20260508 04:39:14.528 | INFO | Analyzing chunk frames = [5302272,5302784) | 20260508 04:39:14.528 | INFO | MemInfo: { | 20260508 04:39:14.528 | INFO | "gpu" : " 17 GB", | 20260508 04:39:14.528 | INFO | "huge" : "", | 20260508 04:39:14.528 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:14.528 | INFO | "malloc" : " 14 GB", | 20260508 04:39:14.528 | INFO | "pinned" : " 29 GB" | 20260508 04:39:14.528 | INFO | } | 20260508 04:39:14.528 | INFO | Perf report for the last 5.14522 seconds: | 20260508 04:39:14.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:14.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:39:14.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:39:14.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:14.528 | INFO | Uploaded 36GiB to GPU in 2.32269s (15.4993GiB/s) | 20260508 04:39:14.528 | INFO | Downloaded 34GiB to GPU in 2.56511s (13.2548GiB/s) | 20260508 04:39:19.666 | INFO | Analyzing chunk frames = [5302784,5303296) | 20260508 04:39:19.666 | INFO | MemInfo: { | 20260508 04:39:19.666 | INFO | "gpu" : " 17 GB", | 20260508 04:39:19.666 | INFO | "huge" : "", | 20260508 04:39:19.666 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:19.666 | INFO | "malloc" : " 14 GB", | 20260508 04:39:19.666 | INFO | "pinned" : " 29 GB" | 20260508 04:39:19.667 | INFO | } | 20260508 04:39:19.667 | INFO | Perf report for the last 5.13874 seconds: | 20260508 04:39:19.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:19.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:39:19.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:39:19.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:19.667 | INFO | Uploaded 36GiB to GPU in 2.42933s (14.8189GiB/s) | 20260508 04:39:19.667 | INFO | Downloaded 34GiB to GPU in 2.41966s (14.0516GiB/s) | 20260508 04:39:24.805 | INFO | Analyzing chunk frames = [5303296,5303808) | 20260508 04:39:24.805 | INFO | MemInfo: { | 20260508 04:39:24.805 | INFO | "gpu" : " 17 GB", | 20260508 04:39:24.806 | INFO | "huge" : "", | 20260508 04:39:24.806 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:24.806 | INFO | "malloc" : " 14 GB", | 20260508 04:39:24.806 | INFO | "pinned" : " 29 GB" | 20260508 04:39:24.806 | INFO | } | 20260508 04:39:24.806 | INFO | Perf report for the last 5.1391 seconds: | 20260508 04:39:24.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:24.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:39:24.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 04:39:24.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:24.806 | INFO | Uploaded 36GiB to GPU in 2.4564s (14.6556GiB/s) | 20260508 04:39:24.806 | INFO | Downloaded 34GiB to GPU in 2.43447s (13.9661GiB/s) | 20260508 04:39:29.939 | INFO | Analyzing chunk frames = [5303808,5304320) | 20260508 04:39:29.939 | INFO | MemInfo: { | 20260508 04:39:29.939 | INFO | "gpu" : " 17 GB", | 20260508 04:39:29.940 | INFO | "huge" : "", | 20260508 04:39:29.940 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:29.940 | INFO | "malloc" : " 14 GB", | 20260508 04:39:29.940 | INFO | "pinned" : " 29 GB" | 20260508 04:39:29.940 | INFO | } | 20260508 04:39:29.940 | INFO | Perf report for the last 5.134 seconds: | 20260508 04:39:29.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:29.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:39:29.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 04:39:29.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:29.940 | INFO | Uploaded 36GiB to GPU in 2.43844s (14.7635GiB/s) | 20260508 04:39:29.940 | INFO | Downloaded 34GiB to GPU in 2.4296s (13.9941GiB/s) | 20260508 04:39:30.037 | INFO | Starting a round of Baz IO | 20260508 04:39:33.471 | INFO | Populating RTMetrics report | 20260508 04:39:33.471 | INFO | RTMetrics report populated | 20260508 04:39:33.471 | INFO | Opening tmp RTMetrics json file | 20260508 04:39:33.479 | INFO | Serializing RTMetrics to json | 20260508 04:39:33.487 | INFO | Serialization done, swapping tmp file now... | 20260508 04:39:33.487 | INFO | RTMetrics done | 20260508 04:39:33.545 | INFO | Finished a round of Baz IO: Wrote 1.593 GiB in 3.508 seconds | 20260508 04:39:33.545 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.02% / 0.00% / 58.98% | 20260508 04:39:33.545 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.522129% / 7.94729e-06% / 7.15256e-05% | 20260508 04:39:33.545 | INFO | Baz IO overall: | 20260508 04:39:33.545 | INFO | BytesWritten : 2372554072232 | 20260508 04:39:33.546 | INFO | NumEvents : 1145310995690 | 20260508 04:39:35.079 | INFO | Analyzing chunk frames = [5304320,5304832) | 20260508 04:39:35.079 | INFO | MemInfo: { | 20260508 04:39:35.079 | INFO | "gpu" : " 17 GB", | 20260508 04:39:35.079 | INFO | "huge" : "", | 20260508 04:39:35.079 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:35.079 | INFO | "malloc" : " 14 GB", | 20260508 04:39:35.079 | INFO | "pinned" : " 29 GB" | 20260508 04:39:35.079 | INFO | } | 20260508 04:39:35.080 | INFO | Perf report for the last 5.13983 seconds: | 20260508 04:39:35.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:35.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:35.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:35.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:35.080 | INFO | ANALYSIS : 68.0%, 3.0, 106.3ms, 0 | 20260508 04:39:35.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:35.080 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:39:35.080 | INFO | PRE_HQ : 65.5%, 1.6, 55.4ms, 0 | 20260508 04:39:35.080 | INFO | BAZWRITER : 67.7%, 2.7, 94.1ms, 0 | 20260508 04:39:35.080 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:39:35.080 | INFO | Uploaded 36GiB to GPU in 2.559s (14.068GiB/s) | 20260508 04:39:35.080 | INFO | Downloaded 38GiB to GPU in 2.67805s (14.1894GiB/s) | 20260508 04:39:40.220 | INFO | Analyzing chunk frames = [5304832,5305344) | 20260508 04:39:40.220 | INFO | MemInfo: { | 20260508 04:39:40.220 | INFO | "gpu" : " 17 GB", | 20260508 04:39:40.220 | INFO | "huge" : "", | 20260508 04:39:40.220 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:40.220 | INFO | "malloc" : " 14 GB", | 20260508 04:39:40.220 | INFO | "pinned" : " 29 GB" | 20260508 04:39:40.220 | INFO | } | 20260508 04:39:40.220 | INFO | Perf report for the last 5.14024 seconds: | 20260508 04:39:40.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:40.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:39:40.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:39:40.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:40.220 | INFO | Uploaded 36GiB to GPU in 2.5764s (13.973GiB/s) | 20260508 04:39:40.220 | INFO | Downloaded 34GiB to GPU in 2.44215s (13.9221GiB/s) | 20260508 04:39:45.363 | INFO | Analyzing chunk frames = [5305344,5305856) | 20260508 04:39:45.363 | INFO | MemInfo: { | 20260508 04:39:45.363 | INFO | "gpu" : " 17 GB", | 20260508 04:39:45.363 | INFO | "huge" : "", | 20260508 04:39:45.363 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:45.364 | INFO | "malloc" : " 14 GB", | 20260508 04:39:45.364 | INFO | "pinned" : " 29 GB" | 20260508 04:39:45.364 | INFO | } | 20260508 04:39:45.364 | INFO | Perf report for the last 5.14385 seconds: | 20260508 04:39:45.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:45.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 04:39:45.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:39:45.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:45.364 | INFO | Uploaded 36GiB to GPU in 2.54451s (14.1481GiB/s) | 20260508 04:39:45.364 | INFO | Downloaded 34GiB to GPU in 2.43869s (13.9419GiB/s) | 20260508 04:39:50.506 | INFO | Analyzing chunk frames = [5305856,5306368) | 20260508 04:39:50.506 | INFO | MemInfo: { | 20260508 04:39:50.506 | INFO | "gpu" : " 17 GB", | 20260508 04:39:50.506 | INFO | "huge" : "", | 20260508 04:39:50.506 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:50.506 | INFO | "malloc" : " 14 GB", | 20260508 04:39:50.506 | INFO | "pinned" : " 29 GB" | 20260508 04:39:50.506 | INFO | } | 20260508 04:39:50.506 | INFO | Perf report for the last 5.14241 seconds: | 20260508 04:39:50.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:50.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | ANALYSIS : 67.0%, 2.9, 103.8ms, 0 | 20260508 04:39:50.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:39:50.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:50.506 | INFO | Uploaded 36GiB to GPU in 2.57253s (13.994GiB/s) | 20260508 04:39:50.506 | INFO | Downloaded 34GiB to GPU in 2.42309s (14.0317GiB/s) | 20260508 04:39:55.639 | INFO | Analyzing chunk frames = [5306368,5306880) | 20260508 04:39:55.639 | INFO | MemInfo: { | 20260508 04:39:55.639 | INFO | "gpu" : " 17 GB", | 20260508 04:39:55.639 | INFO | "huge" : "", | 20260508 04:39:55.639 | INFO | "huge_shared" : " 29 GB", | 20260508 04:39:55.639 | INFO | "malloc" : " 14 GB", | 20260508 04:39:55.639 | INFO | "pinned" : " 29 GB" | 20260508 04:39:55.639 | INFO | } | 20260508 04:39:55.639 | INFO | Perf report for the last 5.13312 seconds: | 20260508 04:39:55.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:39:55.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.639 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 04:39:55.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.639 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 04:39:55.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:39:55.640 | INFO | Uploaded 36GiB to GPU in 2.31398s (15.5576GiB/s) | 20260508 04:39:55.640 | INFO | Downloaded 34GiB to GPU in 2.55263s (13.3196GiB/s) | 20260508 04:40:00.782 | INFO | Analyzing chunk frames = [5306880,5307392) | 20260508 04:40:00.782 | INFO | MemInfo: { | 20260508 04:40:00.782 | INFO | "gpu" : " 17 GB", | 20260508 04:40:00.782 | INFO | "huge" : "", | 20260508 04:40:00.782 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:00.782 | INFO | "malloc" : " 14 GB", | 20260508 04:40:00.782 | INFO | "pinned" : " 29 GB" | 20260508 04:40:00.782 | INFO | } | 20260508 04:40:00.782 | INFO | Perf report for the last 5.14318 seconds: | 20260508 04:40:00.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:00.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.782 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:40:00.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.783 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260508 04:40:00.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:00.783 | INFO | Uploaded 36GiB to GPU in 2.4118s (14.9266GiB/s) | 20260508 04:40:00.783 | INFO | Downloaded 34GiB to GPU in 2.44132s (13.9269GiB/s) | 20260508 04:40:05.916 | INFO | Analyzing chunk frames = [5307392,5307904) | 20260508 04:40:05.916 | INFO | MemInfo: { | 20260508 04:40:05.916 | INFO | "gpu" : " 17 GB", | 20260508 04:40:05.916 | INFO | "huge" : "", | 20260508 04:40:05.916 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:05.916 | INFO | "malloc" : " 14 GB", | 20260508 04:40:05.917 | INFO | "pinned" : " 29 GB" | 20260508 04:40:05.917 | INFO | } | 20260508 04:40:05.917 | INFO | Perf report for the last 5.1342 seconds: | 20260508 04:40:05.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:05.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:40:05.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:40:05.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:05.917 | INFO | Uploaded 36GiB to GPU in 2.39077s (15.0579GiB/s) | 20260508 04:40:05.917 | INFO | Downloaded 34GiB to GPU in 2.44162s (13.9252GiB/s) | 20260508 04:40:11.056 | INFO | Analyzing chunk frames = [5307904,5308416) | 20260508 04:40:11.056 | INFO | MemInfo: { | 20260508 04:40:11.056 | INFO | "gpu" : " 17 GB", | 20260508 04:40:11.056 | INFO | "huge" : "", | 20260508 04:40:11.057 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:11.057 | INFO | "malloc" : " 14 GB", | 20260508 04:40:11.057 | INFO | "pinned" : " 29 GB" | 20260508 04:40:11.057 | INFO | } | 20260508 04:40:11.057 | INFO | Perf report for the last 5.14 seconds: | 20260508 04:40:11.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:11.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:40:11.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:40:11.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:11.057 | INFO | Uploaded 36GiB to GPU in 2.42321s (14.8563GiB/s) | 20260508 04:40:11.057 | INFO | Downloaded 34GiB to GPU in 2.44238s (13.9208GiB/s) | 20260508 04:40:11.057 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:40:11.057 | INFO | Total mem usage: 16.7 GiB | 20260508 04:40:11.057 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:40:11.057 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.4%) | 20260508 04:40:11.057 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:40:11.057 | INFO | High Water Breakdown: | 20260508 04:40:11.057 | INFO | 59.7% : 8.63 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:40:11.057 | INFO | 40.2% : 5.81 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:40:11.057 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:40:11.057 | INFO | ------------------------------------------ | 20260508 04:40:11.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:40:11.057 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:40:11.057 | INFO | Total mem usage: 26.5 GiB | 20260508 04:40:11.057 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:40:11.057 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:40:11.057 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:40:11.057 | INFO | High Water Breakdown: | 20260508 04:40:11.057 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:40:11.057 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:40:11.057 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:40:11.057 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:40:11.057 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:40:11.057 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:40:11.057 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:40:11.057 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:40:11.057 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:40:11.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:40:11.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:40:11.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:40:11.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:40:11.057 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:40:11.058 | INFO | ------------------------------------------ | 20260508 04:40:11.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:40:11.058 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:40:11.058 | INFO | Total mem usage: 27.8 GiB | 20260508 04:40:11.058 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:40:11.058 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:40:11.058 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:40:11.058 | INFO | High Water Breakdown: | 20260508 04:40:11.058 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:40:11.058 | INFO | ------------------------------------------ | 20260508 04:40:11.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:40:11.058 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:40:11.058 | INFO | Total mem usage: 15.7 GiB | 20260508 04:40:11.058 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:40:11.058 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:40:11.058 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:40:11.058 | INFO | High Water Breakdown: | 20260508 04:40:11.058 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:40:11.058 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:40:11.058 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:40:11.058 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:40:11.058 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:40:11.058 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:40:11.058 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:40:11.058 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:40:11.058 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:40:11.058 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:40:11.058 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:40:11.058 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:40:11.058 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:40:11.058 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:40:11.058 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:40:11.058 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:40:11.058 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:40:11.058 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:40:11.058 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:40:11.058 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:40:11.058 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:40:11.058 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:40:11.058 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:40:11.058 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:40:11.058 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:40:11.058 | INFO | ------------------------------------------ | 20260508 04:40:11.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:40:11.058 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:40:11.058 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:40:11.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:40:11.058 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 1.55e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.10e-02 / 6.74e-03 | 20260508 04:40:11.058 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.46e+00, 5.81e-01 / 2.04e-01 | 20260508 04:40:11.058 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.16e-01 | 20260508 04:40:11.058 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.52e+01, 3.60e+00 / 1.77e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.79e+00 / 1.40e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.58e+00, 4.20e-01 / 2.69e-01 | 20260508 04:40:11.058 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.23e+00, 2.65e-01 / 1.54e-01 | 20260508 04:40:11.058 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.54e-01, 5.14e-02 / 2.57e-02 | 20260508 04:40:11.058 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:40:11.058 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:40:11.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:40:11.058 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 24.28%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 24.35%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 2.08%, LockUpload -- 2.21e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 2.76%, LockDownload -- 2.93e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:40:11.058 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.57e-02 / 0.00e+00 | 20260508 04:40:11.154 | INFO | Starting a round of Baz IO | 20260508 04:40:14.583 | INFO | Populating RTMetrics report | 20260508 04:40:14.583 | INFO | RTMetrics report populated | 20260508 04:40:14.583 | INFO | Opening tmp RTMetrics json file | 20260508 04:40:14.599 | INFO | Serializing RTMetrics to json | 20260508 04:40:14.601 | INFO | Serialization done, swapping tmp file now... | 20260508 04:40:14.601 | INFO | RTMetrics done | 20260508 04:40:14.658 | INFO | Finished a round of Baz IO: Wrote 1.59 GiB in 3.504 seconds | 20260508 04:40:14.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.02% / 0.00% / 58.98% | 20260508 04:40:14.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.519133% / 7.94729e-06% / 6.75519e-05% | 20260508 04:40:14.658 | INFO | Baz IO overall: | 20260508 04:40:14.658 | INFO | BytesWritten : 2374261590194 | 20260508 04:40:14.658 | INFO | NumEvents : 1146126053808 | 20260508 04:40:16.191 | INFO | Analyzing chunk frames = [5308416,5308928) | 20260508 04:40:16.191 | INFO | MemInfo: { | 20260508 04:40:16.191 | INFO | "gpu" : " 17 GB", | 20260508 04:40:16.191 | INFO | "huge" : "", | 20260508 04:40:16.191 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:16.191 | INFO | "malloc" : " 14 GB", | 20260508 04:40:16.191 | INFO | "pinned" : " 29 GB" | 20260508 04:40:16.191 | INFO | } | 20260508 04:40:16.191 | INFO | Perf report for the last 5.13463 seconds: | 20260508 04:40:16.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:16.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:16.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:16.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:16.191 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 04:40:16.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:16.191 | INFO | RT_METRICS : 39.8%, 1.0, 20.8ms, 0 | 20260508 04:40:16.191 | INFO | PRE_HQ : 65.5%, 1.6, 55.8ms, 0 | 20260508 04:40:16.192 | INFO | BAZWRITER : 67.8%, 2.7, 93.7ms, 0 | 20260508 04:40:16.192 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:40:16.192 | INFO | Uploaded 36GiB to GPU in 2.55207s (14.1062GiB/s) | 20260508 04:40:16.192 | INFO | Downloaded 38GiB to GPU in 2.70688s (14.0383GiB/s) | 20260508 04:40:21.337 | INFO | Analyzing chunk frames = [5308928,5309440) | 20260508 04:40:21.337 | INFO | MemInfo: { | 20260508 04:40:21.337 | INFO | "gpu" : " 17 GB", | 20260508 04:40:21.337 | INFO | "huge" : "", | 20260508 04:40:21.337 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:21.337 | INFO | "malloc" : " 14 GB", | 20260508 04:40:21.337 | INFO | "pinned" : " 29 GB" | 20260508 04:40:21.337 | INFO | } | 20260508 04:40:21.337 | INFO | Perf report for the last 5.14598 seconds: | 20260508 04:40:21.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:21.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.337 | INFO | ANALYSIS : 67.2%, 3.0, 104.8ms, 0 | 20260508 04:40:21.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.337 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:40:21.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:21.338 | INFO | Uploaded 36GiB to GPU in 2.53134s (14.2217GiB/s) | 20260508 04:40:21.338 | INFO | Downloaded 34GiB to GPU in 2.439s (13.9401GiB/s) | 20260508 04:40:26.476 | INFO | Analyzing chunk frames = [5309440,5309952) | 20260508 04:40:26.476 | INFO | MemInfo: { | 20260508 04:40:26.476 | INFO | "gpu" : " 17 GB", | 20260508 04:40:26.477 | INFO | "huge" : "", | 20260508 04:40:26.477 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:26.477 | INFO | "malloc" : " 14 GB", | 20260508 04:40:26.477 | INFO | "pinned" : " 29 GB" | 20260508 04:40:26.477 | INFO | } | 20260508 04:40:26.477 | INFO | Perf report for the last 5.13948 seconds: | 20260508 04:40:26.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:26.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 04:40:26.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:40:26.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:26.477 | INFO | Uploaded 36GiB to GPU in 2.50674s (14.3613GiB/s) | 20260508 04:40:26.477 | INFO | Downloaded 34GiB to GPU in 2.43265s (13.9765GiB/s) | 20260508 04:40:31.617 | INFO | Analyzing chunk frames = [5309952,5310464) | 20260508 04:40:31.617 | INFO | MemInfo: { | 20260508 04:40:31.617 | INFO | "gpu" : " 17 GB", | 20260508 04:40:31.617 | INFO | "huge" : "", | 20260508 04:40:31.617 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:31.617 | INFO | "malloc" : " 14 GB", | 20260508 04:40:31.617 | INFO | "pinned" : " 29 GB" | 20260508 04:40:31.617 | INFO | } | 20260508 04:40:31.617 | INFO | Perf report for the last 5.14065 seconds: | 20260508 04:40:31.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:31.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.617 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:40:31.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.618 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:40:31.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:31.618 | INFO | Uploaded 36GiB to GPU in 2.54443s (14.1485GiB/s) | 20260508 04:40:31.618 | INFO | Downloaded 34GiB to GPU in 2.43705s (13.9513GiB/s) | 20260508 04:40:36.753 | INFO | Analyzing chunk frames = [5310464,5310976) | 20260508 04:40:36.753 | INFO | MemInfo: { | 20260508 04:40:36.753 | INFO | "gpu" : " 17 GB", | 20260508 04:40:36.753 | INFO | "huge" : "", | 20260508 04:40:36.753 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:36.753 | INFO | "malloc" : " 14 GB", | 20260508 04:40:36.753 | INFO | "pinned" : " 29 GB" | 20260508 04:40:36.753 | INFO | } | 20260508 04:40:36.753 | INFO | Perf report for the last 5.13574 seconds: | 20260508 04:40:36.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:36.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.753 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 04:40:36.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.753 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:40:36.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:36.754 | INFO | Uploaded 36GiB to GPU in 2.31705s (15.537GiB/s) | 20260508 04:40:36.754 | INFO | Downloaded 34GiB to GPU in 2.55773s (13.293GiB/s) | 20260508 04:40:41.890 | INFO | Analyzing chunk frames = [5310976,5311488) | 20260508 04:40:41.890 | INFO | MemInfo: { | 20260508 04:40:41.890 | INFO | "gpu" : " 17 GB", | 20260508 04:40:41.890 | INFO | "huge" : "", | 20260508 04:40:41.890 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:41.890 | INFO | "malloc" : " 14 GB", | 20260508 04:40:41.890 | INFO | "pinned" : " 29 GB" | 20260508 04:40:41.890 | INFO | } | 20260508 04:40:41.890 | INFO | Perf report for the last 5.137 seconds: | 20260508 04:40:41.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:41.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 04:40:41.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:40:41.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:41.890 | INFO | Uploaded 36GiB to GPU in 2.38823s (15.0739GiB/s) | 20260508 04:40:41.890 | INFO | Downloaded 34GiB to GPU in 2.43656s (13.9541GiB/s) | 20260508 04:40:47.035 | INFO | Analyzing chunk frames = [5311488,5312000) | 20260508 04:40:47.035 | INFO | MemInfo: { | 20260508 04:40:47.035 | INFO | "gpu" : " 17 GB", | 20260508 04:40:47.035 | INFO | "huge" : "", | 20260508 04:40:47.035 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:47.035 | INFO | "malloc" : " 14 GB", | 20260508 04:40:47.035 | INFO | "pinned" : " 29 GB" | 20260508 04:40:47.035 | INFO | } | 20260508 04:40:47.035 | INFO | Perf report for the last 5.14505 seconds: | 20260508 04:40:47.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:47.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:40:47.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 04:40:47.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:47.035 | INFO | Uploaded 36GiB to GPU in 2.39161s (15.0526GiB/s) | 20260508 04:40:47.036 | INFO | Downloaded 34GiB to GPU in 2.445s (13.906GiB/s) | 20260508 04:40:52.172 | INFO | Analyzing chunk frames = [5312000,5312512) | 20260508 04:40:52.172 | INFO | MemInfo: { | 20260508 04:40:52.172 | INFO | "gpu" : " 17 GB", | 20260508 04:40:52.172 | INFO | "huge" : "", | 20260508 04:40:52.172 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:52.172 | INFO | "malloc" : " 14 GB", | 20260508 04:40:52.172 | INFO | "pinned" : " 29 GB" | 20260508 04:40:52.172 | INFO | } | 20260508 04:40:52.172 | INFO | Perf report for the last 5.13702 seconds: | 20260508 04:40:52.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:52.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.172 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:40:52.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.172 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:40:52.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:52.173 | INFO | Uploaded 36GiB to GPU in 2.40494s (14.9692GiB/s) | 20260508 04:40:52.173 | INFO | Downloaded 34GiB to GPU in 2.43081s (13.9871GiB/s) | 20260508 04:40:52.270 | INFO | Starting a round of Baz IO | 20260508 04:40:55.704 | INFO | Populating RTMetrics report | 20260508 04:40:55.704 | INFO | RTMetrics report populated | 20260508 04:40:55.704 | INFO | Opening tmp RTMetrics json file | 20260508 04:40:55.714 | INFO | Serializing RTMetrics to json | 20260508 04:40:55.719 | INFO | Serialization done, swapping tmp file now... | 20260508 04:40:55.719 | INFO | RTMetrics done | 20260508 04:40:55.772 | INFO | Finished a round of Baz IO: Wrote 1.59 GiB in 3.502 seconds | 20260508 04:40:55.772 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.02% / 0.00% / 58.98% | 20260508 04:40:55.772 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.518978% / 1.19209e-05% / 8.74201e-05% | 20260508 04:40:55.772 | INFO | Baz IO overall: | 20260508 04:40:55.772 | INFO | BytesWritten : 2375969305971 | 20260508 04:40:55.772 | INFO | NumEvents : 1146941162558 | 20260508 04:40:57.317 | INFO | Analyzing chunk frames = [5312512,5313024) | 20260508 04:40:57.317 | INFO | MemInfo: { | 20260508 04:40:57.317 | INFO | "gpu" : " 17 GB", | 20260508 04:40:57.317 | INFO | "huge" : "", | 20260508 04:40:57.317 | INFO | "huge_shared" : " 29 GB", | 20260508 04:40:57.317 | INFO | "malloc" : " 14 GB", | 20260508 04:40:57.317 | INFO | "pinned" : " 29 GB" | 20260508 04:40:57.317 | INFO | } | 20260508 04:40:57.317 | INFO | Perf report for the last 5.14513 seconds: | 20260508 04:40:57.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:40:57.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:57.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:57.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:57.317 | INFO | ANALYSIS : 67.7%, 3.0, 106.0ms, 0 | 20260508 04:40:57.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:40:57.318 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 04:40:57.318 | INFO | PRE_HQ : 65.3%, 1.7, 56.0ms, 0 | 20260508 04:40:57.318 | INFO | BAZWRITER : 67.6%, 2.7, 93.8ms, 0 | 20260508 04:40:57.318 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 04:40:57.318 | INFO | Uploaded 36GiB to GPU in 2.5419s (14.1627GiB/s) | 20260508 04:40:57.318 | INFO | Downloaded 38GiB to GPU in 2.68905s (14.1314GiB/s) | 20260508 04:41:02.451 | INFO | Analyzing chunk frames = [5313024,5313536) | 20260508 04:41:02.451 | INFO | MemInfo: { | 20260508 04:41:02.451 | INFO | "gpu" : " 17 GB", | 20260508 04:41:02.451 | INFO | "huge" : "", | 20260508 04:41:02.451 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:02.451 | INFO | "malloc" : " 14 GB", | 20260508 04:41:02.451 | INFO | "pinned" : " 29 GB" | 20260508 04:41:02.451 | INFO | } | 20260508 04:41:02.451 | INFO | Perf report for the last 5.13399 seconds: | 20260508 04:41:02.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:02.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.451 | INFO | ANALYSIS : 67.5%, 3.0, 105.0ms, 0 | 20260508 04:41:02.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.452 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:41:02.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:02.452 | INFO | Uploaded 36GiB to GPU in 2.55303s (14.1009GiB/s) | 20260508 04:41:02.452 | INFO | Downloaded 34GiB to GPU in 2.43952s (13.9372GiB/s) | 20260508 04:41:07.589 | INFO | Analyzing chunk frames = [5313536,5314048) | 20260508 04:41:07.589 | INFO | MemInfo: { | 20260508 04:41:07.589 | INFO | "gpu" : " 17 GB", | 20260508 04:41:07.589 | INFO | "huge" : "", | 20260508 04:41:07.589 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:07.589 | INFO | "malloc" : " 14 GB", | 20260508 04:41:07.589 | INFO | "pinned" : " 29 GB" | 20260508 04:41:07.589 | INFO | } | 20260508 04:41:07.589 | INFO | Perf report for the last 5.13736 seconds: | 20260508 04:41:07.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:07.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:41:07.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:41:07.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:07.589 | INFO | Uploaded 36GiB to GPU in 2.51471s (14.3158GiB/s) | 20260508 04:41:07.589 | INFO | Downloaded 34GiB to GPU in 2.44359s (13.914GiB/s) | 20260508 04:41:12.732 | INFO | Analyzing chunk frames = [5314048,5314560) | 20260508 04:41:12.732 | INFO | MemInfo: { | 20260508 04:41:12.732 | INFO | "gpu" : " 17 GB", | 20260508 04:41:12.732 | INFO | "huge" : "", | 20260508 04:41:12.732 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:12.732 | INFO | "malloc" : " 14 GB", | 20260508 04:41:12.732 | INFO | "pinned" : " 29 GB" | 20260508 04:41:12.732 | INFO | } | 20260508 04:41:12.732 | INFO | Perf report for the last 5.14364 seconds: | 20260508 04:41:12.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:12.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.733 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 04:41:12.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.733 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 04:41:12.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:12.733 | INFO | Uploaded 36GiB to GPU in 2.52732s (14.2443GiB/s) | 20260508 04:41:12.733 | INFO | Downloaded 34GiB to GPU in 2.43331s (13.9728GiB/s) | 20260508 04:41:17.870 | INFO | Analyzing chunk frames = [5314560,5315072) | 20260508 04:41:17.870 | INFO | MemInfo: { | 20260508 04:41:17.870 | INFO | "gpu" : " 17 GB", | 20260508 04:41:17.870 | INFO | "huge" : "", | 20260508 04:41:17.870 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:17.870 | INFO | "malloc" : " 14 GB", | 20260508 04:41:17.871 | INFO | "pinned" : " 29 GB" | 20260508 04:41:17.871 | INFO | } | 20260508 04:41:17.871 | INFO | Perf report for the last 5.13816 seconds: | 20260508 04:41:17.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:17.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:41:17.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:41:17.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:17.871 | INFO | Uploaded 36GiB to GPU in 2.3073s (15.6027GiB/s) | 20260508 04:41:17.871 | INFO | Downloaded 34GiB to GPU in 2.56419s (13.2595GiB/s) | 20260508 04:41:23.008 | INFO | Analyzing chunk frames = [5315072,5315584) | 20260508 04:41:23.008 | INFO | MemInfo: { | 20260508 04:41:23.008 | INFO | "gpu" : " 17 GB", | 20260508 04:41:23.008 | INFO | "huge" : "", | 20260508 04:41:23.008 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:23.008 | INFO | "malloc" : " 14 GB", | 20260508 04:41:23.008 | INFO | "pinned" : " 29 GB" | 20260508 04:41:23.008 | INFO | } | 20260508 04:41:23.008 | INFO | Perf report for the last 5.1376 seconds: | 20260508 04:41:23.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:23.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.008 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 04:41:23.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.008 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:41:23.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:23.009 | INFO | Uploaded 36GiB to GPU in 2.39582s (15.0262GiB/s) | 20260508 04:41:23.009 | INFO | Downloaded 34GiB to GPU in 2.44756s (13.8914GiB/s) | 20260508 04:41:28.142 | INFO | Analyzing chunk frames = [5315584,5316096) | 20260508 04:41:28.142 | INFO | MemInfo: { | 20260508 04:41:28.142 | INFO | "gpu" : " 17 GB", | 20260508 04:41:28.142 | INFO | "huge" : "", | 20260508 04:41:28.142 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:28.142 | INFO | "malloc" : " 14 GB", | 20260508 04:41:28.142 | INFO | "pinned" : " 29 GB" | 20260508 04:41:28.142 | INFO | } | 20260508 04:41:28.142 | INFO | Perf report for the last 5.13398 seconds: | 20260508 04:41:28.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:28.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.142 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:41:28.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.142 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:41:28.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:28.143 | INFO | Uploaded 36GiB to GPU in 2.40202s (14.9874GiB/s) | 20260508 04:41:28.143 | INFO | Downloaded 34GiB to GPU in 2.43921s (13.939GiB/s) | 20260508 04:41:33.285 | INFO | Analyzing chunk frames = [5316096,5316608) | 20260508 04:41:33.285 | INFO | MemInfo: { | 20260508 04:41:33.285 | INFO | "gpu" : " 17 GB", | 20260508 04:41:33.285 | INFO | "huge" : "", | 20260508 04:41:33.285 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:33.285 | INFO | "malloc" : " 14 GB", | 20260508 04:41:33.285 | INFO | "pinned" : " 29 GB" | 20260508 04:41:33.285 | INFO | } | 20260508 04:41:33.285 | INFO | Perf report for the last 5.1431 seconds: | 20260508 04:41:33.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:33.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.285 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:41:33.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.285 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:41:33.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:33.286 | INFO | Uploaded 36GiB to GPU in 2.4212s (14.8687GiB/s) | 20260508 04:41:33.286 | INFO | Downloaded 34GiB to GPU in 2.43272s (13.9761GiB/s) | 20260508 04:41:33.286 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:41:33.286 | INFO | Total mem usage: 16.7 GiB | 20260508 04:41:33.286 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:41:33.286 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 04:41:33.286 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:41:33.286 | INFO | High Water Breakdown: | 20260508 04:41:33.286 | INFO | 59.5% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:41:33.286 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:41:33.286 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:41:33.286 | INFO | ------------------------------------------ | 20260508 04:41:33.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:41:33.286 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:41:33.286 | INFO | Total mem usage: 26.5 GiB | 20260508 04:41:33.286 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:41:33.286 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:41:33.286 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:41:33.286 | INFO | High Water Breakdown: | 20260508 04:41:33.286 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:41:33.286 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:41:33.286 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:41:33.286 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:41:33.286 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:41:33.286 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:41:33.286 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:41:33.286 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:41:33.286 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:41:33.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:41:33.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:41:33.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:41:33.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:41:33.286 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:41:33.286 | INFO | ------------------------------------------ | 20260508 04:41:33.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:41:33.286 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:41:33.286 | INFO | Total mem usage: 27.8 GiB | 20260508 04:41:33.286 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:41:33.286 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:41:33.286 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:41:33.286 | INFO | High Water Breakdown: | 20260508 04:41:33.286 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:41:33.286 | INFO | ------------------------------------------ | 20260508 04:41:33.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:41:33.286 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:41:33.286 | INFO | Total mem usage: 15.7 GiB | 20260508 04:41:33.286 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:41:33.286 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:41:33.286 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:41:33.286 | INFO | High Water Breakdown: | 20260508 04:41:33.286 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:41:33.286 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:41:33.286 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:41:33.286 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:41:33.286 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:41:33.286 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:41:33.286 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:41:33.286 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:41:33.286 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:41:33.286 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:41:33.286 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:41:33.286 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:41:33.286 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:41:33.286 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:41:33.286 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:41:33.286 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:41:33.286 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:41:33.286 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:41:33.286 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:41:33.286 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:41:33.286 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:41:33.286 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:41:33.286 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:41:33.286 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:41:33.286 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:41:33.286 | INFO | ------------------------------------------ | 20260508 04:41:33.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:41:33.286 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:41:33.286 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:41:33.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:41:33.286 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.39e+01 / 1.33e+00 | 20260508 04:41:33.286 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.09e-02 / 4.37e-03 | 20260508 04:41:33.286 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.48e+00, 5.92e-01 / 2.10e-01 | 20260508 04:41:33.286 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.18e-01 | 20260508 04:41:33.287 | INFO | PerfReport: 15.87%, DME -- 4.16e+01 / 4.50e+01, 3.55e+00 / 1.75e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.32e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.56e+00, 4.22e-01 / 2.25e-01 | 20260508 04:41:33.287 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.21e+00, 2.64e-01 / 1.52e-01 | 20260508 04:41:33.287 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.51e-01, 5.04e-02 / 2.16e-02 | 20260508 04:41:33.287 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:41:33.287 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:41:33.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:41:33.287 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 24.11%, Upload -- 2.56e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 24.40%, Download -- 2.59e+01 / 0.00e+00, 8.35e+00 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 30.87%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 1.93%, LockUpload -- 2.05e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 2.96%, LockDownload -- 3.14e+00 / 0.00e+00, 4.55e+00 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 15.72%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:41:33.287 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260508 04:41:33.387 | INFO | Starting a round of Baz IO | 20260508 04:41:36.823 | INFO | Populating RTMetrics report | 20260508 04:41:36.823 | INFO | RTMetrics report populated | 20260508 04:41:36.823 | INFO | Opening tmp RTMetrics json file | 20260508 04:41:36.839 | INFO | Serializing RTMetrics to json | 20260508 04:41:36.841 | INFO | Serialization done, swapping tmp file now... | 20260508 04:41:36.842 | INFO | RTMetrics done | 20260508 04:41:36.893 | INFO | Finished a round of Baz IO: Wrote 1.592 GiB in 3.505 seconds | 20260508 04:41:36.893 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.01% / 0.00% / 58.98% | 20260508 04:41:36.893 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.519808% / 7.94729e-06% / 6.35783e-05% | 20260508 04:41:36.893 | INFO | Baz IO overall: | 20260508 04:41:36.893 | INFO | BytesWritten : 2377678766995 | 20260508 04:41:36.893 | INFO | NumEvents : 1147757804910 | 20260508 04:41:38.427 | INFO | Analyzing chunk frames = [5316608,5317120) | 20260508 04:41:38.427 | INFO | MemInfo: { | 20260508 04:41:38.427 | INFO | "gpu" : " 17 GB", | 20260508 04:41:38.427 | INFO | "huge" : "", | 20260508 04:41:38.427 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:38.427 | INFO | "malloc" : " 14 GB", | 20260508 04:41:38.427 | INFO | "pinned" : " 29 GB" | 20260508 04:41:38.427 | INFO | } | 20260508 04:41:38.427 | INFO | Perf report for the last 5.14197 seconds: | 20260508 04:41:38.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:38.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:38.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:38.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:38.427 | INFO | ANALYSIS : 67.9%, 3.0, 106.2ms, 0 | 20260508 04:41:38.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:38.427 | INFO | RT_METRICS : 40.3%, 1.0, 21.1ms, 0 | 20260508 04:41:38.427 | INFO | PRE_HQ : 65.4%, 1.6, 56.0ms, 0 | 20260508 04:41:38.427 | INFO | BAZWRITER : 67.8%, 2.7, 93.3ms, 0 | 20260508 04:41:38.427 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 04:41:38.428 | INFO | Uploaded 36GiB to GPU in 2.56819s (14.0176GiB/s) | 20260508 04:41:38.428 | INFO | Downloaded 38GiB to GPU in 2.69115s (14.1204GiB/s) | 20260508 04:41:43.570 | INFO | Analyzing chunk frames = [5317120,5317632) | 20260508 04:41:43.570 | INFO | MemInfo: { | 20260508 04:41:43.570 | INFO | "gpu" : " 17 GB", | 20260508 04:41:43.570 | INFO | "huge" : "", | 20260508 04:41:43.570 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:43.570 | INFO | "malloc" : " 14 GB", | 20260508 04:41:43.570 | INFO | "pinned" : " 29 GB" | 20260508 04:41:43.570 | INFO | } | 20260508 04:41:43.570 | INFO | Perf report for the last 5.14291 seconds: | 20260508 04:41:43.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:43.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.570 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 04:41:43.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.570 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:41:43.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:43.571 | INFO | Uploaded 36GiB to GPU in 2.55433s (14.0937GiB/s) | 20260508 04:41:43.571 | INFO | Downloaded 34GiB to GPU in 2.44394s (13.9119GiB/s) | 20260508 04:41:48.704 | INFO | Analyzing chunk frames = [5317632,5318144) | 20260508 04:41:48.704 | INFO | MemInfo: { | 20260508 04:41:48.704 | INFO | "gpu" : " 17 GB", | 20260508 04:41:48.704 | INFO | "huge" : "", | 20260508 04:41:48.704 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:48.704 | INFO | "malloc" : " 14 GB", | 20260508 04:41:48.704 | INFO | "pinned" : " 29 GB" | 20260508 04:41:48.704 | INFO | } | 20260508 04:41:48.704 | INFO | Perf report for the last 5.13407 seconds: | 20260508 04:41:48.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:48.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.704 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 04:41:48.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.704 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:41:48.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:48.705 | INFO | Uploaded 36GiB to GPU in 2.50898s (14.3485GiB/s) | 20260508 04:41:48.705 | INFO | Downloaded 34GiB to GPU in 2.42863s (13.9997GiB/s) | 20260508 04:41:53.847 | INFO | Analyzing chunk frames = [5318144,5318656) | 20260508 04:41:53.847 | INFO | MemInfo: { | 20260508 04:41:53.847 | INFO | "gpu" : " 17 GB", | 20260508 04:41:53.847 | INFO | "huge" : "", | 20260508 04:41:53.847 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:53.847 | INFO | "malloc" : " 14 GB", | 20260508 04:41:53.847 | INFO | "pinned" : " 29 GB" | 20260508 04:41:53.847 | INFO | } | 20260508 04:41:53.847 | INFO | Perf report for the last 5.1426 seconds: | 20260508 04:41:53.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:53.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.847 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 04:41:53.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.847 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:41:53.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:53.848 | INFO | Uploaded 36GiB to GPU in 2.55271s (14.1027GiB/s) | 20260508 04:41:53.848 | INFO | Downloaded 34GiB to GPU in 2.44763s (13.891GiB/s) | 20260508 04:41:58.986 | INFO | Analyzing chunk frames = [5318656,5319168) | 20260508 04:41:58.986 | INFO | MemInfo: { | 20260508 04:41:58.986 | INFO | "gpu" : " 17 GB", | 20260508 04:41:58.986 | INFO | "huge" : "", | 20260508 04:41:58.986 | INFO | "huge_shared" : " 29 GB", | 20260508 04:41:58.986 | INFO | "malloc" : " 14 GB", | 20260508 04:41:58.986 | INFO | "pinned" : " 29 GB" | 20260508 04:41:58.986 | INFO | } | 20260508 04:41:58.987 | INFO | Perf report for the last 5.13954 seconds: | 20260508 04:41:58.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:41:58.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:41:58.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:41:58.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:41:58.987 | INFO | Uploaded 36GiB to GPU in 2.33474s (15.4192GiB/s) | 20260508 04:41:58.987 | INFO | Downloaded 34GiB to GPU in 2.56412s (13.2599GiB/s) | 20260508 04:42:04.123 | INFO | Analyzing chunk frames = [5319168,5319680) | 20260508 04:42:04.123 | INFO | MemInfo: { | 20260508 04:42:04.123 | INFO | "gpu" : " 17 GB", | 20260508 04:42:04.123 | INFO | "huge" : "", | 20260508 04:42:04.123 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:04.123 | INFO | "malloc" : " 14 GB", | 20260508 04:42:04.123 | INFO | "pinned" : " 29 GB" | 20260508 04:42:04.123 | INFO | } | 20260508 04:42:04.124 | INFO | Perf report for the last 5.13699 seconds: | 20260508 04:42:04.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:04.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 04:42:04.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:42:04.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:04.124 | INFO | Uploaded 36GiB to GPU in 2.42614s (14.8384GiB/s) | 20260508 04:42:04.124 | INFO | Downloaded 34GiB to GPU in 2.41515s (14.0778GiB/s) | 20260508 04:42:09.264 | INFO | Analyzing chunk frames = [5319680,5320192) | 20260508 04:42:09.264 | INFO | MemInfo: { | 20260508 04:42:09.264 | INFO | "gpu" : " 17 GB", | 20260508 04:42:09.264 | INFO | "huge" : "", | 20260508 04:42:09.264 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:09.264 | INFO | "malloc" : " 14 GB", | 20260508 04:42:09.264 | INFO | "pinned" : " 29 GB" | 20260508 04:42:09.264 | INFO | } | 20260508 04:42:09.265 | INFO | Perf report for the last 5.14103 seconds: | 20260508 04:42:09.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:09.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:42:09.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:42:09.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:09.265 | INFO | Uploaded 36GiB to GPU in 2.4441s (14.7293GiB/s) | 20260508 04:42:09.265 | INFO | Downloaded 34GiB to GPU in 2.43092s (13.9865GiB/s) | 20260508 04:42:14.400 | INFO | Analyzing chunk frames = [5320192,5320704) | 20260508 04:42:14.400 | INFO | MemInfo: { | 20260508 04:42:14.400 | INFO | "gpu" : " 17 GB", | 20260508 04:42:14.400 | INFO | "huge" : "", | 20260508 04:42:14.400 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:14.400 | INFO | "malloc" : " 14 GB", | 20260508 04:42:14.400 | INFO | "pinned" : " 29 GB" | 20260508 04:42:14.400 | INFO | } | 20260508 04:42:14.400 | INFO | Perf report for the last 5.13554 seconds: | 20260508 04:42:14.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:14.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.400 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:42:14.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.400 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 04:42:14.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:14.401 | INFO | Uploaded 36GiB to GPU in 2.4321s (14.802GiB/s) | 20260508 04:42:14.401 | INFO | Downloaded 34GiB to GPU in 2.42647s (14.0121GiB/s) | 20260508 04:42:14.497 | INFO | Starting a round of Baz IO | 20260508 04:42:17.930 | INFO | Populating RTMetrics report | 20260508 04:42:17.930 | INFO | RTMetrics report populated | 20260508 04:42:17.930 | INFO | Opening tmp RTMetrics json file | 20260508 04:42:17.939 | INFO | Serializing RTMetrics to json | 20260508 04:42:17.944 | INFO | Serialization done, swapping tmp file now... | 20260508 04:42:17.945 | INFO | RTMetrics done | 20260508 04:42:18.007 | INFO | Finished a round of Baz IO: Wrote 1.591 GiB in 3.509 seconds | 20260508 04:42:18.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.01% / 0.00% / 58.99% | 20260508 04:42:18.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.518068% / 7.94729e-06% / 9.93411e-05% | 20260508 04:42:18.007 | INFO | Baz IO overall: | 20260508 04:42:18.007 | INFO | BytesWritten : 2379387196710 | 20260508 04:42:18.007 | INFO | NumEvents : 1148573723860 | 20260508 04:42:19.541 | INFO | Analyzing chunk frames = [5320704,5321216) | 20260508 04:42:19.541 | INFO | MemInfo: { | 20260508 04:42:19.541 | INFO | "gpu" : " 17 GB", | 20260508 04:42:19.541 | INFO | "huge" : "", | 20260508 04:42:19.541 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:19.541 | INFO | "malloc" : " 14 GB", | 20260508 04:42:19.541 | INFO | "pinned" : " 29 GB" | 20260508 04:42:19.541 | INFO | } | 20260508 04:42:19.541 | INFO | Perf report for the last 5.14115 seconds: | 20260508 04:42:19.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:19.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:19.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:19.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:19.541 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 04:42:19.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:19.541 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:42:19.541 | INFO | PRE_HQ : 65.4%, 1.6, 55.6ms, 0 | 20260508 04:42:19.541 | INFO | BAZWRITER : 67.7%, 2.7, 93.8ms, 0 | 20260508 04:42:19.541 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:42:19.542 | INFO | Uploaded 36GiB to GPU in 2.5931s (13.883GiB/s) | 20260508 04:42:19.542 | INFO | Downloaded 38GiB to GPU in 2.6968s (14.0908GiB/s) | 20260508 04:42:24.684 | INFO | Analyzing chunk frames = [5321216,5321728) | 20260508 04:42:24.684 | INFO | MemInfo: { | 20260508 04:42:24.684 | INFO | "gpu" : " 17 GB", | 20260508 04:42:24.684 | INFO | "huge" : "", | 20260508 04:42:24.684 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:24.684 | INFO | "malloc" : " 14 GB", | 20260508 04:42:24.684 | INFO | "pinned" : " 29 GB" | 20260508 04:42:24.684 | INFO | } | 20260508 04:42:24.684 | INFO | Perf report for the last 5.14278 seconds: | 20260508 04:42:24.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:24.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.684 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 04:42:24.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.684 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:42:24.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:24.685 | INFO | Uploaded 36GiB to GPU in 2.5926s (13.8856GiB/s) | 20260508 04:42:24.685 | INFO | Downloaded 34GiB to GPU in 2.43245s (13.9777GiB/s) | 20260508 04:42:29.820 | INFO | Analyzing chunk frames = [5321728,5322240) | 20260508 04:42:29.820 | INFO | MemInfo: { | 20260508 04:42:29.820 | INFO | "gpu" : " 17 GB", | 20260508 04:42:29.820 | INFO | "huge" : "", | 20260508 04:42:29.820 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:29.820 | INFO | "malloc" : " 14 GB", | 20260508 04:42:29.820 | INFO | "pinned" : " 29 GB" | 20260508 04:42:29.820 | INFO | } | 20260508 04:42:29.820 | INFO | Perf report for the last 5.13575 seconds: | 20260508 04:42:29.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:29.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 04:42:29.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:42:29.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:29.820 | INFO | Uploaded 36GiB to GPU in 2.5723s (13.9953GiB/s) | 20260508 04:42:29.820 | INFO | Downloaded 34GiB to GPU in 2.42147s (14.041GiB/s) | 20260508 04:42:34.956 | INFO | Analyzing chunk frames = [5322240,5322752) | 20260508 04:42:34.956 | INFO | MemInfo: { | 20260508 04:42:34.956 | INFO | "gpu" : " 17 GB", | 20260508 04:42:34.956 | INFO | "huge" : "", | 20260508 04:42:34.956 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:34.956 | INFO | "malloc" : " 14 GB", | 20260508 04:42:34.956 | INFO | "pinned" : " 29 GB" | 20260508 04:42:34.956 | INFO | } | 20260508 04:42:34.956 | INFO | Perf report for the last 5.13642 seconds: | 20260508 04:42:34.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:34.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | ANALYSIS : 67.2%, 2.9, 104.0ms, 0 | 20260508 04:42:34.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:42:34.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:34.956 | INFO | Uploaded 36GiB to GPU in 2.59892s (13.8519GiB/s) | 20260508 04:42:34.956 | INFO | Downloaded 34GiB to GPU in 2.4117s (14.0979GiB/s) | 20260508 04:42:40.099 | INFO | Analyzing chunk frames = [5322752,5323264) | 20260508 04:42:40.099 | INFO | MemInfo: { | 20260508 04:42:40.099 | INFO | "gpu" : " 17 GB", | 20260508 04:42:40.099 | INFO | "huge" : "", | 20260508 04:42:40.099 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:40.099 | INFO | "malloc" : " 14 GB", | 20260508 04:42:40.099 | INFO | "pinned" : " 29 GB" | 20260508 04:42:40.099 | INFO | } | 20260508 04:42:40.099 | INFO | Perf report for the last 5.14309 seconds: | 20260508 04:42:40.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:40.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.099 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 04:42:40.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.099 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:42:40.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:40.100 | INFO | Uploaded 36GiB to GPU in 2.37461s (15.1604GiB/s) | 20260508 04:42:40.100 | INFO | Downloaded 34GiB to GPU in 2.55257s (13.3199GiB/s) | 20260508 04:42:45.238 | INFO | Analyzing chunk frames = [5323264,5323776) | 20260508 04:42:45.238 | INFO | MemInfo: { | 20260508 04:42:45.238 | INFO | "gpu" : " 17 GB", | 20260508 04:42:45.238 | INFO | "huge" : "", | 20260508 04:42:45.238 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:45.238 | INFO | "malloc" : " 14 GB", | 20260508 04:42:45.238 | INFO | "pinned" : " 29 GB" | 20260508 04:42:45.238 | INFO | } | 20260508 04:42:45.238 | INFO | Perf report for the last 5.13859 seconds: | 20260508 04:42:45.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:45.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:42:45.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:42:45.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:45.238 | INFO | Uploaded 36GiB to GPU in 2.43736s (14.7701GiB/s) | 20260508 04:42:45.238 | INFO | Downloaded 34GiB to GPU in 2.41498s (14.0788GiB/s) | 20260508 04:42:50.377 | INFO | Analyzing chunk frames = [5323776,5324288) | 20260508 04:42:50.377 | INFO | MemInfo: { | 20260508 04:42:50.377 | INFO | "gpu" : " 17 GB", | 20260508 04:42:50.377 | INFO | "huge" : "", | 20260508 04:42:50.377 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:50.377 | INFO | "malloc" : " 14 GB", | 20260508 04:42:50.377 | INFO | "pinned" : " 29 GB" | 20260508 04:42:50.377 | INFO | } | 20260508 04:42:50.377 | INFO | Perf report for the last 5.13946 seconds: | 20260508 04:42:50.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:50.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.378 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:42:50.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.378 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 04:42:50.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:50.378 | INFO | Uploaded 36GiB to GPU in 2.45036s (14.6917GiB/s) | 20260508 04:42:50.378 | INFO | Downloaded 34GiB to GPU in 2.42716s (14.0081GiB/s) | 20260508 04:42:55.517 | INFO | Analyzing chunk frames = [5324288,5324800) | 20260508 04:42:55.517 | INFO | MemInfo: { | 20260508 04:42:55.517 | INFO | "gpu" : " 17 GB", | 20260508 04:42:55.517 | INFO | "huge" : "", | 20260508 04:42:55.517 | INFO | "huge_shared" : " 29 GB", | 20260508 04:42:55.518 | INFO | "malloc" : " 14 GB", | 20260508 04:42:55.518 | INFO | "pinned" : " 29 GB" | 20260508 04:42:55.518 | INFO | } | 20260508 04:42:55.518 | INFO | Perf report for the last 5.14021 seconds: | 20260508 04:42:55.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:42:55.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:42:55.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 04:42:55.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:42:55.518 | INFO | Uploaded 36GiB to GPU in 2.44361s (14.7323GiB/s) | 20260508 04:42:55.518 | INFO | Downloaded 34GiB to GPU in 2.42565s (14.0169GiB/s) | 20260508 04:42:55.519 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:42:55.519 | INFO | Total mem usage: 16.7 GiB | 20260508 04:42:55.519 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:42:55.519 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 04:42:55.519 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:42:55.519 | INFO | High Water Breakdown: | 20260508 04:42:55.519 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:42:55.519 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:42:55.519 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:42:55.519 | INFO | ------------------------------------------ | 20260508 04:42:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:42:55.519 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:42:55.519 | INFO | Total mem usage: 26.5 GiB | 20260508 04:42:55.519 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:42:55.519 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:42:55.519 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:42:55.519 | INFO | High Water Breakdown: | 20260508 04:42:55.519 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:42:55.519 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:42:55.519 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:42:55.519 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:42:55.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:42:55.519 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:42:55.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:42:55.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:42:55.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:42:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:42:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:42:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:42:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:42:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:42:55.519 | INFO | ------------------------------------------ | 20260508 04:42:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:42:55.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:42:55.519 | INFO | Total mem usage: 27.8 GiB | 20260508 04:42:55.519 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:42:55.519 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:42:55.519 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:42:55.519 | INFO | High Water Breakdown: | 20260508 04:42:55.519 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:42:55.519 | INFO | ------------------------------------------ | 20260508 04:42:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:42:55.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:42:55.519 | INFO | Total mem usage: 15.7 GiB | 20260508 04:42:55.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:42:55.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:42:55.519 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:42:55.519 | INFO | High Water Breakdown: | 20260508 04:42:55.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:42:55.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:42:55.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:42:55.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:42:55.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:42:55.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:42:55.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:42:55.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:42:55.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:42:55.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:42:55.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:42:55.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:42:55.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:42:55.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:42:55.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:42:55.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:42:55.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:42:55.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:42:55.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:42:55.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:42:55.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:42:55.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:42:55.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:42:55.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:42:55.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:42:55.519 | INFO | ------------------------------------------ | 20260508 04:42:55.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:42:55.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:42:55.519 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:42:55.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:42:55.520 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.48e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.20e-02 / 5.74e-03 | 20260508 04:42:55.520 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.48e+00, 5.84e-01 / 2.06e-01 | 20260508 04:42:55.520 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.36e-01 | 20260508 04:42:55.520 | INFO | PerfReport: 15.91%, DME -- 4.16e+01 / 4.51e+01, 3.85e+00 / 1.69e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.78e+00 / 1.37e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.18e-01 / 2.52e-01 | 20260508 04:42:55.520 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.25e+00, 2.63e-01 / 1.59e-01 | 20260508 04:42:55.520 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.62e-01, 5.06e-02 / 2.62e-02 | 20260508 04:42:55.520 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:42:55.520 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:42:55.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:42:55.520 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 24.42%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 24.27%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 2.20%, LockUpload -- 2.34e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 2.64%, LockDownload -- 2.81e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 15.67%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:42:55.520 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260508 04:42:55.629 | INFO | Starting a round of Baz IO | 20260508 04:42:59.042 | INFO | Populating RTMetrics report | 20260508 04:42:59.043 | INFO | RTMetrics report populated | 20260508 04:42:59.043 | INFO | Opening tmp RTMetrics json file | 20260508 04:42:59.056 | INFO | Serializing RTMetrics to json | 20260508 04:42:59.058 | INFO | Serialization done, swapping tmp file now... | 20260508 04:42:59.058 | INFO | RTMetrics done | 20260508 04:42:59.119 | INFO | Finished a round of Baz IO: Wrote 1.59 GiB in 3.49 seconds | 20260508 04:42:59.119 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.01% / 0.00% / 58.99% | 20260508 04:42:59.119 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.517209% / 7.94729e-06% / 8.74201e-05% | 20260508 04:42:59.119 | INFO | Baz IO overall: | 20260508 04:42:59.119 | INFO | BytesWritten : 2381094365849 | 20260508 04:42:59.119 | INFO | NumEvents : 1149388968376 | 20260508 04:43:00.653 | INFO | Analyzing chunk frames = [5324800,5325312) | 20260508 04:43:00.654 | INFO | MemInfo: { | 20260508 04:43:00.654 | INFO | "gpu" : " 17 GB", | 20260508 04:43:00.654 | INFO | "huge" : "", | 20260508 04:43:00.654 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:00.654 | INFO | "malloc" : " 14 GB", | 20260508 04:43:00.654 | INFO | "pinned" : " 29 GB" | 20260508 04:43:00.654 | INFO | } | 20260508 04:43:00.654 | INFO | Perf report for the last 5.13591 seconds: | 20260508 04:43:00.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:00.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:00.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:00.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:00.654 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 04:43:00.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:00.654 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260508 04:43:00.654 | INFO | PRE_HQ : 66.8%, 1.8, 62.3ms, 0 | 20260508 04:43:00.654 | INFO | BAZWRITER : 67.2%, 2.9, 101.4ms, 0 | 20260508 04:43:00.654 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:43:00.654 | INFO | Uploaded 36GiB to GPU in 2.56009s (14.062GiB/s) | 20260508 04:43:00.654 | INFO | Downloaded 38GiB to GPU in 2.69774s (14.0859GiB/s) | 20260508 04:43:05.802 | INFO | Analyzing chunk frames = [5325312,5325824) | 20260508 04:43:05.802 | INFO | MemInfo: { | 20260508 04:43:05.802 | INFO | "gpu" : " 17 GB", | 20260508 04:43:05.802 | INFO | "huge" : "", | 20260508 04:43:05.802 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:05.802 | INFO | "malloc" : " 14 GB", | 20260508 04:43:05.802 | INFO | "pinned" : " 29 GB" | 20260508 04:43:05.802 | INFO | } | 20260508 04:43:05.802 | INFO | Perf report for the last 5.14831 seconds: | 20260508 04:43:05.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:05.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 04:43:05.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:43:05.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:05.802 | INFO | Uploaded 36GiB to GPU in 2.57839s (13.9622GiB/s) | 20260508 04:43:05.802 | INFO | Downloaded 34GiB to GPU in 2.42613s (14.0141GiB/s) | 20260508 04:43:10.938 | INFO | Analyzing chunk frames = [5325824,5326336) | 20260508 04:43:10.938 | INFO | MemInfo: { | 20260508 04:43:10.938 | INFO | "gpu" : " 17 GB", | 20260508 04:43:10.938 | INFO | "huge" : "", | 20260508 04:43:10.938 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:10.938 | INFO | "malloc" : " 14 GB", | 20260508 04:43:10.938 | INFO | "pinned" : " 29 GB" | 20260508 04:43:10.938 | INFO | } | 20260508 04:43:10.938 | INFO | Perf report for the last 5.13646 seconds: | 20260508 04:43:10.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:10.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.938 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 04:43:10.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.939 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:43:10.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:10.939 | INFO | Uploaded 36GiB to GPU in 2.54353s (14.1536GiB/s) | 20260508 04:43:10.939 | INFO | Downloaded 34GiB to GPU in 2.39909s (14.172GiB/s) | 20260508 04:43:16.076 | INFO | Analyzing chunk frames = [5326336,5326848) | 20260508 04:43:16.076 | INFO | MemInfo: { | 20260508 04:43:16.076 | INFO | "gpu" : " 17 GB", | 20260508 04:43:16.076 | INFO | "huge" : "", | 20260508 04:43:16.076 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:16.076 | INFO | "malloc" : " 14 GB", | 20260508 04:43:16.076 | INFO | "pinned" : " 29 GB" | 20260508 04:43:16.077 | INFO | } | 20260508 04:43:16.077 | INFO | Perf report for the last 5.13819 seconds: | 20260508 04:43:16.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:16.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:43:16.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:43:16.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:16.077 | INFO | Uploaded 36GiB to GPU in 2.54345s (14.154GiB/s) | 20260508 04:43:16.077 | INFO | Downloaded 34GiB to GPU in 2.39851s (14.1754GiB/s) | 20260508 04:43:21.212 | INFO | Analyzing chunk frames = [5326848,5327360) | 20260508 04:43:21.212 | INFO | MemInfo: { | 20260508 04:43:21.212 | INFO | "gpu" : " 17 GB", | 20260508 04:43:21.212 | INFO | "huge" : "", | 20260508 04:43:21.212 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:21.212 | INFO | "malloc" : " 14 GB", | 20260508 04:43:21.212 | INFO | "pinned" : " 29 GB" | 20260508 04:43:21.212 | INFO | } | 20260508 04:43:21.212 | INFO | Perf report for the last 5.13525 seconds: | 20260508 04:43:21.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:21.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 04:43:21.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:43:21.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:21.212 | INFO | Uploaded 36GiB to GPU in 2.30245s (15.6355GiB/s) | 20260508 04:43:21.213 | INFO | Downloaded 34GiB to GPU in 2.46961s (13.7673GiB/s) | 20260508 04:43:26.353 | INFO | Analyzing chunk frames = [5327360,5327872) | 20260508 04:43:26.353 | INFO | MemInfo: { | 20260508 04:43:26.353 | INFO | "gpu" : " 17 GB", | 20260508 04:43:26.353 | INFO | "huge" : "", | 20260508 04:43:26.353 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:26.353 | INFO | "malloc" : " 14 GB", | 20260508 04:43:26.353 | INFO | "pinned" : " 29 GB" | 20260508 04:43:26.353 | INFO | } | 20260508 04:43:26.353 | INFO | Perf report for the last 5.14126 seconds: | 20260508 04:43:26.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:26.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 04:43:26.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:43:26.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:26.353 | INFO | Uploaded 36GiB to GPU in 2.45265s (14.678GiB/s) | 20260508 04:43:26.353 | INFO | Downloaded 34GiB to GPU in 2.41612s (14.0721GiB/s) | 20260508 04:43:31.492 | INFO | Analyzing chunk frames = [5327872,5328384) | 20260508 04:43:31.492 | INFO | MemInfo: { | 20260508 04:43:31.492 | INFO | "gpu" : " 17 GB", | 20260508 04:43:31.492 | INFO | "huge" : "", | 20260508 04:43:31.492 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:31.492 | INFO | "malloc" : " 14 GB", | 20260508 04:43:31.492 | INFO | "pinned" : " 29 GB" | 20260508 04:43:31.492 | INFO | } | 20260508 04:43:31.492 | INFO | Perf report for the last 5.1392 seconds: | 20260508 04:43:31.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:31.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.492 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:43:31.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.493 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 04:43:31.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:31.493 | INFO | Uploaded 36GiB to GPU in 2.45205s (14.6816GiB/s) | 20260508 04:43:31.493 | INFO | Downloaded 34GiB to GPU in 2.416s (14.0729GiB/s) | 20260508 04:43:36.636 | INFO | Analyzing chunk frames = [5328384,5328896) | 20260508 04:43:36.636 | INFO | MemInfo: { | 20260508 04:43:36.636 | INFO | "gpu" : " 17 GB", | 20260508 04:43:36.636 | INFO | "huge" : "", | 20260508 04:43:36.636 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:36.636 | INFO | "malloc" : " 14 GB", | 20260508 04:43:36.636 | INFO | "pinned" : " 29 GB" | 20260508 04:43:36.636 | INFO | } | 20260508 04:43:36.636 | INFO | Perf report for the last 5.14404 seconds: | 20260508 04:43:36.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:36.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.636 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:43:36.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.636 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260508 04:43:36.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:36.637 | INFO | Uploaded 36GiB to GPU in 2.48523s (14.4856GiB/s) | 20260508 04:43:36.637 | INFO | Downloaded 34GiB to GPU in 2.4326s (13.9768GiB/s) | 20260508 04:43:36.733 | INFO | Starting a round of Baz IO | 20260508 04:43:40.167 | INFO | Populating RTMetrics report | 20260508 04:43:40.168 | INFO | RTMetrics report populated | 20260508 04:43:40.168 | INFO | Opening tmp RTMetrics json file | 20260508 04:43:40.179 | INFO | Serializing RTMetrics to json | 20260508 04:43:40.180 | INFO | Serialization done, swapping tmp file now... | 20260508 04:43:40.181 | INFO | RTMetrics done | 20260508 04:43:40.237 | INFO | Finished a round of Baz IO: Wrote 1.589 GiB in 3.503 seconds | 20260508 04:43:40.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.01% / 0.00% / 58.99% | 20260508 04:43:40.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.515632% / 0% / 7.15256e-05% | 20260508 04:43:40.237 | INFO | Baz IO overall: | 20260508 04:43:40.237 | INFO | BytesWritten : 2382800550513 | 20260508 04:43:40.237 | INFO | NumEvents : 1150203224962 | 20260508 04:43:41.768 | INFO | Analyzing chunk frames = [5328896,5329408) | 20260508 04:43:41.768 | INFO | MemInfo: { | 20260508 04:43:41.768 | INFO | "gpu" : " 17 GB", | 20260508 04:43:41.768 | INFO | "huge" : "", | 20260508 04:43:41.768 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:41.768 | INFO | "malloc" : " 14 GB", | 20260508 04:43:41.768 | INFO | "pinned" : " 29 GB" | 20260508 04:43:41.768 | INFO | } | 20260508 04:43:41.768 | INFO | Perf report for the last 5.13144 seconds: | 20260508 04:43:41.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:41.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:41.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:41.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:41.768 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 04:43:41.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:41.768 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 04:43:41.768 | INFO | PRE_HQ : 65.4%, 1.6, 55.4ms, 0 | 20260508 04:43:41.768 | INFO | BAZWRITER : 67.8%, 2.7, 95.4ms, 0 | 20260508 04:43:41.768 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 04:43:41.768 | INFO | Uploaded 36GiB to GPU in 2.54617s (14.1389GiB/s) | 20260508 04:43:41.768 | INFO | Downloaded 38GiB to GPU in 2.68739s (14.1401GiB/s) | 20260508 04:43:46.907 | INFO | Analyzing chunk frames = [5329408,5329920) | 20260508 04:43:46.907 | INFO | MemInfo: { | 20260508 04:43:46.907 | INFO | "gpu" : " 17 GB", | 20260508 04:43:46.907 | INFO | "huge" : "", | 20260508 04:43:46.907 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:46.907 | INFO | "malloc" : " 14 GB", | 20260508 04:43:46.907 | INFO | "pinned" : " 29 GB" | 20260508 04:43:46.907 | INFO | } | 20260508 04:43:46.907 | INFO | Perf report for the last 5.13954 seconds: | 20260508 04:43:46.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:46.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.907 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 04:43:46.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.908 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:43:46.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:46.908 | INFO | Uploaded 36GiB to GPU in 2.59399s (13.8782GiB/s) | 20260508 04:43:46.908 | INFO | Downloaded 34GiB to GPU in 2.4274s (14.0067GiB/s) | 20260508 04:43:52.048 | INFO | Analyzing chunk frames = [5329920,5330432) | 20260508 04:43:52.048 | INFO | MemInfo: { | 20260508 04:43:52.048 | INFO | "gpu" : " 17 GB", | 20260508 04:43:52.048 | INFO | "huge" : "", | 20260508 04:43:52.048 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:52.048 | INFO | "malloc" : " 14 GB", | 20260508 04:43:52.048 | INFO | "pinned" : " 29 GB" | 20260508 04:43:52.048 | INFO | } | 20260508 04:43:52.049 | INFO | Perf report for the last 5.14113 seconds: | 20260508 04:43:52.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:52.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 04:43:52.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:43:52.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:52.049 | INFO | Uploaded 36GiB to GPU in 2.55298s (14.1012GiB/s) | 20260508 04:43:52.049 | INFO | Downloaded 34GiB to GPU in 2.40517s (14.1362GiB/s) | 20260508 04:43:57.190 | INFO | Analyzing chunk frames = [5330432,5330944) | 20260508 04:43:57.190 | INFO | MemInfo: { | 20260508 04:43:57.190 | INFO | "gpu" : " 17 GB", | 20260508 04:43:57.190 | INFO | "huge" : "", | 20260508 04:43:57.190 | INFO | "huge_shared" : " 29 GB", | 20260508 04:43:57.190 | INFO | "malloc" : " 14 GB", | 20260508 04:43:57.190 | INFO | "pinned" : " 29 GB" | 20260508 04:43:57.190 | INFO | } | 20260508 04:43:57.190 | INFO | Perf report for the last 5.14188 seconds: | 20260508 04:43:57.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:43:57.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.190 | INFO | ANALYSIS : 67.1%, 2.9, 103.9ms, 0 | 20260508 04:43:57.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.191 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:43:57.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:43:57.191 | INFO | Uploaded 36GiB to GPU in 2.57995s (13.9537GiB/s) | 20260508 04:43:57.191 | INFO | Downloaded 34GiB to GPU in 2.40511s (14.1366GiB/s) | 20260508 04:44:02.331 | INFO | Analyzing chunk frames = [5330944,5331456) | 20260508 04:44:02.331 | INFO | MemInfo: { | 20260508 04:44:02.331 | INFO | "gpu" : " 17 GB", | 20260508 04:44:02.331 | INFO | "huge" : "", | 20260508 04:44:02.331 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:02.331 | INFO | "malloc" : " 14 GB", | 20260508 04:44:02.331 | INFO | "pinned" : " 29 GB" | 20260508 04:44:02.331 | INFO | } | 20260508 04:44:02.331 | INFO | Perf report for the last 5.14076 seconds: | 20260508 04:44:02.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:02.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 04:44:02.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:44:02.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:02.331 | INFO | Uploaded 36GiB to GPU in 2.39513s (15.0305GiB/s) | 20260508 04:44:02.332 | INFO | Downloaded 34GiB to GPU in 2.54729s (13.3475GiB/s) | 20260508 04:44:07.468 | INFO | Analyzing chunk frames = [5331456,5331968) | 20260508 04:44:07.468 | INFO | MemInfo: { | 20260508 04:44:07.468 | INFO | "gpu" : " 17 GB", | 20260508 04:44:07.468 | INFO | "huge" : "", | 20260508 04:44:07.468 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:07.468 | INFO | "malloc" : " 14 GB", | 20260508 04:44:07.468 | INFO | "pinned" : " 29 GB" | 20260508 04:44:07.468 | INFO | } | 20260508 04:44:07.468 | INFO | Perf report for the last 5.13675 seconds: | 20260508 04:44:07.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:07.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:44:07.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:44:07.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:07.468 | INFO | Uploaded 36GiB to GPU in 2.45334s (14.6739GiB/s) | 20260508 04:44:07.469 | INFO | Downloaded 34GiB to GPU in 2.40792s (14.1201GiB/s) | 20260508 04:44:12.609 | INFO | Analyzing chunk frames = [5331968,5332480) | 20260508 04:44:12.609 | INFO | MemInfo: { | 20260508 04:44:12.609 | INFO | "gpu" : " 17 GB", | 20260508 04:44:12.609 | INFO | "huge" : "", | 20260508 04:44:12.609 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:12.610 | INFO | "malloc" : " 14 GB", | 20260508 04:44:12.610 | INFO | "pinned" : " 29 GB" | 20260508 04:44:12.610 | INFO | } | 20260508 04:44:12.610 | INFO | Perf report for the last 5.14165 seconds: | 20260508 04:44:12.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:12.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:44:12.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 04:44:12.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:12.610 | INFO | Uploaded 36GiB to GPU in 2.46309s (14.6158GiB/s) | 20260508 04:44:12.610 | INFO | Downloaded 34GiB to GPU in 2.41533s (14.0768GiB/s) | 20260508 04:44:17.748 | INFO | Analyzing chunk frames = [5332480,5332992) | 20260508 04:44:17.748 | INFO | MemInfo: { | 20260508 04:44:17.748 | INFO | "gpu" : " 17 GB", | 20260508 04:44:17.748 | INFO | "huge" : "", | 20260508 04:44:17.748 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:17.748 | INFO | "malloc" : " 14 GB", | 20260508 04:44:17.748 | INFO | "pinned" : " 29 GB" | 20260508 04:44:17.748 | INFO | } | 20260508 04:44:17.748 | INFO | Perf report for the last 5.13854 seconds: | 20260508 04:44:17.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:17.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.748 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:44:17.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.748 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:44:17.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:17.749 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:44:17.749 | INFO | Total mem usage: 16.7 GiB | 20260508 04:44:17.749 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:44:17.749 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:44:17.749 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:44:17.749 | INFO | High Water Breakdown: | 20260508 04:44:17.749 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:44:17.749 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:44:17.749 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:44:17.749 | INFO | ------------------------------------------ | 20260508 04:44:17.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:44:17.749 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:44:17.749 | INFO | Total mem usage: 26.5 GiB | 20260508 04:44:17.749 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:44:17.749 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:44:17.749 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:44:17.749 | INFO | High Water Breakdown: | 20260508 04:44:17.749 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:44:17.749 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:44:17.749 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:44:17.749 | INFO | Uploaded 36GiB to GPU in 2.4545s (14.667GiB/s) | 20260508 04:44:17.749 | INFO | Downloaded 34GiB to GPU in 2.4232s (14.031GiB/s) | 20260508 04:44:17.749 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:44:17.749 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:44:17.749 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:44:17.749 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:44:17.749 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:44:17.749 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:44:17.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:44:17.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:44:17.749 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:44:17.749 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:44:17.749 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:44:17.749 | INFO | ------------------------------------------ | 20260508 04:44:17.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:44:17.749 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:44:17.749 | INFO | Total mem usage: 27.8 GiB | 20260508 04:44:17.749 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:44:17.749 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:44:17.749 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:44:17.749 | INFO | High Water Breakdown: | 20260508 04:44:17.749 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:44:17.749 | INFO | ------------------------------------------ | 20260508 04:44:17.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:44:17.749 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:44:17.749 | INFO | Total mem usage: 15.7 GiB | 20260508 04:44:17.749 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:44:17.749 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:44:17.749 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:44:17.749 | INFO | High Water Breakdown: | 20260508 04:44:17.749 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:44:17.749 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:44:17.749 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:44:17.749 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:44:17.749 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:44:17.749 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:44:17.749 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:44:17.749 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:44:17.749 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:44:17.749 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:44:17.749 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:44:17.749 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:44:17.749 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:44:17.749 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:44:17.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:44:17.750 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:44:17.750 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:44:17.750 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:44:17.750 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:44:17.750 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:44:17.750 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:44:17.750 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:44:17.750 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:44:17.750 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:44:17.750 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:44:17.750 | INFO | ------------------------------------------ | 20260508 04:44:17.750 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:44:17.750 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:44:17.750 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:44:17.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:44:17.750 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 1.27e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.13e-02 / 1.01e-02 | 20260508 04:44:17.750 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.45e+00, 5.91e-01 / 2.50e-01 | 20260508 04:44:17.750 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.53e-01 | 20260508 04:44:17.750 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.52e+01, 3.78e+00 / 1.61e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.54e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.18e-01 / 2.68e-01 | 20260508 04:44:17.750 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.22e+00, 2.60e-01 / 1.07e-01 | 20260508 04:44:17.750 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.55e-01, 5.04e-02 / 2.09e-02 | 20260508 04:44:17.750 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:44:17.750 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:44:17.750 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:44:17.750 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 24.10%, Download -- 2.56e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 30.80%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 2.44%, LockUpload -- 2.60e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 2.46%, LockDownload -- 2.62e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 15.72%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:44:17.750 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 4.54e-02 / 0.00e+00 | 20260508 04:44:17.846 | INFO | Starting a round of Baz IO | 20260508 04:44:21.289 | INFO | Populating RTMetrics report | 20260508 04:44:21.289 | INFO | RTMetrics report populated | 20260508 04:44:21.289 | INFO | Opening tmp RTMetrics json file | 20260508 04:44:21.301 | INFO | Serializing RTMetrics to json | 20260508 04:44:21.303 | INFO | Serialization done, swapping tmp file now... | 20260508 04:44:21.303 | INFO | RTMetrics done | 20260508 04:44:21.355 | INFO | Finished a round of Baz IO: Wrote 1.589 GiB in 3.509 seconds | 20260508 04:44:21.355 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.00% / 0.00% / 58.99% | 20260508 04:44:21.355 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.514917% / 0% / 6.35783e-05% | 20260508 04:44:21.355 | INFO | Baz IO overall: | 20260508 04:44:21.355 | INFO | BytesWritten : 2384506476167 | 20260508 04:44:21.355 | INFO | NumEvents : 1151017400853 | 20260508 04:44:22.887 | INFO | Analyzing chunk frames = [5332992,5333504) | 20260508 04:44:22.887 | INFO | MemInfo: { | 20260508 04:44:22.887 | INFO | "gpu" : " 17 GB", | 20260508 04:44:22.887 | INFO | "huge" : "", | 20260508 04:44:22.887 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:22.887 | INFO | "malloc" : " 14 GB", | 20260508 04:44:22.887 | INFO | "pinned" : " 29 GB" | 20260508 04:44:22.887 | INFO | } | 20260508 04:44:22.887 | INFO | Perf report for the last 5.139 seconds: | 20260508 04:44:22.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:22.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:22.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:22.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:22.887 | INFO | ANALYSIS : 67.9%, 3.0, 106.3ms, 0 | 20260508 04:44:22.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:22.887 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:44:22.887 | INFO | PRE_HQ : 65.3%, 1.6, 55.3ms, 0 | 20260508 04:44:22.887 | INFO | BAZWRITER : 67.8%, 2.7, 93.7ms, 0 | 20260508 04:44:22.887 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 04:44:22.888 | INFO | Uploaded 36GiB to GPU in 2.55406s (14.0952GiB/s) | 20260508 04:44:22.888 | INFO | Downloaded 38GiB to GPU in 2.6929s (14.1112GiB/s) | 20260508 04:44:28.027 | INFO | Analyzing chunk frames = [5333504,5334016) | 20260508 04:44:28.027 | INFO | MemInfo: { | 20260508 04:44:28.027 | INFO | "gpu" : " 17 GB", | 20260508 04:44:28.027 | INFO | "huge" : "", | 20260508 04:44:28.027 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:28.027 | INFO | "malloc" : " 14 GB", | 20260508 04:44:28.027 | INFO | "pinned" : " 29 GB" | 20260508 04:44:28.027 | INFO | } | 20260508 04:44:28.027 | INFO | Perf report for the last 5.13988 seconds: | 20260508 04:44:28.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:28.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.027 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:44:28.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.027 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 04:44:28.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:28.028 | INFO | Uploaded 36GiB to GPU in 2.59183s (13.8898GiB/s) | 20260508 04:44:28.028 | INFO | Downloaded 34GiB to GPU in 2.42862s (13.9997GiB/s) | 20260508 04:44:33.162 | INFO | Analyzing chunk frames = [5334016,5334528) | 20260508 04:44:33.162 | INFO | MemInfo: { | 20260508 04:44:33.162 | INFO | "gpu" : " 17 GB", | 20260508 04:44:33.162 | INFO | "huge" : "", | 20260508 04:44:33.162 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:33.162 | INFO | "malloc" : " 14 GB", | 20260508 04:44:33.162 | INFO | "pinned" : " 29 GB" | 20260508 04:44:33.162 | INFO | } | 20260508 04:44:33.162 | INFO | Perf report for the last 5.13534 seconds: | 20260508 04:44:33.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:33.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.162 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 04:44:33.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.163 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:44:33.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:33.163 | INFO | Uploaded 36GiB to GPU in 2.55105s (14.1118GiB/s) | 20260508 04:44:33.163 | INFO | Downloaded 34GiB to GPU in 2.39235s (14.212GiB/s) | 20260508 04:44:38.304 | INFO | Analyzing chunk frames = [5334528,5335040) | 20260508 04:44:38.304 | INFO | MemInfo: { | 20260508 04:44:38.304 | INFO | "gpu" : " 17 GB", | 20260508 04:44:38.304 | INFO | "huge" : "", | 20260508 04:44:38.304 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:38.305 | INFO | "malloc" : " 14 GB", | 20260508 04:44:38.305 | INFO | "pinned" : " 29 GB" | 20260508 04:44:38.305 | INFO | } | 20260508 04:44:38.305 | INFO | Perf report for the last 5.14266 seconds: | 20260508 04:44:38.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:38.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | ANALYSIS : 67.2%, 2.9, 104.1ms, 0 | 20260508 04:44:38.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:44:38.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:38.305 | INFO | Uploaded 36GiB to GPU in 2.58542s (13.9243GiB/s) | 20260508 04:44:38.305 | INFO | Downloaded 34GiB to GPU in 2.41966s (14.0516GiB/s) | 20260508 04:44:43.444 | INFO | Analyzing chunk frames = [5335040,5335552) | 20260508 04:44:43.444 | INFO | MemInfo: { | 20260508 04:44:43.444 | INFO | "gpu" : " 17 GB", | 20260508 04:44:43.444 | INFO | "huge" : "", | 20260508 04:44:43.444 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:43.444 | INFO | "malloc" : " 14 GB", | 20260508 04:44:43.444 | INFO | "pinned" : " 29 GB" | 20260508 04:44:43.444 | INFO | } | 20260508 04:44:43.445 | INFO | Perf report for the last 5.13938 seconds: | 20260508 04:44:43.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:43.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 04:44:43.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:44:43.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:43.445 | INFO | Uploaded 36GiB to GPU in 2.4306s (14.8112GiB/s) | 20260508 04:44:43.445 | INFO | Downloaded 34GiB to GPU in 2.54656s (13.3514GiB/s) | 20260508 04:44:48.584 | INFO | Analyzing chunk frames = [5335552,5336064) | 20260508 04:44:48.584 | INFO | MemInfo: { | 20260508 04:44:48.584 | INFO | "gpu" : " 17 GB", | 20260508 04:44:48.584 | INFO | "huge" : "", | 20260508 04:44:48.584 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:48.584 | INFO | "malloc" : " 14 GB", | 20260508 04:44:48.584 | INFO | "pinned" : " 29 GB" | 20260508 04:44:48.584 | INFO | } | 20260508 04:44:48.584 | INFO | Perf report for the last 5.13981 seconds: | 20260508 04:44:48.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:48.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.584 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:44:48.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.584 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260508 04:44:48.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:48.585 | INFO | Uploaded 36GiB to GPU in 2.43495s (14.7847GiB/s) | 20260508 04:44:48.585 | INFO | Downloaded 34GiB to GPU in 2.42529s (14.019GiB/s) | 20260508 04:44:53.718 | INFO | Analyzing chunk frames = [5336064,5336576) | 20260508 04:44:53.718 | INFO | MemInfo: { | 20260508 04:44:53.718 | INFO | "gpu" : " 17 GB", | 20260508 04:44:53.718 | INFO | "huge" : "", | 20260508 04:44:53.718 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:53.718 | INFO | "malloc" : " 14 GB", | 20260508 04:44:53.718 | INFO | "pinned" : " 29 GB" | 20260508 04:44:53.718 | INFO | } | 20260508 04:44:53.718 | INFO | Perf report for the last 5.13353 seconds: | 20260508 04:44:53.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:53.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:44:53.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260508 04:44:53.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:53.718 | INFO | Uploaded 36GiB to GPU in 2.4476s (14.7083GiB/s) | 20260508 04:44:53.718 | INFO | Downloaded 34GiB to GPU in 2.42708s (14.0086GiB/s) | 20260508 04:44:58.862 | INFO | Analyzing chunk frames = [5336576,5337088) | 20260508 04:44:58.862 | INFO | MemInfo: { | 20260508 04:44:58.862 | INFO | "gpu" : " 17 GB", | 20260508 04:44:58.862 | INFO | "huge" : "", | 20260508 04:44:58.862 | INFO | "huge_shared" : " 29 GB", | 20260508 04:44:58.862 | INFO | "malloc" : " 14 GB", | 20260508 04:44:58.862 | INFO | "pinned" : " 29 GB" | 20260508 04:44:58.862 | INFO | } | 20260508 04:44:58.862 | INFO | Perf report for the last 5.14447 seconds: | 20260508 04:44:58.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:44:58.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.862 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:44:58.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.862 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:44:58.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:44:58.863 | INFO | Uploaded 36GiB to GPU in 2.44298s (14.7361GiB/s) | 20260508 04:44:58.863 | INFO | Downloaded 34GiB to GPU in 2.42327s (14.0306GiB/s) | 20260508 04:44:58.960 | INFO | Starting a round of Baz IO | 20260508 04:45:02.375 | INFO | Populating RTMetrics report | 20260508 04:45:02.375 | INFO | RTMetrics report populated | 20260508 04:45:02.375 | INFO | Opening tmp RTMetrics json file | 20260508 04:45:02.376 | INFO | Serializing RTMetrics to json | 20260508 04:45:02.379 | INFO | Serialization done, swapping tmp file now... | 20260508 04:45:02.380 | INFO | RTMetrics done | 20260508 04:45:02.514 | INFO | Finished a round of Baz IO: Wrote 1.59 GiB in 3.554 seconds | 20260508 04:45:02.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.00% / 0.00% / 59.00% | 20260508 04:45:02.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.515143% / 0% / 5.5631e-05% | 20260508 04:45:02.514 | INFO | Baz IO overall: | 20260508 04:45:02.514 | INFO | BytesWritten : 2386213910224 | 20260508 04:45:02.514 | INFO | NumEvents : 1151832729760 | 20260508 04:45:04.001 | INFO | Analyzing chunk frames = [5337088,5337600) | 20260508 04:45:04.001 | INFO | MemInfo: { | 20260508 04:45:04.001 | INFO | "gpu" : " 17 GB", | 20260508 04:45:04.001 | INFO | "huge" : "", | 20260508 04:45:04.001 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:04.001 | INFO | "malloc" : " 14 GB", | 20260508 04:45:04.001 | INFO | "pinned" : " 29 GB" | 20260508 04:45:04.001 | INFO | } | 20260508 04:45:04.001 | INFO | Perf report for the last 5.13907 seconds: | 20260508 04:45:04.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:04.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:04.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:04.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:04.001 | INFO | ANALYSIS : 68.0%, 3.0, 105.1ms, 0 | 20260508 04:45:04.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:04.002 | INFO | RT_METRICS : 39.9%, 1.0, 20.7ms, 0 | 20260508 04:45:04.002 | INFO | PRE_HQ : 65.8%, 1.6, 54.9ms, 0 | 20260508 04:45:04.002 | INFO | BAZWRITER : 68.6%, 2.6, 92.2ms, 0 | 20260508 04:45:04.002 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:45:04.002 | INFO | Uploaded 36GiB to GPU in 2.56225s (14.0502GiB/s) | 20260508 04:45:04.002 | INFO | Downloaded 38GiB to GPU in 2.70612s (14.0422GiB/s) | 20260508 04:45:09.141 | INFO | Analyzing chunk frames = [5337600,5338112) | 20260508 04:45:09.141 | INFO | MemInfo: { | 20260508 04:45:09.141 | INFO | "gpu" : " 17 GB", | 20260508 04:45:09.141 | INFO | "huge" : "", | 20260508 04:45:09.141 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:09.141 | INFO | "malloc" : " 14 GB", | 20260508 04:45:09.141 | INFO | "pinned" : " 29 GB" | 20260508 04:45:09.141 | INFO | } | 20260508 04:45:09.141 | INFO | Perf report for the last 5.13994 seconds: | 20260508 04:45:09.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:09.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.141 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:45:09.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.142 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:45:09.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:09.142 | INFO | Uploaded 36GiB to GPU in 2.58073s (13.9496GiB/s) | 20260508 04:45:09.142 | INFO | Downloaded 34GiB to GPU in 2.42735s (14.007GiB/s) | 20260508 04:45:14.279 | INFO | Analyzing chunk frames = [5338112,5338624) | 20260508 04:45:14.279 | INFO | MemInfo: { | 20260508 04:45:14.279 | INFO | "gpu" : " 17 GB", | 20260508 04:45:14.279 | INFO | "huge" : "", | 20260508 04:45:14.279 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:14.279 | INFO | "malloc" : " 14 GB", | 20260508 04:45:14.279 | INFO | "pinned" : " 29 GB" | 20260508 04:45:14.279 | INFO | } | 20260508 04:45:14.279 | INFO | Perf report for the last 5.1378 seconds: | 20260508 04:45:14.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:14.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.279 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:45:14.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.279 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:45:14.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:14.280 | INFO | Uploaded 36GiB to GPU in 2.53702s (14.1899GiB/s) | 20260508 04:45:14.280 | INFO | Downloaded 34GiB to GPU in 2.41897s (14.0556GiB/s) | 20260508 04:45:19.419 | INFO | Analyzing chunk frames = [5338624,5339136) | 20260508 04:45:19.419 | INFO | MemInfo: { | 20260508 04:45:19.419 | INFO | "gpu" : " 17 GB", | 20260508 04:45:19.419 | INFO | "huge" : "", | 20260508 04:45:19.419 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:19.419 | INFO | "malloc" : " 14 GB", | 20260508 04:45:19.419 | INFO | "pinned" : " 29 GB" | 20260508 04:45:19.419 | INFO | } | 20260508 04:45:19.420 | INFO | Perf report for the last 5.14036 seconds: | 20260508 04:45:19.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:19.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 04:45:19.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:45:19.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:19.420 | INFO | Uploaded 36GiB to GPU in 2.58445s (13.9295GiB/s) | 20260508 04:45:19.420 | INFO | Downloaded 34GiB to GPU in 2.4167s (14.0688GiB/s) | 20260508 04:45:24.563 | INFO | Analyzing chunk frames = [5339136,5339648) | 20260508 04:45:24.563 | INFO | MemInfo: { | 20260508 04:45:24.563 | INFO | "gpu" : " 17 GB", | 20260508 04:45:24.563 | INFO | "huge" : "", | 20260508 04:45:24.563 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:24.563 | INFO | "malloc" : " 14 GB", | 20260508 04:45:24.563 | INFO | "pinned" : " 29 GB" | 20260508 04:45:24.563 | INFO | } | 20260508 04:45:24.563 | INFO | Perf report for the last 5.14366 seconds: | 20260508 04:45:24.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:24.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.563 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:45:24.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.564 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:45:24.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:24.564 | INFO | Uploaded 36GiB to GPU in 2.34899s (15.3257GiB/s) | 20260508 04:45:24.564 | INFO | Downloaded 34GiB to GPU in 2.54255s (13.3724GiB/s) | 20260508 04:45:29.700 | INFO | Analyzing chunk frames = [5339648,5340160) | 20260508 04:45:29.700 | INFO | MemInfo: { | 20260508 04:45:29.700 | INFO | "gpu" : " 17 GB", | 20260508 04:45:29.700 | INFO | "huge" : "", | 20260508 04:45:29.700 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:29.700 | INFO | "malloc" : " 14 GB", | 20260508 04:45:29.700 | INFO | "pinned" : " 29 GB" | 20260508 04:45:29.700 | INFO | } | 20260508 04:45:29.701 | INFO | Perf report for the last 5.13718 seconds: | 20260508 04:45:29.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:29.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 04:45:29.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 04:45:29.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:29.701 | INFO | Uploaded 36GiB to GPU in 2.43712s (14.7715GiB/s) | 20260508 04:45:29.701 | INFO | Downloaded 34GiB to GPU in 2.42091s (14.0443GiB/s) | 20260508 04:45:34.837 | INFO | Analyzing chunk frames = [5340160,5340672) | 20260508 04:45:34.837 | INFO | MemInfo: { | 20260508 04:45:34.837 | INFO | "gpu" : " 17 GB", | 20260508 04:45:34.837 | INFO | "huge" : "", | 20260508 04:45:34.837 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:34.837 | INFO | "malloc" : " 14 GB", | 20260508 04:45:34.837 | INFO | "pinned" : " 29 GB" | 20260508 04:45:34.837 | INFO | } | 20260508 04:45:34.837 | INFO | Perf report for the last 5.13665 seconds: | 20260508 04:45:34.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:34.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 04:45:34.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 04:45:34.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:34.837 | INFO | Uploaded 36GiB to GPU in 2.44773s (14.7075GiB/s) | 20260508 04:45:34.838 | INFO | Downloaded 34GiB to GPU in 2.43548s (13.9603GiB/s) | 20260508 04:45:39.971 | INFO | Analyzing chunk frames = [5340672,5341184) | 20260508 04:45:39.971 | INFO | MemInfo: { | 20260508 04:45:39.971 | INFO | "gpu" : " 17 GB", | 20260508 04:45:39.971 | INFO | "huge" : "", | 20260508 04:45:39.971 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:39.971 | INFO | "malloc" : " 14 GB", | 20260508 04:45:39.971 | INFO | "pinned" : " 29 GB" | 20260508 04:45:39.971 | INFO | } | 20260508 04:45:39.971 | INFO | Perf report for the last 5.13414 seconds: | 20260508 04:45:39.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:39.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.971 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:45:39.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.971 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 04:45:39.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:39.972 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:45:39.972 | INFO | Total mem usage: 16.7 GiB | 20260508 04:45:39.972 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:45:39.972 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:45:39.972 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:45:39.972 | INFO | High Water Breakdown: | 20260508 04:45:39.972 | INFO | 59.5% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:45:39.972 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:45:39.972 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:45:39.972 | INFO | Uploaded 36GiB to GPU in 2.46877s (14.5821GiB/s) | 20260508 04:45:39.972 | INFO | ------------------------------------------ | 20260508 04:45:39.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:45:39.972 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:45:39.972 | INFO | Total mem usage: 26.5 GiB | 20260508 04:45:39.972 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:45:39.972 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 04:45:39.972 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 04:45:39.972 | INFO | High Water Breakdown: | 20260508 04:45:39.972 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:45:39.972 | INFO | Downloaded 34GiB to GPU in 2.4354s (13.9607GiB/s) | 20260508 04:45:39.972 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:45:39.972 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:45:39.972 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:45:39.972 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:45:39.972 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:45:39.972 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:45:39.972 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:45:39.972 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:45:39.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:45:39.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:45:39.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:45:39.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:45:39.973 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:45:39.973 | INFO | ------------------------------------------ | 20260508 04:45:39.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:45:39.973 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:45:39.973 | INFO | Total mem usage: 27.8 GiB | 20260508 04:45:39.973 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:45:39.973 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:45:39.973 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:45:39.973 | INFO | High Water Breakdown: | 20260508 04:45:39.973 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:45:39.973 | INFO | ------------------------------------------ | 20260508 04:45:39.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:45:39.973 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:45:39.973 | INFO | Total mem usage: 15.7 GiB | 20260508 04:45:39.973 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:45:39.973 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:45:39.973 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:45:39.973 | INFO | High Water Breakdown: | 20260508 04:45:39.973 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:45:39.973 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:45:39.973 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:45:39.973 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:45:39.973 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:45:39.973 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:45:39.973 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:45:39.973 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:45:39.973 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:45:39.973 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:45:39.973 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:45:39.973 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:45:39.973 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:45:39.973 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:45:39.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:45:39.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:45:39.973 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:45:39.973 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:45:39.973 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:45:39.973 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:45:39.973 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:45:39.973 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:45:39.973 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:45:39.973 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:45:39.973 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:45:39.973 | INFO | ------------------------------------------ | 20260508 04:45:39.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:45:39.973 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:45:39.973 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:45:39.973 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:45:39.973 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.57e+01, 1.39e+01 / 1.41e+00 | 20260508 04:45:39.973 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.04e-01, 1.37e-02 / 6.65e-03 | 20260508 04:45:39.973 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 5.85e-01 / 1.99e-01 | 20260508 04:45:39.973 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.20e-01 | 20260508 04:45:39.973 | INFO | PerfReport: 15.89%, DME -- 4.16e+01 / 4.51e+01, 3.48e+00 / 1.72e+00 | 20260508 04:45:39.973 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.80e+00 / 1.36e+00 | 20260508 04:45:39.973 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.39e+00 / 2.55e+00, 4.18e-01 / 2.39e-01 | 20260508 04:45:39.973 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.23e+00, 2.63e-01 / 1.41e-01 | 20260508 04:45:39.973 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.24e-01 / 3.61e-01, 5.16e-02 / 2.78e-02 | 20260508 04:45:39.973 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:45:39.973 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:45:39.973 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:45:39.973 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:45:39.973 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 04:45:39.973 | INFO | PerfReport: 24.20%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:45:39.973 | INFO | PerfReport: 30.78%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:45:39.974 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 04:45:39.974 | INFO | PerfReport: 2.55%, LockDownload -- 2.72e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 04:45:39.974 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:45:39.974 | INFO | PerfReport: 0.02%, Misc -- 1.60e-02 / 0.00e+00, 3.15e-02 / 0.00e+00 | 20260508 04:45:40.084 | INFO | Starting a round of Baz IO | 20260508 04:45:43.513 | INFO | Populating RTMetrics report | 20260508 04:45:43.514 | INFO | RTMetrics report populated | 20260508 04:45:43.514 | INFO | Opening tmp RTMetrics json file | 20260508 04:45:43.515 | INFO | Serializing RTMetrics to json | 20260508 04:45:43.516 | INFO | Serialization done, swapping tmp file now... | 20260508 04:45:43.516 | INFO | RTMetrics done | 20260508 04:45:43.585 | INFO | Finished a round of Baz IO: Wrote 1.59 GiB in 3.501 seconds | 20260508 04:45:43.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.00% / 0.00% / 59.00% | 20260508 04:45:43.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.51446% / 3.97364e-06% / 5.96046e-05% | 20260508 04:45:43.585 | INFO | Baz IO overall: | 20260508 04:45:43.585 | INFO | BytesWritten : 2387921465349 | 20260508 04:45:43.585 | INFO | NumEvents : 1152648407611 | 20260508 04:45:45.116 | INFO | Analyzing chunk frames = [5341184,5341696) | 20260508 04:45:45.116 | INFO | MemInfo: { | 20260508 04:45:45.116 | INFO | "gpu" : " 17 GB", | 20260508 04:45:45.116 | INFO | "huge" : "", | 20260508 04:45:45.116 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:45.116 | INFO | "malloc" : " 14 GB", | 20260508 04:45:45.116 | INFO | "pinned" : " 29 GB" | 20260508 04:45:45.116 | INFO | } | 20260508 04:45:45.116 | INFO | Perf report for the last 5.14504 seconds: | 20260508 04:45:45.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:45.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:45.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:45.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:45.116 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 04:45:45.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:45.117 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 04:45:45.117 | INFO | PRE_HQ : 66.5%, 1.8, 61.8ms, 0 | 20260508 04:45:45.117 | INFO | BAZWRITER : 67.5%, 2.9, 99.4ms, 0 | 20260508 04:45:45.117 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 04:45:45.117 | INFO | Uploaded 36GiB to GPU in 2.51677s (14.304GiB/s) | 20260508 04:45:45.117 | INFO | Downloaded 38GiB to GPU in 2.70333s (14.0567GiB/s) | 20260508 04:45:50.259 | INFO | Analyzing chunk frames = [5341696,5342208) | 20260508 04:45:50.259 | INFO | MemInfo: { | 20260508 04:45:50.259 | INFO | "gpu" : " 17 GB", | 20260508 04:45:50.259 | INFO | "huge" : "", | 20260508 04:45:50.259 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:50.259 | INFO | "malloc" : " 14 GB", | 20260508 04:45:50.259 | INFO | "pinned" : " 29 GB" | 20260508 04:45:50.259 | INFO | } | 20260508 04:45:50.259 | INFO | Perf report for the last 5.14298 seconds: | 20260508 04:45:50.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:50.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.259 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:45:50.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.259 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:45:50.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:50.260 | INFO | Uploaded 36GiB to GPU in 2.57064s (14.0043GiB/s) | 20260508 04:45:50.260 | INFO | Downloaded 34GiB to GPU in 2.43136s (13.9839GiB/s) | 20260508 04:45:55.394 | INFO | Analyzing chunk frames = [5342208,5342720) | 20260508 04:45:55.395 | INFO | MemInfo: { | 20260508 04:45:55.395 | INFO | "gpu" : " 17 GB", | 20260508 04:45:55.395 | INFO | "huge" : "", | 20260508 04:45:55.395 | INFO | "huge_shared" : " 29 GB", | 20260508 04:45:55.395 | INFO | "malloc" : " 14 GB", | 20260508 04:45:55.395 | INFO | "pinned" : " 29 GB" | 20260508 04:45:55.395 | INFO | } | 20260508 04:45:55.395 | INFO | Perf report for the last 5.13548 seconds: | 20260508 04:45:55.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:45:55.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:45:55.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:45:55.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:45:55.395 | INFO | Uploaded 36GiB to GPU in 2.53417s (14.2058GiB/s) | 20260508 04:45:55.395 | INFO | Downloaded 34GiB to GPU in 2.41637s (14.0707GiB/s) | 20260508 04:46:00.531 | INFO | Analyzing chunk frames = [5342720,5343232) | 20260508 04:46:00.531 | INFO | MemInfo: { | 20260508 04:46:00.531 | INFO | "gpu" : " 17 GB", | 20260508 04:46:00.531 | INFO | "huge" : "", | 20260508 04:46:00.531 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:00.531 | INFO | "malloc" : " 14 GB", | 20260508 04:46:00.531 | INFO | "pinned" : " 29 GB" | 20260508 04:46:00.531 | INFO | } | 20260508 04:46:00.531 | INFO | Perf report for the last 5.13665 seconds: | 20260508 04:46:00.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:00.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.532 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:46:00.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.532 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 04:46:00.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:00.532 | INFO | Uploaded 36GiB to GPU in 2.58165s (13.9446GiB/s) | 20260508 04:46:00.532 | INFO | Downloaded 34GiB to GPU in 2.41528s (14.077GiB/s) | 20260508 04:46:05.671 | INFO | Analyzing chunk frames = [5343232,5343744) | 20260508 04:46:05.671 | INFO | MemInfo: { | 20260508 04:46:05.671 | INFO | "gpu" : " 17 GB", | 20260508 04:46:05.671 | INFO | "huge" : "", | 20260508 04:46:05.671 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:05.671 | INFO | "malloc" : " 14 GB", | 20260508 04:46:05.671 | INFO | "pinned" : " 29 GB" | 20260508 04:46:05.671 | INFO | } | 20260508 04:46:05.671 | INFO | Perf report for the last 5.13983 seconds: | 20260508 04:46:05.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:05.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.671 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:46:05.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.671 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:46:05.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:05.672 | INFO | Uploaded 36GiB to GPU in 2.35046s (15.3161GiB/s) | 20260508 04:46:05.672 | INFO | Downloaded 34GiB to GPU in 2.55252s (13.3202GiB/s) | 20260508 04:46:10.814 | INFO | Analyzing chunk frames = [5343744,5344256) | 20260508 04:46:10.814 | INFO | MemInfo: { | 20260508 04:46:10.814 | INFO | "gpu" : " 17 GB", | 20260508 04:46:10.814 | INFO | "huge" : "", | 20260508 04:46:10.814 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:10.814 | INFO | "malloc" : " 14 GB", | 20260508 04:46:10.814 | INFO | "pinned" : " 29 GB" | 20260508 04:46:10.814 | INFO | } | 20260508 04:46:10.814 | INFO | Perf report for the last 5.14274 seconds: | 20260508 04:46:10.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:10.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 04:46:10.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:46:10.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:10.814 | INFO | Uploaded 36GiB to GPU in 2.45482s (14.665GiB/s) | 20260508 04:46:10.814 | INFO | Downloaded 34GiB to GPU in 2.41565s (14.0749GiB/s) | 20260508 04:46:15.950 | INFO | Analyzing chunk frames = [5344256,5344768) | 20260508 04:46:15.950 | INFO | MemInfo: { | 20260508 04:46:15.950 | INFO | "gpu" : " 17 GB", | 20260508 04:46:15.950 | INFO | "huge" : "", | 20260508 04:46:15.950 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:15.950 | INFO | "malloc" : " 14 GB", | 20260508 04:46:15.950 | INFO | "pinned" : " 29 GB" | 20260508 04:46:15.950 | INFO | } | 20260508 04:46:15.950 | INFO | Perf report for the last 5.13601 seconds: | 20260508 04:46:15.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:15.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:46:15.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 04:46:15.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:15.950 | INFO | Uploaded 36GiB to GPU in 2.43928s (14.7585GiB/s) | 20260508 04:46:15.950 | INFO | Downloaded 34GiB to GPU in 2.40636s (14.1292GiB/s) | 20260508 04:46:21.090 | INFO | Analyzing chunk frames = [5344768,5345280) | 20260508 04:46:21.090 | INFO | MemInfo: { | 20260508 04:46:21.090 | INFO | "gpu" : " 17 GB", | 20260508 04:46:21.090 | INFO | "huge" : "", | 20260508 04:46:21.090 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:21.090 | INFO | "malloc" : " 14 GB", | 20260508 04:46:21.090 | INFO | "pinned" : " 29 GB" | 20260508 04:46:21.090 | INFO | } | 20260508 04:46:21.090 | INFO | Perf report for the last 5.13975 seconds: | 20260508 04:46:21.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:21.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:46:21.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:46:21.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:21.090 | INFO | Uploaded 36GiB to GPU in 2.46883s (14.5818GiB/s) | 20260508 04:46:21.090 | INFO | Downloaded 34GiB to GPU in 2.41453s (14.0814GiB/s) | 20260508 04:46:21.187 | INFO | Starting a round of Baz IO | 20260508 04:46:24.631 | INFO | Populating RTMetrics report | 20260508 04:46:24.632 | INFO | RTMetrics report populated | 20260508 04:46:24.632 | INFO | Opening tmp RTMetrics json file | 20260508 04:46:24.643 | INFO | Serializing RTMetrics to json | 20260508 04:46:24.645 | INFO | Serialization done, swapping tmp file now... | 20260508 04:46:24.646 | INFO | RTMetrics done | 20260508 04:46:24.701 | INFO | Finished a round of Baz IO: Wrote 1.589 GiB in 3.514 seconds | 20260508 04:46:24.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.00% / 0.00% / 59.00% | 20260508 04:46:24.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.513645% / 3.97364e-06% / 4.76837e-05% | 20260508 04:46:24.701 | INFO | Baz IO overall: | 20260508 04:46:24.701 | INFO | BytesWritten : 2389627476693 | 20260508 04:46:24.701 | INFO | NumEvents : 1153462763075 | 20260508 04:46:26.228 | INFO | Analyzing chunk frames = [5345280,5345792) | 20260508 04:46:26.228 | INFO | MemInfo: { | 20260508 04:46:26.228 | INFO | "gpu" : " 17 GB", | 20260508 04:46:26.228 | INFO | "huge" : "", | 20260508 04:46:26.228 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:26.228 | INFO | "malloc" : " 14 GB", | 20260508 04:46:26.228 | INFO | "pinned" : " 29 GB" | 20260508 04:46:26.229 | INFO | } | 20260508 04:46:26.229 | INFO | Perf report for the last 5.13874 seconds: | 20260508 04:46:26.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:26.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:26.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:26.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:26.229 | INFO | ANALYSIS : 68.0%, 3.0, 106.3ms, 0 | 20260508 04:46:26.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:26.229 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:46:26.229 | INFO | PRE_HQ : 65.9%, 1.6, 55.8ms, 0 | 20260508 04:46:26.229 | INFO | BAZWRITER : 67.8%, 2.7, 93.2ms, 0 | 20260508 04:46:26.229 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 04:46:26.229 | INFO | Uploaded 36GiB to GPU in 2.55569s (14.0862GiB/s) | 20260508 04:46:26.229 | INFO | Downloaded 38GiB to GPU in 2.6881s (14.1364GiB/s) | 20260508 04:46:31.375 | INFO | Analyzing chunk frames = [5345792,5346304) | 20260508 04:46:31.375 | INFO | MemInfo: { | 20260508 04:46:31.375 | INFO | "gpu" : " 17 GB", | 20260508 04:46:31.375 | INFO | "huge" : "", | 20260508 04:46:31.375 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:31.375 | INFO | "malloc" : " 14 GB", | 20260508 04:46:31.375 | INFO | "pinned" : " 29 GB" | 20260508 04:46:31.375 | INFO | } | 20260508 04:46:31.375 | INFO | Perf report for the last 5.14632 seconds: | 20260508 04:46:31.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:31.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 04:46:31.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:46:31.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:31.375 | INFO | Uploaded 36GiB to GPU in 2.5672s (14.0231GiB/s) | 20260508 04:46:31.375 | INFO | Downloaded 34GiB to GPU in 2.44452s (13.9087GiB/s) | 20260508 04:46:36.515 | INFO | Analyzing chunk frames = [5346304,5346816) | 20260508 04:46:36.515 | INFO | MemInfo: { | 20260508 04:46:36.515 | INFO | "gpu" : " 17 GB", | 20260508 04:46:36.515 | INFO | "huge" : "", | 20260508 04:46:36.515 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:36.515 | INFO | "malloc" : " 14 GB", | 20260508 04:46:36.515 | INFO | "pinned" : " 29 GB" | 20260508 04:46:36.515 | INFO | } | 20260508 04:46:36.515 | INFO | Perf report for the last 5.1399 seconds: | 20260508 04:46:36.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:36.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 04:46:36.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 04:46:36.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:36.515 | INFO | Uploaded 36GiB to GPU in 2.52152s (14.2771GiB/s) | 20260508 04:46:36.515 | INFO | Downloaded 34GiB to GPU in 2.40977s (14.1092GiB/s) | 20260508 04:46:41.649 | INFO | Analyzing chunk frames = [5346816,5347328) | 20260508 04:46:41.649 | INFO | MemInfo: { | 20260508 04:46:41.649 | INFO | "gpu" : " 17 GB", | 20260508 04:46:41.649 | INFO | "huge" : "", | 20260508 04:46:41.649 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:41.649 | INFO | "malloc" : " 14 GB", | 20260508 04:46:41.649 | INFO | "pinned" : " 29 GB" | 20260508 04:46:41.649 | INFO | } | 20260508 04:46:41.649 | INFO | Perf report for the last 5.13425 seconds: | 20260508 04:46:41.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:41.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 04:46:41.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:46:41.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:41.649 | INFO | Uploaded 36GiB to GPU in 2.54141s (14.1654GiB/s) | 20260508 04:46:41.649 | INFO | Downloaded 34GiB to GPU in 2.3941s (14.2016GiB/s) | 20260508 04:46:46.786 | INFO | Analyzing chunk frames = [5347328,5347840) | 20260508 04:46:46.786 | INFO | MemInfo: { | 20260508 04:46:46.786 | INFO | "gpu" : " 17 GB", | 20260508 04:46:46.786 | INFO | "huge" : "", | 20260508 04:46:46.786 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:46.786 | INFO | "malloc" : " 14 GB", | 20260508 04:46:46.786 | INFO | "pinned" : " 29 GB" | 20260508 04:46:46.786 | INFO | } | 20260508 04:46:46.786 | INFO | Perf report for the last 5.13681 seconds: | 20260508 04:46:46.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:46.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 04:46:46.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:46:46.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:46.786 | INFO | Uploaded 36GiB to GPU in 2.40318s (14.9801GiB/s) | 20260508 04:46:46.786 | INFO | Downloaded 34GiB to GPU in 2.56766s (13.2416GiB/s) | 20260508 04:46:51.923 | INFO | Analyzing chunk frames = [5347840,5348352) | 20260508 04:46:51.923 | INFO | MemInfo: { | 20260508 04:46:51.923 | INFO | "gpu" : " 17 GB", | 20260508 04:46:51.923 | INFO | "huge" : "", | 20260508 04:46:51.923 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:51.923 | INFO | "malloc" : " 14 GB", | 20260508 04:46:51.923 | INFO | "pinned" : " 29 GB" | 20260508 04:46:51.923 | INFO | } | 20260508 04:46:51.923 | INFO | Perf report for the last 5.13744 seconds: | 20260508 04:46:51.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:51.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.923 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:46:51.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.923 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 04:46:51.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:51.924 | INFO | Uploaded 36GiB to GPU in 2.44032s (14.7522GiB/s) | 20260508 04:46:51.924 | INFO | Downloaded 34GiB to GPU in 2.43936s (13.9381GiB/s) | 20260508 04:46:57.067 | INFO | Analyzing chunk frames = [5348352,5348864) | 20260508 04:46:57.067 | INFO | MemInfo: { | 20260508 04:46:57.067 | INFO | "gpu" : " 17 GB", | 20260508 04:46:57.067 | INFO | "huge" : "", | 20260508 04:46:57.067 | INFO | "huge_shared" : " 29 GB", | 20260508 04:46:57.067 | INFO | "malloc" : " 14 GB", | 20260508 04:46:57.067 | INFO | "pinned" : " 29 GB" | 20260508 04:46:57.067 | INFO | } | 20260508 04:46:57.067 | INFO | Perf report for the last 5.14399 seconds: | 20260508 04:46:57.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:46:57.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.067 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:46:57.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.068 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:46:57.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:46:57.068 | INFO | Uploaded 36GiB to GPU in 2.44995s (14.6942GiB/s) | 20260508 04:46:57.068 | INFO | Downloaded 34GiB to GPU in 2.43407s (13.9684GiB/s) | 20260508 04:47:02.215 | INFO | Analyzing chunk frames = [5348864,5349376) | 20260508 04:47:02.215 | INFO | MemInfo: { | 20260508 04:47:02.215 | INFO | "gpu" : " 17 GB", | 20260508 04:47:02.215 | INFO | "huge" : "", | 20260508 04:47:02.215 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:02.215 | INFO | "malloc" : " 14 GB", | 20260508 04:47:02.215 | INFO | "pinned" : " 29 GB" | 20260508 04:47:02.215 | INFO | } | 20260508 04:47:02.215 | INFO | Perf report for the last 5.14781 seconds: | 20260508 04:47:02.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:02.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.215 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 04:47:02.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.215 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:47:02.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:02.216 | INFO | Uploaded 36GiB to GPU in 2.44957s (14.6964GiB/s) | 20260508 04:47:02.216 | INFO | Downloaded 34GiB to GPU in 2.42664s (14.0112GiB/s) | 20260508 04:47:02.216 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:47:02.216 | INFO | Total mem usage: 16.7 GiB | 20260508 04:47:02.216 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:47:02.216 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:47:02.216 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:47:02.216 | INFO | High Water Breakdown: | 20260508 04:47:02.216 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:47:02.216 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:47:02.216 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:47:02.216 | INFO | ------------------------------------------ | 20260508 04:47:02.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:47:02.216 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:47:02.216 | INFO | Total mem usage: 26.5 GiB | 20260508 04:47:02.216 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:47:02.216 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:47:02.216 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:47:02.216 | INFO | High Water Breakdown: | 20260508 04:47:02.216 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:47:02.216 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:47:02.216 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:47:02.216 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:47:02.216 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:47:02.216 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:47:02.216 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:47:02.216 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:47:02.216 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:47:02.216 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:47:02.216 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:47:02.216 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:47:02.216 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:47:02.216 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:47:02.216 | INFO | ------------------------------------------ | 20260508 04:47:02.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:47:02.216 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:47:02.216 | INFO | Total mem usage: 27.8 GiB | 20260508 04:47:02.217 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:47:02.217 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:47:02.217 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:47:02.217 | INFO | High Water Breakdown: | 20260508 04:47:02.217 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:47:02.217 | INFO | ------------------------------------------ | 20260508 04:47:02.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:47:02.217 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:47:02.217 | INFO | Total mem usage: 15.7 GiB | 20260508 04:47:02.217 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:47:02.217 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:47:02.217 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:47:02.217 | INFO | High Water Breakdown: | 20260508 04:47:02.217 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:47:02.217 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:47:02.217 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:47:02.217 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:47:02.217 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:47:02.217 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:47:02.217 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:47:02.217 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:47:02.217 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:47:02.217 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:47:02.217 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:47:02.217 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:47:02.217 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:47:02.217 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:47:02.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:47:02.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:47:02.217 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:47:02.217 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:47:02.217 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:47:02.217 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:47:02.217 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:47:02.217 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:47:02.217 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:47:02.217 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:47:02.217 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:47:02.217 | INFO | ------------------------------------------ | 20260508 04:47:02.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:47:02.217 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:47:02.217 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:47:02.217 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:47:02.217 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.39e+01 / 1.68e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.13e-02 / 4.51e-03 | 20260508 04:47:02.217 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.46e+00, 5.90e-01 / 2.51e-01 | 20260508 04:47:02.217 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.17e-01 | 20260508 04:47:02.217 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.52e+01, 3.79e+00 / 1.93e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.84e+00 / 1.35e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.17e-01 / 2.84e-01 | 20260508 04:47:02.217 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.23e+00, 2.63e-01 / 1.51e-01 | 20260508 04:47:02.217 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.57e-01, 5.08e-02 / 2.89e-02 | 20260508 04:47:02.217 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:47:02.217 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:47:02.217 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:47:02.217 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 24.21%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 30.80%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 2.26%, LockUpload -- 2.41e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:47:02.217 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260508 04:47:02.312 | INFO | Starting a round of Baz IO | 20260508 04:47:05.734 | INFO | Populating RTMetrics report | 20260508 04:47:05.734 | INFO | RTMetrics report populated | 20260508 04:47:05.734 | INFO | Opening tmp RTMetrics json file | 20260508 04:47:05.747 | INFO | Serializing RTMetrics to json | 20260508 04:47:05.748 | INFO | Serialization done, swapping tmp file now... | 20260508 04:47:05.749 | INFO | RTMetrics done | 20260508 04:47:05.808 | INFO | Finished a round of Baz IO: Wrote 1.588 GiB in 3.495 seconds | 20260508 04:47:05.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.00% / 0.00% / 59.00% | 20260508 04:47:05.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.512449% / 3.97364e-06% / 5.5631e-05% | 20260508 04:47:05.808 | INFO | Baz IO overall: | 20260508 04:47:05.808 | INFO | BytesWritten : 2391332615459 | 20260508 04:47:05.808 | INFO | NumEvents : 1154276447037 | 20260508 04:47:07.348 | INFO | Analyzing chunk frames = [5349376,5349888) | 20260508 04:47:07.348 | INFO | MemInfo: { | 20260508 04:47:07.348 | INFO | "gpu" : " 17 GB", | 20260508 04:47:07.348 | INFO | "huge" : "", | 20260508 04:47:07.348 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:07.348 | INFO | "malloc" : " 14 GB", | 20260508 04:47:07.348 | INFO | "pinned" : " 29 GB" | 20260508 04:47:07.348 | INFO | } | 20260508 04:47:07.348 | INFO | Perf report for the last 5.13308 seconds: | 20260508 04:47:07.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:07.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:07.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:07.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:07.348 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 04:47:07.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:07.348 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:47:07.349 | INFO | PRE_HQ : 65.8%, 1.6, 55.8ms, 0 | 20260508 04:47:07.349 | INFO | BAZWRITER : 67.6%, 2.7, 93.8ms, 0 | 20260508 04:47:07.349 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:47:07.349 | INFO | Uploaded 36GiB to GPU in 2.56023s (14.0612GiB/s) | 20260508 04:47:07.349 | INFO | Downloaded 38GiB to GPU in 2.68247s (14.1661GiB/s) | 20260508 04:47:12.482 | INFO | Analyzing chunk frames = [5349888,5350400) | 20260508 04:47:12.482 | INFO | MemInfo: { | 20260508 04:47:12.482 | INFO | "gpu" : " 17 GB", | 20260508 04:47:12.482 | INFO | "huge" : "", | 20260508 04:47:12.482 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:12.482 | INFO | "malloc" : " 14 GB", | 20260508 04:47:12.482 | INFO | "pinned" : " 29 GB" | 20260508 04:47:12.482 | INFO | } | 20260508 04:47:12.482 | INFO | Perf report for the last 5.13355 seconds: | 20260508 04:47:12.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:12.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 04:47:12.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:47:12.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:12.482 | INFO | Uploaded 36GiB to GPU in 2.59249s (13.8863GiB/s) | 20260508 04:47:12.482 | INFO | Downloaded 34GiB to GPU in 2.43327s (13.973GiB/s) | 20260508 04:47:17.626 | INFO | Analyzing chunk frames = [5350400,5350912) | 20260508 04:47:17.626 | INFO | MemInfo: { | 20260508 04:47:17.626 | INFO | "gpu" : " 17 GB", | 20260508 04:47:17.626 | INFO | "huge" : "", | 20260508 04:47:17.626 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:17.626 | INFO | "malloc" : " 14 GB", | 20260508 04:47:17.626 | INFO | "pinned" : " 29 GB" | 20260508 04:47:17.626 | INFO | } | 20260508 04:47:17.626 | INFO | Perf report for the last 5.14404 seconds: | 20260508 04:47:17.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:17.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 04:47:17.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:47:17.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:17.626 | INFO | Uploaded 36GiB to GPU in 2.54095s (14.1679GiB/s) | 20260508 04:47:17.626 | INFO | Downloaded 34GiB to GPU in 2.40434s (14.1411GiB/s) | 20260508 04:47:22.764 | INFO | Analyzing chunk frames = [5350912,5351424) | 20260508 04:47:22.764 | INFO | MemInfo: { | 20260508 04:47:22.764 | INFO | "gpu" : " 17 GB", | 20260508 04:47:22.764 | INFO | "huge" : "", | 20260508 04:47:22.765 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:22.765 | INFO | "malloc" : " 14 GB", | 20260508 04:47:22.765 | INFO | "pinned" : " 29 GB" | 20260508 04:47:22.765 | INFO | } | 20260508 04:47:22.765 | INFO | Perf report for the last 5.13885 seconds: | 20260508 04:47:22.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:22.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.765 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:47:22.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.765 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:47:22.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:22.766 | INFO | Uploaded 36GiB to GPU in 2.56123s (14.0558GiB/s) | 20260508 04:47:22.766 | INFO | Downloaded 34GiB to GPU in 2.38259s (14.2702GiB/s) | 20260508 04:47:27.901 | INFO | Analyzing chunk frames = [5351424,5351936) | 20260508 04:47:27.901 | INFO | MemInfo: { | 20260508 04:47:27.901 | INFO | "gpu" : " 17 GB", | 20260508 04:47:27.901 | INFO | "huge" : "", | 20260508 04:47:27.901 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:27.901 | INFO | "malloc" : " 14 GB", | 20260508 04:47:27.901 | INFO | "pinned" : " 29 GB" | 20260508 04:47:27.902 | INFO | } | 20260508 04:47:27.902 | INFO | Perf report for the last 5.13676 seconds: | 20260508 04:47:27.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:27.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 04:47:27.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:47:27.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:27.902 | INFO | Uploaded 36GiB to GPU in 2.41183s (14.9265GiB/s) | 20260508 04:47:27.902 | INFO | Downloaded 34GiB to GPU in 2.55359s (13.3146GiB/s) | 20260508 04:47:33.040 | INFO | Analyzing chunk frames = [5351936,5352448) | 20260508 04:47:33.040 | INFO | MemInfo: { | 20260508 04:47:33.040 | INFO | "gpu" : " 17 GB", | 20260508 04:47:33.040 | INFO | "huge" : "", | 20260508 04:47:33.040 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:33.040 | INFO | "malloc" : " 14 GB", | 20260508 04:47:33.040 | INFO | "pinned" : " 29 GB" | 20260508 04:47:33.040 | INFO | } | 20260508 04:47:33.040 | INFO | Perf report for the last 5.13862 seconds: | 20260508 04:47:33.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:33.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.040 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:47:33.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.040 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:47:33.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:33.041 | INFO | Uploaded 36GiB to GPU in 2.45559s (14.6604GiB/s) | 20260508 04:47:33.041 | INFO | Downloaded 34GiB to GPU in 2.4128s (14.0915GiB/s) | 20260508 04:47:38.182 | INFO | Analyzing chunk frames = [5352448,5352960) | 20260508 04:47:38.182 | INFO | MemInfo: { | 20260508 04:47:38.182 | INFO | "gpu" : " 17 GB", | 20260508 04:47:38.182 | INFO | "huge" : "", | 20260508 04:47:38.182 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:38.182 | INFO | "malloc" : " 14 GB", | 20260508 04:47:38.182 | INFO | "pinned" : " 29 GB" | 20260508 04:47:38.182 | INFO | } | 20260508 04:47:38.182 | INFO | Perf report for the last 5.14161 seconds: | 20260508 04:47:38.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:38.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:47:38.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:47:38.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:38.182 | INFO | Uploaded 36GiB to GPU in 2.47307s (14.5568GiB/s) | 20260508 04:47:38.182 | INFO | Downloaded 34GiB to GPU in 2.42227s (14.0364GiB/s) | 20260508 04:47:43.325 | INFO | Analyzing chunk frames = [5352960,5353472) | 20260508 04:47:43.325 | INFO | MemInfo: { | 20260508 04:47:43.325 | INFO | "gpu" : " 17 GB", | 20260508 04:47:43.325 | INFO | "huge" : "", | 20260508 04:47:43.325 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:43.325 | INFO | "malloc" : " 14 GB", | 20260508 04:47:43.325 | INFO | "pinned" : " 29 GB" | 20260508 04:47:43.325 | INFO | } | 20260508 04:47:43.325 | INFO | Perf report for the last 5.14311 seconds: | 20260508 04:47:43.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:43.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.325 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:47:43.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.325 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:47:43.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:43.326 | INFO | Uploaded 36GiB to GPU in 2.44772s (14.7076GiB/s) | 20260508 04:47:43.326 | INFO | Downloaded 34GiB to GPU in 2.4113s (14.1003GiB/s) | 20260508 04:47:43.422 | INFO | Starting a round of Baz IO | 20260508 04:47:46.854 | INFO | Populating RTMetrics report | 20260508 04:47:46.854 | INFO | RTMetrics report populated | 20260508 04:47:46.854 | INFO | Opening tmp RTMetrics json file | 20260508 04:47:46.866 | INFO | Serializing RTMetrics to json | 20260508 04:47:46.868 | INFO | Serialization done, swapping tmp file now... | 20260508 04:47:46.868 | INFO | RTMetrics done | 20260508 04:47:46.928 | INFO | Finished a round of Baz IO: Wrote 1.588 GiB in 3.505 seconds | 20260508 04:47:46.928 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.99% / 0.00% / 59.00% | 20260508 04:47:46.928 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.511344% / 0% / 6.35783e-05% | 20260508 04:47:46.928 | INFO | Baz IO overall: | 20260508 04:47:46.928 | INFO | BytesWritten : 2393037624771 | 20260508 04:47:46.928 | INFO | NumEvents : 1155090156883 | 20260508 04:47:48.462 | INFO | Analyzing chunk frames = [5353472,5353984) | 20260508 04:47:48.462 | INFO | MemInfo: { | 20260508 04:47:48.462 | INFO | "gpu" : " 17 GB", | 20260508 04:47:48.462 | INFO | "huge" : "", | 20260508 04:47:48.462 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:48.462 | INFO | "malloc" : " 14 GB", | 20260508 04:47:48.462 | INFO | "pinned" : " 29 GB" | 20260508 04:47:48.462 | INFO | } | 20260508 04:47:48.462 | INFO | Perf report for the last 5.13705 seconds: | 20260508 04:47:48.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:48.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:48.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:48.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:48.462 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 04:47:48.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:48.462 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:47:48.462 | INFO | PRE_HQ : 65.7%, 1.6, 55.6ms, 0 | 20260508 04:47:48.462 | INFO | BAZWRITER : 67.7%, 2.7, 92.3ms, 0 | 20260508 04:47:48.462 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:47:48.463 | INFO | Uploaded 36GiB to GPU in 2.57669s (13.9714GiB/s) | 20260508 04:47:48.463 | INFO | Downloaded 38GiB to GPU in 2.70464s (14.0499GiB/s) | 20260508 04:47:53.599 | INFO | Analyzing chunk frames = [5353984,5354496) | 20260508 04:47:53.599 | INFO | MemInfo: { | 20260508 04:47:53.599 | INFO | "gpu" : " 17 GB", | 20260508 04:47:53.599 | INFO | "huge" : "", | 20260508 04:47:53.599 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:53.599 | INFO | "malloc" : " 14 GB", | 20260508 04:47:53.599 | INFO | "pinned" : " 29 GB" | 20260508 04:47:53.599 | INFO | } | 20260508 04:47:53.599 | INFO | Perf report for the last 5.13724 seconds: | 20260508 04:47:53.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:53.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.599 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 04:47:53.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.599 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:47:53.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:53.600 | INFO | Uploaded 36GiB to GPU in 2.60196s (13.8357GiB/s) | 20260508 04:47:53.600 | INFO | Downloaded 34GiB to GPU in 2.43679s (13.9528GiB/s) | 20260508 04:47:58.736 | INFO | Analyzing chunk frames = [5354496,5355008) | 20260508 04:47:58.736 | INFO | MemInfo: { | 20260508 04:47:58.736 | INFO | "gpu" : " 17 GB", | 20260508 04:47:58.736 | INFO | "huge" : "", | 20260508 04:47:58.736 | INFO | "huge_shared" : " 29 GB", | 20260508 04:47:58.736 | INFO | "malloc" : " 14 GB", | 20260508 04:47:58.736 | INFO | "pinned" : " 29 GB" | 20260508 04:47:58.737 | INFO | } | 20260508 04:47:58.737 | INFO | Perf report for the last 5.13749 seconds: | 20260508 04:47:58.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:47:58.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:47:58.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:47:58.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:47:58.737 | INFO | Uploaded 36GiB to GPU in 2.54465s (14.1473GiB/s) | 20260508 04:47:58.737 | INFO | Downloaded 34GiB to GPU in 2.39908s (14.1721GiB/s) | 20260508 04:48:03.877 | INFO | Analyzing chunk frames = [5355008,5355520) | 20260508 04:48:03.877 | INFO | MemInfo: { | 20260508 04:48:03.877 | INFO | "gpu" : " 17 GB", | 20260508 04:48:03.877 | INFO | "huge" : "", | 20260508 04:48:03.877 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:03.877 | INFO | "malloc" : " 14 GB", | 20260508 04:48:03.877 | INFO | "pinned" : " 29 GB" | 20260508 04:48:03.877 | INFO | } | 20260508 04:48:03.877 | INFO | Perf report for the last 5.1401 seconds: | 20260508 04:48:03.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:03.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 04:48:03.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:48:03.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:03.877 | INFO | Uploaded 36GiB to GPU in 2.54302s (14.1564GiB/s) | 20260508 04:48:03.877 | INFO | Downloaded 34GiB to GPU in 2.37665s (14.3059GiB/s) | 20260508 04:48:09.023 | INFO | Analyzing chunk frames = [5355520,5356032) | 20260508 04:48:09.023 | INFO | MemInfo: { | 20260508 04:48:09.023 | INFO | "gpu" : " 17 GB", | 20260508 04:48:09.023 | INFO | "huge" : "", | 20260508 04:48:09.023 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:09.023 | INFO | "malloc" : " 14 GB", | 20260508 04:48:09.023 | INFO | "pinned" : " 29 GB" | 20260508 04:48:09.023 | INFO | } | 20260508 04:48:09.023 | INFO | Perf report for the last 5.14603 seconds: | 20260508 04:48:09.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:09.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:48:09.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 04:48:09.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:09.023 | INFO | Uploaded 36GiB to GPU in 2.37275s (15.1723GiB/s) | 20260508 04:48:09.023 | INFO | Downloaded 34GiB to GPU in 2.54695s (13.3493GiB/s) | 20260508 04:48:14.161 | INFO | Analyzing chunk frames = [5356032,5356544) | 20260508 04:48:14.161 | INFO | MemInfo: { | 20260508 04:48:14.161 | INFO | "gpu" : " 17 GB", | 20260508 04:48:14.161 | INFO | "huge" : "", | 20260508 04:48:14.161 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:14.161 | INFO | "malloc" : " 14 GB", | 20260508 04:48:14.161 | INFO | "pinned" : " 29 GB" | 20260508 04:48:14.161 | INFO | } | 20260508 04:48:14.161 | INFO | Perf report for the last 5.13787 seconds: | 20260508 04:48:14.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:14.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:48:14.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:48:14.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:14.161 | INFO | Uploaded 36GiB to GPU in 2.44139s (14.7457GiB/s) | 20260508 04:48:14.161 | INFO | Downloaded 34GiB to GPU in 2.41987s (14.0503GiB/s) | 20260508 04:48:19.296 | INFO | Analyzing chunk frames = [5356544,5357056) | 20260508 04:48:19.297 | INFO | MemInfo: { | 20260508 04:48:19.297 | INFO | "gpu" : " 17 GB", | 20260508 04:48:19.297 | INFO | "huge" : "", | 20260508 04:48:19.297 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:19.297 | INFO | "malloc" : " 14 GB", | 20260508 04:48:19.297 | INFO | "pinned" : " 29 GB" | 20260508 04:48:19.297 | INFO | } | 20260508 04:48:19.297 | INFO | Perf report for the last 5.13597 seconds: | 20260508 04:48:19.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:19.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 04:48:19.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 04:48:19.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:19.297 | INFO | Uploaded 36GiB to GPU in 2.42439s (14.8491GiB/s) | 20260508 04:48:19.297 | INFO | Downloaded 34GiB to GPU in 2.41462s (14.0809GiB/s) | 20260508 04:48:24.438 | INFO | Analyzing chunk frames = [5357056,5357568) | 20260508 04:48:24.438 | INFO | MemInfo: { | 20260508 04:48:24.438 | INFO | "gpu" : " 17 GB", | 20260508 04:48:24.438 | INFO | "huge" : "", | 20260508 04:48:24.438 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:24.438 | INFO | "malloc" : " 14 GB", | 20260508 04:48:24.438 | INFO | "pinned" : " 29 GB" | 20260508 04:48:24.438 | INFO | } | 20260508 04:48:24.438 | INFO | Perf report for the last 5.1412 seconds: | 20260508 04:48:24.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:24.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:48:24.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:48:24.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:24.438 | INFO | Uploaded 36GiB to GPU in 2.4467s (14.7137GiB/s) | 20260508 04:48:24.438 | INFO | Downloaded 34GiB to GPU in 2.418s (14.0612GiB/s) | 20260508 04:48:24.438 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:48:24.438 | INFO | Total mem usage: 16.7 GiB | 20260508 04:48:24.438 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:48:24.438 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 04:48:24.438 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:48:24.438 | INFO | High Water Breakdown: | 20260508 04:48:24.438 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:48:24.438 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:48:24.438 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:48:24.438 | INFO | ------------------------------------------ | 20260508 04:48:24.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:48:24.438 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:48:24.438 | INFO | Total mem usage: 26.5 GiB | 20260508 04:48:24.438 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:48:24.438 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:48:24.438 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:48:24.438 | INFO | High Water Breakdown: | 20260508 04:48:24.438 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:48:24.438 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:48:24.438 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:48:24.438 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:48:24.438 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:48:24.438 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:48:24.438 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:48:24.438 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:48:24.438 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:48:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:48:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:48:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:48:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:48:24.438 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:48:24.438 | INFO | ------------------------------------------ | 20260508 04:48:24.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:48:24.438 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:48:24.438 | INFO | Total mem usage: 27.8 GiB | 20260508 04:48:24.438 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:48:24.438 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:48:24.438 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:48:24.438 | INFO | High Water Breakdown: | 20260508 04:48:24.438 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:48:24.438 | INFO | ------------------------------------------ | 20260508 04:48:24.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:48:24.438 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:48:24.438 | INFO | Total mem usage: 15.7 GiB | 20260508 04:48:24.438 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:48:24.438 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:48:24.438 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:48:24.438 | INFO | High Water Breakdown: | 20260508 04:48:24.438 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:48:24.438 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:48:24.438 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:48:24.438 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:48:24.438 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:48:24.438 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:48:24.438 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:48:24.439 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:48:24.439 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:48:24.439 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:48:24.439 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:48:24.439 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:48:24.439 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:48:24.439 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:48:24.439 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:48:24.439 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:48:24.439 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:48:24.439 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:48:24.439 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:48:24.439 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:48:24.439 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:48:24.439 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:48:24.439 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:48:24.439 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:48:24.439 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:48:24.439 | INFO | ------------------------------------------ | 20260508 04:48:24.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:48:24.439 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:48:24.439 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:48:24.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:48:24.439 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.39e+01 / 1.81e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.11e-02 / 8.98e-03 | 20260508 04:48:24.439 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.43e+00, 5.89e-01 / 2.69e-01 | 20260508 04:48:24.439 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.10e-01 / 1.60e-01 | 20260508 04:48:24.439 | INFO | PerfReport: 15.97%, DME -- 4.17e+01 / 4.56e+01, 3.96e+00 / 2.10e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.70e+00 / 1.39e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.59e+00, 4.20e-01 / 2.38e-01 | 20260508 04:48:24.439 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.22e+00, 2.64e-01 / 1.22e-01 | 20260508 04:48:24.439 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.54e-01, 5.08e-02 / 2.34e-02 | 20260508 04:48:24.439 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:48:24.439 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:48:24.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:48:24.439 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 24.49%, Upload -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 24.13%, Download -- 2.57e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 2.47%, LockUpload -- 2.62e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 2.54%, LockDownload -- 2.70e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:48:24.439 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.25e-02 / 0.00e+00 | 20260508 04:48:24.535 | INFO | Starting a round of Baz IO | 20260508 04:48:27.965 | INFO | Populating RTMetrics report | 20260508 04:48:27.965 | INFO | RTMetrics report populated | 20260508 04:48:27.965 | INFO | Opening tmp RTMetrics json file | 20260508 04:48:27.978 | INFO | Serializing RTMetrics to json | 20260508 04:48:27.979 | INFO | Serialization done, swapping tmp file now... | 20260508 04:48:27.980 | INFO | RTMetrics done | 20260508 04:48:28.032 | INFO | Finished a round of Baz IO: Wrote 1.589 GiB in 3.497 seconds | 20260508 04:48:28.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.99% / 0.00% / 59.01% | 20260508 04:48:28.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.511976% / 7.94729e-06% / 6.35783e-05% | 20260508 04:48:28.032 | INFO | Baz IO overall: | 20260508 04:48:28.032 | INFO | BytesWritten : 2394744082326 | 20260508 04:48:28.033 | INFO | NumEvents : 1155904981443 | 20260508 04:48:29.576 | INFO | Analyzing chunk frames = [5357568,5358080) | 20260508 04:48:29.576 | INFO | MemInfo: { | 20260508 04:48:29.576 | INFO | "gpu" : " 17 GB", | 20260508 04:48:29.576 | INFO | "huge" : "", | 20260508 04:48:29.576 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:29.576 | INFO | "malloc" : " 14 GB", | 20260508 04:48:29.576 | INFO | "pinned" : " 29 GB" | 20260508 04:48:29.576 | INFO | } | 20260508 04:48:29.576 | INFO | Perf report for the last 5.13761 seconds: | 20260508 04:48:29.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:29.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:29.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:29.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:29.576 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 04:48:29.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:29.576 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 04:48:29.576 | INFO | PRE_HQ : 66.1%, 1.6, 55.5ms, 0 | 20260508 04:48:29.576 | INFO | BAZWRITER : 67.6%, 2.7, 92.7ms, 0 | 20260508 04:48:29.576 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:48:29.576 | INFO | Uploaded 36GiB to GPU in 2.52264s (14.2708GiB/s) | 20260508 04:48:29.577 | INFO | Downloaded 38GiB to GPU in 2.68485s (14.1535GiB/s) | 20260508 04:48:34.712 | INFO | Analyzing chunk frames = [5358080,5358592) | 20260508 04:48:34.712 | INFO | MemInfo: { | 20260508 04:48:34.712 | INFO | "gpu" : " 17 GB", | 20260508 04:48:34.712 | INFO | "huge" : "", | 20260508 04:48:34.712 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:34.712 | INFO | "malloc" : " 14 GB", | 20260508 04:48:34.712 | INFO | "pinned" : " 29 GB" | 20260508 04:48:34.712 | INFO | } | 20260508 04:48:34.712 | INFO | Perf report for the last 5.13665 seconds: | 20260508 04:48:34.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:34.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.712 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:48:34.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.712 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 04:48:34.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:34.713 | INFO | Uploaded 36GiB to GPU in 2.57627s (13.9737GiB/s) | 20260508 04:48:34.713 | INFO | Downloaded 34GiB to GPU in 2.44324s (13.916GiB/s) | 20260508 04:48:39.851 | INFO | Analyzing chunk frames = [5358592,5359104) | 20260508 04:48:39.851 | INFO | MemInfo: { | 20260508 04:48:39.851 | INFO | "gpu" : " 17 GB", | 20260508 04:48:39.851 | INFO | "huge" : "", | 20260508 04:48:39.851 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:39.851 | INFO | "malloc" : " 14 GB", | 20260508 04:48:39.851 | INFO | "pinned" : " 29 GB" | 20260508 04:48:39.851 | INFO | } | 20260508 04:48:39.851 | INFO | Perf report for the last 5.13876 seconds: | 20260508 04:48:39.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:39.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 04:48:39.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:48:39.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:39.851 | INFO | Uploaded 36GiB to GPU in 2.55081s (14.1132GiB/s) | 20260508 04:48:39.852 | INFO | Downloaded 34GiB to GPU in 2.43046s (13.9891GiB/s) | 20260508 04:48:44.989 | INFO | Analyzing chunk frames = [5359104,5359616) | 20260508 04:48:44.989 | INFO | MemInfo: { | 20260508 04:48:44.989 | INFO | "gpu" : " 17 GB", | 20260508 04:48:44.989 | INFO | "huge" : "", | 20260508 04:48:44.989 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:44.989 | INFO | "malloc" : " 14 GB", | 20260508 04:48:44.989 | INFO | "pinned" : " 29 GB" | 20260508 04:48:44.989 | INFO | } | 20260508 04:48:44.989 | INFO | Perf report for the last 5.13793 seconds: | 20260508 04:48:44.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:44.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.989 | INFO | ANALYSIS : 67.1%, 3.0, 103.9ms, 0 | 20260508 04:48:44.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.989 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:48:44.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:44.990 | INFO | Uploaded 36GiB to GPU in 2.56864s (14.0152GiB/s) | 20260508 04:48:44.990 | INFO | Downloaded 34GiB to GPU in 2.42297s (14.0323GiB/s) | 20260508 04:48:50.137 | INFO | Analyzing chunk frames = [5359616,5360128) | 20260508 04:48:50.137 | INFO | MemInfo: { | 20260508 04:48:50.137 | INFO | "gpu" : " 17 GB", | 20260508 04:48:50.137 | INFO | "huge" : "", | 20260508 04:48:50.137 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:50.137 | INFO | "malloc" : " 14 GB", | 20260508 04:48:50.137 | INFO | "pinned" : " 29 GB" | 20260508 04:48:50.137 | INFO | } | 20260508 04:48:50.137 | INFO | Perf report for the last 5.14818 seconds: | 20260508 04:48:50.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:50.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.137 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 04:48:50.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.137 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 04:48:50.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:50.138 | INFO | Uploaded 36GiB to GPU in 2.3955s (15.0282GiB/s) | 20260508 04:48:50.138 | INFO | Downloaded 34GiB to GPU in 2.55124s (13.3269GiB/s) | 20260508 04:48:55.278 | INFO | Analyzing chunk frames = [5360128,5360640) | 20260508 04:48:55.278 | INFO | MemInfo: { | 20260508 04:48:55.278 | INFO | "gpu" : " 17 GB", | 20260508 04:48:55.278 | INFO | "huge" : "", | 20260508 04:48:55.278 | INFO | "huge_shared" : " 29 GB", | 20260508 04:48:55.278 | INFO | "malloc" : " 14 GB", | 20260508 04:48:55.278 | INFO | "pinned" : " 29 GB" | 20260508 04:48:55.278 | INFO | } | 20260508 04:48:55.278 | INFO | Perf report for the last 5.14082 seconds: | 20260508 04:48:55.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:48:55.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:48:55.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:48:55.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:48:55.278 | INFO | Uploaded 36GiB to GPU in 2.42203s (14.8635GiB/s) | 20260508 04:48:55.278 | INFO | Downloaded 34GiB to GPU in 2.4173s (14.0653GiB/s) | 20260508 04:49:00.411 | INFO | Analyzing chunk frames = [5360640,5361152) | 20260508 04:49:00.411 | INFO | MemInfo: { | 20260508 04:49:00.411 | INFO | "gpu" : " 17 GB", | 20260508 04:49:00.411 | INFO | "huge" : "", | 20260508 04:49:00.411 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:00.411 | INFO | "malloc" : " 14 GB", | 20260508 04:49:00.411 | INFO | "pinned" : " 29 GB" | 20260508 04:49:00.411 | INFO | } | 20260508 04:49:00.411 | INFO | Perf report for the last 5.13315 seconds: | 20260508 04:49:00.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:00.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.411 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:49:00.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.411 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 04:49:00.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:00.412 | INFO | Uploaded 36GiB to GPU in 2.43875s (14.7616GiB/s) | 20260508 04:49:00.412 | INFO | Downloaded 34GiB to GPU in 2.43159s (13.9826GiB/s) | 20260508 04:49:05.547 | INFO | Analyzing chunk frames = [5361152,5361664) | 20260508 04:49:05.547 | INFO | MemInfo: { | 20260508 04:49:05.547 | INFO | "gpu" : " 17 GB", | 20260508 04:49:05.547 | INFO | "huge" : "", | 20260508 04:49:05.547 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:05.547 | INFO | "malloc" : " 14 GB", | 20260508 04:49:05.547 | INFO | "pinned" : " 29 GB" | 20260508 04:49:05.547 | INFO | } | 20260508 04:49:05.547 | INFO | Perf report for the last 5.13589 seconds: | 20260508 04:49:05.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:05.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.547 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:49:05.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.547 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 04:49:05.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:05.548 | INFO | Uploaded 36GiB to GPU in 2.46164s (14.6244GiB/s) | 20260508 04:49:05.548 | INFO | Downloaded 34GiB to GPU in 2.4347s (13.9647GiB/s) | 20260508 04:49:05.645 | INFO | Starting a round of Baz IO | 20260508 04:49:09.067 | INFO | Populating RTMetrics report | 20260508 04:49:09.067 | INFO | RTMetrics report populated | 20260508 04:49:09.067 | INFO | Opening tmp RTMetrics json file | 20260508 04:49:09.068 | INFO | Serializing RTMetrics to json | 20260508 04:49:09.083 | INFO | Serialization done, swapping tmp file now... | 20260508 04:49:09.085 | INFO | RTMetrics done | 20260508 04:49:09.143 | INFO | Finished a round of Baz IO: Wrote 1.589 GiB in 3.498 seconds | 20260508 04:49:09.143 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.99% / 0.00% / 59.01% | 20260508 04:49:09.143 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.511797% / 3.97364e-06% / 8.34465e-05% | 20260508 04:49:09.143 | INFO | Baz IO overall: | 20260508 04:49:09.143 | INFO | BytesWritten : 2396450725917 | 20260508 04:49:09.143 | INFO | NumEvents : 1156719848133 | 20260508 04:49:10.688 | INFO | Analyzing chunk frames = [5361664,5362176) | 20260508 04:49:10.688 | INFO | MemInfo: { | 20260508 04:49:10.688 | INFO | "gpu" : " 17 GB", | 20260508 04:49:10.688 | INFO | "huge" : "", | 20260508 04:49:10.688 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:10.688 | INFO | "malloc" : " 14 GB", | 20260508 04:49:10.688 | INFO | "pinned" : " 29 GB" | 20260508 04:49:10.692 | INFO | } | 20260508 04:49:10.692 | INFO | Perf report for the last 5.14507 seconds: | 20260508 04:49:10.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:10.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:10.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:10.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:10.692 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 04:49:10.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:10.692 | INFO | RT_METRICS : 39.7%, 1.0, 20.9ms, 0 | 20260508 04:49:10.692 | INFO | PRE_HQ : 65.5%, 1.6, 55.5ms, 0 | 20260508 04:49:10.692 | INFO | BAZWRITER : 67.5%, 2.6, 91.8ms, 0 | 20260508 04:49:10.692 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:49:10.692 | INFO | Uploaded 36GiB to GPU in 2.5377s (14.1861GiB/s) | 20260508 04:49:10.692 | INFO | Downloaded 38GiB to GPU in 2.70307s (14.0581GiB/s) | 20260508 04:49:15.829 | INFO | Analyzing chunk frames = [5362176,5362688) | 20260508 04:49:15.829 | INFO | MemInfo: { | 20260508 04:49:15.829 | INFO | "gpu" : " 17 GB", | 20260508 04:49:15.829 | INFO | "huge" : "", | 20260508 04:49:15.829 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:15.829 | INFO | "malloc" : " 14 GB", | 20260508 04:49:15.829 | INFO | "pinned" : " 29 GB" | 20260508 04:49:15.829 | INFO | } | 20260508 04:49:15.829 | INFO | Perf report for the last 5.13688 seconds: | 20260508 04:49:15.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:15.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 04:49:15.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:49:15.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:15.829 | INFO | Uploaded 36GiB to GPU in 2.48814s (14.4686GiB/s) | 20260508 04:49:15.829 | INFO | Downloaded 34GiB to GPU in 2.43362s (13.971GiB/s) | 20260508 04:49:20.977 | INFO | Analyzing chunk frames = [5362688,5363200) | 20260508 04:49:20.977 | INFO | MemInfo: { | 20260508 04:49:20.977 | INFO | "gpu" : " 17 GB", | 20260508 04:49:20.977 | INFO | "huge" : "", | 20260508 04:49:20.977 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:20.977 | INFO | "malloc" : " 14 GB", | 20260508 04:49:20.977 | INFO | "pinned" : " 29 GB" | 20260508 04:49:20.977 | INFO | } | 20260508 04:49:20.977 | INFO | Perf report for the last 5.14779 seconds: | 20260508 04:49:20.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:20.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 04:49:20.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:49:20.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:20.977 | INFO | Uploaded 36GiB to GPU in 2.52992s (14.2297GiB/s) | 20260508 04:49:20.977 | INFO | Downloaded 34GiB to GPU in 2.41698s (14.0671GiB/s) | 20260508 04:49:26.110 | INFO | Analyzing chunk frames = [5363200,5363712) | 20260508 04:49:26.110 | INFO | MemInfo: { | 20260508 04:49:26.110 | INFO | "gpu" : " 17 GB", | 20260508 04:49:26.110 | INFO | "huge" : "", | 20260508 04:49:26.110 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:26.110 | INFO | "malloc" : " 14 GB", | 20260508 04:49:26.110 | INFO | "pinned" : " 29 GB" | 20260508 04:49:26.110 | INFO | } | 20260508 04:49:26.110 | INFO | Perf report for the last 5.13358 seconds: | 20260508 04:49:26.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:26.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.111 | INFO | ANALYSIS : 66.9%, 3.0, 103.3ms, 0 | 20260508 04:49:26.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.111 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:49:26.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:26.111 | INFO | Uploaded 36GiB to GPU in 2.53089s (14.2242GiB/s) | 20260508 04:49:26.111 | INFO | Downloaded 34GiB to GPU in 2.39494s (14.1966GiB/s) | 20260508 04:49:31.248 | INFO | Analyzing chunk frames = [5363712,5364224) | 20260508 04:49:31.248 | INFO | MemInfo: { | 20260508 04:49:31.248 | INFO | "gpu" : " 17 GB", | 20260508 04:49:31.248 | INFO | "huge" : "", | 20260508 04:49:31.248 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:31.248 | INFO | "malloc" : " 14 GB", | 20260508 04:49:31.248 | INFO | "pinned" : " 29 GB" | 20260508 04:49:31.248 | INFO | } | 20260508 04:49:31.248 | INFO | Perf report for the last 5.13739 seconds: | 20260508 04:49:31.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:31.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 04:49:31.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:49:31.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:31.248 | INFO | Uploaded 36GiB to GPU in 2.37072s (15.1852GiB/s) | 20260508 04:49:31.249 | INFO | Downloaded 34GiB to GPU in 2.56365s (13.2623GiB/s) | 20260508 04:49:36.385 | INFO | Analyzing chunk frames = [5364224,5364736) | 20260508 04:49:36.385 | INFO | MemInfo: { | 20260508 04:49:36.385 | INFO | "gpu" : " 17 GB", | 20260508 04:49:36.385 | INFO | "huge" : "", | 20260508 04:49:36.385 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:36.385 | INFO | "malloc" : " 14 GB", | 20260508 04:49:36.385 | INFO | "pinned" : " 29 GB" | 20260508 04:49:36.385 | INFO | } | 20260508 04:49:36.386 | INFO | Perf report for the last 5.13767 seconds: | 20260508 04:49:36.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:36.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 04:49:36.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:49:36.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:36.386 | INFO | Uploaded 36GiB to GPU in 2.45476s (14.6654GiB/s) | 20260508 04:49:36.386 | INFO | Downloaded 34GiB to GPU in 2.41373s (14.0861GiB/s) | 20260508 04:49:41.529 | INFO | Analyzing chunk frames = [5364736,5365248) | 20260508 04:49:41.529 | INFO | MemInfo: { | 20260508 04:49:41.529 | INFO | "gpu" : " 17 GB", | 20260508 04:49:41.529 | INFO | "huge" : "", | 20260508 04:49:41.529 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:41.529 | INFO | "malloc" : " 14 GB", | 20260508 04:49:41.529 | INFO | "pinned" : " 29 GB" | 20260508 04:49:41.529 | INFO | } | 20260508 04:49:41.529 | INFO | Perf report for the last 5.14346 seconds: | 20260508 04:49:41.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:41.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:49:41.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260508 04:49:41.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:41.529 | INFO | Uploaded 36GiB to GPU in 2.44202s (14.7419GiB/s) | 20260508 04:49:41.529 | INFO | Downloaded 34GiB to GPU in 2.40495s (14.1375GiB/s) | 20260508 04:49:46.666 | INFO | Analyzing chunk frames = [5365248,5365760) | 20260508 04:49:46.666 | INFO | MemInfo: { | 20260508 04:49:46.666 | INFO | "gpu" : " 17 GB", | 20260508 04:49:46.666 | INFO | "huge" : "", | 20260508 04:49:46.666 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:46.666 | INFO | "malloc" : " 14 GB", | 20260508 04:49:46.666 | INFO | "pinned" : " 29 GB" | 20260508 04:49:46.666 | INFO | } | 20260508 04:49:46.666 | INFO | Perf report for the last 5.13734 seconds: | 20260508 04:49:46.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:46.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.666 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:49:46.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.666 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:49:46.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:46.667 | INFO | Uploaded 36GiB to GPU in 2.46813s (14.5859GiB/s) | 20260508 04:49:46.667 | INFO | Downloaded 34GiB to GPU in 2.42019s (14.0485GiB/s) | 20260508 04:49:46.667 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:49:46.667 | INFO | Total mem usage: 16.7 GiB | 20260508 04:49:46.667 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:49:46.667 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 04:49:46.667 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:49:46.667 | INFO | High Water Breakdown: | 20260508 04:49:46.667 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:49:46.667 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:49:46.667 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:49:46.667 | INFO | ------------------------------------------ | 20260508 04:49:46.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:49:46.667 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:49:46.667 | INFO | Total mem usage: 26.5 GiB | 20260508 04:49:46.667 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:49:46.667 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 04:49:46.667 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 04:49:46.667 | INFO | High Water Breakdown: | 20260508 04:49:46.667 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:49:46.667 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:49:46.667 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:49:46.667 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:49:46.667 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:49:46.667 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:49:46.667 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:49:46.667 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:49:46.667 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:49:46.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:49:46.668 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:49:46.668 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:49:46.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:49:46.668 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:49:46.668 | INFO | ------------------------------------------ | 20260508 04:49:46.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:49:46.668 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:49:46.668 | INFO | Total mem usage: 27.8 GiB | 20260508 04:49:46.668 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:49:46.668 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:49:46.668 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:49:46.668 | INFO | High Water Breakdown: | 20260508 04:49:46.668 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:49:46.668 | INFO | ------------------------------------------ | 20260508 04:49:46.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:49:46.668 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:49:46.668 | INFO | Total mem usage: 15.7 GiB | 20260508 04:49:46.668 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:49:46.668 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:49:46.668 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:49:46.668 | INFO | High Water Breakdown: | 20260508 04:49:46.668 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:49:46.668 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:49:46.668 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:49:46.668 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:49:46.668 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:49:46.668 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:49:46.668 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:49:46.668 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:49:46.668 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:49:46.668 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:49:46.668 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:49:46.668 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:49:46.668 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:49:46.668 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:49:46.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:49:46.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:49:46.668 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:49:46.668 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:49:46.668 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:49:46.668 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:49:46.668 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:49:46.668 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:49:46.668 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:49:46.668 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:49:46.668 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:49:46.668 | INFO | ------------------------------------------ | 20260508 04:49:46.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:49:46.668 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:49:46.668 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:49:46.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:49:46.668 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.42e+01 / 1.70e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.20e-02 / 6.49e-03 | 20260508 04:49:46.668 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.47e+00, 5.94e-01 / 2.17e-01 | 20260508 04:49:46.668 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.20e-01 | 20260508 04:49:46.668 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.50e+01, 3.78e+00 / 2.02e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 1.37e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.61e+00, 4.20e-01 / 2.38e-01 | 20260508 04:49:46.668 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.25e+00, 2.66e-01 / 1.45e-01 | 20260508 04:49:46.668 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.60e-01, 5.09e-02 / 2.63e-02 | 20260508 04:49:46.668 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:49:46.668 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:49:46.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:49:46.668 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 24.35%, Upload -- 2.59e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 24.24%, Download -- 2.58e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 30.82%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 2.22%, LockUpload -- 2.36e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 2.66%, LockDownload -- 2.82e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 04:49:46.668 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260508 04:49:46.767 | INFO | Starting a round of Baz IO | 20260508 04:49:50.192 | INFO | Populating RTMetrics report | 20260508 04:49:50.192 | INFO | RTMetrics report populated | 20260508 04:49:50.192 | INFO | Opening tmp RTMetrics json file | 20260508 04:49:50.205 | INFO | Serializing RTMetrics to json | 20260508 04:49:50.206 | INFO | Serialization done, swapping tmp file now... | 20260508 04:49:50.207 | INFO | RTMetrics done | 20260508 04:49:50.269 | INFO | Finished a round of Baz IO: Wrote 1.588 GiB in 3.502 seconds | 20260508 04:49:50.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.99% / 0.00% / 59.01% | 20260508 04:49:50.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.508515% / 0% / 4.76837e-05% | 20260508 04:49:50.270 | INFO | Baz IO overall: | 20260508 04:49:50.270 | INFO | BytesWritten : 2398156311518 | 20260508 04:49:50.270 | INFO | NumEvents : 1157533804860 | 20260508 04:49:51.802 | INFO | Analyzing chunk frames = [5365760,5366272) | 20260508 04:49:51.802 | INFO | MemInfo: { | 20260508 04:49:51.802 | INFO | "gpu" : " 17 GB", | 20260508 04:49:51.802 | INFO | "huge" : "", | 20260508 04:49:51.802 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:51.802 | INFO | "malloc" : " 14 GB", | 20260508 04:49:51.802 | INFO | "pinned" : " 29 GB" | 20260508 04:49:51.802 | INFO | } | 20260508 04:49:51.802 | INFO | Perf report for the last 5.13568 seconds: | 20260508 04:49:51.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:51.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:51.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:51.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:51.802 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 04:49:51.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:51.802 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:49:51.802 | INFO | PRE_HQ : 65.8%, 1.6, 55.8ms, 0 | 20260508 04:49:51.802 | INFO | BAZWRITER : 67.8%, 2.7, 92.5ms, 0 | 20260508 04:49:51.802 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:49:51.802 | INFO | Uploaded 36GiB to GPU in 2.57993s (13.9539GiB/s) | 20260508 04:49:51.803 | INFO | Downloaded 38GiB to GPU in 2.69004s (14.1262GiB/s) | 20260508 04:49:56.941 | INFO | Analyzing chunk frames = [5366272,5366784) | 20260508 04:49:56.941 | INFO | MemInfo: { | 20260508 04:49:56.941 | INFO | "gpu" : " 17 GB", | 20260508 04:49:56.942 | INFO | "huge" : "", | 20260508 04:49:56.942 | INFO | "huge_shared" : " 29 GB", | 20260508 04:49:56.942 | INFO | "malloc" : " 14 GB", | 20260508 04:49:56.942 | INFO | "pinned" : " 29 GB" | 20260508 04:49:56.942 | INFO | } | 20260508 04:49:56.942 | INFO | Perf report for the last 5.13965 seconds: | 20260508 04:49:56.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:49:56.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:49:56.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:49:56.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:49:56.942 | INFO | Uploaded 36GiB to GPU in 2.60008s (13.8457GiB/s) | 20260508 04:49:56.942 | INFO | Downloaded 34GiB to GPU in 2.43579s (13.9585GiB/s) | 20260508 04:50:02.085 | INFO | Analyzing chunk frames = [5366784,5367296) | 20260508 04:50:02.085 | INFO | MemInfo: { | 20260508 04:50:02.085 | INFO | "gpu" : " 17 GB", | 20260508 04:50:02.085 | INFO | "huge" : "", | 20260508 04:50:02.085 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:02.085 | INFO | "malloc" : " 14 GB", | 20260508 04:50:02.085 | INFO | "pinned" : " 29 GB" | 20260508 04:50:02.085 | INFO | } | 20260508 04:50:02.085 | INFO | Perf report for the last 5.14364 seconds: | 20260508 04:50:02.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:02.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.085 | INFO | ANALYSIS : 67.5%, 3.0, 104.9ms, 0 | 20260508 04:50:02.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.085 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:50:02.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:02.086 | INFO | Uploaded 36GiB to GPU in 2.56511s (14.0345GiB/s) | 20260508 04:50:02.086 | INFO | Downloaded 34GiB to GPU in 2.42015s (14.0487GiB/s) | 20260508 04:50:07.225 | INFO | Analyzing chunk frames = [5367296,5367808) | 20260508 04:50:07.225 | INFO | MemInfo: { | 20260508 04:50:07.225 | INFO | "gpu" : " 17 GB", | 20260508 04:50:07.225 | INFO | "huge" : "", | 20260508 04:50:07.225 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:07.225 | INFO | "malloc" : " 14 GB", | 20260508 04:50:07.225 | INFO | "pinned" : " 29 GB" | 20260508 04:50:07.225 | INFO | } | 20260508 04:50:07.225 | INFO | Perf report for the last 5.13995 seconds: | 20260508 04:50:07.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:07.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.225 | INFO | ANALYSIS : 67.2%, 2.9, 103.9ms, 0 | 20260508 04:50:07.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.225 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:50:07.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:07.226 | INFO | Uploaded 36GiB to GPU in 2.57946s (13.9564GiB/s) | 20260508 04:50:07.226 | INFO | Downloaded 34GiB to GPU in 2.41001s (14.1078GiB/s) | 20260508 04:50:12.364 | INFO | Analyzing chunk frames = [5367808,5368320) | 20260508 04:50:12.364 | INFO | MemInfo: { | 20260508 04:50:12.364 | INFO | "gpu" : " 17 GB", | 20260508 04:50:12.364 | INFO | "huge" : "", | 20260508 04:50:12.364 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:12.364 | INFO | "malloc" : " 14 GB", | 20260508 04:50:12.364 | INFO | "pinned" : " 29 GB" | 20260508 04:50:12.364 | INFO | } | 20260508 04:50:12.364 | INFO | Perf report for the last 5.13896 seconds: | 20260508 04:50:12.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:12.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.364 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 04:50:12.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.364 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:50:12.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:12.365 | INFO | Uploaded 36GiB to GPU in 2.41563s (14.903GiB/s) | 20260508 04:50:12.365 | INFO | Downloaded 34GiB to GPU in 2.5303s (13.4371GiB/s) | 20260508 04:50:17.501 | INFO | Analyzing chunk frames = [5368320,5368832) | 20260508 04:50:17.501 | INFO | MemInfo: { | 20260508 04:50:17.501 | INFO | "gpu" : " 17 GB", | 20260508 04:50:17.501 | INFO | "huge" : "", | 20260508 04:50:17.501 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:17.501 | INFO | "malloc" : " 14 GB", | 20260508 04:50:17.501 | INFO | "pinned" : " 29 GB" | 20260508 04:50:17.501 | INFO | } | 20260508 04:50:17.501 | INFO | Perf report for the last 5.13701 seconds: | 20260508 04:50:17.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:17.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.501 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:50:17.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.501 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:50:17.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:17.502 | INFO | Uploaded 36GiB to GPU in 2.47033s (14.5729GiB/s) | 20260508 04:50:17.502 | INFO | Downloaded 34GiB to GPU in 2.40022s (14.1654GiB/s) | 20260508 04:50:22.641 | INFO | Analyzing chunk frames = [5368832,5369344) | 20260508 04:50:22.641 | INFO | MemInfo: { | 20260508 04:50:22.641 | INFO | "gpu" : " 17 GB", | 20260508 04:50:22.641 | INFO | "huge" : "", | 20260508 04:50:22.641 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:22.641 | INFO | "malloc" : " 14 GB", | 20260508 04:50:22.641 | INFO | "pinned" : " 29 GB" | 20260508 04:50:22.641 | INFO | } | 20260508 04:50:22.641 | INFO | Perf report for the last 5.13999 seconds: | 20260508 04:50:22.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:22.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.641 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:50:22.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.641 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:50:22.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:22.642 | INFO | Uploaded 36GiB to GPU in 2.4816s (14.5068GiB/s) | 20260508 04:50:22.642 | INFO | Downloaded 34GiB to GPU in 2.41779s (14.0624GiB/s) | 20260508 04:50:27.779 | INFO | Analyzing chunk frames = [5369344,5369856) | 20260508 04:50:27.779 | INFO | MemInfo: { | 20260508 04:50:27.779 | INFO | "gpu" : " 17 GB", | 20260508 04:50:27.779 | INFO | "huge" : "", | 20260508 04:50:27.779 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:27.779 | INFO | "malloc" : " 14 GB", | 20260508 04:50:27.779 | INFO | "pinned" : " 29 GB" | 20260508 04:50:27.779 | INFO | } | 20260508 04:50:27.779 | INFO | Perf report for the last 5.13774 seconds: | 20260508 04:50:27.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:27.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 04:50:27.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:50:27.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:27.779 | INFO | Uploaded 36GiB to GPU in 2.4529s (14.6765GiB/s) | 20260508 04:50:27.779 | INFO | Downloaded 34GiB to GPU in 2.40982s (14.1089GiB/s) | 20260508 04:50:27.876 | INFO | Starting a round of Baz IO | 20260508 04:50:31.318 | INFO | Populating RTMetrics report | 20260508 04:50:31.318 | INFO | RTMetrics report populated | 20260508 04:50:31.318 | INFO | Opening tmp RTMetrics json file | 20260508 04:50:31.329 | INFO | Serializing RTMetrics to json | 20260508 04:50:31.330 | INFO | Serialization done, swapping tmp file now... | 20260508 04:50:31.331 | INFO | RTMetrics done | 20260508 04:50:31.386 | INFO | Finished a round of Baz IO: Wrote 1.588 GiB in 3.509 seconds | 20260508 04:50:31.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.98% / 0.00% / 59.01% | 20260508 04:50:31.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.508845% / 3.97364e-06% / 7.15256e-05% | 20260508 04:50:31.386 | INFO | Baz IO overall: | 20260508 04:50:31.386 | INFO | BytesWritten : 2399861805930 | 20260508 04:50:31.386 | INFO | NumEvents : 1158348113524 | 20260508 04:50:32.923 | INFO | Analyzing chunk frames = [5369856,5370368) | 20260508 04:50:32.923 | INFO | MemInfo: { | 20260508 04:50:32.923 | INFO | "gpu" : " 17 GB", | 20260508 04:50:32.923 | INFO | "huge" : "", | 20260508 04:50:32.923 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:32.923 | INFO | "malloc" : " 14 GB", | 20260508 04:50:32.923 | INFO | "pinned" : " 29 GB" | 20260508 04:50:32.923 | INFO | } | 20260508 04:50:32.923 | INFO | Perf report for the last 5.14438 seconds: | 20260508 04:50:32.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:32.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:32.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:32.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:32.924 | INFO | ANALYSIS : 67.9%, 3.0, 106.3ms, 0 | 20260508 04:50:32.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:32.924 | INFO | RT_METRICS : 39.5%, 1.0, 20.8ms, 0 | 20260508 04:50:32.924 | INFO | PRE_HQ : 65.6%, 1.6, 55.7ms, 0 | 20260508 04:50:32.924 | INFO | BAZWRITER : 67.6%, 2.6, 92.1ms, 0 | 20260508 04:50:32.924 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:50:32.924 | INFO | Uploaded 36GiB to GPU in 2.58595s (13.9214GiB/s) | 20260508 04:50:32.924 | INFO | Downloaded 38GiB to GPU in 2.69542s (14.098GiB/s) | 20260508 04:50:38.060 | INFO | Analyzing chunk frames = [5370368,5370880) | 20260508 04:50:38.060 | INFO | MemInfo: { | 20260508 04:50:38.060 | INFO | "gpu" : " 17 GB", | 20260508 04:50:38.060 | INFO | "huge" : "", | 20260508 04:50:38.060 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:38.060 | INFO | "malloc" : " 14 GB", | 20260508 04:50:38.060 | INFO | "pinned" : " 29 GB" | 20260508 04:50:38.060 | INFO | } | 20260508 04:50:38.060 | INFO | Perf report for the last 5.13643 seconds: | 20260508 04:50:38.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:38.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 04:50:38.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:50:38.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:38.060 | INFO | Uploaded 36GiB to GPU in 2.57499s (13.9806GiB/s) | 20260508 04:50:38.060 | INFO | Downloaded 34GiB to GPU in 2.45306s (13.8603GiB/s) | 20260508 04:50:43.202 | INFO | Analyzing chunk frames = [5370880,5371392) | 20260508 04:50:43.202 | INFO | MemInfo: { | 20260508 04:50:43.202 | INFO | "gpu" : " 17 GB", | 20260508 04:50:43.202 | INFO | "huge" : "", | 20260508 04:50:43.202 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:43.202 | INFO | "malloc" : " 14 GB", | 20260508 04:50:43.202 | INFO | "pinned" : " 29 GB" | 20260508 04:50:43.202 | INFO | } | 20260508 04:50:43.202 | INFO | Perf report for the last 5.14241 seconds: | 20260508 04:50:43.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:43.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.202 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 04:50:43.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.202 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 04:50:43.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:43.203 | INFO | Uploaded 36GiB to GPU in 2.52175s (14.2758GiB/s) | 20260508 04:50:43.203 | INFO | Downloaded 34GiB to GPU in 2.4156s (14.0752GiB/s) | 20260508 04:50:48.339 | INFO | Analyzing chunk frames = [5371392,5371904) | 20260508 04:50:48.339 | INFO | MemInfo: { | 20260508 04:50:48.339 | INFO | "gpu" : " 17 GB", | 20260508 04:50:48.339 | INFO | "huge" : "", | 20260508 04:50:48.339 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:48.339 | INFO | "malloc" : " 14 GB", | 20260508 04:50:48.339 | INFO | "pinned" : " 29 GB" | 20260508 04:50:48.339 | INFO | } | 20260508 04:50:48.339 | INFO | Perf report for the last 5.1367 seconds: | 20260508 04:50:48.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:48.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 04:50:48.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 04:50:48.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:48.339 | INFO | Uploaded 36GiB to GPU in 2.54178s (14.1633GiB/s) | 20260508 04:50:48.340 | INFO | Downloaded 34GiB to GPU in 2.41652s (14.0698GiB/s) | 20260508 04:50:53.477 | INFO | Analyzing chunk frames = [5371904,5372416) | 20260508 04:50:53.477 | INFO | MemInfo: { | 20260508 04:50:53.477 | INFO | "gpu" : " 17 GB", | 20260508 04:50:53.477 | INFO | "huge" : "", | 20260508 04:50:53.477 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:53.477 | INFO | "malloc" : " 14 GB", | 20260508 04:50:53.477 | INFO | "pinned" : " 29 GB" | 20260508 04:50:53.477 | INFO | } | 20260508 04:50:53.477 | INFO | Perf report for the last 5.13834 seconds: | 20260508 04:50:53.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:53.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.477 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 04:50:53.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.478 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:50:53.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:53.478 | INFO | Uploaded 36GiB to GPU in 2.3713s (15.1815GiB/s) | 20260508 04:50:53.478 | INFO | Downloaded 34GiB to GPU in 2.56192s (13.2713GiB/s) | 20260508 04:50:58.613 | INFO | Analyzing chunk frames = [5372416,5372928) | 20260508 04:50:58.613 | INFO | MemInfo: { | 20260508 04:50:58.613 | INFO | "gpu" : " 17 GB", | 20260508 04:50:58.613 | INFO | "huge" : "", | 20260508 04:50:58.613 | INFO | "huge_shared" : " 29 GB", | 20260508 04:50:58.613 | INFO | "malloc" : " 14 GB", | 20260508 04:50:58.614 | INFO | "pinned" : " 29 GB" | 20260508 04:50:58.614 | INFO | } | 20260508 04:50:58.614 | INFO | Perf report for the last 5.13637 seconds: | 20260508 04:50:58.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:50:58.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:50:58.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:50:58.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:50:58.614 | INFO | Uploaded 36GiB to GPU in 2.42751s (14.83GiB/s) | 20260508 04:50:58.614 | INFO | Downloaded 34GiB to GPU in 2.4249s (14.0212GiB/s) | 20260508 04:51:03.753 | INFO | Analyzing chunk frames = [5372928,5373440) | 20260508 04:51:03.753 | INFO | MemInfo: { | 20260508 04:51:03.753 | INFO | "gpu" : " 17 GB", | 20260508 04:51:03.753 | INFO | "huge" : "", | 20260508 04:51:03.753 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:03.753 | INFO | "malloc" : " 14 GB", | 20260508 04:51:03.753 | INFO | "pinned" : " 29 GB" | 20260508 04:51:03.753 | INFO | } | 20260508 04:51:03.753 | INFO | Perf report for the last 5.13895 seconds: | 20260508 04:51:03.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:03.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:51:03.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:51:03.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:03.753 | INFO | Uploaded 36GiB to GPU in 2.45902s (14.64GiB/s) | 20260508 04:51:03.753 | INFO | Downloaded 34GiB to GPU in 2.43406s (13.9685GiB/s) | 20260508 04:51:08.895 | INFO | Analyzing chunk frames = [5373440,5373952) | 20260508 04:51:08.895 | INFO | MemInfo: { | 20260508 04:51:08.895 | INFO | "gpu" : " 17 GB", | 20260508 04:51:08.895 | INFO | "huge" : "", | 20260508 04:51:08.895 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:08.895 | INFO | "malloc" : " 14 GB", | 20260508 04:51:08.895 | INFO | "pinned" : " 29 GB" | 20260508 04:51:08.895 | INFO | } | 20260508 04:51:08.895 | INFO | Perf report for the last 5.14228 seconds: | 20260508 04:51:08.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:08.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:51:08.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:51:08.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:08.895 | INFO | Uploaded 36GiB to GPU in 2.44384s (14.7309GiB/s) | 20260508 04:51:08.895 | INFO | Downloaded 34GiB to GPU in 2.42138s (14.0416GiB/s) | 20260508 04:51:08.896 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:51:08.896 | INFO | Total mem usage: 16.7 GiB | 20260508 04:51:08.896 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:51:08.896 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 04:51:08.896 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:51:08.896 | INFO | High Water Breakdown: | 20260508 04:51:08.896 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:51:08.896 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:51:08.896 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:51:08.896 | INFO | ------------------------------------------ | 20260508 04:51:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:51:08.896 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:51:08.896 | INFO | Total mem usage: 26.5 GiB | 20260508 04:51:08.896 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:51:08.896 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:51:08.896 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:51:08.896 | INFO | High Water Breakdown: | 20260508 04:51:08.896 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:51:08.896 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:51:08.896 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:51:08.896 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:51:08.896 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:51:08.896 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:51:08.896 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:51:08.896 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:51:08.896 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:51:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:51:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:51:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:51:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:51:08.896 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:51:08.896 | INFO | ------------------------------------------ | 20260508 04:51:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:51:08.896 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:51:08.896 | INFO | Total mem usage: 27.8 GiB | 20260508 04:51:08.896 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:51:08.896 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:51:08.896 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:51:08.896 | INFO | High Water Breakdown: | 20260508 04:51:08.896 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:51:08.896 | INFO | ------------------------------------------ | 20260508 04:51:08.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:51:08.896 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:51:08.896 | INFO | Total mem usage: 15.7 GiB | 20260508 04:51:08.896 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:51:08.896 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:51:08.896 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:51:08.896 | INFO | High Water Breakdown: | 20260508 04:51:08.896 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:51:08.896 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:51:08.896 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:51:08.896 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:51:08.896 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:51:08.897 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:51:08.897 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:51:08.897 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:51:08.897 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:51:08.897 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:51:08.897 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:51:08.897 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:51:08.897 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:51:08.897 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:51:08.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:51:08.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:51:08.897 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:51:08.897 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:51:08.897 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:51:08.897 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:51:08.897 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:51:08.897 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:51:08.897 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:51:08.897 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:51:08.897 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:51:08.897 | INFO | ------------------------------------------ | 20260508 04:51:08.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:51:08.897 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:51:08.897 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:51:08.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:51:08.897 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.57e+01, 1.42e+01 / 1.31e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.02e-01, 1.06e-02 / 4.66e-03 | 20260508 04:51:08.897 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.47e+00, 5.89e-01 / 2.20e-01 | 20260508 04:51:08.897 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.06e-01 / 1.23e-01 | 20260508 04:51:08.897 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.50e+01, 3.70e+00 / 1.53e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.51e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.55e+00, 4.16e-01 / 2.47e-01 | 20260508 04:51:08.897 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.21e+00, 2.61e-01 / 1.42e-01 | 20260508 04:51:08.897 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.56e-01, 5.16e-02 / 2.71e-02 | 20260508 04:51:08.897 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:51:08.897 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:51:08.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:51:08.897 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 24.19%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 2.36%, LockUpload -- 2.51e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 2.52%, LockDownload -- 2.68e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 15.67%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:51:08.897 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 04:51:08.992 | INFO | Starting a round of Baz IO | 20260508 04:51:12.432 | INFO | Populating RTMetrics report | 20260508 04:51:12.433 | INFO | RTMetrics report populated | 20260508 04:51:12.433 | INFO | Opening tmp RTMetrics json file | 20260508 04:51:12.445 | INFO | Serializing RTMetrics to json | 20260508 04:51:12.446 | INFO | Serialization done, swapping tmp file now... | 20260508 04:51:12.446 | INFO | RTMetrics done | 20260508 04:51:12.501 | INFO | Finished a round of Baz IO: Wrote 1.588 GiB in 3.508 seconds | 20260508 04:51:12.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.98% / 0.00% / 59.02% | 20260508 04:51:12.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.506814% / 3.97364e-06% / 7.15256e-05% | 20260508 04:51:12.501 | INFO | Baz IO overall: | 20260508 04:51:12.501 | INFO | BytesWritten : 2401566902497 | 20260508 04:51:12.501 | INFO | NumEvents : 1159161965242 | 20260508 04:51:14.039 | INFO | Analyzing chunk frames = [5373952,5374464) | 20260508 04:51:14.039 | INFO | MemInfo: { | 20260508 04:51:14.039 | INFO | "gpu" : " 17 GB", | 20260508 04:51:14.039 | INFO | "huge" : "", | 20260508 04:51:14.039 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:14.039 | INFO | "malloc" : " 14 GB", | 20260508 04:51:14.039 | INFO | "pinned" : " 29 GB" | 20260508 04:51:14.039 | INFO | } | 20260508 04:51:14.039 | INFO | Perf report for the last 5.1443 seconds: | 20260508 04:51:14.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:14.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:14.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:14.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:14.039 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 04:51:14.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:14.039 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:51:14.039 | INFO | PRE_HQ : 65.6%, 1.6, 56.0ms, 0 | 20260508 04:51:14.039 | INFO | BAZWRITER : 67.7%, 2.7, 93.0ms, 0 | 20260508 04:51:14.039 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 04:51:14.040 | INFO | Uploaded 36GiB to GPU in 2.53669s (14.1917GiB/s) | 20260508 04:51:14.040 | INFO | Downloaded 38GiB to GPU in 2.68628s (14.146GiB/s) | 20260508 04:51:19.174 | INFO | Analyzing chunk frames = [5374464,5374976) | 20260508 04:51:19.174 | INFO | MemInfo: { | 20260508 04:51:19.174 | INFO | "gpu" : " 17 GB", | 20260508 04:51:19.174 | INFO | "huge" : "", | 20260508 04:51:19.174 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:19.174 | INFO | "malloc" : " 14 GB", | 20260508 04:51:19.174 | INFO | "pinned" : " 29 GB" | 20260508 04:51:19.174 | INFO | } | 20260508 04:51:19.174 | INFO | Perf report for the last 5.13442 seconds: | 20260508 04:51:19.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:19.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:51:19.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:51:19.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:19.174 | INFO | Uploaded 36GiB to GPU in 2.58545s (13.9241GiB/s) | 20260508 04:51:19.174 | INFO | Downloaded 34GiB to GPU in 2.44929s (13.8816GiB/s) | 20260508 04:51:24.312 | INFO | Analyzing chunk frames = [5374976,5375488) | 20260508 04:51:24.312 | INFO | MemInfo: { | 20260508 04:51:24.312 | INFO | "gpu" : " 17 GB", | 20260508 04:51:24.312 | INFO | "huge" : "", | 20260508 04:51:24.312 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:24.312 | INFO | "malloc" : " 14 GB", | 20260508 04:51:24.312 | INFO | "pinned" : " 29 GB" | 20260508 04:51:24.312 | INFO | } | 20260508 04:51:24.312 | INFO | Perf report for the last 5.13805 seconds: | 20260508 04:51:24.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:24.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 04:51:24.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:51:24.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:24.312 | INFO | Uploaded 36GiB to GPU in 2.5235s (14.2659GiB/s) | 20260508 04:51:24.312 | INFO | Downloaded 34GiB to GPU in 2.41169s (14.098GiB/s) | 20260508 04:51:29.455 | INFO | Analyzing chunk frames = [5375488,5376000) | 20260508 04:51:29.455 | INFO | MemInfo: { | 20260508 04:51:29.455 | INFO | "gpu" : " 17 GB", | 20260508 04:51:29.455 | INFO | "huge" : "", | 20260508 04:51:29.455 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:29.455 | INFO | "malloc" : " 14 GB", | 20260508 04:51:29.455 | INFO | "pinned" : " 29 GB" | 20260508 04:51:29.455 | INFO | } | 20260508 04:51:29.456 | INFO | Perf report for the last 5.14365 seconds: | 20260508 04:51:29.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:29.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 04:51:29.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:51:29.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:29.456 | INFO | Uploaded 36GiB to GPU in 2.56423s (14.0393GiB/s) | 20260508 04:51:29.456 | INFO | Downloaded 34GiB to GPU in 2.41554s (14.0756GiB/s) | 20260508 04:51:34.593 | INFO | Analyzing chunk frames = [5376000,5376512) | 20260508 04:51:34.593 | INFO | MemInfo: { | 20260508 04:51:34.593 | INFO | "gpu" : " 17 GB", | 20260508 04:51:34.593 | INFO | "huge" : "", | 20260508 04:51:34.593 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:34.593 | INFO | "malloc" : " 14 GB", | 20260508 04:51:34.593 | INFO | "pinned" : " 29 GB" | 20260508 04:51:34.593 | INFO | } | 20260508 04:51:34.593 | INFO | Perf report for the last 5.1373 seconds: | 20260508 04:51:34.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:34.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 04:51:34.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 04:51:34.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:34.593 | INFO | Uploaded 36GiB to GPU in 2.45511s (14.6633GiB/s) | 20260508 04:51:34.593 | INFO | Downloaded 34GiB to GPU in 2.53863s (13.3931GiB/s) | 20260508 04:51:39.736 | INFO | Analyzing chunk frames = [5376512,5377024) | 20260508 04:51:39.736 | INFO | MemInfo: { | 20260508 04:51:39.736 | INFO | "gpu" : " 17 GB", | 20260508 04:51:39.736 | INFO | "huge" : "", | 20260508 04:51:39.736 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:39.736 | INFO | "malloc" : " 14 GB", | 20260508 04:51:39.736 | INFO | "pinned" : " 29 GB" | 20260508 04:51:39.736 | INFO | } | 20260508 04:51:39.736 | INFO | Perf report for the last 5.14335 seconds: | 20260508 04:51:39.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:39.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 04:51:39.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:51:39.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:39.736 | INFO | Uploaded 36GiB to GPU in 2.41057s (14.9342GiB/s) | 20260508 04:51:39.736 | INFO | Downloaded 34GiB to GPU in 2.43871s (13.9418GiB/s) | 20260508 04:51:44.876 | INFO | Analyzing chunk frames = [5377024,5377536) | 20260508 04:51:44.876 | INFO | MemInfo: { | 20260508 04:51:44.876 | INFO | "gpu" : " 17 GB", | 20260508 04:51:44.876 | INFO | "huge" : "", | 20260508 04:51:44.876 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:44.876 | INFO | "malloc" : " 14 GB", | 20260508 04:51:44.876 | INFO | "pinned" : " 29 GB" | 20260508 04:51:44.876 | INFO | } | 20260508 04:51:44.876 | INFO | Perf report for the last 5.13983 seconds: | 20260508 04:51:44.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:44.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.876 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 04:51:44.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.876 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:51:44.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:44.877 | INFO | Uploaded 36GiB to GPU in 2.4042s (14.9738GiB/s) | 20260508 04:51:44.877 | INFO | Downloaded 34GiB to GPU in 2.44801s (13.8889GiB/s) | 20260508 04:51:50.008 | INFO | Analyzing chunk frames = [5377536,5378048) | 20260508 04:51:50.008 | INFO | MemInfo: { | 20260508 04:51:50.008 | INFO | "gpu" : " 17 GB", | 20260508 04:51:50.008 | INFO | "huge" : "", | 20260508 04:51:50.008 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:50.008 | INFO | "malloc" : " 14 GB", | 20260508 04:51:50.008 | INFO | "pinned" : " 29 GB" | 20260508 04:51:50.009 | INFO | } | 20260508 04:51:50.009 | INFO | Perf report for the last 5.13297 seconds: | 20260508 04:51:50.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:50.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 04:51:50.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:51:50.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:50.009 | INFO | Uploaded 36GiB to GPU in 2.42314s (14.8568GiB/s) | 20260508 04:51:50.009 | INFO | Downloaded 34GiB to GPU in 2.444s (13.9116GiB/s) | 20260508 04:51:50.106 | INFO | Starting a round of Baz IO | 20260508 04:51:53.544 | INFO | Populating RTMetrics report | 20260508 04:51:53.544 | INFO | RTMetrics report populated | 20260508 04:51:53.544 | INFO | Opening tmp RTMetrics json file | 20260508 04:51:53.554 | INFO | Serializing RTMetrics to json | 20260508 04:51:53.556 | INFO | Serialization done, swapping tmp file now... | 20260508 04:51:53.556 | INFO | RTMetrics done | 20260508 04:51:53.614 | INFO | Finished a round of Baz IO: Wrote 1.587 GiB in 3.507 seconds | 20260508 04:51:53.614 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.98% / 0.00% / 59.02% | 20260508 04:51:53.614 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.505463% / 0% / 4.76837e-05% | 20260508 04:51:53.614 | INFO | Baz IO overall: | 20260508 04:51:53.614 | INFO | BytesWritten : 2403271150765 | 20260508 04:51:53.614 | INFO | NumEvents : 1159975446813 | 20260508 04:51:55.152 | INFO | Analyzing chunk frames = [5378048,5378560) | 20260508 04:51:55.152 | INFO | MemInfo: { | 20260508 04:51:55.152 | INFO | "gpu" : " 17 GB", | 20260508 04:51:55.152 | INFO | "huge" : "", | 20260508 04:51:55.152 | INFO | "huge_shared" : " 29 GB", | 20260508 04:51:55.152 | INFO | "malloc" : " 14 GB", | 20260508 04:51:55.152 | INFO | "pinned" : " 29 GB" | 20260508 04:51:55.152 | INFO | } | 20260508 04:51:55.152 | INFO | Perf report for the last 5.14321 seconds: | 20260508 04:51:55.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:51:55.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:55.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:55.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:55.152 | INFO | ANALYSIS : 67.8%, 3.0, 106.1ms, 0 | 20260508 04:51:55.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:51:55.152 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 04:51:55.152 | INFO | PRE_HQ : 66.0%, 1.7, 59.8ms, 0 | 20260508 04:51:55.152 | INFO | BAZWRITER : 67.8%, 2.8, 96.2ms, 0 | 20260508 04:51:55.152 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 04:51:55.153 | INFO | Uploaded 36GiB to GPU in 2.55142s (14.1098GiB/s) | 20260508 04:51:55.153 | INFO | Downloaded 38GiB to GPU in 2.69362s (14.1074GiB/s) | 20260508 04:52:00.289 | INFO | Analyzing chunk frames = [5378560,5379072) | 20260508 04:52:00.289 | INFO | MemInfo: { | 20260508 04:52:00.289 | INFO | "gpu" : " 17 GB", | 20260508 04:52:00.289 | INFO | "huge" : "", | 20260508 04:52:00.289 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:00.289 | INFO | "malloc" : " 14 GB", | 20260508 04:52:00.289 | INFO | "pinned" : " 29 GB" | 20260508 04:52:00.289 | INFO | } | 20260508 04:52:00.289 | INFO | Perf report for the last 5.13712 seconds: | 20260508 04:52:00.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:00.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.289 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:52:00.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.289 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:52:00.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:00.290 | INFO | Uploaded 36GiB to GPU in 2.57909s (13.9584GiB/s) | 20260508 04:52:00.290 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.975GiB/s) | 20260508 04:52:05.429 | INFO | Analyzing chunk frames = [5379072,5379584) | 20260508 04:52:05.429 | INFO | MemInfo: { | 20260508 04:52:05.429 | INFO | "gpu" : " 17 GB", | 20260508 04:52:05.429 | INFO | "huge" : "", | 20260508 04:52:05.429 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:05.429 | INFO | "malloc" : " 14 GB", | 20260508 04:52:05.429 | INFO | "pinned" : " 29 GB" | 20260508 04:52:05.429 | INFO | } | 20260508 04:52:05.429 | INFO | Perf report for the last 5.13987 seconds: | 20260508 04:52:05.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:05.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 04:52:05.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:52:05.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:05.429 | INFO | Uploaded 36GiB to GPU in 2.54144s (14.1652GiB/s) | 20260508 04:52:05.430 | INFO | Downloaded 34GiB to GPU in 2.41707s (14.0666GiB/s) | 20260508 04:52:10.565 | INFO | Analyzing chunk frames = [5379584,5380096) | 20260508 04:52:10.565 | INFO | MemInfo: { | 20260508 04:52:10.565 | INFO | "gpu" : " 17 GB", | 20260508 04:52:10.565 | INFO | "huge" : "", | 20260508 04:52:10.565 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:10.565 | INFO | "malloc" : " 14 GB", | 20260508 04:52:10.565 | INFO | "pinned" : " 29 GB" | 20260508 04:52:10.565 | INFO | } | 20260508 04:52:10.565 | INFO | Perf report for the last 5.13575 seconds: | 20260508 04:52:10.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:10.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 04:52:10.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:52:10.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:10.565 | INFO | Uploaded 36GiB to GPU in 2.57971s (13.9551GiB/s) | 20260508 04:52:10.565 | INFO | Downloaded 34GiB to GPU in 2.4166s (14.0694GiB/s) | 20260508 04:52:15.708 | INFO | Analyzing chunk frames = [5380096,5380608) | 20260508 04:52:15.708 | INFO | MemInfo: { | 20260508 04:52:15.708 | INFO | "gpu" : " 17 GB", | 20260508 04:52:15.708 | INFO | "huge" : "", | 20260508 04:52:15.708 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:15.708 | INFO | "malloc" : " 14 GB", | 20260508 04:52:15.708 | INFO | "pinned" : " 29 GB" | 20260508 04:52:15.708 | INFO | } | 20260508 04:52:15.708 | INFO | Perf report for the last 5.14342 seconds: | 20260508 04:52:15.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:15.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.708 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 04:52:15.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.708 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 04:52:15.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:15.709 | INFO | Uploaded 36GiB to GPU in 2.3613s (15.2459GiB/s) | 20260508 04:52:15.709 | INFO | Downloaded 34GiB to GPU in 2.55719s (13.2958GiB/s) | 20260508 04:52:20.846 | INFO | Analyzing chunk frames = [5380608,5381120) | 20260508 04:52:20.846 | INFO | MemInfo: { | 20260508 04:52:20.846 | INFO | "gpu" : " 17 GB", | 20260508 04:52:20.846 | INFO | "huge" : "", | 20260508 04:52:20.846 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:20.846 | INFO | "malloc" : " 14 GB", | 20260508 04:52:20.846 | INFO | "pinned" : " 29 GB" | 20260508 04:52:20.846 | INFO | } | 20260508 04:52:20.846 | INFO | Perf report for the last 5.13782 seconds: | 20260508 04:52:20.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:20.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.846 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:52:20.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.846 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:52:20.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:20.847 | INFO | Uploaded 36GiB to GPU in 2.45471s (14.6657GiB/s) | 20260508 04:52:20.847 | INFO | Downloaded 34GiB to GPU in 2.41529s (14.077GiB/s) | 20260508 04:52:25.986 | INFO | Analyzing chunk frames = [5381120,5381632) | 20260508 04:52:25.986 | INFO | MemInfo: { | 20260508 04:52:25.986 | INFO | "gpu" : " 17 GB", | 20260508 04:52:25.986 | INFO | "huge" : "", | 20260508 04:52:25.986 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:25.987 | INFO | "malloc" : " 14 GB", | 20260508 04:52:25.987 | INFO | "pinned" : " 29 GB" | 20260508 04:52:25.987 | INFO | } | 20260508 04:52:25.987 | INFO | Perf report for the last 5.14049 seconds: | 20260508 04:52:25.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:25.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 04:52:25.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:52:25.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:25.987 | INFO | Uploaded 36GiB to GPU in 2.45342s (14.6734GiB/s) | 20260508 04:52:25.987 | INFO | Downloaded 34GiB to GPU in 2.41449s (14.0816GiB/s) | 20260508 04:52:31.125 | INFO | Analyzing chunk frames = [5381632,5382144) | 20260508 04:52:31.125 | INFO | MemInfo: { | 20260508 04:52:31.125 | INFO | "gpu" : " 17 GB", | 20260508 04:52:31.125 | INFO | "huge" : "", | 20260508 04:52:31.125 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:31.125 | INFO | "malloc" : " 14 GB", | 20260508 04:52:31.125 | INFO | "pinned" : " 29 GB" | 20260508 04:52:31.125 | INFO | } | 20260508 04:52:31.125 | INFO | Perf report for the last 5.13828 seconds: | 20260508 04:52:31.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:31.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 04:52:31.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 04:52:31.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:31.125 | INFO | Uploaded 36GiB to GPU in 2.46807s (14.5863GiB/s) | 20260508 04:52:31.125 | INFO | Downloaded 34GiB to GPU in 2.42066s (14.0457GiB/s) | 20260508 04:52:31.126 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:52:31.126 | INFO | Total mem usage: 16.7 GiB | 20260508 04:52:31.126 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:52:31.126 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 04:52:31.126 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:52:31.126 | INFO | High Water Breakdown: | 20260508 04:52:31.126 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:52:31.126 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:52:31.126 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:52:31.126 | INFO | ------------------------------------------ | 20260508 04:52:31.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:52:31.126 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:52:31.126 | INFO | Total mem usage: 26.5 GiB | 20260508 04:52:31.126 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:52:31.126 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:52:31.126 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:52:31.126 | INFO | High Water Breakdown: | 20260508 04:52:31.126 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:52:31.126 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:52:31.126 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:52:31.126 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:52:31.126 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:52:31.126 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:52:31.126 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:52:31.126 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:52:31.126 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:52:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:52:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:52:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:52:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:52:31.126 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:52:31.126 | INFO | ------------------------------------------ | 20260508 04:52:31.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:52:31.126 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:52:31.126 | INFO | Total mem usage: 27.8 GiB | 20260508 04:52:31.126 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:52:31.126 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:52:31.126 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:52:31.126 | INFO | High Water Breakdown: | 20260508 04:52:31.126 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:52:31.126 | INFO | ------------------------------------------ | 20260508 04:52:31.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:52:31.126 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:52:31.126 | INFO | Total mem usage: 15.7 GiB | 20260508 04:52:31.126 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:52:31.126 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:52:31.126 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:52:31.126 | INFO | High Water Breakdown: | 20260508 04:52:31.126 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:52:31.126 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:52:31.126 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:52:31.126 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:52:31.126 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:52:31.126 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:52:31.126 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:52:31.126 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:52:31.126 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:52:31.126 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:52:31.126 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:52:31.126 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:52:31.126 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:52:31.126 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:52:31.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:52:31.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:52:31.126 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:52:31.126 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:52:31.126 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:52:31.126 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:52:31.126 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:52:31.126 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:52:31.126 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:52:31.127 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:52:31.127 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:52:31.127 | INFO | ------------------------------------------ | 20260508 04:52:31.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:52:31.127 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:52:31.127 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:52:31.127 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:52:31.127 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.57e+01, 1.39e+01 / 1.44e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.06e-01, 1.11e-02 / 9.99e-03 | 20260508 04:52:31.127 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.46e+00, 5.91e-01 / 2.14e-01 | 20260508 04:52:31.127 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.42e+00, 2.10e-01 / 1.38e-01 | 20260508 04:52:31.127 | INFO | PerfReport: 15.88%, DME -- 4.16e+01 / 4.50e+01, 3.66e+00 / 1.91e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.83e+00 / 1.53e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.56e+00, 4.17e-01 / 2.58e-01 | 20260508 04:52:31.127 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.27e+00, 2.66e-01 / 1.51e-01 | 20260508 04:52:31.127 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.60e-01, 5.07e-02 / 2.72e-02 | 20260508 04:52:31.127 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:52:31.127 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:52:31.127 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:52:31.127 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 24.42%, Upload -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 24.24%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 30.79%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 2.23%, LockUpload -- 2.37e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 15.68%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:52:31.127 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 04:52:31.222 | INFO | Starting a round of Baz IO | 20260508 04:52:34.650 | INFO | Populating RTMetrics report | 20260508 04:52:34.650 | INFO | RTMetrics report populated | 20260508 04:52:34.650 | INFO | Opening tmp RTMetrics json file | 20260508 04:52:34.662 | INFO | Serializing RTMetrics to json | 20260508 04:52:34.663 | INFO | Serialization done, swapping tmp file now... | 20260508 04:52:34.664 | INFO | RTMetrics done | 20260508 04:52:34.727 | INFO | Finished a round of Baz IO: Wrote 1.587 GiB in 3.504 seconds | 20260508 04:52:34.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.98% / 0.00% / 59.02% | 20260508 04:52:34.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.505801% / 0% / 6.75519e-05% | 20260508 04:52:34.727 | INFO | Baz IO overall: | 20260508 04:52:34.727 | INFO | BytesWritten : 2404975419313 | 20260508 04:52:34.727 | INFO | NumEvents : 1160789021072 | 20260508 04:52:36.264 | INFO | Analyzing chunk frames = [5382144,5382656) | 20260508 04:52:36.264 | INFO | MemInfo: { | 20260508 04:52:36.264 | INFO | "gpu" : " 17 GB", | 20260508 04:52:36.264 | INFO | "huge" : "", | 20260508 04:52:36.264 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:36.264 | INFO | "malloc" : " 14 GB", | 20260508 04:52:36.264 | INFO | "pinned" : " 29 GB" | 20260508 04:52:36.264 | INFO | } | 20260508 04:52:36.264 | INFO | Perf report for the last 5.13891 seconds: | 20260508 04:52:36.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:36.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:36.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:36.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:36.264 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 04:52:36.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:36.264 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 04:52:36.264 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 04:52:36.264 | INFO | BAZWRITER : 67.7%, 2.7, 92.9ms, 0 | 20260508 04:52:36.264 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:52:36.264 | INFO | Uploaded 36GiB to GPU in 2.56689s (14.0248GiB/s) | 20260508 04:52:36.264 | INFO | Downloaded 38GiB to GPU in 2.69722s (14.0886GiB/s) | 20260508 04:52:41.405 | INFO | Analyzing chunk frames = [5382656,5383168) | 20260508 04:52:41.405 | INFO | MemInfo: { | 20260508 04:52:41.405 | INFO | "gpu" : " 17 GB", | 20260508 04:52:41.405 | INFO | "huge" : "", | 20260508 04:52:41.405 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:41.405 | INFO | "malloc" : " 14 GB", | 20260508 04:52:41.405 | INFO | "pinned" : " 29 GB" | 20260508 04:52:41.405 | INFO | } | 20260508 04:52:41.405 | INFO | Perf report for the last 5.14144 seconds: | 20260508 04:52:41.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:41.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.406 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 04:52:41.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.406 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 04:52:41.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:41.406 | INFO | Uploaded 36GiB to GPU in 2.59311s (13.8829GiB/s) | 20260508 04:52:41.406 | INFO | Downloaded 34GiB to GPU in 2.43324s (13.9731GiB/s) | 20260508 04:52:46.547 | INFO | Analyzing chunk frames = [5383168,5383680) | 20260508 04:52:46.547 | INFO | MemInfo: { | 20260508 04:52:46.547 | INFO | "gpu" : " 17 GB", | 20260508 04:52:46.547 | INFO | "huge" : "", | 20260508 04:52:46.547 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:46.547 | INFO | "malloc" : " 14 GB", | 20260508 04:52:46.548 | INFO | "pinned" : " 29 GB" | 20260508 04:52:46.548 | INFO | } | 20260508 04:52:46.548 | INFO | Perf report for the last 5.14221 seconds: | 20260508 04:52:46.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:46.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 04:52:46.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:52:46.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:46.548 | INFO | Uploaded 36GiB to GPU in 2.54988s (14.1183GiB/s) | 20260508 04:52:46.548 | INFO | Downloaded 34GiB to GPU in 2.40585s (14.1322GiB/s) | 20260508 04:52:51.687 | INFO | Analyzing chunk frames = [5383680,5384192) | 20260508 04:52:51.687 | INFO | MemInfo: { | 20260508 04:52:51.687 | INFO | "gpu" : " 17 GB", | 20260508 04:52:51.687 | INFO | "huge" : "", | 20260508 04:52:51.687 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:51.687 | INFO | "malloc" : " 14 GB", | 20260508 04:52:51.687 | INFO | "pinned" : " 29 GB" | 20260508 04:52:51.687 | INFO | } | 20260508 04:52:51.687 | INFO | Perf report for the last 5.13932 seconds: | 20260508 04:52:51.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:51.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.688 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 04:52:51.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.688 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:52:51.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:51.688 | INFO | Uploaded 36GiB to GPU in 2.5493s (14.1215GiB/s) | 20260508 04:52:51.688 | INFO | Downloaded 34GiB to GPU in 2.37226s (14.3323GiB/s) | 20260508 04:52:56.826 | INFO | Analyzing chunk frames = [5384192,5384704) | 20260508 04:52:56.826 | INFO | MemInfo: { | 20260508 04:52:56.826 | INFO | "gpu" : " 17 GB", | 20260508 04:52:56.826 | INFO | "huge" : "", | 20260508 04:52:56.826 | INFO | "huge_shared" : " 29 GB", | 20260508 04:52:56.826 | INFO | "malloc" : " 14 GB", | 20260508 04:52:56.826 | INFO | "pinned" : " 29 GB" | 20260508 04:52:56.826 | INFO | } | 20260508 04:52:56.827 | INFO | Perf report for the last 5.13953 seconds: | 20260508 04:52:56.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:52:56.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 04:52:56.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 04:52:56.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:52:56.827 | INFO | Uploaded 36GiB to GPU in 2.412s (14.9254GiB/s) | 20260508 04:52:56.827 | INFO | Downloaded 34GiB to GPU in 2.48784s (13.6665GiB/s) | 20260508 04:53:01.962 | INFO | Analyzing chunk frames = [5384704,5385216) | 20260508 04:53:01.962 | INFO | MemInfo: { | 20260508 04:53:01.962 | INFO | "gpu" : " 17 GB", | 20260508 04:53:01.962 | INFO | "huge" : "", | 20260508 04:53:01.962 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:01.962 | INFO | "malloc" : " 14 GB", | 20260508 04:53:01.962 | INFO | "pinned" : " 29 GB" | 20260508 04:53:01.962 | INFO | } | 20260508 04:53:01.962 | INFO | Perf report for the last 5.1357 seconds: | 20260508 04:53:01.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:01.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.962 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:53:01.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.962 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:53:01.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:01.963 | INFO | Uploaded 36GiB to GPU in 2.44182s (14.7431GiB/s) | 20260508 04:53:01.963 | INFO | Downloaded 34GiB to GPU in 2.42816s (14.0024GiB/s) | 20260508 04:53:07.102 | INFO | Analyzing chunk frames = [5385216,5385728) | 20260508 04:53:07.102 | INFO | MemInfo: { | 20260508 04:53:07.102 | INFO | "gpu" : " 17 GB", | 20260508 04:53:07.102 | INFO | "huge" : "", | 20260508 04:53:07.102 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:07.102 | INFO | "malloc" : " 14 GB", | 20260508 04:53:07.102 | INFO | "pinned" : " 29 GB" | 20260508 04:53:07.102 | INFO | } | 20260508 04:53:07.102 | INFO | Perf report for the last 5.13989 seconds: | 20260508 04:53:07.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:07.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.102 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:53:07.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.103 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260508 04:53:07.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:07.103 | INFO | Uploaded 36GiB to GPU in 2.45073s (14.6895GiB/s) | 20260508 04:53:07.103 | INFO | Downloaded 34GiB to GPU in 2.42728s (14.0075GiB/s) | 20260508 04:53:12.242 | INFO | Analyzing chunk frames = [5385728,5386240) | 20260508 04:53:12.242 | INFO | MemInfo: { | 20260508 04:53:12.242 | INFO | "gpu" : " 17 GB", | 20260508 04:53:12.242 | INFO | "huge" : "", | 20260508 04:53:12.242 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:12.242 | INFO | "malloc" : " 14 GB", | 20260508 04:53:12.242 | INFO | "pinned" : " 29 GB" | 20260508 04:53:12.242 | INFO | } | 20260508 04:53:12.242 | INFO | Perf report for the last 5.1402 seconds: | 20260508 04:53:12.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:12.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.242 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 04:53:12.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.243 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 04:53:12.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:12.243 | INFO | Uploaded 36GiB to GPU in 2.4459s (14.7185GiB/s) | 20260508 04:53:12.243 | INFO | Downloaded 34GiB to GPU in 2.43711s (13.951GiB/s) | 20260508 04:53:12.355 | INFO | Starting a round of Baz IO | 20260508 04:53:15.778 | INFO | Populating RTMetrics report | 20260508 04:53:15.779 | INFO | RTMetrics report populated | 20260508 04:53:15.779 | INFO | Opening tmp RTMetrics json file | 20260508 04:53:15.789 | INFO | Serializing RTMetrics to json | 20260508 04:53:15.790 | INFO | Serialization done, swapping tmp file now... | 20260508 04:53:15.791 | INFO | RTMetrics done | 20260508 04:53:15.849 | INFO | Finished a round of Baz IO: Wrote 1.587 GiB in 3.493 seconds | 20260508 04:53:15.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.97% / 0.00% / 59.02% | 20260508 04:53:15.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.503425% / 0% / 9.13938e-05% | 20260508 04:53:15.849 | INFO | Baz IO overall: | 20260508 04:53:15.849 | INFO | BytesWritten : 2406679406833 | 20260508 04:53:15.849 | INFO | NumEvents : 1161602763422 | 20260508 04:53:17.376 | INFO | Analyzing chunk frames = [5386240,5386752) | 20260508 04:53:17.376 | INFO | MemInfo: { | 20260508 04:53:17.376 | INFO | "gpu" : " 17 GB", | 20260508 04:53:17.376 | INFO | "huge" : "", | 20260508 04:53:17.376 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:17.376 | INFO | "malloc" : " 14 GB", | 20260508 04:53:17.376 | INFO | "pinned" : " 29 GB" | 20260508 04:53:17.376 | INFO | } | 20260508 04:53:17.376 | INFO | Perf report for the last 5.13385 seconds: | 20260508 04:53:17.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:17.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:17.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:17.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:17.376 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 04:53:17.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:17.376 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 04:53:17.376 | INFO | PRE_HQ : 66.3%, 1.8, 61.8ms, 0 | 20260508 04:53:17.376 | INFO | BAZWRITER : 67.6%, 2.8, 97.4ms, 0 | 20260508 04:53:17.376 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:53:17.377 | INFO | Uploaded 36GiB to GPU in 2.53909s (14.1783GiB/s) | 20260508 04:53:17.377 | INFO | Downloaded 38GiB to GPU in 2.70573s (14.0443GiB/s) | 20260508 04:53:22.515 | INFO | Analyzing chunk frames = [5386752,5387264) | 20260508 04:53:22.515 | INFO | MemInfo: { | 20260508 04:53:22.515 | INFO | "gpu" : " 17 GB", | 20260508 04:53:22.515 | INFO | "huge" : "", | 20260508 04:53:22.515 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:22.515 | INFO | "malloc" : " 14 GB", | 20260508 04:53:22.515 | INFO | "pinned" : " 29 GB" | 20260508 04:53:22.515 | INFO | } | 20260508 04:53:22.515 | INFO | Perf report for the last 5.13859 seconds: | 20260508 04:53:22.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:22.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 04:53:22.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 04:53:22.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:22.515 | INFO | Uploaded 36GiB to GPU in 2.58856s (13.9073GiB/s) | 20260508 04:53:22.515 | INFO | Downloaded 34GiB to GPU in 2.44024s (13.9331GiB/s) | 20260508 04:53:27.658 | INFO | Analyzing chunk frames = [5387264,5387776) | 20260508 04:53:27.658 | INFO | MemInfo: { | 20260508 04:53:27.658 | INFO | "gpu" : " 17 GB", | 20260508 04:53:27.658 | INFO | "huge" : "", | 20260508 04:53:27.658 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:27.658 | INFO | "malloc" : " 14 GB", | 20260508 04:53:27.658 | INFO | "pinned" : " 29 GB" | 20260508 04:53:27.658 | INFO | } | 20260508 04:53:27.659 | INFO | Perf report for the last 5.14362 seconds: | 20260508 04:53:27.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:27.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 04:53:27.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:53:27.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:27.659 | INFO | Uploaded 36GiB to GPU in 2.55018s (14.1167GiB/s) | 20260508 04:53:27.659 | INFO | Downloaded 34GiB to GPU in 2.4303s (13.99GiB/s) | 20260508 04:53:32.801 | INFO | Analyzing chunk frames = [5387776,5388288) | 20260508 04:53:32.801 | INFO | MemInfo: { | 20260508 04:53:32.801 | INFO | "gpu" : " 17 GB", | 20260508 04:53:32.801 | INFO | "huge" : "", | 20260508 04:53:32.801 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:32.802 | INFO | "malloc" : " 14 GB", | 20260508 04:53:32.802 | INFO | "pinned" : " 29 GB" | 20260508 04:53:32.802 | INFO | } | 20260508 04:53:32.802 | INFO | Perf report for the last 5.14352 seconds: | 20260508 04:53:32.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:32.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 04:53:32.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:53:32.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:32.802 | INFO | Uploaded 36GiB to GPU in 2.5877s (13.912GiB/s) | 20260508 04:53:32.802 | INFO | Downloaded 34GiB to GPU in 2.41985s (14.0504GiB/s) | 20260508 04:53:37.937 | INFO | Analyzing chunk frames = [5388288,5388800) | 20260508 04:53:37.937 | INFO | MemInfo: { | 20260508 04:53:37.937 | INFO | "gpu" : " 17 GB", | 20260508 04:53:37.937 | INFO | "huge" : "", | 20260508 04:53:37.937 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:37.937 | INFO | "malloc" : " 14 GB", | 20260508 04:53:37.937 | INFO | "pinned" : " 29 GB" | 20260508 04:53:37.938 | INFO | } | 20260508 04:53:37.938 | INFO | Perf report for the last 5.13557 seconds: | 20260508 04:53:37.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:37.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:53:37.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 04:53:37.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:37.938 | INFO | Uploaded 36GiB to GPU in 2.36886s (15.1972GiB/s) | 20260508 04:53:37.938 | INFO | Downloaded 34GiB to GPU in 2.5498s (13.3344GiB/s) | 20260508 04:53:43.073 | INFO | Analyzing chunk frames = [5388800,5389312) | 20260508 04:53:43.073 | INFO | MemInfo: { | 20260508 04:53:43.073 | INFO | "gpu" : " 17 GB", | 20260508 04:53:43.073 | INFO | "huge" : "", | 20260508 04:53:43.073 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:43.073 | INFO | "malloc" : " 14 GB", | 20260508 04:53:43.073 | INFO | "pinned" : " 29 GB" | 20260508 04:53:43.073 | INFO | } | 20260508 04:53:43.073 | INFO | Perf report for the last 5.13576 seconds: | 20260508 04:53:43.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:43.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.073 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:53:43.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.074 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:53:43.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:43.074 | INFO | Uploaded 36GiB to GPU in 2.45089s (14.6885GiB/s) | 20260508 04:53:43.074 | INFO | Downloaded 34GiB to GPU in 2.41105s (14.1017GiB/s) | 20260508 04:53:48.213 | INFO | Analyzing chunk frames = [5389312,5389824) | 20260508 04:53:48.213 | INFO | MemInfo: { | 20260508 04:53:48.213 | INFO | "gpu" : " 17 GB", | 20260508 04:53:48.214 | INFO | "huge" : "", | 20260508 04:53:48.214 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:48.214 | INFO | "malloc" : " 14 GB", | 20260508 04:53:48.214 | INFO | "pinned" : " 29 GB" | 20260508 04:53:48.214 | INFO | } | 20260508 04:53:48.214 | INFO | Perf report for the last 5.14044 seconds: | 20260508 04:53:48.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:48.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:53:48.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 04:53:48.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:48.214 | INFO | Uploaded 36GiB to GPU in 2.47988s (14.5168GiB/s) | 20260508 04:53:48.215 | INFO | Downloaded 34GiB to GPU in 2.41869s (14.0572GiB/s) | 20260508 04:53:53.357 | INFO | Analyzing chunk frames = [5389824,5390336) | 20260508 04:53:53.357 | INFO | MemInfo: { | 20260508 04:53:53.357 | INFO | "gpu" : " 17 GB", | 20260508 04:53:53.357 | INFO | "huge" : "", | 20260508 04:53:53.357 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:53.358 | INFO | "malloc" : " 14 GB", | 20260508 04:53:53.358 | INFO | "pinned" : " 29 GB" | 20260508 04:53:53.358 | INFO | } | 20260508 04:53:53.358 | INFO | Perf report for the last 5.14409 seconds: | 20260508 04:53:53.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:53.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.358 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 04:53:53.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.358 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 04:53:53.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:53.359 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:53:53.359 | INFO | Total mem usage: 16.7 GiB | 20260508 04:53:53.359 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:53:53.359 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:53:53.359 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:53:53.359 | INFO | High Water Breakdown: | 20260508 04:53:53.359 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:53:53.359 | INFO | Uploaded 36GiB to GPU in 2.46857s (14.5834GiB/s) | 20260508 04:53:53.359 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:53:53.359 | INFO | Downloaded 34GiB to GPU in 2.41714s (14.0662GiB/s) | 20260508 04:53:53.359 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:53:53.359 | INFO | ------------------------------------------ | 20260508 04:53:53.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:53:53.359 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:53:53.359 | INFO | Total mem usage: 26.5 GiB | 20260508 04:53:53.359 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:53:53.359 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:53:53.359 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:53:53.359 | INFO | High Water Breakdown: | 20260508 04:53:53.359 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:53:53.359 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:53:53.359 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:53:53.359 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:53:53.359 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:53:53.359 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:53:53.359 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:53:53.359 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:53:53.359 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:53:53.359 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:53:53.359 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:53:53.359 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:53:53.359 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:53:53.359 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:53:53.359 | INFO | ------------------------------------------ | 20260508 04:53:53.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:53:53.359 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:53:53.359 | INFO | Total mem usage: 27.8 GiB | 20260508 04:53:53.359 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:53:53.359 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:53:53.359 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:53:53.359 | INFO | High Water Breakdown: | 20260508 04:53:53.359 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:53:53.359 | INFO | ------------------------------------------ | 20260508 04:53:53.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:53:53.359 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:53:53.359 | INFO | Total mem usage: 15.7 GiB | 20260508 04:53:53.359 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:53:53.359 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:53:53.359 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:53:53.359 | INFO | High Water Breakdown: | 20260508 04:53:53.359 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:53:53.359 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:53:53.359 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:53:53.359 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:53:53.359 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:53:53.359 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:53:53.359 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:53:53.359 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:53:53.360 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:53:53.360 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:53:53.360 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:53:53.360 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:53:53.360 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:53:53.360 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:53:53.360 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:53:53.360 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:53:53.360 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:53:53.360 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:53:53.360 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:53:53.360 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:53:53.360 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:53:53.360 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:53:53.360 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:53:53.360 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:53:53.360 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:53:53.360 | INFO | ------------------------------------------ | 20260508 04:53:53.360 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:53:53.360 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:53:53.360 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:53:53.360 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:53:53.360 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 1.14e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.33e-02 / 6.11e-03 | 20260508 04:53:53.360 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.36e+00, 5.88e-01 / 2.57e-01 | 20260508 04:53:53.360 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.07e-01 / 1.51e-01 | 20260508 04:53:53.360 | INFO | PerfReport: 15.88%, DME -- 4.16e+01 / 4.48e+01, 3.57e+00 / 1.50e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.79e+00 / 1.40e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.55e+00, 4.12e-01 / 2.51e-01 | 20260508 04:53:53.360 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.22e+00, 2.61e-01 / 1.38e-01 | 20260508 04:53:53.360 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.23e-01 / 3.54e-01, 5.11e-02 / 2.51e-02 | 20260508 04:53:53.360 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:53:53.360 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:53:53.360 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:53:53.360 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 24.51%, Upload -- 2.61e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 24.17%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 2.37%, LockUpload -- 2.53e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 2.52%, LockDownload -- 2.68e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:53:53.360 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260508 04:53:53.472 | INFO | Starting a round of Baz IO | 20260508 04:53:56.884 | INFO | Populating RTMetrics report | 20260508 04:53:56.884 | INFO | RTMetrics report populated | 20260508 04:53:56.884 | INFO | Opening tmp RTMetrics json file | 20260508 04:53:56.896 | INFO | Serializing RTMetrics to json | 20260508 04:53:56.897 | INFO | Serialization done, swapping tmp file now... | 20260508 04:53:56.898 | INFO | RTMetrics done | 20260508 04:53:56.958 | INFO | Finished a round of Baz IO: Wrote 1.588 GiB in 3.486 seconds | 20260508 04:53:56.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.97% / 0.00% / 59.03% | 20260508 04:53:56.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.504827% / 0% / 5.16574e-05% | 20260508 04:53:56.958 | INFO | Baz IO overall: | 20260508 04:53:56.958 | INFO | BytesWritten : 2408384707602 | 20260508 04:53:56.958 | INFO | NumEvents : 1162417664061 | 20260508 04:53:58.501 | INFO | Analyzing chunk frames = [5390336,5390848) | 20260508 04:53:58.501 | INFO | MemInfo: { | 20260508 04:53:58.501 | INFO | "gpu" : " 17 GB", | 20260508 04:53:58.501 | INFO | "huge" : "", | 20260508 04:53:58.501 | INFO | "huge_shared" : " 29 GB", | 20260508 04:53:58.501 | INFO | "malloc" : " 14 GB", | 20260508 04:53:58.501 | INFO | "pinned" : " 29 GB" | 20260508 04:53:58.501 | INFO | } | 20260508 04:53:58.501 | INFO | Perf report for the last 5.14332 seconds: | 20260508 04:53:58.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:53:58.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:58.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:58.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:58.501 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 04:53:58.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:53:58.501 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 04:53:58.501 | INFO | PRE_HQ : 66.4%, 1.8, 61.6ms, 0 | 20260508 04:53:58.501 | INFO | BAZWRITER : 67.2%, 2.8, 97.6ms, 0 | 20260508 04:53:58.501 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:53:58.502 | INFO | Uploaded 36GiB to GPU in 2.59808s (13.8564GiB/s) | 20260508 04:53:58.502 | INFO | Downloaded 38GiB to GPU in 2.71922s (13.9746GiB/s) | 20260508 04:54:03.636 | INFO | Analyzing chunk frames = [5390848,5391360) | 20260508 04:54:03.636 | INFO | MemInfo: { | 20260508 04:54:03.636 | INFO | "gpu" : " 17 GB", | 20260508 04:54:03.636 | INFO | "huge" : "", | 20260508 04:54:03.636 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:03.636 | INFO | "malloc" : " 14 GB", | 20260508 04:54:03.636 | INFO | "pinned" : " 29 GB" | 20260508 04:54:03.636 | INFO | } | 20260508 04:54:03.637 | INFO | Perf report for the last 5.13523 seconds: | 20260508 04:54:03.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:03.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 04:54:03.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:54:03.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:03.637 | INFO | Uploaded 36GiB to GPU in 2.47797s (14.528GiB/s) | 20260508 04:54:03.637 | INFO | Downloaded 34GiB to GPU in 2.44042s (13.932GiB/s) | 20260508 04:54:08.772 | INFO | Analyzing chunk frames = [5391360,5391872) | 20260508 04:54:08.772 | INFO | MemInfo: { | 20260508 04:54:08.772 | INFO | "gpu" : " 17 GB", | 20260508 04:54:08.772 | INFO | "huge" : "", | 20260508 04:54:08.772 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:08.772 | INFO | "malloc" : " 14 GB", | 20260508 04:54:08.772 | INFO | "pinned" : " 29 GB" | 20260508 04:54:08.772 | INFO | } | 20260508 04:54:08.772 | INFO | Perf report for the last 5.13581 seconds: | 20260508 04:54:08.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:08.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.772 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 04:54:08.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.772 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 04:54:08.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:08.773 | INFO | Uploaded 36GiB to GPU in 2.53052s (14.2263GiB/s) | 20260508 04:54:08.773 | INFO | Downloaded 34GiB to GPU in 2.40683s (14.1264GiB/s) | 20260508 04:54:13.916 | INFO | Analyzing chunk frames = [5391872,5392384) | 20260508 04:54:13.916 | INFO | MemInfo: { | 20260508 04:54:13.916 | INFO | "gpu" : " 17 GB", | 20260508 04:54:13.916 | INFO | "huge" : "", | 20260508 04:54:13.916 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:13.916 | INFO | "malloc" : " 14 GB", | 20260508 04:54:13.916 | INFO | "pinned" : " 29 GB" | 20260508 04:54:13.916 | INFO | } | 20260508 04:54:13.916 | INFO | Perf report for the last 5.14369 seconds: | 20260508 04:54:13.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:13.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 04:54:13.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:54:13.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:13.916 | INFO | Uploaded 36GiB to GPU in 2.55881s (14.069GiB/s) | 20260508 04:54:13.916 | INFO | Downloaded 34GiB to GPU in 2.41278s (14.0916GiB/s) | 20260508 04:54:19.054 | INFO | Analyzing chunk frames = [5392384,5392896) | 20260508 04:54:19.054 | INFO | MemInfo: { | 20260508 04:54:19.054 | INFO | "gpu" : " 17 GB", | 20260508 04:54:19.054 | INFO | "huge" : "", | 20260508 04:54:19.054 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:19.054 | INFO | "malloc" : " 14 GB", | 20260508 04:54:19.054 | INFO | "pinned" : " 29 GB" | 20260508 04:54:19.054 | INFO | } | 20260508 04:54:19.054 | INFO | Perf report for the last 5.13794 seconds: | 20260508 04:54:19.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:19.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.054 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:54:19.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.054 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 04:54:19.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:19.055 | INFO | Uploaded 36GiB to GPU in 2.42595s (14.8395GiB/s) | 20260508 04:54:19.055 | INFO | Downloaded 34GiB to GPU in 2.55013s (13.3326GiB/s) | 20260508 04:54:24.196 | INFO | Analyzing chunk frames = [5392896,5393408) | 20260508 04:54:24.196 | INFO | MemInfo: { | 20260508 04:54:24.196 | INFO | "gpu" : " 17 GB", | 20260508 04:54:24.196 | INFO | "huge" : "", | 20260508 04:54:24.196 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:24.196 | INFO | "malloc" : " 14 GB", | 20260508 04:54:24.196 | INFO | "pinned" : " 29 GB" | 20260508 04:54:24.196 | INFO | } | 20260508 04:54:24.196 | INFO | Perf report for the last 5.14247 seconds: | 20260508 04:54:24.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:24.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:54:24.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:54:24.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:24.197 | INFO | Uploaded 36GiB to GPU in 2.40982s (14.9389GiB/s) | 20260508 04:54:24.197 | INFO | Downloaded 34GiB to GPU in 2.42526s (14.0191GiB/s) | 20260508 04:54:29.328 | INFO | Analyzing chunk frames = [5393408,5393920) | 20260508 04:54:29.328 | INFO | MemInfo: { | 20260508 04:54:29.328 | INFO | "gpu" : " 17 GB", | 20260508 04:54:29.328 | INFO | "huge" : "", | 20260508 04:54:29.328 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:29.328 | INFO | "malloc" : " 14 GB", | 20260508 04:54:29.329 | INFO | "pinned" : " 29 GB" | 20260508 04:54:29.329 | INFO | } | 20260508 04:54:29.329 | INFO | Perf report for the last 5.13214 seconds: | 20260508 04:54:29.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:29.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 04:54:29.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 04:54:29.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:29.329 | INFO | Uploaded 36GiB to GPU in 2.44304s (14.7358GiB/s) | 20260508 04:54:29.329 | INFO | Downloaded 34GiB to GPU in 2.4361s (13.9567GiB/s) | 20260508 04:54:34.472 | INFO | Analyzing chunk frames = [5393920,5394432) | 20260508 04:54:34.472 | INFO | MemInfo: { | 20260508 04:54:34.472 | INFO | "gpu" : " 17 GB", | 20260508 04:54:34.472 | INFO | "huge" : "", | 20260508 04:54:34.472 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:34.472 | INFO | "malloc" : " 14 GB", | 20260508 04:54:34.472 | INFO | "pinned" : " 29 GB" | 20260508 04:54:34.472 | INFO | } | 20260508 04:54:34.472 | INFO | Perf report for the last 5.14371 seconds: | 20260508 04:54:34.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:34.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.473 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 04:54:34.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.473 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 04:54:34.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:34.473 | INFO | Uploaded 36GiB to GPU in 2.44065s (14.7501GiB/s) | 20260508 04:54:34.473 | INFO | Downloaded 34GiB to GPU in 2.42977s (13.9931GiB/s) | 20260508 04:54:34.570 | INFO | Starting a round of Baz IO | 20260508 04:54:38.004 | INFO | Populating RTMetrics report | 20260508 04:54:38.004 | INFO | RTMetrics report populated | 20260508 04:54:38.004 | INFO | Opening tmp RTMetrics json file | 20260508 04:54:38.006 | INFO | Serializing RTMetrics to json | 20260508 04:54:38.021 | INFO | Serialization done, swapping tmp file now... | 20260508 04:54:38.021 | INFO | RTMetrics done | 20260508 04:54:38.072 | INFO | Finished a round of Baz IO: Wrote 1.585 GiB in 3.501 seconds | 20260508 04:54:38.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.97% / 0.00% / 59.03% | 20260508 04:54:38.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.501625% / 3.97364e-06% / 7.54992e-05% | 20260508 04:54:38.072 | INFO | Baz IO overall: | 20260508 04:54:38.072 | INFO | BytesWritten : 2410086514025 | 20260508 04:54:38.072 | INFO | NumEvents : 1163230096924 | 20260508 04:54:39.610 | INFO | Analyzing chunk frames = [5394432,5394944) | 20260508 04:54:39.610 | INFO | MemInfo: { | 20260508 04:54:39.610 | INFO | "gpu" : " 17 GB", | 20260508 04:54:39.610 | INFO | "huge" : "", | 20260508 04:54:39.610 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:39.610 | INFO | "malloc" : " 14 GB", | 20260508 04:54:39.610 | INFO | "pinned" : " 29 GB" | 20260508 04:54:39.610 | INFO | } | 20260508 04:54:39.610 | INFO | Perf report for the last 5.13784 seconds: | 20260508 04:54:39.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:39.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:39.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:39.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:39.610 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 04:54:39.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:39.610 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:54:39.610 | INFO | PRE_HQ : 65.9%, 1.6, 55.8ms, 0 | 20260508 04:54:39.610 | INFO | BAZWRITER : 67.6%, 2.7, 92.9ms, 0 | 20260508 04:54:39.610 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:54:39.611 | INFO | Uploaded 36GiB to GPU in 2.54471s (14.147GiB/s) | 20260508 04:54:39.611 | INFO | Downloaded 38GiB to GPU in 2.67812s (14.189GiB/s) | 20260508 04:54:44.752 | INFO | Analyzing chunk frames = [5394944,5395456) | 20260508 04:54:44.752 | INFO | MemInfo: { | 20260508 04:54:44.752 | INFO | "gpu" : " 17 GB", | 20260508 04:54:44.752 | INFO | "huge" : "", | 20260508 04:54:44.752 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:44.752 | INFO | "malloc" : " 14 GB", | 20260508 04:54:44.752 | INFO | "pinned" : " 29 GB" | 20260508 04:54:44.752 | INFO | } | 20260508 04:54:44.752 | INFO | Perf report for the last 5.14214 seconds: | 20260508 04:54:44.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:44.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.752 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 04:54:44.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.753 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:54:44.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:44.753 | INFO | Uploaded 36GiB to GPU in 2.579s (13.9589GiB/s) | 20260508 04:54:44.753 | INFO | Downloaded 34GiB to GPU in 2.44378s (13.9128GiB/s) | 20260508 04:54:49.887 | INFO | Analyzing chunk frames = [5395456,5395968) | 20260508 04:54:49.887 | INFO | MemInfo: { | 20260508 04:54:49.887 | INFO | "gpu" : " 17 GB", | 20260508 04:54:49.887 | INFO | "huge" : "", | 20260508 04:54:49.887 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:49.887 | INFO | "malloc" : " 14 GB", | 20260508 04:54:49.887 | INFO | "pinned" : " 29 GB" | 20260508 04:54:49.887 | INFO | } | 20260508 04:54:49.887 | INFO | Perf report for the last 5.13489 seconds: | 20260508 04:54:49.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:49.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.887 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 04:54:49.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.887 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:54:49.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:49.888 | INFO | Uploaded 36GiB to GPU in 2.51243s (14.3288GiB/s) | 20260508 04:54:49.888 | INFO | Downloaded 34GiB to GPU in 2.41172s (14.0978GiB/s) | 20260508 04:54:55.026 | INFO | Analyzing chunk frames = [5395968,5396480) | 20260508 04:54:55.026 | INFO | MemInfo: { | 20260508 04:54:55.026 | INFO | "gpu" : " 17 GB", | 20260508 04:54:55.026 | INFO | "huge" : "", | 20260508 04:54:55.026 | INFO | "huge_shared" : " 29 GB", | 20260508 04:54:55.026 | INFO | "malloc" : " 14 GB", | 20260508 04:54:55.026 | INFO | "pinned" : " 29 GB" | 20260508 04:54:55.026 | INFO | } | 20260508 04:54:55.026 | INFO | Perf report for the last 5.13857 seconds: | 20260508 04:54:55.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:54:55.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 04:54:55.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:54:55.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:54:55.026 | INFO | Uploaded 36GiB to GPU in 2.54001s (14.1732GiB/s) | 20260508 04:54:55.026 | INFO | Downloaded 34GiB to GPU in 2.42212s (14.0373GiB/s) | 20260508 04:55:00.170 | INFO | Analyzing chunk frames = [5396480,5396992) | 20260508 04:55:00.170 | INFO | MemInfo: { | 20260508 04:55:00.170 | INFO | "gpu" : " 17 GB", | 20260508 04:55:00.170 | INFO | "huge" : "", | 20260508 04:55:00.170 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:00.170 | INFO | "malloc" : " 14 GB", | 20260508 04:55:00.170 | INFO | "pinned" : " 29 GB" | 20260508 04:55:00.170 | INFO | } | 20260508 04:55:00.170 | INFO | Perf report for the last 5.14444 seconds: | 20260508 04:55:00.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:00.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.170 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:55:00.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.170 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 04:55:00.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:00.171 | INFO | Uploaded 36GiB to GPU in 2.32347s (15.4941GiB/s) | 20260508 04:55:00.171 | INFO | Downloaded 34GiB to GPU in 2.5541s (13.3119GiB/s) | 20260508 04:55:05.307 | INFO | Analyzing chunk frames = [5396992,5397504) | 20260508 04:55:05.307 | INFO | MemInfo: { | 20260508 04:55:05.307 | INFO | "gpu" : " 17 GB", | 20260508 04:55:05.307 | INFO | "huge" : "", | 20260508 04:55:05.307 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:05.307 | INFO | "malloc" : " 14 GB", | 20260508 04:55:05.308 | INFO | "pinned" : " 29 GB" | 20260508 04:55:05.308 | INFO | } | 20260508 04:55:05.308 | INFO | Perf report for the last 5.13755 seconds: | 20260508 04:55:05.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:05.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:55:05.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260508 04:55:05.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:05.308 | INFO | Uploaded 36GiB to GPU in 2.43299s (14.7966GiB/s) | 20260508 04:55:05.308 | INFO | Downloaded 34GiB to GPU in 2.43857s (13.9426GiB/s) | 20260508 04:55:10.447 | INFO | Analyzing chunk frames = [5397504,5398016) | 20260508 04:55:10.447 | INFO | MemInfo: { | 20260508 04:55:10.447 | INFO | "gpu" : " 17 GB", | 20260508 04:55:10.447 | INFO | "huge" : "", | 20260508 04:55:10.447 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:10.447 | INFO | "malloc" : " 14 GB", | 20260508 04:55:10.447 | INFO | "pinned" : " 29 GB" | 20260508 04:55:10.447 | INFO | } | 20260508 04:55:10.447 | INFO | Perf report for the last 5.13905 seconds: | 20260508 04:55:10.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:10.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.447 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:55:10.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.447 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 04:55:10.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:10.448 | INFO | Uploaded 36GiB to GPU in 2.48257s (14.5011GiB/s) | 20260508 04:55:10.448 | INFO | Downloaded 34GiB to GPU in 2.45621s (13.8424GiB/s) | 20260508 04:55:15.580 | INFO | Analyzing chunk frames = [5398016,5398528) | 20260508 04:55:15.580 | INFO | MemInfo: { | 20260508 04:55:15.580 | INFO | "gpu" : " 17 GB", | 20260508 04:55:15.580 | INFO | "huge" : "", | 20260508 04:55:15.580 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:15.580 | INFO | "malloc" : " 14 GB", | 20260508 04:55:15.580 | INFO | "pinned" : " 29 GB" | 20260508 04:55:15.580 | INFO | } | 20260508 04:55:15.581 | INFO | Perf report for the last 5.13368 seconds: | 20260508 04:55:15.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:15.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:55:15.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | PRE_HQ : 34.2%, 1.0, 17.7ms, 0 | 20260508 04:55:15.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:15.581 | INFO | Uploaded 36GiB to GPU in 2.44774s (14.7074GiB/s) | 20260508 04:55:15.581 | INFO | Downloaded 34GiB to GPU in 2.4311s (13.9854GiB/s) | 20260508 04:55:15.581 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:55:15.581 | INFO | Total mem usage: 16.7 GiB | 20260508 04:55:15.581 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:55:15.581 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:55:15.581 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:55:15.581 | INFO | High Water Breakdown: | 20260508 04:55:15.581 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:55:15.581 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:55:15.581 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:55:15.581 | INFO | ------------------------------------------ | 20260508 04:55:15.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:55:15.581 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:55:15.581 | INFO | Total mem usage: 26.5 GiB | 20260508 04:55:15.581 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:55:15.582 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:55:15.582 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:55:15.582 | INFO | High Water Breakdown: | 20260508 04:55:15.582 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:55:15.582 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:55:15.582 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:55:15.582 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:55:15.582 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:55:15.582 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:55:15.582 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:55:15.582 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:55:15.582 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:55:15.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:55:15.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:55:15.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:55:15.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:55:15.582 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:55:15.582 | INFO | ------------------------------------------ | 20260508 04:55:15.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:55:15.582 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:55:15.582 | INFO | Total mem usage: 27.8 GiB | 20260508 04:55:15.582 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:55:15.582 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:55:15.582 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:55:15.582 | INFO | High Water Breakdown: | 20260508 04:55:15.582 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:55:15.582 | INFO | ------------------------------------------ | 20260508 04:55:15.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:55:15.582 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:55:15.582 | INFO | Total mem usage: 15.7 GiB | 20260508 04:55:15.582 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:55:15.582 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:55:15.582 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:55:15.582 | INFO | High Water Breakdown: | 20260508 04:55:15.582 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:55:15.582 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:55:15.582 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:55:15.582 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:55:15.582 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:55:15.582 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:55:15.582 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:55:15.582 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:55:15.582 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:55:15.582 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:55:15.582 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:55:15.582 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:55:15.582 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:55:15.582 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:55:15.582 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:55:15.582 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:55:15.582 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:55:15.582 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:55:15.582 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:55:15.582 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:55:15.582 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:55:15.582 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:55:15.582 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:55:15.582 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:55:15.582 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:55:15.582 | INFO | ------------------------------------------ | 20260508 04:55:15.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:55:15.582 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:55:15.582 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:55:15.582 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:55:15.582 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 1.11e+00 | 20260508 04:55:15.582 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.05e-01, 1.20e-02 / 6.21e-03 | 20260508 04:55:15.582 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.51e+00, 5.85e-01 / 2.50e-01 | 20260508 04:55:15.583 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.07e-01 / 1.02e-01 | 20260508 04:55:15.583 | INFO | PerfReport: 15.87%, DME -- 4.15e+01 / 4.50e+01, 3.73e+00 / 1.61e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.78e+00 / 1.44e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.65e+00, 4.19e-01 / 2.10e-01 | 20260508 04:55:15.583 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.25e+00, 2.60e-01 / 1.37e-01 | 20260508 04:55:15.583 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.65e-01, 5.16e-02 / 2.68e-02 | 20260508 04:55:15.583 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:55:15.583 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:55:15.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:55:15.583 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 24.31%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 2.20%, LockUpload -- 2.34e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 2.64%, LockDownload -- 2.80e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:55:15.583 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260508 04:55:15.694 | INFO | Starting a round of Baz IO | 20260508 04:55:19.113 | INFO | Populating RTMetrics report | 20260508 04:55:19.114 | INFO | RTMetrics report populated | 20260508 04:55:19.114 | INFO | Opening tmp RTMetrics json file | 20260508 04:55:19.125 | INFO | Serializing RTMetrics to json | 20260508 04:55:19.126 | INFO | Serialization done, swapping tmp file now... | 20260508 04:55:19.127 | INFO | RTMetrics done | 20260508 04:55:19.183 | INFO | Finished a round of Baz IO: Wrote 1.587 GiB in 3.489 seconds | 20260508 04:55:19.183 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.97% / 0.00% / 59.03% | 20260508 04:55:19.183 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.501049% / 0% / 6.35783e-05% | 20260508 04:55:19.183 | INFO | Baz IO overall: | 20260508 04:55:19.183 | INFO | BytesWritten : 2411790683851 | 20260508 04:55:19.183 | INFO | NumEvents : 1164044297528 | 20260508 04:55:20.729 | INFO | Analyzing chunk frames = [5398528,5399040) | 20260508 04:55:20.729 | INFO | MemInfo: { | 20260508 04:55:20.729 | INFO | "gpu" : " 17 GB", | 20260508 04:55:20.729 | INFO | "huge" : "", | 20260508 04:55:20.729 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:20.729 | INFO | "malloc" : " 14 GB", | 20260508 04:55:20.729 | INFO | "pinned" : " 29 GB" | 20260508 04:55:20.729 | INFO | } | 20260508 04:55:20.729 | INFO | Perf report for the last 5.14877 seconds: | 20260508 04:55:20.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:20.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:20.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:20.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:20.730 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 04:55:20.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:20.730 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 04:55:20.730 | INFO | PRE_HQ : 66.0%, 1.8, 62.0ms, 0 | 20260508 04:55:20.730 | INFO | BAZWRITER : 67.2%, 2.8, 97.7ms, 0 | 20260508 04:55:20.730 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:55:20.730 | INFO | Uploaded 36GiB to GPU in 2.56546s (14.0326GiB/s) | 20260508 04:55:20.730 | INFO | Downloaded 38GiB to GPU in 2.69355s (14.1078GiB/s) | 20260508 04:55:25.864 | INFO | Analyzing chunk frames = [5399040,5399552) | 20260508 04:55:25.864 | INFO | MemInfo: { | 20260508 04:55:25.864 | INFO | "gpu" : " 17 GB", | 20260508 04:55:25.864 | INFO | "huge" : "", | 20260508 04:55:25.864 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:25.864 | INFO | "malloc" : " 14 GB", | 20260508 04:55:25.864 | INFO | "pinned" : " 29 GB" | 20260508 04:55:25.864 | INFO | } | 20260508 04:55:25.864 | INFO | Perf report for the last 5.13475 seconds: | 20260508 04:55:25.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:25.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.864 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 04:55:25.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.864 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:55:25.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:25.865 | INFO | Uploaded 36GiB to GPU in 2.47539s (14.5432GiB/s) | 20260508 04:55:25.865 | INFO | Downloaded 34GiB to GPU in 2.4368s (13.9527GiB/s) | 20260508 04:55:31.005 | INFO | Analyzing chunk frames = [5399552,5400064) | 20260508 04:55:31.005 | INFO | MemInfo: { | 20260508 04:55:31.005 | INFO | "gpu" : " 17 GB", | 20260508 04:55:31.005 | INFO | "huge" : "", | 20260508 04:55:31.005 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:31.005 | INFO | "malloc" : " 14 GB", | 20260508 04:55:31.005 | INFO | "pinned" : " 29 GB" | 20260508 04:55:31.005 | INFO | } | 20260508 04:55:31.005 | INFO | Perf report for the last 5.1409 seconds: | 20260508 04:55:31.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:31.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.005 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 04:55:31.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.005 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:55:31.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:31.006 | INFO | Uploaded 36GiB to GPU in 2.51814s (14.2963GiB/s) | 20260508 04:55:31.006 | INFO | Downloaded 34GiB to GPU in 2.41837s (14.0591GiB/s) | 20260508 04:55:36.140 | INFO | Analyzing chunk frames = [5400064,5400576) | 20260508 04:55:36.140 | INFO | MemInfo: { | 20260508 04:55:36.140 | INFO | "gpu" : " 17 GB", | 20260508 04:55:36.140 | INFO | "huge" : "", | 20260508 04:55:36.140 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:36.140 | INFO | "malloc" : " 14 GB", | 20260508 04:55:36.140 | INFO | "pinned" : " 29 GB" | 20260508 04:55:36.140 | INFO | } | 20260508 04:55:36.140 | INFO | Perf report for the last 5.13513 seconds: | 20260508 04:55:36.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:36.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.140 | INFO | ANALYSIS : 67.1%, 3.0, 103.7ms, 0 | 20260508 04:55:36.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.140 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:55:36.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:36.141 | INFO | Uploaded 36GiB to GPU in 2.56997s (14.008GiB/s) | 20260508 04:55:36.141 | INFO | Downloaded 34GiB to GPU in 2.42706s (14.0087GiB/s) | 20260508 04:55:41.278 | INFO | Analyzing chunk frames = [5400576,5401088) | 20260508 04:55:41.278 | INFO | MemInfo: { | 20260508 04:55:41.278 | INFO | "gpu" : " 17 GB", | 20260508 04:55:41.278 | INFO | "huge" : "", | 20260508 04:55:41.278 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:41.278 | INFO | "malloc" : " 14 GB", | 20260508 04:55:41.278 | INFO | "pinned" : " 29 GB" | 20260508 04:55:41.278 | INFO | } | 20260508 04:55:41.278 | INFO | Perf report for the last 5.13737 seconds: | 20260508 04:55:41.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:41.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 04:55:41.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 04:55:41.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:41.278 | INFO | Uploaded 36GiB to GPU in 2.33482s (15.4188GiB/s) | 20260508 04:55:41.278 | INFO | Downloaded 34GiB to GPU in 2.56294s (13.266GiB/s) | 20260508 04:55:46.423 | INFO | Analyzing chunk frames = [5401088,5401600) | 20260508 04:55:46.423 | INFO | MemInfo: { | 20260508 04:55:46.423 | INFO | "gpu" : " 17 GB", | 20260508 04:55:46.423 | INFO | "huge" : "", | 20260508 04:55:46.423 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:46.423 | INFO | "malloc" : " 14 GB", | 20260508 04:55:46.423 | INFO | "pinned" : " 29 GB" | 20260508 04:55:46.423 | INFO | } | 20260508 04:55:46.423 | INFO | Perf report for the last 5.14521 seconds: | 20260508 04:55:46.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:46.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 04:55:46.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:55:46.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:46.423 | INFO | Uploaded 36GiB to GPU in 2.43094s (14.8091GiB/s) | 20260508 04:55:46.423 | INFO | Downloaded 34GiB to GPU in 2.43418s (13.9678GiB/s) | 20260508 04:55:51.568 | INFO | Analyzing chunk frames = [5401600,5402112) | 20260508 04:55:51.568 | INFO | MemInfo: { | 20260508 04:55:51.568 | INFO | "gpu" : " 17 GB", | 20260508 04:55:51.568 | INFO | "huge" : "", | 20260508 04:55:51.568 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:51.568 | INFO | "malloc" : " 14 GB", | 20260508 04:55:51.568 | INFO | "pinned" : " 29 GB" | 20260508 04:55:51.568 | INFO | } | 20260508 04:55:51.568 | INFO | Perf report for the last 5.14505 seconds: | 20260508 04:55:51.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:51.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.568 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 04:55:51.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.568 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 04:55:51.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:51.569 | INFO | Uploaded 36GiB to GPU in 2.45428s (14.6683GiB/s) | 20260508 04:55:51.569 | INFO | Downloaded 34GiB to GPU in 2.43697s (13.9518GiB/s) | 20260508 04:55:56.702 | INFO | Analyzing chunk frames = [5402112,5402624) | 20260508 04:55:56.702 | INFO | MemInfo: { | 20260508 04:55:56.702 | INFO | "gpu" : " 17 GB", | 20260508 04:55:56.702 | INFO | "huge" : "", | 20260508 04:55:56.702 | INFO | "huge_shared" : " 29 GB", | 20260508 04:55:56.702 | INFO | "malloc" : " 14 GB", | 20260508 04:55:56.703 | INFO | "pinned" : " 29 GB" | 20260508 04:55:56.703 | INFO | } | 20260508 04:55:56.703 | INFO | Perf report for the last 5.13461 seconds: | 20260508 04:55:56.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:55:56.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:55:56.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260508 04:55:56.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:55:56.703 | INFO | Uploaded 36GiB to GPU in 2.44137s (14.7458GiB/s) | 20260508 04:55:56.703 | INFO | Downloaded 34GiB to GPU in 2.44682s (13.8956GiB/s) | 20260508 04:55:56.800 | INFO | Starting a round of Baz IO | 20260508 04:56:00.227 | INFO | Populating RTMetrics report | 20260508 04:56:00.227 | INFO | RTMetrics report populated | 20260508 04:56:00.227 | INFO | Opening tmp RTMetrics json file | 20260508 04:56:00.229 | INFO | Serializing RTMetrics to json | 20260508 04:56:00.243 | INFO | Serialization done, swapping tmp file now... | 20260508 04:56:00.245 | INFO | RTMetrics done | 20260508 04:56:00.300 | INFO | Finished a round of Baz IO: Wrote 1.586 GiB in 3.5 seconds | 20260508 04:56:00.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.97% / 0.00% / 59.03% | 20260508 04:56:00.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.499682% / 3.97364e-06% / 5.5631e-05% | 20260508 04:56:00.300 | INFO | Baz IO overall: | 20260508 04:56:00.300 | INFO | BytesWritten : 2413493184201 | 20260508 04:56:00.300 | INFO | NumEvents : 1164857477037 | 20260508 04:56:01.837 | INFO | Analyzing chunk frames = [5402624,5403136) | 20260508 04:56:01.837 | INFO | MemInfo: { | 20260508 04:56:01.837 | INFO | "gpu" : " 17 GB", | 20260508 04:56:01.837 | INFO | "huge" : "", | 20260508 04:56:01.837 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:01.837 | INFO | "malloc" : " 14 GB", | 20260508 04:56:01.837 | INFO | "pinned" : " 29 GB" | 20260508 04:56:01.837 | INFO | } | 20260508 04:56:01.837 | INFO | Perf report for the last 5.13424 seconds: | 20260508 04:56:01.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:01.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:01.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:01.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:01.837 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 04:56:01.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:01.837 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 04:56:01.837 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 04:56:01.837 | INFO | BAZWRITER : 67.6%, 2.6, 91.2ms, 0 | 20260508 04:56:01.837 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:56:01.837 | INFO | Uploaded 36GiB to GPU in 2.57892s (13.9593GiB/s) | 20260508 04:56:01.837 | INFO | Downloaded 38GiB to GPU in 2.69741s (14.0876GiB/s) | 20260508 04:56:06.977 | INFO | Analyzing chunk frames = [5403136,5403648) | 20260508 04:56:06.977 | INFO | MemInfo: { | 20260508 04:56:06.977 | INFO | "gpu" : " 17 GB", | 20260508 04:56:06.977 | INFO | "huge" : "", | 20260508 04:56:06.977 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:06.977 | INFO | "malloc" : " 14 GB", | 20260508 04:56:06.977 | INFO | "pinned" : " 29 GB" | 20260508 04:56:06.977 | INFO | } | 20260508 04:56:06.977 | INFO | Perf report for the last 5.14049 seconds: | 20260508 04:56:06.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:06.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.977 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 04:56:06.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.977 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 04:56:06.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:06.978 | INFO | Uploaded 36GiB to GPU in 2.59699s (13.8622GiB/s) | 20260508 04:56:06.978 | INFO | Downloaded 34GiB to GPU in 2.4294s (13.9952GiB/s) | 20260508 04:56:12.118 | INFO | Analyzing chunk frames = [5403648,5404160) | 20260508 04:56:12.118 | INFO | MemInfo: { | 20260508 04:56:12.118 | INFO | "gpu" : " 17 GB", | 20260508 04:56:12.118 | INFO | "huge" : "", | 20260508 04:56:12.118 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:12.118 | INFO | "malloc" : " 14 GB", | 20260508 04:56:12.118 | INFO | "pinned" : " 29 GB" | 20260508 04:56:12.118 | INFO | } | 20260508 04:56:12.118 | INFO | Perf report for the last 5.14042 seconds: | 20260508 04:56:12.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:12.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.118 | INFO | ANALYSIS : 67.5%, 3.0, 104.8ms, 0 | 20260508 04:56:12.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.118 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:56:12.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:12.119 | INFO | Uploaded 36GiB to GPU in 2.55835s (14.0715GiB/s) | 20260508 04:56:12.119 | INFO | Downloaded 34GiB to GPU in 2.42501s (14.0206GiB/s) | 20260508 04:56:17.260 | INFO | Analyzing chunk frames = [5404160,5404672) | 20260508 04:56:17.260 | INFO | MemInfo: { | 20260508 04:56:17.260 | INFO | "gpu" : " 17 GB", | 20260508 04:56:17.260 | INFO | "huge" : "", | 20260508 04:56:17.260 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:17.260 | INFO | "malloc" : " 14 GB", | 20260508 04:56:17.260 | INFO | "pinned" : " 29 GB" | 20260508 04:56:17.260 | INFO | } | 20260508 04:56:17.260 | INFO | Perf report for the last 5.14247 seconds: | 20260508 04:56:17.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:17.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 04:56:17.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:56:17.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:17.260 | INFO | Uploaded 36GiB to GPU in 2.58475s (13.9279GiB/s) | 20260508 04:56:17.261 | INFO | Downloaded 34GiB to GPU in 2.50338s (13.5817GiB/s) | 20260508 04:56:22.397 | INFO | Analyzing chunk frames = [5404672,5405184) | 20260508 04:56:22.397 | INFO | MemInfo: { | 20260508 04:56:22.397 | INFO | "gpu" : " 17 GB", | 20260508 04:56:22.397 | INFO | "huge" : "", | 20260508 04:56:22.397 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:22.397 | INFO | "malloc" : " 14 GB", | 20260508 04:56:22.397 | INFO | "pinned" : " 29 GB" | 20260508 04:56:22.397 | INFO | } | 20260508 04:56:22.397 | INFO | Perf report for the last 5.13685 seconds: | 20260508 04:56:22.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:22.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.397 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 04:56:22.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.397 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:56:22.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:22.398 | INFO | Uploaded 36GiB to GPU in 2.42646s (14.8364GiB/s) | 20260508 04:56:22.398 | INFO | Downloaded 34GiB to GPU in 2.58164s (13.1699GiB/s) | 20260508 04:56:27.535 | INFO | Analyzing chunk frames = [5405184,5405696) | 20260508 04:56:27.535 | INFO | MemInfo: { | 20260508 04:56:27.535 | INFO | "gpu" : " 17 GB", | 20260508 04:56:27.535 | INFO | "huge" : "", | 20260508 04:56:27.535 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:27.535 | INFO | "malloc" : " 14 GB", | 20260508 04:56:27.535 | INFO | "pinned" : " 29 GB" | 20260508 04:56:27.535 | INFO | } | 20260508 04:56:27.535 | INFO | Perf report for the last 5.13816 seconds: | 20260508 04:56:27.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:27.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.535 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 04:56:27.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.535 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:56:27.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:27.536 | INFO | Uploaded 36GiB to GPU in 2.43552s (14.7812GiB/s) | 20260508 04:56:27.536 | INFO | Downloaded 34GiB to GPU in 2.42581s (14.0159GiB/s) | 20260508 04:56:32.678 | INFO | Analyzing chunk frames = [5405696,5406208) | 20260508 04:56:32.678 | INFO | MemInfo: { | 20260508 04:56:32.678 | INFO | "gpu" : " 17 GB", | 20260508 04:56:32.678 | INFO | "huge" : "", | 20260508 04:56:32.678 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:32.678 | INFO | "malloc" : " 14 GB", | 20260508 04:56:32.678 | INFO | "pinned" : " 29 GB" | 20260508 04:56:32.678 | INFO | } | 20260508 04:56:32.678 | INFO | Perf report for the last 5.14315 seconds: | 20260508 04:56:32.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:32.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.678 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:56:32.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.679 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:56:32.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:32.679 | INFO | Uploaded 36GiB to GPU in 2.41962s (14.8784GiB/s) | 20260508 04:56:32.679 | INFO | Downloaded 34GiB to GPU in 2.43257s (13.977GiB/s) | 20260508 04:56:37.816 | INFO | Analyzing chunk frames = [5406208,5406720) | 20260508 04:56:37.816 | INFO | MemInfo: { | 20260508 04:56:37.816 | INFO | "gpu" : " 17 GB", | 20260508 04:56:37.816 | INFO | "huge" : "", | 20260508 04:56:37.816 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:37.816 | INFO | "malloc" : " 14 GB", | 20260508 04:56:37.816 | INFO | "pinned" : " 29 GB" | 20260508 04:56:37.816 | INFO | } | 20260508 04:56:37.816 | INFO | Perf report for the last 5.13805 seconds: | 20260508 04:56:37.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:37.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:56:37.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 04:56:37.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:37.817 | INFO | Uploaded 36GiB to GPU in 2.4449s (14.7246GiB/s) | 20260508 04:56:37.817 | INFO | Downloaded 34GiB to GPU in 2.4325s (13.9774GiB/s) | 20260508 04:56:37.817 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:56:37.817 | INFO | Total mem usage: 16.7 GiB | 20260508 04:56:37.817 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:56:37.817 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:56:37.817 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:56:37.817 | INFO | High Water Breakdown: | 20260508 04:56:37.817 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:56:37.817 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:56:37.817 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:56:37.817 | INFO | ------------------------------------------ | 20260508 04:56:37.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:56:37.817 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:56:37.817 | INFO | Total mem usage: 26.5 GiB | 20260508 04:56:37.817 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:56:37.817 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:56:37.817 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:56:37.817 | INFO | High Water Breakdown: | 20260508 04:56:37.817 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:56:37.817 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:56:37.817 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:56:37.817 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:56:37.817 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:56:37.817 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:56:37.817 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:56:37.817 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:56:37.817 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:56:37.817 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:56:37.817 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:56:37.817 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:56:37.817 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:56:37.817 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:56:37.817 | INFO | ------------------------------------------ | 20260508 04:56:37.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:56:37.817 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:56:37.817 | INFO | Total mem usage: 27.8 GiB | 20260508 04:56:37.817 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:56:37.817 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:56:37.817 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:56:37.817 | INFO | High Water Breakdown: | 20260508 04:56:37.817 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:56:37.817 | INFO | ------------------------------------------ | 20260508 04:56:37.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:56:37.817 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:56:37.818 | INFO | Total mem usage: 15.7 GiB | 20260508 04:56:37.818 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:56:37.818 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:56:37.818 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:56:37.818 | INFO | High Water Breakdown: | 20260508 04:56:37.818 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:56:37.818 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:56:37.818 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:56:37.818 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:56:37.818 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:56:37.818 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:56:37.818 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:56:37.818 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:56:37.818 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:56:37.818 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:56:37.818 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:56:37.818 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:56:37.818 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:56:37.818 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:56:37.818 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:56:37.818 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:56:37.818 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:56:37.818 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:56:37.818 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:56:37.818 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:56:37.818 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:56:37.818 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:56:37.818 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:56:37.818 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:56:37.818 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:56:37.818 | INFO | ------------------------------------------ | 20260508 04:56:37.818 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:56:37.818 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:56:37.818 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:56:37.818 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:56:37.818 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.63e+01, 1.43e+01 / 1.94e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.16e-02 / 6.32e-03 | 20260508 04:56:37.818 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.45e+00, 5.81e-01 / 2.41e-01 | 20260508 04:56:37.818 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.40e+00, 2.08e-01 / 1.26e-01 | 20260508 04:56:37.818 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.56e+01, 3.90e+00 / 2.25e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.75e+00 / 1.31e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.54e+00, 4.17e-01 / 2.32e-01 | 20260508 04:56:37.818 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.22e+00, 2.61e-01 / 1.43e-01 | 20260508 04:56:37.818 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.51e-01, 5.11e-02 / 2.31e-02 | 20260508 04:56:37.818 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:56:37.818 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:56:37.818 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:56:37.818 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 24.36%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 2.64%, LockDownload -- 2.81e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:56:37.818 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260508 04:56:37.914 | INFO | Starting a round of Baz IO | 20260508 04:56:41.347 | INFO | Populating RTMetrics report | 20260508 04:56:41.347 | INFO | RTMetrics report populated | 20260508 04:56:41.347 | INFO | Opening tmp RTMetrics json file | 20260508 04:56:41.359 | INFO | Serializing RTMetrics to json | 20260508 04:56:41.360 | INFO | Serialization done, swapping tmp file now... | 20260508 04:56:41.361 | INFO | RTMetrics done | 20260508 04:56:41.417 | INFO | Finished a round of Baz IO: Wrote 1.586 GiB in 3.503 seconds | 20260508 04:56:41.417 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.96% / 0.00% / 59.03% | 20260508 04:56:41.417 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.499125% / 0% / 5.96046e-05% | 20260508 04:56:41.417 | INFO | Baz IO overall: | 20260508 04:56:41.417 | INFO | BytesWritten : 2415196222521 | 20260508 04:56:41.417 | INFO | NumEvents : 1165671005337 | 20260508 04:56:42.956 | INFO | Analyzing chunk frames = [5406720,5407232) | 20260508 04:56:42.956 | INFO | MemInfo: { | 20260508 04:56:42.956 | INFO | "gpu" : " 17 GB", | 20260508 04:56:42.956 | INFO | "huge" : "", | 20260508 04:56:42.956 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:42.956 | INFO | "malloc" : " 14 GB", | 20260508 04:56:42.956 | INFO | "pinned" : " 29 GB" | 20260508 04:56:42.956 | INFO | } | 20260508 04:56:42.956 | INFO | Perf report for the last 5.13913 seconds: | 20260508 04:56:42.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:42.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:42.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:42.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:42.956 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 04:56:42.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:42.956 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 04:56:42.956 | INFO | PRE_HQ : 65.5%, 1.6, 55.5ms, 0 | 20260508 04:56:42.956 | INFO | BAZWRITER : 67.7%, 2.6, 91.1ms, 0 | 20260508 04:56:42.956 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:56:42.956 | INFO | Uploaded 36GiB to GPU in 2.56391s (14.0411GiB/s) | 20260508 04:56:42.956 | INFO | Downloaded 38GiB to GPU in 2.70351s (14.0558GiB/s) | 20260508 04:56:48.092 | INFO | Analyzing chunk frames = [5407232,5407744) | 20260508 04:56:48.092 | INFO | MemInfo: { | 20260508 04:56:48.092 | INFO | "gpu" : " 17 GB", | 20260508 04:56:48.092 | INFO | "huge" : "", | 20260508 04:56:48.092 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:48.092 | INFO | "malloc" : " 14 GB", | 20260508 04:56:48.092 | INFO | "pinned" : " 29 GB" | 20260508 04:56:48.092 | INFO | } | 20260508 04:56:48.092 | INFO | Perf report for the last 5.1367 seconds: | 20260508 04:56:48.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:48.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.092 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 04:56:48.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.092 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:56:48.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:48.093 | INFO | Uploaded 36GiB to GPU in 2.56711s (14.0235GiB/s) | 20260508 04:56:48.093 | INFO | Downloaded 34GiB to GPU in 2.44744s (13.8921GiB/s) | 20260508 04:56:53.234 | INFO | Analyzing chunk frames = [5407744,5408256) | 20260508 04:56:53.234 | INFO | MemInfo: { | 20260508 04:56:53.234 | INFO | "gpu" : " 17 GB", | 20260508 04:56:53.234 | INFO | "huge" : "", | 20260508 04:56:53.234 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:53.234 | INFO | "malloc" : " 14 GB", | 20260508 04:56:53.234 | INFO | "pinned" : " 29 GB" | 20260508 04:56:53.234 | INFO | } | 20260508 04:56:53.234 | INFO | Perf report for the last 5.14155 seconds: | 20260508 04:56:53.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:53.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 04:56:53.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:56:53.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:53.234 | INFO | Uploaded 36GiB to GPU in 2.51533s (14.3122GiB/s) | 20260508 04:56:53.234 | INFO | Downloaded 34GiB to GPU in 2.42105s (14.0435GiB/s) | 20260508 04:56:58.377 | INFO | Analyzing chunk frames = [5408256,5408768) | 20260508 04:56:58.377 | INFO | MemInfo: { | 20260508 04:56:58.377 | INFO | "gpu" : " 17 GB", | 20260508 04:56:58.377 | INFO | "huge" : "", | 20260508 04:56:58.377 | INFO | "huge_shared" : " 29 GB", | 20260508 04:56:58.377 | INFO | "malloc" : " 14 GB", | 20260508 04:56:58.377 | INFO | "pinned" : " 29 GB" | 20260508 04:56:58.377 | INFO | } | 20260508 04:56:58.377 | INFO | Perf report for the last 5.14308 seconds: | 20260508 04:56:58.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:56:58.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.377 | INFO | ANALYSIS : 67.0%, 2.9, 103.7ms, 0 | 20260508 04:56:58.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.377 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:56:58.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:56:58.378 | INFO | Uploaded 36GiB to GPU in 2.55594s (14.0849GiB/s) | 20260508 04:56:58.378 | INFO | Downloaded 34GiB to GPU in 2.41643s (14.0704GiB/s) | 20260508 04:57:03.513 | INFO | Analyzing chunk frames = [5408768,5409280) | 20260508 04:57:03.513 | INFO | MemInfo: { | 20260508 04:57:03.513 | INFO | "gpu" : " 17 GB", | 20260508 04:57:03.513 | INFO | "huge" : "", | 20260508 04:57:03.513 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:03.513 | INFO | "malloc" : " 14 GB", | 20260508 04:57:03.513 | INFO | "pinned" : " 29 GB" | 20260508 04:57:03.513 | INFO | } | 20260508 04:57:03.513 | INFO | Perf report for the last 5.13603 seconds: | 20260508 04:57:03.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:03.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 04:57:03.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 04:57:03.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:03.513 | INFO | Uploaded 36GiB to GPU in 2.37021s (15.1885GiB/s) | 20260508 04:57:03.514 | INFO | Downloaded 34GiB to GPU in 2.55301s (13.3176GiB/s) | 20260508 04:57:08.653 | INFO | Analyzing chunk frames = [5409280,5409792) | 20260508 04:57:08.653 | INFO | MemInfo: { | 20260508 04:57:08.653 | INFO | "gpu" : " 17 GB", | 20260508 04:57:08.653 | INFO | "huge" : "", | 20260508 04:57:08.653 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:08.653 | INFO | "malloc" : " 14 GB", | 20260508 04:57:08.653 | INFO | "pinned" : " 29 GB" | 20260508 04:57:08.653 | INFO | } | 20260508 04:57:08.653 | INFO | Perf report for the last 5.14028 seconds: | 20260508 04:57:08.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:08.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.653 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:57:08.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.653 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:57:08.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:08.654 | INFO | Uploaded 36GiB to GPU in 2.41675s (14.896GiB/s) | 20260508 04:57:08.654 | INFO | Downloaded 34GiB to GPU in 2.4172s (14.0658GiB/s) | 20260508 04:57:13.789 | INFO | Analyzing chunk frames = [5409792,5410304) | 20260508 04:57:13.789 | INFO | MemInfo: { | 20260508 04:57:13.789 | INFO | "gpu" : " 17 GB", | 20260508 04:57:13.789 | INFO | "huge" : "", | 20260508 04:57:13.789 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:13.789 | INFO | "malloc" : " 14 GB", | 20260508 04:57:13.789 | INFO | "pinned" : " 29 GB" | 20260508 04:57:13.789 | INFO | } | 20260508 04:57:13.789 | INFO | Perf report for the last 5.13587 seconds: | 20260508 04:57:13.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:13.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.789 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 04:57:13.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.789 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 04:57:13.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:13.790 | INFO | Uploaded 36GiB to GPU in 2.4434s (14.7335GiB/s) | 20260508 04:57:13.790 | INFO | Downloaded 34GiB to GPU in 2.43334s (13.9726GiB/s) | 20260508 04:57:18.934 | INFO | Analyzing chunk frames = [5410304,5410816) | 20260508 04:57:18.934 | INFO | MemInfo: { | 20260508 04:57:18.934 | INFO | "gpu" : " 17 GB", | 20260508 04:57:18.934 | INFO | "huge" : "", | 20260508 04:57:18.934 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:18.934 | INFO | "malloc" : " 14 GB", | 20260508 04:57:18.934 | INFO | "pinned" : " 29 GB" | 20260508 04:57:18.934 | INFO | } | 20260508 04:57:18.934 | INFO | Perf report for the last 5.14471 seconds: | 20260508 04:57:18.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:18.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.934 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:57:18.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.934 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 04:57:18.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:18.935 | INFO | Uploaded 36GiB to GPU in 2.4251s (14.8447GiB/s) | 20260508 04:57:18.935 | INFO | Downloaded 34GiB to GPU in 2.42843s (14.0008GiB/s) | 20260508 04:57:19.031 | INFO | Starting a round of Baz IO | 20260508 04:57:22.462 | INFO | Populating RTMetrics report | 20260508 04:57:22.462 | INFO | RTMetrics report populated | 20260508 04:57:22.462 | INFO | Opening tmp RTMetrics json file | 20260508 04:57:22.471 | INFO | Serializing RTMetrics to json | 20260508 04:57:22.476 | INFO | Serialization done, swapping tmp file now... | 20260508 04:57:22.477 | INFO | RTMetrics done | 20260508 04:57:22.529 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.497 seconds | 20260508 04:57:22.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.96% / 0.00% / 59.04% | 20260508 04:57:22.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.498732% / 0% / 6.75519e-05% | 20260508 04:57:22.529 | INFO | Baz IO overall: | 20260508 04:57:22.529 | INFO | BytesWritten : 2416897417609 | 20260508 04:57:22.529 | INFO | NumEvents : 1166483069838 | 20260508 04:57:24.069 | INFO | Analyzing chunk frames = [5410816,5411328) | 20260508 04:57:24.069 | INFO | MemInfo: { | 20260508 04:57:24.069 | INFO | "gpu" : " 17 GB", | 20260508 04:57:24.069 | INFO | "huge" : "", | 20260508 04:57:24.069 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:24.069 | INFO | "malloc" : " 14 GB", | 20260508 04:57:24.069 | INFO | "pinned" : " 29 GB" | 20260508 04:57:24.069 | INFO | } | 20260508 04:57:24.069 | INFO | Perf report for the last 5.13493 seconds: | 20260508 04:57:24.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:24.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:24.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:24.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:24.069 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 04:57:24.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:24.069 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 04:57:24.069 | INFO | PRE_HQ : 66.2%, 1.6, 55.8ms, 0 | 20260508 04:57:24.069 | INFO | BAZWRITER : 67.7%, 2.6, 91.7ms, 0 | 20260508 04:57:24.069 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:57:24.069 | INFO | Uploaded 36GiB to GPU in 2.52682s (14.2471GiB/s) | 20260508 04:57:24.069 | INFO | Downloaded 38GiB to GPU in 2.69443s (14.1032GiB/s) | 20260508 04:57:29.210 | INFO | Analyzing chunk frames = [5411328,5411840) | 20260508 04:57:29.210 | INFO | MemInfo: { | 20260508 04:57:29.210 | INFO | "gpu" : " 17 GB", | 20260508 04:57:29.210 | INFO | "huge" : "", | 20260508 04:57:29.210 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:29.210 | INFO | "malloc" : " 14 GB", | 20260508 04:57:29.210 | INFO | "pinned" : " 29 GB" | 20260508 04:57:29.211 | INFO | } | 20260508 04:57:29.211 | INFO | Perf report for the last 5.14166 seconds: | 20260508 04:57:29.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:29.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 04:57:29.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:57:29.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:29.211 | INFO | Uploaded 36GiB to GPU in 2.45103s (14.6877GiB/s) | 20260508 04:57:29.211 | INFO | Downloaded 34GiB to GPU in 2.45085s (13.8727GiB/s) | 20260508 04:57:34.346 | INFO | Analyzing chunk frames = [5411840,5412352) | 20260508 04:57:34.346 | INFO | MemInfo: { | 20260508 04:57:34.346 | INFO | "gpu" : " 17 GB", | 20260508 04:57:34.346 | INFO | "huge" : "", | 20260508 04:57:34.346 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:34.346 | INFO | "malloc" : " 14 GB", | 20260508 04:57:34.346 | INFO | "pinned" : " 29 GB" | 20260508 04:57:34.346 | INFO | } | 20260508 04:57:34.346 | INFO | Perf report for the last 5.13573 seconds: | 20260508 04:57:34.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:34.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.346 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 04:57:34.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.346 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 04:57:34.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:34.347 | INFO | Uploaded 36GiB to GPU in 2.48817s (14.4685GiB/s) | 20260508 04:57:34.347 | INFO | Downloaded 34GiB to GPU in 2.43407s (13.9684GiB/s) | 20260508 04:57:39.486 | INFO | Analyzing chunk frames = [5412352,5412864) | 20260508 04:57:39.486 | INFO | MemInfo: { | 20260508 04:57:39.486 | INFO | "gpu" : " 17 GB", | 20260508 04:57:39.486 | INFO | "huge" : "", | 20260508 04:57:39.486 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:39.486 | INFO | "malloc" : " 14 GB", | 20260508 04:57:39.486 | INFO | "pinned" : " 29 GB" | 20260508 04:57:39.486 | INFO | } | 20260508 04:57:39.486 | INFO | Perf report for the last 5.13971 seconds: | 20260508 04:57:39.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:39.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 04:57:39.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:57:39.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:39.486 | INFO | Uploaded 36GiB to GPU in 2.49072s (14.4536GiB/s) | 20260508 04:57:39.486 | INFO | Downloaded 34GiB to GPU in 2.4135s (14.0874GiB/s) | 20260508 04:57:44.630 | INFO | Analyzing chunk frames = [5412864,5413376) | 20260508 04:57:44.630 | INFO | MemInfo: { | 20260508 04:57:44.630 | INFO | "gpu" : " 17 GB", | 20260508 04:57:44.630 | INFO | "huge" : "", | 20260508 04:57:44.630 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:44.630 | INFO | "malloc" : " 14 GB", | 20260508 04:57:44.630 | INFO | "pinned" : " 29 GB" | 20260508 04:57:44.630 | INFO | } | 20260508 04:57:44.630 | INFO | Perf report for the last 5.14443 seconds: | 20260508 04:57:44.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:44.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.631 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 04:57:44.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.631 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:57:44.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:44.631 | INFO | Uploaded 36GiB to GPU in 2.35748s (15.2705GiB/s) | 20260508 04:57:44.631 | INFO | Downloaded 34GiB to GPU in 2.54643s (13.352GiB/s) | 20260508 04:57:49.768 | INFO | Analyzing chunk frames = [5413376,5413888) | 20260508 04:57:49.768 | INFO | MemInfo: { | 20260508 04:57:49.768 | INFO | "gpu" : " 17 GB", | 20260508 04:57:49.768 | INFO | "huge" : "", | 20260508 04:57:49.768 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:49.768 | INFO | "malloc" : " 14 GB", | 20260508 04:57:49.768 | INFO | "pinned" : " 29 GB" | 20260508 04:57:49.768 | INFO | } | 20260508 04:57:49.769 | INFO | Perf report for the last 5.13808 seconds: | 20260508 04:57:49.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:49.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 04:57:49.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:57:49.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:49.769 | INFO | Uploaded 36GiB to GPU in 2.41434s (14.9109GiB/s) | 20260508 04:57:49.769 | INFO | Downloaded 34GiB to GPU in 2.43516s (13.9621GiB/s) | 20260508 04:57:54.905 | INFO | Analyzing chunk frames = [5413888,5414400) | 20260508 04:57:54.905 | INFO | MemInfo: { | 20260508 04:57:54.905 | INFO | "gpu" : " 17 GB", | 20260508 04:57:54.905 | INFO | "huge" : "", | 20260508 04:57:54.905 | INFO | "huge_shared" : " 29 GB", | 20260508 04:57:54.905 | INFO | "malloc" : " 14 GB", | 20260508 04:57:54.905 | INFO | "pinned" : " 29 GB" | 20260508 04:57:54.905 | INFO | } | 20260508 04:57:54.905 | INFO | Perf report for the last 5.13679 seconds: | 20260508 04:57:54.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:57:54.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.905 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 04:57:54.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.905 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 04:57:54.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:57:54.906 | INFO | Uploaded 36GiB to GPU in 2.46507s (14.6041GiB/s) | 20260508 04:57:54.906 | INFO | Downloaded 34GiB to GPU in 2.44977s (13.8788GiB/s) | 20260508 04:58:00.044 | INFO | Analyzing chunk frames = [5414400,5414912) | 20260508 04:58:00.044 | INFO | MemInfo: { | 20260508 04:58:00.044 | INFO | "gpu" : " 17 GB", | 20260508 04:58:00.044 | INFO | "huge" : "", | 20260508 04:58:00.044 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:00.044 | INFO | "malloc" : " 14 GB", | 20260508 04:58:00.044 | INFO | "pinned" : " 29 GB" | 20260508 04:58:00.044 | INFO | } | 20260508 04:58:00.044 | INFO | Perf report for the last 5.13887 seconds: | 20260508 04:58:00.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:00.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.044 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 04:58:00.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.044 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 04:58:00.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:00.045 | INFO | Uploaded 36GiB to GPU in 2.43863s (14.7624GiB/s) | 20260508 04:58:00.045 | INFO | Downloaded 34GiB to GPU in 2.43864s (13.9422GiB/s) | 20260508 04:58:00.045 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:58:00.045 | INFO | Total mem usage: 16.7 GiB | 20260508 04:58:00.045 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:58:00.045 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17.1%) | 20260508 04:58:00.045 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:58:00.045 | INFO | High Water Breakdown: | 20260508 04:58:00.045 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:58:00.045 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:58:00.045 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:58:00.045 | INFO | ------------------------------------------ | 20260508 04:58:00.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:58:00.045 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:58:00.045 | INFO | Total mem usage: 26.5 GiB | 20260508 04:58:00.045 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:58:00.045 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 04:58:00.045 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 04:58:00.045 | INFO | High Water Breakdown: | 20260508 04:58:00.045 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:58:00.045 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:58:00.045 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:58:00.045 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:58:00.045 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:58:00.045 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:58:00.045 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:58:00.045 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:58:00.045 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:58:00.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:58:00.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:58:00.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:58:00.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:58:00.045 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:58:00.045 | INFO | ------------------------------------------ | 20260508 04:58:00.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:58:00.045 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:58:00.045 | INFO | Total mem usage: 27.8 GiB | 20260508 04:58:00.045 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:58:00.045 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 04:58:00.045 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 04:58:00.045 | INFO | High Water Breakdown: | 20260508 04:58:00.045 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 04:58:00.045 | INFO | ------------------------------------------ | 20260508 04:58:00.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:58:00.046 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:58:00.046 | INFO | Total mem usage: 15.7 GiB | 20260508 04:58:00.046 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:58:00.046 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:58:00.046 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:58:00.046 | INFO | High Water Breakdown: | 20260508 04:58:00.046 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:58:00.046 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:58:00.046 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:58:00.046 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:58:00.046 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:58:00.046 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:58:00.046 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:58:00.046 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:58:00.046 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:58:00.046 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:58:00.046 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:58:00.046 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:58:00.046 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:58:00.046 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:58:00.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:58:00.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:58:00.046 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:58:00.046 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:58:00.046 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:58:00.046 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:58:00.046 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:58:00.046 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:58:00.046 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:58:00.046 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:58:00.046 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:58:00.046 | INFO | ------------------------------------------ | 20260508 04:58:00.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:58:00.046 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:58:00.046 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:58:00.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:58:00.046 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.57e+01, 1.40e+01 / 1.23e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 0.29%, AggregateStats -- 9.65e-02 / 1.01e-01, 1.12e-02 / 5.00e-03 | 20260508 04:58:00.046 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.41e+00, 5.90e-01 / 2.02e-01 | 20260508 04:58:00.046 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.06e-01 | 20260508 04:58:00.046 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.49e+01, 3.65e+00 / 1.59e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.86e+01 / 2.07e+01, 1.84e+00 / 1.45e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.38e+00 / 2.57e+00, 4.20e-01 / 1.97e-01 | 20260508 04:58:00.046 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.26e+00, 2.61e-01 / 1.37e-01 | 20260508 04:58:00.046 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.57e-01, 5.19e-02 / 2.27e-02 | 20260508 04:58:00.046 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:58:00.046 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:58:00.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:58:00.046 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 24.22%, Upload -- 2.57e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 30.87%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 2.03%, LockUpload -- 2.15e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 2.80%, LockDownload -- 2.98e+00 / 0.00e+00, 4.39e+00 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 15.73%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 04:58:00.046 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260508 04:58:00.142 | INFO | Starting a round of Baz IO | 20260508 04:58:03.567 | INFO | Populating RTMetrics report | 20260508 04:58:03.567 | INFO | RTMetrics report populated | 20260508 04:58:03.567 | INFO | Opening tmp RTMetrics json file | 20260508 04:58:03.579 | INFO | Serializing RTMetrics to json | 20260508 04:58:03.580 | INFO | Serialization done, swapping tmp file now... | 20260508 04:58:03.581 | INFO | RTMetrics done | 20260508 04:58:03.641 | INFO | Finished a round of Baz IO: Wrote 1.585 GiB in 3.498 seconds | 20260508 04:58:03.641 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.96% / 0.00% / 59.04% | 20260508 04:58:03.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.49783% / 3.97364e-06% / 7.54992e-05% | 20260508 04:58:03.641 | INFO | Baz IO overall: | 20260508 04:58:03.641 | INFO | BytesWritten : 2418599121860 | 20260508 04:58:03.641 | INFO | NumEvents : 1167295465193 | 20260508 04:58:05.182 | INFO | Analyzing chunk frames = [5414912,5415424) | 20260508 04:58:05.182 | INFO | MemInfo: { | 20260508 04:58:05.182 | INFO | "gpu" : " 17 GB", | 20260508 04:58:05.182 | INFO | "huge" : "", | 20260508 04:58:05.182 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:05.182 | INFO | "malloc" : " 14 GB", | 20260508 04:58:05.182 | INFO | "pinned" : " 29 GB" | 20260508 04:58:05.182 | INFO | } | 20260508 04:58:05.182 | INFO | Perf report for the last 5.13808 seconds: | 20260508 04:58:05.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:05.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:05.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:05.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:05.182 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 04:58:05.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:05.183 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 04:58:05.183 | INFO | PRE_HQ : 65.5%, 1.6, 55.5ms, 0 | 20260508 04:58:05.183 | INFO | BAZWRITER : 67.5%, 2.7, 92.4ms, 0 | 20260508 04:58:05.183 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:58:05.183 | INFO | Uploaded 36GiB to GPU in 2.58289s (13.9379GiB/s) | 20260508 04:58:05.183 | INFO | Downloaded 38GiB to GPU in 2.69414s (14.1047GiB/s) | 20260508 04:58:10.326 | INFO | Analyzing chunk frames = [5415424,5415936) | 20260508 04:58:10.326 | INFO | MemInfo: { | 20260508 04:58:10.326 | INFO | "gpu" : " 17 GB", | 20260508 04:58:10.326 | INFO | "huge" : "", | 20260508 04:58:10.326 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:10.326 | INFO | "malloc" : " 14 GB", | 20260508 04:58:10.326 | INFO | "pinned" : " 29 GB" | 20260508 04:58:10.326 | INFO | } | 20260508 04:58:10.326 | INFO | Perf report for the last 5.14401 seconds: | 20260508 04:58:10.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:10.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.326 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 04:58:10.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.326 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 04:58:10.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:10.327 | INFO | Uploaded 36GiB to GPU in 2.56021s (14.0614GiB/s) | 20260508 04:58:10.327 | INFO | Downloaded 34GiB to GPU in 2.44576s (13.9016GiB/s) | 20260508 04:58:15.467 | INFO | Analyzing chunk frames = [5415936,5416448) | 20260508 04:58:15.467 | INFO | MemInfo: { | 20260508 04:58:15.467 | INFO | "gpu" : " 17 GB", | 20260508 04:58:15.467 | INFO | "huge" : "", | 20260508 04:58:15.467 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:15.467 | INFO | "malloc" : " 14 GB", | 20260508 04:58:15.467 | INFO | "pinned" : " 29 GB" | 20260508 04:58:15.467 | INFO | } | 20260508 04:58:15.467 | INFO | Perf report for the last 5.14081 seconds: | 20260508 04:58:15.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:15.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 04:58:15.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:58:15.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:15.467 | INFO | Uploaded 36GiB to GPU in 2.52035s (14.2837GiB/s) | 20260508 04:58:15.467 | INFO | Downloaded 34GiB to GPU in 2.41224s (14.0948GiB/s) | 20260508 04:58:20.601 | INFO | Analyzing chunk frames = [5416448,5416960) | 20260508 04:58:20.601 | INFO | MemInfo: { | 20260508 04:58:20.601 | INFO | "gpu" : " 17 GB", | 20260508 04:58:20.601 | INFO | "huge" : "", | 20260508 04:58:20.601 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:20.601 | INFO | "malloc" : " 14 GB", | 20260508 04:58:20.601 | INFO | "pinned" : " 29 GB" | 20260508 04:58:20.601 | INFO | } | 20260508 04:58:20.602 | INFO | Perf report for the last 5.13434 seconds: | 20260508 04:58:20.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:20.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 04:58:20.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:58:20.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:20.602 | INFO | Uploaded 36GiB to GPU in 2.54063s (14.1697GiB/s) | 20260508 04:58:20.603 | INFO | Downloaded 34GiB to GPU in 2.40092s (14.1613GiB/s) | 20260508 04:58:25.740 | INFO | Analyzing chunk frames = [5416960,5417472) | 20260508 04:58:25.740 | INFO | MemInfo: { | 20260508 04:58:25.741 | INFO | "gpu" : " 17 GB", | 20260508 04:58:25.741 | INFO | "huge" : "", | 20260508 04:58:25.741 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:25.741 | INFO | "malloc" : " 14 GB", | 20260508 04:58:25.741 | INFO | "pinned" : " 29 GB" | 20260508 04:58:25.741 | INFO | } | 20260508 04:58:25.741 | INFO | Perf report for the last 5.13922 seconds: | 20260508 04:58:25.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:25.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 04:58:25.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 04:58:25.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:25.741 | INFO | Uploaded 36GiB to GPU in 2.37812s (15.138GiB/s) | 20260508 04:58:25.741 | INFO | Downloaded 34GiB to GPU in 2.56748s (13.2425GiB/s) | 20260508 04:58:30.883 | INFO | Analyzing chunk frames = [5417472,5417984) | 20260508 04:58:30.883 | INFO | MemInfo: { | 20260508 04:58:30.883 | INFO | "gpu" : " 17 GB", | 20260508 04:58:30.883 | INFO | "huge" : "", | 20260508 04:58:30.883 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:30.883 | INFO | "malloc" : " 14 GB", | 20260508 04:58:30.883 | INFO | "pinned" : " 29 GB" | 20260508 04:58:30.883 | INFO | } | 20260508 04:58:30.883 | INFO | Perf report for the last 5.14233 seconds: | 20260508 04:58:30.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:30.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 04:58:30.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:58:30.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:30.883 | INFO | Uploaded 36GiB to GPU in 2.41816s (14.8874GiB/s) | 20260508 04:58:30.883 | INFO | Downloaded 34GiB to GPU in 2.42672s (14.0107GiB/s) | 20260508 04:58:36.021 | INFO | Analyzing chunk frames = [5417984,5418496) | 20260508 04:58:36.021 | INFO | MemInfo: { | 20260508 04:58:36.021 | INFO | "gpu" : " 17 GB", | 20260508 04:58:36.021 | INFO | "huge" : "", | 20260508 04:58:36.021 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:36.021 | INFO | "malloc" : " 14 GB", | 20260508 04:58:36.021 | INFO | "pinned" : " 29 GB" | 20260508 04:58:36.021 | INFO | } | 20260508 04:58:36.021 | INFO | Perf report for the last 5.13826 seconds: | 20260508 04:58:36.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:36.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.021 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:58:36.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.021 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 04:58:36.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:36.022 | INFO | Uploaded 36GiB to GPU in 2.42429s (14.8497GiB/s) | 20260508 04:58:36.022 | INFO | Downloaded 34GiB to GPU in 2.42578s (14.0161GiB/s) | 20260508 04:58:41.156 | INFO | Analyzing chunk frames = [5418496,5419008) | 20260508 04:58:41.156 | INFO | MemInfo: { | 20260508 04:58:41.156 | INFO | "gpu" : " 17 GB", | 20260508 04:58:41.156 | INFO | "huge" : "", | 20260508 04:58:41.156 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:41.156 | INFO | "malloc" : " 14 GB", | 20260508 04:58:41.156 | INFO | "pinned" : " 29 GB" | 20260508 04:58:41.156 | INFO | } | 20260508 04:58:41.156 | INFO | Perf report for the last 5.13499 seconds: | 20260508 04:58:41.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:41.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.156 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 04:58:41.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.156 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 04:58:41.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:41.157 | INFO | Uploaded 36GiB to GPU in 2.43968s (14.756GiB/s) | 20260508 04:58:41.157 | INFO | Downloaded 34GiB to GPU in 2.43414s (13.968GiB/s) | 20260508 04:58:41.267 | INFO | Starting a round of Baz IO | 20260508 04:58:44.685 | INFO | Populating RTMetrics report | 20260508 04:58:44.685 | INFO | RTMetrics report populated | 20260508 04:58:44.685 | INFO | Opening tmp RTMetrics json file | 20260508 04:58:44.697 | INFO | Serializing RTMetrics to json | 20260508 04:58:44.698 | INFO | Serialization done, swapping tmp file now... | 20260508 04:58:44.699 | INFO | RTMetrics done | 20260508 04:58:44.757 | INFO | Finished a round of Baz IO: Wrote 1.585 GiB in 3.49 seconds | 20260508 04:58:44.757 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.96% / 0.00% / 59.04% | 20260508 04:58:44.758 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.498056% / 0% / 7.94729e-05% | 20260508 04:58:44.758 | INFO | Baz IO overall: | 20260508 04:58:44.758 | INFO | BytesWritten : 2420301055813 | 20260508 04:58:44.758 | INFO | NumEvents : 1168108044547 | 20260508 04:58:46.300 | INFO | Analyzing chunk frames = [5419008,5419520) | 20260508 04:58:46.300 | INFO | MemInfo: { | 20260508 04:58:46.300 | INFO | "gpu" : " 17 GB", | 20260508 04:58:46.300 | INFO | "huge" : "", | 20260508 04:58:46.300 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:46.300 | INFO | "malloc" : " 14 GB", | 20260508 04:58:46.300 | INFO | "pinned" : " 29 GB" | 20260508 04:58:46.300 | INFO | } | 20260508 04:58:46.300 | INFO | Perf report for the last 5.14387 seconds: | 20260508 04:58:46.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:46.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:46.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:46.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:46.300 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 04:58:46.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:46.300 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 04:58:46.300 | INFO | PRE_HQ : 66.5%, 1.8, 62.2ms, 0 | 20260508 04:58:46.300 | INFO | BAZWRITER : 67.4%, 2.8, 97.6ms, 0 | 20260508 04:58:46.300 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:58:46.301 | INFO | Uploaded 36GiB to GPU in 2.575s (13.9806GiB/s) | 20260508 04:58:46.301 | INFO | Downloaded 38GiB to GPU in 2.69059s (14.1233GiB/s) | 20260508 04:58:51.436 | INFO | Analyzing chunk frames = [5419520,5420032) | 20260508 04:58:51.436 | INFO | MemInfo: { | 20260508 04:58:51.436 | INFO | "gpu" : " 17 GB", | 20260508 04:58:51.436 | INFO | "huge" : "", | 20260508 04:58:51.437 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:51.437 | INFO | "malloc" : " 14 GB", | 20260508 04:58:51.437 | INFO | "pinned" : " 29 GB" | 20260508 04:58:51.437 | INFO | } | 20260508 04:58:51.437 | INFO | Perf report for the last 5.1364 seconds: | 20260508 04:58:51.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:51.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 04:58:51.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 04:58:51.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:51.437 | INFO | Uploaded 36GiB to GPU in 2.59557s (13.8698GiB/s) | 20260508 04:58:51.437 | INFO | Downloaded 34GiB to GPU in 2.43571s (13.959GiB/s) | 20260508 04:58:56.585 | INFO | Analyzing chunk frames = [5420032,5420544) | 20260508 04:58:56.585 | INFO | MemInfo: { | 20260508 04:58:56.585 | INFO | "gpu" : " 17 GB", | 20260508 04:58:56.585 | INFO | "huge" : "", | 20260508 04:58:56.585 | INFO | "huge_shared" : " 29 GB", | 20260508 04:58:56.585 | INFO | "malloc" : " 14 GB", | 20260508 04:58:56.585 | INFO | "pinned" : " 29 GB" | 20260508 04:58:56.585 | INFO | } | 20260508 04:58:56.585 | INFO | Perf report for the last 5.14841 seconds: | 20260508 04:58:56.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:58:56.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.585 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 04:58:56.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.585 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 04:58:56.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:58:56.586 | INFO | Uploaded 36GiB to GPU in 2.55849s (14.0708GiB/s) | 20260508 04:58:56.586 | INFO | Downloaded 34GiB to GPU in 2.41836s (14.0591GiB/s) | 20260508 04:59:01.722 | INFO | Analyzing chunk frames = [5420544,5421056) | 20260508 04:59:01.722 | INFO | MemInfo: { | 20260508 04:59:01.722 | INFO | "gpu" : " 17 GB", | 20260508 04:59:01.722 | INFO | "huge" : "", | 20260508 04:59:01.722 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:01.722 | INFO | "malloc" : " 14 GB", | 20260508 04:59:01.722 | INFO | "pinned" : " 29 GB" | 20260508 04:59:01.722 | INFO | } | 20260508 04:59:01.722 | INFO | Perf report for the last 5.13713 seconds: | 20260508 04:59:01.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:01.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.722 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 04:59:01.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.722 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:59:01.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:01.723 | INFO | Uploaded 36GiB to GPU in 2.56808s (14.0183GiB/s) | 20260508 04:59:01.723 | INFO | Downloaded 34GiB to GPU in 2.39068s (14.2219GiB/s) | 20260508 04:59:06.853 | INFO | Analyzing chunk frames = [5421056,5421568) | 20260508 04:59:06.853 | INFO | MemInfo: { | 20260508 04:59:06.853 | INFO | "gpu" : " 17 GB", | 20260508 04:59:06.853 | INFO | "huge" : "", | 20260508 04:59:06.854 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:06.854 | INFO | "malloc" : " 14 GB", | 20260508 04:59:06.854 | INFO | "pinned" : " 29 GB" | 20260508 04:59:06.854 | INFO | } | 20260508 04:59:06.854 | INFO | Perf report for the last 5.13145 seconds: | 20260508 04:59:06.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:06.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 04:59:06.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 04:59:06.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:06.854 | INFO | Uploaded 36GiB to GPU in 2.3336s (15.4268GiB/s) | 20260508 04:59:06.854 | INFO | Downloaded 34GiB to GPU in 2.49632s (13.6201GiB/s) | 20260508 04:59:12.001 | INFO | Analyzing chunk frames = [5421568,5422080) | 20260508 04:59:12.001 | INFO | MemInfo: { | 20260508 04:59:12.001 | INFO | "gpu" : " 17 GB", | 20260508 04:59:12.001 | INFO | "huge" : "", | 20260508 04:59:12.001 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:12.001 | INFO | "malloc" : " 14 GB", | 20260508 04:59:12.001 | INFO | "pinned" : " 29 GB" | 20260508 04:59:12.001 | INFO | } | 20260508 04:59:12.001 | INFO | Perf report for the last 5.14757 seconds: | 20260508 04:59:12.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:12.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.001 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 04:59:12.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.001 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 04:59:12.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:12.002 | INFO | Uploaded 36GiB to GPU in 2.42622s (14.8379GiB/s) | 20260508 04:59:12.002 | INFO | Downloaded 34GiB to GPU in 2.42165s (14.04GiB/s) | 20260508 04:59:17.140 | INFO | Analyzing chunk frames = [5422080,5422592) | 20260508 04:59:17.141 | INFO | MemInfo: { | 20260508 04:59:17.141 | INFO | "gpu" : " 17 GB", | 20260508 04:59:17.141 | INFO | "huge" : "", | 20260508 04:59:17.141 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:17.141 | INFO | "malloc" : " 14 GB", | 20260508 04:59:17.141 | INFO | "pinned" : " 29 GB" | 20260508 04:59:17.141 | INFO | } | 20260508 04:59:17.141 | INFO | Perf report for the last 5.13967 seconds: | 20260508 04:59:17.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:17.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:59:17.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 04:59:17.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:17.141 | INFO | Uploaded 36GiB to GPU in 2.44926s (14.6983GiB/s) | 20260508 04:59:17.141 | INFO | Downloaded 34GiB to GPU in 2.4264s (14.0126GiB/s) | 20260508 04:59:22.277 | INFO | Analyzing chunk frames = [5422592,5423104) | 20260508 04:59:22.277 | INFO | MemInfo: { | 20260508 04:59:22.277 | INFO | "gpu" : " 17 GB", | 20260508 04:59:22.277 | INFO | "huge" : "", | 20260508 04:59:22.277 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:22.277 | INFO | "malloc" : " 14 GB", | 20260508 04:59:22.277 | INFO | "pinned" : " 29 GB" | 20260508 04:59:22.277 | INFO | } | 20260508 04:59:22.277 | INFO | Perf report for the last 5.13633 seconds: | 20260508 04:59:22.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:22.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.277 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 04:59:22.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.278 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 04:59:22.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:22.278 | INFO | Uploaded 36GiB to GPU in 2.47107s (14.5686GiB/s) | 20260508 04:59:22.278 | INFO | Downloaded 34GiB to GPU in 2.44781s (13.89GiB/s) | 20260508 04:59:22.278 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 04:59:22.278 | INFO | Total mem usage: 16.7 GiB | 20260508 04:59:22.278 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 04:59:22.278 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 04:59:22.278 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 04:59:22.278 | INFO | High Water Breakdown: | 20260508 04:59:22.278 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 04:59:22.278 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 04:59:22.278 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 04:59:22.278 | INFO | ------------------------------------------ | 20260508 04:59:22.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:59:22.278 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 04:59:22.278 | INFO | Total mem usage: 26.5 GiB | 20260508 04:59:22.278 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 04:59:22.278 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 04:59:22.278 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 04:59:22.278 | INFO | High Water Breakdown: | 20260508 04:59:22.278 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:59:22.278 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:59:22.278 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:59:22.278 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:59:22.278 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:59:22.278 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:59:22.278 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 04:59:22.278 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 04:59:22.278 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:59:22.278 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 04:59:22.278 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 04:59:22.278 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 04:59:22.278 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 04:59:22.278 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 04:59:22.278 | INFO | ------------------------------------------ | 20260508 04:59:22.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:59:22.278 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 04:59:22.278 | INFO | Total mem usage: 27.8 GiB | 20260508 04:59:22.278 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 04:59:22.278 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 04:59:22.278 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 04:59:22.278 | INFO | High Water Breakdown: | 20260508 04:59:22.278 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 04:59:22.278 | INFO | ------------------------------------------ | 20260508 04:59:22.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:59:22.278 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 04:59:22.278 | INFO | Total mem usage: 15.7 GiB | 20260508 04:59:22.278 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 04:59:22.278 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 04:59:22.278 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 04:59:22.278 | INFO | High Water Breakdown: | 20260508 04:59:22.278 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 04:59:22.278 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 04:59:22.278 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 04:59:22.278 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 04:59:22.278 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 04:59:22.278 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 04:59:22.278 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 04:59:22.278 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 04:59:22.278 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 04:59:22.278 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 04:59:22.278 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 04:59:22.278 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 04:59:22.278 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 04:59:22.278 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 04:59:22.278 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 04:59:22.278 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 04:59:22.278 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 04:59:22.278 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 04:59:22.278 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 04:59:22.278 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 04:59:22.278 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 04:59:22.278 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 04:59:22.278 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 04:59:22.278 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 04:59:22.278 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 04:59:22.278 | INFO | ------------------------------------------ | 20260508 04:59:22.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 04:59:22.278 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 04:59:22.279 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:59:22.279 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:59:22.279 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.23e+00 | 20260508 04:59:22.279 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.14e-02 / 5.88e-03 | 20260508 04:59:22.279 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.42e+00, 5.89e-01 / 2.22e-01 | 20260508 04:59:22.279 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.42e+00, 2.09e-01 / 1.36e-01 | 20260508 04:59:22.279 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.57e+01, 3.97e+00 / 1.70e+00 | 20260508 04:59:22.279 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.75e+00 / 1.35e+00 | 20260508 04:59:22.279 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.60e+00, 4.16e-01 / 2.59e-01 | 20260508 04:59:22.279 | INFO | PerfReport: 3.21%, Metrics -- 1.05e+00 / 1.22e+00, 2.61e-01 / 1.43e-01 | 20260508 04:59:22.280 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.55e-01, 5.18e-02 / 2.56e-02 | 20260508 04:59:22.280 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 04:59:22.280 | INFO | PerfReport: Gathered 1536 samples. | 20260508 04:59:22.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 04:59:22.280 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 24.40%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 2.32%, LockUpload -- 2.47e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 2.58%, LockDownload -- 2.74e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 04:59:22.280 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 04:59:22.390 | INFO | Starting a round of Baz IO | 20260508 04:59:25.816 | INFO | Populating RTMetrics report | 20260508 04:59:25.817 | INFO | RTMetrics report populated | 20260508 04:59:25.817 | INFO | Opening tmp RTMetrics json file | 20260508 04:59:25.829 | INFO | Serializing RTMetrics to json | 20260508 04:59:25.830 | INFO | Serialization done, swapping tmp file now... | 20260508 04:59:25.831 | INFO | RTMetrics done | 20260508 04:59:25.887 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.497 seconds | 20260508 04:59:25.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.95% / 0.00% / 59.04% | 20260508 04:59:25.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.49684% / 3.97364e-06% / 8.34465e-05% | 20260508 04:59:25.887 | INFO | Baz IO overall: | 20260508 04:59:25.887 | INFO | BytesWritten : 2422001875024 | 20260508 04:59:25.887 | INFO | NumEvents : 1168919807186 | 20260508 04:59:27.412 | INFO | Analyzing chunk frames = [5423104,5423616) | 20260508 04:59:27.412 | INFO | MemInfo: { | 20260508 04:59:27.412 | INFO | "gpu" : " 17 GB", | 20260508 04:59:27.412 | INFO | "huge" : "", | 20260508 04:59:27.412 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:27.412 | INFO | "malloc" : " 14 GB", | 20260508 04:59:27.412 | INFO | "pinned" : " 29 GB" | 20260508 04:59:27.412 | INFO | } | 20260508 04:59:27.412 | INFO | Perf report for the last 5.13469 seconds: | 20260508 04:59:27.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:27.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:27.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:27.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:27.412 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 04:59:27.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:27.412 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 04:59:27.412 | INFO | PRE_HQ : 66.4%, 1.8, 61.6ms, 0 | 20260508 04:59:27.412 | INFO | BAZWRITER : 67.6%, 2.8, 97.5ms, 0 | 20260508 04:59:27.412 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 04:59:27.413 | INFO | Uploaded 36GiB to GPU in 2.56202s (14.0514GiB/s) | 20260508 04:59:27.413 | INFO | Downloaded 38GiB to GPU in 2.69722s (14.0886GiB/s) | 20260508 04:59:32.551 | INFO | Analyzing chunk frames = [5423616,5424128) | 20260508 04:59:32.551 | INFO | MemInfo: { | 20260508 04:59:32.551 | INFO | "gpu" : " 17 GB", | 20260508 04:59:32.551 | INFO | "huge" : "", | 20260508 04:59:32.551 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:32.551 | INFO | "malloc" : " 14 GB", | 20260508 04:59:32.551 | INFO | "pinned" : " 29 GB" | 20260508 04:59:32.551 | INFO | } | 20260508 04:59:32.551 | INFO | Perf report for the last 5.13928 seconds: | 20260508 04:59:32.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:32.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.551 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 04:59:32.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.551 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 04:59:32.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:32.552 | INFO | Uploaded 36GiB to GPU in 2.56517s (14.0341GiB/s) | 20260508 04:59:32.552 | INFO | Downloaded 34GiB to GPU in 2.45632s (13.8418GiB/s) | 20260508 04:59:37.691 | INFO | Analyzing chunk frames = [5424128,5424640) | 20260508 04:59:37.691 | INFO | MemInfo: { | 20260508 04:59:37.691 | INFO | "gpu" : " 17 GB", | 20260508 04:59:37.691 | INFO | "huge" : "", | 20260508 04:59:37.691 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:37.691 | INFO | "malloc" : " 14 GB", | 20260508 04:59:37.691 | INFO | "pinned" : " 29 GB" | 20260508 04:59:37.691 | INFO | } | 20260508 04:59:37.691 | INFO | Perf report for the last 5.13999 seconds: | 20260508 04:59:37.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:37.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.691 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 04:59:37.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.691 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:59:37.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:37.692 | INFO | Uploaded 36GiB to GPU in 2.51731s (14.301GiB/s) | 20260508 04:59:37.692 | INFO | Downloaded 34GiB to GPU in 2.45014s (13.8768GiB/s) | 20260508 04:59:42.832 | INFO | Analyzing chunk frames = [5424640,5425152) | 20260508 04:59:42.832 | INFO | MemInfo: { | 20260508 04:59:42.832 | INFO | "gpu" : " 17 GB", | 20260508 04:59:42.832 | INFO | "huge" : "", | 20260508 04:59:42.832 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:42.832 | INFO | "malloc" : " 14 GB", | 20260508 04:59:42.832 | INFO | "pinned" : " 29 GB" | 20260508 04:59:42.832 | INFO | } | 20260508 04:59:42.832 | INFO | Perf report for the last 5.1407 seconds: | 20260508 04:59:42.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:42.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 04:59:42.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 04:59:42.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:42.832 | INFO | Uploaded 36GiB to GPU in 2.50271s (14.3844GiB/s) | 20260508 04:59:42.833 | INFO | Downloaded 34GiB to GPU in 2.39834s (14.1765GiB/s) | 20260508 04:59:47.971 | INFO | Analyzing chunk frames = [5425152,5425664) | 20260508 04:59:47.971 | INFO | MemInfo: { | 20260508 04:59:47.971 | INFO | "gpu" : " 17 GB", | 20260508 04:59:47.971 | INFO | "huge" : "", | 20260508 04:59:47.971 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:47.971 | INFO | "malloc" : " 14 GB", | 20260508 04:59:47.971 | INFO | "pinned" : " 29 GB" | 20260508 04:59:47.972 | INFO | } | 20260508 04:59:47.972 | INFO | Perf report for the last 5.13961 seconds: | 20260508 04:59:47.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:47.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 04:59:47.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 04:59:47.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:47.972 | INFO | Uploaded 36GiB to GPU in 2.35108s (15.3121GiB/s) | 20260508 04:59:47.972 | INFO | Downloaded 34GiB to GPU in 2.56763s (13.2418GiB/s) | 20260508 04:59:53.107 | INFO | Analyzing chunk frames = [5425664,5426176) | 20260508 04:59:53.107 | INFO | MemInfo: { | 20260508 04:59:53.107 | INFO | "gpu" : " 17 GB", | 20260508 04:59:53.107 | INFO | "huge" : "", | 20260508 04:59:53.107 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:53.107 | INFO | "malloc" : " 14 GB", | 20260508 04:59:53.107 | INFO | "pinned" : " 29 GB" | 20260508 04:59:53.107 | INFO | } | 20260508 04:59:53.107 | INFO | Perf report for the last 5.13529 seconds: | 20260508 04:59:53.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:53.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 04:59:53.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 04:59:53.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:53.107 | INFO | Uploaded 36GiB to GPU in 2.43191s (14.8032GiB/s) | 20260508 04:59:53.107 | INFO | Downloaded 34GiB to GPU in 2.42767s (14.0052GiB/s) | 20260508 04:59:58.248 | INFO | Analyzing chunk frames = [5426176,5426688) | 20260508 04:59:58.248 | INFO | MemInfo: { | 20260508 04:59:58.248 | INFO | "gpu" : " 17 GB", | 20260508 04:59:58.248 | INFO | "huge" : "", | 20260508 04:59:58.248 | INFO | "huge_shared" : " 29 GB", | 20260508 04:59:58.248 | INFO | "malloc" : " 14 GB", | 20260508 04:59:58.248 | INFO | "pinned" : " 29 GB" | 20260508 04:59:58.248 | INFO | } | 20260508 04:59:58.248 | INFO | Perf report for the last 5.14122 seconds: | 20260508 04:59:58.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 04:59:58.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.248 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 04:59:58.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.248 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 04:59:58.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 04:59:58.249 | INFO | Uploaded 36GiB to GPU in 2.41845s (14.8856GiB/s) | 20260508 04:59:58.249 | INFO | Downloaded 34GiB to GPU in 2.427s (14.0091GiB/s) | 20260508 05:00:03.389 | INFO | Analyzing chunk frames = [5426688,5427200) | 20260508 05:00:03.389 | INFO | MemInfo: { | 20260508 05:00:03.389 | INFO | "gpu" : " 17 GB", | 20260508 05:00:03.389 | INFO | "huge" : "", | 20260508 05:00:03.390 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:03.390 | INFO | "malloc" : " 14 GB", | 20260508 05:00:03.390 | INFO | "pinned" : " 29 GB" | 20260508 05:00:03.390 | INFO | } | 20260508 05:00:03.390 | INFO | Perf report for the last 5.14146 seconds: | 20260508 05:00:03.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:03.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.390 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:00:03.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.390 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260508 05:00:03.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:03.391 | INFO | Uploaded 36GiB to GPU in 2.43853s (14.763GiB/s) | 20260508 05:00:03.391 | INFO | Downloaded 34GiB to GPU in 2.43326s (13.973GiB/s) | 20260508 05:00:03.504 | INFO | Starting a round of Baz IO | 20260508 05:00:06.913 | INFO | Populating RTMetrics report | 20260508 05:00:06.914 | INFO | RTMetrics report populated | 20260508 05:00:06.914 | INFO | Opening tmp RTMetrics json file | 20260508 05:00:06.925 | INFO | Serializing RTMetrics to json | 20260508 05:00:06.927 | INFO | Serialization done, swapping tmp file now... | 20260508 05:00:06.927 | INFO | RTMetrics done | 20260508 05:00:06.988 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.484 seconds | 20260508 05:00:06.988 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.95% / 0.00% / 59.05% | 20260508 05:00:06.988 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.496777% / 3.97364e-06% / 5.5631e-05% | 20260508 05:00:06.988 | INFO | Baz IO overall: | 20260508 05:00:06.988 | INFO | BytesWritten : 2423703017160 | 20260508 05:00:06.988 | INFO | NumEvents : 1169731934448 | 20260508 05:00:08.535 | INFO | Analyzing chunk frames = [5427200,5427712) | 20260508 05:00:08.535 | INFO | MemInfo: { | 20260508 05:00:08.535 | INFO | "gpu" : " 17 GB", | 20260508 05:00:08.535 | INFO | "huge" : "", | 20260508 05:00:08.535 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:08.535 | INFO | "malloc" : " 14 GB", | 20260508 05:00:08.535 | INFO | "pinned" : " 29 GB" | 20260508 05:00:08.535 | INFO | } | 20260508 05:00:08.535 | INFO | Perf report for the last 5.14562 seconds: | 20260508 05:00:08.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:08.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:08.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:08.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:08.535 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:00:08.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:08.535 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 05:00:08.535 | INFO | PRE_HQ : 66.3%, 1.8, 61.9ms, 0 | 20260508 05:00:08.536 | INFO | BAZWRITER : 67.2%, 2.8, 97.6ms, 0 | 20260508 05:00:08.536 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:00:08.536 | INFO | Uploaded 36GiB to GPU in 2.55351s (14.0982GiB/s) | 20260508 05:00:08.536 | INFO | Downloaded 38GiB to GPU in 2.69282s (14.1116GiB/s) | 20260508 05:00:13.669 | INFO | Analyzing chunk frames = [5427712,5428224) | 20260508 05:00:13.669 | INFO | MemInfo: { | 20260508 05:00:13.669 | INFO | "gpu" : " 17 GB", | 20260508 05:00:13.669 | INFO | "huge" : "", | 20260508 05:00:13.669 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:13.669 | INFO | "malloc" : " 14 GB", | 20260508 05:00:13.670 | INFO | "pinned" : " 29 GB" | 20260508 05:00:13.670 | INFO | } | 20260508 05:00:13.670 | INFO | Perf report for the last 5.13424 seconds: | 20260508 05:00:13.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:13.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 05:00:13.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:00:13.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:13.670 | INFO | Uploaded 36GiB to GPU in 2.48506s (14.4866GiB/s) | 20260508 05:00:13.670 | INFO | Downloaded 34GiB to GPU in 2.44221s (13.9218GiB/s) | 20260508 05:00:18.808 | INFO | Analyzing chunk frames = [5428224,5428736) | 20260508 05:00:18.808 | INFO | MemInfo: { | 20260508 05:00:18.808 | INFO | "gpu" : " 17 GB", | 20260508 05:00:18.808 | INFO | "huge" : "", | 20260508 05:00:18.808 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:18.808 | INFO | "malloc" : " 14 GB", | 20260508 05:00:18.808 | INFO | "pinned" : " 29 GB" | 20260508 05:00:18.808 | INFO | } | 20260508 05:00:18.808 | INFO | Perf report for the last 5.13835 seconds: | 20260508 05:00:18.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:18.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.808 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:00:18.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.808 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:00:18.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:18.809 | INFO | Uploaded 36GiB to GPU in 2.52619s (14.2507GiB/s) | 20260508 05:00:18.809 | INFO | Downloaded 34GiB to GPU in 2.40649s (14.1285GiB/s) | 20260508 05:00:23.948 | INFO | Analyzing chunk frames = [5428736,5429248) | 20260508 05:00:23.948 | INFO | MemInfo: { | 20260508 05:00:23.948 | INFO | "gpu" : " 17 GB", | 20260508 05:00:23.948 | INFO | "huge" : "", | 20260508 05:00:23.948 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:23.948 | INFO | "malloc" : " 14 GB", | 20260508 05:00:23.948 | INFO | "pinned" : " 29 GB" | 20260508 05:00:23.948 | INFO | } | 20260508 05:00:23.948 | INFO | Perf report for the last 5.14026 seconds: | 20260508 05:00:23.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:23.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.948 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:00:23.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.948 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:00:23.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:23.949 | INFO | Uploaded 36GiB to GPU in 2.52847s (14.2379GiB/s) | 20260508 05:00:23.949 | INFO | Downloaded 34GiB to GPU in 2.3909s (14.2206GiB/s) | 20260508 05:00:29.084 | INFO | Analyzing chunk frames = [5429248,5429760) | 20260508 05:00:29.084 | INFO | MemInfo: { | 20260508 05:00:29.084 | INFO | "gpu" : " 17 GB", | 20260508 05:00:29.084 | INFO | "huge" : "", | 20260508 05:00:29.084 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:29.084 | INFO | "malloc" : " 14 GB", | 20260508 05:00:29.084 | INFO | "pinned" : " 29 GB" | 20260508 05:00:29.084 | INFO | } | 20260508 05:00:29.084 | INFO | Perf report for the last 5.13589 seconds: | 20260508 05:00:29.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:29.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 05:00:29.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:00:29.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:29.084 | INFO | Uploaded 36GiB to GPU in 2.38201s (15.1133GiB/s) | 20260508 05:00:29.085 | INFO | Downloaded 34GiB to GPU in 2.56911s (13.2341GiB/s) | 20260508 05:00:34.224 | INFO | Analyzing chunk frames = [5429760,5430272) | 20260508 05:00:34.224 | INFO | MemInfo: { | 20260508 05:00:34.224 | INFO | "gpu" : " 17 GB", | 20260508 05:00:34.225 | INFO | "huge" : "", | 20260508 05:00:34.225 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:34.225 | INFO | "malloc" : " 14 GB", | 20260508 05:00:34.225 | INFO | "pinned" : " 29 GB" | 20260508 05:00:34.225 | INFO | } | 20260508 05:00:34.225 | INFO | Perf report for the last 5.14058 seconds: | 20260508 05:00:34.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:34.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 05:00:34.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:00:34.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:34.225 | INFO | Uploaded 36GiB to GPU in 2.38883s (15.0701GiB/s) | 20260508 05:00:34.225 | INFO | Downloaded 34GiB to GPU in 2.44872s (13.8848GiB/s) | 20260508 05:00:39.359 | INFO | Analyzing chunk frames = [5430272,5430784) | 20260508 05:00:39.359 | INFO | MemInfo: { | 20260508 05:00:39.359 | INFO | "gpu" : " 17 GB", | 20260508 05:00:39.359 | INFO | "huge" : "", | 20260508 05:00:39.360 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:39.360 | INFO | "malloc" : " 14 GB", | 20260508 05:00:39.360 | INFO | "pinned" : " 29 GB" | 20260508 05:00:39.360 | INFO | } | 20260508 05:00:39.360 | INFO | Perf report for the last 5.1349 seconds: | 20260508 05:00:39.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:39.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:00:39.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:00:39.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:39.360 | INFO | Uploaded 36GiB to GPU in 2.38775s (15.0769GiB/s) | 20260508 05:00:39.361 | INFO | Downloaded 34GiB to GPU in 2.42232s (14.0361GiB/s) | 20260508 05:00:44.507 | INFO | Analyzing chunk frames = [5430784,5431296) | 20260508 05:00:44.507 | INFO | MemInfo: { | 20260508 05:00:44.507 | INFO | "gpu" : " 17 GB", | 20260508 05:00:44.507 | INFO | "huge" : "", | 20260508 05:00:44.508 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:44.508 | INFO | "malloc" : " 14 GB", | 20260508 05:00:44.508 | INFO | "pinned" : " 29 GB" | 20260508 05:00:44.508 | INFO | } | 20260508 05:00:44.508 | INFO | Perf report for the last 5.14798 seconds: | 20260508 05:00:44.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:44.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:00:44.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 05:00:44.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:44.508 | INFO | Uploaded 36GiB to GPU in 2.43569s (14.7802GiB/s) | 20260508 05:00:44.508 | INFO | Downloaded 34GiB to GPU in 2.4514s (13.8696GiB/s) | 20260508 05:00:44.508 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:00:44.508 | INFO | Total mem usage: 16.7 GiB | 20260508 05:00:44.508 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:00:44.508 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:00:44.508 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:00:44.508 | INFO | High Water Breakdown: | 20260508 05:00:44.508 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:00:44.508 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:00:44.508 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:00:44.508 | INFO | ------------------------------------------ | 20260508 05:00:44.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:00:44.508 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:00:44.508 | INFO | Total mem usage: 26.5 GiB | 20260508 05:00:44.508 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:00:44.508 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:00:44.509 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:00:44.509 | INFO | High Water Breakdown: | 20260508 05:00:44.509 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:00:44.509 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:00:44.509 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:00:44.509 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:00:44.509 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:00:44.509 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:00:44.509 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:00:44.509 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:00:44.509 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:00:44.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:00:44.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:00:44.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:00:44.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:00:44.509 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:00:44.509 | INFO | ------------------------------------------ | 20260508 05:00:44.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:00:44.509 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:00:44.509 | INFO | Total mem usage: 27.8 GiB | 20260508 05:00:44.509 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:00:44.509 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:00:44.509 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:00:44.509 | INFO | High Water Breakdown: | 20260508 05:00:44.509 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:00:44.509 | INFO | ------------------------------------------ | 20260508 05:00:44.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:00:44.509 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:00:44.509 | INFO | Total mem usage: 15.7 GiB | 20260508 05:00:44.509 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:00:44.509 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:00:44.509 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:00:44.509 | INFO | High Water Breakdown: | 20260508 05:00:44.509 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:00:44.509 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:00:44.509 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:00:44.509 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:00:44.509 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:00:44.509 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:00:44.509 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:00:44.509 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:00:44.509 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:00:44.509 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:00:44.509 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:00:44.509 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:00:44.510 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:00:44.510 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:00:44.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:00:44.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:00:44.510 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:00:44.510 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:00:44.510 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:00:44.510 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:00:44.510 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:00:44.510 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:00:44.510 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:00:44.510 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:00:44.510 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:00:44.510 | INFO | ------------------------------------------ | 20260508 05:00:44.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:00:44.510 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:00:44.510 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:00:44.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:00:44.510 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 2.14e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.06e-01, 1.23e-02 / 4.63e-03 | 20260508 05:00:44.510 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.42e+00, 5.88e-01 / 2.23e-01 | 20260508 05:00:44.510 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.11e-01 | 20260508 05:00:44.510 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.58e+01, 3.91e+00 / 2.47e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.76e+00 / 1.35e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.21e-01 / 2.23e-01 | 20260508 05:00:44.510 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.22e+00, 2.62e-01 / 1.14e-01 | 20260508 05:00:44.510 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.22e-01 / 3.57e-01, 5.20e-02 / 2.79e-02 | 20260508 05:00:44.510 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:00:44.510 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:00:44.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:00:44.510 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 24.21%, Upload -- 2.57e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 30.85%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 2.82%, LockDownload -- 2.99e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 15.68%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:00:44.510 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260508 05:00:44.622 | INFO | Starting a round of Baz IO | 20260508 05:00:48.048 | INFO | Populating RTMetrics report | 20260508 05:00:48.048 | INFO | RTMetrics report populated | 20260508 05:00:48.048 | INFO | Opening tmp RTMetrics json file | 20260508 05:00:48.059 | INFO | Serializing RTMetrics to json | 20260508 05:00:48.061 | INFO | Serialization done, swapping tmp file now... | 20260508 05:00:48.061 | INFO | RTMetrics done | 20260508 05:00:48.116 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.494 seconds | 20260508 05:00:48.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.95% / 0.00% / 59.05% | 20260508 05:00:48.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.494202% / 1.19209e-05% / 7.54992e-05% | 20260508 05:00:48.116 | INFO | Baz IO overall: | 20260508 05:00:48.116 | INFO | BytesWritten : 2425403588580 | 20260508 05:00:48.116 | INFO | NumEvents : 1170543473235 | 20260508 05:00:49.643 | INFO | Analyzing chunk frames = [5431296,5431808) | 20260508 05:00:49.643 | INFO | MemInfo: { | 20260508 05:00:49.643 | INFO | "gpu" : " 17 GB", | 20260508 05:00:49.643 | INFO | "huge" : "", | 20260508 05:00:49.643 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:49.643 | INFO | "malloc" : " 14 GB", | 20260508 05:00:49.643 | INFO | "pinned" : " 29 GB" | 20260508 05:00:49.643 | INFO | } | 20260508 05:00:49.643 | INFO | Perf report for the last 5.13531 seconds: | 20260508 05:00:49.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:49.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:49.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:49.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:49.643 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 05:00:49.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:49.643 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 05:00:49.643 | INFO | PRE_HQ : 66.5%, 1.8, 62.1ms, 0 | 20260508 05:00:49.643 | INFO | BAZWRITER : 67.5%, 2.8, 97.2ms, 0 | 20260508 05:00:49.643 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:00:49.643 | INFO | Uploaded 36GiB to GPU in 2.5746s (13.9828GiB/s) | 20260508 05:00:49.643 | INFO | Downloaded 38GiB to GPU in 2.69997s (14.0742GiB/s) | 20260508 05:00:54.782 | INFO | Analyzing chunk frames = [5431808,5432320) | 20260508 05:00:54.782 | INFO | MemInfo: { | 20260508 05:00:54.782 | INFO | "gpu" : " 17 GB", | 20260508 05:00:54.782 | INFO | "huge" : "", | 20260508 05:00:54.782 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:54.782 | INFO | "malloc" : " 14 GB", | 20260508 05:00:54.782 | INFO | "pinned" : " 29 GB" | 20260508 05:00:54.782 | INFO | } | 20260508 05:00:54.782 | INFO | Perf report for the last 5.13911 seconds: | 20260508 05:00:54.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:54.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.782 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:00:54.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.782 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:00:54.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:54.783 | INFO | Uploaded 36GiB to GPU in 2.48925s (14.4622GiB/s) | 20260508 05:00:54.783 | INFO | Downloaded 34GiB to GPU in 2.43803s (13.9457GiB/s) | 20260508 05:00:59.917 | INFO | Analyzing chunk frames = [5432320,5432832) | 20260508 05:00:59.917 | INFO | MemInfo: { | 20260508 05:00:59.917 | INFO | "gpu" : " 17 GB", | 20260508 05:00:59.918 | INFO | "huge" : "", | 20260508 05:00:59.918 | INFO | "huge_shared" : " 29 GB", | 20260508 05:00:59.918 | INFO | "malloc" : " 14 GB", | 20260508 05:00:59.918 | INFO | "pinned" : " 29 GB" | 20260508 05:00:59.918 | INFO | } | 20260508 05:00:59.918 | INFO | Perf report for the last 5.13558 seconds: | 20260508 05:00:59.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:00:59.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 05:00:59.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:00:59.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:00:59.918 | INFO | Uploaded 36GiB to GPU in 2.54521s (14.1442GiB/s) | 20260508 05:00:59.918 | INFO | Downloaded 34GiB to GPU in 2.43122s (13.9847GiB/s) | 20260508 05:01:05.059 | INFO | Analyzing chunk frames = [5432832,5433344) | 20260508 05:01:05.060 | INFO | MemInfo: { | 20260508 05:01:05.060 | INFO | "gpu" : " 17 GB", | 20260508 05:01:05.060 | INFO | "huge" : "", | 20260508 05:01:05.060 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:05.060 | INFO | "malloc" : " 14 GB", | 20260508 05:01:05.060 | INFO | "pinned" : " 29 GB" | 20260508 05:01:05.060 | INFO | } | 20260508 05:01:05.060 | INFO | Perf report for the last 5.14189 seconds: | 20260508 05:01:05.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:05.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 05:01:05.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:01:05.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:05.060 | INFO | Uploaded 36GiB to GPU in 2.57902s (13.9588GiB/s) | 20260508 05:01:05.060 | INFO | Downloaded 34GiB to GPU in 2.42207s (14.0376GiB/s) | 20260508 05:01:10.201 | INFO | Analyzing chunk frames = [5433344,5433856) | 20260508 05:01:10.201 | INFO | MemInfo: { | 20260508 05:01:10.201 | INFO | "gpu" : " 17 GB", | 20260508 05:01:10.201 | INFO | "huge" : "", | 20260508 05:01:10.201 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:10.201 | INFO | "malloc" : " 14 GB", | 20260508 05:01:10.202 | INFO | "pinned" : " 29 GB" | 20260508 05:01:10.202 | INFO | } | 20260508 05:01:10.202 | INFO | Perf report for the last 5.14201 seconds: | 20260508 05:01:10.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:10.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:01:10.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:01:10.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:10.202 | INFO | Uploaded 36GiB to GPU in 2.32185s (15.5049GiB/s) | 20260508 05:01:10.202 | INFO | Downloaded 34GiB to GPU in 2.47402s (13.7428GiB/s) | 20260508 05:01:15.344 | INFO | Analyzing chunk frames = [5433856,5434368) | 20260508 05:01:15.344 | INFO | MemInfo: { | 20260508 05:01:15.344 | INFO | "gpu" : " 17 GB", | 20260508 05:01:15.344 | INFO | "huge" : "", | 20260508 05:01:15.344 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:15.344 | INFO | "malloc" : " 14 GB", | 20260508 05:01:15.344 | INFO | "pinned" : " 29 GB" | 20260508 05:01:15.344 | INFO | } | 20260508 05:01:15.344 | INFO | Perf report for the last 5.14231 seconds: | 20260508 05:01:15.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:15.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 05:01:15.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:01:15.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:15.344 | INFO | Uploaded 36GiB to GPU in 2.45224s (14.6805GiB/s) | 20260508 05:01:15.344 | INFO | Downloaded 34GiB to GPU in 2.419s (14.0554GiB/s) | 20260508 05:01:20.483 | INFO | Analyzing chunk frames = [5434368,5434880) | 20260508 05:01:20.483 | INFO | MemInfo: { | 20260508 05:01:20.483 | INFO | "gpu" : " 17 GB", | 20260508 05:01:20.483 | INFO | "huge" : "", | 20260508 05:01:20.483 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:20.483 | INFO | "malloc" : " 14 GB", | 20260508 05:01:20.483 | INFO | "pinned" : " 29 GB" | 20260508 05:01:20.483 | INFO | } | 20260508 05:01:20.483 | INFO | Perf report for the last 5.13891 seconds: | 20260508 05:01:20.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:20.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:01:20.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 05:01:20.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:20.483 | INFO | Uploaded 36GiB to GPU in 2.43515s (14.7835GiB/s) | 20260508 05:01:20.483 | INFO | Downloaded 34GiB to GPU in 2.41668s (14.0689GiB/s) | 20260508 05:01:25.614 | INFO | Analyzing chunk frames = [5434880,5435392) | 20260508 05:01:25.614 | INFO | MemInfo: { | 20260508 05:01:25.614 | INFO | "gpu" : " 17 GB", | 20260508 05:01:25.614 | INFO | "huge" : "", | 20260508 05:01:25.614 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:25.614 | INFO | "malloc" : " 14 GB", | 20260508 05:01:25.614 | INFO | "pinned" : " 29 GB" | 20260508 05:01:25.614 | INFO | } | 20260508 05:01:25.614 | INFO | Perf report for the last 5.13132 seconds: | 20260508 05:01:25.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:25.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.614 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 05:01:25.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.615 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 05:01:25.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:25.615 | INFO | Uploaded 36GiB to GPU in 2.50006s (14.3997GiB/s) | 20260508 05:01:25.616 | INFO | Downloaded 34GiB to GPU in 2.43997s (13.9346GiB/s) | 20260508 05:01:25.728 | INFO | Starting a round of Baz IO | 20260508 05:01:29.146 | INFO | Populating RTMetrics report | 20260508 05:01:29.146 | INFO | RTMetrics report populated | 20260508 05:01:29.146 | INFO | Opening tmp RTMetrics json file | 20260508 05:01:29.148 | INFO | Serializing RTMetrics to json | 20260508 05:01:29.162 | INFO | Serialization done, swapping tmp file now... | 20260508 05:01:29.164 | INFO | RTMetrics done | 20260508 05:01:29.220 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.492 seconds | 20260508 05:01:29.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.95% / 0.00% / 59.05% | 20260508 05:01:29.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493924% / 0% / 6.35783e-05% | 20260508 05:01:29.220 | INFO | Baz IO overall: | 20260508 05:01:29.220 | INFO | BytesWritten : 2427104037576 | 20260508 05:01:29.220 | INFO | NumEvents : 1171354820094 | 20260508 05:01:30.757 | INFO | Analyzing chunk frames = [5435392,5435904) | 20260508 05:01:30.757 | INFO | MemInfo: { | 20260508 05:01:30.757 | INFO | "gpu" : " 17 GB", | 20260508 05:01:30.757 | INFO | "huge" : "", | 20260508 05:01:30.757 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:30.757 | INFO | "malloc" : " 14 GB", | 20260508 05:01:30.757 | INFO | "pinned" : " 29 GB" | 20260508 05:01:30.757 | INFO | } | 20260508 05:01:30.758 | INFO | Perf report for the last 5.1433 seconds: | 20260508 05:01:30.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:30.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:30.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:30.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:30.758 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 05:01:30.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:30.758 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260508 05:01:30.758 | INFO | PRE_HQ : 66.4%, 1.8, 60.9ms, 0 | 20260508 05:01:30.758 | INFO | BAZWRITER : 67.3%, 2.8, 95.6ms, 0 | 20260508 05:01:30.758 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:01:30.758 | INFO | Uploaded 36GiB to GPU in 2.57949s (13.9563GiB/s) | 20260508 05:01:30.758 | INFO | Downloaded 38GiB to GPU in 2.6893s (14.1301GiB/s) | 20260508 05:01:35.900 | INFO | Analyzing chunk frames = [5435904,5436416) | 20260508 05:01:35.900 | INFO | MemInfo: { | 20260508 05:01:35.900 | INFO | "gpu" : " 17 GB", | 20260508 05:01:35.900 | INFO | "huge" : "", | 20260508 05:01:35.900 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:35.900 | INFO | "malloc" : " 14 GB", | 20260508 05:01:35.900 | INFO | "pinned" : " 29 GB" | 20260508 05:01:35.900 | INFO | } | 20260508 05:01:35.900 | INFO | Perf report for the last 5.14269 seconds: | 20260508 05:01:35.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:35.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 05:01:35.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:01:35.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:35.900 | INFO | Uploaded 36GiB to GPU in 2.58191s (13.9432GiB/s) | 20260508 05:01:35.901 | INFO | Downloaded 34GiB to GPU in 2.43716s (13.9507GiB/s) | 20260508 05:01:41.035 | INFO | Analyzing chunk frames = [5436416,5436928) | 20260508 05:01:41.035 | INFO | MemInfo: { | 20260508 05:01:41.035 | INFO | "gpu" : " 17 GB", | 20260508 05:01:41.035 | INFO | "huge" : "", | 20260508 05:01:41.035 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:41.035 | INFO | "malloc" : " 14 GB", | 20260508 05:01:41.035 | INFO | "pinned" : " 29 GB" | 20260508 05:01:41.035 | INFO | } | 20260508 05:01:41.035 | INFO | Perf report for the last 5.13488 seconds: | 20260508 05:01:41.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:41.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 05:01:41.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:01:41.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:41.035 | INFO | Uploaded 36GiB to GPU in 2.55455s (14.0925GiB/s) | 20260508 05:01:41.036 | INFO | Downloaded 34GiB to GPU in 2.41028s (14.1062GiB/s) | 20260508 05:01:46.172 | INFO | Analyzing chunk frames = [5436928,5437440) | 20260508 05:01:46.172 | INFO | MemInfo: { | 20260508 05:01:46.172 | INFO | "gpu" : " 17 GB", | 20260508 05:01:46.172 | INFO | "huge" : "", | 20260508 05:01:46.172 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:46.172 | INFO | "malloc" : " 14 GB", | 20260508 05:01:46.172 | INFO | "pinned" : " 29 GB" | 20260508 05:01:46.172 | INFO | } | 20260508 05:01:46.172 | INFO | Perf report for the last 5.13671 seconds: | 20260508 05:01:46.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:46.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | ANALYSIS : 67.2%, 2.9, 103.8ms, 0 | 20260508 05:01:46.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:01:46.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:46.172 | INFO | Uploaded 36GiB to GPU in 2.59923s (13.8502GiB/s) | 20260508 05:01:46.172 | INFO | Downloaded 34GiB to GPU in 2.41242s (14.0937GiB/s) | 20260508 05:01:51.316 | INFO | Analyzing chunk frames = [5437440,5437952) | 20260508 05:01:51.316 | INFO | MemInfo: { | 20260508 05:01:51.316 | INFO | "gpu" : " 17 GB", | 20260508 05:01:51.316 | INFO | "huge" : "", | 20260508 05:01:51.316 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:51.316 | INFO | "malloc" : " 14 GB", | 20260508 05:01:51.316 | INFO | "pinned" : " 29 GB" | 20260508 05:01:51.316 | INFO | } | 20260508 05:01:51.316 | INFO | Perf report for the last 5.14402 seconds: | 20260508 05:01:51.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:51.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:01:51.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:01:51.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:51.316 | INFO | Uploaded 36GiB to GPU in 2.40094s (14.9942GiB/s) | 20260508 05:01:51.316 | INFO | Downloaded 34GiB to GPU in 2.54903s (13.3384GiB/s) | 20260508 05:01:56.454 | INFO | Analyzing chunk frames = [5437952,5438464) | 20260508 05:01:56.454 | INFO | MemInfo: { | 20260508 05:01:56.454 | INFO | "gpu" : " 17 GB", | 20260508 05:01:56.454 | INFO | "huge" : "", | 20260508 05:01:56.454 | INFO | "huge_shared" : " 29 GB", | 20260508 05:01:56.454 | INFO | "malloc" : " 14 GB", | 20260508 05:01:56.454 | INFO | "pinned" : " 29 GB" | 20260508 05:01:56.454 | INFO | } | 20260508 05:01:56.454 | INFO | Perf report for the last 5.13809 seconds: | 20260508 05:01:56.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:01:56.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:01:56.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 05:01:56.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:01:56.454 | INFO | Uploaded 36GiB to GPU in 2.44598s (14.718GiB/s) | 20260508 05:01:56.454 | INFO | Downloaded 34GiB to GPU in 2.41299s (14.0904GiB/s) | 20260508 05:02:01.596 | INFO | Analyzing chunk frames = [5438464,5438976) | 20260508 05:02:01.596 | INFO | MemInfo: { | 20260508 05:02:01.596 | INFO | "gpu" : " 17 GB", | 20260508 05:02:01.596 | INFO | "huge" : "", | 20260508 05:02:01.596 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:01.596 | INFO | "malloc" : " 14 GB", | 20260508 05:02:01.596 | INFO | "pinned" : " 29 GB" | 20260508 05:02:01.596 | INFO | } | 20260508 05:02:01.596 | INFO | Perf report for the last 5.14234 seconds: | 20260508 05:02:01.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:01.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.596 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 05:02:01.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.597 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 05:02:01.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:01.597 | INFO | Uploaded 36GiB to GPU in 2.42663s (14.8354GiB/s) | 20260508 05:02:01.597 | INFO | Downloaded 34GiB to GPU in 2.43413s (13.968GiB/s) | 20260508 05:02:06.731 | INFO | Analyzing chunk frames = [5438976,5439488) | 20260508 05:02:06.731 | INFO | MemInfo: { | 20260508 05:02:06.731 | INFO | "gpu" : " 17 GB", | 20260508 05:02:06.731 | INFO | "huge" : "", | 20260508 05:02:06.731 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:06.731 | INFO | "malloc" : " 14 GB", | 20260508 05:02:06.731 | INFO | "pinned" : " 29 GB" | 20260508 05:02:06.731 | INFO | } | 20260508 05:02:06.731 | INFO | Perf report for the last 5.13503 seconds: | 20260508 05:02:06.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:06.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.731 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:02:06.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.731 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 05:02:06.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:06.732 | INFO | Uploaded 36GiB to GPU in 2.45429s (14.6682GiB/s) | 20260508 05:02:06.732 | INFO | Downloaded 34GiB to GPU in 2.43154s (13.9829GiB/s) | 20260508 05:02:06.732 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:02:06.732 | INFO | Total mem usage: 16.7 GiB | 20260508 05:02:06.732 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:02:06.732 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:02:06.732 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:02:06.732 | INFO | High Water Breakdown: | 20260508 05:02:06.732 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:02:06.732 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:02:06.732 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:02:06.732 | INFO | ------------------------------------------ | 20260508 05:02:06.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:02:06.732 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:02:06.732 | INFO | Total mem usage: 26.5 GiB | 20260508 05:02:06.732 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:02:06.732 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:02:06.732 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:02:06.732 | INFO | High Water Breakdown: | 20260508 05:02:06.732 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:02:06.732 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:02:06.732 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:02:06.732 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:02:06.732 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:02:06.732 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:02:06.732 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:02:06.732 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:02:06.732 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:02:06.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:02:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:02:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:02:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:02:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:02:06.733 | INFO | ------------------------------------------ | 20260508 05:02:06.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:02:06.733 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:02:06.733 | INFO | Total mem usage: 27.8 GiB | 20260508 05:02:06.733 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:02:06.733 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:02:06.733 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:02:06.733 | INFO | High Water Breakdown: | 20260508 05:02:06.733 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:02:06.733 | INFO | ------------------------------------------ | 20260508 05:02:06.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:02:06.733 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:02:06.733 | INFO | Total mem usage: 15.7 GiB | 20260508 05:02:06.733 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:02:06.733 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:02:06.733 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:02:06.733 | INFO | High Water Breakdown: | 20260508 05:02:06.733 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:02:06.733 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:02:06.733 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:02:06.733 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:02:06.733 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:02:06.733 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:02:06.733 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:02:06.733 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:02:06.733 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:02:06.733 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:02:06.733 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:02:06.733 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:02:06.733 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:02:06.733 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:02:06.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:02:06.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:02:06.733 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:02:06.733 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:02:06.733 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:02:06.733 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:02:06.733 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:02:06.733 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:02:06.733 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:02:06.733 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:02:06.733 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:02:06.733 | INFO | ------------------------------------------ | 20260508 05:02:06.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:02:06.733 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:02:06.733 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:02:06.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:02:06.733 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.42e+01 / 1.22e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.04e-01, 1.09e-02 / 5.20e-03 | 20260508 05:02:06.733 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.44e+00, 5.93e-01 / 2.28e-01 | 20260508 05:02:06.733 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.13e-01 | 20260508 05:02:06.733 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.51e+01, 3.60e+00 / 1.54e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.80e+00 / 1.29e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.62e+00, 4.13e-01 / 2.30e-01 | 20260508 05:02:06.733 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.23e+00, 2.61e-01 / 1.51e-01 | 20260508 05:02:06.733 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.57e-01, 5.13e-02 / 2.46e-02 | 20260508 05:02:06.733 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:02:06.733 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:02:06.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:02:06.733 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 30.81%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 2.35%, LockUpload -- 2.50e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 2.51%, LockDownload -- 2.67e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 15.66%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:02:06.733 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260508 05:02:06.847 | INFO | Starting a round of Baz IO | 20260508 05:02:10.260 | INFO | Populating RTMetrics report | 20260508 05:02:10.260 | INFO | RTMetrics report populated | 20260508 05:02:10.260 | INFO | Opening tmp RTMetrics json file | 20260508 05:02:10.272 | INFO | Serializing RTMetrics to json | 20260508 05:02:10.273 | INFO | Serialization done, swapping tmp file now... | 20260508 05:02:10.274 | INFO | RTMetrics done | 20260508 05:02:10.331 | INFO | Finished a round of Baz IO: Wrote 1.583 GiB in 3.483 seconds | 20260508 05:02:10.331 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.95% / 0.00% / 59.05% | 20260508 05:02:10.331 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493073% / 3.97364e-06% / 7.15256e-05% | 20260508 05:02:10.331 | INFO | Baz IO overall: | 20260508 05:02:10.331 | INFO | BytesWritten : 2428803994169 | 20260508 05:02:10.331 | INFO | NumEvents : 1172166199160 | 20260508 05:02:11.869 | INFO | Analyzing chunk frames = [5439488,5440000) | 20260508 05:02:11.869 | INFO | MemInfo: { | 20260508 05:02:11.869 | INFO | "gpu" : " 17 GB", | 20260508 05:02:11.869 | INFO | "huge" : "", | 20260508 05:02:11.869 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:11.869 | INFO | "malloc" : " 14 GB", | 20260508 05:02:11.869 | INFO | "pinned" : " 29 GB" | 20260508 05:02:11.869 | INFO | } | 20260508 05:02:11.869 | INFO | Perf report for the last 5.13732 seconds: | 20260508 05:02:11.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:11.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:11.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:11.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:11.869 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 05:02:11.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:11.869 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:02:11.869 | INFO | PRE_HQ : 66.4%, 1.8, 62.1ms, 0 | 20260508 05:02:11.869 | INFO | BAZWRITER : 67.1%, 2.8, 97.0ms, 0 | 20260508 05:02:11.869 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:02:11.869 | INFO | Uploaded 36GiB to GPU in 2.54567s (14.1416GiB/s) | 20260508 05:02:11.869 | INFO | Downloaded 38GiB to GPU in 2.71533s (13.9946GiB/s) | 20260508 05:02:17.010 | INFO | Analyzing chunk frames = [5440000,5440512) | 20260508 05:02:17.010 | INFO | MemInfo: { | 20260508 05:02:17.010 | INFO | "gpu" : " 17 GB", | 20260508 05:02:17.010 | INFO | "huge" : "", | 20260508 05:02:17.010 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:17.010 | INFO | "malloc" : " 14 GB", | 20260508 05:02:17.010 | INFO | "pinned" : " 29 GB" | 20260508 05:02:17.010 | INFO | } | 20260508 05:02:17.010 | INFO | Perf report for the last 5.14148 seconds: | 20260508 05:02:17.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:17.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.010 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 05:02:17.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.010 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:02:17.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:17.011 | INFO | Uploaded 36GiB to GPU in 2.46431s (14.6085GiB/s) | 20260508 05:02:17.011 | INFO | Downloaded 34GiB to GPU in 2.4517s (13.8679GiB/s) | 20260508 05:02:22.154 | INFO | Analyzing chunk frames = [5440512,5441024) | 20260508 05:02:22.154 | INFO | MemInfo: { | 20260508 05:02:22.154 | INFO | "gpu" : " 17 GB", | 20260508 05:02:22.154 | INFO | "huge" : "", | 20260508 05:02:22.154 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:22.155 | INFO | "malloc" : " 14 GB", | 20260508 05:02:22.155 | INFO | "pinned" : " 29 GB" | 20260508 05:02:22.155 | INFO | } | 20260508 05:02:22.155 | INFO | Perf report for the last 5.14438 seconds: | 20260508 05:02:22.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:22.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 05:02:22.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:02:22.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:22.155 | INFO | Uploaded 36GiB to GPU in 2.49556s (14.4256GiB/s) | 20260508 05:02:22.155 | INFO | Downloaded 34GiB to GPU in 2.4273s (14.0073GiB/s) | 20260508 05:02:27.293 | INFO | Analyzing chunk frames = [5441024,5441536) | 20260508 05:02:27.293 | INFO | MemInfo: { | 20260508 05:02:27.293 | INFO | "gpu" : " 17 GB", | 20260508 05:02:27.293 | INFO | "huge" : "", | 20260508 05:02:27.293 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:27.293 | INFO | "malloc" : " 14 GB", | 20260508 05:02:27.293 | INFO | "pinned" : " 29 GB" | 20260508 05:02:27.293 | INFO | } | 20260508 05:02:27.293 | INFO | Perf report for the last 5.13855 seconds: | 20260508 05:02:27.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:27.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.293 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:02:27.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.293 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:02:27.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:27.294 | INFO | Uploaded 36GiB to GPU in 2.52938s (14.2327GiB/s) | 20260508 05:02:27.294 | INFO | Downloaded 34GiB to GPU in 2.41452s (14.0815GiB/s) | 20260508 05:02:32.434 | INFO | Analyzing chunk frames = [5441536,5442048) | 20260508 05:02:32.434 | INFO | MemInfo: { | 20260508 05:02:32.434 | INFO | "gpu" : " 17 GB", | 20260508 05:02:32.434 | INFO | "huge" : "", | 20260508 05:02:32.434 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:32.434 | INFO | "malloc" : " 14 GB", | 20260508 05:02:32.434 | INFO | "pinned" : " 29 GB" | 20260508 05:02:32.434 | INFO | } | 20260508 05:02:32.434 | INFO | Perf report for the last 5.14099 seconds: | 20260508 05:02:32.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:32.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 05:02:32.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:02:32.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:32.434 | INFO | Uploaded 36GiB to GPU in 2.41921s (14.8809GiB/s) | 20260508 05:02:32.435 | INFO | Downloaded 34GiB to GPU in 2.55085s (13.3289GiB/s) | 20260508 05:02:37.572 | INFO | Analyzing chunk frames = [5442048,5442560) | 20260508 05:02:37.572 | INFO | MemInfo: { | 20260508 05:02:37.572 | INFO | "gpu" : " 17 GB", | 20260508 05:02:37.572 | INFO | "huge" : "", | 20260508 05:02:37.572 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:37.573 | INFO | "malloc" : " 14 GB", | 20260508 05:02:37.573 | INFO | "pinned" : " 29 GB" | 20260508 05:02:37.573 | INFO | } | 20260508 05:02:37.573 | INFO | Perf report for the last 5.1385 seconds: | 20260508 05:02:37.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:37.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:02:37.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:02:37.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:37.573 | INFO | Uploaded 36GiB to GPU in 2.42105s (14.8696GiB/s) | 20260508 05:02:37.573 | INFO | Downloaded 34GiB to GPU in 2.44443s (13.9092GiB/s) | 20260508 05:02:42.712 | INFO | Analyzing chunk frames = [5442560,5443072) | 20260508 05:02:42.712 | INFO | MemInfo: { | 20260508 05:02:42.712 | INFO | "gpu" : " 17 GB", | 20260508 05:02:42.712 | INFO | "huge" : "", | 20260508 05:02:42.712 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:42.713 | INFO | "malloc" : " 14 GB", | 20260508 05:02:42.713 | INFO | "pinned" : " 29 GB" | 20260508 05:02:42.713 | INFO | } | 20260508 05:02:42.713 | INFO | Perf report for the last 5.14011 seconds: | 20260508 05:02:42.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:42.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.713 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:02:42.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.713 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:02:42.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:42.714 | INFO | Uploaded 36GiB to GPU in 2.41676s (14.896GiB/s) | 20260508 05:02:42.714 | INFO | Downloaded 34GiB to GPU in 2.4488s (13.8843GiB/s) | 20260508 05:02:47.849 | INFO | Analyzing chunk frames = [5443072,5443584) | 20260508 05:02:47.849 | INFO | MemInfo: { | 20260508 05:02:47.849 | INFO | "gpu" : " 17 GB", | 20260508 05:02:47.849 | INFO | "huge" : "", | 20260508 05:02:47.849 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:47.849 | INFO | "malloc" : " 14 GB", | 20260508 05:02:47.849 | INFO | "pinned" : " 29 GB" | 20260508 05:02:47.849 | INFO | } | 20260508 05:02:47.849 | INFO | Perf report for the last 5.13642 seconds: | 20260508 05:02:47.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:47.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.849 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:02:47.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.849 | INFO | PRE_HQ : 34.2%, 1.0, 17.7ms, 0 | 20260508 05:02:47.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:47.850 | INFO | Uploaded 36GiB to GPU in 2.43288s (14.7973GiB/s) | 20260508 05:02:47.850 | INFO | Downloaded 34GiB to GPU in 2.44343s (13.9149GiB/s) | 20260508 05:02:47.962 | INFO | Starting a round of Baz IO | 20260508 05:02:51.372 | INFO | Populating RTMetrics report | 20260508 05:02:51.373 | INFO | RTMetrics report populated | 20260508 05:02:51.373 | INFO | Opening tmp RTMetrics json file | 20260508 05:02:51.376 | INFO | Serializing RTMetrics to json | 20260508 05:02:51.389 | INFO | Serialization done, swapping tmp file now... | 20260508 05:02:51.389 | INFO | RTMetrics done | 20260508 05:02:51.448 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.486 seconds | 20260508 05:02:51.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.94% / 0.00% / 59.05% | 20260508 05:02:51.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.494154% / 0% / 9.13938e-05% | 20260508 05:02:51.449 | INFO | Baz IO overall: | 20260508 05:02:51.449 | INFO | BytesWritten : 2430504568069 | 20260508 05:02:51.449 | INFO | NumEvents : 1172977982504 | 20260508 05:02:52.983 | INFO | Analyzing chunk frames = [5443584,5444096) | 20260508 05:02:52.983 | INFO | MemInfo: { | 20260508 05:02:52.983 | INFO | "gpu" : " 17 GB", | 20260508 05:02:52.983 | INFO | "huge" : "", | 20260508 05:02:52.983 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:52.984 | INFO | "malloc" : " 14 GB", | 20260508 05:02:52.984 | INFO | "pinned" : " 29 GB" | 20260508 05:02:52.984 | INFO | } | 20260508 05:02:52.984 | INFO | Perf report for the last 5.13439 seconds: | 20260508 05:02:52.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:52.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:52.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:52.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:52.984 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 05:02:52.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:52.984 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 05:02:52.984 | INFO | PRE_HQ : 66.6%, 1.8, 61.1ms, 0 | 20260508 05:02:52.984 | INFO | BAZWRITER : 67.4%, 2.8, 96.8ms, 0 | 20260508 05:02:52.984 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:02:52.984 | INFO | Uploaded 36GiB to GPU in 2.53276s (14.2138GiB/s) | 20260508 05:02:52.984 | INFO | Downloaded 38GiB to GPU in 2.69635s (14.0931GiB/s) | 20260508 05:02:58.121 | INFO | Analyzing chunk frames = [5444096,5444608) | 20260508 05:02:58.122 | INFO | MemInfo: { | 20260508 05:02:58.122 | INFO | "gpu" : " 17 GB", | 20260508 05:02:58.122 | INFO | "huge" : "", | 20260508 05:02:58.122 | INFO | "huge_shared" : " 29 GB", | 20260508 05:02:58.122 | INFO | "malloc" : " 14 GB", | 20260508 05:02:58.122 | INFO | "pinned" : " 29 GB" | 20260508 05:02:58.122 | INFO | } | 20260508 05:02:58.122 | INFO | Perf report for the last 5.13815 seconds: | 20260508 05:02:58.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:02:58.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 05:02:58.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:02:58.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:02:58.122 | INFO | Uploaded 36GiB to GPU in 2.57183s (13.9978GiB/s) | 20260508 05:02:58.122 | INFO | Downloaded 34GiB to GPU in 2.44406s (13.9113GiB/s) | 20260508 05:03:03.271 | INFO | Analyzing chunk frames = [5444608,5445120) | 20260508 05:03:03.271 | INFO | MemInfo: { | 20260508 05:03:03.271 | INFO | "gpu" : " 17 GB", | 20260508 05:03:03.271 | INFO | "huge" : "", | 20260508 05:03:03.271 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:03.271 | INFO | "malloc" : " 14 GB", | 20260508 05:03:03.271 | INFO | "pinned" : " 29 GB" | 20260508 05:03:03.272 | INFO | } | 20260508 05:03:03.272 | INFO | Perf report for the last 5.14988 seconds: | 20260508 05:03:03.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:03.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:03:03.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:03:03.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:03.272 | INFO | Uploaded 36GiB to GPU in 2.52994s (14.2296GiB/s) | 20260508 05:03:03.273 | INFO | Downloaded 34GiB to GPU in 2.42668s (14.0109GiB/s) | 20260508 05:03:08.405 | INFO | Analyzing chunk frames = [5445120,5445632) | 20260508 05:03:08.405 | INFO | MemInfo: { | 20260508 05:03:08.405 | INFO | "gpu" : " 17 GB", | 20260508 05:03:08.405 | INFO | "huge" : "", | 20260508 05:03:08.405 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:08.405 | INFO | "malloc" : " 14 GB", | 20260508 05:03:08.405 | INFO | "pinned" : " 29 GB" | 20260508 05:03:08.405 | INFO | } | 20260508 05:03:08.405 | INFO | Perf report for the last 5.13357 seconds: | 20260508 05:03:08.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:08.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.405 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 05:03:08.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.405 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:03:08.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:08.406 | INFO | Uploaded 36GiB to GPU in 2.54068s (14.1695GiB/s) | 20260508 05:03:08.406 | INFO | Downloaded 34GiB to GPU in 2.40738s (14.1232GiB/s) | 20260508 05:03:13.544 | INFO | Analyzing chunk frames = [5445632,5446144) | 20260508 05:03:13.544 | INFO | MemInfo: { | 20260508 05:03:13.544 | INFO | "gpu" : " 17 GB", | 20260508 05:03:13.544 | INFO | "huge" : "", | 20260508 05:03:13.544 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:13.544 | INFO | "malloc" : " 14 GB", | 20260508 05:03:13.544 | INFO | "pinned" : " 29 GB" | 20260508 05:03:13.544 | INFO | } | 20260508 05:03:13.544 | INFO | Perf report for the last 5.1389 seconds: | 20260508 05:03:13.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:13.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.544 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 05:03:13.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.544 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:03:13.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:13.545 | INFO | Uploaded 36GiB to GPU in 2.3837s (15.1026GiB/s) | 20260508 05:03:13.545 | INFO | Downloaded 34GiB to GPU in 2.54529s (13.358GiB/s) | 20260508 05:03:18.679 | INFO | Analyzing chunk frames = [5446144,5446656) | 20260508 05:03:18.679 | INFO | MemInfo: { | 20260508 05:03:18.679 | INFO | "gpu" : " 17 GB", | 20260508 05:03:18.679 | INFO | "huge" : "", | 20260508 05:03:18.679 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:18.679 | INFO | "malloc" : " 14 GB", | 20260508 05:03:18.679 | INFO | "pinned" : " 29 GB" | 20260508 05:03:18.679 | INFO | } | 20260508 05:03:18.679 | INFO | Perf report for the last 5.13477 seconds: | 20260508 05:03:18.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:18.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:03:18.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:03:18.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:18.679 | INFO | Uploaded 36GiB to GPU in 2.44916s (14.6989GiB/s) | 20260508 05:03:18.679 | INFO | Downloaded 34GiB to GPU in 2.42266s (14.0341GiB/s) | 20260508 05:03:23.821 | INFO | Analyzing chunk frames = [5446656,5447168) | 20260508 05:03:23.821 | INFO | MemInfo: { | 20260508 05:03:23.821 | INFO | "gpu" : " 17 GB", | 20260508 05:03:23.821 | INFO | "huge" : "", | 20260508 05:03:23.821 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:23.821 | INFO | "malloc" : " 14 GB", | 20260508 05:03:23.821 | INFO | "pinned" : " 29 GB" | 20260508 05:03:23.821 | INFO | } | 20260508 05:03:23.821 | INFO | Perf report for the last 5.14174 seconds: | 20260508 05:03:23.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:23.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:03:23.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 05:03:23.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:23.821 | INFO | Uploaded 36GiB to GPU in 2.45251s (14.6789GiB/s) | 20260508 05:03:23.822 | INFO | Downloaded 34GiB to GPU in 2.40954s (14.1106GiB/s) | 20260508 05:03:28.961 | INFO | Analyzing chunk frames = [5447168,5447680) | 20260508 05:03:28.961 | INFO | MemInfo: { | 20260508 05:03:28.961 | INFO | "gpu" : " 17 GB", | 20260508 05:03:28.961 | INFO | "huge" : "", | 20260508 05:03:28.961 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:28.961 | INFO | "malloc" : " 14 GB", | 20260508 05:03:28.961 | INFO | "pinned" : " 29 GB" | 20260508 05:03:28.961 | INFO | } | 20260508 05:03:28.961 | INFO | Perf report for the last 5.14037 seconds: | 20260508 05:03:28.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:28.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 05:03:28.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 05:03:28.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:28.961 | INFO | Uploaded 36GiB to GPU in 2.46315s (14.6155GiB/s) | 20260508 05:03:28.962 | INFO | Downloaded 34GiB to GPU in 2.42808s (14.0029GiB/s) | 20260508 05:03:28.962 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:03:28.962 | INFO | Total mem usage: 16.7 GiB | 20260508 05:03:28.962 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:03:28.962 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:03:28.962 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:03:28.962 | INFO | High Water Breakdown: | 20260508 05:03:28.962 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:03:28.962 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:03:28.962 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:03:28.962 | INFO | ------------------------------------------ | 20260508 05:03:28.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:03:28.962 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:03:28.962 | INFO | Total mem usage: 26.5 GiB | 20260508 05:03:28.962 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:03:28.962 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:03:28.962 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:03:28.962 | INFO | High Water Breakdown: | 20260508 05:03:28.962 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:03:28.962 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:03:28.962 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:03:28.962 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:03:28.962 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:03:28.962 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:03:28.962 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:03:28.962 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:03:28.962 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:03:28.962 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:03:28.962 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:03:28.962 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:03:28.962 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:03:28.962 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:03:28.962 | INFO | ------------------------------------------ | 20260508 05:03:28.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:03:28.962 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:03:28.962 | INFO | Total mem usage: 27.8 GiB | 20260508 05:03:28.962 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:03:28.962 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:03:28.962 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:03:28.962 | INFO | High Water Breakdown: | 20260508 05:03:28.962 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:03:28.962 | INFO | ------------------------------------------ | 20260508 05:03:28.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:03:28.962 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:03:28.962 | INFO | Total mem usage: 15.7 GiB | 20260508 05:03:28.962 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:03:28.962 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:03:28.962 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:03:28.962 | INFO | High Water Breakdown: | 20260508 05:03:28.962 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:03:28.963 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:03:28.963 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:03:28.963 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:03:28.963 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:03:28.963 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:03:28.963 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:03:28.963 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:03:28.963 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:03:28.963 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:03:28.963 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:03:28.963 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:03:28.963 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:03:28.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:03:28.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:03:28.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:03:28.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:03:28.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:03:28.963 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:03:28.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:03:28.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:03:28.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:03:28.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:03:28.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:03:28.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:03:28.963 | INFO | ------------------------------------------ | 20260508 05:03:28.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:03:28.963 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:03:28.963 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:03:28.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:03:28.963 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 1.06e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.14e-02 / 5.95e-03 | 20260508 05:03:28.963 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.52e+00, 5.83e-01 / 1.78e-01 | 20260508 05:03:28.963 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.36e-01 | 20260508 05:03:28.963 | INFO | PerfReport: 15.87%, DME -- 4.16e+01 / 4.47e+01, 3.68e+00 / 1.43e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.54e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.58e+00, 4.16e-01 / 2.65e-01 | 20260508 05:03:28.963 | INFO | PerfReport: 3.19%, Metrics -- 1.05e+00 / 1.23e+00, 2.61e-01 / 1.44e-01 | 20260508 05:03:28.963 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.54e-01, 5.08e-02 / 2.88e-02 | 20260508 05:03:28.963 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:03:28.963 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:03:28.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:03:28.963 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 24.30%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 24.32%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 2.77%, LockDownload -- 2.94e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 15.67%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:03:28.963 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.73e-02 / 0.00e+00 | 20260508 05:03:29.070 | INFO | Starting a round of Baz IO | 20260508 05:03:32.494 | INFO | Populating RTMetrics report | 20260508 05:03:32.494 | INFO | RTMetrics report populated | 20260508 05:03:32.494 | INFO | Opening tmp RTMetrics json file | 20260508 05:03:32.495 | INFO | Serializing RTMetrics to json | 20260508 05:03:32.496 | INFO | Serialization done, swapping tmp file now... | 20260508 05:03:32.496 | INFO | RTMetrics done | 20260508 05:03:32.566 | INFO | Finished a round of Baz IO: Wrote 1.584 GiB in 3.496 seconds | 20260508 05:03:32.566 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.94% / 0.00% / 59.06% | 20260508 05:03:32.566 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.492831% / 1.19209e-05% / 5.16574e-05% | 20260508 05:03:32.566 | INFO | Baz IO overall: | 20260508 05:03:32.566 | INFO | BytesWritten : 2432205095425 | 20260508 05:03:32.566 | INFO | NumEvents : 1173789804117 | 20260508 05:03:34.108 | INFO | Analyzing chunk frames = [5447680,5448192) | 20260508 05:03:34.108 | INFO | MemInfo: { | 20260508 05:03:34.108 | INFO | "gpu" : " 17 GB", | 20260508 05:03:34.108 | INFO | "huge" : "", | 20260508 05:03:34.108 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:34.108 | INFO | "malloc" : " 14 GB", | 20260508 05:03:34.108 | INFO | "pinned" : " 29 GB" | 20260508 05:03:34.108 | INFO | } | 20260508 05:03:34.109 | INFO | Perf report for the last 5.14753 seconds: | 20260508 05:03:34.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:34.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:34.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:34.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:34.109 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:03:34.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:34.109 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 05:03:34.109 | INFO | PRE_HQ : 66.3%, 1.8, 62.5ms, 0 | 20260508 05:03:34.109 | INFO | BAZWRITER : 67.3%, 2.9, 99.1ms, 0 | 20260508 05:03:34.109 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 05:03:34.109 | INFO | Uploaded 36GiB to GPU in 2.56443s (14.0382GiB/s) | 20260508 05:03:34.109 | INFO | Downloaded 38GiB to GPU in 2.70345s (14.0561GiB/s) | 20260508 05:03:39.243 | INFO | Analyzing chunk frames = [5448192,5448704) | 20260508 05:03:39.243 | INFO | MemInfo: { | 20260508 05:03:39.243 | INFO | "gpu" : " 17 GB", | 20260508 05:03:39.243 | INFO | "huge" : "", | 20260508 05:03:39.243 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:39.243 | INFO | "malloc" : " 14 GB", | 20260508 05:03:39.243 | INFO | "pinned" : " 29 GB" | 20260508 05:03:39.243 | INFO | } | 20260508 05:03:39.243 | INFO | Perf report for the last 5.13427 seconds: | 20260508 05:03:39.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:39.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.243 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 05:03:39.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.243 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:03:39.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:39.244 | INFO | Uploaded 36GiB to GPU in 2.49191s (14.4467GiB/s) | 20260508 05:03:39.244 | INFO | Downloaded 34GiB to GPU in 2.44138s (13.9266GiB/s) | 20260508 05:03:44.383 | INFO | Analyzing chunk frames = [5448704,5449216) | 20260508 05:03:44.383 | INFO | MemInfo: { | 20260508 05:03:44.383 | INFO | "gpu" : " 17 GB", | 20260508 05:03:44.383 | INFO | "huge" : "", | 20260508 05:03:44.383 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:44.383 | INFO | "malloc" : " 14 GB", | 20260508 05:03:44.383 | INFO | "pinned" : " 29 GB" | 20260508 05:03:44.383 | INFO | } | 20260508 05:03:44.383 | INFO | Perf report for the last 5.14019 seconds: | 20260508 05:03:44.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:44.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:03:44.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:03:44.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:44.383 | INFO | Uploaded 36GiB to GPU in 2.52864s (14.2369GiB/s) | 20260508 05:03:44.384 | INFO | Downloaded 34GiB to GPU in 2.41671s (14.0687GiB/s) | 20260508 05:03:49.522 | INFO | Analyzing chunk frames = [5449216,5449728) | 20260508 05:03:49.522 | INFO | MemInfo: { | 20260508 05:03:49.522 | INFO | "gpu" : " 17 GB", | 20260508 05:03:49.522 | INFO | "huge" : "", | 20260508 05:03:49.522 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:49.522 | INFO | "malloc" : " 14 GB", | 20260508 05:03:49.522 | INFO | "pinned" : " 29 GB" | 20260508 05:03:49.522 | INFO | } | 20260508 05:03:49.522 | INFO | Perf report for the last 5.13864 seconds: | 20260508 05:03:49.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:49.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:03:49.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:03:49.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:49.522 | INFO | Uploaded 36GiB to GPU in 2.54709s (14.1338GiB/s) | 20260508 05:03:49.522 | INFO | Downloaded 34GiB to GPU in 2.40551s (14.1342GiB/s) | 20260508 05:03:54.661 | INFO | Analyzing chunk frames = [5449728,5450240) | 20260508 05:03:54.661 | INFO | MemInfo: { | 20260508 05:03:54.661 | INFO | "gpu" : " 17 GB", | 20260508 05:03:54.661 | INFO | "huge" : "", | 20260508 05:03:54.661 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:54.661 | INFO | "malloc" : " 14 GB", | 20260508 05:03:54.661 | INFO | "pinned" : " 29 GB" | 20260508 05:03:54.661 | INFO | } | 20260508 05:03:54.661 | INFO | Perf report for the last 5.13899 seconds: | 20260508 05:03:54.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:54.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 05:03:54.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:03:54.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:54.661 | INFO | Uploaded 36GiB to GPU in 2.42022s (14.8747GiB/s) | 20260508 05:03:54.661 | INFO | Downloaded 34GiB to GPU in 2.52629s (13.4585GiB/s) | 20260508 05:03:59.799 | INFO | Analyzing chunk frames = [5450240,5450752) | 20260508 05:03:59.799 | INFO | MemInfo: { | 20260508 05:03:59.799 | INFO | "gpu" : " 17 GB", | 20260508 05:03:59.799 | INFO | "huge" : "", | 20260508 05:03:59.799 | INFO | "huge_shared" : " 29 GB", | 20260508 05:03:59.800 | INFO | "malloc" : " 14 GB", | 20260508 05:03:59.800 | INFO | "pinned" : " 29 GB" | 20260508 05:03:59.800 | INFO | } | 20260508 05:03:59.800 | INFO | Perf report for the last 5.13881 seconds: | 20260508 05:03:59.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:03:59.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:03:59.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:03:59.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:03:59.800 | INFO | Uploaded 36GiB to GPU in 2.42151s (14.8667GiB/s) | 20260508 05:03:59.800 | INFO | Downloaded 34GiB to GPU in 2.43236s (13.9782GiB/s) | 20260508 05:04:04.937 | INFO | Analyzing chunk frames = [5450752,5451264) | 20260508 05:04:04.937 | INFO | MemInfo: { | 20260508 05:04:04.937 | INFO | "gpu" : " 17 GB", | 20260508 05:04:04.937 | INFO | "huge" : "", | 20260508 05:04:04.937 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:04.937 | INFO | "malloc" : " 14 GB", | 20260508 05:04:04.937 | INFO | "pinned" : " 29 GB" | 20260508 05:04:04.937 | INFO | } | 20260508 05:04:04.937 | INFO | Perf report for the last 5.13744 seconds: | 20260508 05:04:04.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:04.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:04:04.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 05:04:04.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:04.937 | INFO | Uploaded 36GiB to GPU in 2.40937s (14.9417GiB/s) | 20260508 05:04:04.938 | INFO | Downloaded 34GiB to GPU in 2.43094s (13.9863GiB/s) | 20260508 05:04:10.074 | INFO | Analyzing chunk frames = [5451264,5451776) | 20260508 05:04:10.074 | INFO | MemInfo: { | 20260508 05:04:10.074 | INFO | "gpu" : " 17 GB", | 20260508 05:04:10.074 | INFO | "huge" : "", | 20260508 05:04:10.074 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:10.074 | INFO | "malloc" : " 14 GB", | 20260508 05:04:10.074 | INFO | "pinned" : " 29 GB" | 20260508 05:04:10.074 | INFO | } | 20260508 05:04:10.075 | INFO | Perf report for the last 5.13741 seconds: | 20260508 05:04:10.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:10.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:04:10.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 05:04:10.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:10.075 | INFO | Uploaded 36GiB to GPU in 2.4624s (14.6199GiB/s) | 20260508 05:04:10.076 | INFO | Downloaded 34GiB to GPU in 2.44799s (13.889GiB/s) | 20260508 05:04:10.188 | INFO | Starting a round of Baz IO | 20260508 05:04:13.609 | INFO | Populating RTMetrics report | 20260508 05:04:13.609 | INFO | RTMetrics report populated | 20260508 05:04:13.609 | INFO | Opening tmp RTMetrics json file | 20260508 05:04:13.620 | INFO | Serializing RTMetrics to json | 20260508 05:04:13.621 | INFO | Serialization done, swapping tmp file now... | 20260508 05:04:13.622 | INFO | RTMetrics done | 20260508 05:04:13.678 | INFO | Finished a round of Baz IO: Wrote 1.583 GiB in 3.489 seconds | 20260508 05:04:13.678 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.94% / 0.00% / 59.06% | 20260508 05:04:13.678 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.491635% / 7.94729e-06% / 5.5631e-05% | 20260508 05:04:13.678 | INFO | Baz IO overall: | 20260508 05:04:13.678 | INFO | BytesWritten : 2433904598023 | 20260508 05:04:13.678 | INFO | NumEvents : 1174601036617 | 20260508 05:04:15.215 | INFO | Analyzing chunk frames = [5451776,5452288) | 20260508 05:04:15.215 | INFO | MemInfo: { | 20260508 05:04:15.215 | INFO | "gpu" : " 17 GB", | 20260508 05:04:15.215 | INFO | "huge" : "", | 20260508 05:04:15.215 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:15.215 | INFO | "malloc" : " 14 GB", | 20260508 05:04:15.215 | INFO | "pinned" : " 29 GB" | 20260508 05:04:15.215 | INFO | } | 20260508 05:04:15.215 | INFO | Perf report for the last 5.14022 seconds: | 20260508 05:04:15.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:15.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:15.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:15.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:15.215 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:04:15.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:15.215 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 05:04:15.215 | INFO | PRE_HQ : 66.2%, 1.8, 61.6ms, 0 | 20260508 05:04:15.215 | INFO | BAZWRITER : 67.3%, 2.8, 97.3ms, 0 | 20260508 05:04:15.215 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 05:04:15.215 | INFO | Uploaded 36GiB to GPU in 2.55019s (14.1166GiB/s) | 20260508 05:04:15.215 | INFO | Downloaded 38GiB to GPU in 2.68536s (14.1508GiB/s) | 20260508 05:04:20.360 | INFO | Analyzing chunk frames = [5452288,5452800) | 20260508 05:04:20.360 | INFO | MemInfo: { | 20260508 05:04:20.360 | INFO | "gpu" : " 17 GB", | 20260508 05:04:20.360 | INFO | "huge" : "", | 20260508 05:04:20.360 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:20.360 | INFO | "malloc" : " 14 GB", | 20260508 05:04:20.360 | INFO | "pinned" : " 29 GB" | 20260508 05:04:20.360 | INFO | } | 20260508 05:04:20.360 | INFO | Perf report for the last 5.14536 seconds: | 20260508 05:04:20.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:20.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 05:04:20.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:04:20.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:20.360 | INFO | Uploaded 36GiB to GPU in 2.57557s (13.9775GiB/s) | 20260508 05:04:20.360 | INFO | Downloaded 34GiB to GPU in 2.43505s (13.9628GiB/s) | 20260508 05:04:25.495 | INFO | Analyzing chunk frames = [5452800,5453312) | 20260508 05:04:25.495 | INFO | MemInfo: { | 20260508 05:04:25.495 | INFO | "gpu" : " 17 GB", | 20260508 05:04:25.495 | INFO | "huge" : "", | 20260508 05:04:25.495 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:25.495 | INFO | "malloc" : " 14 GB", | 20260508 05:04:25.495 | INFO | "pinned" : " 29 GB" | 20260508 05:04:25.495 | INFO | } | 20260508 05:04:25.495 | INFO | Perf report for the last 5.13466 seconds: | 20260508 05:04:25.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:25.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 05:04:25.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:04:25.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:25.495 | INFO | Uploaded 36GiB to GPU in 2.55192s (14.107GiB/s) | 20260508 05:04:25.496 | INFO | Downloaded 34GiB to GPU in 2.40028s (14.165GiB/s) | 20260508 05:04:30.631 | INFO | Analyzing chunk frames = [5453312,5453824) | 20260508 05:04:30.631 | INFO | MemInfo: { | 20260508 05:04:30.631 | INFO | "gpu" : " 17 GB", | 20260508 05:04:30.631 | INFO | "huge" : "", | 20260508 05:04:30.631 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:30.631 | INFO | "malloc" : " 14 GB", | 20260508 05:04:30.632 | INFO | "pinned" : " 29 GB" | 20260508 05:04:30.632 | INFO | } | 20260508 05:04:30.632 | INFO | Perf report for the last 5.1368 seconds: | 20260508 05:04:30.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:30.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 05:04:30.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:04:30.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:30.632 | INFO | Uploaded 36GiB to GPU in 2.57041s (14.0055GiB/s) | 20260508 05:04:30.632 | INFO | Downloaded 34GiB to GPU in 2.39312s (14.2074GiB/s) | 20260508 05:04:35.774 | INFO | Analyzing chunk frames = [5453824,5454336) | 20260508 05:04:35.774 | INFO | MemInfo: { | 20260508 05:04:35.774 | INFO | "gpu" : " 17 GB", | 20260508 05:04:35.774 | INFO | "huge" : "", | 20260508 05:04:35.774 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:35.774 | INFO | "malloc" : " 14 GB", | 20260508 05:04:35.774 | INFO | "pinned" : " 29 GB" | 20260508 05:04:35.774 | INFO | } | 20260508 05:04:35.775 | INFO | Perf report for the last 5.14287 seconds: | 20260508 05:04:35.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:35.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:04:35.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:04:35.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:35.775 | INFO | Uploaded 36GiB to GPU in 2.39221s (15.0488GiB/s) | 20260508 05:04:35.775 | INFO | Downloaded 34GiB to GPU in 2.53847s (13.3939GiB/s) | 20260508 05:04:40.916 | INFO | Analyzing chunk frames = [5454336,5454848) | 20260508 05:04:40.916 | INFO | MemInfo: { | 20260508 05:04:40.916 | INFO | "gpu" : " 17 GB", | 20260508 05:04:40.916 | INFO | "huge" : "", | 20260508 05:04:40.916 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:40.916 | INFO | "malloc" : " 14 GB", | 20260508 05:04:40.916 | INFO | "pinned" : " 29 GB" | 20260508 05:04:40.916 | INFO | } | 20260508 05:04:40.916 | INFO | Perf report for the last 5.1415 seconds: | 20260508 05:04:40.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:40.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 05:04:40.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:04:40.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:40.916 | INFO | Uploaded 36GiB to GPU in 2.43631s (14.7765GiB/s) | 20260508 05:04:40.916 | INFO | Downloaded 34GiB to GPU in 2.42354s (14.0291GiB/s) | 20260508 05:04:46.055 | INFO | Analyzing chunk frames = [5454848,5455360) | 20260508 05:04:46.055 | INFO | MemInfo: { | 20260508 05:04:46.055 | INFO | "gpu" : " 17 GB", | 20260508 05:04:46.055 | INFO | "huge" : "", | 20260508 05:04:46.055 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:46.055 | INFO | "malloc" : " 14 GB", | 20260508 05:04:46.055 | INFO | "pinned" : " 29 GB" | 20260508 05:04:46.055 | INFO | } | 20260508 05:04:46.055 | INFO | Perf report for the last 5.13943 seconds: | 20260508 05:04:46.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:46.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 05:04:46.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:04:46.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:46.056 | INFO | Uploaded 36GiB to GPU in 2.40656s (14.9591GiB/s) | 20260508 05:04:46.056 | INFO | Downloaded 34GiB to GPU in 2.42369s (14.0282GiB/s) | 20260508 05:04:51.191 | INFO | Analyzing chunk frames = [5455360,5455872) | 20260508 05:04:51.191 | INFO | MemInfo: { | 20260508 05:04:51.191 | INFO | "gpu" : " 17 GB", | 20260508 05:04:51.191 | INFO | "huge" : "", | 20260508 05:04:51.191 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:51.191 | INFO | "malloc" : " 14 GB", | 20260508 05:04:51.191 | INFO | "pinned" : " 29 GB" | 20260508 05:04:51.191 | INFO | } | 20260508 05:04:51.191 | INFO | Perf report for the last 5.13562 seconds: | 20260508 05:04:51.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:51.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 05:04:51.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 05:04:51.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:51.191 | INFO | Uploaded 36GiB to GPU in 2.45676s (14.6534GiB/s) | 20260508 05:04:51.191 | INFO | Downloaded 34GiB to GPU in 2.44125s (13.9273GiB/s) | 20260508 05:04:51.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:04:51.192 | INFO | Total mem usage: 16.7 GiB | 20260508 05:04:51.192 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:04:51.192 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:04:51.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:04:51.192 | INFO | High Water Breakdown: | 20260508 05:04:51.192 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:04:51.192 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:04:51.192 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:04:51.192 | INFO | ------------------------------------------ | 20260508 05:04:51.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:04:51.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:04:51.192 | INFO | Total mem usage: 26.5 GiB | 20260508 05:04:51.192 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:04:51.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:04:51.192 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:04:51.192 | INFO | High Water Breakdown: | 20260508 05:04:51.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:04:51.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:04:51.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:04:51.192 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:04:51.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:04:51.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:04:51.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:04:51.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:04:51.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:04:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:04:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:04:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:04:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:04:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:04:51.192 | INFO | ------------------------------------------ | 20260508 05:04:51.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:04:51.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:04:51.192 | INFO | Total mem usage: 27.8 GiB | 20260508 05:04:51.192 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:04:51.192 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:04:51.192 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:04:51.192 | INFO | High Water Breakdown: | 20260508 05:04:51.192 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:04:51.192 | INFO | ------------------------------------------ | 20260508 05:04:51.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:04:51.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:04:51.192 | INFO | Total mem usage: 15.7 GiB | 20260508 05:04:51.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:04:51.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:04:51.192 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:04:51.192 | INFO | High Water Breakdown: | 20260508 05:04:51.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:04:51.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:04:51.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:04:51.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:04:51.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:04:51.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:04:51.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:04:51.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:04:51.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:04:51.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:04:51.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:04:51.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:04:51.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:04:51.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:04:51.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:04:51.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:04:51.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:04:51.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:04:51.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:04:51.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:04:51.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:04:51.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:04:51.192 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:04:51.192 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:04:51.192 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:04:51.193 | INFO | ------------------------------------------ | 20260508 05:04:51.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:04:51.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:04:51.193 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:04:51.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:04:51.193 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.42e+01 / 1.22e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.13e-02 / 8.35e-03 | 20260508 05:04:51.193 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.46e+00, 5.92e-01 / 2.04e-01 | 20260508 05:04:51.193 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.20e-01 | 20260508 05:04:51.193 | INFO | PerfReport: 15.85%, DME -- 4.15e+01 / 4.48e+01, 3.68e+00 / 1.54e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.84e+00 / 1.35e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.57e+00, 4.16e-01 / 2.35e-01 | 20260508 05:04:51.193 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.25e+00, 2.62e-01 / 1.50e-01 | 20260508 05:04:51.193 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.56e-01, 5.05e-02 / 2.54e-02 | 20260508 05:04:51.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:04:51.193 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:04:51.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:04:51.193 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 24.24%, Download -- 2.57e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 2.28%, LockUpload -- 2.42e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 2.58%, LockDownload -- 2.74e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:04:51.193 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260508 05:04:51.289 | INFO | Starting a round of Baz IO | 20260508 05:04:54.718 | INFO | Populating RTMetrics report | 20260508 05:04:54.718 | INFO | RTMetrics report populated | 20260508 05:04:54.718 | INFO | Opening tmp RTMetrics json file | 20260508 05:04:54.731 | INFO | Serializing RTMetrics to json | 20260508 05:04:54.732 | INFO | Serialization done, swapping tmp file now... | 20260508 05:04:54.733 | INFO | RTMetrics done | 20260508 05:04:54.791 | INFO | Finished a round of Baz IO: Wrote 1.583 GiB in 3.502 seconds | 20260508 05:04:54.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.94% / 0.00% / 59.06% | 20260508 05:04:54.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.490053% / 3.97364e-06% / 5.5631e-05% | 20260508 05:04:54.791 | INFO | Baz IO overall: | 20260508 05:04:54.791 | INFO | BytesWritten : 2435603965523 | 20260508 05:04:54.791 | INFO | NumEvents : 1175412125941 | 20260508 05:04:56.335 | INFO | Analyzing chunk frames = [5455872,5456384) | 20260508 05:04:56.335 | INFO | MemInfo: { | 20260508 05:04:56.335 | INFO | "gpu" : " 17 GB", | 20260508 05:04:56.335 | INFO | "huge" : "", | 20260508 05:04:56.335 | INFO | "huge_shared" : " 29 GB", | 20260508 05:04:56.335 | INFO | "malloc" : " 14 GB", | 20260508 05:04:56.335 | INFO | "pinned" : " 29 GB" | 20260508 05:04:56.335 | INFO | } | 20260508 05:04:56.335 | INFO | Perf report for the last 5.14402 seconds: | 20260508 05:04:56.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:04:56.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:56.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:56.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:56.335 | INFO | ANALYSIS : 67.6%, 3.0, 105.9ms, 0 | 20260508 05:04:56.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:04:56.335 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:04:56.335 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 05:04:56.335 | INFO | BAZWRITER : 67.6%, 2.7, 92.2ms, 0 | 20260508 05:04:56.335 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 05:04:56.336 | INFO | Uploaded 36GiB to GPU in 2.51452s (14.3168GiB/s) | 20260508 05:04:56.336 | INFO | Downloaded 38GiB to GPU in 2.68895s (14.1319GiB/s) | 20260508 05:05:01.476 | INFO | Analyzing chunk frames = [5456384,5456896) | 20260508 05:05:01.476 | INFO | MemInfo: { | 20260508 05:05:01.476 | INFO | "gpu" : " 17 GB", | 20260508 05:05:01.476 | INFO | "huge" : "", | 20260508 05:05:01.476 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:01.476 | INFO | "malloc" : " 14 GB", | 20260508 05:05:01.476 | INFO | "pinned" : " 29 GB" | 20260508 05:05:01.476 | INFO | } | 20260508 05:05:01.476 | INFO | Perf report for the last 5.1412 seconds: | 20260508 05:05:01.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:01.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.476 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 05:05:01.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.476 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:05:01.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:01.477 | INFO | Uploaded 36GiB to GPU in 2.57675s (13.9711GiB/s) | 20260508 05:05:01.477 | INFO | Downloaded 34GiB to GPU in 2.44232s (13.9212GiB/s) | 20260508 05:05:06.608 | INFO | Analyzing chunk frames = [5456896,5457408) | 20260508 05:05:06.608 | INFO | MemInfo: { | 20260508 05:05:06.608 | INFO | "gpu" : " 17 GB", | 20260508 05:05:06.609 | INFO | "huge" : "", | 20260508 05:05:06.609 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:06.609 | INFO | "malloc" : " 14 GB", | 20260508 05:05:06.609 | INFO | "pinned" : " 29 GB" | 20260508 05:05:06.609 | INFO | } | 20260508 05:05:06.609 | INFO | Perf report for the last 5.13239 seconds: | 20260508 05:05:06.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:06.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 05:05:06.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:05:06.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:06.609 | INFO | Uploaded 36GiB to GPU in 2.52829s (14.2389GiB/s) | 20260508 05:05:06.609 | INFO | Downloaded 34GiB to GPU in 2.41385s (14.0854GiB/s) | 20260508 05:05:11.751 | INFO | Analyzing chunk frames = [5457408,5457920) | 20260508 05:05:11.751 | INFO | MemInfo: { | 20260508 05:05:11.751 | INFO | "gpu" : " 17 GB", | 20260508 05:05:11.751 | INFO | "huge" : "", | 20260508 05:05:11.751 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:11.751 | INFO | "malloc" : " 14 GB", | 20260508 05:05:11.751 | INFO | "pinned" : " 29 GB" | 20260508 05:05:11.751 | INFO | } | 20260508 05:05:11.751 | INFO | Perf report for the last 5.14252 seconds: | 20260508 05:05:11.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:11.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.751 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 05:05:11.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.751 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:05:11.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:11.752 | INFO | Uploaded 36GiB to GPU in 2.56582s (14.0306GiB/s) | 20260508 05:05:11.752 | INFO | Downloaded 34GiB to GPU in 2.4101s (14.1073GiB/s) | 20260508 05:05:16.886 | INFO | Analyzing chunk frames = [5457920,5458432) | 20260508 05:05:16.886 | INFO | MemInfo: { | 20260508 05:05:16.886 | INFO | "gpu" : " 17 GB", | 20260508 05:05:16.886 | INFO | "huge" : "", | 20260508 05:05:16.886 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:16.886 | INFO | "malloc" : " 14 GB", | 20260508 05:05:16.886 | INFO | "pinned" : " 29 GB" | 20260508 05:05:16.886 | INFO | } | 20260508 05:05:16.887 | INFO | Perf report for the last 5.13521 seconds: | 20260508 05:05:16.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:16.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:05:16.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:05:16.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:16.887 | INFO | Uploaded 36GiB to GPU in 2.4107s (14.9334GiB/s) | 20260508 05:05:16.887 | INFO | Downloaded 34GiB to GPU in 2.55828s (13.2902GiB/s) | 20260508 05:05:22.031 | INFO | Analyzing chunk frames = [5458432,5458944) | 20260508 05:05:22.031 | INFO | MemInfo: { | 20260508 05:05:22.031 | INFO | "gpu" : " 17 GB", | 20260508 05:05:22.031 | INFO | "huge" : "", | 20260508 05:05:22.031 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:22.031 | INFO | "malloc" : " 14 GB", | 20260508 05:05:22.031 | INFO | "pinned" : " 29 GB" | 20260508 05:05:22.031 | INFO | } | 20260508 05:05:22.031 | INFO | Perf report for the last 5.14432 seconds: | 20260508 05:05:22.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:22.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:05:22.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:05:22.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:22.031 | INFO | Uploaded 36GiB to GPU in 2.44111s (14.7474GiB/s) | 20260508 05:05:22.031 | INFO | Downloaded 34GiB to GPU in 2.427s (14.0091GiB/s) | 20260508 05:05:27.173 | INFO | Analyzing chunk frames = [5458944,5459456) | 20260508 05:05:27.173 | INFO | MemInfo: { | 20260508 05:05:27.173 | INFO | "gpu" : " 17 GB", | 20260508 05:05:27.173 | INFO | "huge" : "", | 20260508 05:05:27.173 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:27.173 | INFO | "malloc" : " 14 GB", | 20260508 05:05:27.173 | INFO | "pinned" : " 29 GB" | 20260508 05:05:27.173 | INFO | } | 20260508 05:05:27.173 | INFO | Perf report for the last 5.14233 seconds: | 20260508 05:05:27.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:27.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:05:27.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 05:05:27.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:27.173 | INFO | Uploaded 36GiB to GPU in 2.43442s (14.7879GiB/s) | 20260508 05:05:27.173 | INFO | Downloaded 34GiB to GPU in 2.42824s (14.0019GiB/s) | 20260508 05:05:32.309 | INFO | Analyzing chunk frames = [5459456,5459968) | 20260508 05:05:32.309 | INFO | MemInfo: { | 20260508 05:05:32.309 | INFO | "gpu" : " 17 GB", | 20260508 05:05:32.309 | INFO | "huge" : "", | 20260508 05:05:32.309 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:32.309 | INFO | "malloc" : " 14 GB", | 20260508 05:05:32.309 | INFO | "pinned" : " 29 GB" | 20260508 05:05:32.309 | INFO | } | 20260508 05:05:32.309 | INFO | Perf report for the last 5.13554 seconds: | 20260508 05:05:32.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:32.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:05:32.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:05:32.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:32.309 | INFO | Uploaded 36GiB to GPU in 2.45597s (14.6582GiB/s) | 20260508 05:05:32.309 | INFO | Downloaded 34GiB to GPU in 2.4385s (13.943GiB/s) | 20260508 05:05:32.407 | INFO | Starting a round of Baz IO | 20260508 05:05:35.821 | INFO | Populating RTMetrics report | 20260508 05:05:35.821 | INFO | RTMetrics report populated | 20260508 05:05:35.821 | INFO | Opening tmp RTMetrics json file | 20260508 05:05:35.832 | INFO | Serializing RTMetrics to json | 20260508 05:05:35.834 | INFO | Serialization done, swapping tmp file now... | 20260508 05:05:35.835 | INFO | RTMetrics done | 20260508 05:05:35.895 | INFO | Finished a round of Baz IO: Wrote 1.581 GiB in 3.488 seconds | 20260508 05:05:35.895 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.94% / 0.00% / 59.06% | 20260508 05:05:35.895 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.489342% / 0% / 5.96046e-05% | 20260508 05:05:35.895 | INFO | Baz IO overall: | 20260508 05:05:35.895 | INFO | BytesWritten : 2437301352315 | 20260508 05:05:35.895 | INFO | NumEvents : 1176221734095 | 20260508 05:05:37.441 | INFO | Analyzing chunk frames = [5459968,5460480) | 20260508 05:05:37.441 | INFO | MemInfo: { | 20260508 05:05:37.441 | INFO | "gpu" : " 17 GB", | 20260508 05:05:37.441 | INFO | "huge" : "", | 20260508 05:05:37.441 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:37.441 | INFO | "malloc" : " 14 GB", | 20260508 05:05:37.441 | INFO | "pinned" : " 29 GB" | 20260508 05:05:37.441 | INFO | } | 20260508 05:05:37.441 | INFO | Perf report for the last 5.13239 seconds: | 20260508 05:05:37.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:37.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:37.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:37.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:37.442 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 05:05:37.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:37.442 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:05:37.442 | INFO | PRE_HQ : 65.9%, 1.6, 55.4ms, 0 | 20260508 05:05:37.442 | INFO | BAZWRITER : 67.5%, 2.7, 94.7ms, 0 | 20260508 05:05:37.442 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:05:37.442 | INFO | Uploaded 36GiB to GPU in 2.53885s (14.1796GiB/s) | 20260508 05:05:37.442 | INFO | Downloaded 38GiB to GPU in 2.69188s (14.1165GiB/s) | 20260508 05:05:42.588 | INFO | Analyzing chunk frames = [5460480,5460992) | 20260508 05:05:42.588 | INFO | MemInfo: { | 20260508 05:05:42.588 | INFO | "gpu" : " 17 GB", | 20260508 05:05:42.588 | INFO | "huge" : "", | 20260508 05:05:42.588 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:42.588 | INFO | "malloc" : " 14 GB", | 20260508 05:05:42.588 | INFO | "pinned" : " 29 GB" | 20260508 05:05:42.588 | INFO | } | 20260508 05:05:42.589 | INFO | Perf report for the last 5.14737 seconds: | 20260508 05:05:42.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:42.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 05:05:42.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:05:42.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:42.589 | INFO | Uploaded 36GiB to GPU in 2.5762s (13.9741GiB/s) | 20260508 05:05:42.589 | INFO | Downloaded 34GiB to GPU in 2.44575s (13.9017GiB/s) | 20260508 05:05:47.727 | INFO | Analyzing chunk frames = [5460992,5461504) | 20260508 05:05:47.727 | INFO | MemInfo: { | 20260508 05:05:47.727 | INFO | "gpu" : " 17 GB", | 20260508 05:05:47.727 | INFO | "huge" : "", | 20260508 05:05:47.727 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:47.727 | INFO | "malloc" : " 14 GB", | 20260508 05:05:47.727 | INFO | "pinned" : " 29 GB" | 20260508 05:05:47.728 | INFO | } | 20260508 05:05:47.728 | INFO | Perf report for the last 5.139 seconds: | 20260508 05:05:47.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:47.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 05:05:47.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:05:47.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:47.728 | INFO | Uploaded 36GiB to GPU in 2.55606s (14.0842GiB/s) | 20260508 05:05:47.728 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0225GiB/s) | 20260508 05:05:52.872 | INFO | Analyzing chunk frames = [5461504,5462016) | 20260508 05:05:52.872 | INFO | MemInfo: { | 20260508 05:05:52.872 | INFO | "gpu" : " 17 GB", | 20260508 05:05:52.872 | INFO | "huge" : "", | 20260508 05:05:52.872 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:52.872 | INFO | "malloc" : " 14 GB", | 20260508 05:05:52.872 | INFO | "pinned" : " 29 GB" | 20260508 05:05:52.872 | INFO | } | 20260508 05:05:52.872 | INFO | Perf report for the last 5.1444 seconds: | 20260508 05:05:52.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:52.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.872 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 05:05:52.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.872 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:05:52.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:52.873 | INFO | Uploaded 36GiB to GPU in 2.58027s (13.9521GiB/s) | 20260508 05:05:52.873 | INFO | Downloaded 34GiB to GPU in 2.42305s (14.0319GiB/s) | 20260508 05:05:58.007 | INFO | Analyzing chunk frames = [5462016,5462528) | 20260508 05:05:58.007 | INFO | MemInfo: { | 20260508 05:05:58.007 | INFO | "gpu" : " 17 GB", | 20260508 05:05:58.007 | INFO | "huge" : "", | 20260508 05:05:58.007 | INFO | "huge_shared" : " 29 GB", | 20260508 05:05:58.007 | INFO | "malloc" : " 14 GB", | 20260508 05:05:58.007 | INFO | "pinned" : " 29 GB" | 20260508 05:05:58.007 | INFO | } | 20260508 05:05:58.008 | INFO | Perf report for the last 5.13553 seconds: | 20260508 05:05:58.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:05:58.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 05:05:58.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:05:58.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:05:58.008 | INFO | Uploaded 36GiB to GPU in 2.31251s (15.5675GiB/s) | 20260508 05:05:58.008 | INFO | Downloaded 34GiB to GPU in 2.47172s (13.7556GiB/s) | 20260508 05:06:03.143 | INFO | Analyzing chunk frames = [5462528,5463040) | 20260508 05:06:03.143 | INFO | MemInfo: { | 20260508 05:06:03.143 | INFO | "gpu" : " 17 GB", | 20260508 05:06:03.143 | INFO | "huge" : "", | 20260508 05:06:03.143 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:03.143 | INFO | "malloc" : " 14 GB", | 20260508 05:06:03.143 | INFO | "pinned" : " 29 GB" | 20260508 05:06:03.143 | INFO | } | 20260508 05:06:03.143 | INFO | Perf report for the last 5.1356 seconds: | 20260508 05:06:03.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:03.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:06:03.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:06:03.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:03.143 | INFO | Uploaded 36GiB to GPU in 2.43867s (14.7621GiB/s) | 20260508 05:06:03.143 | INFO | Downloaded 34GiB to GPU in 2.42663s (14.0112GiB/s) | 20260508 05:06:08.283 | INFO | Analyzing chunk frames = [5463040,5463552) | 20260508 05:06:08.283 | INFO | MemInfo: { | 20260508 05:06:08.283 | INFO | "gpu" : " 17 GB", | 20260508 05:06:08.283 | INFO | "huge" : "", | 20260508 05:06:08.283 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:08.283 | INFO | "malloc" : " 14 GB", | 20260508 05:06:08.283 | INFO | "pinned" : " 29 GB" | 20260508 05:06:08.283 | INFO | } | 20260508 05:06:08.283 | INFO | Perf report for the last 5.13976 seconds: | 20260508 05:06:08.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:08.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:06:08.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:06:08.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:08.283 | INFO | Uploaded 36GiB to GPU in 2.42251s (14.8606GiB/s) | 20260508 05:06:08.283 | INFO | Downloaded 34GiB to GPU in 2.42202s (14.0379GiB/s) | 20260508 05:06:13.425 | INFO | Analyzing chunk frames = [5463552,5464064) | 20260508 05:06:13.425 | INFO | MemInfo: { | 20260508 05:06:13.425 | INFO | "gpu" : " 17 GB", | 20260508 05:06:13.425 | INFO | "huge" : "", | 20260508 05:06:13.425 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:13.425 | INFO | "malloc" : " 14 GB", | 20260508 05:06:13.425 | INFO | "pinned" : " 29 GB" | 20260508 05:06:13.425 | INFO | } | 20260508 05:06:13.426 | INFO | Perf report for the last 5.14264 seconds: | 20260508 05:06:13.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:13.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:06:13.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:06:13.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:13.426 | INFO | Uploaded 36GiB to GPU in 2.43978s (14.7554GiB/s) | 20260508 05:06:13.426 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:06:13.426 | INFO | Total mem usage: 16.7 GiB | 20260508 05:06:13.426 | INFO | Downloaded 34GiB to GPU in 2.42348s (14.0294GiB/s) | 20260508 05:06:13.426 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:06:13.426 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 05:06:13.426 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:06:13.426 | INFO | High Water Breakdown: | 20260508 05:06:13.426 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:06:13.426 | INFO | 40.4% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:06:13.426 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:06:13.426 | INFO | ------------------------------------------ | 20260508 05:06:13.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:06:13.426 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:06:13.426 | INFO | Total mem usage: 26.5 GiB | 20260508 05:06:13.426 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:06:13.426 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 05:06:13.426 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 05:06:13.426 | INFO | High Water Breakdown: | 20260508 05:06:13.427 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:06:13.427 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:06:13.427 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:06:13.427 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:06:13.427 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:06:13.427 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:06:13.427 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:06:13.427 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:06:13.427 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:06:13.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:06:13.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:06:13.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:06:13.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:06:13.427 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:06:13.427 | INFO | ------------------------------------------ | 20260508 05:06:13.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:06:13.427 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:06:13.427 | INFO | Total mem usage: 27.8 GiB | 20260508 05:06:13.427 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:06:13.427 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:06:13.427 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:06:13.427 | INFO | High Water Breakdown: | 20260508 05:06:13.427 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:06:13.427 | INFO | ------------------------------------------ | 20260508 05:06:13.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:06:13.427 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:06:13.427 | INFO | Total mem usage: 15.7 GiB | 20260508 05:06:13.427 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:06:13.427 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:06:13.427 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:06:13.427 | INFO | High Water Breakdown: | 20260508 05:06:13.427 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:06:13.427 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:06:13.427 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:06:13.427 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:06:13.427 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:06:13.427 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:06:13.427 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:06:13.427 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:06:13.427 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:06:13.427 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:06:13.427 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:06:13.427 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:06:13.427 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:06:13.427 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:06:13.427 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:06:13.427 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:06:13.427 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:06:13.427 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:06:13.427 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:06:13.428 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:06:13.428 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:06:13.428 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:06:13.428 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:06:13.428 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:06:13.428 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:06:13.428 | INFO | ------------------------------------------ | 20260508 05:06:13.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:06:13.428 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:06:13.428 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:06:13.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:06:13.428 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 1.13e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.08e-02 / 4.10e-03 | 20260508 05:06:13.428 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.47e+00, 5.90e-01 / 2.20e-01 | 20260508 05:06:13.428 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.35e-01 | 20260508 05:06:13.428 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.52e+01, 3.81e+00 / 1.61e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.78e+00 / 1.58e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.50e+00, 4.18e-01 / 2.87e-01 | 20260508 05:06:13.428 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.23e+00, 2.62e-01 / 1.66e-01 | 20260508 05:06:13.428 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.51e-01, 5.09e-02 / 2.56e-02 | 20260508 05:06:13.428 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:06:13.428 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:06:13.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:06:13.428 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 24.23%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 2.20%, LockUpload -- 2.34e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 2.67%, LockDownload -- 2.83e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 15.72%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:06:13.428 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260508 05:06:13.523 | INFO | Starting a round of Baz IO | 20260508 05:06:16.952 | INFO | Populating RTMetrics report | 20260508 05:06:16.953 | INFO | RTMetrics report populated | 20260508 05:06:16.953 | INFO | Opening tmp RTMetrics json file | 20260508 05:06:16.965 | INFO | Serializing RTMetrics to json | 20260508 05:06:16.966 | INFO | Serialization done, swapping tmp file now... | 20260508 05:06:16.968 | INFO | RTMetrics done | 20260508 05:06:17.023 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.5 seconds | 20260508 05:06:17.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.93% / 0.00% / 59.06% | 20260508 05:06:17.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.486823% / 0% / 7.54992e-05% | 20260508 05:06:17.023 | INFO | Baz IO overall: | 20260508 05:06:17.023 | INFO | BytesWritten : 2438997652304 | 20260508 05:06:17.023 | INFO | NumEvents : 1177030924876 | 20260508 05:06:18.563 | INFO | Analyzing chunk frames = [5464064,5464576) | 20260508 05:06:18.563 | INFO | MemInfo: { | 20260508 05:06:18.563 | INFO | "gpu" : " 17 GB", | 20260508 05:06:18.563 | INFO | "huge" : "", | 20260508 05:06:18.563 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:18.563 | INFO | "malloc" : " 14 GB", | 20260508 05:06:18.563 | INFO | "pinned" : " 29 GB" | 20260508 05:06:18.563 | INFO | } | 20260508 05:06:18.563 | INFO | Perf report for the last 5.13718 seconds: | 20260508 05:06:18.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:18.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:18.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:18.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:18.563 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 05:06:18.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:18.563 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 05:06:18.563 | INFO | PRE_HQ : 66.0%, 1.6, 55.6ms, 0 | 20260508 05:06:18.563 | INFO | BAZWRITER : 67.6%, 2.6, 91.8ms, 0 | 20260508 05:06:18.563 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:06:18.563 | INFO | Uploaded 36GiB to GPU in 2.5135s (14.3227GiB/s) | 20260508 05:06:18.564 | INFO | Downloaded 38GiB to GPU in 2.69586s (14.0957GiB/s) | 20260508 05:06:23.697 | INFO | Analyzing chunk frames = [5464576,5465088) | 20260508 05:06:23.697 | INFO | MemInfo: { | 20260508 05:06:23.697 | INFO | "gpu" : " 17 GB", | 20260508 05:06:23.697 | INFO | "huge" : "", | 20260508 05:06:23.697 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:23.697 | INFO | "malloc" : " 14 GB", | 20260508 05:06:23.697 | INFO | "pinned" : " 29 GB" | 20260508 05:06:23.697 | INFO | } | 20260508 05:06:23.698 | INFO | Perf report for the last 5.13497 seconds: | 20260508 05:06:23.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:23.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 05:06:23.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:06:23.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:23.698 | INFO | Uploaded 36GiB to GPU in 2.56118s (14.056GiB/s) | 20260508 05:06:23.698 | INFO | Downloaded 34GiB to GPU in 2.45293s (13.861GiB/s) | 20260508 05:06:28.841 | INFO | Analyzing chunk frames = [5465088,5465600) | 20260508 05:06:28.841 | INFO | MemInfo: { | 20260508 05:06:28.841 | INFO | "gpu" : " 17 GB", | 20260508 05:06:28.841 | INFO | "huge" : "", | 20260508 05:06:28.841 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:28.841 | INFO | "malloc" : " 14 GB", | 20260508 05:06:28.841 | INFO | "pinned" : " 29 GB" | 20260508 05:06:28.841 | INFO | } | 20260508 05:06:28.841 | INFO | Perf report for the last 5.14323 seconds: | 20260508 05:06:28.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:28.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:06:28.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:06:28.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:28.841 | INFO | Uploaded 36GiB to GPU in 2.50075s (14.3957GiB/s) | 20260508 05:06:28.841 | INFO | Downloaded 34GiB to GPU in 2.43302s (13.9744GiB/s) | 20260508 05:06:33.980 | INFO | Analyzing chunk frames = [5465600,5466112) | 20260508 05:06:33.980 | INFO | MemInfo: { | 20260508 05:06:33.980 | INFO | "gpu" : " 17 GB", | 20260508 05:06:33.980 | INFO | "huge" : "", | 20260508 05:06:33.980 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:33.980 | INFO | "malloc" : " 14 GB", | 20260508 05:06:33.980 | INFO | "pinned" : " 29 GB" | 20260508 05:06:33.980 | INFO | } | 20260508 05:06:33.980 | INFO | Perf report for the last 5.13896 seconds: | 20260508 05:06:33.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:33.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 05:06:33.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 05:06:33.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:33.980 | INFO | Uploaded 36GiB to GPU in 2.53019s (14.2282GiB/s) | 20260508 05:06:33.981 | INFO | Downloaded 34GiB to GPU in 2.42877s (13.9988GiB/s) | 20260508 05:06:39.122 | INFO | Analyzing chunk frames = [5466112,5466624) | 20260508 05:06:39.122 | INFO | MemInfo: { | 20260508 05:06:39.122 | INFO | "gpu" : " 17 GB", | 20260508 05:06:39.123 | INFO | "huge" : "", | 20260508 05:06:39.123 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:39.123 | INFO | "malloc" : " 14 GB", | 20260508 05:06:39.123 | INFO | "pinned" : " 29 GB" | 20260508 05:06:39.123 | INFO | } | 20260508 05:06:39.123 | INFO | Perf report for the last 5.14279 seconds: | 20260508 05:06:39.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:39.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 05:06:39.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:06:39.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:39.123 | INFO | Uploaded 36GiB to GPU in 2.35612s (15.2794GiB/s) | 20260508 05:06:39.123 | INFO | Downloaded 34GiB to GPU in 2.57944s (13.1812GiB/s) | 20260508 05:06:44.262 | INFO | Analyzing chunk frames = [5466624,5467136) | 20260508 05:06:44.262 | INFO | MemInfo: { | 20260508 05:06:44.262 | INFO | "gpu" : " 17 GB", | 20260508 05:06:44.262 | INFO | "huge" : "", | 20260508 05:06:44.262 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:44.262 | INFO | "malloc" : " 14 GB", | 20260508 05:06:44.262 | INFO | "pinned" : " 29 GB" | 20260508 05:06:44.262 | INFO | } | 20260508 05:06:44.262 | INFO | Perf report for the last 5.13908 seconds: | 20260508 05:06:44.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:44.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:06:44.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:06:44.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:44.262 | INFO | Uploaded 36GiB to GPU in 2.42911s (14.8202GiB/s) | 20260508 05:06:44.262 | INFO | Downloaded 34GiB to GPU in 2.4294s (13.9952GiB/s) | 20260508 05:06:49.398 | INFO | Analyzing chunk frames = [5467136,5467648) | 20260508 05:06:49.398 | INFO | MemInfo: { | 20260508 05:06:49.398 | INFO | "gpu" : " 17 GB", | 20260508 05:06:49.398 | INFO | "huge" : "", | 20260508 05:06:49.398 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:49.398 | INFO | "malloc" : " 14 GB", | 20260508 05:06:49.398 | INFO | "pinned" : " 29 GB" | 20260508 05:06:49.398 | INFO | } | 20260508 05:06:49.398 | INFO | Perf report for the last 5.13617 seconds: | 20260508 05:06:49.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:49.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:06:49.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:06:49.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:49.398 | INFO | Uploaded 36GiB to GPU in 2.45133s (14.6859GiB/s) | 20260508 05:06:49.398 | INFO | Downloaded 34GiB to GPU in 2.44141s (13.9264GiB/s) | 20260508 05:06:54.537 | INFO | Analyzing chunk frames = [5467648,5468160) | 20260508 05:06:54.537 | INFO | MemInfo: { | 20260508 05:06:54.537 | INFO | "gpu" : " 17 GB", | 20260508 05:06:54.537 | INFO | "huge" : "", | 20260508 05:06:54.537 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:54.537 | INFO | "malloc" : " 14 GB", | 20260508 05:06:54.537 | INFO | "pinned" : " 29 GB" | 20260508 05:06:54.537 | INFO | } | 20260508 05:06:54.537 | INFO | Perf report for the last 5.13929 seconds: | 20260508 05:06:54.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:54.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.537 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:06:54.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.537 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:06:54.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:54.538 | INFO | Uploaded 36GiB to GPU in 2.43845s (14.7635GiB/s) | 20260508 05:06:54.538 | INFO | Downloaded 34GiB to GPU in 2.43044s (13.9892GiB/s) | 20260508 05:06:54.635 | INFO | Starting a round of Baz IO | 20260508 05:06:58.056 | INFO | Populating RTMetrics report | 20260508 05:06:58.056 | INFO | RTMetrics report populated | 20260508 05:06:58.056 | INFO | Opening tmp RTMetrics json file | 20260508 05:06:58.062 | INFO | Serializing RTMetrics to json | 20260508 05:06:58.072 | INFO | Serialization done, swapping tmp file now... | 20260508 05:06:58.073 | INFO | RTMetrics done | 20260508 05:06:58.132 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.497 seconds | 20260508 05:06:58.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.93% / 0.00% / 59.07% | 20260508 05:06:58.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.485746% / 0% / 5.96046e-05% | 20260508 05:06:58.132 | INFO | Baz IO overall: | 20260508 05:06:58.132 | INFO | BytesWritten : 2440694068860 | 20260508 05:06:58.132 | INFO | NumEvents : 1177840208472 | 20260508 05:06:59.677 | INFO | Analyzing chunk frames = [5468160,5468672) | 20260508 05:06:59.677 | INFO | MemInfo: { | 20260508 05:06:59.677 | INFO | "gpu" : " 17 GB", | 20260508 05:06:59.677 | INFO | "huge" : "", | 20260508 05:06:59.677 | INFO | "huge_shared" : " 29 GB", | 20260508 05:06:59.677 | INFO | "malloc" : " 14 GB", | 20260508 05:06:59.678 | INFO | "pinned" : " 29 GB" | 20260508 05:06:59.678 | INFO | } | 20260508 05:06:59.678 | INFO | Perf report for the last 5.14031 seconds: | 20260508 05:06:59.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:06:59.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:59.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:59.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:59.678 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 05:06:59.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:06:59.678 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 05:06:59.678 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 05:06:59.678 | INFO | BAZWRITER : 67.4%, 2.7, 92.6ms, 0 | 20260508 05:06:59.678 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:06:59.678 | INFO | Uploaded 36GiB to GPU in 2.55736s (14.077GiB/s) | 20260508 05:06:59.678 | INFO | Downloaded 38GiB to GPU in 2.70909s (14.0268GiB/s) | 20260508 05:07:04.817 | INFO | Analyzing chunk frames = [5468672,5469184) | 20260508 05:07:04.817 | INFO | MemInfo: { | 20260508 05:07:04.817 | INFO | "gpu" : " 17 GB", | 20260508 05:07:04.817 | INFO | "huge" : "", | 20260508 05:07:04.817 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:04.817 | INFO | "malloc" : " 14 GB", | 20260508 05:07:04.817 | INFO | "pinned" : " 29 GB" | 20260508 05:07:04.817 | INFO | } | 20260508 05:07:04.817 | INFO | Perf report for the last 5.13949 seconds: | 20260508 05:07:04.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:04.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 05:07:04.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:07:04.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:04.817 | INFO | Uploaded 36GiB to GPU in 2.56274s (14.0475GiB/s) | 20260508 05:07:04.818 | INFO | Downloaded 34GiB to GPU in 2.44864s (13.8853GiB/s) | 20260508 05:07:09.951 | INFO | Analyzing chunk frames = [5469184,5469696) | 20260508 05:07:09.951 | INFO | MemInfo: { | 20260508 05:07:09.951 | INFO | "gpu" : " 17 GB", | 20260508 05:07:09.951 | INFO | "huge" : "", | 20260508 05:07:09.951 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:09.951 | INFO | "malloc" : " 14 GB", | 20260508 05:07:09.952 | INFO | "pinned" : " 29 GB" | 20260508 05:07:09.952 | INFO | } | 20260508 05:07:09.952 | INFO | Perf report for the last 5.13456 seconds: | 20260508 05:07:09.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:09.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | ANALYSIS : 67.1%, 3.0, 104.0ms, 0 | 20260508 05:07:09.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:07:09.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:09.952 | INFO | Uploaded 36GiB to GPU in 2.53393s (14.2072GiB/s) | 20260508 05:07:09.952 | INFO | Downloaded 34GiB to GPU in 2.4218s (14.0392GiB/s) | 20260508 05:07:15.096 | INFO | Analyzing chunk frames = [5469696,5470208) | 20260508 05:07:15.096 | INFO | MemInfo: { | 20260508 05:07:15.096 | INFO | "gpu" : " 17 GB", | 20260508 05:07:15.096 | INFO | "huge" : "", | 20260508 05:07:15.096 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:15.096 | INFO | "malloc" : " 14 GB", | 20260508 05:07:15.096 | INFO | "pinned" : " 29 GB" | 20260508 05:07:15.096 | INFO | } | 20260508 05:07:15.097 | INFO | Perf report for the last 5.14477 seconds: | 20260508 05:07:15.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:15.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 05:07:15.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:07:15.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:15.097 | INFO | Uploaded 36GiB to GPU in 2.57105s (14.002GiB/s) | 20260508 05:07:15.097 | INFO | Downloaded 34GiB to GPU in 2.42185s (14.0388GiB/s) | 20260508 05:07:20.235 | INFO | Analyzing chunk frames = [5470208,5470720) | 20260508 05:07:20.235 | INFO | MemInfo: { | 20260508 05:07:20.235 | INFO | "gpu" : " 17 GB", | 20260508 05:07:20.235 | INFO | "huge" : "", | 20260508 05:07:20.235 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:20.235 | INFO | "malloc" : " 14 GB", | 20260508 05:07:20.235 | INFO | "pinned" : " 29 GB" | 20260508 05:07:20.235 | INFO | } | 20260508 05:07:20.235 | INFO | Perf report for the last 5.13896 seconds: | 20260508 05:07:20.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:20.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.236 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 05:07:20.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.236 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:07:20.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:20.236 | INFO | Uploaded 36GiB to GPU in 2.31412s (15.5567GiB/s) | 20260508 05:07:20.236 | INFO | Downloaded 34GiB to GPU in 2.56097s (13.2762GiB/s) | 20260508 05:07:25.378 | INFO | Analyzing chunk frames = [5470720,5471232) | 20260508 05:07:25.378 | INFO | MemInfo: { | 20260508 05:07:25.378 | INFO | "gpu" : " 17 GB", | 20260508 05:07:25.378 | INFO | "huge" : "", | 20260508 05:07:25.378 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:25.378 | INFO | "malloc" : " 14 GB", | 20260508 05:07:25.378 | INFO | "pinned" : " 29 GB" | 20260508 05:07:25.378 | INFO | } | 20260508 05:07:25.378 | INFO | Perf report for the last 5.14244 seconds: | 20260508 05:07:25.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:25.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:07:25.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:07:25.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:25.378 | INFO | Uploaded 36GiB to GPU in 2.43037s (14.8125GiB/s) | 20260508 05:07:25.378 | INFO | Downloaded 34GiB to GPU in 2.43702s (13.9514GiB/s) | 20260508 05:07:30.511 | INFO | Analyzing chunk frames = [5471232,5471744) | 20260508 05:07:30.511 | INFO | MemInfo: { | 20260508 05:07:30.511 | INFO | "gpu" : " 17 GB", | 20260508 05:07:30.511 | INFO | "huge" : "", | 20260508 05:07:30.511 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:30.511 | INFO | "malloc" : " 14 GB", | 20260508 05:07:30.511 | INFO | "pinned" : " 29 GB" | 20260508 05:07:30.511 | INFO | } | 20260508 05:07:30.511 | INFO | Perf report for the last 5.13309 seconds: | 20260508 05:07:30.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:30.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 05:07:30.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:07:30.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:30.511 | INFO | Uploaded 36GiB to GPU in 2.42471s (14.8471GiB/s) | 20260508 05:07:30.511 | INFO | Downloaded 34GiB to GPU in 2.43449s (13.966GiB/s) | 20260508 05:07:35.651 | INFO | Analyzing chunk frames = [5471744,5472256) | 20260508 05:07:35.651 | INFO | MemInfo: { | 20260508 05:07:35.651 | INFO | "gpu" : " 17 GB", | 20260508 05:07:35.651 | INFO | "huge" : "", | 20260508 05:07:35.651 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:35.651 | INFO | "malloc" : " 14 GB", | 20260508 05:07:35.651 | INFO | "pinned" : " 29 GB" | 20260508 05:07:35.651 | INFO | } | 20260508 05:07:35.651 | INFO | Perf report for the last 5.14045 seconds: | 20260508 05:07:35.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:35.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:07:35.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 05:07:35.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:35.652 | INFO | Uploaded 36GiB to GPU in 2.43064s (14.8109GiB/s) | 20260508 05:07:35.652 | INFO | Downloaded 34GiB to GPU in 2.43536s (13.961GiB/s) | 20260508 05:07:35.652 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:07:35.652 | INFO | Total mem usage: 16.7 GiB | 20260508 05:07:35.652 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:07:35.652 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 05:07:35.652 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:07:35.652 | INFO | High Water Breakdown: | 20260508 05:07:35.652 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:07:35.652 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:07:35.652 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:07:35.652 | INFO | ------------------------------------------ | 20260508 05:07:35.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:07:35.652 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:07:35.652 | INFO | Total mem usage: 26.5 GiB | 20260508 05:07:35.652 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:07:35.652 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:07:35.652 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:07:35.652 | INFO | High Water Breakdown: | 20260508 05:07:35.652 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:07:35.652 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:07:35.652 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:07:35.652 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:07:35.652 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:07:35.652 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:07:35.652 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:07:35.652 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:07:35.652 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:07:35.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:07:35.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:07:35.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:07:35.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:07:35.652 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:07:35.652 | INFO | ------------------------------------------ | 20260508 05:07:35.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:07:35.652 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:07:35.652 | INFO | Total mem usage: 27.8 GiB | 20260508 05:07:35.652 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:07:35.652 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:07:35.652 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:07:35.652 | INFO | High Water Breakdown: | 20260508 05:07:35.652 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:07:35.652 | INFO | ------------------------------------------ | 20260508 05:07:35.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:07:35.652 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:07:35.652 | INFO | Total mem usage: 15.7 GiB | 20260508 05:07:35.652 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:07:35.652 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:07:35.652 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:07:35.652 | INFO | High Water Breakdown: | 20260508 05:07:35.652 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:07:35.652 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:07:35.653 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:07:35.653 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:07:35.653 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:07:35.653 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:07:35.653 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:07:35.653 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:07:35.653 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:07:35.653 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:07:35.653 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:07:35.653 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:07:35.653 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:07:35.653 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:07:35.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:07:35.653 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:07:35.653 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:07:35.653 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:07:35.653 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:07:35.653 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:07:35.653 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:07:35.653 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:07:35.653 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:07:35.653 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:07:35.653 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:07:35.653 | INFO | ------------------------------------------ | 20260508 05:07:35.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:07:35.653 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:07:35.653 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:07:35.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:07:35.653 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 1.69e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.01e-01, 1.31e-02 / 4.56e-03 | 20260508 05:07:35.653 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.44e+00, 5.86e-01 / 1.77e-01 | 20260508 05:07:35.653 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.11e-01 / 1.18e-01 | 20260508 05:07:35.653 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.53e+01, 3.79e+00 / 1.97e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.38e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.14e-01 / 2.39e-01 | 20260508 05:07:35.653 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.19e+00, 2.61e-01 / 1.19e-01 | 20260508 05:07:35.653 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.52e-01, 5.09e-02 / 2.18e-02 | 20260508 05:07:35.653 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:07:35.653 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:07:35.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:07:35.653 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 24.28%, Upload -- 2.58e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 24.37%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 30.79%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 2.05%, LockUpload -- 2.18e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 2.86%, LockDownload -- 3.04e+00 / 0.00e+00, 4.36e+00 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:07:35.653 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.79e-02 / 0.00e+00 | 20260508 05:07:35.753 | INFO | Starting a round of Baz IO | 20260508 05:07:39.181 | INFO | Populating RTMetrics report | 20260508 05:07:39.181 | INFO | RTMetrics report populated | 20260508 05:07:39.181 | INFO | Opening tmp RTMetrics json file | 20260508 05:07:39.193 | INFO | Serializing RTMetrics to json | 20260508 05:07:39.195 | INFO | Serialization done, swapping tmp file now... | 20260508 05:07:39.195 | INFO | RTMetrics done | 20260508 05:07:39.255 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.502 seconds | 20260508 05:07:39.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.93% / 0.00% / 59.07% | 20260508 05:07:39.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.485758% / 1.19209e-05% / 5.96046e-05% | 20260508 05:07:39.256 | INFO | Baz IO overall: | 20260508 05:07:39.256 | INFO | BytesWritten : 2442390519005 | 20260508 05:07:39.256 | INFO | NumEvents : 1178649573602 | 20260508 05:07:40.790 | INFO | Analyzing chunk frames = [5472256,5472768) | 20260508 05:07:40.790 | INFO | MemInfo: { | 20260508 05:07:40.790 | INFO | "gpu" : " 17 GB", | 20260508 05:07:40.790 | INFO | "huge" : "", | 20260508 05:07:40.790 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:40.790 | INFO | "malloc" : " 14 GB", | 20260508 05:07:40.790 | INFO | "pinned" : " 29 GB" | 20260508 05:07:40.790 | INFO | } | 20260508 05:07:40.790 | INFO | Perf report for the last 5.13853 seconds: | 20260508 05:07:40.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:40.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:40.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:40.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:40.790 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 05:07:40.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:40.790 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 05:07:40.790 | INFO | PRE_HQ : 66.1%, 1.8, 60.4ms, 0 | 20260508 05:07:40.790 | INFO | BAZWRITER : 67.5%, 2.8, 96.2ms, 0 | 20260508 05:07:40.790 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:07:40.790 | INFO | Uploaded 36GiB to GPU in 2.58493s (13.9269GiB/s) | 20260508 05:07:40.790 | INFO | Downloaded 38GiB to GPU in 2.6874s (14.14GiB/s) | 20260508 05:07:45.932 | INFO | Analyzing chunk frames = [5472768,5473280) | 20260508 05:07:45.932 | INFO | MemInfo: { | 20260508 05:07:45.932 | INFO | "gpu" : " 17 GB", | 20260508 05:07:45.932 | INFO | "huge" : "", | 20260508 05:07:45.932 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:45.932 | INFO | "malloc" : " 14 GB", | 20260508 05:07:45.933 | INFO | "pinned" : " 29 GB" | 20260508 05:07:45.933 | INFO | } | 20260508 05:07:45.933 | INFO | Perf report for the last 5.14263 seconds: | 20260508 05:07:45.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:45.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 05:07:45.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:07:45.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:45.933 | INFO | Uploaded 36GiB to GPU in 2.57884s (13.9598GiB/s) | 20260508 05:07:45.933 | INFO | Downloaded 34GiB to GPU in 2.43295s (13.9748GiB/s) | 20260508 05:07:51.075 | INFO | Analyzing chunk frames = [5473280,5473792) | 20260508 05:07:51.075 | INFO | MemInfo: { | 20260508 05:07:51.075 | INFO | "gpu" : " 17 GB", | 20260508 05:07:51.075 | INFO | "huge" : "", | 20260508 05:07:51.075 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:51.075 | INFO | "malloc" : " 14 GB", | 20260508 05:07:51.075 | INFO | "pinned" : " 29 GB" | 20260508 05:07:51.075 | INFO | } | 20260508 05:07:51.075 | INFO | Perf report for the last 5.1426 seconds: | 20260508 05:07:51.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:51.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.075 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 05:07:51.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.075 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:07:51.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:51.076 | INFO | Uploaded 36GiB to GPU in 2.55199s (14.1067GiB/s) | 20260508 05:07:51.076 | INFO | Downloaded 34GiB to GPU in 2.40494s (14.1376GiB/s) | 20260508 05:07:56.205 | INFO | Analyzing chunk frames = [5473792,5474304) | 20260508 05:07:56.205 | INFO | MemInfo: { | 20260508 05:07:56.205 | INFO | "gpu" : " 17 GB", | 20260508 05:07:56.205 | INFO | "huge" : "", | 20260508 05:07:56.205 | INFO | "huge_shared" : " 29 GB", | 20260508 05:07:56.205 | INFO | "malloc" : " 14 GB", | 20260508 05:07:56.205 | INFO | "pinned" : " 29 GB" | 20260508 05:07:56.205 | INFO | } | 20260508 05:07:56.205 | INFO | Perf report for the last 5.12994 seconds: | 20260508 05:07:56.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:07:56.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.205 | INFO | ANALYSIS : 67.0%, 2.9, 103.4ms, 0 | 20260508 05:07:56.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.205 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:07:56.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:07:56.206 | INFO | Uploaded 36GiB to GPU in 2.5602s (14.0614GiB/s) | 20260508 05:07:56.206 | INFO | Downloaded 34GiB to GPU in 2.39684s (14.1853GiB/s) | 20260508 05:08:01.349 | INFO | Analyzing chunk frames = [5474304,5474816) | 20260508 05:08:01.349 | INFO | MemInfo: { | 20260508 05:08:01.349 | INFO | "gpu" : " 17 GB", | 20260508 05:08:01.349 | INFO | "huge" : "", | 20260508 05:08:01.349 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:01.349 | INFO | "malloc" : " 14 GB", | 20260508 05:08:01.349 | INFO | "pinned" : " 29 GB" | 20260508 05:08:01.349 | INFO | } | 20260508 05:08:01.349 | INFO | Perf report for the last 5.14365 seconds: | 20260508 05:08:01.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:01.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:08:01.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:08:01.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:01.349 | INFO | Uploaded 36GiB to GPU in 2.41185s (14.9263GiB/s) | 20260508 05:08:01.349 | INFO | Downloaded 34GiB to GPU in 2.56499s (13.2554GiB/s) | 20260508 05:08:06.487 | INFO | Analyzing chunk frames = [5474816,5475328) | 20260508 05:08:06.487 | INFO | MemInfo: { | 20260508 05:08:06.487 | INFO | "gpu" : " 17 GB", | 20260508 05:08:06.487 | INFO | "huge" : "", | 20260508 05:08:06.487 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:06.487 | INFO | "malloc" : " 14 GB", | 20260508 05:08:06.487 | INFO | "pinned" : " 29 GB" | 20260508 05:08:06.487 | INFO | } | 20260508 05:08:06.487 | INFO | Perf report for the last 5.13831 seconds: | 20260508 05:08:06.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:06.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.487 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:08:06.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.487 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:08:06.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:06.488 | INFO | Uploaded 36GiB to GPU in 2.45558s (14.6605GiB/s) | 20260508 05:08:06.488 | INFO | Downloaded 34GiB to GPU in 2.41284s (14.0913GiB/s) | 20260508 05:08:11.630 | INFO | Analyzing chunk frames = [5475328,5475840) | 20260508 05:08:11.630 | INFO | MemInfo: { | 20260508 05:08:11.630 | INFO | "gpu" : " 17 GB", | 20260508 05:08:11.630 | INFO | "huge" : "", | 20260508 05:08:11.630 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:11.630 | INFO | "malloc" : " 14 GB", | 20260508 05:08:11.630 | INFO | "pinned" : " 29 GB" | 20260508 05:08:11.630 | INFO | } | 20260508 05:08:11.630 | INFO | Perf report for the last 5.14276 seconds: | 20260508 05:08:11.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:11.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.630 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:08:11.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.630 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 05:08:11.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:11.631 | INFO | Uploaded 36GiB to GPU in 2.4456s (14.7203GiB/s) | 20260508 05:08:11.631 | INFO | Downloaded 34GiB to GPU in 2.41713s (14.0663GiB/s) | 20260508 05:08:16.764 | INFO | Analyzing chunk frames = [5475840,5476352) | 20260508 05:08:16.764 | INFO | MemInfo: { | 20260508 05:08:16.764 | INFO | "gpu" : " 17 GB", | 20260508 05:08:16.764 | INFO | "huge" : "", | 20260508 05:08:16.764 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:16.764 | INFO | "malloc" : " 14 GB", | 20260508 05:08:16.764 | INFO | "pinned" : " 29 GB" | 20260508 05:08:16.764 | INFO | } | 20260508 05:08:16.764 | INFO | Perf report for the last 5.13421 seconds: | 20260508 05:08:16.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:16.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.764 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:08:16.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.764 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 05:08:16.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:16.765 | INFO | Uploaded 36GiB to GPU in 2.47064s (14.5711GiB/s) | 20260508 05:08:16.765 | INFO | Downloaded 34GiB to GPU in 2.41927s (14.0538GiB/s) | 20260508 05:08:16.860 | INFO | Starting a round of Baz IO | 20260508 05:08:20.288 | INFO | Populating RTMetrics report | 20260508 05:08:20.289 | INFO | RTMetrics report populated | 20260508 05:08:20.289 | INFO | Opening tmp RTMetrics json file | 20260508 05:08:20.298 | INFO | Serializing RTMetrics to json | 20260508 05:08:20.302 | INFO | Serialization done, swapping tmp file now... | 20260508 05:08:20.302 | INFO | RTMetrics done | 20260508 05:08:20.362 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.501 seconds | 20260508 05:08:20.362 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.93% / 0.00% / 59.07% | 20260508 05:08:20.362 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.4861% / 0% / 5.5631e-05% | 20260508 05:08:20.362 | INFO | Baz IO overall: | 20260508 05:08:20.362 | INFO | BytesWritten : 2444086929428 | 20260508 05:08:20.362 | INFO | NumEvents : 1179458764174 | 20260508 05:08:21.905 | INFO | Analyzing chunk frames = [5476352,5476864) | 20260508 05:08:21.905 | INFO | MemInfo: { | 20260508 05:08:21.905 | INFO | "gpu" : " 17 GB", | 20260508 05:08:21.905 | INFO | "huge" : "", | 20260508 05:08:21.905 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:21.905 | INFO | "malloc" : " 14 GB", | 20260508 05:08:21.905 | INFO | "pinned" : " 29 GB" | 20260508 05:08:21.905 | INFO | } | 20260508 05:08:21.905 | INFO | Perf report for the last 5.14059 seconds: | 20260508 05:08:21.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:21.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:21.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:21.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:21.905 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 05:08:21.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:21.905 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:08:21.905 | INFO | PRE_HQ : 65.7%, 1.6, 55.3ms, 0 | 20260508 05:08:21.905 | INFO | BAZWRITER : 67.6%, 2.6, 90.5ms, 0 | 20260508 05:08:21.905 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:08:21.905 | INFO | Uploaded 36GiB to GPU in 2.58111s (13.9475GiB/s) | 20260508 05:08:21.905 | INFO | Downloaded 38GiB to GPU in 2.69362s (14.1074GiB/s) | 20260508 05:08:27.045 | INFO | Analyzing chunk frames = [5476864,5477376) | 20260508 05:08:27.045 | INFO | MemInfo: { | 20260508 05:08:27.045 | INFO | "gpu" : " 17 GB", | 20260508 05:08:27.045 | INFO | "huge" : "", | 20260508 05:08:27.045 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:27.045 | INFO | "malloc" : " 14 GB", | 20260508 05:08:27.045 | INFO | "pinned" : " 29 GB" | 20260508 05:08:27.045 | INFO | } | 20260508 05:08:27.045 | INFO | Perf report for the last 5.14048 seconds: | 20260508 05:08:27.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:27.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.045 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 05:08:27.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.045 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:08:27.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:27.046 | INFO | Uploaded 36GiB to GPU in 2.58237s (13.9407GiB/s) | 20260508 05:08:27.046 | INFO | Downloaded 34GiB to GPU in 2.42541s (14.0182GiB/s) | 20260508 05:08:32.189 | INFO | Analyzing chunk frames = [5477376,5477888) | 20260508 05:08:32.189 | INFO | MemInfo: { | 20260508 05:08:32.189 | INFO | "gpu" : " 17 GB", | 20260508 05:08:32.189 | INFO | "huge" : "", | 20260508 05:08:32.189 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:32.189 | INFO | "malloc" : " 14 GB", | 20260508 05:08:32.189 | INFO | "pinned" : " 29 GB" | 20260508 05:08:32.189 | INFO | } | 20260508 05:08:32.189 | INFO | Perf report for the last 5.1439 seconds: | 20260508 05:08:32.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:32.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.189 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 05:08:32.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.189 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:08:32.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:32.190 | INFO | Uploaded 36GiB to GPU in 2.55519s (14.0889GiB/s) | 20260508 05:08:32.190 | INFO | Downloaded 34GiB to GPU in 2.41118s (14.101GiB/s) | 20260508 05:08:37.330 | INFO | Analyzing chunk frames = [5477888,5478400) | 20260508 05:08:37.330 | INFO | MemInfo: { | 20260508 05:08:37.330 | INFO | "gpu" : " 17 GB", | 20260508 05:08:37.330 | INFO | "huge" : "", | 20260508 05:08:37.330 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:37.330 | INFO | "malloc" : " 14 GB", | 20260508 05:08:37.330 | INFO | "pinned" : " 29 GB" | 20260508 05:08:37.330 | INFO | } | 20260508 05:08:37.330 | INFO | Perf report for the last 5.14066 seconds: | 20260508 05:08:37.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:37.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.330 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 05:08:37.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.330 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:08:37.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:37.331 | INFO | Uploaded 36GiB to GPU in 2.55838s (14.0714GiB/s) | 20260508 05:08:37.331 | INFO | Downloaded 34GiB to GPU in 2.38895s (14.2322GiB/s) | 20260508 05:08:42.460 | INFO | Analyzing chunk frames = [5478400,5478912) | 20260508 05:08:42.460 | INFO | MemInfo: { | 20260508 05:08:42.460 | INFO | "gpu" : " 17 GB", | 20260508 05:08:42.460 | INFO | "huge" : "", | 20260508 05:08:42.460 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:42.460 | INFO | "malloc" : " 14 GB", | 20260508 05:08:42.460 | INFO | "pinned" : " 29 GB" | 20260508 05:08:42.460 | INFO | } | 20260508 05:08:42.460 | INFO | Perf report for the last 5.12986 seconds: | 20260508 05:08:42.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:42.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:08:42.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | PRE_HQ : 25.6%, 1.0, 13.2ms, 0 | 20260508 05:08:42.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:42.460 | INFO | Uploaded 36GiB to GPU in 2.37306s (15.1703GiB/s) | 20260508 05:08:42.460 | INFO | Downloaded 34GiB to GPU in 2.50286s (13.5844GiB/s) | 20260508 05:08:47.606 | INFO | Analyzing chunk frames = [5478912,5479424) | 20260508 05:08:47.606 | INFO | MemInfo: { | 20260508 05:08:47.606 | INFO | "gpu" : " 17 GB", | 20260508 05:08:47.606 | INFO | "huge" : "", | 20260508 05:08:47.606 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:47.606 | INFO | "malloc" : " 14 GB", | 20260508 05:08:47.606 | INFO | "pinned" : " 29 GB" | 20260508 05:08:47.606 | INFO | } | 20260508 05:08:47.606 | INFO | Perf report for the last 5.14665 seconds: | 20260508 05:08:47.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:47.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 05:08:47.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 05:08:47.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:47.607 | INFO | Uploaded 36GiB to GPU in 2.44891s (14.7004GiB/s) | 20260508 05:08:47.607 | INFO | Downloaded 34GiB to GPU in 2.43396s (13.969GiB/s) | 20260508 05:08:52.744 | INFO | Analyzing chunk frames = [5479424,5479936) | 20260508 05:08:52.744 | INFO | MemInfo: { | 20260508 05:08:52.744 | INFO | "gpu" : " 17 GB", | 20260508 05:08:52.744 | INFO | "huge" : "", | 20260508 05:08:52.744 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:52.744 | INFO | "malloc" : " 14 GB", | 20260508 05:08:52.744 | INFO | "pinned" : " 29 GB" | 20260508 05:08:52.744 | INFO | } | 20260508 05:08:52.744 | INFO | Perf report for the last 5.13778 seconds: | 20260508 05:08:52.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:52.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.744 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:08:52.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.744 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 05:08:52.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:52.745 | INFO | Uploaded 36GiB to GPU in 2.44621s (14.7166GiB/s) | 20260508 05:08:52.745 | INFO | Downloaded 34GiB to GPU in 2.42804s (14.0031GiB/s) | 20260508 05:08:57.882 | INFO | Analyzing chunk frames = [5479936,5480448) | 20260508 05:08:57.882 | INFO | MemInfo: { | 20260508 05:08:57.882 | INFO | "gpu" : " 17 GB", | 20260508 05:08:57.882 | INFO | "huge" : "", | 20260508 05:08:57.882 | INFO | "huge_shared" : " 29 GB", | 20260508 05:08:57.882 | INFO | "malloc" : " 14 GB", | 20260508 05:08:57.882 | INFO | "pinned" : " 29 GB" | 20260508 05:08:57.882 | INFO | } | 20260508 05:08:57.882 | INFO | Perf report for the last 5.13781 seconds: | 20260508 05:08:57.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:08:57.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 05:08:57.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 05:08:57.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:08:57.882 | INFO | Uploaded 36GiB to GPU in 2.45705s (14.6517GiB/s) | 20260508 05:08:57.883 | INFO | Downloaded 34GiB to GPU in 2.43914s (13.9393GiB/s) | 20260508 05:08:57.883 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:08:57.883 | INFO | Total mem usage: 16.7 GiB | 20260508 05:08:57.883 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:08:57.883 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:08:57.883 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:08:57.883 | INFO | High Water Breakdown: | 20260508 05:08:57.883 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:08:57.883 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:08:57.883 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:08:57.883 | INFO | ------------------------------------------ | 20260508 05:08:57.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:08:57.883 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:08:57.883 | INFO | Total mem usage: 26.5 GiB | 20260508 05:08:57.883 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:08:57.883 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:08:57.883 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:08:57.883 | INFO | High Water Breakdown: | 20260508 05:08:57.883 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:08:57.883 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:08:57.883 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:08:57.883 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:08:57.883 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:08:57.883 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:08:57.883 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:08:57.883 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:08:57.883 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:08:57.883 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:08:57.883 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:08:57.883 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:08:57.883 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:08:57.883 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:08:57.883 | INFO | ------------------------------------------ | 20260508 05:08:57.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:08:57.883 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:08:57.883 | INFO | Total mem usage: 27.8 GiB | 20260508 05:08:57.883 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:08:57.883 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:08:57.883 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:08:57.883 | INFO | High Water Breakdown: | 20260508 05:08:57.883 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:08:57.883 | INFO | ------------------------------------------ | 20260508 05:08:57.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:08:57.883 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:08:57.883 | INFO | Total mem usage: 15.7 GiB | 20260508 05:08:57.883 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:08:57.883 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:08:57.883 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:08:57.883 | INFO | High Water Breakdown: | 20260508 05:08:57.883 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:08:57.883 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:08:57.883 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:08:57.883 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:08:57.884 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:08:57.884 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:08:57.884 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:08:57.884 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:08:57.884 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:08:57.884 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:08:57.884 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:08:57.884 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:08:57.884 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:08:57.884 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:08:57.884 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:08:57.884 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:08:57.884 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:08:57.884 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:08:57.884 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:08:57.884 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:08:57.884 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:08:57.884 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:08:57.884 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:08:57.884 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:08:57.884 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:08:57.884 | INFO | ------------------------------------------ | 20260508 05:08:57.884 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:08:57.884 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:08:57.884 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:08:57.884 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:08:57.884 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.57e+01, 1.40e+01 / 1.36e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.04e-01, 1.15e-02 / 5.43e-03 | 20260508 05:08:57.884 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.41e+00, 5.88e-01 / 1.95e-01 | 20260508 05:08:57.884 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.10e-01 / 1.15e-01 | 20260508 05:08:57.884 | INFO | PerfReport: 15.93%, DME -- 4.16e+01 / 4.53e+01, 3.77e+00 / 1.66e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.77e+00 / 1.27e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.61e+00, 4.17e-01 / 2.19e-01 | 20260508 05:08:57.884 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.23e+00, 2.60e-01 / 1.16e-01 | 20260508 05:08:57.884 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.59e-01, 5.09e-02 / 2.23e-02 | 20260508 05:08:57.884 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:08:57.884 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:08:57.884 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:08:57.884 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 24.53%, Upload -- 2.61e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 24.16%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 2.42%, LockUpload -- 2.58e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 2.48%, LockDownload -- 2.63e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:08:57.884 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 05:08:57.994 | INFO | Starting a round of Baz IO | 20260508 05:09:01.410 | INFO | Populating RTMetrics report | 20260508 05:09:01.411 | INFO | RTMetrics report populated | 20260508 05:09:01.411 | INFO | Opening tmp RTMetrics json file | 20260508 05:09:01.422 | INFO | Serializing RTMetrics to json | 20260508 05:09:01.423 | INFO | Serialization done, swapping tmp file now... | 20260508 05:09:01.423 | INFO | RTMetrics done | 20260508 05:09:01.487 | INFO | Finished a round of Baz IO: Wrote 1.581 GiB in 3.492 seconds | 20260508 05:09:01.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.92% / 0.00% / 59.07% | 20260508 05:09:01.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.485051% / 7.94729e-06% / 6.75519e-05% | 20260508 05:09:01.487 | INFO | Baz IO overall: | 20260508 05:09:01.487 | INFO | BytesWritten : 2445784388818 | 20260508 05:09:01.487 | INFO | NumEvents : 1180268802993 | 20260508 05:09:03.020 | INFO | Analyzing chunk frames = [5480448,5480960) | 20260508 05:09:03.020 | INFO | MemInfo: { | 20260508 05:09:03.020 | INFO | "gpu" : " 17 GB", | 20260508 05:09:03.020 | INFO | "huge" : "", | 20260508 05:09:03.020 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:03.020 | INFO | "malloc" : " 14 GB", | 20260508 05:09:03.020 | INFO | "pinned" : " 29 GB" | 20260508 05:09:03.020 | INFO | } | 20260508 05:09:03.020 | INFO | Perf report for the last 5.13796 seconds: | 20260508 05:09:03.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:03.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:03.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:03.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:03.020 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 05:09:03.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:03.020 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 05:09:03.020 | INFO | PRE_HQ : 66.5%, 1.8, 62.2ms, 0 | 20260508 05:09:03.020 | INFO | BAZWRITER : 67.6%, 2.9, 100.1ms, 0 | 20260508 05:09:03.020 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:09:03.021 | INFO | Uploaded 36GiB to GPU in 2.54928s (14.1216GiB/s) | 20260508 05:09:03.021 | INFO | Downloaded 38GiB to GPU in 2.70057s (14.0711GiB/s) | 20260508 05:09:08.161 | INFO | Analyzing chunk frames = [5480960,5481472) | 20260508 05:09:08.161 | INFO | MemInfo: { | 20260508 05:09:08.161 | INFO | "gpu" : " 17 GB", | 20260508 05:09:08.161 | INFO | "huge" : "", | 20260508 05:09:08.161 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:08.162 | INFO | "malloc" : " 14 GB", | 20260508 05:09:08.162 | INFO | "pinned" : " 29 GB" | 20260508 05:09:08.162 | INFO | } | 20260508 05:09:08.162 | INFO | Perf report for the last 5.14165 seconds: | 20260508 05:09:08.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:08.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 05:09:08.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:09:08.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:08.162 | INFO | Uploaded 36GiB to GPU in 2.55635s (14.0826GiB/s) | 20260508 05:09:08.162 | INFO | Downloaded 34GiB to GPU in 2.45774s (13.8338GiB/s) | 20260508 05:09:13.299 | INFO | Analyzing chunk frames = [5481472,5481984) | 20260508 05:09:13.299 | INFO | MemInfo: { | 20260508 05:09:13.299 | INFO | "gpu" : " 17 GB", | 20260508 05:09:13.299 | INFO | "huge" : "", | 20260508 05:09:13.299 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:13.299 | INFO | "malloc" : " 14 GB", | 20260508 05:09:13.299 | INFO | "pinned" : " 29 GB" | 20260508 05:09:13.299 | INFO | } | 20260508 05:09:13.299 | INFO | Perf report for the last 5.13754 seconds: | 20260508 05:09:13.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:13.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.299 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:09:13.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.299 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:09:13.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:13.300 | INFO | Uploaded 36GiB to GPU in 2.49399s (14.4347GiB/s) | 20260508 05:09:13.300 | INFO | Downloaded 34GiB to GPU in 2.41652s (14.0698GiB/s) | 20260508 05:09:18.438 | INFO | Analyzing chunk frames = [5481984,5482496) | 20260508 05:09:18.438 | INFO | MemInfo: { | 20260508 05:09:18.438 | INFO | "gpu" : " 17 GB", | 20260508 05:09:18.438 | INFO | "huge" : "", | 20260508 05:09:18.438 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:18.438 | INFO | "malloc" : " 14 GB", | 20260508 05:09:18.438 | INFO | "pinned" : " 29 GB" | 20260508 05:09:18.439 | INFO | } | 20260508 05:09:18.439 | INFO | Perf report for the last 5.13926 seconds: | 20260508 05:09:18.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:18.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 05:09:18.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:09:18.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:18.439 | INFO | Uploaded 36GiB to GPU in 2.55125s (14.1107GiB/s) | 20260508 05:09:18.439 | INFO | Downloaded 34GiB to GPU in 2.43044s (13.9893GiB/s) | 20260508 05:09:23.576 | INFO | Analyzing chunk frames = [5482496,5483008) | 20260508 05:09:23.576 | INFO | MemInfo: { | 20260508 05:09:23.576 | INFO | "gpu" : " 17 GB", | 20260508 05:09:23.576 | INFO | "huge" : "", | 20260508 05:09:23.576 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:23.576 | INFO | "malloc" : " 14 GB", | 20260508 05:09:23.576 | INFO | "pinned" : " 29 GB" | 20260508 05:09:23.576 | INFO | } | 20260508 05:09:23.576 | INFO | Perf report for the last 5.13733 seconds: | 20260508 05:09:23.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:23.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:09:23.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:09:23.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:23.576 | INFO | Uploaded 36GiB to GPU in 2.37112s (15.1827GiB/s) | 20260508 05:09:23.576 | INFO | Downloaded 34GiB to GPU in 2.57156s (13.2216GiB/s) | 20260508 05:09:28.713 | INFO | Analyzing chunk frames = [5483008,5483520) | 20260508 05:09:28.713 | INFO | MemInfo: { | 20260508 05:09:28.713 | INFO | "gpu" : " 17 GB", | 20260508 05:09:28.713 | INFO | "huge" : "", | 20260508 05:09:28.713 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:28.713 | INFO | "malloc" : " 14 GB", | 20260508 05:09:28.713 | INFO | "pinned" : " 29 GB" | 20260508 05:09:28.713 | INFO | } | 20260508 05:09:28.713 | INFO | Perf report for the last 5.13758 seconds: | 20260508 05:09:28.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:28.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.714 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:09:28.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.714 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:09:28.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:28.714 | INFO | Uploaded 36GiB to GPU in 2.43686s (14.7731GiB/s) | 20260508 05:09:28.714 | INFO | Downloaded 34GiB to GPU in 2.41929s (14.0537GiB/s) | 20260508 05:09:33.860 | INFO | Analyzing chunk frames = [5483520,5484032) | 20260508 05:09:33.860 | INFO | MemInfo: { | 20260508 05:09:33.860 | INFO | "gpu" : " 17 GB", | 20260508 05:09:33.860 | INFO | "huge" : "", | 20260508 05:09:33.860 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:33.860 | INFO | "malloc" : " 14 GB", | 20260508 05:09:33.860 | INFO | "pinned" : " 29 GB" | 20260508 05:09:33.860 | INFO | } | 20260508 05:09:33.860 | INFO | Perf report for the last 5.14663 seconds: | 20260508 05:09:33.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:33.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.860 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:09:33.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.860 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 05:09:33.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:33.861 | INFO | Uploaded 36GiB to GPU in 2.41948s (14.8792GiB/s) | 20260508 05:09:33.861 | INFO | Downloaded 34GiB to GPU in 2.4234s (14.0299GiB/s) | 20260508 05:09:39.003 | INFO | Analyzing chunk frames = [5484032,5484544) | 20260508 05:09:39.003 | INFO | MemInfo: { | 20260508 05:09:39.003 | INFO | "gpu" : " 17 GB", | 20260508 05:09:39.003 | INFO | "huge" : "", | 20260508 05:09:39.003 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:39.003 | INFO | "malloc" : " 14 GB", | 20260508 05:09:39.003 | INFO | "pinned" : " 29 GB" | 20260508 05:09:39.003 | INFO | } | 20260508 05:09:39.003 | INFO | Perf report for the last 5.14316 seconds: | 20260508 05:09:39.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:39.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.003 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:09:39.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.004 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 05:09:39.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:39.004 | INFO | Uploaded 36GiB to GPU in 2.4608s (14.6294GiB/s) | 20260508 05:09:39.004 | INFO | Downloaded 34GiB to GPU in 2.43403s (13.9686GiB/s) | 20260508 05:09:39.115 | INFO | Starting a round of Baz IO | 20260508 05:09:42.521 | INFO | Populating RTMetrics report | 20260508 05:09:42.522 | INFO | RTMetrics report populated | 20260508 05:09:42.522 | INFO | Opening tmp RTMetrics json file | 20260508 05:09:42.530 | INFO | Serializing RTMetrics to json | 20260508 05:09:42.536 | INFO | Serialization done, swapping tmp file now... | 20260508 05:09:42.537 | INFO | RTMetrics done | 20260508 05:09:42.597 | INFO | Finished a round of Baz IO: Wrote 1.581 GiB in 3.481 seconds | 20260508 05:09:42.597 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.92% / 0.00% / 59.08% | 20260508 05:09:42.597 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.484085% / 1.19209e-05% / 6.75519e-05% | 20260508 05:09:42.597 | INFO | Baz IO overall: | 20260508 05:09:42.597 | INFO | BytesWritten : 2447481524700 | 20260508 05:09:42.597 | INFO | NumEvents : 1181078664025 | 20260508 05:09:44.136 | INFO | Analyzing chunk frames = [5484544,5485056) | 20260508 05:09:44.136 | INFO | MemInfo: { | 20260508 05:09:44.136 | INFO | "gpu" : " 17 GB", | 20260508 05:09:44.136 | INFO | "huge" : "", | 20260508 05:09:44.136 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:44.136 | INFO | "malloc" : " 14 GB", | 20260508 05:09:44.137 | INFO | "pinned" : " 29 GB" | 20260508 05:09:44.137 | INFO | } | 20260508 05:09:44.137 | INFO | Perf report for the last 5.13331 seconds: | 20260508 05:09:44.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:44.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:44.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:44.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:44.137 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 05:09:44.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:44.137 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:09:44.137 | INFO | PRE_HQ : 66.4%, 1.8, 61.6ms, 0 | 20260508 05:09:44.137 | INFO | BAZWRITER : 67.4%, 2.9, 101.0ms, 0 | 20260508 05:09:44.137 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:09:44.137 | INFO | Uploaded 36GiB to GPU in 2.57718s (13.9688GiB/s) | 20260508 05:09:44.137 | INFO | Downloaded 38GiB to GPU in 2.68862s (14.1337GiB/s) | 20260508 05:09:49.273 | INFO | Analyzing chunk frames = [5485056,5485568) | 20260508 05:09:49.273 | INFO | MemInfo: { | 20260508 05:09:49.273 | INFO | "gpu" : " 17 GB", | 20260508 05:09:49.273 | INFO | "huge" : "", | 20260508 05:09:49.273 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:49.273 | INFO | "malloc" : " 14 GB", | 20260508 05:09:49.273 | INFO | "pinned" : " 29 GB" | 20260508 05:09:49.273 | INFO | } | 20260508 05:09:49.273 | INFO | Perf report for the last 5.13639 seconds: | 20260508 05:09:49.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:49.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 05:09:49.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:09:49.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:49.273 | INFO | Uploaded 36GiB to GPU in 2.57405s (13.9858GiB/s) | 20260508 05:09:49.273 | INFO | Downloaded 34GiB to GPU in 2.44832s (13.8871GiB/s) | 20260508 05:09:54.412 | INFO | Analyzing chunk frames = [5485568,5486080) | 20260508 05:09:54.412 | INFO | MemInfo: { | 20260508 05:09:54.412 | INFO | "gpu" : " 17 GB", | 20260508 05:09:54.412 | INFO | "huge" : "", | 20260508 05:09:54.412 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:54.412 | INFO | "malloc" : " 14 GB", | 20260508 05:09:54.412 | INFO | "pinned" : " 29 GB" | 20260508 05:09:54.412 | INFO | } | 20260508 05:09:54.412 | INFO | Perf report for the last 5.13895 seconds: | 20260508 05:09:54.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:54.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:09:54.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:09:54.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:54.412 | INFO | Uploaded 36GiB to GPU in 2.52118s (14.279GiB/s) | 20260508 05:09:54.412 | INFO | Downloaded 34GiB to GPU in 2.414s (14.0845GiB/s) | 20260508 05:09:59.554 | INFO | Analyzing chunk frames = [5486080,5486592) | 20260508 05:09:59.554 | INFO | MemInfo: { | 20260508 05:09:59.554 | INFO | "gpu" : " 17 GB", | 20260508 05:09:59.554 | INFO | "huge" : "", | 20260508 05:09:59.554 | INFO | "huge_shared" : " 29 GB", | 20260508 05:09:59.554 | INFO | "malloc" : " 14 GB", | 20260508 05:09:59.554 | INFO | "pinned" : " 29 GB" | 20260508 05:09:59.554 | INFO | } | 20260508 05:09:59.554 | INFO | Perf report for the last 5.14211 seconds: | 20260508 05:09:59.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:09:59.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | ANALYSIS : 67.0%, 3.0, 103.8ms, 0 | 20260508 05:09:59.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:09:59.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:09:59.554 | INFO | Uploaded 36GiB to GPU in 2.56815s (14.0179GiB/s) | 20260508 05:09:59.554 | INFO | Downloaded 34GiB to GPU in 2.43106s (13.9857GiB/s) | 20260508 05:10:04.700 | INFO | Analyzing chunk frames = [5486592,5487104) | 20260508 05:10:04.700 | INFO | MemInfo: { | 20260508 05:10:04.700 | INFO | "gpu" : " 17 GB", | 20260508 05:10:04.700 | INFO | "huge" : "", | 20260508 05:10:04.700 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:04.700 | INFO | "malloc" : " 14 GB", | 20260508 05:10:04.700 | INFO | "pinned" : " 29 GB" | 20260508 05:10:04.700 | INFO | } | 20260508 05:10:04.700 | INFO | Perf report for the last 5.14617 seconds: | 20260508 05:10:04.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:04.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.700 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 05:10:04.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.700 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:10:04.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:04.701 | INFO | Uploaded 36GiB to GPU in 2.41635s (14.8985GiB/s) | 20260508 05:10:04.701 | INFO | Downloaded 34GiB to GPU in 2.55406s (13.3121GiB/s) | 20260508 05:10:09.834 | INFO | Analyzing chunk frames = [5487104,5487616) | 20260508 05:10:09.834 | INFO | MemInfo: { | 20260508 05:10:09.834 | INFO | "gpu" : " 17 GB", | 20260508 05:10:09.834 | INFO | "huge" : "", | 20260508 05:10:09.834 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:09.834 | INFO | "malloc" : " 14 GB", | 20260508 05:10:09.834 | INFO | "pinned" : " 29 GB" | 20260508 05:10:09.834 | INFO | } | 20260508 05:10:09.834 | INFO | Perf report for the last 5.13398 seconds: | 20260508 05:10:09.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:09.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.834 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 05:10:09.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.834 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:10:09.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:09.835 | INFO | Uploaded 36GiB to GPU in 2.39536s (15.0291GiB/s) | 20260508 05:10:09.835 | INFO | Downloaded 34GiB to GPU in 2.43989s (13.935GiB/s) | 20260508 05:10:14.967 | INFO | Analyzing chunk frames = [5487616,5488128) | 20260508 05:10:14.967 | INFO | MemInfo: { | 20260508 05:10:14.967 | INFO | "gpu" : " 17 GB", | 20260508 05:10:14.968 | INFO | "huge" : "", | 20260508 05:10:14.968 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:14.968 | INFO | "malloc" : " 14 GB", | 20260508 05:10:14.968 | INFO | "pinned" : " 29 GB" | 20260508 05:10:14.968 | INFO | } | 20260508 05:10:14.968 | INFO | Perf report for the last 5.13365 seconds: | 20260508 05:10:14.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:14.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 05:10:14.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 05:10:14.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:14.968 | INFO | Uploaded 36GiB to GPU in 2.39285s (15.0448GiB/s) | 20260508 05:10:14.968 | INFO | Downloaded 34GiB to GPU in 2.43668s (13.9534GiB/s) | 20260508 05:10:20.112 | INFO | Analyzing chunk frames = [5488128,5488640) | 20260508 05:10:20.112 | INFO | MemInfo: { | 20260508 05:10:20.112 | INFO | "gpu" : " 17 GB", | 20260508 05:10:20.112 | INFO | "huge" : "", | 20260508 05:10:20.112 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:20.112 | INFO | "malloc" : " 14 GB", | 20260508 05:10:20.113 | INFO | "pinned" : " 29 GB" | 20260508 05:10:20.113 | INFO | } | 20260508 05:10:20.113 | INFO | Perf report for the last 5.14468 seconds: | 20260508 05:10:20.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:20.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:10:20.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 05:10:20.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:20.113 | INFO | Uploaded 36GiB to GPU in 2.41944s (14.8795GiB/s) | 20260508 05:10:20.113 | INFO | Downloaded 34GiB to GPU in 2.4393s (13.9385GiB/s) | 20260508 05:10:20.113 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:10:20.113 | INFO | Total mem usage: 16.7 GiB | 20260508 05:10:20.113 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:10:20.113 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:10:20.114 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:10:20.114 | INFO | High Water Breakdown: | 20260508 05:10:20.114 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:10:20.114 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:10:20.114 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:10:20.114 | INFO | ------------------------------------------ | 20260508 05:10:20.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:10:20.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:10:20.114 | INFO | Total mem usage: 26.5 GiB | 20260508 05:10:20.114 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:10:20.114 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 05:10:20.114 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 05:10:20.114 | INFO | High Water Breakdown: | 20260508 05:10:20.114 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:10:20.114 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:10:20.114 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:10:20.114 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:10:20.114 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:10:20.114 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:10:20.114 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:10:20.114 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:10:20.114 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:10:20.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:10:20.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:10:20.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:10:20.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:10:20.114 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:10:20.114 | INFO | ------------------------------------------ | 20260508 05:10:20.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:10:20.114 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:10:20.114 | INFO | Total mem usage: 27.8 GiB | 20260508 05:10:20.114 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:10:20.114 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:10:20.114 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:10:20.114 | INFO | High Water Breakdown: | 20260508 05:10:20.114 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:10:20.114 | INFO | ------------------------------------------ | 20260508 05:10:20.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:10:20.114 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:10:20.114 | INFO | Total mem usage: 15.7 GiB | 20260508 05:10:20.114 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:10:20.114 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:10:20.114 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:10:20.114 | INFO | High Water Breakdown: | 20260508 05:10:20.114 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:10:20.114 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:10:20.114 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:10:20.114 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:10:20.114 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:10:20.114 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:10:20.114 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:10:20.114 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:10:20.114 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:10:20.114 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:10:20.114 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:10:20.114 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:10:20.114 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:10:20.114 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:10:20.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:10:20.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:10:20.114 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:10:20.114 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:10:20.114 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:10:20.114 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:10:20.114 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:10:20.114 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:10:20.114 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:10:20.114 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:10:20.114 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:10:20.114 | INFO | ------------------------------------------ | 20260508 05:10:20.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:10:20.115 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:10:20.115 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:10:20.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:10:20.115 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.38e+01 / 1.83e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.03e-01, 1.17e-02 / 6.05e-03 | 20260508 05:10:20.115 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.54e+00, 5.84e-01 / 1.50e-01 | 20260508 05:10:20.115 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.42e+00, 2.08e-01 / 1.35e-01 | 20260508 05:10:20.115 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.55e+01, 3.87e+00 / 2.11e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.78e+00 / 1.47e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.61e+00, 4.18e-01 / 2.24e-01 | 20260508 05:10:20.115 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.21e+00, 2.62e-01 / 9.03e-02 | 20260508 05:10:20.115 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.60e-01, 5.07e-02 / 2.09e-02 | 20260508 05:10:20.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:10:20.115 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:10:20.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:10:20.115 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 24.34%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 24.34%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 2.79%, LockDownload -- 2.96e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 05:10:20.115 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260508 05:10:20.226 | INFO | Starting a round of Baz IO | 20260508 05:10:23.647 | INFO | Populating RTMetrics report | 20260508 05:10:23.647 | INFO | RTMetrics report populated | 20260508 05:10:23.647 | INFO | Opening tmp RTMetrics json file | 20260508 05:10:23.657 | INFO | Serializing RTMetrics to json | 20260508 05:10:23.658 | INFO | Serialization done, swapping tmp file now... | 20260508 05:10:23.659 | INFO | RTMetrics done | 20260508 05:10:23.718 | INFO | Finished a round of Baz IO: Wrote 1.579 GiB in 3.492 seconds | 20260508 05:10:23.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.92% / 0.00% / 59.08% | 20260508 05:10:23.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.481705% / 7.94729e-06% / 8.34465e-05% | 20260508 05:10:23.718 | INFO | Baz IO overall: | 20260508 05:10:23.718 | INFO | BytesWritten : 2449176612183 | 20260508 05:10:23.718 | INFO | NumEvents : 1181887062272 | 20260508 05:10:25.251 | INFO | Analyzing chunk frames = [5488640,5489152) | 20260508 05:10:25.252 | INFO | MemInfo: { | 20260508 05:10:25.252 | INFO | "gpu" : " 17 GB", | 20260508 05:10:25.252 | INFO | "huge" : "", | 20260508 05:10:25.252 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:25.252 | INFO | "malloc" : " 14 GB", | 20260508 05:10:25.252 | INFO | "pinned" : " 29 GB" | 20260508 05:10:25.252 | INFO | } | 20260508 05:10:25.252 | INFO | Perf report for the last 5.13911 seconds: | 20260508 05:10:25.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:25.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:25.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:25.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:25.252 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 05:10:25.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:25.252 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:10:25.252 | INFO | PRE_HQ : 66.2%, 1.8, 61.8ms, 0 | 20260508 05:10:25.252 | INFO | BAZWRITER : 67.5%, 2.9, 99.4ms, 0 | 20260508 05:10:25.252 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:10:25.252 | INFO | Uploaded 36GiB to GPU in 2.56293s (14.0464GiB/s) | 20260508 05:10:25.252 | INFO | Downloaded 38GiB to GPU in 2.69594s (14.0953GiB/s) | 20260508 05:10:30.388 | INFO | Analyzing chunk frames = [5489152,5489664) | 20260508 05:10:30.388 | INFO | MemInfo: { | 20260508 05:10:30.388 | INFO | "gpu" : " 17 GB", | 20260508 05:10:30.388 | INFO | "huge" : "", | 20260508 05:10:30.388 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:30.388 | INFO | "malloc" : " 14 GB", | 20260508 05:10:30.388 | INFO | "pinned" : " 29 GB" | 20260508 05:10:30.388 | INFO | } | 20260508 05:10:30.388 | INFO | Perf report for the last 5.13637 seconds: | 20260508 05:10:30.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:30.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.388 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 05:10:30.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.388 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:10:30.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:30.389 | INFO | Uploaded 36GiB to GPU in 2.55041s (14.1154GiB/s) | 20260508 05:10:30.389 | INFO | Downloaded 34GiB to GPU in 2.45997s (13.8213GiB/s) | 20260508 05:10:35.526 | INFO | Analyzing chunk frames = [5489664,5490176) | 20260508 05:10:35.526 | INFO | MemInfo: { | 20260508 05:10:35.526 | INFO | "gpu" : " 17 GB", | 20260508 05:10:35.526 | INFO | "huge" : "", | 20260508 05:10:35.526 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:35.526 | INFO | "malloc" : " 14 GB", | 20260508 05:10:35.526 | INFO | "pinned" : " 29 GB" | 20260508 05:10:35.526 | INFO | } | 20260508 05:10:35.526 | INFO | Perf report for the last 5.13826 seconds: | 20260508 05:10:35.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:35.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.526 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:10:35.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.527 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:10:35.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:35.527 | INFO | Uploaded 36GiB to GPU in 2.51488s (14.3148GiB/s) | 20260508 05:10:35.527 | INFO | Downloaded 34GiB to GPU in 2.43128s (13.9844GiB/s) | 20260508 05:10:40.667 | INFO | Analyzing chunk frames = [5490176,5490688) | 20260508 05:10:40.667 | INFO | MemInfo: { | 20260508 05:10:40.667 | INFO | "gpu" : " 17 GB", | 20260508 05:10:40.667 | INFO | "huge" : "", | 20260508 05:10:40.667 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:40.667 | INFO | "malloc" : " 14 GB", | 20260508 05:10:40.667 | INFO | "pinned" : " 29 GB" | 20260508 05:10:40.667 | INFO | } | 20260508 05:10:40.667 | INFO | Perf report for the last 5.1407 seconds: | 20260508 05:10:40.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:40.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 05:10:40.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:10:40.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:40.667 | INFO | Uploaded 36GiB to GPU in 2.53126s (14.2222GiB/s) | 20260508 05:10:40.667 | INFO | Downloaded 34GiB to GPU in 2.42097s (14.044GiB/s) | 20260508 05:10:45.809 | INFO | Analyzing chunk frames = [5490688,5491200) | 20260508 05:10:45.809 | INFO | MemInfo: { | 20260508 05:10:45.809 | INFO | "gpu" : " 17 GB", | 20260508 05:10:45.809 | INFO | "huge" : "", | 20260508 05:10:45.809 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:45.809 | INFO | "malloc" : " 14 GB", | 20260508 05:10:45.809 | INFO | "pinned" : " 29 GB" | 20260508 05:10:45.809 | INFO | } | 20260508 05:10:45.809 | INFO | Perf report for the last 5.14171 seconds: | 20260508 05:10:45.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:45.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:10:45.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:10:45.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:45.809 | INFO | Uploaded 36GiB to GPU in 2.37708s (15.1446GiB/s) | 20260508 05:10:45.809 | INFO | Downloaded 34GiB to GPU in 2.56712s (13.2444GiB/s) | 20260508 05:10:50.950 | INFO | Analyzing chunk frames = [5491200,5491712) | 20260508 05:10:50.950 | INFO | MemInfo: { | 20260508 05:10:50.950 | INFO | "gpu" : " 17 GB", | 20260508 05:10:50.950 | INFO | "huge" : "", | 20260508 05:10:50.950 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:50.950 | INFO | "malloc" : " 14 GB", | 20260508 05:10:50.950 | INFO | "pinned" : " 29 GB" | 20260508 05:10:50.950 | INFO | } | 20260508 05:10:50.950 | INFO | Perf report for the last 5.14124 seconds: | 20260508 05:10:50.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:50.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:10:50.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:10:50.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:50.950 | INFO | Uploaded 36GiB to GPU in 2.40358s (14.9776GiB/s) | 20260508 05:10:50.950 | INFO | Downloaded 34GiB to GPU in 2.438s (13.9459GiB/s) | 20260508 05:10:56.085 | INFO | Analyzing chunk frames = [5491712,5492224) | 20260508 05:10:56.085 | INFO | MemInfo: { | 20260508 05:10:56.085 | INFO | "gpu" : " 17 GB", | 20260508 05:10:56.085 | INFO | "huge" : "", | 20260508 05:10:56.085 | INFO | "huge_shared" : " 29 GB", | 20260508 05:10:56.085 | INFO | "malloc" : " 14 GB", | 20260508 05:10:56.086 | INFO | "pinned" : " 29 GB" | 20260508 05:10:56.086 | INFO | } | 20260508 05:10:56.086 | INFO | Perf report for the last 5.13574 seconds: | 20260508 05:10:56.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:10:56.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.086 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 05:10:56.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.086 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 05:10:56.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:10:56.087 | INFO | Uploaded 36GiB to GPU in 2.39173s (15.0518GiB/s) | 20260508 05:10:56.087 | INFO | Downloaded 34GiB to GPU in 2.42951s (13.9946GiB/s) | 20260508 05:11:01.222 | INFO | Analyzing chunk frames = [5492224,5492736) | 20260508 05:11:01.222 | INFO | MemInfo: { | 20260508 05:11:01.222 | INFO | "gpu" : " 17 GB", | 20260508 05:11:01.222 | INFO | "huge" : "", | 20260508 05:11:01.222 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:01.222 | INFO | "malloc" : " 14 GB", | 20260508 05:11:01.223 | INFO | "pinned" : " 29 GB" | 20260508 05:11:01.223 | INFO | } | 20260508 05:11:01.223 | INFO | Perf report for the last 5.13676 seconds: | 20260508 05:11:01.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:01.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:11:01.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260508 05:11:01.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:01.223 | INFO | Uploaded 36GiB to GPU in 2.42016s (14.875GiB/s) | 20260508 05:11:01.223 | INFO | Downloaded 34GiB to GPU in 2.45593s (13.8441GiB/s) | 20260508 05:11:01.319 | INFO | Starting a round of Baz IO | 20260508 05:11:04.753 | INFO | Populating RTMetrics report | 20260508 05:11:04.753 | INFO | RTMetrics report populated | 20260508 05:11:04.753 | INFO | Opening tmp RTMetrics json file | 20260508 05:11:04.763 | INFO | Serializing RTMetrics to json | 20260508 05:11:04.765 | INFO | Serialization done, swapping tmp file now... | 20260508 05:11:04.766 | INFO | RTMetrics done | 20260508 05:11:04.823 | INFO | Finished a round of Baz IO: Wrote 1.579 GiB in 3.504 seconds | 20260508 05:11:04.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.92% / 0.00% / 59.08% | 20260508 05:11:04.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.48217% / 0% / 5.96046e-05% | 20260508 05:11:04.823 | INFO | Baz IO overall: | 20260508 05:11:04.824 | INFO | BytesWritten : 2450871924104 | 20260508 05:11:04.824 | INFO | NumEvents : 1182695526173 | 20260508 05:11:06.364 | INFO | Analyzing chunk frames = [5492736,5493248) | 20260508 05:11:06.364 | INFO | MemInfo: { | 20260508 05:11:06.364 | INFO | "gpu" : " 17 GB", | 20260508 05:11:06.364 | INFO | "huge" : "", | 20260508 05:11:06.364 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:06.364 | INFO | "malloc" : " 14 GB", | 20260508 05:11:06.364 | INFO | "pinned" : " 29 GB" | 20260508 05:11:06.364 | INFO | } | 20260508 05:11:06.364 | INFO | Perf report for the last 5.1414 seconds: | 20260508 05:11:06.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:06.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:06.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:06.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:06.364 | INFO | ANALYSIS : 67.7%, 3.0, 106.0ms, 0 | 20260508 05:11:06.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:06.364 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:11:06.364 | INFO | PRE_HQ : 65.4%, 1.6, 55.2ms, 0 | 20260508 05:11:06.364 | INFO | BAZWRITER : 67.6%, 2.7, 95.3ms, 0 | 20260508 05:11:06.364 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 05:11:06.364 | INFO | Uploaded 36GiB to GPU in 2.53372s (14.2083GiB/s) | 20260508 05:11:06.364 | INFO | Downloaded 38GiB to GPU in 2.69905s (14.079GiB/s) | 20260508 05:11:11.504 | INFO | Analyzing chunk frames = [5493248,5493760) | 20260508 05:11:11.504 | INFO | MemInfo: { | 20260508 05:11:11.504 | INFO | "gpu" : " 17 GB", | 20260508 05:11:11.504 | INFO | "huge" : "", | 20260508 05:11:11.504 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:11.504 | INFO | "malloc" : " 14 GB", | 20260508 05:11:11.504 | INFO | "pinned" : " 29 GB" | 20260508 05:11:11.504 | INFO | } | 20260508 05:11:11.505 | INFO | Perf report for the last 5.14046 seconds: | 20260508 05:11:11.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:11.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | ANALYSIS : 67.3%, 3.0, 104.9ms, 0 | 20260508 05:11:11.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:11:11.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:11.505 | INFO | Uploaded 36GiB to GPU in 2.56328s (14.0445GiB/s) | 20260508 05:11:11.505 | INFO | Downloaded 34GiB to GPU in 2.43951s (13.9372GiB/s) | 20260508 05:11:16.646 | INFO | Analyzing chunk frames = [5493760,5494272) | 20260508 05:11:16.646 | INFO | MemInfo: { | 20260508 05:11:16.646 | INFO | "gpu" : " 17 GB", | 20260508 05:11:16.646 | INFO | "huge" : "", | 20260508 05:11:16.646 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:16.646 | INFO | "malloc" : " 14 GB", | 20260508 05:11:16.646 | INFO | "pinned" : " 29 GB" | 20260508 05:11:16.646 | INFO | } | 20260508 05:11:16.646 | INFO | Perf report for the last 5.14177 seconds: | 20260508 05:11:16.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:16.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.646 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 05:11:16.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.646 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:11:16.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:16.647 | INFO | Uploaded 36GiB to GPU in 2.53807s (14.184GiB/s) | 20260508 05:11:16.647 | INFO | Downloaded 34GiB to GPU in 2.43686s (13.9524GiB/s) | 20260508 05:11:21.781 | INFO | Analyzing chunk frames = [5494272,5494784) | 20260508 05:11:21.781 | INFO | MemInfo: { | 20260508 05:11:21.781 | INFO | "gpu" : " 17 GB", | 20260508 05:11:21.781 | INFO | "huge" : "", | 20260508 05:11:21.781 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:21.781 | INFO | "malloc" : " 14 GB", | 20260508 05:11:21.781 | INFO | "pinned" : " 29 GB" | 20260508 05:11:21.781 | INFO | } | 20260508 05:11:21.781 | INFO | Perf report for the last 5.13483 seconds: | 20260508 05:11:21.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:21.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | ANALYSIS : 67.4%, 3.0, 104.2ms, 0 | 20260508 05:11:21.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:11:21.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:21.781 | INFO | Uploaded 36GiB to GPU in 2.55793s (14.0739GiB/s) | 20260508 05:11:21.781 | INFO | Downloaded 34GiB to GPU in 2.50665s (13.5639GiB/s) | 20260508 05:11:26.922 | INFO | Analyzing chunk frames = [5494784,5495296) | 20260508 05:11:26.922 | INFO | MemInfo: { | 20260508 05:11:26.922 | INFO | "gpu" : " 17 GB", | 20260508 05:11:26.922 | INFO | "huge" : "", | 20260508 05:11:26.922 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:26.922 | INFO | "malloc" : " 14 GB", | 20260508 05:11:26.922 | INFO | "pinned" : " 29 GB" | 20260508 05:11:26.922 | INFO | } | 20260508 05:11:26.922 | INFO | Perf report for the last 5.14118 seconds: | 20260508 05:11:26.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:26.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.922 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:11:26.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.922 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:11:26.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:26.923 | INFO | Uploaded 36GiB to GPU in 2.34825s (15.3305GiB/s) | 20260508 05:11:26.923 | INFO | Downloaded 34GiB to GPU in 2.54361s (13.3668GiB/s) | 20260508 05:11:32.065 | INFO | Analyzing chunk frames = [5495296,5495808) | 20260508 05:11:32.065 | INFO | MemInfo: { | 20260508 05:11:32.065 | INFO | "gpu" : " 17 GB", | 20260508 05:11:32.065 | INFO | "huge" : "", | 20260508 05:11:32.065 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:32.065 | INFO | "malloc" : " 14 GB", | 20260508 05:11:32.065 | INFO | "pinned" : " 29 GB" | 20260508 05:11:32.065 | INFO | } | 20260508 05:11:32.065 | INFO | Perf report for the last 5.1427 seconds: | 20260508 05:11:32.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:32.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.065 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 05:11:32.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.066 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:11:32.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:32.066 | INFO | Uploaded 36GiB to GPU in 2.45609s (14.6574GiB/s) | 20260508 05:11:32.066 | INFO | Downloaded 34GiB to GPU in 2.41493s (14.0791GiB/s) | 20260508 05:11:37.201 | INFO | Analyzing chunk frames = [5495808,5496320) | 20260508 05:11:37.201 | INFO | MemInfo: { | 20260508 05:11:37.201 | INFO | "gpu" : " 17 GB", | 20260508 05:11:37.201 | INFO | "huge" : "", | 20260508 05:11:37.201 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:37.201 | INFO | "malloc" : " 14 GB", | 20260508 05:11:37.201 | INFO | "pinned" : " 29 GB" | 20260508 05:11:37.201 | INFO | } | 20260508 05:11:37.201 | INFO | Perf report for the last 5.13611 seconds: | 20260508 05:11:37.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:37.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.201 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:11:37.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.201 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 05:11:37.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:37.202 | INFO | Uploaded 36GiB to GPU in 2.49445s (14.4321GiB/s) | 20260508 05:11:37.202 | INFO | Downloaded 34GiB to GPU in 2.4334s (13.9722GiB/s) | 20260508 05:11:42.338 | INFO | Analyzing chunk frames = [5496320,5496832) | 20260508 05:11:42.338 | INFO | MemInfo: { | 20260508 05:11:42.338 | INFO | "gpu" : " 17 GB", | 20260508 05:11:42.338 | INFO | "huge" : "", | 20260508 05:11:42.338 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:42.338 | INFO | "malloc" : " 14 GB", | 20260508 05:11:42.338 | INFO | "pinned" : " 29 GB" | 20260508 05:11:42.338 | INFO | } | 20260508 05:11:42.338 | INFO | Perf report for the last 5.13677 seconds: | 20260508 05:11:42.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:42.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.338 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 05:11:42.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.338 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 05:11:42.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:42.339 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:11:42.339 | INFO | Total mem usage: 16.7 GiB | 20260508 05:11:42.339 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:11:42.339 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:11:42.339 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:11:42.339 | INFO | High Water Breakdown: | 20260508 05:11:42.339 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:11:42.339 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:11:42.339 | INFO | Uploaded 36GiB to GPU in 2.48011s (14.5155GiB/s) | 20260508 05:11:42.339 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:11:42.339 | INFO | ------------------------------------------ | 20260508 05:11:42.339 | INFO | Downloaded 34GiB to GPU in 2.4327s (13.9763GiB/s) | 20260508 05:11:42.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:11:42.339 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:11:42.339 | INFO | Total mem usage: 26.5 GiB | 20260508 05:11:42.339 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:11:42.339 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 05:11:42.339 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 05:11:42.339 | INFO | High Water Breakdown: | 20260508 05:11:42.339 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:11:42.339 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:11:42.339 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:11:42.339 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:11:42.339 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:11:42.339 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:11:42.339 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:11:42.339 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:11:42.339 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:11:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:11:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:11:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:11:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:11:42.339 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:11:42.339 | INFO | ------------------------------------------ | 20260508 05:11:42.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:11:42.339 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:11:42.339 | INFO | Total mem usage: 27.8 GiB | 20260508 05:11:42.339 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:11:42.339 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:11:42.339 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:11:42.339 | INFO | High Water Breakdown: | 20260508 05:11:42.339 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:11:42.339 | INFO | ------------------------------------------ | 20260508 05:11:42.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:11:42.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:11:42.339 | INFO | Total mem usage: 15.7 GiB | 20260508 05:11:42.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:11:42.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:11:42.339 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:11:42.339 | INFO | High Water Breakdown: | 20260508 05:11:42.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:11:42.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:11:42.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:11:42.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:11:42.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:11:42.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:11:42.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:11:42.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:11:42.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:11:42.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:11:42.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:11:42.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:11:42.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:11:42.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:11:42.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:11:42.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:11:42.339 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:11:42.339 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:11:42.339 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:11:42.339 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:11:42.339 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:11:42.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:11:42.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:11:42.340 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:11:42.340 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:11:42.340 | INFO | ------------------------------------------ | 20260508 05:11:42.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:11:42.340 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:11:42.340 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:11:42.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:11:42.340 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.41e+01 / 1.67e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.00e-01, 1.12e-02 / 7.37e-03 | 20260508 05:11:42.340 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.49e+00, 5.85e-01 / 1.83e-01 | 20260508 05:11:42.340 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.09e-01 / 1.35e-01 | 20260508 05:11:42.340 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.56e+01, 3.70e+00 / 2.10e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.76e+00 / 1.48e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.55e+00, 4.17e-01 / 2.60e-01 | 20260508 05:11:42.340 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.21e+00, 2.57e-01 / 1.37e-01 | 20260508 05:11:42.340 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.19e-01 / 3.50e-01, 5.09e-02 / 2.42e-02 | 20260508 05:11:42.340 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:11:42.340 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:11:42.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:11:42.340 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 24.33%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 24.38%, Download -- 2.59e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 30.75%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 2.14%, LockUpload -- 2.28e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 2.77%, LockDownload -- 2.95e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:11:42.340 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 05:11:42.451 | INFO | Starting a round of Baz IO | 20260508 05:11:45.865 | INFO | Populating RTMetrics report | 20260508 05:11:45.865 | INFO | RTMetrics report populated | 20260508 05:11:45.865 | INFO | Opening tmp RTMetrics json file | 20260508 05:11:45.878 | INFO | Serializing RTMetrics to json | 20260508 05:11:45.879 | INFO | Serialization done, swapping tmp file now... | 20260508 05:11:45.880 | INFO | RTMetrics done | 20260508 05:11:45.941 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.49 seconds | 20260508 05:11:45.942 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.92% / 0.00% / 59.08% | 20260508 05:11:45.942 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.481665% / 3.97364e-06% / 7.94729e-05% | 20260508 05:11:45.942 | INFO | Baz IO overall: | 20260508 05:11:45.942 | INFO | BytesWritten : 2452567910800 | 20260508 05:11:45.942 | INFO | NumEvents : 1183504613885 | 20260508 05:11:47.476 | INFO | Analyzing chunk frames = [5496832,5497344) | 20260508 05:11:47.476 | INFO | MemInfo: { | 20260508 05:11:47.476 | INFO | "gpu" : " 17 GB", | 20260508 05:11:47.476 | INFO | "huge" : "", | 20260508 05:11:47.476 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:47.476 | INFO | "malloc" : " 14 GB", | 20260508 05:11:47.476 | INFO | "pinned" : " 29 GB" | 20260508 05:11:47.476 | INFO | } | 20260508 05:11:47.476 | INFO | Perf report for the last 5.13827 seconds: | 20260508 05:11:47.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:47.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:47.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:47.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:47.476 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 05:11:47.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:47.477 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 05:11:47.477 | INFO | PRE_HQ : 66.6%, 1.8, 62.3ms, 0 | 20260508 05:11:47.477 | INFO | BAZWRITER : 67.5%, 2.9, 102.0ms, 0 | 20260508 05:11:47.477 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:11:47.477 | INFO | Uploaded 36GiB to GPU in 2.58512s (13.9259GiB/s) | 20260508 05:11:47.477 | INFO | Downloaded 38GiB to GPU in 2.7018s (14.0647GiB/s) | 20260508 05:11:52.619 | INFO | Analyzing chunk frames = [5497344,5497856) | 20260508 05:11:52.619 | INFO | MemInfo: { | 20260508 05:11:52.619 | INFO | "gpu" : " 17 GB", | 20260508 05:11:52.619 | INFO | "huge" : "", | 20260508 05:11:52.619 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:52.619 | INFO | "malloc" : " 14 GB", | 20260508 05:11:52.619 | INFO | "pinned" : " 29 GB" | 20260508 05:11:52.619 | INFO | } | 20260508 05:11:52.619 | INFO | Perf report for the last 5.14276 seconds: | 20260508 05:11:52.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:52.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 05:11:52.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:11:52.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:52.619 | INFO | Uploaded 36GiB to GPU in 2.58616s (13.9202GiB/s) | 20260508 05:11:52.619 | INFO | Downloaded 34GiB to GPU in 2.43477s (13.9644GiB/s) | 20260508 05:11:57.768 | INFO | Analyzing chunk frames = [5497856,5498368) | 20260508 05:11:57.768 | INFO | MemInfo: { | 20260508 05:11:57.768 | INFO | "gpu" : " 17 GB", | 20260508 05:11:57.768 | INFO | "huge" : "", | 20260508 05:11:57.768 | INFO | "huge_shared" : " 29 GB", | 20260508 05:11:57.768 | INFO | "malloc" : " 14 GB", | 20260508 05:11:57.768 | INFO | "pinned" : " 29 GB" | 20260508 05:11:57.768 | INFO | } | 20260508 05:11:57.768 | INFO | Perf report for the last 5.14935 seconds: | 20260508 05:11:57.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:11:57.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.768 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 05:11:57.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.769 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:11:57.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:11:57.769 | INFO | Uploaded 36GiB to GPU in 2.56122s (14.0558GiB/s) | 20260508 05:11:57.769 | INFO | Downloaded 34GiB to GPU in 2.42309s (14.0317GiB/s) | 20260508 05:12:02.902 | INFO | Analyzing chunk frames = [5498368,5498880) | 20260508 05:12:02.902 | INFO | MemInfo: { | 20260508 05:12:02.902 | INFO | "gpu" : " 17 GB", | 20260508 05:12:02.902 | INFO | "huge" : "", | 20260508 05:12:02.902 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:02.902 | INFO | "malloc" : " 14 GB", | 20260508 05:12:02.902 | INFO | "pinned" : " 29 GB" | 20260508 05:12:02.902 | INFO | } | 20260508 05:12:02.902 | INFO | Perf report for the last 5.13344 seconds: | 20260508 05:12:02.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:02.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | ANALYSIS : 67.0%, 3.0, 103.5ms, 0 | 20260508 05:12:02.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:12:02.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:02.902 | INFO | Uploaded 36GiB to GPU in 2.5817s (13.9443GiB/s) | 20260508 05:12:02.902 | INFO | Downloaded 34GiB to GPU in 2.38978s (14.2273GiB/s) | 20260508 05:12:08.035 | INFO | Analyzing chunk frames = [5498880,5499392) | 20260508 05:12:08.035 | INFO | MemInfo: { | 20260508 05:12:08.035 | INFO | "gpu" : " 17 GB", | 20260508 05:12:08.035 | INFO | "huge" : "", | 20260508 05:12:08.035 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:08.035 | INFO | "malloc" : " 14 GB", | 20260508 05:12:08.035 | INFO | "pinned" : " 29 GB" | 20260508 05:12:08.035 | INFO | } | 20260508 05:12:08.035 | INFO | Perf report for the last 5.13278 seconds: | 20260508 05:12:08.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:08.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.035 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 05:12:08.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.035 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:12:08.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:08.036 | INFO | Uploaded 36GiB to GPU in 2.48719s (14.4742GiB/s) | 20260508 05:12:08.036 | INFO | Downloaded 34GiB to GPU in 2.5278s (13.4504GiB/s) | 20260508 05:12:13.174 | INFO | Analyzing chunk frames = [5499392,5499904) | 20260508 05:12:13.174 | INFO | MemInfo: { | 20260508 05:12:13.174 | INFO | "gpu" : " 17 GB", | 20260508 05:12:13.174 | INFO | "huge" : "", | 20260508 05:12:13.174 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:13.174 | INFO | "malloc" : " 14 GB", | 20260508 05:12:13.174 | INFO | "pinned" : " 29 GB" | 20260508 05:12:13.174 | INFO | } | 20260508 05:12:13.174 | INFO | Perf report for the last 5.13934 seconds: | 20260508 05:12:13.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:13.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:12:13.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:12:13.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:13.174 | INFO | Uploaded 36GiB to GPU in 2.44861s (14.7022GiB/s) | 20260508 05:12:13.174 | INFO | Downloaded 34GiB to GPU in 2.42168s (14.0398GiB/s) | 20260508 05:12:18.315 | INFO | Analyzing chunk frames = [5499904,5500416) | 20260508 05:12:18.315 | INFO | MemInfo: { | 20260508 05:12:18.315 | INFO | "gpu" : " 17 GB", | 20260508 05:12:18.315 | INFO | "huge" : "", | 20260508 05:12:18.316 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:18.316 | INFO | "malloc" : " 14 GB", | 20260508 05:12:18.316 | INFO | "pinned" : " 29 GB" | 20260508 05:12:18.316 | INFO | } | 20260508 05:12:18.316 | INFO | Perf report for the last 5.14187 seconds: | 20260508 05:12:18.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:18.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.316 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:12:18.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.316 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 05:12:18.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:18.317 | INFO | Uploaded 36GiB to GPU in 2.45091s (14.6884GiB/s) | 20260508 05:12:18.317 | INFO | Downloaded 34GiB to GPU in 2.43098s (13.9862GiB/s) | 20260508 05:12:23.457 | INFO | Analyzing chunk frames = [5500416,5500928) | 20260508 05:12:23.457 | INFO | MemInfo: { | 20260508 05:12:23.457 | INFO | "gpu" : " 17 GB", | 20260508 05:12:23.457 | INFO | "huge" : "", | 20260508 05:12:23.457 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:23.457 | INFO | "malloc" : " 14 GB", | 20260508 05:12:23.457 | INFO | "pinned" : " 29 GB" | 20260508 05:12:23.457 | INFO | } | 20260508 05:12:23.457 | INFO | Perf report for the last 5.14124 seconds: | 20260508 05:12:23.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:23.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.457 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:12:23.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.458 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260508 05:12:23.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:23.458 | INFO | Uploaded 36GiB to GPU in 2.47187s (14.5638GiB/s) | 20260508 05:12:23.458 | INFO | Downloaded 34GiB to GPU in 2.4421s (13.9224GiB/s) | 20260508 05:12:23.566 | INFO | Starting a round of Baz IO | 20260508 05:12:26.984 | INFO | Populating RTMetrics report | 20260508 05:12:26.984 | INFO | RTMetrics report populated | 20260508 05:12:26.984 | INFO | Opening tmp RTMetrics json file | 20260508 05:12:26.992 | INFO | Serializing RTMetrics to json | 20260508 05:12:26.999 | INFO | Serialization done, swapping tmp file now... | 20260508 05:12:26.999 | INFO | RTMetrics done | 20260508 05:12:27.061 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.495 seconds | 20260508 05:12:27.061 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.91% / 0.00% / 59.08% | 20260508 05:12:27.061 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.481164% / 3.97364e-06% / 5.96046e-05% | 20260508 05:12:27.061 | INFO | Baz IO overall: | 20260508 05:12:27.061 | INFO | BytesWritten : 2454264564430 | 20260508 05:12:27.061 | INFO | NumEvents : 1184314305134 | 20260508 05:12:28.599 | INFO | Analyzing chunk frames = [5500928,5501440) | 20260508 05:12:28.599 | INFO | MemInfo: { | 20260508 05:12:28.599 | INFO | "gpu" : " 17 GB", | 20260508 05:12:28.599 | INFO | "huge" : "", | 20260508 05:12:28.599 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:28.599 | INFO | "malloc" : " 14 GB", | 20260508 05:12:28.599 | INFO | "pinned" : " 29 GB" | 20260508 05:12:28.599 | INFO | } | 20260508 05:12:28.599 | INFO | Perf report for the last 5.1416 seconds: | 20260508 05:12:28.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:28.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:28.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:28.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:28.599 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 05:12:28.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:28.599 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 05:12:28.599 | INFO | PRE_HQ : 66.4%, 1.8, 61.6ms, 0 | 20260508 05:12:28.599 | INFO | BAZWRITER : 67.4%, 2.9, 101.2ms, 0 | 20260508 05:12:28.599 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:12:28.599 | INFO | Uploaded 36GiB to GPU in 2.56471s (14.0367GiB/s) | 20260508 05:12:28.599 | INFO | Downloaded 38GiB to GPU in 2.70516s (14.0472GiB/s) | 20260508 05:12:33.734 | INFO | Analyzing chunk frames = [5501440,5501952) | 20260508 05:12:33.734 | INFO | MemInfo: { | 20260508 05:12:33.734 | INFO | "gpu" : " 17 GB", | 20260508 05:12:33.734 | INFO | "huge" : "", | 20260508 05:12:33.734 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:33.734 | INFO | "malloc" : " 14 GB", | 20260508 05:12:33.734 | INFO | "pinned" : " 29 GB" | 20260508 05:12:33.734 | INFO | } | 20260508 05:12:33.734 | INFO | Perf report for the last 5.13534 seconds: | 20260508 05:12:33.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:33.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 05:12:33.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:12:33.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:33.734 | INFO | Uploaded 36GiB to GPU in 2.49497s (14.429GiB/s) | 20260508 05:12:33.735 | INFO | Downloaded 34GiB to GPU in 2.44075s (13.9301GiB/s) | 20260508 05:12:38.875 | INFO | Analyzing chunk frames = [5501952,5502464) | 20260508 05:12:38.875 | INFO | MemInfo: { | 20260508 05:12:38.875 | INFO | "gpu" : " 17 GB", | 20260508 05:12:38.875 | INFO | "huge" : "", | 20260508 05:12:38.875 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:38.875 | INFO | "malloc" : " 14 GB", | 20260508 05:12:38.875 | INFO | "pinned" : " 29 GB" | 20260508 05:12:38.875 | INFO | } | 20260508 05:12:38.875 | INFO | Perf report for the last 5.14115 seconds: | 20260508 05:12:38.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:38.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.875 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 05:12:38.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.875 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:12:38.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:38.876 | INFO | Uploaded 36GiB to GPU in 2.54042s (14.1709GiB/s) | 20260508 05:12:38.876 | INFO | Downloaded 34GiB to GPU in 2.4159s (14.0734GiB/s) | 20260508 05:12:44.016 | INFO | Analyzing chunk frames = [5502464,5502976) | 20260508 05:12:44.016 | INFO | MemInfo: { | 20260508 05:12:44.016 | INFO | "gpu" : " 17 GB", | 20260508 05:12:44.016 | INFO | "huge" : "", | 20260508 05:12:44.016 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:44.016 | INFO | "malloc" : " 14 GB", | 20260508 05:12:44.016 | INFO | "pinned" : " 29 GB" | 20260508 05:12:44.016 | INFO | } | 20260508 05:12:44.016 | INFO | Perf report for the last 5.14062 seconds: | 20260508 05:12:44.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:44.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 05:12:44.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:12:44.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:44.016 | INFO | Uploaded 36GiB to GPU in 2.54639s (14.1377GiB/s) | 20260508 05:12:44.016 | INFO | Downloaded 34GiB to GPU in 2.39577s (14.1917GiB/s) | 20260508 05:12:49.154 | INFO | Analyzing chunk frames = [5502976,5503488) | 20260508 05:12:49.154 | INFO | MemInfo: { | 20260508 05:12:49.154 | INFO | "gpu" : " 17 GB", | 20260508 05:12:49.154 | INFO | "huge" : "", | 20260508 05:12:49.154 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:49.154 | INFO | "malloc" : " 14 GB", | 20260508 05:12:49.154 | INFO | "pinned" : " 29 GB" | 20260508 05:12:49.154 | INFO | } | 20260508 05:12:49.154 | INFO | Perf report for the last 5.13843 seconds: | 20260508 05:12:49.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:49.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.154 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:12:49.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.154 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:12:49.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:49.155 | INFO | Uploaded 36GiB to GPU in 2.31521s (15.5493GiB/s) | 20260508 05:12:49.155 | INFO | Downloaded 34GiB to GPU in 2.46611s (13.7869GiB/s) | 20260508 05:12:54.286 | INFO | Analyzing chunk frames = [5503488,5504000) | 20260508 05:12:54.286 | INFO | MemInfo: { | 20260508 05:12:54.286 | INFO | "gpu" : " 17 GB", | 20260508 05:12:54.286 | INFO | "huge" : "", | 20260508 05:12:54.286 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:54.287 | INFO | "malloc" : " 14 GB", | 20260508 05:12:54.287 | INFO | "pinned" : " 29 GB" | 20260508 05:12:54.287 | INFO | } | 20260508 05:12:54.287 | INFO | Perf report for the last 5.13216 seconds: | 20260508 05:12:54.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:54.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:12:54.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:12:54.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:54.287 | INFO | Uploaded 36GiB to GPU in 2.45404s (14.6697GiB/s) | 20260508 05:12:54.287 | INFO | Downloaded 34GiB to GPU in 2.41154s (14.0989GiB/s) | 20260508 05:12:59.427 | INFO | Analyzing chunk frames = [5504000,5504512) | 20260508 05:12:59.427 | INFO | MemInfo: { | 20260508 05:12:59.427 | INFO | "gpu" : " 17 GB", | 20260508 05:12:59.427 | INFO | "huge" : "", | 20260508 05:12:59.427 | INFO | "huge_shared" : " 29 GB", | 20260508 05:12:59.427 | INFO | "malloc" : " 14 GB", | 20260508 05:12:59.428 | INFO | "pinned" : " 29 GB" | 20260508 05:12:59.428 | INFO | } | 20260508 05:12:59.428 | INFO | Perf report for the last 5.14126 seconds: | 20260508 05:12:59.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:12:59.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:12:59.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 05:12:59.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:12:59.428 | INFO | Uploaded 36GiB to GPU in 2.48884s (14.4646GiB/s) | 20260508 05:12:59.428 | INFO | Downloaded 34GiB to GPU in 2.43069s (13.9878GiB/s) | 20260508 05:13:04.571 | INFO | Analyzing chunk frames = [5504512,5505024) | 20260508 05:13:04.571 | INFO | MemInfo: { | 20260508 05:13:04.571 | INFO | "gpu" : " 17 GB", | 20260508 05:13:04.572 | INFO | "huge" : "", | 20260508 05:13:04.572 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:04.572 | INFO | "malloc" : " 14 GB", | 20260508 05:13:04.572 | INFO | "pinned" : " 29 GB" | 20260508 05:13:04.572 | INFO | } | 20260508 05:13:04.572 | INFO | Perf report for the last 5.1438 seconds: | 20260508 05:13:04.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:04.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:13:04.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260508 05:13:04.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:04.572 | INFO | Uploaded 36GiB to GPU in 2.46588s (14.5992GiB/s) | 20260508 05:13:04.572 | INFO | Downloaded 34GiB to GPU in 2.42328s (14.0306GiB/s) | 20260508 05:13:04.572 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:13:04.572 | INFO | Total mem usage: 16.7 GiB | 20260508 05:13:04.572 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:13:04.572 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:13:04.572 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:13:04.572 | INFO | High Water Breakdown: | 20260508 05:13:04.572 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:13:04.572 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:13:04.572 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:13:04.572 | INFO | ------------------------------------------ | 20260508 05:13:04.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:13:04.572 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:13:04.572 | INFO | Total mem usage: 26.5 GiB | 20260508 05:13:04.572 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:13:04.572 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:13:04.572 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:13:04.572 | INFO | High Water Breakdown: | 20260508 05:13:04.572 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:13:04.572 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:13:04.572 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:13:04.572 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:13:04.572 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:13:04.572 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:13:04.572 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:13:04.572 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:13:04.572 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:13:04.572 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:13:04.572 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:13:04.572 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:13:04.572 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:13:04.572 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:13:04.572 | INFO | ------------------------------------------ | 20260508 05:13:04.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:13:04.572 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:13:04.572 | INFO | Total mem usage: 27.8 GiB | 20260508 05:13:04.572 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:13:04.572 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:13:04.572 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:13:04.572 | INFO | High Water Breakdown: | 20260508 05:13:04.572 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:13:04.572 | INFO | ------------------------------------------ | 20260508 05:13:04.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:13:04.572 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:13:04.572 | INFO | Total mem usage: 15.7 GiB | 20260508 05:13:04.572 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:13:04.572 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:13:04.572 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:13:04.572 | INFO | High Water Breakdown: | 20260508 05:13:04.572 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:13:04.572 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:13:04.572 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:13:04.572 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:13:04.572 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:13:04.572 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:13:04.572 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:13:04.572 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:13:04.572 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:13:04.572 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:13:04.572 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:13:04.572 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:13:04.572 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:13:04.572 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:13:04.572 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:13:04.572 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:13:04.572 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:13:04.572 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:13:04.572 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:13:04.572 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:13:04.572 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:13:04.572 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:13:04.572 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:13:04.572 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:13:04.572 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:13:04.572 | INFO | ------------------------------------------ | 20260508 05:13:04.572 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:13:04.572 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:13:04.572 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:13:04.572 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:13:04.572 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.38e+01 / 6.62e-01 | 20260508 05:13:04.572 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.23e-02 / 5.90e-03 | 20260508 05:13:04.572 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.42e+00, 5.90e-01 / 1.94e-01 | 20260508 05:13:04.572 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.26e-01 | 20260508 05:13:04.572 | INFO | PerfReport: 15.88%, DME -- 4.16e+01 / 4.48e+01, 3.68e+00 / 1.06e+00 | 20260508 05:13:04.572 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.82e+00 / 1.42e+00 | 20260508 05:13:04.572 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.39e+00 / 2.53e+00, 4.16e-01 / 2.46e-01 | 20260508 05:13:04.572 | INFO | PerfReport: 3.19%, Metrics -- 1.05e+00 / 1.18e+00, 2.62e-01 / 1.25e-01 | 20260508 05:13:04.572 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.21e-01 / 3.59e-01, 5.11e-02 / 2.54e-02 | 20260508 05:13:04.572 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:13:04.572 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:13:04.572 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:13:04.572 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 05:13:04.572 | INFO | PerfReport: 24.53%, Upload -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:13:04.572 | INFO | PerfReport: 24.17%, Download -- 2.57e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 05:13:04.572 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 05:13:04.572 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 05:13:04.573 | INFO | PerfReport: 2.46%, LockDownload -- 2.61e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260508 05:13:04.573 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:13:04.573 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 05:13:04.669 | INFO | Starting a round of Baz IO | 20260508 05:13:08.094 | INFO | Populating RTMetrics report | 20260508 05:13:08.094 | INFO | RTMetrics report populated | 20260508 05:13:08.094 | INFO | Opening tmp RTMetrics json file | 20260508 05:13:08.108 | INFO | Serializing RTMetrics to json | 20260508 05:13:08.109 | INFO | Serialization done, swapping tmp file now... | 20260508 05:13:08.110 | INFO | RTMetrics done | 20260508 05:13:08.173 | INFO | Finished a round of Baz IO: Wrote 1.581 GiB in 3.504 seconds | 20260508 05:13:08.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.91% / 0.00% / 59.09% | 20260508 05:13:08.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.480676% / 3.97364e-06% / 4.37101e-05% | 20260508 05:13:08.173 | INFO | Baz IO overall: | 20260508 05:13:08.173 | INFO | BytesWritten : 2455961834737 | 20260508 05:13:08.173 | INFO | NumEvents : 1185124541344 | 20260508 05:13:09.714 | INFO | Analyzing chunk frames = [5505024,5505536) | 20260508 05:13:09.714 | INFO | MemInfo: { | 20260508 05:13:09.714 | INFO | "gpu" : " 17 GB", | 20260508 05:13:09.714 | INFO | "huge" : "", | 20260508 05:13:09.714 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:09.714 | INFO | "malloc" : " 14 GB", | 20260508 05:13:09.715 | INFO | "pinned" : " 29 GB" | 20260508 05:13:09.715 | INFO | } | 20260508 05:13:09.715 | INFO | Perf report for the last 5.14292 seconds: | 20260508 05:13:09.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:09.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:09.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:09.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:09.715 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 05:13:09.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:09.715 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:13:09.715 | INFO | PRE_HQ : 65.4%, 1.6, 55.4ms, 0 | 20260508 05:13:09.715 | INFO | BAZWRITER : 67.6%, 2.7, 94.9ms, 0 | 20260508 05:13:09.715 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:13:09.715 | INFO | Uploaded 36GiB to GPU in 2.55551s (14.0872GiB/s) | 20260508 05:13:09.715 | INFO | Downloaded 38GiB to GPU in 2.70236s (14.0618GiB/s) | 20260508 05:13:14.851 | INFO | Analyzing chunk frames = [5505536,5506048) | 20260508 05:13:14.851 | INFO | MemInfo: { | 20260508 05:13:14.851 | INFO | "gpu" : " 17 GB", | 20260508 05:13:14.851 | INFO | "huge" : "", | 20260508 05:13:14.851 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:14.851 | INFO | "malloc" : " 14 GB", | 20260508 05:13:14.851 | INFO | "pinned" : " 29 GB" | 20260508 05:13:14.851 | INFO | } | 20260508 05:13:14.851 | INFO | Perf report for the last 5.13629 seconds: | 20260508 05:13:14.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:14.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:13:14.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:13:14.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:14.851 | INFO | Uploaded 36GiB to GPU in 2.51192s (14.3317GiB/s) | 20260508 05:13:14.851 | INFO | Downloaded 34GiB to GPU in 2.4226s (14.0345GiB/s) | 20260508 05:13:19.986 | INFO | Analyzing chunk frames = [5506048,5506560) | 20260508 05:13:19.986 | INFO | MemInfo: { | 20260508 05:13:19.986 | INFO | "gpu" : " 17 GB", | 20260508 05:13:19.986 | INFO | "huge" : "", | 20260508 05:13:19.986 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:19.986 | INFO | "malloc" : " 14 GB", | 20260508 05:13:19.986 | INFO | "pinned" : " 29 GB" | 20260508 05:13:19.986 | INFO | } | 20260508 05:13:19.986 | INFO | Perf report for the last 5.13549 seconds: | 20260508 05:13:19.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:19.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.986 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 05:13:19.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.987 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:13:19.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:19.987 | INFO | Uploaded 36GiB to GPU in 2.54508s (14.1449GiB/s) | 20260508 05:13:19.987 | INFO | Downloaded 34GiB to GPU in 2.40283s (14.15GiB/s) | 20260508 05:13:25.133 | INFO | Analyzing chunk frames = [5506560,5507072) | 20260508 05:13:25.133 | INFO | MemInfo: { | 20260508 05:13:25.133 | INFO | "gpu" : " 17 GB", | 20260508 05:13:25.133 | INFO | "huge" : "", | 20260508 05:13:25.133 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:25.133 | INFO | "malloc" : " 14 GB", | 20260508 05:13:25.133 | INFO | "pinned" : " 29 GB" | 20260508 05:13:25.133 | INFO | } | 20260508 05:13:25.134 | INFO | Perf report for the last 5.14712 seconds: | 20260508 05:13:25.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:25.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 05:13:25.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:13:25.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:25.134 | INFO | Uploaded 36GiB to GPU in 2.55488s (14.0907GiB/s) | 20260508 05:13:25.134 | INFO | Downloaded 34GiB to GPU in 2.39114s (14.2191GiB/s) | 20260508 05:13:30.271 | INFO | Analyzing chunk frames = [5507072,5507584) | 20260508 05:13:30.271 | INFO | MemInfo: { | 20260508 05:13:30.271 | INFO | "gpu" : " 17 GB", | 20260508 05:13:30.271 | INFO | "huge" : "", | 20260508 05:13:30.271 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:30.271 | INFO | "malloc" : " 14 GB", | 20260508 05:13:30.271 | INFO | "pinned" : " 29 GB" | 20260508 05:13:30.271 | INFO | } | 20260508 05:13:30.271 | INFO | Perf report for the last 5.13772 seconds: | 20260508 05:13:30.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:30.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.271 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:13:30.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.271 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:13:30.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:30.272 | INFO | Uploaded 36GiB to GPU in 2.29809s (15.6652GiB/s) | 20260508 05:13:30.272 | INFO | Downloaded 34GiB to GPU in 2.48655s (13.6735GiB/s) | 20260508 05:13:35.413 | INFO | Analyzing chunk frames = [5507584,5508096) | 20260508 05:13:35.413 | INFO | MemInfo: { | 20260508 05:13:35.413 | INFO | "gpu" : " 17 GB", | 20260508 05:13:35.413 | INFO | "huge" : "", | 20260508 05:13:35.414 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:35.414 | INFO | "malloc" : " 14 GB", | 20260508 05:13:35.414 | INFO | "pinned" : " 29 GB" | 20260508 05:13:35.414 | INFO | } | 20260508 05:13:35.414 | INFO | Perf report for the last 5.14243 seconds: | 20260508 05:13:35.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:35.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:13:35.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:13:35.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:35.414 | INFO | Uploaded 36GiB to GPU in 2.45256s (14.6785GiB/s) | 20260508 05:13:35.414 | INFO | Downloaded 34GiB to GPU in 2.41708s (14.0666GiB/s) | 20260508 05:13:40.545 | INFO | Analyzing chunk frames = [5508096,5508608) | 20260508 05:13:40.545 | INFO | MemInfo: { | 20260508 05:13:40.545 | INFO | "gpu" : " 17 GB", | 20260508 05:13:40.545 | INFO | "huge" : "", | 20260508 05:13:40.545 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:40.545 | INFO | "malloc" : " 14 GB", | 20260508 05:13:40.545 | INFO | "pinned" : " 29 GB" | 20260508 05:13:40.545 | INFO | } | 20260508 05:13:40.545 | INFO | Perf report for the last 5.13162 seconds: | 20260508 05:13:40.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:40.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.545 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:13:40.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.545 | INFO | PRE_HQ : 25.4%, 1.0, 13.1ms, 0 | 20260508 05:13:40.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:40.546 | INFO | Uploaded 36GiB to GPU in 2.48408s (14.4923GiB/s) | 20260508 05:13:40.546 | INFO | Downloaded 34GiB to GPU in 2.42667s (14.011GiB/s) | 20260508 05:13:45.687 | INFO | Analyzing chunk frames = [5508608,5509120) | 20260508 05:13:45.687 | INFO | MemInfo: { | 20260508 05:13:45.687 | INFO | "gpu" : " 17 GB", | 20260508 05:13:45.687 | INFO | "huge" : "", | 20260508 05:13:45.687 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:45.687 | INFO | "malloc" : " 14 GB", | 20260508 05:13:45.687 | INFO | "pinned" : " 29 GB" | 20260508 05:13:45.687 | INFO | } | 20260508 05:13:45.687 | INFO | Perf report for the last 5.14168 seconds: | 20260508 05:13:45.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:45.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 05:13:45.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:13:45.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:45.687 | INFO | Uploaded 36GiB to GPU in 2.49877s (14.4071GiB/s) | 20260508 05:13:45.687 | INFO | Downloaded 34GiB to GPU in 2.42129s (14.0421GiB/s) | 20260508 05:13:45.784 | INFO | Starting a round of Baz IO | 20260508 05:13:49.213 | INFO | Populating RTMetrics report | 20260508 05:13:49.214 | INFO | RTMetrics report populated | 20260508 05:13:49.214 | INFO | Opening tmp RTMetrics json file | 20260508 05:13:49.224 | INFO | Serializing RTMetrics to json | 20260508 05:13:49.225 | INFO | Serialization done, swapping tmp file now... | 20260508 05:13:49.226 | INFO | RTMetrics done | 20260508 05:13:49.290 | INFO | Finished a round of Baz IO: Wrote 1.579 GiB in 3.505 seconds | 20260508 05:13:49.290 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.91% / 0.00% / 59.09% | 20260508 05:13:49.290 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.478323% / 0% / 5.5631e-05% | 20260508 05:13:49.290 | INFO | Baz IO overall: | 20260508 05:13:49.290 | INFO | BytesWritten : 2457657075920 | 20260508 05:13:49.290 | INFO | NumEvents : 1185933222244 | 20260508 05:13:50.829 | INFO | Analyzing chunk frames = [5509120,5509632) | 20260508 05:13:50.829 | INFO | MemInfo: { | 20260508 05:13:50.829 | INFO | "gpu" : " 17 GB", | 20260508 05:13:50.829 | INFO | "huge" : "", | 20260508 05:13:50.829 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:50.829 | INFO | "malloc" : " 14 GB", | 20260508 05:13:50.829 | INFO | "pinned" : " 29 GB" | 20260508 05:13:50.829 | INFO | } | 20260508 05:13:50.829 | INFO | Perf report for the last 5.14198 seconds: | 20260508 05:13:50.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:50.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:50.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:50.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:50.829 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 05:13:50.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:50.829 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 05:13:50.829 | INFO | PRE_HQ : 65.9%, 1.6, 55.8ms, 0 | 20260508 05:13:50.829 | INFO | BAZWRITER : 67.6%, 2.7, 94.5ms, 0 | 20260508 05:13:50.829 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:13:50.830 | INFO | Uploaded 36GiB to GPU in 2.57859s (13.9611GiB/s) | 20260508 05:13:50.830 | INFO | Downloaded 38GiB to GPU in 2.7096s (14.0242GiB/s) | 20260508 05:13:55.971 | INFO | Analyzing chunk frames = [5509632,5510144) | 20260508 05:13:55.971 | INFO | MemInfo: { | 20260508 05:13:55.971 | INFO | "gpu" : " 17 GB", | 20260508 05:13:55.971 | INFO | "huge" : "", | 20260508 05:13:55.971 | INFO | "huge_shared" : " 29 GB", | 20260508 05:13:55.971 | INFO | "malloc" : " 14 GB", | 20260508 05:13:55.972 | INFO | "pinned" : " 29 GB" | 20260508 05:13:55.972 | INFO | } | 20260508 05:13:55.972 | INFO | Perf report for the last 5.14267 seconds: | 20260508 05:13:55.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:13:55.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 05:13:55.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:13:55.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:13:55.972 | INFO | Uploaded 36GiB to GPU in 2.49269s (14.4422GiB/s) | 20260508 05:13:55.972 | INFO | Downloaded 34GiB to GPU in 2.44891s (13.8837GiB/s) | 20260508 05:14:01.099 | INFO | Analyzing chunk frames = [5510144,5510656) | 20260508 05:14:01.099 | INFO | MemInfo: { | 20260508 05:14:01.099 | INFO | "gpu" : " 17 GB", | 20260508 05:14:01.099 | INFO | "huge" : "", | 20260508 05:14:01.099 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:01.099 | INFO | "malloc" : " 14 GB", | 20260508 05:14:01.099 | INFO | "pinned" : " 29 GB" | 20260508 05:14:01.099 | INFO | } | 20260508 05:14:01.100 | INFO | Perf report for the last 5.12781 seconds: | 20260508 05:14:01.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:01.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 05:14:01.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:14:01.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:01.100 | INFO | Uploaded 36GiB to GPU in 2.54081s (14.1687GiB/s) | 20260508 05:14:01.100 | INFO | Downloaded 34GiB to GPU in 2.42376s (14.0278GiB/s) | 20260508 05:14:06.237 | INFO | Analyzing chunk frames = [5510656,5511168) | 20260508 05:14:06.237 | INFO | MemInfo: { | 20260508 05:14:06.237 | INFO | "gpu" : " 17 GB", | 20260508 05:14:06.237 | INFO | "huge" : "", | 20260508 05:14:06.237 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:06.237 | INFO | "malloc" : " 14 GB", | 20260508 05:14:06.237 | INFO | "pinned" : " 29 GB" | 20260508 05:14:06.237 | INFO | } | 20260508 05:14:06.237 | INFO | Perf report for the last 5.13744 seconds: | 20260508 05:14:06.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:06.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | ANALYSIS : 67.2%, 3.0, 104.0ms, 0 | 20260508 05:14:06.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:14:06.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:06.237 | INFO | Uploaded 36GiB to GPU in 2.58688s (13.9164GiB/s) | 20260508 05:14:06.237 | INFO | Downloaded 34GiB to GPU in 2.43364s (13.9708GiB/s) | 20260508 05:14:11.381 | INFO | Analyzing chunk frames = [5511168,5511680) | 20260508 05:14:11.381 | INFO | MemInfo: { | 20260508 05:14:11.381 | INFO | "gpu" : " 17 GB", | 20260508 05:14:11.381 | INFO | "huge" : "", | 20260508 05:14:11.381 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:11.381 | INFO | "malloc" : " 14 GB", | 20260508 05:14:11.381 | INFO | "pinned" : " 29 GB" | 20260508 05:14:11.381 | INFO | } | 20260508 05:14:11.381 | INFO | Perf report for the last 5.14403 seconds: | 20260508 05:14:11.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:11.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.381 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 05:14:11.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.381 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260508 05:14:11.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:11.382 | INFO | Uploaded 36GiB to GPU in 2.37397s (15.1645GiB/s) | 20260508 05:14:11.382 | INFO | Downloaded 34GiB to GPU in 2.56096s (13.2763GiB/s) | 20260508 05:14:16.530 | INFO | Analyzing chunk frames = [5511680,5512192) | 20260508 05:14:16.530 | INFO | MemInfo: { | 20260508 05:14:16.530 | INFO | "gpu" : " 17 GB", | 20260508 05:14:16.530 | INFO | "huge" : "", | 20260508 05:14:16.530 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:16.530 | INFO | "malloc" : " 14 GB", | 20260508 05:14:16.531 | INFO | "pinned" : " 29 GB" | 20260508 05:14:16.531 | INFO | } | 20260508 05:14:16.531 | INFO | Perf report for the last 5.14963 seconds: | 20260508 05:14:16.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:16.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 05:14:16.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:14:16.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:16.531 | INFO | Uploaded 36GiB to GPU in 2.44076s (14.7495GiB/s) | 20260508 05:14:16.531 | INFO | Downloaded 34GiB to GPU in 2.42944s (13.995GiB/s) | 20260508 05:14:21.666 | INFO | Analyzing chunk frames = [5512192,5512704) | 20260508 05:14:21.666 | INFO | MemInfo: { | 20260508 05:14:21.666 | INFO | "gpu" : " 17 GB", | 20260508 05:14:21.667 | INFO | "huge" : "", | 20260508 05:14:21.667 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:21.667 | INFO | "malloc" : " 14 GB", | 20260508 05:14:21.667 | INFO | "pinned" : " 29 GB" | 20260508 05:14:21.667 | INFO | } | 20260508 05:14:21.667 | INFO | Perf report for the last 5.13604 seconds: | 20260508 05:14:21.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:21.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 05:14:21.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:14:21.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:21.667 | INFO | Uploaded 36GiB to GPU in 2.43741s (14.7698GiB/s) | 20260508 05:14:21.668 | INFO | Downloaded 34GiB to GPU in 2.43786s (13.9467GiB/s) | 20260508 05:14:26.798 | INFO | Analyzing chunk frames = [5512704,5513216) | 20260508 05:14:26.798 | INFO | MemInfo: { | 20260508 05:14:26.798 | INFO | "gpu" : " 17 GB", | 20260508 05:14:26.798 | INFO | "huge" : "", | 20260508 05:14:26.798 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:26.798 | INFO | "malloc" : " 14 GB", | 20260508 05:14:26.798 | INFO | "pinned" : " 29 GB" | 20260508 05:14:26.798 | INFO | } | 20260508 05:14:26.798 | INFO | Perf report for the last 5.13156 seconds: | 20260508 05:14:26.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:26.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.798 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:14:26.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.798 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:14:26.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:26.799 | INFO | Uploaded 36GiB to GPU in 2.43818s (14.7651GiB/s) | 20260508 05:14:26.799 | INFO | Downloaded 34GiB to GPU in 2.43725s (13.9502GiB/s) | 20260508 05:14:26.799 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:14:26.799 | INFO | Total mem usage: 16.7 GiB | 20260508 05:14:26.799 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:14:26.799 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.3%) | 20260508 05:14:26.799 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:14:26.799 | INFO | High Water Breakdown: | 20260508 05:14:26.799 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:14:26.799 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:14:26.799 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:14:26.799 | INFO | ------------------------------------------ | 20260508 05:14:26.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:14:26.799 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:14:26.799 | INFO | Total mem usage: 26.5 GiB | 20260508 05:14:26.799 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:14:26.799 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:14:26.799 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:14:26.799 | INFO | High Water Breakdown: | 20260508 05:14:26.799 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:14:26.799 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:14:26.799 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:14:26.799 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:14:26.799 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:14:26.799 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:14:26.799 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:14:26.799 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:14:26.799 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:14:26.799 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:14:26.799 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:14:26.799 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:14:26.799 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:14:26.799 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:14:26.799 | INFO | ------------------------------------------ | 20260508 05:14:26.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:14:26.799 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:14:26.799 | INFO | Total mem usage: 27.8 GiB | 20260508 05:14:26.799 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:14:26.800 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:14:26.800 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:14:26.800 | INFO | High Water Breakdown: | 20260508 05:14:26.800 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:14:26.800 | INFO | ------------------------------------------ | 20260508 05:14:26.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:14:26.800 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:14:26.800 | INFO | Total mem usage: 15.7 GiB | 20260508 05:14:26.800 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:14:26.800 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:14:26.800 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:14:26.800 | INFO | High Water Breakdown: | 20260508 05:14:26.800 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:14:26.800 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:14:26.800 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:14:26.800 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:14:26.800 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:14:26.800 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:14:26.800 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:14:26.800 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:14:26.800 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:14:26.800 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:14:26.800 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:14:26.800 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:14:26.800 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:14:26.800 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:14:26.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:14:26.800 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:14:26.800 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:14:26.800 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:14:26.800 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:14:26.800 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:14:26.800 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:14:26.800 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:14:26.800 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:14:26.800 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:14:26.800 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:14:26.800 | INFO | ------------------------------------------ | 20260508 05:14:26.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:14:26.800 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:14:26.800 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:14:26.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:14:26.800 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 1.22e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.19e-02 / 7.14e-03 | 20260508 05:14:26.800 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.47e+00, 5.93e-01 / 1.80e-01 | 20260508 05:14:26.800 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.24e-01 | 20260508 05:14:26.800 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.51e+01, 3.76e+00 / 1.42e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.77e+00 / 1.39e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.18e-01 / 2.29e-01 | 20260508 05:14:26.800 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.25e+00, 2.60e-01 / 1.78e-01 | 20260508 05:14:26.800 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.51e-01, 5.04e-02 / 2.29e-02 | 20260508 05:14:26.800 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:14:26.800 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:14:26.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:14:26.800 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 24.25%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 30.79%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 2.33%, LockUpload -- 2.47e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 2.57%, LockDownload -- 2.73e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:14:26.800 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.10e-02 / 0.00e+00 | 20260508 05:14:26.896 | INFO | Starting a round of Baz IO | 20260508 05:14:30.324 | INFO | Populating RTMetrics report | 20260508 05:14:30.324 | INFO | RTMetrics report populated | 20260508 05:14:30.324 | INFO | Opening tmp RTMetrics json file | 20260508 05:14:30.338 | INFO | Serializing RTMetrics to json | 20260508 05:14:30.339 | INFO | Serialization done, swapping tmp file now... | 20260508 05:14:30.340 | INFO | RTMetrics done | 20260508 05:14:30.401 | INFO | Finished a round of Baz IO: Wrote 1.58 GiB in 3.505 seconds | 20260508 05:14:30.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.91% / 0.00% / 59.09% | 20260508 05:14:30.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.479853% / 3.97364e-06% / 7.54992e-05% | 20260508 05:14:30.401 | INFO | Baz IO overall: | 20260508 05:14:30.401 | INFO | BytesWritten : 2459353234377 | 20260508 05:14:30.401 | INFO | NumEvents : 1186742687678 | 20260508 05:14:31.940 | INFO | Analyzing chunk frames = [5513216,5513728) | 20260508 05:14:31.940 | INFO | MemInfo: { | 20260508 05:14:31.940 | INFO | "gpu" : " 17 GB", | 20260508 05:14:31.940 | INFO | "huge" : "", | 20260508 05:14:31.940 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:31.940 | INFO | "malloc" : " 14 GB", | 20260508 05:14:31.940 | INFO | "pinned" : " 29 GB" | 20260508 05:14:31.940 | INFO | } | 20260508 05:14:31.940 | INFO | Perf report for the last 5.14188 seconds: | 20260508 05:14:31.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:31.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:31.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:31.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:31.940 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 05:14:31.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:31.940 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 05:14:31.940 | INFO | PRE_HQ : 65.7%, 1.6, 55.4ms, 0 | 20260508 05:14:31.940 | INFO | BAZWRITER : 67.7%, 2.7, 94.8ms, 0 | 20260508 05:14:31.940 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:14:31.940 | INFO | Uploaded 36GiB to GPU in 2.59709s (13.8617GiB/s) | 20260508 05:14:31.940 | INFO | Downloaded 38GiB to GPU in 2.71244s (14.0095GiB/s) | 20260508 05:14:37.078 | INFO | Analyzing chunk frames = [5513728,5514240) | 20260508 05:14:37.078 | INFO | MemInfo: { | 20260508 05:14:37.078 | INFO | "gpu" : " 17 GB", | 20260508 05:14:37.078 | INFO | "huge" : "", | 20260508 05:14:37.078 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:37.078 | INFO | "malloc" : " 14 GB", | 20260508 05:14:37.078 | INFO | "pinned" : " 29 GB" | 20260508 05:14:37.078 | INFO | } | 20260508 05:14:37.078 | INFO | Perf report for the last 5.13785 seconds: | 20260508 05:14:37.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:37.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 05:14:37.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:14:37.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:37.078 | INFO | Uploaded 36GiB to GPU in 2.48281s (14.4997GiB/s) | 20260508 05:14:37.078 | INFO | Downloaded 34GiB to GPU in 2.43573s (13.9589GiB/s) | 20260508 05:14:42.223 | INFO | Analyzing chunk frames = [5514240,5514752) | 20260508 05:14:42.223 | INFO | MemInfo: { | 20260508 05:14:42.223 | INFO | "gpu" : " 17 GB", | 20260508 05:14:42.223 | INFO | "huge" : "", | 20260508 05:14:42.223 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:42.223 | INFO | "malloc" : " 14 GB", | 20260508 05:14:42.223 | INFO | "pinned" : " 29 GB" | 20260508 05:14:42.223 | INFO | } | 20260508 05:14:42.223 | INFO | Perf report for the last 5.14509 seconds: | 20260508 05:14:42.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:42.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.223 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 05:14:42.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.223 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:14:42.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:42.224 | INFO | Uploaded 36GiB to GPU in 2.5335s (14.2096GiB/s) | 20260508 05:14:42.224 | INFO | Downloaded 34GiB to GPU in 2.41581s (14.074GiB/s) | 20260508 05:14:47.358 | INFO | Analyzing chunk frames = [5514752,5515264) | 20260508 05:14:47.358 | INFO | MemInfo: { | 20260508 05:14:47.358 | INFO | "gpu" : " 17 GB", | 20260508 05:14:47.358 | INFO | "huge" : "", | 20260508 05:14:47.358 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:47.358 | INFO | "malloc" : " 14 GB", | 20260508 05:14:47.358 | INFO | "pinned" : " 29 GB" | 20260508 05:14:47.358 | INFO | } | 20260508 05:14:47.358 | INFO | Perf report for the last 5.13484 seconds: | 20260508 05:14:47.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:47.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 05:14:47.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:14:47.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:47.358 | INFO | Uploaded 36GiB to GPU in 2.53021s (14.2281GiB/s) | 20260508 05:14:47.359 | INFO | Downloaded 34GiB to GPU in 2.39914s (14.1717GiB/s) | 20260508 05:14:52.497 | INFO | Analyzing chunk frames = [5515264,5515776) | 20260508 05:14:52.497 | INFO | MemInfo: { | 20260508 05:14:52.497 | INFO | "gpu" : " 17 GB", | 20260508 05:14:52.497 | INFO | "huge" : "", | 20260508 05:14:52.497 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:52.497 | INFO | "malloc" : " 14 GB", | 20260508 05:14:52.497 | INFO | "pinned" : " 29 GB" | 20260508 05:14:52.497 | INFO | } | 20260508 05:14:52.497 | INFO | Perf report for the last 5.13911 seconds: | 20260508 05:14:52.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:52.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 05:14:52.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:14:52.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:52.497 | INFO | Uploaded 36GiB to GPU in 2.38686s (15.0826GiB/s) | 20260508 05:14:52.497 | INFO | Downloaded 34GiB to GPU in 2.49423s (13.6315GiB/s) | 20260508 05:14:57.640 | INFO | Analyzing chunk frames = [5515776,5516288) | 20260508 05:14:57.640 | INFO | MemInfo: { | 20260508 05:14:57.640 | INFO | "gpu" : " 17 GB", | 20260508 05:14:57.640 | INFO | "huge" : "", | 20260508 05:14:57.640 | INFO | "huge_shared" : " 29 GB", | 20260508 05:14:57.640 | INFO | "malloc" : " 14 GB", | 20260508 05:14:57.640 | INFO | "pinned" : " 29 GB" | 20260508 05:14:57.640 | INFO | } | 20260508 05:14:57.640 | INFO | Perf report for the last 5.14289 seconds: | 20260508 05:14:57.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:14:57.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:14:57.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:14:57.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:14:57.640 | INFO | Uploaded 36GiB to GPU in 2.43428s (14.7888GiB/s) | 20260508 05:14:57.640 | INFO | Downloaded 34GiB to GPU in 2.42984s (13.9927GiB/s) | 20260508 05:15:02.779 | INFO | Analyzing chunk frames = [5516288,5516800) | 20260508 05:15:02.779 | INFO | MemInfo: { | 20260508 05:15:02.779 | INFO | "gpu" : " 17 GB", | 20260508 05:15:02.779 | INFO | "huge" : "", | 20260508 05:15:02.779 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:02.779 | INFO | "malloc" : " 14 GB", | 20260508 05:15:02.779 | INFO | "pinned" : " 29 GB" | 20260508 05:15:02.779 | INFO | } | 20260508 05:15:02.779 | INFO | Perf report for the last 5.13891 seconds: | 20260508 05:15:02.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:02.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:15:02.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:15:02.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:02.779 | INFO | Uploaded 36GiB to GPU in 2.44584s (14.7189GiB/s) | 20260508 05:15:02.779 | INFO | Downloaded 34GiB to GPU in 2.43334s (13.9726GiB/s) | 20260508 05:15:07.916 | INFO | Analyzing chunk frames = [5516800,5517312) | 20260508 05:15:07.916 | INFO | MemInfo: { | 20260508 05:15:07.916 | INFO | "gpu" : " 17 GB", | 20260508 05:15:07.916 | INFO | "huge" : "", | 20260508 05:15:07.916 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:07.916 | INFO | "malloc" : " 14 GB", | 20260508 05:15:07.916 | INFO | "pinned" : " 29 GB" | 20260508 05:15:07.916 | INFO | } | 20260508 05:15:07.916 | INFO | Perf report for the last 5.1374 seconds: | 20260508 05:15:07.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:07.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.916 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:15:07.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.916 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:15:07.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:07.917 | INFO | Uploaded 36GiB to GPU in 2.43634s (14.7762GiB/s) | 20260508 05:15:07.917 | INFO | Downloaded 34GiB to GPU in 2.42789s (14.0039GiB/s) | 20260508 05:15:08.014 | INFO | Starting a round of Baz IO | 20260508 05:15:11.448 | INFO | Populating RTMetrics report | 20260508 05:15:11.448 | INFO | RTMetrics report populated | 20260508 05:15:11.448 | INFO | Opening tmp RTMetrics json file | 20260508 05:15:11.449 | INFO | Serializing RTMetrics to json | 20260508 05:15:11.450 | INFO | Serialization done, swapping tmp file now... | 20260508 05:15:11.451 | INFO | RTMetrics done | 20260508 05:15:11.516 | INFO | Finished a round of Baz IO: Wrote 1.578 GiB in 3.501 seconds | 20260508 05:15:11.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.90% / 0.00% / 59.09% | 20260508 05:15:11.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.476587% / 1.19209e-05% / 5.96046e-05% | 20260508 05:15:11.516 | INFO | Baz IO overall: | 20260508 05:15:11.516 | INFO | BytesWritten : 2461047141217 | 20260508 05:15:11.516 | INFO | NumEvents : 1187550744954 | 20260508 05:15:13.051 | INFO | Analyzing chunk frames = [5517312,5517824) | 20260508 05:15:13.051 | INFO | MemInfo: { | 20260508 05:15:13.051 | INFO | "gpu" : " 17 GB", | 20260508 05:15:13.051 | INFO | "huge" : "", | 20260508 05:15:13.051 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:13.051 | INFO | "malloc" : " 14 GB", | 20260508 05:15:13.051 | INFO | "pinned" : " 29 GB" | 20260508 05:15:13.051 | INFO | } | 20260508 05:15:13.051 | INFO | Perf report for the last 5.135 seconds: | 20260508 05:15:13.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:13.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:13.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:13.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:13.051 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 05:15:13.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:13.051 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 05:15:13.051 | INFO | PRE_HQ : 66.0%, 1.6, 55.2ms, 0 | 20260508 05:15:13.052 | INFO | BAZWRITER : 67.7%, 2.7, 94.7ms, 0 | 20260508 05:15:13.052 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:15:13.052 | INFO | Uploaded 36GiB to GPU in 2.53023s (14.2279GiB/s) | 20260508 05:15:13.052 | INFO | Downloaded 38GiB to GPU in 2.69621s (14.0938GiB/s) | 20260508 05:15:18.196 | INFO | Analyzing chunk frames = [5517824,5518336) | 20260508 05:15:18.196 | INFO | MemInfo: { | 20260508 05:15:18.196 | INFO | "gpu" : " 17 GB", | 20260508 05:15:18.196 | INFO | "huge" : "", | 20260508 05:15:18.196 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:18.196 | INFO | "malloc" : " 14 GB", | 20260508 05:15:18.196 | INFO | "pinned" : " 29 GB" | 20260508 05:15:18.196 | INFO | } | 20260508 05:15:18.196 | INFO | Perf report for the last 5.14492 seconds: | 20260508 05:15:18.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:18.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.196 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:15:18.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.196 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:15:18.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:18.197 | INFO | Uploaded 36GiB to GPU in 2.51161s (14.3334GiB/s) | 20260508 05:15:18.197 | INFO | Downloaded 34GiB to GPU in 2.42713s (14.0083GiB/s) | 20260508 05:15:23.336 | INFO | Analyzing chunk frames = [5518336,5518848) | 20260508 05:15:23.336 | INFO | MemInfo: { | 20260508 05:15:23.336 | INFO | "gpu" : " 17 GB", | 20260508 05:15:23.336 | INFO | "huge" : "", | 20260508 05:15:23.337 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:23.337 | INFO | "malloc" : " 14 GB", | 20260508 05:15:23.337 | INFO | "pinned" : " 29 GB" | 20260508 05:15:23.337 | INFO | } | 20260508 05:15:23.337 | INFO | Perf report for the last 5.14051 seconds: | 20260508 05:15:23.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:23.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 05:15:23.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:15:23.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:23.337 | INFO | Uploaded 36GiB to GPU in 2.55202s (14.1065GiB/s) | 20260508 05:15:23.337 | INFO | Downloaded 34GiB to GPU in 2.39691s (14.1849GiB/s) | 20260508 05:15:28.474 | INFO | Analyzing chunk frames = [5518848,5519360) | 20260508 05:15:28.474 | INFO | MemInfo: { | 20260508 05:15:28.474 | INFO | "gpu" : " 17 GB", | 20260508 05:15:28.474 | INFO | "huge" : "", | 20260508 05:15:28.474 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:28.474 | INFO | "malloc" : " 14 GB", | 20260508 05:15:28.474 | INFO | "pinned" : " 29 GB" | 20260508 05:15:28.474 | INFO | } | 20260508 05:15:28.474 | INFO | Perf report for the last 5.137 seconds: | 20260508 05:15:28.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:28.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 05:15:28.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:15:28.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:28.474 | INFO | Uploaded 36GiB to GPU in 2.57091s (14.0028GiB/s) | 20260508 05:15:28.474 | INFO | Downloaded 34GiB to GPU in 2.38995s (14.2262GiB/s) | 20260508 05:15:33.609 | INFO | Analyzing chunk frames = [5519360,5519872) | 20260508 05:15:33.609 | INFO | MemInfo: { | 20260508 05:15:33.609 | INFO | "gpu" : " 17 GB", | 20260508 05:15:33.609 | INFO | "huge" : "", | 20260508 05:15:33.610 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:33.610 | INFO | "malloc" : " 14 GB", | 20260508 05:15:33.610 | INFO | "pinned" : " 29 GB" | 20260508 05:15:33.610 | INFO | } | 20260508 05:15:33.610 | INFO | Perf report for the last 5.13584 seconds: | 20260508 05:15:33.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:33.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 05:15:33.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:15:33.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:33.610 | INFO | Uploaded 36GiB to GPU in 2.37567s (15.1536GiB/s) | 20260508 05:15:33.610 | INFO | Downloaded 34GiB to GPU in 2.53801s (13.3963GiB/s) | 20260508 05:15:38.747 | INFO | Analyzing chunk frames = [5519872,5520384) | 20260508 05:15:38.747 | INFO | MemInfo: { | 20260508 05:15:38.747 | INFO | "gpu" : " 17 GB", | 20260508 05:15:38.747 | INFO | "huge" : "", | 20260508 05:15:38.747 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:38.747 | INFO | "malloc" : " 14 GB", | 20260508 05:15:38.747 | INFO | "pinned" : " 29 GB" | 20260508 05:15:38.747 | INFO | } | 20260508 05:15:38.747 | INFO | Perf report for the last 5.13758 seconds: | 20260508 05:15:38.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:38.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.747 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:15:38.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.747 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:15:38.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:38.748 | INFO | Uploaded 36GiB to GPU in 2.44438s (14.7276GiB/s) | 20260508 05:15:38.748 | INFO | Downloaded 34GiB to GPU in 2.42201s (14.0379GiB/s) | 20260508 05:15:43.892 | INFO | Analyzing chunk frames = [5520384,5520896) | 20260508 05:15:43.892 | INFO | MemInfo: { | 20260508 05:15:43.892 | INFO | "gpu" : " 17 GB", | 20260508 05:15:43.892 | INFO | "huge" : "", | 20260508 05:15:43.892 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:43.892 | INFO | "malloc" : " 14 GB", | 20260508 05:15:43.892 | INFO | "pinned" : " 29 GB" | 20260508 05:15:43.892 | INFO | } | 20260508 05:15:43.892 | INFO | Perf report for the last 5.14485 seconds: | 20260508 05:15:43.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:43.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:15:43.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:15:43.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:43.892 | INFO | Uploaded 36GiB to GPU in 2.41964s (14.8782GiB/s) | 20260508 05:15:43.892 | INFO | Downloaded 34GiB to GPU in 2.41855s (14.058GiB/s) | 20260508 05:15:49.032 | INFO | Analyzing chunk frames = [5520896,5521408) | 20260508 05:15:49.032 | INFO | MemInfo: { | 20260508 05:15:49.032 | INFO | "gpu" : " 17 GB", | 20260508 05:15:49.032 | INFO | "huge" : "", | 20260508 05:15:49.032 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:49.032 | INFO | "malloc" : " 14 GB", | 20260508 05:15:49.032 | INFO | "pinned" : " 29 GB" | 20260508 05:15:49.032 | INFO | } | 20260508 05:15:49.032 | INFO | Perf report for the last 5.13974 seconds: | 20260508 05:15:49.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:49.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:15:49.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 05:15:49.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:49.032 | INFO | Uploaded 36GiB to GPU in 2.43255s (14.7993GiB/s) | 20260508 05:15:49.032 | INFO | Downloaded 34GiB to GPU in 2.42063s (14.0459GiB/s) | 20260508 05:15:49.032 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:15:49.032 | INFO | Total mem usage: 16.7 GiB | 20260508 05:15:49.032 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:15:49.032 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.6%) | 20260508 05:15:49.032 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:15:49.032 | INFO | High Water Breakdown: | 20260508 05:15:49.032 | INFO | 59.6% : 8.52 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:15:49.032 | INFO | 40.3% : 5.77 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:15:49.032 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:15:49.032 | INFO | ------------------------------------------ | 20260508 05:15:49.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:15:49.032 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:15:49.032 | INFO | Total mem usage: 26.5 GiB | 20260508 05:15:49.032 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:15:49.032 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 05:15:49.032 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 05:15:49.032 | INFO | High Water Breakdown: | 20260508 05:15:49.032 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:15:49.032 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:15:49.032 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:15:49.032 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:15:49.032 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:15:49.032 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:15:49.032 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:15:49.032 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:15:49.032 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:15:49.032 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:15:49.032 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:15:49.032 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:15:49.032 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:15:49.032 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:15:49.032 | INFO | ------------------------------------------ | 20260508 05:15:49.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:15:49.032 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:15:49.032 | INFO | Total mem usage: 27.8 GiB | 20260508 05:15:49.032 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:15:49.032 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:15:49.032 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:15:49.032 | INFO | High Water Breakdown: | 20260508 05:15:49.032 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:15:49.032 | INFO | ------------------------------------------ | 20260508 05:15:49.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:15:49.032 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:15:49.032 | INFO | Total mem usage: 15.7 GiB | 20260508 05:15:49.032 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:15:49.032 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:15:49.032 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:15:49.032 | INFO | High Water Breakdown: | 20260508 05:15:49.032 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:15:49.032 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:15:49.032 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:15:49.032 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:15:49.032 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:15:49.032 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:15:49.032 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:15:49.032 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:15:49.032 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:15:49.032 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:15:49.032 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:15:49.032 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:15:49.032 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:15:49.032 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:15:49.032 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:15:49.032 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:15:49.032 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:15:49.032 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:15:49.032 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:15:49.032 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:15:49.032 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:15:49.032 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:15:49.032 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:15:49.032 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:15:49.032 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:15:49.032 | INFO | ------------------------------------------ | 20260508 05:15:49.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:15:49.032 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:15:49.032 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:15:49.032 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:15:49.032 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 1.22e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.08e-02 / 6.84e-03 | 20260508 05:15:49.032 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.47e+00, 5.88e-01 / 1.95e-01 | 20260508 05:15:49.032 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.45e+00, 2.09e-01 / 1.25e-01 | 20260508 05:15:49.032 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.52e+01, 3.86e+00 / 1.50e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.31e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.62e+00, 4.13e-01 / 1.78e-01 | 20260508 05:15:49.032 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.24e+00, 2.59e-01 / 1.34e-01 | 20260508 05:15:49.032 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.54e-01, 4.99e-02 / 1.86e-02 | 20260508 05:15:49.032 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:15:49.032 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:15:49.032 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:15:49.032 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 24.34%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 30.85%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 2.28%, LockUpload -- 2.42e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 05:15:49.032 | INFO | PerfReport: 2.58%, LockDownload -- 2.74e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 05:15:49.033 | INFO | PerfReport: 15.74%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:15:49.033 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 05:15:49.129 | INFO | Starting a round of Baz IO | 20260508 05:15:52.561 | INFO | Populating RTMetrics report | 20260508 05:15:52.562 | INFO | RTMetrics report populated | 20260508 05:15:52.562 | INFO | Opening tmp RTMetrics json file | 20260508 05:15:52.563 | INFO | Serializing RTMetrics to json | 20260508 05:15:52.564 | INFO | Serialization done, swapping tmp file now... | 20260508 05:15:52.564 | INFO | RTMetrics done | 20260508 05:15:52.632 | INFO | Finished a round of Baz IO: Wrote 1.578 GiB in 3.503 seconds | 20260508 05:15:52.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.90% / 0.00% / 59.10% | 20260508 05:15:52.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.476885% / 0% / 7.15256e-05% | 20260508 05:15:52.633 | INFO | Baz IO overall: | 20260508 05:15:52.633 | INFO | BytesWritten : 2462741805315 | 20260508 05:15:52.633 | INFO | NumEvents : 1188359349070 | 20260508 05:15:54.173 | INFO | Analyzing chunk frames = [5521408,5521920) | 20260508 05:15:54.173 | INFO | MemInfo: { | 20260508 05:15:54.173 | INFO | "gpu" : " 17 GB", | 20260508 05:15:54.173 | INFO | "huge" : "", | 20260508 05:15:54.173 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:54.173 | INFO | "malloc" : " 14 GB", | 20260508 05:15:54.173 | INFO | "pinned" : " 29 GB" | 20260508 05:15:54.173 | INFO | } | 20260508 05:15:54.173 | INFO | Perf report for the last 5.14122 seconds: | 20260508 05:15:54.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:54.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:54.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:54.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:54.173 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 05:15:54.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:54.173 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 05:15:54.173 | INFO | PRE_HQ : 65.8%, 1.6, 55.4ms, 0 | 20260508 05:15:54.173 | INFO | BAZWRITER : 67.7%, 2.7, 94.3ms, 0 | 20260508 05:15:54.173 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 05:15:54.174 | INFO | Uploaded 36GiB to GPU in 2.51257s (14.328GiB/s) | 20260508 05:15:54.174 | INFO | Downloaded 38GiB to GPU in 2.71199s (14.0119GiB/s) | 20260508 05:15:59.308 | INFO | Analyzing chunk frames = [5521920,5522432) | 20260508 05:15:59.308 | INFO | MemInfo: { | 20260508 05:15:59.308 | INFO | "gpu" : " 17 GB", | 20260508 05:15:59.308 | INFO | "huge" : "", | 20260508 05:15:59.308 | INFO | "huge_shared" : " 29 GB", | 20260508 05:15:59.308 | INFO | "malloc" : " 14 GB", | 20260508 05:15:59.308 | INFO | "pinned" : " 29 GB" | 20260508 05:15:59.308 | INFO | } | 20260508 05:15:59.309 | INFO | Perf report for the last 5.13539 seconds: | 20260508 05:15:59.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:15:59.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 05:15:59.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:15:59.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:15:59.309 | INFO | Uploaded 36GiB to GPU in 2.45387s (14.6707GiB/s) | 20260508 05:15:59.309 | INFO | Downloaded 34GiB to GPU in 2.44546s (13.9033GiB/s) | 20260508 05:16:04.451 | INFO | Analyzing chunk frames = [5522432,5522944) | 20260508 05:16:04.451 | INFO | MemInfo: { | 20260508 05:16:04.451 | INFO | "gpu" : " 17 GB", | 20260508 05:16:04.451 | INFO | "huge" : "", | 20260508 05:16:04.451 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:04.451 | INFO | "malloc" : " 14 GB", | 20260508 05:16:04.451 | INFO | "pinned" : " 29 GB" | 20260508 05:16:04.451 | INFO | } | 20260508 05:16:04.451 | INFO | Perf report for the last 5.14226 seconds: | 20260508 05:16:04.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:04.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:16:04.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:16:04.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:04.451 | INFO | Uploaded 36GiB to GPU in 2.47511s (14.5448GiB/s) | 20260508 05:16:04.451 | INFO | Downloaded 34GiB to GPU in 2.41699s (14.0671GiB/s) | 20260508 05:16:09.593 | INFO | Analyzing chunk frames = [5522944,5523456) | 20260508 05:16:09.593 | INFO | MemInfo: { | 20260508 05:16:09.593 | INFO | "gpu" : " 17 GB", | 20260508 05:16:09.593 | INFO | "huge" : "", | 20260508 05:16:09.593 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:09.593 | INFO | "malloc" : " 14 GB", | 20260508 05:16:09.593 | INFO | "pinned" : " 29 GB" | 20260508 05:16:09.593 | INFO | } | 20260508 05:16:09.593 | INFO | Perf report for the last 5.14216 seconds: | 20260508 05:16:09.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:09.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | ANALYSIS : 66.6%, 3.0, 103.0ms, 0 | 20260508 05:16:09.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:16:09.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:09.593 | INFO | Uploaded 36GiB to GPU in 2.50124s (14.3928GiB/s) | 20260508 05:16:09.593 | INFO | Downloaded 34GiB to GPU in 2.40204s (14.1546GiB/s) | 20260508 05:16:14.728 | INFO | Analyzing chunk frames = [5523456,5523968) | 20260508 05:16:14.728 | INFO | MemInfo: { | 20260508 05:16:14.728 | INFO | "gpu" : " 17 GB", | 20260508 05:16:14.728 | INFO | "huge" : "", | 20260508 05:16:14.728 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:14.728 | INFO | "malloc" : " 14 GB", | 20260508 05:16:14.728 | INFO | "pinned" : " 29 GB" | 20260508 05:16:14.728 | INFO | } | 20260508 05:16:14.728 | INFO | Perf report for the last 5.13541 seconds: | 20260508 05:16:14.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:14.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 05:16:14.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:16:14.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:14.729 | INFO | Uploaded 36GiB to GPU in 2.28602s (15.7479GiB/s) | 20260508 05:16:14.729 | INFO | Downloaded 34GiB to GPU in 2.484s (13.6876GiB/s) | 20260508 05:16:19.864 | INFO | Analyzing chunk frames = [5523968,5524480) | 20260508 05:16:19.864 | INFO | MemInfo: { | 20260508 05:16:19.864 | INFO | "gpu" : " 17 GB", | 20260508 05:16:19.864 | INFO | "huge" : "", | 20260508 05:16:19.864 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:19.864 | INFO | "malloc" : " 14 GB", | 20260508 05:16:19.864 | INFO | "pinned" : " 29 GB" | 20260508 05:16:19.864 | INFO | } | 20260508 05:16:19.864 | INFO | Perf report for the last 5.13561 seconds: | 20260508 05:16:19.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:19.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.864 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:16:19.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.864 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:16:19.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:19.865 | INFO | Uploaded 36GiB to GPU in 2.41921s (14.8809GiB/s) | 20260508 05:16:19.865 | INFO | Downloaded 34GiB to GPU in 2.42967s (13.9937GiB/s) | 20260508 05:16:25.003 | INFO | Analyzing chunk frames = [5524480,5524992) | 20260508 05:16:25.003 | INFO | MemInfo: { | 20260508 05:16:25.003 | INFO | "gpu" : " 17 GB", | 20260508 05:16:25.003 | INFO | "huge" : "", | 20260508 05:16:25.003 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:25.003 | INFO | "malloc" : " 14 GB", | 20260508 05:16:25.003 | INFO | "pinned" : " 29 GB" | 20260508 05:16:25.003 | INFO | } | 20260508 05:16:25.003 | INFO | Perf report for the last 5.13908 seconds: | 20260508 05:16:25.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:25.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.003 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:16:25.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.003 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 05:16:25.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:25.004 | INFO | Uploaded 36GiB to GPU in 2.41641s (14.8982GiB/s) | 20260508 05:16:25.004 | INFO | Downloaded 34GiB to GPU in 2.43512s (13.9624GiB/s) | 20260508 05:16:30.144 | INFO | Analyzing chunk frames = [5524992,5525504) | 20260508 05:16:30.144 | INFO | MemInfo: { | 20260508 05:16:30.144 | INFO | "gpu" : " 17 GB", | 20260508 05:16:30.144 | INFO | "huge" : "", | 20260508 05:16:30.144 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:30.144 | INFO | "malloc" : " 14 GB", | 20260508 05:16:30.144 | INFO | "pinned" : " 29 GB" | 20260508 05:16:30.144 | INFO | } | 20260508 05:16:30.144 | INFO | Perf report for the last 5.14089 seconds: | 20260508 05:16:30.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:30.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.144 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:16:30.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.144 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 05:16:30.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:30.145 | INFO | Uploaded 36GiB to GPU in 2.41973s (14.8777GiB/s) | 20260508 05:16:30.145 | INFO | Downloaded 34GiB to GPU in 2.42553s (14.0176GiB/s) | 20260508 05:16:30.242 | INFO | Starting a round of Baz IO | 20260508 05:16:33.679 | INFO | Populating RTMetrics report | 20260508 05:16:33.679 | INFO | RTMetrics report populated | 20260508 05:16:33.679 | INFO | Opening tmp RTMetrics json file | 20260508 05:16:33.680 | INFO | Serializing RTMetrics to json | 20260508 05:16:33.691 | INFO | Serialization done, swapping tmp file now... | 20260508 05:16:33.697 | INFO | RTMetrics done | 20260508 05:16:33.751 | INFO | Finished a round of Baz IO: Wrote 1.577 GiB in 3.509 seconds | 20260508 05:16:33.751 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.90% / 0.00% / 59.10% | 20260508 05:16:33.751 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.47431% / 0% / 7.94729e-05% | 20260508 05:16:33.751 | INFO | Baz IO overall: | 20260508 05:16:33.751 | INFO | BytesWritten : 2464434651008 | 20260508 05:16:33.751 | INFO | NumEvents : 1189166733499 | 20260508 05:16:35.284 | INFO | Analyzing chunk frames = [5525504,5526016) | 20260508 05:16:35.284 | INFO | MemInfo: { | 20260508 05:16:35.284 | INFO | "gpu" : " 17 GB", | 20260508 05:16:35.284 | INFO | "huge" : "", | 20260508 05:16:35.284 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:35.284 | INFO | "malloc" : " 14 GB", | 20260508 05:16:35.284 | INFO | "pinned" : " 29 GB" | 20260508 05:16:35.284 | INFO | } | 20260508 05:16:35.284 | INFO | Perf report for the last 5.13982 seconds: | 20260508 05:16:35.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:35.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:35.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:35.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:35.284 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 05:16:35.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:35.284 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 05:16:35.284 | INFO | PRE_HQ : 65.8%, 1.6, 55.7ms, 0 | 20260508 05:16:35.284 | INFO | BAZWRITER : 67.7%, 2.7, 94.3ms, 0 | 20260508 05:16:35.284 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:16:35.284 | INFO | Uploaded 36GiB to GPU in 2.56051s (14.0597GiB/s) | 20260508 05:16:35.284 | INFO | Downloaded 38GiB to GPU in 2.70415s (14.0525GiB/s) | 20260508 05:16:40.424 | INFO | Analyzing chunk frames = [5526016,5526528) | 20260508 05:16:40.424 | INFO | MemInfo: { | 20260508 05:16:40.424 | INFO | "gpu" : " 17 GB", | 20260508 05:16:40.424 | INFO | "huge" : "", | 20260508 05:16:40.424 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:40.424 | INFO | "malloc" : " 14 GB", | 20260508 05:16:40.424 | INFO | "pinned" : " 29 GB" | 20260508 05:16:40.424 | INFO | } | 20260508 05:16:40.424 | INFO | Perf report for the last 5.14055 seconds: | 20260508 05:16:40.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:40.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.425 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 05:16:40.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.425 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:16:40.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:40.425 | INFO | Uploaded 36GiB to GPU in 2.5923s (13.8873GiB/s) | 20260508 05:16:40.425 | INFO | Downloaded 34GiB to GPU in 2.43419s (13.9677GiB/s) | 20260508 05:16:45.560 | INFO | Analyzing chunk frames = [5526528,5527040) | 20260508 05:16:45.560 | INFO | MemInfo: { | 20260508 05:16:45.560 | INFO | "gpu" : " 17 GB", | 20260508 05:16:45.560 | INFO | "huge" : "", | 20260508 05:16:45.560 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:45.560 | INFO | "malloc" : " 14 GB", | 20260508 05:16:45.560 | INFO | "pinned" : " 29 GB" | 20260508 05:16:45.560 | INFO | } | 20260508 05:16:45.560 | INFO | Perf report for the last 5.1359 seconds: | 20260508 05:16:45.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:45.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.560 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 05:16:45.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.560 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:16:45.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:45.561 | INFO | Uploaded 36GiB to GPU in 2.56865s (14.0151GiB/s) | 20260508 05:16:45.561 | INFO | Downloaded 34GiB to GPU in 2.42357s (14.0289GiB/s) | 20260508 05:16:50.703 | INFO | Analyzing chunk frames = [5527040,5527552) | 20260508 05:16:50.703 | INFO | MemInfo: { | 20260508 05:16:50.703 | INFO | "gpu" : " 17 GB", | 20260508 05:16:50.703 | INFO | "huge" : "", | 20260508 05:16:50.703 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:50.703 | INFO | "malloc" : " 14 GB", | 20260508 05:16:50.703 | INFO | "pinned" : " 29 GB" | 20260508 05:16:50.703 | INFO | } | 20260508 05:16:50.703 | INFO | Perf report for the last 5.14257 seconds: | 20260508 05:16:50.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:50.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 05:16:50.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:16:50.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:50.703 | INFO | Uploaded 36GiB to GPU in 2.55174s (14.108GiB/s) | 20260508 05:16:50.703 | INFO | Downloaded 34GiB to GPU in 2.38317s (14.2667GiB/s) | 20260508 05:16:55.844 | INFO | Analyzing chunk frames = [5527552,5528064) | 20260508 05:16:55.844 | INFO | MemInfo: { | 20260508 05:16:55.844 | INFO | "gpu" : " 17 GB", | 20260508 05:16:55.844 | INFO | "huge" : "", | 20260508 05:16:55.844 | INFO | "huge_shared" : " 29 GB", | 20260508 05:16:55.844 | INFO | "malloc" : " 14 GB", | 20260508 05:16:55.844 | INFO | "pinned" : " 29 GB" | 20260508 05:16:55.844 | INFO | } | 20260508 05:16:55.844 | INFO | Perf report for the last 5.14107 seconds: | 20260508 05:16:55.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:16:55.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:16:55.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:16:55.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:16:55.844 | INFO | Uploaded 36GiB to GPU in 2.32454s (15.4869GiB/s) | 20260508 05:16:55.845 | INFO | Downloaded 34GiB to GPU in 2.47975s (13.7111GiB/s) | 20260508 05:17:00.981 | INFO | Analyzing chunk frames = [5528064,5528576) | 20260508 05:17:00.981 | INFO | MemInfo: { | 20260508 05:17:00.981 | INFO | "gpu" : " 17 GB", | 20260508 05:17:00.981 | INFO | "huge" : "", | 20260508 05:17:00.981 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:00.981 | INFO | "malloc" : " 14 GB", | 20260508 05:17:00.981 | INFO | "pinned" : " 29 GB" | 20260508 05:17:00.981 | INFO | } | 20260508 05:17:00.981 | INFO | Perf report for the last 5.13696 seconds: | 20260508 05:17:00.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:00.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:17:00.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:17:00.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:00.981 | INFO | Uploaded 36GiB to GPU in 2.4338s (14.7917GiB/s) | 20260508 05:17:00.981 | INFO | Downloaded 34GiB to GPU in 2.4213s (14.0421GiB/s) | 20260508 05:17:06.124 | INFO | Analyzing chunk frames = [5528576,5529088) | 20260508 05:17:06.124 | INFO | MemInfo: { | 20260508 05:17:06.124 | INFO | "gpu" : " 17 GB", | 20260508 05:17:06.124 | INFO | "huge" : "", | 20260508 05:17:06.124 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:06.124 | INFO | "malloc" : " 14 GB", | 20260508 05:17:06.124 | INFO | "pinned" : " 29 GB" | 20260508 05:17:06.125 | INFO | } | 20260508 05:17:06.125 | INFO | Perf report for the last 5.14369 seconds: | 20260508 05:17:06.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:06.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 05:17:06.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 05:17:06.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:06.125 | INFO | Uploaded 36GiB to GPU in 2.41549s (14.9038GiB/s) | 20260508 05:17:06.125 | INFO | Downloaded 34GiB to GPU in 2.42529s (14.0189GiB/s) | 20260508 05:17:11.259 | INFO | Analyzing chunk frames = [5529088,5529600) | 20260508 05:17:11.259 | INFO | MemInfo: { | 20260508 05:17:11.259 | INFO | "gpu" : " 17 GB", | 20260508 05:17:11.259 | INFO | "huge" : "", | 20260508 05:17:11.259 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:11.259 | INFO | "malloc" : " 14 GB", | 20260508 05:17:11.259 | INFO | "pinned" : " 29 GB" | 20260508 05:17:11.259 | INFO | } | 20260508 05:17:11.259 | INFO | Perf report for the last 5.13432 seconds: | 20260508 05:17:11.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:11.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:17:11.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 05:17:11.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:11.259 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:17:11.259 | INFO | Total mem usage: 16.7 GiB | 20260508 05:17:11.259 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:17:11.259 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 05:17:11.259 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:17:11.259 | INFO | High Water Breakdown: | 20260508 05:17:11.259 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:17:11.259 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:17:11.259 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:17:11.259 | INFO | ------------------------------------------ | 20260508 05:17:11.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:17:11.259 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:17:11.259 | INFO | Total mem usage: 26.5 GiB | 20260508 05:17:11.259 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:17:11.259 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 05:17:11.259 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 05:17:11.259 | INFO | High Water Breakdown: | 20260508 05:17:11.259 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:17:11.260 | INFO | Uploaded 36GiB to GPU in 2.45936s (14.6379GiB/s) | 20260508 05:17:11.260 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:17:11.260 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:17:11.260 | INFO | Downloaded 34GiB to GPU in 2.43433s (13.9669GiB/s) | 20260508 05:17:11.260 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:17:11.260 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:17:11.260 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:17:11.260 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:17:11.260 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:17:11.260 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:17:11.260 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:17:11.260 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:17:11.260 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:17:11.260 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:17:11.260 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:17:11.260 | INFO | ------------------------------------------ | 20260508 05:17:11.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:17:11.260 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:17:11.260 | INFO | Total mem usage: 27.8 GiB | 20260508 05:17:11.260 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:17:11.260 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:17:11.260 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:17:11.260 | INFO | High Water Breakdown: | 20260508 05:17:11.260 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:17:11.260 | INFO | ------------------------------------------ | 20260508 05:17:11.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:17:11.260 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:17:11.260 | INFO | Total mem usage: 15.7 GiB | 20260508 05:17:11.260 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:17:11.260 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:17:11.260 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:17:11.260 | INFO | High Water Breakdown: | 20260508 05:17:11.260 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:17:11.260 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:17:11.260 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:17:11.260 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:17:11.260 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:17:11.260 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:17:11.260 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:17:11.260 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:17:11.260 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:17:11.260 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:17:11.260 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:17:11.260 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:17:11.260 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:17:11.260 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:17:11.260 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:17:11.260 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:17:11.261 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:17:11.261 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:17:11.261 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:17:11.261 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:17:11.261 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:17:11.261 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:17:11.261 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:17:11.261 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:17:11.261 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:17:11.261 | INFO | ------------------------------------------ | 20260508 05:17:11.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:17:11.261 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:17:11.261 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:17:11.261 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:17:11.261 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 6.39e-01 | 20260508 05:17:11.261 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.11e-02 / 5.86e-03 | 20260508 05:17:11.261 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.39e+00, 5.92e-01 / 2.00e-01 | 20260508 05:17:11.261 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.44e+00, 2.13e-01 / 1.40e-01 | 20260508 05:17:11.261 | INFO | PerfReport: 15.89%, DME -- 4.17e+01 / 4.53e+01, 3.92e+00 / 1.22e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.82e+00 / 1.41e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.60e+00, 4.12e-01 / 2.27e-01 | 20260508 05:17:11.261 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.22e+00, 2.58e-01 / 1.63e-01 | 20260508 05:17:11.261 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.56e-01, 5.01e-02 / 1.96e-02 | 20260508 05:17:11.261 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:17:11.261 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:17:11.261 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:17:11.261 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.57e+01 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 24.17%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 30.90%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 2.23%, LockUpload -- 2.37e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 2.65%, LockDownload -- 2.81e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 15.83%, LockCompute -- 1.68e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:17:11.261 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260508 05:17:11.356 | INFO | Starting a round of Baz IO | 20260508 05:17:14.777 | INFO | Populating RTMetrics report | 20260508 05:17:14.778 | INFO | RTMetrics report populated | 20260508 05:17:14.778 | INFO | Opening tmp RTMetrics json file | 20260508 05:17:14.790 | INFO | Serializing RTMetrics to json | 20260508 05:17:14.791 | INFO | Serialization done, swapping tmp file now... | 20260508 05:17:14.792 | INFO | RTMetrics done | 20260508 05:17:14.854 | INFO | Finished a round of Baz IO: Wrote 1.578 GiB in 3.497 seconds | 20260508 05:17:14.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.90% / 0.00% / 59.10% | 20260508 05:17:14.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.475085% / 3.97364e-06% / 6.35783e-05% | 20260508 05:17:14.854 | INFO | Baz IO overall: | 20260508 05:17:14.854 | INFO | BytesWritten : 2466129387501 | 20260508 05:17:14.854 | INFO | NumEvents : 1189975698692 | 20260508 05:17:16.402 | INFO | Analyzing chunk frames = [5529600,5530112) | 20260508 05:17:16.402 | INFO | MemInfo: { | 20260508 05:17:16.402 | INFO | "gpu" : " 17 GB", | 20260508 05:17:16.402 | INFO | "huge" : "", | 20260508 05:17:16.402 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:16.402 | INFO | "malloc" : " 14 GB", | 20260508 05:17:16.402 | INFO | "pinned" : " 29 GB" | 20260508 05:17:16.402 | INFO | } | 20260508 05:17:16.402 | INFO | Perf report for the last 5.14328 seconds: | 20260508 05:17:16.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:16.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:16.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:16.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:16.402 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 05:17:16.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:16.402 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:17:16.402 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 05:17:16.402 | INFO | BAZWRITER : 67.4%, 2.7, 94.4ms, 0 | 20260508 05:17:16.402 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:17:16.403 | INFO | Uploaded 36GiB to GPU in 2.549s (14.1232GiB/s) | 20260508 05:17:16.403 | INFO | Downloaded 38GiB to GPU in 2.69244s (14.1136GiB/s) | 20260508 05:17:21.540 | INFO | Analyzing chunk frames = [5530112,5530624) | 20260508 05:17:21.540 | INFO | MemInfo: { | 20260508 05:17:21.540 | INFO | "gpu" : " 17 GB", | 20260508 05:17:21.540 | INFO | "huge" : "", | 20260508 05:17:21.540 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:21.540 | INFO | "malloc" : " 14 GB", | 20260508 05:17:21.540 | INFO | "pinned" : " 29 GB" | 20260508 05:17:21.540 | INFO | } | 20260508 05:17:21.540 | INFO | Perf report for the last 5.13795 seconds: | 20260508 05:17:21.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:21.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.540 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:17:21.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.540 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:17:21.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:21.541 | INFO | Uploaded 36GiB to GPU in 2.49384s (14.4356GiB/s) | 20260508 05:17:21.541 | INFO | Downloaded 34GiB to GPU in 2.41682s (14.068GiB/s) | 20260508 05:17:26.680 | INFO | Analyzing chunk frames = [5530624,5531136) | 20260508 05:17:26.680 | INFO | MemInfo: { | 20260508 05:17:26.680 | INFO | "gpu" : " 17 GB", | 20260508 05:17:26.680 | INFO | "huge" : "", | 20260508 05:17:26.680 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:26.681 | INFO | "malloc" : " 14 GB", | 20260508 05:17:26.681 | INFO | "pinned" : " 29 GB" | 20260508 05:17:26.681 | INFO | } | 20260508 05:17:26.681 | INFO | Perf report for the last 5.14029 seconds: | 20260508 05:17:26.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:26.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:17:26.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:17:26.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:26.681 | INFO | Uploaded 36GiB to GPU in 2.53018s (14.2282GiB/s) | 20260508 05:17:26.681 | INFO | Downloaded 34GiB to GPU in 2.41164s (14.0983GiB/s) | 20260508 05:17:31.813 | INFO | Analyzing chunk frames = [5531136,5531648) | 20260508 05:17:31.813 | INFO | MemInfo: { | 20260508 05:17:31.813 | INFO | "gpu" : " 17 GB", | 20260508 05:17:31.813 | INFO | "huge" : "", | 20260508 05:17:31.813 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:31.813 | INFO | "malloc" : " 14 GB", | 20260508 05:17:31.813 | INFO | "pinned" : " 29 GB" | 20260508 05:17:31.813 | INFO | } | 20260508 05:17:31.813 | INFO | Perf report for the last 5.13276 seconds: | 20260508 05:17:31.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:31.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.813 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 05:17:31.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.813 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260508 05:17:31.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:31.814 | INFO | Uploaded 36GiB to GPU in 2.53595s (14.1958GiB/s) | 20260508 05:17:31.814 | INFO | Downloaded 34GiB to GPU in 2.41062s (14.1043GiB/s) | 20260508 05:17:36.956 | INFO | Analyzing chunk frames = [5531648,5532160) | 20260508 05:17:36.956 | INFO | MemInfo: { | 20260508 05:17:36.956 | INFO | "gpu" : " 17 GB", | 20260508 05:17:36.956 | INFO | "huge" : "", | 20260508 05:17:36.956 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:36.957 | INFO | "malloc" : " 14 GB", | 20260508 05:17:36.957 | INFO | "pinned" : " 29 GB" | 20260508 05:17:36.957 | INFO | } | 20260508 05:17:36.957 | INFO | Perf report for the last 5.14327 seconds: | 20260508 05:17:36.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:36.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:17:36.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 05:17:36.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:36.957 | INFO | Uploaded 36GiB to GPU in 2.29598s (15.6796GiB/s) | 20260508 05:17:36.957 | INFO | Downloaded 34GiB to GPU in 2.47616s (13.7309GiB/s) | 20260508 05:17:42.092 | INFO | Analyzing chunk frames = [5532160,5532672) | 20260508 05:17:42.092 | INFO | MemInfo: { | 20260508 05:17:42.092 | INFO | "gpu" : " 17 GB", | 20260508 05:17:42.092 | INFO | "huge" : "", | 20260508 05:17:42.092 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:42.092 | INFO | "malloc" : " 14 GB", | 20260508 05:17:42.092 | INFO | "pinned" : " 29 GB" | 20260508 05:17:42.092 | INFO | } | 20260508 05:17:42.092 | INFO | Perf report for the last 5.13574 seconds: | 20260508 05:17:42.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:42.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.092 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:17:42.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.093 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:17:42.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:42.093 | INFO | Uploaded 36GiB to GPU in 2.44294s (14.7364GiB/s) | 20260508 05:17:42.093 | INFO | Downloaded 34GiB to GPU in 2.42172s (14.0396GiB/s) | 20260508 05:17:47.238 | INFO | Analyzing chunk frames = [5532672,5533184) | 20260508 05:17:47.238 | INFO | MemInfo: { | 20260508 05:17:47.238 | INFO | "gpu" : " 17 GB", | 20260508 05:17:47.238 | INFO | "huge" : "", | 20260508 05:17:47.238 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:47.238 | INFO | "malloc" : " 14 GB", | 20260508 05:17:47.238 | INFO | "pinned" : " 29 GB" | 20260508 05:17:47.238 | INFO | } | 20260508 05:17:47.238 | INFO | Perf report for the last 5.14571 seconds: | 20260508 05:17:47.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:47.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | ANALYSIS : 66.2%, 3.0, 103.7ms, 0 | 20260508 05:17:47.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:17:47.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:47.238 | INFO | Uploaded 36GiB to GPU in 2.44746s (14.7091GiB/s) | 20260508 05:17:47.239 | INFO | Downloaded 34GiB to GPU in 2.43366s (13.9707GiB/s) | 20260508 05:17:52.372 | INFO | Analyzing chunk frames = [5533184,5533696) | 20260508 05:17:52.372 | INFO | MemInfo: { | 20260508 05:17:52.372 | INFO | "gpu" : " 17 GB", | 20260508 05:17:52.372 | INFO | "huge" : "", | 20260508 05:17:52.372 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:52.372 | INFO | "malloc" : " 14 GB", | 20260508 05:17:52.373 | INFO | "pinned" : " 29 GB" | 20260508 05:17:52.373 | INFO | } | 20260508 05:17:52.373 | INFO | Perf report for the last 5.13454 seconds: | 20260508 05:17:52.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:52.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:17:52.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 05:17:52.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:52.373 | INFO | Uploaded 36GiB to GPU in 2.44385s (14.7308GiB/s) | 20260508 05:17:52.373 | INFO | Downloaded 34GiB to GPU in 2.43169s (13.982GiB/s) | 20260508 05:17:52.482 | INFO | Starting a round of Baz IO | 20260508 05:17:55.886 | INFO | Populating RTMetrics report | 20260508 05:17:55.887 | INFO | RTMetrics report populated | 20260508 05:17:55.887 | INFO | Opening tmp RTMetrics json file | 20260508 05:17:55.896 | INFO | Serializing RTMetrics to json | 20260508 05:17:55.900 | INFO | Serialization done, swapping tmp file now... | 20260508 05:17:55.900 | INFO | RTMetrics done | 20260508 05:17:55.962 | INFO | Finished a round of Baz IO: Wrote 1.576 GiB in 3.479 seconds | 20260508 05:17:55.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.90% / 0.00% / 59.10% | 20260508 05:17:55.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.473416% / 3.97364e-06% / 5.96046e-05% | 20260508 05:17:55.962 | INFO | Baz IO overall: | 20260508 05:17:55.962 | INFO | BytesWritten : 2467822006233 | 20260508 05:17:55.962 | INFO | NumEvents : 1190782977348 | 20260508 05:17:57.512 | INFO | Analyzing chunk frames = [5533696,5534208) | 20260508 05:17:57.512 | INFO | MemInfo: { | 20260508 05:17:57.512 | INFO | "gpu" : " 17 GB", | 20260508 05:17:57.512 | INFO | "huge" : "", | 20260508 05:17:57.512 | INFO | "huge_shared" : " 29 GB", | 20260508 05:17:57.512 | INFO | "malloc" : " 14 GB", | 20260508 05:17:57.512 | INFO | "pinned" : " 29 GB" | 20260508 05:17:57.512 | INFO | } | 20260508 05:17:57.513 | INFO | Perf report for the last 5.13982 seconds: | 20260508 05:17:57.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:17:57.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:57.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:57.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:57.513 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 05:17:57.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:17:57.513 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 05:17:57.513 | INFO | PRE_HQ : 65.3%, 1.6, 55.5ms, 0 | 20260508 05:17:57.513 | INFO | BAZWRITER : 67.3%, 2.7, 91.9ms, 0 | 20260508 05:17:57.513 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:17:57.513 | INFO | Uploaded 36GiB to GPU in 2.54656s (14.1367GiB/s) | 20260508 05:17:57.513 | INFO | Downloaded 38GiB to GPU in 2.68731s (14.1405GiB/s) | 20260508 05:18:02.656 | INFO | Analyzing chunk frames = [5534208,5534720) | 20260508 05:18:02.656 | INFO | MemInfo: { | 20260508 05:18:02.656 | INFO | "gpu" : " 17 GB", | 20260508 05:18:02.656 | INFO | "huge" : "", | 20260508 05:18:02.656 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:02.656 | INFO | "malloc" : " 14 GB", | 20260508 05:18:02.656 | INFO | "pinned" : " 29 GB" | 20260508 05:18:02.656 | INFO | } | 20260508 05:18:02.656 | INFO | Perf report for the last 5.14335 seconds: | 20260508 05:18:02.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:02.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 05:18:02.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 05:18:02.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:02.656 | INFO | Uploaded 36GiB to GPU in 2.57546s (13.9781GiB/s) | 20260508 05:18:02.656 | INFO | Downloaded 34GiB to GPU in 2.44409s (13.9111GiB/s) | 20260508 05:18:07.797 | INFO | Analyzing chunk frames = [5534720,5535232) | 20260508 05:18:07.797 | INFO | MemInfo: { | 20260508 05:18:07.797 | INFO | "gpu" : " 17 GB", | 20260508 05:18:07.797 | INFO | "huge" : "", | 20260508 05:18:07.801 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:07.801 | INFO | "malloc" : " 14 GB", | 20260508 05:18:07.801 | INFO | "pinned" : " 29 GB" | 20260508 05:18:07.801 | INFO | } | 20260508 05:18:07.801 | INFO | Perf report for the last 5.14536 seconds: | 20260508 05:18:07.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:07.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.801 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 05:18:07.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.801 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:18:07.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:07.802 | INFO | Uploaded 36GiB to GPU in 2.53798s (14.1845GiB/s) | 20260508 05:18:07.802 | INFO | Downloaded 34GiB to GPU in 2.40435s (14.141GiB/s) | 20260508 05:18:12.933 | INFO | Analyzing chunk frames = [5535232,5535744) | 20260508 05:18:12.933 | INFO | MemInfo: { | 20260508 05:18:12.933 | INFO | "gpu" : " 17 GB", | 20260508 05:18:12.933 | INFO | "huge" : "", | 20260508 05:18:12.933 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:12.933 | INFO | "malloc" : " 14 GB", | 20260508 05:18:12.933 | INFO | "pinned" : " 29 GB" | 20260508 05:18:12.933 | INFO | } | 20260508 05:18:12.933 | INFO | Perf report for the last 5.13207 seconds: | 20260508 05:18:12.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:12.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.933 | INFO | ANALYSIS : 66.9%, 3.0, 103.4ms, 0 | 20260508 05:18:12.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.933 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:18:12.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:12.934 | INFO | Uploaded 36GiB to GPU in 2.56798s (14.0188GiB/s) | 20260508 05:18:12.934 | INFO | Downloaded 34GiB to GPU in 2.41188s (14.0969GiB/s) | 20260508 05:18:18.071 | INFO | Analyzing chunk frames = [5535744,5536256) | 20260508 05:18:18.071 | INFO | MemInfo: { | 20260508 05:18:18.071 | INFO | "gpu" : " 17 GB", | 20260508 05:18:18.071 | INFO | "huge" : "", | 20260508 05:18:18.071 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:18.071 | INFO | "malloc" : " 14 GB", | 20260508 05:18:18.071 | INFO | "pinned" : " 29 GB" | 20260508 05:18:18.071 | INFO | } | 20260508 05:18:18.072 | INFO | Perf report for the last 5.13821 seconds: | 20260508 05:18:18.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:18.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 05:18:18.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:18:18.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:18.072 | INFO | Uploaded 36GiB to GPU in 2.37661s (15.1476GiB/s) | 20260508 05:18:18.072 | INFO | Downloaded 34GiB to GPU in 2.57179s (13.2203GiB/s) | 20260508 05:18:23.214 | INFO | Analyzing chunk frames = [5536256,5536768) | 20260508 05:18:23.214 | INFO | MemInfo: { | 20260508 05:18:23.214 | INFO | "gpu" : " 17 GB", | 20260508 05:18:23.214 | INFO | "huge" : "", | 20260508 05:18:23.214 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:23.214 | INFO | "malloc" : " 14 GB", | 20260508 05:18:23.214 | INFO | "pinned" : " 29 GB" | 20260508 05:18:23.214 | INFO | } | 20260508 05:18:23.214 | INFO | Perf report for the last 5.14242 seconds: | 20260508 05:18:23.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:23.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:18:23.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:18:23.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:23.214 | INFO | Uploaded 36GiB to GPU in 2.43162s (14.8049GiB/s) | 20260508 05:18:23.214 | INFO | Downloaded 34GiB to GPU in 2.4187s (14.0571GiB/s) | 20260508 05:18:28.355 | INFO | Analyzing chunk frames = [5536768,5537280) | 20260508 05:18:28.355 | INFO | MemInfo: { | 20260508 05:18:28.355 | INFO | "gpu" : " 17 GB", | 20260508 05:18:28.355 | INFO | "huge" : "", | 20260508 05:18:28.355 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:28.355 | INFO | "malloc" : " 14 GB", | 20260508 05:18:28.355 | INFO | "pinned" : " 29 GB" | 20260508 05:18:28.355 | INFO | } | 20260508 05:18:28.355 | INFO | Perf report for the last 5.14129 seconds: | 20260508 05:18:28.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:28.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:18:28.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:18:28.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:28.356 | INFO | Uploaded 36GiB to GPU in 2.41628s (14.8989GiB/s) | 20260508 05:18:28.356 | INFO | Downloaded 34GiB to GPU in 2.42026s (14.0481GiB/s) | 20260508 05:18:33.491 | INFO | Analyzing chunk frames = [5537280,5537792) | 20260508 05:18:33.491 | INFO | MemInfo: { | 20260508 05:18:33.491 | INFO | "gpu" : " 17 GB", | 20260508 05:18:33.491 | INFO | "huge" : "", | 20260508 05:18:33.492 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:33.492 | INFO | "malloc" : " 14 GB", | 20260508 05:18:33.492 | INFO | "pinned" : " 29 GB" | 20260508 05:18:33.492 | INFO | } | 20260508 05:18:33.492 | INFO | Perf report for the last 5.13625 seconds: | 20260508 05:18:33.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:33.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:18:33.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 05:18:33.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:33.492 | INFO | Uploaded 36GiB to GPU in 2.43949s (14.7572GiB/s) | 20260508 05:18:33.492 | INFO | Downloaded 34GiB to GPU in 2.42129s (14.0421GiB/s) | 20260508 05:18:33.492 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:18:33.492 | INFO | Total mem usage: 16.7 GiB | 20260508 05:18:33.492 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:18:33.492 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 05:18:33.492 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:18:33.492 | INFO | High Water Breakdown: | 20260508 05:18:33.492 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:18:33.492 | INFO | 40.4% : 5.76 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:18:33.492 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:18:33.492 | INFO | ------------------------------------------ | 20260508 05:18:33.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:18:33.492 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:18:33.492 | INFO | Total mem usage: 26.5 GiB | 20260508 05:18:33.492 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:18:33.492 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:18:33.492 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:18:33.492 | INFO | High Water Breakdown: | 20260508 05:18:33.492 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:18:33.492 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:18:33.492 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:18:33.492 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:18:33.492 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:18:33.492 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:18:33.492 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:18:33.492 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:18:33.492 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:18:33.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:18:33.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:18:33.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:18:33.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:18:33.492 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:18:33.492 | INFO | ------------------------------------------ | 20260508 05:18:33.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:18:33.492 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:18:33.492 | INFO | Total mem usage: 27.8 GiB | 20260508 05:18:33.492 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:18:33.492 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:18:33.492 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:18:33.492 | INFO | High Water Breakdown: | 20260508 05:18:33.492 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:18:33.492 | INFO | ------------------------------------------ | 20260508 05:18:33.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:18:33.492 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:18:33.492 | INFO | Total mem usage: 15.7 GiB | 20260508 05:18:33.492 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:18:33.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:18:33.492 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:18:33.492 | INFO | High Water Breakdown: | 20260508 05:18:33.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:18:33.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:18:33.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:18:33.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:18:33.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:18:33.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:18:33.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:18:33.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:18:33.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:18:33.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:18:33.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:18:33.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:18:33.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:18:33.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:18:33.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:18:33.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:18:33.492 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:18:33.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:18:33.492 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:18:33.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:18:33.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:18:33.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:18:33.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:18:33.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:18:33.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:18:33.492 | INFO | ------------------------------------------ | 20260508 05:18:33.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:18:33.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:18:33.492 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:18:33.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:18:33.492 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 1.37e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.04e-01, 1.10e-02 / 2.84e-03 | 20260508 05:18:33.492 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.45e+00, 5.95e-01 / 2.30e-01 | 20260508 05:18:33.492 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.14e-01 | 20260508 05:18:33.492 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.54e+01, 3.79e+00 / 1.73e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.76e+00 / 1.38e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.61e+00, 4.17e-01 / 2.54e-01 | 20260508 05:18:33.492 | INFO | PerfReport: 3.20%, Metrics -- 1.05e+00 / 1.23e+00, 2.60e-01 / 1.32e-01 | 20260508 05:18:33.492 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.53e-01, 5.04e-02 / 2.62e-02 | 20260508 05:18:33.492 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:18:33.492 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:18:33.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:18:33.492 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 24.31%, Upload -- 2.58e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.33e+00 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 2.21%, LockUpload -- 2.34e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 2.72%, LockDownload -- 2.89e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 15.73%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:18:33.492 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.74e-02 / 0.00e+00 | 20260508 05:18:33.589 | INFO | Starting a round of Baz IO | 20260508 05:18:37.023 | INFO | Populating RTMetrics report | 20260508 05:18:37.023 | INFO | RTMetrics report populated | 20260508 05:18:37.023 | INFO | Opening tmp RTMetrics json file | 20260508 05:18:37.035 | INFO | Serializing RTMetrics to json | 20260508 05:18:37.036 | INFO | Serialization done, swapping tmp file now... | 20260508 05:18:37.037 | INFO | RTMetrics done | 20260508 05:18:37.093 | INFO | Finished a round of Baz IO: Wrote 1.577 GiB in 3.504 seconds | 20260508 05:18:37.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.89% / 0.00% / 59.10% | 20260508 05:18:37.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.473591% / 0% / 5.96046e-05% | 20260508 05:18:37.093 | INFO | Baz IO overall: | 20260508 05:18:37.093 | INFO | BytesWritten : 2469515512151 | 20260508 05:18:37.093 | INFO | NumEvents : 1191591134161 | 20260508 05:18:38.630 | INFO | Analyzing chunk frames = [5537792,5538304) | 20260508 05:18:38.630 | INFO | MemInfo: { | 20260508 05:18:38.630 | INFO | "gpu" : " 17 GB", | 20260508 05:18:38.630 | INFO | "huge" : "", | 20260508 05:18:38.630 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:38.630 | INFO | "malloc" : " 14 GB", | 20260508 05:18:38.630 | INFO | "pinned" : " 29 GB" | 20260508 05:18:38.630 | INFO | } | 20260508 05:18:38.630 | INFO | Perf report for the last 5.13885 seconds: | 20260508 05:18:38.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:38.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:38.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:38.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:38.631 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:18:38.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:38.631 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 05:18:38.631 | INFO | PRE_HQ : 65.3%, 1.7, 56.9ms, 0 | 20260508 05:18:38.631 | INFO | BAZWRITER : 67.6%, 2.7, 93.5ms, 0 | 20260508 05:18:38.631 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:18:38.631 | INFO | Uploaded 36GiB to GPU in 2.54576s (14.1411GiB/s) | 20260508 05:18:38.631 | INFO | Downloaded 38GiB to GPU in 2.69417s (14.1046GiB/s) | 20260508 05:18:43.765 | INFO | Analyzing chunk frames = [5538304,5538816) | 20260508 05:18:43.765 | INFO | MemInfo: { | 20260508 05:18:43.765 | INFO | "gpu" : " 17 GB", | 20260508 05:18:43.765 | INFO | "huge" : "", | 20260508 05:18:43.765 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:43.765 | INFO | "malloc" : " 14 GB", | 20260508 05:18:43.765 | INFO | "pinned" : " 29 GB" | 20260508 05:18:43.765 | INFO | } | 20260508 05:18:43.765 | INFO | Perf report for the last 5.13429 seconds: | 20260508 05:18:43.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:43.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:18:43.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:18:43.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:43.765 | INFO | Uploaded 36GiB to GPU in 2.4593s (14.6383GiB/s) | 20260508 05:18:43.765 | INFO | Downloaded 34GiB to GPU in 2.44287s (13.9181GiB/s) | 20260508 05:18:48.905 | INFO | Analyzing chunk frames = [5538816,5539328) | 20260508 05:18:48.905 | INFO | MemInfo: { | 20260508 05:18:48.905 | INFO | "gpu" : " 17 GB", | 20260508 05:18:48.905 | INFO | "huge" : "", | 20260508 05:18:48.905 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:48.905 | INFO | "malloc" : " 14 GB", | 20260508 05:18:48.905 | INFO | "pinned" : " 29 GB" | 20260508 05:18:48.905 | INFO | } | 20260508 05:18:48.905 | INFO | Perf report for the last 5.14071 seconds: | 20260508 05:18:48.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:48.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:18:48.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:18:48.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:48.906 | INFO | Uploaded 36GiB to GPU in 2.51223s (14.3299GiB/s) | 20260508 05:18:48.906 | INFO | Downloaded 34GiB to GPU in 2.42933s (13.9956GiB/s) | 20260508 05:18:54.046 | INFO | Analyzing chunk frames = [5539328,5539840) | 20260508 05:18:54.046 | INFO | MemInfo: { | 20260508 05:18:54.046 | INFO | "gpu" : " 17 GB", | 20260508 05:18:54.046 | INFO | "huge" : "", | 20260508 05:18:54.046 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:54.046 | INFO | "malloc" : " 14 GB", | 20260508 05:18:54.046 | INFO | "pinned" : " 29 GB" | 20260508 05:18:54.046 | INFO | } | 20260508 05:18:54.046 | INFO | Perf report for the last 5.141 seconds: | 20260508 05:18:54.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:54.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 05:18:54.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:18:54.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:54.047 | INFO | Uploaded 36GiB to GPU in 2.55241s (14.1043GiB/s) | 20260508 05:18:54.047 | INFO | Downloaded 34GiB to GPU in 2.43928s (13.9385GiB/s) | 20260508 05:18:59.189 | INFO | Analyzing chunk frames = [5539840,5540352) | 20260508 05:18:59.189 | INFO | MemInfo: { | 20260508 05:18:59.189 | INFO | "gpu" : " 17 GB", | 20260508 05:18:59.189 | INFO | "huge" : "", | 20260508 05:18:59.189 | INFO | "huge_shared" : " 29 GB", | 20260508 05:18:59.189 | INFO | "malloc" : " 14 GB", | 20260508 05:18:59.189 | INFO | "pinned" : " 29 GB" | 20260508 05:18:59.189 | INFO | } | 20260508 05:18:59.189 | INFO | Perf report for the last 5.1424 seconds: | 20260508 05:18:59.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:18:59.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:18:59.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:18:59.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:18:59.189 | INFO | Uploaded 36GiB to GPU in 2.35833s (15.265GiB/s) | 20260508 05:18:59.189 | INFO | Downloaded 34GiB to GPU in 2.5718s (13.2203GiB/s) | 20260508 05:19:04.324 | INFO | Analyzing chunk frames = [5540352,5540864) | 20260508 05:19:04.324 | INFO | MemInfo: { | 20260508 05:19:04.324 | INFO | "gpu" : " 17 GB", | 20260508 05:19:04.324 | INFO | "huge" : "", | 20260508 05:19:04.324 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:04.324 | INFO | "malloc" : " 14 GB", | 20260508 05:19:04.324 | INFO | "pinned" : " 29 GB" | 20260508 05:19:04.324 | INFO | } | 20260508 05:19:04.324 | INFO | Perf report for the last 5.13545 seconds: | 20260508 05:19:04.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:04.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.327 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:19:04.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.327 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:19:04.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:04.328 | INFO | Uploaded 36GiB to GPU in 2.43175s (14.8042GiB/s) | 20260508 05:19:04.328 | INFO | Downloaded 34GiB to GPU in 2.44887s (13.8839GiB/s) | 20260508 05:19:09.472 | INFO | Analyzing chunk frames = [5540864,5541376) | 20260508 05:19:09.472 | INFO | MemInfo: { | 20260508 05:19:09.472 | INFO | "gpu" : " 17 GB", | 20260508 05:19:09.472 | INFO | "huge" : "", | 20260508 05:19:09.472 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:09.472 | INFO | "malloc" : " 14 GB", | 20260508 05:19:09.472 | INFO | "pinned" : " 29 GB" | 20260508 05:19:09.472 | INFO | } | 20260508 05:19:09.472 | INFO | Perf report for the last 5.14739 seconds: | 20260508 05:19:09.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:09.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | ANALYSIS : 65.9%, 3.0, 103.2ms, 0 | 20260508 05:19:09.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 05:19:09.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:09.472 | INFO | Uploaded 36GiB to GPU in 2.41969s (14.8779GiB/s) | 20260508 05:19:09.472 | INFO | Downloaded 34GiB to GPU in 2.43904s (13.9399GiB/s) | 20260508 05:19:14.608 | INFO | Analyzing chunk frames = [5541376,5541888) | 20260508 05:19:14.608 | INFO | MemInfo: { | 20260508 05:19:14.608 | INFO | "gpu" : " 17 GB", | 20260508 05:19:14.608 | INFO | "huge" : "", | 20260508 05:19:14.608 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:14.608 | INFO | "malloc" : " 14 GB", | 20260508 05:19:14.608 | INFO | "pinned" : " 29 GB" | 20260508 05:19:14.608 | INFO | } | 20260508 05:19:14.608 | INFO | Perf report for the last 5.13619 seconds: | 20260508 05:19:14.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:14.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:19:14.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:19:14.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:14.608 | INFO | Uploaded 36GiB to GPU in 2.41698s (14.8946GiB/s) | 20260508 05:19:14.608 | INFO | Downloaded 34GiB to GPU in 2.45229s (13.8646GiB/s) | 20260508 05:19:14.705 | INFO | Starting a round of Baz IO | 20260508 05:19:18.130 | INFO | Populating RTMetrics report | 20260508 05:19:18.130 | INFO | RTMetrics report populated | 20260508 05:19:18.130 | INFO | Opening tmp RTMetrics json file | 20260508 05:19:18.138 | INFO | Serializing RTMetrics to json | 20260508 05:19:18.140 | INFO | Serialization done, swapping tmp file now... | 20260508 05:19:18.140 | INFO | RTMetrics done | 20260508 05:19:18.204 | INFO | Finished a round of Baz IO: Wrote 1.578 GiB in 3.499 seconds | 20260508 05:19:18.204 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.89% / 0.00% / 59.11% | 20260508 05:19:18.204 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.473448% / 0% / 6.75519e-05% | 20260508 05:19:18.204 | INFO | Baz IO overall: | 20260508 05:19:18.204 | INFO | BytesWritten : 2471209657719 | 20260508 05:19:18.204 | INFO | NumEvents : 1192399652137 | 20260508 05:19:19.745 | INFO | Analyzing chunk frames = [5541888,5542400) | 20260508 05:19:19.745 | INFO | MemInfo: { | 20260508 05:19:19.745 | INFO | "gpu" : " 17 GB", | 20260508 05:19:19.745 | INFO | "huge" : "", | 20260508 05:19:19.745 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:19.745 | INFO | "malloc" : " 14 GB", | 20260508 05:19:19.745 | INFO | "pinned" : " 29 GB" | 20260508 05:19:19.745 | INFO | } | 20260508 05:19:19.745 | INFO | Perf report for the last 5.13716 seconds: | 20260508 05:19:19.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:19.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:19.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:19.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:19.745 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 05:19:19.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:19.745 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:19:19.745 | INFO | PRE_HQ : 65.5%, 1.6, 55.3ms, 0 | 20260508 05:19:19.745 | INFO | BAZWRITER : 67.6%, 2.7, 94.0ms, 0 | 20260508 05:19:19.745 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:19:19.745 | INFO | Uploaded 36GiB to GPU in 2.51312s (14.3248GiB/s) | 20260508 05:19:19.745 | INFO | Downloaded 38GiB to GPU in 2.68855s (14.134GiB/s) | 20260508 05:19:24.883 | INFO | Analyzing chunk frames = [5542400,5542912) | 20260508 05:19:24.883 | INFO | MemInfo: { | 20260508 05:19:24.883 | INFO | "gpu" : " 17 GB", | 20260508 05:19:24.883 | INFO | "huge" : "", | 20260508 05:19:24.883 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:24.883 | INFO | "malloc" : " 14 GB", | 20260508 05:19:24.883 | INFO | "pinned" : " 29 GB" | 20260508 05:19:24.883 | INFO | } | 20260508 05:19:24.883 | INFO | Perf report for the last 5.1377 seconds: | 20260508 05:19:24.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:24.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 05:19:24.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:19:24.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:24.883 | INFO | Uploaded 36GiB to GPU in 2.54689s (14.1349GiB/s) | 20260508 05:19:24.883 | INFO | Downloaded 34GiB to GPU in 2.44899s (13.8833GiB/s) | 20260508 05:19:30.023 | INFO | Analyzing chunk frames = [5542912,5543424) | 20260508 05:19:30.023 | INFO | MemInfo: { | 20260508 05:19:30.023 | INFO | "gpu" : " 17 GB", | 20260508 05:19:30.023 | INFO | "huge" : "", | 20260508 05:19:30.023 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:30.023 | INFO | "malloc" : " 14 GB", | 20260508 05:19:30.023 | INFO | "pinned" : " 29 GB" | 20260508 05:19:30.023 | INFO | } | 20260508 05:19:30.023 | INFO | Perf report for the last 5.13982 seconds: | 20260508 05:19:30.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:30.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:19:30.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:19:30.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:30.023 | INFO | Uploaded 36GiB to GPU in 2.48739s (14.473GiB/s) | 20260508 05:19:30.023 | INFO | Downloaded 34GiB to GPU in 2.42209s (14.0375GiB/s) | 20260508 05:19:35.161 | INFO | Analyzing chunk frames = [5543424,5543936) | 20260508 05:19:35.161 | INFO | MemInfo: { | 20260508 05:19:35.161 | INFO | "gpu" : " 17 GB", | 20260508 05:19:35.161 | INFO | "huge" : "", | 20260508 05:19:35.161 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:35.161 | INFO | "malloc" : " 14 GB", | 20260508 05:19:35.161 | INFO | "pinned" : " 29 GB" | 20260508 05:19:35.161 | INFO | } | 20260508 05:19:35.161 | INFO | Perf report for the last 5.13872 seconds: | 20260508 05:19:35.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:35.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 05:19:35.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:19:35.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:35.162 | INFO | Uploaded 36GiB to GPU in 2.53963s (14.1753GiB/s) | 20260508 05:19:35.162 | INFO | Downloaded 34GiB to GPU in 2.42744s (14.0065GiB/s) | 20260508 05:19:40.305 | INFO | Analyzing chunk frames = [5543936,5544448) | 20260508 05:19:40.305 | INFO | MemInfo: { | 20260508 05:19:40.305 | INFO | "gpu" : " 17 GB", | 20260508 05:19:40.305 | INFO | "huge" : "", | 20260508 05:19:40.305 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:40.305 | INFO | "malloc" : " 14 GB", | 20260508 05:19:40.305 | INFO | "pinned" : " 29 GB" | 20260508 05:19:40.305 | INFO | } | 20260508 05:19:40.305 | INFO | Perf report for the last 5.14378 seconds: | 20260508 05:19:40.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:40.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.305 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:19:40.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.305 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:19:40.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:40.306 | INFO | Uploaded 36GiB to GPU in 2.32299s (15.4973GiB/s) | 20260508 05:19:40.306 | INFO | Downloaded 34GiB to GPU in 2.50516s (13.572GiB/s) | 20260508 05:19:45.439 | INFO | Analyzing chunk frames = [5544448,5544960) | 20260508 05:19:45.439 | INFO | MemInfo: { | 20260508 05:19:45.439 | INFO | "gpu" : " 17 GB", | 20260508 05:19:45.439 | INFO | "huge" : "", | 20260508 05:19:45.439 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:45.439 | INFO | "malloc" : " 14 GB", | 20260508 05:19:45.439 | INFO | "pinned" : " 29 GB" | 20260508 05:19:45.439 | INFO | } | 20260508 05:19:45.439 | INFO | Perf report for the last 5.1342 seconds: | 20260508 05:19:45.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:45.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:19:45.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 05:19:45.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:45.440 | INFO | Uploaded 36GiB to GPU in 2.4268s (14.8344GiB/s) | 20260508 05:19:45.440 | INFO | Downloaded 34GiB to GPU in 2.4422s (13.9219GiB/s) | 20260508 05:19:50.574 | INFO | Analyzing chunk frames = [5544960,5545472) | 20260508 05:19:50.574 | INFO | MemInfo: { | 20260508 05:19:50.574 | INFO | "gpu" : " 17 GB", | 20260508 05:19:50.574 | INFO | "huge" : "", | 20260508 05:19:50.574 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:50.574 | INFO | "malloc" : " 14 GB", | 20260508 05:19:50.574 | INFO | "pinned" : " 29 GB" | 20260508 05:19:50.574 | INFO | } | 20260508 05:19:50.574 | INFO | Perf report for the last 5.13505 seconds: | 20260508 05:19:50.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:50.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:19:50.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:19:50.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:50.575 | INFO | Uploaded 36GiB to GPU in 2.41635s (14.8985GiB/s) | 20260508 05:19:50.575 | INFO | Downloaded 34GiB to GPU in 2.42943s (13.9951GiB/s) | 20260508 05:19:55.721 | INFO | Analyzing chunk frames = [5545472,5545984) | 20260508 05:19:55.721 | INFO | MemInfo: { | 20260508 05:19:55.721 | INFO | "gpu" : " 17 GB", | 20260508 05:19:55.721 | INFO | "huge" : "", | 20260508 05:19:55.721 | INFO | "huge_shared" : " 29 GB", | 20260508 05:19:55.721 | INFO | "malloc" : " 14 GB", | 20260508 05:19:55.721 | INFO | "pinned" : " 29 GB" | 20260508 05:19:55.721 | INFO | } | 20260508 05:19:55.721 | INFO | Perf report for the last 5.14616 seconds: | 20260508 05:19:55.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:19:55.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 05:19:55.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:19:55.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:19:55.721 | INFO | Uploaded 36GiB to GPU in 2.4515s (14.6849GiB/s) | 20260508 05:19:55.721 | INFO | Downloaded 34GiB to GPU in 2.43444s (13.9662GiB/s) | 20260508 05:19:55.721 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:19:55.721 | INFO | Total mem usage: 16.7 GiB | 20260508 05:19:55.721 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:19:55.721 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 05:19:55.721 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:19:55.721 | INFO | High Water Breakdown: | 20260508 05:19:55.721 | INFO | 59.5% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:19:55.721 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:19:55.721 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:19:55.721 | INFO | ------------------------------------------ | 20260508 05:19:55.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:19:55.721 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:19:55.721 | INFO | Total mem usage: 26.5 GiB | 20260508 05:19:55.721 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:19:55.721 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:19:55.721 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:19:55.721 | INFO | High Water Breakdown: | 20260508 05:19:55.721 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:19:55.721 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:19:55.721 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:19:55.721 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:19:55.721 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:19:55.721 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:19:55.721 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:19:55.721 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:19:55.721 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:19:55.721 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:19:55.721 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:19:55.721 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:19:55.721 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:19:55.721 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:19:55.721 | INFO | ------------------------------------------ | 20260508 05:19:55.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:19:55.721 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:19:55.721 | INFO | Total mem usage: 27.8 GiB | 20260508 05:19:55.721 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:19:55.721 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:19:55.721 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:19:55.721 | INFO | High Water Breakdown: | 20260508 05:19:55.721 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:19:55.721 | INFO | ------------------------------------------ | 20260508 05:19:55.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:19:55.721 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:19:55.721 | INFO | Total mem usage: 15.7 GiB | 20260508 05:19:55.721 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:19:55.721 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:19:55.721 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:19:55.721 | INFO | High Water Breakdown: | 20260508 05:19:55.721 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:19:55.721 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:19:55.721 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:19:55.721 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:19:55.721 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:19:55.721 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:19:55.721 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:19:55.721 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:19:55.721 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:19:55.721 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:19:55.721 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:19:55.722 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:19:55.722 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:19:55.722 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:19:55.722 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:19:55.722 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:19:55.722 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:19:55.722 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:19:55.722 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:19:55.722 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:19:55.722 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:19:55.722 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:19:55.722 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:19:55.722 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:19:55.722 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:19:55.722 | INFO | ------------------------------------------ | 20260508 05:19:55.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:19:55.722 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:19:55.722 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:19:55.722 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:19:55.722 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.58e+01, 1.40e+01 / 1.60e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.01e-01, 1.23e-02 / 6.15e-03 | 20260508 05:19:55.722 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.43e+00, 5.90e-01 / 1.88e-01 | 20260508 05:19:55.722 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.42e+00, 2.10e-01 / 1.42e-01 | 20260508 05:19:55.722 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.55e+01, 4.01e+00 / 2.13e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.83e+00 / 1.30e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.15e-01 / 1.99e-01 | 20260508 05:19:55.722 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.21e+00, 2.59e-01 / 1.37e-01 | 20260508 05:19:55.722 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.50e-01, 5.08e-02 / 1.81e-02 | 20260508 05:19:55.722 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:19:55.722 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:19:55.722 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:19:55.722 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 24.18%, Upload -- 2.57e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 24.38%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 30.88%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 2.02%, LockUpload -- 2.14e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 2.82%, LockDownload -- 3.00e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 15.71%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:19:55.722 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260508 05:19:55.819 | INFO | Starting a round of Baz IO | 20260508 05:19:59.259 | INFO | Populating RTMetrics report | 20260508 05:19:59.259 | INFO | RTMetrics report populated | 20260508 05:19:59.259 | INFO | Opening tmp RTMetrics json file | 20260508 05:19:59.270 | INFO | Serializing RTMetrics to json | 20260508 05:19:59.271 | INFO | Serialization done, swapping tmp file now... | 20260508 05:19:59.271 | INFO | RTMetrics done | 20260508 05:19:59.330 | INFO | Finished a round of Baz IO: Wrote 1.577 GiB in 3.51 seconds | 20260508 05:19:59.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.89% / 0.00% / 59.11% | 20260508 05:19:59.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.472319% / 0% / 7.15256e-05% | 20260508 05:19:59.330 | INFO | Baz IO overall: | 20260508 05:19:59.330 | INFO | BytesWritten : 2472902851956 | 20260508 05:19:59.330 | INFO | NumEvents : 1193207810712 | 20260508 05:20:00.862 | INFO | Analyzing chunk frames = [5545984,5546496) | 20260508 05:20:00.862 | INFO | MemInfo: { | 20260508 05:20:00.862 | INFO | "gpu" : " 17 GB", | 20260508 05:20:00.862 | INFO | "huge" : "", | 20260508 05:20:00.862 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:00.862 | INFO | "malloc" : " 14 GB", | 20260508 05:20:00.862 | INFO | "pinned" : " 29 GB" | 20260508 05:20:00.862 | INFO | } | 20260508 05:20:00.862 | INFO | Perf report for the last 5.1416 seconds: | 20260508 05:20:00.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:00.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:00.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:00.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:00.862 | INFO | ANALYSIS : 67.9%, 3.0, 106.2ms, 0 | 20260508 05:20:00.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:00.862 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:20:00.862 | INFO | PRE_HQ : 65.8%, 1.6, 55.8ms, 0 | 20260508 05:20:00.862 | INFO | BAZWRITER : 67.7%, 2.7, 93.7ms, 0 | 20260508 05:20:00.862 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:20:00.862 | INFO | Uploaded 36GiB to GPU in 2.55606s (14.0842GiB/s) | 20260508 05:20:00.863 | INFO | Downloaded 38GiB to GPU in 2.68323s (14.162GiB/s) | 20260508 05:20:06.000 | INFO | Analyzing chunk frames = [5546496,5547008) | 20260508 05:20:06.000 | INFO | MemInfo: { | 20260508 05:20:06.000 | INFO | "gpu" : " 17 GB", | 20260508 05:20:06.000 | INFO | "huge" : "", | 20260508 05:20:06.000 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:06.000 | INFO | "malloc" : " 14 GB", | 20260508 05:20:06.000 | INFO | "pinned" : " 29 GB" | 20260508 05:20:06.001 | INFO | } | 20260508 05:20:06.001 | INFO | Perf report for the last 5.1384 seconds: | 20260508 05:20:06.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:06.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:20:06.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:20:06.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:06.001 | INFO | Uploaded 36GiB to GPU in 2.47491s (14.546GiB/s) | 20260508 05:20:06.001 | INFO | Downloaded 34GiB to GPU in 2.43689s (13.9522GiB/s) | 20260508 05:20:11.138 | INFO | Analyzing chunk frames = [5547008,5547520) | 20260508 05:20:11.138 | INFO | MemInfo: { | 20260508 05:20:11.138 | INFO | "gpu" : " 17 GB", | 20260508 05:20:11.138 | INFO | "huge" : "", | 20260508 05:20:11.138 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:11.138 | INFO | "malloc" : " 14 GB", | 20260508 05:20:11.138 | INFO | "pinned" : " 29 GB" | 20260508 05:20:11.138 | INFO | } | 20260508 05:20:11.138 | INFO | Perf report for the last 5.13757 seconds: | 20260508 05:20:11.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:11.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.138 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 05:20:11.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.138 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:20:11.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:11.139 | INFO | Uploaded 36GiB to GPU in 2.52415s (14.2623GiB/s) | 20260508 05:20:11.139 | INFO | Downloaded 34GiB to GPU in 2.41121s (14.1008GiB/s) | 20260508 05:20:16.274 | INFO | Analyzing chunk frames = [5547520,5548032) | 20260508 05:20:16.274 | INFO | MemInfo: { | 20260508 05:20:16.274 | INFO | "gpu" : " 17 GB", | 20260508 05:20:16.274 | INFO | "huge" : "", | 20260508 05:20:16.274 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:16.274 | INFO | "malloc" : " 14 GB", | 20260508 05:20:16.274 | INFO | "pinned" : " 29 GB" | 20260508 05:20:16.275 | INFO | } | 20260508 05:20:16.275 | INFO | Perf report for the last 5.13631 seconds: | 20260508 05:20:16.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:16.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 05:20:16.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:20:16.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:16.275 | INFO | Uploaded 36GiB to GPU in 2.54062s (14.1697GiB/s) | 20260508 05:20:16.275 | INFO | Downloaded 34GiB to GPU in 2.39794s (14.1789GiB/s) | 20260508 05:20:21.420 | INFO | Analyzing chunk frames = [5548032,5548544) | 20260508 05:20:21.420 | INFO | MemInfo: { | 20260508 05:20:21.420 | INFO | "gpu" : " 17 GB", | 20260508 05:20:21.420 | INFO | "huge" : "", | 20260508 05:20:21.420 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:21.420 | INFO | "malloc" : " 14 GB", | 20260508 05:20:21.420 | INFO | "pinned" : " 29 GB" | 20260508 05:20:21.421 | INFO | } | 20260508 05:20:21.421 | INFO | Perf report for the last 5.14609 seconds: | 20260508 05:20:21.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:21.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:20:21.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 05:20:21.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:21.421 | INFO | Uploaded 36GiB to GPU in 2.35779s (15.2685GiB/s) | 20260508 05:20:21.421 | INFO | Downloaded 34GiB to GPU in 2.55917s (13.2856GiB/s) | 20260508 05:20:26.556 | INFO | Analyzing chunk frames = [5548544,5549056) | 20260508 05:20:26.556 | INFO | MemInfo: { | 20260508 05:20:26.556 | INFO | "gpu" : " 17 GB", | 20260508 05:20:26.556 | INFO | "huge" : "", | 20260508 05:20:26.556 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:26.556 | INFO | "malloc" : " 14 GB", | 20260508 05:20:26.556 | INFO | "pinned" : " 29 GB" | 20260508 05:20:26.557 | INFO | } | 20260508 05:20:26.557 | INFO | Perf report for the last 5.13601 seconds: | 20260508 05:20:26.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:26.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:20:26.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:20:26.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:26.557 | INFO | Uploaded 36GiB to GPU in 2.41106s (14.9312GiB/s) | 20260508 05:20:26.557 | INFO | Downloaded 34GiB to GPU in 2.44936s (13.8812GiB/s) | 20260508 05:20:31.698 | INFO | Analyzing chunk frames = [5549056,5549568) | 20260508 05:20:31.698 | INFO | MemInfo: { | 20260508 05:20:31.698 | INFO | "gpu" : " 17 GB", | 20260508 05:20:31.698 | INFO | "huge" : "", | 20260508 05:20:31.698 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:31.698 | INFO | "malloc" : " 14 GB", | 20260508 05:20:31.698 | INFO | "pinned" : " 29 GB" | 20260508 05:20:31.698 | INFO | } | 20260508 05:20:31.698 | INFO | Perf report for the last 5.14134 seconds: | 20260508 05:20:31.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:31.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:20:31.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 05:20:31.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:31.698 | INFO | Uploaded 36GiB to GPU in 2.43583s (14.7794GiB/s) | 20260508 05:20:31.698 | INFO | Downloaded 34GiB to GPU in 2.44654s (13.8972GiB/s) | 20260508 05:20:36.829 | INFO | Analyzing chunk frames = [5549568,5550080) | 20260508 05:20:36.829 | INFO | MemInfo: { | 20260508 05:20:36.829 | INFO | "gpu" : " 17 GB", | 20260508 05:20:36.829 | INFO | "huge" : "", | 20260508 05:20:36.829 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:36.829 | INFO | "malloc" : " 14 GB", | 20260508 05:20:36.829 | INFO | "pinned" : " 29 GB" | 20260508 05:20:36.829 | INFO | } | 20260508 05:20:36.829 | INFO | Perf report for the last 5.13099 seconds: | 20260508 05:20:36.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:36.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.829 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 05:20:36.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.829 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 05:20:36.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:36.830 | INFO | Uploaded 36GiB to GPU in 2.4028s (14.9825GiB/s) | 20260508 05:20:36.830 | INFO | Downloaded 34GiB to GPU in 2.43899s (13.9402GiB/s) | 20260508 05:20:36.927 | INFO | Starting a round of Baz IO | 20260508 05:20:40.345 | INFO | Populating RTMetrics report | 20260508 05:20:40.346 | INFO | RTMetrics report populated | 20260508 05:20:40.346 | INFO | Opening tmp RTMetrics json file | 20260508 05:20:40.357 | INFO | Serializing RTMetrics to json | 20260508 05:20:40.358 | INFO | Serialization done, swapping tmp file now... | 20260508 05:20:40.359 | INFO | RTMetrics done | 20260508 05:20:40.422 | INFO | Finished a round of Baz IO: Wrote 1.577 GiB in 3.494 seconds | 20260508 05:20:40.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.89% / 0.00% / 59.11% | 20260508 05:20:40.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.471258% / 0% / 5.5631e-05% | 20260508 05:20:40.422 | INFO | Baz IO overall: | 20260508 05:20:40.422 | INFO | BytesWritten : 2474596435791 | 20260508 05:20:40.422 | INFO | NumEvents : 1194016287857 | 20260508 05:20:41.974 | INFO | Analyzing chunk frames = [5550080,5550592) | 20260508 05:20:41.974 | INFO | MemInfo: { | 20260508 05:20:41.974 | INFO | "gpu" : " 17 GB", | 20260508 05:20:41.974 | INFO | "huge" : "", | 20260508 05:20:41.974 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:41.974 | INFO | "malloc" : " 14 GB", | 20260508 05:20:41.974 | INFO | "pinned" : " 29 GB" | 20260508 05:20:41.974 | INFO | } | 20260508 05:20:41.974 | INFO | Perf report for the last 5.14532 seconds: | 20260508 05:20:41.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:41.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:41.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:41.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:41.974 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 05:20:41.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:41.974 | INFO | RT_METRICS : 39.5%, 1.0, 20.8ms, 0 | 20260508 05:20:41.974 | INFO | PRE_HQ : 65.4%, 1.6, 55.4ms, 0 | 20260508 05:20:41.975 | INFO | BAZWRITER : 67.4%, 2.7, 93.3ms, 0 | 20260508 05:20:41.975 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:20:41.975 | INFO | Uploaded 36GiB to GPU in 2.54288s (14.1572GiB/s) | 20260508 05:20:41.975 | INFO | Downloaded 38GiB to GPU in 2.69494s (14.1005GiB/s) | 20260508 05:20:47.120 | INFO | Analyzing chunk frames = [5550592,5551104) | 20260508 05:20:47.120 | INFO | MemInfo: { | 20260508 05:20:47.120 | INFO | "gpu" : " 17 GB", | 20260508 05:20:47.120 | INFO | "huge" : "", | 20260508 05:20:47.120 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:47.120 | INFO | "malloc" : " 14 GB", | 20260508 05:20:47.120 | INFO | "pinned" : " 29 GB" | 20260508 05:20:47.121 | INFO | } | 20260508 05:20:47.121 | INFO | Perf report for the last 5.14618 seconds: | 20260508 05:20:47.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:47.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 05:20:47.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:20:47.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:47.121 | INFO | Uploaded 36GiB to GPU in 2.58685s (13.9166GiB/s) | 20260508 05:20:47.121 | INFO | Downloaded 34GiB to GPU in 2.44584s (13.9012GiB/s) | 20260508 05:20:52.254 | INFO | Analyzing chunk frames = [5551104,5551616) | 20260508 05:20:52.254 | INFO | MemInfo: { | 20260508 05:20:52.254 | INFO | "gpu" : " 17 GB", | 20260508 05:20:52.254 | INFO | "huge" : "", | 20260508 05:20:52.254 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:52.254 | INFO | "malloc" : " 14 GB", | 20260508 05:20:52.254 | INFO | "pinned" : " 29 GB" | 20260508 05:20:52.255 | INFO | } | 20260508 05:20:52.255 | INFO | Perf report for the last 5.13459 seconds: | 20260508 05:20:52.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:52.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.255 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 05:20:52.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.255 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 05:20:52.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:52.256 | INFO | Uploaded 36GiB to GPU in 2.53512s (14.2005GiB/s) | 20260508 05:20:52.256 | INFO | Downloaded 34GiB to GPU in 2.41182s (14.0973GiB/s) | 20260508 05:20:57.389 | INFO | Analyzing chunk frames = [5551616,5552128) | 20260508 05:20:57.389 | INFO | MemInfo: { | 20260508 05:20:57.389 | INFO | "gpu" : " 17 GB", | 20260508 05:20:57.389 | INFO | "huge" : "", | 20260508 05:20:57.389 | INFO | "huge_shared" : " 29 GB", | 20260508 05:20:57.389 | INFO | "malloc" : " 14 GB", | 20260508 05:20:57.389 | INFO | "pinned" : " 29 GB" | 20260508 05:20:57.389 | INFO | } | 20260508 05:20:57.389 | INFO | Perf report for the last 5.13379 seconds: | 20260508 05:20:57.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:20:57.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.389 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 05:20:57.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.389 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:20:57.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:20:57.390 | INFO | Uploaded 36GiB to GPU in 2.57527s (13.9791GiB/s) | 20260508 05:20:57.390 | INFO | Downloaded 34GiB to GPU in 2.41021s (14.1067GiB/s) | 20260508 05:21:02.532 | INFO | Analyzing chunk frames = [5552128,5552640) | 20260508 05:21:02.532 | INFO | MemInfo: { | 20260508 05:21:02.532 | INFO | "gpu" : " 17 GB", | 20260508 05:21:02.532 | INFO | "huge" : "", | 20260508 05:21:02.532 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:02.532 | INFO | "malloc" : " 14 GB", | 20260508 05:21:02.532 | INFO | "pinned" : " 29 GB" | 20260508 05:21:02.532 | INFO | } | 20260508 05:21:02.532 | INFO | Perf report for the last 5.14284 seconds: | 20260508 05:21:02.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:02.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:21:02.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:21:02.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:02.532 | INFO | Uploaded 36GiB to GPU in 2.38735s (15.0795GiB/s) | 20260508 05:21:02.532 | INFO | Downloaded 34GiB to GPU in 2.54815s (13.343GiB/s) | 20260508 05:21:07.669 | INFO | Analyzing chunk frames = [5552640,5553152) | 20260508 05:21:07.669 | INFO | MemInfo: { | 20260508 05:21:07.669 | INFO | "gpu" : " 17 GB", | 20260508 05:21:07.669 | INFO | "huge" : "", | 20260508 05:21:07.669 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:07.669 | INFO | "malloc" : " 14 GB", | 20260508 05:21:07.669 | INFO | "pinned" : " 29 GB" | 20260508 05:21:07.669 | INFO | } | 20260508 05:21:07.669 | INFO | Perf report for the last 5.1369 seconds: | 20260508 05:21:07.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:07.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:21:07.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:21:07.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:07.669 | INFO | Uploaded 36GiB to GPU in 2.4574s (14.6496GiB/s) | 20260508 05:21:07.669 | INFO | Downloaded 34GiB to GPU in 2.41228s (14.0945GiB/s) | 20260508 05:21:12.809 | INFO | Analyzing chunk frames = [5553152,5553664) | 20260508 05:21:12.809 | INFO | MemInfo: { | 20260508 05:21:12.809 | INFO | "gpu" : " 17 GB", | 20260508 05:21:12.810 | INFO | "huge" : "", | 20260508 05:21:12.810 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:12.810 | INFO | "malloc" : " 14 GB", | 20260508 05:21:12.810 | INFO | "pinned" : " 29 GB" | 20260508 05:21:12.810 | INFO | } | 20260508 05:21:12.810 | INFO | Perf report for the last 5.14097 seconds: | 20260508 05:21:12.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:12.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:21:12.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:21:12.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:12.810 | INFO | Uploaded 36GiB to GPU in 2.46091s (14.6287GiB/s) | 20260508 05:21:12.810 | INFO | Downloaded 34GiB to GPU in 2.4167s (14.0688GiB/s) | 20260508 05:21:17.952 | INFO | Analyzing chunk frames = [5553664,5554176) | 20260508 05:21:17.952 | INFO | MemInfo: { | 20260508 05:21:17.952 | INFO | "gpu" : " 17 GB", | 20260508 05:21:17.952 | INFO | "huge" : "", | 20260508 05:21:17.952 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:17.952 | INFO | "malloc" : " 14 GB", | 20260508 05:21:17.952 | INFO | "pinned" : " 29 GB" | 20260508 05:21:17.952 | INFO | } | 20260508 05:21:17.952 | INFO | Perf report for the last 5.1422 seconds: | 20260508 05:21:17.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:17.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:21:17.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:21:17.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:17.952 | INFO | Uploaded 36GiB to GPU in 2.45216s (14.681GiB/s) | 20260508 05:21:17.952 | INFO | Downloaded 34GiB to GPU in 2.41385s (14.0854GiB/s) | 20260508 05:21:17.952 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:21:17.952 | INFO | Total mem usage: 16.7 GiB | 20260508 05:21:17.952 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:21:17.952 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 05:21:17.953 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:21:17.953 | INFO | High Water Breakdown: | 20260508 05:21:17.953 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:21:17.953 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:21:17.953 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:21:17.953 | INFO | ------------------------------------------ | 20260508 05:21:17.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:21:17.953 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:21:17.953 | INFO | Total mem usage: 26.5 GiB | 20260508 05:21:17.953 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:21:17.953 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 05:21:17.953 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 05:21:17.953 | INFO | High Water Breakdown: | 20260508 05:21:17.953 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:21:17.953 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:21:17.953 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:21:17.953 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:21:17.953 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:21:17.953 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:21:17.953 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:21:17.953 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:21:17.953 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:21:17.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:21:17.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:21:17.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:21:17.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:21:17.953 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:21:17.953 | INFO | ------------------------------------------ | 20260508 05:21:17.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:21:17.953 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:21:17.953 | INFO | Total mem usage: 27.8 GiB | 20260508 05:21:17.953 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:21:17.953 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:21:17.953 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:21:17.953 | INFO | High Water Breakdown: | 20260508 05:21:17.953 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:21:17.953 | INFO | ------------------------------------------ | 20260508 05:21:17.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:21:17.953 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:21:17.953 | INFO | Total mem usage: 15.7 GiB | 20260508 05:21:17.953 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:21:17.953 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:21:17.953 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:21:17.953 | INFO | High Water Breakdown: | 20260508 05:21:17.953 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:21:17.953 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:21:17.953 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:21:17.953 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:21:17.953 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:21:17.953 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:21:17.953 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:21:17.953 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:21:17.953 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:21:17.953 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:21:17.953 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:21:17.953 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:21:17.953 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:21:17.953 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:21:17.953 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:21:17.953 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:21:17.953 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:21:17.953 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:21:17.953 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:21:17.953 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:21:17.953 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:21:17.953 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:21:17.953 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:21:17.953 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:21:17.953 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:21:17.953 | INFO | ------------------------------------------ | 20260508 05:21:17.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:21:17.954 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:21:17.954 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:21:17.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:21:17.954 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.41e+01 / 1.74e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.17e-02 / 4.03e-03 | 20260508 05:21:17.954 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.48e+00, 5.88e-01 / 1.82e-01 | 20260508 05:21:17.954 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.09e-01 / 1.22e-01 | 20260508 05:21:17.954 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.52e+01, 3.80e+00 / 1.91e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.77e+00 / 1.38e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.60e+00, 4.18e-01 / 2.43e-01 | 20260508 05:21:17.954 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.21e+00, 2.58e-01 / 1.34e-01 | 20260508 05:21:17.954 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.52e-01, 4.99e-02 / 2.20e-02 | 20260508 05:21:17.954 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:21:17.954 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:21:17.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:21:17.954 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 24.34%, Upload -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 24.26%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 2.75%, LockDownload -- 2.92e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:21:17.954 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 05:21:18.049 | INFO | Starting a round of Baz IO | 20260508 05:21:21.467 | INFO | Populating RTMetrics report | 20260508 05:21:21.467 | INFO | RTMetrics report populated | 20260508 05:21:21.467 | INFO | Opening tmp RTMetrics json file | 20260508 05:21:21.480 | INFO | Serializing RTMetrics to json | 20260508 05:21:21.482 | INFO | Serialization done, swapping tmp file now... | 20260508 05:21:21.482 | INFO | RTMetrics done | 20260508 05:21:21.544 | INFO | Finished a round of Baz IO: Wrote 1.575 GiB in 3.494 seconds | 20260508 05:21:21.544 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.89% / 0.00% / 59.11% | 20260508 05:21:21.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.469542% / 1.58946e-05% / 9.53674e-05% | 20260508 05:21:21.544 | INFO | Baz IO overall: | 20260508 05:21:21.544 | INFO | BytesWritten : 2476287914666 | 20260508 05:21:21.544 | INFO | NumEvents : 1194823214565 | 20260508 05:21:23.086 | INFO | Analyzing chunk frames = [5554176,5554688) | 20260508 05:21:23.086 | INFO | MemInfo: { | 20260508 05:21:23.086 | INFO | "gpu" : " 17 GB", | 20260508 05:21:23.086 | INFO | "huge" : "", | 20260508 05:21:23.086 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:23.086 | INFO | "malloc" : " 14 GB", | 20260508 05:21:23.086 | INFO | "pinned" : " 29 GB" | 20260508 05:21:23.086 | INFO | } | 20260508 05:21:23.086 | INFO | Perf report for the last 5.13449 seconds: | 20260508 05:21:23.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:23.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:23.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:23.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:23.087 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:21:23.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:23.087 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:21:23.087 | INFO | PRE_HQ : 65.7%, 1.6, 55.4ms, 0 | 20260508 05:21:23.087 | INFO | BAZWRITER : 67.4%, 2.7, 92.9ms, 0 | 20260508 05:21:23.087 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:21:23.087 | INFO | Uploaded 36GiB to GPU in 2.55536s (14.0881GiB/s) | 20260508 05:21:23.087 | INFO | Downloaded 38GiB to GPU in 2.68486s (14.1534GiB/s) | 20260508 05:21:28.228 | INFO | Analyzing chunk frames = [5554688,5555200) | 20260508 05:21:28.228 | INFO | MemInfo: { | 20260508 05:21:28.228 | INFO | "gpu" : " 17 GB", | 20260508 05:21:28.228 | INFO | "huge" : "", | 20260508 05:21:28.228 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:28.228 | INFO | "malloc" : " 14 GB", | 20260508 05:21:28.228 | INFO | "pinned" : " 29 GB" | 20260508 05:21:28.228 | INFO | } | 20260508 05:21:28.228 | INFO | Perf report for the last 5.14181 seconds: | 20260508 05:21:28.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:28.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.228 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 05:21:28.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.228 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 05:21:28.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:28.229 | INFO | Uploaded 36GiB to GPU in 2.52439s (14.2609GiB/s) | 20260508 05:21:28.229 | INFO | Downloaded 34GiB to GPU in 2.43284s (13.9754GiB/s) | 20260508 05:21:33.368 | INFO | Analyzing chunk frames = [5555200,5555712) | 20260508 05:21:33.368 | INFO | MemInfo: { | 20260508 05:21:33.368 | INFO | "gpu" : " 17 GB", | 20260508 05:21:33.368 | INFO | "huge" : "", | 20260508 05:21:33.368 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:33.368 | INFO | "malloc" : " 14 GB", | 20260508 05:21:33.368 | INFO | "pinned" : " 29 GB" | 20260508 05:21:33.368 | INFO | } | 20260508 05:21:33.368 | INFO | Perf report for the last 5.13946 seconds: | 20260508 05:21:33.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:33.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 05:21:33.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:21:33.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:33.368 | INFO | Uploaded 36GiB to GPU in 2.55647s (14.0819GiB/s) | 20260508 05:21:33.368 | INFO | Downloaded 34GiB to GPU in 2.39833s (14.1765GiB/s) | 20260508 05:21:38.504 | INFO | Analyzing chunk frames = [5555712,5556224) | 20260508 05:21:38.505 | INFO | MemInfo: { | 20260508 05:21:38.505 | INFO | "gpu" : " 17 GB", | 20260508 05:21:38.505 | INFO | "huge" : "", | 20260508 05:21:38.505 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:38.505 | INFO | "malloc" : " 14 GB", | 20260508 05:21:38.505 | INFO | "pinned" : " 29 GB" | 20260508 05:21:38.505 | INFO | } | 20260508 05:21:38.505 | INFO | Perf report for the last 5.13685 seconds: | 20260508 05:21:38.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:38.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | ANALYSIS : 67.2%, 3.0, 103.9ms, 0 | 20260508 05:21:38.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:21:38.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:38.505 | INFO | Uploaded 36GiB to GPU in 2.60528s (13.8181GiB/s) | 20260508 05:21:38.505 | INFO | Downloaded 34GiB to GPU in 2.41174s (14.0977GiB/s) | 20260508 05:21:43.640 | INFO | Analyzing chunk frames = [5556224,5556736) | 20260508 05:21:43.640 | INFO | MemInfo: { | 20260508 05:21:43.640 | INFO | "gpu" : " 17 GB", | 20260508 05:21:43.640 | INFO | "huge" : "", | 20260508 05:21:43.640 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:43.640 | INFO | "malloc" : " 14 GB", | 20260508 05:21:43.640 | INFO | "pinned" : " 29 GB" | 20260508 05:21:43.640 | INFO | } | 20260508 05:21:43.640 | INFO | Perf report for the last 5.13557 seconds: | 20260508 05:21:43.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:43.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.640 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 05:21:43.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.640 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:21:43.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:43.641 | INFO | Uploaded 36GiB to GPU in 2.37549s (15.1547GiB/s) | 20260508 05:21:43.641 | INFO | Downloaded 34GiB to GPU in 2.54719s (13.348GiB/s) | 20260508 05:21:48.784 | INFO | Analyzing chunk frames = [5556736,5557248) | 20260508 05:21:48.784 | INFO | MemInfo: { | 20260508 05:21:48.784 | INFO | "gpu" : " 17 GB", | 20260508 05:21:48.784 | INFO | "huge" : "", | 20260508 05:21:48.784 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:48.784 | INFO | "malloc" : " 14 GB", | 20260508 05:21:48.784 | INFO | "pinned" : " 29 GB" | 20260508 05:21:48.784 | INFO | } | 20260508 05:21:48.784 | INFO | Perf report for the last 5.14409 seconds: | 20260508 05:21:48.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:48.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.784 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 05:21:48.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.784 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:21:48.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:48.785 | INFO | Uploaded 36GiB to GPU in 2.45627s (14.6564GiB/s) | 20260508 05:21:48.785 | INFO | Downloaded 34GiB to GPU in 2.41605s (14.0725GiB/s) | 20260508 05:21:53.925 | INFO | Analyzing chunk frames = [5557248,5557760) | 20260508 05:21:53.926 | INFO | MemInfo: { | 20260508 05:21:53.926 | INFO | "gpu" : " 17 GB", | 20260508 05:21:53.926 | INFO | "huge" : "", | 20260508 05:21:53.926 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:53.926 | INFO | "malloc" : " 14 GB", | 20260508 05:21:53.926 | INFO | "pinned" : " 29 GB" | 20260508 05:21:53.926 | INFO | } | 20260508 05:21:53.926 | INFO | Perf report for the last 5.14129 seconds: | 20260508 05:21:53.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:53.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:21:53.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:21:53.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:53.926 | INFO | Uploaded 36GiB to GPU in 2.45697s (14.6522GiB/s) | 20260508 05:21:53.926 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0225GiB/s) | 20260508 05:21:59.070 | INFO | Analyzing chunk frames = [5557760,5558272) | 20260508 05:21:59.070 | INFO | MemInfo: { | 20260508 05:21:59.070 | INFO | "gpu" : " 17 GB", | 20260508 05:21:59.070 | INFO | "huge" : "", | 20260508 05:21:59.070 | INFO | "huge_shared" : " 29 GB", | 20260508 05:21:59.070 | INFO | "malloc" : " 14 GB", | 20260508 05:21:59.070 | INFO | "pinned" : " 29 GB" | 20260508 05:21:59.071 | INFO | } | 20260508 05:21:59.071 | INFO | Perf report for the last 5.14499 seconds: | 20260508 05:21:59.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:21:59.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 05:21:59.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:21:59.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:21:59.071 | INFO | Uploaded 36GiB to GPU in 2.4682s (14.5855GiB/s) | 20260508 05:21:59.071 | INFO | Downloaded 34GiB to GPU in 2.41446s (14.0818GiB/s) | 20260508 05:21:59.184 | INFO | Starting a round of Baz IO | 20260508 05:22:02.599 | INFO | Populating RTMetrics report | 20260508 05:22:02.599 | INFO | RTMetrics report populated | 20260508 05:22:02.599 | INFO | Opening tmp RTMetrics json file | 20260508 05:22:02.610 | INFO | Serializing RTMetrics to json | 20260508 05:22:02.611 | INFO | Serialization done, swapping tmp file now... | 20260508 05:22:02.612 | INFO | RTMetrics done | 20260508 05:22:02.671 | INFO | Finished a round of Baz IO: Wrote 1.576 GiB in 3.486 seconds | 20260508 05:22:02.671 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.88% / 0.00% / 59.11% | 20260508 05:22:02.671 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.468405% / 0% / 7.15256e-05% | 20260508 05:22:02.671 | INFO | Baz IO overall: | 20260508 05:22:02.671 | INFO | BytesWritten : 2477980156666 | 20260508 05:22:02.671 | INFO | NumEvents : 1195630444212 | 20260508 05:22:04.204 | INFO | Analyzing chunk frames = [5558272,5558784) | 20260508 05:22:04.204 | INFO | MemInfo: { | 20260508 05:22:04.204 | INFO | "gpu" : " 17 GB", | 20260508 05:22:04.204 | INFO | "huge" : "", | 20260508 05:22:04.204 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:04.204 | INFO | "malloc" : " 14 GB", | 20260508 05:22:04.204 | INFO | "pinned" : " 29 GB" | 20260508 05:22:04.204 | INFO | } | 20260508 05:22:04.204 | INFO | Perf report for the last 5.13316 seconds: | 20260508 05:22:04.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:04.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:04.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:04.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:04.204 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 05:22:04.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:04.204 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260508 05:22:04.204 | INFO | PRE_HQ : 66.0%, 1.7, 58.0ms, 0 | 20260508 05:22:04.204 | INFO | BAZWRITER : 67.4%, 2.8, 95.7ms, 0 | 20260508 05:22:04.204 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:22:04.204 | INFO | Uploaded 36GiB to GPU in 2.60301s (13.8302GiB/s) | 20260508 05:22:04.204 | INFO | Downloaded 38GiB to GPU in 2.69783s (14.0854GiB/s) | 20260508 05:22:09.339 | INFO | Analyzing chunk frames = [5558784,5559296) | 20260508 05:22:09.339 | INFO | MemInfo: { | 20260508 05:22:09.339 | INFO | "gpu" : " 17 GB", | 20260508 05:22:09.339 | INFO | "huge" : "", | 20260508 05:22:09.339 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:09.339 | INFO | "malloc" : " 14 GB", | 20260508 05:22:09.339 | INFO | "pinned" : " 29 GB" | 20260508 05:22:09.339 | INFO | } | 20260508 05:22:09.339 | INFO | Perf report for the last 5.13514 seconds: | 20260508 05:22:09.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:09.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 05:22:09.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:22:09.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:09.339 | INFO | Uploaded 36GiB to GPU in 2.51607s (14.308GiB/s) | 20260508 05:22:09.339 | INFO | Downloaded 34GiB to GPU in 2.42754s (14.0059GiB/s) | 20260508 05:22:14.484 | INFO | Analyzing chunk frames = [5559296,5559808) | 20260508 05:22:14.484 | INFO | MemInfo: { | 20260508 05:22:14.484 | INFO | "gpu" : " 17 GB", | 20260508 05:22:14.484 | INFO | "huge" : "", | 20260508 05:22:14.484 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:14.484 | INFO | "malloc" : " 14 GB", | 20260508 05:22:14.484 | INFO | "pinned" : " 29 GB" | 20260508 05:22:14.484 | INFO | } | 20260508 05:22:14.484 | INFO | Perf report for the last 5.14554 seconds: | 20260508 05:22:14.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:14.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.485 | INFO | ANALYSIS : 67.4%, 3.0, 104.8ms, 0 | 20260508 05:22:14.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.485 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:22:14.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:14.485 | INFO | Uploaded 36GiB to GPU in 2.55732s (14.0772GiB/s) | 20260508 05:22:14.485 | INFO | Downloaded 34GiB to GPU in 2.41894s (14.0557GiB/s) | 20260508 05:22:19.621 | INFO | Analyzing chunk frames = [5559808,5560320) | 20260508 05:22:19.621 | INFO | MemInfo: { | 20260508 05:22:19.621 | INFO | "gpu" : " 17 GB", | 20260508 05:22:19.621 | INFO | "huge" : "", | 20260508 05:22:19.622 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:19.622 | INFO | "malloc" : " 14 GB", | 20260508 05:22:19.622 | INFO | "pinned" : " 29 GB" | 20260508 05:22:19.622 | INFO | } | 20260508 05:22:19.622 | INFO | Perf report for the last 5.13722 seconds: | 20260508 05:22:19.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:19.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 05:22:19.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:22:19.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:19.622 | INFO | Uploaded 36GiB to GPU in 2.56212s (14.0508GiB/s) | 20260508 05:22:19.622 | INFO | Downloaded 34GiB to GPU in 2.37531s (14.3139GiB/s) | 20260508 05:22:24.762 | INFO | Analyzing chunk frames = [5560320,5560832) | 20260508 05:22:24.762 | INFO | MemInfo: { | 20260508 05:22:24.762 | INFO | "gpu" : " 17 GB", | 20260508 05:22:24.762 | INFO | "huge" : "", | 20260508 05:22:24.762 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:24.762 | INFO | "malloc" : " 14 GB", | 20260508 05:22:24.762 | INFO | "pinned" : " 29 GB" | 20260508 05:22:24.762 | INFO | } | 20260508 05:22:24.762 | INFO | Perf report for the last 5.1406 seconds: | 20260508 05:22:24.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:24.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.762 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 05:22:24.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.763 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:22:24.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:24.763 | INFO | Uploaded 36GiB to GPU in 2.3589s (15.2613GiB/s) | 20260508 05:22:24.763 | INFO | Downloaded 34GiB to GPU in 2.54832s (13.3421GiB/s) | 20260508 05:22:29.897 | INFO | Analyzing chunk frames = [5560832,5561344) | 20260508 05:22:29.897 | INFO | MemInfo: { | 20260508 05:22:29.897 | INFO | "gpu" : " 17 GB", | 20260508 05:22:29.897 | INFO | "huge" : "", | 20260508 05:22:29.897 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:29.897 | INFO | "malloc" : " 14 GB", | 20260508 05:22:29.897 | INFO | "pinned" : " 29 GB" | 20260508 05:22:29.897 | INFO | } | 20260508 05:22:29.898 | INFO | Perf report for the last 5.1352 seconds: | 20260508 05:22:29.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:29.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:22:29.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:22:29.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:29.898 | INFO | Uploaded 36GiB to GPU in 2.44232s (14.7401GiB/s) | 20260508 05:22:29.898 | INFO | Downloaded 34GiB to GPU in 2.43456s (13.9656GiB/s) | 20260508 05:22:35.039 | INFO | Analyzing chunk frames = [5561344,5561856) | 20260508 05:22:35.039 | INFO | MemInfo: { | 20260508 05:22:35.039 | INFO | "gpu" : " 17 GB", | 20260508 05:22:35.039 | INFO | "huge" : "", | 20260508 05:22:35.039 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:35.039 | INFO | "malloc" : " 14 GB", | 20260508 05:22:35.039 | INFO | "pinned" : " 29 GB" | 20260508 05:22:35.039 | INFO | } | 20260508 05:22:35.039 | INFO | Perf report for the last 5.14181 seconds: | 20260508 05:22:35.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:35.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.039 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:22:35.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.039 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:22:35.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:35.040 | INFO | Uploaded 36GiB to GPU in 2.43847s (14.7634GiB/s) | 20260508 05:22:35.040 | INFO | Downloaded 34GiB to GPU in 2.43109s (13.9855GiB/s) | 20260508 05:22:40.179 | INFO | Analyzing chunk frames = [5561856,5562368) | 20260508 05:22:40.179 | INFO | MemInfo: { | 20260508 05:22:40.179 | INFO | "gpu" : " 17 GB", | 20260508 05:22:40.179 | INFO | "huge" : "", | 20260508 05:22:40.179 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:40.179 | INFO | "malloc" : " 14 GB", | 20260508 05:22:40.179 | INFO | "pinned" : " 29 GB" | 20260508 05:22:40.179 | INFO | } | 20260508 05:22:40.179 | INFO | Perf report for the last 5.13951 seconds: | 20260508 05:22:40.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:40.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:22:40.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:22:40.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:40.179 | INFO | Uploaded 36GiB to GPU in 2.43623s (14.7769GiB/s) | 20260508 05:22:40.179 | INFO | Downloaded 34GiB to GPU in 2.42357s (14.0289GiB/s) | 20260508 05:22:40.179 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:22:40.179 | INFO | Total mem usage: 16.7 GiB | 20260508 05:22:40.179 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:22:40.179 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.7%) | 20260508 05:22:40.179 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:22:40.180 | INFO | High Water Breakdown: | 20260508 05:22:40.180 | INFO | 59.7% : 8.6 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:22:40.180 | INFO | 40.2% : 5.79 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:22:40.180 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:22:40.180 | INFO | ------------------------------------------ | 20260508 05:22:40.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:22:40.180 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:22:40.180 | INFO | Total mem usage: 26.5 GiB | 20260508 05:22:40.180 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:22:40.180 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:22:40.180 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:22:40.180 | INFO | High Water Breakdown: | 20260508 05:22:40.180 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:22:40.180 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:22:40.180 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:22:40.180 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:22:40.180 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:22:40.180 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:22:40.180 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:22:40.180 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:22:40.180 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:22:40.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:22:40.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:22:40.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:22:40.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:22:40.180 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:22:40.180 | INFO | ------------------------------------------ | 20260508 05:22:40.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:22:40.180 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:22:40.180 | INFO | Total mem usage: 27.8 GiB | 20260508 05:22:40.180 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:22:40.180 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:22:40.180 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:22:40.180 | INFO | High Water Breakdown: | 20260508 05:22:40.180 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:22:40.180 | INFO | ------------------------------------------ | 20260508 05:22:40.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:22:40.180 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:22:40.180 | INFO | Total mem usage: 15.7 GiB | 20260508 05:22:40.180 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:22:40.180 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:22:40.180 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:22:40.180 | INFO | High Water Breakdown: | 20260508 05:22:40.180 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:22:40.180 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:22:40.180 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:22:40.180 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:22:40.180 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:22:40.180 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:22:40.180 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:22:40.180 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:22:40.180 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:22:40.180 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:22:40.180 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:22:40.180 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:22:40.180 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:22:40.180 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:22:40.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:22:40.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:22:40.180 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:22:40.180 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:22:40.180 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:22:40.180 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:22:40.180 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:22:40.180 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:22:40.180 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:22:40.180 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:22:40.180 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:22:40.180 | INFO | ------------------------------------------ | 20260508 05:22:40.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:22:40.180 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:22:40.180 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:22:40.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:22:40.180 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.42e+01 / 1.30e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.17e-02 / 5.37e-03 | 20260508 05:22:40.180 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.51e+00, 5.97e-01 / 1.60e-01 | 20260508 05:22:40.180 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.11e-01 / 1.21e-01 | 20260508 05:22:40.180 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.48e+01, 3.78e+00 / 1.53e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.30e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.37e+00 / 2.60e+00, 4.18e-01 / 2.61e-01 | 20260508 05:22:40.180 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.25e+00, 2.63e-01 / 1.26e-01 | 20260508 05:22:40.180 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.58e-01, 5.21e-02 / 2.99e-02 | 20260508 05:22:40.180 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:22:40.180 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:22:40.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:22:40.180 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 24.45%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 24.19%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 2.43%, LockUpload -- 2.58e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 05:22:40.180 | INFO | PerfReport: 2.48%, LockDownload -- 2.63e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 05:22:40.181 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:22:40.181 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 05:22:40.275 | INFO | Starting a round of Baz IO | 20260508 05:22:43.696 | INFO | Populating RTMetrics report | 20260508 05:22:43.697 | INFO | RTMetrics report populated | 20260508 05:22:43.697 | INFO | Opening tmp RTMetrics json file | 20260508 05:22:43.711 | INFO | Serializing RTMetrics to json | 20260508 05:22:43.712 | INFO | Serialization done, swapping tmp file now... | 20260508 05:22:43.713 | INFO | RTMetrics done | 20260508 05:22:43.777 | INFO | Finished a round of Baz IO: Wrote 1.577 GiB in 3.501 seconds | 20260508 05:22:43.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.88% / 0.00% / 59.12% | 20260508 05:22:43.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.468512% / 0% / 8.34465e-05% | 20260508 05:22:43.777 | INFO | Baz IO overall: | 20260508 05:22:43.777 | INFO | BytesWritten : 2479673475220 | 20260508 05:22:43.777 | INFO | NumEvents : 1196438331720 | 20260508 05:22:45.316 | INFO | Analyzing chunk frames = [5562368,5562880) | 20260508 05:22:45.316 | INFO | MemInfo: { | 20260508 05:22:45.316 | INFO | "gpu" : " 17 GB", | 20260508 05:22:45.316 | INFO | "huge" : "", | 20260508 05:22:45.316 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:45.316 | INFO | "malloc" : " 14 GB", | 20260508 05:22:45.316 | INFO | "pinned" : " 29 GB" | 20260508 05:22:45.316 | INFO | } | 20260508 05:22:45.316 | INFO | Perf report for the last 5.13708 seconds: | 20260508 05:22:45.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:45.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:45.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:45.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:45.316 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 05:22:45.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:45.316 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:22:45.316 | INFO | PRE_HQ : 65.3%, 1.6, 55.2ms, 0 | 20260508 05:22:45.316 | INFO | BAZWRITER : 67.6%, 2.7, 93.8ms, 0 | 20260508 05:22:45.316 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:22:45.317 | INFO | Uploaded 36GiB to GPU in 2.53804s (14.1842GiB/s) | 20260508 05:22:45.317 | INFO | Downloaded 38GiB to GPU in 2.71041s (14.02GiB/s) | 20260508 05:22:50.453 | INFO | Analyzing chunk frames = [5562880,5563392) | 20260508 05:22:50.453 | INFO | MemInfo: { | 20260508 05:22:50.453 | INFO | "gpu" : " 17 GB", | 20260508 05:22:50.453 | INFO | "huge" : "", | 20260508 05:22:50.453 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:50.453 | INFO | "malloc" : " 14 GB", | 20260508 05:22:50.453 | INFO | "pinned" : " 29 GB" | 20260508 05:22:50.453 | INFO | } | 20260508 05:22:50.453 | INFO | Perf report for the last 5.13676 seconds: | 20260508 05:22:50.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:50.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 05:22:50.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:22:50.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:50.453 | INFO | Uploaded 36GiB to GPU in 2.55868s (14.0697GiB/s) | 20260508 05:22:50.453 | INFO | Downloaded 34GiB to GPU in 2.42931s (13.9957GiB/s) | 20260508 05:22:55.591 | INFO | Analyzing chunk frames = [5563392,5563904) | 20260508 05:22:55.591 | INFO | MemInfo: { | 20260508 05:22:55.591 | INFO | "gpu" : " 17 GB", | 20260508 05:22:55.591 | INFO | "huge" : "", | 20260508 05:22:55.592 | INFO | "huge_shared" : " 29 GB", | 20260508 05:22:55.592 | INFO | "malloc" : " 14 GB", | 20260508 05:22:55.592 | INFO | "pinned" : " 29 GB" | 20260508 05:22:55.592 | INFO | } | 20260508 05:22:55.592 | INFO | Perf report for the last 5.13896 seconds: | 20260508 05:22:55.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:22:55.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:22:55.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:22:55.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:22:55.592 | INFO | Uploaded 36GiB to GPU in 2.5351s (14.2006GiB/s) | 20260508 05:22:55.592 | INFO | Downloaded 34GiB to GPU in 2.41627s (14.0713GiB/s) | 20260508 05:23:00.737 | INFO | Analyzing chunk frames = [5563904,5564416) | 20260508 05:23:00.737 | INFO | MemInfo: { | 20260508 05:23:00.737 | INFO | "gpu" : " 17 GB", | 20260508 05:23:00.737 | INFO | "huge" : "", | 20260508 05:23:00.737 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:00.737 | INFO | "malloc" : " 14 GB", | 20260508 05:23:00.737 | INFO | "pinned" : " 29 GB" | 20260508 05:23:00.737 | INFO | } | 20260508 05:23:00.737 | INFO | Perf report for the last 5.14553 seconds: | 20260508 05:23:00.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:00.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.737 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 05:23:00.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.737 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:23:00.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:00.738 | INFO | Uploaded 36GiB to GPU in 2.56709s (14.0237GiB/s) | 20260508 05:23:00.738 | INFO | Downloaded 34GiB to GPU in 2.4132s (14.0891GiB/s) | 20260508 05:23:05.881 | INFO | Analyzing chunk frames = [5564416,5564928) | 20260508 05:23:05.881 | INFO | MemInfo: { | 20260508 05:23:05.881 | INFO | "gpu" : " 17 GB", | 20260508 05:23:05.881 | INFO | "huge" : "", | 20260508 05:23:05.881 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:05.881 | INFO | "malloc" : " 14 GB", | 20260508 05:23:05.881 | INFO | "pinned" : " 29 GB" | 20260508 05:23:05.881 | INFO | } | 20260508 05:23:05.881 | INFO | Perf report for the last 5.14423 seconds: | 20260508 05:23:05.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:05.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | ANALYSIS : 66.6%, 3.0, 104.2ms, 0 | 20260508 05:23:05.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:23:05.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:05.882 | INFO | Uploaded 36GiB to GPU in 2.46976s (14.5763GiB/s) | 20260508 05:23:05.882 | INFO | Downloaded 34GiB to GPU in 2.5234s (13.4739GiB/s) | 20260508 05:23:11.019 | INFO | Analyzing chunk frames = [5564928,5565440) | 20260508 05:23:11.019 | INFO | MemInfo: { | 20260508 05:23:11.019 | INFO | "gpu" : " 17 GB", | 20260508 05:23:11.019 | INFO | "huge" : "", | 20260508 05:23:11.019 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:11.019 | INFO | "malloc" : " 14 GB", | 20260508 05:23:11.019 | INFO | "pinned" : " 29 GB" | 20260508 05:23:11.019 | INFO | } | 20260508 05:23:11.019 | INFO | Perf report for the last 5.13726 seconds: | 20260508 05:23:11.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:11.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:23:11.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:23:11.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:11.019 | INFO | Uploaded 36GiB to GPU in 2.42984s (14.8158GiB/s) | 20260508 05:23:11.019 | INFO | Downloaded 34GiB to GPU in 2.41667s (14.0689GiB/s) | 20260508 05:23:16.153 | INFO | Analyzing chunk frames = [5565440,5565952) | 20260508 05:23:16.153 | INFO | MemInfo: { | 20260508 05:23:16.153 | INFO | "gpu" : " 17 GB", | 20260508 05:23:16.153 | INFO | "huge" : "", | 20260508 05:23:16.153 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:16.153 | INFO | "malloc" : " 14 GB", | 20260508 05:23:16.153 | INFO | "pinned" : " 29 GB" | 20260508 05:23:16.153 | INFO | } | 20260508 05:23:16.153 | INFO | Perf report for the last 5.13468 seconds: | 20260508 05:23:16.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:16.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.154 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:23:16.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.154 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:23:16.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:16.154 | INFO | Uploaded 36GiB to GPU in 2.45225s (14.6804GiB/s) | 20260508 05:23:16.154 | INFO | Downloaded 34GiB to GPU in 2.42934s (13.9956GiB/s) | 20260508 05:23:21.295 | INFO | Analyzing chunk frames = [5565952,5566464) | 20260508 05:23:21.295 | INFO | MemInfo: { | 20260508 05:23:21.295 | INFO | "gpu" : " 17 GB", | 20260508 05:23:21.295 | INFO | "huge" : "", | 20260508 05:23:21.295 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:21.295 | INFO | "malloc" : " 14 GB", | 20260508 05:23:21.295 | INFO | "pinned" : " 29 GB" | 20260508 05:23:21.295 | INFO | } | 20260508 05:23:21.295 | INFO | Perf report for the last 5.14159 seconds: | 20260508 05:23:21.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:21.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:23:21.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:23:21.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:21.295 | INFO | Uploaded 36GiB to GPU in 2.47285s (14.5581GiB/s) | 20260508 05:23:21.295 | INFO | Downloaded 34GiB to GPU in 2.43793s (13.9463GiB/s) | 20260508 05:23:21.393 | INFO | Starting a round of Baz IO | 20260508 05:23:24.820 | INFO | Populating RTMetrics report | 20260508 05:23:24.821 | INFO | RTMetrics report populated | 20260508 05:23:24.821 | INFO | Opening tmp RTMetrics json file | 20260508 05:23:24.835 | INFO | Serializing RTMetrics to json | 20260508 05:23:24.836 | INFO | Serialization done, swapping tmp file now... | 20260508 05:23:24.837 | INFO | RTMetrics done | 20260508 05:23:24.896 | INFO | Finished a round of Baz IO: Wrote 1.582 GiB in 3.503 seconds | 20260508 05:23:24.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.88% / 0.00% / 59.12% | 20260508 05:23:24.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.470889% / 7.94729e-06% / 6.75519e-05% | 20260508 05:23:24.896 | INFO | Baz IO overall: | 20260508 05:23:24.896 | INFO | BytesWritten : 2481372125354 | 20260508 05:23:24.896 | INFO | NumEvents : 1197247733052 | 20260508 05:23:26.433 | INFO | Analyzing chunk frames = [5566464,5566976) | 20260508 05:23:26.433 | INFO | MemInfo: { | 20260508 05:23:26.433 | INFO | "gpu" : " 17 GB", | 20260508 05:23:26.433 | INFO | "huge" : "", | 20260508 05:23:26.433 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:26.433 | INFO | "malloc" : " 14 GB", | 20260508 05:23:26.433 | INFO | "pinned" : " 29 GB" | 20260508 05:23:26.433 | INFO | } | 20260508 05:23:26.433 | INFO | Perf report for the last 5.13795 seconds: | 20260508 05:23:26.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:26.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:26.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:26.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:26.433 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 05:23:26.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:26.433 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:23:26.433 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 05:23:26.433 | INFO | BAZWRITER : 67.7%, 2.7, 93.7ms, 0 | 20260508 05:23:26.433 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:23:26.434 | INFO | Uploaded 36GiB to GPU in 2.5422s (14.161GiB/s) | 20260508 05:23:26.434 | INFO | Downloaded 38GiB to GPU in 2.70119s (14.0679GiB/s) | 20260508 05:23:31.572 | INFO | Analyzing chunk frames = [5566976,5567488) | 20260508 05:23:31.572 | INFO | MemInfo: { | 20260508 05:23:31.572 | INFO | "gpu" : " 17 GB", | 20260508 05:23:31.572 | INFO | "huge" : "", | 20260508 05:23:31.572 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:31.572 | INFO | "malloc" : " 14 GB", | 20260508 05:23:31.572 | INFO | "pinned" : " 29 GB" | 20260508 05:23:31.572 | INFO | } | 20260508 05:23:31.572 | INFO | Perf report for the last 5.13927 seconds: | 20260508 05:23:31.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:31.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.572 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 05:23:31.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.572 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:23:31.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:31.573 | INFO | Uploaded 36GiB to GPU in 2.57286s (13.9922GiB/s) | 20260508 05:23:31.573 | INFO | Downloaded 34GiB to GPU in 2.44026s (13.933GiB/s) | 20260508 05:23:36.711 | INFO | Analyzing chunk frames = [5567488,5568000) | 20260508 05:23:36.711 | INFO | MemInfo: { | 20260508 05:23:36.711 | INFO | "gpu" : " 17 GB", | 20260508 05:23:36.711 | INFO | "huge" : "", | 20260508 05:23:36.711 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:36.711 | INFO | "malloc" : " 14 GB", | 20260508 05:23:36.711 | INFO | "pinned" : " 29 GB" | 20260508 05:23:36.711 | INFO | } | 20260508 05:23:36.711 | INFO | Perf report for the last 5.13856 seconds: | 20260508 05:23:36.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:36.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.711 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 05:23:36.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.711 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:23:36.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:36.712 | INFO | Uploaded 36GiB to GPU in 2.53684s (14.1909GiB/s) | 20260508 05:23:36.712 | INFO | Downloaded 34GiB to GPU in 2.42705s (14.0088GiB/s) | 20260508 05:23:41.849 | INFO | Analyzing chunk frames = [5568000,5568512) | 20260508 05:23:41.849 | INFO | MemInfo: { | 20260508 05:23:41.849 | INFO | "gpu" : " 17 GB", | 20260508 05:23:41.849 | INFO | "huge" : "", | 20260508 05:23:41.849 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:41.849 | INFO | "malloc" : " 14 GB", | 20260508 05:23:41.849 | INFO | "pinned" : " 29 GB" | 20260508 05:23:41.849 | INFO | } | 20260508 05:23:41.850 | INFO | Perf report for the last 5.13863 seconds: | 20260508 05:23:41.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:41.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 05:23:41.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 05:23:41.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:41.850 | INFO | Uploaded 36GiB to GPU in 2.54914s (14.1224GiB/s) | 20260508 05:23:41.850 | INFO | Downloaded 34GiB to GPU in 2.38921s (14.2306GiB/s) | 20260508 05:23:46.993 | INFO | Analyzing chunk frames = [5568512,5569024) | 20260508 05:23:46.993 | INFO | MemInfo: { | 20260508 05:23:46.993 | INFO | "gpu" : " 17 GB", | 20260508 05:23:46.993 | INFO | "huge" : "", | 20260508 05:23:46.993 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:46.993 | INFO | "malloc" : " 14 GB", | 20260508 05:23:46.993 | INFO | "pinned" : " 29 GB" | 20260508 05:23:46.993 | INFO | } | 20260508 05:23:46.993 | INFO | Perf report for the last 5.14368 seconds: | 20260508 05:23:46.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:46.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:23:46.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:23:46.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:46.993 | INFO | Uploaded 36GiB to GPU in 2.33s (15.4506GiB/s) | 20260508 05:23:46.994 | INFO | Downloaded 34GiB to GPU in 2.50339s (13.5816GiB/s) | 20260508 05:23:52.130 | INFO | Analyzing chunk frames = [5569024,5569536) | 20260508 05:23:52.130 | INFO | MemInfo: { | 20260508 05:23:52.130 | INFO | "gpu" : " 17 GB", | 20260508 05:23:52.130 | INFO | "huge" : "", | 20260508 05:23:52.131 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:52.131 | INFO | "malloc" : " 14 GB", | 20260508 05:23:52.131 | INFO | "pinned" : " 29 GB" | 20260508 05:23:52.131 | INFO | } | 20260508 05:23:52.131 | INFO | Perf report for the last 5.13749 seconds: | 20260508 05:23:52.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:52.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:23:52.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:23:52.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:52.131 | INFO | Uploaded 36GiB to GPU in 2.45576s (14.6594GiB/s) | 20260508 05:23:52.131 | INFO | Downloaded 34GiB to GPU in 2.41987s (14.0504GiB/s) | 20260508 05:23:57.268 | INFO | Analyzing chunk frames = [5569536,5570048) | 20260508 05:23:57.268 | INFO | MemInfo: { | 20260508 05:23:57.268 | INFO | "gpu" : " 17 GB", | 20260508 05:23:57.268 | INFO | "huge" : "", | 20260508 05:23:57.268 | INFO | "huge_shared" : " 29 GB", | 20260508 05:23:57.268 | INFO | "malloc" : " 14 GB", | 20260508 05:23:57.268 | INFO | "pinned" : " 29 GB" | 20260508 05:23:57.268 | INFO | } | 20260508 05:23:57.268 | INFO | Perf report for the last 5.13746 seconds: | 20260508 05:23:57.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:23:57.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.268 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 05:23:57.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.268 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:23:57.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:23:57.269 | INFO | Uploaded 36GiB to GPU in 2.48466s (14.4889GiB/s) | 20260508 05:23:57.269 | INFO | Downloaded 34GiB to GPU in 2.42129s (14.0421GiB/s) | 20260508 05:24:02.405 | INFO | Analyzing chunk frames = [5570048,5570560) | 20260508 05:24:02.405 | INFO | MemInfo: { | 20260508 05:24:02.405 | INFO | "gpu" : " 17 GB", | 20260508 05:24:02.405 | INFO | "huge" : "", | 20260508 05:24:02.405 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:02.405 | INFO | "malloc" : " 14 GB", | 20260508 05:24:02.405 | INFO | "pinned" : " 29 GB" | 20260508 05:24:02.405 | INFO | } | 20260508 05:24:02.405 | INFO | Perf report for the last 5.13659 seconds: | 20260508 05:24:02.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:02.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:24:02.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 05:24:02.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:02.405 | INFO | Uploaded 36GiB to GPU in 2.48083s (14.5113GiB/s) | 20260508 05:24:02.405 | INFO | Downloaded 34GiB to GPU in 2.42889s (13.9982GiB/s) | 20260508 05:24:02.405 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:24:02.405 | INFO | Total mem usage: 16.7 GiB | 20260508 05:24:02.405 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:24:02.405 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 05:24:02.405 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:24:02.405 | INFO | High Water Breakdown: | 20260508 05:24:02.405 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:24:02.405 | INFO | 40.4% : 5.76 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:24:02.405 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:24:02.405 | INFO | ------------------------------------------ | 20260508 05:24:02.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:24:02.406 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:24:02.406 | INFO | Total mem usage: 26.5 GiB | 20260508 05:24:02.406 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:24:02.406 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 05:24:02.406 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 05:24:02.406 | INFO | High Water Breakdown: | 20260508 05:24:02.406 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:24:02.406 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:24:02.406 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:24:02.406 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:24:02.406 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:24:02.406 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:24:02.406 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:24:02.406 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:24:02.406 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:24:02.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:24:02.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:24:02.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:24:02.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:24:02.406 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:24:02.406 | INFO | ------------------------------------------ | 20260508 05:24:02.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:24:02.406 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:24:02.406 | INFO | Total mem usage: 27.8 GiB | 20260508 05:24:02.406 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:24:02.406 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:24:02.406 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:24:02.406 | INFO | High Water Breakdown: | 20260508 05:24:02.406 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:24:02.406 | INFO | ------------------------------------------ | 20260508 05:24:02.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:24:02.406 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:24:02.406 | INFO | Total mem usage: 15.7 GiB | 20260508 05:24:02.406 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:24:02.406 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:24:02.406 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:24:02.406 | INFO | High Water Breakdown: | 20260508 05:24:02.406 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:24:02.406 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:24:02.406 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:24:02.406 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:24:02.406 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:24:02.406 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:24:02.406 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:24:02.406 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:24:02.406 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:24:02.406 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:24:02.406 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:24:02.406 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:24:02.406 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:24:02.406 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:24:02.406 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:24:02.406 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:24:02.406 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:24:02.406 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:24:02.406 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:24:02.406 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:24:02.406 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:24:02.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:24:02.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:24:02.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:24:02.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:24:02.407 | INFO | ------------------------------------------ | 20260508 05:24:02.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:24:02.407 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:24:02.407 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:24:02.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:24:02.407 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 6.53e-01 | 20260508 05:24:02.407 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.03e-01, 1.20e-02 / 6.05e-03 | 20260508 05:24:02.407 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.41e+00, 5.89e-01 / 2.52e-01 | 20260508 05:24:02.407 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 1.31e-01 | 20260508 05:24:02.407 | INFO | PerfReport: 15.89%, DME -- 4.16e+01 / 4.48e+01, 3.64e+00 / 1.32e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 1.50e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.53e+00, 4.19e-01 / 2.69e-01 | 20260508 05:24:02.407 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.20e+00, 2.62e-01 / 1.39e-01 | 20260508 05:24:02.407 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.58e-01, 5.13e-02 / 2.77e-02 | 20260508 05:24:02.407 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:24:02.407 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:24:02.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:24:02.407 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 24.49%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 2.29%, LockUpload -- 2.44e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 2.59%, LockDownload -- 2.76e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.98e+01 / 0.00e+00 | 20260508 05:24:02.407 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260508 05:24:02.503 | INFO | Starting a round of Baz IO | 20260508 05:24:05.929 | INFO | Populating RTMetrics report | 20260508 05:24:05.929 | INFO | RTMetrics report populated | 20260508 05:24:05.929 | INFO | Opening tmp RTMetrics json file | 20260508 05:24:05.944 | INFO | Serializing RTMetrics to json | 20260508 05:24:05.947 | INFO | Serialization done, swapping tmp file now... | 20260508 05:24:05.947 | INFO | RTMetrics done | 20260508 05:24:06.007 | INFO | Finished a round of Baz IO: Wrote 1.579 GiB in 3.504 seconds | 20260508 05:24:06.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.88% / 0.00% / 59.12% | 20260508 05:24:06.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.468258% / 0% / 7.54992e-05% | 20260508 05:24:06.007 | INFO | Baz IO overall: | 20260508 05:24:06.007 | INFO | BytesWritten : 2483067739120 | 20260508 05:24:06.007 | INFO | NumEvents : 1198055516504 | 20260508 05:24:07.549 | INFO | Analyzing chunk frames = [5570560,5571072) | 20260508 05:24:07.549 | INFO | MemInfo: { | 20260508 05:24:07.550 | INFO | "gpu" : " 17 GB", | 20260508 05:24:07.550 | INFO | "huge" : "", | 20260508 05:24:07.550 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:07.550 | INFO | "malloc" : " 14 GB", | 20260508 05:24:07.550 | INFO | "pinned" : " 29 GB" | 20260508 05:24:07.550 | INFO | } | 20260508 05:24:07.550 | INFO | Perf report for the last 5.14477 seconds: | 20260508 05:24:07.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:07.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:07.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:07.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:07.550 | INFO | ANALYSIS : 67.8%, 3.0, 106.1ms, 0 | 20260508 05:24:07.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:07.550 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:24:07.550 | INFO | PRE_HQ : 65.5%, 1.6, 55.3ms, 0 | 20260508 05:24:07.550 | INFO | BAZWRITER : 67.6%, 2.7, 94.4ms, 0 | 20260508 05:24:07.550 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:24:07.550 | INFO | Uploaded 36GiB to GPU in 2.57924s (13.9576GiB/s) | 20260508 05:24:07.550 | INFO | Downloaded 38GiB to GPU in 2.68866s (14.1334GiB/s) | 20260508 05:24:12.693 | INFO | Analyzing chunk frames = [5571072,5571584) | 20260508 05:24:12.693 | INFO | MemInfo: { | 20260508 05:24:12.693 | INFO | "gpu" : " 17 GB", | 20260508 05:24:12.693 | INFO | "huge" : "", | 20260508 05:24:12.693 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:12.693 | INFO | "malloc" : " 14 GB", | 20260508 05:24:12.693 | INFO | "pinned" : " 29 GB" | 20260508 05:24:12.693 | INFO | } | 20260508 05:24:12.693 | INFO | Perf report for the last 5.14341 seconds: | 20260508 05:24:12.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:12.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:24:12.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:24:12.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:12.693 | INFO | Uploaded 36GiB to GPU in 2.52073s (14.2816GiB/s) | 20260508 05:24:12.693 | INFO | Downloaded 34GiB to GPU in 2.43089s (13.9866GiB/s) | 20260508 05:24:17.830 | INFO | Analyzing chunk frames = [5571584,5572096) | 20260508 05:24:17.830 | INFO | MemInfo: { | 20260508 05:24:17.830 | INFO | "gpu" : " 17 GB", | 20260508 05:24:17.830 | INFO | "huge" : "", | 20260508 05:24:17.830 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:17.830 | INFO | "malloc" : " 14 GB", | 20260508 05:24:17.830 | INFO | "pinned" : " 29 GB" | 20260508 05:24:17.830 | INFO | } | 20260508 05:24:17.830 | INFO | Perf report for the last 5.13738 seconds: | 20260508 05:24:17.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:17.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.830 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 05:24:17.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.831 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:24:17.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:17.831 | INFO | Uploaded 36GiB to GPU in 2.5579s (14.074GiB/s) | 20260508 05:24:17.831 | INFO | Downloaded 34GiB to GPU in 2.4048s (14.1384GiB/s) | 20260508 05:24:22.964 | INFO | Analyzing chunk frames = [5572096,5572608) | 20260508 05:24:22.964 | INFO | MemInfo: { | 20260508 05:24:22.964 | INFO | "gpu" : " 17 GB", | 20260508 05:24:22.964 | INFO | "huge" : "", | 20260508 05:24:22.964 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:22.964 | INFO | "malloc" : " 14 GB", | 20260508 05:24:22.964 | INFO | "pinned" : " 29 GB" | 20260508 05:24:22.964 | INFO | } | 20260508 05:24:22.964 | INFO | Perf report for the last 5.13399 seconds: | 20260508 05:24:22.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:22.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.964 | INFO | ANALYSIS : 67.2%, 3.0, 103.8ms, 0 | 20260508 05:24:22.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.964 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:24:22.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:22.965 | INFO | Uploaded 36GiB to GPU in 2.59337s (13.8816GiB/s) | 20260508 05:24:22.965 | INFO | Downloaded 34GiB to GPU in 2.39807s (14.1781GiB/s) | 20260508 05:24:28.100 | INFO | Analyzing chunk frames = [5572608,5573120) | 20260508 05:24:28.100 | INFO | MemInfo: { | 20260508 05:24:28.100 | INFO | "gpu" : " 17 GB", | 20260508 05:24:28.100 | INFO | "huge" : "", | 20260508 05:24:28.100 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:28.100 | INFO | "malloc" : " 14 GB", | 20260508 05:24:28.100 | INFO | "pinned" : " 29 GB" | 20260508 05:24:28.100 | INFO | } | 20260508 05:24:28.100 | INFO | Perf report for the last 5.13573 seconds: | 20260508 05:24:28.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:28.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.100 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 05:24:28.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.100 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:24:28.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:28.101 | INFO | Uploaded 36GiB to GPU in 2.47166s (14.5651GiB/s) | 20260508 05:24:28.101 | INFO | Downloaded 34GiB to GPU in 2.52503s (13.4652GiB/s) | 20260508 05:24:33.245 | INFO | Analyzing chunk frames = [5573120,5573632) | 20260508 05:24:33.245 | INFO | MemInfo: { | 20260508 05:24:33.245 | INFO | "gpu" : " 17 GB", | 20260508 05:24:33.245 | INFO | "huge" : "", | 20260508 05:24:33.245 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:33.245 | INFO | "malloc" : " 14 GB", | 20260508 05:24:33.246 | INFO | "pinned" : " 29 GB" | 20260508 05:24:33.246 | INFO | } | 20260508 05:24:33.246 | INFO | Perf report for the last 5.14555 seconds: | 20260508 05:24:33.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:33.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:24:33.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:24:33.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:33.246 | INFO | Uploaded 36GiB to GPU in 2.45489s (14.6646GiB/s) | 20260508 05:24:33.246 | INFO | Downloaded 34GiB to GPU in 2.41319s (14.0893GiB/s) | 20260508 05:24:38.386 | INFO | Analyzing chunk frames = [5573632,5574144) | 20260508 05:24:38.386 | INFO | MemInfo: { | 20260508 05:24:38.386 | INFO | "gpu" : " 17 GB", | 20260508 05:24:38.386 | INFO | "huge" : "", | 20260508 05:24:38.386 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:38.386 | INFO | "malloc" : " 14 GB", | 20260508 05:24:38.386 | INFO | "pinned" : " 29 GB" | 20260508 05:24:38.386 | INFO | } | 20260508 05:24:38.386 | INFO | Perf report for the last 5.14051 seconds: | 20260508 05:24:38.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:38.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.386 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:24:38.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.386 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:24:38.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:38.387 | INFO | Uploaded 36GiB to GPU in 2.44381s (14.7311GiB/s) | 20260508 05:24:38.387 | INFO | Downloaded 34GiB to GPU in 2.41576s (14.0742GiB/s) | 20260508 05:24:43.528 | INFO | Analyzing chunk frames = [5574144,5574656) | 20260508 05:24:43.528 | INFO | MemInfo: { | 20260508 05:24:43.528 | INFO | "gpu" : " 17 GB", | 20260508 05:24:43.528 | INFO | "huge" : "", | 20260508 05:24:43.528 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:43.528 | INFO | "malloc" : " 14 GB", | 20260508 05:24:43.528 | INFO | "pinned" : " 29 GB" | 20260508 05:24:43.528 | INFO | } | 20260508 05:24:43.528 | INFO | Perf report for the last 5.14178 seconds: | 20260508 05:24:43.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:43.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.528 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:24:43.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.528 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:24:43.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:43.529 | INFO | Uploaded 36GiB to GPU in 2.4796s (14.5185GiB/s) | 20260508 05:24:43.529 | INFO | Downloaded 34GiB to GPU in 2.43337s (13.9724GiB/s) | 20260508 05:24:43.626 | INFO | Starting a round of Baz IO | 20260508 05:24:47.045 | INFO | Populating RTMetrics report | 20260508 05:24:47.045 | INFO | RTMetrics report populated | 20260508 05:24:47.045 | INFO | Opening tmp RTMetrics json file | 20260508 05:24:47.053 | INFO | Serializing RTMetrics to json | 20260508 05:24:47.059 | INFO | Serialization done, swapping tmp file now... | 20260508 05:24:47.060 | INFO | RTMetrics done | 20260508 05:24:47.122 | INFO | Finished a round of Baz IO: Wrote 1.578 GiB in 3.496 seconds | 20260508 05:24:47.122 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.88% / 0.00% / 59.12% | 20260508 05:24:47.122 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.468596% / 3.97364e-06% / 8.34465e-05% | 20260508 05:24:47.122 | INFO | Baz IO overall: | 20260508 05:24:47.122 | INFO | BytesWritten : 2484761617896 | 20260508 05:24:47.122 | INFO | NumEvents : 1198863030466 | 20260508 05:24:48.663 | INFO | Analyzing chunk frames = [5574656,5575168) | 20260508 05:24:48.663 | INFO | MemInfo: { | 20260508 05:24:48.663 | INFO | "gpu" : " 17 GB", | 20260508 05:24:48.663 | INFO | "huge" : "", | 20260508 05:24:48.663 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:48.663 | INFO | "malloc" : " 14 GB", | 20260508 05:24:48.663 | INFO | "pinned" : " 29 GB" | 20260508 05:24:48.663 | INFO | } | 20260508 05:24:48.663 | INFO | Perf report for the last 5.13519 seconds: | 20260508 05:24:48.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:48.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:48.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:48.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:48.663 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:24:48.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:48.663 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 05:24:48.663 | INFO | PRE_HQ : 66.3%, 1.7, 59.4ms, 0 | 20260508 05:24:48.663 | INFO | BAZWRITER : 67.7%, 2.9, 99.4ms, 0 | 20260508 05:24:48.663 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:24:48.664 | INFO | Uploaded 36GiB to GPU in 2.56033s (14.0607GiB/s) | 20260508 05:24:48.664 | INFO | Downloaded 38GiB to GPU in 2.67829s (14.1882GiB/s) | 20260508 05:24:53.802 | INFO | Analyzing chunk frames = [5575168,5575680) | 20260508 05:24:53.802 | INFO | MemInfo: { | 20260508 05:24:53.802 | INFO | "gpu" : " 17 GB", | 20260508 05:24:53.802 | INFO | "huge" : "", | 20260508 05:24:53.802 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:53.802 | INFO | "malloc" : " 14 GB", | 20260508 05:24:53.802 | INFO | "pinned" : " 29 GB" | 20260508 05:24:53.802 | INFO | } | 20260508 05:24:53.802 | INFO | Perf report for the last 5.13893 seconds: | 20260508 05:24:53.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:53.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.802 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 05:24:53.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.802 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:24:53.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:53.803 | INFO | Uploaded 36GiB to GPU in 2.48429s (14.491GiB/s) | 20260508 05:24:53.803 | INFO | Downloaded 34GiB to GPU in 2.44156s (13.9255GiB/s) | 20260508 05:24:58.942 | INFO | Analyzing chunk frames = [5575680,5576192) | 20260508 05:24:58.942 | INFO | MemInfo: { | 20260508 05:24:58.942 | INFO | "gpu" : " 17 GB", | 20260508 05:24:58.942 | INFO | "huge" : "", | 20260508 05:24:58.942 | INFO | "huge_shared" : " 29 GB", | 20260508 05:24:58.942 | INFO | "malloc" : " 14 GB", | 20260508 05:24:58.942 | INFO | "pinned" : " 29 GB" | 20260508 05:24:58.942 | INFO | } | 20260508 05:24:58.942 | INFO | Perf report for the last 5.13973 seconds: | 20260508 05:24:58.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:24:58.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 05:24:58.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:24:58.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:24:58.942 | INFO | Uploaded 36GiB to GPU in 2.52942s (14.2325GiB/s) | 20260508 05:24:58.942 | INFO | Downloaded 34GiB to GPU in 2.40349s (14.1461GiB/s) | 20260508 05:25:04.080 | INFO | Analyzing chunk frames = [5576192,5576704) | 20260508 05:25:04.080 | INFO | MemInfo: { | 20260508 05:25:04.080 | INFO | "gpu" : " 17 GB", | 20260508 05:25:04.080 | INFO | "huge" : "", | 20260508 05:25:04.080 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:04.080 | INFO | "malloc" : " 14 GB", | 20260508 05:25:04.080 | INFO | "pinned" : " 29 GB" | 20260508 05:25:04.080 | INFO | } | 20260508 05:25:04.080 | INFO | Perf report for the last 5.13843 seconds: | 20260508 05:25:04.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:04.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 05:25:04.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:25:04.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:04.081 | INFO | Uploaded 36GiB to GPU in 2.54044s (14.1708GiB/s) | 20260508 05:25:04.081 | INFO | Downloaded 34GiB to GPU in 2.40654s (14.1281GiB/s) | 20260508 05:25:09.220 | INFO | Analyzing chunk frames = [5576704,5577216) | 20260508 05:25:09.220 | INFO | MemInfo: { | 20260508 05:25:09.220 | INFO | "gpu" : " 17 GB", | 20260508 05:25:09.220 | INFO | "huge" : "", | 20260508 05:25:09.220 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:09.220 | INFO | "malloc" : " 14 GB", | 20260508 05:25:09.220 | INFO | "pinned" : " 29 GB" | 20260508 05:25:09.220 | INFO | } | 20260508 05:25:09.220 | INFO | Perf report for the last 5.13983 seconds: | 20260508 05:25:09.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:09.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.220 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 05:25:09.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.220 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:25:09.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:09.221 | INFO | Uploaded 36GiB to GPU in 2.386s (15.088GiB/s) | 20260508 05:25:09.221 | INFO | Downloaded 34GiB to GPU in 2.57001s (13.2295GiB/s) | 20260508 05:25:14.354 | INFO | Analyzing chunk frames = [5577216,5577728) | 20260508 05:25:14.354 | INFO | MemInfo: { | 20260508 05:25:14.354 | INFO | "gpu" : " 17 GB", | 20260508 05:25:14.354 | INFO | "huge" : "", | 20260508 05:25:14.354 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:14.354 | INFO | "malloc" : " 14 GB", | 20260508 05:25:14.354 | INFO | "pinned" : " 29 GB" | 20260508 05:25:14.354 | INFO | } | 20260508 05:25:14.354 | INFO | Perf report for the last 5.13412 seconds: | 20260508 05:25:14.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:14.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.354 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:25:14.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.354 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:25:14.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:14.355 | INFO | Uploaded 36GiB to GPU in 2.40544s (14.9661GiB/s) | 20260508 05:25:14.355 | INFO | Downloaded 34GiB to GPU in 2.44268s (13.9191GiB/s) | 20260508 05:25:19.499 | INFO | Analyzing chunk frames = [5577728,5578240) | 20260508 05:25:19.499 | INFO | MemInfo: { | 20260508 05:25:19.499 | INFO | "gpu" : " 17 GB", | 20260508 05:25:19.499 | INFO | "huge" : "", | 20260508 05:25:19.499 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:19.499 | INFO | "malloc" : " 14 GB", | 20260508 05:25:19.499 | INFO | "pinned" : " 29 GB" | 20260508 05:25:19.499 | INFO | } | 20260508 05:25:19.499 | INFO | Perf report for the last 5.14507 seconds: | 20260508 05:25:19.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:19.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.499 | INFO | ANALYSIS : 65.9%, 3.0, 103.2ms, 0 | 20260508 05:25:19.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.500 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:25:19.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:19.500 | INFO | Uploaded 36GiB to GPU in 2.37585s (15.1525GiB/s) | 20260508 05:25:19.500 | INFO | Downloaded 34GiB to GPU in 2.44035s (13.9324GiB/s) | 20260508 05:25:24.642 | INFO | Analyzing chunk frames = [5578240,5578752) | 20260508 05:25:24.642 | INFO | MemInfo: { | 20260508 05:25:24.642 | INFO | "gpu" : " 17 GB", | 20260508 05:25:24.642 | INFO | "huge" : "", | 20260508 05:25:24.643 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:24.643 | INFO | "malloc" : " 14 GB", | 20260508 05:25:24.643 | INFO | "pinned" : " 29 GB" | 20260508 05:25:24.643 | INFO | } | 20260508 05:25:24.643 | INFO | Perf report for the last 5.14335 seconds: | 20260508 05:25:24.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:24.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:25:24.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 05:25:24.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:24.643 | INFO | Uploaded 36GiB to GPU in 2.4393s (14.7584GiB/s) | 20260508 05:25:24.643 | INFO | Downloaded 34GiB to GPU in 2.45027s (13.876GiB/s) | 20260508 05:25:24.643 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:25:24.643 | INFO | Total mem usage: 16.7 GiB | 20260508 05:25:24.643 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:25:24.643 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17%) | 20260508 05:25:24.643 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:25:24.643 | INFO | High Water Breakdown: | 20260508 05:25:24.643 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:25:24.643 | INFO | 40.4% : 5.76 GiB ( 85.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:25:24.643 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:25:24.643 | INFO | ------------------------------------------ | 20260508 05:25:24.643 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:25:24.643 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:25:24.643 | INFO | Total mem usage: 26.5 GiB | 20260508 05:25:24.643 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:25:24.643 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:25:24.643 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:25:24.643 | INFO | High Water Breakdown: | 20260508 05:25:24.643 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:25:24.643 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:25:24.643 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:25:24.643 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:25:24.643 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:25:24.643 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:25:24.644 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:25:24.644 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:25:24.644 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:25:24.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:25:24.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:25:24.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:25:24.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:25:24.644 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:25:24.644 | INFO | ------------------------------------------ | 20260508 05:25:24.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:25:24.644 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:25:24.644 | INFO | Total mem usage: 27.8 GiB | 20260508 05:25:24.644 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:25:24.644 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:25:24.644 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:25:24.644 | INFO | High Water Breakdown: | 20260508 05:25:24.644 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:25:24.644 | INFO | ------------------------------------------ | 20260508 05:25:24.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:25:24.644 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:25:24.644 | INFO | Total mem usage: 15.7 GiB | 20260508 05:25:24.644 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:25:24.644 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:25:24.644 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:25:24.644 | INFO | High Water Breakdown: | 20260508 05:25:24.644 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:25:24.644 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:25:24.644 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:25:24.644 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:25:24.644 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:25:24.644 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:25:24.644 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:25:24.644 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:25:24.644 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:25:24.644 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:25:24.644 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:25:24.644 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:25:24.644 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:25:24.644 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:25:24.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:25:24.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:25:24.644 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:25:24.644 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:25:24.644 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:25:24.644 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:25:24.644 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:25:24.644 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:25:24.644 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:25:24.644 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:25:24.644 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:25:24.644 | INFO | ------------------------------------------ | 20260508 05:25:24.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:25:24.645 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:25:24.645 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:25:24.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:25:24.645 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.37e+01 / 1.74e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.06e-01, 1.12e-02 / 1.06e-02 | 20260508 05:25:24.645 | INFO | PerfReport: 12.11%, Baseline -- 3.97e+00 / 4.37e+00, 5.84e-01 / 2.71e-01 | 20260508 05:25:24.645 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.21e-01 | 20260508 05:25:24.645 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.50e+01, 3.81e+00 / 2.09e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.82e+00 / 1.47e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.17e-01 / 2.59e-01 | 20260508 05:25:24.645 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.22e+00, 2.62e-01 / 1.61e-01 | 20260508 05:25:24.645 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.55e-01, 5.03e-02 / 2.34e-02 | 20260508 05:25:24.645 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:25:24.645 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:25:24.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:25:24.645 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 24.41%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 24.24%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 2.29%, LockUpload -- 2.43e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 2.54%, LockDownload -- 2.70e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 15.67%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:25:24.645 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260508 05:25:24.755 | INFO | Starting a round of Baz IO | 20260508 05:25:28.164 | INFO | Populating RTMetrics report | 20260508 05:25:28.165 | INFO | RTMetrics report populated | 20260508 05:25:28.165 | INFO | Opening tmp RTMetrics json file | 20260508 05:25:28.166 | INFO | Serializing RTMetrics to json | 20260508 05:25:28.167 | INFO | Serialization done, swapping tmp file now... | 20260508 05:25:28.167 | INFO | RTMetrics done | 20260508 05:25:28.238 | INFO | Finished a round of Baz IO: Wrote 1.577 GiB in 3.482 seconds | 20260508 05:25:28.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.00% / 59.12% | 20260508 05:25:28.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.467213% / 7.94729e-06% / 6.35783e-05% | 20260508 05:25:28.238 | INFO | Baz IO overall: | 20260508 05:25:28.238 | INFO | BytesWritten : 2486454735427 | 20260508 05:25:28.238 | INFO | NumEvents : 1199670407338 | 20260508 05:25:29.783 | INFO | Analyzing chunk frames = [5578752,5579264) | 20260508 05:25:29.783 | INFO | MemInfo: { | 20260508 05:25:29.783 | INFO | "gpu" : " 17 GB", | 20260508 05:25:29.783 | INFO | "huge" : "", | 20260508 05:25:29.783 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:29.783 | INFO | "malloc" : " 14 GB", | 20260508 05:25:29.783 | INFO | "pinned" : " 29 GB" | 20260508 05:25:29.783 | INFO | } | 20260508 05:25:29.783 | INFO | Perf report for the last 5.14008 seconds: | 20260508 05:25:29.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:29.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:29.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:29.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:29.783 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 05:25:29.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:29.783 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 05:25:29.783 | INFO | PRE_HQ : 65.7%, 1.6, 55.3ms, 0 | 20260508 05:25:29.783 | INFO | BAZWRITER : 67.7%, 2.7, 94.3ms, 0 | 20260508 05:25:29.783 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:25:29.783 | INFO | Uploaded 36GiB to GPU in 2.51718s (14.3017GiB/s) | 20260508 05:25:29.783 | INFO | Downloaded 38GiB to GPU in 2.69s (14.1264GiB/s) | 20260508 05:25:34.914 | INFO | Analyzing chunk frames = [5579264,5579776) | 20260508 05:25:34.914 | INFO | MemInfo: { | 20260508 05:25:34.914 | INFO | "gpu" : " 17 GB", | 20260508 05:25:34.914 | INFO | "huge" : "", | 20260508 05:25:34.914 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:34.914 | INFO | "malloc" : " 14 GB", | 20260508 05:25:34.914 | INFO | "pinned" : " 29 GB" | 20260508 05:25:34.914 | INFO | } | 20260508 05:25:34.914 | INFO | Perf report for the last 5.13138 seconds: | 20260508 05:25:34.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:34.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.914 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 05:25:34.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.915 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:25:34.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:34.915 | INFO | Uploaded 36GiB to GPU in 2.48781s (14.4705GiB/s) | 20260508 05:25:34.915 | INFO | Downloaded 34GiB to GPU in 2.43714s (13.9508GiB/s) | 20260508 05:25:40.054 | INFO | Analyzing chunk frames = [5579776,5580288) | 20260508 05:25:40.054 | INFO | MemInfo: { | 20260508 05:25:40.054 | INFO | "gpu" : " 17 GB", | 20260508 05:25:40.054 | INFO | "huge" : "", | 20260508 05:25:40.054 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:40.054 | INFO | "malloc" : " 14 GB", | 20260508 05:25:40.054 | INFO | "pinned" : " 29 GB" | 20260508 05:25:40.054 | INFO | } | 20260508 05:25:40.054 | INFO | Perf report for the last 5.13979 seconds: | 20260508 05:25:40.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:40.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.054 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:25:40.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.054 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:25:40.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:40.055 | INFO | Uploaded 36GiB to GPU in 2.55304s (14.1008GiB/s) | 20260508 05:25:40.055 | INFO | Downloaded 34GiB to GPU in 2.41114s (14.1012GiB/s) | 20260508 05:25:45.193 | INFO | Analyzing chunk frames = [5580288,5580800) | 20260508 05:25:45.193 | INFO | MemInfo: { | 20260508 05:25:45.193 | INFO | "gpu" : " 17 GB", | 20260508 05:25:45.193 | INFO | "huge" : "", | 20260508 05:25:45.193 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:45.193 | INFO | "malloc" : " 14 GB", | 20260508 05:25:45.193 | INFO | "pinned" : " 29 GB" | 20260508 05:25:45.193 | INFO | } | 20260508 05:25:45.193 | INFO | Perf report for the last 5.13905 seconds: | 20260508 05:25:45.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:45.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.193 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:25:45.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.193 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:25:45.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:45.194 | INFO | Uploaded 36GiB to GPU in 2.5278s (14.2416GiB/s) | 20260508 05:25:45.194 | INFO | Downloaded 34GiB to GPU in 2.38191s (14.2743GiB/s) | 20260508 05:25:50.339 | INFO | Analyzing chunk frames = [5580800,5581312) | 20260508 05:25:50.339 | INFO | MemInfo: { | 20260508 05:25:50.339 | INFO | "gpu" : " 17 GB", | 20260508 05:25:50.339 | INFO | "huge" : "", | 20260508 05:25:50.339 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:50.339 | INFO | "malloc" : " 14 GB", | 20260508 05:25:50.339 | INFO | "pinned" : " 29 GB" | 20260508 05:25:50.339 | INFO | } | 20260508 05:25:50.339 | INFO | Perf report for the last 5.14605 seconds: | 20260508 05:25:50.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:50.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.339 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 05:25:50.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.339 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 05:25:50.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:50.340 | INFO | Uploaded 36GiB to GPU in 2.38547s (15.0914GiB/s) | 20260508 05:25:50.340 | INFO | Downloaded 34GiB to GPU in 2.56153s (13.2733GiB/s) | 20260508 05:25:55.474 | INFO | Analyzing chunk frames = [5581312,5581824) | 20260508 05:25:55.474 | INFO | MemInfo: { | 20260508 05:25:55.474 | INFO | "gpu" : " 17 GB", | 20260508 05:25:55.474 | INFO | "huge" : "", | 20260508 05:25:55.474 | INFO | "huge_shared" : " 29 GB", | 20260508 05:25:55.474 | INFO | "malloc" : " 14 GB", | 20260508 05:25:55.474 | INFO | "pinned" : " 29 GB" | 20260508 05:25:55.474 | INFO | } | 20260508 05:25:55.475 | INFO | Perf report for the last 5.13532 seconds: | 20260508 05:25:55.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:25:55.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:25:55.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:25:55.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:25:55.475 | INFO | Uploaded 36GiB to GPU in 2.44483s (14.7249GiB/s) | 20260508 05:25:55.475 | INFO | Downloaded 34GiB to GPU in 2.43286s (13.9753GiB/s) | 20260508 05:26:00.616 | INFO | Analyzing chunk frames = [5581824,5582336) | 20260508 05:26:00.616 | INFO | MemInfo: { | 20260508 05:26:00.616 | INFO | "gpu" : " 17 GB", | 20260508 05:26:00.616 | INFO | "huge" : "", | 20260508 05:26:00.616 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:00.616 | INFO | "malloc" : " 14 GB", | 20260508 05:26:00.616 | INFO | "pinned" : " 29 GB" | 20260508 05:26:00.616 | INFO | } | 20260508 05:26:00.616 | INFO | Perf report for the last 5.14181 seconds: | 20260508 05:26:00.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:00.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.616 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:26:00.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.617 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 05:26:00.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:00.617 | INFO | Uploaded 36GiB to GPU in 2.44531s (14.722GiB/s) | 20260508 05:26:00.617 | INFO | Downloaded 34GiB to GPU in 2.42438s (14.0242GiB/s) | 20260508 05:26:05.757 | INFO | Analyzing chunk frames = [5582336,5582848) | 20260508 05:26:05.757 | INFO | MemInfo: { | 20260508 05:26:05.757 | INFO | "gpu" : " 17 GB", | 20260508 05:26:05.757 | INFO | "huge" : "", | 20260508 05:26:05.757 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:05.757 | INFO | "malloc" : " 14 GB", | 20260508 05:26:05.757 | INFO | "pinned" : " 29 GB" | 20260508 05:26:05.757 | INFO | } | 20260508 05:26:05.757 | INFO | Perf report for the last 5.14072 seconds: | 20260508 05:26:05.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:05.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:26:05.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:26:05.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:05.757 | INFO | Uploaded 36GiB to GPU in 2.46747s (14.5898GiB/s) | 20260508 05:26:05.758 | INFO | Downloaded 34GiB to GPU in 2.43601s (13.9573GiB/s) | 20260508 05:26:05.855 | INFO | Starting a round of Baz IO | 20260508 05:26:09.293 | INFO | Populating RTMetrics report | 20260508 05:26:09.293 | INFO | RTMetrics report populated | 20260508 05:26:09.293 | INFO | Opening tmp RTMetrics json file | 20260508 05:26:09.305 | INFO | Serializing RTMetrics to json | 20260508 05:26:09.306 | INFO | Serialization done, swapping tmp file now... | 20260508 05:26:09.306 | INFO | RTMetrics done | 20260508 05:26:09.373 | INFO | Finished a round of Baz IO: Wrote 1.576 GiB in 3.518 seconds | 20260508 05:26:09.373 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.00% / 59.13% | 20260508 05:26:09.373 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.466347% / 3.97364e-06% / 7.94729e-05% | 20260508 05:26:09.373 | INFO | Baz IO overall: | 20260508 05:26:09.373 | INFO | BytesWritten : 2488146642022 | 20260508 05:26:09.373 | INFO | NumEvents : 1200477366035 | 20260508 05:26:10.894 | INFO | Analyzing chunk frames = [5582848,5583360) | 20260508 05:26:10.894 | INFO | MemInfo: { | 20260508 05:26:10.895 | INFO | "gpu" : " 17 GB", | 20260508 05:26:10.895 | INFO | "huge" : "", | 20260508 05:26:10.895 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:10.895 | INFO | "malloc" : " 14 GB", | 20260508 05:26:10.895 | INFO | "pinned" : " 29 GB" | 20260508 05:26:10.895 | INFO | } | 20260508 05:26:10.895 | INFO | Perf report for the last 5.1376 seconds: | 20260508 05:26:10.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:10.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:10.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:10.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:10.895 | INFO | ANALYSIS : 68.1%, 3.0, 106.4ms, 0 | 20260508 05:26:10.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:10.895 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:26:10.895 | INFO | PRE_HQ : 65.9%, 1.6, 55.3ms, 0 | 20260508 05:26:10.895 | INFO | BAZWRITER : 68.0%, 2.7, 93.2ms, 0 | 20260508 05:26:10.895 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:26:10.895 | INFO | Uploaded 36GiB to GPU in 2.54059s (14.17GiB/s) | 20260508 05:26:10.895 | INFO | Downloaded 38GiB to GPU in 2.70748s (14.0352GiB/s) | 20260508 05:26:16.029 | INFO | Analyzing chunk frames = [5583360,5583872) | 20260508 05:26:16.029 | INFO | MemInfo: { | 20260508 05:26:16.029 | INFO | "gpu" : " 17 GB", | 20260508 05:26:16.029 | INFO | "huge" : "", | 20260508 05:26:16.029 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:16.029 | INFO | "malloc" : " 14 GB", | 20260508 05:26:16.030 | INFO | "pinned" : " 29 GB" | 20260508 05:26:16.030 | INFO | } | 20260508 05:26:16.030 | INFO | Perf report for the last 5.13494 seconds: | 20260508 05:26:16.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:16.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 05:26:16.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:26:16.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:16.030 | INFO | Uploaded 36GiB to GPU in 2.55596s (14.0848GiB/s) | 20260508 05:26:16.030 | INFO | Downloaded 34GiB to GPU in 2.45977s (13.8225GiB/s) | 20260508 05:26:21.166 | INFO | Analyzing chunk frames = [5583872,5584384) | 20260508 05:26:21.166 | INFO | MemInfo: { | 20260508 05:26:21.166 | INFO | "gpu" : " 17 GB", | 20260508 05:26:21.166 | INFO | "huge" : "", | 20260508 05:26:21.166 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:21.166 | INFO | "malloc" : " 14 GB", | 20260508 05:26:21.166 | INFO | "pinned" : " 29 GB" | 20260508 05:26:21.166 | INFO | } | 20260508 05:26:21.166 | INFO | Perf report for the last 5.13657 seconds: | 20260508 05:26:21.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:21.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.166 | INFO | ANALYSIS : 66.8%, 3.0, 103.6ms, 0 | 20260508 05:26:21.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.166 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:26:21.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:21.167 | INFO | Uploaded 36GiB to GPU in 2.48345s (14.4959GiB/s) | 20260508 05:26:21.167 | INFO | Downloaded 34GiB to GPU in 2.42011s (14.0489GiB/s) | 20260508 05:26:26.308 | INFO | Analyzing chunk frames = [5584384,5584896) | 20260508 05:26:26.308 | INFO | MemInfo: { | 20260508 05:26:26.308 | INFO | "gpu" : " 17 GB", | 20260508 05:26:26.308 | INFO | "huge" : "", | 20260508 05:26:26.308 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:26.308 | INFO | "malloc" : " 14 GB", | 20260508 05:26:26.308 | INFO | "pinned" : " 29 GB" | 20260508 05:26:26.308 | INFO | } | 20260508 05:26:26.308 | INFO | Perf report for the last 5.14181 seconds: | 20260508 05:26:26.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:26.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.308 | INFO | ANALYSIS : 67.4%, 3.0, 104.4ms, 0 | 20260508 05:26:26.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.308 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:26:26.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:26.309 | INFO | Uploaded 36GiB to GPU in 2.5875s (13.913GiB/s) | 20260508 05:26:26.309 | INFO | Downloaded 34GiB to GPU in 2.51634s (13.5117GiB/s) | 20260508 05:26:31.456 | INFO | Analyzing chunk frames = [5584896,5585408) | 20260508 05:26:31.456 | INFO | MemInfo: { | 20260508 05:26:31.456 | INFO | "gpu" : " 17 GB", | 20260508 05:26:31.456 | INFO | "huge" : "", | 20260508 05:26:31.456 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:31.456 | INFO | "malloc" : " 14 GB", | 20260508 05:26:31.456 | INFO | "pinned" : " 29 GB" | 20260508 05:26:31.456 | INFO | } | 20260508 05:26:31.456 | INFO | Perf report for the last 5.14806 seconds: | 20260508 05:26:31.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:31.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.456 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 05:26:31.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.456 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:26:31.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:31.457 | INFO | Uploaded 36GiB to GPU in 2.36966s (15.1921GiB/s) | 20260508 05:26:31.457 | INFO | Downloaded 34GiB to GPU in 2.57096s (13.2246GiB/s) | 20260508 05:26:36.593 | INFO | Analyzing chunk frames = [5585408,5585920) | 20260508 05:26:36.594 | INFO | MemInfo: { | 20260508 05:26:36.594 | INFO | "gpu" : " 17 GB", | 20260508 05:26:36.594 | INFO | "huge" : "", | 20260508 05:26:36.594 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:36.594 | INFO | "malloc" : " 14 GB", | 20260508 05:26:36.594 | INFO | "pinned" : " 29 GB" | 20260508 05:26:36.594 | INFO | } | 20260508 05:26:36.594 | INFO | Perf report for the last 5.13761 seconds: | 20260508 05:26:36.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:36.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:26:36.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 05:26:36.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:36.594 | INFO | Uploaded 36GiB to GPU in 2.45395s (14.6702GiB/s) | 20260508 05:26:36.594 | INFO | Downloaded 34GiB to GPU in 2.44457s (13.9084GiB/s) | 20260508 05:26:41.727 | INFO | Analyzing chunk frames = [5585920,5586432) | 20260508 05:26:41.727 | INFO | MemInfo: { | 20260508 05:26:41.727 | INFO | "gpu" : " 17 GB", | 20260508 05:26:41.727 | INFO | "huge" : "", | 20260508 05:26:41.727 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:41.727 | INFO | "malloc" : " 14 GB", | 20260508 05:26:41.727 | INFO | "pinned" : " 29 GB" | 20260508 05:26:41.727 | INFO | } | 20260508 05:26:41.727 | INFO | Perf report for the last 5.13367 seconds: | 20260508 05:26:41.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:41.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.728 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:26:41.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.728 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 05:26:41.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:41.728 | INFO | Uploaded 36GiB to GPU in 2.43238s (14.8003GiB/s) | 20260508 05:26:41.728 | INFO | Downloaded 34GiB to GPU in 2.43658s (13.954GiB/s) | 20260508 05:26:46.865 | INFO | Analyzing chunk frames = [5586432,5586944) | 20260508 05:26:46.866 | INFO | MemInfo: { | 20260508 05:26:46.866 | INFO | "gpu" : " 17 GB", | 20260508 05:26:46.866 | INFO | "huge" : "", | 20260508 05:26:46.866 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:46.866 | INFO | "malloc" : " 14 GB", | 20260508 05:26:46.866 | INFO | "pinned" : " 29 GB" | 20260508 05:26:46.866 | INFO | } | 20260508 05:26:46.866 | INFO | Perf report for the last 5.13823 seconds: | 20260508 05:26:46.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:46.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:26:46.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 05:26:46.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:46.866 | INFO | Uploaded 36GiB to GPU in 2.43229s (14.8009GiB/s) | 20260508 05:26:46.866 | INFO | Downloaded 34GiB to GPU in 2.43183s (13.9812GiB/s) | 20260508 05:26:46.866 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:26:46.866 | INFO | Total mem usage: 16.7 GiB | 20260508 05:26:46.866 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:26:46.866 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17.1%) | 20260508 05:26:46.866 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:26:46.866 | INFO | High Water Breakdown: | 20260508 05:26:46.866 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:26:46.866 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:26:46.866 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:26:46.866 | INFO | ------------------------------------------ | 20260508 05:26:46.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:26:46.866 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:26:46.866 | INFO | Total mem usage: 26.5 GiB | 20260508 05:26:46.866 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:26:46.866 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 05:26:46.867 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 05:26:46.867 | INFO | High Water Breakdown: | 20260508 05:26:46.867 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:26:46.867 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:26:46.867 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:26:46.867 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:26:46.867 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:26:46.867 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:26:46.867 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:26:46.867 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:26:46.867 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:26:46.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:26:46.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:26:46.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:26:46.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:26:46.867 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:26:46.867 | INFO | ------------------------------------------ | 20260508 05:26:46.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:26:46.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:26:46.867 | INFO | Total mem usage: 27.8 GiB | 20260508 05:26:46.867 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:26:46.867 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:26:46.867 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:26:46.867 | INFO | High Water Breakdown: | 20260508 05:26:46.867 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:26:46.867 | INFO | ------------------------------------------ | 20260508 05:26:46.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:26:46.867 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:26:46.867 | INFO | Total mem usage: 15.7 GiB | 20260508 05:26:46.867 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:26:46.867 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:26:46.867 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:26:46.867 | INFO | High Water Breakdown: | 20260508 05:26:46.867 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:26:46.867 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:26:46.867 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:26:46.867 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:26:46.867 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:26:46.867 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:26:46.867 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:26:46.867 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:26:46.867 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:26:46.867 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:26:46.867 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:26:46.867 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:26:46.867 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:26:46.867 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:26:46.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:26:46.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:26:46.867 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:26:46.867 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:26:46.867 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:26:46.867 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:26:46.867 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:26:46.867 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:26:46.867 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:26:46.867 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:26:46.867 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:26:46.867 | INFO | ------------------------------------------ | 20260508 05:26:46.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:26:46.867 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:26:46.867 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:26:46.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:26:46.867 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.60e+01, 1.40e+01 / 1.75e+00 | 20260508 05:26:46.867 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.19e-02 / 5.31e-03 | 20260508 05:26:46.867 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.46e+00, 5.85e-01 / 1.96e-01 | 20260508 05:26:46.867 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 1.25e-01 | 20260508 05:26:46.867 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.59e+01, 4.01e+00 / 2.03e+00 | 20260508 05:26:46.867 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.85e+01 / 1.99e+01, 1.75e+00 / 1.52e+00 | 20260508 05:26:46.867 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.61e+00, 4.20e-01 / 2.79e-01 | 20260508 05:26:46.867 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.26e+00, 2.59e-01 / 1.87e-01 | 20260508 05:26:46.867 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.61e-01, 5.04e-02 / 2.91e-02 | 20260508 05:26:46.867 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:26:46.867 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:26:46.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:26:46.867 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 24.31%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 24.36%, Download -- 2.59e+01 / 0.00e+00, 8.34e+00 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 30.73%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 2.14%, LockUpload -- 2.28e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 2.83%, LockDownload -- 3.00e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:26:46.868 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260508 05:26:46.965 | INFO | Starting a round of Baz IO | 20260508 05:26:50.389 | INFO | Populating RTMetrics report | 20260508 05:26:50.390 | INFO | RTMetrics report populated | 20260508 05:26:50.390 | INFO | Opening tmp RTMetrics json file | 20260508 05:26:50.402 | INFO | Serializing RTMetrics to json | 20260508 05:26:50.403 | INFO | Serialization done, swapping tmp file now... | 20260508 05:26:50.404 | INFO | RTMetrics done | 20260508 05:26:50.471 | INFO | Finished a round of Baz IO: Wrote 1.574 GiB in 3.505 seconds | 20260508 05:26:50.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.00% / 59.13% | 20260508 05:26:50.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.464749% / 0% / 7.54992e-05% | 20260508 05:26:50.471 | INFO | Baz IO overall: | 20260508 05:26:50.471 | INFO | BytesWritten : 2489836965575 | 20260508 05:26:50.471 | INFO | NumEvents : 1201283299024 | 20260508 05:26:52.010 | INFO | Analyzing chunk frames = [5586944,5587456) | 20260508 05:26:52.010 | INFO | MemInfo: { | 20260508 05:26:52.010 | INFO | "gpu" : " 17 GB", | 20260508 05:26:52.010 | INFO | "huge" : "", | 20260508 05:26:52.010 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:52.010 | INFO | "malloc" : " 14 GB", | 20260508 05:26:52.010 | INFO | "pinned" : " 29 GB" | 20260508 05:26:52.010 | INFO | } | 20260508 05:26:52.010 | INFO | Perf report for the last 5.14426 seconds: | 20260508 05:26:52.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:52.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:52.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:52.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:52.010 | INFO | ANALYSIS : 67.8%, 3.0, 106.1ms, 0 | 20260508 05:26:52.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:52.010 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:26:52.010 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 05:26:52.010 | INFO | BAZWRITER : 67.7%, 2.7, 93.8ms, 0 | 20260508 05:26:52.010 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:26:52.011 | INFO | Uploaded 36GiB to GPU in 2.59561s (13.8696GiB/s) | 20260508 05:26:52.011 | INFO | Downloaded 38GiB to GPU in 2.70258s (14.0606GiB/s) | 20260508 05:26:57.149 | INFO | Analyzing chunk frames = [5587456,5587968) | 20260508 05:26:57.149 | INFO | MemInfo: { | 20260508 05:26:57.149 | INFO | "gpu" : " 17 GB", | 20260508 05:26:57.149 | INFO | "huge" : "", | 20260508 05:26:57.149 | INFO | "huge_shared" : " 29 GB", | 20260508 05:26:57.149 | INFO | "malloc" : " 14 GB", | 20260508 05:26:57.149 | INFO | "pinned" : " 29 GB" | 20260508 05:26:57.149 | INFO | } | 20260508 05:26:57.149 | INFO | Perf report for the last 5.13868 seconds: | 20260508 05:26:57.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:26:57.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:26:57.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:26:57.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:26:57.149 | INFO | Uploaded 36GiB to GPU in 2.48357s (14.4953GiB/s) | 20260508 05:26:57.149 | INFO | Downloaded 34GiB to GPU in 2.4348s (13.9642GiB/s) | 20260508 05:27:02.293 | INFO | Analyzing chunk frames = [5587968,5588480) | 20260508 05:27:02.293 | INFO | MemInfo: { | 20260508 05:27:02.293 | INFO | "gpu" : " 17 GB", | 20260508 05:27:02.293 | INFO | "huge" : "", | 20260508 05:27:02.293 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:02.293 | INFO | "malloc" : " 14 GB", | 20260508 05:27:02.293 | INFO | "pinned" : " 29 GB" | 20260508 05:27:02.294 | INFO | } | 20260508 05:27:02.294 | INFO | Perf report for the last 5.14493 seconds: | 20260508 05:27:02.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:02.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:27:02.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:27:02.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:02.294 | INFO | Uploaded 36GiB to GPU in 2.53252s (14.2151GiB/s) | 20260508 05:27:02.294 | INFO | Downloaded 34GiB to GPU in 2.40923s (14.1124GiB/s) | 20260508 05:27:07.424 | INFO | Analyzing chunk frames = [5588480,5588992) | 20260508 05:27:07.425 | INFO | MemInfo: { | 20260508 05:27:07.425 | INFO | "gpu" : " 17 GB", | 20260508 05:27:07.425 | INFO | "huge" : "", | 20260508 05:27:07.425 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:07.425 | INFO | "malloc" : " 14 GB", | 20260508 05:27:07.425 | INFO | "pinned" : " 29 GB" | 20260508 05:27:07.425 | INFO | } | 20260508 05:27:07.425 | INFO | Perf report for the last 5.1311 seconds: | 20260508 05:27:07.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:07.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | ANALYSIS : 66.8%, 3.0, 103.2ms, 0 | 20260508 05:27:07.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:27:07.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:07.425 | INFO | Uploaded 36GiB to GPU in 2.55259s (14.1033GiB/s) | 20260508 05:27:07.425 | INFO | Downloaded 34GiB to GPU in 2.4086s (14.1161GiB/s) | 20260508 05:27:12.568 | INFO | Analyzing chunk frames = [5588992,5589504) | 20260508 05:27:12.568 | INFO | MemInfo: { | 20260508 05:27:12.568 | INFO | "gpu" : " 17 GB", | 20260508 05:27:12.568 | INFO | "huge" : "", | 20260508 05:27:12.568 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:12.568 | INFO | "malloc" : " 14 GB", | 20260508 05:27:12.568 | INFO | "pinned" : " 29 GB" | 20260508 05:27:12.568 | INFO | } | 20260508 05:27:12.568 | INFO | Perf report for the last 5.14326 seconds: | 20260508 05:27:12.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:12.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.568 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 05:27:12.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.568 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:27:12.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:12.569 | INFO | Uploaded 36GiB to GPU in 2.42196s (14.864GiB/s) | 20260508 05:27:12.569 | INFO | Downloaded 34GiB to GPU in 2.52412s (13.47GiB/s) | 20260508 05:27:17.705 | INFO | Analyzing chunk frames = [5589504,5590016) | 20260508 05:27:17.705 | INFO | MemInfo: { | 20260508 05:27:17.705 | INFO | "gpu" : " 17 GB", | 20260508 05:27:17.705 | INFO | "huge" : "", | 20260508 05:27:17.705 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:17.705 | INFO | "malloc" : " 14 GB", | 20260508 05:27:17.705 | INFO | "pinned" : " 29 GB" | 20260508 05:27:17.705 | INFO | } | 20260508 05:27:17.705 | INFO | Perf report for the last 5.13735 seconds: | 20260508 05:27:17.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:17.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:27:17.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:27:17.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:17.706 | INFO | Uploaded 36GiB to GPU in 2.42722s (14.8318GiB/s) | 20260508 05:27:17.706 | INFO | Downloaded 34GiB to GPU in 2.43479s (13.9643GiB/s) | 20260508 05:27:22.847 | INFO | Analyzing chunk frames = [5590016,5590528) | 20260508 05:27:22.847 | INFO | MemInfo: { | 20260508 05:27:22.847 | INFO | "gpu" : " 17 GB", | 20260508 05:27:22.847 | INFO | "huge" : "", | 20260508 05:27:22.847 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:22.847 | INFO | "malloc" : " 14 GB", | 20260508 05:27:22.847 | INFO | "pinned" : " 29 GB" | 20260508 05:27:22.847 | INFO | } | 20260508 05:27:22.847 | INFO | Perf report for the last 5.14185 seconds: | 20260508 05:27:22.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:22.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.847 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:27:22.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.848 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:27:22.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:22.848 | INFO | Uploaded 36GiB to GPU in 2.45008s (14.6934GiB/s) | 20260508 05:27:22.848 | INFO | Downloaded 34GiB to GPU in 2.42842s (14.0009GiB/s) | 20260508 05:27:27.980 | INFO | Analyzing chunk frames = [5590528,5591040) | 20260508 05:27:27.980 | INFO | MemInfo: { | 20260508 05:27:27.980 | INFO | "gpu" : " 17 GB", | 20260508 05:27:27.980 | INFO | "huge" : "", | 20260508 05:27:27.980 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:27.980 | INFO | "malloc" : " 14 GB", | 20260508 05:27:27.980 | INFO | "pinned" : " 29 GB" | 20260508 05:27:27.980 | INFO | } | 20260508 05:27:27.980 | INFO | Perf report for the last 5.13258 seconds: | 20260508 05:27:27.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:27.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 05:27:27.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:27:27.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:27.980 | INFO | Uploaded 36GiB to GPU in 2.45576s (14.6594GiB/s) | 20260508 05:27:27.980 | INFO | Downloaded 34GiB to GPU in 2.44654s (13.8972GiB/s) | 20260508 05:27:28.077 | INFO | Starting a round of Baz IO | 20260508 05:27:31.511 | INFO | Populating RTMetrics report | 20260508 05:27:31.511 | INFO | RTMetrics report populated | 20260508 05:27:31.511 | INFO | Opening tmp RTMetrics json file | 20260508 05:27:31.520 | INFO | Serializing RTMetrics to json | 20260508 05:27:31.525 | INFO | Serialization done, swapping tmp file now... | 20260508 05:27:31.525 | INFO | RTMetrics done | 20260508 05:27:31.586 | INFO | Finished a round of Baz IO: Wrote 1.575 GiB in 3.509 seconds | 20260508 05:27:31.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.00% / 59.13% | 20260508 05:27:31.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.464416% / 7.94729e-06% / 8.34465e-05% | 20260508 05:27:31.586 | INFO | Baz IO overall: | 20260508 05:27:31.587 | INFO | BytesWritten : 2491527710515 | 20260508 05:27:31.587 | INFO | NumEvents : 1202089737359 | 20260508 05:27:33.116 | INFO | Analyzing chunk frames = [5591040,5591552) | 20260508 05:27:33.116 | INFO | MemInfo: { | 20260508 05:27:33.116 | INFO | "gpu" : " 17 GB", | 20260508 05:27:33.116 | INFO | "huge" : "", | 20260508 05:27:33.116 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:33.116 | INFO | "malloc" : " 14 GB", | 20260508 05:27:33.116 | INFO | "pinned" : " 29 GB" | 20260508 05:27:33.117 | INFO | } | 20260508 05:27:33.117 | INFO | Perf report for the last 5.13669 seconds: | 20260508 05:27:33.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:33.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:33.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:33.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:33.117 | INFO | ANALYSIS : 68.1%, 3.0, 106.3ms, 0 | 20260508 05:27:33.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:33.117 | INFO | RT_METRICS : 39.6%, 1.0, 20.7ms, 0 | 20260508 05:27:33.117 | INFO | PRE_HQ : 65.8%, 1.6, 55.3ms, 0 | 20260508 05:27:33.117 | INFO | BAZWRITER : 67.8%, 2.7, 93.5ms, 0 | 20260508 05:27:33.117 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:27:33.117 | INFO | Uploaded 36GiB to GPU in 2.5887s (13.9066GiB/s) | 20260508 05:27:33.117 | INFO | Downloaded 38GiB to GPU in 2.7075s (14.0351GiB/s) | 20260508 05:27:38.261 | INFO | Analyzing chunk frames = [5591552,5592064) | 20260508 05:27:38.261 | INFO | MemInfo: { | 20260508 05:27:38.261 | INFO | "gpu" : " 17 GB", | 20260508 05:27:38.261 | INFO | "huge" : "", | 20260508 05:27:38.261 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:38.261 | INFO | "malloc" : " 14 GB", | 20260508 05:27:38.261 | INFO | "pinned" : " 29 GB" | 20260508 05:27:38.261 | INFO | } | 20260508 05:27:38.261 | INFO | Perf report for the last 5.14488 seconds: | 20260508 05:27:38.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:38.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.262 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 05:27:38.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.262 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:27:38.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:38.262 | INFO | Uploaded 36GiB to GPU in 2.59631s (13.8658GiB/s) | 20260508 05:27:38.262 | INFO | Downloaded 34GiB to GPU in 2.43916s (13.9392GiB/s) | 20260508 05:27:43.404 | INFO | Analyzing chunk frames = [5592064,5592576) | 20260508 05:27:43.404 | INFO | MemInfo: { | 20260508 05:27:43.404 | INFO | "gpu" : " 17 GB", | 20260508 05:27:43.404 | INFO | "huge" : "", | 20260508 05:27:43.404 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:43.404 | INFO | "malloc" : " 14 GB", | 20260508 05:27:43.404 | INFO | "pinned" : " 29 GB" | 20260508 05:27:43.404 | INFO | } | 20260508 05:27:43.404 | INFO | Perf report for the last 5.14283 seconds: | 20260508 05:27:43.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:43.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.404 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 05:27:43.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.404 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 05:27:43.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:43.405 | INFO | Uploaded 36GiB to GPU in 2.564s (14.0405GiB/s) | 20260508 05:27:43.405 | INFO | Downloaded 34GiB to GPU in 2.40925s (14.1123GiB/s) | 20260508 05:27:48.541 | INFO | Analyzing chunk frames = [5592576,5593088) | 20260508 05:27:48.541 | INFO | MemInfo: { | 20260508 05:27:48.541 | INFO | "gpu" : " 17 GB", | 20260508 05:27:48.541 | INFO | "huge" : "", | 20260508 05:27:48.541 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:48.541 | INFO | "malloc" : " 14 GB", | 20260508 05:27:48.541 | INFO | "pinned" : " 29 GB" | 20260508 05:27:48.541 | INFO | } | 20260508 05:27:48.541 | INFO | Perf report for the last 5.13708 seconds: | 20260508 05:27:48.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:48.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.541 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 05:27:48.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.542 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:27:48.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:48.542 | INFO | Uploaded 36GiB to GPU in 2.55711s (14.0784GiB/s) | 20260508 05:27:48.542 | INFO | Downloaded 34GiB to GPU in 2.37981s (14.2868GiB/s) | 20260508 05:27:53.678 | INFO | Analyzing chunk frames = [5593088,5593600) | 20260508 05:27:53.678 | INFO | MemInfo: { | 20260508 05:27:53.678 | INFO | "gpu" : " 17 GB", | 20260508 05:27:53.678 | INFO | "huge" : "", | 20260508 05:27:53.678 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:53.678 | INFO | "malloc" : " 14 GB", | 20260508 05:27:53.678 | INFO | "pinned" : " 29 GB" | 20260508 05:27:53.678 | INFO | } | 20260508 05:27:53.678 | INFO | Perf report for the last 5.13642 seconds: | 20260508 05:27:53.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:53.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 05:27:53.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:27:53.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:53.678 | INFO | Uploaded 36GiB to GPU in 2.4302s (14.8136GiB/s) | 20260508 05:27:53.678 | INFO | Downloaded 34GiB to GPU in 2.55766s (13.2934GiB/s) | 20260508 05:27:58.821 | INFO | Analyzing chunk frames = [5593600,5594112) | 20260508 05:27:58.821 | INFO | MemInfo: { | 20260508 05:27:58.821 | INFO | "gpu" : " 17 GB", | 20260508 05:27:58.821 | INFO | "huge" : "", | 20260508 05:27:58.821 | INFO | "huge_shared" : " 29 GB", | 20260508 05:27:58.821 | INFO | "malloc" : " 14 GB", | 20260508 05:27:58.821 | INFO | "pinned" : " 29 GB" | 20260508 05:27:58.821 | INFO | } | 20260508 05:27:58.822 | INFO | Perf report for the last 5.1437 seconds: | 20260508 05:27:58.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:27:58.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:27:58.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:27:58.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:27:58.822 | INFO | Uploaded 36GiB to GPU in 2.47951s (14.519GiB/s) | 20260508 05:27:58.822 | INFO | Downloaded 34GiB to GPU in 2.43517s (13.9621GiB/s) | 20260508 05:28:03.957 | INFO | Analyzing chunk frames = [5594112,5594624) | 20260508 05:28:03.957 | INFO | MemInfo: { | 20260508 05:28:03.957 | INFO | "gpu" : " 17 GB", | 20260508 05:28:03.957 | INFO | "huge" : "", | 20260508 05:28:03.957 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:03.957 | INFO | "malloc" : " 14 GB", | 20260508 05:28:03.957 | INFO | "pinned" : " 29 GB" | 20260508 05:28:03.957 | INFO | } | 20260508 05:28:03.957 | INFO | Perf report for the last 5.13578 seconds: | 20260508 05:28:03.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:03.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.957 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:28:03.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.957 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:28:03.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:03.958 | INFO | Uploaded 36GiB to GPU in 2.48072s (14.5119GiB/s) | 20260508 05:28:03.958 | INFO | Downloaded 34GiB to GPU in 2.42934s (13.9956GiB/s) | 20260508 05:28:09.097 | INFO | Analyzing chunk frames = [5594624,5595136) | 20260508 05:28:09.097 | INFO | MemInfo: { | 20260508 05:28:09.097 | INFO | "gpu" : " 17 GB", | 20260508 05:28:09.097 | INFO | "huge" : "", | 20260508 05:28:09.098 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:09.098 | INFO | "malloc" : " 14 GB", | 20260508 05:28:09.098 | INFO | "pinned" : " 29 GB" | 20260508 05:28:09.098 | INFO | } | 20260508 05:28:09.098 | INFO | Perf report for the last 5.14036 seconds: | 20260508 05:28:09.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:09.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:28:09.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:28:09.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:09.098 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:28:09.098 | INFO | Total mem usage: 16.7 GiB | 20260508 05:28:09.098 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:28:09.098 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17.1%) | 20260508 05:28:09.098 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:28:09.098 | INFO | High Water Breakdown: | 20260508 05:28:09.098 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:28:09.098 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:28:09.098 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:28:09.098 | INFO | ------------------------------------------ | 20260508 05:28:09.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:28:09.099 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:28:09.099 | INFO | Total mem usage: 26.5 GiB | 20260508 05:28:09.099 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:28:09.099 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:28:09.099 | INFO | Uploaded 36GiB to GPU in 2.47848s (14.525GiB/s) | 20260508 05:28:09.099 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:28:09.099 | INFO | High Water Breakdown: | 20260508 05:28:09.099 | INFO | Downloaded 34GiB to GPU in 2.41621s (14.0716GiB/s) | 20260508 05:28:09.099 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:28:09.099 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:28:09.099 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:28:09.099 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:28:09.099 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:28:09.099 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:28:09.099 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:28:09.099 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:28:09.099 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:28:09.099 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:28:09.099 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:28:09.099 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:28:09.099 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:28:09.099 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:28:09.099 | INFO | ------------------------------------------ | 20260508 05:28:09.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:28:09.099 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:28:09.099 | INFO | Total mem usage: 27.8 GiB | 20260508 05:28:09.099 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:28:09.099 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:28:09.099 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:28:09.099 | INFO | High Water Breakdown: | 20260508 05:28:09.099 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:28:09.099 | INFO | ------------------------------------------ | 20260508 05:28:09.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:28:09.099 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:28:09.099 | INFO | Total mem usage: 15.7 GiB | 20260508 05:28:09.099 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:28:09.099 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:28:09.099 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:28:09.099 | INFO | High Water Breakdown: | 20260508 05:28:09.099 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:28:09.099 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:28:09.099 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:28:09.099 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:28:09.099 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:28:09.099 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:28:09.099 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:28:09.099 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:28:09.099 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:28:09.099 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:28:09.099 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:28:09.099 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:28:09.099 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:28:09.099 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:28:09.099 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:28:09.099 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:28:09.099 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:28:09.099 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:28:09.099 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:28:09.099 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:28:09.099 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:28:09.099 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:28:09.099 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:28:09.099 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:28:09.099 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:28:09.099 | INFO | ------------------------------------------ | 20260508 05:28:09.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:28:09.099 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:28:09.099 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:28:09.099 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:28:09.100 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.40e+01 / 1.66e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.18e-02 / 5.91e-03 | 20260508 05:28:09.100 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.39e+00, 5.82e-01 / 2.18e-01 | 20260508 05:28:09.100 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.44e+00, 2.06e-01 / 1.27e-01 | 20260508 05:28:09.100 | INFO | PerfReport: 15.94%, DME -- 4.16e+01 / 4.50e+01, 3.75e+00 / 1.90e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.85e+01 / 2.03e+01, 1.75e+00 / 1.42e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.39e+00 / 2.59e+00, 4.19e-01 / 2.36e-01 | 20260508 05:28:09.100 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.22e+00, 2.55e-01 / 1.32e-01 | 20260508 05:28:09.100 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.60e-01, 4.96e-02 / 2.70e-02 | 20260508 05:28:09.100 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:28:09.100 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:28:09.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:28:09.100 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 24.56%, Upload -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 24.23%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 30.71%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 2.35%, LockUpload -- 2.49e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 2.56%, LockDownload -- 2.72e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 15.57%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 05:28:09.100 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260508 05:28:09.195 | INFO | Starting a round of Baz IO | 20260508 05:28:12.629 | INFO | Populating RTMetrics report | 20260508 05:28:12.630 | INFO | RTMetrics report populated | 20260508 05:28:12.630 | INFO | Opening tmp RTMetrics json file | 20260508 05:28:12.643 | INFO | Serializing RTMetrics to json | 20260508 05:28:12.644 | INFO | Serialization done, swapping tmp file now... | 20260508 05:28:12.644 | INFO | RTMetrics done | 20260508 05:28:12.701 | INFO | Finished a round of Baz IO: Wrote 1.574 GiB in 3.505 seconds | 20260508 05:28:12.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.00% / 59.13% | 20260508 05:28:12.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.463617% / 3.97364e-06% / 8.74201e-05% | 20260508 05:28:12.701 | INFO | Baz IO overall: | 20260508 05:28:12.701 | INFO | BytesWritten : 2493217401898 | 20260508 05:28:12.701 | INFO | NumEvents : 1202895388038 | 20260508 05:28:14.236 | INFO | Analyzing chunk frames = [5595136,5595648) | 20260508 05:28:14.236 | INFO | MemInfo: { | 20260508 05:28:14.236 | INFO | "gpu" : " 17 GB", | 20260508 05:28:14.236 | INFO | "huge" : "", | 20260508 05:28:14.236 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:14.236 | INFO | "malloc" : " 14 GB", | 20260508 05:28:14.236 | INFO | "pinned" : " 29 GB" | 20260508 05:28:14.236 | INFO | } | 20260508 05:28:14.237 | INFO | Perf report for the last 5.13889 seconds: | 20260508 05:28:14.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:14.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:14.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:14.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:14.237 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 05:28:14.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:14.237 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:28:14.237 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 05:28:14.237 | INFO | BAZWRITER : 67.7%, 2.7, 94.3ms, 0 | 20260508 05:28:14.237 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:28:14.237 | INFO | Uploaded 36GiB to GPU in 2.51467s (14.316GiB/s) | 20260508 05:28:14.237 | INFO | Downloaded 38GiB to GPU in 2.69867s (14.081GiB/s) | 20260508 05:28:19.375 | INFO | Analyzing chunk frames = [5595648,5596160) | 20260508 05:28:19.375 | INFO | MemInfo: { | 20260508 05:28:19.375 | INFO | "gpu" : " 17 GB", | 20260508 05:28:19.375 | INFO | "huge" : "", | 20260508 05:28:19.375 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:19.375 | INFO | "malloc" : " 14 GB", | 20260508 05:28:19.375 | INFO | "pinned" : " 29 GB" | 20260508 05:28:19.375 | INFO | } | 20260508 05:28:19.375 | INFO | Perf report for the last 5.13853 seconds: | 20260508 05:28:19.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:19.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.375 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:28:19.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.375 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:28:19.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:19.376 | INFO | Uploaded 36GiB to GPU in 2.49057s (14.4545GiB/s) | 20260508 05:28:19.376 | INFO | Downloaded 34GiB to GPU in 2.44621s (13.8991GiB/s) | 20260508 05:28:24.523 | INFO | Analyzing chunk frames = [5596160,5596672) | 20260508 05:28:24.523 | INFO | MemInfo: { | 20260508 05:28:24.523 | INFO | "gpu" : " 17 GB", | 20260508 05:28:24.523 | INFO | "huge" : "", | 20260508 05:28:24.523 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:24.523 | INFO | "malloc" : " 14 GB", | 20260508 05:28:24.523 | INFO | "pinned" : " 29 GB" | 20260508 05:28:24.523 | INFO | } | 20260508 05:28:24.523 | INFO | Perf report for the last 5.14801 seconds: | 20260508 05:28:24.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:24.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:28:24.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:28:24.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:24.523 | INFO | Uploaded 36GiB to GPU in 2.52196s (14.2746GiB/s) | 20260508 05:28:24.524 | INFO | Downloaded 34GiB to GPU in 2.41047s (14.1051GiB/s) | 20260508 05:28:29.661 | INFO | Analyzing chunk frames = [5596672,5597184) | 20260508 05:28:29.661 | INFO | MemInfo: { | 20260508 05:28:29.661 | INFO | "gpu" : " 17 GB", | 20260508 05:28:29.661 | INFO | "huge" : "", | 20260508 05:28:29.661 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:29.661 | INFO | "malloc" : " 14 GB", | 20260508 05:28:29.661 | INFO | "pinned" : " 29 GB" | 20260508 05:28:29.661 | INFO | } | 20260508 05:28:29.661 | INFO | Perf report for the last 5.13801 seconds: | 20260508 05:28:29.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:29.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.661 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 05:28:29.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.661 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:28:29.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:29.662 | INFO | Uploaded 36GiB to GPU in 2.56425s (14.0392GiB/s) | 20260508 05:28:29.662 | INFO | Downloaded 34GiB to GPU in 2.42615s (14.014GiB/s) | 20260508 05:28:34.792 | INFO | Analyzing chunk frames = [5597184,5597696) | 20260508 05:28:34.792 | INFO | MemInfo: { | 20260508 05:28:34.792 | INFO | "gpu" : " 17 GB", | 20260508 05:28:34.792 | INFO | "huge" : "", | 20260508 05:28:34.792 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:34.792 | INFO | "malloc" : " 14 GB", | 20260508 05:28:34.792 | INFO | "pinned" : " 29 GB" | 20260508 05:28:34.792 | INFO | } | 20260508 05:28:34.792 | INFO | Perf report for the last 5.13128 seconds: | 20260508 05:28:34.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:34.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.793 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 05:28:34.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.793 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:28:34.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:34.793 | INFO | Uploaded 36GiB to GPU in 2.38869s (15.071GiB/s) | 20260508 05:28:34.793 | INFO | Downloaded 34GiB to GPU in 2.51075s (13.5418GiB/s) | 20260508 05:28:39.929 | INFO | Analyzing chunk frames = [5597696,5598208) | 20260508 05:28:39.929 | INFO | MemInfo: { | 20260508 05:28:39.929 | INFO | "gpu" : " 17 GB", | 20260508 05:28:39.929 | INFO | "huge" : "", | 20260508 05:28:39.929 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:39.930 | INFO | "malloc" : " 14 GB", | 20260508 05:28:39.930 | INFO | "pinned" : " 29 GB" | 20260508 05:28:39.930 | INFO | } | 20260508 05:28:39.930 | INFO | Perf report for the last 5.13716 seconds: | 20260508 05:28:39.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:39.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:28:39.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:28:39.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:39.930 | INFO | Uploaded 36GiB to GPU in 2.41306s (14.9188GiB/s) | 20260508 05:28:39.930 | INFO | Downloaded 34GiB to GPU in 2.43906s (13.9398GiB/s) | 20260508 05:28:45.070 | INFO | Analyzing chunk frames = [5598208,5598720) | 20260508 05:28:45.070 | INFO | MemInfo: { | 20260508 05:28:45.070 | INFO | "gpu" : " 17 GB", | 20260508 05:28:45.070 | INFO | "huge" : "", | 20260508 05:28:45.070 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:45.070 | INFO | "malloc" : " 14 GB", | 20260508 05:28:45.070 | INFO | "pinned" : " 29 GB" | 20260508 05:28:45.070 | INFO | } | 20260508 05:28:45.070 | INFO | Perf report for the last 5.14019 seconds: | 20260508 05:28:45.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:45.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 05:28:45.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:28:45.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:45.070 | INFO | Uploaded 36GiB to GPU in 2.41092s (14.9321GiB/s) | 20260508 05:28:45.070 | INFO | Downloaded 34GiB to GPU in 2.43903s (13.94GiB/s) | 20260508 05:28:50.213 | INFO | Analyzing chunk frames = [5598720,5599232) | 20260508 05:28:50.213 | INFO | MemInfo: { | 20260508 05:28:50.213 | INFO | "gpu" : " 17 GB", | 20260508 05:28:50.213 | INFO | "huge" : "", | 20260508 05:28:50.213 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:50.213 | INFO | "malloc" : " 14 GB", | 20260508 05:28:50.213 | INFO | "pinned" : " 29 GB" | 20260508 05:28:50.213 | INFO | } | 20260508 05:28:50.213 | INFO | Perf report for the last 5.14294 seconds: | 20260508 05:28:50.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:50.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:28:50.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 05:28:50.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:50.213 | INFO | Uploaded 36GiB to GPU in 2.4312s (14.8075GiB/s) | 20260508 05:28:50.213 | INFO | Downloaded 34GiB to GPU in 2.44466s (13.9079GiB/s) | 20260508 05:28:50.328 | INFO | Starting a round of Baz IO | 20260508 05:28:53.743 | INFO | Populating RTMetrics report | 20260508 05:28:53.743 | INFO | RTMetrics report populated | 20260508 05:28:53.744 | INFO | Opening tmp RTMetrics json file | 20260508 05:28:53.754 | INFO | Serializing RTMetrics to json | 20260508 05:28:53.755 | INFO | Serialization done, swapping tmp file now... | 20260508 05:28:53.756 | INFO | RTMetrics done | 20260508 05:28:53.813 | INFO | Finished a round of Baz IO: Wrote 1.574 GiB in 3.485 seconds | 20260508 05:28:53.813 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.86% / 0.00% / 59.13% | 20260508 05:28:53.813 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.462981% / 0% / 5.96046e-05% | 20260508 05:28:53.813 | INFO | Baz IO overall: | 20260508 05:28:53.813 | INFO | BytesWritten : 2494906991812 | 20260508 05:28:53.813 | INFO | NumEvents : 1203701031809 | 20260508 05:28:55.355 | INFO | Analyzing chunk frames = [5599232,5599744) | 20260508 05:28:55.355 | INFO | MemInfo: { | 20260508 05:28:55.355 | INFO | "gpu" : " 17 GB", | 20260508 05:28:55.355 | INFO | "huge" : "", | 20260508 05:28:55.355 | INFO | "huge_shared" : " 29 GB", | 20260508 05:28:55.355 | INFO | "malloc" : " 14 GB", | 20260508 05:28:55.355 | INFO | "pinned" : " 29 GB" | 20260508 05:28:55.355 | INFO | } | 20260508 05:28:55.355 | INFO | Perf report for the last 5.14239 seconds: | 20260508 05:28:55.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:28:55.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:55.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:55.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:55.355 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 05:28:55.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:28:55.355 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260508 05:28:55.355 | INFO | PRE_HQ : 66.2%, 1.8, 62.4ms, 0 | 20260508 05:28:55.355 | INFO | BAZWRITER : 67.3%, 2.8, 98.6ms, 0 | 20260508 05:28:55.355 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 05:28:55.356 | INFO | Uploaded 36GiB to GPU in 2.55668s (14.0808GiB/s) | 20260508 05:28:55.356 | INFO | Downloaded 38GiB to GPU in 2.68604s (14.1472GiB/s) | 20260508 05:29:00.490 | INFO | Analyzing chunk frames = [5599744,5600256) | 20260508 05:29:00.490 | INFO | MemInfo: { | 20260508 05:29:00.490 | INFO | "gpu" : " 17 GB", | 20260508 05:29:00.490 | INFO | "huge" : "", | 20260508 05:29:00.490 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:00.490 | INFO | "malloc" : " 14 GB", | 20260508 05:29:00.490 | INFO | "pinned" : " 29 GB" | 20260508 05:29:00.490 | INFO | } | 20260508 05:29:00.490 | INFO | Perf report for the last 5.13526 seconds: | 20260508 05:29:00.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:00.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 05:29:00.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:29:00.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:00.491 | INFO | Uploaded 36GiB to GPU in 2.56318s (14.045GiB/s) | 20260508 05:29:00.491 | INFO | Downloaded 34GiB to GPU in 2.43829s (13.9442GiB/s) | 20260508 05:29:05.628 | INFO | Analyzing chunk frames = [5600256,5600768) | 20260508 05:29:05.628 | INFO | MemInfo: { | 20260508 05:29:05.628 | INFO | "gpu" : " 17 GB", | 20260508 05:29:05.628 | INFO | "huge" : "", | 20260508 05:29:05.628 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:05.628 | INFO | "malloc" : " 14 GB", | 20260508 05:29:05.629 | INFO | "pinned" : " 29 GB" | 20260508 05:29:05.629 | INFO | } | 20260508 05:29:05.629 | INFO | Perf report for the last 5.13809 seconds: | 20260508 05:29:05.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:05.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 05:29:05.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:29:05.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:05.629 | INFO | Uploaded 36GiB to GPU in 2.54101s (14.1676GiB/s) | 20260508 05:29:05.629 | INFO | Downloaded 34GiB to GPU in 2.41674s (14.0686GiB/s) | 20260508 05:29:10.769 | INFO | Analyzing chunk frames = [5600768,5601280) | 20260508 05:29:10.769 | INFO | MemInfo: { | 20260508 05:29:10.769 | INFO | "gpu" : " 17 GB", | 20260508 05:29:10.769 | INFO | "huge" : "", | 20260508 05:29:10.769 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:10.769 | INFO | "malloc" : " 14 GB", | 20260508 05:29:10.769 | INFO | "pinned" : " 29 GB" | 20260508 05:29:10.769 | INFO | } | 20260508 05:29:10.769 | INFO | Perf report for the last 5.14054 seconds: | 20260508 05:29:10.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:10.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 05:29:10.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:29:10.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:10.769 | INFO | Uploaded 36GiB to GPU in 2.56987s (14.0085GiB/s) | 20260508 05:29:10.769 | INFO | Downloaded 34GiB to GPU in 2.41422s (14.0832GiB/s) | 20260508 05:29:15.908 | INFO | Analyzing chunk frames = [5601280,5601792) | 20260508 05:29:15.908 | INFO | MemInfo: { | 20260508 05:29:15.908 | INFO | "gpu" : " 17 GB", | 20260508 05:29:15.908 | INFO | "huge" : "", | 20260508 05:29:15.908 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:15.908 | INFO | "malloc" : " 14 GB", | 20260508 05:29:15.908 | INFO | "pinned" : " 29 GB" | 20260508 05:29:15.908 | INFO | } | 20260508 05:29:15.908 | INFO | Perf report for the last 5.13934 seconds: | 20260508 05:29:15.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:15.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.908 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:29:15.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.909 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:29:15.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:15.909 | INFO | Uploaded 36GiB to GPU in 2.38026s (15.1244GiB/s) | 20260508 05:29:15.909 | INFO | Downloaded 34GiB to GPU in 2.54771s (13.3453GiB/s) | 20260508 05:29:21.053 | INFO | Analyzing chunk frames = [5601792,5602304) | 20260508 05:29:21.053 | INFO | MemInfo: { | 20260508 05:29:21.053 | INFO | "gpu" : " 17 GB", | 20260508 05:29:21.053 | INFO | "huge" : "", | 20260508 05:29:21.053 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:21.053 | INFO | "malloc" : " 14 GB", | 20260508 05:29:21.053 | INFO | "pinned" : " 29 GB" | 20260508 05:29:21.053 | INFO | } | 20260508 05:29:21.053 | INFO | Perf report for the last 5.14449 seconds: | 20260508 05:29:21.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:21.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.053 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:29:21.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.053 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:29:21.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:21.054 | INFO | Uploaded 36GiB to GPU in 2.45235s (14.6798GiB/s) | 20260508 05:29:21.054 | INFO | Downloaded 34GiB to GPU in 2.41838s (14.059GiB/s) | 20260508 05:29:26.187 | INFO | Analyzing chunk frames = [5602304,5602816) | 20260508 05:29:26.187 | INFO | MemInfo: { | 20260508 05:29:26.187 | INFO | "gpu" : " 17 GB", | 20260508 05:29:26.187 | INFO | "huge" : "", | 20260508 05:29:26.188 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:26.188 | INFO | "malloc" : " 14 GB", | 20260508 05:29:26.188 | INFO | "pinned" : " 29 GB" | 20260508 05:29:26.188 | INFO | } | 20260508 05:29:26.188 | INFO | Perf report for the last 5.1346 seconds: | 20260508 05:29:26.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:26.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:29:26.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:29:26.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:26.188 | INFO | Uploaded 36GiB to GPU in 2.48463s (14.4891GiB/s) | 20260508 05:29:26.188 | INFO | Downloaded 34GiB to GPU in 2.43039s (13.9895GiB/s) | 20260508 05:29:31.333 | INFO | Analyzing chunk frames = [5602816,5603328) | 20260508 05:29:31.333 | INFO | MemInfo: { | 20260508 05:29:31.333 | INFO | "gpu" : " 17 GB", | 20260508 05:29:31.333 | INFO | "huge" : "", | 20260508 05:29:31.333 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:31.333 | INFO | "malloc" : " 14 GB", | 20260508 05:29:31.333 | INFO | "pinned" : " 29 GB" | 20260508 05:29:31.333 | INFO | } | 20260508 05:29:31.334 | INFO | Perf report for the last 5.14586 seconds: | 20260508 05:29:31.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:31.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:29:31.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 05:29:31.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:31.334 | INFO | Uploaded 36GiB to GPU in 2.48s (14.5161GiB/s) | 20260508 05:29:31.334 | INFO | Downloaded 34GiB to GPU in 2.43777s (13.9472GiB/s) | 20260508 05:29:31.334 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:29:31.334 | INFO | Total mem usage: 16.7 GiB | 20260508 05:29:31.334 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:29:31.334 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:29:31.334 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:29:31.334 | INFO | High Water Breakdown: | 20260508 05:29:31.334 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:29:31.334 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:29:31.335 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:29:31.335 | INFO | ------------------------------------------ | 20260508 05:29:31.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:29:31.335 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:29:31.335 | INFO | Total mem usage: 26.5 GiB | 20260508 05:29:31.335 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:29:31.335 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 05:29:31.335 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 05:29:31.335 | INFO | High Water Breakdown: | 20260508 05:29:31.335 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:29:31.335 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:29:31.335 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:29:31.335 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:29:31.335 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:29:31.335 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:29:31.335 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:29:31.335 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:29:31.335 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:29:31.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:29:31.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:29:31.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:29:31.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:29:31.335 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:29:31.335 | INFO | ------------------------------------------ | 20260508 05:29:31.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:29:31.335 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:29:31.335 | INFO | Total mem usage: 27.8 GiB | 20260508 05:29:31.335 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:29:31.335 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:29:31.335 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:29:31.335 | INFO | High Water Breakdown: | 20260508 05:29:31.335 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:29:31.335 | INFO | ------------------------------------------ | 20260508 05:29:31.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:29:31.335 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:29:31.335 | INFO | Total mem usage: 15.7 GiB | 20260508 05:29:31.335 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:29:31.335 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:29:31.335 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:29:31.335 | INFO | High Water Breakdown: | 20260508 05:29:31.335 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:29:31.335 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:29:31.335 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:29:31.335 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:29:31.335 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:29:31.335 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:29:31.335 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:29:31.335 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:29:31.335 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:29:31.335 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:29:31.335 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:29:31.335 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:29:31.335 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:29:31.335 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:29:31.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:29:31.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:29:31.335 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:29:31.335 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:29:31.335 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:29:31.335 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:29:31.335 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:29:31.335 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:29:31.335 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:29:31.335 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:29:31.335 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:29:31.335 | INFO | ------------------------------------------ | 20260508 05:29:31.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:29:31.336 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:29:31.336 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:29:31.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:29:31.336 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 1.17e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.05e-01, 1.12e-02 / 7.69e-03 | 20260508 05:29:31.336 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.40e+00, 5.83e-01 / 2.35e-01 | 20260508 05:29:31.336 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.42e+00, 2.07e-01 / 1.54e-01 | 20260508 05:29:31.336 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.50e+01, 3.76e+00 / 1.58e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.82e+00 / 1.29e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.14e-01 / 2.01e-01 | 20260508 05:29:31.336 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.24e+00, 2.57e-01 / 1.60e-01 | 20260508 05:29:31.336 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.53e-01, 4.91e-02 / 2.47e-02 | 20260508 05:29:31.336 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:29:31.336 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:29:31.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:29:31.336 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 24.37%, Upload -- 2.59e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 24.27%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 2.19%, LockUpload -- 2.32e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 2.65%, LockDownload -- 2.82e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 15.68%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:29:31.336 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 3.09e-02 / 0.00e+00 | 20260508 05:29:31.447 | INFO | Starting a round of Baz IO | 20260508 05:29:34.867 | INFO | Populating RTMetrics report | 20260508 05:29:34.867 | INFO | RTMetrics report populated | 20260508 05:29:34.867 | INFO | Opening tmp RTMetrics json file | 20260508 05:29:34.880 | INFO | Serializing RTMetrics to json | 20260508 05:29:34.881 | INFO | Serialization done, swapping tmp file now... | 20260508 05:29:34.882 | INFO | RTMetrics done | 20260508 05:29:34.938 | INFO | Finished a round of Baz IO: Wrote 1.573 GiB in 3.491 seconds | 20260508 05:29:34.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.86% / 0.00% / 59.14% | 20260508 05:29:34.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.461487% / 3.97364e-06% / 6.75519e-05% | 20260508 05:29:34.938 | INFO | Baz IO overall: | 20260508 05:29:34.938 | INFO | BytesWritten : 2496596009458 | 20260508 05:29:34.938 | INFO | NumEvents : 1204506306841 | 20260508 05:29:36.472 | INFO | Analyzing chunk frames = [5603328,5603840) | 20260508 05:29:36.472 | INFO | MemInfo: { | 20260508 05:29:36.472 | INFO | "gpu" : " 17 GB", | 20260508 05:29:36.472 | INFO | "huge" : "", | 20260508 05:29:36.472 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:36.472 | INFO | "malloc" : " 14 GB", | 20260508 05:29:36.472 | INFO | "pinned" : " 29 GB" | 20260508 05:29:36.472 | INFO | } | 20260508 05:29:36.473 | INFO | Perf report for the last 5.13899 seconds: | 20260508 05:29:36.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:36.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:36.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:36.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:36.473 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 05:29:36.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:36.473 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 05:29:36.473 | INFO | PRE_HQ : 66.5%, 1.8, 62.1ms, 0 | 20260508 05:29:36.473 | INFO | BAZWRITER : 67.3%, 2.9, 100.7ms, 0 | 20260508 05:29:36.473 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:29:36.473 | INFO | Uploaded 36GiB to GPU in 2.54873s (14.1247GiB/s) | 20260508 05:29:36.473 | INFO | Downloaded 38GiB to GPU in 2.70067s (14.0706GiB/s) | 20260508 05:29:41.607 | INFO | Analyzing chunk frames = [5603840,5604352) | 20260508 05:29:41.607 | INFO | MemInfo: { | 20260508 05:29:41.607 | INFO | "gpu" : " 17 GB", | 20260508 05:29:41.607 | INFO | "huge" : "", | 20260508 05:29:41.607 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:41.608 | INFO | "malloc" : " 14 GB", | 20260508 05:29:41.608 | INFO | "pinned" : " 29 GB" | 20260508 05:29:41.608 | INFO | } | 20260508 05:29:41.608 | INFO | Perf report for the last 5.13535 seconds: | 20260508 05:29:41.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:41.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 05:29:41.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:29:41.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:41.608 | INFO | Uploaded 36GiB to GPU in 2.51003s (14.3424GiB/s) | 20260508 05:29:41.608 | INFO | Downloaded 34GiB to GPU in 2.43459s (13.9654GiB/s) | 20260508 05:29:46.747 | INFO | Analyzing chunk frames = [5604352,5604864) | 20260508 05:29:46.747 | INFO | MemInfo: { | 20260508 05:29:46.747 | INFO | "gpu" : " 17 GB", | 20260508 05:29:46.747 | INFO | "huge" : "", | 20260508 05:29:46.747 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:46.747 | INFO | "malloc" : " 14 GB", | 20260508 05:29:46.747 | INFO | "pinned" : " 29 GB" | 20260508 05:29:46.747 | INFO | } | 20260508 05:29:46.747 | INFO | Perf report for the last 5.13942 seconds: | 20260508 05:29:46.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:46.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.747 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 05:29:46.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.747 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:29:46.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:46.748 | INFO | Uploaded 36GiB to GPU in 2.56236s (14.0496GiB/s) | 20260508 05:29:46.748 | INFO | Downloaded 34GiB to GPU in 2.40682s (14.1265GiB/s) | 20260508 05:29:51.886 | INFO | Analyzing chunk frames = [5604864,5605376) | 20260508 05:29:51.886 | INFO | MemInfo: { | 20260508 05:29:51.886 | INFO | "gpu" : " 17 GB", | 20260508 05:29:51.886 | INFO | "huge" : "", | 20260508 05:29:51.886 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:51.886 | INFO | "malloc" : " 14 GB", | 20260508 05:29:51.886 | INFO | "pinned" : " 29 GB" | 20260508 05:29:51.886 | INFO | } | 20260508 05:29:51.886 | INFO | Perf report for the last 5.13852 seconds: | 20260508 05:29:51.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:51.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 05:29:51.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:29:51.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:51.886 | INFO | Uploaded 36GiB to GPU in 2.58853s (13.9075GiB/s) | 20260508 05:29:51.886 | INFO | Downloaded 34GiB to GPU in 2.39617s (14.1893GiB/s) | 20260508 05:29:57.021 | INFO | Analyzing chunk frames = [5605376,5605888) | 20260508 05:29:57.021 | INFO | MemInfo: { | 20260508 05:29:57.021 | INFO | "gpu" : " 17 GB", | 20260508 05:29:57.021 | INFO | "huge" : "", | 20260508 05:29:57.021 | INFO | "huge_shared" : " 29 GB", | 20260508 05:29:57.021 | INFO | "malloc" : " 14 GB", | 20260508 05:29:57.021 | INFO | "pinned" : " 29 GB" | 20260508 05:29:57.021 | INFO | } | 20260508 05:29:57.021 | INFO | Perf report for the last 5.13521 seconds: | 20260508 05:29:57.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:29:57.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 05:29:57.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:29:57.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:29:57.021 | INFO | Uploaded 36GiB to GPU in 2.33797s (15.398GiB/s) | 20260508 05:29:57.021 | INFO | Downloaded 34GiB to GPU in 2.48227s (13.6971GiB/s) | 20260508 05:30:02.165 | INFO | Analyzing chunk frames = [5605888,5606400) | 20260508 05:30:02.165 | INFO | MemInfo: { | 20260508 05:30:02.165 | INFO | "gpu" : " 17 GB", | 20260508 05:30:02.165 | INFO | "huge" : "", | 20260508 05:30:02.165 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:02.165 | INFO | "malloc" : " 14 GB", | 20260508 05:30:02.165 | INFO | "pinned" : " 29 GB" | 20260508 05:30:02.165 | INFO | } | 20260508 05:30:02.165 | INFO | Perf report for the last 5.14383 seconds: | 20260508 05:30:02.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:02.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:30:02.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:30:02.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:02.165 | INFO | Uploaded 36GiB to GPU in 2.45288s (14.6766GiB/s) | 20260508 05:30:02.165 | INFO | Downloaded 34GiB to GPU in 2.40839s (14.1173GiB/s) | 20260508 05:30:07.305 | INFO | Analyzing chunk frames = [5606400,5606912) | 20260508 05:30:07.305 | INFO | MemInfo: { | 20260508 05:30:07.305 | INFO | "gpu" : " 17 GB", | 20260508 05:30:07.305 | INFO | "huge" : "", | 20260508 05:30:07.305 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:07.305 | INFO | "malloc" : " 14 GB", | 20260508 05:30:07.306 | INFO | "pinned" : " 29 GB" | 20260508 05:30:07.306 | INFO | } | 20260508 05:30:07.306 | INFO | Perf report for the last 5.14073 seconds: | 20260508 05:30:07.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:07.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:30:07.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 05:30:07.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:07.306 | INFO | Uploaded 36GiB to GPU in 2.45347s (14.6731GiB/s) | 20260508 05:30:07.306 | INFO | Downloaded 34GiB to GPU in 2.41401s (14.0845GiB/s) | 20260508 05:30:12.442 | INFO | Analyzing chunk frames = [5606912,5607424) | 20260508 05:30:12.442 | INFO | MemInfo: { | 20260508 05:30:12.442 | INFO | "gpu" : " 17 GB", | 20260508 05:30:12.442 | INFO | "huge" : "", | 20260508 05:30:12.442 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:12.442 | INFO | "malloc" : " 14 GB", | 20260508 05:30:12.442 | INFO | "pinned" : " 29 GB" | 20260508 05:30:12.442 | INFO | } | 20260508 05:30:12.442 | INFO | Perf report for the last 5.13674 seconds: | 20260508 05:30:12.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:12.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.443 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:30:12.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.443 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 05:30:12.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:12.443 | INFO | Uploaded 36GiB to GPU in 2.49343s (14.438GiB/s) | 20260508 05:30:12.444 | INFO | Downloaded 34GiB to GPU in 2.43585s (13.9582GiB/s) | 20260508 05:30:12.556 | INFO | Starting a round of Baz IO | 20260508 05:30:15.963 | INFO | Populating RTMetrics report | 20260508 05:30:15.964 | INFO | RTMetrics report populated | 20260508 05:30:15.964 | INFO | Opening tmp RTMetrics json file | 20260508 05:30:15.975 | INFO | Serializing RTMetrics to json | 20260508 05:30:15.977 | INFO | Serialization done, swapping tmp file now... | 20260508 05:30:15.977 | INFO | RTMetrics done | 20260508 05:30:16.041 | INFO | Finished a round of Baz IO: Wrote 1.575 GiB in 3.484 seconds | 20260508 05:30:16.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.86% / 0.00% / 59.14% | 20260508 05:30:16.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.462898% / 3.97364e-06% / 5.16574e-05% | 20260508 05:30:16.041 | INFO | Baz IO overall: | 20260508 05:30:16.041 | INFO | BytesWritten : 2498286736994 | 20260508 05:30:16.041 | INFO | NumEvents : 1205312862765 | 20260508 05:30:17.585 | INFO | Analyzing chunk frames = [5607424,5607936) | 20260508 05:30:17.585 | INFO | MemInfo: { | 20260508 05:30:17.585 | INFO | "gpu" : " 17 GB", | 20260508 05:30:17.585 | INFO | "huge" : "", | 20260508 05:30:17.585 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:17.585 | INFO | "malloc" : " 14 GB", | 20260508 05:30:17.585 | INFO | "pinned" : " 29 GB" | 20260508 05:30:17.585 | INFO | } | 20260508 05:30:17.586 | INFO | Perf report for the last 5.14308 seconds: | 20260508 05:30:17.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:17.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:17.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:17.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:17.586 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 05:30:17.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:17.586 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:30:17.586 | INFO | PRE_HQ : 66.3%, 1.8, 61.5ms, 0 | 20260508 05:30:17.586 | INFO | BAZWRITER : 67.3%, 2.9, 99.8ms, 0 | 20260508 05:30:17.586 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:30:17.586 | INFO | Uploaded 36GiB to GPU in 2.57771s (13.9659GiB/s) | 20260508 05:30:17.586 | INFO | Downloaded 38GiB to GPU in 2.68055s (14.1762GiB/s) | 20260508 05:30:22.724 | INFO | Analyzing chunk frames = [5607936,5608448) | 20260508 05:30:22.724 | INFO | MemInfo: { | 20260508 05:30:22.724 | INFO | "gpu" : " 17 GB", | 20260508 05:30:22.724 | INFO | "huge" : "", | 20260508 05:30:22.724 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:22.724 | INFO | "malloc" : " 14 GB", | 20260508 05:30:22.724 | INFO | "pinned" : " 29 GB" | 20260508 05:30:22.724 | INFO | } | 20260508 05:30:22.724 | INFO | Perf report for the last 5.13845 seconds: | 20260508 05:30:22.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:22.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 05:30:22.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:30:22.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:22.724 | INFO | Uploaded 36GiB to GPU in 2.51231s (14.3294GiB/s) | 20260508 05:30:22.724 | INFO | Downloaded 34GiB to GPU in 2.43544s (13.9605GiB/s) | 20260508 05:30:27.862 | INFO | Analyzing chunk frames = [5608448,5608960) | 20260508 05:30:27.862 | INFO | MemInfo: { | 20260508 05:30:27.862 | INFO | "gpu" : " 17 GB", | 20260508 05:30:27.862 | INFO | "huge" : "", | 20260508 05:30:27.862 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:27.862 | INFO | "malloc" : " 14 GB", | 20260508 05:30:27.862 | INFO | "pinned" : " 29 GB" | 20260508 05:30:27.862 | INFO | } | 20260508 05:30:27.862 | INFO | Perf report for the last 5.13823 seconds: | 20260508 05:30:27.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:27.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.862 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:30:27.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.862 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:30:27.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:27.863 | INFO | Uploaded 36GiB to GPU in 2.56659s (14.0264GiB/s) | 20260508 05:30:27.863 | INFO | Downloaded 34GiB to GPU in 2.41696s (14.0672GiB/s) | 20260508 05:30:33.000 | INFO | Analyzing chunk frames = [5608960,5609472) | 20260508 05:30:33.000 | INFO | MemInfo: { | 20260508 05:30:33.000 | INFO | "gpu" : " 17 GB", | 20260508 05:30:33.000 | INFO | "huge" : "", | 20260508 05:30:33.000 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:33.000 | INFO | "malloc" : " 14 GB", | 20260508 05:30:33.000 | INFO | "pinned" : " 29 GB" | 20260508 05:30:33.000 | INFO | } | 20260508 05:30:33.000 | INFO | Perf report for the last 5.13807 seconds: | 20260508 05:30:33.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:33.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.000 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 05:30:33.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.000 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:30:33.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:33.001 | INFO | Uploaded 36GiB to GPU in 2.55949s (14.0653GiB/s) | 20260508 05:30:33.001 | INFO | Downloaded 34GiB to GPU in 2.38588s (14.2505GiB/s) | 20260508 05:30:38.137 | INFO | Analyzing chunk frames = [5609472,5609984) | 20260508 05:30:38.137 | INFO | MemInfo: { | 20260508 05:30:38.137 | INFO | "gpu" : " 17 GB", | 20260508 05:30:38.137 | INFO | "huge" : "", | 20260508 05:30:38.137 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:38.137 | INFO | "malloc" : " 14 GB", | 20260508 05:30:38.137 | INFO | "pinned" : " 29 GB" | 20260508 05:30:38.137 | INFO | } | 20260508 05:30:38.138 | INFO | Perf report for the last 5.13719 seconds: | 20260508 05:30:38.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:38.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:30:38.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:30:38.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:38.138 | INFO | Uploaded 36GiB to GPU in 2.28997s (15.7207GiB/s) | 20260508 05:30:38.138 | INFO | Downloaded 34GiB to GPU in 2.48613s (13.6759GiB/s) | 20260508 05:30:43.277 | INFO | Analyzing chunk frames = [5609984,5610496) | 20260508 05:30:43.277 | INFO | MemInfo: { | 20260508 05:30:43.277 | INFO | "gpu" : " 17 GB", | 20260508 05:30:43.277 | INFO | "huge" : "", | 20260508 05:30:43.277 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:43.277 | INFO | "malloc" : " 14 GB", | 20260508 05:30:43.277 | INFO | "pinned" : " 29 GB" | 20260508 05:30:43.277 | INFO | } | 20260508 05:30:43.277 | INFO | Perf report for the last 5.13927 seconds: | 20260508 05:30:43.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:43.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:30:43.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:30:43.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:43.277 | INFO | Uploaded 36GiB to GPU in 2.43394s (14.7908GiB/s) | 20260508 05:30:43.277 | INFO | Downloaded 34GiB to GPU in 2.41741s (14.0646GiB/s) | 20260508 05:30:48.421 | INFO | Analyzing chunk frames = [5610496,5611008) | 20260508 05:30:48.421 | INFO | MemInfo: { | 20260508 05:30:48.421 | INFO | "gpu" : " 17 GB", | 20260508 05:30:48.421 | INFO | "huge" : "", | 20260508 05:30:48.422 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:48.422 | INFO | "malloc" : " 14 GB", | 20260508 05:30:48.422 | INFO | "pinned" : " 29 GB" | 20260508 05:30:48.422 | INFO | } | 20260508 05:30:48.422 | INFO | Perf report for the last 5.14508 seconds: | 20260508 05:30:48.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:48.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 05:30:48.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 05:30:48.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:48.422 | INFO | Uploaded 36GiB to GPU in 2.42185s (14.8647GiB/s) | 20260508 05:30:48.422 | INFO | Downloaded 34GiB to GPU in 2.42311s (14.0316GiB/s) | 20260508 05:30:53.561 | INFO | Analyzing chunk frames = [5611008,5611520) | 20260508 05:30:53.561 | INFO | MemInfo: { | 20260508 05:30:53.562 | INFO | "gpu" : " 17 GB", | 20260508 05:30:53.562 | INFO | "huge" : "", | 20260508 05:30:53.562 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:53.562 | INFO | "malloc" : " 14 GB", | 20260508 05:30:53.562 | INFO | "pinned" : " 29 GB" | 20260508 05:30:53.562 | INFO | } | 20260508 05:30:53.562 | INFO | Perf report for the last 5.14007 seconds: | 20260508 05:30:53.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:53.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.562 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:30:53.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.562 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 05:30:53.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:53.563 | INFO | Uploaded 36GiB to GPU in 2.46055s (14.6309GiB/s) | 20260508 05:30:53.563 | INFO | Downloaded 34GiB to GPU in 2.43986s (13.9353GiB/s) | 20260508 05:30:53.563 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:30:53.563 | INFO | Total mem usage: 16.7 GiB | 20260508 05:30:53.563 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:30:53.563 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:30:53.563 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:30:53.563 | INFO | High Water Breakdown: | 20260508 05:30:53.563 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:30:53.563 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:30:53.563 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:30:53.563 | INFO | ------------------------------------------ | 20260508 05:30:53.563 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:30:53.563 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:30:53.563 | INFO | Total mem usage: 26.5 GiB | 20260508 05:30:53.563 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:30:53.563 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 05:30:53.563 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 05:30:53.563 | INFO | High Water Breakdown: | 20260508 05:30:53.563 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:30:53.563 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:30:53.563 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:30:53.563 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:30:53.563 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:30:53.563 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:30:53.563 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:30:53.563 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:30:53.563 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:30:53.563 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:30:53.563 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:30:53.563 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:30:53.563 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:30:53.563 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:30:53.563 | INFO | ------------------------------------------ | 20260508 05:30:53.563 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:30:53.563 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:30:53.563 | INFO | Total mem usage: 27.8 GiB | 20260508 05:30:53.563 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:30:53.563 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:30:53.563 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:30:53.563 | INFO | High Water Breakdown: | 20260508 05:30:53.563 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:30:53.563 | INFO | ------------------------------------------ | 20260508 05:30:53.563 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:30:53.563 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:30:53.563 | INFO | Total mem usage: 15.7 GiB | 20260508 05:30:53.563 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:30:53.563 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:30:53.563 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:30:53.563 | INFO | High Water Breakdown: | 20260508 05:30:53.563 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:30:53.564 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:30:53.564 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:30:53.564 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:30:53.564 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:30:53.564 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:30:53.564 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:30:53.564 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:30:53.564 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:30:53.564 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:30:53.564 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:30:53.564 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:30:53.564 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:30:53.564 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:30:53.564 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:30:53.564 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:30:53.564 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:30:53.564 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:30:53.564 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:30:53.564 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:30:53.564 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:30:53.564 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:30:53.564 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:30:53.564 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:30:53.564 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:30:53.564 | INFO | ------------------------------------------ | 20260508 05:30:53.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:30:53.564 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:30:53.564 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:30:53.564 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:30:53.564 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.93e-01 | 20260508 05:30:53.564 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.06e-01, 1.17e-02 / 8.48e-03 | 20260508 05:30:53.564 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.41e+00, 5.87e-01 / 2.25e-01 | 20260508 05:30:53.564 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.20e-01 | 20260508 05:30:53.564 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.51e+01, 3.78e+00 / 1.20e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.36e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.57e+00, 4.14e-01 / 2.81e-01 | 20260508 05:30:53.564 | INFO | PerfReport: 3.19%, Metrics -- 1.05e+00 / 1.24e+00, 2.56e-01 / 1.34e-01 | 20260508 05:30:53.564 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.56e-01, 5.01e-02 / 2.63e-02 | 20260508 05:30:53.564 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:30:53.564 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:30:53.564 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:30:53.564 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 24.40%, Upload -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 24.15%, Download -- 2.56e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 30.87%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 2.44%, LockUpload -- 2.59e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 2.44%, LockDownload -- 2.59e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 15.69%, LockCompute -- 1.67e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 05:30:53.564 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260508 05:30:53.676 | INFO | Starting a round of Baz IO | 20260508 05:30:57.093 | INFO | Populating RTMetrics report | 20260508 05:30:57.093 | INFO | RTMetrics report populated | 20260508 05:30:57.093 | INFO | Opening tmp RTMetrics json file | 20260508 05:30:57.094 | INFO | Serializing RTMetrics to json | 20260508 05:30:57.096 | INFO | Serialization done, swapping tmp file now... | 20260508 05:30:57.096 | INFO | RTMetrics done | 20260508 05:30:57.168 | INFO | Finished a round of Baz IO: Wrote 1.574 GiB in 3.491 seconds | 20260508 05:30:57.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.86% / 0.00% / 59.14% | 20260508 05:30:57.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.460446% / 3.97364e-06% / 4.76837e-05% | 20260508 05:30:57.168 | INFO | Baz IO overall: | 20260508 05:30:57.168 | INFO | BytesWritten : 2499976528044 | 20260508 05:30:57.168 | INFO | NumEvents : 1206118942907 | 20260508 05:30:58.692 | INFO | Analyzing chunk frames = [5611520,5612032) | 20260508 05:30:58.692 | INFO | MemInfo: { | 20260508 05:30:58.692 | INFO | "gpu" : " 17 GB", | 20260508 05:30:58.692 | INFO | "huge" : "", | 20260508 05:30:58.692 | INFO | "huge_shared" : " 29 GB", | 20260508 05:30:58.692 | INFO | "malloc" : " 14 GB", | 20260508 05:30:58.692 | INFO | "pinned" : " 29 GB" | 20260508 05:30:58.692 | INFO | } | 20260508 05:30:58.692 | INFO | Perf report for the last 5.13008 seconds: | 20260508 05:30:58.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:30:58.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:58.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:58.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:58.692 | INFO | ANALYSIS : 68.4%, 3.0, 106.0ms, 0 | 20260508 05:30:58.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:30:58.692 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260508 05:30:58.692 | INFO | PRE_HQ : 66.8%, 1.8, 61.6ms, 0 | 20260508 05:30:58.692 | INFO | BAZWRITER : 67.5%, 2.9, 100.0ms, 0 | 20260508 05:30:58.692 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 05:30:58.693 | INFO | Uploaded 36GiB to GPU in 2.54355s (14.1535GiB/s) | 20260508 05:30:58.693 | INFO | Downloaded 38GiB to GPU in 2.71412s (14.0009GiB/s) | 20260508 05:31:03.833 | INFO | Analyzing chunk frames = [5612032,5612544) | 20260508 05:31:03.833 | INFO | MemInfo: { | 20260508 05:31:03.833 | INFO | "gpu" : " 17 GB", | 20260508 05:31:03.833 | INFO | "huge" : "", | 20260508 05:31:03.833 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:03.833 | INFO | "malloc" : " 14 GB", | 20260508 05:31:03.833 | INFO | "pinned" : " 29 GB" | 20260508 05:31:03.833 | INFO | } | 20260508 05:31:03.833 | INFO | Perf report for the last 5.14081 seconds: | 20260508 05:31:03.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:03.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 05:31:03.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:31:03.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:03.833 | INFO | Uploaded 36GiB to GPU in 2.54394s (14.1513GiB/s) | 20260508 05:31:03.833 | INFO | Downloaded 34GiB to GPU in 2.45571s (13.8453GiB/s) | 20260508 05:31:08.974 | INFO | Analyzing chunk frames = [5612544,5613056) | 20260508 05:31:08.974 | INFO | MemInfo: { | 20260508 05:31:08.974 | INFO | "gpu" : " 17 GB", | 20260508 05:31:08.974 | INFO | "huge" : "", | 20260508 05:31:08.974 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:08.974 | INFO | "malloc" : " 14 GB", | 20260508 05:31:08.974 | INFO | "pinned" : " 29 GB" | 20260508 05:31:08.974 | INFO | } | 20260508 05:31:08.974 | INFO | Perf report for the last 5.14094 seconds: | 20260508 05:31:08.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:08.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:31:08.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:31:08.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:08.974 | INFO | Uploaded 36GiB to GPU in 2.50161s (14.3907GiB/s) | 20260508 05:31:08.974 | INFO | Downloaded 34GiB to GPU in 2.43177s (13.9816GiB/s) | 20260508 05:31:14.114 | INFO | Analyzing chunk frames = [5613056,5613568) | 20260508 05:31:14.114 | INFO | MemInfo: { | 20260508 05:31:14.114 | INFO | "gpu" : " 17 GB", | 20260508 05:31:14.114 | INFO | "huge" : "", | 20260508 05:31:14.115 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:14.115 | INFO | "malloc" : " 14 GB", | 20260508 05:31:14.115 | INFO | "pinned" : " 29 GB" | 20260508 05:31:14.115 | INFO | } | 20260508 05:31:14.115 | INFO | Perf report for the last 5.14084 seconds: | 20260508 05:31:14.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:14.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 05:31:14.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:31:14.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:14.115 | INFO | Uploaded 36GiB to GPU in 2.5095s (14.3455GiB/s) | 20260508 05:31:14.115 | INFO | Downloaded 34GiB to GPU in 2.4101s (14.1073GiB/s) | 20260508 05:31:19.256 | INFO | Analyzing chunk frames = [5613568,5614080) | 20260508 05:31:19.256 | INFO | MemInfo: { | 20260508 05:31:19.256 | INFO | "gpu" : " 17 GB", | 20260508 05:31:19.256 | INFO | "huge" : "", | 20260508 05:31:19.256 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:19.256 | INFO | "malloc" : " 14 GB", | 20260508 05:31:19.256 | INFO | "pinned" : " 29 GB" | 20260508 05:31:19.256 | INFO | } | 20260508 05:31:19.256 | INFO | Perf report for the last 5.14156 seconds: | 20260508 05:31:19.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:19.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.256 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:31:19.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.256 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:31:19.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:19.257 | INFO | Uploaded 36GiB to GPU in 2.40994s (14.9382GiB/s) | 20260508 05:31:19.257 | INFO | Downloaded 34GiB to GPU in 2.55046s (13.3309GiB/s) | 20260508 05:31:24.392 | INFO | Analyzing chunk frames = [5614080,5614592) | 20260508 05:31:24.392 | INFO | MemInfo: { | 20260508 05:31:24.392 | INFO | "gpu" : " 17 GB", | 20260508 05:31:24.392 | INFO | "huge" : "", | 20260508 05:31:24.392 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:24.393 | INFO | "malloc" : " 14 GB", | 20260508 05:31:24.393 | INFO | "pinned" : " 29 GB" | 20260508 05:31:24.393 | INFO | } | 20260508 05:31:24.393 | INFO | Perf report for the last 5.13639 seconds: | 20260508 05:31:24.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:24.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 05:31:24.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:31:24.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:24.393 | INFO | Uploaded 36GiB to GPU in 2.40612s (14.9618GiB/s) | 20260508 05:31:24.393 | INFO | Downloaded 34GiB to GPU in 2.45608s (13.8432GiB/s) | 20260508 05:31:29.535 | INFO | Analyzing chunk frames = [5614592,5615104) | 20260508 05:31:29.535 | INFO | MemInfo: { | 20260508 05:31:29.535 | INFO | "gpu" : " 17 GB", | 20260508 05:31:29.535 | INFO | "huge" : "", | 20260508 05:31:29.535 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:29.535 | INFO | "malloc" : " 14 GB", | 20260508 05:31:29.535 | INFO | "pinned" : " 29 GB" | 20260508 05:31:29.535 | INFO | } | 20260508 05:31:29.536 | INFO | Perf report for the last 5.14277 seconds: | 20260508 05:31:29.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:29.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 05:31:29.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:31:29.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:29.536 | INFO | Uploaded 36GiB to GPU in 2.37652s (15.1482GiB/s) | 20260508 05:31:29.536 | INFO | Downloaded 34GiB to GPU in 2.44262s (13.9195GiB/s) | 20260508 05:31:34.673 | INFO | Analyzing chunk frames = [5615104,5615616) | 20260508 05:31:34.673 | INFO | MemInfo: { | 20260508 05:31:34.673 | INFO | "gpu" : " 17 GB", | 20260508 05:31:34.673 | INFO | "huge" : "", | 20260508 05:31:34.673 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:34.673 | INFO | "malloc" : " 14 GB", | 20260508 05:31:34.673 | INFO | "pinned" : " 29 GB" | 20260508 05:31:34.673 | INFO | } | 20260508 05:31:34.673 | INFO | Perf report for the last 5.13783 seconds: | 20260508 05:31:34.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:34.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.673 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:31:34.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.674 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 05:31:34.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:34.674 | INFO | Uploaded 36GiB to GPU in 2.4523s (14.6801GiB/s) | 20260508 05:31:34.674 | INFO | Downloaded 34GiB to GPU in 2.45885s (13.8276GiB/s) | 20260508 05:31:34.788 | INFO | Starting a round of Baz IO | 20260508 05:31:38.204 | INFO | Populating RTMetrics report | 20260508 05:31:38.204 | INFO | RTMetrics report populated | 20260508 05:31:38.204 | INFO | Opening tmp RTMetrics json file | 20260508 05:31:38.215 | INFO | Serializing RTMetrics to json | 20260508 05:31:38.217 | INFO | Serialization done, swapping tmp file now... | 20260508 05:31:38.217 | INFO | RTMetrics done | 20260508 05:31:38.274 | INFO | Finished a round of Baz IO: Wrote 1.572 GiB in 3.485 seconds | 20260508 05:31:38.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.85% / 0.00% / 59.14% | 20260508 05:31:38.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.458634% / 3.97364e-06% / 4.37101e-05% | 20260508 05:31:38.274 | INFO | Baz IO overall: | 20260508 05:31:38.274 | INFO | BytesWritten : 2501664806262 | 20260508 05:31:38.274 | INFO | NumEvents : 1206923856887 | 20260508 05:31:39.810 | INFO | Analyzing chunk frames = [5615616,5616128) | 20260508 05:31:39.810 | INFO | MemInfo: { | 20260508 05:31:39.810 | INFO | "gpu" : " 17 GB", | 20260508 05:31:39.810 | INFO | "huge" : "", | 20260508 05:31:39.810 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:39.810 | INFO | "malloc" : " 14 GB", | 20260508 05:31:39.810 | INFO | "pinned" : " 29 GB" | 20260508 05:31:39.810 | INFO | } | 20260508 05:31:39.810 | INFO | Perf report for the last 5.13716 seconds: | 20260508 05:31:39.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:39.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:39.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:39.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:39.811 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260508 05:31:39.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:39.811 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 05:31:39.811 | INFO | PRE_HQ : 66.4%, 1.8, 62.0ms, 0 | 20260508 05:31:39.811 | INFO | BAZWRITER : 67.2%, 2.9, 100.1ms, 0 | 20260508 05:31:39.811 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:31:39.811 | INFO | Uploaded 36GiB to GPU in 2.5373s (14.1883GiB/s) | 20260508 05:31:39.811 | INFO | Downloaded 38GiB to GPU in 2.6998s (14.0751GiB/s) | 20260508 05:31:44.946 | INFO | Analyzing chunk frames = [5616128,5616640) | 20260508 05:31:44.946 | INFO | MemInfo: { | 20260508 05:31:44.946 | INFO | "gpu" : " 17 GB", | 20260508 05:31:44.946 | INFO | "huge" : "", | 20260508 05:31:44.946 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:44.946 | INFO | "malloc" : " 14 GB", | 20260508 05:31:44.946 | INFO | "pinned" : " 29 GB" | 20260508 05:31:44.946 | INFO | } | 20260508 05:31:44.946 | INFO | Perf report for the last 5.13573 seconds: | 20260508 05:31:44.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:44.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.946 | INFO | ANALYSIS : 67.3%, 3.0, 104.8ms, 0 | 20260508 05:31:44.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.946 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:31:44.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:44.947 | INFO | Uploaded 36GiB to GPU in 2.55009s (14.1171GiB/s) | 20260508 05:31:44.947 | INFO | Downloaded 34GiB to GPU in 2.45254s (13.8632GiB/s) | 20260508 05:31:50.090 | INFO | Analyzing chunk frames = [5616640,5617152) | 20260508 05:31:50.090 | INFO | MemInfo: { | 20260508 05:31:50.090 | INFO | "gpu" : " 17 GB", | 20260508 05:31:50.090 | INFO | "huge" : "", | 20260508 05:31:50.090 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:50.090 | INFO | "malloc" : " 14 GB", | 20260508 05:31:50.090 | INFO | "pinned" : " 29 GB" | 20260508 05:31:50.091 | INFO | } | 20260508 05:31:50.091 | INFO | Perf report for the last 5.14457 seconds: | 20260508 05:31:50.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:50.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:31:50.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:31:50.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:50.091 | INFO | Uploaded 36GiB to GPU in 2.50841s (14.3517GiB/s) | 20260508 05:31:50.091 | INFO | Downloaded 34GiB to GPU in 2.42953s (13.9945GiB/s) | 20260508 05:31:55.228 | INFO | Analyzing chunk frames = [5617152,5617664) | 20260508 05:31:55.228 | INFO | MemInfo: { | 20260508 05:31:55.228 | INFO | "gpu" : " 17 GB", | 20260508 05:31:55.229 | INFO | "huge" : "", | 20260508 05:31:55.229 | INFO | "huge_shared" : " 29 GB", | 20260508 05:31:55.229 | INFO | "malloc" : " 14 GB", | 20260508 05:31:55.229 | INFO | "pinned" : " 29 GB" | 20260508 05:31:55.229 | INFO | } | 20260508 05:31:55.229 | INFO | Perf report for the last 5.13793 seconds: | 20260508 05:31:55.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:31:55.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 05:31:55.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:31:55.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:31:55.229 | INFO | Uploaded 36GiB to GPU in 2.50995s (14.3429GiB/s) | 20260508 05:31:55.229 | INFO | Downloaded 34GiB to GPU in 2.42997s (13.992GiB/s) | 20260508 05:32:00.370 | INFO | Analyzing chunk frames = [5617664,5618176) | 20260508 05:32:00.370 | INFO | MemInfo: { | 20260508 05:32:00.370 | INFO | "gpu" : " 17 GB", | 20260508 05:32:00.370 | INFO | "huge" : "", | 20260508 05:32:00.370 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:00.370 | INFO | "malloc" : " 14 GB", | 20260508 05:32:00.370 | INFO | "pinned" : " 29 GB" | 20260508 05:32:00.370 | INFO | } | 20260508 05:32:00.371 | INFO | Perf report for the last 5.14176 seconds: | 20260508 05:32:00.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:00.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:32:00.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:32:00.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:00.371 | INFO | Uploaded 36GiB to GPU in 2.32395s (15.4909GiB/s) | 20260508 05:32:00.371 | INFO | Downloaded 34GiB to GPU in 2.56052s (13.2786GiB/s) | 20260508 05:32:05.505 | INFO | Analyzing chunk frames = [5618176,5618688) | 20260508 05:32:05.505 | INFO | MemInfo: { | 20260508 05:32:05.505 | INFO | "gpu" : " 17 GB", | 20260508 05:32:05.505 | INFO | "huge" : "", | 20260508 05:32:05.505 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:05.505 | INFO | "malloc" : " 14 GB", | 20260508 05:32:05.505 | INFO | "pinned" : " 29 GB" | 20260508 05:32:05.505 | INFO | } | 20260508 05:32:05.505 | INFO | Perf report for the last 5.13452 seconds: | 20260508 05:32:05.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:05.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 05:32:05.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:32:05.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:05.505 | INFO | Uploaded 36GiB to GPU in 2.40048s (14.997GiB/s) | 20260508 05:32:05.505 | INFO | Downloaded 34GiB to GPU in 2.44522s (13.9047GiB/s) | 20260508 05:32:10.646 | INFO | Analyzing chunk frames = [5618688,5619200) | 20260508 05:32:10.646 | INFO | MemInfo: { | 20260508 05:32:10.646 | INFO | "gpu" : " 17 GB", | 20260508 05:32:10.646 | INFO | "huge" : "", | 20260508 05:32:10.646 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:10.646 | INFO | "malloc" : " 14 GB", | 20260508 05:32:10.646 | INFO | "pinned" : " 29 GB" | 20260508 05:32:10.646 | INFO | } | 20260508 05:32:10.646 | INFO | Perf report for the last 5.1413 seconds: | 20260508 05:32:10.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:10.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.646 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 05:32:10.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.647 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 05:32:10.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:10.647 | INFO | Uploaded 36GiB to GPU in 2.38483s (15.0954GiB/s) | 20260508 05:32:10.647 | INFO | Downloaded 34GiB to GPU in 2.42957s (13.9942GiB/s) | 20260508 05:32:15.786 | INFO | Analyzing chunk frames = [5619200,5619712) | 20260508 05:32:15.786 | INFO | MemInfo: { | 20260508 05:32:15.786 | INFO | "gpu" : " 17 GB", | 20260508 05:32:15.786 | INFO | "huge" : "", | 20260508 05:32:15.787 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:15.787 | INFO | "malloc" : " 14 GB", | 20260508 05:32:15.787 | INFO | "pinned" : " 29 GB" | 20260508 05:32:15.787 | INFO | } | 20260508 05:32:15.787 | INFO | Perf report for the last 5.14033 seconds: | 20260508 05:32:15.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:15.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.787 | INFO | ANALYSIS : 66.4%, 3.0, 102.5ms, 0 | 20260508 05:32:15.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.787 | INFO | PRE_HQ : 34.1%, 1.0, 17.6ms, 0 | 20260508 05:32:15.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:15.788 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:32:15.788 | INFO | Total mem usage: 16.7 GiB | 20260508 05:32:15.788 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:32:15.788 | INFO | Uploaded 36GiB to GPU in 2.42447s (14.8486GiB/s) | 20260508 05:32:15.788 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:32:15.788 | INFO | Downloaded 34GiB to GPU in 2.45685s (13.8389GiB/s) | 20260508 05:32:15.788 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:32:15.788 | INFO | High Water Breakdown: | 20260508 05:32:15.788 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:32:15.788 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:32:15.788 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:32:15.788 | INFO | ------------------------------------------ | 20260508 05:32:15.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:32:15.788 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:32:15.788 | INFO | Total mem usage: 26.5 GiB | 20260508 05:32:15.788 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:32:15.788 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:32:15.788 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:32:15.788 | INFO | High Water Breakdown: | 20260508 05:32:15.788 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:32:15.788 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:32:15.788 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:32:15.788 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:32:15.788 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:32:15.788 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:32:15.788 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:32:15.788 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:32:15.788 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:32:15.788 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:32:15.788 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:32:15.788 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:32:15.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:32:15.789 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:32:15.789 | INFO | ------------------------------------------ | 20260508 05:32:15.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:32:15.789 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:32:15.789 | INFO | Total mem usage: 27.8 GiB | 20260508 05:32:15.789 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:32:15.789 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:32:15.789 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:32:15.789 | INFO | High Water Breakdown: | 20260508 05:32:15.789 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:32:15.789 | INFO | ------------------------------------------ | 20260508 05:32:15.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:32:15.789 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:32:15.789 | INFO | Total mem usage: 15.7 GiB | 20260508 05:32:15.789 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:32:15.789 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:32:15.789 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:32:15.789 | INFO | High Water Breakdown: | 20260508 05:32:15.789 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:32:15.789 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:32:15.789 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:32:15.789 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:32:15.789 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:32:15.789 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:32:15.789 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:32:15.789 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:32:15.789 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:32:15.789 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:32:15.789 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:32:15.789 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:32:15.789 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:32:15.789 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:32:15.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:32:15.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:32:15.789 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:32:15.789 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:32:15.789 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:32:15.789 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:32:15.789 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:32:15.789 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:32:15.789 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:32:15.789 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:32:15.789 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:32:15.789 | INFO | ------------------------------------------ | 20260508 05:32:15.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:32:15.789 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:32:15.789 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:32:15.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:32:15.789 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 8.74e-01 | 20260508 05:32:15.789 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.21e-02 / 7.54e-03 | 20260508 05:32:15.789 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.44e+00, 5.84e-01 / 1.93e-01 | 20260508 05:32:15.789 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.21e-01 | 20260508 05:32:15.789 | INFO | PerfReport: 15.89%, DME -- 4.16e+01 / 4.51e+01, 3.68e+00 / 1.25e+00 | 20260508 05:32:15.789 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.80e+00 / 1.34e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.16e-01 / 2.11e-01 | 20260508 05:32:15.790 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.24e+00, 2.57e-01 / 1.19e-01 | 20260508 05:32:15.790 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.19e-01 / 3.57e-01, 5.04e-02 / 2.01e-02 | 20260508 05:32:15.790 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:32:15.790 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:32:15.790 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:32:15.790 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 24.18%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 24.45%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 1.91%, LockUpload -- 2.02e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 2.99%, LockDownload -- 3.17e+00 / 0.00e+00, 4.54e+00 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:32:15.790 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 05:32:15.901 | INFO | Starting a round of Baz IO | 20260508 05:32:19.306 | INFO | Populating RTMetrics report | 20260508 05:32:19.306 | INFO | RTMetrics report populated | 20260508 05:32:19.306 | INFO | Opening tmp RTMetrics json file | 20260508 05:32:19.319 | INFO | Serializing RTMetrics to json | 20260508 05:32:19.320 | INFO | Serialization done, swapping tmp file now... | 20260508 05:32:19.321 | INFO | RTMetrics done | 20260508 05:32:19.386 | INFO | Finished a round of Baz IO: Wrote 1.573 GiB in 3.484 seconds | 20260508 05:32:19.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.85% / 0.00% / 59.15% | 20260508 05:32:19.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.458054% / 3.97364e-06% / 5.5631e-05% | 20260508 05:32:19.386 | INFO | Baz IO overall: | 20260508 05:32:19.386 | INFO | BytesWritten : 2503353498924 | 20260508 05:32:19.386 | INFO | NumEvents : 1207728968143 | 20260508 05:32:20.929 | INFO | Analyzing chunk frames = [5619712,5620224) | 20260508 05:32:20.929 | INFO | MemInfo: { | 20260508 05:32:20.929 | INFO | "gpu" : " 17 GB", | 20260508 05:32:20.929 | INFO | "huge" : "", | 20260508 05:32:20.929 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:20.929 | INFO | "malloc" : " 14 GB", | 20260508 05:32:20.929 | INFO | "pinned" : " 29 GB" | 20260508 05:32:20.929 | INFO | } | 20260508 05:32:20.929 | INFO | Perf report for the last 5.14215 seconds: | 20260508 05:32:20.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:20.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:20.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:20.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:20.929 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 05:32:20.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:20.930 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 05:32:20.930 | INFO | PRE_HQ : 66.5%, 1.8, 61.4ms, 0 | 20260508 05:32:20.930 | INFO | BAZWRITER : 67.0%, 2.9, 98.9ms, 0 | 20260508 05:32:20.930 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:32:20.930 | INFO | Uploaded 36GiB to GPU in 2.53972s (14.1748GiB/s) | 20260508 05:32:20.930 | INFO | Downloaded 38GiB to GPU in 2.68558s (14.1496GiB/s) | 20260508 05:32:26.065 | INFO | Analyzing chunk frames = [5620224,5620736) | 20260508 05:32:26.065 | INFO | MemInfo: { | 20260508 05:32:26.065 | INFO | "gpu" : " 17 GB", | 20260508 05:32:26.065 | INFO | "huge" : "", | 20260508 05:32:26.065 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:26.065 | INFO | "malloc" : " 14 GB", | 20260508 05:32:26.065 | INFO | "pinned" : " 29 GB" | 20260508 05:32:26.065 | INFO | } | 20260508 05:32:26.065 | INFO | Perf report for the last 5.13638 seconds: | 20260508 05:32:26.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:26.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.065 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 05:32:26.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.065 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:32:26.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:26.066 | INFO | Uploaded 36GiB to GPU in 2.57449s (13.9834GiB/s) | 20260508 05:32:26.066 | INFO | Downloaded 34GiB to GPU in 2.45222s (13.865GiB/s) | 20260508 05:32:31.201 | INFO | Analyzing chunk frames = [5620736,5621248) | 20260508 05:32:31.201 | INFO | MemInfo: { | 20260508 05:32:31.201 | INFO | "gpu" : " 17 GB", | 20260508 05:32:31.201 | INFO | "huge" : "", | 20260508 05:32:31.201 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:31.201 | INFO | "malloc" : " 14 GB", | 20260508 05:32:31.201 | INFO | "pinned" : " 29 GB" | 20260508 05:32:31.201 | INFO | } | 20260508 05:32:31.202 | INFO | Perf report for the last 5.13622 seconds: | 20260508 05:32:31.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:31.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 05:32:31.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:32:31.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:31.202 | INFO | Uploaded 36GiB to GPU in 2.51616s (14.3075GiB/s) | 20260508 05:32:31.202 | INFO | Downloaded 34GiB to GPU in 2.40651s (14.1284GiB/s) | 20260508 05:32:36.345 | INFO | Analyzing chunk frames = [5621248,5621760) | 20260508 05:32:36.345 | INFO | MemInfo: { | 20260508 05:32:36.345 | INFO | "gpu" : " 17 GB", | 20260508 05:32:36.345 | INFO | "huge" : "", | 20260508 05:32:36.345 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:36.345 | INFO | "malloc" : " 14 GB", | 20260508 05:32:36.345 | INFO | "pinned" : " 29 GB" | 20260508 05:32:36.345 | INFO | } | 20260508 05:32:36.345 | INFO | Perf report for the last 5.14333 seconds: | 20260508 05:32:36.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:36.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.345 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 05:32:36.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.345 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:32:36.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:36.346 | INFO | Uploaded 36GiB to GPU in 2.57228s (13.9954GiB/s) | 20260508 05:32:36.346 | INFO | Downloaded 34GiB to GPU in 2.39949s (14.1697GiB/s) | 20260508 05:32:41.485 | INFO | Analyzing chunk frames = [5621760,5622272) | 20260508 05:32:41.485 | INFO | MemInfo: { | 20260508 05:32:41.485 | INFO | "gpu" : " 17 GB", | 20260508 05:32:41.485 | INFO | "huge" : "", | 20260508 05:32:41.485 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:41.485 | INFO | "malloc" : " 14 GB", | 20260508 05:32:41.485 | INFO | "pinned" : " 29 GB" | 20260508 05:32:41.485 | INFO | } | 20260508 05:32:41.485 | INFO | Perf report for the last 5.1399 seconds: | 20260508 05:32:41.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:41.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.485 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:32:41.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.485 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:32:41.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:41.486 | INFO | Uploaded 36GiB to GPU in 2.32052s (15.5138GiB/s) | 20260508 05:32:41.486 | INFO | Downloaded 34GiB to GPU in 2.47742s (13.724GiB/s) | 20260508 05:32:46.624 | INFO | Analyzing chunk frames = [5622272,5622784) | 20260508 05:32:46.624 | INFO | MemInfo: { | 20260508 05:32:46.624 | INFO | "gpu" : " 17 GB", | 20260508 05:32:46.624 | INFO | "huge" : "", | 20260508 05:32:46.624 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:46.624 | INFO | "malloc" : " 14 GB", | 20260508 05:32:46.624 | INFO | "pinned" : " 29 GB" | 20260508 05:32:46.624 | INFO | } | 20260508 05:32:46.624 | INFO | Perf report for the last 5.13927 seconds: | 20260508 05:32:46.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:46.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:32:46.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:32:46.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:46.624 | INFO | Uploaded 36GiB to GPU in 2.44545s (14.7212GiB/s) | 20260508 05:32:46.624 | INFO | Downloaded 34GiB to GPU in 2.42648s (14.0121GiB/s) | 20260508 05:32:51.760 | INFO | Analyzing chunk frames = [5622784,5623296) | 20260508 05:32:51.760 | INFO | MemInfo: { | 20260508 05:32:51.760 | INFO | "gpu" : " 17 GB", | 20260508 05:32:51.760 | INFO | "huge" : "", | 20260508 05:32:51.760 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:51.760 | INFO | "malloc" : " 14 GB", | 20260508 05:32:51.761 | INFO | "pinned" : " 29 GB" | 20260508 05:32:51.761 | INFO | } | 20260508 05:32:51.761 | INFO | Perf report for the last 5.13662 seconds: | 20260508 05:32:51.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:51.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:32:51.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:32:51.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:51.761 | INFO | Uploaded 36GiB to GPU in 2.47545s (14.5428GiB/s) | 20260508 05:32:51.761 | INFO | Downloaded 34GiB to GPU in 2.4281s (14.0027GiB/s) | 20260508 05:32:56.899 | INFO | Analyzing chunk frames = [5623296,5623808) | 20260508 05:32:56.899 | INFO | MemInfo: { | 20260508 05:32:56.899 | INFO | "gpu" : " 17 GB", | 20260508 05:32:56.900 | INFO | "huge" : "", | 20260508 05:32:56.900 | INFO | "huge_shared" : " 29 GB", | 20260508 05:32:56.900 | INFO | "malloc" : " 14 GB", | 20260508 05:32:56.900 | INFO | "pinned" : " 29 GB" | 20260508 05:32:56.900 | INFO | } | 20260508 05:32:56.900 | INFO | Perf report for the last 5.13904 seconds: | 20260508 05:32:56.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:32:56.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:32:56.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 05:32:56.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:32:56.900 | INFO | Uploaded 36GiB to GPU in 2.45072s (14.6896GiB/s) | 20260508 05:32:56.900 | INFO | Downloaded 34GiB to GPU in 2.43721s (13.9504GiB/s) | 20260508 05:32:57.012 | INFO | Starting a round of Baz IO | 20260508 05:33:00.424 | INFO | Populating RTMetrics report | 20260508 05:33:00.425 | INFO | RTMetrics report populated | 20260508 05:33:00.425 | INFO | Opening tmp RTMetrics json file | 20260508 05:33:00.436 | INFO | Serializing RTMetrics to json | 20260508 05:33:00.437 | INFO | Serialization done, swapping tmp file now... | 20260508 05:33:00.438 | INFO | RTMetrics done | 20260508 05:33:00.502 | INFO | Finished a round of Baz IO: Wrote 1.573 GiB in 3.49 seconds | 20260508 05:33:00.503 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.85% / 0.00% / 59.15% | 20260508 05:33:00.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.458654% / 7.94729e-06% / 3.97364e-05% | 20260508 05:33:00.503 | INFO | Baz IO overall: | 20260508 05:33:00.503 | INFO | BytesWritten : 2505042271320 | 20260508 05:33:00.503 | INFO | NumEvents : 1208534150253 | 20260508 05:33:02.042 | INFO | Analyzing chunk frames = [5623808,5624320) | 20260508 05:33:02.042 | INFO | MemInfo: { | 20260508 05:33:02.042 | INFO | "gpu" : " 17 GB", | 20260508 05:33:02.043 | INFO | "huge" : "", | 20260508 05:33:02.043 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:02.043 | INFO | "malloc" : " 14 GB", | 20260508 05:33:02.043 | INFO | "pinned" : " 29 GB" | 20260508 05:33:02.043 | INFO | } | 20260508 05:33:02.043 | INFO | Perf report for the last 5.14295 seconds: | 20260508 05:33:02.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:02.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:02.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:02.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:02.043 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 05:33:02.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:02.043 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 05:33:02.043 | INFO | PRE_HQ : 66.4%, 1.8, 61.6ms, 0 | 20260508 05:33:02.043 | INFO | BAZWRITER : 67.3%, 2.9, 99.4ms, 0 | 20260508 05:33:02.043 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:33:02.043 | INFO | Uploaded 36GiB to GPU in 2.54774s (14.1302GiB/s) | 20260508 05:33:02.043 | INFO | Downloaded 38GiB to GPU in 2.70253s (14.0609GiB/s) | 20260508 05:33:07.182 | INFO | Analyzing chunk frames = [5624320,5624832) | 20260508 05:33:07.182 | INFO | MemInfo: { | 20260508 05:33:07.182 | INFO | "gpu" : " 17 GB", | 20260508 05:33:07.182 | INFO | "huge" : "", | 20260508 05:33:07.183 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:07.183 | INFO | "malloc" : " 14 GB", | 20260508 05:33:07.183 | INFO | "pinned" : " 29 GB" | 20260508 05:33:07.183 | INFO | } | 20260508 05:33:07.183 | INFO | Perf report for the last 5.13998 seconds: | 20260508 05:33:07.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:07.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:33:07.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:33:07.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:07.183 | INFO | Uploaded 36GiB to GPU in 2.45758s (14.6486GiB/s) | 20260508 05:33:07.183 | INFO | Downloaded 34GiB to GPU in 2.45551s (13.8464GiB/s) | 20260508 05:33:12.321 | INFO | Analyzing chunk frames = [5624832,5625344) | 20260508 05:33:12.321 | INFO | MemInfo: { | 20260508 05:33:12.321 | INFO | "gpu" : " 17 GB", | 20260508 05:33:12.322 | INFO | "huge" : "", | 20260508 05:33:12.322 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:12.322 | INFO | "malloc" : " 14 GB", | 20260508 05:33:12.322 | INFO | "pinned" : " 29 GB" | 20260508 05:33:12.322 | INFO | } | 20260508 05:33:12.322 | INFO | Perf report for the last 5.13896 seconds: | 20260508 05:33:12.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:12.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:33:12.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:33:12.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:12.322 | INFO | Uploaded 36GiB to GPU in 2.49947s (14.403GiB/s) | 20260508 05:33:12.322 | INFO | Downloaded 34GiB to GPU in 2.42426s (14.0249GiB/s) | 20260508 05:33:17.455 | INFO | Analyzing chunk frames = [5625344,5625856) | 20260508 05:33:17.455 | INFO | MemInfo: { | 20260508 05:33:17.455 | INFO | "gpu" : " 17 GB", | 20260508 05:33:17.455 | INFO | "huge" : "", | 20260508 05:33:17.455 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:17.455 | INFO | "malloc" : " 14 GB", | 20260508 05:33:17.455 | INFO | "pinned" : " 29 GB" | 20260508 05:33:17.455 | INFO | } | 20260508 05:33:17.455 | INFO | Perf report for the last 5.13344 seconds: | 20260508 05:33:17.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:17.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 05:33:17.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:33:17.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:17.455 | INFO | Uploaded 36GiB to GPU in 2.51779s (14.2982GiB/s) | 20260508 05:33:17.455 | INFO | Downloaded 34GiB to GPU in 2.42551s (14.0177GiB/s) | 20260508 05:33:22.598 | INFO | Analyzing chunk frames = [5625856,5626368) | 20260508 05:33:22.598 | INFO | MemInfo: { | 20260508 05:33:22.598 | INFO | "gpu" : " 17 GB", | 20260508 05:33:22.598 | INFO | "huge" : "", | 20260508 05:33:22.598 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:22.598 | INFO | "malloc" : " 14 GB", | 20260508 05:33:22.598 | INFO | "pinned" : " 29 GB" | 20260508 05:33:22.598 | INFO | } | 20260508 05:33:22.598 | INFO | Perf report for the last 5.14298 seconds: | 20260508 05:33:22.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:22.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.598 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:33:22.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.598 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:33:22.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:22.599 | INFO | Uploaded 36GiB to GPU in 2.29911s (15.6582GiB/s) | 20260508 05:33:22.599 | INFO | Downloaded 34GiB to GPU in 2.48992s (13.6551GiB/s) | 20260508 05:33:27.738 | INFO | Analyzing chunk frames = [5626368,5626880) | 20260508 05:33:27.738 | INFO | MemInfo: { | 20260508 05:33:27.738 | INFO | "gpu" : " 17 GB", | 20260508 05:33:27.738 | INFO | "huge" : "", | 20260508 05:33:27.738 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:27.738 | INFO | "malloc" : " 14 GB", | 20260508 05:33:27.739 | INFO | "pinned" : " 29 GB" | 20260508 05:33:27.739 | INFO | } | 20260508 05:33:27.739 | INFO | Perf report for the last 5.14053 seconds: | 20260508 05:33:27.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:27.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:33:27.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:33:27.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:27.739 | INFO | Uploaded 36GiB to GPU in 2.40967s (14.9398GiB/s) | 20260508 05:33:27.739 | INFO | Downloaded 34GiB to GPU in 2.4539s (13.8555GiB/s) | 20260508 05:33:32.880 | INFO | Analyzing chunk frames = [5626880,5627392) | 20260508 05:33:32.880 | INFO | MemInfo: { | 20260508 05:33:32.880 | INFO | "gpu" : " 17 GB", | 20260508 05:33:32.880 | INFO | "huge" : "", | 20260508 05:33:32.880 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:32.880 | INFO | "malloc" : " 14 GB", | 20260508 05:33:32.880 | INFO | "pinned" : " 29 GB" | 20260508 05:33:32.880 | INFO | } | 20260508 05:33:32.880 | INFO | Perf report for the last 5.14135 seconds: | 20260508 05:33:32.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:32.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 05:33:32.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 05:33:32.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:32.880 | INFO | Uploaded 36GiB to GPU in 2.41191s (14.9259GiB/s) | 20260508 05:33:32.880 | INFO | Downloaded 34GiB to GPU in 2.45079s (13.8731GiB/s) | 20260508 05:33:38.015 | INFO | Analyzing chunk frames = [5627392,5627904) | 20260508 05:33:38.015 | INFO | MemInfo: { | 20260508 05:33:38.015 | INFO | "gpu" : " 17 GB", | 20260508 05:33:38.015 | INFO | "huge" : "", | 20260508 05:33:38.015 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:38.016 | INFO | "malloc" : " 14 GB", | 20260508 05:33:38.016 | INFO | "pinned" : " 29 GB" | 20260508 05:33:38.016 | INFO | } | 20260508 05:33:38.016 | INFO | Perf report for the last 5.1358 seconds: | 20260508 05:33:38.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:38.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:33:38.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 05:33:38.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:38.016 | INFO | Uploaded 36GiB to GPU in 2.44044s (14.7515GiB/s) | 20260508 05:33:38.016 | INFO | Downloaded 34GiB to GPU in 2.46228s (13.8083GiB/s) | 20260508 05:33:38.017 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:33:38.017 | INFO | Total mem usage: 16.7 GiB | 20260508 05:33:38.017 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:33:38.017 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:33:38.017 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:33:38.017 | INFO | High Water Breakdown: | 20260508 05:33:38.017 | INFO | 59.6% : 8.56 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:33:38.017 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:33:38.017 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:33:38.017 | INFO | ------------------------------------------ | 20260508 05:33:38.017 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:33:38.017 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:33:38.017 | INFO | Total mem usage: 26.5 GiB | 20260508 05:33:38.017 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:33:38.017 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:33:38.017 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:33:38.017 | INFO | High Water Breakdown: | 20260508 05:33:38.017 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:33:38.017 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:33:38.017 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:33:38.017 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:33:38.017 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:33:38.017 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:33:38.017 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:33:38.017 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:33:38.017 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:33:38.017 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:33:38.017 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:33:38.017 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:33:38.017 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:33:38.017 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:33:38.017 | INFO | ------------------------------------------ | 20260508 05:33:38.017 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:33:38.017 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:33:38.017 | INFO | Total mem usage: 27.8 GiB | 20260508 05:33:38.017 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:33:38.017 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:33:38.017 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:33:38.017 | INFO | High Water Breakdown: | 20260508 05:33:38.017 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:33:38.017 | INFO | ------------------------------------------ | 20260508 05:33:38.017 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:33:38.017 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:33:38.017 | INFO | Total mem usage: 15.7 GiB | 20260508 05:33:38.017 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:33:38.017 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:33:38.017 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:33:38.017 | INFO | High Water Breakdown: | 20260508 05:33:38.017 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:33:38.017 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:33:38.017 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:33:38.017 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:33:38.017 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:33:38.017 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:33:38.017 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:33:38.017 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:33:38.017 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:33:38.017 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:33:38.017 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:33:38.017 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:33:38.017 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:33:38.017 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:33:38.017 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:33:38.017 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:33:38.017 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:33:38.017 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:33:38.017 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:33:38.017 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:33:38.017 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:33:38.018 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:33:38.018 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:33:38.018 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:33:38.018 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:33:38.018 | INFO | ------------------------------------------ | 20260508 05:33:38.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:33:38.018 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:33:38.018 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:33:38.018 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:33:38.018 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.35e-01 | 20260508 05:33:38.018 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.18e-02 / 5.97e-03 | 20260508 05:33:38.018 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.82e-01 / 1.97e-01 | 20260508 05:33:38.018 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.32e-01 | 20260508 05:33:38.018 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.52e+01, 3.62e+00 / 1.19e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.42e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.60e+00, 4.12e-01 / 2.48e-01 | 20260508 05:33:38.018 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.21e+00, 2.55e-01 / 1.42e-01 | 20260508 05:33:38.018 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.20e-01 / 3.55e-01, 5.04e-02 / 2.45e-02 | 20260508 05:33:38.018 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:33:38.018 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:33:38.018 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:33:38.018 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 24.23%, Upload -- 2.57e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 2.10%, LockUpload -- 2.23e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 2.76%, LockDownload -- 2.93e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 15.74%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:33:38.018 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260508 05:33:38.130 | INFO | Starting a round of Baz IO | 20260508 05:33:41.544 | INFO | Populating RTMetrics report | 20260508 05:33:41.544 | INFO | RTMetrics report populated | 20260508 05:33:41.544 | INFO | Opening tmp RTMetrics json file | 20260508 05:33:41.557 | INFO | Serializing RTMetrics to json | 20260508 05:33:41.558 | INFO | Serialization done, swapping tmp file now... | 20260508 05:33:41.558 | INFO | RTMetrics done | 20260508 05:33:41.618 | INFO | Finished a round of Baz IO: Wrote 1.572 GiB in 3.488 seconds | 20260508 05:33:41.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.85% / 0.00% / 59.15% | 20260508 05:33:41.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.45677% / 3.97364e-06% / 5.16574e-05% | 20260508 05:33:41.618 | INFO | Baz IO overall: | 20260508 05:33:41.618 | INFO | BytesWritten : 2506729898792 | 20260508 05:33:41.618 | INFO | NumEvents : 1209338599735 | 20260508 05:33:43.154 | INFO | Analyzing chunk frames = [5627904,5628416) | 20260508 05:33:43.154 | INFO | MemInfo: { | 20260508 05:33:43.154 | INFO | "gpu" : " 17 GB", | 20260508 05:33:43.154 | INFO | "huge" : "", | 20260508 05:33:43.154 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:43.154 | INFO | "malloc" : " 14 GB", | 20260508 05:33:43.154 | INFO | "pinned" : " 29 GB" | 20260508 05:33:43.154 | INFO | } | 20260508 05:33:43.154 | INFO | Perf report for the last 5.13786 seconds: | 20260508 05:33:43.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:43.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:43.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:43.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:43.154 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 05:33:43.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:43.154 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 05:33:43.154 | INFO | PRE_HQ : 66.3%, 1.8, 61.6ms, 0 | 20260508 05:33:43.154 | INFO | BAZWRITER : 67.4%, 2.9, 99.7ms, 0 | 20260508 05:33:43.154 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 05:33:43.154 | INFO | Uploaded 36GiB to GPU in 2.53404s (14.2066GiB/s) | 20260508 05:33:43.154 | INFO | Downloaded 38GiB to GPU in 2.6965s (14.0923GiB/s) | 20260508 05:33:48.294 | INFO | Analyzing chunk frames = [5628416,5628928) | 20260508 05:33:48.294 | INFO | MemInfo: { | 20260508 05:33:48.294 | INFO | "gpu" : " 17 GB", | 20260508 05:33:48.294 | INFO | "huge" : "", | 20260508 05:33:48.294 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:48.294 | INFO | "malloc" : " 14 GB", | 20260508 05:33:48.294 | INFO | "pinned" : " 29 GB" | 20260508 05:33:48.295 | INFO | } | 20260508 05:33:48.295 | INFO | Perf report for the last 5.14092 seconds: | 20260508 05:33:48.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:48.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:33:48.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:33:48.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:48.295 | INFO | Uploaded 36GiB to GPU in 2.45401s (14.6698GiB/s) | 20260508 05:33:48.295 | INFO | Downloaded 34GiB to GPU in 2.45225s (13.8648GiB/s) | 20260508 05:33:53.435 | INFO | Analyzing chunk frames = [5628928,5629440) | 20260508 05:33:53.435 | INFO | MemInfo: { | 20260508 05:33:53.435 | INFO | "gpu" : " 17 GB", | 20260508 05:33:53.435 | INFO | "huge" : "", | 20260508 05:33:53.435 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:53.435 | INFO | "malloc" : " 14 GB", | 20260508 05:33:53.435 | INFO | "pinned" : " 29 GB" | 20260508 05:33:53.435 | INFO | } | 20260508 05:33:53.435 | INFO | Perf report for the last 5.14003 seconds: | 20260508 05:33:53.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:53.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:33:53.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:33:53.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:53.435 | INFO | Uploaded 36GiB to GPU in 2.48789s (14.4701GiB/s) | 20260508 05:33:53.436 | INFO | Downloaded 34GiB to GPU in 2.4244s (14.0241GiB/s) | 20260508 05:33:58.572 | INFO | Analyzing chunk frames = [5629440,5629952) | 20260508 05:33:58.572 | INFO | MemInfo: { | 20260508 05:33:58.572 | INFO | "gpu" : " 17 GB", | 20260508 05:33:58.572 | INFO | "huge" : "", | 20260508 05:33:58.572 | INFO | "huge_shared" : " 29 GB", | 20260508 05:33:58.572 | INFO | "malloc" : " 14 GB", | 20260508 05:33:58.572 | INFO | "pinned" : " 29 GB" | 20260508 05:33:58.572 | INFO | } | 20260508 05:33:58.572 | INFO | Perf report for the last 5.13747 seconds: | 20260508 05:33:58.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:33:58.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:33:58.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:33:58.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:33:58.572 | INFO | Uploaded 36GiB to GPU in 2.52663s (14.2482GiB/s) | 20260508 05:33:58.572 | INFO | Downloaded 34GiB to GPU in 2.42185s (14.0388GiB/s) | 20260508 05:34:03.710 | INFO | Analyzing chunk frames = [5629952,5630464) | 20260508 05:34:03.710 | INFO | MemInfo: { | 20260508 05:34:03.710 | INFO | "gpu" : " 17 GB", | 20260508 05:34:03.710 | INFO | "huge" : "", | 20260508 05:34:03.710 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:03.710 | INFO | "malloc" : " 14 GB", | 20260508 05:34:03.710 | INFO | "pinned" : " 29 GB" | 20260508 05:34:03.710 | INFO | } | 20260508 05:34:03.710 | INFO | Perf report for the last 5.13792 seconds: | 20260508 05:34:03.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:03.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.710 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:34:03.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.710 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 05:34:03.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:03.711 | INFO | Uploaded 36GiB to GPU in 2.27539s (15.8215GiB/s) | 20260508 05:34:03.711 | INFO | Downloaded 34GiB to GPU in 2.48996s (13.6548GiB/s) | 20260508 05:34:08.856 | INFO | Analyzing chunk frames = [5630464,5630976) | 20260508 05:34:08.856 | INFO | MemInfo: { | 20260508 05:34:08.856 | INFO | "gpu" : " 17 GB", | 20260508 05:34:08.856 | INFO | "huge" : "", | 20260508 05:34:08.856 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:08.856 | INFO | "malloc" : " 14 GB", | 20260508 05:34:08.856 | INFO | "pinned" : " 29 GB" | 20260508 05:34:08.856 | INFO | } | 20260508 05:34:08.856 | INFO | Perf report for the last 5.14599 seconds: | 20260508 05:34:08.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:08.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.856 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:34:08.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.856 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:34:08.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:08.857 | INFO | Uploaded 36GiB to GPU in 2.3997s (15.0019GiB/s) | 20260508 05:34:08.857 | INFO | Downloaded 34GiB to GPU in 2.45076s (13.8732GiB/s) | 20260508 05:34:13.996 | INFO | Analyzing chunk frames = [5630976,5631488) | 20260508 05:34:13.996 | INFO | MemInfo: { | 20260508 05:34:13.996 | INFO | "gpu" : " 17 GB", | 20260508 05:34:13.996 | INFO | "huge" : "", | 20260508 05:34:13.996 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:13.996 | INFO | "malloc" : " 14 GB", | 20260508 05:34:13.996 | INFO | "pinned" : " 29 GB" | 20260508 05:34:13.996 | INFO | } | 20260508 05:34:13.996 | INFO | Perf report for the last 5.13978 seconds: | 20260508 05:34:13.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:13.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 05:34:13.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 05:34:13.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:13.996 | INFO | Uploaded 36GiB to GPU in 2.4037s (14.9769GiB/s) | 20260508 05:34:13.996 | INFO | Downloaded 34GiB to GPU in 2.44611s (13.8996GiB/s) | 20260508 05:34:19.131 | INFO | Analyzing chunk frames = [5631488,5632000) | 20260508 05:34:19.131 | INFO | MemInfo: { | 20260508 05:34:19.131 | INFO | "gpu" : " 17 GB", | 20260508 05:34:19.131 | INFO | "huge" : "", | 20260508 05:34:19.131 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:19.131 | INFO | "malloc" : " 14 GB", | 20260508 05:34:19.131 | INFO | "pinned" : " 29 GB" | 20260508 05:34:19.131 | INFO | } | 20260508 05:34:19.132 | INFO | Perf report for the last 5.13565 seconds: | 20260508 05:34:19.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:19.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:34:19.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 05:34:19.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:19.132 | INFO | Uploaded 36GiB to GPU in 2.42858s (14.8235GiB/s) | 20260508 05:34:19.132 | INFO | Downloaded 34GiB to GPU in 2.45222s (13.865GiB/s) | 20260508 05:34:19.229 | INFO | Starting a round of Baz IO | 20260508 05:34:22.664 | INFO | Populating RTMetrics report | 20260508 05:34:22.665 | INFO | RTMetrics report populated | 20260508 05:34:22.665 | INFO | Opening tmp RTMetrics json file | 20260508 05:34:22.674 | INFO | Serializing RTMetrics to json | 20260508 05:34:22.676 | INFO | Serialization done, swapping tmp file now... | 20260508 05:34:22.676 | INFO | RTMetrics done | 20260508 05:34:22.734 | INFO | Finished a round of Baz IO: Wrote 1.572 GiB in 3.505 seconds | 20260508 05:34:22.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.85% / 0.00% / 59.15% | 20260508 05:34:22.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.455856% / 0% / 5.96046e-05% | 20260508 05:34:22.734 | INFO | Baz IO overall: | 20260508 05:34:22.734 | INFO | BytesWritten : 2508417296013 | 20260508 05:34:22.734 | INFO | NumEvents : 1210142960896 | 20260508 05:34:24.268 | INFO | Analyzing chunk frames = [5632000,5632512) | 20260508 05:34:24.268 | INFO | MemInfo: { | 20260508 05:34:24.268 | INFO | "gpu" : " 17 GB", | 20260508 05:34:24.268 | INFO | "huge" : "", | 20260508 05:34:24.268 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:24.268 | INFO | "malloc" : " 14 GB", | 20260508 05:34:24.268 | INFO | "pinned" : " 29 GB" | 20260508 05:34:24.268 | INFO | } | 20260508 05:34:24.268 | INFO | Perf report for the last 5.13656 seconds: | 20260508 05:34:24.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:24.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:24.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:24.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:24.268 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 05:34:24.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:24.268 | INFO | RT_METRICS : 39.4%, 1.0, 20.6ms, 0 | 20260508 05:34:24.268 | INFO | PRE_HQ : 66.0%, 1.6, 55.6ms, 0 | 20260508 05:34:24.268 | INFO | BAZWRITER : 67.7%, 2.7, 93.3ms, 0 | 20260508 05:34:24.268 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:34:24.269 | INFO | Uploaded 36GiB to GPU in 2.54011s (14.1726GiB/s) | 20260508 05:34:24.269 | INFO | Downloaded 38GiB to GPU in 2.68735s (14.1403GiB/s) | 20260508 05:34:29.411 | INFO | Analyzing chunk frames = [5632512,5633024) | 20260508 05:34:29.411 | INFO | MemInfo: { | 20260508 05:34:29.411 | INFO | "gpu" : " 17 GB", | 20260508 05:34:29.411 | INFO | "huge" : "", | 20260508 05:34:29.411 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:29.411 | INFO | "malloc" : " 14 GB", | 20260508 05:34:29.411 | INFO | "pinned" : " 29 GB" | 20260508 05:34:29.411 | INFO | } | 20260508 05:34:29.411 | INFO | Perf report for the last 5.14269 seconds: | 20260508 05:34:29.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:29.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 05:34:29.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:34:29.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:29.411 | INFO | Uploaded 36GiB to GPU in 2.48934s (14.4617GiB/s) | 20260508 05:34:29.411 | INFO | Downloaded 34GiB to GPU in 2.4453s (13.9042GiB/s) | 20260508 05:34:34.548 | INFO | Analyzing chunk frames = [5633024,5633536) | 20260508 05:34:34.548 | INFO | MemInfo: { | 20260508 05:34:34.548 | INFO | "gpu" : " 17 GB", | 20260508 05:34:34.548 | INFO | "huge" : "", | 20260508 05:34:34.548 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:34.549 | INFO | "malloc" : " 14 GB", | 20260508 05:34:34.549 | INFO | "pinned" : " 29 GB" | 20260508 05:34:34.549 | INFO | } | 20260508 05:34:34.549 | INFO | Perf report for the last 5.13771 seconds: | 20260508 05:34:34.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:34.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:34:34.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:34:34.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:34.549 | INFO | Uploaded 36GiB to GPU in 2.50162s (14.3907GiB/s) | 20260508 05:34:34.549 | INFO | Downloaded 34GiB to GPU in 2.40357s (14.1457GiB/s) | 20260508 05:34:39.690 | INFO | Analyzing chunk frames = [5633536,5634048) | 20260508 05:34:39.690 | INFO | MemInfo: { | 20260508 05:34:39.690 | INFO | "gpu" : " 17 GB", | 20260508 05:34:39.690 | INFO | "huge" : "", | 20260508 05:34:39.690 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:39.690 | INFO | "malloc" : " 14 GB", | 20260508 05:34:39.690 | INFO | "pinned" : " 29 GB" | 20260508 05:34:39.690 | INFO | } | 20260508 05:34:39.690 | INFO | Perf report for the last 5.14152 seconds: | 20260508 05:34:39.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:39.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.690 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 05:34:39.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.690 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:34:39.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:39.691 | INFO | Uploaded 36GiB to GPU in 2.53273s (14.2139GiB/s) | 20260508 05:34:39.691 | INFO | Downloaded 34GiB to GPU in 2.38669s (14.2457GiB/s) | 20260508 05:34:44.827 | INFO | Analyzing chunk frames = [5634048,5634560) | 20260508 05:34:44.827 | INFO | MemInfo: { | 20260508 05:34:44.827 | INFO | "gpu" : " 17 GB", | 20260508 05:34:44.827 | INFO | "huge" : "", | 20260508 05:34:44.827 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:44.827 | INFO | "malloc" : " 14 GB", | 20260508 05:34:44.827 | INFO | "pinned" : " 29 GB" | 20260508 05:34:44.827 | INFO | } | 20260508 05:34:44.827 | INFO | Perf report for the last 5.1371 seconds: | 20260508 05:34:44.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:44.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.827 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:34:44.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.827 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:34:44.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:44.828 | INFO | Uploaded 36GiB to GPU in 2.28399s (15.7619GiB/s) | 20260508 05:34:44.828 | INFO | Downloaded 34GiB to GPU in 2.47239s (13.7519GiB/s) | 20260508 05:34:49.964 | INFO | Analyzing chunk frames = [5634560,5635072) | 20260508 05:34:49.964 | INFO | MemInfo: { | 20260508 05:34:49.964 | INFO | "gpu" : " 17 GB", | 20260508 05:34:49.964 | INFO | "huge" : "", | 20260508 05:34:49.964 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:49.964 | INFO | "malloc" : " 14 GB", | 20260508 05:34:49.964 | INFO | "pinned" : " 29 GB" | 20260508 05:34:49.964 | INFO | } | 20260508 05:34:49.964 | INFO | Perf report for the last 5.13673 seconds: | 20260508 05:34:49.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:49.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:34:49.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260508 05:34:49.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:49.964 | INFO | Uploaded 36GiB to GPU in 2.44027s (14.7524GiB/s) | 20260508 05:34:49.964 | INFO | Downloaded 34GiB to GPU in 2.42299s (14.0323GiB/s) | 20260508 05:34:55.110 | INFO | Analyzing chunk frames = [5635072,5635584) | 20260508 05:34:55.110 | INFO | MemInfo: { | 20260508 05:34:55.110 | INFO | "gpu" : " 17 GB", | 20260508 05:34:55.110 | INFO | "huge" : "", | 20260508 05:34:55.110 | INFO | "huge_shared" : " 29 GB", | 20260508 05:34:55.110 | INFO | "malloc" : " 14 GB", | 20260508 05:34:55.110 | INFO | "pinned" : " 29 GB" | 20260508 05:34:55.110 | INFO | } | 20260508 05:34:55.110 | INFO | Perf report for the last 5.14632 seconds: | 20260508 05:34:55.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:34:55.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.110 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:34:55.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.110 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:34:55.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:34:55.111 | INFO | Uploaded 36GiB to GPU in 2.47245s (14.5605GiB/s) | 20260508 05:34:55.111 | INFO | Downloaded 34GiB to GPU in 2.42359s (14.0287GiB/s) | 20260508 05:35:00.245 | INFO | Analyzing chunk frames = [5635584,5636096) | 20260508 05:35:00.245 | INFO | MemInfo: { | 20260508 05:35:00.245 | INFO | "gpu" : " 17 GB", | 20260508 05:35:00.245 | INFO | "huge" : "", | 20260508 05:35:00.245 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:00.245 | INFO | "malloc" : " 14 GB", | 20260508 05:35:00.246 | INFO | "pinned" : " 29 GB" | 20260508 05:35:00.246 | INFO | } | 20260508 05:35:00.246 | INFO | Perf report for the last 5.13526 seconds: | 20260508 05:35:00.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:00.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:35:00.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 05:35:00.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:00.246 | INFO | Uploaded 36GiB to GPU in 2.47944s (14.5194GiB/s) | 20260508 05:35:00.246 | INFO | Downloaded 34GiB to GPU in 2.43568s (13.9592GiB/s) | 20260508 05:35:00.246 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:35:00.246 | INFO | Total mem usage: 16.7 GiB | 20260508 05:35:00.246 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:35:00.246 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:35:00.246 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:35:00.246 | INFO | High Water Breakdown: | 20260508 05:35:00.246 | INFO | 59.6% : 8.56 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:35:00.246 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:35:00.246 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:35:00.246 | INFO | ------------------------------------------ | 20260508 05:35:00.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:35:00.246 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:35:00.246 | INFO | Total mem usage: 26.5 GiB | 20260508 05:35:00.246 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:35:00.246 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:35:00.246 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:35:00.246 | INFO | High Water Breakdown: | 20260508 05:35:00.246 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:35:00.246 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:35:00.246 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:35:00.246 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:35:00.246 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:35:00.246 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:35:00.246 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:35:00.246 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:35:00.246 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:35:00.246 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:35:00.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:35:00.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:35:00.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:35:00.247 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:35:00.247 | INFO | ------------------------------------------ | 20260508 05:35:00.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:35:00.247 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:35:00.247 | INFO | Total mem usage: 27.8 GiB | 20260508 05:35:00.247 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:35:00.247 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:35:00.247 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:35:00.247 | INFO | High Water Breakdown: | 20260508 05:35:00.247 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:35:00.247 | INFO | ------------------------------------------ | 20260508 05:35:00.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:35:00.247 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:35:00.247 | INFO | Total mem usage: 15.7 GiB | 20260508 05:35:00.247 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:35:00.247 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:35:00.247 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:35:00.247 | INFO | High Water Breakdown: | 20260508 05:35:00.247 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:35:00.247 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:35:00.247 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:35:00.247 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:35:00.247 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:35:00.247 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:35:00.247 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:35:00.247 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:35:00.247 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:35:00.247 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:35:00.247 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:35:00.247 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:35:00.247 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:35:00.247 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:35:00.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:35:00.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:35:00.247 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:35:00.247 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:35:00.247 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:35:00.247 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:35:00.247 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:35:00.247 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:35:00.247 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:35:00.247 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:35:00.247 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:35:00.247 | INFO | ------------------------------------------ | 20260508 05:35:00.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:35:00.247 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:35:00.247 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:35:00.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:35:00.247 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.37e+01 / 7.75e-01 | 20260508 05:35:00.247 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.05e-01, 1.19e-02 / 6.60e-03 | 20260508 05:35:00.247 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.52e+00, 5.93e-01 / 2.05e-01 | 20260508 05:35:00.247 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.12e-01 / 1.23e-01 | 20260508 05:35:00.247 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.51e+01, 3.66e+00 / 1.24e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.83e+00 / 1.50e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.60e+00, 4.16e-01 / 2.48e-01 | 20260508 05:35:00.247 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.24e+00, 2.55e-01 / 1.38e-01 | 20260508 05:35:00.247 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.62e-01, 5.08e-02 / 3.00e-02 | 20260508 05:35:00.247 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:35:00.247 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:35:00.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:35:00.247 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 24.10%, Upload -- 2.56e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 30.92%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 2.13%, LockUpload -- 2.25e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 2.75%, LockDownload -- 2.91e+00 / 0.00e+00, 4.37e+00 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 15.83%, LockCompute -- 1.68e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:35:00.247 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260508 05:35:00.343 | INFO | Starting a round of Baz IO | 20260508 05:35:03.785 | INFO | Populating RTMetrics report | 20260508 05:35:03.785 | INFO | RTMetrics report populated | 20260508 05:35:03.785 | INFO | Opening tmp RTMetrics json file | 20260508 05:35:03.799 | INFO | Serializing RTMetrics to json | 20260508 05:35:03.800 | INFO | Serialization done, swapping tmp file now... | 20260508 05:35:03.800 | INFO | RTMetrics done | 20260508 05:35:03.854 | INFO | Finished a round of Baz IO: Wrote 1.571 GiB in 3.511 seconds | 20260508 05:35:03.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.84% / 0.00% / 59.15% | 20260508 05:35:03.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.455117% / 3.97364e-06% / 7.15256e-05% | 20260508 05:35:03.855 | INFO | Baz IO overall: | 20260508 05:35:03.855 | INFO | BytesWritten : 2510104289204 | 20260508 05:35:03.855 | INFO | NumEvents : 1210947071091 | 20260508 05:35:05.386 | INFO | Analyzing chunk frames = [5636096,5636608) | 20260508 05:35:05.386 | INFO | MemInfo: { | 20260508 05:35:05.386 | INFO | "gpu" : " 17 GB", | 20260508 05:35:05.386 | INFO | "huge" : "", | 20260508 05:35:05.386 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:05.386 | INFO | "malloc" : " 14 GB", | 20260508 05:35:05.386 | INFO | "pinned" : " 29 GB" | 20260508 05:35:05.386 | INFO | } | 20260508 05:35:05.387 | INFO | Perf report for the last 5.14115 seconds: | 20260508 05:35:05.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:05.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:05.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:05.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:05.387 | INFO | ANALYSIS : 68.2%, 3.0, 106.3ms, 0 | 20260508 05:35:05.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:05.387 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:35:05.387 | INFO | PRE_HQ : 65.6%, 1.6, 55.4ms, 0 | 20260508 05:35:05.387 | INFO | BAZWRITER : 67.7%, 2.7, 93.7ms, 0 | 20260508 05:35:05.387 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:35:05.387 | INFO | Uploaded 36GiB to GPU in 2.55301s (14.101GiB/s) | 20260508 05:35:05.387 | INFO | Downloaded 38GiB to GPU in 2.69519s (14.0992GiB/s) | 20260508 05:35:10.522 | INFO | Analyzing chunk frames = [5636608,5637120) | 20260508 05:35:10.522 | INFO | MemInfo: { | 20260508 05:35:10.522 | INFO | "gpu" : " 17 GB", | 20260508 05:35:10.522 | INFO | "huge" : "", | 20260508 05:35:10.522 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:10.522 | INFO | "malloc" : " 14 GB", | 20260508 05:35:10.522 | INFO | "pinned" : " 29 GB" | 20260508 05:35:10.522 | INFO | } | 20260508 05:35:10.522 | INFO | Perf report for the last 5.1355 seconds: | 20260508 05:35:10.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:10.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 05:35:10.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:35:10.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:10.522 | INFO | Uploaded 36GiB to GPU in 2.518s (14.2971GiB/s) | 20260508 05:35:10.523 | INFO | Downloaded 34GiB to GPU in 2.44323s (13.916GiB/s) | 20260508 05:35:15.662 | INFO | Analyzing chunk frames = [5637120,5637632) | 20260508 05:35:15.662 | INFO | MemInfo: { | 20260508 05:35:15.662 | INFO | "gpu" : " 17 GB", | 20260508 05:35:15.662 | INFO | "huge" : "", | 20260508 05:35:15.662 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:15.662 | INFO | "malloc" : " 14 GB", | 20260508 05:35:15.662 | INFO | "pinned" : " 29 GB" | 20260508 05:35:15.662 | INFO | } | 20260508 05:35:15.662 | INFO | Perf report for the last 5.13984 seconds: | 20260508 05:35:15.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:15.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:35:15.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:35:15.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:15.662 | INFO | Uploaded 36GiB to GPU in 2.53687s (14.1907GiB/s) | 20260508 05:35:15.663 | INFO | Downloaded 34GiB to GPU in 2.40405s (14.1428GiB/s) | 20260508 05:35:20.805 | INFO | Analyzing chunk frames = [5637632,5638144) | 20260508 05:35:20.805 | INFO | MemInfo: { | 20260508 05:35:20.805 | INFO | "gpu" : " 17 GB", | 20260508 05:35:20.805 | INFO | "huge" : "", | 20260508 05:35:20.805 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:20.805 | INFO | "malloc" : " 14 GB", | 20260508 05:35:20.805 | INFO | "pinned" : " 29 GB" | 20260508 05:35:20.805 | INFO | } | 20260508 05:35:20.805 | INFO | Perf report for the last 5.14288 seconds: | 20260508 05:35:20.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:20.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.805 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 05:35:20.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.805 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:35:20.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:20.806 | INFO | Uploaded 36GiB to GPU in 2.59801s (13.8568GiB/s) | 20260508 05:35:20.806 | INFO | Downloaded 34GiB to GPU in 2.40468s (14.1391GiB/s) | 20260508 05:35:25.948 | INFO | Analyzing chunk frames = [5638144,5638656) | 20260508 05:35:25.948 | INFO | MemInfo: { | 20260508 05:35:25.948 | INFO | "gpu" : " 17 GB", | 20260508 05:35:25.948 | INFO | "huge" : "", | 20260508 05:35:25.948 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:25.949 | INFO | "malloc" : " 14 GB", | 20260508 05:35:25.949 | INFO | "pinned" : " 29 GB" | 20260508 05:35:25.949 | INFO | } | 20260508 05:35:25.949 | INFO | Perf report for the last 5.14365 seconds: | 20260508 05:35:25.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:25.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 05:35:25.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260508 05:35:25.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:25.949 | INFO | Uploaded 36GiB to GPU in 2.44995s (14.6942GiB/s) | 20260508 05:35:25.949 | INFO | Downloaded 34GiB to GPU in 2.52838s (13.4474GiB/s) | 20260508 05:35:31.083 | INFO | Analyzing chunk frames = [5638656,5639168) | 20260508 05:35:31.083 | INFO | MemInfo: { | 20260508 05:35:31.083 | INFO | "gpu" : " 17 GB", | 20260508 05:35:31.083 | INFO | "huge" : "", | 20260508 05:35:31.083 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:31.083 | INFO | "malloc" : " 14 GB", | 20260508 05:35:31.083 | INFO | "pinned" : " 29 GB" | 20260508 05:35:31.083 | INFO | } | 20260508 05:35:31.084 | INFO | Perf report for the last 5.13485 seconds: | 20260508 05:35:31.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:31.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:35:31.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:35:31.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:31.084 | INFO | Uploaded 36GiB to GPU in 2.43556s (14.781GiB/s) | 20260508 05:35:31.084 | INFO | Downloaded 34GiB to GPU in 2.42641s (14.0125GiB/s) | 20260508 05:35:36.219 | INFO | Analyzing chunk frames = [5639168,5639680) | 20260508 05:35:36.219 | INFO | MemInfo: { | 20260508 05:35:36.219 | INFO | "gpu" : " 17 GB", | 20260508 05:35:36.219 | INFO | "huge" : "", | 20260508 05:35:36.219 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:36.219 | INFO | "malloc" : " 14 GB", | 20260508 05:35:36.219 | INFO | "pinned" : " 29 GB" | 20260508 05:35:36.219 | INFO | } | 20260508 05:35:36.219 | INFO | Perf report for the last 5.13534 seconds: | 20260508 05:35:36.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:36.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:35:36.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:35:36.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:36.219 | INFO | Uploaded 36GiB to GPU in 2.43496s (14.7847GiB/s) | 20260508 05:35:36.219 | INFO | Downloaded 34GiB to GPU in 2.4251s (14.02GiB/s) | 20260508 05:35:41.359 | INFO | Analyzing chunk frames = [5639680,5640192) | 20260508 05:35:41.359 | INFO | MemInfo: { | 20260508 05:35:41.359 | INFO | "gpu" : " 17 GB", | 20260508 05:35:41.359 | INFO | "huge" : "", | 20260508 05:35:41.359 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:41.359 | INFO | "malloc" : " 14 GB", | 20260508 05:35:41.359 | INFO | "pinned" : " 29 GB" | 20260508 05:35:41.359 | INFO | } | 20260508 05:35:41.359 | INFO | Perf report for the last 5.13991 seconds: | 20260508 05:35:41.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:41.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:35:41.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:35:41.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:41.359 | INFO | Uploaded 36GiB to GPU in 2.4509s (14.6885GiB/s) | 20260508 05:35:41.359 | INFO | Downloaded 34GiB to GPU in 2.43416s (13.9678GiB/s) | 20260508 05:35:41.456 | INFO | Starting a round of Baz IO | 20260508 05:35:44.875 | INFO | Populating RTMetrics report | 20260508 05:35:44.875 | INFO | RTMetrics report populated | 20260508 05:35:44.875 | INFO | Opening tmp RTMetrics json file | 20260508 05:35:44.881 | INFO | Serializing RTMetrics to json | 20260508 05:35:44.890 | INFO | Serialization done, swapping tmp file now... | 20260508 05:35:44.890 | INFO | RTMetrics done | 20260508 05:35:44.954 | INFO | Finished a round of Baz IO: Wrote 1.572 GiB in 3.497 seconds | 20260508 05:35:44.954 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.84% / 0.00% / 59.16% | 20260508 05:35:44.954 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.455527% / 0% / 7.94729e-05% | 20260508 05:35:44.954 | INFO | Baz IO overall: | 20260508 05:35:44.954 | INFO | BytesWritten : 2511792536013 | 20260508 05:35:44.954 | INFO | NumEvents : 1211752223494 | 20260508 05:35:46.501 | INFO | Analyzing chunk frames = [5640192,5640704) | 20260508 05:35:46.501 | INFO | MemInfo: { | 20260508 05:35:46.501 | INFO | "gpu" : " 17 GB", | 20260508 05:35:46.501 | INFO | "huge" : "", | 20260508 05:35:46.501 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:46.501 | INFO | "malloc" : " 14 GB", | 20260508 05:35:46.501 | INFO | "pinned" : " 29 GB" | 20260508 05:35:46.501 | INFO | } | 20260508 05:35:46.501 | INFO | Perf report for the last 5.14253 seconds: | 20260508 05:35:46.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:46.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:46.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:46.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:46.501 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 05:35:46.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:46.501 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:35:46.501 | INFO | PRE_HQ : 65.4%, 1.6, 55.3ms, 0 | 20260508 05:35:46.502 | INFO | BAZWRITER : 67.6%, 2.7, 92.7ms, 0 | 20260508 05:35:46.502 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:35:46.502 | INFO | Uploaded 36GiB to GPU in 2.5277s (14.2422GiB/s) | 20260508 05:35:46.502 | INFO | Downloaded 38GiB to GPU in 2.70952s (14.0246GiB/s) | 20260508 05:35:51.640 | INFO | Analyzing chunk frames = [5640704,5641216) | 20260508 05:35:51.640 | INFO | MemInfo: { | 20260508 05:35:51.640 | INFO | "gpu" : " 17 GB", | 20260508 05:35:51.640 | INFO | "huge" : "", | 20260508 05:35:51.640 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:51.640 | INFO | "malloc" : " 14 GB", | 20260508 05:35:51.640 | INFO | "pinned" : " 29 GB" | 20260508 05:35:51.640 | INFO | } | 20260508 05:35:51.640 | INFO | Perf report for the last 5.13875 seconds: | 20260508 05:35:51.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:51.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 05:35:51.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:35:51.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:51.640 | INFO | Uploaded 36GiB to GPU in 2.56765s (14.0206GiB/s) | 20260508 05:35:51.641 | INFO | Downloaded 34GiB to GPU in 2.44157s (13.9255GiB/s) | 20260508 05:35:56.780 | INFO | Analyzing chunk frames = [5641216,5641728) | 20260508 05:35:56.780 | INFO | MemInfo: { | 20260508 05:35:56.780 | INFO | "gpu" : " 17 GB", | 20260508 05:35:56.780 | INFO | "huge" : "", | 20260508 05:35:56.780 | INFO | "huge_shared" : " 29 GB", | 20260508 05:35:56.780 | INFO | "malloc" : " 14 GB", | 20260508 05:35:56.780 | INFO | "pinned" : " 29 GB" | 20260508 05:35:56.780 | INFO | } | 20260508 05:35:56.780 | INFO | Perf report for the last 5.14017 seconds: | 20260508 05:35:56.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:35:56.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.780 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:35:56.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.780 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:35:56.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:35:56.781 | INFO | Uploaded 36GiB to GPU in 2.53137s (14.2216GiB/s) | 20260508 05:35:56.781 | INFO | Downloaded 34GiB to GPU in 2.4127s (14.0921GiB/s) | 20260508 05:36:01.921 | INFO | Analyzing chunk frames = [5641728,5642240) | 20260508 05:36:01.921 | INFO | MemInfo: { | 20260508 05:36:01.921 | INFO | "gpu" : " 17 GB", | 20260508 05:36:01.921 | INFO | "huge" : "", | 20260508 05:36:01.921 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:01.921 | INFO | "malloc" : " 14 GB", | 20260508 05:36:01.921 | INFO | "pinned" : " 29 GB" | 20260508 05:36:01.921 | INFO | } | 20260508 05:36:01.921 | INFO | Perf report for the last 5.14059 seconds: | 20260508 05:36:01.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:01.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 05:36:01.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:36:01.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:01.921 | INFO | Uploaded 36GiB to GPU in 2.55778s (14.0747GiB/s) | 20260508 05:36:01.921 | INFO | Downloaded 34GiB to GPU in 2.42201s (14.0379GiB/s) | 20260508 05:36:07.057 | INFO | Analyzing chunk frames = [5642240,5642752) | 20260508 05:36:07.057 | INFO | MemInfo: { | 20260508 05:36:07.057 | INFO | "gpu" : " 17 GB", | 20260508 05:36:07.057 | INFO | "huge" : "", | 20260508 05:36:07.057 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:07.057 | INFO | "malloc" : " 14 GB", | 20260508 05:36:07.057 | INFO | "pinned" : " 29 GB" | 20260508 05:36:07.057 | INFO | } | 20260508 05:36:07.057 | INFO | Perf report for the last 5.13647 seconds: | 20260508 05:36:07.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:07.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.057 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:36:07.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.058 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:36:07.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:07.058 | INFO | Uploaded 36GiB to GPU in 2.28622s (15.7465GiB/s) | 20260508 05:36:07.058 | INFO | Downloaded 34GiB to GPU in 2.49106s (13.6488GiB/s) | 20260508 05:36:12.196 | INFO | Analyzing chunk frames = [5642752,5643264) | 20260508 05:36:12.196 | INFO | MemInfo: { | 20260508 05:36:12.196 | INFO | "gpu" : " 17 GB", | 20260508 05:36:12.196 | INFO | "huge" : "", | 20260508 05:36:12.196 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:12.196 | INFO | "malloc" : " 14 GB", | 20260508 05:36:12.196 | INFO | "pinned" : " 29 GB" | 20260508 05:36:12.197 | INFO | } | 20260508 05:36:12.197 | INFO | Perf report for the last 5.13935 seconds: | 20260508 05:36:12.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:12.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:36:12.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:36:12.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:12.197 | INFO | Uploaded 36GiB to GPU in 2.45765s (14.6481GiB/s) | 20260508 05:36:12.197 | INFO | Downloaded 34GiB to GPU in 2.42172s (14.0396GiB/s) | 20260508 05:36:17.335 | INFO | Analyzing chunk frames = [5643264,5643776) | 20260508 05:36:17.335 | INFO | MemInfo: { | 20260508 05:36:17.335 | INFO | "gpu" : " 17 GB", | 20260508 05:36:17.335 | INFO | "huge" : "", | 20260508 05:36:17.335 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:17.335 | INFO | "malloc" : " 14 GB", | 20260508 05:36:17.335 | INFO | "pinned" : " 29 GB" | 20260508 05:36:17.335 | INFO | } | 20260508 05:36:17.335 | INFO | Perf report for the last 5.13811 seconds: | 20260508 05:36:17.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:17.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:36:17.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:36:17.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:17.335 | INFO | Uploaded 36GiB to GPU in 2.46415s (14.6095GiB/s) | 20260508 05:36:17.335 | INFO | Downloaded 34GiB to GPU in 2.41802s (14.0611GiB/s) | 20260508 05:36:22.475 | INFO | Analyzing chunk frames = [5643776,5644288) | 20260508 05:36:22.475 | INFO | MemInfo: { | 20260508 05:36:22.475 | INFO | "gpu" : " 17 GB", | 20260508 05:36:22.475 | INFO | "huge" : "", | 20260508 05:36:22.475 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:22.475 | INFO | "malloc" : " 14 GB", | 20260508 05:36:22.475 | INFO | "pinned" : " 29 GB" | 20260508 05:36:22.475 | INFO | } | 20260508 05:36:22.475 | INFO | Perf report for the last 5.14044 seconds: | 20260508 05:36:22.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:22.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.475 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:36:22.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.475 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:36:22.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:22.476 | INFO | Uploaded 36GiB to GPU in 2.46194s (14.6226GiB/s) | 20260508 05:36:22.476 | INFO | Downloaded 34GiB to GPU in 2.42539s (14.0183GiB/s) | 20260508 05:36:22.476 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:36:22.476 | INFO | Total mem usage: 16.7 GiB | 20260508 05:36:22.476 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:36:22.476 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -17.1%) | 20260508 05:36:22.476 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:36:22.476 | INFO | High Water Breakdown: | 20260508 05:36:22.476 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:36:22.476 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:36:22.476 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:36:22.476 | INFO | ------------------------------------------ | 20260508 05:36:22.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:36:22.476 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:36:22.476 | INFO | Total mem usage: 26.5 GiB | 20260508 05:36:22.476 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:36:22.476 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:36:22.476 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:36:22.476 | INFO | High Water Breakdown: | 20260508 05:36:22.476 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:36:22.476 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:36:22.476 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:36:22.476 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:36:22.476 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:36:22.476 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:36:22.476 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:36:22.476 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:36:22.476 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:36:22.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:36:22.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:36:22.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:36:22.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:36:22.476 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:36:22.476 | INFO | ------------------------------------------ | 20260508 05:36:22.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:36:22.476 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:36:22.476 | INFO | Total mem usage: 27.8 GiB | 20260508 05:36:22.476 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:36:22.476 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:36:22.476 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:36:22.476 | INFO | High Water Breakdown: | 20260508 05:36:22.476 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:36:22.476 | INFO | ------------------------------------------ | 20260508 05:36:22.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:36:22.476 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:36:22.476 | INFO | Total mem usage: 15.7 GiB | 20260508 05:36:22.477 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:36:22.477 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:36:22.477 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:36:22.477 | INFO | High Water Breakdown: | 20260508 05:36:22.477 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:36:22.477 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:36:22.477 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:36:22.477 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:36:22.477 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:36:22.477 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:36:22.477 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:36:22.477 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:36:22.477 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:36:22.477 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:36:22.477 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:36:22.477 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:36:22.477 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:36:22.477 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:36:22.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:36:22.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:36:22.477 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:36:22.477 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:36:22.477 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:36:22.477 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:36:22.477 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:36:22.477 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:36:22.477 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:36:22.477 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:36:22.477 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:36:22.477 | INFO | ------------------------------------------ | 20260508 05:36:22.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:36:22.477 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:36:22.477 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:36:22.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:36:22.477 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.48e-01 | 20260508 05:36:22.477 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.09e-02 / 7.94e-03 | 20260508 05:36:22.477 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.40e+00, 5.85e-01 / 2.27e-01 | 20260508 05:36:22.477 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.26e-01 | 20260508 05:36:22.477 | INFO | PerfReport: 15.87%, DME -- 4.15e+01 / 4.49e+01, 3.87e+00 / 9.78e-01 | 20260508 05:36:22.477 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.78e+00 / 1.30e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.52e+00, 4.13e-01 / 2.66e-01 | 20260508 05:36:22.477 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.23e+00, 2.55e-01 / 1.32e-01 | 20260508 05:36:22.477 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.54e-01, 4.88e-02 / 2.28e-02 | 20260508 05:36:22.477 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:36:22.477 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:36:22.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:36:22.477 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 24.41%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 30.77%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 2.31%, LockUpload -- 2.45e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 2.62%, LockDownload -- 2.78e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:36:22.477 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.85e-02 / 0.00e+00 | 20260508 05:36:22.573 | INFO | Starting a round of Baz IO | 20260508 05:36:25.999 | INFO | Populating RTMetrics report | 20260508 05:36:25.999 | INFO | RTMetrics report populated | 20260508 05:36:25.999 | INFO | Opening tmp RTMetrics json file | 20260508 05:36:26.012 | INFO | Serializing RTMetrics to json | 20260508 05:36:26.013 | INFO | Serialization done, swapping tmp file now... | 20260508 05:36:26.014 | INFO | RTMetrics done | 20260508 05:36:26.075 | INFO | Finished a round of Baz IO: Wrote 1.571 GiB in 3.502 seconds | 20260508 05:36:26.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.84% / 0.00% / 59.16% | 20260508 05:36:26.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.453913% / 7.94729e-06% / 7.54992e-05% | 20260508 05:36:26.075 | INFO | Baz IO overall: | 20260508 05:36:26.075 | INFO | BytesWritten : 2513479672483 | 20260508 05:36:26.075 | INFO | NumEvents : 1212556628896 | 20260508 05:36:27.615 | INFO | Analyzing chunk frames = [5644288,5644800) | 20260508 05:36:27.615 | INFO | MemInfo: { | 20260508 05:36:27.615 | INFO | "gpu" : " 17 GB", | 20260508 05:36:27.615 | INFO | "huge" : "", | 20260508 05:36:27.615 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:27.615 | INFO | "malloc" : " 14 GB", | 20260508 05:36:27.615 | INFO | "pinned" : " 29 GB" | 20260508 05:36:27.615 | INFO | } | 20260508 05:36:27.615 | INFO | Perf report for the last 5.14012 seconds: | 20260508 05:36:27.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:27.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:27.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:27.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:27.616 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 05:36:27.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:27.616 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 05:36:27.616 | INFO | PRE_HQ : 65.7%, 1.6, 55.2ms, 0 | 20260508 05:36:27.616 | INFO | BAZWRITER : 67.6%, 2.6, 91.7ms, 0 | 20260508 05:36:27.616 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:36:27.616 | INFO | Uploaded 36GiB to GPU in 2.56397s (14.0407GiB/s) | 20260508 05:36:27.616 | INFO | Downloaded 38GiB to GPU in 2.69178s (14.117GiB/s) | 20260508 05:36:32.759 | INFO | Analyzing chunk frames = [5644800,5645312) | 20260508 05:36:32.759 | INFO | MemInfo: { | 20260508 05:36:32.759 | INFO | "gpu" : " 17 GB", | 20260508 05:36:32.759 | INFO | "huge" : "", | 20260508 05:36:32.759 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:32.759 | INFO | "malloc" : " 14 GB", | 20260508 05:36:32.759 | INFO | "pinned" : " 29 GB" | 20260508 05:36:32.759 | INFO | } | 20260508 05:36:32.759 | INFO | Perf report for the last 5.14382 seconds: | 20260508 05:36:32.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:32.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.759 | INFO | ANALYSIS : 67.1%, 3.0, 104.7ms, 0 | 20260508 05:36:32.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.759 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:36:32.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:32.760 | INFO | Uploaded 36GiB to GPU in 2.50629s (14.3639GiB/s) | 20260508 05:36:32.760 | INFO | Downloaded 34GiB to GPU in 2.43335s (13.9725GiB/s) | 20260508 05:36:37.896 | INFO | Analyzing chunk frames = [5645312,5645824) | 20260508 05:36:37.896 | INFO | MemInfo: { | 20260508 05:36:37.896 | INFO | "gpu" : " 17 GB", | 20260508 05:36:37.896 | INFO | "huge" : "", | 20260508 05:36:37.896 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:37.896 | INFO | "malloc" : " 14 GB", | 20260508 05:36:37.896 | INFO | "pinned" : " 29 GB" | 20260508 05:36:37.896 | INFO | } | 20260508 05:36:37.896 | INFO | Perf report for the last 5.13649 seconds: | 20260508 05:36:37.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:37.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 05:36:37.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:36:37.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:37.896 | INFO | Uploaded 36GiB to GPU in 2.54476s (14.1467GiB/s) | 20260508 05:36:37.896 | INFO | Downloaded 34GiB to GPU in 2.40391s (14.1436GiB/s) | 20260508 05:36:43.030 | INFO | Analyzing chunk frames = [5645824,5646336) | 20260508 05:36:43.030 | INFO | MemInfo: { | 20260508 05:36:43.030 | INFO | "gpu" : " 17 GB", | 20260508 05:36:43.030 | INFO | "huge" : "", | 20260508 05:36:43.030 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:43.030 | INFO | "malloc" : " 14 GB", | 20260508 05:36:43.031 | INFO | "pinned" : " 29 GB" | 20260508 05:36:43.031 | INFO | } | 20260508 05:36:43.031 | INFO | Perf report for the last 5.13494 seconds: | 20260508 05:36:43.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:43.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 05:36:43.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:36:43.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:43.031 | INFO | Uploaded 36GiB to GPU in 2.56469s (14.0368GiB/s) | 20260508 05:36:43.031 | INFO | Downloaded 34GiB to GPU in 2.38879s (14.2331GiB/s) | 20260508 05:36:48.173 | INFO | Analyzing chunk frames = [5646336,5646848) | 20260508 05:36:48.173 | INFO | MemInfo: { | 20260508 05:36:48.173 | INFO | "gpu" : " 17 GB", | 20260508 05:36:48.173 | INFO | "huge" : "", | 20260508 05:36:48.173 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:48.173 | INFO | "malloc" : " 14 GB", | 20260508 05:36:48.173 | INFO | "pinned" : " 29 GB" | 20260508 05:36:48.173 | INFO | } | 20260508 05:36:48.173 | INFO | Perf report for the last 5.14229 seconds: | 20260508 05:36:48.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:48.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.173 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:36:48.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.173 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:36:48.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:48.174 | INFO | Uploaded 36GiB to GPU in 2.29893s (15.6595GiB/s) | 20260508 05:36:48.174 | INFO | Downloaded 34GiB to GPU in 2.48997s (13.6548GiB/s) | 20260508 05:36:53.314 | INFO | Analyzing chunk frames = [5646848,5647360) | 20260508 05:36:53.314 | INFO | MemInfo: { | 20260508 05:36:53.314 | INFO | "gpu" : " 17 GB", | 20260508 05:36:53.314 | INFO | "huge" : "", | 20260508 05:36:53.314 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:53.314 | INFO | "malloc" : " 14 GB", | 20260508 05:36:53.314 | INFO | "pinned" : " 29 GB" | 20260508 05:36:53.314 | INFO | } | 20260508 05:36:53.314 | INFO | Perf report for the last 5.14124 seconds: | 20260508 05:36:53.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:53.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.314 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:36:53.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.314 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:36:53.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:53.315 | INFO | Uploaded 36GiB to GPU in 2.42189s (14.8645GiB/s) | 20260508 05:36:53.315 | INFO | Downloaded 34GiB to GPU in 2.42494s (14.021GiB/s) | 20260508 05:36:58.452 | INFO | Analyzing chunk frames = [5647360,5647872) | 20260508 05:36:58.452 | INFO | MemInfo: { | 20260508 05:36:58.452 | INFO | "gpu" : " 17 GB", | 20260508 05:36:58.452 | INFO | "huge" : "", | 20260508 05:36:58.452 | INFO | "huge_shared" : " 29 GB", | 20260508 05:36:58.452 | INFO | "malloc" : " 14 GB", | 20260508 05:36:58.452 | INFO | "pinned" : " 29 GB" | 20260508 05:36:58.452 | INFO | } | 20260508 05:36:58.452 | INFO | Perf report for the last 5.13765 seconds: | 20260508 05:36:58.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:36:58.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:36:58.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 05:36:58.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:36:58.452 | INFO | Uploaded 36GiB to GPU in 2.43888s (14.7609GiB/s) | 20260508 05:36:58.452 | INFO | Downloaded 34GiB to GPU in 2.42943s (13.9951GiB/s) | 20260508 05:37:03.592 | INFO | Analyzing chunk frames = [5647872,5648384) | 20260508 05:37:03.592 | INFO | MemInfo: { | 20260508 05:37:03.592 | INFO | "gpu" : " 17 GB", | 20260508 05:37:03.592 | INFO | "huge" : "", | 20260508 05:37:03.592 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:03.592 | INFO | "malloc" : " 14 GB", | 20260508 05:37:03.592 | INFO | "pinned" : " 29 GB" | 20260508 05:37:03.592 | INFO | } | 20260508 05:37:03.592 | INFO | Perf report for the last 5.14042 seconds: | 20260508 05:37:03.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:03.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.592 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:37:03.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.593 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 05:37:03.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:03.593 | INFO | Uploaded 36GiB to GPU in 2.43367s (14.7925GiB/s) | 20260508 05:37:03.593 | INFO | Downloaded 34GiB to GPU in 2.43284s (13.9754GiB/s) | 20260508 05:37:03.705 | INFO | Starting a round of Baz IO | 20260508 05:37:07.111 | INFO | Populating RTMetrics report | 20260508 05:37:07.111 | INFO | RTMetrics report populated | 20260508 05:37:07.111 | INFO | Opening tmp RTMetrics json file | 20260508 05:37:07.121 | INFO | Serializing RTMetrics to json | 20260508 05:37:07.124 | INFO | Serialization done, swapping tmp file now... | 20260508 05:37:07.125 | INFO | RTMetrics done | 20260508 05:37:07.187 | INFO | Finished a round of Baz IO: Wrote 1.571 GiB in 3.481 seconds | 20260508 05:37:07.187 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.84% / 0.00% / 59.16% | 20260508 05:37:07.187 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.453222% / 1.19209e-05% / 4.76837e-05% | 20260508 05:37:07.187 | INFO | Baz IO overall: | 20260508 05:37:07.187 | INFO | BytesWritten : 2515166450192 | 20260508 05:37:07.187 | INFO | NumEvents : 1213360977938 | 20260508 05:37:08.725 | INFO | Analyzing chunk frames = [5648384,5648896) | 20260508 05:37:08.725 | INFO | MemInfo: { | 20260508 05:37:08.725 | INFO | "gpu" : " 17 GB", | 20260508 05:37:08.725 | INFO | "huge" : "", | 20260508 05:37:08.725 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:08.725 | INFO | "malloc" : " 14 GB", | 20260508 05:37:08.725 | INFO | "pinned" : " 29 GB" | 20260508 05:37:08.725 | INFO | } | 20260508 05:37:08.725 | INFO | Perf report for the last 5.1329 seconds: | 20260508 05:37:08.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:08.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:08.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:08.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:08.725 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 05:37:08.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:08.725 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260508 05:37:08.725 | INFO | PRE_HQ : 66.4%, 1.8, 61.7ms, 0 | 20260508 05:37:08.725 | INFO | BAZWRITER : 67.1%, 2.9, 98.8ms, 0 | 20260508 05:37:08.725 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:37:08.726 | INFO | Uploaded 36GiB to GPU in 2.56364s (14.0426GiB/s) | 20260508 05:37:08.726 | INFO | Downloaded 38GiB to GPU in 2.69928s (14.0778GiB/s) | 20260508 05:37:13.871 | INFO | Analyzing chunk frames = [5648896,5649408) | 20260508 05:37:13.871 | INFO | MemInfo: { | 20260508 05:37:13.871 | INFO | "gpu" : " 17 GB", | 20260508 05:37:13.871 | INFO | "huge" : "", | 20260508 05:37:13.871 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:13.871 | INFO | "malloc" : " 14 GB", | 20260508 05:37:13.871 | INFO | "pinned" : " 29 GB" | 20260508 05:37:13.871 | INFO | } | 20260508 05:37:13.871 | INFO | Perf report for the last 5.14547 seconds: | 20260508 05:37:13.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:13.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 05:37:13.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 05:37:13.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:13.871 | INFO | Uploaded 36GiB to GPU in 2.56715s (14.0233GiB/s) | 20260508 05:37:13.871 | INFO | Downloaded 34GiB to GPU in 2.44736s (13.8925GiB/s) | 20260508 05:37:19.011 | INFO | Analyzing chunk frames = [5649408,5649920) | 20260508 05:37:19.011 | INFO | MemInfo: { | 20260508 05:37:19.011 | INFO | "gpu" : " 17 GB", | 20260508 05:37:19.011 | INFO | "huge" : "", | 20260508 05:37:19.011 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:19.011 | INFO | "malloc" : " 14 GB", | 20260508 05:37:19.011 | INFO | "pinned" : " 29 GB" | 20260508 05:37:19.011 | INFO | } | 20260508 05:37:19.011 | INFO | Perf report for the last 5.14021 seconds: | 20260508 05:37:19.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:19.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:37:19.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:37:19.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:19.011 | INFO | Uploaded 36GiB to GPU in 2.52978s (14.2305GiB/s) | 20260508 05:37:19.011 | INFO | Downloaded 34GiB to GPU in 2.4241s (14.0258GiB/s) | 20260508 05:37:24.149 | INFO | Analyzing chunk frames = [5649920,5650432) | 20260508 05:37:24.149 | INFO | MemInfo: { | 20260508 05:37:24.149 | INFO | "gpu" : " 17 GB", | 20260508 05:37:24.149 | INFO | "huge" : "", | 20260508 05:37:24.149 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:24.149 | INFO | "malloc" : " 14 GB", | 20260508 05:37:24.149 | INFO | "pinned" : " 29 GB" | 20260508 05:37:24.149 | INFO | } | 20260508 05:37:24.149 | INFO | Perf report for the last 5.13843 seconds: | 20260508 05:37:24.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:24.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.150 | INFO | ANALYSIS : 66.7%, 3.0, 103.1ms, 0 | 20260508 05:37:24.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.150 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:37:24.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:24.150 | INFO | Uploaded 36GiB to GPU in 2.52215s (14.2736GiB/s) | 20260508 05:37:24.150 | INFO | Downloaded 34GiB to GPU in 2.3957s (14.1921GiB/s) | 20260508 05:37:29.282 | INFO | Analyzing chunk frames = [5650432,5650944) | 20260508 05:37:29.282 | INFO | MemInfo: { | 20260508 05:37:29.282 | INFO | "gpu" : " 17 GB", | 20260508 05:37:29.282 | INFO | "huge" : "", | 20260508 05:37:29.282 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:29.282 | INFO | "malloc" : " 14 GB", | 20260508 05:37:29.282 | INFO | "pinned" : " 29 GB" | 20260508 05:37:29.282 | INFO | } | 20260508 05:37:29.282 | INFO | Perf report for the last 5.13281 seconds: | 20260508 05:37:29.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:29.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.282 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 05:37:29.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.282 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:37:29.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:29.283 | INFO | Uploaded 36GiB to GPU in 2.31331s (15.5621GiB/s) | 20260508 05:37:29.283 | INFO | Downloaded 34GiB to GPU in 2.4916s (13.6459GiB/s) | 20260508 05:37:34.426 | INFO | Analyzing chunk frames = [5650944,5651456) | 20260508 05:37:34.426 | INFO | MemInfo: { | 20260508 05:37:34.426 | INFO | "gpu" : " 17 GB", | 20260508 05:37:34.426 | INFO | "huge" : "", | 20260508 05:37:34.426 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:34.426 | INFO | "malloc" : " 14 GB", | 20260508 05:37:34.426 | INFO | "pinned" : " 29 GB" | 20260508 05:37:34.426 | INFO | } | 20260508 05:37:34.426 | INFO | Perf report for the last 5.14385 seconds: | 20260508 05:37:34.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:34.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 05:37:34.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:37:34.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:34.426 | INFO | Uploaded 36GiB to GPU in 2.4565s (14.655GiB/s) | 20260508 05:37:34.426 | INFO | Downloaded 34GiB to GPU in 2.4251s (14.0201GiB/s) | 20260508 05:37:39.567 | INFO | Analyzing chunk frames = [5651456,5651968) | 20260508 05:37:39.567 | INFO | MemInfo: { | 20260508 05:37:39.567 | INFO | "gpu" : " 17 GB", | 20260508 05:37:39.567 | INFO | "huge" : "", | 20260508 05:37:39.567 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:39.567 | INFO | "malloc" : " 14 GB", | 20260508 05:37:39.567 | INFO | "pinned" : " 29 GB" | 20260508 05:37:39.568 | INFO | } | 20260508 05:37:39.568 | INFO | Perf report for the last 5.14145 seconds: | 20260508 05:37:39.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:39.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:37:39.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:37:39.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:39.568 | INFO | Uploaded 36GiB to GPU in 2.45736s (14.6499GiB/s) | 20260508 05:37:39.568 | INFO | Downloaded 34GiB to GPU in 2.42293s (14.0326GiB/s) | 20260508 05:37:44.711 | INFO | Analyzing chunk frames = [5651968,5652480) | 20260508 05:37:44.711 | INFO | MemInfo: { | 20260508 05:37:44.711 | INFO | "gpu" : " 17 GB", | 20260508 05:37:44.711 | INFO | "huge" : "", | 20260508 05:37:44.711 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:44.711 | INFO | "malloc" : " 14 GB", | 20260508 05:37:44.711 | INFO | "pinned" : " 29 GB" | 20260508 05:37:44.711 | INFO | } | 20260508 05:37:44.712 | INFO | Perf report for the last 5.14391 seconds: | 20260508 05:37:44.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:44.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:37:44.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 05:37:44.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:44.712 | INFO | Uploaded 36GiB to GPU in 2.48529s (14.4852GiB/s) | 20260508 05:37:44.712 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:37:44.712 | INFO | Total mem usage: 16.7 GiB | 20260508 05:37:44.712 | INFO | Downloaded 34GiB to GPU in 2.44195s (13.9233GiB/s) | 20260508 05:37:44.712 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:37:44.712 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:37:44.712 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:37:44.712 | INFO | High Water Breakdown: | 20260508 05:37:44.712 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:37:44.712 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:37:44.713 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:37:44.713 | INFO | ------------------------------------------ | 20260508 05:37:44.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:37:44.713 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:37:44.713 | INFO | Total mem usage: 26.5 GiB | 20260508 05:37:44.713 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:37:44.713 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:37:44.713 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:37:44.713 | INFO | High Water Breakdown: | 20260508 05:37:44.713 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:37:44.713 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:37:44.713 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:37:44.713 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:37:44.713 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:37:44.713 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:37:44.713 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:37:44.713 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:37:44.713 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:37:44.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:37:44.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:37:44.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:37:44.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:37:44.713 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:37:44.713 | INFO | ------------------------------------------ | 20260508 05:37:44.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:37:44.713 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:37:44.713 | INFO | Total mem usage: 27.8 GiB | 20260508 05:37:44.713 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:37:44.713 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:37:44.713 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:37:44.713 | INFO | High Water Breakdown: | 20260508 05:37:44.713 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:37:44.713 | INFO | ------------------------------------------ | 20260508 05:37:44.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:37:44.713 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:37:44.713 | INFO | Total mem usage: 15.7 GiB | 20260508 05:37:44.713 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:37:44.713 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:37:44.713 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:37:44.713 | INFO | High Water Breakdown: | 20260508 05:37:44.713 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:37:44.713 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:37:44.713 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:37:44.713 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:37:44.713 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:37:44.713 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:37:44.713 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:37:44.713 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:37:44.713 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:37:44.713 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:37:44.713 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:37:44.713 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:37:44.713 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:37:44.713 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:37:44.713 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:37:44.713 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:37:44.713 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:37:44.713 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:37:44.713 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:37:44.713 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:37:44.713 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:37:44.713 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:37:44.713 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:37:44.713 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:37:44.713 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:37:44.713 | INFO | ------------------------------------------ | 20260508 05:37:44.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:37:44.714 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:37:44.714 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:37:44.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:37:44.714 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.27e-01 | 20260508 05:37:44.714 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.05e-01, 1.12e-02 / 5.81e-03 | 20260508 05:37:44.714 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.40e+00, 5.89e-01 / 2.40e-01 | 20260508 05:37:44.714 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.11e-01 / 1.17e-01 | 20260508 05:37:44.714 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.52e+01, 3.83e+00 / 1.25e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.77e+00 / 1.43e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.57e+00, 4.15e-01 / 2.60e-01 | 20260508 05:37:44.714 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.24e+00, 2.57e-01 / 1.36e-01 | 20260508 05:37:44.714 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.58e-01, 5.00e-02 / 2.82e-02 | 20260508 05:37:44.714 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:37:44.714 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:37:44.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:37:44.714 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 24.33%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 2.29%, LockUpload -- 2.43e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 2.60%, LockDownload -- 2.76e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 15.74%, LockCompute -- 1.67e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:37:44.714 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260508 05:37:44.827 | INFO | Starting a round of Baz IO | 20260508 05:37:48.241 | INFO | Populating RTMetrics report | 20260508 05:37:48.241 | INFO | RTMetrics report populated | 20260508 05:37:48.241 | INFO | Opening tmp RTMetrics json file | 20260508 05:37:48.252 | INFO | Serializing RTMetrics to json | 20260508 05:37:48.253 | INFO | Serialization done, swapping tmp file now... | 20260508 05:37:48.254 | INFO | RTMetrics done | 20260508 05:37:48.311 | INFO | Finished a round of Baz IO: Wrote 1.57 GiB in 3.484 seconds | 20260508 05:37:48.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.84% / 0.00% / 59.16% | 20260508 05:37:48.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.451636% / 3.97364e-06% / 8.74201e-05% | 20260508 05:37:48.311 | INFO | Baz IO overall: | 20260508 05:37:48.311 | INFO | BytesWritten : 2516851853538 | 20260508 05:37:48.311 | INFO | NumEvents : 1214164510299 | 20260508 05:37:49.846 | INFO | Analyzing chunk frames = [5652480,5652992) | 20260508 05:37:49.846 | INFO | MemInfo: { | 20260508 05:37:49.846 | INFO | "gpu" : " 17 GB", | 20260508 05:37:49.846 | INFO | "huge" : "", | 20260508 05:37:49.846 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:49.846 | INFO | "malloc" : " 14 GB", | 20260508 05:37:49.846 | INFO | "pinned" : " 29 GB" | 20260508 05:37:49.846 | INFO | } | 20260508 05:37:49.846 | INFO | Perf report for the last 5.13467 seconds: | 20260508 05:37:49.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:49.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:49.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:49.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:49.846 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 05:37:49.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:49.846 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 05:37:49.846 | INFO | PRE_HQ : 66.7%, 1.8, 61.9ms, 0 | 20260508 05:37:49.846 | INFO | BAZWRITER : 67.1%, 2.9, 99.2ms, 0 | 20260508 05:37:49.846 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:37:49.846 | INFO | Uploaded 36GiB to GPU in 2.55011s (14.1171GiB/s) | 20260508 05:37:49.846 | INFO | Downloaded 38GiB to GPU in 2.71113s (14.0163GiB/s) | 20260508 05:37:54.984 | INFO | Analyzing chunk frames = [5652992,5653504) | 20260508 05:37:54.984 | INFO | MemInfo: { | 20260508 05:37:54.984 | INFO | "gpu" : " 17 GB", | 20260508 05:37:54.984 | INFO | "huge" : "", | 20260508 05:37:54.984 | INFO | "huge_shared" : " 29 GB", | 20260508 05:37:54.985 | INFO | "malloc" : " 14 GB", | 20260508 05:37:54.985 | INFO | "pinned" : " 29 GB" | 20260508 05:37:54.985 | INFO | } | 20260508 05:37:54.985 | INFO | Perf report for the last 5.13838 seconds: | 20260508 05:37:54.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:37:54.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:37:54.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:37:54.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:37:54.985 | INFO | Uploaded 36GiB to GPU in 2.49544s (14.4263GiB/s) | 20260508 05:37:54.985 | INFO | Downloaded 34GiB to GPU in 2.44766s (13.8908GiB/s) | 20260508 05:38:00.122 | INFO | Analyzing chunk frames = [5653504,5654016) | 20260508 05:38:00.122 | INFO | MemInfo: { | 20260508 05:38:00.122 | INFO | "gpu" : " 17 GB", | 20260508 05:38:00.122 | INFO | "huge" : "", | 20260508 05:38:00.122 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:00.122 | INFO | "malloc" : " 14 GB", | 20260508 05:38:00.122 | INFO | "pinned" : " 29 GB" | 20260508 05:38:00.122 | INFO | } | 20260508 05:38:00.122 | INFO | Perf report for the last 5.13778 seconds: | 20260508 05:38:00.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:00.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.123 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 05:38:00.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.123 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:38:00.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:00.123 | INFO | Uploaded 36GiB to GPU in 2.5408s (14.1688GiB/s) | 20260508 05:38:00.123 | INFO | Downloaded 34GiB to GPU in 2.42202s (14.0379GiB/s) | 20260508 05:38:05.264 | INFO | Analyzing chunk frames = [5654016,5654528) | 20260508 05:38:05.264 | INFO | MemInfo: { | 20260508 05:38:05.264 | INFO | "gpu" : " 17 GB", | 20260508 05:38:05.264 | INFO | "huge" : "", | 20260508 05:38:05.264 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:05.264 | INFO | "malloc" : " 14 GB", | 20260508 05:38:05.264 | INFO | "pinned" : " 29 GB" | 20260508 05:38:05.264 | INFO | } | 20260508 05:38:05.264 | INFO | Perf report for the last 5.14155 seconds: | 20260508 05:38:05.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:05.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.264 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 05:38:05.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.264 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:38:05.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:05.265 | INFO | Uploaded 36GiB to GPU in 2.53416s (14.2059GiB/s) | 20260508 05:38:05.265 | INFO | Downloaded 34GiB to GPU in 2.40246s (14.1521GiB/s) | 20260508 05:38:10.404 | INFO | Analyzing chunk frames = [5654528,5655040) | 20260508 05:38:10.404 | INFO | MemInfo: { | 20260508 05:38:10.404 | INFO | "gpu" : " 17 GB", | 20260508 05:38:10.404 | INFO | "huge" : "", | 20260508 05:38:10.404 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:10.404 | INFO | "malloc" : " 14 GB", | 20260508 05:38:10.404 | INFO | "pinned" : " 29 GB" | 20260508 05:38:10.404 | INFO | } | 20260508 05:38:10.404 | INFO | Perf report for the last 5.14001 seconds: | 20260508 05:38:10.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:10.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.404 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 05:38:10.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.404 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:38:10.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:10.405 | INFO | Uploaded 36GiB to GPU in 2.34553s (15.3484GiB/s) | 20260508 05:38:10.405 | INFO | Downloaded 34GiB to GPU in 2.55653s (13.2993GiB/s) | 20260508 05:38:15.538 | INFO | Analyzing chunk frames = [5655040,5655552) | 20260508 05:38:15.538 | INFO | MemInfo: { | 20260508 05:38:15.538 | INFO | "gpu" : " 17 GB", | 20260508 05:38:15.538 | INFO | "huge" : "", | 20260508 05:38:15.538 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:15.538 | INFO | "malloc" : " 14 GB", | 20260508 05:38:15.538 | INFO | "pinned" : " 29 GB" | 20260508 05:38:15.538 | INFO | } | 20260508 05:38:15.538 | INFO | Perf report for the last 5.1342 seconds: | 20260508 05:38:15.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:15.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.538 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:38:15.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.538 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:38:15.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:15.539 | INFO | Uploaded 36GiB to GPU in 2.41489s (14.9075GiB/s) | 20260508 05:38:15.539 | INFO | Downloaded 34GiB to GPU in 2.43559s (13.9597GiB/s) | 20260508 05:38:20.681 | INFO | Analyzing chunk frames = [5655552,5656064) | 20260508 05:38:20.681 | INFO | MemInfo: { | 20260508 05:38:20.681 | INFO | "gpu" : " 17 GB", | 20260508 05:38:20.681 | INFO | "huge" : "", | 20260508 05:38:20.681 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:20.681 | INFO | "malloc" : " 14 GB", | 20260508 05:38:20.681 | INFO | "pinned" : " 29 GB" | 20260508 05:38:20.681 | INFO | } | 20260508 05:38:20.682 | INFO | Perf report for the last 5.14333 seconds: | 20260508 05:38:20.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:20.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 05:38:20.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 05:38:20.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:20.682 | INFO | Uploaded 36GiB to GPU in 2.3982s (15.0113GiB/s) | 20260508 05:38:20.682 | INFO | Downloaded 34GiB to GPU in 2.43593s (13.9577GiB/s) | 20260508 05:38:25.822 | INFO | Analyzing chunk frames = [5656064,5656576) | 20260508 05:38:25.822 | INFO | MemInfo: { | 20260508 05:38:25.822 | INFO | "gpu" : " 17 GB", | 20260508 05:38:25.822 | INFO | "huge" : "", | 20260508 05:38:25.822 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:25.822 | INFO | "malloc" : " 14 GB", | 20260508 05:38:25.822 | INFO | "pinned" : " 29 GB" | 20260508 05:38:25.822 | INFO | } | 20260508 05:38:25.822 | INFO | Perf report for the last 5.14056 seconds: | 20260508 05:38:25.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:25.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:38:25.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260508 05:38:25.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:25.822 | INFO | Uploaded 36GiB to GPU in 2.44934s (14.6979GiB/s) | 20260508 05:38:25.823 | INFO | Downloaded 34GiB to GPU in 2.44916s (13.8823GiB/s) | 20260508 05:38:25.920 | INFO | Starting a round of Baz IO | 20260508 05:38:29.354 | INFO | Populating RTMetrics report | 20260508 05:38:29.354 | INFO | RTMetrics report populated | 20260508 05:38:29.354 | INFO | Opening tmp RTMetrics json file | 20260508 05:38:29.366 | INFO | Serializing RTMetrics to json | 20260508 05:38:29.368 | INFO | Serialization done, swapping tmp file now... | 20260508 05:38:29.368 | INFO | RTMetrics done | 20260508 05:38:29.424 | INFO | Finished a round of Baz IO: Wrote 1.57 GiB in 3.504 seconds | 20260508 05:38:29.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.83% / 0.00% / 59.16% | 20260508 05:38:29.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.451648% / 7.94729e-06% / 6.75519e-05% | 20260508 05:38:29.424 | INFO | Baz IO overall: | 20260508 05:38:29.424 | INFO | BytesWritten : 2518537660773 | 20260508 05:38:29.424 | INFO | NumEvents : 1214968519758 | 20260508 05:38:30.959 | INFO | Analyzing chunk frames = [5656576,5657088) | 20260508 05:38:30.959 | INFO | MemInfo: { | 20260508 05:38:30.959 | INFO | "gpu" : " 17 GB", | 20260508 05:38:30.959 | INFO | "huge" : "", | 20260508 05:38:30.959 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:30.959 | INFO | "malloc" : " 14 GB", | 20260508 05:38:30.959 | INFO | "pinned" : " 29 GB" | 20260508 05:38:30.959 | INFO | } | 20260508 05:38:30.960 | INFO | Perf report for the last 5.13736 seconds: | 20260508 05:38:30.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:30.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:30.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:30.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:30.960 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 05:38:30.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:30.960 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:38:30.960 | INFO | PRE_HQ : 66.0%, 1.6, 55.3ms, 0 | 20260508 05:38:30.960 | INFO | BAZWRITER : 67.6%, 2.7, 93.0ms, 0 | 20260508 05:38:30.960 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:38:30.960 | INFO | Uploaded 36GiB to GPU in 2.5481s (14.1282GiB/s) | 20260508 05:38:30.960 | INFO | Downloaded 38GiB to GPU in 2.71972s (13.972GiB/s) | 20260508 05:38:36.095 | INFO | Analyzing chunk frames = [5657088,5657600) | 20260508 05:38:36.095 | INFO | MemInfo: { | 20260508 05:38:36.095 | INFO | "gpu" : " 17 GB", | 20260508 05:38:36.095 | INFO | "huge" : "", | 20260508 05:38:36.095 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:36.095 | INFO | "malloc" : " 14 GB", | 20260508 05:38:36.095 | INFO | "pinned" : " 29 GB" | 20260508 05:38:36.095 | INFO | } | 20260508 05:38:36.095 | INFO | Perf report for the last 5.13574 seconds: | 20260508 05:38:36.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:36.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.095 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:38:36.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.095 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:38:36.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:36.096 | INFO | Uploaded 36GiB to GPU in 2.46221s (14.621GiB/s) | 20260508 05:38:36.096 | INFO | Downloaded 34GiB to GPU in 2.45278s (13.8618GiB/s) | 20260508 05:38:41.233 | INFO | Analyzing chunk frames = [5657600,5658112) | 20260508 05:38:41.233 | INFO | MemInfo: { | 20260508 05:38:41.233 | INFO | "gpu" : " 17 GB", | 20260508 05:38:41.233 | INFO | "huge" : "", | 20260508 05:38:41.233 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:41.234 | INFO | "malloc" : " 14 GB", | 20260508 05:38:41.234 | INFO | "pinned" : " 29 GB" | 20260508 05:38:41.234 | INFO | } | 20260508 05:38:41.234 | INFO | Perf report for the last 5.13838 seconds: | 20260508 05:38:41.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:41.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:38:41.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:38:41.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:41.234 | INFO | Uploaded 36GiB to GPU in 2.49293s (14.4409GiB/s) | 20260508 05:38:41.234 | INFO | Downloaded 34GiB to GPU in 2.4192s (14.0543GiB/s) | 20260508 05:38:46.379 | INFO | Analyzing chunk frames = [5658112,5658624) | 20260508 05:38:46.379 | INFO | MemInfo: { | 20260508 05:38:46.379 | INFO | "gpu" : " 17 GB", | 20260508 05:38:46.379 | INFO | "huge" : "", | 20260508 05:38:46.379 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:46.379 | INFO | "malloc" : " 14 GB", | 20260508 05:38:46.379 | INFO | "pinned" : " 29 GB" | 20260508 05:38:46.379 | INFO | } | 20260508 05:38:46.379 | INFO | Perf report for the last 5.1458 seconds: | 20260508 05:38:46.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:46.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 05:38:46.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:38:46.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:46.380 | INFO | Uploaded 36GiB to GPU in 2.52917s (14.2339GiB/s) | 20260508 05:38:46.380 | INFO | Downloaded 34GiB to GPU in 2.41741s (14.0646GiB/s) | 20260508 05:38:51.522 | INFO | Analyzing chunk frames = [5658624,5659136) | 20260508 05:38:51.522 | INFO | MemInfo: { | 20260508 05:38:51.522 | INFO | "gpu" : " 17 GB", | 20260508 05:38:51.522 | INFO | "huge" : "", | 20260508 05:38:51.522 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:51.522 | INFO | "malloc" : " 14 GB", | 20260508 05:38:51.522 | INFO | "pinned" : " 29 GB" | 20260508 05:38:51.522 | INFO | } | 20260508 05:38:51.523 | INFO | Perf report for the last 5.14308 seconds: | 20260508 05:38:51.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:51.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:38:51.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:38:51.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:51.523 | INFO | Uploaded 36GiB to GPU in 2.35187s (15.3069GiB/s) | 20260508 05:38:51.523 | INFO | Downloaded 34GiB to GPU in 2.4992s (13.6043GiB/s) | 20260508 05:38:56.659 | INFO | Analyzing chunk frames = [5659136,5659648) | 20260508 05:38:56.659 | INFO | MemInfo: { | 20260508 05:38:56.659 | INFO | "gpu" : " 17 GB", | 20260508 05:38:56.660 | INFO | "huge" : "", | 20260508 05:38:56.660 | INFO | "huge_shared" : " 29 GB", | 20260508 05:38:56.660 | INFO | "malloc" : " 14 GB", | 20260508 05:38:56.660 | INFO | "pinned" : " 29 GB" | 20260508 05:38:56.660 | INFO | } | 20260508 05:38:56.660 | INFO | Perf report for the last 5.13738 seconds: | 20260508 05:38:56.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:38:56.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:38:56.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:38:56.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:38:56.660 | INFO | Uploaded 36GiB to GPU in 2.4609s (14.6288GiB/s) | 20260508 05:38:56.661 | INFO | Downloaded 34GiB to GPU in 2.43227s (13.9787GiB/s) | 20260508 05:39:01.791 | INFO | Analyzing chunk frames = [5659648,5660160) | 20260508 05:39:01.791 | INFO | MemInfo: { | 20260508 05:39:01.791 | INFO | "gpu" : " 17 GB", | 20260508 05:39:01.791 | INFO | "huge" : "", | 20260508 05:39:01.791 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:01.791 | INFO | "malloc" : " 14 GB", | 20260508 05:39:01.791 | INFO | "pinned" : " 29 GB" | 20260508 05:39:01.791 | INFO | } | 20260508 05:39:01.792 | INFO | Perf report for the last 5.13146 seconds: | 20260508 05:39:01.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:01.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:39:01.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 05:39:01.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:01.792 | INFO | Uploaded 36GiB to GPU in 2.45051s (14.6908GiB/s) | 20260508 05:39:01.792 | INFO | Downloaded 34GiB to GPU in 2.43916s (13.9392GiB/s) | 20260508 05:39:06.935 | INFO | Analyzing chunk frames = [5660160,5660672) | 20260508 05:39:06.935 | INFO | MemInfo: { | 20260508 05:39:06.935 | INFO | "gpu" : " 17 GB", | 20260508 05:39:06.935 | INFO | "huge" : "", | 20260508 05:39:06.935 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:06.935 | INFO | "malloc" : " 14 GB", | 20260508 05:39:06.935 | INFO | "pinned" : " 29 GB" | 20260508 05:39:06.935 | INFO | } | 20260508 05:39:06.935 | INFO | Perf report for the last 5.14395 seconds: | 20260508 05:39:06.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:06.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.935 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 05:39:06.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.936 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 05:39:06.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:06.936 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:39:06.936 | INFO | Total mem usage: 16.7 GiB | 20260508 05:39:06.936 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:39:06.936 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:39:06.936 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:39:06.936 | INFO | High Water Breakdown: | 20260508 05:39:06.936 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:39:06.936 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:39:06.936 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:39:06.936 | INFO | ------------------------------------------ | 20260508 05:39:06.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:39:06.936 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:39:06.936 | INFO | Total mem usage: 26.5 GiB | 20260508 05:39:06.936 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:39:06.936 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:39:06.936 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:39:06.936 | INFO | High Water Breakdown: | 20260508 05:39:06.936 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:39:06.936 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:39:06.936 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:39:06.936 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:39:06.936 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:39:06.936 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:39:06.936 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:39:06.936 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:39:06.936 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:39:06.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:39:06.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:39:06.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:39:06.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:39:06.936 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:39:06.936 | INFO | ------------------------------------------ | 20260508 05:39:06.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:39:06.936 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:39:06.936 | INFO | Total mem usage: 27.8 GiB | 20260508 05:39:06.936 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:39:06.936 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:39:06.936 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:39:06.936 | INFO | High Water Breakdown: | 20260508 05:39:06.936 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:39:06.936 | INFO | ------------------------------------------ | 20260508 05:39:06.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:39:06.936 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:39:06.936 | INFO | Total mem usage: 15.7 GiB | 20260508 05:39:06.936 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:39:06.936 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:39:06.936 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:39:06.936 | INFO | High Water Breakdown: | 20260508 05:39:06.936 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:39:06.936 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:39:06.936 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:39:06.936 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:39:06.936 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:39:06.936 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:39:06.936 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:39:06.936 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:39:06.936 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:39:06.936 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:39:06.936 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:39:06.936 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:39:06.936 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:39:06.936 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:39:06.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:39:06.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:39:06.936 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:39:06.936 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:39:06.936 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:39:06.936 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:39:06.936 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:39:06.936 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:39:06.937 | INFO | Uploaded 36GiB to GPU in 2.46525s (14.603GiB/s) | 20260508 05:39:06.937 | INFO | Downloaded 34GiB to GPU in 2.44144s (13.9262GiB/s) | 20260508 05:39:06.936 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:39:06.937 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:39:06.937 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:39:06.937 | INFO | ------------------------------------------ | 20260508 05:39:06.937 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:39:06.937 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:39:06.937 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:39:06.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:39:06.937 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.40e+01 / 1.07e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.06e-01, 1.19e-02 / 7.27e-03 | 20260508 05:39:06.937 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.39e+00, 5.89e-01 / 2.73e-01 | 20260508 05:39:06.937 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.11e-01 / 1.29e-01 | 20260508 05:39:06.937 | INFO | PerfReport: 15.88%, DME -- 4.17e+01 / 4.50e+01, 3.64e+00 / 1.45e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.86e+00 / 1.51e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.16e-01 / 2.65e-01 | 20260508 05:39:06.937 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.24e+00, 2.56e-01 / 1.55e-01 | 20260508 05:39:06.937 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.19e-01 / 3.58e-01, 4.99e-02 / 2.94e-02 | 20260508 05:39:06.937 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:39:06.937 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:39:06.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:39:06.937 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 24.23%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 24.35%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 30.91%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 2.76%, LockDownload -- 2.92e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:39:06.937 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 05:39:07.049 | INFO | Starting a round of Baz IO | 20260508 05:39:10.480 | INFO | Populating RTMetrics report | 20260508 05:39:10.480 | INFO | RTMetrics report populated | 20260508 05:39:10.480 | INFO | Opening tmp RTMetrics json file | 20260508 05:39:10.481 | INFO | Serializing RTMetrics to json | 20260508 05:39:10.482 | INFO | Serialization done, swapping tmp file now... | 20260508 05:39:10.482 | INFO | RTMetrics done | 20260508 05:39:10.550 | INFO | Finished a round of Baz IO: Wrote 1.57 GiB in 3.501 seconds | 20260508 05:39:10.550 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.83% / 0.00% / 59.17% | 20260508 05:39:10.550 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.450011% / 0% / 5.5631e-05% | 20260508 05:39:10.550 | INFO | Baz IO overall: | 20260508 05:39:10.550 | INFO | BytesWritten : 2520223126867 | 20260508 05:39:10.550 | INFO | NumEvents : 1215772527812 | 20260508 05:39:12.075 | INFO | Analyzing chunk frames = [5660672,5661184) | 20260508 05:39:12.075 | INFO | MemInfo: { | 20260508 05:39:12.075 | INFO | "gpu" : " 17 GB", | 20260508 05:39:12.075 | INFO | "huge" : "", | 20260508 05:39:12.075 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:12.075 | INFO | "malloc" : " 14 GB", | 20260508 05:39:12.075 | INFO | "pinned" : " 29 GB" | 20260508 05:39:12.075 | INFO | } | 20260508 05:39:12.075 | INFO | Perf report for the last 5.1392 seconds: | 20260508 05:39:12.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:12.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:12.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:12.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:12.075 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260508 05:39:12.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:12.075 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 05:39:12.075 | INFO | PRE_HQ : 67.0%, 1.8, 61.9ms, 0 | 20260508 05:39:12.075 | INFO | BAZWRITER : 67.6%, 2.9, 100.1ms, 0 | 20260508 05:39:12.075 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 05:39:12.075 | INFO | Uploaded 36GiB to GPU in 2.55495s (14.0903GiB/s) | 20260508 05:39:12.075 | INFO | Downloaded 38GiB to GPU in 2.7042s (14.0522GiB/s) | 20260508 05:39:17.215 | INFO | Analyzing chunk frames = [5661184,5661696) | 20260508 05:39:17.215 | INFO | MemInfo: { | 20260508 05:39:17.215 | INFO | "gpu" : " 17 GB", | 20260508 05:39:17.215 | INFO | "huge" : "", | 20260508 05:39:17.215 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:17.215 | INFO | "malloc" : " 14 GB", | 20260508 05:39:17.215 | INFO | "pinned" : " 29 GB" | 20260508 05:39:17.215 | INFO | } | 20260508 05:39:17.215 | INFO | Perf report for the last 5.14039 seconds: | 20260508 05:39:17.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:17.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:39:17.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:39:17.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:17.215 | INFO | Uploaded 36GiB to GPU in 2.487s (14.4753GiB/s) | 20260508 05:39:17.215 | INFO | Downloaded 34GiB to GPU in 2.44994s (13.8779GiB/s) | 20260508 05:39:22.360 | INFO | Analyzing chunk frames = [5661696,5662208) | 20260508 05:39:22.360 | INFO | MemInfo: { | 20260508 05:39:22.360 | INFO | "gpu" : " 17 GB", | 20260508 05:39:22.360 | INFO | "huge" : "", | 20260508 05:39:22.360 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:22.360 | INFO | "malloc" : " 14 GB", | 20260508 05:39:22.361 | INFO | "pinned" : " 29 GB" | 20260508 05:39:22.361 | INFO | } | 20260508 05:39:22.361 | INFO | Perf report for the last 5.14554 seconds: | 20260508 05:39:22.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:22.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 05:39:22.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:39:22.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:22.361 | INFO | Uploaded 36GiB to GPU in 2.52869s (14.2366GiB/s) | 20260508 05:39:22.361 | INFO | Downloaded 34GiB to GPU in 2.4081s (14.119GiB/s) | 20260508 05:39:27.491 | INFO | Analyzing chunk frames = [5662208,5662720) | 20260508 05:39:27.491 | INFO | MemInfo: { | 20260508 05:39:27.491 | INFO | "gpu" : " 17 GB", | 20260508 05:39:27.491 | INFO | "huge" : "", | 20260508 05:39:27.491 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:27.491 | INFO | "malloc" : " 14 GB", | 20260508 05:39:27.491 | INFO | "pinned" : " 29 GB" | 20260508 05:39:27.491 | INFO | } | 20260508 05:39:27.491 | INFO | Perf report for the last 5.13044 seconds: | 20260508 05:39:27.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:27.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.491 | INFO | ANALYSIS : 66.9%, 2.9, 103.2ms, 0 | 20260508 05:39:27.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.491 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:39:27.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:27.492 | INFO | Uploaded 36GiB to GPU in 2.54234s (14.1602GiB/s) | 20260508 05:39:27.492 | INFO | Downloaded 34GiB to GPU in 2.41533s (14.0768GiB/s) | 20260508 05:39:32.632 | INFO | Analyzing chunk frames = [5662720,5663232) | 20260508 05:39:32.632 | INFO | MemInfo: { | 20260508 05:39:32.632 | INFO | "gpu" : " 17 GB", | 20260508 05:39:32.632 | INFO | "huge" : "", | 20260508 05:39:32.633 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:32.633 | INFO | "malloc" : " 14 GB", | 20260508 05:39:32.633 | INFO | "pinned" : " 29 GB" | 20260508 05:39:32.633 | INFO | } | 20260508 05:39:32.633 | INFO | Perf report for the last 5.14158 seconds: | 20260508 05:39:32.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:32.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 05:39:32.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:39:32.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:32.633 | INFO | Uploaded 36GiB to GPU in 2.3349s (15.4182GiB/s) | 20260508 05:39:32.633 | INFO | Downloaded 34GiB to GPU in 2.5669s (13.2456GiB/s) | 20260508 05:39:37.772 | INFO | Analyzing chunk frames = [5663232,5663744) | 20260508 05:39:37.772 | INFO | MemInfo: { | 20260508 05:39:37.772 | INFO | "gpu" : " 17 GB", | 20260508 05:39:37.772 | INFO | "huge" : "", | 20260508 05:39:37.772 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:37.772 | INFO | "malloc" : " 14 GB", | 20260508 05:39:37.772 | INFO | "pinned" : " 29 GB" | 20260508 05:39:37.772 | INFO | } | 20260508 05:39:37.772 | INFO | Perf report for the last 5.13941 seconds: | 20260508 05:39:37.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:37.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:39:37.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:39:37.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:37.772 | INFO | Uploaded 36GiB to GPU in 2.42448s (14.8486GiB/s) | 20260508 05:39:37.772 | INFO | Downloaded 34GiB to GPU in 2.43988s (13.9351GiB/s) | 20260508 05:39:42.911 | INFO | Analyzing chunk frames = [5663744,5664256) | 20260508 05:39:42.911 | INFO | MemInfo: { | 20260508 05:39:42.911 | INFO | "gpu" : " 17 GB", | 20260508 05:39:42.911 | INFO | "huge" : "", | 20260508 05:39:42.912 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:42.912 | INFO | "malloc" : " 14 GB", | 20260508 05:39:42.912 | INFO | "pinned" : " 29 GB" | 20260508 05:39:42.912 | INFO | } | 20260508 05:39:42.912 | INFO | Perf report for the last 5.13974 seconds: | 20260508 05:39:42.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:42.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:39:42.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 05:39:42.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:42.912 | INFO | Uploaded 36GiB to GPU in 2.42029s (14.8743GiB/s) | 20260508 05:39:42.912 | INFO | Downloaded 34GiB to GPU in 2.43366s (13.9707GiB/s) | 20260508 05:39:48.045 | INFO | Analyzing chunk frames = [5664256,5664768) | 20260508 05:39:48.045 | INFO | MemInfo: { | 20260508 05:39:48.045 | INFO | "gpu" : " 17 GB", | 20260508 05:39:48.045 | INFO | "huge" : "", | 20260508 05:39:48.045 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:48.045 | INFO | "malloc" : " 14 GB", | 20260508 05:39:48.045 | INFO | "pinned" : " 29 GB" | 20260508 05:39:48.045 | INFO | } | 20260508 05:39:48.045 | INFO | Perf report for the last 5.13339 seconds: | 20260508 05:39:48.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:48.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.045 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:39:48.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.045 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 05:39:48.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:48.046 | INFO | Uploaded 36GiB to GPU in 2.4679s (14.5873GiB/s) | 20260508 05:39:48.046 | INFO | Downloaded 34GiB to GPU in 2.4524s (13.864GiB/s) | 20260508 05:39:48.159 | INFO | Starting a round of Baz IO | 20260508 05:39:51.578 | INFO | Populating RTMetrics report | 20260508 05:39:51.579 | INFO | RTMetrics report populated | 20260508 05:39:51.579 | INFO | Opening tmp RTMetrics json file | 20260508 05:39:51.579 | INFO | Serializing RTMetrics to json | 20260508 05:39:51.580 | INFO | Serialization done, swapping tmp file now... | 20260508 05:39:51.581 | INFO | RTMetrics done | 20260508 05:39:51.651 | INFO | Finished a round of Baz IO: Wrote 1.57 GiB in 3.492 seconds | 20260508 05:39:51.651 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.83% / 0.00% / 59.17% | 20260508 05:39:51.651 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.449272% / 3.97364e-06% / 5.16574e-05% | 20260508 05:39:51.651 | INFO | Baz IO overall: | 20260508 05:39:51.651 | INFO | BytesWritten : 2521908679954 | 20260508 05:39:51.652 | INFO | NumEvents : 1216576449195 | 20260508 05:39:53.192 | INFO | Analyzing chunk frames = [5664768,5665280) | 20260508 05:39:53.192 | INFO | MemInfo: { | 20260508 05:39:53.192 | INFO | "gpu" : " 17 GB", | 20260508 05:39:53.192 | INFO | "huge" : "", | 20260508 05:39:53.192 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:53.192 | INFO | "malloc" : " 14 GB", | 20260508 05:39:53.192 | INFO | "pinned" : " 29 GB" | 20260508 05:39:53.192 | INFO | } | 20260508 05:39:53.192 | INFO | Perf report for the last 5.14722 seconds: | 20260508 05:39:53.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:53.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:53.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:53.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:53.193 | INFO | ANALYSIS : 68.3%, 3.0, 106.1ms, 0 | 20260508 05:39:53.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:53.193 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 05:39:53.193 | INFO | PRE_HQ : 66.6%, 1.8, 61.8ms, 0 | 20260508 05:39:53.193 | INFO | BAZWRITER : 67.2%, 2.9, 98.9ms, 0 | 20260508 05:39:53.193 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 05:39:53.193 | INFO | Uploaded 36GiB to GPU in 2.5165s (14.3056GiB/s) | 20260508 05:39:53.193 | INFO | Downloaded 38GiB to GPU in 2.68492s (14.1531GiB/s) | 20260508 05:39:58.331 | INFO | Analyzing chunk frames = [5665280,5665792) | 20260508 05:39:58.331 | INFO | MemInfo: { | 20260508 05:39:58.331 | INFO | "gpu" : " 17 GB", | 20260508 05:39:58.331 | INFO | "huge" : "", | 20260508 05:39:58.331 | INFO | "huge_shared" : " 29 GB", | 20260508 05:39:58.331 | INFO | "malloc" : " 14 GB", | 20260508 05:39:58.331 | INFO | "pinned" : " 29 GB" | 20260508 05:39:58.331 | INFO | } | 20260508 05:39:58.331 | INFO | Perf report for the last 5.13887 seconds: | 20260508 05:39:58.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:39:58.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.331 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:39:58.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.331 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:39:58.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:39:58.332 | INFO | Uploaded 36GiB to GPU in 2.50824s (14.3527GiB/s) | 20260508 05:39:58.332 | INFO | Downloaded 34GiB to GPU in 2.44795s (13.8892GiB/s) | 20260508 05:40:03.472 | INFO | Analyzing chunk frames = [5665792,5666304) | 20260508 05:40:03.472 | INFO | MemInfo: { | 20260508 05:40:03.472 | INFO | "gpu" : " 17 GB", | 20260508 05:40:03.472 | INFO | "huge" : "", | 20260508 05:40:03.472 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:03.472 | INFO | "malloc" : " 14 GB", | 20260508 05:40:03.472 | INFO | "pinned" : " 29 GB" | 20260508 05:40:03.472 | INFO | } | 20260508 05:40:03.472 | INFO | Perf report for the last 5.14113 seconds: | 20260508 05:40:03.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:03.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.473 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:40:03.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.473 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:40:03.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:03.473 | INFO | Uploaded 36GiB to GPU in 2.54807s (14.1283GiB/s) | 20260508 05:40:03.473 | INFO | Downloaded 34GiB to GPU in 2.40898s (14.1138GiB/s) | 20260508 05:40:08.606 | INFO | Analyzing chunk frames = [5666304,5666816) | 20260508 05:40:08.606 | INFO | MemInfo: { | 20260508 05:40:08.606 | INFO | "gpu" : " 17 GB", | 20260508 05:40:08.606 | INFO | "huge" : "", | 20260508 05:40:08.606 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:08.606 | INFO | "malloc" : " 14 GB", | 20260508 05:40:08.606 | INFO | "pinned" : " 29 GB" | 20260508 05:40:08.607 | INFO | } | 20260508 05:40:08.607 | INFO | Perf report for the last 5.13408 seconds: | 20260508 05:40:08.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:08.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | ANALYSIS : 66.9%, 3.0, 103.4ms, 0 | 20260508 05:40:08.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:40:08.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:08.607 | INFO | Uploaded 36GiB to GPU in 2.55427s (14.094GiB/s) | 20260508 05:40:08.607 | INFO | Downloaded 34GiB to GPU in 2.38449s (14.2588GiB/s) | 20260508 05:40:13.749 | INFO | Analyzing chunk frames = [5666816,5667328) | 20260508 05:40:13.749 | INFO | MemInfo: { | 20260508 05:40:13.749 | INFO | "gpu" : " 17 GB", | 20260508 05:40:13.749 | INFO | "huge" : "", | 20260508 05:40:13.749 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:13.749 | INFO | "malloc" : " 14 GB", | 20260508 05:40:13.749 | INFO | "pinned" : " 29 GB" | 20260508 05:40:13.749 | INFO | } | 20260508 05:40:13.749 | INFO | Perf report for the last 5.14243 seconds: | 20260508 05:40:13.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:13.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.749 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 05:40:13.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.749 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:40:13.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:13.750 | INFO | Uploaded 36GiB to GPU in 2.41914s (14.8813GiB/s) | 20260508 05:40:13.750 | INFO | Downloaded 34GiB to GPU in 2.51301s (13.5296GiB/s) | 20260508 05:40:18.884 | INFO | Analyzing chunk frames = [5667328,5667840) | 20260508 05:40:18.884 | INFO | MemInfo: { | 20260508 05:40:18.884 | INFO | "gpu" : " 17 GB", | 20260508 05:40:18.884 | INFO | "huge" : "", | 20260508 05:40:18.884 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:18.884 | INFO | "malloc" : " 14 GB", | 20260508 05:40:18.884 | INFO | "pinned" : " 29 GB" | 20260508 05:40:18.884 | INFO | } | 20260508 05:40:18.884 | INFO | Perf report for the last 5.13532 seconds: | 20260508 05:40:18.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:18.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.884 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:40:18.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.884 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:40:18.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:18.885 | INFO | Uploaded 36GiB to GPU in 2.4277s (14.8288GiB/s) | 20260508 05:40:18.885 | INFO | Downloaded 34GiB to GPU in 2.42467s (14.0225GiB/s) | 20260508 05:40:24.024 | INFO | Analyzing chunk frames = [5667840,5668352) | 20260508 05:40:24.024 | INFO | MemInfo: { | 20260508 05:40:24.024 | INFO | "gpu" : " 17 GB", | 20260508 05:40:24.024 | INFO | "huge" : "", | 20260508 05:40:24.024 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:24.024 | INFO | "malloc" : " 14 GB", | 20260508 05:40:24.024 | INFO | "pinned" : " 29 GB" | 20260508 05:40:24.025 | INFO | } | 20260508 05:40:24.025 | INFO | Perf report for the last 5.14021 seconds: | 20260508 05:40:24.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:24.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:40:24.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 05:40:24.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:24.025 | INFO | Uploaded 36GiB to GPU in 2.43318s (14.7955GiB/s) | 20260508 05:40:24.025 | INFO | Downloaded 34GiB to GPU in 2.43261s (13.9767GiB/s) | 20260508 05:40:29.167 | INFO | Analyzing chunk frames = [5668352,5668864) | 20260508 05:40:29.167 | INFO | MemInfo: { | 20260508 05:40:29.167 | INFO | "gpu" : " 17 GB", | 20260508 05:40:29.167 | INFO | "huge" : "", | 20260508 05:40:29.167 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:29.167 | INFO | "malloc" : " 14 GB", | 20260508 05:40:29.168 | INFO | "pinned" : " 29 GB" | 20260508 05:40:29.168 | INFO | } | 20260508 05:40:29.168 | INFO | Perf report for the last 5.14306 seconds: | 20260508 05:40:29.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:29.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 05:40:29.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260508 05:40:29.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:29.168 | INFO | Uploaded 36GiB to GPU in 2.462s (14.6222GiB/s) | 20260508 05:40:29.168 | INFO | Downloaded 34GiB to GPU in 2.44233s (13.9211GiB/s) | 20260508 05:40:29.168 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:40:29.168 | INFO | Total mem usage: 16.7 GiB | 20260508 05:40:29.168 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:40:29.168 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:40:29.168 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:40:29.168 | INFO | High Water Breakdown: | 20260508 05:40:29.168 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:40:29.168 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:40:29.168 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:40:29.168 | INFO | ------------------------------------------ | 20260508 05:40:29.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:40:29.168 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:40:29.168 | INFO | Total mem usage: 26.5 GiB | 20260508 05:40:29.169 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:40:29.169 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:40:29.169 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:40:29.169 | INFO | High Water Breakdown: | 20260508 05:40:29.169 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:40:29.169 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:40:29.169 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:40:29.169 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:40:29.169 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:40:29.169 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:40:29.169 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:40:29.169 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:40:29.169 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:40:29.169 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:40:29.169 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:40:29.169 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:40:29.169 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:40:29.169 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:40:29.169 | INFO | ------------------------------------------ | 20260508 05:40:29.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:40:29.169 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:40:29.169 | INFO | Total mem usage: 27.8 GiB | 20260508 05:40:29.169 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:40:29.169 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:40:29.169 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:40:29.169 | INFO | High Water Breakdown: | 20260508 05:40:29.169 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:40:29.169 | INFO | ------------------------------------------ | 20260508 05:40:29.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:40:29.169 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:40:29.169 | INFO | Total mem usage: 15.7 GiB | 20260508 05:40:29.169 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:40:29.169 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:40:29.169 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:40:29.169 | INFO | High Water Breakdown: | 20260508 05:40:29.169 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:40:29.169 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:40:29.169 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:40:29.169 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:40:29.169 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:40:29.169 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:40:29.169 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:40:29.169 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:40:29.169 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:40:29.169 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:40:29.169 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:40:29.169 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:40:29.169 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:40:29.169 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:40:29.169 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:40:29.169 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:40:29.169 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:40:29.169 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:40:29.169 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:40:29.169 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:40:29.169 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:40:29.169 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:40:29.169 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:40:29.169 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:40:29.169 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:40:29.169 | INFO | ------------------------------------------ | 20260508 05:40:29.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:40:29.169 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:40:29.169 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:40:29.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:40:29.169 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.37e+01 / 1.23e+00 | 20260508 05:40:29.169 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.08e-01, 1.20e-02 / 6.72e-03 | 20260508 05:40:29.169 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.40e+00, 5.88e-01 / 2.55e-01 | 20260508 05:40:29.169 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.28e-01 | 20260508 05:40:29.169 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.53e+01, 3.87e+00 / 1.50e+00 | 20260508 05:40:29.169 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.83e+00 / 1.31e+00 | 20260508 05:40:29.169 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.22e-01 / 2.37e-01 | 20260508 05:40:29.169 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.27e+00, 2.60e-01 / 1.48e-01 | 20260508 05:40:29.169 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.55e-01, 5.03e-02 / 2.69e-02 | 20260508 05:40:29.169 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:40:29.169 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:40:29.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:40:29.170 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 24.30%, Upload -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 30.85%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 15.70%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:40:29.170 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260508 05:40:29.282 | INFO | Starting a round of Baz IO | 20260508 05:40:32.702 | INFO | Populating RTMetrics report | 20260508 05:40:32.703 | INFO | RTMetrics report populated | 20260508 05:40:32.703 | INFO | Opening tmp RTMetrics json file | 20260508 05:40:32.703 | INFO | Serializing RTMetrics to json | 20260508 05:40:32.705 | INFO | Serialization done, swapping tmp file now... | 20260508 05:40:32.705 | INFO | RTMetrics done | 20260508 05:40:32.776 | INFO | Finished a round of Baz IO: Wrote 1.57 GiB in 3.494 seconds | 20260508 05:40:32.776 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.83% / 0.00% / 59.17% | 20260508 05:40:32.776 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.449316% / 0% / 4.76837e-05% | 20260508 05:40:32.777 | INFO | Baz IO overall: | 20260508 05:40:32.777 | INFO | BytesWritten : 2523594258660 | 20260508 05:40:32.777 | INFO | NumEvents : 1217380308979 | 20260508 05:40:34.301 | INFO | Analyzing chunk frames = [5668864,5669376) | 20260508 05:40:34.301 | INFO | MemInfo: { | 20260508 05:40:34.301 | INFO | "gpu" : " 17 GB", | 20260508 05:40:34.301 | INFO | "huge" : "", | 20260508 05:40:34.301 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:34.301 | INFO | "malloc" : " 14 GB", | 20260508 05:40:34.301 | INFO | "pinned" : " 29 GB" | 20260508 05:40:34.301 | INFO | } | 20260508 05:40:34.301 | INFO | Perf report for the last 5.1332 seconds: | 20260508 05:40:34.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:34.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:34.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:34.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:34.301 | INFO | ANALYSIS : 68.5%, 3.0, 106.2ms, 0 | 20260508 05:40:34.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:34.301 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 05:40:34.301 | INFO | PRE_HQ : 66.7%, 1.8, 61.8ms, 0 | 20260508 05:40:34.301 | INFO | BAZWRITER : 67.2%, 2.9, 99.9ms, 0 | 20260508 05:40:34.301 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 05:40:34.301 | INFO | Uploaded 36GiB to GPU in 2.53983s (14.1742GiB/s) | 20260508 05:40:34.301 | INFO | Downloaded 38GiB to GPU in 2.71218s (14.0109GiB/s) | 20260508 05:40:39.450 | INFO | Analyzing chunk frames = [5669376,5669888) | 20260508 05:40:39.450 | INFO | MemInfo: { | 20260508 05:40:39.450 | INFO | "gpu" : " 17 GB", | 20260508 05:40:39.450 | INFO | "huge" : "", | 20260508 05:40:39.450 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:39.450 | INFO | "malloc" : " 14 GB", | 20260508 05:40:39.450 | INFO | "pinned" : " 29 GB" | 20260508 05:40:39.450 | INFO | } | 20260508 05:40:39.450 | INFO | Perf report for the last 5.1492 seconds: | 20260508 05:40:39.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:39.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.450 | INFO | ANALYSIS : 67.2%, 3.0, 104.9ms, 0 | 20260508 05:40:39.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.450 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 05:40:39.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:39.451 | INFO | Uploaded 36GiB to GPU in 2.57268s (13.9932GiB/s) | 20260508 05:40:39.451 | INFO | Downloaded 34GiB to GPU in 2.44212s (13.9224GiB/s) | 20260508 05:40:44.588 | INFO | Analyzing chunk frames = [5669888,5670400) | 20260508 05:40:44.588 | INFO | MemInfo: { | 20260508 05:40:44.588 | INFO | "gpu" : " 17 GB", | 20260508 05:40:44.588 | INFO | "huge" : "", | 20260508 05:40:44.588 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:44.588 | INFO | "malloc" : " 14 GB", | 20260508 05:40:44.588 | INFO | "pinned" : " 29 GB" | 20260508 05:40:44.588 | INFO | } | 20260508 05:40:44.588 | INFO | Perf report for the last 5.13794 seconds: | 20260508 05:40:44.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:44.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:40:44.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:40:44.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:44.588 | INFO | Uploaded 36GiB to GPU in 2.56174s (14.0529GiB/s) | 20260508 05:40:44.588 | INFO | Downloaded 34GiB to GPU in 2.43756s (13.9484GiB/s) | 20260508 05:40:49.721 | INFO | Analyzing chunk frames = [5670400,5670912) | 20260508 05:40:49.721 | INFO | MemInfo: { | 20260508 05:40:49.721 | INFO | "gpu" : " 17 GB", | 20260508 05:40:49.721 | INFO | "huge" : "", | 20260508 05:40:49.721 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:49.721 | INFO | "malloc" : " 14 GB", | 20260508 05:40:49.721 | INFO | "pinned" : " 29 GB" | 20260508 05:40:49.721 | INFO | } | 20260508 05:40:49.722 | INFO | Perf report for the last 5.13349 seconds: | 20260508 05:40:49.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:49.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | ANALYSIS : 66.7%, 2.9, 103.1ms, 0 | 20260508 05:40:49.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:40:49.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:49.722 | INFO | Uploaded 36GiB to GPU in 2.53495s (14.2015GiB/s) | 20260508 05:40:49.722 | INFO | Downloaded 34GiB to GPU in 2.40547s (14.1345GiB/s) | 20260508 05:40:54.857 | INFO | Analyzing chunk frames = [5670912,5671424) | 20260508 05:40:54.857 | INFO | MemInfo: { | 20260508 05:40:54.857 | INFO | "gpu" : " 17 GB", | 20260508 05:40:54.857 | INFO | "huge" : "", | 20260508 05:40:54.858 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:54.858 | INFO | "malloc" : " 14 GB", | 20260508 05:40:54.858 | INFO | "pinned" : " 29 GB" | 20260508 05:40:54.858 | INFO | } | 20260508 05:40:54.858 | INFO | Perf report for the last 5.13608 seconds: | 20260508 05:40:54.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:54.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:40:54.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 05:40:54.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:54.858 | INFO | Uploaded 36GiB to GPU in 2.41247s (14.9224GiB/s) | 20260508 05:40:54.858 | INFO | Downloaded 34GiB to GPU in 2.56518s (13.2544GiB/s) | 20260508 05:40:59.996 | INFO | Analyzing chunk frames = [5671424,5671936) | 20260508 05:40:59.996 | INFO | MemInfo: { | 20260508 05:40:59.996 | INFO | "gpu" : " 17 GB", | 20260508 05:40:59.996 | INFO | "huge" : "", | 20260508 05:40:59.996 | INFO | "huge_shared" : " 29 GB", | 20260508 05:40:59.996 | INFO | "malloc" : " 14 GB", | 20260508 05:40:59.996 | INFO | "pinned" : " 29 GB" | 20260508 05:40:59.996 | INFO | } | 20260508 05:40:59.996 | INFO | Perf report for the last 5.13879 seconds: | 20260508 05:40:59.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:40:59.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.997 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:40:59.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.997 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:40:59.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:40:59.997 | INFO | Uploaded 36GiB to GPU in 2.45472s (14.6656GiB/s) | 20260508 05:40:59.997 | INFO | Downloaded 34GiB to GPU in 2.41806s (14.0609GiB/s) | 20260508 05:41:05.142 | INFO | Analyzing chunk frames = [5671936,5672448) | 20260508 05:41:05.142 | INFO | MemInfo: { | 20260508 05:41:05.142 | INFO | "gpu" : " 17 GB", | 20260508 05:41:05.142 | INFO | "huge" : "", | 20260508 05:41:05.142 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:05.142 | INFO | "malloc" : " 14 GB", | 20260508 05:41:05.142 | INFO | "pinned" : " 29 GB" | 20260508 05:41:05.142 | INFO | } | 20260508 05:41:05.142 | INFO | Perf report for the last 5.14567 seconds: | 20260508 05:41:05.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:05.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.142 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 05:41:05.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.142 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 05:41:05.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:05.143 | INFO | Uploaded 36GiB to GPU in 2.44791s (14.7064GiB/s) | 20260508 05:41:05.143 | INFO | Downloaded 34GiB to GPU in 2.42127s (14.0422GiB/s) | 20260508 05:41:10.282 | INFO | Analyzing chunk frames = [5672448,5672960) | 20260508 05:41:10.282 | INFO | MemInfo: { | 20260508 05:41:10.282 | INFO | "gpu" : " 17 GB", | 20260508 05:41:10.282 | INFO | "huge" : "", | 20260508 05:41:10.282 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:10.282 | INFO | "malloc" : " 14 GB", | 20260508 05:41:10.282 | INFO | "pinned" : " 29 GB" | 20260508 05:41:10.282 | INFO | } | 20260508 05:41:10.282 | INFO | Perf report for the last 5.13994 seconds: | 20260508 05:41:10.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:10.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.282 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:41:10.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.282 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 05:41:10.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:10.283 | INFO | Uploaded 36GiB to GPU in 2.48279s (14.4998GiB/s) | 20260508 05:41:10.283 | INFO | Downloaded 34GiB to GPU in 2.43541s (13.9607GiB/s) | 20260508 05:41:10.395 | INFO | Starting a round of Baz IO | 20260508 05:41:13.815 | INFO | Populating RTMetrics report | 20260508 05:41:13.815 | INFO | RTMetrics report populated | 20260508 05:41:13.815 | INFO | Opening tmp RTMetrics json file | 20260508 05:41:13.827 | INFO | Serializing RTMetrics to json | 20260508 05:41:13.828 | INFO | Serialization done, swapping tmp file now... | 20260508 05:41:13.828 | INFO | RTMetrics done | 20260508 05:41:13.887 | INFO | Finished a round of Baz IO: Wrote 1.569 GiB in 3.491 seconds | 20260508 05:41:13.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.83% / 0.00% / 59.17% | 20260508 05:41:13.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.447381% / 3.97364e-06% / 5.16574e-05% | 20260508 05:41:13.887 | INFO | Baz IO overall: | 20260508 05:41:13.887 | INFO | BytesWritten : 2525278705014 | 20260508 05:41:13.887 | INFO | NumEvents : 1218183548930 | 20260508 05:41:15.422 | INFO | Analyzing chunk frames = [5672960,5673472) | 20260508 05:41:15.422 | INFO | MemInfo: { | 20260508 05:41:15.423 | INFO | "gpu" : " 17 GB", | 20260508 05:41:15.423 | INFO | "huge" : "", | 20260508 05:41:15.423 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:15.423 | INFO | "malloc" : " 14 GB", | 20260508 05:41:15.423 | INFO | "pinned" : " 29 GB" | 20260508 05:41:15.423 | INFO | } | 20260508 05:41:15.423 | INFO | Perf report for the last 5.14061 seconds: | 20260508 05:41:15.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:15.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:15.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:15.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:15.423 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 05:41:15.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:15.423 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:41:15.423 | INFO | PRE_HQ : 66.2%, 1.8, 61.8ms, 0 | 20260508 05:41:15.423 | INFO | BAZWRITER : 67.2%, 2.9, 99.4ms, 0 | 20260508 05:41:15.423 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:41:15.423 | INFO | Uploaded 36GiB to GPU in 2.54339s (14.1544GiB/s) | 20260508 05:41:15.423 | INFO | Downloaded 38GiB to GPU in 2.68994s (14.1267GiB/s) | 20260508 05:41:20.555 | INFO | Analyzing chunk frames = [5673472,5673984) | 20260508 05:41:20.555 | INFO | MemInfo: { | 20260508 05:41:20.555 | INFO | "gpu" : " 17 GB", | 20260508 05:41:20.555 | INFO | "huge" : "", | 20260508 05:41:20.555 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:20.555 | INFO | "malloc" : " 14 GB", | 20260508 05:41:20.555 | INFO | "pinned" : " 29 GB" | 20260508 05:41:20.555 | INFO | } | 20260508 05:41:20.555 | INFO | Perf report for the last 5.13223 seconds: | 20260508 05:41:20.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:20.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 05:41:20.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:41:20.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:20.555 | INFO | Uploaded 36GiB to GPU in 2.50398s (14.3771GiB/s) | 20260508 05:41:20.555 | INFO | Downloaded 34GiB to GPU in 2.43495s (13.9633GiB/s) | 20260508 05:41:25.700 | INFO | Analyzing chunk frames = [5673984,5674496) | 20260508 05:41:25.700 | INFO | MemInfo: { | 20260508 05:41:25.700 | INFO | "gpu" : " 17 GB", | 20260508 05:41:25.700 | INFO | "huge" : "", | 20260508 05:41:25.700 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:25.700 | INFO | "malloc" : " 14 GB", | 20260508 05:41:25.700 | INFO | "pinned" : " 29 GB" | 20260508 05:41:25.700 | INFO | } | 20260508 05:41:25.700 | INFO | Perf report for the last 5.1449 seconds: | 20260508 05:41:25.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:25.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:41:25.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:41:25.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:25.700 | INFO | Uploaded 36GiB to GPU in 2.56064s (14.059GiB/s) | 20260508 05:41:25.700 | INFO | Downloaded 34GiB to GPU in 2.42014s (14.0488GiB/s) | 20260508 05:41:30.838 | INFO | Analyzing chunk frames = [5674496,5675008) | 20260508 05:41:30.838 | INFO | MemInfo: { | 20260508 05:41:30.838 | INFO | "gpu" : " 17 GB", | 20260508 05:41:30.838 | INFO | "huge" : "", | 20260508 05:41:30.838 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:30.838 | INFO | "malloc" : " 14 GB", | 20260508 05:41:30.838 | INFO | "pinned" : " 29 GB" | 20260508 05:41:30.839 | INFO | } | 20260508 05:41:30.839 | INFO | Perf report for the last 5.13868 seconds: | 20260508 05:41:30.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:30.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 05:41:30.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:41:30.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:30.839 | INFO | Uploaded 36GiB to GPU in 2.5683s (14.017GiB/s) | 20260508 05:41:30.839 | INFO | Downloaded 34GiB to GPU in 2.39366s (14.2042GiB/s) | 20260508 05:41:35.977 | INFO | Analyzing chunk frames = [5675008,5675520) | 20260508 05:41:35.977 | INFO | MemInfo: { | 20260508 05:41:35.977 | INFO | "gpu" : " 17 GB", | 20260508 05:41:35.977 | INFO | "huge" : "", | 20260508 05:41:35.977 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:35.977 | INFO | "malloc" : " 14 GB", | 20260508 05:41:35.977 | INFO | "pinned" : " 29 GB" | 20260508 05:41:35.977 | INFO | } | 20260508 05:41:35.977 | INFO | Perf report for the last 5.13848 seconds: | 20260508 05:41:35.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:35.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.977 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 05:41:35.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.977 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:41:35.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:35.978 | INFO | Uploaded 36GiB to GPU in 2.37021s (15.1885GiB/s) | 20260508 05:41:35.978 | INFO | Downloaded 34GiB to GPU in 2.55416s (13.3116GiB/s) | 20260508 05:41:41.118 | INFO | Analyzing chunk frames = [5675520,5676032) | 20260508 05:41:41.118 | INFO | MemInfo: { | 20260508 05:41:41.118 | INFO | "gpu" : " 17 GB", | 20260508 05:41:41.118 | INFO | "huge" : "", | 20260508 05:41:41.118 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:41.118 | INFO | "malloc" : " 14 GB", | 20260508 05:41:41.118 | INFO | "pinned" : " 29 GB" | 20260508 05:41:41.118 | INFO | } | 20260508 05:41:41.118 | INFO | Perf report for the last 5.14112 seconds: | 20260508 05:41:41.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:41.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.118 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 05:41:41.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.118 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:41:41.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:41.119 | INFO | Uploaded 36GiB to GPU in 2.43563s (14.7806GiB/s) | 20260508 05:41:41.119 | INFO | Downloaded 34GiB to GPU in 2.4336s (13.9711GiB/s) | 20260508 05:41:46.255 | INFO | Analyzing chunk frames = [5676032,5676544) | 20260508 05:41:46.255 | INFO | MemInfo: { | 20260508 05:41:46.255 | INFO | "gpu" : " 17 GB", | 20260508 05:41:46.255 | INFO | "huge" : "", | 20260508 05:41:46.255 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:46.255 | INFO | "malloc" : " 14 GB", | 20260508 05:41:46.255 | INFO | "pinned" : " 29 GB" | 20260508 05:41:46.255 | INFO | } | 20260508 05:41:46.255 | INFO | Perf report for the last 5.13675 seconds: | 20260508 05:41:46.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:46.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.255 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:41:46.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.255 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:41:46.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:46.256 | INFO | Uploaded 36GiB to GPU in 2.46179s (14.6235GiB/s) | 20260508 05:41:46.256 | INFO | Downloaded 34GiB to GPU in 2.44231s (13.9213GiB/s) | 20260508 05:41:51.397 | INFO | Analyzing chunk frames = [5676544,5677056) | 20260508 05:41:51.397 | INFO | MemInfo: { | 20260508 05:41:51.397 | INFO | "gpu" : " 17 GB", | 20260508 05:41:51.397 | INFO | "huge" : "", | 20260508 05:41:51.397 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:51.397 | INFO | "malloc" : " 14 GB", | 20260508 05:41:51.397 | INFO | "pinned" : " 29 GB" | 20260508 05:41:51.397 | INFO | } | 20260508 05:41:51.397 | INFO | Perf report for the last 5.1424 seconds: | 20260508 05:41:51.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:51.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.397 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:41:51.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.397 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:41:51.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:51.398 | INFO | Uploaded 36GiB to GPU in 2.44805s (14.7056GiB/s) | 20260508 05:41:51.398 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:41:51.398 | INFO | Total mem usage: 16.7 GiB | 20260508 05:41:51.398 | INFO | Downloaded 34GiB to GPU in 2.43816s (13.9449GiB/s) | 20260508 05:41:51.398 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:41:51.398 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:41:51.398 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:41:51.398 | INFO | High Water Breakdown: | 20260508 05:41:51.398 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:41:51.398 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:41:51.398 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:41:51.398 | INFO | ------------------------------------------ | 20260508 05:41:51.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:41:51.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:41:51.398 | INFO | Total mem usage: 26.5 GiB | 20260508 05:41:51.398 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:41:51.398 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:41:51.398 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:41:51.398 | INFO | High Water Breakdown: | 20260508 05:41:51.398 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:41:51.398 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:41:51.398 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:41:51.398 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:41:51.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:41:51.398 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:41:51.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:41:51.398 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:41:51.398 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:41:51.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:41:51.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:41:51.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:41:51.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:41:51.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:41:51.398 | INFO | ------------------------------------------ | 20260508 05:41:51.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:41:51.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:41:51.398 | INFO | Total mem usage: 27.8 GiB | 20260508 05:41:51.398 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:41:51.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:41:51.398 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:41:51.398 | INFO | High Water Breakdown: | 20260508 05:41:51.398 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:41:51.398 | INFO | ------------------------------------------ | 20260508 05:41:51.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:41:51.398 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:41:51.398 | INFO | Total mem usage: 15.7 GiB | 20260508 05:41:51.398 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:41:51.398 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:41:51.398 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:41:51.399 | INFO | High Water Breakdown: | 20260508 05:41:51.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:41:51.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:41:51.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:41:51.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:41:51.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:41:51.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:41:51.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:41:51.399 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:41:51.399 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:41:51.399 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:41:51.399 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:41:51.399 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:41:51.399 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:41:51.399 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:41:51.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:41:51.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:41:51.399 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:41:51.399 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:41:51.399 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:41:51.399 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:41:51.399 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:41:51.399 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:41:51.399 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:41:51.399 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:41:51.399 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:41:51.399 | INFO | ------------------------------------------ | 20260508 05:41:51.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:41:51.399 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:41:51.399 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:41:51.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:41:51.399 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.59e+01, 1.40e+01 / 1.57e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.12e-02 / 7.16e-03 | 20260508 05:41:51.399 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.43e+00, 5.86e-01 / 2.37e-01 | 20260508 05:41:51.399 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.10e-01 / 1.48e-01 | 20260508 05:41:51.399 | INFO | PerfReport: 15.93%, DME -- 4.16e+01 / 4.54e+01, 3.87e+00 / 1.88e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.77e+00 / 1.33e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.58e+00, 4.19e-01 / 2.25e-01 | 20260508 05:41:51.399 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.21e+00, 2.60e-01 / 1.05e-01 | 20260508 05:41:51.399 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.56e-01, 5.00e-02 / 2.38e-02 | 20260508 05:41:51.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:41:51.399 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:41:51.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:41:51.399 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 24.30%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 2.28%, LockUpload -- 2.42e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 2.64%, LockDownload -- 2.80e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:41:51.399 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260508 05:41:51.495 | INFO | Starting a round of Baz IO | 20260508 05:41:54.926 | INFO | Populating RTMetrics report | 20260508 05:41:54.926 | INFO | RTMetrics report populated | 20260508 05:41:54.926 | INFO | Opening tmp RTMetrics json file | 20260508 05:41:54.939 | INFO | Serializing RTMetrics to json | 20260508 05:41:54.940 | INFO | Serialization done, swapping tmp file now... | 20260508 05:41:54.940 | INFO | RTMetrics done | 20260508 05:41:54.996 | INFO | Finished a round of Baz IO: Wrote 1.568 GiB in 3.5 seconds | 20260508 05:41:54.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.82% / 0.00% / 59.17% | 20260508 05:41:54.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.446157% / 0% / 8.34465e-05% | 20260508 05:41:54.996 | INFO | Baz IO overall: | 20260508 05:41:54.996 | INFO | BytesWritten : 2526962608323 | 20260508 05:41:54.996 | INFO | NumEvents : 1218986351224 | 20260508 05:41:56.534 | INFO | Analyzing chunk frames = [5677056,5677568) | 20260508 05:41:56.534 | INFO | MemInfo: { | 20260508 05:41:56.534 | INFO | "gpu" : " 17 GB", | 20260508 05:41:56.534 | INFO | "huge" : "", | 20260508 05:41:56.534 | INFO | "huge_shared" : " 29 GB", | 20260508 05:41:56.534 | INFO | "malloc" : " 14 GB", | 20260508 05:41:56.534 | INFO | "pinned" : " 29 GB" | 20260508 05:41:56.534 | INFO | } | 20260508 05:41:56.534 | INFO | Perf report for the last 5.13681 seconds: | 20260508 05:41:56.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:41:56.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:56.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:56.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:56.534 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 05:41:56.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:41:56.534 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 05:41:56.534 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 05:41:56.534 | INFO | BAZWRITER : 67.6%, 2.7, 93.5ms, 0 | 20260508 05:41:56.534 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:41:56.534 | INFO | Uploaded 36GiB to GPU in 2.54928s (14.1216GiB/s) | 20260508 05:41:56.535 | INFO | Downloaded 38GiB to GPU in 2.69323s (14.1094GiB/s) | 20260508 05:42:01.673 | INFO | Analyzing chunk frames = [5677568,5678080) | 20260508 05:42:01.673 | INFO | MemInfo: { | 20260508 05:42:01.673 | INFO | "gpu" : " 17 GB", | 20260508 05:42:01.673 | INFO | "huge" : "", | 20260508 05:42:01.673 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:01.673 | INFO | "malloc" : " 14 GB", | 20260508 05:42:01.673 | INFO | "pinned" : " 29 GB" | 20260508 05:42:01.673 | INFO | } | 20260508 05:42:01.673 | INFO | Perf report for the last 5.13886 seconds: | 20260508 05:42:01.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:01.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 05:42:01.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:42:01.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:01.673 | INFO | Uploaded 36GiB to GPU in 2.50166s (14.3905GiB/s) | 20260508 05:42:01.673 | INFO | Downloaded 34GiB to GPU in 2.43935s (13.9381GiB/s) | 20260508 05:42:06.808 | INFO | Analyzing chunk frames = [5678080,5678592) | 20260508 05:42:06.808 | INFO | MemInfo: { | 20260508 05:42:06.808 | INFO | "gpu" : " 17 GB", | 20260508 05:42:06.808 | INFO | "huge" : "", | 20260508 05:42:06.808 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:06.808 | INFO | "malloc" : " 14 GB", | 20260508 05:42:06.808 | INFO | "pinned" : " 29 GB" | 20260508 05:42:06.808 | INFO | } | 20260508 05:42:06.808 | INFO | Perf report for the last 5.13499 seconds: | 20260508 05:42:06.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:06.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 05:42:06.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:42:06.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:06.808 | INFO | Uploaded 36GiB to GPU in 2.5237s (14.2648GiB/s) | 20260508 05:42:06.808 | INFO | Downloaded 34GiB to GPU in 2.4125s (14.0933GiB/s) | 20260508 05:42:11.949 | INFO | Analyzing chunk frames = [5678592,5679104) | 20260508 05:42:11.949 | INFO | MemInfo: { | 20260508 05:42:11.949 | INFO | "gpu" : " 17 GB", | 20260508 05:42:11.949 | INFO | "huge" : "", | 20260508 05:42:11.949 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:11.949 | INFO | "malloc" : " 14 GB", | 20260508 05:42:11.949 | INFO | "pinned" : " 29 GB" | 20260508 05:42:11.949 | INFO | } | 20260508 05:42:11.949 | INFO | Perf report for the last 5.14104 seconds: | 20260508 05:42:11.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:11.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 05:42:11.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:42:11.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:11.949 | INFO | Uploaded 36GiB to GPU in 2.56495s (14.0354GiB/s) | 20260508 05:42:11.949 | INFO | Downloaded 34GiB to GPU in 2.40291s (14.1495GiB/s) | 20260508 05:42:17.091 | INFO | Analyzing chunk frames = [5679104,5679616) | 20260508 05:42:17.091 | INFO | MemInfo: { | 20260508 05:42:17.091 | INFO | "gpu" : " 17 GB", | 20260508 05:42:17.091 | INFO | "huge" : "", | 20260508 05:42:17.091 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:17.091 | INFO | "malloc" : " 14 GB", | 20260508 05:42:17.091 | INFO | "pinned" : " 29 GB" | 20260508 05:42:17.091 | INFO | } | 20260508 05:42:17.091 | INFO | Perf report for the last 5.14212 seconds: | 20260508 05:42:17.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:17.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.091 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:42:17.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.091 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:42:17.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:17.092 | INFO | Uploaded 36GiB to GPU in 2.39257s (15.0466GiB/s) | 20260508 05:42:17.092 | INFO | Downloaded 34GiB to GPU in 2.56359s (13.2627GiB/s) | 20260508 05:42:22.228 | INFO | Analyzing chunk frames = [5679616,5680128) | 20260508 05:42:22.228 | INFO | MemInfo: { | 20260508 05:42:22.228 | INFO | "gpu" : " 17 GB", | 20260508 05:42:22.228 | INFO | "huge" : "", | 20260508 05:42:22.229 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:22.229 | INFO | "malloc" : " 14 GB", | 20260508 05:42:22.229 | INFO | "pinned" : " 29 GB" | 20260508 05:42:22.229 | INFO | } | 20260508 05:42:22.229 | INFO | Perf report for the last 5.13743 seconds: | 20260508 05:42:22.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:22.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:42:22.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 05:42:22.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:22.229 | INFO | Uploaded 36GiB to GPU in 2.42859s (14.8234GiB/s) | 20260508 05:42:22.229 | INFO | Downloaded 34GiB to GPU in 2.42738s (14.0068GiB/s) | 20260508 05:42:27.367 | INFO | Analyzing chunk frames = [5680128,5680640) | 20260508 05:42:27.367 | INFO | MemInfo: { | 20260508 05:42:27.367 | INFO | "gpu" : " 17 GB", | 20260508 05:42:27.367 | INFO | "huge" : "", | 20260508 05:42:27.367 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:27.367 | INFO | "malloc" : " 14 GB", | 20260508 05:42:27.368 | INFO | "pinned" : " 29 GB" | 20260508 05:42:27.368 | INFO | } | 20260508 05:42:27.368 | INFO | Perf report for the last 5.13892 seconds: | 20260508 05:42:27.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:27.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:42:27.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260508 05:42:27.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:27.368 | INFO | Uploaded 36GiB to GPU in 2.45334s (14.6739GiB/s) | 20260508 05:42:27.368 | INFO | Downloaded 34GiB to GPU in 2.42663s (14.0112GiB/s) | 20260508 05:42:32.510 | INFO | Analyzing chunk frames = [5680640,5681152) | 20260508 05:42:32.510 | INFO | MemInfo: { | 20260508 05:42:32.510 | INFO | "gpu" : " 17 GB", | 20260508 05:42:32.510 | INFO | "huge" : "", | 20260508 05:42:32.510 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:32.510 | INFO | "malloc" : " 14 GB", | 20260508 05:42:32.510 | INFO | "pinned" : " 29 GB" | 20260508 05:42:32.510 | INFO | } | 20260508 05:42:32.510 | INFO | Perf report for the last 5.14245 seconds: | 20260508 05:42:32.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:32.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.510 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 05:42:32.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.510 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 05:42:32.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:32.511 | INFO | Uploaded 36GiB to GPU in 2.46534s (14.6024GiB/s) | 20260508 05:42:32.511 | INFO | Downloaded 34GiB to GPU in 2.44449s (13.9088GiB/s) | 20260508 05:42:32.648 | INFO | Starting a round of Baz IO | 20260508 05:42:36.042 | INFO | Populating RTMetrics report | 20260508 05:42:36.042 | INFO | RTMetrics report populated | 20260508 05:42:36.042 | INFO | Opening tmp RTMetrics json file | 20260508 05:42:36.054 | INFO | Serializing RTMetrics to json | 20260508 05:42:36.055 | INFO | Serialization done, swapping tmp file now... | 20260508 05:42:36.055 | INFO | RTMetrics done | 20260508 05:42:36.118 | INFO | Finished a round of Baz IO: Wrote 1.567 GiB in 3.469 seconds | 20260508 05:42:36.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.82% / 0.00% / 59.18% | 20260508 05:42:36.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.444734% / 3.97364e-06% / 6.75519e-05% | 20260508 05:42:36.118 | INFO | Baz IO overall: | 20260508 05:42:36.118 | INFO | BytesWritten : 2528644977986 | 20260508 05:42:36.118 | INFO | NumEvents : 1219788271662 | 20260508 05:42:37.649 | INFO | Analyzing chunk frames = [5681152,5681664) | 20260508 05:42:37.649 | INFO | MemInfo: { | 20260508 05:42:37.649 | INFO | "gpu" : " 17 GB", | 20260508 05:42:37.649 | INFO | "huge" : "", | 20260508 05:42:37.649 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:37.649 | INFO | "malloc" : " 14 GB", | 20260508 05:42:37.649 | INFO | "pinned" : " 29 GB" | 20260508 05:42:37.649 | INFO | } | 20260508 05:42:37.650 | INFO | Perf report for the last 5.13943 seconds: | 20260508 05:42:37.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:37.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:37.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:37.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:37.650 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 05:42:37.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:37.650 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 05:42:37.650 | INFO | PRE_HQ : 66.5%, 1.8, 62.6ms, 0 | 20260508 05:42:37.650 | INFO | BAZWRITER : 67.4%, 2.9, 100.0ms, 0 | 20260508 05:42:37.650 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:42:37.650 | INFO | Uploaded 36GiB to GPU in 2.57808s (13.9639GiB/s) | 20260508 05:42:37.650 | INFO | Downloaded 38GiB to GPU in 2.70788s (14.0331GiB/s) | 20260508 05:42:42.788 | INFO | Analyzing chunk frames = [5681664,5682176) | 20260508 05:42:42.788 | INFO | MemInfo: { | 20260508 05:42:42.788 | INFO | "gpu" : " 17 GB", | 20260508 05:42:42.788 | INFO | "huge" : "", | 20260508 05:42:42.788 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:42.789 | INFO | "malloc" : " 14 GB", | 20260508 05:42:42.789 | INFO | "pinned" : " 29 GB" | 20260508 05:42:42.789 | INFO | } | 20260508 05:42:42.789 | INFO | Perf report for the last 5.13919 seconds: | 20260508 05:42:42.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:42.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:42:42.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:42:42.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:42.789 | INFO | Uploaded 36GiB to GPU in 2.47759s (14.5302GiB/s) | 20260508 05:42:42.789 | INFO | Downloaded 34GiB to GPU in 2.44523s (13.9046GiB/s) | 20260508 05:42:47.930 | INFO | Analyzing chunk frames = [5682176,5682688) | 20260508 05:42:47.930 | INFO | MemInfo: { | 20260508 05:42:47.930 | INFO | "gpu" : " 17 GB", | 20260508 05:42:47.930 | INFO | "huge" : "", | 20260508 05:42:47.930 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:47.930 | INFO | "malloc" : " 14 GB", | 20260508 05:42:47.930 | INFO | "pinned" : " 29 GB" | 20260508 05:42:47.930 | INFO | } | 20260508 05:42:47.930 | INFO | Perf report for the last 5.14141 seconds: | 20260508 05:42:47.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:47.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.930 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:42:47.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.930 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:42:47.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:47.931 | INFO | Uploaded 36GiB to GPU in 2.54112s (14.167GiB/s) | 20260508 05:42:47.931 | INFO | Downloaded 34GiB to GPU in 2.42481s (14.0217GiB/s) | 20260508 05:42:53.064 | INFO | Analyzing chunk frames = [5682688,5683200) | 20260508 05:42:53.064 | INFO | MemInfo: { | 20260508 05:42:53.064 | INFO | "gpu" : " 17 GB", | 20260508 05:42:53.064 | INFO | "huge" : "", | 20260508 05:42:53.065 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:53.065 | INFO | "malloc" : " 14 GB", | 20260508 05:42:53.065 | INFO | "pinned" : " 29 GB" | 20260508 05:42:53.065 | INFO | } | 20260508 05:42:53.065 | INFO | Perf report for the last 5.13456 seconds: | 20260508 05:42:53.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:53.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 05:42:53.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:42:53.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:53.065 | INFO | Uploaded 36GiB to GPU in 2.56262s (14.0481GiB/s) | 20260508 05:42:53.065 | INFO | Downloaded 34GiB to GPU in 2.40459s (14.1396GiB/s) | 20260508 05:42:58.207 | INFO | Analyzing chunk frames = [5683200,5683712) | 20260508 05:42:58.207 | INFO | MemInfo: { | 20260508 05:42:58.207 | INFO | "gpu" : " 17 GB", | 20260508 05:42:58.207 | INFO | "huge" : "", | 20260508 05:42:58.207 | INFO | "huge_shared" : " 29 GB", | 20260508 05:42:58.207 | INFO | "malloc" : " 14 GB", | 20260508 05:42:58.207 | INFO | "pinned" : " 29 GB" | 20260508 05:42:58.207 | INFO | } | 20260508 05:42:58.207 | INFO | Perf report for the last 5.14219 seconds: | 20260508 05:42:58.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:42:58.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.207 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:42:58.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.207 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:42:58.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:42:58.208 | INFO | Uploaded 36GiB to GPU in 2.3657s (15.2175GiB/s) | 20260508 05:42:58.208 | INFO | Downloaded 34GiB to GPU in 2.55614s (13.3013GiB/s) | 20260508 05:43:03.347 | INFO | Analyzing chunk frames = [5683712,5684224) | 20260508 05:43:03.347 | INFO | MemInfo: { | 20260508 05:43:03.347 | INFO | "gpu" : " 17 GB", | 20260508 05:43:03.347 | INFO | "huge" : "", | 20260508 05:43:03.348 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:03.348 | INFO | "malloc" : " 14 GB", | 20260508 05:43:03.348 | INFO | "pinned" : " 29 GB" | 20260508 05:43:03.348 | INFO | } | 20260508 05:43:03.348 | INFO | Perf report for the last 5.14079 seconds: | 20260508 05:43:03.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:03.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:43:03.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:43:03.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:03.348 | INFO | Uploaded 36GiB to GPU in 2.46338s (14.614GiB/s) | 20260508 05:43:03.348 | INFO | Downloaded 34GiB to GPU in 2.42704s (14.0088GiB/s) | 20260508 05:43:08.485 | INFO | Analyzing chunk frames = [5684224,5684736) | 20260508 05:43:08.485 | INFO | MemInfo: { | 20260508 05:43:08.485 | INFO | "gpu" : " 17 GB", | 20260508 05:43:08.486 | INFO | "huge" : "", | 20260508 05:43:08.486 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:08.486 | INFO | "malloc" : " 14 GB", | 20260508 05:43:08.486 | INFO | "pinned" : " 29 GB" | 20260508 05:43:08.486 | INFO | } | 20260508 05:43:08.486 | INFO | Perf report for the last 5.13822 seconds: | 20260508 05:43:08.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:08.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.486 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:43:08.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.486 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 05:43:08.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:08.487 | INFO | Uploaded 36GiB to GPU in 2.45401s (14.6699GiB/s) | 20260508 05:43:08.487 | INFO | Downloaded 34GiB to GPU in 2.42299s (14.0322GiB/s) | 20260508 05:43:13.619 | INFO | Analyzing chunk frames = [5684736,5685248) | 20260508 05:43:13.619 | INFO | MemInfo: { | 20260508 05:43:13.619 | INFO | "gpu" : " 17 GB", | 20260508 05:43:13.619 | INFO | "huge" : "", | 20260508 05:43:13.619 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:13.619 | INFO | "malloc" : " 14 GB", | 20260508 05:43:13.619 | INFO | "pinned" : " 29 GB" | 20260508 05:43:13.619 | INFO | } | 20260508 05:43:13.619 | INFO | Perf report for the last 5.13277 seconds: | 20260508 05:43:13.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:13.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 05:43:13.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260508 05:43:13.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:13.619 | INFO | Uploaded 36GiB to GPU in 2.45218s (14.6808GiB/s) | 20260508 05:43:13.619 | INFO | Downloaded 34GiB to GPU in 2.42403s (14.0262GiB/s) | 20260508 05:43:13.619 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:43:13.619 | INFO | Total mem usage: 16.7 GiB | 20260508 05:43:13.619 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:43:13.619 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:43:13.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:43:13.619 | INFO | High Water Breakdown: | 20260508 05:43:13.619 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:43:13.620 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:43:13.620 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:43:13.620 | INFO | ------------------------------------------ | 20260508 05:43:13.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:43:13.620 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:43:13.620 | INFO | Total mem usage: 26.5 GiB | 20260508 05:43:13.620 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:43:13.620 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 05:43:13.620 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 05:43:13.620 | INFO | High Water Breakdown: | 20260508 05:43:13.620 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:43:13.620 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:43:13.620 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:43:13.620 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:43:13.620 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:43:13.620 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:43:13.620 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:43:13.620 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:43:13.620 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:43:13.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:43:13.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:43:13.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:43:13.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:43:13.620 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:43:13.620 | INFO | ------------------------------------------ | 20260508 05:43:13.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:43:13.620 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:43:13.620 | INFO | Total mem usage: 27.8 GiB | 20260508 05:43:13.620 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:43:13.620 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:43:13.620 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:43:13.620 | INFO | High Water Breakdown: | 20260508 05:43:13.620 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:43:13.620 | INFO | ------------------------------------------ | 20260508 05:43:13.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:43:13.620 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:43:13.620 | INFO | Total mem usage: 15.7 GiB | 20260508 05:43:13.620 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:43:13.620 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:43:13.620 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:43:13.620 | INFO | High Water Breakdown: | 20260508 05:43:13.620 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:43:13.620 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:43:13.620 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:43:13.620 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:43:13.620 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:43:13.620 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:43:13.620 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:43:13.620 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:43:13.620 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:43:13.620 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:43:13.620 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:43:13.620 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:43:13.620 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:43:13.620 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:43:13.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:43:13.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:43:13.620 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:43:13.620 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:43:13.620 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:43:13.620 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:43:13.620 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:43:13.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:43:13.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:43:13.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:43:13.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:43:13.620 | INFO | ------------------------------------------ | 20260508 05:43:13.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:43:13.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:43:13.620 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:43:13.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:43:13.620 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.59e+01, 1.41e+01 / 2.12e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.19e-02 / 7.05e-03 | 20260508 05:43:13.621 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.45e+00, 5.92e-01 / 2.43e-01 | 20260508 05:43:13.621 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.32e-01 | 20260508 05:43:13.621 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.54e+01, 3.93e+00 / 2.34e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.82e+00 / 1.28e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.38e+00 / 2.58e+00, 4.18e-01 / 2.61e-01 | 20260508 05:43:13.621 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.23e+00, 2.61e-01 / 1.52e-01 | 20260508 05:43:13.621 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.19e-01 / 3.58e-01, 5.02e-02 / 2.35e-02 | 20260508 05:43:13.621 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:43:13.621 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:43:13.621 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:43:13.621 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 24.38%, Upload -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 30.84%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 2.20%, LockUpload -- 2.33e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 2.64%, LockDownload -- 2.80e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:43:13.621 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260508 05:43:13.716 | INFO | Starting a round of Baz IO | 20260508 05:43:17.137 | INFO | Populating RTMetrics report | 20260508 05:43:17.137 | INFO | RTMetrics report populated | 20260508 05:43:17.138 | INFO | Opening tmp RTMetrics json file | 20260508 05:43:17.153 | INFO | Serializing RTMetrics to json | 20260508 05:43:17.156 | INFO | Serialization done, swapping tmp file now... | 20260508 05:43:17.156 | INFO | RTMetrics done | 20260508 05:43:17.215 | INFO | Finished a round of Baz IO: Wrote 1.568 GiB in 3.498 seconds | 20260508 05:43:17.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.82% / 0.00% / 59.18% | 20260508 05:43:17.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.445151% / 3.97364e-06% / 5.5631e-05% | 20260508 05:43:17.215 | INFO | Baz IO overall: | 20260508 05:43:17.215 | INFO | BytesWritten : 2530328819143 | 20260508 05:43:17.215 | INFO | NumEvents : 1220591047847 | 20260508 05:43:18.760 | INFO | Analyzing chunk frames = [5685248,5685760) | 20260508 05:43:18.760 | INFO | MemInfo: { | 20260508 05:43:18.760 | INFO | "gpu" : " 17 GB", | 20260508 05:43:18.760 | INFO | "huge" : "", | 20260508 05:43:18.760 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:18.760 | INFO | "malloc" : " 14 GB", | 20260508 05:43:18.760 | INFO | "pinned" : " 29 GB" | 20260508 05:43:18.760 | INFO | } | 20260508 05:43:18.760 | INFO | Perf report for the last 5.14153 seconds: | 20260508 05:43:18.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:18.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:18.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:18.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:18.760 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 05:43:18.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:18.760 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 05:43:18.760 | INFO | PRE_HQ : 65.5%, 1.6, 55.5ms, 0 | 20260508 05:43:18.760 | INFO | BAZWRITER : 67.5%, 2.7, 94.2ms, 0 | 20260508 05:43:18.761 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:43:18.761 | INFO | Uploaded 36GiB to GPU in 2.58018s (13.9525GiB/s) | 20260508 05:43:18.761 | INFO | Downloaded 38GiB to GPU in 2.70364s (14.0551GiB/s) | 20260508 05:43:23.902 | INFO | Analyzing chunk frames = [5685760,5686272) | 20260508 05:43:23.902 | INFO | MemInfo: { | 20260508 05:43:23.902 | INFO | "gpu" : " 17 GB", | 20260508 05:43:23.902 | INFO | "huge" : "", | 20260508 05:43:23.902 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:23.902 | INFO | "malloc" : " 14 GB", | 20260508 05:43:23.902 | INFO | "pinned" : " 29 GB" | 20260508 05:43:23.902 | INFO | } | 20260508 05:43:23.902 | INFO | Perf report for the last 5.14161 seconds: | 20260508 05:43:23.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:23.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 05:43:23.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:43:23.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:23.902 | INFO | Uploaded 36GiB to GPU in 2.4832s (14.4974GiB/s) | 20260508 05:43:23.902 | INFO | Downloaded 34GiB to GPU in 2.44003s (13.9343GiB/s) | 20260508 05:43:29.045 | INFO | Analyzing chunk frames = [5686272,5686784) | 20260508 05:43:29.045 | INFO | MemInfo: { | 20260508 05:43:29.045 | INFO | "gpu" : " 17 GB", | 20260508 05:43:29.045 | INFO | "huge" : "", | 20260508 05:43:29.045 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:29.045 | INFO | "malloc" : " 14 GB", | 20260508 05:43:29.045 | INFO | "pinned" : " 29 GB" | 20260508 05:43:29.045 | INFO | } | 20260508 05:43:29.045 | INFO | Perf report for the last 5.14322 seconds: | 20260508 05:43:29.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:29.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.045 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 05:43:29.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.045 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:43:29.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:29.046 | INFO | Uploaded 36GiB to GPU in 2.53999s (14.1733GiB/s) | 20260508 05:43:29.046 | INFO | Downloaded 34GiB to GPU in 2.42597s (14.015GiB/s) | 20260508 05:43:34.182 | INFO | Analyzing chunk frames = [5686784,5687296) | 20260508 05:43:34.182 | INFO | MemInfo: { | 20260508 05:43:34.182 | INFO | "gpu" : " 17 GB", | 20260508 05:43:34.182 | INFO | "huge" : "", | 20260508 05:43:34.183 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:34.183 | INFO | "malloc" : " 14 GB", | 20260508 05:43:34.183 | INFO | "pinned" : " 29 GB" | 20260508 05:43:34.183 | INFO | } | 20260508 05:43:34.183 | INFO | Perf report for the last 5.13751 seconds: | 20260508 05:43:34.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:34.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 05:43:34.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:43:34.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:34.183 | INFO | Uploaded 36GiB to GPU in 2.54871s (14.1248GiB/s) | 20260508 05:43:34.183 | INFO | Downloaded 34GiB to GPU in 2.3932s (14.2069GiB/s) | 20260508 05:43:39.315 | INFO | Analyzing chunk frames = [5687296,5687808) | 20260508 05:43:39.315 | INFO | MemInfo: { | 20260508 05:43:39.315 | INFO | "gpu" : " 17 GB", | 20260508 05:43:39.315 | INFO | "huge" : "", | 20260508 05:43:39.315 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:39.315 | INFO | "malloc" : " 14 GB", | 20260508 05:43:39.315 | INFO | "pinned" : " 29 GB" | 20260508 05:43:39.315 | INFO | } | 20260508 05:43:39.315 | INFO | Perf report for the last 5.13212 seconds: | 20260508 05:43:39.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:39.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 05:43:39.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260508 05:43:39.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:39.315 | INFO | Uploaded 36GiB to GPU in 2.27813s (15.8024GiB/s) | 20260508 05:43:39.315 | INFO | Downloaded 34GiB to GPU in 2.49663s (13.6184GiB/s) | 20260508 05:43:44.463 | INFO | Analyzing chunk frames = [5687808,5688320) | 20260508 05:43:44.463 | INFO | MemInfo: { | 20260508 05:43:44.463 | INFO | "gpu" : " 17 GB", | 20260508 05:43:44.463 | INFO | "huge" : "", | 20260508 05:43:44.463 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:44.463 | INFO | "malloc" : " 14 GB", | 20260508 05:43:44.463 | INFO | "pinned" : " 29 GB" | 20260508 05:43:44.463 | INFO | } | 20260508 05:43:44.463 | INFO | Perf report for the last 5.1486 seconds: | 20260508 05:43:44.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:44.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.464 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 05:43:44.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.464 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 05:43:44.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:44.464 | INFO | Uploaded 36GiB to GPU in 2.4172s (14.8933GiB/s) | 20260508 05:43:44.464 | INFO | Downloaded 34GiB to GPU in 2.44128s (13.9271GiB/s) | 20260508 05:43:49.605 | INFO | Analyzing chunk frames = [5688320,5688832) | 20260508 05:43:49.605 | INFO | MemInfo: { | 20260508 05:43:49.605 | INFO | "gpu" : " 17 GB", | 20260508 05:43:49.605 | INFO | "huge" : "", | 20260508 05:43:49.605 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:49.605 | INFO | "malloc" : " 14 GB", | 20260508 05:43:49.605 | INFO | "pinned" : " 29 GB" | 20260508 05:43:49.605 | INFO | } | 20260508 05:43:49.605 | INFO | Perf report for the last 5.14158 seconds: | 20260508 05:43:49.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:49.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.605 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:43:49.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.605 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 05:43:49.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:49.606 | INFO | Uploaded 36GiB to GPU in 2.4426s (14.7384GiB/s) | 20260508 05:43:49.606 | INFO | Downloaded 34GiB to GPU in 2.44771s (13.8906GiB/s) | 20260508 05:43:54.743 | INFO | Analyzing chunk frames = [5688832,5689344) | 20260508 05:43:54.743 | INFO | MemInfo: { | 20260508 05:43:54.743 | INFO | "gpu" : " 17 GB", | 20260508 05:43:54.743 | INFO | "huge" : "", | 20260508 05:43:54.743 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:54.743 | INFO | "malloc" : " 14 GB", | 20260508 05:43:54.743 | INFO | "pinned" : " 29 GB" | 20260508 05:43:54.743 | INFO | } | 20260508 05:43:54.743 | INFO | Perf report for the last 5.13799 seconds: | 20260508 05:43:54.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:54.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.743 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:43:54.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.743 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 05:43:54.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:54.744 | INFO | Uploaded 36GiB to GPU in 2.4535s (14.6729GiB/s) | 20260508 05:43:54.744 | INFO | Downloaded 34GiB to GPU in 2.45615s (13.8428GiB/s) | 20260508 05:43:54.857 | INFO | Starting a round of Baz IO | 20260508 05:43:58.274 | INFO | Populating RTMetrics report | 20260508 05:43:58.274 | INFO | RTMetrics report populated | 20260508 05:43:58.274 | INFO | Opening tmp RTMetrics json file | 20260508 05:43:58.283 | INFO | Serializing RTMetrics to json | 20260508 05:43:58.284 | INFO | Serialization done, swapping tmp file now... | 20260508 05:43:58.285 | INFO | RTMetrics done | 20260508 05:43:58.345 | INFO | Finished a round of Baz IO: Wrote 1.568 GiB in 3.487 seconds | 20260508 05:43:58.345 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.82% / 0.00% / 59.18% | 20260508 05:43:58.345 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.444595% / 1.19209e-05% / 5.16574e-05% | 20260508 05:43:58.345 | INFO | Baz IO overall: | 20260508 05:43:58.345 | INFO | BytesWritten : 2532012036979 | 20260508 05:43:58.345 | INFO | NumEvents : 1221393444341 | 20260508 05:43:59.879 | INFO | Analyzing chunk frames = [5689344,5689856) | 20260508 05:43:59.879 | INFO | MemInfo: { | 20260508 05:43:59.879 | INFO | "gpu" : " 17 GB", | 20260508 05:43:59.879 | INFO | "huge" : "", | 20260508 05:43:59.879 | INFO | "huge_shared" : " 29 GB", | 20260508 05:43:59.879 | INFO | "malloc" : " 14 GB", | 20260508 05:43:59.879 | INFO | "pinned" : " 29 GB" | 20260508 05:43:59.879 | INFO | } | 20260508 05:43:59.879 | INFO | Perf report for the last 5.13569 seconds: | 20260508 05:43:59.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:43:59.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:59.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:59.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:59.879 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 05:43:59.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:43:59.879 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 05:43:59.879 | INFO | PRE_HQ : 65.9%, 1.7, 57.0ms, 0 | 20260508 05:43:59.879 | INFO | BAZWRITER : 67.3%, 2.7, 94.6ms, 0 | 20260508 05:43:59.879 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:43:59.879 | INFO | Uploaded 36GiB to GPU in 2.56724s (14.0229GiB/s) | 20260508 05:43:59.879 | INFO | Downloaded 38GiB to GPU in 2.68878s (14.1328GiB/s) | 20260508 05:44:05.017 | INFO | Analyzing chunk frames = [5689856,5690368) | 20260508 05:44:05.017 | INFO | MemInfo: { | 20260508 05:44:05.017 | INFO | "gpu" : " 17 GB", | 20260508 05:44:05.017 | INFO | "huge" : "", | 20260508 05:44:05.017 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:05.017 | INFO | "malloc" : " 14 GB", | 20260508 05:44:05.017 | INFO | "pinned" : " 29 GB" | 20260508 05:44:05.017 | INFO | } | 20260508 05:44:05.017 | INFO | Perf report for the last 5.13859 seconds: | 20260508 05:44:05.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:05.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.017 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:44:05.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.017 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:44:05.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:05.018 | INFO | Uploaded 36GiB to GPU in 2.48766s (14.4714GiB/s) | 20260508 05:44:05.018 | INFO | Downloaded 34GiB to GPU in 2.43583s (13.9583GiB/s) | 20260508 05:44:10.156 | INFO | Analyzing chunk frames = [5690368,5690880) | 20260508 05:44:10.156 | INFO | MemInfo: { | 20260508 05:44:10.156 | INFO | "gpu" : " 17 GB", | 20260508 05:44:10.156 | INFO | "huge" : "", | 20260508 05:44:10.156 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:10.156 | INFO | "malloc" : " 14 GB", | 20260508 05:44:10.156 | INFO | "pinned" : " 29 GB" | 20260508 05:44:10.156 | INFO | } | 20260508 05:44:10.157 | INFO | Perf report for the last 5.1392 seconds: | 20260508 05:44:10.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:10.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:44:10.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:44:10.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:10.157 | INFO | Uploaded 36GiB to GPU in 2.54898s (14.1233GiB/s) | 20260508 05:44:10.157 | INFO | Downloaded 34GiB to GPU in 2.4201s (14.049GiB/s) | 20260508 05:44:15.296 | INFO | Analyzing chunk frames = [5690880,5691392) | 20260508 05:44:15.296 | INFO | MemInfo: { | 20260508 05:44:15.296 | INFO | "gpu" : " 17 GB", | 20260508 05:44:15.296 | INFO | "huge" : "", | 20260508 05:44:15.296 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:15.296 | INFO | "malloc" : " 14 GB", | 20260508 05:44:15.296 | INFO | "pinned" : " 29 GB" | 20260508 05:44:15.296 | INFO | } | 20260508 05:44:15.296 | INFO | Perf report for the last 5.13923 seconds: | 20260508 05:44:15.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:15.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 05:44:15.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:44:15.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:15.296 | INFO | Uploaded 36GiB to GPU in 2.53514s (14.2004GiB/s) | 20260508 05:44:15.296 | INFO | Downloaded 34GiB to GPU in 2.40595s (14.1317GiB/s) | 20260508 05:44:20.434 | INFO | Analyzing chunk frames = [5691392,5691904) | 20260508 05:44:20.434 | INFO | MemInfo: { | 20260508 05:44:20.434 | INFO | "gpu" : " 17 GB", | 20260508 05:44:20.434 | INFO | "huge" : "", | 20260508 05:44:20.435 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:20.435 | INFO | "malloc" : " 14 GB", | 20260508 05:44:20.435 | INFO | "pinned" : " 29 GB" | 20260508 05:44:20.435 | INFO | } | 20260508 05:44:20.435 | INFO | Perf report for the last 5.13892 seconds: | 20260508 05:44:20.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:20.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 05:44:20.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:44:20.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:20.435 | INFO | Uploaded 36GiB to GPU in 2.38751s (15.0785GiB/s) | 20260508 05:44:20.435 | INFO | Downloaded 34GiB to GPU in 2.549s (13.3386GiB/s) | 20260508 05:44:25.579 | INFO | Analyzing chunk frames = [5691904,5692416) | 20260508 05:44:25.579 | INFO | MemInfo: { | 20260508 05:44:25.579 | INFO | "gpu" : " 17 GB", | 20260508 05:44:25.579 | INFO | "huge" : "", | 20260508 05:44:25.579 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:25.579 | INFO | "malloc" : " 14 GB", | 20260508 05:44:25.579 | INFO | "pinned" : " 29 GB" | 20260508 05:44:25.579 | INFO | } | 20260508 05:44:25.580 | INFO | Perf report for the last 5.14474 seconds: | 20260508 05:44:25.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:25.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:44:25.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:44:25.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:25.580 | INFO | Uploaded 36GiB to GPU in 2.4557s (14.6598GiB/s) | 20260508 05:44:25.580 | INFO | Downloaded 34GiB to GPU in 2.42243s (14.0355GiB/s) | 20260508 05:44:30.718 | INFO | Analyzing chunk frames = [5692416,5692928) | 20260508 05:44:30.718 | INFO | MemInfo: { | 20260508 05:44:30.718 | INFO | "gpu" : " 17 GB", | 20260508 05:44:30.718 | INFO | "huge" : "", | 20260508 05:44:30.718 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:30.719 | INFO | "malloc" : " 14 GB", | 20260508 05:44:30.719 | INFO | "pinned" : " 29 GB" | 20260508 05:44:30.719 | INFO | } | 20260508 05:44:30.719 | INFO | Perf report for the last 5.13918 seconds: | 20260508 05:44:30.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:30.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:44:30.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 05:44:30.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:30.719 | INFO | Uploaded 36GiB to GPU in 2.47459s (14.5479GiB/s) | 20260508 05:44:30.719 | INFO | Downloaded 34GiB to GPU in 2.42196s (14.0382GiB/s) | 20260508 05:44:35.857 | INFO | Analyzing chunk frames = [5692928,5693440) | 20260508 05:44:35.857 | INFO | MemInfo: { | 20260508 05:44:35.857 | INFO | "gpu" : " 17 GB", | 20260508 05:44:35.857 | INFO | "huge" : "", | 20260508 05:44:35.857 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:35.857 | INFO | "malloc" : " 14 GB", | 20260508 05:44:35.857 | INFO | "pinned" : " 29 GB" | 20260508 05:44:35.857 | INFO | } | 20260508 05:44:35.858 | INFO | Perf report for the last 5.13885 seconds: | 20260508 05:44:35.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:35.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.858 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:44:35.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.858 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:44:35.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:35.859 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:44:35.859 | INFO | Total mem usage: 16.7 GiB | 20260508 05:44:35.859 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:44:35.859 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:44:35.859 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:44:35.859 | INFO | High Water Breakdown: | 20260508 05:44:35.859 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:44:35.859 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:44:35.859 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:44:35.859 | INFO | ------------------------------------------ | 20260508 05:44:35.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:44:35.859 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:44:35.859 | INFO | Total mem usage: 26.5 GiB | 20260508 05:44:35.859 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:44:35.859 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:44:35.859 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:44:35.859 | INFO | High Water Breakdown: | 20260508 05:44:35.859 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:44:35.859 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:44:35.859 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:44:35.859 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:44:35.859 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:44:35.859 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:44:35.859 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:44:35.859 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:44:35.859 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:44:35.859 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:44:35.859 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:44:35.859 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:44:35.859 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:44:35.859 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:44:35.859 | INFO | ------------------------------------------ | 20260508 05:44:35.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:44:35.859 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:44:35.859 | INFO | Total mem usage: 27.8 GiB | 20260508 05:44:35.859 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:44:35.859 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:44:35.859 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:44:35.859 | INFO | High Water Breakdown: | 20260508 05:44:35.859 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:44:35.859 | INFO | ------------------------------------------ | 20260508 05:44:35.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:44:35.859 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:44:35.859 | INFO | Total mem usage: 15.7 GiB | 20260508 05:44:35.859 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:44:35.859 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:44:35.859 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:44:35.859 | INFO | High Water Breakdown: | 20260508 05:44:35.859 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:44:35.859 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:44:35.859 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:44:35.859 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:44:35.859 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:44:35.859 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:44:35.859 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:44:35.859 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:44:35.859 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:44:35.859 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:44:35.859 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:44:35.859 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:44:35.859 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:44:35.859 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:44:35.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:44:35.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:44:35.859 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:44:35.859 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:44:35.859 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:44:35.859 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:44:35.859 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:44:35.859 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:44:35.859 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:44:35.859 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:44:35.859 | INFO | Uploaded 36GiB to GPU in 2.48193s (14.5049GiB/s) | 20260508 05:44:35.859 | INFO | Downloaded 34GiB to GPU in 2.43211s (13.9797GiB/s) | 20260508 05:44:35.859 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:44:35.859 | INFO | ------------------------------------------ | 20260508 05:44:35.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:44:35.859 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:44:35.859 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:44:35.859 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:44:35.859 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 1.24e+00 | 20260508 05:44:35.859 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.15e-02 / 8.18e-03 | 20260508 05:44:35.860 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.49e+00, 5.88e-01 / 1.94e-01 | 20260508 05:44:35.860 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.20e-01 | 20260508 05:44:35.860 | INFO | PerfReport: 15.91%, DME -- 4.16e+01 / 4.50e+01, 3.77e+00 / 1.67e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.54e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.57e+00, 4.16e-01 / 2.60e-01 | 20260508 05:44:35.860 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.21e+00, 2.55e-01 / 1.39e-01 | 20260508 05:44:35.860 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.50e-01, 5.02e-02 / 2.26e-02 | 20260508 05:44:35.860 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:44:35.860 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:44:35.860 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:44:35.860 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 24.35%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 2.19%, LockUpload -- 2.32e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 2.66%, LockDownload -- 2.83e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 15.67%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:44:35.860 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260508 05:44:35.956 | INFO | Starting a round of Baz IO | 20260508 05:44:39.379 | INFO | Populating RTMetrics report | 20260508 05:44:39.379 | INFO | RTMetrics report populated | 20260508 05:44:39.379 | INFO | Opening tmp RTMetrics json file | 20260508 05:44:39.391 | INFO | Serializing RTMetrics to json | 20260508 05:44:39.393 | INFO | Serialization done, swapping tmp file now... | 20260508 05:44:39.393 | INFO | RTMetrics done | 20260508 05:44:39.456 | INFO | Finished a round of Baz IO: Wrote 1.567 GiB in 3.5 seconds | 20260508 05:44:39.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.82% / 0.00% / 59.18% | 20260508 05:44:39.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.44328% / 3.97364e-06% / 6.75519e-05% | 20260508 05:44:39.456 | INFO | Baz IO overall: | 20260508 05:44:39.456 | INFO | BytesWritten : 2533694860129 | 20260508 05:44:39.456 | INFO | NumEvents : 1222195411014 | 20260508 05:44:40.996 | INFO | Analyzing chunk frames = [5693440,5693952) | 20260508 05:44:40.996 | INFO | MemInfo: { | 20260508 05:44:40.996 | INFO | "gpu" : " 17 GB", | 20260508 05:44:40.996 | INFO | "huge" : "", | 20260508 05:44:40.996 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:40.996 | INFO | "malloc" : " 14 GB", | 20260508 05:44:40.996 | INFO | "pinned" : " 29 GB" | 20260508 05:44:40.996 | INFO | } | 20260508 05:44:40.996 | INFO | Perf report for the last 5.13872 seconds: | 20260508 05:44:40.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:40.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:40.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:40.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:40.996 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 05:44:40.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:40.996 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 05:44:40.996 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 05:44:40.996 | INFO | BAZWRITER : 67.6%, 2.7, 93.6ms, 0 | 20260508 05:44:40.997 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:44:40.997 | INFO | Uploaded 36GiB to GPU in 2.58453s (13.9291GiB/s) | 20260508 05:44:40.997 | INFO | Downloaded 38GiB to GPU in 2.69165s (14.1177GiB/s) | 20260508 05:44:46.129 | INFO | Analyzing chunk frames = [5693952,5694464) | 20260508 05:44:46.129 | INFO | MemInfo: { | 20260508 05:44:46.129 | INFO | "gpu" : " 17 GB", | 20260508 05:44:46.129 | INFO | "huge" : "", | 20260508 05:44:46.129 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:46.129 | INFO | "malloc" : " 14 GB", | 20260508 05:44:46.129 | INFO | "pinned" : " 29 GB" | 20260508 05:44:46.129 | INFO | } | 20260508 05:44:46.129 | INFO | Perf report for the last 5.13299 seconds: | 20260508 05:44:46.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:46.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.129 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 05:44:46.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.129 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:44:46.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:46.130 | INFO | Uploaded 36GiB to GPU in 2.50748s (14.3571GiB/s) | 20260508 05:44:46.130 | INFO | Downloaded 34GiB to GPU in 2.43588s (13.958GiB/s) | 20260508 05:44:51.270 | INFO | Analyzing chunk frames = [5694464,5694976) | 20260508 05:44:51.270 | INFO | MemInfo: { | 20260508 05:44:51.270 | INFO | "gpu" : " 17 GB", | 20260508 05:44:51.270 | INFO | "huge" : "", | 20260508 05:44:51.270 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:51.270 | INFO | "malloc" : " 14 GB", | 20260508 05:44:51.270 | INFO | "pinned" : " 29 GB" | 20260508 05:44:51.270 | INFO | } | 20260508 05:44:51.270 | INFO | Perf report for the last 5.1409 seconds: | 20260508 05:44:51.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:51.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.270 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 05:44:51.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.270 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:44:51.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:51.271 | INFO | Uploaded 36GiB to GPU in 2.56219s (14.0505GiB/s) | 20260508 05:44:51.271 | INFO | Downloaded 34GiB to GPU in 2.41077s (14.1034GiB/s) | 20260508 05:44:56.415 | INFO | Analyzing chunk frames = [5694976,5695488) | 20260508 05:44:56.415 | INFO | MemInfo: { | 20260508 05:44:56.415 | INFO | "gpu" : " 17 GB", | 20260508 05:44:56.415 | INFO | "huge" : "", | 20260508 05:44:56.415 | INFO | "huge_shared" : " 29 GB", | 20260508 05:44:56.415 | INFO | "malloc" : " 14 GB", | 20260508 05:44:56.415 | INFO | "pinned" : " 29 GB" | 20260508 05:44:56.415 | INFO | } | 20260508 05:44:56.415 | INFO | Perf report for the last 5.14473 seconds: | 20260508 05:44:56.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:44:56.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 05:44:56.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:44:56.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:44:56.415 | INFO | Uploaded 36GiB to GPU in 2.56011s (14.0619GiB/s) | 20260508 05:44:56.415 | INFO | Downloaded 34GiB to GPU in 2.38671s (14.2455GiB/s) | 20260508 05:45:01.552 | INFO | Analyzing chunk frames = [5695488,5696000) | 20260508 05:45:01.552 | INFO | MemInfo: { | 20260508 05:45:01.552 | INFO | "gpu" : " 17 GB", | 20260508 05:45:01.552 | INFO | "huge" : "", | 20260508 05:45:01.552 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:01.552 | INFO | "malloc" : " 14 GB", | 20260508 05:45:01.552 | INFO | "pinned" : " 29 GB" | 20260508 05:45:01.553 | INFO | } | 20260508 05:45:01.553 | INFO | Perf report for the last 5.13767 seconds: | 20260508 05:45:01.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:01.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:45:01.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:45:01.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:01.553 | INFO | Uploaded 36GiB to GPU in 2.31179s (15.5723GiB/s) | 20260508 05:45:01.553 | INFO | Downloaded 34GiB to GPU in 2.49029s (13.653GiB/s) | 20260508 05:45:06.691 | INFO | Analyzing chunk frames = [5696000,5696512) | 20260508 05:45:06.691 | INFO | MemInfo: { | 20260508 05:45:06.691 | INFO | "gpu" : " 17 GB", | 20260508 05:45:06.691 | INFO | "huge" : "", | 20260508 05:45:06.691 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:06.691 | INFO | "malloc" : " 14 GB", | 20260508 05:45:06.691 | INFO | "pinned" : " 29 GB" | 20260508 05:45:06.691 | INFO | } | 20260508 05:45:06.691 | INFO | Perf report for the last 5.13851 seconds: | 20260508 05:45:06.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:06.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.692 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:45:06.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.692 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260508 05:45:06.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:06.692 | INFO | Uploaded 36GiB to GPU in 2.44182s (14.7431GiB/s) | 20260508 05:45:06.692 | INFO | Downloaded 34GiB to GPU in 2.4262s (14.0137GiB/s) | 20260508 05:45:11.824 | INFO | Analyzing chunk frames = [5696512,5697024) | 20260508 05:45:11.824 | INFO | MemInfo: { | 20260508 05:45:11.824 | INFO | "gpu" : " 17 GB", | 20260508 05:45:11.824 | INFO | "huge" : "", | 20260508 05:45:11.824 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:11.824 | INFO | "malloc" : " 14 GB", | 20260508 05:45:11.824 | INFO | "pinned" : " 29 GB" | 20260508 05:45:11.825 | INFO | } | 20260508 05:45:11.825 | INFO | Perf report for the last 5.13356 seconds: | 20260508 05:45:11.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:11.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.825 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:45:11.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.825 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 05:45:11.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:11.826 | INFO | Uploaded 36GiB to GPU in 2.47482s (14.5465GiB/s) | 20260508 05:45:11.826 | INFO | Downloaded 34GiB to GPU in 2.43902s (13.94GiB/s) | 20260508 05:45:16.972 | INFO | Analyzing chunk frames = [5697024,5697536) | 20260508 05:45:16.972 | INFO | MemInfo: { | 20260508 05:45:16.972 | INFO | "gpu" : " 17 GB", | 20260508 05:45:16.972 | INFO | "huge" : "", | 20260508 05:45:16.972 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:16.973 | INFO | "malloc" : " 14 GB", | 20260508 05:45:16.973 | INFO | "pinned" : " 29 GB" | 20260508 05:45:16.973 | INFO | } | 20260508 05:45:16.973 | INFO | Perf report for the last 5.14797 seconds: | 20260508 05:45:16.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:16.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 05:45:16.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 05:45:16.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:16.973 | INFO | Uploaded 36GiB to GPU in 2.46403s (14.6102GiB/s) | 20260508 05:45:16.974 | INFO | Downloaded 34GiB to GPU in 2.44039s (13.9322GiB/s) | 20260508 05:45:17.087 | INFO | Starting a round of Baz IO | 20260508 05:45:20.499 | INFO | Populating RTMetrics report | 20260508 05:45:20.499 | INFO | RTMetrics report populated | 20260508 05:45:20.499 | INFO | Opening tmp RTMetrics json file | 20260508 05:45:20.509 | INFO | Serializing RTMetrics to json | 20260508 05:45:20.510 | INFO | Serialization done, swapping tmp file now... | 20260508 05:45:20.511 | INFO | RTMetrics done | 20260508 05:45:20.572 | INFO | Finished a round of Baz IO: Wrote 1.568 GiB in 3.485 seconds | 20260508 05:45:20.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.81% / 0.00% / 59.18% | 20260508 05:45:20.573 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.442561% / 3.97364e-06% / 7.15256e-05% | 20260508 05:45:20.573 | INFO | Baz IO overall: | 20260508 05:45:20.573 | INFO | BytesWritten : 2535378140678 | 20260508 05:45:20.573 | INFO | NumEvents : 1222997920325 | 20260508 05:45:22.109 | INFO | Analyzing chunk frames = [5697536,5698048) | 20260508 05:45:22.109 | INFO | MemInfo: { | 20260508 05:45:22.109 | INFO | "gpu" : " 17 GB", | 20260508 05:45:22.109 | INFO | "huge" : "", | 20260508 05:45:22.109 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:22.109 | INFO | "malloc" : " 14 GB", | 20260508 05:45:22.109 | INFO | "pinned" : " 29 GB" | 20260508 05:45:22.109 | INFO | } | 20260508 05:45:22.109 | INFO | Perf report for the last 5.13656 seconds: | 20260508 05:45:22.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:22.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:22.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:22.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:22.109 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 05:45:22.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:22.109 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 05:45:22.109 | INFO | PRE_HQ : 66.6%, 1.8, 61.7ms, 0 | 20260508 05:45:22.109 | INFO | BAZWRITER : 67.4%, 2.9, 99.5ms, 0 | 20260508 05:45:22.109 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:45:22.110 | INFO | Uploaded 36GiB to GPU in 2.54684s (14.1352GiB/s) | 20260508 05:45:22.110 | INFO | Downloaded 38GiB to GPU in 2.70883s (14.0282GiB/s) | 20260508 05:45:27.249 | INFO | Analyzing chunk frames = [5698048,5698560) | 20260508 05:45:27.249 | INFO | MemInfo: { | 20260508 05:45:27.249 | INFO | "gpu" : " 17 GB", | 20260508 05:45:27.249 | INFO | "huge" : "", | 20260508 05:45:27.249 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:27.249 | INFO | "malloc" : " 14 GB", | 20260508 05:45:27.249 | INFO | "pinned" : " 29 GB" | 20260508 05:45:27.249 | INFO | } | 20260508 05:45:27.249 | INFO | Perf report for the last 5.14011 seconds: | 20260508 05:45:27.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:27.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.250 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:45:27.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.250 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:45:27.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:27.250 | INFO | Uploaded 36GiB to GPU in 2.47176s (14.5645GiB/s) | 20260508 05:45:27.250 | INFO | Downloaded 34GiB to GPU in 2.4567s (13.8397GiB/s) | 20260508 05:45:32.386 | INFO | Analyzing chunk frames = [5698560,5699072) | 20260508 05:45:32.386 | INFO | MemInfo: { | 20260508 05:45:32.386 | INFO | "gpu" : " 17 GB", | 20260508 05:45:32.386 | INFO | "huge" : "", | 20260508 05:45:32.386 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:32.386 | INFO | "malloc" : " 14 GB", | 20260508 05:45:32.386 | INFO | "pinned" : " 29 GB" | 20260508 05:45:32.386 | INFO | } | 20260508 05:45:32.386 | INFO | Perf report for the last 5.13691 seconds: | 20260508 05:45:32.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:32.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.386 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 05:45:32.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.386 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:45:32.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:32.387 | INFO | Uploaded 36GiB to GPU in 2.50788s (14.3547GiB/s) | 20260508 05:45:32.387 | INFO | Downloaded 34GiB to GPU in 2.42319s (14.0311GiB/s) | 20260508 05:45:37.528 | INFO | Analyzing chunk frames = [5699072,5699584) | 20260508 05:45:37.528 | INFO | MemInfo: { | 20260508 05:45:37.528 | INFO | "gpu" : " 17 GB", | 20260508 05:45:37.528 | INFO | "huge" : "", | 20260508 05:45:37.528 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:37.528 | INFO | "malloc" : " 14 GB", | 20260508 05:45:37.528 | INFO | "pinned" : " 29 GB" | 20260508 05:45:37.528 | INFO | } | 20260508 05:45:37.528 | INFO | Perf report for the last 5.14199 seconds: | 20260508 05:45:37.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:37.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.528 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 05:45:37.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.528 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:45:37.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:37.529 | INFO | Uploaded 36GiB to GPU in 2.51955s (14.2883GiB/s) | 20260508 05:45:37.529 | INFO | Downloaded 34GiB to GPU in 2.4173s (14.0653GiB/s) | 20260508 05:45:42.667 | INFO | Analyzing chunk frames = [5699584,5700096) | 20260508 05:45:42.667 | INFO | MemInfo: { | 20260508 05:45:42.667 | INFO | "gpu" : " 17 GB", | 20260508 05:45:42.667 | INFO | "huge" : "", | 20260508 05:45:42.667 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:42.667 | INFO | "malloc" : " 14 GB", | 20260508 05:45:42.667 | INFO | "pinned" : " 29 GB" | 20260508 05:45:42.667 | INFO | } | 20260508 05:45:42.667 | INFO | Perf report for the last 5.13876 seconds: | 20260508 05:45:42.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:42.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:45:42.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:45:42.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:42.667 | INFO | Uploaded 36GiB to GPU in 2.26782s (15.8743GiB/s) | 20260508 05:45:42.667 | INFO | Downloaded 34GiB to GPU in 2.49002s (13.6545GiB/s) | 20260508 05:45:47.805 | INFO | Analyzing chunk frames = [5700096,5700608) | 20260508 05:45:47.805 | INFO | MemInfo: { | 20260508 05:45:47.805 | INFO | "gpu" : " 17 GB", | 20260508 05:45:47.805 | INFO | "huge" : "", | 20260508 05:45:47.805 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:47.805 | INFO | "malloc" : " 14 GB", | 20260508 05:45:47.805 | INFO | "pinned" : " 29 GB" | 20260508 05:45:47.805 | INFO | } | 20260508 05:45:47.805 | INFO | Perf report for the last 5.13806 seconds: | 20260508 05:45:47.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:47.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.805 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:45:47.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.805 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 05:45:47.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:47.806 | INFO | Uploaded 36GiB to GPU in 2.43165s (14.8048GiB/s) | 20260508 05:45:47.806 | INFO | Downloaded 34GiB to GPU in 2.43475s (13.9645GiB/s) | 20260508 05:45:52.946 | INFO | Analyzing chunk frames = [5700608,5701120) | 20260508 05:45:52.946 | INFO | MemInfo: { | 20260508 05:45:52.946 | INFO | "gpu" : " 17 GB", | 20260508 05:45:52.946 | INFO | "huge" : "", | 20260508 05:45:52.946 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:52.946 | INFO | "malloc" : " 14 GB", | 20260508 05:45:52.946 | INFO | "pinned" : " 29 GB" | 20260508 05:45:52.947 | INFO | } | 20260508 05:45:52.947 | INFO | Perf report for the last 5.1414 seconds: | 20260508 05:45:52.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:52.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 05:45:52.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:45:52.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:52.947 | INFO | Uploaded 36GiB to GPU in 2.43192s (14.8031GiB/s) | 20260508 05:45:52.947 | INFO | Downloaded 34GiB to GPU in 2.42552s (14.0176GiB/s) | 20260508 05:45:58.078 | INFO | Analyzing chunk frames = [5701120,5701632) | 20260508 05:45:58.078 | INFO | MemInfo: { | 20260508 05:45:58.078 | INFO | "gpu" : " 17 GB", | 20260508 05:45:58.078 | INFO | "huge" : "", | 20260508 05:45:58.078 | INFO | "huge_shared" : " 29 GB", | 20260508 05:45:58.078 | INFO | "malloc" : " 14 GB", | 20260508 05:45:58.078 | INFO | "pinned" : " 29 GB" | 20260508 05:45:58.078 | INFO | } | 20260508 05:45:58.078 | INFO | Perf report for the last 5.13158 seconds: | 20260508 05:45:58.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:45:58.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.078 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 05:45:58.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.079 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 05:45:58.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:45:58.079 | INFO | Uploaded 36GiB to GPU in 2.48572s (14.4827GiB/s) | 20260508 05:45:58.079 | INFO | Downloaded 34GiB to GPU in 2.45331s (13.8588GiB/s) | 20260508 05:45:58.079 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:45:58.079 | INFO | Total mem usage: 16.7 GiB | 20260508 05:45:58.079 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:45:58.079 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:45:58.079 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:45:58.079 | INFO | High Water Breakdown: | 20260508 05:45:58.079 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:45:58.079 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:45:58.079 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:45:58.079 | INFO | ------------------------------------------ | 20260508 05:45:58.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:45:58.079 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:45:58.079 | INFO | Total mem usage: 26.5 GiB | 20260508 05:45:58.079 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:45:58.079 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:45:58.079 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:45:58.079 | INFO | High Water Breakdown: | 20260508 05:45:58.079 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:45:58.079 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:45:58.079 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:45:58.079 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:45:58.079 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:45:58.079 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:45:58.079 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:45:58.079 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:45:58.079 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:45:58.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:45:58.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:45:58.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:45:58.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:45:58.079 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:45:58.079 | INFO | ------------------------------------------ | 20260508 05:45:58.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:45:58.079 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:45:58.079 | INFO | Total mem usage: 27.8 GiB | 20260508 05:45:58.079 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:45:58.079 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:45:58.079 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:45:58.079 | INFO | High Water Breakdown: | 20260508 05:45:58.079 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:45:58.079 | INFO | ------------------------------------------ | 20260508 05:45:58.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:45:58.079 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:45:58.079 | INFO | Total mem usage: 15.7 GiB | 20260508 05:45:58.079 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:45:58.079 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:45:58.079 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:45:58.079 | INFO | High Water Breakdown: | 20260508 05:45:58.079 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:45:58.079 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:45:58.079 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:45:58.079 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:45:58.079 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:45:58.079 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:45:58.079 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:45:58.079 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:45:58.079 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:45:58.079 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:45:58.079 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:45:58.079 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:45:58.079 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:45:58.079 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:45:58.079 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:45:58.079 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:45:58.079 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:45:58.079 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:45:58.079 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:45:58.079 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:45:58.079 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:45:58.079 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:45:58.079 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:45:58.079 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:45:58.079 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:45:58.079 | INFO | ------------------------------------------ | 20260508 05:45:58.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:45:58.079 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:45:58.080 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:45:58.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:45:58.080 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 7.29e-01 | 20260508 05:45:58.080 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.07e-01, 1.23e-02 / 6.63e-03 | 20260508 05:45:58.080 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.42e+00, 5.91e-01 / 2.04e-01 | 20260508 05:45:58.080 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.20e-01 | 20260508 05:45:58.080 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.53e+01, 3.74e+00 / 1.02e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.76e+00 / 1.20e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.40e+00 / 2.63e+00, 4.18e-01 / 2.34e-01 | 20260508 05:45:58.080 | INFO | PerfReport: 3.19%, Metrics -- 1.04e+00 / 1.26e+00, 2.56e-01 / 1.68e-01 | 20260508 05:45:58.080 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.19e-01 / 3.58e-01, 5.01e-02 / 2.52e-02 | 20260508 05:45:58.080 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:45:58.080 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:45:58.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:45:58.080 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 24.29%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 24.27%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 2.23%, LockUpload -- 2.37e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 2.65%, LockDownload -- 2.81e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 15.70%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:45:58.080 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.21e-02 / 0.00e+00 | 20260508 05:45:58.193 | INFO | Starting a round of Baz IO | 20260508 05:46:01.600 | INFO | Populating RTMetrics report | 20260508 05:46:01.600 | INFO | RTMetrics report populated | 20260508 05:46:01.600 | INFO | Opening tmp RTMetrics json file | 20260508 05:46:01.613 | INFO | Serializing RTMetrics to json | 20260508 05:46:01.614 | INFO | Serialization done, swapping tmp file now... | 20260508 05:46:01.615 | INFO | RTMetrics done | 20260508 05:46:01.672 | INFO | Finished a round of Baz IO: Wrote 1.568 GiB in 3.479 seconds | 20260508 05:46:01.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.81% / 0.00% / 59.19% | 20260508 05:46:01.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.443296% / 0% / 8.34465e-05% | 20260508 05:46:01.672 | INFO | Baz IO overall: | 20260508 05:46:01.672 | INFO | BytesWritten : 2537061737685 | 20260508 05:46:01.672 | INFO | NumEvents : 1223800640879 | 20260508 05:46:03.222 | INFO | Analyzing chunk frames = [5701632,5702144) | 20260508 05:46:03.222 | INFO | MemInfo: { | 20260508 05:46:03.222 | INFO | "gpu" : " 17 GB", | 20260508 05:46:03.222 | INFO | "huge" : "", | 20260508 05:46:03.222 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:03.222 | INFO | "malloc" : " 14 GB", | 20260508 05:46:03.222 | INFO | "pinned" : " 29 GB" | 20260508 05:46:03.222 | INFO | } | 20260508 05:46:03.222 | INFO | Perf report for the last 5.1441 seconds: | 20260508 05:46:03.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:03.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:03.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:03.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:03.223 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 05:46:03.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:03.223 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 05:46:03.223 | INFO | PRE_HQ : 66.1%, 1.8, 62.1ms, 0 | 20260508 05:46:03.223 | INFO | BAZWRITER : 67.0%, 2.9, 99.1ms, 0 | 20260508 05:46:03.223 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:46:03.223 | INFO | Uploaded 36GiB to GPU in 2.49959s (14.4024GiB/s) | 20260508 05:46:03.223 | INFO | Downloaded 38GiB to GPU in 2.6865s (14.1448GiB/s) | 20260508 05:46:08.361 | INFO | Analyzing chunk frames = [5702144,5702656) | 20260508 05:46:08.361 | INFO | MemInfo: { | 20260508 05:46:08.361 | INFO | "gpu" : " 17 GB", | 20260508 05:46:08.361 | INFO | "huge" : "", | 20260508 05:46:08.361 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:08.361 | INFO | "malloc" : " 14 GB", | 20260508 05:46:08.361 | INFO | "pinned" : " 29 GB" | 20260508 05:46:08.361 | INFO | } | 20260508 05:46:08.361 | INFO | Perf report for the last 5.13867 seconds: | 20260508 05:46:08.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:08.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:46:08.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:46:08.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:08.361 | INFO | Uploaded 36GiB to GPU in 2.51646s (14.3058GiB/s) | 20260508 05:46:08.361 | INFO | Downloaded 34GiB to GPU in 2.42647s (14.0122GiB/s) | 20260508 05:46:13.502 | INFO | Analyzing chunk frames = [5702656,5703168) | 20260508 05:46:13.502 | INFO | MemInfo: { | 20260508 05:46:13.502 | INFO | "gpu" : " 17 GB", | 20260508 05:46:13.502 | INFO | "huge" : "", | 20260508 05:46:13.503 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:13.503 | INFO | "malloc" : " 14 GB", | 20260508 05:46:13.503 | INFO | "pinned" : " 29 GB" | 20260508 05:46:13.503 | INFO | } | 20260508 05:46:13.503 | INFO | Perf report for the last 5.14174 seconds: | 20260508 05:46:13.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:13.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 05:46:13.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:46:13.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:13.503 | INFO | Uploaded 36GiB to GPU in 2.54414s (14.1501GiB/s) | 20260508 05:46:13.503 | INFO | Downloaded 34GiB to GPU in 2.39698s (14.1845GiB/s) | 20260508 05:46:18.642 | INFO | Analyzing chunk frames = [5703168,5703680) | 20260508 05:46:18.642 | INFO | MemInfo: { | 20260508 05:46:18.642 | INFO | "gpu" : " 17 GB", | 20260508 05:46:18.642 | INFO | "huge" : "", | 20260508 05:46:18.642 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:18.642 | INFO | "malloc" : " 14 GB", | 20260508 05:46:18.643 | INFO | "pinned" : " 29 GB" | 20260508 05:46:18.643 | INFO | } | 20260508 05:46:18.643 | INFO | Perf report for the last 5.13996 seconds: | 20260508 05:46:18.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:18.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | ANALYSIS : 66.8%, 2.9, 102.4ms, 0 | 20260508 05:46:18.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | PRE_HQ : 24.6%, 1.0, 12.6ms, 0 | 20260508 05:46:18.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:18.643 | INFO | Uploaded 36GiB to GPU in 2.56189s (14.0521GiB/s) | 20260508 05:46:18.643 | INFO | Downloaded 34GiB to GPU in 2.38805s (14.2375GiB/s) | 20260508 05:46:23.782 | INFO | Analyzing chunk frames = [5703680,5704192) | 20260508 05:46:23.782 | INFO | MemInfo: { | 20260508 05:46:23.782 | INFO | "gpu" : " 17 GB", | 20260508 05:46:23.782 | INFO | "huge" : "", | 20260508 05:46:23.782 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:23.782 | INFO | "malloc" : " 14 GB", | 20260508 05:46:23.782 | INFO | "pinned" : " 29 GB" | 20260508 05:46:23.782 | INFO | } | 20260508 05:46:23.782 | INFO | Perf report for the last 5.13907 seconds: | 20260508 05:46:23.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:23.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.782 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 05:46:23.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.782 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:46:23.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:23.783 | INFO | Uploaded 36GiB to GPU in 2.39695s (15.0191GiB/s) | 20260508 05:46:23.783 | INFO | Downloaded 34GiB to GPU in 2.51554s (13.516GiB/s) | 20260508 05:46:28.919 | INFO | Analyzing chunk frames = [5704192,5704704) | 20260508 05:46:28.919 | INFO | MemInfo: { | 20260508 05:46:28.919 | INFO | "gpu" : " 17 GB", | 20260508 05:46:28.919 | INFO | "huge" : "", | 20260508 05:46:28.919 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:28.919 | INFO | "malloc" : " 14 GB", | 20260508 05:46:28.919 | INFO | "pinned" : " 29 GB" | 20260508 05:46:28.919 | INFO | } | 20260508 05:46:28.919 | INFO | Perf report for the last 5.13727 seconds: | 20260508 05:46:28.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:28.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:46:28.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:46:28.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:28.919 | INFO | Uploaded 36GiB to GPU in 2.45155s (14.6846GiB/s) | 20260508 05:46:28.919 | INFO | Downloaded 34GiB to GPU in 2.42681s (14.0102GiB/s) | 20260508 05:46:34.063 | INFO | Analyzing chunk frames = [5704704,5705216) | 20260508 05:46:34.063 | INFO | MemInfo: { | 20260508 05:46:34.063 | INFO | "gpu" : " 17 GB", | 20260508 05:46:34.063 | INFO | "huge" : "", | 20260508 05:46:34.063 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:34.064 | INFO | "malloc" : " 14 GB", | 20260508 05:46:34.064 | INFO | "pinned" : " 29 GB" | 20260508 05:46:34.064 | INFO | } | 20260508 05:46:34.064 | INFO | Perf report for the last 5.14456 seconds: | 20260508 05:46:34.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:34.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:46:34.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:46:34.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:34.064 | INFO | Uploaded 36GiB to GPU in 2.43053s (14.8116GiB/s) | 20260508 05:46:34.064 | INFO | Downloaded 34GiB to GPU in 2.42359s (14.0288GiB/s) | 20260508 05:46:39.199 | INFO | Analyzing chunk frames = [5705216,5705728) | 20260508 05:46:39.199 | INFO | MemInfo: { | 20260508 05:46:39.199 | INFO | "gpu" : " 17 GB", | 20260508 05:46:39.199 | INFO | "huge" : "", | 20260508 05:46:39.199 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:39.199 | INFO | "malloc" : " 14 GB", | 20260508 05:46:39.199 | INFO | "pinned" : " 29 GB" | 20260508 05:46:39.199 | INFO | } | 20260508 05:46:39.199 | INFO | Perf report for the last 5.13543 seconds: | 20260508 05:46:39.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:39.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.199 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 05:46:39.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.199 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 05:46:39.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:39.200 | INFO | Uploaded 36GiB to GPU in 2.48704s (14.475GiB/s) | 20260508 05:46:39.200 | INFO | Downloaded 34GiB to GPU in 2.44416s (13.9107GiB/s) | 20260508 05:46:39.313 | INFO | Starting a round of Baz IO | 20260508 05:46:42.741 | INFO | Populating RTMetrics report | 20260508 05:46:42.742 | INFO | RTMetrics report populated | 20260508 05:46:42.742 | INFO | Opening tmp RTMetrics json file | 20260508 05:46:42.743 | INFO | Serializing RTMetrics to json | 20260508 05:46:42.744 | INFO | Serialization done, swapping tmp file now... | 20260508 05:46:42.744 | INFO | RTMetrics done | 20260508 05:46:42.814 | INFO | Finished a round of Baz IO: Wrote 1.567 GiB in 3.5 seconds | 20260508 05:46:42.814 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.81% / 0.00% / 59.19% | 20260508 05:46:42.814 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.442378% / 1.19209e-05% / 7.94729e-05% | 20260508 05:46:42.814 | INFO | Baz IO overall: | 20260508 05:46:42.814 | INFO | BytesWritten : 2538744234496 | 20260508 05:46:42.814 | INFO | NumEvents : 1224602677264 | 20260508 05:46:44.340 | INFO | Analyzing chunk frames = [5705728,5706240) | 20260508 05:46:44.340 | INFO | MemInfo: { | 20260508 05:46:44.340 | INFO | "gpu" : " 17 GB", | 20260508 05:46:44.340 | INFO | "huge" : "", | 20260508 05:46:44.340 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:44.340 | INFO | "malloc" : " 14 GB", | 20260508 05:46:44.340 | INFO | "pinned" : " 29 GB" | 20260508 05:46:44.340 | INFO | } | 20260508 05:46:44.340 | INFO | Perf report for the last 5.14077 seconds: | 20260508 05:46:44.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:44.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:44.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:44.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:44.340 | INFO | ANALYSIS : 68.6%, 3.0, 106.4ms, 0 | 20260508 05:46:44.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:44.340 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 05:46:44.340 | INFO | PRE_HQ : 66.8%, 1.8, 61.7ms, 0 | 20260508 05:46:44.340 | INFO | BAZWRITER : 67.5%, 2.9, 99.4ms, 0 | 20260508 05:46:44.340 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 05:46:44.340 | INFO | Uploaded 36GiB to GPU in 2.54788s (14.1294GiB/s) | 20260508 05:46:44.340 | INFO | Downloaded 38GiB to GPU in 2.71983s (13.9714GiB/s) | 20260508 05:46:49.477 | INFO | Analyzing chunk frames = [5706240,5706752) | 20260508 05:46:49.477 | INFO | MemInfo: { | 20260508 05:46:49.477 | INFO | "gpu" : " 17 GB", | 20260508 05:46:49.477 | INFO | "huge" : "", | 20260508 05:46:49.477 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:49.477 | INFO | "malloc" : " 14 GB", | 20260508 05:46:49.477 | INFO | "pinned" : " 29 GB" | 20260508 05:46:49.477 | INFO | } | 20260508 05:46:49.477 | INFO | Perf report for the last 5.13752 seconds: | 20260508 05:46:49.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:49.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.478 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:46:49.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.478 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:46:49.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:49.478 | INFO | Uploaded 36GiB to GPU in 2.4598s (14.6353GiB/s) | 20260508 05:46:49.478 | INFO | Downloaded 34GiB to GPU in 2.45114s (13.8711GiB/s) | 20260508 05:46:54.623 | INFO | Analyzing chunk frames = [5706752,5707264) | 20260508 05:46:54.623 | INFO | MemInfo: { | 20260508 05:46:54.623 | INFO | "gpu" : " 17 GB", | 20260508 05:46:54.623 | INFO | "huge" : "", | 20260508 05:46:54.623 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:54.623 | INFO | "malloc" : " 14 GB", | 20260508 05:46:54.623 | INFO | "pinned" : " 29 GB" | 20260508 05:46:54.623 | INFO | } | 20260508 05:46:54.623 | INFO | Perf report for the last 5.14555 seconds: | 20260508 05:46:54.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:54.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 05:46:54.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:46:54.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:54.623 | INFO | Uploaded 36GiB to GPU in 2.50814s (14.3533GiB/s) | 20260508 05:46:54.623 | INFO | Downloaded 34GiB to GPU in 2.42426s (14.0249GiB/s) | 20260508 05:46:59.760 | INFO | Analyzing chunk frames = [5707264,5707776) | 20260508 05:46:59.760 | INFO | MemInfo: { | 20260508 05:46:59.760 | INFO | "gpu" : " 17 GB", | 20260508 05:46:59.760 | INFO | "huge" : "", | 20260508 05:46:59.760 | INFO | "huge_shared" : " 29 GB", | 20260508 05:46:59.760 | INFO | "malloc" : " 14 GB", | 20260508 05:46:59.760 | INFO | "pinned" : " 29 GB" | 20260508 05:46:59.760 | INFO | } | 20260508 05:46:59.760 | INFO | Perf report for the last 5.1371 seconds: | 20260508 05:46:59.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:46:59.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | ANALYSIS : 66.7%, 2.9, 103.1ms, 0 | 20260508 05:46:59.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:46:59.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:46:59.760 | INFO | Uploaded 36GiB to GPU in 2.51111s (14.3363GiB/s) | 20260508 05:46:59.760 | INFO | Downloaded 34GiB to GPU in 2.40496s (14.1375GiB/s) | 20260508 05:47:04.891 | INFO | Analyzing chunk frames = [5707776,5708288) | 20260508 05:47:04.891 | INFO | MemInfo: { | 20260508 05:47:04.891 | INFO | "gpu" : " 17 GB", | 20260508 05:47:04.891 | INFO | "huge" : "", | 20260508 05:47:04.891 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:04.891 | INFO | "malloc" : " 14 GB", | 20260508 05:47:04.891 | INFO | "pinned" : " 29 GB" | 20260508 05:47:04.891 | INFO | } | 20260508 05:47:04.891 | INFO | Perf report for the last 5.13109 seconds: | 20260508 05:47:04.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:04.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.891 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 05:47:04.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.891 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:47:04.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:04.892 | INFO | Uploaded 36GiB to GPU in 2.26676s (15.8817GiB/s) | 20260508 05:47:04.892 | INFO | Downloaded 34GiB to GPU in 2.4826s (13.6953GiB/s) | 20260508 05:47:10.032 | INFO | Analyzing chunk frames = [5708288,5708800) | 20260508 05:47:10.032 | INFO | MemInfo: { | 20260508 05:47:10.032 | INFO | "gpu" : " 17 GB", | 20260508 05:47:10.032 | INFO | "huge" : "", | 20260508 05:47:10.032 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:10.032 | INFO | "malloc" : " 14 GB", | 20260508 05:47:10.032 | INFO | "pinned" : " 29 GB" | 20260508 05:47:10.032 | INFO | } | 20260508 05:47:10.032 | INFO | Perf report for the last 5.14106 seconds: | 20260508 05:47:10.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:10.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.032 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:47:10.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.032 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:47:10.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:10.033 | INFO | Uploaded 36GiB to GPU in 2.41641s (14.8981GiB/s) | 20260508 05:47:10.033 | INFO | Downloaded 34GiB to GPU in 2.42462s (14.0228GiB/s) | 20260508 05:47:15.176 | INFO | Analyzing chunk frames = [5708800,5709312) | 20260508 05:47:15.176 | INFO | MemInfo: { | 20260508 05:47:15.176 | INFO | "gpu" : " 17 GB", | 20260508 05:47:15.176 | INFO | "huge" : "", | 20260508 05:47:15.176 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:15.176 | INFO | "malloc" : " 14 GB", | 20260508 05:47:15.176 | INFO | "pinned" : " 29 GB" | 20260508 05:47:15.176 | INFO | } | 20260508 05:47:15.176 | INFO | Perf report for the last 5.14398 seconds: | 20260508 05:47:15.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:15.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.176 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:47:15.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.176 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 05:47:15.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:15.177 | INFO | Uploaded 36GiB to GPU in 2.42115s (14.869GiB/s) | 20260508 05:47:15.177 | INFO | Downloaded 34GiB to GPU in 2.42911s (13.9969GiB/s) | 20260508 05:47:20.315 | INFO | Analyzing chunk frames = [5709312,5709824) | 20260508 05:47:20.315 | INFO | MemInfo: { | 20260508 05:47:20.315 | INFO | "gpu" : " 17 GB", | 20260508 05:47:20.315 | INFO | "huge" : "", | 20260508 05:47:20.315 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:20.315 | INFO | "malloc" : " 14 GB", | 20260508 05:47:20.315 | INFO | "pinned" : " 29 GB" | 20260508 05:47:20.315 | INFO | } | 20260508 05:47:20.315 | INFO | Perf report for the last 5.1391 seconds: | 20260508 05:47:20.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:20.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.316 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:47:20.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.316 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 05:47:20.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:20.316 | INFO | Uploaded 36GiB to GPU in 2.46387s (14.6112GiB/s) | 20260508 05:47:20.316 | INFO | Downloaded 34GiB to GPU in 2.44367s (13.9135GiB/s) | 20260508 05:47:20.316 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:47:20.316 | INFO | Total mem usage: 16.7 GiB | 20260508 05:47:20.316 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:47:20.316 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:47:20.316 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:47:20.316 | INFO | High Water Breakdown: | 20260508 05:47:20.316 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:47:20.316 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:47:20.316 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:47:20.316 | INFO | ------------------------------------------ | 20260508 05:47:20.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:47:20.316 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:47:20.316 | INFO | Total mem usage: 26.5 GiB | 20260508 05:47:20.316 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:47:20.316 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 05:47:20.316 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 05:47:20.316 | INFO | High Water Breakdown: | 20260508 05:47:20.316 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:47:20.316 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:47:20.316 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:47:20.316 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:47:20.316 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:47:20.316 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:47:20.316 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:47:20.316 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:47:20.316 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:47:20.316 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:47:20.316 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:47:20.316 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:47:20.316 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:47:20.316 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:47:20.316 | INFO | ------------------------------------------ | 20260508 05:47:20.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:47:20.316 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:47:20.316 | INFO | Total mem usage: 27.8 GiB | 20260508 05:47:20.316 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:47:20.316 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:47:20.316 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:47:20.316 | INFO | High Water Breakdown: | 20260508 05:47:20.316 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:47:20.317 | INFO | ------------------------------------------ | 20260508 05:47:20.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:47:20.317 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:47:20.317 | INFO | Total mem usage: 15.7 GiB | 20260508 05:47:20.317 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:47:20.317 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:47:20.317 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:47:20.317 | INFO | High Water Breakdown: | 20260508 05:47:20.317 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:47:20.317 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:47:20.317 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:47:20.317 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:47:20.317 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:47:20.317 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:47:20.317 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:47:20.317 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:47:20.317 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:47:20.317 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:47:20.317 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:47:20.317 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:47:20.317 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:47:20.317 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:47:20.317 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:47:20.317 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:47:20.317 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:47:20.317 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:47:20.317 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:47:20.317 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:47:20.317 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:47:20.317 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:47:20.317 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:47:20.317 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:47:20.317 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:47:20.317 | INFO | ------------------------------------------ | 20260508 05:47:20.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:47:20.317 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:47:20.317 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:47:20.317 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:47:20.317 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 5.19e-01 | 20260508 05:47:20.317 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.05e-01, 1.16e-02 / 4.09e-03 | 20260508 05:47:20.317 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.40e+00, 5.89e-01 / 2.46e-01 | 20260508 05:47:20.317 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.12e-01 / 1.17e-01 | 20260508 05:47:20.317 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.51e+01, 3.84e+00 / 1.19e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.85e+00 / 1.52e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.13e-01 / 2.26e-01 | 20260508 05:47:20.317 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.25e+00, 2.58e-01 / 1.42e-01 | 20260508 05:47:20.317 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.53e-01, 5.05e-02 / 1.86e-02 | 20260508 05:47:20.317 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:47:20.317 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:47:20.317 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:47:20.317 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 24.23%, Upload -- 2.57e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 24.24%, Download -- 2.57e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 30.91%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 2.23%, LockUpload -- 2.37e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 2.64%, LockDownload -- 2.81e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 15.73%, LockCompute -- 1.67e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:47:20.317 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260508 05:47:20.446 | INFO | Starting a round of Baz IO | 20260508 05:47:23.849 | INFO | Populating RTMetrics report | 20260508 05:47:23.849 | INFO | RTMetrics report populated | 20260508 05:47:23.849 | INFO | Opening tmp RTMetrics json file | 20260508 05:47:23.860 | INFO | Serializing RTMetrics to json | 20260508 05:47:23.862 | INFO | Serialization done, swapping tmp file now... | 20260508 05:47:23.862 | INFO | RTMetrics done | 20260508 05:47:23.920 | INFO | Finished a round of Baz IO: Wrote 1.566 GiB in 3.474 seconds | 20260508 05:47:23.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.81% / 0.00% / 59.19% | 20260508 05:47:23.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.440772% / 3.97364e-06% / 5.16574e-05% | 20260508 05:47:23.920 | INFO | Baz IO overall: | 20260508 05:47:23.920 | INFO | BytesWritten : 2540425797343 | 20260508 05:47:23.920 | INFO | NumEvents : 1225404349851 | 20260508 05:47:25.453 | INFO | Analyzing chunk frames = [5709824,5710336) | 20260508 05:47:25.453 | INFO | MemInfo: { | 20260508 05:47:25.453 | INFO | "gpu" : " 17 GB", | 20260508 05:47:25.453 | INFO | "huge" : "", | 20260508 05:47:25.453 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:25.453 | INFO | "malloc" : " 14 GB", | 20260508 05:47:25.453 | INFO | "pinned" : " 29 GB" | 20260508 05:47:25.453 | INFO | } | 20260508 05:47:25.453 | INFO | Perf report for the last 5.13805 seconds: | 20260508 05:47:25.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:25.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:25.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:25.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:25.454 | INFO | ANALYSIS : 68.1%, 3.0, 106.0ms, 0 | 20260508 05:47:25.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:25.454 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:47:25.454 | INFO | PRE_HQ : 66.5%, 1.8, 61.6ms, 0 | 20260508 05:47:25.454 | INFO | BAZWRITER : 67.4%, 2.8, 98.0ms, 0 | 20260508 05:47:25.454 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:47:25.454 | INFO | Uploaded 36GiB to GPU in 2.54166s (14.164GiB/s) | 20260508 05:47:25.454 | INFO | Downloaded 38GiB to GPU in 2.70076s (14.0701GiB/s) | 20260508 05:47:30.590 | INFO | Analyzing chunk frames = [5710336,5710848) | 20260508 05:47:30.590 | INFO | MemInfo: { | 20260508 05:47:30.590 | INFO | "gpu" : " 17 GB", | 20260508 05:47:30.590 | INFO | "huge" : "", | 20260508 05:47:30.590 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:30.590 | INFO | "malloc" : " 14 GB", | 20260508 05:47:30.590 | INFO | "pinned" : " 29 GB" | 20260508 05:47:30.590 | INFO | } | 20260508 05:47:30.590 | INFO | Perf report for the last 5.13634 seconds: | 20260508 05:47:30.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:30.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:47:30.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:47:30.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:30.590 | INFO | Uploaded 36GiB to GPU in 2.49165s (14.4483GiB/s) | 20260508 05:47:30.590 | INFO | Downloaded 34GiB to GPU in 2.45512s (13.8486GiB/s) | 20260508 05:47:35.732 | INFO | Analyzing chunk frames = [5710848,5711360) | 20260508 05:47:35.732 | INFO | MemInfo: { | 20260508 05:47:35.732 | INFO | "gpu" : " 17 GB", | 20260508 05:47:35.732 | INFO | "huge" : "", | 20260508 05:47:35.732 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:35.732 | INFO | "malloc" : " 14 GB", | 20260508 05:47:35.732 | INFO | "pinned" : " 29 GB" | 20260508 05:47:35.732 | INFO | } | 20260508 05:47:35.732 | INFO | Perf report for the last 5.14215 seconds: | 20260508 05:47:35.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:35.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.732 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 05:47:35.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.732 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:47:35.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:35.733 | INFO | Uploaded 36GiB to GPU in 2.53352s (14.2095GiB/s) | 20260508 05:47:35.733 | INFO | Downloaded 34GiB to GPU in 2.43101s (13.986GiB/s) | 20260508 05:47:40.870 | INFO | Analyzing chunk frames = [5711360,5711872) | 20260508 05:47:40.870 | INFO | MemInfo: { | 20260508 05:47:40.870 | INFO | "gpu" : " 17 GB", | 20260508 05:47:40.870 | INFO | "huge" : "", | 20260508 05:47:40.870 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:40.870 | INFO | "malloc" : " 14 GB", | 20260508 05:47:40.870 | INFO | "pinned" : " 29 GB" | 20260508 05:47:40.870 | INFO | } | 20260508 05:47:40.870 | INFO | Perf report for the last 5.13783 seconds: | 20260508 05:47:40.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:40.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 05:47:40.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:47:40.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:40.870 | INFO | Uploaded 36GiB to GPU in 2.53153s (14.2206GiB/s) | 20260508 05:47:40.870 | INFO | Downloaded 34GiB to GPU in 2.39575s (14.1918GiB/s) | 20260508 05:47:46.018 | INFO | Analyzing chunk frames = [5711872,5712384) | 20260508 05:47:46.018 | INFO | MemInfo: { | 20260508 05:47:46.018 | INFO | "gpu" : " 17 GB", | 20260508 05:47:46.018 | INFO | "huge" : "", | 20260508 05:47:46.018 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:46.018 | INFO | "malloc" : " 14 GB", | 20260508 05:47:46.018 | INFO | "pinned" : " 29 GB" | 20260508 05:47:46.018 | INFO | } | 20260508 05:47:46.018 | INFO | Perf report for the last 5.14806 seconds: | 20260508 05:47:46.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:46.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:47:46.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:47:46.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:46.018 | INFO | Uploaded 36GiB to GPU in 2.3484s (15.3296GiB/s) | 20260508 05:47:46.018 | INFO | Downloaded 34GiB to GPU in 2.51548s (13.5163GiB/s) | 20260508 05:47:51.148 | INFO | Analyzing chunk frames = [5712384,5712896) | 20260508 05:47:51.148 | INFO | MemInfo: { | 20260508 05:47:51.148 | INFO | "gpu" : " 17 GB", | 20260508 05:47:51.149 | INFO | "huge" : "", | 20260508 05:47:51.149 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:51.149 | INFO | "malloc" : " 14 GB", | 20260508 05:47:51.149 | INFO | "pinned" : " 29 GB" | 20260508 05:47:51.149 | INFO | } | 20260508 05:47:51.149 | INFO | Perf report for the last 5.13085 seconds: | 20260508 05:47:51.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:51.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 05:47:51.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:47:51.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:51.149 | INFO | Uploaded 36GiB to GPU in 2.45774s (14.6476GiB/s) | 20260508 05:47:51.149 | INFO | Downloaded 34GiB to GPU in 2.45183s (13.8672GiB/s) | 20260508 05:47:56.293 | INFO | Analyzing chunk frames = [5712896,5713408) | 20260508 05:47:56.293 | INFO | MemInfo: { | 20260508 05:47:56.293 | INFO | "gpu" : " 17 GB", | 20260508 05:47:56.293 | INFO | "huge" : "", | 20260508 05:47:56.293 | INFO | "huge_shared" : " 29 GB", | 20260508 05:47:56.294 | INFO | "malloc" : " 14 GB", | 20260508 05:47:56.294 | INFO | "pinned" : " 29 GB" | 20260508 05:47:56.294 | INFO | } | 20260508 05:47:56.294 | INFO | Perf report for the last 5.14493 seconds: | 20260508 05:47:56.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:47:56.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 05:47:56.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:47:56.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:47:56.294 | INFO | Uploaded 36GiB to GPU in 2.45639s (14.6557GiB/s) | 20260508 05:47:56.294 | INFO | Downloaded 34GiB to GPU in 2.43467s (13.9649GiB/s) | 20260508 05:48:01.430 | INFO | Analyzing chunk frames = [5713408,5713920) | 20260508 05:48:01.430 | INFO | MemInfo: { | 20260508 05:48:01.430 | INFO | "gpu" : " 17 GB", | 20260508 05:48:01.430 | INFO | "huge" : "", | 20260508 05:48:01.430 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:01.430 | INFO | "malloc" : " 14 GB", | 20260508 05:48:01.430 | INFO | "pinned" : " 29 GB" | 20260508 05:48:01.430 | INFO | } | 20260508 05:48:01.430 | INFO | Perf report for the last 5.13622 seconds: | 20260508 05:48:01.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:01.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 05:48:01.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 05:48:01.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:01.430 | INFO | Uploaded 36GiB to GPU in 2.48424s (14.4913GiB/s) | 20260508 05:48:01.430 | INFO | Downloaded 34GiB to GPU in 2.45537s (13.8472GiB/s) | 20260508 05:48:01.543 | INFO | Starting a round of Baz IO | 20260508 05:48:04.950 | INFO | Populating RTMetrics report | 20260508 05:48:04.950 | INFO | RTMetrics report populated | 20260508 05:48:04.950 | INFO | Opening tmp RTMetrics json file | 20260508 05:48:04.962 | INFO | Serializing RTMetrics to json | 20260508 05:48:04.963 | INFO | Serialization done, swapping tmp file now... | 20260508 05:48:04.964 | INFO | RTMetrics done | 20260508 05:48:05.029 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.486 seconds | 20260508 05:48:05.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.81% / 0.00% / 59.19% | 20260508 05:48:05.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.440137% / 0% / 3.97364e-05% | 20260508 05:48:05.029 | INFO | Baz IO overall: | 20260508 05:48:05.029 | INFO | BytesWritten : 2542106547723 | 20260508 05:48:05.029 | INFO | NumEvents : 1226205332494 | 20260508 05:48:06.568 | INFO | Analyzing chunk frames = [5713920,5714432) | 20260508 05:48:06.568 | INFO | MemInfo: { | 20260508 05:48:06.568 | INFO | "gpu" : " 17 GB", | 20260508 05:48:06.568 | INFO | "huge" : "", | 20260508 05:48:06.568 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:06.568 | INFO | "malloc" : " 14 GB", | 20260508 05:48:06.568 | INFO | "pinned" : " 29 GB" | 20260508 05:48:06.568 | INFO | } | 20260508 05:48:06.568 | INFO | Perf report for the last 5.13845 seconds: | 20260508 05:48:06.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:06.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:06.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:06.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:06.568 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 05:48:06.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:06.568 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260508 05:48:06.569 | INFO | PRE_HQ : 66.7%, 1.8, 62.8ms, 0 | 20260508 05:48:06.569 | INFO | BAZWRITER : 67.2%, 2.9, 100.0ms, 0 | 20260508 05:48:06.569 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:48:06.569 | INFO | Uploaded 36GiB to GPU in 2.58795s (13.9106GiB/s) | 20260508 05:48:06.569 | INFO | Downloaded 38GiB to GPU in 2.69881s (14.0803GiB/s) | 20260508 05:48:11.705 | INFO | Analyzing chunk frames = [5714432,5714944) | 20260508 05:48:11.705 | INFO | MemInfo: { | 20260508 05:48:11.705 | INFO | "gpu" : " 17 GB", | 20260508 05:48:11.705 | INFO | "huge" : "", | 20260508 05:48:11.705 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:11.706 | INFO | "malloc" : " 14 GB", | 20260508 05:48:11.706 | INFO | "pinned" : " 29 GB" | 20260508 05:48:11.706 | INFO | } | 20260508 05:48:11.706 | INFO | Perf report for the last 5.13722 seconds: | 20260508 05:48:11.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:11.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | ANALYSIS : 67.4%, 3.0, 104.9ms, 0 | 20260508 05:48:11.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 05:48:11.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:11.706 | INFO | Uploaded 36GiB to GPU in 2.55956s (14.0649GiB/s) | 20260508 05:48:11.706 | INFO | Downloaded 34GiB to GPU in 2.43314s (13.9737GiB/s) | 20260508 05:48:16.842 | INFO | Analyzing chunk frames = [5714944,5715456) | 20260508 05:48:16.842 | INFO | MemInfo: { | 20260508 05:48:16.842 | INFO | "gpu" : " 17 GB", | 20260508 05:48:16.842 | INFO | "huge" : "", | 20260508 05:48:16.842 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:16.842 | INFO | "malloc" : " 14 GB", | 20260508 05:48:16.842 | INFO | "pinned" : " 29 GB" | 20260508 05:48:16.842 | INFO | } | 20260508 05:48:16.842 | INFO | Perf report for the last 5.13635 seconds: | 20260508 05:48:16.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:16.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:48:16.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:48:16.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:16.842 | INFO | Uploaded 36GiB to GPU in 2.49767s (14.4134GiB/s) | 20260508 05:48:16.842 | INFO | Downloaded 34GiB to GPU in 2.40942s (14.1113GiB/s) | 20260508 05:48:21.985 | INFO | Analyzing chunk frames = [5715456,5715968) | 20260508 05:48:21.985 | INFO | MemInfo: { | 20260508 05:48:21.985 | INFO | "gpu" : " 17 GB", | 20260508 05:48:21.985 | INFO | "huge" : "", | 20260508 05:48:21.985 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:21.985 | INFO | "malloc" : " 14 GB", | 20260508 05:48:21.985 | INFO | "pinned" : " 29 GB" | 20260508 05:48:21.985 | INFO | } | 20260508 05:48:21.986 | INFO | Perf report for the last 5.14352 seconds: | 20260508 05:48:21.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:21.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 05:48:21.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:48:21.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:21.986 | INFO | Uploaded 36GiB to GPU in 2.54379s (14.1521GiB/s) | 20260508 05:48:21.986 | INFO | Downloaded 34GiB to GPU in 2.41512s (14.078GiB/s) | 20260508 05:48:27.127 | INFO | Analyzing chunk frames = [5715968,5716480) | 20260508 05:48:27.127 | INFO | MemInfo: { | 20260508 05:48:27.127 | INFO | "gpu" : " 17 GB", | 20260508 05:48:27.127 | INFO | "huge" : "", | 20260508 05:48:27.127 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:27.128 | INFO | "malloc" : " 14 GB", | 20260508 05:48:27.128 | INFO | "pinned" : " 29 GB" | 20260508 05:48:27.128 | INFO | } | 20260508 05:48:27.128 | INFO | Perf report for the last 5.14203 seconds: | 20260508 05:48:27.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:27.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:48:27.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 05:48:27.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:27.128 | INFO | Uploaded 36GiB to GPU in 2.3102s (15.583GiB/s) | 20260508 05:48:27.128 | INFO | Downloaded 34GiB to GPU in 2.50548s (13.5702GiB/s) | 20260508 05:48:32.266 | INFO | Analyzing chunk frames = [5716480,5716992) | 20260508 05:48:32.266 | INFO | MemInfo: { | 20260508 05:48:32.266 | INFO | "gpu" : " 17 GB", | 20260508 05:48:32.266 | INFO | "huge" : "", | 20260508 05:48:32.266 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:32.266 | INFO | "malloc" : " 14 GB", | 20260508 05:48:32.266 | INFO | "pinned" : " 29 GB" | 20260508 05:48:32.266 | INFO | } | 20260508 05:48:32.266 | INFO | Perf report for the last 5.13831 seconds: | 20260508 05:48:32.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:32.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:48:32.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:48:32.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:32.266 | INFO | Uploaded 36GiB to GPU in 2.42075s (14.8714GiB/s) | 20260508 05:48:32.266 | INFO | Downloaded 34GiB to GPU in 2.43056s (13.9886GiB/s) | 20260508 05:48:37.400 | INFO | Analyzing chunk frames = [5716992,5717504) | 20260508 05:48:37.400 | INFO | MemInfo: { | 20260508 05:48:37.400 | INFO | "gpu" : " 17 GB", | 20260508 05:48:37.400 | INFO | "huge" : "", | 20260508 05:48:37.400 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:37.400 | INFO | "malloc" : " 14 GB", | 20260508 05:48:37.400 | INFO | "pinned" : " 29 GB" | 20260508 05:48:37.400 | INFO | } | 20260508 05:48:37.400 | INFO | Perf report for the last 5.13412 seconds: | 20260508 05:48:37.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:37.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.400 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 05:48:37.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.400 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 05:48:37.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:37.401 | INFO | Uploaded 36GiB to GPU in 2.41438s (14.9106GiB/s) | 20260508 05:48:37.401 | INFO | Downloaded 34GiB to GPU in 2.43204s (13.98GiB/s) | 20260508 05:48:42.541 | INFO | Analyzing chunk frames = [5717504,5718016) | 20260508 05:48:42.541 | INFO | MemInfo: { | 20260508 05:48:42.541 | INFO | "gpu" : " 17 GB", | 20260508 05:48:42.541 | INFO | "huge" : "", | 20260508 05:48:42.541 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:42.541 | INFO | "malloc" : " 14 GB", | 20260508 05:48:42.541 | INFO | "pinned" : " 29 GB" | 20260508 05:48:42.541 | INFO | } | 20260508 05:48:42.541 | INFO | Perf report for the last 5.14141 seconds: | 20260508 05:48:42.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:42.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.541 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:48:42.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.542 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 05:48:42.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:42.542 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:48:42.542 | INFO | Total mem usage: 16.7 GiB | 20260508 05:48:42.542 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:48:42.542 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:48:42.542 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:48:42.542 | INFO | High Water Breakdown: | 20260508 05:48:42.542 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:48:42.542 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:48:42.542 | INFO | Uploaded 36GiB to GPU in 2.45784s (14.647GiB/s) | 20260508 05:48:42.542 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:48:42.542 | INFO | Downloaded 34GiB to GPU in 2.44594s (13.9006GiB/s) | 20260508 05:48:42.542 | INFO | ------------------------------------------ | 20260508 05:48:42.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:48:42.542 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:48:42.542 | INFO | Total mem usage: 26.5 GiB | 20260508 05:48:42.542 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:48:42.542 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:48:42.542 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:48:42.542 | INFO | High Water Breakdown: | 20260508 05:48:42.542 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:48:42.542 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:48:42.542 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:48:42.542 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:48:42.542 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:48:42.542 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:48:42.542 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:48:42.542 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:48:42.542 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:48:42.542 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:48:42.542 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:48:42.542 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:48:42.543 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:48:42.543 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:48:42.543 | INFO | ------------------------------------------ | 20260508 05:48:42.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:48:42.543 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:48:42.543 | INFO | Total mem usage: 27.8 GiB | 20260508 05:48:42.543 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:48:42.543 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:48:42.543 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:48:42.543 | INFO | High Water Breakdown: | 20260508 05:48:42.543 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:48:42.543 | INFO | ------------------------------------------ | 20260508 05:48:42.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:48:42.543 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:48:42.543 | INFO | Total mem usage: 15.7 GiB | 20260508 05:48:42.543 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:48:42.543 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:48:42.543 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:48:42.543 | INFO | High Water Breakdown: | 20260508 05:48:42.543 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:48:42.543 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:48:42.543 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:48:42.543 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:48:42.543 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:48:42.543 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:48:42.543 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:48:42.543 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:48:42.543 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:48:42.543 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:48:42.543 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:48:42.543 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:48:42.543 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:48:42.543 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:48:42.543 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:48:42.543 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:48:42.543 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:48:42.543 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:48:42.543 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:48:42.543 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:48:42.543 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:48:42.543 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:48:42.543 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:48:42.543 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:48:42.543 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:48:42.543 | INFO | ------------------------------------------ | 20260508 05:48:42.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:48:42.543 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:48:42.543 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:48:42.543 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:48:42.543 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 5.46e-01 | 20260508 05:48:42.543 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.21e-02 / 8.23e-03 | 20260508 05:48:42.543 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.42e+00, 5.84e-01 / 2.20e-01 | 20260508 05:48:42.543 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.39e+00, 2.09e-01 / 1.37e-01 | 20260508 05:48:42.543 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.53e+01, 3.84e+00 / 1.27e+00 | 20260508 05:48:42.543 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.47e+00 | 20260508 05:48:42.543 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.56e+00, 4.09e-01 / 2.58e-01 | 20260508 05:48:42.543 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.19e+00, 2.57e-01 / 9.55e-02 | 20260508 05:48:42.543 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.47e-01, 5.05e-02 / 2.46e-02 | 20260508 05:48:42.543 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:48:42.543 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:48:42.543 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:48:42.544 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 24.31%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 24.30%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 2.62%, LockDownload -- 2.79e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 15.69%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:48:42.544 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260508 05:48:42.654 | INFO | Starting a round of Baz IO | 20260508 05:48:46.070 | INFO | Populating RTMetrics report | 20260508 05:48:46.071 | INFO | RTMetrics report populated | 20260508 05:48:46.071 | INFO | Opening tmp RTMetrics json file | 20260508 05:48:46.084 | INFO | Serializing RTMetrics to json | 20260508 05:48:46.085 | INFO | Serialization done, swapping tmp file now... | 20260508 05:48:46.085 | INFO | RTMetrics done | 20260508 05:48:46.142 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.487 seconds | 20260508 05:48:46.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.80% / 0.00% / 59.19% | 20260508 05:48:46.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.439692% / 7.94729e-06% / 5.96046e-05% | 20260508 05:48:46.142 | INFO | Baz IO overall: | 20260508 05:48:46.142 | INFO | BytesWritten : 2543787259172 | 20260508 05:48:46.142 | INFO | NumEvents : 1227006525260 | 20260508 05:48:47.683 | INFO | Analyzing chunk frames = [5718016,5718528) | 20260508 05:48:47.683 | INFO | MemInfo: { | 20260508 05:48:47.683 | INFO | "gpu" : " 17 GB", | 20260508 05:48:47.683 | INFO | "huge" : "", | 20260508 05:48:47.683 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:47.683 | INFO | "malloc" : " 14 GB", | 20260508 05:48:47.683 | INFO | "pinned" : " 29 GB" | 20260508 05:48:47.683 | INFO | } | 20260508 05:48:47.683 | INFO | Perf report for the last 5.1415 seconds: | 20260508 05:48:47.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:47.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:47.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:47.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:47.683 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 05:48:47.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:47.683 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:48:47.683 | INFO | PRE_HQ : 66.4%, 1.8, 62.1ms, 0 | 20260508 05:48:47.683 | INFO | BAZWRITER : 67.3%, 2.9, 100.7ms, 0 | 20260508 05:48:47.683 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 05:48:47.684 | INFO | Uploaded 36GiB to GPU in 2.55212s (14.1059GiB/s) | 20260508 05:48:47.684 | INFO | Downloaded 38GiB to GPU in 2.68118s (14.1729GiB/s) | 20260508 05:48:52.821 | INFO | Analyzing chunk frames = [5718528,5719040) | 20260508 05:48:52.821 | INFO | MemInfo: { | 20260508 05:48:52.821 | INFO | "gpu" : " 17 GB", | 20260508 05:48:52.821 | INFO | "huge" : "", | 20260508 05:48:52.821 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:52.821 | INFO | "malloc" : " 14 GB", | 20260508 05:48:52.821 | INFO | "pinned" : " 29 GB" | 20260508 05:48:52.821 | INFO | } | 20260508 05:48:52.821 | INFO | Perf report for the last 5.13801 seconds: | 20260508 05:48:52.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:52.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:48:52.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:48:52.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:52.821 | INFO | Uploaded 36GiB to GPU in 2.50803s (14.3539GiB/s) | 20260508 05:48:52.821 | INFO | Downloaded 34GiB to GPU in 2.43312s (13.9738GiB/s) | 20260508 05:48:57.959 | INFO | Analyzing chunk frames = [5719040,5719552) | 20260508 05:48:57.960 | INFO | MemInfo: { | 20260508 05:48:57.960 | INFO | "gpu" : " 17 GB", | 20260508 05:48:57.960 | INFO | "huge" : "", | 20260508 05:48:57.960 | INFO | "huge_shared" : " 29 GB", | 20260508 05:48:57.960 | INFO | "malloc" : " 14 GB", | 20260508 05:48:57.960 | INFO | "pinned" : " 29 GB" | 20260508 05:48:57.960 | INFO | } | 20260508 05:48:57.960 | INFO | Perf report for the last 5.13881 seconds: | 20260508 05:48:57.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:48:57.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 05:48:57.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:48:57.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:48:57.960 | INFO | Uploaded 36GiB to GPU in 2.56199s (14.0516GiB/s) | 20260508 05:48:57.961 | INFO | Downloaded 34GiB to GPU in 2.40817s (14.1186GiB/s) | 20260508 05:49:03.096 | INFO | Analyzing chunk frames = [5719552,5720064) | 20260508 05:49:03.096 | INFO | MemInfo: { | 20260508 05:49:03.096 | INFO | "gpu" : " 17 GB", | 20260508 05:49:03.096 | INFO | "huge" : "", | 20260508 05:49:03.096 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:03.096 | INFO | "malloc" : " 14 GB", | 20260508 05:49:03.096 | INFO | "pinned" : " 29 GB" | 20260508 05:49:03.096 | INFO | } | 20260508 05:49:03.096 | INFO | Perf report for the last 5.13652 seconds: | 20260508 05:49:03.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:03.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.096 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 05:49:03.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.096 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:49:03.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:03.097 | INFO | Uploaded 36GiB to GPU in 2.57914s (13.9581GiB/s) | 20260508 05:49:03.097 | INFO | Downloaded 34GiB to GPU in 2.40242s (14.1524GiB/s) | 20260508 05:49:08.246 | INFO | Analyzing chunk frames = [5720064,5720576) | 20260508 05:49:08.246 | INFO | MemInfo: { | 20260508 05:49:08.246 | INFO | "gpu" : " 17 GB", | 20260508 05:49:08.246 | INFO | "huge" : "", | 20260508 05:49:08.246 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:08.246 | INFO | "malloc" : " 14 GB", | 20260508 05:49:08.246 | INFO | "pinned" : " 29 GB" | 20260508 05:49:08.246 | INFO | } | 20260508 05:49:08.246 | INFO | Perf report for the last 5.14993 seconds: | 20260508 05:49:08.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:08.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.246 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:49:08.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.246 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:49:08.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:08.247 | INFO | Uploaded 36GiB to GPU in 2.38236s (15.1111GiB/s) | 20260508 05:49:08.247 | INFO | Downloaded 34GiB to GPU in 2.54822s (13.3426GiB/s) | 20260508 05:49:13.383 | INFO | Analyzing chunk frames = [5720576,5721088) | 20260508 05:49:13.383 | INFO | MemInfo: { | 20260508 05:49:13.383 | INFO | "gpu" : " 17 GB", | 20260508 05:49:13.383 | INFO | "huge" : "", | 20260508 05:49:13.383 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:13.383 | INFO | "malloc" : " 14 GB", | 20260508 05:49:13.383 | INFO | "pinned" : " 29 GB" | 20260508 05:49:13.383 | INFO | } | 20260508 05:49:13.383 | INFO | Perf report for the last 5.13719 seconds: | 20260508 05:49:13.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:13.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 05:49:13.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:49:13.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:13.384 | INFO | Uploaded 36GiB to GPU in 2.47716s (14.5328GiB/s) | 20260508 05:49:13.384 | INFO | Downloaded 34GiB to GPU in 2.4378s (13.947GiB/s) | 20260508 05:49:18.516 | INFO | Analyzing chunk frames = [5721088,5721600) | 20260508 05:49:18.516 | INFO | MemInfo: { | 20260508 05:49:18.516 | INFO | "gpu" : " 17 GB", | 20260508 05:49:18.516 | INFO | "huge" : "", | 20260508 05:49:18.516 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:18.516 | INFO | "malloc" : " 14 GB", | 20260508 05:49:18.516 | INFO | "pinned" : " 29 GB" | 20260508 05:49:18.516 | INFO | } | 20260508 05:49:18.517 | INFO | Perf report for the last 5.13309 seconds: | 20260508 05:49:18.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:18.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:49:18.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 05:49:18.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:18.517 | INFO | Uploaded 36GiB to GPU in 2.48418s (14.4917GiB/s) | 20260508 05:49:18.517 | INFO | Downloaded 34GiB to GPU in 2.43405s (13.9685GiB/s) | 20260508 05:49:23.653 | INFO | Analyzing chunk frames = [5721600,5722112) | 20260508 05:49:23.653 | INFO | MemInfo: { | 20260508 05:49:23.653 | INFO | "gpu" : " 17 GB", | 20260508 05:49:23.653 | INFO | "huge" : "", | 20260508 05:49:23.653 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:23.653 | INFO | "malloc" : " 14 GB", | 20260508 05:49:23.653 | INFO | "pinned" : " 29 GB" | 20260508 05:49:23.653 | INFO | } | 20260508 05:49:23.654 | INFO | Perf report for the last 5.13684 seconds: | 20260508 05:49:23.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:23.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 05:49:23.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 05:49:23.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:23.654 | INFO | Uploaded 36GiB to GPU in 2.50026s (14.3985GiB/s) | 20260508 05:49:23.654 | INFO | Downloaded 34GiB to GPU in 2.44084s (13.9296GiB/s) | 20260508 05:49:23.767 | INFO | Starting a round of Baz IO | 20260508 05:49:27.174 | INFO | Populating RTMetrics report | 20260508 05:49:27.174 | INFO | RTMetrics report populated | 20260508 05:49:27.174 | INFO | Opening tmp RTMetrics json file | 20260508 05:49:27.184 | INFO | Serializing RTMetrics to json | 20260508 05:49:27.185 | INFO | Serialization done, swapping tmp file now... | 20260508 05:49:27.185 | INFO | RTMetrics done | 20260508 05:49:27.246 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.478 seconds | 20260508 05:49:27.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.80% / 0.00% / 59.20% | 20260508 05:49:27.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.438865% / 0% / 7.15256e-05% | 20260508 05:49:27.246 | INFO | Baz IO overall: | 20260508 05:49:27.246 | INFO | BytesWritten : 2545467766448 | 20260508 05:49:27.246 | INFO | NumEvents : 1227807584533 | 20260508 05:49:28.795 | INFO | Analyzing chunk frames = [5722112,5722624) | 20260508 05:49:28.795 | INFO | MemInfo: { | 20260508 05:49:28.795 | INFO | "gpu" : " 17 GB", | 20260508 05:49:28.795 | INFO | "huge" : "", | 20260508 05:49:28.795 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:28.795 | INFO | "malloc" : " 14 GB", | 20260508 05:49:28.795 | INFO | "pinned" : " 29 GB" | 20260508 05:49:28.795 | INFO | } | 20260508 05:49:28.795 | INFO | Perf report for the last 5.1414 seconds: | 20260508 05:49:28.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:28.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:28.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:28.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:28.795 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 05:49:28.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:28.795 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 05:49:28.795 | INFO | PRE_HQ : 66.4%, 1.8, 61.2ms, 0 | 20260508 05:49:28.795 | INFO | BAZWRITER : 67.1%, 2.8, 97.3ms, 0 | 20260508 05:49:28.795 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:49:28.795 | INFO | Uploaded 36GiB to GPU in 2.56049s (14.0598GiB/s) | 20260508 05:49:28.795 | INFO | Downloaded 38GiB to GPU in 2.6863s (14.1459GiB/s) | 20260508 05:49:33.935 | INFO | Analyzing chunk frames = [5722624,5723136) | 20260508 05:49:33.935 | INFO | MemInfo: { | 20260508 05:49:33.935 | INFO | "gpu" : " 17 GB", | 20260508 05:49:33.935 | INFO | "huge" : "", | 20260508 05:49:33.935 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:33.935 | INFO | "malloc" : " 14 GB", | 20260508 05:49:33.935 | INFO | "pinned" : " 29 GB" | 20260508 05:49:33.935 | INFO | } | 20260508 05:49:33.935 | INFO | Perf report for the last 5.14014 seconds: | 20260508 05:49:33.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:33.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 05:49:33.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:49:33.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:33.935 | INFO | Uploaded 36GiB to GPU in 2.58818s (13.9094GiB/s) | 20260508 05:49:33.936 | INFO | Downloaded 34GiB to GPU in 2.4407s (13.9304GiB/s) | 20260508 05:49:39.077 | INFO | Analyzing chunk frames = [5723136,5723648) | 20260508 05:49:39.077 | INFO | MemInfo: { | 20260508 05:49:39.077 | INFO | "gpu" : " 17 GB", | 20260508 05:49:39.077 | INFO | "huge" : "", | 20260508 05:49:39.077 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:39.077 | INFO | "malloc" : " 14 GB", | 20260508 05:49:39.077 | INFO | "pinned" : " 29 GB" | 20260508 05:49:39.077 | INFO | } | 20260508 05:49:39.077 | INFO | Perf report for the last 5.14205 seconds: | 20260508 05:49:39.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:39.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 05:49:39.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:49:39.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:39.077 | INFO | Uploaded 36GiB to GPU in 2.54949s (14.1205GiB/s) | 20260508 05:49:39.077 | INFO | Downloaded 34GiB to GPU in 2.40436s (14.141GiB/s) | 20260508 05:49:44.217 | INFO | Analyzing chunk frames = [5723648,5724160) | 20260508 05:49:44.217 | INFO | MemInfo: { | 20260508 05:49:44.217 | INFO | "gpu" : " 17 GB", | 20260508 05:49:44.217 | INFO | "huge" : "", | 20260508 05:49:44.217 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:44.217 | INFO | "malloc" : " 14 GB", | 20260508 05:49:44.218 | INFO | "pinned" : " 29 GB" | 20260508 05:49:44.218 | INFO | } | 20260508 05:49:44.218 | INFO | Perf report for the last 5.14052 seconds: | 20260508 05:49:44.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:44.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 05:49:44.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:49:44.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:44.218 | INFO | Uploaded 36GiB to GPU in 2.55251s (14.1038GiB/s) | 20260508 05:49:44.218 | INFO | Downloaded 34GiB to GPU in 2.38676s (14.2453GiB/s) | 20260508 05:49:49.352 | INFO | Analyzing chunk frames = [5724160,5724672) | 20260508 05:49:49.352 | INFO | MemInfo: { | 20260508 05:49:49.352 | INFO | "gpu" : " 17 GB", | 20260508 05:49:49.352 | INFO | "huge" : "", | 20260508 05:49:49.352 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:49.352 | INFO | "malloc" : " 14 GB", | 20260508 05:49:49.352 | INFO | "pinned" : " 29 GB" | 20260508 05:49:49.352 | INFO | } | 20260508 05:49:49.352 | INFO | Perf report for the last 5.13459 seconds: | 20260508 05:49:49.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:49.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 05:49:49.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:49:49.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:49.352 | INFO | Uploaded 36GiB to GPU in 2.31095s (15.578GiB/s) | 20260508 05:49:49.353 | INFO | Downloaded 34GiB to GPU in 2.48618s (13.6756GiB/s) | 20260508 05:49:54.495 | INFO | Analyzing chunk frames = [5724672,5725184) | 20260508 05:49:54.495 | INFO | MemInfo: { | 20260508 05:49:54.495 | INFO | "gpu" : " 17 GB", | 20260508 05:49:54.495 | INFO | "huge" : "", | 20260508 05:49:54.495 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:54.495 | INFO | "malloc" : " 14 GB", | 20260508 05:49:54.495 | INFO | "pinned" : " 29 GB" | 20260508 05:49:54.495 | INFO | } | 20260508 05:49:54.495 | INFO | Perf report for the last 5.14294 seconds: | 20260508 05:49:54.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:54.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:49:54.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:49:54.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:54.495 | INFO | Uploaded 36GiB to GPU in 2.43693s (14.7727GiB/s) | 20260508 05:49:54.495 | INFO | Downloaded 34GiB to GPU in 2.42646s (14.0122GiB/s) | 20260508 05:49:59.632 | INFO | Analyzing chunk frames = [5725184,5725696) | 20260508 05:49:59.632 | INFO | MemInfo: { | 20260508 05:49:59.632 | INFO | "gpu" : " 17 GB", | 20260508 05:49:59.632 | INFO | "huge" : "", | 20260508 05:49:59.632 | INFO | "huge_shared" : " 29 GB", | 20260508 05:49:59.632 | INFO | "malloc" : " 14 GB", | 20260508 05:49:59.632 | INFO | "pinned" : " 29 GB" | 20260508 05:49:59.632 | INFO | } | 20260508 05:49:59.632 | INFO | Perf report for the last 5.13723 seconds: | 20260508 05:49:59.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:49:59.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 05:49:59.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:49:59.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:49:59.633 | INFO | Uploaded 36GiB to GPU in 2.46884s (14.5817GiB/s) | 20260508 05:49:59.633 | INFO | Downloaded 34GiB to GPU in 2.43243s (13.9778GiB/s) | 20260508 05:50:04.774 | INFO | Analyzing chunk frames = [5725696,5726208) | 20260508 05:50:04.774 | INFO | MemInfo: { | 20260508 05:50:04.774 | INFO | "gpu" : " 17 GB", | 20260508 05:50:04.774 | INFO | "huge" : "", | 20260508 05:50:04.774 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:04.774 | INFO | "malloc" : " 14 GB", | 20260508 05:50:04.774 | INFO | "pinned" : " 29 GB" | 20260508 05:50:04.774 | INFO | } | 20260508 05:50:04.774 | INFO | Perf report for the last 5.14182 seconds: | 20260508 05:50:04.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:04.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.774 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:50:04.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.774 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 05:50:04.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:04.775 | INFO | Uploaded 36GiB to GPU in 2.44931s (14.698GiB/s) | 20260508 05:50:04.775 | INFO | Downloaded 34GiB to GPU in 2.43983s (13.9354GiB/s) | 20260508 05:50:04.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:50:04.775 | INFO | Total mem usage: 16.7 GiB | 20260508 05:50:04.775 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:50:04.775 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:50:04.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:50:04.775 | INFO | High Water Breakdown: | 20260508 05:50:04.775 | INFO | 59.6% : 8.56 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:50:04.775 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:50:04.775 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:50:04.775 | INFO | ------------------------------------------ | 20260508 05:50:04.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:50:04.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:50:04.775 | INFO | Total mem usage: 26.5 GiB | 20260508 05:50:04.775 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:50:04.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:50:04.775 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:50:04.775 | INFO | High Water Breakdown: | 20260508 05:50:04.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:50:04.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:50:04.775 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:50:04.775 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:50:04.775 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:50:04.775 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:50:04.775 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:50:04.775 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:50:04.775 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:50:04.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:50:04.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:50:04.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:50:04.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:50:04.775 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:50:04.775 | INFO | ------------------------------------------ | 20260508 05:50:04.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:50:04.775 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:50:04.775 | INFO | Total mem usage: 27.8 GiB | 20260508 05:50:04.775 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:50:04.775 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:50:04.775 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:50:04.775 | INFO | High Water Breakdown: | 20260508 05:50:04.775 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:50:04.775 | INFO | ------------------------------------------ | 20260508 05:50:04.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:50:04.775 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:50:04.775 | INFO | Total mem usage: 15.7 GiB | 20260508 05:50:04.775 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:50:04.775 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:50:04.775 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:50:04.775 | INFO | High Water Breakdown: | 20260508 05:50:04.775 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:50:04.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:50:04.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:50:04.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:50:04.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:50:04.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:50:04.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:50:04.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:50:04.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:50:04.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:50:04.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:50:04.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:50:04.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:50:04.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:50:04.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:50:04.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:50:04.776 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:50:04.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:50:04.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:50:04.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:50:04.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:50:04.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:50:04.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:50:04.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:50:04.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:50:04.776 | INFO | ------------------------------------------ | 20260508 05:50:04.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:50:04.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:50:04.776 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:50:04.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:50:04.776 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 1.07e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.07e-01, 1.19e-02 / 2.08e-02 | 20260508 05:50:04.776 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.42e+00, 5.89e-01 / 2.39e-01 | 20260508 05:50:04.776 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.08e-01 / 1.43e-01 | 20260508 05:50:04.776 | INFO | PerfReport: 15.94%, DME -- 4.16e+01 / 4.51e+01, 3.82e+00 / 1.38e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.77e+00 / 1.36e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.14e-01 / 2.53e-01 | 20260508 05:50:04.776 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.24e+00, 2.57e-01 / 2.05e-01 | 20260508 05:50:04.776 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.52e-01, 5.05e-02 / 2.52e-02 | 20260508 05:50:04.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:50:04.776 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:50:04.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:50:04.776 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 24.50%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 30.74%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 2.52%, LockDownload -- 2.68e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 15.61%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:50:04.776 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260508 05:50:04.887 | INFO | Starting a round of Baz IO | 20260508 05:50:08.297 | INFO | Populating RTMetrics report | 20260508 05:50:08.298 | INFO | RTMetrics report populated | 20260508 05:50:08.298 | INFO | Opening tmp RTMetrics json file | 20260508 05:50:08.311 | INFO | Serializing RTMetrics to json | 20260508 05:50:08.312 | INFO | Serialization done, swapping tmp file now... | 20260508 05:50:08.313 | INFO | RTMetrics done | 20260508 05:50:08.364 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.477 seconds | 20260508 05:50:08.364 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.80% / 0.00% / 59.20% | 20260508 05:50:08.364 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.438162% / 3.97364e-06% / 6.35783e-05% | 20260508 05:50:08.364 | INFO | Baz IO overall: | 20260508 05:50:08.364 | INFO | BytesWritten : 2547148621091 | 20260508 05:50:08.364 | INFO | NumEvents : 1228608877387 | 20260508 05:50:09.910 | INFO | Analyzing chunk frames = [5726208,5726720) | 20260508 05:50:09.910 | INFO | MemInfo: { | 20260508 05:50:09.910 | INFO | "gpu" : " 17 GB", | 20260508 05:50:09.910 | INFO | "huge" : "", | 20260508 05:50:09.910 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:09.910 | INFO | "malloc" : " 14 GB", | 20260508 05:50:09.910 | INFO | "pinned" : " 29 GB" | 20260508 05:50:09.910 | INFO | } | 20260508 05:50:09.910 | INFO | Perf report for the last 5.13595 seconds: | 20260508 05:50:09.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:09.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:09.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:09.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:09.910 | INFO | ANALYSIS : 67.9%, 3.0, 105.5ms, 0 | 20260508 05:50:09.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:09.910 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 05:50:09.910 | INFO | PRE_HQ : 66.6%, 1.8, 61.9ms, 0 | 20260508 05:50:09.910 | INFO | BAZWRITER : 67.2%, 2.9, 98.7ms, 0 | 20260508 05:50:09.911 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:50:09.911 | INFO | Uploaded 36GiB to GPU in 2.54584s (14.1407GiB/s) | 20260508 05:50:09.911 | INFO | Downloaded 38GiB to GPU in 2.71755s (13.9832GiB/s) | 20260508 05:50:15.050 | INFO | Analyzing chunk frames = [5726720,5727232) | 20260508 05:50:15.050 | INFO | MemInfo: { | 20260508 05:50:15.050 | INFO | "gpu" : " 17 GB", | 20260508 05:50:15.050 | INFO | "huge" : "", | 20260508 05:50:15.050 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:15.050 | INFO | "malloc" : " 14 GB", | 20260508 05:50:15.050 | INFO | "pinned" : " 29 GB" | 20260508 05:50:15.050 | INFO | } | 20260508 05:50:15.050 | INFO | Perf report for the last 5.13958 seconds: | 20260508 05:50:15.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:15.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 05:50:15.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:50:15.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:15.050 | INFO | Uploaded 36GiB to GPU in 2.44531s (14.7221GiB/s) | 20260508 05:50:15.051 | INFO | Downloaded 34GiB to GPU in 2.44782s (13.8899GiB/s) | 20260508 05:50:20.192 | INFO | Analyzing chunk frames = [5727232,5727744) | 20260508 05:50:20.192 | INFO | MemInfo: { | 20260508 05:50:20.192 | INFO | "gpu" : " 17 GB", | 20260508 05:50:20.192 | INFO | "huge" : "", | 20260508 05:50:20.192 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:20.192 | INFO | "malloc" : " 14 GB", | 20260508 05:50:20.192 | INFO | "pinned" : " 29 GB" | 20260508 05:50:20.192 | INFO | } | 20260508 05:50:20.192 | INFO | Perf report for the last 5.14227 seconds: | 20260508 05:50:20.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:20.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 05:50:20.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:50:20.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:20.192 | INFO | Uploaded 36GiB to GPU in 2.50487s (14.372GiB/s) | 20260508 05:50:20.192 | INFO | Downloaded 34GiB to GPU in 2.4236s (14.0287GiB/s) | 20260508 05:50:25.331 | INFO | Analyzing chunk frames = [5727744,5728256) | 20260508 05:50:25.331 | INFO | MemInfo: { | 20260508 05:50:25.331 | INFO | "gpu" : " 17 GB", | 20260508 05:50:25.331 | INFO | "huge" : "", | 20260508 05:50:25.331 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:25.331 | INFO | "malloc" : " 14 GB", | 20260508 05:50:25.331 | INFO | "pinned" : " 29 GB" | 20260508 05:50:25.331 | INFO | } | 20260508 05:50:25.332 | INFO | Perf report for the last 5.13941 seconds: | 20260508 05:50:25.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:25.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 05:50:25.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:50:25.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:25.332 | INFO | Uploaded 36GiB to GPU in 2.52107s (14.2796GiB/s) | 20260508 05:50:25.332 | INFO | Downloaded 34GiB to GPU in 2.4319s (13.9808GiB/s) | 20260508 05:50:30.469 | INFO | Analyzing chunk frames = [5728256,5728768) | 20260508 05:50:30.469 | INFO | MemInfo: { | 20260508 05:50:30.469 | INFO | "gpu" : " 17 GB", | 20260508 05:50:30.469 | INFO | "huge" : "", | 20260508 05:50:30.469 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:30.469 | INFO | "malloc" : " 14 GB", | 20260508 05:50:30.469 | INFO | "pinned" : " 29 GB" | 20260508 05:50:30.469 | INFO | } | 20260508 05:50:30.469 | INFO | Perf report for the last 5.13761 seconds: | 20260508 05:50:30.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:30.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.469 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:50:30.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.469 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:50:30.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:30.470 | INFO | Uploaded 36GiB to GPU in 2.36492s (15.2225GiB/s) | 20260508 05:50:30.470 | INFO | Downloaded 34GiB to GPU in 2.51703s (13.508GiB/s) | 20260508 05:50:35.605 | INFO | Analyzing chunk frames = [5728768,5729280) | 20260508 05:50:35.605 | INFO | MemInfo: { | 20260508 05:50:35.605 | INFO | "gpu" : " 17 GB", | 20260508 05:50:35.605 | INFO | "huge" : "", | 20260508 05:50:35.605 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:35.605 | INFO | "malloc" : " 14 GB", | 20260508 05:50:35.605 | INFO | "pinned" : " 29 GB" | 20260508 05:50:35.605 | INFO | } | 20260508 05:50:35.605 | INFO | Perf report for the last 5.13586 seconds: | 20260508 05:50:35.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:35.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 05:50:35.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 05:50:35.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:35.605 | INFO | Uploaded 36GiB to GPU in 2.42997s (14.815GiB/s) | 20260508 05:50:35.605 | INFO | Downloaded 34GiB to GPU in 2.43378s (13.97GiB/s) | 20260508 05:50:40.747 | INFO | Analyzing chunk frames = [5729280,5729792) | 20260508 05:50:40.747 | INFO | MemInfo: { | 20260508 05:50:40.747 | INFO | "gpu" : " 17 GB", | 20260508 05:50:40.747 | INFO | "huge" : "", | 20260508 05:50:40.747 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:40.747 | INFO | "malloc" : " 14 GB", | 20260508 05:50:40.748 | INFO | "pinned" : " 29 GB" | 20260508 05:50:40.748 | INFO | } | 20260508 05:50:40.748 | INFO | Perf report for the last 5.14262 seconds: | 20260508 05:50:40.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:40.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 05:50:40.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 05:50:40.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:40.748 | INFO | Uploaded 36GiB to GPU in 2.41887s (14.883GiB/s) | 20260508 05:50:40.748 | INFO | Downloaded 34GiB to GPU in 2.42581s (14.0159GiB/s) | 20260508 05:50:45.886 | INFO | Analyzing chunk frames = [5729792,5730304) | 20260508 05:50:45.887 | INFO | MemInfo: { | 20260508 05:50:45.887 | INFO | "gpu" : " 17 GB", | 20260508 05:50:45.887 | INFO | "huge" : "", | 20260508 05:50:45.887 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:45.887 | INFO | "malloc" : " 14 GB", | 20260508 05:50:45.887 | INFO | "pinned" : " 29 GB" | 20260508 05:50:45.887 | INFO | } | 20260508 05:50:45.887 | INFO | Perf report for the last 5.13916 seconds: | 20260508 05:50:45.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:45.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:50:45.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:50:45.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:45.887 | INFO | Uploaded 36GiB to GPU in 2.47567s (14.5415GiB/s) | 20260508 05:50:45.887 | INFO | Downloaded 34GiB to GPU in 2.45289s (13.8612GiB/s) | 20260508 05:50:45.985 | INFO | Starting a round of Baz IO | 20260508 05:50:49.424 | INFO | Populating RTMetrics report | 20260508 05:50:49.425 | INFO | RTMetrics report populated | 20260508 05:50:49.425 | INFO | Opening tmp RTMetrics json file | 20260508 05:50:49.425 | INFO | Serializing RTMetrics to json | 20260508 05:50:49.427 | INFO | Serialization done, swapping tmp file now... | 20260508 05:50:49.427 | INFO | RTMetrics done | 20260508 05:50:49.493 | INFO | Finished a round of Baz IO: Wrote 1.564 GiB in 3.508 seconds | 20260508 05:50:49.493 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.80% / 0.00% / 59.20% | 20260508 05:50:49.493 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.436139% / 0% / 5.16574e-05% | 20260508 05:50:49.493 | INFO | Baz IO overall: | 20260508 05:50:49.494 | INFO | BytesWritten : 2548827434846 | 20260508 05:50:49.494 | INFO | NumEvents : 1229408658432 | 20260508 05:50:51.027 | INFO | Analyzing chunk frames = [5730304,5730816) | 20260508 05:50:51.027 | INFO | MemInfo: { | 20260508 05:50:51.027 | INFO | "gpu" : " 17 GB", | 20260508 05:50:51.027 | INFO | "huge" : "", | 20260508 05:50:51.027 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:51.027 | INFO | "malloc" : " 14 GB", | 20260508 05:50:51.027 | INFO | "pinned" : " 29 GB" | 20260508 05:50:51.027 | INFO | } | 20260508 05:50:51.027 | INFO | Perf report for the last 5.1401 seconds: | 20260508 05:50:51.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:51.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:51.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:51.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:51.027 | INFO | ANALYSIS : 68.2%, 3.0, 106.4ms, 0 | 20260508 05:50:51.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:51.027 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 05:50:51.027 | INFO | PRE_HQ : 65.8%, 1.6, 55.2ms, 0 | 20260508 05:50:51.027 | INFO | BAZWRITER : 67.7%, 2.7, 94.0ms, 0 | 20260508 05:50:51.027 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 05:50:51.028 | INFO | Uploaded 36GiB to GPU in 2.58114s (13.9473GiB/s) | 20260508 05:50:51.028 | INFO | Downloaded 38GiB to GPU in 2.70654s (14.0401GiB/s) | 20260508 05:50:56.163 | INFO | Analyzing chunk frames = [5730816,5731328) | 20260508 05:50:56.163 | INFO | MemInfo: { | 20260508 05:50:56.163 | INFO | "gpu" : " 17 GB", | 20260508 05:50:56.163 | INFO | "huge" : "", | 20260508 05:50:56.163 | INFO | "huge_shared" : " 29 GB", | 20260508 05:50:56.163 | INFO | "malloc" : " 14 GB", | 20260508 05:50:56.163 | INFO | "pinned" : " 29 GB" | 20260508 05:50:56.163 | INFO | } | 20260508 05:50:56.163 | INFO | Perf report for the last 5.13587 seconds: | 20260508 05:50:56.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:50:56.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:50:56.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:50:56.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:50:56.163 | INFO | Uploaded 36GiB to GPU in 2.47999s (14.5162GiB/s) | 20260508 05:50:56.163 | INFO | Downloaded 34GiB to GPU in 2.44022s (13.9332GiB/s) | 20260508 05:51:01.306 | INFO | Analyzing chunk frames = [5731328,5731840) | 20260508 05:51:01.306 | INFO | MemInfo: { | 20260508 05:51:01.306 | INFO | "gpu" : " 17 GB", | 20260508 05:51:01.306 | INFO | "huge" : "", | 20260508 05:51:01.306 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:01.306 | INFO | "malloc" : " 14 GB", | 20260508 05:51:01.306 | INFO | "pinned" : " 29 GB" | 20260508 05:51:01.306 | INFO | } | 20260508 05:51:01.306 | INFO | Perf report for the last 5.14313 seconds: | 20260508 05:51:01.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:01.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.306 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:51:01.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.306 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:51:01.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:01.307 | INFO | Uploaded 36GiB to GPU in 2.54148s (14.165GiB/s) | 20260508 05:51:01.307 | INFO | Downloaded 34GiB to GPU in 2.42193s (14.0384GiB/s) | 20260508 05:51:06.447 | INFO | Analyzing chunk frames = [5731840,5732352) | 20260508 05:51:06.447 | INFO | MemInfo: { | 20260508 05:51:06.447 | INFO | "gpu" : " 17 GB", | 20260508 05:51:06.447 | INFO | "huge" : "", | 20260508 05:51:06.447 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:06.447 | INFO | "malloc" : " 14 GB", | 20260508 05:51:06.447 | INFO | "pinned" : " 29 GB" | 20260508 05:51:06.447 | INFO | } | 20260508 05:51:06.447 | INFO | Perf report for the last 5.14111 seconds: | 20260508 05:51:06.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:06.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | ANALYSIS : 66.7%, 2.9, 103.3ms, 0 | 20260508 05:51:06.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:51:06.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:06.447 | INFO | Uploaded 36GiB to GPU in 2.52682s (14.2471GiB/s) | 20260508 05:51:06.447 | INFO | Downloaded 34GiB to GPU in 2.40407s (14.1427GiB/s) | 20260508 05:51:11.584 | INFO | Analyzing chunk frames = [5732352,5732864) | 20260508 05:51:11.585 | INFO | MemInfo: { | 20260508 05:51:11.585 | INFO | "gpu" : " 17 GB", | 20260508 05:51:11.585 | INFO | "huge" : "", | 20260508 05:51:11.585 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:11.585 | INFO | "malloc" : " 14 GB", | 20260508 05:51:11.585 | INFO | "pinned" : " 29 GB" | 20260508 05:51:11.585 | INFO | } | 20260508 05:51:11.585 | INFO | Perf report for the last 5.13775 seconds: | 20260508 05:51:11.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:11.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.585 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:51:11.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.585 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:51:11.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:11.586 | INFO | Uploaded 36GiB to GPU in 2.25092s (15.9934GiB/s) | 20260508 05:51:11.586 | INFO | Downloaded 34GiB to GPU in 2.49476s (13.6285GiB/s) | 20260508 05:51:16.722 | INFO | Analyzing chunk frames = [5732864,5733376) | 20260508 05:51:16.722 | INFO | MemInfo: { | 20260508 05:51:16.722 | INFO | "gpu" : " 17 GB", | 20260508 05:51:16.722 | INFO | "huge" : "", | 20260508 05:51:16.722 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:16.722 | INFO | "malloc" : " 14 GB", | 20260508 05:51:16.722 | INFO | "pinned" : " 29 GB" | 20260508 05:51:16.722 | INFO | } | 20260508 05:51:16.722 | INFO | Perf report for the last 5.13742 seconds: | 20260508 05:51:16.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:16.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.722 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 05:51:16.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.723 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260508 05:51:16.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:16.723 | INFO | Uploaded 36GiB to GPU in 2.41784s (14.8893GiB/s) | 20260508 05:51:16.723 | INFO | Downloaded 34GiB to GPU in 2.45391s (13.8555GiB/s) | 20260508 05:51:21.861 | INFO | Analyzing chunk frames = [5733376,5733888) | 20260508 05:51:21.861 | INFO | MemInfo: { | 20260508 05:51:21.861 | INFO | "gpu" : " 17 GB", | 20260508 05:51:21.861 | INFO | "huge" : "", | 20260508 05:51:21.861 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:21.861 | INFO | "malloc" : " 14 GB", | 20260508 05:51:21.861 | INFO | "pinned" : " 29 GB" | 20260508 05:51:21.861 | INFO | } | 20260508 05:51:21.861 | INFO | Perf report for the last 5.13872 seconds: | 20260508 05:51:21.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:21.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.861 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 05:51:21.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.861 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 05:51:21.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:21.862 | INFO | Uploaded 36GiB to GPU in 2.46053s (14.631GiB/s) | 20260508 05:51:21.862 | INFO | Downloaded 34GiB to GPU in 2.4627s (13.806GiB/s) | 20260508 05:51:27.004 | INFO | Analyzing chunk frames = [5733888,5734400) | 20260508 05:51:27.004 | INFO | MemInfo: { | 20260508 05:51:27.004 | INFO | "gpu" : " 17 GB", | 20260508 05:51:27.004 | INFO | "huge" : "", | 20260508 05:51:27.004 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:27.004 | INFO | "malloc" : " 14 GB", | 20260508 05:51:27.004 | INFO | "pinned" : " 29 GB" | 20260508 05:51:27.004 | INFO | } | 20260508 05:51:27.004 | INFO | Perf report for the last 5.14318 seconds: | 20260508 05:51:27.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:27.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.004 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:51:27.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.004 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 05:51:27.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:27.005 | INFO | Uploaded 36GiB to GPU in 2.42096s (14.8701GiB/s) | 20260508 05:51:27.005 | INFO | Downloaded 34GiB to GPU in 2.44848s (13.8862GiB/s) | 20260508 05:51:27.005 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:51:27.005 | INFO | Total mem usage: 16.7 GiB | 20260508 05:51:27.005 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:51:27.005 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:51:27.005 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:51:27.005 | INFO | High Water Breakdown: | 20260508 05:51:27.005 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:51:27.005 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:51:27.005 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:51:27.005 | INFO | ------------------------------------------ | 20260508 05:51:27.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:51:27.005 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:51:27.005 | INFO | Total mem usage: 26.5 GiB | 20260508 05:51:27.005 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:51:27.005 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 05:51:27.005 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 05:51:27.005 | INFO | High Water Breakdown: | 20260508 05:51:27.005 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:51:27.005 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:51:27.005 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:51:27.005 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:51:27.005 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:51:27.005 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:51:27.005 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:51:27.005 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:51:27.005 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:51:27.005 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:51:27.005 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:51:27.005 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:51:27.005 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:51:27.005 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:51:27.005 | INFO | ------------------------------------------ | 20260508 05:51:27.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:51:27.005 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:51:27.005 | INFO | Total mem usage: 27.8 GiB | 20260508 05:51:27.005 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:51:27.005 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 05:51:27.005 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 05:51:27.005 | INFO | High Water Breakdown: | 20260508 05:51:27.006 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 05:51:27.006 | INFO | ------------------------------------------ | 20260508 05:51:27.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:51:27.006 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:51:27.006 | INFO | Total mem usage: 15.7 GiB | 20260508 05:51:27.006 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:51:27.006 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:51:27.006 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:51:27.006 | INFO | High Water Breakdown: | 20260508 05:51:27.006 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:51:27.006 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:51:27.006 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:51:27.006 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:51:27.006 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:51:27.006 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:51:27.006 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:51:27.006 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:51:27.006 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:51:27.006 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:51:27.006 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:51:27.006 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:51:27.006 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:51:27.006 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:51:27.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:51:27.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:51:27.006 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:51:27.006 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:51:27.006 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:51:27.006 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:51:27.006 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:51:27.006 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:51:27.006 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:51:27.006 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:51:27.006 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:51:27.006 | INFO | ------------------------------------------ | 20260508 05:51:27.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:51:27.006 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:51:27.006 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:51:27.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:51:27.006 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 7.44e-01 | 20260508 05:51:27.006 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.14e-02 / 6.56e-03 | 20260508 05:51:27.006 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.43e+00, 5.87e-01 / 2.20e-01 | 20260508 05:51:27.006 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.12e-01 / 1.23e-01 | 20260508 05:51:27.006 | INFO | PerfReport: 15.87%, DME -- 4.16e+01 / 4.52e+01, 3.67e+00 / 1.21e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.40e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.63e+00, 4.14e-01 / 1.94e-01 | 20260508 05:51:27.006 | INFO | PerfReport: 3.18%, Metrics -- 1.04e+00 / 1.25e+00, 2.56e-01 / 1.44e-01 | 20260508 05:51:27.006 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.57e-01, 4.99e-02 / 1.99e-02 | 20260508 05:51:27.006 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:51:27.006 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:51:27.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:51:27.006 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 24.20%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 24.38%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 30.88%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 2.06%, LockUpload -- 2.18e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 2.77%, LockDownload -- 2.94e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 15.69%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:51:27.006 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 05:51:27.119 | INFO | Starting a round of Baz IO | 20260508 05:51:30.550 | INFO | Populating RTMetrics report | 20260508 05:51:30.550 | INFO | RTMetrics report populated | 20260508 05:51:30.550 | INFO | Opening tmp RTMetrics json file | 20260508 05:51:30.551 | INFO | Serializing RTMetrics to json | 20260508 05:51:30.552 | INFO | Serialization done, swapping tmp file now... | 20260508 05:51:30.553 | INFO | RTMetrics done | 20260508 05:51:30.622 | INFO | Finished a round of Baz IO: Wrote 1.566 GiB in 3.502 seconds | 20260508 05:51:30.622 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.80% / 0.00% / 59.20% | 20260508 05:51:30.622 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.437331% / 0% / 4.76837e-05% | 20260508 05:51:30.622 | INFO | Baz IO overall: | 20260508 05:51:30.622 | INFO | BytesWritten : 2550508697012 | 20260508 05:51:30.622 | INFO | NumEvents : 1230210270711 | 20260508 05:51:32.145 | INFO | Analyzing chunk frames = [5734400,5734912) | 20260508 05:51:32.145 | INFO | MemInfo: { | 20260508 05:51:32.145 | INFO | "gpu" : " 17 GB", | 20260508 05:51:32.145 | INFO | "huge" : "", | 20260508 05:51:32.145 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:32.145 | INFO | "malloc" : " 14 GB", | 20260508 05:51:32.145 | INFO | "pinned" : " 29 GB" | 20260508 05:51:32.145 | INFO | } | 20260508 05:51:32.145 | INFO | Perf report for the last 5.14076 seconds: | 20260508 05:51:32.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:32.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:32.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:32.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:32.145 | INFO | ANALYSIS : 68.6%, 3.0, 106.5ms, 0 | 20260508 05:51:32.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:32.145 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 05:51:32.145 | INFO | PRE_HQ : 66.7%, 1.8, 61.7ms, 0 | 20260508 05:51:32.145 | INFO | BAZWRITER : 67.7%, 2.9, 99.4ms, 0 | 20260508 05:51:32.145 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 05:51:32.145 | INFO | Uploaded 36GiB to GPU in 2.54557s (14.1422GiB/s) | 20260508 05:51:32.145 | INFO | Downloaded 38GiB to GPU in 2.70897s (14.0275GiB/s) | 20260508 05:51:37.282 | INFO | Analyzing chunk frames = [5734912,5735424) | 20260508 05:51:37.282 | INFO | MemInfo: { | 20260508 05:51:37.282 | INFO | "gpu" : " 17 GB", | 20260508 05:51:37.282 | INFO | "huge" : "", | 20260508 05:51:37.282 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:37.282 | INFO | "malloc" : " 14 GB", | 20260508 05:51:37.282 | INFO | "pinned" : " 29 GB" | 20260508 05:51:37.282 | INFO | } | 20260508 05:51:37.282 | INFO | Perf report for the last 5.13742 seconds: | 20260508 05:51:37.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:37.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.283 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:51:37.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.283 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:51:37.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:37.283 | INFO | Uploaded 36GiB to GPU in 2.49127s (14.4505GiB/s) | 20260508 05:51:37.283 | INFO | Downloaded 34GiB to GPU in 2.43269s (13.9763GiB/s) | 20260508 05:51:42.414 | INFO | Analyzing chunk frames = [5735424,5735936) | 20260508 05:51:42.414 | INFO | MemInfo: { | 20260508 05:51:42.414 | INFO | "gpu" : " 17 GB", | 20260508 05:51:42.414 | INFO | "huge" : "", | 20260508 05:51:42.414 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:42.414 | INFO | "malloc" : " 14 GB", | 20260508 05:51:42.414 | INFO | "pinned" : " 29 GB" | 20260508 05:51:42.414 | INFO | } | 20260508 05:51:42.414 | INFO | Perf report for the last 5.13158 seconds: | 20260508 05:51:42.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:42.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.414 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 05:51:42.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.414 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:51:42.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:42.415 | INFO | Uploaded 36GiB to GPU in 2.54155s (14.1646GiB/s) | 20260508 05:51:42.415 | INFO | Downloaded 34GiB to GPU in 2.41734s (14.065GiB/s) | 20260508 05:51:47.563 | INFO | Analyzing chunk frames = [5735936,5736448) | 20260508 05:51:47.563 | INFO | MemInfo: { | 20260508 05:51:47.563 | INFO | "gpu" : " 17 GB", | 20260508 05:51:47.563 | INFO | "huge" : "", | 20260508 05:51:47.563 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:47.563 | INFO | "malloc" : " 14 GB", | 20260508 05:51:47.563 | INFO | "pinned" : " 29 GB" | 20260508 05:51:47.563 | INFO | } | 20260508 05:51:47.563 | INFO | Perf report for the last 5.14876 seconds: | 20260508 05:51:47.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:47.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | ANALYSIS : 66.8%, 3.0, 103.5ms, 0 | 20260508 05:51:47.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:51:47.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:47.563 | INFO | Uploaded 36GiB to GPU in 2.56552s (14.0322GiB/s) | 20260508 05:51:47.563 | INFO | Downloaded 34GiB to GPU in 2.41014s (14.1071GiB/s) | 20260508 05:51:52.704 | INFO | Analyzing chunk frames = [5736448,5736960) | 20260508 05:51:52.704 | INFO | MemInfo: { | 20260508 05:51:52.704 | INFO | "gpu" : " 17 GB", | 20260508 05:51:52.704 | INFO | "huge" : "", | 20260508 05:51:52.704 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:52.704 | INFO | "malloc" : " 14 GB", | 20260508 05:51:52.704 | INFO | "pinned" : " 29 GB" | 20260508 05:51:52.704 | INFO | } | 20260508 05:51:52.704 | INFO | Perf report for the last 5.14128 seconds: | 20260508 05:51:52.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:52.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.704 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 05:51:52.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.704 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:51:52.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:52.705 | INFO | Uploaded 36GiB to GPU in 2.29663s (15.6751GiB/s) | 20260508 05:51:52.705 | INFO | Downloaded 34GiB to GPU in 2.48097s (13.7043GiB/s) | 20260508 05:51:57.844 | INFO | Analyzing chunk frames = [5736960,5737472) | 20260508 05:51:57.844 | INFO | MemInfo: { | 20260508 05:51:57.844 | INFO | "gpu" : " 17 GB", | 20260508 05:51:57.844 | INFO | "huge" : "", | 20260508 05:51:57.844 | INFO | "huge_shared" : " 29 GB", | 20260508 05:51:57.844 | INFO | "malloc" : " 14 GB", | 20260508 05:51:57.844 | INFO | "pinned" : " 29 GB" | 20260508 05:51:57.844 | INFO | } | 20260508 05:51:57.844 | INFO | Perf report for the last 5.13998 seconds: | 20260508 05:51:57.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:51:57.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:51:57.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:51:57.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:51:57.844 | INFO | Uploaded 36GiB to GPU in 2.45787s (14.6468GiB/s) | 20260508 05:51:57.844 | INFO | Downloaded 34GiB to GPU in 2.41107s (14.1016GiB/s) | 20260508 05:52:02.981 | INFO | Analyzing chunk frames = [5737472,5737984) | 20260508 05:52:02.981 | INFO | MemInfo: { | 20260508 05:52:02.981 | INFO | "gpu" : " 17 GB", | 20260508 05:52:02.981 | INFO | "huge" : "", | 20260508 05:52:02.981 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:02.981 | INFO | "malloc" : " 14 GB", | 20260508 05:52:02.981 | INFO | "pinned" : " 29 GB" | 20260508 05:52:02.981 | INFO | } | 20260508 05:52:02.981 | INFO | Perf report for the last 5.1365 seconds: | 20260508 05:52:02.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:02.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:52:02.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 05:52:02.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:02.981 | INFO | Uploaded 36GiB to GPU in 2.49122s (14.4508GiB/s) | 20260508 05:52:02.981 | INFO | Downloaded 34GiB to GPU in 2.42811s (14.0027GiB/s) | 20260508 05:52:08.118 | INFO | Analyzing chunk frames = [5737984,5738496) | 20260508 05:52:08.118 | INFO | MemInfo: { | 20260508 05:52:08.118 | INFO | "gpu" : " 17 GB", | 20260508 05:52:08.118 | INFO | "huge" : "", | 20260508 05:52:08.118 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:08.118 | INFO | "malloc" : " 14 GB", | 20260508 05:52:08.118 | INFO | "pinned" : " 29 GB" | 20260508 05:52:08.118 | INFO | } | 20260508 05:52:08.118 | INFO | Perf report for the last 5.13725 seconds: | 20260508 05:52:08.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:08.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:52:08.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 05:52:08.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:08.118 | INFO | Uploaded 36GiB to GPU in 2.49147s (14.4493GiB/s) | 20260508 05:52:08.118 | INFO | Downloaded 34GiB to GPU in 2.42521s (14.0194GiB/s) | 20260508 05:52:08.215 | INFO | Starting a round of Baz IO | 20260508 05:52:11.646 | INFO | Populating RTMetrics report | 20260508 05:52:11.646 | INFO | RTMetrics report populated | 20260508 05:52:11.646 | INFO | Opening tmp RTMetrics json file | 20260508 05:52:11.657 | INFO | Serializing RTMetrics to json | 20260508 05:52:11.658 | INFO | Serialization done, swapping tmp file now... | 20260508 05:52:11.659 | INFO | RTMetrics done | 20260508 05:52:11.715 | INFO | Finished a round of Baz IO: Wrote 1.564 GiB in 3.499 seconds | 20260508 05:52:11.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.80% / 0.00% / 59.20% | 20260508 05:52:11.715 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.435797% / 3.97364e-06% / 4.76837e-05% | 20260508 05:52:11.715 | INFO | Baz IO overall: | 20260508 05:52:11.715 | INFO | BytesWritten : 2552188516932 | 20260508 05:52:11.715 | INFO | NumEvents : 1231010871923 | 20260508 05:52:13.257 | INFO | Analyzing chunk frames = [5738496,5739008) | 20260508 05:52:13.257 | INFO | MemInfo: { | 20260508 05:52:13.257 | INFO | "gpu" : " 17 GB", | 20260508 05:52:13.257 | INFO | "huge" : "", | 20260508 05:52:13.257 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:13.257 | INFO | "malloc" : " 14 GB", | 20260508 05:52:13.257 | INFO | "pinned" : " 29 GB" | 20260508 05:52:13.257 | INFO | } | 20260508 05:52:13.257 | INFO | Perf report for the last 5.13951 seconds: | 20260508 05:52:13.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:13.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:13.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:13.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:13.258 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 05:52:13.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:13.258 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:52:13.258 | INFO | PRE_HQ : 65.8%, 1.6, 55.5ms, 0 | 20260508 05:52:13.258 | INFO | BAZWRITER : 67.5%, 2.7, 92.2ms, 0 | 20260508 05:52:13.258 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:52:13.258 | INFO | Uploaded 36GiB to GPU in 2.56986s (14.0086GiB/s) | 20260508 05:52:13.258 | INFO | Downloaded 38GiB to GPU in 2.68461s (14.1548GiB/s) | 20260508 05:52:18.396 | INFO | Analyzing chunk frames = [5739008,5739520) | 20260508 05:52:18.396 | INFO | MemInfo: { | 20260508 05:52:18.396 | INFO | "gpu" : " 17 GB", | 20260508 05:52:18.396 | INFO | "huge" : "", | 20260508 05:52:18.396 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:18.396 | INFO | "malloc" : " 14 GB", | 20260508 05:52:18.396 | INFO | "pinned" : " 29 GB" | 20260508 05:52:18.396 | INFO | } | 20260508 05:52:18.396 | INFO | Perf report for the last 5.13857 seconds: | 20260508 05:52:18.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:18.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.396 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 05:52:18.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.396 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:52:18.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:18.397 | INFO | Uploaded 36GiB to GPU in 2.48294s (14.4989GiB/s) | 20260508 05:52:18.397 | INFO | Downloaded 34GiB to GPU in 2.43974s (13.9359GiB/s) | 20260508 05:52:23.535 | INFO | Analyzing chunk frames = [5739520,5740032) | 20260508 05:52:23.535 | INFO | MemInfo: { | 20260508 05:52:23.535 | INFO | "gpu" : " 17 GB", | 20260508 05:52:23.535 | INFO | "huge" : "", | 20260508 05:52:23.535 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:23.535 | INFO | "malloc" : " 14 GB", | 20260508 05:52:23.535 | INFO | "pinned" : " 29 GB" | 20260508 05:52:23.535 | INFO | } | 20260508 05:52:23.535 | INFO | Perf report for the last 5.1389 seconds: | 20260508 05:52:23.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:23.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 05:52:23.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:52:23.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:23.535 | INFO | Uploaded 36GiB to GPU in 2.5159s (14.309GiB/s) | 20260508 05:52:23.535 | INFO | Downloaded 34GiB to GPU in 2.40714s (14.1246GiB/s) | 20260508 05:52:28.673 | INFO | Analyzing chunk frames = [5740032,5740544) | 20260508 05:52:28.673 | INFO | MemInfo: { | 20260508 05:52:28.673 | INFO | "gpu" : " 17 GB", | 20260508 05:52:28.673 | INFO | "huge" : "", | 20260508 05:52:28.673 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:28.673 | INFO | "malloc" : " 14 GB", | 20260508 05:52:28.673 | INFO | "pinned" : " 29 GB" | 20260508 05:52:28.674 | INFO | } | 20260508 05:52:28.674 | INFO | Perf report for the last 5.13875 seconds: | 20260508 05:52:28.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:28.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:52:28.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:52:28.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:28.674 | INFO | Uploaded 36GiB to GPU in 2.53563s (14.1977GiB/s) | 20260508 05:52:28.674 | INFO | Downloaded 34GiB to GPU in 2.38612s (14.2491GiB/s) | 20260508 05:52:33.816 | INFO | Analyzing chunk frames = [5740544,5741056) | 20260508 05:52:33.816 | INFO | MemInfo: { | 20260508 05:52:33.816 | INFO | "gpu" : " 17 GB", | 20260508 05:52:33.816 | INFO | "huge" : "", | 20260508 05:52:33.816 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:33.817 | INFO | "malloc" : " 14 GB", | 20260508 05:52:33.817 | INFO | "pinned" : " 29 GB" | 20260508 05:52:33.817 | INFO | } | 20260508 05:52:33.817 | INFO | Perf report for the last 5.143 seconds: | 20260508 05:52:33.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:33.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:52:33.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 05:52:33.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:33.817 | INFO | Uploaded 36GiB to GPU in 2.31151s (15.5742GiB/s) | 20260508 05:52:33.817 | INFO | Downloaded 34GiB to GPU in 2.50557s (13.5698GiB/s) | 20260508 05:52:38.957 | INFO | Analyzing chunk frames = [5741056,5741568) | 20260508 05:52:38.957 | INFO | MemInfo: { | 20260508 05:52:38.957 | INFO | "gpu" : " 17 GB", | 20260508 05:52:38.957 | INFO | "huge" : "", | 20260508 05:52:38.957 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:38.957 | INFO | "malloc" : " 14 GB", | 20260508 05:52:38.957 | INFO | "pinned" : " 29 GB" | 20260508 05:52:38.957 | INFO | } | 20260508 05:52:38.957 | INFO | Perf report for the last 5.14047 seconds: | 20260508 05:52:38.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:38.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.957 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:52:38.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.957 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 05:52:38.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:38.958 | INFO | Uploaded 36GiB to GPU in 2.42842s (14.8245GiB/s) | 20260508 05:52:38.958 | INFO | Downloaded 34GiB to GPU in 2.44937s (13.8811GiB/s) | 20260508 05:52:44.095 | INFO | Analyzing chunk frames = [5741568,5742080) | 20260508 05:52:44.095 | INFO | MemInfo: { | 20260508 05:52:44.096 | INFO | "gpu" : " 17 GB", | 20260508 05:52:44.096 | INFO | "huge" : "", | 20260508 05:52:44.096 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:44.096 | INFO | "malloc" : " 14 GB", | 20260508 05:52:44.096 | INFO | "pinned" : " 29 GB" | 20260508 05:52:44.096 | INFO | } | 20260508 05:52:44.096 | INFO | Perf report for the last 5.1387 seconds: | 20260508 05:52:44.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:44.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.096 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:52:44.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.096 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 05:52:44.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:44.097 | INFO | Uploaded 36GiB to GPU in 2.4431s (14.7354GiB/s) | 20260508 05:52:44.097 | INFO | Downloaded 34GiB to GPU in 2.4508s (13.873GiB/s) | 20260508 05:52:49.230 | INFO | Analyzing chunk frames = [5742080,5742592) | 20260508 05:52:49.231 | INFO | MemInfo: { | 20260508 05:52:49.231 | INFO | "gpu" : " 17 GB", | 20260508 05:52:49.231 | INFO | "huge" : "", | 20260508 05:52:49.231 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:49.231 | INFO | "malloc" : " 14 GB", | 20260508 05:52:49.231 | INFO | "pinned" : " 29 GB" | 20260508 05:52:49.231 | INFO | } | 20260508 05:52:49.231 | INFO | Perf report for the last 5.13472 seconds: | 20260508 05:52:49.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:49.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:52:49.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 05:52:49.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:49.231 | INFO | Uploaded 36GiB to GPU in 2.37767s (15.1409GiB/s) | 20260508 05:52:49.231 | INFO | Downloaded 34GiB to GPU in 2.45286s (13.8614GiB/s) | 20260508 05:52:49.231 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:52:49.231 | INFO | Total mem usage: 16.7 GiB | 20260508 05:52:49.231 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:52:49.231 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.2%) | 20260508 05:52:49.231 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:52:49.231 | INFO | High Water Breakdown: | 20260508 05:52:49.231 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:52:49.231 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:52:49.231 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:52:49.231 | INFO | ------------------------------------------ | 20260508 05:52:49.231 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:52:49.231 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:52:49.232 | INFO | Total mem usage: 26.5 GiB | 20260508 05:52:49.232 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:52:49.232 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:52:49.232 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:52:49.232 | INFO | High Water Breakdown: | 20260508 05:52:49.232 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:52:49.232 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:52:49.232 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:52:49.232 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:52:49.232 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:52:49.232 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:52:49.232 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:52:49.232 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:52:49.232 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:52:49.232 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:52:49.232 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:52:49.232 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:52:49.232 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:52:49.232 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:52:49.232 | INFO | ------------------------------------------ | 20260508 05:52:49.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:52:49.232 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:52:49.232 | INFO | Total mem usage: 27.8 GiB | 20260508 05:52:49.232 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:52:49.232 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:52:49.232 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:52:49.232 | INFO | High Water Breakdown: | 20260508 05:52:49.232 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:52:49.232 | INFO | ------------------------------------------ | 20260508 05:52:49.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:52:49.232 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:52:49.232 | INFO | Total mem usage: 15.7 GiB | 20260508 05:52:49.232 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:52:49.232 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:52:49.232 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:52:49.232 | INFO | High Water Breakdown: | 20260508 05:52:49.232 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:52:49.232 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:52:49.232 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:52:49.232 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:52:49.232 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:52:49.232 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:52:49.232 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:52:49.232 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:52:49.232 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:52:49.232 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:52:49.232 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:52:49.232 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:52:49.232 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:52:49.232 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:52:49.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:52:49.232 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:52:49.232 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:52:49.232 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:52:49.232 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:52:49.232 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:52:49.232 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:52:49.232 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:52:49.232 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:52:49.232 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:52:49.232 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:52:49.232 | INFO | ------------------------------------------ | 20260508 05:52:49.232 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:52:49.232 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:52:49.232 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:52:49.232 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:52:49.232 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.40e+01 / 7.69e-01 | 20260508 05:52:49.232 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.05e-01, 1.13e-02 / 6.65e-03 | 20260508 05:52:49.232 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.44e+00, 5.95e-01 / 2.66e-01 | 20260508 05:52:49.232 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.12e-01 | 20260508 05:52:49.232 | INFO | PerfReport: 15.92%, DME -- 4.18e+01 / 4.54e+01, 3.85e+00 / 1.02e+00 | 20260508 05:52:49.232 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.87e+00 / 1.25e+00 | 20260508 05:52:49.232 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.37e+00 / 2.62e+00, 4.15e-01 / 1.76e-01 | 20260508 05:52:49.232 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.27e+00, 2.60e-01 / 1.47e-01 | 20260508 05:52:49.232 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.19e-01 / 3.59e-01, 5.05e-02 / 2.26e-02 | 20260508 05:52:49.233 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:52:49.233 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:52:49.233 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:52:49.233 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 24.26%, Upload -- 2.57e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 24.23%, Download -- 2.57e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 30.94%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 2.25%, LockUpload -- 2.39e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 2.54%, LockDownload -- 2.70e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 15.76%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:52:49.233 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260508 05:52:49.344 | INFO | Starting a round of Baz IO | 20260508 05:52:52.746 | INFO | Populating RTMetrics report | 20260508 05:52:52.747 | INFO | RTMetrics report populated | 20260508 05:52:52.747 | INFO | Opening tmp RTMetrics json file | 20260508 05:52:52.761 | INFO | Serializing RTMetrics to json | 20260508 05:52:52.762 | INFO | Serialization done, swapping tmp file now... | 20260508 05:52:52.763 | INFO | RTMetrics done | 20260508 05:52:52.823 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.479 seconds | 20260508 05:52:52.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.79% / 0.00% / 59.20% | 20260508 05:52:52.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.435479% / 0% / 5.16574e-05% | 20260508 05:52:52.823 | INFO | Baz IO overall: | 20260508 05:52:52.823 | INFO | BytesWritten : 2553869362743 | 20260508 05:52:52.823 | INFO | NumEvents : 1231812246719 | 20260508 05:52:54.371 | INFO | Analyzing chunk frames = [5742592,5743104) | 20260508 05:52:54.371 | INFO | MemInfo: { | 20260508 05:52:54.371 | INFO | "gpu" : " 17 GB", | 20260508 05:52:54.371 | INFO | "huge" : "", | 20260508 05:52:54.371 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:54.371 | INFO | "malloc" : " 14 GB", | 20260508 05:52:54.371 | INFO | "pinned" : " 29 GB" | 20260508 05:52:54.371 | INFO | } | 20260508 05:52:54.371 | INFO | Perf report for the last 5.14041 seconds: | 20260508 05:52:54.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:54.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:54.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:54.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:54.371 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 05:52:54.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:54.371 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 05:52:54.371 | INFO | PRE_HQ : 66.4%, 1.8, 61.7ms, 0 | 20260508 05:52:54.371 | INFO | BAZWRITER : 67.1%, 2.9, 98.6ms, 0 | 20260508 05:52:54.371 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:52:54.372 | INFO | Uploaded 36GiB to GPU in 2.58112s (13.9474GiB/s) | 20260508 05:52:54.372 | INFO | Downloaded 38GiB to GPU in 2.71473s (13.9977GiB/s) | 20260508 05:52:59.516 | INFO | Analyzing chunk frames = [5743104,5743616) | 20260508 05:52:59.516 | INFO | MemInfo: { | 20260508 05:52:59.516 | INFO | "gpu" : " 17 GB", | 20260508 05:52:59.516 | INFO | "huge" : "", | 20260508 05:52:59.516 | INFO | "huge_shared" : " 29 GB", | 20260508 05:52:59.516 | INFO | "malloc" : " 14 GB", | 20260508 05:52:59.516 | INFO | "pinned" : " 29 GB" | 20260508 05:52:59.516 | INFO | } | 20260508 05:52:59.517 | INFO | Perf report for the last 5.14545 seconds: | 20260508 05:52:59.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:52:59.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 05:52:59.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 05:52:59.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:52:59.517 | INFO | Uploaded 36GiB to GPU in 2.49711s (14.4167GiB/s) | 20260508 05:52:59.517 | INFO | Downloaded 34GiB to GPU in 2.43397s (13.9689GiB/s) | 20260508 05:53:04.654 | INFO | Analyzing chunk frames = [5743616,5744128) | 20260508 05:53:04.654 | INFO | MemInfo: { | 20260508 05:53:04.654 | INFO | "gpu" : " 17 GB", | 20260508 05:53:04.654 | INFO | "huge" : "", | 20260508 05:53:04.654 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:04.654 | INFO | "malloc" : " 14 GB", | 20260508 05:53:04.654 | INFO | "pinned" : " 29 GB" | 20260508 05:53:04.654 | INFO | } | 20260508 05:53:04.654 | INFO | Perf report for the last 5.13782 seconds: | 20260508 05:53:04.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:04.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.654 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:53:04.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.655 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:53:04.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:04.655 | INFO | Uploaded 36GiB to GPU in 2.54365s (14.1529GiB/s) | 20260508 05:53:04.655 | INFO | Downloaded 34GiB to GPU in 2.41007s (14.1075GiB/s) | 20260508 05:53:09.790 | INFO | Analyzing chunk frames = [5744128,5744640) | 20260508 05:53:09.790 | INFO | MemInfo: { | 20260508 05:53:09.790 | INFO | "gpu" : " 17 GB", | 20260508 05:53:09.790 | INFO | "huge" : "", | 20260508 05:53:09.790 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:09.790 | INFO | "malloc" : " 14 GB", | 20260508 05:53:09.790 | INFO | "pinned" : " 29 GB" | 20260508 05:53:09.790 | INFO | } | 20260508 05:53:09.790 | INFO | Perf report for the last 5.13596 seconds: | 20260508 05:53:09.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:09.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.790 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 05:53:09.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.790 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:53:09.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:09.791 | INFO | Uploaded 36GiB to GPU in 2.54598s (14.1399GiB/s) | 20260508 05:53:09.791 | INFO | Downloaded 34GiB to GPU in 2.40553s (14.1341GiB/s) | 20260508 05:53:14.923 | INFO | Analyzing chunk frames = [5744640,5745152) | 20260508 05:53:14.923 | INFO | MemInfo: { | 20260508 05:53:14.923 | INFO | "gpu" : " 17 GB", | 20260508 05:53:14.923 | INFO | "huge" : "", | 20260508 05:53:14.923 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:14.923 | INFO | "malloc" : " 14 GB", | 20260508 05:53:14.923 | INFO | "pinned" : " 29 GB" | 20260508 05:53:14.923 | INFO | } | 20260508 05:53:14.923 | INFO | Perf report for the last 5.1327 seconds: | 20260508 05:53:14.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:14.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 05:53:14.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 05:53:14.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:14.923 | INFO | Uploaded 36GiB to GPU in 2.37058s (15.1861GiB/s) | 20260508 05:53:14.923 | INFO | Downloaded 34GiB to GPU in 2.56731s (13.2434GiB/s) | 20260508 05:53:20.066 | INFO | Analyzing chunk frames = [5745152,5745664) | 20260508 05:53:20.066 | INFO | MemInfo: { | 20260508 05:53:20.066 | INFO | "gpu" : " 17 GB", | 20260508 05:53:20.066 | INFO | "huge" : "", | 20260508 05:53:20.066 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:20.066 | INFO | "malloc" : " 14 GB", | 20260508 05:53:20.066 | INFO | "pinned" : " 29 GB" | 20260508 05:53:20.066 | INFO | } | 20260508 05:53:20.066 | INFO | Perf report for the last 5.1428 seconds: | 20260508 05:53:20.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:20.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 05:53:20.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:53:20.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:20.066 | INFO | Uploaded 36GiB to GPU in 2.45243s (14.6793GiB/s) | 20260508 05:53:20.066 | INFO | Downloaded 34GiB to GPU in 2.41292s (14.0908GiB/s) | 20260508 05:53:25.205 | INFO | Analyzing chunk frames = [5745664,5746176) | 20260508 05:53:25.205 | INFO | MemInfo: { | 20260508 05:53:25.205 | INFO | "gpu" : " 17 GB", | 20260508 05:53:25.205 | INFO | "huge" : "", | 20260508 05:53:25.205 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:25.205 | INFO | "malloc" : " 14 GB", | 20260508 05:53:25.205 | INFO | "pinned" : " 29 GB" | 20260508 05:53:25.206 | INFO | } | 20260508 05:53:25.206 | INFO | Perf report for the last 5.13971 seconds: | 20260508 05:53:25.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:25.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 05:53:25.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 05:53:25.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:25.206 | INFO | Uploaded 36GiB to GPU in 2.45369s (14.6718GiB/s) | 20260508 05:53:25.206 | INFO | Downloaded 34GiB to GPU in 2.41441s (14.0821GiB/s) | 20260508 05:53:30.344 | INFO | Analyzing chunk frames = [5746176,5746688) | 20260508 05:53:30.344 | INFO | MemInfo: { | 20260508 05:53:30.344 | INFO | "gpu" : " 17 GB", | 20260508 05:53:30.344 | INFO | "huge" : "", | 20260508 05:53:30.344 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:30.345 | INFO | "malloc" : " 14 GB", | 20260508 05:53:30.345 | INFO | "pinned" : " 29 GB" | 20260508 05:53:30.345 | INFO | } | 20260508 05:53:30.345 | INFO | Perf report for the last 5.13902 seconds: | 20260508 05:53:30.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:30.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 05:53:30.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260508 05:53:30.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:30.345 | INFO | Uploaded 36GiB to GPU in 2.48484s (14.4879GiB/s) | 20260508 05:53:30.345 | INFO | Downloaded 34GiB to GPU in 2.42628s (14.0132GiB/s) | 20260508 05:53:30.442 | INFO | Starting a round of Baz IO | 20260508 05:53:33.878 | INFO | Populating RTMetrics report | 20260508 05:53:33.879 | INFO | RTMetrics report populated | 20260508 05:53:33.879 | INFO | Opening tmp RTMetrics json file | 20260508 05:53:33.888 | INFO | Serializing RTMetrics to json | 20260508 05:53:33.889 | INFO | Serialization done, swapping tmp file now... | 20260508 05:53:33.890 | INFO | RTMetrics done | 20260508 05:53:33.947 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.505 seconds | 20260508 05:53:33.947 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.79% / 0.00% / 59.21% | 20260508 05:53:33.948 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.436203% / 7.94729e-06% / 4.76837e-05% | 20260508 05:53:33.948 | INFO | Baz IO overall: | 20260508 05:53:33.948 | INFO | BytesWritten : 2555549901728 | 20260508 05:53:33.948 | INFO | NumEvents : 1232613434777 | 20260508 05:53:35.484 | INFO | Analyzing chunk frames = [5746688,5747200) | 20260508 05:53:35.484 | INFO | MemInfo: { | 20260508 05:53:35.484 | INFO | "gpu" : " 17 GB", | 20260508 05:53:35.485 | INFO | "huge" : "", | 20260508 05:53:35.485 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:35.485 | INFO | "malloc" : " 14 GB", | 20260508 05:53:35.485 | INFO | "pinned" : " 29 GB" | 20260508 05:53:35.485 | INFO | } | 20260508 05:53:35.485 | INFO | Perf report for the last 5.14 seconds: | 20260508 05:53:35.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:35.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:35.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:35.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:35.485 | INFO | ANALYSIS : 67.8%, 3.0, 106.1ms, 0 | 20260508 05:53:35.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:35.485 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 05:53:35.485 | INFO | PRE_HQ : 65.4%, 1.6, 55.2ms, 0 | 20260508 05:53:35.485 | INFO | BAZWRITER : 67.6%, 2.7, 92.2ms, 0 | 20260508 05:53:35.485 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 05:53:35.485 | INFO | Uploaded 36GiB to GPU in 2.57488s (13.9812GiB/s) | 20260508 05:53:35.485 | INFO | Downloaded 38GiB to GPU in 2.68561s (14.1495GiB/s) | 20260508 05:53:40.623 | INFO | Analyzing chunk frames = [5747200,5747712) | 20260508 05:53:40.623 | INFO | MemInfo: { | 20260508 05:53:40.623 | INFO | "gpu" : " 17 GB", | 20260508 05:53:40.623 | INFO | "huge" : "", | 20260508 05:53:40.623 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:40.623 | INFO | "malloc" : " 14 GB", | 20260508 05:53:40.623 | INFO | "pinned" : " 29 GB" | 20260508 05:53:40.623 | INFO | } | 20260508 05:53:40.623 | INFO | Perf report for the last 5.13845 seconds: | 20260508 05:53:40.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:40.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | ANALYSIS : 67.2%, 3.0, 104.7ms, 0 | 20260508 05:53:40.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:53:40.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:40.623 | INFO | Uploaded 36GiB to GPU in 2.51938s (14.2892GiB/s) | 20260508 05:53:40.623 | INFO | Downloaded 34GiB to GPU in 2.42229s (14.0363GiB/s) | 20260508 05:53:45.766 | INFO | Analyzing chunk frames = [5747712,5748224) | 20260508 05:53:45.766 | INFO | MemInfo: { | 20260508 05:53:45.766 | INFO | "gpu" : " 17 GB", | 20260508 05:53:45.766 | INFO | "huge" : "", | 20260508 05:53:45.766 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:45.766 | INFO | "malloc" : " 14 GB", | 20260508 05:53:45.766 | INFO | "pinned" : " 29 GB" | 20260508 05:53:45.767 | INFO | } | 20260508 05:53:45.767 | INFO | Perf report for the last 5.14344 seconds: | 20260508 05:53:45.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:45.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 05:53:45.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:53:45.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:45.767 | INFO | Uploaded 36GiB to GPU in 2.54512s (14.1447GiB/s) | 20260508 05:53:45.767 | INFO | Downloaded 34GiB to GPU in 2.39524s (14.1948GiB/s) | 20260508 05:53:50.908 | INFO | Analyzing chunk frames = [5748224,5748736) | 20260508 05:53:50.908 | INFO | MemInfo: { | 20260508 05:53:50.908 | INFO | "gpu" : " 17 GB", | 20260508 05:53:50.908 | INFO | "huge" : "", | 20260508 05:53:50.908 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:50.908 | INFO | "malloc" : " 14 GB", | 20260508 05:53:50.908 | INFO | "pinned" : " 29 GB" | 20260508 05:53:50.909 | INFO | } | 20260508 05:53:50.909 | INFO | Perf report for the last 5.14205 seconds: | 20260508 05:53:50.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:50.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 05:53:50.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:53:50.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:50.909 | INFO | Uploaded 36GiB to GPU in 2.58177s (13.9439GiB/s) | 20260508 05:53:50.909 | INFO | Downloaded 34GiB to GPU in 2.38849s (14.2349GiB/s) | 20260508 05:53:56.045 | INFO | Analyzing chunk frames = [5748736,5749248) | 20260508 05:53:56.045 | INFO | MemInfo: { | 20260508 05:53:56.045 | INFO | "gpu" : " 17 GB", | 20260508 05:53:56.045 | INFO | "huge" : "", | 20260508 05:53:56.045 | INFO | "huge_shared" : " 29 GB", | 20260508 05:53:56.045 | INFO | "malloc" : " 14 GB", | 20260508 05:53:56.045 | INFO | "pinned" : " 29 GB" | 20260508 05:53:56.045 | INFO | } | 20260508 05:53:56.045 | INFO | Perf report for the last 5.13626 seconds: | 20260508 05:53:56.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:53:56.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 05:53:56.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:53:56.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:53:56.045 | INFO | Uploaded 36GiB to GPU in 2.32369s (15.4926GiB/s) | 20260508 05:53:56.045 | INFO | Downloaded 34GiB to GPU in 2.49217s (13.6427GiB/s) | 20260508 05:54:01.177 | INFO | Analyzing chunk frames = [5749248,5749760) | 20260508 05:54:01.177 | INFO | MemInfo: { | 20260508 05:54:01.177 | INFO | "gpu" : " 17 GB", | 20260508 05:54:01.177 | INFO | "huge" : "", | 20260508 05:54:01.177 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:01.177 | INFO | "malloc" : " 14 GB", | 20260508 05:54:01.177 | INFO | "pinned" : " 29 GB" | 20260508 05:54:01.177 | INFO | } | 20260508 05:54:01.177 | INFO | Perf report for the last 5.13217 seconds: | 20260508 05:54:01.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:01.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.177 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 05:54:01.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.177 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:54:01.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:01.178 | INFO | Uploaded 36GiB to GPU in 2.45762s (14.6483GiB/s) | 20260508 05:54:01.178 | INFO | Downloaded 34GiB to GPU in 2.41151s (14.099GiB/s) | 20260508 05:54:06.321 | INFO | Analyzing chunk frames = [5749760,5750272) | 20260508 05:54:06.321 | INFO | MemInfo: { | 20260508 05:54:06.321 | INFO | "gpu" : " 17 GB", | 20260508 05:54:06.321 | INFO | "huge" : "", | 20260508 05:54:06.321 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:06.321 | INFO | "malloc" : " 14 GB", | 20260508 05:54:06.322 | INFO | "pinned" : " 29 GB" | 20260508 05:54:06.322 | INFO | } | 20260508 05:54:06.322 | INFO | Perf report for the last 5.14464 seconds: | 20260508 05:54:06.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:06.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | ANALYSIS : 66.2%, 3.0, 103.7ms, 0 | 20260508 05:54:06.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:54:06.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:06.322 | INFO | Uploaded 36GiB to GPU in 2.47398s (14.5514GiB/s) | 20260508 05:54:06.322 | INFO | Downloaded 34GiB to GPU in 2.4199s (14.0502GiB/s) | 20260508 05:54:11.462 | INFO | Analyzing chunk frames = [5750272,5750784) | 20260508 05:54:11.462 | INFO | MemInfo: { | 20260508 05:54:11.462 | INFO | "gpu" : " 17 GB", | 20260508 05:54:11.462 | INFO | "huge" : "", | 20260508 05:54:11.463 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:11.463 | INFO | "malloc" : " 14 GB", | 20260508 05:54:11.463 | INFO | "pinned" : " 29 GB" | 20260508 05:54:11.463 | INFO | } | 20260508 05:54:11.463 | INFO | Perf report for the last 5.14095 seconds: | 20260508 05:54:11.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:11.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:54:11.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:54:11.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:11.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:54:11.463 | INFO | Total mem usage: 16.7 GiB | 20260508 05:54:11.463 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:54:11.463 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 05:54:11.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:54:11.463 | INFO | High Water Breakdown: | 20260508 05:54:11.463 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:54:11.463 | INFO | 40.3% : 5.78 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:54:11.463 | INFO | Uploaded 36GiB to GPU in 2.48873s (14.4652GiB/s) | 20260508 05:54:11.463 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:54:11.463 | INFO | ------------------------------------------ | 20260508 05:54:11.463 | INFO | Downloaded 34GiB to GPU in 2.43314s (13.9737GiB/s) | 20260508 05:54:11.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:54:11.464 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:54:11.464 | INFO | Total mem usage: 26.5 GiB | 20260508 05:54:11.464 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:54:11.464 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:54:11.464 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:54:11.464 | INFO | High Water Breakdown: | 20260508 05:54:11.464 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:54:11.464 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:54:11.464 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:54:11.464 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:54:11.464 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:54:11.464 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:54:11.464 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:54:11.464 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:54:11.464 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:54:11.464 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:54:11.464 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:54:11.464 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:54:11.464 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:54:11.464 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:54:11.464 | INFO | ------------------------------------------ | 20260508 05:54:11.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:54:11.464 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:54:11.464 | INFO | Total mem usage: 27.8 GiB | 20260508 05:54:11.464 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:54:11.464 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:54:11.464 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:54:11.464 | INFO | High Water Breakdown: | 20260508 05:54:11.464 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:54:11.464 | INFO | ------------------------------------------ | 20260508 05:54:11.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:54:11.464 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:54:11.464 | INFO | Total mem usage: 15.7 GiB | 20260508 05:54:11.464 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:54:11.464 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:54:11.464 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:54:11.464 | INFO | High Water Breakdown: | 20260508 05:54:11.464 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:54:11.464 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:54:11.464 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:54:11.464 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:54:11.464 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:54:11.464 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:54:11.464 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:54:11.464 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:54:11.464 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:54:11.464 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:54:11.464 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:54:11.464 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:54:11.464 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:54:11.464 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:54:11.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:54:11.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:54:11.464 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:54:11.464 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:54:11.464 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:54:11.465 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:54:11.465 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:54:11.465 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:54:11.465 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:54:11.465 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:54:11.465 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:54:11.465 | INFO | ------------------------------------------ | 20260508 05:54:11.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:54:11.465 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:54:11.465 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:54:11.465 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:54:11.465 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.40e+01 / 1.63e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.06e-01, 1.15e-02 / 1.10e-02 | 20260508 05:54:11.465 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.45e+00, 5.91e-01 / 2.22e-01 | 20260508 05:54:11.465 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.16e-01 | 20260508 05:54:11.465 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.53e+01, 4.02e+00 / 1.80e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.75e+00 / 1.39e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.55e+00, 4.15e-01 / 2.65e-01 | 20260508 05:54:11.465 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.22e+00, 2.56e-01 / 1.34e-01 | 20260508 05:54:11.465 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.16e-01 / 3.52e-01, 4.98e-02 / 2.59e-02 | 20260508 05:54:11.465 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:54:11.465 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:54:11.465 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:54:11.465 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 24.19%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 2.38%, LockUpload -- 2.53e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 2.55%, LockDownload -- 2.70e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 15.59%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:54:11.465 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260508 05:54:11.560 | INFO | Starting a round of Baz IO | 20260508 05:54:14.986 | INFO | Populating RTMetrics report | 20260508 05:54:14.986 | INFO | RTMetrics report populated | 20260508 05:54:14.986 | INFO | Opening tmp RTMetrics json file | 20260508 05:54:14.999 | INFO | Serializing RTMetrics to json | 20260508 05:54:15.000 | INFO | Serialization done, swapping tmp file now... | 20260508 05:54:15.001 | INFO | RTMetrics done | 20260508 05:54:15.063 | INFO | Finished a round of Baz IO: Wrote 1.565 GiB in 3.503 seconds | 20260508 05:54:15.063 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.79% / 0.00% / 59.21% | 20260508 05:54:15.063 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.434955% / 3.97364e-06% / 5.16574e-05% | 20260508 05:54:15.063 | INFO | Baz IO overall: | 20260508 05:54:15.063 | INFO | BytesWritten : 2557229904967 | 20260508 05:54:15.063 | INFO | NumEvents : 1233414269674 | 20260508 05:54:16.603 | INFO | Analyzing chunk frames = [5750784,5751296) | 20260508 05:54:16.603 | INFO | MemInfo: { | 20260508 05:54:16.603 | INFO | "gpu" : " 17 GB", | 20260508 05:54:16.603 | INFO | "huge" : "", | 20260508 05:54:16.603 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:16.603 | INFO | "malloc" : " 14 GB", | 20260508 05:54:16.603 | INFO | "pinned" : " 29 GB" | 20260508 05:54:16.603 | INFO | } | 20260508 05:54:16.603 | INFO | Perf report for the last 5.14003 seconds: | 20260508 05:54:16.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:16.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:16.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:16.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:16.603 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 05:54:16.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:16.603 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:54:16.603 | INFO | PRE_HQ : 65.8%, 1.6, 55.4ms, 0 | 20260508 05:54:16.603 | INFO | BAZWRITER : 67.6%, 2.7, 92.8ms, 0 | 20260508 05:54:16.603 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:54:16.603 | INFO | Uploaded 36GiB to GPU in 2.58489s (13.9271GiB/s) | 20260508 05:54:16.603 | INFO | Downloaded 38GiB to GPU in 2.68355s (14.1604GiB/s) | 20260508 05:54:21.738 | INFO | Analyzing chunk frames = [5751296,5751808) | 20260508 05:54:21.738 | INFO | MemInfo: { | 20260508 05:54:21.738 | INFO | "gpu" : " 17 GB", | 20260508 05:54:21.738 | INFO | "huge" : "", | 20260508 05:54:21.738 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:21.738 | INFO | "malloc" : " 14 GB", | 20260508 05:54:21.739 | INFO | "pinned" : " 29 GB" | 20260508 05:54:21.739 | INFO | } | 20260508 05:54:21.739 | INFO | Perf report for the last 5.13647 seconds: | 20260508 05:54:21.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:21.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 05:54:21.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:54:21.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:21.739 | INFO | Uploaded 36GiB to GPU in 2.59588s (13.8681GiB/s) | 20260508 05:54:21.740 | INFO | Downloaded 34GiB to GPU in 2.43801s (13.9458GiB/s) | 20260508 05:54:26.880 | INFO | Analyzing chunk frames = [5751808,5752320) | 20260508 05:54:26.880 | INFO | MemInfo: { | 20260508 05:54:26.880 | INFO | "gpu" : " 17 GB", | 20260508 05:54:26.880 | INFO | "huge" : "", | 20260508 05:54:26.880 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:26.880 | INFO | "malloc" : " 14 GB", | 20260508 05:54:26.880 | INFO | "pinned" : " 29 GB" | 20260508 05:54:26.880 | INFO | } | 20260508 05:54:26.880 | INFO | Perf report for the last 5.14126 seconds: | 20260508 05:54:26.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:26.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.881 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 05:54:26.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.881 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:54:26.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:26.881 | INFO | Uploaded 36GiB to GPU in 2.55728s (14.0775GiB/s) | 20260508 05:54:26.881 | INFO | Downloaded 34GiB to GPU in 2.40891s (14.1142GiB/s) | 20260508 05:54:32.019 | INFO | Analyzing chunk frames = [5752320,5752832) | 20260508 05:54:32.019 | INFO | MemInfo: { | 20260508 05:54:32.019 | INFO | "gpu" : " 17 GB", | 20260508 05:54:32.019 | INFO | "huge" : "", | 20260508 05:54:32.019 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:32.019 | INFO | "malloc" : " 14 GB", | 20260508 05:54:32.019 | INFO | "pinned" : " 29 GB" | 20260508 05:54:32.019 | INFO | } | 20260508 05:54:32.020 | INFO | Perf report for the last 5.13903 seconds: | 20260508 05:54:32.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:32.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 05:54:32.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:54:32.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:32.020 | INFO | Uploaded 36GiB to GPU in 2.57651s (13.9724GiB/s) | 20260508 05:54:32.020 | INFO | Downloaded 34GiB to GPU in 2.39137s (14.2178GiB/s) | 20260508 05:54:37.160 | INFO | Analyzing chunk frames = [5752832,5753344) | 20260508 05:54:37.160 | INFO | MemInfo: { | 20260508 05:54:37.160 | INFO | "gpu" : " 17 GB", | 20260508 05:54:37.160 | INFO | "huge" : "", | 20260508 05:54:37.160 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:37.160 | INFO | "malloc" : " 14 GB", | 20260508 05:54:37.160 | INFO | "pinned" : " 29 GB" | 20260508 05:54:37.160 | INFO | } | 20260508 05:54:37.160 | INFO | Perf report for the last 5.14054 seconds: | 20260508 05:54:37.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:37.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 05:54:37.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 05:54:37.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:37.160 | INFO | Uploaded 36GiB to GPU in 2.46078s (14.6295GiB/s) | 20260508 05:54:37.160 | INFO | Downloaded 34GiB to GPU in 2.52545s (13.4629GiB/s) | 20260508 05:54:42.294 | INFO | Analyzing chunk frames = [5753344,5753856) | 20260508 05:54:42.294 | INFO | MemInfo: { | 20260508 05:54:42.294 | INFO | "gpu" : " 17 GB", | 20260508 05:54:42.294 | INFO | "huge" : "", | 20260508 05:54:42.294 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:42.294 | INFO | "malloc" : " 14 GB", | 20260508 05:54:42.294 | INFO | "pinned" : " 29 GB" | 20260508 05:54:42.294 | INFO | } | 20260508 05:54:42.294 | INFO | Perf report for the last 5.13391 seconds: | 20260508 05:54:42.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:42.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:54:42.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:54:42.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:42.294 | INFO | Uploaded 36GiB to GPU in 2.44252s (14.7389GiB/s) | 20260508 05:54:42.294 | INFO | Downloaded 34GiB to GPU in 2.42409s (14.0259GiB/s) | 20260508 05:54:47.432 | INFO | Analyzing chunk frames = [5753856,5754368) | 20260508 05:54:47.432 | INFO | MemInfo: { | 20260508 05:54:47.432 | INFO | "gpu" : " 17 GB", | 20260508 05:54:47.432 | INFO | "huge" : "", | 20260508 05:54:47.432 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:47.432 | INFO | "malloc" : " 14 GB", | 20260508 05:54:47.432 | INFO | "pinned" : " 29 GB" | 20260508 05:54:47.432 | INFO | } | 20260508 05:54:47.432 | INFO | Perf report for the last 5.1381 seconds: | 20260508 05:54:47.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:47.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.432 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:54:47.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.432 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:54:47.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:47.433 | INFO | Uploaded 36GiB to GPU in 2.46073s (14.6298GiB/s) | 20260508 05:54:47.433 | INFO | Downloaded 34GiB to GPU in 2.43628s (13.9557GiB/s) | 20260508 05:54:52.576 | INFO | Analyzing chunk frames = [5754368,5754880) | 20260508 05:54:52.576 | INFO | MemInfo: { | 20260508 05:54:52.576 | INFO | "gpu" : " 17 GB", | 20260508 05:54:52.576 | INFO | "huge" : "", | 20260508 05:54:52.577 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:52.577 | INFO | "malloc" : " 14 GB", | 20260508 05:54:52.577 | INFO | "pinned" : " 29 GB" | 20260508 05:54:52.577 | INFO | } | 20260508 05:54:52.577 | INFO | Perf report for the last 5.14455 seconds: | 20260508 05:54:52.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:52.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:54:52.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:54:52.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:52.577 | INFO | Uploaded 36GiB to GPU in 2.45663s (14.6542GiB/s) | 20260508 05:54:52.577 | INFO | Downloaded 34GiB to GPU in 2.43076s (13.9874GiB/s) | 20260508 05:54:52.676 | INFO | Starting a round of Baz IO | 20260508 05:54:56.090 | INFO | Populating RTMetrics report | 20260508 05:54:56.091 | INFO | RTMetrics report populated | 20260508 05:54:56.091 | INFO | Opening tmp RTMetrics json file | 20260508 05:54:56.101 | INFO | Serializing RTMetrics to json | 20260508 05:54:56.104 | INFO | Serialization done, swapping tmp file now... | 20260508 05:54:56.104 | INFO | RTMetrics done | 20260508 05:54:56.165 | INFO | Finished a round of Baz IO: Wrote 1.563 GiB in 3.489 seconds | 20260508 05:54:56.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.79% / 0.00% / 59.21% | 20260508 05:54:56.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.431585% / 0% / 3.17891e-05% | 20260508 05:54:56.165 | INFO | Baz IO overall: | 20260508 05:54:56.165 | INFO | BytesWritten : 2558908600121 | 20260508 05:54:56.165 | INFO | NumEvents : 1234214168531 | 20260508 05:54:57.723 | INFO | Analyzing chunk frames = [5754880,5755392) | 20260508 05:54:57.723 | INFO | MemInfo: { | 20260508 05:54:57.723 | INFO | "gpu" : " 17 GB", | 20260508 05:54:57.723 | INFO | "huge" : "", | 20260508 05:54:57.724 | INFO | "huge_shared" : " 29 GB", | 20260508 05:54:57.724 | INFO | "malloc" : " 14 GB", | 20260508 05:54:57.724 | INFO | "pinned" : " 29 GB" | 20260508 05:54:57.724 | INFO | } | 20260508 05:54:57.724 | INFO | Perf report for the last 5.14698 seconds: | 20260508 05:54:57.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:54:57.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:57.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:57.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:57.724 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 05:54:57.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:54:57.724 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:54:57.724 | INFO | PRE_HQ : 65.4%, 1.6, 55.5ms, 0 | 20260508 05:54:57.724 | INFO | BAZWRITER : 67.4%, 2.7, 92.0ms, 0 | 20260508 05:54:57.724 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:54:57.724 | INFO | Uploaded 36GiB to GPU in 2.55298s (14.1012GiB/s) | 20260508 05:54:57.724 | INFO | Downloaded 38GiB to GPU in 2.68305s (14.163GiB/s) | 20260508 05:55:02.859 | INFO | Analyzing chunk frames = [5755392,5755904) | 20260508 05:55:02.859 | INFO | MemInfo: { | 20260508 05:55:02.859 | INFO | "gpu" : " 17 GB", | 20260508 05:55:02.859 | INFO | "huge" : "", | 20260508 05:55:02.860 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:02.860 | INFO | "malloc" : " 14 GB", | 20260508 05:55:02.860 | INFO | "pinned" : " 29 GB" | 20260508 05:55:02.860 | INFO | } | 20260508 05:55:02.860 | INFO | Perf report for the last 5.13604 seconds: | 20260508 05:55:02.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:02.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.860 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:55:02.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.860 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:55:02.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:02.861 | INFO | Uploaded 36GiB to GPU in 2.49733s (14.4154GiB/s) | 20260508 05:55:02.861 | INFO | Downloaded 34GiB to GPU in 2.43643s (13.9549GiB/s) | 20260508 05:55:07.992 | INFO | Analyzing chunk frames = [5755904,5756416) | 20260508 05:55:07.992 | INFO | MemInfo: { | 20260508 05:55:07.992 | INFO | "gpu" : " 17 GB", | 20260508 05:55:07.992 | INFO | "huge" : "", | 20260508 05:55:07.992 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:07.992 | INFO | "malloc" : " 14 GB", | 20260508 05:55:07.992 | INFO | "pinned" : " 29 GB" | 20260508 05:55:07.992 | INFO | } | 20260508 05:55:07.992 | INFO | Perf report for the last 5.13234 seconds: | 20260508 05:55:07.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:07.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.992 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 05:55:07.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.992 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 05:55:07.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:07.993 | INFO | Uploaded 36GiB to GPU in 2.54685s (14.1351GiB/s) | 20260508 05:55:07.993 | INFO | Downloaded 34GiB to GPU in 2.41539s (14.0764GiB/s) | 20260508 05:55:13.131 | INFO | Analyzing chunk frames = [5756416,5756928) | 20260508 05:55:13.131 | INFO | MemInfo: { | 20260508 05:55:13.131 | INFO | "gpu" : " 17 GB", | 20260508 05:55:13.131 | INFO | "huge" : "", | 20260508 05:55:13.131 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:13.131 | INFO | "malloc" : " 14 GB", | 20260508 05:55:13.131 | INFO | "pinned" : " 29 GB" | 20260508 05:55:13.131 | INFO | } | 20260508 05:55:13.131 | INFO | Perf report for the last 5.13886 seconds: | 20260508 05:55:13.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:13.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 05:55:13.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:55:13.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:13.131 | INFO | Uploaded 36GiB to GPU in 2.53726s (14.1885GiB/s) | 20260508 05:55:13.131 | INFO | Downloaded 34GiB to GPU in 2.40106s (14.1604GiB/s) | 20260508 05:55:18.274 | INFO | Analyzing chunk frames = [5756928,5757440) | 20260508 05:55:18.274 | INFO | MemInfo: { | 20260508 05:55:18.274 | INFO | "gpu" : " 17 GB", | 20260508 05:55:18.274 | INFO | "huge" : "", | 20260508 05:55:18.274 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:18.274 | INFO | "malloc" : " 14 GB", | 20260508 05:55:18.275 | INFO | "pinned" : " 29 GB" | 20260508 05:55:18.275 | INFO | } | 20260508 05:55:18.275 | INFO | Perf report for the last 5.14377 seconds: | 20260508 05:55:18.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:18.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | ANALYSIS : 66.7%, 3.0, 104.3ms, 0 | 20260508 05:55:18.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 05:55:18.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:18.275 | INFO | Uploaded 36GiB to GPU in 2.47216s (14.5622GiB/s) | 20260508 05:55:18.275 | INFO | Downloaded 34GiB to GPU in 2.52883s (13.4449GiB/s) | 20260508 05:55:23.413 | INFO | Analyzing chunk frames = [5757440,5757952) | 20260508 05:55:23.413 | INFO | MemInfo: { | 20260508 05:55:23.413 | INFO | "gpu" : " 17 GB", | 20260508 05:55:23.413 | INFO | "huge" : "", | 20260508 05:55:23.413 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:23.413 | INFO | "malloc" : " 14 GB", | 20260508 05:55:23.413 | INFO | "pinned" : " 29 GB" | 20260508 05:55:23.413 | INFO | } | 20260508 05:55:23.413 | INFO | Perf report for the last 5.13826 seconds: | 20260508 05:55:23.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:23.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:55:23.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:55:23.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:23.413 | INFO | Uploaded 36GiB to GPU in 2.44145s (14.7454GiB/s) | 20260508 05:55:23.413 | INFO | Downloaded 34GiB to GPU in 2.43101s (13.9859GiB/s) | 20260508 05:55:28.556 | INFO | Analyzing chunk frames = [5757952,5758464) | 20260508 05:55:28.556 | INFO | MemInfo: { | 20260508 05:55:28.556 | INFO | "gpu" : " 17 GB", | 20260508 05:55:28.556 | INFO | "huge" : "", | 20260508 05:55:28.556 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:28.556 | INFO | "malloc" : " 14 GB", | 20260508 05:55:28.556 | INFO | "pinned" : " 29 GB" | 20260508 05:55:28.556 | INFO | } | 20260508 05:55:28.556 | INFO | Perf report for the last 5.14336 seconds: | 20260508 05:55:28.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:28.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.556 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:55:28.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.556 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 05:55:28.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:28.557 | INFO | Uploaded 36GiB to GPU in 2.44419s (14.7288GiB/s) | 20260508 05:55:28.557 | INFO | Downloaded 34GiB to GPU in 2.42406s (14.026GiB/s) | 20260508 05:55:33.690 | INFO | Analyzing chunk frames = [5758464,5758976) | 20260508 05:55:33.690 | INFO | MemInfo: { | 20260508 05:55:33.690 | INFO | "gpu" : " 17 GB", | 20260508 05:55:33.690 | INFO | "huge" : "", | 20260508 05:55:33.690 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:33.690 | INFO | "malloc" : " 14 GB", | 20260508 05:55:33.690 | INFO | "pinned" : " 29 GB" | 20260508 05:55:33.690 | INFO | } | 20260508 05:55:33.690 | INFO | Perf report for the last 5.13402 seconds: | 20260508 05:55:33.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:33.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.690 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 05:55:33.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.691 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 05:55:33.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:33.691 | INFO | Uploaded 36GiB to GPU in 2.47252s (14.5601GiB/s) | 20260508 05:55:33.691 | INFO | Downloaded 34GiB to GPU in 2.43858s (13.9426GiB/s) | 20260508 05:55:33.691 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:55:33.691 | INFO | Total mem usage: 16.7 GiB | 20260508 05:55:33.691 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:55:33.691 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 05:55:33.691 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:55:33.691 | INFO | High Water Breakdown: | 20260508 05:55:33.691 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:55:33.691 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:55:33.691 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:55:33.691 | INFO | ------------------------------------------ | 20260508 05:55:33.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:55:33.691 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:55:33.691 | INFO | Total mem usage: 26.5 GiB | 20260508 05:55:33.691 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:55:33.691 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:55:33.691 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:55:33.691 | INFO | High Water Breakdown: | 20260508 05:55:33.691 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:55:33.691 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:55:33.691 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:55:33.691 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:55:33.691 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:55:33.691 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:55:33.691 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:55:33.691 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:55:33.691 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:55:33.691 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:55:33.691 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:55:33.691 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:55:33.691 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:55:33.691 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:55:33.691 | INFO | ------------------------------------------ | 20260508 05:55:33.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:55:33.692 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:55:33.692 | INFO | Total mem usage: 27.8 GiB | 20260508 05:55:33.692 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:55:33.692 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:55:33.692 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:55:33.692 | INFO | High Water Breakdown: | 20260508 05:55:33.692 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:55:33.692 | INFO | ------------------------------------------ | 20260508 05:55:33.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:55:33.692 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:55:33.692 | INFO | Total mem usage: 15.7 GiB | 20260508 05:55:33.692 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:55:33.692 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:55:33.692 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:55:33.692 | INFO | High Water Breakdown: | 20260508 05:55:33.692 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:55:33.692 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:55:33.692 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:55:33.692 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:55:33.692 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:55:33.692 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:55:33.692 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:55:33.692 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:55:33.692 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:55:33.692 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:55:33.692 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:55:33.692 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:55:33.692 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:55:33.692 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:55:33.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:55:33.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:55:33.692 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:55:33.692 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:55:33.692 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:55:33.692 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:55:33.692 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:55:33.692 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:55:33.692 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:55:33.692 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:55:33.692 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:55:33.692 | INFO | ------------------------------------------ | 20260508 05:55:33.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:55:33.692 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:55:33.692 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:55:33.692 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:55:33.692 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 6.48e-01 | 20260508 05:55:33.692 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.12e-02 / 6.23e-03 | 20260508 05:55:33.692 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.40e+00, 5.83e-01 / 2.44e-01 | 20260508 05:55:33.692 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.44e+00, 2.09e-01 / 1.40e-01 | 20260508 05:55:33.692 | INFO | PerfReport: 15.89%, DME -- 4.15e+01 / 4.46e+01, 3.66e+00 / 1.13e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.79e+00 / 1.52e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.37e+00 / 2.50e+00, 4.12e-01 / 2.67e-01 | 20260508 05:55:33.692 | INFO | PerfReport: 3.17%, Metrics -- 1.03e+00 / 1.22e+00, 2.57e-01 / 1.39e-01 | 20260508 05:55:33.692 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.15e-01 / 3.51e-01, 4.97e-02 / 2.31e-02 | 20260508 05:55:33.692 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:55:33.692 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:55:33.692 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:55:33.692 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 24.57%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 2.34%, LockUpload -- 2.48e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 2.55%, LockDownload -- 2.71e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 15.58%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 05:55:33.692 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260508 05:55:33.787 | INFO | Starting a round of Baz IO | 20260508 05:55:37.203 | INFO | Populating RTMetrics report | 20260508 05:55:37.203 | INFO | RTMetrics report populated | 20260508 05:55:37.203 | INFO | Opening tmp RTMetrics json file | 20260508 05:55:37.217 | INFO | Serializing RTMetrics to json | 20260508 05:55:37.218 | INFO | Serialization done, swapping tmp file now... | 20260508 05:55:37.219 | INFO | RTMetrics done | 20260508 05:55:37.279 | INFO | Finished a round of Baz IO: Wrote 1.562 GiB in 3.491 seconds | 20260508 05:55:37.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.79% / 0.00% / 59.21% | 20260508 05:55:37.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.430822% / 0% / 5.16574e-05% | 20260508 05:55:37.279 | INFO | Baz IO overall: | 20260508 05:55:37.279 | INFO | BytesWritten : 2560585930827 | 20260508 05:55:37.279 | INFO | NumEvents : 1235012999184 | 20260508 05:55:38.831 | INFO | Analyzing chunk frames = [5758976,5759488) | 20260508 05:55:38.831 | INFO | MemInfo: { | 20260508 05:55:38.831 | INFO | "gpu" : " 17 GB", | 20260508 05:55:38.831 | INFO | "huge" : "", | 20260508 05:55:38.831 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:38.831 | INFO | "malloc" : " 14 GB", | 20260508 05:55:38.831 | INFO | "pinned" : " 29 GB" | 20260508 05:55:38.831 | INFO | } | 20260508 05:55:38.831 | INFO | Perf report for the last 5.14093 seconds: | 20260508 05:55:38.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:38.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:38.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:38.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:38.832 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 05:55:38.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:38.832 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 05:55:38.832 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 05:55:38.832 | INFO | BAZWRITER : 67.2%, 2.7, 91.7ms, 0 | 20260508 05:55:38.832 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 05:55:38.832 | INFO | Uploaded 36GiB to GPU in 2.53471s (14.2028GiB/s) | 20260508 05:55:38.832 | INFO | Downloaded 38GiB to GPU in 2.68387s (14.1587GiB/s) | 20260508 05:55:43.973 | INFO | Analyzing chunk frames = [5759488,5760000) | 20260508 05:55:43.973 | INFO | MemInfo: { | 20260508 05:55:43.973 | INFO | "gpu" : " 17 GB", | 20260508 05:55:43.973 | INFO | "huge" : "", | 20260508 05:55:43.973 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:43.973 | INFO | "malloc" : " 14 GB", | 20260508 05:55:43.973 | INFO | "pinned" : " 29 GB" | 20260508 05:55:43.973 | INFO | } | 20260508 05:55:43.973 | INFO | Perf report for the last 5.14166 seconds: | 20260508 05:55:43.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:43.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:55:43.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 05:55:43.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:43.973 | INFO | Uploaded 36GiB to GPU in 2.5094s (14.3461GiB/s) | 20260508 05:55:43.973 | INFO | Downloaded 34GiB to GPU in 2.44353s (13.9143GiB/s) | 20260508 05:55:49.112 | INFO | Analyzing chunk frames = [5760000,5760512) | 20260508 05:55:49.112 | INFO | MemInfo: { | 20260508 05:55:49.112 | INFO | "gpu" : " 17 GB", | 20260508 05:55:49.112 | INFO | "huge" : "", | 20260508 05:55:49.112 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:49.112 | INFO | "malloc" : " 14 GB", | 20260508 05:55:49.112 | INFO | "pinned" : " 29 GB" | 20260508 05:55:49.112 | INFO | } | 20260508 05:55:49.112 | INFO | Perf report for the last 5.13919 seconds: | 20260508 05:55:49.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:49.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.112 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 05:55:49.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.112 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:55:49.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:49.113 | INFO | Uploaded 36GiB to GPU in 2.54541s (14.1431GiB/s) | 20260508 05:55:49.113 | INFO | Downloaded 34GiB to GPU in 2.41404s (14.0843GiB/s) | 20260508 05:55:54.247 | INFO | Analyzing chunk frames = [5760512,5761024) | 20260508 05:55:54.247 | INFO | MemInfo: { | 20260508 05:55:54.247 | INFO | "gpu" : " 17 GB", | 20260508 05:55:54.247 | INFO | "huge" : "", | 20260508 05:55:54.247 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:54.247 | INFO | "malloc" : " 14 GB", | 20260508 05:55:54.247 | INFO | "pinned" : " 29 GB" | 20260508 05:55:54.247 | INFO | } | 20260508 05:55:54.247 | INFO | Perf report for the last 5.13495 seconds: | 20260508 05:55:54.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:54.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.247 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:55:54.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.247 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:55:54.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:54.248 | INFO | Uploaded 36GiB to GPU in 2.56505s (14.0348GiB/s) | 20260508 05:55:54.248 | INFO | Downloaded 34GiB to GPU in 2.40839s (14.1173GiB/s) | 20260508 05:55:59.385 | INFO | Analyzing chunk frames = [5761024,5761536) | 20260508 05:55:59.385 | INFO | MemInfo: { | 20260508 05:55:59.385 | INFO | "gpu" : " 17 GB", | 20260508 05:55:59.385 | INFO | "huge" : "", | 20260508 05:55:59.385 | INFO | "huge_shared" : " 29 GB", | 20260508 05:55:59.385 | INFO | "malloc" : " 14 GB", | 20260508 05:55:59.385 | INFO | "pinned" : " 29 GB" | 20260508 05:55:59.386 | INFO | } | 20260508 05:55:59.386 | INFO | Perf report for the last 5.13843 seconds: | 20260508 05:55:59.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:55:59.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 05:55:59.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:55:59.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:55:59.386 | INFO | Uploaded 36GiB to GPU in 2.38366s (15.1028GiB/s) | 20260508 05:55:59.386 | INFO | Downloaded 34GiB to GPU in 2.54711s (13.3485GiB/s) | 20260508 05:56:04.528 | INFO | Analyzing chunk frames = [5761536,5762048) | 20260508 05:56:04.528 | INFO | MemInfo: { | 20260508 05:56:04.528 | INFO | "gpu" : " 17 GB", | 20260508 05:56:04.528 | INFO | "huge" : "", | 20260508 05:56:04.528 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:04.528 | INFO | "malloc" : " 14 GB", | 20260508 05:56:04.528 | INFO | "pinned" : " 29 GB" | 20260508 05:56:04.528 | INFO | } | 20260508 05:56:04.528 | INFO | Perf report for the last 5.14251 seconds: | 20260508 05:56:04.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:04.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 05:56:04.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:56:04.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:04.528 | INFO | Uploaded 36GiB to GPU in 2.46801s (14.5867GiB/s) | 20260508 05:56:04.528 | INFO | Downloaded 34GiB to GPU in 2.41202s (14.096GiB/s) | 20260508 05:56:09.671 | INFO | Analyzing chunk frames = [5762048,5762560) | 20260508 05:56:09.671 | INFO | MemInfo: { | 20260508 05:56:09.671 | INFO | "gpu" : " 17 GB", | 20260508 05:56:09.671 | INFO | "huge" : "", | 20260508 05:56:09.671 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:09.671 | INFO | "malloc" : " 14 GB", | 20260508 05:56:09.671 | INFO | "pinned" : " 29 GB" | 20260508 05:56:09.671 | INFO | } | 20260508 05:56:09.671 | INFO | Perf report for the last 5.14291 seconds: | 20260508 05:56:09.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:09.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.671 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:56:09.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.671 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 05:56:09.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:09.672 | INFO | Uploaded 36GiB to GPU in 2.48388s (14.4935GiB/s) | 20260508 05:56:09.672 | INFO | Downloaded 34GiB to GPU in 2.42696s (14.0093GiB/s) | 20260508 05:56:14.808 | INFO | Analyzing chunk frames = [5762560,5763072) | 20260508 05:56:14.808 | INFO | MemInfo: { | 20260508 05:56:14.808 | INFO | "gpu" : " 17 GB", | 20260508 05:56:14.808 | INFO | "huge" : "", | 20260508 05:56:14.809 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:14.809 | INFO | "malloc" : " 14 GB", | 20260508 05:56:14.809 | INFO | "pinned" : " 29 GB" | 20260508 05:56:14.809 | INFO | } | 20260508 05:56:14.809 | INFO | Perf report for the last 5.1379 seconds: | 20260508 05:56:14.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:14.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.809 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:56:14.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.809 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 05:56:14.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:14.810 | INFO | Uploaded 36GiB to GPU in 2.49815s (14.4107GiB/s) | 20260508 05:56:14.810 | INFO | Downloaded 34GiB to GPU in 2.4309s (13.9866GiB/s) | 20260508 05:56:14.923 | INFO | Starting a round of Baz IO | 20260508 05:56:18.338 | INFO | Populating RTMetrics report | 20260508 05:56:18.338 | INFO | RTMetrics report populated | 20260508 05:56:18.338 | INFO | Opening tmp RTMetrics json file | 20260508 05:56:18.339 | INFO | Serializing RTMetrics to json | 20260508 05:56:18.340 | INFO | Serialization done, swapping tmp file now... | 20260508 05:56:18.340 | INFO | RTMetrics done | 20260508 05:56:18.407 | INFO | Finished a round of Baz IO: Wrote 1.562 GiB in 3.484 seconds | 20260508 05:56:18.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.00% / 59.21% | 20260508 05:56:18.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.429829% / 0% / 5.16574e-05% | 20260508 05:56:18.408 | INFO | Baz IO overall: | 20260508 05:56:18.408 | INFO | BytesWritten : 2562263422180 | 20260508 05:56:18.408 | INFO | NumEvents : 1235812207120 | 20260508 05:56:19.943 | INFO | Analyzing chunk frames = [5763072,5763584) | 20260508 05:56:19.943 | INFO | MemInfo: { | 20260508 05:56:19.943 | INFO | "gpu" : " 17 GB", | 20260508 05:56:19.943 | INFO | "huge" : "", | 20260508 05:56:19.943 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:19.943 | INFO | "malloc" : " 14 GB", | 20260508 05:56:19.943 | INFO | "pinned" : " 29 GB" | 20260508 05:56:19.943 | INFO | } | 20260508 05:56:19.944 | INFO | Perf report for the last 5.13456 seconds: | 20260508 05:56:19.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:19.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:19.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:19.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:19.944 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 05:56:19.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:19.944 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 05:56:19.944 | INFO | PRE_HQ : 66.6%, 1.8, 61.6ms, 0 | 20260508 05:56:19.944 | INFO | BAZWRITER : 67.2%, 2.8, 96.6ms, 0 | 20260508 05:56:19.944 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 05:56:19.944 | INFO | Uploaded 36GiB to GPU in 2.53894s (14.1791GiB/s) | 20260508 05:56:19.944 | INFO | Downloaded 38GiB to GPU in 2.71423s (14.0003GiB/s) | 20260508 05:56:25.082 | INFO | Analyzing chunk frames = [5763584,5764096) | 20260508 05:56:25.082 | INFO | MemInfo: { | 20260508 05:56:25.082 | INFO | "gpu" : " 17 GB", | 20260508 05:56:25.082 | INFO | "huge" : "", | 20260508 05:56:25.082 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:25.082 | INFO | "malloc" : " 14 GB", | 20260508 05:56:25.082 | INFO | "pinned" : " 29 GB" | 20260508 05:56:25.082 | INFO | } | 20260508 05:56:25.082 | INFO | Perf report for the last 5.13845 seconds: | 20260508 05:56:25.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:25.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 05:56:25.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:56:25.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:25.082 | INFO | Uploaded 36GiB to GPU in 2.48665s (14.4773GiB/s) | 20260508 05:56:25.082 | INFO | Downloaded 34GiB to GPU in 2.43461s (13.9653GiB/s) | 20260508 05:56:30.222 | INFO | Analyzing chunk frames = [5764096,5764608) | 20260508 05:56:30.222 | INFO | MemInfo: { | 20260508 05:56:30.222 | INFO | "gpu" : " 17 GB", | 20260508 05:56:30.222 | INFO | "huge" : "", | 20260508 05:56:30.222 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:30.222 | INFO | "malloc" : " 14 GB", | 20260508 05:56:30.222 | INFO | "pinned" : " 29 GB" | 20260508 05:56:30.222 | INFO | } | 20260508 05:56:30.222 | INFO | Perf report for the last 5.14025 seconds: | 20260508 05:56:30.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:30.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.222 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:56:30.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.222 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:56:30.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:30.223 | INFO | Uploaded 36GiB to GPU in 2.52342s (14.2664GiB/s) | 20260508 05:56:30.223 | INFO | Downloaded 34GiB to GPU in 2.41206s (14.0959GiB/s) | 20260508 05:56:35.365 | INFO | Analyzing chunk frames = [5764608,5765120) | 20260508 05:56:35.365 | INFO | MemInfo: { | 20260508 05:56:35.365 | INFO | "gpu" : " 17 GB", | 20260508 05:56:35.365 | INFO | "huge" : "", | 20260508 05:56:35.365 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:35.365 | INFO | "malloc" : " 14 GB", | 20260508 05:56:35.365 | INFO | "pinned" : " 29 GB" | 20260508 05:56:35.365 | INFO | } | 20260508 05:56:35.365 | INFO | Perf report for the last 5.14318 seconds: | 20260508 05:56:35.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:35.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.365 | INFO | ANALYSIS : 66.7%, 2.9, 103.3ms, 0 | 20260508 05:56:35.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.365 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:56:35.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:35.366 | INFO | Uploaded 36GiB to GPU in 2.54255s (14.159GiB/s) | 20260508 05:56:35.366 | INFO | Downloaded 34GiB to GPU in 2.39803s (14.1783GiB/s) | 20260508 05:56:40.501 | INFO | Analyzing chunk frames = [5765120,5765632) | 20260508 05:56:40.501 | INFO | MemInfo: { | 20260508 05:56:40.501 | INFO | "gpu" : " 17 GB", | 20260508 05:56:40.501 | INFO | "huge" : "", | 20260508 05:56:40.501 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:40.501 | INFO | "malloc" : " 14 GB", | 20260508 05:56:40.501 | INFO | "pinned" : " 29 GB" | 20260508 05:56:40.501 | INFO | } | 20260508 05:56:40.501 | INFO | Perf report for the last 5.13592 seconds: | 20260508 05:56:40.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:40.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.501 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 05:56:40.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.501 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260508 05:56:40.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:40.502 | INFO | Uploaded 36GiB to GPU in 2.40887s (14.9448GiB/s) | 20260508 05:56:40.502 | INFO | Downloaded 34GiB to GPU in 2.52632s (13.4583GiB/s) | 20260508 05:56:45.647 | INFO | Analyzing chunk frames = [5765632,5766144) | 20260508 05:56:45.647 | INFO | MemInfo: { | 20260508 05:56:45.647 | INFO | "gpu" : " 17 GB", | 20260508 05:56:45.647 | INFO | "huge" : "", | 20260508 05:56:45.648 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:45.648 | INFO | "malloc" : " 14 GB", | 20260508 05:56:45.648 | INFO | "pinned" : " 29 GB" | 20260508 05:56:45.648 | INFO | } | 20260508 05:56:45.648 | INFO | Perf report for the last 5.14629 seconds: | 20260508 05:56:45.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:45.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 05:56:45.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:56:45.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:45.648 | INFO | Uploaded 36GiB to GPU in 2.42389s (14.8522GiB/s) | 20260508 05:56:45.648 | INFO | Downloaded 34GiB to GPU in 2.44978s (13.8788GiB/s) | 20260508 05:56:50.787 | INFO | Analyzing chunk frames = [5766144,5766656) | 20260508 05:56:50.787 | INFO | MemInfo: { | 20260508 05:56:50.787 | INFO | "gpu" : " 17 GB", | 20260508 05:56:50.787 | INFO | "huge" : "", | 20260508 05:56:50.787 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:50.788 | INFO | "malloc" : " 14 GB", | 20260508 05:56:50.788 | INFO | "pinned" : " 29 GB" | 20260508 05:56:50.788 | INFO | } | 20260508 05:56:50.788 | INFO | Perf report for the last 5.13999 seconds: | 20260508 05:56:50.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:50.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 05:56:50.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 05:56:50.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:50.788 | INFO | Uploaded 36GiB to GPU in 2.42649s (14.8363GiB/s) | 20260508 05:56:50.788 | INFO | Downloaded 34GiB to GPU in 2.43972s (13.936GiB/s) | 20260508 05:56:55.925 | INFO | Analyzing chunk frames = [5766656,5767168) | 20260508 05:56:55.925 | INFO | MemInfo: { | 20260508 05:56:55.925 | INFO | "gpu" : " 17 GB", | 20260508 05:56:55.925 | INFO | "huge" : "", | 20260508 05:56:55.925 | INFO | "huge_shared" : " 29 GB", | 20260508 05:56:55.925 | INFO | "malloc" : " 14 GB", | 20260508 05:56:55.925 | INFO | "pinned" : " 29 GB" | 20260508 05:56:55.925 | INFO | } | 20260508 05:56:55.926 | INFO | Perf report for the last 5.13774 seconds: | 20260508 05:56:55.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:56:55.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 05:56:55.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 05:56:55.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:56:55.926 | INFO | Uploaded 36GiB to GPU in 2.40668s (14.9584GiB/s) | 20260508 05:56:55.926 | INFO | Downloaded 34GiB to GPU in 2.44925s (13.8818GiB/s) | 20260508 05:56:55.926 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:56:55.926 | INFO | Total mem usage: 16.7 GiB | 20260508 05:56:55.926 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:56:55.926 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 05:56:55.926 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:56:55.926 | INFO | High Water Breakdown: | 20260508 05:56:55.926 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:56:55.926 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:56:55.926 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:56:55.926 | INFO | ------------------------------------------ | 20260508 05:56:55.926 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:56:55.926 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:56:55.926 | INFO | Total mem usage: 26.5 GiB | 20260508 05:56:55.926 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:56:55.926 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:56:55.926 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:56:55.926 | INFO | High Water Breakdown: | 20260508 05:56:55.926 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:56:55.926 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:56:55.927 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:56:55.927 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:56:55.927 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:56:55.927 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:56:55.927 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:56:55.927 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:56:55.927 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:56:55.927 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:56:55.927 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:56:55.927 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:56:55.927 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:56:55.927 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:56:55.927 | INFO | ------------------------------------------ | 20260508 05:56:55.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:56:55.927 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:56:55.927 | INFO | Total mem usage: 27.8 GiB | 20260508 05:56:55.927 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:56:55.927 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:56:55.927 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:56:55.927 | INFO | High Water Breakdown: | 20260508 05:56:55.927 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:56:55.927 | INFO | ------------------------------------------ | 20260508 05:56:55.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:56:55.927 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:56:55.927 | INFO | Total mem usage: 15.7 GiB | 20260508 05:56:55.927 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:56:55.927 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:56:55.927 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:56:55.927 | INFO | High Water Breakdown: | 20260508 05:56:55.927 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:56:55.927 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:56:55.927 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:56:55.927 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:56:55.927 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:56:55.927 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:56:55.927 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:56:55.927 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:56:55.927 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:56:55.927 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:56:55.927 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:56:55.927 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:56:55.927 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:56:55.927 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:56:55.927 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:56:55.927 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:56:55.927 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:56:55.927 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:56:55.927 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:56:55.927 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:56:55.927 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:56:55.927 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:56:55.927 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:56:55.927 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:56:55.927 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:56:55.927 | INFO | ------------------------------------------ | 20260508 05:56:55.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:56:55.927 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:56:55.927 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:56:55.927 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:56:55.927 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 1.30e+00 | 20260508 05:56:55.927 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.36e-02 / 6.24e-03 | 20260508 05:56:55.927 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.40e+00, 5.85e-01 / 2.43e-01 | 20260508 05:56:55.927 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.14e-01 | 20260508 05:56:55.927 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.52e+01, 3.90e+00 / 1.66e+00 | 20260508 05:56:55.927 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.82e+00 / 1.47e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.55e+00, 4.16e-01 / 2.53e-01 | 20260508 05:56:55.928 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.22e+00, 2.56e-01 / 1.50e-01 | 20260508 05:56:55.928 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.16e-01 / 3.55e-01, 4.99e-02 / 2.43e-02 | 20260508 05:56:55.928 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:56:55.928 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:56:55.928 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:56:55.928 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 24.38%, Upload -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 2.24%, LockUpload -- 2.38e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 2.59%, LockDownload -- 2.75e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 05:56:55.928 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 05:56:56.038 | INFO | Starting a round of Baz IO | 20260508 05:56:59.454 | INFO | Populating RTMetrics report | 20260508 05:56:59.454 | INFO | RTMetrics report populated | 20260508 05:56:59.454 | INFO | Opening tmp RTMetrics json file | 20260508 05:56:59.467 | INFO | Serializing RTMetrics to json | 20260508 05:56:59.468 | INFO | Serialization done, swapping tmp file now... | 20260508 05:56:59.468 | INFO | RTMetrics done | 20260508 05:56:59.528 | INFO | Finished a round of Baz IO: Wrote 1.563 GiB in 3.49 seconds | 20260508 05:56:59.528 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.00% / 59.22% | 20260508 05:56:59.528 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.430437% / 0% / 4.37101e-05% | 20260508 05:56:59.528 | INFO | Baz IO overall: | 20260508 05:56:59.528 | INFO | BytesWritten : 2563941928897 | 20260508 05:56:59.528 | INFO | NumEvents : 1236611948823 | 20260508 05:57:01.056 | INFO | Analyzing chunk frames = [5767168,5767680) | 20260508 05:57:01.056 | INFO | MemInfo: { | 20260508 05:57:01.056 | INFO | "gpu" : " 17 GB", | 20260508 05:57:01.056 | INFO | "huge" : "", | 20260508 05:57:01.056 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:01.056 | INFO | "malloc" : " 14 GB", | 20260508 05:57:01.056 | INFO | "pinned" : " 29 GB" | 20260508 05:57:01.056 | INFO | } | 20260508 05:57:01.056 | INFO | Perf report for the last 5.13061 seconds: | 20260508 05:57:01.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:01.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:01.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:01.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:01.056 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 05:57:01.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:01.056 | INFO | RT_METRICS : 43.5%, 1.0, 22.7ms, 0 | 20260508 05:57:01.056 | INFO | PRE_HQ : 66.5%, 1.8, 61.5ms, 0 | 20260508 05:57:01.056 | INFO | BAZWRITER : 67.5%, 2.8, 97.4ms, 0 | 20260508 05:57:01.056 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:57:01.057 | INFO | Uploaded 36GiB to GPU in 2.57977s (13.9547GiB/s) | 20260508 05:57:01.057 | INFO | Downloaded 38GiB to GPU in 2.71355s (14.0038GiB/s) | 20260508 05:57:06.194 | INFO | Analyzing chunk frames = [5767680,5768192) | 20260508 05:57:06.194 | INFO | MemInfo: { | 20260508 05:57:06.194 | INFO | "gpu" : " 17 GB", | 20260508 05:57:06.194 | INFO | "huge" : "", | 20260508 05:57:06.194 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:06.194 | INFO | "malloc" : " 14 GB", | 20260508 05:57:06.194 | INFO | "pinned" : " 29 GB" | 20260508 05:57:06.194 | INFO | } | 20260508 05:57:06.194 | INFO | Perf report for the last 5.13792 seconds: | 20260508 05:57:06.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:06.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:57:06.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 05:57:06.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:06.194 | INFO | Uploaded 36GiB to GPU in 2.4661s (14.5979GiB/s) | 20260508 05:57:06.194 | INFO | Downloaded 34GiB to GPU in 2.43939s (13.9379GiB/s) | 20260508 05:57:11.337 | INFO | Analyzing chunk frames = [5768192,5768704) | 20260508 05:57:11.337 | INFO | MemInfo: { | 20260508 05:57:11.337 | INFO | "gpu" : " 17 GB", | 20260508 05:57:11.337 | INFO | "huge" : "", | 20260508 05:57:11.337 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:11.337 | INFO | "malloc" : " 14 GB", | 20260508 05:57:11.337 | INFO | "pinned" : " 29 GB" | 20260508 05:57:11.337 | INFO | } | 20260508 05:57:11.337 | INFO | Perf report for the last 5.14318 seconds: | 20260508 05:57:11.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:11.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.337 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 05:57:11.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.337 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 05:57:11.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:11.338 | INFO | Uploaded 36GiB to GPU in 2.48789s (14.4701GiB/s) | 20260508 05:57:11.338 | INFO | Downloaded 34GiB to GPU in 2.43442s (13.9664GiB/s) | 20260508 05:57:16.481 | INFO | Analyzing chunk frames = [5768704,5769216) | 20260508 05:57:16.481 | INFO | MemInfo: { | 20260508 05:57:16.481 | INFO | "gpu" : " 17 GB", | 20260508 05:57:16.481 | INFO | "huge" : "", | 20260508 05:57:16.481 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:16.481 | INFO | "malloc" : " 14 GB", | 20260508 05:57:16.481 | INFO | "pinned" : " 29 GB" | 20260508 05:57:16.481 | INFO | } | 20260508 05:57:16.481 | INFO | Perf report for the last 5.14399 seconds: | 20260508 05:57:16.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:16.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.481 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 05:57:16.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.482 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 05:57:16.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:16.482 | INFO | Uploaded 36GiB to GPU in 2.5442s (14.1498GiB/s) | 20260508 05:57:16.482 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.975GiB/s) | 20260508 05:57:21.618 | INFO | Analyzing chunk frames = [5769216,5769728) | 20260508 05:57:21.618 | INFO | MemInfo: { | 20260508 05:57:21.618 | INFO | "gpu" : " 17 GB", | 20260508 05:57:21.618 | INFO | "huge" : "", | 20260508 05:57:21.618 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:21.618 | INFO | "malloc" : " 14 GB", | 20260508 05:57:21.618 | INFO | "pinned" : " 29 GB" | 20260508 05:57:21.619 | INFO | } | 20260508 05:57:21.619 | INFO | Perf report for the last 5.13744 seconds: | 20260508 05:57:21.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:21.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 05:57:21.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:57:21.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:21.619 | INFO | Uploaded 36GiB to GPU in 2.32734s (15.4683GiB/s) | 20260508 05:57:21.619 | INFO | Downloaded 34GiB to GPU in 2.5159s (13.514GiB/s) | 20260508 05:57:26.753 | INFO | Analyzing chunk frames = [5769728,5770240) | 20260508 05:57:26.753 | INFO | MemInfo: { | 20260508 05:57:26.753 | INFO | "gpu" : " 17 GB", | 20260508 05:57:26.753 | INFO | "huge" : "", | 20260508 05:57:26.753 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:26.753 | INFO | "malloc" : " 14 GB", | 20260508 05:57:26.753 | INFO | "pinned" : " 29 GB" | 20260508 05:57:26.753 | INFO | } | 20260508 05:57:26.753 | INFO | Perf report for the last 5.13469 seconds: | 20260508 05:57:26.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:26.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.753 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 05:57:26.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.753 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:57:26.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:26.754 | INFO | Uploaded 36GiB to GPU in 2.43833s (14.7642GiB/s) | 20260508 05:57:26.754 | INFO | Downloaded 34GiB to GPU in 2.44394s (13.912GiB/s) | 20260508 05:57:31.894 | INFO | Analyzing chunk frames = [5770240,5770752) | 20260508 05:57:31.894 | INFO | MemInfo: { | 20260508 05:57:31.894 | INFO | "gpu" : " 17 GB", | 20260508 05:57:31.894 | INFO | "huge" : "", | 20260508 05:57:31.894 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:31.894 | INFO | "malloc" : " 14 GB", | 20260508 05:57:31.894 | INFO | "pinned" : " 29 GB" | 20260508 05:57:31.894 | INFO | } | 20260508 05:57:31.894 | INFO | Perf report for the last 5.14073 seconds: | 20260508 05:57:31.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:31.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.895 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 05:57:31.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.895 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 05:57:31.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:31.895 | INFO | Uploaded 36GiB to GPU in 2.45845s (14.6434GiB/s) | 20260508 05:57:31.895 | INFO | Downloaded 34GiB to GPU in 2.43971s (13.9361GiB/s) | 20260508 05:57:37.037 | INFO | Analyzing chunk frames = [5770752,5771264) | 20260508 05:57:37.037 | INFO | MemInfo: { | 20260508 05:57:37.037 | INFO | "gpu" : " 17 GB", | 20260508 05:57:37.037 | INFO | "huge" : "", | 20260508 05:57:37.038 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:37.038 | INFO | "malloc" : " 14 GB", | 20260508 05:57:37.038 | INFO | "pinned" : " 29 GB" | 20260508 05:57:37.038 | INFO | } | 20260508 05:57:37.038 | INFO | Perf report for the last 5.14363 seconds: | 20260508 05:57:37.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:37.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:57:37.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 05:57:37.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:37.038 | INFO | Uploaded 36GiB to GPU in 2.44562s (14.7202GiB/s) | 20260508 05:57:37.038 | INFO | Downloaded 34GiB to GPU in 2.44483s (13.9069GiB/s) | 20260508 05:57:37.151 | INFO | Starting a round of Baz IO | 20260508 05:57:40.574 | INFO | Populating RTMetrics report | 20260508 05:57:40.574 | INFO | RTMetrics report populated | 20260508 05:57:40.574 | INFO | Opening tmp RTMetrics json file | 20260508 05:57:40.585 | INFO | Serializing RTMetrics to json | 20260508 05:57:40.587 | INFO | Serialization done, swapping tmp file now... | 20260508 05:57:40.587 | INFO | RTMetrics done | 20260508 05:57:40.642 | INFO | Finished a round of Baz IO: Wrote 1.562 GiB in 3.491 seconds | 20260508 05:57:40.642 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.00% / 59.22% | 20260508 05:57:40.642 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.428641% / 0% / 3.97364e-05% | 20260508 05:57:40.642 | INFO | Baz IO overall: | 20260508 05:57:40.642 | INFO | BytesWritten : 2565619319504 | 20260508 05:57:40.642 | INFO | NumEvents : 1237411109010 | 20260508 05:57:42.184 | INFO | Analyzing chunk frames = [5771264,5771776) | 20260508 05:57:42.184 | INFO | MemInfo: { | 20260508 05:57:42.184 | INFO | "gpu" : " 17 GB", | 20260508 05:57:42.184 | INFO | "huge" : "", | 20260508 05:57:42.184 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:42.184 | INFO | "malloc" : " 14 GB", | 20260508 05:57:42.184 | INFO | "pinned" : " 29 GB" | 20260508 05:57:42.184 | INFO | } | 20260508 05:57:42.184 | INFO | Perf report for the last 5.14664 seconds: | 20260508 05:57:42.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:42.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:42.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:42.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:42.185 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 05:57:42.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:42.185 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 05:57:42.185 | INFO | PRE_HQ : 66.6%, 1.8, 61.4ms, 0 | 20260508 05:57:42.185 | INFO | BAZWRITER : 67.1%, 2.8, 96.8ms, 0 | 20260508 05:57:42.185 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:57:42.185 | INFO | Uploaded 36GiB to GPU in 2.59495s (13.8731GiB/s) | 20260508 05:57:42.185 | INFO | Downloaded 38GiB to GPU in 2.70664s (14.0395GiB/s) | 20260508 05:57:47.320 | INFO | Analyzing chunk frames = [5771776,5772288) | 20260508 05:57:47.320 | INFO | MemInfo: { | 20260508 05:57:47.320 | INFO | "gpu" : " 17 GB", | 20260508 05:57:47.320 | INFO | "huge" : "", | 20260508 05:57:47.320 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:47.320 | INFO | "malloc" : " 14 GB", | 20260508 05:57:47.320 | INFO | "pinned" : " 29 GB" | 20260508 05:57:47.320 | INFO | } | 20260508 05:57:47.321 | INFO | Perf report for the last 5.13631 seconds: | 20260508 05:57:47.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:47.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 05:57:47.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:57:47.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:47.321 | INFO | Uploaded 36GiB to GPU in 2.47431s (14.5495GiB/s) | 20260508 05:57:47.321 | INFO | Downloaded 34GiB to GPU in 2.44657s (13.897GiB/s) | 20260508 05:57:52.451 | INFO | Analyzing chunk frames = [5772288,5772800) | 20260508 05:57:52.451 | INFO | MemInfo: { | 20260508 05:57:52.451 | INFO | "gpu" : " 17 GB", | 20260508 05:57:52.451 | INFO | "huge" : "", | 20260508 05:57:52.452 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:52.452 | INFO | "malloc" : " 14 GB", | 20260508 05:57:52.452 | INFO | "pinned" : " 29 GB" | 20260508 05:57:52.452 | INFO | } | 20260508 05:57:52.452 | INFO | Perf report for the last 5.13089 seconds: | 20260508 05:57:52.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:52.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | ANALYSIS : 67.2%, 3.0, 104.1ms, 0 | 20260508 05:57:52.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:57:52.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:52.452 | INFO | Uploaded 36GiB to GPU in 2.53289s (14.213GiB/s) | 20260508 05:57:52.452 | INFO | Downloaded 34GiB to GPU in 2.43247s (13.9775GiB/s) | 20260508 05:57:57.597 | INFO | Analyzing chunk frames = [5772800,5773312) | 20260508 05:57:57.597 | INFO | MemInfo: { | 20260508 05:57:57.597 | INFO | "gpu" : " 17 GB", | 20260508 05:57:57.597 | INFO | "huge" : "", | 20260508 05:57:57.597 | INFO | "huge_shared" : " 29 GB", | 20260508 05:57:57.597 | INFO | "malloc" : " 14 GB", | 20260508 05:57:57.597 | INFO | "pinned" : " 29 GB" | 20260508 05:57:57.597 | INFO | } | 20260508 05:57:57.597 | INFO | Perf report for the last 5.14519 seconds: | 20260508 05:57:57.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:57:57.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.597 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 05:57:57.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.597 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:57:57.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:57:57.598 | INFO | Uploaded 36GiB to GPU in 2.53858s (14.1812GiB/s) | 20260508 05:57:57.598 | INFO | Downloaded 34GiB to GPU in 2.38355s (14.2644GiB/s) | 20260508 05:58:02.735 | INFO | Analyzing chunk frames = [5773312,5773824) | 20260508 05:58:02.735 | INFO | MemInfo: { | 20260508 05:58:02.735 | INFO | "gpu" : " 17 GB", | 20260508 05:58:02.735 | INFO | "huge" : "", | 20260508 05:58:02.735 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:02.735 | INFO | "malloc" : " 14 GB", | 20260508 05:58:02.735 | INFO | "pinned" : " 29 GB" | 20260508 05:58:02.735 | INFO | } | 20260508 05:58:02.735 | INFO | Perf report for the last 5.13824 seconds: | 20260508 05:58:02.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:02.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.735 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 05:58:02.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.735 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 05:58:02.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:02.736 | INFO | Uploaded 36GiB to GPU in 2.42854s (14.8237GiB/s) | 20260508 05:58:02.736 | INFO | Downloaded 34GiB to GPU in 2.53698s (13.4017GiB/s) | 20260508 05:58:07.872 | INFO | Analyzing chunk frames = [5773824,5774336) | 20260508 05:58:07.872 | INFO | MemInfo: { | 20260508 05:58:07.872 | INFO | "gpu" : " 17 GB", | 20260508 05:58:07.872 | INFO | "huge" : "", | 20260508 05:58:07.872 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:07.872 | INFO | "malloc" : " 14 GB", | 20260508 05:58:07.872 | INFO | "pinned" : " 29 GB" | 20260508 05:58:07.872 | INFO | } | 20260508 05:58:07.872 | INFO | Perf report for the last 5.13672 seconds: | 20260508 05:58:07.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:07.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 05:58:07.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 05:58:07.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:07.872 | INFO | Uploaded 36GiB to GPU in 2.40909s (14.9434GiB/s) | 20260508 05:58:07.872 | INFO | Downloaded 34GiB to GPU in 2.44741s (13.8923GiB/s) | 20260508 05:58:13.008 | INFO | Analyzing chunk frames = [5774336,5774848) | 20260508 05:58:13.008 | INFO | MemInfo: { | 20260508 05:58:13.008 | INFO | "gpu" : " 17 GB", | 20260508 05:58:13.008 | INFO | "huge" : "", | 20260508 05:58:13.008 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:13.009 | INFO | "malloc" : " 14 GB", | 20260508 05:58:13.009 | INFO | "pinned" : " 29 GB" | 20260508 05:58:13.009 | INFO | } | 20260508 05:58:13.009 | INFO | Perf report for the last 5.13745 seconds: | 20260508 05:58:13.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:13.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.009 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 05:58:13.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.010 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 05:58:13.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:13.010 | INFO | Uploaded 36GiB to GPU in 2.37878s (15.1338GiB/s) | 20260508 05:58:13.010 | INFO | Downloaded 34GiB to GPU in 2.44188s (13.9237GiB/s) | 20260508 05:58:18.151 | INFO | Analyzing chunk frames = [5774848,5775360) | 20260508 05:58:18.151 | INFO | MemInfo: { | 20260508 05:58:18.151 | INFO | "gpu" : " 17 GB", | 20260508 05:58:18.151 | INFO | "huge" : "", | 20260508 05:58:18.151 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:18.152 | INFO | "malloc" : " 14 GB", | 20260508 05:58:18.152 | INFO | "pinned" : " 29 GB" | 20260508 05:58:18.152 | INFO | } | 20260508 05:58:18.152 | INFO | Perf report for the last 5.14253 seconds: | 20260508 05:58:18.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:18.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 05:58:18.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 05:58:18.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:18.152 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:58:18.152 | INFO | Total mem usage: 16.7 GiB | 20260508 05:58:18.152 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:58:18.152 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 05:58:18.152 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:58:18.152 | INFO | High Water Breakdown: | 20260508 05:58:18.152 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:58:18.153 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:58:18.153 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:58:18.153 | INFO | ------------------------------------------ | 20260508 05:58:18.152 | INFO | Uploaded 36GiB to GPU in 2.44295s (14.7363GiB/s) | 20260508 05:58:18.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:58:18.153 | INFO | Downloaded 34GiB to GPU in 2.45679s (13.8392GiB/s) | 20260508 05:58:18.153 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:58:18.153 | INFO | Total mem usage: 26.5 GiB | 20260508 05:58:18.153 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:58:18.153 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:58:18.153 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:58:18.153 | INFO | High Water Breakdown: | 20260508 05:58:18.153 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:58:18.153 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:58:18.153 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:58:18.153 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:58:18.153 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:58:18.153 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:58:18.153 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:58:18.153 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:58:18.153 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:58:18.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:58:18.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:58:18.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:58:18.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:58:18.153 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:58:18.153 | INFO | ------------------------------------------ | 20260508 05:58:18.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:58:18.153 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:58:18.153 | INFO | Total mem usage: 27.8 GiB | 20260508 05:58:18.153 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:58:18.153 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:58:18.153 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:58:18.153 | INFO | High Water Breakdown: | 20260508 05:58:18.153 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:58:18.153 | INFO | ------------------------------------------ | 20260508 05:58:18.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:58:18.153 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:58:18.153 | INFO | Total mem usage: 15.7 GiB | 20260508 05:58:18.153 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:58:18.153 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:58:18.153 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:58:18.153 | INFO | High Water Breakdown: | 20260508 05:58:18.153 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:58:18.153 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:58:18.153 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:58:18.153 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:58:18.153 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:58:18.153 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:58:18.153 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:58:18.153 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:58:18.153 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:58:18.153 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:58:18.153 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:58:18.153 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:58:18.153 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:58:18.153 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:58:18.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:58:18.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:58:18.153 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:58:18.153 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:58:18.153 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:58:18.153 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:58:18.153 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:58:18.153 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:58:18.153 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:58:18.153 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:58:18.154 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:58:18.154 | INFO | ------------------------------------------ | 20260508 05:58:18.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:58:18.154 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:58:18.154 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:58:18.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:58:18.154 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.39e+01 / 5.96e-01 | 20260508 05:58:18.154 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.15e-02 / 6.73e-03 | 20260508 05:58:18.154 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.48e+00, 5.87e-01 / 2.02e-01 | 20260508 05:58:18.154 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.32e-01 | 20260508 05:58:18.154 | INFO | PerfReport: 15.87%, DME -- 4.16e+01 / 4.49e+01, 3.61e+00 / 1.04e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.88e+00 / 1.43e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.58e+00, 4.13e-01 / 2.34e-01 | 20260508 05:58:18.154 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.18e+00, 2.57e-01 / 9.99e-02 | 20260508 05:58:18.154 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.58e-01, 5.01e-02 / 2.74e-02 | 20260508 05:58:18.154 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:58:18.154 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:58:18.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:58:18.154 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 24.27%, Upload -- 2.57e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 24.37%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 30.88%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 2.07%, LockUpload -- 2.19e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 2.75%, LockDownload -- 2.91e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 05:58:18.154 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260508 05:58:18.263 | INFO | Starting a round of Baz IO | 20260508 05:58:21.682 | INFO | Populating RTMetrics report | 20260508 05:58:21.683 | INFO | RTMetrics report populated | 20260508 05:58:21.683 | INFO | Opening tmp RTMetrics json file | 20260508 05:58:21.695 | INFO | Serializing RTMetrics to json | 20260508 05:58:21.697 | INFO | Serialization done, swapping tmp file now... | 20260508 05:58:21.697 | INFO | RTMetrics done | 20260508 05:58:21.752 | INFO | Finished a round of Baz IO: Wrote 1.563 GiB in 3.488 seconds | 20260508 05:58:21.752 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.00% / 59.22% | 20260508 05:58:21.752 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.427671% / 0% / 6.35783e-05% | 20260508 05:58:21.752 | INFO | Baz IO overall: | 20260508 05:58:21.752 | INFO | BytesWritten : 2567297459674 | 20260508 05:58:21.752 | INFO | NumEvents : 1238210764735 | 20260508 05:58:23.293 | INFO | Analyzing chunk frames = [5775360,5775872) | 20260508 05:58:23.293 | INFO | MemInfo: { | 20260508 05:58:23.293 | INFO | "gpu" : " 17 GB", | 20260508 05:58:23.293 | INFO | "huge" : "", | 20260508 05:58:23.293 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:23.293 | INFO | "malloc" : " 14 GB", | 20260508 05:58:23.293 | INFO | "pinned" : " 29 GB" | 20260508 05:58:23.293 | INFO | } | 20260508 05:58:23.293 | INFO | Perf report for the last 5.14075 seconds: | 20260508 05:58:23.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:23.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:23.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:23.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:23.293 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 05:58:23.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:23.293 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 05:58:23.293 | INFO | PRE_HQ : 66.5%, 1.8, 61.1ms, 0 | 20260508 05:58:23.293 | INFO | BAZWRITER : 67.3%, 2.8, 97.3ms, 0 | 20260508 05:58:23.293 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:58:23.293 | INFO | Uploaded 36GiB to GPU in 2.54965s (14.1196GiB/s) | 20260508 05:58:23.293 | INFO | Downloaded 38GiB to GPU in 2.70213s (14.063GiB/s) | 20260508 05:58:28.429 | INFO | Analyzing chunk frames = [5775872,5776384) | 20260508 05:58:28.429 | INFO | MemInfo: { | 20260508 05:58:28.429 | INFO | "gpu" : " 17 GB", | 20260508 05:58:28.429 | INFO | "huge" : "", | 20260508 05:58:28.429 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:28.429 | INFO | "malloc" : " 14 GB", | 20260508 05:58:28.429 | INFO | "pinned" : " 29 GB" | 20260508 05:58:28.430 | INFO | } | 20260508 05:58:28.430 | INFO | Perf report for the last 5.13708 seconds: | 20260508 05:58:28.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:28.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 05:58:28.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 05:58:28.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:28.430 | INFO | Uploaded 36GiB to GPU in 2.46761s (14.589GiB/s) | 20260508 05:58:28.430 | INFO | Downloaded 34GiB to GPU in 2.44817s (13.8879GiB/s) | 20260508 05:58:33.565 | INFO | Analyzing chunk frames = [5776384,5776896) | 20260508 05:58:33.565 | INFO | MemInfo: { | 20260508 05:58:33.565 | INFO | "gpu" : " 17 GB", | 20260508 05:58:33.565 | INFO | "huge" : "", | 20260508 05:58:33.565 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:33.565 | INFO | "malloc" : " 14 GB", | 20260508 05:58:33.565 | INFO | "pinned" : " 29 GB" | 20260508 05:58:33.565 | INFO | } | 20260508 05:58:33.565 | INFO | Perf report for the last 5.13555 seconds: | 20260508 05:58:33.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:33.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.565 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 05:58:33.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.565 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 05:58:33.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:33.566 | INFO | Uploaded 36GiB to GPU in 2.5069s (14.3604GiB/s) | 20260508 05:58:33.566 | INFO | Downloaded 34GiB to GPU in 2.42753s (14.006GiB/s) | 20260508 05:58:38.706 | INFO | Analyzing chunk frames = [5776896,5777408) | 20260508 05:58:38.706 | INFO | MemInfo: { | 20260508 05:58:38.706 | INFO | "gpu" : " 17 GB", | 20260508 05:58:38.706 | INFO | "huge" : "", | 20260508 05:58:38.706 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:38.706 | INFO | "malloc" : " 14 GB", | 20260508 05:58:38.706 | INFO | "pinned" : " 29 GB" | 20260508 05:58:38.706 | INFO | } | 20260508 05:58:38.706 | INFO | Perf report for the last 5.14077 seconds: | 20260508 05:58:38.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:38.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.706 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 05:58:38.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.706 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 05:58:38.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:38.707 | INFO | Uploaded 36GiB to GPU in 2.52926s (14.2334GiB/s) | 20260508 05:58:38.707 | INFO | Downloaded 34GiB to GPU in 2.42594s (14.0152GiB/s) | 20260508 05:58:43.846 | INFO | Analyzing chunk frames = [5777408,5777920) | 20260508 05:58:43.846 | INFO | MemInfo: { | 20260508 05:58:43.846 | INFO | "gpu" : " 17 GB", | 20260508 05:58:43.846 | INFO | "huge" : "", | 20260508 05:58:43.846 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:43.846 | INFO | "malloc" : " 14 GB", | 20260508 05:58:43.846 | INFO | "pinned" : " 29 GB" | 20260508 05:58:43.846 | INFO | } | 20260508 05:58:43.846 | INFO | Perf report for the last 5.1401 seconds: | 20260508 05:58:43.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:43.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 05:58:43.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 05:58:43.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:43.846 | INFO | Uploaded 36GiB to GPU in 2.27522s (15.8226GiB/s) | 20260508 05:58:43.846 | INFO | Downloaded 34GiB to GPU in 2.48489s (13.6827GiB/s) | 20260508 05:58:48.991 | INFO | Analyzing chunk frames = [5777920,5778432) | 20260508 05:58:48.991 | INFO | MemInfo: { | 20260508 05:58:48.991 | INFO | "gpu" : " 17 GB", | 20260508 05:58:48.991 | INFO | "huge" : "", | 20260508 05:58:48.991 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:48.991 | INFO | "malloc" : " 14 GB", | 20260508 05:58:48.991 | INFO | "pinned" : " 29 GB" | 20260508 05:58:48.991 | INFO | } | 20260508 05:58:48.991 | INFO | Perf report for the last 5.14519 seconds: | 20260508 05:58:48.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:48.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.991 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:58:48.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.992 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:58:48.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:48.992 | INFO | Uploaded 36GiB to GPU in 2.40343s (14.9786GiB/s) | 20260508 05:58:48.992 | INFO | Downloaded 34GiB to GPU in 2.4438s (13.9127GiB/s) | 20260508 05:58:54.128 | INFO | Analyzing chunk frames = [5778432,5778944) | 20260508 05:58:54.128 | INFO | MemInfo: { | 20260508 05:58:54.128 | INFO | "gpu" : " 17 GB", | 20260508 05:58:54.128 | INFO | "huge" : "", | 20260508 05:58:54.128 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:54.128 | INFO | "malloc" : " 14 GB", | 20260508 05:58:54.128 | INFO | "pinned" : " 29 GB" | 20260508 05:58:54.128 | INFO | } | 20260508 05:58:54.129 | INFO | Perf report for the last 5.13714 seconds: | 20260508 05:58:54.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:54.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 05:58:54.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 05:58:54.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:54.129 | INFO | Uploaded 36GiB to GPU in 2.44052s (14.7509GiB/s) | 20260508 05:58:54.129 | INFO | Downloaded 34GiB to GPU in 2.44486s (13.9067GiB/s) | 20260508 05:58:59.265 | INFO | Analyzing chunk frames = [5778944,5779456) | 20260508 05:58:59.265 | INFO | MemInfo: { | 20260508 05:58:59.265 | INFO | "gpu" : " 17 GB", | 20260508 05:58:59.265 | INFO | "huge" : "", | 20260508 05:58:59.265 | INFO | "huge_shared" : " 29 GB", | 20260508 05:58:59.266 | INFO | "malloc" : " 14 GB", | 20260508 05:58:59.266 | INFO | "pinned" : " 29 GB" | 20260508 05:58:59.266 | INFO | } | 20260508 05:58:59.266 | INFO | Perf report for the last 5.13715 seconds: | 20260508 05:58:59.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:58:59.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.266 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 05:58:59.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.266 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 05:58:59.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:58:59.267 | INFO | Uploaded 36GiB to GPU in 2.44183s (14.743GiB/s) | 20260508 05:58:59.267 | INFO | Downloaded 34GiB to GPU in 2.44242s (13.9206GiB/s) | 20260508 05:58:59.364 | INFO | Starting a round of Baz IO | 20260508 05:59:02.792 | INFO | Populating RTMetrics report | 20260508 05:59:02.793 | INFO | RTMetrics report populated | 20260508 05:59:02.793 | INFO | Opening tmp RTMetrics json file | 20260508 05:59:02.793 | INFO | Serializing RTMetrics to json | 20260508 05:59:02.794 | INFO | Serialization done, swapping tmp file now... | 20260508 05:59:02.795 | INFO | RTMetrics done | 20260508 05:59:02.861 | INFO | Finished a round of Baz IO: Wrote 1.564 GiB in 3.496 seconds | 20260508 05:59:02.861 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.00% / 59.22% | 20260508 05:59:02.861 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.428144% / 0% / 3.57628e-05% | 20260508 05:59:02.861 | INFO | Baz IO overall: | 20260508 05:59:02.861 | INFO | BytesWritten : 2568976365584 | 20260508 05:59:02.861 | INFO | NumEvents : 1239011422428 | 20260508 05:59:04.406 | INFO | Analyzing chunk frames = [5779456,5779968) | 20260508 05:59:04.406 | INFO | MemInfo: { | 20260508 05:59:04.406 | INFO | "gpu" : " 17 GB", | 20260508 05:59:04.406 | INFO | "huge" : "", | 20260508 05:59:04.406 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:04.406 | INFO | "malloc" : " 14 GB", | 20260508 05:59:04.406 | INFO | "pinned" : " 29 GB" | 20260508 05:59:04.406 | INFO | } | 20260508 05:59:04.406 | INFO | Perf report for the last 5.14072 seconds: | 20260508 05:59:04.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:04.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:04.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:04.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:04.407 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 05:59:04.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:04.407 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 05:59:04.407 | INFO | PRE_HQ : 65.7%, 1.6, 55.0ms, 0 | 20260508 05:59:04.407 | INFO | BAZWRITER : 67.4%, 2.6, 91.0ms, 0 | 20260508 05:59:04.407 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:59:04.407 | INFO | Uploaded 36GiB to GPU in 2.5493s (14.1215GiB/s) | 20260508 05:59:04.407 | INFO | Downloaded 38GiB to GPU in 2.71727s (13.9846GiB/s) | 20260508 05:59:09.544 | INFO | Analyzing chunk frames = [5779968,5780480) | 20260508 05:59:09.544 | INFO | MemInfo: { | 20260508 05:59:09.544 | INFO | "gpu" : " 17 GB", | 20260508 05:59:09.544 | INFO | "huge" : "", | 20260508 05:59:09.544 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:09.544 | INFO | "malloc" : " 14 GB", | 20260508 05:59:09.544 | INFO | "pinned" : " 29 GB" | 20260508 05:59:09.544 | INFO | } | 20260508 05:59:09.545 | INFO | Perf report for the last 5.13808 seconds: | 20260508 05:59:09.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:09.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 05:59:09.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 05:59:09.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:09.545 | INFO | Uploaded 36GiB to GPU in 2.45724s (14.6506GiB/s) | 20260508 05:59:09.545 | INFO | Downloaded 34GiB to GPU in 2.45134s (13.87GiB/s) | 20260508 05:59:14.685 | INFO | Analyzing chunk frames = [5780480,5780992) | 20260508 05:59:14.685 | INFO | MemInfo: { | 20260508 05:59:14.685 | INFO | "gpu" : " 17 GB", | 20260508 05:59:14.685 | INFO | "huge" : "", | 20260508 05:59:14.685 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:14.685 | INFO | "malloc" : " 14 GB", | 20260508 05:59:14.685 | INFO | "pinned" : " 29 GB" | 20260508 05:59:14.685 | INFO | } | 20260508 05:59:14.685 | INFO | Perf report for the last 5.14086 seconds: | 20260508 05:59:14.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:14.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.685 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 05:59:14.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.686 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 05:59:14.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:14.686 | INFO | Uploaded 36GiB to GPU in 2.49261s (14.4427GiB/s) | 20260508 05:59:14.686 | INFO | Downloaded 34GiB to GPU in 2.41808s (14.0607GiB/s) | 20260508 05:59:19.822 | INFO | Analyzing chunk frames = [5780992,5781504) | 20260508 05:59:19.822 | INFO | MemInfo: { | 20260508 05:59:19.822 | INFO | "gpu" : " 17 GB", | 20260508 05:59:19.822 | INFO | "huge" : "", | 20260508 05:59:19.822 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:19.823 | INFO | "malloc" : " 14 GB", | 20260508 05:59:19.823 | INFO | "pinned" : " 29 GB" | 20260508 05:59:19.823 | INFO | } | 20260508 05:59:19.823 | INFO | Perf report for the last 5.13722 seconds: | 20260508 05:59:19.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:19.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | ANALYSIS : 66.7%, 3.0, 103.1ms, 0 | 20260508 05:59:19.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 05:59:19.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:19.823 | INFO | Uploaded 36GiB to GPU in 2.496s (14.4231GiB/s) | 20260508 05:59:19.823 | INFO | Downloaded 34GiB to GPU in 2.40973s (14.1094GiB/s) | 20260508 05:59:24.958 | INFO | Analyzing chunk frames = [5781504,5782016) | 20260508 05:59:24.958 | INFO | MemInfo: { | 20260508 05:59:24.958 | INFO | "gpu" : " 17 GB", | 20260508 05:59:24.958 | INFO | "huge" : "", | 20260508 05:59:24.958 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:24.958 | INFO | "malloc" : " 14 GB", | 20260508 05:59:24.958 | INFO | "pinned" : " 29 GB" | 20260508 05:59:24.958 | INFO | } | 20260508 05:59:24.958 | INFO | Perf report for the last 5.13556 seconds: | 20260508 05:59:24.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:24.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.958 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 05:59:24.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.958 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 05:59:24.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:24.959 | INFO | Uploaded 36GiB to GPU in 2.40865s (14.9461GiB/s) | 20260508 05:59:24.959 | INFO | Downloaded 34GiB to GPU in 2.52908s (13.4436GiB/s) | 20260508 05:59:30.105 | INFO | Analyzing chunk frames = [5782016,5782528) | 20260508 05:59:30.105 | INFO | MemInfo: { | 20260508 05:59:30.105 | INFO | "gpu" : " 17 GB", | 20260508 05:59:30.105 | INFO | "huge" : "", | 20260508 05:59:30.105 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:30.105 | INFO | "malloc" : " 14 GB", | 20260508 05:59:30.105 | INFO | "pinned" : " 29 GB" | 20260508 05:59:30.105 | INFO | } | 20260508 05:59:30.105 | INFO | Perf report for the last 5.14722 seconds: | 20260508 05:59:30.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:30.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.106 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 05:59:30.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.106 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 05:59:30.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:30.106 | INFO | Uploaded 36GiB to GPU in 2.41209s (14.9248GiB/s) | 20260508 05:59:30.106 | INFO | Downloaded 34GiB to GPU in 2.44156s (13.9255GiB/s) | 20260508 05:59:35.245 | INFO | Analyzing chunk frames = [5782528,5783040) | 20260508 05:59:35.245 | INFO | MemInfo: { | 20260508 05:59:35.245 | INFO | "gpu" : " 17 GB", | 20260508 05:59:35.245 | INFO | "huge" : "", | 20260508 05:59:35.245 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:35.245 | INFO | "malloc" : " 14 GB", | 20260508 05:59:35.245 | INFO | "pinned" : " 29 GB" | 20260508 05:59:35.245 | INFO | } | 20260508 05:59:35.245 | INFO | Perf report for the last 5.13975 seconds: | 20260508 05:59:35.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:35.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.245 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 05:59:35.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.246 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260508 05:59:35.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:35.246 | INFO | Uploaded 36GiB to GPU in 2.4183s (14.8865GiB/s) | 20260508 05:59:35.246 | INFO | Downloaded 34GiB to GPU in 2.44522s (13.9047GiB/s) | 20260508 05:59:40.381 | INFO | Analyzing chunk frames = [5783040,5783552) | 20260508 05:59:40.381 | INFO | MemInfo: { | 20260508 05:59:40.381 | INFO | "gpu" : " 17 GB", | 20260508 05:59:40.381 | INFO | "huge" : "", | 20260508 05:59:40.381 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:40.381 | INFO | "malloc" : " 14 GB", | 20260508 05:59:40.381 | INFO | "pinned" : " 29 GB" | 20260508 05:59:40.381 | INFO | } | 20260508 05:59:40.381 | INFO | Perf report for the last 5.13606 seconds: | 20260508 05:59:40.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:40.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.381 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 05:59:40.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.381 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 05:59:40.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:40.382 | INFO | Uploaded 36GiB to GPU in 2.45913s (14.6393GiB/s) | 20260508 05:59:40.382 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 05:59:40.382 | INFO | Total mem usage: 16.7 GiB | 20260508 05:59:40.382 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 05:59:40.382 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 05:59:40.382 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 05:59:40.382 | INFO | High Water Breakdown: | 20260508 05:59:40.382 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 05:59:40.382 | INFO | Downloaded 34GiB to GPU in 2.46443s (13.7963GiB/s) | 20260508 05:59:40.382 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 05:59:40.382 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 05:59:40.382 | INFO | ------------------------------------------ | 20260508 05:59:40.382 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:59:40.382 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 05:59:40.382 | INFO | Total mem usage: 26.5 GiB | 20260508 05:59:40.382 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 05:59:40.382 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 05:59:40.382 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 05:59:40.382 | INFO | High Water Breakdown: | 20260508 05:59:40.382 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:59:40.382 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:59:40.382 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:59:40.382 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:59:40.382 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:59:40.382 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:59:40.382 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 05:59:40.382 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 05:59:40.382 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:59:40.382 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 05:59:40.382 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 05:59:40.382 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 05:59:40.382 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 05:59:40.382 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 05:59:40.382 | INFO | ------------------------------------------ | 20260508 05:59:40.382 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:59:40.382 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 05:59:40.382 | INFO | Total mem usage: 27.8 GiB | 20260508 05:59:40.383 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 05:59:40.383 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 05:59:40.383 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 05:59:40.383 | INFO | High Water Breakdown: | 20260508 05:59:40.383 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 05:59:40.383 | INFO | ------------------------------------------ | 20260508 05:59:40.383 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:59:40.383 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 05:59:40.383 | INFO | Total mem usage: 15.7 GiB | 20260508 05:59:40.383 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 05:59:40.383 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 05:59:40.383 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 05:59:40.383 | INFO | High Water Breakdown: | 20260508 05:59:40.383 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 05:59:40.383 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 05:59:40.383 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 05:59:40.383 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 05:59:40.383 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 05:59:40.383 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 05:59:40.383 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 05:59:40.383 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 05:59:40.383 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 05:59:40.383 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 05:59:40.383 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 05:59:40.383 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 05:59:40.383 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 05:59:40.383 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 05:59:40.383 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 05:59:40.383 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 05:59:40.383 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 05:59:40.383 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 05:59:40.383 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 05:59:40.383 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 05:59:40.383 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 05:59:40.383 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 05:59:40.383 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 05:59:40.383 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 05:59:40.383 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 05:59:40.384 | INFO | ------------------------------------------ | 20260508 05:59:40.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 05:59:40.384 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 05:59:40.384 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:59:40.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:59:40.384 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.39e+01 / 1.09e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.04e-01, 1.23e-02 / 5.81e-03 | 20260508 05:59:40.384 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.46e+00, 5.90e-01 / 2.36e-01 | 20260508 05:59:40.384 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.05e-01 | 20260508 05:59:40.384 | INFO | PerfReport: 15.86%, DME -- 4.16e+01 / 4.51e+01, 3.72e+00 / 1.38e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.85e+00 / 1.24e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.61e+00, 4.10e-01 / 2.19e-01 | 20260508 05:59:40.384 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.26e+00, 2.61e-01 / 1.53e-01 | 20260508 05:59:40.384 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.64e-01, 5.10e-02 / 3.16e-02 | 20260508 05:59:40.384 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 05:59:40.384 | INFO | PerfReport: Gathered 1536 samples. | 20260508 05:59:40.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 05:59:40.384 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 24.14%, Upload -- 2.56e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 24.38%, Download -- 2.58e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 30.93%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 2.01%, LockUpload -- 2.13e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 2.81%, LockDownload -- 2.98e+00 / 0.00e+00, 4.40e+00 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 15.70%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 05:59:40.384 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260508 05:59:40.493 | INFO | Starting a round of Baz IO | 20260508 05:59:43.911 | INFO | Populating RTMetrics report | 20260508 05:59:43.911 | INFO | RTMetrics report populated | 20260508 05:59:43.911 | INFO | Opening tmp RTMetrics json file | 20260508 05:59:43.912 | INFO | Serializing RTMetrics to json | 20260508 05:59:43.916 | INFO | Serialization done, swapping tmp file now... | 20260508 05:59:43.925 | INFO | RTMetrics done | 20260508 05:59:43.989 | INFO | Finished a round of Baz IO: Wrote 1.562 GiB in 3.495 seconds | 20260508 05:59:43.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.77% / 0.00% / 59.22% | 20260508 05:59:43.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.427675% / 3.97364e-06% / 6.75519e-05% | 20260508 05:59:43.989 | INFO | Baz IO overall: | 20260508 05:59:43.989 | INFO | BytesWritten : 2570654054132 | 20260508 05:59:43.989 | INFO | NumEvents : 1239811249561 | 20260508 05:59:45.514 | INFO | Analyzing chunk frames = [5783552,5784064) | 20260508 05:59:45.514 | INFO | MemInfo: { | 20260508 05:59:45.514 | INFO | "gpu" : " 17 GB", | 20260508 05:59:45.514 | INFO | "huge" : "", | 20260508 05:59:45.514 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:45.514 | INFO | "malloc" : " 14 GB", | 20260508 05:59:45.514 | INFO | "pinned" : " 29 GB" | 20260508 05:59:45.514 | INFO | } | 20260508 05:59:45.514 | INFO | Perf report for the last 5.13286 seconds: | 20260508 05:59:45.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:45.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:45.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:45.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:45.514 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 05:59:45.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:45.514 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 05:59:45.514 | INFO | PRE_HQ : 66.9%, 1.8, 61.5ms, 0 | 20260508 05:59:45.514 | INFO | BAZWRITER : 67.4%, 2.8, 96.9ms, 0 | 20260508 05:59:45.514 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 05:59:45.515 | INFO | Uploaded 36GiB to GPU in 2.52316s (14.2678GiB/s) | 20260508 05:59:45.515 | INFO | Downloaded 38GiB to GPU in 2.71002s (14.022GiB/s) | 20260508 05:59:50.659 | INFO | Analyzing chunk frames = [5784064,5784576) | 20260508 05:59:50.659 | INFO | MemInfo: { | 20260508 05:59:50.659 | INFO | "gpu" : " 17 GB", | 20260508 05:59:50.659 | INFO | "huge" : "", | 20260508 05:59:50.659 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:50.659 | INFO | "malloc" : " 14 GB", | 20260508 05:59:50.659 | INFO | "pinned" : " 29 GB" | 20260508 05:59:50.659 | INFO | } | 20260508 05:59:50.659 | INFO | Perf report for the last 5.14465 seconds: | 20260508 05:59:50.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:50.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 05:59:50.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 05:59:50.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:50.659 | INFO | Uploaded 36GiB to GPU in 2.49385s (14.4355GiB/s) | 20260508 05:59:50.659 | INFO | Downloaded 34GiB to GPU in 2.42928s (13.9959GiB/s) | 20260508 05:59:55.798 | INFO | Analyzing chunk frames = [5784576,5785088) | 20260508 05:59:55.798 | INFO | MemInfo: { | 20260508 05:59:55.798 | INFO | "gpu" : " 17 GB", | 20260508 05:59:55.798 | INFO | "huge" : "", | 20260508 05:59:55.798 | INFO | "huge_shared" : " 29 GB", | 20260508 05:59:55.798 | INFO | "malloc" : " 14 GB", | 20260508 05:59:55.798 | INFO | "pinned" : " 29 GB" | 20260508 05:59:55.798 | INFO | } | 20260508 05:59:55.799 | INFO | Perf report for the last 5.13968 seconds: | 20260508 05:59:55.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 05:59:55.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 05:59:55.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 05:59:55.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 05:59:55.799 | INFO | Uploaded 36GiB to GPU in 2.5202s (14.2846GiB/s) | 20260508 05:59:55.799 | INFO | Downloaded 34GiB to GPU in 2.40216s (14.154GiB/s) | 20260508 06:00:00.944 | INFO | Analyzing chunk frames = [5785088,5785600) | 20260508 06:00:00.944 | INFO | MemInfo: { | 20260508 06:00:00.944 | INFO | "gpu" : " 17 GB", | 20260508 06:00:00.944 | INFO | "huge" : "", | 20260508 06:00:00.944 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:00.944 | INFO | "malloc" : " 14 GB", | 20260508 06:00:00.944 | INFO | "pinned" : " 29 GB" | 20260508 06:00:00.944 | INFO | } | 20260508 06:00:00.944 | INFO | Perf report for the last 5.14589 seconds: | 20260508 06:00:00.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:00.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.945 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 06:00:00.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.945 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:00:00.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:00.945 | INFO | Uploaded 36GiB to GPU in 2.5876s (13.9125GiB/s) | 20260508 06:00:00.945 | INFO | Downloaded 34GiB to GPU in 2.40803s (14.1194GiB/s) | 20260508 06:00:06.078 | INFO | Analyzing chunk frames = [5785600,5786112) | 20260508 06:00:06.078 | INFO | MemInfo: { | 20260508 06:00:06.078 | INFO | "gpu" : " 17 GB", | 20260508 06:00:06.078 | INFO | "huge" : "", | 20260508 06:00:06.078 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:06.078 | INFO | "malloc" : " 14 GB", | 20260508 06:00:06.078 | INFO | "pinned" : " 29 GB" | 20260508 06:00:06.079 | INFO | } | 20260508 06:00:06.079 | INFO | Perf report for the last 5.13414 seconds: | 20260508 06:00:06.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:06.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 06:00:06.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:00:06.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:06.079 | INFO | Uploaded 36GiB to GPU in 2.40199s (14.9876GiB/s) | 20260508 06:00:06.079 | INFO | Downloaded 34GiB to GPU in 2.51254s (13.5321GiB/s) | 20260508 06:00:11.213 | INFO | Analyzing chunk frames = [5786112,5786624) | 20260508 06:00:11.213 | INFO | MemInfo: { | 20260508 06:00:11.213 | INFO | "gpu" : " 17 GB", | 20260508 06:00:11.213 | INFO | "huge" : "", | 20260508 06:00:11.213 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:11.213 | INFO | "malloc" : " 14 GB", | 20260508 06:00:11.213 | INFO | "pinned" : " 29 GB" | 20260508 06:00:11.213 | INFO | } | 20260508 06:00:11.214 | INFO | Perf report for the last 5.13505 seconds: | 20260508 06:00:11.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:11.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 06:00:11.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:00:11.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:11.214 | INFO | Uploaded 36GiB to GPU in 2.47478s (14.5468GiB/s) | 20260508 06:00:11.214 | INFO | Downloaded 34GiB to GPU in 2.44473s (13.9074GiB/s) | 20260508 06:00:16.354 | INFO | Analyzing chunk frames = [5786624,5787136) | 20260508 06:00:16.354 | INFO | MemInfo: { | 20260508 06:00:16.354 | INFO | "gpu" : " 17 GB", | 20260508 06:00:16.354 | INFO | "huge" : "", | 20260508 06:00:16.354 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:16.354 | INFO | "malloc" : " 14 GB", | 20260508 06:00:16.354 | INFO | "pinned" : " 29 GB" | 20260508 06:00:16.354 | INFO | } | 20260508 06:00:16.354 | INFO | Perf report for the last 5.14058 seconds: | 20260508 06:00:16.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:16.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:00:16.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 06:00:16.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:16.354 | INFO | Uploaded 36GiB to GPU in 2.45091s (14.6884GiB/s) | 20260508 06:00:16.354 | INFO | Downloaded 34GiB to GPU in 2.42994s (13.9921GiB/s) | 20260508 06:00:21.495 | INFO | Analyzing chunk frames = [5787136,5787648) | 20260508 06:00:21.495 | INFO | MemInfo: { | 20260508 06:00:21.495 | INFO | "gpu" : " 17 GB", | 20260508 06:00:21.495 | INFO | "huge" : "", | 20260508 06:00:21.495 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:21.495 | INFO | "malloc" : " 14 GB", | 20260508 06:00:21.495 | INFO | "pinned" : " 29 GB" | 20260508 06:00:21.495 | INFO | } | 20260508 06:00:21.495 | INFO | Perf report for the last 5.14057 seconds: | 20260508 06:00:21.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:21.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:00:21.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 06:00:21.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:21.495 | INFO | Uploaded 36GiB to GPU in 2.4675s (14.5897GiB/s) | 20260508 06:00:21.495 | INFO | Downloaded 34GiB to GPU in 2.43743s (13.9491GiB/s) | 20260508 06:00:21.592 | INFO | Starting a round of Baz IO | 20260508 06:00:25.023 | INFO | Populating RTMetrics report | 20260508 06:00:25.023 | INFO | RTMetrics report populated | 20260508 06:00:25.023 | INFO | Opening tmp RTMetrics json file | 20260508 06:00:25.033 | INFO | Serializing RTMetrics to json | 20260508 06:00:25.034 | INFO | Serialization done, swapping tmp file now... | 20260508 06:00:25.034 | INFO | RTMetrics done | 20260508 06:00:25.092 | INFO | Finished a round of Baz IO: Wrote 1.562 GiB in 3.5 seconds | 20260508 06:00:25.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.77% / 0.00% / 59.23% | 20260508 06:00:25.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.427155% / 3.97364e-06% / 6.75519e-05% | 20260508 06:00:25.092 | INFO | Baz IO overall: | 20260508 06:00:25.092 | INFO | BytesWritten : 2572331142636 | 20260508 06:00:25.092 | INFO | NumEvents : 1240610797996 | 20260508 06:00:26.638 | INFO | Analyzing chunk frames = [5787648,5788160) | 20260508 06:00:26.638 | INFO | MemInfo: { | 20260508 06:00:26.638 | INFO | "gpu" : " 17 GB", | 20260508 06:00:26.638 | INFO | "huge" : "", | 20260508 06:00:26.638 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:26.638 | INFO | "malloc" : " 14 GB", | 20260508 06:00:26.638 | INFO | "pinned" : " 29 GB" | 20260508 06:00:26.638 | INFO | } | 20260508 06:00:26.638 | INFO | Perf report for the last 5.14356 seconds: | 20260508 06:00:26.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:26.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:26.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:26.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:26.638 | INFO | ANALYSIS : 67.7%, 3.0, 106.0ms, 0 | 20260508 06:00:26.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:26.639 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 06:00:26.639 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 06:00:26.639 | INFO | BAZWRITER : 67.5%, 2.6, 90.6ms, 0 | 20260508 06:00:26.639 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:00:26.639 | INFO | Uploaded 36GiB to GPU in 2.5398s (14.1743GiB/s) | 20260508 06:00:26.639 | INFO | Downloaded 38GiB to GPU in 2.68601s (14.1474GiB/s) | 20260508 06:00:31.774 | INFO | Analyzing chunk frames = [5788160,5788672) | 20260508 06:00:31.774 | INFO | MemInfo: { | 20260508 06:00:31.774 | INFO | "gpu" : " 17 GB", | 20260508 06:00:31.774 | INFO | "huge" : "", | 20260508 06:00:31.774 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:31.774 | INFO | "malloc" : " 14 GB", | 20260508 06:00:31.774 | INFO | "pinned" : " 29 GB" | 20260508 06:00:31.774 | INFO | } | 20260508 06:00:31.774 | INFO | Perf report for the last 5.13553 seconds: | 20260508 06:00:31.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:31.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:00:31.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:00:31.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:31.774 | INFO | Uploaded 36GiB to GPU in 2.50419s (14.3759GiB/s) | 20260508 06:00:31.774 | INFO | Downloaded 34GiB to GPU in 2.43715s (13.9507GiB/s) | 20260508 06:00:36.913 | INFO | Analyzing chunk frames = [5788672,5789184) | 20260508 06:00:36.913 | INFO | MemInfo: { | 20260508 06:00:36.913 | INFO | "gpu" : " 17 GB", | 20260508 06:00:36.913 | INFO | "huge" : "", | 20260508 06:00:36.913 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:36.913 | INFO | "malloc" : " 14 GB", | 20260508 06:00:36.913 | INFO | "pinned" : " 29 GB" | 20260508 06:00:36.913 | INFO | } | 20260508 06:00:36.913 | INFO | Perf report for the last 5.13926 seconds: | 20260508 06:00:36.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:36.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.913 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:00:36.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.913 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:00:36.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:36.914 | INFO | Uploaded 36GiB to GPU in 2.53628s (14.194GiB/s) | 20260508 06:00:36.914 | INFO | Downloaded 34GiB to GPU in 2.40888s (14.1144GiB/s) | 20260508 06:00:42.053 | INFO | Analyzing chunk frames = [5789184,5789696) | 20260508 06:00:42.053 | INFO | MemInfo: { | 20260508 06:00:42.053 | INFO | "gpu" : " 17 GB", | 20260508 06:00:42.053 | INFO | "huge" : "", | 20260508 06:00:42.053 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:42.053 | INFO | "malloc" : " 14 GB", | 20260508 06:00:42.053 | INFO | "pinned" : " 29 GB" | 20260508 06:00:42.053 | INFO | } | 20260508 06:00:42.053 | INFO | Perf report for the last 5.13976 seconds: | 20260508 06:00:42.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:42.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.053 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 06:00:42.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.053 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:00:42.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:42.054 | INFO | Uploaded 36GiB to GPU in 2.54487s (14.1461GiB/s) | 20260508 06:00:42.054 | INFO | Downloaded 34GiB to GPU in 2.40419s (14.142GiB/s) | 20260508 06:00:47.194 | INFO | Analyzing chunk frames = [5789696,5790208) | 20260508 06:00:47.194 | INFO | MemInfo: { | 20260508 06:00:47.194 | INFO | "gpu" : " 17 GB", | 20260508 06:00:47.194 | INFO | "huge" : "", | 20260508 06:00:47.194 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:47.194 | INFO | "malloc" : " 14 GB", | 20260508 06:00:47.194 | INFO | "pinned" : " 29 GB" | 20260508 06:00:47.194 | INFO | } | 20260508 06:00:47.194 | INFO | Perf report for the last 5.1413 seconds: | 20260508 06:00:47.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:47.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.194 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 06:00:47.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.194 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:00:47.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:47.195 | INFO | Uploaded 36GiB to GPU in 2.4286s (14.8234GiB/s) | 20260508 06:00:47.195 | INFO | Downloaded 34GiB to GPU in 2.50713s (13.5613GiB/s) | 20260508 06:00:52.326 | INFO | Analyzing chunk frames = [5790208,5790720) | 20260508 06:00:52.326 | INFO | MemInfo: { | 20260508 06:00:52.326 | INFO | "gpu" : " 17 GB", | 20260508 06:00:52.326 | INFO | "huge" : "", | 20260508 06:00:52.326 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:52.327 | INFO | "malloc" : " 14 GB", | 20260508 06:00:52.327 | INFO | "pinned" : " 29 GB" | 20260508 06:00:52.327 | INFO | } | 20260508 06:00:52.327 | INFO | Perf report for the last 5.13242 seconds: | 20260508 06:00:52.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:52.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.327 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 06:00:52.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.327 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260508 06:00:52.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:52.328 | INFO | Uploaded 36GiB to GPU in 2.47209s (14.5626GiB/s) | 20260508 06:00:52.328 | INFO | Downloaded 34GiB to GPU in 2.42342s (14.0298GiB/s) | 20260508 06:00:57.468 | INFO | Analyzing chunk frames = [5790720,5791232) | 20260508 06:00:57.468 | INFO | MemInfo: { | 20260508 06:00:57.468 | INFO | "gpu" : " 17 GB", | 20260508 06:00:57.468 | INFO | "huge" : "", | 20260508 06:00:57.468 | INFO | "huge_shared" : " 29 GB", | 20260508 06:00:57.468 | INFO | "malloc" : " 14 GB", | 20260508 06:00:57.468 | INFO | "pinned" : " 29 GB" | 20260508 06:00:57.468 | INFO | } | 20260508 06:00:57.468 | INFO | Perf report for the last 5.1413 seconds: | 20260508 06:00:57.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:00:57.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:00:57.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260508 06:00:57.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:00:57.468 | INFO | Uploaded 36GiB to GPU in 2.47197s (14.5633GiB/s) | 20260508 06:00:57.468 | INFO | Downloaded 34GiB to GPU in 2.42923s (13.9962GiB/s) | 20260508 06:01:02.609 | INFO | Analyzing chunk frames = [5791232,5791744) | 20260508 06:01:02.609 | INFO | MemInfo: { | 20260508 06:01:02.609 | INFO | "gpu" : " 17 GB", | 20260508 06:01:02.609 | INFO | "huge" : "", | 20260508 06:01:02.609 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:02.609 | INFO | "malloc" : " 14 GB", | 20260508 06:01:02.609 | INFO | "pinned" : " 29 GB" | 20260508 06:01:02.609 | INFO | } | 20260508 06:01:02.609 | INFO | Perf report for the last 5.14141 seconds: | 20260508 06:01:02.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:02.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:01:02.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:01:02.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:02.610 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:01:02.610 | INFO | Total mem usage: 16.7 GiB | 20260508 06:01:02.610 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:01:02.610 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:01:02.610 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:01:02.610 | INFO | High Water Breakdown: | 20260508 06:01:02.610 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:01:02.610 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:01:02.610 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:01:02.610 | INFO | ------------------------------------------ | 20260508 06:01:02.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:01:02.610 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:01:02.610 | INFO | Total mem usage: 26.5 GiB | 20260508 06:01:02.610 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:01:02.610 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:01:02.610 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:01:02.610 | INFO | High Water Breakdown: | 20260508 06:01:02.610 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:01:02.610 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:01:02.610 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:01:02.610 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:01:02.610 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:01:02.610 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:01:02.610 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:01:02.610 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:01:02.610 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:01:02.610 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:01:02.610 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:01:02.610 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:01:02.610 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:01:02.610 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:01:02.610 | INFO | ------------------------------------------ | 20260508 06:01:02.610 | INFO | Uploaded 36GiB to GPU in 2.47225s (14.5616GiB/s) | 20260508 06:01:02.610 | INFO | Downloaded 34GiB to GPU in 2.4267s (14.0108GiB/s) | 20260508 06:01:02.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:01:02.610 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:01:02.610 | INFO | Total mem usage: 27.8 GiB | 20260508 06:01:02.610 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:01:02.610 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:01:02.610 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:01:02.610 | INFO | High Water Breakdown: | 20260508 06:01:02.610 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:01:02.610 | INFO | ------------------------------------------ | 20260508 06:01:02.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:01:02.610 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:01:02.610 | INFO | Total mem usage: 15.7 GiB | 20260508 06:01:02.610 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:01:02.610 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:01:02.611 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:01:02.611 | INFO | High Water Breakdown: | 20260508 06:01:02.611 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:01:02.611 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:01:02.611 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:01:02.611 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:01:02.611 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:01:02.611 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:01:02.611 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:01:02.611 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:01:02.611 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:01:02.611 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:01:02.611 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:01:02.611 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:01:02.611 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:01:02.611 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:01:02.611 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:01:02.611 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:01:02.611 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:01:02.611 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:01:02.611 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:01:02.611 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:01:02.611 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:01:02.611 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:01:02.611 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:01:02.611 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:01:02.611 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:01:02.611 | INFO | ------------------------------------------ | 20260508 06:01:02.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:01:02.611 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:01:02.611 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:01:02.611 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:01:02.611 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 6.11e-01 | 20260508 06:01:02.611 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.11e-02 / 7.11e-03 | 20260508 06:01:02.611 | INFO | PerfReport: 12.12%, Baseline -- 3.97e+00 / 4.33e+00, 5.86e-01 / 2.58e-01 | 20260508 06:01:02.611 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.29e-01 | 20260508 06:01:02.611 | INFO | PerfReport: 15.88%, DME -- 4.16e+01 / 4.48e+01, 3.79e+00 / 1.41e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.82e+00 / 1.55e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.51e+00, 4.12e-01 / 2.78e-01 | 20260508 06:01:02.611 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.22e+00, 2.57e-01 / 1.22e-01 | 20260508 06:01:02.611 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.50e-01, 4.97e-02 / 2.45e-02 | 20260508 06:01:02.611 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:01:02.611 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:01:02.611 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:01:02.611 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 24.46%, Upload -- 2.60e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 24.22%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 2.28%, LockUpload -- 2.43e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 2.55%, LockDownload -- 2.71e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:01:02.611 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.89e-02 / 0.00e+00 | 20260508 06:01:02.707 | INFO | Starting a round of Baz IO | 20260508 06:01:06.133 | INFO | Populating RTMetrics report | 20260508 06:01:06.134 | INFO | RTMetrics report populated | 20260508 06:01:06.134 | INFO | Opening tmp RTMetrics json file | 20260508 06:01:06.150 | INFO | Serializing RTMetrics to json | 20260508 06:01:06.152 | INFO | Serialization done, swapping tmp file now... | 20260508 06:01:06.153 | INFO | RTMetrics done | 20260508 06:01:06.207 | INFO | Finished a round of Baz IO: Wrote 1.561 GiB in 3.5 seconds | 20260508 06:01:06.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.77% / 0.00% / 59.23% | 20260508 06:01:06.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.426312% / 3.97364e-06% / 7.94729e-05% | 20260508 06:01:06.207 | INFO | Baz IO overall: | 20260508 06:01:06.207 | INFO | BytesWritten : 2574007542962 | 20260508 06:01:06.207 | INFO | NumEvents : 1241409865192 | 20260508 06:01:07.759 | INFO | Analyzing chunk frames = [5791744,5792256) | 20260508 06:01:07.759 | INFO | MemInfo: { | 20260508 06:01:07.759 | INFO | "gpu" : " 17 GB", | 20260508 06:01:07.759 | INFO | "huge" : "", | 20260508 06:01:07.759 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:07.759 | INFO | "malloc" : " 14 GB", | 20260508 06:01:07.759 | INFO | "pinned" : " 29 GB" | 20260508 06:01:07.759 | INFO | } | 20260508 06:01:07.759 | INFO | Perf report for the last 5.14938 seconds: | 20260508 06:01:07.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:07.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:07.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:07.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:07.759 | INFO | ANALYSIS : 67.8%, 3.0, 106.1ms, 0 | 20260508 06:01:07.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:07.759 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:01:07.759 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 06:01:07.759 | INFO | BAZWRITER : 67.4%, 2.6, 91.0ms, 0 | 20260508 06:01:07.759 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:01:07.760 | INFO | Uploaded 36GiB to GPU in 2.57294s (13.9918GiB/s) | 20260508 06:01:07.760 | INFO | Downloaded 38GiB to GPU in 2.70125s (14.0676GiB/s) | 20260508 06:01:12.892 | INFO | Analyzing chunk frames = [5792256,5792768) | 20260508 06:01:12.892 | INFO | MemInfo: { | 20260508 06:01:12.892 | INFO | "gpu" : " 17 GB", | 20260508 06:01:12.892 | INFO | "huge" : "", | 20260508 06:01:12.892 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:12.892 | INFO | "malloc" : " 14 GB", | 20260508 06:01:12.892 | INFO | "pinned" : " 29 GB" | 20260508 06:01:12.892 | INFO | } | 20260508 06:01:12.892 | INFO | Perf report for the last 5.13357 seconds: | 20260508 06:01:12.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:12.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.892 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 06:01:12.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.892 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:01:12.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:12.893 | INFO | Uploaded 36GiB to GPU in 2.49712s (14.4166GiB/s) | 20260508 06:01:12.893 | INFO | Downloaded 34GiB to GPU in 2.43374s (13.9703GiB/s) | 20260508 06:01:18.026 | INFO | Analyzing chunk frames = [5792768,5793280) | 20260508 06:01:18.026 | INFO | MemInfo: { | 20260508 06:01:18.026 | INFO | "gpu" : " 17 GB", | 20260508 06:01:18.026 | INFO | "huge" : "", | 20260508 06:01:18.026 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:18.026 | INFO | "malloc" : " 14 GB", | 20260508 06:01:18.027 | INFO | "pinned" : " 29 GB" | 20260508 06:01:18.027 | INFO | } | 20260508 06:01:18.027 | INFO | Perf report for the last 5.13467 seconds: | 20260508 06:01:18.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:18.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 06:01:18.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:01:18.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:18.027 | INFO | Uploaded 36GiB to GPU in 2.55661s (14.0812GiB/s) | 20260508 06:01:18.027 | INFO | Downloaded 34GiB to GPU in 2.40277s (14.1503GiB/s) | 20260508 06:01:23.165 | INFO | Analyzing chunk frames = [5793280,5793792) | 20260508 06:01:23.165 | INFO | MemInfo: { | 20260508 06:01:23.165 | INFO | "gpu" : " 17 GB", | 20260508 06:01:23.165 | INFO | "huge" : "", | 20260508 06:01:23.165 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:23.165 | INFO | "malloc" : " 14 GB", | 20260508 06:01:23.165 | INFO | "pinned" : " 29 GB" | 20260508 06:01:23.165 | INFO | } | 20260508 06:01:23.165 | INFO | Perf report for the last 5.13801 seconds: | 20260508 06:01:23.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:23.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:01:23.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:01:23.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:23.165 | INFO | Uploaded 36GiB to GPU in 2.56639s (14.0275GiB/s) | 20260508 06:01:23.165 | INFO | Downloaded 34GiB to GPU in 2.38288s (14.2684GiB/s) | 20260508 06:01:28.309 | INFO | Analyzing chunk frames = [5793792,5794304) | 20260508 06:01:28.309 | INFO | MemInfo: { | 20260508 06:01:28.309 | INFO | "gpu" : " 17 GB", | 20260508 06:01:28.309 | INFO | "huge" : "", | 20260508 06:01:28.309 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:28.309 | INFO | "malloc" : " 14 GB", | 20260508 06:01:28.309 | INFO | "pinned" : " 29 GB" | 20260508 06:01:28.309 | INFO | } | 20260508 06:01:28.309 | INFO | Perf report for the last 5.14361 seconds: | 20260508 06:01:28.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:28.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:01:28.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 06:01:28.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:28.309 | INFO | Uploaded 36GiB to GPU in 2.33795s (15.3981GiB/s) | 20260508 06:01:28.309 | INFO | Downloaded 34GiB to GPU in 2.49753s (13.6134GiB/s) | 20260508 06:01:33.448 | INFO | Analyzing chunk frames = [5794304,5794816) | 20260508 06:01:33.448 | INFO | MemInfo: { | 20260508 06:01:33.448 | INFO | "gpu" : " 17 GB", | 20260508 06:01:33.448 | INFO | "huge" : "", | 20260508 06:01:33.448 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:33.448 | INFO | "malloc" : " 14 GB", | 20260508 06:01:33.448 | INFO | "pinned" : " 29 GB" | 20260508 06:01:33.448 | INFO | } | 20260508 06:01:33.448 | INFO | Perf report for the last 5.13965 seconds: | 20260508 06:01:33.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:33.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.448 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 06:01:33.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.449 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:01:33.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:33.449 | INFO | Uploaded 36GiB to GPU in 2.45429s (14.6682GiB/s) | 20260508 06:01:33.449 | INFO | Downloaded 34GiB to GPU in 2.41524s (14.0773GiB/s) | 20260508 06:01:38.587 | INFO | Analyzing chunk frames = [5794816,5795328) | 20260508 06:01:38.587 | INFO | MemInfo: { | 20260508 06:01:38.587 | INFO | "gpu" : " 17 GB", | 20260508 06:01:38.587 | INFO | "huge" : "", | 20260508 06:01:38.587 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:38.587 | INFO | "malloc" : " 14 GB", | 20260508 06:01:38.587 | INFO | "pinned" : " 29 GB" | 20260508 06:01:38.587 | INFO | } | 20260508 06:01:38.587 | INFO | Perf report for the last 5.13837 seconds: | 20260508 06:01:38.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:38.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:01:38.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:01:38.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:38.587 | INFO | Uploaded 36GiB to GPU in 2.46416s (14.6095GiB/s) | 20260508 06:01:38.587 | INFO | Downloaded 34GiB to GPU in 2.41197s (14.0964GiB/s) | 20260508 06:01:43.723 | INFO | Analyzing chunk frames = [5795328,5795840) | 20260508 06:01:43.723 | INFO | MemInfo: { | 20260508 06:01:43.723 | INFO | "gpu" : " 17 GB", | 20260508 06:01:43.723 | INFO | "huge" : "", | 20260508 06:01:43.723 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:43.723 | INFO | "malloc" : " 14 GB", | 20260508 06:01:43.723 | INFO | "pinned" : " 29 GB" | 20260508 06:01:43.723 | INFO | } | 20260508 06:01:43.723 | INFO | Perf report for the last 5.1361 seconds: | 20260508 06:01:43.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:43.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:01:43.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:01:43.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:43.723 | INFO | Uploaded 36GiB to GPU in 2.48544s (14.4843GiB/s) | 20260508 06:01:43.723 | INFO | Downloaded 34GiB to GPU in 2.42539s (14.0184GiB/s) | 20260508 06:01:43.820 | INFO | Starting a round of Baz IO | 20260508 06:01:47.241 | INFO | Populating RTMetrics report | 20260508 06:01:47.241 | INFO | RTMetrics report populated | 20260508 06:01:47.241 | INFO | Opening tmp RTMetrics json file | 20260508 06:01:47.247 | INFO | Serializing RTMetrics to json | 20260508 06:01:47.257 | INFO | Serialization done, swapping tmp file now... | 20260508 06:01:47.258 | INFO | RTMetrics done | 20260508 06:01:47.313 | INFO | Finished a round of Baz IO: Wrote 1.561 GiB in 3.493 seconds | 20260508 06:01:47.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.77% / 0.00% / 59.23% | 20260508 06:01:47.313 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.424612% / 0% / 4.37101e-05% | 20260508 06:01:47.313 | INFO | Baz IO overall: | 20260508 06:01:47.313 | INFO | BytesWritten : 2575683346374 | 20260508 06:01:47.313 | INFO | NumEvents : 1242208420063 | 20260508 06:01:48.866 | INFO | Analyzing chunk frames = [5795840,5796352) | 20260508 06:01:48.866 | INFO | MemInfo: { | 20260508 06:01:48.866 | INFO | "gpu" : " 17 GB", | 20260508 06:01:48.866 | INFO | "huge" : "", | 20260508 06:01:48.866 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:48.866 | INFO | "malloc" : " 14 GB", | 20260508 06:01:48.866 | INFO | "pinned" : " 29 GB" | 20260508 06:01:48.866 | INFO | } | 20260508 06:01:48.866 | INFO | Perf report for the last 5.14344 seconds: | 20260508 06:01:48.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:48.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:48.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:48.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:48.866 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 06:01:48.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:48.866 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:01:48.867 | INFO | PRE_HQ : 65.6%, 1.6, 55.2ms, 0 | 20260508 06:01:48.867 | INFO | BAZWRITER : 67.2%, 2.6, 91.0ms, 0 | 20260508 06:01:48.867 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:01:48.867 | INFO | Uploaded 36GiB to GPU in 2.56559s (14.0319GiB/s) | 20260508 06:01:48.867 | INFO | Downloaded 38GiB to GPU in 2.68969s (14.128GiB/s) | 20260508 06:01:54.005 | INFO | Analyzing chunk frames = [5796352,5796864) | 20260508 06:01:54.005 | INFO | MemInfo: { | 20260508 06:01:54.005 | INFO | "gpu" : " 17 GB", | 20260508 06:01:54.005 | INFO | "huge" : "", | 20260508 06:01:54.005 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:54.005 | INFO | "malloc" : " 14 GB", | 20260508 06:01:54.005 | INFO | "pinned" : " 29 GB" | 20260508 06:01:54.005 | INFO | } | 20260508 06:01:54.005 | INFO | Perf report for the last 5.13876 seconds: | 20260508 06:01:54.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:54.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.005 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:01:54.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.005 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:01:54.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:54.006 | INFO | Uploaded 36GiB to GPU in 2.51447s (14.3171GiB/s) | 20260508 06:01:54.006 | INFO | Downloaded 34GiB to GPU in 2.42866s (13.9995GiB/s) | 20260508 06:01:59.144 | INFO | Analyzing chunk frames = [5796864,5797376) | 20260508 06:01:59.144 | INFO | MemInfo: { | 20260508 06:01:59.144 | INFO | "gpu" : " 17 GB", | 20260508 06:01:59.145 | INFO | "huge" : "", | 20260508 06:01:59.145 | INFO | "huge_shared" : " 29 GB", | 20260508 06:01:59.145 | INFO | "malloc" : " 14 GB", | 20260508 06:01:59.145 | INFO | "pinned" : " 29 GB" | 20260508 06:01:59.145 | INFO | } | 20260508 06:01:59.145 | INFO | Perf report for the last 5.13956 seconds: | 20260508 06:01:59.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:01:59.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:01:59.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:01:59.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:01:59.145 | INFO | Uploaded 36GiB to GPU in 2.57448s (13.9834GiB/s) | 20260508 06:01:59.145 | INFO | Downloaded 34GiB to GPU in 2.40973s (14.1095GiB/s) | 20260508 06:02:04.280 | INFO | Analyzing chunk frames = [5797376,5797888) | 20260508 06:02:04.280 | INFO | MemInfo: { | 20260508 06:02:04.280 | INFO | "gpu" : " 17 GB", | 20260508 06:02:04.280 | INFO | "huge" : "", | 20260508 06:02:04.280 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:04.280 | INFO | "malloc" : " 14 GB", | 20260508 06:02:04.280 | INFO | "pinned" : " 29 GB" | 20260508 06:02:04.280 | INFO | } | 20260508 06:02:04.280 | INFO | Perf report for the last 5.13512 seconds: | 20260508 06:02:04.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:04.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 06:02:04.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 06:02:04.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:04.280 | INFO | Uploaded 36GiB to GPU in 2.59207s (13.8885GiB/s) | 20260508 06:02:04.280 | INFO | Downloaded 34GiB to GPU in 2.39406s (14.2018GiB/s) | 20260508 06:02:09.423 | INFO | Analyzing chunk frames = [5797888,5798400) | 20260508 06:02:09.423 | INFO | MemInfo: { | 20260508 06:02:09.423 | INFO | "gpu" : " 17 GB", | 20260508 06:02:09.423 | INFO | "huge" : "", | 20260508 06:02:09.423 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:09.423 | INFO | "malloc" : " 14 GB", | 20260508 06:02:09.423 | INFO | "pinned" : " 29 GB" | 20260508 06:02:09.423 | INFO | } | 20260508 06:02:09.423 | INFO | Perf report for the last 5.14321 seconds: | 20260508 06:02:09.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:09.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 06:02:09.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:02:09.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:09.423 | INFO | Uploaded 36GiB to GPU in 2.3105s (15.581GiB/s) | 20260508 06:02:09.423 | INFO | Downloaded 34GiB to GPU in 2.46843s (13.7739GiB/s) | 20260508 06:02:14.560 | INFO | Analyzing chunk frames = [5798400,5798912) | 20260508 06:02:14.560 | INFO | MemInfo: { | 20260508 06:02:14.560 | INFO | "gpu" : " 17 GB", | 20260508 06:02:14.560 | INFO | "huge" : "", | 20260508 06:02:14.560 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:14.560 | INFO | "malloc" : " 14 GB", | 20260508 06:02:14.560 | INFO | "pinned" : " 29 GB" | 20260508 06:02:14.560 | INFO | } | 20260508 06:02:14.560 | INFO | Perf report for the last 5.13738 seconds: | 20260508 06:02:14.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:14.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.561 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:02:14.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.561 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:02:14.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:14.561 | INFO | Uploaded 36GiB to GPU in 2.4524s (14.6795GiB/s) | 20260508 06:02:14.561 | INFO | Downloaded 34GiB to GPU in 2.43019s (13.9907GiB/s) | 20260508 06:02:19.700 | INFO | Analyzing chunk frames = [5798912,5799424) | 20260508 06:02:19.700 | INFO | MemInfo: { | 20260508 06:02:19.700 | INFO | "gpu" : " 17 GB", | 20260508 06:02:19.700 | INFO | "huge" : "", | 20260508 06:02:19.700 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:19.700 | INFO | "malloc" : " 14 GB", | 20260508 06:02:19.700 | INFO | "pinned" : " 29 GB" | 20260508 06:02:19.700 | INFO | } | 20260508 06:02:19.701 | INFO | Perf report for the last 5.14001 seconds: | 20260508 06:02:19.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:19.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 06:02:19.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:02:19.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:19.701 | INFO | Uploaded 36GiB to GPU in 2.43627s (14.7767GiB/s) | 20260508 06:02:19.701 | INFO | Downloaded 34GiB to GPU in 2.41689s (14.0676GiB/s) | 20260508 06:02:24.838 | INFO | Analyzing chunk frames = [5799424,5799936) | 20260508 06:02:24.838 | INFO | MemInfo: { | 20260508 06:02:24.838 | INFO | "gpu" : " 17 GB", | 20260508 06:02:24.838 | INFO | "huge" : "", | 20260508 06:02:24.838 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:24.838 | INFO | "malloc" : " 14 GB", | 20260508 06:02:24.838 | INFO | "pinned" : " 29 GB" | 20260508 06:02:24.838 | INFO | } | 20260508 06:02:24.838 | INFO | Perf report for the last 5.13734 seconds: | 20260508 06:02:24.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:24.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:02:24.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 06:02:24.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:24.838 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:02:24.838 | INFO | Total mem usage: 16.7 GiB | 20260508 06:02:24.838 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:02:24.838 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:02:24.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:02:24.838 | INFO | High Water Breakdown: | 20260508 06:02:24.838 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:02:24.838 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:02:24.838 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:02:24.838 | INFO | ------------------------------------------ | 20260508 06:02:24.838 | INFO | Uploaded 36GiB to GPU in 2.44402s (14.7298GiB/s) | 20260508 06:02:24.838 | INFO | Downloaded 34GiB to GPU in 2.41732s (14.0651GiB/s) | 20260508 06:02:24.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:02:24.839 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:02:24.839 | INFO | Total mem usage: 26.5 GiB | 20260508 06:02:24.839 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:02:24.839 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:02:24.839 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:02:24.839 | INFO | High Water Breakdown: | 20260508 06:02:24.839 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:02:24.839 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:02:24.839 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:02:24.839 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:02:24.839 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:02:24.839 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:02:24.839 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:02:24.839 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:02:24.839 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:02:24.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:02:24.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:02:24.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:02:24.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:02:24.839 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:02:24.839 | INFO | ------------------------------------------ | 20260508 06:02:24.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:02:24.839 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:02:24.839 | INFO | Total mem usage: 27.8 GiB | 20260508 06:02:24.839 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:02:24.839 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:02:24.839 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:02:24.839 | INFO | High Water Breakdown: | 20260508 06:02:24.839 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:02:24.839 | INFO | ------------------------------------------ | 20260508 06:02:24.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:02:24.839 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:02:24.839 | INFO | Total mem usage: 15.7 GiB | 20260508 06:02:24.839 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:02:24.839 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:02:24.839 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:02:24.839 | INFO | High Water Breakdown: | 20260508 06:02:24.839 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:02:24.839 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:02:24.839 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:02:24.839 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:02:24.839 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:02:24.839 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:02:24.839 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:02:24.839 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:02:24.839 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:02:24.839 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:02:24.839 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:02:24.839 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:02:24.839 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:02:24.839 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:02:24.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:02:24.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:02:24.839 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:02:24.839 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:02:24.839 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:02:24.839 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:02:24.839 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:02:24.839 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:02:24.839 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:02:24.839 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:02:24.839 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:02:24.839 | INFO | ------------------------------------------ | 20260508 06:02:24.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:02:24.839 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:02:24.839 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:02:24.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:02:24.839 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.91e-01 | 20260508 06:02:24.839 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.09e-01, 1.13e-02 / 1.28e-02 | 20260508 06:02:24.839 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.40e+00, 5.85e-01 / 2.34e-01 | 20260508 06:02:24.840 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.17e-01 | 20260508 06:02:24.840 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.53e+01, 3.82e+00 / 1.09e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.85e+01 / 1.98e+01, 1.74e+00 / 1.32e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.60e+00, 4.10e-01 / 2.41e-01 | 20260508 06:02:24.840 | INFO | PerfReport: 3.17%, Metrics -- 1.03e+00 / 1.23e+00, 2.54e-01 / 1.34e-01 | 20260508 06:02:24.840 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.51e-01, 4.92e-02 / 2.32e-02 | 20260508 06:02:24.840 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:02:24.840 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:02:24.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:02:24.840 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 24.44%, Upload -- 2.59e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 24.13%, Download -- 2.56e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 30.79%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 2.45%, LockUpload -- 2.60e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 2.49%, LockDownload -- 2.64e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 15.69%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:02:24.840 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 9.09e-03 / 0.00e+00 | 20260508 06:02:24.935 | INFO | Starting a round of Baz IO | 20260508 06:02:28.364 | INFO | Populating RTMetrics report | 20260508 06:02:28.364 | INFO | RTMetrics report populated | 20260508 06:02:28.364 | INFO | Opening tmp RTMetrics json file | 20260508 06:02:28.379 | INFO | Serializing RTMetrics to json | 20260508 06:02:28.380 | INFO | Serialization done, swapping tmp file now... | 20260508 06:02:28.380 | INFO | RTMetrics done | 20260508 06:02:28.431 | INFO | Finished a round of Baz IO: Wrote 1.561 GiB in 3.495 seconds | 20260508 06:02:28.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.77% / 0.00% / 59.23% | 20260508 06:02:28.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.425299% / 7.94729e-06% / 4.37101e-05% | 20260508 06:02:28.431 | INFO | Baz IO overall: | 20260508 06:02:28.431 | INFO | BytesWritten : 2577359785911 | 20260508 06:02:28.431 | INFO | NumEvents : 1243007392959 | 20260508 06:02:29.979 | INFO | Analyzing chunk frames = [5799936,5800448) | 20260508 06:02:29.979 | INFO | MemInfo: { | 20260508 06:02:29.979 | INFO | "gpu" : " 17 GB", | 20260508 06:02:29.979 | INFO | "huge" : "", | 20260508 06:02:29.979 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:29.979 | INFO | "malloc" : " 14 GB", | 20260508 06:02:29.979 | INFO | "pinned" : " 29 GB" | 20260508 06:02:29.979 | INFO | } | 20260508 06:02:29.979 | INFO | Perf report for the last 5.14087 seconds: | 20260508 06:02:29.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:29.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:29.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:29.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:29.979 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:02:29.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:29.979 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:02:29.979 | INFO | PRE_HQ : 65.8%, 1.6, 55.4ms, 0 | 20260508 06:02:29.979 | INFO | BAZWRITER : 67.5%, 2.6, 91.1ms, 0 | 20260508 06:02:29.979 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:02:29.979 | INFO | Uploaded 36GiB to GPU in 2.51211s (14.3306GiB/s) | 20260508 06:02:29.979 | INFO | Downloaded 38GiB to GPU in 2.68833s (14.1352GiB/s) | 20260508 06:02:35.119 | INFO | Analyzing chunk frames = [5800448,5800960) | 20260508 06:02:35.119 | INFO | MemInfo: { | 20260508 06:02:35.119 | INFO | "gpu" : " 17 GB", | 20260508 06:02:35.119 | INFO | "huge" : "", | 20260508 06:02:35.119 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:35.119 | INFO | "malloc" : " 14 GB", | 20260508 06:02:35.119 | INFO | "pinned" : " 29 GB" | 20260508 06:02:35.119 | INFO | } | 20260508 06:02:35.120 | INFO | Perf report for the last 5.14078 seconds: | 20260508 06:02:35.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:35.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:02:35.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:02:35.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:35.120 | INFO | Uploaded 36GiB to GPU in 2.49468s (14.4307GiB/s) | 20260508 06:02:35.120 | INFO | Downloaded 34GiB to GPU in 2.44124s (13.9273GiB/s) | 20260508 06:02:40.258 | INFO | Analyzing chunk frames = [5800960,5801472) | 20260508 06:02:40.258 | INFO | MemInfo: { | 20260508 06:02:40.258 | INFO | "gpu" : " 17 GB", | 20260508 06:02:40.258 | INFO | "huge" : "", | 20260508 06:02:40.258 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:40.258 | INFO | "malloc" : " 14 GB", | 20260508 06:02:40.258 | INFO | "pinned" : " 29 GB" | 20260508 06:02:40.258 | INFO | } | 20260508 06:02:40.258 | INFO | Perf report for the last 5.13839 seconds: | 20260508 06:02:40.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:40.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 06:02:40.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:02:40.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:40.258 | INFO | Uploaded 36GiB to GPU in 2.52643s (14.2494GiB/s) | 20260508 06:02:40.258 | INFO | Downloaded 34GiB to GPU in 2.41171s (14.0979GiB/s) | 20260508 06:02:45.403 | INFO | Analyzing chunk frames = [5801472,5801984) | 20260508 06:02:45.403 | INFO | MemInfo: { | 20260508 06:02:45.403 | INFO | "gpu" : " 17 GB", | 20260508 06:02:45.403 | INFO | "huge" : "", | 20260508 06:02:45.403 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:45.403 | INFO | "malloc" : " 14 GB", | 20260508 06:02:45.404 | INFO | "pinned" : " 29 GB" | 20260508 06:02:45.404 | INFO | } | 20260508 06:02:45.404 | INFO | Perf report for the last 5.14567 seconds: | 20260508 06:02:45.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:45.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 06:02:45.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:02:45.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:45.404 | INFO | Uploaded 36GiB to GPU in 2.52346s (14.2661GiB/s) | 20260508 06:02:45.404 | INFO | Downloaded 34GiB to GPU in 2.38708s (14.2433GiB/s) | 20260508 06:02:50.534 | INFO | Analyzing chunk frames = [5801984,5802496) | 20260508 06:02:50.534 | INFO | MemInfo: { | 20260508 06:02:50.534 | INFO | "gpu" : " 17 GB", | 20260508 06:02:50.534 | INFO | "huge" : "", | 20260508 06:02:50.534 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:50.534 | INFO | "malloc" : " 14 GB", | 20260508 06:02:50.534 | INFO | "pinned" : " 29 GB" | 20260508 06:02:50.534 | INFO | } | 20260508 06:02:50.534 | INFO | Perf report for the last 5.13074 seconds: | 20260508 06:02:50.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:50.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.534 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 06:02:50.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.534 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:02:50.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:50.535 | INFO | Uploaded 36GiB to GPU in 2.29611s (15.6787GiB/s) | 20260508 06:02:50.535 | INFO | Downloaded 34GiB to GPU in 2.48853s (13.6627GiB/s) | 20260508 06:02:55.677 | INFO | Analyzing chunk frames = [5802496,5803008) | 20260508 06:02:55.677 | INFO | MemInfo: { | 20260508 06:02:55.677 | INFO | "gpu" : " 17 GB", | 20260508 06:02:55.677 | INFO | "huge" : "", | 20260508 06:02:55.677 | INFO | "huge_shared" : " 29 GB", | 20260508 06:02:55.677 | INFO | "malloc" : " 14 GB", | 20260508 06:02:55.677 | INFO | "pinned" : " 29 GB" | 20260508 06:02:55.677 | INFO | } | 20260508 06:02:55.677 | INFO | Perf report for the last 5.14306 seconds: | 20260508 06:02:55.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:02:55.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.678 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:02:55.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.678 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:02:55.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:02:55.678 | INFO | Uploaded 36GiB to GPU in 2.46955s (14.5775GiB/s) | 20260508 06:02:55.678 | INFO | Downloaded 34GiB to GPU in 2.43319s (13.9734GiB/s) | 20260508 06:03:00.818 | INFO | Analyzing chunk frames = [5803008,5803520) | 20260508 06:03:00.818 | INFO | MemInfo: { | 20260508 06:03:00.818 | INFO | "gpu" : " 17 GB", | 20260508 06:03:00.818 | INFO | "huge" : "", | 20260508 06:03:00.818 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:00.818 | INFO | "malloc" : " 14 GB", | 20260508 06:03:00.818 | INFO | "pinned" : " 29 GB" | 20260508 06:03:00.818 | INFO | } | 20260508 06:03:00.818 | INFO | Perf report for the last 5.14092 seconds: | 20260508 06:03:00.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:00.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.818 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 06:03:00.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.819 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:03:00.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:00.819 | INFO | Uploaded 36GiB to GPU in 2.42582s (14.8403GiB/s) | 20260508 06:03:00.819 | INFO | Downloaded 34GiB to GPU in 2.4177s (14.0629GiB/s) | 20260508 06:03:05.959 | INFO | Analyzing chunk frames = [5803520,5804032) | 20260508 06:03:05.959 | INFO | MemInfo: { | 20260508 06:03:05.959 | INFO | "gpu" : " 17 GB", | 20260508 06:03:05.959 | INFO | "huge" : "", | 20260508 06:03:05.959 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:05.959 | INFO | "malloc" : " 14 GB", | 20260508 06:03:05.959 | INFO | "pinned" : " 29 GB" | 20260508 06:03:05.959 | INFO | } | 20260508 06:03:05.959 | INFO | Perf report for the last 5.1411 seconds: | 20260508 06:03:05.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:05.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:03:05.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:03:05.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:05.960 | INFO | Uploaded 36GiB to GPU in 2.47208s (14.5626GiB/s) | 20260508 06:03:05.960 | INFO | Downloaded 34GiB to GPU in 2.435s (13.963GiB/s) | 20260508 06:03:06.057 | INFO | Starting a round of Baz IO | 20260508 06:03:09.488 | INFO | Populating RTMetrics report | 20260508 06:03:09.488 | INFO | RTMetrics report populated | 20260508 06:03:09.488 | INFO | Opening tmp RTMetrics json file | 20260508 06:03:09.498 | INFO | Serializing RTMetrics to json | 20260508 06:03:09.499 | INFO | Serialization done, swapping tmp file now... | 20260508 06:03:09.500 | INFO | RTMetrics done | 20260508 06:03:09.558 | INFO | Finished a round of Baz IO: Wrote 1.56 GiB in 3.5 seconds | 20260508 06:03:09.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.00% / 59.23% | 20260508 06:03:09.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.424135% / 0% / 8.34465e-05% | 20260508 06:03:09.558 | INFO | Baz IO overall: | 20260508 06:03:09.558 | INFO | BytesWritten : 2579035237981 | 20260508 06:03:09.558 | INFO | NumEvents : 1243805929397 | 20260508 06:03:11.090 | INFO | Analyzing chunk frames = [5804032,5804544) | 20260508 06:03:11.090 | INFO | MemInfo: { | 20260508 06:03:11.090 | INFO | "gpu" : " 17 GB", | 20260508 06:03:11.090 | INFO | "huge" : "", | 20260508 06:03:11.090 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:11.090 | INFO | "malloc" : " 14 GB", | 20260508 06:03:11.091 | INFO | "pinned" : " 29 GB" | 20260508 06:03:11.091 | INFO | } | 20260508 06:03:11.091 | INFO | Perf report for the last 5.13113 seconds: | 20260508 06:03:11.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:11.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:11.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:11.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:11.091 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 06:03:11.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:11.091 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260508 06:03:11.091 | INFO | PRE_HQ : 66.1%, 1.7, 58.8ms, 0 | 20260508 06:03:11.091 | INFO | BAZWRITER : 67.6%, 2.7, 94.5ms, 0 | 20260508 06:03:11.091 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:03:11.091 | INFO | Uploaded 36GiB to GPU in 2.58971s (13.9012GiB/s) | 20260508 06:03:11.091 | INFO | Downloaded 38GiB to GPU in 2.69947s (14.0768GiB/s) | 20260508 06:03:16.228 | INFO | Analyzing chunk frames = [5804544,5805056) | 20260508 06:03:16.228 | INFO | MemInfo: { | 20260508 06:03:16.228 | INFO | "gpu" : " 17 GB", | 20260508 06:03:16.228 | INFO | "huge" : "", | 20260508 06:03:16.228 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:16.228 | INFO | "malloc" : " 14 GB", | 20260508 06:03:16.228 | INFO | "pinned" : " 29 GB" | 20260508 06:03:16.228 | INFO | } | 20260508 06:03:16.228 | INFO | Perf report for the last 5.1378 seconds: | 20260508 06:03:16.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:16.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.229 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 06:03:16.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.229 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:03:16.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:16.229 | INFO | Uploaded 36GiB to GPU in 2.57591s (13.9757GiB/s) | 20260508 06:03:16.229 | INFO | Downloaded 34GiB to GPU in 2.44888s (13.8839GiB/s) | 20260508 06:03:21.371 | INFO | Analyzing chunk frames = [5805056,5805568) | 20260508 06:03:21.371 | INFO | MemInfo: { | 20260508 06:03:21.371 | INFO | "gpu" : " 17 GB", | 20260508 06:03:21.371 | INFO | "huge" : "", | 20260508 06:03:21.371 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:21.372 | INFO | "malloc" : " 14 GB", | 20260508 06:03:21.372 | INFO | "pinned" : " 29 GB" | 20260508 06:03:21.372 | INFO | } | 20260508 06:03:21.372 | INFO | Perf report for the last 5.14328 seconds: | 20260508 06:03:21.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:21.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:03:21.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:03:21.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:21.372 | INFO | Uploaded 36GiB to GPU in 2.5398s (14.1743GiB/s) | 20260508 06:03:21.372 | INFO | Downloaded 34GiB to GPU in 2.41187s (14.097GiB/s) | 20260508 06:03:26.513 | INFO | Analyzing chunk frames = [5805568,5806080) | 20260508 06:03:26.513 | INFO | MemInfo: { | 20260508 06:03:26.513 | INFO | "gpu" : " 17 GB", | 20260508 06:03:26.513 | INFO | "huge" : "", | 20260508 06:03:26.513 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:26.513 | INFO | "malloc" : " 14 GB", | 20260508 06:03:26.514 | INFO | "pinned" : " 29 GB" | 20260508 06:03:26.514 | INFO | } | 20260508 06:03:26.514 | INFO | Perf report for the last 5.14191 seconds: | 20260508 06:03:26.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:26.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | ANALYSIS : 66.7%, 2.9, 103.3ms, 0 | 20260508 06:03:26.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:03:26.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:26.514 | INFO | Uploaded 36GiB to GPU in 2.5395s (14.176GiB/s) | 20260508 06:03:26.514 | INFO | Downloaded 34GiB to GPU in 2.38844s (14.2352GiB/s) | 20260508 06:03:31.651 | INFO | Analyzing chunk frames = [5806080,5806592) | 20260508 06:03:31.651 | INFO | MemInfo: { | 20260508 06:03:31.651 | INFO | "gpu" : " 17 GB", | 20260508 06:03:31.651 | INFO | "huge" : "", | 20260508 06:03:31.651 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:31.651 | INFO | "malloc" : " 14 GB", | 20260508 06:03:31.651 | INFO | "pinned" : " 29 GB" | 20260508 06:03:31.651 | INFO | } | 20260508 06:03:31.651 | INFO | Perf report for the last 5.13775 seconds: | 20260508 06:03:31.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:31.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.651 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:03:31.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.652 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:03:31.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:31.652 | INFO | Uploaded 36GiB to GPU in 2.29822s (15.6643GiB/s) | 20260508 06:03:31.652 | INFO | Downloaded 34GiB to GPU in 2.47558s (13.7342GiB/s) | 20260508 06:03:36.787 | INFO | Analyzing chunk frames = [5806592,5807104) | 20260508 06:03:36.787 | INFO | MemInfo: { | 20260508 06:03:36.787 | INFO | "gpu" : " 17 GB", | 20260508 06:03:36.787 | INFO | "huge" : "", | 20260508 06:03:36.787 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:36.787 | INFO | "malloc" : " 14 GB", | 20260508 06:03:36.787 | INFO | "pinned" : " 29 GB" | 20260508 06:03:36.787 | INFO | } | 20260508 06:03:36.787 | INFO | Perf report for the last 5.13563 seconds: | 20260508 06:03:36.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:36.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.787 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:03:36.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.787 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:03:36.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:36.788 | INFO | Uploaded 36GiB to GPU in 2.46919s (14.5797GiB/s) | 20260508 06:03:36.788 | INFO | Downloaded 34GiB to GPU in 2.43354s (13.9714GiB/s) | 20260508 06:03:41.926 | INFO | Analyzing chunk frames = [5807104,5807616) | 20260508 06:03:41.926 | INFO | MemInfo: { | 20260508 06:03:41.926 | INFO | "gpu" : " 17 GB", | 20260508 06:03:41.926 | INFO | "huge" : "", | 20260508 06:03:41.926 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:41.926 | INFO | "malloc" : " 14 GB", | 20260508 06:03:41.926 | INFO | "pinned" : " 29 GB" | 20260508 06:03:41.926 | INFO | } | 20260508 06:03:41.926 | INFO | Perf report for the last 5.13912 seconds: | 20260508 06:03:41.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:41.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.926 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:03:41.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.926 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:03:41.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:41.927 | INFO | Uploaded 36GiB to GPU in 2.45767s (14.648GiB/s) | 20260508 06:03:41.927 | INFO | Downloaded 34GiB to GPU in 2.43435s (13.9667GiB/s) | 20260508 06:03:47.071 | INFO | Analyzing chunk frames = [5807616,5808128) | 20260508 06:03:47.071 | INFO | MemInfo: { | 20260508 06:03:47.071 | INFO | "gpu" : " 17 GB", | 20260508 06:03:47.071 | INFO | "huge" : "", | 20260508 06:03:47.071 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:47.071 | INFO | "malloc" : " 14 GB", | 20260508 06:03:47.071 | INFO | "pinned" : " 29 GB" | 20260508 06:03:47.071 | INFO | } | 20260508 06:03:47.072 | INFO | Perf report for the last 5.14528 seconds: | 20260508 06:03:47.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:47.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:03:47.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 06:03:47.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:47.072 | INFO | Uploaded 36GiB to GPU in 2.45919s (14.639GiB/s) | 20260508 06:03:47.072 | INFO | Downloaded 34GiB to GPU in 2.44103s (13.9286GiB/s) | 20260508 06:03:47.072 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:03:47.072 | INFO | Total mem usage: 16.7 GiB | 20260508 06:03:47.072 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:03:47.072 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:03:47.072 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:03:47.072 | INFO | High Water Breakdown: | 20260508 06:03:47.072 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:03:47.072 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:03:47.072 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:03:47.072 | INFO | ------------------------------------------ | 20260508 06:03:47.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:03:47.072 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:03:47.072 | INFO | Total mem usage: 26.5 GiB | 20260508 06:03:47.072 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:03:47.072 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:03:47.072 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:03:47.072 | INFO | High Water Breakdown: | 20260508 06:03:47.073 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:03:47.073 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:03:47.073 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:03:47.073 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:03:47.073 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:03:47.073 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:03:47.073 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:03:47.073 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:03:47.073 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:03:47.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:03:47.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:03:47.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:03:47.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:03:47.073 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:03:47.073 | INFO | ------------------------------------------ | 20260508 06:03:47.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:03:47.073 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:03:47.073 | INFO | Total mem usage: 27.8 GiB | 20260508 06:03:47.073 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:03:47.073 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:03:47.073 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:03:47.073 | INFO | High Water Breakdown: | 20260508 06:03:47.073 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:03:47.073 | INFO | ------------------------------------------ | 20260508 06:03:47.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:03:47.073 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:03:47.073 | INFO | Total mem usage: 15.7 GiB | 20260508 06:03:47.073 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:03:47.073 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:03:47.073 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:03:47.073 | INFO | High Water Breakdown: | 20260508 06:03:47.073 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:03:47.073 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:03:47.073 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:03:47.073 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:03:47.073 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:03:47.073 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:03:47.073 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:03:47.073 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:03:47.073 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:03:47.073 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:03:47.073 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:03:47.073 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:03:47.073 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:03:47.073 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:03:47.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:03:47.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:03:47.073 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:03:47.073 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:03:47.073 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:03:47.073 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:03:47.073 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:03:47.073 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:03:47.073 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:03:47.073 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:03:47.073 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:03:47.073 | INFO | ------------------------------------------ | 20260508 06:03:47.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:03:47.073 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:03:47.073 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:03:47.073 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:03:47.073 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.46e-01 | 20260508 06:03:47.073 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.23e-02 / 5.20e-03 | 20260508 06:03:47.073 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.42e+00, 5.87e-01 / 2.15e-01 | 20260508 06:03:47.074 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.25e-01 | 20260508 06:03:47.074 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.52e+01, 3.76e+00 / 1.09e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.78e+00 / 1.35e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.60e+00, 4.14e-01 / 2.49e-01 | 20260508 06:03:47.074 | INFO | PerfReport: 3.17%, Metrics -- 1.04e+00 / 1.22e+00, 2.55e-01 / 1.18e-01 | 20260508 06:03:47.074 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.16e-01 / 3.50e-01, 4.97e-02 / 2.69e-02 | 20260508 06:03:47.074 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:03:47.074 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:03:47.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:03:47.074 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 24.33%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 2.27%, LockUpload -- 2.41e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 2.67%, LockDownload -- 2.83e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 15.71%, LockCompute -- 1.67e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 06:03:47.074 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260508 06:03:47.186 | INFO | Starting a round of Baz IO | 20260508 06:03:50.594 | INFO | Populating RTMetrics report | 20260508 06:03:50.594 | INFO | RTMetrics report populated | 20260508 06:03:50.594 | INFO | Opening tmp RTMetrics json file | 20260508 06:03:50.607 | INFO | Serializing RTMetrics to json | 20260508 06:03:50.608 | INFO | Serialization done, swapping tmp file now... | 20260508 06:03:50.608 | INFO | RTMetrics done | 20260508 06:03:50.666 | INFO | Finished a round of Baz IO: Wrote 1.56 GiB in 3.48 seconds | 20260508 06:03:50.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.00% / 59.24% | 20260508 06:03:50.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.423288% / 3.97364e-06% / 7.54992e-05% | 20260508 06:03:50.666 | INFO | Baz IO overall: | 20260508 06:03:50.666 | INFO | BytesWritten : 2580709991922 | 20260508 06:03:50.666 | INFO | NumEvents : 1244603814304 | 20260508 06:03:52.214 | INFO | Analyzing chunk frames = [5808128,5808640) | 20260508 06:03:52.214 | INFO | MemInfo: { | 20260508 06:03:52.214 | INFO | "gpu" : " 17 GB", | 20260508 06:03:52.214 | INFO | "huge" : "", | 20260508 06:03:52.214 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:52.214 | INFO | "malloc" : " 14 GB", | 20260508 06:03:52.214 | INFO | "pinned" : " 29 GB" | 20260508 06:03:52.214 | INFO | } | 20260508 06:03:52.214 | INFO | Perf report for the last 5.14266 seconds: | 20260508 06:03:52.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:52.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:52.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:52.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:52.214 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 06:03:52.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:52.214 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 06:03:52.214 | INFO | PRE_HQ : 66.4%, 1.8, 61.9ms, 0 | 20260508 06:03:52.214 | INFO | BAZWRITER : 67.1%, 2.8, 96.4ms, 0 | 20260508 06:03:52.214 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:03:52.215 | INFO | Uploaded 36GiB to GPU in 2.57532s (13.9788GiB/s) | 20260508 06:03:52.215 | INFO | Downloaded 38GiB to GPU in 2.70234s (14.0619GiB/s) | 20260508 06:03:57.354 | INFO | Analyzing chunk frames = [5808640,5809152) | 20260508 06:03:57.354 | INFO | MemInfo: { | 20260508 06:03:57.354 | INFO | "gpu" : " 17 GB", | 20260508 06:03:57.354 | INFO | "huge" : "", | 20260508 06:03:57.354 | INFO | "huge_shared" : " 29 GB", | 20260508 06:03:57.354 | INFO | "malloc" : " 14 GB", | 20260508 06:03:57.354 | INFO | "pinned" : " 29 GB" | 20260508 06:03:57.354 | INFO | } | 20260508 06:03:57.354 | INFO | Perf report for the last 5.13997 seconds: | 20260508 06:03:57.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:03:57.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.354 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 06:03:57.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.354 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:03:57.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:03:57.355 | INFO | Uploaded 36GiB to GPU in 2.45563s (14.6602GiB/s) | 20260508 06:03:57.355 | INFO | Downloaded 34GiB to GPU in 2.4577s (13.8341GiB/s) | 20260508 06:04:02.488 | INFO | Analyzing chunk frames = [5809152,5809664) | 20260508 06:04:02.488 | INFO | MemInfo: { | 20260508 06:04:02.488 | INFO | "gpu" : " 17 GB", | 20260508 06:04:02.488 | INFO | "huge" : "", | 20260508 06:04:02.488 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:02.488 | INFO | "malloc" : " 14 GB", | 20260508 06:04:02.488 | INFO | "pinned" : " 29 GB" | 20260508 06:04:02.488 | INFO | } | 20260508 06:04:02.488 | INFO | Perf report for the last 5.13364 seconds: | 20260508 06:04:02.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:02.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 06:04:02.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:04:02.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:02.488 | INFO | Uploaded 36GiB to GPU in 2.54836s (14.1267GiB/s) | 20260508 06:04:02.488 | INFO | Downloaded 34GiB to GPU in 2.4588s (13.8279GiB/s) | 20260508 06:04:07.630 | INFO | Analyzing chunk frames = [5809664,5810176) | 20260508 06:04:07.630 | INFO | MemInfo: { | 20260508 06:04:07.630 | INFO | "gpu" : " 17 GB", | 20260508 06:04:07.630 | INFO | "huge" : "", | 20260508 06:04:07.630 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:07.630 | INFO | "malloc" : " 14 GB", | 20260508 06:04:07.630 | INFO | "pinned" : " 29 GB" | 20260508 06:04:07.630 | INFO | } | 20260508 06:04:07.631 | INFO | Perf report for the last 5.14274 seconds: | 20260508 06:04:07.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:07.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 06:04:07.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:04:07.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:07.631 | INFO | Uploaded 36GiB to GPU in 2.50881s (14.3494GiB/s) | 20260508 06:04:07.631 | INFO | Downloaded 34GiB to GPU in 2.41864s (14.0575GiB/s) | 20260508 06:04:12.769 | INFO | Analyzing chunk frames = [5810176,5810688) | 20260508 06:04:12.769 | INFO | MemInfo: { | 20260508 06:04:12.769 | INFO | "gpu" : " 17 GB", | 20260508 06:04:12.769 | INFO | "huge" : "", | 20260508 06:04:12.769 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:12.769 | INFO | "malloc" : " 14 GB", | 20260508 06:04:12.769 | INFO | "pinned" : " 29 GB" | 20260508 06:04:12.769 | INFO | } | 20260508 06:04:12.769 | INFO | Perf report for the last 5.13836 seconds: | 20260508 06:04:12.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:12.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:04:12.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:04:12.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:12.769 | INFO | Uploaded 36GiB to GPU in 2.33782s (15.399GiB/s) | 20260508 06:04:12.769 | INFO | Downloaded 34GiB to GPU in 2.51285s (13.5304GiB/s) | 20260508 06:04:17.905 | INFO | Analyzing chunk frames = [5810688,5811200) | 20260508 06:04:17.905 | INFO | MemInfo: { | 20260508 06:04:17.905 | INFO | "gpu" : " 17 GB", | 20260508 06:04:17.905 | INFO | "huge" : "", | 20260508 06:04:17.905 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:17.905 | INFO | "malloc" : " 14 GB", | 20260508 06:04:17.905 | INFO | "pinned" : " 29 GB" | 20260508 06:04:17.905 | INFO | } | 20260508 06:04:17.905 | INFO | Perf report for the last 5.13625 seconds: | 20260508 06:04:17.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:17.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.905 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 06:04:17.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.905 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:04:17.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:17.906 | INFO | Uploaded 36GiB to GPU in 2.41516s (14.9059GiB/s) | 20260508 06:04:17.906 | INFO | Downloaded 34GiB to GPU in 2.42783s (14.0043GiB/s) | 20260508 06:04:23.040 | INFO | Analyzing chunk frames = [5811200,5811712) | 20260508 06:04:23.040 | INFO | MemInfo: { | 20260508 06:04:23.040 | INFO | "gpu" : " 17 GB", | 20260508 06:04:23.040 | INFO | "huge" : "", | 20260508 06:04:23.040 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:23.040 | INFO | "malloc" : " 14 GB", | 20260508 06:04:23.040 | INFO | "pinned" : " 29 GB" | 20260508 06:04:23.040 | INFO | } | 20260508 06:04:23.040 | INFO | Perf report for the last 5.13471 seconds: | 20260508 06:04:23.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:23.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:04:23.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 06:04:23.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:23.040 | INFO | Uploaded 36GiB to GPU in 2.41731s (14.8926GiB/s) | 20260508 06:04:23.040 | INFO | Downloaded 34GiB to GPU in 2.43796s (13.9461GiB/s) | 20260508 06:04:28.186 | INFO | Analyzing chunk frames = [5811712,5812224) | 20260508 06:04:28.186 | INFO | MemInfo: { | 20260508 06:04:28.186 | INFO | "gpu" : " 17 GB", | 20260508 06:04:28.186 | INFO | "huge" : "", | 20260508 06:04:28.186 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:28.186 | INFO | "malloc" : " 14 GB", | 20260508 06:04:28.186 | INFO | "pinned" : " 29 GB" | 20260508 06:04:28.186 | INFO | } | 20260508 06:04:28.186 | INFO | Perf report for the last 5.14646 seconds: | 20260508 06:04:28.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:28.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.186 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:04:28.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.187 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:04:28.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:28.187 | INFO | Uploaded 36GiB to GPU in 2.45009s (14.6933GiB/s) | 20260508 06:04:28.187 | INFO | Downloaded 34GiB to GPU in 2.43352s (13.9715GiB/s) | 20260508 06:04:28.284 | INFO | Starting a round of Baz IO | 20260508 06:04:31.717 | INFO | Populating RTMetrics report | 20260508 06:04:31.718 | INFO | RTMetrics report populated | 20260508 06:04:31.718 | INFO | Opening tmp RTMetrics json file | 20260508 06:04:31.729 | INFO | Serializing RTMetrics to json | 20260508 06:04:31.730 | INFO | Serialization done, swapping tmp file now... | 20260508 06:04:31.731 | INFO | RTMetrics done | 20260508 06:04:31.786 | INFO | Finished a round of Baz IO: Wrote 1.559 GiB in 3.502 seconds | 20260508 06:04:31.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.00% / 59.24% | 20260508 06:04:31.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.422629% / 0% / 5.16574e-05% | 20260508 06:04:31.786 | INFO | Baz IO overall: | 20260508 06:04:31.786 | INFO | BytesWritten : 2582384409659 | 20260508 06:04:31.786 | INFO | NumEvents : 1245401556417 | 20260508 06:04:33.322 | INFO | Analyzing chunk frames = [5812224,5812736) | 20260508 06:04:33.322 | INFO | MemInfo: { | 20260508 06:04:33.322 | INFO | "gpu" : " 17 GB", | 20260508 06:04:33.322 | INFO | "huge" : "", | 20260508 06:04:33.322 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:33.322 | INFO | "malloc" : " 14 GB", | 20260508 06:04:33.322 | INFO | "pinned" : " 29 GB" | 20260508 06:04:33.322 | INFO | } | 20260508 06:04:33.322 | INFO | Perf report for the last 5.13576 seconds: | 20260508 06:04:33.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:33.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:33.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:33.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:33.322 | INFO | ANALYSIS : 68.0%, 3.0, 106.1ms, 0 | 20260508 06:04:33.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:33.322 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:04:33.322 | INFO | PRE_HQ : 65.9%, 1.6, 55.0ms, 0 | 20260508 06:04:33.322 | INFO | BAZWRITER : 67.5%, 2.6, 90.6ms, 0 | 20260508 06:04:33.322 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:04:33.322 | INFO | Uploaded 36GiB to GPU in 2.56582s (14.0306GiB/s) | 20260508 06:04:33.323 | INFO | Downloaded 38GiB to GPU in 2.6978s (14.0855GiB/s) | 20260508 06:04:38.464 | INFO | Analyzing chunk frames = [5812736,5813248) | 20260508 06:04:38.464 | INFO | MemInfo: { | 20260508 06:04:38.464 | INFO | "gpu" : " 17 GB", | 20260508 06:04:38.464 | INFO | "huge" : "", | 20260508 06:04:38.464 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:38.464 | INFO | "malloc" : " 14 GB", | 20260508 06:04:38.464 | INFO | "pinned" : " 29 GB" | 20260508 06:04:38.464 | INFO | } | 20260508 06:04:38.464 | INFO | Perf report for the last 5.14222 seconds: | 20260508 06:04:38.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:38.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.464 | INFO | ANALYSIS : 67.4%, 3.0, 105.0ms, 0 | 20260508 06:04:38.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.465 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:04:38.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:38.465 | INFO | Uploaded 36GiB to GPU in 2.58905s (13.9047GiB/s) | 20260508 06:04:38.465 | INFO | Downloaded 34GiB to GPU in 2.43405s (13.9685GiB/s) | 20260508 06:04:43.600 | INFO | Analyzing chunk frames = [5813248,5813760) | 20260508 06:04:43.600 | INFO | MemInfo: { | 20260508 06:04:43.600 | INFO | "gpu" : " 17 GB", | 20260508 06:04:43.600 | INFO | "huge" : "", | 20260508 06:04:43.600 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:43.600 | INFO | "malloc" : " 14 GB", | 20260508 06:04:43.600 | INFO | "pinned" : " 29 GB" | 20260508 06:04:43.600 | INFO | } | 20260508 06:04:43.600 | INFO | Perf report for the last 5.13554 seconds: | 20260508 06:04:43.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:43.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.600 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 06:04:43.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.600 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:04:43.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:43.601 | INFO | Uploaded 36GiB to GPU in 2.55702s (14.0789GiB/s) | 20260508 06:04:43.601 | INFO | Downloaded 34GiB to GPU in 2.43202s (13.9801GiB/s) | 20260508 06:04:48.741 | INFO | Analyzing chunk frames = [5813760,5814272) | 20260508 06:04:48.741 | INFO | MemInfo: { | 20260508 06:04:48.741 | INFO | "gpu" : " 17 GB", | 20260508 06:04:48.741 | INFO | "huge" : "", | 20260508 06:04:48.741 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:48.741 | INFO | "malloc" : " 14 GB", | 20260508 06:04:48.741 | INFO | "pinned" : " 29 GB" | 20260508 06:04:48.741 | INFO | } | 20260508 06:04:48.741 | INFO | Perf report for the last 5.14143 seconds: | 20260508 06:04:48.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:48.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.741 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 06:04:48.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.742 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:04:48.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:48.742 | INFO | Uploaded 36GiB to GPU in 2.56408s (14.0401GiB/s) | 20260508 06:04:48.742 | INFO | Downloaded 34GiB to GPU in 2.3842s (14.2605GiB/s) | 20260508 06:04:53.883 | INFO | Analyzing chunk frames = [5814272,5814784) | 20260508 06:04:53.883 | INFO | MemInfo: { | 20260508 06:04:53.883 | INFO | "gpu" : " 17 GB", | 20260508 06:04:53.883 | INFO | "huge" : "", | 20260508 06:04:53.883 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:53.883 | INFO | "malloc" : " 14 GB", | 20260508 06:04:53.883 | INFO | "pinned" : " 29 GB" | 20260508 06:04:53.883 | INFO | } | 20260508 06:04:53.883 | INFO | Perf report for the last 5.1418 seconds: | 20260508 06:04:53.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:53.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:04:53.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 06:04:53.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:53.883 | INFO | Uploaded 36GiB to GPU in 2.34232s (15.3693GiB/s) | 20260508 06:04:53.884 | INFO | Downloaded 34GiB to GPU in 2.50355s (13.5807GiB/s) | 20260508 06:04:59.024 | INFO | Analyzing chunk frames = [5814784,5815296) | 20260508 06:04:59.024 | INFO | MemInfo: { | 20260508 06:04:59.024 | INFO | "gpu" : " 17 GB", | 20260508 06:04:59.024 | INFO | "huge" : "", | 20260508 06:04:59.024 | INFO | "huge_shared" : " 29 GB", | 20260508 06:04:59.025 | INFO | "malloc" : " 14 GB", | 20260508 06:04:59.025 | INFO | "pinned" : " 29 GB" | 20260508 06:04:59.025 | INFO | } | 20260508 06:04:59.025 | INFO | Perf report for the last 5.1416 seconds: | 20260508 06:04:59.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:04:59.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:04:59.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 06:04:59.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:04:59.025 | INFO | Uploaded 36GiB to GPU in 2.49796s (14.4117GiB/s) | 20260508 06:04:59.025 | INFO | Downloaded 34GiB to GPU in 2.43563s (13.9594GiB/s) | 20260508 06:05:04.163 | INFO | Analyzing chunk frames = [5815296,5815808) | 20260508 06:05:04.163 | INFO | MemInfo: { | 20260508 06:05:04.163 | INFO | "gpu" : " 17 GB", | 20260508 06:05:04.163 | INFO | "huge" : "", | 20260508 06:05:04.163 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:04.163 | INFO | "malloc" : " 14 GB", | 20260508 06:05:04.163 | INFO | "pinned" : " 29 GB" | 20260508 06:05:04.163 | INFO | } | 20260508 06:05:04.163 | INFO | Perf report for the last 5.13858 seconds: | 20260508 06:05:04.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:04.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.163 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 06:05:04.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.164 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 06:05:04.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:04.164 | INFO | Uploaded 36GiB to GPU in 2.51171s (14.3329GiB/s) | 20260508 06:05:04.164 | INFO | Downloaded 34GiB to GPU in 2.43274s (13.976GiB/s) | 20260508 06:05:09.295 | INFO | Analyzing chunk frames = [5815808,5816320) | 20260508 06:05:09.295 | INFO | MemInfo: { | 20260508 06:05:09.295 | INFO | "gpu" : " 17 GB", | 20260508 06:05:09.295 | INFO | "huge" : "", | 20260508 06:05:09.295 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:09.295 | INFO | "malloc" : " 14 GB", | 20260508 06:05:09.295 | INFO | "pinned" : " 29 GB" | 20260508 06:05:09.295 | INFO | } | 20260508 06:05:09.295 | INFO | Perf report for the last 5.13148 seconds: | 20260508 06:05:09.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:09.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 06:05:09.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:05:09.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:09.296 | INFO | Uploaded 36GiB to GPU in 2.48507s (14.4865GiB/s) | 20260508 06:05:09.296 | INFO | Downloaded 34GiB to GPU in 2.43456s (13.9656GiB/s) | 20260508 06:05:09.296 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:05:09.296 | INFO | Total mem usage: 16.7 GiB | 20260508 06:05:09.296 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:05:09.296 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:05:09.296 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:05:09.296 | INFO | High Water Breakdown: | 20260508 06:05:09.296 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:05:09.296 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:05:09.296 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:05:09.296 | INFO | ------------------------------------------ | 20260508 06:05:09.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:05:09.296 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:05:09.296 | INFO | Total mem usage: 26.5 GiB | 20260508 06:05:09.296 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:05:09.296 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:05:09.296 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:05:09.296 | INFO | High Water Breakdown: | 20260508 06:05:09.296 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:05:09.296 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:05:09.296 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:05:09.296 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:05:09.296 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:05:09.296 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:05:09.296 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:05:09.296 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:05:09.296 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:05:09.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:05:09.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:05:09.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:05:09.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:05:09.296 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:05:09.296 | INFO | ------------------------------------------ | 20260508 06:05:09.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:05:09.296 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:05:09.296 | INFO | Total mem usage: 27.8 GiB | 20260508 06:05:09.296 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:05:09.296 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:05:09.296 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:05:09.296 | INFO | High Water Breakdown: | 20260508 06:05:09.296 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:05:09.296 | INFO | ------------------------------------------ | 20260508 06:05:09.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:05:09.296 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:05:09.296 | INFO | Total mem usage: 15.7 GiB | 20260508 06:05:09.296 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:05:09.296 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:05:09.296 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:05:09.296 | INFO | High Water Breakdown: | 20260508 06:05:09.296 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:05:09.296 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:05:09.296 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:05:09.296 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:05:09.296 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:05:09.296 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:05:09.296 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:05:09.296 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:05:09.296 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:05:09.296 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:05:09.296 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:05:09.296 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:05:09.296 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:05:09.296 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:05:09.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:05:09.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:05:09.296 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:05:09.296 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:05:09.296 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:05:09.296 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:05:09.297 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:05:09.297 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:05:09.297 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:05:09.297 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:05:09.297 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:05:09.297 | INFO | ------------------------------------------ | 20260508 06:05:09.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:05:09.297 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:05:09.297 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:05:09.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:05:09.297 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.40e+01 / 6.18e-01 | 20260508 06:05:09.297 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.06e-01, 1.17e-02 / 5.71e-03 | 20260508 06:05:09.297 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.39e+00, 5.87e-01 / 2.13e-01 | 20260508 06:05:09.297 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.42e+00, 2.07e-01 / 1.25e-01 | 20260508 06:05:09.297 | INFO | PerfReport: 15.91%, DME -- 4.15e+01 / 4.52e+01, 3.76e+00 / 1.20e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.75e+00 / 1.42e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.56e+00, 4.16e-01 / 2.24e-01 | 20260508 06:05:09.297 | INFO | PerfReport: 3.17%, Metrics -- 1.03e+00 / 1.25e+00, 2.55e-01 / 1.08e-01 | 20260508 06:05:09.297 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.16e-01 / 3.56e-01, 5.01e-02 / 2.92e-02 | 20260508 06:05:09.297 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:05:09.297 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:05:09.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:05:09.297 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 24.43%, Upload -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 24.30%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 30.76%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 2.31%, LockUpload -- 2.45e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 2.61%, LockDownload -- 2.77e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:05:09.297 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260508 06:05:09.392 | INFO | Starting a round of Baz IO | 20260508 06:05:12.828 | INFO | Populating RTMetrics report | 20260508 06:05:12.829 | INFO | RTMetrics report populated | 20260508 06:05:12.829 | INFO | Opening tmp RTMetrics json file | 20260508 06:05:12.839 | INFO | Serializing RTMetrics to json | 20260508 06:05:12.840 | INFO | Serialization done, swapping tmp file now... | 20260508 06:05:12.841 | INFO | RTMetrics done | 20260508 06:05:12.896 | INFO | Finished a round of Baz IO: Wrote 1.559 GiB in 3.503 seconds | 20260508 06:05:12.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.00% / 59.24% | 20260508 06:05:12.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.420713% / 3.97364e-06% / 5.5631e-05% | 20260508 06:05:12.896 | INFO | Baz IO overall: | 20260508 06:05:12.896 | INFO | BytesWritten : 2584057946211 | 20260508 06:05:12.896 | INFO | NumEvents : 1246198658999 | 20260508 06:05:14.442 | INFO | Analyzing chunk frames = [5816320,5816832) | 20260508 06:05:14.442 | INFO | MemInfo: { | 20260508 06:05:14.442 | INFO | "gpu" : " 17 GB", | 20260508 06:05:14.442 | INFO | "huge" : "", | 20260508 06:05:14.442 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:14.442 | INFO | "malloc" : " 14 GB", | 20260508 06:05:14.442 | INFO | "pinned" : " 29 GB" | 20260508 06:05:14.442 | INFO | } | 20260508 06:05:14.442 | INFO | Perf report for the last 5.14709 seconds: | 20260508 06:05:14.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:14.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:14.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:14.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:14.442 | INFO | ANALYSIS : 67.7%, 3.0, 106.1ms, 0 | 20260508 06:05:14.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:14.442 | INFO | RT_METRICS : 39.5%, 1.0, 20.8ms, 0 | 20260508 06:05:14.442 | INFO | PRE_HQ : 65.4%, 1.6, 55.0ms, 0 | 20260508 06:05:14.442 | INFO | BAZWRITER : 67.6%, 2.6, 90.1ms, 0 | 20260508 06:05:14.442 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 06:05:14.443 | INFO | Uploaded 36GiB to GPU in 2.57074s (14.0037GiB/s) | 20260508 06:05:14.443 | INFO | Downloaded 38GiB to GPU in 2.6879s (14.1374GiB/s) | 20260508 06:05:19.580 | INFO | Analyzing chunk frames = [5816832,5817344) | 20260508 06:05:19.580 | INFO | MemInfo: { | 20260508 06:05:19.580 | INFO | "gpu" : " 17 GB", | 20260508 06:05:19.580 | INFO | "huge" : "", | 20260508 06:05:19.580 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:19.580 | INFO | "malloc" : " 14 GB", | 20260508 06:05:19.580 | INFO | "pinned" : " 29 GB" | 20260508 06:05:19.580 | INFO | } | 20260508 06:05:19.580 | INFO | Perf report for the last 5.13833 seconds: | 20260508 06:05:19.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:19.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.580 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:05:19.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.580 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:05:19.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:19.581 | INFO | Uploaded 36GiB to GPU in 2.51658s (14.3051GiB/s) | 20260508 06:05:19.581 | INFO | Downloaded 34GiB to GPU in 2.43147s (13.9833GiB/s) | 20260508 06:05:24.717 | INFO | Analyzing chunk frames = [5817344,5817856) | 20260508 06:05:24.717 | INFO | MemInfo: { | 20260508 06:05:24.717 | INFO | "gpu" : " 17 GB", | 20260508 06:05:24.717 | INFO | "huge" : "", | 20260508 06:05:24.717 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:24.717 | INFO | "malloc" : " 14 GB", | 20260508 06:05:24.717 | INFO | "pinned" : " 29 GB" | 20260508 06:05:24.717 | INFO | } | 20260508 06:05:24.717 | INFO | Perf report for the last 5.13695 seconds: | 20260508 06:05:24.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:24.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.717 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:05:24.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.717 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:05:24.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:24.718 | INFO | Uploaded 36GiB to GPU in 2.55609s (14.084GiB/s) | 20260508 06:05:24.718 | INFO | Downloaded 34GiB to GPU in 2.40077s (14.1621GiB/s) | 20260508 06:05:29.851 | INFO | Analyzing chunk frames = [5817856,5818368) | 20260508 06:05:29.851 | INFO | MemInfo: { | 20260508 06:05:29.851 | INFO | "gpu" : " 17 GB", | 20260508 06:05:29.851 | INFO | "huge" : "", | 20260508 06:05:29.851 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:29.851 | INFO | "malloc" : " 14 GB", | 20260508 06:05:29.851 | INFO | "pinned" : " 29 GB" | 20260508 06:05:29.851 | INFO | } | 20260508 06:05:29.851 | INFO | Perf report for the last 5.13373 seconds: | 20260508 06:05:29.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:29.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.851 | INFO | ANALYSIS : 67.0%, 3.0, 103.5ms, 0 | 20260508 06:05:29.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.851 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:05:29.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:29.852 | INFO | Uploaded 36GiB to GPU in 2.57579s (13.9763GiB/s) | 20260508 06:05:29.852 | INFO | Downloaded 34GiB to GPU in 2.38671s (14.2455GiB/s) | 20260508 06:05:34.993 | INFO | Analyzing chunk frames = [5818368,5818880) | 20260508 06:05:34.993 | INFO | MemInfo: { | 20260508 06:05:34.993 | INFO | "gpu" : " 17 GB", | 20260508 06:05:34.993 | INFO | "huge" : "", | 20260508 06:05:34.993 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:34.993 | INFO | "malloc" : " 14 GB", | 20260508 06:05:34.993 | INFO | "pinned" : " 29 GB" | 20260508 06:05:34.993 | INFO | } | 20260508 06:05:34.993 | INFO | Perf report for the last 5.14225 seconds: | 20260508 06:05:34.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:34.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.993 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:05:34.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.993 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:05:34.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:34.994 | INFO | Uploaded 36GiB to GPU in 2.40681s (14.9576GiB/s) | 20260508 06:05:34.994 | INFO | Downloaded 34GiB to GPU in 2.55438s (13.3105GiB/s) | 20260508 06:05:40.134 | INFO | Analyzing chunk frames = [5818880,5819392) | 20260508 06:05:40.134 | INFO | MemInfo: { | 20260508 06:05:40.134 | INFO | "gpu" : " 17 GB", | 20260508 06:05:40.134 | INFO | "huge" : "", | 20260508 06:05:40.135 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:40.135 | INFO | "malloc" : " 14 GB", | 20260508 06:05:40.135 | INFO | "pinned" : " 29 GB" | 20260508 06:05:40.135 | INFO | } | 20260508 06:05:40.135 | INFO | Perf report for the last 5.14134 seconds: | 20260508 06:05:40.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:40.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 06:05:40.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260508 06:05:40.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:40.135 | INFO | Uploaded 36GiB to GPU in 2.46044s (14.6315GiB/s) | 20260508 06:05:40.135 | INFO | Downloaded 34GiB to GPU in 2.43226s (13.9788GiB/s) | 20260508 06:05:45.274 | INFO | Analyzing chunk frames = [5819392,5819904) | 20260508 06:05:45.275 | INFO | MemInfo: { | 20260508 06:05:45.275 | INFO | "gpu" : " 17 GB", | 20260508 06:05:45.275 | INFO | "huge" : "", | 20260508 06:05:45.275 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:45.275 | INFO | "malloc" : " 14 GB", | 20260508 06:05:45.275 | INFO | "pinned" : " 29 GB" | 20260508 06:05:45.275 | INFO | } | 20260508 06:05:45.275 | INFO | Perf report for the last 5.14008 seconds: | 20260508 06:05:45.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:45.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:05:45.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 06:05:45.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:45.275 | INFO | Uploaded 36GiB to GPU in 2.4618s (14.6234GiB/s) | 20260508 06:05:45.275 | INFO | Downloaded 34GiB to GPU in 2.43318s (13.9735GiB/s) | 20260508 06:05:50.418 | INFO | Analyzing chunk frames = [5819904,5820416) | 20260508 06:05:50.418 | INFO | MemInfo: { | 20260508 06:05:50.418 | INFO | "gpu" : " 17 GB", | 20260508 06:05:50.418 | INFO | "huge" : "", | 20260508 06:05:50.418 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:50.418 | INFO | "malloc" : " 14 GB", | 20260508 06:05:50.418 | INFO | "pinned" : " 29 GB" | 20260508 06:05:50.418 | INFO | } | 20260508 06:05:50.418 | INFO | Perf report for the last 5.14361 seconds: | 20260508 06:05:50.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:50.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.419 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:05:50.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.419 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 06:05:50.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:50.419 | INFO | Uploaded 36GiB to GPU in 2.48267s (14.5005GiB/s) | 20260508 06:05:50.419 | INFO | Downloaded 34GiB to GPU in 2.44713s (13.8938GiB/s) | 20260508 06:05:50.532 | INFO | Starting a round of Baz IO | 20260508 06:05:53.934 | INFO | Populating RTMetrics report | 20260508 06:05:53.935 | INFO | RTMetrics report populated | 20260508 06:05:53.935 | INFO | Opening tmp RTMetrics json file | 20260508 06:05:53.945 | INFO | Serializing RTMetrics to json | 20260508 06:05:53.948 | INFO | Serialization done, swapping tmp file now... | 20260508 06:05:53.948 | INFO | RTMetrics done | 20260508 06:05:54.008 | INFO | Finished a round of Baz IO: Wrote 1.56 GiB in 3.475 seconds | 20260508 06:05:54.008 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.00% / 59.24% | 20260508 06:05:54.008 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.421937% / 3.97364e-06% / 3.97364e-05% | 20260508 06:05:54.008 | INFO | Baz IO overall: | 20260508 06:05:54.008 | INFO | BytesWritten : 2585732807042 | 20260508 06:05:54.008 | INFO | NumEvents : 1246996809744 | 20260508 06:05:55.550 | INFO | Analyzing chunk frames = [5820416,5820928) | 20260508 06:05:55.550 | INFO | MemInfo: { | 20260508 06:05:55.550 | INFO | "gpu" : " 17 GB", | 20260508 06:05:55.550 | INFO | "huge" : "", | 20260508 06:05:55.550 | INFO | "huge_shared" : " 29 GB", | 20260508 06:05:55.550 | INFO | "malloc" : " 14 GB", | 20260508 06:05:55.550 | INFO | "pinned" : " 29 GB" | 20260508 06:05:55.550 | INFO | } | 20260508 06:05:55.550 | INFO | Perf report for the last 5.1314 seconds: | 20260508 06:05:55.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:05:55.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:55.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:55.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:55.550 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 06:05:55.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:05:55.550 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 06:05:55.550 | INFO | PRE_HQ : 66.3%, 1.8, 61.3ms, 0 | 20260508 06:05:55.550 | INFO | BAZWRITER : 67.1%, 2.8, 95.6ms, 0 | 20260508 06:05:55.550 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:05:55.550 | INFO | Uploaded 36GiB to GPU in 2.56427s (14.0391GiB/s) | 20260508 06:05:55.550 | INFO | Downloaded 38GiB to GPU in 2.69917s (14.0784GiB/s) | 20260508 06:06:00.691 | INFO | Analyzing chunk frames = [5820928,5821440) | 20260508 06:06:00.691 | INFO | MemInfo: { | 20260508 06:06:00.691 | INFO | "gpu" : " 17 GB", | 20260508 06:06:00.691 | INFO | "huge" : "", | 20260508 06:06:00.691 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:00.691 | INFO | "malloc" : " 14 GB", | 20260508 06:06:00.691 | INFO | "pinned" : " 29 GB" | 20260508 06:06:00.691 | INFO | } | 20260508 06:06:00.691 | INFO | Perf report for the last 5.14096 seconds: | 20260508 06:06:00.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:00.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:06:00.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:06:00.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:00.691 | INFO | Uploaded 36GiB to GPU in 2.46989s (14.5756GiB/s) | 20260508 06:06:00.691 | INFO | Downloaded 34GiB to GPU in 2.44452s (13.9087GiB/s) | 20260508 06:06:05.834 | INFO | Analyzing chunk frames = [5821440,5821952) | 20260508 06:06:05.834 | INFO | MemInfo: { | 20260508 06:06:05.834 | INFO | "gpu" : " 17 GB", | 20260508 06:06:05.834 | INFO | "huge" : "", | 20260508 06:06:05.834 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:05.834 | INFO | "malloc" : " 14 GB", | 20260508 06:06:05.834 | INFO | "pinned" : " 29 GB" | 20260508 06:06:05.834 | INFO | } | 20260508 06:06:05.834 | INFO | Perf report for the last 5.14313 seconds: | 20260508 06:06:05.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:05.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 06:06:05.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:06:05.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:05.834 | INFO | Uploaded 36GiB to GPU in 2.4969s (14.4179GiB/s) | 20260508 06:06:05.834 | INFO | Downloaded 34GiB to GPU in 2.42515s (14.0197GiB/s) | 20260508 06:06:10.970 | INFO | Analyzing chunk frames = [5821952,5822464) | 20260508 06:06:10.970 | INFO | MemInfo: { | 20260508 06:06:10.970 | INFO | "gpu" : " 17 GB", | 20260508 06:06:10.970 | INFO | "huge" : "", | 20260508 06:06:10.970 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:10.970 | INFO | "malloc" : " 14 GB", | 20260508 06:06:10.970 | INFO | "pinned" : " 29 GB" | 20260508 06:06:10.970 | INFO | } | 20260508 06:06:10.970 | INFO | Perf report for the last 5.13625 seconds: | 20260508 06:06:10.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:10.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | ANALYSIS : 66.6%, 3.0, 103.0ms, 0 | 20260508 06:06:10.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:06:10.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:10.970 | INFO | Uploaded 36GiB to GPU in 2.51545s (14.3115GiB/s) | 20260508 06:06:10.970 | INFO | Downloaded 34GiB to GPU in 2.41987s (14.0503GiB/s) | 20260508 06:06:16.109 | INFO | Analyzing chunk frames = [5822464,5822976) | 20260508 06:06:16.109 | INFO | MemInfo: { | 20260508 06:06:16.109 | INFO | "gpu" : " 17 GB", | 20260508 06:06:16.109 | INFO | "huge" : "", | 20260508 06:06:16.109 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:16.109 | INFO | "malloc" : " 14 GB", | 20260508 06:06:16.109 | INFO | "pinned" : " 29 GB" | 20260508 06:06:16.109 | INFO | } | 20260508 06:06:16.109 | INFO | Perf report for the last 5.13892 seconds: | 20260508 06:06:16.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:16.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.109 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 06:06:16.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.109 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:06:16.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:16.110 | INFO | Uploaded 36GiB to GPU in 2.28955s (15.7236GiB/s) | 20260508 06:06:16.110 | INFO | Downloaded 34GiB to GPU in 2.49741s (13.6141GiB/s) | 20260508 06:06:21.250 | INFO | Analyzing chunk frames = [5822976,5823488) | 20260508 06:06:21.250 | INFO | MemInfo: { | 20260508 06:06:21.250 | INFO | "gpu" : " 17 GB", | 20260508 06:06:21.250 | INFO | "huge" : "", | 20260508 06:06:21.250 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:21.250 | INFO | "malloc" : " 14 GB", | 20260508 06:06:21.250 | INFO | "pinned" : " 29 GB" | 20260508 06:06:21.250 | INFO | } | 20260508 06:06:21.250 | INFO | Perf report for the last 5.14075 seconds: | 20260508 06:06:21.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:21.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 06:06:21.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:06:21.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:21.250 | INFO | Uploaded 36GiB to GPU in 2.41716s (14.8935GiB/s) | 20260508 06:06:21.250 | INFO | Downloaded 34GiB to GPU in 2.45357s (13.8574GiB/s) | 20260508 06:06:26.393 | INFO | Analyzing chunk frames = [5823488,5824000) | 20260508 06:06:26.393 | INFO | MemInfo: { | 20260508 06:06:26.393 | INFO | "gpu" : " 17 GB", | 20260508 06:06:26.393 | INFO | "huge" : "", | 20260508 06:06:26.393 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:26.394 | INFO | "malloc" : " 14 GB", | 20260508 06:06:26.394 | INFO | "pinned" : " 29 GB" | 20260508 06:06:26.394 | INFO | } | 20260508 06:06:26.394 | INFO | Perf report for the last 5.14379 seconds: | 20260508 06:06:26.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:26.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.394 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 06:06:26.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.394 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:06:26.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:26.395 | INFO | Uploaded 36GiB to GPU in 2.41178s (14.9267GiB/s) | 20260508 06:06:26.395 | INFO | Downloaded 34GiB to GPU in 2.44101s (13.9287GiB/s) | 20260508 06:06:31.530 | INFO | Analyzing chunk frames = [5824000,5824512) | 20260508 06:06:31.530 | INFO | MemInfo: { | 20260508 06:06:31.530 | INFO | "gpu" : " 17 GB", | 20260508 06:06:31.530 | INFO | "huge" : "", | 20260508 06:06:31.530 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:31.530 | INFO | "malloc" : " 14 GB", | 20260508 06:06:31.530 | INFO | "pinned" : " 29 GB" | 20260508 06:06:31.530 | INFO | } | 20260508 06:06:31.530 | INFO | Perf report for the last 5.13663 seconds: | 20260508 06:06:31.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:31.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.530 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 06:06:31.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.531 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 06:06:31.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:31.531 | INFO | Uploaded 36GiB to GPU in 2.47008s (14.5744GiB/s) | 20260508 06:06:31.531 | INFO | Downloaded 34GiB to GPU in 2.46643s (13.7851GiB/s) | 20260508 06:06:31.531 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:06:31.531 | INFO | Total mem usage: 16.7 GiB | 20260508 06:06:31.531 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:06:31.531 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:06:31.531 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:06:31.531 | INFO | High Water Breakdown: | 20260508 06:06:31.531 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:06:31.531 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:06:31.531 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:06:31.531 | INFO | ------------------------------------------ | 20260508 06:06:31.531 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:06:31.531 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:06:31.531 | INFO | Total mem usage: 26.5 GiB | 20260508 06:06:31.531 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:06:31.531 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:06:31.531 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:06:31.531 | INFO | High Water Breakdown: | 20260508 06:06:31.531 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:06:31.532 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:06:31.532 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:06:31.532 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:06:31.532 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:06:31.532 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:06:31.532 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:06:31.532 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:06:31.532 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:06:31.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:06:31.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:06:31.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:06:31.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:06:31.532 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:06:31.532 | INFO | ------------------------------------------ | 20260508 06:06:31.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:06:31.532 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:06:31.532 | INFO | Total mem usage: 27.8 GiB | 20260508 06:06:31.532 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:06:31.532 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:06:31.532 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:06:31.532 | INFO | High Water Breakdown: | 20260508 06:06:31.532 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:06:31.532 | INFO | ------------------------------------------ | 20260508 06:06:31.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:06:31.532 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:06:31.532 | INFO | Total mem usage: 15.7 GiB | 20260508 06:06:31.532 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:06:31.532 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:06:31.532 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:06:31.532 | INFO | High Water Breakdown: | 20260508 06:06:31.532 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:06:31.532 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:06:31.532 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:06:31.532 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:06:31.532 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:06:31.532 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:06:31.532 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:06:31.532 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:06:31.532 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:06:31.532 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:06:31.532 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:06:31.532 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:06:31.532 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:06:31.532 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:06:31.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:06:31.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:06:31.532 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:06:31.532 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:06:31.532 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:06:31.532 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:06:31.532 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:06:31.532 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:06:31.532 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:06:31.532 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:06:31.532 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:06:31.532 | INFO | ------------------------------------------ | 20260508 06:06:31.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:06:31.532 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:06:31.532 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:06:31.532 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:06:31.532 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.40e+01 / 1.49e+00 | 20260508 06:06:31.532 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.05e-01, 1.19e-02 / 7.21e-03 | 20260508 06:06:31.532 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.50e+00, 5.85e-01 / 2.10e-01 | 20260508 06:06:31.532 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.47e+00, 2.09e-01 / 1.27e-01 | 20260508 06:06:31.532 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.54e+01, 3.94e+00 / 1.87e+00 | 20260508 06:06:31.532 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.78e+00 / 1.37e+00 | 20260508 06:06:31.532 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.39e+00 / 2.62e+00, 4.17e-01 / 2.51e-01 | 20260508 06:06:31.532 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.22e+00, 2.54e-01 / 1.26e-01 | 20260508 06:06:31.533 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.18e-01 / 3.57e-01, 5.02e-02 / 2.40e-02 | 20260508 06:06:31.533 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:06:31.533 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:06:31.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:06:31.533 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 24.34%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 24.32%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 2.17%, LockUpload -- 2.31e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 2.69%, LockDownload -- 2.85e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:06:31.533 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260508 06:06:31.645 | INFO | Starting a round of Baz IO | 20260508 06:06:35.060 | INFO | Populating RTMetrics report | 20260508 06:06:35.061 | INFO | RTMetrics report populated | 20260508 06:06:35.061 | INFO | Opening tmp RTMetrics json file | 20260508 06:06:35.072 | INFO | Serializing RTMetrics to json | 20260508 06:06:35.074 | INFO | Serialization done, swapping tmp file now... | 20260508 06:06:35.074 | INFO | RTMetrics done | 20260508 06:06:35.128 | INFO | Finished a round of Baz IO: Wrote 1.56 GiB in 3.483 seconds | 20260508 06:06:35.128 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.00% / 59.24% | 20260508 06:06:35.128 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.420304% / 1.19209e-05% / 5.16574e-05% | 20260508 06:06:35.128 | INFO | Baz IO overall: | 20260508 06:06:35.129 | INFO | BytesWritten : 2587407633837 | 20260508 06:06:35.129 | INFO | NumEvents : 1247794958551 | 20260508 06:06:36.668 | INFO | Analyzing chunk frames = [5824512,5825024) | 20260508 06:06:36.668 | INFO | MemInfo: { | 20260508 06:06:36.668 | INFO | "gpu" : " 17 GB", | 20260508 06:06:36.668 | INFO | "huge" : "", | 20260508 06:06:36.668 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:36.668 | INFO | "malloc" : " 14 GB", | 20260508 06:06:36.668 | INFO | "pinned" : " 29 GB" | 20260508 06:06:36.668 | INFO | } | 20260508 06:06:36.668 | INFO | Perf report for the last 5.13745 seconds: | 20260508 06:06:36.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:36.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:36.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:36.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:36.668 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 06:06:36.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:36.668 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 06:06:36.668 | INFO | PRE_HQ : 66.3%, 1.8, 62.4ms, 0 | 20260508 06:06:36.668 | INFO | BAZWRITER : 67.2%, 2.8, 96.2ms, 0 | 20260508 06:06:36.668 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 06:06:36.668 | INFO | Uploaded 36GiB to GPU in 2.53608s (14.1952GiB/s) | 20260508 06:06:36.668 | INFO | Downloaded 38GiB to GPU in 2.67785s (14.1905GiB/s) | 20260508 06:06:41.802 | INFO | Analyzing chunk frames = [5825024,5825536) | 20260508 06:06:41.802 | INFO | MemInfo: { | 20260508 06:06:41.802 | INFO | "gpu" : " 17 GB", | 20260508 06:06:41.802 | INFO | "huge" : "", | 20260508 06:06:41.802 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:41.802 | INFO | "malloc" : " 14 GB", | 20260508 06:06:41.802 | INFO | "pinned" : " 29 GB" | 20260508 06:06:41.802 | INFO | } | 20260508 06:06:41.802 | INFO | Perf report for the last 5.13431 seconds: | 20260508 06:06:41.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:41.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 06:06:41.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:06:41.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:41.803 | INFO | Uploaded 36GiB to GPU in 2.48952s (14.4606GiB/s) | 20260508 06:06:41.803 | INFO | Downloaded 34GiB to GPU in 2.43964s (13.9365GiB/s) | 20260508 06:06:46.950 | INFO | Analyzing chunk frames = [5825536,5826048) | 20260508 06:06:46.950 | INFO | MemInfo: { | 20260508 06:06:46.950 | INFO | "gpu" : " 17 GB", | 20260508 06:06:46.950 | INFO | "huge" : "", | 20260508 06:06:46.950 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:46.950 | INFO | "malloc" : " 14 GB", | 20260508 06:06:46.950 | INFO | "pinned" : " 29 GB" | 20260508 06:06:46.950 | INFO | } | 20260508 06:06:46.951 | INFO | Perf report for the last 5.14839 seconds: | 20260508 06:06:46.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:46.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 06:06:46.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:06:46.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:46.951 | INFO | Uploaded 36GiB to GPU in 2.53328s (14.2108GiB/s) | 20260508 06:06:46.951 | INFO | Downloaded 34GiB to GPU in 2.41814s (14.0604GiB/s) | 20260508 06:06:52.088 | INFO | Analyzing chunk frames = [5826048,5826560) | 20260508 06:06:52.088 | INFO | MemInfo: { | 20260508 06:06:52.088 | INFO | "gpu" : " 17 GB", | 20260508 06:06:52.088 | INFO | "huge" : "", | 20260508 06:06:52.088 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:52.088 | INFO | "malloc" : " 14 GB", | 20260508 06:06:52.088 | INFO | "pinned" : " 29 GB" | 20260508 06:06:52.088 | INFO | } | 20260508 06:06:52.088 | INFO | Perf report for the last 5.13744 seconds: | 20260508 06:06:52.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:52.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 06:06:52.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:06:52.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:52.088 | INFO | Uploaded 36GiB to GPU in 2.54207s (14.1617GiB/s) | 20260508 06:06:52.088 | INFO | Downloaded 34GiB to GPU in 2.40732s (14.1236GiB/s) | 20260508 06:06:57.229 | INFO | Analyzing chunk frames = [5826560,5827072) | 20260508 06:06:57.229 | INFO | MemInfo: { | 20260508 06:06:57.229 | INFO | "gpu" : " 17 GB", | 20260508 06:06:57.229 | INFO | "huge" : "", | 20260508 06:06:57.229 | INFO | "huge_shared" : " 29 GB", | 20260508 06:06:57.229 | INFO | "malloc" : " 14 GB", | 20260508 06:06:57.229 | INFO | "pinned" : " 29 GB" | 20260508 06:06:57.229 | INFO | } | 20260508 06:06:57.229 | INFO | Perf report for the last 5.141 seconds: | 20260508 06:06:57.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:06:57.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:06:57.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:06:57.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:06:57.229 | INFO | Uploaded 36GiB to GPU in 2.32194s (15.5043GiB/s) | 20260508 06:06:57.229 | INFO | Downloaded 34GiB to GPU in 2.49305s (13.6379GiB/s) | 20260508 06:07:02.364 | INFO | Analyzing chunk frames = [5827072,5827584) | 20260508 06:07:02.364 | INFO | MemInfo: { | 20260508 06:07:02.364 | INFO | "gpu" : " 17 GB", | 20260508 06:07:02.364 | INFO | "huge" : "", | 20260508 06:07:02.364 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:02.364 | INFO | "malloc" : " 14 GB", | 20260508 06:07:02.364 | INFO | "pinned" : " 29 GB" | 20260508 06:07:02.364 | INFO | } | 20260508 06:07:02.364 | INFO | Perf report for the last 5.13484 seconds: | 20260508 06:07:02.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:02.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:07:02.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:07:02.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:02.364 | INFO | Uploaded 36GiB to GPU in 2.44923s (14.6985GiB/s) | 20260508 06:07:02.364 | INFO | Downloaded 34GiB to GPU in 2.43449s (13.9659GiB/s) | 20260508 06:07:07.505 | INFO | Analyzing chunk frames = [5827584,5828096) | 20260508 06:07:07.505 | INFO | MemInfo: { | 20260508 06:07:07.505 | INFO | "gpu" : " 17 GB", | 20260508 06:07:07.505 | INFO | "huge" : "", | 20260508 06:07:07.505 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:07.505 | INFO | "malloc" : " 14 GB", | 20260508 06:07:07.505 | INFO | "pinned" : " 29 GB" | 20260508 06:07:07.505 | INFO | } | 20260508 06:07:07.505 | INFO | Perf report for the last 5.14126 seconds: | 20260508 06:07:07.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:07.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.505 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:07:07.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.505 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 06:07:07.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:07.506 | INFO | Uploaded 36GiB to GPU in 2.46452s (14.6073GiB/s) | 20260508 06:07:07.506 | INFO | Downloaded 34GiB to GPU in 2.4352s (13.9619GiB/s) | 20260508 06:07:12.645 | INFO | Analyzing chunk frames = [5828096,5828608) | 20260508 06:07:12.645 | INFO | MemInfo: { | 20260508 06:07:12.645 | INFO | "gpu" : " 17 GB", | 20260508 06:07:12.645 | INFO | "huge" : "", | 20260508 06:07:12.645 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:12.645 | INFO | "malloc" : " 14 GB", | 20260508 06:07:12.645 | INFO | "pinned" : " 29 GB" | 20260508 06:07:12.645 | INFO | } | 20260508 06:07:12.646 | INFO | Perf report for the last 5.14039 seconds: | 20260508 06:07:12.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:12.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:07:12.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260508 06:07:12.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:12.646 | INFO | Uploaded 36GiB to GPU in 2.50261s (14.385GiB/s) | 20260508 06:07:12.646 | INFO | Downloaded 34GiB to GPU in 2.45779s (13.8336GiB/s) | 20260508 06:07:12.757 | INFO | Starting a round of Baz IO | 20260508 06:07:16.177 | INFO | Populating RTMetrics report | 20260508 06:07:16.177 | INFO | RTMetrics report populated | 20260508 06:07:16.177 | INFO | Opening tmp RTMetrics json file | 20260508 06:07:16.185 | INFO | Serializing RTMetrics to json | 20260508 06:07:16.192 | INFO | Serialization done, swapping tmp file now... | 20260508 06:07:16.193 | INFO | RTMetrics done | 20260508 06:07:16.245 | INFO | Finished a round of Baz IO: Wrote 1.56 GiB in 3.487 seconds | 20260508 06:07:16.245 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.75% / 0.00% / 59.24% | 20260508 06:07:16.245 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.420539% / 7.94729e-06% / 6.75519e-05% | 20260508 06:07:16.245 | INFO | Baz IO overall: | 20260508 06:07:16.245 | INFO | BytesWritten : 2589082349021 | 20260508 06:07:16.245 | INFO | NumEvents : 1248593018426 | 20260508 06:07:17.782 | INFO | Analyzing chunk frames = [5828608,5829120) | 20260508 06:07:17.782 | INFO | MemInfo: { | 20260508 06:07:17.782 | INFO | "gpu" : " 17 GB", | 20260508 06:07:17.782 | INFO | "huge" : "", | 20260508 06:07:17.782 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:17.782 | INFO | "malloc" : " 14 GB", | 20260508 06:07:17.783 | INFO | "pinned" : " 29 GB" | 20260508 06:07:17.783 | INFO | } | 20260508 06:07:17.783 | INFO | Perf report for the last 5.137 seconds: | 20260508 06:07:17.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:17.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:17.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:17.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:17.783 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 06:07:17.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:17.783 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 06:07:17.783 | INFO | PRE_HQ : 66.3%, 1.8, 62.1ms, 0 | 20260508 06:07:17.783 | INFO | BAZWRITER : 67.3%, 2.8, 97.0ms, 0 | 20260508 06:07:17.783 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 06:07:17.783 | INFO | Uploaded 36GiB to GPU in 2.57354s (13.9885GiB/s) | 20260508 06:07:17.783 | INFO | Downloaded 38GiB to GPU in 2.69192s (14.1163GiB/s) | 20260508 06:07:22.925 | INFO | Analyzing chunk frames = [5829120,5829632) | 20260508 06:07:22.925 | INFO | MemInfo: { | 20260508 06:07:22.925 | INFO | "gpu" : " 17 GB", | 20260508 06:07:22.925 | INFO | "huge" : "", | 20260508 06:07:22.925 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:22.925 | INFO | "malloc" : " 14 GB", | 20260508 06:07:22.925 | INFO | "pinned" : " 29 GB" | 20260508 06:07:22.925 | INFO | } | 20260508 06:07:22.926 | INFO | Perf report for the last 5.14291 seconds: | 20260508 06:07:22.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:22.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:07:22.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:07:22.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:22.926 | INFO | Uploaded 36GiB to GPU in 2.49038s (14.4556GiB/s) | 20260508 06:07:22.926 | INFO | Downloaded 34GiB to GPU in 2.43518s (13.962GiB/s) | 20260508 06:07:28.060 | INFO | Analyzing chunk frames = [5829632,5830144) | 20260508 06:07:28.060 | INFO | MemInfo: { | 20260508 06:07:28.060 | INFO | "gpu" : " 17 GB", | 20260508 06:07:28.060 | INFO | "huge" : "", | 20260508 06:07:28.060 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:28.060 | INFO | "malloc" : " 14 GB", | 20260508 06:07:28.060 | INFO | "pinned" : " 29 GB" | 20260508 06:07:28.060 | INFO | } | 20260508 06:07:28.060 | INFO | Perf report for the last 5.13471 seconds: | 20260508 06:07:28.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:28.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 06:07:28.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:07:28.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:28.060 | INFO | Uploaded 36GiB to GPU in 2.54395s (14.1512GiB/s) | 20260508 06:07:28.061 | INFO | Downloaded 34GiB to GPU in 2.41856s (14.0579GiB/s) | 20260508 06:07:33.201 | INFO | Analyzing chunk frames = [5830144,5830656) | 20260508 06:07:33.201 | INFO | MemInfo: { | 20260508 06:07:33.201 | INFO | "gpu" : " 17 GB", | 20260508 06:07:33.201 | INFO | "huge" : "", | 20260508 06:07:33.201 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:33.201 | INFO | "malloc" : " 14 GB", | 20260508 06:07:33.201 | INFO | "pinned" : " 29 GB" | 20260508 06:07:33.201 | INFO | } | 20260508 06:07:33.202 | INFO | Perf report for the last 5.14126 seconds: | 20260508 06:07:33.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:33.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 06:07:33.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:07:33.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:33.202 | INFO | Uploaded 36GiB to GPU in 2.54706s (14.1339GiB/s) | 20260508 06:07:33.202 | INFO | Downloaded 34GiB to GPU in 2.40848s (14.1168GiB/s) | 20260508 06:07:38.341 | INFO | Analyzing chunk frames = [5830656,5831168) | 20260508 06:07:38.341 | INFO | MemInfo: { | 20260508 06:07:38.341 | INFO | "gpu" : " 17 GB", | 20260508 06:07:38.341 | INFO | "huge" : "", | 20260508 06:07:38.341 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:38.341 | INFO | "malloc" : " 14 GB", | 20260508 06:07:38.342 | INFO | "pinned" : " 29 GB" | 20260508 06:07:38.342 | INFO | } | 20260508 06:07:38.342 | INFO | Perf report for the last 5.14007 seconds: | 20260508 06:07:38.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:38.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:07:38.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:07:38.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:38.342 | INFO | Uploaded 36GiB to GPU in 2.32361s (15.4931GiB/s) | 20260508 06:07:38.342 | INFO | Downloaded 34GiB to GPU in 2.48112s (13.7035GiB/s) | 20260508 06:07:43.481 | INFO | Analyzing chunk frames = [5831168,5831680) | 20260508 06:07:43.481 | INFO | MemInfo: { | 20260508 06:07:43.481 | INFO | "gpu" : " 17 GB", | 20260508 06:07:43.481 | INFO | "huge" : "", | 20260508 06:07:43.481 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:43.481 | INFO | "malloc" : " 14 GB", | 20260508 06:07:43.481 | INFO | "pinned" : " 29 GB" | 20260508 06:07:43.481 | INFO | } | 20260508 06:07:43.481 | INFO | Perf report for the last 5.13972 seconds: | 20260508 06:07:43.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:43.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.481 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:07:43.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.481 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:07:43.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:43.482 | INFO | Uploaded 36GiB to GPU in 2.464s (14.6104GiB/s) | 20260508 06:07:43.482 | INFO | Downloaded 34GiB to GPU in 2.43101s (13.986GiB/s) | 20260508 06:07:48.617 | INFO | Analyzing chunk frames = [5831680,5832192) | 20260508 06:07:48.617 | INFO | MemInfo: { | 20260508 06:07:48.617 | INFO | "gpu" : " 17 GB", | 20260508 06:07:48.617 | INFO | "huge" : "", | 20260508 06:07:48.617 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:48.617 | INFO | "malloc" : " 14 GB", | 20260508 06:07:48.617 | INFO | "pinned" : " 29 GB" | 20260508 06:07:48.617 | INFO | } | 20260508 06:07:48.617 | INFO | Perf report for the last 5.13567 seconds: | 20260508 06:07:48.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:48.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.617 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:07:48.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.617 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 06:07:48.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:48.618 | INFO | Uploaded 36GiB to GPU in 2.4827s (14.5003GiB/s) | 20260508 06:07:48.618 | INFO | Downloaded 34GiB to GPU in 2.4303s (13.9901GiB/s) | 20260508 06:07:53.757 | INFO | Analyzing chunk frames = [5832192,5832704) | 20260508 06:07:53.757 | INFO | MemInfo: { | 20260508 06:07:53.757 | INFO | "gpu" : " 17 GB", | 20260508 06:07:53.757 | INFO | "huge" : "", | 20260508 06:07:53.758 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:53.758 | INFO | "malloc" : " 14 GB", | 20260508 06:07:53.758 | INFO | "pinned" : " 29 GB" | 20260508 06:07:53.758 | INFO | } | 20260508 06:07:53.758 | INFO | Perf report for the last 5.14071 seconds: | 20260508 06:07:53.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:53.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:07:53.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | PRE_HQ : 34.2%, 1.0, 17.7ms, 0 | 20260508 06:07:53.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:53.758 | INFO | Uploaded 36GiB to GPU in 2.50823s (14.3527GiB/s) | 20260508 06:07:53.758 | INFO | Downloaded 34GiB to GPU in 2.4423s (13.9213GiB/s) | 20260508 06:07:53.759 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:07:53.759 | INFO | Total mem usage: 16.7 GiB | 20260508 06:07:53.759 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:07:53.759 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:07:53.759 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:07:53.759 | INFO | High Water Breakdown: | 20260508 06:07:53.759 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:07:53.759 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:07:53.759 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:07:53.759 | INFO | ------------------------------------------ | 20260508 06:07:53.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:07:53.759 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:07:53.759 | INFO | Total mem usage: 26.5 GiB | 20260508 06:07:53.759 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:07:53.759 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 06:07:53.759 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 06:07:53.759 | INFO | High Water Breakdown: | 20260508 06:07:53.759 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:07:53.759 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:07:53.759 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:07:53.759 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:07:53.759 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:07:53.759 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:07:53.759 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:07:53.759 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:07:53.759 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:07:53.759 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:07:53.759 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:07:53.759 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:07:53.759 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:07:53.759 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:07:53.759 | INFO | ------------------------------------------ | 20260508 06:07:53.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:07:53.759 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:07:53.759 | INFO | Total mem usage: 27.8 GiB | 20260508 06:07:53.759 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:07:53.759 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:07:53.759 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:07:53.759 | INFO | High Water Breakdown: | 20260508 06:07:53.759 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:07:53.759 | INFO | ------------------------------------------ | 20260508 06:07:53.759 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:07:53.759 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:07:53.759 | INFO | Total mem usage: 15.7 GiB | 20260508 06:07:53.759 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:07:53.759 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:07:53.759 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:07:53.759 | INFO | High Water Breakdown: | 20260508 06:07:53.759 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:07:53.759 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:07:53.759 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:07:53.759 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:07:53.759 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:07:53.759 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:07:53.759 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:07:53.759 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:07:53.759 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:07:53.759 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:07:53.759 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:07:53.759 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:07:53.759 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:07:53.759 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:07:53.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:07:53.759 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:07:53.759 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:07:53.759 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:07:53.759 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:07:53.759 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:07:53.759 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:07:53.759 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:07:53.759 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:07:53.760 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:07:53.760 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:07:53.760 | INFO | ------------------------------------------ | 20260508 06:07:53.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:07:53.760 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:07:53.760 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:07:53.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:07:53.760 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 5.49e-01 | 20260508 06:07:53.760 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.04e-01, 1.18e-02 / 6.77e-03 | 20260508 06:07:53.760 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.40e+00, 5.86e-01 / 2.34e-01 | 20260508 06:07:53.760 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.07e-01 / 1.33e-01 | 20260508 06:07:53.760 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.52e+01, 3.85e+00 / 1.30e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.61e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.53e+00, 4.11e-01 / 2.90e-01 | 20260508 06:07:53.760 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.25e+00, 2.56e-01 / 1.58e-01 | 20260508 06:07:53.760 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.51e-01, 5.03e-02 / 2.62e-02 | 20260508 06:07:53.760 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:07:53.760 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:07:53.760 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:07:53.760 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 24.40%, Upload -- 2.59e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 24.24%, Download -- 2.57e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 2.27%, LockUpload -- 2.41e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 2.58%, LockDownload -- 2.73e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 15.67%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:07:53.760 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260508 06:07:53.878 | INFO | Starting a round of Baz IO | 20260508 06:07:57.294 | INFO | Populating RTMetrics report | 20260508 06:07:57.294 | INFO | RTMetrics report populated | 20260508 06:07:57.294 | INFO | Opening tmp RTMetrics json file | 20260508 06:07:57.295 | INFO | Serializing RTMetrics to json | 20260508 06:07:57.296 | INFO | Serialization done, swapping tmp file now... | 20260508 06:07:57.297 | INFO | RTMetrics done | 20260508 06:07:57.363 | INFO | Finished a round of Baz IO: Wrote 1.56 GiB in 3.484 seconds | 20260508 06:07:57.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.75% / 0.00% / 59.25% | 20260508 06:07:57.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.417805% / 0% / 7.54992e-05% | 20260508 06:07:57.363 | INFO | Baz IO overall: | 20260508 06:07:57.363 | INFO | BytesWritten : 2590756922584 | 20260508 06:07:57.363 | INFO | NumEvents : 1249391174850 | 20260508 06:07:58.897 | INFO | Analyzing chunk frames = [5832704,5833216) | 20260508 06:07:58.897 | INFO | MemInfo: { | 20260508 06:07:58.897 | INFO | "gpu" : " 17 GB", | 20260508 06:07:58.897 | INFO | "huge" : "", | 20260508 06:07:58.897 | INFO | "huge_shared" : " 29 GB", | 20260508 06:07:58.898 | INFO | "malloc" : " 14 GB", | 20260508 06:07:58.898 | INFO | "pinned" : " 29 GB" | 20260508 06:07:58.898 | INFO | } | 20260508 06:07:58.898 | INFO | Perf report for the last 5.13986 seconds: | 20260508 06:07:58.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:07:58.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:58.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:58.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:58.898 | INFO | ANALYSIS : 68.5%, 3.0, 106.0ms, 0 | 20260508 06:07:58.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:07:58.898 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 06:07:58.898 | INFO | PRE_HQ : 66.8%, 1.8, 62.4ms, 0 | 20260508 06:07:58.898 | INFO | BAZWRITER : 67.1%, 2.8, 96.5ms, 0 | 20260508 06:07:58.898 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 06:07:58.898 | INFO | Uploaded 36GiB to GPU in 2.56107s (14.0566GiB/s) | 20260508 06:07:58.898 | INFO | Downloaded 38GiB to GPU in 2.69812s (14.0839GiB/s) | 20260508 06:08:04.039 | INFO | Analyzing chunk frames = [5833216,5833728) | 20260508 06:08:04.039 | INFO | MemInfo: { | 20260508 06:08:04.039 | INFO | "gpu" : " 17 GB", | 20260508 06:08:04.039 | INFO | "huge" : "", | 20260508 06:08:04.039 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:04.039 | INFO | "malloc" : " 14 GB", | 20260508 06:08:04.039 | INFO | "pinned" : " 29 GB" | 20260508 06:08:04.039 | INFO | } | 20260508 06:08:04.039 | INFO | Perf report for the last 5.14177 seconds: | 20260508 06:08:04.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:04.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.039 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:08:04.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.040 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:08:04.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:04.040 | INFO | Uploaded 36GiB to GPU in 2.48231s (14.5026GiB/s) | 20260508 06:08:04.040 | INFO | Downloaded 34GiB to GPU in 2.43596s (13.9575GiB/s) | 20260508 06:08:09.178 | INFO | Analyzing chunk frames = [5833728,5834240) | 20260508 06:08:09.178 | INFO | MemInfo: { | 20260508 06:08:09.178 | INFO | "gpu" : " 17 GB", | 20260508 06:08:09.178 | INFO | "huge" : "", | 20260508 06:08:09.178 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:09.178 | INFO | "malloc" : " 14 GB", | 20260508 06:08:09.178 | INFO | "pinned" : " 29 GB" | 20260508 06:08:09.178 | INFO | } | 20260508 06:08:09.178 | INFO | Perf report for the last 5.139 seconds: | 20260508 06:08:09.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:09.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.179 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:08:09.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.179 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:08:09.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:09.179 | INFO | Uploaded 36GiB to GPU in 2.53032s (14.2275GiB/s) | 20260508 06:08:09.179 | INFO | Downloaded 34GiB to GPU in 2.41744s (14.0645GiB/s) | 20260508 06:08:14.313 | INFO | Analyzing chunk frames = [5834240,5834752) | 20260508 06:08:14.313 | INFO | MemInfo: { | 20260508 06:08:14.313 | INFO | "gpu" : " 17 GB", | 20260508 06:08:14.313 | INFO | "huge" : "", | 20260508 06:08:14.313 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:14.313 | INFO | "malloc" : " 14 GB", | 20260508 06:08:14.313 | INFO | "pinned" : " 29 GB" | 20260508 06:08:14.313 | INFO | } | 20260508 06:08:14.313 | INFO | Perf report for the last 5.13507 seconds: | 20260508 06:08:14.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:14.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 06:08:14.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:08:14.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:14.314 | INFO | Uploaded 36GiB to GPU in 2.55481s (14.0911GiB/s) | 20260508 06:08:14.314 | INFO | Downloaded 34GiB to GPU in 2.40425s (14.1416GiB/s) | 20260508 06:08:19.457 | INFO | Analyzing chunk frames = [5834752,5835264) | 20260508 06:08:19.457 | INFO | MemInfo: { | 20260508 06:08:19.457 | INFO | "gpu" : " 17 GB", | 20260508 06:08:19.457 | INFO | "huge" : "", | 20260508 06:08:19.458 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:19.458 | INFO | "malloc" : " 14 GB", | 20260508 06:08:19.458 | INFO | "pinned" : " 29 GB" | 20260508 06:08:19.458 | INFO | } | 20260508 06:08:19.458 | INFO | Perf report for the last 5.14417 seconds: | 20260508 06:08:19.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:19.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 06:08:19.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:08:19.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:19.458 | INFO | Uploaded 36GiB to GPU in 2.42985s (14.8157GiB/s) | 20260508 06:08:19.458 | INFO | Downloaded 34GiB to GPU in 2.51251s (13.5323GiB/s) | 20260508 06:08:24.598 | INFO | Analyzing chunk frames = [5835264,5835776) | 20260508 06:08:24.598 | INFO | MemInfo: { | 20260508 06:08:24.598 | INFO | "gpu" : " 17 GB", | 20260508 06:08:24.598 | INFO | "huge" : "", | 20260508 06:08:24.598 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:24.598 | INFO | "malloc" : " 14 GB", | 20260508 06:08:24.598 | INFO | "pinned" : " 29 GB" | 20260508 06:08:24.598 | INFO | } | 20260508 06:08:24.598 | INFO | Perf report for the last 5.14026 seconds: | 20260508 06:08:24.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:24.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:08:24.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:08:24.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:24.598 | INFO | Uploaded 36GiB to GPU in 2.44652s (14.7148GiB/s) | 20260508 06:08:24.598 | INFO | Downloaded 34GiB to GPU in 2.44235s (13.921GiB/s) | 20260508 06:08:29.732 | INFO | Analyzing chunk frames = [5835776,5836288) | 20260508 06:08:29.732 | INFO | MemInfo: { | 20260508 06:08:29.732 | INFO | "gpu" : " 17 GB", | 20260508 06:08:29.732 | INFO | "huge" : "", | 20260508 06:08:29.732 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:29.732 | INFO | "malloc" : " 14 GB", | 20260508 06:08:29.732 | INFO | "pinned" : " 29 GB" | 20260508 06:08:29.732 | INFO | } | 20260508 06:08:29.732 | INFO | Perf report for the last 5.1341 seconds: | 20260508 06:08:29.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:29.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.732 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:08:29.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.732 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 06:08:29.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:29.733 | INFO | Uploaded 36GiB to GPU in 2.46081s (14.6293GiB/s) | 20260508 06:08:29.733 | INFO | Downloaded 34GiB to GPU in 2.44132s (13.9269GiB/s) | 20260508 06:08:34.871 | INFO | Analyzing chunk frames = [5836288,5836800) | 20260508 06:08:34.871 | INFO | MemInfo: { | 20260508 06:08:34.871 | INFO | "gpu" : " 17 GB", | 20260508 06:08:34.871 | INFO | "huge" : "", | 20260508 06:08:34.871 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:34.871 | INFO | "malloc" : " 14 GB", | 20260508 06:08:34.871 | INFO | "pinned" : " 29 GB" | 20260508 06:08:34.871 | INFO | } | 20260508 06:08:34.871 | INFO | Perf report for the last 5.1389 seconds: | 20260508 06:08:34.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:34.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:08:34.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | PRE_HQ : 34.7%, 1.0, 18.0ms, 0 | 20260508 06:08:34.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:34.871 | INFO | Uploaded 36GiB to GPU in 2.45567s (14.66GiB/s) | 20260508 06:08:34.872 | INFO | Downloaded 34GiB to GPU in 2.44258s (13.9197GiB/s) | 20260508 06:08:34.986 | INFO | Starting a round of Baz IO | 20260508 06:08:38.389 | INFO | Populating RTMetrics report | 20260508 06:08:38.390 | INFO | RTMetrics report populated | 20260508 06:08:38.390 | INFO | Opening tmp RTMetrics json file | 20260508 06:08:38.401 | INFO | Serializing RTMetrics to json | 20260508 06:08:38.403 | INFO | Serialization done, swapping tmp file now... | 20260508 06:08:38.403 | INFO | RTMetrics done | 20260508 06:08:38.463 | INFO | Finished a round of Baz IO: Wrote 1.559 GiB in 3.477 seconds | 20260508 06:08:38.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.75% / 0.00% / 59.25% | 20260508 06:08:38.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.418051% / 3.97364e-06% / 5.16574e-05% | 20260508 06:08:38.463 | INFO | Baz IO overall: | 20260508 06:08:38.463 | INFO | BytesWritten : 2592430819783 | 20260508 06:08:38.463 | INFO | NumEvents : 1250188821581 | 20260508 06:08:40.012 | INFO | Analyzing chunk frames = [5836800,5837312) | 20260508 06:08:40.012 | INFO | MemInfo: { | 20260508 06:08:40.012 | INFO | "gpu" : " 17 GB", | 20260508 06:08:40.012 | INFO | "huge" : "", | 20260508 06:08:40.012 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:40.012 | INFO | "malloc" : " 14 GB", | 20260508 06:08:40.012 | INFO | "pinned" : " 29 GB" | 20260508 06:08:40.012 | INFO | } | 20260508 06:08:40.012 | INFO | Perf report for the last 5.14113 seconds: | 20260508 06:08:40.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:40.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:40.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:40.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:40.012 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 06:08:40.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:40.012 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 06:08:40.012 | INFO | PRE_HQ : 66.2%, 1.8, 61.3ms, 0 | 20260508 06:08:40.012 | INFO | BAZWRITER : 67.1%, 2.8, 95.9ms, 0 | 20260508 06:08:40.012 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:08:40.013 | INFO | Uploaded 36GiB to GPU in 2.56212s (14.0509GiB/s) | 20260508 06:08:40.013 | INFO | Downloaded 38GiB to GPU in 2.68707s (14.1418GiB/s) | 20260508 06:08:45.155 | INFO | Analyzing chunk frames = [5837312,5837824) | 20260508 06:08:45.155 | INFO | MemInfo: { | 20260508 06:08:45.155 | INFO | "gpu" : " 17 GB", | 20260508 06:08:45.155 | INFO | "huge" : "", | 20260508 06:08:45.156 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:45.156 | INFO | "malloc" : " 14 GB", | 20260508 06:08:45.156 | INFO | "pinned" : " 29 GB" | 20260508 06:08:45.156 | INFO | } | 20260508 06:08:45.156 | INFO | Perf report for the last 5.14357 seconds: | 20260508 06:08:45.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:45.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:08:45.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:08:45.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:45.156 | INFO | Uploaded 36GiB to GPU in 2.48429s (14.4911GiB/s) | 20260508 06:08:45.156 | INFO | Downloaded 34GiB to GPU in 2.45218s (13.8652GiB/s) | 20260508 06:08:50.293 | INFO | Analyzing chunk frames = [5837824,5838336) | 20260508 06:08:50.293 | INFO | MemInfo: { | 20260508 06:08:50.293 | INFO | "gpu" : " 17 GB", | 20260508 06:08:50.293 | INFO | "huge" : "", | 20260508 06:08:50.293 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:50.293 | INFO | "malloc" : " 14 GB", | 20260508 06:08:50.293 | INFO | "pinned" : " 29 GB" | 20260508 06:08:50.293 | INFO | } | 20260508 06:08:50.293 | INFO | Perf report for the last 5.13751 seconds: | 20260508 06:08:50.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:50.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.293 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 06:08:50.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.294 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:08:50.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:50.294 | INFO | Uploaded 36GiB to GPU in 2.55553s (14.0871GiB/s) | 20260508 06:08:50.294 | INFO | Downloaded 34GiB to GPU in 2.43789s (13.9465GiB/s) | 20260508 06:08:55.427 | INFO | Analyzing chunk frames = [5838336,5838848) | 20260508 06:08:55.427 | INFO | MemInfo: { | 20260508 06:08:55.427 | INFO | "gpu" : " 17 GB", | 20260508 06:08:55.427 | INFO | "huge" : "", | 20260508 06:08:55.427 | INFO | "huge_shared" : " 29 GB", | 20260508 06:08:55.427 | INFO | "malloc" : " 14 GB", | 20260508 06:08:55.427 | INFO | "pinned" : " 29 GB" | 20260508 06:08:55.427 | INFO | } | 20260508 06:08:55.427 | INFO | Perf report for the last 5.13355 seconds: | 20260508 06:08:55.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:08:55.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 06:08:55.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:08:55.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:08:55.427 | INFO | Uploaded 36GiB to GPU in 2.53337s (14.2103GiB/s) | 20260508 06:08:55.427 | INFO | Downloaded 34GiB to GPU in 2.39448s (14.1993GiB/s) | 20260508 06:09:00.565 | INFO | Analyzing chunk frames = [5838848,5839360) | 20260508 06:09:00.565 | INFO | MemInfo: { | 20260508 06:09:00.565 | INFO | "gpu" : " 17 GB", | 20260508 06:09:00.565 | INFO | "huge" : "", | 20260508 06:09:00.565 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:00.565 | INFO | "malloc" : " 14 GB", | 20260508 06:09:00.565 | INFO | "pinned" : " 29 GB" | 20260508 06:09:00.565 | INFO | } | 20260508 06:09:00.565 | INFO | Perf report for the last 5.13856 seconds: | 20260508 06:09:00.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:00.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 06:09:00.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:09:00.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:00.566 | INFO | Uploaded 36GiB to GPU in 2.42085s (14.8708GiB/s) | 20260508 06:09:00.566 | INFO | Downloaded 34GiB to GPU in 2.52204s (13.4812GiB/s) | 20260508 06:09:05.714 | INFO | Analyzing chunk frames = [5839360,5839872) | 20260508 06:09:05.714 | INFO | MemInfo: { | 20260508 06:09:05.714 | INFO | "gpu" : " 17 GB", | 20260508 06:09:05.714 | INFO | "huge" : "", | 20260508 06:09:05.714 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:05.714 | INFO | "malloc" : " 14 GB", | 20260508 06:09:05.714 | INFO | "pinned" : " 29 GB" | 20260508 06:09:05.714 | INFO | } | 20260508 06:09:05.714 | INFO | Perf report for the last 5.14855 seconds: | 20260508 06:09:05.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:05.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.714 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 06:09:05.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.714 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:09:05.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:05.715 | INFO | Uploaded 36GiB to GPU in 2.4337s (14.7923GiB/s) | 20260508 06:09:05.715 | INFO | Downloaded 34GiB to GPU in 2.43469s (13.9648GiB/s) | 20260508 06:09:10.852 | INFO | Analyzing chunk frames = [5839872,5840384) | 20260508 06:09:10.852 | INFO | MemInfo: { | 20260508 06:09:10.852 | INFO | "gpu" : " 17 GB", | 20260508 06:09:10.852 | INFO | "huge" : "", | 20260508 06:09:10.852 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:10.852 | INFO | "malloc" : " 14 GB", | 20260508 06:09:10.852 | INFO | "pinned" : " 29 GB" | 20260508 06:09:10.852 | INFO | } | 20260508 06:09:10.852 | INFO | Perf report for the last 5.13831 seconds: | 20260508 06:09:10.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:10.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.852 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:09:10.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.852 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 06:09:10.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:10.853 | INFO | Uploaded 36GiB to GPU in 2.47079s (14.5702GiB/s) | 20260508 06:09:10.853 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9296GiB/s) | 20260508 06:09:15.995 | INFO | Analyzing chunk frames = [5840384,5840896) | 20260508 06:09:15.995 | INFO | MemInfo: { | 20260508 06:09:15.995 | INFO | "gpu" : " 17 GB", | 20260508 06:09:15.995 | INFO | "huge" : "", | 20260508 06:09:15.995 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:15.995 | INFO | "malloc" : " 14 GB", | 20260508 06:09:15.995 | INFO | "pinned" : " 29 GB" | 20260508 06:09:15.995 | INFO | } | 20260508 06:09:15.995 | INFO | Perf report for the last 5.14311 seconds: | 20260508 06:09:15.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:15.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.996 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:09:15.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.996 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 06:09:15.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:15.996 | INFO | Uploaded 36GiB to GPU in 2.46448s (14.6075GiB/s) | 20260508 06:09:15.996 | INFO | Downloaded 34GiB to GPU in 2.45046s (13.8749GiB/s) | 20260508 06:09:15.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:09:15.996 | INFO | Total mem usage: 16.7 GiB | 20260508 06:09:15.996 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:09:15.996 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.4%) | 20260508 06:09:15.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:09:15.996 | INFO | High Water Breakdown: | 20260508 06:09:15.996 | INFO | 59.7% : 8.64 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:09:15.996 | INFO | 40.2% : 5.81 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:09:15.996 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:09:15.996 | INFO | ------------------------------------------ | 20260508 06:09:15.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:09:15.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:09:15.996 | INFO | Total mem usage: 26.5 GiB | 20260508 06:09:15.996 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:09:15.997 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 06:09:15.997 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 06:09:15.997 | INFO | High Water Breakdown: | 20260508 06:09:15.997 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:09:15.997 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:09:15.997 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:09:15.997 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:09:15.997 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:09:15.997 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:09:15.997 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:09:15.997 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:09:15.997 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:09:15.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:09:15.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:09:15.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:09:15.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:09:15.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:09:15.997 | INFO | ------------------------------------------ | 20260508 06:09:15.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:09:15.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:09:15.997 | INFO | Total mem usage: 27.8 GiB | 20260508 06:09:15.997 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:09:15.997 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:09:15.997 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:09:15.997 | INFO | High Water Breakdown: | 20260508 06:09:15.997 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:09:15.997 | INFO | ------------------------------------------ | 20260508 06:09:15.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:09:15.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:09:15.997 | INFO | Total mem usage: 15.7 GiB | 20260508 06:09:15.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:09:15.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:09:15.997 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:09:15.997 | INFO | High Water Breakdown: | 20260508 06:09:15.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:09:15.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:09:15.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:09:15.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:09:15.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:09:15.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:09:15.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:09:15.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:09:15.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:09:15.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:09:15.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:09:15.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:09:15.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:09:15.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:09:15.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:09:15.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:09:15.997 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:09:15.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:09:15.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:09:15.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:09:15.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:09:15.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:09:15.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:09:15.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:09:15.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:09:15.997 | INFO | ------------------------------------------ | 20260508 06:09:15.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:09:15.998 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:09:15.998 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:09:15.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:09:15.998 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.79e-01 | 20260508 06:09:15.998 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.07e-01, 1.14e-02 / 5.53e-03 | 20260508 06:09:15.998 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.44e+00, 5.80e-01 / 2.13e-01 | 20260508 06:09:15.998 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.07e-01 | 20260508 06:09:15.998 | INFO | PerfReport: 15.88%, DME -- 4.16e+01 / 4.48e+01, 3.76e+00 / 1.05e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.82e+00 / 1.51e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.56e+00, 4.15e-01 / 2.43e-01 | 20260508 06:09:15.998 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.23e+00, 2.58e-01 / 1.11e-01 | 20260508 06:09:15.998 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.16e-01 / 3.56e-01, 5.02e-02 / 2.40e-02 | 20260508 06:09:15.998 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:09:15.998 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:09:15.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:09:15.998 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 24.44%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 24.30%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 2.20%, LockUpload -- 2.34e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 2.58%, LockDownload -- 2.73e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 06:09:15.998 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 06:09:16.110 | INFO | Starting a round of Baz IO | 20260508 06:09:19.529 | INFO | Populating RTMetrics report | 20260508 06:09:19.529 | INFO | RTMetrics report populated | 20260508 06:09:19.529 | INFO | Opening tmp RTMetrics json file | 20260508 06:09:19.541 | INFO | Serializing RTMetrics to json | 20260508 06:09:19.542 | INFO | Serialization done, swapping tmp file now... | 20260508 06:09:19.543 | INFO | RTMetrics done | 20260508 06:09:19.600 | INFO | Finished a round of Baz IO: Wrote 1.558 GiB in 3.49 seconds | 20260508 06:09:19.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.75% / 0.00% / 59.25% | 20260508 06:09:19.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.416485% / 0% / 5.16574e-05% | 20260508 06:09:19.600 | INFO | Baz IO overall: | 20260508 06:09:19.600 | INFO | BytesWritten : 2594104218547 | 20260508 06:09:19.600 | INFO | NumEvents : 1250985987881 | 20260508 06:09:21.127 | INFO | Analyzing chunk frames = [5840896,5841408) | 20260508 06:09:21.127 | INFO | MemInfo: { | 20260508 06:09:21.127 | INFO | "gpu" : " 17 GB", | 20260508 06:09:21.127 | INFO | "huge" : "", | 20260508 06:09:21.127 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:21.127 | INFO | "malloc" : " 14 GB", | 20260508 06:09:21.127 | INFO | "pinned" : " 29 GB" | 20260508 06:09:21.127 | INFO | } | 20260508 06:09:21.127 | INFO | Perf report for the last 5.13122 seconds: | 20260508 06:09:21.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:21.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:21.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:21.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:21.127 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 06:09:21.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:21.127 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 06:09:21.127 | INFO | PRE_HQ : 66.4%, 1.8, 61.9ms, 0 | 20260508 06:09:21.127 | INFO | BAZWRITER : 67.3%, 2.8, 97.2ms, 0 | 20260508 06:09:21.127 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:09:21.127 | INFO | Uploaded 36GiB to GPU in 2.55885s (14.0688GiB/s) | 20260508 06:09:21.127 | INFO | Downloaded 38GiB to GPU in 2.70644s (14.0406GiB/s) | 20260508 06:09:26.269 | INFO | Analyzing chunk frames = [5841408,5841920) | 20260508 06:09:26.269 | INFO | MemInfo: { | 20260508 06:09:26.269 | INFO | "gpu" : " 17 GB", | 20260508 06:09:26.269 | INFO | "huge" : "", | 20260508 06:09:26.269 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:26.269 | INFO | "malloc" : " 14 GB", | 20260508 06:09:26.269 | INFO | "pinned" : " 29 GB" | 20260508 06:09:26.269 | INFO | } | 20260508 06:09:26.270 | INFO | Perf report for the last 5.14296 seconds: | 20260508 06:09:26.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:26.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:09:26.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:09:26.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:26.270 | INFO | Uploaded 36GiB to GPU in 2.51186s (14.332GiB/s) | 20260508 06:09:26.270 | INFO | Downloaded 34GiB to GPU in 2.43926s (13.9387GiB/s) | 20260508 06:09:31.405 | INFO | Analyzing chunk frames = [5841920,5842432) | 20260508 06:09:31.405 | INFO | MemInfo: { | 20260508 06:09:31.405 | INFO | "gpu" : " 17 GB", | 20260508 06:09:31.405 | INFO | "huge" : "", | 20260508 06:09:31.405 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:31.405 | INFO | "malloc" : " 14 GB", | 20260508 06:09:31.405 | INFO | "pinned" : " 29 GB" | 20260508 06:09:31.405 | INFO | } | 20260508 06:09:31.405 | INFO | Perf report for the last 5.13551 seconds: | 20260508 06:09:31.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:31.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 06:09:31.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:09:31.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:31.405 | INFO | Uploaded 36GiB to GPU in 2.57807s (13.9639GiB/s) | 20260508 06:09:31.405 | INFO | Downloaded 34GiB to GPU in 2.41869s (14.0572GiB/s) | 20260508 06:09:36.545 | INFO | Analyzing chunk frames = [5842432,5842944) | 20260508 06:09:36.545 | INFO | MemInfo: { | 20260508 06:09:36.545 | INFO | "gpu" : " 17 GB", | 20260508 06:09:36.545 | INFO | "huge" : "", | 20260508 06:09:36.545 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:36.545 | INFO | "malloc" : " 14 GB", | 20260508 06:09:36.545 | INFO | "pinned" : " 29 GB" | 20260508 06:09:36.545 | INFO | } | 20260508 06:09:36.545 | INFO | Perf report for the last 5.1398 seconds: | 20260508 06:09:36.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:36.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 06:09:36.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:09:36.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:36.545 | INFO | Uploaded 36GiB to GPU in 2.58261s (13.9394GiB/s) | 20260508 06:09:36.545 | INFO | Downloaded 34GiB to GPU in 2.38835s (14.2357GiB/s) | 20260508 06:09:41.683 | INFO | Analyzing chunk frames = [5842944,5843456) | 20260508 06:09:41.683 | INFO | MemInfo: { | 20260508 06:09:41.683 | INFO | "gpu" : " 17 GB", | 20260508 06:09:41.683 | INFO | "huge" : "", | 20260508 06:09:41.683 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:41.683 | INFO | "malloc" : " 14 GB", | 20260508 06:09:41.683 | INFO | "pinned" : " 29 GB" | 20260508 06:09:41.683 | INFO | } | 20260508 06:09:41.683 | INFO | Perf report for the last 5.13838 seconds: | 20260508 06:09:41.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:41.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.683 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 06:09:41.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.683 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:09:41.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:41.684 | INFO | Uploaded 36GiB to GPU in 2.38046s (15.1231GiB/s) | 20260508 06:09:41.684 | INFO | Downloaded 34GiB to GPU in 2.55854s (13.2888GiB/s) | 20260508 06:09:46.823 | INFO | Analyzing chunk frames = [5843456,5843968) | 20260508 06:09:46.823 | INFO | MemInfo: { | 20260508 06:09:46.823 | INFO | "gpu" : " 17 GB", | 20260508 06:09:46.823 | INFO | "huge" : "", | 20260508 06:09:46.823 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:46.823 | INFO | "malloc" : " 14 GB", | 20260508 06:09:46.823 | INFO | "pinned" : " 29 GB" | 20260508 06:09:46.823 | INFO | } | 20260508 06:09:46.823 | INFO | Perf report for the last 5.13989 seconds: | 20260508 06:09:46.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:46.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.823 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:09:46.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.823 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:09:46.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:46.824 | INFO | Uploaded 36GiB to GPU in 2.4571s (14.6514GiB/s) | 20260508 06:09:46.824 | INFO | Downloaded 34GiB to GPU in 2.42635s (14.0128GiB/s) | 20260508 06:09:51.964 | INFO | Analyzing chunk frames = [5843968,5844480) | 20260508 06:09:51.964 | INFO | MemInfo: { | 20260508 06:09:51.964 | INFO | "gpu" : " 17 GB", | 20260508 06:09:51.964 | INFO | "huge" : "", | 20260508 06:09:51.964 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:51.964 | INFO | "malloc" : " 14 GB", | 20260508 06:09:51.964 | INFO | "pinned" : " 29 GB" | 20260508 06:09:51.964 | INFO | } | 20260508 06:09:51.965 | INFO | Perf report for the last 5.14125 seconds: | 20260508 06:09:51.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:51.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:09:51.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 06:09:51.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:51.965 | INFO | Uploaded 36GiB to GPU in 2.46397s (14.6106GiB/s) | 20260508 06:09:51.965 | INFO | Downloaded 34GiB to GPU in 2.43034s (13.9898GiB/s) | 20260508 06:09:57.104 | INFO | Analyzing chunk frames = [5844480,5844992) | 20260508 06:09:57.104 | INFO | MemInfo: { | 20260508 06:09:57.104 | INFO | "gpu" : " 17 GB", | 20260508 06:09:57.104 | INFO | "huge" : "", | 20260508 06:09:57.104 | INFO | "huge_shared" : " 29 GB", | 20260508 06:09:57.105 | INFO | "malloc" : " 14 GB", | 20260508 06:09:57.105 | INFO | "pinned" : " 29 GB" | 20260508 06:09:57.105 | INFO | } | 20260508 06:09:57.105 | INFO | Perf report for the last 5.14021 seconds: | 20260508 06:09:57.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:09:57.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 06:09:57.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 06:09:57.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:09:57.105 | INFO | Uploaded 36GiB to GPU in 2.48732s (14.4734GiB/s) | 20260508 06:09:57.105 | INFO | Downloaded 34GiB to GPU in 2.44661s (13.8968GiB/s) | 20260508 06:09:57.217 | INFO | Starting a round of Baz IO | 20260508 06:10:00.632 | INFO | Populating RTMetrics report | 20260508 06:10:00.633 | INFO | RTMetrics report populated | 20260508 06:10:00.633 | INFO | Opening tmp RTMetrics json file | 20260508 06:10:00.643 | INFO | Serializing RTMetrics to json | 20260508 06:10:00.645 | INFO | Serialization done, swapping tmp file now... | 20260508 06:10:00.645 | INFO | RTMetrics done | 20260508 06:10:00.701 | INFO | Finished a round of Baz IO: Wrote 1.558 GiB in 3.483 seconds | 20260508 06:10:00.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.75% / 0.00% / 59.25% | 20260508 06:10:00.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415663% / 7.94729e-06% / 6.35783e-05% | 20260508 06:10:00.701 | INFO | Baz IO overall: | 20260508 06:10:00.701 | INFO | BytesWritten : 2595777167045 | 20260508 06:10:00.701 | INFO | NumEvents : 1251783193785 | 20260508 06:10:02.242 | INFO | Analyzing chunk frames = [5844992,5845504) | 20260508 06:10:02.242 | INFO | MemInfo: { | 20260508 06:10:02.242 | INFO | "gpu" : " 17 GB", | 20260508 06:10:02.242 | INFO | "huge" : "", | 20260508 06:10:02.242 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:02.242 | INFO | "malloc" : " 14 GB", | 20260508 06:10:02.242 | INFO | "pinned" : " 29 GB" | 20260508 06:10:02.242 | INFO | } | 20260508 06:10:02.242 | INFO | Perf report for the last 5.13735 seconds: | 20260508 06:10:02.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:02.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:02.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:02.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:02.242 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260508 06:10:02.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:02.242 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 06:10:02.242 | INFO | PRE_HQ : 66.6%, 1.8, 61.8ms, 0 | 20260508 06:10:02.243 | INFO | BAZWRITER : 67.1%, 2.8, 97.6ms, 0 | 20260508 06:10:02.243 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:10:02.243 | INFO | Uploaded 36GiB to GPU in 2.48684s (14.4762GiB/s) | 20260508 06:10:02.243 | INFO | Downloaded 38GiB to GPU in 2.69789s (14.0851GiB/s) | 20260508 06:10:07.381 | INFO | Analyzing chunk frames = [5845504,5846016) | 20260508 06:10:07.381 | INFO | MemInfo: { | 20260508 06:10:07.381 | INFO | "gpu" : " 17 GB", | 20260508 06:10:07.381 | INFO | "huge" : "", | 20260508 06:10:07.381 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:07.381 | INFO | "malloc" : " 14 GB", | 20260508 06:10:07.381 | INFO | "pinned" : " 29 GB" | 20260508 06:10:07.381 | INFO | } | 20260508 06:10:07.381 | INFO | Perf report for the last 5.13898 seconds: | 20260508 06:10:07.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:07.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:10:07.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:10:07.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:07.381 | INFO | Uploaded 36GiB to GPU in 2.49402s (14.4345GiB/s) | 20260508 06:10:07.381 | INFO | Downloaded 34GiB to GPU in 2.43114s (13.9852GiB/s) | 20260508 06:10:12.527 | INFO | Analyzing chunk frames = [5846016,5846528) | 20260508 06:10:12.527 | INFO | MemInfo: { | 20260508 06:10:12.527 | INFO | "gpu" : " 17 GB", | 20260508 06:10:12.527 | INFO | "huge" : "", | 20260508 06:10:12.527 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:12.527 | INFO | "malloc" : " 14 GB", | 20260508 06:10:12.527 | INFO | "pinned" : " 29 GB" | 20260508 06:10:12.527 | INFO | } | 20260508 06:10:12.528 | INFO | Perf report for the last 5.14646 seconds: | 20260508 06:10:12.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:12.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 06:10:12.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:10:12.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:12.528 | INFO | Uploaded 36GiB to GPU in 2.52534s (14.2555GiB/s) | 20260508 06:10:12.528 | INFO | Downloaded 34GiB to GPU in 2.41856s (14.058GiB/s) | 20260508 06:10:17.664 | INFO | Analyzing chunk frames = [5846528,5847040) | 20260508 06:10:17.664 | INFO | MemInfo: { | 20260508 06:10:17.664 | INFO | "gpu" : " 17 GB", | 20260508 06:10:17.664 | INFO | "huge" : "", | 20260508 06:10:17.664 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:17.664 | INFO | "malloc" : " 14 GB", | 20260508 06:10:17.664 | INFO | "pinned" : " 29 GB" | 20260508 06:10:17.664 | INFO | } | 20260508 06:10:17.665 | INFO | Perf report for the last 5.13699 seconds: | 20260508 06:10:17.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:17.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 06:10:17.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:10:17.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:17.665 | INFO | Uploaded 36GiB to GPU in 2.56505s (14.0348GiB/s) | 20260508 06:10:17.665 | INFO | Downloaded 34GiB to GPU in 2.42121s (14.0425GiB/s) | 20260508 06:10:22.802 | INFO | Analyzing chunk frames = [5847040,5847552) | 20260508 06:10:22.802 | INFO | MemInfo: { | 20260508 06:10:22.802 | INFO | "gpu" : " 17 GB", | 20260508 06:10:22.802 | INFO | "huge" : "", | 20260508 06:10:22.802 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:22.802 | INFO | "malloc" : " 14 GB", | 20260508 06:10:22.802 | INFO | "pinned" : " 29 GB" | 20260508 06:10:22.802 | INFO | } | 20260508 06:10:22.802 | INFO | Perf report for the last 5.13757 seconds: | 20260508 06:10:22.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:22.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.802 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:10:22.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.802 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:10:22.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:22.803 | INFO | Uploaded 36GiB to GPU in 2.29554s (15.6826GiB/s) | 20260508 06:10:22.803 | INFO | Downloaded 34GiB to GPU in 2.48255s (13.6956GiB/s) | 20260508 06:10:27.936 | INFO | Analyzing chunk frames = [5847552,5848064) | 20260508 06:10:27.936 | INFO | MemInfo: { | 20260508 06:10:27.936 | INFO | "gpu" : " 17 GB", | 20260508 06:10:27.936 | INFO | "huge" : "", | 20260508 06:10:27.936 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:27.937 | INFO | "malloc" : " 14 GB", | 20260508 06:10:27.937 | INFO | "pinned" : " 29 GB" | 20260508 06:10:27.937 | INFO | } | 20260508 06:10:27.937 | INFO | Perf report for the last 5.13455 seconds: | 20260508 06:10:27.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:27.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:10:27.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:10:27.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:27.937 | INFO | Uploaded 36GiB to GPU in 2.46336s (14.6142GiB/s) | 20260508 06:10:27.937 | INFO | Downloaded 34GiB to GPU in 2.4375s (13.9487GiB/s) | 20260508 06:10:33.073 | INFO | Analyzing chunk frames = [5848064,5848576) | 20260508 06:10:33.073 | INFO | MemInfo: { | 20260508 06:10:33.073 | INFO | "gpu" : " 17 GB", | 20260508 06:10:33.073 | INFO | "huge" : "", | 20260508 06:10:33.073 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:33.073 | INFO | "malloc" : " 14 GB", | 20260508 06:10:33.074 | INFO | "pinned" : " 29 GB" | 20260508 06:10:33.074 | INFO | } | 20260508 06:10:33.074 | INFO | Perf report for the last 5.13694 seconds: | 20260508 06:10:33.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:33.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.074 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 06:10:33.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.074 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:10:33.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:33.075 | INFO | Uploaded 36GiB to GPU in 2.43111s (14.808GiB/s) | 20260508 06:10:33.075 | INFO | Downloaded 34GiB to GPU in 2.41875s (14.0568GiB/s) | 20260508 06:10:38.216 | INFO | Analyzing chunk frames = [5848576,5849088) | 20260508 06:10:38.216 | INFO | MemInfo: { | 20260508 06:10:38.216 | INFO | "gpu" : " 17 GB", | 20260508 06:10:38.216 | INFO | "huge" : "", | 20260508 06:10:38.216 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:38.216 | INFO | "malloc" : " 14 GB", | 20260508 06:10:38.216 | INFO | "pinned" : " 29 GB" | 20260508 06:10:38.216 | INFO | } | 20260508 06:10:38.216 | INFO | Perf report for the last 5.14245 seconds: | 20260508 06:10:38.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:38.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.216 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:10:38.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.216 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 06:10:38.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:38.217 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:10:38.217 | INFO | Total mem usage: 16.7 GiB | 20260508 06:10:38.217 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:10:38.217 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.6%) | 20260508 06:10:38.217 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:10:38.217 | INFO | High Water Breakdown: | 20260508 06:10:38.217 | INFO | 59.8% : 8.63 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:10:38.217 | INFO | 40.1% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:10:38.217 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:10:38.217 | INFO | ------------------------------------------ | 20260508 06:10:38.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:10:38.217 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:10:38.217 | INFO | Uploaded 36GiB to GPU in 2.47359s (14.5538GiB/s) | 20260508 06:10:38.217 | INFO | Total mem usage: 26.5 GiB | 20260508 06:10:38.217 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:10:38.217 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:10:38.217 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:10:38.217 | INFO | High Water Breakdown: | 20260508 06:10:38.217 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:10:38.217 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:10:38.217 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:10:38.217 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:10:38.217 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:10:38.217 | INFO | Downloaded 34GiB to GPU in 2.43535s (13.961GiB/s) | 20260508 06:10:38.217 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:10:38.217 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:10:38.217 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:10:38.217 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:10:38.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:10:38.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:10:38.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:10:38.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:10:38.217 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:10:38.217 | INFO | ------------------------------------------ | 20260508 06:10:38.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:10:38.217 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:10:38.217 | INFO | Total mem usage: 27.8 GiB | 20260508 06:10:38.217 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:10:38.217 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:10:38.217 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:10:38.217 | INFO | High Water Breakdown: | 20260508 06:10:38.217 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:10:38.217 | INFO | ------------------------------------------ | 20260508 06:10:38.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:10:38.217 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:10:38.217 | INFO | Total mem usage: 15.7 GiB | 20260508 06:10:38.217 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:10:38.217 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:10:38.217 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 06:10:38.217 | INFO | High Water Breakdown: | 20260508 06:10:38.217 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:10:38.217 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:10:38.217 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:10:38.217 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:10:38.217 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:10:38.217 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:10:38.217 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:10:38.217 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:10:38.217 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:10:38.217 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:10:38.217 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:10:38.217 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:10:38.217 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:10:38.217 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:10:38.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:10:38.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:10:38.218 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:10:38.218 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:10:38.218 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:10:38.218 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:10:38.218 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:10:38.218 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:10:38.218 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:10:38.218 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:10:38.218 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:10:38.218 | INFO | ------------------------------------------ | 20260508 06:10:38.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:10:38.218 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:10:38.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:10:38.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:10:38.218 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.57e+01, 1.42e+01 / 1.37e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.07e-01, 1.15e-02 / 1.53e-02 | 20260508 06:10:38.218 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.43e+00, 5.89e-01 / 2.27e-01 | 20260508 06:10:38.218 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.13e-01 | 20260508 06:10:38.218 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.55e+01, 3.88e+00 / 1.60e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.23e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.13e-01 / 2.46e-01 | 20260508 06:10:38.218 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.22e+00, 2.54e-01 / 1.22e-01 | 20260508 06:10:38.218 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.56e-01, 5.02e-02 / 2.43e-02 | 20260508 06:10:38.218 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:10:38.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:10:38.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:10:38.218 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 24.38%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 24.26%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 2.32%, LockUpload -- 2.46e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 2.59%, LockDownload -- 2.75e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 06:10:38.218 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260508 06:10:38.329 | INFO | Starting a round of Baz IO | 20260508 06:10:41.746 | INFO | Populating RTMetrics report | 20260508 06:10:41.747 | INFO | RTMetrics report populated | 20260508 06:10:41.747 | INFO | Opening tmp RTMetrics json file | 20260508 06:10:41.760 | INFO | Serializing RTMetrics to json | 20260508 06:10:41.761 | INFO | Serialization done, swapping tmp file now... | 20260508 06:10:41.762 | INFO | RTMetrics done | 20260508 06:10:41.819 | INFO | Finished a round of Baz IO: Wrote 1.557 GiB in 3.489 seconds | 20260508 06:10:41.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.00% / 59.25% | 20260508 06:10:41.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415222% / 3.97364e-06% / 6.75519e-05% | 20260508 06:10:41.819 | INFO | Baz IO overall: | 20260508 06:10:41.819 | INFO | BytesWritten : 2597448536793 | 20260508 06:10:41.819 | INFO | NumEvents : 1252579121606 | 20260508 06:10:43.362 | INFO | Analyzing chunk frames = [5849088,5849600) | 20260508 06:10:43.362 | INFO | MemInfo: { | 20260508 06:10:43.362 | INFO | "gpu" : " 17 GB", | 20260508 06:10:43.362 | INFO | "huge" : "", | 20260508 06:10:43.362 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:43.362 | INFO | "malloc" : " 14 GB", | 20260508 06:10:43.362 | INFO | "pinned" : " 29 GB" | 20260508 06:10:43.362 | INFO | } | 20260508 06:10:43.362 | INFO | Perf report for the last 5.14589 seconds: | 20260508 06:10:43.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:43.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:43.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:43.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:43.362 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 06:10:43.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:43.362 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 06:10:43.362 | INFO | PRE_HQ : 66.5%, 1.8, 61.4ms, 0 | 20260508 06:10:43.362 | INFO | BAZWRITER : 67.3%, 2.8, 96.9ms, 0 | 20260508 06:10:43.362 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:10:43.362 | INFO | Uploaded 36GiB to GPU in 2.55291s (14.1015GiB/s) | 20260508 06:10:43.362 | INFO | Downloaded 38GiB to GPU in 2.69409s (14.1049GiB/s) | 20260508 06:10:48.499 | INFO | Analyzing chunk frames = [5849600,5850112) | 20260508 06:10:48.499 | INFO | MemInfo: { | 20260508 06:10:48.499 | INFO | "gpu" : " 17 GB", | 20260508 06:10:48.499 | INFO | "huge" : "", | 20260508 06:10:48.499 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:48.499 | INFO | "malloc" : " 14 GB", | 20260508 06:10:48.499 | INFO | "pinned" : " 29 GB" | 20260508 06:10:48.500 | INFO | } | 20260508 06:10:48.500 | INFO | Perf report for the last 5.13758 seconds: | 20260508 06:10:48.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:48.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:10:48.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:10:48.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:48.500 | INFO | Uploaded 36GiB to GPU in 2.4811s (14.5097GiB/s) | 20260508 06:10:48.500 | INFO | Downloaded 34GiB to GPU in 2.43582s (13.9584GiB/s) | 20260508 06:10:53.635 | INFO | Analyzing chunk frames = [5850112,5850624) | 20260508 06:10:53.635 | INFO | MemInfo: { | 20260508 06:10:53.635 | INFO | "gpu" : " 17 GB", | 20260508 06:10:53.635 | INFO | "huge" : "", | 20260508 06:10:53.635 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:53.635 | INFO | "malloc" : " 14 GB", | 20260508 06:10:53.635 | INFO | "pinned" : " 29 GB" | 20260508 06:10:53.635 | INFO | } | 20260508 06:10:53.635 | INFO | Perf report for the last 5.13566 seconds: | 20260508 06:10:53.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:53.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.635 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 06:10:53.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.636 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:10:53.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:53.636 | INFO | Uploaded 36GiB to GPU in 2.5489s (14.1237GiB/s) | 20260508 06:10:53.636 | INFO | Downloaded 34GiB to GPU in 2.42034s (14.0476GiB/s) | 20260508 06:10:58.775 | INFO | Analyzing chunk frames = [5850624,5851136) | 20260508 06:10:58.775 | INFO | MemInfo: { | 20260508 06:10:58.775 | INFO | "gpu" : " 17 GB", | 20260508 06:10:58.775 | INFO | "huge" : "", | 20260508 06:10:58.775 | INFO | "huge_shared" : " 29 GB", | 20260508 06:10:58.775 | INFO | "malloc" : " 14 GB", | 20260508 06:10:58.775 | INFO | "pinned" : " 29 GB" | 20260508 06:10:58.775 | INFO | } | 20260508 06:10:58.775 | INFO | Perf report for the last 5.13954 seconds: | 20260508 06:10:58.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:10:58.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 06:10:58.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:10:58.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:10:58.775 | INFO | Uploaded 36GiB to GPU in 2.5637s (14.0422GiB/s) | 20260508 06:10:58.775 | INFO | Downloaded 34GiB to GPU in 2.41199s (14.0963GiB/s) | 20260508 06:11:03.917 | INFO | Analyzing chunk frames = [5851136,5851648) | 20260508 06:11:03.917 | INFO | MemInfo: { | 20260508 06:11:03.917 | INFO | "gpu" : " 17 GB", | 20260508 06:11:03.917 | INFO | "huge" : "", | 20260508 06:11:03.917 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:03.917 | INFO | "malloc" : " 14 GB", | 20260508 06:11:03.917 | INFO | "pinned" : " 29 GB" | 20260508 06:11:03.917 | INFO | } | 20260508 06:11:03.917 | INFO | Perf report for the last 5.142 seconds: | 20260508 06:11:03.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:03.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:11:03.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:11:03.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:03.917 | INFO | Uploaded 36GiB to GPU in 2.38725s (15.0801GiB/s) | 20260508 06:11:03.917 | INFO | Downloaded 34GiB to GPU in 2.48877s (13.6614GiB/s) | 20260508 06:11:09.058 | INFO | Analyzing chunk frames = [5851648,5852160) | 20260508 06:11:09.058 | INFO | MemInfo: { | 20260508 06:11:09.058 | INFO | "gpu" : " 17 GB", | 20260508 06:11:09.058 | INFO | "huge" : "", | 20260508 06:11:09.058 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:09.058 | INFO | "malloc" : " 14 GB", | 20260508 06:11:09.058 | INFO | "pinned" : " 29 GB" | 20260508 06:11:09.058 | INFO | } | 20260508 06:11:09.058 | INFO | Perf report for the last 5.14103 seconds: | 20260508 06:11:09.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:09.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:11:09.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:11:09.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:09.058 | INFO | Uploaded 36GiB to GPU in 2.45924s (14.6386GiB/s) | 20260508 06:11:09.058 | INFO | Downloaded 34GiB to GPU in 2.42149s (14.041GiB/s) | 20260508 06:11:14.192 | INFO | Analyzing chunk frames = [5852160,5852672) | 20260508 06:11:14.192 | INFO | MemInfo: { | 20260508 06:11:14.192 | INFO | "gpu" : " 17 GB", | 20260508 06:11:14.192 | INFO | "huge" : "", | 20260508 06:11:14.192 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:14.192 | INFO | "malloc" : " 14 GB", | 20260508 06:11:14.193 | INFO | "pinned" : " 29 GB" | 20260508 06:11:14.193 | INFO | } | 20260508 06:11:14.193 | INFO | Perf report for the last 5.13471 seconds: | 20260508 06:11:14.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:14.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:11:14.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:11:14.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:14.193 | INFO | Uploaded 36GiB to GPU in 2.49523s (14.4275GiB/s) | 20260508 06:11:14.193 | INFO | Downloaded 34GiB to GPU in 2.43089s (13.9867GiB/s) | 20260508 06:11:19.330 | INFO | Analyzing chunk frames = [5852672,5853184) | 20260508 06:11:19.330 | INFO | MemInfo: { | 20260508 06:11:19.330 | INFO | "gpu" : " 17 GB", | 20260508 06:11:19.330 | INFO | "huge" : "", | 20260508 06:11:19.330 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:19.330 | INFO | "malloc" : " 14 GB", | 20260508 06:11:19.330 | INFO | "pinned" : " 29 GB" | 20260508 06:11:19.330 | INFO | } | 20260508 06:11:19.330 | INFO | Perf report for the last 5.13752 seconds: | 20260508 06:11:19.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:19.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.330 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:11:19.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.330 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 06:11:19.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:19.331 | INFO | Uploaded 36GiB to GPU in 2.48339s (14.4963GiB/s) | 20260508 06:11:19.331 | INFO | Downloaded 34GiB to GPU in 2.4338s (13.9699GiB/s) | 20260508 06:11:19.443 | INFO | Starting a round of Baz IO | 20260508 06:11:22.867 | INFO | Populating RTMetrics report | 20260508 06:11:22.867 | INFO | RTMetrics report populated | 20260508 06:11:22.867 | INFO | Opening tmp RTMetrics json file | 20260508 06:11:22.878 | INFO | Serializing RTMetrics to json | 20260508 06:11:22.879 | INFO | Serialization done, swapping tmp file now... | 20260508 06:11:22.879 | INFO | RTMetrics done | 20260508 06:11:22.936 | INFO | Finished a round of Baz IO: Wrote 1.556 GiB in 3.493 seconds | 20260508 06:11:22.936 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.00% / 59.26% | 20260508 06:11:22.936 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.41426% / 0% / 5.96046e-05% | 20260508 06:11:22.936 | INFO | Baz IO overall: | 20260508 06:11:22.936 | INFO | BytesWritten : 2599119404060 | 20260508 06:11:22.936 | INFO | NumEvents : 1253374729532 | 20260508 06:11:24.472 | INFO | Analyzing chunk frames = [5853184,5853696) | 20260508 06:11:24.472 | INFO | MemInfo: { | 20260508 06:11:24.472 | INFO | "gpu" : " 17 GB", | 20260508 06:11:24.472 | INFO | "huge" : "", | 20260508 06:11:24.472 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:24.472 | INFO | "malloc" : " 14 GB", | 20260508 06:11:24.472 | INFO | "pinned" : " 29 GB" | 20260508 06:11:24.472 | INFO | } | 20260508 06:11:24.472 | INFO | Perf report for the last 5.14212 seconds: | 20260508 06:11:24.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:24.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:24.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:24.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:24.472 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 06:11:24.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:24.472 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 06:11:24.472 | INFO | PRE_HQ : 66.4%, 1.8, 61.4ms, 0 | 20260508 06:11:24.472 | INFO | BAZWRITER : 67.4%, 2.8, 96.8ms, 0 | 20260508 06:11:24.472 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 06:11:24.473 | INFO | Uploaded 36GiB to GPU in 2.54532s (14.1436GiB/s) | 20260508 06:11:24.473 | INFO | Downloaded 38GiB to GPU in 2.69039s (14.1244GiB/s) | 20260508 06:11:29.619 | INFO | Analyzing chunk frames = [5853696,5854208) | 20260508 06:11:29.619 | INFO | MemInfo: { | 20260508 06:11:29.619 | INFO | "gpu" : " 17 GB", | 20260508 06:11:29.619 | INFO | "huge" : "", | 20260508 06:11:29.619 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:29.619 | INFO | "malloc" : " 14 GB", | 20260508 06:11:29.619 | INFO | "pinned" : " 29 GB" | 20260508 06:11:29.619 | INFO | } | 20260508 06:11:29.619 | INFO | Perf report for the last 5.14682 seconds: | 20260508 06:11:29.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:29.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | ANALYSIS : 66.9%, 3.0, 104.5ms, 0 | 20260508 06:11:29.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:11:29.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:29.619 | INFO | Uploaded 36GiB to GPU in 2.51608s (14.308GiB/s) | 20260508 06:11:29.619 | INFO | Downloaded 34GiB to GPU in 2.43941s (13.9378GiB/s) | 20260508 06:11:34.754 | INFO | Analyzing chunk frames = [5854208,5854720) | 20260508 06:11:34.754 | INFO | MemInfo: { | 20260508 06:11:34.754 | INFO | "gpu" : " 17 GB", | 20260508 06:11:34.754 | INFO | "huge" : "", | 20260508 06:11:34.754 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:34.754 | INFO | "malloc" : " 14 GB", | 20260508 06:11:34.754 | INFO | "pinned" : " 29 GB" | 20260508 06:11:34.754 | INFO | } | 20260508 06:11:34.754 | INFO | Perf report for the last 5.13531 seconds: | 20260508 06:11:34.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:34.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.755 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 06:11:34.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.755 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:11:34.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:34.756 | INFO | Uploaded 36GiB to GPU in 2.57573s (13.9766GiB/s) | 20260508 06:11:34.756 | INFO | Downloaded 34GiB to GPU in 2.42718s (14.008GiB/s) | 20260508 06:11:39.884 | INFO | Analyzing chunk frames = [5854720,5855232) | 20260508 06:11:39.884 | INFO | MemInfo: { | 20260508 06:11:39.884 | INFO | "gpu" : " 17 GB", | 20260508 06:11:39.884 | INFO | "huge" : "", | 20260508 06:11:39.884 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:39.884 | INFO | "malloc" : " 14 GB", | 20260508 06:11:39.884 | INFO | "pinned" : " 29 GB" | 20260508 06:11:39.884 | INFO | } | 20260508 06:11:39.884 | INFO | Perf report for the last 5.13001 seconds: | 20260508 06:11:39.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:39.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:11:39.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:11:39.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:39.885 | INFO | Uploaded 36GiB to GPU in 2.56876s (14.0145GiB/s) | 20260508 06:11:39.885 | INFO | Downloaded 34GiB to GPU in 2.3904s (14.2236GiB/s) | 20260508 06:11:45.032 | INFO | Analyzing chunk frames = [5855232,5855744) | 20260508 06:11:45.032 | INFO | MemInfo: { | 20260508 06:11:45.032 | INFO | "gpu" : " 17 GB", | 20260508 06:11:45.032 | INFO | "huge" : "", | 20260508 06:11:45.032 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:45.032 | INFO | "malloc" : " 14 GB", | 20260508 06:11:45.032 | INFO | "pinned" : " 29 GB" | 20260508 06:11:45.032 | INFO | } | 20260508 06:11:45.032 | INFO | Perf report for the last 5.14751 seconds: | 20260508 06:11:45.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:45.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:11:45.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 06:11:45.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:45.032 | INFO | Uploaded 36GiB to GPU in 2.32847s (15.4608GiB/s) | 20260508 06:11:45.032 | INFO | Downloaded 34GiB to GPU in 2.479s (13.7152GiB/s) | 20260508 06:11:50.169 | INFO | Analyzing chunk frames = [5855744,5856256) | 20260508 06:11:50.169 | INFO | MemInfo: { | 20260508 06:11:50.169 | INFO | "gpu" : " 17 GB", | 20260508 06:11:50.169 | INFO | "huge" : "", | 20260508 06:11:50.170 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:50.170 | INFO | "malloc" : " 14 GB", | 20260508 06:11:50.170 | INFO | "pinned" : " 29 GB" | 20260508 06:11:50.170 | INFO | } | 20260508 06:11:50.170 | INFO | Perf report for the last 5.13776 seconds: | 20260508 06:11:50.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:50.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 06:11:50.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:11:50.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:50.170 | INFO | Uploaded 36GiB to GPU in 2.45783s (14.6471GiB/s) | 20260508 06:11:50.170 | INFO | Downloaded 34GiB to GPU in 2.42379s (14.0276GiB/s) | 20260508 06:11:55.308 | INFO | Analyzing chunk frames = [5856256,5856768) | 20260508 06:11:55.308 | INFO | MemInfo: { | 20260508 06:11:55.308 | INFO | "gpu" : " 17 GB", | 20260508 06:11:55.308 | INFO | "huge" : "", | 20260508 06:11:55.308 | INFO | "huge_shared" : " 29 GB", | 20260508 06:11:55.308 | INFO | "malloc" : " 14 GB", | 20260508 06:11:55.308 | INFO | "pinned" : " 29 GB" | 20260508 06:11:55.308 | INFO | } | 20260508 06:11:55.308 | INFO | Perf report for the last 5.13862 seconds: | 20260508 06:11:55.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:11:55.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.309 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:11:55.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.309 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:11:55.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:11:55.309 | INFO | Uploaded 36GiB to GPU in 2.47027s (14.5733GiB/s) | 20260508 06:11:55.309 | INFO | Downloaded 34GiB to GPU in 2.42653s (14.0118GiB/s) | 20260508 06:12:00.446 | INFO | Analyzing chunk frames = [5856768,5857280) | 20260508 06:12:00.446 | INFO | MemInfo: { | 20260508 06:12:00.446 | INFO | "gpu" : " 17 GB", | 20260508 06:12:00.446 | INFO | "huge" : "", | 20260508 06:12:00.446 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:00.446 | INFO | "malloc" : " 14 GB", | 20260508 06:12:00.446 | INFO | "pinned" : " 29 GB" | 20260508 06:12:00.446 | INFO | } | 20260508 06:12:00.447 | INFO | Perf report for the last 5.13813 seconds: | 20260508 06:12:00.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:00.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:12:00.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:12:00.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:00.447 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:12:00.447 | INFO | Total mem usage: 16.7 GiB | 20260508 06:12:00.447 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:12:00.447 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:12:00.447 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:12:00.447 | INFO | High Water Breakdown: | 20260508 06:12:00.447 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:12:00.447 | INFO | Uploaded 36GiB to GPU in 2.46074s (14.6298GiB/s) | 20260508 06:12:00.447 | INFO | Downloaded 34GiB to GPU in 2.4298s (13.9929GiB/s) | 20260508 06:12:00.447 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:12:00.447 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:12:00.447 | INFO | ------------------------------------------ | 20260508 06:12:00.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:12:00.447 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:12:00.448 | INFO | Total mem usage: 26.5 GiB | 20260508 06:12:00.448 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:12:00.448 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:12:00.448 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:12:00.448 | INFO | High Water Breakdown: | 20260508 06:12:00.448 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:12:00.448 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:12:00.448 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:12:00.448 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:12:00.448 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:12:00.448 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:12:00.448 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:12:00.448 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:12:00.448 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:12:00.448 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:12:00.448 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:12:00.448 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:12:00.448 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:12:00.448 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:12:00.448 | INFO | ------------------------------------------ | 20260508 06:12:00.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:12:00.448 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:12:00.448 | INFO | Total mem usage: 27.8 GiB | 20260508 06:12:00.448 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:12:00.448 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:12:00.448 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:12:00.448 | INFO | High Water Breakdown: | 20260508 06:12:00.448 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:12:00.448 | INFO | ------------------------------------------ | 20260508 06:12:00.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:12:00.448 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:12:00.448 | INFO | Total mem usage: 15.7 GiB | 20260508 06:12:00.448 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:12:00.448 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:12:00.448 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:12:00.448 | INFO | High Water Breakdown: | 20260508 06:12:00.448 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:12:00.448 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:12:00.448 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:12:00.448 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:12:00.448 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:12:00.448 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:12:00.448 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:12:00.448 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:12:00.448 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:12:00.448 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:12:00.448 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:12:00.448 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:12:00.448 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:12:00.448 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:12:00.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:12:00.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:12:00.448 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:12:00.448 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:12:00.448 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:12:00.448 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:12:00.448 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:12:00.448 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:12:00.448 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:12:00.448 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:12:00.448 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:12:00.448 | INFO | ------------------------------------------ | 20260508 06:12:00.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:12:00.449 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:12:00.449 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:12:00.449 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:12:00.449 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 7.23e-01 | 20260508 06:12:00.449 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.05e-01, 1.10e-02 / 4.82e-03 | 20260508 06:12:00.449 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.37e+00, 5.87e-01 / 2.75e-01 | 20260508 06:12:00.449 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.10e-01 / 1.13e-01 | 20260508 06:12:00.449 | INFO | PerfReport: 15.93%, DME -- 4.16e+01 / 4.51e+01, 3.65e+00 / 9.70e-01 | 20260508 06:12:00.449 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.78e+00 / 1.24e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.39e+00 / 2.55e+00, 4.12e-01 / 2.13e-01 | 20260508 06:12:00.449 | INFO | PerfReport: 3.17%, Metrics -- 1.03e+00 / 1.26e+00, 2.57e-01 / 1.12e-01 | 20260508 06:12:00.449 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.16e-01 / 3.58e-01, 4.95e-02 / 1.74e-02 | 20260508 06:12:00.449 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:12:00.449 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:12:00.449 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:12:00.449 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 24.19%, Download -- 2.57e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 2.52%, LockDownload -- 2.67e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 15.63%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 06:12:00.449 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260508 06:12:00.544 | INFO | Starting a round of Baz IO | 20260508 06:12:03.970 | INFO | Populating RTMetrics report | 20260508 06:12:03.970 | INFO | RTMetrics report populated | 20260508 06:12:03.970 | INFO | Opening tmp RTMetrics json file | 20260508 06:12:03.982 | INFO | Serializing RTMetrics to json | 20260508 06:12:03.983 | INFO | Serialization done, swapping tmp file now... | 20260508 06:12:03.983 | INFO | RTMetrics done | 20260508 06:12:04.043 | INFO | Finished a round of Baz IO: Wrote 1.557 GiB in 3.498 seconds | 20260508 06:12:04.043 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.00% / 59.26% | 20260508 06:12:04.043 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.414232% / 3.97364e-06% / 7.15256e-05% | 20260508 06:12:04.043 | INFO | Baz IO overall: | 20260508 06:12:04.043 | INFO | BytesWritten : 2600791124345 | 20260508 06:12:04.043 | INFO | NumEvents : 1254171045896 | 20260508 06:12:05.587 | INFO | Analyzing chunk frames = [5857280,5857792) | 20260508 06:12:05.587 | INFO | MemInfo: { | 20260508 06:12:05.587 | INFO | "gpu" : " 17 GB", | 20260508 06:12:05.587 | INFO | "huge" : "", | 20260508 06:12:05.587 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:05.587 | INFO | "malloc" : " 14 GB", | 20260508 06:12:05.587 | INFO | "pinned" : " 29 GB" | 20260508 06:12:05.587 | INFO | } | 20260508 06:12:05.587 | INFO | Perf report for the last 5.14007 seconds: | 20260508 06:12:05.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:05.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:05.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:05.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:05.587 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:12:05.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:05.587 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 06:12:05.587 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 06:12:05.587 | INFO | BAZWRITER : 67.5%, 2.6, 91.3ms, 0 | 20260508 06:12:05.587 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:12:05.587 | INFO | Uploaded 36GiB to GPU in 2.59889s (13.8521GiB/s) | 20260508 06:12:05.587 | INFO | Downloaded 38GiB to GPU in 2.68728s (14.1407GiB/s) | 20260508 06:12:10.727 | INFO | Analyzing chunk frames = [5857792,5858304) | 20260508 06:12:10.727 | INFO | MemInfo: { | 20260508 06:12:10.727 | INFO | "gpu" : " 17 GB", | 20260508 06:12:10.727 | INFO | "huge" : "", | 20260508 06:12:10.727 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:10.727 | INFO | "malloc" : " 14 GB", | 20260508 06:12:10.727 | INFO | "pinned" : " 29 GB" | 20260508 06:12:10.727 | INFO | } | 20260508 06:12:10.727 | INFO | Perf report for the last 5.14055 seconds: | 20260508 06:12:10.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:10.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.727 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:12:10.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.727 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:12:10.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:10.728 | INFO | Uploaded 36GiB to GPU in 2.51943s (14.289GiB/s) | 20260508 06:12:10.728 | INFO | Downloaded 34GiB to GPU in 2.43607s (13.9569GiB/s) | 20260508 06:12:15.866 | INFO | Analyzing chunk frames = [5858304,5858816) | 20260508 06:12:15.866 | INFO | MemInfo: { | 20260508 06:12:15.866 | INFO | "gpu" : " 17 GB", | 20260508 06:12:15.866 | INFO | "huge" : "", | 20260508 06:12:15.866 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:15.866 | INFO | "malloc" : " 14 GB", | 20260508 06:12:15.866 | INFO | "pinned" : " 29 GB" | 20260508 06:12:15.867 | INFO | } | 20260508 06:12:15.867 | INFO | Perf report for the last 5.13936 seconds: | 20260508 06:12:15.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:15.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 06:12:15.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:12:15.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:15.867 | INFO | Uploaded 36GiB to GPU in 2.55247s (14.104GiB/s) | 20260508 06:12:15.867 | INFO | Downloaded 34GiB to GPU in 2.40473s (14.1388GiB/s) | 20260508 06:12:21.004 | INFO | Analyzing chunk frames = [5858816,5859328) | 20260508 06:12:21.004 | INFO | MemInfo: { | 20260508 06:12:21.004 | INFO | "gpu" : " 17 GB", | 20260508 06:12:21.004 | INFO | "huge" : "", | 20260508 06:12:21.004 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:21.004 | INFO | "malloc" : " 14 GB", | 20260508 06:12:21.004 | INFO | "pinned" : " 29 GB" | 20260508 06:12:21.004 | INFO | } | 20260508 06:12:21.004 | INFO | Perf report for the last 5.13709 seconds: | 20260508 06:12:21.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:21.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:12:21.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:12:21.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:21.004 | INFO | Uploaded 36GiB to GPU in 2.56949s (14.0106GiB/s) | 20260508 06:12:21.004 | INFO | Downloaded 34GiB to GPU in 2.38627s (14.2482GiB/s) | 20260508 06:12:26.140 | INFO | Analyzing chunk frames = [5859328,5859840) | 20260508 06:12:26.140 | INFO | MemInfo: { | 20260508 06:12:26.140 | INFO | "gpu" : " 17 GB", | 20260508 06:12:26.140 | INFO | "huge" : "", | 20260508 06:12:26.140 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:26.140 | INFO | "malloc" : " 14 GB", | 20260508 06:12:26.140 | INFO | "pinned" : " 29 GB" | 20260508 06:12:26.140 | INFO | } | 20260508 06:12:26.140 | INFO | Perf report for the last 5.13601 seconds: | 20260508 06:12:26.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:26.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 06:12:26.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:12:26.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:26.140 | INFO | Uploaded 36GiB to GPU in 2.42232s (14.8618GiB/s) | 20260508 06:12:26.140 | INFO | Downloaded 34GiB to GPU in 2.56978s (13.2307GiB/s) | 20260508 06:12:31.281 | INFO | Analyzing chunk frames = [5859840,5860352) | 20260508 06:12:31.281 | INFO | MemInfo: { | 20260508 06:12:31.281 | INFO | "gpu" : " 17 GB", | 20260508 06:12:31.281 | INFO | "huge" : "", | 20260508 06:12:31.281 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:31.281 | INFO | "malloc" : " 14 GB", | 20260508 06:12:31.281 | INFO | "pinned" : " 29 GB" | 20260508 06:12:31.281 | INFO | } | 20260508 06:12:31.281 | INFO | Perf report for the last 5.14093 seconds: | 20260508 06:12:31.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:31.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 06:12:31.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 06:12:31.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:31.281 | INFO | Uploaded 36GiB to GPU in 2.45758s (14.6485GiB/s) | 20260508 06:12:31.281 | INFO | Downloaded 34GiB to GPU in 2.42034s (14.0476GiB/s) | 20260508 06:12:36.427 | INFO | Analyzing chunk frames = [5860352,5860864) | 20260508 06:12:36.427 | INFO | MemInfo: { | 20260508 06:12:36.427 | INFO | "gpu" : " 17 GB", | 20260508 06:12:36.427 | INFO | "huge" : "", | 20260508 06:12:36.427 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:36.427 | INFO | "malloc" : " 14 GB", | 20260508 06:12:36.427 | INFO | "pinned" : " 29 GB" | 20260508 06:12:36.427 | INFO | } | 20260508 06:12:36.427 | INFO | Perf report for the last 5.14676 seconds: | 20260508 06:12:36.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:36.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 06:12:36.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:12:36.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:36.428 | INFO | Uploaded 36GiB to GPU in 2.46278s (14.6176GiB/s) | 20260508 06:12:36.428 | INFO | Downloaded 34GiB to GPU in 2.41422s (14.0832GiB/s) | 20260508 06:12:41.563 | INFO | Analyzing chunk frames = [5860864,5861376) | 20260508 06:12:41.563 | INFO | MemInfo: { | 20260508 06:12:41.563 | INFO | "gpu" : " 17 GB", | 20260508 06:12:41.563 | INFO | "huge" : "", | 20260508 06:12:41.563 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:41.563 | INFO | "malloc" : " 14 GB", | 20260508 06:12:41.563 | INFO | "pinned" : " 29 GB" | 20260508 06:12:41.563 | INFO | } | 20260508 06:12:41.563 | INFO | Perf report for the last 5.13554 seconds: | 20260508 06:12:41.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:41.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:12:41.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:12:41.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:41.563 | INFO | Uploaded 36GiB to GPU in 2.48055s (14.5129GiB/s) | 20260508 06:12:41.563 | INFO | Downloaded 34GiB to GPU in 2.43254s (13.9771GiB/s) | 20260508 06:12:41.673 | INFO | Starting a round of Baz IO | 20260508 06:12:45.090 | INFO | Populating RTMetrics report | 20260508 06:12:45.090 | INFO | RTMetrics report populated | 20260508 06:12:45.090 | INFO | Opening tmp RTMetrics json file | 20260508 06:12:45.100 | INFO | Serializing RTMetrics to json | 20260508 06:12:45.102 | INFO | Serialization done, swapping tmp file now... | 20260508 06:12:45.102 | INFO | RTMetrics done | 20260508 06:12:45.159 | INFO | Finished a round of Baz IO: Wrote 1.558 GiB in 3.485 seconds | 20260508 06:12:45.159 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.00% / 59.26% | 20260508 06:12:45.159 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415051% / 0% / 5.96046e-05% | 20260508 06:12:45.159 | INFO | Baz IO overall: | 20260508 06:12:45.159 | INFO | BytesWritten : 2602463510128 | 20260508 06:12:45.159 | INFO | NumEvents : 1254967965498 | 20260508 06:12:46.702 | INFO | Analyzing chunk frames = [5861376,5861888) | 20260508 06:12:46.702 | INFO | MemInfo: { | 20260508 06:12:46.702 | INFO | "gpu" : " 17 GB", | 20260508 06:12:46.702 | INFO | "huge" : "", | 20260508 06:12:46.702 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:46.702 | INFO | "malloc" : " 14 GB", | 20260508 06:12:46.702 | INFO | "pinned" : " 29 GB" | 20260508 06:12:46.702 | INFO | } | 20260508 06:12:46.702 | INFO | Perf report for the last 5.13917 seconds: | 20260508 06:12:46.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:46.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:46.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:46.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:46.702 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 06:12:46.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:46.702 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 06:12:46.702 | INFO | PRE_HQ : 65.5%, 1.6, 55.6ms, 0 | 20260508 06:12:46.702 | INFO | BAZWRITER : 67.4%, 2.6, 91.2ms, 0 | 20260508 06:12:46.702 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 06:12:46.702 | INFO | Uploaded 36GiB to GPU in 2.55444s (14.0931GiB/s) | 20260508 06:12:46.702 | INFO | Downloaded 38GiB to GPU in 2.67721s (14.1939GiB/s) | 20260508 06:12:51.838 | INFO | Analyzing chunk frames = [5861888,5862400) | 20260508 06:12:51.838 | INFO | MemInfo: { | 20260508 06:12:51.838 | INFO | "gpu" : " 17 GB", | 20260508 06:12:51.838 | INFO | "huge" : "", | 20260508 06:12:51.838 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:51.838 | INFO | "malloc" : " 14 GB", | 20260508 06:12:51.838 | INFO | "pinned" : " 29 GB" | 20260508 06:12:51.838 | INFO | } | 20260508 06:12:51.838 | INFO | Perf report for the last 5.13583 seconds: | 20260508 06:12:51.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:51.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 06:12:51.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:12:51.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:51.838 | INFO | Uploaded 36GiB to GPU in 2.51433s (14.3179GiB/s) | 20260508 06:12:51.838 | INFO | Downloaded 34GiB to GPU in 2.4318s (13.9814GiB/s) | 20260508 06:12:56.981 | INFO | Analyzing chunk frames = [5862400,5862912) | 20260508 06:12:56.981 | INFO | MemInfo: { | 20260508 06:12:56.981 | INFO | "gpu" : " 17 GB", | 20260508 06:12:56.981 | INFO | "huge" : "", | 20260508 06:12:56.981 | INFO | "huge_shared" : " 29 GB", | 20260508 06:12:56.981 | INFO | "malloc" : " 14 GB", | 20260508 06:12:56.981 | INFO | "pinned" : " 29 GB" | 20260508 06:12:56.981 | INFO | } | 20260508 06:12:56.981 | INFO | Perf report for the last 5.14323 seconds: | 20260508 06:12:56.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:12:56.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.981 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:12:56.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.981 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:12:56.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:12:56.982 | INFO | Uploaded 36GiB to GPU in 2.55995s (14.0628GiB/s) | 20260508 06:12:56.982 | INFO | Downloaded 34GiB to GPU in 2.40807s (14.1192GiB/s) | 20260508 06:13:02.125 | INFO | Analyzing chunk frames = [5862912,5863424) | 20260508 06:13:02.125 | INFO | MemInfo: { | 20260508 06:13:02.125 | INFO | "gpu" : " 17 GB", | 20260508 06:13:02.125 | INFO | "huge" : "", | 20260508 06:13:02.125 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:02.125 | INFO | "malloc" : " 14 GB", | 20260508 06:13:02.125 | INFO | "pinned" : " 29 GB" | 20260508 06:13:02.125 | INFO | } | 20260508 06:13:02.125 | INFO | Perf report for the last 5.1441 seconds: | 20260508 06:13:02.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:02.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.125 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 06:13:02.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.125 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:13:02.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:02.126 | INFO | Uploaded 36GiB to GPU in 2.55999s (14.0625GiB/s) | 20260508 06:13:02.126 | INFO | Downloaded 34GiB to GPU in 2.39305s (14.2078GiB/s) | 20260508 06:13:07.261 | INFO | Analyzing chunk frames = [5863424,5863936) | 20260508 06:13:07.261 | INFO | MemInfo: { | 20260508 06:13:07.261 | INFO | "gpu" : " 17 GB", | 20260508 06:13:07.261 | INFO | "huge" : "", | 20260508 06:13:07.262 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:07.262 | INFO | "malloc" : " 14 GB", | 20260508 06:13:07.262 | INFO | "pinned" : " 29 GB" | 20260508 06:13:07.262 | INFO | } | 20260508 06:13:07.262 | INFO | Perf report for the last 5.13621 seconds: | 20260508 06:13:07.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:07.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:13:07.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 06:13:07.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:07.262 | INFO | Uploaded 36GiB to GPU in 2.28475s (15.7567GiB/s) | 20260508 06:13:07.262 | INFO | Downloaded 34GiB to GPU in 2.49648s (13.6192GiB/s) | 20260508 06:13:12.396 | INFO | Analyzing chunk frames = [5863936,5864448) | 20260508 06:13:12.396 | INFO | MemInfo: { | 20260508 06:13:12.396 | INFO | "gpu" : " 17 GB", | 20260508 06:13:12.396 | INFO | "huge" : "", | 20260508 06:13:12.396 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:12.396 | INFO | "malloc" : " 14 GB", | 20260508 06:13:12.396 | INFO | "pinned" : " 29 GB" | 20260508 06:13:12.396 | INFO | } | 20260508 06:13:12.396 | INFO | Perf report for the last 5.13449 seconds: | 20260508 06:13:12.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:12.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.396 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:13:12.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.396 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:13:12.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:12.397 | INFO | Uploaded 36GiB to GPU in 2.45874s (14.6416GiB/s) | 20260508 06:13:12.397 | INFO | Downloaded 34GiB to GPU in 2.42491s (14.0211GiB/s) | 20260508 06:13:17.534 | INFO | Analyzing chunk frames = [5864448,5864960) | 20260508 06:13:17.534 | INFO | MemInfo: { | 20260508 06:13:17.534 | INFO | "gpu" : " 17 GB", | 20260508 06:13:17.534 | INFO | "huge" : "", | 20260508 06:13:17.534 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:17.534 | INFO | "malloc" : " 14 GB", | 20260508 06:13:17.534 | INFO | "pinned" : " 29 GB" | 20260508 06:13:17.534 | INFO | } | 20260508 06:13:17.535 | INFO | Perf report for the last 5.13842 seconds: | 20260508 06:13:17.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:17.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:13:17.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:13:17.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:17.535 | INFO | Uploaded 36GiB to GPU in 2.46985s (14.5758GiB/s) | 20260508 06:13:17.535 | INFO | Downloaded 34GiB to GPU in 2.42755s (14.0059GiB/s) | 20260508 06:13:22.677 | INFO | Analyzing chunk frames = [5864960,5865472) | 20260508 06:13:22.677 | INFO | MemInfo: { | 20260508 06:13:22.677 | INFO | "gpu" : " 17 GB", | 20260508 06:13:22.677 | INFO | "huge" : "", | 20260508 06:13:22.677 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:22.677 | INFO | "malloc" : " 14 GB", | 20260508 06:13:22.677 | INFO | "pinned" : " 29 GB" | 20260508 06:13:22.677 | INFO | } | 20260508 06:13:22.677 | INFO | Perf report for the last 5.14282 seconds: | 20260508 06:13:22.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:22.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.677 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:13:22.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.677 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:13:22.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:22.678 | INFO | Uploaded 36GiB to GPU in 2.44579s (14.7192GiB/s) | 20260508 06:13:22.678 | INFO | Downloaded 34GiB to GPU in 2.42341s (14.0298GiB/s) | 20260508 06:13:22.678 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:13:22.678 | INFO | Total mem usage: 16.7 GiB | 20260508 06:13:22.678 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:13:22.678 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 06:13:22.678 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:13:22.678 | INFO | High Water Breakdown: | 20260508 06:13:22.678 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:13:22.678 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:13:22.678 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:13:22.678 | INFO | ------------------------------------------ | 20260508 06:13:22.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:13:22.678 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:13:22.678 | INFO | Total mem usage: 26.5 GiB | 20260508 06:13:22.678 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:13:22.678 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:13:22.678 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:13:22.678 | INFO | High Water Breakdown: | 20260508 06:13:22.678 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:13:22.678 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:13:22.678 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:13:22.678 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:13:22.678 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:13:22.678 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:13:22.678 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:13:22.678 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:13:22.678 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:13:22.678 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:13:22.678 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:13:22.678 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:13:22.678 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:13:22.678 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:13:22.678 | INFO | ------------------------------------------ | 20260508 06:13:22.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:13:22.678 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:13:22.678 | INFO | Total mem usage: 27.8 GiB | 20260508 06:13:22.678 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:13:22.678 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:13:22.678 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:13:22.678 | INFO | High Water Breakdown: | 20260508 06:13:22.678 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:13:22.678 | INFO | ------------------------------------------ | 20260508 06:13:22.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:13:22.678 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:13:22.678 | INFO | Total mem usage: 15.7 GiB | 20260508 06:13:22.678 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:13:22.678 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:13:22.678 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:13:22.678 | INFO | High Water Breakdown: | 20260508 06:13:22.678 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:13:22.678 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:13:22.678 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:13:22.678 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:13:22.679 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:13:22.679 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:13:22.679 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:13:22.679 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:13:22.679 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:13:22.679 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:13:22.679 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:13:22.679 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:13:22.679 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:13:22.679 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:13:22.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:13:22.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:13:22.679 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:13:22.679 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:13:22.679 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:13:22.679 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:13:22.679 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:13:22.679 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:13:22.679 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:13:22.679 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:13:22.679 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:13:22.679 | INFO | ------------------------------------------ | 20260508 06:13:22.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:13:22.679 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:13:22.679 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:13:22.679 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:13:22.679 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.58e+01, 1.41e+01 / 1.24e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.05e-01, 1.09e-02 / 4.48e-03 | 20260508 06:13:22.679 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.84e-01 / 2.13e-01 | 20260508 06:13:22.679 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.46e+00, 2.09e-01 / 1.07e-01 | 20260508 06:13:22.679 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.56e+01, 3.84e+00 / 1.72e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.39e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.57e+00, 4.13e-01 / 2.33e-01 | 20260508 06:13:22.679 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.24e+00, 2.57e-01 / 1.31e-01 | 20260508 06:13:22.679 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.50e-01, 4.92e-02 / 2.33e-02 | 20260508 06:13:22.679 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:13:22.679 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:13:22.679 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:13:22.679 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 24.18%, Download -- 2.57e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 2.44%, LockUpload -- 2.59e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 2.47%, LockDownload -- 2.62e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:13:22.679 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260508 06:13:22.778 | INFO | Starting a round of Baz IO | 20260508 06:13:26.208 | INFO | Populating RTMetrics report | 20260508 06:13:26.209 | INFO | RTMetrics report populated | 20260508 06:13:26.209 | INFO | Opening tmp RTMetrics json file | 20260508 06:13:26.220 | INFO | Serializing RTMetrics to json | 20260508 06:13:26.221 | INFO | Serialization done, swapping tmp file now... | 20260508 06:13:26.222 | INFO | RTMetrics done | 20260508 06:13:26.276 | INFO | Finished a round of Baz IO: Wrote 1.557 GiB in 3.497 seconds | 20260508 06:13:26.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.00% / 59.26% | 20260508 06:13:26.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.413656% / 3.97364e-06% / 6.75519e-05% | 20260508 06:13:26.276 | INFO | Baz IO overall: | 20260508 06:13:26.276 | INFO | BytesWritten : 2604135672401 | 20260508 06:13:26.276 | INFO | NumEvents : 1255764772907 | 20260508 06:13:27.812 | INFO | Analyzing chunk frames = [5865472,5865984) | 20260508 06:13:27.812 | INFO | MemInfo: { | 20260508 06:13:27.812 | INFO | "gpu" : " 17 GB", | 20260508 06:13:27.812 | INFO | "huge" : "", | 20260508 06:13:27.812 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:27.812 | INFO | "malloc" : " 14 GB", | 20260508 06:13:27.812 | INFO | "pinned" : " 29 GB" | 20260508 06:13:27.812 | INFO | } | 20260508 06:13:27.812 | INFO | Perf report for the last 5.1347 seconds: | 20260508 06:13:27.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:27.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:27.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:27.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:27.812 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:13:27.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:27.812 | INFO | RT_METRICS : 39.4%, 1.0, 20.6ms, 0 | 20260508 06:13:27.812 | INFO | PRE_HQ : 65.6%, 1.6, 55.5ms, 0 | 20260508 06:13:27.812 | INFO | BAZWRITER : 67.6%, 2.6, 91.1ms, 0 | 20260508 06:13:27.812 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 06:13:27.812 | INFO | Uploaded 36GiB to GPU in 2.60202s (13.8354GiB/s) | 20260508 06:13:27.812 | INFO | Downloaded 38GiB to GPU in 2.67585s (14.2011GiB/s) | 20260508 06:13:32.951 | INFO | Analyzing chunk frames = [5865984,5866496) | 20260508 06:13:32.951 | INFO | MemInfo: { | 20260508 06:13:32.951 | INFO | "gpu" : " 17 GB", | 20260508 06:13:32.951 | INFO | "huge" : "", | 20260508 06:13:32.951 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:32.951 | INFO | "malloc" : " 14 GB", | 20260508 06:13:32.951 | INFO | "pinned" : " 29 GB" | 20260508 06:13:32.951 | INFO | } | 20260508 06:13:32.951 | INFO | Perf report for the last 5.13921 seconds: | 20260508 06:13:32.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:32.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.951 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 06:13:32.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.951 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:13:32.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:32.952 | INFO | Uploaded 36GiB to GPU in 2.5215s (14.2772GiB/s) | 20260508 06:13:32.952 | INFO | Downloaded 34GiB to GPU in 2.42652s (14.0119GiB/s) | 20260508 06:13:38.095 | INFO | Analyzing chunk frames = [5866496,5867008) | 20260508 06:13:38.095 | INFO | MemInfo: { | 20260508 06:13:38.095 | INFO | "gpu" : " 17 GB", | 20260508 06:13:38.095 | INFO | "huge" : "", | 20260508 06:13:38.096 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:38.096 | INFO | "malloc" : " 14 GB", | 20260508 06:13:38.096 | INFO | "pinned" : " 29 GB" | 20260508 06:13:38.096 | INFO | } | 20260508 06:13:38.096 | INFO | Perf report for the last 5.14433 seconds: | 20260508 06:13:38.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:38.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 06:13:38.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 06:13:38.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:38.096 | INFO | Uploaded 36GiB to GPU in 2.54629s (14.1382GiB/s) | 20260508 06:13:38.096 | INFO | Downloaded 34GiB to GPU in 2.40433s (14.1411GiB/s) | 20260508 06:13:43.236 | INFO | Analyzing chunk frames = [5867008,5867520) | 20260508 06:13:43.236 | INFO | MemInfo: { | 20260508 06:13:43.236 | INFO | "gpu" : " 17 GB", | 20260508 06:13:43.236 | INFO | "huge" : "", | 20260508 06:13:43.236 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:43.236 | INFO | "malloc" : " 14 GB", | 20260508 06:13:43.236 | INFO | "pinned" : " 29 GB" | 20260508 06:13:43.236 | INFO | } | 20260508 06:13:43.236 | INFO | Perf report for the last 5.14059 seconds: | 20260508 06:13:43.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:43.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 06:13:43.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:13:43.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:43.236 | INFO | Uploaded 36GiB to GPU in 2.5919s (13.8894GiB/s) | 20260508 06:13:43.237 | INFO | Downloaded 34GiB to GPU in 2.40095s (14.1611GiB/s) | 20260508 06:13:48.378 | INFO | Analyzing chunk frames = [5867520,5868032) | 20260508 06:13:48.378 | INFO | MemInfo: { | 20260508 06:13:48.378 | INFO | "gpu" : " 17 GB", | 20260508 06:13:48.378 | INFO | "huge" : "", | 20260508 06:13:48.378 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:48.378 | INFO | "malloc" : " 14 GB", | 20260508 06:13:48.378 | INFO | "pinned" : " 29 GB" | 20260508 06:13:48.378 | INFO | } | 20260508 06:13:48.378 | INFO | Perf report for the last 5.14198 seconds: | 20260508 06:13:48.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:48.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.378 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:13:48.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.378 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:13:48.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:48.379 | INFO | Uploaded 36GiB to GPU in 2.31599s (15.5441GiB/s) | 20260508 06:13:48.379 | INFO | Downloaded 34GiB to GPU in 2.48789s (13.6662GiB/s) | 20260508 06:13:53.514 | INFO | Analyzing chunk frames = [5868032,5868544) | 20260508 06:13:53.514 | INFO | MemInfo: { | 20260508 06:13:53.514 | INFO | "gpu" : " 17 GB", | 20260508 06:13:53.514 | INFO | "huge" : "", | 20260508 06:13:53.514 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:53.514 | INFO | "malloc" : " 14 GB", | 20260508 06:13:53.514 | INFO | "pinned" : " 29 GB" | 20260508 06:13:53.514 | INFO | } | 20260508 06:13:53.514 | INFO | Perf report for the last 5.13609 seconds: | 20260508 06:13:53.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:53.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.514 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:13:53.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.514 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:13:53.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:53.515 | INFO | Uploaded 36GiB to GPU in 2.4489s (14.7005GiB/s) | 20260508 06:13:53.515 | INFO | Downloaded 34GiB to GPU in 2.42893s (13.998GiB/s) | 20260508 06:13:58.649 | INFO | Analyzing chunk frames = [5868544,5869056) | 20260508 06:13:58.649 | INFO | MemInfo: { | 20260508 06:13:58.649 | INFO | "gpu" : " 17 GB", | 20260508 06:13:58.649 | INFO | "huge" : "", | 20260508 06:13:58.649 | INFO | "huge_shared" : " 29 GB", | 20260508 06:13:58.649 | INFO | "malloc" : " 14 GB", | 20260508 06:13:58.649 | INFO | "pinned" : " 29 GB" | 20260508 06:13:58.649 | INFO | } | 20260508 06:13:58.649 | INFO | Perf report for the last 5.13502 seconds: | 20260508 06:13:58.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:13:58.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.649 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 06:13:58.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.649 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:13:58.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:13:58.650 | INFO | Uploaded 36GiB to GPU in 2.46183s (14.6233GiB/s) | 20260508 06:13:58.650 | INFO | Downloaded 34GiB to GPU in 2.4182s (14.06GiB/s) | 20260508 06:14:03.790 | INFO | Analyzing chunk frames = [5869056,5869568) | 20260508 06:14:03.790 | INFO | MemInfo: { | 20260508 06:14:03.790 | INFO | "gpu" : " 17 GB", | 20260508 06:14:03.790 | INFO | "huge" : "", | 20260508 06:14:03.791 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:03.791 | INFO | "malloc" : " 14 GB", | 20260508 06:14:03.791 | INFO | "pinned" : " 29 GB" | 20260508 06:14:03.791 | INFO | } | 20260508 06:14:03.791 | INFO | Perf report for the last 5.14132 seconds: | 20260508 06:14:03.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:03.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.791 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:14:03.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.791 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:14:03.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:03.792 | INFO | Uploaded 36GiB to GPU in 2.48229s (14.5027GiB/s) | 20260508 06:14:03.792 | INFO | Downloaded 34GiB to GPU in 2.44328s (13.9157GiB/s) | 20260508 06:14:03.888 | INFO | Starting a round of Baz IO | 20260508 06:14:07.307 | INFO | Populating RTMetrics report | 20260508 06:14:07.308 | INFO | RTMetrics report populated | 20260508 06:14:07.308 | INFO | Opening tmp RTMetrics json file | 20260508 06:14:07.316 | INFO | Serializing RTMetrics to json | 20260508 06:14:07.318 | INFO | Serialization done, swapping tmp file now... | 20260508 06:14:07.318 | INFO | RTMetrics done | 20260508 06:14:07.377 | INFO | Finished a round of Baz IO: Wrote 1.556 GiB in 3.488 seconds | 20260508 06:14:07.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.00% / 59.26% | 20260508 06:14:07.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.412329% / 0% / 4.76837e-05% | 20260508 06:14:07.377 | INFO | Baz IO overall: | 20260508 06:14:07.377 | INFO | BytesWritten : 2605806668360 | 20260508 06:14:07.377 | INFO | NumEvents : 1256560620010 | 20260508 06:14:08.934 | INFO | Analyzing chunk frames = [5869568,5870080) | 20260508 06:14:08.934 | INFO | MemInfo: { | 20260508 06:14:08.934 | INFO | "gpu" : " 17 GB", | 20260508 06:14:08.934 | INFO | "huge" : "", | 20260508 06:14:08.934 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:08.934 | INFO | "malloc" : " 14 GB", | 20260508 06:14:08.934 | INFO | "pinned" : " 29 GB" | 20260508 06:14:08.934 | INFO | } | 20260508 06:14:08.934 | INFO | Perf report for the last 5.14326 seconds: | 20260508 06:14:08.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:08.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:08.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:08.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:08.934 | INFO | ANALYSIS : 67.4%, 3.0, 105.6ms, 0 | 20260508 06:14:08.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:08.934 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:14:08.934 | INFO | PRE_HQ : 65.6%, 1.6, 55.5ms, 0 | 20260508 06:14:08.934 | INFO | BAZWRITER : 67.2%, 2.7, 92.0ms, 0 | 20260508 06:14:08.934 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:14:08.934 | INFO | Uploaded 36GiB to GPU in 2.58056s (13.9505GiB/s) | 20260508 06:14:08.934 | INFO | Downloaded 38GiB to GPU in 2.6889s (14.1322GiB/s) | 20260508 06:14:14.077 | INFO | Analyzing chunk frames = [5870080,5870592) | 20260508 06:14:14.077 | INFO | MemInfo: { | 20260508 06:14:14.077 | INFO | "gpu" : " 17 GB", | 20260508 06:14:14.077 | INFO | "huge" : "", | 20260508 06:14:14.077 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:14.077 | INFO | "malloc" : " 14 GB", | 20260508 06:14:14.077 | INFO | "pinned" : " 29 GB" | 20260508 06:14:14.077 | INFO | } | 20260508 06:14:14.078 | INFO | Perf report for the last 5.14352 seconds: | 20260508 06:14:14.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:14.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:14:14.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 06:14:14.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:14.078 | INFO | Uploaded 36GiB to GPU in 2.50039s (14.3978GiB/s) | 20260508 06:14:14.078 | INFO | Downloaded 34GiB to GPU in 2.44271s (13.919GiB/s) | 20260508 06:14:19.212 | INFO | Analyzing chunk frames = [5870592,5871104) | 20260508 06:14:19.212 | INFO | MemInfo: { | 20260508 06:14:19.212 | INFO | "gpu" : " 17 GB", | 20260508 06:14:19.212 | INFO | "huge" : "", | 20260508 06:14:19.213 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:19.213 | INFO | "malloc" : " 14 GB", | 20260508 06:14:19.213 | INFO | "pinned" : " 29 GB" | 20260508 06:14:19.213 | INFO | } | 20260508 06:14:19.213 | INFO | Perf report for the last 5.13518 seconds: | 20260508 06:14:19.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:19.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.213 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 06:14:19.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.213 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 06:14:19.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:19.214 | INFO | Uploaded 36GiB to GPU in 2.48535s (14.4849GiB/s) | 20260508 06:14:19.214 | INFO | Downloaded 34GiB to GPU in 2.4414s (13.9265GiB/s) | 20260508 06:14:24.348 | INFO | Analyzing chunk frames = [5871104,5871616) | 20260508 06:14:24.348 | INFO | MemInfo: { | 20260508 06:14:24.348 | INFO | "gpu" : " 17 GB", | 20260508 06:14:24.348 | INFO | "huge" : "", | 20260508 06:14:24.348 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:24.348 | INFO | "malloc" : " 14 GB", | 20260508 06:14:24.348 | INFO | "pinned" : " 29 GB" | 20260508 06:14:24.348 | INFO | } | 20260508 06:14:24.348 | INFO | Perf report for the last 5.13529 seconds: | 20260508 06:14:24.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:24.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | ANALYSIS : 66.9%, 3.0, 103.4ms, 0 | 20260508 06:14:24.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 06:14:24.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:24.348 | INFO | Uploaded 36GiB to GPU in 2.57138s (14.0003GiB/s) | 20260508 06:14:24.349 | INFO | Downloaded 34GiB to GPU in 2.43506s (13.9627GiB/s) | 20260508 06:14:29.489 | INFO | Analyzing chunk frames = [5871616,5872128) | 20260508 06:14:29.489 | INFO | MemInfo: { | 20260508 06:14:29.489 | INFO | "gpu" : " 17 GB", | 20260508 06:14:29.489 | INFO | "huge" : "", | 20260508 06:14:29.489 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:29.489 | INFO | "malloc" : " 14 GB", | 20260508 06:14:29.489 | INFO | "pinned" : " 29 GB" | 20260508 06:14:29.490 | INFO | } | 20260508 06:14:29.490 | INFO | Perf report for the last 5.14154 seconds: | 20260508 06:14:29.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:29.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 06:14:29.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260508 06:14:29.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:29.490 | INFO | Uploaded 36GiB to GPU in 2.3018s (15.6399GiB/s) | 20260508 06:14:29.490 | INFO | Downloaded 34GiB to GPU in 2.50045s (13.5975GiB/s) | 20260508 06:14:34.630 | INFO | Analyzing chunk frames = [5872128,5872640) | 20260508 06:14:34.630 | INFO | MemInfo: { | 20260508 06:14:34.630 | INFO | "gpu" : " 17 GB", | 20260508 06:14:34.630 | INFO | "huge" : "", | 20260508 06:14:34.630 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:34.631 | INFO | "malloc" : " 14 GB", | 20260508 06:14:34.631 | INFO | "pinned" : " 29 GB" | 20260508 06:14:34.631 | INFO | } | 20260508 06:14:34.631 | INFO | Perf report for the last 5.141 seconds: | 20260508 06:14:34.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:34.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:14:34.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:14:34.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:34.631 | INFO | Uploaded 36GiB to GPU in 2.46071s (14.6299GiB/s) | 20260508 06:14:34.631 | INFO | Downloaded 34GiB to GPU in 2.42424s (14.025GiB/s) | 20260508 06:14:39.766 | INFO | Analyzing chunk frames = [5872640,5873152) | 20260508 06:14:39.766 | INFO | MemInfo: { | 20260508 06:14:39.766 | INFO | "gpu" : " 17 GB", | 20260508 06:14:39.766 | INFO | "huge" : "", | 20260508 06:14:39.766 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:39.766 | INFO | "malloc" : " 14 GB", | 20260508 06:14:39.766 | INFO | "pinned" : " 29 GB" | 20260508 06:14:39.766 | INFO | } | 20260508 06:14:39.766 | INFO | Perf report for the last 5.13525 seconds: | 20260508 06:14:39.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:39.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.766 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:14:39.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.766 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:14:39.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:39.767 | INFO | Uploaded 36GiB to GPU in 2.47996s (14.5164GiB/s) | 20260508 06:14:39.767 | INFO | Downloaded 34GiB to GPU in 2.42202s (14.0379GiB/s) | 20260508 06:14:44.902 | INFO | Analyzing chunk frames = [5873152,5873664) | 20260508 06:14:44.902 | INFO | MemInfo: { | 20260508 06:14:44.902 | INFO | "gpu" : " 17 GB", | 20260508 06:14:44.902 | INFO | "huge" : "", | 20260508 06:14:44.902 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:44.902 | INFO | "malloc" : " 14 GB", | 20260508 06:14:44.903 | INFO | "pinned" : " 29 GB" | 20260508 06:14:44.903 | INFO | } | 20260508 06:14:44.903 | INFO | Perf report for the last 5.13681 seconds: | 20260508 06:14:44.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:44.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:14:44.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:14:44.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:44.903 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:14:44.903 | INFO | Total mem usage: 16.7 GiB | 20260508 06:14:44.903 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:14:44.903 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 06:14:44.903 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:14:44.903 | INFO | High Water Breakdown: | 20260508 06:14:44.903 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:14:44.903 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:14:44.903 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:14:44.903 | INFO | ------------------------------------------ | 20260508 06:14:44.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:14:44.903 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:14:44.903 | INFO | Total mem usage: 26.5 GiB | 20260508 06:14:44.903 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:14:44.903 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:14:44.903 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:14:44.903 | INFO | High Water Breakdown: | 20260508 06:14:44.903 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:14:44.903 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:14:44.903 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:14:44.903 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:14:44.903 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:14:44.903 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:14:44.903 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:14:44.903 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:14:44.903 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:14:44.903 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:14:44.903 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:14:44.903 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:14:44.903 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:14:44.903 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:14:44.903 | INFO | ------------------------------------------ | 20260508 06:14:44.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:14:44.903 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:14:44.903 | INFO | Total mem usage: 27.8 GiB | 20260508 06:14:44.903 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:14:44.903 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:14:44.903 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:14:44.903 | INFO | High Water Breakdown: | 20260508 06:14:44.903 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:14:44.903 | INFO | ------------------------------------------ | 20260508 06:14:44.903 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:14:44.903 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:14:44.903 | INFO | Total mem usage: 15.7 GiB | 20260508 06:14:44.903 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:14:44.903 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:14:44.903 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:14:44.903 | INFO | High Water Breakdown: | 20260508 06:14:44.903 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:14:44.903 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:14:44.903 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:14:44.903 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:14:44.903 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:14:44.903 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:14:44.903 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:14:44.903 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:14:44.903 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:14:44.903 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:14:44.903 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:14:44.903 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:14:44.903 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:14:44.903 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:14:44.903 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:14:44.903 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:14:44.903 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:14:44.903 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:14:44.903 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:14:44.903 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:14:44.904 | INFO | Uploaded 36GiB to GPU in 2.46403s (14.6102GiB/s) | 20260508 06:14:44.904 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:14:44.904 | INFO | Downloaded 34GiB to GPU in 2.43124s (13.9846GiB/s) | 20260508 06:14:44.904 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:14:44.904 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:14:44.904 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:14:44.904 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:14:44.904 | INFO | ------------------------------------------ | 20260508 06:14:44.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:14:44.904 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:14:44.904 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:14:44.904 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:14:44.904 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 7.57e-01 | 20260508 06:14:44.904 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.04e-01, 1.15e-02 / 3.57e-03 | 20260508 06:14:44.904 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.41e+00, 5.90e-01 / 2.40e-01 | 20260508 06:14:44.904 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.23e-01 | 20260508 06:14:44.904 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.51e+01, 3.83e+00 / 1.15e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 1.30e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.56e+00, 4.08e-01 / 2.16e-01 | 20260508 06:14:44.904 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.24e+00, 2.55e-01 / 1.54e-01 | 20260508 06:14:44.904 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.47e-01, 5.00e-02 / 2.50e-02 | 20260508 06:14:44.904 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:14:44.904 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:14:44.904 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:14:44.904 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 24.43%, Upload -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 24.22%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 2.38%, LockUpload -- 2.52e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 2.47%, LockDownload -- 2.62e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:14:44.904 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260508 06:14:45.000 | INFO | Starting a round of Baz IO | 20260508 06:14:48.431 | INFO | Populating RTMetrics report | 20260508 06:14:48.431 | INFO | RTMetrics report populated | 20260508 06:14:48.431 | INFO | Opening tmp RTMetrics json file | 20260508 06:14:48.442 | INFO | Serializing RTMetrics to json | 20260508 06:14:48.443 | INFO | Serialization done, swapping tmp file now... | 20260508 06:14:48.443 | INFO | RTMetrics done | 20260508 06:14:48.501 | INFO | Finished a round of Baz IO: Wrote 1.555 GiB in 3.501 seconds | 20260508 06:14:48.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.73% / 0.00% / 59.26% | 20260508 06:14:48.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.411165% / 0% / 5.5631e-05% | 20260508 06:14:48.501 | INFO | Baz IO overall: | 20260508 06:14:48.501 | INFO | BytesWritten : 2607476227413 | 20260508 06:14:48.501 | INFO | NumEvents : 1257355508119 | 20260508 06:14:50.049 | INFO | Analyzing chunk frames = [5873664,5874176) | 20260508 06:14:50.049 | INFO | MemInfo: { | 20260508 06:14:50.049 | INFO | "gpu" : " 17 GB", | 20260508 06:14:50.049 | INFO | "huge" : "", | 20260508 06:14:50.049 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:50.049 | INFO | "malloc" : " 14 GB", | 20260508 06:14:50.049 | INFO | "pinned" : " 29 GB" | 20260508 06:14:50.049 | INFO | } | 20260508 06:14:50.049 | INFO | Perf report for the last 5.14617 seconds: | 20260508 06:14:50.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:50.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:50.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:50.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:50.049 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:14:50.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:50.049 | INFO | RT_METRICS : 39.7%, 1.0, 20.9ms, 0 | 20260508 06:14:50.049 | INFO | PRE_HQ : 65.7%, 1.6, 55.0ms, 0 | 20260508 06:14:50.049 | INFO | BAZWRITER : 67.5%, 2.6, 90.8ms, 0 | 20260508 06:14:50.049 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:14:50.049 | INFO | Uploaded 36GiB to GPU in 2.55486s (14.0908GiB/s) | 20260508 06:14:50.049 | INFO | Downloaded 38GiB to GPU in 2.67282s (14.2172GiB/s) | 20260508 06:14:55.189 | INFO | Analyzing chunk frames = [5874176,5874688) | 20260508 06:14:55.189 | INFO | MemInfo: { | 20260508 06:14:55.189 | INFO | "gpu" : " 17 GB", | 20260508 06:14:55.189 | INFO | "huge" : "", | 20260508 06:14:55.189 | INFO | "huge_shared" : " 29 GB", | 20260508 06:14:55.189 | INFO | "malloc" : " 14 GB", | 20260508 06:14:55.189 | INFO | "pinned" : " 29 GB" | 20260508 06:14:55.190 | INFO | } | 20260508 06:14:55.190 | INFO | Perf report for the last 5.14069 seconds: | 20260508 06:14:55.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:14:55.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 06:14:55.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:14:55.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:14:55.190 | INFO | Uploaded 36GiB to GPU in 2.51409s (14.3193GiB/s) | 20260508 06:14:55.190 | INFO | Downloaded 34GiB to GPU in 2.43084s (13.987GiB/s) | 20260508 06:15:00.328 | INFO | Analyzing chunk frames = [5874688,5875200) | 20260508 06:15:00.328 | INFO | MemInfo: { | 20260508 06:15:00.328 | INFO | "gpu" : " 17 GB", | 20260508 06:15:00.328 | INFO | "huge" : "", | 20260508 06:15:00.328 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:00.328 | INFO | "malloc" : " 14 GB", | 20260508 06:15:00.329 | INFO | "pinned" : " 29 GB" | 20260508 06:15:00.329 | INFO | } | 20260508 06:15:00.329 | INFO | Perf report for the last 5.13905 seconds: | 20260508 06:15:00.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:00.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 06:15:00.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:15:00.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:00.329 | INFO | Uploaded 36GiB to GPU in 2.56806s (14.0184GiB/s) | 20260508 06:15:00.329 | INFO | Downloaded 34GiB to GPU in 2.41779s (14.0625GiB/s) | 20260508 06:15:05.467 | INFO | Analyzing chunk frames = [5875200,5875712) | 20260508 06:15:05.467 | INFO | MemInfo: { | 20260508 06:15:05.467 | INFO | "gpu" : " 17 GB", | 20260508 06:15:05.467 | INFO | "huge" : "", | 20260508 06:15:05.467 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:05.467 | INFO | "malloc" : " 14 GB", | 20260508 06:15:05.467 | INFO | "pinned" : " 29 GB" | 20260508 06:15:05.467 | INFO | } | 20260508 06:15:05.467 | INFO | Perf report for the last 5.13869 seconds: | 20260508 06:15:05.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:05.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.467 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 06:15:05.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.468 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 06:15:05.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:05.468 | INFO | Uploaded 36GiB to GPU in 2.59048s (13.897GiB/s) | 20260508 06:15:05.468 | INFO | Downloaded 34GiB to GPU in 2.43109s (13.9855GiB/s) | 20260508 06:15:10.602 | INFO | Analyzing chunk frames = [5875712,5876224) | 20260508 06:15:10.602 | INFO | MemInfo: { | 20260508 06:15:10.602 | INFO | "gpu" : " 17 GB", | 20260508 06:15:10.602 | INFO | "huge" : "", | 20260508 06:15:10.602 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:10.602 | INFO | "malloc" : " 14 GB", | 20260508 06:15:10.602 | INFO | "pinned" : " 29 GB" | 20260508 06:15:10.602 | INFO | } | 20260508 06:15:10.602 | INFO | Perf report for the last 5.13487 seconds: | 20260508 06:15:10.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:10.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.602 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:15:10.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.602 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260508 06:15:10.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:10.603 | INFO | Uploaded 36GiB to GPU in 2.27385s (15.8322GiB/s) | 20260508 06:15:10.603 | INFO | Downloaded 34GiB to GPU in 2.49002s (13.6545GiB/s) | 20260508 06:15:15.743 | INFO | Analyzing chunk frames = [5876224,5876736) | 20260508 06:15:15.743 | INFO | MemInfo: { | 20260508 06:15:15.743 | INFO | "gpu" : " 17 GB", | 20260508 06:15:15.743 | INFO | "huge" : "", | 20260508 06:15:15.743 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:15.743 | INFO | "malloc" : " 14 GB", | 20260508 06:15:15.743 | INFO | "pinned" : " 29 GB" | 20260508 06:15:15.743 | INFO | } | 20260508 06:15:15.743 | INFO | Perf report for the last 5.14098 seconds: | 20260508 06:15:15.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:15.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.743 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 06:15:15.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.743 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:15:15.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:15.744 | INFO | Uploaded 36GiB to GPU in 2.44417s (14.7289GiB/s) | 20260508 06:15:15.744 | INFO | Downloaded 34GiB to GPU in 2.4318s (13.9814GiB/s) | 20260508 06:15:20.884 | INFO | Analyzing chunk frames = [5876736,5877248) | 20260508 06:15:20.884 | INFO | MemInfo: { | 20260508 06:15:20.884 | INFO | "gpu" : " 17 GB", | 20260508 06:15:20.884 | INFO | "huge" : "", | 20260508 06:15:20.884 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:20.884 | INFO | "malloc" : " 14 GB", | 20260508 06:15:20.884 | INFO | "pinned" : " 29 GB" | 20260508 06:15:20.885 | INFO | } | 20260508 06:15:20.885 | INFO | Perf report for the last 5.14138 seconds: | 20260508 06:15:20.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:20.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:15:20.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:15:20.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:20.885 | INFO | Uploaded 36GiB to GPU in 2.46859s (14.5832GiB/s) | 20260508 06:15:20.885 | INFO | Downloaded 34GiB to GPU in 2.44152s (13.9257GiB/s) | 20260508 06:15:26.024 | INFO | Analyzing chunk frames = [5877248,5877760) | 20260508 06:15:26.024 | INFO | MemInfo: { | 20260508 06:15:26.024 | INFO | "gpu" : " 17 GB", | 20260508 06:15:26.024 | INFO | "huge" : "", | 20260508 06:15:26.024 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:26.024 | INFO | "malloc" : " 14 GB", | 20260508 06:15:26.024 | INFO | "pinned" : " 29 GB" | 20260508 06:15:26.024 | INFO | } | 20260508 06:15:26.025 | INFO | Perf report for the last 5.13991 seconds: | 20260508 06:15:26.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:26.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:15:26.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:15:26.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:26.025 | INFO | Uploaded 36GiB to GPU in 2.45319s (14.6748GiB/s) | 20260508 06:15:26.025 | INFO | Downloaded 34GiB to GPU in 2.43028s (13.9901GiB/s) | 20260508 06:15:26.122 | INFO | Starting a round of Baz IO | 20260508 06:15:29.550 | INFO | Populating RTMetrics report | 20260508 06:15:29.550 | INFO | RTMetrics report populated | 20260508 06:15:29.550 | INFO | Opening tmp RTMetrics json file | 20260508 06:15:29.551 | INFO | Serializing RTMetrics to json | 20260508 06:15:29.552 | INFO | Serialization done, swapping tmp file now... | 20260508 06:15:29.553 | INFO | RTMetrics done | 20260508 06:15:29.616 | INFO | Finished a round of Baz IO: Wrote 1.555 GiB in 3.494 seconds | 20260508 06:15:29.616 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.73% / 0.00% / 59.27% | 20260508 06:15:29.616 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.410223% / 0% / 7.15256e-05% | 20260508 06:15:29.616 | INFO | Baz IO overall: | 20260508 06:15:29.616 | INFO | BytesWritten : 2609145921149 | 20260508 06:15:29.616 | INFO | NumEvents : 1258150445384 | 20260508 06:15:31.159 | INFO | Analyzing chunk frames = [5877760,5878272) | 20260508 06:15:31.159 | INFO | MemInfo: { | 20260508 06:15:31.159 | INFO | "gpu" : " 17 GB", | 20260508 06:15:31.159 | INFO | "huge" : "", | 20260508 06:15:31.159 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:31.159 | INFO | "malloc" : " 14 GB", | 20260508 06:15:31.159 | INFO | "pinned" : " 29 GB" | 20260508 06:15:31.159 | INFO | } | 20260508 06:15:31.159 | INFO | Perf report for the last 5.13495 seconds: | 20260508 06:15:31.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:31.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:31.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:31.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:31.160 | INFO | ANALYSIS : 68.3%, 3.0, 106.1ms, 0 | 20260508 06:15:31.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:31.160 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 06:15:31.160 | INFO | PRE_HQ : 66.2%, 1.6, 55.1ms, 0 | 20260508 06:15:31.160 | INFO | BAZWRITER : 67.3%, 3.0, 102.2ms, 0 | 20260508 06:15:31.160 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 06:15:31.160 | INFO | Uploaded 36GiB to GPU in 2.52324s (14.2674GiB/s) | 20260508 06:15:31.160 | INFO | Downloaded 38GiB to GPU in 2.69903s (14.0791GiB/s) | 20260508 06:15:36.305 | INFO | Analyzing chunk frames = [5878272,5878784) | 20260508 06:15:36.305 | INFO | MemInfo: { | 20260508 06:15:36.305 | INFO | "gpu" : " 17 GB", | 20260508 06:15:36.305 | INFO | "huge" : "", | 20260508 06:15:36.305 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:36.305 | INFO | "malloc" : " 14 GB", | 20260508 06:15:36.305 | INFO | "pinned" : " 29 GB" | 20260508 06:15:36.305 | INFO | } | 20260508 06:15:36.305 | INFO | Perf report for the last 5.14572 seconds: | 20260508 06:15:36.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:36.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.305 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:15:36.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.305 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:15:36.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:36.306 | INFO | Uploaded 36GiB to GPU in 2.48972s (14.4594GiB/s) | 20260508 06:15:36.306 | INFO | Downloaded 34GiB to GPU in 2.42608s (14.0144GiB/s) | 20260508 06:15:41.440 | INFO | Analyzing chunk frames = [5878784,5879296) | 20260508 06:15:41.440 | INFO | MemInfo: { | 20260508 06:15:41.440 | INFO | "gpu" : " 17 GB", | 20260508 06:15:41.440 | INFO | "huge" : "", | 20260508 06:15:41.441 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:41.441 | INFO | "malloc" : " 14 GB", | 20260508 06:15:41.441 | INFO | "pinned" : " 29 GB" | 20260508 06:15:41.441 | INFO | } | 20260508 06:15:41.441 | INFO | Perf report for the last 5.13557 seconds: | 20260508 06:15:41.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:41.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.441 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 06:15:41.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.441 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:15:41.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:41.442 | INFO | Uploaded 36GiB to GPU in 2.49356s (14.4372GiB/s) | 20260508 06:15:41.442 | INFO | Downloaded 34GiB to GPU in 2.42771s (14.0049GiB/s) | 20260508 06:15:46.575 | INFO | Analyzing chunk frames = [5879296,5879808) | 20260508 06:15:46.575 | INFO | MemInfo: { | 20260508 06:15:46.575 | INFO | "gpu" : " 17 GB", | 20260508 06:15:46.575 | INFO | "huge" : "", | 20260508 06:15:46.575 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:46.575 | INFO | "malloc" : " 14 GB", | 20260508 06:15:46.575 | INFO | "pinned" : " 29 GB" | 20260508 06:15:46.576 | INFO | } | 20260508 06:15:46.576 | INFO | Perf report for the last 5.1348 seconds: | 20260508 06:15:46.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:46.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 06:15:46.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 06:15:46.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:46.576 | INFO | Uploaded 36GiB to GPU in 2.57328s (13.99GiB/s) | 20260508 06:15:46.576 | INFO | Downloaded 34GiB to GPU in 2.41073s (14.1036GiB/s) | 20260508 06:15:51.716 | INFO | Analyzing chunk frames = [5879808,5880320) | 20260508 06:15:51.716 | INFO | MemInfo: { | 20260508 06:15:51.716 | INFO | "gpu" : " 17 GB", | 20260508 06:15:51.716 | INFO | "huge" : "", | 20260508 06:15:51.716 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:51.716 | INFO | "malloc" : " 14 GB", | 20260508 06:15:51.716 | INFO | "pinned" : " 29 GB" | 20260508 06:15:51.717 | INFO | } | 20260508 06:15:51.717 | INFO | Perf report for the last 5.14092 seconds: | 20260508 06:15:51.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:51.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 06:15:51.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | PRE_HQ : 31.5%, 1.0, 16.5ms, 0 | 20260508 06:15:51.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:51.717 | INFO | Uploaded 36GiB to GPU in 2.31476s (15.5524GiB/s) | 20260508 06:15:51.717 | INFO | Downloaded 34GiB to GPU in 2.47064s (13.7616GiB/s) | 20260508 06:15:56.856 | INFO | Analyzing chunk frames = [5880320,5880832) | 20260508 06:15:56.856 | INFO | MemInfo: { | 20260508 06:15:56.856 | INFO | "gpu" : " 17 GB", | 20260508 06:15:56.856 | INFO | "huge" : "", | 20260508 06:15:56.856 | INFO | "huge_shared" : " 29 GB", | 20260508 06:15:56.856 | INFO | "malloc" : " 14 GB", | 20260508 06:15:56.856 | INFO | "pinned" : " 29 GB" | 20260508 06:15:56.856 | INFO | } | 20260508 06:15:56.856 | INFO | Perf report for the last 5.13974 seconds: | 20260508 06:15:56.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:15:56.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.856 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 06:15:56.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.856 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:15:56.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:15:56.857 | INFO | Uploaded 36GiB to GPU in 2.45752s (14.6489GiB/s) | 20260508 06:15:56.857 | INFO | Downloaded 34GiB to GPU in 2.41069s (14.1039GiB/s) | 20260508 06:16:01.998 | INFO | Analyzing chunk frames = [5880832,5881344) | 20260508 06:16:01.998 | INFO | MemInfo: { | 20260508 06:16:01.998 | INFO | "gpu" : " 17 GB", | 20260508 06:16:01.998 | INFO | "huge" : "", | 20260508 06:16:01.998 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:01.999 | INFO | "malloc" : " 14 GB", | 20260508 06:16:01.999 | INFO | "pinned" : " 29 GB" | 20260508 06:16:01.999 | INFO | } | 20260508 06:16:01.999 | INFO | Perf report for the last 5.14243 seconds: | 20260508 06:16:01.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:01.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:16:01.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:16:01.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:01.999 | INFO | Uploaded 36GiB to GPU in 2.47485s (14.5463GiB/s) | 20260508 06:16:01.999 | INFO | Downloaded 34GiB to GPU in 2.41938s (14.0532GiB/s) | 20260508 06:16:07.141 | INFO | Analyzing chunk frames = [5881344,5881856) | 20260508 06:16:07.141 | INFO | MemInfo: { | 20260508 06:16:07.141 | INFO | "gpu" : " 17 GB", | 20260508 06:16:07.141 | INFO | "huge" : "", | 20260508 06:16:07.141 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:07.141 | INFO | "malloc" : " 14 GB", | 20260508 06:16:07.141 | INFO | "pinned" : " 29 GB" | 20260508 06:16:07.141 | INFO | } | 20260508 06:16:07.141 | INFO | Perf report for the last 5.14258 seconds: | 20260508 06:16:07.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:07.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.141 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:16:07.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.141 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:16:07.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:07.142 | INFO | Uploaded 36GiB to GPU in 2.46073s (14.6298GiB/s) | 20260508 06:16:07.142 | INFO | Downloaded 34GiB to GPU in 2.42623s (14.0135GiB/s) | 20260508 06:16:07.142 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:16:07.142 | INFO | Total mem usage: 16.7 GiB | 20260508 06:16:07.142 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:16:07.142 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.9%) | 20260508 06:16:07.142 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:16:07.142 | INFO | High Water Breakdown: | 20260508 06:16:07.142 | INFO | 60% : 8.87 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:16:07.142 | INFO | 39.8% : 5.88 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:16:07.142 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:16:07.142 | INFO | ------------------------------------------ | 20260508 06:16:07.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:16:07.142 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:16:07.142 | INFO | Total mem usage: 26.5 GiB | 20260508 06:16:07.142 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:16:07.142 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 06:16:07.142 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 06:16:07.142 | INFO | High Water Breakdown: | 20260508 06:16:07.142 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:16:07.142 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:16:07.142 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:16:07.142 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:16:07.142 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:16:07.142 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:16:07.142 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:16:07.142 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:16:07.142 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:16:07.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:16:07.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:16:07.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:16:07.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:16:07.142 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:16:07.142 | INFO | ------------------------------------------ | 20260508 06:16:07.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:16:07.142 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:16:07.142 | INFO | Total mem usage: 27.8 GiB | 20260508 06:16:07.142 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:16:07.142 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:16:07.142 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:16:07.142 | INFO | High Water Breakdown: | 20260508 06:16:07.142 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:16:07.142 | INFO | ------------------------------------------ | 20260508 06:16:07.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:16:07.142 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:16:07.142 | INFO | Total mem usage: 15.7 GiB | 20260508 06:16:07.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:16:07.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:16:07.142 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 06:16:07.142 | INFO | High Water Breakdown: | 20260508 06:16:07.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:16:07.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:16:07.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:16:07.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:16:07.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:16:07.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:16:07.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:16:07.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:16:07.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:16:07.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:16:07.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:16:07.143 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:16:07.143 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:16:07.143 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:16:07.143 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:16:07.143 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:16:07.143 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:16:07.143 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:16:07.143 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:16:07.143 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:16:07.143 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:16:07.143 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:16:07.143 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:16:07.143 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:16:07.143 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:16:07.143 | INFO | ------------------------------------------ | 20260508 06:16:07.143 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:16:07.143 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:16:07.143 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:16:07.143 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:16:07.143 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.38e-01 | 20260508 06:16:07.143 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.20e-02 / 4.94e-03 | 20260508 06:16:07.143 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.49e+00, 5.98e-01 / 2.09e-01 | 20260508 06:16:07.143 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.12e-01 / 1.08e-01 | 20260508 06:16:07.143 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.52e+01, 3.74e+00 / 1.22e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.51e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.14e-01 / 2.48e-01 | 20260508 06:16:07.143 | INFO | PerfReport: 3.16%, Metrics -- 1.04e+00 / 1.21e+00, 2.53e-01 / 1.20e-01 | 20260508 06:16:07.143 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.17e-01 / 3.49e-01, 5.01e-02 / 2.04e-02 | 20260508 06:16:07.143 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:16:07.143 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:16:07.143 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:16:07.143 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 24.33%, Upload -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 2.38%, LockUpload -- 2.52e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 2.48%, LockDownload -- 2.63e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 15.74%, LockCompute -- 1.67e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 06:16:07.143 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 06:16:07.238 | INFO | Starting a round of Baz IO | 20260508 06:16:10.662 | INFO | Populating RTMetrics report | 20260508 06:16:10.663 | INFO | RTMetrics report populated | 20260508 06:16:10.663 | INFO | Opening tmp RTMetrics json file | 20260508 06:16:10.664 | INFO | Serializing RTMetrics to json | 20260508 06:16:10.671 | INFO | Serialization done, swapping tmp file now... | 20260508 06:16:10.681 | INFO | RTMetrics done | 20260508 06:16:10.731 | INFO | Finished a round of Baz IO: Wrote 1.556 GiB in 3.492 seconds | 20260508 06:16:10.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.73% / 0.00% / 59.27% | 20260508 06:16:10.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.408888% / 7.94729e-06% / 6.35783e-05% | 20260508 06:16:10.731 | INFO | Baz IO overall: | 20260508 06:16:10.731 | INFO | BytesWritten : 2610816401513 | 20260508 06:16:10.731 | INFO | NumEvents : 1258945904753 | 20260508 06:16:12.274 | INFO | Analyzing chunk frames = [5881856,5882368) | 20260508 06:16:12.274 | INFO | MemInfo: { | 20260508 06:16:12.274 | INFO | "gpu" : " 17 GB", | 20260508 06:16:12.274 | INFO | "huge" : "", | 20260508 06:16:12.274 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:12.274 | INFO | "malloc" : " 14 GB", | 20260508 06:16:12.274 | INFO | "pinned" : " 29 GB" | 20260508 06:16:12.274 | INFO | } | 20260508 06:16:12.274 | INFO | Perf report for the last 5.13305 seconds: | 20260508 06:16:12.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:12.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:12.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:12.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:12.275 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 06:16:12.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:12.275 | INFO | RT_METRICS : 39.4%, 1.0, 20.6ms, 0 | 20260508 06:16:12.275 | INFO | PRE_HQ : 65.6%, 1.6, 55.1ms, 0 | 20260508 06:16:12.275 | INFO | BAZWRITER : 67.4%, 2.6, 90.3ms, 0 | 20260508 06:16:12.275 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:16:12.275 | INFO | Uploaded 36GiB to GPU in 2.59852s (13.854GiB/s) | 20260508 06:16:12.275 | INFO | Downloaded 38GiB to GPU in 2.70825s (14.0312GiB/s) | 20260508 06:16:17.410 | INFO | Analyzing chunk frames = [5882368,5882880) | 20260508 06:16:17.410 | INFO | MemInfo: { | 20260508 06:16:17.410 | INFO | "gpu" : " 17 GB", | 20260508 06:16:17.410 | INFO | "huge" : "", | 20260508 06:16:17.410 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:17.410 | INFO | "malloc" : " 14 GB", | 20260508 06:16:17.410 | INFO | "pinned" : " 29 GB" | 20260508 06:16:17.410 | INFO | } | 20260508 06:16:17.410 | INFO | Perf report for the last 5.13588 seconds: | 20260508 06:16:17.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:17.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.410 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 06:16:17.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.410 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:16:17.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:17.411 | INFO | Uploaded 36GiB to GPU in 2.51578s (14.3097GiB/s) | 20260508 06:16:17.411 | INFO | Downloaded 34GiB to GPU in 2.44309s (13.9168GiB/s) | 20260508 06:16:22.555 | INFO | Analyzing chunk frames = [5882880,5883392) | 20260508 06:16:22.555 | INFO | MemInfo: { | 20260508 06:16:22.555 | INFO | "gpu" : " 17 GB", | 20260508 06:16:22.555 | INFO | "huge" : "", | 20260508 06:16:22.555 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:22.555 | INFO | "malloc" : " 14 GB", | 20260508 06:16:22.555 | INFO | "pinned" : " 29 GB" | 20260508 06:16:22.555 | INFO | } | 20260508 06:16:22.555 | INFO | Perf report for the last 5.14471 seconds: | 20260508 06:16:22.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:22.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:16:22.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:16:22.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:22.555 | INFO | Uploaded 36GiB to GPU in 2.55077s (14.1134GiB/s) | 20260508 06:16:22.555 | INFO | Downloaded 34GiB to GPU in 2.40278s (14.1503GiB/s) | 20260508 06:16:27.697 | INFO | Analyzing chunk frames = [5883392,5883904) | 20260508 06:16:27.697 | INFO | MemInfo: { | 20260508 06:16:27.697 | INFO | "gpu" : " 17 GB", | 20260508 06:16:27.697 | INFO | "huge" : "", | 20260508 06:16:27.697 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:27.697 | INFO | "malloc" : " 14 GB", | 20260508 06:16:27.697 | INFO | "pinned" : " 29 GB" | 20260508 06:16:27.697 | INFO | } | 20260508 06:16:27.697 | INFO | Perf report for the last 5.1422 seconds: | 20260508 06:16:27.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:27.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.697 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 06:16:27.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.697 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:16:27.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:27.698 | INFO | Uploaded 36GiB to GPU in 2.59135s (13.8924GiB/s) | 20260508 06:16:27.698 | INFO | Downloaded 34GiB to GPU in 2.39437s (14.2GiB/s) | 20260508 06:16:32.838 | INFO | Analyzing chunk frames = [5883904,5884416) | 20260508 06:16:32.838 | INFO | MemInfo: { | 20260508 06:16:32.838 | INFO | "gpu" : " 17 GB", | 20260508 06:16:32.838 | INFO | "huge" : "", | 20260508 06:16:32.838 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:32.838 | INFO | "malloc" : " 14 GB", | 20260508 06:16:32.838 | INFO | "pinned" : " 29 GB" | 20260508 06:16:32.838 | INFO | } | 20260508 06:16:32.838 | INFO | Perf report for the last 5.14067 seconds: | 20260508 06:16:32.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:32.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.838 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 06:16:32.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.838 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:16:32.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:32.839 | INFO | Uploaded 36GiB to GPU in 2.39985s (15.0009GiB/s) | 20260508 06:16:32.839 | INFO | Downloaded 34GiB to GPU in 2.49883s (13.6064GiB/s) | 20260508 06:16:37.969 | INFO | Analyzing chunk frames = [5884416,5884928) | 20260508 06:16:37.969 | INFO | MemInfo: { | 20260508 06:16:37.969 | INFO | "gpu" : " 17 GB", | 20260508 06:16:37.969 | INFO | "huge" : "", | 20260508 06:16:37.969 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:37.969 | INFO | "malloc" : " 14 GB", | 20260508 06:16:37.969 | INFO | "pinned" : " 29 GB" | 20260508 06:16:37.969 | INFO | } | 20260508 06:16:37.969 | INFO | Perf report for the last 5.1314 seconds: | 20260508 06:16:37.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:37.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.969 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 06:16:37.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.969 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:16:37.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:37.970 | INFO | Uploaded 36GiB to GPU in 2.464s (14.6104GiB/s) | 20260508 06:16:37.970 | INFO | Downloaded 34GiB to GPU in 2.41898s (14.0555GiB/s) | 20260508 06:16:43.110 | INFO | Analyzing chunk frames = [5884928,5885440) | 20260508 06:16:43.110 | INFO | MemInfo: { | 20260508 06:16:43.110 | INFO | "gpu" : " 17 GB", | 20260508 06:16:43.110 | INFO | "huge" : "", | 20260508 06:16:43.110 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:43.110 | INFO | "malloc" : " 14 GB", | 20260508 06:16:43.110 | INFO | "pinned" : " 29 GB" | 20260508 06:16:43.110 | INFO | } | 20260508 06:16:43.110 | INFO | Perf report for the last 5.14043 seconds: | 20260508 06:16:43.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:43.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.110 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:16:43.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.110 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:16:43.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:43.111 | INFO | Uploaded 36GiB to GPU in 2.48394s (14.4931GiB/s) | 20260508 06:16:43.111 | INFO | Downloaded 34GiB to GPU in 2.4337s (13.9705GiB/s) | 20260508 06:16:48.249 | INFO | Analyzing chunk frames = [5885440,5885952) | 20260508 06:16:48.249 | INFO | MemInfo: { | 20260508 06:16:48.249 | INFO | "gpu" : " 17 GB", | 20260508 06:16:48.249 | INFO | "huge" : "", | 20260508 06:16:48.250 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:48.250 | INFO | "malloc" : " 14 GB", | 20260508 06:16:48.250 | INFO | "pinned" : " 29 GB" | 20260508 06:16:48.250 | INFO | } | 20260508 06:16:48.250 | INFO | Perf report for the last 5.13991 seconds: | 20260508 06:16:48.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:48.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:16:48.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 06:16:48.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:48.250 | INFO | Uploaded 36GiB to GPU in 2.47454s (14.5481GiB/s) | 20260508 06:16:48.250 | INFO | Downloaded 34GiB to GPU in 2.42498s (14.0207GiB/s) | 20260508 06:16:48.347 | INFO | Starting a round of Baz IO | 20260508 06:16:51.772 | INFO | Populating RTMetrics report | 20260508 06:16:51.772 | INFO | RTMetrics report populated | 20260508 06:16:51.772 | INFO | Opening tmp RTMetrics json file | 20260508 06:16:51.782 | INFO | Serializing RTMetrics to json | 20260508 06:16:51.786 | INFO | Serialization done, swapping tmp file now... | 20260508 06:16:51.786 | INFO | RTMetrics done | 20260508 06:16:51.844 | INFO | Finished a round of Baz IO: Wrote 1.556 GiB in 3.497 seconds | 20260508 06:16:51.844 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.73% / 0.00% / 59.27% | 20260508 06:16:51.844 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.410004% / 3.97364e-06% / 6.35783e-05% | 20260508 06:16:51.844 | INFO | Baz IO overall: | 20260508 06:16:51.844 | INFO | BytesWritten : 2612487306135 | 20260508 06:16:51.844 | INFO | NumEvents : 1259741806424 | 20260508 06:16:53.391 | INFO | Analyzing chunk frames = [5885952,5886464) | 20260508 06:16:53.391 | INFO | MemInfo: { | 20260508 06:16:53.391 | INFO | "gpu" : " 17 GB", | 20260508 06:16:53.391 | INFO | "huge" : "", | 20260508 06:16:53.391 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:53.391 | INFO | "malloc" : " 14 GB", | 20260508 06:16:53.391 | INFO | "pinned" : " 29 GB" | 20260508 06:16:53.391 | INFO | } | 20260508 06:16:53.391 | INFO | Perf report for the last 5.14168 seconds: | 20260508 06:16:53.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:53.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:53.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:53.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:53.392 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 06:16:53.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:53.392 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:16:53.392 | INFO | PRE_HQ : 65.6%, 1.6, 55.1ms, 0 | 20260508 06:16:53.392 | INFO | BAZWRITER : 67.3%, 2.6, 90.8ms, 0 | 20260508 06:16:53.392 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:16:53.392 | INFO | Uploaded 36GiB to GPU in 2.56948s (14.0106GiB/s) | 20260508 06:16:53.392 | INFO | Downloaded 38GiB to GPU in 2.68681s (14.1431GiB/s) | 20260508 06:16:58.531 | INFO | Analyzing chunk frames = [5886464,5886976) | 20260508 06:16:58.531 | INFO | MemInfo: { | 20260508 06:16:58.531 | INFO | "gpu" : " 17 GB", | 20260508 06:16:58.531 | INFO | "huge" : "", | 20260508 06:16:58.531 | INFO | "huge_shared" : " 29 GB", | 20260508 06:16:58.531 | INFO | "malloc" : " 14 GB", | 20260508 06:16:58.531 | INFO | "pinned" : " 29 GB" | 20260508 06:16:58.531 | INFO | } | 20260508 06:16:58.531 | INFO | Perf report for the last 5.13953 seconds: | 20260508 06:16:58.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:16:58.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 06:16:58.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 06:16:58.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:16:58.531 | INFO | Uploaded 36GiB to GPU in 2.51994s (14.2861GiB/s) | 20260508 06:16:58.531 | INFO | Downloaded 34GiB to GPU in 2.44458s (13.9083GiB/s) | 20260508 06:17:03.666 | INFO | Analyzing chunk frames = [5886976,5887488) | 20260508 06:17:03.666 | INFO | MemInfo: { | 20260508 06:17:03.666 | INFO | "gpu" : " 17 GB", | 20260508 06:17:03.666 | INFO | "huge" : "", | 20260508 06:17:03.666 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:03.666 | INFO | "malloc" : " 14 GB", | 20260508 06:17:03.666 | INFO | "pinned" : " 29 GB" | 20260508 06:17:03.666 | INFO | } | 20260508 06:17:03.666 | INFO | Perf report for the last 5.13545 seconds: | 20260508 06:17:03.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:03.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.666 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 06:17:03.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.667 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:17:03.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:03.667 | INFO | Uploaded 36GiB to GPU in 2.57943s (13.9566GiB/s) | 20260508 06:17:03.667 | INFO | Downloaded 34GiB to GPU in 2.42948s (13.9947GiB/s) | 20260508 06:17:08.812 | INFO | Analyzing chunk frames = [5887488,5888000) | 20260508 06:17:08.812 | INFO | MemInfo: { | 20260508 06:17:08.812 | INFO | "gpu" : " 17 GB", | 20260508 06:17:08.812 | INFO | "huge" : "", | 20260508 06:17:08.812 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:08.812 | INFO | "malloc" : " 14 GB", | 20260508 06:17:08.812 | INFO | "pinned" : " 29 GB" | 20260508 06:17:08.812 | INFO | } | 20260508 06:17:08.812 | INFO | Perf report for the last 5.14597 seconds: | 20260508 06:17:08.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:08.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.812 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 06:17:08.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.813 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:17:08.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:08.813 | INFO | Uploaded 36GiB to GPU in 2.57407s (13.9856GiB/s) | 20260508 06:17:08.813 | INFO | Downloaded 34GiB to GPU in 2.39628s (14.1887GiB/s) | 20260508 06:17:13.953 | INFO | Analyzing chunk frames = [5888000,5888512) | 20260508 06:17:13.953 | INFO | MemInfo: { | 20260508 06:17:13.953 | INFO | "gpu" : " 17 GB", | 20260508 06:17:13.953 | INFO | "huge" : "", | 20260508 06:17:13.953 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:13.953 | INFO | "malloc" : " 14 GB", | 20260508 06:17:13.953 | INFO | "pinned" : " 29 GB" | 20260508 06:17:13.953 | INFO | } | 20260508 06:17:13.953 | INFO | Perf report for the last 5.14062 seconds: | 20260508 06:17:13.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:13.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 06:17:13.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:17:13.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:13.953 | INFO | Uploaded 36GiB to GPU in 2.43392s (14.791GiB/s) | 20260508 06:17:13.953 | INFO | Downloaded 34GiB to GPU in 2.5222s (13.4803GiB/s) | 20260508 06:17:19.087 | INFO | Analyzing chunk frames = [5888512,5889024) | 20260508 06:17:19.087 | INFO | MemInfo: { | 20260508 06:17:19.087 | INFO | "gpu" : " 17 GB", | 20260508 06:17:19.087 | INFO | "huge" : "", | 20260508 06:17:19.087 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:19.087 | INFO | "malloc" : " 14 GB", | 20260508 06:17:19.087 | INFO | "pinned" : " 29 GB" | 20260508 06:17:19.087 | INFO | } | 20260508 06:17:19.088 | INFO | Perf report for the last 5.13449 seconds: | 20260508 06:17:19.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:19.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:17:19.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:17:19.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:19.088 | INFO | Uploaded 36GiB to GPU in 2.47016s (14.574GiB/s) | 20260508 06:17:19.088 | INFO | Downloaded 34GiB to GPU in 2.42789s (14.004GiB/s) | 20260508 06:17:24.226 | INFO | Analyzing chunk frames = [5889024,5889536) | 20260508 06:17:24.226 | INFO | MemInfo: { | 20260508 06:17:24.226 | INFO | "gpu" : " 17 GB", | 20260508 06:17:24.226 | INFO | "huge" : "", | 20260508 06:17:24.226 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:24.227 | INFO | "malloc" : " 14 GB", | 20260508 06:17:24.227 | INFO | "pinned" : " 29 GB" | 20260508 06:17:24.227 | INFO | } | 20260508 06:17:24.227 | INFO | Perf report for the last 5.13912 seconds: | 20260508 06:17:24.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:24.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 06:17:24.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:17:24.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:24.227 | INFO | Uploaded 36GiB to GPU in 2.4521s (14.6813GiB/s) | 20260508 06:17:24.227 | INFO | Downloaded 34GiB to GPU in 2.43511s (13.9624GiB/s) | 20260508 06:17:29.365 | INFO | Analyzing chunk frames = [5889536,5890048) | 20260508 06:17:29.365 | INFO | MemInfo: { | 20260508 06:17:29.365 | INFO | "gpu" : " 17 GB", | 20260508 06:17:29.365 | INFO | "huge" : "", | 20260508 06:17:29.365 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:29.365 | INFO | "malloc" : " 14 GB", | 20260508 06:17:29.365 | INFO | "pinned" : " 29 GB" | 20260508 06:17:29.365 | INFO | } | 20260508 06:17:29.365 | INFO | Perf report for the last 5.13864 seconds: | 20260508 06:17:29.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:29.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.366 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:17:29.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.366 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:17:29.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:29.366 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:17:29.366 | INFO | Total mem usage: 16.7 GiB | 20260508 06:17:29.366 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:17:29.366 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:17:29.366 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:17:29.366 | INFO | High Water Breakdown: | 20260508 06:17:29.366 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:17:29.366 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:17:29.366 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:17:29.366 | INFO | ------------------------------------------ | 20260508 06:17:29.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:17:29.366 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:17:29.366 | INFO | Total mem usage: 26.5 GiB | 20260508 06:17:29.366 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:17:29.366 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:17:29.366 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:17:29.366 | INFO | High Water Breakdown: | 20260508 06:17:29.366 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:17:29.366 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:17:29.366 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:17:29.366 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:17:29.366 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:17:29.366 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:17:29.366 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:17:29.366 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:17:29.366 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:17:29.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:17:29.366 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:17:29.366 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:17:29.366 | INFO | Uploaded 36GiB to GPU in 2.48118s (14.5092GiB/s) | 20260508 06:17:29.366 | INFO | Downloaded 34GiB to GPU in 2.42708s (14.0086GiB/s) | 20260508 06:17:29.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:17:29.366 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:17:29.366 | INFO | ------------------------------------------ | 20260508 06:17:29.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:17:29.367 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:17:29.367 | INFO | Total mem usage: 27.8 GiB | 20260508 06:17:29.367 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:17:29.367 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:17:29.367 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:17:29.367 | INFO | High Water Breakdown: | 20260508 06:17:29.367 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:17:29.367 | INFO | ------------------------------------------ | 20260508 06:17:29.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:17:29.367 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:17:29.367 | INFO | Total mem usage: 15.7 GiB | 20260508 06:17:29.367 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:17:29.367 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:17:29.367 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:17:29.367 | INFO | High Water Breakdown: | 20260508 06:17:29.367 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:17:29.367 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:17:29.367 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:17:29.367 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:17:29.367 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:17:29.367 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:17:29.367 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:17:29.367 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:17:29.367 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:17:29.367 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:17:29.367 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:17:29.367 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:17:29.367 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:17:29.367 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:17:29.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:17:29.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:17:29.367 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:17:29.367 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:17:29.367 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:17:29.367 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:17:29.367 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:17:29.367 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:17:29.367 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:17:29.367 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:17:29.367 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:17:29.367 | INFO | ------------------------------------------ | 20260508 06:17:29.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:17:29.367 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:17:29.367 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:17:29.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:17:29.367 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 5.57e-01 | 20260508 06:17:29.367 | INFO | PerfReport: 0.30%, AggregateStats -- 9.63e-02 / 1.05e-01, 1.19e-02 / 7.74e-03 | 20260508 06:17:29.367 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.39e+00, 5.83e-01 / 2.15e-01 | 20260508 06:17:29.367 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.42e+00, 2.06e-01 / 1.14e-01 | 20260508 06:17:29.367 | INFO | PerfReport: 15.89%, DME -- 4.15e+01 / 4.48e+01, 3.66e+00 / 9.93e-01 | 20260508 06:17:29.367 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.32e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.37e+00 / 2.57e+00, 4.14e-01 / 2.14e-01 | 20260508 06:17:29.367 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.25e+00, 2.55e-01 / 1.23e-01 | 20260508 06:17:29.367 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.48e-01, 4.91e-02 / 2.15e-02 | 20260508 06:17:29.367 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:17:29.367 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:17:29.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:17:29.367 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 24.61%, Upload -- 2.61e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 30.73%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 2.49%, LockUpload -- 2.64e+00 / 0.00e+00, 5.87e+00 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 2.40%, LockDownload -- 2.55e+00 / 0.00e+00, 3.85e+00 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:17:29.367 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260508 06:17:29.463 | INFO | Starting a round of Baz IO | 20260508 06:17:32.896 | INFO | Populating RTMetrics report | 20260508 06:17:32.897 | INFO | RTMetrics report populated | 20260508 06:17:32.897 | INFO | Opening tmp RTMetrics json file | 20260508 06:17:32.908 | INFO | Serializing RTMetrics to json | 20260508 06:17:32.910 | INFO | Serialization done, swapping tmp file now... | 20260508 06:17:32.910 | INFO | RTMetrics done | 20260508 06:17:32.968 | INFO | Finished a round of Baz IO: Wrote 1.555 GiB in 3.504 seconds | 20260508 06:17:32.968 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.73% / 0.00% / 59.27% | 20260508 06:17:32.968 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.408518% / 1.19209e-05% / 7.94729e-05% | 20260508 06:17:32.968 | INFO | Baz IO overall: | 20260508 06:17:32.968 | INFO | BytesWritten : 2614157504948 | 20260508 06:17:32.968 | INFO | NumEvents : 1260537363550 | 20260508 06:17:34.507 | INFO | Analyzing chunk frames = [5890048,5890560) | 20260508 06:17:34.507 | INFO | MemInfo: { | 20260508 06:17:34.507 | INFO | "gpu" : " 17 GB", | 20260508 06:17:34.507 | INFO | "huge" : "", | 20260508 06:17:34.507 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:34.507 | INFO | "malloc" : " 14 GB", | 20260508 06:17:34.507 | INFO | "pinned" : " 29 GB" | 20260508 06:17:34.507 | INFO | } | 20260508 06:17:34.507 | INFO | Perf report for the last 5.14158 seconds: | 20260508 06:17:34.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:34.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:34.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:34.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:34.507 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 06:17:34.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:34.507 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 06:17:34.507 | INFO | PRE_HQ : 66.2%, 1.7, 59.4ms, 0 | 20260508 06:17:34.507 | INFO | BAZWRITER : 67.6%, 2.7, 94.5ms, 0 | 20260508 06:17:34.507 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:17:34.507 | INFO | Uploaded 36GiB to GPU in 2.61531s (13.7651GiB/s) | 20260508 06:17:34.507 | INFO | Downloaded 38GiB to GPU in 2.69639s (14.0929GiB/s) | 20260508 06:17:39.645 | INFO | Analyzing chunk frames = [5890560,5891072) | 20260508 06:17:39.645 | INFO | MemInfo: { | 20260508 06:17:39.645 | INFO | "gpu" : " 17 GB", | 20260508 06:17:39.645 | INFO | "huge" : "", | 20260508 06:17:39.645 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:39.645 | INFO | "malloc" : " 14 GB", | 20260508 06:17:39.645 | INFO | "pinned" : " 29 GB" | 20260508 06:17:39.645 | INFO | } | 20260508 06:17:39.646 | INFO | Perf report for the last 5.13855 seconds: | 20260508 06:17:39.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:39.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 06:17:39.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:17:39.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:39.646 | INFO | Uploaded 36GiB to GPU in 2.5139s (14.3204GiB/s) | 20260508 06:17:39.646 | INFO | Downloaded 34GiB to GPU in 2.43772s (13.9475GiB/s) | 20260508 06:17:44.786 | INFO | Analyzing chunk frames = [5891072,5891584) | 20260508 06:17:44.786 | INFO | MemInfo: { | 20260508 06:17:44.786 | INFO | "gpu" : " 17 GB", | 20260508 06:17:44.786 | INFO | "huge" : "", | 20260508 06:17:44.786 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:44.786 | INFO | "malloc" : " 14 GB", | 20260508 06:17:44.786 | INFO | "pinned" : " 29 GB" | 20260508 06:17:44.786 | INFO | } | 20260508 06:17:44.786 | INFO | Perf report for the last 5.14043 seconds: | 20260508 06:17:44.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:44.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:17:44.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:17:44.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:44.786 | INFO | Uploaded 36GiB to GPU in 2.55605s (14.0842GiB/s) | 20260508 06:17:44.786 | INFO | Downloaded 34GiB to GPU in 2.40484s (14.1382GiB/s) | 20260508 06:17:49.922 | INFO | Analyzing chunk frames = [5891584,5892096) | 20260508 06:17:49.922 | INFO | MemInfo: { | 20260508 06:17:49.922 | INFO | "gpu" : " 17 GB", | 20260508 06:17:49.923 | INFO | "huge" : "", | 20260508 06:17:49.923 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:49.923 | INFO | "malloc" : " 14 GB", | 20260508 06:17:49.923 | INFO | "pinned" : " 29 GB" | 20260508 06:17:49.923 | INFO | } | 20260508 06:17:49.923 | INFO | Perf report for the last 5.13681 seconds: | 20260508 06:17:49.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:49.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:17:49.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:17:49.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:49.923 | INFO | Uploaded 36GiB to GPU in 2.57315s (13.9906GiB/s) | 20260508 06:17:49.923 | INFO | Downloaded 34GiB to GPU in 2.38853s (14.2347GiB/s) | 20260508 06:17:55.066 | INFO | Analyzing chunk frames = [5892096,5892608) | 20260508 06:17:55.066 | INFO | MemInfo: { | 20260508 06:17:55.066 | INFO | "gpu" : " 17 GB", | 20260508 06:17:55.066 | INFO | "huge" : "", | 20260508 06:17:55.066 | INFO | "huge_shared" : " 29 GB", | 20260508 06:17:55.066 | INFO | "malloc" : " 14 GB", | 20260508 06:17:55.066 | INFO | "pinned" : " 29 GB" | 20260508 06:17:55.066 | INFO | } | 20260508 06:17:55.066 | INFO | Perf report for the last 5.14308 seconds: | 20260508 06:17:55.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:17:55.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 06:17:55.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:17:55.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:17:55.066 | INFO | Uploaded 36GiB to GPU in 2.30407s (15.6245GiB/s) | 20260508 06:17:55.067 | INFO | Downloaded 34GiB to GPU in 2.48293s (13.6935GiB/s) | 20260508 06:18:00.203 | INFO | Analyzing chunk frames = [5892608,5893120) | 20260508 06:18:00.203 | INFO | MemInfo: { | 20260508 06:18:00.203 | INFO | "gpu" : " 17 GB", | 20260508 06:18:00.203 | INFO | "huge" : "", | 20260508 06:18:00.203 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:00.203 | INFO | "malloc" : " 14 GB", | 20260508 06:18:00.203 | INFO | "pinned" : " 29 GB" | 20260508 06:18:00.203 | INFO | } | 20260508 06:18:00.203 | INFO | Perf report for the last 5.13747 seconds: | 20260508 06:18:00.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:00.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.203 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:18:00.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.203 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:18:00.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:00.204 | INFO | Uploaded 36GiB to GPU in 2.46716s (14.5917GiB/s) | 20260508 06:18:00.204 | INFO | Downloaded 34GiB to GPU in 2.43149s (13.9832GiB/s) | 20260508 06:18:05.343 | INFO | Analyzing chunk frames = [5893120,5893632) | 20260508 06:18:05.343 | INFO | MemInfo: { | 20260508 06:18:05.343 | INFO | "gpu" : " 17 GB", | 20260508 06:18:05.343 | INFO | "huge" : "", | 20260508 06:18:05.343 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:05.343 | INFO | "malloc" : " 14 GB", | 20260508 06:18:05.343 | INFO | "pinned" : " 29 GB" | 20260508 06:18:05.343 | INFO | } | 20260508 06:18:05.343 | INFO | Perf report for the last 5.1395 seconds: | 20260508 06:18:05.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:05.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:18:05.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:18:05.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:05.343 | INFO | Uploaded 36GiB to GPU in 2.46578s (14.5998GiB/s) | 20260508 06:18:05.343 | INFO | Downloaded 34GiB to GPU in 2.43017s (13.9908GiB/s) | 20260508 06:18:10.478 | INFO | Analyzing chunk frames = [5893632,5894144) | 20260508 06:18:10.478 | INFO | MemInfo: { | 20260508 06:18:10.478 | INFO | "gpu" : " 17 GB", | 20260508 06:18:10.478 | INFO | "huge" : "", | 20260508 06:18:10.478 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:10.478 | INFO | "malloc" : " 14 GB", | 20260508 06:18:10.478 | INFO | "pinned" : " 29 GB" | 20260508 06:18:10.478 | INFO | } | 20260508 06:18:10.478 | INFO | Perf report for the last 5.13519 seconds: | 20260508 06:18:10.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:10.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:18:10.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:18:10.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:10.478 | INFO | Uploaded 36GiB to GPU in 2.45409s (14.6694GiB/s) | 20260508 06:18:10.478 | INFO | Downloaded 34GiB to GPU in 2.4134s (14.088GiB/s) | 20260508 06:18:10.584 | INFO | Starting a round of Baz IO | 20260508 06:18:13.998 | INFO | Populating RTMetrics report | 20260508 06:18:13.999 | INFO | RTMetrics report populated | 20260508 06:18:13.999 | INFO | Opening tmp RTMetrics json file | 20260508 06:18:14.009 | INFO | Serializing RTMetrics to json | 20260508 06:18:14.011 | INFO | Serialization done, swapping tmp file now... | 20260508 06:18:14.011 | INFO | RTMetrics done | 20260508 06:18:14.071 | INFO | Finished a round of Baz IO: Wrote 1.555 GiB in 3.486 seconds | 20260508 06:18:14.071 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.00% / 59.27% | 20260508 06:18:14.071 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.407211% / 7.94729e-06% / 7.94729e-05% | 20260508 06:18:14.071 | INFO | Baz IO overall: | 20260508 06:18:14.071 | INFO | BytesWritten : 2615827689696 | 20260508 06:18:14.071 | INFO | NumEvents : 1261332981670 | 20260508 06:18:15.620 | INFO | Analyzing chunk frames = [5894144,5894656) | 20260508 06:18:15.620 | INFO | MemInfo: { | 20260508 06:18:15.620 | INFO | "gpu" : " 17 GB", | 20260508 06:18:15.620 | INFO | "huge" : "", | 20260508 06:18:15.620 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:15.620 | INFO | "malloc" : " 14 GB", | 20260508 06:18:15.620 | INFO | "pinned" : " 29 GB" | 20260508 06:18:15.620 | INFO | } | 20260508 06:18:15.620 | INFO | Perf report for the last 5.14207 seconds: | 20260508 06:18:15.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:15.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:15.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:15.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:15.620 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 06:18:15.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:15.620 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 06:18:15.620 | INFO | PRE_HQ : 65.6%, 1.6, 55.1ms, 0 | 20260508 06:18:15.620 | INFO | BAZWRITER : 67.3%, 2.6, 89.9ms, 0 | 20260508 06:18:15.620 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:18:15.620 | INFO | Uploaded 36GiB to GPU in 2.59985s (13.8469GiB/s) | 20260508 06:18:15.620 | INFO | Downloaded 38GiB to GPU in 2.68775s (14.1382GiB/s) | 20260508 06:18:20.764 | INFO | Analyzing chunk frames = [5894656,5895168) | 20260508 06:18:20.764 | INFO | MemInfo: { | 20260508 06:18:20.764 | INFO | "gpu" : " 17 GB", | 20260508 06:18:20.764 | INFO | "huge" : "", | 20260508 06:18:20.764 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:20.764 | INFO | "malloc" : " 14 GB", | 20260508 06:18:20.764 | INFO | "pinned" : " 29 GB" | 20260508 06:18:20.764 | INFO | } | 20260508 06:18:20.764 | INFO | Perf report for the last 5.14376 seconds: | 20260508 06:18:20.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:20.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:18:20.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:18:20.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:20.764 | INFO | Uploaded 36GiB to GPU in 2.51376s (14.3212GiB/s) | 20260508 06:18:20.764 | INFO | Downloaded 34GiB to GPU in 2.43154s (13.9829GiB/s) | 20260508 06:18:25.901 | INFO | Analyzing chunk frames = [5895168,5895680) | 20260508 06:18:25.901 | INFO | MemInfo: { | 20260508 06:18:25.901 | INFO | "gpu" : " 17 GB", | 20260508 06:18:25.901 | INFO | "huge" : "", | 20260508 06:18:25.901 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:25.901 | INFO | "malloc" : " 14 GB", | 20260508 06:18:25.901 | INFO | "pinned" : " 29 GB" | 20260508 06:18:25.901 | INFO | } | 20260508 06:18:25.901 | INFO | Perf report for the last 5.137 seconds: | 20260508 06:18:25.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:25.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.901 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 06:18:25.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.901 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:18:25.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:25.902 | INFO | Uploaded 36GiB to GPU in 2.57014s (14.007GiB/s) | 20260508 06:18:25.902 | INFO | Downloaded 34GiB to GPU in 2.42416s (14.0255GiB/s) | 20260508 06:18:31.038 | INFO | Analyzing chunk frames = [5895680,5896192) | 20260508 06:18:31.038 | INFO | MemInfo: { | 20260508 06:18:31.038 | INFO | "gpu" : " 17 GB", | 20260508 06:18:31.038 | INFO | "huge" : "", | 20260508 06:18:31.038 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:31.038 | INFO | "malloc" : " 14 GB", | 20260508 06:18:31.038 | INFO | "pinned" : " 29 GB" | 20260508 06:18:31.038 | INFO | } | 20260508 06:18:31.038 | INFO | Perf report for the last 5.13746 seconds: | 20260508 06:18:31.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:31.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.038 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:18:31.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.038 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:18:31.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:31.039 | INFO | Uploaded 36GiB to GPU in 2.57273s (13.9929GiB/s) | 20260508 06:18:31.039 | INFO | Downloaded 34GiB to GPU in 2.39132s (14.2181GiB/s) | 20260508 06:18:36.172 | INFO | Analyzing chunk frames = [5896192,5896704) | 20260508 06:18:36.172 | INFO | MemInfo: { | 20260508 06:18:36.172 | INFO | "gpu" : " 17 GB", | 20260508 06:18:36.172 | INFO | "huge" : "", | 20260508 06:18:36.172 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:36.172 | INFO | "malloc" : " 14 GB", | 20260508 06:18:36.172 | INFO | "pinned" : " 29 GB" | 20260508 06:18:36.172 | INFO | } | 20260508 06:18:36.172 | INFO | Perf report for the last 5.13391 seconds: | 20260508 06:18:36.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:36.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.172 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 06:18:36.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.172 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:18:36.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:36.173 | INFO | Uploaded 36GiB to GPU in 2.43488s (14.7851GiB/s) | 20260508 06:18:36.173 | INFO | Downloaded 34GiB to GPU in 2.49376s (13.634GiB/s) | 20260508 06:18:41.313 | INFO | Analyzing chunk frames = [5896704,5897216) | 20260508 06:18:41.313 | INFO | MemInfo: { | 20260508 06:18:41.313 | INFO | "gpu" : " 17 GB", | 20260508 06:18:41.313 | INFO | "huge" : "", | 20260508 06:18:41.313 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:41.313 | INFO | "malloc" : " 14 GB", | 20260508 06:18:41.313 | INFO | "pinned" : " 29 GB" | 20260508 06:18:41.313 | INFO | } | 20260508 06:18:41.313 | INFO | Perf report for the last 5.14097 seconds: | 20260508 06:18:41.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:41.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.313 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:18:41.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.313 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 06:18:41.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:41.314 | INFO | Uploaded 36GiB to GPU in 2.45181s (14.683GiB/s) | 20260508 06:18:41.314 | INFO | Downloaded 34GiB to GPU in 2.42089s (14.0444GiB/s) | 20260508 06:18:46.461 | INFO | Analyzing chunk frames = [5897216,5897728) | 20260508 06:18:46.461 | INFO | MemInfo: { | 20260508 06:18:46.461 | INFO | "gpu" : " 17 GB", | 20260508 06:18:46.461 | INFO | "huge" : "", | 20260508 06:18:46.461 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:46.461 | INFO | "malloc" : " 14 GB", | 20260508 06:18:46.461 | INFO | "pinned" : " 29 GB" | 20260508 06:18:46.461 | INFO | } | 20260508 06:18:46.461 | INFO | Perf report for the last 5.14746 seconds: | 20260508 06:18:46.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:46.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:18:46.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 06:18:46.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:46.461 | INFO | Uploaded 36GiB to GPU in 2.46263s (14.6185GiB/s) | 20260508 06:18:46.461 | INFO | Downloaded 34GiB to GPU in 2.43088s (13.9867GiB/s) | 20260508 06:18:51.601 | INFO | Analyzing chunk frames = [5897728,5898240) | 20260508 06:18:51.601 | INFO | MemInfo: { | 20260508 06:18:51.601 | INFO | "gpu" : " 17 GB", | 20260508 06:18:51.602 | INFO | "huge" : "", | 20260508 06:18:51.602 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:51.602 | INFO | "malloc" : " 14 GB", | 20260508 06:18:51.602 | INFO | "pinned" : " 29 GB" | 20260508 06:18:51.602 | INFO | } | 20260508 06:18:51.602 | INFO | Perf report for the last 5.14096 seconds: | 20260508 06:18:51.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:51.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:18:51.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:18:51.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:51.602 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:18:51.602 | INFO | Total mem usage: 16.7 GiB | 20260508 06:18:51.602 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:18:51.602 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:18:51.602 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:18:51.602 | INFO | High Water Breakdown: | 20260508 06:18:51.602 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:18:51.602 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:18:51.602 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:18:51.602 | INFO | ------------------------------------------ | 20260508 06:18:51.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:18:51.602 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:18:51.602 | INFO | Total mem usage: 26.5 GiB | 20260508 06:18:51.602 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:18:51.602 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:18:51.602 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:18:51.602 | INFO | High Water Breakdown: | 20260508 06:18:51.602 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:18:51.602 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:18:51.602 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:18:51.602 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:18:51.602 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:18:51.602 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:18:51.602 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:18:51.602 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:18:51.602 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:18:51.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:18:51.602 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:18:51.602 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:18:51.602 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:18:51.602 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:18:51.602 | INFO | ------------------------------------------ | 20260508 06:18:51.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:18:51.602 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:18:51.602 | INFO | Total mem usage: 27.8 GiB | 20260508 06:18:51.602 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:18:51.602 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:18:51.602 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:18:51.602 | INFO | High Water Breakdown: | 20260508 06:18:51.602 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:18:51.602 | INFO | ------------------------------------------ | 20260508 06:18:51.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:18:51.602 | INFO | Uploaded 36GiB to GPU in 2.45577s (14.6593GiB/s) | 20260508 06:18:51.602 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:18:51.603 | INFO | Downloaded 34GiB to GPU in 2.42827s (14.0017GiB/s) | 20260508 06:18:51.603 | INFO | Total mem usage: 15.7 GiB | 20260508 06:18:51.603 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:18:51.603 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:18:51.603 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:18:51.603 | INFO | High Water Breakdown: | 20260508 06:18:51.603 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:18:51.603 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:18:51.603 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:18:51.603 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:18:51.603 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:18:51.603 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:18:51.603 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:18:51.603 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:18:51.603 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:18:51.603 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:18:51.603 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:18:51.603 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:18:51.603 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:18:51.603 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:18:51.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:18:51.603 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:18:51.603 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:18:51.603 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:18:51.603 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:18:51.603 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:18:51.603 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:18:51.603 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:18:51.603 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:18:51.603 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:18:51.603 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:18:51.603 | INFO | ------------------------------------------ | 20260508 06:18:51.603 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:18:51.603 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:18:51.603 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:18:51.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:18:51.603 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.09e-01 | 20260508 06:18:51.603 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.03e-01, 1.12e-02 / 6.41e-03 | 20260508 06:18:51.603 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.42e+00, 5.82e-01 / 2.21e-01 | 20260508 06:18:51.603 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.08e-01 / 1.36e-01 | 20260508 06:18:51.603 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.51e+01, 3.67e+00 / 1.22e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 1.49e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.56e+00, 4.13e-01 / 2.47e-01 | 20260508 06:18:51.603 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.22e+00, 2.54e-01 / 9.63e-02 | 20260508 06:18:51.603 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.49e-01, 4.99e-02 / 2.44e-02 | 20260508 06:18:51.603 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:18:51.603 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:18:51.603 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:18:51.603 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 24.53%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 24.15%, Download -- 2.56e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 2.45%, LockUpload -- 2.60e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 2.41%, LockDownload -- 2.56e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:18:51.603 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 3.02e-02 / 0.00e+00 | 20260508 06:18:51.700 | INFO | Starting a round of Baz IO | 20260508 06:18:55.115 | INFO | Populating RTMetrics report | 20260508 06:18:55.115 | INFO | RTMetrics report populated | 20260508 06:18:55.115 | INFO | Opening tmp RTMetrics json file | 20260508 06:18:55.128 | INFO | Serializing RTMetrics to json | 20260508 06:18:55.130 | INFO | Serialization done, swapping tmp file now... | 20260508 06:18:55.130 | INFO | RTMetrics done | 20260508 06:18:55.187 | INFO | Finished a round of Baz IO: Wrote 1.555 GiB in 3.487 seconds | 20260508 06:18:55.187 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.00% / 59.28% | 20260508 06:18:55.187 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.407418% / 7.94729e-06% / 8.74201e-05% | 20260508 06:18:55.187 | INFO | Baz IO overall: | 20260508 06:18:55.187 | INFO | BytesWritten : 2617497641762 | 20260508 06:18:55.187 | INFO | NumEvents : 1262128398671 | 20260508 06:18:56.730 | INFO | Analyzing chunk frames = [5898240,5898752) | 20260508 06:18:56.730 | INFO | MemInfo: { | 20260508 06:18:56.730 | INFO | "gpu" : " 17 GB", | 20260508 06:18:56.730 | INFO | "huge" : "", | 20260508 06:18:56.730 | INFO | "huge_shared" : " 29 GB", | 20260508 06:18:56.730 | INFO | "malloc" : " 14 GB", | 20260508 06:18:56.730 | INFO | "pinned" : " 29 GB" | 20260508 06:18:56.730 | INFO | } | 20260508 06:18:56.730 | INFO | Perf report for the last 5.12868 seconds: | 20260508 06:18:56.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:18:56.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:56.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:56.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:56.731 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 06:18:56.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:18:56.731 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:18:56.731 | INFO | PRE_HQ : 65.6%, 1.6, 55.1ms, 0 | 20260508 06:18:56.731 | INFO | BAZWRITER : 67.3%, 2.6, 89.4ms, 0 | 20260508 06:18:56.731 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:18:56.731 | INFO | Uploaded 36GiB to GPU in 2.61375s (13.7733GiB/s) | 20260508 06:18:56.731 | INFO | Downloaded 38GiB to GPU in 2.69638s (14.093GiB/s) | 20260508 06:19:01.878 | INFO | Analyzing chunk frames = [5898752,5899264) | 20260508 06:19:01.878 | INFO | MemInfo: { | 20260508 06:19:01.878 | INFO | "gpu" : " 17 GB", | 20260508 06:19:01.878 | INFO | "huge" : "", | 20260508 06:19:01.878 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:01.878 | INFO | "malloc" : " 14 GB", | 20260508 06:19:01.878 | INFO | "pinned" : " 29 GB" | 20260508 06:19:01.878 | INFO | } | 20260508 06:19:01.878 | INFO | Perf report for the last 5.14734 seconds: | 20260508 06:19:01.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:01.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.878 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:19:01.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.878 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:19:01.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:01.879 | INFO | Uploaded 36GiB to GPU in 2.51088s (14.3376GiB/s) | 20260508 06:19:01.879 | INFO | Downloaded 34GiB to GPU in 2.4289s (13.9981GiB/s) | 20260508 06:19:07.014 | INFO | Analyzing chunk frames = [5899264,5899776) | 20260508 06:19:07.014 | INFO | MemInfo: { | 20260508 06:19:07.014 | INFO | "gpu" : " 17 GB", | 20260508 06:19:07.014 | INFO | "huge" : "", | 20260508 06:19:07.014 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:07.014 | INFO | "malloc" : " 14 GB", | 20260508 06:19:07.014 | INFO | "pinned" : " 29 GB" | 20260508 06:19:07.014 | INFO | } | 20260508 06:19:07.014 | INFO | Perf report for the last 5.13653 seconds: | 20260508 06:19:07.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:07.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.014 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 06:19:07.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.014 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:19:07.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:07.015 | INFO | Uploaded 36GiB to GPU in 2.57558s (13.9774GiB/s) | 20260508 06:19:07.015 | INFO | Downloaded 34GiB to GPU in 2.43129s (13.9843GiB/s) | 20260508 06:19:12.157 | INFO | Analyzing chunk frames = [5899776,5900288) | 20260508 06:19:12.157 | INFO | MemInfo: { | 20260508 06:19:12.157 | INFO | "gpu" : " 17 GB", | 20260508 06:19:12.157 | INFO | "huge" : "", | 20260508 06:19:12.157 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:12.157 | INFO | "malloc" : " 14 GB", | 20260508 06:19:12.157 | INFO | "pinned" : " 29 GB" | 20260508 06:19:12.157 | INFO | } | 20260508 06:19:12.157 | INFO | Perf report for the last 5.14274 seconds: | 20260508 06:19:12.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:12.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 06:19:12.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:19:12.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:12.157 | INFO | Uploaded 36GiB to GPU in 2.58705s (13.9154GiB/s) | 20260508 06:19:12.157 | INFO | Downloaded 34GiB to GPU in 2.4002s (14.1655GiB/s) | 20260508 06:19:17.300 | INFO | Analyzing chunk frames = [5900288,5900800) | 20260508 06:19:17.300 | INFO | MemInfo: { | 20260508 06:19:17.300 | INFO | "gpu" : " 17 GB", | 20260508 06:19:17.300 | INFO | "huge" : "", | 20260508 06:19:17.300 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:17.300 | INFO | "malloc" : " 14 GB", | 20260508 06:19:17.300 | INFO | "pinned" : " 29 GB" | 20260508 06:19:17.300 | INFO | } | 20260508 06:19:17.300 | INFO | Perf report for the last 5.14269 seconds: | 20260508 06:19:17.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:17.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 06:19:17.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 06:19:17.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:17.300 | INFO | Uploaded 36GiB to GPU in 2.30656s (15.6076GiB/s) | 20260508 06:19:17.300 | INFO | Downloaded 34GiB to GPU in 2.49585s (13.6226GiB/s) | 20260508 06:19:22.432 | INFO | Analyzing chunk frames = [5900800,5901312) | 20260508 06:19:22.432 | INFO | MemInfo: { | 20260508 06:19:22.432 | INFO | "gpu" : " 17 GB", | 20260508 06:19:22.432 | INFO | "huge" : "", | 20260508 06:19:22.432 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:22.432 | INFO | "malloc" : " 14 GB", | 20260508 06:19:22.432 | INFO | "pinned" : " 29 GB" | 20260508 06:19:22.432 | INFO | } | 20260508 06:19:22.432 | INFO | Perf report for the last 5.13253 seconds: | 20260508 06:19:22.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:22.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.432 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 06:19:22.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.432 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:19:22.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:22.433 | INFO | Uploaded 36GiB to GPU in 2.47069s (14.5709GiB/s) | 20260508 06:19:22.433 | INFO | Downloaded 34GiB to GPU in 2.43426s (13.9673GiB/s) | 20260508 06:19:27.571 | INFO | Analyzing chunk frames = [5901312,5901824) | 20260508 06:19:27.571 | INFO | MemInfo: { | 20260508 06:19:27.571 | INFO | "gpu" : " 17 GB", | 20260508 06:19:27.571 | INFO | "huge" : "", | 20260508 06:19:27.571 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:27.571 | INFO | "malloc" : " 14 GB", | 20260508 06:19:27.571 | INFO | "pinned" : " 29 GB" | 20260508 06:19:27.571 | INFO | } | 20260508 06:19:27.571 | INFO | Perf report for the last 5.13876 seconds: | 20260508 06:19:27.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:27.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.571 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:19:27.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.571 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:19:27.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:27.572 | INFO | Uploaded 36GiB to GPU in 2.49172s (14.4479GiB/s) | 20260508 06:19:27.572 | INFO | Downloaded 34GiB to GPU in 2.42645s (14.0122GiB/s) | 20260508 06:19:32.713 | INFO | Analyzing chunk frames = [5901824,5902336) | 20260508 06:19:32.713 | INFO | MemInfo: { | 20260508 06:19:32.713 | INFO | "gpu" : " 17 GB", | 20260508 06:19:32.713 | INFO | "huge" : "", | 20260508 06:19:32.713 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:32.713 | INFO | "malloc" : " 14 GB", | 20260508 06:19:32.713 | INFO | "pinned" : " 29 GB" | 20260508 06:19:32.713 | INFO | } | 20260508 06:19:32.713 | INFO | Perf report for the last 5.14181 seconds: | 20260508 06:19:32.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:32.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.713 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:19:32.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.713 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:19:32.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:32.714 | INFO | Uploaded 36GiB to GPU in 2.47782s (14.5289GiB/s) | 20260508 06:19:32.714 | INFO | Downloaded 34GiB to GPU in 2.43862s (13.9423GiB/s) | 20260508 06:19:32.810 | INFO | Starting a round of Baz IO | 20260508 06:19:36.241 | INFO | Populating RTMetrics report | 20260508 06:19:36.241 | INFO | RTMetrics report populated | 20260508 06:19:36.241 | INFO | Opening tmp RTMetrics json file | 20260508 06:19:36.252 | INFO | Serializing RTMetrics to json | 20260508 06:19:36.253 | INFO | Serialization done, swapping tmp file now... | 20260508 06:19:36.254 | INFO | RTMetrics done | 20260508 06:19:36.311 | INFO | Finished a round of Baz IO: Wrote 1.553 GiB in 3.501 seconds | 20260508 06:19:36.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.00% / 59.28% | 20260508 06:19:36.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.40551% / 0% / 5.5631e-05% | 20260508 06:19:36.311 | INFO | Baz IO overall: | 20260508 06:19:36.311 | INFO | BytesWritten : 2619165388434 | 20260508 06:19:36.311 | INFO | NumEvents : 1262922162863 | 20260508 06:19:37.845 | INFO | Analyzing chunk frames = [5902336,5902848) | 20260508 06:19:37.845 | INFO | MemInfo: { | 20260508 06:19:37.845 | INFO | "gpu" : " 17 GB", | 20260508 06:19:37.845 | INFO | "huge" : "", | 20260508 06:19:37.845 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:37.845 | INFO | "malloc" : " 14 GB", | 20260508 06:19:37.845 | INFO | "pinned" : " 29 GB" | 20260508 06:19:37.845 | INFO | } | 20260508 06:19:37.845 | INFO | Perf report for the last 5.13178 seconds: | 20260508 06:19:37.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:37.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:37.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:37.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:37.845 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 06:19:37.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:37.845 | INFO | RT_METRICS : 39.4%, 1.0, 20.6ms, 0 | 20260508 06:19:37.845 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 06:19:37.845 | INFO | BAZWRITER : 67.6%, 2.9, 99.2ms, 0 | 20260508 06:19:37.845 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 06:19:37.845 | INFO | Uploaded 36GiB to GPU in 2.576s (13.9752GiB/s) | 20260508 06:19:37.845 | INFO | Downloaded 38GiB to GPU in 2.67534s (14.2038GiB/s) | 20260508 06:19:42.983 | INFO | Analyzing chunk frames = [5902848,5903360) | 20260508 06:19:42.983 | INFO | MemInfo: { | 20260508 06:19:42.983 | INFO | "gpu" : " 17 GB", | 20260508 06:19:42.983 | INFO | "huge" : "", | 20260508 06:19:42.983 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:42.983 | INFO | "malloc" : " 14 GB", | 20260508 06:19:42.983 | INFO | "pinned" : " 29 GB" | 20260508 06:19:42.983 | INFO | } | 20260508 06:19:42.984 | INFO | Perf report for the last 5.13873 seconds: | 20260508 06:19:42.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:42.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | ANALYSIS : 67.5%, 3.0, 105.1ms, 0 | 20260508 06:19:42.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:19:42.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:42.984 | INFO | Uploaded 36GiB to GPU in 2.58648s (13.9186GiB/s) | 20260508 06:19:42.984 | INFO | Downloaded 34GiB to GPU in 2.43773s (13.9474GiB/s) | 20260508 06:19:48.128 | INFO | Analyzing chunk frames = [5903360,5903872) | 20260508 06:19:48.128 | INFO | MemInfo: { | 20260508 06:19:48.128 | INFO | "gpu" : " 17 GB", | 20260508 06:19:48.128 | INFO | "huge" : "", | 20260508 06:19:48.128 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:48.128 | INFO | "malloc" : " 14 GB", | 20260508 06:19:48.128 | INFO | "pinned" : " 29 GB" | 20260508 06:19:48.128 | INFO | } | 20260508 06:19:48.128 | INFO | Perf report for the last 5.14463 seconds: | 20260508 06:19:48.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:48.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 06:19:48.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:19:48.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:48.128 | INFO | Uploaded 36GiB to GPU in 2.56033s (14.0607GiB/s) | 20260508 06:19:48.129 | INFO | Downloaded 34GiB to GPU in 2.40373s (14.1447GiB/s) | 20260508 06:19:53.270 | INFO | Analyzing chunk frames = [5903872,5904384) | 20260508 06:19:53.270 | INFO | MemInfo: { | 20260508 06:19:53.270 | INFO | "gpu" : " 17 GB", | 20260508 06:19:53.270 | INFO | "huge" : "", | 20260508 06:19:53.270 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:53.270 | INFO | "malloc" : " 14 GB", | 20260508 06:19:53.270 | INFO | "pinned" : " 29 GB" | 20260508 06:19:53.270 | INFO | } | 20260508 06:19:53.270 | INFO | Perf report for the last 5.14193 seconds: | 20260508 06:19:53.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:53.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.270 | INFO | ANALYSIS : 67.0%, 2.9, 103.7ms, 0 | 20260508 06:19:53.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.270 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 06:19:53.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:53.271 | INFO | Uploaded 36GiB to GPU in 2.58354s (13.9344GiB/s) | 20260508 06:19:53.271 | INFO | Downloaded 34GiB to GPU in 2.41694s (14.0674GiB/s) | 20260508 06:19:58.411 | INFO | Analyzing chunk frames = [5904384,5904896) | 20260508 06:19:58.411 | INFO | MemInfo: { | 20260508 06:19:58.411 | INFO | "gpu" : " 17 GB", | 20260508 06:19:58.411 | INFO | "huge" : "", | 20260508 06:19:58.411 | INFO | "huge_shared" : " 29 GB", | 20260508 06:19:58.411 | INFO | "malloc" : " 14 GB", | 20260508 06:19:58.411 | INFO | "pinned" : " 29 GB" | 20260508 06:19:58.411 | INFO | } | 20260508 06:19:58.412 | INFO | Perf report for the last 5.1414 seconds: | 20260508 06:19:58.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:19:58.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:19:58.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | PRE_HQ : 31.4%, 1.0, 16.4ms, 0 | 20260508 06:19:58.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:19:58.412 | INFO | Uploaded 36GiB to GPU in 2.36787s (15.2035GiB/s) | 20260508 06:19:58.412 | INFO | Downloaded 34GiB to GPU in 2.49599s (13.6219GiB/s) | 20260508 06:20:03.545 | INFO | Analyzing chunk frames = [5904896,5905408) | 20260508 06:20:03.545 | INFO | MemInfo: { | 20260508 06:20:03.545 | INFO | "gpu" : " 17 GB", | 20260508 06:20:03.545 | INFO | "huge" : "", | 20260508 06:20:03.545 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:03.545 | INFO | "malloc" : " 14 GB", | 20260508 06:20:03.545 | INFO | "pinned" : " 29 GB" | 20260508 06:20:03.545 | INFO | } | 20260508 06:20:03.545 | INFO | Perf report for the last 5.13373 seconds: | 20260508 06:20:03.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:03.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.545 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 06:20:03.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.545 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:20:03.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:03.546 | INFO | Uploaded 36GiB to GPU in 2.44971s (14.6956GiB/s) | 20260508 06:20:03.546 | INFO | Downloaded 34GiB to GPU in 2.42643s (14.0123GiB/s) | 20260508 06:20:08.682 | INFO | Analyzing chunk frames = [5905408,5905920) | 20260508 06:20:08.682 | INFO | MemInfo: { | 20260508 06:20:08.682 | INFO | "gpu" : " 17 GB", | 20260508 06:20:08.682 | INFO | "huge" : "", | 20260508 06:20:08.682 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:08.682 | INFO | "malloc" : " 14 GB", | 20260508 06:20:08.682 | INFO | "pinned" : " 29 GB" | 20260508 06:20:08.682 | INFO | } | 20260508 06:20:08.682 | INFO | Perf report for the last 5.13714 seconds: | 20260508 06:20:08.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:08.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.682 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:20:08.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.683 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:20:08.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:08.683 | INFO | Uploaded 36GiB to GPU in 2.44993s (14.6943GiB/s) | 20260508 06:20:08.683 | INFO | Downloaded 34GiB to GPU in 2.43301s (13.9745GiB/s) | 20260508 06:20:13.829 | INFO | Analyzing chunk frames = [5905920,5906432) | 20260508 06:20:13.829 | INFO | MemInfo: { | 20260508 06:20:13.829 | INFO | "gpu" : " 17 GB", | 20260508 06:20:13.829 | INFO | "huge" : "", | 20260508 06:20:13.829 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:13.829 | INFO | "malloc" : " 14 GB", | 20260508 06:20:13.829 | INFO | "pinned" : " 29 GB" | 20260508 06:20:13.829 | INFO | } | 20260508 06:20:13.829 | INFO | Perf report for the last 5.1463 seconds: | 20260508 06:20:13.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:13.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:20:13.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:20:13.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:13.829 | INFO | Uploaded 36GiB to GPU in 2.45229s (14.6802GiB/s) | 20260508 06:20:13.829 | INFO | Downloaded 34GiB to GPU in 2.43683s (13.9526GiB/s) | 20260508 06:20:13.829 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:20:13.829 | INFO | Total mem usage: 16.7 GiB | 20260508 06:20:13.829 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:20:13.829 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.3%) | 20260508 06:20:13.829 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:20:13.829 | INFO | High Water Breakdown: | 20260508 06:20:13.829 | INFO | 59.9% : 8.74 GiB ( 84.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:20:13.829 | INFO | 40% : 5.84 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:20:13.829 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:20:13.829 | INFO | ------------------------------------------ | 20260508 06:20:13.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:20:13.829 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:20:13.829 | INFO | Total mem usage: 26.5 GiB | 20260508 06:20:13.829 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:20:13.829 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:20:13.829 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:20:13.829 | INFO | High Water Breakdown: | 20260508 06:20:13.829 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:20:13.829 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:20:13.829 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:20:13.829 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:20:13.829 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:20:13.829 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:20:13.829 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:20:13.829 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:20:13.829 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:20:13.829 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:20:13.829 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:20:13.829 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:20:13.829 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:20:13.829 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:20:13.830 | INFO | ------------------------------------------ | 20260508 06:20:13.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:20:13.830 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:20:13.830 | INFO | Total mem usage: 27.8 GiB | 20260508 06:20:13.830 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:20:13.830 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:20:13.830 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:20:13.830 | INFO | High Water Breakdown: | 20260508 06:20:13.830 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:20:13.830 | INFO | ------------------------------------------ | 20260508 06:20:13.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:20:13.830 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:20:13.830 | INFO | Total mem usage: 15.7 GiB | 20260508 06:20:13.830 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:20:13.830 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:20:13.830 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:20:13.830 | INFO | High Water Breakdown: | 20260508 06:20:13.830 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:20:13.830 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:20:13.830 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:20:13.830 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:20:13.830 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:20:13.830 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:20:13.830 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:20:13.830 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:20:13.830 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:20:13.830 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:20:13.830 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:20:13.830 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:20:13.830 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:20:13.830 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:20:13.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:20:13.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:20:13.830 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:20:13.830 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:20:13.830 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:20:13.830 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:20:13.830 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:20:13.830 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:20:13.830 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:20:13.830 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:20:13.830 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:20:13.830 | INFO | ------------------------------------------ | 20260508 06:20:13.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:20:13.830 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:20:13.830 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:20:13.830 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:20:13.830 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.39e+01 / 6.64e-01 | 20260508 06:20:13.830 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.10e-02 / 4.71e-03 | 20260508 06:20:13.830 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.39e+00, 5.87e-01 / 2.21e-01 | 20260508 06:20:13.830 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.44e+00, 2.10e-01 / 1.24e-01 | 20260508 06:20:13.830 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.51e+01, 3.57e+00 / 9.44e-01 | 20260508 06:20:13.830 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.77e+00 / 1.30e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.48e+00, 4.13e-01 / 2.38e-01 | 20260508 06:20:13.830 | INFO | PerfReport: 3.16%, Metrics -- 1.03e+00 / 1.20e+00, 2.56e-01 / 1.21e-01 | 20260508 06:20:13.830 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.48e-01, 4.98e-02 / 2.76e-02 | 20260508 06:20:13.830 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:20:13.830 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:20:13.830 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:20:13.830 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.56e+01 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 24.56%, Upload -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 30.74%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 2.46%, LockUpload -- 2.61e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 2.43%, LockDownload -- 2.58e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 15.59%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:20:13.830 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.92e-02 / 0.00e+00 | 20260508 06:20:13.927 | INFO | Starting a round of Baz IO | 20260508 06:20:17.340 | INFO | Populating RTMetrics report | 20260508 06:20:17.340 | INFO | RTMetrics report populated | 20260508 06:20:17.340 | INFO | Opening tmp RTMetrics json file | 20260508 06:20:17.351 | INFO | Serializing RTMetrics to json | 20260508 06:20:17.352 | INFO | Serialization done, swapping tmp file now... | 20260508 06:20:17.353 | INFO | RTMetrics done | 20260508 06:20:17.414 | INFO | Finished a round of Baz IO: Wrote 1.554 GiB in 3.487 seconds | 20260508 06:20:17.414 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.00% / 59.28% | 20260508 06:20:17.414 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.405582% / 0% / 6.75519e-05% | 20260508 06:20:17.414 | INFO | Baz IO overall: | 20260508 06:20:17.414 | INFO | BytesWritten : 2620833855069 | 20260508 06:20:17.414 | INFO | NumEvents : 1263716745424 | 20260508 06:20:18.969 | INFO | Analyzing chunk frames = [5906432,5906944) | 20260508 06:20:18.969 | INFO | MemInfo: { | 20260508 06:20:18.969 | INFO | "gpu" : " 17 GB", | 20260508 06:20:18.969 | INFO | "huge" : "", | 20260508 06:20:18.969 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:18.969 | INFO | "malloc" : " 14 GB", | 20260508 06:20:18.969 | INFO | "pinned" : " 29 GB" | 20260508 06:20:18.969 | INFO | } | 20260508 06:20:18.969 | INFO | Perf report for the last 5.13994 seconds: | 20260508 06:20:18.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:18.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:18.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:18.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:18.969 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 06:20:18.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:18.969 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 06:20:18.969 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 06:20:18.969 | INFO | BAZWRITER : 67.2%, 2.7, 91.8ms, 0 | 20260508 06:20:18.969 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:20:18.969 | INFO | Uploaded 36GiB to GPU in 2.58005s (13.9532GiB/s) | 20260508 06:20:18.969 | INFO | Downloaded 38GiB to GPU in 2.6982s (14.0835GiB/s) | 20260508 06:20:24.104 | INFO | Analyzing chunk frames = [5906944,5907456) | 20260508 06:20:24.104 | INFO | MemInfo: { | 20260508 06:20:24.104 | INFO | "gpu" : " 17 GB", | 20260508 06:20:24.104 | INFO | "huge" : "", | 20260508 06:20:24.104 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:24.104 | INFO | "malloc" : " 14 GB", | 20260508 06:20:24.104 | INFO | "pinned" : " 29 GB" | 20260508 06:20:24.104 | INFO | } | 20260508 06:20:24.104 | INFO | Perf report for the last 5.13542 seconds: | 20260508 06:20:24.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:24.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.104 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:20:24.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.104 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:20:24.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:24.105 | INFO | Uploaded 36GiB to GPU in 2.49983s (14.401GiB/s) | 20260508 06:20:24.105 | INFO | Downloaded 34GiB to GPU in 2.44488s (13.9066GiB/s) | 20260508 06:20:29.243 | INFO | Analyzing chunk frames = [5907456,5907968) | 20260508 06:20:29.243 | INFO | MemInfo: { | 20260508 06:20:29.243 | INFO | "gpu" : " 17 GB", | 20260508 06:20:29.243 | INFO | "huge" : "", | 20260508 06:20:29.243 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:29.243 | INFO | "malloc" : " 14 GB", | 20260508 06:20:29.243 | INFO | "pinned" : " 29 GB" | 20260508 06:20:29.243 | INFO | } | 20260508 06:20:29.243 | INFO | Perf report for the last 5.13893 seconds: | 20260508 06:20:29.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:29.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.243 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:20:29.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.243 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:20:29.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:29.244 | INFO | Uploaded 36GiB to GPU in 2.5405s (14.1704GiB/s) | 20260508 06:20:29.244 | INFO | Downloaded 34GiB to GPU in 2.41473s (14.0803GiB/s) | 20260508 06:20:34.385 | INFO | Analyzing chunk frames = [5907968,5908480) | 20260508 06:20:34.385 | INFO | MemInfo: { | 20260508 06:20:34.385 | INFO | "gpu" : " 17 GB", | 20260508 06:20:34.385 | INFO | "huge" : "", | 20260508 06:20:34.385 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:34.385 | INFO | "malloc" : " 14 GB", | 20260508 06:20:34.385 | INFO | "pinned" : " 29 GB" | 20260508 06:20:34.385 | INFO | } | 20260508 06:20:34.385 | INFO | Perf report for the last 5.14208 seconds: | 20260508 06:20:34.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:34.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.385 | INFO | ANALYSIS : 66.7%, 2.9, 103.3ms, 0 | 20260508 06:20:34.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.385 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:20:34.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:34.386 | INFO | Uploaded 36GiB to GPU in 2.55164s (14.1086GiB/s) | 20260508 06:20:34.386 | INFO | Downloaded 34GiB to GPU in 2.40488s (14.1379GiB/s) | 20260508 06:20:39.525 | INFO | Analyzing chunk frames = [5908480,5908992) | 20260508 06:20:39.525 | INFO | MemInfo: { | 20260508 06:20:39.525 | INFO | "gpu" : " 17 GB", | 20260508 06:20:39.525 | INFO | "huge" : "", | 20260508 06:20:39.525 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:39.525 | INFO | "malloc" : " 14 GB", | 20260508 06:20:39.525 | INFO | "pinned" : " 29 GB" | 20260508 06:20:39.525 | INFO | } | 20260508 06:20:39.525 | INFO | Perf report for the last 5.14033 seconds: | 20260508 06:20:39.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:39.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:20:39.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 06:20:39.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:39.526 | INFO | Uploaded 36GiB to GPU in 2.38479s (15.0956GiB/s) | 20260508 06:20:39.526 | INFO | Downloaded 34GiB to GPU in 2.50263s (13.5857GiB/s) | 20260508 06:20:44.662 | INFO | Analyzing chunk frames = [5908992,5909504) | 20260508 06:20:44.662 | INFO | MemInfo: { | 20260508 06:20:44.662 | INFO | "gpu" : " 17 GB", | 20260508 06:20:44.662 | INFO | "huge" : "", | 20260508 06:20:44.662 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:44.662 | INFO | "malloc" : " 14 GB", | 20260508 06:20:44.662 | INFO | "pinned" : " 29 GB" | 20260508 06:20:44.662 | INFO | } | 20260508 06:20:44.662 | INFO | Perf report for the last 5.13682 seconds: | 20260508 06:20:44.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:44.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.662 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:20:44.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.662 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:20:44.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:44.663 | INFO | Uploaded 36GiB to GPU in 2.48036s (14.514GiB/s) | 20260508 06:20:44.663 | INFO | Downloaded 34GiB to GPU in 2.4222s (14.0368GiB/s) | 20260508 06:20:49.801 | INFO | Analyzing chunk frames = [5909504,5910016) | 20260508 06:20:49.801 | INFO | MemInfo: { | 20260508 06:20:49.801 | INFO | "gpu" : " 17 GB", | 20260508 06:20:49.801 | INFO | "huge" : "", | 20260508 06:20:49.801 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:49.801 | INFO | "malloc" : " 14 GB", | 20260508 06:20:49.801 | INFO | "pinned" : " 29 GB" | 20260508 06:20:49.801 | INFO | } | 20260508 06:20:49.801 | INFO | Perf report for the last 5.13912 seconds: | 20260508 06:20:49.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:49.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.801 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:20:49.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.802 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:20:49.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:49.802 | INFO | Uploaded 36GiB to GPU in 2.46786s (14.5875GiB/s) | 20260508 06:20:49.802 | INFO | Downloaded 34GiB to GPU in 2.41679s (14.0682GiB/s) | 20260508 06:20:54.935 | INFO | Analyzing chunk frames = [5910016,5910528) | 20260508 06:20:54.935 | INFO | MemInfo: { | 20260508 06:20:54.935 | INFO | "gpu" : " 17 GB", | 20260508 06:20:54.935 | INFO | "huge" : "", | 20260508 06:20:54.935 | INFO | "huge_shared" : " 29 GB", | 20260508 06:20:54.935 | INFO | "malloc" : " 14 GB", | 20260508 06:20:54.935 | INFO | "pinned" : " 29 GB" | 20260508 06:20:54.935 | INFO | } | 20260508 06:20:54.935 | INFO | Perf report for the last 5.13357 seconds: | 20260508 06:20:54.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:20:54.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.935 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:20:54.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.935 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:20:54.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:20:54.936 | INFO | Uploaded 36GiB to GPU in 2.46807s (14.5863GiB/s) | 20260508 06:20:54.936 | INFO | Downloaded 34GiB to GPU in 2.43051s (13.9888GiB/s) | 20260508 06:20:55.033 | INFO | Starting a round of Baz IO | 20260508 06:20:58.455 | INFO | Populating RTMetrics report | 20260508 06:20:58.456 | INFO | RTMetrics report populated | 20260508 06:20:58.456 | INFO | Opening tmp RTMetrics json file | 20260508 06:20:58.467 | INFO | Serializing RTMetrics to json | 20260508 06:20:58.468 | INFO | Serialization done, swapping tmp file now... | 20260508 06:20:58.469 | INFO | RTMetrics done | 20260508 06:20:58.530 | INFO | Finished a round of Baz IO: Wrote 1.553 GiB in 3.497 seconds | 20260508 06:20:58.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.00% / 59.28% | 20260508 06:20:58.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.404688% / 1.19209e-05% / 7.94729e-05% | 20260508 06:20:58.530 | INFO | Baz IO overall: | 20260508 06:20:58.530 | INFO | BytesWritten : 2622501130521 | 20260508 06:20:58.530 | INFO | NumEvents : 1264510652501 | 20260508 06:21:00.079 | INFO | Analyzing chunk frames = [5910528,5911040) | 20260508 06:21:00.079 | INFO | MemInfo: { | 20260508 06:21:00.079 | INFO | "gpu" : " 17 GB", | 20260508 06:21:00.079 | INFO | "huge" : "", | 20260508 06:21:00.079 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:00.079 | INFO | "malloc" : " 14 GB", | 20260508 06:21:00.079 | INFO | "pinned" : " 29 GB" | 20260508 06:21:00.079 | INFO | } | 20260508 06:21:00.079 | INFO | Perf report for the last 5.14432 seconds: | 20260508 06:21:00.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:00.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:00.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:00.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:00.080 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 06:21:00.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:00.080 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:21:00.080 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 06:21:00.080 | INFO | BAZWRITER : 67.4%, 2.6, 91.1ms, 0 | 20260508 06:21:00.080 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:21:00.080 | INFO | Uploaded 36GiB to GPU in 2.58065s (13.9499GiB/s) | 20260508 06:21:00.080 | INFO | Downloaded 38GiB to GPU in 2.69713s (14.0891GiB/s) | 20260508 06:21:05.223 | INFO | Analyzing chunk frames = [5911040,5911552) | 20260508 06:21:05.223 | INFO | MemInfo: { | 20260508 06:21:05.223 | INFO | "gpu" : " 17 GB", | 20260508 06:21:05.223 | INFO | "huge" : "", | 20260508 06:21:05.223 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:05.223 | INFO | "malloc" : " 14 GB", | 20260508 06:21:05.223 | INFO | "pinned" : " 29 GB" | 20260508 06:21:05.223 | INFO | } | 20260508 06:21:05.224 | INFO | Perf report for the last 5.14415 seconds: | 20260508 06:21:05.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:05.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 06:21:05.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 06:21:05.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:05.224 | INFO | Uploaded 36GiB to GPU in 2.52555s (14.2543GiB/s) | 20260508 06:21:05.224 | INFO | Downloaded 34GiB to GPU in 2.44253s (13.92GiB/s) | 20260508 06:21:10.366 | INFO | Analyzing chunk frames = [5911552,5912064) | 20260508 06:21:10.366 | INFO | MemInfo: { | 20260508 06:21:10.366 | INFO | "gpu" : " 17 GB", | 20260508 06:21:10.366 | INFO | "huge" : "", | 20260508 06:21:10.366 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:10.366 | INFO | "malloc" : " 14 GB", | 20260508 06:21:10.366 | INFO | "pinned" : " 29 GB" | 20260508 06:21:10.366 | INFO | } | 20260508 06:21:10.366 | INFO | Perf report for the last 5.1429 seconds: | 20260508 06:21:10.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:10.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.367 | INFO | ANALYSIS : 67.2%, 3.0, 104.5ms, 0 | 20260508 06:21:10.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.367 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:21:10.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:10.367 | INFO | Uploaded 36GiB to GPU in 2.55062s (14.1142GiB/s) | 20260508 06:21:10.367 | INFO | Downloaded 34GiB to GPU in 2.39857s (14.1751GiB/s) | 20260508 06:21:15.498 | INFO | Analyzing chunk frames = [5912064,5912576) | 20260508 06:21:15.498 | INFO | MemInfo: { | 20260508 06:21:15.498 | INFO | "gpu" : " 17 GB", | 20260508 06:21:15.498 | INFO | "huge" : "", | 20260508 06:21:15.498 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:15.498 | INFO | "malloc" : " 14 GB", | 20260508 06:21:15.498 | INFO | "pinned" : " 29 GB" | 20260508 06:21:15.498 | INFO | } | 20260508 06:21:15.499 | INFO | Perf report for the last 5.13208 seconds: | 20260508 06:21:15.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:15.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | ANALYSIS : 67.1%, 2.9, 103.6ms, 0 | 20260508 06:21:15.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:21:15.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:15.499 | INFO | Uploaded 36GiB to GPU in 2.5932s (13.8825GiB/s) | 20260508 06:21:15.499 | INFO | Downloaded 34GiB to GPU in 2.40639s (14.1291GiB/s) | 20260508 06:21:20.641 | INFO | Analyzing chunk frames = [5912576,5913088) | 20260508 06:21:20.641 | INFO | MemInfo: { | 20260508 06:21:20.641 | INFO | "gpu" : " 17 GB", | 20260508 06:21:20.641 | INFO | "huge" : "", | 20260508 06:21:20.641 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:20.641 | INFO | "malloc" : " 14 GB", | 20260508 06:21:20.641 | INFO | "pinned" : " 29 GB" | 20260508 06:21:20.641 | INFO | } | 20260508 06:21:20.642 | INFO | Perf report for the last 5.14295 seconds: | 20260508 06:21:20.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:20.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:21:20.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:21:20.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:20.642 | INFO | Uploaded 36GiB to GPU in 2.42653s (14.836GiB/s) | 20260508 06:21:20.642 | INFO | Downloaded 34GiB to GPU in 2.52389s (13.4713GiB/s) | 20260508 06:21:25.778 | INFO | Analyzing chunk frames = [5913088,5913600) | 20260508 06:21:25.778 | INFO | MemInfo: { | 20260508 06:21:25.778 | INFO | "gpu" : " 17 GB", | 20260508 06:21:25.778 | INFO | "huge" : "", | 20260508 06:21:25.778 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:25.778 | INFO | "malloc" : " 14 GB", | 20260508 06:21:25.779 | INFO | "pinned" : " 29 GB" | 20260508 06:21:25.779 | INFO | } | 20260508 06:21:25.779 | INFO | Perf report for the last 5.13713 seconds: | 20260508 06:21:25.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:25.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:21:25.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:21:25.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:25.779 | INFO | Uploaded 36GiB to GPU in 2.4915s (14.4491GiB/s) | 20260508 06:21:25.779 | INFO | Downloaded 34GiB to GPU in 2.42928s (13.9959GiB/s) | 20260508 06:21:30.917 | INFO | Analyzing chunk frames = [5913600,5914112) | 20260508 06:21:30.917 | INFO | MemInfo: { | 20260508 06:21:30.917 | INFO | "gpu" : " 17 GB", | 20260508 06:21:30.917 | INFO | "huge" : "", | 20260508 06:21:30.917 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:30.917 | INFO | "malloc" : " 14 GB", | 20260508 06:21:30.917 | INFO | "pinned" : " 29 GB" | 20260508 06:21:30.917 | INFO | } | 20260508 06:21:30.917 | INFO | Perf report for the last 5.13875 seconds: | 20260508 06:21:30.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:30.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.917 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:21:30.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.918 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:21:30.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:30.918 | INFO | Uploaded 36GiB to GPU in 2.4883s (14.4677GiB/s) | 20260508 06:21:30.918 | INFO | Downloaded 34GiB to GPU in 2.42742s (14.0067GiB/s) | 20260508 06:21:36.056 | INFO | Analyzing chunk frames = [5914112,5914624) | 20260508 06:21:36.056 | INFO | MemInfo: { | 20260508 06:21:36.056 | INFO | "gpu" : " 17 GB", | 20260508 06:21:36.056 | INFO | "huge" : "", | 20260508 06:21:36.056 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:36.056 | INFO | "malloc" : " 14 GB", | 20260508 06:21:36.056 | INFO | "pinned" : " 29 GB" | 20260508 06:21:36.056 | INFO | } | 20260508 06:21:36.056 | INFO | Perf report for the last 5.13872 seconds: | 20260508 06:21:36.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:36.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.056 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:21:36.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.056 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:21:36.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:36.057 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:21:36.057 | INFO | Total mem usage: 16.7 GiB | 20260508 06:21:36.057 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:21:36.057 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 06:21:36.057 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:21:36.057 | INFO | High Water Breakdown: | 20260508 06:21:36.057 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:21:36.057 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:21:36.057 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:21:36.057 | INFO | ------------------------------------------ | 20260508 06:21:36.057 | INFO | Uploaded 36GiB to GPU in 2.4694s (14.5785GiB/s) | 20260508 06:21:36.057 | INFO | Downloaded 34GiB to GPU in 2.43479s (13.9642GiB/s) | 20260508 06:21:36.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:21:36.057 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:21:36.057 | INFO | Total mem usage: 26.5 GiB | 20260508 06:21:36.057 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:21:36.057 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:21:36.057 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:21:36.057 | INFO | High Water Breakdown: | 20260508 06:21:36.057 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:21:36.057 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:21:36.057 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:21:36.057 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:21:36.057 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:21:36.057 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:21:36.057 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:21:36.057 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:21:36.057 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:21:36.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:21:36.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:21:36.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:21:36.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:21:36.057 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:21:36.057 | INFO | ------------------------------------------ | 20260508 06:21:36.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:21:36.057 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:21:36.057 | INFO | Total mem usage: 27.8 GiB | 20260508 06:21:36.057 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:21:36.057 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:21:36.057 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:21:36.057 | INFO | High Water Breakdown: | 20260508 06:21:36.057 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:21:36.057 | INFO | ------------------------------------------ | 20260508 06:21:36.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:21:36.057 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:21:36.057 | INFO | Total mem usage: 15.7 GiB | 20260508 06:21:36.057 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:21:36.057 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:21:36.057 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:21:36.057 | INFO | High Water Breakdown: | 20260508 06:21:36.057 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:21:36.057 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:21:36.057 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:21:36.057 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:21:36.057 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:21:36.057 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:21:36.057 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:21:36.057 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:21:36.057 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:21:36.057 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:21:36.057 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:21:36.057 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:21:36.057 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:21:36.057 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:21:36.057 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:21:36.057 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:21:36.057 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:21:36.057 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:21:36.057 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:21:36.057 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:21:36.057 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:21:36.057 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:21:36.057 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:21:36.057 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:21:36.057 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:21:36.057 | INFO | ------------------------------------------ | 20260508 06:21:36.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:21:36.058 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:21:36.058 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:21:36.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:21:36.058 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.78e-01 | 20260508 06:21:36.058 | INFO | PerfReport: 0.29%, AggregateStats -- 9.61e-02 / 1.06e-01, 1.10e-02 / 7.87e-03 | 20260508 06:21:36.058 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.40e+00, 5.86e-01 / 1.94e-01 | 20260508 06:21:36.058 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.08e-01 / 1.34e-01 | 20260508 06:21:36.058 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.47e+01, 3.69e+00 / 1.10e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.32e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.52e+00, 4.11e-01 / 2.04e-01 | 20260508 06:21:36.058 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.24e+00, 2.56e-01 / 1.65e-01 | 20260508 06:21:36.058 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.48e-01, 4.96e-02 / 2.42e-02 | 20260508 06:21:36.058 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:21:36.058 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:21:36.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:21:36.058 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 24.58%, Upload -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 30.76%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 2.42%, LockUpload -- 2.57e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 2.45%, LockDownload -- 2.60e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.97e+01 / 0.00e+00 | 20260508 06:21:36.058 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260508 06:21:36.166 | INFO | Starting a round of Baz IO | 20260508 06:21:39.570 | INFO | Populating RTMetrics report | 20260508 06:21:39.571 | INFO | RTMetrics report populated | 20260508 06:21:39.571 | INFO | Opening tmp RTMetrics json file | 20260508 06:21:39.583 | INFO | Serializing RTMetrics to json | 20260508 06:21:39.584 | INFO | Serialization done, swapping tmp file now... | 20260508 06:21:39.585 | INFO | RTMetrics done | 20260508 06:21:39.645 | INFO | Finished a round of Baz IO: Wrote 1.553 GiB in 3.478 seconds | 20260508 06:21:39.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.00% / 59.28% | 20260508 06:21:39.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.404131% / 0% / 9.93411e-05% | 20260508 06:21:39.645 | INFO | Baz IO overall: | 20260508 06:21:39.645 | INFO | BytesWritten : 2624168313171 | 20260508 06:21:39.645 | INFO | NumEvents : 1265304669935 | 20260508 06:21:41.193 | INFO | Analyzing chunk frames = [5914624,5915136) | 20260508 06:21:41.194 | INFO | MemInfo: { | 20260508 06:21:41.194 | INFO | "gpu" : " 17 GB", | 20260508 06:21:41.194 | INFO | "huge" : "", | 20260508 06:21:41.194 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:41.194 | INFO | "malloc" : " 14 GB", | 20260508 06:21:41.194 | INFO | "pinned" : " 29 GB" | 20260508 06:21:41.194 | INFO | } | 20260508 06:21:41.194 | INFO | Perf report for the last 5.1376 seconds: | 20260508 06:21:41.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:41.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:41.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:41.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:41.194 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 06:21:41.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:41.194 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 06:21:41.194 | INFO | PRE_HQ : 66.3%, 1.7, 59.4ms, 0 | 20260508 06:21:41.194 | INFO | BAZWRITER : 67.1%, 2.8, 95.4ms, 0 | 20260508 06:21:41.194 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:21:41.194 | INFO | Uploaded 36GiB to GPU in 2.6095s (13.7957GiB/s) | 20260508 06:21:41.195 | INFO | Downloaded 38GiB to GPU in 2.69696s (14.09GiB/s) | 20260508 06:21:46.333 | INFO | Analyzing chunk frames = [5915136,5915648) | 20260508 06:21:46.333 | INFO | MemInfo: { | 20260508 06:21:46.333 | INFO | "gpu" : " 17 GB", | 20260508 06:21:46.333 | INFO | "huge" : "", | 20260508 06:21:46.333 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:46.333 | INFO | "malloc" : " 14 GB", | 20260508 06:21:46.334 | INFO | "pinned" : " 29 GB" | 20260508 06:21:46.334 | INFO | } | 20260508 06:21:46.334 | INFO | Perf report for the last 5.13984 seconds: | 20260508 06:21:46.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:46.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:21:46.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:21:46.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:46.334 | INFO | Uploaded 36GiB to GPU in 2.50754s (14.3567GiB/s) | 20260508 06:21:46.334 | INFO | Downloaded 34GiB to GPU in 2.43816s (13.9449GiB/s) | 20260508 06:21:51.475 | INFO | Analyzing chunk frames = [5915648,5916160) | 20260508 06:21:51.475 | INFO | MemInfo: { | 20260508 06:21:51.475 | INFO | "gpu" : " 17 GB", | 20260508 06:21:51.475 | INFO | "huge" : "", | 20260508 06:21:51.475 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:51.475 | INFO | "malloc" : " 14 GB", | 20260508 06:21:51.475 | INFO | "pinned" : " 29 GB" | 20260508 06:21:51.475 | INFO | } | 20260508 06:21:51.475 | INFO | Perf report for the last 5.14169 seconds: | 20260508 06:21:51.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:51.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.475 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:21:51.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.475 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:21:51.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:51.476 | INFO | Uploaded 36GiB to GPU in 2.5558s (14.0856GiB/s) | 20260508 06:21:51.476 | INFO | Downloaded 34GiB to GPU in 2.39684s (14.1853GiB/s) | 20260508 06:21:56.613 | INFO | Analyzing chunk frames = [5916160,5916672) | 20260508 06:21:56.614 | INFO | MemInfo: { | 20260508 06:21:56.614 | INFO | "gpu" : " 17 GB", | 20260508 06:21:56.614 | INFO | "huge" : "", | 20260508 06:21:56.614 | INFO | "huge_shared" : " 29 GB", | 20260508 06:21:56.614 | INFO | "malloc" : " 14 GB", | 20260508 06:21:56.614 | INFO | "pinned" : " 29 GB" | 20260508 06:21:56.614 | INFO | } | 20260508 06:21:56.614 | INFO | Perf report for the last 5.13844 seconds: | 20260508 06:21:56.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:21:56.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:21:56.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:21:56.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:21:56.614 | INFO | Uploaded 36GiB to GPU in 2.58609s (13.9206GiB/s) | 20260508 06:21:56.614 | INFO | Downloaded 34GiB to GPU in 2.40016s (14.1657GiB/s) | 20260508 06:22:01.747 | INFO | Analyzing chunk frames = [5916672,5917184) | 20260508 06:22:01.747 | INFO | MemInfo: { | 20260508 06:22:01.747 | INFO | "gpu" : " 17 GB", | 20260508 06:22:01.747 | INFO | "huge" : "", | 20260508 06:22:01.747 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:01.747 | INFO | "malloc" : " 14 GB", | 20260508 06:22:01.747 | INFO | "pinned" : " 29 GB" | 20260508 06:22:01.748 | INFO | } | 20260508 06:22:01.748 | INFO | Perf report for the last 5.13386 seconds: | 20260508 06:22:01.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:01.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 06:22:01.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:22:01.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:01.748 | INFO | Uploaded 36GiB to GPU in 2.39954s (15.0029GiB/s) | 20260508 06:22:01.748 | INFO | Downloaded 34GiB to GPU in 2.51829s (13.5012GiB/s) | 20260508 06:22:06.894 | INFO | Analyzing chunk frames = [5917184,5917696) | 20260508 06:22:06.894 | INFO | MemInfo: { | 20260508 06:22:06.894 | INFO | "gpu" : " 17 GB", | 20260508 06:22:06.894 | INFO | "huge" : "", | 20260508 06:22:06.894 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:06.894 | INFO | "malloc" : " 14 GB", | 20260508 06:22:06.894 | INFO | "pinned" : " 29 GB" | 20260508 06:22:06.894 | INFO | } | 20260508 06:22:06.894 | INFO | Perf report for the last 5.14673 seconds: | 20260508 06:22:06.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:06.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.894 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 06:22:06.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.894 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:22:06.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:06.895 | INFO | Uploaded 36GiB to GPU in 2.4489s (14.7005GiB/s) | 20260508 06:22:06.895 | INFO | Downloaded 34GiB to GPU in 2.41213s (14.0954GiB/s) | 20260508 06:22:12.032 | INFO | Analyzing chunk frames = [5917696,5918208) | 20260508 06:22:12.032 | INFO | MemInfo: { | 20260508 06:22:12.032 | INFO | "gpu" : " 17 GB", | 20260508 06:22:12.032 | INFO | "huge" : "", | 20260508 06:22:12.032 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:12.032 | INFO | "malloc" : " 14 GB", | 20260508 06:22:12.032 | INFO | "pinned" : " 29 GB" | 20260508 06:22:12.032 | INFO | } | 20260508 06:22:12.033 | INFO | Perf report for the last 5.13809 seconds: | 20260508 06:22:12.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:12.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:22:12.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:22:12.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:12.033 | INFO | Uploaded 36GiB to GPU in 2.48303s (14.4984GiB/s) | 20260508 06:22:12.033 | INFO | Downloaded 34GiB to GPU in 2.42432s (14.0246GiB/s) | 20260508 06:22:17.174 | INFO | Analyzing chunk frames = [5918208,5918720) | 20260508 06:22:17.174 | INFO | MemInfo: { | 20260508 06:22:17.174 | INFO | "gpu" : " 17 GB", | 20260508 06:22:17.174 | INFO | "huge" : "", | 20260508 06:22:17.174 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:17.175 | INFO | "malloc" : " 14 GB", | 20260508 06:22:17.175 | INFO | "pinned" : " 29 GB" | 20260508 06:22:17.175 | INFO | } | 20260508 06:22:17.175 | INFO | Perf report for the last 5.14235 seconds: | 20260508 06:22:17.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:17.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:22:17.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 06:22:17.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:17.175 | INFO | Uploaded 36GiB to GPU in 2.49204s (14.446GiB/s) | 20260508 06:22:17.175 | INFO | Downloaded 34GiB to GPU in 2.4429s (13.9179GiB/s) | 20260508 06:22:17.291 | INFO | Starting a round of Baz IO | 20260508 06:22:20.704 | INFO | Populating RTMetrics report | 20260508 06:22:20.704 | INFO | RTMetrics report populated | 20260508 06:22:20.704 | INFO | Opening tmp RTMetrics json file | 20260508 06:22:20.705 | INFO | Serializing RTMetrics to json | 20260508 06:22:20.706 | INFO | Serialization done, swapping tmp file now... | 20260508 06:22:20.707 | INFO | RTMetrics done | 20260508 06:22:20.773 | INFO | Finished a round of Baz IO: Wrote 1.552 GiB in 3.482 seconds | 20260508 06:22:20.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.71% / 0.00% / 59.28% | 20260508 06:22:20.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.402685% / 3.97364e-06% / 8.74201e-05% | 20260508 06:22:20.773 | INFO | Baz IO overall: | 20260508 06:22:20.773 | INFO | BytesWritten : 2625834726502 | 20260508 06:22:20.773 | INFO | NumEvents : 1266098357784 | 20260508 06:22:22.309 | INFO | Analyzing chunk frames = [5918720,5919232) | 20260508 06:22:22.309 | INFO | MemInfo: { | 20260508 06:22:22.309 | INFO | "gpu" : " 17 GB", | 20260508 06:22:22.309 | INFO | "huge" : "", | 20260508 06:22:22.309 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:22.309 | INFO | "malloc" : " 14 GB", | 20260508 06:22:22.309 | INFO | "pinned" : " 29 GB" | 20260508 06:22:22.309 | INFO | } | 20260508 06:22:22.309 | INFO | Perf report for the last 5.13453 seconds: | 20260508 06:22:22.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:22.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:22.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:22.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:22.309 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 06:22:22.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:22.309 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 06:22:22.310 | INFO | PRE_HQ : 66.5%, 1.8, 61.5ms, 0 | 20260508 06:22:22.310 | INFO | BAZWRITER : 67.2%, 2.8, 98.0ms, 0 | 20260508 06:22:22.310 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:22:22.310 | INFO | Uploaded 36GiB to GPU in 2.57788s (13.965GiB/s) | 20260508 06:22:22.310 | INFO | Downloaded 38GiB to GPU in 2.7001s (14.0736GiB/s) | 20260508 06:22:27.445 | INFO | Analyzing chunk frames = [5919232,5919744) | 20260508 06:22:27.445 | INFO | MemInfo: { | 20260508 06:22:27.445 | INFO | "gpu" : " 17 GB", | 20260508 06:22:27.445 | INFO | "huge" : "", | 20260508 06:22:27.445 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:27.445 | INFO | "malloc" : " 14 GB", | 20260508 06:22:27.445 | INFO | "pinned" : " 29 GB" | 20260508 06:22:27.445 | INFO | } | 20260508 06:22:27.445 | INFO | Perf report for the last 5.13573 seconds: | 20260508 06:22:27.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:27.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 06:22:27.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:22:27.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:27.445 | INFO | Uploaded 36GiB to GPU in 2.50826s (14.3526GiB/s) | 20260508 06:22:27.446 | INFO | Downloaded 34GiB to GPU in 2.4276s (14.0056GiB/s) | 20260508 06:22:32.590 | INFO | Analyzing chunk frames = [5919744,5920256) | 20260508 06:22:32.590 | INFO | MemInfo: { | 20260508 06:22:32.590 | INFO | "gpu" : " 17 GB", | 20260508 06:22:32.590 | INFO | "huge" : "", | 20260508 06:22:32.590 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:32.590 | INFO | "malloc" : " 14 GB", | 20260508 06:22:32.590 | INFO | "pinned" : " 29 GB" | 20260508 06:22:32.590 | INFO | } | 20260508 06:22:32.591 | INFO | Perf report for the last 5.1454 seconds: | 20260508 06:22:32.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:32.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 06:22:32.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:22:32.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:32.591 | INFO | Uploaded 36GiB to GPU in 2.5699s (14.0084GiB/s) | 20260508 06:22:32.591 | INFO | Downloaded 34GiB to GPU in 2.40914s (14.1129GiB/s) | 20260508 06:22:37.730 | INFO | Analyzing chunk frames = [5920256,5920768) | 20260508 06:22:37.730 | INFO | MemInfo: { | 20260508 06:22:37.730 | INFO | "gpu" : " 17 GB", | 20260508 06:22:37.730 | INFO | "huge" : "", | 20260508 06:22:37.730 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:37.730 | INFO | "malloc" : " 14 GB", | 20260508 06:22:37.730 | INFO | "pinned" : " 29 GB" | 20260508 06:22:37.730 | INFO | } | 20260508 06:22:37.730 | INFO | Perf report for the last 5.13981 seconds: | 20260508 06:22:37.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:37.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.730 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 06:22:37.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.731 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:22:37.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:37.731 | INFO | Uploaded 36GiB to GPU in 2.58269s (13.939GiB/s) | 20260508 06:22:37.731 | INFO | Downloaded 34GiB to GPU in 2.39793s (14.1789GiB/s) | 20260508 06:22:42.870 | INFO | Analyzing chunk frames = [5920768,5921280) | 20260508 06:22:42.870 | INFO | MemInfo: { | 20260508 06:22:42.870 | INFO | "gpu" : " 17 GB", | 20260508 06:22:42.870 | INFO | "huge" : "", | 20260508 06:22:42.870 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:42.870 | INFO | "malloc" : " 14 GB", | 20260508 06:22:42.870 | INFO | "pinned" : " 29 GB" | 20260508 06:22:42.870 | INFO | } | 20260508 06:22:42.870 | INFO | Perf report for the last 5.14007 seconds: | 20260508 06:22:42.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:42.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.871 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:22:42.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.871 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:22:42.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:42.871 | INFO | Uploaded 36GiB to GPU in 2.30169s (15.6407GiB/s) | 20260508 06:22:42.871 | INFO | Downloaded 34GiB to GPU in 2.494s (13.6327GiB/s) | 20260508 06:22:48.007 | INFO | Analyzing chunk frames = [5921280,5921792) | 20260508 06:22:48.007 | INFO | MemInfo: { | 20260508 06:22:48.007 | INFO | "gpu" : " 17 GB", | 20260508 06:22:48.007 | INFO | "huge" : "", | 20260508 06:22:48.007 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:48.007 | INFO | "malloc" : " 14 GB", | 20260508 06:22:48.007 | INFO | "pinned" : " 29 GB" | 20260508 06:22:48.007 | INFO | } | 20260508 06:22:48.007 | INFO | Perf report for the last 5.13648 seconds: | 20260508 06:22:48.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:48.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.007 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:22:48.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.007 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:22:48.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:48.008 | INFO | Uploaded 36GiB to GPU in 2.4664s (14.5962GiB/s) | 20260508 06:22:48.008 | INFO | Downloaded 34GiB to GPU in 2.44102s (13.9286GiB/s) | 20260508 06:22:53.143 | INFO | Analyzing chunk frames = [5921792,5922304) | 20260508 06:22:53.143 | INFO | MemInfo: { | 20260508 06:22:53.143 | INFO | "gpu" : " 17 GB", | 20260508 06:22:53.143 | INFO | "huge" : "", | 20260508 06:22:53.143 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:53.143 | INFO | "malloc" : " 14 GB", | 20260508 06:22:53.144 | INFO | "pinned" : " 29 GB" | 20260508 06:22:53.144 | INFO | } | 20260508 06:22:53.144 | INFO | Perf report for the last 5.13668 seconds: | 20260508 06:22:53.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:53.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.144 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:22:53.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.144 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 06:22:53.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:53.145 | INFO | Uploaded 36GiB to GPU in 2.48459s (14.4893GiB/s) | 20260508 06:22:53.145 | INFO | Downloaded 34GiB to GPU in 2.4339s (13.9694GiB/s) | 20260508 06:22:58.285 | INFO | Analyzing chunk frames = [5922304,5922816) | 20260508 06:22:58.285 | INFO | MemInfo: { | 20260508 06:22:58.285 | INFO | "gpu" : " 17 GB", | 20260508 06:22:58.285 | INFO | "huge" : "", | 20260508 06:22:58.285 | INFO | "huge_shared" : " 29 GB", | 20260508 06:22:58.285 | INFO | "malloc" : " 14 GB", | 20260508 06:22:58.286 | INFO | "pinned" : " 29 GB" | 20260508 06:22:58.286 | INFO | } | 20260508 06:22:58.286 | INFO | Perf report for the last 5.14208 seconds: | 20260508 06:22:58.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:22:58.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 06:22:58.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 06:22:58.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:22:58.286 | INFO | Uploaded 36GiB to GPU in 2.47637s (14.5374GiB/s) | 20260508 06:22:58.286 | INFO | Downloaded 34GiB to GPU in 2.43632s (13.9555GiB/s) | 20260508 06:22:58.287 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:22:58.287 | INFO | Total mem usage: 16.7 GiB | 20260508 06:22:58.287 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:22:58.287 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.3%) | 20260508 06:22:58.287 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:22:58.287 | INFO | High Water Breakdown: | 20260508 06:22:58.287 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:22:58.287 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:22:58.287 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:22:58.287 | INFO | ------------------------------------------ | 20260508 06:22:58.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:22:58.287 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:22:58.287 | INFO | Total mem usage: 26.5 GiB | 20260508 06:22:58.287 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:22:58.287 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:22:58.287 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:22:58.287 | INFO | High Water Breakdown: | 20260508 06:22:58.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:22:58.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:22:58.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:22:58.287 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:22:58.287 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:22:58.287 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:22:58.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:22:58.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:22:58.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:22:58.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:22:58.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:22:58.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:22:58.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:22:58.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:22:58.287 | INFO | ------------------------------------------ | 20260508 06:22:58.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:22:58.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:22:58.287 | INFO | Total mem usage: 27.8 GiB | 20260508 06:22:58.287 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:22:58.287 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:22:58.287 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:22:58.287 | INFO | High Water Breakdown: | 20260508 06:22:58.287 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:22:58.287 | INFO | ------------------------------------------ | 20260508 06:22:58.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:22:58.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:22:58.287 | INFO | Total mem usage: 15.7 GiB | 20260508 06:22:58.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:22:58.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:22:58.287 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:22:58.287 | INFO | High Water Breakdown: | 20260508 06:22:58.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:22:58.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:22:58.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:22:58.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:22:58.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:22:58.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:22:58.287 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:22:58.287 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:22:58.287 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:22:58.287 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:22:58.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:22:58.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:22:58.288 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:22:58.288 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:22:58.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:22:58.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:22:58.288 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:22:58.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:22:58.288 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:22:58.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:22:58.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:22:58.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:22:58.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:22:58.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:22:58.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:22:58.288 | INFO | ------------------------------------------ | 20260508 06:22:58.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:22:58.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:22:58.288 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:22:58.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:22:58.288 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.83e-01 | 20260508 06:22:58.288 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.15e-02 / 5.39e-03 | 20260508 06:22:58.288 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.44e+00, 5.84e-01 / 2.32e-01 | 20260508 06:22:58.288 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.05e-01 | 20260508 06:22:58.288 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.50e+01, 3.83e+00 / 9.35e-01 | 20260508 06:22:58.288 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 1.43e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.57e+00, 4.08e-01 / 2.51e-01 | 20260508 06:22:58.288 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.18e+00, 2.53e-01 / 1.25e-01 | 20260508 06:22:58.288 | INFO | PerfReport: 0.97%, PulseSerializing -- 3.16e-01 / 3.48e-01, 5.01e-02 / 2.54e-02 | 20260508 06:22:58.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:22:58.288 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:22:58.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:22:58.288 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 24.56%, Upload -- 2.61e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 24.20%, Download -- 2.57e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 2.45%, LockUpload -- 2.60e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 2.40%, LockDownload -- 2.55e+00 / 0.00e+00, 3.84e+00 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 06:22:58.288 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260508 06:22:58.397 | INFO | Starting a round of Baz IO | 20260508 06:23:01.823 | INFO | Populating RTMetrics report | 20260508 06:23:01.823 | INFO | RTMetrics report populated | 20260508 06:23:01.823 | INFO | Opening tmp RTMetrics json file | 20260508 06:23:01.835 | INFO | Serializing RTMetrics to json | 20260508 06:23:01.837 | INFO | Serialization done, swapping tmp file now... | 20260508 06:23:01.838 | INFO | RTMetrics done | 20260508 06:23:01.890 | INFO | Finished a round of Baz IO: Wrote 1.553 GiB in 3.493 seconds | 20260508 06:23:01.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.71% / 0.00% / 59.29% | 20260508 06:23:01.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.402272% / 1.19209e-05% / 5.96046e-05% | 20260508 06:23:01.890 | INFO | Baz IO overall: | 20260508 06:23:01.890 | INFO | BytesWritten : 2627502208592 | 20260508 06:23:01.890 | INFO | NumEvents : 1266892025614 | 20260508 06:23:03.427 | INFO | Analyzing chunk frames = [5922816,5923328) | 20260508 06:23:03.427 | INFO | MemInfo: { | 20260508 06:23:03.428 | INFO | "gpu" : " 17 GB", | 20260508 06:23:03.428 | INFO | "huge" : "", | 20260508 06:23:03.428 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:03.428 | INFO | "malloc" : " 14 GB", | 20260508 06:23:03.428 | INFO | "pinned" : " 29 GB" | 20260508 06:23:03.428 | INFO | } | 20260508 06:23:03.428 | INFO | Perf report for the last 5.14202 seconds: | 20260508 06:23:03.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:03.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:03.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:03.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:03.428 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 06:23:03.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:03.428 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 06:23:03.428 | INFO | PRE_HQ : 66.6%, 1.8, 62.0ms, 0 | 20260508 06:23:03.428 | INFO | BAZWRITER : 67.3%, 2.8, 96.7ms, 0 | 20260508 06:23:03.428 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 06:23:03.428 | INFO | Uploaded 36GiB to GPU in 2.55948s (14.0653GiB/s) | 20260508 06:23:03.428 | INFO | Downloaded 38GiB to GPU in 2.70242s (14.0615GiB/s) | 20260508 06:23:08.566 | INFO | Analyzing chunk frames = [5923328,5923840) | 20260508 06:23:08.566 | INFO | MemInfo: { | 20260508 06:23:08.566 | INFO | "gpu" : " 17 GB", | 20260508 06:23:08.566 | INFO | "huge" : "", | 20260508 06:23:08.566 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:08.566 | INFO | "malloc" : " 14 GB", | 20260508 06:23:08.566 | INFO | "pinned" : " 29 GB" | 20260508 06:23:08.566 | INFO | } | 20260508 06:23:08.566 | INFO | Perf report for the last 5.1381 seconds: | 20260508 06:23:08.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:08.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:23:08.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:23:08.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:08.566 | INFO | Uploaded 36GiB to GPU in 2.48604s (14.4809GiB/s) | 20260508 06:23:08.566 | INFO | Downloaded 34GiB to GPU in 2.44416s (13.9107GiB/s) | 20260508 06:23:13.699 | INFO | Analyzing chunk frames = [5923840,5924352) | 20260508 06:23:13.699 | INFO | MemInfo: { | 20260508 06:23:13.699 | INFO | "gpu" : " 17 GB", | 20260508 06:23:13.699 | INFO | "huge" : "", | 20260508 06:23:13.699 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:13.699 | INFO | "malloc" : " 14 GB", | 20260508 06:23:13.699 | INFO | "pinned" : " 29 GB" | 20260508 06:23:13.699 | INFO | } | 20260508 06:23:13.699 | INFO | Perf report for the last 5.13324 seconds: | 20260508 06:23:13.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:13.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 06:23:13.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:23:13.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:13.699 | INFO | Uploaded 36GiB to GPU in 2.56173s (14.053GiB/s) | 20260508 06:23:13.699 | INFO | Downloaded 34GiB to GPU in 2.43187s (13.981GiB/s) | 20260508 06:23:18.842 | INFO | Analyzing chunk frames = [5924352,5924864) | 20260508 06:23:18.842 | INFO | MemInfo: { | 20260508 06:23:18.842 | INFO | "gpu" : " 17 GB", | 20260508 06:23:18.842 | INFO | "huge" : "", | 20260508 06:23:18.842 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:18.842 | INFO | "malloc" : " 14 GB", | 20260508 06:23:18.842 | INFO | "pinned" : " 29 GB" | 20260508 06:23:18.842 | INFO | } | 20260508 06:23:18.842 | INFO | Perf report for the last 5.14289 seconds: | 20260508 06:23:18.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:18.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 06:23:18.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:23:18.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:18.842 | INFO | Uploaded 36GiB to GPU in 2.56829s (14.0171GiB/s) | 20260508 06:23:18.842 | INFO | Downloaded 34GiB to GPU in 2.39608s (14.1899GiB/s) | 20260508 06:23:23.986 | INFO | Analyzing chunk frames = [5924864,5925376) | 20260508 06:23:23.986 | INFO | MemInfo: { | 20260508 06:23:23.986 | INFO | "gpu" : " 17 GB", | 20260508 06:23:23.986 | INFO | "huge" : "", | 20260508 06:23:23.986 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:23.986 | INFO | "malloc" : " 14 GB", | 20260508 06:23:23.986 | INFO | "pinned" : " 29 GB" | 20260508 06:23:23.986 | INFO | } | 20260508 06:23:23.986 | INFO | Perf report for the last 5.14402 seconds: | 20260508 06:23:23.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:23.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.986 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 06:23:23.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.986 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:23:23.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:23.987 | INFO | Uploaded 36GiB to GPU in 2.33206s (15.437GiB/s) | 20260508 06:23:23.987 | INFO | Downloaded 34GiB to GPU in 2.47959s (13.7119GiB/s) | 20260508 06:23:29.124 | INFO | Analyzing chunk frames = [5925376,5925888) | 20260508 06:23:29.124 | INFO | MemInfo: { | 20260508 06:23:29.124 | INFO | "gpu" : " 17 GB", | 20260508 06:23:29.124 | INFO | "huge" : "", | 20260508 06:23:29.124 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:29.124 | INFO | "malloc" : " 14 GB", | 20260508 06:23:29.124 | INFO | "pinned" : " 29 GB" | 20260508 06:23:29.124 | INFO | } | 20260508 06:23:29.125 | INFO | Perf report for the last 5.13842 seconds: | 20260508 06:23:29.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:29.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 06:23:29.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:23:29.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:29.125 | INFO | Uploaded 36GiB to GPU in 2.4612s (14.627GiB/s) | 20260508 06:23:29.125 | INFO | Downloaded 34GiB to GPU in 2.4224s (14.0357GiB/s) | 20260508 06:23:34.262 | INFO | Analyzing chunk frames = [5925888,5926400) | 20260508 06:23:34.262 | INFO | MemInfo: { | 20260508 06:23:34.262 | INFO | "gpu" : " 17 GB", | 20260508 06:23:34.262 | INFO | "huge" : "", | 20260508 06:23:34.262 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:34.262 | INFO | "malloc" : " 14 GB", | 20260508 06:23:34.262 | INFO | "pinned" : " 29 GB" | 20260508 06:23:34.263 | INFO | } | 20260508 06:23:34.263 | INFO | Perf report for the last 5.13807 seconds: | 20260508 06:23:34.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:34.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:23:34.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 06:23:34.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:34.263 | INFO | Uploaded 36GiB to GPU in 2.47985s (14.517GiB/s) | 20260508 06:23:34.263 | INFO | Downloaded 34GiB to GPU in 2.43585s (13.9582GiB/s) | 20260508 06:23:39.405 | INFO | Analyzing chunk frames = [5926400,5926912) | 20260508 06:23:39.405 | INFO | MemInfo: { | 20260508 06:23:39.405 | INFO | "gpu" : " 17 GB", | 20260508 06:23:39.405 | INFO | "huge" : "", | 20260508 06:23:39.405 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:39.405 | INFO | "malloc" : " 14 GB", | 20260508 06:23:39.405 | INFO | "pinned" : " 29 GB" | 20260508 06:23:39.405 | INFO | } | 20260508 06:23:39.406 | INFO | Perf report for the last 5.14294 seconds: | 20260508 06:23:39.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:39.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 06:23:39.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | PRE_HQ : 34.4%, 1.0, 17.9ms, 0 | 20260508 06:23:39.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:39.406 | INFO | Uploaded 36GiB to GPU in 2.46786s (14.5875GiB/s) | 20260508 06:23:39.406 | INFO | Downloaded 34GiB to GPU in 2.44597s (13.9004GiB/s) | 20260508 06:23:39.519 | INFO | Starting a round of Baz IO | 20260508 06:23:42.932 | INFO | Populating RTMetrics report | 20260508 06:23:42.933 | INFO | RTMetrics report populated | 20260508 06:23:42.933 | INFO | Opening tmp RTMetrics json file | 20260508 06:23:42.933 | INFO | Serializing RTMetrics to json | 20260508 06:23:42.935 | INFO | Serialization done, swapping tmp file now... | 20260508 06:23:42.935 | INFO | RTMetrics done | 20260508 06:23:43.003 | INFO | Finished a round of Baz IO: Wrote 1.553 GiB in 3.483 seconds | 20260508 06:23:43.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.71% / 0.00% / 59.29% | 20260508 06:23:43.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.400368% / 3.97364e-06% / 7.15256e-05% | 20260508 06:23:43.003 | INFO | Baz IO overall: | 20260508 06:23:43.003 | INFO | BytesWritten : 2629169352131 | 20260508 06:23:43.003 | INFO | NumEvents : 1267685295627 | 20260508 06:23:44.542 | INFO | Analyzing chunk frames = [5926912,5927424) | 20260508 06:23:44.542 | INFO | MemInfo: { | 20260508 06:23:44.542 | INFO | "gpu" : " 17 GB", | 20260508 06:23:44.542 | INFO | "huge" : "", | 20260508 06:23:44.542 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:44.542 | INFO | "malloc" : " 14 GB", | 20260508 06:23:44.542 | INFO | "pinned" : " 29 GB" | 20260508 06:23:44.542 | INFO | } | 20260508 06:23:44.542 | INFO | Perf report for the last 5.13663 seconds: | 20260508 06:23:44.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:44.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:44.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:44.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:44.542 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 06:23:44.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:44.542 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 06:23:44.542 | INFO | PRE_HQ : 66.7%, 1.8, 61.6ms, 0 | 20260508 06:23:44.542 | INFO | BAZWRITER : 66.9%, 2.8, 96.2ms, 0 | 20260508 06:23:44.542 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:23:44.542 | INFO | Uploaded 36GiB to GPU in 2.56009s (14.062GiB/s) | 20260508 06:23:44.543 | INFO | Downloaded 38GiB to GPU in 2.71952s (13.9731GiB/s) | 20260508 06:23:49.681 | INFO | Analyzing chunk frames = [5927424,5927936) | 20260508 06:23:49.681 | INFO | MemInfo: { | 20260508 06:23:49.681 | INFO | "gpu" : " 17 GB", | 20260508 06:23:49.681 | INFO | "huge" : "", | 20260508 06:23:49.681 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:49.681 | INFO | "malloc" : " 14 GB", | 20260508 06:23:49.681 | INFO | "pinned" : " 29 GB" | 20260508 06:23:49.681 | INFO | } | 20260508 06:23:49.681 | INFO | Perf report for the last 5.13924 seconds: | 20260508 06:23:49.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:49.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.681 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:23:49.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.682 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:23:49.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:49.682 | INFO | Uploaded 36GiB to GPU in 2.48565s (14.4831GiB/s) | 20260508 06:23:49.682 | INFO | Downloaded 34GiB to GPU in 2.45028s (13.876GiB/s) | 20260508 06:23:54.816 | INFO | Analyzing chunk frames = [5927936,5928448) | 20260508 06:23:54.816 | INFO | MemInfo: { | 20260508 06:23:54.816 | INFO | "gpu" : " 17 GB", | 20260508 06:23:54.816 | INFO | "huge" : "", | 20260508 06:23:54.816 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:54.816 | INFO | "malloc" : " 14 GB", | 20260508 06:23:54.816 | INFO | "pinned" : " 29 GB" | 20260508 06:23:54.816 | INFO | } | 20260508 06:23:54.816 | INFO | Perf report for the last 5.13484 seconds: | 20260508 06:23:54.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:54.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.816 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 06:23:54.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.816 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:23:54.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:54.817 | INFO | Uploaded 36GiB to GPU in 2.55056s (14.1145GiB/s) | 20260508 06:23:54.817 | INFO | Downloaded 34GiB to GPU in 2.42601s (14.0148GiB/s) | 20260508 06:23:59.956 | INFO | Analyzing chunk frames = [5928448,5928960) | 20260508 06:23:59.956 | INFO | MemInfo: { | 20260508 06:23:59.956 | INFO | "gpu" : " 17 GB", | 20260508 06:23:59.956 | INFO | "huge" : "", | 20260508 06:23:59.956 | INFO | "huge_shared" : " 29 GB", | 20260508 06:23:59.956 | INFO | "malloc" : " 14 GB", | 20260508 06:23:59.956 | INFO | "pinned" : " 29 GB" | 20260508 06:23:59.956 | INFO | } | 20260508 06:23:59.956 | INFO | Perf report for the last 5.13962 seconds: | 20260508 06:23:59.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:23:59.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 06:23:59.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:23:59.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:23:59.956 | INFO | Uploaded 36GiB to GPU in 2.53505s (14.2009GiB/s) | 20260508 06:23:59.956 | INFO | Downloaded 34GiB to GPU in 2.41564s (14.0749GiB/s) | 20260508 06:24:05.098 | INFO | Analyzing chunk frames = [5928960,5929472) | 20260508 06:24:05.098 | INFO | MemInfo: { | 20260508 06:24:05.098 | INFO | "gpu" : " 17 GB", | 20260508 06:24:05.098 | INFO | "huge" : "", | 20260508 06:24:05.098 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:05.098 | INFO | "malloc" : " 14 GB", | 20260508 06:24:05.098 | INFO | "pinned" : " 29 GB" | 20260508 06:24:05.099 | INFO | } | 20260508 06:24:05.099 | INFO | Perf report for the last 5.14263 seconds: | 20260508 06:24:05.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:05.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 06:24:05.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:24:05.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:05.099 | INFO | Uploaded 36GiB to GPU in 2.32258s (15.5GiB/s) | 20260508 06:24:05.099 | INFO | Downloaded 34GiB to GPU in 2.50977s (13.5471GiB/s) | 20260508 06:24:10.237 | INFO | Analyzing chunk frames = [5929472,5929984) | 20260508 06:24:10.238 | INFO | MemInfo: { | 20260508 06:24:10.238 | INFO | "gpu" : " 17 GB", | 20260508 06:24:10.238 | INFO | "huge" : "", | 20260508 06:24:10.238 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:10.238 | INFO | "malloc" : " 14 GB", | 20260508 06:24:10.238 | INFO | "pinned" : " 29 GB" | 20260508 06:24:10.238 | INFO | } | 20260508 06:24:10.238 | INFO | Perf report for the last 5.1392 seconds: | 20260508 06:24:10.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:10.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 06:24:10.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 06:24:10.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:10.238 | INFO | Uploaded 36GiB to GPU in 2.42545s (14.8426GiB/s) | 20260508 06:24:10.238 | INFO | Downloaded 34GiB to GPU in 2.44007s (13.9341GiB/s) | 20260508 06:24:15.378 | INFO | Analyzing chunk frames = [5929984,5930496) | 20260508 06:24:15.378 | INFO | MemInfo: { | 20260508 06:24:15.378 | INFO | "gpu" : " 17 GB", | 20260508 06:24:15.378 | INFO | "huge" : "", | 20260508 06:24:15.378 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:15.378 | INFO | "malloc" : " 14 GB", | 20260508 06:24:15.378 | INFO | "pinned" : " 29 GB" | 20260508 06:24:15.378 | INFO | } | 20260508 06:24:15.378 | INFO | Perf report for the last 5.14037 seconds: | 20260508 06:24:15.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:15.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.378 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:24:15.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.378 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:24:15.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:15.379 | INFO | Uploaded 36GiB to GPU in 2.45609s (14.6574GiB/s) | 20260508 06:24:15.379 | INFO | Downloaded 34GiB to GPU in 2.44073s (13.9303GiB/s) | 20260508 06:24:20.510 | INFO | Analyzing chunk frames = [5930496,5931008) | 20260508 06:24:20.510 | INFO | MemInfo: { | 20260508 06:24:20.510 | INFO | "gpu" : " 17 GB", | 20260508 06:24:20.510 | INFO | "huge" : "", | 20260508 06:24:20.510 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:20.510 | INFO | "malloc" : " 14 GB", | 20260508 06:24:20.510 | INFO | "pinned" : " 29 GB" | 20260508 06:24:20.510 | INFO | } | 20260508 06:24:20.510 | INFO | Perf report for the last 5.13182 seconds: | 20260508 06:24:20.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:20.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.510 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 06:24:20.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.510 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 06:24:20.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:20.511 | INFO | Uploaded 36GiB to GPU in 2.45086s (14.6887GiB/s) | 20260508 06:24:20.511 | INFO | Downloaded 34GiB to GPU in 2.45043s (13.8751GiB/s) | 20260508 06:24:20.511 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:24:20.511 | INFO | Total mem usage: 16.7 GiB | 20260508 06:24:20.511 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:24:20.511 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.8%) | 20260508 06:24:20.511 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:24:20.511 | INFO | High Water Breakdown: | 20260508 06:24:20.511 | INFO | 59.7% : 8.6 GiB ( 83.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:24:20.511 | INFO | 40.2% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:24:20.511 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:24:20.511 | INFO | ------------------------------------------ | 20260508 06:24:20.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:24:20.511 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:24:20.511 | INFO | Total mem usage: 26.5 GiB | 20260508 06:24:20.511 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:24:20.511 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:24:20.511 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:24:20.511 | INFO | High Water Breakdown: | 20260508 06:24:20.511 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:24:20.511 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:24:20.511 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:24:20.511 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:24:20.511 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:24:20.511 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:24:20.511 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:24:20.511 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:24:20.511 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:24:20.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:24:20.511 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:24:20.511 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:24:20.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:24:20.511 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:24:20.511 | INFO | ------------------------------------------ | 20260508 06:24:20.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:24:20.511 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:24:20.511 | INFO | Total mem usage: 27.8 GiB | 20260508 06:24:20.511 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:24:20.511 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:24:20.511 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:24:20.511 | INFO | High Water Breakdown: | 20260508 06:24:20.511 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:24:20.511 | INFO | ------------------------------------------ | 20260508 06:24:20.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:24:20.511 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:24:20.511 | INFO | Total mem usage: 15.7 GiB | 20260508 06:24:20.511 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:24:20.511 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:24:20.511 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:24:20.511 | INFO | High Water Breakdown: | 20260508 06:24:20.511 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:24:20.511 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:24:20.511 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:24:20.511 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:24:20.511 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:24:20.511 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:24:20.511 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:24:20.511 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:24:20.511 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:24:20.511 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:24:20.511 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:24:20.512 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:24:20.512 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:24:20.512 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:24:20.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:24:20.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:24:20.512 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:24:20.512 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:24:20.512 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:24:20.512 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:24:20.512 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:24:20.512 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:24:20.512 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:24:20.512 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:24:20.512 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:24:20.512 | INFO | ------------------------------------------ | 20260508 06:24:20.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:24:20.512 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:24:20.512 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:24:20.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:24:20.512 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.38e+01 / 6.96e-01 | 20260508 06:24:20.512 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.06e-01, 1.19e-02 / 6.68e-03 | 20260508 06:24:20.512 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.39e+00, 5.85e-01 / 2.84e-01 | 20260508 06:24:20.512 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.12e-01 | 20260508 06:24:20.512 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.55e+01, 3.88e+00 / 1.18e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.84e+00 / 1.18e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.62e+00, 4.11e-01 / 2.07e-01 | 20260508 06:24:20.512 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.25e+00, 2.55e-01 / 1.71e-01 | 20260508 06:24:20.512 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.16e-01 / 3.52e-01, 5.02e-02 / 2.23e-02 | 20260508 06:24:20.512 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:24:20.512 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:24:20.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:24:20.512 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 24.37%, Upload -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 24.31%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 30.88%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 2.24%, LockUpload -- 2.38e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 2.56%, LockDownload -- 2.72e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:24:20.512 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.54e-02 / 0.00e+00 | 20260508 06:24:20.623 | INFO | Starting a round of Baz IO | 20260508 06:24:24.040 | INFO | Populating RTMetrics report | 20260508 06:24:24.040 | INFO | RTMetrics report populated | 20260508 06:24:24.040 | INFO | Opening tmp RTMetrics json file | 20260508 06:24:24.052 | INFO | Serializing RTMetrics to json | 20260508 06:24:24.053 | INFO | Serialization done, swapping tmp file now... | 20260508 06:24:24.053 | INFO | RTMetrics done | 20260508 06:24:24.113 | INFO | Finished a round of Baz IO: Wrote 1.551 GiB in 3.489 seconds | 20260508 06:24:24.113 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.71% / 0.00% / 59.29% | 20260508 06:24:24.113 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.399963% / 7.94729e-06% / 7.15256e-05% | 20260508 06:24:24.113 | INFO | Baz IO overall: | 20260508 06:24:24.113 | INFO | BytesWritten : 2630835047342 | 20260508 06:24:24.113 | INFO | NumEvents : 1268477857082 | 20260508 06:24:25.653 | INFO | Analyzing chunk frames = [5931008,5931520) | 20260508 06:24:25.653 | INFO | MemInfo: { | 20260508 06:24:25.653 | INFO | "gpu" : " 17 GB", | 20260508 06:24:25.653 | INFO | "huge" : "", | 20260508 06:24:25.653 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:25.653 | INFO | "malloc" : " 14 GB", | 20260508 06:24:25.653 | INFO | "pinned" : " 29 GB" | 20260508 06:24:25.653 | INFO | } | 20260508 06:24:25.653 | INFO | Perf report for the last 5.14313 seconds: | 20260508 06:24:25.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:25.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:25.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:25.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:25.653 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 06:24:25.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:25.653 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 06:24:25.653 | INFO | PRE_HQ : 66.2%, 1.8, 61.2ms, 0 | 20260508 06:24:25.653 | INFO | BAZWRITER : 67.3%, 2.8, 96.1ms, 0 | 20260508 06:24:25.653 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:24:25.654 | INFO | Uploaded 36GiB to GPU in 2.51268s (14.3274GiB/s) | 20260508 06:24:25.654 | INFO | Downloaded 38GiB to GPU in 2.70096s (14.069GiB/s) | 20260508 06:24:30.796 | INFO | Analyzing chunk frames = [5931520,5932032) | 20260508 06:24:30.796 | INFO | MemInfo: { | 20260508 06:24:30.796 | INFO | "gpu" : " 17 GB", | 20260508 06:24:30.796 | INFO | "huge" : "", | 20260508 06:24:30.796 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:30.796 | INFO | "malloc" : " 14 GB", | 20260508 06:24:30.796 | INFO | "pinned" : " 29 GB" | 20260508 06:24:30.796 | INFO | } | 20260508 06:24:30.796 | INFO | Perf report for the last 5.1429 seconds: | 20260508 06:24:30.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:30.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:24:30.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:24:30.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:30.796 | INFO | Uploaded 36GiB to GPU in 2.50506s (14.3709GiB/s) | 20260508 06:24:30.796 | INFO | Downloaded 34GiB to GPU in 2.43005s (13.9915GiB/s) | 20260508 06:24:35.938 | INFO | Analyzing chunk frames = [5932032,5932544) | 20260508 06:24:35.938 | INFO | MemInfo: { | 20260508 06:24:35.938 | INFO | "gpu" : " 17 GB", | 20260508 06:24:35.938 | INFO | "huge" : "", | 20260508 06:24:35.938 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:35.938 | INFO | "malloc" : " 14 GB", | 20260508 06:24:35.938 | INFO | "pinned" : " 29 GB" | 20260508 06:24:35.938 | INFO | } | 20260508 06:24:35.939 | INFO | Perf report for the last 5.14245 seconds: | 20260508 06:24:35.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:35.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:24:35.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:24:35.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:35.939 | INFO | Uploaded 36GiB to GPU in 2.56875s (14.0146GiB/s) | 20260508 06:24:35.939 | INFO | Downloaded 34GiB to GPU in 2.41321s (14.0891GiB/s) | 20260508 06:24:41.072 | INFO | Analyzing chunk frames = [5932544,5933056) | 20260508 06:24:41.072 | INFO | MemInfo: { | 20260508 06:24:41.072 | INFO | "gpu" : " 17 GB", | 20260508 06:24:41.072 | INFO | "huge" : "", | 20260508 06:24:41.072 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:41.072 | INFO | "malloc" : " 14 GB", | 20260508 06:24:41.072 | INFO | "pinned" : " 29 GB" | 20260508 06:24:41.072 | INFO | } | 20260508 06:24:41.072 | INFO | Perf report for the last 5.1336 seconds: | 20260508 06:24:41.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:41.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 06:24:41.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:24:41.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:41.072 | INFO | Uploaded 36GiB to GPU in 2.57819s (13.9633GiB/s) | 20260508 06:24:41.073 | INFO | Downloaded 34GiB to GPU in 2.39536s (14.1941GiB/s) | 20260508 06:24:46.210 | INFO | Analyzing chunk frames = [5933056,5933568) | 20260508 06:24:46.210 | INFO | MemInfo: { | 20260508 06:24:46.211 | INFO | "gpu" : " 17 GB", | 20260508 06:24:46.211 | INFO | "huge" : "", | 20260508 06:24:46.211 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:46.211 | INFO | "malloc" : " 14 GB", | 20260508 06:24:46.211 | INFO | "pinned" : " 29 GB" | 20260508 06:24:46.211 | INFO | } | 20260508 06:24:46.211 | INFO | Perf report for the last 5.13895 seconds: | 20260508 06:24:46.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:46.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 06:24:46.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:24:46.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:46.211 | INFO | Uploaded 36GiB to GPU in 2.41352s (14.9159GiB/s) | 20260508 06:24:46.211 | INFO | Downloaded 34GiB to GPU in 2.49189s (13.6443GiB/s) | 20260508 06:24:51.352 | INFO | Analyzing chunk frames = [5933568,5934080) | 20260508 06:24:51.352 | INFO | MemInfo: { | 20260508 06:24:51.352 | INFO | "gpu" : " 17 GB", | 20260508 06:24:51.352 | INFO | "huge" : "", | 20260508 06:24:51.352 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:51.352 | INFO | "malloc" : " 14 GB", | 20260508 06:24:51.352 | INFO | "pinned" : " 29 GB" | 20260508 06:24:51.352 | INFO | } | 20260508 06:24:51.352 | INFO | Perf report for the last 5.14112 seconds: | 20260508 06:24:51.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:51.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.352 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 06:24:51.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.352 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:24:51.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:51.353 | INFO | Uploaded 36GiB to GPU in 2.48066s (14.5123GiB/s) | 20260508 06:24:51.353 | INFO | Downloaded 34GiB to GPU in 2.42288s (14.0329GiB/s) | 20260508 06:24:56.489 | INFO | Analyzing chunk frames = [5934080,5934592) | 20260508 06:24:56.489 | INFO | MemInfo: { | 20260508 06:24:56.489 | INFO | "gpu" : " 17 GB", | 20260508 06:24:56.489 | INFO | "huge" : "", | 20260508 06:24:56.489 | INFO | "huge_shared" : " 29 GB", | 20260508 06:24:56.489 | INFO | "malloc" : " 14 GB", | 20260508 06:24:56.489 | INFO | "pinned" : " 29 GB" | 20260508 06:24:56.489 | INFO | } | 20260508 06:24:56.489 | INFO | Perf report for the last 5.1368 seconds: | 20260508 06:24:56.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:24:56.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.489 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:24:56.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.489 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:24:56.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:24:56.490 | INFO | Uploaded 36GiB to GPU in 2.48876s (14.465GiB/s) | 20260508 06:24:56.490 | INFO | Downloaded 34GiB to GPU in 2.43088s (13.9867GiB/s) | 20260508 06:25:01.631 | INFO | Analyzing chunk frames = [5934592,5935104) | 20260508 06:25:01.631 | INFO | MemInfo: { | 20260508 06:25:01.631 | INFO | "gpu" : " 17 GB", | 20260508 06:25:01.631 | INFO | "huge" : "", | 20260508 06:25:01.631 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:01.631 | INFO | "malloc" : " 14 GB", | 20260508 06:25:01.631 | INFO | "pinned" : " 29 GB" | 20260508 06:25:01.631 | INFO | } | 20260508 06:25:01.632 | INFO | Perf report for the last 5.14237 seconds: | 20260508 06:25:01.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:01.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:25:01.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 06:25:01.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:01.632 | INFO | Uploaded 36GiB to GPU in 2.4856s (14.4834GiB/s) | 20260508 06:25:01.632 | INFO | Downloaded 34GiB to GPU in 2.43224s (13.9789GiB/s) | 20260508 06:25:01.745 | INFO | Starting a round of Baz IO | 20260508 06:25:05.152 | INFO | Populating RTMetrics report | 20260508 06:25:05.152 | INFO | RTMetrics report populated | 20260508 06:25:05.152 | INFO | Opening tmp RTMetrics json file | 20260508 06:25:05.161 | INFO | Serializing RTMetrics to json | 20260508 06:25:05.165 | INFO | Serialization done, swapping tmp file now... | 20260508 06:25:05.166 | INFO | RTMetrics done | 20260508 06:25:05.227 | INFO | Finished a round of Baz IO: Wrote 1.552 GiB in 3.481 seconds | 20260508 06:25:05.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.71% / 0.00% / 59.29% | 20260508 06:25:05.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.399876% / 3.97364e-06% / 7.15256e-05% | 20260508 06:25:05.227 | INFO | Baz IO overall: | 20260508 06:25:05.227 | INFO | BytesWritten : 2632501549697 | 20260508 06:25:05.227 | INFO | NumEvents : 1269271425715 | 20260508 06:25:06.769 | INFO | Analyzing chunk frames = [5935104,5935616) | 20260508 06:25:06.769 | INFO | MemInfo: { | 20260508 06:25:06.769 | INFO | "gpu" : " 17 GB", | 20260508 06:25:06.769 | INFO | "huge" : "", | 20260508 06:25:06.769 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:06.769 | INFO | "malloc" : " 14 GB", | 20260508 06:25:06.769 | INFO | "pinned" : " 29 GB" | 20260508 06:25:06.769 | INFO | } | 20260508 06:25:06.769 | INFO | Perf report for the last 5.13797 seconds: | 20260508 06:25:06.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:06.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:06.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:06.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:06.770 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 06:25:06.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:06.770 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 06:25:06.770 | INFO | PRE_HQ : 66.4%, 1.8, 61.4ms, 0 | 20260508 06:25:06.770 | INFO | BAZWRITER : 67.1%, 2.8, 96.8ms, 0 | 20260508 06:25:06.770 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:25:06.770 | INFO | Uploaded 36GiB to GPU in 2.59928s (13.85GiB/s) | 20260508 06:25:06.770 | INFO | Downloaded 38GiB to GPU in 2.70792s (14.0329GiB/s) | 20260508 06:25:11.908 | INFO | Analyzing chunk frames = [5935616,5936128) | 20260508 06:25:11.908 | INFO | MemInfo: { | 20260508 06:25:11.908 | INFO | "gpu" : " 17 GB", | 20260508 06:25:11.908 | INFO | "huge" : "", | 20260508 06:25:11.908 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:11.908 | INFO | "malloc" : " 14 GB", | 20260508 06:25:11.908 | INFO | "pinned" : " 29 GB" | 20260508 06:25:11.908 | INFO | } | 20260508 06:25:11.908 | INFO | Perf report for the last 5.13889 seconds: | 20260508 06:25:11.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:11.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:25:11.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:25:11.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:11.909 | INFO | Uploaded 36GiB to GPU in 2.48416s (14.4918GiB/s) | 20260508 06:25:11.909 | INFO | Downloaded 34GiB to GPU in 2.44126s (13.9272GiB/s) | 20260508 06:25:17.053 | INFO | Analyzing chunk frames = [5936128,5936640) | 20260508 06:25:17.053 | INFO | MemInfo: { | 20260508 06:25:17.053 | INFO | "gpu" : " 17 GB", | 20260508 06:25:17.053 | INFO | "huge" : "", | 20260508 06:25:17.053 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:17.053 | INFO | "malloc" : " 14 GB", | 20260508 06:25:17.053 | INFO | "pinned" : " 29 GB" | 20260508 06:25:17.053 | INFO | } | 20260508 06:25:17.053 | INFO | Perf report for the last 5.14492 seconds: | 20260508 06:25:17.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:17.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.053 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:25:17.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.053 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:25:17.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:17.054 | INFO | Uploaded 36GiB to GPU in 2.52223s (14.2731GiB/s) | 20260508 06:25:17.054 | INFO | Downloaded 34GiB to GPU in 2.4085s (14.1167GiB/s) | 20260508 06:25:22.189 | INFO | Analyzing chunk frames = [5936640,5937152) | 20260508 06:25:22.189 | INFO | MemInfo: { | 20260508 06:25:22.189 | INFO | "gpu" : " 17 GB", | 20260508 06:25:22.189 | INFO | "huge" : "", | 20260508 06:25:22.189 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:22.189 | INFO | "malloc" : " 14 GB", | 20260508 06:25:22.189 | INFO | "pinned" : " 29 GB" | 20260508 06:25:22.189 | INFO | } | 20260508 06:25:22.189 | INFO | Perf report for the last 5.13548 seconds: | 20260508 06:25:22.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:22.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 06:25:22.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:25:22.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:22.189 | INFO | Uploaded 36GiB to GPU in 2.55599s (14.0846GiB/s) | 20260508 06:25:22.189 | INFO | Downloaded 34GiB to GPU in 2.42689s (14.0097GiB/s) | 20260508 06:25:27.322 | INFO | Analyzing chunk frames = [5937152,5937664) | 20260508 06:25:27.322 | INFO | MemInfo: { | 20260508 06:25:27.322 | INFO | "gpu" : " 17 GB", | 20260508 06:25:27.322 | INFO | "huge" : "", | 20260508 06:25:27.323 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:27.323 | INFO | "malloc" : " 14 GB", | 20260508 06:25:27.323 | INFO | "pinned" : " 29 GB" | 20260508 06:25:27.323 | INFO | } | 20260508 06:25:27.323 | INFO | Perf report for the last 5.1338 seconds: | 20260508 06:25:27.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:27.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 06:25:27.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:25:27.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:27.323 | INFO | Uploaded 36GiB to GPU in 2.29958s (15.655GiB/s) | 20260508 06:25:27.323 | INFO | Downloaded 34GiB to GPU in 2.48536s (13.6801GiB/s) | 20260508 06:25:32.464 | INFO | Analyzing chunk frames = [5937664,5938176) | 20260508 06:25:32.464 | INFO | MemInfo: { | 20260508 06:25:32.464 | INFO | "gpu" : " 17 GB", | 20260508 06:25:32.464 | INFO | "huge" : "", | 20260508 06:25:32.464 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:32.464 | INFO | "malloc" : " 14 GB", | 20260508 06:25:32.464 | INFO | "pinned" : " 29 GB" | 20260508 06:25:32.464 | INFO | } | 20260508 06:25:32.464 | INFO | Perf report for the last 5.1414 seconds: | 20260508 06:25:32.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:32.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.464 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 06:25:32.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.464 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:25:32.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:32.465 | INFO | Uploaded 36GiB to GPU in 2.40542s (14.9662GiB/s) | 20260508 06:25:32.465 | INFO | Downloaded 34GiB to GPU in 2.45319s (13.8595GiB/s) | 20260508 06:25:37.607 | INFO | Analyzing chunk frames = [5938176,5938688) | 20260508 06:25:37.607 | INFO | MemInfo: { | 20260508 06:25:37.607 | INFO | "gpu" : " 17 GB", | 20260508 06:25:37.607 | INFO | "huge" : "", | 20260508 06:25:37.607 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:37.607 | INFO | "malloc" : " 14 GB", | 20260508 06:25:37.608 | INFO | "pinned" : " 29 GB" | 20260508 06:25:37.608 | INFO | } | 20260508 06:25:37.608 | INFO | Perf report for the last 5.14436 seconds: | 20260508 06:25:37.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:37.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.608 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 06:25:37.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.608 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:25:37.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:37.609 | INFO | Uploaded 36GiB to GPU in 2.4508s (14.6891GiB/s) | 20260508 06:25:37.609 | INFO | Downloaded 34GiB to GPU in 2.44757s (13.8913GiB/s) | 20260508 06:25:42.744 | INFO | Analyzing chunk frames = [5938688,5939200) | 20260508 06:25:42.744 | INFO | MemInfo: { | 20260508 06:25:42.744 | INFO | "gpu" : " 17 GB", | 20260508 06:25:42.744 | INFO | "huge" : "", | 20260508 06:25:42.744 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:42.744 | INFO | "malloc" : " 14 GB", | 20260508 06:25:42.745 | INFO | "pinned" : " 29 GB" | 20260508 06:25:42.745 | INFO | } | 20260508 06:25:42.745 | INFO | Perf report for the last 5.13641 seconds: | 20260508 06:25:42.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:42.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.745 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:25:42.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.745 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 06:25:42.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:42.746 | INFO | Uploaded 36GiB to GPU in 2.44917s (14.6989GiB/s) | 20260508 06:25:42.746 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:25:42.746 | INFO | Total mem usage: 16.7 GiB | 20260508 06:25:42.746 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:25:42.746 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:25:42.746 | INFO | Downloaded 34GiB to GPU in 2.45281s (13.8616GiB/s) | 20260508 06:25:42.746 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:25:42.746 | INFO | High Water Breakdown: | 20260508 06:25:42.746 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:25:42.746 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:25:42.746 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:25:42.746 | INFO | ------------------------------------------ | 20260508 06:25:42.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:25:42.746 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:25:42.746 | INFO | Total mem usage: 26.5 GiB | 20260508 06:25:42.746 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:25:42.746 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:25:42.746 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:25:42.746 | INFO | High Water Breakdown: | 20260508 06:25:42.746 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:25:42.746 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:25:42.746 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:25:42.746 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:25:42.746 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:25:42.746 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:25:42.746 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:25:42.746 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:25:42.746 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:25:42.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:25:42.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:25:42.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:25:42.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:25:42.746 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:25:42.746 | INFO | ------------------------------------------ | 20260508 06:25:42.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:25:42.746 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:25:42.746 | INFO | Total mem usage: 27.8 GiB | 20260508 06:25:42.746 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:25:42.746 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:25:42.746 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:25:42.746 | INFO | High Water Breakdown: | 20260508 06:25:42.746 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:25:42.746 | INFO | ------------------------------------------ | 20260508 06:25:42.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:25:42.746 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:25:42.746 | INFO | Total mem usage: 15.7 GiB | 20260508 06:25:42.746 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:25:42.746 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:25:42.746 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:25:42.746 | INFO | High Water Breakdown: | 20260508 06:25:42.746 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:25:42.746 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:25:42.746 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:25:42.746 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:25:42.746 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:25:42.747 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:25:42.747 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:25:42.747 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:25:42.747 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:25:42.747 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:25:42.747 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:25:42.747 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:25:42.747 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:25:42.747 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:25:42.747 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:25:42.747 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:25:42.747 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:25:42.747 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:25:42.747 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:25:42.747 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:25:42.747 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:25:42.747 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:25:42.747 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:25:42.747 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:25:42.747 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:25:42.747 | INFO | ------------------------------------------ | 20260508 06:25:42.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:25:42.747 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:25:42.747 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:25:42.747 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:25:42.747 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.18e-01 | 20260508 06:25:42.747 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.07e-01, 1.17e-02 / 5.94e-03 | 20260508 06:25:42.747 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.41e+00, 5.82e-01 / 2.09e-01 | 20260508 06:25:42.747 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.28e-01 | 20260508 06:25:42.747 | INFO | PerfReport: 15.91%, DME -- 4.16e+01 / 4.50e+01, 3.66e+00 / 1.17e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.82e+00 / 1.13e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.57e+00, 4.11e-01 / 1.79e-01 | 20260508 06:25:42.747 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.24e+00, 2.55e-01 / 1.11e-01 | 20260508 06:25:42.747 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.52e-01, 4.97e-02 / 1.93e-02 | 20260508 06:25:42.747 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:25:42.747 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:25:42.747 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:25:42.747 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 24.42%, Upload -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 2.31%, LockUpload -- 2.45e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 2.55%, LockDownload -- 2.70e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 15.59%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 06:25:42.747 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260508 06:25:42.859 | INFO | Starting a round of Baz IO | 20260508 06:25:46.266 | INFO | Populating RTMetrics report | 20260508 06:25:46.266 | INFO | RTMetrics report populated | 20260508 06:25:46.266 | INFO | Opening tmp RTMetrics json file | 20260508 06:25:46.278 | INFO | Serializing RTMetrics to json | 20260508 06:25:46.279 | INFO | Serialization done, swapping tmp file now... | 20260508 06:25:46.280 | INFO | RTMetrics done | 20260508 06:25:46.340 | INFO | Finished a round of Baz IO: Wrote 1.551 GiB in 3.481 seconds | 20260508 06:25:46.340 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.00% / 59.29% | 20260508 06:25:46.340 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39897% / 3.97364e-06% / 7.15256e-05% | 20260508 06:25:46.340 | INFO | Baz IO overall: | 20260508 06:25:46.340 | INFO | BytesWritten : 2634166777120 | 20260508 06:25:46.340 | INFO | NumEvents : 1270064202204 | 20260508 06:25:47.884 | INFO | Analyzing chunk frames = [5939200,5939712) | 20260508 06:25:47.884 | INFO | MemInfo: { | 20260508 06:25:47.884 | INFO | "gpu" : " 17 GB", | 20260508 06:25:47.884 | INFO | "huge" : "", | 20260508 06:25:47.884 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:47.884 | INFO | "malloc" : " 14 GB", | 20260508 06:25:47.884 | INFO | "pinned" : " 29 GB" | 20260508 06:25:47.884 | INFO | } | 20260508 06:25:47.884 | INFO | Perf report for the last 5.13953 seconds: | 20260508 06:25:47.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:47.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:47.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:47.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:47.885 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 06:25:47.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:47.885 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 06:25:47.885 | INFO | PRE_HQ : 66.4%, 1.8, 61.4ms, 0 | 20260508 06:25:47.885 | INFO | BAZWRITER : 67.2%, 2.8, 97.8ms, 0 | 20260508 06:25:47.885 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:25:47.885 | INFO | Uploaded 36GiB to GPU in 2.58642s (13.9188GiB/s) | 20260508 06:25:47.885 | INFO | Downloaded 38GiB to GPU in 2.70311s (14.0579GiB/s) | 20260508 06:25:53.018 | INFO | Analyzing chunk frames = [5939712,5940224) | 20260508 06:25:53.018 | INFO | MemInfo: { | 20260508 06:25:53.018 | INFO | "gpu" : " 17 GB", | 20260508 06:25:53.018 | INFO | "huge" : "", | 20260508 06:25:53.018 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:53.018 | INFO | "malloc" : " 14 GB", | 20260508 06:25:53.018 | INFO | "pinned" : " 29 GB" | 20260508 06:25:53.018 | INFO | } | 20260508 06:25:53.018 | INFO | Perf report for the last 5.13396 seconds: | 20260508 06:25:53.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:53.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.018 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 06:25:53.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.018 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:25:53.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:53.019 | INFO | Uploaded 36GiB to GPU in 2.49778s (14.4128GiB/s) | 20260508 06:25:53.019 | INFO | Downloaded 34GiB to GPU in 2.43727s (13.95GiB/s) | 20260508 06:25:58.160 | INFO | Analyzing chunk frames = [5940224,5940736) | 20260508 06:25:58.160 | INFO | MemInfo: { | 20260508 06:25:58.160 | INFO | "gpu" : " 17 GB", | 20260508 06:25:58.160 | INFO | "huge" : "", | 20260508 06:25:58.160 | INFO | "huge_shared" : " 29 GB", | 20260508 06:25:58.160 | INFO | "malloc" : " 14 GB", | 20260508 06:25:58.160 | INFO | "pinned" : " 29 GB" | 20260508 06:25:58.160 | INFO | } | 20260508 06:25:58.161 | INFO | Perf report for the last 5.14218 seconds: | 20260508 06:25:58.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:25:58.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 06:25:58.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:25:58.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:25:58.161 | INFO | Uploaded 36GiB to GPU in 2.56162s (14.0536GiB/s) | 20260508 06:25:58.161 | INFO | Downloaded 34GiB to GPU in 2.4233s (14.0305GiB/s) | 20260508 06:26:03.302 | INFO | Analyzing chunk frames = [5940736,5941248) | 20260508 06:26:03.302 | INFO | MemInfo: { | 20260508 06:26:03.302 | INFO | "gpu" : " 17 GB", | 20260508 06:26:03.302 | INFO | "huge" : "", | 20260508 06:26:03.302 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:03.302 | INFO | "malloc" : " 14 GB", | 20260508 06:26:03.302 | INFO | "pinned" : " 29 GB" | 20260508 06:26:03.302 | INFO | } | 20260508 06:26:03.302 | INFO | Perf report for the last 5.14158 seconds: | 20260508 06:26:03.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:03.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.302 | INFO | ANALYSIS : 66.9%, 3.0, 102.8ms, 0 | 20260508 06:26:03.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.302 | INFO | PRE_HQ : 23.7%, 1.0, 12.2ms, 0 | 20260508 06:26:03.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:03.303 | INFO | Uploaded 36GiB to GPU in 2.56604s (14.0294GiB/s) | 20260508 06:26:03.303 | INFO | Downloaded 34GiB to GPU in 2.42343s (14.0297GiB/s) | 20260508 06:26:08.444 | INFO | Analyzing chunk frames = [5941248,5941760) | 20260508 06:26:08.444 | INFO | MemInfo: { | 20260508 06:26:08.444 | INFO | "gpu" : " 17 GB", | 20260508 06:26:08.444 | INFO | "huge" : "", | 20260508 06:26:08.444 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:08.444 | INFO | "malloc" : " 14 GB", | 20260508 06:26:08.445 | INFO | "pinned" : " 29 GB" | 20260508 06:26:08.445 | INFO | } | 20260508 06:26:08.445 | INFO | Perf report for the last 5.14247 seconds: | 20260508 06:26:08.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:08.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:26:08.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:26:08.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:08.445 | INFO | Uploaded 36GiB to GPU in 2.38995s (15.0631GiB/s) | 20260508 06:26:08.445 | INFO | Downloaded 34GiB to GPU in 2.49123s (13.6479GiB/s) | 20260508 06:26:13.577 | INFO | Analyzing chunk frames = [5941760,5942272) | 20260508 06:26:13.577 | INFO | MemInfo: { | 20260508 06:26:13.577 | INFO | "gpu" : " 17 GB", | 20260508 06:26:13.577 | INFO | "huge" : "", | 20260508 06:26:13.577 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:13.577 | INFO | "malloc" : " 14 GB", | 20260508 06:26:13.577 | INFO | "pinned" : " 29 GB" | 20260508 06:26:13.577 | INFO | } | 20260508 06:26:13.577 | INFO | Perf report for the last 5.13255 seconds: | 20260508 06:26:13.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:13.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.577 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:26:13.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.577 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:26:13.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:13.578 | INFO | Uploaded 36GiB to GPU in 2.48092s (14.5107GiB/s) | 20260508 06:26:13.578 | INFO | Downloaded 34GiB to GPU in 2.43244s (13.9777GiB/s) | 20260508 06:26:18.721 | INFO | Analyzing chunk frames = [5942272,5942784) | 20260508 06:26:18.721 | INFO | MemInfo: { | 20260508 06:26:18.721 | INFO | "gpu" : " 17 GB", | 20260508 06:26:18.721 | INFO | "huge" : "", | 20260508 06:26:18.721 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:18.721 | INFO | "malloc" : " 14 GB", | 20260508 06:26:18.721 | INFO | "pinned" : " 29 GB" | 20260508 06:26:18.721 | INFO | } | 20260508 06:26:18.721 | INFO | Perf report for the last 5.14379 seconds: | 20260508 06:26:18.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:18.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:26:18.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:26:18.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:18.721 | INFO | Uploaded 36GiB to GPU in 2.48614s (14.4803GiB/s) | 20260508 06:26:18.722 | INFO | Downloaded 34GiB to GPU in 2.42465s (14.0227GiB/s) | 20260508 06:26:23.861 | INFO | Analyzing chunk frames = [5942784,5943296) | 20260508 06:26:23.861 | INFO | MemInfo: { | 20260508 06:26:23.861 | INFO | "gpu" : " 17 GB", | 20260508 06:26:23.861 | INFO | "huge" : "", | 20260508 06:26:23.861 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:23.861 | INFO | "malloc" : " 14 GB", | 20260508 06:26:23.861 | INFO | "pinned" : " 29 GB" | 20260508 06:26:23.861 | INFO | } | 20260508 06:26:23.862 | INFO | Perf report for the last 5.14052 seconds: | 20260508 06:26:23.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:23.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:26:23.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 06:26:23.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:23.862 | INFO | Uploaded 36GiB to GPU in 2.47793s (14.5283GiB/s) | 20260508 06:26:23.862 | INFO | Downloaded 34GiB to GPU in 2.42733s (14.0071GiB/s) | 20260508 06:26:23.959 | INFO | Starting a round of Baz IO | 20260508 06:26:27.383 | INFO | Populating RTMetrics report | 20260508 06:26:27.383 | INFO | RTMetrics report populated | 20260508 06:26:27.383 | INFO | Opening tmp RTMetrics json file | 20260508 06:26:27.392 | INFO | Serializing RTMetrics to json | 20260508 06:26:27.395 | INFO | Serialization done, swapping tmp file now... | 20260508 06:26:27.395 | INFO | RTMetrics done | 20260508 06:26:27.457 | INFO | Finished a round of Baz IO: Wrote 1.551 GiB in 3.498 seconds | 20260508 06:26:27.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.00% / 59.30% | 20260508 06:26:27.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.399319% / 3.97364e-06% / 4.76837e-05% | 20260508 06:26:27.457 | INFO | Baz IO overall: | 20260508 06:26:27.457 | INFO | BytesWritten : 2635832545076 | 20260508 06:26:27.457 | INFO | NumEvents : 1270857360758 | 20260508 06:26:29.002 | INFO | Analyzing chunk frames = [5943296,5943808) | 20260508 06:26:29.002 | INFO | MemInfo: { | 20260508 06:26:29.002 | INFO | "gpu" : " 17 GB", | 20260508 06:26:29.002 | INFO | "huge" : "", | 20260508 06:26:29.002 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:29.002 | INFO | "malloc" : " 14 GB", | 20260508 06:26:29.002 | INFO | "pinned" : " 29 GB" | 20260508 06:26:29.002 | INFO | } | 20260508 06:26:29.002 | INFO | Perf report for the last 5.14059 seconds: | 20260508 06:26:29.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:29.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:29.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:29.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:29.002 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 06:26:29.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:29.002 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 06:26:29.002 | INFO | PRE_HQ : 65.4%, 1.6, 55.0ms, 0 | 20260508 06:26:29.002 | INFO | BAZWRITER : 67.5%, 2.6, 91.1ms, 0 | 20260508 06:26:29.002 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:26:29.002 | INFO | Uploaded 36GiB to GPU in 2.58273s (13.9387GiB/s) | 20260508 06:26:29.002 | INFO | Downloaded 38GiB to GPU in 2.68864s (14.1335GiB/s) | 20260508 06:26:34.139 | INFO | Analyzing chunk frames = [5943808,5944320) | 20260508 06:26:34.139 | INFO | MemInfo: { | 20260508 06:26:34.139 | INFO | "gpu" : " 17 GB", | 20260508 06:26:34.139 | INFO | "huge" : "", | 20260508 06:26:34.139 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:34.139 | INFO | "malloc" : " 14 GB", | 20260508 06:26:34.139 | INFO | "pinned" : " 29 GB" | 20260508 06:26:34.139 | INFO | } | 20260508 06:26:34.139 | INFO | Perf report for the last 5.13678 seconds: | 20260508 06:26:34.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:34.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:26:34.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:26:34.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:34.139 | INFO | Uploaded 36GiB to GPU in 2.50919s (14.3473GiB/s) | 20260508 06:26:34.139 | INFO | Downloaded 34GiB to GPU in 2.43906s (13.9398GiB/s) | 20260508 06:26:39.275 | INFO | Analyzing chunk frames = [5944320,5944832) | 20260508 06:26:39.275 | INFO | MemInfo: { | 20260508 06:26:39.275 | INFO | "gpu" : " 17 GB", | 20260508 06:26:39.275 | INFO | "huge" : "", | 20260508 06:26:39.275 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:39.275 | INFO | "malloc" : " 14 GB", | 20260508 06:26:39.275 | INFO | "pinned" : " 29 GB" | 20260508 06:26:39.275 | INFO | } | 20260508 06:26:39.275 | INFO | Perf report for the last 5.1359 seconds: | 20260508 06:26:39.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:39.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 06:26:39.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:26:39.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:39.275 | INFO | Uploaded 36GiB to GPU in 2.58593s (13.9215GiB/s) | 20260508 06:26:39.275 | INFO | Downloaded 34GiB to GPU in 2.43137s (13.9839GiB/s) | 20260508 06:26:44.418 | INFO | Analyzing chunk frames = [5944832,5945344) | 20260508 06:26:44.418 | INFO | MemInfo: { | 20260508 06:26:44.418 | INFO | "gpu" : " 17 GB", | 20260508 06:26:44.418 | INFO | "huge" : "", | 20260508 06:26:44.418 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:44.418 | INFO | "malloc" : " 14 GB", | 20260508 06:26:44.418 | INFO | "pinned" : " 29 GB" | 20260508 06:26:44.418 | INFO | } | 20260508 06:26:44.418 | INFO | Perf report for the last 5.14323 seconds: | 20260508 06:26:44.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:44.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.418 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 06:26:44.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.418 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:26:44.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:44.419 | INFO | Uploaded 36GiB to GPU in 2.56175s (14.0529GiB/s) | 20260508 06:26:44.419 | INFO | Downloaded 34GiB to GPU in 2.39216s (14.2131GiB/s) | 20260508 06:26:49.555 | INFO | Analyzing chunk frames = [5945344,5945856) | 20260508 06:26:49.555 | INFO | MemInfo: { | 20260508 06:26:49.555 | INFO | "gpu" : " 17 GB", | 20260508 06:26:49.555 | INFO | "huge" : "", | 20260508 06:26:49.555 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:49.555 | INFO | "malloc" : " 14 GB", | 20260508 06:26:49.555 | INFO | "pinned" : " 29 GB" | 20260508 06:26:49.555 | INFO | } | 20260508 06:26:49.555 | INFO | Perf report for the last 5.13723 seconds: | 20260508 06:26:49.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:49.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:26:49.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:26:49.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:49.555 | INFO | Uploaded 36GiB to GPU in 2.476s (14.5396GiB/s) | 20260508 06:26:49.556 | INFO | Downloaded 34GiB to GPU in 2.52566s (13.4618GiB/s) | 20260508 06:26:54.697 | INFO | Analyzing chunk frames = [5945856,5946368) | 20260508 06:26:54.697 | INFO | MemInfo: { | 20260508 06:26:54.697 | INFO | "gpu" : " 17 GB", | 20260508 06:26:54.698 | INFO | "huge" : "", | 20260508 06:26:54.698 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:54.698 | INFO | "malloc" : " 14 GB", | 20260508 06:26:54.698 | INFO | "pinned" : " 29 GB" | 20260508 06:26:54.698 | INFO | } | 20260508 06:26:54.698 | INFO | Perf report for the last 5.14236 seconds: | 20260508 06:26:54.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:54.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 06:26:54.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 06:26:54.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:54.698 | INFO | Uploaded 36GiB to GPU in 2.43256s (14.7992GiB/s) | 20260508 06:26:54.698 | INFO | Downloaded 34GiB to GPU in 2.42215s (14.0371GiB/s) | 20260508 06:26:59.836 | INFO | Analyzing chunk frames = [5946368,5946880) | 20260508 06:26:59.836 | INFO | MemInfo: { | 20260508 06:26:59.836 | INFO | "gpu" : " 17 GB", | 20260508 06:26:59.836 | INFO | "huge" : "", | 20260508 06:26:59.837 | INFO | "huge_shared" : " 29 GB", | 20260508 06:26:59.837 | INFO | "malloc" : " 14 GB", | 20260508 06:26:59.837 | INFO | "pinned" : " 29 GB" | 20260508 06:26:59.837 | INFO | } | 20260508 06:26:59.837 | INFO | Perf report for the last 5.13912 seconds: | 20260508 06:26:59.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:26:59.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:26:59.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 06:26:59.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:26:59.837 | INFO | Uploaded 36GiB to GPU in 2.48179s (14.5057GiB/s) | 20260508 06:26:59.837 | INFO | Downloaded 34GiB to GPU in 2.4398s (13.9356GiB/s) | 20260508 06:27:04.972 | INFO | Analyzing chunk frames = [5946880,5947392) | 20260508 06:27:04.972 | INFO | MemInfo: { | 20260508 06:27:04.972 | INFO | "gpu" : " 17 GB", | 20260508 06:27:04.972 | INFO | "huge" : "", | 20260508 06:27:04.972 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:04.972 | INFO | "malloc" : " 14 GB", | 20260508 06:27:04.972 | INFO | "pinned" : " 29 GB" | 20260508 06:27:04.972 | INFO | } | 20260508 06:27:04.972 | INFO | Perf report for the last 5.13564 seconds: | 20260508 06:27:04.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:04.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.973 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 06:27:04.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.973 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260508 06:27:04.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:04.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:27:04.973 | INFO | Total mem usage: 16.7 GiB | 20260508 06:27:04.973 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:27:04.973 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:27:04.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:27:04.973 | INFO | High Water Breakdown: | 20260508 06:27:04.973 | INFO | Uploaded 36GiB to GPU in 2.49544s (14.4263GiB/s) | 20260508 06:27:04.973 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:27:04.973 | INFO | Downloaded 34GiB to GPU in 2.45788s (13.833GiB/s) | 20260508 06:27:04.973 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:27:04.973 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:27:04.973 | INFO | ------------------------------------------ | 20260508 06:27:04.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:27:04.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:27:04.973 | INFO | Total mem usage: 26.5 GiB | 20260508 06:27:04.973 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:27:04.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:27:04.973 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:27:04.973 | INFO | High Water Breakdown: | 20260508 06:27:04.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:27:04.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:27:04.974 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:27:04.974 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:27:04.974 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:27:04.974 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:27:04.974 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:27:04.974 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:27:04.974 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:27:04.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:27:04.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:27:04.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:27:04.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:27:04.974 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:27:04.974 | INFO | ------------------------------------------ | 20260508 06:27:04.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:27:04.974 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:27:04.974 | INFO | Total mem usage: 27.8 GiB | 20260508 06:27:04.974 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:27:04.974 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:27:04.974 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:27:04.974 | INFO | High Water Breakdown: | 20260508 06:27:04.974 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:27:04.974 | INFO | ------------------------------------------ | 20260508 06:27:04.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:27:04.974 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:27:04.974 | INFO | Total mem usage: 15.7 GiB | 20260508 06:27:04.974 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:27:04.974 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:27:04.974 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:27:04.974 | INFO | High Water Breakdown: | 20260508 06:27:04.974 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:27:04.974 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:27:04.974 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:27:04.974 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:27:04.974 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:27:04.974 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:27:04.974 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:27:04.974 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:27:04.974 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:27:04.974 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:27:04.974 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:27:04.975 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:27:04.975 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:27:04.975 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:27:04.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:27:04.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:27:04.975 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:27:04.975 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:27:04.975 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:27:04.975 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:27:04.975 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:27:04.975 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:27:04.975 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:27:04.975 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:27:04.975 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:27:04.975 | INFO | ------------------------------------------ | 20260508 06:27:04.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:27:04.975 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:27:04.975 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:27:04.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:27:04.975 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.04e-01 | 20260508 06:27:04.975 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.19e-02 / 6.49e-03 | 20260508 06:27:04.975 | INFO | PerfReport: 12.11%, Baseline -- 3.96e+00 / 4.42e+00, 5.79e-01 / 2.49e-01 | 20260508 06:27:04.975 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.08e-01 / 1.21e-01 | 20260508 06:27:04.975 | INFO | PerfReport: 15.89%, DME -- 4.15e+01 / 4.47e+01, 3.55e+00 / 1.17e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.38e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.51e+00, 4.10e-01 / 2.43e-01 | 20260508 06:27:04.975 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.22e+00, 2.53e-01 / 1.23e-01 | 20260508 06:27:04.975 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.52e-01, 4.96e-02 / 2.58e-02 | 20260508 06:27:04.975 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:27:04.975 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:27:04.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:27:04.975 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 24.61%, Upload -- 2.62e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 24.24%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 30.74%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 2.38%, LockUpload -- 2.53e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 2.47%, LockDownload -- 2.62e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:27:04.975 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260508 06:27:05.072 | INFO | Starting a round of Baz IO | 20260508 06:27:08.498 | INFO | Populating RTMetrics report | 20260508 06:27:08.498 | INFO | RTMetrics report populated | 20260508 06:27:08.498 | INFO | Opening tmp RTMetrics json file | 20260508 06:27:08.509 | INFO | Serializing RTMetrics to json | 20260508 06:27:08.510 | INFO | Serialization done, swapping tmp file now... | 20260508 06:27:08.511 | INFO | RTMetrics done | 20260508 06:27:08.567 | INFO | Finished a round of Baz IO: Wrote 1.55 GiB in 3.494 seconds | 20260508 06:27:08.567 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.00% / 59.30% | 20260508 06:27:08.567 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39808% / 0% / 4.76837e-05% | 20260508 06:27:08.567 | INFO | Baz IO overall: | 20260508 06:27:08.567 | INFO | BytesWritten : 2637497043258 | 20260508 06:27:08.567 | INFO | NumEvents : 1271649548836 | 20260508 06:27:10.112 | INFO | Analyzing chunk frames = [5947392,5947904) | 20260508 06:27:10.112 | INFO | MemInfo: { | 20260508 06:27:10.112 | INFO | "gpu" : " 17 GB", | 20260508 06:27:10.112 | INFO | "huge" : "", | 20260508 06:27:10.112 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:10.112 | INFO | "malloc" : " 14 GB", | 20260508 06:27:10.112 | INFO | "pinned" : " 29 GB" | 20260508 06:27:10.112 | INFO | } | 20260508 06:27:10.112 | INFO | Perf report for the last 5.13969 seconds: | 20260508 06:27:10.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:10.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:10.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:10.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:10.112 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 06:27:10.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:10.112 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:27:10.112 | INFO | PRE_HQ : 65.7%, 1.6, 55.3ms, 0 | 20260508 06:27:10.112 | INFO | BAZWRITER : 67.5%, 2.6, 90.9ms, 0 | 20260508 06:27:10.112 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 06:27:10.113 | INFO | Uploaded 36GiB to GPU in 2.51758s (14.2995GiB/s) | 20260508 06:27:10.113 | INFO | Downloaded 38GiB to GPU in 2.67919s (14.1834GiB/s) | 20260508 06:27:15.251 | INFO | Analyzing chunk frames = [5947904,5948416) | 20260508 06:27:15.251 | INFO | MemInfo: { | 20260508 06:27:15.251 | INFO | "gpu" : " 17 GB", | 20260508 06:27:15.251 | INFO | "huge" : "", | 20260508 06:27:15.251 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:15.251 | INFO | "malloc" : " 14 GB", | 20260508 06:27:15.251 | INFO | "pinned" : " 29 GB" | 20260508 06:27:15.251 | INFO | } | 20260508 06:27:15.251 | INFO | Perf report for the last 5.13884 seconds: | 20260508 06:27:15.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:15.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:27:15.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:27:15.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:15.251 | INFO | Uploaded 36GiB to GPU in 2.47344s (14.5546GiB/s) | 20260508 06:27:15.251 | INFO | Downloaded 34GiB to GPU in 2.45287s (13.8613GiB/s) | 20260508 06:27:20.394 | INFO | Analyzing chunk frames = [5948416,5948928) | 20260508 06:27:20.394 | INFO | MemInfo: { | 20260508 06:27:20.394 | INFO | "gpu" : " 17 GB", | 20260508 06:27:20.394 | INFO | "huge" : "", | 20260508 06:27:20.394 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:20.394 | INFO | "malloc" : " 14 GB", | 20260508 06:27:20.394 | INFO | "pinned" : " 29 GB" | 20260508 06:27:20.394 | INFO | } | 20260508 06:27:20.395 | INFO | Perf report for the last 5.1435 seconds: | 20260508 06:27:20.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:20.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 06:27:20.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:27:20.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:20.395 | INFO | Uploaded 36GiB to GPU in 2.52292s (14.2692GiB/s) | 20260508 06:27:20.395 | INFO | Downloaded 34GiB to GPU in 2.42639s (14.0126GiB/s) | 20260508 06:27:25.530 | INFO | Analyzing chunk frames = [5948928,5949440) | 20260508 06:27:25.530 | INFO | MemInfo: { | 20260508 06:27:25.530 | INFO | "gpu" : " 17 GB", | 20260508 06:27:25.531 | INFO | "huge" : "", | 20260508 06:27:25.531 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:25.531 | INFO | "malloc" : " 14 GB", | 20260508 06:27:25.531 | INFO | "pinned" : " 29 GB" | 20260508 06:27:25.531 | INFO | } | 20260508 06:27:25.531 | INFO | Perf report for the last 5.13626 seconds: | 20260508 06:27:25.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:25.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | ANALYSIS : 66.7%, 3.0, 103.1ms, 0 | 20260508 06:27:25.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:27:25.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:25.531 | INFO | Uploaded 36GiB to GPU in 2.51738s (14.3006GiB/s) | 20260508 06:27:25.531 | INFO | Downloaded 34GiB to GPU in 2.43141s (13.9837GiB/s) | 20260508 06:27:30.672 | INFO | Analyzing chunk frames = [5949440,5949952) | 20260508 06:27:30.672 | INFO | MemInfo: { | 20260508 06:27:30.672 | INFO | "gpu" : " 17 GB", | 20260508 06:27:30.672 | INFO | "huge" : "", | 20260508 06:27:30.672 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:30.672 | INFO | "malloc" : " 14 GB", | 20260508 06:27:30.672 | INFO | "pinned" : " 29 GB" | 20260508 06:27:30.672 | INFO | } | 20260508 06:27:30.672 | INFO | Perf report for the last 5.1412 seconds: | 20260508 06:27:30.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:30.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 06:27:30.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:27:30.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:30.672 | INFO | Uploaded 36GiB to GPU in 2.41931s (14.8803GiB/s) | 20260508 06:27:30.672 | INFO | Downloaded 34GiB to GPU in 2.52686s (13.4555GiB/s) | 20260508 06:27:35.811 | INFO | Analyzing chunk frames = [5949952,5950464) | 20260508 06:27:35.811 | INFO | MemInfo: { | 20260508 06:27:35.811 | INFO | "gpu" : " 17 GB", | 20260508 06:27:35.811 | INFO | "huge" : "", | 20260508 06:27:35.811 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:35.811 | INFO | "malloc" : " 14 GB", | 20260508 06:27:35.811 | INFO | "pinned" : " 29 GB" | 20260508 06:27:35.811 | INFO | } | 20260508 06:27:35.811 | INFO | Perf report for the last 5.13899 seconds: | 20260508 06:27:35.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:35.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 06:27:35.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:27:35.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:35.811 | INFO | Uploaded 36GiB to GPU in 2.35578s (15.2816GiB/s) | 20260508 06:27:35.811 | INFO | Downloaded 34GiB to GPU in 2.45067s (13.8738GiB/s) | 20260508 06:27:40.950 | INFO | Analyzing chunk frames = [5950464,5950976) | 20260508 06:27:40.950 | INFO | MemInfo: { | 20260508 06:27:40.950 | INFO | "gpu" : " 17 GB", | 20260508 06:27:40.950 | INFO | "huge" : "", | 20260508 06:27:40.950 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:40.950 | INFO | "malloc" : " 14 GB", | 20260508 06:27:40.951 | INFO | "pinned" : " 29 GB" | 20260508 06:27:40.951 | INFO | } | 20260508 06:27:40.951 | INFO | Perf report for the last 5.13954 seconds: | 20260508 06:27:40.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:40.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 06:27:40.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:27:40.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:40.951 | INFO | Uploaded 36GiB to GPU in 2.4098s (14.939GiB/s) | 20260508 06:27:40.951 | INFO | Downloaded 34GiB to GPU in 2.44489s (13.9066GiB/s) | 20260508 06:27:46.082 | INFO | Analyzing chunk frames = [5950976,5951488) | 20260508 06:27:46.082 | INFO | MemInfo: { | 20260508 06:27:46.082 | INFO | "gpu" : " 17 GB", | 20260508 06:27:46.082 | INFO | "huge" : "", | 20260508 06:27:46.082 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:46.082 | INFO | "malloc" : " 14 GB", | 20260508 06:27:46.083 | INFO | "pinned" : " 29 GB" | 20260508 06:27:46.083 | INFO | } | 20260508 06:27:46.083 | INFO | Perf report for the last 5.1321 seconds: | 20260508 06:27:46.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:46.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 06:27:46.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 06:27:46.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:46.083 | INFO | Uploaded 36GiB to GPU in 2.43668s (14.7742GiB/s) | 20260508 06:27:46.083 | INFO | Downloaded 34GiB to GPU in 2.44946s (13.8806GiB/s) | 20260508 06:27:46.180 | INFO | Starting a round of Baz IO | 20260508 06:27:49.589 | INFO | Populating RTMetrics report | 20260508 06:27:49.589 | INFO | RTMetrics report populated | 20260508 06:27:49.589 | INFO | Opening tmp RTMetrics json file | 20260508 06:27:49.599 | INFO | Serializing RTMetrics to json | 20260508 06:27:49.602 | INFO | Serialization done, swapping tmp file now... | 20260508 06:27:49.602 | INFO | RTMetrics done | 20260508 06:27:49.661 | INFO | Finished a round of Baz IO: Wrote 1.55 GiB in 3.481 seconds | 20260508 06:27:49.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.00% / 59.30% | 20260508 06:27:49.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39618% / 7.94729e-06% / 4.76837e-05% | 20260508 06:27:49.661 | INFO | Baz IO overall: | 20260508 06:27:49.661 | INFO | BytesWritten : 2639160868441 | 20260508 06:27:49.661 | INFO | NumEvents : 1272441147451 | 20260508 06:27:51.225 | INFO | Analyzing chunk frames = [5951488,5952000) | 20260508 06:27:51.225 | INFO | MemInfo: { | 20260508 06:27:51.225 | INFO | "gpu" : " 17 GB", | 20260508 06:27:51.225 | INFO | "huge" : "", | 20260508 06:27:51.225 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:51.225 | INFO | "malloc" : " 14 GB", | 20260508 06:27:51.225 | INFO | "pinned" : " 29 GB" | 20260508 06:27:51.225 | INFO | } | 20260508 06:27:51.225 | INFO | Perf report for the last 5.14229 seconds: | 20260508 06:27:51.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:51.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:51.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:51.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:51.225 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 06:27:51.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:51.225 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 06:27:51.225 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 06:27:51.225 | INFO | BAZWRITER : 67.2%, 2.6, 91.0ms, 0 | 20260508 06:27:51.225 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:27:51.225 | INFO | Uploaded 36GiB to GPU in 2.53062s (14.2257GiB/s) | 20260508 06:27:51.225 | INFO | Downloaded 38GiB to GPU in 2.68616s (14.1466GiB/s) | 20260508 06:27:56.368 | INFO | Analyzing chunk frames = [5952000,5952512) | 20260508 06:27:56.368 | INFO | MemInfo: { | 20260508 06:27:56.368 | INFO | "gpu" : " 17 GB", | 20260508 06:27:56.368 | INFO | "huge" : "", | 20260508 06:27:56.368 | INFO | "huge_shared" : " 29 GB", | 20260508 06:27:56.368 | INFO | "malloc" : " 14 GB", | 20260508 06:27:56.368 | INFO | "pinned" : " 29 GB" | 20260508 06:27:56.368 | INFO | } | 20260508 06:27:56.369 | INFO | Perf report for the last 5.14348 seconds: | 20260508 06:27:56.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:27:56.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:27:56.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:27:56.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:27:56.369 | INFO | Uploaded 36GiB to GPU in 2.49263s (14.4426GiB/s) | 20260508 06:27:56.369 | INFO | Downloaded 34GiB to GPU in 2.44286s (13.9181GiB/s) | 20260508 06:28:01.512 | INFO | Analyzing chunk frames = [5952512,5953024) | 20260508 06:28:01.512 | INFO | MemInfo: { | 20260508 06:28:01.512 | INFO | "gpu" : " 17 GB", | 20260508 06:28:01.512 | INFO | "huge" : "", | 20260508 06:28:01.512 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:01.512 | INFO | "malloc" : " 14 GB", | 20260508 06:28:01.513 | INFO | "pinned" : " 29 GB" | 20260508 06:28:01.513 | INFO | } | 20260508 06:28:01.513 | INFO | Perf report for the last 5.14404 seconds: | 20260508 06:28:01.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:01.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 06:28:01.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:28:01.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:01.513 | INFO | Uploaded 36GiB to GPU in 2.5395s (14.176GiB/s) | 20260508 06:28:01.513 | INFO | Downloaded 34GiB to GPU in 2.41933s (14.0535GiB/s) | 20260508 06:28:06.650 | INFO | Analyzing chunk frames = [5953024,5953536) | 20260508 06:28:06.650 | INFO | MemInfo: { | 20260508 06:28:06.650 | INFO | "gpu" : " 17 GB", | 20260508 06:28:06.650 | INFO | "huge" : "", | 20260508 06:28:06.650 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:06.650 | INFO | "malloc" : " 14 GB", | 20260508 06:28:06.650 | INFO | "pinned" : " 29 GB" | 20260508 06:28:06.650 | INFO | } | 20260508 06:28:06.651 | INFO | Perf report for the last 5.13796 seconds: | 20260508 06:28:06.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:06.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 06:28:06.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:28:06.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:06.651 | INFO | Uploaded 36GiB to GPU in 2.56165s (14.0534GiB/s) | 20260508 06:28:06.651 | INFO | Downloaded 34GiB to GPU in 2.42131s (14.042GiB/s) | 20260508 06:28:11.779 | INFO | Analyzing chunk frames = [5953536,5954048) | 20260508 06:28:11.779 | INFO | MemInfo: { | 20260508 06:28:11.779 | INFO | "gpu" : " 17 GB", | 20260508 06:28:11.779 | INFO | "huge" : "", | 20260508 06:28:11.779 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:11.779 | INFO | "malloc" : " 14 GB", | 20260508 06:28:11.779 | INFO | "pinned" : " 29 GB" | 20260508 06:28:11.779 | INFO | } | 20260508 06:28:11.779 | INFO | Perf report for the last 5.12866 seconds: | 20260508 06:28:11.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:11.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 06:28:11.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 06:28:11.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:11.779 | INFO | Uploaded 36GiB to GPU in 2.28342s (15.7658GiB/s) | 20260508 06:28:11.780 | INFO | Downloaded 34GiB to GPU in 2.4924s (13.6415GiB/s) | 20260508 06:28:16.922 | INFO | Analyzing chunk frames = [5954048,5954560) | 20260508 06:28:16.922 | INFO | MemInfo: { | 20260508 06:28:16.922 | INFO | "gpu" : " 17 GB", | 20260508 06:28:16.922 | INFO | "huge" : "", | 20260508 06:28:16.922 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:16.922 | INFO | "malloc" : " 14 GB", | 20260508 06:28:16.922 | INFO | "pinned" : " 29 GB" | 20260508 06:28:16.922 | INFO | } | 20260508 06:28:16.922 | INFO | Perf report for the last 5.14311 seconds: | 20260508 06:28:16.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:16.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.922 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:28:16.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.922 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:28:16.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:16.923 | INFO | Uploaded 36GiB to GPU in 2.44574s (14.7195GiB/s) | 20260508 06:28:16.923 | INFO | Downloaded 34GiB to GPU in 2.41946s (14.0527GiB/s) | 20260508 06:28:22.064 | INFO | Analyzing chunk frames = [5954560,5955072) | 20260508 06:28:22.064 | INFO | MemInfo: { | 20260508 06:28:22.064 | INFO | "gpu" : " 17 GB", | 20260508 06:28:22.064 | INFO | "huge" : "", | 20260508 06:28:22.064 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:22.064 | INFO | "malloc" : " 14 GB", | 20260508 06:28:22.064 | INFO | "pinned" : " 29 GB" | 20260508 06:28:22.064 | INFO | } | 20260508 06:28:22.064 | INFO | Perf report for the last 5.14191 seconds: | 20260508 06:28:22.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:22.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.064 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:28:22.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.064 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:28:22.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:22.065 | INFO | Uploaded 36GiB to GPU in 2.48461s (14.4892GiB/s) | 20260508 06:28:22.065 | INFO | Downloaded 34GiB to GPU in 2.43053s (13.9887GiB/s) | 20260508 06:28:27.206 | INFO | Analyzing chunk frames = [5955072,5955584) | 20260508 06:28:27.206 | INFO | MemInfo: { | 20260508 06:28:27.206 | INFO | "gpu" : " 17 GB", | 20260508 06:28:27.206 | INFO | "huge" : "", | 20260508 06:28:27.206 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:27.206 | INFO | "malloc" : " 14 GB", | 20260508 06:28:27.206 | INFO | "pinned" : " 29 GB" | 20260508 06:28:27.206 | INFO | } | 20260508 06:28:27.206 | INFO | Perf report for the last 5.1418 seconds: | 20260508 06:28:27.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:27.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.206 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:28:27.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.206 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:28:27.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:27.207 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:28:27.207 | INFO | Total mem usage: 16.7 GiB | 20260508 06:28:27.207 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:28:27.207 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:28:27.207 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:28:27.207 | INFO | High Water Breakdown: | 20260508 06:28:27.207 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:28:27.207 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:28:27.207 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:28:27.207 | INFO | ------------------------------------------ | 20260508 06:28:27.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:28:27.207 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:28:27.207 | INFO | Total mem usage: 26.5 GiB | 20260508 06:28:27.207 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:28:27.207 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:28:27.207 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:28:27.207 | INFO | High Water Breakdown: | 20260508 06:28:27.207 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:28:27.207 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:28:27.207 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:28:27.207 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:28:27.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:28:27.207 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:28:27.207 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:28:27.207 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:28:27.207 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:28:27.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:28:27.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:28:27.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:28:27.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:28:27.207 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:28:27.207 | INFO | ------------------------------------------ | 20260508 06:28:27.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:28:27.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:28:27.207 | INFO | Total mem usage: 27.8 GiB | 20260508 06:28:27.207 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:28:27.207 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:28:27.207 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:28:27.207 | INFO | High Water Breakdown: | 20260508 06:28:27.207 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:28:27.207 | INFO | ------------------------------------------ | 20260508 06:28:27.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:28:27.207 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:28:27.207 | INFO | Total mem usage: 15.7 GiB | 20260508 06:28:27.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:28:27.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:28:27.207 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:28:27.207 | INFO | High Water Breakdown: | 20260508 06:28:27.207 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:28:27.207 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:28:27.207 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:28:27.207 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:28:27.207 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:28:27.207 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:28:27.207 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:28:27.207 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:28:27.207 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:28:27.207 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:28:27.207 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:28:27.207 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:28:27.207 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:28:27.207 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:28:27.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:28:27.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:28:27.207 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:28:27.207 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:28:27.207 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:28:27.207 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:28:27.207 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:28:27.207 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:28:27.207 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:28:27.207 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:28:27.207 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:28:27.207 | INFO | ------------------------------------------ | 20260508 06:28:27.207 | INFO | Uploaded 36GiB to GPU in 2.47626s (14.538GiB/s) | 20260508 06:28:27.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:28:27.207 | INFO | Downloaded 34GiB to GPU in 2.44119s (13.9276GiB/s) | 20260508 06:28:27.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:28:27.208 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:28:27.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:28:27.208 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.58e-01 | 20260508 06:28:27.208 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.06e-01, 1.17e-02 / 6.57e-03 | 20260508 06:28:27.208 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.39e+00, 5.87e-01 / 2.13e-01 | 20260508 06:28:27.208 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.11e-01 / 1.21e-01 | 20260508 06:28:27.208 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.47e+01, 3.65e+00 / 9.60e-01 | 20260508 06:28:27.208 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.81e+00 / 1.29e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.53e+00, 4.11e-01 / 2.43e-01 | 20260508 06:28:27.208 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.21e+00, 2.53e-01 / 1.43e-01 | 20260508 06:28:27.208 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.12e-01 / 3.41e-01, 4.90e-02 / 1.76e-02 | 20260508 06:28:27.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:28:27.208 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:28:27.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:28:27.208 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 24.25%, Upload -- 2.57e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 2.13%, LockUpload -- 2.26e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 2.72%, LockDownload -- 2.88e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 15.67%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:28:27.208 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260508 06:28:27.305 | INFO | Starting a round of Baz IO | 20260508 06:28:30.739 | INFO | Populating RTMetrics report | 20260508 06:28:30.740 | INFO | RTMetrics report populated | 20260508 06:28:30.740 | INFO | Opening tmp RTMetrics json file | 20260508 06:28:30.742 | INFO | Serializing RTMetrics to json | 20260508 06:28:30.743 | INFO | Serialization done, swapping tmp file now... | 20260508 06:28:30.743 | INFO | RTMetrics done | 20260508 06:28:30.819 | INFO | Finished a round of Baz IO: Wrote 1.55 GiB in 3.514 seconds | 20260508 06:28:30.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.00% / 59.30% | 20260508 06:28:30.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.396947% / 0% / 6.35783e-05% | 20260508 06:28:30.819 | INFO | Baz IO overall: | 20260508 06:28:30.819 | INFO | BytesWritten : 2640825287066 | 20260508 06:28:30.819 | INFO | NumEvents : 1273233206937 | 20260508 06:28:32.338 | INFO | Analyzing chunk frames = [5955584,5956096) | 20260508 06:28:32.338 | INFO | MemInfo: { | 20260508 06:28:32.338 | INFO | "gpu" : " 17 GB", | 20260508 06:28:32.338 | INFO | "huge" : "", | 20260508 06:28:32.338 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:32.338 | INFO | "malloc" : " 14 GB", | 20260508 06:28:32.338 | INFO | "pinned" : " 29 GB" | 20260508 06:28:32.338 | INFO | } | 20260508 06:28:32.339 | INFO | Perf report for the last 5.13246 seconds: | 20260508 06:28:32.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:32.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:32.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:32.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:32.339 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 06:28:32.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:32.339 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:28:32.339 | INFO | PRE_HQ : 65.9%, 1.6, 55.4ms, 0 | 20260508 06:28:32.339 | INFO | BAZWRITER : 68.0%, 2.9, 100.7ms, 0 | 20260508 06:28:32.339 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:28:32.339 | INFO | Uploaded 36GiB to GPU in 2.5416s (14.1643GiB/s) | 20260508 06:28:32.339 | INFO | Downloaded 38GiB to GPU in 2.69184s (14.1167GiB/s) | 20260508 06:28:37.485 | INFO | Analyzing chunk frames = [5956096,5956608) | 20260508 06:28:37.485 | INFO | MemInfo: { | 20260508 06:28:37.485 | INFO | "gpu" : " 17 GB", | 20260508 06:28:37.485 | INFO | "huge" : "", | 20260508 06:28:37.485 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:37.485 | INFO | "malloc" : " 14 GB", | 20260508 06:28:37.485 | INFO | "pinned" : " 29 GB" | 20260508 06:28:37.485 | INFO | } | 20260508 06:28:37.485 | INFO | Perf report for the last 5.14687 seconds: | 20260508 06:28:37.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:37.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.485 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:28:37.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.486 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:28:37.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:37.486 | INFO | Uploaded 36GiB to GPU in 2.51526s (14.3127GiB/s) | 20260508 06:28:37.486 | INFO | Downloaded 34GiB to GPU in 2.43594s (13.9576GiB/s) | 20260508 06:28:42.624 | INFO | Analyzing chunk frames = [5956608,5957120) | 20260508 06:28:42.624 | INFO | MemInfo: { | 20260508 06:28:42.624 | INFO | "gpu" : " 17 GB", | 20260508 06:28:42.624 | INFO | "huge" : "", | 20260508 06:28:42.624 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:42.624 | INFO | "malloc" : " 14 GB", | 20260508 06:28:42.625 | INFO | "pinned" : " 29 GB" | 20260508 06:28:42.625 | INFO | } | 20260508 06:28:42.625 | INFO | Perf report for the last 5.13923 seconds: | 20260508 06:28:42.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:42.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 06:28:42.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:28:42.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:42.625 | INFO | Uploaded 36GiB to GPU in 2.57276s (13.9928GiB/s) | 20260508 06:28:42.625 | INFO | Downloaded 34GiB to GPU in 2.41785s (14.0621GiB/s) | 20260508 06:28:47.763 | INFO | Analyzing chunk frames = [5957120,5957632) | 20260508 06:28:47.763 | INFO | MemInfo: { | 20260508 06:28:47.763 | INFO | "gpu" : " 17 GB", | 20260508 06:28:47.763 | INFO | "huge" : "", | 20260508 06:28:47.763 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:47.764 | INFO | "malloc" : " 14 GB", | 20260508 06:28:47.764 | INFO | "pinned" : " 29 GB" | 20260508 06:28:47.764 | INFO | } | 20260508 06:28:47.764 | INFO | Perf report for the last 5.13893 seconds: | 20260508 06:28:47.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:47.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:28:47.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:28:47.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:47.764 | INFO | Uploaded 36GiB to GPU in 2.58731s (13.9141GiB/s) | 20260508 06:28:47.764 | INFO | Downloaded 34GiB to GPU in 2.40233s (14.153GiB/s) | 20260508 06:28:52.901 | INFO | Analyzing chunk frames = [5957632,5958144) | 20260508 06:28:52.901 | INFO | MemInfo: { | 20260508 06:28:52.901 | INFO | "gpu" : " 17 GB", | 20260508 06:28:52.901 | INFO | "huge" : "", | 20260508 06:28:52.901 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:52.901 | INFO | "malloc" : " 14 GB", | 20260508 06:28:52.901 | INFO | "pinned" : " 29 GB" | 20260508 06:28:52.901 | INFO | } | 20260508 06:28:52.901 | INFO | Perf report for the last 5.13718 seconds: | 20260508 06:28:52.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:52.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:28:52.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:28:52.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:52.901 | INFO | Uploaded 36GiB to GPU in 2.29441s (15.6903GiB/s) | 20260508 06:28:52.901 | INFO | Downloaded 34GiB to GPU in 2.49481s (13.6283GiB/s) | 20260508 06:28:58.037 | INFO | Analyzing chunk frames = [5958144,5958656) | 20260508 06:28:58.037 | INFO | MemInfo: { | 20260508 06:28:58.037 | INFO | "gpu" : " 17 GB", | 20260508 06:28:58.037 | INFO | "huge" : "", | 20260508 06:28:58.037 | INFO | "huge_shared" : " 29 GB", | 20260508 06:28:58.037 | INFO | "malloc" : " 14 GB", | 20260508 06:28:58.037 | INFO | "pinned" : " 29 GB" | 20260508 06:28:58.037 | INFO | } | 20260508 06:28:58.037 | INFO | Perf report for the last 5.13604 seconds: | 20260508 06:28:58.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:28:58.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:28:58.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:28:58.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:28:58.037 | INFO | Uploaded 36GiB to GPU in 2.46889s (14.5814GiB/s) | 20260508 06:28:58.038 | INFO | Downloaded 34GiB to GPU in 2.42809s (14.0028GiB/s) | 20260508 06:29:03.181 | INFO | Analyzing chunk frames = [5958656,5959168) | 20260508 06:29:03.181 | INFO | MemInfo: { | 20260508 06:29:03.181 | INFO | "gpu" : " 17 GB", | 20260508 06:29:03.181 | INFO | "huge" : "", | 20260508 06:29:03.181 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:03.181 | INFO | "malloc" : " 14 GB", | 20260508 06:29:03.181 | INFO | "pinned" : " 29 GB" | 20260508 06:29:03.181 | INFO | } | 20260508 06:29:03.182 | INFO | Perf report for the last 5.14463 seconds: | 20260508 06:29:03.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:03.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:29:03.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:29:03.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:03.182 | INFO | Uploaded 36GiB to GPU in 2.47755s (14.5305GiB/s) | 20260508 06:29:03.182 | INFO | Downloaded 34GiB to GPU in 2.43168s (13.9821GiB/s) | 20260508 06:29:08.325 | INFO | Analyzing chunk frames = [5959168,5959680) | 20260508 06:29:08.325 | INFO | MemInfo: { | 20260508 06:29:08.325 | INFO | "gpu" : " 17 GB", | 20260508 06:29:08.325 | INFO | "huge" : "", | 20260508 06:29:08.325 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:08.325 | INFO | "malloc" : " 14 GB", | 20260508 06:29:08.325 | INFO | "pinned" : " 29 GB" | 20260508 06:29:08.325 | INFO | } | 20260508 06:29:08.325 | INFO | Perf report for the last 5.14385 seconds: | 20260508 06:29:08.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:08.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.325 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:29:08.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.326 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:29:08.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:08.326 | INFO | Uploaded 36GiB to GPU in 2.4867s (14.477GiB/s) | 20260508 06:29:08.326 | INFO | Downloaded 34GiB to GPU in 2.4381s (13.9453GiB/s) | 20260508 06:29:08.423 | INFO | Starting a round of Baz IO | 20260508 06:29:11.845 | INFO | Populating RTMetrics report | 20260508 06:29:11.845 | INFO | RTMetrics report populated | 20260508 06:29:11.845 | INFO | Opening tmp RTMetrics json file | 20260508 06:29:11.852 | INFO | Serializing RTMetrics to json | 20260508 06:29:11.854 | INFO | Serialization done, swapping tmp file now... | 20260508 06:29:11.854 | INFO | RTMetrics done | 20260508 06:29:11.913 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.49 seconds | 20260508 06:29:11.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.00% / 59.30% | 20260508 06:29:11.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.394865% / 0% / 7.15256e-05% | 20260508 06:29:11.913 | INFO | Baz IO overall: | 20260508 06:29:11.913 | INFO | BytesWritten : 2642487327895 | 20260508 06:29:11.913 | INFO | NumEvents : 1274023919192 | 20260508 06:29:13.459 | INFO | Analyzing chunk frames = [5959680,5960192) | 20260508 06:29:13.459 | INFO | MemInfo: { | 20260508 06:29:13.459 | INFO | "gpu" : " 17 GB", | 20260508 06:29:13.459 | INFO | "huge" : "", | 20260508 06:29:13.459 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:13.459 | INFO | "malloc" : " 14 GB", | 20260508 06:29:13.460 | INFO | "pinned" : " 29 GB" | 20260508 06:29:13.460 | INFO | } | 20260508 06:29:13.460 | INFO | Perf report for the last 5.13419 seconds: | 20260508 06:29:13.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:13.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:13.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:13.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:13.460 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 06:29:13.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:13.460 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 06:29:13.460 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 06:29:13.460 | INFO | BAZWRITER : 67.3%, 2.6, 90.6ms, 0 | 20260508 06:29:13.460 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:29:13.460 | INFO | Uploaded 36GiB to GPU in 2.58091s (13.9486GiB/s) | 20260508 06:29:13.460 | INFO | Downloaded 38GiB to GPU in 2.71668s (13.9877GiB/s) | 20260508 06:29:18.599 | INFO | Analyzing chunk frames = [5960192,5960704) | 20260508 06:29:18.599 | INFO | MemInfo: { | 20260508 06:29:18.599 | INFO | "gpu" : " 17 GB", | 20260508 06:29:18.599 | INFO | "huge" : "", | 20260508 06:29:18.599 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:18.599 | INFO | "malloc" : " 14 GB", | 20260508 06:29:18.599 | INFO | "pinned" : " 29 GB" | 20260508 06:29:18.599 | INFO | } | 20260508 06:29:18.599 | INFO | Perf report for the last 5.13953 seconds: | 20260508 06:29:18.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:18.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:29:18.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 06:29:18.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:18.599 | INFO | Uploaded 36GiB to GPU in 2.49299s (14.4405GiB/s) | 20260508 06:29:18.599 | INFO | Downloaded 34GiB to GPU in 2.45034s (13.8756GiB/s) | 20260508 06:29:23.738 | INFO | Analyzing chunk frames = [5960704,5961216) | 20260508 06:29:23.738 | INFO | MemInfo: { | 20260508 06:29:23.738 | INFO | "gpu" : " 17 GB", | 20260508 06:29:23.738 | INFO | "huge" : "", | 20260508 06:29:23.738 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:23.738 | INFO | "malloc" : " 14 GB", | 20260508 06:29:23.738 | INFO | "pinned" : " 29 GB" | 20260508 06:29:23.738 | INFO | } | 20260508 06:29:23.738 | INFO | Perf report for the last 5.13908 seconds: | 20260508 06:29:23.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:23.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 06:29:23.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:29:23.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:23.738 | INFO | Uploaded 36GiB to GPU in 2.56899s (14.0133GiB/s) | 20260508 06:29:23.738 | INFO | Downloaded 34GiB to GPU in 2.44475s (13.9073GiB/s) | 20260508 06:29:28.877 | INFO | Analyzing chunk frames = [5961216,5961728) | 20260508 06:29:28.877 | INFO | MemInfo: { | 20260508 06:29:28.877 | INFO | "gpu" : " 17 GB", | 20260508 06:29:28.877 | INFO | "huge" : "", | 20260508 06:29:28.877 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:28.877 | INFO | "malloc" : " 14 GB", | 20260508 06:29:28.877 | INFO | "pinned" : " 29 GB" | 20260508 06:29:28.877 | INFO | } | 20260508 06:29:28.877 | INFO | Perf report for the last 5.13873 seconds: | 20260508 06:29:28.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:28.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 06:29:28.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:29:28.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:28.877 | INFO | Uploaded 36GiB to GPU in 2.54964s (14.1196GiB/s) | 20260508 06:29:28.877 | INFO | Downloaded 34GiB to GPU in 2.39675s (14.1859GiB/s) | 20260508 06:29:34.014 | INFO | Analyzing chunk frames = [5961728,5962240) | 20260508 06:29:34.014 | INFO | MemInfo: { | 20260508 06:29:34.014 | INFO | "gpu" : " 17 GB", | 20260508 06:29:34.014 | INFO | "huge" : "", | 20260508 06:29:34.014 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:34.014 | INFO | "malloc" : " 14 GB", | 20260508 06:29:34.014 | INFO | "pinned" : " 29 GB" | 20260508 06:29:34.014 | INFO | } | 20260508 06:29:34.014 | INFO | Perf report for the last 5.13693 seconds: | 20260508 06:29:34.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:34.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 06:29:34.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:29:34.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:34.014 | INFO | Uploaded 36GiB to GPU in 2.29557s (15.6824GiB/s) | 20260508 06:29:34.014 | INFO | Downloaded 34GiB to GPU in 2.48174s (13.7001GiB/s) | 20260508 06:29:39.155 | INFO | Analyzing chunk frames = [5962240,5962752) | 20260508 06:29:39.155 | INFO | MemInfo: { | 20260508 06:29:39.155 | INFO | "gpu" : " 17 GB", | 20260508 06:29:39.155 | INFO | "huge" : "", | 20260508 06:29:39.155 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:39.155 | INFO | "malloc" : " 14 GB", | 20260508 06:29:39.155 | INFO | "pinned" : " 29 GB" | 20260508 06:29:39.155 | INFO | } | 20260508 06:29:39.155 | INFO | Perf report for the last 5.14158 seconds: | 20260508 06:29:39.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:39.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 06:29:39.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:29:39.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:39.156 | INFO | Uploaded 36GiB to GPU in 2.4351s (14.7838GiB/s) | 20260508 06:29:39.156 | INFO | Downloaded 34GiB to GPU in 2.43284s (13.9754GiB/s) | 20260508 06:29:44.291 | INFO | Analyzing chunk frames = [5962752,5963264) | 20260508 06:29:44.291 | INFO | MemInfo: { | 20260508 06:29:44.291 | INFO | "gpu" : " 17 GB", | 20260508 06:29:44.291 | INFO | "huge" : "", | 20260508 06:29:44.291 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:44.291 | INFO | "malloc" : " 14 GB", | 20260508 06:29:44.291 | INFO | "pinned" : " 29 GB" | 20260508 06:29:44.292 | INFO | } | 20260508 06:29:44.292 | INFO | Perf report for the last 5.1361 seconds: | 20260508 06:29:44.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:44.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 06:29:44.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:29:44.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:44.292 | INFO | Uploaded 36GiB to GPU in 2.44684s (14.7129GiB/s) | 20260508 06:29:44.292 | INFO | Downloaded 34GiB to GPU in 2.44058s (13.9311GiB/s) | 20260508 06:29:49.437 | INFO | Analyzing chunk frames = [5963264,5963776) | 20260508 06:29:49.437 | INFO | MemInfo: { | 20260508 06:29:49.437 | INFO | "gpu" : " 17 GB", | 20260508 06:29:49.437 | INFO | "huge" : "", | 20260508 06:29:49.437 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:49.437 | INFO | "malloc" : " 14 GB", | 20260508 06:29:49.437 | INFO | "pinned" : " 29 GB" | 20260508 06:29:49.437 | INFO | } | 20260508 06:29:49.437 | INFO | Perf report for the last 5.14546 seconds: | 20260508 06:29:49.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:49.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:29:49.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:29:49.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:49.437 | INFO | Uploaded 36GiB to GPU in 2.43215s (14.8017GiB/s) | 20260508 06:29:49.437 | INFO | Downloaded 34GiB to GPU in 2.43211s (13.9796GiB/s) | 20260508 06:29:49.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:29:49.437 | INFO | Total mem usage: 16.7 GiB | 20260508 06:29:49.437 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:29:49.437 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14%) | 20260508 06:29:49.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:29:49.437 | INFO | High Water Breakdown: | 20260508 06:29:49.437 | INFO | 59.9% : 8.76 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:29:49.437 | INFO | 40% : 5.86 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:29:49.437 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:29:49.437 | INFO | ------------------------------------------ | 20260508 06:29:49.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:29:49.437 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:29:49.437 | INFO | Total mem usage: 26.5 GiB | 20260508 06:29:49.437 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:29:49.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:29:49.437 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:29:49.437 | INFO | High Water Breakdown: | 20260508 06:29:49.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:29:49.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:29:49.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:29:49.437 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:29:49.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:29:49.437 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:29:49.437 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:29:49.437 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:29:49.437 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:29:49.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:29:49.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:29:49.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:29:49.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:29:49.437 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:29:49.438 | INFO | ------------------------------------------ | 20260508 06:29:49.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:29:49.438 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:29:49.438 | INFO | Total mem usage: 27.8 GiB | 20260508 06:29:49.438 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:29:49.438 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:29:49.438 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:29:49.438 | INFO | High Water Breakdown: | 20260508 06:29:49.438 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:29:49.438 | INFO | ------------------------------------------ | 20260508 06:29:49.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:29:49.438 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:29:49.438 | INFO | Total mem usage: 15.7 GiB | 20260508 06:29:49.438 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:29:49.438 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:29:49.438 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:29:49.438 | INFO | High Water Breakdown: | 20260508 06:29:49.438 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:29:49.438 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:29:49.438 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:29:49.438 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:29:49.438 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:29:49.438 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:29:49.438 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:29:49.438 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:29:49.438 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:29:49.438 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:29:49.438 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:29:49.438 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:29:49.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:29:49.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:29:49.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:29:49.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:29:49.438 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:29:49.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:29:49.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:29:49.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:29:49.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:29:49.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:29:49.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:29:49.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:29:49.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:29:49.438 | INFO | ------------------------------------------ | 20260508 06:29:49.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:29:49.438 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:29:49.438 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:29:49.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:29:49.438 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.38e+01 / 6.23e-01 | 20260508 06:29:49.438 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.05e-01, 1.15e-02 / 6.38e-03 | 20260508 06:29:49.438 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.46e+00, 5.88e-01 / 2.00e-01 | 20260508 06:29:49.438 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.41e+00, 2.08e-01 / 1.25e-01 | 20260508 06:29:49.438 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.51e+01, 3.84e+00 / 1.05e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.86e+00 / 1.34e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.52e+00, 4.12e-01 / 2.64e-01 | 20260508 06:29:49.438 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.20e+00, 2.51e-01 / 1.22e-01 | 20260508 06:29:49.438 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.47e-01, 4.98e-02 / 2.06e-02 | 20260508 06:29:49.438 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:29:49.438 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:29:49.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:29:49.438 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 24.39%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 2.32%, LockUpload -- 2.46e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 2.55%, LockDownload -- 2.71e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:29:49.438 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 3.82e-02 / 0.00e+00 | 20260508 06:29:49.535 | INFO | Starting a round of Baz IO | 20260508 06:29:52.956 | INFO | Populating RTMetrics report | 20260508 06:29:52.956 | INFO | RTMetrics report populated | 20260508 06:29:52.956 | INFO | Opening tmp RTMetrics json file | 20260508 06:29:52.969 | INFO | Serializing RTMetrics to json | 20260508 06:29:52.970 | INFO | Serialization done, swapping tmp file now... | 20260508 06:29:52.971 | INFO | RTMetrics done | 20260508 06:29:53.033 | INFO | Finished a round of Baz IO: Wrote 1.549 GiB in 3.497 seconds | 20260508 06:29:53.033 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.69% / 0.00% / 59.30% | 20260508 06:29:53.033 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.396593% / 0% / 5.5631e-05% | 20260508 06:29:53.033 | INFO | Baz IO overall: | 20260508 06:29:53.033 | INFO | BytesWritten : 2644150858354 | 20260508 06:29:53.033 | INFO | NumEvents : 1274815672285 | 20260508 06:29:54.575 | INFO | Analyzing chunk frames = [5963776,5964288) | 20260508 06:29:54.575 | INFO | MemInfo: { | 20260508 06:29:54.575 | INFO | "gpu" : " 17 GB", | 20260508 06:29:54.575 | INFO | "huge" : "", | 20260508 06:29:54.575 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:54.575 | INFO | "malloc" : " 14 GB", | 20260508 06:29:54.575 | INFO | "pinned" : " 29 GB" | 20260508 06:29:54.575 | INFO | } | 20260508 06:29:54.575 | INFO | Perf report for the last 5.13806 seconds: | 20260508 06:29:54.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:54.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:54.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:54.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:54.575 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 06:29:54.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:54.575 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:29:54.575 | INFO | PRE_HQ : 66.0%, 1.6, 55.3ms, 0 | 20260508 06:29:54.575 | INFO | BAZWRITER : 67.5%, 2.6, 90.9ms, 0 | 20260508 06:29:54.575 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:29:54.576 | INFO | Uploaded 36GiB to GPU in 2.60282s (13.8311GiB/s) | 20260508 06:29:54.576 | INFO | Downloaded 38GiB to GPU in 2.71673s (13.9874GiB/s) | 20260508 06:29:59.713 | INFO | Analyzing chunk frames = [5964288,5964800) | 20260508 06:29:59.713 | INFO | MemInfo: { | 20260508 06:29:59.713 | INFO | "gpu" : " 17 GB", | 20260508 06:29:59.713 | INFO | "huge" : "", | 20260508 06:29:59.713 | INFO | "huge_shared" : " 29 GB", | 20260508 06:29:59.713 | INFO | "malloc" : " 14 GB", | 20260508 06:29:59.713 | INFO | "pinned" : " 29 GB" | 20260508 06:29:59.713 | INFO | } | 20260508 06:29:59.713 | INFO | Perf report for the last 5.13793 seconds: | 20260508 06:29:59.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:29:59.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:29:59.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:29:59.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:29:59.713 | INFO | Uploaded 36GiB to GPU in 2.47829s (14.5262GiB/s) | 20260508 06:29:59.714 | INFO | Downloaded 34GiB to GPU in 2.4367s (13.9533GiB/s) | 20260508 06:30:04.847 | INFO | Analyzing chunk frames = [5964800,5965312) | 20260508 06:30:04.847 | INFO | MemInfo: { | 20260508 06:30:04.847 | INFO | "gpu" : " 17 GB", | 20260508 06:30:04.847 | INFO | "huge" : "", | 20260508 06:30:04.847 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:04.847 | INFO | "malloc" : " 14 GB", | 20260508 06:30:04.847 | INFO | "pinned" : " 29 GB" | 20260508 06:30:04.847 | INFO | } | 20260508 06:30:04.847 | INFO | Perf report for the last 5.13396 seconds: | 20260508 06:30:04.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:04.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | ANALYSIS : 67.0%, 3.0, 103.9ms, 0 | 20260508 06:30:04.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:30:04.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:04.847 | INFO | Uploaded 36GiB to GPU in 2.55243s (14.1042GiB/s) | 20260508 06:30:04.848 | INFO | Downloaded 34GiB to GPU in 2.43888s (13.9408GiB/s) | 20260508 06:30:09.989 | INFO | Analyzing chunk frames = [5965312,5965824) | 20260508 06:30:09.989 | INFO | MemInfo: { | 20260508 06:30:09.989 | INFO | "gpu" : " 17 GB", | 20260508 06:30:09.989 | INFO | "huge" : "", | 20260508 06:30:09.989 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:09.989 | INFO | "malloc" : " 14 GB", | 20260508 06:30:09.989 | INFO | "pinned" : " 29 GB" | 20260508 06:30:09.989 | INFO | } | 20260508 06:30:09.989 | INFO | Perf report for the last 5.14238 seconds: | 20260508 06:30:09.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:09.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.990 | INFO | ANALYSIS : 66.7%, 2.9, 103.3ms, 0 | 20260508 06:30:09.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.990 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:30:09.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:09.990 | INFO | Uploaded 36GiB to GPU in 2.55756s (14.0759GiB/s) | 20260508 06:30:09.990 | INFO | Downloaded 34GiB to GPU in 2.4199s (14.0502GiB/s) | 20260508 06:30:15.131 | INFO | Analyzing chunk frames = [5965824,5966336) | 20260508 06:30:15.131 | INFO | MemInfo: { | 20260508 06:30:15.131 | INFO | "gpu" : " 17 GB", | 20260508 06:30:15.131 | INFO | "huge" : "", | 20260508 06:30:15.131 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:15.131 | INFO | "malloc" : " 14 GB", | 20260508 06:30:15.131 | INFO | "pinned" : " 29 GB" | 20260508 06:30:15.131 | INFO | } | 20260508 06:30:15.131 | INFO | Perf report for the last 5.14157 seconds: | 20260508 06:30:15.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:15.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 06:30:15.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:30:15.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:15.131 | INFO | Uploaded 36GiB to GPU in 2.41283s (14.9202GiB/s) | 20260508 06:30:15.131 | INFO | Downloaded 34GiB to GPU in 2.52987s (13.4394GiB/s) | 20260508 06:30:20.277 | INFO | Analyzing chunk frames = [5966336,5966848) | 20260508 06:30:20.277 | INFO | MemInfo: { | 20260508 06:30:20.277 | INFO | "gpu" : " 17 GB", | 20260508 06:30:20.277 | INFO | "huge" : "", | 20260508 06:30:20.277 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:20.277 | INFO | "malloc" : " 14 GB", | 20260508 06:30:20.277 | INFO | "pinned" : " 29 GB" | 20260508 06:30:20.277 | INFO | } | 20260508 06:30:20.277 | INFO | Perf report for the last 5.14639 seconds: | 20260508 06:30:20.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:20.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.278 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 06:30:20.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.278 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:30:20.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:20.278 | INFO | Uploaded 36GiB to GPU in 2.42027s (14.8744GiB/s) | 20260508 06:30:20.278 | INFO | Downloaded 34GiB to GPU in 2.44707s (13.8942GiB/s) | 20260508 06:30:25.410 | INFO | Analyzing chunk frames = [5966848,5967360) | 20260508 06:30:25.410 | INFO | MemInfo: { | 20260508 06:30:25.410 | INFO | "gpu" : " 17 GB", | 20260508 06:30:25.410 | INFO | "huge" : "", | 20260508 06:30:25.410 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:25.410 | INFO | "malloc" : " 14 GB", | 20260508 06:30:25.410 | INFO | "pinned" : " 29 GB" | 20260508 06:30:25.410 | INFO | } | 20260508 06:30:25.410 | INFO | Perf report for the last 5.13298 seconds: | 20260508 06:30:25.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:25.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.410 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:30:25.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.411 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:30:25.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:25.411 | INFO | Uploaded 36GiB to GPU in 2.44365s (14.7321GiB/s) | 20260508 06:30:25.411 | INFO | Downloaded 34GiB to GPU in 2.45026s (13.8761GiB/s) | 20260508 06:30:30.546 | INFO | Analyzing chunk frames = [5967360,5967872) | 20260508 06:30:30.546 | INFO | MemInfo: { | 20260508 06:30:30.546 | INFO | "gpu" : " 17 GB", | 20260508 06:30:30.546 | INFO | "huge" : "", | 20260508 06:30:30.546 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:30.546 | INFO | "malloc" : " 14 GB", | 20260508 06:30:30.546 | INFO | "pinned" : " 29 GB" | 20260508 06:30:30.546 | INFO | } | 20260508 06:30:30.546 | INFO | Perf report for the last 5.13534 seconds: | 20260508 06:30:30.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:30.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:30:30.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 06:30:30.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:30.546 | INFO | Uploaded 36GiB to GPU in 2.43641s (14.7758GiB/s) | 20260508 06:30:30.546 | INFO | Downloaded 34GiB to GPU in 2.45721s (13.8368GiB/s) | 20260508 06:30:30.643 | INFO | Starting a round of Baz IO | 20260508 06:30:34.080 | INFO | Populating RTMetrics report | 20260508 06:30:34.081 | INFO | RTMetrics report populated | 20260508 06:30:34.081 | INFO | Opening tmp RTMetrics json file | 20260508 06:30:34.081 | INFO | Serializing RTMetrics to json | 20260508 06:30:34.083 | INFO | Serialization done, swapping tmp file now... | 20260508 06:30:34.083 | INFO | RTMetrics done | 20260508 06:30:34.149 | INFO | Finished a round of Baz IO: Wrote 1.55 GiB in 3.505 seconds | 20260508 06:30:34.149 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.69% / 0.00% / 59.31% | 20260508 06:30:34.149 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.396697% / 3.97364e-06% / 5.96046e-05% | 20260508 06:30:34.149 | INFO | Baz IO overall: | 20260508 06:30:34.149 | INFO | BytesWritten : 2645815555548 | 20260508 06:30:34.149 | INFO | NumEvents : 1275608508847 | 20260508 06:30:35.686 | INFO | Analyzing chunk frames = [5967872,5968384) | 20260508 06:30:35.686 | INFO | MemInfo: { | 20260508 06:30:35.686 | INFO | "gpu" : " 17 GB", | 20260508 06:30:35.686 | INFO | "huge" : "", | 20260508 06:30:35.686 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:35.686 | INFO | "malloc" : " 14 GB", | 20260508 06:30:35.686 | INFO | "pinned" : " 29 GB" | 20260508 06:30:35.686 | INFO | } | 20260508 06:30:35.686 | INFO | Perf report for the last 5.14014 seconds: | 20260508 06:30:35.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:35.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:35.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:35.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:35.686 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 06:30:35.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:35.686 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:30:35.686 | INFO | PRE_HQ : 65.9%, 1.6, 55.3ms, 0 | 20260508 06:30:35.686 | INFO | BAZWRITER : 67.5%, 2.6, 90.2ms, 0 | 20260508 06:30:35.686 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:30:35.686 | INFO | Uploaded 36GiB to GPU in 2.54447s (14.1483GiB/s) | 20260508 06:30:35.686 | INFO | Downloaded 38GiB to GPU in 2.7146s (13.9984GiB/s) | 20260508 06:30:40.828 | INFO | Analyzing chunk frames = [5968384,5968896) | 20260508 06:30:40.828 | INFO | MemInfo: { | 20260508 06:30:40.828 | INFO | "gpu" : " 17 GB", | 20260508 06:30:40.828 | INFO | "huge" : "", | 20260508 06:30:40.828 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:40.828 | INFO | "malloc" : " 14 GB", | 20260508 06:30:40.828 | INFO | "pinned" : " 29 GB" | 20260508 06:30:40.828 | INFO | } | 20260508 06:30:40.828 | INFO | Perf report for the last 5.14203 seconds: | 20260508 06:30:40.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:40.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:30:40.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:30:40.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:40.828 | INFO | Uploaded 36GiB to GPU in 2.46594s (14.5989GiB/s) | 20260508 06:30:40.828 | INFO | Downloaded 34GiB to GPU in 2.4454s (13.9037GiB/s) | 20260508 06:30:45.967 | INFO | Analyzing chunk frames = [5968896,5969408) | 20260508 06:30:45.967 | INFO | MemInfo: { | 20260508 06:30:45.967 | INFO | "gpu" : " 17 GB", | 20260508 06:30:45.967 | INFO | "huge" : "", | 20260508 06:30:45.967 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:45.967 | INFO | "malloc" : " 14 GB", | 20260508 06:30:45.967 | INFO | "pinned" : " 29 GB" | 20260508 06:30:45.967 | INFO | } | 20260508 06:30:45.967 | INFO | Perf report for the last 5.13904 seconds: | 20260508 06:30:45.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:45.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260508 06:30:45.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:30:45.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:45.967 | INFO | Uploaded 36GiB to GPU in 2.48553s (14.4838GiB/s) | 20260508 06:30:45.967 | INFO | Downloaded 34GiB to GPU in 2.44415s (13.9107GiB/s) | 20260508 06:30:51.104 | INFO | Analyzing chunk frames = [5969408,5969920) | 20260508 06:30:51.104 | INFO | MemInfo: { | 20260508 06:30:51.104 | INFO | "gpu" : " 17 GB", | 20260508 06:30:51.104 | INFO | "huge" : "", | 20260508 06:30:51.104 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:51.104 | INFO | "malloc" : " 14 GB", | 20260508 06:30:51.104 | INFO | "pinned" : " 29 GB" | 20260508 06:30:51.104 | INFO | } | 20260508 06:30:51.104 | INFO | Perf report for the last 5.13698 seconds: | 20260508 06:30:51.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:51.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | ANALYSIS : 66.5%, 3.0, 102.9ms, 0 | 20260508 06:30:51.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:30:51.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:51.104 | INFO | Uploaded 36GiB to GPU in 2.49418s (14.4336GiB/s) | 20260508 06:30:51.104 | INFO | Downloaded 34GiB to GPU in 2.42744s (14.0065GiB/s) | 20260508 06:30:56.245 | INFO | Analyzing chunk frames = [5969920,5970432) | 20260508 06:30:56.245 | INFO | MemInfo: { | 20260508 06:30:56.245 | INFO | "gpu" : " 17 GB", | 20260508 06:30:56.245 | INFO | "huge" : "", | 20260508 06:30:56.245 | INFO | "huge_shared" : " 29 GB", | 20260508 06:30:56.245 | INFO | "malloc" : " 14 GB", | 20260508 06:30:56.245 | INFO | "pinned" : " 29 GB" | 20260508 06:30:56.245 | INFO | } | 20260508 06:30:56.245 | INFO | Perf report for the last 5.14144 seconds: | 20260508 06:30:56.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:30:56.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:30:56.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:30:56.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:30:56.246 | INFO | Uploaded 36GiB to GPU in 2.27041s (15.8561GiB/s) | 20260508 06:30:56.246 | INFO | Downloaded 34GiB to GPU in 2.5008s (13.5957GiB/s) | 20260508 06:31:01.386 | INFO | Analyzing chunk frames = [5970432,5970944) | 20260508 06:31:01.386 | INFO | MemInfo: { | 20260508 06:31:01.386 | INFO | "gpu" : " 17 GB", | 20260508 06:31:01.386 | INFO | "huge" : "", | 20260508 06:31:01.386 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:01.386 | INFO | "malloc" : " 14 GB", | 20260508 06:31:01.386 | INFO | "pinned" : " 29 GB" | 20260508 06:31:01.386 | INFO | } | 20260508 06:31:01.386 | INFO | Perf report for the last 5.14064 seconds: | 20260508 06:31:01.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:01.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.386 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:31:01.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.386 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:31:01.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:01.387 | INFO | Uploaded 36GiB to GPU in 2.4414s (14.7456GiB/s) | 20260508 06:31:01.387 | INFO | Downloaded 34GiB to GPU in 2.43724s (13.9502GiB/s) | 20260508 06:31:06.525 | INFO | Analyzing chunk frames = [5970944,5971456) | 20260508 06:31:06.525 | INFO | MemInfo: { | 20260508 06:31:06.525 | INFO | "gpu" : " 17 GB", | 20260508 06:31:06.525 | INFO | "huge" : "", | 20260508 06:31:06.525 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:06.525 | INFO | "malloc" : " 14 GB", | 20260508 06:31:06.525 | INFO | "pinned" : " 29 GB" | 20260508 06:31:06.525 | INFO | } | 20260508 06:31:06.525 | INFO | Perf report for the last 5.13916 seconds: | 20260508 06:31:06.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:06.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.525 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 06:31:06.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.525 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:31:06.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:06.526 | INFO | Uploaded 36GiB to GPU in 2.44408s (14.7295GiB/s) | 20260508 06:31:06.526 | INFO | Downloaded 34GiB to GPU in 2.43751s (13.9487GiB/s) | 20260508 06:31:11.663 | INFO | Analyzing chunk frames = [5971456,5971968) | 20260508 06:31:11.663 | INFO | MemInfo: { | 20260508 06:31:11.663 | INFO | "gpu" : " 17 GB", | 20260508 06:31:11.663 | INFO | "huge" : "", | 20260508 06:31:11.663 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:11.663 | INFO | "malloc" : " 14 GB", | 20260508 06:31:11.663 | INFO | "pinned" : " 29 GB" | 20260508 06:31:11.663 | INFO | } | 20260508 06:31:11.663 | INFO | Perf report for the last 5.13782 seconds: | 20260508 06:31:11.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:11.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:31:11.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:31:11.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:11.663 | INFO | Uploaded 36GiB to GPU in 2.448s (14.7059GiB/s) | 20260508 06:31:11.663 | INFO | Downloaded 34GiB to GPU in 2.44783s (13.8898GiB/s) | 20260508 06:31:11.664 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:31:11.664 | INFO | Total mem usage: 16.7 GiB | 20260508 06:31:11.664 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:31:11.664 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.8%) | 20260508 06:31:11.664 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:31:11.664 | INFO | High Water Breakdown: | 20260508 06:31:11.664 | INFO | 59.5% : 8.51 GiB ( 82.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:31:11.664 | INFO | 40.3% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:31:11.664 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:31:11.664 | INFO | ------------------------------------------ | 20260508 06:31:11.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:31:11.664 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:31:11.664 | INFO | Total mem usage: 26.5 GiB | 20260508 06:31:11.664 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:31:11.664 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:31:11.664 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:31:11.664 | INFO | High Water Breakdown: | 20260508 06:31:11.664 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:31:11.664 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:31:11.664 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:31:11.664 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:31:11.664 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:31:11.664 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:31:11.664 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:31:11.664 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:31:11.664 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:31:11.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:31:11.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:31:11.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:31:11.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:31:11.664 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:31:11.664 | INFO | ------------------------------------------ | 20260508 06:31:11.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:31:11.664 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:31:11.664 | INFO | Total mem usage: 27.8 GiB | 20260508 06:31:11.664 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:31:11.664 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:31:11.664 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:31:11.664 | INFO | High Water Breakdown: | 20260508 06:31:11.664 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:31:11.664 | INFO | ------------------------------------------ | 20260508 06:31:11.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:31:11.664 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:31:11.664 | INFO | Total mem usage: 15.7 GiB | 20260508 06:31:11.664 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:31:11.664 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:31:11.664 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:31:11.664 | INFO | High Water Breakdown: | 20260508 06:31:11.664 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:31:11.664 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:31:11.664 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:31:11.664 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:31:11.664 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:31:11.664 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:31:11.664 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:31:11.664 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:31:11.664 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:31:11.664 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:31:11.664 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:31:11.664 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:31:11.664 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:31:11.664 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:31:11.664 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:31:11.664 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:31:11.664 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:31:11.664 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:31:11.664 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:31:11.664 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:31:11.664 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:31:11.664 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:31:11.665 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:31:11.665 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:31:11.665 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:31:11.665 | INFO | ------------------------------------------ | 20260508 06:31:11.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:31:11.665 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:31:11.665 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:31:11.665 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:31:11.665 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.36e+01 / 6.67e-01 | 20260508 06:31:11.665 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.11e-02 / 8.12e-03 | 20260508 06:31:11.665 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.40e+00, 5.79e-01 / 2.65e-01 | 20260508 06:31:11.665 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.05e-01 / 1.28e-01 | 20260508 06:31:11.665 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.53e+01, 3.84e+00 / 9.55e-01 | 20260508 06:31:11.665 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.39e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.37e+00 / 2.57e+00, 4.10e-01 / 2.21e-01 | 20260508 06:31:11.665 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.21e+00, 2.54e-01 / 1.35e-01 | 20260508 06:31:11.665 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.46e-01, 4.93e-02 / 2.58e-02 | 20260508 06:31:11.665 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:31:11.665 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:31:11.665 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:31:11.665 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 24.26%, Upload -- 2.57e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 24.42%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 2.08%, LockUpload -- 2.20e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 2.77%, LockDownload -- 2.94e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:31:11.665 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260508 06:31:11.761 | INFO | Starting a round of Baz IO | 20260508 06:31:15.192 | INFO | Populating RTMetrics report | 20260508 06:31:15.192 | INFO | RTMetrics report populated | 20260508 06:31:15.192 | INFO | Opening tmp RTMetrics json file | 20260508 06:31:15.201 | INFO | Serializing RTMetrics to json | 20260508 06:31:15.203 | INFO | Serialization done, swapping tmp file now... | 20260508 06:31:15.203 | INFO | RTMetrics done | 20260508 06:31:15.262 | INFO | Finished a round of Baz IO: Wrote 1.547 GiB in 3.5 seconds | 20260508 06:31:15.262 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.69% / 0.00% / 59.31% | 20260508 06:31:15.262 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.393538% / 0% / 9.93411e-05% | 20260508 06:31:15.262 | INFO | Baz IO overall: | 20260508 06:31:15.262 | INFO | BytesWritten : 2647476151394 | 20260508 06:31:15.262 | INFO | NumEvents : 1276398341570 | 20260508 06:31:16.800 | INFO | Analyzing chunk frames = [5971968,5972480) | 20260508 06:31:16.800 | INFO | MemInfo: { | 20260508 06:31:16.800 | INFO | "gpu" : " 17 GB", | 20260508 06:31:16.800 | INFO | "huge" : "", | 20260508 06:31:16.800 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:16.800 | INFO | "malloc" : " 14 GB", | 20260508 06:31:16.800 | INFO | "pinned" : " 29 GB" | 20260508 06:31:16.800 | INFO | } | 20260508 06:31:16.800 | INFO | Perf report for the last 5.13652 seconds: | 20260508 06:31:16.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:16.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:16.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:16.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:16.800 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:31:16.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:16.800 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:31:16.800 | INFO | PRE_HQ : 65.8%, 1.6, 55.4ms, 0 | 20260508 06:31:16.800 | INFO | BAZWRITER : 67.6%, 2.6, 90.8ms, 0 | 20260508 06:31:16.800 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:31:16.800 | INFO | Uploaded 36GiB to GPU in 2.61356s (13.7743GiB/s) | 20260508 06:31:16.800 | INFO | Downloaded 38GiB to GPU in 2.72019s (13.9696GiB/s) | 20260508 06:31:21.939 | INFO | Analyzing chunk frames = [5972480,5972992) | 20260508 06:31:21.939 | INFO | MemInfo: { | 20260508 06:31:21.939 | INFO | "gpu" : " 17 GB", | 20260508 06:31:21.939 | INFO | "huge" : "", | 20260508 06:31:21.939 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:21.939 | INFO | "malloc" : " 14 GB", | 20260508 06:31:21.939 | INFO | "pinned" : " 29 GB" | 20260508 06:31:21.939 | INFO | } | 20260508 06:31:21.939 | INFO | Perf report for the last 5.13921 seconds: | 20260508 06:31:21.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:21.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:31:21.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:31:21.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:21.939 | INFO | Uploaded 36GiB to GPU in 2.48585s (14.4819GiB/s) | 20260508 06:31:21.939 | INFO | Downloaded 34GiB to GPU in 2.45548s (13.8466GiB/s) | 20260508 06:31:27.087 | INFO | Analyzing chunk frames = [5972992,5973504) | 20260508 06:31:27.087 | INFO | MemInfo: { | 20260508 06:31:27.087 | INFO | "gpu" : " 17 GB", | 20260508 06:31:27.087 | INFO | "huge" : "", | 20260508 06:31:27.087 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:27.087 | INFO | "malloc" : " 14 GB", | 20260508 06:31:27.087 | INFO | "pinned" : " 29 GB" | 20260508 06:31:27.087 | INFO | } | 20260508 06:31:27.087 | INFO | Perf report for the last 5.14812 seconds: | 20260508 06:31:27.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:27.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 06:31:27.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:31:27.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:27.087 | INFO | Uploaded 36GiB to GPU in 2.52365s (14.265GiB/s) | 20260508 06:31:27.087 | INFO | Downloaded 34GiB to GPU in 2.40077s (14.1621GiB/s) | 20260508 06:31:32.227 | INFO | Analyzing chunk frames = [5973504,5974016) | 20260508 06:31:32.227 | INFO | MemInfo: { | 20260508 06:31:32.227 | INFO | "gpu" : " 17 GB", | 20260508 06:31:32.227 | INFO | "huge" : "", | 20260508 06:31:32.227 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:32.227 | INFO | "malloc" : " 14 GB", | 20260508 06:31:32.227 | INFO | "pinned" : " 29 GB" | 20260508 06:31:32.227 | INFO | } | 20260508 06:31:32.227 | INFO | Perf report for the last 5.14022 seconds: | 20260508 06:31:32.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:32.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.227 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 06:31:32.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.227 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 06:31:32.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:32.228 | INFO | Uploaded 36GiB to GPU in 2.56469s (14.0368GiB/s) | 20260508 06:31:32.228 | INFO | Downloaded 34GiB to GPU in 2.42151s (14.0408GiB/s) | 20260508 06:31:37.359 | INFO | Analyzing chunk frames = [5974016,5974528) | 20260508 06:31:37.359 | INFO | MemInfo: { | 20260508 06:31:37.359 | INFO | "gpu" : " 17 GB", | 20260508 06:31:37.359 | INFO | "huge" : "", | 20260508 06:31:37.359 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:37.359 | INFO | "malloc" : " 14 GB", | 20260508 06:31:37.359 | INFO | "pinned" : " 29 GB" | 20260508 06:31:37.359 | INFO | } | 20260508 06:31:37.360 | INFO | Perf report for the last 5.13229 seconds: | 20260508 06:31:37.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:37.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 06:31:37.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:31:37.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:37.360 | INFO | Uploaded 36GiB to GPU in 2.26288s (15.9089GiB/s) | 20260508 06:31:37.360 | INFO | Downloaded 34GiB to GPU in 2.51759s (13.505GiB/s) | 20260508 06:31:42.504 | INFO | Analyzing chunk frames = [5974528,5975040) | 20260508 06:31:42.504 | INFO | MemInfo: { | 20260508 06:31:42.504 | INFO | "gpu" : " 17 GB", | 20260508 06:31:42.504 | INFO | "huge" : "", | 20260508 06:31:42.504 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:42.504 | INFO | "malloc" : " 14 GB", | 20260508 06:31:42.504 | INFO | "pinned" : " 29 GB" | 20260508 06:31:42.504 | INFO | } | 20260508 06:31:42.504 | INFO | Perf report for the last 5.14422 seconds: | 20260508 06:31:42.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:42.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 06:31:42.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:31:42.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:42.504 | INFO | Uploaded 36GiB to GPU in 2.39997s (15.0002GiB/s) | 20260508 06:31:42.504 | INFO | Downloaded 34GiB to GPU in 2.44817s (13.8879GiB/s) | 20260508 06:31:47.640 | INFO | Analyzing chunk frames = [5975040,5975552) | 20260508 06:31:47.640 | INFO | MemInfo: { | 20260508 06:31:47.640 | INFO | "gpu" : " 17 GB", | 20260508 06:31:47.640 | INFO | "huge" : "", | 20260508 06:31:47.640 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:47.640 | INFO | "malloc" : " 14 GB", | 20260508 06:31:47.640 | INFO | "pinned" : " 29 GB" | 20260508 06:31:47.640 | INFO | } | 20260508 06:31:47.640 | INFO | Perf report for the last 5.13609 seconds: | 20260508 06:31:47.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:47.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:31:47.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:31:47.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:47.640 | INFO | Uploaded 36GiB to GPU in 2.44829s (14.7041GiB/s) | 20260508 06:31:47.640 | INFO | Downloaded 34GiB to GPU in 2.45022s (13.8763GiB/s) | 20260508 06:31:52.776 | INFO | Analyzing chunk frames = [5975552,5976064) | 20260508 06:31:52.776 | INFO | MemInfo: { | 20260508 06:31:52.776 | INFO | "gpu" : " 17 GB", | 20260508 06:31:52.776 | INFO | "huge" : "", | 20260508 06:31:52.776 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:52.776 | INFO | "malloc" : " 14 GB", | 20260508 06:31:52.776 | INFO | "pinned" : " 29 GB" | 20260508 06:31:52.776 | INFO | } | 20260508 06:31:52.776 | INFO | Perf report for the last 5.13619 seconds: | 20260508 06:31:52.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:52.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:31:52.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:31:52.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:52.776 | INFO | Uploaded 36GiB to GPU in 2.41687s (14.8953GiB/s) | 20260508 06:31:52.776 | INFO | Downloaded 34GiB to GPU in 2.45142s (13.8695GiB/s) | 20260508 06:31:52.873 | INFO | Starting a round of Baz IO | 20260508 06:31:56.290 | INFO | Populating RTMetrics report | 20260508 06:31:56.290 | INFO | RTMetrics report populated | 20260508 06:31:56.290 | INFO | Opening tmp RTMetrics json file | 20260508 06:31:56.298 | INFO | Serializing RTMetrics to json | 20260508 06:31:56.304 | INFO | Serialization done, swapping tmp file now... | 20260508 06:31:56.305 | INFO | RTMetrics done | 20260508 06:31:56.365 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.492 seconds | 20260508 06:31:56.365 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.69% / 0.00% / 59.31% | 20260508 06:31:56.365 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.393748% / 3.97364e-06% / 5.5631e-05% | 20260508 06:31:56.365 | INFO | Baz IO overall: | 20260508 06:31:56.365 | INFO | BytesWritten : 2649138386309 | 20260508 06:31:56.365 | INFO | NumEvents : 1277189485414 | 20260508 06:31:57.915 | INFO | Analyzing chunk frames = [5976064,5976576) | 20260508 06:31:57.915 | INFO | MemInfo: { | 20260508 06:31:57.915 | INFO | "gpu" : " 17 GB", | 20260508 06:31:57.915 | INFO | "huge" : "", | 20260508 06:31:57.916 | INFO | "huge_shared" : " 29 GB", | 20260508 06:31:57.916 | INFO | "malloc" : " 14 GB", | 20260508 06:31:57.916 | INFO | "pinned" : " 29 GB" | 20260508 06:31:57.916 | INFO | } | 20260508 06:31:57.916 | INFO | Perf report for the last 5.13964 seconds: | 20260508 06:31:57.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:31:57.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:57.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:57.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:57.916 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 06:31:57.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:31:57.916 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:31:57.916 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 06:31:57.916 | INFO | BAZWRITER : 67.5%, 2.6, 91.2ms, 0 | 20260508 06:31:57.916 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:31:57.916 | INFO | Uploaded 36GiB to GPU in 2.54852s (14.1258GiB/s) | 20260508 06:31:57.916 | INFO | Downloaded 38GiB to GPU in 2.69477s (14.1014GiB/s) | 20260508 06:32:03.051 | INFO | Analyzing chunk frames = [5976576,5977088) | 20260508 06:32:03.051 | INFO | MemInfo: { | 20260508 06:32:03.051 | INFO | "gpu" : " 17 GB", | 20260508 06:32:03.051 | INFO | "huge" : "", | 20260508 06:32:03.051 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:03.051 | INFO | "malloc" : " 14 GB", | 20260508 06:32:03.051 | INFO | "pinned" : " 29 GB" | 20260508 06:32:03.051 | INFO | } | 20260508 06:32:03.051 | INFO | Perf report for the last 5.13542 seconds: | 20260508 06:32:03.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:03.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.051 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 06:32:03.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.051 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:32:03.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:03.052 | INFO | Uploaded 36GiB to GPU in 2.44645s (14.7152GiB/s) | 20260508 06:32:03.052 | INFO | Downloaded 34GiB to GPU in 2.45723s (13.8367GiB/s) | 20260508 06:32:08.193 | INFO | Analyzing chunk frames = [5977088,5977600) | 20260508 06:32:08.194 | INFO | MemInfo: { | 20260508 06:32:08.194 | INFO | "gpu" : " 17 GB", | 20260508 06:32:08.194 | INFO | "huge" : "", | 20260508 06:32:08.194 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:08.194 | INFO | "malloc" : " 14 GB", | 20260508 06:32:08.194 | INFO | "pinned" : " 29 GB" | 20260508 06:32:08.194 | INFO | } | 20260508 06:32:08.194 | INFO | Perf report for the last 5.14274 seconds: | 20260508 06:32:08.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:08.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 06:32:08.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 06:32:08.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:08.194 | INFO | Uploaded 36GiB to GPU in 2.50138s (14.3921GiB/s) | 20260508 06:32:08.194 | INFO | Downloaded 34GiB to GPU in 2.43185s (13.9811GiB/s) | 20260508 06:32:13.340 | INFO | Analyzing chunk frames = [5977600,5978112) | 20260508 06:32:13.340 | INFO | MemInfo: { | 20260508 06:32:13.340 | INFO | "gpu" : " 17 GB", | 20260508 06:32:13.340 | INFO | "huge" : "", | 20260508 06:32:13.340 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:13.340 | INFO | "malloc" : " 14 GB", | 20260508 06:32:13.340 | INFO | "pinned" : " 29 GB" | 20260508 06:32:13.340 | INFO | } | 20260508 06:32:13.340 | INFO | Perf report for the last 5.14586 seconds: | 20260508 06:32:13.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:13.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 06:32:13.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:32:13.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:13.340 | INFO | Uploaded 36GiB to GPU in 2.50479s (14.3725GiB/s) | 20260508 06:32:13.340 | INFO | Downloaded 34GiB to GPU in 2.42027s (14.048GiB/s) | 20260508 06:32:18.477 | INFO | Analyzing chunk frames = [5978112,5978624) | 20260508 06:32:18.477 | INFO | MemInfo: { | 20260508 06:32:18.477 | INFO | "gpu" : " 17 GB", | 20260508 06:32:18.477 | INFO | "huge" : "", | 20260508 06:32:18.477 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:18.478 | INFO | "malloc" : " 14 GB", | 20260508 06:32:18.478 | INFO | "pinned" : " 29 GB" | 20260508 06:32:18.478 | INFO | } | 20260508 06:32:18.478 | INFO | Perf report for the last 5.13787 seconds: | 20260508 06:32:18.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:18.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:32:18.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:32:18.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:18.478 | INFO | Uploaded 36GiB to GPU in 2.33659s (15.4071GiB/s) | 20260508 06:32:18.478 | INFO | Downloaded 34GiB to GPU in 2.52272s (13.4775GiB/s) | 20260508 06:32:23.608 | INFO | Analyzing chunk frames = [5978624,5979136) | 20260508 06:32:23.608 | INFO | MemInfo: { | 20260508 06:32:23.608 | INFO | "gpu" : " 17 GB", | 20260508 06:32:23.608 | INFO | "huge" : "", | 20260508 06:32:23.609 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:23.609 | INFO | "malloc" : " 14 GB", | 20260508 06:32:23.609 | INFO | "pinned" : " 29 GB" | 20260508 06:32:23.609 | INFO | } | 20260508 06:32:23.609 | INFO | Perf report for the last 5.13089 seconds: | 20260508 06:32:23.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:23.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 06:32:23.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:32:23.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:23.609 | INFO | Uploaded 36GiB to GPU in 2.38506s (15.0939GiB/s) | 20260508 06:32:23.609 | INFO | Downloaded 34GiB to GPU in 2.43854s (13.9428GiB/s) | 20260508 06:32:28.754 | INFO | Analyzing chunk frames = [5979136,5979648) | 20260508 06:32:28.754 | INFO | MemInfo: { | 20260508 06:32:28.754 | INFO | "gpu" : " 17 GB", | 20260508 06:32:28.754 | INFO | "huge" : "", | 20260508 06:32:28.754 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:28.754 | INFO | "malloc" : " 14 GB", | 20260508 06:32:28.754 | INFO | "pinned" : " 29 GB" | 20260508 06:32:28.754 | INFO | } | 20260508 06:32:28.754 | INFO | Perf report for the last 5.14515 seconds: | 20260508 06:32:28.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:28.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 06:32:28.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:32:28.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:28.754 | INFO | Uploaded 36GiB to GPU in 2.43635s (14.7762GiB/s) | 20260508 06:32:28.754 | INFO | Downloaded 34GiB to GPU in 2.43759s (13.9482GiB/s) | 20260508 06:32:33.893 | INFO | Analyzing chunk frames = [5979648,5980160) | 20260508 06:32:33.893 | INFO | MemInfo: { | 20260508 06:32:33.893 | INFO | "gpu" : " 17 GB", | 20260508 06:32:33.893 | INFO | "huge" : "", | 20260508 06:32:33.893 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:33.893 | INFO | "malloc" : " 14 GB", | 20260508 06:32:33.893 | INFO | "pinned" : " 29 GB" | 20260508 06:32:33.893 | INFO | } | 20260508 06:32:33.893 | INFO | Perf report for the last 5.13905 seconds: | 20260508 06:32:33.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:33.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:32:33.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:32:33.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:33.893 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:32:33.893 | INFO | Total mem usage: 16.7 GiB | 20260508 06:32:33.893 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:32:33.893 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:32:33.893 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:32:33.893 | INFO | High Water Breakdown: | 20260508 06:32:33.893 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:32:33.893 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:32:33.893 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:32:33.893 | INFO | Uploaded 36GiB to GPU in 2.45472s (14.6656GiB/s) | 20260508 06:32:33.893 | INFO | ------------------------------------------ | 20260508 06:32:33.894 | INFO | Downloaded 34GiB to GPU in 2.45086s (13.8727GiB/s) | 20260508 06:32:33.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:32:33.894 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:32:33.894 | INFO | Total mem usage: 26.5 GiB | 20260508 06:32:33.894 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:32:33.894 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 06:32:33.894 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 06:32:33.894 | INFO | High Water Breakdown: | 20260508 06:32:33.894 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:32:33.894 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:32:33.894 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:32:33.894 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:32:33.894 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:32:33.894 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:32:33.894 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:32:33.894 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:32:33.894 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:32:33.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:32:33.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:32:33.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:32:33.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:32:33.894 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:32:33.894 | INFO | ------------------------------------------ | 20260508 06:32:33.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:32:33.894 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:32:33.894 | INFO | Total mem usage: 27.8 GiB | 20260508 06:32:33.894 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:32:33.894 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:32:33.894 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:32:33.894 | INFO | High Water Breakdown: | 20260508 06:32:33.894 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:32:33.894 | INFO | ------------------------------------------ | 20260508 06:32:33.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:32:33.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:32:33.894 | INFO | Total mem usage: 15.7 GiB | 20260508 06:32:33.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:32:33.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:32:33.894 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:32:33.894 | INFO | High Water Breakdown: | 20260508 06:32:33.894 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:32:33.894 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:32:33.894 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:32:33.894 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:32:33.894 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:32:33.894 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:32:33.894 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:32:33.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:32:33.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:32:33.894 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:32:33.894 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:32:33.894 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:32:33.894 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:32:33.894 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:32:33.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:32:33.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:32:33.894 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:32:33.894 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:32:33.894 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:32:33.894 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:32:33.894 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:32:33.894 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:32:33.894 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:32:33.894 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:32:33.894 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:32:33.894 | INFO | ------------------------------------------ | 20260508 06:32:33.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:32:33.894 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:32:33.894 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:32:33.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:32:33.894 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 7.28e-01 | 20260508 06:32:33.894 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.18e-02 / 4.33e-03 | 20260508 06:32:33.894 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.41e+00, 5.86e-01 / 2.51e-01 | 20260508 06:32:33.894 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.09e-01 / 1.38e-01 | 20260508 06:32:33.894 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.53e+01, 3.81e+00 / 1.25e+00 | 20260508 06:32:33.894 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.35e+00 | 20260508 06:32:33.894 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.59e+00, 4.13e-01 / 2.04e-01 | 20260508 06:32:33.894 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.21e+00, 2.53e-01 / 1.21e-01 | 20260508 06:32:33.894 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.46e-01, 4.98e-02 / 1.96e-02 | 20260508 06:32:33.895 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:32:33.895 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:32:33.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:32:33.895 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 24.18%, Upload -- 2.56e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 24.42%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 30.87%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 2.05%, LockUpload -- 2.17e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 2.80%, LockDownload -- 2.97e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:32:33.895 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260508 06:32:33.991 | INFO | Starting a round of Baz IO | 20260508 06:32:37.437 | INFO | Populating RTMetrics report | 20260508 06:32:37.437 | INFO | RTMetrics report populated | 20260508 06:32:37.437 | INFO | Opening tmp RTMetrics json file | 20260508 06:32:37.438 | INFO | Serializing RTMetrics to json | 20260508 06:32:37.448 | INFO | Serialization done, swapping tmp file now... | 20260508 06:32:37.456 | INFO | RTMetrics done | 20260508 06:32:37.517 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.526 seconds | 20260508 06:32:37.517 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.69% / 0.00% / 59.31% | 20260508 06:32:37.517 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39312% / 0% / 6.35783e-05% | 20260508 06:32:37.517 | INFO | Baz IO overall: | 20260508 06:32:37.517 | INFO | BytesWritten : 2650800439190 | 20260508 06:32:37.517 | INFO | NumEvents : 1277980575698 | 20260508 06:32:39.038 | INFO | Analyzing chunk frames = [5980160,5980672) | 20260508 06:32:39.038 | INFO | MemInfo: { | 20260508 06:32:39.038 | INFO | "gpu" : " 17 GB", | 20260508 06:32:39.038 | INFO | "huge" : "", | 20260508 06:32:39.038 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:39.038 | INFO | "malloc" : " 14 GB", | 20260508 06:32:39.038 | INFO | "pinned" : " 29 GB" | 20260508 06:32:39.038 | INFO | } | 20260508 06:32:39.038 | INFO | Perf report for the last 5.14516 seconds: | 20260508 06:32:39.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:39.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:39.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:39.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:39.038 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 06:32:39.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:39.039 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 06:32:39.039 | INFO | PRE_HQ : 65.8%, 1.6, 55.6ms, 0 | 20260508 06:32:39.039 | INFO | BAZWRITER : 68.0%, 2.6, 91.8ms, 0 | 20260508 06:32:39.039 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 06:32:39.039 | INFO | Uploaded 36GiB to GPU in 2.59353s (13.8807GiB/s) | 20260508 06:32:39.039 | INFO | Downloaded 38GiB to GPU in 2.69766s (14.0863GiB/s) | 20260508 06:32:44.176 | INFO | Analyzing chunk frames = [5980672,5981184) | 20260508 06:32:44.176 | INFO | MemInfo: { | 20260508 06:32:44.176 | INFO | "gpu" : " 17 GB", | 20260508 06:32:44.176 | INFO | "huge" : "", | 20260508 06:32:44.176 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:44.176 | INFO | "malloc" : " 14 GB", | 20260508 06:32:44.176 | INFO | "pinned" : " 29 GB" | 20260508 06:32:44.176 | INFO | } | 20260508 06:32:44.176 | INFO | Perf report for the last 5.13827 seconds: | 20260508 06:32:44.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:44.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.176 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:32:44.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.176 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:32:44.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:44.177 | INFO | Uploaded 36GiB to GPU in 2.50667s (14.3617GiB/s) | 20260508 06:32:44.177 | INFO | Downloaded 34GiB to GPU in 2.43308s (13.9741GiB/s) | 20260508 06:32:49.309 | INFO | Analyzing chunk frames = [5981184,5981696) | 20260508 06:32:49.309 | INFO | MemInfo: { | 20260508 06:32:49.309 | INFO | "gpu" : " 17 GB", | 20260508 06:32:49.309 | INFO | "huge" : "", | 20260508 06:32:49.309 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:49.309 | INFO | "malloc" : " 14 GB", | 20260508 06:32:49.309 | INFO | "pinned" : " 29 GB" | 20260508 06:32:49.309 | INFO | } | 20260508 06:32:49.309 | INFO | Perf report for the last 5.13253 seconds: | 20260508 06:32:49.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:49.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | ANALYSIS : 67.4%, 3.0, 104.6ms, 0 | 20260508 06:32:49.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:32:49.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:49.309 | INFO | Uploaded 36GiB to GPU in 2.58914s (13.9042GiB/s) | 20260508 06:32:49.309 | INFO | Downloaded 34GiB to GPU in 2.43905s (13.9398GiB/s) | 20260508 06:32:54.449 | INFO | Analyzing chunk frames = [5981696,5982208) | 20260508 06:32:54.449 | INFO | MemInfo: { | 20260508 06:32:54.449 | INFO | "gpu" : " 17 GB", | 20260508 06:32:54.449 | INFO | "huge" : "", | 20260508 06:32:54.450 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:54.450 | INFO | "malloc" : " 14 GB", | 20260508 06:32:54.450 | INFO | "pinned" : " 29 GB" | 20260508 06:32:54.450 | INFO | } | 20260508 06:32:54.450 | INFO | Perf report for the last 5.14088 seconds: | 20260508 06:32:54.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:54.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 06:32:54.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:32:54.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:54.450 | INFO | Uploaded 36GiB to GPU in 2.57104s (14.0021GiB/s) | 20260508 06:32:54.450 | INFO | Downloaded 34GiB to GPU in 2.38489s (14.2564GiB/s) | 20260508 06:32:59.592 | INFO | Analyzing chunk frames = [5982208,5982720) | 20260508 06:32:59.592 | INFO | MemInfo: { | 20260508 06:32:59.592 | INFO | "gpu" : " 17 GB", | 20260508 06:32:59.592 | INFO | "huge" : "", | 20260508 06:32:59.593 | INFO | "huge_shared" : " 29 GB", | 20260508 06:32:59.593 | INFO | "malloc" : " 14 GB", | 20260508 06:32:59.593 | INFO | "pinned" : " 29 GB" | 20260508 06:32:59.593 | INFO | } | 20260508 06:32:59.593 | INFO | Perf report for the last 5.14289 seconds: | 20260508 06:32:59.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:32:59.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:32:59.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:32:59.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:32:59.593 | INFO | Uploaded 36GiB to GPU in 2.31413s (15.5566GiB/s) | 20260508 06:32:59.594 | INFO | Downloaded 34GiB to GPU in 2.48243s (13.6963GiB/s) | 20260508 06:33:04.731 | INFO | Analyzing chunk frames = [5982720,5983232) | 20260508 06:33:04.731 | INFO | MemInfo: { | 20260508 06:33:04.731 | INFO | "gpu" : " 17 GB", | 20260508 06:33:04.731 | INFO | "huge" : "", | 20260508 06:33:04.731 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:04.731 | INFO | "malloc" : " 14 GB", | 20260508 06:33:04.731 | INFO | "pinned" : " 29 GB" | 20260508 06:33:04.731 | INFO | } | 20260508 06:33:04.731 | INFO | Perf report for the last 5.13846 seconds: | 20260508 06:33:04.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:04.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.731 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:33:04.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.731 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:33:04.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:04.732 | INFO | Uploaded 36GiB to GPU in 2.46046s (14.6314GiB/s) | 20260508 06:33:04.732 | INFO | Downloaded 34GiB to GPU in 2.43679s (13.9528GiB/s) | 20260508 06:33:09.864 | INFO | Analyzing chunk frames = [5983232,5983744) | 20260508 06:33:09.864 | INFO | MemInfo: { | 20260508 06:33:09.864 | INFO | "gpu" : " 17 GB", | 20260508 06:33:09.864 | INFO | "huge" : "", | 20260508 06:33:09.864 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:09.864 | INFO | "malloc" : " 14 GB", | 20260508 06:33:09.864 | INFO | "pinned" : " 29 GB" | 20260508 06:33:09.864 | INFO | } | 20260508 06:33:09.864 | INFO | Perf report for the last 5.1331 seconds: | 20260508 06:33:09.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:09.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:33:09.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:33:09.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:09.864 | INFO | Uploaded 36GiB to GPU in 2.47149s (14.5661GiB/s) | 20260508 06:33:09.865 | INFO | Downloaded 34GiB to GPU in 2.43607s (13.9569GiB/s) | 20260508 06:33:15.007 | INFO | Analyzing chunk frames = [5983744,5984256) | 20260508 06:33:15.007 | INFO | MemInfo: { | 20260508 06:33:15.007 | INFO | "gpu" : " 17 GB", | 20260508 06:33:15.007 | INFO | "huge" : "", | 20260508 06:33:15.008 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:15.008 | INFO | "malloc" : " 14 GB", | 20260508 06:33:15.008 | INFO | "pinned" : " 29 GB" | 20260508 06:33:15.008 | INFO | } | 20260508 06:33:15.008 | INFO | Perf report for the last 5.14352 seconds: | 20260508 06:33:15.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:15.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:33:15.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 06:33:15.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:15.008 | INFO | Uploaded 36GiB to GPU in 2.45197s (14.6821GiB/s) | 20260508 06:33:15.008 | INFO | Downloaded 34GiB to GPU in 2.4366s (13.9539GiB/s) | 20260508 06:33:15.122 | INFO | Starting a round of Baz IO | 20260508 06:33:18.532 | INFO | Populating RTMetrics report | 20260508 06:33:18.533 | INFO | RTMetrics report populated | 20260508 06:33:18.533 | INFO | Opening tmp RTMetrics json file | 20260508 06:33:18.533 | INFO | Serializing RTMetrics to json | 20260508 06:33:18.534 | INFO | Serialization done, swapping tmp file now... | 20260508 06:33:18.535 | INFO | RTMetrics done | 20260508 06:33:18.598 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.475 seconds | 20260508 06:33:18.598 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.00% / 59.31% | 20260508 06:33:18.598 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39283% / 7.94729e-06% / 8.74201e-05% | 20260508 06:33:18.598 | INFO | Baz IO overall: | 20260508 06:33:18.598 | INFO | BytesWritten : 2652462445600 | 20260508 06:33:18.598 | INFO | NumEvents : 1278771722412 | 20260508 06:33:20.147 | INFO | Analyzing chunk frames = [5984256,5984768) | 20260508 06:33:20.147 | INFO | MemInfo: { | 20260508 06:33:20.147 | INFO | "gpu" : " 17 GB", | 20260508 06:33:20.147 | INFO | "huge" : "", | 20260508 06:33:20.147 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:20.147 | INFO | "malloc" : " 14 GB", | 20260508 06:33:20.147 | INFO | "pinned" : " 29 GB" | 20260508 06:33:20.147 | INFO | } | 20260508 06:33:20.147 | INFO | Perf report for the last 5.13942 seconds: | 20260508 06:33:20.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:20.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:20.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:20.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:20.147 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260508 06:33:20.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:20.147 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 06:33:20.147 | INFO | PRE_HQ : 67.1%, 1.8, 61.9ms, 0 | 20260508 06:33:20.147 | INFO | BAZWRITER : 66.9%, 2.8, 97.7ms, 0 | 20260508 06:33:20.147 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 06:33:20.147 | INFO | Uploaded 36GiB to GPU in 2.51433s (14.318GiB/s) | 20260508 06:33:20.148 | INFO | Downloaded 38GiB to GPU in 2.68984s (14.1272GiB/s) | 20260508 06:33:25.289 | INFO | Analyzing chunk frames = [5984768,5985280) | 20260508 06:33:25.289 | INFO | MemInfo: { | 20260508 06:33:25.289 | INFO | "gpu" : " 17 GB", | 20260508 06:33:25.289 | INFO | "huge" : "", | 20260508 06:33:25.290 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:25.290 | INFO | "malloc" : " 14 GB", | 20260508 06:33:25.290 | INFO | "pinned" : " 29 GB" | 20260508 06:33:25.290 | INFO | } | 20260508 06:33:25.290 | INFO | Perf report for the last 5.14251 seconds: | 20260508 06:33:25.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:25.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:33:25.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:33:25.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:25.290 | INFO | Uploaded 36GiB to GPU in 2.50625s (14.3641GiB/s) | 20260508 06:33:25.290 | INFO | Downloaded 34GiB to GPU in 2.43901s (13.9401GiB/s) | 20260508 06:33:30.428 | INFO | Analyzing chunk frames = [5985280,5985792) | 20260508 06:33:30.428 | INFO | MemInfo: { | 20260508 06:33:30.428 | INFO | "gpu" : " 17 GB", | 20260508 06:33:30.428 | INFO | "huge" : "", | 20260508 06:33:30.429 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:30.429 | INFO | "malloc" : " 14 GB", | 20260508 06:33:30.429 | INFO | "pinned" : " 29 GB" | 20260508 06:33:30.429 | INFO | } | 20260508 06:33:30.429 | INFO | Perf report for the last 5.13894 seconds: | 20260508 06:33:30.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:30.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:33:30.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:33:30.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:30.429 | INFO | Uploaded 36GiB to GPU in 2.55607s (14.0841GiB/s) | 20260508 06:33:30.429 | INFO | Downloaded 34GiB to GPU in 2.43425s (13.9674GiB/s) | 20260508 06:33:35.562 | INFO | Analyzing chunk frames = [5985792,5986304) | 20260508 06:33:35.562 | INFO | MemInfo: { | 20260508 06:33:35.562 | INFO | "gpu" : " 17 GB", | 20260508 06:33:35.562 | INFO | "huge" : "", | 20260508 06:33:35.562 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:35.562 | INFO | "malloc" : " 14 GB", | 20260508 06:33:35.562 | INFO | "pinned" : " 29 GB" | 20260508 06:33:35.562 | INFO | } | 20260508 06:33:35.562 | INFO | Perf report for the last 5.13375 seconds: | 20260508 06:33:35.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:35.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.563 | INFO | ANALYSIS : 66.9%, 3.0, 103.4ms, 0 | 20260508 06:33:35.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.563 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:33:35.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:35.563 | INFO | Uploaded 36GiB to GPU in 2.56878s (14.0144GiB/s) | 20260508 06:33:35.563 | INFO | Downloaded 34GiB to GPU in 2.4302s (13.9906GiB/s) | 20260508 06:33:40.709 | INFO | Analyzing chunk frames = [5986304,5986816) | 20260508 06:33:40.709 | INFO | MemInfo: { | 20260508 06:33:40.709 | INFO | "gpu" : " 17 GB", | 20260508 06:33:40.709 | INFO | "huge" : "", | 20260508 06:33:40.709 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:40.709 | INFO | "malloc" : " 14 GB", | 20260508 06:33:40.709 | INFO | "pinned" : " 29 GB" | 20260508 06:33:40.709 | INFO | } | 20260508 06:33:40.709 | INFO | Perf report for the last 5.14668 seconds: | 20260508 06:33:40.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:40.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.709 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 06:33:40.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.709 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:33:40.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:40.710 | INFO | Uploaded 36GiB to GPU in 2.29028s (15.7186GiB/s) | 20260508 06:33:40.710 | INFO | Downloaded 34GiB to GPU in 2.4899s (13.6552GiB/s) | 20260508 06:33:45.848 | INFO | Analyzing chunk frames = [5986816,5987328) | 20260508 06:33:45.848 | INFO | MemInfo: { | 20260508 06:33:45.848 | INFO | "gpu" : " 17 GB", | 20260508 06:33:45.848 | INFO | "huge" : "", | 20260508 06:33:45.848 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:45.848 | INFO | "malloc" : " 14 GB", | 20260508 06:33:45.848 | INFO | "pinned" : " 29 GB" | 20260508 06:33:45.848 | INFO | } | 20260508 06:33:45.848 | INFO | Perf report for the last 5.13872 seconds: | 20260508 06:33:45.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:45.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.848 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 06:33:45.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.848 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:33:45.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:45.849 | INFO | Uploaded 36GiB to GPU in 2.48566s (14.4831GiB/s) | 20260508 06:33:45.849 | INFO | Downloaded 34GiB to GPU in 2.43625s (13.9559GiB/s) | 20260508 06:33:50.988 | INFO | Analyzing chunk frames = [5987328,5987840) | 20260508 06:33:50.988 | INFO | MemInfo: { | 20260508 06:33:50.988 | INFO | "gpu" : " 17 GB", | 20260508 06:33:50.988 | INFO | "huge" : "", | 20260508 06:33:50.988 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:50.988 | INFO | "malloc" : " 14 GB", | 20260508 06:33:50.988 | INFO | "pinned" : " 29 GB" | 20260508 06:33:50.988 | INFO | } | 20260508 06:33:50.988 | INFO | Perf report for the last 5.14024 seconds: | 20260508 06:33:50.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:50.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.988 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:33:50.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.988 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 06:33:50.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:50.989 | INFO | Uploaded 36GiB to GPU in 2.49995s (14.4003GiB/s) | 20260508 06:33:50.989 | INFO | Downloaded 34GiB to GPU in 2.42948s (13.9948GiB/s) | 20260508 06:33:56.122 | INFO | Analyzing chunk frames = [5987840,5988352) | 20260508 06:33:56.122 | INFO | MemInfo: { | 20260508 06:33:56.122 | INFO | "gpu" : " 17 GB", | 20260508 06:33:56.122 | INFO | "huge" : "", | 20260508 06:33:56.122 | INFO | "huge_shared" : " 29 GB", | 20260508 06:33:56.122 | INFO | "malloc" : " 14 GB", | 20260508 06:33:56.122 | INFO | "pinned" : " 29 GB" | 20260508 06:33:56.122 | INFO | } | 20260508 06:33:56.122 | INFO | Perf report for the last 5.13389 seconds: | 20260508 06:33:56.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:33:56.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:33:56.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260508 06:33:56.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:33:56.122 | INFO | Uploaded 36GiB to GPU in 2.45498s (14.6641GiB/s) | 20260508 06:33:56.122 | INFO | Downloaded 34GiB to GPU in 2.42956s (13.9943GiB/s) | 20260508 06:33:56.123 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:33:56.123 | INFO | Total mem usage: 16.7 GiB | 20260508 06:33:56.123 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:33:56.123 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:33:56.123 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:33:56.123 | INFO | High Water Breakdown: | 20260508 06:33:56.123 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:33:56.123 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:33:56.123 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:33:56.123 | INFO | ------------------------------------------ | 20260508 06:33:56.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:33:56.123 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:33:56.123 | INFO | Total mem usage: 26.5 GiB | 20260508 06:33:56.123 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:33:56.123 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:33:56.123 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:33:56.123 | INFO | High Water Breakdown: | 20260508 06:33:56.123 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:33:56.123 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:33:56.123 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:33:56.123 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:33:56.123 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:33:56.123 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:33:56.123 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:33:56.123 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:33:56.123 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:33:56.123 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:33:56.123 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:33:56.123 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:33:56.123 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:33:56.123 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:33:56.123 | INFO | ------------------------------------------ | 20260508 06:33:56.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:33:56.123 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:33:56.123 | INFO | Total mem usage: 27.8 GiB | 20260508 06:33:56.123 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:33:56.123 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:33:56.123 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:33:56.123 | INFO | High Water Breakdown: | 20260508 06:33:56.123 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:33:56.123 | INFO | ------------------------------------------ | 20260508 06:33:56.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:33:56.123 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:33:56.123 | INFO | Total mem usage: 15.7 GiB | 20260508 06:33:56.123 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:33:56.123 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:33:56.123 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:33:56.123 | INFO | High Water Breakdown: | 20260508 06:33:56.123 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:33:56.123 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:33:56.123 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:33:56.123 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:33:56.123 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:33:56.123 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:33:56.123 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:33:56.123 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:33:56.123 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:33:56.123 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:33:56.123 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:33:56.123 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:33:56.123 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:33:56.123 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:33:56.123 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:33:56.123 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:33:56.123 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:33:56.123 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:33:56.123 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:33:56.123 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:33:56.123 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:33:56.123 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:33:56.123 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:33:56.123 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:33:56.123 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:33:56.123 | INFO | ------------------------------------------ | 20260508 06:33:56.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:33:56.124 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:33:56.124 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:33:56.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:33:56.124 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 6.65e-01 | 20260508 06:33:56.124 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.16e-02 / 5.19e-03 | 20260508 06:33:56.124 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.42e+00, 5.93e-01 / 2.20e-01 | 20260508 06:33:56.124 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.18e-01 | 20260508 06:33:56.124 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.53e+01, 3.75e+00 / 1.03e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.81e+00 / 1.31e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.38e+00 / 2.59e+00, 4.09e-01 / 2.45e-01 | 20260508 06:33:56.124 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.21e+00, 2.57e-01 / 1.11e-01 | 20260508 06:33:56.124 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.54e-01, 5.05e-02 / 2.35e-02 | 20260508 06:33:56.124 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:33:56.124 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:33:56.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:33:56.124 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 24.45%, Upload -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 2.37%, LockUpload -- 2.52e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 2.47%, LockDownload -- 2.62e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 15.60%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:33:56.124 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 06:33:56.219 | INFO | Starting a round of Baz IO | 20260508 06:33:59.649 | INFO | Populating RTMetrics report | 20260508 06:33:59.649 | INFO | RTMetrics report populated | 20260508 06:33:59.649 | INFO | Opening tmp RTMetrics json file | 20260508 06:33:59.662 | INFO | Serializing RTMetrics to json | 20260508 06:33:59.663 | INFO | Serialization done, swapping tmp file now... | 20260508 06:33:59.664 | INFO | RTMetrics done | 20260508 06:33:59.723 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.504 seconds | 20260508 06:33:59.723 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.00% / 59.32% | 20260508 06:33:59.724 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.391754% / 0% / 8.74201e-05% | 20260508 06:33:59.724 | INFO | Baz IO overall: | 20260508 06:33:59.724 | INFO | BytesWritten : 2654124858004 | 20260508 06:33:59.724 | INFO | NumEvents : 1279562701341 | 20260508 06:34:01.259 | INFO | Analyzing chunk frames = [5988352,5988864) | 20260508 06:34:01.259 | INFO | MemInfo: { | 20260508 06:34:01.259 | INFO | "gpu" : " 17 GB", | 20260508 06:34:01.259 | INFO | "huge" : "", | 20260508 06:34:01.259 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:01.259 | INFO | "malloc" : " 14 GB", | 20260508 06:34:01.259 | INFO | "pinned" : " 29 GB" | 20260508 06:34:01.259 | INFO | } | 20260508 06:34:01.259 | INFO | Perf report for the last 5.13672 seconds: | 20260508 06:34:01.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:01.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:01.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:01.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:01.259 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 06:34:01.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:01.259 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:34:01.259 | INFO | PRE_HQ : 65.7%, 1.6, 55.1ms, 0 | 20260508 06:34:01.259 | INFO | BAZWRITER : 67.6%, 2.6, 91.1ms, 0 | 20260508 06:34:01.259 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:34:01.259 | INFO | Uploaded 36GiB to GPU in 2.58714s (13.915GiB/s) | 20260508 06:34:01.259 | INFO | Downloaded 38GiB to GPU in 2.69694s (14.0901GiB/s) | 20260508 06:34:06.407 | INFO | Analyzing chunk frames = [5988864,5989376) | 20260508 06:34:06.407 | INFO | MemInfo: { | 20260508 06:34:06.407 | INFO | "gpu" : " 17 GB", | 20260508 06:34:06.407 | INFO | "huge" : "", | 20260508 06:34:06.407 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:06.407 | INFO | "malloc" : " 14 GB", | 20260508 06:34:06.407 | INFO | "pinned" : " 29 GB" | 20260508 06:34:06.407 | INFO | } | 20260508 06:34:06.407 | INFO | Perf report for the last 5.14814 seconds: | 20260508 06:34:06.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:06.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | ANALYSIS : 67.0%, 3.0, 104.7ms, 0 | 20260508 06:34:06.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:34:06.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:06.407 | INFO | Uploaded 36GiB to GPU in 2.50953s (14.3453GiB/s) | 20260508 06:34:06.407 | INFO | Downloaded 34GiB to GPU in 2.43152s (13.983GiB/s) | 20260508 06:34:11.542 | INFO | Analyzing chunk frames = [5989376,5989888) | 20260508 06:34:11.542 | INFO | MemInfo: { | 20260508 06:34:11.542 | INFO | "gpu" : " 17 GB", | 20260508 06:34:11.542 | INFO | "huge" : "", | 20260508 06:34:11.542 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:11.542 | INFO | "malloc" : " 14 GB", | 20260508 06:34:11.542 | INFO | "pinned" : " 29 GB" | 20260508 06:34:11.542 | INFO | } | 20260508 06:34:11.542 | INFO | Perf report for the last 5.13534 seconds: | 20260508 06:34:11.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:11.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 06:34:11.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:34:11.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:11.542 | INFO | Uploaded 36GiB to GPU in 2.58021s (13.9523GiB/s) | 20260508 06:34:11.543 | INFO | Downloaded 34GiB to GPU in 2.4339s (13.9694GiB/s) | 20260508 06:34:16.678 | INFO | Analyzing chunk frames = [5989888,5990400) | 20260508 06:34:16.678 | INFO | MemInfo: { | 20260508 06:34:16.678 | INFO | "gpu" : " 17 GB", | 20260508 06:34:16.678 | INFO | "huge" : "", | 20260508 06:34:16.678 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:16.678 | INFO | "malloc" : " 14 GB", | 20260508 06:34:16.678 | INFO | "pinned" : " 29 GB" | 20260508 06:34:16.678 | INFO | } | 20260508 06:34:16.678 | INFO | Perf report for the last 5.13568 seconds: | 20260508 06:34:16.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:16.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:34:16.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:34:16.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:16.678 | INFO | Uploaded 36GiB to GPU in 2.57044s (14.0054GiB/s) | 20260508 06:34:16.678 | INFO | Downloaded 34GiB to GPU in 2.39084s (14.221GiB/s) | 20260508 06:34:21.816 | INFO | Analyzing chunk frames = [5990400,5990912) | 20260508 06:34:21.816 | INFO | MemInfo: { | 20260508 06:34:21.816 | INFO | "gpu" : " 17 GB", | 20260508 06:34:21.816 | INFO | "huge" : "", | 20260508 06:34:21.816 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:21.816 | INFO | "malloc" : " 14 GB", | 20260508 06:34:21.816 | INFO | "pinned" : " 29 GB" | 20260508 06:34:21.816 | INFO | } | 20260508 06:34:21.816 | INFO | Perf report for the last 5.13791 seconds: | 20260508 06:34:21.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:21.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:34:21.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:34:21.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:21.816 | INFO | Uploaded 36GiB to GPU in 2.43994s (14.7545GiB/s) | 20260508 06:34:21.816 | INFO | Downloaded 34GiB to GPU in 2.51572s (13.515GiB/s) | 20260508 06:34:26.961 | INFO | Analyzing chunk frames = [5990912,5991424) | 20260508 06:34:26.961 | INFO | MemInfo: { | 20260508 06:34:26.961 | INFO | "gpu" : " 17 GB", | 20260508 06:34:26.961 | INFO | "huge" : "", | 20260508 06:34:26.962 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:26.962 | INFO | "malloc" : " 14 GB", | 20260508 06:34:26.962 | INFO | "pinned" : " 29 GB" | 20260508 06:34:26.962 | INFO | } | 20260508 06:34:26.962 | INFO | Perf report for the last 5.14585 seconds: | 20260508 06:34:26.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:26.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:34:26.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:34:26.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:26.962 | INFO | Uploaded 36GiB to GPU in 2.4755s (14.5425GiB/s) | 20260508 06:34:26.962 | INFO | Downloaded 34GiB to GPU in 2.41403s (14.0844GiB/s) | 20260508 06:34:32.105 | INFO | Analyzing chunk frames = [5991424,5991936) | 20260508 06:34:32.105 | INFO | MemInfo: { | 20260508 06:34:32.105 | INFO | "gpu" : " 17 GB", | 20260508 06:34:32.105 | INFO | "huge" : "", | 20260508 06:34:32.105 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:32.105 | INFO | "malloc" : " 14 GB", | 20260508 06:34:32.105 | INFO | "pinned" : " 29 GB" | 20260508 06:34:32.105 | INFO | } | 20260508 06:34:32.105 | INFO | Perf report for the last 5.14363 seconds: | 20260508 06:34:32.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:32.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.105 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 06:34:32.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.106 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 06:34:32.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:32.106 | INFO | Uploaded 36GiB to GPU in 2.50799s (14.3541GiB/s) | 20260508 06:34:32.106 | INFO | Downloaded 34GiB to GPU in 2.44344s (13.9148GiB/s) | 20260508 06:34:37.242 | INFO | Analyzing chunk frames = [5991936,5992448) | 20260508 06:34:37.242 | INFO | MemInfo: { | 20260508 06:34:37.242 | INFO | "gpu" : " 17 GB", | 20260508 06:34:37.242 | INFO | "huge" : "", | 20260508 06:34:37.242 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:37.242 | INFO | "malloc" : " 14 GB", | 20260508 06:34:37.242 | INFO | "pinned" : " 29 GB" | 20260508 06:34:37.242 | INFO | } | 20260508 06:34:37.243 | INFO | Perf report for the last 5.13713 seconds: | 20260508 06:34:37.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:37.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 06:34:37.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:34:37.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:37.243 | INFO | Uploaded 36GiB to GPU in 2.50988s (14.3433GiB/s) | 20260508 06:34:37.243 | INFO | Downloaded 34GiB to GPU in 2.44228s (13.9214GiB/s) | 20260508 06:34:37.339 | INFO | Starting a round of Baz IO | 20260508 06:34:40.771 | INFO | Populating RTMetrics report | 20260508 06:34:40.771 | INFO | RTMetrics report populated | 20260508 06:34:40.771 | INFO | Opening tmp RTMetrics json file | 20260508 06:34:40.779 | INFO | Serializing RTMetrics to json | 20260508 06:34:40.780 | INFO | Serialization done, swapping tmp file now... | 20260508 06:34:40.781 | INFO | RTMetrics done | 20260508 06:34:40.841 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.502 seconds | 20260508 06:34:40.841 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.00% / 59.32% | 20260508 06:34:40.841 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.391479% / 1.58946e-05% / 7.54992e-05% | 20260508 06:34:40.841 | INFO | Baz IO overall: | 20260508 06:34:40.841 | INFO | BytesWritten : 2655787145829 | 20260508 06:34:40.841 | INFO | NumEvents : 1280353848822 | 20260508 06:34:42.375 | INFO | Analyzing chunk frames = [5992448,5992960) | 20260508 06:34:42.375 | INFO | MemInfo: { | 20260508 06:34:42.375 | INFO | "gpu" : " 17 GB", | 20260508 06:34:42.375 | INFO | "huge" : "", | 20260508 06:34:42.375 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:42.375 | INFO | "malloc" : " 14 GB", | 20260508 06:34:42.375 | INFO | "pinned" : " 29 GB" | 20260508 06:34:42.375 | INFO | } | 20260508 06:34:42.375 | INFO | Perf report for the last 5.13256 seconds: | 20260508 06:34:42.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:42.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:42.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:42.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:42.375 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:34:42.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:42.375 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 06:34:42.375 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 06:34:42.375 | INFO | BAZWRITER : 67.6%, 2.6, 91.6ms, 0 | 20260508 06:34:42.375 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:34:42.376 | INFO | Uploaded 36GiB to GPU in 2.57968s (13.9552GiB/s) | 20260508 06:34:42.376 | INFO | Downloaded 38GiB to GPU in 2.70259s (14.0606GiB/s) | 20260508 06:34:47.518 | INFO | Analyzing chunk frames = [5992960,5993472) | 20260508 06:34:47.518 | INFO | MemInfo: { | 20260508 06:34:47.518 | INFO | "gpu" : " 17 GB", | 20260508 06:34:47.518 | INFO | "huge" : "", | 20260508 06:34:47.518 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:47.518 | INFO | "malloc" : " 14 GB", | 20260508 06:34:47.518 | INFO | "pinned" : " 29 GB" | 20260508 06:34:47.518 | INFO | } | 20260508 06:34:47.518 | INFO | Perf report for the last 5.14268 seconds: | 20260508 06:34:47.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:47.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:34:47.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:34:47.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:47.518 | INFO | Uploaded 36GiB to GPU in 2.5069s (14.3604GiB/s) | 20260508 06:34:47.518 | INFO | Downloaded 34GiB to GPU in 2.45307s (13.8602GiB/s) | 20260508 06:34:52.657 | INFO | Analyzing chunk frames = [5993472,5993984) | 20260508 06:34:52.657 | INFO | MemInfo: { | 20260508 06:34:52.657 | INFO | "gpu" : " 17 GB", | 20260508 06:34:52.657 | INFO | "huge" : "", | 20260508 06:34:52.657 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:52.657 | INFO | "malloc" : " 14 GB", | 20260508 06:34:52.657 | INFO | "pinned" : " 29 GB" | 20260508 06:34:52.657 | INFO | } | 20260508 06:34:52.657 | INFO | Perf report for the last 5.13892 seconds: | 20260508 06:34:52.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:52.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 06:34:52.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:34:52.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:52.657 | INFO | Uploaded 36GiB to GPU in 2.55566s (14.0864GiB/s) | 20260508 06:34:52.657 | INFO | Downloaded 34GiB to GPU in 2.43329s (13.9728GiB/s) | 20260508 06:34:57.799 | INFO | Analyzing chunk frames = [5993984,5994496) | 20260508 06:34:57.799 | INFO | MemInfo: { | 20260508 06:34:57.799 | INFO | "gpu" : " 17 GB", | 20260508 06:34:57.799 | INFO | "huge" : "", | 20260508 06:34:57.799 | INFO | "huge_shared" : " 29 GB", | 20260508 06:34:57.799 | INFO | "malloc" : " 14 GB", | 20260508 06:34:57.799 | INFO | "pinned" : " 29 GB" | 20260508 06:34:57.799 | INFO | } | 20260508 06:34:57.799 | INFO | Perf report for the last 5.14244 seconds: | 20260508 06:34:57.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:34:57.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 06:34:57.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:34:57.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:34:57.799 | INFO | Uploaded 36GiB to GPU in 2.56366s (14.0424GiB/s) | 20260508 06:34:57.800 | INFO | Downloaded 34GiB to GPU in 2.43452s (13.9658GiB/s) | 20260508 06:35:02.934 | INFO | Analyzing chunk frames = [5994496,5995008) | 20260508 06:35:02.934 | INFO | MemInfo: { | 20260508 06:35:02.934 | INFO | "gpu" : " 17 GB", | 20260508 06:35:02.934 | INFO | "huge" : "", | 20260508 06:35:02.934 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:02.934 | INFO | "malloc" : " 14 GB", | 20260508 06:35:02.934 | INFO | "pinned" : " 29 GB" | 20260508 06:35:02.934 | INFO | } | 20260508 06:35:02.934 | INFO | Perf report for the last 5.1349 seconds: | 20260508 06:35:02.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:02.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:35:02.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:35:02.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:02.934 | INFO | Uploaded 36GiB to GPU in 2.38224s (15.1118GiB/s) | 20260508 06:35:02.934 | INFO | Downloaded 34GiB to GPU in 2.52696s (13.4549GiB/s) | 20260508 06:35:08.072 | INFO | Analyzing chunk frames = [5995008,5995520) | 20260508 06:35:08.072 | INFO | MemInfo: { | 20260508 06:35:08.072 | INFO | "gpu" : " 17 GB", | 20260508 06:35:08.072 | INFO | "huge" : "", | 20260508 06:35:08.072 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:08.072 | INFO | "malloc" : " 14 GB", | 20260508 06:35:08.072 | INFO | "pinned" : " 29 GB" | 20260508 06:35:08.073 | INFO | } | 20260508 06:35:08.073 | INFO | Perf report for the last 5.13847 seconds: | 20260508 06:35:08.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:08.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.073 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 06:35:08.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.073 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260508 06:35:08.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:08.074 | INFO | Uploaded 36GiB to GPU in 2.36446s (15.2254GiB/s) | 20260508 06:35:08.074 | INFO | Downloaded 34GiB to GPU in 2.45754s (13.835GiB/s) | 20260508 06:35:13.215 | INFO | Analyzing chunk frames = [5995520,5996032) | 20260508 06:35:13.215 | INFO | MemInfo: { | 20260508 06:35:13.215 | INFO | "gpu" : " 17 GB", | 20260508 06:35:13.215 | INFO | "huge" : "", | 20260508 06:35:13.215 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:13.216 | INFO | "malloc" : " 14 GB", | 20260508 06:35:13.216 | INFO | "pinned" : " 29 GB" | 20260508 06:35:13.216 | INFO | } | 20260508 06:35:13.216 | INFO | Perf report for the last 5.14306 seconds: | 20260508 06:35:13.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:13.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 06:35:13.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260508 06:35:13.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:13.216 | INFO | Uploaded 36GiB to GPU in 2.43741s (14.7698GiB/s) | 20260508 06:35:13.216 | INFO | Downloaded 34GiB to GPU in 2.45171s (13.8678GiB/s) | 20260508 06:35:18.356 | INFO | Analyzing chunk frames = [5996032,5996544) | 20260508 06:35:18.356 | INFO | MemInfo: { | 20260508 06:35:18.356 | INFO | "gpu" : " 17 GB", | 20260508 06:35:18.356 | INFO | "huge" : "", | 20260508 06:35:18.356 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:18.356 | INFO | "malloc" : " 14 GB", | 20260508 06:35:18.356 | INFO | "pinned" : " 29 GB" | 20260508 06:35:18.356 | INFO | } | 20260508 06:35:18.356 | INFO | Perf report for the last 5.14072 seconds: | 20260508 06:35:18.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:18.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.356 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 06:35:18.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.357 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 06:35:18.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:18.357 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:35:18.357 | INFO | Total mem usage: 16.7 GiB | 20260508 06:35:18.357 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:35:18.357 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:35:18.357 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:35:18.357 | INFO | High Water Breakdown: | 20260508 06:35:18.357 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:35:18.357 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:35:18.357 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:35:18.357 | INFO | ------------------------------------------ | 20260508 06:35:18.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:35:18.357 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:35:18.357 | INFO | Total mem usage: 26.5 GiB | 20260508 06:35:18.357 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:35:18.357 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 06:35:18.357 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 06:35:18.357 | INFO | High Water Breakdown: | 20260508 06:35:18.357 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:35:18.357 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:35:18.357 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:35:18.357 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:35:18.357 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:35:18.357 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:35:18.357 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:35:18.357 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:35:18.357 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:35:18.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:35:18.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:35:18.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:35:18.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:35:18.357 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:35:18.357 | INFO | ------------------------------------------ | 20260508 06:35:18.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:35:18.357 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:35:18.357 | INFO | Uploaded 36GiB to GPU in 2.42764s (14.8292GiB/s) | 20260508 06:35:18.357 | INFO | Downloaded 34GiB to GPU in 2.47248s (13.7514GiB/s) | 20260508 06:35:18.357 | INFO | Total mem usage: 27.8 GiB | 20260508 06:35:18.358 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:35:18.358 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:35:18.358 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:35:18.358 | INFO | High Water Breakdown: | 20260508 06:35:18.358 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:35:18.358 | INFO | ------------------------------------------ | 20260508 06:35:18.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:35:18.358 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:35:18.358 | INFO | Total mem usage: 15.7 GiB | 20260508 06:35:18.358 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:35:18.358 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:35:18.358 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:35:18.358 | INFO | High Water Breakdown: | 20260508 06:35:18.358 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:35:18.358 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:35:18.358 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:35:18.358 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:35:18.358 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:35:18.358 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:35:18.358 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:35:18.358 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:35:18.358 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:35:18.358 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:35:18.358 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:35:18.358 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:35:18.358 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:35:18.358 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:35:18.358 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:35:18.358 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:35:18.358 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:35:18.358 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:35:18.358 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:35:18.358 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:35:18.358 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:35:18.358 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:35:18.358 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:35:18.358 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:35:18.358 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:35:18.358 | INFO | ------------------------------------------ | 20260508 06:35:18.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:35:18.358 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:35:18.358 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:35:18.358 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:35:18.358 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 8.24e-01 | 20260508 06:35:18.358 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.17e-02 / 7.06e-03 | 20260508 06:35:18.358 | INFO | PerfReport: 12.10%, Baseline -- 3.96e+00 / 4.39e+00, 5.77e-01 / 2.34e-01 | 20260508 06:35:18.358 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.43e+00, 2.05e-01 / 1.40e-01 | 20260508 06:35:18.358 | INFO | PerfReport: 15.86%, DME -- 4.15e+01 / 4.48e+01, 3.75e+00 / 1.34e+00 | 20260508 06:35:18.358 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.85e+00 / 1.55e+00 | 20260508 06:35:18.358 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.37e+00 / 2.51e+00, 4.12e-01 / 2.72e-01 | 20260508 06:35:18.358 | INFO | PerfReport: 3.13%, Metrics -- 1.03e+00 / 1.21e+00, 2.53e-01 / 1.42e-01 | 20260508 06:35:18.358 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.51e-01, 4.98e-02 / 2.58e-02 | 20260508 06:35:18.358 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:35:18.358 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:35:18.358 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:35:18.359 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 24.54%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 24.36%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 2.34%, LockUpload -- 2.48e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 2.47%, LockDownload -- 2.62e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 15.45%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:35:18.359 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260508 06:35:18.469 | INFO | Starting a round of Baz IO | 20260508 06:35:21.893 | INFO | Populating RTMetrics report | 20260508 06:35:21.893 | INFO | RTMetrics report populated | 20260508 06:35:21.893 | INFO | Opening tmp RTMetrics json file | 20260508 06:35:21.894 | INFO | Serializing RTMetrics to json | 20260508 06:35:21.895 | INFO | Serialization done, swapping tmp file now... | 20260508 06:35:21.895 | INFO | RTMetrics done | 20260508 06:35:21.963 | INFO | Finished a round of Baz IO: Wrote 1.547 GiB in 3.493 seconds | 20260508 06:35:21.963 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.00% / 59.32% | 20260508 06:35:21.963 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.391237% / 0% / 5.96046e-05% | 20260508 06:35:21.963 | INFO | Baz IO overall: | 20260508 06:35:21.963 | INFO | BytesWritten : 2657448371604 | 20260508 06:35:21.963 | INFO | NumEvents : 1281144157811 | 20260508 06:35:23.495 | INFO | Analyzing chunk frames = [5996544,5997056) | 20260508 06:35:23.495 | INFO | MemInfo: { | 20260508 06:35:23.495 | INFO | "gpu" : " 17 GB", | 20260508 06:35:23.495 | INFO | "huge" : "", | 20260508 06:35:23.495 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:23.495 | INFO | "malloc" : " 14 GB", | 20260508 06:35:23.495 | INFO | "pinned" : " 29 GB" | 20260508 06:35:23.495 | INFO | } | 20260508 06:35:23.495 | INFO | Perf report for the last 5.13883 seconds: | 20260508 06:35:23.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:23.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:23.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:23.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:23.495 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260508 06:35:23.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:23.495 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260508 06:35:23.495 | INFO | PRE_HQ : 66.4%, 1.8, 60.8ms, 0 | 20260508 06:35:23.495 | INFO | BAZWRITER : 67.3%, 2.8, 96.5ms, 0 | 20260508 06:35:23.495 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:35:23.496 | INFO | Uploaded 36GiB to GPU in 2.57075s (14.0037GiB/s) | 20260508 06:35:23.496 | INFO | Downloaded 38GiB to GPU in 2.70695s (14.0379GiB/s) | 20260508 06:35:28.628 | INFO | Analyzing chunk frames = [5997056,5997568) | 20260508 06:35:28.628 | INFO | MemInfo: { | 20260508 06:35:28.628 | INFO | "gpu" : " 17 GB", | 20260508 06:35:28.628 | INFO | "huge" : "", | 20260508 06:35:28.628 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:28.628 | INFO | "malloc" : " 14 GB", | 20260508 06:35:28.628 | INFO | "pinned" : " 29 GB" | 20260508 06:35:28.628 | INFO | } | 20260508 06:35:28.628 | INFO | Perf report for the last 5.13321 seconds: | 20260508 06:35:28.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:28.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 06:35:28.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:35:28.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:28.629 | INFO | Uploaded 36GiB to GPU in 2.44194s (14.7424GiB/s) | 20260508 06:35:28.629 | INFO | Downloaded 34GiB to GPU in 2.45205s (13.866GiB/s) | 20260508 06:35:33.769 | INFO | Analyzing chunk frames = [5997568,5998080) | 20260508 06:35:33.769 | INFO | MemInfo: { | 20260508 06:35:33.769 | INFO | "gpu" : " 17 GB", | 20260508 06:35:33.769 | INFO | "huge" : "", | 20260508 06:35:33.769 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:33.770 | INFO | "malloc" : " 14 GB", | 20260508 06:35:33.770 | INFO | "pinned" : " 29 GB" | 20260508 06:35:33.770 | INFO | } | 20260508 06:35:33.770 | INFO | Perf report for the last 5.14121 seconds: | 20260508 06:35:33.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:33.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:35:33.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:35:33.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:33.770 | INFO | Uploaded 36GiB to GPU in 2.52261s (14.2709GiB/s) | 20260508 06:35:33.770 | INFO | Downloaded 34GiB to GPU in 2.45215s (13.8654GiB/s) | 20260508 06:35:38.913 | INFO | Analyzing chunk frames = [5998080,5998592) | 20260508 06:35:38.913 | INFO | MemInfo: { | 20260508 06:35:38.913 | INFO | "gpu" : " 17 GB", | 20260508 06:35:38.913 | INFO | "huge" : "", | 20260508 06:35:38.913 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:38.913 | INFO | "malloc" : " 14 GB", | 20260508 06:35:38.913 | INFO | "pinned" : " 29 GB" | 20260508 06:35:38.913 | INFO | } | 20260508 06:35:38.913 | INFO | Perf report for the last 5.14343 seconds: | 20260508 06:35:38.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:38.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.913 | INFO | ANALYSIS : 66.5%, 3.0, 102.9ms, 0 | 20260508 06:35:38.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.913 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:35:38.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:38.914 | INFO | Uploaded 36GiB to GPU in 2.49591s (14.4236GiB/s) | 20260508 06:35:38.914 | INFO | Downloaded 34GiB to GPU in 2.41021s (14.1067GiB/s) | 20260508 06:35:44.055 | INFO | Analyzing chunk frames = [5998592,5999104) | 20260508 06:35:44.055 | INFO | MemInfo: { | 20260508 06:35:44.055 | INFO | "gpu" : " 17 GB", | 20260508 06:35:44.055 | INFO | "huge" : "", | 20260508 06:35:44.055 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:44.055 | INFO | "malloc" : " 14 GB", | 20260508 06:35:44.055 | INFO | "pinned" : " 29 GB" | 20260508 06:35:44.055 | INFO | } | 20260508 06:35:44.055 | INFO | Perf report for the last 5.14223 seconds: | 20260508 06:35:44.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:44.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.055 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 06:35:44.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.055 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:35:44.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:44.056 | INFO | Uploaded 36GiB to GPU in 2.3762s (15.1502GiB/s) | 20260508 06:35:44.056 | INFO | Downloaded 34GiB to GPU in 2.53874s (13.3924GiB/s) | 20260508 06:35:49.187 | INFO | Analyzing chunk frames = [5999104,5999616) | 20260508 06:35:49.187 | INFO | MemInfo: { | 20260508 06:35:49.187 | INFO | "gpu" : " 17 GB", | 20260508 06:35:49.187 | INFO | "huge" : "", | 20260508 06:35:49.187 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:49.187 | INFO | "malloc" : " 14 GB", | 20260508 06:35:49.187 | INFO | "pinned" : " 29 GB" | 20260508 06:35:49.187 | INFO | } | 20260508 06:35:49.187 | INFO | Perf report for the last 5.13194 seconds: | 20260508 06:35:49.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:49.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.187 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 06:35:49.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.187 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:35:49.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:49.188 | INFO | Uploaded 36GiB to GPU in 2.35844s (15.2643GiB/s) | 20260508 06:35:49.188 | INFO | Downloaded 34GiB to GPU in 2.43487s (13.9638GiB/s) | 20260508 06:35:54.324 | INFO | Analyzing chunk frames = [5999616,6000128) | 20260508 06:35:54.324 | INFO | MemInfo: { | 20260508 06:35:54.324 | INFO | "gpu" : " 17 GB", | 20260508 06:35:54.324 | INFO | "huge" : "", | 20260508 06:35:54.324 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:54.324 | INFO | "malloc" : " 14 GB", | 20260508 06:35:54.324 | INFO | "pinned" : " 29 GB" | 20260508 06:35:54.324 | INFO | } | 20260508 06:35:54.325 | INFO | Perf report for the last 5.13725 seconds: | 20260508 06:35:54.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:54.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:35:54.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:35:54.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:54.325 | INFO | Uploaded 36GiB to GPU in 2.45315s (14.675GiB/s) | 20260508 06:35:54.325 | INFO | Downloaded 34GiB to GPU in 2.43639s (13.9551GiB/s) | 20260508 06:35:59.471 | INFO | Analyzing chunk frames = [6000128,6000640) | 20260508 06:35:59.471 | INFO | MemInfo: { | 20260508 06:35:59.471 | INFO | "gpu" : " 17 GB", | 20260508 06:35:59.471 | INFO | "huge" : "", | 20260508 06:35:59.471 | INFO | "huge_shared" : " 29 GB", | 20260508 06:35:59.471 | INFO | "malloc" : " 14 GB", | 20260508 06:35:59.471 | INFO | "pinned" : " 29 GB" | 20260508 06:35:59.471 | INFO | } | 20260508 06:35:59.471 | INFO | Perf report for the last 5.14633 seconds: | 20260508 06:35:59.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:35:59.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 06:35:59.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 06:35:59.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:35:59.471 | INFO | Uploaded 36GiB to GPU in 2.48923s (14.4623GiB/s) | 20260508 06:35:59.471 | INFO | Downloaded 34GiB to GPU in 2.45664s (13.84GiB/s) | 20260508 06:35:59.584 | INFO | Starting a round of Baz IO | 20260508 06:36:03.002 | INFO | Populating RTMetrics report | 20260508 06:36:03.003 | INFO | RTMetrics report populated | 20260508 06:36:03.003 | INFO | Opening tmp RTMetrics json file | 20260508 06:36:03.003 | INFO | Serializing RTMetrics to json | 20260508 06:36:03.004 | INFO | Serialization done, swapping tmp file now... | 20260508 06:36:03.005 | INFO | RTMetrics done | 20260508 06:36:03.072 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.488 seconds | 20260508 06:36:03.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.00% / 59.32% | 20260508 06:36:03.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39113% / 3.97364e-06% / 4.76837e-05% | 20260508 06:36:03.072 | INFO | Baz IO overall: | 20260508 06:36:03.072 | INFO | BytesWritten : 2659110796138 | 20260508 06:36:03.072 | INFO | NumEvents : 1281935430655 | 20260508 06:36:04.607 | INFO | Analyzing chunk frames = [6000640,6001152) | 20260508 06:36:04.607 | INFO | MemInfo: { | 20260508 06:36:04.607 | INFO | "gpu" : " 17 GB", | 20260508 06:36:04.607 | INFO | "huge" : "", | 20260508 06:36:04.607 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:04.607 | INFO | "malloc" : " 14 GB", | 20260508 06:36:04.607 | INFO | "pinned" : " 29 GB" | 20260508 06:36:04.607 | INFO | } | 20260508 06:36:04.608 | INFO | Perf report for the last 5.13653 seconds: | 20260508 06:36:04.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:04.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:04.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:04.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:04.608 | INFO | ANALYSIS : 68.4%, 3.0, 106.0ms, 0 | 20260508 06:36:04.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:04.608 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 06:36:04.608 | INFO | PRE_HQ : 66.9%, 1.8, 61.7ms, 0 | 20260508 06:36:04.608 | INFO | BAZWRITER : 67.2%, 2.8, 97.4ms, 0 | 20260508 06:36:04.608 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:36:04.608 | INFO | Uploaded 36GiB to GPU in 2.53144s (14.2212GiB/s) | 20260508 06:36:04.608 | INFO | Downloaded 38GiB to GPU in 2.70812s (14.0319GiB/s) | 20260508 06:36:09.746 | INFO | Analyzing chunk frames = [6001152,6001664) | 20260508 06:36:09.746 | INFO | MemInfo: { | 20260508 06:36:09.746 | INFO | "gpu" : " 17 GB", | 20260508 06:36:09.746 | INFO | "huge" : "", | 20260508 06:36:09.747 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:09.747 | INFO | "malloc" : " 14 GB", | 20260508 06:36:09.747 | INFO | "pinned" : " 29 GB" | 20260508 06:36:09.747 | INFO | } | 20260508 06:36:09.747 | INFO | Perf report for the last 5.13907 seconds: | 20260508 06:36:09.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:09.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:36:09.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:36:09.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:09.747 | INFO | Uploaded 36GiB to GPU in 2.4921s (14.4457GiB/s) | 20260508 06:36:09.747 | INFO | Downloaded 34GiB to GPU in 2.44144s (13.9262GiB/s) | 20260508 06:36:14.883 | INFO | Analyzing chunk frames = [6001664,6002176) | 20260508 06:36:14.883 | INFO | MemInfo: { | 20260508 06:36:14.883 | INFO | "gpu" : " 17 GB", | 20260508 06:36:14.883 | INFO | "huge" : "", | 20260508 06:36:14.883 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:14.883 | INFO | "malloc" : " 14 GB", | 20260508 06:36:14.883 | INFO | "pinned" : " 29 GB" | 20260508 06:36:14.883 | INFO | } | 20260508 06:36:14.883 | INFO | Perf report for the last 5.13624 seconds: | 20260508 06:36:14.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:14.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 06:36:14.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:36:14.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:14.883 | INFO | Uploaded 36GiB to GPU in 2.55249s (14.1039GiB/s) | 20260508 06:36:14.883 | INFO | Downloaded 34GiB to GPU in 2.42583s (14.0158GiB/s) | 20260508 06:36:20.023 | INFO | Analyzing chunk frames = [6002176,6002688) | 20260508 06:36:20.023 | INFO | MemInfo: { | 20260508 06:36:20.023 | INFO | "gpu" : " 17 GB", | 20260508 06:36:20.023 | INFO | "huge" : "", | 20260508 06:36:20.023 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:20.023 | INFO | "malloc" : " 14 GB", | 20260508 06:36:20.023 | INFO | "pinned" : " 29 GB" | 20260508 06:36:20.023 | INFO | } | 20260508 06:36:20.023 | INFO | Perf report for the last 5.1406 seconds: | 20260508 06:36:20.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:20.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.023 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 06:36:20.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.024 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:36:20.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:20.024 | INFO | Uploaded 36GiB to GPU in 2.56123s (14.0558GiB/s) | 20260508 06:36:20.024 | INFO | Downloaded 34GiB to GPU in 2.41994s (14.0499GiB/s) | 20260508 06:36:25.170 | INFO | Analyzing chunk frames = [6002688,6003200) | 20260508 06:36:25.170 | INFO | MemInfo: { | 20260508 06:36:25.170 | INFO | "gpu" : " 17 GB", | 20260508 06:36:25.170 | INFO | "huge" : "", | 20260508 06:36:25.170 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:25.170 | INFO | "malloc" : " 14 GB", | 20260508 06:36:25.170 | INFO | "pinned" : " 29 GB" | 20260508 06:36:25.170 | INFO | } | 20260508 06:36:25.170 | INFO | Perf report for the last 5.14686 seconds: | 20260508 06:36:25.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:25.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.170 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 06:36:25.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.170 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:36:25.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:25.171 | INFO | Uploaded 36GiB to GPU in 2.40111s (14.9931GiB/s) | 20260508 06:36:25.171 | INFO | Downloaded 34GiB to GPU in 2.52881s (13.4451GiB/s) | 20260508 06:36:30.307 | INFO | Analyzing chunk frames = [6003200,6003712) | 20260508 06:36:30.307 | INFO | MemInfo: { | 20260508 06:36:30.307 | INFO | "gpu" : " 17 GB", | 20260508 06:36:30.307 | INFO | "huge" : "", | 20260508 06:36:30.307 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:30.307 | INFO | "malloc" : " 14 GB", | 20260508 06:36:30.307 | INFO | "pinned" : " 29 GB" | 20260508 06:36:30.307 | INFO | } | 20260508 06:36:30.307 | INFO | Perf report for the last 5.13675 seconds: | 20260508 06:36:30.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:30.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:36:30.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:36:30.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:30.307 | INFO | Uploaded 36GiB to GPU in 2.46142s (14.6257GiB/s) | 20260508 06:36:30.307 | INFO | Downloaded 34GiB to GPU in 2.44161s (13.9252GiB/s) | 20260508 06:36:35.440 | INFO | Analyzing chunk frames = [6003712,6004224) | 20260508 06:36:35.440 | INFO | MemInfo: { | 20260508 06:36:35.440 | INFO | "gpu" : " 17 GB", | 20260508 06:36:35.440 | INFO | "huge" : "", | 20260508 06:36:35.440 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:35.440 | INFO | "malloc" : " 14 GB", | 20260508 06:36:35.440 | INFO | "pinned" : " 29 GB" | 20260508 06:36:35.440 | INFO | } | 20260508 06:36:35.440 | INFO | Perf report for the last 5.13297 seconds: | 20260508 06:36:35.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:35.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.440 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 06:36:35.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.440 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 06:36:35.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:35.441 | INFO | Uploaded 36GiB to GPU in 2.406s (14.9626GiB/s) | 20260508 06:36:35.441 | INFO | Downloaded 34GiB to GPU in 2.44001s (13.9344GiB/s) | 20260508 06:36:40.578 | INFO | Analyzing chunk frames = [6004224,6004736) | 20260508 06:36:40.578 | INFO | MemInfo: { | 20260508 06:36:40.578 | INFO | "gpu" : " 17 GB", | 20260508 06:36:40.578 | INFO | "huge" : "", | 20260508 06:36:40.578 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:40.578 | INFO | "malloc" : " 14 GB", | 20260508 06:36:40.578 | INFO | "pinned" : " 29 GB" | 20260508 06:36:40.578 | INFO | } | 20260508 06:36:40.578 | INFO | Perf report for the last 5.13833 seconds: | 20260508 06:36:40.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:40.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:36:40.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 06:36:40.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:40.579 | INFO | Uploaded 36GiB to GPU in 2.49024s (14.4565GiB/s) | 20260508 06:36:40.579 | INFO | Downloaded 34GiB to GPU in 2.45541s (13.847GiB/s) | 20260508 06:36:40.579 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:36:40.579 | INFO | Total mem usage: 16.7 GiB | 20260508 06:36:40.579 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:36:40.579 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:36:40.579 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:36:40.579 | INFO | High Water Breakdown: | 20260508 06:36:40.579 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:36:40.579 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:36:40.579 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:36:40.579 | INFO | ------------------------------------------ | 20260508 06:36:40.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:36:40.579 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:36:40.579 | INFO | Total mem usage: 26.5 GiB | 20260508 06:36:40.579 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:36:40.579 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:36:40.579 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:36:40.579 | INFO | High Water Breakdown: | 20260508 06:36:40.579 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:36:40.579 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:36:40.579 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:36:40.579 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:36:40.579 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:36:40.579 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:36:40.579 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:36:40.579 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:36:40.579 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:36:40.579 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:36:40.579 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:36:40.579 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:36:40.579 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:36:40.579 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:36:40.579 | INFO | ------------------------------------------ | 20260508 06:36:40.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:36:40.579 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:36:40.579 | INFO | Total mem usage: 27.8 GiB | 20260508 06:36:40.579 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:36:40.580 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 06:36:40.580 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 06:36:40.580 | INFO | High Water Breakdown: | 20260508 06:36:40.580 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 06:36:40.580 | INFO | ------------------------------------------ | 20260508 06:36:40.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:36:40.580 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:36:40.580 | INFO | Total mem usage: 15.7 GiB | 20260508 06:36:40.580 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:36:40.580 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:36:40.580 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:36:40.580 | INFO | High Water Breakdown: | 20260508 06:36:40.580 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:36:40.580 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:36:40.580 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:36:40.580 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:36:40.580 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:36:40.580 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:36:40.580 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:36:40.580 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:36:40.580 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:36:40.580 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:36:40.580 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:36:40.580 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:36:40.580 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:36:40.580 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:36:40.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:36:40.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:36:40.580 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:36:40.580 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:36:40.580 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:36:40.580 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:36:40.580 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:36:40.580 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:36:40.580 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:36:40.580 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:36:40.580 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:36:40.580 | INFO | ------------------------------------------ | 20260508 06:36:40.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:36:40.580 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:36:40.580 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:36:40.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:36:40.580 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.41e+01 / 7.38e-01 | 20260508 06:36:40.580 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.10e-02 / 4.13e-03 | 20260508 06:36:40.580 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.37e+00, 5.84e-01 / 2.64e-01 | 20260508 06:36:40.580 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.08e-01 | 20260508 06:36:40.580 | INFO | PerfReport: 15.88%, DME -- 4.16e+01 / 4.50e+01, 3.79e+00 / 1.30e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.33e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.55e+00, 4.13e-01 / 2.20e-01 | 20260508 06:36:40.580 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.23e+00, 2.55e-01 / 1.19e-01 | 20260508 06:36:40.580 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.52e-01, 5.06e-02 / 2.11e-02 | 20260508 06:36:40.580 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:36:40.580 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:36:40.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:36:40.580 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 24.33%, Upload -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 24.42%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 30.91%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 2.13%, LockUpload -- 2.26e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 06:36:40.580 | INFO | PerfReport: 2.65%, LockDownload -- 2.81e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 06:36:40.581 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 06:36:40.581 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260508 06:36:40.694 | INFO | Starting a round of Baz IO | 20260508 06:36:44.099 | INFO | Populating RTMetrics report | 20260508 06:36:44.099 | INFO | RTMetrics report populated | 20260508 06:36:44.099 | INFO | Opening tmp RTMetrics json file | 20260508 06:36:44.100 | INFO | Serializing RTMetrics to json | 20260508 06:36:44.107 | INFO | Serialization done, swapping tmp file now... | 20260508 06:36:44.117 | INFO | RTMetrics done | 20260508 06:36:44.175 | INFO | Finished a round of Baz IO: Wrote 1.547 GiB in 3.48 seconds | 20260508 06:36:44.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.00% / 59.32% | 20260508 06:36:44.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.389647% / 0% / 5.96046e-05% | 20260508 06:36:44.175 | INFO | Baz IO overall: | 20260508 06:36:44.175 | INFO | BytesWritten : 2660771967045 | 20260508 06:36:44.175 | INFO | NumEvents : 1282726102290 | 20260508 06:36:45.720 | INFO | Analyzing chunk frames = [6004736,6005248) | 20260508 06:36:45.720 | INFO | MemInfo: { | 20260508 06:36:45.720 | INFO | "gpu" : " 17 GB", | 20260508 06:36:45.720 | INFO | "huge" : "", | 20260508 06:36:45.720 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:45.720 | INFO | "malloc" : " 14 GB", | 20260508 06:36:45.720 | INFO | "pinned" : " 29 GB" | 20260508 06:36:45.720 | INFO | } | 20260508 06:36:45.720 | INFO | Perf report for the last 5.14155 seconds: | 20260508 06:36:45.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:45.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:45.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:45.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:45.720 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 06:36:45.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:45.720 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 06:36:45.720 | INFO | PRE_HQ : 66.1%, 1.7, 59.3ms, 0 | 20260508 06:36:45.720 | INFO | BAZWRITER : 67.1%, 2.8, 95.3ms, 0 | 20260508 06:36:45.720 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:36:45.720 | INFO | Uploaded 36GiB to GPU in 2.61311s (13.7767GiB/s) | 20260508 06:36:45.720 | INFO | Downloaded 38GiB to GPU in 2.6986s (14.0814GiB/s) | 20260508 06:36:50.863 | INFO | Analyzing chunk frames = [6005248,6005760) | 20260508 06:36:50.863 | INFO | MemInfo: { | 20260508 06:36:50.863 | INFO | "gpu" : " 17 GB", | 20260508 06:36:50.863 | INFO | "huge" : "", | 20260508 06:36:50.863 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:50.863 | INFO | "malloc" : " 14 GB", | 20260508 06:36:50.863 | INFO | "pinned" : " 29 GB" | 20260508 06:36:50.863 | INFO | } | 20260508 06:36:50.863 | INFO | Perf report for the last 5.1433 seconds: | 20260508 06:36:50.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:50.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.863 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:36:50.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.863 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:36:50.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:50.864 | INFO | Uploaded 36GiB to GPU in 2.47556s (14.5421GiB/s) | 20260508 06:36:50.864 | INFO | Downloaded 34GiB to GPU in 2.43377s (13.9701GiB/s) | 20260508 06:36:56.002 | INFO | Analyzing chunk frames = [6005760,6006272) | 20260508 06:36:56.002 | INFO | MemInfo: { | 20260508 06:36:56.002 | INFO | "gpu" : " 17 GB", | 20260508 06:36:56.002 | INFO | "huge" : "", | 20260508 06:36:56.002 | INFO | "huge_shared" : " 29 GB", | 20260508 06:36:56.002 | INFO | "malloc" : " 14 GB", | 20260508 06:36:56.002 | INFO | "pinned" : " 29 GB" | 20260508 06:36:56.002 | INFO | } | 20260508 06:36:56.002 | INFO | Perf report for the last 5.13868 seconds: | 20260508 06:36:56.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:36:56.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 06:36:56.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:36:56.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:36:56.002 | INFO | Uploaded 36GiB to GPU in 2.52619s (14.2507GiB/s) | 20260508 06:36:56.002 | INFO | Downloaded 34GiB to GPU in 2.41983s (14.0506GiB/s) | 20260508 06:37:01.138 | INFO | Analyzing chunk frames = [6006272,6006784) | 20260508 06:37:01.138 | INFO | MemInfo: { | 20260508 06:37:01.138 | INFO | "gpu" : " 17 GB", | 20260508 06:37:01.138 | INFO | "huge" : "", | 20260508 06:37:01.138 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:01.138 | INFO | "malloc" : " 14 GB", | 20260508 06:37:01.138 | INFO | "pinned" : " 29 GB" | 20260508 06:37:01.138 | INFO | } | 20260508 06:37:01.138 | INFO | Perf report for the last 5.13592 seconds: | 20260508 06:37:01.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:01.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 06:37:01.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:37:01.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:01.138 | INFO | Uploaded 36GiB to GPU in 2.55428s (14.094GiB/s) | 20260508 06:37:01.138 | INFO | Downloaded 34GiB to GPU in 2.40101s (14.1607GiB/s) | 20260508 06:37:06.282 | INFO | Analyzing chunk frames = [6006784,6007296) | 20260508 06:37:06.282 | INFO | MemInfo: { | 20260508 06:37:06.282 | INFO | "gpu" : " 17 GB", | 20260508 06:37:06.282 | INFO | "huge" : "", | 20260508 06:37:06.282 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:06.282 | INFO | "malloc" : " 14 GB", | 20260508 06:37:06.282 | INFO | "pinned" : " 29 GB" | 20260508 06:37:06.282 | INFO | } | 20260508 06:37:06.282 | INFO | Perf report for the last 5.1444 seconds: | 20260508 06:37:06.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:06.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.282 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:37:06.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.282 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:37:06.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:06.283 | INFO | Uploaded 36GiB to GPU in 2.39325s (15.0423GiB/s) | 20260508 06:37:06.283 | INFO | Downloaded 34GiB to GPU in 2.52352s (13.4733GiB/s) | 20260508 06:37:11.420 | INFO | Analyzing chunk frames = [6007296,6007808) | 20260508 06:37:11.420 | INFO | MemInfo: { | 20260508 06:37:11.420 | INFO | "gpu" : " 17 GB", | 20260508 06:37:11.420 | INFO | "huge" : "", | 20260508 06:37:11.420 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:11.420 | INFO | "malloc" : " 14 GB", | 20260508 06:37:11.420 | INFO | "pinned" : " 29 GB" | 20260508 06:37:11.421 | INFO | } | 20260508 06:37:11.421 | INFO | Perf report for the last 5.13821 seconds: | 20260508 06:37:11.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:11.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 06:37:11.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:37:11.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:11.421 | INFO | Uploaded 36GiB to GPU in 2.382s (15.1133GiB/s) | 20260508 06:37:11.421 | INFO | Downloaded 34GiB to GPU in 2.43141s (13.9836GiB/s) | 20260508 06:37:16.557 | INFO | Analyzing chunk frames = [6007808,6008320) | 20260508 06:37:16.557 | INFO | MemInfo: { | 20260508 06:37:16.557 | INFO | "gpu" : " 17 GB", | 20260508 06:37:16.557 | INFO | "huge" : "", | 20260508 06:37:16.557 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:16.557 | INFO | "malloc" : " 14 GB", | 20260508 06:37:16.557 | INFO | "pinned" : " 29 GB" | 20260508 06:37:16.557 | INFO | } | 20260508 06:37:16.558 | INFO | Perf report for the last 5.13694 seconds: | 20260508 06:37:16.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:16.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:37:16.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 06:37:16.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:16.558 | INFO | Uploaded 36GiB to GPU in 2.46507s (14.6041GiB/s) | 20260508 06:37:16.558 | INFO | Downloaded 34GiB to GPU in 2.4411s (13.9281GiB/s) | 20260508 06:37:21.696 | INFO | Analyzing chunk frames = [6008320,6008832) | 20260508 06:37:21.696 | INFO | MemInfo: { | 20260508 06:37:21.696 | INFO | "gpu" : " 17 GB", | 20260508 06:37:21.696 | INFO | "huge" : "", | 20260508 06:37:21.696 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:21.696 | INFO | "malloc" : " 14 GB", | 20260508 06:37:21.696 | INFO | "pinned" : " 29 GB" | 20260508 06:37:21.696 | INFO | } | 20260508 06:37:21.696 | INFO | Perf report for the last 5.1385 seconds: | 20260508 06:37:21.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:21.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:37:21.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260508 06:37:21.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:21.696 | INFO | Uploaded 36GiB to GPU in 2.47163s (14.5653GiB/s) | 20260508 06:37:21.696 | INFO | Downloaded 34GiB to GPU in 2.45437s (13.8528GiB/s) | 20260508 06:37:21.793 | INFO | Starting a round of Baz IO | 20260508 06:37:25.204 | INFO | Populating RTMetrics report | 20260508 06:37:25.204 | INFO | RTMetrics report populated | 20260508 06:37:25.204 | INFO | Opening tmp RTMetrics json file | 20260508 06:37:25.216 | INFO | Serializing RTMetrics to json | 20260508 06:37:25.217 | INFO | Serialization done, swapping tmp file now... | 20260508 06:37:25.218 | INFO | RTMetrics done | 20260508 06:37:25.279 | INFO | Finished a round of Baz IO: Wrote 1.546 GiB in 3.485 seconds | 20260508 06:37:25.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.67% / 0.00% / 59.32% | 20260508 06:37:25.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.388126% / 3.97364e-06% / 6.35783e-05% | 20260508 06:37:25.279 | INFO | Baz IO overall: | 20260508 06:37:25.279 | INFO | BytesWritten : 2662432047301 | 20260508 06:37:25.279 | INFO | NumEvents : 1283515919169 | 20260508 06:37:26.836 | INFO | Analyzing chunk frames = [6008832,6009344) | 20260508 06:37:26.837 | INFO | MemInfo: { | 20260508 06:37:26.837 | INFO | "gpu" : " 17 GB", | 20260508 06:37:26.837 | INFO | "huge" : "", | 20260508 06:37:26.837 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:26.837 | INFO | "malloc" : " 14 GB", | 20260508 06:37:26.837 | INFO | "pinned" : " 29 GB" | 20260508 06:37:26.837 | INFO | } | 20260508 06:37:26.837 | INFO | Perf report for the last 5.14067 seconds: | 20260508 06:37:26.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:26.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:26.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:26.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:26.837 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 06:37:26.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:26.837 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:37:26.837 | INFO | PRE_HQ : 65.4%, 1.6, 55.0ms, 0 | 20260508 06:37:26.837 | INFO | BAZWRITER : 67.1%, 2.7, 91.5ms, 0 | 20260508 06:37:26.837 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:37:26.837 | INFO | Uploaded 36GiB to GPU in 2.60832s (13.802GiB/s) | 20260508 06:37:26.837 | INFO | Downloaded 38GiB to GPU in 2.68388s (14.1586GiB/s) | 20260508 06:37:31.978 | INFO | Analyzing chunk frames = [6009344,6009856) | 20260508 06:37:31.978 | INFO | MemInfo: { | 20260508 06:37:31.978 | INFO | "gpu" : " 17 GB", | 20260508 06:37:31.978 | INFO | "huge" : "", | 20260508 06:37:31.978 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:31.978 | INFO | "malloc" : " 14 GB", | 20260508 06:37:31.978 | INFO | "pinned" : " 29 GB" | 20260508 06:37:31.978 | INFO | } | 20260508 06:37:31.978 | INFO | Perf report for the last 5.14173 seconds: | 20260508 06:37:31.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:31.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:37:31.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:37:31.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:31.979 | INFO | Uploaded 36GiB to GPU in 2.49069s (14.4538GiB/s) | 20260508 06:37:31.979 | INFO | Downloaded 34GiB to GPU in 2.43872s (13.9417GiB/s) | 20260508 06:37:37.118 | INFO | Analyzing chunk frames = [6009856,6010368) | 20260508 06:37:37.118 | INFO | MemInfo: { | 20260508 06:37:37.118 | INFO | "gpu" : " 17 GB", | 20260508 06:37:37.118 | INFO | "huge" : "", | 20260508 06:37:37.118 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:37.118 | INFO | "malloc" : " 14 GB", | 20260508 06:37:37.118 | INFO | "pinned" : " 29 GB" | 20260508 06:37:37.118 | INFO | } | 20260508 06:37:37.118 | INFO | Perf report for the last 5.13987 seconds: | 20260508 06:37:37.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:37.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.118 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 06:37:37.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.118 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:37:37.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:37.119 | INFO | Uploaded 36GiB to GPU in 2.5655s (14.0324GiB/s) | 20260508 06:37:37.119 | INFO | Downloaded 34GiB to GPU in 2.429s (13.9975GiB/s) | 20260508 06:37:42.257 | INFO | Analyzing chunk frames = [6010368,6010880) | 20260508 06:37:42.257 | INFO | MemInfo: { | 20260508 06:37:42.257 | INFO | "gpu" : " 17 GB", | 20260508 06:37:42.257 | INFO | "huge" : "", | 20260508 06:37:42.257 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:42.257 | INFO | "malloc" : " 14 GB", | 20260508 06:37:42.257 | INFO | "pinned" : " 29 GB" | 20260508 06:37:42.257 | INFO | } | 20260508 06:37:42.257 | INFO | Perf report for the last 5.13901 seconds: | 20260508 06:37:42.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:42.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.258 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 06:37:42.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.258 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:37:42.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:42.258 | INFO | Uploaded 36GiB to GPU in 2.56299s (14.0461GiB/s) | 20260508 06:37:42.258 | INFO | Downloaded 34GiB to GPU in 2.39608s (14.1898GiB/s) | 20260508 06:37:47.389 | INFO | Analyzing chunk frames = [6010880,6011392) | 20260508 06:37:47.389 | INFO | MemInfo: { | 20260508 06:37:47.389 | INFO | "gpu" : " 17 GB", | 20260508 06:37:47.389 | INFO | "huge" : "", | 20260508 06:37:47.389 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:47.389 | INFO | "malloc" : " 14 GB", | 20260508 06:37:47.389 | INFO | "pinned" : " 29 GB" | 20260508 06:37:47.389 | INFO | } | 20260508 06:37:47.389 | INFO | Perf report for the last 5.13188 seconds: | 20260508 06:37:47.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:47.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.389 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 06:37:47.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.389 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:37:47.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:47.390 | INFO | Uploaded 36GiB to GPU in 2.27237s (15.8425GiB/s) | 20260508 06:37:47.390 | INFO | Downloaded 34GiB to GPU in 2.48987s (13.6553GiB/s) | 20260508 06:37:52.535 | INFO | Analyzing chunk frames = [6011392,6011904) | 20260508 06:37:52.535 | INFO | MemInfo: { | 20260508 06:37:52.535 | INFO | "gpu" : " 17 GB", | 20260508 06:37:52.535 | INFO | "huge" : "", | 20260508 06:37:52.535 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:52.536 | INFO | "malloc" : " 14 GB", | 20260508 06:37:52.536 | INFO | "pinned" : " 29 GB" | 20260508 06:37:52.536 | INFO | } | 20260508 06:37:52.536 | INFO | Perf report for the last 5.14661 seconds: | 20260508 06:37:52.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:52.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:37:52.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 06:37:52.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:52.536 | INFO | Uploaded 36GiB to GPU in 2.47721s (14.5325GiB/s) | 20260508 06:37:52.537 | INFO | Downloaded 34GiB to GPU in 2.44028s (13.9328GiB/s) | 20260508 06:37:57.673 | INFO | Analyzing chunk frames = [6011904,6012416) | 20260508 06:37:57.674 | INFO | MemInfo: { | 20260508 06:37:57.674 | INFO | "gpu" : " 17 GB", | 20260508 06:37:57.674 | INFO | "huge" : "", | 20260508 06:37:57.674 | INFO | "huge_shared" : " 29 GB", | 20260508 06:37:57.674 | INFO | "malloc" : " 14 GB", | 20260508 06:37:57.674 | INFO | "pinned" : " 29 GB" | 20260508 06:37:57.674 | INFO | } | 20260508 06:37:57.674 | INFO | Perf report for the last 5.13785 seconds: | 20260508 06:37:57.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:37:57.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:37:57.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260508 06:37:57.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:37:57.674 | INFO | Uploaded 36GiB to GPU in 2.5092s (14.3472GiB/s) | 20260508 06:37:57.674 | INFO | Downloaded 34GiB to GPU in 2.43992s (13.9349GiB/s) | 20260508 06:38:02.811 | INFO | Analyzing chunk frames = [6012416,6012928) | 20260508 06:38:02.811 | INFO | MemInfo: { | 20260508 06:38:02.811 | INFO | "gpu" : " 17 GB", | 20260508 06:38:02.811 | INFO | "huge" : "", | 20260508 06:38:02.811 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:02.811 | INFO | "malloc" : " 14 GB", | 20260508 06:38:02.811 | INFO | "pinned" : " 29 GB" | 20260508 06:38:02.811 | INFO | } | 20260508 06:38:02.811 | INFO | Perf report for the last 5.13753 seconds: | 20260508 06:38:02.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:02.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.811 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 06:38:02.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.811 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:38:02.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:02.812 | INFO | Uploaded 36GiB to GPU in 2.47343s (14.5547GiB/s) | 20260508 06:38:02.812 | INFO | Downloaded 34GiB to GPU in 2.43547s (13.9603GiB/s) | 20260508 06:38:02.812 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:38:02.812 | INFO | Total mem usage: 16.7 GiB | 20260508 06:38:02.812 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:38:02.812 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:38:02.812 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:38:02.812 | INFO | High Water Breakdown: | 20260508 06:38:02.812 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:38:02.812 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:38:02.812 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:38:02.812 | INFO | ------------------------------------------ | 20260508 06:38:02.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:38:02.812 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:38:02.812 | INFO | Total mem usage: 26.5 GiB | 20260508 06:38:02.812 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:38:02.812 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:38:02.812 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:38:02.812 | INFO | High Water Breakdown: | 20260508 06:38:02.812 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:38:02.812 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:38:02.812 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:38:02.812 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:38:02.812 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:38:02.812 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:38:02.812 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:38:02.812 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:38:02.812 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:38:02.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:38:02.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:38:02.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:38:02.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:38:02.812 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:38:02.812 | INFO | ------------------------------------------ | 20260508 06:38:02.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:38:02.812 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:38:02.812 | INFO | Total mem usage: 27.8 GiB | 20260508 06:38:02.812 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:38:02.812 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:38:02.812 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:38:02.812 | INFO | High Water Breakdown: | 20260508 06:38:02.812 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:38:02.812 | INFO | ------------------------------------------ | 20260508 06:38:02.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:38:02.812 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:38:02.812 | INFO | Total mem usage: 15.7 GiB | 20260508 06:38:02.812 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:38:02.812 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:38:02.812 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:38:02.812 | INFO | High Water Breakdown: | 20260508 06:38:02.812 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:38:02.812 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:38:02.812 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:38:02.812 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:38:02.812 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:38:02.812 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:38:02.812 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:38:02.812 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:38:02.812 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:38:02.812 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:38:02.812 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:38:02.812 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:38:02.813 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:38:02.813 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:38:02.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:38:02.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:38:02.813 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:38:02.813 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:38:02.813 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:38:02.813 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:38:02.813 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:38:02.813 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:38:02.813 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:38:02.813 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:38:02.813 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:38:02.813 | INFO | ------------------------------------------ | 20260508 06:38:02.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:38:02.813 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:38:02.813 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:38:02.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:38:02.813 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.90e-01 | 20260508 06:38:02.813 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.05e-01, 1.20e-02 / 1.01e-02 | 20260508 06:38:02.813 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.44e+00, 5.82e-01 / 2.34e-01 | 20260508 06:38:02.813 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.43e+00, 2.07e-01 / 1.43e-01 | 20260508 06:38:02.813 | INFO | PerfReport: 15.91%, DME -- 4.16e+01 / 4.51e+01, 3.58e+00 / 1.10e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.82e+00 / 1.52e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.56e+00, 4.09e-01 / 2.44e-01 | 20260508 06:38:02.813 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.18e+00, 2.55e-01 / 1.11e-01 | 20260508 06:38:02.813 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.42e-01, 4.88e-02 / 2.28e-02 | 20260508 06:38:02.813 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:38:02.813 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:38:02.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:38:02.813 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 24.46%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 24.28%, Download -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 2.32%, LockUpload -- 2.46e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 2.48%, LockDownload -- 2.62e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 15.61%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 06:38:02.813 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 06:38:02.915 | INFO | Starting a round of Baz IO | 20260508 06:38:06.331 | INFO | Populating RTMetrics report | 20260508 06:38:06.331 | INFO | RTMetrics report populated | 20260508 06:38:06.331 | INFO | Opening tmp RTMetrics json file | 20260508 06:38:06.342 | INFO | Serializing RTMetrics to json | 20260508 06:38:06.343 | INFO | Serialization done, swapping tmp file now... | 20260508 06:38:06.343 | INFO | RTMetrics done | 20260508 06:38:06.402 | INFO | Finished a round of Baz IO: Wrote 1.546 GiB in 3.486 seconds | 20260508 06:38:06.402 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.67% / 0.00% / 59.33% | 20260508 06:38:06.402 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.387311% / 0% / 3.97364e-05% | 20260508 06:38:06.402 | INFO | Baz IO overall: | 20260508 06:38:06.402 | INFO | BytesWritten : 2664091860404 | 20260508 06:38:06.402 | INFO | NumEvents : 1284305598514 | 20260508 06:38:07.949 | INFO | Analyzing chunk frames = [6012928,6013440) | 20260508 06:38:07.949 | INFO | MemInfo: { | 20260508 06:38:07.949 | INFO | "gpu" : " 17 GB", | 20260508 06:38:07.949 | INFO | "huge" : "", | 20260508 06:38:07.949 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:07.949 | INFO | "malloc" : " 14 GB", | 20260508 06:38:07.949 | INFO | "pinned" : " 29 GB" | 20260508 06:38:07.949 | INFO | } | 20260508 06:38:07.949 | INFO | Perf report for the last 5.13803 seconds: | 20260508 06:38:07.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:07.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:07.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:07.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:07.949 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 06:38:07.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:07.949 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:38:07.949 | INFO | PRE_HQ : 65.7%, 1.6, 55.2ms, 0 | 20260508 06:38:07.949 | INFO | BAZWRITER : 67.4%, 2.6, 91.3ms, 0 | 20260508 06:38:07.950 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:38:07.950 | INFO | Uploaded 36GiB to GPU in 2.57729s (13.9681GiB/s) | 20260508 06:38:07.950 | INFO | Downloaded 38GiB to GPU in 2.68214s (14.1678GiB/s) | 20260508 06:38:13.090 | INFO | Analyzing chunk frames = [6013440,6013952) | 20260508 06:38:13.090 | INFO | MemInfo: { | 20260508 06:38:13.090 | INFO | "gpu" : " 17 GB", | 20260508 06:38:13.090 | INFO | "huge" : "", | 20260508 06:38:13.090 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:13.090 | INFO | "malloc" : " 14 GB", | 20260508 06:38:13.090 | INFO | "pinned" : " 29 GB" | 20260508 06:38:13.090 | INFO | } | 20260508 06:38:13.090 | INFO | Perf report for the last 5.1407 seconds: | 20260508 06:38:13.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:13.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:38:13.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:38:13.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:13.090 | INFO | Uploaded 36GiB to GPU in 2.5081s (14.3535GiB/s) | 20260508 06:38:13.090 | INFO | Downloaded 34GiB to GPU in 2.43436s (13.9667GiB/s) | 20260508 06:38:18.233 | INFO | Analyzing chunk frames = [6013952,6014464) | 20260508 06:38:18.233 | INFO | MemInfo: { | 20260508 06:38:18.233 | INFO | "gpu" : " 17 GB", | 20260508 06:38:18.233 | INFO | "huge" : "", | 20260508 06:38:18.233 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:18.233 | INFO | "malloc" : " 14 GB", | 20260508 06:38:18.234 | INFO | "pinned" : " 29 GB" | 20260508 06:38:18.234 | INFO | } | 20260508 06:38:18.234 | INFO | Perf report for the last 5.14353 seconds: | 20260508 06:38:18.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:18.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:38:18.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:38:18.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:18.234 | INFO | Uploaded 36GiB to GPU in 2.56889s (14.0138GiB/s) | 20260508 06:38:18.234 | INFO | Downloaded 34GiB to GPU in 2.42011s (14.049GiB/s) | 20260508 06:38:23.373 | INFO | Analyzing chunk frames = [6014464,6014976) | 20260508 06:38:23.373 | INFO | MemInfo: { | 20260508 06:38:23.373 | INFO | "gpu" : " 17 GB", | 20260508 06:38:23.373 | INFO | "huge" : "", | 20260508 06:38:23.373 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:23.373 | INFO | "malloc" : " 14 GB", | 20260508 06:38:23.373 | INFO | "pinned" : " 29 GB" | 20260508 06:38:23.373 | INFO | } | 20260508 06:38:23.373 | INFO | Perf report for the last 5.13947 seconds: | 20260508 06:38:23.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:23.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.373 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 06:38:23.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.373 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:38:23.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:23.374 | INFO | Uploaded 36GiB to GPU in 2.59037s (13.8976GiB/s) | 20260508 06:38:23.374 | INFO | Downloaded 34GiB to GPU in 2.41035s (14.1058GiB/s) | 20260508 06:38:28.510 | INFO | Analyzing chunk frames = [6014976,6015488) | 20260508 06:38:28.510 | INFO | MemInfo: { | 20260508 06:38:28.510 | INFO | "gpu" : " 17 GB", | 20260508 06:38:28.510 | INFO | "huge" : "", | 20260508 06:38:28.510 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:28.510 | INFO | "malloc" : " 14 GB", | 20260508 06:38:28.511 | INFO | "pinned" : " 29 GB" | 20260508 06:38:28.511 | INFO | } | 20260508 06:38:28.511 | INFO | Perf report for the last 5.13751 seconds: | 20260508 06:38:28.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:28.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 06:38:28.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:38:28.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:28.511 | INFO | Uploaded 36GiB to GPU in 2.25011s (15.9992GiB/s) | 20260508 06:38:28.511 | INFO | Downloaded 34GiB to GPU in 2.47338s (13.7463GiB/s) | 20260508 06:38:33.645 | INFO | Analyzing chunk frames = [6015488,6016000) | 20260508 06:38:33.645 | INFO | MemInfo: { | 20260508 06:38:33.645 | INFO | "gpu" : " 17 GB", | 20260508 06:38:33.645 | INFO | "huge" : "", | 20260508 06:38:33.645 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:33.645 | INFO | "malloc" : " 14 GB", | 20260508 06:38:33.645 | INFO | "pinned" : " 29 GB" | 20260508 06:38:33.645 | INFO | } | 20260508 06:38:33.645 | INFO | Perf report for the last 5.13463 seconds: | 20260508 06:38:33.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:33.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.645 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 06:38:33.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.646 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 06:38:33.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:33.646 | INFO | Uploaded 36GiB to GPU in 2.37753s (15.1418GiB/s) | 20260508 06:38:33.646 | INFO | Downloaded 34GiB to GPU in 2.44332s (13.9155GiB/s) | 20260508 06:38:38.785 | INFO | Analyzing chunk frames = [6016000,6016512) | 20260508 06:38:38.785 | INFO | MemInfo: { | 20260508 06:38:38.785 | INFO | "gpu" : " 17 GB", | 20260508 06:38:38.785 | INFO | "huge" : "", | 20260508 06:38:38.785 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:38.785 | INFO | "malloc" : " 14 GB", | 20260508 06:38:38.785 | INFO | "pinned" : " 29 GB" | 20260508 06:38:38.785 | INFO | } | 20260508 06:38:38.785 | INFO | Perf report for the last 5.14013 seconds: | 20260508 06:38:38.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:38.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.785 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:38:38.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.786 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 06:38:38.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:38.786 | INFO | Uploaded 36GiB to GPU in 2.49452s (14.4316GiB/s) | 20260508 06:38:38.786 | INFO | Downloaded 34GiB to GPU in 2.44905s (13.8829GiB/s) | 20260508 06:38:43.930 | INFO | Analyzing chunk frames = [6016512,6017024) | 20260508 06:38:43.930 | INFO | MemInfo: { | 20260508 06:38:43.930 | INFO | "gpu" : " 17 GB", | 20260508 06:38:43.930 | INFO | "huge" : "", | 20260508 06:38:43.930 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:43.930 | INFO | "malloc" : " 14 GB", | 20260508 06:38:43.930 | INFO | "pinned" : " 29 GB" | 20260508 06:38:43.930 | INFO | } | 20260508 06:38:43.930 | INFO | Perf report for the last 5.14498 seconds: | 20260508 06:38:43.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:43.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.930 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:38:43.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.931 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 06:38:43.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:43.931 | INFO | Uploaded 36GiB to GPU in 2.47519s (14.5443GiB/s) | 20260508 06:38:43.931 | INFO | Downloaded 34GiB to GPU in 2.44044s (13.9319GiB/s) | 20260508 06:38:44.045 | INFO | Starting a round of Baz IO | 20260508 06:38:47.452 | INFO | Populating RTMetrics report | 20260508 06:38:47.452 | INFO | RTMetrics report populated | 20260508 06:38:47.452 | INFO | Opening tmp RTMetrics json file | 20260508 06:38:47.461 | INFO | Serializing RTMetrics to json | 20260508 06:38:47.463 | INFO | Serialization done, swapping tmp file now... | 20260508 06:38:47.463 | INFO | RTMetrics done | 20260508 06:38:47.525 | INFO | Finished a round of Baz IO: Wrote 1.545 GiB in 3.48 seconds | 20260508 06:38:47.525 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.67% / 0.00% / 59.33% | 20260508 06:38:47.525 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.385678% / 7.94729e-06% / 4.37101e-05% | 20260508 06:38:47.525 | INFO | Baz IO overall: | 20260508 06:38:47.525 | INFO | BytesWritten : 2665750501288 | 20260508 06:38:47.525 | INFO | NumEvents : 1285094163970 | 20260508 06:38:49.071 | INFO | Analyzing chunk frames = [6017024,6017536) | 20260508 06:38:49.071 | INFO | MemInfo: { | 20260508 06:38:49.071 | INFO | "gpu" : " 17 GB", | 20260508 06:38:49.071 | INFO | "huge" : "", | 20260508 06:38:49.071 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:49.071 | INFO | "malloc" : " 14 GB", | 20260508 06:38:49.071 | INFO | "pinned" : " 29 GB" | 20260508 06:38:49.071 | INFO | } | 20260508 06:38:49.071 | INFO | Perf report for the last 5.14046 seconds: | 20260508 06:38:49.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:49.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:49.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:49.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:49.071 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 06:38:49.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:49.071 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 06:38:49.071 | INFO | PRE_HQ : 66.3%, 1.8, 61.6ms, 0 | 20260508 06:38:49.071 | INFO | BAZWRITER : 67.2%, 2.8, 97.2ms, 0 | 20260508 06:38:49.071 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:38:49.071 | INFO | Uploaded 36GiB to GPU in 2.56853s (14.0158GiB/s) | 20260508 06:38:49.071 | INFO | Downloaded 38GiB to GPU in 2.71871s (13.9772GiB/s) | 20260508 06:38:54.202 | INFO | Analyzing chunk frames = [6017536,6018048) | 20260508 06:38:54.202 | INFO | MemInfo: { | 20260508 06:38:54.202 | INFO | "gpu" : " 17 GB", | 20260508 06:38:54.202 | INFO | "huge" : "", | 20260508 06:38:54.202 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:54.202 | INFO | "malloc" : " 14 GB", | 20260508 06:38:54.202 | INFO | "pinned" : " 29 GB" | 20260508 06:38:54.202 | INFO | } | 20260508 06:38:54.202 | INFO | Perf report for the last 5.13133 seconds: | 20260508 06:38:54.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:54.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:38:54.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:38:54.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:54.202 | INFO | Uploaded 36GiB to GPU in 2.49819s (14.4104GiB/s) | 20260508 06:38:54.203 | INFO | Downloaded 34GiB to GPU in 2.43715s (13.9507GiB/s) | 20260508 06:38:59.344 | INFO | Analyzing chunk frames = [6018048,6018560) | 20260508 06:38:59.344 | INFO | MemInfo: { | 20260508 06:38:59.344 | INFO | "gpu" : " 17 GB", | 20260508 06:38:59.344 | INFO | "huge" : "", | 20260508 06:38:59.344 | INFO | "huge_shared" : " 29 GB", | 20260508 06:38:59.344 | INFO | "malloc" : " 14 GB", | 20260508 06:38:59.344 | INFO | "pinned" : " 29 GB" | 20260508 06:38:59.344 | INFO | } | 20260508 06:38:59.344 | INFO | Perf report for the last 5.14194 seconds: | 20260508 06:38:59.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:38:59.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.344 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 06:38:59.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.344 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:38:59.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:38:59.345 | INFO | Uploaded 36GiB to GPU in 2.53595s (14.1958GiB/s) | 20260508 06:38:59.345 | INFO | Downloaded 34GiB to GPU in 2.40986s (14.1087GiB/s) | 20260508 06:39:04.484 | INFO | Analyzing chunk frames = [6018560,6019072) | 20260508 06:39:04.484 | INFO | MemInfo: { | 20260508 06:39:04.484 | INFO | "gpu" : " 17 GB", | 20260508 06:39:04.484 | INFO | "huge" : "", | 20260508 06:39:04.484 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:04.484 | INFO | "malloc" : " 14 GB", | 20260508 06:39:04.484 | INFO | "pinned" : " 29 GB" | 20260508 06:39:04.484 | INFO | } | 20260508 06:39:04.484 | INFO | Perf report for the last 5.14015 seconds: | 20260508 06:39:04.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:04.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.484 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 06:39:04.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.484 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:39:04.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:04.485 | INFO | Uploaded 36GiB to GPU in 2.55256s (14.1035GiB/s) | 20260508 06:39:04.485 | INFO | Downloaded 34GiB to GPU in 2.4076s (14.1219GiB/s) | 20260508 06:39:09.621 | INFO | Analyzing chunk frames = [6019072,6019584) | 20260508 06:39:09.621 | INFO | MemInfo: { | 20260508 06:39:09.621 | INFO | "gpu" : " 17 GB", | 20260508 06:39:09.621 | INFO | "huge" : "", | 20260508 06:39:09.621 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:09.621 | INFO | "malloc" : " 14 GB", | 20260508 06:39:09.621 | INFO | "pinned" : " 29 GB" | 20260508 06:39:09.621 | INFO | } | 20260508 06:39:09.621 | INFO | Perf report for the last 5.13687 seconds: | 20260508 06:39:09.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:09.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 06:39:09.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 06:39:09.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:09.621 | INFO | Uploaded 36GiB to GPU in 2.26396s (15.9013GiB/s) | 20260508 06:39:09.622 | INFO | Downloaded 34GiB to GPU in 2.47989s (13.7103GiB/s) | 20260508 06:39:14.765 | INFO | Analyzing chunk frames = [6019584,6020096) | 20260508 06:39:14.765 | INFO | MemInfo: { | 20260508 06:39:14.765 | INFO | "gpu" : " 17 GB", | 20260508 06:39:14.765 | INFO | "huge" : "", | 20260508 06:39:14.765 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:14.765 | INFO | "malloc" : " 14 GB", | 20260508 06:39:14.765 | INFO | "pinned" : " 29 GB" | 20260508 06:39:14.765 | INFO | } | 20260508 06:39:14.765 | INFO | Perf report for the last 5.14406 seconds: | 20260508 06:39:14.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:14.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.765 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 06:39:14.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.765 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:39:14.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:14.766 | INFO | Uploaded 36GiB to GPU in 2.4443s (14.7282GiB/s) | 20260508 06:39:14.766 | INFO | Downloaded 34GiB to GPU in 2.43066s (13.988GiB/s) | 20260508 06:39:19.901 | INFO | Analyzing chunk frames = [6020096,6020608) | 20260508 06:39:19.901 | INFO | MemInfo: { | 20260508 06:39:19.901 | INFO | "gpu" : " 17 GB", | 20260508 06:39:19.901 | INFO | "huge" : "", | 20260508 06:39:19.901 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:19.901 | INFO | "malloc" : " 14 GB", | 20260508 06:39:19.901 | INFO | "pinned" : " 29 GB" | 20260508 06:39:19.901 | INFO | } | 20260508 06:39:19.901 | INFO | Perf report for the last 5.13585 seconds: | 20260508 06:39:19.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:19.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.901 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:39:19.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.901 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 06:39:19.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:19.902 | INFO | Uploaded 36GiB to GPU in 2.47223s (14.5618GiB/s) | 20260508 06:39:19.902 | INFO | Downloaded 34GiB to GPU in 2.43335s (13.9725GiB/s) | 20260508 06:39:25.042 | INFO | Analyzing chunk frames = [6020608,6021120) | 20260508 06:39:25.042 | INFO | MemInfo: { | 20260508 06:39:25.042 | INFO | "gpu" : " 17 GB", | 20260508 06:39:25.042 | INFO | "huge" : "", | 20260508 06:39:25.042 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:25.042 | INFO | "malloc" : " 14 GB", | 20260508 06:39:25.042 | INFO | "pinned" : " 29 GB" | 20260508 06:39:25.042 | INFO | } | 20260508 06:39:25.042 | INFO | Perf report for the last 5.14111 seconds: | 20260508 06:39:25.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:25.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.042 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:39:25.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.042 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:39:25.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:25.043 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:39:25.043 | INFO | Total mem usage: 16.7 GiB | 20260508 06:39:25.043 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:39:25.043 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:39:25.043 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:39:25.043 | INFO | High Water Breakdown: | 20260508 06:39:25.043 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:39:25.043 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:39:25.043 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:39:25.043 | INFO | ------------------------------------------ | 20260508 06:39:25.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:39:25.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:39:25.043 | INFO | Total mem usage: 26.5 GiB | 20260508 06:39:25.043 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:39:25.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:39:25.043 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:39:25.043 | INFO | High Water Breakdown: | 20260508 06:39:25.043 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:39:25.043 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:39:25.043 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:39:25.043 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:39:25.043 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:39:25.043 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:39:25.043 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:39:25.043 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:39:25.043 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:39:25.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:39:25.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:39:25.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:39:25.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:39:25.043 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:39:25.043 | INFO | ------------------------------------------ | 20260508 06:39:25.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:39:25.043 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:39:25.043 | INFO | Total mem usage: 27.8 GiB | 20260508 06:39:25.043 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:39:25.043 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:39:25.043 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:39:25.043 | INFO | High Water Breakdown: | 20260508 06:39:25.043 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:39:25.043 | INFO | ------------------------------------------ | 20260508 06:39:25.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:39:25.043 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:39:25.043 | INFO | Total mem usage: 15.7 GiB | 20260508 06:39:25.043 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:39:25.043 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:39:25.043 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:39:25.043 | INFO | High Water Breakdown: | 20260508 06:39:25.043 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:39:25.043 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:39:25.043 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:39:25.043 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:39:25.043 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:39:25.043 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:39:25.043 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:39:25.043 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:39:25.043 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:39:25.043 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:39:25.043 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:39:25.043 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:39:25.043 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:39:25.043 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:39:25.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:39:25.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:39:25.043 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:39:25.043 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:39:25.043 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:39:25.043 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:39:25.043 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:39:25.043 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:39:25.043 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:39:25.043 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:39:25.043 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:39:25.043 | INFO | ------------------------------------------ | 20260508 06:39:25.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:39:25.043 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:39:25.043 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:39:25.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:39:25.043 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.39e+01 / 5.92e-01 | 20260508 06:39:25.043 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.07e-01, 1.21e-02 / 6.99e-03 | 20260508 06:39:25.043 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.44e+00, 5.87e-01 / 2.17e-01 | 20260508 06:39:25.043 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.11e-01 / 1.02e-01 | 20260508 06:39:25.043 | INFO | PerfReport: 15.94%, DME -- 4.16e+01 / 4.49e+01, 3.51e+00 / 1.18e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.76e+00 / 1.57e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.52e+00, 4.10e-01 / 2.69e-01 | 20260508 06:39:25.043 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.19e+00, 2.50e-01 / 1.21e-01 | 20260508 06:39:25.043 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.15e-01 / 3.48e-01, 4.96e-02 / 2.67e-02 | 20260508 06:39:25.043 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:39:25.043 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:39:25.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:39:25.043 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 24.37%, Upload -- 2.58e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 24.28%, Download -- 2.57e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 2.28%, LockUpload -- 2.41e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 2.61%, LockDownload -- 2.77e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 15.61%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:39:25.043 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260508 06:39:25.043 | INFO | Uploaded 36GiB to GPU in 2.48202s (14.5043GiB/s) | 20260508 06:39:25.043 | INFO | Downloaded 34GiB to GPU in 2.44258s (13.9197GiB/s) | 20260508 06:39:25.140 | INFO | Starting a round of Baz IO | 20260508 06:39:28.556 | INFO | Populating RTMetrics report | 20260508 06:39:28.556 | INFO | RTMetrics report populated | 20260508 06:39:28.556 | INFO | Opening tmp RTMetrics json file | 20260508 06:39:28.565 | INFO | Serializing RTMetrics to json | 20260508 06:39:28.567 | INFO | Serialization done, swapping tmp file now... | 20260508 06:39:28.568 | INFO | RTMetrics done | 20260508 06:39:28.625 | INFO | Finished a round of Baz IO: Wrote 1.547 GiB in 3.485 seconds | 20260508 06:39:28.626 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.67% / 0.00% / 59.33% | 20260508 06:39:28.626 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.387375% / 3.97364e-06% / 8.74201e-05% | 20260508 06:39:28.626 | INFO | Baz IO overall: | 20260508 06:39:28.626 | INFO | BytesWritten : 2667411516321 | 20260508 06:39:28.626 | INFO | NumEvents : 1285884796402 | 20260508 06:39:30.185 | INFO | Analyzing chunk frames = [6021120,6021632) | 20260508 06:39:30.185 | INFO | MemInfo: { | 20260508 06:39:30.185 | INFO | "gpu" : " 17 GB", | 20260508 06:39:30.185 | INFO | "huge" : "", | 20260508 06:39:30.185 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:30.185 | INFO | "malloc" : " 14 GB", | 20260508 06:39:30.185 | INFO | "pinned" : " 29 GB" | 20260508 06:39:30.185 | INFO | } | 20260508 06:39:30.185 | INFO | Perf report for the last 5.14284 seconds: | 20260508 06:39:30.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:30.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:30.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:30.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:30.185 | INFO | ANALYSIS : 67.4%, 3.0, 105.5ms, 0 | 20260508 06:39:30.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:30.185 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 06:39:30.186 | INFO | PRE_HQ : 65.3%, 1.6, 54.9ms, 0 | 20260508 06:39:30.186 | INFO | BAZWRITER : 67.1%, 2.6, 91.0ms, 0 | 20260508 06:39:30.186 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:39:30.186 | INFO | Uploaded 36GiB to GPU in 2.54885s (14.124GiB/s) | 20260508 06:39:30.186 | INFO | Downloaded 38GiB to GPU in 2.69892s (14.0797GiB/s) | 20260508 06:39:35.323 | INFO | Analyzing chunk frames = [6021632,6022144) | 20260508 06:39:35.323 | INFO | MemInfo: { | 20260508 06:39:35.323 | INFO | "gpu" : " 17 GB", | 20260508 06:39:35.323 | INFO | "huge" : "", | 20260508 06:39:35.323 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:35.323 | INFO | "malloc" : " 14 GB", | 20260508 06:39:35.323 | INFO | "pinned" : " 29 GB" | 20260508 06:39:35.323 | INFO | } | 20260508 06:39:35.323 | INFO | Perf report for the last 5.13785 seconds: | 20260508 06:39:35.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:35.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 06:39:35.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:39:35.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:35.323 | INFO | Uploaded 36GiB to GPU in 2.47064s (14.5711GiB/s) | 20260508 06:39:35.323 | INFO | Downloaded 34GiB to GPU in 2.45237s (13.8641GiB/s) | 20260508 06:39:40.455 | INFO | Analyzing chunk frames = [6022144,6022656) | 20260508 06:39:40.455 | INFO | MemInfo: { | 20260508 06:39:40.455 | INFO | "gpu" : " 17 GB", | 20260508 06:39:40.455 | INFO | "huge" : "", | 20260508 06:39:40.455 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:40.455 | INFO | "malloc" : " 14 GB", | 20260508 06:39:40.455 | INFO | "pinned" : " 29 GB" | 20260508 06:39:40.455 | INFO | } | 20260508 06:39:40.455 | INFO | Perf report for the last 5.13237 seconds: | 20260508 06:39:40.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:40.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.455 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 06:39:40.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.455 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:39:40.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:40.456 | INFO | Uploaded 36GiB to GPU in 2.52758s (14.2429GiB/s) | 20260508 06:39:40.456 | INFO | Downloaded 34GiB to GPU in 2.44768s (13.8907GiB/s) | 20260508 06:39:45.597 | INFO | Analyzing chunk frames = [6022656,6023168) | 20260508 06:39:45.597 | INFO | MemInfo: { | 20260508 06:39:45.597 | INFO | "gpu" : " 17 GB", | 20260508 06:39:45.597 | INFO | "huge" : "", | 20260508 06:39:45.597 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:45.597 | INFO | "malloc" : " 14 GB", | 20260508 06:39:45.597 | INFO | "pinned" : " 29 GB" | 20260508 06:39:45.597 | INFO | } | 20260508 06:39:45.597 | INFO | Perf report for the last 5.14195 seconds: | 20260508 06:39:45.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:45.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.597 | INFO | ANALYSIS : 66.5%, 2.9, 103.0ms, 0 | 20260508 06:39:45.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.597 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:39:45.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:45.598 | INFO | Uploaded 36GiB to GPU in 2.52574s (14.2532GiB/s) | 20260508 06:39:45.598 | INFO | Downloaded 34GiB to GPU in 2.41836s (14.0591GiB/s) | 20260508 06:39:50.738 | INFO | Analyzing chunk frames = [6023168,6023680) | 20260508 06:39:50.738 | INFO | MemInfo: { | 20260508 06:39:50.738 | INFO | "gpu" : " 17 GB", | 20260508 06:39:50.738 | INFO | "huge" : "", | 20260508 06:39:50.738 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:50.738 | INFO | "malloc" : " 14 GB", | 20260508 06:39:50.738 | INFO | "pinned" : " 29 GB" | 20260508 06:39:50.738 | INFO | } | 20260508 06:39:50.738 | INFO | Perf report for the last 5.14112 seconds: | 20260508 06:39:50.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:50.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.738 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 06:39:50.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.738 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:39:50.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:50.739 | INFO | Uploaded 36GiB to GPU in 2.29081s (15.7149GiB/s) | 20260508 06:39:50.739 | INFO | Downloaded 34GiB to GPU in 2.50374s (13.5797GiB/s) | 20260508 06:39:55.878 | INFO | Analyzing chunk frames = [6023680,6024192) | 20260508 06:39:55.878 | INFO | MemInfo: { | 20260508 06:39:55.878 | INFO | "gpu" : " 17 GB", | 20260508 06:39:55.878 | INFO | "huge" : "", | 20260508 06:39:55.878 | INFO | "huge_shared" : " 29 GB", | 20260508 06:39:55.879 | INFO | "malloc" : " 14 GB", | 20260508 06:39:55.879 | INFO | "pinned" : " 29 GB" | 20260508 06:39:55.879 | INFO | } | 20260508 06:39:55.879 | INFO | Perf report for the last 5.14035 seconds: | 20260508 06:39:55.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:39:55.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.879 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 06:39:55.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.879 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260508 06:39:55.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:39:55.880 | INFO | Uploaded 36GiB to GPU in 2.33677s (15.4059GiB/s) | 20260508 06:39:55.880 | INFO | Downloaded 34GiB to GPU in 2.44576s (13.9016GiB/s) | 20260508 06:40:01.021 | INFO | Analyzing chunk frames = [6024192,6024704) | 20260508 06:40:01.021 | INFO | MemInfo: { | 20260508 06:40:01.021 | INFO | "gpu" : " 17 GB", | 20260508 06:40:01.021 | INFO | "huge" : "", | 20260508 06:40:01.021 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:01.021 | INFO | "malloc" : " 14 GB", | 20260508 06:40:01.021 | INFO | "pinned" : " 29 GB" | 20260508 06:40:01.022 | INFO | } | 20260508 06:40:01.022 | INFO | Perf report for the last 5.14295 seconds: | 20260508 06:40:01.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:01.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:40:01.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 06:40:01.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:01.022 | INFO | Uploaded 36GiB to GPU in 2.48594s (14.4814GiB/s) | 20260508 06:40:01.022 | INFO | Downloaded 34GiB to GPU in 2.46884s (13.7717GiB/s) | 20260508 06:40:06.153 | INFO | Analyzing chunk frames = [6024704,6025216) | 20260508 06:40:06.153 | INFO | MemInfo: { | 20260508 06:40:06.153 | INFO | "gpu" : " 17 GB", | 20260508 06:40:06.153 | INFO | "huge" : "", | 20260508 06:40:06.153 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:06.153 | INFO | "malloc" : " 14 GB", | 20260508 06:40:06.153 | INFO | "pinned" : " 29 GB" | 20260508 06:40:06.153 | INFO | } | 20260508 06:40:06.153 | INFO | Perf report for the last 5.13134 seconds: | 20260508 06:40:06.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:06.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.153 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 06:40:06.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.153 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 06:40:06.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:06.154 | INFO | Uploaded 36GiB to GPU in 2.4123s (14.9235GiB/s) | 20260508 06:40:06.154 | INFO | Downloaded 34GiB to GPU in 2.46174s (13.8114GiB/s) | 20260508 06:40:06.267 | INFO | Starting a round of Baz IO | 20260508 06:40:09.683 | INFO | Populating RTMetrics report | 20260508 06:40:09.684 | INFO | RTMetrics report populated | 20260508 06:40:09.684 | INFO | Opening tmp RTMetrics json file | 20260508 06:40:09.684 | INFO | Serializing RTMetrics to json | 20260508 06:40:09.686 | INFO | Serialization done, swapping tmp file now... | 20260508 06:40:09.686 | INFO | RTMetrics done | 20260508 06:40:09.749 | INFO | Finished a round of Baz IO: Wrote 1.546 GiB in 3.481 seconds | 20260508 06:40:09.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.67% / 0.00% / 59.33% | 20260508 06:40:09.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.386349% / 3.97364e-06% / 9.53674e-05% | 20260508 06:40:09.749 | INFO | Baz IO overall: | 20260508 06:40:09.749 | INFO | BytesWritten : 2669071426077 | 20260508 06:40:09.749 | INFO | NumEvents : 1286674748103 | 20260508 06:40:11.298 | INFO | Analyzing chunk frames = [6025216,6025728) | 20260508 06:40:11.298 | INFO | MemInfo: { | 20260508 06:40:11.298 | INFO | "gpu" : " 17 GB", | 20260508 06:40:11.298 | INFO | "huge" : "", | 20260508 06:40:11.298 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:11.299 | INFO | "malloc" : " 14 GB", | 20260508 06:40:11.299 | INFO | "pinned" : " 29 GB" | 20260508 06:40:11.299 | INFO | } | 20260508 06:40:11.299 | INFO | Perf report for the last 5.14546 seconds: | 20260508 06:40:11.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:11.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:11.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:11.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:11.299 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260508 06:40:11.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:11.299 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260508 06:40:11.299 | INFO | PRE_HQ : 66.7%, 1.8, 61.4ms, 0 | 20260508 06:40:11.299 | INFO | BAZWRITER : 67.0%, 2.8, 96.4ms, 0 | 20260508 06:40:11.299 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 06:40:11.299 | INFO | Uploaded 36GiB to GPU in 2.49454s (14.4315GiB/s) | 20260508 06:40:11.299 | INFO | Downloaded 38GiB to GPU in 2.70034s (14.0723GiB/s) | 20260508 06:40:16.439 | INFO | Analyzing chunk frames = [6025728,6026240) | 20260508 06:40:16.439 | INFO | MemInfo: { | 20260508 06:40:16.439 | INFO | "gpu" : " 17 GB", | 20260508 06:40:16.439 | INFO | "huge" : "", | 20260508 06:40:16.440 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:16.440 | INFO | "malloc" : " 14 GB", | 20260508 06:40:16.440 | INFO | "pinned" : " 29 GB" | 20260508 06:40:16.440 | INFO | } | 20260508 06:40:16.440 | INFO | Perf report for the last 5.14106 seconds: | 20260508 06:40:16.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:16.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 06:40:16.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:40:16.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:16.440 | INFO | Uploaded 36GiB to GPU in 2.50185s (14.3894GiB/s) | 20260508 06:40:16.440 | INFO | Downloaded 34GiB to GPU in 2.44423s (13.9103GiB/s) | 20260508 06:40:21.578 | INFO | Analyzing chunk frames = [6026240,6026752) | 20260508 06:40:21.578 | INFO | MemInfo: { | 20260508 06:40:21.578 | INFO | "gpu" : " 17 GB", | 20260508 06:40:21.578 | INFO | "huge" : "", | 20260508 06:40:21.578 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:21.578 | INFO | "malloc" : " 14 GB", | 20260508 06:40:21.578 | INFO | "pinned" : " 29 GB" | 20260508 06:40:21.578 | INFO | } | 20260508 06:40:21.578 | INFO | Perf report for the last 5.13855 seconds: | 20260508 06:40:21.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:21.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.578 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:40:21.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.578 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:40:21.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:21.579 | INFO | Uploaded 36GiB to GPU in 2.56069s (14.0587GiB/s) | 20260508 06:40:21.579 | INFO | Downloaded 34GiB to GPU in 2.43807s (13.9455GiB/s) | 20260508 06:40:26.708 | INFO | Analyzing chunk frames = [6026752,6027264) | 20260508 06:40:26.708 | INFO | MemInfo: { | 20260508 06:40:26.708 | INFO | "gpu" : " 17 GB", | 20260508 06:40:26.708 | INFO | "huge" : "", | 20260508 06:40:26.708 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:26.708 | INFO | "malloc" : " 14 GB", | 20260508 06:40:26.708 | INFO | "pinned" : " 29 GB" | 20260508 06:40:26.708 | INFO | } | 20260508 06:40:26.708 | INFO | Perf report for the last 5.13015 seconds: | 20260508 06:40:26.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:26.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.708 | INFO | ANALYSIS : 66.8%, 3.0, 103.2ms, 0 | 20260508 06:40:26.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.709 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:40:26.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:26.709 | INFO | Uploaded 36GiB to GPU in 2.54323s (14.1552GiB/s) | 20260508 06:40:26.709 | INFO | Downloaded 34GiB to GPU in 2.40695s (14.1257GiB/s) | 20260508 06:40:31.852 | INFO | Analyzing chunk frames = [6027264,6027776) | 20260508 06:40:31.852 | INFO | MemInfo: { | 20260508 06:40:31.852 | INFO | "gpu" : " 17 GB", | 20260508 06:40:31.852 | INFO | "huge" : "", | 20260508 06:40:31.852 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:31.852 | INFO | "malloc" : " 14 GB", | 20260508 06:40:31.852 | INFO | "pinned" : " 29 GB" | 20260508 06:40:31.852 | INFO | } | 20260508 06:40:31.853 | INFO | Perf report for the last 5.1441 seconds: | 20260508 06:40:31.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:31.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:40:31.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:40:31.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:31.853 | INFO | Uploaded 36GiB to GPU in 2.30316s (15.6307GiB/s) | 20260508 06:40:31.853 | INFO | Downloaded 34GiB to GPU in 2.49501s (13.6272GiB/s) | 20260508 06:40:36.995 | INFO | Analyzing chunk frames = [6027776,6028288) | 20260508 06:40:36.995 | INFO | MemInfo: { | 20260508 06:40:36.995 | INFO | "gpu" : " 17 GB", | 20260508 06:40:36.995 | INFO | "huge" : "", | 20260508 06:40:36.995 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:36.995 | INFO | "malloc" : " 14 GB", | 20260508 06:40:36.995 | INFO | "pinned" : " 29 GB" | 20260508 06:40:36.995 | INFO | } | 20260508 06:40:36.996 | INFO | Perf report for the last 5.14299 seconds: | 20260508 06:40:36.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:36.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:40:36.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:40:36.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:36.996 | INFO | Uploaded 36GiB to GPU in 2.463s (14.6163GiB/s) | 20260508 06:40:36.996 | INFO | Downloaded 34GiB to GPU in 2.42365s (14.0284GiB/s) | 20260508 06:40:42.139 | INFO | Analyzing chunk frames = [6028288,6028800) | 20260508 06:40:42.139 | INFO | MemInfo: { | 20260508 06:40:42.139 | INFO | "gpu" : " 17 GB", | 20260508 06:40:42.139 | INFO | "huge" : "", | 20260508 06:40:42.139 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:42.139 | INFO | "malloc" : " 14 GB", | 20260508 06:40:42.139 | INFO | "pinned" : " 29 GB" | 20260508 06:40:42.139 | INFO | } | 20260508 06:40:42.139 | INFO | Perf report for the last 5.14382 seconds: | 20260508 06:40:42.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:42.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.139 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:40:42.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.140 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:40:42.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:42.140 | INFO | Uploaded 36GiB to GPU in 2.47827s (14.5263GiB/s) | 20260508 06:40:42.140 | INFO | Downloaded 34GiB to GPU in 2.42973s (13.9933GiB/s) | 20260508 06:40:47.271 | INFO | Analyzing chunk frames = [6028800,6029312) | 20260508 06:40:47.271 | INFO | MemInfo: { | 20260508 06:40:47.271 | INFO | "gpu" : " 17 GB", | 20260508 06:40:47.271 | INFO | "huge" : "", | 20260508 06:40:47.271 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:47.271 | INFO | "malloc" : " 14 GB", | 20260508 06:40:47.271 | INFO | "pinned" : " 29 GB" | 20260508 06:40:47.271 | INFO | } | 20260508 06:40:47.271 | INFO | Perf report for the last 5.13193 seconds: | 20260508 06:40:47.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:47.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.271 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 06:40:47.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.271 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:40:47.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:47.272 | INFO | Uploaded 36GiB to GPU in 2.4377s (14.768GiB/s) | 20260508 06:40:47.272 | INFO | Downloaded 34GiB to GPU in 2.41802s (14.0611GiB/s) | 20260508 06:40:47.272 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:40:47.272 | INFO | Total mem usage: 16.7 GiB | 20260508 06:40:47.272 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:40:47.272 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:40:47.272 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:40:47.272 | INFO | High Water Breakdown: | 20260508 06:40:47.272 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:40:47.272 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:40:47.272 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:40:47.272 | INFO | ------------------------------------------ | 20260508 06:40:47.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:40:47.272 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:40:47.272 | INFO | Total mem usage: 26.5 GiB | 20260508 06:40:47.272 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:40:47.272 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:40:47.272 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:40:47.272 | INFO | High Water Breakdown: | 20260508 06:40:47.272 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:40:47.272 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:40:47.272 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:40:47.272 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:40:47.272 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:40:47.272 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:40:47.272 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:40:47.272 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:40:47.272 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:40:47.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:40:47.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:40:47.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:40:47.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:40:47.272 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:40:47.272 | INFO | ------------------------------------------ | 20260508 06:40:47.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:40:47.272 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:40:47.272 | INFO | Total mem usage: 27.8 GiB | 20260508 06:40:47.272 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:40:47.272 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:40:47.272 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:40:47.272 | INFO | High Water Breakdown: | 20260508 06:40:47.272 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:40:47.272 | INFO | ------------------------------------------ | 20260508 06:40:47.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:40:47.272 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:40:47.272 | INFO | Total mem usage: 15.7 GiB | 20260508 06:40:47.272 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:40:47.272 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:40:47.272 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:40:47.272 | INFO | High Water Breakdown: | 20260508 06:40:47.272 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:40:47.272 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:40:47.272 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:40:47.272 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:40:47.272 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:40:47.272 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:40:47.272 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:40:47.272 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:40:47.272 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:40:47.272 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:40:47.272 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:40:47.272 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:40:47.272 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:40:47.272 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:40:47.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:40:47.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:40:47.272 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:40:47.272 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:40:47.272 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:40:47.272 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:40:47.272 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:40:47.272 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:40:47.272 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:40:47.272 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:40:47.272 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:40:47.272 | INFO | ------------------------------------------ | 20260508 06:40:47.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:40:47.272 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:40:47.272 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:40:47.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:40:47.272 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.39e+01 / 7.61e-01 | 20260508 06:40:47.272 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.15e-02 / 4.81e-03 | 20260508 06:40:47.272 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.46e+00, 5.94e-01 / 2.27e-01 | 20260508 06:40:47.272 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.17e-01 | 20260508 06:40:47.272 | INFO | PerfReport: 15.91%, DME -- 4.18e+01 / 4.56e+01, 3.99e+00 / 1.14e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.87e+00 / 1.24e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.37e+00 / 2.59e+00, 4.14e-01 / 1.91e-01 | 20260508 06:40:47.272 | INFO | PerfReport: 3.13%, Metrics -- 1.03e+00 / 1.20e+00, 2.53e-01 / 1.30e-01 | 20260508 06:40:47.272 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.44e-01, 5.04e-02 / 1.86e-02 | 20260508 06:40:47.272 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:40:47.272 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:40:47.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:40:47.272 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 24.22%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 24.39%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 31.00%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 2.64%, LockDownload -- 2.79e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 06:40:47.272 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260508 06:40:47.369 | INFO | Starting a round of Baz IO | 20260508 06:40:50.804 | INFO | Populating RTMetrics report | 20260508 06:40:50.804 | INFO | RTMetrics report populated | 20260508 06:40:50.804 | INFO | Opening tmp RTMetrics json file | 20260508 06:40:50.805 | INFO | Serializing RTMetrics to json | 20260508 06:40:50.806 | INFO | Serialization done, swapping tmp file now... | 20260508 06:40:50.806 | INFO | RTMetrics done | 20260508 06:40:50.874 | INFO | Finished a round of Baz IO: Wrote 1.544 GiB in 3.505 seconds | 20260508 06:40:50.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.67% / 0.00% / 59.33% | 20260508 06:40:50.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.384514% / 0% / 5.96046e-05% | 20260508 06:40:50.874 | INFO | Baz IO overall: | 20260508 06:40:50.874 | INFO | BytesWritten : 2670729631189 | 20260508 06:40:50.874 | INFO | NumEvents : 1287463261950 | 20260508 06:40:52.406 | INFO | Analyzing chunk frames = [6029312,6029824) | 20260508 06:40:52.406 | INFO | MemInfo: { | 20260508 06:40:52.406 | INFO | "gpu" : " 17 GB", | 20260508 06:40:52.406 | INFO | "huge" : "", | 20260508 06:40:52.406 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:52.406 | INFO | "malloc" : " 14 GB", | 20260508 06:40:52.406 | INFO | "pinned" : " 29 GB" | 20260508 06:40:52.406 | INFO | } | 20260508 06:40:52.406 | INFO | Perf report for the last 5.13431 seconds: | 20260508 06:40:52.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:52.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:52.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:52.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:52.409 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260508 06:40:52.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:52.409 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 06:40:52.409 | INFO | PRE_HQ : 66.0%, 1.6, 54.9ms, 0 | 20260508 06:40:52.409 | INFO | BAZWRITER : 67.6%, 2.6, 90.8ms, 0 | 20260508 06:40:52.409 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 06:40:52.409 | INFO | Uploaded 36GiB to GPU in 2.5471s (14.1337GiB/s) | 20260508 06:40:52.409 | INFO | Downloaded 38GiB to GPU in 2.70959s (14.0243GiB/s) | 20260508 06:40:57.548 | INFO | Analyzing chunk frames = [6029824,6030336) | 20260508 06:40:57.548 | INFO | MemInfo: { | 20260508 06:40:57.548 | INFO | "gpu" : " 17 GB", | 20260508 06:40:57.548 | INFO | "huge" : "", | 20260508 06:40:57.548 | INFO | "huge_shared" : " 29 GB", | 20260508 06:40:57.548 | INFO | "malloc" : " 14 GB", | 20260508 06:40:57.548 | INFO | "pinned" : " 29 GB" | 20260508 06:40:57.548 | INFO | } | 20260508 06:40:57.548 | INFO | Perf report for the last 5.14237 seconds: | 20260508 06:40:57.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:40:57.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:40:57.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:40:57.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:40:57.548 | INFO | Uploaded 36GiB to GPU in 2.48643s (14.4786GiB/s) | 20260508 06:40:57.548 | INFO | Downloaded 34GiB to GPU in 2.45111s (13.8713GiB/s) | 20260508 06:41:02.693 | INFO | Analyzing chunk frames = [6030336,6030848) | 20260508 06:41:02.693 | INFO | MemInfo: { | 20260508 06:41:02.693 | INFO | "gpu" : " 17 GB", | 20260508 06:41:02.693 | INFO | "huge" : "", | 20260508 06:41:02.693 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:02.693 | INFO | "malloc" : " 14 GB", | 20260508 06:41:02.693 | INFO | "pinned" : " 29 GB" | 20260508 06:41:02.693 | INFO | } | 20260508 06:41:02.693 | INFO | Perf report for the last 5.1451 seconds: | 20260508 06:41:02.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:02.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.693 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 06:41:02.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.693 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:41:02.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:02.694 | INFO | Uploaded 36GiB to GPU in 2.52894s (14.2352GiB/s) | 20260508 06:41:02.694 | INFO | Downloaded 34GiB to GPU in 2.40824s (14.1182GiB/s) | 20260508 06:41:07.829 | INFO | Analyzing chunk frames = [6030848,6031360) | 20260508 06:41:07.829 | INFO | MemInfo: { | 20260508 06:41:07.829 | INFO | "gpu" : " 17 GB", | 20260508 06:41:07.829 | INFO | "huge" : "", | 20260508 06:41:07.829 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:07.829 | INFO | "malloc" : " 14 GB", | 20260508 06:41:07.829 | INFO | "pinned" : " 29 GB" | 20260508 06:41:07.829 | INFO | } | 20260508 06:41:07.829 | INFO | Perf report for the last 5.13573 seconds: | 20260508 06:41:07.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:07.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.829 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 06:41:07.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.829 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:41:07.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:07.830 | INFO | Uploaded 36GiB to GPU in 2.56754s (14.0212GiB/s) | 20260508 06:41:07.830 | INFO | Downloaded 34GiB to GPU in 2.41778s (14.0625GiB/s) | 20260508 06:41:12.971 | INFO | Analyzing chunk frames = [6031360,6031872) | 20260508 06:41:12.971 | INFO | MemInfo: { | 20260508 06:41:12.971 | INFO | "gpu" : " 17 GB", | 20260508 06:41:12.971 | INFO | "huge" : "", | 20260508 06:41:12.971 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:12.971 | INFO | "malloc" : " 14 GB", | 20260508 06:41:12.971 | INFO | "pinned" : " 29 GB" | 20260508 06:41:12.971 | INFO | } | 20260508 06:41:12.971 | INFO | Perf report for the last 5.1426 seconds: | 20260508 06:41:12.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:12.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.972 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:41:12.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.972 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:41:12.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:12.972 | INFO | Uploaded 36GiB to GPU in 2.29692s (15.6732GiB/s) | 20260508 06:41:12.972 | INFO | Downloaded 34GiB to GPU in 2.50308s (13.5833GiB/s) | 20260508 06:41:18.111 | INFO | Analyzing chunk frames = [6031872,6032384) | 20260508 06:41:18.111 | INFO | MemInfo: { | 20260508 06:41:18.111 | INFO | "gpu" : " 17 GB", | 20260508 06:41:18.111 | INFO | "huge" : "", | 20260508 06:41:18.111 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:18.111 | INFO | "malloc" : " 14 GB", | 20260508 06:41:18.111 | INFO | "pinned" : " 29 GB" | 20260508 06:41:18.111 | INFO | } | 20260508 06:41:18.111 | INFO | Perf report for the last 5.13995 seconds: | 20260508 06:41:18.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:18.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 06:41:18.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260508 06:41:18.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:18.112 | INFO | Uploaded 36GiB to GPU in 2.3497s (15.3211GiB/s) | 20260508 06:41:18.112 | INFO | Downloaded 34GiB to GPU in 2.43232s (13.9784GiB/s) | 20260508 06:41:23.248 | INFO | Analyzing chunk frames = [6032384,6032896) | 20260508 06:41:23.248 | INFO | MemInfo: { | 20260508 06:41:23.248 | INFO | "gpu" : " 17 GB", | 20260508 06:41:23.248 | INFO | "huge" : "", | 20260508 06:41:23.248 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:23.248 | INFO | "malloc" : " 14 GB", | 20260508 06:41:23.248 | INFO | "pinned" : " 29 GB" | 20260508 06:41:23.248 | INFO | } | 20260508 06:41:23.249 | INFO | Perf report for the last 5.13707 seconds: | 20260508 06:41:23.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:23.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:41:23.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:41:23.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:23.249 | INFO | Uploaded 36GiB to GPU in 2.47s (14.5749GiB/s) | 20260508 06:41:23.249 | INFO | Downloaded 34GiB to GPU in 2.43497s (13.9632GiB/s) | 20260508 06:41:28.386 | INFO | Analyzing chunk frames = [6032896,6033408) | 20260508 06:41:28.386 | INFO | MemInfo: { | 20260508 06:41:28.386 | INFO | "gpu" : " 17 GB", | 20260508 06:41:28.386 | INFO | "huge" : "", | 20260508 06:41:28.386 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:28.386 | INFO | "malloc" : " 14 GB", | 20260508 06:41:28.386 | INFO | "pinned" : " 29 GB" | 20260508 06:41:28.386 | INFO | } | 20260508 06:41:28.386 | INFO | Perf report for the last 5.13763 seconds: | 20260508 06:41:28.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:28.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.386 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:41:28.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.386 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:41:28.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:28.387 | INFO | Uploaded 36GiB to GPU in 2.47259s (14.5596GiB/s) | 20260508 06:41:28.387 | INFO | Downloaded 34GiB to GPU in 2.45016s (13.8767GiB/s) | 20260508 06:41:28.484 | INFO | Starting a round of Baz IO | 20260508 06:41:31.909 | INFO | Populating RTMetrics report | 20260508 06:41:31.910 | INFO | RTMetrics report populated | 20260508 06:41:31.910 | INFO | Opening tmp RTMetrics json file | 20260508 06:41:31.922 | INFO | Serializing RTMetrics to json | 20260508 06:41:31.923 | INFO | Serialization done, swapping tmp file now... | 20260508 06:41:31.923 | INFO | RTMetrics done | 20260508 06:41:31.984 | INFO | Finished a round of Baz IO: Wrote 1.545 GiB in 3.499 seconds | 20260508 06:41:31.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.66% / 0.00% / 59.33% | 20260508 06:41:31.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.384259% / 0% / 6.75519e-05% | 20260508 06:41:31.984 | INFO | Baz IO overall: | 20260508 06:41:31.984 | INFO | BytesWritten : 2672388053873 | 20260508 06:41:31.984 | INFO | NumEvents : 1288251991613 | 20260508 06:41:33.524 | INFO | Analyzing chunk frames = [6033408,6033920) | 20260508 06:41:33.524 | INFO | MemInfo: { | 20260508 06:41:33.524 | INFO | "gpu" : " 17 GB", | 20260508 06:41:33.524 | INFO | "huge" : "", | 20260508 06:41:33.524 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:33.524 | INFO | "malloc" : " 14 GB", | 20260508 06:41:33.524 | INFO | "pinned" : " 29 GB" | 20260508 06:41:33.524 | INFO | } | 20260508 06:41:33.524 | INFO | Perf report for the last 5.13753 seconds: | 20260508 06:41:33.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:33.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:33.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:33.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:33.524 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:41:33.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:33.524 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 06:41:33.524 | INFO | PRE_HQ : 65.5%, 1.6, 55.2ms, 0 | 20260508 06:41:33.524 | INFO | BAZWRITER : 67.6%, 2.6, 91.0ms, 0 | 20260508 06:41:33.524 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:41:33.524 | INFO | Uploaded 36GiB to GPU in 2.60055s (13.8432GiB/s) | 20260508 06:41:33.524 | INFO | Downloaded 38GiB to GPU in 2.687s (14.1422GiB/s) | 20260508 06:41:38.660 | INFO | Analyzing chunk frames = [6033920,6034432) | 20260508 06:41:38.660 | INFO | MemInfo: { | 20260508 06:41:38.660 | INFO | "gpu" : " 17 GB", | 20260508 06:41:38.660 | INFO | "huge" : "", | 20260508 06:41:38.660 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:38.660 | INFO | "malloc" : " 14 GB", | 20260508 06:41:38.660 | INFO | "pinned" : " 29 GB" | 20260508 06:41:38.660 | INFO | } | 20260508 06:41:38.661 | INFO | Perf report for the last 5.13675 seconds: | 20260508 06:41:38.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:38.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 06:41:38.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:41:38.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:38.661 | INFO | Uploaded 36GiB to GPU in 2.51096s (14.3372GiB/s) | 20260508 06:41:38.661 | INFO | Downloaded 34GiB to GPU in 2.43972s (13.936GiB/s) | 20260508 06:41:43.805 | INFO | Analyzing chunk frames = [6034432,6034944) | 20260508 06:41:43.805 | INFO | MemInfo: { | 20260508 06:41:43.805 | INFO | "gpu" : " 17 GB", | 20260508 06:41:43.805 | INFO | "huge" : "", | 20260508 06:41:43.805 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:43.806 | INFO | "malloc" : " 14 GB", | 20260508 06:41:43.806 | INFO | "pinned" : " 29 GB" | 20260508 06:41:43.806 | INFO | } | 20260508 06:41:43.806 | INFO | Perf report for the last 5.14507 seconds: | 20260508 06:41:43.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:43.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 06:41:43.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:41:43.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:43.806 | INFO | Uploaded 36GiB to GPU in 2.55077s (14.1134GiB/s) | 20260508 06:41:43.806 | INFO | Downloaded 34GiB to GPU in 2.41479s (14.0799GiB/s) | 20260508 06:41:48.948 | INFO | Analyzing chunk frames = [6034944,6035456) | 20260508 06:41:48.948 | INFO | MemInfo: { | 20260508 06:41:48.948 | INFO | "gpu" : " 17 GB", | 20260508 06:41:48.948 | INFO | "huge" : "", | 20260508 06:41:48.948 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:48.948 | INFO | "malloc" : " 14 GB", | 20260508 06:41:48.948 | INFO | "pinned" : " 29 GB" | 20260508 06:41:48.948 | INFO | } | 20260508 06:41:48.948 | INFO | Perf report for the last 5.14239 seconds: | 20260508 06:41:48.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:48.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 06:41:48.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:41:48.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:48.948 | INFO | Uploaded 36GiB to GPU in 2.58759s (13.9126GiB/s) | 20260508 06:41:48.948 | INFO | Downloaded 34GiB to GPU in 2.41317s (14.0894GiB/s) | 20260508 06:41:54.083 | INFO | Analyzing chunk frames = [6035456,6035968) | 20260508 06:41:54.083 | INFO | MemInfo: { | 20260508 06:41:54.083 | INFO | "gpu" : " 17 GB", | 20260508 06:41:54.083 | INFO | "huge" : "", | 20260508 06:41:54.083 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:54.083 | INFO | "malloc" : " 14 GB", | 20260508 06:41:54.083 | INFO | "pinned" : " 29 GB" | 20260508 06:41:54.083 | INFO | } | 20260508 06:41:54.083 | INFO | Perf report for the last 5.13518 seconds: | 20260508 06:41:54.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:54.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.083 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:41:54.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.083 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 06:41:54.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:54.084 | INFO | Uploaded 36GiB to GPU in 2.28713s (15.7403GiB/s) | 20260508 06:41:54.084 | INFO | Downloaded 34GiB to GPU in 2.49499s (13.6273GiB/s) | 20260508 06:41:59.219 | INFO | Analyzing chunk frames = [6035968,6036480) | 20260508 06:41:59.219 | INFO | MemInfo: { | 20260508 06:41:59.219 | INFO | "gpu" : " 17 GB", | 20260508 06:41:59.219 | INFO | "huge" : "", | 20260508 06:41:59.219 | INFO | "huge_shared" : " 29 GB", | 20260508 06:41:59.219 | INFO | "malloc" : " 14 GB", | 20260508 06:41:59.220 | INFO | "pinned" : " 29 GB" | 20260508 06:41:59.220 | INFO | } | 20260508 06:41:59.220 | INFO | Perf report for the last 5.13649 seconds: | 20260508 06:41:59.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:41:59.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:41:59.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:41:59.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:41:59.220 | INFO | Uploaded 36GiB to GPU in 2.4777s (14.5296GiB/s) | 20260508 06:41:59.220 | INFO | Downloaded 34GiB to GPU in 2.44911s (13.8826GiB/s) | 20260508 06:42:04.363 | INFO | Analyzing chunk frames = [6036480,6036992) | 20260508 06:42:04.363 | INFO | MemInfo: { | 20260508 06:42:04.363 | INFO | "gpu" : " 17 GB", | 20260508 06:42:04.363 | INFO | "huge" : "", | 20260508 06:42:04.363 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:04.363 | INFO | "malloc" : " 14 GB", | 20260508 06:42:04.363 | INFO | "pinned" : " 29 GB" | 20260508 06:42:04.363 | INFO | } | 20260508 06:42:04.363 | INFO | Perf report for the last 5.14371 seconds: | 20260508 06:42:04.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:04.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.363 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 06:42:04.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.364 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:42:04.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:04.364 | INFO | Uploaded 36GiB to GPU in 2.45625s (14.6565GiB/s) | 20260508 06:42:04.364 | INFO | Downloaded 34GiB to GPU in 2.43064s (13.9881GiB/s) | 20260508 06:42:09.504 | INFO | Analyzing chunk frames = [6036992,6037504) | 20260508 06:42:09.504 | INFO | MemInfo: { | 20260508 06:42:09.504 | INFO | "gpu" : " 17 GB", | 20260508 06:42:09.504 | INFO | "huge" : "", | 20260508 06:42:09.504 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:09.504 | INFO | "malloc" : " 14 GB", | 20260508 06:42:09.504 | INFO | "pinned" : " 29 GB" | 20260508 06:42:09.504 | INFO | } | 20260508 06:42:09.505 | INFO | Perf report for the last 5.14115 seconds: | 20260508 06:42:09.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:09.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:42:09.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:42:09.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:09.505 | INFO | Uploaded 36GiB to GPU in 2.47117s (14.568GiB/s) | 20260508 06:42:09.505 | INFO | Downloaded 34GiB to GPU in 2.44285s (13.9182GiB/s) | 20260508 06:42:09.505 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:42:09.505 | INFO | Total mem usage: 16.7 GiB | 20260508 06:42:09.505 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:42:09.505 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:42:09.505 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:42:09.505 | INFO | High Water Breakdown: | 20260508 06:42:09.505 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:42:09.505 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:42:09.505 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:42:09.505 | INFO | ------------------------------------------ | 20260508 06:42:09.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:42:09.505 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:42:09.505 | INFO | Total mem usage: 26.5 GiB | 20260508 06:42:09.505 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:42:09.505 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:42:09.505 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:42:09.505 | INFO | High Water Breakdown: | 20260508 06:42:09.505 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:42:09.505 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:42:09.505 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:42:09.505 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:42:09.505 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:42:09.505 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:42:09.505 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:42:09.505 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:42:09.505 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:42:09.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:42:09.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:42:09.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:42:09.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:42:09.505 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:42:09.505 | INFO | ------------------------------------------ | 20260508 06:42:09.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:42:09.506 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:42:09.506 | INFO | Total mem usage: 27.8 GiB | 20260508 06:42:09.506 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:42:09.506 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:42:09.506 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:42:09.506 | INFO | High Water Breakdown: | 20260508 06:42:09.506 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:42:09.506 | INFO | ------------------------------------------ | 20260508 06:42:09.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:42:09.506 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:42:09.506 | INFO | Total mem usage: 15.7 GiB | 20260508 06:42:09.506 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:42:09.506 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:42:09.506 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:42:09.506 | INFO | High Water Breakdown: | 20260508 06:42:09.506 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:42:09.506 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:42:09.506 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:42:09.506 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:42:09.506 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:42:09.506 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:42:09.506 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:42:09.506 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:42:09.506 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:42:09.506 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:42:09.506 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:42:09.506 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:42:09.506 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:42:09.506 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:42:09.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:42:09.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:42:09.506 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:42:09.506 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:42:09.506 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:42:09.506 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:42:09.506 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:42:09.506 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:42:09.506 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:42:09.506 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:42:09.506 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:42:09.506 | INFO | ------------------------------------------ | 20260508 06:42:09.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:42:09.506 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:42:09.506 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:42:09.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:42:09.506 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.14e-01 | 20260508 06:42:09.506 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.13e-02 / 6.57e-03 | 20260508 06:42:09.506 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.42e+00, 5.86e-01 / 2.53e-01 | 20260508 06:42:09.506 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.23e-01 | 20260508 06:42:09.506 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.55e+01, 3.98e+00 / 1.14e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.79e+00 / 1.27e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.37e+00 / 2.64e+00, 4.12e-01 / 1.85e-01 | 20260508 06:42:09.506 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.24e+00, 2.52e-01 / 1.05e-01 | 20260508 06:42:09.506 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.12e-01 / 3.48e-01, 4.95e-02 / 1.82e-02 | 20260508 06:42:09.506 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:42:09.506 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:42:09.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:42:09.506 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 24.35%, Upload -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 24.30%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 30.80%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 2.32%, LockUpload -- 2.46e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 2.55%, LockDownload -- 2.70e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 15.67%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:42:09.506 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 06:42:09.602 | INFO | Starting a round of Baz IO | 20260508 06:42:13.030 | INFO | Populating RTMetrics report | 20260508 06:42:13.030 | INFO | RTMetrics report populated | 20260508 06:42:13.030 | INFO | Opening tmp RTMetrics json file | 20260508 06:42:13.040 | INFO | Serializing RTMetrics to json | 20260508 06:42:13.041 | INFO | Serialization done, swapping tmp file now... | 20260508 06:42:13.042 | INFO | RTMetrics done | 20260508 06:42:13.103 | INFO | Finished a round of Baz IO: Wrote 1.543 GiB in 3.501 seconds | 20260508 06:42:13.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.66% / 0.00% / 59.34% | 20260508 06:42:13.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.382857% / 7.94729e-06% / 7.15256e-05% | 20260508 06:42:13.103 | INFO | Baz IO overall: | 20260508 06:42:13.103 | INFO | BytesWritten : 2674045142047 | 20260508 06:42:13.103 | INFO | NumEvents : 1289039884185 | 20260508 06:42:14.649 | INFO | Analyzing chunk frames = [6037504,6038016) | 20260508 06:42:14.649 | INFO | MemInfo: { | 20260508 06:42:14.649 | INFO | "gpu" : " 17 GB", | 20260508 06:42:14.649 | INFO | "huge" : "", | 20260508 06:42:14.649 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:14.649 | INFO | "malloc" : " 14 GB", | 20260508 06:42:14.649 | INFO | "pinned" : " 29 GB" | 20260508 06:42:14.649 | INFO | } | 20260508 06:42:14.649 | INFO | Perf report for the last 5.14429 seconds: | 20260508 06:42:14.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:14.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:14.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:14.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:14.649 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 06:42:14.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:14.649 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:42:14.649 | INFO | PRE_HQ : 65.6%, 1.6, 54.8ms, 0 | 20260508 06:42:14.649 | INFO | BAZWRITER : 67.5%, 2.6, 90.9ms, 0 | 20260508 06:42:14.649 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:42:14.649 | INFO | Uploaded 36GiB to GPU in 2.57144s (14GiB/s) | 20260508 06:42:14.649 | INFO | Downloaded 38GiB to GPU in 2.72984s (13.9202GiB/s) | 20260508 06:42:19.781 | INFO | Analyzing chunk frames = [6038016,6038528) | 20260508 06:42:19.781 | INFO | MemInfo: { | 20260508 06:42:19.781 | INFO | "gpu" : " 17 GB", | 20260508 06:42:19.781 | INFO | "huge" : "", | 20260508 06:42:19.781 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:19.781 | INFO | "malloc" : " 14 GB", | 20260508 06:42:19.781 | INFO | "pinned" : " 29 GB" | 20260508 06:42:19.781 | INFO | } | 20260508 06:42:19.781 | INFO | Perf report for the last 5.13243 seconds: | 20260508 06:42:19.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:19.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.781 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:42:19.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.781 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:42:19.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:19.782 | INFO | Uploaded 36GiB to GPU in 2.48668s (14.4771GiB/s) | 20260508 06:42:19.782 | INFO | Downloaded 34GiB to GPU in 2.45245s (13.8637GiB/s) | 20260508 06:42:24.918 | INFO | Analyzing chunk frames = [6038528,6039040) | 20260508 06:42:24.918 | INFO | MemInfo: { | 20260508 06:42:24.918 | INFO | "gpu" : " 17 GB", | 20260508 06:42:24.918 | INFO | "huge" : "", | 20260508 06:42:24.918 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:24.918 | INFO | "malloc" : " 14 GB", | 20260508 06:42:24.918 | INFO | "pinned" : " 29 GB" | 20260508 06:42:24.918 | INFO | } | 20260508 06:42:24.918 | INFO | Perf report for the last 5.13694 seconds: | 20260508 06:42:24.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:24.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.918 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 06:42:24.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.918 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:42:24.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:24.919 | INFO | Uploaded 36GiB to GPU in 2.53064s (14.2257GiB/s) | 20260508 06:42:24.919 | INFO | Downloaded 34GiB to GPU in 2.44264s (13.9194GiB/s) | 20260508 06:42:30.058 | INFO | Analyzing chunk frames = [6039040,6039552) | 20260508 06:42:30.058 | INFO | MemInfo: { | 20260508 06:42:30.058 | INFO | "gpu" : " 17 GB", | 20260508 06:42:30.058 | INFO | "huge" : "", | 20260508 06:42:30.058 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:30.058 | INFO | "malloc" : " 14 GB", | 20260508 06:42:30.058 | INFO | "pinned" : " 29 GB" | 20260508 06:42:30.058 | INFO | } | 20260508 06:42:30.058 | INFO | Perf report for the last 5.14012 seconds: | 20260508 06:42:30.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:30.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.058 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 06:42:30.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.058 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:42:30.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:30.059 | INFO | Uploaded 36GiB to GPU in 2.48862s (14.4659GiB/s) | 20260508 06:42:30.059 | INFO | Downloaded 34GiB to GPU in 2.40984s (14.1088GiB/s) | 20260508 06:42:35.197 | INFO | Analyzing chunk frames = [6039552,6040064) | 20260508 06:42:35.197 | INFO | MemInfo: { | 20260508 06:42:35.197 | INFO | "gpu" : " 17 GB", | 20260508 06:42:35.197 | INFO | "huge" : "", | 20260508 06:42:35.197 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:35.197 | INFO | "malloc" : " 14 GB", | 20260508 06:42:35.197 | INFO | "pinned" : " 29 GB" | 20260508 06:42:35.197 | INFO | } | 20260508 06:42:35.197 | INFO | Perf report for the last 5.13853 seconds: | 20260508 06:42:35.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:35.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:42:35.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:42:35.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:35.197 | INFO | Uploaded 36GiB to GPU in 2.37187s (15.1779GiB/s) | 20260508 06:42:35.197 | INFO | Downloaded 34GiB to GPU in 2.52839s (13.4473GiB/s) | 20260508 06:42:40.337 | INFO | Analyzing chunk frames = [6040064,6040576) | 20260508 06:42:40.337 | INFO | MemInfo: { | 20260508 06:42:40.337 | INFO | "gpu" : " 17 GB", | 20260508 06:42:40.337 | INFO | "huge" : "", | 20260508 06:42:40.337 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:40.337 | INFO | "malloc" : " 14 GB", | 20260508 06:42:40.338 | INFO | "pinned" : " 29 GB" | 20260508 06:42:40.338 | INFO | } | 20260508 06:42:40.338 | INFO | Perf report for the last 5.14125 seconds: | 20260508 06:42:40.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:40.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.338 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 06:42:40.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.338 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:42:40.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:40.339 | INFO | Uploaded 36GiB to GPU in 2.43144s (14.8061GiB/s) | 20260508 06:42:40.339 | INFO | Downloaded 34GiB to GPU in 2.43258s (13.977GiB/s) | 20260508 06:42:45.473 | INFO | Analyzing chunk frames = [6040576,6041088) | 20260508 06:42:45.473 | INFO | MemInfo: { | 20260508 06:42:45.473 | INFO | "gpu" : " 17 GB", | 20260508 06:42:45.473 | INFO | "huge" : "", | 20260508 06:42:45.473 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:45.473 | INFO | "malloc" : " 14 GB", | 20260508 06:42:45.474 | INFO | "pinned" : " 29 GB" | 20260508 06:42:45.474 | INFO | } | 20260508 06:42:45.474 | INFO | Perf report for the last 5.1355 seconds: | 20260508 06:42:45.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:45.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:42:45.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:42:45.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:45.474 | INFO | Uploaded 36GiB to GPU in 2.44185s (14.7429GiB/s) | 20260508 06:42:45.474 | INFO | Downloaded 34GiB to GPU in 2.44331s (13.9155GiB/s) | 20260508 06:42:50.617 | INFO | Analyzing chunk frames = [6041088,6041600) | 20260508 06:42:50.617 | INFO | MemInfo: { | 20260508 06:42:50.617 | INFO | "gpu" : " 17 GB", | 20260508 06:42:50.617 | INFO | "huge" : "", | 20260508 06:42:50.617 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:50.617 | INFO | "malloc" : " 14 GB", | 20260508 06:42:50.617 | INFO | "pinned" : " 29 GB" | 20260508 06:42:50.617 | INFO | } | 20260508 06:42:50.617 | INFO | Perf report for the last 5.14349 seconds: | 20260508 06:42:50.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:50.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:42:50.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:42:50.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:50.617 | INFO | Uploaded 36GiB to GPU in 2.43043s (14.8122GiB/s) | 20260508 06:42:50.617 | INFO | Downloaded 34GiB to GPU in 2.43498s (13.9631GiB/s) | 20260508 06:42:50.714 | INFO | Starting a round of Baz IO | 20260508 06:42:54.137 | INFO | Populating RTMetrics report | 20260508 06:42:54.137 | INFO | RTMetrics report populated | 20260508 06:42:54.137 | INFO | Opening tmp RTMetrics json file | 20260508 06:42:54.148 | INFO | Serializing RTMetrics to json | 20260508 06:42:54.149 | INFO | Serialization done, swapping tmp file now... | 20260508 06:42:54.150 | INFO | RTMetrics done | 20260508 06:42:54.205 | INFO | Finished a round of Baz IO: Wrote 1.544 GiB in 3.491 seconds | 20260508 06:42:54.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.66% / 0.00% / 59.34% | 20260508 06:42:54.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.382741% / 0% / 5.16574e-05% | 20260508 06:42:54.206 | INFO | Baz IO overall: | 20260508 06:42:54.206 | INFO | BytesWritten : 2675703475201 | 20260508 06:42:54.206 | INFO | NumEvents : 1289829022210 | 20260508 06:42:55.757 | INFO | Analyzing chunk frames = [6041600,6042112) | 20260508 06:42:55.757 | INFO | MemInfo: { | 20260508 06:42:55.757 | INFO | "gpu" : " 17 GB", | 20260508 06:42:55.757 | INFO | "huge" : "", | 20260508 06:42:55.757 | INFO | "huge_shared" : " 29 GB", | 20260508 06:42:55.757 | INFO | "malloc" : " 14 GB", | 20260508 06:42:55.757 | INFO | "pinned" : " 29 GB" | 20260508 06:42:55.757 | INFO | } | 20260508 06:42:55.757 | INFO | Perf report for the last 5.13957 seconds: | 20260508 06:42:55.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:42:55.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:55.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:55.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:55.757 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 06:42:55.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:42:55.757 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 06:42:55.757 | INFO | PRE_HQ : 65.6%, 1.6, 55.2ms, 0 | 20260508 06:42:55.757 | INFO | BAZWRITER : 67.3%, 2.6, 91.5ms, 0 | 20260508 06:42:55.757 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:42:55.757 | INFO | Uploaded 36GiB to GPU in 2.53031s (14.2275GiB/s) | 20260508 06:42:55.757 | INFO | Downloaded 38GiB to GPU in 2.69936s (14.0774GiB/s) | 20260508 06:43:00.893 | INFO | Analyzing chunk frames = [6042112,6042624) | 20260508 06:43:00.893 | INFO | MemInfo: { | 20260508 06:43:00.893 | INFO | "gpu" : " 17 GB", | 20260508 06:43:00.893 | INFO | "huge" : "", | 20260508 06:43:00.893 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:00.893 | INFO | "malloc" : " 14 GB", | 20260508 06:43:00.893 | INFO | "pinned" : " 29 GB" | 20260508 06:43:00.893 | INFO | } | 20260508 06:43:00.893 | INFO | Perf report for the last 5.13631 seconds: | 20260508 06:43:00.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:00.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 06:43:00.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260508 06:43:00.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:00.893 | INFO | Uploaded 36GiB to GPU in 2.48928s (14.462GiB/s) | 20260508 06:43:00.893 | INFO | Downloaded 34GiB to GPU in 2.44409s (13.9111GiB/s) | 20260508 06:43:06.035 | INFO | Analyzing chunk frames = [6042624,6043136) | 20260508 06:43:06.035 | INFO | MemInfo: { | 20260508 06:43:06.035 | INFO | "gpu" : " 17 GB", | 20260508 06:43:06.035 | INFO | "huge" : "", | 20260508 06:43:06.035 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:06.035 | INFO | "malloc" : " 14 GB", | 20260508 06:43:06.035 | INFO | "pinned" : " 29 GB" | 20260508 06:43:06.035 | INFO | } | 20260508 06:43:06.035 | INFO | Perf report for the last 5.14225 seconds: | 20260508 06:43:06.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:06.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.035 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:43:06.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.036 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:43:06.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:06.036 | INFO | Uploaded 36GiB to GPU in 2.52681s (14.2472GiB/s) | 20260508 06:43:06.036 | INFO | Downloaded 34GiB to GPU in 2.41833s (14.0593GiB/s) | 20260508 06:43:11.169 | INFO | Analyzing chunk frames = [6043136,6043648) | 20260508 06:43:11.169 | INFO | MemInfo: { | 20260508 06:43:11.169 | INFO | "gpu" : " 17 GB", | 20260508 06:43:11.169 | INFO | "huge" : "", | 20260508 06:43:11.169 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:11.169 | INFO | "malloc" : " 14 GB", | 20260508 06:43:11.169 | INFO | "pinned" : " 29 GB" | 20260508 06:43:11.169 | INFO | } | 20260508 06:43:11.170 | INFO | Perf report for the last 5.13414 seconds: | 20260508 06:43:11.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:11.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 06:43:11.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:43:11.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:11.170 | INFO | Uploaded 36GiB to GPU in 2.56128s (14.0555GiB/s) | 20260508 06:43:11.170 | INFO | Downloaded 34GiB to GPU in 2.42766s (14.0052GiB/s) | 20260508 06:43:16.316 | INFO | Analyzing chunk frames = [6043648,6044160) | 20260508 06:43:16.316 | INFO | MemInfo: { | 20260508 06:43:16.316 | INFO | "gpu" : " 17 GB", | 20260508 06:43:16.316 | INFO | "huge" : "", | 20260508 06:43:16.316 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:16.316 | INFO | "malloc" : " 14 GB", | 20260508 06:43:16.317 | INFO | "pinned" : " 29 GB" | 20260508 06:43:16.317 | INFO | } | 20260508 06:43:16.317 | INFO | Perf report for the last 5.14707 seconds: | 20260508 06:43:16.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:16.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | ANALYSIS : 66.8%, 3.0, 104.4ms, 0 | 20260508 06:43:16.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:43:16.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:16.317 | INFO | Uploaded 36GiB to GPU in 2.47707s (14.5333GiB/s) | 20260508 06:43:16.317 | INFO | Downloaded 34GiB to GPU in 2.54445s (13.3624GiB/s) | 20260508 06:43:21.458 | INFO | Analyzing chunk frames = [6044160,6044672) | 20260508 06:43:21.458 | INFO | MemInfo: { | 20260508 06:43:21.458 | INFO | "gpu" : " 17 GB", | 20260508 06:43:21.458 | INFO | "huge" : "", | 20260508 06:43:21.458 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:21.458 | INFO | "malloc" : " 14 GB", | 20260508 06:43:21.458 | INFO | "pinned" : " 29 GB" | 20260508 06:43:21.458 | INFO | } | 20260508 06:43:21.458 | INFO | Perf report for the last 5.14116 seconds: | 20260508 06:43:21.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:21.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 06:43:21.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:43:21.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:21.458 | INFO | Uploaded 36GiB to GPU in 2.35024s (15.3176GiB/s) | 20260508 06:43:21.458 | INFO | Downloaded 34GiB to GPU in 2.45411s (13.8543GiB/s) | 20260508 06:43:26.598 | INFO | Analyzing chunk frames = [6044672,6045184) | 20260508 06:43:26.598 | INFO | MemInfo: { | 20260508 06:43:26.598 | INFO | "gpu" : " 17 GB", | 20260508 06:43:26.598 | INFO | "huge" : "", | 20260508 06:43:26.598 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:26.598 | INFO | "malloc" : " 14 GB", | 20260508 06:43:26.598 | INFO | "pinned" : " 29 GB" | 20260508 06:43:26.598 | INFO | } | 20260508 06:43:26.598 | INFO | Perf report for the last 5.14052 seconds: | 20260508 06:43:26.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:26.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.598 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 06:43:26.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.598 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:43:26.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:26.599 | INFO | Uploaded 36GiB to GPU in 2.43228s (14.8009GiB/s) | 20260508 06:43:26.599 | INFO | Downloaded 34GiB to GPU in 2.43992s (13.9349GiB/s) | 20260508 06:43:31.730 | INFO | Analyzing chunk frames = [6045184,6045696) | 20260508 06:43:31.730 | INFO | MemInfo: { | 20260508 06:43:31.730 | INFO | "gpu" : " 17 GB", | 20260508 06:43:31.730 | INFO | "huge" : "", | 20260508 06:43:31.730 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:31.730 | INFO | "malloc" : " 14 GB", | 20260508 06:43:31.731 | INFO | "pinned" : " 29 GB" | 20260508 06:43:31.731 | INFO | } | 20260508 06:43:31.731 | INFO | Perf report for the last 5.13235 seconds: | 20260508 06:43:31.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:31.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 06:43:31.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:43:31.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:31.731 | INFO | Uploaded 36GiB to GPU in 2.48002s (14.516GiB/s) | 20260508 06:43:31.732 | INFO | Downloaded 34GiB to GPU in 2.46241s (13.8076GiB/s) | 20260508 06:43:31.732 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:43:31.732 | INFO | Total mem usage: 16.7 GiB | 20260508 06:43:31.732 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:43:31.732 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:43:31.732 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:43:31.732 | INFO | High Water Breakdown: | 20260508 06:43:31.732 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:43:31.732 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:43:31.732 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:43:31.732 | INFO | ------------------------------------------ | 20260508 06:43:31.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:43:31.732 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:43:31.732 | INFO | Total mem usage: 26.5 GiB | 20260508 06:43:31.732 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:43:31.732 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 06:43:31.732 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 06:43:31.732 | INFO | High Water Breakdown: | 20260508 06:43:31.732 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:43:31.732 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:43:31.732 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:43:31.732 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:43:31.732 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:43:31.732 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:43:31.732 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:43:31.732 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:43:31.732 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:43:31.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:43:31.732 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:43:31.732 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:43:31.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:43:31.732 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:43:31.732 | INFO | ------------------------------------------ | 20260508 06:43:31.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:43:31.732 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:43:31.732 | INFO | Total mem usage: 27.8 GiB | 20260508 06:43:31.732 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:43:31.732 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:43:31.732 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:43:31.732 | INFO | High Water Breakdown: | 20260508 06:43:31.732 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:43:31.732 | INFO | ------------------------------------------ | 20260508 06:43:31.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:43:31.732 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:43:31.732 | INFO | Total mem usage: 15.7 GiB | 20260508 06:43:31.732 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:43:31.732 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:43:31.732 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:43:31.732 | INFO | High Water Breakdown: | 20260508 06:43:31.732 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:43:31.732 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:43:31.732 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:43:31.732 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:43:31.732 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:43:31.732 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:43:31.732 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:43:31.732 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:43:31.732 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:43:31.732 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:43:31.732 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:43:31.732 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:43:31.733 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:43:31.733 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:43:31.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:43:31.733 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:43:31.733 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:43:31.733 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:43:31.733 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:43:31.733 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:43:31.733 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:43:31.733 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:43:31.733 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:43:31.733 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:43:31.733 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:43:31.733 | INFO | ------------------------------------------ | 20260508 06:43:31.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:43:31.733 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:43:31.733 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:43:31.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:43:31.733 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.24e-01 | 20260508 06:43:31.733 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.26e-02 / 2.83e-03 | 20260508 06:43:31.733 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.41e+00, 5.79e-01 / 1.87e-01 | 20260508 06:43:31.733 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.19e-01 | 20260508 06:43:31.733 | INFO | PerfReport: 15.86%, DME -- 4.15e+01 / 4.47e+01, 3.78e+00 / 1.14e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.85e+00 / 1.43e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.54e+00, 4.10e-01 / 2.36e-01 | 20260508 06:43:31.733 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.23e+00, 2.50e-01 / 1.30e-01 | 20260508 06:43:31.733 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.45e-01, 4.96e-02 / 2.68e-02 | 20260508 06:43:31.733 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:43:31.733 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:43:31.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:43:31.733 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 24.33%, Upload -- 2.58e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 24.43%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 2.08%, LockUpload -- 2.20e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 2.69%, LockDownload -- 2.85e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 15.59%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:43:31.733 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260508 06:43:31.846 | INFO | Starting a round of Baz IO | 20260508 06:43:35.239 | INFO | Populating RTMetrics report | 20260508 06:43:35.239 | INFO | RTMetrics report populated | 20260508 06:43:35.239 | INFO | Opening tmp RTMetrics json file | 20260508 06:43:35.252 | INFO | Serializing RTMetrics to json | 20260508 06:43:35.253 | INFO | Serialization done, swapping tmp file now... | 20260508 06:43:35.254 | INFO | RTMetrics done | 20260508 06:43:35.312 | INFO | Finished a round of Baz IO: Wrote 1.542 GiB in 3.466 seconds | 20260508 06:43:35.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.66% / 0.00% / 59.34% | 20260508 06:43:35.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.381887% / 7.94729e-06% / 4.76837e-05% | 20260508 06:43:35.312 | INFO | Baz IO overall: | 20260508 06:43:35.312 | INFO | BytesWritten : 2677359698824 | 20260508 06:43:35.312 | INFO | NumEvents : 1290616658598 | 20260508 06:43:36.870 | INFO | Analyzing chunk frames = [6045696,6046208) | 20260508 06:43:36.870 | INFO | MemInfo: { | 20260508 06:43:36.870 | INFO | "gpu" : " 17 GB", | 20260508 06:43:36.870 | INFO | "huge" : "", | 20260508 06:43:36.870 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:36.870 | INFO | "malloc" : " 14 GB", | 20260508 06:43:36.870 | INFO | "pinned" : " 29 GB" | 20260508 06:43:36.870 | INFO | } | 20260508 06:43:36.870 | INFO | Perf report for the last 5.13953 seconds: | 20260508 06:43:36.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:36.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:36.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:36.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:36.870 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 06:43:36.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:36.870 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 06:43:36.870 | INFO | PRE_HQ : 66.0%, 1.7, 59.5ms, 0 | 20260508 06:43:36.870 | INFO | BAZWRITER : 66.7%, 2.8, 94.6ms, 0 | 20260508 06:43:36.870 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:43:36.871 | INFO | Uploaded 36GiB to GPU in 2.57525s (13.9792GiB/s) | 20260508 06:43:36.871 | INFO | Downloaded 38GiB to GPU in 2.69614s (14.0942GiB/s) | 20260508 06:43:42.009 | INFO | Analyzing chunk frames = [6046208,6046720) | 20260508 06:43:42.009 | INFO | MemInfo: { | 20260508 06:43:42.009 | INFO | "gpu" : " 17 GB", | 20260508 06:43:42.009 | INFO | "huge" : "", | 20260508 06:43:42.009 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:42.009 | INFO | "malloc" : " 14 GB", | 20260508 06:43:42.009 | INFO | "pinned" : " 29 GB" | 20260508 06:43:42.009 | INFO | } | 20260508 06:43:42.009 | INFO | Perf report for the last 5.13902 seconds: | 20260508 06:43:42.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:42.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.009 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:43:42.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.009 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:43:42.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:42.010 | INFO | Uploaded 36GiB to GPU in 2.47662s (14.5359GiB/s) | 20260508 06:43:42.010 | INFO | Downloaded 34GiB to GPU in 2.43435s (13.9668GiB/s) | 20260508 06:43:47.147 | INFO | Analyzing chunk frames = [6046720,6047232) | 20260508 06:43:47.147 | INFO | MemInfo: { | 20260508 06:43:47.147 | INFO | "gpu" : " 17 GB", | 20260508 06:43:47.147 | INFO | "huge" : "", | 20260508 06:43:47.147 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:47.147 | INFO | "malloc" : " 14 GB", | 20260508 06:43:47.147 | INFO | "pinned" : " 29 GB" | 20260508 06:43:47.147 | INFO | } | 20260508 06:43:47.147 | INFO | Perf report for the last 5.13822 seconds: | 20260508 06:43:47.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:47.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.148 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 06:43:47.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.148 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:43:47.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:47.148 | INFO | Uploaded 36GiB to GPU in 2.53723s (14.1887GiB/s) | 20260508 06:43:47.148 | INFO | Downloaded 34GiB to GPU in 2.41097s (14.1022GiB/s) | 20260508 06:43:52.288 | INFO | Analyzing chunk frames = [6047232,6047744) | 20260508 06:43:52.288 | INFO | MemInfo: { | 20260508 06:43:52.288 | INFO | "gpu" : " 17 GB", | 20260508 06:43:52.288 | INFO | "huge" : "", | 20260508 06:43:52.289 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:52.289 | INFO | "malloc" : " 14 GB", | 20260508 06:43:52.289 | INFO | "pinned" : " 29 GB" | 20260508 06:43:52.289 | INFO | } | 20260508 06:43:52.289 | INFO | Perf report for the last 5.14123 seconds: | 20260508 06:43:52.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:52.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 06:43:52.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:43:52.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:52.289 | INFO | Uploaded 36GiB to GPU in 2.54772s (14.1303GiB/s) | 20260508 06:43:52.289 | INFO | Downloaded 34GiB to GPU in 2.40331s (14.1471GiB/s) | 20260508 06:43:57.426 | INFO | Analyzing chunk frames = [6047744,6048256) | 20260508 06:43:57.426 | INFO | MemInfo: { | 20260508 06:43:57.426 | INFO | "gpu" : " 17 GB", | 20260508 06:43:57.426 | INFO | "huge" : "", | 20260508 06:43:57.426 | INFO | "huge_shared" : " 29 GB", | 20260508 06:43:57.426 | INFO | "malloc" : " 14 GB", | 20260508 06:43:57.426 | INFO | "pinned" : " 29 GB" | 20260508 06:43:57.426 | INFO | } | 20260508 06:43:57.426 | INFO | Perf report for the last 5.13711 seconds: | 20260508 06:43:57.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:43:57.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:43:57.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:43:57.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:43:57.426 | INFO | Uploaded 36GiB to GPU in 2.36244s (15.2385GiB/s) | 20260508 06:43:57.426 | INFO | Downloaded 34GiB to GPU in 2.51778s (13.504GiB/s) | 20260508 06:44:02.572 | INFO | Analyzing chunk frames = [6048256,6048768) | 20260508 06:44:02.572 | INFO | MemInfo: { | 20260508 06:44:02.572 | INFO | "gpu" : " 17 GB", | 20260508 06:44:02.572 | INFO | "huge" : "", | 20260508 06:44:02.572 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:02.572 | INFO | "malloc" : " 14 GB", | 20260508 06:44:02.572 | INFO | "pinned" : " 29 GB" | 20260508 06:44:02.572 | INFO | } | 20260508 06:44:02.572 | INFO | Perf report for the last 5.14627 seconds: | 20260508 06:44:02.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:02.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.572 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:44:02.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.572 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:44:02.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:02.573 | INFO | Uploaded 36GiB to GPU in 2.47211s (14.5624GiB/s) | 20260508 06:44:02.573 | INFO | Downloaded 34GiB to GPU in 2.42918s (13.9965GiB/s) | 20260508 06:44:07.709 | INFO | Analyzing chunk frames = [6048768,6049280) | 20260508 06:44:07.709 | INFO | MemInfo: { | 20260508 06:44:07.709 | INFO | "gpu" : " 17 GB", | 20260508 06:44:07.709 | INFO | "huge" : "", | 20260508 06:44:07.709 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:07.709 | INFO | "malloc" : " 14 GB", | 20260508 06:44:07.709 | INFO | "pinned" : " 29 GB" | 20260508 06:44:07.709 | INFO | } | 20260508 06:44:07.709 | INFO | Perf report for the last 5.13728 seconds: | 20260508 06:44:07.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:07.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.709 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:44:07.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.710 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 06:44:07.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:07.710 | INFO | Uploaded 36GiB to GPU in 2.48086s (14.5111GiB/s) | 20260508 06:44:07.710 | INFO | Downloaded 34GiB to GPU in 2.42794s (14.0037GiB/s) | 20260508 06:44:12.846 | INFO | Analyzing chunk frames = [6049280,6049792) | 20260508 06:44:12.846 | INFO | MemInfo: { | 20260508 06:44:12.846 | INFO | "gpu" : " 17 GB", | 20260508 06:44:12.846 | INFO | "huge" : "", | 20260508 06:44:12.846 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:12.846 | INFO | "malloc" : " 14 GB", | 20260508 06:44:12.846 | INFO | "pinned" : " 29 GB" | 20260508 06:44:12.846 | INFO | } | 20260508 06:44:12.846 | INFO | Perf report for the last 5.13629 seconds: | 20260508 06:44:12.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:12.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 06:44:12.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260508 06:44:12.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:12.846 | INFO | Uploaded 36GiB to GPU in 2.5192s (14.2903GiB/s) | 20260508 06:44:12.846 | INFO | Downloaded 34GiB to GPU in 2.45142s (13.8695GiB/s) | 20260508 06:44:12.943 | INFO | Starting a round of Baz IO | 20260508 06:44:16.371 | INFO | Populating RTMetrics report | 20260508 06:44:16.371 | INFO | RTMetrics report populated | 20260508 06:44:16.371 | INFO | Opening tmp RTMetrics json file | 20260508 06:44:16.381 | INFO | Serializing RTMetrics to json | 20260508 06:44:16.383 | INFO | Serialization done, swapping tmp file now... | 20260508 06:44:16.383 | INFO | RTMetrics done | 20260508 06:44:16.441 | INFO | Finished a round of Baz IO: Wrote 1.545 GiB in 3.498 seconds | 20260508 06:44:16.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.66% / 0.00% / 59.34% | 20260508 06:44:16.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.382479% / 3.97364e-06% / 4.76837e-05% | 20260508 06:44:16.441 | INFO | Baz IO overall: | 20260508 06:44:16.441 | INFO | BytesWritten : 2679018388274 | 20260508 06:44:16.441 | INFO | NumEvents : 1291406224782 | 20260508 06:44:17.981 | INFO | Analyzing chunk frames = [6049792,6050304) | 20260508 06:44:17.982 | INFO | MemInfo: { | 20260508 06:44:17.982 | INFO | "gpu" : " 17 GB", | 20260508 06:44:17.982 | INFO | "huge" : "", | 20260508 06:44:17.982 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:17.982 | INFO | "malloc" : " 14 GB", | 20260508 06:44:17.982 | INFO | "pinned" : " 29 GB" | 20260508 06:44:17.982 | INFO | } | 20260508 06:44:17.982 | INFO | Perf report for the last 5.13599 seconds: | 20260508 06:44:17.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:17.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:17.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:17.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:17.982 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 06:44:17.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:17.982 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 06:44:17.982 | INFO | PRE_HQ : 65.4%, 1.6, 55.5ms, 0 | 20260508 06:44:17.982 | INFO | BAZWRITER : 67.6%, 2.6, 91.8ms, 0 | 20260508 06:44:17.982 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 06:44:17.983 | INFO | Uploaded 36GiB to GPU in 2.56425s (14.0392GiB/s) | 20260508 06:44:17.983 | INFO | Downloaded 38GiB to GPU in 2.68477s (14.1539GiB/s) | 20260508 06:44:23.121 | INFO | Analyzing chunk frames = [6050304,6050816) | 20260508 06:44:23.121 | INFO | MemInfo: { | 20260508 06:44:23.121 | INFO | "gpu" : " 17 GB", | 20260508 06:44:23.121 | INFO | "huge" : "", | 20260508 06:44:23.121 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:23.122 | INFO | "malloc" : " 14 GB", | 20260508 06:44:23.122 | INFO | "pinned" : " 29 GB" | 20260508 06:44:23.122 | INFO | } | 20260508 06:44:23.122 | INFO | Perf report for the last 5.13995 seconds: | 20260508 06:44:23.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:23.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 06:44:23.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:44:23.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:23.122 | INFO | Uploaded 36GiB to GPU in 2.51819s (14.296GiB/s) | 20260508 06:44:23.122 | INFO | Downloaded 34GiB to GPU in 2.42823s (14.002GiB/s) | 20260508 06:44:28.266 | INFO | Analyzing chunk frames = [6050816,6051328) | 20260508 06:44:28.266 | INFO | MemInfo: { | 20260508 06:44:28.266 | INFO | "gpu" : " 17 GB", | 20260508 06:44:28.266 | INFO | "huge" : "", | 20260508 06:44:28.266 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:28.266 | INFO | "malloc" : " 14 GB", | 20260508 06:44:28.266 | INFO | "pinned" : " 29 GB" | 20260508 06:44:28.266 | INFO | } | 20260508 06:44:28.266 | INFO | Perf report for the last 5.14444 seconds: | 20260508 06:44:28.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:28.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:44:28.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:44:28.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:28.266 | INFO | Uploaded 36GiB to GPU in 2.55363s (14.0976GiB/s) | 20260508 06:44:28.266 | INFO | Downloaded 34GiB to GPU in 2.40939s (14.1115GiB/s) | 20260508 06:44:33.407 | INFO | Analyzing chunk frames = [6051328,6051840) | 20260508 06:44:33.407 | INFO | MemInfo: { | 20260508 06:44:33.407 | INFO | "gpu" : " 17 GB", | 20260508 06:44:33.407 | INFO | "huge" : "", | 20260508 06:44:33.407 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:33.407 | INFO | "malloc" : " 14 GB", | 20260508 06:44:33.407 | INFO | "pinned" : " 29 GB" | 20260508 06:44:33.407 | INFO | } | 20260508 06:44:33.407 | INFO | Perf report for the last 5.14065 seconds: | 20260508 06:44:33.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:33.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 06:44:33.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:44:33.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:33.407 | INFO | Uploaded 36GiB to GPU in 2.58669s (13.9174GiB/s) | 20260508 06:44:33.407 | INFO | Downloaded 34GiB to GPU in 2.4242s (14.0252GiB/s) | 20260508 06:44:38.546 | INFO | Analyzing chunk frames = [6051840,6052352) | 20260508 06:44:38.546 | INFO | MemInfo: { | 20260508 06:44:38.546 | INFO | "gpu" : " 17 GB", | 20260508 06:44:38.546 | INFO | "huge" : "", | 20260508 06:44:38.546 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:38.546 | INFO | "malloc" : " 14 GB", | 20260508 06:44:38.546 | INFO | "pinned" : " 29 GB" | 20260508 06:44:38.546 | INFO | } | 20260508 06:44:38.546 | INFO | Perf report for the last 5.13919 seconds: | 20260508 06:44:38.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:38.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 06:44:38.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:44:38.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:38.546 | INFO | Uploaded 36GiB to GPU in 2.27572s (15.8192GiB/s) | 20260508 06:44:38.546 | INFO | Downloaded 34GiB to GPU in 2.47592s (13.7323GiB/s) | 20260508 06:44:43.679 | INFO | Analyzing chunk frames = [6052352,6052864) | 20260508 06:44:43.679 | INFO | MemInfo: { | 20260508 06:44:43.679 | INFO | "gpu" : " 17 GB", | 20260508 06:44:43.679 | INFO | "huge" : "", | 20260508 06:44:43.679 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:43.679 | INFO | "malloc" : " 14 GB", | 20260508 06:44:43.679 | INFO | "pinned" : " 29 GB" | 20260508 06:44:43.679 | INFO | } | 20260508 06:44:43.680 | INFO | Perf report for the last 5.13352 seconds: | 20260508 06:44:43.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:43.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 06:44:43.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:44:43.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:43.680 | INFO | Uploaded 36GiB to GPU in 2.36367s (15.2306GiB/s) | 20260508 06:44:43.681 | INFO | Downloaded 34GiB to GPU in 2.42901s (13.9974GiB/s) | 20260508 06:44:48.824 | INFO | Analyzing chunk frames = [6052864,6053376) | 20260508 06:44:48.824 | INFO | MemInfo: { | 20260508 06:44:48.824 | INFO | "gpu" : " 17 GB", | 20260508 06:44:48.824 | INFO | "huge" : "", | 20260508 06:44:48.824 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:48.824 | INFO | "malloc" : " 14 GB", | 20260508 06:44:48.824 | INFO | "pinned" : " 29 GB" | 20260508 06:44:48.824 | INFO | } | 20260508 06:44:48.825 | INFO | Perf report for the last 5.1449 seconds: | 20260508 06:44:48.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:48.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 06:44:48.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 06:44:48.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:48.825 | INFO | Uploaded 36GiB to GPU in 2.46432s (14.6085GiB/s) | 20260508 06:44:48.825 | INFO | Downloaded 34GiB to GPU in 2.45544s (13.8468GiB/s) | 20260508 06:44:53.959 | INFO | Analyzing chunk frames = [6053376,6053888) | 20260508 06:44:53.959 | INFO | MemInfo: { | 20260508 06:44:53.959 | INFO | "gpu" : " 17 GB", | 20260508 06:44:53.959 | INFO | "huge" : "", | 20260508 06:44:53.959 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:53.959 | INFO | "malloc" : " 14 GB", | 20260508 06:44:53.959 | INFO | "pinned" : " 29 GB" | 20260508 06:44:53.959 | INFO | } | 20260508 06:44:53.960 | INFO | Perf report for the last 5.13503 seconds: | 20260508 06:44:53.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:53.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 06:44:53.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260508 06:44:53.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:53.960 | INFO | Uploaded 36GiB to GPU in 2.47522s (14.5441GiB/s) | 20260508 06:44:53.960 | INFO | Downloaded 34GiB to GPU in 2.43929s (13.9385GiB/s) | 20260508 06:44:53.960 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:44:53.960 | INFO | Total mem usage: 16.7 GiB | 20260508 06:44:53.960 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:44:53.960 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:44:53.960 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:44:53.960 | INFO | High Water Breakdown: | 20260508 06:44:53.960 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:44:53.960 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:44:53.960 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:44:53.960 | INFO | ------------------------------------------ | 20260508 06:44:53.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:44:53.960 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:44:53.960 | INFO | Total mem usage: 26.5 GiB | 20260508 06:44:53.960 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:44:53.960 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:44:53.960 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:44:53.960 | INFO | High Water Breakdown: | 20260508 06:44:53.961 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:44:53.961 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:44:53.961 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:44:53.961 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:44:53.961 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:44:53.961 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:44:53.961 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:44:53.961 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:44:53.961 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:44:53.961 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:44:53.961 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:44:53.961 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:44:53.961 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:44:53.961 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:44:53.961 | INFO | ------------------------------------------ | 20260508 06:44:53.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:44:53.961 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:44:53.961 | INFO | Total mem usage: 27.8 GiB | 20260508 06:44:53.961 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:44:53.961 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:44:53.961 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:44:53.961 | INFO | High Water Breakdown: | 20260508 06:44:53.961 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:44:53.961 | INFO | ------------------------------------------ | 20260508 06:44:53.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:44:53.961 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:44:53.961 | INFO | Total mem usage: 15.7 GiB | 20260508 06:44:53.961 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:44:53.961 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:44:53.961 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:44:53.961 | INFO | High Water Breakdown: | 20260508 06:44:53.961 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:44:53.961 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:44:53.961 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:44:53.961 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:44:53.961 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:44:53.961 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:44:53.961 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:44:53.961 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:44:53.961 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:44:53.961 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:44:53.961 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:44:53.961 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:44:53.961 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:44:53.961 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:44:53.961 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:44:53.961 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:44:53.961 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:44:53.961 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:44:53.961 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:44:53.961 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:44:53.961 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:44:53.961 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:44:53.961 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:44:53.961 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:44:53.961 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:44:53.961 | INFO | ------------------------------------------ | 20260508 06:44:53.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:44:53.961 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:44:53.961 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:44:53.961 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:44:53.961 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 5.55e-01 | 20260508 06:44:53.961 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.04e-01, 1.13e-02 / 6.39e-03 | 20260508 06:44:53.961 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.43e+00, 5.82e-01 / 2.01e-01 | 20260508 06:44:53.961 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.45e+00, 2.07e-01 / 1.22e-01 | 20260508 06:44:53.962 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.49e+01, 3.73e+00 / 1.05e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.82e+00 / 1.52e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.54e+00, 4.08e-01 / 2.59e-01 | 20260508 06:44:53.962 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.23e+00, 2.63e-01 / 1.61e-01 | 20260508 06:44:53.962 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.46e-01, 4.95e-02 / 2.97e-02 | 20260508 06:44:53.962 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:44:53.962 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:44:53.962 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:44:53.962 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 24.42%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 24.27%, Download -- 2.57e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 2.34%, LockUpload -- 2.48e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 2.48%, LockDownload -- 2.63e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:44:53.962 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260508 06:44:54.073 | INFO | Starting a round of Baz IO | 20260508 06:44:57.474 | INFO | Populating RTMetrics report | 20260508 06:44:57.474 | INFO | RTMetrics report populated | 20260508 06:44:57.474 | INFO | Opening tmp RTMetrics json file | 20260508 06:44:57.486 | INFO | Serializing RTMetrics to json | 20260508 06:44:57.487 | INFO | Serialization done, swapping tmp file now... | 20260508 06:44:57.488 | INFO | RTMetrics done | 20260508 06:44:57.548 | INFO | Finished a round of Baz IO: Wrote 1.542 GiB in 3.474 seconds | 20260508 06:44:57.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.65% / 0.00% / 59.34% | 20260508 06:44:57.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.380552% / 3.97364e-06% / 4.76837e-05% | 20260508 06:44:57.548 | INFO | Baz IO overall: | 20260508 06:44:57.548 | INFO | BytesWritten : 2680674409090 | 20260508 06:44:57.548 | INFO | NumEvents : 1292193929319 | 20260508 06:44:59.103 | INFO | Analyzing chunk frames = [6053888,6054400) | 20260508 06:44:59.103 | INFO | MemInfo: { | 20260508 06:44:59.103 | INFO | "gpu" : " 17 GB", | 20260508 06:44:59.103 | INFO | "huge" : "", | 20260508 06:44:59.103 | INFO | "huge_shared" : " 29 GB", | 20260508 06:44:59.103 | INFO | "malloc" : " 14 GB", | 20260508 06:44:59.103 | INFO | "pinned" : " 29 GB" | 20260508 06:44:59.103 | INFO | } | 20260508 06:44:59.103 | INFO | Perf report for the last 5.14348 seconds: | 20260508 06:44:59.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:44:59.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:59.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:59.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:59.103 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 06:44:59.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:44:59.103 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 06:44:59.103 | INFO | PRE_HQ : 66.3%, 1.8, 61.7ms, 0 | 20260508 06:44:59.103 | INFO | BAZWRITER : 67.0%, 2.8, 95.3ms, 0 | 20260508 06:44:59.103 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:44:59.103 | INFO | Uploaded 36GiB to GPU in 2.57723s (13.9685GiB/s) | 20260508 06:44:59.104 | INFO | Downloaded 38GiB to GPU in 2.71578s (13.9923GiB/s) | 20260508 06:45:04.236 | INFO | Analyzing chunk frames = [6054400,6054912) | 20260508 06:45:04.236 | INFO | MemInfo: { | 20260508 06:45:04.236 | INFO | "gpu" : " 17 GB", | 20260508 06:45:04.236 | INFO | "huge" : "", | 20260508 06:45:04.236 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:04.236 | INFO | "malloc" : " 14 GB", | 20260508 06:45:04.236 | INFO | "pinned" : " 29 GB" | 20260508 06:45:04.236 | INFO | } | 20260508 06:45:04.236 | INFO | Perf report for the last 5.13289 seconds: | 20260508 06:45:04.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:04.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.236 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 06:45:04.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.236 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:45:04.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:04.237 | INFO | Uploaded 36GiB to GPU in 2.46264s (14.6185GiB/s) | 20260508 06:45:04.237 | INFO | Downloaded 34GiB to GPU in 2.44761s (13.8911GiB/s) | 20260508 06:45:09.377 | INFO | Analyzing chunk frames = [6054912,6055424) | 20260508 06:45:09.377 | INFO | MemInfo: { | 20260508 06:45:09.377 | INFO | "gpu" : " 17 GB", | 20260508 06:45:09.377 | INFO | "huge" : "", | 20260508 06:45:09.377 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:09.377 | INFO | "malloc" : " 14 GB", | 20260508 06:45:09.377 | INFO | "pinned" : " 29 GB" | 20260508 06:45:09.377 | INFO | } | 20260508 06:45:09.377 | INFO | Perf report for the last 5.14125 seconds: | 20260508 06:45:09.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:09.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 06:45:09.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:45:09.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:09.377 | INFO | Uploaded 36GiB to GPU in 2.50025s (14.3986GiB/s) | 20260508 06:45:09.378 | INFO | Downloaded 34GiB to GPU in 2.437s (13.9516GiB/s) | 20260508 06:45:14.517 | INFO | Analyzing chunk frames = [6055424,6055936) | 20260508 06:45:14.518 | INFO | MemInfo: { | 20260508 06:45:14.518 | INFO | "gpu" : " 17 GB", | 20260508 06:45:14.518 | INFO | "huge" : "", | 20260508 06:45:14.518 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:14.518 | INFO | "malloc" : " 14 GB", | 20260508 06:45:14.518 | INFO | "pinned" : " 29 GB" | 20260508 06:45:14.518 | INFO | } | 20260508 06:45:14.518 | INFO | Perf report for the last 5.14054 seconds: | 20260508 06:45:14.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:14.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | ANALYSIS : 66.6%, 2.9, 103.0ms, 0 | 20260508 06:45:14.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:45:14.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:14.518 | INFO | Uploaded 36GiB to GPU in 2.50979s (14.3438GiB/s) | 20260508 06:45:14.518 | INFO | Downloaded 34GiB to GPU in 2.41911s (14.0548GiB/s) | 20260508 06:45:19.658 | INFO | Analyzing chunk frames = [6055936,6056448) | 20260508 06:45:19.658 | INFO | MemInfo: { | 20260508 06:45:19.658 | INFO | "gpu" : " 17 GB", | 20260508 06:45:19.658 | INFO | "huge" : "", | 20260508 06:45:19.658 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:19.658 | INFO | "malloc" : " 14 GB", | 20260508 06:45:19.658 | INFO | "pinned" : " 29 GB" | 20260508 06:45:19.658 | INFO | } | 20260508 06:45:19.658 | INFO | Perf report for the last 5.1405 seconds: | 20260508 06:45:19.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:19.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.658 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:45:19.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.658 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:45:19.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:19.659 | INFO | Uploaded 36GiB to GPU in 2.35934s (15.2585GiB/s) | 20260508 06:45:19.659 | INFO | Downloaded 34GiB to GPU in 2.51974s (13.4935GiB/s) | 20260508 06:45:24.798 | INFO | Analyzing chunk frames = [6056448,6056960) | 20260508 06:45:24.798 | INFO | MemInfo: { | 20260508 06:45:24.798 | INFO | "gpu" : " 17 GB", | 20260508 06:45:24.798 | INFO | "huge" : "", | 20260508 06:45:24.798 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:24.798 | INFO | "malloc" : " 14 GB", | 20260508 06:45:24.798 | INFO | "pinned" : " 29 GB" | 20260508 06:45:24.798 | INFO | } | 20260508 06:45:24.798 | INFO | Perf report for the last 5.13996 seconds: | 20260508 06:45:24.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:24.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.798 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:45:24.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.798 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:45:24.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:24.799 | INFO | Uploaded 36GiB to GPU in 2.42927s (14.8193GiB/s) | 20260508 06:45:24.799 | INFO | Downloaded 34GiB to GPU in 2.46248s (13.8072GiB/s) | 20260508 06:45:29.933 | INFO | Analyzing chunk frames = [6056960,6057472) | 20260508 06:45:29.933 | INFO | MemInfo: { | 20260508 06:45:29.933 | INFO | "gpu" : " 17 GB", | 20260508 06:45:29.933 | INFO | "huge" : "", | 20260508 06:45:29.933 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:29.933 | INFO | "malloc" : " 14 GB", | 20260508 06:45:29.933 | INFO | "pinned" : " 29 GB" | 20260508 06:45:29.933 | INFO | } | 20260508 06:45:29.933 | INFO | Perf report for the last 5.13516 seconds: | 20260508 06:45:29.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:29.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.933 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 06:45:29.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.934 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:45:29.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:29.934 | INFO | Uploaded 36GiB to GPU in 2.42535s (14.8432GiB/s) | 20260508 06:45:29.934 | INFO | Downloaded 34GiB to GPU in 2.44884s (13.8841GiB/s) | 20260508 06:45:35.075 | INFO | Analyzing chunk frames = [6057472,6057984) | 20260508 06:45:35.075 | INFO | MemInfo: { | 20260508 06:45:35.075 | INFO | "gpu" : " 17 GB", | 20260508 06:45:35.075 | INFO | "huge" : "", | 20260508 06:45:35.075 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:35.075 | INFO | "malloc" : " 14 GB", | 20260508 06:45:35.075 | INFO | "pinned" : " 29 GB" | 20260508 06:45:35.075 | INFO | } | 20260508 06:45:35.075 | INFO | Perf report for the last 5.14182 seconds: | 20260508 06:45:35.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:35.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.075 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:45:35.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.075 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 06:45:35.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:35.076 | INFO | Uploaded 36GiB to GPU in 2.48341s (14.4962GiB/s) | 20260508 06:45:35.076 | INFO | Downloaded 34GiB to GPU in 2.47256s (13.7509GiB/s) | 20260508 06:45:35.189 | INFO | Starting a round of Baz IO | 20260508 06:45:38.603 | INFO | Populating RTMetrics report | 20260508 06:45:38.603 | INFO | RTMetrics report populated | 20260508 06:45:38.603 | INFO | Opening tmp RTMetrics json file | 20260508 06:45:38.615 | INFO | Serializing RTMetrics to json | 20260508 06:45:38.616 | INFO | Serialization done, swapping tmp file now... | 20260508 06:45:38.616 | INFO | RTMetrics done | 20260508 06:45:38.669 | INFO | Finished a round of Baz IO: Wrote 1.542 GiB in 3.48 seconds | 20260508 06:45:38.669 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.65% / 0.00% / 59.35% | 20260508 06:45:38.669 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.378935% / 3.97364e-06% / 5.96046e-05% | 20260508 06:45:38.669 | INFO | Baz IO overall: | 20260508 06:45:38.669 | INFO | BytesWritten : 2682330076902 | 20260508 06:45:38.669 | INFO | NumEvents : 1292981391036 | 20260508 06:45:40.215 | INFO | Analyzing chunk frames = [6057984,6058496) | 20260508 06:45:40.215 | INFO | MemInfo: { | 20260508 06:45:40.215 | INFO | "gpu" : " 17 GB", | 20260508 06:45:40.215 | INFO | "huge" : "", | 20260508 06:45:40.216 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:40.216 | INFO | "malloc" : " 14 GB", | 20260508 06:45:40.216 | INFO | "pinned" : " 29 GB" | 20260508 06:45:40.216 | INFO | } | 20260508 06:45:40.216 | INFO | Perf report for the last 5.14047 seconds: | 20260508 06:45:40.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:40.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:40.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:40.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:40.216 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260508 06:45:40.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:40.216 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 06:45:40.216 | INFO | PRE_HQ : 66.5%, 1.8, 61.7ms, 0 | 20260508 06:45:40.216 | INFO | BAZWRITER : 67.1%, 2.8, 95.5ms, 0 | 20260508 06:45:40.216 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 06:45:40.216 | INFO | Uploaded 36GiB to GPU in 2.56647s (14.0271GiB/s) | 20260508 06:45:40.216 | INFO | Downloaded 38GiB to GPU in 2.70369s (14.0549GiB/s) | 20260508 06:45:45.357 | INFO | Analyzing chunk frames = [6058496,6059008) | 20260508 06:45:45.357 | INFO | MemInfo: { | 20260508 06:45:45.357 | INFO | "gpu" : " 17 GB", | 20260508 06:45:45.357 | INFO | "huge" : "", | 20260508 06:45:45.358 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:45.358 | INFO | "malloc" : " 14 GB", | 20260508 06:45:45.358 | INFO | "pinned" : " 29 GB" | 20260508 06:45:45.358 | INFO | } | 20260508 06:45:45.358 | INFO | Perf report for the last 5.14197 seconds: | 20260508 06:45:45.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:45.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:45:45.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:45:45.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:45.358 | INFO | Uploaded 36GiB to GPU in 2.48481s (14.488GiB/s) | 20260508 06:45:45.358 | INFO | Downloaded 34GiB to GPU in 2.4339s (13.9694GiB/s) | 20260508 06:45:50.497 | INFO | Analyzing chunk frames = [6059008,6059520) | 20260508 06:45:50.497 | INFO | MemInfo: { | 20260508 06:45:50.497 | INFO | "gpu" : " 17 GB", | 20260508 06:45:50.497 | INFO | "huge" : "", | 20260508 06:45:50.497 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:50.497 | INFO | "malloc" : " 14 GB", | 20260508 06:45:50.497 | INFO | "pinned" : " 29 GB" | 20260508 06:45:50.497 | INFO | } | 20260508 06:45:50.497 | INFO | Perf report for the last 5.13949 seconds: | 20260508 06:45:50.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:50.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.497 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 06:45:50.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.497 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:45:50.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:50.498 | INFO | Uploaded 36GiB to GPU in 2.53809s (14.1839GiB/s) | 20260508 06:45:50.498 | INFO | Downloaded 34GiB to GPU in 2.4269s (14.0096GiB/s) | 20260508 06:45:55.635 | INFO | Analyzing chunk frames = [6059520,6060032) | 20260508 06:45:55.635 | INFO | MemInfo: { | 20260508 06:45:55.635 | INFO | "gpu" : " 17 GB", | 20260508 06:45:55.635 | INFO | "huge" : "", | 20260508 06:45:55.635 | INFO | "huge_shared" : " 29 GB", | 20260508 06:45:55.635 | INFO | "malloc" : " 14 GB", | 20260508 06:45:55.635 | INFO | "pinned" : " 29 GB" | 20260508 06:45:55.635 | INFO | } | 20260508 06:45:55.635 | INFO | Perf report for the last 5.13798 seconds: | 20260508 06:45:55.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:45:55.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.635 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 06:45:55.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.635 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:45:55.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:45:55.636 | INFO | Uploaded 36GiB to GPU in 2.56461s (14.0372GiB/s) | 20260508 06:45:55.636 | INFO | Downloaded 34GiB to GPU in 2.43437s (13.9667GiB/s) | 20260508 06:46:00.776 | INFO | Analyzing chunk frames = [6060032,6060544) | 20260508 06:46:00.776 | INFO | MemInfo: { | 20260508 06:46:00.776 | INFO | "gpu" : " 17 GB", | 20260508 06:46:00.776 | INFO | "huge" : "", | 20260508 06:46:00.776 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:00.776 | INFO | "malloc" : " 14 GB", | 20260508 06:46:00.776 | INFO | "pinned" : " 29 GB" | 20260508 06:46:00.776 | INFO | } | 20260508 06:46:00.776 | INFO | Perf report for the last 5.14118 seconds: | 20260508 06:46:00.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:00.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.776 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:46:00.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.776 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:46:00.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:00.777 | INFO | Uploaded 36GiB to GPU in 2.28067s (15.7848GiB/s) | 20260508 06:46:00.777 | INFO | Downloaded 34GiB to GPU in 2.48903s (13.6599GiB/s) | 20260508 06:46:05.916 | INFO | Analyzing chunk frames = [6060544,6061056) | 20260508 06:46:05.916 | INFO | MemInfo: { | 20260508 06:46:05.916 | INFO | "gpu" : " 17 GB", | 20260508 06:46:05.916 | INFO | "huge" : "", | 20260508 06:46:05.916 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:05.916 | INFO | "malloc" : " 14 GB", | 20260508 06:46:05.916 | INFO | "pinned" : " 29 GB" | 20260508 06:46:05.916 | INFO | } | 20260508 06:46:05.916 | INFO | Perf report for the last 5.13993 seconds: | 20260508 06:46:05.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:05.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.916 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:46:05.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.916 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 06:46:05.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:05.917 | INFO | Uploaded 36GiB to GPU in 2.45227s (14.6803GiB/s) | 20260508 06:46:05.917 | INFO | Downloaded 34GiB to GPU in 2.43895s (13.9404GiB/s) | 20260508 06:46:11.049 | INFO | Analyzing chunk frames = [6061056,6061568) | 20260508 06:46:11.049 | INFO | MemInfo: { | 20260508 06:46:11.049 | INFO | "gpu" : " 17 GB", | 20260508 06:46:11.049 | INFO | "huge" : "", | 20260508 06:46:11.049 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:11.049 | INFO | "malloc" : " 14 GB", | 20260508 06:46:11.049 | INFO | "pinned" : " 29 GB" | 20260508 06:46:11.049 | INFO | } | 20260508 06:46:11.049 | INFO | Perf report for the last 5.133 seconds: | 20260508 06:46:11.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:11.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.049 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:46:11.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.050 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 06:46:11.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:11.050 | INFO | Uploaded 36GiB to GPU in 2.46714s (14.5918GiB/s) | 20260508 06:46:11.050 | INFO | Downloaded 34GiB to GPU in 2.43611s (13.9567GiB/s) | 20260508 06:46:16.185 | INFO | Analyzing chunk frames = [6061568,6062080) | 20260508 06:46:16.185 | INFO | MemInfo: { | 20260508 06:46:16.185 | INFO | "gpu" : " 17 GB", | 20260508 06:46:16.185 | INFO | "huge" : "", | 20260508 06:46:16.185 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:16.185 | INFO | "malloc" : " 14 GB", | 20260508 06:46:16.185 | INFO | "pinned" : " 29 GB" | 20260508 06:46:16.185 | INFO | } | 20260508 06:46:16.185 | INFO | Perf report for the last 5.13599 seconds: | 20260508 06:46:16.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:16.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.185 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:46:16.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.185 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 06:46:16.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:16.186 | INFO | Uploaded 36GiB to GPU in 2.47233s (14.5612GiB/s) | 20260508 06:46:16.186 | INFO | Downloaded 34GiB to GPU in 2.45514s (13.8485GiB/s) | 20260508 06:46:16.186 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:46:16.186 | INFO | Total mem usage: 16.7 GiB | 20260508 06:46:16.186 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:46:16.186 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.3%) | 20260508 06:46:16.186 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:46:16.186 | INFO | High Water Breakdown: | 20260508 06:46:16.186 | INFO | 59.5% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:46:16.186 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:46:16.186 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:46:16.186 | INFO | ------------------------------------------ | 20260508 06:46:16.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:46:16.186 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:46:16.186 | INFO | Total mem usage: 26.5 GiB | 20260508 06:46:16.186 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:46:16.186 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 06:46:16.186 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 06:46:16.186 | INFO | High Water Breakdown: | 20260508 06:46:16.186 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:46:16.186 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:46:16.187 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:46:16.187 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:46:16.187 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:46:16.187 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:46:16.187 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:46:16.187 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:46:16.187 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:46:16.187 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:46:16.187 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:46:16.187 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:46:16.187 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:46:16.187 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:46:16.187 | INFO | ------------------------------------------ | 20260508 06:46:16.187 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:46:16.187 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:46:16.187 | INFO | Total mem usage: 27.8 GiB | 20260508 06:46:16.187 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:46:16.187 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:46:16.187 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:46:16.187 | INFO | High Water Breakdown: | 20260508 06:46:16.187 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:46:16.187 | INFO | ------------------------------------------ | 20260508 06:46:16.187 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:46:16.187 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:46:16.187 | INFO | Total mem usage: 15.7 GiB | 20260508 06:46:16.187 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:46:16.187 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:46:16.187 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:46:16.187 | INFO | High Water Breakdown: | 20260508 06:46:16.187 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:46:16.187 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:46:16.187 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:46:16.187 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:46:16.187 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:46:16.187 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:46:16.187 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:46:16.187 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:46:16.187 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:46:16.187 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:46:16.187 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:46:16.187 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:46:16.187 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:46:16.187 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:46:16.187 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:46:16.187 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:46:16.187 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:46:16.187 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:46:16.187 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:46:16.187 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:46:16.187 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:46:16.187 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:46:16.187 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:46:16.187 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:46:16.187 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:46:16.187 | INFO | ------------------------------------------ | 20260508 06:46:16.187 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:46:16.187 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:46:16.187 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:46:16.187 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:46:16.187 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 6.55e-01 | 20260508 06:46:16.187 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.05e-01, 1.12e-02 / 4.67e-03 | 20260508 06:46:16.187 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.39e+00, 5.87e-01 / 2.28e-01 | 20260508 06:46:16.187 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.09e-01 / 1.33e-01 | 20260508 06:46:16.187 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.53e+01, 3.78e+00 / 1.07e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.78e+00 / 1.31e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.63e+00, 4.14e-01 / 1.86e-01 | 20260508 06:46:16.187 | INFO | PerfReport: 3.15%, Metrics -- 1.03e+00 / 1.22e+00, 2.55e-01 / 1.05e-01 | 20260508 06:46:16.187 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.56e-01, 5.06e-02 / 2.51e-02 | 20260508 06:46:16.187 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:46:16.187 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:46:16.187 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:46:16.187 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 24.31%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 24.41%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 2.12%, LockUpload -- 2.25e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 2.76%, LockDownload -- 2.92e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:46:16.187 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260508 06:46:16.299 | INFO | Starting a round of Baz IO | 20260508 06:46:19.704 | INFO | Populating RTMetrics report | 20260508 06:46:19.704 | INFO | RTMetrics report populated | 20260508 06:46:19.704 | INFO | Opening tmp RTMetrics json file | 20260508 06:46:19.716 | INFO | Serializing RTMetrics to json | 20260508 06:46:19.717 | INFO | Serialization done, swapping tmp file now... | 20260508 06:46:19.718 | INFO | RTMetrics done | 20260508 06:46:19.781 | INFO | Finished a round of Baz IO: Wrote 1.542 GiB in 3.481 seconds | 20260508 06:46:19.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.65% / 0.00% / 59.35% | 20260508 06:46:19.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.37868% / 0% / 6.35783e-05% | 20260508 06:46:19.781 | INFO | Baz IO overall: | 20260508 06:46:19.781 | INFO | BytesWritten : 2683985430313 | 20260508 06:46:19.781 | INFO | NumEvents : 1293768715735 | 20260508 06:46:21.333 | INFO | Analyzing chunk frames = [6062080,6062592) | 20260508 06:46:21.333 | INFO | MemInfo: { | 20260508 06:46:21.333 | INFO | "gpu" : " 17 GB", | 20260508 06:46:21.333 | INFO | "huge" : "", | 20260508 06:46:21.333 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:21.333 | INFO | "malloc" : " 14 GB", | 20260508 06:46:21.333 | INFO | "pinned" : " 29 GB" | 20260508 06:46:21.333 | INFO | } | 20260508 06:46:21.333 | INFO | Perf report for the last 5.14804 seconds: | 20260508 06:46:21.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:21.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:21.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:21.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:21.334 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 06:46:21.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:21.334 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 06:46:21.334 | INFO | PRE_HQ : 66.3%, 1.8, 62.1ms, 0 | 20260508 06:46:21.334 | INFO | BAZWRITER : 66.8%, 2.8, 96.5ms, 0 | 20260508 06:46:21.334 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:46:21.334 | INFO | Uploaded 36GiB to GPU in 2.59894s (13.8518GiB/s) | 20260508 06:46:21.334 | INFO | Downloaded 38GiB to GPU in 2.69663s (14.0917GiB/s) | 20260508 06:46:26.473 | INFO | Analyzing chunk frames = [6062592,6063104) | 20260508 06:46:26.473 | INFO | MemInfo: { | 20260508 06:46:26.473 | INFO | "gpu" : " 17 GB", | 20260508 06:46:26.473 | INFO | "huge" : "", | 20260508 06:46:26.473 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:26.473 | INFO | "malloc" : " 14 GB", | 20260508 06:46:26.473 | INFO | "pinned" : " 29 GB" | 20260508 06:46:26.473 | INFO | } | 20260508 06:46:26.473 | INFO | Perf report for the last 5.14004 seconds: | 20260508 06:46:26.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:26.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.474 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:46:26.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.474 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:46:26.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:26.474 | INFO | Uploaded 36GiB to GPU in 2.47902s (14.5218GiB/s) | 20260508 06:46:26.474 | INFO | Downloaded 34GiB to GPU in 2.43751s (13.9487GiB/s) | 20260508 06:46:31.609 | INFO | Analyzing chunk frames = [6063104,6063616) | 20260508 06:46:31.609 | INFO | MemInfo: { | 20260508 06:46:31.609 | INFO | "gpu" : " 17 GB", | 20260508 06:46:31.609 | INFO | "huge" : "", | 20260508 06:46:31.609 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:31.609 | INFO | "malloc" : " 14 GB", | 20260508 06:46:31.609 | INFO | "pinned" : " 29 GB" | 20260508 06:46:31.609 | INFO | } | 20260508 06:46:31.609 | INFO | Perf report for the last 5.13556 seconds: | 20260508 06:46:31.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:31.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:46:31.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:46:31.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:31.609 | INFO | Uploaded 36GiB to GPU in 2.54297s (14.1567GiB/s) | 20260508 06:46:31.609 | INFO | Downloaded 34GiB to GPU in 2.42072s (14.0454GiB/s) | 20260508 06:46:36.741 | INFO | Analyzing chunk frames = [6063616,6064128) | 20260508 06:46:36.741 | INFO | MemInfo: { | 20260508 06:46:36.741 | INFO | "gpu" : " 17 GB", | 20260508 06:46:36.741 | INFO | "huge" : "", | 20260508 06:46:36.742 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:36.742 | INFO | "malloc" : " 14 GB", | 20260508 06:46:36.742 | INFO | "pinned" : " 29 GB" | 20260508 06:46:36.742 | INFO | } | 20260508 06:46:36.742 | INFO | Perf report for the last 5.13269 seconds: | 20260508 06:46:36.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:36.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 06:46:36.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:46:36.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:36.742 | INFO | Uploaded 36GiB to GPU in 2.56352s (14.0432GiB/s) | 20260508 06:46:36.742 | INFO | Downloaded 34GiB to GPU in 2.42404s (14.0262GiB/s) | 20260508 06:46:41.888 | INFO | Analyzing chunk frames = [6064128,6064640) | 20260508 06:46:41.888 | INFO | MemInfo: { | 20260508 06:46:41.888 | INFO | "gpu" : " 17 GB", | 20260508 06:46:41.888 | INFO | "huge" : "", | 20260508 06:46:41.888 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:41.888 | INFO | "malloc" : " 14 GB", | 20260508 06:46:41.888 | INFO | "pinned" : " 29 GB" | 20260508 06:46:41.888 | INFO | } | 20260508 06:46:41.888 | INFO | Perf report for the last 5.14668 seconds: | 20260508 06:46:41.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:41.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.889 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:46:41.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.889 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 06:46:41.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:41.889 | INFO | Uploaded 36GiB to GPU in 2.30328s (15.6299GiB/s) | 20260508 06:46:41.889 | INFO | Downloaded 34GiB to GPU in 2.49632s (13.62GiB/s) | 20260508 06:46:47.028 | INFO | Analyzing chunk frames = [6064640,6065152) | 20260508 06:46:47.028 | INFO | MemInfo: { | 20260508 06:46:47.028 | INFO | "gpu" : " 17 GB", | 20260508 06:46:47.028 | INFO | "huge" : "", | 20260508 06:46:47.028 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:47.028 | INFO | "malloc" : " 14 GB", | 20260508 06:46:47.028 | INFO | "pinned" : " 29 GB" | 20260508 06:46:47.028 | INFO | } | 20260508 06:46:47.028 | INFO | Perf report for the last 5.13984 seconds: | 20260508 06:46:47.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:47.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.028 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:46:47.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.028 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:46:47.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:47.029 | INFO | Uploaded 36GiB to GPU in 2.44985s (14.6948GiB/s) | 20260508 06:46:47.029 | INFO | Downloaded 34GiB to GPU in 2.43867s (13.942GiB/s) | 20260508 06:46:52.170 | INFO | Analyzing chunk frames = [6065152,6065664) | 20260508 06:46:52.170 | INFO | MemInfo: { | 20260508 06:46:52.170 | INFO | "gpu" : " 17 GB", | 20260508 06:46:52.170 | INFO | "huge" : "", | 20260508 06:46:52.170 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:52.170 | INFO | "malloc" : " 14 GB", | 20260508 06:46:52.170 | INFO | "pinned" : " 29 GB" | 20260508 06:46:52.170 | INFO | } | 20260508 06:46:52.170 | INFO | Perf report for the last 5.14202 seconds: | 20260508 06:46:52.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:52.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.171 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 06:46:52.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.171 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 06:46:52.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:52.171 | INFO | Uploaded 36GiB to GPU in 2.45349s (14.673GiB/s) | 20260508 06:46:52.171 | INFO | Downloaded 34GiB to GPU in 2.42818s (14.0023GiB/s) | 20260508 06:46:57.302 | INFO | Analyzing chunk frames = [6065664,6066176) | 20260508 06:46:57.302 | INFO | MemInfo: { | 20260508 06:46:57.302 | INFO | "gpu" : " 17 GB", | 20260508 06:46:57.302 | INFO | "huge" : "", | 20260508 06:46:57.302 | INFO | "huge_shared" : " 29 GB", | 20260508 06:46:57.302 | INFO | "malloc" : " 14 GB", | 20260508 06:46:57.302 | INFO | "pinned" : " 29 GB" | 20260508 06:46:57.302 | INFO | } | 20260508 06:46:57.302 | INFO | Perf report for the last 5.13208 seconds: | 20260508 06:46:57.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:46:57.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 06:46:57.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 06:46:57.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:46:57.303 | INFO | Uploaded 36GiB to GPU in 2.49171s (14.4479GiB/s) | 20260508 06:46:57.303 | INFO | Downloaded 34GiB to GPU in 2.45213s (13.8655GiB/s) | 20260508 06:46:57.415 | INFO | Starting a round of Baz IO | 20260508 06:47:00.835 | INFO | Populating RTMetrics report | 20260508 06:47:00.835 | INFO | RTMetrics report populated | 20260508 06:47:00.835 | INFO | Opening tmp RTMetrics json file | 20260508 06:47:00.847 | INFO | Serializing RTMetrics to json | 20260508 06:47:00.848 | INFO | Serialization done, swapping tmp file now... | 20260508 06:47:00.849 | INFO | RTMetrics done | 20260508 06:47:00.905 | INFO | Finished a round of Baz IO: Wrote 1.543 GiB in 3.489 seconds | 20260508 06:47:00.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.65% / 0.00% / 59.35% | 20260508 06:47:00.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.380345% / 0% / 3.97364e-05% | 20260508 06:47:00.905 | INFO | Baz IO overall: | 20260508 06:47:00.905 | INFO | BytesWritten : 2685642590731 | 20260508 06:47:00.905 | INFO | NumEvents : 1294557314258 | 20260508 06:47:02.446 | INFO | Analyzing chunk frames = [6066176,6066688) | 20260508 06:47:02.446 | INFO | MemInfo: { | 20260508 06:47:02.446 | INFO | "gpu" : " 17 GB", | 20260508 06:47:02.446 | INFO | "huge" : "", | 20260508 06:47:02.446 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:02.446 | INFO | "malloc" : " 14 GB", | 20260508 06:47:02.446 | INFO | "pinned" : " 29 GB" | 20260508 06:47:02.446 | INFO | } | 20260508 06:47:02.446 | INFO | Perf report for the last 5.14364 seconds: | 20260508 06:47:02.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:02.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:02.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:02.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:02.446 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 06:47:02.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:02.446 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260508 06:47:02.446 | INFO | PRE_HQ : 66.4%, 1.7, 59.9ms, 0 | 20260508 06:47:02.446 | INFO | BAZWRITER : 67.3%, 2.7, 93.9ms, 0 | 20260508 06:47:02.446 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:47:02.446 | INFO | Uploaded 36GiB to GPU in 2.51463s (14.3162GiB/s) | 20260508 06:47:02.447 | INFO | Downloaded 38GiB to GPU in 2.71784s (13.9817GiB/s) | 20260508 06:47:07.586 | INFO | Analyzing chunk frames = [6066688,6067200) | 20260508 06:47:07.586 | INFO | MemInfo: { | 20260508 06:47:07.586 | INFO | "gpu" : " 17 GB", | 20260508 06:47:07.586 | INFO | "huge" : "", | 20260508 06:47:07.586 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:07.586 | INFO | "malloc" : " 14 GB", | 20260508 06:47:07.586 | INFO | "pinned" : " 29 GB" | 20260508 06:47:07.586 | INFO | } | 20260508 06:47:07.586 | INFO | Perf report for the last 5.14016 seconds: | 20260508 06:47:07.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:07.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.586 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 06:47:07.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.586 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:47:07.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:07.587 | INFO | Uploaded 36GiB to GPU in 2.45425s (14.6684GiB/s) | 20260508 06:47:07.587 | INFO | Downloaded 34GiB to GPU in 2.46085s (13.8163GiB/s) | 20260508 06:47:12.725 | INFO | Analyzing chunk frames = [6067200,6067712) | 20260508 06:47:12.725 | INFO | MemInfo: { | 20260508 06:47:12.725 | INFO | "gpu" : " 17 GB", | 20260508 06:47:12.725 | INFO | "huge" : "", | 20260508 06:47:12.725 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:12.725 | INFO | "malloc" : " 14 GB", | 20260508 06:47:12.725 | INFO | "pinned" : " 29 GB" | 20260508 06:47:12.725 | INFO | } | 20260508 06:47:12.725 | INFO | Perf report for the last 5.13865 seconds: | 20260508 06:47:12.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:12.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 06:47:12.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:47:12.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:12.725 | INFO | Uploaded 36GiB to GPU in 2.51212s (14.3305GiB/s) | 20260508 06:47:12.725 | INFO | Downloaded 34GiB to GPU in 2.4583s (13.8307GiB/s) | 20260508 06:47:17.869 | INFO | Analyzing chunk frames = [6067712,6068224) | 20260508 06:47:17.869 | INFO | MemInfo: { | 20260508 06:47:17.869 | INFO | "gpu" : " 17 GB", | 20260508 06:47:17.869 | INFO | "huge" : "", | 20260508 06:47:17.869 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:17.869 | INFO | "malloc" : " 14 GB", | 20260508 06:47:17.869 | INFO | "pinned" : " 29 GB" | 20260508 06:47:17.869 | INFO | } | 20260508 06:47:17.869 | INFO | Perf report for the last 5.14393 seconds: | 20260508 06:47:17.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:17.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | ANALYSIS : 66.5%, 2.9, 103.0ms, 0 | 20260508 06:47:17.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:47:17.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:17.869 | INFO | Uploaded 36GiB to GPU in 2.50459s (14.3736GiB/s) | 20260508 06:47:17.869 | INFO | Downloaded 34GiB to GPU in 2.43429s (13.9671GiB/s) | 20260508 06:47:22.999 | INFO | Analyzing chunk frames = [6068224,6068736) | 20260508 06:47:22.999 | INFO | MemInfo: { | 20260508 06:47:22.999 | INFO | "gpu" : " 17 GB", | 20260508 06:47:22.999 | INFO | "huge" : "", | 20260508 06:47:22.999 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:22.999 | INFO | "malloc" : " 14 GB", | 20260508 06:47:22.999 | INFO | "pinned" : " 29 GB" | 20260508 06:47:22.999 | INFO | } | 20260508 06:47:22.999 | INFO | Perf report for the last 5.13025 seconds: | 20260508 06:47:22.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:22.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:22.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:22.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:22.999 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:47:22.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:22.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:22.999 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:47:22.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:22.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:23.000 | INFO | Uploaded 36GiB to GPU in 2.34439s (15.3558GiB/s) | 20260508 06:47:23.000 | INFO | Downloaded 34GiB to GPU in 2.51128s (13.5389GiB/s) | 20260508 06:47:28.140 | INFO | Analyzing chunk frames = [6068736,6069248) | 20260508 06:47:28.140 | INFO | MemInfo: { | 20260508 06:47:28.140 | INFO | "gpu" : " 17 GB", | 20260508 06:47:28.140 | INFO | "huge" : "", | 20260508 06:47:28.140 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:28.140 | INFO | "malloc" : " 14 GB", | 20260508 06:47:28.140 | INFO | "pinned" : " 29 GB" | 20260508 06:47:28.140 | INFO | } | 20260508 06:47:28.140 | INFO | Perf report for the last 5.1413 seconds: | 20260508 06:47:28.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:28.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.141 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 06:47:28.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.141 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:47:28.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:28.141 | INFO | Uploaded 36GiB to GPU in 2.40951s (14.9408GiB/s) | 20260508 06:47:28.141 | INFO | Downloaded 34GiB to GPU in 2.44767s (13.8907GiB/s) | 20260508 06:47:33.282 | INFO | Analyzing chunk frames = [6069248,6069760) | 20260508 06:47:33.282 | INFO | MemInfo: { | 20260508 06:47:33.282 | INFO | "gpu" : " 17 GB", | 20260508 06:47:33.283 | INFO | "huge" : "", | 20260508 06:47:33.283 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:33.283 | INFO | "malloc" : " 14 GB", | 20260508 06:47:33.283 | INFO | "pinned" : " 29 GB" | 20260508 06:47:33.283 | INFO | } | 20260508 06:47:33.283 | INFO | Perf report for the last 5.14233 seconds: | 20260508 06:47:33.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:33.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.283 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 06:47:33.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.283 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 06:47:33.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:33.284 | INFO | Uploaded 36GiB to GPU in 2.39531s (15.0294GiB/s) | 20260508 06:47:33.284 | INFO | Downloaded 34GiB to GPU in 2.44793s (13.8893GiB/s) | 20260508 06:47:38.424 | INFO | Analyzing chunk frames = [6069760,6070272) | 20260508 06:47:38.424 | INFO | MemInfo: { | 20260508 06:47:38.424 | INFO | "gpu" : " 17 GB", | 20260508 06:47:38.424 | INFO | "huge" : "", | 20260508 06:47:38.424 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:38.424 | INFO | "malloc" : " 14 GB", | 20260508 06:47:38.424 | INFO | "pinned" : " 29 GB" | 20260508 06:47:38.424 | INFO | } | 20260508 06:47:38.424 | INFO | Perf report for the last 5.14154 seconds: | 20260508 06:47:38.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:38.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.424 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 06:47:38.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.424 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 06:47:38.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:38.425 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:47:38.425 | INFO | Total mem usage: 16.7 GiB | 20260508 06:47:38.425 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:47:38.425 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:47:38.425 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:47:38.425 | INFO | High Water Breakdown: | 20260508 06:47:38.425 | INFO | Uploaded 36GiB to GPU in 2.39728s (15.017GiB/s) | 20260508 06:47:38.425 | INFO | 59.5% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:47:38.425 | INFO | Downloaded 34GiB to GPU in 2.47511s (13.7368GiB/s) | 20260508 06:47:38.425 | INFO | 40.3% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:47:38.425 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:47:38.425 | INFO | ------------------------------------------ | 20260508 06:47:38.425 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:47:38.425 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:47:38.425 | INFO | Total mem usage: 26.5 GiB | 20260508 06:47:38.425 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:47:38.425 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:47:38.425 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:47:38.425 | INFO | High Water Breakdown: | 20260508 06:47:38.425 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:47:38.425 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:47:38.425 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:47:38.425 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:47:38.425 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:47:38.425 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:47:38.425 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:47:38.425 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:47:38.426 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:47:38.426 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:47:38.426 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:47:38.426 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:47:38.426 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:47:38.426 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:47:38.426 | INFO | ------------------------------------------ | 20260508 06:47:38.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:47:38.426 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:47:38.426 | INFO | Total mem usage: 27.8 GiB | 20260508 06:47:38.426 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:47:38.426 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:47:38.426 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:47:38.426 | INFO | High Water Breakdown: | 20260508 06:47:38.426 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:47:38.426 | INFO | ------------------------------------------ | 20260508 06:47:38.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:47:38.426 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:47:38.426 | INFO | Total mem usage: 15.7 GiB | 20260508 06:47:38.426 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:47:38.426 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:47:38.426 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:47:38.426 | INFO | High Water Breakdown: | 20260508 06:47:38.426 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:47:38.426 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:47:38.426 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:47:38.426 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:47:38.426 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:47:38.426 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:47:38.426 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:47:38.426 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:47:38.426 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:47:38.426 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:47:38.426 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:47:38.426 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:47:38.426 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:47:38.426 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:47:38.426 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:47:38.426 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:47:38.426 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:47:38.426 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:47:38.426 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:47:38.426 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:47:38.426 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:47:38.426 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:47:38.426 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:47:38.426 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:47:38.426 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:47:38.426 | INFO | ------------------------------------------ | 20260508 06:47:38.426 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:47:38.426 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:47:38.426 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:47:38.426 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:47:38.426 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.37e+01 / 7.12e-01 | 20260508 06:47:38.426 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.05e-01, 1.12e-02 / 6.27e-03 | 20260508 06:47:38.426 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.39e+00, 5.84e-01 / 2.60e-01 | 20260508 06:47:38.426 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.48e+00, 2.07e-01 / 1.18e-01 | 20260508 06:47:38.426 | INFO | PerfReport: 15.91%, DME -- 4.15e+01 / 4.51e+01, 3.76e+00 / 1.45e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.79e+00 / 1.51e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.38e+00 / 2.57e+00, 4.15e-01 / 2.62e-01 | 20260508 06:47:38.427 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.21e+00, 2.52e-01 / 1.35e-01 | 20260508 06:47:38.427 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.51e-01, 5.01e-02 / 2.79e-02 | 20260508 06:47:38.427 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:47:38.427 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:47:38.427 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:47:38.427 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 24.24%, Upload -- 2.57e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 24.44%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 30.83%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 2.05%, LockUpload -- 2.17e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 2.85%, LockDownload -- 3.02e+00 / 0.00e+00, 4.43e+00 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 15.58%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:47:38.427 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260508 06:47:38.522 | INFO | Starting a round of Baz IO | 20260508 06:47:41.955 | INFO | Populating RTMetrics report | 20260508 06:47:41.955 | INFO | RTMetrics report populated | 20260508 06:47:41.955 | INFO | Opening tmp RTMetrics json file | 20260508 06:47:41.966 | INFO | Serializing RTMetrics to json | 20260508 06:47:41.967 | INFO | Serialization done, swapping tmp file now... | 20260508 06:47:41.968 | INFO | RTMetrics done | 20260508 06:47:42.022 | INFO | Finished a round of Baz IO: Wrote 1.541 GiB in 3.5 seconds | 20260508 06:47:42.022 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.65% / 0.00% / 59.35% | 20260508 06:47:42.022 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.377138% / 0% / 4.76837e-05% | 20260508 06:47:42.022 | INFO | Baz IO overall: | 20260508 06:47:42.022 | INFO | BytesWritten : 2687297254929 | 20260508 06:47:42.022 | INFO | NumEvents : 1295344222109 | 20260508 06:47:43.557 | INFO | Analyzing chunk frames = [6070272,6070784) | 20260508 06:47:43.557 | INFO | MemInfo: { | 20260508 06:47:43.557 | INFO | "gpu" : " 17 GB", | 20260508 06:47:43.557 | INFO | "huge" : "", | 20260508 06:47:43.557 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:43.557 | INFO | "malloc" : " 14 GB", | 20260508 06:47:43.557 | INFO | "pinned" : " 29 GB" | 20260508 06:47:43.557 | INFO | } | 20260508 06:47:43.557 | INFO | Perf report for the last 5.13309 seconds: | 20260508 06:47:43.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:43.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:43.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:43.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:43.558 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 06:47:43.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:43.558 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 06:47:43.558 | INFO | PRE_HQ : 65.9%, 1.6, 55.1ms, 0 | 20260508 06:47:43.558 | INFO | BAZWRITER : 67.6%, 2.6, 89.9ms, 0 | 20260508 06:47:43.558 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:47:43.558 | INFO | Uploaded 36GiB to GPU in 2.52812s (14.2399GiB/s) | 20260508 06:47:43.558 | INFO | Downloaded 38GiB to GPU in 2.71549s (13.9938GiB/s) | 20260508 06:47:48.697 | INFO | Analyzing chunk frames = [6070784,6071296) | 20260508 06:47:48.697 | INFO | MemInfo: { | 20260508 06:47:48.697 | INFO | "gpu" : " 17 GB", | 20260508 06:47:48.697 | INFO | "huge" : "", | 20260508 06:47:48.697 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:48.697 | INFO | "malloc" : " 14 GB", | 20260508 06:47:48.697 | INFO | "pinned" : " 29 GB" | 20260508 06:47:48.697 | INFO | } | 20260508 06:47:48.698 | INFO | Perf report for the last 5.14013 seconds: | 20260508 06:47:48.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:48.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 06:47:48.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:47:48.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:48.698 | INFO | Uploaded 36GiB to GPU in 2.45555s (14.6607GiB/s) | 20260508 06:47:48.698 | INFO | Downloaded 34GiB to GPU in 2.44999s (13.8776GiB/s) | 20260508 06:47:53.838 | INFO | Analyzing chunk frames = [6071296,6071808) | 20260508 06:47:53.838 | INFO | MemInfo: { | 20260508 06:47:53.838 | INFO | "gpu" : " 17 GB", | 20260508 06:47:53.838 | INFO | "huge" : "", | 20260508 06:47:53.838 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:53.838 | INFO | "malloc" : " 14 GB", | 20260508 06:47:53.838 | INFO | "pinned" : " 29 GB" | 20260508 06:47:53.838 | INFO | } | 20260508 06:47:53.838 | INFO | Perf report for the last 5.14048 seconds: | 20260508 06:47:53.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:53.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 06:47:53.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:47:53.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:53.838 | INFO | Uploaded 36GiB to GPU in 2.52891s (14.2354GiB/s) | 20260508 06:47:53.838 | INFO | Downloaded 34GiB to GPU in 2.44229s (13.9213GiB/s) | 20260508 06:47:58.978 | INFO | Analyzing chunk frames = [6071808,6072320) | 20260508 06:47:58.978 | INFO | MemInfo: { | 20260508 06:47:58.978 | INFO | "gpu" : " 17 GB", | 20260508 06:47:58.978 | INFO | "huge" : "", | 20260508 06:47:58.978 | INFO | "huge_shared" : " 29 GB", | 20260508 06:47:58.978 | INFO | "malloc" : " 14 GB", | 20260508 06:47:58.978 | INFO | "pinned" : " 29 GB" | 20260508 06:47:58.978 | INFO | } | 20260508 06:47:58.978 | INFO | Perf report for the last 5.1398 seconds: | 20260508 06:47:58.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:47:58.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 06:47:58.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:47:58.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:47:58.978 | INFO | Uploaded 36GiB to GPU in 2.5329s (14.213GiB/s) | 20260508 06:47:58.978 | INFO | Downloaded 34GiB to GPU in 2.43254s (13.9772GiB/s) | 20260508 06:48:04.119 | INFO | Analyzing chunk frames = [6072320,6072832) | 20260508 06:48:04.119 | INFO | MemInfo: { | 20260508 06:48:04.119 | INFO | "gpu" : " 17 GB", | 20260508 06:48:04.119 | INFO | "huge" : "", | 20260508 06:48:04.119 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:04.119 | INFO | "malloc" : " 14 GB", | 20260508 06:48:04.119 | INFO | "pinned" : " 29 GB" | 20260508 06:48:04.119 | INFO | } | 20260508 06:48:04.119 | INFO | Perf report for the last 5.14142 seconds: | 20260508 06:48:04.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:04.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.119 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:48:04.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.119 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:48:04.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:04.120 | INFO | Uploaded 36GiB to GPU in 2.38124s (15.1182GiB/s) | 20260508 06:48:04.120 | INFO | Downloaded 34GiB to GPU in 2.50849s (13.554GiB/s) | 20260508 06:48:09.254 | INFO | Analyzing chunk frames = [6072832,6073344) | 20260508 06:48:09.254 | INFO | MemInfo: { | 20260508 06:48:09.254 | INFO | "gpu" : " 17 GB", | 20260508 06:48:09.254 | INFO | "huge" : "", | 20260508 06:48:09.254 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:09.254 | INFO | "malloc" : " 14 GB", | 20260508 06:48:09.254 | INFO | "pinned" : " 29 GB" | 20260508 06:48:09.254 | INFO | } | 20260508 06:48:09.254 | INFO | Perf report for the last 5.13498 seconds: | 20260508 06:48:09.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:09.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.254 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 06:48:09.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.255 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:48:09.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:09.255 | INFO | Uploaded 36GiB to GPU in 2.35209s (15.3055GiB/s) | 20260508 06:48:09.255 | INFO | Downloaded 34GiB to GPU in 2.4623s (13.8082GiB/s) | 20260508 06:48:14.397 | INFO | Analyzing chunk frames = [6073344,6073856) | 20260508 06:48:14.397 | INFO | MemInfo: { | 20260508 06:48:14.397 | INFO | "gpu" : " 17 GB", | 20260508 06:48:14.397 | INFO | "huge" : "", | 20260508 06:48:14.397 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:14.397 | INFO | "malloc" : " 14 GB", | 20260508 06:48:14.397 | INFO | "pinned" : " 29 GB" | 20260508 06:48:14.397 | INFO | } | 20260508 06:48:14.397 | INFO | Perf report for the last 5.14278 seconds: | 20260508 06:48:14.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:14.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.397 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:48:14.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.398 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 06:48:14.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:14.398 | INFO | Uploaded 36GiB to GPU in 2.4611s (14.6276GiB/s) | 20260508 06:48:14.398 | INFO | Downloaded 34GiB to GPU in 2.45522s (13.8481GiB/s) | 20260508 06:48:19.537 | INFO | Analyzing chunk frames = [6073856,6074368) | 20260508 06:48:19.537 | INFO | MemInfo: { | 20260508 06:48:19.537 | INFO | "gpu" : " 17 GB", | 20260508 06:48:19.537 | INFO | "huge" : "", | 20260508 06:48:19.537 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:19.537 | INFO | "malloc" : " 14 GB", | 20260508 06:48:19.537 | INFO | "pinned" : " 29 GB" | 20260508 06:48:19.537 | INFO | } | 20260508 06:48:19.538 | INFO | Perf report for the last 5.14042 seconds: | 20260508 06:48:19.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:19.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:48:19.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | PRE_HQ : 34.5%, 1.0, 17.9ms, 0 | 20260508 06:48:19.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:19.538 | INFO | Uploaded 36GiB to GPU in 2.46016s (14.6332GiB/s) | 20260508 06:48:19.538 | INFO | Downloaded 34GiB to GPU in 2.45875s (13.8282GiB/s) | 20260508 06:48:19.651 | INFO | Starting a round of Baz IO | 20260508 06:48:23.068 | INFO | Populating RTMetrics report | 20260508 06:48:23.069 | INFO | RTMetrics report populated | 20260508 06:48:23.069 | INFO | Opening tmp RTMetrics json file | 20260508 06:48:23.069 | INFO | Serializing RTMetrics to json | 20260508 06:48:23.071 | INFO | Serialization done, swapping tmp file now... | 20260508 06:48:23.071 | INFO | RTMetrics done | 20260508 06:48:23.137 | INFO | Finished a round of Baz IO: Wrote 1.541 GiB in 3.485 seconds | 20260508 06:48:23.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.65% / 0.00% / 59.35% | 20260508 06:48:23.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.376936% / 3.97364e-06% / 5.96046e-05% | 20260508 06:48:23.137 | INFO | Baz IO overall: | 20260508 06:48:23.137 | INFO | BytesWritten : 2688952066051 | 20260508 06:48:23.137 | INFO | NumEvents : 1296131297755 | 20260508 06:48:24.677 | INFO | Analyzing chunk frames = [6074368,6074880) | 20260508 06:48:24.677 | INFO | MemInfo: { | 20260508 06:48:24.677 | INFO | "gpu" : " 17 GB", | 20260508 06:48:24.677 | INFO | "huge" : "", | 20260508 06:48:24.677 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:24.677 | INFO | "malloc" : " 14 GB", | 20260508 06:48:24.677 | INFO | "pinned" : " 29 GB" | 20260508 06:48:24.677 | INFO | } | 20260508 06:48:24.677 | INFO | Perf report for the last 5.13943 seconds: | 20260508 06:48:24.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:24.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:24.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:24.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:24.677 | INFO | ANALYSIS : 68.3%, 3.0, 106.1ms, 0 | 20260508 06:48:24.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:24.677 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260508 06:48:24.677 | INFO | PRE_HQ : 66.4%, 1.8, 61.5ms, 0 | 20260508 06:48:24.677 | INFO | BAZWRITER : 67.3%, 2.8, 95.9ms, 0 | 20260508 06:48:24.677 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:48:24.678 | INFO | Uploaded 36GiB to GPU in 2.54461s (14.1476GiB/s) | 20260508 06:48:24.678 | INFO | Downloaded 38GiB to GPU in 2.70746s (14.0353GiB/s) | 20260508 06:48:29.809 | INFO | Analyzing chunk frames = [6074880,6075392) | 20260508 06:48:29.809 | INFO | MemInfo: { | 20260508 06:48:29.809 | INFO | "gpu" : " 17 GB", | 20260508 06:48:29.809 | INFO | "huge" : "", | 20260508 06:48:29.809 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:29.809 | INFO | "malloc" : " 14 GB", | 20260508 06:48:29.809 | INFO | "pinned" : " 29 GB" | 20260508 06:48:29.809 | INFO | } | 20260508 06:48:29.810 | INFO | Perf report for the last 5.13252 seconds: | 20260508 06:48:29.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:29.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:48:29.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:48:29.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:29.810 | INFO | Uploaded 36GiB to GPU in 2.47339s (14.5549GiB/s) | 20260508 06:48:29.810 | INFO | Downloaded 34GiB to GPU in 2.43237s (13.9781GiB/s) | 20260508 06:48:34.949 | INFO | Analyzing chunk frames = [6075392,6075904) | 20260508 06:48:34.949 | INFO | MemInfo: { | 20260508 06:48:34.949 | INFO | "gpu" : " 17 GB", | 20260508 06:48:34.949 | INFO | "huge" : "", | 20260508 06:48:34.949 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:34.949 | INFO | "malloc" : " 14 GB", | 20260508 06:48:34.949 | INFO | "pinned" : " 29 GB" | 20260508 06:48:34.949 | INFO | } | 20260508 06:48:34.949 | INFO | Perf report for the last 5.13989 seconds: | 20260508 06:48:34.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:34.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.949 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:48:34.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.949 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:48:34.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:34.950 | INFO | Uploaded 36GiB to GPU in 2.53582s (14.1966GiB/s) | 20260508 06:48:34.950 | INFO | Downloaded 34GiB to GPU in 2.41638s (14.0706GiB/s) | 20260508 06:48:40.091 | INFO | Analyzing chunk frames = [6075904,6076416) | 20260508 06:48:40.091 | INFO | MemInfo: { | 20260508 06:48:40.091 | INFO | "gpu" : " 17 GB", | 20260508 06:48:40.091 | INFO | "huge" : "", | 20260508 06:48:40.091 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:40.091 | INFO | "malloc" : " 14 GB", | 20260508 06:48:40.091 | INFO | "pinned" : " 29 GB" | 20260508 06:48:40.091 | INFO | } | 20260508 06:48:40.091 | INFO | Perf report for the last 5.142 seconds: | 20260508 06:48:40.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:40.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.092 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 06:48:40.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.092 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:48:40.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:40.092 | INFO | Uploaded 36GiB to GPU in 2.54144s (14.1652GiB/s) | 20260508 06:48:40.092 | INFO | Downloaded 34GiB to GPU in 2.40745s (14.1228GiB/s) | 20260508 06:48:45.236 | INFO | Analyzing chunk frames = [6076416,6076928) | 20260508 06:48:45.237 | INFO | MemInfo: { | 20260508 06:48:45.237 | INFO | "gpu" : " 17 GB", | 20260508 06:48:45.237 | INFO | "huge" : "", | 20260508 06:48:45.237 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:45.237 | INFO | "malloc" : " 14 GB", | 20260508 06:48:45.237 | INFO | "pinned" : " 29 GB" | 20260508 06:48:45.237 | INFO | } | 20260508 06:48:45.237 | INFO | Perf report for the last 5.14531 seconds: | 20260508 06:48:45.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:45.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 06:48:45.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 06:48:45.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:45.237 | INFO | Uploaded 36GiB to GPU in 2.24584s (16.0296GiB/s) | 20260508 06:48:45.237 | INFO | Downloaded 34GiB to GPU in 2.47674s (13.7277GiB/s) | 20260508 06:48:50.369 | INFO | Analyzing chunk frames = [6076928,6077440) | 20260508 06:48:50.369 | INFO | MemInfo: { | 20260508 06:48:50.369 | INFO | "gpu" : " 17 GB", | 20260508 06:48:50.369 | INFO | "huge" : "", | 20260508 06:48:50.369 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:50.369 | INFO | "malloc" : " 14 GB", | 20260508 06:48:50.369 | INFO | "pinned" : " 29 GB" | 20260508 06:48:50.369 | INFO | } | 20260508 06:48:50.369 | INFO | Perf report for the last 5.13191 seconds: | 20260508 06:48:50.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:50.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 06:48:50.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:48:50.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:50.369 | INFO | Uploaded 36GiB to GPU in 2.41001s (14.9377GiB/s) | 20260508 06:48:50.369 | INFO | Downloaded 34GiB to GPU in 2.43461s (13.9653GiB/s) | 20260508 06:48:55.504 | INFO | Analyzing chunk frames = [6077440,6077952) | 20260508 06:48:55.504 | INFO | MemInfo: { | 20260508 06:48:55.504 | INFO | "gpu" : " 17 GB", | 20260508 06:48:55.504 | INFO | "huge" : "", | 20260508 06:48:55.505 | INFO | "huge_shared" : " 29 GB", | 20260508 06:48:55.505 | INFO | "malloc" : " 14 GB", | 20260508 06:48:55.505 | INFO | "pinned" : " 29 GB" | 20260508 06:48:55.505 | INFO | } | 20260508 06:48:55.505 | INFO | Perf report for the last 5.13589 seconds: | 20260508 06:48:55.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:48:55.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:48:55.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 06:48:55.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:48:55.505 | INFO | Uploaded 36GiB to GPU in 2.47646s (14.5369GiB/s) | 20260508 06:48:55.505 | INFO | Downloaded 34GiB to GPU in 2.43072s (13.9876GiB/s) | 20260508 06:49:00.651 | INFO | Analyzing chunk frames = [6077952,6078464) | 20260508 06:49:00.651 | INFO | MemInfo: { | 20260508 06:49:00.651 | INFO | "gpu" : " 17 GB", | 20260508 06:49:00.651 | INFO | "huge" : "", | 20260508 06:49:00.651 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:00.651 | INFO | "malloc" : " 14 GB", | 20260508 06:49:00.651 | INFO | "pinned" : " 29 GB" | 20260508 06:49:00.651 | INFO | } | 20260508 06:49:00.651 | INFO | Perf report for the last 5.14638 seconds: | 20260508 06:49:00.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:00.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:49:00.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 06:49:00.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:00.651 | INFO | Uploaded 36GiB to GPU in 2.4708s (14.5702GiB/s) | 20260508 06:49:00.651 | INFO | Downloaded 34GiB to GPU in 2.43459s (13.9654GiB/s) | 20260508 06:49:00.652 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:49:00.652 | INFO | Total mem usage: 16.7 GiB | 20260508 06:49:00.652 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:49:00.652 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:49:00.652 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:49:00.652 | INFO | High Water Breakdown: | 20260508 06:49:00.652 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:49:00.652 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:49:00.652 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:49:00.652 | INFO | ------------------------------------------ | 20260508 06:49:00.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:49:00.652 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:49:00.652 | INFO | Total mem usage: 26.5 GiB | 20260508 06:49:00.652 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:49:00.652 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:49:00.652 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:49:00.652 | INFO | High Water Breakdown: | 20260508 06:49:00.652 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:49:00.652 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:49:00.652 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:49:00.652 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:49:00.652 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:49:00.652 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:49:00.652 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:49:00.652 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:49:00.652 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:49:00.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:49:00.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:49:00.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:49:00.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:49:00.652 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:49:00.652 | INFO | ------------------------------------------ | 20260508 06:49:00.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:49:00.652 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:49:00.652 | INFO | Total mem usage: 27.8 GiB | 20260508 06:49:00.652 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:49:00.652 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:49:00.652 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:49:00.652 | INFO | High Water Breakdown: | 20260508 06:49:00.652 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:49:00.652 | INFO | ------------------------------------------ | 20260508 06:49:00.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:49:00.652 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:49:00.652 | INFO | Total mem usage: 15.7 GiB | 20260508 06:49:00.652 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:49:00.652 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:49:00.652 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:49:00.652 | INFO | High Water Breakdown: | 20260508 06:49:00.652 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:49:00.652 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:49:00.652 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:49:00.652 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:49:00.652 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:49:00.652 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:49:00.652 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:49:00.652 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:49:00.652 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:49:00.652 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:49:00.652 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:49:00.652 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:49:00.652 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:49:00.652 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:49:00.652 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:49:00.652 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:49:00.652 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:49:00.652 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:49:00.652 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:49:00.652 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:49:00.652 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:49:00.652 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:49:00.652 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:49:00.652 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:49:00.652 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:49:00.652 | INFO | ------------------------------------------ | 20260508 06:49:00.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:49:00.652 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:49:00.652 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:49:00.652 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:49:00.652 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 6.07e-01 | 20260508 06:49:00.652 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.06e-01, 1.13e-02 / 6.12e-03 | 20260508 06:49:00.652 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.44e+00, 5.88e-01 / 1.68e-01 | 20260508 06:49:00.652 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 9.59e-02 | 20260508 06:49:00.653 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.51e+01, 3.67e+00 / 1.04e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.86e+00 / 1.22e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.55e+00, 4.10e-01 / 2.08e-01 | 20260508 06:49:00.653 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.22e+00, 2.50e-01 / 1.09e-01 | 20260508 06:49:00.653 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.50e-01, 4.98e-02 / 2.86e-02 | 20260508 06:49:00.653 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:49:00.653 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:49:00.653 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:49:00.653 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 24.21%, Upload -- 2.57e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 24.37%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 30.93%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 2.12%, LockUpload -- 2.25e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 2.69%, LockDownload -- 2.85e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 06:49:00.653 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260508 06:49:00.756 | INFO | Starting a round of Baz IO | 20260508 06:49:04.173 | INFO | Populating RTMetrics report | 20260508 06:49:04.173 | INFO | RTMetrics report populated | 20260508 06:49:04.173 | INFO | Opening tmp RTMetrics json file | 20260508 06:49:04.185 | INFO | Serializing RTMetrics to json | 20260508 06:49:04.186 | INFO | Serialization done, swapping tmp file now... | 20260508 06:49:04.187 | INFO | RTMetrics done | 20260508 06:49:04.247 | INFO | Finished a round of Baz IO: Wrote 1.542 GiB in 3.49 seconds | 20260508 06:49:04.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.64% / 0.00% / 59.35% | 20260508 06:49:04.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.376411% / 3.97364e-06% / 4.37101e-05% | 20260508 06:49:04.247 | INFO | Baz IO overall: | 20260508 06:49:04.247 | INFO | BytesWritten : 2690607958875 | 20260508 06:49:04.247 | INFO | NumEvents : 1296919263663 | 20260508 06:49:05.790 | INFO | Analyzing chunk frames = [6078464,6078976) | 20260508 06:49:05.790 | INFO | MemInfo: { | 20260508 06:49:05.790 | INFO | "gpu" : " 17 GB", | 20260508 06:49:05.790 | INFO | "huge" : "", | 20260508 06:49:05.790 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:05.790 | INFO | "malloc" : " 14 GB", | 20260508 06:49:05.790 | INFO | "pinned" : " 29 GB" | 20260508 06:49:05.790 | INFO | } | 20260508 06:49:05.791 | INFO | Perf report for the last 5.1395 seconds: | 20260508 06:49:05.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:05.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:05.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:05.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:05.791 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:49:05.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:05.791 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 06:49:05.791 | INFO | PRE_HQ : 65.6%, 1.6, 55.4ms, 0 | 20260508 06:49:05.791 | INFO | BAZWRITER : 67.5%, 2.6, 90.6ms, 0 | 20260508 06:49:05.791 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:49:05.791 | INFO | Uploaded 36GiB to GPU in 2.60556s (13.8166GiB/s) | 20260508 06:49:05.791 | INFO | Downloaded 38GiB to GPU in 2.69541s (14.098GiB/s) | 20260508 06:49:10.928 | INFO | Analyzing chunk frames = [6078976,6079488) | 20260508 06:49:10.929 | INFO | MemInfo: { | 20260508 06:49:10.929 | INFO | "gpu" : " 17 GB", | 20260508 06:49:10.929 | INFO | "huge" : "", | 20260508 06:49:10.929 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:10.929 | INFO | "malloc" : " 14 GB", | 20260508 06:49:10.929 | INFO | "pinned" : " 29 GB" | 20260508 06:49:10.929 | INFO | } | 20260508 06:49:10.929 | INFO | Perf report for the last 5.13818 seconds: | 20260508 06:49:10.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:10.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 06:49:10.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:49:10.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:10.929 | INFO | Uploaded 36GiB to GPU in 2.51145s (14.3344GiB/s) | 20260508 06:49:10.929 | INFO | Downloaded 34GiB to GPU in 2.43763s (13.948GiB/s) | 20260508 06:49:16.070 | INFO | Analyzing chunk frames = [6079488,6080000) | 20260508 06:49:16.070 | INFO | MemInfo: { | 20260508 06:49:16.070 | INFO | "gpu" : " 17 GB", | 20260508 06:49:16.070 | INFO | "huge" : "", | 20260508 06:49:16.070 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:16.070 | INFO | "malloc" : " 14 GB", | 20260508 06:49:16.070 | INFO | "pinned" : " 29 GB" | 20260508 06:49:16.070 | INFO | } | 20260508 06:49:16.070 | INFO | Perf report for the last 5.14132 seconds: | 20260508 06:49:16.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:16.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 06:49:16.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:49:16.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:16.070 | INFO | Uploaded 36GiB to GPU in 2.54893s (14.1236GiB/s) | 20260508 06:49:16.070 | INFO | Downloaded 34GiB to GPU in 2.39968s (14.1686GiB/s) | 20260508 06:49:21.204 | INFO | Analyzing chunk frames = [6080000,6080512) | 20260508 06:49:21.204 | INFO | MemInfo: { | 20260508 06:49:21.204 | INFO | "gpu" : " 17 GB", | 20260508 06:49:21.204 | INFO | "huge" : "", | 20260508 06:49:21.204 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:21.205 | INFO | "malloc" : " 14 GB", | 20260508 06:49:21.205 | INFO | "pinned" : " 29 GB" | 20260508 06:49:21.205 | INFO | } | 20260508 06:49:21.205 | INFO | Perf report for the last 5.13468 seconds: | 20260508 06:49:21.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:21.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 06:49:21.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:49:21.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:21.205 | INFO | Uploaded 36GiB to GPU in 2.58738s (13.9137GiB/s) | 20260508 06:49:21.205 | INFO | Downloaded 34GiB to GPU in 2.42792s (14.0037GiB/s) | 20260508 06:49:26.349 | INFO | Analyzing chunk frames = [6080512,6081024) | 20260508 06:49:26.349 | INFO | MemInfo: { | 20260508 06:49:26.349 | INFO | "gpu" : " 17 GB", | 20260508 06:49:26.349 | INFO | "huge" : "", | 20260508 06:49:26.349 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:26.349 | INFO | "malloc" : " 14 GB", | 20260508 06:49:26.349 | INFO | "pinned" : " 29 GB" | 20260508 06:49:26.349 | INFO | } | 20260508 06:49:26.350 | INFO | Perf report for the last 5.14482 seconds: | 20260508 06:49:26.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:26.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:49:26.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:49:26.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:26.350 | INFO | Uploaded 36GiB to GPU in 2.46815s (14.5858GiB/s) | 20260508 06:49:26.350 | INFO | Downloaded 34GiB to GPU in 2.55443s (13.3102GiB/s) | 20260508 06:49:31.488 | INFO | Analyzing chunk frames = [6081024,6081536) | 20260508 06:49:31.488 | INFO | MemInfo: { | 20260508 06:49:31.488 | INFO | "gpu" : " 17 GB", | 20260508 06:49:31.488 | INFO | "huge" : "", | 20260508 06:49:31.488 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:31.488 | INFO | "malloc" : " 14 GB", | 20260508 06:49:31.488 | INFO | "pinned" : " 29 GB" | 20260508 06:49:31.488 | INFO | } | 20260508 06:49:31.488 | INFO | Perf report for the last 5.13867 seconds: | 20260508 06:49:31.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:31.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.488 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 06:49:31.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.488 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260508 06:49:31.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:31.489 | INFO | Uploaded 36GiB to GPU in 2.38052s (15.1228GiB/s) | 20260508 06:49:31.489 | INFO | Downloaded 34GiB to GPU in 2.42583s (14.0158GiB/s) | 20260508 06:49:36.622 | INFO | Analyzing chunk frames = [6081536,6082048) | 20260508 06:49:36.622 | INFO | MemInfo: { | 20260508 06:49:36.622 | INFO | "gpu" : " 17 GB", | 20260508 06:49:36.622 | INFO | "huge" : "", | 20260508 06:49:36.622 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:36.622 | INFO | "malloc" : " 14 GB", | 20260508 06:49:36.622 | INFO | "pinned" : " 29 GB" | 20260508 06:49:36.622 | INFO | } | 20260508 06:49:36.622 | INFO | Perf report for the last 5.13407 seconds: | 20260508 06:49:36.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:36.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.622 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 06:49:36.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.622 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:49:36.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:36.623 | INFO | Uploaded 36GiB to GPU in 2.47923s (14.5206GiB/s) | 20260508 06:49:36.623 | INFO | Downloaded 34GiB to GPU in 2.42949s (13.9947GiB/s) | 20260508 06:49:41.759 | INFO | Analyzing chunk frames = [6082048,6082560) | 20260508 06:49:41.759 | INFO | MemInfo: { | 20260508 06:49:41.759 | INFO | "gpu" : " 17 GB", | 20260508 06:49:41.759 | INFO | "huge" : "", | 20260508 06:49:41.759 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:41.759 | INFO | "malloc" : " 14 GB", | 20260508 06:49:41.759 | INFO | "pinned" : " 29 GB" | 20260508 06:49:41.759 | INFO | } | 20260508 06:49:41.759 | INFO | Perf report for the last 5.13694 seconds: | 20260508 06:49:41.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:41.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.759 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:49:41.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.759 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 06:49:41.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:41.760 | INFO | Uploaded 36GiB to GPU in 2.48008s (14.5157GiB/s) | 20260508 06:49:41.760 | INFO | Downloaded 34GiB to GPU in 2.44031s (13.9326GiB/s) | 20260508 06:49:41.856 | INFO | Starting a round of Baz IO | 20260508 06:49:45.279 | INFO | Populating RTMetrics report | 20260508 06:49:45.279 | INFO | RTMetrics report populated | 20260508 06:49:45.279 | INFO | Opening tmp RTMetrics json file | 20260508 06:49:45.289 | INFO | Serializing RTMetrics to json | 20260508 06:49:45.292 | INFO | Serialization done, swapping tmp file now... | 20260508 06:49:45.293 | INFO | RTMetrics done | 20260508 06:49:45.351 | INFO | Finished a round of Baz IO: Wrote 1.539 GiB in 3.494 seconds | 20260508 06:49:45.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.64% / 0.00% / 59.36% | 20260508 06:49:45.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374643% / 3.97364e-06% / 4.76837e-05% | 20260508 06:49:45.351 | INFO | Baz IO overall: | 20260508 06:49:45.351 | INFO | BytesWritten : 2692260482023 | 20260508 06:49:45.351 | INFO | NumEvents : 1297704742322 | 20260508 06:49:46.905 | INFO | Analyzing chunk frames = [6082560,6083072) | 20260508 06:49:46.905 | INFO | MemInfo: { | 20260508 06:49:46.905 | INFO | "gpu" : " 17 GB", | 20260508 06:49:46.905 | INFO | "huge" : "", | 20260508 06:49:46.905 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:46.905 | INFO | "malloc" : " 14 GB", | 20260508 06:49:46.905 | INFO | "pinned" : " 29 GB" | 20260508 06:49:46.905 | INFO | } | 20260508 06:49:46.905 | INFO | Perf report for the last 5.14576 seconds: | 20260508 06:49:46.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:46.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:46.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:46.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:46.905 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 06:49:46.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:46.905 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 06:49:46.905 | INFO | PRE_HQ : 65.6%, 1.6, 55.1ms, 0 | 20260508 06:49:46.905 | INFO | BAZWRITER : 67.4%, 2.6, 89.2ms, 0 | 20260508 06:49:46.905 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:49:46.906 | INFO | Uploaded 36GiB to GPU in 2.58782s (13.9113GiB/s) | 20260508 06:49:46.906 | INFO | Downloaded 38GiB to GPU in 2.68414s (14.1572GiB/s) | 20260508 06:49:52.044 | INFO | Analyzing chunk frames = [6083072,6083584) | 20260508 06:49:52.044 | INFO | MemInfo: { | 20260508 06:49:52.044 | INFO | "gpu" : " 17 GB", | 20260508 06:49:52.044 | INFO | "huge" : "", | 20260508 06:49:52.044 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:52.044 | INFO | "malloc" : " 14 GB", | 20260508 06:49:52.044 | INFO | "pinned" : " 29 GB" | 20260508 06:49:52.044 | INFO | } | 20260508 06:49:52.044 | INFO | Perf report for the last 5.13931 seconds: | 20260508 06:49:52.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:52.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.044 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 06:49:52.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.045 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:49:52.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:52.045 | INFO | Uploaded 36GiB to GPU in 2.51229s (14.3296GiB/s) | 20260508 06:49:52.045 | INFO | Downloaded 34GiB to GPU in 2.43238s (13.9781GiB/s) | 20260508 06:49:57.183 | INFO | Analyzing chunk frames = [6083584,6084096) | 20260508 06:49:57.183 | INFO | MemInfo: { | 20260508 06:49:57.183 | INFO | "gpu" : " 17 GB", | 20260508 06:49:57.183 | INFO | "huge" : "", | 20260508 06:49:57.183 | INFO | "huge_shared" : " 29 GB", | 20260508 06:49:57.183 | INFO | "malloc" : " 14 GB", | 20260508 06:49:57.183 | INFO | "pinned" : " 29 GB" | 20260508 06:49:57.183 | INFO | } | 20260508 06:49:57.183 | INFO | Perf report for the last 5.13837 seconds: | 20260508 06:49:57.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:49:57.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 06:49:57.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:49:57.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:49:57.183 | INFO | Uploaded 36GiB to GPU in 2.56577s (14.0309GiB/s) | 20260508 06:49:57.183 | INFO | Downloaded 34GiB to GPU in 2.42131s (14.042GiB/s) | 20260508 06:50:02.319 | INFO | Analyzing chunk frames = [6084096,6084608) | 20260508 06:50:02.319 | INFO | MemInfo: { | 20260508 06:50:02.319 | INFO | "gpu" : " 17 GB", | 20260508 06:50:02.319 | INFO | "huge" : "", | 20260508 06:50:02.319 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:02.319 | INFO | "malloc" : " 14 GB", | 20260508 06:50:02.319 | INFO | "pinned" : " 29 GB" | 20260508 06:50:02.319 | INFO | } | 20260508 06:50:02.319 | INFO | Perf report for the last 5.13652 seconds: | 20260508 06:50:02.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:02.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.319 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:50:02.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.319 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:50:02.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:02.320 | INFO | Uploaded 36GiB to GPU in 2.56217s (14.0506GiB/s) | 20260508 06:50:02.320 | INFO | Downloaded 34GiB to GPU in 2.39176s (14.2155GiB/s) | 20260508 06:50:07.462 | INFO | Analyzing chunk frames = [6084608,6085120) | 20260508 06:50:07.462 | INFO | MemInfo: { | 20260508 06:50:07.462 | INFO | "gpu" : " 17 GB", | 20260508 06:50:07.462 | INFO | "huge" : "", | 20260508 06:50:07.462 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:07.462 | INFO | "malloc" : " 14 GB", | 20260508 06:50:07.462 | INFO | "pinned" : " 29 GB" | 20260508 06:50:07.462 | INFO | } | 20260508 06:50:07.462 | INFO | Perf report for the last 5.14315 seconds: | 20260508 06:50:07.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:07.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.463 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 06:50:07.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.463 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 06:50:07.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:07.463 | INFO | Uploaded 36GiB to GPU in 2.36914s (15.1954GiB/s) | 20260508 06:50:07.463 | INFO | Downloaded 34GiB to GPU in 2.51229s (13.5335GiB/s) | 20260508 06:50:12.598 | INFO | Analyzing chunk frames = [6085120,6085632) | 20260508 06:50:12.598 | INFO | MemInfo: { | 20260508 06:50:12.598 | INFO | "gpu" : " 17 GB", | 20260508 06:50:12.598 | INFO | "huge" : "", | 20260508 06:50:12.598 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:12.598 | INFO | "malloc" : " 14 GB", | 20260508 06:50:12.598 | INFO | "pinned" : " 29 GB" | 20260508 06:50:12.598 | INFO | } | 20260508 06:50:12.598 | INFO | Perf report for the last 5.13548 seconds: | 20260508 06:50:12.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:12.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 06:50:12.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:50:12.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:12.598 | INFO | Uploaded 36GiB to GPU in 2.39272s (15.0456GiB/s) | 20260508 06:50:12.598 | INFO | Downloaded 34GiB to GPU in 2.42788s (14.004GiB/s) | 20260508 06:50:17.743 | INFO | Analyzing chunk frames = [6085632,6086144) | 20260508 06:50:17.743 | INFO | MemInfo: { | 20260508 06:50:17.743 | INFO | "gpu" : " 17 GB", | 20260508 06:50:17.743 | INFO | "huge" : "", | 20260508 06:50:17.743 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:17.743 | INFO | "malloc" : " 14 GB", | 20260508 06:50:17.743 | INFO | "pinned" : " 29 GB" | 20260508 06:50:17.743 | INFO | } | 20260508 06:50:17.743 | INFO | Perf report for the last 5.14545 seconds: | 20260508 06:50:17.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:17.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.744 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:50:17.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.744 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:50:17.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:17.744 | INFO | Uploaded 36GiB to GPU in 2.47695s (14.534GiB/s) | 20260508 06:50:17.744 | INFO | Downloaded 34GiB to GPU in 2.43174s (13.9818GiB/s) | 20260508 06:50:22.881 | INFO | Analyzing chunk frames = [6086144,6086656) | 20260508 06:50:22.881 | INFO | MemInfo: { | 20260508 06:50:22.881 | INFO | "gpu" : " 17 GB", | 20260508 06:50:22.881 | INFO | "huge" : "", | 20260508 06:50:22.881 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:22.881 | INFO | "malloc" : " 14 GB", | 20260508 06:50:22.881 | INFO | "pinned" : " 29 GB" | 20260508 06:50:22.881 | INFO | } | 20260508 06:50:22.881 | INFO | Perf report for the last 5.13768 seconds: | 20260508 06:50:22.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:22.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.881 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:50:22.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.881 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 06:50:22.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:22.882 | INFO | Uploaded 36GiB to GPU in 2.49664s (14.4194GiB/s) | 20260508 06:50:22.882 | INFO | Downloaded 34GiB to GPU in 2.44458s (13.9083GiB/s) | 20260508 06:50:22.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:50:22.882 | INFO | Total mem usage: 16.7 GiB | 20260508 06:50:22.882 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:50:22.882 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 06:50:22.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:50:22.882 | INFO | High Water Breakdown: | 20260508 06:50:22.882 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:50:22.882 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:50:22.882 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:50:22.882 | INFO | ------------------------------------------ | 20260508 06:50:22.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:50:22.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:50:22.882 | INFO | Total mem usage: 26.5 GiB | 20260508 06:50:22.882 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:50:22.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:50:22.882 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:50:22.882 | INFO | High Water Breakdown: | 20260508 06:50:22.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:50:22.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:50:22.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:50:22.882 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:50:22.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:50:22.882 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:50:22.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:50:22.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:50:22.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:50:22.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:50:22.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:50:22.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:50:22.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:50:22.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:50:22.882 | INFO | ------------------------------------------ | 20260508 06:50:22.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:50:22.882 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:50:22.882 | INFO | Total mem usage: 27.8 GiB | 20260508 06:50:22.882 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:50:22.882 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:50:22.882 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:50:22.882 | INFO | High Water Breakdown: | 20260508 06:50:22.882 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:50:22.882 | INFO | ------------------------------------------ | 20260508 06:50:22.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:50:22.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:50:22.883 | INFO | Total mem usage: 15.7 GiB | 20260508 06:50:22.883 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:50:22.883 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:50:22.883 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:50:22.883 | INFO | High Water Breakdown: | 20260508 06:50:22.883 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:50:22.883 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:50:22.883 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:50:22.883 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:50:22.883 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:50:22.883 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:50:22.883 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:50:22.883 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:50:22.883 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:50:22.883 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:50:22.883 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:50:22.883 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:50:22.883 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:50:22.883 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:50:22.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:50:22.883 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:50:22.883 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:50:22.883 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:50:22.883 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:50:22.883 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:50:22.883 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:50:22.883 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:50:22.883 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:50:22.883 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:50:22.883 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:50:22.883 | INFO | ------------------------------------------ | 20260508 06:50:22.883 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:50:22.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:50:22.883 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:50:22.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:50:22.883 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 7.49e-01 | 20260508 06:50:22.883 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.05e-01, 1.13e-02 / 7.44e-03 | 20260508 06:50:22.883 | INFO | PerfReport: 12.11%, Baseline -- 3.95e+00 / 4.38e+00, 5.81e-01 / 2.63e-01 | 20260508 06:50:22.883 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.42e+00, 2.08e-01 / 1.30e-01 | 20260508 06:50:22.883 | INFO | PerfReport: 15.90%, DME -- 4.15e+01 / 4.48e+01, 3.69e+00 / 1.01e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.48e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.55e+00, 4.12e-01 / 2.48e-01 | 20260508 06:50:22.883 | INFO | PerfReport: 3.14%, Metrics -- 1.02e+00 / 1.22e+00, 2.56e-01 / 1.51e-01 | 20260508 06:50:22.883 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.47e-01, 4.92e-02 / 2.67e-02 | 20260508 06:50:22.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:50:22.883 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:50:22.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:50:22.883 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.53e+01 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 24.55%, Upload -- 2.61e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 24.27%, Download -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 30.75%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 2.46%, LockUpload -- 2.62e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 2.43%, LockDownload -- 2.58e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:50:22.883 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.22e-02 / 0.00e+00 | 20260508 06:50:22.978 | INFO | Starting a round of Baz IO | 20260508 06:50:26.404 | INFO | Populating RTMetrics report | 20260508 06:50:26.404 | INFO | RTMetrics report populated | 20260508 06:50:26.404 | INFO | Opening tmp RTMetrics json file | 20260508 06:50:26.416 | INFO | Serializing RTMetrics to json | 20260508 06:50:26.417 | INFO | Serialization done, swapping tmp file now... | 20260508 06:50:26.417 | INFO | RTMetrics done | 20260508 06:50:26.477 | INFO | Finished a round of Baz IO: Wrote 1.541 GiB in 3.498 seconds | 20260508 06:50:26.477 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.64% / 0.00% / 59.36% | 20260508 06:50:26.477 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.375458% / 3.97364e-06% / 4.76837e-05% | 20260508 06:50:26.477 | INFO | Baz IO overall: | 20260508 06:50:26.477 | INFO | BytesWritten : 2693915187851 | 20260508 06:50:26.477 | INFO | NumEvents : 1298492031096 | 20260508 06:50:28.017 | INFO | Analyzing chunk frames = [6086656,6087168) | 20260508 06:50:28.017 | INFO | MemInfo: { | 20260508 06:50:28.017 | INFO | "gpu" : " 17 GB", | 20260508 06:50:28.017 | INFO | "huge" : "", | 20260508 06:50:28.017 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:28.017 | INFO | "malloc" : " 14 GB", | 20260508 06:50:28.017 | INFO | "pinned" : " 29 GB" | 20260508 06:50:28.017 | INFO | } | 20260508 06:50:28.017 | INFO | Perf report for the last 5.13616 seconds: | 20260508 06:50:28.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:28.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:28.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:28.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:28.017 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 06:50:28.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:28.017 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 06:50:28.017 | INFO | PRE_HQ : 65.6%, 1.6, 54.9ms, 0 | 20260508 06:50:28.017 | INFO | BAZWRITER : 67.5%, 2.6, 88.5ms, 0 | 20260508 06:50:28.017 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:50:28.018 | INFO | Uploaded 36GiB to GPU in 2.59873s (13.8529GiB/s) | 20260508 06:50:28.018 | INFO | Downloaded 38GiB to GPU in 2.68613s (14.1468GiB/s) | 20260508 06:50:33.160 | INFO | Analyzing chunk frames = [6087168,6087680) | 20260508 06:50:33.160 | INFO | MemInfo: { | 20260508 06:50:33.160 | INFO | "gpu" : " 17 GB", | 20260508 06:50:33.160 | INFO | "huge" : "", | 20260508 06:50:33.160 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:33.160 | INFO | "malloc" : " 14 GB", | 20260508 06:50:33.160 | INFO | "pinned" : " 29 GB" | 20260508 06:50:33.160 | INFO | } | 20260508 06:50:33.160 | INFO | Perf report for the last 5.14256 seconds: | 20260508 06:50:33.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:33.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:50:33.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:50:33.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:33.160 | INFO | Uploaded 36GiB to GPU in 2.49925s (14.4043GiB/s) | 20260508 06:50:33.161 | INFO | Downloaded 34GiB to GPU in 2.43203s (13.9801GiB/s) | 20260508 06:50:38.302 | INFO | Analyzing chunk frames = [6087680,6088192) | 20260508 06:50:38.302 | INFO | MemInfo: { | 20260508 06:50:38.302 | INFO | "gpu" : " 17 GB", | 20260508 06:50:38.302 | INFO | "huge" : "", | 20260508 06:50:38.302 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:38.302 | INFO | "malloc" : " 14 GB", | 20260508 06:50:38.302 | INFO | "pinned" : " 29 GB" | 20260508 06:50:38.302 | INFO | } | 20260508 06:50:38.302 | INFO | Perf report for the last 5.14202 seconds: | 20260508 06:50:38.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:38.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 06:50:38.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 06:50:38.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:38.302 | INFO | Uploaded 36GiB to GPU in 2.56056s (14.0594GiB/s) | 20260508 06:50:38.302 | INFO | Downloaded 34GiB to GPU in 2.41929s (14.0537GiB/s) | 20260508 06:50:43.437 | INFO | Analyzing chunk frames = [6088192,6088704) | 20260508 06:50:43.437 | INFO | MemInfo: { | 20260508 06:50:43.437 | INFO | "gpu" : " 17 GB", | 20260508 06:50:43.437 | INFO | "huge" : "", | 20260508 06:50:43.437 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:43.437 | INFO | "malloc" : " 14 GB", | 20260508 06:50:43.437 | INFO | "pinned" : " 29 GB" | 20260508 06:50:43.437 | INFO | } | 20260508 06:50:43.437 | INFO | Perf report for the last 5.13547 seconds: | 20260508 06:50:43.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:43.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.437 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 06:50:43.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.437 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:50:43.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:43.438 | INFO | Uploaded 36GiB to GPU in 2.56419s (14.0395GiB/s) | 20260508 06:50:43.438 | INFO | Downloaded 34GiB to GPU in 2.38916s (14.2309GiB/s) | 20260508 06:50:48.573 | INFO | Analyzing chunk frames = [6088704,6089216) | 20260508 06:50:48.573 | INFO | MemInfo: { | 20260508 06:50:48.573 | INFO | "gpu" : " 17 GB", | 20260508 06:50:48.573 | INFO | "huge" : "", | 20260508 06:50:48.573 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:48.573 | INFO | "malloc" : " 14 GB", | 20260508 06:50:48.573 | INFO | "pinned" : " 29 GB" | 20260508 06:50:48.574 | INFO | } | 20260508 06:50:48.574 | INFO | Perf report for the last 5.13618 seconds: | 20260508 06:50:48.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:48.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 06:50:48.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:50:48.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:48.574 | INFO | Uploaded 36GiB to GPU in 2.30698s (15.6048GiB/s) | 20260508 06:50:48.574 | INFO | Downloaded 34GiB to GPU in 2.48276s (13.6944GiB/s) | 20260508 06:50:53.715 | INFO | Analyzing chunk frames = [6089216,6089728) | 20260508 06:50:53.715 | INFO | MemInfo: { | 20260508 06:50:53.715 | INFO | "gpu" : " 17 GB", | 20260508 06:50:53.715 | INFO | "huge" : "", | 20260508 06:50:53.715 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:53.715 | INFO | "malloc" : " 14 GB", | 20260508 06:50:53.715 | INFO | "pinned" : " 29 GB" | 20260508 06:50:53.715 | INFO | } | 20260508 06:50:53.715 | INFO | Perf report for the last 5.14188 seconds: | 20260508 06:50:53.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:53.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.716 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 06:50:53.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.716 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:50:53.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:53.716 | INFO | Uploaded 36GiB to GPU in 2.37769s (15.1408GiB/s) | 20260508 06:50:53.716 | INFO | Downloaded 34GiB to GPU in 2.41953s (14.0523GiB/s) | 20260508 06:50:58.856 | INFO | Analyzing chunk frames = [6089728,6090240) | 20260508 06:50:58.856 | INFO | MemInfo: { | 20260508 06:50:58.856 | INFO | "gpu" : " 17 GB", | 20260508 06:50:58.856 | INFO | "huge" : "", | 20260508 06:50:58.856 | INFO | "huge_shared" : " 29 GB", | 20260508 06:50:58.856 | INFO | "malloc" : " 14 GB", | 20260508 06:50:58.856 | INFO | "pinned" : " 29 GB" | 20260508 06:50:58.856 | INFO | } | 20260508 06:50:58.856 | INFO | Perf report for the last 5.14054 seconds: | 20260508 06:50:58.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:50:58.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:50:58.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:50:58.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:50:58.856 | INFO | Uploaded 36GiB to GPU in 2.48208s (14.504GiB/s) | 20260508 06:50:58.856 | INFO | Downloaded 34GiB to GPU in 2.42897s (13.9977GiB/s) | 20260508 06:51:03.997 | INFO | Analyzing chunk frames = [6090240,6090752) | 20260508 06:51:03.997 | INFO | MemInfo: { | 20260508 06:51:03.997 | INFO | "gpu" : " 17 GB", | 20260508 06:51:03.997 | INFO | "huge" : "", | 20260508 06:51:03.997 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:03.997 | INFO | "malloc" : " 14 GB", | 20260508 06:51:03.997 | INFO | "pinned" : " 29 GB" | 20260508 06:51:03.997 | INFO | } | 20260508 06:51:03.997 | INFO | Perf report for the last 5.14144 seconds: | 20260508 06:51:03.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:03.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.998 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:51:03.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.998 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:51:03.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:03.998 | INFO | Uploaded 36GiB to GPU in 2.46965s (14.5769GiB/s) | 20260508 06:51:03.998 | INFO | Downloaded 34GiB to GPU in 2.42905s (13.9972GiB/s) | 20260508 06:51:04.095 | INFO | Starting a round of Baz IO | 20260508 06:51:07.527 | INFO | Populating RTMetrics report | 20260508 06:51:07.527 | INFO | RTMetrics report populated | 20260508 06:51:07.527 | INFO | Opening tmp RTMetrics json file | 20260508 06:51:07.528 | INFO | Serializing RTMetrics to json | 20260508 06:51:07.529 | INFO | Serialization done, swapping tmp file now... | 20260508 06:51:07.529 | INFO | RTMetrics done | 20260508 06:51:07.592 | INFO | Finished a round of Baz IO: Wrote 1.54 GiB in 3.497 seconds | 20260508 06:51:07.592 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.64% / 0.00% / 59.36% | 20260508 06:51:07.592 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374806% / 3.97364e-06% / 5.5631e-05% | 20260508 06:51:07.592 | INFO | Baz IO overall: | 20260508 06:51:07.592 | INFO | BytesWritten : 2695568943163 | 20260508 06:51:07.592 | INFO | NumEvents : 1299278355858 | 20260508 06:51:09.135 | INFO | Analyzing chunk frames = [6090752,6091264) | 20260508 06:51:09.135 | INFO | MemInfo: { | 20260508 06:51:09.135 | INFO | "gpu" : " 17 GB", | 20260508 06:51:09.135 | INFO | "huge" : "", | 20260508 06:51:09.135 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:09.135 | INFO | "malloc" : " 14 GB", | 20260508 06:51:09.135 | INFO | "pinned" : " 29 GB" | 20260508 06:51:09.135 | INFO | } | 20260508 06:51:09.135 | INFO | Perf report for the last 5.13801 seconds: | 20260508 06:51:09.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:09.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:09.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:09.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:09.136 | INFO | ANALYSIS : 68.3%, 3.0, 106.2ms, 0 | 20260508 06:51:09.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:09.136 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 06:51:09.136 | INFO | PRE_HQ : 66.3%, 1.6, 55.2ms, 0 | 20260508 06:51:09.136 | INFO | BAZWRITER : 67.5%, 2.6, 89.2ms, 0 | 20260508 06:51:09.136 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 06:51:09.136 | INFO | Uploaded 36GiB to GPU in 2.55014s (14.1169GiB/s) | 20260508 06:51:09.136 | INFO | Downloaded 38GiB to GPU in 2.68261s (14.1653GiB/s) | 20260508 06:51:14.270 | INFO | Analyzing chunk frames = [6091264,6091776) | 20260508 06:51:14.270 | INFO | MemInfo: { | 20260508 06:51:14.270 | INFO | "gpu" : " 17 GB", | 20260508 06:51:14.270 | INFO | "huge" : "", | 20260508 06:51:14.270 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:14.270 | INFO | "malloc" : " 14 GB", | 20260508 06:51:14.270 | INFO | "pinned" : " 29 GB" | 20260508 06:51:14.270 | INFO | } | 20260508 06:51:14.270 | INFO | Perf report for the last 5.13437 seconds: | 20260508 06:51:14.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:14.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:51:14.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 06:51:14.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:14.270 | INFO | Uploaded 36GiB to GPU in 2.48767s (14.4714GiB/s) | 20260508 06:51:14.271 | INFO | Downloaded 34GiB to GPU in 2.42329s (14.0305GiB/s) | 20260508 06:51:19.413 | INFO | Analyzing chunk frames = [6091776,6092288) | 20260508 06:51:19.413 | INFO | MemInfo: { | 20260508 06:51:19.413 | INFO | "gpu" : " 17 GB", | 20260508 06:51:19.413 | INFO | "huge" : "", | 20260508 06:51:19.413 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:19.413 | INFO | "malloc" : " 14 GB", | 20260508 06:51:19.413 | INFO | "pinned" : " 29 GB" | 20260508 06:51:19.413 | INFO | } | 20260508 06:51:19.413 | INFO | Perf report for the last 5.14354 seconds: | 20260508 06:51:19.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:19.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.414 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 06:51:19.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.414 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:51:19.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:19.414 | INFO | Uploaded 36GiB to GPU in 2.53081s (14.2247GiB/s) | 20260508 06:51:19.414 | INFO | Downloaded 34GiB to GPU in 2.40062s (14.163GiB/s) | 20260508 06:51:24.551 | INFO | Analyzing chunk frames = [6092288,6092800) | 20260508 06:51:24.551 | INFO | MemInfo: { | 20260508 06:51:24.551 | INFO | "gpu" : " 17 GB", | 20260508 06:51:24.551 | INFO | "huge" : "", | 20260508 06:51:24.551 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:24.551 | INFO | "malloc" : " 14 GB", | 20260508 06:51:24.551 | INFO | "pinned" : " 29 GB" | 20260508 06:51:24.551 | INFO | } | 20260508 06:51:24.551 | INFO | Perf report for the last 5.13753 seconds: | 20260508 06:51:24.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:24.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.551 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 06:51:24.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.551 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:51:24.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:24.552 | INFO | Uploaded 36GiB to GPU in 2.59066s (13.8961GiB/s) | 20260508 06:51:24.552 | INFO | Downloaded 34GiB to GPU in 2.41808s (14.0607GiB/s) | 20260508 06:51:29.695 | INFO | Analyzing chunk frames = [6092800,6093312) | 20260508 06:51:29.695 | INFO | MemInfo: { | 20260508 06:51:29.695 | INFO | "gpu" : " 17 GB", | 20260508 06:51:29.695 | INFO | "huge" : "", | 20260508 06:51:29.695 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:29.695 | INFO | "malloc" : " 14 GB", | 20260508 06:51:29.695 | INFO | "pinned" : " 29 GB" | 20260508 06:51:29.695 | INFO | } | 20260508 06:51:29.695 | INFO | Perf report for the last 5.14442 seconds: | 20260508 06:51:29.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:29.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.695 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 06:51:29.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.696 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 06:51:29.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:29.696 | INFO | Uploaded 36GiB to GPU in 2.41506s (14.9065GiB/s) | 20260508 06:51:29.696 | INFO | Downloaded 34GiB to GPU in 2.51512s (13.5183GiB/s) | 20260508 06:51:34.830 | INFO | Analyzing chunk frames = [6093312,6093824) | 20260508 06:51:34.830 | INFO | MemInfo: { | 20260508 06:51:34.830 | INFO | "gpu" : " 17 GB", | 20260508 06:51:34.830 | INFO | "huge" : "", | 20260508 06:51:34.831 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:34.831 | INFO | "malloc" : " 14 GB", | 20260508 06:51:34.831 | INFO | "pinned" : " 29 GB" | 20260508 06:51:34.831 | INFO | } | 20260508 06:51:34.831 | INFO | Perf report for the last 5.13529 seconds: | 20260508 06:51:34.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:34.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 06:51:34.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:51:34.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:34.831 | INFO | Uploaded 36GiB to GPU in 2.36463s (15.2244GiB/s) | 20260508 06:51:34.831 | INFO | Downloaded 34GiB to GPU in 2.42812s (14.0026GiB/s) | 20260508 06:51:39.976 | INFO | Analyzing chunk frames = [6093824,6094336) | 20260508 06:51:39.976 | INFO | MemInfo: { | 20260508 06:51:39.976 | INFO | "gpu" : " 17 GB", | 20260508 06:51:39.976 | INFO | "huge" : "", | 20260508 06:51:39.976 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:39.976 | INFO | "malloc" : " 14 GB", | 20260508 06:51:39.976 | INFO | "pinned" : " 29 GB" | 20260508 06:51:39.976 | INFO | } | 20260508 06:51:39.976 | INFO | Perf report for the last 5.14566 seconds: | 20260508 06:51:39.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:39.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.976 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 06:51:39.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.976 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:51:39.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:39.977 | INFO | Uploaded 36GiB to GPU in 2.46926s (14.5793GiB/s) | 20260508 06:51:39.977 | INFO | Downloaded 34GiB to GPU in 2.4362s (13.9561GiB/s) | 20260508 06:51:45.109 | INFO | Analyzing chunk frames = [6094336,6094848) | 20260508 06:51:45.109 | INFO | MemInfo: { | 20260508 06:51:45.109 | INFO | "gpu" : " 17 GB", | 20260508 06:51:45.109 | INFO | "huge" : "", | 20260508 06:51:45.109 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:45.109 | INFO | "malloc" : " 14 GB", | 20260508 06:51:45.109 | INFO | "pinned" : " 29 GB" | 20260508 06:51:45.109 | INFO | } | 20260508 06:51:45.109 | INFO | Perf report for the last 5.13292 seconds: | 20260508 06:51:45.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:45.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.109 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:51:45.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.109 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:51:45.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:45.110 | INFO | Uploaded 36GiB to GPU in 2.49068s (14.4539GiB/s) | 20260508 06:51:45.110 | INFO | Downloaded 34GiB to GPU in 2.44516s (13.905GiB/s) | 20260508 06:51:45.110 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:51:45.110 | INFO | Total mem usage: 16.7 GiB | 20260508 06:51:45.110 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:51:45.110 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:51:45.110 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:51:45.110 | INFO | High Water Breakdown: | 20260508 06:51:45.110 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:51:45.110 | INFO | 40.4% : 5.76 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:51:45.110 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:51:45.110 | INFO | ------------------------------------------ | 20260508 06:51:45.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:51:45.110 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:51:45.110 | INFO | Total mem usage: 26.5 GiB | 20260508 06:51:45.110 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:51:45.110 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.14%) | 20260508 06:51:45.110 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260508 06:51:45.110 | INFO | High Water Breakdown: | 20260508 06:51:45.110 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:51:45.110 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:51:45.110 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:51:45.110 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:51:45.110 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:51:45.110 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:51:45.110 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:51:45.110 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:51:45.110 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:51:45.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:51:45.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:51:45.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:51:45.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:51:45.110 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:51:45.110 | INFO | ------------------------------------------ | 20260508 06:51:45.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:51:45.110 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:51:45.110 | INFO | Total mem usage: 27.8 GiB | 20260508 06:51:45.110 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:51:45.110 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 06:51:45.110 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 06:51:45.110 | INFO | High Water Breakdown: | 20260508 06:51:45.110 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 06:51:45.110 | INFO | ------------------------------------------ | 20260508 06:51:45.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:51:45.110 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:51:45.110 | INFO | Total mem usage: 15.7 GiB | 20260508 06:51:45.110 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:51:45.110 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:51:45.110 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:51:45.110 | INFO | High Water Breakdown: | 20260508 06:51:45.110 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:51:45.110 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:51:45.110 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:51:45.110 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:51:45.110 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:51:45.110 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:51:45.110 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:51:45.110 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:51:45.110 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:51:45.110 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:51:45.110 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:51:45.110 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:51:45.110 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:51:45.110 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:51:45.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:51:45.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:51:45.110 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:51:45.110 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:51:45.110 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:51:45.110 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:51:45.110 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:51:45.110 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:51:45.110 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:51:45.110 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:51:45.110 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:51:45.110 | INFO | ------------------------------------------ | 20260508 06:51:45.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:51:45.110 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:51:45.111 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:51:45.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:51:45.111 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.37e+01 / 7.01e-01 | 20260508 06:51:45.111 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.05e-01, 1.09e-02 / 3.64e-03 | 20260508 06:51:45.111 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.43e+00, 5.90e-01 / 2.27e-01 | 20260508 06:51:45.111 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.17e-01 | 20260508 06:51:45.111 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.55e+01, 3.74e+00 / 1.35e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.86e+00 / 1.44e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.64e+00, 4.15e-01 / 1.55e-01 | 20260508 06:51:45.111 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.24e+00, 2.51e-01 / 9.35e-02 | 20260508 06:51:45.111 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.48e-01, 4.97e-02 / 1.73e-02 | 20260508 06:51:45.111 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:51:45.111 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:51:45.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:51:45.111 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 24.41%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 24.21%, Download -- 2.57e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 2.43%, LockUpload -- 2.58e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 2.42%, LockDownload -- 2.57e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 06:51:45.111 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260508 06:51:45.207 | INFO | Starting a round of Baz IO | 20260508 06:51:48.639 | INFO | Populating RTMetrics report | 20260508 06:51:48.640 | INFO | RTMetrics report populated | 20260508 06:51:48.640 | INFO | Opening tmp RTMetrics json file | 20260508 06:51:48.651 | INFO | Serializing RTMetrics to json | 20260508 06:51:48.652 | INFO | Serialization done, swapping tmp file now... | 20260508 06:51:48.653 | INFO | RTMetrics done | 20260508 06:51:48.708 | INFO | Finished a round of Baz IO: Wrote 1.541 GiB in 3.501 seconds | 20260508 06:51:48.709 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.64% / 0.00% / 59.36% | 20260508 06:51:48.709 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374409% / 3.97364e-06% / 4.76837e-05% | 20260508 06:51:48.709 | INFO | Baz IO overall: | 20260508 06:51:48.709 | INFO | BytesWritten : 2697223674019 | 20260508 06:51:48.709 | INFO | NumEvents : 1300065344782 | 20260508 06:51:50.250 | INFO | Analyzing chunk frames = [6094848,6095360) | 20260508 06:51:50.250 | INFO | MemInfo: { | 20260508 06:51:50.250 | INFO | "gpu" : " 17 GB", | 20260508 06:51:50.250 | INFO | "huge" : "", | 20260508 06:51:50.250 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:50.250 | INFO | "malloc" : " 14 GB", | 20260508 06:51:50.250 | INFO | "pinned" : " 29 GB" | 20260508 06:51:50.250 | INFO | } | 20260508 06:51:50.250 | INFO | Perf report for the last 5.14086 seconds: | 20260508 06:51:50.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:50.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:50.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:50.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:50.250 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 06:51:50.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:50.250 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260508 06:51:50.250 | INFO | PRE_HQ : 66.6%, 1.7, 59.3ms, 0 | 20260508 06:51:50.250 | INFO | BAZWRITER : 67.3%, 2.7, 92.9ms, 0 | 20260508 06:51:50.250 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 06:51:50.251 | INFO | Uploaded 36GiB to GPU in 2.5704s (14.0056GiB/s) | 20260508 06:51:50.251 | INFO | Downloaded 38GiB to GPU in 2.69437s (14.1035GiB/s) | 20260508 06:51:55.393 | INFO | Analyzing chunk frames = [6095360,6095872) | 20260508 06:51:55.393 | INFO | MemInfo: { | 20260508 06:51:55.393 | INFO | "gpu" : " 17 GB", | 20260508 06:51:55.393 | INFO | "huge" : "", | 20260508 06:51:55.393 | INFO | "huge_shared" : " 29 GB", | 20260508 06:51:55.393 | INFO | "malloc" : " 14 GB", | 20260508 06:51:55.393 | INFO | "pinned" : " 29 GB" | 20260508 06:51:55.393 | INFO | } | 20260508 06:51:55.393 | INFO | Perf report for the last 5.14281 seconds: | 20260508 06:51:55.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:51:55.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.393 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:51:55.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.393 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:51:55.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:51:55.394 | INFO | Uploaded 36GiB to GPU in 2.48708s (14.4748GiB/s) | 20260508 06:51:55.394 | INFO | Downloaded 34GiB to GPU in 2.43855s (13.9427GiB/s) | 20260508 06:52:00.524 | INFO | Analyzing chunk frames = [6095872,6096384) | 20260508 06:52:00.524 | INFO | MemInfo: { | 20260508 06:52:00.524 | INFO | "gpu" : " 17 GB", | 20260508 06:52:00.524 | INFO | "huge" : "", | 20260508 06:52:00.524 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:00.524 | INFO | "malloc" : " 14 GB", | 20260508 06:52:00.524 | INFO | "pinned" : " 29 GB" | 20260508 06:52:00.524 | INFO | } | 20260508 06:52:00.524 | INFO | Perf report for the last 5.13108 seconds: | 20260508 06:52:00.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:00.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | ANALYSIS : 67.3%, 3.0, 104.3ms, 0 | 20260508 06:52:00.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:52:00.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:00.524 | INFO | Uploaded 36GiB to GPU in 2.56866s (14.0151GiB/s) | 20260508 06:52:00.524 | INFO | Downloaded 34GiB to GPU in 2.43766s (13.9478GiB/s) | 20260508 06:52:05.666 | INFO | Analyzing chunk frames = [6096384,6096896) | 20260508 06:52:05.666 | INFO | MemInfo: { | 20260508 06:52:05.666 | INFO | "gpu" : " 17 GB", | 20260508 06:52:05.666 | INFO | "huge" : "", | 20260508 06:52:05.666 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:05.667 | INFO | "malloc" : " 14 GB", | 20260508 06:52:05.667 | INFO | "pinned" : " 29 GB" | 20260508 06:52:05.667 | INFO | } | 20260508 06:52:05.667 | INFO | Perf report for the last 5.14249 seconds: | 20260508 06:52:05.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:05.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 06:52:05.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:52:05.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:05.667 | INFO | Uploaded 36GiB to GPU in 2.54461s (14.1475GiB/s) | 20260508 06:52:05.667 | INFO | Downloaded 34GiB to GPU in 2.4067s (14.1272GiB/s) | 20260508 06:52:10.812 | INFO | Analyzing chunk frames = [6096896,6097408) | 20260508 06:52:10.812 | INFO | MemInfo: { | 20260508 06:52:10.812 | INFO | "gpu" : " 17 GB", | 20260508 06:52:10.812 | INFO | "huge" : "", | 20260508 06:52:10.812 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:10.812 | INFO | "malloc" : " 14 GB", | 20260508 06:52:10.812 | INFO | "pinned" : " 29 GB" | 20260508 06:52:10.812 | INFO | } | 20260508 06:52:10.812 | INFO | Perf report for the last 5.1451 seconds: | 20260508 06:52:10.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:10.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:52:10.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 06:52:10.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:10.812 | INFO | Uploaded 36GiB to GPU in 2.39186s (15.0511GiB/s) | 20260508 06:52:10.812 | INFO | Downloaded 34GiB to GPU in 2.51334s (13.5278GiB/s) | 20260508 06:52:15.949 | INFO | Analyzing chunk frames = [6097408,6097920) | 20260508 06:52:15.949 | INFO | MemInfo: { | 20260508 06:52:15.949 | INFO | "gpu" : " 17 GB", | 20260508 06:52:15.949 | INFO | "huge" : "", | 20260508 06:52:15.949 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:15.949 | INFO | "malloc" : " 14 GB", | 20260508 06:52:15.949 | INFO | "pinned" : " 29 GB" | 20260508 06:52:15.949 | INFO | } | 20260508 06:52:15.949 | INFO | Perf report for the last 5.13744 seconds: | 20260508 06:52:15.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:15.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.949 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 06:52:15.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.949 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 06:52:15.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:15.950 | INFO | Uploaded 36GiB to GPU in 2.38558s (15.0907GiB/s) | 20260508 06:52:15.950 | INFO | Downloaded 34GiB to GPU in 2.43878s (13.9414GiB/s) | 20260508 06:52:21.085 | INFO | Analyzing chunk frames = [6097920,6098432) | 20260508 06:52:21.085 | INFO | MemInfo: { | 20260508 06:52:21.085 | INFO | "gpu" : " 17 GB", | 20260508 06:52:21.085 | INFO | "huge" : "", | 20260508 06:52:21.085 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:21.085 | INFO | "malloc" : " 14 GB", | 20260508 06:52:21.085 | INFO | "pinned" : " 29 GB" | 20260508 06:52:21.085 | INFO | } | 20260508 06:52:21.086 | INFO | Perf report for the last 5.1363 seconds: | 20260508 06:52:21.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:21.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 06:52:21.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 06:52:21.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:21.086 | INFO | Uploaded 36GiB to GPU in 2.46746s (14.5899GiB/s) | 20260508 06:52:21.086 | INFO | Downloaded 34GiB to GPU in 2.44125s (13.9273GiB/s) | 20260508 06:52:26.225 | INFO | Analyzing chunk frames = [6098432,6098944) | 20260508 06:52:26.225 | INFO | MemInfo: { | 20260508 06:52:26.225 | INFO | "gpu" : " 17 GB", | 20260508 06:52:26.225 | INFO | "huge" : "", | 20260508 06:52:26.225 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:26.225 | INFO | "malloc" : " 14 GB", | 20260508 06:52:26.225 | INFO | "pinned" : " 29 GB" | 20260508 06:52:26.225 | INFO | } | 20260508 06:52:26.225 | INFO | Perf report for the last 5.13979 seconds: | 20260508 06:52:26.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:26.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.225 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 06:52:26.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.226 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260508 06:52:26.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:26.226 | INFO | Uploaded 36GiB to GPU in 2.40448s (14.9721GiB/s) | 20260508 06:52:26.226 | INFO | Downloaded 34GiB to GPU in 2.4249s (14.0212GiB/s) | 20260508 06:52:26.356 | INFO | Starting a round of Baz IO | 20260508 06:52:29.756 | INFO | Populating RTMetrics report | 20260508 06:52:29.756 | INFO | RTMetrics report populated | 20260508 06:52:29.756 | INFO | Opening tmp RTMetrics json file | 20260508 06:52:29.756 | INFO | Serializing RTMetrics to json | 20260508 06:52:29.758 | INFO | Serialization done, swapping tmp file now... | 20260508 06:52:29.758 | INFO | RTMetrics done | 20260508 06:52:29.824 | INFO | Finished a round of Baz IO: Wrote 1.541 GiB in 3.467 seconds | 20260508 06:52:29.824 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.64% / 0.00% / 59.36% | 20260508 06:52:29.824 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374953% / 0% / 4.37101e-05% | 20260508 06:52:29.824 | INFO | Baz IO overall: | 20260508 06:52:29.824 | INFO | BytesWritten : 2698878681174 | 20260508 06:52:29.824 | INFO | NumEvents : 1300852645728 | 20260508 06:52:31.363 | INFO | Analyzing chunk frames = [6098944,6099456) | 20260508 06:52:31.363 | INFO | MemInfo: { | 20260508 06:52:31.363 | INFO | "gpu" : " 17 GB", | 20260508 06:52:31.363 | INFO | "huge" : "", | 20260508 06:52:31.363 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:31.363 | INFO | "malloc" : " 14 GB", | 20260508 06:52:31.363 | INFO | "pinned" : " 29 GB" | 20260508 06:52:31.363 | INFO | } | 20260508 06:52:31.363 | INFO | Perf report for the last 5.13792 seconds: | 20260508 06:52:31.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:31.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:31.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:31.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:31.363 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 06:52:31.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:31.363 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 06:52:31.363 | INFO | PRE_HQ : 66.7%, 1.8, 61.4ms, 0 | 20260508 06:52:31.363 | INFO | BAZWRITER : 67.2%, 2.7, 94.7ms, 0 | 20260508 06:52:31.363 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 06:52:31.364 | INFO | Uploaded 36GiB to GPU in 2.56359s (14.0428GiB/s) | 20260508 06:52:31.364 | INFO | Downloaded 38GiB to GPU in 2.71011s (14.0216GiB/s) | 20260508 06:52:36.500 | INFO | Analyzing chunk frames = [6099456,6099968) | 20260508 06:52:36.500 | INFO | MemInfo: { | 20260508 06:52:36.500 | INFO | "gpu" : " 17 GB", | 20260508 06:52:36.500 | INFO | "huge" : "", | 20260508 06:52:36.500 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:36.500 | INFO | "malloc" : " 14 GB", | 20260508 06:52:36.500 | INFO | "pinned" : " 29 GB" | 20260508 06:52:36.500 | INFO | } | 20260508 06:52:36.500 | INFO | Perf report for the last 5.13699 seconds: | 20260508 06:52:36.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:36.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.500 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:52:36.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.500 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:52:36.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:36.501 | INFO | Uploaded 36GiB to GPU in 2.49516s (14.4279GiB/s) | 20260508 06:52:36.501 | INFO | Downloaded 34GiB to GPU in 2.43294s (13.9749GiB/s) | 20260508 06:52:41.642 | INFO | Analyzing chunk frames = [6099968,6100480) | 20260508 06:52:41.642 | INFO | MemInfo: { | 20260508 06:52:41.642 | INFO | "gpu" : " 17 GB", | 20260508 06:52:41.642 | INFO | "huge" : "", | 20260508 06:52:41.642 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:41.642 | INFO | "malloc" : " 14 GB", | 20260508 06:52:41.642 | INFO | "pinned" : " 29 GB" | 20260508 06:52:41.642 | INFO | } | 20260508 06:52:41.642 | INFO | Perf report for the last 5.1416 seconds: | 20260508 06:52:41.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:41.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 06:52:41.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:52:41.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:41.642 | INFO | Uploaded 36GiB to GPU in 2.53788s (14.1851GiB/s) | 20260508 06:52:41.643 | INFO | Downloaded 34GiB to GPU in 2.41704s (14.0668GiB/s) | 20260508 06:52:46.775 | INFO | Analyzing chunk frames = [6100480,6100992) | 20260508 06:52:46.775 | INFO | MemInfo: { | 20260508 06:52:46.775 | INFO | "gpu" : " 17 GB", | 20260508 06:52:46.775 | INFO | "huge" : "", | 20260508 06:52:46.775 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:46.775 | INFO | "malloc" : " 14 GB", | 20260508 06:52:46.775 | INFO | "pinned" : " 29 GB" | 20260508 06:52:46.775 | INFO | } | 20260508 06:52:46.775 | INFO | Perf report for the last 5.13342 seconds: | 20260508 06:52:46.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:46.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.775 | INFO | ANALYSIS : 66.8%, 2.9, 103.2ms, 0 | 20260508 06:52:46.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.775 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:52:46.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:46.776 | INFO | Uploaded 36GiB to GPU in 2.54907s (14.1228GiB/s) | 20260508 06:52:46.776 | INFO | Downloaded 34GiB to GPU in 2.40643s (14.1288GiB/s) | 20260508 06:52:51.921 | INFO | Analyzing chunk frames = [6100992,6101504) | 20260508 06:52:51.921 | INFO | MemInfo: { | 20260508 06:52:51.921 | INFO | "gpu" : " 17 GB", | 20260508 06:52:51.921 | INFO | "huge" : "", | 20260508 06:52:51.921 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:51.921 | INFO | "malloc" : " 14 GB", | 20260508 06:52:51.921 | INFO | "pinned" : " 29 GB" | 20260508 06:52:51.921 | INFO | } | 20260508 06:52:51.921 | INFO | Perf report for the last 5.14553 seconds: | 20260508 06:52:51.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:51.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 06:52:51.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 06:52:51.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:51.921 | INFO | Uploaded 36GiB to GPU in 2.43196s (14.8029GiB/s) | 20260508 06:52:51.922 | INFO | Downloaded 34GiB to GPU in 2.56439s (13.2585GiB/s) | 20260508 06:52:57.063 | INFO | Analyzing chunk frames = [6101504,6102016) | 20260508 06:52:57.063 | INFO | MemInfo: { | 20260508 06:52:57.063 | INFO | "gpu" : " 17 GB", | 20260508 06:52:57.063 | INFO | "huge" : "", | 20260508 06:52:57.063 | INFO | "huge_shared" : " 29 GB", | 20260508 06:52:57.063 | INFO | "malloc" : " 14 GB", | 20260508 06:52:57.063 | INFO | "pinned" : " 29 GB" | 20260508 06:52:57.063 | INFO | } | 20260508 06:52:57.063 | INFO | Perf report for the last 5.14202 seconds: | 20260508 06:52:57.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:52:57.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:52:57.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:52:57.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:52:57.063 | INFO | Uploaded 36GiB to GPU in 2.45089s (14.6885GiB/s) | 20260508 06:52:57.063 | INFO | Downloaded 34GiB to GPU in 2.42487s (14.0214GiB/s) | 20260508 06:53:02.203 | INFO | Analyzing chunk frames = [6102016,6102528) | 20260508 06:53:02.203 | INFO | MemInfo: { | 20260508 06:53:02.203 | INFO | "gpu" : " 17 GB", | 20260508 06:53:02.203 | INFO | "huge" : "", | 20260508 06:53:02.203 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:02.203 | INFO | "malloc" : " 14 GB", | 20260508 06:53:02.203 | INFO | "pinned" : " 29 GB" | 20260508 06:53:02.203 | INFO | } | 20260508 06:53:02.203 | INFO | Perf report for the last 5.1401 seconds: | 20260508 06:53:02.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:02.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 06:53:02.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 06:53:02.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:02.203 | INFO | Uploaded 36GiB to GPU in 2.49915s (14.4049GiB/s) | 20260508 06:53:02.203 | INFO | Downloaded 34GiB to GPU in 2.43455s (13.9656GiB/s) | 20260508 06:53:07.334 | INFO | Analyzing chunk frames = [6102528,6103040) | 20260508 06:53:07.334 | INFO | MemInfo: { | 20260508 06:53:07.334 | INFO | "gpu" : " 17 GB", | 20260508 06:53:07.334 | INFO | "huge" : "", | 20260508 06:53:07.334 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:07.334 | INFO | "malloc" : " 14 GB", | 20260508 06:53:07.334 | INFO | "pinned" : " 29 GB" | 20260508 06:53:07.334 | INFO | } | 20260508 06:53:07.334 | INFO | Perf report for the last 5.13117 seconds: | 20260508 06:53:07.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:07.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.334 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 06:53:07.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.335 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 06:53:07.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:07.335 | INFO | Uploaded 36GiB to GPU in 2.45949s (14.6372GiB/s) | 20260508 06:53:07.335 | INFO | Downloaded 34GiB to GPU in 2.43174s (13.9818GiB/s) | 20260508 06:53:07.335 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:53:07.335 | INFO | Total mem usage: 16.7 GiB | 20260508 06:53:07.335 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:53:07.335 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:53:07.335 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:53:07.335 | INFO | High Water Breakdown: | 20260508 06:53:07.335 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:53:07.335 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:53:07.335 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:53:07.335 | INFO | ------------------------------------------ | 20260508 06:53:07.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:53:07.335 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:53:07.335 | INFO | Total mem usage: 26.5 GiB | 20260508 06:53:07.335 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:53:07.335 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:53:07.335 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:53:07.335 | INFO | High Water Breakdown: | 20260508 06:53:07.335 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:53:07.335 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:53:07.335 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:53:07.335 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:53:07.335 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:53:07.335 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:53:07.335 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:53:07.335 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:53:07.335 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:53:07.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:53:07.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:53:07.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:53:07.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:53:07.336 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:53:07.336 | INFO | ------------------------------------------ | 20260508 06:53:07.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:53:07.336 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:53:07.336 | INFO | Total mem usage: 27.8 GiB | 20260508 06:53:07.336 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:53:07.336 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:53:07.336 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:53:07.336 | INFO | High Water Breakdown: | 20260508 06:53:07.336 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:53:07.336 | INFO | ------------------------------------------ | 20260508 06:53:07.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:53:07.336 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:53:07.336 | INFO | Total mem usage: 15.7 GiB | 20260508 06:53:07.336 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:53:07.336 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:53:07.336 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:53:07.336 | INFO | High Water Breakdown: | 20260508 06:53:07.336 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:53:07.336 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:53:07.336 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:53:07.336 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:53:07.336 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:53:07.336 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:53:07.336 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:53:07.336 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:53:07.336 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:53:07.336 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:53:07.336 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:53:07.336 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:53:07.336 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:53:07.336 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:53:07.336 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:53:07.336 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:53:07.336 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:53:07.336 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:53:07.336 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:53:07.336 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:53:07.336 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:53:07.336 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:53:07.336 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:53:07.336 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:53:07.336 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:53:07.336 | INFO | ------------------------------------------ | 20260508 06:53:07.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:53:07.336 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:53:07.336 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:53:07.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:53:07.336 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.39e+01 / 1.25e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.18e-02 / 8.57e-03 | 20260508 06:53:07.336 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.47e+00, 5.90e-01 / 2.41e-01 | 20260508 06:53:07.336 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.32e-01 | 20260508 06:53:07.336 | INFO | PerfReport: 15.86%, DME -- 4.16e+01 / 4.50e+01, 3.82e+00 / 1.49e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.89e+00 / 1.38e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.36e+00 / 2.59e+00, 4.11e-01 / 2.46e-01 | 20260508 06:53:07.336 | INFO | PerfReport: 3.14%, Metrics -- 1.03e+00 / 1.34e+00, 2.96e-01 / 6.64e-01 | 20260508 06:53:07.336 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.14e-01 / 3.52e-01, 5.04e-02 / 2.84e-02 | 20260508 06:53:07.336 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:53:07.336 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:53:07.336 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:53:07.336 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 24.45%, Upload -- 2.59e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 24.33%, Download -- 2.58e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 30.93%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 2.22%, LockUpload -- 2.35e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 2.58%, LockDownload -- 2.74e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 06:53:07.336 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 06:53:07.456 | INFO | Starting a round of Baz IO | 20260508 06:53:10.869 | INFO | Populating RTMetrics report | 20260508 06:53:10.870 | INFO | RTMetrics report populated | 20260508 06:53:10.870 | INFO | Opening tmp RTMetrics json file | 20260508 06:53:10.880 | INFO | Serializing RTMetrics to json | 20260508 06:53:10.881 | INFO | Serialization done, swapping tmp file now... | 20260508 06:53:10.882 | INFO | RTMetrics done | 20260508 06:53:10.935 | INFO | Finished a round of Baz IO: Wrote 1.54 GiB in 3.479 seconds | 20260508 06:53:10.935 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.00% / 59.36% | 20260508 06:53:10.935 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374067% / 1.58946e-05% / 3.57628e-05% | 20260508 06:53:10.935 | INFO | Baz IO overall: | 20260508 06:53:10.935 | INFO | BytesWritten : 2700532527507 | 20260508 06:53:10.935 | INFO | NumEvents : 1301638985523 | 20260508 06:53:12.475 | INFO | Analyzing chunk frames = [6103040,6103552) | 20260508 06:53:12.475 | INFO | MemInfo: { | 20260508 06:53:12.475 | INFO | "gpu" : " 17 GB", | 20260508 06:53:12.475 | INFO | "huge" : "", | 20260508 06:53:12.475 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:12.475 | INFO | "malloc" : " 14 GB", | 20260508 06:53:12.475 | INFO | "pinned" : " 29 GB" | 20260508 06:53:12.475 | INFO | } | 20260508 06:53:12.476 | INFO | Perf report for the last 5.14135 seconds: | 20260508 06:53:12.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:12.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:12.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:12.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:12.476 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 06:53:12.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:12.476 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 06:53:12.476 | INFO | PRE_HQ : 66.6%, 1.8, 61.7ms, 0 | 20260508 06:53:12.476 | INFO | BAZWRITER : 67.4%, 2.8, 96.6ms, 0 | 20260508 06:53:12.476 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:53:12.476 | INFO | Uploaded 36GiB to GPU in 2.5622s (14.0504GiB/s) | 20260508 06:53:12.476 | INFO | Downloaded 38GiB to GPU in 2.70478s (14.0492GiB/s) | 20260508 06:53:17.618 | INFO | Analyzing chunk frames = [6103552,6104064) | 20260508 06:53:17.618 | INFO | MemInfo: { | 20260508 06:53:17.618 | INFO | "gpu" : " 17 GB", | 20260508 06:53:17.618 | INFO | "huge" : "", | 20260508 06:53:17.618 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:17.618 | INFO | "malloc" : " 14 GB", | 20260508 06:53:17.618 | INFO | "pinned" : " 29 GB" | 20260508 06:53:17.618 | INFO | } | 20260508 06:53:17.618 | INFO | Perf report for the last 5.14251 seconds: | 20260508 06:53:17.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:17.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:53:17.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:53:17.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:17.618 | INFO | Uploaded 36GiB to GPU in 2.50105s (14.3939GiB/s) | 20260508 06:53:17.618 | INFO | Downloaded 34GiB to GPU in 2.43084s (13.9869GiB/s) | 20260508 06:53:22.763 | INFO | Analyzing chunk frames = [6104064,6104576) | 20260508 06:53:22.763 | INFO | MemInfo: { | 20260508 06:53:22.763 | INFO | "gpu" : " 17 GB", | 20260508 06:53:22.763 | INFO | "huge" : "", | 20260508 06:53:22.763 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:22.763 | INFO | "malloc" : " 14 GB", | 20260508 06:53:22.764 | INFO | "pinned" : " 29 GB" | 20260508 06:53:22.764 | INFO | } | 20260508 06:53:22.764 | INFO | Perf report for the last 5.14552 seconds: | 20260508 06:53:22.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:22.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:53:22.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:53:22.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:22.764 | INFO | Uploaded 36GiB to GPU in 2.56597s (14.0298GiB/s) | 20260508 06:53:22.764 | INFO | Downloaded 34GiB to GPU in 2.41255s (14.093GiB/s) | 20260508 06:53:27.900 | INFO | Analyzing chunk frames = [6104576,6105088) | 20260508 06:53:27.900 | INFO | MemInfo: { | 20260508 06:53:27.900 | INFO | "gpu" : " 17 GB", | 20260508 06:53:27.900 | INFO | "huge" : "", | 20260508 06:53:27.900 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:27.900 | INFO | "malloc" : " 14 GB", | 20260508 06:53:27.900 | INFO | "pinned" : " 29 GB" | 20260508 06:53:27.900 | INFO | } | 20260508 06:53:27.900 | INFO | Perf report for the last 5.13646 seconds: | 20260508 06:53:27.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:27.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | ANALYSIS : 67.0%, 3.0, 103.5ms, 0 | 20260508 06:53:27.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:53:27.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:27.900 | INFO | Uploaded 36GiB to GPU in 2.58942s (13.9027GiB/s) | 20260508 06:53:27.901 | INFO | Downloaded 34GiB to GPU in 2.41682s (14.0681GiB/s) | 20260508 06:53:33.033 | INFO | Analyzing chunk frames = [6105088,6105600) | 20260508 06:53:33.033 | INFO | MemInfo: { | 20260508 06:53:33.033 | INFO | "gpu" : " 17 GB", | 20260508 06:53:33.033 | INFO | "huge" : "", | 20260508 06:53:33.033 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:33.033 | INFO | "malloc" : " 14 GB", | 20260508 06:53:33.033 | INFO | "pinned" : " 29 GB" | 20260508 06:53:33.033 | INFO | } | 20260508 06:53:33.033 | INFO | Perf report for the last 5.13306 seconds: | 20260508 06:53:33.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:33.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 06:53:33.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:53:33.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:33.033 | INFO | Uploaded 36GiB to GPU in 2.44307s (14.7356GiB/s) | 20260508 06:53:33.034 | INFO | Downloaded 34GiB to GPU in 2.52579s (13.4611GiB/s) | 20260508 06:53:38.172 | INFO | Analyzing chunk frames = [6105600,6106112) | 20260508 06:53:38.172 | INFO | MemInfo: { | 20260508 06:53:38.172 | INFO | "gpu" : " 17 GB", | 20260508 06:53:38.172 | INFO | "huge" : "", | 20260508 06:53:38.172 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:38.172 | INFO | "malloc" : " 14 GB", | 20260508 06:53:38.172 | INFO | "pinned" : " 29 GB" | 20260508 06:53:38.172 | INFO | } | 20260508 06:53:38.172 | INFO | Perf report for the last 5.13918 seconds: | 20260508 06:53:38.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:38.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.172 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 06:53:38.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.172 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:53:38.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:38.173 | INFO | Uploaded 36GiB to GPU in 2.4452s (14.7227GiB/s) | 20260508 06:53:38.173 | INFO | Downloaded 34GiB to GPU in 2.4235s (14.0293GiB/s) | 20260508 06:53:43.313 | INFO | Analyzing chunk frames = [6106112,6106624) | 20260508 06:53:43.313 | INFO | MemInfo: { | 20260508 06:53:43.313 | INFO | "gpu" : " 17 GB", | 20260508 06:53:43.313 | INFO | "huge" : "", | 20260508 06:53:43.313 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:43.313 | INFO | "malloc" : " 14 GB", | 20260508 06:53:43.313 | INFO | "pinned" : " 29 GB" | 20260508 06:53:43.313 | INFO | } | 20260508 06:53:43.313 | INFO | Perf report for the last 5.14066 seconds: | 20260508 06:53:43.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:43.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 06:53:43.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:53:43.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:43.313 | INFO | Uploaded 36GiB to GPU in 2.45319s (14.6747GiB/s) | 20260508 06:53:43.314 | INFO | Downloaded 34GiB to GPU in 2.43653s (13.9543GiB/s) | 20260508 06:53:48.451 | INFO | Analyzing chunk frames = [6106624,6107136) | 20260508 06:53:48.451 | INFO | MemInfo: { | 20260508 06:53:48.451 | INFO | "gpu" : " 17 GB", | 20260508 06:53:48.451 | INFO | "huge" : "", | 20260508 06:53:48.451 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:48.451 | INFO | "malloc" : " 14 GB", | 20260508 06:53:48.451 | INFO | "pinned" : " 29 GB" | 20260508 06:53:48.451 | INFO | } | 20260508 06:53:48.451 | INFO | Perf report for the last 5.13799 seconds: | 20260508 06:53:48.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:48.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.451 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 06:53:48.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.451 | INFO | PRE_HQ : 34.8%, 1.0, 18.0ms, 0 | 20260508 06:53:48.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:48.452 | INFO | Uploaded 36GiB to GPU in 2.48628s (14.4795GiB/s) | 20260508 06:53:48.452 | INFO | Downloaded 34GiB to GPU in 2.45503s (13.8491GiB/s) | 20260508 06:53:48.568 | INFO | Starting a round of Baz IO | 20260508 06:53:51.989 | INFO | Populating RTMetrics report | 20260508 06:53:51.989 | INFO | RTMetrics report populated | 20260508 06:53:51.989 | INFO | Opening tmp RTMetrics json file | 20260508 06:53:51.999 | INFO | Serializing RTMetrics to json | 20260508 06:53:52.002 | INFO | Serialization done, swapping tmp file now... | 20260508 06:53:52.002 | INFO | RTMetrics done | 20260508 06:53:52.055 | INFO | Finished a round of Baz IO: Wrote 1.539 GiB in 3.487 seconds | 20260508 06:53:52.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.00% / 59.37% | 20260508 06:53:52.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.372454% / 0% / 5.5631e-05% | 20260508 06:53:52.056 | INFO | Baz IO overall: | 20260508 06:53:52.056 | INFO | BytesWritten : 2702184635185 | 20260508 06:53:52.056 | INFO | NumEvents : 1302424336617 | 20260508 06:53:53.589 | INFO | Analyzing chunk frames = [6107136,6107648) | 20260508 06:53:53.589 | INFO | MemInfo: { | 20260508 06:53:53.589 | INFO | "gpu" : " 17 GB", | 20260508 06:53:53.589 | INFO | "huge" : "", | 20260508 06:53:53.589 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:53.589 | INFO | "malloc" : " 14 GB", | 20260508 06:53:53.589 | INFO | "pinned" : " 29 GB" | 20260508 06:53:53.589 | INFO | } | 20260508 06:53:53.589 | INFO | Perf report for the last 5.13833 seconds: | 20260508 06:53:53.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:53.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:53.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:53.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:53.589 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 06:53:53.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:53.590 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260508 06:53:53.590 | INFO | PRE_HQ : 66.6%, 1.8, 61.8ms, 0 | 20260508 06:53:53.590 | INFO | BAZWRITER : 67.3%, 2.7, 94.8ms, 0 | 20260508 06:53:53.590 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:53:53.590 | INFO | Uploaded 36GiB to GPU in 2.52722s (14.2449GiB/s) | 20260508 06:53:53.590 | INFO | Downloaded 38GiB to GPU in 2.69815s (14.0837GiB/s) | 20260508 06:53:58.733 | INFO | Analyzing chunk frames = [6107648,6108160) | 20260508 06:53:58.733 | INFO | MemInfo: { | 20260508 06:53:58.733 | INFO | "gpu" : " 17 GB", | 20260508 06:53:58.733 | INFO | "huge" : "", | 20260508 06:53:58.733 | INFO | "huge_shared" : " 29 GB", | 20260508 06:53:58.733 | INFO | "malloc" : " 14 GB", | 20260508 06:53:58.734 | INFO | "pinned" : " 29 GB" | 20260508 06:53:58.734 | INFO | } | 20260508 06:53:58.734 | INFO | Perf report for the last 5.1442 seconds: | 20260508 06:53:58.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:53:58.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 06:53:58.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:53:58.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:53:58.734 | INFO | Uploaded 36GiB to GPU in 2.46411s (14.6097GiB/s) | 20260508 06:53:58.734 | INFO | Downloaded 34GiB to GPU in 2.44872s (13.8848GiB/s) | 20260508 06:54:03.877 | INFO | Analyzing chunk frames = [6108160,6108672) | 20260508 06:54:03.877 | INFO | MemInfo: { | 20260508 06:54:03.877 | INFO | "gpu" : " 17 GB", | 20260508 06:54:03.877 | INFO | "huge" : "", | 20260508 06:54:03.877 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:03.877 | INFO | "malloc" : " 14 GB", | 20260508 06:54:03.877 | INFO | "pinned" : " 29 GB" | 20260508 06:54:03.877 | INFO | } | 20260508 06:54:03.877 | INFO | Perf report for the last 5.14325 seconds: | 20260508 06:54:03.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:03.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:54:03.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:54:03.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:03.877 | INFO | Uploaded 36GiB to GPU in 2.53889s (14.1794GiB/s) | 20260508 06:54:03.877 | INFO | Downloaded 34GiB to GPU in 2.45918s (13.8258GiB/s) | 20260508 06:54:09.014 | INFO | Analyzing chunk frames = [6108672,6109184) | 20260508 06:54:09.014 | INFO | MemInfo: { | 20260508 06:54:09.014 | INFO | "gpu" : " 17 GB", | 20260508 06:54:09.014 | INFO | "huge" : "", | 20260508 06:54:09.014 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:09.014 | INFO | "malloc" : " 14 GB", | 20260508 06:54:09.014 | INFO | "pinned" : " 29 GB" | 20260508 06:54:09.014 | INFO | } | 20260508 06:54:09.014 | INFO | Perf report for the last 5.13734 seconds: | 20260508 06:54:09.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:09.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.014 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 06:54:09.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.014 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:54:09.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:09.015 | INFO | Uploaded 36GiB to GPU in 2.54409s (14.1505GiB/s) | 20260508 06:54:09.015 | INFO | Downloaded 34GiB to GPU in 2.44651s (13.8974GiB/s) | 20260508 06:54:14.149 | INFO | Analyzing chunk frames = [6109184,6109696) | 20260508 06:54:14.149 | INFO | MemInfo: { | 20260508 06:54:14.149 | INFO | "gpu" : " 17 GB", | 20260508 06:54:14.149 | INFO | "huge" : "", | 20260508 06:54:14.149 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:14.149 | INFO | "malloc" : " 14 GB", | 20260508 06:54:14.149 | INFO | "pinned" : " 29 GB" | 20260508 06:54:14.149 | INFO | } | 20260508 06:54:14.149 | INFO | Perf report for the last 5.135 seconds: | 20260508 06:54:14.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:14.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.149 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 06:54:14.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.149 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:54:14.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:14.150 | INFO | Uploaded 36GiB to GPU in 2.28194s (15.776GiB/s) | 20260508 06:54:14.150 | INFO | Downloaded 34GiB to GPU in 2.51201s (13.535GiB/s) | 20260508 06:54:19.287 | INFO | Analyzing chunk frames = [6109696,6110208) | 20260508 06:54:19.287 | INFO | MemInfo: { | 20260508 06:54:19.287 | INFO | "gpu" : " 17 GB", | 20260508 06:54:19.287 | INFO | "huge" : "", | 20260508 06:54:19.287 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:19.287 | INFO | "malloc" : " 14 GB", | 20260508 06:54:19.287 | INFO | "pinned" : " 29 GB" | 20260508 06:54:19.287 | INFO | } | 20260508 06:54:19.287 | INFO | Perf report for the last 5.13772 seconds: | 20260508 06:54:19.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:19.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 06:54:19.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:54:19.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:19.287 | INFO | Uploaded 36GiB to GPU in 2.34905s (15.3254GiB/s) | 20260508 06:54:19.287 | INFO | Downloaded 34GiB to GPU in 2.4324s (13.978GiB/s) | 20260508 06:54:24.430 | INFO | Analyzing chunk frames = [6110208,6110720) | 20260508 06:54:24.430 | INFO | MemInfo: { | 20260508 06:54:24.430 | INFO | "gpu" : " 17 GB", | 20260508 06:54:24.430 | INFO | "huge" : "", | 20260508 06:54:24.430 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:24.430 | INFO | "malloc" : " 14 GB", | 20260508 06:54:24.430 | INFO | "pinned" : " 29 GB" | 20260508 06:54:24.430 | INFO | } | 20260508 06:54:24.430 | INFO | Perf report for the last 5.14297 seconds: | 20260508 06:54:24.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:24.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.430 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 06:54:24.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.430 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 06:54:24.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:24.431 | INFO | Uploaded 36GiB to GPU in 2.48589s (14.4817GiB/s) | 20260508 06:54:24.431 | INFO | Downloaded 34GiB to GPU in 2.45139s (13.8697GiB/s) | 20260508 06:54:29.574 | INFO | Analyzing chunk frames = [6110720,6111232) | 20260508 06:54:29.574 | INFO | MemInfo: { | 20260508 06:54:29.574 | INFO | "gpu" : " 17 GB", | 20260508 06:54:29.574 | INFO | "huge" : "", | 20260508 06:54:29.574 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:29.574 | INFO | "malloc" : " 14 GB", | 20260508 06:54:29.574 | INFO | "pinned" : " 29 GB" | 20260508 06:54:29.574 | INFO | } | 20260508 06:54:29.574 | INFO | Perf report for the last 5.14409 seconds: | 20260508 06:54:29.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:29.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.574 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 06:54:29.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.574 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 06:54:29.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:29.575 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:54:29.575 | INFO | Total mem usage: 16.7 GiB | 20260508 06:54:29.575 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:54:29.575 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:54:29.575 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:54:29.575 | INFO | High Water Breakdown: | 20260508 06:54:29.575 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:54:29.575 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:54:29.575 | INFO | Uploaded 36GiB to GPU in 2.45504s (14.6637GiB/s) | 20260508 06:54:29.575 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:54:29.575 | INFO | Downloaded 34GiB to GPU in 2.46525s (13.7917GiB/s) | 20260508 06:54:29.575 | INFO | ------------------------------------------ | 20260508 06:54:29.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:54:29.575 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:54:29.575 | INFO | Total mem usage: 26.5 GiB | 20260508 06:54:29.575 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:54:29.575 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:54:29.575 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:54:29.575 | INFO | High Water Breakdown: | 20260508 06:54:29.575 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:54:29.575 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:54:29.575 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:54:29.575 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:54:29.575 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:54:29.575 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:54:29.575 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:54:29.575 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:54:29.575 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:54:29.575 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:54:29.575 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:54:29.575 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:54:29.575 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:54:29.575 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:54:29.575 | INFO | ------------------------------------------ | 20260508 06:54:29.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:54:29.575 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:54:29.575 | INFO | Total mem usage: 27.8 GiB | 20260508 06:54:29.575 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:54:29.575 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:54:29.575 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:54:29.575 | INFO | High Water Breakdown: | 20260508 06:54:29.575 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:54:29.575 | INFO | ------------------------------------------ | 20260508 06:54:29.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:54:29.575 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:54:29.575 | INFO | Total mem usage: 15.7 GiB | 20260508 06:54:29.575 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:54:29.575 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:54:29.575 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:54:29.575 | INFO | High Water Breakdown: | 20260508 06:54:29.575 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:54:29.575 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:54:29.575 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:54:29.575 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:54:29.575 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:54:29.575 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:54:29.575 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:54:29.576 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:54:29.576 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:54:29.576 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:54:29.576 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:54:29.576 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:54:29.576 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:54:29.576 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:54:29.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:54:29.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:54:29.576 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:54:29.576 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:54:29.576 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:54:29.576 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:54:29.576 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:54:29.576 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:54:29.576 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:54:29.576 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:54:29.576 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:54:29.576 | INFO | ------------------------------------------ | 20260508 06:54:29.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:54:29.576 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:54:29.576 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:54:29.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:54:29.576 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 5.96e-01 | 20260508 06:54:29.576 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.05e-01, 1.12e-02 / 7.85e-03 | 20260508 06:54:29.576 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.36e+00, 5.77e-01 / 2.63e-01 | 20260508 06:54:29.576 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.08e-01 / 1.40e-01 | 20260508 06:54:29.576 | INFO | PerfReport: 15.89%, DME -- 4.16e+01 / 4.50e+01, 3.84e+00 / 1.16e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.86e+00 / 1.41e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.56e+00, 4.11e-01 / 2.18e-01 | 20260508 06:54:29.576 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.22e+00, 2.53e-01 / 1.34e-01 | 20260508 06:54:29.576 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.49e-01, 4.98e-02 / 1.83e-02 | 20260508 06:54:29.576 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:54:29.576 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:54:29.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:54:29.576 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 24.37%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 24.39%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 2.23%, LockUpload -- 2.36e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 2.57%, LockDownload -- 2.73e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.94e+01 / 0.00e+00 | 20260508 06:54:29.576 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 06:54:29.671 | INFO | Starting a round of Baz IO | 20260508 06:54:33.093 | INFO | Populating RTMetrics report | 20260508 06:54:33.093 | INFO | RTMetrics report populated | 20260508 06:54:33.093 | INFO | Opening tmp RTMetrics json file | 20260508 06:54:33.105 | INFO | Serializing RTMetrics to json | 20260508 06:54:33.106 | INFO | Serialization done, swapping tmp file now... | 20260508 06:54:33.106 | INFO | RTMetrics done | 20260508 06:54:33.160 | INFO | Finished a round of Baz IO: Wrote 1.539 GiB in 3.488 seconds | 20260508 06:54:33.160 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.00% / 59.37% | 20260508 06:54:33.160 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.372326% / 3.97364e-06% / 4.37101e-05% | 20260508 06:54:33.160 | INFO | Baz IO overall: | 20260508 06:54:33.160 | INFO | BytesWritten : 2703837649335 | 20260508 06:54:33.160 | INFO | NumEvents : 1303210231822 | 20260508 06:54:34.709 | INFO | Analyzing chunk frames = [6111232,6111744) | 20260508 06:54:34.709 | INFO | MemInfo: { | 20260508 06:54:34.709 | INFO | "gpu" : " 17 GB", | 20260508 06:54:34.709 | INFO | "huge" : "", | 20260508 06:54:34.709 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:34.709 | INFO | "malloc" : " 14 GB", | 20260508 06:54:34.709 | INFO | "pinned" : " 29 GB" | 20260508 06:54:34.709 | INFO | } | 20260508 06:54:34.709 | INFO | Perf report for the last 5.13486 seconds: | 20260508 06:54:34.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:34.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:34.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:34.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:34.709 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 06:54:34.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:34.709 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 06:54:34.709 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 06:54:34.709 | INFO | BAZWRITER : 67.3%, 2.5, 87.2ms, 0 | 20260508 06:54:34.709 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 06:54:34.709 | INFO | Uploaded 36GiB to GPU in 2.58525s (13.9251GiB/s) | 20260508 06:54:34.709 | INFO | Downloaded 38GiB to GPU in 2.69866s (14.0811GiB/s) | 20260508 06:54:39.844 | INFO | Analyzing chunk frames = [6111744,6112256) | 20260508 06:54:39.844 | INFO | MemInfo: { | 20260508 06:54:39.844 | INFO | "gpu" : " 17 GB", | 20260508 06:54:39.844 | INFO | "huge" : "", | 20260508 06:54:39.844 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:39.844 | INFO | "malloc" : " 14 GB", | 20260508 06:54:39.844 | INFO | "pinned" : " 29 GB" | 20260508 06:54:39.844 | INFO | } | 20260508 06:54:39.845 | INFO | Perf report for the last 5.13559 seconds: | 20260508 06:54:39.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:39.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:54:39.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:54:39.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:39.845 | INFO | Uploaded 36GiB to GPU in 2.48674s (14.4768GiB/s) | 20260508 06:54:39.845 | INFO | Downloaded 34GiB to GPU in 2.4317s (13.982GiB/s) | 20260508 06:54:44.986 | INFO | Analyzing chunk frames = [6112256,6112768) | 20260508 06:54:44.986 | INFO | MemInfo: { | 20260508 06:54:44.986 | INFO | "gpu" : " 17 GB", | 20260508 06:54:44.986 | INFO | "huge" : "", | 20260508 06:54:44.986 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:44.986 | INFO | "malloc" : " 14 GB", | 20260508 06:54:44.986 | INFO | "pinned" : " 29 GB" | 20260508 06:54:44.986 | INFO | } | 20260508 06:54:44.986 | INFO | Perf report for the last 5.14187 seconds: | 20260508 06:54:44.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:44.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.987 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 06:54:44.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.987 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:54:44.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:44.987 | INFO | Uploaded 36GiB to GPU in 2.53245s (14.2155GiB/s) | 20260508 06:54:44.987 | INFO | Downloaded 34GiB to GPU in 2.4081s (14.119GiB/s) | 20260508 06:54:50.127 | INFO | Analyzing chunk frames = [6112768,6113280) | 20260508 06:54:50.127 | INFO | MemInfo: { | 20260508 06:54:50.127 | INFO | "gpu" : " 17 GB", | 20260508 06:54:50.127 | INFO | "huge" : "", | 20260508 06:54:50.128 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:50.128 | INFO | "malloc" : " 14 GB", | 20260508 06:54:50.128 | INFO | "pinned" : " 29 GB" | 20260508 06:54:50.128 | INFO | } | 20260508 06:54:50.128 | INFO | Perf report for the last 5.14135 seconds: | 20260508 06:54:50.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:50.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 06:54:50.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:54:50.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:50.128 | INFO | Uploaded 36GiB to GPU in 2.55873s (14.0695GiB/s) | 20260508 06:54:50.128 | INFO | Downloaded 34GiB to GPU in 2.418s (14.0612GiB/s) | 20260508 06:54:55.267 | INFO | Analyzing chunk frames = [6113280,6113792) | 20260508 06:54:55.267 | INFO | MemInfo: { | 20260508 06:54:55.267 | INFO | "gpu" : " 17 GB", | 20260508 06:54:55.267 | INFO | "huge" : "", | 20260508 06:54:55.267 | INFO | "huge_shared" : " 29 GB", | 20260508 06:54:55.267 | INFO | "malloc" : " 14 GB", | 20260508 06:54:55.267 | INFO | "pinned" : " 29 GB" | 20260508 06:54:55.267 | INFO | } | 20260508 06:54:55.268 | INFO | Perf report for the last 5.13974 seconds: | 20260508 06:54:55.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:54:55.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 06:54:55.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:54:55.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:54:55.268 | INFO | Uploaded 36GiB to GPU in 2.36128s (15.2459GiB/s) | 20260508 06:54:55.268 | INFO | Downloaded 34GiB to GPU in 2.5653s (13.2538GiB/s) | 20260508 06:55:00.404 | INFO | Analyzing chunk frames = [6113792,6114304) | 20260508 06:55:00.404 | INFO | MemInfo: { | 20260508 06:55:00.404 | INFO | "gpu" : " 17 GB", | 20260508 06:55:00.404 | INFO | "huge" : "", | 20260508 06:55:00.404 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:00.404 | INFO | "malloc" : " 14 GB", | 20260508 06:55:00.404 | INFO | "pinned" : " 29 GB" | 20260508 06:55:00.404 | INFO | } | 20260508 06:55:00.404 | INFO | Perf report for the last 5.13671 seconds: | 20260508 06:55:00.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:00.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.404 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 06:55:00.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.404 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:55:00.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:00.405 | INFO | Uploaded 36GiB to GPU in 2.34896s (15.3259GiB/s) | 20260508 06:55:00.405 | INFO | Downloaded 34GiB to GPU in 2.4449s (13.9065GiB/s) | 20260508 06:55:05.538 | INFO | Analyzing chunk frames = [6114304,6114816) | 20260508 06:55:05.538 | INFO | MemInfo: { | 20260508 06:55:05.538 | INFO | "gpu" : " 17 GB", | 20260508 06:55:05.538 | INFO | "huge" : "", | 20260508 06:55:05.538 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:05.539 | INFO | "malloc" : " 14 GB", | 20260508 06:55:05.539 | INFO | "pinned" : " 29 GB" | 20260508 06:55:05.539 | INFO | } | 20260508 06:55:05.539 | INFO | Perf report for the last 5.1344 seconds: | 20260508 06:55:05.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:05.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:55:05.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 06:55:05.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:05.539 | INFO | Uploaded 36GiB to GPU in 2.42809s (14.8265GiB/s) | 20260508 06:55:05.539 | INFO | Downloaded 34GiB to GPU in 2.44649s (13.8975GiB/s) | 20260508 06:55:10.683 | INFO | Analyzing chunk frames = [6114816,6115328) | 20260508 06:55:10.683 | INFO | MemInfo: { | 20260508 06:55:10.683 | INFO | "gpu" : " 17 GB", | 20260508 06:55:10.683 | INFO | "huge" : "", | 20260508 06:55:10.683 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:10.683 | INFO | "malloc" : " 14 GB", | 20260508 06:55:10.683 | INFO | "pinned" : " 29 GB" | 20260508 06:55:10.683 | INFO | } | 20260508 06:55:10.683 | INFO | Perf report for the last 5.14481 seconds: | 20260508 06:55:10.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:10.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:55:10.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 06:55:10.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:10.684 | INFO | Uploaded 36GiB to GPU in 2.45298s (14.676GiB/s) | 20260508 06:55:10.684 | INFO | Downloaded 34GiB to GPU in 2.46056s (13.818GiB/s) | 20260508 06:55:10.795 | INFO | Starting a round of Baz IO | 20260508 06:55:14.214 | INFO | Populating RTMetrics report | 20260508 06:55:14.214 | INFO | RTMetrics report populated | 20260508 06:55:14.214 | INFO | Opening tmp RTMetrics json file | 20260508 06:55:14.215 | INFO | Serializing RTMetrics to json | 20260508 06:55:14.216 | INFO | Serialization done, swapping tmp file now... | 20260508 06:55:14.217 | INFO | RTMetrics done | 20260508 06:55:14.285 | INFO | Finished a round of Baz IO: Wrote 1.539 GiB in 3.489 seconds | 20260508 06:55:14.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.00% / 59.37% | 20260508 06:55:14.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.371655% / 0% / 3.17891e-05% | 20260508 06:55:14.285 | INFO | Baz IO overall: | 20260508 06:55:14.285 | INFO | BytesWritten : 2705490430850 | 20260508 06:55:14.285 | INFO | NumEvents : 1303996099358 | 20260508 06:55:15.823 | INFO | Analyzing chunk frames = [6115328,6115840) | 20260508 06:55:15.823 | INFO | MemInfo: { | 20260508 06:55:15.823 | INFO | "gpu" : " 17 GB", | 20260508 06:55:15.823 | INFO | "huge" : "", | 20260508 06:55:15.823 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:15.823 | INFO | "malloc" : " 14 GB", | 20260508 06:55:15.823 | INFO | "pinned" : " 29 GB" | 20260508 06:55:15.823 | INFO | } | 20260508 06:55:15.823 | INFO | Perf report for the last 5.13977 seconds: | 20260508 06:55:15.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:15.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:15.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:15.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:15.823 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 06:55:15.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:15.823 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 06:55:15.823 | INFO | PRE_HQ : 66.5%, 1.8, 61.5ms, 0 | 20260508 06:55:15.823 | INFO | BAZWRITER : 67.3%, 2.7, 94.0ms, 0 | 20260508 06:55:15.823 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 06:55:15.824 | INFO | Uploaded 36GiB to GPU in 2.55224s (14.1053GiB/s) | 20260508 06:55:15.824 | INFO | Downloaded 38GiB to GPU in 2.71807s (13.9805GiB/s) | 20260508 06:55:20.963 | INFO | Analyzing chunk frames = [6115840,6116352) | 20260508 06:55:20.963 | INFO | MemInfo: { | 20260508 06:55:20.963 | INFO | "gpu" : " 17 GB", | 20260508 06:55:20.963 | INFO | "huge" : "", | 20260508 06:55:20.963 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:20.963 | INFO | "malloc" : " 14 GB", | 20260508 06:55:20.963 | INFO | "pinned" : " 29 GB" | 20260508 06:55:20.963 | INFO | } | 20260508 06:55:20.963 | INFO | Perf report for the last 5.13989 seconds: | 20260508 06:55:20.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:20.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.963 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 06:55:20.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.963 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:55:20.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:20.964 | INFO | Uploaded 36GiB to GPU in 2.4526s (14.6783GiB/s) | 20260508 06:55:20.964 | INFO | Downloaded 34GiB to GPU in 2.45194s (13.8666GiB/s) | 20260508 06:55:26.097 | INFO | Analyzing chunk frames = [6116352,6116864) | 20260508 06:55:26.097 | INFO | MemInfo: { | 20260508 06:55:26.097 | INFO | "gpu" : " 17 GB", | 20260508 06:55:26.097 | INFO | "huge" : "", | 20260508 06:55:26.097 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:26.097 | INFO | "malloc" : " 14 GB", | 20260508 06:55:26.097 | INFO | "pinned" : " 29 GB" | 20260508 06:55:26.097 | INFO | } | 20260508 06:55:26.097 | INFO | Perf report for the last 5.13369 seconds: | 20260508 06:55:26.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:26.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 06:55:26.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:55:26.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:26.097 | INFO | Uploaded 36GiB to GPU in 2.52838s (14.2384GiB/s) | 20260508 06:55:26.098 | INFO | Downloaded 34GiB to GPU in 2.45351s (13.8577GiB/s) | 20260508 06:55:31.239 | INFO | Analyzing chunk frames = [6116864,6117376) | 20260508 06:55:31.239 | INFO | MemInfo: { | 20260508 06:55:31.239 | INFO | "gpu" : " 17 GB", | 20260508 06:55:31.239 | INFO | "huge" : "", | 20260508 06:55:31.239 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:31.239 | INFO | "malloc" : " 14 GB", | 20260508 06:55:31.239 | INFO | "pinned" : " 29 GB" | 20260508 06:55:31.239 | INFO | } | 20260508 06:55:31.239 | INFO | Perf report for the last 5.14205 seconds: | 20260508 06:55:31.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:31.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | ANALYSIS : 66.5%, 3.0, 102.9ms, 0 | 20260508 06:55:31.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:55:31.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:31.239 | INFO | Uploaded 36GiB to GPU in 2.51007s (14.3422GiB/s) | 20260508 06:55:31.239 | INFO | Downloaded 34GiB to GPU in 2.42815s (14.0024GiB/s) | 20260508 06:55:36.382 | INFO | Analyzing chunk frames = [6117376,6117888) | 20260508 06:55:36.382 | INFO | MemInfo: { | 20260508 06:55:36.382 | INFO | "gpu" : " 17 GB", | 20260508 06:55:36.382 | INFO | "huge" : "", | 20260508 06:55:36.382 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:36.382 | INFO | "malloc" : " 14 GB", | 20260508 06:55:36.382 | INFO | "pinned" : " 29 GB" | 20260508 06:55:36.382 | INFO | } | 20260508 06:55:36.382 | INFO | Perf report for the last 5.14329 seconds: | 20260508 06:55:36.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:36.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.382 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:55:36.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.382 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:55:36.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:36.383 | INFO | Uploaded 36GiB to GPU in 2.28943s (15.7245GiB/s) | 20260508 06:55:36.383 | INFO | Downloaded 34GiB to GPU in 2.50205s (13.5889GiB/s) | 20260508 06:55:41.522 | INFO | Analyzing chunk frames = [6117888,6118400) | 20260508 06:55:41.522 | INFO | MemInfo: { | 20260508 06:55:41.522 | INFO | "gpu" : " 17 GB", | 20260508 06:55:41.522 | INFO | "huge" : "", | 20260508 06:55:41.522 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:41.522 | INFO | "malloc" : " 14 GB", | 20260508 06:55:41.522 | INFO | "pinned" : " 29 GB" | 20260508 06:55:41.522 | INFO | } | 20260508 06:55:41.522 | INFO | Perf report for the last 5.14014 seconds: | 20260508 06:55:41.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:41.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.522 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 06:55:41.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.523 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 06:55:41.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:41.523 | INFO | Uploaded 36GiB to GPU in 2.41844s (14.8856GiB/s) | 20260508 06:55:41.523 | INFO | Downloaded 34GiB to GPU in 2.45277s (13.8619GiB/s) | 20260508 06:55:46.658 | INFO | Analyzing chunk frames = [6118400,6118912) | 20260508 06:55:46.658 | INFO | MemInfo: { | 20260508 06:55:46.658 | INFO | "gpu" : " 17 GB", | 20260508 06:55:46.659 | INFO | "huge" : "", | 20260508 06:55:46.659 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:46.659 | INFO | "malloc" : " 14 GB", | 20260508 06:55:46.659 | INFO | "pinned" : " 29 GB" | 20260508 06:55:46.659 | INFO | } | 20260508 06:55:46.659 | INFO | Perf report for the last 5.13643 seconds: | 20260508 06:55:46.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:46.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.659 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:55:46.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.659 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 06:55:46.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:46.660 | INFO | Uploaded 36GiB to GPU in 2.43838s (14.7639GiB/s) | 20260508 06:55:46.660 | INFO | Downloaded 34GiB to GPU in 2.45155s (13.8688GiB/s) | 20260508 06:55:51.801 | INFO | Analyzing chunk frames = [6118912,6119424) | 20260508 06:55:51.801 | INFO | MemInfo: { | 20260508 06:55:51.801 | INFO | "gpu" : " 17 GB", | 20260508 06:55:51.801 | INFO | "huge" : "", | 20260508 06:55:51.801 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:51.802 | INFO | "malloc" : " 14 GB", | 20260508 06:55:51.802 | INFO | "pinned" : " 29 GB" | 20260508 06:55:51.802 | INFO | } | 20260508 06:55:51.802 | INFO | Perf report for the last 5.14282 seconds: | 20260508 06:55:51.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:51.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:55:51.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 06:55:51.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:51.802 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:55:51.802 | INFO | Total mem usage: 16.7 GiB | 20260508 06:55:51.802 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:55:51.802 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:55:51.802 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:55:51.802 | INFO | High Water Breakdown: | 20260508 06:55:51.802 | INFO | Uploaded 36GiB to GPU in 2.45229s (14.6802GiB/s) | 20260508 06:55:51.802 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:55:51.802 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:55:51.802 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:55:51.802 | INFO | Downloaded 34GiB to GPU in 2.46704s (13.7817GiB/s) | 20260508 06:55:51.802 | INFO | ------------------------------------------ | 20260508 06:55:51.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:55:51.802 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:55:51.802 | INFO | Total mem usage: 26.5 GiB | 20260508 06:55:51.803 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:55:51.803 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:55:51.803 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:55:51.803 | INFO | High Water Breakdown: | 20260508 06:55:51.803 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:55:51.803 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:55:51.803 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:55:51.803 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:55:51.803 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:55:51.803 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:55:51.803 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:55:51.803 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:55:51.803 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:55:51.803 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:55:51.803 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:55:51.803 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:55:51.803 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:55:51.803 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:55:51.803 | INFO | ------------------------------------------ | 20260508 06:55:51.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:55:51.803 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:55:51.803 | INFO | Total mem usage: 27.8 GiB | 20260508 06:55:51.803 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:55:51.803 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:55:51.803 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:55:51.803 | INFO | High Water Breakdown: | 20260508 06:55:51.803 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:55:51.803 | INFO | ------------------------------------------ | 20260508 06:55:51.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:55:51.803 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:55:51.803 | INFO | Total mem usage: 15.7 GiB | 20260508 06:55:51.803 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:55:51.803 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:55:51.803 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:55:51.803 | INFO | High Water Breakdown: | 20260508 06:55:51.803 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:55:51.803 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:55:51.803 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:55:51.803 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:55:51.803 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:55:51.803 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:55:51.803 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:55:51.803 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:55:51.803 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:55:51.803 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:55:51.803 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:55:51.803 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:55:51.803 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:55:51.803 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:55:51.803 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:55:51.803 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:55:51.803 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:55:51.803 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:55:51.803 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:55:51.803 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:55:51.803 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:55:51.803 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:55:51.803 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:55:51.803 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:55:51.803 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:55:51.803 | INFO | ------------------------------------------ | 20260508 06:55:51.803 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:55:51.803 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:55:51.803 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:55:51.803 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:55:51.803 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 1.09e+00 | 20260508 06:55:51.803 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.08e-01, 1.28e-02 / 1.42e-02 | 20260508 06:55:51.803 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.44e+00, 5.85e-01 / 2.02e-01 | 20260508 06:55:51.803 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.44e+00, 2.07e-01 / 1.27e-01 | 20260508 06:55:51.803 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.52e+01, 3.85e+00 / 1.48e+00 | 20260508 06:55:51.803 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.29e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.58e+00, 4.16e-01 / 2.52e-01 | 20260508 06:55:51.804 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.21e+00, 2.53e-01 / 1.59e-01 | 20260508 06:55:51.804 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.45e-01, 5.01e-02 / 2.83e-02 | 20260508 06:55:51.804 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:55:51.804 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:55:51.804 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:55:51.804 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 24.22%, Upload -- 2.56e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 24.47%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 2.05%, LockUpload -- 2.17e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 2.83%, LockDownload -- 3.00e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 06:55:51.804 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260508 06:55:51.914 | INFO | Starting a round of Baz IO | 20260508 06:55:55.328 | INFO | Populating RTMetrics report | 20260508 06:55:55.328 | INFO | RTMetrics report populated | 20260508 06:55:55.328 | INFO | Opening tmp RTMetrics json file | 20260508 06:55:55.329 | INFO | Serializing RTMetrics to json | 20260508 06:55:55.331 | INFO | Serialization done, swapping tmp file now... | 20260508 06:55:55.331 | INFO | RTMetrics done | 20260508 06:55:55.403 | INFO | Finished a round of Baz IO: Wrote 1.538 GiB in 3.489 seconds | 20260508 06:55:55.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.00% / 59.37% | 20260508 06:55:55.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.370312% / 0% / 5.5631e-05% | 20260508 06:55:55.403 | INFO | Baz IO overall: | 20260508 06:55:55.403 | INFO | BytesWritten : 2707141841793 | 20260508 06:55:55.403 | INFO | NumEvents : 1304781223401 | 20260508 06:55:56.940 | INFO | Analyzing chunk frames = [6119424,6119936) | 20260508 06:55:56.940 | INFO | MemInfo: { | 20260508 06:55:56.940 | INFO | "gpu" : " 17 GB", | 20260508 06:55:56.940 | INFO | "huge" : "", | 20260508 06:55:56.940 | INFO | "huge_shared" : " 29 GB", | 20260508 06:55:56.940 | INFO | "malloc" : " 14 GB", | 20260508 06:55:56.940 | INFO | "pinned" : " 29 GB" | 20260508 06:55:56.940 | INFO | } | 20260508 06:55:56.941 | INFO | Perf report for the last 5.13882 seconds: | 20260508 06:55:56.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:55:56.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:56.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:56.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:56.941 | INFO | ANALYSIS : 68.2%, 3.0, 105.9ms, 0 | 20260508 06:55:56.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:55:56.941 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 06:55:56.941 | INFO | PRE_HQ : 66.4%, 1.8, 61.8ms, 0 | 20260508 06:55:56.941 | INFO | BAZWRITER : 67.2%, 2.8, 97.4ms, 0 | 20260508 06:55:56.941 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 06:55:56.941 | INFO | Uploaded 36GiB to GPU in 2.54343s (14.1541GiB/s) | 20260508 06:55:56.941 | INFO | Downloaded 38GiB to GPU in 2.70343s (14.0562GiB/s) | 20260508 06:56:02.075 | INFO | Analyzing chunk frames = [6119936,6120448) | 20260508 06:56:02.075 | INFO | MemInfo: { | 20260508 06:56:02.075 | INFO | "gpu" : " 17 GB", | 20260508 06:56:02.075 | INFO | "huge" : "", | 20260508 06:56:02.075 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:02.075 | INFO | "malloc" : " 14 GB", | 20260508 06:56:02.075 | INFO | "pinned" : " 29 GB" | 20260508 06:56:02.075 | INFO | } | 20260508 06:56:02.075 | INFO | Perf report for the last 5.13453 seconds: | 20260508 06:56:02.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:02.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.075 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:56:02.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.075 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:56:02.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:02.076 | INFO | Uploaded 36GiB to GPU in 2.49047s (14.4551GiB/s) | 20260508 06:56:02.076 | INFO | Downloaded 34GiB to GPU in 2.44541s (13.9036GiB/s) | 20260508 06:56:07.214 | INFO | Analyzing chunk frames = [6120448,6120960) | 20260508 06:56:07.214 | INFO | MemInfo: { | 20260508 06:56:07.214 | INFO | "gpu" : " 17 GB", | 20260508 06:56:07.214 | INFO | "huge" : "", | 20260508 06:56:07.214 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:07.214 | INFO | "malloc" : " 14 GB", | 20260508 06:56:07.214 | INFO | "pinned" : " 29 GB" | 20260508 06:56:07.215 | INFO | } | 20260508 06:56:07.215 | INFO | Perf report for the last 5.13952 seconds: | 20260508 06:56:07.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:07.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 06:56:07.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:56:07.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:07.215 | INFO | Uploaded 36GiB to GPU in 2.51981s (14.2868GiB/s) | 20260508 06:56:07.215 | INFO | Downloaded 34GiB to GPU in 2.40447s (14.1403GiB/s) | 20260508 06:56:12.352 | INFO | Analyzing chunk frames = [6120960,6121472) | 20260508 06:56:12.352 | INFO | MemInfo: { | 20260508 06:56:12.352 | INFO | "gpu" : " 17 GB", | 20260508 06:56:12.352 | INFO | "huge" : "", | 20260508 06:56:12.352 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:12.352 | INFO | "malloc" : " 14 GB", | 20260508 06:56:12.352 | INFO | "pinned" : " 29 GB" | 20260508 06:56:12.352 | INFO | } | 20260508 06:56:12.352 | INFO | Perf report for the last 5.13777 seconds: | 20260508 06:56:12.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:12.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.352 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 06:56:12.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.353 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:56:12.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:12.353 | INFO | Uploaded 36GiB to GPU in 2.56473s (14.0366GiB/s) | 20260508 06:56:12.353 | INFO | Downloaded 34GiB to GPU in 2.4248s (14.0218GiB/s) | 20260508 06:56:17.493 | INFO | Analyzing chunk frames = [6121472,6121984) | 20260508 06:56:17.493 | INFO | MemInfo: { | 20260508 06:56:17.493 | INFO | "gpu" : " 17 GB", | 20260508 06:56:17.493 | INFO | "huge" : "", | 20260508 06:56:17.493 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:17.493 | INFO | "malloc" : " 14 GB", | 20260508 06:56:17.493 | INFO | "pinned" : " 29 GB" | 20260508 06:56:17.493 | INFO | } | 20260508 06:56:17.493 | INFO | Perf report for the last 5.14058 seconds: | 20260508 06:56:17.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:17.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 06:56:17.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 06:56:17.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:17.493 | INFO | Uploaded 36GiB to GPU in 2.29231s (15.7047GiB/s) | 20260508 06:56:17.493 | INFO | Downloaded 34GiB to GPU in 2.49053s (13.6517GiB/s) | 20260508 06:56:22.636 | INFO | Analyzing chunk frames = [6121984,6122496) | 20260508 06:56:22.636 | INFO | MemInfo: { | 20260508 06:56:22.636 | INFO | "gpu" : " 17 GB", | 20260508 06:56:22.636 | INFO | "huge" : "", | 20260508 06:56:22.636 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:22.636 | INFO | "malloc" : " 14 GB", | 20260508 06:56:22.636 | INFO | "pinned" : " 29 GB" | 20260508 06:56:22.636 | INFO | } | 20260508 06:56:22.636 | INFO | Perf report for the last 5.14339 seconds: | 20260508 06:56:22.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:22.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.636 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 06:56:22.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.637 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:56:22.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:22.637 | INFO | Uploaded 36GiB to GPU in 2.42943s (14.8183GiB/s) | 20260508 06:56:22.637 | INFO | Downloaded 34GiB to GPU in 2.42641s (14.0125GiB/s) | 20260508 06:56:27.773 | INFO | Analyzing chunk frames = [6122496,6123008) | 20260508 06:56:27.773 | INFO | MemInfo: { | 20260508 06:56:27.773 | INFO | "gpu" : " 17 GB", | 20260508 06:56:27.773 | INFO | "huge" : "", | 20260508 06:56:27.773 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:27.773 | INFO | "malloc" : " 14 GB", | 20260508 06:56:27.773 | INFO | "pinned" : " 29 GB" | 20260508 06:56:27.773 | INFO | } | 20260508 06:56:27.773 | INFO | Perf report for the last 5.1369 seconds: | 20260508 06:56:27.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:27.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.773 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:56:27.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.773 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 06:56:27.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:27.774 | INFO | Uploaded 36GiB to GPU in 2.47306s (14.5569GiB/s) | 20260508 06:56:27.774 | INFO | Downloaded 34GiB to GPU in 2.43332s (13.9727GiB/s) | 20260508 06:56:32.911 | INFO | Analyzing chunk frames = [6123008,6123520) | 20260508 06:56:32.911 | INFO | MemInfo: { | 20260508 06:56:32.911 | INFO | "gpu" : " 17 GB", | 20260508 06:56:32.911 | INFO | "huge" : "", | 20260508 06:56:32.911 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:32.911 | INFO | "malloc" : " 14 GB", | 20260508 06:56:32.911 | INFO | "pinned" : " 29 GB" | 20260508 06:56:32.911 | INFO | } | 20260508 06:56:32.911 | INFO | Perf report for the last 5.13789 seconds: | 20260508 06:56:32.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:32.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:56:32.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:56:32.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:32.911 | INFO | Uploaded 36GiB to GPU in 2.45986s (14.635GiB/s) | 20260508 06:56:32.911 | INFO | Downloaded 34GiB to GPU in 2.44179s (13.9242GiB/s) | 20260508 06:56:33.008 | INFO | Starting a round of Baz IO | 20260508 06:56:36.442 | INFO | Populating RTMetrics report | 20260508 06:56:36.443 | INFO | RTMetrics report populated | 20260508 06:56:36.443 | INFO | Opening tmp RTMetrics json file | 20260508 06:56:36.452 | INFO | Serializing RTMetrics to json | 20260508 06:56:36.454 | INFO | Serialization done, swapping tmp file now... | 20260508 06:56:36.454 | INFO | RTMetrics done | 20260508 06:56:36.512 | INFO | Finished a round of Baz IO: Wrote 1.538 GiB in 3.504 seconds | 20260508 06:56:36.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.00% / 59.37% | 20260508 06:56:36.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.370359% / 0% / 5.96046e-05% | 20260508 06:56:36.512 | INFO | Baz IO overall: | 20260508 06:56:36.512 | INFO | BytesWritten : 2708793403171 | 20260508 06:56:36.512 | INFO | NumEvents : 1305566399495 | 20260508 06:56:38.047 | INFO | Analyzing chunk frames = [6123520,6124032) | 20260508 06:56:38.047 | INFO | MemInfo: { | 20260508 06:56:38.047 | INFO | "gpu" : " 17 GB", | 20260508 06:56:38.047 | INFO | "huge" : "", | 20260508 06:56:38.047 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:38.047 | INFO | "malloc" : " 14 GB", | 20260508 06:56:38.047 | INFO | "pinned" : " 29 GB" | 20260508 06:56:38.047 | INFO | } | 20260508 06:56:38.047 | INFO | Perf report for the last 5.136 seconds: | 20260508 06:56:38.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:38.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:38.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:38.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:38.047 | INFO | ANALYSIS : 67.9%, 3.0, 106.1ms, 0 | 20260508 06:56:38.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:38.047 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 06:56:38.047 | INFO | PRE_HQ : 65.7%, 1.6, 54.8ms, 0 | 20260508 06:56:38.047 | INFO | BAZWRITER : 67.8%, 2.6, 89.9ms, 0 | 20260508 06:56:38.047 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:56:38.047 | INFO | Uploaded 36GiB to GPU in 2.59521s (13.8717GiB/s) | 20260508 06:56:38.048 | INFO | Downloaded 38GiB to GPU in 2.69979s (14.0752GiB/s) | 20260508 06:56:43.194 | INFO | Analyzing chunk frames = [6124032,6124544) | 20260508 06:56:43.194 | INFO | MemInfo: { | 20260508 06:56:43.194 | INFO | "gpu" : " 17 GB", | 20260508 06:56:43.194 | INFO | "huge" : "", | 20260508 06:56:43.194 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:43.194 | INFO | "malloc" : " 14 GB", | 20260508 06:56:43.194 | INFO | "pinned" : " 29 GB" | 20260508 06:56:43.194 | INFO | } | 20260508 06:56:43.194 | INFO | Perf report for the last 5.14711 seconds: | 20260508 06:56:43.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:43.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.194 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 06:56:43.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.194 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 06:56:43.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:43.195 | INFO | Uploaded 36GiB to GPU in 2.50282s (14.3838GiB/s) | 20260508 06:56:43.195 | INFO | Downloaded 34GiB to GPU in 2.43864s (13.9422GiB/s) | 20260508 06:56:48.332 | INFO | Analyzing chunk frames = [6124544,6125056) | 20260508 06:56:48.332 | INFO | MemInfo: { | 20260508 06:56:48.332 | INFO | "gpu" : " 17 GB", | 20260508 06:56:48.332 | INFO | "huge" : "", | 20260508 06:56:48.332 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:48.332 | INFO | "malloc" : " 14 GB", | 20260508 06:56:48.332 | INFO | "pinned" : " 29 GB" | 20260508 06:56:48.332 | INFO | } | 20260508 06:56:48.332 | INFO | Perf report for the last 5.13786 seconds: | 20260508 06:56:48.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:48.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:56:48.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 06:56:48.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:48.332 | INFO | Uploaded 36GiB to GPU in 2.55092s (14.1125GiB/s) | 20260508 06:56:48.333 | INFO | Downloaded 34GiB to GPU in 2.42778s (14.0046GiB/s) | 20260508 06:56:53.473 | INFO | Analyzing chunk frames = [6125056,6125568) | 20260508 06:56:53.473 | INFO | MemInfo: { | 20260508 06:56:53.473 | INFO | "gpu" : " 17 GB", | 20260508 06:56:53.473 | INFO | "huge" : "", | 20260508 06:56:53.473 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:53.473 | INFO | "malloc" : " 14 GB", | 20260508 06:56:53.473 | INFO | "pinned" : " 29 GB" | 20260508 06:56:53.473 | INFO | } | 20260508 06:56:53.473 | INFO | Perf report for the last 5.14092 seconds: | 20260508 06:56:53.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:53.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | ANALYSIS : 66.7%, 2.9, 103.1ms, 0 | 20260508 06:56:53.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:56:53.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:53.473 | INFO | Uploaded 36GiB to GPU in 2.54086s (14.1684GiB/s) | 20260508 06:56:53.473 | INFO | Downloaded 34GiB to GPU in 2.40727s (14.1239GiB/s) | 20260508 06:56:58.609 | INFO | Analyzing chunk frames = [6125568,6126080) | 20260508 06:56:58.609 | INFO | MemInfo: { | 20260508 06:56:58.609 | INFO | "gpu" : " 17 GB", | 20260508 06:56:58.609 | INFO | "huge" : "", | 20260508 06:56:58.609 | INFO | "huge_shared" : " 29 GB", | 20260508 06:56:58.609 | INFO | "malloc" : " 14 GB", | 20260508 06:56:58.609 | INFO | "pinned" : " 29 GB" | 20260508 06:56:58.609 | INFO | } | 20260508 06:56:58.609 | INFO | Perf report for the last 5.13579 seconds: | 20260508 06:56:58.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:56:58.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 06:56:58.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:56:58.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:56:58.609 | INFO | Uploaded 36GiB to GPU in 2.29809s (15.6652GiB/s) | 20260508 06:56:58.609 | INFO | Downloaded 34GiB to GPU in 2.49121s (13.648GiB/s) | 20260508 06:57:03.754 | INFO | Analyzing chunk frames = [6126080,6126592) | 20260508 06:57:03.754 | INFO | MemInfo: { | 20260508 06:57:03.754 | INFO | "gpu" : " 17 GB", | 20260508 06:57:03.754 | INFO | "huge" : "", | 20260508 06:57:03.754 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:03.754 | INFO | "malloc" : " 14 GB", | 20260508 06:57:03.754 | INFO | "pinned" : " 29 GB" | 20260508 06:57:03.754 | INFO | } | 20260508 06:57:03.754 | INFO | Perf report for the last 5.14483 seconds: | 20260508 06:57:03.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:03.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 06:57:03.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 06:57:03.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:03.754 | INFO | Uploaded 36GiB to GPU in 2.4697s (14.5766GiB/s) | 20260508 06:57:03.754 | INFO | Downloaded 34GiB to GPU in 2.44027s (13.9329GiB/s) | 20260508 06:57:08.889 | INFO | Analyzing chunk frames = [6126592,6127104) | 20260508 06:57:08.889 | INFO | MemInfo: { | 20260508 06:57:08.889 | INFO | "gpu" : " 17 GB", | 20260508 06:57:08.889 | INFO | "huge" : "", | 20260508 06:57:08.889 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:08.889 | INFO | "malloc" : " 14 GB", | 20260508 06:57:08.889 | INFO | "pinned" : " 29 GB" | 20260508 06:57:08.889 | INFO | } | 20260508 06:57:08.889 | INFO | Perf report for the last 5.13546 seconds: | 20260508 06:57:08.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:08.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.889 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:57:08.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.889 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 06:57:08.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:08.890 | INFO | Uploaded 36GiB to GPU in 2.46779s (14.588GiB/s) | 20260508 06:57:08.890 | INFO | Downloaded 34GiB to GPU in 2.43016s (13.9909GiB/s) | 20260508 06:57:14.029 | INFO | Analyzing chunk frames = [6127104,6127616) | 20260508 06:57:14.029 | INFO | MemInfo: { | 20260508 06:57:14.029 | INFO | "gpu" : " 17 GB", | 20260508 06:57:14.029 | INFO | "huge" : "", | 20260508 06:57:14.029 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:14.029 | INFO | "malloc" : " 14 GB", | 20260508 06:57:14.029 | INFO | "pinned" : " 29 GB" | 20260508 06:57:14.029 | INFO | } | 20260508 06:57:14.029 | INFO | Perf report for the last 5.13977 seconds: | 20260508 06:57:14.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:14.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 06:57:14.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:57:14.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:14.029 | INFO | Uploaded 36GiB to GPU in 2.46816s (14.5858GiB/s) | 20260508 06:57:14.029 | INFO | Downloaded 34GiB to GPU in 2.43355s (13.9714GiB/s) | 20260508 06:57:14.030 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:57:14.030 | INFO | Total mem usage: 16.7 GiB | 20260508 06:57:14.030 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:57:14.030 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:57:14.030 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:57:14.030 | INFO | High Water Breakdown: | 20260508 06:57:14.030 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:57:14.030 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:57:14.030 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:57:14.030 | INFO | ------------------------------------------ | 20260508 06:57:14.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:57:14.030 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:57:14.030 | INFO | Total mem usage: 26.5 GiB | 20260508 06:57:14.030 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:57:14.030 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 06:57:14.030 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 06:57:14.030 | INFO | High Water Breakdown: | 20260508 06:57:14.030 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:57:14.030 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:57:14.030 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:57:14.030 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:57:14.030 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:57:14.030 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:57:14.030 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:57:14.030 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:57:14.030 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:57:14.030 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:57:14.030 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:57:14.030 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:57:14.030 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:57:14.030 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:57:14.030 | INFO | ------------------------------------------ | 20260508 06:57:14.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:57:14.030 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:57:14.030 | INFO | Total mem usage: 27.8 GiB | 20260508 06:57:14.030 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:57:14.030 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:57:14.030 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:57:14.030 | INFO | High Water Breakdown: | 20260508 06:57:14.030 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:57:14.030 | INFO | ------------------------------------------ | 20260508 06:57:14.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:57:14.030 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:57:14.030 | INFO | Total mem usage: 15.7 GiB | 20260508 06:57:14.030 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:57:14.030 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:57:14.030 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:57:14.030 | INFO | High Water Breakdown: | 20260508 06:57:14.030 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:57:14.030 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:57:14.030 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:57:14.030 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:57:14.030 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:57:14.030 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:57:14.030 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:57:14.030 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:57:14.030 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:57:14.030 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:57:14.030 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:57:14.030 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:57:14.030 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:57:14.030 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:57:14.030 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:57:14.030 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:57:14.030 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:57:14.030 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:57:14.030 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:57:14.030 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:57:14.030 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:57:14.030 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:57:14.030 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:57:14.030 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:57:14.030 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:57:14.030 | INFO | ------------------------------------------ | 20260508 06:57:14.030 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:57:14.030 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:57:14.030 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:57:14.030 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:57:14.030 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.41e+01 / 6.09e-01 | 20260508 06:57:14.030 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.13e-02 / 6.52e-03 | 20260508 06:57:14.030 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.34e+00, 5.85e-01 / 2.10e-01 | 20260508 06:57:14.030 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.42e+00, 2.09e-01 / 1.39e-01 | 20260508 06:57:14.030 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.53e+01, 3.72e+00 / 1.16e+00 | 20260508 06:57:14.030 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.79e+00 / 1.29e+00 | 20260508 06:57:14.030 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.37e+00 / 2.52e+00, 4.10e-01 / 2.57e-01 | 20260508 06:57:14.030 | INFO | PerfReport: 3.14%, Metrics -- 1.02e+00 / 1.22e+00, 2.54e-01 / 1.82e-01 | 20260508 06:57:14.030 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.40e-01, 4.94e-02 / 2.43e-02 | 20260508 06:57:14.031 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:57:14.031 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:57:14.031 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:57:14.031 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 24.36%, Upload -- 2.58e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 24.28%, Download -- 2.57e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 30.80%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 2.21%, LockUpload -- 2.35e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 2.69%, LockDownload -- 2.85e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 15.64%, LockCompute -- 1.66e+01 / 0.00e+00, 1.96e+01 / 0.00e+00 | 20260508 06:57:14.031 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 06:57:14.127 | INFO | Starting a round of Baz IO | 20260508 06:57:17.532 | INFO | Populating RTMetrics report | 20260508 06:57:17.533 | INFO | RTMetrics report populated | 20260508 06:57:17.533 | INFO | Opening tmp RTMetrics json file | 20260508 06:57:17.534 | INFO | Serializing RTMetrics to json | 20260508 06:57:17.537 | INFO | Serialization done, swapping tmp file now... | 20260508 06:57:17.538 | INFO | RTMetrics done | 20260508 06:57:17.623 | INFO | Finished a round of Baz IO: Wrote 1.537 GiB in 3.495 seconds | 20260508 06:57:17.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.62% / 0.00% / 59.37% | 20260508 06:57:17.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.368849% / 0% / 4.37101e-05% | 20260508 06:57:17.623 | INFO | Baz IO overall: | 20260508 06:57:17.623 | INFO | BytesWritten : 2710443960932 | 20260508 06:57:17.623 | INFO | NumEvents : 1306350940967 | 20260508 06:57:19.167 | INFO | Analyzing chunk frames = [6127616,6128128) | 20260508 06:57:19.167 | INFO | MemInfo: { | 20260508 06:57:19.167 | INFO | "gpu" : " 17 GB", | 20260508 06:57:19.167 | INFO | "huge" : "", | 20260508 06:57:19.167 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:19.167 | INFO | "malloc" : " 14 GB", | 20260508 06:57:19.167 | INFO | "pinned" : " 29 GB" | 20260508 06:57:19.167 | INFO | } | 20260508 06:57:19.167 | INFO | Perf report for the last 5.13774 seconds: | 20260508 06:57:19.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:19.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:19.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:19.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:19.167 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 06:57:19.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:19.167 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 06:57:19.167 | INFO | PRE_HQ : 65.6%, 1.6, 54.8ms, 0 | 20260508 06:57:19.167 | INFO | BAZWRITER : 67.5%, 2.6, 91.6ms, 0 | 20260508 06:57:19.167 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:57:19.167 | INFO | Uploaded 36GiB to GPU in 2.56432s (14.0388GiB/s) | 20260508 06:57:19.167 | INFO | Downloaded 38GiB to GPU in 2.69867s (14.081GiB/s) | 20260508 06:57:24.301 | INFO | Analyzing chunk frames = [6128128,6128640) | 20260508 06:57:24.301 | INFO | MemInfo: { | 20260508 06:57:24.301 | INFO | "gpu" : " 17 GB", | 20260508 06:57:24.301 | INFO | "huge" : "", | 20260508 06:57:24.301 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:24.301 | INFO | "malloc" : " 14 GB", | 20260508 06:57:24.301 | INFO | "pinned" : " 29 GB" | 20260508 06:57:24.301 | INFO | } | 20260508 06:57:24.302 | INFO | Perf report for the last 5.13473 seconds: | 20260508 06:57:24.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:24.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 06:57:24.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:57:24.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:24.302 | INFO | Uploaded 36GiB to GPU in 2.49804s (14.4113GiB/s) | 20260508 06:57:24.302 | INFO | Downloaded 34GiB to GPU in 2.42616s (14.0139GiB/s) | 20260508 06:57:29.446 | INFO | Analyzing chunk frames = [6128640,6129152) | 20260508 06:57:29.446 | INFO | MemInfo: { | 20260508 06:57:29.446 | INFO | "gpu" : " 17 GB", | 20260508 06:57:29.446 | INFO | "huge" : "", | 20260508 06:57:29.446 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:29.447 | INFO | "malloc" : " 14 GB", | 20260508 06:57:29.447 | INFO | "pinned" : " 29 GB" | 20260508 06:57:29.447 | INFO | } | 20260508 06:57:29.447 | INFO | Perf report for the last 5.1451 seconds: | 20260508 06:57:29.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:29.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 06:57:29.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:57:29.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:29.447 | INFO | Uploaded 36GiB to GPU in 2.54178s (14.1633GiB/s) | 20260508 06:57:29.447 | INFO | Downloaded 34GiB to GPU in 2.42414s (14.0256GiB/s) | 20260508 06:57:34.584 | INFO | Analyzing chunk frames = [6129152,6129664) | 20260508 06:57:34.584 | INFO | MemInfo: { | 20260508 06:57:34.584 | INFO | "gpu" : " 17 GB", | 20260508 06:57:34.584 | INFO | "huge" : "", | 20260508 06:57:34.584 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:34.584 | INFO | "malloc" : " 14 GB", | 20260508 06:57:34.584 | INFO | "pinned" : " 29 GB" | 20260508 06:57:34.584 | INFO | } | 20260508 06:57:34.584 | INFO | Perf report for the last 5.13733 seconds: | 20260508 06:57:34.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:34.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 06:57:34.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:57:34.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:34.584 | INFO | Uploaded 36GiB to GPU in 2.56643s (14.0273GiB/s) | 20260508 06:57:34.584 | INFO | Downloaded 34GiB to GPU in 2.41641s (14.0705GiB/s) | 20260508 06:57:39.723 | INFO | Analyzing chunk frames = [6129664,6130176) | 20260508 06:57:39.723 | INFO | MemInfo: { | 20260508 06:57:39.723 | INFO | "gpu" : " 17 GB", | 20260508 06:57:39.723 | INFO | "huge" : "", | 20260508 06:57:39.723 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:39.723 | INFO | "malloc" : " 14 GB", | 20260508 06:57:39.723 | INFO | "pinned" : " 29 GB" | 20260508 06:57:39.723 | INFO | } | 20260508 06:57:39.723 | INFO | Perf report for the last 5.13883 seconds: | 20260508 06:57:39.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:39.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 06:57:39.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 06:57:39.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:39.723 | INFO | Uploaded 36GiB to GPU in 2.416s (14.9006GiB/s) | 20260508 06:57:39.723 | INFO | Downloaded 34GiB to GPU in 2.52621s (13.4589GiB/s) | 20260508 06:57:44.860 | INFO | Analyzing chunk frames = [6130176,6130688) | 20260508 06:57:44.860 | INFO | MemInfo: { | 20260508 06:57:44.860 | INFO | "gpu" : " 17 GB", | 20260508 06:57:44.860 | INFO | "huge" : "", | 20260508 06:57:44.860 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:44.860 | INFO | "malloc" : " 14 GB", | 20260508 06:57:44.860 | INFO | "pinned" : " 29 GB" | 20260508 06:57:44.860 | INFO | } | 20260508 06:57:44.860 | INFO | Perf report for the last 5.13743 seconds: | 20260508 06:57:44.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:44.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.860 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 06:57:44.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.860 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:57:44.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:44.861 | INFO | Uploaded 36GiB to GPU in 2.43583s (14.7793GiB/s) | 20260508 06:57:44.861 | INFO | Downloaded 34GiB to GPU in 2.42341s (14.0298GiB/s) | 20260508 06:57:50.004 | INFO | Analyzing chunk frames = [6130688,6131200) | 20260508 06:57:50.004 | INFO | MemInfo: { | 20260508 06:57:50.004 | INFO | "gpu" : " 17 GB", | 20260508 06:57:50.004 | INFO | "huge" : "", | 20260508 06:57:50.004 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:50.004 | INFO | "malloc" : " 14 GB", | 20260508 06:57:50.004 | INFO | "pinned" : " 29 GB" | 20260508 06:57:50.004 | INFO | } | 20260508 06:57:50.004 | INFO | Perf report for the last 5.14408 seconds: | 20260508 06:57:50.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:50.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.004 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 06:57:50.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.005 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 06:57:50.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:50.005 | INFO | Uploaded 36GiB to GPU in 2.4485s (14.7029GiB/s) | 20260508 06:57:50.005 | INFO | Downloaded 34GiB to GPU in 2.434s (13.9688GiB/s) | 20260508 06:57:55.144 | INFO | Analyzing chunk frames = [6131200,6131712) | 20260508 06:57:55.144 | INFO | MemInfo: { | 20260508 06:57:55.144 | INFO | "gpu" : " 17 GB", | 20260508 06:57:55.144 | INFO | "huge" : "", | 20260508 06:57:55.144 | INFO | "huge_shared" : " 29 GB", | 20260508 06:57:55.144 | INFO | "malloc" : " 14 GB", | 20260508 06:57:55.144 | INFO | "pinned" : " 29 GB" | 20260508 06:57:55.144 | INFO | } | 20260508 06:57:55.144 | INFO | Perf report for the last 5.13969 seconds: | 20260508 06:57:55.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:57:55.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 06:57:55.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:57:55.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:57:55.144 | INFO | Uploaded 36GiB to GPU in 2.48845s (14.4668GiB/s) | 20260508 06:57:55.144 | INFO | Downloaded 34GiB to GPU in 2.45272s (13.8622GiB/s) | 20260508 06:57:55.241 | INFO | Starting a round of Baz IO | 20260508 06:57:58.662 | INFO | Populating RTMetrics report | 20260508 06:57:58.663 | INFO | RTMetrics report populated | 20260508 06:57:58.663 | INFO | Opening tmp RTMetrics json file | 20260508 06:57:58.674 | INFO | Serializing RTMetrics to json | 20260508 06:57:58.675 | INFO | Serialization done, swapping tmp file now... | 20260508 06:57:58.675 | INFO | RTMetrics done | 20260508 06:57:58.727 | INFO | Finished a round of Baz IO: Wrote 1.538 GiB in 3.486 seconds | 20260508 06:57:58.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.62% / 0.00% / 59.38% | 20260508 06:57:58.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.368965% / 3.97364e-06% / 6.75519e-05% | 20260508 06:57:58.727 | INFO | Baz IO overall: | 20260508 06:57:58.727 | INFO | BytesWritten : 2712094889865 | 20260508 06:57:58.727 | INFO | NumEvents : 1307135803953 | 20260508 06:58:00.287 | INFO | Analyzing chunk frames = [6131712,6132224) | 20260508 06:58:00.287 | INFO | MemInfo: { | 20260508 06:58:00.287 | INFO | "gpu" : " 17 GB", | 20260508 06:58:00.287 | INFO | "huge" : "", | 20260508 06:58:00.287 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:00.287 | INFO | "malloc" : " 14 GB", | 20260508 06:58:00.287 | INFO | "pinned" : " 29 GB" | 20260508 06:58:00.287 | INFO | } | 20260508 06:58:00.287 | INFO | Perf report for the last 5.14332 seconds: | 20260508 06:58:00.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:00.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:00.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:00.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:00.287 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 06:58:00.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:00.288 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 06:58:00.288 | INFO | PRE_HQ : 65.6%, 1.6, 54.9ms, 0 | 20260508 06:58:00.288 | INFO | BAZWRITER : 67.2%, 2.6, 89.6ms, 0 | 20260508 06:58:00.288 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:58:00.288 | INFO | Uploaded 36GiB to GPU in 2.56143s (14.0546GiB/s) | 20260508 06:58:00.288 | INFO | Downloaded 38GiB to GPU in 2.70488s (14.0487GiB/s) | 20260508 06:58:05.427 | INFO | Analyzing chunk frames = [6132224,6132736) | 20260508 06:58:05.427 | INFO | MemInfo: { | 20260508 06:58:05.427 | INFO | "gpu" : " 17 GB", | 20260508 06:58:05.427 | INFO | "huge" : "", | 20260508 06:58:05.427 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:05.428 | INFO | "malloc" : " 14 GB", | 20260508 06:58:05.428 | INFO | "pinned" : " 29 GB" | 20260508 06:58:05.428 | INFO | } | 20260508 06:58:05.428 | INFO | Perf report for the last 5.14023 seconds: | 20260508 06:58:05.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:05.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 06:58:05.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 06:58:05.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:05.428 | INFO | Uploaded 36GiB to GPU in 2.50953s (14.3453GiB/s) | 20260508 06:58:05.428 | INFO | Downloaded 34GiB to GPU in 2.43925s (13.9387GiB/s) | 20260508 06:58:10.562 | INFO | Analyzing chunk frames = [6132736,6133248) | 20260508 06:58:10.562 | INFO | MemInfo: { | 20260508 06:58:10.562 | INFO | "gpu" : " 17 GB", | 20260508 06:58:10.562 | INFO | "huge" : "", | 20260508 06:58:10.562 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:10.563 | INFO | "malloc" : " 14 GB", | 20260508 06:58:10.563 | INFO | "pinned" : " 29 GB" | 20260508 06:58:10.563 | INFO | } | 20260508 06:58:10.563 | INFO | Perf report for the last 5.13498 seconds: | 20260508 06:58:10.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:10.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.563 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 06:58:10.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.563 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 06:58:10.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:10.564 | INFO | Uploaded 36GiB to GPU in 2.47647s (14.5368GiB/s) | 20260508 06:58:10.564 | INFO | Downloaded 34GiB to GPU in 2.42717s (14.0081GiB/s) | 20260508 06:58:15.703 | INFO | Analyzing chunk frames = [6133248,6133760) | 20260508 06:58:15.703 | INFO | MemInfo: { | 20260508 06:58:15.703 | INFO | "gpu" : " 17 GB", | 20260508 06:58:15.703 | INFO | "huge" : "", | 20260508 06:58:15.703 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:15.703 | INFO | "malloc" : " 14 GB", | 20260508 06:58:15.703 | INFO | "pinned" : " 29 GB" | 20260508 06:58:15.703 | INFO | } | 20260508 06:58:15.703 | INFO | Perf report for the last 5.13995 seconds: | 20260508 06:58:15.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:15.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 06:58:15.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:58:15.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:15.703 | INFO | Uploaded 36GiB to GPU in 2.56847s (14.0161GiB/s) | 20260508 06:58:15.703 | INFO | Downloaded 34GiB to GPU in 2.42093s (14.0442GiB/s) | 20260508 06:58:20.838 | INFO | Analyzing chunk frames = [6133760,6134272) | 20260508 06:58:20.838 | INFO | MemInfo: { | 20260508 06:58:20.838 | INFO | "gpu" : " 17 GB", | 20260508 06:58:20.838 | INFO | "huge" : "", | 20260508 06:58:20.838 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:20.838 | INFO | "malloc" : " 14 GB", | 20260508 06:58:20.838 | INFO | "pinned" : " 29 GB" | 20260508 06:58:20.838 | INFO | } | 20260508 06:58:20.838 | INFO | Perf report for the last 5.13542 seconds: | 20260508 06:58:20.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:20.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 06:58:20.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 06:58:20.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:20.838 | INFO | Uploaded 36GiB to GPU in 2.30668s (15.6069GiB/s) | 20260508 06:58:20.838 | INFO | Downloaded 34GiB to GPU in 2.49048s (13.652GiB/s) | 20260508 06:58:25.975 | INFO | Analyzing chunk frames = [6134272,6134784) | 20260508 06:58:25.975 | INFO | MemInfo: { | 20260508 06:58:25.975 | INFO | "gpu" : " 17 GB", | 20260508 06:58:25.975 | INFO | "huge" : "", | 20260508 06:58:25.975 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:25.975 | INFO | "malloc" : " 14 GB", | 20260508 06:58:25.975 | INFO | "pinned" : " 29 GB" | 20260508 06:58:25.975 | INFO | } | 20260508 06:58:25.975 | INFO | Perf report for the last 5.13692 seconds: | 20260508 06:58:25.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:25.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 06:58:25.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:58:25.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:25.975 | INFO | Uploaded 36GiB to GPU in 2.44144s (14.7454GiB/s) | 20260508 06:58:25.975 | INFO | Downloaded 34GiB to GPU in 2.43263s (13.9767GiB/s) | 20260508 06:58:31.111 | INFO | Analyzing chunk frames = [6134784,6135296) | 20260508 06:58:31.111 | INFO | MemInfo: { | 20260508 06:58:31.111 | INFO | "gpu" : " 17 GB", | 20260508 06:58:31.111 | INFO | "huge" : "", | 20260508 06:58:31.111 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:31.112 | INFO | "malloc" : " 14 GB", | 20260508 06:58:31.112 | INFO | "pinned" : " 29 GB" | 20260508 06:58:31.112 | INFO | } | 20260508 06:58:31.112 | INFO | Perf report for the last 5.13665 seconds: | 20260508 06:58:31.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:31.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 06:58:31.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 06:58:31.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:31.112 | INFO | Uploaded 36GiB to GPU in 2.44837s (14.7037GiB/s) | 20260508 06:58:31.112 | INFO | Downloaded 34GiB to GPU in 2.42136s (14.0417GiB/s) | 20260508 06:58:36.254 | INFO | Analyzing chunk frames = [6135296,6135808) | 20260508 06:58:36.254 | INFO | MemInfo: { | 20260508 06:58:36.254 | INFO | "gpu" : " 17 GB", | 20260508 06:58:36.254 | INFO | "huge" : "", | 20260508 06:58:36.254 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:36.254 | INFO | "malloc" : " 14 GB", | 20260508 06:58:36.254 | INFO | "pinned" : " 29 GB" | 20260508 06:58:36.254 | INFO | } | 20260508 06:58:36.254 | INFO | Perf report for the last 5.14265 seconds: | 20260508 06:58:36.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:36.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.254 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 06:58:36.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.254 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 06:58:36.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:36.255 | INFO | Uploaded 36GiB to GPU in 2.45128s (14.6862GiB/s) | 20260508 06:58:36.255 | INFO | Downloaded 34GiB to GPU in 2.43351s (13.9716GiB/s) | 20260508 06:58:36.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:58:36.255 | INFO | Total mem usage: 16.7 GiB | 20260508 06:58:36.255 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:58:36.255 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 06:58:36.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:58:36.255 | INFO | High Water Breakdown: | 20260508 06:58:36.255 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:58:36.255 | INFO | 40.4% : 5.76 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:58:36.255 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:58:36.255 | INFO | ------------------------------------------ | 20260508 06:58:36.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:58:36.255 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:58:36.255 | INFO | Total mem usage: 26.5 GiB | 20260508 06:58:36.255 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:58:36.255 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 06:58:36.255 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 06:58:36.255 | INFO | High Water Breakdown: | 20260508 06:58:36.255 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:58:36.255 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:58:36.255 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:58:36.255 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:58:36.255 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:58:36.255 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:58:36.255 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:58:36.255 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:58:36.255 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:58:36.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:58:36.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:58:36.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:58:36.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:58:36.255 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:58:36.255 | INFO | ------------------------------------------ | 20260508 06:58:36.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:58:36.255 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:58:36.255 | INFO | Total mem usage: 27.8 GiB | 20260508 06:58:36.255 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:58:36.255 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:58:36.255 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:58:36.255 | INFO | High Water Breakdown: | 20260508 06:58:36.255 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:58:36.255 | INFO | ------------------------------------------ | 20260508 06:58:36.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:58:36.255 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:58:36.255 | INFO | Total mem usage: 15.7 GiB | 20260508 06:58:36.255 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:58:36.255 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:58:36.255 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:58:36.255 | INFO | High Water Breakdown: | 20260508 06:58:36.255 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:58:36.255 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:58:36.255 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:58:36.255 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:58:36.255 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:58:36.255 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:58:36.255 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:58:36.255 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:58:36.255 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:58:36.255 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:58:36.255 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:58:36.255 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:58:36.255 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:58:36.255 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:58:36.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:58:36.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:58:36.255 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:58:36.255 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:58:36.255 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:58:36.255 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:58:36.255 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:58:36.255 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:58:36.255 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:58:36.255 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:58:36.255 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:58:36.255 | INFO | ------------------------------------------ | 20260508 06:58:36.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:58:36.255 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:58:36.255 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:58:36.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:58:36.255 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 6.53e-01 | 20260508 06:58:36.255 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.08e-02 / 6.29e-03 | 20260508 06:58:36.255 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.47e+00, 5.84e-01 / 2.24e-01 | 20260508 06:58:36.255 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.06e-01 / 1.21e-01 | 20260508 06:58:36.255 | INFO | PerfReport: 15.91%, DME -- 4.16e+01 / 4.50e+01, 3.71e+00 / 1.26e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.82e+00 / 1.35e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.55e+00, 4.09e-01 / 2.31e-01 | 20260508 06:58:36.255 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.23e+00, 2.50e-01 / 1.33e-01 | 20260508 06:58:36.255 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.44e-01, 5.02e-02 / 2.83e-02 | 20260508 06:58:36.255 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:58:36.255 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:58:36.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:58:36.255 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 24.40%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 24.31%, Download -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 2.19%, LockUpload -- 2.32e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 2.63%, LockDownload -- 2.78e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 06:58:36.255 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260508 06:58:36.351 | INFO | Starting a round of Baz IO | 20260508 06:58:39.789 | INFO | Populating RTMetrics report | 20260508 06:58:39.789 | INFO | RTMetrics report populated | 20260508 06:58:39.789 | INFO | Opening tmp RTMetrics json file | 20260508 06:58:39.801 | INFO | Serializing RTMetrics to json | 20260508 06:58:39.802 | INFO | Serialization done, swapping tmp file now... | 20260508 06:58:39.802 | INFO | RTMetrics done | 20260508 06:58:39.854 | INFO | Finished a round of Baz IO: Wrote 1.538 GiB in 3.503 seconds | 20260508 06:58:39.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.62% / 0.00% / 59.38% | 20260508 06:58:39.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.369084% / 0% / 6.75519e-05% | 20260508 06:58:39.854 | INFO | Baz IO overall: | 20260508 06:58:39.854 | INFO | BytesWritten : 2713746510083 | 20260508 06:58:39.854 | INFO | NumEvents : 1307920996505 | 20260508 06:58:41.398 | INFO | Analyzing chunk frames = [6135808,6136320) | 20260508 06:58:41.398 | INFO | MemInfo: { | 20260508 06:58:41.398 | INFO | "gpu" : " 17 GB", | 20260508 06:58:41.398 | INFO | "huge" : "", | 20260508 06:58:41.398 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:41.398 | INFO | "malloc" : " 14 GB", | 20260508 06:58:41.398 | INFO | "pinned" : " 29 GB" | 20260508 06:58:41.398 | INFO | } | 20260508 06:58:41.398 | INFO | Perf report for the last 5.14397 seconds: | 20260508 06:58:41.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:41.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:41.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:41.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:41.398 | INFO | ANALYSIS : 68.0%, 3.0, 106.2ms, 0 | 20260508 06:58:41.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:41.399 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 06:58:41.399 | INFO | PRE_HQ : 65.7%, 1.6, 54.8ms, 0 | 20260508 06:58:41.399 | INFO | BAZWRITER : 67.5%, 2.6, 88.8ms, 0 | 20260508 06:58:41.399 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 06:58:41.399 | INFO | Uploaded 36GiB to GPU in 2.57238s (13.9948GiB/s) | 20260508 06:58:41.399 | INFO | Downloaded 38GiB to GPU in 2.69066s (14.1229GiB/s) | 20260508 06:58:46.537 | INFO | Analyzing chunk frames = [6136320,6136832) | 20260508 06:58:46.537 | INFO | MemInfo: { | 20260508 06:58:46.537 | INFO | "gpu" : " 17 GB", | 20260508 06:58:46.537 | INFO | "huge" : "", | 20260508 06:58:46.537 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:46.537 | INFO | "malloc" : " 14 GB", | 20260508 06:58:46.537 | INFO | "pinned" : " 29 GB" | 20260508 06:58:46.537 | INFO | } | 20260508 06:58:46.537 | INFO | Perf report for the last 5.13891 seconds: | 20260508 06:58:46.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:46.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.537 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 06:58:46.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.537 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 06:58:46.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:46.538 | INFO | Uploaded 36GiB to GPU in 2.50929s (14.3467GiB/s) | 20260508 06:58:46.538 | INFO | Downloaded 34GiB to GPU in 2.42919s (13.9964GiB/s) | 20260508 06:58:51.672 | INFO | Analyzing chunk frames = [6136832,6137344) | 20260508 06:58:51.672 | INFO | MemInfo: { | 20260508 06:58:51.672 | INFO | "gpu" : " 17 GB", | 20260508 06:58:51.672 | INFO | "huge" : "", | 20260508 06:58:51.672 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:51.672 | INFO | "malloc" : " 14 GB", | 20260508 06:58:51.672 | INFO | "pinned" : " 29 GB" | 20260508 06:58:51.672 | INFO | } | 20260508 06:58:51.672 | INFO | Perf report for the last 5.13461 seconds: | 20260508 06:58:51.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:51.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 06:58:51.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 06:58:51.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:51.672 | INFO | Uploaded 36GiB to GPU in 2.54109s (14.1672GiB/s) | 20260508 06:58:51.672 | INFO | Downloaded 34GiB to GPU in 2.42594s (14.0152GiB/s) | 20260508 06:58:56.812 | INFO | Analyzing chunk frames = [6137344,6137856) | 20260508 06:58:56.812 | INFO | MemInfo: { | 20260508 06:58:56.812 | INFO | "gpu" : " 17 GB", | 20260508 06:58:56.812 | INFO | "huge" : "", | 20260508 06:58:56.812 | INFO | "huge_shared" : " 29 GB", | 20260508 06:58:56.812 | INFO | "malloc" : " 14 GB", | 20260508 06:58:56.812 | INFO | "pinned" : " 29 GB" | 20260508 06:58:56.812 | INFO | } | 20260508 06:58:56.812 | INFO | Perf report for the last 5.13998 seconds: | 20260508 06:58:56.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:58:56.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 06:58:56.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:58:56.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:58:56.812 | INFO | Uploaded 36GiB to GPU in 2.56392s (14.041GiB/s) | 20260508 06:58:56.812 | INFO | Downloaded 34GiB to GPU in 2.43073s (13.9876GiB/s) | 20260508 06:59:01.954 | INFO | Analyzing chunk frames = [6137856,6138368) | 20260508 06:59:01.954 | INFO | MemInfo: { | 20260508 06:59:01.954 | INFO | "gpu" : " 17 GB", | 20260508 06:59:01.954 | INFO | "huge" : "", | 20260508 06:59:01.954 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:01.954 | INFO | "malloc" : " 14 GB", | 20260508 06:59:01.954 | INFO | "pinned" : " 29 GB" | 20260508 06:59:01.954 | INFO | } | 20260508 06:59:01.954 | INFO | Perf report for the last 5.1426 seconds: | 20260508 06:59:01.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:01.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.955 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 06:59:01.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.955 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 06:59:01.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:01.955 | INFO | Uploaded 36GiB to GPU in 2.43437s (14.7882GiB/s) | 20260508 06:59:01.955 | INFO | Downloaded 34GiB to GPU in 2.52777s (13.4506GiB/s) | 20260508 06:59:07.099 | INFO | Analyzing chunk frames = [6138368,6138880) | 20260508 06:59:07.099 | INFO | MemInfo: { | 20260508 06:59:07.099 | INFO | "gpu" : " 17 GB", | 20260508 06:59:07.099 | INFO | "huge" : "", | 20260508 06:59:07.099 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:07.099 | INFO | "malloc" : " 14 GB", | 20260508 06:59:07.099 | INFO | "pinned" : " 29 GB" | 20260508 06:59:07.099 | INFO | } | 20260508 06:59:07.100 | INFO | Perf report for the last 5.14523 seconds: | 20260508 06:59:07.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:07.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:59:07.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:59:07.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:07.100 | INFO | Uploaded 36GiB to GPU in 2.45974s (14.6357GiB/s) | 20260508 06:59:07.100 | INFO | Downloaded 34GiB to GPU in 2.42581s (14.016GiB/s) | 20260508 06:59:12.238 | INFO | Analyzing chunk frames = [6138880,6139392) | 20260508 06:59:12.238 | INFO | MemInfo: { | 20260508 06:59:12.238 | INFO | "gpu" : " 17 GB", | 20260508 06:59:12.238 | INFO | "huge" : "", | 20260508 06:59:12.238 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:12.238 | INFO | "malloc" : " 14 GB", | 20260508 06:59:12.238 | INFO | "pinned" : " 29 GB" | 20260508 06:59:12.238 | INFO | } | 20260508 06:59:12.238 | INFO | Perf report for the last 5.13875 seconds: | 20260508 06:59:12.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:12.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.239 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 06:59:12.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.239 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 06:59:12.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:12.239 | INFO | Uploaded 36GiB to GPU in 2.4332s (14.7953GiB/s) | 20260508 06:59:12.239 | INFO | Downloaded 34GiB to GPU in 2.42935s (13.9955GiB/s) | 20260508 06:59:17.369 | INFO | Analyzing chunk frames = [6139392,6139904) | 20260508 06:59:17.369 | INFO | MemInfo: { | 20260508 06:59:17.369 | INFO | "gpu" : " 17 GB", | 20260508 06:59:17.369 | INFO | "huge" : "", | 20260508 06:59:17.369 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:17.369 | INFO | "malloc" : " 14 GB", | 20260508 06:59:17.369 | INFO | "pinned" : " 29 GB" | 20260508 06:59:17.369 | INFO | } | 20260508 06:59:17.369 | INFO | Perf report for the last 5.13093 seconds: | 20260508 06:59:17.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:17.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.369 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 06:59:17.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.370 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 06:59:17.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:17.370 | INFO | Uploaded 36GiB to GPU in 2.48717s (14.4743GiB/s) | 20260508 06:59:17.370 | INFO | Downloaded 34GiB to GPU in 2.46066s (13.8174GiB/s) | 20260508 06:59:17.483 | INFO | Starting a round of Baz IO | 20260508 06:59:20.895 | INFO | Populating RTMetrics report | 20260508 06:59:20.896 | INFO | RTMetrics report populated | 20260508 06:59:20.896 | INFO | Opening tmp RTMetrics json file | 20260508 06:59:20.896 | INFO | Serializing RTMetrics to json | 20260508 06:59:20.897 | INFO | Serialization done, swapping tmp file now... | 20260508 06:59:20.898 | INFO | RTMetrics done | 20260508 06:59:20.961 | INFO | Finished a round of Baz IO: Wrote 1.537 GiB in 3.478 seconds | 20260508 06:59:20.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.62% / 0.00% / 59.38% | 20260508 06:59:20.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.367777% / 7.94729e-06% / 4.76837e-05% | 20260508 06:59:20.962 | INFO | Baz IO overall: | 20260508 06:59:20.962 | INFO | BytesWritten : 2715396660252 | 20260508 06:59:20.962 | INFO | NumEvents : 1308705229683 | 20260508 06:59:22.516 | INFO | Analyzing chunk frames = [6139904,6140416) | 20260508 06:59:22.516 | INFO | MemInfo: { | 20260508 06:59:22.516 | INFO | "gpu" : " 17 GB", | 20260508 06:59:22.517 | INFO | "huge" : "", | 20260508 06:59:22.517 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:22.517 | INFO | "malloc" : " 14 GB", | 20260508 06:59:22.517 | INFO | "pinned" : " 29 GB" | 20260508 06:59:22.517 | INFO | } | 20260508 06:59:22.517 | INFO | Perf report for the last 5.14734 seconds: | 20260508 06:59:22.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:22.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:22.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:22.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:22.517 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 06:59:22.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:22.517 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 06:59:22.517 | INFO | PRE_HQ : 66.7%, 1.8, 61.5ms, 0 | 20260508 06:59:22.517 | INFO | BAZWRITER : 66.8%, 2.8, 95.6ms, 0 | 20260508 06:59:22.517 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 06:59:22.517 | INFO | Uploaded 36GiB to GPU in 2.52933s (14.233GiB/s) | 20260508 06:59:22.517 | INFO | Downloaded 38GiB to GPU in 2.70495s (14.0483GiB/s) | 20260508 06:59:27.652 | INFO | Analyzing chunk frames = [6140416,6140928) | 20260508 06:59:27.652 | INFO | MemInfo: { | 20260508 06:59:27.652 | INFO | "gpu" : " 17 GB", | 20260508 06:59:27.652 | INFO | "huge" : "", | 20260508 06:59:27.652 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:27.652 | INFO | "malloc" : " 14 GB", | 20260508 06:59:27.652 | INFO | "pinned" : " 29 GB" | 20260508 06:59:27.653 | INFO | } | 20260508 06:59:27.653 | INFO | Perf report for the last 5.13603 seconds: | 20260508 06:59:27.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:27.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 06:59:27.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 06:59:27.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:27.653 | INFO | Uploaded 36GiB to GPU in 2.50992s (14.3431GiB/s) | 20260508 06:59:27.653 | INFO | Downloaded 34GiB to GPU in 2.45364s (13.857GiB/s) | 20260508 06:59:32.792 | INFO | Analyzing chunk frames = [6140928,6141440) | 20260508 06:59:32.792 | INFO | MemInfo: { | 20260508 06:59:32.792 | INFO | "gpu" : " 17 GB", | 20260508 06:59:32.792 | INFO | "huge" : "", | 20260508 06:59:32.792 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:32.792 | INFO | "malloc" : " 14 GB", | 20260508 06:59:32.792 | INFO | "pinned" : " 29 GB" | 20260508 06:59:32.793 | INFO | } | 20260508 06:59:32.793 | INFO | Perf report for the last 5.13977 seconds: | 20260508 06:59:32.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:32.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 06:59:32.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 06:59:32.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:32.793 | INFO | Uploaded 36GiB to GPU in 2.56163s (14.0536GiB/s) | 20260508 06:59:32.793 | INFO | Downloaded 34GiB to GPU in 2.43046s (13.9891GiB/s) | 20260508 06:59:37.927 | INFO | Analyzing chunk frames = [6141440,6141952) | 20260508 06:59:37.927 | INFO | MemInfo: { | 20260508 06:59:37.927 | INFO | "gpu" : " 17 GB", | 20260508 06:59:37.927 | INFO | "huge" : "", | 20260508 06:59:37.927 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:37.927 | INFO | "malloc" : " 14 GB", | 20260508 06:59:37.927 | INFO | "pinned" : " 29 GB" | 20260508 06:59:37.927 | INFO | } | 20260508 06:59:37.927 | INFO | Perf report for the last 5.13448 seconds: | 20260508 06:59:37.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:37.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | ANALYSIS : 66.7%, 3.0, 103.1ms, 0 | 20260508 06:59:37.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 06:59:37.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:37.927 | INFO | Uploaded 36GiB to GPU in 2.5431s (14.156GiB/s) | 20260508 06:59:37.927 | INFO | Downloaded 34GiB to GPU in 2.40325s (14.1475GiB/s) | 20260508 06:59:43.066 | INFO | Analyzing chunk frames = [6141952,6142464) | 20260508 06:59:43.066 | INFO | MemInfo: { | 20260508 06:59:43.066 | INFO | "gpu" : " 17 GB", | 20260508 06:59:43.066 | INFO | "huge" : "", | 20260508 06:59:43.066 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:43.067 | INFO | "malloc" : " 14 GB", | 20260508 06:59:43.067 | INFO | "pinned" : " 29 GB" | 20260508 06:59:43.067 | INFO | } | 20260508 06:59:43.067 | INFO | Perf report for the last 5.1396 seconds: | 20260508 06:59:43.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:43.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 06:59:43.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 06:59:43.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:43.067 | INFO | Uploaded 36GiB to GPU in 2.30184s (15.6397GiB/s) | 20260508 06:59:43.067 | INFO | Downloaded 34GiB to GPU in 2.4776s (13.723GiB/s) | 20260508 06:59:48.207 | INFO | Analyzing chunk frames = [6142464,6142976) | 20260508 06:59:48.207 | INFO | MemInfo: { | 20260508 06:59:48.207 | INFO | "gpu" : " 17 GB", | 20260508 06:59:48.207 | INFO | "huge" : "", | 20260508 06:59:48.207 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:48.207 | INFO | "malloc" : " 14 GB", | 20260508 06:59:48.207 | INFO | "pinned" : " 29 GB" | 20260508 06:59:48.207 | INFO | } | 20260508 06:59:48.207 | INFO | Perf report for the last 5.14029 seconds: | 20260508 06:59:48.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:48.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.207 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 06:59:48.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.207 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 06:59:48.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:48.208 | INFO | Uploaded 36GiB to GPU in 2.45469s (14.6658GiB/s) | 20260508 06:59:48.208 | INFO | Downloaded 34GiB to GPU in 2.43583s (13.9583GiB/s) | 20260508 06:59:53.351 | INFO | Analyzing chunk frames = [6142976,6143488) | 20260508 06:59:53.351 | INFO | MemInfo: { | 20260508 06:59:53.351 | INFO | "gpu" : " 17 GB", | 20260508 06:59:53.351 | INFO | "huge" : "", | 20260508 06:59:53.351 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:53.351 | INFO | "malloc" : " 14 GB", | 20260508 06:59:53.351 | INFO | "pinned" : " 29 GB" | 20260508 06:59:53.351 | INFO | } | 20260508 06:59:53.351 | INFO | Perf report for the last 5.14437 seconds: | 20260508 06:59:53.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:53.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.352 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 06:59:53.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.352 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 06:59:53.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:53.352 | INFO | Uploaded 36GiB to GPU in 2.47639s (14.5373GiB/s) | 20260508 06:59:53.352 | INFO | Downloaded 34GiB to GPU in 2.43553s (13.96GiB/s) | 20260508 06:59:58.487 | INFO | Analyzing chunk frames = [6143488,6144000) | 20260508 06:59:58.487 | INFO | MemInfo: { | 20260508 06:59:58.487 | INFO | "gpu" : " 17 GB", | 20260508 06:59:58.487 | INFO | "huge" : "", | 20260508 06:59:58.487 | INFO | "huge_shared" : " 29 GB", | 20260508 06:59:58.487 | INFO | "malloc" : " 14 GB", | 20260508 06:59:58.487 | INFO | "pinned" : " 29 GB" | 20260508 06:59:58.487 | INFO | } | 20260508 06:59:58.488 | INFO | Perf report for the last 5.13613 seconds: | 20260508 06:59:58.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 06:59:58.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 06:59:58.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 06:59:58.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 06:59:58.488 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 06:59:58.488 | INFO | Total mem usage: 16.7 GiB | 20260508 06:59:58.488 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 06:59:58.488 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 06:59:58.488 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 06:59:58.488 | INFO | High Water Breakdown: | 20260508 06:59:58.488 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 06:59:58.488 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 06:59:58.488 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 06:59:58.488 | INFO | ------------------------------------------ | 20260508 06:59:58.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:59:58.488 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 06:59:58.488 | INFO | Total mem usage: 26.5 GiB | 20260508 06:59:58.488 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 06:59:58.488 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 06:59:58.488 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 06:59:58.488 | INFO | High Water Breakdown: | 20260508 06:59:58.488 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:59:58.488 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:59:58.488 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:59:58.488 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:59:58.488 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:59:58.488 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:59:58.488 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 06:59:58.488 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 06:59:58.488 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:59:58.488 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 06:59:58.488 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 06:59:58.488 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 06:59:58.488 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 06:59:58.488 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 06:59:58.488 | INFO | ------------------------------------------ | 20260508 06:59:58.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:59:58.488 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 06:59:58.488 | INFO | Total mem usage: 27.8 GiB | 20260508 06:59:58.488 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 06:59:58.489 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 06:59:58.489 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 06:59:58.489 | INFO | High Water Breakdown: | 20260508 06:59:58.488 | INFO | Uploaded 36GiB to GPU in 2.43972s (14.7558GiB/s) | 20260508 06:59:58.489 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 06:59:58.489 | INFO | ------------------------------------------ | 20260508 06:59:58.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:59:58.489 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 06:59:58.489 | INFO | Total mem usage: 15.7 GiB | 20260508 06:59:58.489 | INFO | Downloaded 34GiB to GPU in 2.44464s (13.908GiB/s) | 20260508 06:59:58.489 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 06:59:58.489 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 06:59:58.489 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 06:59:58.489 | INFO | High Water Breakdown: | 20260508 06:59:58.489 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 06:59:58.489 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 06:59:58.489 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 06:59:58.489 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 06:59:58.489 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 06:59:58.489 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 06:59:58.489 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 06:59:58.489 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 06:59:58.489 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 06:59:58.489 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 06:59:58.489 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 06:59:58.489 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 06:59:58.489 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 06:59:58.489 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 06:59:58.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 06:59:58.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 06:59:58.489 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 06:59:58.489 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 06:59:58.489 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 06:59:58.489 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 06:59:58.489 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 06:59:58.489 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 06:59:58.489 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 06:59:58.489 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 06:59:58.489 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 06:59:58.489 | INFO | ------------------------------------------ | 20260508 06:59:58.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 06:59:58.490 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 06:59:58.490 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:59:58.490 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:59:58.490 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.26e-01 | 20260508 06:59:58.490 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.23e-02 / 5.53e-03 | 20260508 06:59:58.490 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.48e+00, 5.82e-01 / 1.75e-01 | 20260508 06:59:58.490 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.23e-01 | 20260508 06:59:58.490 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.53e+01, 3.73e+00 / 1.13e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.81e+00 / 1.35e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.37e+00 / 2.58e+00, 4.09e-01 / 2.57e-01 | 20260508 06:59:58.490 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.20e+00, 2.53e-01 / 1.28e-01 | 20260508 06:59:58.490 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.12e-01 / 3.47e-01, 4.93e-02 / 2.27e-02 | 20260508 06:59:58.490 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 06:59:58.490 | INFO | PerfReport: Gathered 1536 samples. | 20260508 06:59:58.490 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 06:59:58.490 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 24.46%, Upload -- 2.59e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 2.22%, LockUpload -- 2.35e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 2.66%, LockDownload -- 2.82e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 06:59:58.490 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260508 06:59:58.600 | INFO | Starting a round of Baz IO | 20260508 07:00:02.016 | INFO | Populating RTMetrics report | 20260508 07:00:02.016 | INFO | RTMetrics report populated | 20260508 07:00:02.016 | INFO | Opening tmp RTMetrics json file | 20260508 07:00:02.017 | INFO | Serializing RTMetrics to json | 20260508 07:00:02.018 | INFO | Serialization done, swapping tmp file now... | 20260508 07:00:02.019 | INFO | RTMetrics done | 20260508 07:00:02.081 | INFO | Finished a round of Baz IO: Wrote 1.537 GiB in 3.48 seconds | 20260508 07:00:02.081 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.62% / 0.00% / 59.38% | 20260508 07:00:02.081 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.366998% / 0% / 4.37101e-05% | 20260508 07:00:02.081 | INFO | Baz IO overall: | 20260508 07:00:02.081 | INFO | BytesWritten : 2717046789833 | 20260508 07:00:02.081 | INFO | NumEvents : 1309489213022 | 20260508 07:00:03.624 | INFO | Analyzing chunk frames = [6144000,6144512) | 20260508 07:00:03.624 | INFO | MemInfo: { | 20260508 07:00:03.624 | INFO | "gpu" : " 17 GB", | 20260508 07:00:03.624 | INFO | "huge" : "", | 20260508 07:00:03.624 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:03.624 | INFO | "malloc" : " 14 GB", | 20260508 07:00:03.624 | INFO | "pinned" : " 29 GB" | 20260508 07:00:03.624 | INFO | } | 20260508 07:00:03.625 | INFO | Perf report for the last 5.13699 seconds: | 20260508 07:00:03.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:03.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:03.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:03.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:03.625 | INFO | ANALYSIS : 68.3%, 3.0, 105.9ms, 0 | 20260508 07:00:03.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:03.625 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 07:00:03.625 | INFO | PRE_HQ : 67.0%, 1.8, 61.9ms, 0 | 20260508 07:00:03.625 | INFO | BAZWRITER : 67.1%, 2.8, 96.3ms, 0 | 20260508 07:00:03.625 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 07:00:03.625 | INFO | Uploaded 36GiB to GPU in 2.51069s (14.3387GiB/s) | 20260508 07:00:03.625 | INFO | Downloaded 38GiB to GPU in 2.70223s (14.0625GiB/s) | 20260508 07:00:08.763 | INFO | Analyzing chunk frames = [6144512,6145024) | 20260508 07:00:08.763 | INFO | MemInfo: { | 20260508 07:00:08.763 | INFO | "gpu" : " 17 GB", | 20260508 07:00:08.763 | INFO | "huge" : "", | 20260508 07:00:08.763 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:08.763 | INFO | "malloc" : " 14 GB", | 20260508 07:00:08.763 | INFO | "pinned" : " 29 GB" | 20260508 07:00:08.763 | INFO | } | 20260508 07:00:08.763 | INFO | Perf report for the last 5.13829 seconds: | 20260508 07:00:08.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:08.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:00:08.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:00:08.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:08.763 | INFO | Uploaded 36GiB to GPU in 2.46615s (14.5976GiB/s) | 20260508 07:00:08.763 | INFO | Downloaded 34GiB to GPU in 2.43726s (13.9501GiB/s) | 20260508 07:00:13.913 | INFO | Analyzing chunk frames = [6145024,6145536) | 20260508 07:00:13.913 | INFO | MemInfo: { | 20260508 07:00:13.913 | INFO | "gpu" : " 17 GB", | 20260508 07:00:13.913 | INFO | "huge" : "", | 20260508 07:00:13.913 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:13.913 | INFO | "malloc" : " 14 GB", | 20260508 07:00:13.913 | INFO | "pinned" : " 29 GB" | 20260508 07:00:13.913 | INFO | } | 20260508 07:00:13.913 | INFO | Perf report for the last 5.1503 seconds: | 20260508 07:00:13.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:13.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:00:13.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:00:13.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:13.913 | INFO | Uploaded 36GiB to GPU in 2.53387s (14.2075GiB/s) | 20260508 07:00:13.913 | INFO | Downloaded 34GiB to GPU in 2.4186s (14.0577GiB/s) | 20260508 07:00:19.046 | INFO | Analyzing chunk frames = [6145536,6146048) | 20260508 07:00:19.046 | INFO | MemInfo: { | 20260508 07:00:19.046 | INFO | "gpu" : " 17 GB", | 20260508 07:00:19.046 | INFO | "huge" : "", | 20260508 07:00:19.046 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:19.046 | INFO | "malloc" : " 14 GB", | 20260508 07:00:19.046 | INFO | "pinned" : " 29 GB" | 20260508 07:00:19.046 | INFO | } | 20260508 07:00:19.046 | INFO | Perf report for the last 5.13324 seconds: | 20260508 07:00:19.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:19.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.047 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 07:00:19.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.047 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:00:19.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:19.047 | INFO | Uploaded 36GiB to GPU in 2.56631s (14.0279GiB/s) | 20260508 07:00:19.047 | INFO | Downloaded 34GiB to GPU in 2.43939s (13.9379GiB/s) | 20260508 07:00:24.184 | INFO | Analyzing chunk frames = [6146048,6146560) | 20260508 07:00:24.184 | INFO | MemInfo: { | 20260508 07:00:24.184 | INFO | "gpu" : " 17 GB", | 20260508 07:00:24.184 | INFO | "huge" : "", | 20260508 07:00:24.184 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:24.184 | INFO | "malloc" : " 14 GB", | 20260508 07:00:24.184 | INFO | "pinned" : " 29 GB" | 20260508 07:00:24.184 | INFO | } | 20260508 07:00:24.184 | INFO | Perf report for the last 5.13797 seconds: | 20260508 07:00:24.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:24.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.184 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 07:00:24.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.185 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:00:24.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:24.185 | INFO | Uploaded 36GiB to GPU in 2.40608s (14.9621GiB/s) | 20260508 07:00:24.185 | INFO | Downloaded 34GiB to GPU in 2.5166s (13.5103GiB/s) | 20260508 07:00:29.324 | INFO | Analyzing chunk frames = [6146560,6147072) | 20260508 07:00:29.324 | INFO | MemInfo: { | 20260508 07:00:29.324 | INFO | "gpu" : " 17 GB", | 20260508 07:00:29.324 | INFO | "huge" : "", | 20260508 07:00:29.324 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:29.324 | INFO | "malloc" : " 14 GB", | 20260508 07:00:29.324 | INFO | "pinned" : " 29 GB" | 20260508 07:00:29.324 | INFO | } | 20260508 07:00:29.324 | INFO | Perf report for the last 5.13985 seconds: | 20260508 07:00:29.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:29.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.324 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:00:29.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.324 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:00:29.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:29.325 | INFO | Uploaded 36GiB to GPU in 2.38436s (15.0984GiB/s) | 20260508 07:00:29.325 | INFO | Downloaded 34GiB to GPU in 2.44325s (13.9159GiB/s) | 20260508 07:00:34.467 | INFO | Analyzing chunk frames = [6147072,6147584) | 20260508 07:00:34.467 | INFO | MemInfo: { | 20260508 07:00:34.467 | INFO | "gpu" : " 17 GB", | 20260508 07:00:34.467 | INFO | "huge" : "", | 20260508 07:00:34.467 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:34.467 | INFO | "malloc" : " 14 GB", | 20260508 07:00:34.467 | INFO | "pinned" : " 29 GB" | 20260508 07:00:34.467 | INFO | } | 20260508 07:00:34.467 | INFO | Perf report for the last 5.14308 seconds: | 20260508 07:00:34.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:34.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.467 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 07:00:34.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.468 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:00:34.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:34.468 | INFO | Uploaded 36GiB to GPU in 2.47075s (14.5704GiB/s) | 20260508 07:00:34.468 | INFO | Downloaded 34GiB to GPU in 2.43587s (13.958GiB/s) | 20260508 07:00:39.602 | INFO | Analyzing chunk frames = [6147584,6148096) | 20260508 07:00:39.602 | INFO | MemInfo: { | 20260508 07:00:39.602 | INFO | "gpu" : " 17 GB", | 20260508 07:00:39.602 | INFO | "huge" : "", | 20260508 07:00:39.603 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:39.603 | INFO | "malloc" : " 14 GB", | 20260508 07:00:39.603 | INFO | "pinned" : " 29 GB" | 20260508 07:00:39.603 | INFO | } | 20260508 07:00:39.603 | INFO | Perf report for the last 5.13545 seconds: | 20260508 07:00:39.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:39.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 07:00:39.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 07:00:39.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:39.603 | INFO | Uploaded 36GiB to GPU in 2.46216s (14.6213GiB/s) | 20260508 07:00:39.604 | INFO | Downloaded 34GiB to GPU in 2.45444s (13.8524GiB/s) | 20260508 07:00:39.716 | INFO | Starting a round of Baz IO | 20260508 07:00:43.160 | INFO | Populating RTMetrics report | 20260508 07:00:43.161 | INFO | RTMetrics report populated | 20260508 07:00:43.161 | INFO | Opening tmp RTMetrics json file | 20260508 07:00:43.161 | INFO | Serializing RTMetrics to json | 20260508 07:00:43.163 | INFO | Serialization done, swapping tmp file now... | 20260508 07:00:43.163 | INFO | RTMetrics done | 20260508 07:00:43.223 | INFO | Finished a round of Baz IO: Wrote 1.537 GiB in 3.506 seconds | 20260508 07:00:43.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.62% / 0.00% / 59.38% | 20260508 07:00:43.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.367355% / 1.19209e-05% / 4.76837e-05% | 20260508 07:00:43.223 | INFO | Baz IO overall: | 20260508 07:00:43.223 | INFO | BytesWritten : 2718697206287 | 20260508 07:00:43.223 | INFO | NumEvents : 1310273673181 | 20260508 07:00:44.739 | INFO | Analyzing chunk frames = [6148096,6148608) | 20260508 07:00:44.739 | INFO | MemInfo: { | 20260508 07:00:44.739 | INFO | "gpu" : " 17 GB", | 20260508 07:00:44.740 | INFO | "huge" : "", | 20260508 07:00:44.740 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:44.740 | INFO | "malloc" : " 14 GB", | 20260508 07:00:44.740 | INFO | "pinned" : " 29 GB" | 20260508 07:00:44.740 | INFO | } | 20260508 07:00:44.740 | INFO | Perf report for the last 5.13682 seconds: | 20260508 07:00:44.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:44.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:44.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:44.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:44.740 | INFO | ANALYSIS : 68.9%, 2.9, 105.2ms, 0 | 20260508 07:00:44.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:44.740 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 07:00:44.740 | INFO | PRE_HQ : 67.7%, 1.8, 62.3ms, 0 | 20260508 07:00:44.740 | INFO | BAZWRITER : 67.4%, 3.1, 107.5ms, 0 | 20260508 07:00:44.740 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 07:00:44.740 | INFO | Uploaded 36GiB to GPU in 2.54654s (14.1368GiB/s) | 20260508 07:00:44.740 | INFO | Downloaded 38GiB to GPU in 2.67544s (14.2032GiB/s) | 20260508 07:00:49.878 | INFO | Analyzing chunk frames = [6148608,6149120) | 20260508 07:00:49.878 | INFO | MemInfo: { | 20260508 07:00:49.878 | INFO | "gpu" : " 17 GB", | 20260508 07:00:49.878 | INFO | "huge" : "", | 20260508 07:00:49.878 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:49.878 | INFO | "malloc" : " 14 GB", | 20260508 07:00:49.878 | INFO | "pinned" : " 29 GB" | 20260508 07:00:49.878 | INFO | } | 20260508 07:00:49.878 | INFO | Perf report for the last 5.13846 seconds: | 20260508 07:00:49.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:49.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.878 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:00:49.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.878 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:00:49.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:49.879 | INFO | Uploaded 36GiB to GPU in 2.49606s (14.4227GiB/s) | 20260508 07:00:49.879 | INFO | Downloaded 34GiB to GPU in 2.44103s (13.9285GiB/s) | 20260508 07:00:55.024 | INFO | Analyzing chunk frames = [6149120,6149632) | 20260508 07:00:55.024 | INFO | MemInfo: { | 20260508 07:00:55.024 | INFO | "gpu" : " 17 GB", | 20260508 07:00:55.024 | INFO | "huge" : "", | 20260508 07:00:55.024 | INFO | "huge_shared" : " 29 GB", | 20260508 07:00:55.024 | INFO | "malloc" : " 14 GB", | 20260508 07:00:55.024 | INFO | "pinned" : " 29 GB" | 20260508 07:00:55.024 | INFO | } | 20260508 07:00:55.024 | INFO | Perf report for the last 5.14622 seconds: | 20260508 07:00:55.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:00:55.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 07:00:55.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:00:55.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:00:55.025 | INFO | Uploaded 36GiB to GPU in 2.54788s (14.1294GiB/s) | 20260508 07:00:55.025 | INFO | Downloaded 34GiB to GPU in 2.41489s (14.0793GiB/s) | 20260508 07:01:00.163 | INFO | Analyzing chunk frames = [6149632,6150144) | 20260508 07:01:00.163 | INFO | MemInfo: { | 20260508 07:01:00.163 | INFO | "gpu" : " 17 GB", | 20260508 07:01:00.163 | INFO | "huge" : "", | 20260508 07:01:00.163 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:00.163 | INFO | "malloc" : " 14 GB", | 20260508 07:01:00.164 | INFO | "pinned" : " 29 GB" | 20260508 07:01:00.164 | INFO | } | 20260508 07:01:00.164 | INFO | Perf report for the last 5.13935 seconds: | 20260508 07:01:00.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:00.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 07:01:00.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:01:00.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:00.164 | INFO | Uploaded 36GiB to GPU in 2.56787s (14.0194GiB/s) | 20260508 07:01:00.164 | INFO | Downloaded 34GiB to GPU in 2.42631s (14.013GiB/s) | 20260508 07:01:05.299 | INFO | Analyzing chunk frames = [6150144,6150656) | 20260508 07:01:05.299 | INFO | MemInfo: { | 20260508 07:01:05.299 | INFO | "gpu" : " 17 GB", | 20260508 07:01:05.299 | INFO | "huge" : "", | 20260508 07:01:05.299 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:05.299 | INFO | "malloc" : " 14 GB", | 20260508 07:01:05.299 | INFO | "pinned" : " 29 GB" | 20260508 07:01:05.299 | INFO | } | 20260508 07:01:05.299 | INFO | Perf report for the last 5.13505 seconds: | 20260508 07:01:05.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:05.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:01:05.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:01:05.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:05.299 | INFO | Uploaded 36GiB to GPU in 2.28488s (15.7558GiB/s) | 20260508 07:01:05.299 | INFO | Downloaded 34GiB to GPU in 2.4914s (13.647GiB/s) | 20260508 07:01:10.433 | INFO | Analyzing chunk frames = [6150656,6151168) | 20260508 07:01:10.433 | INFO | MemInfo: { | 20260508 07:01:10.433 | INFO | "gpu" : " 17 GB", | 20260508 07:01:10.433 | INFO | "huge" : "", | 20260508 07:01:10.433 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:10.433 | INFO | "malloc" : " 14 GB", | 20260508 07:01:10.433 | INFO | "pinned" : " 29 GB" | 20260508 07:01:10.433 | INFO | } | 20260508 07:01:10.433 | INFO | Perf report for the last 5.1344 seconds: | 20260508 07:01:10.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:10.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.433 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 07:01:10.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.434 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:01:10.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:10.434 | INFO | Uploaded 36GiB to GPU in 2.45966s (14.6362GiB/s) | 20260508 07:01:10.434 | INFO | Downloaded 34GiB to GPU in 2.41479s (14.0799GiB/s) | 20260508 07:01:15.575 | INFO | Analyzing chunk frames = [6151168,6151680) | 20260508 07:01:15.575 | INFO | MemInfo: { | 20260508 07:01:15.575 | INFO | "gpu" : " 17 GB", | 20260508 07:01:15.575 | INFO | "huge" : "", | 20260508 07:01:15.575 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:15.575 | INFO | "malloc" : " 14 GB", | 20260508 07:01:15.575 | INFO | "pinned" : " 29 GB" | 20260508 07:01:15.575 | INFO | } | 20260508 07:01:15.575 | INFO | Perf report for the last 5.14202 seconds: | 20260508 07:01:15.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:15.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.575 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:01:15.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.575 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 07:01:15.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:15.576 | INFO | Uploaded 36GiB to GPU in 2.50059s (14.3966GiB/s) | 20260508 07:01:15.576 | INFO | Downloaded 34GiB to GPU in 2.43397s (13.9689GiB/s) | 20260508 07:01:20.716 | INFO | Analyzing chunk frames = [6151680,6152192) | 20260508 07:01:20.716 | INFO | MemInfo: { | 20260508 07:01:20.716 | INFO | "gpu" : " 17 GB", | 20260508 07:01:20.716 | INFO | "huge" : "", | 20260508 07:01:20.716 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:20.716 | INFO | "malloc" : " 14 GB", | 20260508 07:01:20.716 | INFO | "pinned" : " 29 GB" | 20260508 07:01:20.716 | INFO | } | 20260508 07:01:20.716 | INFO | Perf report for the last 5.14076 seconds: | 20260508 07:01:20.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:20.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:01:20.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:01:20.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:20.716 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:01:20.716 | INFO | Total mem usage: 16.7 GiB | 20260508 07:01:20.716 | INFO | Uploaded 36GiB to GPU in 2.48828s (14.4678GiB/s) | 20260508 07:01:20.716 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:01:20.716 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.4%) | 20260508 07:01:20.716 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:01:20.716 | INFO | High Water Breakdown: | 20260508 07:01:20.716 | INFO | Downloaded 34GiB to GPU in 2.43947s (13.9374GiB/s) | 20260508 07:01:20.717 | INFO | 60.4% : 9.13 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:01:20.717 | INFO | 39.4% : 5.96 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:01:20.717 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:01:20.717 | INFO | ------------------------------------------ | 20260508 07:01:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:01:20.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:01:20.717 | INFO | Total mem usage: 26.5 GiB | 20260508 07:01:20.717 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:01:20.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:01:20.717 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:01:20.717 | INFO | High Water Breakdown: | 20260508 07:01:20.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:01:20.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:01:20.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:01:20.717 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:01:20.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:01:20.717 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:01:20.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:01:20.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:01:20.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:01:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:01:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:01:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:01:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:01:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:01:20.717 | INFO | ------------------------------------------ | 20260508 07:01:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:01:20.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:01:20.717 | INFO | Total mem usage: 27.8 GiB | 20260508 07:01:20.717 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:01:20.717 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:01:20.717 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:01:20.717 | INFO | High Water Breakdown: | 20260508 07:01:20.717 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:01:20.717 | INFO | ------------------------------------------ | 20260508 07:01:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:01:20.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:01:20.717 | INFO | Total mem usage: 15.7 GiB | 20260508 07:01:20.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:01:20.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:01:20.717 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:01:20.717 | INFO | High Water Breakdown: | 20260508 07:01:20.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:01:20.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:01:20.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:01:20.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:01:20.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:01:20.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:01:20.717 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:01:20.717 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:01:20.717 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:01:20.717 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:01:20.717 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:01:20.717 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:01:20.717 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:01:20.717 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:01:20.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:01:20.717 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:01:20.717 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:01:20.717 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:01:20.717 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:01:20.717 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:01:20.717 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:01:20.717 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:01:20.717 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:01:20.717 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:01:20.717 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:01:20.717 | INFO | ------------------------------------------ | 20260508 07:01:20.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:01:20.717 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:01:20.717 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:01:20.717 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:01:20.717 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 7.11e-01 | 20260508 07:01:20.717 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.08e-01, 1.09e-02 / 6.86e-03 | 20260508 07:01:20.717 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.44e+00, 5.97e-01 / 2.31e-01 | 20260508 07:01:20.717 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.13e-01 | 20260508 07:01:20.717 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.54e+01, 3.86e+00 / 9.44e-01 | 20260508 07:01:20.717 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.82e+00 / 1.09e+00 | 20260508 07:01:20.717 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.60e+00, 4.07e-01 / 1.63e-01 | 20260508 07:01:20.718 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.22e+00, 2.71e-01 / 9.75e-02 | 20260508 07:01:20.718 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.56e-01, 4.93e-02 / 2.63e-02 | 20260508 07:01:20.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:01:20.718 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:01:20.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:01:20.718 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 24.39%, Upload -- 2.58e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 24.32%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 30.89%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 2.26%, LockUpload -- 2.39e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 2.62%, LockDownload -- 2.77e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 15.51%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:01:20.718 | INFO | PerfReport: 0.01%, Misc -- 1.57e-02 / 0.00e+00, 4.03e-02 / 0.00e+00 | 20260508 07:01:20.813 | INFO | Starting a round of Baz IO | 20260508 07:01:24.247 | INFO | Populating RTMetrics report | 20260508 07:01:24.248 | INFO | RTMetrics report populated | 20260508 07:01:24.248 | INFO | Opening tmp RTMetrics json file | 20260508 07:01:24.259 | INFO | Serializing RTMetrics to json | 20260508 07:01:24.261 | INFO | Serialization done, swapping tmp file now... | 20260508 07:01:24.261 | INFO | RTMetrics done | 20260508 07:01:24.317 | INFO | Finished a round of Baz IO: Wrote 1.536 GiB in 3.503 seconds | 20260508 07:01:24.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.00% / 59.38% | 20260508 07:01:24.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.365698% / 0% / 4.76837e-05% | 20260508 07:01:24.317 | INFO | Baz IO overall: | 20260508 07:01:24.317 | INFO | BytesWritten : 2720346864230 | 20260508 07:01:24.317 | INFO | NumEvents : 1311057708318 | 20260508 07:01:25.859 | INFO | Analyzing chunk frames = [6152192,6152704) | 20260508 07:01:25.859 | INFO | MemInfo: { | 20260508 07:01:25.859 | INFO | "gpu" : " 17 GB", | 20260508 07:01:25.859 | INFO | "huge" : "", | 20260508 07:01:25.859 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:25.859 | INFO | "malloc" : " 14 GB", | 20260508 07:01:25.859 | INFO | "pinned" : " 29 GB" | 20260508 07:01:25.859 | INFO | } | 20260508 07:01:25.859 | INFO | Perf report for the last 5.14309 seconds: | 20260508 07:01:25.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:25.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:25.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:25.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:25.859 | INFO | ANALYSIS : 67.8%, 3.0, 106.1ms, 0 | 20260508 07:01:25.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:25.859 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:01:25.859 | INFO | PRE_HQ : 65.3%, 1.6, 54.9ms, 0 | 20260508 07:01:25.859 | INFO | BAZWRITER : 67.5%, 2.6, 89.1ms, 0 | 20260508 07:01:25.859 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:01:25.860 | INFO | Uploaded 36GiB to GPU in 2.55164s (14.1086GiB/s) | 20260508 07:01:25.860 | INFO | Downloaded 38GiB to GPU in 2.67915s (14.1836GiB/s) | 20260508 07:01:31.001 | INFO | Analyzing chunk frames = [6152704,6153216) | 20260508 07:01:31.001 | INFO | MemInfo: { | 20260508 07:01:31.001 | INFO | "gpu" : " 17 GB", | 20260508 07:01:31.001 | INFO | "huge" : "", | 20260508 07:01:31.001 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:31.001 | INFO | "malloc" : " 14 GB", | 20260508 07:01:31.001 | INFO | "pinned" : " 29 GB" | 20260508 07:01:31.001 | INFO | } | 20260508 07:01:31.001 | INFO | Perf report for the last 5.14162 seconds: | 20260508 07:01:31.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:31.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:01:31.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:01:31.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:31.001 | INFO | Uploaded 36GiB to GPU in 2.49626s (14.4215GiB/s) | 20260508 07:01:31.001 | INFO | Downloaded 34GiB to GPU in 2.43554s (13.96GiB/s) | 20260508 07:01:36.133 | INFO | Analyzing chunk frames = [6153216,6153728) | 20260508 07:01:36.133 | INFO | MemInfo: { | 20260508 07:01:36.133 | INFO | "gpu" : " 17 GB", | 20260508 07:01:36.133 | INFO | "huge" : "", | 20260508 07:01:36.133 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:36.133 | INFO | "malloc" : " 14 GB", | 20260508 07:01:36.133 | INFO | "pinned" : " 29 GB" | 20260508 07:01:36.133 | INFO | } | 20260508 07:01:36.133 | INFO | Perf report for the last 5.1322 seconds: | 20260508 07:01:36.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:36.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | ANALYSIS : 66.8%, 3.0, 103.6ms, 0 | 20260508 07:01:36.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:01:36.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:36.133 | INFO | Uploaded 36GiB to GPU in 2.50285s (14.3836GiB/s) | 20260508 07:01:36.133 | INFO | Downloaded 34GiB to GPU in 2.42948s (13.9948GiB/s) | 20260508 07:01:41.278 | INFO | Analyzing chunk frames = [6153728,6154240) | 20260508 07:01:41.278 | INFO | MemInfo: { | 20260508 07:01:41.278 | INFO | "gpu" : " 17 GB", | 20260508 07:01:41.278 | INFO | "huge" : "", | 20260508 07:01:41.278 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:41.278 | INFO | "malloc" : " 14 GB", | 20260508 07:01:41.278 | INFO | "pinned" : " 29 GB" | 20260508 07:01:41.278 | INFO | } | 20260508 07:01:41.278 | INFO | Perf report for the last 5.14511 seconds: | 20260508 07:01:41.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:41.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.278 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:01:41.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.278 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:01:41.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:41.279 | INFO | Uploaded 36GiB to GPU in 2.58981s (13.9007GiB/s) | 20260508 07:01:41.279 | INFO | Downloaded 34GiB to GPU in 2.41321s (14.0891GiB/s) | 20260508 07:01:46.413 | INFO | Analyzing chunk frames = [6154240,6154752) | 20260508 07:01:46.413 | INFO | MemInfo: { | 20260508 07:01:46.413 | INFO | "gpu" : " 17 GB", | 20260508 07:01:46.413 | INFO | "huge" : "", | 20260508 07:01:46.413 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:46.413 | INFO | "malloc" : " 14 GB", | 20260508 07:01:46.413 | INFO | "pinned" : " 29 GB" | 20260508 07:01:46.413 | INFO | } | 20260508 07:01:46.413 | INFO | Perf report for the last 5.13514 seconds: | 20260508 07:01:46.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:46.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:01:46.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:01:46.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:46.413 | INFO | Uploaded 36GiB to GPU in 2.28505s (15.7546GiB/s) | 20260508 07:01:46.413 | INFO | Downloaded 34GiB to GPU in 2.5038s (13.5794GiB/s) | 20260508 07:01:51.549 | INFO | Analyzing chunk frames = [6154752,6155264) | 20260508 07:01:51.549 | INFO | MemInfo: { | 20260508 07:01:51.549 | INFO | "gpu" : " 17 GB", | 20260508 07:01:51.549 | INFO | "huge" : "", | 20260508 07:01:51.549 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:51.549 | INFO | "malloc" : " 14 GB", | 20260508 07:01:51.549 | INFO | "pinned" : " 29 GB" | 20260508 07:01:51.549 | INFO | } | 20260508 07:01:51.549 | INFO | Perf report for the last 5.13613 seconds: | 20260508 07:01:51.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:51.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.550 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 07:01:51.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.550 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:01:51.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:51.550 | INFO | Uploaded 36GiB to GPU in 2.39204s (15.0499GiB/s) | 20260508 07:01:51.550 | INFO | Downloaded 34GiB to GPU in 2.42575s (14.0163GiB/s) | 20260508 07:01:56.696 | INFO | Analyzing chunk frames = [6155264,6155776) | 20260508 07:01:56.696 | INFO | MemInfo: { | 20260508 07:01:56.696 | INFO | "gpu" : " 17 GB", | 20260508 07:01:56.696 | INFO | "huge" : "", | 20260508 07:01:56.696 | INFO | "huge_shared" : " 29 GB", | 20260508 07:01:56.696 | INFO | "malloc" : " 14 GB", | 20260508 07:01:56.696 | INFO | "pinned" : " 29 GB" | 20260508 07:01:56.696 | INFO | } | 20260508 07:01:56.696 | INFO | Perf report for the last 5.14689 seconds: | 20260508 07:01:56.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:01:56.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.696 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 07:01:56.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.696 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:01:56.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:01:56.697 | INFO | Uploaded 36GiB to GPU in 2.47438s (14.5491GiB/s) | 20260508 07:01:56.697 | INFO | Downloaded 34GiB to GPU in 2.42473s (14.0222GiB/s) | 20260508 07:02:01.832 | INFO | Analyzing chunk frames = [6155776,6156288) | 20260508 07:02:01.832 | INFO | MemInfo: { | 20260508 07:02:01.832 | INFO | "gpu" : " 17 GB", | 20260508 07:02:01.832 | INFO | "huge" : "", | 20260508 07:02:01.832 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:01.832 | INFO | "malloc" : " 14 GB", | 20260508 07:02:01.832 | INFO | "pinned" : " 29 GB" | 20260508 07:02:01.832 | INFO | } | 20260508 07:02:01.832 | INFO | Perf report for the last 5.13605 seconds: | 20260508 07:02:01.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:01.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.832 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 07:02:01.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.833 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:02:01.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:01.833 | INFO | Uploaded 36GiB to GPU in 2.45662s (14.6543GiB/s) | 20260508 07:02:01.833 | INFO | Downloaded 34GiB to GPU in 2.44109s (13.9282GiB/s) | 20260508 07:02:01.938 | INFO | Starting a round of Baz IO | 20260508 07:02:05.357 | INFO | Populating RTMetrics report | 20260508 07:02:05.358 | INFO | RTMetrics report populated | 20260508 07:02:05.358 | INFO | Opening tmp RTMetrics json file | 20260508 07:02:05.358 | INFO | Serializing RTMetrics to json | 20260508 07:02:05.359 | INFO | Serialization done, swapping tmp file now... | 20260508 07:02:05.360 | INFO | RTMetrics done | 20260508 07:02:05.424 | INFO | Finished a round of Baz IO: Wrote 1.536 GiB in 3.485 seconds | 20260508 07:02:05.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.00% / 59.39% | 20260508 07:02:05.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.365631% / 0% / 5.16574e-05% | 20260508 07:02:05.424 | INFO | Baz IO overall: | 20260508 07:02:05.424 | INFO | BytesWritten : 2721996406436 | 20260508 07:02:05.424 | INFO | NumEvents : 1311841671150 | 20260508 07:02:06.972 | INFO | Analyzing chunk frames = [6156288,6156800) | 20260508 07:02:06.973 | INFO | MemInfo: { | 20260508 07:02:06.973 | INFO | "gpu" : " 17 GB", | 20260508 07:02:06.973 | INFO | "huge" : "", | 20260508 07:02:06.973 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:06.973 | INFO | "malloc" : " 14 GB", | 20260508 07:02:06.973 | INFO | "pinned" : " 29 GB" | 20260508 07:02:06.973 | INFO | } | 20260508 07:02:06.973 | INFO | Perf report for the last 5.14037 seconds: | 20260508 07:02:06.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:06.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:06.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:06.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:06.973 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 07:02:06.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:06.973 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:02:06.973 | INFO | PRE_HQ : 65.7%, 1.6, 55.0ms, 0 | 20260508 07:02:06.973 | INFO | BAZWRITER : 67.4%, 2.6, 90.2ms, 0 | 20260508 07:02:06.973 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 07:02:06.973 | INFO | Uploaded 36GiB to GPU in 2.59449s (13.8756GiB/s) | 20260508 07:02:06.973 | INFO | Downloaded 38GiB to GPU in 2.69765s (14.0863GiB/s) | 20260508 07:02:12.111 | INFO | Analyzing chunk frames = [6156800,6157312) | 20260508 07:02:12.111 | INFO | MemInfo: { | 20260508 07:02:12.111 | INFO | "gpu" : " 17 GB", | 20260508 07:02:12.111 | INFO | "huge" : "", | 20260508 07:02:12.111 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:12.111 | INFO | "malloc" : " 14 GB", | 20260508 07:02:12.111 | INFO | "pinned" : " 29 GB" | 20260508 07:02:12.111 | INFO | } | 20260508 07:02:12.111 | INFO | Perf report for the last 5.13836 seconds: | 20260508 07:02:12.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:12.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:02:12.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:02:12.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:12.111 | INFO | Uploaded 36GiB to GPU in 2.51035s (14.3406GiB/s) | 20260508 07:02:12.111 | INFO | Downloaded 34GiB to GPU in 2.44139s (13.9265GiB/s) | 20260508 07:02:17.249 | INFO | Analyzing chunk frames = [6157312,6157824) | 20260508 07:02:17.249 | INFO | MemInfo: { | 20260508 07:02:17.249 | INFO | "gpu" : " 17 GB", | 20260508 07:02:17.249 | INFO | "huge" : "", | 20260508 07:02:17.249 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:17.249 | INFO | "malloc" : " 14 GB", | 20260508 07:02:17.249 | INFO | "pinned" : " 29 GB" | 20260508 07:02:17.249 | INFO | } | 20260508 07:02:17.249 | INFO | Perf report for the last 5.13811 seconds: | 20260508 07:02:17.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:17.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.249 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:02:17.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.249 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:02:17.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:17.250 | INFO | Uploaded 36GiB to GPU in 2.5779s (13.9649GiB/s) | 20260508 07:02:17.250 | INFO | Downloaded 34GiB to GPU in 2.42112s (14.0431GiB/s) | 20260508 07:02:22.387 | INFO | Analyzing chunk frames = [6157824,6158336) | 20260508 07:02:22.387 | INFO | MemInfo: { | 20260508 07:02:22.387 | INFO | "gpu" : " 17 GB", | 20260508 07:02:22.387 | INFO | "huge" : "", | 20260508 07:02:22.387 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:22.387 | INFO | "malloc" : " 14 GB", | 20260508 07:02:22.387 | INFO | "pinned" : " 29 GB" | 20260508 07:02:22.387 | INFO | } | 20260508 07:02:22.387 | INFO | Perf report for the last 5.13779 seconds: | 20260508 07:02:22.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:22.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.387 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:02:22.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.387 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 07:02:22.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:22.388 | INFO | Uploaded 36GiB to GPU in 2.58609s (13.9207GiB/s) | 20260508 07:02:22.388 | INFO | Downloaded 34GiB to GPU in 2.42043s (14.0471GiB/s) | 20260508 07:02:27.527 | INFO | Analyzing chunk frames = [6158336,6158848) | 20260508 07:02:27.527 | INFO | MemInfo: { | 20260508 07:02:27.527 | INFO | "gpu" : " 17 GB", | 20260508 07:02:27.527 | INFO | "huge" : "", | 20260508 07:02:27.527 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:27.527 | INFO | "malloc" : " 14 GB", | 20260508 07:02:27.527 | INFO | "pinned" : " 29 GB" | 20260508 07:02:27.527 | INFO | } | 20260508 07:02:27.527 | INFO | Perf report for the last 5.14034 seconds: | 20260508 07:02:27.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:27.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.527 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 07:02:27.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.528 | INFO | PRE_HQ : 32.7%, 1.0, 17.1ms, 0 | 20260508 07:02:27.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:27.528 | INFO | Uploaded 36GiB to GPU in 2.43512s (14.7836GiB/s) | 20260508 07:02:27.528 | INFO | Downloaded 34GiB to GPU in 2.53272s (13.4243GiB/s) | 20260508 07:02:32.670 | INFO | Analyzing chunk frames = [6158848,6159360) | 20260508 07:02:32.670 | INFO | MemInfo: { | 20260508 07:02:32.670 | INFO | "gpu" : " 17 GB", | 20260508 07:02:32.670 | INFO | "huge" : "", | 20260508 07:02:32.670 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:32.670 | INFO | "malloc" : " 14 GB", | 20260508 07:02:32.671 | INFO | "pinned" : " 29 GB" | 20260508 07:02:32.671 | INFO | } | 20260508 07:02:32.671 | INFO | Perf report for the last 5.14329 seconds: | 20260508 07:02:32.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:32.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.671 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:02:32.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.671 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:02:32.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:32.672 | INFO | Uploaded 36GiB to GPU in 2.47475s (14.5469GiB/s) | 20260508 07:02:32.672 | INFO | Downloaded 34GiB to GPU in 2.43532s (13.9612GiB/s) | 20260508 07:02:37.808 | INFO | Analyzing chunk frames = [6159360,6159872) | 20260508 07:02:37.808 | INFO | MemInfo: { | 20260508 07:02:37.808 | INFO | "gpu" : " 17 GB", | 20260508 07:02:37.808 | INFO | "huge" : "", | 20260508 07:02:37.808 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:37.808 | INFO | "malloc" : " 14 GB", | 20260508 07:02:37.808 | INFO | "pinned" : " 29 GB" | 20260508 07:02:37.808 | INFO | } | 20260508 07:02:37.808 | INFO | Perf report for the last 5.13774 seconds: | 20260508 07:02:37.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:37.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 07:02:37.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:02:37.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:37.809 | INFO | Uploaded 36GiB to GPU in 2.47514s (14.5447GiB/s) | 20260508 07:02:37.809 | INFO | Downloaded 34GiB to GPU in 2.42906s (13.9972GiB/s) | 20260508 07:02:42.947 | INFO | Analyzing chunk frames = [6159872,6160384) | 20260508 07:02:42.947 | INFO | MemInfo: { | 20260508 07:02:42.947 | INFO | "gpu" : " 17 GB", | 20260508 07:02:42.947 | INFO | "huge" : "", | 20260508 07:02:42.947 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:42.947 | INFO | "malloc" : " 14 GB", | 20260508 07:02:42.947 | INFO | "pinned" : " 29 GB" | 20260508 07:02:42.947 | INFO | } | 20260508 07:02:42.947 | INFO | Perf report for the last 5.1388 seconds: | 20260508 07:02:42.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:42.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.947 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:02:42.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.947 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 07:02:42.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:42.948 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:02:42.948 | INFO | Total mem usage: 16.7 GiB | 20260508 07:02:42.948 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:02:42.948 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:02:42.948 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:02:42.948 | INFO | Uploaded 36GiB to GPU in 2.49991s (14.4005GiB/s) | 20260508 07:02:42.948 | INFO | High Water Breakdown: | 20260508 07:02:42.948 | INFO | Downloaded 34GiB to GPU in 2.44531s (13.9042GiB/s) | 20260508 07:02:42.948 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:02:42.948 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:02:42.948 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:02:42.948 | INFO | ------------------------------------------ | 20260508 07:02:42.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:02:42.948 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:02:42.948 | INFO | Total mem usage: 26.5 GiB | 20260508 07:02:42.948 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:02:42.948 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:02:42.948 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:02:42.948 | INFO | High Water Breakdown: | 20260508 07:02:42.948 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:02:42.948 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:02:42.948 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:02:42.948 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:02:42.948 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:02:42.948 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:02:42.948 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:02:42.948 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:02:42.948 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:02:42.948 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:02:42.948 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:02:42.948 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:02:42.948 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:02:42.948 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:02:42.948 | INFO | ------------------------------------------ | 20260508 07:02:42.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:02:42.949 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:02:42.949 | INFO | Total mem usage: 27.8 GiB | 20260508 07:02:42.949 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:02:42.949 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:02:42.949 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:02:42.949 | INFO | High Water Breakdown: | 20260508 07:02:42.949 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:02:42.949 | INFO | ------------------------------------------ | 20260508 07:02:42.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:02:42.949 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:02:42.949 | INFO | Total mem usage: 15.7 GiB | 20260508 07:02:42.949 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:02:42.949 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:02:42.949 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:02:42.949 | INFO | High Water Breakdown: | 20260508 07:02:42.949 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:02:42.949 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:02:42.949 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:02:42.949 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:02:42.949 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:02:42.949 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:02:42.949 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:02:42.949 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:02:42.949 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:02:42.949 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:02:42.949 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:02:42.949 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:02:42.949 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:02:42.949 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:02:42.949 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:02:42.949 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:02:42.949 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:02:42.949 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:02:42.949 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:02:42.949 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:02:42.949 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:02:42.949 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:02:42.949 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:02:42.949 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:02:42.949 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:02:42.949 | INFO | ------------------------------------------ | 20260508 07:02:42.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:02:42.949 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:02:42.949 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:02:42.949 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:02:42.949 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.69e-01 | 20260508 07:02:42.949 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.04e-01, 1.13e-02 / 6.58e-03 | 20260508 07:02:42.949 | INFO | PerfReport: 12.11%, Baseline -- 3.96e+00 / 4.38e+00, 5.87e-01 / 2.84e-01 | 20260508 07:02:42.949 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.46e+00, 2.10e-01 / 1.19e-01 | 20260508 07:02:42.949 | INFO | PerfReport: 15.93%, DME -- 4.16e+01 / 4.52e+01, 3.82e+00 / 1.27e+00 | 20260508 07:02:42.949 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.81e+00 / 1.45e+00 | 20260508 07:02:42.949 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.12e-01 / 2.53e-01 | 20260508 07:02:42.949 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.21e+00, 2.55e-01 / 1.48e-01 | 20260508 07:02:42.949 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.42e-01, 4.99e-02 / 2.36e-02 | 20260508 07:02:42.949 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:02:42.949 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:02:42.949 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:02:42.949 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:02:42.949 | INFO | PerfReport: 24.49%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:02:42.949 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 07:02:42.949 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:02:42.950 | INFO | PerfReport: 2.45%, LockUpload -- 2.60e+00 / 0.00e+00, 5.87e+00 / 0.00e+00 | 20260508 07:02:42.950 | INFO | PerfReport: 2.37%, LockDownload -- 2.51e+00 / 0.00e+00, 3.85e+00 / 0.00e+00 | 20260508 07:02:42.950 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:02:42.950 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260508 07:02:43.044 | INFO | Starting a round of Baz IO | 20260508 07:02:46.459 | INFO | Populating RTMetrics report | 20260508 07:02:46.460 | INFO | RTMetrics report populated | 20260508 07:02:46.460 | INFO | Opening tmp RTMetrics json file | 20260508 07:02:46.472 | INFO | Serializing RTMetrics to json | 20260508 07:02:46.474 | INFO | Serialization done, swapping tmp file now... | 20260508 07:02:46.474 | INFO | RTMetrics done | 20260508 07:02:46.532 | INFO | Finished a round of Baz IO: Wrote 1.535 GiB in 3.487 seconds | 20260508 07:02:46.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.00% / 59.39% | 20260508 07:02:46.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.362774% / 3.97364e-06% / 6.75519e-05% | 20260508 07:02:46.532 | INFO | Baz IO overall: | 20260508 07:02:46.532 | INFO | BytesWritten : 2723644512051 | 20260508 07:02:46.532 | INFO | NumEvents : 1312624451957 | 20260508 07:02:48.089 | INFO | Analyzing chunk frames = [6160384,6160896) | 20260508 07:02:48.089 | INFO | MemInfo: { | 20260508 07:02:48.089 | INFO | "gpu" : " 17 GB", | 20260508 07:02:48.089 | INFO | "huge" : "", | 20260508 07:02:48.089 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:48.089 | INFO | "malloc" : " 14 GB", | 20260508 07:02:48.089 | INFO | "pinned" : " 29 GB" | 20260508 07:02:48.089 | INFO | } | 20260508 07:02:48.089 | INFO | Perf report for the last 5.14157 seconds: | 20260508 07:02:48.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:48.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:48.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:48.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:48.089 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 07:02:48.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:48.089 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 07:02:48.089 | INFO | PRE_HQ : 65.8%, 1.6, 55.4ms, 0 | 20260508 07:02:48.089 | INFO | BAZWRITER : 67.1%, 2.6, 90.2ms, 0 | 20260508 07:02:48.089 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:02:48.089 | INFO | Uploaded 36GiB to GPU in 2.58882s (13.906GiB/s) | 20260508 07:02:48.089 | INFO | Downloaded 38GiB to GPU in 2.70012s (14.0734GiB/s) | 20260508 07:02:53.227 | INFO | Analyzing chunk frames = [6160896,6161408) | 20260508 07:02:53.227 | INFO | MemInfo: { | 20260508 07:02:53.227 | INFO | "gpu" : " 17 GB", | 20260508 07:02:53.227 | INFO | "huge" : "", | 20260508 07:02:53.227 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:53.227 | INFO | "malloc" : " 14 GB", | 20260508 07:02:53.227 | INFO | "pinned" : " 29 GB" | 20260508 07:02:53.227 | INFO | } | 20260508 07:02:53.227 | INFO | Perf report for the last 5.13842 seconds: | 20260508 07:02:53.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:53.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.227 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:02:53.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.227 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:02:53.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:53.228 | INFO | Uploaded 36GiB to GPU in 2.53158s (14.2204GiB/s) | 20260508 07:02:53.228 | INFO | Downloaded 34GiB to GPU in 2.43932s (13.9383GiB/s) | 20260508 07:02:58.364 | INFO | Analyzing chunk frames = [6161408,6161920) | 20260508 07:02:58.364 | INFO | MemInfo: { | 20260508 07:02:58.364 | INFO | "gpu" : " 17 GB", | 20260508 07:02:58.364 | INFO | "huge" : "", | 20260508 07:02:58.364 | INFO | "huge_shared" : " 29 GB", | 20260508 07:02:58.364 | INFO | "malloc" : " 14 GB", | 20260508 07:02:58.364 | INFO | "pinned" : " 29 GB" | 20260508 07:02:58.364 | INFO | } | 20260508 07:02:58.364 | INFO | Perf report for the last 5.13708 seconds: | 20260508 07:02:58.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:02:58.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.365 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 07:02:58.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.365 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:02:58.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:02:58.365 | INFO | Uploaded 36GiB to GPU in 2.57988s (13.9542GiB/s) | 20260508 07:02:58.365 | INFO | Downloaded 34GiB to GPU in 2.43044s (13.9893GiB/s) | 20260508 07:03:03.505 | INFO | Analyzing chunk frames = [6161920,6162432) | 20260508 07:03:03.505 | INFO | MemInfo: { | 20260508 07:03:03.505 | INFO | "gpu" : " 17 GB", | 20260508 07:03:03.505 | INFO | "huge" : "", | 20260508 07:03:03.505 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:03.505 | INFO | "malloc" : " 14 GB", | 20260508 07:03:03.505 | INFO | "pinned" : " 29 GB" | 20260508 07:03:03.505 | INFO | } | 20260508 07:03:03.505 | INFO | Perf report for the last 5.14042 seconds: | 20260508 07:03:03.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:03.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:03:03.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:03:03.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:03.505 | INFO | Uploaded 36GiB to GPU in 2.59368s (13.8799GiB/s) | 20260508 07:03:03.505 | INFO | Downloaded 34GiB to GPU in 2.41705s (14.0668GiB/s) | 20260508 07:03:08.639 | INFO | Analyzing chunk frames = [6162432,6162944) | 20260508 07:03:08.639 | INFO | MemInfo: { | 20260508 07:03:08.639 | INFO | "gpu" : " 17 GB", | 20260508 07:03:08.639 | INFO | "huge" : "", | 20260508 07:03:08.639 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:08.639 | INFO | "malloc" : " 14 GB", | 20260508 07:03:08.639 | INFO | "pinned" : " 29 GB" | 20260508 07:03:08.639 | INFO | } | 20260508 07:03:08.640 | INFO | Perf report for the last 5.1347 seconds: | 20260508 07:03:08.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:08.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:03:08.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:03:08.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:08.640 | INFO | Uploaded 36GiB to GPU in 2.28935s (15.725GiB/s) | 20260508 07:03:08.640 | INFO | Downloaded 34GiB to GPU in 2.49456s (13.6296GiB/s) | 20260508 07:03:13.785 | INFO | Analyzing chunk frames = [6162944,6163456) | 20260508 07:03:13.785 | INFO | MemInfo: { | 20260508 07:03:13.785 | INFO | "gpu" : " 17 GB", | 20260508 07:03:13.785 | INFO | "huge" : "", | 20260508 07:03:13.785 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:13.785 | INFO | "malloc" : " 14 GB", | 20260508 07:03:13.786 | INFO | "pinned" : " 29 GB" | 20260508 07:03:13.786 | INFO | } | 20260508 07:03:13.786 | INFO | Perf report for the last 5.14605 seconds: | 20260508 07:03:13.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:13.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 07:03:13.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:03:13.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:13.786 | INFO | Uploaded 36GiB to GPU in 2.44713s (14.7111GiB/s) | 20260508 07:03:13.786 | INFO | Downloaded 34GiB to GPU in 2.43371s (13.9705GiB/s) | 20260508 07:03:18.925 | INFO | Analyzing chunk frames = [6163456,6163968) | 20260508 07:03:18.925 | INFO | MemInfo: { | 20260508 07:03:18.925 | INFO | "gpu" : " 17 GB", | 20260508 07:03:18.925 | INFO | "huge" : "", | 20260508 07:03:18.925 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:18.925 | INFO | "malloc" : " 14 GB", | 20260508 07:03:18.925 | INFO | "pinned" : " 29 GB" | 20260508 07:03:18.925 | INFO | } | 20260508 07:03:18.925 | INFO | Perf report for the last 5.13928 seconds: | 20260508 07:03:18.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:18.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 07:03:18.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:03:18.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:18.925 | INFO | Uploaded 36GiB to GPU in 2.46421s (14.6091GiB/s) | 20260508 07:03:18.926 | INFO | Downloaded 34GiB to GPU in 2.42455s (14.0232GiB/s) | 20260508 07:03:24.063 | INFO | Analyzing chunk frames = [6163968,6164480) | 20260508 07:03:24.063 | INFO | MemInfo: { | 20260508 07:03:24.063 | INFO | "gpu" : " 17 GB", | 20260508 07:03:24.063 | INFO | "huge" : "", | 20260508 07:03:24.063 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:24.063 | INFO | "malloc" : " 14 GB", | 20260508 07:03:24.063 | INFO | "pinned" : " 29 GB" | 20260508 07:03:24.063 | INFO | } | 20260508 07:03:24.063 | INFO | Perf report for the last 5.1381 seconds: | 20260508 07:03:24.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:24.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.063 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:03:24.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.063 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:03:24.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:24.064 | INFO | Uploaded 36GiB to GPU in 2.47488s (14.5462GiB/s) | 20260508 07:03:24.064 | INFO | Downloaded 34GiB to GPU in 2.44446s (13.909GiB/s) | 20260508 07:03:24.160 | INFO | Starting a round of Baz IO | 20260508 07:03:27.585 | INFO | Populating RTMetrics report | 20260508 07:03:27.585 | INFO | RTMetrics report populated | 20260508 07:03:27.585 | INFO | Opening tmp RTMetrics json file | 20260508 07:03:27.594 | INFO | Serializing RTMetrics to json | 20260508 07:03:27.598 | INFO | Serialization done, swapping tmp file now... | 20260508 07:03:27.599 | INFO | RTMetrics done | 20260508 07:03:27.655 | INFO | Finished a round of Baz IO: Wrote 1.536 GiB in 3.495 seconds | 20260508 07:03:27.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.00% / 59.39% | 20260508 07:03:27.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.363453% / 0% / 4.76837e-05% | 20260508 07:03:27.655 | INFO | Baz IO overall: | 20260508 07:03:27.655 | INFO | BytesWritten : 2725293515716 | 20260508 07:03:27.655 | INFO | NumEvents : 1313408083198 | 20260508 07:03:29.196 | INFO | Analyzing chunk frames = [6164480,6164992) | 20260508 07:03:29.196 | INFO | MemInfo: { | 20260508 07:03:29.196 | INFO | "gpu" : " 17 GB", | 20260508 07:03:29.196 | INFO | "huge" : "", | 20260508 07:03:29.196 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:29.196 | INFO | "malloc" : " 14 GB", | 20260508 07:03:29.196 | INFO | "pinned" : " 29 GB" | 20260508 07:03:29.196 | INFO | } | 20260508 07:03:29.196 | INFO | Perf report for the last 5.13282 seconds: | 20260508 07:03:29.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:29.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:29.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:29.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:29.196 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 07:03:29.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:29.196 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 07:03:29.196 | INFO | PRE_HQ : 65.7%, 1.6, 55.1ms, 0 | 20260508 07:03:29.196 | INFO | BAZWRITER : 67.5%, 2.6, 90.0ms, 0 | 20260508 07:03:29.196 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:03:29.196 | INFO | Uploaded 36GiB to GPU in 2.59052s (13.8968GiB/s) | 20260508 07:03:29.196 | INFO | Downloaded 38GiB to GPU in 2.69596s (14.0951GiB/s) | 20260508 07:03:34.339 | INFO | Analyzing chunk frames = [6164992,6165504) | 20260508 07:03:34.339 | INFO | MemInfo: { | 20260508 07:03:34.339 | INFO | "gpu" : " 17 GB", | 20260508 07:03:34.339 | INFO | "huge" : "", | 20260508 07:03:34.339 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:34.339 | INFO | "malloc" : " 14 GB", | 20260508 07:03:34.339 | INFO | "pinned" : " 29 GB" | 20260508 07:03:34.339 | INFO | } | 20260508 07:03:34.339 | INFO | Perf report for the last 5.14306 seconds: | 20260508 07:03:34.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:34.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 07:03:34.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 07:03:34.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:34.339 | INFO | Uploaded 36GiB to GPU in 2.50557s (14.368GiB/s) | 20260508 07:03:34.339 | INFO | Downloaded 34GiB to GPU in 2.44371s (13.9133GiB/s) | 20260508 07:03:39.477 | INFO | Analyzing chunk frames = [6165504,6166016) | 20260508 07:03:39.477 | INFO | MemInfo: { | 20260508 07:03:39.477 | INFO | "gpu" : " 17 GB", | 20260508 07:03:39.477 | INFO | "huge" : "", | 20260508 07:03:39.477 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:39.477 | INFO | "malloc" : " 14 GB", | 20260508 07:03:39.477 | INFO | "pinned" : " 29 GB" | 20260508 07:03:39.477 | INFO | } | 20260508 07:03:39.477 | INFO | Perf report for the last 5.13849 seconds: | 20260508 07:03:39.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:39.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.478 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 07:03:39.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.478 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:03:39.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:39.478 | INFO | Uploaded 36GiB to GPU in 2.56163s (14.0535GiB/s) | 20260508 07:03:39.478 | INFO | Downloaded 34GiB to GPU in 2.43105s (13.9857GiB/s) | 20260508 07:03:44.621 | INFO | Analyzing chunk frames = [6166016,6166528) | 20260508 07:03:44.621 | INFO | MemInfo: { | 20260508 07:03:44.621 | INFO | "gpu" : " 17 GB", | 20260508 07:03:44.621 | INFO | "huge" : "", | 20260508 07:03:44.621 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:44.621 | INFO | "malloc" : " 14 GB", | 20260508 07:03:44.621 | INFO | "pinned" : " 29 GB" | 20260508 07:03:44.621 | INFO | } | 20260508 07:03:44.621 | INFO | Perf report for the last 5.14338 seconds: | 20260508 07:03:44.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:44.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 07:03:44.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:03:44.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:44.621 | INFO | Uploaded 36GiB to GPU in 2.54895s (14.1235GiB/s) | 20260508 07:03:44.621 | INFO | Downloaded 34GiB to GPU in 2.39548s (14.1934GiB/s) | 20260508 07:03:49.762 | INFO | Analyzing chunk frames = [6166528,6167040) | 20260508 07:03:49.762 | INFO | MemInfo: { | 20260508 07:03:49.762 | INFO | "gpu" : " 17 GB", | 20260508 07:03:49.762 | INFO | "huge" : "", | 20260508 07:03:49.762 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:49.762 | INFO | "malloc" : " 14 GB", | 20260508 07:03:49.762 | INFO | "pinned" : " 29 GB" | 20260508 07:03:49.762 | INFO | } | 20260508 07:03:49.762 | INFO | Perf report for the last 5.14142 seconds: | 20260508 07:03:49.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:49.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.762 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:03:49.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.762 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:03:49.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:49.763 | INFO | Uploaded 36GiB to GPU in 2.38822s (15.074GiB/s) | 20260508 07:03:49.763 | INFO | Downloaded 34GiB to GPU in 2.49807s (13.6105GiB/s) | 20260508 07:03:54.893 | INFO | Analyzing chunk frames = [6167040,6167552) | 20260508 07:03:54.893 | INFO | MemInfo: { | 20260508 07:03:54.893 | INFO | "gpu" : " 17 GB", | 20260508 07:03:54.893 | INFO | "huge" : "", | 20260508 07:03:54.893 | INFO | "huge_shared" : " 29 GB", | 20260508 07:03:54.893 | INFO | "malloc" : " 14 GB", | 20260508 07:03:54.893 | INFO | "pinned" : " 29 GB" | 20260508 07:03:54.893 | INFO | } | 20260508 07:03:54.893 | INFO | Perf report for the last 5.13078 seconds: | 20260508 07:03:54.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:03:54.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.893 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:03:54.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.893 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:03:54.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:03:54.894 | INFO | Uploaded 36GiB to GPU in 2.39526s (15.0297GiB/s) | 20260508 07:03:54.894 | INFO | Downloaded 34GiB to GPU in 2.43397s (13.9689GiB/s) | 20260508 07:04:00.037 | INFO | Analyzing chunk frames = [6167552,6168064) | 20260508 07:04:00.037 | INFO | MemInfo: { | 20260508 07:04:00.037 | INFO | "gpu" : " 17 GB", | 20260508 07:04:00.037 | INFO | "huge" : "", | 20260508 07:04:00.037 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:00.037 | INFO | "malloc" : " 14 GB", | 20260508 07:04:00.037 | INFO | "pinned" : " 29 GB" | 20260508 07:04:00.037 | INFO | } | 20260508 07:04:00.037 | INFO | Perf report for the last 5.14395 seconds: | 20260508 07:04:00.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:00.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.037 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 07:04:00.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.037 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:04:00.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:00.038 | INFO | Uploaded 36GiB to GPU in 2.48013s (14.5154GiB/s) | 20260508 07:04:00.038 | INFO | Downloaded 34GiB to GPU in 2.43384s (13.9697GiB/s) | 20260508 07:04:05.178 | INFO | Analyzing chunk frames = [6168064,6168576) | 20260508 07:04:05.179 | INFO | MemInfo: { | 20260508 07:04:05.179 | INFO | "gpu" : " 17 GB", | 20260508 07:04:05.179 | INFO | "huge" : "", | 20260508 07:04:05.179 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:05.179 | INFO | "malloc" : " 14 GB", | 20260508 07:04:05.179 | INFO | "pinned" : " 29 GB" | 20260508 07:04:05.179 | INFO | } | 20260508 07:04:05.179 | INFO | Perf report for the last 5.14163 seconds: | 20260508 07:04:05.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:05.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:04:05.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:04:05.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:05.179 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:04:05.179 | INFO | Total mem usage: 16.7 GiB | 20260508 07:04:05.179 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:04:05.179 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:04:05.179 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:04:05.179 | INFO | High Water Breakdown: | 20260508 07:04:05.179 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:04:05.179 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:04:05.179 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:04:05.179 | INFO | ------------------------------------------ | 20260508 07:04:05.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:04:05.179 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:04:05.179 | INFO | Total mem usage: 26.5 GiB | 20260508 07:04:05.179 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:04:05.179 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:04:05.179 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:04:05.179 | INFO | Uploaded 36GiB to GPU in 2.48329s (14.4969GiB/s) | 20260508 07:04:05.179 | INFO | High Water Breakdown: | 20260508 07:04:05.179 | INFO | Downloaded 34GiB to GPU in 2.44181s (13.9241GiB/s) | 20260508 07:04:05.179 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:04:05.179 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:04:05.179 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:04:05.179 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:04:05.179 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:04:05.179 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:04:05.179 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:04:05.179 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:04:05.179 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:04:05.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:04:05.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:04:05.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:04:05.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:04:05.179 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:04:05.179 | INFO | ------------------------------------------ | 20260508 07:04:05.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:04:05.180 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:04:05.180 | INFO | Total mem usage: 27.8 GiB | 20260508 07:04:05.180 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:04:05.180 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:04:05.180 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:04:05.180 | INFO | High Water Breakdown: | 20260508 07:04:05.180 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:04:05.180 | INFO | ------------------------------------------ | 20260508 07:04:05.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:04:05.180 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:04:05.180 | INFO | Total mem usage: 15.7 GiB | 20260508 07:04:05.180 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:04:05.180 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:04:05.180 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:04:05.180 | INFO | High Water Breakdown: | 20260508 07:04:05.180 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:04:05.180 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:04:05.180 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:04:05.180 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:04:05.180 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:04:05.180 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:04:05.180 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:04:05.180 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:04:05.180 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:04:05.180 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:04:05.180 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:04:05.180 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:04:05.180 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:04:05.180 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:04:05.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:04:05.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:04:05.180 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:04:05.180 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:04:05.180 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:04:05.180 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:04:05.180 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:04:05.180 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:04:05.180 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:04:05.180 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:04:05.180 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:04:05.180 | INFO | ------------------------------------------ | 20260508 07:04:05.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:04:05.180 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:04:05.180 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:04:05.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:04:05.180 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.04e-01 | 20260508 07:04:05.180 | INFO | PerfReport: 0.29%, AggregateStats -- 9.59e-02 / 1.05e-01, 1.20e-02 / 5.74e-03 | 20260508 07:04:05.180 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.48e+00, 5.81e-01 / 1.73e-01 | 20260508 07:04:05.180 | INFO | PerfReport: 3.33%, Histogram -- 1.09e+00 / 1.43e+00, 2.07e-01 / 1.14e-01 | 20260508 07:04:05.180 | INFO | PerfReport: 15.94%, DME -- 4.16e+01 / 4.51e+01, 3.68e+00 / 1.08e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.77e+00 / 1.24e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.37e+00 / 2.55e+00, 4.11e-01 / 1.89e-01 | 20260508 07:04:05.180 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.18e+00, 2.52e-01 / 9.96e-02 | 20260508 07:04:05.180 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.44e-01, 5.01e-02 / 2.32e-02 | 20260508 07:04:05.180 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:04:05.180 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:04:05.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:04:05.180 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 24.51%, Upload -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 24.28%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 30.76%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 2.35%, LockUpload -- 2.50e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 2.53%, LockDownload -- 2.68e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 07:04:05.180 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260508 07:04:05.293 | INFO | Starting a round of Baz IO | 20260508 07:04:08.698 | INFO | Populating RTMetrics report | 20260508 07:04:08.698 | INFO | RTMetrics report populated | 20260508 07:04:08.698 | INFO | Opening tmp RTMetrics json file | 20260508 07:04:08.700 | INFO | Serializing RTMetrics to json | 20260508 07:04:08.701 | INFO | Serialization done, swapping tmp file now... | 20260508 07:04:08.701 | INFO | RTMetrics done | 20260508 07:04:08.765 | INFO | Finished a round of Baz IO: Wrote 1.537 GiB in 3.471 seconds | 20260508 07:04:08.765 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.00% / 59.39% | 20260508 07:04:08.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.363807% / 0% / 5.96046e-05% | 20260508 07:04:08.765 | INFO | Baz IO overall: | 20260508 07:04:08.765 | INFO | BytesWritten : 2726944378105 | 20260508 07:04:08.765 | INFO | NumEvents : 1314193292586 | 20260508 07:04:10.315 | INFO | Analyzing chunk frames = [6168576,6169088) | 20260508 07:04:10.315 | INFO | MemInfo: { | 20260508 07:04:10.315 | INFO | "gpu" : " 17 GB", | 20260508 07:04:10.315 | INFO | "huge" : "", | 20260508 07:04:10.315 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:10.315 | INFO | "malloc" : " 14 GB", | 20260508 07:04:10.315 | INFO | "pinned" : " 29 GB" | 20260508 07:04:10.315 | INFO | } | 20260508 07:04:10.315 | INFO | Perf report for the last 5.13604 seconds: | 20260508 07:04:10.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:10.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:10.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:10.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:10.315 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 07:04:10.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:10.315 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260508 07:04:10.315 | INFO | PRE_HQ : 66.6%, 1.7, 58.2ms, 0 | 20260508 07:04:10.315 | INFO | BAZWRITER : 67.0%, 2.7, 93.2ms, 0 | 20260508 07:04:10.315 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:04:10.315 | INFO | Uploaded 36GiB to GPU in 2.50602s (14.3654GiB/s) | 20260508 07:04:10.315 | INFO | Downloaded 38GiB to GPU in 2.68297s (14.1634GiB/s) | 20260508 07:04:15.454 | INFO | Analyzing chunk frames = [6169088,6169600) | 20260508 07:04:15.454 | INFO | MemInfo: { | 20260508 07:04:15.454 | INFO | "gpu" : " 17 GB", | 20260508 07:04:15.454 | INFO | "huge" : "", | 20260508 07:04:15.454 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:15.454 | INFO | "malloc" : " 14 GB", | 20260508 07:04:15.454 | INFO | "pinned" : " 29 GB" | 20260508 07:04:15.454 | INFO | } | 20260508 07:04:15.454 | INFO | Perf report for the last 5.13936 seconds: | 20260508 07:04:15.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:15.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.454 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:04:15.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.454 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:04:15.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:15.455 | INFO | Uploaded 36GiB to GPU in 2.47845s (14.5252GiB/s) | 20260508 07:04:15.455 | INFO | Downloaded 34GiB to GPU in 2.44048s (13.9317GiB/s) | 20260508 07:04:20.595 | INFO | Analyzing chunk frames = [6169600,6170112) | 20260508 07:04:20.595 | INFO | MemInfo: { | 20260508 07:04:20.595 | INFO | "gpu" : " 17 GB", | 20260508 07:04:20.595 | INFO | "huge" : "", | 20260508 07:04:20.595 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:20.595 | INFO | "malloc" : " 14 GB", | 20260508 07:04:20.595 | INFO | "pinned" : " 29 GB" | 20260508 07:04:20.595 | INFO | } | 20260508 07:04:20.595 | INFO | Perf report for the last 5.14128 seconds: | 20260508 07:04:20.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:20.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.595 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:04:20.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.596 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:04:20.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:20.596 | INFO | Uploaded 36GiB to GPU in 2.56839s (14.0165GiB/s) | 20260508 07:04:20.596 | INFO | Downloaded 34GiB to GPU in 2.43802s (13.9457GiB/s) | 20260508 07:04:25.734 | INFO | Analyzing chunk frames = [6170112,6170624) | 20260508 07:04:25.734 | INFO | MemInfo: { | 20260508 07:04:25.734 | INFO | "gpu" : " 17 GB", | 20260508 07:04:25.734 | INFO | "huge" : "", | 20260508 07:04:25.734 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:25.734 | INFO | "malloc" : " 14 GB", | 20260508 07:04:25.734 | INFO | "pinned" : " 29 GB" | 20260508 07:04:25.734 | INFO | } | 20260508 07:04:25.734 | INFO | Perf report for the last 5.13859 seconds: | 20260508 07:04:25.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:25.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.734 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:04:25.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.734 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:04:25.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:25.735 | INFO | Uploaded 36GiB to GPU in 2.55946s (14.0655GiB/s) | 20260508 07:04:25.735 | INFO | Downloaded 34GiB to GPU in 2.43036s (13.9897GiB/s) | 20260508 07:04:30.876 | INFO | Analyzing chunk frames = [6170624,6171136) | 20260508 07:04:30.876 | INFO | MemInfo: { | 20260508 07:04:30.876 | INFO | "gpu" : " 17 GB", | 20260508 07:04:30.876 | INFO | "huge" : "", | 20260508 07:04:30.877 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:30.877 | INFO | "malloc" : " 14 GB", | 20260508 07:04:30.877 | INFO | "pinned" : " 29 GB" | 20260508 07:04:30.877 | INFO | } | 20260508 07:04:30.877 | INFO | Perf report for the last 5.14259 seconds: | 20260508 07:04:30.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:30.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:04:30.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:04:30.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:30.877 | INFO | Uploaded 36GiB to GPU in 2.28922s (15.7259GiB/s) | 20260508 07:04:30.877 | INFO | Downloaded 34GiB to GPU in 2.49051s (13.6518GiB/s) | 20260508 07:04:36.012 | INFO | Analyzing chunk frames = [6171136,6171648) | 20260508 07:04:36.012 | INFO | MemInfo: { | 20260508 07:04:36.012 | INFO | "gpu" : " 17 GB", | 20260508 07:04:36.012 | INFO | "huge" : "", | 20260508 07:04:36.012 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:36.012 | INFO | "malloc" : " 14 GB", | 20260508 07:04:36.012 | INFO | "pinned" : " 29 GB" | 20260508 07:04:36.012 | INFO | } | 20260508 07:04:36.012 | INFO | Perf report for the last 5.13562 seconds: | 20260508 07:04:36.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:36.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.012 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:04:36.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.012 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:04:36.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:36.013 | INFO | Uploaded 36GiB to GPU in 2.40407s (14.9746GiB/s) | 20260508 07:04:36.013 | INFO | Downloaded 34GiB to GPU in 2.4198s (14.0507GiB/s) | 20260508 07:04:41.150 | INFO | Analyzing chunk frames = [6171648,6172160) | 20260508 07:04:41.150 | INFO | MemInfo: { | 20260508 07:04:41.150 | INFO | "gpu" : " 17 GB", | 20260508 07:04:41.150 | INFO | "huge" : "", | 20260508 07:04:41.150 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:41.150 | INFO | "malloc" : " 14 GB", | 20260508 07:04:41.150 | INFO | "pinned" : " 29 GB" | 20260508 07:04:41.151 | INFO | } | 20260508 07:04:41.151 | INFO | Perf report for the last 5.13831 seconds: | 20260508 07:04:41.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:41.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 07:04:41.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:04:41.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:41.151 | INFO | Uploaded 36GiB to GPU in 2.42345s (14.8548GiB/s) | 20260508 07:04:41.151 | INFO | Downloaded 34GiB to GPU in 2.43083s (13.987GiB/s) | 20260508 07:04:46.294 | INFO | Analyzing chunk frames = [6172160,6172672) | 20260508 07:04:46.294 | INFO | MemInfo: { | 20260508 07:04:46.294 | INFO | "gpu" : " 17 GB", | 20260508 07:04:46.294 | INFO | "huge" : "", | 20260508 07:04:46.294 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:46.294 | INFO | "malloc" : " 14 GB", | 20260508 07:04:46.294 | INFO | "pinned" : " 29 GB" | 20260508 07:04:46.294 | INFO | } | 20260508 07:04:46.294 | INFO | Perf report for the last 5.14328 seconds: | 20260508 07:04:46.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:46.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 07:04:46.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:04:46.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:46.294 | INFO | Uploaded 36GiB to GPU in 2.47607s (14.5392GiB/s) | 20260508 07:04:46.295 | INFO | Downloaded 34GiB to GPU in 2.43328s (13.9729GiB/s) | 20260508 07:04:46.392 | INFO | Starting a round of Baz IO | 20260508 07:04:49.824 | INFO | Populating RTMetrics report | 20260508 07:04:49.824 | INFO | RTMetrics report populated | 20260508 07:04:49.824 | INFO | Opening tmp RTMetrics json file | 20260508 07:04:49.825 | INFO | Serializing RTMetrics to json | 20260508 07:04:49.826 | INFO | Serialization done, swapping tmp file now... | 20260508 07:04:49.826 | INFO | RTMetrics done | 20260508 07:04:49.895 | INFO | Finished a round of Baz IO: Wrote 1.536 GiB in 3.502 seconds | 20260508 07:04:49.895 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.00% / 59.39% | 20260508 07:04:49.895 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.36397% / 0% / 7.15256e-05% | 20260508 07:04:49.895 | INFO | Baz IO overall: | 20260508 07:04:49.895 | INFO | BytesWritten : 2728593237448 | 20260508 07:04:49.895 | INFO | NumEvents : 1314977275774 | 20260508 07:04:51.434 | INFO | Analyzing chunk frames = [6172672,6173184) | 20260508 07:04:51.434 | INFO | MemInfo: { | 20260508 07:04:51.434 | INFO | "gpu" : " 17 GB", | 20260508 07:04:51.434 | INFO | "huge" : "", | 20260508 07:04:51.434 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:51.434 | INFO | "malloc" : " 14 GB", | 20260508 07:04:51.434 | INFO | "pinned" : " 29 GB" | 20260508 07:04:51.434 | INFO | } | 20260508 07:04:51.434 | INFO | Perf report for the last 5.13999 seconds: | 20260508 07:04:51.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:51.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:51.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:51.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:51.434 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 07:04:51.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:51.434 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:04:51.434 | INFO | PRE_HQ : 66.0%, 1.6, 54.8ms, 0 | 20260508 07:04:51.434 | INFO | BAZWRITER : 67.5%, 2.6, 90.5ms, 0 | 20260508 07:04:51.434 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:04:51.434 | INFO | Uploaded 36GiB to GPU in 2.5667s (14.0258GiB/s) | 20260508 07:04:51.434 | INFO | Downloaded 38GiB to GPU in 2.71494s (13.9966GiB/s) | 20260508 07:04:56.568 | INFO | Analyzing chunk frames = [6173184,6173696) | 20260508 07:04:56.568 | INFO | MemInfo: { | 20260508 07:04:56.568 | INFO | "gpu" : " 17 GB", | 20260508 07:04:56.568 | INFO | "huge" : "", | 20260508 07:04:56.568 | INFO | "huge_shared" : " 29 GB", | 20260508 07:04:56.568 | INFO | "malloc" : " 14 GB", | 20260508 07:04:56.568 | INFO | "pinned" : " 29 GB" | 20260508 07:04:56.568 | INFO | } | 20260508 07:04:56.568 | INFO | Perf report for the last 5.1344 seconds: | 20260508 07:04:56.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:04:56.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.568 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:04:56.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.568 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:04:56.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:04:56.569 | INFO | Uploaded 36GiB to GPU in 2.50069s (14.396GiB/s) | 20260508 07:04:56.569 | INFO | Downloaded 34GiB to GPU in 2.43822s (13.9446GiB/s) | 20260508 07:05:01.706 | INFO | Analyzing chunk frames = [6173696,6174208) | 20260508 07:05:01.706 | INFO | MemInfo: { | 20260508 07:05:01.706 | INFO | "gpu" : " 17 GB", | 20260508 07:05:01.706 | INFO | "huge" : "", | 20260508 07:05:01.706 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:01.706 | INFO | "malloc" : " 14 GB", | 20260508 07:05:01.706 | INFO | "pinned" : " 29 GB" | 20260508 07:05:01.706 | INFO | } | 20260508 07:05:01.706 | INFO | Perf report for the last 5.13752 seconds: | 20260508 07:05:01.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:01.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 07:05:01.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:05:01.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:01.706 | INFO | Uploaded 36GiB to GPU in 2.57506s (13.9802GiB/s) | 20260508 07:05:01.707 | INFO | Downloaded 34GiB to GPU in 2.42915s (13.9967GiB/s) | 20260508 07:05:06.848 | INFO | Analyzing chunk frames = [6174208,6174720) | 20260508 07:05:06.848 | INFO | MemInfo: { | 20260508 07:05:06.848 | INFO | "gpu" : " 17 GB", | 20260508 07:05:06.848 | INFO | "huge" : "", | 20260508 07:05:06.848 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:06.848 | INFO | "malloc" : " 14 GB", | 20260508 07:05:06.848 | INFO | "pinned" : " 29 GB" | 20260508 07:05:06.848 | INFO | } | 20260508 07:05:06.848 | INFO | Perf report for the last 5.14254 seconds: | 20260508 07:05:06.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:06.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.848 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:05:06.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.849 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:05:06.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:06.849 | INFO | Uploaded 36GiB to GPU in 2.55341s (14.0988GiB/s) | 20260508 07:05:06.849 | INFO | Downloaded 34GiB to GPU in 2.39278s (14.2094GiB/s) | 20260508 07:05:11.991 | INFO | Analyzing chunk frames = [6174720,6175232) | 20260508 07:05:11.991 | INFO | MemInfo: { | 20260508 07:05:11.991 | INFO | "gpu" : " 17 GB", | 20260508 07:05:11.991 | INFO | "huge" : "", | 20260508 07:05:11.991 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:11.991 | INFO | "malloc" : " 14 GB", | 20260508 07:05:11.992 | INFO | "pinned" : " 29 GB" | 20260508 07:05:11.992 | INFO | } | 20260508 07:05:11.992 | INFO | Perf report for the last 5.14323 seconds: | 20260508 07:05:11.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:11.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:05:11.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:05:11.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:11.992 | INFO | Uploaded 36GiB to GPU in 2.43333s (14.7946GiB/s) | 20260508 07:05:11.992 | INFO | Downloaded 34GiB to GPU in 2.51341s (13.5274GiB/s) | 20260508 07:05:17.128 | INFO | Analyzing chunk frames = [6175232,6175744) | 20260508 07:05:17.128 | INFO | MemInfo: { | 20260508 07:05:17.128 | INFO | "gpu" : " 17 GB", | 20260508 07:05:17.128 | INFO | "huge" : "", | 20260508 07:05:17.128 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:17.128 | INFO | "malloc" : " 14 GB", | 20260508 07:05:17.128 | INFO | "pinned" : " 29 GB" | 20260508 07:05:17.128 | INFO | } | 20260508 07:05:17.128 | INFO | Perf report for the last 5.13643 seconds: | 20260508 07:05:17.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:17.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.128 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260508 07:05:17.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.128 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:05:17.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:17.129 | INFO | Uploaded 36GiB to GPU in 2.38762s (15.0778GiB/s) | 20260508 07:05:17.129 | INFO | Downloaded 34GiB to GPU in 2.41553s (14.0756GiB/s) | 20260508 07:05:22.265 | INFO | Analyzing chunk frames = [6175744,6176256) | 20260508 07:05:22.265 | INFO | MemInfo: { | 20260508 07:05:22.265 | INFO | "gpu" : " 17 GB", | 20260508 07:05:22.265 | INFO | "huge" : "", | 20260508 07:05:22.265 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:22.265 | INFO | "malloc" : " 14 GB", | 20260508 07:05:22.265 | INFO | "pinned" : " 29 GB" | 20260508 07:05:22.265 | INFO | } | 20260508 07:05:22.265 | INFO | Perf report for the last 5.13687 seconds: | 20260508 07:05:22.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:22.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:05:22.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:05:22.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:22.265 | INFO | Uploaded 36GiB to GPU in 2.48886s (14.4644GiB/s) | 20260508 07:05:22.265 | INFO | Downloaded 34GiB to GPU in 2.42925s (13.9961GiB/s) | 20260508 07:05:27.402 | INFO | Analyzing chunk frames = [6176256,6176768) | 20260508 07:05:27.402 | INFO | MemInfo: { | 20260508 07:05:27.402 | INFO | "gpu" : " 17 GB", | 20260508 07:05:27.402 | INFO | "huge" : "", | 20260508 07:05:27.402 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:27.402 | INFO | "malloc" : " 14 GB", | 20260508 07:05:27.402 | INFO | "pinned" : " 29 GB" | 20260508 07:05:27.403 | INFO | } | 20260508 07:05:27.403 | INFO | Perf report for the last 5.13795 seconds: | 20260508 07:05:27.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:27.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 07:05:27.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:05:27.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:27.403 | INFO | Uploaded 36GiB to GPU in 2.49346s (14.4378GiB/s) | 20260508 07:05:27.403 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9296GiB/s) | 20260508 07:05:27.403 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:05:27.403 | INFO | Total mem usage: 16.7 GiB | 20260508 07:05:27.403 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:05:27.403 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:05:27.404 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:05:27.404 | INFO | High Water Breakdown: | 20260508 07:05:27.404 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:05:27.404 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:05:27.404 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:05:27.404 | INFO | ------------------------------------------ | 20260508 07:05:27.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:05:27.404 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:05:27.404 | INFO | Total mem usage: 26.5 GiB | 20260508 07:05:27.404 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:05:27.404 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:05:27.404 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:05:27.404 | INFO | High Water Breakdown: | 20260508 07:05:27.404 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:05:27.404 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:05:27.404 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:05:27.404 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:05:27.404 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:05:27.404 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:05:27.404 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:05:27.404 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:05:27.404 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:05:27.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:05:27.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:05:27.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:05:27.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:05:27.404 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:05:27.404 | INFO | ------------------------------------------ | 20260508 07:05:27.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:05:27.404 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:05:27.404 | INFO | Total mem usage: 27.8 GiB | 20260508 07:05:27.404 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:05:27.404 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:05:27.404 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:05:27.404 | INFO | High Water Breakdown: | 20260508 07:05:27.404 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:05:27.404 | INFO | ------------------------------------------ | 20260508 07:05:27.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:05:27.404 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:05:27.404 | INFO | Total mem usage: 15.7 GiB | 20260508 07:05:27.404 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:05:27.404 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:05:27.404 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:05:27.404 | INFO | High Water Breakdown: | 20260508 07:05:27.404 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:05:27.404 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:05:27.404 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:05:27.404 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:05:27.404 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:05:27.404 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:05:27.404 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:05:27.404 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:05:27.404 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:05:27.404 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:05:27.404 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:05:27.404 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:05:27.404 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:05:27.404 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:05:27.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:05:27.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:05:27.404 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:05:27.404 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:05:27.404 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:05:27.404 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:05:27.404 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:05:27.404 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:05:27.404 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:05:27.404 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:05:27.404 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:05:27.404 | INFO | ------------------------------------------ | 20260508 07:05:27.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:05:27.405 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:05:27.405 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:05:27.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:05:27.405 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 7.37e-01 | 20260508 07:05:27.405 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.21e-02 / 6.34e-03 | 20260508 07:05:27.405 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.39e+00, 5.93e-01 / 1.95e-01 | 20260508 07:05:27.405 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.09e-01 / 1.12e-01 | 20260508 07:05:27.405 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.56e+01, 3.75e+00 / 1.10e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.87e+01 / 1.96e+01, 1.87e+00 / 1.16e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.36e+00 / 2.61e+00, 4.13e-01 / 1.81e-01 | 20260508 07:05:27.405 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.21e+00, 2.54e-01 / 1.25e-01 | 20260508 07:05:27.405 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.46e-01, 4.97e-02 / 2.03e-02 | 20260508 07:05:27.405 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:05:27.405 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:05:27.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:05:27.405 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 24.41%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 24.31%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 30.94%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 2.33%, LockUpload -- 2.46e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 2.48%, LockDownload -- 2.62e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:05:27.405 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260508 07:05:27.500 | INFO | Starting a round of Baz IO | 20260508 07:05:30.921 | INFO | Populating RTMetrics report | 20260508 07:05:30.922 | INFO | RTMetrics report populated | 20260508 07:05:30.922 | INFO | Opening tmp RTMetrics json file | 20260508 07:05:30.933 | INFO | Serializing RTMetrics to json | 20260508 07:05:30.934 | INFO | Serialization done, swapping tmp file now... | 20260508 07:05:30.935 | INFO | RTMetrics done | 20260508 07:05:30.989 | INFO | Finished a round of Baz IO: Wrote 1.534 GiB in 3.488 seconds | 20260508 07:05:30.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.60% / 0.00% / 59.39% | 20260508 07:05:30.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.361951% / 0% / 7.94729e-05% | 20260508 07:05:30.989 | INFO | Baz IO overall: | 20260508 07:05:30.989 | INFO | BytesWritten : 2730240429283 | 20260508 07:05:30.989 | INFO | NumEvents : 1315760117084 | 20260508 07:05:32.548 | INFO | Analyzing chunk frames = [6176768,6177280) | 20260508 07:05:32.548 | INFO | MemInfo: { | 20260508 07:05:32.548 | INFO | "gpu" : " 17 GB", | 20260508 07:05:32.548 | INFO | "huge" : "", | 20260508 07:05:32.548 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:32.548 | INFO | "malloc" : " 14 GB", | 20260508 07:05:32.548 | INFO | "pinned" : " 29 GB" | 20260508 07:05:32.548 | INFO | } | 20260508 07:05:32.549 | INFO | Perf report for the last 5.14563 seconds: | 20260508 07:05:32.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:32.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:32.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:32.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:32.549 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 07:05:32.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:32.549 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 07:05:32.549 | INFO | PRE_HQ : 65.7%, 1.6, 55.0ms, 0 | 20260508 07:05:32.549 | INFO | BAZWRITER : 67.1%, 2.6, 90.8ms, 0 | 20260508 07:05:32.549 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:05:32.549 | INFO | Uploaded 36GiB to GPU in 2.55567s (14.0863GiB/s) | 20260508 07:05:32.549 | INFO | Downloaded 38GiB to GPU in 2.69372s (14.1069GiB/s) | 20260508 07:05:37.688 | INFO | Analyzing chunk frames = [6177280,6177792) | 20260508 07:05:37.688 | INFO | MemInfo: { | 20260508 07:05:37.688 | INFO | "gpu" : " 17 GB", | 20260508 07:05:37.688 | INFO | "huge" : "", | 20260508 07:05:37.688 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:37.688 | INFO | "malloc" : " 14 GB", | 20260508 07:05:37.688 | INFO | "pinned" : " 29 GB" | 20260508 07:05:37.688 | INFO | } | 20260508 07:05:37.688 | INFO | Perf report for the last 5.13924 seconds: | 20260508 07:05:37.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:37.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:05:37.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:05:37.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:37.688 | INFO | Uploaded 36GiB to GPU in 2.52686s (14.247GiB/s) | 20260508 07:05:37.688 | INFO | Downloaded 34GiB to GPU in 2.4424s (13.9208GiB/s) | 20260508 07:05:42.831 | INFO | Analyzing chunk frames = [6177792,6178304) | 20260508 07:05:42.831 | INFO | MemInfo: { | 20260508 07:05:42.831 | INFO | "gpu" : " 17 GB", | 20260508 07:05:42.831 | INFO | "huge" : "", | 20260508 07:05:42.831 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:42.831 | INFO | "malloc" : " 14 GB", | 20260508 07:05:42.831 | INFO | "pinned" : " 29 GB" | 20260508 07:05:42.831 | INFO | } | 20260508 07:05:42.832 | INFO | Perf report for the last 5.14374 seconds: | 20260508 07:05:42.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:42.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 07:05:42.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:05:42.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:42.832 | INFO | Uploaded 36GiB to GPU in 2.58083s (13.949GiB/s) | 20260508 07:05:42.832 | INFO | Downloaded 34GiB to GPU in 2.43083s (13.987GiB/s) | 20260508 07:05:47.962 | INFO | Analyzing chunk frames = [6178304,6178816) | 20260508 07:05:47.962 | INFO | MemInfo: { | 20260508 07:05:47.962 | INFO | "gpu" : " 17 GB", | 20260508 07:05:47.962 | INFO | "huge" : "", | 20260508 07:05:47.962 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:47.963 | INFO | "malloc" : " 14 GB", | 20260508 07:05:47.963 | INFO | "pinned" : " 29 GB" | 20260508 07:05:47.963 | INFO | } | 20260508 07:05:47.963 | INFO | Perf report for the last 5.13137 seconds: | 20260508 07:05:47.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:47.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.963 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 07:05:47.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.963 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260508 07:05:47.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:47.964 | INFO | Uploaded 36GiB to GPU in 2.58738s (13.9137GiB/s) | 20260508 07:05:47.964 | INFO | Downloaded 34GiB to GPU in 2.41965s (14.0516GiB/s) | 20260508 07:05:53.099 | INFO | Analyzing chunk frames = [6178816,6179328) | 20260508 07:05:53.099 | INFO | MemInfo: { | 20260508 07:05:53.099 | INFO | "gpu" : " 17 GB", | 20260508 07:05:53.099 | INFO | "huge" : "", | 20260508 07:05:53.099 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:53.099 | INFO | "malloc" : " 14 GB", | 20260508 07:05:53.099 | INFO | "pinned" : " 29 GB" | 20260508 07:05:53.099 | INFO | } | 20260508 07:05:53.099 | INFO | Perf report for the last 5.13581 seconds: | 20260508 07:05:53.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:53.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 07:05:53.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | PRE_HQ : 32.3%, 1.0, 16.9ms, 0 | 20260508 07:05:53.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:53.099 | INFO | Uploaded 36GiB to GPU in 2.43317s (14.7955GiB/s) | 20260508 07:05:53.099 | INFO | Downloaded 34GiB to GPU in 2.5363s (13.4054GiB/s) | 20260508 07:05:58.240 | INFO | Analyzing chunk frames = [6179328,6179840) | 20260508 07:05:58.240 | INFO | MemInfo: { | 20260508 07:05:58.240 | INFO | "gpu" : " 17 GB", | 20260508 07:05:58.240 | INFO | "huge" : "", | 20260508 07:05:58.240 | INFO | "huge_shared" : " 29 GB", | 20260508 07:05:58.240 | INFO | "malloc" : " 14 GB", | 20260508 07:05:58.240 | INFO | "pinned" : " 29 GB" | 20260508 07:05:58.241 | INFO | } | 20260508 07:05:58.241 | INFO | Perf report for the last 5.14191 seconds: | 20260508 07:05:58.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:05:58.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:05:58.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:05:58.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:05:58.241 | INFO | Uploaded 36GiB to GPU in 2.47142s (14.5665GiB/s) | 20260508 07:05:58.241 | INFO | Downloaded 34GiB to GPU in 2.43355s (13.9714GiB/s) | 20260508 07:06:03.382 | INFO | Analyzing chunk frames = [6179840,6180352) | 20260508 07:06:03.382 | INFO | MemInfo: { | 20260508 07:06:03.382 | INFO | "gpu" : " 17 GB", | 20260508 07:06:03.382 | INFO | "huge" : "", | 20260508 07:06:03.382 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:03.382 | INFO | "malloc" : " 14 GB", | 20260508 07:06:03.382 | INFO | "pinned" : " 29 GB" | 20260508 07:06:03.383 | INFO | } | 20260508 07:06:03.383 | INFO | Perf report for the last 5.14207 seconds: | 20260508 07:06:03.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:03.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 07:06:03.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:06:03.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:03.383 | INFO | Uploaded 36GiB to GPU in 2.47523s (14.5441GiB/s) | 20260508 07:06:03.383 | INFO | Downloaded 34GiB to GPU in 2.42539s (14.0183GiB/s) | 20260508 07:06:08.521 | INFO | Analyzing chunk frames = [6180352,6180864) | 20260508 07:06:08.521 | INFO | MemInfo: { | 20260508 07:06:08.521 | INFO | "gpu" : " 17 GB", | 20260508 07:06:08.521 | INFO | "huge" : "", | 20260508 07:06:08.521 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:08.521 | INFO | "malloc" : " 14 GB", | 20260508 07:06:08.521 | INFO | "pinned" : " 29 GB" | 20260508 07:06:08.521 | INFO | } | 20260508 07:06:08.521 | INFO | Perf report for the last 5.13869 seconds: | 20260508 07:06:08.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:08.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:06:08.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:06:08.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:08.522 | INFO | Uploaded 36GiB to GPU in 2.50671s (14.3615GiB/s) | 20260508 07:06:08.522 | INFO | Downloaded 34GiB to GPU in 2.44494s (13.9062GiB/s) | 20260508 07:06:08.618 | INFO | Starting a round of Baz IO | 20260508 07:06:12.044 | INFO | Populating RTMetrics report | 20260508 07:06:12.044 | INFO | RTMetrics report populated | 20260508 07:06:12.044 | INFO | Opening tmp RTMetrics json file | 20260508 07:06:12.055 | INFO | Serializing RTMetrics to json | 20260508 07:06:12.056 | INFO | Serialization done, swapping tmp file now... | 20260508 07:06:12.057 | INFO | RTMetrics done | 20260508 07:06:12.112 | INFO | Finished a round of Baz IO: Wrote 1.534 GiB in 3.493 seconds | 20260508 07:06:12.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.60% / 0.00% / 59.40% | 20260508 07:06:12.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.36178% / 3.97364e-06% / 5.16574e-05% | 20260508 07:06:12.112 | INFO | Baz IO overall: | 20260508 07:06:12.112 | INFO | BytesWritten : 2731887635012 | 20260508 07:06:12.112 | INFO | NumEvents : 1316543010981 | 20260508 07:06:13.658 | INFO | Analyzing chunk frames = [6180864,6181376) | 20260508 07:06:13.658 | INFO | MemInfo: { | 20260508 07:06:13.658 | INFO | "gpu" : " 17 GB", | 20260508 07:06:13.658 | INFO | "huge" : "", | 20260508 07:06:13.658 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:13.658 | INFO | "malloc" : " 14 GB", | 20260508 07:06:13.658 | INFO | "pinned" : " 29 GB" | 20260508 07:06:13.658 | INFO | } | 20260508 07:06:13.658 | INFO | Perf report for the last 5.13631 seconds: | 20260508 07:06:13.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:13.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:13.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:13.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:13.658 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 07:06:13.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:13.658 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:06:13.658 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 07:06:13.658 | INFO | BAZWRITER : 67.4%, 2.6, 90.4ms, 0 | 20260508 07:06:13.658 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 07:06:13.658 | INFO | Uploaded 36GiB to GPU in 2.59572s (13.869GiB/s) | 20260508 07:06:13.658 | INFO | Downloaded 38GiB to GPU in 2.68669s (14.1438GiB/s) | 20260508 07:06:18.799 | INFO | Analyzing chunk frames = [6181376,6181888) | 20260508 07:06:18.800 | INFO | MemInfo: { | 20260508 07:06:18.800 | INFO | "gpu" : " 17 GB", | 20260508 07:06:18.800 | INFO | "huge" : "", | 20260508 07:06:18.800 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:18.800 | INFO | "malloc" : " 14 GB", | 20260508 07:06:18.800 | INFO | "pinned" : " 29 GB" | 20260508 07:06:18.800 | INFO | } | 20260508 07:06:18.800 | INFO | Perf report for the last 5.142 seconds: | 20260508 07:06:18.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:18.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:06:18.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:06:18.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:18.800 | INFO | Uploaded 36GiB to GPU in 2.54194s (14.1624GiB/s) | 20260508 07:06:18.800 | INFO | Downloaded 34GiB to GPU in 2.45677s (13.8393GiB/s) | 20260508 07:06:23.938 | INFO | Analyzing chunk frames = [6181888,6182400) | 20260508 07:06:23.938 | INFO | MemInfo: { | 20260508 07:06:23.938 | INFO | "gpu" : " 17 GB", | 20260508 07:06:23.938 | INFO | "huge" : "", | 20260508 07:06:23.938 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:23.938 | INFO | "malloc" : " 14 GB", | 20260508 07:06:23.939 | INFO | "pinned" : " 29 GB" | 20260508 07:06:23.939 | INFO | } | 20260508 07:06:23.939 | INFO | Perf report for the last 5.13877 seconds: | 20260508 07:06:23.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:23.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 07:06:23.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:06:23.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:23.939 | INFO | Uploaded 36GiB to GPU in 2.58488s (13.9272GiB/s) | 20260508 07:06:23.939 | INFO | Downloaded 34GiB to GPU in 2.41837s (14.059GiB/s) | 20260508 07:06:29.076 | INFO | Analyzing chunk frames = [6182400,6182912) | 20260508 07:06:29.076 | INFO | MemInfo: { | 20260508 07:06:29.076 | INFO | "gpu" : " 17 GB", | 20260508 07:06:29.076 | INFO | "huge" : "", | 20260508 07:06:29.076 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:29.076 | INFO | "malloc" : " 14 GB", | 20260508 07:06:29.077 | INFO | "pinned" : " 29 GB" | 20260508 07:06:29.077 | INFO | } | 20260508 07:06:29.077 | INFO | Perf report for the last 5.13817 seconds: | 20260508 07:06:29.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:29.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.077 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:06:29.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.077 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 07:06:29.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:29.078 | INFO | Uploaded 36GiB to GPU in 2.58879s (13.9061GiB/s) | 20260508 07:06:29.078 | INFO | Downloaded 34GiB to GPU in 2.4157s (14.0746GiB/s) | 20260508 07:06:34.215 | INFO | Analyzing chunk frames = [6182912,6183424) | 20260508 07:06:34.215 | INFO | MemInfo: { | 20260508 07:06:34.215 | INFO | "gpu" : " 17 GB", | 20260508 07:06:34.215 | INFO | "huge" : "", | 20260508 07:06:34.215 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:34.215 | INFO | "malloc" : " 14 GB", | 20260508 07:06:34.215 | INFO | "pinned" : " 29 GB" | 20260508 07:06:34.215 | INFO | } | 20260508 07:06:34.217 | INFO | Perf report for the last 5.14046 seconds: | 20260508 07:06:34.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:34.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.217 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 07:06:34.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.217 | INFO | PRE_HQ : 32.0%, 1.0, 16.8ms, 0 | 20260508 07:06:34.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:34.218 | INFO | Uploaded 36GiB to GPU in 2.27233s (15.8428GiB/s) | 20260508 07:06:34.218 | INFO | Downloaded 34GiB to GPU in 2.47688s (13.727GiB/s) | 20260508 07:06:39.357 | INFO | Analyzing chunk frames = [6183424,6183936) | 20260508 07:06:39.357 | INFO | MemInfo: { | 20260508 07:06:39.357 | INFO | "gpu" : " 17 GB", | 20260508 07:06:39.357 | INFO | "huge" : "", | 20260508 07:06:39.357 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:39.357 | INFO | "malloc" : " 14 GB", | 20260508 07:06:39.357 | INFO | "pinned" : " 29 GB" | 20260508 07:06:39.357 | INFO | } | 20260508 07:06:39.357 | INFO | Perf report for the last 5.13989 seconds: | 20260508 07:06:39.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:39.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.357 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 07:06:39.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.357 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:06:39.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:39.358 | INFO | Uploaded 36GiB to GPU in 2.4344s (14.7881GiB/s) | 20260508 07:06:39.358 | INFO | Downloaded 34GiB to GPU in 2.4306s (13.9883GiB/s) | 20260508 07:06:44.502 | INFO | Analyzing chunk frames = [6183936,6184448) | 20260508 07:06:44.502 | INFO | MemInfo: { | 20260508 07:06:44.502 | INFO | "gpu" : " 17 GB", | 20260508 07:06:44.502 | INFO | "huge" : "", | 20260508 07:06:44.502 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:44.502 | INFO | "malloc" : " 14 GB", | 20260508 07:06:44.502 | INFO | "pinned" : " 29 GB" | 20260508 07:06:44.502 | INFO | } | 20260508 07:06:44.502 | INFO | Perf report for the last 5.14471 seconds: | 20260508 07:06:44.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:44.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 07:06:44.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:06:44.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:44.502 | INFO | Uploaded 36GiB to GPU in 2.48641s (14.4787GiB/s) | 20260508 07:06:44.502 | INFO | Downloaded 34GiB to GPU in 2.42864s (13.9996GiB/s) | 20260508 07:06:49.639 | INFO | Analyzing chunk frames = [6184448,6184960) | 20260508 07:06:49.639 | INFO | MemInfo: { | 20260508 07:06:49.639 | INFO | "gpu" : " 17 GB", | 20260508 07:06:49.639 | INFO | "huge" : "", | 20260508 07:06:49.639 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:49.639 | INFO | "malloc" : " 14 GB", | 20260508 07:06:49.639 | INFO | "pinned" : " 29 GB" | 20260508 07:06:49.639 | INFO | } | 20260508 07:06:49.639 | INFO | Perf report for the last 5.13716 seconds: | 20260508 07:06:49.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:49.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 07:06:49.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:06:49.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:49.640 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:06:49.640 | INFO | Total mem usage: 16.7 GiB | 20260508 07:06:49.640 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:06:49.640 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:06:49.640 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:06:49.640 | INFO | High Water Breakdown: | 20260508 07:06:49.640 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:06:49.640 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:06:49.640 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:06:49.640 | INFO | ------------------------------------------ | 20260508 07:06:49.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:06:49.640 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:06:49.640 | INFO | Total mem usage: 26.5 GiB | 20260508 07:06:49.640 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:06:49.640 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:06:49.640 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:06:49.640 | INFO | High Water Breakdown: | 20260508 07:06:49.640 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:06:49.640 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:06:49.640 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:06:49.640 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:06:49.640 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:06:49.640 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:06:49.640 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:06:49.640 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:06:49.640 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:06:49.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:06:49.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:06:49.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:06:49.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:06:49.640 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:06:49.640 | INFO | ------------------------------------------ | 20260508 07:06:49.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:06:49.640 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:06:49.640 | INFO | Total mem usage: 27.8 GiB | 20260508 07:06:49.640 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:06:49.640 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:06:49.640 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:06:49.640 | INFO | High Water Breakdown: | 20260508 07:06:49.640 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:06:49.640 | INFO | ------------------------------------------ | 20260508 07:06:49.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:06:49.640 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:06:49.640 | INFO | Total mem usage: 15.7 GiB | 20260508 07:06:49.640 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:06:49.640 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:06:49.640 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:06:49.640 | INFO | High Water Breakdown: | 20260508 07:06:49.640 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:06:49.640 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:06:49.640 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:06:49.640 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:06:49.641 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:06:49.641 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:06:49.641 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:06:49.641 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:06:49.641 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:06:49.641 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:06:49.641 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:06:49.641 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:06:49.641 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:06:49.641 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:06:49.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:06:49.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:06:49.641 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:06:49.641 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:06:49.641 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:06:49.641 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:06:49.641 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:06:49.641 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:06:49.641 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:06:49.641 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:06:49.641 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:06:49.641 | INFO | ------------------------------------------ | 20260508 07:06:49.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:06:49.641 | INFO | Uploaded 36GiB to GPU in 2.45586s (14.6588GiB/s) | 20260508 07:06:49.641 | INFO | Downloaded 34GiB to GPU in 2.43991s (13.9349GiB/s) | 20260508 07:06:49.641 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:06:49.641 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:06:49.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:06:49.641 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.70e-01 | 20260508 07:06:49.641 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.06e-01, 1.13e-02 / 4.28e-03 | 20260508 07:06:49.641 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.43e+00, 5.85e-01 / 2.35e-01 | 20260508 07:06:49.641 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.13e-01 | 20260508 07:06:49.641 | INFO | PerfReport: 15.87%, DME -- 4.15e+01 / 4.47e+01, 3.65e+00 / 9.16e-01 | 20260508 07:06:49.641 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.86e+00 / 1.40e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.52e+00, 4.09e-01 / 2.59e-01 | 20260508 07:06:49.641 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.23e+00, 2.51e-01 / 1.42e-01 | 20260508 07:06:49.641 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.43e-01, 4.89e-02 / 2.55e-02 | 20260508 07:06:49.641 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:06:49.641 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:06:49.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:06:49.641 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 24.61%, Upload -- 2.61e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 2.45%, LockUpload -- 2.60e+00 / 0.00e+00, 5.81e+00 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 2.38%, LockDownload -- 2.53e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 15.45%, LockCompute -- 1.64e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 07:06:49.641 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 07:06:49.738 | INFO | Starting a round of Baz IO | 20260508 07:06:53.160 | INFO | Populating RTMetrics report | 20260508 07:06:53.160 | INFO | RTMetrics report populated | 20260508 07:06:53.160 | INFO | Opening tmp RTMetrics json file | 20260508 07:06:53.161 | INFO | Serializing RTMetrics to json | 20260508 07:06:53.162 | INFO | Serialization done, swapping tmp file now... | 20260508 07:06:53.162 | INFO | RTMetrics done | 20260508 07:06:53.230 | INFO | Finished a round of Baz IO: Wrote 1.533 GiB in 3.491 seconds | 20260508 07:06:53.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.60% / 0.00% / 59.40% | 20260508 07:06:53.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.359988% / 0% / 6.35783e-05% | 20260508 07:06:53.230 | INFO | Baz IO overall: | 20260508 07:06:53.230 | INFO | BytesWritten : 2733533835147 | 20260508 07:06:53.230 | INFO | NumEvents : 1317325065241 | 20260508 07:06:54.773 | INFO | Analyzing chunk frames = [6184960,6185472) | 20260508 07:06:54.773 | INFO | MemInfo: { | 20260508 07:06:54.773 | INFO | "gpu" : " 17 GB", | 20260508 07:06:54.773 | INFO | "huge" : "", | 20260508 07:06:54.773 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:54.773 | INFO | "malloc" : " 14 GB", | 20260508 07:06:54.773 | INFO | "pinned" : " 29 GB" | 20260508 07:06:54.773 | INFO | } | 20260508 07:06:54.773 | INFO | Perf report for the last 5.13428 seconds: | 20260508 07:06:54.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:54.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:54.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:54.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:54.774 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 07:06:54.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:54.774 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:06:54.774 | INFO | PRE_HQ : 65.7%, 1.6, 54.8ms, 0 | 20260508 07:06:54.774 | INFO | BAZWRITER : 67.3%, 2.6, 90.4ms, 0 | 20260508 07:06:54.774 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:06:54.774 | INFO | Uploaded 36GiB to GPU in 2.56857s (14.0156GiB/s) | 20260508 07:06:54.774 | INFO | Downloaded 38GiB to GPU in 2.70752s (14.035GiB/s) | 20260508 07:06:59.911 | INFO | Analyzing chunk frames = [6185472,6185984) | 20260508 07:06:59.911 | INFO | MemInfo: { | 20260508 07:06:59.911 | INFO | "gpu" : " 17 GB", | 20260508 07:06:59.911 | INFO | "huge" : "", | 20260508 07:06:59.911 | INFO | "huge_shared" : " 29 GB", | 20260508 07:06:59.911 | INFO | "malloc" : " 14 GB", | 20260508 07:06:59.911 | INFO | "pinned" : " 29 GB" | 20260508 07:06:59.911 | INFO | } | 20260508 07:06:59.911 | INFO | Perf report for the last 5.13813 seconds: | 20260508 07:06:59.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:06:59.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.912 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 07:06:59.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.912 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:06:59.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:06:59.912 | INFO | Uploaded 36GiB to GPU in 2.51156s (14.3337GiB/s) | 20260508 07:06:59.912 | INFO | Downloaded 34GiB to GPU in 2.4244s (14.0241GiB/s) | 20260508 07:07:05.052 | INFO | Analyzing chunk frames = [6185984,6186496) | 20260508 07:07:05.052 | INFO | MemInfo: { | 20260508 07:07:05.052 | INFO | "gpu" : " 17 GB", | 20260508 07:07:05.052 | INFO | "huge" : "", | 20260508 07:07:05.052 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:05.052 | INFO | "malloc" : " 14 GB", | 20260508 07:07:05.052 | INFO | "pinned" : " 29 GB" | 20260508 07:07:05.052 | INFO | } | 20260508 07:07:05.052 | INFO | Perf report for the last 5.14038 seconds: | 20260508 07:07:05.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:05.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:07:05.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:07:05.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:05.052 | INFO | Uploaded 36GiB to GPU in 2.58s (13.9535GiB/s) | 20260508 07:07:05.052 | INFO | Downloaded 34GiB to GPU in 2.41321s (14.0891GiB/s) | 20260508 07:07:10.192 | INFO | Analyzing chunk frames = [6186496,6187008) | 20260508 07:07:10.192 | INFO | MemInfo: { | 20260508 07:07:10.192 | INFO | "gpu" : " 17 GB", | 20260508 07:07:10.192 | INFO | "huge" : "", | 20260508 07:07:10.192 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:10.192 | INFO | "malloc" : " 14 GB", | 20260508 07:07:10.192 | INFO | "pinned" : " 29 GB" | 20260508 07:07:10.192 | INFO | } | 20260508 07:07:10.192 | INFO | Perf report for the last 5.1403 seconds: | 20260508 07:07:10.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:10.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.192 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 07:07:10.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.192 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:07:10.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:10.193 | INFO | Uploaded 36GiB to GPU in 2.59022s (13.8985GiB/s) | 20260508 07:07:10.193 | INFO | Downloaded 34GiB to GPU in 2.40892s (14.1142GiB/s) | 20260508 07:07:15.332 | INFO | Analyzing chunk frames = [6187008,6187520) | 20260508 07:07:15.332 | INFO | MemInfo: { | 20260508 07:07:15.332 | INFO | "gpu" : " 17 GB", | 20260508 07:07:15.332 | INFO | "huge" : "", | 20260508 07:07:15.332 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:15.332 | INFO | "malloc" : " 14 GB", | 20260508 07:07:15.332 | INFO | "pinned" : " 29 GB" | 20260508 07:07:15.332 | INFO | } | 20260508 07:07:15.332 | INFO | Perf report for the last 5.14025 seconds: | 20260508 07:07:15.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:15.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.333 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 07:07:15.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.333 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:07:15.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:15.333 | INFO | Uploaded 36GiB to GPU in 2.32783s (15.4651GiB/s) | 20260508 07:07:15.333 | INFO | Downloaded 34GiB to GPU in 2.49175s (13.645GiB/s) | 20260508 07:07:20.470 | INFO | Analyzing chunk frames = [6187520,6188032) | 20260508 07:07:20.470 | INFO | MemInfo: { | 20260508 07:07:20.470 | INFO | "gpu" : " 17 GB", | 20260508 07:07:20.470 | INFO | "huge" : "", | 20260508 07:07:20.470 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:20.470 | INFO | "malloc" : " 14 GB", | 20260508 07:07:20.470 | INFO | "pinned" : " 29 GB" | 20260508 07:07:20.470 | INFO | } | 20260508 07:07:20.470 | INFO | Perf report for the last 5.13739 seconds: | 20260508 07:07:20.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:20.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.470 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:07:20.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.470 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:07:20.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:20.471 | INFO | Uploaded 36GiB to GPU in 2.47806s (14.5275GiB/s) | 20260508 07:07:20.471 | INFO | Downloaded 34GiB to GPU in 2.42437s (14.0243GiB/s) | 20260508 07:07:25.613 | INFO | Analyzing chunk frames = [6188032,6188544) | 20260508 07:07:25.613 | INFO | MemInfo: { | 20260508 07:07:25.613 | INFO | "gpu" : " 17 GB", | 20260508 07:07:25.613 | INFO | "huge" : "", | 20260508 07:07:25.613 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:25.613 | INFO | "malloc" : " 14 GB", | 20260508 07:07:25.613 | INFO | "pinned" : " 29 GB" | 20260508 07:07:25.613 | INFO | } | 20260508 07:07:25.613 | INFO | Perf report for the last 5.14353 seconds: | 20260508 07:07:25.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:25.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.613 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:07:25.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.614 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:07:25.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:25.614 | INFO | Uploaded 36GiB to GPU in 2.49364s (14.4367GiB/s) | 20260508 07:07:25.614 | INFO | Downloaded 34GiB to GPU in 2.43166s (13.9822GiB/s) | 20260508 07:07:30.752 | INFO | Analyzing chunk frames = [6188544,6189056) | 20260508 07:07:30.752 | INFO | MemInfo: { | 20260508 07:07:30.752 | INFO | "gpu" : " 17 GB", | 20260508 07:07:30.752 | INFO | "huge" : "", | 20260508 07:07:30.752 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:30.752 | INFO | "malloc" : " 14 GB", | 20260508 07:07:30.752 | INFO | "pinned" : " 29 GB" | 20260508 07:07:30.752 | INFO | } | 20260508 07:07:30.752 | INFO | Perf report for the last 5.13875 seconds: | 20260508 07:07:30.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:30.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:07:30.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:07:30.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:30.752 | INFO | Uploaded 36GiB to GPU in 2.48254s (14.5013GiB/s) | 20260508 07:07:30.753 | INFO | Downloaded 34GiB to GPU in 2.42278s (14.0334GiB/s) | 20260508 07:07:30.849 | INFO | Starting a round of Baz IO | 20260508 07:07:34.279 | INFO | Populating RTMetrics report | 20260508 07:07:34.279 | INFO | RTMetrics report populated | 20260508 07:07:34.279 | INFO | Opening tmp RTMetrics json file | 20260508 07:07:34.287 | INFO | Serializing RTMetrics to json | 20260508 07:07:34.289 | INFO | Serialization done, swapping tmp file now... | 20260508 07:07:34.289 | INFO | RTMetrics done | 20260508 07:07:34.347 | INFO | Finished a round of Baz IO: Wrote 1.534 GiB in 3.497 seconds | 20260508 07:07:34.347 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.60% / 0.00% / 59.40% | 20260508 07:07:34.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.359575% / 3.97364e-06% / 6.35783e-05% | 20260508 07:07:34.347 | INFO | Baz IO overall: | 20260508 07:07:34.347 | INFO | BytesWritten : 2735180465603 | 20260508 07:07:34.347 | INFO | NumEvents : 1318107548996 | 20260508 07:07:35.889 | INFO | Analyzing chunk frames = [6189056,6189568) | 20260508 07:07:35.889 | INFO | MemInfo: { | 20260508 07:07:35.889 | INFO | "gpu" : " 17 GB", | 20260508 07:07:35.889 | INFO | "huge" : "", | 20260508 07:07:35.889 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:35.889 | INFO | "malloc" : " 14 GB", | 20260508 07:07:35.889 | INFO | "pinned" : " 29 GB" | 20260508 07:07:35.889 | INFO | } | 20260508 07:07:35.889 | INFO | Perf report for the last 5.1367 seconds: | 20260508 07:07:35.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:35.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:35.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:35.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:35.889 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 07:07:35.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:35.889 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:07:35.889 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 07:07:35.889 | INFO | BAZWRITER : 67.4%, 2.6, 90.3ms, 0 | 20260508 07:07:35.889 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:07:35.889 | INFO | Uploaded 36GiB to GPU in 2.60499s (13.8196GiB/s) | 20260508 07:07:35.890 | INFO | Downloaded 38GiB to GPU in 2.6877s (14.1385GiB/s) | 20260508 07:07:41.026 | INFO | Analyzing chunk frames = [6189568,6190080) | 20260508 07:07:41.026 | INFO | MemInfo: { | 20260508 07:07:41.026 | INFO | "gpu" : " 17 GB", | 20260508 07:07:41.026 | INFO | "huge" : "", | 20260508 07:07:41.026 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:41.026 | INFO | "malloc" : " 14 GB", | 20260508 07:07:41.026 | INFO | "pinned" : " 29 GB" | 20260508 07:07:41.026 | INFO | } | 20260508 07:07:41.026 | INFO | Perf report for the last 5.13699 seconds: | 20260508 07:07:41.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:41.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:07:41.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 07:07:41.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:41.026 | INFO | Uploaded 36GiB to GPU in 2.50686s (14.3606GiB/s) | 20260508 07:07:41.026 | INFO | Downloaded 34GiB to GPU in 2.42926s (13.9961GiB/s) | 20260508 07:07:46.167 | INFO | Analyzing chunk frames = [6190080,6190592) | 20260508 07:07:46.167 | INFO | MemInfo: { | 20260508 07:07:46.167 | INFO | "gpu" : " 17 GB", | 20260508 07:07:46.167 | INFO | "huge" : "", | 20260508 07:07:46.167 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:46.167 | INFO | "malloc" : " 14 GB", | 20260508 07:07:46.167 | INFO | "pinned" : " 29 GB" | 20260508 07:07:46.167 | INFO | } | 20260508 07:07:46.167 | INFO | Perf report for the last 5.1414 seconds: | 20260508 07:07:46.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:46.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.167 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:07:46.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.167 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:07:46.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:46.168 | INFO | Uploaded 36GiB to GPU in 2.57857s (13.9612GiB/s) | 20260508 07:07:46.168 | INFO | Downloaded 34GiB to GPU in 2.41761s (14.0635GiB/s) | 20260508 07:07:51.309 | INFO | Analyzing chunk frames = [6190592,6191104) | 20260508 07:07:51.309 | INFO | MemInfo: { | 20260508 07:07:51.309 | INFO | "gpu" : " 17 GB", | 20260508 07:07:51.309 | INFO | "huge" : "", | 20260508 07:07:51.309 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:51.309 | INFO | "malloc" : " 14 GB", | 20260508 07:07:51.309 | INFO | "pinned" : " 29 GB" | 20260508 07:07:51.309 | INFO | } | 20260508 07:07:51.309 | INFO | Perf report for the last 5.14185 seconds: | 20260508 07:07:51.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:51.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | ANALYSIS : 66.8%, 3.0, 103.5ms, 0 | 20260508 07:07:51.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:07:51.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:51.309 | INFO | Uploaded 36GiB to GPU in 2.59169s (13.8905GiB/s) | 20260508 07:07:51.309 | INFO | Downloaded 34GiB to GPU in 2.40691s (14.126GiB/s) | 20260508 07:07:56.454 | INFO | Analyzing chunk frames = [6191104,6191616) | 20260508 07:07:56.454 | INFO | MemInfo: { | 20260508 07:07:56.454 | INFO | "gpu" : " 17 GB", | 20260508 07:07:56.454 | INFO | "huge" : "", | 20260508 07:07:56.454 | INFO | "huge_shared" : " 29 GB", | 20260508 07:07:56.454 | INFO | "malloc" : " 14 GB", | 20260508 07:07:56.454 | INFO | "pinned" : " 29 GB" | 20260508 07:07:56.455 | INFO | } | 20260508 07:07:56.455 | INFO | Perf report for the last 5.14556 seconds: | 20260508 07:07:56.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:07:56.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:07:56.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:07:56.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:07:56.455 | INFO | Uploaded 36GiB to GPU in 2.28193s (15.7761GiB/s) | 20260508 07:07:56.455 | INFO | Downloaded 34GiB to GPU in 2.48599s (13.6766GiB/s) | 20260508 07:08:01.590 | INFO | Analyzing chunk frames = [6191616,6192128) | 20260508 07:08:01.590 | INFO | MemInfo: { | 20260508 07:08:01.590 | INFO | "gpu" : " 17 GB", | 20260508 07:08:01.590 | INFO | "huge" : "", | 20260508 07:08:01.590 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:01.590 | INFO | "malloc" : " 14 GB", | 20260508 07:08:01.590 | INFO | "pinned" : " 29 GB" | 20260508 07:08:01.590 | INFO | } | 20260508 07:08:01.590 | INFO | Perf report for the last 5.13562 seconds: | 20260508 07:08:01.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:01.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.590 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 07:08:01.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.590 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:08:01.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:01.591 | INFO | Uploaded 36GiB to GPU in 2.40588s (14.9634GiB/s) | 20260508 07:08:01.591 | INFO | Downloaded 34GiB to GPU in 2.42433s (14.0245GiB/s) | 20260508 07:08:06.726 | INFO | Analyzing chunk frames = [6192128,6192640) | 20260508 07:08:06.726 | INFO | MemInfo: { | 20260508 07:08:06.726 | INFO | "gpu" : " 17 GB", | 20260508 07:08:06.726 | INFO | "huge" : "", | 20260508 07:08:06.726 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:06.726 | INFO | "malloc" : " 14 GB", | 20260508 07:08:06.726 | INFO | "pinned" : " 29 GB" | 20260508 07:08:06.726 | INFO | } | 20260508 07:08:06.726 | INFO | Perf report for the last 5.13551 seconds: | 20260508 07:08:06.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:06.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:08:06.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:08:06.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:06.726 | INFO | Uploaded 36GiB to GPU in 2.4671s (14.592GiB/s) | 20260508 07:08:06.727 | INFO | Downloaded 34GiB to GPU in 2.42114s (14.043GiB/s) | 20260508 07:08:11.866 | INFO | Analyzing chunk frames = [6192640,6193152) | 20260508 07:08:11.866 | INFO | MemInfo: { | 20260508 07:08:11.866 | INFO | "gpu" : " 17 GB", | 20260508 07:08:11.866 | INFO | "huge" : "", | 20260508 07:08:11.866 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:11.866 | INFO | "malloc" : " 14 GB", | 20260508 07:08:11.866 | INFO | "pinned" : " 29 GB" | 20260508 07:08:11.866 | INFO | } | 20260508 07:08:11.866 | INFO | Perf report for the last 5.14053 seconds: | 20260508 07:08:11.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:11.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.866 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:08:11.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.867 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:08:11.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:11.867 | INFO | Uploaded 36GiB to GPU in 2.48767s (14.4714GiB/s) | 20260508 07:08:11.867 | INFO | Downloaded 34GiB to GPU in 2.43273s (13.9761GiB/s) | 20260508 07:08:11.867 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:08:11.867 | INFO | Total mem usage: 16.7 GiB | 20260508 07:08:11.867 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:08:11.867 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:08:11.867 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:08:11.867 | INFO | High Water Breakdown: | 20260508 07:08:11.867 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:08:11.867 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:08:11.867 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:08:11.867 | INFO | ------------------------------------------ | 20260508 07:08:11.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:08:11.867 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:08:11.867 | INFO | Total mem usage: 26.5 GiB | 20260508 07:08:11.867 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:08:11.867 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:08:11.867 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:08:11.867 | INFO | High Water Breakdown: | 20260508 07:08:11.867 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:08:11.867 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:08:11.867 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:08:11.867 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:08:11.867 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:08:11.867 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:08:11.867 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:08:11.867 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:08:11.867 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:08:11.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:08:11.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:08:11.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:08:11.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:08:11.867 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:08:11.867 | INFO | ------------------------------------------ | 20260508 07:08:11.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:08:11.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:08:11.867 | INFO | Total mem usage: 27.8 GiB | 20260508 07:08:11.867 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:08:11.867 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:08:11.867 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:08:11.868 | INFO | High Water Breakdown: | 20260508 07:08:11.868 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:08:11.868 | INFO | ------------------------------------------ | 20260508 07:08:11.868 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:08:11.868 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:08:11.868 | INFO | Total mem usage: 15.7 GiB | 20260508 07:08:11.868 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:08:11.868 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:08:11.868 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:08:11.868 | INFO | High Water Breakdown: | 20260508 07:08:11.868 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:08:11.868 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:08:11.868 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:08:11.868 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:08:11.868 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:08:11.868 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:08:11.868 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:08:11.868 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:08:11.868 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:08:11.868 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:08:11.868 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:08:11.868 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:08:11.868 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:08:11.868 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:08:11.868 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:08:11.868 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:08:11.868 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:08:11.868 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:08:11.868 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:08:11.868 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:08:11.868 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:08:11.868 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:08:11.868 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:08:11.868 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:08:11.868 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:08:11.868 | INFO | ------------------------------------------ | 20260508 07:08:11.868 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:08:11.868 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:08:11.868 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:08:11.868 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:08:11.868 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 5.75e-01 | 20260508 07:08:11.868 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.02e-01, 1.11e-02 / 5.08e-03 | 20260508 07:08:11.868 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.34e+00, 5.91e-01 / 2.42e-01 | 20260508 07:08:11.868 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.28e-01 | 20260508 07:08:11.868 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.52e+01, 3.72e+00 / 1.31e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.81e+00 / 1.43e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.50e+00, 4.07e-01 / 2.67e-01 | 20260508 07:08:11.868 | INFO | PerfReport: 3.13%, Metrics -- 1.02e+00 / 1.26e+00, 2.54e-01 / 1.60e-01 | 20260508 07:08:11.868 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.48e-01, 4.91e-02 / 3.10e-02 | 20260508 07:08:11.868 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:08:11.868 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:08:11.868 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:08:11.868 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 24.54%, Upload -- 2.60e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 24.22%, Download -- 2.57e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 2.49%, LockUpload -- 2.64e+00 / 0.00e+00, 5.86e+00 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 2.36%, LockDownload -- 2.51e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:08:11.868 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260508 07:08:11.974 | INFO | Starting a round of Baz IO | 20260508 07:08:15.390 | INFO | Populating RTMetrics report | 20260508 07:08:15.390 | INFO | RTMetrics report populated | 20260508 07:08:15.390 | INFO | Opening tmp RTMetrics json file | 20260508 07:08:15.402 | INFO | Serializing RTMetrics to json | 20260508 07:08:15.403 | INFO | Serialization done, swapping tmp file now... | 20260508 07:08:15.403 | INFO | RTMetrics done | 20260508 07:08:15.466 | INFO | Finished a round of Baz IO: Wrote 1.533 GiB in 3.492 seconds | 20260508 07:08:15.466 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.60% / 0.00% / 59.40% | 20260508 07:08:15.466 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.358323% / 0% / 4.37101e-05% | 20260508 07:08:15.466 | INFO | Baz IO overall: | 20260508 07:08:15.466 | INFO | BytesWritten : 2736826674445 | 20260508 07:08:15.466 | INFO | NumEvents : 1318889818798 | 20260508 07:08:17.000 | INFO | Analyzing chunk frames = [6193152,6193664) | 20260508 07:08:17.000 | INFO | MemInfo: { | 20260508 07:08:17.000 | INFO | "gpu" : " 17 GB", | 20260508 07:08:17.000 | INFO | "huge" : "", | 20260508 07:08:17.000 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:17.000 | INFO | "malloc" : " 14 GB", | 20260508 07:08:17.000 | INFO | "pinned" : " 29 GB" | 20260508 07:08:17.000 | INFO | } | 20260508 07:08:17.000 | INFO | Perf report for the last 5.13367 seconds: | 20260508 07:08:17.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:17.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:17.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:17.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:17.000 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 07:08:17.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:17.000 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260508 07:08:17.000 | INFO | PRE_HQ : 66.4%, 1.8, 61.1ms, 0 | 20260508 07:08:17.000 | INFO | BAZWRITER : 67.5%, 2.8, 95.8ms, 0 | 20260508 07:08:17.000 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:08:17.001 | INFO | Uploaded 36GiB to GPU in 2.58343s (13.935GiB/s) | 20260508 07:08:17.001 | INFO | Downloaded 38GiB to GPU in 2.68905s (14.1314GiB/s) | 20260508 07:08:22.142 | INFO | Analyzing chunk frames = [6193664,6194176) | 20260508 07:08:22.142 | INFO | MemInfo: { | 20260508 07:08:22.142 | INFO | "gpu" : " 17 GB", | 20260508 07:08:22.142 | INFO | "huge" : "", | 20260508 07:08:22.142 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:22.142 | INFO | "malloc" : " 14 GB", | 20260508 07:08:22.142 | INFO | "pinned" : " 29 GB" | 20260508 07:08:22.142 | INFO | } | 20260508 07:08:22.142 | INFO | Perf report for the last 5.14192 seconds: | 20260508 07:08:22.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:22.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 07:08:22.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:08:22.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:22.142 | INFO | Uploaded 36GiB to GPU in 2.58951s (13.9022GiB/s) | 20260508 07:08:22.142 | INFO | Downloaded 34GiB to GPU in 2.43748s (13.9488GiB/s) | 20260508 07:08:27.287 | INFO | Analyzing chunk frames = [6194176,6194688) | 20260508 07:08:27.287 | INFO | MemInfo: { | 20260508 07:08:27.287 | INFO | "gpu" : " 17 GB", | 20260508 07:08:27.287 | INFO | "huge" : "", | 20260508 07:08:27.287 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:27.287 | INFO | "malloc" : " 14 GB", | 20260508 07:08:27.287 | INFO | "pinned" : " 29 GB" | 20260508 07:08:27.287 | INFO | } | 20260508 07:08:27.287 | INFO | Perf report for the last 5.14532 seconds: | 20260508 07:08:27.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:27.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.287 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 07:08:27.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.287 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:08:27.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:27.288 | INFO | Uploaded 36GiB to GPU in 2.57763s (13.9663GiB/s) | 20260508 07:08:27.288 | INFO | Downloaded 34GiB to GPU in 2.41889s (14.0561GiB/s) | 20260508 07:08:32.420 | INFO | Analyzing chunk frames = [6194688,6195200) | 20260508 07:08:32.420 | INFO | MemInfo: { | 20260508 07:08:32.420 | INFO | "gpu" : " 17 GB", | 20260508 07:08:32.420 | INFO | "huge" : "", | 20260508 07:08:32.420 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:32.420 | INFO | "malloc" : " 14 GB", | 20260508 07:08:32.420 | INFO | "pinned" : " 29 GB" | 20260508 07:08:32.421 | INFO | } | 20260508 07:08:32.421 | INFO | Perf report for the last 5.13319 seconds: | 20260508 07:08:32.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:32.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 07:08:32.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:08:32.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:32.421 | INFO | Uploaded 36GiB to GPU in 2.52496s (14.2576GiB/s) | 20260508 07:08:32.421 | INFO | Downloaded 34GiB to GPU in 2.44093s (13.9291GiB/s) | 20260508 07:08:37.565 | INFO | Analyzing chunk frames = [6195200,6195712) | 20260508 07:08:37.565 | INFO | MemInfo: { | 20260508 07:08:37.565 | INFO | "gpu" : " 17 GB", | 20260508 07:08:37.565 | INFO | "huge" : "", | 20260508 07:08:37.565 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:37.565 | INFO | "malloc" : " 14 GB", | 20260508 07:08:37.565 | INFO | "pinned" : " 29 GB" | 20260508 07:08:37.565 | INFO | } | 20260508 07:08:37.565 | INFO | Perf report for the last 5.14447 seconds: | 20260508 07:08:37.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:37.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:08:37.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:08:37.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:37.565 | INFO | Uploaded 36GiB to GPU in 2.40864s (14.9462GiB/s) | 20260508 07:08:37.565 | INFO | Downloaded 34GiB to GPU in 2.50143s (13.5922GiB/s) | 20260508 07:08:42.703 | INFO | Analyzing chunk frames = [6195712,6196224) | 20260508 07:08:42.703 | INFO | MemInfo: { | 20260508 07:08:42.703 | INFO | "gpu" : " 17 GB", | 20260508 07:08:42.703 | INFO | "huge" : "", | 20260508 07:08:42.703 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:42.703 | INFO | "malloc" : " 14 GB", | 20260508 07:08:42.703 | INFO | "pinned" : " 29 GB" | 20260508 07:08:42.703 | INFO | } | 20260508 07:08:42.703 | INFO | Perf report for the last 5.13829 seconds: | 20260508 07:08:42.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:42.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.703 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:08:42.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.703 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:08:42.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:42.704 | INFO | Uploaded 36GiB to GPU in 2.38309s (15.1065GiB/s) | 20260508 07:08:42.704 | INFO | Downloaded 34GiB to GPU in 2.40903s (14.1135GiB/s) | 20260508 07:08:47.840 | INFO | Analyzing chunk frames = [6196224,6196736) | 20260508 07:08:47.840 | INFO | MemInfo: { | 20260508 07:08:47.840 | INFO | "gpu" : " 17 GB", | 20260508 07:08:47.840 | INFO | "huge" : "", | 20260508 07:08:47.840 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:47.840 | INFO | "malloc" : " 14 GB", | 20260508 07:08:47.840 | INFO | "pinned" : " 29 GB" | 20260508 07:08:47.840 | INFO | } | 20260508 07:08:47.840 | INFO | Perf report for the last 5.13664 seconds: | 20260508 07:08:47.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:47.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 07:08:47.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 07:08:47.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:47.840 | INFO | Uploaded 36GiB to GPU in 2.43711s (14.7716GiB/s) | 20260508 07:08:47.841 | INFO | Downloaded 34GiB to GPU in 2.44009s (13.9339GiB/s) | 20260508 07:08:52.978 | INFO | Analyzing chunk frames = [6196736,6197248) | 20260508 07:08:52.978 | INFO | MemInfo: { | 20260508 07:08:52.978 | INFO | "gpu" : " 17 GB", | 20260508 07:08:52.978 | INFO | "huge" : "", | 20260508 07:08:52.978 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:52.978 | INFO | "malloc" : " 14 GB", | 20260508 07:08:52.978 | INFO | "pinned" : " 29 GB" | 20260508 07:08:52.978 | INFO | } | 20260508 07:08:52.979 | INFO | Perf report for the last 5.13853 seconds: | 20260508 07:08:52.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:52.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:08:52.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260508 07:08:52.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:52.979 | INFO | Uploaded 36GiB to GPU in 2.51174s (14.3327GiB/s) | 20260508 07:08:52.979 | INFO | Downloaded 34GiB to GPU in 2.44627s (13.8987GiB/s) | 20260508 07:08:53.076 | INFO | Starting a round of Baz IO | 20260508 07:08:56.498 | INFO | Populating RTMetrics report | 20260508 07:08:56.498 | INFO | RTMetrics report populated | 20260508 07:08:56.498 | INFO | Opening tmp RTMetrics json file | 20260508 07:08:56.505 | INFO | Serializing RTMetrics to json | 20260508 07:08:56.507 | INFO | Serialization done, swapping tmp file now... | 20260508 07:08:56.507 | INFO | RTMetrics done | 20260508 07:08:56.569 | INFO | Finished a round of Baz IO: Wrote 1.533 GiB in 3.493 seconds | 20260508 07:08:56.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.60% / 0.00% / 59.40% | 20260508 07:08:56.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.35938% / 3.97364e-06% / 6.35783e-05% | 20260508 07:08:56.569 | INFO | Baz IO overall: | 20260508 07:08:56.569 | INFO | BytesWritten : 2738473152970 | 20260508 07:08:56.569 | INFO | NumEvents : 1319672658744 | 20260508 07:08:58.120 | INFO | Analyzing chunk frames = [6197248,6197760) | 20260508 07:08:58.120 | INFO | MemInfo: { | 20260508 07:08:58.120 | INFO | "gpu" : " 17 GB", | 20260508 07:08:58.120 | INFO | "huge" : "", | 20260508 07:08:58.120 | INFO | "huge_shared" : " 29 GB", | 20260508 07:08:58.120 | INFO | "malloc" : " 14 GB", | 20260508 07:08:58.120 | INFO | "pinned" : " 29 GB" | 20260508 07:08:58.120 | INFO | } | 20260508 07:08:58.120 | INFO | Perf report for the last 5.14187 seconds: | 20260508 07:08:58.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:08:58.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:58.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:58.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:58.121 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 07:08:58.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:08:58.121 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 07:08:58.121 | INFO | PRE_HQ : 65.4%, 1.6, 55.3ms, 0 | 20260508 07:08:58.121 | INFO | BAZWRITER : 67.4%, 2.6, 90.9ms, 0 | 20260508 07:08:58.121 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:08:58.121 | INFO | Uploaded 36GiB to GPU in 2.59665s (13.864GiB/s) | 20260508 07:08:58.121 | INFO | Downloaded 38GiB to GPU in 2.70618s (14.0419GiB/s) | 20260508 07:09:03.262 | INFO | Analyzing chunk frames = [6197760,6198272) | 20260508 07:09:03.262 | INFO | MemInfo: { | 20260508 07:09:03.262 | INFO | "gpu" : " 17 GB", | 20260508 07:09:03.262 | INFO | "huge" : "", | 20260508 07:09:03.262 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:03.262 | INFO | "malloc" : " 14 GB", | 20260508 07:09:03.262 | INFO | "pinned" : " 29 GB" | 20260508 07:09:03.262 | INFO | } | 20260508 07:09:03.262 | INFO | Perf report for the last 5.14179 seconds: | 20260508 07:09:03.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:03.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.262 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:09:03.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.262 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:09:03.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:03.263 | INFO | Uploaded 36GiB to GPU in 2.49285s (14.4413GiB/s) | 20260508 07:09:03.263 | INFO | Downloaded 34GiB to GPU in 2.43038s (13.9896GiB/s) | 20260508 07:09:08.405 | INFO | Analyzing chunk frames = [6198272,6198784) | 20260508 07:09:08.405 | INFO | MemInfo: { | 20260508 07:09:08.405 | INFO | "gpu" : " 17 GB", | 20260508 07:09:08.405 | INFO | "huge" : "", | 20260508 07:09:08.405 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:08.405 | INFO | "malloc" : " 14 GB", | 20260508 07:09:08.405 | INFO | "pinned" : " 29 GB" | 20260508 07:09:08.405 | INFO | } | 20260508 07:09:08.405 | INFO | Perf report for the last 5.14268 seconds: | 20260508 07:09:08.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:08.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 07:09:08.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:09:08.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:08.405 | INFO | Uploaded 36GiB to GPU in 2.57699s (13.9698GiB/s) | 20260508 07:09:08.405 | INFO | Downloaded 34GiB to GPU in 2.41914s (14.0546GiB/s) | 20260508 07:09:13.537 | INFO | Analyzing chunk frames = [6198784,6199296) | 20260508 07:09:13.537 | INFO | MemInfo: { | 20260508 07:09:13.537 | INFO | "gpu" : " 17 GB", | 20260508 07:09:13.537 | INFO | "huge" : "", | 20260508 07:09:13.537 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:13.537 | INFO | "malloc" : " 14 GB", | 20260508 07:09:13.537 | INFO | "pinned" : " 29 GB" | 20260508 07:09:13.538 | INFO | } | 20260508 07:09:13.538 | INFO | Perf report for the last 5.13265 seconds: | 20260508 07:09:13.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:13.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | ANALYSIS : 66.6%, 3.0, 103.0ms, 0 | 20260508 07:09:13.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:09:13.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:13.538 | INFO | Uploaded 36GiB to GPU in 2.50482s (14.3723GiB/s) | 20260508 07:09:13.538 | INFO | Downloaded 34GiB to GPU in 2.43456s (13.9656GiB/s) | 20260508 07:09:18.678 | INFO | Analyzing chunk frames = [6199296,6199808) | 20260508 07:09:18.678 | INFO | MemInfo: { | 20260508 07:09:18.678 | INFO | "gpu" : " 17 GB", | 20260508 07:09:18.678 | INFO | "huge" : "", | 20260508 07:09:18.678 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:18.678 | INFO | "malloc" : " 14 GB", | 20260508 07:09:18.678 | INFO | "pinned" : " 29 GB" | 20260508 07:09:18.678 | INFO | } | 20260508 07:09:18.678 | INFO | Perf report for the last 5.14054 seconds: | 20260508 07:09:18.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:18.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.678 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 07:09:18.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.678 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:09:18.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:18.679 | INFO | Uploaded 36GiB to GPU in 2.42856s (14.8236GiB/s) | 20260508 07:09:18.679 | INFO | Downloaded 34GiB to GPU in 2.52501s (13.4653GiB/s) | 20260508 07:09:23.815 | INFO | Analyzing chunk frames = [6199808,6200320) | 20260508 07:09:23.815 | INFO | MemInfo: { | 20260508 07:09:23.815 | INFO | "gpu" : " 17 GB", | 20260508 07:09:23.815 | INFO | "huge" : "", | 20260508 07:09:23.815 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:23.815 | INFO | "malloc" : " 14 GB", | 20260508 07:09:23.815 | INFO | "pinned" : " 29 GB" | 20260508 07:09:23.815 | INFO | } | 20260508 07:09:23.815 | INFO | Perf report for the last 5.13692 seconds: | 20260508 07:09:23.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:23.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.815 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:09:23.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.815 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:09:23.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:23.816 | INFO | Uploaded 36GiB to GPU in 2.36067s (15.2499GiB/s) | 20260508 07:09:23.816 | INFO | Downloaded 34GiB to GPU in 2.43246s (13.9776GiB/s) | 20260508 07:09:28.961 | INFO | Analyzing chunk frames = [6200320,6200832) | 20260508 07:09:28.962 | INFO | MemInfo: { | 20260508 07:09:28.962 | INFO | "gpu" : " 17 GB", | 20260508 07:09:28.962 | INFO | "huge" : "", | 20260508 07:09:28.962 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:28.962 | INFO | "malloc" : " 14 GB", | 20260508 07:09:28.962 | INFO | "pinned" : " 29 GB" | 20260508 07:09:28.962 | INFO | } | 20260508 07:09:28.962 | INFO | Perf report for the last 5.14668 seconds: | 20260508 07:09:28.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:28.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 07:09:28.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260508 07:09:28.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:28.962 | INFO | Uploaded 36GiB to GPU in 2.48709s (14.4748GiB/s) | 20260508 07:09:28.962 | INFO | Downloaded 34GiB to GPU in 2.44424s (13.9103GiB/s) | 20260508 07:09:34.096 | INFO | Analyzing chunk frames = [6200832,6201344) | 20260508 07:09:34.096 | INFO | MemInfo: { | 20260508 07:09:34.096 | INFO | "gpu" : " 17 GB", | 20260508 07:09:34.096 | INFO | "huge" : "", | 20260508 07:09:34.096 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:34.096 | INFO | "malloc" : " 14 GB", | 20260508 07:09:34.096 | INFO | "pinned" : " 29 GB" | 20260508 07:09:34.096 | INFO | } | 20260508 07:09:34.097 | INFO | Perf report for the last 5.13475 seconds: | 20260508 07:09:34.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:34.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 07:09:34.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 07:09:34.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:34.097 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:09:34.097 | INFO | Total mem usage: 16.7 GiB | 20260508 07:09:34.097 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:09:34.097 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:09:34.097 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:09:34.097 | INFO | High Water Breakdown: | 20260508 07:09:34.097 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:09:34.097 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:09:34.097 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:09:34.097 | INFO | ------------------------------------------ | 20260508 07:09:34.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:09:34.097 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:09:34.097 | INFO | Total mem usage: 26.5 GiB | 20260508 07:09:34.097 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:09:34.097 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:09:34.097 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:09:34.097 | INFO | High Water Breakdown: | 20260508 07:09:34.097 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:09:34.097 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:09:34.097 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:09:34.097 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:09:34.097 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:09:34.097 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:09:34.097 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:09:34.097 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:09:34.097 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:09:34.097 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:09:34.097 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:09:34.097 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:09:34.097 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:09:34.097 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:09:34.097 | INFO | ------------------------------------------ | 20260508 07:09:34.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:09:34.097 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:09:34.097 | INFO | Total mem usage: 27.8 GiB | 20260508 07:09:34.097 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:09:34.097 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:09:34.097 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:09:34.097 | INFO | High Water Breakdown: | 20260508 07:09:34.097 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:09:34.097 | INFO | ------------------------------------------ | 20260508 07:09:34.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:09:34.097 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:09:34.097 | INFO | Total mem usage: 15.7 GiB | 20260508 07:09:34.097 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:09:34.097 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:09:34.097 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:09:34.097 | INFO | High Water Breakdown: | 20260508 07:09:34.097 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:09:34.097 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:09:34.097 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:09:34.097 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:09:34.097 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:09:34.097 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:09:34.097 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:09:34.097 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:09:34.097 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:09:34.097 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:09:34.097 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:09:34.097 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:09:34.097 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:09:34.097 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:09:34.097 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:09:34.097 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:09:34.097 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:09:34.097 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:09:34.097 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:09:34.097 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:09:34.098 | INFO | Uploaded 36GiB to GPU in 2.43016s (14.8139GiB/s) | 20260508 07:09:34.098 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:09:34.098 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:09:34.098 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:09:34.098 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:09:34.098 | INFO | Downloaded 34GiB to GPU in 2.44411s (13.911GiB/s) | 20260508 07:09:34.098 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:09:34.098 | INFO | ------------------------------------------ | 20260508 07:09:34.098 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:09:34.098 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:09:34.098 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:09:34.098 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:09:34.098 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.39e+01 / 5.87e-01 | 20260508 07:09:34.098 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.08e-01, 1.16e-02 / 1.68e-02 | 20260508 07:09:34.098 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.42e+00, 5.91e-01 / 2.59e-01 | 20260508 07:09:34.098 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.07e-01 / 1.22e-01 | 20260508 07:09:34.098 | INFO | PerfReport: 15.89%, DME -- 4.16e+01 / 4.49e+01, 3.61e+00 / 1.01e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.86e+00 / 1.52e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.36e+00 / 2.57e+00, 4.12e-01 / 2.59e-01 | 20260508 07:09:34.098 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.21e+00, 2.53e-01 / 1.30e-01 | 20260508 07:09:34.098 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.50e-01, 4.94e-02 / 2.63e-02 | 20260508 07:09:34.098 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:09:34.098 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:09:34.098 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:09:34.098 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 24.51%, Upload -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 30.92%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 2.41%, LockUpload -- 2.56e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 2.31%, LockDownload -- 2.45e+00 / 0.00e+00, 3.73e+00 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 15.50%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:09:34.098 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260508 07:09:34.209 | INFO | Starting a round of Baz IO | 20260508 07:09:37.628 | INFO | Populating RTMetrics report | 20260508 07:09:37.628 | INFO | RTMetrics report populated | 20260508 07:09:37.628 | INFO | Opening tmp RTMetrics json file | 20260508 07:09:37.629 | INFO | Serializing RTMetrics to json | 20260508 07:09:37.630 | INFO | Serialization done, swapping tmp file now... | 20260508 07:09:37.630 | INFO | RTMetrics done | 20260508 07:09:37.694 | INFO | Finished a round of Baz IO: Wrote 1.534 GiB in 3.484 seconds | 20260508 07:09:37.694 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.00% / 59.40% | 20260508 07:09:37.694 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.358899% / 0% / 6.75519e-05% | 20260508 07:09:37.694 | INFO | Baz IO overall: | 20260508 07:09:37.694 | INFO | BytesWritten : 2740120309781 | 20260508 07:09:37.694 | INFO | NumEvents : 1320455990113 | 20260508 07:09:39.232 | INFO | Analyzing chunk frames = [6201344,6201856) | 20260508 07:09:39.232 | INFO | MemInfo: { | 20260508 07:09:39.232 | INFO | "gpu" : " 17 GB", | 20260508 07:09:39.232 | INFO | "huge" : "", | 20260508 07:09:39.232 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:39.232 | INFO | "malloc" : " 14 GB", | 20260508 07:09:39.232 | INFO | "pinned" : " 29 GB" | 20260508 07:09:39.232 | INFO | } | 20260508 07:09:39.232 | INFO | Perf report for the last 5.13547 seconds: | 20260508 07:09:39.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:39.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:39.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:39.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:39.232 | INFO | ANALYSIS : 68.4%, 3.0, 106.0ms, 0 | 20260508 07:09:39.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:39.232 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 07:09:39.232 | INFO | PRE_HQ : 67.0%, 1.8, 61.8ms, 0 | 20260508 07:09:39.232 | INFO | BAZWRITER : 67.4%, 2.8, 96.3ms, 0 | 20260508 07:09:39.232 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:09:39.233 | INFO | Uploaded 36GiB to GPU in 2.53427s (14.2053GiB/s) | 20260508 07:09:39.233 | INFO | Downloaded 38GiB to GPU in 2.70961s (14.0241GiB/s) | 20260508 07:09:44.370 | INFO | Analyzing chunk frames = [6201856,6202368) | 20260508 07:09:44.370 | INFO | MemInfo: { | 20260508 07:09:44.370 | INFO | "gpu" : " 17 GB", | 20260508 07:09:44.370 | INFO | "huge" : "", | 20260508 07:09:44.370 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:44.370 | INFO | "malloc" : " 14 GB", | 20260508 07:09:44.370 | INFO | "pinned" : " 29 GB" | 20260508 07:09:44.371 | INFO | } | 20260508 07:09:44.371 | INFO | Perf report for the last 5.13861 seconds: | 20260508 07:09:44.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:44.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:09:44.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:09:44.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:44.371 | INFO | Uploaded 36GiB to GPU in 2.47585s (14.5404GiB/s) | 20260508 07:09:44.371 | INFO | Downloaded 34GiB to GPU in 2.43634s (13.9553GiB/s) | 20260508 07:09:49.517 | INFO | Analyzing chunk frames = [6202368,6202880) | 20260508 07:09:49.517 | INFO | MemInfo: { | 20260508 07:09:49.517 | INFO | "gpu" : " 17 GB", | 20260508 07:09:49.517 | INFO | "huge" : "", | 20260508 07:09:49.517 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:49.517 | INFO | "malloc" : " 14 GB", | 20260508 07:09:49.517 | INFO | "pinned" : " 29 GB" | 20260508 07:09:49.517 | INFO | } | 20260508 07:09:49.517 | INFO | Perf report for the last 5.14681 seconds: | 20260508 07:09:49.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:49.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.518 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 07:09:49.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.518 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:09:49.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:49.518 | INFO | Uploaded 36GiB to GPU in 2.54236s (14.1601GiB/s) | 20260508 07:09:49.518 | INFO | Downloaded 34GiB to GPU in 2.41317s (14.0894GiB/s) | 20260508 07:09:54.653 | INFO | Analyzing chunk frames = [6202880,6203392) | 20260508 07:09:54.653 | INFO | MemInfo: { | 20260508 07:09:54.653 | INFO | "gpu" : " 17 GB", | 20260508 07:09:54.653 | INFO | "huge" : "", | 20260508 07:09:54.653 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:54.653 | INFO | "malloc" : " 14 GB", | 20260508 07:09:54.653 | INFO | "pinned" : " 29 GB" | 20260508 07:09:54.653 | INFO | } | 20260508 07:09:54.653 | INFO | Perf report for the last 5.13561 seconds: | 20260508 07:09:54.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:54.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.653 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:09:54.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.653 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:09:54.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:54.654 | INFO | Uploaded 36GiB to GPU in 2.56446s (14.038GiB/s) | 20260508 07:09:54.654 | INFO | Downloaded 34GiB to GPU in 2.42508s (14.0202GiB/s) | 20260508 07:09:59.791 | INFO | Analyzing chunk frames = [6203392,6203904) | 20260508 07:09:59.791 | INFO | MemInfo: { | 20260508 07:09:59.791 | INFO | "gpu" : " 17 GB", | 20260508 07:09:59.791 | INFO | "huge" : "", | 20260508 07:09:59.791 | INFO | "huge_shared" : " 29 GB", | 20260508 07:09:59.791 | INFO | "malloc" : " 14 GB", | 20260508 07:09:59.792 | INFO | "pinned" : " 29 GB" | 20260508 07:09:59.792 | INFO | } | 20260508 07:09:59.792 | INFO | Perf report for the last 5.13854 seconds: | 20260508 07:09:59.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:09:59.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 07:09:59.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 07:09:59.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:09:59.792 | INFO | Uploaded 36GiB to GPU in 2.30725s (15.603GiB/s) | 20260508 07:09:59.792 | INFO | Downloaded 34GiB to GPU in 2.48716s (13.6702GiB/s) | 20260508 07:10:04.929 | INFO | Analyzing chunk frames = [6203904,6204416) | 20260508 07:10:04.929 | INFO | MemInfo: { | 20260508 07:10:04.929 | INFO | "gpu" : " 17 GB", | 20260508 07:10:04.929 | INFO | "huge" : "", | 20260508 07:10:04.929 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:04.929 | INFO | "malloc" : " 14 GB", | 20260508 07:10:04.929 | INFO | "pinned" : " 29 GB" | 20260508 07:10:04.929 | INFO | } | 20260508 07:10:04.929 | INFO | Perf report for the last 5.13753 seconds: | 20260508 07:10:04.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:04.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 07:10:04.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:10:04.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:04.929 | INFO | Uploaded 36GiB to GPU in 2.38646s (15.0851GiB/s) | 20260508 07:10:04.930 | INFO | Downloaded 34GiB to GPU in 2.43001s (13.9917GiB/s) | 20260508 07:10:10.071 | INFO | Analyzing chunk frames = [6204416,6204928) | 20260508 07:10:10.072 | INFO | MemInfo: { | 20260508 07:10:10.072 | INFO | "gpu" : " 17 GB", | 20260508 07:10:10.072 | INFO | "huge" : "", | 20260508 07:10:10.072 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:10.072 | INFO | "malloc" : " 14 GB", | 20260508 07:10:10.072 | INFO | "pinned" : " 29 GB" | 20260508 07:10:10.072 | INFO | } | 20260508 07:10:10.072 | INFO | Perf report for the last 5.14265 seconds: | 20260508 07:10:10.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:10.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 07:10:10.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 07:10:10.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:10.072 | INFO | Uploaded 36GiB to GPU in 2.46876s (14.5822GiB/s) | 20260508 07:10:10.072 | INFO | Downloaded 34GiB to GPU in 2.43456s (13.9656GiB/s) | 20260508 07:10:15.213 | INFO | Analyzing chunk frames = [6204928,6205440) | 20260508 07:10:15.213 | INFO | MemInfo: { | 20260508 07:10:15.213 | INFO | "gpu" : " 17 GB", | 20260508 07:10:15.213 | INFO | "huge" : "", | 20260508 07:10:15.213 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:15.213 | INFO | "malloc" : " 14 GB", | 20260508 07:10:15.213 | INFO | "pinned" : " 29 GB" | 20260508 07:10:15.213 | INFO | } | 20260508 07:10:15.213 | INFO | Perf report for the last 5.14113 seconds: | 20260508 07:10:15.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:15.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 07:10:15.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 07:10:15.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:15.213 | INFO | Uploaded 36GiB to GPU in 2.48657s (14.4778GiB/s) | 20260508 07:10:15.213 | INFO | Downloaded 34GiB to GPU in 2.46618s (13.7865GiB/s) | 20260508 07:10:15.310 | INFO | Starting a round of Baz IO | 20260508 07:10:18.726 | INFO | Populating RTMetrics report | 20260508 07:10:18.727 | INFO | RTMetrics report populated | 20260508 07:10:18.727 | INFO | Opening tmp RTMetrics json file | 20260508 07:10:18.738 | INFO | Serializing RTMetrics to json | 20260508 07:10:18.740 | INFO | Serialization done, swapping tmp file now... | 20260508 07:10:18.740 | INFO | RTMetrics done | 20260508 07:10:18.804 | INFO | Finished a round of Baz IO: Wrote 1.533 GiB in 3.493 seconds | 20260508 07:10:18.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.00% / 59.41% | 20260508 07:10:18.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.358566% / 0% / 5.16574e-05% | 20260508 07:10:18.804 | INFO | Baz IO overall: | 20260508 07:10:18.804 | INFO | BytesWritten : 2741766889381 | 20260508 07:10:18.804 | INFO | NumEvents : 1321238642620 | 20260508 07:10:20.354 | INFO | Analyzing chunk frames = [6205440,6205952) | 20260508 07:10:20.354 | INFO | MemInfo: { | 20260508 07:10:20.354 | INFO | "gpu" : " 17 GB", | 20260508 07:10:20.354 | INFO | "huge" : "", | 20260508 07:10:20.354 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:20.354 | INFO | "malloc" : " 14 GB", | 20260508 07:10:20.354 | INFO | "pinned" : " 29 GB" | 20260508 07:10:20.354 | INFO | } | 20260508 07:10:20.354 | INFO | Perf report for the last 5.14091 seconds: | 20260508 07:10:20.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:20.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:20.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:20.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:20.354 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:10:20.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:20.354 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 07:10:20.354 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 07:10:20.354 | INFO | BAZWRITER : 67.2%, 2.9, 99.0ms, 0 | 20260508 07:10:20.354 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:10:20.354 | INFO | Uploaded 36GiB to GPU in 2.57381s (13.9871GiB/s) | 20260508 07:10:20.354 | INFO | Downloaded 38GiB to GPU in 2.70543s (14.0458GiB/s) | 20260508 07:10:25.482 | INFO | Analyzing chunk frames = [6205952,6206464) | 20260508 07:10:25.482 | INFO | MemInfo: { | 20260508 07:10:25.482 | INFO | "gpu" : " 17 GB", | 20260508 07:10:25.482 | INFO | "huge" : "", | 20260508 07:10:25.482 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:25.482 | INFO | "malloc" : " 14 GB", | 20260508 07:10:25.482 | INFO | "pinned" : " 29 GB" | 20260508 07:10:25.483 | INFO | } | 20260508 07:10:25.483 | INFO | Perf report for the last 5.12864 seconds: | 20260508 07:10:25.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:25.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 07:10:25.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:10:25.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:25.483 | INFO | Uploaded 36GiB to GPU in 2.49377s (14.436GiB/s) | 20260508 07:10:25.483 | INFO | Downloaded 34GiB to GPU in 2.4388s (13.9413GiB/s) | 20260508 07:10:30.631 | INFO | Analyzing chunk frames = [6206464,6206976) | 20260508 07:10:30.631 | INFO | MemInfo: { | 20260508 07:10:30.631 | INFO | "gpu" : " 17 GB", | 20260508 07:10:30.631 | INFO | "huge" : "", | 20260508 07:10:30.631 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:30.631 | INFO | "malloc" : " 14 GB", | 20260508 07:10:30.631 | INFO | "pinned" : " 29 GB" | 20260508 07:10:30.631 | INFO | } | 20260508 07:10:30.631 | INFO | Perf report for the last 5.14879 seconds: | 20260508 07:10:30.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:30.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.631 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 07:10:30.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.632 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:10:30.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:30.632 | INFO | Uploaded 36GiB to GPU in 2.5633s (14.0444GiB/s) | 20260508 07:10:30.632 | INFO | Downloaded 34GiB to GPU in 2.43677s (13.9529GiB/s) | 20260508 07:10:35.772 | INFO | Analyzing chunk frames = [6206976,6207488) | 20260508 07:10:35.772 | INFO | MemInfo: { | 20260508 07:10:35.772 | INFO | "gpu" : " 17 GB", | 20260508 07:10:35.772 | INFO | "huge" : "", | 20260508 07:10:35.772 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:35.772 | INFO | "malloc" : " 14 GB", | 20260508 07:10:35.772 | INFO | "pinned" : " 29 GB" | 20260508 07:10:35.772 | INFO | } | 20260508 07:10:35.772 | INFO | Perf report for the last 5.14087 seconds: | 20260508 07:10:35.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:35.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.773 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:10:35.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.773 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:10:35.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:35.773 | INFO | Uploaded 36GiB to GPU in 2.57555s (13.9776GiB/s) | 20260508 07:10:35.773 | INFO | Downloaded 34GiB to GPU in 2.43201s (13.9802GiB/s) | 20260508 07:10:40.908 | INFO | Analyzing chunk frames = [6207488,6208000) | 20260508 07:10:40.908 | INFO | MemInfo: { | 20260508 07:10:40.908 | INFO | "gpu" : " 17 GB", | 20260508 07:10:40.908 | INFO | "huge" : "", | 20260508 07:10:40.908 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:40.908 | INFO | "malloc" : " 14 GB", | 20260508 07:10:40.908 | INFO | "pinned" : " 29 GB" | 20260508 07:10:40.908 | INFO | } | 20260508 07:10:40.908 | INFO | Perf report for the last 5.13617 seconds: | 20260508 07:10:40.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:40.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.909 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:10:40.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.909 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 07:10:40.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:40.909 | INFO | Uploaded 36GiB to GPU in 2.2922s (15.7054GiB/s) | 20260508 07:10:40.909 | INFO | Downloaded 34GiB to GPU in 2.49897s (13.6056GiB/s) | 20260508 07:10:46.050 | INFO | Analyzing chunk frames = [6208000,6208512) | 20260508 07:10:46.050 | INFO | MemInfo: { | 20260508 07:10:46.050 | INFO | "gpu" : " 17 GB", | 20260508 07:10:46.050 | INFO | "huge" : "", | 20260508 07:10:46.050 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:46.050 | INFO | "malloc" : " 14 GB", | 20260508 07:10:46.050 | INFO | "pinned" : " 29 GB" | 20260508 07:10:46.050 | INFO | } | 20260508 07:10:46.050 | INFO | Perf report for the last 5.14201 seconds: | 20260508 07:10:46.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:46.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:10:46.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:10:46.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:46.051 | INFO | Uploaded 36GiB to GPU in 2.47364s (14.5534GiB/s) | 20260508 07:10:46.051 | INFO | Downloaded 34GiB to GPU in 2.43553s (13.96GiB/s) | 20260508 07:10:51.187 | INFO | Analyzing chunk frames = [6208512,6209024) | 20260508 07:10:51.187 | INFO | MemInfo: { | 20260508 07:10:51.187 | INFO | "gpu" : " 17 GB", | 20260508 07:10:51.187 | INFO | "huge" : "", | 20260508 07:10:51.187 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:51.187 | INFO | "malloc" : " 14 GB", | 20260508 07:10:51.187 | INFO | "pinned" : " 29 GB" | 20260508 07:10:51.188 | INFO | } | 20260508 07:10:51.188 | INFO | Perf report for the last 5.13712 seconds: | 20260508 07:10:51.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:51.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 07:10:51.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260508 07:10:51.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:51.188 | INFO | Uploaded 36GiB to GPU in 2.42692s (14.8336GiB/s) | 20260508 07:10:51.188 | INFO | Downloaded 34GiB to GPU in 2.42814s (14.0025GiB/s) | 20260508 07:10:56.325 | INFO | Analyzing chunk frames = [6209024,6209536) | 20260508 07:10:56.325 | INFO | MemInfo: { | 20260508 07:10:56.326 | INFO | "gpu" : " 17 GB", | 20260508 07:10:56.326 | INFO | "huge" : "", | 20260508 07:10:56.326 | INFO | "huge_shared" : " 29 GB", | 20260508 07:10:56.326 | INFO | "malloc" : " 14 GB", | 20260508 07:10:56.326 | INFO | "pinned" : " 29 GB" | 20260508 07:10:56.326 | INFO | } | 20260508 07:10:56.326 | INFO | Perf report for the last 5.13831 seconds: | 20260508 07:10:56.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:10:56.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 07:10:56.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 07:10:56.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:10:56.326 | INFO | Uploaded 36GiB to GPU in 2.46138s (14.626GiB/s) | 20260508 07:10:56.326 | INFO | Downloaded 34GiB to GPU in 2.42605s (14.0146GiB/s) | 20260508 07:10:56.327 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:10:56.327 | INFO | Total mem usage: 16.7 GiB | 20260508 07:10:56.327 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:10:56.327 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.7%) | 20260508 07:10:56.327 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:10:56.327 | INFO | High Water Breakdown: | 20260508 07:10:56.327 | INFO | 59.8% : 8.69 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:10:56.327 | INFO | 40.1% : 5.83 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:10:56.327 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:10:56.327 | INFO | ------------------------------------------ | 20260508 07:10:56.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:10:56.327 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:10:56.327 | INFO | Total mem usage: 26.5 GiB | 20260508 07:10:56.327 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:10:56.327 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:10:56.327 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:10:56.327 | INFO | High Water Breakdown: | 20260508 07:10:56.327 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:10:56.327 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:10:56.327 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:10:56.327 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:10:56.327 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:10:56.327 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:10:56.327 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:10:56.327 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:10:56.327 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:10:56.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:10:56.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:10:56.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:10:56.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:10:56.327 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:10:56.327 | INFO | ------------------------------------------ | 20260508 07:10:56.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:10:56.327 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:10:56.327 | INFO | Total mem usage: 27.8 GiB | 20260508 07:10:56.327 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:10:56.327 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:10:56.327 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:10:56.327 | INFO | High Water Breakdown: | 20260508 07:10:56.327 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:10:56.327 | INFO | ------------------------------------------ | 20260508 07:10:56.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:10:56.327 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:10:56.328 | INFO | Total mem usage: 15.7 GiB | 20260508 07:10:56.328 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:10:56.328 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:10:56.328 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:10:56.328 | INFO | High Water Breakdown: | 20260508 07:10:56.328 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:10:56.328 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:10:56.328 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:10:56.328 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:10:56.328 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:10:56.328 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:10:56.328 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:10:56.328 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:10:56.328 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:10:56.328 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:10:56.328 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:10:56.328 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:10:56.328 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:10:56.328 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:10:56.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:10:56.328 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:10:56.328 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:10:56.328 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:10:56.328 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:10:56.328 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:10:56.328 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:10:56.328 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:10:56.328 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:10:56.328 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:10:56.328 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:10:56.328 | INFO | ------------------------------------------ | 20260508 07:10:56.328 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:10:56.328 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:10:56.328 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:10:56.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:10:56.328 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 6.79e-01 | 20260508 07:10:56.328 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.15e-02 / 4.35e-03 | 20260508 07:10:56.328 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.41e+00, 5.90e-01 / 2.13e-01 | 20260508 07:10:56.328 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.50e+00, 2.09e-01 / 8.52e-02 | 20260508 07:10:56.328 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.56e+01, 3.82e+00 / 1.25e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.87e+01 / 1.96e+01, 1.84e+00 / 1.33e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.37e+00 / 2.63e+00, 4.11e-01 / 2.38e-01 | 20260508 07:10:56.328 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.20e+00, 2.51e-01 / 1.26e-01 | 20260508 07:10:56.328 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.47e-01, 4.85e-02 / 2.13e-02 | 20260508 07:10:56.328 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:10:56.328 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:10:56.328 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:10:56.328 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 24.37%, Upload -- 2.58e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 24.36%, Download -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 30.94%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 2.23%, LockUpload -- 2.36e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 2.56%, LockDownload -- 2.71e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 15.53%, LockCompute -- 1.64e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 07:10:56.328 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 07:10:56.438 | INFO | Starting a round of Baz IO | 20260508 07:10:59.854 | INFO | Populating RTMetrics report | 20260508 07:10:59.854 | INFO | RTMetrics report populated | 20260508 07:10:59.854 | INFO | Opening tmp RTMetrics json file | 20260508 07:10:59.867 | INFO | Serializing RTMetrics to json | 20260508 07:10:59.869 | INFO | Serialization done, swapping tmp file now... | 20260508 07:10:59.869 | INFO | RTMetrics done | 20260508 07:10:59.922 | INFO | Finished a round of Baz IO: Wrote 1.534 GiB in 3.483 seconds | 20260508 07:10:59.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.00% / 59.41% | 20260508 07:10:59.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.356805% / 3.97364e-06% / 4.76837e-05% | 20260508 07:10:59.922 | INFO | Baz IO overall: | 20260508 07:10:59.922 | INFO | BytesWritten : 2743413645988 | 20260508 07:10:59.922 | INFO | NumEvents : 1322021318178 | 20260508 07:11:01.464 | INFO | Analyzing chunk frames = [6209536,6210048) | 20260508 07:11:01.464 | INFO | MemInfo: { | 20260508 07:11:01.464 | INFO | "gpu" : " 17 GB", | 20260508 07:11:01.464 | INFO | "huge" : "", | 20260508 07:11:01.464 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:01.464 | INFO | "malloc" : " 14 GB", | 20260508 07:11:01.464 | INFO | "pinned" : " 29 GB" | 20260508 07:11:01.464 | INFO | } | 20260508 07:11:01.464 | INFO | Perf report for the last 5.13811 seconds: | 20260508 07:11:01.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:01.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:01.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:01.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:01.464 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 07:11:01.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:01.464 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 07:11:01.464 | INFO | PRE_HQ : 66.4%, 1.8, 61.6ms, 0 | 20260508 07:11:01.464 | INFO | BAZWRITER : 67.1%, 2.8, 96.9ms, 0 | 20260508 07:11:01.464 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:11:01.464 | INFO | Uploaded 36GiB to GPU in 2.60623s (13.813GiB/s) | 20260508 07:11:01.465 | INFO | Downloaded 38GiB to GPU in 2.69236s (14.114GiB/s) | 20260508 07:11:06.601 | INFO | Analyzing chunk frames = [6210048,6210560) | 20260508 07:11:06.601 | INFO | MemInfo: { | 20260508 07:11:06.601 | INFO | "gpu" : " 17 GB", | 20260508 07:11:06.601 | INFO | "huge" : "", | 20260508 07:11:06.601 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:06.601 | INFO | "malloc" : " 14 GB", | 20260508 07:11:06.601 | INFO | "pinned" : " 29 GB" | 20260508 07:11:06.601 | INFO | } | 20260508 07:11:06.601 | INFO | Perf report for the last 5.13719 seconds: | 20260508 07:11:06.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:06.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.601 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:11:06.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.601 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:11:06.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:06.602 | INFO | Uploaded 36GiB to GPU in 2.4788s (14.5231GiB/s) | 20260508 07:11:06.602 | INFO | Downloaded 34GiB to GPU in 2.42363s (14.0286GiB/s) | 20260508 07:11:11.740 | INFO | Analyzing chunk frames = [6210560,6211072) | 20260508 07:11:11.740 | INFO | MemInfo: { | 20260508 07:11:11.740 | INFO | "gpu" : " 17 GB", | 20260508 07:11:11.740 | INFO | "huge" : "", | 20260508 07:11:11.740 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:11.740 | INFO | "malloc" : " 14 GB", | 20260508 07:11:11.740 | INFO | "pinned" : " 29 GB" | 20260508 07:11:11.740 | INFO | } | 20260508 07:11:11.740 | INFO | Perf report for the last 5.13875 seconds: | 20260508 07:11:11.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:11.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 07:11:11.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:11:11.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:11.740 | INFO | Uploaded 36GiB to GPU in 2.566s (14.0296GiB/s) | 20260508 07:11:11.740 | INFO | Downloaded 34GiB to GPU in 2.42471s (14.0223GiB/s) | 20260508 07:11:16.885 | INFO | Analyzing chunk frames = [6211072,6211584) | 20260508 07:11:16.885 | INFO | MemInfo: { | 20260508 07:11:16.885 | INFO | "gpu" : " 17 GB", | 20260508 07:11:16.885 | INFO | "huge" : "", | 20260508 07:11:16.885 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:16.885 | INFO | "malloc" : " 14 GB", | 20260508 07:11:16.885 | INFO | "pinned" : " 29 GB" | 20260508 07:11:16.885 | INFO | } | 20260508 07:11:16.885 | INFO | Perf report for the last 5.14492 seconds: | 20260508 07:11:16.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:16.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.885 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 07:11:16.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.885 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:11:16.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:16.886 | INFO | Uploaded 36GiB to GPU in 2.58345s (13.9348GiB/s) | 20260508 07:11:16.886 | INFO | Downloaded 34GiB to GPU in 2.40825s (14.1181GiB/s) | 20260508 07:11:22.026 | INFO | Analyzing chunk frames = [6211584,6212096) | 20260508 07:11:22.026 | INFO | MemInfo: { | 20260508 07:11:22.026 | INFO | "gpu" : " 17 GB", | 20260508 07:11:22.026 | INFO | "huge" : "", | 20260508 07:11:22.026 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:22.026 | INFO | "malloc" : " 14 GB", | 20260508 07:11:22.026 | INFO | "pinned" : " 29 GB" | 20260508 07:11:22.026 | INFO | } | 20260508 07:11:22.026 | INFO | Perf report for the last 5.14085 seconds: | 20260508 07:11:22.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:22.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 07:11:22.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 07:11:22.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:22.026 | INFO | Uploaded 36GiB to GPU in 2.27974s (15.7913GiB/s) | 20260508 07:11:22.026 | INFO | Downloaded 34GiB to GPU in 2.4973s (13.6147GiB/s) | 20260508 07:11:27.161 | INFO | Analyzing chunk frames = [6212096,6212608) | 20260508 07:11:27.161 | INFO | MemInfo: { | 20260508 07:11:27.161 | INFO | "gpu" : " 17 GB", | 20260508 07:11:27.161 | INFO | "huge" : "", | 20260508 07:11:27.161 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:27.161 | INFO | "malloc" : " 14 GB", | 20260508 07:11:27.161 | INFO | "pinned" : " 29 GB" | 20260508 07:11:27.161 | INFO | } | 20260508 07:11:27.161 | INFO | Perf report for the last 5.13553 seconds: | 20260508 07:11:27.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:27.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.161 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 07:11:27.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.162 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:11:27.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:27.162 | INFO | Uploaded 36GiB to GPU in 2.38635s (15.0858GiB/s) | 20260508 07:11:27.162 | INFO | Downloaded 34GiB to GPU in 2.43245s (13.9777GiB/s) | 20260508 07:11:32.299 | INFO | Analyzing chunk frames = [6212608,6213120) | 20260508 07:11:32.299 | INFO | MemInfo: { | 20260508 07:11:32.299 | INFO | "gpu" : " 17 GB", | 20260508 07:11:32.299 | INFO | "huge" : "", | 20260508 07:11:32.299 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:32.299 | INFO | "malloc" : " 14 GB", | 20260508 07:11:32.299 | INFO | "pinned" : " 29 GB" | 20260508 07:11:32.299 | INFO | } | 20260508 07:11:32.299 | INFO | Perf report for the last 5.13789 seconds: | 20260508 07:11:32.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:32.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.299 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:11:32.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.299 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 07:11:32.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:32.300 | INFO | Uploaded 36GiB to GPU in 2.48436s (14.4907GiB/s) | 20260508 07:11:32.300 | INFO | Downloaded 34GiB to GPU in 2.42658s (14.0115GiB/s) | 20260508 07:11:37.440 | INFO | Analyzing chunk frames = [6213120,6213632) | 20260508 07:11:37.440 | INFO | MemInfo: { | 20260508 07:11:37.440 | INFO | "gpu" : " 17 GB", | 20260508 07:11:37.440 | INFO | "huge" : "", | 20260508 07:11:37.441 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:37.441 | INFO | "malloc" : " 14 GB", | 20260508 07:11:37.441 | INFO | "pinned" : " 29 GB" | 20260508 07:11:37.441 | INFO | } | 20260508 07:11:37.441 | INFO | Perf report for the last 5.14162 seconds: | 20260508 07:11:37.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:37.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 07:11:37.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 07:11:37.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:37.441 | INFO | Uploaded 36GiB to GPU in 2.50652s (14.3626GiB/s) | 20260508 07:11:37.442 | INFO | Downloaded 34GiB to GPU in 2.45976s (13.8225GiB/s) | 20260508 07:11:37.552 | INFO | Starting a round of Baz IO | 20260508 07:11:40.967 | INFO | Populating RTMetrics report | 20260508 07:11:40.967 | INFO | RTMetrics report populated | 20260508 07:11:40.967 | INFO | Opening tmp RTMetrics json file | 20260508 07:11:40.979 | INFO | Serializing RTMetrics to json | 20260508 07:11:40.981 | INFO | Serialization done, swapping tmp file now... | 20260508 07:11:40.981 | INFO | RTMetrics done | 20260508 07:11:41.041 | INFO | Finished a round of Baz IO: Wrote 1.533 GiB in 3.488 seconds | 20260508 07:11:41.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.00% / 59.41% | 20260508 07:11:41.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.35636% / 0% / 3.57628e-05% | 20260508 07:11:41.041 | INFO | Baz IO overall: | 20260508 07:11:41.041 | INFO | BytesWritten : 2745059504034 | 20260508 07:11:41.041 | INFO | NumEvents : 1322803726105 | 20260508 07:11:42.579 | INFO | Analyzing chunk frames = [6213632,6214144) | 20260508 07:11:42.579 | INFO | MemInfo: { | 20260508 07:11:42.579 | INFO | "gpu" : " 17 GB", | 20260508 07:11:42.579 | INFO | "huge" : "", | 20260508 07:11:42.579 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:42.580 | INFO | "malloc" : " 14 GB", | 20260508 07:11:42.580 | INFO | "pinned" : " 29 GB" | 20260508 07:11:42.580 | INFO | } | 20260508 07:11:42.580 | INFO | Perf report for the last 5.13868 seconds: | 20260508 07:11:42.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:42.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:42.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:42.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:42.580 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 07:11:42.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:42.580 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 07:11:42.580 | INFO | PRE_HQ : 66.6%, 1.8, 61.2ms, 0 | 20260508 07:11:42.580 | INFO | BAZWRITER : 67.2%, 2.8, 97.1ms, 0 | 20260508 07:11:42.580 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:11:42.580 | INFO | Uploaded 36GiB to GPU in 2.58558s (13.9234GiB/s) | 20260508 07:11:42.580 | INFO | Downloaded 38GiB to GPU in 2.70275s (14.0598GiB/s) | 20260508 07:11:47.724 | INFO | Analyzing chunk frames = [6214144,6214656) | 20260508 07:11:47.724 | INFO | MemInfo: { | 20260508 07:11:47.724 | INFO | "gpu" : " 17 GB", | 20260508 07:11:47.724 | INFO | "huge" : "", | 20260508 07:11:47.724 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:47.724 | INFO | "malloc" : " 14 GB", | 20260508 07:11:47.724 | INFO | "pinned" : " 29 GB" | 20260508 07:11:47.724 | INFO | } | 20260508 07:11:47.724 | INFO | Perf report for the last 5.14487 seconds: | 20260508 07:11:47.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:47.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.725 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 07:11:47.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.725 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:11:47.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:47.725 | INFO | Uploaded 36GiB to GPU in 2.52172s (14.276GiB/s) | 20260508 07:11:47.725 | INFO | Downloaded 34GiB to GPU in 2.44152s (13.9257GiB/s) | 20260508 07:11:52.860 | INFO | Analyzing chunk frames = [6214656,6215168) | 20260508 07:11:52.860 | INFO | MemInfo: { | 20260508 07:11:52.860 | INFO | "gpu" : " 17 GB", | 20260508 07:11:52.860 | INFO | "huge" : "", | 20260508 07:11:52.861 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:52.861 | INFO | "malloc" : " 14 GB", | 20260508 07:11:52.861 | INFO | "pinned" : " 29 GB" | 20260508 07:11:52.861 | INFO | } | 20260508 07:11:52.861 | INFO | Perf report for the last 5.13621 seconds: | 20260508 07:11:52.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:52.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 07:11:52.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:11:52.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:52.861 | INFO | Uploaded 36GiB to GPU in 2.58558s (13.9234GiB/s) | 20260508 07:11:52.861 | INFO | Downloaded 34GiB to GPU in 2.4263s (14.0131GiB/s) | 20260508 07:11:57.994 | INFO | Analyzing chunk frames = [6215168,6215680) | 20260508 07:11:57.994 | INFO | MemInfo: { | 20260508 07:11:57.994 | INFO | "gpu" : " 17 GB", | 20260508 07:11:57.994 | INFO | "huge" : "", | 20260508 07:11:57.994 | INFO | "huge_shared" : " 29 GB", | 20260508 07:11:57.994 | INFO | "malloc" : " 14 GB", | 20260508 07:11:57.994 | INFO | "pinned" : " 29 GB" | 20260508 07:11:57.994 | INFO | } | 20260508 07:11:57.994 | INFO | Perf report for the last 5.13316 seconds: | 20260508 07:11:57.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:11:57.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:11:57.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:11:57.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:11:57.994 | INFO | Uploaded 36GiB to GPU in 2.58939s (13.9029GiB/s) | 20260508 07:11:57.994 | INFO | Downloaded 34GiB to GPU in 2.40822s (14.1183GiB/s) | 20260508 07:12:03.138 | INFO | Analyzing chunk frames = [6215680,6216192) | 20260508 07:12:03.138 | INFO | MemInfo: { | 20260508 07:12:03.138 | INFO | "gpu" : " 17 GB", | 20260508 07:12:03.138 | INFO | "huge" : "", | 20260508 07:12:03.138 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:03.138 | INFO | "malloc" : " 14 GB", | 20260508 07:12:03.138 | INFO | "pinned" : " 29 GB" | 20260508 07:12:03.138 | INFO | } | 20260508 07:12:03.138 | INFO | Perf report for the last 5.1446 seconds: | 20260508 07:12:03.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:03.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:12:03.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:12:03.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:03.139 | INFO | Uploaded 36GiB to GPU in 2.32015s (15.5162GiB/s) | 20260508 07:12:03.139 | INFO | Downloaded 34GiB to GPU in 2.47869s (13.7169GiB/s) | 20260508 07:12:08.276 | INFO | Analyzing chunk frames = [6216192,6216704) | 20260508 07:12:08.276 | INFO | MemInfo: { | 20260508 07:12:08.276 | INFO | "gpu" : " 17 GB", | 20260508 07:12:08.277 | INFO | "huge" : "", | 20260508 07:12:08.277 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:08.277 | INFO | "malloc" : " 14 GB", | 20260508 07:12:08.277 | INFO | "pinned" : " 29 GB" | 20260508 07:12:08.277 | INFO | } | 20260508 07:12:08.277 | INFO | Perf report for the last 5.13817 seconds: | 20260508 07:12:08.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:08.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 07:12:08.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:12:08.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:08.277 | INFO | Uploaded 36GiB to GPU in 2.38804s (15.0751GiB/s) | 20260508 07:12:08.277 | INFO | Downloaded 34GiB to GPU in 2.42076s (14.0452GiB/s) | 20260508 07:12:13.417 | INFO | Analyzing chunk frames = [6216704,6217216) | 20260508 07:12:13.417 | INFO | MemInfo: { | 20260508 07:12:13.417 | INFO | "gpu" : " 17 GB", | 20260508 07:12:13.417 | INFO | "huge" : "", | 20260508 07:12:13.417 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:13.417 | INFO | "malloc" : " 14 GB", | 20260508 07:12:13.417 | INFO | "pinned" : " 29 GB" | 20260508 07:12:13.417 | INFO | } | 20260508 07:12:13.417 | INFO | Perf report for the last 5.14063 seconds: | 20260508 07:12:13.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:13.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.417 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:12:13.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.418 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 07:12:13.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:13.418 | INFO | Uploaded 36GiB to GPU in 2.49384s (14.4356GiB/s) | 20260508 07:12:13.418 | INFO | Downloaded 34GiB to GPU in 2.42325s (14.0308GiB/s) | 20260508 07:12:18.547 | INFO | Analyzing chunk frames = [6217216,6217728) | 20260508 07:12:18.548 | INFO | MemInfo: { | 20260508 07:12:18.548 | INFO | "gpu" : " 17 GB", | 20260508 07:12:18.548 | INFO | "huge" : "", | 20260508 07:12:18.548 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:18.548 | INFO | "malloc" : " 14 GB", | 20260508 07:12:18.548 | INFO | "pinned" : " 29 GB" | 20260508 07:12:18.548 | INFO | } | 20260508 07:12:18.548 | INFO | Perf report for the last 5.13053 seconds: | 20260508 07:12:18.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:18.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:12:18.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | PRE_HQ : 33.5%, 1.0, 17.3ms, 0 | 20260508 07:12:18.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:18.548 | INFO | Uploaded 36GiB to GPU in 2.52006s (14.2854GiB/s) | 20260508 07:12:18.548 | INFO | Downloaded 34GiB to GPU in 2.45489s (13.8499GiB/s) | 20260508 07:12:18.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:12:18.548 | INFO | Total mem usage: 16.7 GiB | 20260508 07:12:18.548 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:12:18.548 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:12:18.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:12:18.548 | INFO | High Water Breakdown: | 20260508 07:12:18.548 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:12:18.548 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:12:18.548 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:12:18.548 | INFO | ------------------------------------------ | 20260508 07:12:18.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:12:18.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:12:18.549 | INFO | Total mem usage: 26.5 GiB | 20260508 07:12:18.549 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:12:18.549 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:12:18.549 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:12:18.549 | INFO | High Water Breakdown: | 20260508 07:12:18.549 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:12:18.549 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:12:18.549 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:12:18.549 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:12:18.549 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:12:18.549 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:12:18.549 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:12:18.549 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:12:18.549 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:12:18.549 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:12:18.549 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:12:18.549 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:12:18.549 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:12:18.549 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:12:18.549 | INFO | ------------------------------------------ | 20260508 07:12:18.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:12:18.549 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:12:18.549 | INFO | Total mem usage: 27.8 GiB | 20260508 07:12:18.549 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:12:18.549 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:12:18.549 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:12:18.549 | INFO | High Water Breakdown: | 20260508 07:12:18.549 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:12:18.549 | INFO | ------------------------------------------ | 20260508 07:12:18.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:12:18.549 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:12:18.549 | INFO | Total mem usage: 15.7 GiB | 20260508 07:12:18.549 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:12:18.549 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:12:18.549 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:12:18.549 | INFO | High Water Breakdown: | 20260508 07:12:18.549 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:12:18.549 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:12:18.549 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:12:18.549 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:12:18.549 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:12:18.549 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:12:18.549 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:12:18.549 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:12:18.549 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:12:18.549 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:12:18.549 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:12:18.549 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:12:18.549 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:12:18.549 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:12:18.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:12:18.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:12:18.549 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:12:18.549 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:12:18.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:12:18.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:12:18.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:12:18.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:12:18.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:12:18.550 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:12:18.550 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:12:18.550 | INFO | ------------------------------------------ | 20260508 07:12:18.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:12:18.550 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:12:18.550 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:12:18.550 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:12:18.550 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.41e+01 / 6.40e-01 | 20260508 07:12:18.550 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.06e-01, 1.15e-02 / 4.92e-03 | 20260508 07:12:18.550 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.40e+00, 5.91e-01 / 2.74e-01 | 20260508 07:12:18.550 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.09e-01 / 1.18e-01 | 20260508 07:12:18.550 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.58e+01, 3.83e+00 / 1.25e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.79e+00 / 1.31e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.38e+00 / 2.60e+00, 4.11e-01 / 2.40e-01 | 20260508 07:12:18.550 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.21e+00, 2.52e-01 / 1.23e-01 | 20260508 07:12:18.550 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.48e-01, 4.93e-02 / 2.43e-02 | 20260508 07:12:18.550 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:12:18.550 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:12:18.550 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:12:18.550 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 24.49%, Upload -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 24.27%, Download -- 2.57e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 30.82%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 2.49%, LockUpload -- 2.64e+00 / 0.00e+00, 5.88e+00 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 2.38%, LockDownload -- 2.53e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 15.54%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:12:18.550 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260508 07:12:18.662 | INFO | Starting a round of Baz IO | 20260508 07:12:22.076 | INFO | Populating RTMetrics report | 20260508 07:12:22.076 | INFO | RTMetrics report populated | 20260508 07:12:22.076 | INFO | Opening tmp RTMetrics json file | 20260508 07:12:22.086 | INFO | Serializing RTMetrics to json | 20260508 07:12:22.088 | INFO | Serialization done, swapping tmp file now... | 20260508 07:12:22.088 | INFO | RTMetrics done | 20260508 07:12:22.144 | INFO | Finished a round of Baz IO: Wrote 1.532 GiB in 3.481 seconds | 20260508 07:12:22.144 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.00% / 59.41% | 20260508 07:12:22.144 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.355546% / 3.97364e-06% / 5.16574e-05% | 20260508 07:12:22.144 | INFO | Baz IO overall: | 20260508 07:12:22.144 | INFO | BytesWritten : 2746704800001 | 20260508 07:12:22.144 | INFO | NumEvents : 1323585654082 | 20260508 07:12:23.690 | INFO | Analyzing chunk frames = [6217728,6218240) | 20260508 07:12:23.690 | INFO | MemInfo: { | 20260508 07:12:23.690 | INFO | "gpu" : " 17 GB", | 20260508 07:12:23.690 | INFO | "huge" : "", | 20260508 07:12:23.690 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:23.690 | INFO | "malloc" : " 14 GB", | 20260508 07:12:23.690 | INFO | "pinned" : " 29 GB" | 20260508 07:12:23.690 | INFO | } | 20260508 07:12:23.690 | INFO | Perf report for the last 5.1426 seconds: | 20260508 07:12:23.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:23.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:23.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:23.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:23.690 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 07:12:23.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:23.691 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 07:12:23.691 | INFO | PRE_HQ : 66.1%, 1.8, 61.3ms, 0 | 20260508 07:12:23.691 | INFO | BAZWRITER : 67.3%, 2.8, 96.9ms, 0 | 20260508 07:12:23.691 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:12:23.691 | INFO | Uploaded 36GiB to GPU in 2.60323s (13.829GiB/s) | 20260508 07:12:23.691 | INFO | Downloaded 38GiB to GPU in 2.68262s (14.1653GiB/s) | 20260508 07:12:28.832 | INFO | Analyzing chunk frames = [6218240,6218752) | 20260508 07:12:28.832 | INFO | MemInfo: { | 20260508 07:12:28.832 | INFO | "gpu" : " 17 GB", | 20260508 07:12:28.832 | INFO | "huge" : "", | 20260508 07:12:28.832 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:28.832 | INFO | "malloc" : " 14 GB", | 20260508 07:12:28.832 | INFO | "pinned" : " 29 GB" | 20260508 07:12:28.832 | INFO | } | 20260508 07:12:28.832 | INFO | Perf report for the last 5.14155 seconds: | 20260508 07:12:28.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:28.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:12:28.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:12:28.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:28.832 | INFO | Uploaded 36GiB to GPU in 2.51435s (14.3178GiB/s) | 20260508 07:12:28.832 | INFO | Downloaded 34GiB to GPU in 2.43819s (13.9448GiB/s) | 20260508 07:12:33.979 | INFO | Analyzing chunk frames = [6218752,6219264) | 20260508 07:12:33.979 | INFO | MemInfo: { | 20260508 07:12:33.979 | INFO | "gpu" : " 17 GB", | 20260508 07:12:33.979 | INFO | "huge" : "", | 20260508 07:12:33.979 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:33.979 | INFO | "malloc" : " 14 GB", | 20260508 07:12:33.979 | INFO | "pinned" : " 29 GB" | 20260508 07:12:33.979 | INFO | } | 20260508 07:12:33.979 | INFO | Perf report for the last 5.14706 seconds: | 20260508 07:12:33.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:33.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 07:12:33.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:12:33.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:33.979 | INFO | Uploaded 36GiB to GPU in 2.54751s (14.1314GiB/s) | 20260508 07:12:33.979 | INFO | Downloaded 34GiB to GPU in 2.40968s (14.1098GiB/s) | 20260508 07:12:39.115 | INFO | Analyzing chunk frames = [6219264,6219776) | 20260508 07:12:39.115 | INFO | MemInfo: { | 20260508 07:12:39.115 | INFO | "gpu" : " 17 GB", | 20260508 07:12:39.115 | INFO | "huge" : "", | 20260508 07:12:39.115 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:39.115 | INFO | "malloc" : " 14 GB", | 20260508 07:12:39.115 | INFO | "pinned" : " 29 GB" | 20260508 07:12:39.116 | INFO | } | 20260508 07:12:39.116 | INFO | Perf report for the last 5.13648 seconds: | 20260508 07:12:39.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:39.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 07:12:39.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:12:39.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:39.116 | INFO | Uploaded 36GiB to GPU in 2.58901s (13.9049GiB/s) | 20260508 07:12:39.116 | INFO | Downloaded 34GiB to GPU in 2.42617s (14.0139GiB/s) | 20260508 07:12:44.250 | INFO | Analyzing chunk frames = [6219776,6220288) | 20260508 07:12:44.250 | INFO | MemInfo: { | 20260508 07:12:44.250 | INFO | "gpu" : " 17 GB", | 20260508 07:12:44.250 | INFO | "huge" : "", | 20260508 07:12:44.250 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:44.250 | INFO | "malloc" : " 14 GB", | 20260508 07:12:44.250 | INFO | "pinned" : " 29 GB" | 20260508 07:12:44.250 | INFO | } | 20260508 07:12:44.250 | INFO | Perf report for the last 5.13461 seconds: | 20260508 07:12:44.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:44.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 07:12:44.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:12:44.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:44.250 | INFO | Uploaded 36GiB to GPU in 2.30913s (15.5903GiB/s) | 20260508 07:12:44.251 | INFO | Downloaded 34GiB to GPU in 2.5084s (13.5544GiB/s) | 20260508 07:12:49.394 | INFO | Analyzing chunk frames = [6220288,6220800) | 20260508 07:12:49.394 | INFO | MemInfo: { | 20260508 07:12:49.394 | INFO | "gpu" : " 17 GB", | 20260508 07:12:49.394 | INFO | "huge" : "", | 20260508 07:12:49.394 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:49.394 | INFO | "malloc" : " 14 GB", | 20260508 07:12:49.395 | INFO | "pinned" : " 29 GB" | 20260508 07:12:49.395 | INFO | } | 20260508 07:12:49.395 | INFO | Perf report for the last 5.14437 seconds: | 20260508 07:12:49.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:49.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 07:12:49.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:12:49.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:49.395 | INFO | Uploaded 36GiB to GPU in 2.4577s (14.6478GiB/s) | 20260508 07:12:49.395 | INFO | Downloaded 34GiB to GPU in 2.43755s (13.9484GiB/s) | 20260508 07:12:54.528 | INFO | Analyzing chunk frames = [6220800,6221312) | 20260508 07:12:54.528 | INFO | MemInfo: { | 20260508 07:12:54.528 | INFO | "gpu" : " 17 GB", | 20260508 07:12:54.528 | INFO | "huge" : "", | 20260508 07:12:54.528 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:54.529 | INFO | "malloc" : " 14 GB", | 20260508 07:12:54.529 | INFO | "pinned" : " 29 GB" | 20260508 07:12:54.529 | INFO | } | 20260508 07:12:54.529 | INFO | Perf report for the last 5.13405 seconds: | 20260508 07:12:54.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:54.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 07:12:54.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 07:12:54.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:54.529 | INFO | Uploaded 36GiB to GPU in 2.41363s (14.9153GiB/s) | 20260508 07:12:54.529 | INFO | Downloaded 34GiB to GPU in 2.43917s (13.9392GiB/s) | 20260508 07:12:59.673 | INFO | Analyzing chunk frames = [6221312,6221824) | 20260508 07:12:59.673 | INFO | MemInfo: { | 20260508 07:12:59.673 | INFO | "gpu" : " 17 GB", | 20260508 07:12:59.673 | INFO | "huge" : "", | 20260508 07:12:59.673 | INFO | "huge_shared" : " 29 GB", | 20260508 07:12:59.673 | INFO | "malloc" : " 14 GB", | 20260508 07:12:59.673 | INFO | "pinned" : " 29 GB" | 20260508 07:12:59.673 | INFO | } | 20260508 07:12:59.673 | INFO | Perf report for the last 5.14444 seconds: | 20260508 07:12:59.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:12:59.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:12:59.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260508 07:12:59.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:12:59.673 | INFO | Uploaded 36GiB to GPU in 2.4853s (14.4852GiB/s) | 20260508 07:12:59.673 | INFO | Downloaded 34GiB to GPU in 2.45109s (13.8714GiB/s) | 20260508 07:12:59.771 | INFO | Starting a round of Baz IO | 20260508 07:13:03.197 | INFO | Populating RTMetrics report | 20260508 07:13:03.198 | INFO | RTMetrics report populated | 20260508 07:13:03.198 | INFO | Opening tmp RTMetrics json file | 20260508 07:13:03.206 | INFO | Serializing RTMetrics to json | 20260508 07:13:03.208 | INFO | Serialization done, swapping tmp file now... | 20260508 07:13:03.209 | INFO | RTMetrics done | 20260508 07:13:03.267 | INFO | Finished a round of Baz IO: Wrote 1.532 GiB in 3.495 seconds | 20260508 07:13:03.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.00% / 59.41% | 20260508 07:13:03.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.354707% / 0% / 5.96046e-05% | 20260508 07:13:03.267 | INFO | Baz IO overall: | 20260508 07:13:03.267 | INFO | BytesWritten : 2748349354624 | 20260508 07:13:03.267 | INFO | NumEvents : 1324366908546 | 20260508 07:13:04.808 | INFO | Analyzing chunk frames = [6221824,6222336) | 20260508 07:13:04.808 | INFO | MemInfo: { | 20260508 07:13:04.808 | INFO | "gpu" : " 17 GB", | 20260508 07:13:04.808 | INFO | "huge" : "", | 20260508 07:13:04.808 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:04.808 | INFO | "malloc" : " 14 GB", | 20260508 07:13:04.808 | INFO | "pinned" : " 29 GB" | 20260508 07:13:04.808 | INFO | } | 20260508 07:13:04.808 | INFO | Perf report for the last 5.13499 seconds: | 20260508 07:13:04.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:04.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:04.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:04.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:04.808 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 07:13:04.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:04.808 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:13:04.808 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 07:13:04.808 | INFO | BAZWRITER : 67.5%, 2.6, 90.4ms, 0 | 20260508 07:13:04.808 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:13:04.809 | INFO | Uploaded 36GiB to GPU in 2.59928s (13.85GiB/s) | 20260508 07:13:04.809 | INFO | Downloaded 38GiB to GPU in 2.68956s (14.1287GiB/s) | 20260508 07:13:09.946 | INFO | Analyzing chunk frames = [6222336,6222848) | 20260508 07:13:09.946 | INFO | MemInfo: { | 20260508 07:13:09.946 | INFO | "gpu" : " 17 GB", | 20260508 07:13:09.946 | INFO | "huge" : "", | 20260508 07:13:09.946 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:09.946 | INFO | "malloc" : " 14 GB", | 20260508 07:13:09.946 | INFO | "pinned" : " 29 GB" | 20260508 07:13:09.946 | INFO | } | 20260508 07:13:09.946 | INFO | Perf report for the last 5.1383 seconds: | 20260508 07:13:09.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:09.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.947 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:13:09.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.947 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:13:09.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:09.947 | INFO | Uploaded 36GiB to GPU in 2.4771s (14.5331GiB/s) | 20260508 07:13:09.947 | INFO | Downloaded 34GiB to GPU in 2.42945s (13.9949GiB/s) | 20260508 07:13:15.089 | INFO | Analyzing chunk frames = [6222848,6223360) | 20260508 07:13:15.089 | INFO | MemInfo: { | 20260508 07:13:15.089 | INFO | "gpu" : " 17 GB", | 20260508 07:13:15.089 | INFO | "huge" : "", | 20260508 07:13:15.089 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:15.089 | INFO | "malloc" : " 14 GB", | 20260508 07:13:15.089 | INFO | "pinned" : " 29 GB" | 20260508 07:13:15.089 | INFO | } | 20260508 07:13:15.089 | INFO | Perf report for the last 5.14232 seconds: | 20260508 07:13:15.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:15.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:13:15.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:13:15.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:15.089 | INFO | Uploaded 36GiB to GPU in 2.56144s (14.0546GiB/s) | 20260508 07:13:15.089 | INFO | Downloaded 34GiB to GPU in 2.42922s (13.9963GiB/s) | 20260508 07:13:20.226 | INFO | Analyzing chunk frames = [6223360,6223872) | 20260508 07:13:20.226 | INFO | MemInfo: { | 20260508 07:13:20.226 | INFO | "gpu" : " 17 GB", | 20260508 07:13:20.226 | INFO | "huge" : "", | 20260508 07:13:20.226 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:20.226 | INFO | "malloc" : " 14 GB", | 20260508 07:13:20.226 | INFO | "pinned" : " 29 GB" | 20260508 07:13:20.226 | INFO | } | 20260508 07:13:20.226 | INFO | Perf report for the last 5.13693 seconds: | 20260508 07:13:20.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:20.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:13:20.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:13:20.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:20.226 | INFO | Uploaded 36GiB to GPU in 2.55981s (14.0636GiB/s) | 20260508 07:13:20.226 | INFO | Downloaded 34GiB to GPU in 2.3934s (14.2058GiB/s) | 20260508 07:13:25.364 | INFO | Analyzing chunk frames = [6223872,6224384) | 20260508 07:13:25.364 | INFO | MemInfo: { | 20260508 07:13:25.364 | INFO | "gpu" : " 17 GB", | 20260508 07:13:25.364 | INFO | "huge" : "", | 20260508 07:13:25.364 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:25.364 | INFO | "malloc" : " 14 GB", | 20260508 07:13:25.364 | INFO | "pinned" : " 29 GB" | 20260508 07:13:25.364 | INFO | } | 20260508 07:13:25.364 | INFO | Perf report for the last 5.13835 seconds: | 20260508 07:13:25.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:25.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.364 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:13:25.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.364 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:13:25.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:25.365 | INFO | Uploaded 36GiB to GPU in 2.28474s (15.7567GiB/s) | 20260508 07:13:25.365 | INFO | Downloaded 34GiB to GPU in 2.47398s (13.743GiB/s) | 20260508 07:13:30.500 | INFO | Analyzing chunk frames = [6224384,6224896) | 20260508 07:13:30.500 | INFO | MemInfo: { | 20260508 07:13:30.500 | INFO | "gpu" : " 17 GB", | 20260508 07:13:30.500 | INFO | "huge" : "", | 20260508 07:13:30.500 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:30.500 | INFO | "malloc" : " 14 GB", | 20260508 07:13:30.500 | INFO | "pinned" : " 29 GB" | 20260508 07:13:30.500 | INFO | } | 20260508 07:13:30.500 | INFO | Perf report for the last 5.1358 seconds: | 20260508 07:13:30.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:30.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 07:13:30.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:13:30.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:30.500 | INFO | Uploaded 36GiB to GPU in 2.46516s (14.6035GiB/s) | 20260508 07:13:30.500 | INFO | Downloaded 34GiB to GPU in 2.44052s (13.9315GiB/s) | 20260508 07:13:35.647 | INFO | Analyzing chunk frames = [6224896,6225408) | 20260508 07:13:35.647 | INFO | MemInfo: { | 20260508 07:13:35.647 | INFO | "gpu" : " 17 GB", | 20260508 07:13:35.647 | INFO | "huge" : "", | 20260508 07:13:35.647 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:35.647 | INFO | "malloc" : " 14 GB", | 20260508 07:13:35.647 | INFO | "pinned" : " 29 GB" | 20260508 07:13:35.647 | INFO | } | 20260508 07:13:35.647 | INFO | Perf report for the last 5.14713 seconds: | 20260508 07:13:35.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:35.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 07:13:35.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:13:35.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:35.647 | INFO | Uploaded 36GiB to GPU in 2.4623s (14.6205GiB/s) | 20260508 07:13:35.647 | INFO | Downloaded 34GiB to GPU in 2.42805s (14.003GiB/s) | 20260508 07:13:40.785 | INFO | Analyzing chunk frames = [6225408,6225920) | 20260508 07:13:40.785 | INFO | MemInfo: { | 20260508 07:13:40.785 | INFO | "gpu" : " 17 GB", | 20260508 07:13:40.785 | INFO | "huge" : "", | 20260508 07:13:40.785 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:40.785 | INFO | "malloc" : " 14 GB", | 20260508 07:13:40.785 | INFO | "pinned" : " 29 GB" | 20260508 07:13:40.785 | INFO | } | 20260508 07:13:40.785 | INFO | Perf report for the last 5.13782 seconds: | 20260508 07:13:40.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:40.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 07:13:40.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:13:40.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:40.785 | INFO | Uploaded 36GiB to GPU in 2.47043s (14.5724GiB/s) | 20260508 07:13:40.785 | INFO | Downloaded 34GiB to GPU in 2.44437s (13.9095GiB/s) | 20260508 07:13:40.786 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:13:40.786 | INFO | Total mem usage: 16.7 GiB | 20260508 07:13:40.786 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:13:40.786 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:13:40.786 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:13:40.786 | INFO | High Water Breakdown: | 20260508 07:13:40.786 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:13:40.786 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:13:40.786 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:13:40.786 | INFO | ------------------------------------------ | 20260508 07:13:40.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:13:40.786 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:13:40.786 | INFO | Total mem usage: 26.5 GiB | 20260508 07:13:40.786 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:13:40.786 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:13:40.786 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:13:40.786 | INFO | High Water Breakdown: | 20260508 07:13:40.786 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:13:40.786 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:13:40.786 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:13:40.786 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:13:40.786 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:13:40.786 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:13:40.786 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:13:40.786 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:13:40.786 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:13:40.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:13:40.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:13:40.786 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:13:40.786 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:13:40.786 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:13:40.786 | INFO | ------------------------------------------ | 20260508 07:13:40.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:13:40.786 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:13:40.786 | INFO | Total mem usage: 27.8 GiB | 20260508 07:13:40.786 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:13:40.786 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:13:40.786 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:13:40.786 | INFO | High Water Breakdown: | 20260508 07:13:40.786 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:13:40.786 | INFO | ------------------------------------------ | 20260508 07:13:40.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:13:40.786 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:13:40.786 | INFO | Total mem usage: 15.7 GiB | 20260508 07:13:40.786 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:13:40.786 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:13:40.786 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:13:40.786 | INFO | High Water Breakdown: | 20260508 07:13:40.786 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:13:40.786 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:13:40.786 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:13:40.786 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:13:40.786 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:13:40.786 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:13:40.786 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:13:40.786 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:13:40.786 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:13:40.786 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:13:40.786 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:13:40.786 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:13:40.786 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:13:40.786 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:13:40.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:13:40.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:13:40.786 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:13:40.786 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:13:40.786 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:13:40.786 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:13:40.786 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:13:40.786 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:13:40.786 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:13:40.786 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:13:40.786 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:13:40.787 | INFO | ------------------------------------------ | 20260508 07:13:40.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:13:40.787 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:13:40.787 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:13:40.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:13:40.787 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 6.60e-01 | 20260508 07:13:40.787 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.33e-02 / 4.00e-03 | 20260508 07:13:40.787 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.38e+00, 5.88e-01 / 2.31e-01 | 20260508 07:13:40.787 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.10e-01 / 1.02e-01 | 20260508 07:13:40.787 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.52e+01, 3.79e+00 / 1.08e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.83e+00 / 1.14e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.51e+00, 4.12e-01 / 2.41e-01 | 20260508 07:13:40.787 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.17e+00, 2.50e-01 / 1.26e-01 | 20260508 07:13:40.787 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.43e-01, 4.86e-02 / 2.69e-02 | 20260508 07:13:40.787 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:13:40.787 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:13:40.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:13:40.787 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.55e+01 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 24.44%, Upload -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 24.27%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 2.34%, LockUpload -- 2.48e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 2.55%, LockDownload -- 2.70e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:13:40.787 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 07:13:40.884 | INFO | Starting a round of Baz IO | 20260508 07:13:44.316 | INFO | Populating RTMetrics report | 20260508 07:13:44.316 | INFO | RTMetrics report populated | 20260508 07:13:44.316 | INFO | Opening tmp RTMetrics json file | 20260508 07:13:44.317 | INFO | Serializing RTMetrics to json | 20260508 07:13:44.318 | INFO | Serialization done, swapping tmp file now... | 20260508 07:13:44.319 | INFO | RTMetrics done | 20260508 07:13:44.386 | INFO | Finished a round of Baz IO: Wrote 1.532 GiB in 3.501 seconds | 20260508 07:13:44.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.58% / 0.00% / 59.41% | 20260508 07:13:44.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.3534% / 7.94729e-06% / 5.16574e-05% | 20260508 07:13:44.386 | INFO | Baz IO overall: | 20260508 07:13:44.386 | INFO | BytesWritten : 2749994155569 | 20260508 07:13:44.386 | INFO | NumEvents : 1325148312235 | 20260508 07:13:45.930 | INFO | Analyzing chunk frames = [6225920,6226432) | 20260508 07:13:45.930 | INFO | MemInfo: { | 20260508 07:13:45.930 | INFO | "gpu" : " 17 GB", | 20260508 07:13:45.930 | INFO | "huge" : "", | 20260508 07:13:45.930 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:45.930 | INFO | "malloc" : " 14 GB", | 20260508 07:13:45.930 | INFO | "pinned" : " 29 GB" | 20260508 07:13:45.930 | INFO | } | 20260508 07:13:45.930 | INFO | Perf report for the last 5.14519 seconds: | 20260508 07:13:45.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:45.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:45.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:45.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:45.930 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 07:13:45.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:45.930 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:13:45.930 | INFO | PRE_HQ : 65.8%, 1.6, 55.0ms, 0 | 20260508 07:13:45.930 | INFO | BAZWRITER : 67.5%, 2.6, 90.9ms, 0 | 20260508 07:13:45.930 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 07:13:45.930 | INFO | Uploaded 36GiB to GPU in 2.55082s (14.1131GiB/s) | 20260508 07:13:45.930 | INFO | Downloaded 38GiB to GPU in 2.70019s (14.0731GiB/s) | 20260508 07:13:51.060 | INFO | Analyzing chunk frames = [6226432,6226944) | 20260508 07:13:51.060 | INFO | MemInfo: { | 20260508 07:13:51.060 | INFO | "gpu" : " 17 GB", | 20260508 07:13:51.060 | INFO | "huge" : "", | 20260508 07:13:51.060 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:51.060 | INFO | "malloc" : " 14 GB", | 20260508 07:13:51.060 | INFO | "pinned" : " 29 GB" | 20260508 07:13:51.060 | INFO | } | 20260508 07:13:51.060 | INFO | Perf report for the last 5.13029 seconds: | 20260508 07:13:51.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:51.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.060 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:13:51.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.061 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:13:51.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:51.061 | INFO | Uploaded 36GiB to GPU in 2.48391s (14.4933GiB/s) | 20260508 07:13:51.061 | INFO | Downloaded 34GiB to GPU in 2.43588s (13.958GiB/s) | 20260508 07:13:56.197 | INFO | Analyzing chunk frames = [6226944,6227456) | 20260508 07:13:56.197 | INFO | MemInfo: { | 20260508 07:13:56.197 | INFO | "gpu" : " 17 GB", | 20260508 07:13:56.197 | INFO | "huge" : "", | 20260508 07:13:56.197 | INFO | "huge_shared" : " 29 GB", | 20260508 07:13:56.197 | INFO | "malloc" : " 14 GB", | 20260508 07:13:56.197 | INFO | "pinned" : " 29 GB" | 20260508 07:13:56.197 | INFO | } | 20260508 07:13:56.197 | INFO | Perf report for the last 5.13693 seconds: | 20260508 07:13:56.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:13:56.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.197 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 07:13:56.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.197 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:13:56.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:13:56.198 | INFO | Uploaded 36GiB to GPU in 2.56297s (14.0462GiB/s) | 20260508 07:13:56.198 | INFO | Downloaded 34GiB to GPU in 2.43037s (13.9897GiB/s) | 20260508 07:14:01.341 | INFO | Analyzing chunk frames = [6227456,6227968) | 20260508 07:14:01.341 | INFO | MemInfo: { | 20260508 07:14:01.341 | INFO | "gpu" : " 17 GB", | 20260508 07:14:01.341 | INFO | "huge" : "", | 20260508 07:14:01.341 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:01.341 | INFO | "malloc" : " 14 GB", | 20260508 07:14:01.341 | INFO | "pinned" : " 29 GB" | 20260508 07:14:01.341 | INFO | } | 20260508 07:14:01.341 | INFO | Perf report for the last 5.14382 seconds: | 20260508 07:14:01.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:01.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.341 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 07:14:01.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.341 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:14:01.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:01.342 | INFO | Uploaded 36GiB to GPU in 2.54325s (14.1551GiB/s) | 20260508 07:14:01.342 | INFO | Downloaded 34GiB to GPU in 2.39387s (14.2029GiB/s) | 20260508 07:14:06.479 | INFO | Analyzing chunk frames = [6227968,6228480) | 20260508 07:14:06.479 | INFO | MemInfo: { | 20260508 07:14:06.479 | INFO | "gpu" : " 17 GB", | 20260508 07:14:06.479 | INFO | "huge" : "", | 20260508 07:14:06.479 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:06.479 | INFO | "malloc" : " 14 GB", | 20260508 07:14:06.479 | INFO | "pinned" : " 29 GB" | 20260508 07:14:06.479 | INFO | } | 20260508 07:14:06.479 | INFO | Perf report for the last 5.13812 seconds: | 20260508 07:14:06.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:06.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.479 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:14:06.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.479 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 07:14:06.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:06.480 | INFO | Uploaded 36GiB to GPU in 2.28312s (15.7679GiB/s) | 20260508 07:14:06.480 | INFO | Downloaded 34GiB to GPU in 2.4772s (13.7252GiB/s) | 20260508 07:14:11.618 | INFO | Analyzing chunk frames = [6228480,6228992) | 20260508 07:14:11.618 | INFO | MemInfo: { | 20260508 07:14:11.618 | INFO | "gpu" : " 17 GB", | 20260508 07:14:11.618 | INFO | "huge" : "", | 20260508 07:14:11.618 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:11.618 | INFO | "malloc" : " 14 GB", | 20260508 07:14:11.618 | INFO | "pinned" : " 29 GB" | 20260508 07:14:11.618 | INFO | } | 20260508 07:14:11.618 | INFO | Perf report for the last 5.13909 seconds: | 20260508 07:14:11.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:11.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.618 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:14:11.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.619 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:14:11.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:11.619 | INFO | Uploaded 36GiB to GPU in 2.48924s (14.4622GiB/s) | 20260508 07:14:11.619 | INFO | Downloaded 34GiB to GPU in 2.44322s (13.9161GiB/s) | 20260508 07:14:16.757 | INFO | Analyzing chunk frames = [6228992,6229504) | 20260508 07:14:16.757 | INFO | MemInfo: { | 20260508 07:14:16.757 | INFO | "gpu" : " 17 GB", | 20260508 07:14:16.757 | INFO | "huge" : "", | 20260508 07:14:16.757 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:16.757 | INFO | "malloc" : " 14 GB", | 20260508 07:14:16.757 | INFO | "pinned" : " 29 GB" | 20260508 07:14:16.757 | INFO | } | 20260508 07:14:16.757 | INFO | Perf report for the last 5.13903 seconds: | 20260508 07:14:16.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:16.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.758 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:14:16.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.758 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:14:16.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:16.758 | INFO | Uploaded 36GiB to GPU in 2.48947s (14.4609GiB/s) | 20260508 07:14:16.758 | INFO | Downloaded 34GiB to GPU in 2.43242s (13.9778GiB/s) | 20260508 07:14:21.900 | INFO | Analyzing chunk frames = [6229504,6230016) | 20260508 07:14:21.900 | INFO | MemInfo: { | 20260508 07:14:21.900 | INFO | "gpu" : " 17 GB", | 20260508 07:14:21.900 | INFO | "huge" : "", | 20260508 07:14:21.900 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:21.900 | INFO | "malloc" : " 14 GB", | 20260508 07:14:21.900 | INFO | "pinned" : " 29 GB" | 20260508 07:14:21.900 | INFO | } | 20260508 07:14:21.900 | INFO | Perf report for the last 5.14257 seconds: | 20260508 07:14:21.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:21.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:14:21.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:14:21.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:21.900 | INFO | Uploaded 36GiB to GPU in 2.46807s (14.5863GiB/s) | 20260508 07:14:21.900 | INFO | Downloaded 34GiB to GPU in 2.42325s (14.0307GiB/s) | 20260508 07:14:21.997 | INFO | Starting a round of Baz IO | 20260508 07:14:25.431 | INFO | Populating RTMetrics report | 20260508 07:14:25.431 | INFO | RTMetrics report populated | 20260508 07:14:25.432 | INFO | Opening tmp RTMetrics json file | 20260508 07:14:25.442 | INFO | Serializing RTMetrics to json | 20260508 07:14:25.443 | INFO | Serialization done, swapping tmp file now... | 20260508 07:14:25.444 | INFO | RTMetrics done | 20260508 07:14:25.498 | INFO | Finished a round of Baz IO: Wrote 1.531 GiB in 3.501 seconds | 20260508 07:14:25.498 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.58% / 0.00% / 59.42% | 20260508 07:14:25.498 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.352446% / 0% / 2.78155e-05% | 20260508 07:14:25.498 | INFO | Baz IO overall: | 20260508 07:14:25.498 | INFO | BytesWritten : 2751637521630 | 20260508 07:14:25.498 | INFO | NumEvents : 1325928853249 | 20260508 07:14:27.041 | INFO | Analyzing chunk frames = [6230016,6230528) | 20260508 07:14:27.041 | INFO | MemInfo: { | 20260508 07:14:27.041 | INFO | "gpu" : " 17 GB", | 20260508 07:14:27.041 | INFO | "huge" : "", | 20260508 07:14:27.041 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:27.041 | INFO | "malloc" : " 14 GB", | 20260508 07:14:27.041 | INFO | "pinned" : " 29 GB" | 20260508 07:14:27.041 | INFO | } | 20260508 07:14:27.041 | INFO | Perf report for the last 5.14126 seconds: | 20260508 07:14:27.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:27.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:27.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:27.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:27.041 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 07:14:27.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:27.041 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 07:14:27.041 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 07:14:27.041 | INFO | BAZWRITER : 67.6%, 2.6, 90.4ms, 0 | 20260508 07:14:27.041 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:14:27.042 | INFO | Uploaded 36GiB to GPU in 2.61317s (13.7764GiB/s) | 20260508 07:14:27.042 | INFO | Downloaded 38GiB to GPU in 2.67923s (14.1832GiB/s) | 20260508 07:14:32.179 | INFO | Analyzing chunk frames = [6230528,6231040) | 20260508 07:14:32.179 | INFO | MemInfo: { | 20260508 07:14:32.179 | INFO | "gpu" : " 17 GB", | 20260508 07:14:32.179 | INFO | "huge" : "", | 20260508 07:14:32.179 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:32.179 | INFO | "malloc" : " 14 GB", | 20260508 07:14:32.179 | INFO | "pinned" : " 29 GB" | 20260508 07:14:32.179 | INFO | } | 20260508 07:14:32.179 | INFO | Perf report for the last 5.13779 seconds: | 20260508 07:14:32.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:32.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.179 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:14:32.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.179 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:14:32.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:32.180 | INFO | Uploaded 36GiB to GPU in 2.53257s (14.2148GiB/s) | 20260508 07:14:32.180 | INFO | Downloaded 34GiB to GPU in 2.45124s (13.8705GiB/s) | 20260508 07:14:37.311 | INFO | Analyzing chunk frames = [6231040,6231552) | 20260508 07:14:37.311 | INFO | MemInfo: { | 20260508 07:14:37.311 | INFO | "gpu" : " 17 GB", | 20260508 07:14:37.311 | INFO | "huge" : "", | 20260508 07:14:37.311 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:37.311 | INFO | "malloc" : " 14 GB", | 20260508 07:14:37.311 | INFO | "pinned" : " 29 GB" | 20260508 07:14:37.311 | INFO | } | 20260508 07:14:37.311 | INFO | Perf report for the last 5.13222 seconds: | 20260508 07:14:37.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:37.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.311 | INFO | ANALYSIS : 67.4%, 3.0, 104.5ms, 0 | 20260508 07:14:37.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.311 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 07:14:37.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:37.312 | INFO | Uploaded 36GiB to GPU in 2.58353s (13.9344GiB/s) | 20260508 07:14:37.312 | INFO | Downloaded 34GiB to GPU in 2.43241s (13.9779GiB/s) | 20260508 07:14:42.453 | INFO | Analyzing chunk frames = [6231552,6232064) | 20260508 07:14:42.453 | INFO | MemInfo: { | 20260508 07:14:42.453 | INFO | "gpu" : " 17 GB", | 20260508 07:14:42.454 | INFO | "huge" : "", | 20260508 07:14:42.454 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:42.454 | INFO | "malloc" : " 14 GB", | 20260508 07:14:42.454 | INFO | "pinned" : " 29 GB" | 20260508 07:14:42.454 | INFO | } | 20260508 07:14:42.454 | INFO | Perf report for the last 5.14232 seconds: | 20260508 07:14:42.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:42.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 07:14:42.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:14:42.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:42.454 | INFO | Uploaded 36GiB to GPU in 2.58537s (13.9245GiB/s) | 20260508 07:14:42.454 | INFO | Downloaded 34GiB to GPU in 2.406s (14.1313GiB/s) | 20260508 07:14:47.596 | INFO | Analyzing chunk frames = [6232064,6232576) | 20260508 07:14:47.596 | INFO | MemInfo: { | 20260508 07:14:47.596 | INFO | "gpu" : " 17 GB", | 20260508 07:14:47.596 | INFO | "huge" : "", | 20260508 07:14:47.596 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:47.596 | INFO | "malloc" : " 14 GB", | 20260508 07:14:47.596 | INFO | "pinned" : " 29 GB" | 20260508 07:14:47.596 | INFO | } | 20260508 07:14:47.596 | INFO | Perf report for the last 5.1424 seconds: | 20260508 07:14:47.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:47.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 07:14:47.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:14:47.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:47.596 | INFO | Uploaded 36GiB to GPU in 2.43817s (14.7652GiB/s) | 20260508 07:14:47.597 | INFO | Downloaded 34GiB to GPU in 2.52358s (13.4729GiB/s) | 20260508 07:14:52.734 | INFO | Analyzing chunk frames = [6232576,6233088) | 20260508 07:14:52.734 | INFO | MemInfo: { | 20260508 07:14:52.734 | INFO | "gpu" : " 17 GB", | 20260508 07:14:52.734 | INFO | "huge" : "", | 20260508 07:14:52.734 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:52.734 | INFO | "malloc" : " 14 GB", | 20260508 07:14:52.734 | INFO | "pinned" : " 29 GB" | 20260508 07:14:52.734 | INFO | } | 20260508 07:14:52.735 | INFO | Perf report for the last 5.1384 seconds: | 20260508 07:14:52.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:52.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:14:52.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:14:52.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:52.735 | INFO | Uploaded 36GiB to GPU in 2.37514s (15.157GiB/s) | 20260508 07:14:52.735 | INFO | Downloaded 34GiB to GPU in 2.421s (14.0438GiB/s) | 20260508 07:14:57.878 | INFO | Analyzing chunk frames = [6233088,6233600) | 20260508 07:14:57.878 | INFO | MemInfo: { | 20260508 07:14:57.879 | INFO | "gpu" : " 17 GB", | 20260508 07:14:57.879 | INFO | "huge" : "", | 20260508 07:14:57.879 | INFO | "huge_shared" : " 29 GB", | 20260508 07:14:57.879 | INFO | "malloc" : " 14 GB", | 20260508 07:14:57.879 | INFO | "pinned" : " 29 GB" | 20260508 07:14:57.879 | INFO | } | 20260508 07:14:57.879 | INFO | Perf report for the last 5.14426 seconds: | 20260508 07:14:57.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:14:57.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.879 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:14:57.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.879 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:14:57.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:14:57.880 | INFO | Uploaded 36GiB to GPU in 2.48156s (14.507GiB/s) | 20260508 07:14:57.880 | INFO | Downloaded 34GiB to GPU in 2.43597s (13.9575GiB/s) | 20260508 07:15:03.011 | INFO | Analyzing chunk frames = [6233600,6234112) | 20260508 07:15:03.011 | INFO | MemInfo: { | 20260508 07:15:03.011 | INFO | "gpu" : " 17 GB", | 20260508 07:15:03.011 | INFO | "huge" : "", | 20260508 07:15:03.011 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:03.011 | INFO | "malloc" : " 14 GB", | 20260508 07:15:03.011 | INFO | "pinned" : " 29 GB" | 20260508 07:15:03.011 | INFO | } | 20260508 07:15:03.011 | INFO | Perf report for the last 5.13226 seconds: | 20260508 07:15:03.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:03.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 07:15:03.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 07:15:03.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:03.011 | INFO | Uploaded 36GiB to GPU in 2.4499s (14.6945GiB/s) | 20260508 07:15:03.011 | INFO | Downloaded 34GiB to GPU in 2.43357s (13.9712GiB/s) | 20260508 07:15:03.012 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:15:03.012 | INFO | Total mem usage: 16.7 GiB | 20260508 07:15:03.012 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:15:03.012 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:15:03.012 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:15:03.012 | INFO | High Water Breakdown: | 20260508 07:15:03.012 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:15:03.012 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:15:03.012 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:15:03.012 | INFO | ------------------------------------------ | 20260508 07:15:03.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:15:03.012 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:15:03.012 | INFO | Total mem usage: 26.5 GiB | 20260508 07:15:03.012 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:15:03.012 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:15:03.012 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:15:03.012 | INFO | High Water Breakdown: | 20260508 07:15:03.012 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:15:03.012 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:15:03.012 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:15:03.012 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:15:03.012 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:15:03.012 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:15:03.012 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:15:03.012 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:15:03.012 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:15:03.012 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:15:03.012 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:15:03.012 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:15:03.012 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:15:03.012 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:15:03.012 | INFO | ------------------------------------------ | 20260508 07:15:03.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:15:03.012 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:15:03.012 | INFO | Total mem usage: 27.8 GiB | 20260508 07:15:03.012 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:15:03.012 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:15:03.012 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:15:03.012 | INFO | High Water Breakdown: | 20260508 07:15:03.012 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:15:03.012 | INFO | ------------------------------------------ | 20260508 07:15:03.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:15:03.012 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:15:03.012 | INFO | Total mem usage: 15.7 GiB | 20260508 07:15:03.012 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:15:03.012 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:15:03.012 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:15:03.012 | INFO | High Water Breakdown: | 20260508 07:15:03.012 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:15:03.012 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:15:03.012 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:15:03.012 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:15:03.012 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:15:03.012 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:15:03.012 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:15:03.012 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:15:03.012 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:15:03.012 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:15:03.012 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:15:03.012 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:15:03.012 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:15:03.012 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:15:03.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:15:03.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:15:03.012 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:15:03.012 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:15:03.013 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:15:03.013 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:15:03.013 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:15:03.013 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:15:03.013 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:15:03.013 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:15:03.013 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:15:03.013 | INFO | ------------------------------------------ | 20260508 07:15:03.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:15:03.013 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:15:03.013 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:15:03.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:15:03.013 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.53e-01 | 20260508 07:15:03.013 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.04e-01, 1.18e-02 / 6.74e-03 | 20260508 07:15:03.013 | INFO | PerfReport: 12.11%, Baseline -- 3.96e+00 / 4.44e+00, 5.86e-01 / 1.60e-01 | 20260508 07:15:03.013 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.48e+00, 2.10e-01 / 8.95e-02 | 20260508 07:15:03.013 | INFO | PerfReport: 15.93%, DME -- 4.16e+01 / 4.52e+01, 3.74e+00 / 1.17e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.87e+00 / 1.45e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.54e+00, 4.10e-01 / 2.55e-01 | 20260508 07:15:03.013 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.18e+00, 2.50e-01 / 1.55e-01 | 20260508 07:15:03.013 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.43e-01, 4.88e-02 / 2.20e-02 | 20260508 07:15:03.013 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:15:03.013 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:15:03.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:15:03.013 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.54e+01 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 24.51%, Upload -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 30.81%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.86e+00 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 2.46%, LockDownload -- 2.61e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:15:03.013 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260508 07:15:03.108 | INFO | Starting a round of Baz IO | 20260508 07:15:06.537 | INFO | Populating RTMetrics report | 20260508 07:15:06.537 | INFO | RTMetrics report populated | 20260508 07:15:06.537 | INFO | Opening tmp RTMetrics json file | 20260508 07:15:06.548 | INFO | Serializing RTMetrics to json | 20260508 07:15:06.550 | INFO | Serialization done, swapping tmp file now... | 20260508 07:15:06.550 | INFO | RTMetrics done | 20260508 07:15:06.605 | INFO | Finished a round of Baz IO: Wrote 1.529 GiB in 3.496 seconds | 20260508 07:15:06.605 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.58% / 0.00% / 59.42% | 20260508 07:15:06.605 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.35071% / 7.94729e-06% / 6.35783e-05% | 20260508 07:15:06.605 | INFO | Baz IO overall: | 20260508 07:15:06.605 | INFO | BytesWritten : 2753279086371 | 20260508 07:15:06.605 | INFO | NumEvents : 1326708120517 | 20260508 07:15:08.151 | INFO | Analyzing chunk frames = [6234112,6234624) | 20260508 07:15:08.151 | INFO | MemInfo: { | 20260508 07:15:08.151 | INFO | "gpu" : " 17 GB", | 20260508 07:15:08.151 | INFO | "huge" : "", | 20260508 07:15:08.151 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:08.151 | INFO | "malloc" : " 14 GB", | 20260508 07:15:08.151 | INFO | "pinned" : " 29 GB" | 20260508 07:15:08.151 | INFO | } | 20260508 07:15:08.151 | INFO | Perf report for the last 5.14022 seconds: | 20260508 07:15:08.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:08.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:08.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:08.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:08.151 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 07:15:08.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:08.151 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:15:08.151 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 07:15:08.151 | INFO | BAZWRITER : 67.4%, 2.6, 91.0ms, 0 | 20260508 07:15:08.151 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:15:08.152 | INFO | Uploaded 36GiB to GPU in 2.61007s (13.7928GiB/s) | 20260508 07:15:08.152 | INFO | Downloaded 38GiB to GPU in 2.69534s (14.0984GiB/s) | 20260508 07:15:13.291 | INFO | Analyzing chunk frames = [6234624,6235136) | 20260508 07:15:13.291 | INFO | MemInfo: { | 20260508 07:15:13.291 | INFO | "gpu" : " 17 GB", | 20260508 07:15:13.291 | INFO | "huge" : "", | 20260508 07:15:13.291 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:13.291 | INFO | "malloc" : " 14 GB", | 20260508 07:15:13.291 | INFO | "pinned" : " 29 GB" | 20260508 07:15:13.291 | INFO | } | 20260508 07:15:13.291 | INFO | Perf report for the last 5.14017 seconds: | 20260508 07:15:13.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:13.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.292 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 07:15:13.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.292 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 07:15:13.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:13.292 | INFO | Uploaded 36GiB to GPU in 2.49774s (14.413GiB/s) | 20260508 07:15:13.292 | INFO | Downloaded 34GiB to GPU in 2.42404s (14.0262GiB/s) | 20260508 07:15:18.432 | INFO | Analyzing chunk frames = [6235136,6235648) | 20260508 07:15:18.432 | INFO | MemInfo: { | 20260508 07:15:18.432 | INFO | "gpu" : " 17 GB", | 20260508 07:15:18.432 | INFO | "huge" : "", | 20260508 07:15:18.432 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:18.432 | INFO | "malloc" : " 14 GB", | 20260508 07:15:18.432 | INFO | "pinned" : " 29 GB" | 20260508 07:15:18.432 | INFO | } | 20260508 07:15:18.432 | INFO | Perf report for the last 5.14059 seconds: | 20260508 07:15:18.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:18.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.432 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 07:15:18.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.433 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:15:18.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:18.433 | INFO | Uploaded 36GiB to GPU in 2.58793s (13.9107GiB/s) | 20260508 07:15:18.433 | INFO | Downloaded 34GiB to GPU in 2.42127s (14.0422GiB/s) | 20260508 07:15:23.571 | INFO | Analyzing chunk frames = [6235648,6236160) | 20260508 07:15:23.571 | INFO | MemInfo: { | 20260508 07:15:23.571 | INFO | "gpu" : " 17 GB", | 20260508 07:15:23.571 | INFO | "huge" : "", | 20260508 07:15:23.571 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:23.571 | INFO | "malloc" : " 14 GB", | 20260508 07:15:23.571 | INFO | "pinned" : " 29 GB" | 20260508 07:15:23.571 | INFO | } | 20260508 07:15:23.571 | INFO | Perf report for the last 5.13898 seconds: | 20260508 07:15:23.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:23.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:15:23.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:15:23.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:23.571 | INFO | Uploaded 36GiB to GPU in 2.58908s (13.9046GiB/s) | 20260508 07:15:23.571 | INFO | Downloaded 34GiB to GPU in 2.40975s (14.1093GiB/s) | 20260508 07:15:28.714 | INFO | Analyzing chunk frames = [6236160,6236672) | 20260508 07:15:28.714 | INFO | MemInfo: { | 20260508 07:15:28.714 | INFO | "gpu" : " 17 GB", | 20260508 07:15:28.714 | INFO | "huge" : "", | 20260508 07:15:28.714 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:28.714 | INFO | "malloc" : " 14 GB", | 20260508 07:15:28.714 | INFO | "pinned" : " 29 GB" | 20260508 07:15:28.714 | INFO | } | 20260508 07:15:28.714 | INFO | Perf report for the last 5.14281 seconds: | 20260508 07:15:28.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:28.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:15:28.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 07:15:28.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:28.714 | INFO | Uploaded 36GiB to GPU in 2.29271s (15.7019GiB/s) | 20260508 07:15:28.714 | INFO | Downloaded 34GiB to GPU in 2.50146s (13.5921GiB/s) | 20260508 07:15:33.856 | INFO | Analyzing chunk frames = [6236672,6237184) | 20260508 07:15:33.856 | INFO | MemInfo: { | 20260508 07:15:33.856 | INFO | "gpu" : " 17 GB", | 20260508 07:15:33.856 | INFO | "huge" : "", | 20260508 07:15:33.857 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:33.857 | INFO | "malloc" : " 14 GB", | 20260508 07:15:33.857 | INFO | "pinned" : " 29 GB" | 20260508 07:15:33.857 | INFO | } | 20260508 07:15:33.857 | INFO | Perf report for the last 5.14273 seconds: | 20260508 07:15:33.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:33.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:15:33.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:15:33.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:33.857 | INFO | Uploaded 36GiB to GPU in 2.37916s (15.1314GiB/s) | 20260508 07:15:33.857 | INFO | Downloaded 34GiB to GPU in 2.41465s (14.0807GiB/s) | 20260508 07:15:38.991 | INFO | Analyzing chunk frames = [6237184,6237696) | 20260508 07:15:38.991 | INFO | MemInfo: { | 20260508 07:15:38.991 | INFO | "gpu" : " 17 GB", | 20260508 07:15:38.991 | INFO | "huge" : "", | 20260508 07:15:38.991 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:38.991 | INFO | "malloc" : " 14 GB", | 20260508 07:15:38.991 | INFO | "pinned" : " 29 GB" | 20260508 07:15:38.991 | INFO | } | 20260508 07:15:38.991 | INFO | Perf report for the last 5.13467 seconds: | 20260508 07:15:38.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:38.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.991 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 07:15:38.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.991 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:15:38.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:38.992 | INFO | Uploaded 36GiB to GPU in 2.45396s (14.6702GiB/s) | 20260508 07:15:38.992 | INFO | Downloaded 34GiB to GPU in 2.41933s (14.0535GiB/s) | 20260508 07:15:44.129 | INFO | Analyzing chunk frames = [6237696,6238208) | 20260508 07:15:44.129 | INFO | MemInfo: { | 20260508 07:15:44.129 | INFO | "gpu" : " 17 GB", | 20260508 07:15:44.129 | INFO | "huge" : "", | 20260508 07:15:44.130 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:44.130 | INFO | "malloc" : " 14 GB", | 20260508 07:15:44.130 | INFO | "pinned" : " 29 GB" | 20260508 07:15:44.130 | INFO | } | 20260508 07:15:44.130 | INFO | Perf report for the last 5.1384 seconds: | 20260508 07:15:44.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:44.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:15:44.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:15:44.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:44.130 | INFO | Uploaded 36GiB to GPU in 2.49385s (14.4355GiB/s) | 20260508 07:15:44.130 | INFO | Downloaded 34GiB to GPU in 2.44466s (13.9079GiB/s) | 20260508 07:15:44.243 | INFO | Starting a round of Baz IO | 20260508 07:15:47.657 | INFO | Populating RTMetrics report | 20260508 07:15:47.657 | INFO | RTMetrics report populated | 20260508 07:15:47.657 | INFO | Opening tmp RTMetrics json file | 20260508 07:15:47.665 | INFO | Serializing RTMetrics to json | 20260508 07:15:47.667 | INFO | Serialization done, swapping tmp file now... | 20260508 07:15:47.667 | INFO | RTMetrics done | 20260508 07:15:47.723 | INFO | Finished a round of Baz IO: Wrote 1.532 GiB in 3.479 seconds | 20260508 07:15:47.723 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.58% / 0.00% / 59.42% | 20260508 07:15:47.723 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.352689% / 0% / 4.37101e-05% | 20260508 07:15:47.723 | INFO | Baz IO overall: | 20260508 07:15:47.723 | INFO | BytesWritten : 2754923526865 | 20260508 07:15:47.723 | INFO | NumEvents : 1327489727646 | 20260508 07:15:49.261 | INFO | Analyzing chunk frames = [6238208,6238720) | 20260508 07:15:49.261 | INFO | MemInfo: { | 20260508 07:15:49.261 | INFO | "gpu" : " 17 GB", | 20260508 07:15:49.261 | INFO | "huge" : "", | 20260508 07:15:49.261 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:49.261 | INFO | "malloc" : " 14 GB", | 20260508 07:15:49.261 | INFO | "pinned" : " 29 GB" | 20260508 07:15:49.261 | INFO | } | 20260508 07:15:49.261 | INFO | Perf report for the last 5.13168 seconds: | 20260508 07:15:49.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:49.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:49.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:49.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:49.261 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 07:15:49.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:49.262 | INFO | RT_METRICS : 42.9%, 1.0, 22.4ms, 0 | 20260508 07:15:49.262 | INFO | PRE_HQ : 66.4%, 1.8, 60.6ms, 0 | 20260508 07:15:49.262 | INFO | BAZWRITER : 67.2%, 2.8, 96.4ms, 0 | 20260508 07:15:49.262 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:15:49.262 | INFO | Uploaded 36GiB to GPU in 2.61672s (13.7577GiB/s) | 20260508 07:15:49.262 | INFO | Downloaded 38GiB to GPU in 2.69263s (14.1126GiB/s) | 20260508 07:15:54.406 | INFO | Analyzing chunk frames = [6238720,6239232) | 20260508 07:15:54.406 | INFO | MemInfo: { | 20260508 07:15:54.406 | INFO | "gpu" : " 17 GB", | 20260508 07:15:54.406 | INFO | "huge" : "", | 20260508 07:15:54.406 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:54.406 | INFO | "malloc" : " 14 GB", | 20260508 07:15:54.406 | INFO | "pinned" : " 29 GB" | 20260508 07:15:54.406 | INFO | } | 20260508 07:15:54.406 | INFO | Perf report for the last 5.14464 seconds: | 20260508 07:15:54.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:54.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.406 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 07:15:54.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.406 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 07:15:54.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:54.407 | INFO | Uploaded 36GiB to GPU in 2.5149s (14.3147GiB/s) | 20260508 07:15:54.407 | INFO | Downloaded 34GiB to GPU in 2.42961s (13.994GiB/s) | 20260508 07:15:59.547 | INFO | Analyzing chunk frames = [6239232,6239744) | 20260508 07:15:59.547 | INFO | MemInfo: { | 20260508 07:15:59.547 | INFO | "gpu" : " 17 GB", | 20260508 07:15:59.547 | INFO | "huge" : "", | 20260508 07:15:59.547 | INFO | "huge_shared" : " 29 GB", | 20260508 07:15:59.547 | INFO | "malloc" : " 14 GB", | 20260508 07:15:59.547 | INFO | "pinned" : " 29 GB" | 20260508 07:15:59.547 | INFO | } | 20260508 07:15:59.547 | INFO | Perf report for the last 5.14121 seconds: | 20260508 07:15:59.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:15:59.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.547 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 07:15:59.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.547 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:15:59.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:15:59.548 | INFO | Uploaded 36GiB to GPU in 2.5739s (13.9865GiB/s) | 20260508 07:15:59.548 | INFO | Downloaded 34GiB to GPU in 2.42279s (14.0334GiB/s) | 20260508 07:16:04.689 | INFO | Analyzing chunk frames = [6239744,6240256) | 20260508 07:16:04.689 | INFO | MemInfo: { | 20260508 07:16:04.689 | INFO | "gpu" : " 17 GB", | 20260508 07:16:04.689 | INFO | "huge" : "", | 20260508 07:16:04.689 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:04.689 | INFO | "malloc" : " 14 GB", | 20260508 07:16:04.689 | INFO | "pinned" : " 29 GB" | 20260508 07:16:04.689 | INFO | } | 20260508 07:16:04.689 | INFO | Perf report for the last 5.14202 seconds: | 20260508 07:16:04.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:04.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.689 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:16:04.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.689 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:16:04.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:04.690 | INFO | Uploaded 36GiB to GPU in 2.59863s (13.8535GiB/s) | 20260508 07:16:04.690 | INFO | Downloaded 34GiB to GPU in 2.40341s (14.1465GiB/s) | 20260508 07:16:09.826 | INFO | Analyzing chunk frames = [6240256,6240768) | 20260508 07:16:09.826 | INFO | MemInfo: { | 20260508 07:16:09.826 | INFO | "gpu" : " 17 GB", | 20260508 07:16:09.826 | INFO | "huge" : "", | 20260508 07:16:09.826 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:09.826 | INFO | "malloc" : " 14 GB", | 20260508 07:16:09.826 | INFO | "pinned" : " 29 GB" | 20260508 07:16:09.826 | INFO | } | 20260508 07:16:09.826 | INFO | Perf report for the last 5.13667 seconds: | 20260508 07:16:09.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:09.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:16:09.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:16:09.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:09.826 | INFO | Uploaded 36GiB to GPU in 2.28291s (15.7694GiB/s) | 20260508 07:16:09.826 | INFO | Downloaded 34GiB to GPU in 2.49778s (13.6121GiB/s) | 20260508 07:16:14.965 | INFO | Analyzing chunk frames = [6240768,6241280) | 20260508 07:16:14.965 | INFO | MemInfo: { | 20260508 07:16:14.965 | INFO | "gpu" : " 17 GB", | 20260508 07:16:14.966 | INFO | "huge" : "", | 20260508 07:16:14.966 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:14.966 | INFO | "malloc" : " 14 GB", | 20260508 07:16:14.966 | INFO | "pinned" : " 29 GB" | 20260508 07:16:14.966 | INFO | } | 20260508 07:16:14.966 | INFO | Perf report for the last 5.13963 seconds: | 20260508 07:16:14.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:14.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:16:14.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:16:14.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:14.966 | INFO | Uploaded 36GiB to GPU in 2.37143s (15.1807GiB/s) | 20260508 07:16:14.966 | INFO | Downloaded 34GiB to GPU in 2.42616s (14.0139GiB/s) | 20260508 07:16:20.105 | INFO | Analyzing chunk frames = [6241280,6241792) | 20260508 07:16:20.105 | INFO | MemInfo: { | 20260508 07:16:20.105 | INFO | "gpu" : " 17 GB", | 20260508 07:16:20.105 | INFO | "huge" : "", | 20260508 07:16:20.105 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:20.105 | INFO | "malloc" : " 14 GB", | 20260508 07:16:20.105 | INFO | "pinned" : " 29 GB" | 20260508 07:16:20.105 | INFO | } | 20260508 07:16:20.105 | INFO | Perf report for the last 5.13948 seconds: | 20260508 07:16:20.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:20.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.105 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:16:20.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.105 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 07:16:20.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:20.106 | INFO | Uploaded 36GiB to GPU in 2.49228s (14.4446GiB/s) | 20260508 07:16:20.106 | INFO | Downloaded 34GiB to GPU in 2.43307s (13.9741GiB/s) | 20260508 07:16:25.248 | INFO | Analyzing chunk frames = [6241792,6242304) | 20260508 07:16:25.248 | INFO | MemInfo: { | 20260508 07:16:25.248 | INFO | "gpu" : " 17 GB", | 20260508 07:16:25.248 | INFO | "huge" : "", | 20260508 07:16:25.248 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:25.248 | INFO | "malloc" : " 14 GB", | 20260508 07:16:25.248 | INFO | "pinned" : " 29 GB" | 20260508 07:16:25.248 | INFO | } | 20260508 07:16:25.248 | INFO | Perf report for the last 5.14262 seconds: | 20260508 07:16:25.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:25.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:16:25.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 07:16:25.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:25.248 | INFO | Uploaded 36GiB to GPU in 2.49066s (14.454GiB/s) | 20260508 07:16:25.248 | INFO | Downloaded 34GiB to GPU in 2.43794s (13.9462GiB/s) | 20260508 07:16:25.248 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:16:25.248 | INFO | Total mem usage: 16.7 GiB | 20260508 07:16:25.248 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:16:25.248 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:16:25.248 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:16:25.248 | INFO | High Water Breakdown: | 20260508 07:16:25.248 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:16:25.248 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:16:25.248 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:16:25.248 | INFO | ------------------------------------------ | 20260508 07:16:25.248 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:16:25.248 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:16:25.249 | INFO | Total mem usage: 26.5 GiB | 20260508 07:16:25.249 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:16:25.249 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:16:25.249 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:16:25.249 | INFO | High Water Breakdown: | 20260508 07:16:25.249 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:16:25.249 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:16:25.249 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:16:25.249 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:16:25.249 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:16:25.249 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:16:25.249 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:16:25.249 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:16:25.249 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:16:25.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:16:25.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:16:25.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:16:25.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:16:25.249 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:16:25.249 | INFO | ------------------------------------------ | 20260508 07:16:25.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:16:25.249 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:16:25.249 | INFO | Total mem usage: 27.8 GiB | 20260508 07:16:25.249 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:16:25.249 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:16:25.249 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:16:25.249 | INFO | High Water Breakdown: | 20260508 07:16:25.249 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:16:25.249 | INFO | ------------------------------------------ | 20260508 07:16:25.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:16:25.249 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:16:25.249 | INFO | Total mem usage: 15.7 GiB | 20260508 07:16:25.249 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:16:25.249 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:16:25.249 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:16:25.249 | INFO | High Water Breakdown: | 20260508 07:16:25.249 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:16:25.249 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:16:25.249 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:16:25.249 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:16:25.249 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:16:25.249 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:16:25.249 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:16:25.249 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:16:25.249 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:16:25.249 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:16:25.249 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:16:25.249 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:16:25.249 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:16:25.249 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:16:25.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:16:25.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:16:25.249 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:16:25.249 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:16:25.249 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:16:25.249 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:16:25.249 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:16:25.249 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:16:25.249 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:16:25.249 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:16:25.249 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:16:25.249 | INFO | ------------------------------------------ | 20260508 07:16:25.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:16:25.249 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:16:25.249 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:16:25.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:16:25.249 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 5.64e-01 | 20260508 07:16:25.249 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.08e-02 / 5.05e-03 | 20260508 07:16:25.249 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.45e+00, 5.93e-01 / 1.87e-01 | 20260508 07:16:25.249 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.41e+00, 2.08e-01 / 1.46e-01 | 20260508 07:16:25.249 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.53e+01, 3.91e+00 / 1.25e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.52e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.10e-01 / 2.68e-01 | 20260508 07:16:25.249 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.21e+00, 2.53e-01 / 1.57e-01 | 20260508 07:16:25.249 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.41e-01, 4.95e-02 / 2.56e-02 | 20260508 07:16:25.249 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:16:25.249 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:16:25.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:16:25.249 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 24.49%, Upload -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 2.49%, LockUpload -- 2.63e+00 / 0.00e+00, 5.87e+00 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 2.37%, LockDownload -- 2.51e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 07:16:25.249 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 07:16:25.345 | INFO | Starting a round of Baz IO | 20260508 07:16:28.779 | INFO | Populating RTMetrics report | 20260508 07:16:28.779 | INFO | RTMetrics report populated | 20260508 07:16:28.779 | INFO | Opening tmp RTMetrics json file | 20260508 07:16:28.790 | INFO | Serializing RTMetrics to json | 20260508 07:16:28.791 | INFO | Serialization done, swapping tmp file now... | 20260508 07:16:28.792 | INFO | RTMetrics done | 20260508 07:16:28.848 | INFO | Finished a round of Baz IO: Wrote 1.531 GiB in 3.503 seconds | 20260508 07:16:28.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.58% / 0.00% / 59.42% | 20260508 07:16:28.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.352212% / 0% / 5.96046e-05% | 20260508 07:16:28.848 | INFO | Baz IO overall: | 20260508 07:16:28.848 | INFO | BytesWritten : 2756567185037 | 20260508 07:16:28.848 | INFO | NumEvents : 1328270665012 | 20260508 07:16:30.381 | INFO | Analyzing chunk frames = [6242304,6242816) | 20260508 07:16:30.381 | INFO | MemInfo: { | 20260508 07:16:30.381 | INFO | "gpu" : " 17 GB", | 20260508 07:16:30.381 | INFO | "huge" : "", | 20260508 07:16:30.381 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:30.381 | INFO | "malloc" : " 14 GB", | 20260508 07:16:30.381 | INFO | "pinned" : " 29 GB" | 20260508 07:16:30.381 | INFO | } | 20260508 07:16:30.381 | INFO | Perf report for the last 5.13353 seconds: | 20260508 07:16:30.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:30.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:30.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:30.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:30.382 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 07:16:30.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:30.382 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:16:30.382 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 07:16:30.382 | INFO | BAZWRITER : 67.6%, 2.6, 89.3ms, 0 | 20260508 07:16:30.382 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:16:30.382 | INFO | Uploaded 36GiB to GPU in 2.61544s (13.7644GiB/s) | 20260508 07:16:30.382 | INFO | Downloaded 38GiB to GPU in 2.70715s (14.0369GiB/s) | 20260508 07:16:35.519 | INFO | Analyzing chunk frames = [6242816,6243328) | 20260508 07:16:35.519 | INFO | MemInfo: { | 20260508 07:16:35.519 | INFO | "gpu" : " 17 GB", | 20260508 07:16:35.519 | INFO | "huge" : "", | 20260508 07:16:35.519 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:35.519 | INFO | "malloc" : " 14 GB", | 20260508 07:16:35.519 | INFO | "pinned" : " 29 GB" | 20260508 07:16:35.519 | INFO | } | 20260508 07:16:35.519 | INFO | Perf report for the last 5.13802 seconds: | 20260508 07:16:35.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:35.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.519 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:16:35.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.519 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:16:35.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:35.520 | INFO | Uploaded 36GiB to GPU in 2.50058s (14.3966GiB/s) | 20260508 07:16:35.520 | INFO | Downloaded 34GiB to GPU in 2.43046s (13.9891GiB/s) | 20260508 07:16:40.662 | INFO | Analyzing chunk frames = [6243328,6243840) | 20260508 07:16:40.662 | INFO | MemInfo: { | 20260508 07:16:40.662 | INFO | "gpu" : " 17 GB", | 20260508 07:16:40.662 | INFO | "huge" : "", | 20260508 07:16:40.662 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:40.662 | INFO | "malloc" : " 14 GB", | 20260508 07:16:40.662 | INFO | "pinned" : " 29 GB" | 20260508 07:16:40.662 | INFO | } | 20260508 07:16:40.662 | INFO | Perf report for the last 5.14241 seconds: | 20260508 07:16:40.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:40.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 07:16:40.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:16:40.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:40.662 | INFO | Uploaded 36GiB to GPU in 2.56276s (14.0474GiB/s) | 20260508 07:16:40.662 | INFO | Downloaded 34GiB to GPU in 2.43886s (13.9409GiB/s) | 20260508 07:16:45.802 | INFO | Analyzing chunk frames = [6243840,6244352) | 20260508 07:16:45.802 | INFO | MemInfo: { | 20260508 07:16:45.802 | INFO | "gpu" : " 17 GB", | 20260508 07:16:45.802 | INFO | "huge" : "", | 20260508 07:16:45.802 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:45.802 | INFO | "malloc" : " 14 GB", | 20260508 07:16:45.802 | INFO | "pinned" : " 29 GB" | 20260508 07:16:45.802 | INFO | } | 20260508 07:16:45.802 | INFO | Perf report for the last 5.14045 seconds: | 20260508 07:16:45.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:45.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.802 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 07:16:45.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.802 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:16:45.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:45.803 | INFO | Uploaded 36GiB to GPU in 2.56869s (14.0149GiB/s) | 20260508 07:16:45.803 | INFO | Downloaded 34GiB to GPU in 2.3961s (14.1897GiB/s) | 20260508 07:16:50.942 | INFO | Analyzing chunk frames = [6244352,6244864) | 20260508 07:16:50.942 | INFO | MemInfo: { | 20260508 07:16:50.942 | INFO | "gpu" : " 17 GB", | 20260508 07:16:50.942 | INFO | "huge" : "", | 20260508 07:16:50.942 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:50.942 | INFO | "malloc" : " 14 GB", | 20260508 07:16:50.942 | INFO | "pinned" : " 29 GB" | 20260508 07:16:50.942 | INFO | } | 20260508 07:16:50.942 | INFO | Perf report for the last 5.14014 seconds: | 20260508 07:16:50.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:50.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.942 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 07:16:50.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.943 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:16:50.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:50.943 | INFO | Uploaded 36GiB to GPU in 2.28353s (15.7651GiB/s) | 20260508 07:16:50.943 | INFO | Downloaded 34GiB to GPU in 2.49398s (13.6329GiB/s) | 20260508 07:16:56.081 | INFO | Analyzing chunk frames = [6244864,6245376) | 20260508 07:16:56.081 | INFO | MemInfo: { | 20260508 07:16:56.081 | INFO | "gpu" : " 17 GB", | 20260508 07:16:56.081 | INFO | "huge" : "", | 20260508 07:16:56.081 | INFO | "huge_shared" : " 29 GB", | 20260508 07:16:56.081 | INFO | "malloc" : " 14 GB", | 20260508 07:16:56.081 | INFO | "pinned" : " 29 GB" | 20260508 07:16:56.081 | INFO | } | 20260508 07:16:56.081 | INFO | Perf report for the last 5.13833 seconds: | 20260508 07:16:56.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:16:56.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:16:56.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 07:16:56.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:16:56.081 | INFO | Uploaded 36GiB to GPU in 2.38691s (15.0822GiB/s) | 20260508 07:16:56.081 | INFO | Downloaded 34GiB to GPU in 2.42416s (14.0255GiB/s) | 20260508 07:17:01.215 | INFO | Analyzing chunk frames = [6245376,6245888) | 20260508 07:17:01.215 | INFO | MemInfo: { | 20260508 07:17:01.215 | INFO | "gpu" : " 17 GB", | 20260508 07:17:01.215 | INFO | "huge" : "", | 20260508 07:17:01.215 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:01.215 | INFO | "malloc" : " 14 GB", | 20260508 07:17:01.215 | INFO | "pinned" : " 29 GB" | 20260508 07:17:01.215 | INFO | } | 20260508 07:17:01.215 | INFO | Perf report for the last 5.13461 seconds: | 20260508 07:17:01.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:01.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.215 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 07:17:01.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.216 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:17:01.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:01.216 | INFO | Uploaded 36GiB to GPU in 2.44149s (14.7451GiB/s) | 20260508 07:17:01.216 | INFO | Downloaded 34GiB to GPU in 2.42663s (14.0112GiB/s) | 20260508 07:17:06.355 | INFO | Analyzing chunk frames = [6245888,6246400) | 20260508 07:17:06.355 | INFO | MemInfo: { | 20260508 07:17:06.355 | INFO | "gpu" : " 17 GB", | 20260508 07:17:06.355 | INFO | "huge" : "", | 20260508 07:17:06.355 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:06.355 | INFO | "malloc" : " 14 GB", | 20260508 07:17:06.355 | INFO | "pinned" : " 29 GB" | 20260508 07:17:06.355 | INFO | } | 20260508 07:17:06.355 | INFO | Perf report for the last 5.13959 seconds: | 20260508 07:17:06.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:06.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:17:06.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 07:17:06.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:06.355 | INFO | Uploaded 36GiB to GPU in 2.482s (14.5044GiB/s) | 20260508 07:17:06.355 | INFO | Downloaded 34GiB to GPU in 2.42535s (14.0186GiB/s) | 20260508 07:17:06.452 | INFO | Starting a round of Baz IO | 20260508 07:17:09.873 | INFO | Populating RTMetrics report | 20260508 07:17:09.873 | INFO | RTMetrics report populated | 20260508 07:17:09.873 | INFO | Opening tmp RTMetrics json file | 20260508 07:17:09.884 | INFO | Serializing RTMetrics to json | 20260508 07:17:09.886 | INFO | Serialization done, swapping tmp file now... | 20260508 07:17:09.887 | INFO | RTMetrics done | 20260508 07:17:09.946 | INFO | Finished a round of Baz IO: Wrote 1.531 GiB in 3.494 seconds | 20260508 07:17:09.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.58% / 0.00% / 59.42% | 20260508 07:17:09.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.351536% / 3.97364e-06% / 8.74201e-05% | 20260508 07:17:09.946 | INFO | Baz IO overall: | 20260508 07:17:09.946 | INFO | BytesWritten : 2758211187910 | 20260508 07:17:09.946 | INFO | NumEvents : 1329051791108 | 20260508 07:17:11.501 | INFO | Analyzing chunk frames = [6246400,6246912) | 20260508 07:17:11.501 | INFO | MemInfo: { | 20260508 07:17:11.501 | INFO | "gpu" : " 17 GB", | 20260508 07:17:11.501 | INFO | "huge" : "", | 20260508 07:17:11.501 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:11.501 | INFO | "malloc" : " 14 GB", | 20260508 07:17:11.501 | INFO | "pinned" : " 29 GB" | 20260508 07:17:11.501 | INFO | } | 20260508 07:17:11.501 | INFO | Perf report for the last 5.14616 seconds: | 20260508 07:17:11.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:11.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:11.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:11.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:11.501 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 07:17:11.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:11.501 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:17:11.501 | INFO | PRE_HQ : 65.4%, 1.6, 55.2ms, 0 | 20260508 07:17:11.501 | INFO | BAZWRITER : 67.3%, 2.6, 89.9ms, 0 | 20260508 07:17:11.501 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:17:11.502 | INFO | Uploaded 36GiB to GPU in 2.59501s (13.8728GiB/s) | 20260508 07:17:11.502 | INFO | Downloaded 38GiB to GPU in 2.678s (14.1897GiB/s) | 20260508 07:17:16.640 | INFO | Analyzing chunk frames = [6246912,6247424) | 20260508 07:17:16.640 | INFO | MemInfo: { | 20260508 07:17:16.640 | INFO | "gpu" : " 17 GB", | 20260508 07:17:16.640 | INFO | "huge" : "", | 20260508 07:17:16.640 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:16.640 | INFO | "malloc" : " 14 GB", | 20260508 07:17:16.640 | INFO | "pinned" : " 29 GB" | 20260508 07:17:16.640 | INFO | } | 20260508 07:17:16.640 | INFO | Perf report for the last 5.1393 seconds: | 20260508 07:17:16.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:16.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.641 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:17:16.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.641 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:17:16.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:16.641 | INFO | Uploaded 36GiB to GPU in 2.49637s (14.4209GiB/s) | 20260508 07:17:16.641 | INFO | Downloaded 34GiB to GPU in 2.42884s (13.9985GiB/s) | 20260508 07:17:21.775 | INFO | Analyzing chunk frames = [6247424,6247936) | 20260508 07:17:21.775 | INFO | MemInfo: { | 20260508 07:17:21.775 | INFO | "gpu" : " 17 GB", | 20260508 07:17:21.775 | INFO | "huge" : "", | 20260508 07:17:21.775 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:21.775 | INFO | "malloc" : " 14 GB", | 20260508 07:17:21.775 | INFO | "pinned" : " 29 GB" | 20260508 07:17:21.775 | INFO | } | 20260508 07:17:21.775 | INFO | Perf report for the last 5.13449 seconds: | 20260508 07:17:21.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:21.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:17:21.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:17:21.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:21.775 | INFO | Uploaded 36GiB to GPU in 2.57327s (13.99GiB/s) | 20260508 07:17:21.775 | INFO | Downloaded 34GiB to GPU in 2.42632s (14.013GiB/s) | 20260508 07:17:26.923 | INFO | Analyzing chunk frames = [6247936,6248448) | 20260508 07:17:26.923 | INFO | MemInfo: { | 20260508 07:17:26.923 | INFO | "gpu" : " 17 GB", | 20260508 07:17:26.923 | INFO | "huge" : "", | 20260508 07:17:26.923 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:26.923 | INFO | "malloc" : " 14 GB", | 20260508 07:17:26.923 | INFO | "pinned" : " 29 GB" | 20260508 07:17:26.923 | INFO | } | 20260508 07:17:26.923 | INFO | Perf report for the last 5.14833 seconds: | 20260508 07:17:26.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:26.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.923 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 07:17:26.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.923 | INFO | PRE_HQ : 23.7%, 1.0, 12.4ms, 0 | 20260508 07:17:26.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:26.924 | INFO | Uploaded 36GiB to GPU in 2.59163s (13.8909GiB/s) | 20260508 07:17:26.924 | INFO | Downloaded 34GiB to GPU in 2.40878s (14.115GiB/s) | 20260508 07:17:32.058 | INFO | Analyzing chunk frames = [6248448,6248960) | 20260508 07:17:32.058 | INFO | MemInfo: { | 20260508 07:17:32.058 | INFO | "gpu" : " 17 GB", | 20260508 07:17:32.058 | INFO | "huge" : "", | 20260508 07:17:32.058 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:32.058 | INFO | "malloc" : " 14 GB", | 20260508 07:17:32.058 | INFO | "pinned" : " 29 GB" | 20260508 07:17:32.058 | INFO | } | 20260508 07:17:32.059 | INFO | Perf report for the last 5.13519 seconds: | 20260508 07:17:32.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:32.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 07:17:32.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260508 07:17:32.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:32.059 | INFO | Uploaded 36GiB to GPU in 2.40597s (14.9628GiB/s) | 20260508 07:17:32.059 | INFO | Downloaded 34GiB to GPU in 2.52273s (13.4774GiB/s) | 20260508 07:17:37.195 | INFO | Analyzing chunk frames = [6248960,6249472) | 20260508 07:17:37.195 | INFO | MemInfo: { | 20260508 07:17:37.195 | INFO | "gpu" : " 17 GB", | 20260508 07:17:37.195 | INFO | "huge" : "", | 20260508 07:17:37.195 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:37.195 | INFO | "malloc" : " 14 GB", | 20260508 07:17:37.195 | INFO | "pinned" : " 29 GB" | 20260508 07:17:37.195 | INFO | } | 20260508 07:17:37.195 | INFO | Perf report for the last 5.13654 seconds: | 20260508 07:17:37.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:37.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:17:37.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:17:37.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:37.195 | INFO | Uploaded 36GiB to GPU in 2.35608s (15.2796GiB/s) | 20260508 07:17:37.195 | INFO | Downloaded 34GiB to GPU in 2.43253s (13.9772GiB/s) | 20260508 07:17:42.328 | INFO | Analyzing chunk frames = [6249472,6249984) | 20260508 07:17:42.328 | INFO | MemInfo: { | 20260508 07:17:42.329 | INFO | "gpu" : " 17 GB", | 20260508 07:17:42.329 | INFO | "huge" : "", | 20260508 07:17:42.329 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:42.329 | INFO | "malloc" : " 14 GB", | 20260508 07:17:42.329 | INFO | "pinned" : " 29 GB" | 20260508 07:17:42.329 | INFO | } | 20260508 07:17:42.329 | INFO | Perf report for the last 5.1336 seconds: | 20260508 07:17:42.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:42.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 07:17:42.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:17:42.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:42.329 | INFO | Uploaded 36GiB to GPU in 2.47455s (14.5481GiB/s) | 20260508 07:17:42.329 | INFO | Downloaded 34GiB to GPU in 2.43645s (13.9547GiB/s) | 20260508 07:17:47.475 | INFO | Analyzing chunk frames = [6249984,6250496) | 20260508 07:17:47.475 | INFO | MemInfo: { | 20260508 07:17:47.475 | INFO | "gpu" : " 17 GB", | 20260508 07:17:47.475 | INFO | "huge" : "", | 20260508 07:17:47.475 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:47.475 | INFO | "malloc" : " 14 GB", | 20260508 07:17:47.475 | INFO | "pinned" : " 29 GB" | 20260508 07:17:47.475 | INFO | } | 20260508 07:17:47.475 | INFO | Perf report for the last 5.14615 seconds: | 20260508 07:17:47.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:47.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:17:47.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:17:47.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:47.475 | INFO | Uploaded 36GiB to GPU in 2.45747s (14.6492GiB/s) | 20260508 07:17:47.475 | INFO | Downloaded 34GiB to GPU in 2.43514s (13.9622GiB/s) | 20260508 07:17:47.475 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:17:47.475 | INFO | Total mem usage: 16.7 GiB | 20260508 07:17:47.475 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:17:47.475 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:17:47.475 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:17:47.475 | INFO | High Water Breakdown: | 20260508 07:17:47.475 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:17:47.475 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:17:47.475 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:17:47.475 | INFO | ------------------------------------------ | 20260508 07:17:47.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:17:47.475 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:17:47.475 | INFO | Total mem usage: 26.5 GiB | 20260508 07:17:47.476 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:17:47.476 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:17:47.476 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:17:47.476 | INFO | High Water Breakdown: | 20260508 07:17:47.476 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:17:47.476 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:17:47.476 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:17:47.476 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:17:47.476 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:17:47.476 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:17:47.476 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:17:47.476 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:17:47.476 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:17:47.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:17:47.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:17:47.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:17:47.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:17:47.476 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:17:47.476 | INFO | ------------------------------------------ | 20260508 07:17:47.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:17:47.476 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:17:47.476 | INFO | Total mem usage: 27.8 GiB | 20260508 07:17:47.476 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:17:47.476 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:17:47.476 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:17:47.476 | INFO | High Water Breakdown: | 20260508 07:17:47.476 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:17:47.476 | INFO | ------------------------------------------ | 20260508 07:17:47.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:17:47.476 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:17:47.476 | INFO | Total mem usage: 15.7 GiB | 20260508 07:17:47.476 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:17:47.476 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:17:47.476 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:17:47.476 | INFO | High Water Breakdown: | 20260508 07:17:47.476 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:17:47.476 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:17:47.476 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:17:47.476 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:17:47.476 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:17:47.476 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:17:47.476 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:17:47.476 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:17:47.476 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:17:47.476 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:17:47.476 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:17:47.476 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:17:47.476 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:17:47.476 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:17:47.476 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:17:47.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:17:47.477 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:17:47.477 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:17:47.477 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:17:47.477 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:17:47.477 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:17:47.477 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:17:47.477 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:17:47.477 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:17:47.477 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:17:47.477 | INFO | ------------------------------------------ | 20260508 07:17:47.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:17:47.477 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:17:47.477 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:17:47.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:17:47.477 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.53e-01 | 20260508 07:17:47.477 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.08e-02 / 3.26e-03 | 20260508 07:17:47.477 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.46e+00, 5.88e-01 / 1.86e-01 | 20260508 07:17:47.477 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.11e-01 / 1.23e-01 | 20260508 07:17:47.477 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.52e+01, 3.66e+00 / 9.35e-01 | 20260508 07:17:47.477 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.83e+00 / 1.42e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.60e+00, 4.13e-01 / 2.27e-01 | 20260508 07:17:47.477 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.28e+00, 2.56e-01 / 1.48e-01 | 20260508 07:17:47.477 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.09e-01 / 3.54e-01, 4.93e-02 / 2.57e-02 | 20260508 07:17:47.477 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:17:47.477 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:17:47.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:17:47.477 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 24.46%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 24.29%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 30.87%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 2.41%, LockUpload -- 2.55e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 2.40%, LockDownload -- 2.54e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 07:17:47.477 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260508 07:17:47.572 | INFO | Starting a round of Baz IO | 20260508 07:17:50.992 | INFO | Populating RTMetrics report | 20260508 07:17:50.992 | INFO | RTMetrics report populated | 20260508 07:17:50.992 | INFO | Opening tmp RTMetrics json file | 20260508 07:17:50.993 | INFO | Serializing RTMetrics to json | 20260508 07:17:50.994 | INFO | Serialization done, swapping tmp file now... | 20260508 07:17:50.995 | INFO | RTMetrics done | 20260508 07:17:51.060 | INFO | Finished a round of Baz IO: Wrote 1.53 GiB in 3.488 seconds | 20260508 07:17:51.060 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.00% / 59.42% | 20260508 07:17:51.060 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.351143% / 0% / 3.97364e-05% | 20260508 07:17:51.060 | INFO | Baz IO overall: | 20260508 07:17:51.060 | INFO | BytesWritten : 2759854095427 | 20260508 07:17:51.060 | INFO | NumEvents : 1329832359517 | 20260508 07:17:52.615 | INFO | Analyzing chunk frames = [6250496,6251008) | 20260508 07:17:52.615 | INFO | MemInfo: { | 20260508 07:17:52.615 | INFO | "gpu" : " 17 GB", | 20260508 07:17:52.615 | INFO | "huge" : "", | 20260508 07:17:52.615 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:52.615 | INFO | "malloc" : " 14 GB", | 20260508 07:17:52.615 | INFO | "pinned" : " 29 GB" | 20260508 07:17:52.615 | INFO | } | 20260508 07:17:52.616 | INFO | Perf report for the last 5.14067 seconds: | 20260508 07:17:52.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:52.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:52.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:52.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:52.616 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 07:17:52.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:52.616 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:17:52.616 | INFO | PRE_HQ : 65.7%, 1.6, 55.8ms, 0 | 20260508 07:17:52.616 | INFO | BAZWRITER : 67.2%, 2.6, 90.4ms, 0 | 20260508 07:17:52.616 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:17:52.616 | INFO | Uploaded 36GiB to GPU in 2.5328s (14.2135GiB/s) | 20260508 07:17:52.616 | INFO | Downloaded 38GiB to GPU in 2.70192s (14.0641GiB/s) | 20260508 07:17:57.758 | INFO | Analyzing chunk frames = [6251008,6251520) | 20260508 07:17:57.758 | INFO | MemInfo: { | 20260508 07:17:57.758 | INFO | "gpu" : " 17 GB", | 20260508 07:17:57.758 | INFO | "huge" : "", | 20260508 07:17:57.758 | INFO | "huge_shared" : " 29 GB", | 20260508 07:17:57.758 | INFO | "malloc" : " 14 GB", | 20260508 07:17:57.758 | INFO | "pinned" : " 29 GB" | 20260508 07:17:57.758 | INFO | } | 20260508 07:17:57.758 | INFO | Perf report for the last 5.14276 seconds: | 20260508 07:17:57.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:17:57.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.758 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:17:57.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.758 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:17:57.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:17:57.759 | INFO | Uploaded 36GiB to GPU in 2.48474s (14.4884GiB/s) | 20260508 07:17:57.759 | INFO | Downloaded 34GiB to GPU in 2.43943s (13.9377GiB/s) | 20260508 07:18:02.891 | INFO | Analyzing chunk frames = [6251520,6252032) | 20260508 07:18:02.891 | INFO | MemInfo: { | 20260508 07:18:02.891 | INFO | "gpu" : " 17 GB", | 20260508 07:18:02.891 | INFO | "huge" : "", | 20260508 07:18:02.891 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:02.891 | INFO | "malloc" : " 14 GB", | 20260508 07:18:02.891 | INFO | "pinned" : " 29 GB" | 20260508 07:18:02.891 | INFO | } | 20260508 07:18:02.891 | INFO | Perf report for the last 5.13254 seconds: | 20260508 07:18:02.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:02.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 07:18:02.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:18:02.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:02.891 | INFO | Uploaded 36GiB to GPU in 2.48398s (14.4929GiB/s) | 20260508 07:18:02.891 | INFO | Downloaded 34GiB to GPU in 2.4421s (13.9225GiB/s) | 20260508 07:18:08.031 | INFO | Analyzing chunk frames = [6252032,6252544) | 20260508 07:18:08.031 | INFO | MemInfo: { | 20260508 07:18:08.031 | INFO | "gpu" : " 17 GB", | 20260508 07:18:08.031 | INFO | "huge" : "", | 20260508 07:18:08.031 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:08.031 | INFO | "malloc" : " 14 GB", | 20260508 07:18:08.031 | INFO | "pinned" : " 29 GB" | 20260508 07:18:08.031 | INFO | } | 20260508 07:18:08.032 | INFO | Perf report for the last 5.14066 seconds: | 20260508 07:18:08.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:08.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 07:18:08.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:18:08.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:08.032 | INFO | Uploaded 36GiB to GPU in 2.55215s (14.1057GiB/s) | 20260508 07:18:08.032 | INFO | Downloaded 34GiB to GPU in 2.42016s (14.0486GiB/s) | 20260508 07:18:13.172 | INFO | Analyzing chunk frames = [6252544,6253056) | 20260508 07:18:13.172 | INFO | MemInfo: { | 20260508 07:18:13.172 | INFO | "gpu" : " 17 GB", | 20260508 07:18:13.172 | INFO | "huge" : "", | 20260508 07:18:13.172 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:13.172 | INFO | "malloc" : " 14 GB", | 20260508 07:18:13.172 | INFO | "pinned" : " 29 GB" | 20260508 07:18:13.172 | INFO | } | 20260508 07:18:13.172 | INFO | Perf report for the last 5.14052 seconds: | 20260508 07:18:13.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:13.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.172 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:18:13.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.172 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 07:18:13.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:13.173 | INFO | Uploaded 36GiB to GPU in 2.28509s (15.7543GiB/s) | 20260508 07:18:13.173 | INFO | Downloaded 34GiB to GPU in 2.49953s (13.6026GiB/s) | 20260508 07:18:18.313 | INFO | Analyzing chunk frames = [6253056,6253568) | 20260508 07:18:18.313 | INFO | MemInfo: { | 20260508 07:18:18.313 | INFO | "gpu" : " 17 GB", | 20260508 07:18:18.313 | INFO | "huge" : "", | 20260508 07:18:18.313 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:18.313 | INFO | "malloc" : " 14 GB", | 20260508 07:18:18.313 | INFO | "pinned" : " 29 GB" | 20260508 07:18:18.313 | INFO | } | 20260508 07:18:18.313 | INFO | Perf report for the last 5.14092 seconds: | 20260508 07:18:18.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:18.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.313 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:18:18.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.313 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:18:18.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:18.314 | INFO | Uploaded 36GiB to GPU in 2.38014s (15.1251GiB/s) | 20260508 07:18:18.314 | INFO | Downloaded 34GiB to GPU in 2.42624s (14.0134GiB/s) | 20260508 07:18:23.447 | INFO | Analyzing chunk frames = [6253568,6254080) | 20260508 07:18:23.447 | INFO | MemInfo: { | 20260508 07:18:23.447 | INFO | "gpu" : " 17 GB", | 20260508 07:18:23.447 | INFO | "huge" : "", | 20260508 07:18:23.447 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:23.447 | INFO | "malloc" : " 14 GB", | 20260508 07:18:23.447 | INFO | "pinned" : " 29 GB" | 20260508 07:18:23.447 | INFO | } | 20260508 07:18:23.447 | INFO | Perf report for the last 5.13441 seconds: | 20260508 07:18:23.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:23.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.447 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:18:23.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.447 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:18:23.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:23.448 | INFO | Uploaded 36GiB to GPU in 2.49013s (14.4571GiB/s) | 20260508 07:18:23.448 | INFO | Downloaded 34GiB to GPU in 2.43411s (13.9682GiB/s) | 20260508 07:18:28.588 | INFO | Analyzing chunk frames = [6254080,6254592) | 20260508 07:18:28.588 | INFO | MemInfo: { | 20260508 07:18:28.588 | INFO | "gpu" : " 17 GB", | 20260508 07:18:28.588 | INFO | "huge" : "", | 20260508 07:18:28.588 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:28.588 | INFO | "malloc" : " 14 GB", | 20260508 07:18:28.588 | INFO | "pinned" : " 29 GB" | 20260508 07:18:28.588 | INFO | } | 20260508 07:18:28.588 | INFO | Perf report for the last 5.14071 seconds: | 20260508 07:18:28.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:28.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 07:18:28.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 07:18:28.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:28.588 | INFO | Uploaded 36GiB to GPU in 2.47957s (14.5186GiB/s) | 20260508 07:18:28.588 | INFO | Downloaded 34GiB to GPU in 2.43781s (13.9469GiB/s) | 20260508 07:18:28.685 | INFO | Starting a round of Baz IO | 20260508 07:18:32.099 | INFO | Populating RTMetrics report | 20260508 07:18:32.100 | INFO | RTMetrics report populated | 20260508 07:18:32.100 | INFO | Opening tmp RTMetrics json file | 20260508 07:18:32.110 | INFO | Serializing RTMetrics to json | 20260508 07:18:32.112 | INFO | Serialization done, swapping tmp file now... | 20260508 07:18:32.112 | INFO | RTMetrics done | 20260508 07:18:32.171 | INFO | Finished a round of Baz IO: Wrote 1.53 GiB in 3.486 seconds | 20260508 07:18:32.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.00% / 59.43% | 20260508 07:18:32.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.350654% / 0% / 3.17891e-05% | 20260508 07:18:32.172 | INFO | Baz IO overall: | 20260508 07:18:32.172 | INFO | BytesWritten : 2761497093225 | 20260508 07:18:32.172 | INFO | NumEvents : 1330613051350 | 20260508 07:18:33.728 | INFO | Analyzing chunk frames = [6254592,6255104) | 20260508 07:18:33.728 | INFO | MemInfo: { | 20260508 07:18:33.728 | INFO | "gpu" : " 17 GB", | 20260508 07:18:33.729 | INFO | "huge" : "", | 20260508 07:18:33.729 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:33.729 | INFO | "malloc" : " 14 GB", | 20260508 07:18:33.729 | INFO | "pinned" : " 29 GB" | 20260508 07:18:33.729 | INFO | } | 20260508 07:18:33.729 | INFO | Perf report for the last 5.14049 seconds: | 20260508 07:18:33.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:33.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:33.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:33.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:33.729 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:18:33.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:33.729 | INFO | RT_METRICS : 39.4%, 1.0, 20.7ms, 0 | 20260508 07:18:33.729 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 07:18:33.729 | INFO | BAZWRITER : 67.2%, 2.6, 90.9ms, 0 | 20260508 07:18:33.729 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:18:33.729 | INFO | Uploaded 36GiB to GPU in 2.6075s (13.8063GiB/s) | 20260508 07:18:33.729 | INFO | Downloaded 38GiB to GPU in 2.70051s (14.0714GiB/s) | 20260508 07:18:38.868 | INFO | Analyzing chunk frames = [6255104,6255616) | 20260508 07:18:38.868 | INFO | MemInfo: { | 20260508 07:18:38.868 | INFO | "gpu" : " 17 GB", | 20260508 07:18:38.868 | INFO | "huge" : "", | 20260508 07:18:38.868 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:38.868 | INFO | "malloc" : " 14 GB", | 20260508 07:18:38.868 | INFO | "pinned" : " 29 GB" | 20260508 07:18:38.868 | INFO | } | 20260508 07:18:38.868 | INFO | Perf report for the last 5.13949 seconds: | 20260508 07:18:38.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:38.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.868 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:18:38.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.868 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 07:18:38.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:38.869 | INFO | Uploaded 36GiB to GPU in 2.50712s (14.3591GiB/s) | 20260508 07:18:38.869 | INFO | Downloaded 34GiB to GPU in 2.43452s (13.9658GiB/s) | 20260508 07:18:44.007 | INFO | Analyzing chunk frames = [6255616,6256128) | 20260508 07:18:44.007 | INFO | MemInfo: { | 20260508 07:18:44.007 | INFO | "gpu" : " 17 GB", | 20260508 07:18:44.007 | INFO | "huge" : "", | 20260508 07:18:44.007 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:44.007 | INFO | "malloc" : " 14 GB", | 20260508 07:18:44.007 | INFO | "pinned" : " 29 GB" | 20260508 07:18:44.007 | INFO | } | 20260508 07:18:44.007 | INFO | Perf report for the last 5.13895 seconds: | 20260508 07:18:44.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:44.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 07:18:44.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:18:44.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:44.007 | INFO | Uploaded 36GiB to GPU in 2.58719s (13.9147GiB/s) | 20260508 07:18:44.007 | INFO | Downloaded 34GiB to GPU in 2.42758s (14.0057GiB/s) | 20260508 07:18:49.144 | INFO | Analyzing chunk frames = [6256128,6256640) | 20260508 07:18:49.144 | INFO | MemInfo: { | 20260508 07:18:49.144 | INFO | "gpu" : " 17 GB", | 20260508 07:18:49.144 | INFO | "huge" : "", | 20260508 07:18:49.144 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:49.144 | INFO | "malloc" : " 14 GB", | 20260508 07:18:49.144 | INFO | "pinned" : " 29 GB" | 20260508 07:18:49.144 | INFO | } | 20260508 07:18:49.144 | INFO | Perf report for the last 5.13686 seconds: | 20260508 07:18:49.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:49.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 07:18:49.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:18:49.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:49.144 | INFO | Uploaded 36GiB to GPU in 2.58584s (13.9219GiB/s) | 20260508 07:18:49.144 | INFO | Downloaded 34GiB to GPU in 2.41065s (14.1041GiB/s) | 20260508 07:18:54.280 | INFO | Analyzing chunk frames = [6256640,6257152) | 20260508 07:18:54.280 | INFO | MemInfo: { | 20260508 07:18:54.280 | INFO | "gpu" : " 17 GB", | 20260508 07:18:54.281 | INFO | "huge" : "", | 20260508 07:18:54.281 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:54.281 | INFO | "malloc" : " 14 GB", | 20260508 07:18:54.281 | INFO | "pinned" : " 29 GB" | 20260508 07:18:54.281 | INFO | } | 20260508 07:18:54.281 | INFO | Perf report for the last 5.13665 seconds: | 20260508 07:18:54.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:54.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 07:18:54.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:18:54.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:54.281 | INFO | Uploaded 36GiB to GPU in 2.41609s (14.9001GiB/s) | 20260508 07:18:54.281 | INFO | Downloaded 34GiB to GPU in 2.51479s (13.52GiB/s) | 20260508 07:18:59.424 | INFO | Analyzing chunk frames = [6257152,6257664) | 20260508 07:18:59.424 | INFO | MemInfo: { | 20260508 07:18:59.424 | INFO | "gpu" : " 17 GB", | 20260508 07:18:59.424 | INFO | "huge" : "", | 20260508 07:18:59.424 | INFO | "huge_shared" : " 29 GB", | 20260508 07:18:59.424 | INFO | "malloc" : " 14 GB", | 20260508 07:18:59.424 | INFO | "pinned" : " 29 GB" | 20260508 07:18:59.424 | INFO | } | 20260508 07:18:59.424 | INFO | Perf report for the last 5.1431 seconds: | 20260508 07:18:59.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:18:59.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:18:59.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:18:59.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:18:59.424 | INFO | Uploaded 36GiB to GPU in 2.47671s (14.5354GiB/s) | 20260508 07:18:59.424 | INFO | Downloaded 34GiB to GPU in 2.425s (14.0206GiB/s) | 20260508 07:19:04.565 | INFO | Analyzing chunk frames = [6257664,6258176) | 20260508 07:19:04.565 | INFO | MemInfo: { | 20260508 07:19:04.565 | INFO | "gpu" : " 17 GB", | 20260508 07:19:04.565 | INFO | "huge" : "", | 20260508 07:19:04.565 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:04.565 | INFO | "malloc" : " 14 GB", | 20260508 07:19:04.565 | INFO | "pinned" : " 29 GB" | 20260508 07:19:04.565 | INFO | } | 20260508 07:19:04.565 | INFO | Perf report for the last 5.14129 seconds: | 20260508 07:19:04.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:04.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.565 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:19:04.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.565 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:19:04.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:04.566 | INFO | Uploaded 36GiB to GPU in 2.5066s (14.3621GiB/s) | 20260508 07:19:04.566 | INFO | Downloaded 34GiB to GPU in 2.43816s (13.9449GiB/s) | 20260508 07:19:09.713 | INFO | Analyzing chunk frames = [6258176,6258688) | 20260508 07:19:09.713 | INFO | MemInfo: { | 20260508 07:19:09.713 | INFO | "gpu" : " 17 GB", | 20260508 07:19:09.713 | INFO | "huge" : "", | 20260508 07:19:09.713 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:09.713 | INFO | "malloc" : " 14 GB", | 20260508 07:19:09.713 | INFO | "pinned" : " 29 GB" | 20260508 07:19:09.713 | INFO | } | 20260508 07:19:09.713 | INFO | Perf report for the last 5.14808 seconds: | 20260508 07:19:09.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:09.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:19:09.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:19:09.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:09.713 | INFO | Uploaded 36GiB to GPU in 2.49282s (14.4415GiB/s) | 20260508 07:19:09.713 | INFO | Downloaded 34GiB to GPU in 2.43475s (13.9645GiB/s) | 20260508 07:19:09.714 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:19:09.714 | INFO | Total mem usage: 16.7 GiB | 20260508 07:19:09.714 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:19:09.714 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:19:09.714 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:19:09.714 | INFO | High Water Breakdown: | 20260508 07:19:09.714 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:19:09.714 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:19:09.714 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:19:09.714 | INFO | ------------------------------------------ | 20260508 07:19:09.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:19:09.714 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:19:09.714 | INFO | Total mem usage: 26.5 GiB | 20260508 07:19:09.714 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:19:09.714 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:19:09.714 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:19:09.714 | INFO | High Water Breakdown: | 20260508 07:19:09.714 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:19:09.714 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:19:09.714 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:19:09.714 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:19:09.714 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:19:09.714 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:19:09.714 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:19:09.714 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:19:09.714 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:19:09.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:19:09.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:19:09.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:19:09.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:19:09.714 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:19:09.714 | INFO | ------------------------------------------ | 20260508 07:19:09.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:19:09.714 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:19:09.714 | INFO | Total mem usage: 27.8 GiB | 20260508 07:19:09.714 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:19:09.714 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:19:09.714 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:19:09.714 | INFO | High Water Breakdown: | 20260508 07:19:09.714 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:19:09.714 | INFO | ------------------------------------------ | 20260508 07:19:09.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:19:09.714 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:19:09.714 | INFO | Total mem usage: 15.7 GiB | 20260508 07:19:09.714 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:19:09.714 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:19:09.714 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:19:09.714 | INFO | High Water Breakdown: | 20260508 07:19:09.714 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:19:09.714 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:19:09.714 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:19:09.714 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:19:09.714 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:19:09.714 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:19:09.714 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:19:09.714 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:19:09.714 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:19:09.714 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:19:09.714 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:19:09.714 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:19:09.714 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:19:09.714 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:19:09.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:19:09.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:19:09.714 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:19:09.714 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:19:09.714 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:19:09.714 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:19:09.714 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:19:09.714 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:19:09.714 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:19:09.714 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:19:09.714 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:19:09.714 | INFO | ------------------------------------------ | 20260508 07:19:09.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:19:09.714 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:19:09.714 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:19:09.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:19:09.714 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.37e+01 / 6.84e-01 | 20260508 07:19:09.714 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.04e-01, 1.11e-02 / 3.39e-03 | 20260508 07:19:09.714 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.42e+00, 5.88e-01 / 2.57e-01 | 20260508 07:19:09.714 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.09e-01 | 20260508 07:19:09.714 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.52e+01, 3.78e+00 / 1.03e+00 | 20260508 07:19:09.714 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.84e+00 / 1.34e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.56e+00, 4.10e-01 / 1.81e-01 | 20260508 07:19:09.715 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.16e+00, 2.51e-01 / 1.19e-01 | 20260508 07:19:09.715 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.44e-01, 4.98e-02 / 2.26e-02 | 20260508 07:19:09.715 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:19:09.715 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:19:09.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:19:09.715 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 24.48%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 24.30%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 2.39%, LockUpload -- 2.54e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 2.42%, LockDownload -- 2.57e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 15.56%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:19:09.715 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 07:19:09.822 | INFO | Starting a round of Baz IO | 20260508 07:19:13.220 | INFO | Populating RTMetrics report | 20260508 07:19:13.220 | INFO | RTMetrics report populated | 20260508 07:19:13.220 | INFO | Opening tmp RTMetrics json file | 20260508 07:19:13.233 | INFO | Serializing RTMetrics to json | 20260508 07:19:13.234 | INFO | Serialization done, swapping tmp file now... | 20260508 07:19:13.235 | INFO | RTMetrics done | 20260508 07:19:13.292 | INFO | Finished a round of Baz IO: Wrote 1.528 GiB in 3.469 seconds | 20260508 07:19:13.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.00% / 59.43% | 20260508 07:19:13.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.349764% / 3.97364e-06% / 3.97364e-05% | 20260508 07:19:13.292 | INFO | Baz IO overall: | 20260508 07:19:13.292 | INFO | BytesWritten : 2763137895462 | 20260508 07:19:13.292 | INFO | NumEvents : 1331392186088 | 20260508 07:19:14.840 | INFO | Analyzing chunk frames = [6258688,6259200) | 20260508 07:19:14.840 | INFO | MemInfo: { | 20260508 07:19:14.840 | INFO | "gpu" : " 17 GB", | 20260508 07:19:14.840 | INFO | "huge" : "", | 20260508 07:19:14.840 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:14.840 | INFO | "malloc" : " 14 GB", | 20260508 07:19:14.840 | INFO | "pinned" : " 29 GB" | 20260508 07:19:14.840 | INFO | } | 20260508 07:19:14.840 | INFO | Perf report for the last 5.12711 seconds: | 20260508 07:19:14.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:14.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:14.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:14.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:14.840 | INFO | ANALYSIS : 67.8%, 3.0, 105.3ms, 0 | 20260508 07:19:14.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:14.840 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 07:19:14.840 | INFO | PRE_HQ : 66.7%, 1.8, 61.6ms, 0 | 20260508 07:19:14.840 | INFO | BAZWRITER : 67.0%, 2.8, 95.6ms, 0 | 20260508 07:19:14.840 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:19:14.841 | INFO | Uploaded 36GiB to GPU in 2.61534s (13.7649GiB/s) | 20260508 07:19:14.841 | INFO | Downloaded 38GiB to GPU in 2.70741s (14.0355GiB/s) | 20260508 07:19:19.986 | INFO | Analyzing chunk frames = [6259200,6259712) | 20260508 07:19:19.986 | INFO | MemInfo: { | 20260508 07:19:19.986 | INFO | "gpu" : " 17 GB", | 20260508 07:19:19.986 | INFO | "huge" : "", | 20260508 07:19:19.986 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:19.986 | INFO | "malloc" : " 14 GB", | 20260508 07:19:19.986 | INFO | "pinned" : " 29 GB" | 20260508 07:19:19.986 | INFO | } | 20260508 07:19:19.986 | INFO | Perf report for the last 5.14605 seconds: | 20260508 07:19:19.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:19.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.986 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 07:19:19.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.986 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:19:19.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:19.987 | INFO | Uploaded 36GiB to GPU in 2.50252s (14.3855GiB/s) | 20260508 07:19:19.987 | INFO | Downloaded 34GiB to GPU in 2.42744s (14.0065GiB/s) | 20260508 07:19:25.119 | INFO | Analyzing chunk frames = [6259712,6260224) | 20260508 07:19:25.119 | INFO | MemInfo: { | 20260508 07:19:25.119 | INFO | "gpu" : " 17 GB", | 20260508 07:19:25.119 | INFO | "huge" : "", | 20260508 07:19:25.119 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:25.119 | INFO | "malloc" : " 14 GB", | 20260508 07:19:25.119 | INFO | "pinned" : " 29 GB" | 20260508 07:19:25.119 | INFO | } | 20260508 07:19:25.119 | INFO | Perf report for the last 5.13287 seconds: | 20260508 07:19:25.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:25.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | ANALYSIS : 67.2%, 3.0, 104.2ms, 0 | 20260508 07:19:25.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:19:25.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:25.119 | INFO | Uploaded 36GiB to GPU in 2.57388s (13.9867GiB/s) | 20260508 07:19:25.120 | INFO | Downloaded 34GiB to GPU in 2.43545s (13.9605GiB/s) | 20260508 07:19:30.259 | INFO | Analyzing chunk frames = [6260224,6260736) | 20260508 07:19:30.259 | INFO | MemInfo: { | 20260508 07:19:30.259 | INFO | "gpu" : " 17 GB", | 20260508 07:19:30.259 | INFO | "huge" : "", | 20260508 07:19:30.259 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:30.259 | INFO | "malloc" : " 14 GB", | 20260508 07:19:30.259 | INFO | "pinned" : " 29 GB" | 20260508 07:19:30.259 | INFO | } | 20260508 07:19:30.259 | INFO | Perf report for the last 5.13998 seconds: | 20260508 07:19:30.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:30.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.259 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 07:19:30.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.259 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:19:30.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:30.260 | INFO | Uploaded 36GiB to GPU in 2.58428s (13.9304GiB/s) | 20260508 07:19:30.260 | INFO | Downloaded 34GiB to GPU in 2.4076s (14.122GiB/s) | 20260508 07:19:35.399 | INFO | Analyzing chunk frames = [6260736,6261248) | 20260508 07:19:35.399 | INFO | MemInfo: { | 20260508 07:19:35.399 | INFO | "gpu" : " 17 GB", | 20260508 07:19:35.399 | INFO | "huge" : "", | 20260508 07:19:35.399 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:35.399 | INFO | "malloc" : " 14 GB", | 20260508 07:19:35.399 | INFO | "pinned" : " 29 GB" | 20260508 07:19:35.399 | INFO | } | 20260508 07:19:35.399 | INFO | Perf report for the last 5.1402 seconds: | 20260508 07:19:35.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:35.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.400 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:19:35.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.400 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:19:35.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:35.400 | INFO | Uploaded 36GiB to GPU in 2.30788s (15.5988GiB/s) | 20260508 07:19:35.400 | INFO | Downloaded 34GiB to GPU in 2.48905s (13.6598GiB/s) | 20260508 07:19:40.538 | INFO | Analyzing chunk frames = [6261248,6261760) | 20260508 07:19:40.538 | INFO | MemInfo: { | 20260508 07:19:40.538 | INFO | "gpu" : " 17 GB", | 20260508 07:19:40.538 | INFO | "huge" : "", | 20260508 07:19:40.538 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:40.538 | INFO | "malloc" : " 14 GB", | 20260508 07:19:40.538 | INFO | "pinned" : " 29 GB" | 20260508 07:19:40.538 | INFO | } | 20260508 07:19:40.538 | INFO | Perf report for the last 5.13838 seconds: | 20260508 07:19:40.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:40.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:19:40.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:19:40.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:40.538 | INFO | Uploaded 36GiB to GPU in 2.46786s (14.5876GiB/s) | 20260508 07:19:40.538 | INFO | Downloaded 34GiB to GPU in 2.44106s (13.9284GiB/s) | 20260508 07:19:45.680 | INFO | Analyzing chunk frames = [6261760,6262272) | 20260508 07:19:45.680 | INFO | MemInfo: { | 20260508 07:19:45.680 | INFO | "gpu" : " 17 GB", | 20260508 07:19:45.681 | INFO | "huge" : "", | 20260508 07:19:45.681 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:45.681 | INFO | "malloc" : " 14 GB", | 20260508 07:19:45.681 | INFO | "pinned" : " 29 GB" | 20260508 07:19:45.681 | INFO | } | 20260508 07:19:45.681 | INFO | Perf report for the last 5.1429 seconds: | 20260508 07:19:45.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:45.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 07:19:45.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 07:19:45.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:45.681 | INFO | Uploaded 36GiB to GPU in 2.4633s (14.6145GiB/s) | 20260508 07:19:45.681 | INFO | Downloaded 34GiB to GPU in 2.43361s (13.971GiB/s) | 20260508 07:19:50.820 | INFO | Analyzing chunk frames = [6262272,6262784) | 20260508 07:19:50.820 | INFO | MemInfo: { | 20260508 07:19:50.820 | INFO | "gpu" : " 17 GB", | 20260508 07:19:50.820 | INFO | "huge" : "", | 20260508 07:19:50.820 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:50.821 | INFO | "malloc" : " 14 GB", | 20260508 07:19:50.821 | INFO | "pinned" : " 29 GB" | 20260508 07:19:50.821 | INFO | } | 20260508 07:19:50.821 | INFO | Perf report for the last 5.13991 seconds: | 20260508 07:19:50.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:50.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 07:19:50.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 07:19:50.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:50.821 | INFO | Uploaded 36GiB to GPU in 2.41819s (14.8871GiB/s) | 20260508 07:19:50.821 | INFO | Downloaded 34GiB to GPU in 2.43019s (13.9907GiB/s) | 20260508 07:19:50.935 | INFO | Starting a round of Baz IO | 20260508 07:19:54.342 | INFO | Populating RTMetrics report | 20260508 07:19:54.343 | INFO | RTMetrics report populated | 20260508 07:19:54.343 | INFO | Opening tmp RTMetrics json file | 20260508 07:19:54.351 | INFO | Serializing RTMetrics to json | 20260508 07:19:54.353 | INFO | Serialization done, swapping tmp file now... | 20260508 07:19:54.353 | INFO | RTMetrics done | 20260508 07:19:54.411 | INFO | Finished a round of Baz IO: Wrote 1.528 GiB in 3.476 seconds | 20260508 07:19:54.411 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.00% / 59.43% | 20260508 07:19:54.411 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.348032% / 0% / 4.37101e-05% | 20260508 07:19:54.411 | INFO | Baz IO overall: | 20260508 07:19:54.411 | INFO | BytesWritten : 2764778528751 | 20260508 07:19:54.411 | INFO | NumEvents : 1332171179201 | 20260508 07:19:55.956 | INFO | Analyzing chunk frames = [6262784,6263296) | 20260508 07:19:55.956 | INFO | MemInfo: { | 20260508 07:19:55.956 | INFO | "gpu" : " 17 GB", | 20260508 07:19:55.956 | INFO | "huge" : "", | 20260508 07:19:55.956 | INFO | "huge_shared" : " 29 GB", | 20260508 07:19:55.956 | INFO | "malloc" : " 14 GB", | 20260508 07:19:55.956 | INFO | "pinned" : " 29 GB" | 20260508 07:19:55.956 | INFO | } | 20260508 07:19:55.956 | INFO | Perf report for the last 5.13563 seconds: | 20260508 07:19:55.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:19:55.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:55.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:55.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:55.956 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 07:19:55.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:19:55.956 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 07:19:55.956 | INFO | PRE_HQ : 66.4%, 1.8, 61.7ms, 0 | 20260508 07:19:55.957 | INFO | BAZWRITER : 67.1%, 2.8, 97.0ms, 0 | 20260508 07:19:55.957 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:19:55.957 | INFO | Uploaded 36GiB to GPU in 2.58541s (13.9243GiB/s) | 20260508 07:19:55.957 | INFO | Downloaded 38GiB to GPU in 2.70156s (14.0659GiB/s) | 20260508 07:20:01.092 | INFO | Analyzing chunk frames = [6263296,6263808) | 20260508 07:20:01.092 | INFO | MemInfo: { | 20260508 07:20:01.092 | INFO | "gpu" : " 17 GB", | 20260508 07:20:01.093 | INFO | "huge" : "", | 20260508 07:20:01.093 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:01.093 | INFO | "malloc" : " 14 GB", | 20260508 07:20:01.093 | INFO | "pinned" : " 29 GB" | 20260508 07:20:01.093 | INFO | } | 20260508 07:20:01.093 | INFO | Perf report for the last 5.13633 seconds: | 20260508 07:20:01.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:01.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:20:01.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:20:01.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:01.093 | INFO | Uploaded 36GiB to GPU in 2.49824s (14.4102GiB/s) | 20260508 07:20:01.093 | INFO | Downloaded 34GiB to GPU in 2.43803s (13.9457GiB/s) | 20260508 07:20:06.232 | INFO | Analyzing chunk frames = [6263808,6264320) | 20260508 07:20:06.232 | INFO | MemInfo: { | 20260508 07:20:06.232 | INFO | "gpu" : " 17 GB", | 20260508 07:20:06.232 | INFO | "huge" : "", | 20260508 07:20:06.232 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:06.232 | INFO | "malloc" : " 14 GB", | 20260508 07:20:06.232 | INFO | "pinned" : " 29 GB" | 20260508 07:20:06.232 | INFO | } | 20260508 07:20:06.232 | INFO | Perf report for the last 5.13966 seconds: | 20260508 07:20:06.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:06.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.232 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:20:06.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.232 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:20:06.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:06.233 | INFO | Uploaded 36GiB to GPU in 2.56175s (14.0529GiB/s) | 20260508 07:20:06.233 | INFO | Downloaded 34GiB to GPU in 2.42283s (14.0332GiB/s) | 20260508 07:20:11.380 | INFO | Analyzing chunk frames = [6264320,6264832) | 20260508 07:20:11.380 | INFO | MemInfo: { | 20260508 07:20:11.380 | INFO | "gpu" : " 17 GB", | 20260508 07:20:11.380 | INFO | "huge" : "", | 20260508 07:20:11.380 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:11.380 | INFO | "malloc" : " 14 GB", | 20260508 07:20:11.380 | INFO | "pinned" : " 29 GB" | 20260508 07:20:11.380 | INFO | } | 20260508 07:20:11.381 | INFO | Perf report for the last 5.14841 seconds: | 20260508 07:20:11.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:11.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | ANALYSIS : 66.6%, 2.9, 103.2ms, 0 | 20260508 07:20:11.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:20:11.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:11.381 | INFO | Uploaded 36GiB to GPU in 2.56334s (14.0442GiB/s) | 20260508 07:20:11.381 | INFO | Downloaded 34GiB to GPU in 2.4081s (14.119GiB/s) | 20260508 07:20:16.518 | INFO | Analyzing chunk frames = [6264832,6265344) | 20260508 07:20:16.518 | INFO | MemInfo: { | 20260508 07:20:16.518 | INFO | "gpu" : " 17 GB", | 20260508 07:20:16.518 | INFO | "huge" : "", | 20260508 07:20:16.518 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:16.518 | INFO | "malloc" : " 14 GB", | 20260508 07:20:16.519 | INFO | "pinned" : " 29 GB" | 20260508 07:20:16.519 | INFO | } | 20260508 07:20:16.519 | INFO | Perf report for the last 5.13794 seconds: | 20260508 07:20:16.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:16.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 07:20:16.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:20:16.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:16.519 | INFO | Uploaded 36GiB to GPU in 2.26568s (15.8893GiB/s) | 20260508 07:20:16.519 | INFO | Downloaded 34GiB to GPU in 2.48151s (13.7013GiB/s) | 20260508 07:20:21.654 | INFO | Analyzing chunk frames = [6265344,6265856) | 20260508 07:20:21.654 | INFO | MemInfo: { | 20260508 07:20:21.654 | INFO | "gpu" : " 17 GB", | 20260508 07:20:21.654 | INFO | "huge" : "", | 20260508 07:20:21.654 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:21.654 | INFO | "malloc" : " 14 GB", | 20260508 07:20:21.654 | INFO | "pinned" : " 29 GB" | 20260508 07:20:21.654 | INFO | } | 20260508 07:20:21.654 | INFO | Perf report for the last 5.13534 seconds: | 20260508 07:20:21.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:21.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:20:21.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:20:21.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:21.654 | INFO | Uploaded 36GiB to GPU in 2.36765s (15.2049GiB/s) | 20260508 07:20:21.654 | INFO | Downloaded 34GiB to GPU in 2.43476s (13.9644GiB/s) | 20260508 07:20:26.792 | INFO | Analyzing chunk frames = [6265856,6266368) | 20260508 07:20:26.792 | INFO | MemInfo: { | 20260508 07:20:26.792 | INFO | "gpu" : " 17 GB", | 20260508 07:20:26.792 | INFO | "huge" : "", | 20260508 07:20:26.792 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:26.793 | INFO | "malloc" : " 14 GB", | 20260508 07:20:26.793 | INFO | "pinned" : " 29 GB" | 20260508 07:20:26.793 | INFO | } | 20260508 07:20:26.793 | INFO | Perf report for the last 5.13865 seconds: | 20260508 07:20:26.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:26.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 07:20:26.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:20:26.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:26.793 | INFO | Uploaded 36GiB to GPU in 2.453s (14.6759GiB/s) | 20260508 07:20:26.793 | INFO | Downloaded 34GiB to GPU in 2.42653s (14.0118GiB/s) | 20260508 07:20:31.932 | INFO | Analyzing chunk frames = [6266368,6266880) | 20260508 07:20:31.932 | INFO | MemInfo: { | 20260508 07:20:31.932 | INFO | "gpu" : " 17 GB", | 20260508 07:20:31.932 | INFO | "huge" : "", | 20260508 07:20:31.932 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:31.932 | INFO | "malloc" : " 14 GB", | 20260508 07:20:31.932 | INFO | "pinned" : " 29 GB" | 20260508 07:20:31.932 | INFO | } | 20260508 07:20:31.932 | INFO | Perf report for the last 5.13939 seconds: | 20260508 07:20:31.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:31.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 07:20:31.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 07:20:31.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:31.932 | INFO | Uploaded 36GiB to GPU in 2.46536s (14.6023GiB/s) | 20260508 07:20:31.933 | INFO | Downloaded 34GiB to GPU in 2.46242s (13.8076GiB/s) | 20260508 07:20:31.933 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:20:31.933 | INFO | Total mem usage: 16.7 GiB | 20260508 07:20:31.933 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:20:31.933 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:20:31.933 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:20:31.933 | INFO | High Water Breakdown: | 20260508 07:20:31.933 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:20:31.933 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:20:31.933 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:20:31.933 | INFO | ------------------------------------------ | 20260508 07:20:31.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:20:31.933 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:20:31.933 | INFO | Total mem usage: 26.5 GiB | 20260508 07:20:31.933 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:20:31.933 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:20:31.933 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:20:31.933 | INFO | High Water Breakdown: | 20260508 07:20:31.933 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:20:31.933 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:20:31.933 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:20:31.933 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:20:31.933 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:20:31.933 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:20:31.933 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:20:31.933 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:20:31.933 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:20:31.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:20:31.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:20:31.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:20:31.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:20:31.933 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:20:31.933 | INFO | ------------------------------------------ | 20260508 07:20:31.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:20:31.933 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:20:31.933 | INFO | Total mem usage: 27.8 GiB | 20260508 07:20:31.933 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:20:31.933 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:20:31.933 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:20:31.933 | INFO | High Water Breakdown: | 20260508 07:20:31.933 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:20:31.933 | INFO | ------------------------------------------ | 20260508 07:20:31.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:20:31.933 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:20:31.933 | INFO | Total mem usage: 15.7 GiB | 20260508 07:20:31.933 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:20:31.933 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:20:31.933 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:20:31.933 | INFO | High Water Breakdown: | 20260508 07:20:31.933 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:20:31.933 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:20:31.933 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:20:31.933 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:20:31.933 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:20:31.933 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:20:31.933 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:20:31.933 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:20:31.933 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:20:31.933 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:20:31.933 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:20:31.933 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:20:31.933 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:20:31.933 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:20:31.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:20:31.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:20:31.934 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:20:31.934 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:20:31.934 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:20:31.934 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:20:31.934 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:20:31.934 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:20:31.934 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:20:31.934 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:20:31.934 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:20:31.934 | INFO | ------------------------------------------ | 20260508 07:20:31.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:20:31.934 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:20:31.934 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:20:31.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:20:31.934 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 6.61e-01 | 20260508 07:20:31.934 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.07e-01, 1.15e-02 / 5.71e-03 | 20260508 07:20:31.934 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 5.92e-01 / 2.28e-01 | 20260508 07:20:31.934 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.15e-01 | 20260508 07:20:31.934 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.52e+01, 3.81e+00 / 8.85e-01 | 20260508 07:20:31.934 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.83e+00 / 1.30e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.08e-01 / 2.19e-01 | 20260508 07:20:31.934 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.18e+00, 2.49e-01 / 1.10e-01 | 20260508 07:20:31.934 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.48e-01, 4.92e-02 / 2.74e-02 | 20260508 07:20:31.934 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:20:31.934 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:20:31.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:20:31.934 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 24.42%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 24.33%, Download -- 2.57e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 30.90%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 2.26%, LockUpload -- 2.39e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 2.56%, LockDownload -- 2.71e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 15.51%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:20:31.934 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260508 07:20:32.055 | INFO | Starting a round of Baz IO | 20260508 07:20:35.457 | INFO | Populating RTMetrics report | 20260508 07:20:35.457 | INFO | RTMetrics report populated | 20260508 07:20:35.457 | INFO | Opening tmp RTMetrics json file | 20260508 07:20:35.472 | INFO | Serializing RTMetrics to json | 20260508 07:20:35.475 | INFO | Serialization done, swapping tmp file now... | 20260508 07:20:35.475 | INFO | RTMetrics done | 20260508 07:20:35.524 | INFO | Finished a round of Baz IO: Wrote 1.526 GiB in 3.469 seconds | 20260508 07:20:35.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.00% / 59.43% | 20260508 07:20:35.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.347253% / 3.97364e-06% / 4.37101e-05% | 20260508 07:20:35.524 | INFO | Baz IO overall: | 20260508 07:20:35.524 | INFO | BytesWritten : 2766417411443 | 20260508 07:20:35.524 | INFO | NumEvents : 1332948812574 | 20260508 07:20:37.073 | INFO | Analyzing chunk frames = [6266880,6267392) | 20260508 07:20:37.073 | INFO | MemInfo: { | 20260508 07:20:37.073 | INFO | "gpu" : " 17 GB", | 20260508 07:20:37.073 | INFO | "huge" : "", | 20260508 07:20:37.073 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:37.073 | INFO | "malloc" : " 14 GB", | 20260508 07:20:37.073 | INFO | "pinned" : " 29 GB" | 20260508 07:20:37.073 | INFO | } | 20260508 07:20:37.073 | INFO | Perf report for the last 5.14067 seconds: | 20260508 07:20:37.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:37.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:37.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:37.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:37.073 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 07:20:37.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:37.073 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 07:20:37.073 | INFO | PRE_HQ : 66.5%, 1.8, 62.0ms, 0 | 20260508 07:20:37.073 | INFO | BAZWRITER : 67.2%, 2.8, 96.4ms, 0 | 20260508 07:20:37.073 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:20:37.073 | INFO | Uploaded 36GiB to GPU in 2.5661s (14.0291GiB/s) | 20260508 07:20:37.073 | INFO | Downloaded 38GiB to GPU in 2.70113s (14.0682GiB/s) | 20260508 07:20:42.212 | INFO | Analyzing chunk frames = [6267392,6267904) | 20260508 07:20:42.212 | INFO | MemInfo: { | 20260508 07:20:42.212 | INFO | "gpu" : " 17 GB", | 20260508 07:20:42.212 | INFO | "huge" : "", | 20260508 07:20:42.212 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:42.212 | INFO | "malloc" : " 14 GB", | 20260508 07:20:42.212 | INFO | "pinned" : " 29 GB" | 20260508 07:20:42.212 | INFO | } | 20260508 07:20:42.212 | INFO | Perf report for the last 5.13898 seconds: | 20260508 07:20:42.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:42.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:20:42.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:20:42.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:42.212 | INFO | Uploaded 36GiB to GPU in 2.48898s (14.4637GiB/s) | 20260508 07:20:42.212 | INFO | Downloaded 34GiB to GPU in 2.44677s (13.8959GiB/s) | 20260508 07:20:47.347 | INFO | Analyzing chunk frames = [6267904,6268416) | 20260508 07:20:47.347 | INFO | MemInfo: { | 20260508 07:20:47.347 | INFO | "gpu" : " 17 GB", | 20260508 07:20:47.347 | INFO | "huge" : "", | 20260508 07:20:47.347 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:47.347 | INFO | "malloc" : " 14 GB", | 20260508 07:20:47.347 | INFO | "pinned" : " 29 GB" | 20260508 07:20:47.347 | INFO | } | 20260508 07:20:47.347 | INFO | Perf report for the last 5.13504 seconds: | 20260508 07:20:47.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:47.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 07:20:47.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:20:47.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:47.347 | INFO | Uploaded 36GiB to GPU in 2.55751s (14.0762GiB/s) | 20260508 07:20:47.347 | INFO | Downloaded 34GiB to GPU in 2.42419s (14.0253GiB/s) | 20260508 07:20:52.491 | INFO | Analyzing chunk frames = [6268416,6268928) | 20260508 07:20:52.491 | INFO | MemInfo: { | 20260508 07:20:52.491 | INFO | "gpu" : " 17 GB", | 20260508 07:20:52.491 | INFO | "huge" : "", | 20260508 07:20:52.491 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:52.491 | INFO | "malloc" : " 14 GB", | 20260508 07:20:52.491 | INFO | "pinned" : " 29 GB" | 20260508 07:20:52.491 | INFO | } | 20260508 07:20:52.491 | INFO | Perf report for the last 5.14421 seconds: | 20260508 07:20:52.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:52.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 07:20:52.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:20:52.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:52.491 | INFO | Uploaded 36GiB to GPU in 2.55739s (14.0769GiB/s) | 20260508 07:20:52.491 | INFO | Downloaded 34GiB to GPU in 2.41266s (14.0923GiB/s) | 20260508 07:20:57.630 | INFO | Analyzing chunk frames = [6268928,6269440) | 20260508 07:20:57.630 | INFO | MemInfo: { | 20260508 07:20:57.630 | INFO | "gpu" : " 17 GB", | 20260508 07:20:57.630 | INFO | "huge" : "", | 20260508 07:20:57.630 | INFO | "huge_shared" : " 29 GB", | 20260508 07:20:57.630 | INFO | "malloc" : " 14 GB", | 20260508 07:20:57.630 | INFO | "pinned" : " 29 GB" | 20260508 07:20:57.630 | INFO | } | 20260508 07:20:57.631 | INFO | Perf report for the last 5.13939 seconds: | 20260508 07:20:57.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:20:57.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:20:57.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:20:57.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:20:57.631 | INFO | Uploaded 36GiB to GPU in 2.27279s (15.8396GiB/s) | 20260508 07:20:57.631 | INFO | Downloaded 34GiB to GPU in 2.48472s (13.6836GiB/s) | 20260508 07:21:02.771 | INFO | Analyzing chunk frames = [6269440,6269952) | 20260508 07:21:02.771 | INFO | MemInfo: { | 20260508 07:21:02.771 | INFO | "gpu" : " 17 GB", | 20260508 07:21:02.771 | INFO | "huge" : "", | 20260508 07:21:02.771 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:02.771 | INFO | "malloc" : " 14 GB", | 20260508 07:21:02.771 | INFO | "pinned" : " 29 GB" | 20260508 07:21:02.771 | INFO | } | 20260508 07:21:02.771 | INFO | Perf report for the last 5.14092 seconds: | 20260508 07:21:02.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:02.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.772 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:21:02.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.772 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:21:02.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:02.772 | INFO | Uploaded 36GiB to GPU in 2.46931s (14.579GiB/s) | 20260508 07:21:02.772 | INFO | Downloaded 34GiB to GPU in 2.43465s (13.9651GiB/s) | 20260508 07:21:07.906 | INFO | Analyzing chunk frames = [6269952,6270464) | 20260508 07:21:07.906 | INFO | MemInfo: { | 20260508 07:21:07.906 | INFO | "gpu" : " 17 GB", | 20260508 07:21:07.906 | INFO | "huge" : "", | 20260508 07:21:07.906 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:07.906 | INFO | "malloc" : " 14 GB", | 20260508 07:21:07.906 | INFO | "pinned" : " 29 GB" | 20260508 07:21:07.906 | INFO | } | 20260508 07:21:07.906 | INFO | Perf report for the last 5.13487 seconds: | 20260508 07:21:07.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:07.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.906 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:21:07.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.907 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 07:21:07.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:07.907 | INFO | Uploaded 36GiB to GPU in 2.48051s (14.5131GiB/s) | 20260508 07:21:07.907 | INFO | Downloaded 34GiB to GPU in 2.42492s (14.0211GiB/s) | 20260508 07:21:13.045 | INFO | Analyzing chunk frames = [6270464,6270976) | 20260508 07:21:13.045 | INFO | MemInfo: { | 20260508 07:21:13.045 | INFO | "gpu" : " 17 GB", | 20260508 07:21:13.046 | INFO | "huge" : "", | 20260508 07:21:13.046 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:13.046 | INFO | "malloc" : " 14 GB", | 20260508 07:21:13.046 | INFO | "pinned" : " 29 GB" | 20260508 07:21:13.046 | INFO | } | 20260508 07:21:13.046 | INFO | Perf report for the last 5.13937 seconds: | 20260508 07:21:13.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:13.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 07:21:13.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260508 07:21:13.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:13.046 | INFO | Uploaded 36GiB to GPU in 2.4493s (14.6981GiB/s) | 20260508 07:21:13.046 | INFO | Downloaded 34GiB to GPU in 2.44182s (13.924GiB/s) | 20260508 07:21:13.156 | INFO | Starting a round of Baz IO | 20260508 07:21:16.569 | INFO | Populating RTMetrics report | 20260508 07:21:16.570 | INFO | RTMetrics report populated | 20260508 07:21:16.570 | INFO | Opening tmp RTMetrics json file | 20260508 07:21:16.580 | INFO | Serializing RTMetrics to json | 20260508 07:21:16.582 | INFO | Serialization done, swapping tmp file now... | 20260508 07:21:16.583 | INFO | RTMetrics done | 20260508 07:21:16.641 | INFO | Finished a round of Baz IO: Wrote 1.528 GiB in 3.485 seconds | 20260508 07:21:16.641 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.00% / 59.43% | 20260508 07:21:16.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.346613% / 3.97364e-06% / 7.15256e-05% | 20260508 07:21:16.641 | INFO | Baz IO overall: | 20260508 07:21:16.641 | INFO | BytesWritten : 2768058581386 | 20260508 07:21:16.641 | INFO | NumEvents : 1333728272586 | 20260508 07:21:18.187 | INFO | Analyzing chunk frames = [6270976,6271488) | 20260508 07:21:18.187 | INFO | MemInfo: { | 20260508 07:21:18.187 | INFO | "gpu" : " 17 GB", | 20260508 07:21:18.187 | INFO | "huge" : "", | 20260508 07:21:18.187 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:18.187 | INFO | "malloc" : " 14 GB", | 20260508 07:21:18.187 | INFO | "pinned" : " 29 GB" | 20260508 07:21:18.187 | INFO | } | 20260508 07:21:18.187 | INFO | Perf report for the last 5.14105 seconds: | 20260508 07:21:18.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:18.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:18.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:18.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:18.187 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 07:21:18.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:18.187 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260508 07:21:18.187 | INFO | PRE_HQ : 66.5%, 1.8, 62.5ms, 0 | 20260508 07:21:18.187 | INFO | BAZWRITER : 67.1%, 3.1, 107.3ms, 0 | 20260508 07:21:18.187 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 07:21:18.187 | INFO | Uploaded 36GiB to GPU in 2.60271s (13.8318GiB/s) | 20260508 07:21:18.187 | INFO | Downloaded 38GiB to GPU in 2.6932s (14.1096GiB/s) | 20260508 07:21:23.328 | INFO | Analyzing chunk frames = [6271488,6272000) | 20260508 07:21:23.328 | INFO | MemInfo: { | 20260508 07:21:23.328 | INFO | "gpu" : " 17 GB", | 20260508 07:21:23.328 | INFO | "huge" : "", | 20260508 07:21:23.328 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:23.328 | INFO | "malloc" : " 14 GB", | 20260508 07:21:23.328 | INFO | "pinned" : " 29 GB" | 20260508 07:21:23.328 | INFO | } | 20260508 07:21:23.328 | INFO | Perf report for the last 5.1412 seconds: | 20260508 07:21:23.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:23.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:21:23.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:21:23.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:23.328 | INFO | Uploaded 36GiB to GPU in 2.50453s (14.374GiB/s) | 20260508 07:21:23.328 | INFO | Downloaded 34GiB to GPU in 2.43591s (13.9578GiB/s) | 20260508 07:21:28.467 | INFO | Analyzing chunk frames = [6272000,6272512) | 20260508 07:21:28.467 | INFO | MemInfo: { | 20260508 07:21:28.467 | INFO | "gpu" : " 17 GB", | 20260508 07:21:28.467 | INFO | "huge" : "", | 20260508 07:21:28.467 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:28.467 | INFO | "malloc" : " 14 GB", | 20260508 07:21:28.467 | INFO | "pinned" : " 29 GB" | 20260508 07:21:28.467 | INFO | } | 20260508 07:21:28.467 | INFO | Perf report for the last 5.139 seconds: | 20260508 07:21:28.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:28.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:21:28.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:21:28.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:28.467 | INFO | Uploaded 36GiB to GPU in 2.56779s (14.0198GiB/s) | 20260508 07:21:28.467 | INFO | Downloaded 34GiB to GPU in 2.40972s (14.1095GiB/s) | 20260508 07:21:33.605 | INFO | Analyzing chunk frames = [6272512,6273024) | 20260508 07:21:33.605 | INFO | MemInfo: { | 20260508 07:21:33.605 | INFO | "gpu" : " 17 GB", | 20260508 07:21:33.605 | INFO | "huge" : "", | 20260508 07:21:33.606 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:33.606 | INFO | "malloc" : " 14 GB", | 20260508 07:21:33.606 | INFO | "pinned" : " 29 GB" | 20260508 07:21:33.606 | INFO | } | 20260508 07:21:33.606 | INFO | Perf report for the last 5.13868 seconds: | 20260508 07:21:33.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:33.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:21:33.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:21:33.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:33.606 | INFO | Uploaded 36GiB to GPU in 2.57799s (13.9644GiB/s) | 20260508 07:21:33.606 | INFO | Downloaded 34GiB to GPU in 2.39739s (14.1821GiB/s) | 20260508 07:21:38.744 | INFO | Analyzing chunk frames = [6273024,6273536) | 20260508 07:21:38.744 | INFO | MemInfo: { | 20260508 07:21:38.744 | INFO | "gpu" : " 17 GB", | 20260508 07:21:38.744 | INFO | "huge" : "", | 20260508 07:21:38.744 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:38.744 | INFO | "malloc" : " 14 GB", | 20260508 07:21:38.744 | INFO | "pinned" : " 29 GB" | 20260508 07:21:38.744 | INFO | } | 20260508 07:21:38.744 | INFO | Perf report for the last 5.13853 seconds: | 20260508 07:21:38.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:38.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.744 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:21:38.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.744 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:21:38.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:38.745 | INFO | Uploaded 36GiB to GPU in 2.31386s (15.5584GiB/s) | 20260508 07:21:38.745 | INFO | Downloaded 34GiB to GPU in 2.50073s (13.596GiB/s) | 20260508 07:21:43.887 | INFO | Analyzing chunk frames = [6273536,6274048) | 20260508 07:21:43.887 | INFO | MemInfo: { | 20260508 07:21:43.887 | INFO | "gpu" : " 17 GB", | 20260508 07:21:43.888 | INFO | "huge" : "", | 20260508 07:21:43.888 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:43.888 | INFO | "malloc" : " 14 GB", | 20260508 07:21:43.888 | INFO | "pinned" : " 29 GB" | 20260508 07:21:43.888 | INFO | } | 20260508 07:21:43.888 | INFO | Perf report for the last 5.14336 seconds: | 20260508 07:21:43.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:43.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 07:21:43.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:21:43.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:43.888 | INFO | Uploaded 36GiB to GPU in 2.46452s (14.6073GiB/s) | 20260508 07:21:43.888 | INFO | Downloaded 34GiB to GPU in 2.43445s (13.9662GiB/s) | 20260508 07:21:49.021 | INFO | Analyzing chunk frames = [6274048,6274560) | 20260508 07:21:49.021 | INFO | MemInfo: { | 20260508 07:21:49.021 | INFO | "gpu" : " 17 GB", | 20260508 07:21:49.021 | INFO | "huge" : "", | 20260508 07:21:49.021 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:49.022 | INFO | "malloc" : " 14 GB", | 20260508 07:21:49.022 | INFO | "pinned" : " 29 GB" | 20260508 07:21:49.022 | INFO | } | 20260508 07:21:49.022 | INFO | Perf report for the last 5.1341 seconds: | 20260508 07:21:49.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:49.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:21:49.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 07:21:49.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:49.022 | INFO | Uploaded 36GiB to GPU in 2.50483s (14.3722GiB/s) | 20260508 07:21:49.022 | INFO | Downloaded 34GiB to GPU in 2.43643s (13.9548GiB/s) | 20260508 07:21:54.158 | INFO | Analyzing chunk frames = [6274560,6275072) | 20260508 07:21:54.158 | INFO | MemInfo: { | 20260508 07:21:54.158 | INFO | "gpu" : " 17 GB", | 20260508 07:21:54.158 | INFO | "huge" : "", | 20260508 07:21:54.158 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:54.158 | INFO | "malloc" : " 14 GB", | 20260508 07:21:54.158 | INFO | "pinned" : " 29 GB" | 20260508 07:21:54.158 | INFO | } | 20260508 07:21:54.158 | INFO | Perf report for the last 5.13618 seconds: | 20260508 07:21:54.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:54.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.158 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 07:21:54.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.158 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 07:21:54.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:54.159 | INFO | Uploaded 36GiB to GPU in 2.50125s (14.3928GiB/s) | 20260508 07:21:54.159 | INFO | Downloaded 34GiB to GPU in 2.44371s (13.9133GiB/s) | 20260508 07:21:54.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:21:54.159 | INFO | Total mem usage: 16.7 GiB | 20260508 07:21:54.159 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:21:54.159 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14%) | 20260508 07:21:54.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:21:54.159 | INFO | High Water Breakdown: | 20260508 07:21:54.159 | INFO | 59.9% : 8.76 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:21:54.159 | INFO | 40% : 5.86 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:21:54.159 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:21:54.159 | INFO | ------------------------------------------ | 20260508 07:21:54.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:21:54.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:21:54.159 | INFO | Total mem usage: 26.5 GiB | 20260508 07:21:54.159 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:21:54.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:21:54.159 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:21:54.159 | INFO | High Water Breakdown: | 20260508 07:21:54.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:21:54.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:21:54.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:21:54.159 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:21:54.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:21:54.159 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:21:54.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:21:54.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:21:54.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:21:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:21:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:21:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:21:54.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:21:54.160 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:21:54.160 | INFO | ------------------------------------------ | 20260508 07:21:54.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:21:54.160 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:21:54.160 | INFO | Total mem usage: 27.8 GiB | 20260508 07:21:54.160 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:21:54.160 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:21:54.160 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:21:54.160 | INFO | High Water Breakdown: | 20260508 07:21:54.160 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:21:54.160 | INFO | ------------------------------------------ | 20260508 07:21:54.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:21:54.160 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:21:54.160 | INFO | Total mem usage: 15.7 GiB | 20260508 07:21:54.160 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:21:54.160 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:21:54.160 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:21:54.160 | INFO | High Water Breakdown: | 20260508 07:21:54.160 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:21:54.160 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:21:54.160 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:21:54.160 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:21:54.160 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:21:54.160 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:21:54.160 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:21:54.160 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:21:54.160 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:21:54.160 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:21:54.160 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:21:54.160 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:21:54.160 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:21:54.160 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:21:54.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:21:54.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:21:54.160 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:21:54.160 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:21:54.160 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:21:54.160 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:21:54.160 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:21:54.160 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:21:54.160 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:21:54.160 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:21:54.160 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:21:54.160 | INFO | ------------------------------------------ | 20260508 07:21:54.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:21:54.161 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:21:54.161 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:21:54.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:21:54.161 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 5.11e-01 | 20260508 07:21:54.161 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.06e-01, 1.14e-02 / 3.78e-03 | 20260508 07:21:54.161 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.40e+00, 5.92e-01 / 2.45e-01 | 20260508 07:21:54.161 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.07e-01 / 1.24e-01 | 20260508 07:21:54.161 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.54e+01, 3.67e+00 / 9.65e-01 | 20260508 07:21:54.161 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.24e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.56e+00, 4.12e-01 / 2.41e-01 | 20260508 07:21:54.161 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.19e+00, 2.52e-01 / 1.43e-01 | 20260508 07:21:54.161 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.51e-01, 5.01e-02 / 3.47e-02 | 20260508 07:21:54.161 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:21:54.161 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:21:54.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:21:54.161 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 24.48%, Upload -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 24.27%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 2.40%, LockUpload -- 2.55e+00 / 0.00e+00, 5.81e+00 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 2.48%, LockDownload -- 2.63e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:21:54.161 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260508 07:21:54.269 | INFO | Starting a round of Baz IO | 20260508 07:21:57.690 | INFO | Populating RTMetrics report | 20260508 07:21:57.690 | INFO | RTMetrics report populated | 20260508 07:21:57.690 | INFO | Opening tmp RTMetrics json file | 20260508 07:21:57.703 | INFO | Serializing RTMetrics to json | 20260508 07:21:57.704 | INFO | Serialization done, swapping tmp file now... | 20260508 07:21:57.704 | INFO | RTMetrics done | 20260508 07:21:57.757 | INFO | Finished a round of Baz IO: Wrote 1.529 GiB in 3.488 seconds | 20260508 07:21:57.757 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.43% | 20260508 07:21:57.757 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.347559% / 0% / 5.16574e-05% | 20260508 07:21:57.757 | INFO | Baz IO overall: | 20260508 07:21:57.757 | INFO | BytesWritten : 2769700519087 | 20260508 07:21:57.757 | INFO | NumEvents : 1334508492183 | 20260508 07:21:59.298 | INFO | Analyzing chunk frames = [6275072,6275584) | 20260508 07:21:59.298 | INFO | MemInfo: { | 20260508 07:21:59.298 | INFO | "gpu" : " 17 GB", | 20260508 07:21:59.298 | INFO | "huge" : "", | 20260508 07:21:59.298 | INFO | "huge_shared" : " 29 GB", | 20260508 07:21:59.298 | INFO | "malloc" : " 14 GB", | 20260508 07:21:59.298 | INFO | "pinned" : " 29 GB" | 20260508 07:21:59.298 | INFO | } | 20260508 07:21:59.298 | INFO | Perf report for the last 5.14054 seconds: | 20260508 07:21:59.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:21:59.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:59.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:59.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:59.299 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 07:21:59.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:21:59.299 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 07:21:59.299 | INFO | PRE_HQ : 66.6%, 1.8, 62.4ms, 0 | 20260508 07:21:59.299 | INFO | BAZWRITER : 67.4%, 2.8, 98.5ms, 0 | 20260508 07:21:59.299 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:21:59.299 | INFO | Uploaded 36GiB to GPU in 2.5641s (14.04GiB/s) | 20260508 07:21:59.299 | INFO | Downloaded 38GiB to GPU in 2.69291s (14.1111GiB/s) | 20260508 07:22:04.443 | INFO | Analyzing chunk frames = [6275584,6276096) | 20260508 07:22:04.443 | INFO | MemInfo: { | 20260508 07:22:04.443 | INFO | "gpu" : " 17 GB", | 20260508 07:22:04.443 | INFO | "huge" : "", | 20260508 07:22:04.443 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:04.443 | INFO | "malloc" : " 14 GB", | 20260508 07:22:04.443 | INFO | "pinned" : " 29 GB" | 20260508 07:22:04.443 | INFO | } | 20260508 07:22:04.443 | INFO | Perf report for the last 5.14443 seconds: | 20260508 07:22:04.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:04.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:22:04.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:22:04.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:04.443 | INFO | Uploaded 36GiB to GPU in 2.51281s (14.3266GiB/s) | 20260508 07:22:04.443 | INFO | Downloaded 34GiB to GPU in 2.431s (13.986GiB/s) | 20260508 07:22:09.580 | INFO | Analyzing chunk frames = [6276096,6276608) | 20260508 07:22:09.580 | INFO | MemInfo: { | 20260508 07:22:09.580 | INFO | "gpu" : " 17 GB", | 20260508 07:22:09.580 | INFO | "huge" : "", | 20260508 07:22:09.580 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:09.580 | INFO | "malloc" : " 14 GB", | 20260508 07:22:09.580 | INFO | "pinned" : " 29 GB" | 20260508 07:22:09.580 | INFO | } | 20260508 07:22:09.580 | INFO | Perf report for the last 5.13697 seconds: | 20260508 07:22:09.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:09.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:22:09.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:22:09.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:09.580 | INFO | Uploaded 36GiB to GPU in 2.57367s (13.9878GiB/s) | 20260508 07:22:09.580 | INFO | Downloaded 34GiB to GPU in 2.42768s (14.0051GiB/s) | 20260508 07:22:14.722 | INFO | Analyzing chunk frames = [6276608,6277120) | 20260508 07:22:14.722 | INFO | MemInfo: { | 20260508 07:22:14.722 | INFO | "gpu" : " 17 GB", | 20260508 07:22:14.722 | INFO | "huge" : "", | 20260508 07:22:14.722 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:14.722 | INFO | "malloc" : " 14 GB", | 20260508 07:22:14.722 | INFO | "pinned" : " 29 GB" | 20260508 07:22:14.722 | INFO | } | 20260508 07:22:14.722 | INFO | Perf report for the last 5.14256 seconds: | 20260508 07:22:14.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:14.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.723 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 07:22:14.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.723 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:22:14.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:14.723 | INFO | Uploaded 36GiB to GPU in 2.58146s (13.9456GiB/s) | 20260508 07:22:14.723 | INFO | Downloaded 34GiB to GPU in 2.40187s (14.1556GiB/s) | 20260508 07:22:19.856 | INFO | Analyzing chunk frames = [6277120,6277632) | 20260508 07:22:19.856 | INFO | MemInfo: { | 20260508 07:22:19.856 | INFO | "gpu" : " 17 GB", | 20260508 07:22:19.856 | INFO | "huge" : "", | 20260508 07:22:19.856 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:19.856 | INFO | "malloc" : " 14 GB", | 20260508 07:22:19.856 | INFO | "pinned" : " 29 GB" | 20260508 07:22:19.856 | INFO | } | 20260508 07:22:19.856 | INFO | Perf report for the last 5.13351 seconds: | 20260508 07:22:19.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:19.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 07:22:19.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 07:22:19.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:19.856 | INFO | Uploaded 36GiB to GPU in 2.27605s (15.8168GiB/s) | 20260508 07:22:19.856 | INFO | Downloaded 34GiB to GPU in 2.48656s (13.6735GiB/s) | 20260508 07:22:24.999 | INFO | Analyzing chunk frames = [6277632,6278144) | 20260508 07:22:24.999 | INFO | MemInfo: { | 20260508 07:22:24.999 | INFO | "gpu" : " 17 GB", | 20260508 07:22:24.999 | INFO | "huge" : "", | 20260508 07:22:24.999 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:24.999 | INFO | "malloc" : " 14 GB", | 20260508 07:22:24.999 | INFO | "pinned" : " 29 GB" | 20260508 07:22:24.999 | INFO | } | 20260508 07:22:24.999 | INFO | Perf report for the last 5.14292 seconds: | 20260508 07:22:24.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:24.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 07:22:24.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:22:24.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:24.999 | INFO | Uploaded 36GiB to GPU in 2.39642s (15.0224GiB/s) | 20260508 07:22:24.999 | INFO | Downloaded 34GiB to GPU in 2.4222s (14.0368GiB/s) | 20260508 07:22:30.141 | INFO | Analyzing chunk frames = [6278144,6278656) | 20260508 07:22:30.141 | INFO | MemInfo: { | 20260508 07:22:30.141 | INFO | "gpu" : " 17 GB", | 20260508 07:22:30.141 | INFO | "huge" : "", | 20260508 07:22:30.141 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:30.141 | INFO | "malloc" : " 14 GB", | 20260508 07:22:30.141 | INFO | "pinned" : " 29 GB" | 20260508 07:22:30.141 | INFO | } | 20260508 07:22:30.141 | INFO | Perf report for the last 5.14207 seconds: | 20260508 07:22:30.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:30.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:22:30.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:22:30.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:30.141 | INFO | Uploaded 36GiB to GPU in 2.49025s (14.4564GiB/s) | 20260508 07:22:30.141 | INFO | Downloaded 34GiB to GPU in 2.42585s (14.0157GiB/s) | 20260508 07:22:35.278 | INFO | Analyzing chunk frames = [6278656,6279168) | 20260508 07:22:35.278 | INFO | MemInfo: { | 20260508 07:22:35.278 | INFO | "gpu" : " 17 GB", | 20260508 07:22:35.279 | INFO | "huge" : "", | 20260508 07:22:35.279 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:35.279 | INFO | "malloc" : " 14 GB", | 20260508 07:22:35.279 | INFO | "pinned" : " 29 GB" | 20260508 07:22:35.279 | INFO | } | 20260508 07:22:35.279 | INFO | Perf report for the last 5.13765 seconds: | 20260508 07:22:35.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:35.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 07:22:35.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 07:22:35.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:35.279 | INFO | Uploaded 36GiB to GPU in 2.45246s (14.6792GiB/s) | 20260508 07:22:35.279 | INFO | Downloaded 34GiB to GPU in 2.42598s (14.0149GiB/s) | 20260508 07:22:35.391 | INFO | Starting a round of Baz IO | 20260508 07:22:38.797 | INFO | Populating RTMetrics report | 20260508 07:22:38.797 | INFO | RTMetrics report populated | 20260508 07:22:38.797 | INFO | Opening tmp RTMetrics json file | 20260508 07:22:38.809 | INFO | Serializing RTMetrics to json | 20260508 07:22:38.810 | INFO | Serialization done, swapping tmp file now... | 20260508 07:22:38.810 | INFO | RTMetrics done | 20260508 07:22:38.869 | INFO | Finished a round of Baz IO: Wrote 1.527 GiB in 3.477 seconds | 20260508 07:22:38.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.44% | 20260508 07:22:38.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345091% / 0% / 3.57628e-05% | 20260508 07:22:38.869 | INFO | Baz IO overall: | 20260508 07:22:38.869 | INFO | BytesWritten : 2771340396213 | 20260508 07:22:38.869 | INFO | NumEvents : 1335286945499 | 20260508 07:22:40.414 | INFO | Analyzing chunk frames = [6279168,6279680) | 20260508 07:22:40.414 | INFO | MemInfo: { | 20260508 07:22:40.414 | INFO | "gpu" : " 17 GB", | 20260508 07:22:40.415 | INFO | "huge" : "", | 20260508 07:22:40.415 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:40.415 | INFO | "malloc" : " 14 GB", | 20260508 07:22:40.415 | INFO | "pinned" : " 29 GB" | 20260508 07:22:40.415 | INFO | } | 20260508 07:22:40.415 | INFO | Perf report for the last 5.13598 seconds: | 20260508 07:22:40.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:40.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:40.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:40.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:40.415 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 07:22:40.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:40.415 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 07:22:40.415 | INFO | PRE_HQ : 66.5%, 1.8, 62.4ms, 0 | 20260508 07:22:40.415 | INFO | BAZWRITER : 67.1%, 2.8, 97.7ms, 0 | 20260508 07:22:40.415 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:22:40.415 | INFO | Uploaded 36GiB to GPU in 2.62996s (13.6884GiB/s) | 20260508 07:22:40.415 | INFO | Downloaded 38GiB to GPU in 2.71202s (14.0117GiB/s) | 20260508 07:22:45.555 | INFO | Analyzing chunk frames = [6279680,6280192) | 20260508 07:22:45.555 | INFO | MemInfo: { | 20260508 07:22:45.555 | INFO | "gpu" : " 17 GB", | 20260508 07:22:45.555 | INFO | "huge" : "", | 20260508 07:22:45.555 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:45.555 | INFO | "malloc" : " 14 GB", | 20260508 07:22:45.555 | INFO | "pinned" : " 29 GB" | 20260508 07:22:45.555 | INFO | } | 20260508 07:22:45.555 | INFO | Perf report for the last 5.14057 seconds: | 20260508 07:22:45.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:45.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.555 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:22:45.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.555 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:22:45.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:45.556 | INFO | Uploaded 36GiB to GPU in 2.5027s (14.3845GiB/s) | 20260508 07:22:45.556 | INFO | Downloaded 34GiB to GPU in 2.43601s (13.9572GiB/s) | 20260508 07:22:50.696 | INFO | Analyzing chunk frames = [6280192,6280704) | 20260508 07:22:50.697 | INFO | MemInfo: { | 20260508 07:22:50.697 | INFO | "gpu" : " 17 GB", | 20260508 07:22:50.697 | INFO | "huge" : "", | 20260508 07:22:50.697 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:50.697 | INFO | "malloc" : " 14 GB", | 20260508 07:22:50.697 | INFO | "pinned" : " 29 GB" | 20260508 07:22:50.697 | INFO | } | 20260508 07:22:50.697 | INFO | Perf report for the last 5.14141 seconds: | 20260508 07:22:50.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:50.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:22:50.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:22:50.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:50.697 | INFO | Uploaded 36GiB to GPU in 2.57513s (13.9799GiB/s) | 20260508 07:22:50.697 | INFO | Downloaded 34GiB to GPU in 2.42831s (14.0015GiB/s) | 20260508 07:22:55.840 | INFO | Analyzing chunk frames = [6280704,6281216) | 20260508 07:22:55.840 | INFO | MemInfo: { | 20260508 07:22:55.840 | INFO | "gpu" : " 17 GB", | 20260508 07:22:55.840 | INFO | "huge" : "", | 20260508 07:22:55.840 | INFO | "huge_shared" : " 29 GB", | 20260508 07:22:55.840 | INFO | "malloc" : " 14 GB", | 20260508 07:22:55.840 | INFO | "pinned" : " 29 GB" | 20260508 07:22:55.840 | INFO | } | 20260508 07:22:55.840 | INFO | Perf report for the last 5.1431 seconds: | 20260508 07:22:55.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:22:55.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:22:55.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:22:55.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:22:55.840 | INFO | Uploaded 36GiB to GPU in 2.58996s (13.8998GiB/s) | 20260508 07:22:55.840 | INFO | Downloaded 34GiB to GPU in 2.42191s (14.0385GiB/s) | 20260508 07:23:00.971 | INFO | Analyzing chunk frames = [6281216,6281728) | 20260508 07:23:00.971 | INFO | MemInfo: { | 20260508 07:23:00.971 | INFO | "gpu" : " 17 GB", | 20260508 07:23:00.971 | INFO | "huge" : "", | 20260508 07:23:00.971 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:00.971 | INFO | "malloc" : " 14 GB", | 20260508 07:23:00.971 | INFO | "pinned" : " 29 GB" | 20260508 07:23:00.971 | INFO | } | 20260508 07:23:00.971 | INFO | Perf report for the last 5.13156 seconds: | 20260508 07:23:00.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:00.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.971 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:23:00.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.971 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:23:00.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:00.972 | INFO | Uploaded 36GiB to GPU in 2.43328s (14.7949GiB/s) | 20260508 07:23:00.972 | INFO | Downloaded 34GiB to GPU in 2.52328s (13.4745GiB/s) | 20260508 07:23:06.108 | INFO | Analyzing chunk frames = [6281728,6282240) | 20260508 07:23:06.108 | INFO | MemInfo: { | 20260508 07:23:06.108 | INFO | "gpu" : " 17 GB", | 20260508 07:23:06.108 | INFO | "huge" : "", | 20260508 07:23:06.108 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:06.108 | INFO | "malloc" : " 14 GB", | 20260508 07:23:06.108 | INFO | "pinned" : " 29 GB" | 20260508 07:23:06.108 | INFO | } | 20260508 07:23:06.108 | INFO | Perf report for the last 5.13698 seconds: | 20260508 07:23:06.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:06.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.108 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:23:06.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.108 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:23:06.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:06.109 | INFO | Uploaded 36GiB to GPU in 2.34737s (15.3363GiB/s) | 20260508 07:23:06.109 | INFO | Downloaded 34GiB to GPU in 2.42638s (14.0126GiB/s) | 20260508 07:23:11.251 | INFO | Analyzing chunk frames = [6282240,6282752) | 20260508 07:23:11.251 | INFO | MemInfo: { | 20260508 07:23:11.251 | INFO | "gpu" : " 17 GB", | 20260508 07:23:11.251 | INFO | "huge" : "", | 20260508 07:23:11.251 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:11.251 | INFO | "malloc" : " 14 GB", | 20260508 07:23:11.251 | INFO | "pinned" : " 29 GB" | 20260508 07:23:11.251 | INFO | } | 20260508 07:23:11.252 | INFO | Perf report for the last 5.14322 seconds: | 20260508 07:23:11.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:11.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.252 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 07:23:11.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.252 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:23:11.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:11.253 | INFO | Uploaded 36GiB to GPU in 2.45672s (14.6537GiB/s) | 20260508 07:23:11.253 | INFO | Downloaded 34GiB to GPU in 2.43213s (13.9795GiB/s) | 20260508 07:23:16.395 | INFO | Analyzing chunk frames = [6282752,6283264) | 20260508 07:23:16.395 | INFO | MemInfo: { | 20260508 07:23:16.395 | INFO | "gpu" : " 17 GB", | 20260508 07:23:16.395 | INFO | "huge" : "", | 20260508 07:23:16.395 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:16.395 | INFO | "malloc" : " 14 GB", | 20260508 07:23:16.395 | INFO | "pinned" : " 29 GB" | 20260508 07:23:16.395 | INFO | } | 20260508 07:23:16.395 | INFO | Perf report for the last 5.14338 seconds: | 20260508 07:23:16.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:16.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.395 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 07:23:16.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.395 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 07:23:16.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:16.396 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:23:16.396 | INFO | Total mem usage: 16.7 GiB | 20260508 07:23:16.396 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:23:16.396 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:23:16.396 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:23:16.396 | INFO | High Water Breakdown: | 20260508 07:23:16.396 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:23:16.396 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:23:16.396 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:23:16.396 | INFO | ------------------------------------------ | 20260508 07:23:16.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:23:16.396 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:23:16.396 | INFO | Total mem usage: 26.5 GiB | 20260508 07:23:16.396 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:23:16.396 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:23:16.396 | INFO | Uploaded 36GiB to GPU in 2.51968s (14.2875GiB/s) | 20260508 07:23:16.396 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:23:16.396 | INFO | Downloaded 34GiB to GPU in 2.46248s (13.8072GiB/s) | 20260508 07:23:16.396 | INFO | High Water Breakdown: | 20260508 07:23:16.396 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:23:16.396 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:23:16.396 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:23:16.396 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:23:16.396 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:23:16.396 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:23:16.396 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:23:16.396 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:23:16.396 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:23:16.396 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:23:16.396 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:23:16.396 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:23:16.396 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:23:16.396 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:23:16.396 | INFO | ------------------------------------------ | 20260508 07:23:16.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:23:16.396 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:23:16.396 | INFO | Total mem usage: 27.8 GiB | 20260508 07:23:16.396 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:23:16.396 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:23:16.396 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:23:16.396 | INFO | High Water Breakdown: | 20260508 07:23:16.396 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:23:16.396 | INFO | ------------------------------------------ | 20260508 07:23:16.396 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:23:16.396 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:23:16.396 | INFO | Total mem usage: 15.7 GiB | 20260508 07:23:16.396 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:23:16.396 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:23:16.396 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:23:16.396 | INFO | High Water Breakdown: | 20260508 07:23:16.396 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:23:16.396 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:23:16.396 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:23:16.397 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:23:16.397 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:23:16.397 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:23:16.397 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:23:16.397 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:23:16.397 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:23:16.397 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:23:16.397 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:23:16.397 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:23:16.397 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:23:16.397 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:23:16.397 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:23:16.397 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:23:16.397 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:23:16.397 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:23:16.397 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:23:16.397 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:23:16.397 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:23:16.397 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:23:16.397 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:23:16.397 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:23:16.397 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:23:16.397 | INFO | ------------------------------------------ | 20260508 07:23:16.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:23:16.397 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:23:16.397 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:23:16.397 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:23:16.397 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.38e+01 / 6.58e-01 | 20260508 07:23:16.397 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.08e-02 / 5.20e-03 | 20260508 07:23:16.397 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.45e+00, 5.93e-01 / 2.34e-01 | 20260508 07:23:16.397 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.08e-01 | 20260508 07:23:16.397 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.51e+01, 3.73e+00 / 1.07e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.88e+00 / 1.38e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.36e+00 / 2.57e+00, 4.12e-01 / 2.16e-01 | 20260508 07:23:16.397 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.21e+00, 2.52e-01 / 1.38e-01 | 20260508 07:23:16.397 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.46e-01, 4.96e-02 / 2.41e-02 | 20260508 07:23:16.397 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:23:16.397 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:23:16.397 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:23:16.397 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 24.50%, Upload -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 24.29%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 30.89%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 2.41%, LockUpload -- 2.56e+00 / 0.00e+00, 5.81e+00 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 2.37%, LockDownload -- 2.52e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 07:23:16.397 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.42e-02 / 0.00e+00 | 20260508 07:23:16.522 | INFO | Starting a round of Baz IO | 20260508 07:23:19.918 | INFO | Populating RTMetrics report | 20260508 07:23:19.918 | INFO | RTMetrics report populated | 20260508 07:23:19.918 | INFO | Opening tmp RTMetrics json file | 20260508 07:23:19.928 | INFO | Serializing RTMetrics to json | 20260508 07:23:19.929 | INFO | Serialization done, swapping tmp file now... | 20260508 07:23:19.930 | INFO | RTMetrics done | 20260508 07:23:19.987 | INFO | Finished a round of Baz IO: Wrote 1.532 GiB in 3.465 seconds | 20260508 07:23:19.987 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.44% | 20260508 07:23:19.987 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345699% / 3.97364e-06% / 3.97364e-05% | 20260508 07:23:19.987 | INFO | Baz IO overall: | 20260508 07:23:19.987 | INFO | BytesWritten : 2772985464794 | 20260508 07:23:19.987 | INFO | NumEvents : 1336067363253 | 20260508 07:23:21.537 | INFO | Analyzing chunk frames = [6283264,6283776) | 20260508 07:23:21.537 | INFO | MemInfo: { | 20260508 07:23:21.537 | INFO | "gpu" : " 17 GB", | 20260508 07:23:21.537 | INFO | "huge" : "", | 20260508 07:23:21.537 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:21.537 | INFO | "malloc" : " 14 GB", | 20260508 07:23:21.537 | INFO | "pinned" : " 29 GB" | 20260508 07:23:21.537 | INFO | } | 20260508 07:23:21.537 | INFO | Perf report for the last 5.14164 seconds: | 20260508 07:23:21.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:21.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:21.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:21.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:21.537 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 07:23:21.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:21.537 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 07:23:21.537 | INFO | PRE_HQ : 66.4%, 1.8, 62.5ms, 0 | 20260508 07:23:21.537 | INFO | BAZWRITER : 67.3%, 2.8, 97.7ms, 0 | 20260508 07:23:21.537 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:23:21.537 | INFO | Uploaded 36GiB to GPU in 2.5648s (14.0362GiB/s) | 20260508 07:23:21.537 | INFO | Downloaded 38GiB to GPU in 2.69638s (14.093GiB/s) | 20260508 07:23:26.670 | INFO | Analyzing chunk frames = [6283776,6284288) | 20260508 07:23:26.670 | INFO | MemInfo: { | 20260508 07:23:26.670 | INFO | "gpu" : " 17 GB", | 20260508 07:23:26.670 | INFO | "huge" : "", | 20260508 07:23:26.670 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:26.670 | INFO | "malloc" : " 14 GB", | 20260508 07:23:26.670 | INFO | "pinned" : " 29 GB" | 20260508 07:23:26.670 | INFO | } | 20260508 07:23:26.670 | INFO | Perf report for the last 5.13351 seconds: | 20260508 07:23:26.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:26.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.670 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:23:26.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.670 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:23:26.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:26.671 | INFO | Uploaded 36GiB to GPU in 2.52339s (14.2665GiB/s) | 20260508 07:23:26.671 | INFO | Downloaded 34GiB to GPU in 2.4495s (13.8804GiB/s) | 20260508 07:23:31.809 | INFO | Analyzing chunk frames = [6284288,6284800) | 20260508 07:23:31.809 | INFO | MemInfo: { | 20260508 07:23:31.809 | INFO | "gpu" : " 17 GB", | 20260508 07:23:31.809 | INFO | "huge" : "", | 20260508 07:23:31.809 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:31.809 | INFO | "malloc" : " 14 GB", | 20260508 07:23:31.809 | INFO | "pinned" : " 29 GB" | 20260508 07:23:31.809 | INFO | } | 20260508 07:23:31.809 | INFO | Perf report for the last 5.13927 seconds: | 20260508 07:23:31.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:31.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 07:23:31.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:23:31.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:31.810 | INFO | Uploaded 36GiB to GPU in 2.49354s (14.4373GiB/s) | 20260508 07:23:31.810 | INFO | Downloaded 34GiB to GPU in 2.451s (13.8719GiB/s) | 20260508 07:23:36.948 | INFO | Analyzing chunk frames = [6284800,6285312) | 20260508 07:23:36.948 | INFO | MemInfo: { | 20260508 07:23:36.948 | INFO | "gpu" : " 17 GB", | 20260508 07:23:36.948 | INFO | "huge" : "", | 20260508 07:23:36.948 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:36.948 | INFO | "malloc" : " 14 GB", | 20260508 07:23:36.948 | INFO | "pinned" : " 29 GB" | 20260508 07:23:36.948 | INFO | } | 20260508 07:23:36.948 | INFO | Perf report for the last 5.13835 seconds: | 20260508 07:23:36.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:36.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 07:23:36.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:23:36.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:36.948 | INFO | Uploaded 36GiB to GPU in 2.55866s (14.0699GiB/s) | 20260508 07:23:36.948 | INFO | Downloaded 34GiB to GPU in 2.42037s (14.0474GiB/s) | 20260508 07:23:42.087 | INFO | Analyzing chunk frames = [6285312,6285824) | 20260508 07:23:42.087 | INFO | MemInfo: { | 20260508 07:23:42.087 | INFO | "gpu" : " 17 GB", | 20260508 07:23:42.088 | INFO | "huge" : "", | 20260508 07:23:42.088 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:42.088 | INFO | "malloc" : " 14 GB", | 20260508 07:23:42.088 | INFO | "pinned" : " 29 GB" | 20260508 07:23:42.088 | INFO | } | 20260508 07:23:42.088 | INFO | Perf report for the last 5.13983 seconds: | 20260508 07:23:42.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:42.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 07:23:42.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:23:42.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:42.088 | INFO | Uploaded 36GiB to GPU in 2.29195s (15.7071GiB/s) | 20260508 07:23:42.088 | INFO | Downloaded 34GiB to GPU in 2.49884s (13.6063GiB/s) | 20260508 07:23:47.232 | INFO | Analyzing chunk frames = [6285824,6286336) | 20260508 07:23:47.232 | INFO | MemInfo: { | 20260508 07:23:47.232 | INFO | "gpu" : " 17 GB", | 20260508 07:23:47.232 | INFO | "huge" : "", | 20260508 07:23:47.232 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:47.232 | INFO | "malloc" : " 14 GB", | 20260508 07:23:47.232 | INFO | "pinned" : " 29 GB" | 20260508 07:23:47.232 | INFO | } | 20260508 07:23:47.232 | INFO | Perf report for the last 5.14436 seconds: | 20260508 07:23:47.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:47.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:23:47.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:23:47.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:47.232 | INFO | Uploaded 36GiB to GPU in 2.47528s (14.5438GiB/s) | 20260508 07:23:47.232 | INFO | Downloaded 34GiB to GPU in 2.43599s (13.9573GiB/s) | 20260508 07:23:52.366 | INFO | Analyzing chunk frames = [6286336,6286848) | 20260508 07:23:52.366 | INFO | MemInfo: { | 20260508 07:23:52.366 | INFO | "gpu" : " 17 GB", | 20260508 07:23:52.366 | INFO | "huge" : "", | 20260508 07:23:52.366 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:52.366 | INFO | "malloc" : " 14 GB", | 20260508 07:23:52.366 | INFO | "pinned" : " 29 GB" | 20260508 07:23:52.366 | INFO | } | 20260508 07:23:52.366 | INFO | Perf report for the last 5.13431 seconds: | 20260508 07:23:52.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:52.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.366 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:23:52.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.366 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 07:23:52.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:52.367 | INFO | Uploaded 36GiB to GPU in 2.50221s (14.3873GiB/s) | 20260508 07:23:52.367 | INFO | Downloaded 34GiB to GPU in 2.43746s (13.949GiB/s) | 20260508 07:23:57.510 | INFO | Analyzing chunk frames = [6286848,6287360) | 20260508 07:23:57.510 | INFO | MemInfo: { | 20260508 07:23:57.510 | INFO | "gpu" : " 17 GB", | 20260508 07:23:57.510 | INFO | "huge" : "", | 20260508 07:23:57.510 | INFO | "huge_shared" : " 29 GB", | 20260508 07:23:57.510 | INFO | "malloc" : " 14 GB", | 20260508 07:23:57.510 | INFO | "pinned" : " 29 GB" | 20260508 07:23:57.510 | INFO | } | 20260508 07:23:57.510 | INFO | Perf report for the last 5.1437 seconds: | 20260508 07:23:57.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:23:57.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 07:23:57.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260508 07:23:57.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:23:57.511 | INFO | Uploaded 36GiB to GPU in 2.44404s (14.7297GiB/s) | 20260508 07:23:57.511 | INFO | Downloaded 34GiB to GPU in 2.4272s (14.0079GiB/s) | 20260508 07:23:57.623 | INFO | Starting a round of Baz IO | 20260508 07:24:01.045 | INFO | Populating RTMetrics report | 20260508 07:24:01.045 | INFO | RTMetrics report populated | 20260508 07:24:01.045 | INFO | Opening tmp RTMetrics json file | 20260508 07:24:01.055 | INFO | Serializing RTMetrics to json | 20260508 07:24:01.057 | INFO | Serialization done, swapping tmp file now... | 20260508 07:24:01.057 | INFO | RTMetrics done | 20260508 07:24:01.116 | INFO | Finished a round of Baz IO: Wrote 1.531 GiB in 3.492 seconds | 20260508 07:24:01.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.44% | 20260508 07:24:01.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.346116% / 0% / 3.57628e-05% | 20260508 07:24:01.116 | INFO | Baz IO overall: | 20260508 07:24:01.116 | INFO | BytesWritten : 2774629625470 | 20260508 07:24:01.116 | INFO | NumEvents : 1336847138815 | 20260508 07:24:02.650 | INFO | Analyzing chunk frames = [6287360,6287872) | 20260508 07:24:02.650 | INFO | MemInfo: { | 20260508 07:24:02.650 | INFO | "gpu" : " 17 GB", | 20260508 07:24:02.650 | INFO | "huge" : "", | 20260508 07:24:02.650 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:02.650 | INFO | "malloc" : " 14 GB", | 20260508 07:24:02.650 | INFO | "pinned" : " 29 GB" | 20260508 07:24:02.651 | INFO | } | 20260508 07:24:02.651 | INFO | Perf report for the last 5.14046 seconds: | 20260508 07:24:02.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:02.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:02.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:02.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:02.651 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 07:24:02.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:02.651 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 07:24:02.651 | INFO | PRE_HQ : 66.6%, 1.8, 62.3ms, 0 | 20260508 07:24:02.651 | INFO | BAZWRITER : 67.5%, 2.8, 97.5ms, 0 | 20260508 07:24:02.651 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:24:02.651 | INFO | Uploaded 36GiB to GPU in 2.60932s (13.7967GiB/s) | 20260508 07:24:02.651 | INFO | Downloaded 38GiB to GPU in 2.70112s (14.0682GiB/s) | 20260508 07:24:07.786 | INFO | Analyzing chunk frames = [6287872,6288384) | 20260508 07:24:07.786 | INFO | MemInfo: { | 20260508 07:24:07.786 | INFO | "gpu" : " 17 GB", | 20260508 07:24:07.786 | INFO | "huge" : "", | 20260508 07:24:07.786 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:07.786 | INFO | "malloc" : " 14 GB", | 20260508 07:24:07.786 | INFO | "pinned" : " 29 GB" | 20260508 07:24:07.786 | INFO | } | 20260508 07:24:07.786 | INFO | Perf report for the last 5.1357 seconds: | 20260508 07:24:07.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:07.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.786 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:24:07.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.786 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:24:07.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:07.787 | INFO | Uploaded 36GiB to GPU in 2.50367s (14.3789GiB/s) | 20260508 07:24:07.787 | INFO | Downloaded 34GiB to GPU in 2.43157s (13.9827GiB/s) | 20260508 07:24:12.919 | INFO | Analyzing chunk frames = [6288384,6288896) | 20260508 07:24:12.919 | INFO | MemInfo: { | 20260508 07:24:12.919 | INFO | "gpu" : " 17 GB", | 20260508 07:24:12.919 | INFO | "huge" : "", | 20260508 07:24:12.919 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:12.919 | INFO | "malloc" : " 14 GB", | 20260508 07:24:12.919 | INFO | "pinned" : " 29 GB" | 20260508 07:24:12.919 | INFO | } | 20260508 07:24:12.919 | INFO | Perf report for the last 5.13256 seconds: | 20260508 07:24:12.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:12.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:24:12.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:24:12.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:12.919 | INFO | Uploaded 36GiB to GPU in 2.56793s (14.0191GiB/s) | 20260508 07:24:12.919 | INFO | Downloaded 34GiB to GPU in 2.41815s (14.0603GiB/s) | 20260508 07:24:18.064 | INFO | Analyzing chunk frames = [6288896,6289408) | 20260508 07:24:18.064 | INFO | MemInfo: { | 20260508 07:24:18.064 | INFO | "gpu" : " 17 GB", | 20260508 07:24:18.064 | INFO | "huge" : "", | 20260508 07:24:18.064 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:18.064 | INFO | "malloc" : " 14 GB", | 20260508 07:24:18.064 | INFO | "pinned" : " 29 GB" | 20260508 07:24:18.064 | INFO | } | 20260508 07:24:18.064 | INFO | Perf report for the last 5.14493 seconds: | 20260508 07:24:18.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:18.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 07:24:18.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:24:18.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:18.064 | INFO | Uploaded 36GiB to GPU in 2.56626s (14.0282GiB/s) | 20260508 07:24:18.064 | INFO | Downloaded 34GiB to GPU in 2.3929s (14.2087GiB/s) | 20260508 07:24:23.204 | INFO | Analyzing chunk frames = [6289408,6289920) | 20260508 07:24:23.204 | INFO | MemInfo: { | 20260508 07:24:23.204 | INFO | "gpu" : " 17 GB", | 20260508 07:24:23.204 | INFO | "huge" : "", | 20260508 07:24:23.204 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:23.204 | INFO | "malloc" : " 14 GB", | 20260508 07:24:23.204 | INFO | "pinned" : " 29 GB" | 20260508 07:24:23.204 | INFO | } | 20260508 07:24:23.204 | INFO | Perf report for the last 5.14046 seconds: | 20260508 07:24:23.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:23.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.204 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 07:24:23.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.204 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:24:23.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:23.205 | INFO | Uploaded 36GiB to GPU in 2.44392s (14.7304GiB/s) | 20260508 07:24:23.205 | INFO | Downloaded 34GiB to GPU in 2.53431s (13.4159GiB/s) | 20260508 07:24:28.344 | INFO | Analyzing chunk frames = [6289920,6290432) | 20260508 07:24:28.344 | INFO | MemInfo: { | 20260508 07:24:28.344 | INFO | "gpu" : " 17 GB", | 20260508 07:24:28.344 | INFO | "huge" : "", | 20260508 07:24:28.344 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:28.344 | INFO | "malloc" : " 14 GB", | 20260508 07:24:28.345 | INFO | "pinned" : " 29 GB" | 20260508 07:24:28.345 | INFO | } | 20260508 07:24:28.345 | INFO | Perf report for the last 5.14038 seconds: | 20260508 07:24:28.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:28.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:24:28.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:24:28.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:28.345 | INFO | Uploaded 36GiB to GPU in 2.37919s (15.1312GiB/s) | 20260508 07:24:28.345 | INFO | Downloaded 34GiB to GPU in 2.41652s (14.0698GiB/s) | 20260508 07:24:33.483 | INFO | Analyzing chunk frames = [6290432,6290944) | 20260508 07:24:33.483 | INFO | MemInfo: { | 20260508 07:24:33.483 | INFO | "gpu" : " 17 GB", | 20260508 07:24:33.483 | INFO | "huge" : "", | 20260508 07:24:33.483 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:33.483 | INFO | "malloc" : " 14 GB", | 20260508 07:24:33.483 | INFO | "pinned" : " 29 GB" | 20260508 07:24:33.484 | INFO | } | 20260508 07:24:33.484 | INFO | Perf report for the last 5.13889 seconds: | 20260508 07:24:33.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:33.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:24:33.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:24:33.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:33.484 | INFO | Uploaded 36GiB to GPU in 2.50208s (14.388GiB/s) | 20260508 07:24:33.484 | INFO | Downloaded 34GiB to GPU in 2.44035s (13.9324GiB/s) | 20260508 07:24:38.617 | INFO | Analyzing chunk frames = [6290944,6291456) | 20260508 07:24:38.617 | INFO | MemInfo: { | 20260508 07:24:38.617 | INFO | "gpu" : " 17 GB", | 20260508 07:24:38.617 | INFO | "huge" : "", | 20260508 07:24:38.617 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:38.617 | INFO | "malloc" : " 14 GB", | 20260508 07:24:38.617 | INFO | "pinned" : " 29 GB" | 20260508 07:24:38.617 | INFO | } | 20260508 07:24:38.617 | INFO | Perf report for the last 5.13329 seconds: | 20260508 07:24:38.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:38.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 07:24:38.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 07:24:38.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:38.617 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:24:38.617 | INFO | Total mem usage: 16.7 GiB | 20260508 07:24:38.617 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:24:38.617 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:24:38.617 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:24:38.617 | INFO | High Water Breakdown: | 20260508 07:24:38.617 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:24:38.617 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:24:38.617 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:24:38.617 | INFO | ------------------------------------------ | 20260508 07:24:38.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:24:38.617 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:24:38.617 | INFO | Total mem usage: 26.5 GiB | 20260508 07:24:38.617 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:24:38.617 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:24:38.617 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:24:38.617 | INFO | High Water Breakdown: | 20260508 07:24:38.617 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:24:38.617 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:24:38.617 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:24:38.617 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:24:38.617 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:24:38.617 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:24:38.617 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:24:38.617 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:24:38.617 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:24:38.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:24:38.617 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:24:38.617 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:24:38.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:24:38.617 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:24:38.617 | INFO | ------------------------------------------ | 20260508 07:24:38.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:24:38.617 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:24:38.617 | INFO | Total mem usage: 27.8 GiB | 20260508 07:24:38.617 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:24:38.617 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:24:38.617 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:24:38.617 | INFO | High Water Breakdown: | 20260508 07:24:38.617 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:24:38.617 | INFO | ------------------------------------------ | 20260508 07:24:38.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:24:38.617 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:24:38.617 | INFO | Total mem usage: 15.7 GiB | 20260508 07:24:38.617 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:24:38.617 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:24:38.617 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:24:38.617 | INFO | High Water Breakdown: | 20260508 07:24:38.617 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:24:38.617 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:24:38.617 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:24:38.617 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:24:38.617 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:24:38.617 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:24:38.617 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:24:38.617 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:24:38.617 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:24:38.617 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:24:38.617 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:24:38.617 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:24:38.617 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:24:38.617 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:24:38.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:24:38.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:24:38.617 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:24:38.617 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:24:38.617 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:24:38.617 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:24:38.617 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:24:38.617 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:24:38.617 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:24:38.617 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:24:38.617 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:24:38.617 | INFO | ------------------------------------------ | 20260508 07:24:38.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:24:38.617 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:24:38.617 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:24:38.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:24:38.617 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 7.41e-01 | 20260508 07:24:38.617 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.07e-01, 1.16e-02 / 1.20e-02 | 20260508 07:24:38.617 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.44e+00, 5.82e-01 / 2.17e-01 | 20260508 07:24:38.617 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.07e-01 / 1.16e-01 | 20260508 07:24:38.617 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.51e+01, 3.82e+00 / 1.07e+00 | 20260508 07:24:38.618 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.84e+00 / 1.44e+00 | 20260508 07:24:38.618 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.36e+00 / 2.55e+00, 4.10e-01 / 2.24e-01 | 20260508 07:24:38.618 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.23e+00, 2.49e-01 / 1.29e-01 | 20260508 07:24:38.618 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.45e-01, 4.87e-02 / 2.55e-02 | 20260508 07:24:38.618 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:24:38.618 | INFO | Uploaded 36GiB to GPU in 2.50225s (14.3871GiB/s) | 20260508 07:24:38.619 | INFO | Downloaded 34GiB to GPU in 2.4467s (13.8963GiB/s) | 20260508 07:24:38.619 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:24:38.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:24:38.619 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 24.51%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 24.31%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 2.37%, LockUpload -- 2.51e+00 / 0.00e+00, 5.80e+00 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 2.42%, LockDownload -- 2.56e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 15.51%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:24:38.619 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.42e-02 / 0.00e+00 | 20260508 07:24:38.715 | INFO | Starting a round of Baz IO | 20260508 07:24:42.134 | INFO | Populating RTMetrics report | 20260508 07:24:42.135 | INFO | RTMetrics report populated | 20260508 07:24:42.135 | INFO | Opening tmp RTMetrics json file | 20260508 07:24:42.151 | INFO | Serializing RTMetrics to json | 20260508 07:24:42.153 | INFO | Serialization done, swapping tmp file now... | 20260508 07:24:42.153 | INFO | RTMetrics done | 20260508 07:24:42.209 | INFO | Finished a round of Baz IO: Wrote 1.531 GiB in 3.493 seconds | 20260508 07:24:42.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.44% | 20260508 07:24:42.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345826% / 0% / 5.5631e-05% | 20260508 07:24:42.209 | INFO | Baz IO overall: | 20260508 07:24:42.209 | INFO | BytesWritten : 2776273323204 | 20260508 07:24:42.209 | INFO | NumEvents : 1337627370868 | 20260508 07:24:43.759 | INFO | Analyzing chunk frames = [6291456,6291968) | 20260508 07:24:43.759 | INFO | MemInfo: { | 20260508 07:24:43.759 | INFO | "gpu" : " 17 GB", | 20260508 07:24:43.759 | INFO | "huge" : "", | 20260508 07:24:43.759 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:43.759 | INFO | "malloc" : " 14 GB", | 20260508 07:24:43.759 | INFO | "pinned" : " 29 GB" | 20260508 07:24:43.759 | INFO | } | 20260508 07:24:43.759 | INFO | Perf report for the last 5.14257 seconds: | 20260508 07:24:43.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:43.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:43.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:43.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:43.760 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 07:24:43.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:43.760 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:24:43.760 | INFO | PRE_HQ : 65.5%, 1.6, 55.1ms, 0 | 20260508 07:24:43.760 | INFO | BAZWRITER : 67.4%, 2.9, 101.0ms, 0 | 20260508 07:24:43.760 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:24:43.760 | INFO | Uploaded 36GiB to GPU in 2.5993s (13.8499GiB/s) | 20260508 07:24:43.760 | INFO | Downloaded 38GiB to GPU in 2.69663s (14.0916GiB/s) | 20260508 07:24:48.902 | INFO | Analyzing chunk frames = [6291968,6292480) | 20260508 07:24:48.902 | INFO | MemInfo: { | 20260508 07:24:48.902 | INFO | "gpu" : " 17 GB", | 20260508 07:24:48.902 | INFO | "huge" : "", | 20260508 07:24:48.902 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:48.902 | INFO | "malloc" : " 14 GB", | 20260508 07:24:48.902 | INFO | "pinned" : " 29 GB" | 20260508 07:24:48.902 | INFO | } | 20260508 07:24:48.902 | INFO | Perf report for the last 5.14294 seconds: | 20260508 07:24:48.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:48.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.902 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:24:48.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.902 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:24:48.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:48.903 | INFO | Uploaded 36GiB to GPU in 2.51674s (14.3042GiB/s) | 20260508 07:24:48.903 | INFO | Downloaded 34GiB to GPU in 2.43435s (13.9668GiB/s) | 20260508 07:24:54.041 | INFO | Analyzing chunk frames = [6292480,6292992) | 20260508 07:24:54.041 | INFO | MemInfo: { | 20260508 07:24:54.041 | INFO | "gpu" : " 17 GB", | 20260508 07:24:54.041 | INFO | "huge" : "", | 20260508 07:24:54.041 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:54.041 | INFO | "malloc" : " 14 GB", | 20260508 07:24:54.041 | INFO | "pinned" : " 29 GB" | 20260508 07:24:54.041 | INFO | } | 20260508 07:24:54.041 | INFO | Perf report for the last 5.13872 seconds: | 20260508 07:24:54.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:54.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.041 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:24:54.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.041 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:24:54.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:54.042 | INFO | Uploaded 36GiB to GPU in 2.58229s (13.9411GiB/s) | 20260508 07:24:54.042 | INFO | Downloaded 34GiB to GPU in 2.43029s (13.9901GiB/s) | 20260508 07:24:59.181 | INFO | Analyzing chunk frames = [6292992,6293504) | 20260508 07:24:59.181 | INFO | MemInfo: { | 20260508 07:24:59.181 | INFO | "gpu" : " 17 GB", | 20260508 07:24:59.181 | INFO | "huge" : "", | 20260508 07:24:59.181 | INFO | "huge_shared" : " 29 GB", | 20260508 07:24:59.182 | INFO | "malloc" : " 14 GB", | 20260508 07:24:59.182 | INFO | "pinned" : " 29 GB" | 20260508 07:24:59.182 | INFO | } | 20260508 07:24:59.182 | INFO | Perf report for the last 5.14043 seconds: | 20260508 07:24:59.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:24:59.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 07:24:59.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:24:59.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:24:59.182 | INFO | Uploaded 36GiB to GPU in 2.58813s (13.9096GiB/s) | 20260508 07:24:59.182 | INFO | Downloaded 34GiB to GPU in 2.41343s (14.0879GiB/s) | 20260508 07:25:04.320 | INFO | Analyzing chunk frames = [6293504,6294016) | 20260508 07:25:04.320 | INFO | MemInfo: { | 20260508 07:25:04.320 | INFO | "gpu" : " 17 GB", | 20260508 07:25:04.320 | INFO | "huge" : "", | 20260508 07:25:04.320 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:04.320 | INFO | "malloc" : " 14 GB", | 20260508 07:25:04.320 | INFO | "pinned" : " 29 GB" | 20260508 07:25:04.320 | INFO | } | 20260508 07:25:04.320 | INFO | Perf report for the last 5.13875 seconds: | 20260508 07:25:04.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:04.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.320 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:25:04.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.320 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:25:04.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:04.321 | INFO | Uploaded 36GiB to GPU in 2.4113s (14.9297GiB/s) | 20260508 07:25:04.321 | INFO | Downloaded 34GiB to GPU in 2.52084s (13.4876GiB/s) | 20260508 07:25:09.460 | INFO | Analyzing chunk frames = [6294016,6294528) | 20260508 07:25:09.460 | INFO | MemInfo: { | 20260508 07:25:09.460 | INFO | "gpu" : " 17 GB", | 20260508 07:25:09.460 | INFO | "huge" : "", | 20260508 07:25:09.460 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:09.460 | INFO | "malloc" : " 14 GB", | 20260508 07:25:09.460 | INFO | "pinned" : " 29 GB" | 20260508 07:25:09.460 | INFO | } | 20260508 07:25:09.460 | INFO | Perf report for the last 5.1399 seconds: | 20260508 07:25:09.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:09.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.460 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:25:09.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.460 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:25:09.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:09.461 | INFO | Uploaded 36GiB to GPU in 2.37293s (15.1711GiB/s) | 20260508 07:25:09.461 | INFO | Downloaded 34GiB to GPU in 2.4255s (14.0177GiB/s) | 20260508 07:25:14.602 | INFO | Analyzing chunk frames = [6294528,6295040) | 20260508 07:25:14.602 | INFO | MemInfo: { | 20260508 07:25:14.602 | INFO | "gpu" : " 17 GB", | 20260508 07:25:14.602 | INFO | "huge" : "", | 20260508 07:25:14.602 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:14.602 | INFO | "malloc" : " 14 GB", | 20260508 07:25:14.602 | INFO | "pinned" : " 29 GB" | 20260508 07:25:14.602 | INFO | } | 20260508 07:25:14.602 | INFO | Perf report for the last 5.14163 seconds: | 20260508 07:25:14.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:14.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 07:25:14.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:25:14.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:14.602 | INFO | Uploaded 36GiB to GPU in 2.50639s (14.3633GiB/s) | 20260508 07:25:14.602 | INFO | Downloaded 34GiB to GPU in 2.4354s (13.9608GiB/s) | 20260508 07:25:19.737 | INFO | Analyzing chunk frames = [6295040,6295552) | 20260508 07:25:19.737 | INFO | MemInfo: { | 20260508 07:25:19.737 | INFO | "gpu" : " 17 GB", | 20260508 07:25:19.737 | INFO | "huge" : "", | 20260508 07:25:19.737 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:19.737 | INFO | "malloc" : " 14 GB", | 20260508 07:25:19.737 | INFO | "pinned" : " 29 GB" | 20260508 07:25:19.737 | INFO | } | 20260508 07:25:19.737 | INFO | Perf report for the last 5.13541 seconds: | 20260508 07:25:19.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:19.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.737 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 07:25:19.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.737 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 07:25:19.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:19.738 | INFO | Uploaded 36GiB to GPU in 2.47555s (14.5423GiB/s) | 20260508 07:25:19.738 | INFO | Downloaded 34GiB to GPU in 2.44558s (13.9026GiB/s) | 20260508 07:25:19.835 | INFO | Starting a round of Baz IO | 20260508 07:25:23.267 | INFO | Populating RTMetrics report | 20260508 07:25:23.267 | INFO | RTMetrics report populated | 20260508 07:25:23.267 | INFO | Opening tmp RTMetrics json file | 20260508 07:25:23.279 | INFO | Serializing RTMetrics to json | 20260508 07:25:23.280 | INFO | Serialization done, swapping tmp file now... | 20260508 07:25:23.280 | INFO | RTMetrics done | 20260508 07:25:23.335 | INFO | Finished a round of Baz IO: Wrote 1.53 GiB in 3.5 seconds | 20260508 07:25:23.335 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.44% | 20260508 07:25:23.335 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.343808% / 0% / 3.57628e-05% | 20260508 07:25:23.335 | INFO | Baz IO overall: | 20260508 07:25:23.336 | INFO | BytesWritten : 2777915622101 | 20260508 07:25:23.336 | INFO | NumEvents : 1338406914513 | 20260508 07:25:24.870 | INFO | Analyzing chunk frames = [6295552,6296064) | 20260508 07:25:24.870 | INFO | MemInfo: { | 20260508 07:25:24.870 | INFO | "gpu" : " 17 GB", | 20260508 07:25:24.870 | INFO | "huge" : "", | 20260508 07:25:24.870 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:24.870 | INFO | "malloc" : " 14 GB", | 20260508 07:25:24.870 | INFO | "pinned" : " 29 GB" | 20260508 07:25:24.870 | INFO | } | 20260508 07:25:24.870 | INFO | Perf report for the last 5.13275 seconds: | 20260508 07:25:24.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:24.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:24.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:24.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:24.870 | INFO | ANALYSIS : 68.1%, 3.0, 106.1ms, 0 | 20260508 07:25:24.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:24.870 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 07:25:24.870 | INFO | PRE_HQ : 65.8%, 1.6, 55.0ms, 0 | 20260508 07:25:24.870 | INFO | BAZWRITER : 67.7%, 2.6, 90.2ms, 0 | 20260508 07:25:24.870 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:25:24.871 | INFO | Uploaded 36GiB to GPU in 2.56227s (14.05GiB/s) | 20260508 07:25:24.871 | INFO | Downloaded 38GiB to GPU in 2.7017s (14.0652GiB/s) | 20260508 07:25:30.014 | INFO | Analyzing chunk frames = [6296064,6296576) | 20260508 07:25:30.014 | INFO | MemInfo: { | 20260508 07:25:30.014 | INFO | "gpu" : " 17 GB", | 20260508 07:25:30.014 | INFO | "huge" : "", | 20260508 07:25:30.014 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:30.014 | INFO | "malloc" : " 14 GB", | 20260508 07:25:30.014 | INFO | "pinned" : " 29 GB" | 20260508 07:25:30.014 | INFO | } | 20260508 07:25:30.014 | INFO | Perf report for the last 5.14401 seconds: | 20260508 07:25:30.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:30.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.014 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:25:30.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.014 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:25:30.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:30.015 | INFO | Uploaded 36GiB to GPU in 2.51678s (14.304GiB/s) | 20260508 07:25:30.015 | INFO | Downloaded 34GiB to GPU in 2.44007s (13.934GiB/s) | 20260508 07:25:35.153 | INFO | Analyzing chunk frames = [6296576,6297088) | 20260508 07:25:35.153 | INFO | MemInfo: { | 20260508 07:25:35.153 | INFO | "gpu" : " 17 GB", | 20260508 07:25:35.153 | INFO | "huge" : "", | 20260508 07:25:35.153 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:35.153 | INFO | "malloc" : " 14 GB", | 20260508 07:25:35.153 | INFO | "pinned" : " 29 GB" | 20260508 07:25:35.153 | INFO | } | 20260508 07:25:35.153 | INFO | Perf report for the last 5.13921 seconds: | 20260508 07:25:35.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:35.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.153 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:25:35.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.153 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:25:35.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:35.154 | INFO | Uploaded 36GiB to GPU in 2.57636s (13.9732GiB/s) | 20260508 07:25:35.154 | INFO | Downloaded 34GiB to GPU in 2.42967s (13.9937GiB/s) | 20260508 07:25:40.291 | INFO | Analyzing chunk frames = [6297088,6297600) | 20260508 07:25:40.291 | INFO | MemInfo: { | 20260508 07:25:40.291 | INFO | "gpu" : " 17 GB", | 20260508 07:25:40.291 | INFO | "huge" : "", | 20260508 07:25:40.291 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:40.292 | INFO | "malloc" : " 14 GB", | 20260508 07:25:40.292 | INFO | "pinned" : " 29 GB" | 20260508 07:25:40.292 | INFO | } | 20260508 07:25:40.292 | INFO | Perf report for the last 5.1384 seconds: | 20260508 07:25:40.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:40.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:25:40.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 07:25:40.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:40.292 | INFO | Uploaded 36GiB to GPU in 2.56433s (14.0387GiB/s) | 20260508 07:25:40.292 | INFO | Downloaded 34GiB to GPU in 2.39523s (14.1949GiB/s) | 20260508 07:25:45.432 | INFO | Analyzing chunk frames = [6297600,6298112) | 20260508 07:25:45.432 | INFO | MemInfo: { | 20260508 07:25:45.432 | INFO | "gpu" : " 17 GB", | 20260508 07:25:45.432 | INFO | "huge" : "", | 20260508 07:25:45.433 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:45.433 | INFO | "malloc" : " 14 GB", | 20260508 07:25:45.433 | INFO | "pinned" : " 29 GB" | 20260508 07:25:45.433 | INFO | } | 20260508 07:25:45.433 | INFO | Perf report for the last 5.14096 seconds: | 20260508 07:25:45.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:45.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:25:45.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 07:25:45.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:45.433 | INFO | Uploaded 36GiB to GPU in 2.29493s (15.6868GiB/s) | 20260508 07:25:45.433 | INFO | Downloaded 34GiB to GPU in 2.5062s (13.5664GiB/s) | 20260508 07:25:50.572 | INFO | Analyzing chunk frames = [6298112,6298624) | 20260508 07:25:50.572 | INFO | MemInfo: { | 20260508 07:25:50.572 | INFO | "gpu" : " 17 GB", | 20260508 07:25:50.572 | INFO | "huge" : "", | 20260508 07:25:50.572 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:50.572 | INFO | "malloc" : " 14 GB", | 20260508 07:25:50.572 | INFO | "pinned" : " 29 GB" | 20260508 07:25:50.572 | INFO | } | 20260508 07:25:50.572 | INFO | Perf report for the last 5.13926 seconds: | 20260508 07:25:50.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:50.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.572 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:25:50.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.572 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:25:50.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:50.573 | INFO | Uploaded 36GiB to GPU in 2.34958s (15.3219GiB/s) | 20260508 07:25:50.573 | INFO | Downloaded 34GiB to GPU in 2.43288s (13.9752GiB/s) | 20260508 07:25:55.713 | INFO | Analyzing chunk frames = [6298624,6299136) | 20260508 07:25:55.713 | INFO | MemInfo: { | 20260508 07:25:55.713 | INFO | "gpu" : " 17 GB", | 20260508 07:25:55.713 | INFO | "huge" : "", | 20260508 07:25:55.713 | INFO | "huge_shared" : " 29 GB", | 20260508 07:25:55.713 | INFO | "malloc" : " 14 GB", | 20260508 07:25:55.713 | INFO | "pinned" : " 29 GB" | 20260508 07:25:55.713 | INFO | } | 20260508 07:25:55.713 | INFO | Perf report for the last 5.1415 seconds: | 20260508 07:25:55.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:25:55.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:25:55.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:25:55.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:25:55.714 | INFO | Uploaded 36GiB to GPU in 2.46845s (14.584GiB/s) | 20260508 07:25:55.714 | INFO | Downloaded 34GiB to GPU in 2.43242s (13.9779GiB/s) | 20260508 07:26:00.857 | INFO | Analyzing chunk frames = [6299136,6299648) | 20260508 07:26:00.857 | INFO | MemInfo: { | 20260508 07:26:00.857 | INFO | "gpu" : " 17 GB", | 20260508 07:26:00.857 | INFO | "huge" : "", | 20260508 07:26:00.857 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:00.857 | INFO | "malloc" : " 14 GB", | 20260508 07:26:00.857 | INFO | "pinned" : " 29 GB" | 20260508 07:26:00.857 | INFO | } | 20260508 07:26:00.857 | INFO | Perf report for the last 5.14385 seconds: | 20260508 07:26:00.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:00.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.857 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:26:00.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.858 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:26:00.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:00.858 | INFO | Uploaded 36GiB to GPU in 2.47642s (14.5371GiB/s) | 20260508 07:26:00.858 | INFO | Downloaded 34GiB to GPU in 2.44781s (13.8899GiB/s) | 20260508 07:26:00.858 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:26:00.858 | INFO | Total mem usage: 16.7 GiB | 20260508 07:26:00.858 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:26:00.858 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -13.9%) | 20260508 07:26:00.858 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:26:00.858 | INFO | High Water Breakdown: | 20260508 07:26:00.858 | INFO | 59.9% : 8.77 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:26:00.858 | INFO | 40% : 5.86 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:26:00.858 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:26:00.858 | INFO | ------------------------------------------ | 20260508 07:26:00.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:26:00.858 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:26:00.858 | INFO | Total mem usage: 26.5 GiB | 20260508 07:26:00.858 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:26:00.858 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:26:00.858 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:26:00.858 | INFO | High Water Breakdown: | 20260508 07:26:00.858 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:26:00.858 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:26:00.858 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:26:00.858 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:26:00.858 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:26:00.858 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:26:00.858 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:26:00.858 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:26:00.858 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:26:00.858 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:26:00.858 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:26:00.858 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:26:00.858 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:26:00.858 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:26:00.858 | INFO | ------------------------------------------ | 20260508 07:26:00.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:26:00.858 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:26:00.858 | INFO | Total mem usage: 27.8 GiB | 20260508 07:26:00.858 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:26:00.858 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:26:00.858 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:26:00.858 | INFO | High Water Breakdown: | 20260508 07:26:00.858 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:26:00.858 | INFO | ------------------------------------------ | 20260508 07:26:00.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:26:00.858 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:26:00.858 | INFO | Total mem usage: 15.7 GiB | 20260508 07:26:00.858 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:26:00.858 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:26:00.858 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:26:00.859 | INFO | High Water Breakdown: | 20260508 07:26:00.859 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:26:00.859 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:26:00.859 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:26:00.859 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:26:00.859 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:26:00.859 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:26:00.859 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:26:00.859 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:26:00.859 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:26:00.859 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:26:00.859 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:26:00.859 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:26:00.859 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:26:00.859 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:26:00.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:26:00.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:26:00.859 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:26:00.859 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:26:00.859 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:26:00.859 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:26:00.859 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:26:00.859 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:26:00.859 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:26:00.859 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:26:00.859 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:26:00.859 | INFO | ------------------------------------------ | 20260508 07:26:00.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:26:00.859 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:26:00.859 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:26:00.859 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:26:00.859 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 7.30e-01 | 20260508 07:26:00.859 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.11e-02 / 4.22e-03 | 20260508 07:26:00.859 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.40e+00, 5.88e-01 / 2.08e-01 | 20260508 07:26:00.859 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.28e-01 | 20260508 07:26:00.859 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.51e+01, 3.78e+00 / 8.90e-01 | 20260508 07:26:00.859 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.28e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.36e+00 / 2.54e+00, 4.11e-01 / 2.58e-01 | 20260508 07:26:00.859 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.21e+00, 2.51e-01 / 1.25e-01 | 20260508 07:26:00.859 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.49e-01, 4.95e-02 / 1.87e-02 | 20260508 07:26:00.859 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:26:00.859 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:26:00.859 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:26:00.859 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 24.31%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 30.87%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 2.42%, LockUpload -- 2.57e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 2.36%, LockDownload -- 2.50e+00 / 0.00e+00, 3.84e+00 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 15.55%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:26:00.859 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 4.88e-02 / 0.00e+00 | 20260508 07:26:00.955 | INFO | Starting a round of Baz IO | 20260508 07:26:04.367 | INFO | Populating RTMetrics report | 20260508 07:26:04.367 | INFO | RTMetrics report populated | 20260508 07:26:04.367 | INFO | Opening tmp RTMetrics json file | 20260508 07:26:04.379 | INFO | Serializing RTMetrics to json | 20260508 07:26:04.380 | INFO | Serialization done, swapping tmp file now... | 20260508 07:26:04.381 | INFO | RTMetrics done | 20260508 07:26:04.436 | INFO | Finished a round of Baz IO: Wrote 1.531 GiB in 3.48 seconds | 20260508 07:26:04.436 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.56% / 0.00% / 59.44% | 20260508 07:26:04.436 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345508% / 3.97364e-06% / 4.76837e-05% | 20260508 07:26:04.436 | INFO | Baz IO overall: | 20260508 07:26:04.436 | INFO | BytesWritten : 2779559091849 | 20260508 07:26:04.436 | INFO | NumEvents : 1339187846399 | 20260508 07:26:05.993 | INFO | Analyzing chunk frames = [6299648,6300160) | 20260508 07:26:05.993 | INFO | MemInfo: { | 20260508 07:26:05.993 | INFO | "gpu" : " 17 GB", | 20260508 07:26:05.993 | INFO | "huge" : "", | 20260508 07:26:05.993 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:05.993 | INFO | "malloc" : " 14 GB", | 20260508 07:26:05.993 | INFO | "pinned" : " 29 GB" | 20260508 07:26:05.993 | INFO | } | 20260508 07:26:05.993 | INFO | Perf report for the last 5.13545 seconds: | 20260508 07:26:05.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:05.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:05.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:05.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:05.993 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 07:26:05.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:05.993 | INFO | RT_METRICS : 39.4%, 1.0, 20.6ms, 0 | 20260508 07:26:05.993 | INFO | PRE_HQ : 65.6%, 1.6, 55.2ms, 0 | 20260508 07:26:05.993 | INFO | BAZWRITER : 67.1%, 2.6, 90.7ms, 0 | 20260508 07:26:05.993 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:26:05.993 | INFO | Uploaded 36GiB to GPU in 2.55388s (14.0962GiB/s) | 20260508 07:26:05.993 | INFO | Downloaded 38GiB to GPU in 2.70669s (14.0393GiB/s) | 20260508 07:26:11.128 | INFO | Analyzing chunk frames = [6300160,6300672) | 20260508 07:26:11.128 | INFO | MemInfo: { | 20260508 07:26:11.128 | INFO | "gpu" : " 17 GB", | 20260508 07:26:11.128 | INFO | "huge" : "", | 20260508 07:26:11.128 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:11.128 | INFO | "malloc" : " 14 GB", | 20260508 07:26:11.128 | INFO | "pinned" : " 29 GB" | 20260508 07:26:11.128 | INFO | } | 20260508 07:26:11.129 | INFO | Perf report for the last 5.13571 seconds: | 20260508 07:26:11.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:11.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:26:11.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:26:11.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:11.129 | INFO | Uploaded 36GiB to GPU in 2.50489s (14.3719GiB/s) | 20260508 07:26:11.129 | INFO | Downloaded 34GiB to GPU in 2.4342s (13.9676GiB/s) | 20260508 07:26:16.272 | INFO | Analyzing chunk frames = [6300672,6301184) | 20260508 07:26:16.272 | INFO | MemInfo: { | 20260508 07:26:16.272 | INFO | "gpu" : " 17 GB", | 20260508 07:26:16.272 | INFO | "huge" : "", | 20260508 07:26:16.272 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:16.272 | INFO | "malloc" : " 14 GB", | 20260508 07:26:16.272 | INFO | "pinned" : " 29 GB" | 20260508 07:26:16.272 | INFO | } | 20260508 07:26:16.272 | INFO | Perf report for the last 5.14373 seconds: | 20260508 07:26:16.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:16.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.272 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 07:26:16.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.273 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:26:16.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:16.273 | INFO | Uploaded 36GiB to GPU in 2.54843s (14.1263GiB/s) | 20260508 07:26:16.273 | INFO | Downloaded 34GiB to GPU in 2.41785s (14.0621GiB/s) | 20260508 07:26:21.410 | INFO | Analyzing chunk frames = [6301184,6301696) | 20260508 07:26:21.410 | INFO | MemInfo: { | 20260508 07:26:21.410 | INFO | "gpu" : " 17 GB", | 20260508 07:26:21.410 | INFO | "huge" : "", | 20260508 07:26:21.410 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:21.410 | INFO | "malloc" : " 14 GB", | 20260508 07:26:21.410 | INFO | "pinned" : " 29 GB" | 20260508 07:26:21.410 | INFO | } | 20260508 07:26:21.410 | INFO | Perf report for the last 5.13796 seconds: | 20260508 07:26:21.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:21.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.410 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:26:21.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.410 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:26:21.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:21.411 | INFO | Uploaded 36GiB to GPU in 2.56812s (14.018GiB/s) | 20260508 07:26:21.411 | INFO | Downloaded 34GiB to GPU in 2.4116s (14.0985GiB/s) | 20260508 07:26:26.551 | INFO | Analyzing chunk frames = [6301696,6302208) | 20260508 07:26:26.551 | INFO | MemInfo: { | 20260508 07:26:26.551 | INFO | "gpu" : " 17 GB", | 20260508 07:26:26.551 | INFO | "huge" : "", | 20260508 07:26:26.551 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:26.551 | INFO | "malloc" : " 14 GB", | 20260508 07:26:26.551 | INFO | "pinned" : " 29 GB" | 20260508 07:26:26.551 | INFO | } | 20260508 07:26:26.551 | INFO | Perf report for the last 5.14088 seconds: | 20260508 07:26:26.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:26.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:26:26.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:26:26.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:26.551 | INFO | Uploaded 36GiB to GPU in 2.28479s (15.7564GiB/s) | 20260508 07:26:26.551 | INFO | Downloaded 34GiB to GPU in 2.50164s (13.5911GiB/s) | 20260508 07:26:31.680 | INFO | Analyzing chunk frames = [6302208,6302720) | 20260508 07:26:31.680 | INFO | MemInfo: { | 20260508 07:26:31.680 | INFO | "gpu" : " 17 GB", | 20260508 07:26:31.681 | INFO | "huge" : "", | 20260508 07:26:31.681 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:31.681 | INFO | "malloc" : " 14 GB", | 20260508 07:26:31.681 | INFO | "pinned" : " 29 GB" | 20260508 07:26:31.681 | INFO | } | 20260508 07:26:31.681 | INFO | Perf report for the last 5.12954 seconds: | 20260508 07:26:31.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:31.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 07:26:31.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:26:31.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:31.681 | INFO | Uploaded 36GiB to GPU in 2.3713s (15.1815GiB/s) | 20260508 07:26:31.681 | INFO | Downloaded 34GiB to GPU in 2.42209s (14.0375GiB/s) | 20260508 07:26:36.826 | INFO | Analyzing chunk frames = [6302720,6303232) | 20260508 07:26:36.826 | INFO | MemInfo: { | 20260508 07:26:36.826 | INFO | "gpu" : " 17 GB", | 20260508 07:26:36.826 | INFO | "huge" : "", | 20260508 07:26:36.826 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:36.826 | INFO | "malloc" : " 14 GB", | 20260508 07:26:36.826 | INFO | "pinned" : " 29 GB" | 20260508 07:26:36.826 | INFO | } | 20260508 07:26:36.826 | INFO | Perf report for the last 5.14541 seconds: | 20260508 07:26:36.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:36.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.826 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:26:36.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.826 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:26:36.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:36.827 | INFO | Uploaded 36GiB to GPU in 2.48007s (14.5157GiB/s) | 20260508 07:26:36.827 | INFO | Downloaded 34GiB to GPU in 2.42478s (14.0219GiB/s) | 20260508 07:26:41.967 | INFO | Analyzing chunk frames = [6303232,6303744) | 20260508 07:26:41.967 | INFO | MemInfo: { | 20260508 07:26:41.967 | INFO | "gpu" : " 17 GB", | 20260508 07:26:41.967 | INFO | "huge" : "", | 20260508 07:26:41.967 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:41.967 | INFO | "malloc" : " 14 GB", | 20260508 07:26:41.968 | INFO | "pinned" : " 29 GB" | 20260508 07:26:41.968 | INFO | } | 20260508 07:26:41.968 | INFO | Perf report for the last 5.14159 seconds: | 20260508 07:26:41.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:41.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:26:41.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:26:41.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:41.968 | INFO | Uploaded 36GiB to GPU in 2.48143s (14.5078GiB/s) | 20260508 07:26:41.968 | INFO | Downloaded 34GiB to GPU in 2.43555s (13.9599GiB/s) | 20260508 07:26:42.065 | INFO | Starting a round of Baz IO | 20260508 07:26:45.487 | INFO | Populating RTMetrics report | 20260508 07:26:45.487 | INFO | RTMetrics report populated | 20260508 07:26:45.487 | INFO | Opening tmp RTMetrics json file | 20260508 07:26:45.497 | INFO | Serializing RTMetrics to json | 20260508 07:26:45.500 | INFO | Serialization done, swapping tmp file now... | 20260508 07:26:45.500 | INFO | RTMetrics done | 20260508 07:26:45.554 | INFO | Finished a round of Baz IO: Wrote 1.529 GiB in 3.489 seconds | 20260508 07:26:45.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.00% / 59.44% | 20260508 07:26:45.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34428% / 1.19209e-05% / 4.76837e-05% | 20260508 07:26:45.554 | INFO | Baz IO overall: | 20260508 07:26:45.554 | INFO | BytesWritten : 2781200306829 | 20260508 07:26:45.554 | INFO | NumEvents : 1339967443002 | 20260508 07:26:47.109 | INFO | Analyzing chunk frames = [6303744,6304256) | 20260508 07:26:47.109 | INFO | MemInfo: { | 20260508 07:26:47.109 | INFO | "gpu" : " 17 GB", | 20260508 07:26:47.110 | INFO | "huge" : "", | 20260508 07:26:47.110 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:47.110 | INFO | "malloc" : " 14 GB", | 20260508 07:26:47.110 | INFO | "pinned" : " 29 GB" | 20260508 07:26:47.110 | INFO | } | 20260508 07:26:47.110 | INFO | Perf report for the last 5.14197 seconds: | 20260508 07:26:47.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:47.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:47.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:47.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:47.110 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 07:26:47.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:47.110 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 07:26:47.110 | INFO | PRE_HQ : 65.6%, 1.6, 54.8ms, 0 | 20260508 07:26:47.110 | INFO | BAZWRITER : 67.2%, 2.6, 90.1ms, 0 | 20260508 07:26:47.110 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:26:47.110 | INFO | Uploaded 36GiB to GPU in 2.57587s (13.9759GiB/s) | 20260508 07:26:47.110 | INFO | Downloaded 38GiB to GPU in 2.69481s (14.1012GiB/s) | 20260508 07:26:52.245 | INFO | Analyzing chunk frames = [6304256,6304768) | 20260508 07:26:52.245 | INFO | MemInfo: { | 20260508 07:26:52.245 | INFO | "gpu" : " 17 GB", | 20260508 07:26:52.245 | INFO | "huge" : "", | 20260508 07:26:52.245 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:52.245 | INFO | "malloc" : " 14 GB", | 20260508 07:26:52.245 | INFO | "pinned" : " 29 GB" | 20260508 07:26:52.245 | INFO | } | 20260508 07:26:52.245 | INFO | Perf report for the last 5.13537 seconds: | 20260508 07:26:52.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:52.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.245 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 07:26:52.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.245 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 07:26:52.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:52.246 | INFO | Uploaded 36GiB to GPU in 2.54784s (14.1296GiB/s) | 20260508 07:26:52.246 | INFO | Downloaded 34GiB to GPU in 2.4398s (13.9356GiB/s) | 20260508 07:26:57.383 | INFO | Analyzing chunk frames = [6304768,6305280) | 20260508 07:26:57.383 | INFO | MemInfo: { | 20260508 07:26:57.383 | INFO | "gpu" : " 17 GB", | 20260508 07:26:57.383 | INFO | "huge" : "", | 20260508 07:26:57.383 | INFO | "huge_shared" : " 29 GB", | 20260508 07:26:57.383 | INFO | "malloc" : " 14 GB", | 20260508 07:26:57.383 | INFO | "pinned" : " 29 GB" | 20260508 07:26:57.383 | INFO | } | 20260508 07:26:57.383 | INFO | Perf report for the last 5.13819 seconds: | 20260508 07:26:57.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:26:57.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.383 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:26:57.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.383 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:26:57.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:26:57.384 | INFO | Uploaded 36GiB to GPU in 2.57643s (13.9728GiB/s) | 20260508 07:26:57.384 | INFO | Downloaded 34GiB to GPU in 2.42234s (14.036GiB/s) | 20260508 07:27:02.530 | INFO | Analyzing chunk frames = [6305280,6305792) | 20260508 07:27:02.530 | INFO | MemInfo: { | 20260508 07:27:02.530 | INFO | "gpu" : " 17 GB", | 20260508 07:27:02.530 | INFO | "huge" : "", | 20260508 07:27:02.530 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:02.530 | INFO | "malloc" : " 14 GB", | 20260508 07:27:02.530 | INFO | "pinned" : " 29 GB" | 20260508 07:27:02.530 | INFO | } | 20260508 07:27:02.530 | INFO | Perf report for the last 5.14644 seconds: | 20260508 07:27:02.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:02.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 07:27:02.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:27:02.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:02.530 | INFO | Uploaded 36GiB to GPU in 2.59161s (13.891GiB/s) | 20260508 07:27:02.530 | INFO | Downloaded 34GiB to GPU in 2.41018s (14.1068GiB/s) | 20260508 07:27:07.667 | INFO | Analyzing chunk frames = [6305792,6306304) | 20260508 07:27:07.667 | INFO | MemInfo: { | 20260508 07:27:07.667 | INFO | "gpu" : " 17 GB", | 20260508 07:27:07.667 | INFO | "huge" : "", | 20260508 07:27:07.667 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:07.667 | INFO | "malloc" : " 14 GB", | 20260508 07:27:07.667 | INFO | "pinned" : " 29 GB" | 20260508 07:27:07.667 | INFO | } | 20260508 07:27:07.667 | INFO | Perf report for the last 5.13735 seconds: | 20260508 07:27:07.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:07.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:27:07.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:27:07.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:07.667 | INFO | Uploaded 36GiB to GPU in 2.29704s (15.6724GiB/s) | 20260508 07:27:07.667 | INFO | Downloaded 34GiB to GPU in 2.49805s (13.6106GiB/s) | 20260508 07:27:12.805 | INFO | Analyzing chunk frames = [6306304,6306816) | 20260508 07:27:12.805 | INFO | MemInfo: { | 20260508 07:27:12.805 | INFO | "gpu" : " 17 GB", | 20260508 07:27:12.805 | INFO | "huge" : "", | 20260508 07:27:12.805 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:12.805 | INFO | "malloc" : " 14 GB", | 20260508 07:27:12.806 | INFO | "pinned" : " 29 GB" | 20260508 07:27:12.806 | INFO | } | 20260508 07:27:12.806 | INFO | Perf report for the last 5.13845 seconds: | 20260508 07:27:12.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:12.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 07:27:12.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:27:12.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:12.806 | INFO | Uploaded 36GiB to GPU in 2.41609s (14.9001GiB/s) | 20260508 07:27:12.806 | INFO | Downloaded 34GiB to GPU in 2.42925s (13.9961GiB/s) | 20260508 07:27:17.941 | INFO | Analyzing chunk frames = [6306816,6307328) | 20260508 07:27:17.941 | INFO | MemInfo: { | 20260508 07:27:17.941 | INFO | "gpu" : " 17 GB", | 20260508 07:27:17.941 | INFO | "huge" : "", | 20260508 07:27:17.941 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:17.941 | INFO | "malloc" : " 14 GB", | 20260508 07:27:17.941 | INFO | "pinned" : " 29 GB" | 20260508 07:27:17.941 | INFO | } | 20260508 07:27:17.941 | INFO | Perf report for the last 5.13529 seconds: | 20260508 07:27:17.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:17.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.941 | INFO | ANALYSIS : 65.7%, 3.0, 101.3ms, 0 | 20260508 07:27:17.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.941 | INFO | PRE_HQ : 23.9%, 1.0, 12.3ms, 0 | 20260508 07:27:17.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:17.942 | INFO | Uploaded 36GiB to GPU in 2.43453s (14.7873GiB/s) | 20260508 07:27:17.942 | INFO | Downloaded 34GiB to GPU in 2.42798s (14.0034GiB/s) | 20260508 07:27:23.081 | INFO | Analyzing chunk frames = [6307328,6307840) | 20260508 07:27:23.081 | INFO | MemInfo: { | 20260508 07:27:23.081 | INFO | "gpu" : " 17 GB", | 20260508 07:27:23.082 | INFO | "huge" : "", | 20260508 07:27:23.082 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:23.082 | INFO | "malloc" : " 14 GB", | 20260508 07:27:23.082 | INFO | "pinned" : " 29 GB" | 20260508 07:27:23.082 | INFO | } | 20260508 07:27:23.082 | INFO | Perf report for the last 5.14083 seconds: | 20260508 07:27:23.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:23.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 07:27:23.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:27:23.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:23.082 | INFO | Uploaded 36GiB to GPU in 2.51482s (14.3152GiB/s) | 20260508 07:27:23.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:27:23.082 | INFO | Total mem usage: 16.7 GiB | 20260508 07:27:23.082 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:27:23.082 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:27:23.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:27:23.082 | INFO | High Water Breakdown: | 20260508 07:27:23.082 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:27:23.082 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:27:23.082 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:27:23.082 | INFO | Downloaded 34GiB to GPU in 2.44751s (13.8917GiB/s) | 20260508 07:27:23.082 | INFO | ------------------------------------------ | 20260508 07:27:23.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:27:23.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:27:23.082 | INFO | Total mem usage: 26.5 GiB | 20260508 07:27:23.082 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:27:23.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:27:23.082 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:27:23.082 | INFO | High Water Breakdown: | 20260508 07:27:23.082 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:27:23.082 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:27:23.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:27:23.082 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:27:23.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:27:23.082 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:27:23.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:27:23.082 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:27:23.082 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:27:23.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:27:23.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:27:23.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:27:23.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:27:23.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:27:23.083 | INFO | ------------------------------------------ | 20260508 07:27:23.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:27:23.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:27:23.083 | INFO | Total mem usage: 27.8 GiB | 20260508 07:27:23.083 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:27:23.083 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:27:23.083 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:27:23.083 | INFO | High Water Breakdown: | 20260508 07:27:23.083 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:27:23.083 | INFO | ------------------------------------------ | 20260508 07:27:23.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:27:23.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:27:23.083 | INFO | Total mem usage: 15.7 GiB | 20260508 07:27:23.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:27:23.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:27:23.083 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 07:27:23.083 | INFO | High Water Breakdown: | 20260508 07:27:23.083 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:27:23.083 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:27:23.083 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:27:23.083 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:27:23.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:27:23.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:27:23.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:27:23.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:27:23.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:27:23.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:27:23.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:27:23.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:27:23.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:27:23.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:27:23.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:27:23.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:27:23.083 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:27:23.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:27:23.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:27:23.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:27:23.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:27:23.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:27:23.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:27:23.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:27:23.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:27:23.083 | INFO | ------------------------------------------ | 20260508 07:27:23.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:27:23.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:27:23.083 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:27:23.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:27:23.083 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.39e+01 / 7.78e-01 | 20260508 07:27:23.083 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.02e-01, 1.08e-02 / 6.71e-03 | 20260508 07:27:23.083 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.40e+00, 5.97e-01 / 2.09e-01 | 20260508 07:27:23.083 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.43e+00, 2.11e-01 / 1.43e-01 | 20260508 07:27:23.083 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.55e+01, 3.84e+00 / 1.49e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.83e+00 / 1.46e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.57e+00, 4.09e-01 / 2.69e-01 | 20260508 07:27:23.083 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.21e+00, 2.55e-01 / 1.62e-01 | 20260508 07:27:23.083 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.44e-01, 4.94e-02 / 2.44e-02 | 20260508 07:27:23.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:27:23.083 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:27:23.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:27:23.083 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 24.43%, Upload -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 24.29%, Download -- 2.57e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 30.89%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 2.39%, LockUpload -- 2.53e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 2.47%, LockDownload -- 2.61e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 15.52%, LockCompute -- 1.64e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 07:27:23.083 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260508 07:27:23.178 | INFO | Starting a round of Baz IO | 20260508 07:27:26.607 | INFO | Populating RTMetrics report | 20260508 07:27:26.607 | INFO | RTMetrics report populated | 20260508 07:27:26.607 | INFO | Opening tmp RTMetrics json file | 20260508 07:27:26.617 | INFO | Serializing RTMetrics to json | 20260508 07:27:26.619 | INFO | Serialization done, swapping tmp file now... | 20260508 07:27:26.619 | INFO | RTMetrics done | 20260508 07:27:26.675 | INFO | Finished a round of Baz IO: Wrote 1.528 GiB in 3.496 seconds | 20260508 07:27:26.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.00% / 59.45% | 20260508 07:27:26.675 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345182% / 0% / 4.37101e-05% | 20260508 07:27:26.675 | INFO | Baz IO overall: | 20260508 07:27:26.675 | INFO | BytesWritten : 2782841054320 | 20260508 07:27:26.675 | INFO | NumEvents : 1340746771816 | 20260508 07:27:28.220 | INFO | Analyzing chunk frames = [6307840,6308352) | 20260508 07:27:28.220 | INFO | MemInfo: { | 20260508 07:27:28.220 | INFO | "gpu" : " 17 GB", | 20260508 07:27:28.220 | INFO | "huge" : "", | 20260508 07:27:28.220 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:28.220 | INFO | "malloc" : " 14 GB", | 20260508 07:27:28.220 | INFO | "pinned" : " 29 GB" | 20260508 07:27:28.220 | INFO | } | 20260508 07:27:28.220 | INFO | Perf report for the last 5.13852 seconds: | 20260508 07:27:28.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:28.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:28.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:28.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:28.220 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:27:28.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:28.220 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 07:27:28.220 | INFO | PRE_HQ : 66.1%, 1.7, 59.9ms, 0 | 20260508 07:27:28.220 | INFO | BAZWRITER : 67.5%, 2.7, 94.8ms, 0 | 20260508 07:27:28.220 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:27:28.221 | INFO | Uploaded 36GiB to GPU in 2.59586s (13.8682GiB/s) | 20260508 07:27:28.221 | INFO | Downloaded 38GiB to GPU in 2.69409s (14.1049GiB/s) | 20260508 07:27:33.363 | INFO | Analyzing chunk frames = [6308352,6308864) | 20260508 07:27:33.363 | INFO | MemInfo: { | 20260508 07:27:33.363 | INFO | "gpu" : " 17 GB", | 20260508 07:27:33.363 | INFO | "huge" : "", | 20260508 07:27:33.363 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:33.363 | INFO | "malloc" : " 14 GB", | 20260508 07:27:33.363 | INFO | "pinned" : " 29 GB" | 20260508 07:27:33.363 | INFO | } | 20260508 07:27:33.363 | INFO | Perf report for the last 5.14282 seconds: | 20260508 07:27:33.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:33.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 07:27:33.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:27:33.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:33.363 | INFO | Uploaded 36GiB to GPU in 2.50574s (14.367GiB/s) | 20260508 07:27:33.363 | INFO | Downloaded 34GiB to GPU in 2.43499s (13.9631GiB/s) | 20260508 07:27:38.501 | INFO | Analyzing chunk frames = [6308864,6309376) | 20260508 07:27:38.501 | INFO | MemInfo: { | 20260508 07:27:38.501 | INFO | "gpu" : " 17 GB", | 20260508 07:27:38.501 | INFO | "huge" : "", | 20260508 07:27:38.501 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:38.501 | INFO | "malloc" : " 14 GB", | 20260508 07:27:38.501 | INFO | "pinned" : " 29 GB" | 20260508 07:27:38.501 | INFO | } | 20260508 07:27:38.501 | INFO | Perf report for the last 5.13826 seconds: | 20260508 07:27:38.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:38.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.501 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 07:27:38.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.501 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:27:38.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:38.502 | INFO | Uploaded 36GiB to GPU in 2.57891s (13.9594GiB/s) | 20260508 07:27:38.502 | INFO | Downloaded 34GiB to GPU in 2.43191s (13.9808GiB/s) | 20260508 07:27:43.636 | INFO | Analyzing chunk frames = [6309376,6309888) | 20260508 07:27:43.636 | INFO | MemInfo: { | 20260508 07:27:43.636 | INFO | "gpu" : " 17 GB", | 20260508 07:27:43.636 | INFO | "huge" : "", | 20260508 07:27:43.636 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:43.636 | INFO | "malloc" : " 14 GB", | 20260508 07:27:43.636 | INFO | "pinned" : " 29 GB" | 20260508 07:27:43.636 | INFO | } | 20260508 07:27:43.636 | INFO | Perf report for the last 5.13486 seconds: | 20260508 07:27:43.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:43.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.636 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 07:27:43.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.636 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:27:43.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:43.637 | INFO | Uploaded 36GiB to GPU in 2.55781s (14.0746GiB/s) | 20260508 07:27:43.637 | INFO | Downloaded 34GiB to GPU in 2.40665s (14.1275GiB/s) | 20260508 07:27:48.776 | INFO | Analyzing chunk frames = [6309888,6310400) | 20260508 07:27:48.776 | INFO | MemInfo: { | 20260508 07:27:48.776 | INFO | "gpu" : " 17 GB", | 20260508 07:27:48.776 | INFO | "huge" : "", | 20260508 07:27:48.776 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:48.776 | INFO | "malloc" : " 14 GB", | 20260508 07:27:48.776 | INFO | "pinned" : " 29 GB" | 20260508 07:27:48.777 | INFO | } | 20260508 07:27:48.777 | INFO | Perf report for the last 5.14032 seconds: | 20260508 07:27:48.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:48.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:27:48.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:27:48.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:48.777 | INFO | Uploaded 36GiB to GPU in 2.28447s (15.7586GiB/s) | 20260508 07:27:48.777 | INFO | Downloaded 34GiB to GPU in 2.4927s (13.6398GiB/s) | 20260508 07:27:53.920 | INFO | Analyzing chunk frames = [6310400,6310912) | 20260508 07:27:53.920 | INFO | MemInfo: { | 20260508 07:27:53.920 | INFO | "gpu" : " 17 GB", | 20260508 07:27:53.920 | INFO | "huge" : "", | 20260508 07:27:53.920 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:53.920 | INFO | "malloc" : " 14 GB", | 20260508 07:27:53.920 | INFO | "pinned" : " 29 GB" | 20260508 07:27:53.920 | INFO | } | 20260508 07:27:53.920 | INFO | Perf report for the last 5.14392 seconds: | 20260508 07:27:53.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:53.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:27:53.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:27:53.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:53.921 | INFO | Uploaded 36GiB to GPU in 2.39758s (15.0151GiB/s) | 20260508 07:27:53.921 | INFO | Downloaded 34GiB to GPU in 2.41701s (14.0669GiB/s) | 20260508 07:27:59.060 | INFO | Analyzing chunk frames = [6310912,6311424) | 20260508 07:27:59.060 | INFO | MemInfo: { | 20260508 07:27:59.060 | INFO | "gpu" : " 17 GB", | 20260508 07:27:59.060 | INFO | "huge" : "", | 20260508 07:27:59.060 | INFO | "huge_shared" : " 29 GB", | 20260508 07:27:59.060 | INFO | "malloc" : " 14 GB", | 20260508 07:27:59.060 | INFO | "pinned" : " 29 GB" | 20260508 07:27:59.060 | INFO | } | 20260508 07:27:59.060 | INFO | Perf report for the last 5.13943 seconds: | 20260508 07:27:59.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:27:59.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.060 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 07:27:59.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.060 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:27:59.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:27:59.061 | INFO | Uploaded 36GiB to GPU in 2.44742s (14.7094GiB/s) | 20260508 07:27:59.061 | INFO | Downloaded 34GiB to GPU in 2.42276s (14.0336GiB/s) | 20260508 07:28:04.193 | INFO | Analyzing chunk frames = [6311424,6311936) | 20260508 07:28:04.193 | INFO | MemInfo: { | 20260508 07:28:04.194 | INFO | "gpu" : " 17 GB", | 20260508 07:28:04.194 | INFO | "huge" : "", | 20260508 07:28:04.194 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:04.194 | INFO | "malloc" : " 14 GB", | 20260508 07:28:04.194 | INFO | "pinned" : " 29 GB" | 20260508 07:28:04.194 | INFO | } | 20260508 07:28:04.194 | INFO | Perf report for the last 5.13377 seconds: | 20260508 07:28:04.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:04.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 07:28:04.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:28:04.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:04.194 | INFO | Uploaded 36GiB to GPU in 2.52277s (14.27GiB/s) | 20260508 07:28:04.194 | INFO | Downloaded 34GiB to GPU in 2.44895s (13.8835GiB/s) | 20260508 07:28:04.291 | INFO | Starting a round of Baz IO | 20260508 07:28:07.710 | INFO | Populating RTMetrics report | 20260508 07:28:07.710 | INFO | RTMetrics report populated | 20260508 07:28:07.710 | INFO | Opening tmp RTMetrics json file | 20260508 07:28:07.721 | INFO | Serializing RTMetrics to json | 20260508 07:28:07.722 | INFO | Serialization done, swapping tmp file now... | 20260508 07:28:07.722 | INFO | RTMetrics done | 20260508 07:28:07.784 | INFO | Finished a round of Baz IO: Wrote 1.528 GiB in 3.493 seconds | 20260508 07:28:07.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.00% / 59.45% | 20260508 07:28:07.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.343796% / 3.97364e-06% / 4.37101e-05% | 20260508 07:28:07.784 | INFO | Baz IO overall: | 20260508 07:28:07.784 | INFO | BytesWritten : 2784481400201 | 20260508 07:28:07.784 | INFO | NumEvents : 1341525993698 | 20260508 07:28:09.342 | INFO | Analyzing chunk frames = [6311936,6312448) | 20260508 07:28:09.342 | INFO | MemInfo: { | 20260508 07:28:09.342 | INFO | "gpu" : " 17 GB", | 20260508 07:28:09.342 | INFO | "huge" : "", | 20260508 07:28:09.342 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:09.342 | INFO | "malloc" : " 14 GB", | 20260508 07:28:09.342 | INFO | "pinned" : " 29 GB" | 20260508 07:28:09.342 | INFO | } | 20260508 07:28:09.342 | INFO | Perf report for the last 5.14871 seconds: | 20260508 07:28:09.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:09.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:09.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:09.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:09.343 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 07:28:09.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:09.343 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:28:09.343 | INFO | PRE_HQ : 65.3%, 1.6, 54.8ms, 0 | 20260508 07:28:09.343 | INFO | BAZWRITER : 67.3%, 2.6, 90.8ms, 0 | 20260508 07:28:09.343 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:28:09.343 | INFO | Uploaded 36GiB to GPU in 2.60773s (13.8051GiB/s) | 20260508 07:28:09.343 | INFO | Downloaded 38GiB to GPU in 2.69667s (14.0914GiB/s) | 20260508 07:28:14.474 | INFO | Analyzing chunk frames = [6312448,6312960) | 20260508 07:28:14.474 | INFO | MemInfo: { | 20260508 07:28:14.474 | INFO | "gpu" : " 17 GB", | 20260508 07:28:14.475 | INFO | "huge" : "", | 20260508 07:28:14.475 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:14.475 | INFO | "malloc" : " 14 GB", | 20260508 07:28:14.475 | INFO | "pinned" : " 29 GB" | 20260508 07:28:14.475 | INFO | } | 20260508 07:28:14.475 | INFO | Perf report for the last 5.13226 seconds: | 20260508 07:28:14.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:14.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 07:28:14.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:28:14.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:14.475 | INFO | Uploaded 36GiB to GPU in 2.54832s (14.127GiB/s) | 20260508 07:28:14.475 | INFO | Downloaded 34GiB to GPU in 2.44896s (13.8834GiB/s) | 20260508 07:28:19.614 | INFO | Analyzing chunk frames = [6312960,6313472) | 20260508 07:28:19.614 | INFO | MemInfo: { | 20260508 07:28:19.614 | INFO | "gpu" : " 17 GB", | 20260508 07:28:19.614 | INFO | "huge" : "", | 20260508 07:28:19.614 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:19.614 | INFO | "malloc" : " 14 GB", | 20260508 07:28:19.614 | INFO | "pinned" : " 29 GB" | 20260508 07:28:19.614 | INFO | } | 20260508 07:28:19.614 | INFO | Perf report for the last 5.13924 seconds: | 20260508 07:28:19.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:19.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:28:19.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:28:19.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:19.614 | INFO | Uploaded 36GiB to GPU in 2.55963s (14.0645GiB/s) | 20260508 07:28:19.614 | INFO | Downloaded 34GiB to GPU in 2.42256s (14.0347GiB/s) | 20260508 07:28:24.750 | INFO | Analyzing chunk frames = [6313472,6313984) | 20260508 07:28:24.750 | INFO | MemInfo: { | 20260508 07:28:24.750 | INFO | "gpu" : " 17 GB", | 20260508 07:28:24.750 | INFO | "huge" : "", | 20260508 07:28:24.750 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:24.750 | INFO | "malloc" : " 14 GB", | 20260508 07:28:24.750 | INFO | "pinned" : " 29 GB" | 20260508 07:28:24.750 | INFO | } | 20260508 07:28:24.750 | INFO | Perf report for the last 5.13627 seconds: | 20260508 07:28:24.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:24.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.750 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 07:28:24.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.750 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:28:24.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:24.751 | INFO | Uploaded 36GiB to GPU in 2.57961s (13.9556GiB/s) | 20260508 07:28:24.751 | INFO | Downloaded 34GiB to GPU in 2.41517s (14.0777GiB/s) | 20260508 07:28:29.891 | INFO | Analyzing chunk frames = [6313984,6314496) | 20260508 07:28:29.891 | INFO | MemInfo: { | 20260508 07:28:29.891 | INFO | "gpu" : " 17 GB", | 20260508 07:28:29.891 | INFO | "huge" : "", | 20260508 07:28:29.891 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:29.891 | INFO | "malloc" : " 14 GB", | 20260508 07:28:29.891 | INFO | "pinned" : " 29 GB" | 20260508 07:28:29.891 | INFO | } | 20260508 07:28:29.891 | INFO | Perf report for the last 5.14076 seconds: | 20260508 07:28:29.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:29.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:28:29.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 07:28:29.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:29.891 | INFO | Uploaded 36GiB to GPU in 2.2975s (15.6692GiB/s) | 20260508 07:28:29.891 | INFO | Downloaded 34GiB to GPU in 2.49761s (13.613GiB/s) | 20260508 07:28:35.034 | INFO | Analyzing chunk frames = [6314496,6315008) | 20260508 07:28:35.034 | INFO | MemInfo: { | 20260508 07:28:35.034 | INFO | "gpu" : " 17 GB", | 20260508 07:28:35.034 | INFO | "huge" : "", | 20260508 07:28:35.034 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:35.034 | INFO | "malloc" : " 14 GB", | 20260508 07:28:35.034 | INFO | "pinned" : " 29 GB" | 20260508 07:28:35.034 | INFO | } | 20260508 07:28:35.034 | INFO | Perf report for the last 5.14292 seconds: | 20260508 07:28:35.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:35.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:28:35.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:28:35.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:35.034 | INFO | Uploaded 36GiB to GPU in 2.37568s (15.1535GiB/s) | 20260508 07:28:35.034 | INFO | Downloaded 34GiB to GPU in 2.43553s (13.96GiB/s) | 20260508 07:28:40.174 | INFO | Analyzing chunk frames = [6315008,6315520) | 20260508 07:28:40.174 | INFO | MemInfo: { | 20260508 07:28:40.174 | INFO | "gpu" : " 17 GB", | 20260508 07:28:40.174 | INFO | "huge" : "", | 20260508 07:28:40.174 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:40.174 | INFO | "malloc" : " 14 GB", | 20260508 07:28:40.175 | INFO | "pinned" : " 29 GB" | 20260508 07:28:40.175 | INFO | } | 20260508 07:28:40.175 | INFO | Perf report for the last 5.14066 seconds: | 20260508 07:28:40.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:40.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:28:40.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:28:40.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:40.175 | INFO | Uploaded 36GiB to GPU in 2.48181s (14.5055GiB/s) | 20260508 07:28:40.175 | INFO | Downloaded 34GiB to GPU in 2.44227s (13.9214GiB/s) | 20260508 07:28:45.310 | INFO | Analyzing chunk frames = [6315520,6316032) | 20260508 07:28:45.310 | INFO | MemInfo: { | 20260508 07:28:45.310 | INFO | "gpu" : " 17 GB", | 20260508 07:28:45.311 | INFO | "huge" : "", | 20260508 07:28:45.311 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:45.311 | INFO | "malloc" : " 14 GB", | 20260508 07:28:45.311 | INFO | "pinned" : " 29 GB" | 20260508 07:28:45.311 | INFO | } | 20260508 07:28:45.311 | INFO | Perf report for the last 5.13601 seconds: | 20260508 07:28:45.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:45.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 07:28:45.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:28:45.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:45.311 | INFO | Uploaded 36GiB to GPU in 2.43208s (14.8022GiB/s) | 20260508 07:28:45.311 | INFO | Downloaded 34GiB to GPU in 2.453s (13.8606GiB/s) | 20260508 07:28:45.311 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:28:45.311 | INFO | Total mem usage: 16.7 GiB | 20260508 07:28:45.311 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:28:45.311 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:28:45.311 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:28:45.311 | INFO | High Water Breakdown: | 20260508 07:28:45.311 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:28:45.311 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:28:45.311 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:28:45.311 | INFO | ------------------------------------------ | 20260508 07:28:45.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:28:45.311 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:28:45.311 | INFO | Total mem usage: 26.5 GiB | 20260508 07:28:45.311 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:28:45.311 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:28:45.311 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:28:45.311 | INFO | High Water Breakdown: | 20260508 07:28:45.311 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:28:45.311 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:28:45.311 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:28:45.311 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:28:45.311 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:28:45.311 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:28:45.311 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:28:45.311 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:28:45.311 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:28:45.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:28:45.311 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:28:45.311 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:28:45.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:28:45.311 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:28:45.311 | INFO | ------------------------------------------ | 20260508 07:28:45.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:28:45.311 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:28:45.311 | INFO | Total mem usage: 27.8 GiB | 20260508 07:28:45.312 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:28:45.312 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:28:45.312 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:28:45.312 | INFO | High Water Breakdown: | 20260508 07:28:45.312 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:28:45.312 | INFO | ------------------------------------------ | 20260508 07:28:45.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:28:45.312 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:28:45.312 | INFO | Total mem usage: 15.7 GiB | 20260508 07:28:45.312 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:28:45.312 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:28:45.312 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:28:45.312 | INFO | High Water Breakdown: | 20260508 07:28:45.312 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:28:45.312 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:28:45.312 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:28:45.312 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:28:45.312 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:28:45.312 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:28:45.312 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:28:45.312 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:28:45.312 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:28:45.312 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:28:45.312 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:28:45.312 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:28:45.312 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:28:45.312 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:28:45.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:28:45.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:28:45.312 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:28:45.312 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:28:45.312 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:28:45.312 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:28:45.312 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:28:45.312 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:28:45.312 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:28:45.312 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:28:45.312 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:28:45.312 | INFO | ------------------------------------------ | 20260508 07:28:45.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:28:45.312 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:28:45.312 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:28:45.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:28:45.312 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.74e-01 | 20260508 07:28:45.312 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.04e-01, 1.09e-02 / 6.26e-03 | 20260508 07:28:45.312 | INFO | PerfReport: 12.11%, Baseline -- 3.96e+00 / 4.41e+00, 5.87e-01 / 2.21e-01 | 20260508 07:28:45.312 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.22e-01 | 20260508 07:28:45.312 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.54e+01, 3.89e+00 / 9.07e-01 | 20260508 07:28:45.312 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.82e+00 / 1.20e+00 | 20260508 07:28:45.312 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.58e+00, 4.15e-01 / 2.09e-01 | 20260508 07:28:45.312 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.18e+00, 2.53e-01 / 1.26e-01 | 20260508 07:28:45.312 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.46e-01, 5.06e-02 / 2.61e-02 | 20260508 07:28:45.313 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:28:45.313 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:28:45.313 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:28:45.313 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 24.46%, Upload -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 24.33%, Download -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 30.89%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 2.42%, LockUpload -- 2.56e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 2.39%, LockDownload -- 2.53e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 15.51%, LockCompute -- 1.64e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 07:28:45.313 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260508 07:28:45.408 | INFO | Starting a round of Baz IO | 20260508 07:28:48.820 | INFO | Populating RTMetrics report | 20260508 07:28:48.820 | INFO | RTMetrics report populated | 20260508 07:28:48.820 | INFO | Opening tmp RTMetrics json file | 20260508 07:28:48.831 | INFO | Serializing RTMetrics to json | 20260508 07:28:48.832 | INFO | Serialization done, swapping tmp file now... | 20260508 07:28:48.833 | INFO | RTMetrics done | 20260508 07:28:48.896 | INFO | Finished a round of Baz IO: Wrote 1.528 GiB in 3.487 seconds | 20260508 07:28:48.896 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.00% / 59.45% | 20260508 07:28:48.896 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.343513% / 0% / 5.16574e-05% | 20260508 07:28:48.896 | INFO | Baz IO overall: | 20260508 07:28:48.896 | INFO | BytesWritten : 2786122368360 | 20260508 07:28:48.896 | INFO | NumEvents : 1342305807023 | 20260508 07:28:50.447 | INFO | Analyzing chunk frames = [6316032,6316544) | 20260508 07:28:50.447 | INFO | MemInfo: { | 20260508 07:28:50.447 | INFO | "gpu" : " 17 GB", | 20260508 07:28:50.447 | INFO | "huge" : "", | 20260508 07:28:50.447 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:50.447 | INFO | "malloc" : " 14 GB", | 20260508 07:28:50.447 | INFO | "pinned" : " 29 GB" | 20260508 07:28:50.447 | INFO | } | 20260508 07:28:50.447 | INFO | Perf report for the last 5.13628 seconds: | 20260508 07:28:50.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:50.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:50.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:50.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:50.447 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 07:28:50.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:50.447 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:28:50.447 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 07:28:50.447 | INFO | BAZWRITER : 67.4%, 2.6, 90.6ms, 0 | 20260508 07:28:50.447 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:28:50.447 | INFO | Uploaded 36GiB to GPU in 2.56676s (14.0255GiB/s) | 20260508 07:28:50.447 | INFO | Downloaded 38GiB to GPU in 2.70239s (14.0616GiB/s) | 20260508 07:28:55.592 | INFO | Analyzing chunk frames = [6316544,6317056) | 20260508 07:28:55.592 | INFO | MemInfo: { | 20260508 07:28:55.592 | INFO | "gpu" : " 17 GB", | 20260508 07:28:55.592 | INFO | "huge" : "", | 20260508 07:28:55.592 | INFO | "huge_shared" : " 29 GB", | 20260508 07:28:55.592 | INFO | "malloc" : " 14 GB", | 20260508 07:28:55.592 | INFO | "pinned" : " 29 GB" | 20260508 07:28:55.592 | INFO | } | 20260508 07:28:55.592 | INFO | Perf report for the last 5.14546 seconds: | 20260508 07:28:55.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:28:55.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.592 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 07:28:55.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.593 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:28:55.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:28:55.593 | INFO | Uploaded 36GiB to GPU in 2.49049s (14.455GiB/s) | 20260508 07:28:55.593 | INFO | Downloaded 34GiB to GPU in 2.43475s (13.9645GiB/s) | 20260508 07:29:00.732 | INFO | Analyzing chunk frames = [6317056,6317568) | 20260508 07:29:00.732 | INFO | MemInfo: { | 20260508 07:29:00.732 | INFO | "gpu" : " 17 GB", | 20260508 07:29:00.733 | INFO | "huge" : "", | 20260508 07:29:00.733 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:00.733 | INFO | "malloc" : " 14 GB", | 20260508 07:29:00.733 | INFO | "pinned" : " 29 GB" | 20260508 07:29:00.733 | INFO | } | 20260508 07:29:00.733 | INFO | Perf report for the last 5.14022 seconds: | 20260508 07:29:00.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:00.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 07:29:00.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:29:00.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:00.733 | INFO | Uploaded 36GiB to GPU in 2.49052s (14.4548GiB/s) | 20260508 07:29:00.733 | INFO | Downloaded 34GiB to GPU in 2.43697s (13.9517GiB/s) | 20260508 07:29:05.868 | INFO | Analyzing chunk frames = [6317568,6318080) | 20260508 07:29:05.868 | INFO | MemInfo: { | 20260508 07:29:05.868 | INFO | "gpu" : " 17 GB", | 20260508 07:29:05.868 | INFO | "huge" : "", | 20260508 07:29:05.868 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:05.868 | INFO | "malloc" : " 14 GB", | 20260508 07:29:05.868 | INFO | "pinned" : " 29 GB" | 20260508 07:29:05.868 | INFO | } | 20260508 07:29:05.868 | INFO | Perf report for the last 5.1355 seconds: | 20260508 07:29:05.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:05.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.868 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 07:29:05.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.868 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260508 07:29:05.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:05.869 | INFO | Uploaded 36GiB to GPU in 2.5718s (13.998GiB/s) | 20260508 07:29:05.869 | INFO | Downloaded 34GiB to GPU in 2.42688s (14.0098GiB/s) | 20260508 07:29:11.012 | INFO | Analyzing chunk frames = [6318080,6318592) | 20260508 07:29:11.012 | INFO | MemInfo: { | 20260508 07:29:11.012 | INFO | "gpu" : " 17 GB", | 20260508 07:29:11.012 | INFO | "huge" : "", | 20260508 07:29:11.012 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:11.012 | INFO | "malloc" : " 14 GB", | 20260508 07:29:11.012 | INFO | "pinned" : " 29 GB" | 20260508 07:29:11.012 | INFO | } | 20260508 07:29:11.012 | INFO | Perf report for the last 5.14383 seconds: | 20260508 07:29:11.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:11.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.012 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 07:29:11.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.012 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 07:29:11.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:11.013 | INFO | Uploaded 36GiB to GPU in 2.29046s (15.7174GiB/s) | 20260508 07:29:11.013 | INFO | Downloaded 34GiB to GPU in 2.49941s (13.6032GiB/s) | 20260508 07:29:16.143 | INFO | Analyzing chunk frames = [6318592,6319104) | 20260508 07:29:16.143 | INFO | MemInfo: { | 20260508 07:29:16.143 | INFO | "gpu" : " 17 GB", | 20260508 07:29:16.143 | INFO | "huge" : "", | 20260508 07:29:16.143 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:16.143 | INFO | "malloc" : " 14 GB", | 20260508 07:29:16.143 | INFO | "pinned" : " 29 GB" | 20260508 07:29:16.143 | INFO | } | 20260508 07:29:16.143 | INFO | Perf report for the last 5.1311 seconds: | 20260508 07:29:16.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:16.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 07:29:16.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:29:16.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:16.143 | INFO | Uploaded 36GiB to GPU in 2.38473s (15.0961GiB/s) | 20260508 07:29:16.143 | INFO | Downloaded 34GiB to GPU in 2.42571s (14.0165GiB/s) | 20260508 07:29:21.286 | INFO | Analyzing chunk frames = [6319104,6319616) | 20260508 07:29:21.286 | INFO | MemInfo: { | 20260508 07:29:21.286 | INFO | "gpu" : " 17 GB", | 20260508 07:29:21.286 | INFO | "huge" : "", | 20260508 07:29:21.286 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:21.286 | INFO | "malloc" : " 14 GB", | 20260508 07:29:21.286 | INFO | "pinned" : " 29 GB" | 20260508 07:29:21.286 | INFO | } | 20260508 07:29:21.286 | INFO | Perf report for the last 5.1431 seconds: | 20260508 07:29:21.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:21.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.286 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:29:21.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.286 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:29:21.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:21.287 | INFO | Uploaded 36GiB to GPU in 2.48449s (14.4899GiB/s) | 20260508 07:29:21.287 | INFO | Downloaded 34GiB to GPU in 2.4286s (13.9998GiB/s) | 20260508 07:29:26.431 | INFO | Analyzing chunk frames = [6319616,6320128) | 20260508 07:29:26.431 | INFO | MemInfo: { | 20260508 07:29:26.431 | INFO | "gpu" : " 17 GB", | 20260508 07:29:26.431 | INFO | "huge" : "", | 20260508 07:29:26.431 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:26.431 | INFO | "malloc" : " 14 GB", | 20260508 07:29:26.431 | INFO | "pinned" : " 29 GB" | 20260508 07:29:26.431 | INFO | } | 20260508 07:29:26.432 | INFO | Perf report for the last 5.14528 seconds: | 20260508 07:29:26.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:26.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:29:26.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:29:26.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:26.432 | INFO | Uploaded 36GiB to GPU in 2.50535s (14.3692GiB/s) | 20260508 07:29:26.432 | INFO | Downloaded 34GiB to GPU in 2.4476s (13.8912GiB/s) | 20260508 07:29:26.529 | INFO | Starting a round of Baz IO | 20260508 07:29:29.949 | INFO | Populating RTMetrics report | 20260508 07:29:29.949 | INFO | RTMetrics report populated | 20260508 07:29:29.949 | INFO | Opening tmp RTMetrics json file | 20260508 07:29:29.957 | INFO | Serializing RTMetrics to json | 20260508 07:29:29.959 | INFO | Serialization done, swapping tmp file now... | 20260508 07:29:29.959 | INFO | RTMetrics done | 20260508 07:29:30.016 | INFO | Finished a round of Baz IO: Wrote 1.527 GiB in 3.487 seconds | 20260508 07:29:30.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.00% / 59.45% | 20260508 07:29:30.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.343188% / 3.97364e-06% / 3.17891e-05% | 20260508 07:29:30.017 | INFO | Baz IO overall: | 20260508 07:29:30.017 | INFO | BytesWritten : 2787762170223 | 20260508 07:29:30.017 | INFO | NumEvents : 1343085028366 | 20260508 07:29:31.564 | INFO | Analyzing chunk frames = [6320128,6320640) | 20260508 07:29:31.564 | INFO | MemInfo: { | 20260508 07:29:31.564 | INFO | "gpu" : " 17 GB", | 20260508 07:29:31.564 | INFO | "huge" : "", | 20260508 07:29:31.564 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:31.564 | INFO | "malloc" : " 14 GB", | 20260508 07:29:31.564 | INFO | "pinned" : " 29 GB" | 20260508 07:29:31.564 | INFO | } | 20260508 07:29:31.564 | INFO | Perf report for the last 5.13285 seconds: | 20260508 07:29:31.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:31.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:31.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:31.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:31.564 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 07:29:31.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:31.564 | INFO | RT_METRICS : 39.4%, 1.0, 20.6ms, 0 | 20260508 07:29:31.565 | INFO | PRE_HQ : 65.6%, 1.6, 55.0ms, 0 | 20260508 07:29:31.565 | INFO | BAZWRITER : 67.3%, 2.6, 89.7ms, 0 | 20260508 07:29:31.565 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:29:31.565 | INFO | Uploaded 36GiB to GPU in 2.61329s (13.7757GiB/s) | 20260508 07:29:31.565 | INFO | Downloaded 38GiB to GPU in 2.69434s (14.1036GiB/s) | 20260508 07:29:36.699 | INFO | Analyzing chunk frames = [6320640,6321152) | 20260508 07:29:36.699 | INFO | MemInfo: { | 20260508 07:29:36.699 | INFO | "gpu" : " 17 GB", | 20260508 07:29:36.699 | INFO | "huge" : "", | 20260508 07:29:36.699 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:36.699 | INFO | "malloc" : " 14 GB", | 20260508 07:29:36.699 | INFO | "pinned" : " 29 GB" | 20260508 07:29:36.699 | INFO | } | 20260508 07:29:36.699 | INFO | Perf report for the last 5.13497 seconds: | 20260508 07:29:36.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:36.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.699 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:29:36.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.699 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:29:36.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:36.700 | INFO | Uploaded 36GiB to GPU in 2.53173s (14.2195GiB/s) | 20260508 07:29:36.700 | INFO | Downloaded 34GiB to GPU in 2.43848s (13.9431GiB/s) | 20260508 07:29:41.846 | INFO | Analyzing chunk frames = [6321152,6321664) | 20260508 07:29:41.846 | INFO | MemInfo: { | 20260508 07:29:41.846 | INFO | "gpu" : " 17 GB", | 20260508 07:29:41.846 | INFO | "huge" : "", | 20260508 07:29:41.846 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:41.846 | INFO | "malloc" : " 14 GB", | 20260508 07:29:41.846 | INFO | "pinned" : " 29 GB" | 20260508 07:29:41.846 | INFO | } | 20260508 07:29:41.846 | INFO | Perf report for the last 5.14668 seconds: | 20260508 07:29:41.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:41.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.846 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:29:41.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.846 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260508 07:29:41.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:41.847 | INFO | Uploaded 36GiB to GPU in 2.56039s (14.0603GiB/s) | 20260508 07:29:41.847 | INFO | Downloaded 34GiB to GPU in 2.41965s (14.0516GiB/s) | 20260508 07:29:46.984 | INFO | Analyzing chunk frames = [6321664,6322176) | 20260508 07:29:46.984 | INFO | MemInfo: { | 20260508 07:29:46.984 | INFO | "gpu" : " 17 GB", | 20260508 07:29:46.985 | INFO | "huge" : "", | 20260508 07:29:46.985 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:46.985 | INFO | "malloc" : " 14 GB", | 20260508 07:29:46.985 | INFO | "pinned" : " 29 GB" | 20260508 07:29:46.985 | INFO | } | 20260508 07:29:46.985 | INFO | Perf report for the last 5.13867 seconds: | 20260508 07:29:46.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:46.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 07:29:46.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:29:46.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:46.985 | INFO | Uploaded 36GiB to GPU in 2.58529s (13.9249GiB/s) | 20260508 07:29:46.985 | INFO | Downloaded 34GiB to GPU in 2.42855s (14.0001GiB/s) | 20260508 07:29:52.128 | INFO | Analyzing chunk frames = [6322176,6322688) | 20260508 07:29:52.128 | INFO | MemInfo: { | 20260508 07:29:52.128 | INFO | "gpu" : " 17 GB", | 20260508 07:29:52.128 | INFO | "huge" : "", | 20260508 07:29:52.128 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:52.128 | INFO | "malloc" : " 14 GB", | 20260508 07:29:52.128 | INFO | "pinned" : " 29 GB" | 20260508 07:29:52.128 | INFO | } | 20260508 07:29:52.128 | INFO | Perf report for the last 5.14358 seconds: | 20260508 07:29:52.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:52.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.128 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:29:52.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.128 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:29:52.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:52.129 | INFO | Uploaded 36GiB to GPU in 2.39547s (15.0283GiB/s) | 20260508 07:29:52.129 | INFO | Downloaded 34GiB to GPU in 2.52497s (13.4655GiB/s) | 20260508 07:29:57.262 | INFO | Analyzing chunk frames = [6322688,6323200) | 20260508 07:29:57.262 | INFO | MemInfo: { | 20260508 07:29:57.262 | INFO | "gpu" : " 17 GB", | 20260508 07:29:57.262 | INFO | "huge" : "", | 20260508 07:29:57.262 | INFO | "huge_shared" : " 29 GB", | 20260508 07:29:57.262 | INFO | "malloc" : " 14 GB", | 20260508 07:29:57.262 | INFO | "pinned" : " 29 GB" | 20260508 07:29:57.262 | INFO | } | 20260508 07:29:57.262 | INFO | Perf report for the last 5.13395 seconds: | 20260508 07:29:57.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:29:57.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.262 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:29:57.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.262 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:29:57.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:29:57.263 | INFO | Uploaded 36GiB to GPU in 2.37022s (15.1884GiB/s) | 20260508 07:29:57.263 | INFO | Downloaded 34GiB to GPU in 2.42986s (13.9926GiB/s) | 20260508 07:30:02.404 | INFO | Analyzing chunk frames = [6323200,6323712) | 20260508 07:30:02.404 | INFO | MemInfo: { | 20260508 07:30:02.404 | INFO | "gpu" : " 17 GB", | 20260508 07:30:02.404 | INFO | "huge" : "", | 20260508 07:30:02.404 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:02.404 | INFO | "malloc" : " 14 GB", | 20260508 07:30:02.404 | INFO | "pinned" : " 29 GB" | 20260508 07:30:02.404 | INFO | } | 20260508 07:30:02.405 | INFO | Perf report for the last 5.14219 seconds: | 20260508 07:30:02.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:02.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 07:30:02.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:30:02.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:02.405 | INFO | Uploaded 36GiB to GPU in 2.47224s (14.5617GiB/s) | 20260508 07:30:02.405 | INFO | Downloaded 34GiB to GPU in 2.43022s (13.9905GiB/s) | 20260508 07:30:07.541 | INFO | Analyzing chunk frames = [6323712,6324224) | 20260508 07:30:07.541 | INFO | MemInfo: { | 20260508 07:30:07.541 | INFO | "gpu" : " 17 GB", | 20260508 07:30:07.541 | INFO | "huge" : "", | 20260508 07:30:07.541 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:07.541 | INFO | "malloc" : " 14 GB", | 20260508 07:30:07.541 | INFO | "pinned" : " 29 GB" | 20260508 07:30:07.541 | INFO | } | 20260508 07:30:07.541 | INFO | Perf report for the last 5.13653 seconds: | 20260508 07:30:07.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:07.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 07:30:07.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:30:07.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:07.541 | INFO | Uploaded 36GiB to GPU in 2.44528s (14.7223GiB/s) | 20260508 07:30:07.541 | INFO | Downloaded 34GiB to GPU in 2.45202s (13.8661GiB/s) | 20260508 07:30:07.542 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:30:07.542 | INFO | Total mem usage: 16.7 GiB | 20260508 07:30:07.542 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:30:07.542 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:30:07.542 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:30:07.542 | INFO | High Water Breakdown: | 20260508 07:30:07.542 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:30:07.542 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:30:07.542 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:30:07.542 | INFO | ------------------------------------------ | 20260508 07:30:07.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:30:07.542 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:30:07.542 | INFO | Total mem usage: 26.5 GiB | 20260508 07:30:07.542 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:30:07.542 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:30:07.542 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:30:07.542 | INFO | High Water Breakdown: | 20260508 07:30:07.542 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:30:07.542 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:30:07.542 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:30:07.542 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:30:07.542 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:30:07.542 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:30:07.542 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:30:07.542 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:30:07.542 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:30:07.542 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:30:07.542 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:30:07.542 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:30:07.542 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:30:07.542 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:30:07.542 | INFO | ------------------------------------------ | 20260508 07:30:07.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:30:07.542 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:30:07.542 | INFO | Total mem usage: 27.8 GiB | 20260508 07:30:07.542 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:30:07.542 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:30:07.542 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:30:07.542 | INFO | High Water Breakdown: | 20260508 07:30:07.542 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:30:07.542 | INFO | ------------------------------------------ | 20260508 07:30:07.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:30:07.542 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:30:07.542 | INFO | Total mem usage: 15.7 GiB | 20260508 07:30:07.542 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:30:07.542 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:30:07.542 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:30:07.542 | INFO | High Water Breakdown: | 20260508 07:30:07.542 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:30:07.542 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:30:07.542 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:30:07.542 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:30:07.542 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:30:07.542 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:30:07.542 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:30:07.542 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:30:07.542 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:30:07.542 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:30:07.542 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:30:07.542 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:30:07.542 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:30:07.542 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:30:07.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:30:07.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:30:07.542 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:30:07.542 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:30:07.542 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:30:07.542 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:30:07.542 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:30:07.542 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:30:07.542 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:30:07.542 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:30:07.542 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:30:07.543 | INFO | ------------------------------------------ | 20260508 07:30:07.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:30:07.543 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:30:07.543 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:30:07.543 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:30:07.543 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.88e-01 | 20260508 07:30:07.543 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.28e-02 / 3.66e-03 | 20260508 07:30:07.543 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.39e+00, 5.88e-01 / 2.10e-01 | 20260508 07:30:07.543 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 1.11e-01 | 20260508 07:30:07.543 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.51e+01, 3.68e+00 / 1.24e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 1.54e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.36e+00 / 2.52e+00, 4.09e-01 / 2.57e-01 | 20260508 07:30:07.543 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.20e+00, 2.54e-01 / 1.09e-01 | 20260508 07:30:07.543 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.09e-01 / 3.40e-01, 5.01e-02 / 2.34e-02 | 20260508 07:30:07.543 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:30:07.543 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:30:07.543 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:30:07.543 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 24.44%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 24.36%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 30.90%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 2.31%, LockUpload -- 2.45e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 2.44%, LockDownload -- 2.58e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 15.53%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:30:07.543 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260508 07:30:07.639 | INFO | Starting a round of Baz IO | 20260508 07:30:11.066 | INFO | Populating RTMetrics report | 20260508 07:30:11.067 | INFO | RTMetrics report populated | 20260508 07:30:11.067 | INFO | Opening tmp RTMetrics json file | 20260508 07:30:11.068 | INFO | Serializing RTMetrics to json | 20260508 07:30:11.069 | INFO | Serialization done, swapping tmp file now... | 20260508 07:30:11.069 | INFO | RTMetrics done | 20260508 07:30:11.137 | INFO | Finished a round of Baz IO: Wrote 1.529 GiB in 3.497 seconds | 20260508 07:30:11.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.00% / 59.45% | 20260508 07:30:11.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.343454% / 0% / 4.76837e-05% | 20260508 07:30:11.137 | INFO | Baz IO overall: | 20260508 07:30:11.138 | INFO | BytesWritten : 2789404143433 | 20260508 07:30:11.138 | INFO | NumEvents : 1343865878318 | 20260508 07:30:12.680 | INFO | Analyzing chunk frames = [6324224,6324736) | 20260508 07:30:12.680 | INFO | MemInfo: { | 20260508 07:30:12.680 | INFO | "gpu" : " 17 GB", | 20260508 07:30:12.680 | INFO | "huge" : "", | 20260508 07:30:12.680 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:12.680 | INFO | "malloc" : " 14 GB", | 20260508 07:30:12.680 | INFO | "pinned" : " 29 GB" | 20260508 07:30:12.680 | INFO | } | 20260508 07:30:12.680 | INFO | Perf report for the last 5.13925 seconds: | 20260508 07:30:12.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:12.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:12.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:12.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:12.680 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 07:30:12.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:12.680 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:30:12.680 | INFO | PRE_HQ : 66.0%, 1.6, 54.8ms, 0 | 20260508 07:30:12.680 | INFO | BAZWRITER : 67.4%, 2.6, 89.5ms, 0 | 20260508 07:30:12.680 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:30:12.681 | INFO | Uploaded 36GiB to GPU in 2.55788s (14.0742GiB/s) | 20260508 07:30:12.681 | INFO | Downloaded 38GiB to GPU in 2.71913s (13.975GiB/s) | 20260508 07:30:17.816 | INFO | Analyzing chunk frames = [6324736,6325248) | 20260508 07:30:17.816 | INFO | MemInfo: { | 20260508 07:30:17.816 | INFO | "gpu" : " 17 GB", | 20260508 07:30:17.816 | INFO | "huge" : "", | 20260508 07:30:17.816 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:17.816 | INFO | "malloc" : " 14 GB", | 20260508 07:30:17.817 | INFO | "pinned" : " 29 GB" | 20260508 07:30:17.817 | INFO | } | 20260508 07:30:17.817 | INFO | Perf report for the last 5.13628 seconds: | 20260508 07:30:17.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:17.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:30:17.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260508 07:30:17.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:17.817 | INFO | Uploaded 36GiB to GPU in 2.49225s (14.4448GiB/s) | 20260508 07:30:17.817 | INFO | Downloaded 34GiB to GPU in 2.43753s (13.9486GiB/s) | 20260508 07:30:22.956 | INFO | Analyzing chunk frames = [6325248,6325760) | 20260508 07:30:22.956 | INFO | MemInfo: { | 20260508 07:30:22.956 | INFO | "gpu" : " 17 GB", | 20260508 07:30:22.956 | INFO | "huge" : "", | 20260508 07:30:22.956 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:22.956 | INFO | "malloc" : " 14 GB", | 20260508 07:30:22.956 | INFO | "pinned" : " 29 GB" | 20260508 07:30:22.956 | INFO | } | 20260508 07:30:22.957 | INFO | Perf report for the last 5.13988 seconds: | 20260508 07:30:22.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:22.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:30:22.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 07:30:22.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:22.957 | INFO | Uploaded 36GiB to GPU in 2.56402s (14.0404GiB/s) | 20260508 07:30:22.957 | INFO | Downloaded 34GiB to GPU in 2.42892s (13.998GiB/s) | 20260508 07:30:28.099 | INFO | Analyzing chunk frames = [6325760,6326272) | 20260508 07:30:28.099 | INFO | MemInfo: { | 20260508 07:30:28.099 | INFO | "gpu" : " 17 GB", | 20260508 07:30:28.099 | INFO | "huge" : "", | 20260508 07:30:28.099 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:28.099 | INFO | "malloc" : " 14 GB", | 20260508 07:30:28.099 | INFO | "pinned" : " 29 GB" | 20260508 07:30:28.099 | INFO | } | 20260508 07:30:28.099 | INFO | Perf report for the last 5.14269 seconds: | 20260508 07:30:28.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:28.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.099 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:30:28.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.100 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 07:30:28.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:28.100 | INFO | Uploaded 36GiB to GPU in 2.57561s (13.9773GiB/s) | 20260508 07:30:28.100 | INFO | Downloaded 34GiB to GPU in 2.42698s (14.0092GiB/s) | 20260508 07:30:33.240 | INFO | Analyzing chunk frames = [6326272,6326784) | 20260508 07:30:33.240 | INFO | MemInfo: { | 20260508 07:30:33.240 | INFO | "gpu" : " 17 GB", | 20260508 07:30:33.241 | INFO | "huge" : "", | 20260508 07:30:33.241 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:33.241 | INFO | "malloc" : " 14 GB", | 20260508 07:30:33.241 | INFO | "pinned" : " 29 GB" | 20260508 07:30:33.241 | INFO | } | 20260508 07:30:33.241 | INFO | Perf report for the last 5.1414 seconds: | 20260508 07:30:33.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:33.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 07:30:33.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260508 07:30:33.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:33.241 | INFO | Uploaded 36GiB to GPU in 2.39566s (15.0272GiB/s) | 20260508 07:30:33.241 | INFO | Downloaded 34GiB to GPU in 2.52336s (13.4741GiB/s) | 20260508 07:30:38.377 | INFO | Analyzing chunk frames = [6326784,6327296) | 20260508 07:30:38.377 | INFO | MemInfo: { | 20260508 07:30:38.377 | INFO | "gpu" : " 17 GB", | 20260508 07:30:38.377 | INFO | "huge" : "", | 20260508 07:30:38.377 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:38.377 | INFO | "malloc" : " 14 GB", | 20260508 07:30:38.377 | INFO | "pinned" : " 29 GB" | 20260508 07:30:38.377 | INFO | } | 20260508 07:30:38.377 | INFO | Perf report for the last 5.13678 seconds: | 20260508 07:30:38.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:38.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.378 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 07:30:38.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.378 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:30:38.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:38.378 | INFO | Uploaded 36GiB to GPU in 2.39275s (15.0455GiB/s) | 20260508 07:30:38.378 | INFO | Downloaded 34GiB to GPU in 2.4277s (14.005GiB/s) | 20260508 07:30:43.515 | INFO | Analyzing chunk frames = [6327296,6327808) | 20260508 07:30:43.515 | INFO | MemInfo: { | 20260508 07:30:43.515 | INFO | "gpu" : " 17 GB", | 20260508 07:30:43.515 | INFO | "huge" : "", | 20260508 07:30:43.515 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:43.515 | INFO | "malloc" : " 14 GB", | 20260508 07:30:43.515 | INFO | "pinned" : " 29 GB" | 20260508 07:30:43.515 | INFO | } | 20260508 07:30:43.516 | INFO | Perf report for the last 5.13808 seconds: | 20260508 07:30:43.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:43.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:30:43.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:30:43.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:43.516 | INFO | Uploaded 36GiB to GPU in 2.49438s (14.4324GiB/s) | 20260508 07:30:43.516 | INFO | Downloaded 34GiB to GPU in 2.42715s (14.0082GiB/s) | 20260508 07:30:48.653 | INFO | Analyzing chunk frames = [6327808,6328320) | 20260508 07:30:48.654 | INFO | MemInfo: { | 20260508 07:30:48.654 | INFO | "gpu" : " 17 GB", | 20260508 07:30:48.654 | INFO | "huge" : "", | 20260508 07:30:48.654 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:48.654 | INFO | "malloc" : " 14 GB", | 20260508 07:30:48.654 | INFO | "pinned" : " 29 GB" | 20260508 07:30:48.654 | INFO | } | 20260508 07:30:48.654 | INFO | Perf report for the last 5.13824 seconds: | 20260508 07:30:48.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:48.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:30:48.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 07:30:48.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:48.654 | INFO | Uploaded 36GiB to GPU in 2.50664s (14.3619GiB/s) | 20260508 07:30:48.654 | INFO | Downloaded 34GiB to GPU in 2.44247s (13.9203GiB/s) | 20260508 07:30:48.751 | INFO | Starting a round of Baz IO | 20260508 07:30:52.169 | INFO | Populating RTMetrics report | 20260508 07:30:52.169 | INFO | RTMetrics report populated | 20260508 07:30:52.169 | INFO | Opening tmp RTMetrics json file | 20260508 07:30:52.181 | INFO | Serializing RTMetrics to json | 20260508 07:30:52.182 | INFO | Serialization done, swapping tmp file now... | 20260508 07:30:52.182 | INFO | RTMetrics done | 20260508 07:30:52.243 | INFO | Finished a round of Baz IO: Wrote 1.526 GiB in 3.492 seconds | 20260508 07:30:52.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.54% / 0.00% / 59.45% | 20260508 07:30:52.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.3407% / 3.97364e-06% / 1.58946e-05% | 20260508 07:30:52.243 | INFO | Baz IO overall: | 20260508 07:30:52.243 | INFO | BytesWritten : 2791043167187 | 20260508 07:30:52.243 | INFO | NumEvents : 1344643919104 | 20260508 07:30:53.802 | INFO | Analyzing chunk frames = [6328320,6328832) | 20260508 07:30:53.802 | INFO | MemInfo: { | 20260508 07:30:53.802 | INFO | "gpu" : " 17 GB", | 20260508 07:30:53.802 | INFO | "huge" : "", | 20260508 07:30:53.802 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:53.802 | INFO | "malloc" : " 14 GB", | 20260508 07:30:53.802 | INFO | "pinned" : " 29 GB" | 20260508 07:30:53.802 | INFO | } | 20260508 07:30:53.803 | INFO | Perf report for the last 5.14873 seconds: | 20260508 07:30:53.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:53.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:53.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:53.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:53.803 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 07:30:53.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:53.803 | INFO | RT_METRICS : 39.1%, 1.0, 20.6ms, 0 | 20260508 07:30:53.803 | INFO | PRE_HQ : 65.3%, 1.6, 54.9ms, 0 | 20260508 07:30:53.803 | INFO | BAZWRITER : 67.3%, 2.6, 90.4ms, 0 | 20260508 07:30:53.803 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:30:53.803 | INFO | Uploaded 36GiB to GPU in 2.60279s (13.8313GiB/s) | 20260508 07:30:53.803 | INFO | Downloaded 38GiB to GPU in 2.69095s (14.1214GiB/s) | 20260508 07:30:58.941 | INFO | Analyzing chunk frames = [6328832,6329344) | 20260508 07:30:58.941 | INFO | MemInfo: { | 20260508 07:30:58.941 | INFO | "gpu" : " 17 GB", | 20260508 07:30:58.941 | INFO | "huge" : "", | 20260508 07:30:58.941 | INFO | "huge_shared" : " 29 GB", | 20260508 07:30:58.941 | INFO | "malloc" : " 14 GB", | 20260508 07:30:58.941 | INFO | "pinned" : " 29 GB" | 20260508 07:30:58.941 | INFO | } | 20260508 07:30:58.941 | INFO | Perf report for the last 5.13839 seconds: | 20260508 07:30:58.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:30:58.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:30:58.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:30:58.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:30:58.941 | INFO | Uploaded 36GiB to GPU in 2.53907s (14.1784GiB/s) | 20260508 07:30:58.941 | INFO | Downloaded 34GiB to GPU in 2.45652s (13.8407GiB/s) | 20260508 07:31:04.080 | INFO | Analyzing chunk frames = [6329344,6329856) | 20260508 07:31:04.080 | INFO | MemInfo: { | 20260508 07:31:04.080 | INFO | "gpu" : " 17 GB", | 20260508 07:31:04.080 | INFO | "huge" : "", | 20260508 07:31:04.080 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:04.080 | INFO | "malloc" : " 14 GB", | 20260508 07:31:04.080 | INFO | "pinned" : " 29 GB" | 20260508 07:31:04.080 | INFO | } | 20260508 07:31:04.080 | INFO | Perf report for the last 5.13919 seconds: | 20260508 07:31:04.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:04.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 07:31:04.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:31:04.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:04.080 | INFO | Uploaded 36GiB to GPU in 2.51498s (14.3142GiB/s) | 20260508 07:31:04.081 | INFO | Downloaded 34GiB to GPU in 2.44013s (13.9337GiB/s) | 20260508 07:31:09.210 | INFO | Analyzing chunk frames = [6329856,6330368) | 20260508 07:31:09.210 | INFO | MemInfo: { | 20260508 07:31:09.210 | INFO | "gpu" : " 17 GB", | 20260508 07:31:09.210 | INFO | "huge" : "", | 20260508 07:31:09.210 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:09.210 | INFO | "malloc" : " 14 GB", | 20260508 07:31:09.210 | INFO | "pinned" : " 29 GB" | 20260508 07:31:09.210 | INFO | } | 20260508 07:31:09.210 | INFO | Perf report for the last 5.13016 seconds: | 20260508 07:31:09.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:09.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.210 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 07:31:09.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.211 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 07:31:09.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:09.211 | INFO | Uploaded 36GiB to GPU in 2.53933s (14.177GiB/s) | 20260508 07:31:09.211 | INFO | Downloaded 34GiB to GPU in 2.43765s (13.9479GiB/s) | 20260508 07:31:14.352 | INFO | Analyzing chunk frames = [6330368,6330880) | 20260508 07:31:14.352 | INFO | MemInfo: { | 20260508 07:31:14.352 | INFO | "gpu" : " 17 GB", | 20260508 07:31:14.352 | INFO | "huge" : "", | 20260508 07:31:14.352 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:14.352 | INFO | "malloc" : " 14 GB", | 20260508 07:31:14.352 | INFO | "pinned" : " 29 GB" | 20260508 07:31:14.352 | INFO | } | 20260508 07:31:14.352 | INFO | Perf report for the last 5.14185 seconds: | 20260508 07:31:14.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:14.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:31:14.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260508 07:31:14.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:14.352 | INFO | Uploaded 36GiB to GPU in 2.27017s (15.8579GiB/s) | 20260508 07:31:14.353 | INFO | Downloaded 34GiB to GPU in 2.49765s (13.6128GiB/s) | 20260508 07:31:19.491 | INFO | Analyzing chunk frames = [6330880,6331392) | 20260508 07:31:19.491 | INFO | MemInfo: { | 20260508 07:31:19.491 | INFO | "gpu" : " 17 GB", | 20260508 07:31:19.491 | INFO | "huge" : "", | 20260508 07:31:19.491 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:19.491 | INFO | "malloc" : " 14 GB", | 20260508 07:31:19.491 | INFO | "pinned" : " 29 GB" | 20260508 07:31:19.491 | INFO | } | 20260508 07:31:19.492 | INFO | Perf report for the last 5.13938 seconds: | 20260508 07:31:19.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:19.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:31:19.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:31:19.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:19.492 | INFO | Uploaded 36GiB to GPU in 2.45589s (14.6586GiB/s) | 20260508 07:31:19.492 | INFO | Downloaded 34GiB to GPU in 2.42009s (14.049GiB/s) | 20260508 07:31:24.633 | INFO | Analyzing chunk frames = [6331392,6331904) | 20260508 07:31:24.633 | INFO | MemInfo: { | 20260508 07:31:24.633 | INFO | "gpu" : " 17 GB", | 20260508 07:31:24.633 | INFO | "huge" : "", | 20260508 07:31:24.633 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:24.633 | INFO | "malloc" : " 14 GB", | 20260508 07:31:24.633 | INFO | "pinned" : " 29 GB" | 20260508 07:31:24.633 | INFO | } | 20260508 07:31:24.633 | INFO | Perf report for the last 5.14168 seconds: | 20260508 07:31:24.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:24.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.633 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 07:31:24.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.633 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:31:24.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:24.634 | INFO | Uploaded 36GiB to GPU in 2.50936s (14.3463GiB/s) | 20260508 07:31:24.634 | INFO | Downloaded 34GiB to GPU in 2.437s (13.9516GiB/s) | 20260508 07:31:29.769 | INFO | Analyzing chunk frames = [6331904,6332416) | 20260508 07:31:29.769 | INFO | MemInfo: { | 20260508 07:31:29.769 | INFO | "gpu" : " 17 GB", | 20260508 07:31:29.769 | INFO | "huge" : "", | 20260508 07:31:29.769 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:29.769 | INFO | "malloc" : " 14 GB", | 20260508 07:31:29.769 | INFO | "pinned" : " 29 GB" | 20260508 07:31:29.769 | INFO | } | 20260508 07:31:29.770 | INFO | Perf report for the last 5.13629 seconds: | 20260508 07:31:29.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:29.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 07:31:29.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:31:29.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:29.770 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:31:29.770 | INFO | Total mem usage: 16.7 GiB | 20260508 07:31:29.770 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:31:29.770 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:31:29.770 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:31:29.770 | INFO | High Water Breakdown: | 20260508 07:31:29.770 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:31:29.770 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:31:29.770 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:31:29.770 | INFO | ------------------------------------------ | 20260508 07:31:29.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:31:29.770 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:31:29.770 | INFO | Total mem usage: 26.5 GiB | 20260508 07:31:29.770 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:31:29.770 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:31:29.770 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:31:29.770 | INFO | High Water Breakdown: | 20260508 07:31:29.770 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:31:29.770 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:31:29.770 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:31:29.770 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:31:29.770 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:31:29.770 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:31:29.770 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:31:29.770 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:31:29.770 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:31:29.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:31:29.770 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:31:29.770 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:31:29.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:31:29.770 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:31:29.770 | INFO | ------------------------------------------ | 20260508 07:31:29.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:31:29.770 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:31:29.770 | INFO | Total mem usage: 27.8 GiB | 20260508 07:31:29.770 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:31:29.770 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:31:29.770 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:31:29.770 | INFO | High Water Breakdown: | 20260508 07:31:29.770 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:31:29.770 | INFO | ------------------------------------------ | 20260508 07:31:29.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:31:29.770 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:31:29.770 | INFO | Total mem usage: 15.7 GiB | 20260508 07:31:29.770 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:31:29.770 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:31:29.770 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:31:29.770 | INFO | High Water Breakdown: | 20260508 07:31:29.770 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:31:29.770 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:31:29.770 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:31:29.770 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:31:29.770 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:31:29.770 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:31:29.770 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:31:29.770 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:31:29.770 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:31:29.770 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:31:29.770 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:31:29.770 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:31:29.770 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:31:29.770 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:31:29.770 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:31:29.770 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:31:29.770 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:31:29.770 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:31:29.770 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:31:29.770 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:31:29.770 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:31:29.770 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:31:29.770 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:31:29.770 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:31:29.770 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:31:29.770 | INFO | ------------------------------------------ | 20260508 07:31:29.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:31:29.770 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:31:29.770 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:31:29.770 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:31:29.770 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.41e+01 / 6.96e-01 | 20260508 07:31:29.770 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.07e-01, 1.13e-02 / 5.97e-03 | 20260508 07:31:29.770 | INFO | PerfReport: 12.12%, Baseline -- 3.97e+00 / 4.46e+00, 5.81e-01 / 2.21e-01 | 20260508 07:31:29.770 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.14e-01 | 20260508 07:31:29.770 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.53e+01, 3.70e+00 / 1.09e+00 | 20260508 07:31:29.770 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.84e+00 / 1.28e+00 | 20260508 07:31:29.770 | INFO | Uploaded 36GiB to GPU in 2.46298s (14.6164GiB/s) | 20260508 07:31:29.770 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.57e+00, 4.14e-01 / 2.53e-01 | 20260508 07:31:29.771 | INFO | Downloaded 34GiB to GPU in 2.44361s (13.9138GiB/s) | 20260508 07:31:29.771 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.19e+00, 2.52e-01 / 1.24e-01 | 20260508 07:31:29.771 | INFO | PerfReport: 0.96%, PulseSerializing -- 3.13e-01 / 3.49e-01, 5.01e-02 / 2.91e-02 | 20260508 07:31:29.771 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:31:29.771 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:31:29.771 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:31:29.771 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 24.47%, Upload -- 2.60e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 30.88%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 2.38%, LockUpload -- 2.52e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 2.39%, LockDownload -- 2.53e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 15.52%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:31:29.771 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260508 07:31:29.868 | INFO | Starting a round of Baz IO | 20260508 07:31:33.281 | INFO | Populating RTMetrics report | 20260508 07:31:33.282 | INFO | RTMetrics report populated | 20260508 07:31:33.282 | INFO | Opening tmp RTMetrics json file | 20260508 07:31:33.293 | INFO | Serializing RTMetrics to json | 20260508 07:31:33.294 | INFO | Serialization done, swapping tmp file now... | 20260508 07:31:33.295 | INFO | RTMetrics done | 20260508 07:31:33.354 | INFO | Finished a round of Baz IO: Wrote 1.527 GiB in 3.486 seconds | 20260508 07:31:33.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.54% / 0.00% / 59.46% | 20260508 07:31:33.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.340704% / 0% / 3.97364e-05% | 20260508 07:31:33.354 | INFO | Baz IO overall: | 20260508 07:31:33.354 | INFO | BytesWritten : 2792682796996 | 20260508 07:31:33.354 | INFO | NumEvents : 1345422488158 | 20260508 07:31:34.909 | INFO | Analyzing chunk frames = [6332416,6332928) | 20260508 07:31:34.909 | INFO | MemInfo: { | 20260508 07:31:34.909 | INFO | "gpu" : " 17 GB", | 20260508 07:31:34.909 | INFO | "huge" : "", | 20260508 07:31:34.909 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:34.909 | INFO | "malloc" : " 14 GB", | 20260508 07:31:34.909 | INFO | "pinned" : " 29 GB" | 20260508 07:31:34.909 | INFO | } | 20260508 07:31:34.909 | INFO | Perf report for the last 5.13959 seconds: | 20260508 07:31:34.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:34.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:34.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:34.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:34.909 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:31:34.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:34.909 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 07:31:34.909 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 07:31:34.909 | INFO | BAZWRITER : 67.2%, 2.6, 90.5ms, 0 | 20260508 07:31:34.909 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:31:34.910 | INFO | Uploaded 36GiB to GPU in 2.60617s (13.8134GiB/s) | 20260508 07:31:34.910 | INFO | Downloaded 38GiB to GPU in 2.70258s (14.0606GiB/s) | 20260508 07:31:40.050 | INFO | Analyzing chunk frames = [6332928,6333440) | 20260508 07:31:40.050 | INFO | MemInfo: { | 20260508 07:31:40.050 | INFO | "gpu" : " 17 GB", | 20260508 07:31:40.050 | INFO | "huge" : "", | 20260508 07:31:40.051 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:40.051 | INFO | "malloc" : " 14 GB", | 20260508 07:31:40.051 | INFO | "pinned" : " 29 GB" | 20260508 07:31:40.051 | INFO | } | 20260508 07:31:40.051 | INFO | Perf report for the last 5.14152 seconds: | 20260508 07:31:40.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:40.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | ANALYSIS : 67.1%, 3.0, 104.6ms, 0 | 20260508 07:31:40.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:31:40.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:40.051 | INFO | Uploaded 36GiB to GPU in 2.53484s (14.2021GiB/s) | 20260508 07:31:40.051 | INFO | Downloaded 34GiB to GPU in 2.44433s (13.9098GiB/s) | 20260508 07:31:45.191 | INFO | Analyzing chunk frames = [6333440,6333952) | 20260508 07:31:45.191 | INFO | MemInfo: { | 20260508 07:31:45.191 | INFO | "gpu" : " 17 GB", | 20260508 07:31:45.191 | INFO | "huge" : "", | 20260508 07:31:45.192 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:45.192 | INFO | "malloc" : " 14 GB", | 20260508 07:31:45.192 | INFO | "pinned" : " 29 GB" | 20260508 07:31:45.192 | INFO | } | 20260508 07:31:45.192 | INFO | Perf report for the last 5.14104 seconds: | 20260508 07:31:45.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:45.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:31:45.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:31:45.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:45.192 | INFO | Uploaded 36GiB to GPU in 2.57372s (13.9876GiB/s) | 20260508 07:31:45.192 | INFO | Downloaded 34GiB to GPU in 2.42877s (13.9989GiB/s) | 20260508 07:31:50.329 | INFO | Analyzing chunk frames = [6333952,6334464) | 20260508 07:31:50.330 | INFO | MemInfo: { | 20260508 07:31:50.330 | INFO | "gpu" : " 17 GB", | 20260508 07:31:50.330 | INFO | "huge" : "", | 20260508 07:31:50.330 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:50.330 | INFO | "malloc" : " 14 GB", | 20260508 07:31:50.330 | INFO | "pinned" : " 29 GB" | 20260508 07:31:50.330 | INFO | } | 20260508 07:31:50.330 | INFO | Perf report for the last 5.13798 seconds: | 20260508 07:31:50.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:50.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 07:31:50.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 07:31:50.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:50.330 | INFO | Uploaded 36GiB to GPU in 2.59284s (13.8844GiB/s) | 20260508 07:31:50.330 | INFO | Downloaded 34GiB to GPU in 2.42407s (14.026GiB/s) | 20260508 07:31:55.465 | INFO | Analyzing chunk frames = [6334464,6334976) | 20260508 07:31:55.465 | INFO | MemInfo: { | 20260508 07:31:55.465 | INFO | "gpu" : " 17 GB", | 20260508 07:31:55.465 | INFO | "huge" : "", | 20260508 07:31:55.465 | INFO | "huge_shared" : " 29 GB", | 20260508 07:31:55.465 | INFO | "malloc" : " 14 GB", | 20260508 07:31:55.465 | INFO | "pinned" : " 29 GB" | 20260508 07:31:55.465 | INFO | } | 20260508 07:31:55.465 | INFO | Perf report for the last 5.13561 seconds: | 20260508 07:31:55.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:31:55.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.465 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 07:31:55.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.466 | INFO | PRE_HQ : 30.6%, 1.0, 16.0ms, 0 | 20260508 07:31:55.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:31:55.466 | INFO | Uploaded 36GiB to GPU in 2.30161s (15.6413GiB/s) | 20260508 07:31:55.466 | INFO | Downloaded 34GiB to GPU in 2.49266s (13.64GiB/s) | 20260508 07:32:00.607 | INFO | Analyzing chunk frames = [6334976,6335488) | 20260508 07:32:00.607 | INFO | MemInfo: { | 20260508 07:32:00.607 | INFO | "gpu" : " 17 GB", | 20260508 07:32:00.607 | INFO | "huge" : "", | 20260508 07:32:00.607 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:00.607 | INFO | "malloc" : " 14 GB", | 20260508 07:32:00.607 | INFO | "pinned" : " 29 GB" | 20260508 07:32:00.607 | INFO | } | 20260508 07:32:00.607 | INFO | Perf report for the last 5.14172 seconds: | 20260508 07:32:00.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:00.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:32:00.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:32:00.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:00.607 | INFO | Uploaded 36GiB to GPU in 2.38656s (15.0845GiB/s) | 20260508 07:32:00.607 | INFO | Downloaded 34GiB to GPU in 2.42842s (14.0009GiB/s) | 20260508 07:32:05.745 | INFO | Analyzing chunk frames = [6335488,6336000) | 20260508 07:32:05.745 | INFO | MemInfo: { | 20260508 07:32:05.745 | INFO | "gpu" : " 17 GB", | 20260508 07:32:05.745 | INFO | "huge" : "", | 20260508 07:32:05.745 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:05.745 | INFO | "malloc" : " 14 GB", | 20260508 07:32:05.745 | INFO | "pinned" : " 29 GB" | 20260508 07:32:05.745 | INFO | } | 20260508 07:32:05.745 | INFO | Perf report for the last 5.13801 seconds: | 20260508 07:32:05.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:05.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.745 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 07:32:05.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.745 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:32:05.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:05.746 | INFO | Uploaded 36GiB to GPU in 2.45183s (14.6829GiB/s) | 20260508 07:32:05.746 | INFO | Downloaded 34GiB to GPU in 2.42906s (13.9972GiB/s) | 20260508 07:32:10.888 | INFO | Analyzing chunk frames = [6336000,6336512) | 20260508 07:32:10.888 | INFO | MemInfo: { | 20260508 07:32:10.888 | INFO | "gpu" : " 17 GB", | 20260508 07:32:10.888 | INFO | "huge" : "", | 20260508 07:32:10.888 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:10.888 | INFO | "malloc" : " 14 GB", | 20260508 07:32:10.888 | INFO | "pinned" : " 29 GB" | 20260508 07:32:10.888 | INFO | } | 20260508 07:32:10.888 | INFO | Perf report for the last 5.14327 seconds: | 20260508 07:32:10.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:10.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.889 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:32:10.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.889 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:32:10.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:10.889 | INFO | Uploaded 36GiB to GPU in 2.48175s (14.5059GiB/s) | 20260508 07:32:10.889 | INFO | Downloaded 34GiB to GPU in 2.44036s (13.9324GiB/s) | 20260508 07:32:10.985 | INFO | Starting a round of Baz IO | 20260508 07:32:14.412 | INFO | Populating RTMetrics report | 20260508 07:32:14.413 | INFO | RTMetrics report populated | 20260508 07:32:14.413 | INFO | Opening tmp RTMetrics json file | 20260508 07:32:14.422 | INFO | Serializing RTMetrics to json | 20260508 07:32:14.423 | INFO | Serialization done, swapping tmp file now... | 20260508 07:32:14.424 | INFO | RTMetrics done | 20260508 07:32:14.483 | INFO | Finished a round of Baz IO: Wrote 1.527 GiB in 3.497 seconds | 20260508 07:32:14.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.54% / 0.00% / 59.46% | 20260508 07:32:14.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.340792% / 7.94729e-06% / 7.94729e-05% | 20260508 07:32:14.483 | INFO | Baz IO overall: | 20260508 07:32:14.483 | INFO | BytesWritten : 2794322612828 | 20260508 07:32:14.483 | INFO | NumEvents : 1346201366238 | 20260508 07:32:16.024 | INFO | Analyzing chunk frames = [6336512,6337024) | 20260508 07:32:16.024 | INFO | MemInfo: { | 20260508 07:32:16.024 | INFO | "gpu" : " 17 GB", | 20260508 07:32:16.024 | INFO | "huge" : "", | 20260508 07:32:16.024 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:16.024 | INFO | "malloc" : " 14 GB", | 20260508 07:32:16.024 | INFO | "pinned" : " 29 GB" | 20260508 07:32:16.024 | INFO | } | 20260508 07:32:16.024 | INFO | Perf report for the last 5.13539 seconds: | 20260508 07:32:16.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:16.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:16.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:16.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:16.024 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 07:32:16.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:16.024 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:32:16.024 | INFO | PRE_HQ : 65.8%, 1.6, 54.9ms, 0 | 20260508 07:32:16.024 | INFO | BAZWRITER : 67.5%, 2.7, 92.0ms, 0 | 20260508 07:32:16.024 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:32:16.024 | INFO | Uploaded 36GiB to GPU in 2.55255s (14.1035GiB/s) | 20260508 07:32:16.025 | INFO | Downloaded 38GiB to GPU in 2.70743s (14.0355GiB/s) | 20260508 07:32:21.162 | INFO | Analyzing chunk frames = [6337024,6337536) | 20260508 07:32:21.162 | INFO | MemInfo: { | 20260508 07:32:21.162 | INFO | "gpu" : " 17 GB", | 20260508 07:32:21.162 | INFO | "huge" : "", | 20260508 07:32:21.162 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:21.162 | INFO | "malloc" : " 14 GB", | 20260508 07:32:21.162 | INFO | "pinned" : " 29 GB" | 20260508 07:32:21.162 | INFO | } | 20260508 07:32:21.162 | INFO | Perf report for the last 5.13845 seconds: | 20260508 07:32:21.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:21.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.162 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:32:21.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.162 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 07:32:21.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:21.163 | INFO | Uploaded 36GiB to GPU in 2.48337s (14.4965GiB/s) | 20260508 07:32:21.163 | INFO | Downloaded 34GiB to GPU in 2.43884s (13.9411GiB/s) | 20260508 07:32:26.302 | INFO | Analyzing chunk frames = [6337536,6338048) | 20260508 07:32:26.302 | INFO | MemInfo: { | 20260508 07:32:26.302 | INFO | "gpu" : " 17 GB", | 20260508 07:32:26.303 | INFO | "huge" : "", | 20260508 07:32:26.303 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:26.303 | INFO | "malloc" : " 14 GB", | 20260508 07:32:26.303 | INFO | "pinned" : " 29 GB" | 20260508 07:32:26.303 | INFO | } | 20260508 07:32:26.303 | INFO | Perf report for the last 5.14051 seconds: | 20260508 07:32:26.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:26.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:32:26.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:32:26.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:26.303 | INFO | Uploaded 36GiB to GPU in 2.54853s (14.1258GiB/s) | 20260508 07:32:26.303 | INFO | Downloaded 34GiB to GPU in 2.43273s (13.976GiB/s) | 20260508 07:32:31.439 | INFO | Analyzing chunk frames = [6338048,6338560) | 20260508 07:32:31.439 | INFO | MemInfo: { | 20260508 07:32:31.439 | INFO | "gpu" : " 17 GB", | 20260508 07:32:31.439 | INFO | "huge" : "", | 20260508 07:32:31.439 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:31.439 | INFO | "malloc" : " 14 GB", | 20260508 07:32:31.439 | INFO | "pinned" : " 29 GB" | 20260508 07:32:31.439 | INFO | } | 20260508 07:32:31.440 | INFO | Perf report for the last 5.13672 seconds: | 20260508 07:32:31.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:31.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 07:32:31.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260508 07:32:31.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:31.440 | INFO | Uploaded 36GiB to GPU in 2.55117s (14.1111GiB/s) | 20260508 07:32:31.440 | INFO | Downloaded 34GiB to GPU in 2.43331s (13.9727GiB/s) | 20260508 07:32:36.581 | INFO | Analyzing chunk frames = [6338560,6339072) | 20260508 07:32:36.581 | INFO | MemInfo: { | 20260508 07:32:36.581 | INFO | "gpu" : " 17 GB", | 20260508 07:32:36.581 | INFO | "huge" : "", | 20260508 07:32:36.581 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:36.581 | INFO | "malloc" : " 14 GB", | 20260508 07:32:36.581 | INFO | "pinned" : " 29 GB" | 20260508 07:32:36.581 | INFO | } | 20260508 07:32:36.581 | INFO | Perf report for the last 5.14116 seconds: | 20260508 07:32:36.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:36.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 07:32:36.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | PRE_HQ : 31.7%, 1.0, 16.6ms, 0 | 20260508 07:32:36.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:36.581 | INFO | Uploaded 36GiB to GPU in 2.41434s (14.9109GiB/s) | 20260508 07:32:36.581 | INFO | Downloaded 34GiB to GPU in 2.52276s (13.4773GiB/s) | 20260508 07:32:41.717 | INFO | Analyzing chunk frames = [6339072,6339584) | 20260508 07:32:41.717 | INFO | MemInfo: { | 20260508 07:32:41.717 | INFO | "gpu" : " 17 GB", | 20260508 07:32:41.717 | INFO | "huge" : "", | 20260508 07:32:41.717 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:41.717 | INFO | "malloc" : " 14 GB", | 20260508 07:32:41.717 | INFO | "pinned" : " 29 GB" | 20260508 07:32:41.717 | INFO | } | 20260508 07:32:41.717 | INFO | Perf report for the last 5.1363 seconds: | 20260508 07:32:41.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:41.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:32:41.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:32:41.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:41.717 | INFO | Uploaded 36GiB to GPU in 2.47034s (14.5729GiB/s) | 20260508 07:32:41.717 | INFO | Downloaded 34GiB to GPU in 2.4292s (13.9964GiB/s) | 20260508 07:32:46.863 | INFO | Analyzing chunk frames = [6339584,6340096) | 20260508 07:32:46.863 | INFO | MemInfo: { | 20260508 07:32:46.863 | INFO | "gpu" : " 17 GB", | 20260508 07:32:46.863 | INFO | "huge" : "", | 20260508 07:32:46.863 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:46.863 | INFO | "malloc" : " 14 GB", | 20260508 07:32:46.863 | INFO | "pinned" : " 29 GB" | 20260508 07:32:46.863 | INFO | } | 20260508 07:32:46.863 | INFO | Perf report for the last 5.14608 seconds: | 20260508 07:32:46.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:46.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | ANALYSIS : 66.2%, 3.0, 103.7ms, 0 | 20260508 07:32:46.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:32:46.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:46.863 | INFO | Uploaded 36GiB to GPU in 2.49345s (14.4378GiB/s) | 20260508 07:32:46.863 | INFO | Downloaded 34GiB to GPU in 2.42934s (13.9955GiB/s) | 20260508 07:32:52.000 | INFO | Analyzing chunk frames = [6340096,6340608) | 20260508 07:32:52.000 | INFO | MemInfo: { | 20260508 07:32:52.000 | INFO | "gpu" : " 17 GB", | 20260508 07:32:52.000 | INFO | "huge" : "", | 20260508 07:32:52.000 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:52.000 | INFO | "malloc" : " 14 GB", | 20260508 07:32:52.000 | INFO | "pinned" : " 29 GB" | 20260508 07:32:52.000 | INFO | } | 20260508 07:32:52.000 | INFO | Perf report for the last 5.13725 seconds: | 20260508 07:32:52.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:52.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.000 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:32:52.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.001 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:32:52.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:52.001 | INFO | Uploaded 36GiB to GPU in 2.48746s (14.4726GiB/s) | 20260508 07:32:52.001 | INFO | Downloaded 34GiB to GPU in 2.44139s (13.9265GiB/s) | 20260508 07:32:52.001 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:32:52.001 | INFO | Total mem usage: 16.7 GiB | 20260508 07:32:52.001 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:32:52.001 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:32:52.001 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:32:52.001 | INFO | High Water Breakdown: | 20260508 07:32:52.001 | INFO | 59.4% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:32:52.001 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:32:52.001 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:32:52.001 | INFO | ------------------------------------------ | 20260508 07:32:52.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:32:52.001 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:32:52.001 | INFO | Total mem usage: 26.5 GiB | 20260508 07:32:52.001 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:32:52.001 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:32:52.001 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:32:52.001 | INFO | High Water Breakdown: | 20260508 07:32:52.001 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:32:52.001 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:32:52.001 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:32:52.001 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:32:52.001 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:32:52.001 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:32:52.001 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:32:52.001 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:32:52.001 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:32:52.001 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:32:52.001 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:32:52.001 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:32:52.001 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:32:52.001 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:32:52.001 | INFO | ------------------------------------------ | 20260508 07:32:52.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:32:52.001 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:32:52.001 | INFO | Total mem usage: 27.8 GiB | 20260508 07:32:52.001 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:32:52.001 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:32:52.001 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:32:52.001 | INFO | High Water Breakdown: | 20260508 07:32:52.001 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:32:52.001 | INFO | ------------------------------------------ | 20260508 07:32:52.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:32:52.001 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:32:52.001 | INFO | Total mem usage: 15.7 GiB | 20260508 07:32:52.001 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:32:52.001 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:32:52.001 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:32:52.001 | INFO | High Water Breakdown: | 20260508 07:32:52.001 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:32:52.001 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:32:52.001 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:32:52.001 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:32:52.001 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:32:52.001 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:32:52.001 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:32:52.001 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:32:52.001 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:32:52.001 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:32:52.001 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:32:52.001 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:32:52.001 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:32:52.001 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:32:52.001 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:32:52.001 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:32:52.001 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:32:52.001 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:32:52.001 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:32:52.001 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:32:52.001 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:32:52.001 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:32:52.001 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:32:52.001 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:32:52.001 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:32:52.001 | INFO | ------------------------------------------ | 20260508 07:32:52.001 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:32:52.001 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:32:52.001 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:32:52.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:32:52.001 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 8.35e-01 | 20260508 07:32:52.001 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.06e-01, 1.14e-02 / 6.26e-03 | 20260508 07:32:52.001 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.42e+00, 5.83e-01 / 2.48e-01 | 20260508 07:32:52.001 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 1.32e-01 | 20260508 07:32:52.001 | INFO | PerfReport: 15.92%, DME -- 4.16e+01 / 4.48e+01, 3.65e+00 / 1.10e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.81e+00 / 1.47e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.37e+00 / 2.52e+00, 4.09e-01 / 2.47e-01 | 20260508 07:32:52.001 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.22e+00, 2.53e-01 / 1.39e-01 | 20260508 07:32:52.001 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.12e-01 / 3.47e-01, 4.95e-02 / 2.46e-02 | 20260508 07:32:52.001 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:32:52.001 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:32:52.001 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:32:52.001 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 24.50%, Upload -- 2.60e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 24.32%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 30.78%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 2.35%, LockUpload -- 2.49e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 2.50%, LockDownload -- 2.65e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 15.53%, LockCompute -- 1.65e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:32:52.001 | INFO | PerfReport: 0.01%, Misc -- 1.52e-02 / 0.00e+00, 4.48e-02 / 0.00e+00 | 20260508 07:32:52.097 | INFO | Starting a round of Baz IO | 20260508 07:32:55.511 | INFO | Populating RTMetrics report | 20260508 07:32:55.512 | INFO | RTMetrics report populated | 20260508 07:32:55.512 | INFO | Opening tmp RTMetrics json file | 20260508 07:32:55.524 | INFO | Serializing RTMetrics to json | 20260508 07:32:55.525 | INFO | Serialization done, swapping tmp file now... | 20260508 07:32:55.525 | INFO | RTMetrics done | 20260508 07:32:55.588 | INFO | Finished a round of Baz IO: Wrote 1.527 GiB in 3.49 seconds | 20260508 07:32:55.588 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.54% / 0.00% / 59.46% | 20260508 07:32:55.588 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.339365% / 3.97364e-06% / 6.75519e-05% | 20260508 07:32:55.588 | INFO | Baz IO overall: | 20260508 07:32:55.588 | INFO | BytesWritten : 2795962193991 | 20260508 07:32:55.588 | INFO | NumEvents : 1346980218506 | 20260508 07:32:57.141 | INFO | Analyzing chunk frames = [6340608,6341120) | 20260508 07:32:57.141 | INFO | MemInfo: { | 20260508 07:32:57.141 | INFO | "gpu" : " 17 GB", | 20260508 07:32:57.141 | INFO | "huge" : "", | 20260508 07:32:57.141 | INFO | "huge_shared" : " 29 GB", | 20260508 07:32:57.141 | INFO | "malloc" : " 14 GB", | 20260508 07:32:57.141 | INFO | "pinned" : " 29 GB" | 20260508 07:32:57.141 | INFO | } | 20260508 07:32:57.141 | INFO | Perf report for the last 5.14097 seconds: | 20260508 07:32:57.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:32:57.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:57.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:57.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:57.142 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:32:57.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:32:57.142 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:32:57.142 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 07:32:57.142 | INFO | BAZWRITER : 67.2%, 2.6, 91.0ms, 0 | 20260508 07:32:57.142 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:32:57.142 | INFO | Uploaded 36GiB to GPU in 2.59606s (13.8672GiB/s) | 20260508 07:32:57.142 | INFO | Downloaded 38GiB to GPU in 2.69038s (14.1244GiB/s) | 20260508 07:33:02.277 | INFO | Analyzing chunk frames = [6341120,6341632) | 20260508 07:33:02.277 | INFO | MemInfo: { | 20260508 07:33:02.277 | INFO | "gpu" : " 17 GB", | 20260508 07:33:02.277 | INFO | "huge" : "", | 20260508 07:33:02.277 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:02.277 | INFO | "malloc" : " 14 GB", | 20260508 07:33:02.277 | INFO | "pinned" : " 29 GB" | 20260508 07:33:02.277 | INFO | } | 20260508 07:33:02.277 | INFO | Perf report for the last 5.13564 seconds: | 20260508 07:33:02.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:02.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.277 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:33:02.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.277 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:33:02.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:02.278 | INFO | Uploaded 36GiB to GPU in 2.4909s (14.4526GiB/s) | 20260508 07:33:02.278 | INFO | Downloaded 34GiB to GPU in 2.42578s (14.0161GiB/s) | 20260508 07:33:07.413 | INFO | Analyzing chunk frames = [6341632,6342144) | 20260508 07:33:07.413 | INFO | MemInfo: { | 20260508 07:33:07.413 | INFO | "gpu" : " 17 GB", | 20260508 07:33:07.413 | INFO | "huge" : "", | 20260508 07:33:07.413 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:07.413 | INFO | "malloc" : " 14 GB", | 20260508 07:33:07.413 | INFO | "pinned" : " 29 GB" | 20260508 07:33:07.413 | INFO | } | 20260508 07:33:07.413 | INFO | Perf report for the last 5.13641 seconds: | 20260508 07:33:07.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:07.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 07:33:07.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:33:07.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:07.414 | INFO | Uploaded 36GiB to GPU in 2.58506s (13.9262GiB/s) | 20260508 07:33:07.414 | INFO | Downloaded 34GiB to GPU in 2.42241s (14.0356GiB/s) | 20260508 07:33:12.557 | INFO | Analyzing chunk frames = [6342144,6342656) | 20260508 07:33:12.557 | INFO | MemInfo: { | 20260508 07:33:12.557 | INFO | "gpu" : " 17 GB", | 20260508 07:33:12.557 | INFO | "huge" : "", | 20260508 07:33:12.557 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:12.557 | INFO | "malloc" : " 14 GB", | 20260508 07:33:12.557 | INFO | "pinned" : " 29 GB" | 20260508 07:33:12.557 | INFO | } | 20260508 07:33:12.557 | INFO | Perf report for the last 5.14317 seconds: | 20260508 07:33:12.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:12.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:33:12.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:33:12.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:12.557 | INFO | Uploaded 36GiB to GPU in 2.58958s (13.9019GiB/s) | 20260508 07:33:12.557 | INFO | Downloaded 34GiB to GPU in 2.40503s (14.137GiB/s) | 20260508 07:33:17.700 | INFO | Analyzing chunk frames = [6342656,6343168) | 20260508 07:33:17.700 | INFO | MemInfo: { | 20260508 07:33:17.700 | INFO | "gpu" : " 17 GB", | 20260508 07:33:17.700 | INFO | "huge" : "", | 20260508 07:33:17.700 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:17.700 | INFO | "malloc" : " 14 GB", | 20260508 07:33:17.700 | INFO | "pinned" : " 29 GB" | 20260508 07:33:17.700 | INFO | } | 20260508 07:33:17.700 | INFO | Perf report for the last 5.1437 seconds: | 20260508 07:33:17.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:17.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.700 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 07:33:17.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.700 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:33:17.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:17.701 | INFO | Uploaded 36GiB to GPU in 2.239s (16.0786GiB/s) | 20260508 07:33:17.701 | INFO | Downloaded 34GiB to GPU in 2.4839s (13.6881GiB/s) | 20260508 07:33:22.841 | INFO | Analyzing chunk frames = [6343168,6343680) | 20260508 07:33:22.841 | INFO | MemInfo: { | 20260508 07:33:22.841 | INFO | "gpu" : " 17 GB", | 20260508 07:33:22.841 | INFO | "huge" : "", | 20260508 07:33:22.841 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:22.841 | INFO | "malloc" : " 14 GB", | 20260508 07:33:22.841 | INFO | "pinned" : " 29 GB" | 20260508 07:33:22.841 | INFO | } | 20260508 07:33:22.841 | INFO | Perf report for the last 5.14071 seconds: | 20260508 07:33:22.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:22.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 07:33:22.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:33:22.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:22.841 | INFO | Uploaded 36GiB to GPU in 2.36418s (15.2273GiB/s) | 20260508 07:33:22.841 | INFO | Downloaded 34GiB to GPU in 2.43185s (13.9812GiB/s) | 20260508 07:33:27.973 | INFO | Analyzing chunk frames = [6343680,6344192) | 20260508 07:33:27.973 | INFO | MemInfo: { | 20260508 07:33:27.973 | INFO | "gpu" : " 17 GB", | 20260508 07:33:27.973 | INFO | "huge" : "", | 20260508 07:33:27.973 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:27.973 | INFO | "malloc" : " 14 GB", | 20260508 07:33:27.973 | INFO | "pinned" : " 29 GB" | 20260508 07:33:27.973 | INFO | } | 20260508 07:33:27.973 | INFO | Perf report for the last 5.13228 seconds: | 20260508 07:33:27.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:27.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.973 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:33:27.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.974 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:33:27.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:27.974 | INFO | Uploaded 36GiB to GPU in 2.41476s (14.9083GiB/s) | 20260508 07:33:27.974 | INFO | Downloaded 34GiB to GPU in 2.4422s (13.9219GiB/s) | 20260508 07:33:33.118 | INFO | Analyzing chunk frames = [6344192,6344704) | 20260508 07:33:33.118 | INFO | MemInfo: { | 20260508 07:33:33.118 | INFO | "gpu" : " 17 GB", | 20260508 07:33:33.118 | INFO | "huge" : "", | 20260508 07:33:33.118 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:33.118 | INFO | "malloc" : " 14 GB", | 20260508 07:33:33.118 | INFO | "pinned" : " 29 GB" | 20260508 07:33:33.118 | INFO | } | 20260508 07:33:33.118 | INFO | Perf report for the last 5.14444 seconds: | 20260508 07:33:33.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:33.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 07:33:33.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:33:33.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:33.118 | INFO | Uploaded 36GiB to GPU in 2.50535s (14.3693GiB/s) | 20260508 07:33:33.118 | INFO | Downloaded 34GiB to GPU in 2.45589s (13.8443GiB/s) | 20260508 07:33:33.215 | INFO | Starting a round of Baz IO | 20260508 07:33:36.637 | INFO | Populating RTMetrics report | 20260508 07:33:36.637 | INFO | RTMetrics report populated | 20260508 07:33:36.637 | INFO | Opening tmp RTMetrics json file | 20260508 07:33:36.638 | INFO | Serializing RTMetrics to json | 20260508 07:33:36.639 | INFO | Serialization done, swapping tmp file now... | 20260508 07:33:36.640 | INFO | RTMetrics done | 20260508 07:33:36.701 | INFO | Finished a round of Baz IO: Wrote 1.525 GiB in 3.486 seconds | 20260508 07:33:36.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.54% / 0.00% / 59.46% | 20260508 07:33:36.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.338777% / 0% / 4.76837e-05% | 20260508 07:33:36.702 | INFO | Baz IO overall: | 20260508 07:33:36.702 | INFO | BytesWritten : 2797599844037 | 20260508 07:33:36.702 | INFO | NumEvents : 1347757752311 | 20260508 07:33:38.255 | INFO | Analyzing chunk frames = [6344704,6345216) | 20260508 07:33:38.255 | INFO | MemInfo: { | 20260508 07:33:38.255 | INFO | "gpu" : " 17 GB", | 20260508 07:33:38.255 | INFO | "huge" : "", | 20260508 07:33:38.255 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:38.255 | INFO | "malloc" : " 14 GB", | 20260508 07:33:38.255 | INFO | "pinned" : " 29 GB" | 20260508 07:33:38.256 | INFO | } | 20260508 07:33:38.256 | INFO | Perf report for the last 5.13773 seconds: | 20260508 07:33:38.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:38.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:38.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:38.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:38.256 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 07:33:38.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:38.256 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:33:38.256 | INFO | PRE_HQ : 66.2%, 1.6, 55.2ms, 0 | 20260508 07:33:38.256 | INFO | BAZWRITER : 67.3%, 2.6, 91.2ms, 0 | 20260508 07:33:38.256 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 07:33:38.256 | INFO | Uploaded 36GiB to GPU in 2.49716s (14.4164GiB/s) | 20260508 07:33:38.256 | INFO | Downloaded 38GiB to GPU in 2.70061s (14.0709GiB/s) | 20260508 07:33:43.395 | INFO | Analyzing chunk frames = [6345216,6345728) | 20260508 07:33:43.395 | INFO | MemInfo: { | 20260508 07:33:43.395 | INFO | "gpu" : " 17 GB", | 20260508 07:33:43.395 | INFO | "huge" : "", | 20260508 07:33:43.395 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:43.395 | INFO | "malloc" : " 14 GB", | 20260508 07:33:43.395 | INFO | "pinned" : " 29 GB" | 20260508 07:33:43.395 | INFO | } | 20260508 07:33:43.395 | INFO | Perf report for the last 5.13955 seconds: | 20260508 07:33:43.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:43.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:33:43.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 07:33:43.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:43.395 | INFO | Uploaded 36GiB to GPU in 2.49363s (14.4368GiB/s) | 20260508 07:33:43.396 | INFO | Downloaded 34GiB to GPU in 2.43675s (13.953GiB/s) | 20260508 07:33:48.533 | INFO | Analyzing chunk frames = [6345728,6346240) | 20260508 07:33:48.533 | INFO | MemInfo: { | 20260508 07:33:48.533 | INFO | "gpu" : " 17 GB", | 20260508 07:33:48.533 | INFO | "huge" : "", | 20260508 07:33:48.533 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:48.533 | INFO | "malloc" : " 14 GB", | 20260508 07:33:48.533 | INFO | "pinned" : " 29 GB" | 20260508 07:33:48.533 | INFO | } | 20260508 07:33:48.533 | INFO | Perf report for the last 5.13808 seconds: | 20260508 07:33:48.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:48.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.533 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:33:48.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.533 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:33:48.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:48.534 | INFO | Uploaded 36GiB to GPU in 2.55965s (14.0644GiB/s) | 20260508 07:33:48.534 | INFO | Downloaded 34GiB to GPU in 2.43988s (13.9351GiB/s) | 20260508 07:33:53.669 | INFO | Analyzing chunk frames = [6346240,6346752) | 20260508 07:33:53.669 | INFO | MemInfo: { | 20260508 07:33:53.669 | INFO | "gpu" : " 17 GB", | 20260508 07:33:53.669 | INFO | "huge" : "", | 20260508 07:33:53.669 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:53.669 | INFO | "malloc" : " 14 GB", | 20260508 07:33:53.669 | INFO | "pinned" : " 29 GB" | 20260508 07:33:53.669 | INFO | } | 20260508 07:33:53.669 | INFO | Perf report for the last 5.13587 seconds: | 20260508 07:33:53.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:53.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | ANALYSIS : 66.9%, 3.0, 103.4ms, 0 | 20260508 07:33:53.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:33:53.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:53.669 | INFO | Uploaded 36GiB to GPU in 2.5713s (14.0007GiB/s) | 20260508 07:33:53.669 | INFO | Downloaded 34GiB to GPU in 2.43627s (13.9557GiB/s) | 20260508 07:33:58.812 | INFO | Analyzing chunk frames = [6346752,6347264) | 20260508 07:33:58.812 | INFO | MemInfo: { | 20260508 07:33:58.812 | INFO | "gpu" : " 17 GB", | 20260508 07:33:58.812 | INFO | "huge" : "", | 20260508 07:33:58.812 | INFO | "huge_shared" : " 29 GB", | 20260508 07:33:58.812 | INFO | "malloc" : " 14 GB", | 20260508 07:33:58.812 | INFO | "pinned" : " 29 GB" | 20260508 07:33:58.812 | INFO | } | 20260508 07:33:58.812 | INFO | Perf report for the last 5.14275 seconds: | 20260508 07:33:58.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:33:58.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 07:33:58.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 07:33:58.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:33:58.812 | INFO | Uploaded 36GiB to GPU in 2.43919s (14.759GiB/s) | 20260508 07:33:58.812 | INFO | Downloaded 34GiB to GPU in 2.52448s (13.4681GiB/s) | 20260508 07:34:03.949 | INFO | Analyzing chunk frames = [6347264,6347776) | 20260508 07:34:03.949 | INFO | MemInfo: { | 20260508 07:34:03.949 | INFO | "gpu" : " 17 GB", | 20260508 07:34:03.949 | INFO | "huge" : "", | 20260508 07:34:03.949 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:03.949 | INFO | "malloc" : " 14 GB", | 20260508 07:34:03.949 | INFO | "pinned" : " 29 GB" | 20260508 07:34:03.949 | INFO | } | 20260508 07:34:03.949 | INFO | Perf report for the last 5.13752 seconds: | 20260508 07:34:03.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:03.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.949 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 07:34:03.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.950 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:34:03.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:03.950 | INFO | Uploaded 36GiB to GPU in 2.41631s (14.8987GiB/s) | 20260508 07:34:03.950 | INFO | Downloaded 34GiB to GPU in 2.43327s (13.973GiB/s) | 20260508 07:34:09.087 | INFO | Analyzing chunk frames = [6347776,6348288) | 20260508 07:34:09.087 | INFO | MemInfo: { | 20260508 07:34:09.087 | INFO | "gpu" : " 17 GB", | 20260508 07:34:09.087 | INFO | "huge" : "", | 20260508 07:34:09.087 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:09.087 | INFO | "malloc" : " 14 GB", | 20260508 07:34:09.087 | INFO | "pinned" : " 29 GB" | 20260508 07:34:09.088 | INFO | } | 20260508 07:34:09.088 | INFO | Perf report for the last 5.13818 seconds: | 20260508 07:34:09.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:09.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:34:09.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 07:34:09.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:09.088 | INFO | Uploaded 36GiB to GPU in 2.48038s (14.5139GiB/s) | 20260508 07:34:09.088 | INFO | Downloaded 34GiB to GPU in 2.42893s (13.9979GiB/s) | 20260508 07:34:14.225 | INFO | Analyzing chunk frames = [6348288,6348800) | 20260508 07:34:14.225 | INFO | MemInfo: { | 20260508 07:34:14.225 | INFO | "gpu" : " 17 GB", | 20260508 07:34:14.225 | INFO | "huge" : "", | 20260508 07:34:14.225 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:14.225 | INFO | "malloc" : " 14 GB", | 20260508 07:34:14.225 | INFO | "pinned" : " 29 GB" | 20260508 07:34:14.225 | INFO | } | 20260508 07:34:14.225 | INFO | Perf report for the last 5.13758 seconds: | 20260508 07:34:14.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:14.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.225 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:34:14.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.226 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 07:34:14.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:14.226 | INFO | Uploaded 36GiB to GPU in 2.48152s (14.5073GiB/s) | 20260508 07:34:14.226 | INFO | Downloaded 34GiB to GPU in 2.43047s (13.9891GiB/s) | 20260508 07:34:14.226 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:34:14.226 | INFO | Total mem usage: 16.7 GiB | 20260508 07:34:14.226 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:34:14.226 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:34:14.226 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:34:14.226 | INFO | High Water Breakdown: | 20260508 07:34:14.226 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:34:14.226 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:34:14.226 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:34:14.226 | INFO | ------------------------------------------ | 20260508 07:34:14.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:34:14.226 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:34:14.226 | INFO | Total mem usage: 26.5 GiB | 20260508 07:34:14.226 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:34:14.226 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:34:14.226 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:34:14.226 | INFO | High Water Breakdown: | 20260508 07:34:14.226 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:34:14.226 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:34:14.226 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:34:14.226 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:34:14.226 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:34:14.226 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:34:14.226 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:34:14.226 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:34:14.226 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:34:14.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:34:14.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:34:14.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:34:14.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:34:14.226 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:34:14.226 | INFO | ------------------------------------------ | 20260508 07:34:14.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:34:14.226 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:34:14.226 | INFO | Total mem usage: 27.8 GiB | 20260508 07:34:14.226 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:34:14.226 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:34:14.226 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:34:14.226 | INFO | High Water Breakdown: | 20260508 07:34:14.226 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:34:14.227 | INFO | ------------------------------------------ | 20260508 07:34:14.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:34:14.227 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:34:14.227 | INFO | Total mem usage: 15.7 GiB | 20260508 07:34:14.227 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:34:14.227 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:34:14.227 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:34:14.227 | INFO | High Water Breakdown: | 20260508 07:34:14.227 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:34:14.227 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:34:14.227 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:34:14.227 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:34:14.227 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:34:14.227 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:34:14.227 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:34:14.227 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:34:14.227 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:34:14.227 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:34:14.227 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:34:14.227 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:34:14.227 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:34:14.227 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:34:14.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:34:14.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:34:14.227 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:34:14.227 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:34:14.227 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:34:14.227 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:34:14.227 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:34:14.227 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:34:14.227 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:34:14.227 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:34:14.227 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:34:14.227 | INFO | ------------------------------------------ | 20260508 07:34:14.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:34:14.227 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:34:14.227 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:34:14.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:34:14.227 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 7.50e-01 | 20260508 07:34:14.227 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.04e-01, 1.07e-02 / 5.10e-03 | 20260508 07:34:14.227 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.48e+00, 5.88e-01 / 2.15e-01 | 20260508 07:34:14.227 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.11e-01 / 1.19e-01 | 20260508 07:34:14.227 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.53e+01, 3.83e+00 / 1.08e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.21e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.59e+00, 4.09e-01 / 2.14e-01 | 20260508 07:34:14.227 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.22e+00, 2.51e-01 / 1.41e-01 | 20260508 07:34:14.227 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.46e-01, 4.90e-02 / 2.11e-02 | 20260508 07:34:14.227 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:34:14.227 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:34:14.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:34:14.227 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 24.42%, Upload -- 2.59e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 24.33%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 2.33%, LockUpload -- 2.46e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 2.52%, LockDownload -- 2.67e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 15.54%, LockCompute -- 1.65e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:34:14.227 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260508 07:34:14.339 | INFO | Starting a round of Baz IO | 20260508 07:34:17.755 | INFO | Populating RTMetrics report | 20260508 07:34:17.755 | INFO | RTMetrics report populated | 20260508 07:34:17.755 | INFO | Opening tmp RTMetrics json file | 20260508 07:34:17.756 | INFO | Serializing RTMetrics to json | 20260508 07:34:17.757 | INFO | Serialization done, swapping tmp file now... | 20260508 07:34:17.758 | INFO | RTMetrics done | 20260508 07:34:17.823 | INFO | Finished a round of Baz IO: Wrote 1.525 GiB in 3.483 seconds | 20260508 07:34:17.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.54% / 0.00% / 59.46% | 20260508 07:34:17.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.337986% / 3.97364e-06% / 5.5631e-05% | 20260508 07:34:17.823 | INFO | Baz IO overall: | 20260508 07:34:17.823 | INFO | BytesWritten : 2799237530702 | 20260508 07:34:17.823 | INFO | NumEvents : 1348535499351 | 20260508 07:34:19.368 | INFO | Analyzing chunk frames = [6348800,6349312) | 20260508 07:34:19.368 | INFO | MemInfo: { | 20260508 07:34:19.368 | INFO | "gpu" : " 17 GB", | 20260508 07:34:19.368 | INFO | "huge" : "", | 20260508 07:34:19.368 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:19.368 | INFO | "malloc" : " 14 GB", | 20260508 07:34:19.368 | INFO | "pinned" : " 29 GB" | 20260508 07:34:19.368 | INFO | } | 20260508 07:34:19.368 | INFO | Perf report for the last 5.14277 seconds: | 20260508 07:34:19.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:19.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:19.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:19.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:19.368 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 07:34:19.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:19.369 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260508 07:34:19.369 | INFO | PRE_HQ : 66.3%, 1.7, 59.4ms, 0 | 20260508 07:34:19.369 | INFO | BAZWRITER : 67.2%, 2.8, 95.9ms, 0 | 20260508 07:34:19.369 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 07:34:19.369 | INFO | Uploaded 36GiB to GPU in 2.54209s (14.1616GiB/s) | 20260508 07:34:19.369 | INFO | Downloaded 38GiB to GPU in 2.7s (14.0741GiB/s) | 20260508 07:34:24.508 | INFO | Analyzing chunk frames = [6349312,6349824) | 20260508 07:34:24.508 | INFO | MemInfo: { | 20260508 07:34:24.508 | INFO | "gpu" : " 17 GB", | 20260508 07:34:24.508 | INFO | "huge" : "", | 20260508 07:34:24.508 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:24.508 | INFO | "malloc" : " 14 GB", | 20260508 07:34:24.508 | INFO | "pinned" : " 29 GB" | 20260508 07:34:24.508 | INFO | } | 20260508 07:34:24.508 | INFO | Perf report for the last 5.14038 seconds: | 20260508 07:34:24.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:24.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.508 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:34:24.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.508 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:34:24.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:24.509 | INFO | Uploaded 36GiB to GPU in 2.52503s (14.2573GiB/s) | 20260508 07:34:24.509 | INFO | Downloaded 34GiB to GPU in 2.44317s (13.9164GiB/s) | 20260508 07:34:29.653 | INFO | Analyzing chunk frames = [6349824,6350336) | 20260508 07:34:29.653 | INFO | MemInfo: { | 20260508 07:34:29.653 | INFO | "gpu" : " 17 GB", | 20260508 07:34:29.653 | INFO | "huge" : "", | 20260508 07:34:29.653 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:29.653 | INFO | "malloc" : " 14 GB", | 20260508 07:34:29.653 | INFO | "pinned" : " 29 GB" | 20260508 07:34:29.653 | INFO | } | 20260508 07:34:29.653 | INFO | Perf report for the last 5.14451 seconds: | 20260508 07:34:29.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:29.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 07:34:29.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:34:29.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:29.653 | INFO | Uploaded 36GiB to GPU in 2.57229s (13.9953GiB/s) | 20260508 07:34:29.653 | INFO | Downloaded 34GiB to GPU in 2.42835s (14.0013GiB/s) | 20260508 07:34:34.784 | INFO | Analyzing chunk frames = [6350336,6350848) | 20260508 07:34:34.784 | INFO | MemInfo: { | 20260508 07:34:34.784 | INFO | "gpu" : " 17 GB", | 20260508 07:34:34.784 | INFO | "huge" : "", | 20260508 07:34:34.784 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:34.784 | INFO | "malloc" : " 14 GB", | 20260508 07:34:34.784 | INFO | "pinned" : " 29 GB" | 20260508 07:34:34.784 | INFO | } | 20260508 07:34:34.784 | INFO | Perf report for the last 5.1308 seconds: | 20260508 07:34:34.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:34.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 07:34:34.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:34:34.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:34.784 | INFO | Uploaded 36GiB to GPU in 2.58508s (13.9261GiB/s) | 20260508 07:34:34.784 | INFO | Downloaded 34GiB to GPU in 2.41892s (14.0559GiB/s) | 20260508 07:34:39.929 | INFO | Analyzing chunk frames = [6350848,6351360) | 20260508 07:34:39.929 | INFO | MemInfo: { | 20260508 07:34:39.929 | INFO | "gpu" : " 17 GB", | 20260508 07:34:39.929 | INFO | "huge" : "", | 20260508 07:34:39.929 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:39.929 | INFO | "malloc" : " 14 GB", | 20260508 07:34:39.929 | INFO | "pinned" : " 29 GB" | 20260508 07:34:39.929 | INFO | } | 20260508 07:34:39.929 | INFO | Perf report for the last 5.14551 seconds: | 20260508 07:34:39.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:39.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 07:34:39.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:34:39.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:39.929 | INFO | Uploaded 36GiB to GPU in 2.2929s (15.7007GiB/s) | 20260508 07:34:39.929 | INFO | Downloaded 34GiB to GPU in 2.50036s (13.5981GiB/s) | 20260508 07:34:45.065 | INFO | Analyzing chunk frames = [6351360,6351872) | 20260508 07:34:45.065 | INFO | MemInfo: { | 20260508 07:34:45.065 | INFO | "gpu" : " 17 GB", | 20260508 07:34:45.065 | INFO | "huge" : "", | 20260508 07:34:45.065 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:45.065 | INFO | "malloc" : " 14 GB", | 20260508 07:34:45.065 | INFO | "pinned" : " 29 GB" | 20260508 07:34:45.065 | INFO | } | 20260508 07:34:45.065 | INFO | Perf report for the last 5.13615 seconds: | 20260508 07:34:45.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:45.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.065 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 07:34:45.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.066 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:34:45.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:45.066 | INFO | Uploaded 36GiB to GPU in 2.3665s (15.2124GiB/s) | 20260508 07:34:45.066 | INFO | Downloaded 34GiB to GPU in 2.42545s (14.018GiB/s) | 20260508 07:34:50.207 | INFO | Analyzing chunk frames = [6351872,6352384) | 20260508 07:34:50.207 | INFO | MemInfo: { | 20260508 07:34:50.207 | INFO | "gpu" : " 17 GB", | 20260508 07:34:50.208 | INFO | "huge" : "", | 20260508 07:34:50.208 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:50.208 | INFO | "malloc" : " 14 GB", | 20260508 07:34:50.208 | INFO | "pinned" : " 29 GB" | 20260508 07:34:50.208 | INFO | } | 20260508 07:34:50.208 | INFO | Perf report for the last 5.14258 seconds: | 20260508 07:34:50.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:50.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.208 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:34:50.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.208 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 07:34:50.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:50.209 | INFO | Uploaded 36GiB to GPU in 2.47141s (14.5666GiB/s) | 20260508 07:34:50.209 | INFO | Downloaded 34GiB to GPU in 2.44077s (13.93GiB/s) | 20260508 07:34:55.346 | INFO | Analyzing chunk frames = [6352384,6352896) | 20260508 07:34:55.346 | INFO | MemInfo: { | 20260508 07:34:55.346 | INFO | "gpu" : " 17 GB", | 20260508 07:34:55.346 | INFO | "huge" : "", | 20260508 07:34:55.346 | INFO | "huge_shared" : " 29 GB", | 20260508 07:34:55.347 | INFO | "malloc" : " 14 GB", | 20260508 07:34:55.347 | INFO | "pinned" : " 29 GB" | 20260508 07:34:55.347 | INFO | } | 20260508 07:34:55.347 | INFO | Perf report for the last 5.13874 seconds: | 20260508 07:34:55.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:34:55.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 07:34:55.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260508 07:34:55.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:34:55.347 | INFO | Uploaded 36GiB to GPU in 2.50588s (14.3662GiB/s) | 20260508 07:34:55.347 | INFO | Downloaded 34GiB to GPU in 2.46042s (13.8188GiB/s) | 20260508 07:34:55.444 | INFO | Starting a round of Baz IO | 20260508 07:34:58.872 | INFO | Populating RTMetrics report | 20260508 07:34:58.872 | INFO | RTMetrics report populated | 20260508 07:34:58.872 | INFO | Opening tmp RTMetrics json file | 20260508 07:34:58.873 | INFO | Serializing RTMetrics to json | 20260508 07:34:58.874 | INFO | Serialization done, swapping tmp file now... | 20260508 07:34:58.875 | INFO | RTMetrics done | 20260508 07:34:58.949 | INFO | Finished a round of Baz IO: Wrote 1.526 GiB in 3.505 seconds | 20260508 07:34:58.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.46% | 20260508 07:34:58.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.337621% / 0% / 7.15256e-05% | 20260508 07:34:58.949 | INFO | Baz IO overall: | 20260508 07:34:58.949 | INFO | BytesWritten : 2800875786642 | 20260508 07:34:58.949 | INFO | NumEvents : 1349313732312 | 20260508 07:35:00.478 | INFO | Analyzing chunk frames = [6352896,6353408) | 20260508 07:35:00.478 | INFO | MemInfo: { | 20260508 07:35:00.478 | INFO | "gpu" : " 17 GB", | 20260508 07:35:00.478 | INFO | "huge" : "", | 20260508 07:35:00.478 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:00.478 | INFO | "malloc" : " 14 GB", | 20260508 07:35:00.478 | INFO | "pinned" : " 29 GB" | 20260508 07:35:00.478 | INFO | } | 20260508 07:35:00.478 | INFO | Perf report for the last 5.13103 seconds: | 20260508 07:35:00.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:00.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:00.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:00.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:00.478 | INFO | ANALYSIS : 68.3%, 3.0, 106.1ms, 0 | 20260508 07:35:00.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:00.478 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 07:35:00.478 | INFO | PRE_HQ : 66.0%, 1.6, 54.7ms, 0 | 20260508 07:35:00.478 | INFO | BAZWRITER : 67.7%, 2.6, 91.1ms, 0 | 20260508 07:35:00.478 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:35:00.478 | INFO | Uploaded 36GiB to GPU in 2.55017s (14.1167GiB/s) | 20260508 07:35:00.478 | INFO | Downloaded 38GiB to GPU in 2.71127s (14.0156GiB/s) | 20260508 07:35:05.625 | INFO | Analyzing chunk frames = [6353408,6353920) | 20260508 07:35:05.625 | INFO | MemInfo: { | 20260508 07:35:05.625 | INFO | "gpu" : " 17 GB", | 20260508 07:35:05.625 | INFO | "huge" : "", | 20260508 07:35:05.625 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:05.625 | INFO | "malloc" : " 14 GB", | 20260508 07:35:05.625 | INFO | "pinned" : " 29 GB" | 20260508 07:35:05.625 | INFO | } | 20260508 07:35:05.626 | INFO | Perf report for the last 5.14773 seconds: | 20260508 07:35:05.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:05.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 07:35:05.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 07:35:05.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:05.626 | INFO | Uploaded 36GiB to GPU in 2.4981s (14.411GiB/s) | 20260508 07:35:05.626 | INFO | Downloaded 34GiB to GPU in 2.43512s (13.9623GiB/s) | 20260508 07:35:10.766 | INFO | Analyzing chunk frames = [6353920,6354432) | 20260508 07:35:10.766 | INFO | MemInfo: { | 20260508 07:35:10.766 | INFO | "gpu" : " 17 GB", | 20260508 07:35:10.766 | INFO | "huge" : "", | 20260508 07:35:10.766 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:10.766 | INFO | "malloc" : " 14 GB", | 20260508 07:35:10.766 | INFO | "pinned" : " 29 GB" | 20260508 07:35:10.766 | INFO | } | 20260508 07:35:10.766 | INFO | Perf report for the last 5.14048 seconds: | 20260508 07:35:10.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:10.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 07:35:10.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:35:10.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:10.766 | INFO | Uploaded 36GiB to GPU in 2.4827s (14.5003GiB/s) | 20260508 07:35:10.766 | INFO | Downloaded 34GiB to GPU in 2.4345s (13.9659GiB/s) | 20260508 07:35:15.903 | INFO | Analyzing chunk frames = [6354432,6354944) | 20260508 07:35:15.903 | INFO | MemInfo: { | 20260508 07:35:15.903 | INFO | "gpu" : " 17 GB", | 20260508 07:35:15.903 | INFO | "huge" : "", | 20260508 07:35:15.903 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:15.903 | INFO | "malloc" : " 14 GB", | 20260508 07:35:15.903 | INFO | "pinned" : " 29 GB" | 20260508 07:35:15.903 | INFO | } | 20260508 07:35:15.903 | INFO | Perf report for the last 5.1368 seconds: | 20260508 07:35:15.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:15.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 07:35:15.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:35:15.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:15.903 | INFO | Uploaded 36GiB to GPU in 2.5734s (13.9893GiB/s) | 20260508 07:35:15.903 | INFO | Downloaded 34GiB to GPU in 2.42737s (14.0069GiB/s) | 20260508 07:35:21.033 | INFO | Analyzing chunk frames = [6354944,6355456) | 20260508 07:35:21.033 | INFO | MemInfo: { | 20260508 07:35:21.033 | INFO | "gpu" : " 17 GB", | 20260508 07:35:21.033 | INFO | "huge" : "", | 20260508 07:35:21.033 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:21.034 | INFO | "malloc" : " 14 GB", | 20260508 07:35:21.034 | INFO | "pinned" : " 29 GB" | 20260508 07:35:21.034 | INFO | } | 20260508 07:35:21.034 | INFO | Perf report for the last 5.13084 seconds: | 20260508 07:35:21.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:21.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 07:35:21.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:35:21.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:21.034 | INFO | Uploaded 36GiB to GPU in 2.28471s (15.7569GiB/s) | 20260508 07:35:21.034 | INFO | Downloaded 34GiB to GPU in 2.49431s (13.631GiB/s) | 20260508 07:35:26.178 | INFO | Analyzing chunk frames = [6355456,6355968) | 20260508 07:35:26.178 | INFO | MemInfo: { | 20260508 07:35:26.178 | INFO | "gpu" : " 17 GB", | 20260508 07:35:26.178 | INFO | "huge" : "", | 20260508 07:35:26.178 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:26.178 | INFO | "malloc" : " 14 GB", | 20260508 07:35:26.178 | INFO | "pinned" : " 29 GB" | 20260508 07:35:26.178 | INFO | } | 20260508 07:35:26.178 | INFO | Perf report for the last 5.14466 seconds: | 20260508 07:35:26.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:26.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.178 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 07:35:26.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.178 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:35:26.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:26.179 | INFO | Uploaded 36GiB to GPU in 2.44566s (14.72GiB/s) | 20260508 07:35:26.179 | INFO | Downloaded 34GiB to GPU in 2.43367s (13.9707GiB/s) | 20260508 07:35:31.317 | INFO | Analyzing chunk frames = [6355968,6356480) | 20260508 07:35:31.317 | INFO | MemInfo: { | 20260508 07:35:31.317 | INFO | "gpu" : " 17 GB", | 20260508 07:35:31.317 | INFO | "huge" : "", | 20260508 07:35:31.317 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:31.317 | INFO | "malloc" : " 14 GB", | 20260508 07:35:31.318 | INFO | "pinned" : " 29 GB" | 20260508 07:35:31.318 | INFO | } | 20260508 07:35:31.318 | INFO | Perf report for the last 5.13929 seconds: | 20260508 07:35:31.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:31.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:35:31.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 07:35:31.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:31.318 | INFO | Uploaded 36GiB to GPU in 2.47413s (14.5506GiB/s) | 20260508 07:35:31.318 | INFO | Downloaded 34GiB to GPU in 2.44353s (13.9143GiB/s) | 20260508 07:35:36.463 | INFO | Analyzing chunk frames = [6356480,6356992) | 20260508 07:35:36.463 | INFO | MemInfo: { | 20260508 07:35:36.463 | INFO | "gpu" : " 17 GB", | 20260508 07:35:36.463 | INFO | "huge" : "", | 20260508 07:35:36.463 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:36.463 | INFO | "malloc" : " 14 GB", | 20260508 07:35:36.463 | INFO | "pinned" : " 29 GB" | 20260508 07:35:36.463 | INFO | } | 20260508 07:35:36.463 | INFO | Perf report for the last 5.14579 seconds: | 20260508 07:35:36.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:36.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:35:36.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:35:36.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:36.464 | INFO | Uploaded 36GiB to GPU in 2.45477s (14.6653GiB/s) | 20260508 07:35:36.464 | INFO | Downloaded 34GiB to GPU in 2.43464s (13.9651GiB/s) | 20260508 07:35:36.464 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:35:36.464 | INFO | Total mem usage: 16.7 GiB | 20260508 07:35:36.464 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:35:36.464 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:35:36.464 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:35:36.464 | INFO | High Water Breakdown: | 20260508 07:35:36.464 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:35:36.464 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:35:36.464 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:35:36.464 | INFO | ------------------------------------------ | 20260508 07:35:36.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:35:36.464 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:35:36.464 | INFO | Total mem usage: 26.5 GiB | 20260508 07:35:36.464 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:35:36.464 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:35:36.464 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:35:36.464 | INFO | High Water Breakdown: | 20260508 07:35:36.464 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:35:36.464 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:35:36.464 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:35:36.464 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:35:36.464 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:35:36.464 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:35:36.464 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:35:36.465 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:35:36.465 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:35:36.465 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:35:36.465 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:35:36.465 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:35:36.465 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:35:36.465 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:35:36.465 | INFO | ------------------------------------------ | 20260508 07:35:36.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:35:36.465 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:35:36.465 | INFO | Total mem usage: 27.8 GiB | 20260508 07:35:36.465 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:35:36.465 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:35:36.465 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:35:36.465 | INFO | High Water Breakdown: | 20260508 07:35:36.465 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:35:36.465 | INFO | ------------------------------------------ | 20260508 07:35:36.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:35:36.465 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:35:36.465 | INFO | Total mem usage: 15.7 GiB | 20260508 07:35:36.465 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:35:36.465 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:35:36.465 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:35:36.465 | INFO | High Water Breakdown: | 20260508 07:35:36.465 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:35:36.465 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:35:36.465 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:35:36.465 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:35:36.465 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:35:36.465 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:35:36.465 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:35:36.465 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:35:36.465 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:35:36.465 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:35:36.465 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:35:36.465 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:35:36.465 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:35:36.465 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:35:36.465 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:35:36.465 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:35:36.465 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:35:36.465 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:35:36.465 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:35:36.465 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:35:36.465 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:35:36.465 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:35:36.465 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:35:36.465 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:35:36.465 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:35:36.465 | INFO | ------------------------------------------ | 20260508 07:35:36.465 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:35:36.465 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:35:36.465 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:35:36.465 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:35:36.465 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.41e+01 / 6.91e-01 | 20260508 07:35:36.465 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.06e-02 / 4.14e-03 | 20260508 07:35:36.465 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.40e+00, 5.86e-01 / 2.57e-01 | 20260508 07:35:36.465 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.10e-01 | 20260508 07:35:36.465 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.58e+01, 3.93e+00 / 1.27e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.81e+00 / 1.19e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.57e+00, 4.08e-01 / 2.14e-01 | 20260508 07:35:36.465 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.21e+00, 2.51e-01 / 1.35e-01 | 20260508 07:35:36.465 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.39e-01, 4.93e-02 / 1.94e-02 | 20260508 07:35:36.465 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:35:36.465 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:35:36.465 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:35:36.465 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 24.34%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 24.35%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 30.85%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 2.30%, LockUpload -- 2.44e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 2.53%, LockDownload -- 2.69e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 15.62%, LockCompute -- 1.66e+01 / 0.00e+00, 1.93e+01 / 0.00e+00 | 20260508 07:35:36.465 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260508 07:35:36.561 | INFO | Starting a round of Baz IO | 20260508 07:35:39.990 | INFO | Populating RTMetrics report | 20260508 07:35:39.991 | INFO | RTMetrics report populated | 20260508 07:35:39.991 | INFO | Opening tmp RTMetrics json file | 20260508 07:35:40.004 | INFO | Serializing RTMetrics to json | 20260508 07:35:40.005 | INFO | Serialization done, swapping tmp file now... | 20260508 07:35:40.006 | INFO | RTMetrics done | 20260508 07:35:40.058 | INFO | Finished a round of Baz IO: Wrote 1.525 GiB in 3.497 seconds | 20260508 07:35:40.058 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.47% | 20260508 07:35:40.058 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.33613% / 3.97364e-06% / 4.76837e-05% | 20260508 07:35:40.058 | INFO | Baz IO overall: | 20260508 07:35:40.058 | INFO | BytesWritten : 2802512747017 | 20260508 07:35:40.058 | INFO | NumEvents : 1350091007141 | 20260508 07:35:41.603 | INFO | Analyzing chunk frames = [6356992,6357504) | 20260508 07:35:41.603 | INFO | MemInfo: { | 20260508 07:35:41.603 | INFO | "gpu" : " 17 GB", | 20260508 07:35:41.603 | INFO | "huge" : "", | 20260508 07:35:41.603 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:41.603 | INFO | "malloc" : " 14 GB", | 20260508 07:35:41.603 | INFO | "pinned" : " 29 GB" | 20260508 07:35:41.603 | INFO | } | 20260508 07:35:41.603 | INFO | Perf report for the last 5.13981 seconds: | 20260508 07:35:41.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:41.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:41.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:41.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:41.604 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 07:35:41.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:41.604 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 07:35:41.604 | INFO | PRE_HQ : 65.7%, 1.6, 54.8ms, 0 | 20260508 07:35:41.604 | INFO | BAZWRITER : 67.4%, 2.6, 90.4ms, 0 | 20260508 07:35:41.604 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:35:41.604 | INFO | Uploaded 36GiB to GPU in 2.56531s (14.0334GiB/s) | 20260508 07:35:41.604 | INFO | Downloaded 38GiB to GPU in 2.69684s (14.0905GiB/s) | 20260508 07:35:46.737 | INFO | Analyzing chunk frames = [6357504,6358016) | 20260508 07:35:46.737 | INFO | MemInfo: { | 20260508 07:35:46.737 | INFO | "gpu" : " 17 GB", | 20260508 07:35:46.737 | INFO | "huge" : "", | 20260508 07:35:46.737 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:46.737 | INFO | "malloc" : " 14 GB", | 20260508 07:35:46.737 | INFO | "pinned" : " 29 GB" | 20260508 07:35:46.737 | INFO | } | 20260508 07:35:46.737 | INFO | Perf report for the last 5.13369 seconds: | 20260508 07:35:46.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:46.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:35:46.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:35:46.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:46.737 | INFO | Uploaded 36GiB to GPU in 2.52326s (14.2673GiB/s) | 20260508 07:35:46.737 | INFO | Downloaded 34GiB to GPU in 2.44306s (13.917GiB/s) | 20260508 07:35:51.875 | INFO | Analyzing chunk frames = [6358016,6358528) | 20260508 07:35:51.875 | INFO | MemInfo: { | 20260508 07:35:51.875 | INFO | "gpu" : " 17 GB", | 20260508 07:35:51.875 | INFO | "huge" : "", | 20260508 07:35:51.875 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:51.875 | INFO | "malloc" : " 14 GB", | 20260508 07:35:51.875 | INFO | "pinned" : " 29 GB" | 20260508 07:35:51.875 | INFO | } | 20260508 07:35:51.875 | INFO | Perf report for the last 5.1381 seconds: | 20260508 07:35:51.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:51.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.875 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:35:51.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.875 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:35:51.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:51.876 | INFO | Uploaded 36GiB to GPU in 2.56027s (14.061GiB/s) | 20260508 07:35:51.876 | INFO | Downloaded 34GiB to GPU in 2.42705s (14.0088GiB/s) | 20260508 07:35:57.016 | INFO | Analyzing chunk frames = [6358528,6359040) | 20260508 07:35:57.016 | INFO | MemInfo: { | 20260508 07:35:57.016 | INFO | "gpu" : " 17 GB", | 20260508 07:35:57.016 | INFO | "huge" : "", | 20260508 07:35:57.016 | INFO | "huge_shared" : " 29 GB", | 20260508 07:35:57.016 | INFO | "malloc" : " 14 GB", | 20260508 07:35:57.016 | INFO | "pinned" : " 29 GB" | 20260508 07:35:57.016 | INFO | } | 20260508 07:35:57.016 | INFO | Perf report for the last 5.14095 seconds: | 20260508 07:35:57.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:35:57.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.016 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 07:35:57.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.016 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:35:57.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:35:57.017 | INFO | Uploaded 36GiB to GPU in 2.57086s (14.0031GiB/s) | 20260508 07:35:57.017 | INFO | Downloaded 34GiB to GPU in 2.41519s (14.0775GiB/s) | 20260508 07:36:02.156 | INFO | Analyzing chunk frames = [6359040,6359552) | 20260508 07:36:02.156 | INFO | MemInfo: { | 20260508 07:36:02.156 | INFO | "gpu" : " 17 GB", | 20260508 07:36:02.156 | INFO | "huge" : "", | 20260508 07:36:02.156 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:02.156 | INFO | "malloc" : " 14 GB", | 20260508 07:36:02.156 | INFO | "pinned" : " 29 GB" | 20260508 07:36:02.156 | INFO | } | 20260508 07:36:02.156 | INFO | Perf report for the last 5.13994 seconds: | 20260508 07:36:02.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:02.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.156 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:36:02.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.156 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 07:36:02.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:02.157 | INFO | Uploaded 36GiB to GPU in 2.28544s (15.7519GiB/s) | 20260508 07:36:02.157 | INFO | Downloaded 34GiB to GPU in 2.49029s (13.653GiB/s) | 20260508 07:36:07.296 | INFO | Analyzing chunk frames = [6359552,6360064) | 20260508 07:36:07.296 | INFO | MemInfo: { | 20260508 07:36:07.296 | INFO | "gpu" : " 17 GB", | 20260508 07:36:07.296 | INFO | "huge" : "", | 20260508 07:36:07.296 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:07.296 | INFO | "malloc" : " 14 GB", | 20260508 07:36:07.296 | INFO | "pinned" : " 29 GB" | 20260508 07:36:07.296 | INFO | } | 20260508 07:36:07.296 | INFO | Perf report for the last 5.13971 seconds: | 20260508 07:36:07.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:07.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:36:07.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:36:07.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:07.296 | INFO | Uploaded 36GiB to GPU in 2.46984s (14.5759GiB/s) | 20260508 07:36:07.296 | INFO | Downloaded 34GiB to GPU in 2.42006s (14.0492GiB/s) | 20260508 07:36:12.432 | INFO | Analyzing chunk frames = [6360064,6360576) | 20260508 07:36:12.432 | INFO | MemInfo: { | 20260508 07:36:12.432 | INFO | "gpu" : " 17 GB", | 20260508 07:36:12.432 | INFO | "huge" : "", | 20260508 07:36:12.432 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:12.432 | INFO | "malloc" : " 14 GB", | 20260508 07:36:12.432 | INFO | "pinned" : " 29 GB" | 20260508 07:36:12.432 | INFO | } | 20260508 07:36:12.432 | INFO | Perf report for the last 5.13649 seconds: | 20260508 07:36:12.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:12.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.432 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 07:36:12.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.432 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:36:12.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:12.433 | INFO | Uploaded 36GiB to GPU in 2.4276s (14.8294GiB/s) | 20260508 07:36:12.433 | INFO | Downloaded 34GiB to GPU in 2.4262s (14.0137GiB/s) | 20260508 07:36:17.577 | INFO | Analyzing chunk frames = [6360576,6361088) | 20260508 07:36:17.577 | INFO | MemInfo: { | 20260508 07:36:17.577 | INFO | "gpu" : " 17 GB", | 20260508 07:36:17.577 | INFO | "huge" : "", | 20260508 07:36:17.577 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:17.577 | INFO | "malloc" : " 14 GB", | 20260508 07:36:17.577 | INFO | "pinned" : " 29 GB" | 20260508 07:36:17.577 | INFO | } | 20260508 07:36:17.577 | INFO | Perf report for the last 5.14456 seconds: | 20260508 07:36:17.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:17.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 07:36:17.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260508 07:36:17.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:17.577 | INFO | Uploaded 36GiB to GPU in 2.49653s (14.42GiB/s) | 20260508 07:36:17.577 | INFO | Downloaded 34GiB to GPU in 2.44157s (13.9255GiB/s) | 20260508 07:36:17.674 | INFO | Starting a round of Baz IO | 20260508 07:36:21.084 | INFO | Populating RTMetrics report | 20260508 07:36:21.085 | INFO | RTMetrics report populated | 20260508 07:36:21.085 | INFO | Opening tmp RTMetrics json file | 20260508 07:36:21.097 | INFO | Serializing RTMetrics to json | 20260508 07:36:21.098 | INFO | Serialization done, swapping tmp file now... | 20260508 07:36:21.098 | INFO | RTMetrics done | 20260508 07:36:21.160 | INFO | Finished a round of Baz IO: Wrote 1.523 GiB in 3.486 seconds | 20260508 07:36:21.160 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.47% | 20260508 07:36:21.160 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.335439% / 3.97364e-06% / 6.35783e-05% | 20260508 07:36:21.160 | INFO | Baz IO overall: | 20260508 07:36:21.160 | INFO | BytesWritten : 2804148259852 | 20260508 07:36:21.160 | INFO | NumEvents : 1350867340354 | 20260508 07:36:22.713 | INFO | Analyzing chunk frames = [6361088,6361600) | 20260508 07:36:22.713 | INFO | MemInfo: { | 20260508 07:36:22.713 | INFO | "gpu" : " 17 GB", | 20260508 07:36:22.713 | INFO | "huge" : "", | 20260508 07:36:22.713 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:22.713 | INFO | "malloc" : " 14 GB", | 20260508 07:36:22.713 | INFO | "pinned" : " 29 GB" | 20260508 07:36:22.713 | INFO | } | 20260508 07:36:22.713 | INFO | Perf report for the last 5.13599 seconds: | 20260508 07:36:22.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:22.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:22.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:22.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:22.713 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 07:36:22.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:22.713 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 07:36:22.713 | INFO | PRE_HQ : 65.6%, 1.6, 55.3ms, 0 | 20260508 07:36:22.713 | INFO | BAZWRITER : 67.2%, 2.6, 90.2ms, 0 | 20260508 07:36:22.713 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:36:22.713 | INFO | Uploaded 36GiB to GPU in 2.59876s (13.8528GiB/s) | 20260508 07:36:22.713 | INFO | Downloaded 38GiB to GPU in 2.69358s (14.1076GiB/s) | 20260508 07:36:27.849 | INFO | Analyzing chunk frames = [6361600,6362112) | 20260508 07:36:27.850 | INFO | MemInfo: { | 20260508 07:36:27.850 | INFO | "gpu" : " 17 GB", | 20260508 07:36:27.850 | INFO | "huge" : "", | 20260508 07:36:27.850 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:27.850 | INFO | "malloc" : " 14 GB", | 20260508 07:36:27.850 | INFO | "pinned" : " 29 GB" | 20260508 07:36:27.850 | INFO | } | 20260508 07:36:27.850 | INFO | Perf report for the last 5.13695 seconds: | 20260508 07:36:27.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:27.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 07:36:27.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:36:27.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:27.850 | INFO | Uploaded 36GiB to GPU in 2.50144s (14.3917GiB/s) | 20260508 07:36:27.850 | INFO | Downloaded 34GiB to GPU in 2.43214s (13.9794GiB/s) | 20260508 07:36:32.987 | INFO | Analyzing chunk frames = [6362112,6362624) | 20260508 07:36:32.987 | INFO | MemInfo: { | 20260508 07:36:32.987 | INFO | "gpu" : " 17 GB", | 20260508 07:36:32.987 | INFO | "huge" : "", | 20260508 07:36:32.987 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:32.988 | INFO | "malloc" : " 14 GB", | 20260508 07:36:32.988 | INFO | "pinned" : " 29 GB" | 20260508 07:36:32.988 | INFO | } | 20260508 07:36:32.988 | INFO | Perf report for the last 5.1378 seconds: | 20260508 07:36:32.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:32.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:36:32.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:36:32.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:32.988 | INFO | Uploaded 36GiB to GPU in 2.5783s (13.9627GiB/s) | 20260508 07:36:32.988 | INFO | Downloaded 34GiB to GPU in 2.41018s (14.1068GiB/s) | 20260508 07:36:38.134 | INFO | Analyzing chunk frames = [6362624,6363136) | 20260508 07:36:38.134 | INFO | MemInfo: { | 20260508 07:36:38.134 | INFO | "gpu" : " 17 GB", | 20260508 07:36:38.134 | INFO | "huge" : "", | 20260508 07:36:38.134 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:38.134 | INFO | "malloc" : " 14 GB", | 20260508 07:36:38.134 | INFO | "pinned" : " 29 GB" | 20260508 07:36:38.134 | INFO | } | 20260508 07:36:38.134 | INFO | Perf report for the last 5.14638 seconds: | 20260508 07:36:38.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:38.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:36:38.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 07:36:38.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:38.134 | INFO | Uploaded 36GiB to GPU in 2.58868s (13.9067GiB/s) | 20260508 07:36:38.134 | INFO | Downloaded 34GiB to GPU in 2.42041s (14.0472GiB/s) | 20260508 07:36:43.271 | INFO | Analyzing chunk frames = [6363136,6363648) | 20260508 07:36:43.271 | INFO | MemInfo: { | 20260508 07:36:43.271 | INFO | "gpu" : " 17 GB", | 20260508 07:36:43.271 | INFO | "huge" : "", | 20260508 07:36:43.272 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:43.272 | INFO | "malloc" : " 14 GB", | 20260508 07:36:43.272 | INFO | "pinned" : " 29 GB" | 20260508 07:36:43.272 | INFO | } | 20260508 07:36:43.272 | INFO | Perf report for the last 5.13773 seconds: | 20260508 07:36:43.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:43.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 07:36:43.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:36:43.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:43.272 | INFO | Uploaded 36GiB to GPU in 2.29083s (15.7148GiB/s) | 20260508 07:36:43.272 | INFO | Downloaded 34GiB to GPU in 2.48012s (13.709GiB/s) | 20260508 07:36:48.413 | INFO | Analyzing chunk frames = [6363648,6364160) | 20260508 07:36:48.413 | INFO | MemInfo: { | 20260508 07:36:48.413 | INFO | "gpu" : " 17 GB", | 20260508 07:36:48.413 | INFO | "huge" : "", | 20260508 07:36:48.413 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:48.413 | INFO | "malloc" : " 14 GB", | 20260508 07:36:48.413 | INFO | "pinned" : " 29 GB" | 20260508 07:36:48.413 | INFO | } | 20260508 07:36:48.413 | INFO | Perf report for the last 5.14168 seconds: | 20260508 07:36:48.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:48.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.414 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:36:48.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.414 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:36:48.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:48.414 | INFO | Uploaded 36GiB to GPU in 2.48452s (14.4897GiB/s) | 20260508 07:36:48.414 | INFO | Downloaded 34GiB to GPU in 2.43104s (13.9858GiB/s) | 20260508 07:36:53.548 | INFO | Analyzing chunk frames = [6364160,6364672) | 20260508 07:36:53.548 | INFO | MemInfo: { | 20260508 07:36:53.548 | INFO | "gpu" : " 17 GB", | 20260508 07:36:53.548 | INFO | "huge" : "", | 20260508 07:36:53.548 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:53.548 | INFO | "malloc" : " 14 GB", | 20260508 07:36:53.548 | INFO | "pinned" : " 29 GB" | 20260508 07:36:53.548 | INFO | } | 20260508 07:36:53.548 | INFO | Perf report for the last 5.13455 seconds: | 20260508 07:36:53.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:53.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.548 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 07:36:53.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.548 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:36:53.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:53.549 | INFO | Uploaded 36GiB to GPU in 2.49942s (14.4033GiB/s) | 20260508 07:36:53.549 | INFO | Downloaded 34GiB to GPU in 2.43041s (13.9894GiB/s) | 20260508 07:36:58.687 | INFO | Analyzing chunk frames = [6364672,6365184) | 20260508 07:36:58.687 | INFO | MemInfo: { | 20260508 07:36:58.687 | INFO | "gpu" : " 17 GB", | 20260508 07:36:58.687 | INFO | "huge" : "", | 20260508 07:36:58.687 | INFO | "huge_shared" : " 29 GB", | 20260508 07:36:58.687 | INFO | "malloc" : " 14 GB", | 20260508 07:36:58.687 | INFO | "pinned" : " 29 GB" | 20260508 07:36:58.687 | INFO | } | 20260508 07:36:58.687 | INFO | Perf report for the last 5.13905 seconds: | 20260508 07:36:58.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:36:58.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:36:58.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:36:58.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:36:58.687 | INFO | Uploaded 36GiB to GPU in 2.50469s (14.373GiB/s) | 20260508 07:36:58.687 | INFO | Downloaded 34GiB to GPU in 2.43712s (13.9509GiB/s) | 20260508 07:36:58.687 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:36:58.687 | INFO | Total mem usage: 16.7 GiB | 20260508 07:36:58.687 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:36:58.687 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:36:58.687 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:36:58.687 | INFO | High Water Breakdown: | 20260508 07:36:58.687 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:36:58.687 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:36:58.687 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:36:58.687 | INFO | ------------------------------------------ | 20260508 07:36:58.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:36:58.687 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:36:58.687 | INFO | Total mem usage: 26.5 GiB | 20260508 07:36:58.687 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:36:58.687 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:36:58.687 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:36:58.687 | INFO | High Water Breakdown: | 20260508 07:36:58.687 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:36:58.687 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:36:58.687 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:36:58.687 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:36:58.687 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:36:58.687 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:36:58.687 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:36:58.687 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:36:58.687 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:36:58.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:36:58.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:36:58.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:36:58.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:36:58.687 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:36:58.687 | INFO | ------------------------------------------ | 20260508 07:36:58.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:36:58.687 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:36:58.687 | INFO | Total mem usage: 27.8 GiB | 20260508 07:36:58.687 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:36:58.687 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:36:58.687 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:36:58.687 | INFO | High Water Breakdown: | 20260508 07:36:58.687 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:36:58.687 | INFO | ------------------------------------------ | 20260508 07:36:58.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:36:58.688 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:36:58.688 | INFO | Total mem usage: 15.7 GiB | 20260508 07:36:58.688 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:36:58.688 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:36:58.688 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:36:58.688 | INFO | High Water Breakdown: | 20260508 07:36:58.688 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:36:58.688 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:36:58.688 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:36:58.688 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:36:58.688 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:36:58.688 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:36:58.688 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:36:58.688 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:36:58.688 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:36:58.688 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:36:58.688 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:36:58.688 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:36:58.688 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:36:58.688 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:36:58.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:36:58.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:36:58.688 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:36:58.688 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:36:58.688 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:36:58.688 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:36:58.688 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:36:58.688 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:36:58.688 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:36:58.688 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:36:58.688 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:36:58.688 | INFO | ------------------------------------------ | 20260508 07:36:58.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:36:58.688 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:36:58.688 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:36:58.688 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:36:58.688 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 5.88e-01 | 20260508 07:36:58.688 | INFO | PerfReport: 0.29%, AggregateStats -- 9.60e-02 / 1.03e-01, 1.11e-02 / 5.16e-03 | 20260508 07:36:58.688 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.45e+00, 5.88e-01 / 2.15e-01 | 20260508 07:36:58.688 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.46e+00, 2.07e-01 / 1.12e-01 | 20260508 07:36:58.688 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.52e+01, 3.81e+00 / 1.13e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.79e+00 / 1.37e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.37e+00 / 2.55e+00, 4.12e-01 / 2.61e-01 | 20260508 07:36:58.688 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.22e+00, 2.53e-01 / 1.48e-01 | 20260508 07:36:58.688 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.09e-01 / 3.49e-01, 4.96e-02 / 2.56e-02 | 20260508 07:36:58.688 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:36:58.688 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:36:58.688 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:36:58.688 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 24.52%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 24.24%, Download -- 2.57e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 30.76%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 2.43%, LockUpload -- 2.58e+00 / 0.00e+00, 5.81e+00 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 2.46%, LockDownload -- 2.61e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 15.57%, LockCompute -- 1.65e+01 / 0.00e+00, 1.95e+01 / 0.00e+00 | 20260508 07:36:58.688 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.08e-02 / 0.00e+00 | 20260508 07:36:58.786 | INFO | Starting a round of Baz IO | 20260508 07:37:02.197 | INFO | Populating RTMetrics report | 20260508 07:37:02.197 | INFO | RTMetrics report populated | 20260508 07:37:02.197 | INFO | Opening tmp RTMetrics json file | 20260508 07:37:02.208 | INFO | Serializing RTMetrics to json | 20260508 07:37:02.209 | INFO | Serialization done, swapping tmp file now... | 20260508 07:37:02.210 | INFO | RTMetrics done | 20260508 07:37:02.271 | INFO | Finished a round of Baz IO: Wrote 1.523 GiB in 3.484 seconds | 20260508 07:37:02.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.47% | 20260508 07:37:02.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.334632% / 0% / 5.5631e-05% | 20260508 07:37:02.271 | INFO | Baz IO overall: | 20260508 07:37:02.271 | INFO | BytesWritten : 2805784020101 | 20260508 07:37:02.271 | INFO | NumEvents : 1351644119402 | 20260508 07:37:03.828 | INFO | Analyzing chunk frames = [6365184,6365696) | 20260508 07:37:03.828 | INFO | MemInfo: { | 20260508 07:37:03.828 | INFO | "gpu" : " 17 GB", | 20260508 07:37:03.828 | INFO | "huge" : "", | 20260508 07:37:03.828 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:03.828 | INFO | "malloc" : " 14 GB", | 20260508 07:37:03.828 | INFO | "pinned" : " 29 GB" | 20260508 07:37:03.828 | INFO | } | 20260508 07:37:03.828 | INFO | Perf report for the last 5.14105 seconds: | 20260508 07:37:03.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:03.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:03.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:03.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:03.828 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 07:37:03.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:03.828 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 07:37:03.828 | INFO | PRE_HQ : 65.7%, 1.6, 55.4ms, 0 | 20260508 07:37:03.828 | INFO | BAZWRITER : 67.2%, 2.6, 91.1ms, 0 | 20260508 07:37:03.828 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:37:03.829 | INFO | Uploaded 36GiB to GPU in 2.60802s (13.8036GiB/s) | 20260508 07:37:03.829 | INFO | Downloaded 38GiB to GPU in 2.67937s (14.1824GiB/s) | 20260508 07:37:08.964 | INFO | Analyzing chunk frames = [6365696,6366208) | 20260508 07:37:08.964 | INFO | MemInfo: { | 20260508 07:37:08.964 | INFO | "gpu" : " 17 GB", | 20260508 07:37:08.964 | INFO | "huge" : "", | 20260508 07:37:08.964 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:08.964 | INFO | "malloc" : " 14 GB", | 20260508 07:37:08.964 | INFO | "pinned" : " 29 GB" | 20260508 07:37:08.964 | INFO | } | 20260508 07:37:08.965 | INFO | Perf report for the last 5.13633 seconds: | 20260508 07:37:08.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:08.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:37:08.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:37:08.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:08.965 | INFO | Uploaded 36GiB to GPU in 2.50133s (14.3924GiB/s) | 20260508 07:37:08.965 | INFO | Downloaded 34GiB to GPU in 2.42139s (14.0415GiB/s) | 20260508 07:37:14.104 | INFO | Analyzing chunk frames = [6366208,6366720) | 20260508 07:37:14.104 | INFO | MemInfo: { | 20260508 07:37:14.104 | INFO | "gpu" : " 17 GB", | 20260508 07:37:14.104 | INFO | "huge" : "", | 20260508 07:37:14.104 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:14.104 | INFO | "malloc" : " 14 GB", | 20260508 07:37:14.104 | INFO | "pinned" : " 29 GB" | 20260508 07:37:14.104 | INFO | } | 20260508 07:37:14.104 | INFO | Perf report for the last 5.13942 seconds: | 20260508 07:37:14.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:14.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 07:37:14.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260508 07:37:14.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:14.104 | INFO | Uploaded 36GiB to GPU in 2.58438s (13.9298GiB/s) | 20260508 07:37:14.104 | INFO | Downloaded 34GiB to GPU in 2.4291s (13.997GiB/s) | 20260508 07:37:19.245 | INFO | Analyzing chunk frames = [6366720,6367232) | 20260508 07:37:19.246 | INFO | MemInfo: { | 20260508 07:37:19.246 | INFO | "gpu" : " 17 GB", | 20260508 07:37:19.246 | INFO | "huge" : "", | 20260508 07:37:19.246 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:19.246 | INFO | "malloc" : " 14 GB", | 20260508 07:37:19.246 | INFO | "pinned" : " 29 GB" | 20260508 07:37:19.246 | INFO | } | 20260508 07:37:19.246 | INFO | Perf report for the last 5.14169 seconds: | 20260508 07:37:19.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:19.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | ANALYSIS : 66.8%, 3.0, 103.5ms, 0 | 20260508 07:37:19.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:37:19.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:19.246 | INFO | Uploaded 36GiB to GPU in 2.5864s (13.9189GiB/s) | 20260508 07:37:19.246 | INFO | Downloaded 34GiB to GPU in 2.40602s (14.1312GiB/s) | 20260508 07:37:24.385 | INFO | Analyzing chunk frames = [6367232,6367744) | 20260508 07:37:24.385 | INFO | MemInfo: { | 20260508 07:37:24.385 | INFO | "gpu" : " 17 GB", | 20260508 07:37:24.385 | INFO | "huge" : "", | 20260508 07:37:24.385 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:24.385 | INFO | "malloc" : " 14 GB", | 20260508 07:37:24.385 | INFO | "pinned" : " 29 GB" | 20260508 07:37:24.385 | INFO | } | 20260508 07:37:24.386 | INFO | Perf report for the last 5.13991 seconds: | 20260508 07:37:24.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:24.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 07:37:24.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:37:24.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:24.386 | INFO | Uploaded 36GiB to GPU in 2.40382s (14.9762GiB/s) | 20260508 07:37:24.386 | INFO | Downloaded 34GiB to GPU in 2.49726s (13.6149GiB/s) | 20260508 07:37:29.527 | INFO | Analyzing chunk frames = [6367744,6368256) | 20260508 07:37:29.527 | INFO | MemInfo: { | 20260508 07:37:29.527 | INFO | "gpu" : " 17 GB", | 20260508 07:37:29.527 | INFO | "huge" : "", | 20260508 07:37:29.527 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:29.527 | INFO | "malloc" : " 14 GB", | 20260508 07:37:29.527 | INFO | "pinned" : " 29 GB" | 20260508 07:37:29.527 | INFO | } | 20260508 07:37:29.527 | INFO | Perf report for the last 5.1416 seconds: | 20260508 07:37:29.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:29.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:37:29.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:37:29.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:29.527 | INFO | Uploaded 36GiB to GPU in 2.47036s (14.5728GiB/s) | 20260508 07:37:29.527 | INFO | Downloaded 34GiB to GPU in 2.426s (14.0148GiB/s) | 20260508 07:37:34.667 | INFO | Analyzing chunk frames = [6368256,6368768) | 20260508 07:37:34.667 | INFO | MemInfo: { | 20260508 07:37:34.667 | INFO | "gpu" : " 17 GB", | 20260508 07:37:34.667 | INFO | "huge" : "", | 20260508 07:37:34.667 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:34.667 | INFO | "malloc" : " 14 GB", | 20260508 07:37:34.667 | INFO | "pinned" : " 29 GB" | 20260508 07:37:34.667 | INFO | } | 20260508 07:37:34.667 | INFO | Perf report for the last 5.14001 seconds: | 20260508 07:37:34.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:34.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 07:37:34.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:37:34.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:34.667 | INFO | Uploaded 36GiB to GPU in 2.42841s (14.8245GiB/s) | 20260508 07:37:34.668 | INFO | Downloaded 34GiB to GPU in 2.41517s (14.0777GiB/s) | 20260508 07:37:39.798 | INFO | Analyzing chunk frames = [6368768,6369280) | 20260508 07:37:39.798 | INFO | MemInfo: { | 20260508 07:37:39.798 | INFO | "gpu" : " 17 GB", | 20260508 07:37:39.798 | INFO | "huge" : "", | 20260508 07:37:39.799 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:39.799 | INFO | "malloc" : " 14 GB", | 20260508 07:37:39.799 | INFO | "pinned" : " 29 GB" | 20260508 07:37:39.799 | INFO | } | 20260508 07:37:39.799 | INFO | Perf report for the last 5.13137 seconds: | 20260508 07:37:39.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:39.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.799 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 07:37:39.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.799 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 07:37:39.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:39.800 | INFO | Uploaded 36GiB to GPU in 2.5199s (14.2863GiB/s) | 20260508 07:37:39.800 | INFO | Downloaded 34GiB to GPU in 2.45013s (13.8768GiB/s) | 20260508 07:37:39.896 | INFO | Starting a round of Baz IO | 20260508 07:37:43.322 | INFO | Populating RTMetrics report | 20260508 07:37:43.323 | INFO | RTMetrics report populated | 20260508 07:37:43.323 | INFO | Opening tmp RTMetrics json file | 20260508 07:37:43.330 | INFO | Serializing RTMetrics to json | 20260508 07:37:43.331 | INFO | Serialization done, swapping tmp file now... | 20260508 07:37:43.332 | INFO | RTMetrics done | 20260508 07:37:43.391 | INFO | Finished a round of Baz IO: Wrote 1.523 GiB in 3.494 seconds | 20260508 07:37:43.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.47% | 20260508 07:37:43.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.334426% / 3.97364e-06% / 4.37101e-05% | 20260508 07:37:43.391 | INFO | Baz IO overall: | 20260508 07:37:43.391 | INFO | BytesWritten : 2807419824502 | 20260508 07:37:43.391 | INFO | NumEvents : 1352420917048 | 20260508 07:37:44.940 | INFO | Analyzing chunk frames = [6369280,6369792) | 20260508 07:37:44.940 | INFO | MemInfo: { | 20260508 07:37:44.940 | INFO | "gpu" : " 17 GB", | 20260508 07:37:44.940 | INFO | "huge" : "", | 20260508 07:37:44.940 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:44.940 | INFO | "malloc" : " 14 GB", | 20260508 07:37:44.940 | INFO | "pinned" : " 29 GB" | 20260508 07:37:44.940 | INFO | } | 20260508 07:37:44.940 | INFO | Perf report for the last 5.14158 seconds: | 20260508 07:37:44.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:44.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:44.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:44.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:44.940 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 07:37:44.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:44.940 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 07:37:44.940 | INFO | PRE_HQ : 65.3%, 1.6, 54.9ms, 0 | 20260508 07:37:44.940 | INFO | BAZWRITER : 67.4%, 2.6, 90.4ms, 0 | 20260508 07:37:44.940 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:37:44.940 | INFO | Uploaded 36GiB to GPU in 2.61605s (13.7612GiB/s) | 20260508 07:37:44.940 | INFO | Downloaded 38GiB to GPU in 2.69096s (14.1214GiB/s) | 20260508 07:37:50.080 | INFO | Analyzing chunk frames = [6369792,6370304) | 20260508 07:37:50.080 | INFO | MemInfo: { | 20260508 07:37:50.080 | INFO | "gpu" : " 17 GB", | 20260508 07:37:50.080 | INFO | "huge" : "", | 20260508 07:37:50.080 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:50.080 | INFO | "malloc" : " 14 GB", | 20260508 07:37:50.081 | INFO | "pinned" : " 29 GB" | 20260508 07:37:50.081 | INFO | } | 20260508 07:37:50.081 | INFO | Perf report for the last 5.14052 seconds: | 20260508 07:37:50.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:50.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | ANALYSIS : 67.0%, 3.0, 104.5ms, 0 | 20260508 07:37:50.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260508 07:37:50.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:50.081 | INFO | Uploaded 36GiB to GPU in 2.50615s (14.3646GiB/s) | 20260508 07:37:50.081 | INFO | Downloaded 34GiB to GPU in 2.4282s (14.0022GiB/s) | 20260508 07:37:55.222 | INFO | Analyzing chunk frames = [6370304,6370816) | 20260508 07:37:55.222 | INFO | MemInfo: { | 20260508 07:37:55.222 | INFO | "gpu" : " 17 GB", | 20260508 07:37:55.222 | INFO | "huge" : "", | 20260508 07:37:55.222 | INFO | "huge_shared" : " 29 GB", | 20260508 07:37:55.223 | INFO | "malloc" : " 14 GB", | 20260508 07:37:55.223 | INFO | "pinned" : " 29 GB" | 20260508 07:37:55.223 | INFO | } | 20260508 07:37:55.223 | INFO | Perf report for the last 5.1419 seconds: | 20260508 07:37:55.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:37:55.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 07:37:55.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:37:55.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:37:55.223 | INFO | Uploaded 36GiB to GPU in 2.58058s (13.9503GiB/s) | 20260508 07:37:55.223 | INFO | Downloaded 34GiB to GPU in 2.40957s (14.1104GiB/s) | 20260508 07:38:00.366 | INFO | Analyzing chunk frames = [6370816,6371328) | 20260508 07:38:00.367 | INFO | MemInfo: { | 20260508 07:38:00.367 | INFO | "gpu" : " 17 GB", | 20260508 07:38:00.367 | INFO | "huge" : "", | 20260508 07:38:00.367 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:00.367 | INFO | "malloc" : " 14 GB", | 20260508 07:38:00.367 | INFO | "pinned" : " 29 GB" | 20260508 07:38:00.367 | INFO | } | 20260508 07:38:00.367 | INFO | Perf report for the last 5.14398 seconds: | 20260508 07:38:00.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:00.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 07:38:00.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:38:00.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:00.367 | INFO | Uploaded 36GiB to GPU in 2.59162s (13.8909GiB/s) | 20260508 07:38:00.367 | INFO | Downloaded 34GiB to GPU in 2.4087s (14.1155GiB/s) | 20260508 07:38:05.499 | INFO | Analyzing chunk frames = [6371328,6371840) | 20260508 07:38:05.499 | INFO | MemInfo: { | 20260508 07:38:05.499 | INFO | "gpu" : " 17 GB", | 20260508 07:38:05.499 | INFO | "huge" : "", | 20260508 07:38:05.499 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:05.499 | INFO | "malloc" : " 14 GB", | 20260508 07:38:05.499 | INFO | "pinned" : " 29 GB" | 20260508 07:38:05.499 | INFO | } | 20260508 07:38:05.499 | INFO | Perf report for the last 5.13225 seconds: | 20260508 07:38:05.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:05.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 07:38:05.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 07:38:05.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:05.499 | INFO | Uploaded 36GiB to GPU in 2.28458s (15.7578GiB/s) | 20260508 07:38:05.500 | INFO | Downloaded 34GiB to GPU in 2.50143s (13.5922GiB/s) | 20260508 07:38:10.640 | INFO | Analyzing chunk frames = [6371840,6372352) | 20260508 07:38:10.640 | INFO | MemInfo: { | 20260508 07:38:10.640 | INFO | "gpu" : " 17 GB", | 20260508 07:38:10.640 | INFO | "huge" : "", | 20260508 07:38:10.640 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:10.641 | INFO | "malloc" : " 14 GB", | 20260508 07:38:10.641 | INFO | "pinned" : " 29 GB" | 20260508 07:38:10.641 | INFO | } | 20260508 07:38:10.641 | INFO | Perf report for the last 5.14183 seconds: | 20260508 07:38:10.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:10.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:38:10.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:38:10.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:10.641 | INFO | Uploaded 36GiB to GPU in 2.37755s (15.1417GiB/s) | 20260508 07:38:10.641 | INFO | Downloaded 34GiB to GPU in 2.43682s (13.9526GiB/s) | 20260508 07:38:15.780 | INFO | Analyzing chunk frames = [6372352,6372864) | 20260508 07:38:15.780 | INFO | MemInfo: { | 20260508 07:38:15.780 | INFO | "gpu" : " 17 GB", | 20260508 07:38:15.780 | INFO | "huge" : "", | 20260508 07:38:15.780 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:15.780 | INFO | "malloc" : " 14 GB", | 20260508 07:38:15.780 | INFO | "pinned" : " 29 GB" | 20260508 07:38:15.780 | INFO | } | 20260508 07:38:15.780 | INFO | Perf report for the last 5.13965 seconds: | 20260508 07:38:15.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:15.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.780 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:38:15.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.781 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 07:38:15.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:15.781 | INFO | Uploaded 36GiB to GPU in 2.48682s (14.4763GiB/s) | 20260508 07:38:15.781 | INFO | Downloaded 34GiB to GPU in 2.44063s (13.9308GiB/s) | 20260508 07:38:20.919 | INFO | Analyzing chunk frames = [6372864,6373376) | 20260508 07:38:20.919 | INFO | MemInfo: { | 20260508 07:38:20.919 | INFO | "gpu" : " 17 GB", | 20260508 07:38:20.919 | INFO | "huge" : "", | 20260508 07:38:20.919 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:20.919 | INFO | "malloc" : " 14 GB", | 20260508 07:38:20.919 | INFO | "pinned" : " 29 GB" | 20260508 07:38:20.919 | INFO | } | 20260508 07:38:20.919 | INFO | Perf report for the last 5.13836 seconds: | 20260508 07:38:20.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:20.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 07:38:20.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:38:20.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:20.919 | INFO | Uploaded 36GiB to GPU in 2.43279s (14.7978GiB/s) | 20260508 07:38:20.919 | INFO | Downloaded 34GiB to GPU in 2.4409s (13.9293GiB/s) | 20260508 07:38:20.919 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:38:20.919 | INFO | Total mem usage: 16.7 GiB | 20260508 07:38:20.919 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:38:20.919 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:38:20.919 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:38:20.919 | INFO | High Water Breakdown: | 20260508 07:38:20.919 | INFO | 59.4% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:38:20.919 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:38:20.919 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:38:20.919 | INFO | ------------------------------------------ | 20260508 07:38:20.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:38:20.919 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:38:20.919 | INFO | Total mem usage: 26.5 GiB | 20260508 07:38:20.919 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:38:20.919 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:38:20.919 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:38:20.919 | INFO | High Water Breakdown: | 20260508 07:38:20.919 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:38:20.919 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:38:20.919 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:38:20.919 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:38:20.919 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:38:20.919 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:38:20.919 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:38:20.919 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:38:20.919 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:38:20.919 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:38:20.919 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:38:20.919 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:38:20.919 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:38:20.919 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:38:20.919 | INFO | ------------------------------------------ | 20260508 07:38:20.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:38:20.919 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:38:20.919 | INFO | Total mem usage: 27.8 GiB | 20260508 07:38:20.919 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:38:20.919 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:38:20.919 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:38:20.919 | INFO | High Water Breakdown: | 20260508 07:38:20.919 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:38:20.919 | INFO | ------------------------------------------ | 20260508 07:38:20.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:38:20.919 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:38:20.919 | INFO | Total mem usage: 15.7 GiB | 20260508 07:38:20.919 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:38:20.919 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:38:20.920 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:38:20.920 | INFO | High Water Breakdown: | 20260508 07:38:20.920 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:38:20.920 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:38:20.920 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:38:20.920 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:38:20.920 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:38:20.920 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:38:20.920 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:38:20.920 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:38:20.920 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:38:20.920 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:38:20.920 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:38:20.920 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:38:20.921 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:38:20.921 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:38:20.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:38:20.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:38:20.921 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:38:20.921 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:38:20.921 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:38:20.921 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:38:20.921 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:38:20.921 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:38:20.921 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:38:20.921 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:38:20.921 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:38:20.921 | INFO | ------------------------------------------ | 20260508 07:38:20.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:38:20.921 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:38:20.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:38:20.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:38:20.921 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 6.82e-01 | 20260508 07:38:20.921 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.10e-02 / 4.46e-03 | 20260508 07:38:20.921 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.47e+00, 5.85e-01 / 1.80e-01 | 20260508 07:38:20.921 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.18e-01 | 20260508 07:38:20.921 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.50e+01, 3.65e+00 / 8.85e-01 | 20260508 07:38:20.921 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.81e+00 / 1.47e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.55e+00, 4.06e-01 / 2.70e-01 | 20260508 07:38:20.921 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.21e+00, 2.53e-01 / 1.45e-01 | 20260508 07:38:20.921 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.09e-01 / 3.42e-01, 5.00e-02 / 2.72e-02 | 20260508 07:38:20.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:38:20.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:38:20.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:38:20.921 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.52e+01 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 24.56%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 24.26%, Download -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 2.46%, LockUpload -- 2.60e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 2.41%, LockDownload -- 2.56e+00 / 0.00e+00, 3.85e+00 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 15.46%, LockCompute -- 1.64e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 07:38:20.921 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260508 07:38:21.016 | INFO | Starting a round of Baz IO | 20260508 07:38:24.442 | INFO | Populating RTMetrics report | 20260508 07:38:24.442 | INFO | RTMetrics report populated | 20260508 07:38:24.442 | INFO | Opening tmp RTMetrics json file | 20260508 07:38:24.455 | INFO | Serializing RTMetrics to json | 20260508 07:38:24.456 | INFO | Serialization done, swapping tmp file now... | 20260508 07:38:24.457 | INFO | RTMetrics done | 20260508 07:38:24.510 | INFO | Finished a round of Baz IO: Wrote 1.524 GiB in 3.493 seconds | 20260508 07:38:24.510 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.47% | 20260508 07:38:24.510 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.334883% / 3.97364e-06% / 5.5631e-05% | 20260508 07:38:24.510 | INFO | Baz IO overall: | 20260508 07:38:24.510 | INFO | BytesWritten : 2809056164937 | 20260508 07:38:24.510 | INFO | NumEvents : 1353198259181 | 20260508 07:38:26.058 | INFO | Analyzing chunk frames = [6373376,6373888) | 20260508 07:38:26.058 | INFO | MemInfo: { | 20260508 07:38:26.058 | INFO | "gpu" : " 17 GB", | 20260508 07:38:26.058 | INFO | "huge" : "", | 20260508 07:38:26.058 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:26.058 | INFO | "malloc" : " 14 GB", | 20260508 07:38:26.058 | INFO | "pinned" : " 29 GB" | 20260508 07:38:26.058 | INFO | } | 20260508 07:38:26.058 | INFO | Perf report for the last 5.13968 seconds: | 20260508 07:38:26.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:26.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:26.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:26.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:26.059 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 07:38:26.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:26.059 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 07:38:26.059 | INFO | PRE_HQ : 66.6%, 1.7, 59.0ms, 0 | 20260508 07:38:26.059 | INFO | BAZWRITER : 67.6%, 2.7, 94.0ms, 0 | 20260508 07:38:26.059 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:38:26.059 | INFO | Uploaded 36GiB to GPU in 2.53943s (14.1764GiB/s) | 20260508 07:38:26.059 | INFO | Downloaded 38GiB to GPU in 2.70957s (14.0244GiB/s) | 20260508 07:38:31.198 | INFO | Analyzing chunk frames = [6373888,6374400) | 20260508 07:38:31.198 | INFO | MemInfo: { | 20260508 07:38:31.198 | INFO | "gpu" : " 17 GB", | 20260508 07:38:31.198 | INFO | "huge" : "", | 20260508 07:38:31.198 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:31.198 | INFO | "malloc" : " 14 GB", | 20260508 07:38:31.198 | INFO | "pinned" : " 29 GB" | 20260508 07:38:31.198 | INFO | } | 20260508 07:38:31.198 | INFO | Perf report for the last 5.13954 seconds: | 20260508 07:38:31.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:31.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:38:31.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:38:31.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:31.198 | INFO | Uploaded 36GiB to GPU in 2.53424s (14.2055GiB/s) | 20260508 07:38:31.198 | INFO | Downloaded 34GiB to GPU in 2.44324s (13.9159GiB/s) | 20260508 07:38:36.337 | INFO | Analyzing chunk frames = [6374400,6374912) | 20260508 07:38:36.337 | INFO | MemInfo: { | 20260508 07:38:36.337 | INFO | "gpu" : " 17 GB", | 20260508 07:38:36.337 | INFO | "huge" : "", | 20260508 07:38:36.337 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:36.337 | INFO | "malloc" : " 14 GB", | 20260508 07:38:36.337 | INFO | "pinned" : " 29 GB" | 20260508 07:38:36.337 | INFO | } | 20260508 07:38:36.337 | INFO | Perf report for the last 5.13943 seconds: | 20260508 07:38:36.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:36.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.338 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 07:38:36.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.338 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:38:36.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:36.338 | INFO | Uploaded 36GiB to GPU in 2.54746s (14.1317GiB/s) | 20260508 07:38:36.338 | INFO | Downloaded 34GiB to GPU in 2.43726s (13.9501GiB/s) | 20260508 07:38:41.476 | INFO | Analyzing chunk frames = [6374912,6375424) | 20260508 07:38:41.476 | INFO | MemInfo: { | 20260508 07:38:41.476 | INFO | "gpu" : " 17 GB", | 20260508 07:38:41.476 | INFO | "huge" : "", | 20260508 07:38:41.476 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:41.476 | INFO | "malloc" : " 14 GB", | 20260508 07:38:41.476 | INFO | "pinned" : " 29 GB" | 20260508 07:38:41.476 | INFO | } | 20260508 07:38:41.476 | INFO | Perf report for the last 5.13859 seconds: | 20260508 07:38:41.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:41.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.476 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 07:38:41.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.476 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:38:41.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:41.477 | INFO | Uploaded 36GiB to GPU in 2.56687s (14.0249GiB/s) | 20260508 07:38:41.477 | INFO | Downloaded 34GiB to GPU in 2.42399s (14.0265GiB/s) | 20260508 07:38:46.615 | INFO | Analyzing chunk frames = [6375424,6375936) | 20260508 07:38:46.615 | INFO | MemInfo: { | 20260508 07:38:46.615 | INFO | "gpu" : " 17 GB", | 20260508 07:38:46.615 | INFO | "huge" : "", | 20260508 07:38:46.616 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:46.616 | INFO | "malloc" : " 14 GB", | 20260508 07:38:46.616 | INFO | "pinned" : " 29 GB" | 20260508 07:38:46.616 | INFO | } | 20260508 07:38:46.616 | INFO | Perf report for the last 5.13963 seconds: | 20260508 07:38:46.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:46.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:38:46.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:38:46.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:46.616 | INFO | Uploaded 36GiB to GPU in 2.28242s (15.7727GiB/s) | 20260508 07:38:46.616 | INFO | Downloaded 34GiB to GPU in 2.49627s (13.6203GiB/s) | 20260508 07:38:51.751 | INFO | Analyzing chunk frames = [6375936,6376448) | 20260508 07:38:51.752 | INFO | MemInfo: { | 20260508 07:38:51.752 | INFO | "gpu" : " 17 GB", | 20260508 07:38:51.752 | INFO | "huge" : "", | 20260508 07:38:51.752 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:51.752 | INFO | "malloc" : " 14 GB", | 20260508 07:38:51.752 | INFO | "pinned" : " 29 GB" | 20260508 07:38:51.752 | INFO | } | 20260508 07:38:51.752 | INFO | Perf report for the last 5.13612 seconds: | 20260508 07:38:51.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:51.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:38:51.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:38:51.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:51.752 | INFO | Uploaded 36GiB to GPU in 2.39501s (15.0312GiB/s) | 20260508 07:38:51.752 | INFO | Downloaded 34GiB to GPU in 2.42123s (14.0424GiB/s) | 20260508 07:38:56.892 | INFO | Analyzing chunk frames = [6376448,6376960) | 20260508 07:38:56.892 | INFO | MemInfo: { | 20260508 07:38:56.892 | INFO | "gpu" : " 17 GB", | 20260508 07:38:56.892 | INFO | "huge" : "", | 20260508 07:38:56.892 | INFO | "huge_shared" : " 29 GB", | 20260508 07:38:56.892 | INFO | "malloc" : " 14 GB", | 20260508 07:38:56.892 | INFO | "pinned" : " 29 GB" | 20260508 07:38:56.892 | INFO | } | 20260508 07:38:56.893 | INFO | Perf report for the last 5.14067 seconds: | 20260508 07:38:56.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:38:56.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 07:38:56.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:38:56.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:38:56.893 | INFO | Uploaded 36GiB to GPU in 2.49445s (14.432GiB/s) | 20260508 07:38:56.893 | INFO | Downloaded 34GiB to GPU in 2.42916s (13.9966GiB/s) | 20260508 07:39:02.038 | INFO | Analyzing chunk frames = [6376960,6377472) | 20260508 07:39:02.038 | INFO | MemInfo: { | 20260508 07:39:02.038 | INFO | "gpu" : " 17 GB", | 20260508 07:39:02.038 | INFO | "huge" : "", | 20260508 07:39:02.038 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:02.038 | INFO | "malloc" : " 14 GB", | 20260508 07:39:02.039 | INFO | "pinned" : " 29 GB" | 20260508 07:39:02.039 | INFO | } | 20260508 07:39:02.039 | INFO | Perf report for the last 5.14614 seconds: | 20260508 07:39:02.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:02.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 07:39:02.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260508 07:39:02.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:02.039 | INFO | Uploaded 36GiB to GPU in 2.50958s (14.3451GiB/s) | 20260508 07:39:02.039 | INFO | Downloaded 34GiB to GPU in 2.44965s (13.8795GiB/s) | 20260508 07:39:02.153 | INFO | Starting a round of Baz IO | 20260508 07:39:05.568 | INFO | Populating RTMetrics report | 20260508 07:39:05.568 | INFO | RTMetrics report populated | 20260508 07:39:05.568 | INFO | Opening tmp RTMetrics json file | 20260508 07:39:05.577 | INFO | Serializing RTMetrics to json | 20260508 07:39:05.581 | INFO | Serialization done, swapping tmp file now... | 20260508 07:39:05.582 | INFO | RTMetrics done | 20260508 07:39:05.635 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.482 seconds | 20260508 07:39:05.635 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.00% / 59.47% | 20260508 07:39:05.635 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.333639% / 0% / 5.16574e-05% | 20260508 07:39:05.635 | INFO | Baz IO overall: | 20260508 07:39:05.635 | INFO | BytesWritten : 2810690381743 | 20260508 07:39:05.635 | INFO | NumEvents : 1353973562031 | 20260508 07:39:07.175 | INFO | Analyzing chunk frames = [6377472,6377984) | 20260508 07:39:07.175 | INFO | MemInfo: { | 20260508 07:39:07.175 | INFO | "gpu" : " 17 GB", | 20260508 07:39:07.175 | INFO | "huge" : "", | 20260508 07:39:07.176 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:07.176 | INFO | "malloc" : " 14 GB", | 20260508 07:39:07.176 | INFO | "pinned" : " 29 GB" | 20260508 07:39:07.176 | INFO | } | 20260508 07:39:07.176 | INFO | Perf report for the last 5.13696 seconds: | 20260508 07:39:07.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:07.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:07.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:07.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:07.176 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 07:39:07.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:07.176 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260508 07:39:07.176 | INFO | PRE_HQ : 66.7%, 1.8, 62.4ms, 0 | 20260508 07:39:07.176 | INFO | BAZWRITER : 67.3%, 2.8, 97.1ms, 0 | 20260508 07:39:07.176 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:39:07.176 | INFO | Uploaded 36GiB to GPU in 2.59528s (13.8713GiB/s) | 20260508 07:39:07.176 | INFO | Downloaded 38GiB to GPU in 2.70537s (14.0462GiB/s) | 20260508 07:39:12.310 | INFO | Analyzing chunk frames = [6377984,6378496) | 20260508 07:39:12.310 | INFO | MemInfo: { | 20260508 07:39:12.310 | INFO | "gpu" : " 17 GB", | 20260508 07:39:12.310 | INFO | "huge" : "", | 20260508 07:39:12.310 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:12.310 | INFO | "malloc" : " 14 GB", | 20260508 07:39:12.310 | INFO | "pinned" : " 29 GB" | 20260508 07:39:12.310 | INFO | } | 20260508 07:39:12.310 | INFO | Perf report for the last 5.13411 seconds: | 20260508 07:39:12.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:12.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:39:12.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:39:12.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:12.310 | INFO | Uploaded 36GiB to GPU in 2.51018s (14.3416GiB/s) | 20260508 07:39:12.310 | INFO | Downloaded 34GiB to GPU in 2.44567s (13.9021GiB/s) | 20260508 07:39:17.450 | INFO | Analyzing chunk frames = [6378496,6379008) | 20260508 07:39:17.450 | INFO | MemInfo: { | 20260508 07:39:17.450 | INFO | "gpu" : " 17 GB", | 20260508 07:39:17.450 | INFO | "huge" : "", | 20260508 07:39:17.450 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:17.450 | INFO | "malloc" : " 14 GB", | 20260508 07:39:17.450 | INFO | "pinned" : " 29 GB" | 20260508 07:39:17.450 | INFO | } | 20260508 07:39:17.450 | INFO | Perf report for the last 5.14058 seconds: | 20260508 07:39:17.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:17.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.450 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:39:17.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.450 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:39:17.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:17.451 | INFO | Uploaded 36GiB to GPU in 2.56439s (14.0384GiB/s) | 20260508 07:39:17.451 | INFO | Downloaded 34GiB to GPU in 2.42332s (14.0303GiB/s) | 20260508 07:39:22.592 | INFO | Analyzing chunk frames = [6379008,6379520) | 20260508 07:39:22.592 | INFO | MemInfo: { | 20260508 07:39:22.592 | INFO | "gpu" : " 17 GB", | 20260508 07:39:22.592 | INFO | "huge" : "", | 20260508 07:39:22.592 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:22.593 | INFO | "malloc" : " 14 GB", | 20260508 07:39:22.593 | INFO | "pinned" : " 29 GB" | 20260508 07:39:22.593 | INFO | } | 20260508 07:39:22.593 | INFO | Perf report for the last 5.14221 seconds: | 20260508 07:39:22.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:22.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 07:39:22.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:39:22.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:22.593 | INFO | Uploaded 36GiB to GPU in 2.59135s (13.8924GiB/s) | 20260508 07:39:22.593 | INFO | Downloaded 34GiB to GPU in 2.41007s (14.1075GiB/s) | 20260508 07:39:27.732 | INFO | Analyzing chunk frames = [6379520,6380032) | 20260508 07:39:27.732 | INFO | MemInfo: { | 20260508 07:39:27.732 | INFO | "gpu" : " 17 GB", | 20260508 07:39:27.732 | INFO | "huge" : "", | 20260508 07:39:27.732 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:27.732 | INFO | "malloc" : " 14 GB", | 20260508 07:39:27.732 | INFO | "pinned" : " 29 GB" | 20260508 07:39:27.732 | INFO | } | 20260508 07:39:27.732 | INFO | Perf report for the last 5.13944 seconds: | 20260508 07:39:27.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:27.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:39:27.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 07:39:27.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:27.732 | INFO | Uploaded 36GiB to GPU in 2.30151s (15.6419GiB/s) | 20260508 07:39:27.732 | INFO | Downloaded 34GiB to GPU in 2.50194s (13.5895GiB/s) | 20260508 07:39:32.867 | INFO | Analyzing chunk frames = [6380032,6380544) | 20260508 07:39:32.867 | INFO | MemInfo: { | 20260508 07:39:32.867 | INFO | "gpu" : " 17 GB", | 20260508 07:39:32.867 | INFO | "huge" : "", | 20260508 07:39:32.867 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:32.867 | INFO | "malloc" : " 14 GB", | 20260508 07:39:32.867 | INFO | "pinned" : " 29 GB" | 20260508 07:39:32.867 | INFO | } | 20260508 07:39:32.867 | INFO | Perf report for the last 5.13519 seconds: | 20260508 07:39:32.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:32.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.867 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 07:39:32.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.867 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 07:39:32.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:32.868 | INFO | Uploaded 36GiB to GPU in 2.39691s (15.0194GiB/s) | 20260508 07:39:32.868 | INFO | Downloaded 34GiB to GPU in 2.42043s (14.0471GiB/s) | 20260508 07:39:38.005 | INFO | Analyzing chunk frames = [6380544,6381056) | 20260508 07:39:38.005 | INFO | MemInfo: { | 20260508 07:39:38.005 | INFO | "gpu" : " 17 GB", | 20260508 07:39:38.005 | INFO | "huge" : "", | 20260508 07:39:38.005 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:38.005 | INFO | "malloc" : " 14 GB", | 20260508 07:39:38.005 | INFO | "pinned" : " 29 GB" | 20260508 07:39:38.005 | INFO | } | 20260508 07:39:38.005 | INFO | Perf report for the last 5.13805 seconds: | 20260508 07:39:38.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:38.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.006 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:39:38.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.006 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 07:39:38.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:38.006 | INFO | Uploaded 36GiB to GPU in 2.5057s (14.3672GiB/s) | 20260508 07:39:38.006 | INFO | Downloaded 34GiB to GPU in 2.42658s (14.0115GiB/s) | 20260508 07:39:43.153 | INFO | Analyzing chunk frames = [6381056,6381568) | 20260508 07:39:43.153 | INFO | MemInfo: { | 20260508 07:39:43.153 | INFO | "gpu" : " 17 GB", | 20260508 07:39:43.153 | INFO | "huge" : "", | 20260508 07:39:43.153 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:43.153 | INFO | "malloc" : " 14 GB", | 20260508 07:39:43.153 | INFO | "pinned" : " 29 GB" | 20260508 07:39:43.153 | INFO | } | 20260508 07:39:43.153 | INFO | Perf report for the last 5.1477 seconds: | 20260508 07:39:43.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:43.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | ANALYSIS : 66.4%, 3.0, 103.9ms, 0 | 20260508 07:39:43.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 07:39:43.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:43.153 | INFO | Uploaded 36GiB to GPU in 2.51601s (14.3084GiB/s) | 20260508 07:39:43.154 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:39:43.154 | INFO | Total mem usage: 16.7 GiB | 20260508 07:39:43.154 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:39:43.154 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:39:43.154 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:39:43.154 | INFO | High Water Breakdown: | 20260508 07:39:43.154 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:39:43.154 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:39:43.154 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:39:43.154 | INFO | ------------------------------------------ | 20260508 07:39:43.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:39:43.154 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:39:43.154 | INFO | Total mem usage: 26.5 GiB | 20260508 07:39:43.154 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:39:43.154 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:39:43.154 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:39:43.154 | INFO | High Water Breakdown: | 20260508 07:39:43.154 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:39:43.154 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:39:43.154 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:39:43.154 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:39:43.154 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:39:43.154 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:39:43.154 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:39:43.154 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:39:43.154 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:39:43.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:39:43.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:39:43.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:39:43.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:39:43.154 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:39:43.154 | INFO | ------------------------------------------ | 20260508 07:39:43.154 | INFO | Downloaded 34GiB to GPU in 2.45177s (13.8675GiB/s) | 20260508 07:39:43.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:39:43.154 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:39:43.154 | INFO | Total mem usage: 27.8 GiB | 20260508 07:39:43.154 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:39:43.154 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:39:43.154 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:39:43.154 | INFO | High Water Breakdown: | 20260508 07:39:43.154 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:39:43.154 | INFO | ------------------------------------------ | 20260508 07:39:43.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:39:43.154 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:39:43.154 | INFO | Total mem usage: 15.7 GiB | 20260508 07:39:43.154 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:39:43.154 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:39:43.154 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:39:43.154 | INFO | High Water Breakdown: | 20260508 07:39:43.154 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:39:43.154 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:39:43.154 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:39:43.154 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:39:43.154 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:39:43.154 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:39:43.154 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:39:43.154 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:39:43.154 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:39:43.154 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:39:43.154 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:39:43.154 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:39:43.154 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:39:43.154 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:39:43.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:39:43.155 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:39:43.155 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:39:43.155 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:39:43.155 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:39:43.155 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:39:43.155 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:39:43.155 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:39:43.155 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:39:43.155 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:39:43.155 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:39:43.155 | INFO | ------------------------------------------ | 20260508 07:39:43.155 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:39:43.155 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:39:43.155 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:39:43.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:39:43.155 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 6.47e-01 | 20260508 07:39:43.155 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.21e-02 / 3.93e-03 | 20260508 07:39:43.155 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.46e+00, 5.86e-01 / 2.08e-01 | 20260508 07:39:43.155 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.42e+00, 2.10e-01 / 1.24e-01 | 20260508 07:39:43.155 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.54e+01, 3.83e+00 / 1.22e+00 | 20260508 07:39:43.155 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.11e+00 | 20260508 07:39:43.155 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.57e+00, 4.07e-01 / 1.25e-01 | 20260508 07:39:43.155 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.17e+00, 2.50e-01 / 7.70e-02 | 20260508 07:39:43.155 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.11e-01 / 3.41e-01, 4.99e-02 / 1.04e-02 | 20260508 07:39:43.155 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:39:43.155 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:39:43.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:39:43.155 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:39:43.155 | INFO | PerfReport: 24.47%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 07:39:43.155 | INFO | PerfReport: 24.31%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 07:39:43.155 | INFO | PerfReport: 30.84%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:39:43.156 | INFO | PerfReport: 2.41%, LockUpload -- 2.55e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 07:39:43.156 | INFO | PerfReport: 2.46%, LockDownload -- 2.60e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 07:39:43.156 | INFO | PerfReport: 15.51%, LockCompute -- 1.64e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 07:39:43.156 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260508 07:39:43.266 | INFO | Starting a round of Baz IO | 20260508 07:39:46.673 | INFO | Populating RTMetrics report | 20260508 07:39:46.673 | INFO | RTMetrics report populated | 20260508 07:39:46.673 | INFO | Opening tmp RTMetrics json file | 20260508 07:39:46.688 | INFO | Serializing RTMetrics to json | 20260508 07:39:46.689 | INFO | Serialization done, swapping tmp file now... | 20260508 07:39:46.689 | INFO | RTMetrics done | 20260508 07:39:46.743 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.477 seconds | 20260508 07:39:46.743 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.52% / 0.00% / 59.47% | 20260508 07:39:46.743 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.333365% / 0% / 4.37101e-05% | 20260508 07:39:46.743 | INFO | Baz IO overall: | 20260508 07:39:46.743 | INFO | BytesWritten : 2812324801604 | 20260508 07:39:46.743 | INFO | NumEvents : 1354749409225 | 20260508 07:39:48.292 | INFO | Analyzing chunk frames = [6381568,6382080) | 20260508 07:39:48.292 | INFO | MemInfo: { | 20260508 07:39:48.292 | INFO | "gpu" : " 17 GB", | 20260508 07:39:48.292 | INFO | "huge" : "", | 20260508 07:39:48.292 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:48.292 | INFO | "malloc" : " 14 GB", | 20260508 07:39:48.292 | INFO | "pinned" : " 29 GB" | 20260508 07:39:48.292 | INFO | } | 20260508 07:39:48.292 | INFO | Perf report for the last 5.1388 seconds: | 20260508 07:39:48.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:48.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:48.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:48.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:48.292 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 07:39:48.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:48.292 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260508 07:39:48.292 | INFO | PRE_HQ : 66.5%, 1.8, 63.2ms, 0 | 20260508 07:39:48.292 | INFO | BAZWRITER : 67.0%, 2.8, 97.9ms, 0 | 20260508 07:39:48.292 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 07:39:48.292 | INFO | Uploaded 36GiB to GPU in 2.57318s (13.9904GiB/s) | 20260508 07:39:48.292 | INFO | Downloaded 38GiB to GPU in 2.6988s (14.0803GiB/s) | 20260508 07:39:53.430 | INFO | Analyzing chunk frames = [6382080,6382592) | 20260508 07:39:53.430 | INFO | MemInfo: { | 20260508 07:39:53.430 | INFO | "gpu" : " 17 GB", | 20260508 07:39:53.430 | INFO | "huge" : "", | 20260508 07:39:53.430 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:53.430 | INFO | "malloc" : " 14 GB", | 20260508 07:39:53.430 | INFO | "pinned" : " 29 GB" | 20260508 07:39:53.430 | INFO | } | 20260508 07:39:53.430 | INFO | Perf report for the last 5.13835 seconds: | 20260508 07:39:53.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:53.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.430 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:39:53.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.430 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:39:53.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:53.431 | INFO | Uploaded 36GiB to GPU in 2.52305s (14.2685GiB/s) | 20260508 07:39:53.431 | INFO | Downloaded 34GiB to GPU in 2.43553s (13.96GiB/s) | 20260508 07:39:58.565 | INFO | Analyzing chunk frames = [6382592,6383104) | 20260508 07:39:58.565 | INFO | MemInfo: { | 20260508 07:39:58.565 | INFO | "gpu" : " 17 GB", | 20260508 07:39:58.565 | INFO | "huge" : "", | 20260508 07:39:58.565 | INFO | "huge_shared" : " 29 GB", | 20260508 07:39:58.565 | INFO | "malloc" : " 14 GB", | 20260508 07:39:58.565 | INFO | "pinned" : " 29 GB" | 20260508 07:39:58.565 | INFO | } | 20260508 07:39:58.565 | INFO | Perf report for the last 5.13506 seconds: | 20260508 07:39:58.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:39:58.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.565 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 07:39:58.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.565 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:39:58.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:39:58.566 | INFO | Uploaded 36GiB to GPU in 2.5132s (14.3244GiB/s) | 20260508 07:39:58.566 | INFO | Downloaded 34GiB to GPU in 2.43806s (13.9455GiB/s) | 20260508 07:40:03.703 | INFO | Analyzing chunk frames = [6383104,6383616) | 20260508 07:40:03.703 | INFO | MemInfo: { | 20260508 07:40:03.703 | INFO | "gpu" : " 17 GB", | 20260508 07:40:03.703 | INFO | "huge" : "", | 20260508 07:40:03.703 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:03.703 | INFO | "malloc" : " 14 GB", | 20260508 07:40:03.703 | INFO | "pinned" : " 29 GB" | 20260508 07:40:03.703 | INFO | } | 20260508 07:40:03.703 | INFO | Perf report for the last 5.13798 seconds: | 20260508 07:40:03.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:03.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.703 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 07:40:03.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.703 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:40:03.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:03.704 | INFO | Uploaded 36GiB to GPU in 2.58606s (13.9208GiB/s) | 20260508 07:40:03.704 | INFO | Downloaded 34GiB to GPU in 2.42337s (14.03GiB/s) | 20260508 07:40:08.841 | INFO | Analyzing chunk frames = [6383616,6384128) | 20260508 07:40:08.841 | INFO | MemInfo: { | 20260508 07:40:08.841 | INFO | "gpu" : " 17 GB", | 20260508 07:40:08.841 | INFO | "huge" : "", | 20260508 07:40:08.841 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:08.842 | INFO | "malloc" : " 14 GB", | 20260508 07:40:08.842 | INFO | "pinned" : " 29 GB" | 20260508 07:40:08.842 | INFO | } | 20260508 07:40:08.842 | INFO | Perf report for the last 5.13844 seconds: | 20260508 07:40:08.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:08.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:40:08.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:40:08.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:08.842 | INFO | Uploaded 36GiB to GPU in 2.28619s (15.7467GiB/s) | 20260508 07:40:08.842 | INFO | Downloaded 34GiB to GPU in 2.48414s (13.6868GiB/s) | 20260508 07:40:13.984 | INFO | Analyzing chunk frames = [6384128,6384640) | 20260508 07:40:13.984 | INFO | MemInfo: { | 20260508 07:40:13.984 | INFO | "gpu" : " 17 GB", | 20260508 07:40:13.984 | INFO | "huge" : "", | 20260508 07:40:13.984 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:13.984 | INFO | "malloc" : " 14 GB", | 20260508 07:40:13.984 | INFO | "pinned" : " 29 GB" | 20260508 07:40:13.984 | INFO | } | 20260508 07:40:13.984 | INFO | Perf report for the last 5.1425 seconds: | 20260508 07:40:13.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:13.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 07:40:13.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:40:13.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:13.984 | INFO | Uploaded 36GiB to GPU in 2.48251s (14.5014GiB/s) | 20260508 07:40:13.984 | INFO | Downloaded 34GiB to GPU in 2.43638s (13.9551GiB/s) | 20260508 07:40:19.123 | INFO | Analyzing chunk frames = [6384640,6385152) | 20260508 07:40:19.123 | INFO | MemInfo: { | 20260508 07:40:19.123 | INFO | "gpu" : " 17 GB", | 20260508 07:40:19.123 | INFO | "huge" : "", | 20260508 07:40:19.123 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:19.123 | INFO | "malloc" : " 14 GB", | 20260508 07:40:19.124 | INFO | "pinned" : " 29 GB" | 20260508 07:40:19.124 | INFO | } | 20260508 07:40:19.124 | INFO | Perf report for the last 5.13949 seconds: | 20260508 07:40:19.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:19.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:40:19.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:40:19.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:19.124 | INFO | Uploaded 36GiB to GPU in 2.4814s (14.508GiB/s) | 20260508 07:40:19.124 | INFO | Downloaded 34GiB to GPU in 2.42736s (14.007GiB/s) | 20260508 07:40:24.264 | INFO | Analyzing chunk frames = [6385152,6385664) | 20260508 07:40:24.264 | INFO | MemInfo: { | 20260508 07:40:24.264 | INFO | "gpu" : " 17 GB", | 20260508 07:40:24.264 | INFO | "huge" : "", | 20260508 07:40:24.264 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:24.264 | INFO | "malloc" : " 14 GB", | 20260508 07:40:24.264 | INFO | "pinned" : " 29 GB" | 20260508 07:40:24.264 | INFO | } | 20260508 07:40:24.264 | INFO | Perf report for the last 5.1403 seconds: | 20260508 07:40:24.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:24.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 07:40:24.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:40:24.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:24.264 | INFO | Uploaded 36GiB to GPU in 2.51549s (14.3113GiB/s) | 20260508 07:40:24.264 | INFO | Downloaded 34GiB to GPU in 2.44566s (13.9022GiB/s) | 20260508 07:40:24.361 | INFO | Starting a round of Baz IO | 20260508 07:40:27.781 | INFO | Populating RTMetrics report | 20260508 07:40:27.781 | INFO | RTMetrics report populated | 20260508 07:40:27.781 | INFO | Opening tmp RTMetrics json file | 20260508 07:40:27.792 | INFO | Serializing RTMetrics to json | 20260508 07:40:27.793 | INFO | Serialization done, swapping tmp file now... | 20260508 07:40:27.794 | INFO | RTMetrics done | 20260508 07:40:27.856 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.494 seconds | 20260508 07:40:27.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.52% / 0.00% / 59.48% | 20260508 07:40:27.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.332729% / 3.97364e-06% / 5.96046e-05% | 20260508 07:40:27.856 | INFO | Baz IO overall: | 20260508 07:40:27.856 | INFO | BytesWritten : 2813958963330 | 20260508 07:40:27.856 | INFO | NumEvents : 1355525225097 | 20260508 07:40:29.405 | INFO | Analyzing chunk frames = [6385664,6386176) | 20260508 07:40:29.405 | INFO | MemInfo: { | 20260508 07:40:29.405 | INFO | "gpu" : " 17 GB", | 20260508 07:40:29.405 | INFO | "huge" : "", | 20260508 07:40:29.405 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:29.405 | INFO | "malloc" : " 14 GB", | 20260508 07:40:29.405 | INFO | "pinned" : " 29 GB" | 20260508 07:40:29.406 | INFO | } | 20260508 07:40:29.406 | INFO | Perf report for the last 5.14157 seconds: | 20260508 07:40:29.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:29.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:29.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:29.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:29.406 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 07:40:29.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:29.406 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:40:29.406 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 07:40:29.406 | INFO | BAZWRITER : 67.4%, 2.6, 89.5ms, 0 | 20260508 07:40:29.406 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:40:29.406 | INFO | Uploaded 36GiB to GPU in 2.59116s (13.8934GiB/s) | 20260508 07:40:29.406 | INFO | Downloaded 38GiB to GPU in 2.71671s (13.9875GiB/s) | 20260508 07:40:34.543 | INFO | Analyzing chunk frames = [6386176,6386688) | 20260508 07:40:34.543 | INFO | MemInfo: { | 20260508 07:40:34.543 | INFO | "gpu" : " 17 GB", | 20260508 07:40:34.543 | INFO | "huge" : "", | 20260508 07:40:34.543 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:34.543 | INFO | "malloc" : " 14 GB", | 20260508 07:40:34.543 | INFO | "pinned" : " 29 GB" | 20260508 07:40:34.543 | INFO | } | 20260508 07:40:34.543 | INFO | Perf report for the last 5.13781 seconds: | 20260508 07:40:34.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:34.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.543 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:40:34.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.544 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 07:40:34.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:34.544 | INFO | Uploaded 36GiB to GPU in 2.49674s (14.4188GiB/s) | 20260508 07:40:34.544 | INFO | Downloaded 34GiB to GPU in 2.43209s (13.9797GiB/s) | 20260508 07:40:39.681 | INFO | Analyzing chunk frames = [6386688,6387200) | 20260508 07:40:39.681 | INFO | MemInfo: { | 20260508 07:40:39.681 | INFO | "gpu" : " 17 GB", | 20260508 07:40:39.681 | INFO | "huge" : "", | 20260508 07:40:39.681 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:39.681 | INFO | "malloc" : " 14 GB", | 20260508 07:40:39.681 | INFO | "pinned" : " 29 GB" | 20260508 07:40:39.681 | INFO | } | 20260508 07:40:39.681 | INFO | Perf report for the last 5.13774 seconds: | 20260508 07:40:39.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:39.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | ANALYSIS : 67.2%, 3.0, 104.3ms, 0 | 20260508 07:40:39.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:40:39.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:39.681 | INFO | Uploaded 36GiB to GPU in 2.57219s (13.9959GiB/s) | 20260508 07:40:39.682 | INFO | Downloaded 34GiB to GPU in 2.4221s (14.0374GiB/s) | 20260508 07:40:44.815 | INFO | Analyzing chunk frames = [6387200,6387712) | 20260508 07:40:44.815 | INFO | MemInfo: { | 20260508 07:40:44.815 | INFO | "gpu" : " 17 GB", | 20260508 07:40:44.815 | INFO | "huge" : "", | 20260508 07:40:44.815 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:44.815 | INFO | "malloc" : " 14 GB", | 20260508 07:40:44.815 | INFO | "pinned" : " 29 GB" | 20260508 07:40:44.815 | INFO | } | 20260508 07:40:44.815 | INFO | Perf report for the last 5.13368 seconds: | 20260508 07:40:44.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:44.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 07:40:44.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:40:44.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:44.815 | INFO | Uploaded 36GiB to GPU in 2.59119s (13.8932GiB/s) | 20260508 07:40:44.815 | INFO | Downloaded 34GiB to GPU in 2.40946s (14.1111GiB/s) | 20260508 07:40:49.961 | INFO | Analyzing chunk frames = [6387712,6388224) | 20260508 07:40:49.961 | INFO | MemInfo: { | 20260508 07:40:49.961 | INFO | "gpu" : " 17 GB", | 20260508 07:40:49.961 | INFO | "huge" : "", | 20260508 07:40:49.961 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:49.961 | INFO | "malloc" : " 14 GB", | 20260508 07:40:49.961 | INFO | "pinned" : " 29 GB" | 20260508 07:40:49.961 | INFO | } | 20260508 07:40:49.961 | INFO | Perf report for the last 5.14619 seconds: | 20260508 07:40:49.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:49.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.961 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 07:40:49.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.961 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:40:49.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:49.962 | INFO | Uploaded 36GiB to GPU in 2.29893s (15.6595GiB/s) | 20260508 07:40:49.962 | INFO | Downloaded 34GiB to GPU in 2.5047s (13.5745GiB/s) | 20260508 07:40:55.108 | INFO | Analyzing chunk frames = [6388224,6388736) | 20260508 07:40:55.108 | INFO | MemInfo: { | 20260508 07:40:55.108 | INFO | "gpu" : " 17 GB", | 20260508 07:40:55.108 | INFO | "huge" : "", | 20260508 07:40:55.108 | INFO | "huge_shared" : " 29 GB", | 20260508 07:40:55.108 | INFO | "malloc" : " 14 GB", | 20260508 07:40:55.108 | INFO | "pinned" : " 29 GB" | 20260508 07:40:55.108 | INFO | } | 20260508 07:40:55.108 | INFO | Perf report for the last 5.14685 seconds: | 20260508 07:40:55.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:40:55.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.108 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:40:55.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.108 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 07:40:55.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:40:55.109 | INFO | Uploaded 36GiB to GPU in 2.36707s (15.2087GiB/s) | 20260508 07:40:55.109 | INFO | Downloaded 34GiB to GPU in 2.4344s (13.9665GiB/s) | 20260508 07:41:00.244 | INFO | Analyzing chunk frames = [6388736,6389248) | 20260508 07:41:00.244 | INFO | MemInfo: { | 20260508 07:41:00.244 | INFO | "gpu" : " 17 GB", | 20260508 07:41:00.244 | INFO | "huge" : "", | 20260508 07:41:00.244 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:00.244 | INFO | "malloc" : " 14 GB", | 20260508 07:41:00.244 | INFO | "pinned" : " 29 GB" | 20260508 07:41:00.244 | INFO | } | 20260508 07:41:00.244 | INFO | Perf report for the last 5.13635 seconds: | 20260508 07:41:00.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:00.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.244 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 07:41:00.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.244 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:41:00.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:00.245 | INFO | Uploaded 36GiB to GPU in 2.40371s (14.9768GiB/s) | 20260508 07:41:00.245 | INFO | Downloaded 34GiB to GPU in 2.43326s (13.973GiB/s) | 20260508 07:41:05.377 | INFO | Analyzing chunk frames = [6389248,6389760) | 20260508 07:41:05.377 | INFO | MemInfo: { | 20260508 07:41:05.377 | INFO | "gpu" : " 17 GB", | 20260508 07:41:05.377 | INFO | "huge" : "", | 20260508 07:41:05.377 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:05.377 | INFO | "malloc" : " 14 GB", | 20260508 07:41:05.377 | INFO | "pinned" : " 29 GB" | 20260508 07:41:05.377 | INFO | } | 20260508 07:41:05.377 | INFO | Perf report for the last 5.13309 seconds: | 20260508 07:41:05.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:05.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.377 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 07:41:05.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.378 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:41:05.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:05.378 | INFO | Uploaded 36GiB to GPU in 2.49548s (14.4261GiB/s) | 20260508 07:41:05.378 | INFO | Downloaded 34GiB to GPU in 2.46276s (13.8056GiB/s) | 20260508 07:41:05.378 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:41:05.378 | INFO | Total mem usage: 16.7 GiB | 20260508 07:41:05.378 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:41:05.378 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.9%) | 20260508 07:41:05.378 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:41:05.378 | INFO | High Water Breakdown: | 20260508 07:41:05.378 | INFO | 59.7% : 8.59 GiB ( 83.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:41:05.378 | INFO | 40.2% : 5.79 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:41:05.378 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:41:05.378 | INFO | ------------------------------------------ | 20260508 07:41:05.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:41:05.378 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:41:05.378 | INFO | Total mem usage: 26.5 GiB | 20260508 07:41:05.378 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:41:05.378 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 07:41:05.378 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 07:41:05.378 | INFO | High Water Breakdown: | 20260508 07:41:05.378 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:41:05.378 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:41:05.378 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:41:05.378 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:41:05.378 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:41:05.378 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:41:05.378 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:41:05.378 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:41:05.378 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:41:05.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:41:05.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:41:05.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:41:05.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:41:05.379 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:41:05.379 | INFO | ------------------------------------------ | 20260508 07:41:05.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:41:05.379 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:41:05.379 | INFO | Total mem usage: 27.8 GiB | 20260508 07:41:05.379 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:41:05.379 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:41:05.379 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:41:05.379 | INFO | High Water Breakdown: | 20260508 07:41:05.379 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:41:05.379 | INFO | ------------------------------------------ | 20260508 07:41:05.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:41:05.379 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:41:05.379 | INFO | Total mem usage: 15.7 GiB | 20260508 07:41:05.379 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:41:05.379 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:41:05.379 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:41:05.379 | INFO | High Water Breakdown: | 20260508 07:41:05.379 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:41:05.379 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:41:05.379 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:41:05.379 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:41:05.379 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:41:05.379 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:41:05.379 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:41:05.379 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:41:05.379 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:41:05.379 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:41:05.379 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:41:05.379 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:41:05.379 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:41:05.379 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:41:05.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:41:05.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:41:05.379 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:41:05.379 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:41:05.379 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:41:05.379 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:41:05.379 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:41:05.379 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:41:05.379 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:41:05.379 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:41:05.379 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:41:05.379 | INFO | ------------------------------------------ | 20260508 07:41:05.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:41:05.379 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:41:05.379 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:41:05.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:41:05.379 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 7.05e-01 | 20260508 07:41:05.379 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.04e-01, 1.11e-02 / 5.98e-03 | 20260508 07:41:05.379 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.51e+00, 5.95e-01 / 1.87e-01 | 20260508 07:41:05.379 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.08e-01 / 1.37e-01 | 20260508 07:41:05.379 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.52e+01, 3.80e+00 / 1.01e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.35e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.37e+00 / 2.54e+00, 4.14e-01 / 2.80e-01 | 20260508 07:41:05.379 | INFO | PerfReport: 3.12%, Metrics -- 1.02e+00 / 1.20e+00, 2.55e-01 / 1.38e-01 | 20260508 07:41:05.379 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.46e-01, 5.06e-02 / 2.61e-02 | 20260508 07:41:05.379 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:41:05.379 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:41:05.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:41:05.379 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.51e+01 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 24.44%, Upload -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 24.34%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 30.83%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 2.43%, LockUpload -- 2.57e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 2.43%, LockDownload -- 2.57e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 15.52%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:41:05.379 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260508 07:41:05.474 | INFO | Starting a round of Baz IO | 20260508 07:41:08.895 | INFO | Populating RTMetrics report | 20260508 07:41:08.895 | INFO | RTMetrics report populated | 20260508 07:41:08.895 | INFO | Opening tmp RTMetrics json file | 20260508 07:41:08.906 | INFO | Serializing RTMetrics to json | 20260508 07:41:08.907 | INFO | Serialization done, swapping tmp file now... | 20260508 07:41:08.907 | INFO | RTMetrics done | 20260508 07:41:08.962 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.488 seconds | 20260508 07:41:08.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.52% / 0.00% / 59.48% | 20260508 07:41:08.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330834% / 7.94729e-06% / 5.96046e-05% | 20260508 07:41:08.962 | INFO | Baz IO overall: | 20260508 07:41:08.963 | INFO | BytesWritten : 2815592938202 | 20260508 07:41:08.963 | INFO | NumEvents : 1356300725346 | 20260508 07:41:10.516 | INFO | Analyzing chunk frames = [6389760,6390272) | 20260508 07:41:10.516 | INFO | MemInfo: { | 20260508 07:41:10.516 | INFO | "gpu" : " 17 GB", | 20260508 07:41:10.516 | INFO | "huge" : "", | 20260508 07:41:10.516 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:10.516 | INFO | "malloc" : " 14 GB", | 20260508 07:41:10.516 | INFO | "pinned" : " 29 GB" | 20260508 07:41:10.516 | INFO | } | 20260508 07:41:10.516 | INFO | Perf report for the last 5.13871 seconds: | 20260508 07:41:10.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:10.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:10.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:10.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:10.516 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 07:41:10.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:10.516 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:41:10.516 | INFO | PRE_HQ : 65.7%, 1.6, 54.8ms, 0 | 20260508 07:41:10.516 | INFO | BAZWRITER : 67.2%, 2.6, 90.2ms, 0 | 20260508 07:41:10.516 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:41:10.517 | INFO | Uploaded 36GiB to GPU in 2.53322s (14.2112GiB/s) | 20260508 07:41:10.517 | INFO | Downloaded 38GiB to GPU in 2.70513s (14.0474GiB/s) | 20260508 07:41:15.657 | INFO | Analyzing chunk frames = [6390272,6390784) | 20260508 07:41:15.657 | INFO | MemInfo: { | 20260508 07:41:15.657 | INFO | "gpu" : " 17 GB", | 20260508 07:41:15.657 | INFO | "huge" : "", | 20260508 07:41:15.657 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:15.657 | INFO | "malloc" : " 14 GB", | 20260508 07:41:15.657 | INFO | "pinned" : " 29 GB" | 20260508 07:41:15.657 | INFO | } | 20260508 07:41:15.657 | INFO | Perf report for the last 5.14088 seconds: | 20260508 07:41:15.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:15.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.657 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:41:15.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.657 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:41:15.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:15.658 | INFO | Uploaded 36GiB to GPU in 2.48223s (14.5031GiB/s) | 20260508 07:41:15.658 | INFO | Downloaded 34GiB to GPU in 2.45327s (13.8591GiB/s) | 20260508 07:41:20.795 | INFO | Analyzing chunk frames = [6390784,6391296) | 20260508 07:41:20.795 | INFO | MemInfo: { | 20260508 07:41:20.795 | INFO | "gpu" : " 17 GB", | 20260508 07:41:20.795 | INFO | "huge" : "", | 20260508 07:41:20.795 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:20.795 | INFO | "malloc" : " 14 GB", | 20260508 07:41:20.795 | INFO | "pinned" : " 29 GB" | 20260508 07:41:20.795 | INFO | } | 20260508 07:41:20.795 | INFO | Perf report for the last 5.13843 seconds: | 20260508 07:41:20.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:20.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.796 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 07:41:20.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.796 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:41:20.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:20.796 | INFO | Uploaded 36GiB to GPU in 2.51945s (14.2889GiB/s) | 20260508 07:41:20.796 | INFO | Downloaded 34GiB to GPU in 2.45484s (13.8502GiB/s) | 20260508 07:41:25.939 | INFO | Analyzing chunk frames = [6391296,6391808) | 20260508 07:41:25.939 | INFO | MemInfo: { | 20260508 07:41:25.939 | INFO | "gpu" : " 17 GB", | 20260508 07:41:25.939 | INFO | "huge" : "", | 20260508 07:41:25.939 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:25.939 | INFO | "malloc" : " 14 GB", | 20260508 07:41:25.939 | INFO | "pinned" : " 29 GB" | 20260508 07:41:25.940 | INFO | } | 20260508 07:41:25.940 | INFO | Perf report for the last 5.14426 seconds: | 20260508 07:41:25.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:25.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 07:41:25.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:41:25.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:25.940 | INFO | Uploaded 36GiB to GPU in 2.53542s (14.1988GiB/s) | 20260508 07:41:25.940 | INFO | Downloaded 34GiB to GPU in 2.4328s (13.9757GiB/s) | 20260508 07:41:31.074 | INFO | Analyzing chunk frames = [6391808,6392320) | 20260508 07:41:31.074 | INFO | MemInfo: { | 20260508 07:41:31.074 | INFO | "gpu" : " 17 GB", | 20260508 07:41:31.074 | INFO | "huge" : "", | 20260508 07:41:31.074 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:31.074 | INFO | "malloc" : " 14 GB", | 20260508 07:41:31.074 | INFO | "pinned" : " 29 GB" | 20260508 07:41:31.074 | INFO | } | 20260508 07:41:31.074 | INFO | Perf report for the last 5.13449 seconds: | 20260508 07:41:31.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:31.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.074 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 07:41:31.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.074 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:41:31.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:31.075 | INFO | Uploaded 36GiB to GPU in 2.38522s (15.0929GiB/s) | 20260508 07:41:31.075 | INFO | Downloaded 34GiB to GPU in 2.54028s (13.3843GiB/s) | 20260508 07:41:36.218 | INFO | Analyzing chunk frames = [6392320,6392832) | 20260508 07:41:36.218 | INFO | MemInfo: { | 20260508 07:41:36.218 | INFO | "gpu" : " 17 GB", | 20260508 07:41:36.218 | INFO | "huge" : "", | 20260508 07:41:36.218 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:36.218 | INFO | "malloc" : " 14 GB", | 20260508 07:41:36.218 | INFO | "pinned" : " 29 GB" | 20260508 07:41:36.218 | INFO | } | 20260508 07:41:36.218 | INFO | Perf report for the last 5.14377 seconds: | 20260508 07:41:36.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:36.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 07:41:36.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:41:36.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:36.218 | INFO | Uploaded 36GiB to GPU in 2.43774s (14.7678GiB/s) | 20260508 07:41:36.218 | INFO | Downloaded 34GiB to GPU in 2.44919s (13.8822GiB/s) | 20260508 07:41:41.357 | INFO | Analyzing chunk frames = [6392832,6393344) | 20260508 07:41:41.357 | INFO | MemInfo: { | 20260508 07:41:41.357 | INFO | "gpu" : " 17 GB", | 20260508 07:41:41.357 | INFO | "huge" : "", | 20260508 07:41:41.357 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:41.357 | INFO | "malloc" : " 14 GB", | 20260508 07:41:41.357 | INFO | "pinned" : " 29 GB" | 20260508 07:41:41.357 | INFO | } | 20260508 07:41:41.357 | INFO | Perf report for the last 5.13914 seconds: | 20260508 07:41:41.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:41.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 07:41:41.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:41:41.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:41.357 | INFO | Uploaded 36GiB to GPU in 2.43833s (14.7642GiB/s) | 20260508 07:41:41.357 | INFO | Downloaded 34GiB to GPU in 2.4509s (13.8725GiB/s) | 20260508 07:41:46.497 | INFO | Analyzing chunk frames = [6393344,6393856) | 20260508 07:41:46.497 | INFO | MemInfo: { | 20260508 07:41:46.497 | INFO | "gpu" : " 17 GB", | 20260508 07:41:46.497 | INFO | "huge" : "", | 20260508 07:41:46.497 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:46.497 | INFO | "malloc" : " 14 GB", | 20260508 07:41:46.498 | INFO | "pinned" : " 29 GB" | 20260508 07:41:46.498 | INFO | } | 20260508 07:41:46.498 | INFO | Perf report for the last 5.14045 seconds: | 20260508 07:41:46.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:46.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 07:41:46.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:41:46.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:46.498 | INFO | Uploaded 36GiB to GPU in 2.40166s (14.9896GiB/s) | 20260508 07:41:46.498 | INFO | Downloaded 34GiB to GPU in 2.46333s (13.8024GiB/s) | 20260508 07:41:46.596 | INFO | Starting a round of Baz IO | 20260508 07:41:50.000 | INFO | Populating RTMetrics report | 20260508 07:41:50.000 | INFO | RTMetrics report populated | 20260508 07:41:50.000 | INFO | Opening tmp RTMetrics json file | 20260508 07:41:50.010 | INFO | Serializing RTMetrics to json | 20260508 07:41:50.011 | INFO | Serialization done, swapping tmp file now... | 20260508 07:41:50.011 | INFO | RTMetrics done | 20260508 07:41:50.072 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.476 seconds | 20260508 07:41:50.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.52% / 0.00% / 59.48% | 20260508 07:41:50.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.331418% / 3.97364e-06% / 3.97364e-05% | 20260508 07:41:50.072 | INFO | Baz IO overall: | 20260508 07:41:50.072 | INFO | BytesWritten : 2817226681480 | 20260508 07:41:50.072 | INFO | NumEvents : 1357076030205 | 20260508 07:41:51.630 | INFO | Analyzing chunk frames = [6393856,6394368) | 20260508 07:41:51.630 | INFO | MemInfo: { | 20260508 07:41:51.630 | INFO | "gpu" : " 17 GB", | 20260508 07:41:51.630 | INFO | "huge" : "", | 20260508 07:41:51.630 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:51.630 | INFO | "malloc" : " 14 GB", | 20260508 07:41:51.630 | INFO | "pinned" : " 29 GB" | 20260508 07:41:51.630 | INFO | } | 20260508 07:41:51.631 | INFO | Perf report for the last 5.13287 seconds: | 20260508 07:41:51.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:51.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:51.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:51.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:51.631 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 07:41:51.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:51.631 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 07:41:51.631 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 07:41:51.631 | INFO | BAZWRITER : 67.3%, 2.6, 89.8ms, 0 | 20260508 07:41:51.631 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:41:51.631 | INFO | Uploaded 36GiB to GPU in 2.55937s (14.066GiB/s) | 20260508 07:41:51.631 | INFO | Downloaded 38GiB to GPU in 2.7069s (14.0382GiB/s) | 20260508 07:41:56.770 | INFO | Analyzing chunk frames = [6394368,6394880) | 20260508 07:41:56.770 | INFO | MemInfo: { | 20260508 07:41:56.770 | INFO | "gpu" : " 17 GB", | 20260508 07:41:56.770 | INFO | "huge" : "", | 20260508 07:41:56.770 | INFO | "huge_shared" : " 29 GB", | 20260508 07:41:56.770 | INFO | "malloc" : " 14 GB", | 20260508 07:41:56.770 | INFO | "pinned" : " 29 GB" | 20260508 07:41:56.770 | INFO | } | 20260508 07:41:56.770 | INFO | Perf report for the last 5.13968 seconds: | 20260508 07:41:56.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:41:56.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.770 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 07:41:56.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.770 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:41:56.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:41:56.771 | INFO | Uploaded 36GiB to GPU in 2.47848s (14.525GiB/s) | 20260508 07:41:56.771 | INFO | Downloaded 34GiB to GPU in 2.43605s (13.957GiB/s) | 20260508 07:42:01.908 | INFO | Analyzing chunk frames = [6394880,6395392) | 20260508 07:42:01.908 | INFO | MemInfo: { | 20260508 07:42:01.908 | INFO | "gpu" : " 17 GB", | 20260508 07:42:01.908 | INFO | "huge" : "", | 20260508 07:42:01.908 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:01.908 | INFO | "malloc" : " 14 GB", | 20260508 07:42:01.909 | INFO | "pinned" : " 29 GB" | 20260508 07:42:01.909 | INFO | } | 20260508 07:42:01.909 | INFO | Perf report for the last 5.13832 seconds: | 20260508 07:42:01.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:01.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:42:01.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 07:42:01.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:01.909 | INFO | Uploaded 36GiB to GPU in 2.56942s (14.0109GiB/s) | 20260508 07:42:01.909 | INFO | Downloaded 34GiB to GPU in 2.4457s (13.902GiB/s) | 20260508 07:42:07.055 | INFO | Analyzing chunk frames = [6395392,6395904) | 20260508 07:42:07.055 | INFO | MemInfo: { | 20260508 07:42:07.055 | INFO | "gpu" : " 17 GB", | 20260508 07:42:07.055 | INFO | "huge" : "", | 20260508 07:42:07.055 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:07.055 | INFO | "malloc" : " 14 GB", | 20260508 07:42:07.055 | INFO | "pinned" : " 29 GB" | 20260508 07:42:07.055 | INFO | } | 20260508 07:42:07.055 | INFO | Perf report for the last 5.14671 seconds: | 20260508 07:42:07.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:07.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.055 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 07:42:07.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.055 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:42:07.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:07.056 | INFO | Uploaded 36GiB to GPU in 2.55918s (14.067GiB/s) | 20260508 07:42:07.056 | INFO | Downloaded 34GiB to GPU in 2.41873s (14.057GiB/s) | 20260508 07:42:12.194 | INFO | Analyzing chunk frames = [6395904,6396416) | 20260508 07:42:12.194 | INFO | MemInfo: { | 20260508 07:42:12.194 | INFO | "gpu" : " 17 GB", | 20260508 07:42:12.194 | INFO | "huge" : "", | 20260508 07:42:12.194 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:12.194 | INFO | "malloc" : " 14 GB", | 20260508 07:42:12.194 | INFO | "pinned" : " 29 GB" | 20260508 07:42:12.194 | INFO | } | 20260508 07:42:12.194 | INFO | Perf report for the last 5.13862 seconds: | 20260508 07:42:12.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:12.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:42:12.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:42:12.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:12.194 | INFO | Uploaded 36GiB to GPU in 2.36082s (15.2489GiB/s) | 20260508 07:42:12.194 | INFO | Downloaded 34GiB to GPU in 2.5089s (13.5518GiB/s) | 20260508 07:42:17.329 | INFO | Analyzing chunk frames = [6396416,6396928) | 20260508 07:42:17.329 | INFO | MemInfo: { | 20260508 07:42:17.329 | INFO | "gpu" : " 17 GB", | 20260508 07:42:17.329 | INFO | "huge" : "", | 20260508 07:42:17.329 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:17.329 | INFO | "malloc" : " 14 GB", | 20260508 07:42:17.329 | INFO | "pinned" : " 29 GB" | 20260508 07:42:17.329 | INFO | } | 20260508 07:42:17.329 | INFO | Perf report for the last 5.13519 seconds: | 20260508 07:42:17.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:17.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.329 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:42:17.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.329 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:42:17.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:17.330 | INFO | Uploaded 36GiB to GPU in 2.39257s (15.0466GiB/s) | 20260508 07:42:17.330 | INFO | Downloaded 34GiB to GPU in 2.42044s (14.047GiB/s) | 20260508 07:42:22.467 | INFO | Analyzing chunk frames = [6396928,6397440) | 20260508 07:42:22.467 | INFO | MemInfo: { | 20260508 07:42:22.467 | INFO | "gpu" : " 17 GB", | 20260508 07:42:22.467 | INFO | "huge" : "", | 20260508 07:42:22.467 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:22.467 | INFO | "malloc" : " 14 GB", | 20260508 07:42:22.467 | INFO | "pinned" : " 29 GB" | 20260508 07:42:22.467 | INFO | } | 20260508 07:42:22.467 | INFO | Perf report for the last 5.13827 seconds: | 20260508 07:42:22.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:22.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.467 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 07:42:22.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.468 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:42:22.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:22.468 | INFO | Uploaded 36GiB to GPU in 2.44125s (14.7465GiB/s) | 20260508 07:42:22.468 | INFO | Downloaded 34GiB to GPU in 2.43304s (13.9743GiB/s) | 20260508 07:42:27.608 | INFO | Analyzing chunk frames = [6397440,6397952) | 20260508 07:42:27.608 | INFO | MemInfo: { | 20260508 07:42:27.608 | INFO | "gpu" : " 17 GB", | 20260508 07:42:27.608 | INFO | "huge" : "", | 20260508 07:42:27.608 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:27.608 | INFO | "malloc" : " 14 GB", | 20260508 07:42:27.608 | INFO | "pinned" : " 29 GB" | 20260508 07:42:27.608 | INFO | } | 20260508 07:42:27.608 | INFO | Perf report for the last 5.1404 seconds: | 20260508 07:42:27.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:27.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:42:27.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:42:27.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:27.608 | INFO | Uploaded 36GiB to GPU in 2.49474s (14.4304GiB/s) | 20260508 07:42:27.608 | INFO | Downloaded 34GiB to GPU in 2.44839s (13.8867GiB/s) | 20260508 07:42:27.608 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:42:27.608 | INFO | Total mem usage: 16.7 GiB | 20260508 07:42:27.608 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:42:27.608 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:42:27.608 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:42:27.608 | INFO | High Water Breakdown: | 20260508 07:42:27.608 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:42:27.608 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:42:27.608 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:42:27.608 | INFO | ------------------------------------------ | 20260508 07:42:27.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:42:27.609 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:42:27.609 | INFO | Total mem usage: 26.5 GiB | 20260508 07:42:27.609 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:42:27.609 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:42:27.609 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:42:27.609 | INFO | High Water Breakdown: | 20260508 07:42:27.609 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:42:27.609 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:42:27.609 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:42:27.609 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:42:27.609 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:42:27.609 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:42:27.609 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:42:27.609 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:42:27.609 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:42:27.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:42:27.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:42:27.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:42:27.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:42:27.609 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:42:27.609 | INFO | ------------------------------------------ | 20260508 07:42:27.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:42:27.609 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:42:27.609 | INFO | Total mem usage: 27.8 GiB | 20260508 07:42:27.609 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:42:27.609 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:42:27.609 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:42:27.609 | INFO | High Water Breakdown: | 20260508 07:42:27.609 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:42:27.609 | INFO | ------------------------------------------ | 20260508 07:42:27.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:42:27.609 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:42:27.609 | INFO | Total mem usage: 15.7 GiB | 20260508 07:42:27.609 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:42:27.609 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.94%) | 20260508 07:42:27.609 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:42:27.609 | INFO | High Water Breakdown: | 20260508 07:42:27.609 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:42:27.609 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:42:27.609 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:42:27.609 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:42:27.609 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:42:27.609 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:42:27.609 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:42:27.609 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:42:27.609 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:42:27.609 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:42:27.609 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:42:27.609 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:42:27.609 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:42:27.609 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:42:27.609 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:42:27.609 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:42:27.609 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:42:27.609 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:42:27.609 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:42:27.609 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:42:27.609 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:42:27.609 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:42:27.609 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:42:27.609 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:42:27.609 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:42:27.609 | INFO | ------------------------------------------ | 20260508 07:42:27.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:42:27.610 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:42:27.610 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:42:27.610 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:42:27.610 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.40e+01 / 6.23e-01 | 20260508 07:42:27.610 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.03e-01, 1.25e-02 / 5.39e-03 | 20260508 07:42:27.610 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.37e+00, 5.83e-01 / 2.74e-01 | 20260508 07:42:27.610 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.45e+00, 2.09e-01 / 1.16e-01 | 20260508 07:42:27.610 | INFO | PerfReport: 15.93%, DME -- 4.16e+01 / 4.50e+01, 3.77e+00 / 1.13e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 1.39e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.51e+00, 4.10e-01 / 2.52e-01 | 20260508 07:42:27.610 | INFO | PerfReport: 3.11%, Metrics -- 1.01e+00 / 1.23e+00, 2.52e-01 / 1.42e-01 | 20260508 07:42:27.610 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.42e-01, 4.95e-02 / 2.57e-02 | 20260508 07:42:27.610 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:42:27.610 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:42:27.610 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:42:27.610 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 24.36%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 24.47%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 30.85%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 2.11%, LockUpload -- 2.23e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 2.73%, LockDownload -- 2.89e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:42:27.610 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260508 07:42:27.705 | INFO | Starting a round of Baz IO | 20260508 07:42:31.141 | INFO | Populating RTMetrics report | 20260508 07:42:31.141 | INFO | RTMetrics report populated | 20260508 07:42:31.141 | INFO | Opening tmp RTMetrics json file | 20260508 07:42:31.142 | INFO | Serializing RTMetrics to json | 20260508 07:42:31.143 | INFO | Serialization done, swapping tmp file now... | 20260508 07:42:31.143 | INFO | RTMetrics done | 20260508 07:42:31.212 | INFO | Finished a round of Baz IO: Wrote 1.521 GiB in 3.507 seconds | 20260508 07:42:31.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.52% / 0.00% / 59.48% | 20260508 07:42:31.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329987% / 0% / 7.15256e-05% | 20260508 07:42:31.212 | INFO | Baz IO overall: | 20260508 07:42:31.212 | INFO | BytesWritten : 2818859610785 | 20260508 07:42:31.212 | INFO | NumEvents : 1357850823413 | 20260508 07:42:32.746 | INFO | Analyzing chunk frames = [6397952,6398464) | 20260508 07:42:32.747 | INFO | MemInfo: { | 20260508 07:42:32.747 | INFO | "gpu" : " 17 GB", | 20260508 07:42:32.747 | INFO | "huge" : "", | 20260508 07:42:32.747 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:32.747 | INFO | "malloc" : " 14 GB", | 20260508 07:42:32.747 | INFO | "pinned" : " 29 GB" | 20260508 07:42:32.747 | INFO | } | 20260508 07:42:32.747 | INFO | Perf report for the last 5.13875 seconds: | 20260508 07:42:32.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:32.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:32.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:32.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:32.747 | INFO | ANALYSIS : 68.4%, 3.0, 106.4ms, 0 | 20260508 07:42:32.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:32.747 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:42:32.747 | INFO | PRE_HQ : 65.9%, 1.6, 55.1ms, 0 | 20260508 07:42:32.747 | INFO | BAZWRITER : 67.7%, 2.6, 89.9ms, 0 | 20260508 07:42:32.747 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:42:32.747 | INFO | Uploaded 36GiB to GPU in 2.56077s (14.0582GiB/s) | 20260508 07:42:32.747 | INFO | Downloaded 38GiB to GPU in 2.71218s (14.0109GiB/s) | 20260508 07:42:37.883 | INFO | Analyzing chunk frames = [6398464,6398976) | 20260508 07:42:37.883 | INFO | MemInfo: { | 20260508 07:42:37.883 | INFO | "gpu" : " 17 GB", | 20260508 07:42:37.883 | INFO | "huge" : "", | 20260508 07:42:37.883 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:37.883 | INFO | "malloc" : " 14 GB", | 20260508 07:42:37.883 | INFO | "pinned" : " 29 GB" | 20260508 07:42:37.883 | INFO | } | 20260508 07:42:37.883 | INFO | Perf report for the last 5.13667 seconds: | 20260508 07:42:37.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:37.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.883 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:42:37.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.883 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:42:37.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:37.884 | INFO | Uploaded 36GiB to GPU in 2.47841s (14.5254GiB/s) | 20260508 07:42:37.884 | INFO | Downloaded 34GiB to GPU in 2.44608s (13.8998GiB/s) | 20260508 07:42:43.024 | INFO | Analyzing chunk frames = [6398976,6399488) | 20260508 07:42:43.024 | INFO | MemInfo: { | 20260508 07:42:43.024 | INFO | "gpu" : " 17 GB", | 20260508 07:42:43.024 | INFO | "huge" : "", | 20260508 07:42:43.024 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:43.024 | INFO | "malloc" : " 14 GB", | 20260508 07:42:43.025 | INFO | "pinned" : " 29 GB" | 20260508 07:42:43.025 | INFO | } | 20260508 07:42:43.025 | INFO | Perf report for the last 5.14129 seconds: | 20260508 07:42:43.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:43.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:42:43.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:42:43.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:43.025 | INFO | Uploaded 36GiB to GPU in 2.55393s (14.0959GiB/s) | 20260508 07:42:43.025 | INFO | Downloaded 34GiB to GPU in 2.43834s (13.9439GiB/s) | 20260508 07:42:48.170 | INFO | Analyzing chunk frames = [6399488,6400000) | 20260508 07:42:48.170 | INFO | MemInfo: { | 20260508 07:42:48.170 | INFO | "gpu" : " 17 GB", | 20260508 07:42:48.170 | INFO | "huge" : "", | 20260508 07:42:48.170 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:48.170 | INFO | "malloc" : " 14 GB", | 20260508 07:42:48.170 | INFO | "pinned" : " 29 GB" | 20260508 07:42:48.170 | INFO | } | 20260508 07:42:48.170 | INFO | Perf report for the last 5.14585 seconds: | 20260508 07:42:48.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:48.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.170 | INFO | ANALYSIS : 66.7%, 2.9, 103.4ms, 0 | 20260508 07:42:48.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.171 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:42:48.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:48.171 | INFO | Uploaded 36GiB to GPU in 2.56677s (14.0254GiB/s) | 20260508 07:42:48.171 | INFO | Downloaded 34GiB to GPU in 2.43612s (13.9566GiB/s) | 20260508 07:42:53.306 | INFO | Analyzing chunk frames = [6400000,6400512) | 20260508 07:42:53.306 | INFO | MemInfo: { | 20260508 07:42:53.306 | INFO | "gpu" : " 17 GB", | 20260508 07:42:53.306 | INFO | "huge" : "", | 20260508 07:42:53.306 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:53.306 | INFO | "malloc" : " 14 GB", | 20260508 07:42:53.306 | INFO | "pinned" : " 29 GB" | 20260508 07:42:53.306 | INFO | } | 20260508 07:42:53.306 | INFO | Perf report for the last 5.13581 seconds: | 20260508 07:42:53.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:53.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.306 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 07:42:53.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.306 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:42:53.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:53.307 | INFO | Uploaded 36GiB to GPU in 2.35433s (15.291GiB/s) | 20260508 07:42:53.307 | INFO | Downloaded 34GiB to GPU in 2.52422s (13.4695GiB/s) | 20260508 07:42:58.441 | INFO | Analyzing chunk frames = [6400512,6401024) | 20260508 07:42:58.441 | INFO | MemInfo: { | 20260508 07:42:58.441 | INFO | "gpu" : " 17 GB", | 20260508 07:42:58.441 | INFO | "huge" : "", | 20260508 07:42:58.441 | INFO | "huge_shared" : " 29 GB", | 20260508 07:42:58.441 | INFO | "malloc" : " 14 GB", | 20260508 07:42:58.441 | INFO | "pinned" : " 29 GB" | 20260508 07:42:58.441 | INFO | } | 20260508 07:42:58.441 | INFO | Perf report for the last 5.13478 seconds: | 20260508 07:42:58.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:42:58.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 07:42:58.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:42:58.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:42:58.441 | INFO | Uploaded 36GiB to GPU in 2.41461s (14.9092GiB/s) | 20260508 07:42:58.441 | INFO | Downloaded 34GiB to GPU in 2.44783s (13.8898GiB/s) | 20260508 07:43:03.577 | INFO | Analyzing chunk frames = [6401024,6401536) | 20260508 07:43:03.577 | INFO | MemInfo: { | 20260508 07:43:03.577 | INFO | "gpu" : " 17 GB", | 20260508 07:43:03.577 | INFO | "huge" : "", | 20260508 07:43:03.577 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:03.577 | INFO | "malloc" : " 14 GB", | 20260508 07:43:03.577 | INFO | "pinned" : " 29 GB" | 20260508 07:43:03.577 | INFO | } | 20260508 07:43:03.577 | INFO | Perf report for the last 5.13611 seconds: | 20260508 07:43:03.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:03.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 07:43:03.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:43:03.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:03.577 | INFO | Uploaded 36GiB to GPU in 2.43614s (14.7775GiB/s) | 20260508 07:43:03.577 | INFO | Downloaded 34GiB to GPU in 2.44927s (13.8817GiB/s) | 20260508 07:43:08.722 | INFO | Analyzing chunk frames = [6401536,6402048) | 20260508 07:43:08.722 | INFO | MemInfo: { | 20260508 07:43:08.722 | INFO | "gpu" : " 17 GB", | 20260508 07:43:08.722 | INFO | "huge" : "", | 20260508 07:43:08.722 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:08.722 | INFO | "malloc" : " 14 GB", | 20260508 07:43:08.722 | INFO | "pinned" : " 29 GB" | 20260508 07:43:08.722 | INFO | } | 20260508 07:43:08.722 | INFO | Perf report for the last 5.14449 seconds: | 20260508 07:43:08.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:08.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 07:43:08.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260508 07:43:08.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:08.722 | INFO | Uploaded 36GiB to GPU in 2.4649s (14.605GiB/s) | 20260508 07:43:08.722 | INFO | Downloaded 34GiB to GPU in 2.4571s (13.8375GiB/s) | 20260508 07:43:08.834 | INFO | Starting a round of Baz IO | 20260508 07:43:12.256 | INFO | Populating RTMetrics report | 20260508 07:43:12.256 | INFO | RTMetrics report populated | 20260508 07:43:12.256 | INFO | Opening tmp RTMetrics json file | 20260508 07:43:12.257 | INFO | Serializing RTMetrics to json | 20260508 07:43:12.258 | INFO | Serialization done, swapping tmp file now... | 20260508 07:43:12.258 | INFO | RTMetrics done | 20260508 07:43:12.334 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.499 seconds | 20260508 07:43:12.334 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.52% / 0.00% / 59.48% | 20260508 07:43:12.334 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329943% / 3.97364e-06% / 3.57628e-05% | 20260508 07:43:12.334 | INFO | Baz IO overall: | 20260508 07:43:12.334 | INFO | BytesWritten : 2820493532710 | 20260508 07:43:12.334 | INFO | NumEvents : 1358625932367 | 20260508 07:43:13.863 | INFO | Analyzing chunk frames = [6402048,6402560) | 20260508 07:43:13.863 | INFO | MemInfo: { | 20260508 07:43:13.863 | INFO | "gpu" : " 17 GB", | 20260508 07:43:13.863 | INFO | "huge" : "", | 20260508 07:43:13.863 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:13.863 | INFO | "malloc" : " 14 GB", | 20260508 07:43:13.863 | INFO | "pinned" : " 29 GB" | 20260508 07:43:13.863 | INFO | } | 20260508 07:43:13.863 | INFO | Perf report for the last 5.14113 seconds: | 20260508 07:43:13.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:13.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:13.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:13.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:13.863 | INFO | ANALYSIS : 68.5%, 3.0, 106.4ms, 0 | 20260508 07:43:13.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:13.863 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260508 07:43:13.863 | INFO | PRE_HQ : 67.0%, 1.8, 61.0ms, 0 | 20260508 07:43:13.863 | INFO | BAZWRITER : 67.5%, 3.1, 108.3ms, 0 | 20260508 07:43:13.863 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:43:13.863 | INFO | Uploaded 36GiB to GPU in 2.51558s (14.3108GiB/s) | 20260508 07:43:13.863 | INFO | Downloaded 38GiB to GPU in 2.73849s (13.8763GiB/s) | 20260508 07:43:19.006 | INFO | Analyzing chunk frames = [6402560,6403072) | 20260508 07:43:19.006 | INFO | MemInfo: { | 20260508 07:43:19.006 | INFO | "gpu" : " 17 GB", | 20260508 07:43:19.006 | INFO | "huge" : "", | 20260508 07:43:19.006 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:19.006 | INFO | "malloc" : " 14 GB", | 20260508 07:43:19.007 | INFO | "pinned" : " 29 GB" | 20260508 07:43:19.007 | INFO | } | 20260508 07:43:19.007 | INFO | Perf report for the last 5.14379 seconds: | 20260508 07:43:19.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:19.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 07:43:19.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:43:19.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:19.007 | INFO | Uploaded 36GiB to GPU in 2.45014s (14.693GiB/s) | 20260508 07:43:19.007 | INFO | Downloaded 34GiB to GPU in 2.45984s (13.8221GiB/s) | 20260508 07:43:24.142 | INFO | Analyzing chunk frames = [6403072,6403584) | 20260508 07:43:24.142 | INFO | MemInfo: { | 20260508 07:43:24.142 | INFO | "gpu" : " 17 GB", | 20260508 07:43:24.142 | INFO | "huge" : "", | 20260508 07:43:24.142 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:24.142 | INFO | "malloc" : " 14 GB", | 20260508 07:43:24.142 | INFO | "pinned" : " 29 GB" | 20260508 07:43:24.142 | INFO | } | 20260508 07:43:24.142 | INFO | Perf report for the last 5.13552 seconds: | 20260508 07:43:24.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:24.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 07:43:24.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:43:24.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:24.142 | INFO | Uploaded 36GiB to GPU in 2.47131s (14.5672GiB/s) | 20260508 07:43:24.142 | INFO | Downloaded 34GiB to GPU in 2.45516s (13.8484GiB/s) | 20260508 07:43:29.280 | INFO | Analyzing chunk frames = [6403584,6404096) | 20260508 07:43:29.280 | INFO | MemInfo: { | 20260508 07:43:29.280 | INFO | "gpu" : " 17 GB", | 20260508 07:43:29.280 | INFO | "huge" : "", | 20260508 07:43:29.280 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:29.280 | INFO | "malloc" : " 14 GB", | 20260508 07:43:29.280 | INFO | "pinned" : " 29 GB" | 20260508 07:43:29.280 | INFO | } | 20260508 07:43:29.280 | INFO | Perf report for the last 5.13764 seconds: | 20260508 07:43:29.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:29.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | ANALYSIS : 66.5%, 3.0, 102.9ms, 0 | 20260508 07:43:29.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:43:29.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:29.280 | INFO | Uploaded 36GiB to GPU in 2.46824s (14.5853GiB/s) | 20260508 07:43:29.280 | INFO | Downloaded 34GiB to GPU in 2.44907s (13.8828GiB/s) | 20260508 07:43:34.420 | INFO | Analyzing chunk frames = [6404096,6404608) | 20260508 07:43:34.420 | INFO | MemInfo: { | 20260508 07:43:34.420 | INFO | "gpu" : " 17 GB", | 20260508 07:43:34.420 | INFO | "huge" : "", | 20260508 07:43:34.420 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:34.420 | INFO | "malloc" : " 14 GB", | 20260508 07:43:34.420 | INFO | "pinned" : " 29 GB" | 20260508 07:43:34.420 | INFO | } | 20260508 07:43:34.421 | INFO | Perf report for the last 5.14066 seconds: | 20260508 07:43:34.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:34.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:43:34.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:43:34.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:34.421 | INFO | Uploaded 36GiB to GPU in 2.24644s (16.0253GiB/s) | 20260508 07:43:34.421 | INFO | Downloaded 34GiB to GPU in 2.50499s (13.5729GiB/s) | 20260508 07:43:39.558 | INFO | Analyzing chunk frames = [6404608,6405120) | 20260508 07:43:39.558 | INFO | MemInfo: { | 20260508 07:43:39.558 | INFO | "gpu" : " 17 GB", | 20260508 07:43:39.558 | INFO | "huge" : "", | 20260508 07:43:39.558 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:39.558 | INFO | "malloc" : " 14 GB", | 20260508 07:43:39.558 | INFO | "pinned" : " 29 GB" | 20260508 07:43:39.558 | INFO | } | 20260508 07:43:39.558 | INFO | Perf report for the last 5.13795 seconds: | 20260508 07:43:39.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:39.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.559 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 07:43:39.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.559 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:43:39.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:39.559 | INFO | Uploaded 36GiB to GPU in 2.35189s (15.3068GiB/s) | 20260508 07:43:39.559 | INFO | Downloaded 34GiB to GPU in 2.43674s (13.9531GiB/s) | 20260508 07:43:44.696 | INFO | Analyzing chunk frames = [6405120,6405632) | 20260508 07:43:44.696 | INFO | MemInfo: { | 20260508 07:43:44.696 | INFO | "gpu" : " 17 GB", | 20260508 07:43:44.697 | INFO | "huge" : "", | 20260508 07:43:44.697 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:44.697 | INFO | "malloc" : " 14 GB", | 20260508 07:43:44.697 | INFO | "pinned" : " 29 GB" | 20260508 07:43:44.697 | INFO | } | 20260508 07:43:44.697 | INFO | Perf report for the last 5.13825 seconds: | 20260508 07:43:44.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:44.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 07:43:44.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 07:43:44.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:44.697 | INFO | Uploaded 36GiB to GPU in 2.47571s (14.5413GiB/s) | 20260508 07:43:44.697 | INFO | Downloaded 34GiB to GPU in 2.43832s (13.944GiB/s) | 20260508 07:43:49.839 | INFO | Analyzing chunk frames = [6405632,6406144) | 20260508 07:43:49.839 | INFO | MemInfo: { | 20260508 07:43:49.839 | INFO | "gpu" : " 17 GB", | 20260508 07:43:49.839 | INFO | "huge" : "", | 20260508 07:43:49.839 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:49.839 | INFO | "malloc" : " 14 GB", | 20260508 07:43:49.839 | INFO | "pinned" : " 29 GB" | 20260508 07:43:49.839 | INFO | } | 20260508 07:43:49.839 | INFO | Perf report for the last 5.14228 seconds: | 20260508 07:43:49.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:49.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 07:43:49.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 07:43:49.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:49.839 | INFO | Uploaded 36GiB to GPU in 2.43277s (14.7979GiB/s) | 20260508 07:43:49.839 | INFO | Downloaded 34GiB to GPU in 2.45388s (13.8556GiB/s) | 20260508 07:43:49.840 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:43:49.840 | INFO | Total mem usage: 16.7 GiB | 20260508 07:43:49.840 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:43:49.840 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13.1%) | 20260508 07:43:49.840 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:43:49.840 | INFO | High Water Breakdown: | 20260508 07:43:49.840 | INFO | 60% : 8.85 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:43:49.840 | INFO | 39.9% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:43:49.840 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:43:49.840 | INFO | ------------------------------------------ | 20260508 07:43:49.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:43:49.840 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:43:49.840 | INFO | Total mem usage: 26.5 GiB | 20260508 07:43:49.840 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:43:49.840 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 07:43:49.840 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 07:43:49.840 | INFO | High Water Breakdown: | 20260508 07:43:49.840 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:43:49.840 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:43:49.840 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:43:49.840 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:43:49.840 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:43:49.840 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:43:49.840 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:43:49.840 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:43:49.840 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:43:49.840 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:43:49.840 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:43:49.840 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:43:49.840 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:43:49.840 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:43:49.840 | INFO | ------------------------------------------ | 20260508 07:43:49.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:43:49.840 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:43:49.840 | INFO | Total mem usage: 27.8 GiB | 20260508 07:43:49.840 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:43:49.840 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:43:49.840 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:43:49.840 | INFO | High Water Breakdown: | 20260508 07:43:49.840 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:43:49.840 | INFO | ------------------------------------------ | 20260508 07:43:49.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:43:49.840 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:43:49.840 | INFO | Total mem usage: 15.7 GiB | 20260508 07:43:49.840 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:43:49.840 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.94%) | 20260508 07:43:49.840 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:43:49.840 | INFO | High Water Breakdown: | 20260508 07:43:49.840 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:43:49.840 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:43:49.840 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:43:49.840 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:43:49.840 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:43:49.840 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:43:49.840 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:43:49.840 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:43:49.840 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:43:49.840 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:43:49.840 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:43:49.840 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:43:49.840 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:43:49.840 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:43:49.840 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:43:49.840 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:43:49.840 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:43:49.840 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:43:49.840 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:43:49.840 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:43:49.840 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:43:49.840 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:43:49.840 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:43:49.840 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:43:49.840 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:43:49.840 | INFO | ------------------------------------------ | 20260508 07:43:49.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:43:49.840 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:43:49.840 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:43:49.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:43:49.840 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 6.56e-01 | 20260508 07:43:49.840 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.05e-01, 1.12e-02 / 4.11e-03 | 20260508 07:43:49.841 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.48e+00, 5.93e-01 / 1.85e-01 | 20260508 07:43:49.841 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.11e-01 / 1.06e-01 | 20260508 07:43:49.841 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.51e+01, 3.63e+00 / 1.11e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.88e+00 / 1.50e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.60e+00, 4.09e-01 / 2.18e-01 | 20260508 07:43:49.841 | INFO | PerfReport: 3.10%, Metrics -- 1.02e+00 / 1.20e+00, 2.52e-01 / 1.15e-01 | 20260508 07:43:49.841 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.49e-01, 5.03e-02 / 2.04e-02 | 20260508 07:43:49.841 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:43:49.841 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:43:49.841 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:43:49.841 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 24.13%, Upload -- 2.55e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 24.50%, Download -- 2.59e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 30.95%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 2.09%, LockUpload -- 2.22e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 2.66%, LockDownload -- 2.81e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 15.66%, LockCompute -- 1.66e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:43:49.841 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260508 07:43:49.936 | INFO | Starting a round of Baz IO | 20260508 07:43:53.356 | INFO | Populating RTMetrics report | 20260508 07:43:53.356 | INFO | RTMetrics report populated | 20260508 07:43:53.356 | INFO | Opening tmp RTMetrics json file | 20260508 07:43:53.367 | INFO | Serializing RTMetrics to json | 20260508 07:43:53.368 | INFO | Serialization done, swapping tmp file now... | 20260508 07:43:53.369 | INFO | RTMetrics done | 20260508 07:43:53.423 | INFO | Finished a round of Baz IO: Wrote 1.52 GiB in 3.486 seconds | 20260508 07:43:53.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.48% | 20260508 07:43:53.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.328827% / 3.97364e-06% / 7.15256e-05% | 20260508 07:43:53.423 | INFO | Baz IO overall: | 20260508 07:43:53.423 | INFO | BytesWritten : 2822125939223 | 20260508 07:43:53.423 | INFO | NumEvents : 1359399905880 | 20260508 07:43:54.975 | INFO | Analyzing chunk frames = [6406144,6406656) | 20260508 07:43:54.975 | INFO | MemInfo: { | 20260508 07:43:54.975 | INFO | "gpu" : " 17 GB", | 20260508 07:43:54.975 | INFO | "huge" : "", | 20260508 07:43:54.975 | INFO | "huge_shared" : " 29 GB", | 20260508 07:43:54.975 | INFO | "malloc" : " 14 GB", | 20260508 07:43:54.975 | INFO | "pinned" : " 29 GB" | 20260508 07:43:54.975 | INFO | } | 20260508 07:43:54.975 | INFO | Perf report for the last 5.13561 seconds: | 20260508 07:43:54.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:43:54.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:54.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:54.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:54.975 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 07:43:54.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:43:54.975 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:43:54.975 | INFO | PRE_HQ : 65.6%, 1.6, 55.0ms, 0 | 20260508 07:43:54.975 | INFO | BAZWRITER : 67.2%, 2.6, 90.5ms, 0 | 20260508 07:43:54.975 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:43:54.975 | INFO | Uploaded 36GiB to GPU in 2.55835s (14.0716GiB/s) | 20260508 07:43:54.975 | INFO | Downloaded 38GiB to GPU in 2.68771s (14.1384GiB/s) | 20260508 07:44:00.118 | INFO | Analyzing chunk frames = [6406656,6407168) | 20260508 07:44:00.118 | INFO | MemInfo: { | 20260508 07:44:00.118 | INFO | "gpu" : " 17 GB", | 20260508 07:44:00.118 | INFO | "huge" : "", | 20260508 07:44:00.119 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:00.119 | INFO | "malloc" : " 14 GB", | 20260508 07:44:00.119 | INFO | "pinned" : " 29 GB" | 20260508 07:44:00.119 | INFO | } | 20260508 07:44:00.119 | INFO | Perf report for the last 5.14405 seconds: | 20260508 07:44:00.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:00.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:44:00.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:44:00.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:00.119 | INFO | Uploaded 36GiB to GPU in 2.50103s (14.394GiB/s) | 20260508 07:44:00.119 | INFO | Downloaded 34GiB to GPU in 2.42833s (14.0014GiB/s) | 20260508 07:44:05.256 | INFO | Analyzing chunk frames = [6407168,6407680) | 20260508 07:44:05.256 | INFO | MemInfo: { | 20260508 07:44:05.256 | INFO | "gpu" : " 17 GB", | 20260508 07:44:05.256 | INFO | "huge" : "", | 20260508 07:44:05.257 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:05.257 | INFO | "malloc" : " 14 GB", | 20260508 07:44:05.257 | INFO | "pinned" : " 29 GB" | 20260508 07:44:05.257 | INFO | } | 20260508 07:44:05.257 | INFO | Perf report for the last 5.13794 seconds: | 20260508 07:44:05.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:05.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 07:44:05.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:44:05.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:05.257 | INFO | Uploaded 36GiB to GPU in 2.57058s (14.0046GiB/s) | 20260508 07:44:05.257 | INFO | Downloaded 34GiB to GPU in 2.44151s (13.9258GiB/s) | 20260508 07:44:10.393 | INFO | Analyzing chunk frames = [6407680,6408192) | 20260508 07:44:10.393 | INFO | MemInfo: { | 20260508 07:44:10.393 | INFO | "gpu" : " 17 GB", | 20260508 07:44:10.393 | INFO | "huge" : "", | 20260508 07:44:10.393 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:10.393 | INFO | "malloc" : " 14 GB", | 20260508 07:44:10.393 | INFO | "pinned" : " 29 GB" | 20260508 07:44:10.393 | INFO | } | 20260508 07:44:10.393 | INFO | Perf report for the last 5.13621 seconds: | 20260508 07:44:10.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:10.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:44:10.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:44:10.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:10.393 | INFO | Uploaded 36GiB to GPU in 2.58791s (13.9108GiB/s) | 20260508 07:44:10.393 | INFO | Downloaded 34GiB to GPU in 2.42624s (14.0135GiB/s) | 20260508 07:44:15.534 | INFO | Analyzing chunk frames = [6408192,6408704) | 20260508 07:44:15.534 | INFO | MemInfo: { | 20260508 07:44:15.534 | INFO | "gpu" : " 17 GB", | 20260508 07:44:15.534 | INFO | "huge" : "", | 20260508 07:44:15.534 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:15.534 | INFO | "malloc" : " 14 GB", | 20260508 07:44:15.534 | INFO | "pinned" : " 29 GB" | 20260508 07:44:15.534 | INFO | } | 20260508 07:44:15.534 | INFO | Perf report for the last 5.14107 seconds: | 20260508 07:44:15.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:15.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:44:15.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:44:15.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:15.534 | INFO | Uploaded 36GiB to GPU in 2.27533s (15.8219GiB/s) | 20260508 07:44:15.534 | INFO | Downloaded 34GiB to GPU in 2.50244s (13.5867GiB/s) | 20260508 07:44:20.674 | INFO | Analyzing chunk frames = [6408704,6409216) | 20260508 07:44:20.674 | INFO | MemInfo: { | 20260508 07:44:20.674 | INFO | "gpu" : " 17 GB", | 20260508 07:44:20.674 | INFO | "huge" : "", | 20260508 07:44:20.674 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:20.674 | INFO | "malloc" : " 14 GB", | 20260508 07:44:20.674 | INFO | "pinned" : " 29 GB" | 20260508 07:44:20.674 | INFO | } | 20260508 07:44:20.675 | INFO | Perf report for the last 5.14049 seconds: | 20260508 07:44:20.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:20.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:44:20.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:44:20.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:20.675 | INFO | Uploaded 36GiB to GPU in 2.46429s (14.6087GiB/s) | 20260508 07:44:20.675 | INFO | Downloaded 34GiB to GPU in 2.44016s (13.9335GiB/s) | 20260508 07:44:25.815 | INFO | Analyzing chunk frames = [6409216,6409728) | 20260508 07:44:25.815 | INFO | MemInfo: { | 20260508 07:44:25.815 | INFO | "gpu" : " 17 GB", | 20260508 07:44:25.815 | INFO | "huge" : "", | 20260508 07:44:25.816 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:25.816 | INFO | "malloc" : " 14 GB", | 20260508 07:44:25.816 | INFO | "pinned" : " 29 GB" | 20260508 07:44:25.816 | INFO | } | 20260508 07:44:25.816 | INFO | Perf report for the last 5.14116 seconds: | 20260508 07:44:25.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:25.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 07:44:25.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:44:25.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:25.816 | INFO | Uploaded 36GiB to GPU in 2.39747s (15.0158GiB/s) | 20260508 07:44:25.816 | INFO | Downloaded 34GiB to GPU in 2.42332s (14.0303GiB/s) | 20260508 07:44:30.959 | INFO | Analyzing chunk frames = [6409728,6410240) | 20260508 07:44:30.959 | INFO | MemInfo: { | 20260508 07:44:30.959 | INFO | "gpu" : " 17 GB", | 20260508 07:44:30.959 | INFO | "huge" : "", | 20260508 07:44:30.959 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:30.959 | INFO | "malloc" : " 14 GB", | 20260508 07:44:30.959 | INFO | "pinned" : " 29 GB" | 20260508 07:44:30.959 | INFO | } | 20260508 07:44:30.959 | INFO | Perf report for the last 5.14322 seconds: | 20260508 07:44:30.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:30.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 07:44:30.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:44:30.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:30.959 | INFO | Uploaded 36GiB to GPU in 2.41336s (14.917GiB/s) | 20260508 07:44:30.959 | INFO | Downloaded 34GiB to GPU in 2.43079s (13.9873GiB/s) | 20260508 07:44:31.055 | INFO | Starting a round of Baz IO | 20260508 07:44:34.478 | INFO | Populating RTMetrics report | 20260508 07:44:34.478 | INFO | RTMetrics report populated | 20260508 07:44:34.478 | INFO | Opening tmp RTMetrics json file | 20260508 07:44:34.487 | INFO | Serializing RTMetrics to json | 20260508 07:44:34.489 | INFO | Serialization done, swapping tmp file now... | 20260508 07:44:34.489 | INFO | RTMetrics done | 20260508 07:44:34.547 | INFO | Finished a round of Baz IO: Wrote 1.522 GiB in 3.491 seconds | 20260508 07:44:34.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.49% | 20260508 07:44:34.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330329% / 0% / 4.37101e-05% | 20260508 07:44:34.547 | INFO | Baz IO overall: | 20260508 07:44:34.547 | INFO | BytesWritten : 2823760642467 | 20260508 07:44:34.547 | INFO | NumEvents : 1360175744519 | 20260508 07:44:36.088 | INFO | Analyzing chunk frames = [6410240,6410752) | 20260508 07:44:36.088 | INFO | MemInfo: { | 20260508 07:44:36.088 | INFO | "gpu" : " 17 GB", | 20260508 07:44:36.088 | INFO | "huge" : "", | 20260508 07:44:36.088 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:36.088 | INFO | "malloc" : " 14 GB", | 20260508 07:44:36.088 | INFO | "pinned" : " 29 GB" | 20260508 07:44:36.088 | INFO | } | 20260508 07:44:36.088 | INFO | Perf report for the last 5.12939 seconds: | 20260508 07:44:36.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:36.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:36.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:36.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:36.088 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:44:36.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:36.088 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 07:44:36.088 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 07:44:36.088 | INFO | BAZWRITER : 67.5%, 2.6, 89.5ms, 0 | 20260508 07:44:36.088 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:44:36.089 | INFO | Uploaded 36GiB to GPU in 2.55628s (14.083GiB/s) | 20260508 07:44:36.089 | INFO | Downloaded 38GiB to GPU in 2.71881s (13.9767GiB/s) | 20260508 07:44:41.238 | INFO | Analyzing chunk frames = [6410752,6411264) | 20260508 07:44:41.238 | INFO | MemInfo: { | 20260508 07:44:41.238 | INFO | "gpu" : " 17 GB", | 20260508 07:44:41.238 | INFO | "huge" : "", | 20260508 07:44:41.238 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:41.238 | INFO | "malloc" : " 14 GB", | 20260508 07:44:41.238 | INFO | "pinned" : " 29 GB" | 20260508 07:44:41.238 | INFO | } | 20260508 07:44:41.238 | INFO | Perf report for the last 5.14973 seconds: | 20260508 07:44:41.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:41.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.238 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 07:44:41.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.238 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 07:44:41.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:41.239 | INFO | Uploaded 36GiB to GPU in 2.5106s (14.3392GiB/s) | 20260508 07:44:41.239 | INFO | Downloaded 34GiB to GPU in 2.44642s (13.8979GiB/s) | 20260508 07:44:46.370 | INFO | Analyzing chunk frames = [6411264,6411776) | 20260508 07:44:46.370 | INFO | MemInfo: { | 20260508 07:44:46.370 | INFO | "gpu" : " 17 GB", | 20260508 07:44:46.370 | INFO | "huge" : "", | 20260508 07:44:46.370 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:46.370 | INFO | "malloc" : " 14 GB", | 20260508 07:44:46.370 | INFO | "pinned" : " 29 GB" | 20260508 07:44:46.370 | INFO | } | 20260508 07:44:46.370 | INFO | Perf report for the last 5.13224 seconds: | 20260508 07:44:46.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:46.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.370 | INFO | ANALYSIS : 66.7%, 3.0, 103.4ms, 0 | 20260508 07:44:46.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.370 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:44:46.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:46.371 | INFO | Uploaded 36GiB to GPU in 2.47456s (14.548GiB/s) | 20260508 07:44:46.371 | INFO | Downloaded 34GiB to GPU in 2.43767s (13.9478GiB/s) | 20260508 07:44:51.509 | INFO | Analyzing chunk frames = [6411776,6412288) | 20260508 07:44:51.509 | INFO | MemInfo: { | 20260508 07:44:51.509 | INFO | "gpu" : " 17 GB", | 20260508 07:44:51.509 | INFO | "huge" : "", | 20260508 07:44:51.509 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:51.509 | INFO | "malloc" : " 14 GB", | 20260508 07:44:51.509 | INFO | "pinned" : " 29 GB" | 20260508 07:44:51.509 | INFO | } | 20260508 07:44:51.509 | INFO | Perf report for the last 5.13912 seconds: | 20260508 07:44:51.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:51.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.509 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 07:44:51.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.510 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:44:51.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:51.510 | INFO | Uploaded 36GiB to GPU in 2.57138s (14.0003GiB/s) | 20260508 07:44:51.510 | INFO | Downloaded 34GiB to GPU in 2.42718s (14.008GiB/s) | 20260508 07:44:56.646 | INFO | Analyzing chunk frames = [6412288,6412800) | 20260508 07:44:56.646 | INFO | MemInfo: { | 20260508 07:44:56.646 | INFO | "gpu" : " 17 GB", | 20260508 07:44:56.646 | INFO | "huge" : "", | 20260508 07:44:56.646 | INFO | "huge_shared" : " 29 GB", | 20260508 07:44:56.646 | INFO | "malloc" : " 14 GB", | 20260508 07:44:56.646 | INFO | "pinned" : " 29 GB" | 20260508 07:44:56.646 | INFO | } | 20260508 07:44:56.646 | INFO | Perf report for the last 5.1364 seconds: | 20260508 07:44:56.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:44:56.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:44:56.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 07:44:56.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:44:56.646 | INFO | Uploaded 36GiB to GPU in 2.28141s (15.7797GiB/s) | 20260508 07:44:56.646 | INFO | Downloaded 34GiB to GPU in 2.48663s (13.6731GiB/s) | 20260508 07:45:01.785 | INFO | Analyzing chunk frames = [6412800,6413312) | 20260508 07:45:01.785 | INFO | MemInfo: { | 20260508 07:45:01.785 | INFO | "gpu" : " 17 GB", | 20260508 07:45:01.785 | INFO | "huge" : "", | 20260508 07:45:01.785 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:01.785 | INFO | "malloc" : " 14 GB", | 20260508 07:45:01.785 | INFO | "pinned" : " 29 GB" | 20260508 07:45:01.785 | INFO | } | 20260508 07:45:01.785 | INFO | Perf report for the last 5.13896 seconds: | 20260508 07:45:01.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:01.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 07:45:01.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:45:01.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:01.785 | INFO | Uploaded 36GiB to GPU in 2.3933s (15.042GiB/s) | 20260508 07:45:01.785 | INFO | Downloaded 34GiB to GPU in 2.42633s (14.013GiB/s) | 20260508 07:45:06.928 | INFO | Analyzing chunk frames = [6413312,6413824) | 20260508 07:45:06.928 | INFO | MemInfo: { | 20260508 07:45:06.928 | INFO | "gpu" : " 17 GB", | 20260508 07:45:06.928 | INFO | "huge" : "", | 20260508 07:45:06.928 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:06.928 | INFO | "malloc" : " 14 GB", | 20260508 07:45:06.928 | INFO | "pinned" : " 29 GB" | 20260508 07:45:06.928 | INFO | } | 20260508 07:45:06.928 | INFO | Perf report for the last 5.14339 seconds: | 20260508 07:45:06.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:06.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.928 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:45:06.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.928 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:45:06.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:06.929 | INFO | Uploaded 36GiB to GPU in 2.50268s (14.3846GiB/s) | 20260508 07:45:06.929 | INFO | Downloaded 34GiB to GPU in 2.43343s (13.9721GiB/s) | 20260508 07:45:12.068 | INFO | Analyzing chunk frames = [6413824,6414336) | 20260508 07:45:12.068 | INFO | MemInfo: { | 20260508 07:45:12.068 | INFO | "gpu" : " 17 GB", | 20260508 07:45:12.068 | INFO | "huge" : "", | 20260508 07:45:12.069 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:12.069 | INFO | "malloc" : " 14 GB", | 20260508 07:45:12.069 | INFO | "pinned" : " 29 GB" | 20260508 07:45:12.069 | INFO | } | 20260508 07:45:12.069 | INFO | Perf report for the last 5.14048 seconds: | 20260508 07:45:12.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:12.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 07:45:12.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:45:12.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:12.069 | INFO | Uploaded 36GiB to GPU in 2.52332s (14.2669GiB/s) | 20260508 07:45:12.069 | INFO | Downloaded 34GiB to GPU in 2.45903s (13.8266GiB/s) | 20260508 07:45:12.069 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:45:12.069 | INFO | Total mem usage: 16.7 GiB | 20260508 07:45:12.069 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:45:12.069 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:45:12.069 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:45:12.069 | INFO | High Water Breakdown: | 20260508 07:45:12.069 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:45:12.069 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:45:12.069 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:45:12.069 | INFO | ------------------------------------------ | 20260508 07:45:12.069 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:45:12.069 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:45:12.069 | INFO | Total mem usage: 26.5 GiB | 20260508 07:45:12.069 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:45:12.069 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:45:12.069 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:45:12.069 | INFO | High Water Breakdown: | 20260508 07:45:12.069 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:45:12.069 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:45:12.069 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:45:12.069 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:45:12.069 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:45:12.069 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:45:12.070 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:45:12.070 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:45:12.070 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:45:12.070 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:45:12.070 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:45:12.070 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:45:12.070 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:45:12.070 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:45:12.070 | INFO | ------------------------------------------ | 20260508 07:45:12.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:45:12.070 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:45:12.070 | INFO | Total mem usage: 27.8 GiB | 20260508 07:45:12.070 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:45:12.070 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:45:12.070 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:45:12.070 | INFO | High Water Breakdown: | 20260508 07:45:12.070 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:45:12.070 | INFO | ------------------------------------------ | 20260508 07:45:12.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:45:12.070 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:45:12.070 | INFO | Total mem usage: 15.7 GiB | 20260508 07:45:12.070 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:45:12.070 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:45:12.070 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:45:12.070 | INFO | High Water Breakdown: | 20260508 07:45:12.070 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:45:12.070 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:45:12.070 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:45:12.070 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:45:12.070 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:45:12.070 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:45:12.070 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:45:12.070 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:45:12.070 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:45:12.070 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:45:12.070 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:45:12.070 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:45:12.070 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:45:12.070 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:45:12.070 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:45:12.070 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:45:12.070 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:45:12.070 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:45:12.070 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:45:12.070 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:45:12.070 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:45:12.070 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:45:12.070 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:45:12.070 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:45:12.070 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:45:12.070 | INFO | ------------------------------------------ | 20260508 07:45:12.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:45:12.070 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:45:12.070 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:45:12.070 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:45:12.070 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.39e+01 / 5.98e-01 | 20260508 07:45:12.070 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.05e-01, 1.21e-02 / 3.56e-03 | 20260508 07:45:12.070 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.46e+00, 5.94e-01 / 2.55e-01 | 20260508 07:45:12.070 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.23e-01 | 20260508 07:45:12.070 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.52e+01, 3.75e+00 / 8.22e-01 | 20260508 07:45:12.070 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.82e+00 / 1.13e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.36e+00 / 2.55e+00, 4.08e-01 / 2.35e-01 | 20260508 07:45:12.070 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.18e+00, 2.53e-01 / 1.04e-01 | 20260508 07:45:12.070 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.09e-01 / 3.39e-01, 4.92e-02 / 2.21e-02 | 20260508 07:45:12.070 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:45:12.070 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:45:12.070 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:45:12.070 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 24.35%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 24.37%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 30.88%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 2.33%, LockUpload -- 2.47e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 2.53%, LockDownload -- 2.68e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 15.51%, LockCompute -- 1.64e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 07:45:12.070 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260508 07:45:12.168 | INFO | Starting a round of Baz IO | 20260508 07:45:15.597 | INFO | Populating RTMetrics report | 20260508 07:45:15.598 | INFO | RTMetrics report populated | 20260508 07:45:15.598 | INFO | Opening tmp RTMetrics json file | 20260508 07:45:15.599 | INFO | Serializing RTMetrics to json | 20260508 07:45:15.600 | INFO | Serialization done, swapping tmp file now... | 20260508 07:45:15.600 | INFO | RTMetrics done | 20260508 07:45:15.662 | INFO | Finished a round of Baz IO: Wrote 1.521 GiB in 3.493 seconds | 20260508 07:45:15.662 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.49% | 20260508 07:45:15.662 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329248% / 0% / 4.76837e-05% | 20260508 07:45:15.662 | INFO | Baz IO overall: | 20260508 07:45:15.662 | INFO | BytesWritten : 2825393577398 | 20260508 07:45:15.662 | INFO | NumEvents : 1360950475918 | 20260508 07:45:17.204 | INFO | Analyzing chunk frames = [6414336,6414848) | 20260508 07:45:17.204 | INFO | MemInfo: { | 20260508 07:45:17.204 | INFO | "gpu" : " 17 GB", | 20260508 07:45:17.204 | INFO | "huge" : "", | 20260508 07:45:17.204 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:17.204 | INFO | "malloc" : " 14 GB", | 20260508 07:45:17.204 | INFO | "pinned" : " 29 GB" | 20260508 07:45:17.204 | INFO | } | 20260508 07:45:17.204 | INFO | Perf report for the last 5.13521 seconds: | 20260508 07:45:17.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:17.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:17.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:17.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:17.204 | INFO | ANALYSIS : 68.4%, 3.0, 106.2ms, 0 | 20260508 07:45:17.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:17.204 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:45:17.204 | INFO | PRE_HQ : 66.3%, 1.6, 55.1ms, 0 | 20260508 07:45:17.204 | INFO | BAZWRITER : 67.4%, 2.6, 89.7ms, 0 | 20260508 07:45:17.204 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 07:45:17.204 | INFO | Uploaded 36GiB to GPU in 2.52606s (14.2514GiB/s) | 20260508 07:45:17.205 | INFO | Downloaded 38GiB to GPU in 2.71121s (14.0159GiB/s) | 20260508 07:45:22.342 | INFO | Analyzing chunk frames = [6414848,6415360) | 20260508 07:45:22.342 | INFO | MemInfo: { | 20260508 07:45:22.342 | INFO | "gpu" : " 17 GB", | 20260508 07:45:22.342 | INFO | "huge" : "", | 20260508 07:45:22.342 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:22.342 | INFO | "malloc" : " 14 GB", | 20260508 07:45:22.342 | INFO | "pinned" : " 29 GB" | 20260508 07:45:22.342 | INFO | } | 20260508 07:45:22.342 | INFO | Perf report for the last 5.13839 seconds: | 20260508 07:45:22.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:22.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.342 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:45:22.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.342 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:45:22.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:22.343 | INFO | Uploaded 36GiB to GPU in 2.45945s (14.6374GiB/s) | 20260508 07:45:22.343 | INFO | Downloaded 34GiB to GPU in 2.44315s (13.9164GiB/s) | 20260508 07:45:27.487 | INFO | Analyzing chunk frames = [6415360,6415872) | 20260508 07:45:27.487 | INFO | MemInfo: { | 20260508 07:45:27.487 | INFO | "gpu" : " 17 GB", | 20260508 07:45:27.487 | INFO | "huge" : "", | 20260508 07:45:27.487 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:27.487 | INFO | "malloc" : " 14 GB", | 20260508 07:45:27.487 | INFO | "pinned" : " 29 GB" | 20260508 07:45:27.487 | INFO | } | 20260508 07:45:27.487 | INFO | Perf report for the last 5.14467 seconds: | 20260508 07:45:27.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:27.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 07:45:27.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:45:27.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:27.487 | INFO | Uploaded 36GiB to GPU in 2.55865s (14.0699GiB/s) | 20260508 07:45:27.487 | INFO | Downloaded 34GiB to GPU in 2.43834s (13.9439GiB/s) | 20260508 07:45:32.626 | INFO | Analyzing chunk frames = [6415872,6416384) | 20260508 07:45:32.626 | INFO | MemInfo: { | 20260508 07:45:32.626 | INFO | "gpu" : " 17 GB", | 20260508 07:45:32.626 | INFO | "huge" : "", | 20260508 07:45:32.626 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:32.627 | INFO | "malloc" : " 14 GB", | 20260508 07:45:32.627 | INFO | "pinned" : " 29 GB" | 20260508 07:45:32.627 | INFO | } | 20260508 07:45:32.627 | INFO | Perf report for the last 5.13956 seconds: | 20260508 07:45:32.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:32.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 07:45:32.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 07:45:32.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:32.627 | INFO | Uploaded 36GiB to GPU in 2.47729s (14.532GiB/s) | 20260508 07:45:32.627 | INFO | Downloaded 34GiB to GPU in 2.43118s (13.985GiB/s) | 20260508 07:45:37.768 | INFO | Analyzing chunk frames = [6416384,6416896) | 20260508 07:45:37.768 | INFO | MemInfo: { | 20260508 07:45:37.768 | INFO | "gpu" : " 17 GB", | 20260508 07:45:37.768 | INFO | "huge" : "", | 20260508 07:45:37.768 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:37.768 | INFO | "malloc" : " 14 GB", | 20260508 07:45:37.768 | INFO | "pinned" : " 29 GB" | 20260508 07:45:37.768 | INFO | } | 20260508 07:45:37.768 | INFO | Perf report for the last 5.14158 seconds: | 20260508 07:45:37.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:37.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 07:45:37.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:45:37.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:37.768 | INFO | Uploaded 36GiB to GPU in 2.26749s (15.8766GiB/s) | 20260508 07:45:37.768 | INFO | Downloaded 34GiB to GPU in 2.50117s (13.5936GiB/s) | 20260508 07:45:42.904 | INFO | Analyzing chunk frames = [6416896,6417408) | 20260508 07:45:42.904 | INFO | MemInfo: { | 20260508 07:45:42.904 | INFO | "gpu" : " 17 GB", | 20260508 07:45:42.904 | INFO | "huge" : "", | 20260508 07:45:42.904 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:42.905 | INFO | "malloc" : " 14 GB", | 20260508 07:45:42.905 | INFO | "pinned" : " 29 GB" | 20260508 07:45:42.905 | INFO | } | 20260508 07:45:42.905 | INFO | Perf report for the last 5.13652 seconds: | 20260508 07:45:42.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:42.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:45:42.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:45:42.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:42.905 | INFO | Uploaded 36GiB to GPU in 2.35984s (15.2553GiB/s) | 20260508 07:45:42.905 | INFO | Downloaded 34GiB to GPU in 2.44994s (13.8779GiB/s) | 20260508 07:45:48.039 | INFO | Analyzing chunk frames = [6417408,6417920) | 20260508 07:45:48.039 | INFO | MemInfo: { | 20260508 07:45:48.039 | INFO | "gpu" : " 17 GB", | 20260508 07:45:48.039 | INFO | "huge" : "", | 20260508 07:45:48.039 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:48.039 | INFO | "malloc" : " 14 GB", | 20260508 07:45:48.039 | INFO | "pinned" : " 29 GB" | 20260508 07:45:48.039 | INFO | } | 20260508 07:45:48.039 | INFO | Perf report for the last 5.13411 seconds: | 20260508 07:45:48.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:48.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 07:45:48.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:45:48.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:48.039 | INFO | Uploaded 36GiB to GPU in 2.41027s (14.9361GiB/s) | 20260508 07:45:48.039 | INFO | Downloaded 34GiB to GPU in 2.46049s (13.8184GiB/s) | 20260508 07:45:53.182 | INFO | Analyzing chunk frames = [6417920,6418432) | 20260508 07:45:53.182 | INFO | MemInfo: { | 20260508 07:45:53.182 | INFO | "gpu" : " 17 GB", | 20260508 07:45:53.182 | INFO | "huge" : "", | 20260508 07:45:53.182 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:53.182 | INFO | "malloc" : " 14 GB", | 20260508 07:45:53.182 | INFO | "pinned" : " 29 GB" | 20260508 07:45:53.182 | INFO | } | 20260508 07:45:53.182 | INFO | Perf report for the last 5.14352 seconds: | 20260508 07:45:53.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:53.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.182 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:45:53.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.182 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 07:45:53.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:53.183 | INFO | Uploaded 36GiB to GPU in 2.44477s (14.7253GiB/s) | 20260508 07:45:53.183 | INFO | Downloaded 34GiB to GPU in 2.46597s (13.7877GiB/s) | 20260508 07:45:53.279 | INFO | Starting a round of Baz IO | 20260508 07:45:56.707 | INFO | Populating RTMetrics report | 20260508 07:45:56.708 | INFO | RTMetrics report populated | 20260508 07:45:56.708 | INFO | Opening tmp RTMetrics json file | 20260508 07:45:56.719 | INFO | Serializing RTMetrics to json | 20260508 07:45:56.720 | INFO | Serialization done, swapping tmp file now... | 20260508 07:45:56.721 | INFO | RTMetrics done | 20260508 07:45:56.777 | INFO | Finished a round of Baz IO: Wrote 1.521 GiB in 3.497 seconds | 20260508 07:45:56.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.49% | 20260508 07:45:56.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.328517% / 3.97364e-06% / 3.17891e-05% | 20260508 07:45:56.777 | INFO | Baz IO overall: | 20260508 07:45:56.777 | INFO | BytesWritten : 2827027159371 | 20260508 07:45:56.777 | INFO | NumEvents : 1361725403297 | 20260508 07:45:58.322 | INFO | Analyzing chunk frames = [6418432,6418944) | 20260508 07:45:58.322 | INFO | MemInfo: { | 20260508 07:45:58.322 | INFO | "gpu" : " 17 GB", | 20260508 07:45:58.322 | INFO | "huge" : "", | 20260508 07:45:58.322 | INFO | "huge_shared" : " 29 GB", | 20260508 07:45:58.322 | INFO | "malloc" : " 14 GB", | 20260508 07:45:58.322 | INFO | "pinned" : " 29 GB" | 20260508 07:45:58.322 | INFO | } | 20260508 07:45:58.322 | INFO | Perf report for the last 5.13954 seconds: | 20260508 07:45:58.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:45:58.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:58.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:58.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:58.322 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 07:45:58.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:45:58.322 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:45:58.322 | INFO | PRE_HQ : 65.8%, 1.6, 55.0ms, 0 | 20260508 07:45:58.322 | INFO | BAZWRITER : 67.3%, 2.6, 89.2ms, 0 | 20260508 07:45:58.322 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 07:45:58.322 | INFO | Uploaded 36GiB to GPU in 2.47834s (14.5259GiB/s) | 20260508 07:45:58.322 | INFO | Downloaded 38GiB to GPU in 2.70619s (14.0419GiB/s) | 20260508 07:46:03.460 | INFO | Analyzing chunk frames = [6418944,6419456) | 20260508 07:46:03.460 | INFO | MemInfo: { | 20260508 07:46:03.460 | INFO | "gpu" : " 17 GB", | 20260508 07:46:03.460 | INFO | "huge" : "", | 20260508 07:46:03.460 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:03.460 | INFO | "malloc" : " 14 GB", | 20260508 07:46:03.460 | INFO | "pinned" : " 29 GB" | 20260508 07:46:03.460 | INFO | } | 20260508 07:46:03.460 | INFO | Perf report for the last 5.13823 seconds: | 20260508 07:46:03.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:03.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.460 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:46:03.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.460 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:46:03.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:03.461 | INFO | Uploaded 36GiB to GPU in 2.49038s (14.4556GiB/s) | 20260508 07:46:03.461 | INFO | Downloaded 34GiB to GPU in 2.44209s (13.9225GiB/s) | 20260508 07:46:08.600 | INFO | Analyzing chunk frames = [6419456,6419968) | 20260508 07:46:08.600 | INFO | MemInfo: { | 20260508 07:46:08.600 | INFO | "gpu" : " 17 GB", | 20260508 07:46:08.600 | INFO | "huge" : "", | 20260508 07:46:08.600 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:08.600 | INFO | "malloc" : " 14 GB", | 20260508 07:46:08.600 | INFO | "pinned" : " 29 GB" | 20260508 07:46:08.600 | INFO | } | 20260508 07:46:08.600 | INFO | Perf report for the last 5.13978 seconds: | 20260508 07:46:08.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:08.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 07:46:08.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:46:08.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:08.600 | INFO | Uploaded 36GiB to GPU in 2.48102s (14.5101GiB/s) | 20260508 07:46:08.600 | INFO | Downloaded 34GiB to GPU in 2.437s (13.9516GiB/s) | 20260508 07:46:13.744 | INFO | Analyzing chunk frames = [6419968,6420480) | 20260508 07:46:13.744 | INFO | MemInfo: { | 20260508 07:46:13.744 | INFO | "gpu" : " 17 GB", | 20260508 07:46:13.744 | INFO | "huge" : "", | 20260508 07:46:13.744 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:13.744 | INFO | "malloc" : " 14 GB", | 20260508 07:46:13.744 | INFO | "pinned" : " 29 GB" | 20260508 07:46:13.744 | INFO | } | 20260508 07:46:13.744 | INFO | Perf report for the last 5.14427 seconds: | 20260508 07:46:13.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:13.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.744 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 07:46:13.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.744 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:46:13.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:13.745 | INFO | Uploaded 36GiB to GPU in 2.52118s (14.279GiB/s) | 20260508 07:46:13.745 | INFO | Downloaded 34GiB to GPU in 2.44546s (13.9033GiB/s) | 20260508 07:46:18.879 | INFO | Analyzing chunk frames = [6420480,6420992) | 20260508 07:46:18.879 | INFO | MemInfo: { | 20260508 07:46:18.879 | INFO | "gpu" : " 17 GB", | 20260508 07:46:18.879 | INFO | "huge" : "", | 20260508 07:46:18.879 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:18.879 | INFO | "malloc" : " 14 GB", | 20260508 07:46:18.879 | INFO | "pinned" : " 29 GB" | 20260508 07:46:18.879 | INFO | } | 20260508 07:46:18.879 | INFO | Perf report for the last 5.13515 seconds: | 20260508 07:46:18.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:18.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.879 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:46:18.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.880 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 07:46:18.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:18.880 | INFO | Uploaded 36GiB to GPU in 2.29125s (15.7119GiB/s) | 20260508 07:46:18.880 | INFO | Downloaded 34GiB to GPU in 2.50215s (13.5883GiB/s) | 20260508 07:46:24.019 | INFO | Analyzing chunk frames = [6420992,6421504) | 20260508 07:46:24.019 | INFO | MemInfo: { | 20260508 07:46:24.019 | INFO | "gpu" : " 17 GB", | 20260508 07:46:24.019 | INFO | "huge" : "", | 20260508 07:46:24.019 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:24.019 | INFO | "malloc" : " 14 GB", | 20260508 07:46:24.019 | INFO | "pinned" : " 29 GB" | 20260508 07:46:24.019 | INFO | } | 20260508 07:46:24.019 | INFO | Perf report for the last 5.13942 seconds: | 20260508 07:46:24.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:24.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:46:24.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:46:24.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:24.019 | INFO | Uploaded 36GiB to GPU in 2.41334s (14.9171GiB/s) | 20260508 07:46:24.019 | INFO | Downloaded 34GiB to GPU in 2.42679s (14.0103GiB/s) | 20260508 07:46:29.153 | INFO | Analyzing chunk frames = [6421504,6422016) | 20260508 07:46:29.153 | INFO | MemInfo: { | 20260508 07:46:29.153 | INFO | "gpu" : " 17 GB", | 20260508 07:46:29.153 | INFO | "huge" : "", | 20260508 07:46:29.154 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:29.154 | INFO | "malloc" : " 14 GB", | 20260508 07:46:29.154 | INFO | "pinned" : " 29 GB" | 20260508 07:46:29.154 | INFO | } | 20260508 07:46:29.154 | INFO | Perf report for the last 5.13489 seconds: | 20260508 07:46:29.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:29.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:46:29.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:46:29.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:29.154 | INFO | Uploaded 36GiB to GPU in 2.49654s (14.4199GiB/s) | 20260508 07:46:29.154 | INFO | Downloaded 34GiB to GPU in 2.43914s (13.9394GiB/s) | 20260508 07:46:34.297 | INFO | Analyzing chunk frames = [6422016,6422528) | 20260508 07:46:34.297 | INFO | MemInfo: { | 20260508 07:46:34.297 | INFO | "gpu" : " 17 GB", | 20260508 07:46:34.297 | INFO | "huge" : "", | 20260508 07:46:34.297 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:34.298 | INFO | "malloc" : " 14 GB", | 20260508 07:46:34.298 | INFO | "pinned" : " 29 GB" | 20260508 07:46:34.298 | INFO | } | 20260508 07:46:34.298 | INFO | Perf report for the last 5.14395 seconds: | 20260508 07:46:34.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:34.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:46:34.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:46:34.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:34.298 | INFO | Uploaded 36GiB to GPU in 2.49401s (14.4346GiB/s) | 20260508 07:46:34.298 | INFO | Downloaded 34GiB to GPU in 2.44334s (13.9154GiB/s) | 20260508 07:46:34.298 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:46:34.298 | INFO | Total mem usage: 16.7 GiB | 20260508 07:46:34.298 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:46:34.298 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:46:34.298 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:46:34.298 | INFO | High Water Breakdown: | 20260508 07:46:34.298 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:46:34.298 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:46:34.298 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:46:34.298 | INFO | ------------------------------------------ | 20260508 07:46:34.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:46:34.298 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:46:34.298 | INFO | Total mem usage: 26.5 GiB | 20260508 07:46:34.298 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:46:34.298 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.14%) | 20260508 07:46:34.298 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260508 07:46:34.298 | INFO | High Water Breakdown: | 20260508 07:46:34.298 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:46:34.298 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:46:34.298 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:46:34.298 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:46:34.298 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:46:34.298 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:46:34.298 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:46:34.298 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:46:34.298 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:46:34.298 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:46:34.298 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:46:34.298 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:46:34.298 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:46:34.298 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:46:34.298 | INFO | ------------------------------------------ | 20260508 07:46:34.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:46:34.298 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:46:34.298 | INFO | Total mem usage: 27.8 GiB | 20260508 07:46:34.299 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:46:34.299 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:46:34.299 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:46:34.299 | INFO | High Water Breakdown: | 20260508 07:46:34.299 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:46:34.299 | INFO | ------------------------------------------ | 20260508 07:46:34.299 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:46:34.299 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:46:34.299 | INFO | Total mem usage: 15.7 GiB | 20260508 07:46:34.299 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:46:34.299 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:46:34.299 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:46:34.299 | INFO | High Water Breakdown: | 20260508 07:46:34.299 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:46:34.299 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:46:34.299 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:46:34.299 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:46:34.299 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:46:34.299 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:46:34.299 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:46:34.299 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:46:34.299 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:46:34.299 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:46:34.299 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:46:34.299 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:46:34.299 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:46:34.299 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:46:34.299 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:46:34.299 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:46:34.299 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:46:34.299 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:46:34.299 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:46:34.299 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:46:34.299 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:46:34.299 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:46:34.299 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:46:34.299 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:46:34.299 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:46:34.299 | INFO | ------------------------------------------ | 20260508 07:46:34.299 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:46:34.299 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:46:34.299 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:46:34.299 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:46:34.299 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 6.45e-01 | 20260508 07:46:34.299 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.04e-02 / 5.72e-03 | 20260508 07:46:34.299 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.32e+00, 5.93e-01 / 3.00e-01 | 20260508 07:46:34.299 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.43e+00, 2.10e-01 / 1.26e-01 | 20260508 07:46:34.299 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.55e+01, 3.92e+00 / 1.16e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.87e+00 / 1.31e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.56e+00, 4.11e-01 / 2.39e-01 | 20260508 07:46:34.299 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.25e+00, 2.50e-01 / 1.46e-01 | 20260508 07:46:34.299 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.47e-01, 4.95e-02 / 2.25e-02 | 20260508 07:46:34.299 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:46:34.299 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:46:34.299 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:46:34.299 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 24.11%, Upload -- 2.55e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 24.46%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 31.00%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 2.21%, LockUpload -- 2.33e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 2.56%, LockDownload -- 2.71e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 15.65%, LockCompute -- 1.66e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:46:34.299 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260508 07:46:34.394 | INFO | Starting a round of Baz IO | 20260508 07:46:37.802 | INFO | Populating RTMetrics report | 20260508 07:46:37.803 | INFO | RTMetrics report populated | 20260508 07:46:37.803 | INFO | Opening tmp RTMetrics json file | 20260508 07:46:37.816 | INFO | Serializing RTMetrics to json | 20260508 07:46:37.817 | INFO | Serialization done, swapping tmp file now... | 20260508 07:46:37.818 | INFO | RTMetrics done | 20260508 07:46:37.876 | INFO | Finished a round of Baz IO: Wrote 1.52 GiB in 3.481 seconds | 20260508 07:46:37.876 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.49% | 20260508 07:46:37.876 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.327794% / 7.94729e-06% / 5.5631e-05% | 20260508 07:46:37.876 | INFO | Baz IO overall: | 20260508 07:46:37.876 | INFO | BytesWritten : 2828659387325 | 20260508 07:46:37.876 | INFO | NumEvents : 1362499617482 | 20260508 07:46:39.437 | INFO | Analyzing chunk frames = [6422528,6423040) | 20260508 07:46:39.437 | INFO | MemInfo: { | 20260508 07:46:39.437 | INFO | "gpu" : " 17 GB", | 20260508 07:46:39.437 | INFO | "huge" : "", | 20260508 07:46:39.437 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:39.437 | INFO | "malloc" : " 14 GB", | 20260508 07:46:39.437 | INFO | "pinned" : " 29 GB" | 20260508 07:46:39.437 | INFO | } | 20260508 07:46:39.437 | INFO | Perf report for the last 5.13975 seconds: | 20260508 07:46:39.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:39.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:39.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:39.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:39.438 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 07:46:39.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:39.438 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 07:46:39.438 | INFO | PRE_HQ : 65.7%, 1.6, 56.1ms, 0 | 20260508 07:46:39.438 | INFO | BAZWRITER : 67.1%, 2.6, 91.4ms, 0 | 20260508 07:46:39.438 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:46:39.438 | INFO | Uploaded 36GiB to GPU in 2.59498s (13.8729GiB/s) | 20260508 07:46:39.438 | INFO | Downloaded 38GiB to GPU in 2.70535s (14.0462GiB/s) | 20260508 07:46:44.577 | INFO | Analyzing chunk frames = [6423040,6423552) | 20260508 07:46:44.577 | INFO | MemInfo: { | 20260508 07:46:44.577 | INFO | "gpu" : " 17 GB", | 20260508 07:46:44.577 | INFO | "huge" : "", | 20260508 07:46:44.577 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:44.577 | INFO | "malloc" : " 14 GB", | 20260508 07:46:44.577 | INFO | "pinned" : " 29 GB" | 20260508 07:46:44.577 | INFO | } | 20260508 07:46:44.578 | INFO | Perf report for the last 5.14003 seconds: | 20260508 07:46:44.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:44.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 07:46:44.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:46:44.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:44.578 | INFO | Uploaded 36GiB to GPU in 2.47217s (14.5621GiB/s) | 20260508 07:46:44.578 | INFO | Downloaded 34GiB to GPU in 2.44286s (13.9181GiB/s) | 20260508 07:46:49.713 | INFO | Analyzing chunk frames = [6423552,6424064) | 20260508 07:46:49.713 | INFO | MemInfo: { | 20260508 07:46:49.713 | INFO | "gpu" : " 17 GB", | 20260508 07:46:49.713 | INFO | "huge" : "", | 20260508 07:46:49.713 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:49.713 | INFO | "malloc" : " 14 GB", | 20260508 07:46:49.713 | INFO | "pinned" : " 29 GB" | 20260508 07:46:49.713 | INFO | } | 20260508 07:46:49.713 | INFO | Perf report for the last 5.1355 seconds: | 20260508 07:46:49.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:49.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 07:46:49.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:46:49.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:49.713 | INFO | Uploaded 36GiB to GPU in 2.47108s (14.5685GiB/s) | 20260508 07:46:49.713 | INFO | Downloaded 34GiB to GPU in 2.44162s (13.9252GiB/s) | 20260508 07:46:54.847 | INFO | Analyzing chunk frames = [6424064,6424576) | 20260508 07:46:54.847 | INFO | MemInfo: { | 20260508 07:46:54.847 | INFO | "gpu" : " 17 GB", | 20260508 07:46:54.847 | INFO | "huge" : "", | 20260508 07:46:54.847 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:54.847 | INFO | "malloc" : " 14 GB", | 20260508 07:46:54.848 | INFO | "pinned" : " 29 GB" | 20260508 07:46:54.848 | INFO | } | 20260508 07:46:54.848 | INFO | Perf report for the last 5.13461 seconds: | 20260508 07:46:54.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:54.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 07:46:54.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:46:54.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:54.848 | INFO | Uploaded 36GiB to GPU in 2.56502s (14.035GiB/s) | 20260508 07:46:54.848 | INFO | Downloaded 34GiB to GPU in 2.42551s (14.0177GiB/s) | 20260508 07:46:59.993 | INFO | Analyzing chunk frames = [6424576,6425088) | 20260508 07:46:59.993 | INFO | MemInfo: { | 20260508 07:46:59.993 | INFO | "gpu" : " 17 GB", | 20260508 07:46:59.993 | INFO | "huge" : "", | 20260508 07:46:59.993 | INFO | "huge_shared" : " 29 GB", | 20260508 07:46:59.993 | INFO | "malloc" : " 14 GB", | 20260508 07:46:59.993 | INFO | "pinned" : " 29 GB" | 20260508 07:46:59.993 | INFO | } | 20260508 07:46:59.993 | INFO | Perf report for the last 5.14552 seconds: | 20260508 07:46:59.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:46:59.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 07:46:59.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 07:46:59.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:46:59.993 | INFO | Uploaded 36GiB to GPU in 2.27359s (15.834GiB/s) | 20260508 07:46:59.993 | INFO | Downloaded 34GiB to GPU in 2.50239s (13.587GiB/s) | 20260508 07:47:05.133 | INFO | Analyzing chunk frames = [6425088,6425600) | 20260508 07:47:05.133 | INFO | MemInfo: { | 20260508 07:47:05.133 | INFO | "gpu" : " 17 GB", | 20260508 07:47:05.133 | INFO | "huge" : "", | 20260508 07:47:05.133 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:05.133 | INFO | "malloc" : " 14 GB", | 20260508 07:47:05.133 | INFO | "pinned" : " 29 GB" | 20260508 07:47:05.133 | INFO | } | 20260508 07:47:05.133 | INFO | Perf report for the last 5.14018 seconds: | 20260508 07:47:05.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:05.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.133 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 07:47:05.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.133 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:47:05.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:05.134 | INFO | Uploaded 36GiB to GPU in 2.35192s (15.3066GiB/s) | 20260508 07:47:05.134 | INFO | Downloaded 34GiB to GPU in 2.45253s (13.8632GiB/s) | 20260508 07:47:10.281 | INFO | Analyzing chunk frames = [6425600,6426112) | 20260508 07:47:10.281 | INFO | MemInfo: { | 20260508 07:47:10.281 | INFO | "gpu" : " 17 GB", | 20260508 07:47:10.281 | INFO | "huge" : "", | 20260508 07:47:10.281 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:10.282 | INFO | "malloc" : " 14 GB", | 20260508 07:47:10.282 | INFO | "pinned" : " 29 GB" | 20260508 07:47:10.282 | INFO | } | 20260508 07:47:10.282 | INFO | Perf report for the last 5.1483 seconds: | 20260508 07:47:10.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:10.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.282 | INFO | ANALYSIS : 65.9%, 3.0, 103.3ms, 0 | 20260508 07:47:10.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.282 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:47:10.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:10.283 | INFO | Uploaded 36GiB to GPU in 2.4473s (14.7101GiB/s) | 20260508 07:47:10.283 | INFO | Downloaded 34GiB to GPU in 2.45221s (13.8651GiB/s) | 20260508 07:47:15.412 | INFO | Analyzing chunk frames = [6426112,6426624) | 20260508 07:47:15.412 | INFO | MemInfo: { | 20260508 07:47:15.412 | INFO | "gpu" : " 17 GB", | 20260508 07:47:15.412 | INFO | "huge" : "", | 20260508 07:47:15.412 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:15.412 | INFO | "malloc" : " 14 GB", | 20260508 07:47:15.412 | INFO | "pinned" : " 29 GB" | 20260508 07:47:15.412 | INFO | } | 20260508 07:47:15.412 | INFO | Perf report for the last 5.13073 seconds: | 20260508 07:47:15.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:15.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.413 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 07:47:15.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.413 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260508 07:47:15.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:15.413 | INFO | Uploaded 36GiB to GPU in 2.39729s (15.017GiB/s) | 20260508 07:47:15.413 | INFO | Downloaded 34GiB to GPU in 2.44263s (13.9194GiB/s) | 20260508 07:47:15.529 | INFO | Starting a round of Baz IO | 20260508 07:47:18.919 | INFO | Populating RTMetrics report | 20260508 07:47:18.920 | INFO | RTMetrics report populated | 20260508 07:47:18.920 | INFO | Opening tmp RTMetrics json file | 20260508 07:47:18.928 | INFO | Serializing RTMetrics to json | 20260508 07:47:18.933 | INFO | Serialization done, swapping tmp file now... | 20260508 07:47:18.933 | INFO | RTMetrics done | 20260508 07:47:18.994 | INFO | Finished a round of Baz IO: Wrote 1.52 GiB in 3.465 seconds | 20260508 07:47:18.994 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.49% | 20260508 07:47:18.995 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.325986% / 0% / 3.97364e-05% | 20260508 07:47:18.995 | INFO | Baz IO overall: | 20260508 07:47:18.995 | INFO | BytesWritten : 2830291302987 | 20260508 07:47:18.995 | INFO | NumEvents : 1363273617130 | 20260508 07:47:20.553 | INFO | Analyzing chunk frames = [6426624,6427136) | 20260508 07:47:20.553 | INFO | MemInfo: { | 20260508 07:47:20.553 | INFO | "gpu" : " 17 GB", | 20260508 07:47:20.553 | INFO | "huge" : "", | 20260508 07:47:20.553 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:20.553 | INFO | "malloc" : " 14 GB", | 20260508 07:47:20.553 | INFO | "pinned" : " 29 GB" | 20260508 07:47:20.553 | INFO | } | 20260508 07:47:20.553 | INFO | Perf report for the last 5.14068 seconds: | 20260508 07:47:20.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:20.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:20.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:20.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:20.553 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 07:47:20.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:20.553 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260508 07:47:20.553 | INFO | PRE_HQ : 66.4%, 1.8, 63.2ms, 0 | 20260508 07:47:20.553 | INFO | BAZWRITER : 66.9%, 2.8, 97.8ms, 0 | 20260508 07:47:20.553 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:47:20.554 | INFO | Uploaded 36GiB to GPU in 2.5753s (13.979GiB/s) | 20260508 07:47:20.554 | INFO | Downloaded 38GiB to GPU in 2.71068s (14.0186GiB/s) | 20260508 07:47:25.689 | INFO | Analyzing chunk frames = [6427136,6427648) | 20260508 07:47:25.689 | INFO | MemInfo: { | 20260508 07:47:25.689 | INFO | "gpu" : " 17 GB", | 20260508 07:47:25.689 | INFO | "huge" : "", | 20260508 07:47:25.689 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:25.689 | INFO | "malloc" : " 14 GB", | 20260508 07:47:25.689 | INFO | "pinned" : " 29 GB" | 20260508 07:47:25.689 | INFO | } | 20260508 07:47:25.689 | INFO | Perf report for the last 5.13565 seconds: | 20260508 07:47:25.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:25.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:47:25.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:47:25.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:25.689 | INFO | Uploaded 36GiB to GPU in 2.52283s (14.2697GiB/s) | 20260508 07:47:25.689 | INFO | Downloaded 34GiB to GPU in 2.44769s (13.8907GiB/s) | 20260508 07:47:30.830 | INFO | Analyzing chunk frames = [6427648,6428160) | 20260508 07:47:30.831 | INFO | MemInfo: { | 20260508 07:47:30.831 | INFO | "gpu" : " 17 GB", | 20260508 07:47:30.831 | INFO | "huge" : "", | 20260508 07:47:30.831 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:30.831 | INFO | "malloc" : " 14 GB", | 20260508 07:47:30.831 | INFO | "pinned" : " 29 GB" | 20260508 07:47:30.831 | INFO | } | 20260508 07:47:30.831 | INFO | Perf report for the last 5.1419 seconds: | 20260508 07:47:30.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:30.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 07:47:30.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:47:30.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:30.831 | INFO | Uploaded 36GiB to GPU in 2.55731s (14.0773GiB/s) | 20260508 07:47:30.831 | INFO | Downloaded 34GiB to GPU in 2.43177s (13.9816GiB/s) | 20260508 07:47:35.969 | INFO | Analyzing chunk frames = [6428160,6428672) | 20260508 07:47:35.969 | INFO | MemInfo: { | 20260508 07:47:35.969 | INFO | "gpu" : " 17 GB", | 20260508 07:47:35.969 | INFO | "huge" : "", | 20260508 07:47:35.969 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:35.969 | INFO | "malloc" : " 14 GB", | 20260508 07:47:35.969 | INFO | "pinned" : " 29 GB" | 20260508 07:47:35.969 | INFO | } | 20260508 07:47:35.969 | INFO | Perf report for the last 5.13803 seconds: | 20260508 07:47:35.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:35.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 07:47:35.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:47:35.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:35.969 | INFO | Uploaded 36GiB to GPU in 2.56215s (14.0507GiB/s) | 20260508 07:47:35.969 | INFO | Downloaded 34GiB to GPU in 2.43485s (13.9639GiB/s) | 20260508 07:47:41.107 | INFO | Analyzing chunk frames = [6428672,6429184) | 20260508 07:47:41.107 | INFO | MemInfo: { | 20260508 07:47:41.107 | INFO | "gpu" : " 17 GB", | 20260508 07:47:41.107 | INFO | "huge" : "", | 20260508 07:47:41.107 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:41.107 | INFO | "malloc" : " 14 GB", | 20260508 07:47:41.107 | INFO | "pinned" : " 29 GB" | 20260508 07:47:41.107 | INFO | } | 20260508 07:47:41.107 | INFO | Perf report for the last 5.13846 seconds: | 20260508 07:47:41.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:41.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.107 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:47:41.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.107 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:47:41.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:41.108 | INFO | Uploaded 36GiB to GPU in 2.36056s (15.2506GiB/s) | 20260508 07:47:41.108 | INFO | Downloaded 34GiB to GPU in 2.51449s (13.5216GiB/s) | 20260508 07:47:46.245 | INFO | Analyzing chunk frames = [6429184,6429696) | 20260508 07:47:46.245 | INFO | MemInfo: { | 20260508 07:47:46.245 | INFO | "gpu" : " 17 GB", | 20260508 07:47:46.245 | INFO | "huge" : "", | 20260508 07:47:46.245 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:46.245 | INFO | "malloc" : " 14 GB", | 20260508 07:47:46.245 | INFO | "pinned" : " 29 GB" | 20260508 07:47:46.245 | INFO | } | 20260508 07:47:46.245 | INFO | Perf report for the last 5.13756 seconds: | 20260508 07:47:46.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:46.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:47:46.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:47:46.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:46.245 | INFO | Uploaded 36GiB to GPU in 2.37581s (15.1527GiB/s) | 20260508 07:47:46.245 | INFO | Downloaded 34GiB to GPU in 2.43953s (13.9371GiB/s) | 20260508 07:47:51.392 | INFO | Analyzing chunk frames = [6429696,6430208) | 20260508 07:47:51.392 | INFO | MemInfo: { | 20260508 07:47:51.392 | INFO | "gpu" : " 17 GB", | 20260508 07:47:51.392 | INFO | "huge" : "", | 20260508 07:47:51.392 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:51.392 | INFO | "malloc" : " 14 GB", | 20260508 07:47:51.392 | INFO | "pinned" : " 29 GB" | 20260508 07:47:51.392 | INFO | } | 20260508 07:47:51.392 | INFO | Perf report for the last 5.14731 seconds: | 20260508 07:47:51.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:51.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.392 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 07:47:51.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.392 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:47:51.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:51.393 | INFO | Uploaded 36GiB to GPU in 2.46077s (14.6295GiB/s) | 20260508 07:47:51.393 | INFO | Downloaded 34GiB to GPU in 2.43752s (13.9486GiB/s) | 20260508 07:47:56.524 | INFO | Analyzing chunk frames = [6430208,6430720) | 20260508 07:47:56.524 | INFO | MemInfo: { | 20260508 07:47:56.524 | INFO | "gpu" : " 17 GB", | 20260508 07:47:56.524 | INFO | "huge" : "", | 20260508 07:47:56.524 | INFO | "huge_shared" : " 29 GB", | 20260508 07:47:56.524 | INFO | "malloc" : " 14 GB", | 20260508 07:47:56.524 | INFO | "pinned" : " 29 GB" | 20260508 07:47:56.524 | INFO | } | 20260508 07:47:56.524 | INFO | Perf report for the last 5.13203 seconds: | 20260508 07:47:56.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:47:56.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.524 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:47:56.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.524 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:47:56.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:47:56.525 | INFO | Uploaded 36GiB to GPU in 2.48124s (14.5089GiB/s) | 20260508 07:47:56.525 | INFO | Downloaded 34GiB to GPU in 2.46408s (13.7983GiB/s) | 20260508 07:47:56.525 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:47:56.525 | INFO | Total mem usage: 16.7 GiB | 20260508 07:47:56.525 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:47:56.525 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:47:56.525 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:47:56.525 | INFO | High Water Breakdown: | 20260508 07:47:56.525 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:47:56.525 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:47:56.525 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:47:56.525 | INFO | ------------------------------------------ | 20260508 07:47:56.525 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:47:56.525 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:47:56.525 | INFO | Total mem usage: 26.5 GiB | 20260508 07:47:56.525 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:47:56.525 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:47:56.525 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:47:56.525 | INFO | High Water Breakdown: | 20260508 07:47:56.525 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:47:56.525 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:47:56.525 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:47:56.525 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:47:56.525 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:47:56.525 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:47:56.525 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:47:56.525 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:47:56.525 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:47:56.525 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:47:56.525 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:47:56.525 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:47:56.525 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:47:56.525 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:47:56.525 | INFO | ------------------------------------------ | 20260508 07:47:56.525 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:47:56.525 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:47:56.525 | INFO | Total mem usage: 27.8 GiB | 20260508 07:47:56.525 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:47:56.525 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:47:56.525 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:47:56.525 | INFO | High Water Breakdown: | 20260508 07:47:56.525 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:47:56.525 | INFO | ------------------------------------------ | 20260508 07:47:56.525 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:47:56.525 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:47:56.525 | INFO | Total mem usage: 15.7 GiB | 20260508 07:47:56.525 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:47:56.525 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:47:56.525 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:47:56.525 | INFO | High Water Breakdown: | 20260508 07:47:56.525 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:47:56.525 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:47:56.525 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:47:56.525 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:47:56.525 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:47:56.525 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:47:56.525 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:47:56.525 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:47:56.525 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:47:56.525 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:47:56.525 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:47:56.525 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:47:56.525 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:47:56.526 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:47:56.526 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:47:56.526 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:47:56.526 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:47:56.526 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:47:56.526 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:47:56.526 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:47:56.526 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:47:56.526 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:47:56.526 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:47:56.526 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:47:56.526 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:47:56.526 | INFO | ------------------------------------------ | 20260508 07:47:56.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:47:56.526 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:47:56.526 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:47:56.526 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:47:56.526 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.72e-01 | 20260508 07:47:56.526 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.06e-01, 1.14e-02 / 4.97e-03 | 20260508 07:47:56.526 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.43e+00, 5.89e-01 / 1.37e-01 | 20260508 07:47:56.526 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.06e-01 | 20260508 07:47:56.526 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.51e+01, 3.79e+00 / 8.90e-01 | 20260508 07:47:56.526 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.82e+00 / 1.17e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.36e+00 / 2.53e+00, 4.08e-01 / 1.78e-01 | 20260508 07:47:56.526 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.14e+00, 2.52e-01 / 9.92e-02 | 20260508 07:47:56.526 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.09e-01 / 3.35e-01, 4.93e-02 / 1.96e-02 | 20260508 07:47:56.526 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:47:56.526 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:47:56.526 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:47:56.526 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 24.30%, Upload -- 2.57e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 24.48%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 30.92%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 2.17%, LockUpload -- 2.29e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 2.60%, LockDownload -- 2.75e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 15.52%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 07:47:56.526 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260508 07:47:56.622 | INFO | Starting a round of Baz IO | 20260508 07:48:00.055 | INFO | Populating RTMetrics report | 20260508 07:48:00.055 | INFO | RTMetrics report populated | 20260508 07:48:00.055 | INFO | Opening tmp RTMetrics json file | 20260508 07:48:00.068 | INFO | Serializing RTMetrics to json | 20260508 07:48:00.069 | INFO | Serialization done, swapping tmp file now... | 20260508 07:48:00.069 | INFO | RTMetrics done | 20260508 07:48:00.122 | INFO | Finished a round of Baz IO: Wrote 1.52 GiB in 3.499 seconds | 20260508 07:48:00.122 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.00% / 59.49% | 20260508 07:48:00.122 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.327059% / 1.19209e-05% / 5.16574e-05% | 20260508 07:48:00.122 | INFO | Baz IO overall: | 20260508 07:48:00.122 | INFO | BytesWritten : 2831923190134 | 20260508 07:48:00.122 | INFO | NumEvents : 1364048166229 | 20260508 07:48:01.660 | INFO | Analyzing chunk frames = [6430720,6431232) | 20260508 07:48:01.660 | INFO | MemInfo: { | 20260508 07:48:01.660 | INFO | "gpu" : " 17 GB", | 20260508 07:48:01.660 | INFO | "huge" : "", | 20260508 07:48:01.660 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:01.660 | INFO | "malloc" : " 14 GB", | 20260508 07:48:01.660 | INFO | "pinned" : " 29 GB" | 20260508 07:48:01.660 | INFO | } | 20260508 07:48:01.660 | INFO | Perf report for the last 5.13609 seconds: | 20260508 07:48:01.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:01.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:01.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:01.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:01.660 | INFO | ANALYSIS : 68.1%, 3.0, 106.2ms, 0 | 20260508 07:48:01.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:01.660 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 07:48:01.660 | INFO | PRE_HQ : 65.7%, 1.6, 54.7ms, 0 | 20260508 07:48:01.660 | INFO | BAZWRITER : 67.4%, 2.6, 88.8ms, 0 | 20260508 07:48:01.660 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:48:01.661 | INFO | Uploaded 36GiB to GPU in 2.51132s (14.3351GiB/s) | 20260508 07:48:01.661 | INFO | Downloaded 38GiB to GPU in 2.69633s (14.0932GiB/s) | 20260508 07:48:06.806 | INFO | Analyzing chunk frames = [6431232,6431744) | 20260508 07:48:06.806 | INFO | MemInfo: { | 20260508 07:48:06.806 | INFO | "gpu" : " 17 GB", | 20260508 07:48:06.806 | INFO | "huge" : "", | 20260508 07:48:06.806 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:06.806 | INFO | "malloc" : " 14 GB", | 20260508 07:48:06.806 | INFO | "pinned" : " 29 GB" | 20260508 07:48:06.806 | INFO | } | 20260508 07:48:06.806 | INFO | Perf report for the last 5.14555 seconds: | 20260508 07:48:06.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:06.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 07:48:06.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:48:06.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:06.806 | INFO | Uploaded 36GiB to GPU in 2.44513s (14.7231GiB/s) | 20260508 07:48:06.807 | INFO | Downloaded 34GiB to GPU in 2.45387s (13.8557GiB/s) | 20260508 07:48:11.951 | INFO | Analyzing chunk frames = [6431744,6432256) | 20260508 07:48:11.951 | INFO | MemInfo: { | 20260508 07:48:11.951 | INFO | "gpu" : " 17 GB", | 20260508 07:48:11.951 | INFO | "huge" : "", | 20260508 07:48:11.951 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:11.951 | INFO | "malloc" : " 14 GB", | 20260508 07:48:11.951 | INFO | "pinned" : " 29 GB" | 20260508 07:48:11.951 | INFO | } | 20260508 07:48:11.951 | INFO | Perf report for the last 5.14539 seconds: | 20260508 07:48:11.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:11.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.951 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 07:48:11.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.951 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:48:11.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:11.952 | INFO | Uploaded 36GiB to GPU in 2.55326s (14.0996GiB/s) | 20260508 07:48:11.952 | INFO | Downloaded 34GiB to GPU in 2.46174s (13.8114GiB/s) | 20260508 07:48:17.089 | INFO | Analyzing chunk frames = [6432256,6432768) | 20260508 07:48:17.090 | INFO | MemInfo: { | 20260508 07:48:17.090 | INFO | "gpu" : " 17 GB", | 20260508 07:48:17.090 | INFO | "huge" : "", | 20260508 07:48:17.090 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:17.090 | INFO | "malloc" : " 14 GB", | 20260508 07:48:17.090 | INFO | "pinned" : " 29 GB" | 20260508 07:48:17.090 | INFO | } | 20260508 07:48:17.090 | INFO | Perf report for the last 5.13848 seconds: | 20260508 07:48:17.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:17.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 07:48:17.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:48:17.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:17.090 | INFO | Uploaded 36GiB to GPU in 2.54152s (14.1648GiB/s) | 20260508 07:48:17.090 | INFO | Downloaded 34GiB to GPU in 2.4529s (13.8611GiB/s) | 20260508 07:48:22.227 | INFO | Analyzing chunk frames = [6432768,6433280) | 20260508 07:48:22.227 | INFO | MemInfo: { | 20260508 07:48:22.227 | INFO | "gpu" : " 17 GB", | 20260508 07:48:22.227 | INFO | "huge" : "", | 20260508 07:48:22.227 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:22.227 | INFO | "malloc" : " 14 GB", | 20260508 07:48:22.227 | INFO | "pinned" : " 29 GB" | 20260508 07:48:22.227 | INFO | } | 20260508 07:48:22.227 | INFO | Perf report for the last 5.13721 seconds: | 20260508 07:48:22.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:22.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.227 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 07:48:22.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.227 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:48:22.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:22.228 | INFO | Uploaded 36GiB to GPU in 2.27929s (15.7944GiB/s) | 20260508 07:48:22.228 | INFO | Downloaded 34GiB to GPU in 2.51161s (13.5371GiB/s) | 20260508 07:48:27.360 | INFO | Analyzing chunk frames = [6433280,6433792) | 20260508 07:48:27.361 | INFO | MemInfo: { | 20260508 07:48:27.361 | INFO | "gpu" : " 17 GB", | 20260508 07:48:27.361 | INFO | "huge" : "", | 20260508 07:48:27.361 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:27.361 | INFO | "malloc" : " 14 GB", | 20260508 07:48:27.361 | INFO | "pinned" : " 29 GB" | 20260508 07:48:27.361 | INFO | } | 20260508 07:48:27.361 | INFO | Perf report for the last 5.13394 seconds: | 20260508 07:48:27.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:27.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 07:48:27.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 07:48:27.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:27.361 | INFO | Uploaded 36GiB to GPU in 2.34895s (15.326GiB/s) | 20260508 07:48:27.361 | INFO | Downloaded 34GiB to GPU in 2.45185s (13.8671GiB/s) | 20260508 07:48:32.507 | INFO | Analyzing chunk frames = [6433792,6434304) | 20260508 07:48:32.507 | INFO | MemInfo: { | 20260508 07:48:32.507 | INFO | "gpu" : " 17 GB", | 20260508 07:48:32.507 | INFO | "huge" : "", | 20260508 07:48:32.507 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:32.507 | INFO | "malloc" : " 14 GB", | 20260508 07:48:32.507 | INFO | "pinned" : " 29 GB" | 20260508 07:48:32.507 | INFO | } | 20260508 07:48:32.507 | INFO | Perf report for the last 5.14644 seconds: | 20260508 07:48:32.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:32.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.507 | INFO | ANALYSIS : 66.1%, 3.0, 103.6ms, 0 | 20260508 07:48:32.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.507 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 07:48:32.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:32.508 | INFO | Uploaded 36GiB to GPU in 2.47404s (14.5511GiB/s) | 20260508 07:48:32.508 | INFO | Downloaded 34GiB to GPU in 2.46173s (13.8115GiB/s) | 20260508 07:48:37.640 | INFO | Analyzing chunk frames = [6434304,6434816) | 20260508 07:48:37.640 | INFO | MemInfo: { | 20260508 07:48:37.640 | INFO | "gpu" : " 17 GB", | 20260508 07:48:37.641 | INFO | "huge" : "", | 20260508 07:48:37.641 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:37.641 | INFO | "malloc" : " 14 GB", | 20260508 07:48:37.641 | INFO | "pinned" : " 29 GB" | 20260508 07:48:37.641 | INFO | } | 20260508 07:48:37.641 | INFO | Perf report for the last 5.13345 seconds: | 20260508 07:48:37.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:37.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 07:48:37.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260508 07:48:37.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:37.641 | INFO | Uploaded 36GiB to GPU in 2.37869s (15.1344GiB/s) | 20260508 07:48:37.641 | INFO | Downloaded 34GiB to GPU in 2.45623s (13.8423GiB/s) | 20260508 07:48:37.755 | INFO | Starting a round of Baz IO | 20260508 07:48:41.158 | INFO | Populating RTMetrics report | 20260508 07:48:41.159 | INFO | RTMetrics report populated | 20260508 07:48:41.159 | INFO | Opening tmp RTMetrics json file | 20260508 07:48:41.169 | INFO | Serializing RTMetrics to json | 20260508 07:48:41.171 | INFO | Serialization done, swapping tmp file now... | 20260508 07:48:41.171 | INFO | RTMetrics done | 20260508 07:48:41.235 | INFO | Finished a round of Baz IO: Wrote 1.52 GiB in 3.479 seconds | 20260508 07:48:41.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.00% / 59.49% | 20260508 07:48:41.235 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.326757% / 0% / 5.5631e-05% | 20260508 07:48:41.235 | INFO | Baz IO overall: | 20260508 07:48:41.235 | INFO | BytesWritten : 2833554799759 | 20260508 07:48:41.235 | INFO | NumEvents : 1364822736108 | 20260508 07:48:42.783 | INFO | Analyzing chunk frames = [6434816,6435328) | 20260508 07:48:42.783 | INFO | MemInfo: { | 20260508 07:48:42.783 | INFO | "gpu" : " 17 GB", | 20260508 07:48:42.783 | INFO | "huge" : "", | 20260508 07:48:42.783 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:42.783 | INFO | "malloc" : " 14 GB", | 20260508 07:48:42.783 | INFO | "pinned" : " 29 GB" | 20260508 07:48:42.783 | INFO | } | 20260508 07:48:42.783 | INFO | Perf report for the last 5.14241 seconds: | 20260508 07:48:42.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:42.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:42.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:42.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:42.783 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 07:48:42.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:42.783 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 07:48:42.783 | INFO | PRE_HQ : 66.5%, 1.8, 61.6ms, 0 | 20260508 07:48:42.783 | INFO | BAZWRITER : 67.1%, 2.8, 96.4ms, 0 | 20260508 07:48:42.783 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:48:42.784 | INFO | Uploaded 36GiB to GPU in 2.58359s (13.9341GiB/s) | 20260508 07:48:42.784 | INFO | Downloaded 38GiB to GPU in 2.70687s (14.0384GiB/s) | 20260508 07:48:47.916 | INFO | Analyzing chunk frames = [6435328,6435840) | 20260508 07:48:47.916 | INFO | MemInfo: { | 20260508 07:48:47.916 | INFO | "gpu" : " 17 GB", | 20260508 07:48:47.916 | INFO | "huge" : "", | 20260508 07:48:47.916 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:47.916 | INFO | "malloc" : " 14 GB", | 20260508 07:48:47.916 | INFO | "pinned" : " 29 GB" | 20260508 07:48:47.916 | INFO | } | 20260508 07:48:47.917 | INFO | Perf report for the last 5.13322 seconds: | 20260508 07:48:47.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:47.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 07:48:47.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:48:47.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:47.917 | INFO | Uploaded 36GiB to GPU in 2.50867s (14.3502GiB/s) | 20260508 07:48:47.917 | INFO | Downloaded 34GiB to GPU in 2.45007s (13.8771GiB/s) | 20260508 07:48:53.059 | INFO | Analyzing chunk frames = [6435840,6436352) | 20260508 07:48:53.059 | INFO | MemInfo: { | 20260508 07:48:53.059 | INFO | "gpu" : " 17 GB", | 20260508 07:48:53.059 | INFO | "huge" : "", | 20260508 07:48:53.059 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:53.059 | INFO | "malloc" : " 14 GB", | 20260508 07:48:53.059 | INFO | "pinned" : " 29 GB" | 20260508 07:48:53.059 | INFO | } | 20260508 07:48:53.059 | INFO | Perf report for the last 5.14281 seconds: | 20260508 07:48:53.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:53.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.059 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 07:48:53.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.059 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:48:53.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:53.060 | INFO | Uploaded 36GiB to GPU in 2.48268s (14.5004GiB/s) | 20260508 07:48:53.060 | INFO | Downloaded 34GiB to GPU in 2.44061s (13.9309GiB/s) | 20260508 07:48:58.200 | INFO | Analyzing chunk frames = [6436352,6436864) | 20260508 07:48:58.200 | INFO | MemInfo: { | 20260508 07:48:58.200 | INFO | "gpu" : " 17 GB", | 20260508 07:48:58.200 | INFO | "huge" : "", | 20260508 07:48:58.200 | INFO | "huge_shared" : " 29 GB", | 20260508 07:48:58.200 | INFO | "malloc" : " 14 GB", | 20260508 07:48:58.200 | INFO | "pinned" : " 29 GB" | 20260508 07:48:58.200 | INFO | } | 20260508 07:48:58.200 | INFO | Perf report for the last 5.14084 seconds: | 20260508 07:48:58.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:48:58.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 07:48:58.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:48:58.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:48:58.200 | INFO | Uploaded 36GiB to GPU in 2.56723s (14.0229GiB/s) | 20260508 07:48:58.200 | INFO | Downloaded 34GiB to GPU in 2.43466s (13.965GiB/s) | 20260508 07:49:03.340 | INFO | Analyzing chunk frames = [6436864,6437376) | 20260508 07:49:03.340 | INFO | MemInfo: { | 20260508 07:49:03.340 | INFO | "gpu" : " 17 GB", | 20260508 07:49:03.340 | INFO | "huge" : "", | 20260508 07:49:03.340 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:03.340 | INFO | "malloc" : " 14 GB", | 20260508 07:49:03.340 | INFO | "pinned" : " 29 GB" | 20260508 07:49:03.340 | INFO | } | 20260508 07:49:03.340 | INFO | Perf report for the last 5.13994 seconds: | 20260508 07:49:03.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:03.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.340 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:49:03.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.340 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:49:03.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:03.341 | INFO | Uploaded 36GiB to GPU in 2.41012s (14.937GiB/s) | 20260508 07:49:03.341 | INFO | Downloaded 34GiB to GPU in 2.50543s (13.5705GiB/s) | 20260508 07:49:08.476 | INFO | Analyzing chunk frames = [6437376,6437888) | 20260508 07:49:08.476 | INFO | MemInfo: { | 20260508 07:49:08.476 | INFO | "gpu" : " 17 GB", | 20260508 07:49:08.476 | INFO | "huge" : "", | 20260508 07:49:08.476 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:08.476 | INFO | "malloc" : " 14 GB", | 20260508 07:49:08.476 | INFO | "pinned" : " 29 GB" | 20260508 07:49:08.476 | INFO | } | 20260508 07:49:08.477 | INFO | Perf report for the last 5.13642 seconds: | 20260508 07:49:08.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:08.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:49:08.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:49:08.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:08.477 | INFO | Uploaded 36GiB to GPU in 2.393s (15.0439GiB/s) | 20260508 07:49:08.477 | INFO | Downloaded 34GiB to GPU in 2.42669s (14.0109GiB/s) | 20260508 07:49:13.615 | INFO | Analyzing chunk frames = [6437888,6438400) | 20260508 07:49:13.615 | INFO | MemInfo: { | 20260508 07:49:13.615 | INFO | "gpu" : " 17 GB", | 20260508 07:49:13.615 | INFO | "huge" : "", | 20260508 07:49:13.616 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:13.616 | INFO | "malloc" : " 14 GB", | 20260508 07:49:13.616 | INFO | "pinned" : " 29 GB" | 20260508 07:49:13.616 | INFO | } | 20260508 07:49:13.616 | INFO | Perf report for the last 5.13907 seconds: | 20260508 07:49:13.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:13.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 07:49:13.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 07:49:13.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:13.616 | INFO | Uploaded 36GiB to GPU in 2.43375s (14.792GiB/s) | 20260508 07:49:13.616 | INFO | Downloaded 34GiB to GPU in 2.43483s (13.964GiB/s) | 20260508 07:49:18.754 | INFO | Analyzing chunk frames = [6438400,6438912) | 20260508 07:49:18.754 | INFO | MemInfo: { | 20260508 07:49:18.754 | INFO | "gpu" : " 17 GB", | 20260508 07:49:18.754 | INFO | "huge" : "", | 20260508 07:49:18.754 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:18.755 | INFO | "malloc" : " 14 GB", | 20260508 07:49:18.755 | INFO | "pinned" : " 29 GB" | 20260508 07:49:18.755 | INFO | } | 20260508 07:49:18.755 | INFO | Perf report for the last 5.139 seconds: | 20260508 07:49:18.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:18.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 07:49:18.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 07:49:18.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:18.755 | INFO | Uploaded 36GiB to GPU in 2.44574s (14.7195GiB/s) | 20260508 07:49:18.755 | INFO | Downloaded 34GiB to GPU in 2.44132s (13.9269GiB/s) | 20260508 07:49:18.756 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:49:18.756 | INFO | Total mem usage: 16.7 GiB | 20260508 07:49:18.756 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:49:18.756 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 07:49:18.756 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:49:18.756 | INFO | High Water Breakdown: | 20260508 07:49:18.756 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:49:18.756 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:49:18.756 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:49:18.756 | INFO | ------------------------------------------ | 20260508 07:49:18.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:49:18.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:49:18.756 | INFO | Total mem usage: 26.5 GiB | 20260508 07:49:18.756 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:49:18.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:49:18.756 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:49:18.756 | INFO | High Water Breakdown: | 20260508 07:49:18.756 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:49:18.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:49:18.756 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:49:18.756 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:49:18.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:49:18.756 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:49:18.756 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:49:18.756 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:49:18.756 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:49:18.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:49:18.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:49:18.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:49:18.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:49:18.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:49:18.756 | INFO | ------------------------------------------ | 20260508 07:49:18.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:49:18.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:49:18.756 | INFO | Total mem usage: 27.8 GiB | 20260508 07:49:18.756 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:49:18.756 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:49:18.756 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:49:18.756 | INFO | High Water Breakdown: | 20260508 07:49:18.756 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:49:18.756 | INFO | ------------------------------------------ | 20260508 07:49:18.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:49:18.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:49:18.756 | INFO | Total mem usage: 15.7 GiB | 20260508 07:49:18.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:49:18.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:49:18.756 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:49:18.756 | INFO | High Water Breakdown: | 20260508 07:49:18.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:49:18.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:49:18.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:49:18.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:49:18.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:49:18.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:49:18.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:49:18.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:49:18.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:49:18.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:49:18.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:49:18.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:49:18.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:49:18.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:49:18.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:49:18.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:49:18.756 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:49:18.756 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:49:18.756 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:49:18.756 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:49:18.757 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:49:18.757 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:49:18.757 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:49:18.757 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:49:18.757 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:49:18.757 | INFO | ------------------------------------------ | 20260508 07:49:18.757 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:49:18.757 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:49:18.757 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:49:18.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:49:18.757 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.41e+01 / 7.24e-01 | 20260508 07:49:18.757 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.05e-01, 1.21e-02 / 5.33e-03 | 20260508 07:49:18.757 | INFO | PerfReport: 12.11%, Baseline -- 3.97e+00 / 4.47e+00, 5.88e-01 / 2.68e-01 | 20260508 07:49:18.757 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.35e-01 | 20260508 07:49:18.757 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.53e+01, 3.80e+00 / 1.14e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.87e+00 / 1.28e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.58e+00, 4.03e-01 / 2.57e-01 | 20260508 07:49:18.757 | INFO | PerfReport: 3.09%, Metrics -- 1.01e+00 / 1.20e+00, 2.51e-01 / 1.40e-01 | 20260508 07:49:18.757 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.55e-01, 5.00e-02 / 3.45e-02 | 20260508 07:49:18.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:49:18.757 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:49:18.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:49:18.757 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 24.22%, Upload -- 2.56e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 24.49%, Download -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 30.99%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 2.11%, LockUpload -- 2.24e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 2.61%, LockDownload -- 2.76e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 15.56%, LockCompute -- 1.65e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 07:49:18.757 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260508 07:49:18.867 | INFO | Starting a round of Baz IO | 20260508 07:49:22.273 | INFO | Populating RTMetrics report | 20260508 07:49:22.273 | INFO | RTMetrics report populated | 20260508 07:49:22.273 | INFO | Opening tmp RTMetrics json file | 20260508 07:49:22.287 | INFO | Serializing RTMetrics to json | 20260508 07:49:22.289 | INFO | Serialization done, swapping tmp file now... | 20260508 07:49:22.289 | INFO | RTMetrics done | 20260508 07:49:22.350 | INFO | Finished a round of Baz IO: Wrote 1.517 GiB in 3.483 seconds | 20260508 07:49:22.350 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.00% / 59.50% | 20260508 07:49:22.350 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.323784% / 0% / 4.76837e-05% | 20260508 07:49:22.350 | INFO | Baz IO overall: | 20260508 07:49:22.350 | INFO | BytesWritten : 2835184046072 | 20260508 07:49:22.350 | INFO | NumEvents : 1365595667176 | 20260508 07:49:23.899 | INFO | Analyzing chunk frames = [6438912,6439424) | 20260508 07:49:23.899 | INFO | MemInfo: { | 20260508 07:49:23.899 | INFO | "gpu" : " 17 GB", | 20260508 07:49:23.899 | INFO | "huge" : "", | 20260508 07:49:23.899 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:23.899 | INFO | "malloc" : " 14 GB", | 20260508 07:49:23.899 | INFO | "pinned" : " 29 GB" | 20260508 07:49:23.899 | INFO | } | 20260508 07:49:23.899 | INFO | Perf report for the last 5.14452 seconds: | 20260508 07:49:23.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:23.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:23.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:23.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:23.899 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 07:49:23.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:23.899 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 07:49:23.899 | INFO | PRE_HQ : 66.4%, 1.8, 61.8ms, 0 | 20260508 07:49:23.899 | INFO | BAZWRITER : 67.1%, 2.8, 97.1ms, 0 | 20260508 07:49:23.899 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:49:23.900 | INFO | Uploaded 36GiB to GPU in 2.595s (13.8728GiB/s) | 20260508 07:49:23.900 | INFO | Downloaded 38GiB to GPU in 2.68773s (14.1383GiB/s) | 20260508 07:49:29.038 | INFO | Analyzing chunk frames = [6439424,6439936) | 20260508 07:49:29.038 | INFO | MemInfo: { | 20260508 07:49:29.038 | INFO | "gpu" : " 17 GB", | 20260508 07:49:29.038 | INFO | "huge" : "", | 20260508 07:49:29.038 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:29.038 | INFO | "malloc" : " 14 GB", | 20260508 07:49:29.038 | INFO | "pinned" : " 29 GB" | 20260508 07:49:29.038 | INFO | } | 20260508 07:49:29.038 | INFO | Perf report for the last 5.13891 seconds: | 20260508 07:49:29.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:29.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:49:29.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:49:29.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:29.038 | INFO | Uploaded 36GiB to GPU in 2.53698s (14.1901GiB/s) | 20260508 07:49:29.038 | INFO | Downloaded 34GiB to GPU in 2.4458s (13.9014GiB/s) | 20260508 07:49:34.172 | INFO | Analyzing chunk frames = [6439936,6440448) | 20260508 07:49:34.172 | INFO | MemInfo: { | 20260508 07:49:34.172 | INFO | "gpu" : " 17 GB", | 20260508 07:49:34.172 | INFO | "huge" : "", | 20260508 07:49:34.172 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:34.172 | INFO | "malloc" : " 14 GB", | 20260508 07:49:34.172 | INFO | "pinned" : " 29 GB" | 20260508 07:49:34.172 | INFO | } | 20260508 07:49:34.172 | INFO | Perf report for the last 5.1339 seconds: | 20260508 07:49:34.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:34.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 07:49:34.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:49:34.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:34.172 | INFO | Uploaded 36GiB to GPU in 2.51697s (14.3029GiB/s) | 20260508 07:49:34.172 | INFO | Downloaded 34GiB to GPU in 2.43931s (13.9384GiB/s) | 20260508 07:49:39.315 | INFO | Analyzing chunk frames = [6440448,6440960) | 20260508 07:49:39.315 | INFO | MemInfo: { | 20260508 07:49:39.315 | INFO | "gpu" : " 17 GB", | 20260508 07:49:39.315 | INFO | "huge" : "", | 20260508 07:49:39.315 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:39.315 | INFO | "malloc" : " 14 GB", | 20260508 07:49:39.315 | INFO | "pinned" : " 29 GB" | 20260508 07:49:39.315 | INFO | } | 20260508 07:49:39.315 | INFO | Perf report for the last 5.14313 seconds: | 20260508 07:49:39.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:39.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.315 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 07:49:39.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.315 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:49:39.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:39.316 | INFO | Uploaded 36GiB to GPU in 2.58776s (13.9117GiB/s) | 20260508 07:49:39.316 | INFO | Downloaded 34GiB to GPU in 2.42419s (14.0253GiB/s) | 20260508 07:49:44.452 | INFO | Analyzing chunk frames = [6440960,6441472) | 20260508 07:49:44.452 | INFO | MemInfo: { | 20260508 07:49:44.452 | INFO | "gpu" : " 17 GB", | 20260508 07:49:44.452 | INFO | "huge" : "", | 20260508 07:49:44.452 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:44.452 | INFO | "malloc" : " 14 GB", | 20260508 07:49:44.452 | INFO | "pinned" : " 29 GB" | 20260508 07:49:44.452 | INFO | } | 20260508 07:49:44.452 | INFO | Perf report for the last 5.13715 seconds: | 20260508 07:49:44.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:44.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.452 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 07:49:44.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.452 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:49:44.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:44.453 | INFO | Uploaded 36GiB to GPU in 2.4377s (14.768GiB/s) | 20260508 07:49:44.453 | INFO | Downloaded 34GiB to GPU in 2.54355s (13.3672GiB/s) | 20260508 07:49:49.595 | INFO | Analyzing chunk frames = [6441472,6441984) | 20260508 07:49:49.595 | INFO | MemInfo: { | 20260508 07:49:49.595 | INFO | "gpu" : " 17 GB", | 20260508 07:49:49.595 | INFO | "huge" : "", | 20260508 07:49:49.595 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:49.595 | INFO | "malloc" : " 14 GB", | 20260508 07:49:49.595 | INFO | "pinned" : " 29 GB" | 20260508 07:49:49.595 | INFO | } | 20260508 07:49:49.595 | INFO | Perf report for the last 5.14278 seconds: | 20260508 07:49:49.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:49.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.595 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:49:49.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.595 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:49:49.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:49.596 | INFO | Uploaded 36GiB to GPU in 2.39281s (15.0451GiB/s) | 20260508 07:49:49.596 | INFO | Downloaded 34GiB to GPU in 2.43145s (13.9834GiB/s) | 20260508 07:49:54.735 | INFO | Analyzing chunk frames = [6441984,6442496) | 20260508 07:49:54.735 | INFO | MemInfo: { | 20260508 07:49:54.735 | INFO | "gpu" : " 17 GB", | 20260508 07:49:54.735 | INFO | "huge" : "", | 20260508 07:49:54.735 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:54.735 | INFO | "malloc" : " 14 GB", | 20260508 07:49:54.735 | INFO | "pinned" : " 29 GB" | 20260508 07:49:54.736 | INFO | } | 20260508 07:49:54.736 | INFO | Perf report for the last 5.14047 seconds: | 20260508 07:49:54.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:54.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.736 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:49:54.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.736 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:49:54.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:54.737 | INFO | Uploaded 36GiB to GPU in 2.483s (14.4986GiB/s) | 20260508 07:49:54.737 | INFO | Downloaded 34GiB to GPU in 2.43125s (13.9846GiB/s) | 20260508 07:49:59.869 | INFO | Analyzing chunk frames = [6442496,6443008) | 20260508 07:49:59.869 | INFO | MemInfo: { | 20260508 07:49:59.869 | INFO | "gpu" : " 17 GB", | 20260508 07:49:59.869 | INFO | "huge" : "", | 20260508 07:49:59.869 | INFO | "huge_shared" : " 29 GB", | 20260508 07:49:59.869 | INFO | "malloc" : " 14 GB", | 20260508 07:49:59.869 | INFO | "pinned" : " 29 GB" | 20260508 07:49:59.869 | INFO | } | 20260508 07:49:59.869 | INFO | Perf report for the last 5.13354 seconds: | 20260508 07:49:59.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:49:59.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.869 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 07:49:59.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.869 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260508 07:49:59.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:49:59.870 | INFO | Uploaded 36GiB to GPU in 2.43199s (14.8027GiB/s) | 20260508 07:49:59.870 | INFO | Downloaded 34GiB to GPU in 2.43689s (13.9522GiB/s) | 20260508 07:49:59.996 | INFO | Starting a round of Baz IO | 20260508 07:50:03.383 | INFO | Populating RTMetrics report | 20260508 07:50:03.384 | INFO | RTMetrics report populated | 20260508 07:50:03.384 | INFO | Opening tmp RTMetrics json file | 20260508 07:50:03.393 | INFO | Serializing RTMetrics to json | 20260508 07:50:03.396 | INFO | Serialization done, swapping tmp file now... | 20260508 07:50:03.397 | INFO | RTMetrics done | 20260508 07:50:03.457 | INFO | Finished a round of Baz IO: Wrote 1.52 GiB in 3.46 seconds | 20260508 07:50:03.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.00% / 59.50% | 20260508 07:50:03.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.324957% / 3.97364e-06% / 5.16574e-05% | 20260508 07:50:03.457 | INFO | Baz IO overall: | 20260508 07:50:03.457 | INFO | BytesWritten : 2836816319832 | 20260508 07:50:03.457 | INFO | NumEvents : 1366371109923 | 20260508 07:50:05.013 | INFO | Analyzing chunk frames = [6443008,6443520) | 20260508 07:50:05.013 | INFO | MemInfo: { | 20260508 07:50:05.013 | INFO | "gpu" : " 17 GB", | 20260508 07:50:05.013 | INFO | "huge" : "", | 20260508 07:50:05.013 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:05.013 | INFO | "malloc" : " 14 GB", | 20260508 07:50:05.014 | INFO | "pinned" : " 29 GB" | 20260508 07:50:05.014 | INFO | } | 20260508 07:50:05.014 | INFO | Perf report for the last 5.14451 seconds: | 20260508 07:50:05.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:05.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:05.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:05.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:05.014 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 07:50:05.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:05.014 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 07:50:05.014 | INFO | PRE_HQ : 66.4%, 1.8, 62.4ms, 0 | 20260508 07:50:05.014 | INFO | BAZWRITER : 66.8%, 2.8, 96.8ms, 0 | 20260508 07:50:05.014 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:50:05.014 | INFO | Uploaded 36GiB to GPU in 2.59484s (13.8737GiB/s) | 20260508 07:50:05.014 | INFO | Downloaded 38GiB to GPU in 2.69364s (14.1073GiB/s) | 20260508 07:50:10.149 | INFO | Analyzing chunk frames = [6443520,6444032) | 20260508 07:50:10.149 | INFO | MemInfo: { | 20260508 07:50:10.149 | INFO | "gpu" : " 17 GB", | 20260508 07:50:10.149 | INFO | "huge" : "", | 20260508 07:50:10.149 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:10.149 | INFO | "malloc" : " 14 GB", | 20260508 07:50:10.149 | INFO | "pinned" : " 29 GB" | 20260508 07:50:10.149 | INFO | } | 20260508 07:50:10.149 | INFO | Perf report for the last 5.13559 seconds: | 20260508 07:50:10.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:10.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.149 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 07:50:10.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.149 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:50:10.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:10.150 | INFO | Uploaded 36GiB to GPU in 2.53086s (14.2244GiB/s) | 20260508 07:50:10.150 | INFO | Downloaded 34GiB to GPU in 2.45419s (13.8539GiB/s) | 20260508 07:50:15.286 | INFO | Analyzing chunk frames = [6444032,6444544) | 20260508 07:50:15.286 | INFO | MemInfo: { | 20260508 07:50:15.286 | INFO | "gpu" : " 17 GB", | 20260508 07:50:15.286 | INFO | "huge" : "", | 20260508 07:50:15.286 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:15.286 | INFO | "malloc" : " 14 GB", | 20260508 07:50:15.286 | INFO | "pinned" : " 29 GB" | 20260508 07:50:15.287 | INFO | } | 20260508 07:50:15.287 | INFO | Perf report for the last 5.13725 seconds: | 20260508 07:50:15.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:15.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 07:50:15.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:50:15.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:15.287 | INFO | Uploaded 36GiB to GPU in 2.56024s (14.0612GiB/s) | 20260508 07:50:15.287 | INFO | Downloaded 34GiB to GPU in 2.44496s (13.9061GiB/s) | 20260508 07:50:20.422 | INFO | Analyzing chunk frames = [6444544,6445056) | 20260508 07:50:20.422 | INFO | MemInfo: { | 20260508 07:50:20.422 | INFO | "gpu" : " 17 GB", | 20260508 07:50:20.422 | INFO | "huge" : "", | 20260508 07:50:20.422 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:20.422 | INFO | "malloc" : " 14 GB", | 20260508 07:50:20.422 | INFO | "pinned" : " 29 GB" | 20260508 07:50:20.422 | INFO | } | 20260508 07:50:20.422 | INFO | Perf report for the last 5.1357 seconds: | 20260508 07:50:20.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:20.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 07:50:20.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:50:20.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:20.422 | INFO | Uploaded 36GiB to GPU in 2.5442s (14.1498GiB/s) | 20260508 07:50:20.422 | INFO | Downloaded 34GiB to GPU in 2.41072s (14.1037GiB/s) | 20260508 07:50:25.565 | INFO | Analyzing chunk frames = [6445056,6445568) | 20260508 07:50:25.565 | INFO | MemInfo: { | 20260508 07:50:25.565 | INFO | "gpu" : " 17 GB", | 20260508 07:50:25.565 | INFO | "huge" : "", | 20260508 07:50:25.565 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:25.565 | INFO | "malloc" : " 14 GB", | 20260508 07:50:25.565 | INFO | "pinned" : " 29 GB" | 20260508 07:50:25.565 | INFO | } | 20260508 07:50:25.566 | INFO | Perf report for the last 5.14322 seconds: | 20260508 07:50:25.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:25.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:50:25.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:50:25.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:25.566 | INFO | Uploaded 36GiB to GPU in 2.28419s (15.7605GiB/s) | 20260508 07:50:25.566 | INFO | Downloaded 34GiB to GPU in 2.4934s (13.636GiB/s) | 20260508 07:50:30.713 | INFO | Analyzing chunk frames = [6445568,6446080) | 20260508 07:50:30.713 | INFO | MemInfo: { | 20260508 07:50:30.713 | INFO | "gpu" : " 17 GB", | 20260508 07:50:30.713 | INFO | "huge" : "", | 20260508 07:50:30.713 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:30.713 | INFO | "malloc" : " 14 GB", | 20260508 07:50:30.713 | INFO | "pinned" : " 29 GB" | 20260508 07:50:30.713 | INFO | } | 20260508 07:50:30.714 | INFO | Perf report for the last 5.14796 seconds: | 20260508 07:50:30.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:30.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 07:50:30.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:50:30.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:30.714 | INFO | Uploaded 36GiB to GPU in 2.44597s (14.7181GiB/s) | 20260508 07:50:30.714 | INFO | Downloaded 34GiB to GPU in 2.45442s (13.8526GiB/s) | 20260508 07:50:35.850 | INFO | Analyzing chunk frames = [6446080,6446592) | 20260508 07:50:35.850 | INFO | MemInfo: { | 20260508 07:50:35.850 | INFO | "gpu" : " 17 GB", | 20260508 07:50:35.850 | INFO | "huge" : "", | 20260508 07:50:35.850 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:35.850 | INFO | "malloc" : " 14 GB", | 20260508 07:50:35.851 | INFO | "pinned" : " 29 GB" | 20260508 07:50:35.851 | INFO | } | 20260508 07:50:35.851 | INFO | Perf report for the last 5.13726 seconds: | 20260508 07:50:35.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:35.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 07:50:35.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 07:50:35.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:35.851 | INFO | Uploaded 36GiB to GPU in 2.3973s (15.0169GiB/s) | 20260508 07:50:35.851 | INFO | Downloaded 34GiB to GPU in 2.46067s (13.8174GiB/s) | 20260508 07:50:40.982 | INFO | Analyzing chunk frames = [6446592,6447104) | 20260508 07:50:40.982 | INFO | MemInfo: { | 20260508 07:50:40.982 | INFO | "gpu" : " 17 GB", | 20260508 07:50:40.982 | INFO | "huge" : "", | 20260508 07:50:40.982 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:40.982 | INFO | "malloc" : " 14 GB", | 20260508 07:50:40.982 | INFO | "pinned" : " 29 GB" | 20260508 07:50:40.982 | INFO | } | 20260508 07:50:40.982 | INFO | Perf report for the last 5.13164 seconds: | 20260508 07:50:40.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:40.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 07:50:40.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 07:50:40.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:40.983 | INFO | Uploaded 36GiB to GPU in 2.42244s (14.861GiB/s) | 20260508 07:50:40.983 | INFO | Downloaded 34GiB to GPU in 2.46168s (13.8117GiB/s) | 20260508 07:50:40.983 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:50:40.983 | INFO | Total mem usage: 16.7 GiB | 20260508 07:50:40.983 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:50:40.983 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 07:50:40.983 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:50:40.983 | INFO | High Water Breakdown: | 20260508 07:50:40.983 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:50:40.983 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:50:40.983 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:50:40.983 | INFO | ------------------------------------------ | 20260508 07:50:40.983 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:50:40.983 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:50:40.983 | INFO | Total mem usage: 26.5 GiB | 20260508 07:50:40.983 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:50:40.983 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 07:50:40.983 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 07:50:40.983 | INFO | High Water Breakdown: | 20260508 07:50:40.984 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:50:40.984 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:50:40.984 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:50:40.984 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:50:40.984 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:50:40.984 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:50:40.984 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:50:40.984 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:50:40.984 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:50:40.984 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:50:40.984 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:50:40.984 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:50:40.984 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:50:40.984 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:50:40.984 | INFO | ------------------------------------------ | 20260508 07:50:40.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:50:40.984 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:50:40.984 | INFO | Total mem usage: 27.8 GiB | 20260508 07:50:40.984 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:50:40.984 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:50:40.984 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:50:40.984 | INFO | High Water Breakdown: | 20260508 07:50:40.984 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:50:40.984 | INFO | ------------------------------------------ | 20260508 07:50:40.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:50:40.984 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:50:40.984 | INFO | Total mem usage: 15.7 GiB | 20260508 07:50:40.984 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:50:40.984 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:50:40.984 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:50:40.984 | INFO | High Water Breakdown: | 20260508 07:50:40.984 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:50:40.984 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:50:40.984 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:50:40.984 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:50:40.984 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:50:40.984 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:50:40.984 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:50:40.984 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:50:40.984 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:50:40.984 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:50:40.984 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:50:40.984 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:50:40.984 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:50:40.984 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:50:40.984 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:50:40.984 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:50:40.984 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:50:40.984 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:50:40.984 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:50:40.984 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:50:40.984 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:50:40.984 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:50:40.984 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:50:40.984 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:50:40.984 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:50:40.984 | INFO | ------------------------------------------ | 20260508 07:50:40.984 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:50:40.985 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:50:40.985 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:50:40.985 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:50:40.985 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.39e+01 / 6.28e-01 | 20260508 07:50:40.985 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.05e-01, 1.13e-02 / 4.56e-03 | 20260508 07:50:40.985 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.38e+00, 5.84e-01 / 2.42e-01 | 20260508 07:50:40.985 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.42e+00, 2.09e-01 / 9.98e-02 | 20260508 07:50:40.985 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.51e+01, 3.78e+00 / 1.10e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.78e+00 / 1.12e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.36e+00 / 2.51e+00, 4.05e-01 / 1.74e-01 | 20260508 07:50:40.985 | INFO | PerfReport: 3.11%, Metrics -- 1.02e+00 / 1.20e+00, 2.56e-01 / 1.31e-01 | 20260508 07:50:40.985 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.48e-01, 5.01e-02 / 2.81e-02 | 20260508 07:50:40.985 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:50:40.985 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:50:40.985 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:50:40.985 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 24.47%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 24.44%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 30.86%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 2.30%, LockUpload -- 2.43e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 2.53%, LockDownload -- 2.67e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 15.39%, LockCompute -- 1.63e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 07:50:40.985 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260508 07:50:41.097 | INFO | Starting a round of Baz IO | 20260508 07:50:44.490 | INFO | Populating RTMetrics report | 20260508 07:50:44.490 | INFO | RTMetrics report populated | 20260508 07:50:44.490 | INFO | Opening tmp RTMetrics json file | 20260508 07:50:44.503 | INFO | Serializing RTMetrics to json | 20260508 07:50:44.504 | INFO | Serialization done, swapping tmp file now... | 20260508 07:50:44.505 | INFO | RTMetrics done | 20260508 07:50:44.567 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.47 seconds | 20260508 07:50:44.567 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.00% / 59.50% | 20260508 07:50:44.567 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.321416% / 0% / 4.37101e-05% | 20260508 07:50:44.567 | INFO | Baz IO overall: | 20260508 07:50:44.567 | INFO | BytesWritten : 2838443217005 | 20260508 07:50:44.567 | INFO | NumEvents : 1367142285383 | 20260508 07:50:46.121 | INFO | Analyzing chunk frames = [6447104,6447616) | 20260508 07:50:46.121 | INFO | MemInfo: { | 20260508 07:50:46.121 | INFO | "gpu" : " 17 GB", | 20260508 07:50:46.121 | INFO | "huge" : "", | 20260508 07:50:46.121 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:46.121 | INFO | "malloc" : " 14 GB", | 20260508 07:50:46.121 | INFO | "pinned" : " 29 GB" | 20260508 07:50:46.121 | INFO | } | 20260508 07:50:46.121 | INFO | Perf report for the last 5.13885 seconds: | 20260508 07:50:46.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:46.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:46.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:46.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:46.121 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 07:50:46.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:46.121 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260508 07:50:46.122 | INFO | PRE_HQ : 66.5%, 1.8, 62.3ms, 0 | 20260508 07:50:46.122 | INFO | BAZWRITER : 66.9%, 2.8, 96.3ms, 0 | 20260508 07:50:46.122 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:50:46.122 | INFO | Uploaded 36GiB to GPU in 2.55894s (14.0683GiB/s) | 20260508 07:50:46.122 | INFO | Downloaded 38GiB to GPU in 2.71443s (13.9993GiB/s) | 20260508 07:50:51.262 | INFO | Analyzing chunk frames = [6447616,6448128) | 20260508 07:50:51.262 | INFO | MemInfo: { | 20260508 07:50:51.262 | INFO | "gpu" : " 17 GB", | 20260508 07:50:51.262 | INFO | "huge" : "", | 20260508 07:50:51.262 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:51.262 | INFO | "malloc" : " 14 GB", | 20260508 07:50:51.262 | INFO | "pinned" : " 29 GB" | 20260508 07:50:51.262 | INFO | } | 20260508 07:50:51.262 | INFO | Perf report for the last 5.14115 seconds: | 20260508 07:50:51.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:51.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 07:50:51.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 07:50:51.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:51.263 | INFO | Uploaded 36GiB to GPU in 2.45998s (14.6343GiB/s) | 20260508 07:50:51.263 | INFO | Downloaded 34GiB to GPU in 2.44785s (13.8897GiB/s) | 20260508 07:50:56.408 | INFO | Analyzing chunk frames = [6448128,6448640) | 20260508 07:50:56.408 | INFO | MemInfo: { | 20260508 07:50:56.408 | INFO | "gpu" : " 17 GB", | 20260508 07:50:56.408 | INFO | "huge" : "", | 20260508 07:50:56.408 | INFO | "huge_shared" : " 29 GB", | 20260508 07:50:56.408 | INFO | "malloc" : " 14 GB", | 20260508 07:50:56.408 | INFO | "pinned" : " 29 GB" | 20260508 07:50:56.408 | INFO | } | 20260508 07:50:56.408 | INFO | Perf report for the last 5.14557 seconds: | 20260508 07:50:56.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:50:56.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 07:50:56.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:50:56.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:50:56.408 | INFO | Uploaded 36GiB to GPU in 2.52093s (14.2804GiB/s) | 20260508 07:50:56.408 | INFO | Downloaded 34GiB to GPU in 2.45032s (13.8758GiB/s) | 20260508 07:51:01.544 | INFO | Analyzing chunk frames = [6448640,6449152) | 20260508 07:51:01.544 | INFO | MemInfo: { | 20260508 07:51:01.544 | INFO | "gpu" : " 17 GB", | 20260508 07:51:01.544 | INFO | "huge" : "", | 20260508 07:51:01.544 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:01.544 | INFO | "malloc" : " 14 GB", | 20260508 07:51:01.544 | INFO | "pinned" : " 29 GB" | 20260508 07:51:01.544 | INFO | } | 20260508 07:51:01.545 | INFO | Perf report for the last 5.13642 seconds: | 20260508 07:51:01.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:01.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | ANALYSIS : 66.5%, 3.0, 102.9ms, 0 | 20260508 07:51:01.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:51:01.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:01.545 | INFO | Uploaded 36GiB to GPU in 2.48118s (14.5092GiB/s) | 20260508 07:51:01.545 | INFO | Downloaded 34GiB to GPU in 2.46263s (13.8064GiB/s) | 20260508 07:51:06.682 | INFO | Analyzing chunk frames = [6449152,6449664) | 20260508 07:51:06.682 | INFO | MemInfo: { | 20260508 07:51:06.682 | INFO | "gpu" : " 17 GB", | 20260508 07:51:06.682 | INFO | "huge" : "", | 20260508 07:51:06.682 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:06.682 | INFO | "malloc" : " 14 GB", | 20260508 07:51:06.682 | INFO | "pinned" : " 29 GB" | 20260508 07:51:06.682 | INFO | } | 20260508 07:51:06.683 | INFO | Perf report for the last 5.13814 seconds: | 20260508 07:51:06.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:06.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 07:51:06.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:51:06.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:06.683 | INFO | Uploaded 36GiB to GPU in 2.40343s (14.9786GiB/s) | 20260508 07:51:06.683 | INFO | Downloaded 34GiB to GPU in 2.53915s (13.3903GiB/s) | 20260508 07:51:11.826 | INFO | Analyzing chunk frames = [6449664,6450176) | 20260508 07:51:11.826 | INFO | MemInfo: { | 20260508 07:51:11.826 | INFO | "gpu" : " 17 GB", | 20260508 07:51:11.826 | INFO | "huge" : "", | 20260508 07:51:11.826 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:11.826 | INFO | "malloc" : " 14 GB", | 20260508 07:51:11.826 | INFO | "pinned" : " 29 GB" | 20260508 07:51:11.826 | INFO | } | 20260508 07:51:11.826 | INFO | Perf report for the last 5.14335 seconds: | 20260508 07:51:11.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:11.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:51:11.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:51:11.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:11.826 | INFO | Uploaded 36GiB to GPU in 2.37627s (15.1498GiB/s) | 20260508 07:51:11.826 | INFO | Downloaded 34GiB to GPU in 2.43349s (13.9717GiB/s) | 20260508 07:51:16.962 | INFO | Analyzing chunk frames = [6450176,6450688) | 20260508 07:51:16.962 | INFO | MemInfo: { | 20260508 07:51:16.962 | INFO | "gpu" : " 17 GB", | 20260508 07:51:16.962 | INFO | "huge" : "", | 20260508 07:51:16.962 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:16.962 | INFO | "malloc" : " 14 GB", | 20260508 07:51:16.962 | INFO | "pinned" : " 29 GB" | 20260508 07:51:16.962 | INFO | } | 20260508 07:51:16.962 | INFO | Perf report for the last 5.1363 seconds: | 20260508 07:51:16.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:16.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.962 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 07:51:16.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.962 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:51:16.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:16.963 | INFO | Uploaded 36GiB to GPU in 2.40069s (14.9957GiB/s) | 20260508 07:51:16.963 | INFO | Downloaded 34GiB to GPU in 2.44463s (13.9081GiB/s) | 20260508 07:51:22.099 | INFO | Analyzing chunk frames = [6450688,6451200) | 20260508 07:51:22.099 | INFO | MemInfo: { | 20260508 07:51:22.099 | INFO | "gpu" : " 17 GB", | 20260508 07:51:22.099 | INFO | "huge" : "", | 20260508 07:51:22.099 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:22.099 | INFO | "malloc" : " 14 GB", | 20260508 07:51:22.099 | INFO | "pinned" : " 29 GB" | 20260508 07:51:22.099 | INFO | } | 20260508 07:51:22.099 | INFO | Perf report for the last 5.13652 seconds: | 20260508 07:51:22.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:22.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 07:51:22.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:51:22.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:22.099 | INFO | Uploaded 36GiB to GPU in 2.43487s (14.7852GiB/s) | 20260508 07:51:22.099 | INFO | Downloaded 34GiB to GPU in 2.45141s (13.8696GiB/s) | 20260508 07:51:22.196 | INFO | Starting a round of Baz IO | 20260508 07:51:25.615 | INFO | Populating RTMetrics report | 20260508 07:51:25.615 | INFO | RTMetrics report populated | 20260508 07:51:25.615 | INFO | Opening tmp RTMetrics json file | 20260508 07:51:25.624 | INFO | Serializing RTMetrics to json | 20260508 07:51:25.626 | INFO | Serialization done, swapping tmp file now... | 20260508 07:51:25.626 | INFO | RTMetrics done | 20260508 07:51:25.684 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.488 seconds | 20260508 07:51:25.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.00% / 59.50% | 20260508 07:51:25.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.321158% / 3.97364e-06% / 5.16574e-05% | 20260508 07:51:25.685 | INFO | Baz IO overall: | 20260508 07:51:25.685 | INFO | BytesWritten : 2840069917693 | 20260508 07:51:25.685 | INFO | NumEvents : 1367913496863 | 20260508 07:51:27.240 | INFO | Analyzing chunk frames = [6451200,6451712) | 20260508 07:51:27.240 | INFO | MemInfo: { | 20260508 07:51:27.240 | INFO | "gpu" : " 17 GB", | 20260508 07:51:27.240 | INFO | "huge" : "", | 20260508 07:51:27.240 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:27.240 | INFO | "malloc" : " 14 GB", | 20260508 07:51:27.240 | INFO | "pinned" : " 29 GB" | 20260508 07:51:27.240 | INFO | } | 20260508 07:51:27.240 | INFO | Perf report for the last 5.14135 seconds: | 20260508 07:51:27.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:27.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:27.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:27.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:27.240 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 07:51:27.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:27.240 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:51:27.240 | INFO | PRE_HQ : 65.2%, 1.6, 54.6ms, 0 | 20260508 07:51:27.240 | INFO | BAZWRITER : 67.2%, 2.6, 89.8ms, 0 | 20260508 07:51:27.240 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:51:27.241 | INFO | Uploaded 36GiB to GPU in 2.5833s (13.9357GiB/s) | 20260508 07:51:27.241 | INFO | Downloaded 38GiB to GPU in 2.71364s (14.0033GiB/s) | 20260508 07:51:32.381 | INFO | Analyzing chunk frames = [6451712,6452224) | 20260508 07:51:32.381 | INFO | MemInfo: { | 20260508 07:51:32.381 | INFO | "gpu" : " 17 GB", | 20260508 07:51:32.381 | INFO | "huge" : "", | 20260508 07:51:32.381 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:32.381 | INFO | "malloc" : " 14 GB", | 20260508 07:51:32.381 | INFO | "pinned" : " 29 GB" | 20260508 07:51:32.381 | INFO | } | 20260508 07:51:32.381 | INFO | Perf report for the last 5.14052 seconds: | 20260508 07:51:32.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:32.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:51:32.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:51:32.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:32.381 | INFO | Uploaded 36GiB to GPU in 2.50716s (14.3588GiB/s) | 20260508 07:51:32.381 | INFO | Downloaded 34GiB to GPU in 2.45304s (13.8604GiB/s) | 20260508 07:51:37.516 | INFO | Analyzing chunk frames = [6452224,6452736) | 20260508 07:51:37.516 | INFO | MemInfo: { | 20260508 07:51:37.516 | INFO | "gpu" : " 17 GB", | 20260508 07:51:37.516 | INFO | "huge" : "", | 20260508 07:51:37.516 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:37.516 | INFO | "malloc" : " 14 GB", | 20260508 07:51:37.516 | INFO | "pinned" : " 29 GB" | 20260508 07:51:37.516 | INFO | } | 20260508 07:51:37.516 | INFO | Perf report for the last 5.13505 seconds: | 20260508 07:51:37.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:37.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 07:51:37.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:51:37.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:37.516 | INFO | Uploaded 36GiB to GPU in 2.46514s (14.6037GiB/s) | 20260508 07:51:37.516 | INFO | Downloaded 34GiB to GPU in 2.44499s (13.906GiB/s) | 20260508 07:51:42.666 | INFO | Analyzing chunk frames = [6452736,6453248) | 20260508 07:51:42.666 | INFO | MemInfo: { | 20260508 07:51:42.666 | INFO | "gpu" : " 17 GB", | 20260508 07:51:42.666 | INFO | "huge" : "", | 20260508 07:51:42.666 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:42.666 | INFO | "malloc" : " 14 GB", | 20260508 07:51:42.666 | INFO | "pinned" : " 29 GB" | 20260508 07:51:42.666 | INFO | } | 20260508 07:51:42.666 | INFO | Perf report for the last 5.1506 seconds: | 20260508 07:51:42.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:42.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.666 | INFO | ANALYSIS : 66.5%, 2.9, 103.1ms, 0 | 20260508 07:51:42.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.667 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:51:42.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:42.667 | INFO | Uploaded 36GiB to GPU in 2.53975s (14.1746GiB/s) | 20260508 07:51:42.667 | INFO | Downloaded 34GiB to GPU in 2.40524s (14.1358GiB/s) | 20260508 07:51:47.798 | INFO | Analyzing chunk frames = [6453248,6453760) | 20260508 07:51:47.799 | INFO | MemInfo: { | 20260508 07:51:47.799 | INFO | "gpu" : " 17 GB", | 20260508 07:51:47.799 | INFO | "huge" : "", | 20260508 07:51:47.799 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:47.799 | INFO | "malloc" : " 14 GB", | 20260508 07:51:47.799 | INFO | "pinned" : " 29 GB" | 20260508 07:51:47.799 | INFO | } | 20260508 07:51:47.799 | INFO | Perf report for the last 5.13216 seconds: | 20260508 07:51:47.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:47.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 07:51:47.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:51:47.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:47.799 | INFO | Uploaded 36GiB to GPU in 2.28078s (15.784GiB/s) | 20260508 07:51:47.799 | INFO | Downloaded 34GiB to GPU in 2.50889s (13.5518GiB/s) | 20260508 07:51:52.933 | INFO | Analyzing chunk frames = [6453760,6454272) | 20260508 07:51:52.933 | INFO | MemInfo: { | 20260508 07:51:52.933 | INFO | "gpu" : " 17 GB", | 20260508 07:51:52.933 | INFO | "huge" : "", | 20260508 07:51:52.933 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:52.933 | INFO | "malloc" : " 14 GB", | 20260508 07:51:52.933 | INFO | "pinned" : " 29 GB" | 20260508 07:51:52.933 | INFO | } | 20260508 07:51:52.933 | INFO | Perf report for the last 5.13479 seconds: | 20260508 07:51:52.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:52.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 07:51:52.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 07:51:52.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:52.934 | INFO | Uploaded 36GiB to GPU in 2.33474s (15.4193GiB/s) | 20260508 07:51:52.934 | INFO | Downloaded 34GiB to GPU in 2.44992s (13.878GiB/s) | 20260508 07:51:58.077 | INFO | Analyzing chunk frames = [6454272,6454784) | 20260508 07:51:58.077 | INFO | MemInfo: { | 20260508 07:51:58.077 | INFO | "gpu" : " 17 GB", | 20260508 07:51:58.077 | INFO | "huge" : "", | 20260508 07:51:58.077 | INFO | "huge_shared" : " 29 GB", | 20260508 07:51:58.077 | INFO | "malloc" : " 14 GB", | 20260508 07:51:58.077 | INFO | "pinned" : " 29 GB" | 20260508 07:51:58.077 | INFO | } | 20260508 07:51:58.077 | INFO | Perf report for the last 5.14396 seconds: | 20260508 07:51:58.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:51:58.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 07:51:58.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:51:58.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:51:58.078 | INFO | Uploaded 36GiB to GPU in 2.46219s (14.6212GiB/s) | 20260508 07:51:58.078 | INFO | Downloaded 34GiB to GPU in 2.46135s (13.8135GiB/s) | 20260508 07:52:03.218 | INFO | Analyzing chunk frames = [6454784,6455296) | 20260508 07:52:03.218 | INFO | MemInfo: { | 20260508 07:52:03.218 | INFO | "gpu" : " 17 GB", | 20260508 07:52:03.218 | INFO | "huge" : "", | 20260508 07:52:03.218 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:03.218 | INFO | "malloc" : " 14 GB", | 20260508 07:52:03.218 | INFO | "pinned" : " 29 GB" | 20260508 07:52:03.218 | INFO | } | 20260508 07:52:03.218 | INFO | Perf report for the last 5.14034 seconds: | 20260508 07:52:03.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:03.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 07:52:03.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:52:03.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:03.218 | INFO | Uploaded 36GiB to GPU in 2.411s (14.9315GiB/s) | 20260508 07:52:03.218 | INFO | Downloaded 34GiB to GPU in 2.46845s (13.7738GiB/s) | 20260508 07:52:03.218 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:52:03.218 | INFO | Total mem usage: 16.7 GiB | 20260508 07:52:03.218 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:52:03.218 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 07:52:03.218 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:52:03.218 | INFO | High Water Breakdown: | 20260508 07:52:03.218 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:52:03.218 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:52:03.218 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:52:03.218 | INFO | ------------------------------------------ | 20260508 07:52:03.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:52:03.219 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:52:03.219 | INFO | Total mem usage: 26.5 GiB | 20260508 07:52:03.219 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:52:03.219 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:52:03.219 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:52:03.219 | INFO | High Water Breakdown: | 20260508 07:52:03.219 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:52:03.219 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:52:03.219 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:52:03.219 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:52:03.219 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:52:03.219 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:52:03.219 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:52:03.219 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:52:03.219 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:52:03.219 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:52:03.219 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:52:03.219 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:52:03.219 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:52:03.219 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:52:03.219 | INFO | ------------------------------------------ | 20260508 07:52:03.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:52:03.219 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:52:03.219 | INFO | Total mem usage: 27.8 GiB | 20260508 07:52:03.219 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:52:03.219 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:52:03.219 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:52:03.219 | INFO | High Water Breakdown: | 20260508 07:52:03.219 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:52:03.219 | INFO | ------------------------------------------ | 20260508 07:52:03.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:52:03.219 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:52:03.219 | INFO | Total mem usage: 15.7 GiB | 20260508 07:52:03.219 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:52:03.219 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:52:03.219 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:52:03.219 | INFO | High Water Breakdown: | 20260508 07:52:03.219 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:52:03.219 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:52:03.219 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:52:03.219 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:52:03.219 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:52:03.219 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:52:03.219 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:52:03.219 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:52:03.219 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:52:03.219 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:52:03.219 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:52:03.219 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:52:03.219 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:52:03.219 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:52:03.219 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:52:03.219 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:52:03.219 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:52:03.219 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:52:03.219 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:52:03.219 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:52:03.219 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:52:03.219 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:52:03.219 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:52:03.219 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:52:03.219 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:52:03.219 | INFO | ------------------------------------------ | 20260508 07:52:03.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:52:03.219 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:52:03.219 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:52:03.219 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:52:03.219 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.42e+01 / 7.33e-01 | 20260508 07:52:03.219 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.12e-02 / 1.51e-02 | 20260508 07:52:03.219 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.40e+00, 5.94e-01 / 2.85e-01 | 20260508 07:52:03.219 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.43e+00, 2.09e-01 / 1.42e-01 | 20260508 07:52:03.219 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.51e+01, 3.67e+00 / 1.20e+00 | 20260508 07:52:03.219 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.87e+00 / 1.32e+00 | 20260508 07:52:03.219 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.50e+00, 4.06e-01 / 2.74e-01 | 20260508 07:52:03.219 | INFO | PerfReport: 3.09%, Metrics -- 1.01e+00 / 1.21e+00, 2.55e-01 / 1.12e-01 | 20260508 07:52:03.219 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.41e-01, 4.98e-02 / 2.18e-02 | 20260508 07:52:03.219 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:52:03.219 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:52:03.219 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:52:03.219 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:52:03.219 | INFO | PerfReport: 24.19%, Upload -- 2.55e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 07:52:03.219 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 07:52:03.220 | INFO | PerfReport: 31.04%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 07:52:03.220 | INFO | PerfReport: 2.05%, LockUpload -- 2.16e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 07:52:03.220 | INFO | PerfReport: 2.68%, LockDownload -- 2.83e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 07:52:03.220 | INFO | PerfReport: 15.45%, LockCompute -- 1.63e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 07:52:03.220 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260508 07:52:03.315 | INFO | Starting a round of Baz IO | 20260508 07:52:06.747 | INFO | Populating RTMetrics report | 20260508 07:52:06.747 | INFO | RTMetrics report populated | 20260508 07:52:06.747 | INFO | Opening tmp RTMetrics json file | 20260508 07:52:06.748 | INFO | Serializing RTMetrics to json | 20260508 07:52:06.749 | INFO | Serialization done, swapping tmp file now... | 20260508 07:52:06.750 | INFO | RTMetrics done | 20260508 07:52:06.818 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.502 seconds | 20260508 07:52:06.818 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.50% / 0.00% / 59.50% | 20260508 07:52:06.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.320709% / 0% / 5.5631e-05% | 20260508 07:52:06.818 | INFO | Baz IO overall: | 20260508 07:52:06.818 | INFO | BytesWritten : 2841696854961 | 20260508 07:52:06.818 | INFO | NumEvents : 1368684589196 | 20260508 07:52:08.361 | INFO | Analyzing chunk frames = [6455296,6455808) | 20260508 07:52:08.361 | INFO | MemInfo: { | 20260508 07:52:08.361 | INFO | "gpu" : " 17 GB", | 20260508 07:52:08.361 | INFO | "huge" : "", | 20260508 07:52:08.361 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:08.361 | INFO | "malloc" : " 14 GB", | 20260508 07:52:08.361 | INFO | "pinned" : " 29 GB" | 20260508 07:52:08.361 | INFO | } | 20260508 07:52:08.361 | INFO | Perf report for the last 5.14335 seconds: | 20260508 07:52:08.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:08.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:08.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:08.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:08.361 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 07:52:08.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:08.361 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 07:52:08.361 | INFO | PRE_HQ : 65.8%, 1.6, 54.7ms, 0 | 20260508 07:52:08.361 | INFO | BAZWRITER : 67.5%, 2.6, 90.0ms, 0 | 20260508 07:52:08.361 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:52:08.361 | INFO | Uploaded 36GiB to GPU in 2.545s (14.1454GiB/s) | 20260508 07:52:08.362 | INFO | Downloaded 38GiB to GPU in 2.71958s (13.9728GiB/s) | 20260508 07:52:13.496 | INFO | Analyzing chunk frames = [6455808,6456320) | 20260508 07:52:13.496 | INFO | MemInfo: { | 20260508 07:52:13.496 | INFO | "gpu" : " 17 GB", | 20260508 07:52:13.496 | INFO | "huge" : "", | 20260508 07:52:13.496 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:13.496 | INFO | "malloc" : " 14 GB", | 20260508 07:52:13.496 | INFO | "pinned" : " 29 GB" | 20260508 07:52:13.496 | INFO | } | 20260508 07:52:13.496 | INFO | Perf report for the last 5.13496 seconds: | 20260508 07:52:13.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:13.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 07:52:13.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:52:13.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:13.496 | INFO | Uploaded 36GiB to GPU in 2.5114s (14.3346GiB/s) | 20260508 07:52:13.497 | INFO | Downloaded 34GiB to GPU in 2.45004s (13.8773GiB/s) | 20260508 07:52:18.632 | INFO | Analyzing chunk frames = [6456320,6456832) | 20260508 07:52:18.632 | INFO | MemInfo: { | 20260508 07:52:18.632 | INFO | "gpu" : " 17 GB", | 20260508 07:52:18.632 | INFO | "huge" : "", | 20260508 07:52:18.632 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:18.632 | INFO | "malloc" : " 14 GB", | 20260508 07:52:18.632 | INFO | "pinned" : " 29 GB" | 20260508 07:52:18.632 | INFO | } | 20260508 07:52:18.632 | INFO | Perf report for the last 5.13571 seconds: | 20260508 07:52:18.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:18.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 07:52:18.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:52:18.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:18.632 | INFO | Uploaded 36GiB to GPU in 2.43945s (14.7574GiB/s) | 20260508 07:52:18.632 | INFO | Downloaded 34GiB to GPU in 2.44669s (13.8963GiB/s) | 20260508 07:52:23.779 | INFO | Analyzing chunk frames = [6456832,6457344) | 20260508 07:52:23.779 | INFO | MemInfo: { | 20260508 07:52:23.779 | INFO | "gpu" : " 17 GB", | 20260508 07:52:23.779 | INFO | "huge" : "", | 20260508 07:52:23.779 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:23.779 | INFO | "malloc" : " 14 GB", | 20260508 07:52:23.779 | INFO | "pinned" : " 29 GB" | 20260508 07:52:23.779 | INFO | } | 20260508 07:52:23.780 | INFO | Perf report for the last 5.14791 seconds: | 20260508 07:52:23.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:23.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | ANALYSIS : 66.5%, 2.9, 103.0ms, 0 | 20260508 07:52:23.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 07:52:23.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:23.780 | INFO | Uploaded 36GiB to GPU in 2.51902s (14.2913GiB/s) | 20260508 07:52:23.780 | INFO | Downloaded 34GiB to GPU in 2.42119s (14.0427GiB/s) | 20260508 07:52:28.911 | INFO | Analyzing chunk frames = [6457344,6457856) | 20260508 07:52:28.911 | INFO | MemInfo: { | 20260508 07:52:28.911 | INFO | "gpu" : " 17 GB", | 20260508 07:52:28.911 | INFO | "huge" : "", | 20260508 07:52:28.911 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:28.911 | INFO | "malloc" : " 14 GB", | 20260508 07:52:28.911 | INFO | "pinned" : " 29 GB" | 20260508 07:52:28.911 | INFO | } | 20260508 07:52:28.911 | INFO | Perf report for the last 5.13127 seconds: | 20260508 07:52:28.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:28.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 07:52:28.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:52:28.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:28.911 | INFO | Uploaded 36GiB to GPU in 2.39334s (15.0418GiB/s) | 20260508 07:52:28.911 | INFO | Downloaded 34GiB to GPU in 2.54126s (13.3792GiB/s) | 20260508 07:52:34.056 | INFO | Analyzing chunk frames = [6457856,6458368) | 20260508 07:52:34.056 | INFO | MemInfo: { | 20260508 07:52:34.056 | INFO | "gpu" : " 17 GB", | 20260508 07:52:34.056 | INFO | "huge" : "", | 20260508 07:52:34.056 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:34.056 | INFO | "malloc" : " 14 GB", | 20260508 07:52:34.056 | INFO | "pinned" : " 29 GB" | 20260508 07:52:34.056 | INFO | } | 20260508 07:52:34.056 | INFO | Perf report for the last 5.14492 seconds: | 20260508 07:52:34.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:34.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.056 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 07:52:34.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.056 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:52:34.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:34.057 | INFO | Uploaded 36GiB to GPU in 2.34994s (15.3195GiB/s) | 20260508 07:52:34.057 | INFO | Downloaded 34GiB to GPU in 2.42718s (14.0081GiB/s) | 20260508 07:52:39.188 | INFO | Analyzing chunk frames = [6458368,6458880) | 20260508 07:52:39.188 | INFO | MemInfo: { | 20260508 07:52:39.188 | INFO | "gpu" : " 17 GB", | 20260508 07:52:39.189 | INFO | "huge" : "", | 20260508 07:52:39.189 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:39.189 | INFO | "malloc" : " 14 GB", | 20260508 07:52:39.189 | INFO | "pinned" : " 29 GB" | 20260508 07:52:39.189 | INFO | } | 20260508 07:52:39.189 | INFO | Perf report for the last 5.13283 seconds: | 20260508 07:52:39.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:39.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:52:39.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:52:39.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:39.189 | INFO | Uploaded 36GiB to GPU in 2.41118s (14.9305GiB/s) | 20260508 07:52:39.189 | INFO | Downloaded 34GiB to GPU in 2.4428s (13.9185GiB/s) | 20260508 07:52:44.331 | INFO | Analyzing chunk frames = [6458880,6459392) | 20260508 07:52:44.331 | INFO | MemInfo: { | 20260508 07:52:44.331 | INFO | "gpu" : " 17 GB", | 20260508 07:52:44.331 | INFO | "huge" : "", | 20260508 07:52:44.331 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:44.331 | INFO | "malloc" : " 14 GB", | 20260508 07:52:44.331 | INFO | "pinned" : " 29 GB" | 20260508 07:52:44.331 | INFO | } | 20260508 07:52:44.331 | INFO | Perf report for the last 5.1422 seconds: | 20260508 07:52:44.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:44.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 07:52:44.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 07:52:44.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:44.331 | INFO | Uploaded 36GiB to GPU in 2.48045s (14.5135GiB/s) | 20260508 07:52:44.331 | INFO | Downloaded 34GiB to GPU in 2.45923s (13.8255GiB/s) | 20260508 07:52:44.428 | INFO | Starting a round of Baz IO | 20260508 07:52:47.854 | INFO | Populating RTMetrics report | 20260508 07:52:47.854 | INFO | RTMetrics report populated | 20260508 07:52:47.854 | INFO | Opening tmp RTMetrics json file | 20260508 07:52:47.864 | INFO | Serializing RTMetrics to json | 20260508 07:52:47.865 | INFO | Serialization done, swapping tmp file now... | 20260508 07:52:47.866 | INFO | RTMetrics done | 20260508 07:52:47.923 | INFO | Finished a round of Baz IO: Wrote 1.516 GiB in 3.494 seconds | 20260508 07:52:47.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.50% | 20260508 07:52:47.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.321174% / 7.94729e-06% / 3.57628e-05% | 20260508 07:52:47.923 | INFO | Baz IO overall: | 20260508 07:52:47.923 | INFO | BytesWritten : 2843324257339 | 20260508 07:52:47.923 | INFO | NumEvents : 1369456080383 | 20260508 07:52:49.470 | INFO | Analyzing chunk frames = [6459392,6459904) | 20260508 07:52:49.470 | INFO | MemInfo: { | 20260508 07:52:49.470 | INFO | "gpu" : " 17 GB", | 20260508 07:52:49.470 | INFO | "huge" : "", | 20260508 07:52:49.470 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:49.470 | INFO | "malloc" : " 14 GB", | 20260508 07:52:49.470 | INFO | "pinned" : " 29 GB" | 20260508 07:52:49.470 | INFO | } | 20260508 07:52:49.470 | INFO | Perf report for the last 5.13949 seconds: | 20260508 07:52:49.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:49.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:49.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:49.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:49.471 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 07:52:49.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:49.471 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:52:49.471 | INFO | PRE_HQ : 65.3%, 1.6, 54.7ms, 0 | 20260508 07:52:49.471 | INFO | BAZWRITER : 67.4%, 2.6, 90.2ms, 0 | 20260508 07:52:49.471 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:52:49.471 | INFO | Uploaded 36GiB to GPU in 2.61488s (13.7673GiB/s) | 20260508 07:52:49.471 | INFO | Downloaded 38GiB to GPU in 2.67907s (14.184GiB/s) | 20260508 07:52:54.612 | INFO | Analyzing chunk frames = [6459904,6460416) | 20260508 07:52:54.612 | INFO | MemInfo: { | 20260508 07:52:54.612 | INFO | "gpu" : " 17 GB", | 20260508 07:52:54.612 | INFO | "huge" : "", | 20260508 07:52:54.612 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:54.612 | INFO | "malloc" : " 14 GB", | 20260508 07:52:54.612 | INFO | "pinned" : " 29 GB" | 20260508 07:52:54.612 | INFO | } | 20260508 07:52:54.612 | INFO | Perf report for the last 5.14158 seconds: | 20260508 07:52:54.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:54.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:52:54.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:52:54.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:54.612 | INFO | Uploaded 36GiB to GPU in 2.52011s (14.2851GiB/s) | 20260508 07:52:54.612 | INFO | Downloaded 34GiB to GPU in 2.43511s (13.9624GiB/s) | 20260508 07:52:59.743 | INFO | Analyzing chunk frames = [6460416,6460928) | 20260508 07:52:59.743 | INFO | MemInfo: { | 20260508 07:52:59.743 | INFO | "gpu" : " 17 GB", | 20260508 07:52:59.743 | INFO | "huge" : "", | 20260508 07:52:59.743 | INFO | "huge_shared" : " 29 GB", | 20260508 07:52:59.743 | INFO | "malloc" : " 14 GB", | 20260508 07:52:59.743 | INFO | "pinned" : " 29 GB" | 20260508 07:52:59.743 | INFO | } | 20260508 07:52:59.743 | INFO | Perf report for the last 5.13116 seconds: | 20260508 07:52:59.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:52:59.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.743 | INFO | ANALYSIS : 66.9%, 3.0, 103.8ms, 0 | 20260508 07:52:59.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.743 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 07:52:59.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:52:59.744 | INFO | Uploaded 36GiB to GPU in 2.51344s (14.323GiB/s) | 20260508 07:52:59.744 | INFO | Downloaded 34GiB to GPU in 2.43234s (13.9783GiB/s) | 20260508 07:53:04.884 | INFO | Analyzing chunk frames = [6460928,6461440) | 20260508 07:53:04.884 | INFO | MemInfo: { | 20260508 07:53:04.884 | INFO | "gpu" : " 17 GB", | 20260508 07:53:04.884 | INFO | "huge" : "", | 20260508 07:53:04.884 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:04.884 | INFO | "malloc" : " 14 GB", | 20260508 07:53:04.884 | INFO | "pinned" : " 29 GB" | 20260508 07:53:04.884 | INFO | } | 20260508 07:53:04.884 | INFO | Perf report for the last 5.14053 seconds: | 20260508 07:53:04.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:04.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 07:53:04.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:53:04.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:04.884 | INFO | Uploaded 36GiB to GPU in 2.57947s (13.9563GiB/s) | 20260508 07:53:04.884 | INFO | Downloaded 34GiB to GPU in 2.39546s (14.1935GiB/s) | 20260508 07:53:10.027 | INFO | Analyzing chunk frames = [6461440,6461952) | 20260508 07:53:10.027 | INFO | MemInfo: { | 20260508 07:53:10.027 | INFO | "gpu" : " 17 GB", | 20260508 07:53:10.027 | INFO | "huge" : "", | 20260508 07:53:10.027 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:10.027 | INFO | "malloc" : " 14 GB", | 20260508 07:53:10.027 | INFO | "pinned" : " 29 GB" | 20260508 07:53:10.027 | INFO | } | 20260508 07:53:10.027 | INFO | Perf report for the last 5.14327 seconds: | 20260508 07:53:10.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:10.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 07:53:10.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 07:53:10.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:10.027 | INFO | Uploaded 36GiB to GPU in 2.29064s (15.7161GiB/s) | 20260508 07:53:10.027 | INFO | Downloaded 34GiB to GPU in 2.46194s (13.8102GiB/s) | 20260508 07:53:15.169 | INFO | Analyzing chunk frames = [6461952,6462464) | 20260508 07:53:15.169 | INFO | MemInfo: { | 20260508 07:53:15.169 | INFO | "gpu" : " 17 GB", | 20260508 07:53:15.169 | INFO | "huge" : "", | 20260508 07:53:15.169 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:15.169 | INFO | "malloc" : " 14 GB", | 20260508 07:53:15.169 | INFO | "pinned" : " 29 GB" | 20260508 07:53:15.169 | INFO | } | 20260508 07:53:15.169 | INFO | Perf report for the last 5.14168 seconds: | 20260508 07:53:15.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:15.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:53:15.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:53:15.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:15.169 | INFO | Uploaded 36GiB to GPU in 2.38517s (15.0933GiB/s) | 20260508 07:53:15.169 | INFO | Downloaded 34GiB to GPU in 2.42144s (14.0412GiB/s) | 20260508 07:53:20.311 | INFO | Analyzing chunk frames = [6462464,6462976) | 20260508 07:53:20.311 | INFO | MemInfo: { | 20260508 07:53:20.311 | INFO | "gpu" : " 17 GB", | 20260508 07:53:20.311 | INFO | "huge" : "", | 20260508 07:53:20.311 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:20.311 | INFO | "malloc" : " 14 GB", | 20260508 07:53:20.311 | INFO | "pinned" : " 29 GB" | 20260508 07:53:20.311 | INFO | } | 20260508 07:53:20.312 | INFO | Perf report for the last 5.14271 seconds: | 20260508 07:53:20.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:20.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 07:53:20.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:53:20.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:20.312 | INFO | Uploaded 36GiB to GPU in 2.42723s (14.8317GiB/s) | 20260508 07:53:20.312 | INFO | Downloaded 34GiB to GPU in 2.42731s (14.0073GiB/s) | 20260508 07:53:25.446 | INFO | Analyzing chunk frames = [6462976,6463488) | 20260508 07:53:25.446 | INFO | MemInfo: { | 20260508 07:53:25.446 | INFO | "gpu" : " 17 GB", | 20260508 07:53:25.446 | INFO | "huge" : "", | 20260508 07:53:25.446 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:25.446 | INFO | "malloc" : " 14 GB", | 20260508 07:53:25.446 | INFO | "pinned" : " 29 GB" | 20260508 07:53:25.446 | INFO | } | 20260508 07:53:25.446 | INFO | Perf report for the last 5.13424 seconds: | 20260508 07:53:25.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:25.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 07:53:25.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:53:25.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:25.446 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:53:25.446 | INFO | Total mem usage: 16.7 GiB | 20260508 07:53:25.446 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:53:25.446 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:53:25.446 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:53:25.446 | INFO | High Water Breakdown: | 20260508 07:53:25.446 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:53:25.446 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:53:25.446 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:53:25.446 | INFO | ------------------------------------------ | 20260508 07:53:25.446 | INFO | Uploaded 36GiB to GPU in 2.44796s (14.7061GiB/s) | 20260508 07:53:25.447 | INFO | Downloaded 34GiB to GPU in 2.4174s (14.0647GiB/s) | 20260508 07:53:25.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:53:25.447 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:53:25.447 | INFO | Total mem usage: 26.5 GiB | 20260508 07:53:25.447 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:53:25.447 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:53:25.447 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:53:25.447 | INFO | High Water Breakdown: | 20260508 07:53:25.447 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:53:25.447 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:53:25.447 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:53:25.447 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:53:25.447 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:53:25.447 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:53:25.447 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:53:25.447 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:53:25.447 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:53:25.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:53:25.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:53:25.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:53:25.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:53:25.447 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:53:25.447 | INFO | ------------------------------------------ | 20260508 07:53:25.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:53:25.447 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:53:25.447 | INFO | Total mem usage: 27.8 GiB | 20260508 07:53:25.447 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:53:25.447 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:53:25.447 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:53:25.447 | INFO | High Water Breakdown: | 20260508 07:53:25.447 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:53:25.447 | INFO | ------------------------------------------ | 20260508 07:53:25.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:53:25.447 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:53:25.447 | INFO | Total mem usage: 15.7 GiB | 20260508 07:53:25.447 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:53:25.447 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:53:25.447 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:53:25.447 | INFO | High Water Breakdown: | 20260508 07:53:25.447 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:53:25.447 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:53:25.447 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:53:25.447 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:53:25.447 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:53:25.447 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:53:25.447 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:53:25.447 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:53:25.447 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:53:25.447 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:53:25.447 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:53:25.447 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:53:25.447 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:53:25.447 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:53:25.447 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:53:25.447 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:53:25.447 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:53:25.447 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:53:25.447 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:53:25.447 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:53:25.447 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:53:25.447 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:53:25.447 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:53:25.447 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:53:25.447 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:53:25.447 | INFO | ------------------------------------------ | 20260508 07:53:25.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:53:25.447 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:53:25.447 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:53:25.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:53:25.447 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.51e+01, 1.36e+01 / 4.94e-01 | 20260508 07:53:25.447 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.03e-01, 1.04e-02 / 3.91e-03 | 20260508 07:53:25.447 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.51e+00, 5.93e-01 / 1.57e-01 | 20260508 07:53:25.447 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.48e+00, 2.09e-01 / 1.01e-01 | 20260508 07:53:25.447 | INFO | PerfReport: 15.91%, DME -- 4.18e+01 / 4.49e+01, 3.59e+00 / 1.01e+00 | 20260508 07:53:25.447 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.91e+00 / 1.32e+00 | 20260508 07:53:25.447 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.55e+00, 4.09e-01 / 2.55e-01 | 20260508 07:53:25.447 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.19e+00, 2.51e-01 / 1.10e-01 | 20260508 07:53:25.447 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.43e-01, 4.97e-02 / 2.61e-02 | 20260508 07:53:25.447 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:53:25.447 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:53:25.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:53:25.447 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 07:53:25.447 | INFO | PerfReport: 24.30%, Upload -- 2.57e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 07:53:25.448 | INFO | PerfReport: 24.39%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:53:25.448 | INFO | PerfReport: 31.06%, Compute -- 3.28e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260508 07:53:25.448 | INFO | PerfReport: 2.20%, LockUpload -- 2.33e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 07:53:25.448 | INFO | PerfReport: 2.50%, LockDownload -- 2.64e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 07:53:25.448 | INFO | PerfReport: 15.54%, LockCompute -- 1.64e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 07:53:25.448 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 07:53:25.544 | INFO | Starting a round of Baz IO | 20260508 07:53:28.957 | INFO | Populating RTMetrics report | 20260508 07:53:28.957 | INFO | RTMetrics report populated | 20260508 07:53:28.957 | INFO | Opening tmp RTMetrics json file | 20260508 07:53:28.973 | INFO | Serializing RTMetrics to json | 20260508 07:53:28.975 | INFO | Serialization done, swapping tmp file now... | 20260508 07:53:28.975 | INFO | RTMetrics done | 20260508 07:53:29.036 | INFO | Finished a round of Baz IO: Wrote 1.517 GiB in 3.492 seconds | 20260508 07:53:29.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.51% | 20260508 07:53:29.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.321074% / 3.97364e-06% / 5.96046e-05% | 20260508 07:53:29.036 | INFO | Baz IO overall: | 20260508 07:53:29.036 | INFO | BytesWritten : 2844953110205 | 20260508 07:53:29.036 | INFO | NumEvents : 1370228961465 | 20260508 07:53:30.584 | INFO | Analyzing chunk frames = [6463488,6464000) | 20260508 07:53:30.584 | INFO | MemInfo: { | 20260508 07:53:30.584 | INFO | "gpu" : " 17 GB", | 20260508 07:53:30.584 | INFO | "huge" : "", | 20260508 07:53:30.584 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:30.584 | INFO | "malloc" : " 14 GB", | 20260508 07:53:30.584 | INFO | "pinned" : " 29 GB" | 20260508 07:53:30.584 | INFO | } | 20260508 07:53:30.584 | INFO | Perf report for the last 5.13839 seconds: | 20260508 07:53:30.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:30.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:30.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:30.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:30.584 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 07:53:30.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:30.584 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 07:53:30.584 | INFO | PRE_HQ : 65.3%, 1.6, 54.7ms, 0 | 20260508 07:53:30.584 | INFO | BAZWRITER : 67.2%, 2.6, 90.9ms, 0 | 20260508 07:53:30.584 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:53:30.585 | INFO | Uploaded 36GiB to GPU in 2.60594s (13.8146GiB/s) | 20260508 07:53:30.585 | INFO | Downloaded 38GiB to GPU in 2.70272s (14.0599GiB/s) | 20260508 07:53:35.725 | INFO | Analyzing chunk frames = [6464000,6464512) | 20260508 07:53:35.725 | INFO | MemInfo: { | 20260508 07:53:35.725 | INFO | "gpu" : " 17 GB", | 20260508 07:53:35.725 | INFO | "huge" : "", | 20260508 07:53:35.725 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:35.725 | INFO | "malloc" : " 14 GB", | 20260508 07:53:35.725 | INFO | "pinned" : " 29 GB" | 20260508 07:53:35.725 | INFO | } | 20260508 07:53:35.725 | INFO | Perf report for the last 5.14096 seconds: | 20260508 07:53:35.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:35.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 07:53:35.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:53:35.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:35.725 | INFO | Uploaded 36GiB to GPU in 2.50117s (14.3933GiB/s) | 20260508 07:53:35.725 | INFO | Downloaded 34GiB to GPU in 2.45275s (13.862GiB/s) | 20260508 07:53:40.867 | INFO | Analyzing chunk frames = [6464512,6465024) | 20260508 07:53:40.867 | INFO | MemInfo: { | 20260508 07:53:40.867 | INFO | "gpu" : " 17 GB", | 20260508 07:53:40.867 | INFO | "huge" : "", | 20260508 07:53:40.867 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:40.867 | INFO | "malloc" : " 14 GB", | 20260508 07:53:40.867 | INFO | "pinned" : " 29 GB" | 20260508 07:53:40.867 | INFO | } | 20260508 07:53:40.867 | INFO | Perf report for the last 5.14173 seconds: | 20260508 07:53:40.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:40.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.867 | INFO | ANALYSIS : 67.0%, 3.0, 104.0ms, 0 | 20260508 07:53:40.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.867 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:53:40.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:40.868 | INFO | Uploaded 36GiB to GPU in 2.54026s (14.1718GiB/s) | 20260508 07:53:40.868 | INFO | Downloaded 34GiB to GPU in 2.43582s (13.9584GiB/s) | 20260508 07:53:46.000 | INFO | Analyzing chunk frames = [6465024,6465536) | 20260508 07:53:46.000 | INFO | MemInfo: { | 20260508 07:53:46.000 | INFO | "gpu" : " 17 GB", | 20260508 07:53:46.000 | INFO | "huge" : "", | 20260508 07:53:46.000 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:46.000 | INFO | "malloc" : " 14 GB", | 20260508 07:53:46.000 | INFO | "pinned" : " 29 GB" | 20260508 07:53:46.000 | INFO | } | 20260508 07:53:46.000 | INFO | Perf report for the last 5.13298 seconds: | 20260508 07:53:46.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:46.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 07:53:46.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:53:46.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:46.000 | INFO | Uploaded 36GiB to GPU in 2.5639s (14.0411GiB/s) | 20260508 07:53:46.000 | INFO | Downloaded 34GiB to GPU in 2.43826s (13.9443GiB/s) | 20260508 07:53:51.142 | INFO | Analyzing chunk frames = [6465536,6466048) | 20260508 07:53:51.142 | INFO | MemInfo: { | 20260508 07:53:51.142 | INFO | "gpu" : " 17 GB", | 20260508 07:53:51.142 | INFO | "huge" : "", | 20260508 07:53:51.142 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:51.142 | INFO | "malloc" : " 14 GB", | 20260508 07:53:51.142 | INFO | "pinned" : " 29 GB" | 20260508 07:53:51.142 | INFO | } | 20260508 07:53:51.142 | INFO | Perf report for the last 5.14229 seconds: | 20260508 07:53:51.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:51.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.142 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 07:53:51.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.142 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:53:51.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:51.143 | INFO | Uploaded 36GiB to GPU in 2.2678s (15.8744GiB/s) | 20260508 07:53:51.143 | INFO | Downloaded 34GiB to GPU in 2.51394s (13.5246GiB/s) | 20260508 07:53:56.284 | INFO | Analyzing chunk frames = [6466048,6466560) | 20260508 07:53:56.284 | INFO | MemInfo: { | 20260508 07:53:56.284 | INFO | "gpu" : " 17 GB", | 20260508 07:53:56.284 | INFO | "huge" : "", | 20260508 07:53:56.284 | INFO | "huge_shared" : " 29 GB", | 20260508 07:53:56.284 | INFO | "malloc" : " 14 GB", | 20260508 07:53:56.284 | INFO | "pinned" : " 29 GB" | 20260508 07:53:56.284 | INFO | } | 20260508 07:53:56.284 | INFO | Perf report for the last 5.14203 seconds: | 20260508 07:53:56.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:53:56.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.284 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:53:56.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.284 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:53:56.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:53:56.285 | INFO | Uploaded 36GiB to GPU in 2.36422s (15.227GiB/s) | 20260508 07:53:56.285 | INFO | Downloaded 34GiB to GPU in 2.4369s (13.9522GiB/s) | 20260508 07:54:01.423 | INFO | Analyzing chunk frames = [6466560,6467072) | 20260508 07:54:01.423 | INFO | MemInfo: { | 20260508 07:54:01.423 | INFO | "gpu" : " 17 GB", | 20260508 07:54:01.423 | INFO | "huge" : "", | 20260508 07:54:01.423 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:01.423 | INFO | "malloc" : " 14 GB", | 20260508 07:54:01.423 | INFO | "pinned" : " 29 GB" | 20260508 07:54:01.423 | INFO | } | 20260508 07:54:01.423 | INFO | Perf report for the last 5.13865 seconds: | 20260508 07:54:01.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:01.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 07:54:01.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:54:01.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:01.423 | INFO | Uploaded 36GiB to GPU in 2.40141s (14.9912GiB/s) | 20260508 07:54:01.423 | INFO | Downloaded 34GiB to GPU in 2.43685s (13.9524GiB/s) | 20260508 07:54:06.556 | INFO | Analyzing chunk frames = [6467072,6467584) | 20260508 07:54:06.556 | INFO | MemInfo: { | 20260508 07:54:06.556 | INFO | "gpu" : " 17 GB", | 20260508 07:54:06.557 | INFO | "huge" : "", | 20260508 07:54:06.557 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:06.557 | INFO | "malloc" : " 14 GB", | 20260508 07:54:06.557 | INFO | "pinned" : " 29 GB" | 20260508 07:54:06.557 | INFO | } | 20260508 07:54:06.557 | INFO | Perf report for the last 5.13388 seconds: | 20260508 07:54:06.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:06.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.557 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 07:54:06.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.557 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 07:54:06.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:06.558 | INFO | Uploaded 36GiB to GPU in 2.44472s (14.7256GiB/s) | 20260508 07:54:06.558 | INFO | Downloaded 34GiB to GPU in 2.46757s (13.7787GiB/s) | 20260508 07:54:06.670 | INFO | Starting a round of Baz IO | 20260508 07:54:10.080 | INFO | Populating RTMetrics report | 20260508 07:54:10.080 | INFO | RTMetrics report populated | 20260508 07:54:10.080 | INFO | Opening tmp RTMetrics json file | 20260508 07:54:10.087 | INFO | Serializing RTMetrics to json | 20260508 07:54:10.089 | INFO | Serialization done, swapping tmp file now... | 20260508 07:54:10.089 | INFO | RTMetrics done | 20260508 07:54:10.150 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.479 seconds | 20260508 07:54:10.150 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.51% | 20260508 07:54:10.150 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319449% / 0% / 5.96046e-05% | 20260508 07:54:10.150 | INFO | Baz IO overall: | 20260508 07:54:10.150 | INFO | BytesWritten : 2846579568528 | 20260508 07:54:10.150 | INFO | NumEvents : 1371000065108 | 20260508 07:54:11.698 | INFO | Analyzing chunk frames = [6467584,6468096) | 20260508 07:54:11.698 | INFO | MemInfo: { | 20260508 07:54:11.698 | INFO | "gpu" : " 17 GB", | 20260508 07:54:11.698 | INFO | "huge" : "", | 20260508 07:54:11.698 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:11.698 | INFO | "malloc" : " 14 GB", | 20260508 07:54:11.698 | INFO | "pinned" : " 29 GB" | 20260508 07:54:11.698 | INFO | } | 20260508 07:54:11.698 | INFO | Perf report for the last 5.14141 seconds: | 20260508 07:54:11.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:11.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:11.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:11.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:11.698 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 07:54:11.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:11.698 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 07:54:11.698 | INFO | PRE_HQ : 66.4%, 1.8, 62.0ms, 0 | 20260508 07:54:11.698 | INFO | BAZWRITER : 67.1%, 2.8, 96.3ms, 0 | 20260508 07:54:11.698 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:54:11.698 | INFO | Uploaded 36GiB to GPU in 2.58723s (13.9145GiB/s) | 20260508 07:54:11.698 | INFO | Downloaded 38GiB to GPU in 2.68987s (14.1271GiB/s) | 20260508 07:54:16.841 | INFO | Analyzing chunk frames = [6468096,6468608) | 20260508 07:54:16.841 | INFO | MemInfo: { | 20260508 07:54:16.841 | INFO | "gpu" : " 17 GB", | 20260508 07:54:16.841 | INFO | "huge" : "", | 20260508 07:54:16.841 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:16.841 | INFO | "malloc" : " 14 GB", | 20260508 07:54:16.841 | INFO | "pinned" : " 29 GB" | 20260508 07:54:16.841 | INFO | } | 20260508 07:54:16.841 | INFO | Perf report for the last 5.14306 seconds: | 20260508 07:54:16.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:16.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.841 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:54:16.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.841 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:54:16.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:16.842 | INFO | Uploaded 36GiB to GPU in 2.53487s (14.2019GiB/s) | 20260508 07:54:16.842 | INFO | Downloaded 34GiB to GPU in 2.43366s (13.9708GiB/s) | 20260508 07:54:21.983 | INFO | Analyzing chunk frames = [6468608,6469120) | 20260508 07:54:21.983 | INFO | MemInfo: { | 20260508 07:54:21.983 | INFO | "gpu" : " 17 GB", | 20260508 07:54:21.983 | INFO | "huge" : "", | 20260508 07:54:21.983 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:21.983 | INFO | "malloc" : " 14 GB", | 20260508 07:54:21.983 | INFO | "pinned" : " 29 GB" | 20260508 07:54:21.983 | INFO | } | 20260508 07:54:21.983 | INFO | Perf report for the last 5.14141 seconds: | 20260508 07:54:21.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:21.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 07:54:21.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:54:21.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:21.983 | INFO | Uploaded 36GiB to GPU in 2.57568s (13.9769GiB/s) | 20260508 07:54:21.983 | INFO | Downloaded 34GiB to GPU in 2.42267s (14.0341GiB/s) | 20260508 07:54:27.122 | INFO | Analyzing chunk frames = [6469120,6469632) | 20260508 07:54:27.122 | INFO | MemInfo: { | 20260508 07:54:27.122 | INFO | "gpu" : " 17 GB", | 20260508 07:54:27.122 | INFO | "huge" : "", | 20260508 07:54:27.122 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:27.122 | INFO | "malloc" : " 14 GB", | 20260508 07:54:27.122 | INFO | "pinned" : " 29 GB" | 20260508 07:54:27.122 | INFO | } | 20260508 07:54:27.122 | INFO | Perf report for the last 5.13934 seconds: | 20260508 07:54:27.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:27.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 07:54:27.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:54:27.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:27.122 | INFO | Uploaded 36GiB to GPU in 2.51406s (14.3195GiB/s) | 20260508 07:54:27.122 | INFO | Downloaded 34GiB to GPU in 2.4298s (13.9929GiB/s) | 20260508 07:54:32.255 | INFO | Analyzing chunk frames = [6469632,6470144) | 20260508 07:54:32.255 | INFO | MemInfo: { | 20260508 07:54:32.255 | INFO | "gpu" : " 17 GB", | 20260508 07:54:32.255 | INFO | "huge" : "", | 20260508 07:54:32.255 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:32.255 | INFO | "malloc" : " 14 GB", | 20260508 07:54:32.255 | INFO | "pinned" : " 29 GB" | 20260508 07:54:32.255 | INFO | } | 20260508 07:54:32.255 | INFO | Perf report for the last 5.1327 seconds: | 20260508 07:54:32.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:32.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 07:54:32.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 07:54:32.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:32.255 | INFO | Uploaded 36GiB to GPU in 2.29738s (15.67GiB/s) | 20260508 07:54:32.255 | INFO | Downloaded 34GiB to GPU in 2.49965s (13.6019GiB/s) | 20260508 07:54:37.394 | INFO | Analyzing chunk frames = [6470144,6470656) | 20260508 07:54:37.394 | INFO | MemInfo: { | 20260508 07:54:37.394 | INFO | "gpu" : " 17 GB", | 20260508 07:54:37.394 | INFO | "huge" : "", | 20260508 07:54:37.394 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:37.394 | INFO | "malloc" : " 14 GB", | 20260508 07:54:37.394 | INFO | "pinned" : " 29 GB" | 20260508 07:54:37.394 | INFO | } | 20260508 07:54:37.395 | INFO | Perf report for the last 5.13974 seconds: | 20260508 07:54:37.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:37.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 07:54:37.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:54:37.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:37.395 | INFO | Uploaded 36GiB to GPU in 2.42091s (14.8704GiB/s) | 20260508 07:54:37.395 | INFO | Downloaded 34GiB to GPU in 2.43178s (13.9815GiB/s) | 20260508 07:54:42.534 | INFO | Analyzing chunk frames = [6470656,6471168) | 20260508 07:54:42.534 | INFO | MemInfo: { | 20260508 07:54:42.534 | INFO | "gpu" : " 17 GB", | 20260508 07:54:42.534 | INFO | "huge" : "", | 20260508 07:54:42.534 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:42.534 | INFO | "malloc" : " 14 GB", | 20260508 07:54:42.534 | INFO | "pinned" : " 29 GB" | 20260508 07:54:42.535 | INFO | } | 20260508 07:54:42.535 | INFO | Perf report for the last 5.1407 seconds: | 20260508 07:54:42.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:42.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.535 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 07:54:42.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.536 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:54:42.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:42.536 | INFO | Uploaded 36GiB to GPU in 2.42702s (14.833GiB/s) | 20260508 07:54:42.536 | INFO | Downloaded 34GiB to GPU in 2.43113s (13.9853GiB/s) | 20260508 07:54:47.677 | INFO | Analyzing chunk frames = [6471168,6471680) | 20260508 07:54:47.678 | INFO | MemInfo: { | 20260508 07:54:47.678 | INFO | "gpu" : " 17 GB", | 20260508 07:54:47.678 | INFO | "huge" : "", | 20260508 07:54:47.678 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:47.678 | INFO | "malloc" : " 14 GB", | 20260508 07:54:47.678 | INFO | "pinned" : " 29 GB" | 20260508 07:54:47.678 | INFO | } | 20260508 07:54:47.678 | INFO | Perf report for the last 5.14269 seconds: | 20260508 07:54:47.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:47.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.678 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 07:54:47.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.678 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 07:54:47.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:47.679 | INFO | Uploaded 36GiB to GPU in 2.46187s (14.623GiB/s) | 20260508 07:54:47.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:54:47.679 | INFO | Total mem usage: 16.7 GiB | 20260508 07:54:47.679 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:54:47.679 | INFO | Downloaded 34GiB to GPU in 2.45029s (13.8759GiB/s) | 20260508 07:54:47.679 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 07:54:47.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:54:47.679 | INFO | High Water Breakdown: | 20260508 07:54:47.679 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:54:47.679 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:54:47.679 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:54:47.679 | INFO | ------------------------------------------ | 20260508 07:54:47.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:54:47.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:54:47.679 | INFO | Total mem usage: 26.5 GiB | 20260508 07:54:47.679 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:54:47.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:54:47.679 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:54:47.679 | INFO | High Water Breakdown: | 20260508 07:54:47.679 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:54:47.679 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:54:47.679 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:54:47.679 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:54:47.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:54:47.679 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:54:47.679 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:54:47.679 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:54:47.679 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:54:47.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:54:47.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:54:47.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:54:47.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:54:47.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:54:47.679 | INFO | ------------------------------------------ | 20260508 07:54:47.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:54:47.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:54:47.679 | INFO | Total mem usage: 27.8 GiB | 20260508 07:54:47.679 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:54:47.679 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:54:47.679 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:54:47.679 | INFO | High Water Breakdown: | 20260508 07:54:47.679 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:54:47.679 | INFO | ------------------------------------------ | 20260508 07:54:47.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:54:47.679 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:54:47.679 | INFO | Total mem usage: 15.7 GiB | 20260508 07:54:47.679 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:54:47.679 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:54:47.680 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:54:47.680 | INFO | High Water Breakdown: | 20260508 07:54:47.680 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:54:47.680 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:54:47.680 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:54:47.680 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:54:47.680 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:54:47.680 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:54:47.680 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:54:47.680 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:54:47.680 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:54:47.680 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:54:47.680 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:54:47.680 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:54:47.680 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:54:47.680 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:54:47.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:54:47.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:54:47.680 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:54:47.680 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:54:47.680 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:54:47.680 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:54:47.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:54:47.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:54:47.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:54:47.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:54:47.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:54:47.680 | INFO | ------------------------------------------ | 20260508 07:54:47.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:54:47.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:54:47.680 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:54:47.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:54:47.680 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.40e+01 / 6.57e-01 | 20260508 07:54:47.680 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.15e-02 / 8.28e-03 | 20260508 07:54:47.680 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.48e+00, 5.95e-01 / 2.23e-01 | 20260508 07:54:47.680 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.17e-01 | 20260508 07:54:47.680 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.54e+01, 3.76e+00 / 1.02e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.90e+00 / 1.24e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.35e+00 / 2.57e+00, 4.09e-01 / 2.56e-01 | 20260508 07:54:47.680 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.16e+00, 2.51e-01 / 1.45e-01 | 20260508 07:54:47.680 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.39e-01, 4.89e-02 / 2.43e-02 | 20260508 07:54:47.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:54:47.680 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:54:47.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:54:47.680 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 24.34%, Upload -- 2.57e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 24.45%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 31.04%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 2.24%, LockUpload -- 2.37e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 2.44%, LockDownload -- 2.58e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 07:54:47.680 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 07:54:47.791 | INFO | Starting a round of Baz IO | 20260508 07:54:51.212 | INFO | Populating RTMetrics report | 20260508 07:54:51.212 | INFO | RTMetrics report populated | 20260508 07:54:51.212 | INFO | Opening tmp RTMetrics json file | 20260508 07:54:51.219 | INFO | Serializing RTMetrics to json | 20260508 07:54:51.220 | INFO | Serialization done, swapping tmp file now... | 20260508 07:54:51.220 | INFO | RTMetrics done | 20260508 07:54:51.279 | INFO | Finished a round of Baz IO: Wrote 1.516 GiB in 3.488 seconds | 20260508 07:54:51.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.51% | 20260508 07:54:51.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31991% / 3.97364e-06% / 7.15256e-05% | 20260508 07:54:51.279 | INFO | Baz IO overall: | 20260508 07:54:51.279 | INFO | BytesWritten : 2848206982967 | 20260508 07:54:51.279 | INFO | NumEvents : 1371771733025 | 20260508 07:54:52.822 | INFO | Analyzing chunk frames = [6471680,6472192) | 20260508 07:54:52.822 | INFO | MemInfo: { | 20260508 07:54:52.822 | INFO | "gpu" : " 17 GB", | 20260508 07:54:52.822 | INFO | "huge" : "", | 20260508 07:54:52.822 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:52.822 | INFO | "malloc" : " 14 GB", | 20260508 07:54:52.822 | INFO | "pinned" : " 29 GB" | 20260508 07:54:52.822 | INFO | } | 20260508 07:54:52.823 | INFO | Perf report for the last 5.14467 seconds: | 20260508 07:54:52.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:52.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:52.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:52.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:52.823 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 07:54:52.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:52.823 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 07:54:52.823 | INFO | PRE_HQ : 66.3%, 1.8, 62.4ms, 0 | 20260508 07:54:52.823 | INFO | BAZWRITER : 67.3%, 2.8, 96.6ms, 0 | 20260508 07:54:52.823 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:54:52.823 | INFO | Uploaded 36GiB to GPU in 2.60156s (13.8378GiB/s) | 20260508 07:54:52.823 | INFO | Downloaded 38GiB to GPU in 2.70478s (14.0492GiB/s) | 20260508 07:54:57.953 | INFO | Analyzing chunk frames = [6472192,6472704) | 20260508 07:54:57.953 | INFO | MemInfo: { | 20260508 07:54:57.953 | INFO | "gpu" : " 17 GB", | 20260508 07:54:57.953 | INFO | "huge" : "", | 20260508 07:54:57.953 | INFO | "huge_shared" : " 29 GB", | 20260508 07:54:57.953 | INFO | "malloc" : " 14 GB", | 20260508 07:54:57.953 | INFO | "pinned" : " 29 GB" | 20260508 07:54:57.953 | INFO | } | 20260508 07:54:57.953 | INFO | Perf report for the last 5.13044 seconds: | 20260508 07:54:57.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:54:57.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 07:54:57.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:54:57.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:54:57.953 | INFO | Uploaded 36GiB to GPU in 2.5392s (14.1777GiB/s) | 20260508 07:54:57.953 | INFO | Downloaded 34GiB to GPU in 2.44743s (13.8921GiB/s) | 20260508 07:55:03.095 | INFO | Analyzing chunk frames = [6472704,6473216) | 20260508 07:55:03.095 | INFO | MemInfo: { | 20260508 07:55:03.095 | INFO | "gpu" : " 17 GB", | 20260508 07:55:03.095 | INFO | "huge" : "", | 20260508 07:55:03.095 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:03.095 | INFO | "malloc" : " 14 GB", | 20260508 07:55:03.095 | INFO | "pinned" : " 29 GB" | 20260508 07:55:03.095 | INFO | } | 20260508 07:55:03.095 | INFO | Perf report for the last 5.14219 seconds: | 20260508 07:55:03.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:03.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.095 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 07:55:03.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.095 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:55:03.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:03.096 | INFO | Uploaded 36GiB to GPU in 2.48886s (14.4645GiB/s) | 20260508 07:55:03.096 | INFO | Downloaded 34GiB to GPU in 2.42112s (14.0431GiB/s) | 20260508 07:55:08.233 | INFO | Analyzing chunk frames = [6473216,6473728) | 20260508 07:55:08.233 | INFO | MemInfo: { | 20260508 07:55:08.233 | INFO | "gpu" : " 17 GB", | 20260508 07:55:08.233 | INFO | "huge" : "", | 20260508 07:55:08.233 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:08.233 | INFO | "malloc" : " 14 GB", | 20260508 07:55:08.233 | INFO | "pinned" : " 29 GB" | 20260508 07:55:08.233 | INFO | } | 20260508 07:55:08.233 | INFO | Perf report for the last 5.1374 seconds: | 20260508 07:55:08.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:08.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 07:55:08.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:55:08.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:08.233 | INFO | Uploaded 36GiB to GPU in 2.58353s (13.9344GiB/s) | 20260508 07:55:08.233 | INFO | Downloaded 34GiB to GPU in 2.4262s (14.0137GiB/s) | 20260508 07:55:13.369 | INFO | Analyzing chunk frames = [6473728,6474240) | 20260508 07:55:13.369 | INFO | MemInfo: { | 20260508 07:55:13.369 | INFO | "gpu" : " 17 GB", | 20260508 07:55:13.369 | INFO | "huge" : "", | 20260508 07:55:13.369 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:13.369 | INFO | "malloc" : " 14 GB", | 20260508 07:55:13.369 | INFO | "pinned" : " 29 GB" | 20260508 07:55:13.369 | INFO | } | 20260508 07:55:13.369 | INFO | Perf report for the last 5.13615 seconds: | 20260508 07:55:13.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:13.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.369 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 07:55:13.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.369 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:55:13.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:13.370 | INFO | Uploaded 36GiB to GPU in 2.40746s (14.9535GiB/s) | 20260508 07:55:13.370 | INFO | Downloaded 34GiB to GPU in 2.51253s (13.5322GiB/s) | 20260508 07:55:18.509 | INFO | Analyzing chunk frames = [6474240,6474752) | 20260508 07:55:18.509 | INFO | MemInfo: { | 20260508 07:55:18.509 | INFO | "gpu" : " 17 GB", | 20260508 07:55:18.509 | INFO | "huge" : "", | 20260508 07:55:18.509 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:18.509 | INFO | "malloc" : " 14 GB", | 20260508 07:55:18.509 | INFO | "pinned" : " 29 GB" | 20260508 07:55:18.509 | INFO | } | 20260508 07:55:18.509 | INFO | Perf report for the last 5.1406 seconds: | 20260508 07:55:18.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:18.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:55:18.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:55:18.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:18.510 | INFO | Uploaded 36GiB to GPU in 2.39484s (15.0323GiB/s) | 20260508 07:55:18.510 | INFO | Downloaded 34GiB to GPU in 2.42622s (14.0136GiB/s) | 20260508 07:55:23.650 | INFO | Analyzing chunk frames = [6474752,6475264) | 20260508 07:55:23.650 | INFO | MemInfo: { | 20260508 07:55:23.650 | INFO | "gpu" : " 17 GB", | 20260508 07:55:23.650 | INFO | "huge" : "", | 20260508 07:55:23.650 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:23.650 | INFO | "malloc" : " 14 GB", | 20260508 07:55:23.650 | INFO | "pinned" : " 29 GB" | 20260508 07:55:23.650 | INFO | } | 20260508 07:55:23.650 | INFO | Perf report for the last 5.14068 seconds: | 20260508 07:55:23.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:23.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.650 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:55:23.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.650 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 07:55:23.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:23.651 | INFO | Uploaded 36GiB to GPU in 2.49037s (14.4557GiB/s) | 20260508 07:55:23.651 | INFO | Downloaded 34GiB to GPU in 2.42762s (14.0055GiB/s) | 20260508 07:55:28.789 | INFO | Analyzing chunk frames = [6475264,6475776) | 20260508 07:55:28.789 | INFO | MemInfo: { | 20260508 07:55:28.789 | INFO | "gpu" : " 17 GB", | 20260508 07:55:28.789 | INFO | "huge" : "", | 20260508 07:55:28.789 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:28.789 | INFO | "malloc" : " 14 GB", | 20260508 07:55:28.789 | INFO | "pinned" : " 29 GB" | 20260508 07:55:28.789 | INFO | } | 20260508 07:55:28.789 | INFO | Perf report for the last 5.13891 seconds: | 20260508 07:55:28.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:28.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.789 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 07:55:28.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.789 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:55:28.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:28.790 | INFO | Uploaded 36GiB to GPU in 2.43828s (14.7645GiB/s) | 20260508 07:55:28.790 | INFO | Downloaded 34GiB to GPU in 2.43304s (13.9743GiB/s) | 20260508 07:55:28.892 | INFO | Starting a round of Baz IO | 20260508 07:55:32.320 | INFO | Populating RTMetrics report | 20260508 07:55:32.320 | INFO | RTMetrics report populated | 20260508 07:55:32.320 | INFO | Opening tmp RTMetrics json file | 20260508 07:55:32.321 | INFO | Serializing RTMetrics to json | 20260508 07:55:32.323 | INFO | Serialization done, swapping tmp file now... | 20260508 07:55:32.324 | INFO | RTMetrics done | 20260508 07:55:32.397 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.504 seconds | 20260508 07:55:32.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.51% | 20260508 07:55:32.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319473% / 3.97364e-06% / 9.93411e-05% | 20260508 07:55:32.397 | INFO | Baz IO overall: | 20260508 07:55:32.397 | INFO | BytesWritten : 2849833461500 | 20260508 07:55:32.397 | INFO | NumEvents : 1372542849918 | 20260508 07:55:33.928 | INFO | Analyzing chunk frames = [6475776,6476288) | 20260508 07:55:33.928 | INFO | MemInfo: { | 20260508 07:55:33.928 | INFO | "gpu" : " 17 GB", | 20260508 07:55:33.928 | INFO | "huge" : "", | 20260508 07:55:33.928 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:33.928 | INFO | "malloc" : " 14 GB", | 20260508 07:55:33.928 | INFO | "pinned" : " 29 GB" | 20260508 07:55:33.928 | INFO | } | 20260508 07:55:33.928 | INFO | Perf report for the last 5.13899 seconds: | 20260508 07:55:33.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:33.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:33.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:33.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:33.928 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 07:55:33.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:33.928 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 07:55:33.928 | INFO | PRE_HQ : 65.2%, 1.6, 54.9ms, 0 | 20260508 07:55:33.928 | INFO | BAZWRITER : 67.7%, 2.6, 90.1ms, 0 | 20260508 07:55:33.928 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 07:55:33.928 | INFO | Uploaded 36GiB to GPU in 2.59357s (13.8805GiB/s) | 20260508 07:55:33.928 | INFO | Downloaded 38GiB to GPU in 2.68599s (14.1475GiB/s) | 20260508 07:55:39.069 | INFO | Analyzing chunk frames = [6476288,6476800) | 20260508 07:55:39.069 | INFO | MemInfo: { | 20260508 07:55:39.069 | INFO | "gpu" : " 17 GB", | 20260508 07:55:39.069 | INFO | "huge" : "", | 20260508 07:55:39.069 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:39.069 | INFO | "malloc" : " 14 GB", | 20260508 07:55:39.069 | INFO | "pinned" : " 29 GB" | 20260508 07:55:39.069 | INFO | } | 20260508 07:55:39.069 | INFO | Perf report for the last 5.14112 seconds: | 20260508 07:55:39.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:39.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.069 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 07:55:39.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.069 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 07:55:39.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:39.070 | INFO | Uploaded 36GiB to GPU in 2.49023s (14.4565GiB/s) | 20260508 07:55:39.070 | INFO | Downloaded 34GiB to GPU in 2.45008s (13.8771GiB/s) | 20260508 07:55:44.204 | INFO | Analyzing chunk frames = [6476800,6477312) | 20260508 07:55:44.204 | INFO | MemInfo: { | 20260508 07:55:44.204 | INFO | "gpu" : " 17 GB", | 20260508 07:55:44.204 | INFO | "huge" : "", | 20260508 07:55:44.204 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:44.204 | INFO | "malloc" : " 14 GB", | 20260508 07:55:44.204 | INFO | "pinned" : " 29 GB" | 20260508 07:55:44.204 | INFO | } | 20260508 07:55:44.204 | INFO | Perf report for the last 5.13459 seconds: | 20260508 07:55:44.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:44.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 07:55:44.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:55:44.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:44.204 | INFO | Uploaded 36GiB to GPU in 2.47199s (14.5631GiB/s) | 20260508 07:55:44.204 | INFO | Downloaded 34GiB to GPU in 2.44519s (13.9048GiB/s) | 20260508 07:55:49.348 | INFO | Analyzing chunk frames = [6477312,6477824) | 20260508 07:55:49.348 | INFO | MemInfo: { | 20260508 07:55:49.348 | INFO | "gpu" : " 17 GB", | 20260508 07:55:49.348 | INFO | "huge" : "", | 20260508 07:55:49.348 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:49.348 | INFO | "malloc" : " 14 GB", | 20260508 07:55:49.348 | INFO | "pinned" : " 29 GB" | 20260508 07:55:49.348 | INFO | } | 20260508 07:55:49.348 | INFO | Perf report for the last 5.14402 seconds: | 20260508 07:55:49.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:49.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | ANALYSIS : 66.7%, 3.0, 103.4ms, 0 | 20260508 07:55:49.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:55:49.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:49.348 | INFO | Uploaded 36GiB to GPU in 2.5676s (14.0209GiB/s) | 20260508 07:55:49.348 | INFO | Downloaded 34GiB to GPU in 2.42992s (13.9922GiB/s) | 20260508 07:55:54.492 | INFO | Analyzing chunk frames = [6477824,6478336) | 20260508 07:55:54.492 | INFO | MemInfo: { | 20260508 07:55:54.492 | INFO | "gpu" : " 17 GB", | 20260508 07:55:54.492 | INFO | "huge" : "", | 20260508 07:55:54.492 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:54.492 | INFO | "malloc" : " 14 GB", | 20260508 07:55:54.492 | INFO | "pinned" : " 29 GB" | 20260508 07:55:54.492 | INFO | } | 20260508 07:55:54.492 | INFO | Perf report for the last 5.14435 seconds: | 20260508 07:55:54.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:54.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.492 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:55:54.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.492 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:55:54.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:54.493 | INFO | Uploaded 36GiB to GPU in 2.25749s (15.9469GiB/s) | 20260508 07:55:54.493 | INFO | Downloaded 34GiB to GPU in 2.49985s (13.6008GiB/s) | 20260508 07:55:59.634 | INFO | Analyzing chunk frames = [6478336,6478848) | 20260508 07:55:59.634 | INFO | MemInfo: { | 20260508 07:55:59.634 | INFO | "gpu" : " 17 GB", | 20260508 07:55:59.634 | INFO | "huge" : "", | 20260508 07:55:59.634 | INFO | "huge_shared" : " 29 GB", | 20260508 07:55:59.634 | INFO | "malloc" : " 14 GB", | 20260508 07:55:59.634 | INFO | "pinned" : " 29 GB" | 20260508 07:55:59.634 | INFO | } | 20260508 07:55:59.634 | INFO | Perf report for the last 5.14209 seconds: | 20260508 07:55:59.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:55:59.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.634 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 07:55:59.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.635 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:55:59.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:55:59.635 | INFO | Uploaded 36GiB to GPU in 2.37753s (15.1418GiB/s) | 20260508 07:55:59.635 | INFO | Downloaded 34GiB to GPU in 2.44152s (13.9258GiB/s) | 20260508 07:56:04.763 | INFO | Analyzing chunk frames = [6478848,6479360) | 20260508 07:56:04.763 | INFO | MemInfo: { | 20260508 07:56:04.763 | INFO | "gpu" : " 17 GB", | 20260508 07:56:04.763 | INFO | "huge" : "", | 20260508 07:56:04.763 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:04.763 | INFO | "malloc" : " 14 GB", | 20260508 07:56:04.763 | INFO | "pinned" : " 29 GB" | 20260508 07:56:04.763 | INFO | } | 20260508 07:56:04.763 | INFO | Perf report for the last 5.12906 seconds: | 20260508 07:56:04.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:04.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 07:56:04.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:56:04.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:04.764 | INFO | Uploaded 36GiB to GPU in 2.4061s (14.9619GiB/s) | 20260508 07:56:04.764 | INFO | Downloaded 34GiB to GPU in 2.43752s (13.9486GiB/s) | 20260508 07:56:09.907 | INFO | Analyzing chunk frames = [6479360,6479872) | 20260508 07:56:09.907 | INFO | MemInfo: { | 20260508 07:56:09.907 | INFO | "gpu" : " 17 GB", | 20260508 07:56:09.907 | INFO | "huge" : "", | 20260508 07:56:09.907 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:09.907 | INFO | "malloc" : " 14 GB", | 20260508 07:56:09.907 | INFO | "pinned" : " 29 GB" | 20260508 07:56:09.907 | INFO | } | 20260508 07:56:09.907 | INFO | Perf report for the last 5.14355 seconds: | 20260508 07:56:09.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:09.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.907 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 07:56:09.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.908 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:56:09.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:09.908 | INFO | Uploaded 36GiB to GPU in 2.43521s (14.7831GiB/s) | 20260508 07:56:09.908 | INFO | Downloaded 34GiB to GPU in 2.44864s (13.8853GiB/s) | 20260508 07:56:09.908 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:56:09.908 | INFO | Total mem usage: 16.7 GiB | 20260508 07:56:09.908 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:56:09.908 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 07:56:09.908 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:56:09.908 | INFO | High Water Breakdown: | 20260508 07:56:09.908 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:56:09.908 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:56:09.908 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:56:09.908 | INFO | ------------------------------------------ | 20260508 07:56:09.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:56:09.908 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:56:09.908 | INFO | Total mem usage: 26.5 GiB | 20260508 07:56:09.908 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:56:09.908 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:56:09.908 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:56:09.908 | INFO | High Water Breakdown: | 20260508 07:56:09.908 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:56:09.908 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:56:09.908 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:56:09.908 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:56:09.908 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:56:09.908 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:56:09.908 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:56:09.908 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:56:09.908 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:56:09.908 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:56:09.908 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:56:09.908 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:56:09.908 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:56:09.908 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:56:09.908 | INFO | ------------------------------------------ | 20260508 07:56:09.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:56:09.908 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:56:09.908 | INFO | Total mem usage: 27.8 GiB | 20260508 07:56:09.908 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:56:09.908 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:56:09.908 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:56:09.908 | INFO | High Water Breakdown: | 20260508 07:56:09.908 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:56:09.908 | INFO | ------------------------------------------ | 20260508 07:56:09.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:56:09.908 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:56:09.908 | INFO | Total mem usage: 15.7 GiB | 20260508 07:56:09.908 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:56:09.908 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:56:09.908 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:56:09.908 | INFO | High Water Breakdown: | 20260508 07:56:09.908 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:56:09.908 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:56:09.908 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:56:09.908 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:56:09.908 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:56:09.908 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:56:09.908 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:56:09.908 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:56:09.908 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:56:09.908 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:56:09.908 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:56:09.908 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:56:09.908 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:56:09.908 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:56:09.908 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:56:09.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:56:09.909 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:56:09.909 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:56:09.909 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:56:09.909 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:56:09.909 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:56:09.909 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:56:09.909 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:56:09.909 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:56:09.909 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:56:09.909 | INFO | ------------------------------------------ | 20260508 07:56:09.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:56:09.909 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:56:09.909 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:56:09.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:56:09.909 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.41e+01 / 6.42e-01 | 20260508 07:56:09.909 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.05e-01, 1.09e-02 / 3.92e-03 | 20260508 07:56:09.909 | INFO | PerfReport: 12.12%, Baseline -- 3.97e+00 / 4.49e+00, 5.92e-01 / 2.01e-01 | 20260508 07:56:09.909 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.12e-01 / 1.15e-01 | 20260508 07:56:09.909 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.50e+01, 3.69e+00 / 1.01e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.86e+00 / 1.18e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.55e+00, 4.08e-01 / 1.82e-01 | 20260508 07:56:09.909 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.17e+00, 2.49e-01 / 1.34e-01 | 20260508 07:56:09.909 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.41e-01, 4.89e-02 / 2.41e-02 | 20260508 07:56:09.909 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:56:09.909 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:56:09.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:56:09.909 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 24.35%, Upload -- 2.57e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 24.41%, Download -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 30.99%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 2.29%, LockUpload -- 2.42e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 2.41%, LockDownload -- 2.55e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 15.54%, LockCompute -- 1.64e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 07:56:09.909 | INFO | PerfReport: 0.01%, Misc -- 1.17e-02 / 0.00e+00, 1.08e-02 / 0.00e+00 | 20260508 07:56:10.005 | INFO | Starting a round of Baz IO | 20260508 07:56:13.421 | INFO | Populating RTMetrics report | 20260508 07:56:13.421 | INFO | RTMetrics report populated | 20260508 07:56:13.421 | INFO | Opening tmp RTMetrics json file | 20260508 07:56:13.433 | INFO | Serializing RTMetrics to json | 20260508 07:56:13.434 | INFO | Serialization done, swapping tmp file now... | 20260508 07:56:13.435 | INFO | RTMetrics done | 20260508 07:56:13.493 | INFO | Finished a round of Baz IO: Wrote 1.514 GiB in 3.488 seconds | 20260508 07:56:13.493 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.51% | 20260508 07:56:13.493 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31811% / 0% / 5.5631e-05% | 20260508 07:56:13.493 | INFO | Baz IO overall: | 20260508 07:56:13.493 | INFO | BytesWritten : 2851459609871 | 20260508 07:56:13.493 | INFO | NumEvents : 1373313645586 | 20260508 07:56:15.046 | INFO | Analyzing chunk frames = [6479872,6480384) | 20260508 07:56:15.046 | INFO | MemInfo: { | 20260508 07:56:15.046 | INFO | "gpu" : " 17 GB", | 20260508 07:56:15.046 | INFO | "huge" : "", | 20260508 07:56:15.046 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:15.046 | INFO | "malloc" : " 14 GB", | 20260508 07:56:15.046 | INFO | "pinned" : " 29 GB" | 20260508 07:56:15.046 | INFO | } | 20260508 07:56:15.046 | INFO | Perf report for the last 5.13937 seconds: | 20260508 07:56:15.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:15.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:15.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:15.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:15.046 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 07:56:15.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:15.046 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:56:15.046 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 07:56:15.046 | INFO | BAZWRITER : 67.2%, 2.6, 89.5ms, 0 | 20260508 07:56:15.047 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:56:15.047 | INFO | Uploaded 36GiB to GPU in 2.60097s (13.841GiB/s) | 20260508 07:56:15.047 | INFO | Downloaded 38GiB to GPU in 2.7142s (14.0004GiB/s) | 20260508 07:56:20.188 | INFO | Analyzing chunk frames = [6480384,6480896) | 20260508 07:56:20.188 | INFO | MemInfo: { | 20260508 07:56:20.188 | INFO | "gpu" : " 17 GB", | 20260508 07:56:20.188 | INFO | "huge" : "", | 20260508 07:56:20.188 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:20.188 | INFO | "malloc" : " 14 GB", | 20260508 07:56:20.188 | INFO | "pinned" : " 29 GB" | 20260508 07:56:20.188 | INFO | } | 20260508 07:56:20.188 | INFO | Perf report for the last 5.14151 seconds: | 20260508 07:56:20.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:20.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 07:56:20.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 07:56:20.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:20.188 | INFO | Uploaded 36GiB to GPU in 2.44506s (14.7235GiB/s) | 20260508 07:56:20.188 | INFO | Downloaded 34GiB to GPU in 2.44183s (13.924GiB/s) | 20260508 07:56:25.324 | INFO | Analyzing chunk frames = [6480896,6481408) | 20260508 07:56:25.324 | INFO | MemInfo: { | 20260508 07:56:25.324 | INFO | "gpu" : " 17 GB", | 20260508 07:56:25.324 | INFO | "huge" : "", | 20260508 07:56:25.324 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:25.324 | INFO | "malloc" : " 14 GB", | 20260508 07:56:25.324 | INFO | "pinned" : " 29 GB" | 20260508 07:56:25.324 | INFO | } | 20260508 07:56:25.324 | INFO | Perf report for the last 5.13629 seconds: | 20260508 07:56:25.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:25.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 07:56:25.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 07:56:25.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:25.324 | INFO | Uploaded 36GiB to GPU in 2.47374s (14.5529GiB/s) | 20260508 07:56:25.324 | INFO | Downloaded 34GiB to GPU in 2.4345s (13.9659GiB/s) | 20260508 07:56:30.457 | INFO | Analyzing chunk frames = [6481408,6481920) | 20260508 07:56:30.457 | INFO | MemInfo: { | 20260508 07:56:30.457 | INFO | "gpu" : " 17 GB", | 20260508 07:56:30.457 | INFO | "huge" : "", | 20260508 07:56:30.457 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:30.457 | INFO | "malloc" : " 14 GB", | 20260508 07:56:30.457 | INFO | "pinned" : " 29 GB" | 20260508 07:56:30.457 | INFO | } | 20260508 07:56:30.458 | INFO | Perf report for the last 5.13338 seconds: | 20260508 07:56:30.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:30.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.458 | INFO | ANALYSIS : 66.9%, 2.9, 103.3ms, 0 | 20260508 07:56:30.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.458 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 07:56:30.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:30.459 | INFO | Uploaded 36GiB to GPU in 2.56765s (14.0206GiB/s) | 20260508 07:56:30.459 | INFO | Downloaded 34GiB to GPU in 2.43376s (13.9702GiB/s) | 20260508 07:56:35.607 | INFO | Analyzing chunk frames = [6481920,6482432) | 20260508 07:56:35.607 | INFO | MemInfo: { | 20260508 07:56:35.607 | INFO | "gpu" : " 17 GB", | 20260508 07:56:35.607 | INFO | "huge" : "", | 20260508 07:56:35.607 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:35.607 | INFO | "malloc" : " 14 GB", | 20260508 07:56:35.607 | INFO | "pinned" : " 29 GB" | 20260508 07:56:35.608 | INFO | } | 20260508 07:56:35.608 | INFO | Perf report for the last 5.15009 seconds: | 20260508 07:56:35.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:35.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 07:56:35.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 07:56:35.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:35.608 | INFO | Uploaded 36GiB to GPU in 2.258s (15.9433GiB/s) | 20260508 07:56:35.608 | INFO | Downloaded 34GiB to GPU in 2.50825s (13.5553GiB/s) | 20260508 07:56:40.743 | INFO | Analyzing chunk frames = [6482432,6482944) | 20260508 07:56:40.743 | INFO | MemInfo: { | 20260508 07:56:40.743 | INFO | "gpu" : " 17 GB", | 20260508 07:56:40.743 | INFO | "huge" : "", | 20260508 07:56:40.743 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:40.743 | INFO | "malloc" : " 14 GB", | 20260508 07:56:40.743 | INFO | "pinned" : " 29 GB" | 20260508 07:56:40.743 | INFO | } | 20260508 07:56:40.743 | INFO | Perf report for the last 5.13556 seconds: | 20260508 07:56:40.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:40.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.743 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 07:56:40.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.743 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 07:56:40.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:40.744 | INFO | Uploaded 36GiB to GPU in 2.35874s (15.2624GiB/s) | 20260508 07:56:40.744 | INFO | Downloaded 34GiB to GPU in 2.44378s (13.9129GiB/s) | 20260508 07:56:45.878 | INFO | Analyzing chunk frames = [6482944,6483456) | 20260508 07:56:45.878 | INFO | MemInfo: { | 20260508 07:56:45.878 | INFO | "gpu" : " 17 GB", | 20260508 07:56:45.878 | INFO | "huge" : "", | 20260508 07:56:45.878 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:45.878 | INFO | "malloc" : " 14 GB", | 20260508 07:56:45.878 | INFO | "pinned" : " 29 GB" | 20260508 07:56:45.878 | INFO | } | 20260508 07:56:45.878 | INFO | Perf report for the last 5.13507 seconds: | 20260508 07:56:45.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:45.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.878 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 07:56:45.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.878 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:56:45.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:45.879 | INFO | Uploaded 36GiB to GPU in 2.46668s (14.5945GiB/s) | 20260508 07:56:45.879 | INFO | Downloaded 34GiB to GPU in 2.45337s (13.8585GiB/s) | 20260508 07:56:51.017 | INFO | Analyzing chunk frames = [6483456,6483968) | 20260508 07:56:51.017 | INFO | MemInfo: { | 20260508 07:56:51.017 | INFO | "gpu" : " 17 GB", | 20260508 07:56:51.017 | INFO | "huge" : "", | 20260508 07:56:51.017 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:51.017 | INFO | "malloc" : " 14 GB", | 20260508 07:56:51.017 | INFO | "pinned" : " 29 GB" | 20260508 07:56:51.017 | INFO | } | 20260508 07:56:51.017 | INFO | Perf report for the last 5.13857 seconds: | 20260508 07:56:51.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:51.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 07:56:51.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 07:56:51.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:51.017 | INFO | Uploaded 36GiB to GPU in 2.45304s (14.6757GiB/s) | 20260508 07:56:51.017 | INFO | Downloaded 34GiB to GPU in 2.46543s (13.7907GiB/s) | 20260508 07:56:51.115 | INFO | Starting a round of Baz IO | 20260508 07:56:54.529 | INFO | Populating RTMetrics report | 20260508 07:56:54.529 | INFO | RTMetrics report populated | 20260508 07:56:54.529 | INFO | Opening tmp RTMetrics json file | 20260508 07:56:54.540 | INFO | Serializing RTMetrics to json | 20260508 07:56:54.542 | INFO | Serialization done, swapping tmp file now... | 20260508 07:56:54.542 | INFO | RTMetrics done | 20260508 07:56:54.602 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.487 seconds | 20260508 07:56:54.602 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.00% / 59.51% | 20260508 07:56:54.602 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31805% / 3.97364e-06% / 7.15256e-05% | 20260508 07:56:54.602 | INFO | Baz IO overall: | 20260508 07:56:54.602 | INFO | BytesWritten : 2853086300062 | 20260508 07:56:54.602 | INFO | NumEvents : 1374084949046 | 20260508 07:56:56.158 | INFO | Analyzing chunk frames = [6483968,6484480) | 20260508 07:56:56.158 | INFO | MemInfo: { | 20260508 07:56:56.158 | INFO | "gpu" : " 17 GB", | 20260508 07:56:56.158 | INFO | "huge" : "", | 20260508 07:56:56.158 | INFO | "huge_shared" : " 29 GB", | 20260508 07:56:56.158 | INFO | "malloc" : " 14 GB", | 20260508 07:56:56.158 | INFO | "pinned" : " 29 GB" | 20260508 07:56:56.158 | INFO | } | 20260508 07:56:56.159 | INFO | Perf report for the last 5.14174 seconds: | 20260508 07:56:56.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:56:56.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:56.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:56.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:56.159 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 07:56:56.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:56:56.159 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:56:56.159 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 07:56:56.159 | INFO | BAZWRITER : 67.2%, 2.6, 88.9ms, 0 | 20260508 07:56:56.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:56:56.159 | INFO | Uploaded 36GiB to GPU in 2.56147s (14.0544GiB/s) | 20260508 07:56:56.159 | INFO | Downloaded 38GiB to GPU in 2.7169s (13.9865GiB/s) | 20260508 07:57:01.298 | INFO | Analyzing chunk frames = [6484480,6484992) | 20260508 07:57:01.298 | INFO | MemInfo: { | 20260508 07:57:01.298 | INFO | "gpu" : " 17 GB", | 20260508 07:57:01.298 | INFO | "huge" : "", | 20260508 07:57:01.298 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:01.298 | INFO | "malloc" : " 14 GB", | 20260508 07:57:01.298 | INFO | "pinned" : " 29 GB" | 20260508 07:57:01.299 | INFO | } | 20260508 07:57:01.299 | INFO | Perf report for the last 5.13995 seconds: | 20260508 07:57:01.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:01.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 07:57:01.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 07:57:01.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:01.299 | INFO | Uploaded 36GiB to GPU in 2.48346s (14.4959GiB/s) | 20260508 07:57:01.299 | INFO | Downloaded 34GiB to GPU in 2.44823s (13.8876GiB/s) | 20260508 07:57:06.441 | INFO | Analyzing chunk frames = [6484992,6485504) | 20260508 07:57:06.441 | INFO | MemInfo: { | 20260508 07:57:06.441 | INFO | "gpu" : " 17 GB", | 20260508 07:57:06.441 | INFO | "huge" : "", | 20260508 07:57:06.441 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:06.441 | INFO | "malloc" : " 14 GB", | 20260508 07:57:06.441 | INFO | "pinned" : " 29 GB" | 20260508 07:57:06.441 | INFO | } | 20260508 07:57:06.441 | INFO | Perf report for the last 5.14258 seconds: | 20260508 07:57:06.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:06.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.441 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 07:57:06.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.441 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 07:57:06.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:06.442 | INFO | Uploaded 36GiB to GPU in 2.52712s (14.2455GiB/s) | 20260508 07:57:06.442 | INFO | Downloaded 34GiB to GPU in 2.45128s (13.8703GiB/s) | 20260508 07:57:11.579 | INFO | Analyzing chunk frames = [6485504,6486016) | 20260508 07:57:11.579 | INFO | MemInfo: { | 20260508 07:57:11.579 | INFO | "gpu" : " 17 GB", | 20260508 07:57:11.579 | INFO | "huge" : "", | 20260508 07:57:11.579 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:11.579 | INFO | "malloc" : " 14 GB", | 20260508 07:57:11.579 | INFO | "pinned" : " 29 GB" | 20260508 07:57:11.579 | INFO | } | 20260508 07:57:11.579 | INFO | Perf report for the last 5.13763 seconds: | 20260508 07:57:11.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:11.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.579 | INFO | ANALYSIS : 66.7%, 2.9, 103.2ms, 0 | 20260508 07:57:11.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.579 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:57:11.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:11.580 | INFO | Uploaded 36GiB to GPU in 2.53575s (14.197GiB/s) | 20260508 07:57:11.580 | INFO | Downloaded 34GiB to GPU in 2.44302s (13.9172GiB/s) | 20260508 07:57:16.715 | INFO | Analyzing chunk frames = [6486016,6486528) | 20260508 07:57:16.715 | INFO | MemInfo: { | 20260508 07:57:16.715 | INFO | "gpu" : " 17 GB", | 20260508 07:57:16.715 | INFO | "huge" : "", | 20260508 07:57:16.715 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:16.715 | INFO | "malloc" : " 14 GB", | 20260508 07:57:16.715 | INFO | "pinned" : " 29 GB" | 20260508 07:57:16.715 | INFO | } | 20260508 07:57:16.715 | INFO | Perf report for the last 5.13652 seconds: | 20260508 07:57:16.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:16.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:57:16.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:57:16.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:16.716 | INFO | Uploaded 36GiB to GPU in 2.27123s (15.8504GiB/s) | 20260508 07:57:16.716 | INFO | Downloaded 34GiB to GPU in 2.51415s (13.5235GiB/s) | 20260508 07:57:21.852 | INFO | Analyzing chunk frames = [6486528,6487040) | 20260508 07:57:21.852 | INFO | MemInfo: { | 20260508 07:57:21.852 | INFO | "gpu" : " 17 GB", | 20260508 07:57:21.852 | INFO | "huge" : "", | 20260508 07:57:21.852 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:21.852 | INFO | "malloc" : " 14 GB", | 20260508 07:57:21.852 | INFO | "pinned" : " 29 GB" | 20260508 07:57:21.852 | INFO | } | 20260508 07:57:21.852 | INFO | Perf report for the last 5.13695 seconds: | 20260508 07:57:21.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:21.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.853 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:57:21.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.853 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:57:21.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:21.853 | INFO | Uploaded 36GiB to GPU in 2.36039s (15.2517GiB/s) | 20260508 07:57:21.853 | INFO | Downloaded 34GiB to GPU in 2.42814s (14.0025GiB/s) | 20260508 07:57:26.996 | INFO | Analyzing chunk frames = [6487040,6487552) | 20260508 07:57:26.996 | INFO | MemInfo: { | 20260508 07:57:26.996 | INFO | "gpu" : " 17 GB", | 20260508 07:57:26.996 | INFO | "huge" : "", | 20260508 07:57:26.996 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:26.996 | INFO | "malloc" : " 14 GB", | 20260508 07:57:26.996 | INFO | "pinned" : " 29 GB" | 20260508 07:57:26.996 | INFO | } | 20260508 07:57:26.996 | INFO | Perf report for the last 5.14345 seconds: | 20260508 07:57:26.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:26.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:57:26.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:57:26.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:26.996 | INFO | Uploaded 36GiB to GPU in 2.47687s (14.5345GiB/s) | 20260508 07:57:26.996 | INFO | Downloaded 34GiB to GPU in 2.43979s (13.9356GiB/s) | 20260508 07:57:32.138 | INFO | Analyzing chunk frames = [6487552,6488064) | 20260508 07:57:32.138 | INFO | MemInfo: { | 20260508 07:57:32.138 | INFO | "gpu" : " 17 GB", | 20260508 07:57:32.138 | INFO | "huge" : "", | 20260508 07:57:32.138 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:32.138 | INFO | "malloc" : " 14 GB", | 20260508 07:57:32.138 | INFO | "pinned" : " 29 GB" | 20260508 07:57:32.138 | INFO | } | 20260508 07:57:32.138 | INFO | Perf report for the last 5.14191 seconds: | 20260508 07:57:32.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:32.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 07:57:32.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 07:57:32.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:32.138 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:57:32.138 | INFO | Total mem usage: 16.7 GiB | 20260508 07:57:32.138 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:57:32.138 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:57:32.138 | INFO | Uploaded 36GiB to GPU in 2.49057s (14.4545GiB/s) | 20260508 07:57:32.138 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:57:32.138 | INFO | High Water Breakdown: | 20260508 07:57:32.138 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:57:32.138 | INFO | Downloaded 34GiB to GPU in 2.46852s (13.7734GiB/s) | 20260508 07:57:32.138 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:57:32.139 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:57:32.139 | INFO | ------------------------------------------ | 20260508 07:57:32.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:57:32.139 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:57:32.139 | INFO | Total mem usage: 26.5 GiB | 20260508 07:57:32.139 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:57:32.139 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 07:57:32.139 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 07:57:32.139 | INFO | High Water Breakdown: | 20260508 07:57:32.139 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:57:32.139 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:57:32.139 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:57:32.139 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:57:32.139 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:57:32.139 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:57:32.139 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:57:32.139 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:57:32.139 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:57:32.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:57:32.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:57:32.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:57:32.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:57:32.139 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:57:32.139 | INFO | ------------------------------------------ | 20260508 07:57:32.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:57:32.139 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:57:32.139 | INFO | Total mem usage: 27.8 GiB | 20260508 07:57:32.139 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:57:32.139 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:57:32.139 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:57:32.139 | INFO | High Water Breakdown: | 20260508 07:57:32.139 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:57:32.139 | INFO | ------------------------------------------ | 20260508 07:57:32.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:57:32.139 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:57:32.139 | INFO | Total mem usage: 15.7 GiB | 20260508 07:57:32.139 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:57:32.139 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:57:32.139 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:57:32.139 | INFO | High Water Breakdown: | 20260508 07:57:32.139 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:57:32.139 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:57:32.139 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:57:32.139 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:57:32.139 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:57:32.139 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:57:32.139 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:57:32.139 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:57:32.139 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:57:32.139 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:57:32.139 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:57:32.139 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:57:32.139 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:57:32.139 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:57:32.139 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:57:32.139 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:57:32.139 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:57:32.139 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:57:32.139 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:57:32.139 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:57:32.139 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:57:32.139 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:57:32.139 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:57:32.139 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:57:32.139 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:57:32.139 | INFO | ------------------------------------------ | 20260508 07:57:32.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:57:32.139 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:57:32.139 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:57:32.139 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:57:32.139 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.42e+01 / 6.59e-01 | 20260508 07:57:32.139 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.02e-01, 1.16e-02 / 6.15e-03 | 20260508 07:57:32.139 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.45e+00, 5.86e-01 / 2.35e-01 | 20260508 07:57:32.139 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.13e-01 | 20260508 07:57:32.139 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.54e+01, 3.85e+00 / 1.15e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.83e+00 / 1.13e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.35e+00 / 2.56e+00, 4.13e-01 / 2.29e-01 | 20260508 07:57:32.139 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.19e+00, 2.49e-01 / 1.31e-01 | 20260508 07:57:32.139 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.40e-01, 4.94e-02 / 2.57e-02 | 20260508 07:57:32.139 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:57:32.139 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:57:32.139 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:57:32.139 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 24.22%, Upload -- 2.56e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 24.51%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 30.91%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 2.11%, LockUpload -- 2.23e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 2.68%, LockDownload -- 2.84e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 15.56%, LockCompute -- 1.65e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 07:57:32.139 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260508 07:57:32.235 | INFO | Starting a round of Baz IO | 20260508 07:57:35.644 | INFO | Populating RTMetrics report | 20260508 07:57:35.644 | INFO | RTMetrics report populated | 20260508 07:57:35.644 | INFO | Opening tmp RTMetrics json file | 20260508 07:57:35.656 | INFO | Serializing RTMetrics to json | 20260508 07:57:35.657 | INFO | Serialization done, swapping tmp file now... | 20260508 07:57:35.658 | INFO | RTMetrics done | 20260508 07:57:35.720 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.485 seconds | 20260508 07:57:35.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.51% | 20260508 07:57:35.720 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317836% / 0% / 7.15256e-05% | 20260508 07:57:35.720 | INFO | Baz IO overall: | 20260508 07:57:35.720 | INFO | BytesWritten : 2854713434787 | 20260508 07:57:35.720 | INFO | NumEvents : 1374856773151 | 20260508 07:57:37.274 | INFO | Analyzing chunk frames = [6488064,6488576) | 20260508 07:57:37.274 | INFO | MemInfo: { | 20260508 07:57:37.274 | INFO | "gpu" : " 17 GB", | 20260508 07:57:37.274 | INFO | "huge" : "", | 20260508 07:57:37.274 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:37.274 | INFO | "malloc" : " 14 GB", | 20260508 07:57:37.274 | INFO | "pinned" : " 29 GB" | 20260508 07:57:37.274 | INFO | } | 20260508 07:57:37.274 | INFO | Perf report for the last 5.13611 seconds: | 20260508 07:57:37.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:37.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:37.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:37.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:37.274 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 07:57:37.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:37.274 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:57:37.274 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 07:57:37.274 | INFO | BAZWRITER : 67.2%, 2.6, 89.4ms, 0 | 20260508 07:57:37.274 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:57:37.275 | INFO | Uploaded 36GiB to GPU in 2.58557s (13.9234GiB/s) | 20260508 07:57:37.275 | INFO | Downloaded 38GiB to GPU in 2.70574s (14.0442GiB/s) | 20260508 07:57:42.415 | INFO | Analyzing chunk frames = [6488576,6489088) | 20260508 07:57:42.415 | INFO | MemInfo: { | 20260508 07:57:42.415 | INFO | "gpu" : " 17 GB", | 20260508 07:57:42.415 | INFO | "huge" : "", | 20260508 07:57:42.415 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:42.415 | INFO | "malloc" : " 14 GB", | 20260508 07:57:42.415 | INFO | "pinned" : " 29 GB" | 20260508 07:57:42.415 | INFO | } | 20260508 07:57:42.415 | INFO | Perf report for the last 5.14149 seconds: | 20260508 07:57:42.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:42.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.415 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:57:42.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.416 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 07:57:42.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:42.416 | INFO | Uploaded 36GiB to GPU in 2.53632s (14.1938GiB/s) | 20260508 07:57:42.416 | INFO | Downloaded 34GiB to GPU in 2.43889s (13.9407GiB/s) | 20260508 07:57:47.556 | INFO | Analyzing chunk frames = [6489088,6489600) | 20260508 07:57:47.556 | INFO | MemInfo: { | 20260508 07:57:47.556 | INFO | "gpu" : " 17 GB", | 20260508 07:57:47.556 | INFO | "huge" : "", | 20260508 07:57:47.556 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:47.556 | INFO | "malloc" : " 14 GB", | 20260508 07:57:47.556 | INFO | "pinned" : " 29 GB" | 20260508 07:57:47.556 | INFO | } | 20260508 07:57:47.556 | INFO | Perf report for the last 5.14069 seconds: | 20260508 07:57:47.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:47.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.556 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 07:57:47.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.556 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 07:57:47.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:47.557 | INFO | Uploaded 36GiB to GPU in 2.48853s (14.4664GiB/s) | 20260508 07:57:47.557 | INFO | Downloaded 34GiB to GPU in 2.43675s (13.953GiB/s) | 20260508 07:57:52.692 | INFO | Analyzing chunk frames = [6489600,6490112) | 20260508 07:57:52.692 | INFO | MemInfo: { | 20260508 07:57:52.692 | INFO | "gpu" : " 17 GB", | 20260508 07:57:52.692 | INFO | "huge" : "", | 20260508 07:57:52.692 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:52.692 | INFO | "malloc" : " 14 GB", | 20260508 07:57:52.692 | INFO | "pinned" : " 29 GB" | 20260508 07:57:52.693 | INFO | } | 20260508 07:57:52.693 | INFO | Perf report for the last 5.13649 seconds: | 20260508 07:57:52.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:52.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 07:57:52.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 07:57:52.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:52.693 | INFO | Uploaded 36GiB to GPU in 2.53614s (14.1948GiB/s) | 20260508 07:57:52.693 | INFO | Downloaded 34GiB to GPU in 2.43231s (13.9785GiB/s) | 20260508 07:57:57.831 | INFO | Analyzing chunk frames = [6490112,6490624) | 20260508 07:57:57.831 | INFO | MemInfo: { | 20260508 07:57:57.831 | INFO | "gpu" : " 17 GB", | 20260508 07:57:57.831 | INFO | "huge" : "", | 20260508 07:57:57.831 | INFO | "huge_shared" : " 29 GB", | 20260508 07:57:57.831 | INFO | "malloc" : " 14 GB", | 20260508 07:57:57.831 | INFO | "pinned" : " 29 GB" | 20260508 07:57:57.831 | INFO | } | 20260508 07:57:57.832 | INFO | Perf report for the last 5.13902 seconds: | 20260508 07:57:57.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:57:57.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 07:57:57.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260508 07:57:57.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:57:57.832 | INFO | Uploaded 36GiB to GPU in 2.26582s (15.8883GiB/s) | 20260508 07:57:57.832 | INFO | Downloaded 34GiB to GPU in 2.49524s (13.6259GiB/s) | 20260508 07:58:02.967 | INFO | Analyzing chunk frames = [6490624,6491136) | 20260508 07:58:02.967 | INFO | MemInfo: { | 20260508 07:58:02.967 | INFO | "gpu" : " 17 GB", | 20260508 07:58:02.967 | INFO | "huge" : "", | 20260508 07:58:02.967 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:02.967 | INFO | "malloc" : " 14 GB", | 20260508 07:58:02.967 | INFO | "pinned" : " 29 GB" | 20260508 07:58:02.967 | INFO | } | 20260508 07:58:02.967 | INFO | Perf report for the last 5.13502 seconds: | 20260508 07:58:02.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:02.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 07:58:02.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 07:58:02.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:02.967 | INFO | Uploaded 36GiB to GPU in 2.375s (15.1579GiB/s) | 20260508 07:58:02.967 | INFO | Downloaded 34GiB to GPU in 2.43412s (13.9681GiB/s) | 20260508 07:58:08.108 | INFO | Analyzing chunk frames = [6491136,6491648) | 20260508 07:58:08.108 | INFO | MemInfo: { | 20260508 07:58:08.108 | INFO | "gpu" : " 17 GB", | 20260508 07:58:08.108 | INFO | "huge" : "", | 20260508 07:58:08.108 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:08.108 | INFO | "malloc" : " 14 GB", | 20260508 07:58:08.109 | INFO | "pinned" : " 29 GB" | 20260508 07:58:08.109 | INFO | } | 20260508 07:58:08.109 | INFO | Perf report for the last 5.14189 seconds: | 20260508 07:58:08.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:08.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 07:58:08.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 07:58:08.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:08.109 | INFO | Uploaded 36GiB to GPU in 2.46054s (14.631GiB/s) | 20260508 07:58:08.109 | INFO | Downloaded 34GiB to GPU in 2.43279s (13.9757GiB/s) | 20260508 07:58:13.256 | INFO | Analyzing chunk frames = [6491648,6492160) | 20260508 07:58:13.256 | INFO | MemInfo: { | 20260508 07:58:13.256 | INFO | "gpu" : " 17 GB", | 20260508 07:58:13.256 | INFO | "huge" : "", | 20260508 07:58:13.256 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:13.256 | INFO | "malloc" : " 14 GB", | 20260508 07:58:13.256 | INFO | "pinned" : " 29 GB" | 20260508 07:58:13.256 | INFO | } | 20260508 07:58:13.256 | INFO | Perf report for the last 5.14776 seconds: | 20260508 07:58:13.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:13.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.256 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 07:58:13.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.256 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:58:13.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:13.257 | INFO | Uploaded 36GiB to GPU in 2.47583s (14.5406GiB/s) | 20260508 07:58:13.257 | INFO | Downloaded 34GiB to GPU in 2.43732s (13.9498GiB/s) | 20260508 07:58:13.354 | INFO | Starting a round of Baz IO | 20260508 07:58:16.773 | INFO | Populating RTMetrics report | 20260508 07:58:16.773 | INFO | RTMetrics report populated | 20260508 07:58:16.773 | INFO | Opening tmp RTMetrics json file | 20260508 07:58:16.781 | INFO | Serializing RTMetrics to json | 20260508 07:58:16.783 | INFO | Serialization done, swapping tmp file now... | 20260508 07:58:16.783 | INFO | RTMetrics done | 20260508 07:58:16.843 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.489 seconds | 20260508 07:58:16.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.52% | 20260508 07:58:16.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.318344% / 3.97364e-06% / 4.76837e-05% | 20260508 07:58:16.843 | INFO | Baz IO overall: | 20260508 07:58:16.843 | INFO | BytesWritten : 2856340445637 | 20260508 07:58:16.843 | INFO | NumEvents : 1375628462190 | 20260508 07:58:18.392 | INFO | Analyzing chunk frames = [6492160,6492672) | 20260508 07:58:18.392 | INFO | MemInfo: { | 20260508 07:58:18.392 | INFO | "gpu" : " 17 GB", | 20260508 07:58:18.392 | INFO | "huge" : "", | 20260508 07:58:18.392 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:18.392 | INFO | "malloc" : " 14 GB", | 20260508 07:58:18.392 | INFO | "pinned" : " 29 GB" | 20260508 07:58:18.392 | INFO | } | 20260508 07:58:18.392 | INFO | Perf report for the last 5.13571 seconds: | 20260508 07:58:18.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:18.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:18.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:18.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:18.392 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 07:58:18.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:18.392 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 07:58:18.392 | INFO | PRE_HQ : 65.5%, 1.6, 55.0ms, 0 | 20260508 07:58:18.392 | INFO | BAZWRITER : 67.3%, 2.6, 89.6ms, 0 | 20260508 07:58:18.392 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 07:58:18.392 | INFO | Uploaded 36GiB to GPU in 2.57465s (13.9825GiB/s) | 20260508 07:58:18.393 | INFO | Downloaded 38GiB to GPU in 2.7095s (14.0247GiB/s) | 20260508 07:58:23.523 | INFO | Analyzing chunk frames = [6492672,6493184) | 20260508 07:58:23.523 | INFO | MemInfo: { | 20260508 07:58:23.523 | INFO | "gpu" : " 17 GB", | 20260508 07:58:23.523 | INFO | "huge" : "", | 20260508 07:58:23.523 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:23.523 | INFO | "malloc" : " 14 GB", | 20260508 07:58:23.523 | INFO | "pinned" : " 29 GB" | 20260508 07:58:23.523 | INFO | } | 20260508 07:58:23.523 | INFO | Perf report for the last 5.13106 seconds: | 20260508 07:58:23.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:23.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.523 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 07:58:23.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.523 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 07:58:23.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:23.524 | INFO | Uploaded 36GiB to GPU in 2.4883s (14.4677GiB/s) | 20260508 07:58:23.524 | INFO | Downloaded 34GiB to GPU in 2.46227s (13.8084GiB/s) | 20260508 07:58:28.668 | INFO | Analyzing chunk frames = [6493184,6493696) | 20260508 07:58:28.668 | INFO | MemInfo: { | 20260508 07:58:28.668 | INFO | "gpu" : " 17 GB", | 20260508 07:58:28.668 | INFO | "huge" : "", | 20260508 07:58:28.668 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:28.668 | INFO | "malloc" : " 14 GB", | 20260508 07:58:28.668 | INFO | "pinned" : " 29 GB" | 20260508 07:58:28.668 | INFO | } | 20260508 07:58:28.668 | INFO | Perf report for the last 5.14474 seconds: | 20260508 07:58:28.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:28.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 07:58:28.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 07:58:28.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:28.668 | INFO | Uploaded 36GiB to GPU in 2.52723s (14.2449GiB/s) | 20260508 07:58:28.668 | INFO | Downloaded 34GiB to GPU in 2.44587s (13.901GiB/s) | 20260508 07:58:33.809 | INFO | Analyzing chunk frames = [6493696,6494208) | 20260508 07:58:33.809 | INFO | MemInfo: { | 20260508 07:58:33.809 | INFO | "gpu" : " 17 GB", | 20260508 07:58:33.809 | INFO | "huge" : "", | 20260508 07:58:33.809 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:33.809 | INFO | "malloc" : " 14 GB", | 20260508 07:58:33.809 | INFO | "pinned" : " 29 GB" | 20260508 07:58:33.809 | INFO | } | 20260508 07:58:33.809 | INFO | Perf report for the last 5.14123 seconds: | 20260508 07:58:33.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:33.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.809 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 07:58:33.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.809 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:58:33.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:33.810 | INFO | Uploaded 36GiB to GPU in 2.54192s (14.1625GiB/s) | 20260508 07:58:33.810 | INFO | Downloaded 34GiB to GPU in 2.44648s (13.8975GiB/s) | 20260508 07:58:38.957 | INFO | Analyzing chunk frames = [6494208,6494720) | 20260508 07:58:38.957 | INFO | MemInfo: { | 20260508 07:58:38.957 | INFO | "gpu" : " 17 GB", | 20260508 07:58:38.957 | INFO | "huge" : "", | 20260508 07:58:38.957 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:38.957 | INFO | "malloc" : " 14 GB", | 20260508 07:58:38.957 | INFO | "pinned" : " 29 GB" | 20260508 07:58:38.957 | INFO | } | 20260508 07:58:38.957 | INFO | Perf report for the last 5.14814 seconds: | 20260508 07:58:38.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:38.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.957 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 07:58:38.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.957 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 07:58:38.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:38.958 | INFO | Uploaded 36GiB to GPU in 2.32611s (15.4765GiB/s) | 20260508 07:58:38.958 | INFO | Downloaded 34GiB to GPU in 2.52169s (13.483GiB/s) | 20260508 07:58:44.085 | INFO | Analyzing chunk frames = [6494720,6495232) | 20260508 07:58:44.085 | INFO | MemInfo: { | 20260508 07:58:44.085 | INFO | "gpu" : " 17 GB", | 20260508 07:58:44.085 | INFO | "huge" : "", | 20260508 07:58:44.085 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:44.085 | INFO | "malloc" : " 14 GB", | 20260508 07:58:44.086 | INFO | "pinned" : " 29 GB" | 20260508 07:58:44.086 | INFO | } | 20260508 07:58:44.086 | INFO | Perf report for the last 5.12841 seconds: | 20260508 07:58:44.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:44.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 07:58:44.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 07:58:44.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:44.086 | INFO | Uploaded 36GiB to GPU in 2.34207s (15.371GiB/s) | 20260508 07:58:44.086 | INFO | Downloaded 34GiB to GPU in 2.45202s (13.8661GiB/s) | 20260508 07:58:49.226 | INFO | Analyzing chunk frames = [6495232,6495744) | 20260508 07:58:49.226 | INFO | MemInfo: { | 20260508 07:58:49.226 | INFO | "gpu" : " 17 GB", | 20260508 07:58:49.226 | INFO | "huge" : "", | 20260508 07:58:49.226 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:49.226 | INFO | "malloc" : " 14 GB", | 20260508 07:58:49.226 | INFO | "pinned" : " 29 GB" | 20260508 07:58:49.226 | INFO | } | 20260508 07:58:49.226 | INFO | Perf report for the last 5.14073 seconds: | 20260508 07:58:49.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:49.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.226 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 07:58:49.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.227 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:58:49.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:49.227 | INFO | Uploaded 36GiB to GPU in 2.37168s (15.1791GiB/s) | 20260508 07:58:49.227 | INFO | Downloaded 34GiB to GPU in 2.45332s (13.8588GiB/s) | 20260508 07:58:54.366 | INFO | Analyzing chunk frames = [6495744,6496256) | 20260508 07:58:54.366 | INFO | MemInfo: { | 20260508 07:58:54.366 | INFO | "gpu" : " 17 GB", | 20260508 07:58:54.366 | INFO | "huge" : "", | 20260508 07:58:54.366 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:54.366 | INFO | "malloc" : " 14 GB", | 20260508 07:58:54.366 | INFO | "pinned" : " 29 GB" | 20260508 07:58:54.367 | INFO | } | 20260508 07:58:54.367 | INFO | Perf report for the last 5.14014 seconds: | 20260508 07:58:54.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:54.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 07:58:54.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 07:58:54.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:54.367 | INFO | Uploaded 36GiB to GPU in 2.41718s (14.8934GiB/s) | 20260508 07:58:54.367 | INFO | Downloaded 34GiB to GPU in 2.45882s (13.8278GiB/s) | 20260508 07:58:54.367 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 07:58:54.367 | INFO | Total mem usage: 16.7 GiB | 20260508 07:58:54.367 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 07:58:54.368 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 07:58:54.368 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 07:58:54.368 | INFO | High Water Breakdown: | 20260508 07:58:54.368 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 07:58:54.368 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 07:58:54.368 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 07:58:54.368 | INFO | ------------------------------------------ | 20260508 07:58:54.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:58:54.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 07:58:54.368 | INFO | Total mem usage: 26.5 GiB | 20260508 07:58:54.368 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 07:58:54.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 07:58:54.368 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 07:58:54.368 | INFO | High Water Breakdown: | 20260508 07:58:54.368 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:58:54.368 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:58:54.368 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:58:54.368 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:58:54.368 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:58:54.368 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:58:54.368 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 07:58:54.368 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 07:58:54.368 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:58:54.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 07:58:54.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 07:58:54.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 07:58:54.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 07:58:54.368 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 07:58:54.368 | INFO | ------------------------------------------ | 20260508 07:58:54.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:58:54.368 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 07:58:54.368 | INFO | Total mem usage: 27.8 GiB | 20260508 07:58:54.368 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 07:58:54.368 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 07:58:54.368 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 07:58:54.368 | INFO | High Water Breakdown: | 20260508 07:58:54.368 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 07:58:54.368 | INFO | ------------------------------------------ | 20260508 07:58:54.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:58:54.368 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 07:58:54.368 | INFO | Total mem usage: 15.7 GiB | 20260508 07:58:54.368 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 07:58:54.368 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 07:58:54.368 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 07:58:54.368 | INFO | High Water Breakdown: | 20260508 07:58:54.368 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 07:58:54.368 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 07:58:54.368 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 07:58:54.368 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 07:58:54.368 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 07:58:54.368 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 07:58:54.368 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 07:58:54.368 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 07:58:54.368 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 07:58:54.368 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 07:58:54.368 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 07:58:54.368 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 07:58:54.368 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 07:58:54.368 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 07:58:54.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 07:58:54.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 07:58:54.368 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 07:58:54.368 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 07:58:54.368 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 07:58:54.368 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 07:58:54.368 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 07:58:54.368 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 07:58:54.368 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 07:58:54.368 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 07:58:54.368 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 07:58:54.368 | INFO | ------------------------------------------ | 20260508 07:58:54.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 07:58:54.368 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 07:58:54.368 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:58:54.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:58:54.368 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.39e-01 | 20260508 07:58:54.368 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.07e-01, 1.12e-02 / 6.55e-03 | 20260508 07:58:54.368 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.40e+00, 5.92e-01 / 2.10e-01 | 20260508 07:58:54.368 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.11e-01 | 20260508 07:58:54.368 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.49e+01, 3.60e+00 / 9.92e-01 | 20260508 07:58:54.368 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.82e+00 / 1.46e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.36e+00 / 2.51e+00, 4.08e-01 / 2.51e-01 | 20260508 07:58:54.368 | INFO | PerfReport: 3.10%, Metrics -- 1.01e+00 / 1.18e+00, 2.51e-01 / 1.29e-01 | 20260508 07:58:54.368 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.10e-01 / 3.39e-01, 4.93e-02 / 2.49e-02 | 20260508 07:58:54.368 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 07:58:54.368 | INFO | PerfReport: Gathered 1536 samples. | 20260508 07:58:54.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 07:58:54.368 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 24.23%, Upload -- 2.56e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 24.51%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 30.98%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 2.12%, LockUpload -- 2.24e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 2.65%, LockDownload -- 2.80e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 07:58:54.368 | INFO | PerfReport: 0.01%, Misc -- 1.53e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260508 07:58:54.464 | INFO | Starting a round of Baz IO | 20260508 07:58:57.881 | INFO | Populating RTMetrics report | 20260508 07:58:57.881 | INFO | RTMetrics report populated | 20260508 07:58:57.881 | INFO | Opening tmp RTMetrics json file | 20260508 07:58:57.889 | INFO | Serializing RTMetrics to json | 20260508 07:58:57.891 | INFO | Serialization done, swapping tmp file now... | 20260508 07:58:57.892 | INFO | RTMetrics done | 20260508 07:58:57.953 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.488 seconds | 20260508 07:58:57.953 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.52% | 20260508 07:58:57.953 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317069% / 3.97364e-06% / 3.97364e-05% | 20260508 07:58:57.953 | INFO | Baz IO overall: | 20260508 07:58:57.953 | INFO | BytesWritten : 2857967069304 | 20260508 07:58:57.953 | INFO | NumEvents : 1376400092947 | 20260508 07:58:59.504 | INFO | Analyzing chunk frames = [6496256,6496768) | 20260508 07:58:59.504 | INFO | MemInfo: { | 20260508 07:58:59.504 | INFO | "gpu" : " 17 GB", | 20260508 07:58:59.504 | INFO | "huge" : "", | 20260508 07:58:59.504 | INFO | "huge_shared" : " 29 GB", | 20260508 07:58:59.504 | INFO | "malloc" : " 14 GB", | 20260508 07:58:59.504 | INFO | "pinned" : " 29 GB" | 20260508 07:58:59.504 | INFO | } | 20260508 07:58:59.504 | INFO | Perf report for the last 5.13753 seconds: | 20260508 07:58:59.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:58:59.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:59.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:59.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:59.504 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 07:58:59.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:58:59.504 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 07:58:59.504 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 07:58:59.504 | INFO | BAZWRITER : 67.4%, 2.6, 89.8ms, 0 | 20260508 07:58:59.504 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 07:58:59.505 | INFO | Uploaded 36GiB to GPU in 2.53694s (14.1903GiB/s) | 20260508 07:58:59.505 | INFO | Downloaded 38GiB to GPU in 2.7134s (14.0046GiB/s) | 20260508 07:59:04.644 | INFO | Analyzing chunk frames = [6496768,6497280) | 20260508 07:59:04.644 | INFO | MemInfo: { | 20260508 07:59:04.644 | INFO | "gpu" : " 17 GB", | 20260508 07:59:04.644 | INFO | "huge" : "", | 20260508 07:59:04.644 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:04.644 | INFO | "malloc" : " 14 GB", | 20260508 07:59:04.644 | INFO | "pinned" : " 29 GB" | 20260508 07:59:04.644 | INFO | } | 20260508 07:59:04.644 | INFO | Perf report for the last 5.13953 seconds: | 20260508 07:59:04.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:04.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 07:59:04.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 07:59:04.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:04.644 | INFO | Uploaded 36GiB to GPU in 2.48603s (14.4809GiB/s) | 20260508 07:59:04.644 | INFO | Downloaded 34GiB to GPU in 2.43979s (13.9356GiB/s) | 20260508 07:59:09.781 | INFO | Analyzing chunk frames = [6497280,6497792) | 20260508 07:59:09.781 | INFO | MemInfo: { | 20260508 07:59:09.781 | INFO | "gpu" : " 17 GB", | 20260508 07:59:09.781 | INFO | "huge" : "", | 20260508 07:59:09.781 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:09.781 | INFO | "malloc" : " 14 GB", | 20260508 07:59:09.781 | INFO | "pinned" : " 29 GB" | 20260508 07:59:09.781 | INFO | } | 20260508 07:59:09.781 | INFO | Perf report for the last 5.13694 seconds: | 20260508 07:59:09.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:09.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 07:59:09.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:59:09.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:09.781 | INFO | Uploaded 36GiB to GPU in 2.56014s (14.0617GiB/s) | 20260508 07:59:09.781 | INFO | Downloaded 34GiB to GPU in 2.43979s (13.9356GiB/s) | 20260508 07:59:14.922 | INFO | Analyzing chunk frames = [6497792,6498304) | 20260508 07:59:14.922 | INFO | MemInfo: { | 20260508 07:59:14.922 | INFO | "gpu" : " 17 GB", | 20260508 07:59:14.922 | INFO | "huge" : "", | 20260508 07:59:14.922 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:14.922 | INFO | "malloc" : " 14 GB", | 20260508 07:59:14.922 | INFO | "pinned" : " 29 GB" | 20260508 07:59:14.922 | INFO | } | 20260508 07:59:14.922 | INFO | Perf report for the last 5.14179 seconds: | 20260508 07:59:14.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:14.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 07:59:14.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 07:59:14.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:14.923 | INFO | Uploaded 36GiB to GPU in 2.56775s (14.0201GiB/s) | 20260508 07:59:14.923 | INFO | Downloaded 34GiB to GPU in 2.43757s (13.9483GiB/s) | 20260508 07:59:20.061 | INFO | Analyzing chunk frames = [6498304,6498816) | 20260508 07:59:20.061 | INFO | MemInfo: { | 20260508 07:59:20.061 | INFO | "gpu" : " 17 GB", | 20260508 07:59:20.061 | INFO | "huge" : "", | 20260508 07:59:20.061 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:20.061 | INFO | "malloc" : " 14 GB", | 20260508 07:59:20.061 | INFO | "pinned" : " 29 GB" | 20260508 07:59:20.061 | INFO | } | 20260508 07:59:20.061 | INFO | Perf report for the last 5.13897 seconds: | 20260508 07:59:20.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:20.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.061 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 07:59:20.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.061 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 07:59:20.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:20.062 | INFO | Uploaded 36GiB to GPU in 2.25666s (15.9528GiB/s) | 20260508 07:59:20.062 | INFO | Downloaded 34GiB to GPU in 2.49686s (13.6171GiB/s) | 20260508 07:59:25.199 | INFO | Analyzing chunk frames = [6498816,6499328) | 20260508 07:59:25.199 | INFO | MemInfo: { | 20260508 07:59:25.199 | INFO | "gpu" : " 17 GB", | 20260508 07:59:25.199 | INFO | "huge" : "", | 20260508 07:59:25.199 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:25.199 | INFO | "malloc" : " 14 GB", | 20260508 07:59:25.199 | INFO | "pinned" : " 29 GB" | 20260508 07:59:25.199 | INFO | } | 20260508 07:59:25.199 | INFO | Perf report for the last 5.13717 seconds: | 20260508 07:59:25.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:25.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 07:59:25.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 07:59:25.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:25.199 | INFO | Uploaded 36GiB to GPU in 2.40816s (14.9492GiB/s) | 20260508 07:59:25.199 | INFO | Downloaded 34GiB to GPU in 2.42358s (14.0288GiB/s) | 20260508 07:59:30.338 | INFO | Analyzing chunk frames = [6499328,6499840) | 20260508 07:59:30.338 | INFO | MemInfo: { | 20260508 07:59:30.338 | INFO | "gpu" : " 17 GB", | 20260508 07:59:30.338 | INFO | "huge" : "", | 20260508 07:59:30.338 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:30.338 | INFO | "malloc" : " 14 GB", | 20260508 07:59:30.338 | INFO | "pinned" : " 29 GB" | 20260508 07:59:30.338 | INFO | } | 20260508 07:59:30.339 | INFO | Perf report for the last 5.13992 seconds: | 20260508 07:59:30.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:30.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 07:59:30.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:59:30.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:30.339 | INFO | Uploaded 36GiB to GPU in 2.49888s (14.4065GiB/s) | 20260508 07:59:30.339 | INFO | Downloaded 34GiB to GPU in 2.43475s (13.9645GiB/s) | 20260508 07:59:35.474 | INFO | Analyzing chunk frames = [6499840,6500352) | 20260508 07:59:35.474 | INFO | MemInfo: { | 20260508 07:59:35.474 | INFO | "gpu" : " 17 GB", | 20260508 07:59:35.474 | INFO | "huge" : "", | 20260508 07:59:35.474 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:35.474 | INFO | "malloc" : " 14 GB", | 20260508 07:59:35.474 | INFO | "pinned" : " 29 GB" | 20260508 07:59:35.474 | INFO | } | 20260508 07:59:35.474 | INFO | Perf report for the last 5.1357 seconds: | 20260508 07:59:35.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:35.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 07:59:35.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 07:59:35.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:35.475 | INFO | Uploaded 36GiB to GPU in 2.43646s (14.7755GiB/s) | 20260508 07:59:35.475 | INFO | Downloaded 34GiB to GPU in 2.43655s (13.9541GiB/s) | 20260508 07:59:35.572 | INFO | Starting a round of Baz IO | 20260508 07:59:39.005 | INFO | Populating RTMetrics report | 20260508 07:59:39.005 | INFO | RTMetrics report populated | 20260508 07:59:39.005 | INFO | Opening tmp RTMetrics json file | 20260508 07:59:39.006 | INFO | Serializing RTMetrics to json | 20260508 07:59:39.007 | INFO | Serialization done, swapping tmp file now... | 20260508 07:59:39.007 | INFO | RTMetrics done | 20260508 07:59:39.075 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.503 seconds | 20260508 07:59:39.076 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.52% | 20260508 07:59:39.076 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.316262% / 0% / 4.76837e-05% | 20260508 07:59:39.076 | INFO | Baz IO overall: | 20260508 07:59:39.076 | INFO | BytesWritten : 2859593811706 | 20260508 07:59:39.076 | INFO | NumEvents : 1377171804571 | 20260508 07:59:40.618 | INFO | Analyzing chunk frames = [6500352,6500864) | 20260508 07:59:40.618 | INFO | MemInfo: { | 20260508 07:59:40.618 | INFO | "gpu" : " 17 GB", | 20260508 07:59:40.618 | INFO | "huge" : "", | 20260508 07:59:40.618 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:40.618 | INFO | "malloc" : " 14 GB", | 20260508 07:59:40.618 | INFO | "pinned" : " 29 GB" | 20260508 07:59:40.618 | INFO | } | 20260508 07:59:40.618 | INFO | Perf report for the last 5.14402 seconds: | 20260508 07:59:40.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:40.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:40.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:40.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:40.618 | INFO | ANALYSIS : 68.3%, 3.0, 106.0ms, 0 | 20260508 07:59:40.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:40.618 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 07:59:40.618 | INFO | PRE_HQ : 66.4%, 1.7, 60.4ms, 0 | 20260508 07:59:40.619 | INFO | BAZWRITER : 67.2%, 2.8, 95.2ms, 0 | 20260508 07:59:40.619 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 07:59:40.619 | INFO | Uploaded 36GiB to GPU in 2.5841s (13.9314GiB/s) | 20260508 07:59:40.619 | INFO | Downloaded 38GiB to GPU in 2.70314s (14.0577GiB/s) | 20260508 07:59:45.763 | INFO | Analyzing chunk frames = [6500864,6501376) | 20260508 07:59:45.763 | INFO | MemInfo: { | 20260508 07:59:45.763 | INFO | "gpu" : " 17 GB", | 20260508 07:59:45.763 | INFO | "huge" : "", | 20260508 07:59:45.763 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:45.763 | INFO | "malloc" : " 14 GB", | 20260508 07:59:45.763 | INFO | "pinned" : " 29 GB" | 20260508 07:59:45.763 | INFO | } | 20260508 07:59:45.764 | INFO | Perf report for the last 5.14532 seconds: | 20260508 07:59:45.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:45.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 07:59:45.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 07:59:45.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:45.764 | INFO | Uploaded 36GiB to GPU in 2.50467s (14.3732GiB/s) | 20260508 07:59:45.764 | INFO | Downloaded 34GiB to GPU in 2.43056s (13.9885GiB/s) | 20260508 07:59:50.898 | INFO | Analyzing chunk frames = [6501376,6501888) | 20260508 07:59:50.898 | INFO | MemInfo: { | 20260508 07:59:50.898 | INFO | "gpu" : " 17 GB", | 20260508 07:59:50.898 | INFO | "huge" : "", | 20260508 07:59:50.898 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:50.898 | INFO | "malloc" : " 14 GB", | 20260508 07:59:50.898 | INFO | "pinned" : " 29 GB" | 20260508 07:59:50.898 | INFO | } | 20260508 07:59:50.898 | INFO | Perf report for the last 5.1345 seconds: | 20260508 07:59:50.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:50.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.898 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260508 07:59:50.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.898 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 07:59:50.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:50.899 | INFO | Uploaded 36GiB to GPU in 2.50746s (14.3572GiB/s) | 20260508 07:59:50.899 | INFO | Downloaded 34GiB to GPU in 2.43794s (13.9462GiB/s) | 20260508 07:59:56.032 | INFO | Analyzing chunk frames = [6501888,6502400) | 20260508 07:59:56.032 | INFO | MemInfo: { | 20260508 07:59:56.032 | INFO | "gpu" : " 17 GB", | 20260508 07:59:56.032 | INFO | "huge" : "", | 20260508 07:59:56.032 | INFO | "huge_shared" : " 29 GB", | 20260508 07:59:56.032 | INFO | "malloc" : " 14 GB", | 20260508 07:59:56.032 | INFO | "pinned" : " 29 GB" | 20260508 07:59:56.032 | INFO | } | 20260508 07:59:56.032 | INFO | Perf report for the last 5.13401 seconds: | 20260508 07:59:56.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 07:59:56.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | ANALYSIS : 67.0%, 2.9, 103.5ms, 0 | 20260508 07:59:56.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 07:59:56.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 07:59:56.032 | INFO | Uploaded 36GiB to GPU in 2.58348s (13.9347GiB/s) | 20260508 07:59:56.032 | INFO | Downloaded 34GiB to GPU in 2.41565s (14.0749GiB/s) | 20260508 08:00:01.175 | INFO | Analyzing chunk frames = [6502400,6502912) | 20260508 08:00:01.175 | INFO | MemInfo: { | 20260508 08:00:01.175 | INFO | "gpu" : " 17 GB", | 20260508 08:00:01.175 | INFO | "huge" : "", | 20260508 08:00:01.175 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:01.175 | INFO | "malloc" : " 14 GB", | 20260508 08:00:01.175 | INFO | "pinned" : " 29 GB" | 20260508 08:00:01.175 | INFO | } | 20260508 08:00:01.175 | INFO | Perf report for the last 5.14317 seconds: | 20260508 08:00:01.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:01.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 08:00:01.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:00:01.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:01.175 | INFO | Uploaded 36GiB to GPU in 2.30022s (15.6506GiB/s) | 20260508 08:00:01.175 | INFO | Downloaded 34GiB to GPU in 2.50047s (13.5974GiB/s) | 20260508 08:00:06.319 | INFO | Analyzing chunk frames = [6502912,6503424) | 20260508 08:00:06.319 | INFO | MemInfo: { | 20260508 08:00:06.319 | INFO | "gpu" : " 17 GB", | 20260508 08:00:06.320 | INFO | "huge" : "", | 20260508 08:00:06.320 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:06.320 | INFO | "malloc" : " 14 GB", | 20260508 08:00:06.320 | INFO | "pinned" : " 29 GB" | 20260508 08:00:06.320 | INFO | } | 20260508 08:00:06.320 | INFO | Perf report for the last 5.14431 seconds: | 20260508 08:00:06.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:06.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:00:06.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:00:06.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:06.320 | INFO | Uploaded 36GiB to GPU in 2.37899s (15.1324GiB/s) | 20260508 08:00:06.320 | INFO | Downloaded 34GiB to GPU in 2.43708s (13.9511GiB/s) | 20260508 08:00:11.460 | INFO | Analyzing chunk frames = [6503424,6503936) | 20260508 08:00:11.460 | INFO | MemInfo: { | 20260508 08:00:11.460 | INFO | "gpu" : " 17 GB", | 20260508 08:00:11.460 | INFO | "huge" : "", | 20260508 08:00:11.460 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:11.460 | INFO | "malloc" : " 14 GB", | 20260508 08:00:11.460 | INFO | "pinned" : " 29 GB" | 20260508 08:00:11.461 | INFO | } | 20260508 08:00:11.461 | INFO | Perf report for the last 5.14093 seconds: | 20260508 08:00:11.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:11.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 08:00:11.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 08:00:11.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:11.461 | INFO | Uploaded 36GiB to GPU in 2.40392s (14.9756GiB/s) | 20260508 08:00:11.461 | INFO | Downloaded 34GiB to GPU in 2.43508s (13.9626GiB/s) | 20260508 08:00:16.601 | INFO | Analyzing chunk frames = [6503936,6504448) | 20260508 08:00:16.601 | INFO | MemInfo: { | 20260508 08:00:16.601 | INFO | "gpu" : " 17 GB", | 20260508 08:00:16.601 | INFO | "huge" : "", | 20260508 08:00:16.601 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:16.601 | INFO | "malloc" : " 14 GB", | 20260508 08:00:16.601 | INFO | "pinned" : " 29 GB" | 20260508 08:00:16.601 | INFO | } | 20260508 08:00:16.601 | INFO | Perf report for the last 5.14018 seconds: | 20260508 08:00:16.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:16.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 08:00:16.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 08:00:16.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:16.601 | INFO | Uploaded 36GiB to GPU in 2.43208s (14.8022GiB/s) | 20260508 08:00:16.601 | INFO | Downloaded 34GiB to GPU in 2.43229s (13.9786GiB/s) | 20260508 08:00:16.601 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:00:16.601 | INFO | Total mem usage: 16.7 GiB | 20260508 08:00:16.601 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:00:16.601 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 08:00:16.601 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:00:16.601 | INFO | High Water Breakdown: | 20260508 08:00:16.601 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:00:16.601 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:00:16.601 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:00:16.601 | INFO | ------------------------------------------ | 20260508 08:00:16.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:00:16.601 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:00:16.601 | INFO | Total mem usage: 26.5 GiB | 20260508 08:00:16.601 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:00:16.601 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:00:16.601 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:00:16.601 | INFO | High Water Breakdown: | 20260508 08:00:16.601 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:00:16.601 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:00:16.601 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:00:16.601 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:00:16.601 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:00:16.601 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:00:16.601 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:00:16.601 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:00:16.601 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:00:16.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:00:16.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:00:16.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:00:16.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:00:16.601 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:00:16.601 | INFO | ------------------------------------------ | 20260508 08:00:16.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:00:16.602 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:00:16.602 | INFO | Total mem usage: 27.8 GiB | 20260508 08:00:16.602 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:00:16.602 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:00:16.602 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:00:16.602 | INFO | High Water Breakdown: | 20260508 08:00:16.602 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:00:16.602 | INFO | ------------------------------------------ | 20260508 08:00:16.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:00:16.602 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:00:16.602 | INFO | Total mem usage: 15.7 GiB | 20260508 08:00:16.602 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:00:16.602 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:00:16.602 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:00:16.602 | INFO | High Water Breakdown: | 20260508 08:00:16.602 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:00:16.602 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:00:16.602 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:00:16.602 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:00:16.602 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:00:16.602 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:00:16.602 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:00:16.602 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:00:16.602 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:00:16.602 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:00:16.602 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:00:16.602 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:00:16.602 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:00:16.602 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:00:16.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:00:16.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:00:16.602 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:00:16.602 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:00:16.602 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:00:16.602 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:00:16.602 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:00:16.602 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:00:16.602 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:00:16.602 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:00:16.602 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:00:16.602 | INFO | ------------------------------------------ | 20260508 08:00:16.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:00:16.602 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:00:16.602 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:00:16.602 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:00:16.602 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.40e+01 / 5.93e-01 | 20260508 08:00:16.602 | INFO | PerfReport: 0.29%, AggregateStats -- 9.66e-02 / 1.06e-01, 1.11e-02 / 5.50e-03 | 20260508 08:00:16.602 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.45e+00, 5.97e-01 / 2.41e-01 | 20260508 08:00:16.602 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.12e-01 / 1.07e-01 | 20260508 08:00:16.602 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.55e+01, 3.80e+00 / 1.12e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.88e+00 / 1.35e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.56e+00, 4.11e-01 / 2.66e-01 | 20260508 08:00:16.602 | INFO | PerfReport: 3.09%, Metrics -- 1.01e+00 / 1.18e+00, 2.52e-01 / 1.25e-01 | 20260508 08:00:16.602 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.43e-01, 4.98e-02 / 2.40e-02 | 20260508 08:00:16.602 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:00:16.602 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:00:16.602 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:00:16.602 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 24.29%, Upload -- 2.57e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 24.40%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 31.00%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 2.27%, LockUpload -- 2.40e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 2.48%, LockDownload -- 2.63e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 15.54%, LockCompute -- 1.64e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:00:16.602 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 08:00:16.697 | INFO | Starting a round of Baz IO | 20260508 08:00:20.106 | INFO | Populating RTMetrics report | 20260508 08:00:20.107 | INFO | RTMetrics report populated | 20260508 08:00:20.107 | INFO | Opening tmp RTMetrics json file | 20260508 08:00:20.117 | INFO | Serializing RTMetrics to json | 20260508 08:00:20.118 | INFO | Serialization done, swapping tmp file now... | 20260508 08:00:20.118 | INFO | RTMetrics done | 20260508 08:00:20.181 | INFO | Finished a round of Baz IO: Wrote 1.515 GiB in 3.483 seconds | 20260508 08:00:20.181 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.52% | 20260508 08:00:20.181 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.316536% / 0% / 4.37101e-05% | 20260508 08:00:20.181 | INFO | Baz IO overall: | 20260508 08:00:20.181 | INFO | BytesWritten : 2861220355539 | 20260508 08:00:20.181 | INFO | NumEvents : 1377943287012 | 20260508 08:00:21.733 | INFO | Analyzing chunk frames = [6504448,6504960) | 20260508 08:00:21.733 | INFO | MemInfo: { | 20260508 08:00:21.733 | INFO | "gpu" : " 17 GB", | 20260508 08:00:21.733 | INFO | "huge" : "", | 20260508 08:00:21.733 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:21.733 | INFO | "malloc" : " 14 GB", | 20260508 08:00:21.733 | INFO | "pinned" : " 29 GB" | 20260508 08:00:21.733 | INFO | } | 20260508 08:00:21.733 | INFO | Perf report for the last 5.13243 seconds: | 20260508 08:00:21.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:21.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:21.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:21.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:21.733 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:00:21.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:21.733 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:00:21.733 | INFO | PRE_HQ : 65.8%, 1.6, 55.1ms, 0 | 20260508 08:00:21.733 | INFO | BAZWRITER : 67.2%, 2.6, 90.4ms, 0 | 20260508 08:00:21.733 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:00:21.734 | INFO | Uploaded 36GiB to GPU in 2.576s (13.9752GiB/s) | 20260508 08:00:21.734 | INFO | Downloaded 38GiB to GPU in 2.71471s (13.9978GiB/s) | 20260508 08:00:26.872 | INFO | Analyzing chunk frames = [6504960,6505472) | 20260508 08:00:26.872 | INFO | MemInfo: { | 20260508 08:00:26.872 | INFO | "gpu" : " 17 GB", | 20260508 08:00:26.872 | INFO | "huge" : "", | 20260508 08:00:26.872 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:26.872 | INFO | "malloc" : " 14 GB", | 20260508 08:00:26.872 | INFO | "pinned" : " 29 GB" | 20260508 08:00:26.872 | INFO | } | 20260508 08:00:26.872 | INFO | Perf report for the last 5.13856 seconds: | 20260508 08:00:26.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:26.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 08:00:26.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:00:26.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:26.872 | INFO | Uploaded 36GiB to GPU in 2.53984s (14.1741GiB/s) | 20260508 08:00:26.872 | INFO | Downloaded 34GiB to GPU in 2.45398s (13.8551GiB/s) | 20260508 08:00:32.016 | INFO | Analyzing chunk frames = [6505472,6505984) | 20260508 08:00:32.016 | INFO | MemInfo: { | 20260508 08:00:32.016 | INFO | "gpu" : " 17 GB", | 20260508 08:00:32.016 | INFO | "huge" : "", | 20260508 08:00:32.016 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:32.016 | INFO | "malloc" : " 14 GB", | 20260508 08:00:32.016 | INFO | "pinned" : " 29 GB" | 20260508 08:00:32.016 | INFO | } | 20260508 08:00:32.016 | INFO | Perf report for the last 5.14429 seconds: | 20260508 08:00:32.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:32.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 08:00:32.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:00:32.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:32.016 | INFO | Uploaded 36GiB to GPU in 2.57601s (13.9751GiB/s) | 20260508 08:00:32.016 | INFO | Downloaded 34GiB to GPU in 2.43808s (13.9454GiB/s) | 20260508 08:00:37.156 | INFO | Analyzing chunk frames = [6505984,6506496) | 20260508 08:00:37.156 | INFO | MemInfo: { | 20260508 08:00:37.156 | INFO | "gpu" : " 17 GB", | 20260508 08:00:37.156 | INFO | "huge" : "", | 20260508 08:00:37.156 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:37.156 | INFO | "malloc" : " 14 GB", | 20260508 08:00:37.156 | INFO | "pinned" : " 29 GB" | 20260508 08:00:37.156 | INFO | } | 20260508 08:00:37.156 | INFO | Perf report for the last 5.14004 seconds: | 20260508 08:00:37.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:37.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | ANALYSIS : 66.8%, 3.0, 103.3ms, 0 | 20260508 08:00:37.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:00:37.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:37.156 | INFO | Uploaded 36GiB to GPU in 2.56821s (14.0175GiB/s) | 20260508 08:00:37.157 | INFO | Downloaded 34GiB to GPU in 2.42956s (13.9943GiB/s) | 20260508 08:00:42.286 | INFO | Analyzing chunk frames = [6506496,6507008) | 20260508 08:00:42.286 | INFO | MemInfo: { | 20260508 08:00:42.286 | INFO | "gpu" : " 17 GB", | 20260508 08:00:42.286 | INFO | "huge" : "", | 20260508 08:00:42.286 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:42.286 | INFO | "malloc" : " 14 GB", | 20260508 08:00:42.286 | INFO | "pinned" : " 29 GB" | 20260508 08:00:42.286 | INFO | } | 20260508 08:00:42.286 | INFO | Perf report for the last 5.13019 seconds: | 20260508 08:00:42.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:42.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.286 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 08:00:42.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.286 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:00:42.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:42.287 | INFO | Uploaded 36GiB to GPU in 2.29815s (15.6648GiB/s) | 20260508 08:00:42.287 | INFO | Downloaded 34GiB to GPU in 2.5006s (13.5967GiB/s) | 20260508 08:00:47.429 | INFO | Analyzing chunk frames = [6507008,6507520) | 20260508 08:00:47.429 | INFO | MemInfo: { | 20260508 08:00:47.429 | INFO | "gpu" : " 17 GB", | 20260508 08:00:47.429 | INFO | "huge" : "", | 20260508 08:00:47.429 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:47.429 | INFO | "malloc" : " 14 GB", | 20260508 08:00:47.429 | INFO | "pinned" : " 29 GB" | 20260508 08:00:47.429 | INFO | } | 20260508 08:00:47.429 | INFO | Perf report for the last 5.14262 seconds: | 20260508 08:00:47.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:47.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:00:47.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 08:00:47.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:47.429 | INFO | Uploaded 36GiB to GPU in 2.49441s (14.4323GiB/s) | 20260508 08:00:47.430 | INFO | Downloaded 34GiB to GPU in 2.44682s (13.8956GiB/s) | 20260508 08:00:52.570 | INFO | Analyzing chunk frames = [6507520,6508032) | 20260508 08:00:52.570 | INFO | MemInfo: { | 20260508 08:00:52.570 | INFO | "gpu" : " 17 GB", | 20260508 08:00:52.570 | INFO | "huge" : "", | 20260508 08:00:52.570 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:52.570 | INFO | "malloc" : " 14 GB", | 20260508 08:00:52.570 | INFO | "pinned" : " 29 GB" | 20260508 08:00:52.570 | INFO | } | 20260508 08:00:52.570 | INFO | Perf report for the last 5.14066 seconds: | 20260508 08:00:52.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:52.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:00:52.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 08:00:52.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:52.570 | INFO | Uploaded 36GiB to GPU in 2.45419s (14.6688GiB/s) | 20260508 08:00:52.570 | INFO | Downloaded 34GiB to GPU in 2.42486s (14.0214GiB/s) | 20260508 08:00:57.707 | INFO | Analyzing chunk frames = [6508032,6508544) | 20260508 08:00:57.707 | INFO | MemInfo: { | 20260508 08:00:57.707 | INFO | "gpu" : " 17 GB", | 20260508 08:00:57.707 | INFO | "huge" : "", | 20260508 08:00:57.707 | INFO | "huge_shared" : " 29 GB", | 20260508 08:00:57.708 | INFO | "malloc" : " 14 GB", | 20260508 08:00:57.708 | INFO | "pinned" : " 29 GB" | 20260508 08:00:57.708 | INFO | } | 20260508 08:00:57.708 | INFO | Perf report for the last 5.13793 seconds: | 20260508 08:00:57.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:00:57.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:00:57.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:00:57.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:00:57.708 | INFO | Uploaded 36GiB to GPU in 2.51171s (14.3329GiB/s) | 20260508 08:00:57.708 | INFO | Downloaded 34GiB to GPU in 2.44188s (13.9237GiB/s) | 20260508 08:00:57.805 | INFO | Starting a round of Baz IO | 20260508 08:01:01.223 | INFO | Populating RTMetrics report | 20260508 08:01:01.223 | INFO | RTMetrics report populated | 20260508 08:01:01.223 | INFO | Opening tmp RTMetrics json file | 20260508 08:01:01.234 | INFO | Serializing RTMetrics to json | 20260508 08:01:01.236 | INFO | Serialization done, swapping tmp file now... | 20260508 08:01:01.236 | INFO | RTMetrics done | 20260508 08:01:01.301 | INFO | Finished a round of Baz IO: Wrote 1.514 GiB in 3.495 seconds | 20260508 08:01:01.301 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.52% | 20260508 08:01:01.301 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.315674% / 0% / 5.96046e-05% | 20260508 08:01:01.301 | INFO | Baz IO overall: | 20260508 08:01:01.301 | INFO | BytesWritten : 2862846516368 | 20260508 08:01:01.301 | INFO | NumEvents : 1378714675187 | 20260508 08:01:02.844 | INFO | Analyzing chunk frames = [6508544,6509056) | 20260508 08:01:02.844 | INFO | MemInfo: { | 20260508 08:01:02.844 | INFO | "gpu" : " 17 GB", | 20260508 08:01:02.844 | INFO | "huge" : "", | 20260508 08:01:02.845 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:02.845 | INFO | "malloc" : " 14 GB", | 20260508 08:01:02.845 | INFO | "pinned" : " 29 GB" | 20260508 08:01:02.845 | INFO | } | 20260508 08:01:02.845 | INFO | Perf report for the last 5.13701 seconds: | 20260508 08:01:02.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:02.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:02.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:02.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:02.845 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 08:01:02.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:02.845 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:01:02.845 | INFO | PRE_HQ : 65.8%, 1.6, 54.9ms, 0 | 20260508 08:01:02.845 | INFO | BAZWRITER : 67.5%, 2.6, 90.1ms, 0 | 20260508 08:01:02.845 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:01:02.845 | INFO | Uploaded 36GiB to GPU in 2.59873s (13.8529GiB/s) | 20260508 08:01:02.845 | INFO | Downloaded 38GiB to GPU in 2.72738s (13.9328GiB/s) | 20260508 08:01:07.986 | INFO | Analyzing chunk frames = [6509056,6509568) | 20260508 08:01:07.986 | INFO | MemInfo: { | 20260508 08:01:07.986 | INFO | "gpu" : " 17 GB", | 20260508 08:01:07.986 | INFO | "huge" : "", | 20260508 08:01:07.986 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:07.986 | INFO | "malloc" : " 14 GB", | 20260508 08:01:07.986 | INFO | "pinned" : " 29 GB" | 20260508 08:01:07.986 | INFO | } | 20260508 08:01:07.986 | INFO | Perf report for the last 5.14171 seconds: | 20260508 08:01:07.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:07.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.986 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 08:01:07.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.986 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:01:07.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:07.987 | INFO | Uploaded 36GiB to GPU in 2.51484s (14.315GiB/s) | 20260508 08:01:07.987 | INFO | Downloaded 34GiB to GPU in 2.42666s (14.011GiB/s) | 20260508 08:01:13.131 | INFO | Analyzing chunk frames = [6509568,6510080) | 20260508 08:01:13.131 | INFO | MemInfo: { | 20260508 08:01:13.131 | INFO | "gpu" : " 17 GB", | 20260508 08:01:13.131 | INFO | "huge" : "", | 20260508 08:01:13.131 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:13.131 | INFO | "malloc" : " 14 GB", | 20260508 08:01:13.131 | INFO | "pinned" : " 29 GB" | 20260508 08:01:13.131 | INFO | } | 20260508 08:01:13.131 | INFO | Perf report for the last 5.14512 seconds: | 20260508 08:01:13.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:13.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 08:01:13.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:01:13.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:13.132 | INFO | Uploaded 36GiB to GPU in 2.57715s (13.9689GiB/s) | 20260508 08:01:13.132 | INFO | Downloaded 34GiB to GPU in 2.43404s (13.9686GiB/s) | 20260508 08:01:18.273 | INFO | Analyzing chunk frames = [6510080,6510592) | 20260508 08:01:18.273 | INFO | MemInfo: { | 20260508 08:01:18.273 | INFO | "gpu" : " 17 GB", | 20260508 08:01:18.273 | INFO | "huge" : "", | 20260508 08:01:18.273 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:18.273 | INFO | "malloc" : " 14 GB", | 20260508 08:01:18.273 | INFO | "pinned" : " 29 GB" | 20260508 08:01:18.273 | INFO | } | 20260508 08:01:18.273 | INFO | Perf report for the last 5.14135 seconds: | 20260508 08:01:18.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:18.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 08:01:18.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:01:18.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:18.273 | INFO | Uploaded 36GiB to GPU in 2.5909s (13.8948GiB/s) | 20260508 08:01:18.273 | INFO | Downloaded 34GiB to GPU in 2.43304s (13.9743GiB/s) | 20260508 08:01:23.408 | INFO | Analyzing chunk frames = [6510592,6511104) | 20260508 08:01:23.408 | INFO | MemInfo: { | 20260508 08:01:23.408 | INFO | "gpu" : " 17 GB", | 20260508 08:01:23.408 | INFO | "huge" : "", | 20260508 08:01:23.408 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:23.408 | INFO | "malloc" : " 14 GB", | 20260508 08:01:23.408 | INFO | "pinned" : " 29 GB" | 20260508 08:01:23.408 | INFO | } | 20260508 08:01:23.408 | INFO | Perf report for the last 5.13557 seconds: | 20260508 08:01:23.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:23.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.409 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 08:01:23.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.409 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:01:23.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:23.409 | INFO | Uploaded 36GiB to GPU in 2.307s (15.6047GiB/s) | 20260508 08:01:23.409 | INFO | Downloaded 34GiB to GPU in 2.49416s (13.6318GiB/s) | 20260508 08:01:28.540 | INFO | Analyzing chunk frames = [6511104,6511616) | 20260508 08:01:28.540 | INFO | MemInfo: { | 20260508 08:01:28.540 | INFO | "gpu" : " 17 GB", | 20260508 08:01:28.540 | INFO | "huge" : "", | 20260508 08:01:28.540 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:28.540 | INFO | "malloc" : " 14 GB", | 20260508 08:01:28.540 | INFO | "pinned" : " 29 GB" | 20260508 08:01:28.540 | INFO | } | 20260508 08:01:28.541 | INFO | Perf report for the last 5.1322 seconds: | 20260508 08:01:28.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:28.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.541 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:01:28.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.541 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260508 08:01:28.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:28.542 | INFO | Uploaded 36GiB to GPU in 2.40103s (14.9936GiB/s) | 20260508 08:01:28.542 | INFO | Downloaded 34GiB to GPU in 2.43601s (13.9573GiB/s) | 20260508 08:01:33.684 | INFO | Analyzing chunk frames = [6511616,6512128) | 20260508 08:01:33.684 | INFO | MemInfo: { | 20260508 08:01:33.684 | INFO | "gpu" : " 17 GB", | 20260508 08:01:33.684 | INFO | "huge" : "", | 20260508 08:01:33.684 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:33.684 | INFO | "malloc" : " 14 GB", | 20260508 08:01:33.684 | INFO | "pinned" : " 29 GB" | 20260508 08:01:33.684 | INFO | } | 20260508 08:01:33.684 | INFO | Perf report for the last 5.14357 seconds: | 20260508 08:01:33.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:33.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.684 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 08:01:33.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.684 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:01:33.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:33.685 | INFO | Uploaded 36GiB to GPU in 2.47265s (14.5593GiB/s) | 20260508 08:01:33.685 | INFO | Downloaded 34GiB to GPU in 2.42412s (14.0257GiB/s) | 20260508 08:01:38.820 | INFO | Analyzing chunk frames = [6512128,6512640) | 20260508 08:01:38.820 | INFO | MemInfo: { | 20260508 08:01:38.820 | INFO | "gpu" : " 17 GB", | 20260508 08:01:38.820 | INFO | "huge" : "", | 20260508 08:01:38.820 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:38.820 | INFO | "malloc" : " 14 GB", | 20260508 08:01:38.820 | INFO | "pinned" : " 29 GB" | 20260508 08:01:38.820 | INFO | } | 20260508 08:01:38.820 | INFO | Perf report for the last 5.13582 seconds: | 20260508 08:01:38.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:38.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.820 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:01:38.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.820 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:01:38.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:38.821 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:01:38.821 | INFO | Total mem usage: 16.7 GiB | 20260508 08:01:38.821 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:01:38.821 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:01:38.821 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:01:38.821 | INFO | High Water Breakdown: | 20260508 08:01:38.821 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:01:38.821 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:01:38.821 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:01:38.821 | INFO | ------------------------------------------ | 20260508 08:01:38.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:01:38.821 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:01:38.821 | INFO | Total mem usage: 26.5 GiB | 20260508 08:01:38.821 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:01:38.821 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:01:38.821 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:01:38.821 | INFO | High Water Breakdown: | 20260508 08:01:38.821 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:01:38.821 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:01:38.821 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:01:38.821 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:01:38.821 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:01:38.821 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:01:38.821 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:01:38.821 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:01:38.821 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:01:38.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:01:38.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:01:38.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:01:38.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:01:38.821 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:01:38.821 | INFO | ------------------------------------------ | 20260508 08:01:38.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:01:38.821 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:01:38.821 | INFO | Total mem usage: 27.8 GiB | 20260508 08:01:38.821 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:01:38.821 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:01:38.821 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:01:38.821 | INFO | High Water Breakdown: | 20260508 08:01:38.821 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:01:38.821 | INFO | ------------------------------------------ | 20260508 08:01:38.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:01:38.821 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:01:38.821 | INFO | Total mem usage: 15.7 GiB | 20260508 08:01:38.821 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:01:38.821 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:01:38.821 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:01:38.821 | INFO | High Water Breakdown: | 20260508 08:01:38.821 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:01:38.821 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:01:38.821 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:01:38.821 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:01:38.821 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:01:38.821 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:01:38.821 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:01:38.821 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:01:38.821 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:01:38.821 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:01:38.821 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:01:38.821 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:01:38.821 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:01:38.821 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:01:38.821 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:01:38.821 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:01:38.821 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:01:38.821 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:01:38.821 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:01:38.821 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:01:38.821 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:01:38.821 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:01:38.821 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:01:38.821 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:01:38.821 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:01:38.821 | INFO | ------------------------------------------ | 20260508 08:01:38.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:01:38.821 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:01:38.821 | INFO | Uploaded 36GiB to GPU in 2.47096s (14.5692GiB/s) | 20260508 08:01:38.821 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:01:38.822 | INFO | Downloaded 34GiB to GPU in 2.43748s (13.9488GiB/s) | 20260508 08:01:38.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:01:38.822 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.74e-01 | 20260508 08:01:38.822 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.20e-02 / 4.34e-03 | 20260508 08:01:38.822 | INFO | PerfReport: 12.15%, Baseline -- 3.96e+00 / 4.43e+00, 5.88e-01 / 2.28e-01 | 20260508 08:01:38.822 | INFO | PerfReport: 3.34%, Histogram -- 1.09e+00 / 1.43e+00, 2.07e-01 / 1.29e-01 | 20260508 08:01:38.822 | INFO | PerfReport: 15.96%, DME -- 4.16e+01 / 4.53e+01, 3.78e+00 / 1.25e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.76e+00 / 1.42e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.37e+00 / 2.56e+00, 4.09e-01 / 2.62e-01 | 20260508 08:01:38.822 | INFO | PerfReport: 3.11%, Metrics -- 1.01e+00 / 1.16e+00, 2.51e-01 / 1.16e-01 | 20260508 08:01:38.822 | INFO | PerfReport: 0.95%, PulseSerializing -- 3.08e-01 / 3.44e-01, 5.05e-02 / 2.28e-02 | 20260508 08:01:38.822 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:01:38.822 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:01:38.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:01:38.822 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 24.54%, Upload -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 24.37%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 30.76%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 2.41%, LockUpload -- 2.55e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 2.49%, LockDownload -- 2.64e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 15.42%, LockCompute -- 1.63e+01 / 0.00e+00, 1.92e+01 / 0.00e+00 | 20260508 08:01:38.822 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260508 08:01:38.918 | INFO | Starting a round of Baz IO | 20260508 08:01:42.323 | INFO | Populating RTMetrics report | 20260508 08:01:42.323 | INFO | RTMetrics report populated | 20260508 08:01:42.323 | INFO | Opening tmp RTMetrics json file | 20260508 08:01:42.338 | INFO | Serializing RTMetrics to json | 20260508 08:01:42.341 | INFO | Serialization done, swapping tmp file now... | 20260508 08:01:42.341 | INFO | RTMetrics done | 20260508 08:01:42.396 | INFO | Finished a round of Baz IO: Wrote 1.513 GiB in 3.477 seconds | 20260508 08:01:42.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.48% / 0.00% / 59.52% | 20260508 08:01:42.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.314307% / 0% / 6.35783e-05% | 20260508 08:01:42.396 | INFO | Baz IO overall: | 20260508 08:01:42.396 | INFO | BytesWritten : 2864470912129 | 20260508 08:01:42.396 | INFO | NumEvents : 1379484948261 | 20260508 08:01:43.969 | INFO | Analyzing chunk frames = [6512640,6513152) | 20260508 08:01:43.969 | INFO | MemInfo: { | 20260508 08:01:43.969 | INFO | "gpu" : " 17 GB", | 20260508 08:01:43.969 | INFO | "huge" : "", | 20260508 08:01:43.969 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:43.969 | INFO | "malloc" : " 14 GB", | 20260508 08:01:43.969 | INFO | "pinned" : " 29 GB" | 20260508 08:01:43.969 | INFO | } | 20260508 08:01:43.969 | INFO | Perf report for the last 5.14918 seconds: | 20260508 08:01:43.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:43.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:43.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:43.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:43.969 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 08:01:43.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:43.969 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:01:43.969 | INFO | PRE_HQ : 65.3%, 1.6, 55.1ms, 0 | 20260508 08:01:43.970 | INFO | BAZWRITER : 66.9%, 2.6, 89.7ms, 0 | 20260508 08:01:43.970 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 08:01:43.970 | INFO | Uploaded 36GiB to GPU in 2.58863s (13.9069GiB/s) | 20260508 08:01:43.970 | INFO | Downloaded 38GiB to GPU in 2.70071s (14.0704GiB/s) | 20260508 08:01:49.099 | INFO | Analyzing chunk frames = [6513152,6513664) | 20260508 08:01:49.099 | INFO | MemInfo: { | 20260508 08:01:49.099 | INFO | "gpu" : " 17 GB", | 20260508 08:01:49.099 | INFO | "huge" : "", | 20260508 08:01:49.099 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:49.099 | INFO | "malloc" : " 14 GB", | 20260508 08:01:49.099 | INFO | "pinned" : " 29 GB" | 20260508 08:01:49.099 | INFO | } | 20260508 08:01:49.099 | INFO | Perf report for the last 5.12999 seconds: | 20260508 08:01:49.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:49.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.099 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 08:01:49.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.100 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:01:49.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:49.100 | INFO | Uploaded 36GiB to GPU in 2.50203s (14.3883GiB/s) | 20260508 08:01:49.100 | INFO | Downloaded 34GiB to GPU in 2.45433s (13.8531GiB/s) | 20260508 08:01:54.243 | INFO | Analyzing chunk frames = [6513664,6514176) | 20260508 08:01:54.243 | INFO | MemInfo: { | 20260508 08:01:54.243 | INFO | "gpu" : " 17 GB", | 20260508 08:01:54.243 | INFO | "huge" : "", | 20260508 08:01:54.243 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:54.243 | INFO | "malloc" : " 14 GB", | 20260508 08:01:54.243 | INFO | "pinned" : " 29 GB" | 20260508 08:01:54.243 | INFO | } | 20260508 08:01:54.244 | INFO | Perf report for the last 5.14425 seconds: | 20260508 08:01:54.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:54.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 08:01:54.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:01:54.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:54.244 | INFO | Uploaded 36GiB to GPU in 2.54364s (14.153GiB/s) | 20260508 08:01:54.244 | INFO | Downloaded 34GiB to GPU in 2.4368s (13.9527GiB/s) | 20260508 08:01:59.382 | INFO | Analyzing chunk frames = [6514176,6514688) | 20260508 08:01:59.382 | INFO | MemInfo: { | 20260508 08:01:59.382 | INFO | "gpu" : " 17 GB", | 20260508 08:01:59.382 | INFO | "huge" : "", | 20260508 08:01:59.382 | INFO | "huge_shared" : " 29 GB", | 20260508 08:01:59.382 | INFO | "malloc" : " 14 GB", | 20260508 08:01:59.382 | INFO | "pinned" : " 29 GB" | 20260508 08:01:59.382 | INFO | } | 20260508 08:01:59.382 | INFO | Perf report for the last 5.13876 seconds: | 20260508 08:01:59.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:01:59.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.383 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:01:59.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.383 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:01:59.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:01:59.383 | INFO | Uploaded 36GiB to GPU in 2.49984s (14.4009GiB/s) | 20260508 08:01:59.383 | INFO | Downloaded 34GiB to GPU in 2.43427s (13.9672GiB/s) | 20260508 08:02:04.523 | INFO | Analyzing chunk frames = [6514688,6515200) | 20260508 08:02:04.523 | INFO | MemInfo: { | 20260508 08:02:04.523 | INFO | "gpu" : " 17 GB", | 20260508 08:02:04.523 | INFO | "huge" : "", | 20260508 08:02:04.523 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:04.523 | INFO | "malloc" : " 14 GB", | 20260508 08:02:04.523 | INFO | "pinned" : " 29 GB" | 20260508 08:02:04.523 | INFO | } | 20260508 08:02:04.523 | INFO | Perf report for the last 5.1405 seconds: | 20260508 08:02:04.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:04.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 08:02:04.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 08:02:04.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:04.523 | INFO | Uploaded 36GiB to GPU in 2.34921s (15.3243GiB/s) | 20260508 08:02:04.523 | INFO | Downloaded 34GiB to GPU in 2.51963s (13.4941GiB/s) | 20260508 08:02:09.661 | INFO | Analyzing chunk frames = [6515200,6515712) | 20260508 08:02:09.661 | INFO | MemInfo: { | 20260508 08:02:09.661 | INFO | "gpu" : " 17 GB", | 20260508 08:02:09.661 | INFO | "huge" : "", | 20260508 08:02:09.661 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:09.661 | INFO | "malloc" : " 14 GB", | 20260508 08:02:09.661 | INFO | "pinned" : " 29 GB" | 20260508 08:02:09.661 | INFO | } | 20260508 08:02:09.661 | INFO | Perf report for the last 5.1383 seconds: | 20260508 08:02:09.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:09.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.661 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:02:09.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.661 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:02:09.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:09.662 | INFO | Uploaded 36GiB to GPU in 2.37793s (15.1392GiB/s) | 20260508 08:02:09.662 | INFO | Downloaded 34GiB to GPU in 2.43977s (13.9357GiB/s) | 20260508 08:02:14.798 | INFO | Analyzing chunk frames = [6515712,6516224) | 20260508 08:02:14.798 | INFO | MemInfo: { | 20260508 08:02:14.798 | INFO | "gpu" : " 17 GB", | 20260508 08:02:14.798 | INFO | "huge" : "", | 20260508 08:02:14.798 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:14.798 | INFO | "malloc" : " 14 GB", | 20260508 08:02:14.798 | INFO | "pinned" : " 29 GB" | 20260508 08:02:14.798 | INFO | } | 20260508 08:02:14.798 | INFO | Perf report for the last 5.13653 seconds: | 20260508 08:02:14.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:14.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 08:02:14.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:02:14.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:14.798 | INFO | Uploaded 36GiB to GPU in 2.46536s (14.6023GiB/s) | 20260508 08:02:14.798 | INFO | Downloaded 34GiB to GPU in 2.43656s (13.9541GiB/s) | 20260508 08:02:19.940 | INFO | Analyzing chunk frames = [6516224,6516736) | 20260508 08:02:19.940 | INFO | MemInfo: { | 20260508 08:02:19.940 | INFO | "gpu" : " 17 GB", | 20260508 08:02:19.940 | INFO | "huge" : "", | 20260508 08:02:19.940 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:19.940 | INFO | "malloc" : " 14 GB", | 20260508 08:02:19.940 | INFO | "pinned" : " 29 GB" | 20260508 08:02:19.940 | INFO | } | 20260508 08:02:19.940 | INFO | Perf report for the last 5.14204 seconds: | 20260508 08:02:19.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:19.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:02:19.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:02:19.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:19.940 | INFO | Uploaded 36GiB to GPU in 2.50682s (14.3608GiB/s) | 20260508 08:02:19.941 | INFO | Downloaded 34GiB to GPU in 2.46053s (13.8181GiB/s) | 20260508 08:02:20.037 | INFO | Starting a round of Baz IO | 20260508 08:02:23.454 | INFO | Populating RTMetrics report | 20260508 08:02:23.455 | INFO | RTMetrics report populated | 20260508 08:02:23.455 | INFO | Opening tmp RTMetrics json file | 20260508 08:02:23.466 | INFO | Serializing RTMetrics to json | 20260508 08:02:23.467 | INFO | Serialization done, swapping tmp file now... | 20260508 08:02:23.468 | INFO | RTMetrics done | 20260508 08:02:23.527 | INFO | Finished a round of Baz IO: Wrote 1.512 GiB in 3.489 seconds | 20260508 08:02:23.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.52% | 20260508 08:02:23.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31333% / 0% / 4.76837e-05% | 20260508 08:02:23.527 | INFO | Baz IO overall: | 20260508 08:02:23.527 | INFO | BytesWritten : 2866094560331 | 20260508 08:02:23.527 | INFO | NumEvents : 1380254945722 | 20260508 08:02:25.079 | INFO | Analyzing chunk frames = [6516736,6517248) | 20260508 08:02:25.079 | INFO | MemInfo: { | 20260508 08:02:25.079 | INFO | "gpu" : " 17 GB", | 20260508 08:02:25.079 | INFO | "huge" : "", | 20260508 08:02:25.079 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:25.079 | INFO | "malloc" : " 14 GB", | 20260508 08:02:25.079 | INFO | "pinned" : " 29 GB" | 20260508 08:02:25.079 | INFO | } | 20260508 08:02:25.079 | INFO | Perf report for the last 5.13946 seconds: | 20260508 08:02:25.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:25.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:25.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:25.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:25.079 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:02:25.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:25.079 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:02:25.079 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 08:02:25.079 | INFO | BAZWRITER : 67.4%, 2.6, 90.1ms, 0 | 20260508 08:02:25.079 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:02:25.080 | INFO | Uploaded 36GiB to GPU in 2.59518s (13.8719GiB/s) | 20260508 08:02:25.080 | INFO | Downloaded 38GiB to GPU in 2.72195s (13.9606GiB/s) | 20260508 08:02:30.218 | INFO | Analyzing chunk frames = [6517248,6517760) | 20260508 08:02:30.218 | INFO | MemInfo: { | 20260508 08:02:30.218 | INFO | "gpu" : " 17 GB", | 20260508 08:02:30.218 | INFO | "huge" : "", | 20260508 08:02:30.218 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:30.218 | INFO | "malloc" : " 14 GB", | 20260508 08:02:30.218 | INFO | "pinned" : " 29 GB" | 20260508 08:02:30.218 | INFO | } | 20260508 08:02:30.219 | INFO | Perf report for the last 5.1393 seconds: | 20260508 08:02:30.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:30.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 08:02:30.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:02:30.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:30.219 | INFO | Uploaded 36GiB to GPU in 2.52628s (14.2502GiB/s) | 20260508 08:02:30.219 | INFO | Downloaded 34GiB to GPU in 2.436s (13.9573GiB/s) | 20260508 08:02:35.360 | INFO | Analyzing chunk frames = [6517760,6518272) | 20260508 08:02:35.360 | INFO | MemInfo: { | 20260508 08:02:35.360 | INFO | "gpu" : " 17 GB", | 20260508 08:02:35.360 | INFO | "huge" : "", | 20260508 08:02:35.360 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:35.360 | INFO | "malloc" : " 14 GB", | 20260508 08:02:35.360 | INFO | "pinned" : " 29 GB" | 20260508 08:02:35.360 | INFO | } | 20260508 08:02:35.360 | INFO | Perf report for the last 5.14119 seconds: | 20260508 08:02:35.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:35.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | ANALYSIS : 66.8%, 3.0, 103.8ms, 0 | 20260508 08:02:35.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:02:35.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:35.360 | INFO | Uploaded 36GiB to GPU in 2.50062s (14.3964GiB/s) | 20260508 08:02:35.360 | INFO | Downloaded 34GiB to GPU in 2.43792s (13.9463GiB/s) | 20260508 08:02:40.491 | INFO | Analyzing chunk frames = [6518272,6518784) | 20260508 08:02:40.491 | INFO | MemInfo: { | 20260508 08:02:40.491 | INFO | "gpu" : " 17 GB", | 20260508 08:02:40.491 | INFO | "huge" : "", | 20260508 08:02:40.491 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:40.491 | INFO | "malloc" : " 14 GB", | 20260508 08:02:40.491 | INFO | "pinned" : " 29 GB" | 20260508 08:02:40.491 | INFO | } | 20260508 08:02:40.491 | INFO | Perf report for the last 5.13145 seconds: | 20260508 08:02:40.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:40.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.491 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 08:02:40.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.491 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 08:02:40.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:40.492 | INFO | Uploaded 36GiB to GPU in 2.51289s (14.3261GiB/s) | 20260508 08:02:40.492 | INFO | Downloaded 34GiB to GPU in 2.42283s (14.0332GiB/s) | 20260508 08:02:45.636 | INFO | Analyzing chunk frames = [6518784,6519296) | 20260508 08:02:45.636 | INFO | MemInfo: { | 20260508 08:02:45.636 | INFO | "gpu" : " 17 GB", | 20260508 08:02:45.636 | INFO | "huge" : "", | 20260508 08:02:45.636 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:45.636 | INFO | "malloc" : " 14 GB", | 20260508 08:02:45.636 | INFO | "pinned" : " 29 GB" | 20260508 08:02:45.636 | INFO | } | 20260508 08:02:45.636 | INFO | Perf report for the last 5.14465 seconds: | 20260508 08:02:45.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:45.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 08:02:45.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:02:45.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:45.636 | INFO | Uploaded 36GiB to GPU in 2.3004s (15.6495GiB/s) | 20260508 08:02:45.636 | INFO | Downloaded 34GiB to GPU in 2.48619s (13.6756GiB/s) | 20260508 08:02:50.783 | INFO | Analyzing chunk frames = [6519296,6519808) | 20260508 08:02:50.783 | INFO | MemInfo: { | 20260508 08:02:50.783 | INFO | "gpu" : " 17 GB", | 20260508 08:02:50.783 | INFO | "huge" : "", | 20260508 08:02:50.783 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:50.783 | INFO | "malloc" : " 14 GB", | 20260508 08:02:50.783 | INFO | "pinned" : " 29 GB" | 20260508 08:02:50.783 | INFO | } | 20260508 08:02:50.783 | INFO | Perf report for the last 5.14688 seconds: | 20260508 08:02:50.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:50.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 08:02:50.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:02:50.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:50.783 | INFO | Uploaded 36GiB to GPU in 2.47122s (14.5677GiB/s) | 20260508 08:02:50.783 | INFO | Downloaded 34GiB to GPU in 2.43928s (13.9386GiB/s) | 20260508 08:02:55.915 | INFO | Analyzing chunk frames = [6519808,6520320) | 20260508 08:02:55.915 | INFO | MemInfo: { | 20260508 08:02:55.915 | INFO | "gpu" : " 17 GB", | 20260508 08:02:55.915 | INFO | "huge" : "", | 20260508 08:02:55.915 | INFO | "huge_shared" : " 29 GB", | 20260508 08:02:55.915 | INFO | "malloc" : " 14 GB", | 20260508 08:02:55.915 | INFO | "pinned" : " 29 GB" | 20260508 08:02:55.915 | INFO | } | 20260508 08:02:55.915 | INFO | Perf report for the last 5.13266 seconds: | 20260508 08:02:55.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:02:55.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.915 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:02:55.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.916 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:02:55.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:02:55.916 | INFO | Uploaded 36GiB to GPU in 2.42304s (14.8574GiB/s) | 20260508 08:02:55.916 | INFO | Downloaded 34GiB to GPU in 2.43124s (13.9846GiB/s) | 20260508 08:03:01.054 | INFO | Analyzing chunk frames = [6520320,6520832) | 20260508 08:03:01.054 | INFO | MemInfo: { | 20260508 08:03:01.054 | INFO | "gpu" : " 17 GB", | 20260508 08:03:01.054 | INFO | "huge" : "", | 20260508 08:03:01.054 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:01.054 | INFO | "malloc" : " 14 GB", | 20260508 08:03:01.054 | INFO | "pinned" : " 29 GB" | 20260508 08:03:01.054 | INFO | } | 20260508 08:03:01.054 | INFO | Perf report for the last 5.13881 seconds: | 20260508 08:03:01.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:01.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 08:03:01.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:03:01.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:01.054 | INFO | Uploaded 36GiB to GPU in 2.43859s (14.7627GiB/s) | 20260508 08:03:01.054 | INFO | Downloaded 34GiB to GPU in 2.44894s (13.8836GiB/s) | 20260508 08:03:01.055 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:03:01.055 | INFO | Total mem usage: 16.7 GiB | 20260508 08:03:01.055 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:03:01.055 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:03:01.055 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:03:01.055 | INFO | High Water Breakdown: | 20260508 08:03:01.055 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:03:01.055 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:03:01.055 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:03:01.055 | INFO | ------------------------------------------ | 20260508 08:03:01.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:03:01.055 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:03:01.055 | INFO | Total mem usage: 26.5 GiB | 20260508 08:03:01.055 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:03:01.055 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:03:01.055 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:03:01.055 | INFO | High Water Breakdown: | 20260508 08:03:01.055 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:03:01.055 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:03:01.055 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:03:01.055 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:03:01.055 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:03:01.055 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:03:01.055 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:03:01.055 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:03:01.055 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:03:01.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:03:01.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:03:01.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:03:01.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:03:01.055 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:03:01.055 | INFO | ------------------------------------------ | 20260508 08:03:01.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:03:01.055 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:03:01.055 | INFO | Total mem usage: 27.8 GiB | 20260508 08:03:01.055 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:03:01.055 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:03:01.055 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:03:01.055 | INFO | High Water Breakdown: | 20260508 08:03:01.055 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:03:01.055 | INFO | ------------------------------------------ | 20260508 08:03:01.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:03:01.055 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:03:01.055 | INFO | Total mem usage: 15.7 GiB | 20260508 08:03:01.055 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:03:01.055 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:03:01.055 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:03:01.055 | INFO | High Water Breakdown: | 20260508 08:03:01.055 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:03:01.055 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:03:01.055 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:03:01.055 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:03:01.055 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:03:01.055 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:03:01.055 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:03:01.055 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:03:01.055 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:03:01.055 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:03:01.055 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:03:01.055 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:03:01.055 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:03:01.055 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:03:01.055 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:03:01.055 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:03:01.055 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:03:01.056 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:03:01.056 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:03:01.056 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:03:01.056 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:03:01.056 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:03:01.056 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:03:01.056 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:03:01.056 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:03:01.056 | INFO | ------------------------------------------ | 20260508 08:03:01.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:03:01.056 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:03:01.056 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:03:01.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:03:01.056 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.40e+01 / 5.55e-01 | 20260508 08:03:01.056 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.12e-02 / 6.66e-03 | 20260508 08:03:01.056 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.45e+00, 5.94e-01 / 2.29e-01 | 20260508 08:03:01.056 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.42e+00, 2.09e-01 / 1.35e-01 | 20260508 08:03:01.056 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.52e+01, 3.52e+00 / 9.20e-01 | 20260508 08:03:01.056 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.87e+00 / 1.32e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.35e+00 / 2.56e+00, 4.10e-01 / 2.15e-01 | 20260508 08:03:01.056 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.14e+00, 2.54e-01 / 8.72e-02 | 20260508 08:03:01.056 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.42e-01, 4.98e-02 / 2.27e-02 | 20260508 08:03:01.056 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:03:01.056 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:03:01.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:03:01.056 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 24.38%, Upload -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 24.44%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 30.97%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 2.27%, LockUpload -- 2.40e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 2.44%, LockDownload -- 2.58e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 15.48%, LockCompute -- 1.64e+01 / 0.00e+00, 1.90e+01 / 0.00e+00 | 20260508 08:03:01.056 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 3.96e-02 / 0.00e+00 | 20260508 08:03:01.151 | INFO | Starting a round of Baz IO | 20260508 08:03:04.571 | INFO | Populating RTMetrics report | 20260508 08:03:04.571 | INFO | RTMetrics report populated | 20260508 08:03:04.571 | INFO | Opening tmp RTMetrics json file | 20260508 08:03:04.572 | INFO | Serializing RTMetrics to json | 20260508 08:03:04.573 | INFO | Serialization done, swapping tmp file now... | 20260508 08:03:04.574 | INFO | RTMetrics done | 20260508 08:03:04.636 | INFO | Finished a round of Baz IO: Wrote 1.512 GiB in 3.485 seconds | 20260508 08:03:04.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.53% | 20260508 08:03:04.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.311887% / 0% / 6.35783e-05% | 20260508 08:03:04.636 | INFO | Baz IO overall: | 20260508 08:03:04.636 | INFO | BytesWritten : 2867717575547 | 20260508 08:03:04.636 | INFO | NumEvents : 1381024350962 | 20260508 08:03:06.198 | INFO | Analyzing chunk frames = [6520832,6521344) | 20260508 08:03:06.198 | INFO | MemInfo: { | 20260508 08:03:06.198 | INFO | "gpu" : " 17 GB", | 20260508 08:03:06.198 | INFO | "huge" : "", | 20260508 08:03:06.198 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:06.198 | INFO | "malloc" : " 14 GB", | 20260508 08:03:06.198 | INFO | "pinned" : " 29 GB" | 20260508 08:03:06.198 | INFO | } | 20260508 08:03:06.198 | INFO | Perf report for the last 5.14415 seconds: | 20260508 08:03:06.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:06.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:06.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:06.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:06.198 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 08:03:06.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:06.198 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 08:03:06.198 | INFO | PRE_HQ : 65.9%, 1.6, 54.6ms, 0 | 20260508 08:03:06.199 | INFO | BAZWRITER : 67.1%, 2.6, 89.9ms, 0 | 20260508 08:03:06.199 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 08:03:06.199 | INFO | Uploaded 36GiB to GPU in 2.49703s (14.4171GiB/s) | 20260508 08:03:06.199 | INFO | Downloaded 38GiB to GPU in 2.71542s (13.9942GiB/s) | 20260508 08:03:11.336 | INFO | Analyzing chunk frames = [6521344,6521856) | 20260508 08:03:11.336 | INFO | MemInfo: { | 20260508 08:03:11.336 | INFO | "gpu" : " 17 GB", | 20260508 08:03:11.336 | INFO | "huge" : "", | 20260508 08:03:11.336 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:11.336 | INFO | "malloc" : " 14 GB", | 20260508 08:03:11.336 | INFO | "pinned" : " 29 GB" | 20260508 08:03:11.336 | INFO | } | 20260508 08:03:11.336 | INFO | Perf report for the last 5.13787 seconds: | 20260508 08:03:11.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:11.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.336 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:03:11.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.336 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:03:11.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:11.337 | INFO | Uploaded 36GiB to GPU in 2.52583s (14.2528GiB/s) | 20260508 08:03:11.337 | INFO | Downloaded 34GiB to GPU in 2.45682s (13.839GiB/s) | 20260508 08:03:16.470 | INFO | Analyzing chunk frames = [6521856,6522368) | 20260508 08:03:16.470 | INFO | MemInfo: { | 20260508 08:03:16.470 | INFO | "gpu" : " 17 GB", | 20260508 08:03:16.470 | INFO | "huge" : "", | 20260508 08:03:16.470 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:16.470 | INFO | "malloc" : " 14 GB", | 20260508 08:03:16.470 | INFO | "pinned" : " 29 GB" | 20260508 08:03:16.470 | INFO | } | 20260508 08:03:16.470 | INFO | Perf report for the last 5.13395 seconds: | 20260508 08:03:16.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:16.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.470 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260508 08:03:16.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.470 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 08:03:16.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:16.471 | INFO | Uploaded 36GiB to GPU in 2.46752s (14.5895GiB/s) | 20260508 08:03:16.471 | INFO | Downloaded 34GiB to GPU in 2.43871s (13.9418GiB/s) | 20260508 08:03:21.614 | INFO | Analyzing chunk frames = [6522368,6522880) | 20260508 08:03:21.614 | INFO | MemInfo: { | 20260508 08:03:21.614 | INFO | "gpu" : " 17 GB", | 20260508 08:03:21.614 | INFO | "huge" : "", | 20260508 08:03:21.614 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:21.614 | INFO | "malloc" : " 14 GB", | 20260508 08:03:21.614 | INFO | "pinned" : " 29 GB" | 20260508 08:03:21.614 | INFO | } | 20260508 08:03:21.614 | INFO | Perf report for the last 5.14361 seconds: | 20260508 08:03:21.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:21.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | ANALYSIS : 66.7%, 2.9, 103.3ms, 0 | 20260508 08:03:21.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260508 08:03:21.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:21.614 | INFO | Uploaded 36GiB to GPU in 2.56044s (14.0601GiB/s) | 20260508 08:03:21.614 | INFO | Downloaded 34GiB to GPU in 2.44243s (13.9205GiB/s) | 20260508 08:03:26.747 | INFO | Analyzing chunk frames = [6522880,6523392) | 20260508 08:03:26.747 | INFO | MemInfo: { | 20260508 08:03:26.747 | INFO | "gpu" : " 17 GB", | 20260508 08:03:26.747 | INFO | "huge" : "", | 20260508 08:03:26.747 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:26.747 | INFO | "malloc" : " 14 GB", | 20260508 08:03:26.747 | INFO | "pinned" : " 29 GB" | 20260508 08:03:26.747 | INFO | } | 20260508 08:03:26.747 | INFO | Perf report for the last 5.13293 seconds: | 20260508 08:03:26.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:26.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.747 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:03:26.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.747 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:03:26.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:26.748 | INFO | Uploaded 36GiB to GPU in 2.4065s (14.9595GiB/s) | 20260508 08:03:26.748 | INFO | Downloaded 34GiB to GPU in 2.53092s (13.4338GiB/s) | 20260508 08:03:31.894 | INFO | Analyzing chunk frames = [6523392,6523904) | 20260508 08:03:31.894 | INFO | MemInfo: { | 20260508 08:03:31.894 | INFO | "gpu" : " 17 GB", | 20260508 08:03:31.894 | INFO | "huge" : "", | 20260508 08:03:31.894 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:31.894 | INFO | "malloc" : " 14 GB", | 20260508 08:03:31.894 | INFO | "pinned" : " 29 GB" | 20260508 08:03:31.894 | INFO | } | 20260508 08:03:31.894 | INFO | Perf report for the last 5.14755 seconds: | 20260508 08:03:31.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:31.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:03:31.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:03:31.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:31.895 | INFO | Uploaded 36GiB to GPU in 2.46421s (14.6091GiB/s) | 20260508 08:03:31.895 | INFO | Downloaded 34GiB to GPU in 2.44519s (13.9049GiB/s) | 20260508 08:03:37.030 | INFO | Analyzing chunk frames = [6523904,6524416) | 20260508 08:03:37.030 | INFO | MemInfo: { | 20260508 08:03:37.030 | INFO | "gpu" : " 17 GB", | 20260508 08:03:37.030 | INFO | "huge" : "", | 20260508 08:03:37.030 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:37.030 | INFO | "malloc" : " 14 GB", | 20260508 08:03:37.030 | INFO | "pinned" : " 29 GB" | 20260508 08:03:37.030 | INFO | } | 20260508 08:03:37.030 | INFO | Perf report for the last 5.13607 seconds: | 20260508 08:03:37.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:37.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:03:37.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:03:37.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:37.031 | INFO | Uploaded 36GiB to GPU in 2.41405s (14.9127GiB/s) | 20260508 08:03:37.031 | INFO | Downloaded 34GiB to GPU in 2.4439s (13.9122GiB/s) | 20260508 08:03:42.168 | INFO | Analyzing chunk frames = [6524416,6524928) | 20260508 08:03:42.168 | INFO | MemInfo: { | 20260508 08:03:42.168 | INFO | "gpu" : " 17 GB", | 20260508 08:03:42.168 | INFO | "huge" : "", | 20260508 08:03:42.168 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:42.168 | INFO | "malloc" : " 14 GB", | 20260508 08:03:42.168 | INFO | "pinned" : " 29 GB" | 20260508 08:03:42.168 | INFO | } | 20260508 08:03:42.168 | INFO | Perf report for the last 5.13791 seconds: | 20260508 08:03:42.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:42.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 08:03:42.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:03:42.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:42.169 | INFO | Uploaded 36GiB to GPU in 2.41635s (14.8985GiB/s) | 20260508 08:03:42.169 | INFO | Downloaded 34GiB to GPU in 2.44191s (13.9235GiB/s) | 20260508 08:03:42.265 | INFO | Starting a round of Baz IO | 20260508 08:03:45.693 | INFO | Populating RTMetrics report | 20260508 08:03:45.693 | INFO | RTMetrics report populated | 20260508 08:03:45.693 | INFO | Opening tmp RTMetrics json file | 20260508 08:03:45.703 | INFO | Serializing RTMetrics to json | 20260508 08:03:45.705 | INFO | Serialization done, swapping tmp file now... | 20260508 08:03:45.705 | INFO | RTMetrics done | 20260508 08:03:45.761 | INFO | Finished a round of Baz IO: Wrote 1.514 GiB in 3.495 seconds | 20260508 08:03:45.761 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.53% | 20260508 08:03:45.761 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.313397% / 1.19209e-05% / 3.97364e-05% | 20260508 08:03:45.761 | INFO | Baz IO overall: | 20260508 08:03:45.761 | INFO | BytesWritten : 2869343291702 | 20260508 08:03:45.761 | INFO | NumEvents : 1381795712173 | 20260508 08:03:47.305 | INFO | Analyzing chunk frames = [6524928,6525440) | 20260508 08:03:47.305 | INFO | MemInfo: { | 20260508 08:03:47.305 | INFO | "gpu" : " 17 GB", | 20260508 08:03:47.305 | INFO | "huge" : "", | 20260508 08:03:47.305 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:47.305 | INFO | "malloc" : " 14 GB", | 20260508 08:03:47.305 | INFO | "pinned" : " 29 GB" | 20260508 08:03:47.305 | INFO | } | 20260508 08:03:47.305 | INFO | Perf report for the last 5.13633 seconds: | 20260508 08:03:47.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:47.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:47.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:47.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:47.305 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:03:47.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:47.305 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:03:47.305 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 08:03:47.305 | INFO | BAZWRITER : 67.5%, 2.6, 89.6ms, 0 | 20260508 08:03:47.305 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:03:47.305 | INFO | Uploaded 36GiB to GPU in 2.55654s (14.0815GiB/s) | 20260508 08:03:47.305 | INFO | Downloaded 38GiB to GPU in 2.71356s (14.0037GiB/s) | 20260508 08:03:52.443 | INFO | Analyzing chunk frames = [6525440,6525952) | 20260508 08:03:52.443 | INFO | MemInfo: { | 20260508 08:03:52.443 | INFO | "gpu" : " 17 GB", | 20260508 08:03:52.443 | INFO | "huge" : "", | 20260508 08:03:52.443 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:52.443 | INFO | "malloc" : " 14 GB", | 20260508 08:03:52.443 | INFO | "pinned" : " 29 GB" | 20260508 08:03:52.443 | INFO | } | 20260508 08:03:52.443 | INFO | Perf report for the last 5.13859 seconds: | 20260508 08:03:52.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:52.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.443 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:03:52.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.443 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:03:52.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:52.444 | INFO | Uploaded 36GiB to GPU in 2.48815s (14.4686GiB/s) | 20260508 08:03:52.444 | INFO | Downloaded 34GiB to GPU in 2.45936s (13.8248GiB/s) | 20260508 08:03:57.585 | INFO | Analyzing chunk frames = [6525952,6526464) | 20260508 08:03:57.585 | INFO | MemInfo: { | 20260508 08:03:57.585 | INFO | "gpu" : " 17 GB", | 20260508 08:03:57.585 | INFO | "huge" : "", | 20260508 08:03:57.585 | INFO | "huge_shared" : " 29 GB", | 20260508 08:03:57.585 | INFO | "malloc" : " 14 GB", | 20260508 08:03:57.585 | INFO | "pinned" : " 29 GB" | 20260508 08:03:57.585 | INFO | } | 20260508 08:03:57.585 | INFO | Perf report for the last 5.14214 seconds: | 20260508 08:03:57.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:03:57.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.586 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 08:03:57.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.586 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260508 08:03:57.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:03:57.586 | INFO | Uploaded 36GiB to GPU in 2.45883s (14.6411GiB/s) | 20260508 08:03:57.586 | INFO | Downloaded 34GiB to GPU in 2.45156s (13.8687GiB/s) | 20260508 08:04:02.737 | INFO | Analyzing chunk frames = [6526464,6526976) | 20260508 08:04:02.737 | INFO | MemInfo: { | 20260508 08:04:02.737 | INFO | "gpu" : " 17 GB", | 20260508 08:04:02.737 | INFO | "huge" : "", | 20260508 08:04:02.737 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:02.737 | INFO | "malloc" : " 14 GB", | 20260508 08:04:02.737 | INFO | "pinned" : " 29 GB" | 20260508 08:04:02.737 | INFO | } | 20260508 08:04:02.737 | INFO | Perf report for the last 5.15116 seconds: | 20260508 08:04:02.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:02.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | ANALYSIS : 66.6%, 2.9, 103.2ms, 0 | 20260508 08:04:02.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 08:04:02.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:02.737 | INFO | Uploaded 36GiB to GPU in 2.53334s (14.2105GiB/s) | 20260508 08:04:02.737 | INFO | Downloaded 34GiB to GPU in 2.44922s (13.882GiB/s) | 20260508 08:04:07.866 | INFO | Analyzing chunk frames = [6526976,6527488) | 20260508 08:04:07.866 | INFO | MemInfo: { | 20260508 08:04:07.866 | INFO | "gpu" : " 17 GB", | 20260508 08:04:07.866 | INFO | "huge" : "", | 20260508 08:04:07.866 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:07.866 | INFO | "malloc" : " 14 GB", | 20260508 08:04:07.866 | INFO | "pinned" : " 29 GB" | 20260508 08:04:07.866 | INFO | } | 20260508 08:04:07.867 | INFO | Perf report for the last 5.12989 seconds: | 20260508 08:04:07.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:07.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 08:04:07.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:04:07.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:07.867 | INFO | Uploaded 36GiB to GPU in 2.3828s (15.1083GiB/s) | 20260508 08:04:07.867 | INFO | Downloaded 34GiB to GPU in 2.53478s (13.4134GiB/s) | 20260508 08:04:13.002 | INFO | Analyzing chunk frames = [6527488,6528000) | 20260508 08:04:13.002 | INFO | MemInfo: { | 20260508 08:04:13.002 | INFO | "gpu" : " 17 GB", | 20260508 08:04:13.002 | INFO | "huge" : "", | 20260508 08:04:13.003 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:13.003 | INFO | "malloc" : " 14 GB", | 20260508 08:04:13.003 | INFO | "pinned" : " 29 GB" | 20260508 08:04:13.003 | INFO | } | 20260508 08:04:13.003 | INFO | Perf report for the last 5.13606 seconds: | 20260508 08:04:13.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:13.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:04:13.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:04:13.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:13.003 | INFO | Uploaded 36GiB to GPU in 2.37185s (15.1781GiB/s) | 20260508 08:04:13.003 | INFO | Downloaded 34GiB to GPU in 2.45391s (13.8555GiB/s) | 20260508 08:04:18.143 | INFO | Analyzing chunk frames = [6528000,6528512) | 20260508 08:04:18.143 | INFO | MemInfo: { | 20260508 08:04:18.143 | INFO | "gpu" : " 17 GB", | 20260508 08:04:18.143 | INFO | "huge" : "", | 20260508 08:04:18.143 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:18.143 | INFO | "malloc" : " 14 GB", | 20260508 08:04:18.143 | INFO | "pinned" : " 29 GB" | 20260508 08:04:18.143 | INFO | } | 20260508 08:04:18.143 | INFO | Perf report for the last 5.14043 seconds: | 20260508 08:04:18.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:18.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.143 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 08:04:18.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.143 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:04:18.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:18.144 | INFO | Uploaded 36GiB to GPU in 2.4446s (14.7263GiB/s) | 20260508 08:04:18.144 | INFO | Downloaded 34GiB to GPU in 2.4474s (13.8923GiB/s) | 20260508 08:04:23.281 | INFO | Analyzing chunk frames = [6528512,6529024) | 20260508 08:04:23.281 | INFO | MemInfo: { | 20260508 08:04:23.281 | INFO | "gpu" : " 17 GB", | 20260508 08:04:23.281 | INFO | "huge" : "", | 20260508 08:04:23.281 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:23.281 | INFO | "malloc" : " 14 GB", | 20260508 08:04:23.281 | INFO | "pinned" : " 29 GB" | 20260508 08:04:23.281 | INFO | } | 20260508 08:04:23.281 | INFO | Perf report for the last 5.13772 seconds: | 20260508 08:04:23.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:23.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 08:04:23.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:04:23.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:23.281 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:04:23.281 | INFO | Total mem usage: 16.7 GiB | 20260508 08:04:23.281 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:04:23.281 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:04:23.281 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:04:23.281 | INFO | High Water Breakdown: | 20260508 08:04:23.281 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:04:23.281 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:04:23.281 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:04:23.281 | INFO | ------------------------------------------ | 20260508 08:04:23.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:04:23.281 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:04:23.281 | INFO | Total mem usage: 26.5 GiB | 20260508 08:04:23.281 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:04:23.281 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:04:23.281 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:04:23.281 | INFO | High Water Breakdown: | 20260508 08:04:23.281 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:04:23.281 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:04:23.281 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:04:23.281 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:04:23.281 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:04:23.281 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:04:23.281 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:04:23.281 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:04:23.281 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:04:23.281 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:04:23.281 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:04:23.281 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:04:23.281 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:04:23.281 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:04:23.281 | INFO | ------------------------------------------ | 20260508 08:04:23.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:04:23.281 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:04:23.281 | INFO | Total mem usage: 27.8 GiB | 20260508 08:04:23.281 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:04:23.281 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:04:23.281 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:04:23.281 | INFO | High Water Breakdown: | 20260508 08:04:23.281 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:04:23.281 | INFO | ------------------------------------------ | 20260508 08:04:23.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:04:23.281 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:04:23.281 | INFO | Total mem usage: 15.7 GiB | 20260508 08:04:23.281 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:04:23.281 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:04:23.281 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:04:23.281 | INFO | High Water Breakdown: | 20260508 08:04:23.281 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:04:23.281 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:04:23.281 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:04:23.281 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:04:23.281 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:04:23.281 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:04:23.281 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:04:23.281 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:04:23.281 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:04:23.281 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:04:23.281 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:04:23.281 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:04:23.281 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:04:23.281 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:04:23.281 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:04:23.281 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:04:23.281 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:04:23.281 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:04:23.281 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:04:23.281 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:04:23.281 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:04:23.281 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:04:23.281 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:04:23.281 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:04:23.281 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:04:23.281 | INFO | ------------------------------------------ | 20260508 08:04:23.281 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:04:23.281 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:04:23.281 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:04:23.281 | INFO | Uploaded 36GiB to GPU in 2.39229s (15.0484GiB/s) | 20260508 08:04:23.282 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:04:23.282 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 7.50e-01 | 20260508 08:04:23.282 | INFO | Downloaded 34GiB to GPU in 2.44931s (13.8814GiB/s) | 20260508 08:04:23.282 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.04e-01, 1.14e-02 / 3.49e-03 | 20260508 08:04:23.282 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.46e+00, 5.99e-01 / 2.77e-01 | 20260508 08:04:23.282 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.11e-01 / 1.19e-01 | 20260508 08:04:23.282 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.49e+01, 3.74e+00 / 1.13e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.89e+00 / 1.41e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.35e+00 / 2.51e+00, 4.06e-01 / 2.73e-01 | 20260508 08:04:23.282 | INFO | PerfReport: 3.09%, Metrics -- 1.01e+00 / 1.20e+00, 2.52e-01 / 1.43e-01 | 20260508 08:04:23.282 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.09e-01 / 3.41e-01, 4.99e-02 / 2.70e-02 | 20260508 08:04:23.282 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:04:23.282 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:04:23.282 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:04:23.282 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 24.25%, Upload -- 2.56e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 24.55%, Download -- 2.60e+01 / 0.00e+00, 8.32e+00 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 30.99%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 2.05%, LockUpload -- 2.17e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 2.68%, LockDownload -- 2.84e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 15.46%, LockCompute -- 1.63e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:04:23.282 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260508 08:04:23.379 | INFO | Starting a round of Baz IO | 20260508 08:04:26.793 | INFO | Populating RTMetrics report | 20260508 08:04:26.793 | INFO | RTMetrics report populated | 20260508 08:04:26.793 | INFO | Opening tmp RTMetrics json file | 20260508 08:04:26.804 | INFO | Serializing RTMetrics to json | 20260508 08:04:26.806 | INFO | Serialization done, swapping tmp file now... | 20260508 08:04:26.806 | INFO | RTMetrics done | 20260508 08:04:26.871 | INFO | Finished a round of Baz IO: Wrote 1.512 GiB in 3.491 seconds | 20260508 08:04:26.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.53% | 20260508 08:04:26.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.312233% / 7.94729e-06% / 6.35783e-05% | 20260508 08:04:26.871 | INFO | Baz IO overall: | 20260508 08:04:26.871 | INFO | BytesWritten : 2870967018309 | 20260508 08:04:26.871 | INFO | NumEvents : 1382565909674 | 20260508 08:04:28.420 | INFO | Analyzing chunk frames = [6529024,6529536) | 20260508 08:04:28.420 | INFO | MemInfo: { | 20260508 08:04:28.420 | INFO | "gpu" : " 17 GB", | 20260508 08:04:28.420 | INFO | "huge" : "", | 20260508 08:04:28.420 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:28.420 | INFO | "malloc" : " 14 GB", | 20260508 08:04:28.420 | INFO | "pinned" : " 29 GB" | 20260508 08:04:28.420 | INFO | } | 20260508 08:04:28.420 | INFO | Perf report for the last 5.13961 seconds: | 20260508 08:04:28.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:28.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:28.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:28.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:28.421 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 08:04:28.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:28.421 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:04:28.421 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:04:28.421 | INFO | BAZWRITER : 67.4%, 2.6, 89.7ms, 0 | 20260508 08:04:28.421 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:04:28.421 | INFO | Uploaded 36GiB to GPU in 2.5489s (14.1237GiB/s) | 20260508 08:04:28.421 | INFO | Downloaded 38GiB to GPU in 2.722s (13.9603GiB/s) | 20260508 08:04:33.557 | INFO | Analyzing chunk frames = [6529536,6530048) | 20260508 08:04:33.557 | INFO | MemInfo: { | 20260508 08:04:33.557 | INFO | "gpu" : " 17 GB", | 20260508 08:04:33.557 | INFO | "huge" : "", | 20260508 08:04:33.557 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:33.557 | INFO | "malloc" : " 14 GB", | 20260508 08:04:33.557 | INFO | "pinned" : " 29 GB" | 20260508 08:04:33.558 | INFO | } | 20260508 08:04:33.558 | INFO | Perf report for the last 5.13728 seconds: | 20260508 08:04:33.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:33.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:04:33.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:04:33.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:33.558 | INFO | Uploaded 36GiB to GPU in 2.46096s (14.6285GiB/s) | 20260508 08:04:33.558 | INFO | Downloaded 34GiB to GPU in 2.45401s (13.8549GiB/s) | 20260508 08:04:38.701 | INFO | Analyzing chunk frames = [6530048,6530560) | 20260508 08:04:38.701 | INFO | MemInfo: { | 20260508 08:04:38.701 | INFO | "gpu" : " 17 GB", | 20260508 08:04:38.701 | INFO | "huge" : "", | 20260508 08:04:38.701 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:38.701 | INFO | "malloc" : " 14 GB", | 20260508 08:04:38.701 | INFO | "pinned" : " 29 GB" | 20260508 08:04:38.701 | INFO | } | 20260508 08:04:38.701 | INFO | Perf report for the last 5.14324 seconds: | 20260508 08:04:38.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:38.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 08:04:38.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 08:04:38.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:38.701 | INFO | Uploaded 36GiB to GPU in 2.49035s (14.4558GiB/s) | 20260508 08:04:38.701 | INFO | Downloaded 34GiB to GPU in 2.46976s (13.7665GiB/s) | 20260508 08:04:43.841 | INFO | Analyzing chunk frames = [6530560,6531072) | 20260508 08:04:43.841 | INFO | MemInfo: { | 20260508 08:04:43.841 | INFO | "gpu" : " 17 GB", | 20260508 08:04:43.841 | INFO | "huge" : "", | 20260508 08:04:43.841 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:43.841 | INFO | "malloc" : " 14 GB", | 20260508 08:04:43.841 | INFO | "pinned" : " 29 GB" | 20260508 08:04:43.841 | INFO | } | 20260508 08:04:43.841 | INFO | Perf report for the last 5.14039 seconds: | 20260508 08:04:43.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:43.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.841 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 08:04:43.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.841 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:04:43.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:43.842 | INFO | Uploaded 36GiB to GPU in 2.52488s (14.2581GiB/s) | 20260508 08:04:43.842 | INFO | Downloaded 34GiB to GPU in 2.45707s (13.8376GiB/s) | 20260508 08:04:48.982 | INFO | Analyzing chunk frames = [6531072,6531584) | 20260508 08:04:48.982 | INFO | MemInfo: { | 20260508 08:04:48.982 | INFO | "gpu" : " 17 GB", | 20260508 08:04:48.982 | INFO | "huge" : "", | 20260508 08:04:48.982 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:48.982 | INFO | "malloc" : " 14 GB", | 20260508 08:04:48.982 | INFO | "pinned" : " 29 GB" | 20260508 08:04:48.982 | INFO | } | 20260508 08:04:48.982 | INFO | Perf report for the last 5.14045 seconds: | 20260508 08:04:48.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:48.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:04:48.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 08:04:48.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:48.982 | INFO | Uploaded 36GiB to GPU in 2.3617s (15.2433GiB/s) | 20260508 08:04:48.982 | INFO | Downloaded 34GiB to GPU in 2.52729s (13.4532GiB/s) | 20260508 08:04:54.114 | INFO | Analyzing chunk frames = [6531584,6532096) | 20260508 08:04:54.114 | INFO | MemInfo: { | 20260508 08:04:54.114 | INFO | "gpu" : " 17 GB", | 20260508 08:04:54.114 | INFO | "huge" : "", | 20260508 08:04:54.114 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:54.114 | INFO | "malloc" : " 14 GB", | 20260508 08:04:54.114 | INFO | "pinned" : " 29 GB" | 20260508 08:04:54.114 | INFO | } | 20260508 08:04:54.114 | INFO | Perf report for the last 5.13244 seconds: | 20260508 08:04:54.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:54.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:04:54.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:04:54.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:54.114 | INFO | Uploaded 36GiB to GPU in 2.36433s (15.2263GiB/s) | 20260508 08:04:54.115 | INFO | Downloaded 34GiB to GPU in 2.44425s (13.9102GiB/s) | 20260508 08:04:59.258 | INFO | Analyzing chunk frames = [6532096,6532608) | 20260508 08:04:59.258 | INFO | MemInfo: { | 20260508 08:04:59.258 | INFO | "gpu" : " 17 GB", | 20260508 08:04:59.258 | INFO | "huge" : "", | 20260508 08:04:59.258 | INFO | "huge_shared" : " 29 GB", | 20260508 08:04:59.258 | INFO | "malloc" : " 14 GB", | 20260508 08:04:59.258 | INFO | "pinned" : " 29 GB" | 20260508 08:04:59.258 | INFO | } | 20260508 08:04:59.258 | INFO | Perf report for the last 5.14403 seconds: | 20260508 08:04:59.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:04:59.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.258 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 08:04:59.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.258 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:04:59.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:04:59.259 | INFO | Uploaded 36GiB to GPU in 2.49779s (14.4128GiB/s) | 20260508 08:04:59.259 | INFO | Downloaded 34GiB to GPU in 2.45168s (13.868GiB/s) | 20260508 08:05:04.400 | INFO | Analyzing chunk frames = [6532608,6533120) | 20260508 08:05:04.400 | INFO | MemInfo: { | 20260508 08:05:04.400 | INFO | "gpu" : " 17 GB", | 20260508 08:05:04.400 | INFO | "huge" : "", | 20260508 08:05:04.400 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:04.400 | INFO | "malloc" : " 14 GB", | 20260508 08:05:04.400 | INFO | "pinned" : " 29 GB" | 20260508 08:05:04.400 | INFO | } | 20260508 08:05:04.401 | INFO | Perf report for the last 5.14219 seconds: | 20260508 08:05:04.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:04.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:05:04.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:05:04.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:04.401 | INFO | Uploaded 36GiB to GPU in 2.42203s (14.8636GiB/s) | 20260508 08:05:04.401 | INFO | Downloaded 34GiB to GPU in 2.44914s (13.8824GiB/s) | 20260508 08:05:04.499 | INFO | Starting a round of Baz IO | 20260508 08:05:07.929 | INFO | Populating RTMetrics report | 20260508 08:05:07.930 | INFO | RTMetrics report populated | 20260508 08:05:07.930 | INFO | Opening tmp RTMetrics json file | 20260508 08:05:07.931 | INFO | Serializing RTMetrics to json | 20260508 08:05:07.932 | INFO | Serialization done, swapping tmp file now... | 20260508 08:05:07.933 | INFO | RTMetrics done | 20260508 08:05:08.010 | INFO | Finished a round of Baz IO: Wrote 1.512 GiB in 3.51 seconds | 20260508 08:05:08.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.53% | 20260508 08:05:08.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.311744% / 0% / 7.15256e-05% | 20260508 08:05:08.010 | INFO | Baz IO overall: | 20260508 08:05:08.010 | INFO | BytesWritten : 2872590865840 | 20260508 08:05:08.010 | INFO | NumEvents : 1383336034418 | 20260508 08:05:09.538 | INFO | Analyzing chunk frames = [6533120,6533632) | 20260508 08:05:09.539 | INFO | MemInfo: { | 20260508 08:05:09.539 | INFO | "gpu" : " 17 GB", | 20260508 08:05:09.539 | INFO | "huge" : "", | 20260508 08:05:09.539 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:09.539 | INFO | "malloc" : " 14 GB", | 20260508 08:05:09.539 | INFO | "pinned" : " 29 GB" | 20260508 08:05:09.539 | INFO | } | 20260508 08:05:09.539 | INFO | Perf report for the last 5.13812 seconds: | 20260508 08:05:09.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:09.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:09.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:09.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:09.539 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 08:05:09.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:09.539 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 08:05:09.539 | INFO | PRE_HQ : 66.0%, 1.7, 58.6ms, 0 | 20260508 08:05:09.539 | INFO | BAZWRITER : 67.8%, 2.7, 94.5ms, 0 | 20260508 08:05:09.539 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 08:05:09.539 | INFO | Uploaded 36GiB to GPU in 2.59036s (13.8977GiB/s) | 20260508 08:05:09.539 | INFO | Downloaded 38GiB to GPU in 2.71574s (13.9925GiB/s) | 20260508 08:05:14.679 | INFO | Analyzing chunk frames = [6533632,6534144) | 20260508 08:05:14.679 | INFO | MemInfo: { | 20260508 08:05:14.679 | INFO | "gpu" : " 17 GB", | 20260508 08:05:14.679 | INFO | "huge" : "", | 20260508 08:05:14.679 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:14.679 | INFO | "malloc" : " 14 GB", | 20260508 08:05:14.679 | INFO | "pinned" : " 29 GB" | 20260508 08:05:14.679 | INFO | } | 20260508 08:05:14.679 | INFO | Perf report for the last 5.14043 seconds: | 20260508 08:05:14.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:14.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.679 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:05:14.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.679 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:05:14.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:14.680 | INFO | Uploaded 36GiB to GPU in 2.52502s (14.2573GiB/s) | 20260508 08:05:14.680 | INFO | Downloaded 34GiB to GPU in 2.4342s (13.9676GiB/s) | 20260508 08:05:19.813 | INFO | Analyzing chunk frames = [6534144,6534656) | 20260508 08:05:19.813 | INFO | MemInfo: { | 20260508 08:05:19.813 | INFO | "gpu" : " 17 GB", | 20260508 08:05:19.813 | INFO | "huge" : "", | 20260508 08:05:19.813 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:19.814 | INFO | "malloc" : " 14 GB", | 20260508 08:05:19.814 | INFO | "pinned" : " 29 GB" | 20260508 08:05:19.814 | INFO | } | 20260508 08:05:19.814 | INFO | Perf report for the last 5.13463 seconds: | 20260508 08:05:19.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:19.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 08:05:19.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:05:19.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:19.814 | INFO | Uploaded 36GiB to GPU in 2.51783s (14.298GiB/s) | 20260508 08:05:19.814 | INFO | Downloaded 34GiB to GPU in 2.4346s (13.9653GiB/s) | 20260508 08:05:24.955 | INFO | Analyzing chunk frames = [6534656,6535168) | 20260508 08:05:24.955 | INFO | MemInfo: { | 20260508 08:05:24.955 | INFO | "gpu" : " 17 GB", | 20260508 08:05:24.955 | INFO | "huge" : "", | 20260508 08:05:24.955 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:24.955 | INFO | "malloc" : " 14 GB", | 20260508 08:05:24.955 | INFO | "pinned" : " 29 GB" | 20260508 08:05:24.955 | INFO | } | 20260508 08:05:24.955 | INFO | Perf report for the last 5.14137 seconds: | 20260508 08:05:24.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:24.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | ANALYSIS : 67.0%, 2.9, 103.7ms, 0 | 20260508 08:05:24.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:05:24.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:24.955 | INFO | Uploaded 36GiB to GPU in 2.59018s (13.8986GiB/s) | 20260508 08:05:24.955 | INFO | Downloaded 34GiB to GPU in 2.41779s (14.0624GiB/s) | 20260508 08:05:30.096 | INFO | Analyzing chunk frames = [6535168,6535680) | 20260508 08:05:30.096 | INFO | MemInfo: { | 20260508 08:05:30.096 | INFO | "gpu" : " 17 GB", | 20260508 08:05:30.096 | INFO | "huge" : "", | 20260508 08:05:30.096 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:30.096 | INFO | "malloc" : " 14 GB", | 20260508 08:05:30.096 | INFO | "pinned" : " 29 GB" | 20260508 08:05:30.096 | INFO | } | 20260508 08:05:30.096 | INFO | Perf report for the last 5.14115 seconds: | 20260508 08:05:30.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:30.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 08:05:30.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:05:30.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:30.096 | INFO | Uploaded 36GiB to GPU in 2.41675s (14.8961GiB/s) | 20260508 08:05:30.097 | INFO | Downloaded 34GiB to GPU in 2.52839s (13.4473GiB/s) | 20260508 08:05:35.235 | INFO | Analyzing chunk frames = [6535680,6536192) | 20260508 08:05:35.235 | INFO | MemInfo: { | 20260508 08:05:35.235 | INFO | "gpu" : " 17 GB", | 20260508 08:05:35.235 | INFO | "huge" : "", | 20260508 08:05:35.235 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:35.235 | INFO | "malloc" : " 14 GB", | 20260508 08:05:35.235 | INFO | "pinned" : " 29 GB" | 20260508 08:05:35.235 | INFO | } | 20260508 08:05:35.235 | INFO | Perf report for the last 5.13853 seconds: | 20260508 08:05:35.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:35.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:05:35.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:05:35.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:35.235 | INFO | Uploaded 36GiB to GPU in 2.39623s (15.0236GiB/s) | 20260508 08:05:35.235 | INFO | Downloaded 34GiB to GPU in 2.44334s (13.9154GiB/s) | 20260508 08:05:40.373 | INFO | Analyzing chunk frames = [6536192,6536704) | 20260508 08:05:40.373 | INFO | MemInfo: { | 20260508 08:05:40.373 | INFO | "gpu" : " 17 GB", | 20260508 08:05:40.373 | INFO | "huge" : "", | 20260508 08:05:40.373 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:40.373 | INFO | "malloc" : " 14 GB", | 20260508 08:05:40.373 | INFO | "pinned" : " 29 GB" | 20260508 08:05:40.373 | INFO | } | 20260508 08:05:40.373 | INFO | Perf report for the last 5.13818 seconds: | 20260508 08:05:40.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:40.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:05:40.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:05:40.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:40.373 | INFO | Uploaded 36GiB to GPU in 2.41233s (14.9233GiB/s) | 20260508 08:05:40.374 | INFO | Downloaded 34GiB to GPU in 2.43701s (13.9515GiB/s) | 20260508 08:05:45.509 | INFO | Analyzing chunk frames = [6536704,6537216) | 20260508 08:05:45.509 | INFO | MemInfo: { | 20260508 08:05:45.509 | INFO | "gpu" : " 17 GB", | 20260508 08:05:45.509 | INFO | "huge" : "", | 20260508 08:05:45.509 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:45.509 | INFO | "malloc" : " 14 GB", | 20260508 08:05:45.509 | INFO | "pinned" : " 29 GB" | 20260508 08:05:45.509 | INFO | } | 20260508 08:05:45.509 | INFO | Perf report for the last 5.13583 seconds: | 20260508 08:05:45.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:45.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 08:05:45.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 08:05:45.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:45.509 | INFO | Uploaded 36GiB to GPU in 2.43382s (14.7915GiB/s) | 20260508 08:05:45.509 | INFO | Downloaded 34GiB to GPU in 2.43944s (13.9376GiB/s) | 20260508 08:05:45.509 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:05:45.509 | INFO | Total mem usage: 16.7 GiB | 20260508 08:05:45.509 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:05:45.509 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:05:45.509 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:05:45.509 | INFO | High Water Breakdown: | 20260508 08:05:45.509 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:05:45.509 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:05:45.509 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:05:45.509 | INFO | ------------------------------------------ | 20260508 08:05:45.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:05:45.509 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:05:45.509 | INFO | Total mem usage: 26.5 GiB | 20260508 08:05:45.510 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:05:45.510 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:05:45.510 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:05:45.510 | INFO | High Water Breakdown: | 20260508 08:05:45.510 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:05:45.510 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:05:45.510 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:05:45.510 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:05:45.510 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:05:45.510 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:05:45.510 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:05:45.510 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:05:45.510 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:05:45.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:05:45.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:05:45.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:05:45.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:05:45.510 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:05:45.510 | INFO | ------------------------------------------ | 20260508 08:05:45.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:05:45.510 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:05:45.510 | INFO | Total mem usage: 27.8 GiB | 20260508 08:05:45.510 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:05:45.510 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:05:45.510 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:05:45.510 | INFO | High Water Breakdown: | 20260508 08:05:45.510 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:05:45.510 | INFO | ------------------------------------------ | 20260508 08:05:45.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:05:45.510 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:05:45.510 | INFO | Total mem usage: 15.7 GiB | 20260508 08:05:45.510 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:05:45.510 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:05:45.510 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:05:45.510 | INFO | High Water Breakdown: | 20260508 08:05:45.510 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:05:45.510 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:05:45.510 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:05:45.510 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:05:45.510 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:05:45.510 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:05:45.510 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:05:45.510 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:05:45.510 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:05:45.510 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:05:45.510 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:05:45.510 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:05:45.510 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:05:45.510 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:05:45.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:05:45.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:05:45.510 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:05:45.510 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:05:45.510 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:05:45.510 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:05:45.510 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:05:45.510 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:05:45.510 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:05:45.510 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:05:45.510 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:05:45.510 | INFO | ------------------------------------------ | 20260508 08:05:45.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:05:45.510 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:05:45.510 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:05:45.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:05:45.510 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.40e+01 / 6.09e-01 | 20260508 08:05:45.510 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.11e-02 / 6.54e-03 | 20260508 08:05:45.510 | INFO | PerfReport: 12.09%, Baseline -- 3.97e+00 / 4.43e+00, 5.88e-01 / 2.17e-01 | 20260508 08:05:45.510 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.44e+00, 2.07e-01 / 1.33e-01 | 20260508 08:05:45.510 | INFO | PerfReport: 15.86%, DME -- 4.16e+01 / 4.47e+01, 3.70e+00 / 1.06e+00 | 20260508 08:05:45.510 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.91e+00 / 1.33e+00 | 20260508 08:05:45.510 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.47e+00, 4.12e-01 / 2.54e-01 | 20260508 08:05:45.510 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.16e+00, 2.51e-01 / 1.41e-01 | 20260508 08:05:45.510 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.07e-01 / 3.44e-01, 4.98e-02 / 2.72e-02 | 20260508 08:05:45.510 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:05:45.510 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:05:45.510 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:05:45.510 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:05:45.510 | INFO | PerfReport: 24.37%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 08:05:45.510 | INFO | PerfReport: 24.54%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 08:05:45.510 | INFO | PerfReport: 31.04%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:05:45.511 | INFO | PerfReport: 2.14%, LockUpload -- 2.26e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 08:05:45.511 | INFO | PerfReport: 2.50%, LockDownload -- 2.64e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 08:05:45.511 | INFO | PerfReport: 15.40%, LockCompute -- 1.63e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 08:05:45.511 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 3.81e-02 / 0.00e+00 | 20260508 08:05:45.606 | INFO | Starting a round of Baz IO | 20260508 08:05:49.036 | INFO | Populating RTMetrics report | 20260508 08:05:49.037 | INFO | RTMetrics report populated | 20260508 08:05:49.037 | INFO | Opening tmp RTMetrics json file | 20260508 08:05:49.047 | INFO | Serializing RTMetrics to json | 20260508 08:05:49.048 | INFO | Serialization done, swapping tmp file now... | 20260508 08:05:49.049 | INFO | RTMetrics done | 20260508 08:05:49.105 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.498 seconds | 20260508 08:05:49.105 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.53% | 20260508 08:05:49.105 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.310473% / 0% / 7.54992e-05% | 20260508 08:05:49.105 | INFO | Baz IO overall: | 20260508 08:05:49.105 | INFO | BytesWritten : 2874213606267 | 20260508 08:05:49.105 | INFO | NumEvents : 1384105261288 | 20260508 08:05:50.653 | INFO | Analyzing chunk frames = [6537216,6537728) | 20260508 08:05:50.653 | INFO | MemInfo: { | 20260508 08:05:50.653 | INFO | "gpu" : " 17 GB", | 20260508 08:05:50.653 | INFO | "huge" : "", | 20260508 08:05:50.653 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:50.653 | INFO | "malloc" : " 14 GB", | 20260508 08:05:50.653 | INFO | "pinned" : " 29 GB" | 20260508 08:05:50.653 | INFO | } | 20260508 08:05:50.653 | INFO | Perf report for the last 5.14418 seconds: | 20260508 08:05:50.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:50.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:50.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:50.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:50.653 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 08:05:50.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:50.653 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 08:05:50.653 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 08:05:50.653 | INFO | BAZWRITER : 67.4%, 2.6, 90.4ms, 0 | 20260508 08:05:50.653 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 08:05:50.653 | INFO | Uploaded 36GiB to GPU in 2.57649s (13.9725GiB/s) | 20260508 08:05:50.653 | INFO | Downloaded 38GiB to GPU in 2.70814s (14.0318GiB/s) | 20260508 08:05:55.792 | INFO | Analyzing chunk frames = [6537728,6538240) | 20260508 08:05:55.792 | INFO | MemInfo: { | 20260508 08:05:55.792 | INFO | "gpu" : " 17 GB", | 20260508 08:05:55.792 | INFO | "huge" : "", | 20260508 08:05:55.792 | INFO | "huge_shared" : " 29 GB", | 20260508 08:05:55.792 | INFO | "malloc" : " 14 GB", | 20260508 08:05:55.792 | INFO | "pinned" : " 29 GB" | 20260508 08:05:55.792 | INFO | } | 20260508 08:05:55.792 | INFO | Perf report for the last 5.13879 seconds: | 20260508 08:05:55.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:05:55.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:05:55.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:05:55.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:05:55.792 | INFO | Uploaded 36GiB to GPU in 2.46683s (14.5936GiB/s) | 20260508 08:05:55.792 | INFO | Downloaded 34GiB to GPU in 2.43228s (13.9786GiB/s) | 20260508 08:06:00.931 | INFO | Analyzing chunk frames = [6538240,6538752) | 20260508 08:06:00.931 | INFO | MemInfo: { | 20260508 08:06:00.931 | INFO | "gpu" : " 17 GB", | 20260508 08:06:00.931 | INFO | "huge" : "", | 20260508 08:06:00.931 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:00.931 | INFO | "malloc" : " 14 GB", | 20260508 08:06:00.931 | INFO | "pinned" : " 29 GB" | 20260508 08:06:00.931 | INFO | } | 20260508 08:06:00.931 | INFO | Perf report for the last 5.13935 seconds: | 20260508 08:06:00.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:00.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.931 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 08:06:00.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.931 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:06:00.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:00.932 | INFO | Uploaded 36GiB to GPU in 2.48566s (14.4831GiB/s) | 20260508 08:06:00.932 | INFO | Downloaded 34GiB to GPU in 2.436s (13.9573GiB/s) | 20260508 08:06:06.066 | INFO | Analyzing chunk frames = [6538752,6539264) | 20260508 08:06:06.066 | INFO | MemInfo: { | 20260508 08:06:06.066 | INFO | "gpu" : " 17 GB", | 20260508 08:06:06.066 | INFO | "huge" : "", | 20260508 08:06:06.066 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:06.066 | INFO | "malloc" : " 14 GB", | 20260508 08:06:06.066 | INFO | "pinned" : " 29 GB" | 20260508 08:06:06.066 | INFO | } | 20260508 08:06:06.067 | INFO | Perf report for the last 5.13529 seconds: | 20260508 08:06:06.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:06.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 08:06:06.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:06:06.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:06.067 | INFO | Uploaded 36GiB to GPU in 2.49579s (14.4243GiB/s) | 20260508 08:06:06.067 | INFO | Downloaded 34GiB to GPU in 2.44211s (13.9224GiB/s) | 20260508 08:06:11.210 | INFO | Analyzing chunk frames = [6539264,6539776) | 20260508 08:06:11.210 | INFO | MemInfo: { | 20260508 08:06:11.210 | INFO | "gpu" : " 17 GB", | 20260508 08:06:11.210 | INFO | "huge" : "", | 20260508 08:06:11.210 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:11.210 | INFO | "malloc" : " 14 GB", | 20260508 08:06:11.210 | INFO | "pinned" : " 29 GB" | 20260508 08:06:11.210 | INFO | } | 20260508 08:06:11.210 | INFO | Perf report for the last 5.14385 seconds: | 20260508 08:06:11.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:11.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.210 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 08:06:11.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.210 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:06:11.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:11.211 | INFO | Uploaded 36GiB to GPU in 2.45065s (14.69GiB/s) | 20260508 08:06:11.211 | INFO | Downloaded 34GiB to GPU in 2.52376s (13.472GiB/s) | 20260508 08:06:16.354 | INFO | Analyzing chunk frames = [6539776,6540288) | 20260508 08:06:16.354 | INFO | MemInfo: { | 20260508 08:06:16.354 | INFO | "gpu" : " 17 GB", | 20260508 08:06:16.354 | INFO | "huge" : "", | 20260508 08:06:16.354 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:16.354 | INFO | "malloc" : " 14 GB", | 20260508 08:06:16.354 | INFO | "pinned" : " 29 GB" | 20260508 08:06:16.354 | INFO | } | 20260508 08:06:16.354 | INFO | Perf report for the last 5.14384 seconds: | 20260508 08:06:16.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:16.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.354 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:06:16.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.354 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 08:06:16.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:16.355 | INFO | Uploaded 36GiB to GPU in 2.36968s (15.1919GiB/s) | 20260508 08:06:16.355 | INFO | Downloaded 34GiB to GPU in 2.43159s (13.9826GiB/s) | 20260508 08:06:21.494 | INFO | Analyzing chunk frames = [6540288,6540800) | 20260508 08:06:21.494 | INFO | MemInfo: { | 20260508 08:06:21.494 | INFO | "gpu" : " 17 GB", | 20260508 08:06:21.494 | INFO | "huge" : "", | 20260508 08:06:21.494 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:21.494 | INFO | "malloc" : " 14 GB", | 20260508 08:06:21.494 | INFO | "pinned" : " 29 GB" | 20260508 08:06:21.494 | INFO | } | 20260508 08:06:21.494 | INFO | Perf report for the last 5.14019 seconds: | 20260508 08:06:21.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:21.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.494 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:06:21.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.495 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:06:21.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:21.495 | INFO | Uploaded 36GiB to GPU in 2.41169s (14.9273GiB/s) | 20260508 08:06:21.495 | INFO | Downloaded 34GiB to GPU in 2.4308s (13.9872GiB/s) | 20260508 08:06:26.629 | INFO | Analyzing chunk frames = [6540800,6541312) | 20260508 08:06:26.629 | INFO | MemInfo: { | 20260508 08:06:26.629 | INFO | "gpu" : " 17 GB", | 20260508 08:06:26.629 | INFO | "huge" : "", | 20260508 08:06:26.629 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:26.629 | INFO | "malloc" : " 14 GB", | 20260508 08:06:26.629 | INFO | "pinned" : " 29 GB" | 20260508 08:06:26.629 | INFO | } | 20260508 08:06:26.629 | INFO | Perf report for the last 5.13486 seconds: | 20260508 08:06:26.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:26.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.629 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 08:06:26.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.629 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:06:26.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:26.630 | INFO | Uploaded 36GiB to GPU in 2.39735s (15.0166GiB/s) | 20260508 08:06:26.630 | INFO | Downloaded 34GiB to GPU in 2.42509s (14.0201GiB/s) | 20260508 08:06:26.727 | INFO | Starting a round of Baz IO | 20260508 08:06:30.149 | INFO | Populating RTMetrics report | 20260508 08:06:30.149 | INFO | RTMetrics report populated | 20260508 08:06:30.149 | INFO | Opening tmp RTMetrics json file | 20260508 08:06:30.157 | INFO | Serializing RTMetrics to json | 20260508 08:06:30.158 | INFO | Serialization done, swapping tmp file now... | 20260508 08:06:30.158 | INFO | RTMetrics done | 20260508 08:06:30.215 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.488 seconds | 20260508 08:06:30.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.00% / 59.53% | 20260508 08:06:30.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31058% / 1.19209e-05% / 6.75519e-05% | 20260508 08:06:30.215 | INFO | Baz IO overall: | 20260508 08:06:30.215 | INFO | BytesWritten : 2875836563872 | 20260508 08:06:30.215 | INFO | NumEvents : 1384874689496 | 20260508 08:06:31.770 | INFO | Analyzing chunk frames = [6541312,6541824) | 20260508 08:06:31.770 | INFO | MemInfo: { | 20260508 08:06:31.770 | INFO | "gpu" : " 17 GB", | 20260508 08:06:31.770 | INFO | "huge" : "", | 20260508 08:06:31.770 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:31.770 | INFO | "malloc" : " 14 GB", | 20260508 08:06:31.770 | INFO | "pinned" : " 29 GB" | 20260508 08:06:31.770 | INFO | } | 20260508 08:06:31.770 | INFO | Perf report for the last 5.14095 seconds: | 20260508 08:06:31.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:31.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:31.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:31.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:31.770 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:06:31.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:31.770 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:06:31.770 | INFO | PRE_HQ : 65.3%, 1.6, 55.0ms, 0 | 20260508 08:06:31.770 | INFO | BAZWRITER : 67.2%, 2.6, 88.8ms, 0 | 20260508 08:06:31.770 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:06:31.771 | INFO | Uploaded 36GiB to GPU in 2.58631s (13.9195GiB/s) | 20260508 08:06:31.771 | INFO | Downloaded 38GiB to GPU in 2.71137s (14.0151GiB/s) | 20260508 08:06:36.906 | INFO | Analyzing chunk frames = [6541824,6542336) | 20260508 08:06:36.906 | INFO | MemInfo: { | 20260508 08:06:36.906 | INFO | "gpu" : " 17 GB", | 20260508 08:06:36.906 | INFO | "huge" : "", | 20260508 08:06:36.906 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:36.906 | INFO | "malloc" : " 14 GB", | 20260508 08:06:36.906 | INFO | "pinned" : " 29 GB" | 20260508 08:06:36.907 | INFO | } | 20260508 08:06:36.907 | INFO | Perf report for the last 5.13636 seconds: | 20260508 08:06:36.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:36.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 08:06:36.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:06:36.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:36.907 | INFO | Uploaded 36GiB to GPU in 2.51538s (14.312GiB/s) | 20260508 08:06:36.907 | INFO | Downloaded 34GiB to GPU in 2.44891s (13.8837GiB/s) | 20260508 08:06:42.047 | INFO | Analyzing chunk frames = [6542336,6542848) | 20260508 08:06:42.047 | INFO | MemInfo: { | 20260508 08:06:42.047 | INFO | "gpu" : " 17 GB", | 20260508 08:06:42.047 | INFO | "huge" : "", | 20260508 08:06:42.047 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:42.047 | INFO | "malloc" : " 14 GB", | 20260508 08:06:42.047 | INFO | "pinned" : " 29 GB" | 20260508 08:06:42.047 | INFO | } | 20260508 08:06:42.047 | INFO | Perf report for the last 5.14011 seconds: | 20260508 08:06:42.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:42.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 08:06:42.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:06:42.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:42.047 | INFO | Uploaded 36GiB to GPU in 2.55706s (14.0787GiB/s) | 20260508 08:06:42.047 | INFO | Downloaded 34GiB to GPU in 2.43412s (13.9681GiB/s) | 20260508 08:06:47.185 | INFO | Analyzing chunk frames = [6542848,6543360) | 20260508 08:06:47.185 | INFO | MemInfo: { | 20260508 08:06:47.185 | INFO | "gpu" : " 17 GB", | 20260508 08:06:47.185 | INFO | "huge" : "", | 20260508 08:06:47.185 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:47.185 | INFO | "malloc" : " 14 GB", | 20260508 08:06:47.185 | INFO | "pinned" : " 29 GB" | 20260508 08:06:47.185 | INFO | } | 20260508 08:06:47.186 | INFO | Perf report for the last 5.13874 seconds: | 20260508 08:06:47.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:47.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 08:06:47.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:06:47.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:47.186 | INFO | Uploaded 36GiB to GPU in 2.55565s (14.0865GiB/s) | 20260508 08:06:47.186 | INFO | Downloaded 34GiB to GPU in 2.40954s (14.1106GiB/s) | 20260508 08:06:52.321 | INFO | Analyzing chunk frames = [6543360,6543872) | 20260508 08:06:52.321 | INFO | MemInfo: { | 20260508 08:06:52.321 | INFO | "gpu" : " 17 GB", | 20260508 08:06:52.321 | INFO | "huge" : "", | 20260508 08:06:52.321 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:52.321 | INFO | "malloc" : " 14 GB", | 20260508 08:06:52.321 | INFO | "pinned" : " 29 GB" | 20260508 08:06:52.321 | INFO | } | 20260508 08:06:52.321 | INFO | Perf report for the last 5.13554 seconds: | 20260508 08:06:52.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:52.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 08:06:52.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 08:06:52.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:52.321 | INFO | Uploaded 36GiB to GPU in 2.30804s (15.5977GiB/s) | 20260508 08:06:52.321 | INFO | Downloaded 34GiB to GPU in 2.50258s (13.586GiB/s) | 20260508 08:06:57.466 | INFO | Analyzing chunk frames = [6543872,6544384) | 20260508 08:06:57.466 | INFO | MemInfo: { | 20260508 08:06:57.466 | INFO | "gpu" : " 17 GB", | 20260508 08:06:57.466 | INFO | "huge" : "", | 20260508 08:06:57.466 | INFO | "huge_shared" : " 29 GB", | 20260508 08:06:57.466 | INFO | "malloc" : " 14 GB", | 20260508 08:06:57.466 | INFO | "pinned" : " 29 GB" | 20260508 08:06:57.466 | INFO | } | 20260508 08:06:57.466 | INFO | Perf report for the last 5.14511 seconds: | 20260508 08:06:57.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:06:57.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.466 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 08:06:57.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.466 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:06:57.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:06:57.467 | INFO | Uploaded 36GiB to GPU in 2.42949s (14.8179GiB/s) | 20260508 08:06:57.467 | INFO | Downloaded 34GiB to GPU in 2.43119s (13.9849GiB/s) | 20260508 08:07:02.602 | INFO | Analyzing chunk frames = [6544384,6544896) | 20260508 08:07:02.602 | INFO | MemInfo: { | 20260508 08:07:02.602 | INFO | "gpu" : " 17 GB", | 20260508 08:07:02.602 | INFO | "huge" : "", | 20260508 08:07:02.602 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:02.602 | INFO | "malloc" : " 14 GB", | 20260508 08:07:02.602 | INFO | "pinned" : " 29 GB" | 20260508 08:07:02.602 | INFO | } | 20260508 08:07:02.602 | INFO | Perf report for the last 5.13595 seconds: | 20260508 08:07:02.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:02.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 08:07:02.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:07:02.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:02.602 | INFO | Uploaded 36GiB to GPU in 2.46093s (14.6286GiB/s) | 20260508 08:07:02.602 | INFO | Downloaded 34GiB to GPU in 2.44209s (13.9225GiB/s) | 20260508 08:07:07.741 | INFO | Analyzing chunk frames = [6544896,6545408) | 20260508 08:07:07.741 | INFO | MemInfo: { | 20260508 08:07:07.741 | INFO | "gpu" : " 17 GB", | 20260508 08:07:07.741 | INFO | "huge" : "", | 20260508 08:07:07.741 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:07.742 | INFO | "malloc" : " 14 GB", | 20260508 08:07:07.742 | INFO | "pinned" : " 29 GB" | 20260508 08:07:07.742 | INFO | } | 20260508 08:07:07.742 | INFO | Perf report for the last 5.13942 seconds: | 20260508 08:07:07.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:07.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 08:07:07.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:07:07.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:07.742 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:07:07.742 | INFO | Total mem usage: 16.7 GiB | 20260508 08:07:07.742 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:07:07.742 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:07:07.742 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:07:07.742 | INFO | High Water Breakdown: | 20260508 08:07:07.742 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:07:07.742 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:07:07.742 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:07:07.742 | INFO | ------------------------------------------ | 20260508 08:07:07.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:07:07.742 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:07:07.742 | INFO | Total mem usage: 26.5 GiB | 20260508 08:07:07.742 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:07:07.742 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:07:07.742 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:07:07.742 | INFO | High Water Breakdown: | 20260508 08:07:07.742 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:07:07.742 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:07:07.742 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:07:07.742 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:07:07.742 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:07:07.742 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:07:07.742 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:07:07.742 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:07:07.742 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:07:07.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:07:07.742 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:07:07.742 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:07:07.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:07:07.742 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:07:07.742 | INFO | ------------------------------------------ | 20260508 08:07:07.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:07:07.742 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:07:07.742 | INFO | Total mem usage: 27.8 GiB | 20260508 08:07:07.742 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:07:07.742 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:07:07.742 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:07:07.742 | INFO | High Water Breakdown: | 20260508 08:07:07.742 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:07:07.742 | INFO | ------------------------------------------ | 20260508 08:07:07.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:07:07.742 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:07:07.742 | INFO | Total mem usage: 15.7 GiB | 20260508 08:07:07.742 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:07:07.742 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:07:07.742 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:07:07.742 | INFO | High Water Breakdown: | 20260508 08:07:07.742 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:07:07.742 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:07:07.742 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:07:07.742 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:07:07.742 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:07:07.742 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:07:07.742 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:07:07.742 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:07:07.742 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:07:07.742 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:07:07.742 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:07:07.742 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:07:07.742 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:07:07.742 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:07:07.742 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:07:07.742 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:07:07.742 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:07:07.742 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:07:07.742 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:07:07.742 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:07:07.742 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:07:07.742 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:07:07.742 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:07:07.742 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:07:07.742 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:07:07.742 | INFO | ------------------------------------------ | 20260508 08:07:07.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:07:07.742 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:07:07.742 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:07:07.742 | INFO | Uploaded 36GiB to GPU in 2.50715s (14.3589GiB/s) | 20260508 08:07:07.742 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:07:07.743 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 6.53e-01 | 20260508 08:07:07.743 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.12e-02 / 4.56e-03 | 20260508 08:07:07.743 | INFO | Downloaded 34GiB to GPU in 2.44784s (13.8898GiB/s) | 20260508 08:07:07.743 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.45e+00, 5.96e-01 / 2.10e-01 | 20260508 08:07:07.743 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.44e+00, 2.09e-01 / 1.15e-01 | 20260508 08:07:07.743 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.52e+01, 3.73e+00 / 1.11e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.86e+00 / 1.23e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.35e+00 / 2.56e+00, 4.06e-01 / 1.94e-01 | 20260508 08:07:07.743 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.17e+00, 2.51e-01 / 9.76e-02 | 20260508 08:07:07.743 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.39e-01, 4.98e-02 / 2.68e-02 | 20260508 08:07:07.743 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:07:07.743 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:07:07.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:07:07.743 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 24.37%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 24.42%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 30.98%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 2.25%, LockUpload -- 2.38e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 2.51%, LockDownload -- 2.66e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 15.46%, LockCompute -- 1.63e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:07:07.743 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 08:07:07.840 | INFO | Starting a round of Baz IO | 20260508 08:07:11.253 | INFO | Populating RTMetrics report | 20260508 08:07:11.253 | INFO | RTMetrics report populated | 20260508 08:07:11.253 | INFO | Opening tmp RTMetrics json file | 20260508 08:07:11.264 | INFO | Serializing RTMetrics to json | 20260508 08:07:11.265 | INFO | Serialization done, swapping tmp file now... | 20260508 08:07:11.265 | INFO | RTMetrics done | 20260508 08:07:11.327 | INFO | Finished a round of Baz IO: Wrote 1.512 GiB in 3.486 seconds | 20260508 08:07:11.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.46% / 0.00% / 59.53% | 20260508 08:07:11.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.311085% / 0% / 7.94729e-05% | 20260508 08:07:11.327 | INFO | Baz IO overall: | 20260508 08:07:11.327 | INFO | BytesWritten : 2877460500769 | 20260508 08:07:11.327 | INFO | NumEvents : 1385644586861 | 20260508 08:07:12.876 | INFO | Analyzing chunk frames = [6545408,6545920) | 20260508 08:07:12.876 | INFO | MemInfo: { | 20260508 08:07:12.876 | INFO | "gpu" : " 17 GB", | 20260508 08:07:12.876 | INFO | "huge" : "", | 20260508 08:07:12.876 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:12.876 | INFO | "malloc" : " 14 GB", | 20260508 08:07:12.876 | INFO | "pinned" : " 29 GB" | 20260508 08:07:12.876 | INFO | } | 20260508 08:07:12.876 | INFO | Perf report for the last 5.13449 seconds: | 20260508 08:07:12.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:12.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:12.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:12.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:12.876 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 08:07:12.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:12.876 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:07:12.876 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 08:07:12.876 | INFO | BAZWRITER : 67.3%, 2.6, 88.6ms, 0 | 20260508 08:07:12.876 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:07:12.876 | INFO | Uploaded 36GiB to GPU in 2.58722s (13.9146GiB/s) | 20260508 08:07:12.877 | INFO | Downloaded 38GiB to GPU in 2.71557s (13.9934GiB/s) | 20260508 08:07:18.019 | INFO | Analyzing chunk frames = [6545920,6546432) | 20260508 08:07:18.019 | INFO | MemInfo: { | 20260508 08:07:18.019 | INFO | "gpu" : " 17 GB", | 20260508 08:07:18.019 | INFO | "huge" : "", | 20260508 08:07:18.019 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:18.019 | INFO | "malloc" : " 14 GB", | 20260508 08:07:18.019 | INFO | "pinned" : " 29 GB" | 20260508 08:07:18.019 | INFO | } | 20260508 08:07:18.019 | INFO | Perf report for the last 5.14327 seconds: | 20260508 08:07:18.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:18.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.019 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 08:07:18.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.020 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:07:18.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:18.020 | INFO | Uploaded 36GiB to GPU in 2.52461s (14.2596GiB/s) | 20260508 08:07:18.020 | INFO | Downloaded 34GiB to GPU in 2.43683s (13.9525GiB/s) | 20260508 08:07:23.164 | INFO | Analyzing chunk frames = [6546432,6546944) | 20260508 08:07:23.164 | INFO | MemInfo: { | 20260508 08:07:23.164 | INFO | "gpu" : " 17 GB", | 20260508 08:07:23.164 | INFO | "huge" : "", | 20260508 08:07:23.164 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:23.164 | INFO | "malloc" : " 14 GB", | 20260508 08:07:23.164 | INFO | "pinned" : " 29 GB" | 20260508 08:07:23.164 | INFO | } | 20260508 08:07:23.164 | INFO | Perf report for the last 5.14464 seconds: | 20260508 08:07:23.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:23.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.164 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 08:07:23.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.164 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:07:23.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:23.165 | INFO | Uploaded 36GiB to GPU in 2.57651s (13.9724GiB/s) | 20260508 08:07:23.165 | INFO | Downloaded 34GiB to GPU in 2.42944s (13.995GiB/s) | 20260508 08:07:28.308 | INFO | Analyzing chunk frames = [6546944,6547456) | 20260508 08:07:28.308 | INFO | MemInfo: { | 20260508 08:07:28.308 | INFO | "gpu" : " 17 GB", | 20260508 08:07:28.308 | INFO | "huge" : "", | 20260508 08:07:28.308 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:28.308 | INFO | "malloc" : " 14 GB", | 20260508 08:07:28.308 | INFO | "pinned" : " 29 GB" | 20260508 08:07:28.308 | INFO | } | 20260508 08:07:28.308 | INFO | Perf report for the last 5.1438 seconds: | 20260508 08:07:28.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:28.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:07:28.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:07:28.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:28.308 | INFO | Uploaded 36GiB to GPU in 2.52428s (14.2615GiB/s) | 20260508 08:07:28.308 | INFO | Downloaded 34GiB to GPU in 2.43581s (13.9584GiB/s) | 20260508 08:07:33.439 | INFO | Analyzing chunk frames = [6547456,6547968) | 20260508 08:07:33.439 | INFO | MemInfo: { | 20260508 08:07:33.439 | INFO | "gpu" : " 17 GB", | 20260508 08:07:33.439 | INFO | "huge" : "", | 20260508 08:07:33.439 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:33.439 | INFO | "malloc" : " 14 GB", | 20260508 08:07:33.439 | INFO | "pinned" : " 29 GB" | 20260508 08:07:33.439 | INFO | } | 20260508 08:07:33.439 | INFO | Perf report for the last 5.13081 seconds: | 20260508 08:07:33.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:33.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:07:33.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 08:07:33.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:33.439 | INFO | Uploaded 36GiB to GPU in 2.45325s (14.6744GiB/s) | 20260508 08:07:33.439 | INFO | Downloaded 34GiB to GPU in 2.53252s (13.4254GiB/s) | 20260508 08:07:38.578 | INFO | Analyzing chunk frames = [6547968,6548480) | 20260508 08:07:38.578 | INFO | MemInfo: { | 20260508 08:07:38.578 | INFO | "gpu" : " 17 GB", | 20260508 08:07:38.578 | INFO | "huge" : "", | 20260508 08:07:38.578 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:38.578 | INFO | "malloc" : " 14 GB", | 20260508 08:07:38.578 | INFO | "pinned" : " 29 GB" | 20260508 08:07:38.578 | INFO | } | 20260508 08:07:38.578 | INFO | Perf report for the last 5.13917 seconds: | 20260508 08:07:38.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:38.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:07:38.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:07:38.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:38.578 | INFO | Uploaded 36GiB to GPU in 2.37901s (15.1324GiB/s) | 20260508 08:07:38.578 | INFO | Downloaded 34GiB to GPU in 2.43438s (13.9666GiB/s) | 20260508 08:07:43.717 | INFO | Analyzing chunk frames = [6548480,6548992) | 20260508 08:07:43.717 | INFO | MemInfo: { | 20260508 08:07:43.717 | INFO | "gpu" : " 17 GB", | 20260508 08:07:43.717 | INFO | "huge" : "", | 20260508 08:07:43.717 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:43.717 | INFO | "malloc" : " 14 GB", | 20260508 08:07:43.717 | INFO | "pinned" : " 29 GB" | 20260508 08:07:43.717 | INFO | } | 20260508 08:07:43.717 | INFO | Perf report for the last 5.1394 seconds: | 20260508 08:07:43.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:43.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.717 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:07:43.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.717 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:07:43.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:43.718 | INFO | Uploaded 36GiB to GPU in 2.43794s (14.7666GiB/s) | 20260508 08:07:43.718 | INFO | Downloaded 34GiB to GPU in 2.43745s (13.949GiB/s) | 20260508 08:07:48.862 | INFO | Analyzing chunk frames = [6548992,6549504) | 20260508 08:07:48.862 | INFO | MemInfo: { | 20260508 08:07:48.862 | INFO | "gpu" : " 17 GB", | 20260508 08:07:48.862 | INFO | "huge" : "", | 20260508 08:07:48.862 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:48.862 | INFO | "malloc" : " 14 GB", | 20260508 08:07:48.862 | INFO | "pinned" : " 29 GB" | 20260508 08:07:48.862 | INFO | } | 20260508 08:07:48.862 | INFO | Perf report for the last 5.14508 seconds: | 20260508 08:07:48.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:48.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:07:48.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:07:48.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:48.863 | INFO | Uploaded 36GiB to GPU in 2.43722s (14.7709GiB/s) | 20260508 08:07:48.863 | INFO | Downloaded 34GiB to GPU in 2.44617s (13.8993GiB/s) | 20260508 08:07:48.960 | INFO | Starting a round of Baz IO | 20260508 08:07:52.365 | INFO | Populating RTMetrics report | 20260508 08:07:52.366 | INFO | RTMetrics report populated | 20260508 08:07:52.366 | INFO | Opening tmp RTMetrics json file | 20260508 08:07:52.377 | INFO | Serializing RTMetrics to json | 20260508 08:07:52.378 | INFO | Serialization done, swapping tmp file now... | 20260508 08:07:52.378 | INFO | RTMetrics done | 20260508 08:07:52.438 | INFO | Finished a round of Baz IO: Wrote 1.513 GiB in 3.478 seconds | 20260508 08:07:52.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.46% / 0.00% / 59.54% | 20260508 08:07:52.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.311613% / 3.97364e-06% / 6.35783e-05% | 20260508 08:07:52.438 | INFO | Baz IO overall: | 20260508 08:07:52.438 | INFO | BytesWritten : 2879084903739 | 20260508 08:07:52.438 | INFO | NumEvents : 1386415173807 | 20260508 08:07:53.999 | INFO | Analyzing chunk frames = [6549504,6550016) | 20260508 08:07:53.999 | INFO | MemInfo: { | 20260508 08:07:53.999 | INFO | "gpu" : " 17 GB", | 20260508 08:07:53.999 | INFO | "huge" : "", | 20260508 08:07:53.999 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:53.999 | INFO | "malloc" : " 14 GB", | 20260508 08:07:53.999 | INFO | "pinned" : " 29 GB" | 20260508 08:07:53.999 | INFO | } | 20260508 08:07:53.999 | INFO | Perf report for the last 5.1364 seconds: | 20260508 08:07:53.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:53.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:53.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:53.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:53.999 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 08:07:53.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:53.999 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:07:53.999 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:07:53.999 | INFO | BAZWRITER : 67.2%, 2.6, 88.3ms, 0 | 20260508 08:07:53.999 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 08:07:53.999 | INFO | Uploaded 36GiB to GPU in 2.57285s (13.9923GiB/s) | 20260508 08:07:53.999 | INFO | Downloaded 38GiB to GPU in 2.71711s (13.9855GiB/s) | 20260508 08:07:59.143 | INFO | Analyzing chunk frames = [6550016,6550528) | 20260508 08:07:59.143 | INFO | MemInfo: { | 20260508 08:07:59.143 | INFO | "gpu" : " 17 GB", | 20260508 08:07:59.143 | INFO | "huge" : "", | 20260508 08:07:59.143 | INFO | "huge_shared" : " 29 GB", | 20260508 08:07:59.143 | INFO | "malloc" : " 14 GB", | 20260508 08:07:59.143 | INFO | "pinned" : " 29 GB" | 20260508 08:07:59.143 | INFO | } | 20260508 08:07:59.143 | INFO | Perf report for the last 5.14425 seconds: | 20260508 08:07:59.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:07:59.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.143 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 08:07:59.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.143 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 08:07:59.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:07:59.144 | INFO | Uploaded 36GiB to GPU in 2.50464s (14.3733GiB/s) | 20260508 08:07:59.144 | INFO | Downloaded 34GiB to GPU in 2.44154s (13.9256GiB/s) | 20260508 08:08:04.278 | INFO | Analyzing chunk frames = [6550528,6551040) | 20260508 08:08:04.278 | INFO | MemInfo: { | 20260508 08:08:04.278 | INFO | "gpu" : " 17 GB", | 20260508 08:08:04.279 | INFO | "huge" : "", | 20260508 08:08:04.279 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:04.279 | INFO | "malloc" : " 14 GB", | 20260508 08:08:04.279 | INFO | "pinned" : " 29 GB" | 20260508 08:08:04.279 | INFO | } | 20260508 08:08:04.279 | INFO | Perf report for the last 5.1357 seconds: | 20260508 08:08:04.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:04.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 08:08:04.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | PRE_HQ : 33.6%, 1.0, 17.4ms, 0 | 20260508 08:08:04.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:04.279 | INFO | Uploaded 36GiB to GPU in 2.48631s (14.4793GiB/s) | 20260508 08:08:04.279 | INFO | Downloaded 34GiB to GPU in 2.43986s (13.9352GiB/s) | 20260508 08:08:09.413 | INFO | Analyzing chunk frames = [6551040,6551552) | 20260508 08:08:09.413 | INFO | MemInfo: { | 20260508 08:08:09.413 | INFO | "gpu" : " 17 GB", | 20260508 08:08:09.413 | INFO | "huge" : "", | 20260508 08:08:09.414 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:09.414 | INFO | "malloc" : " 14 GB", | 20260508 08:08:09.414 | INFO | "pinned" : " 29 GB" | 20260508 08:08:09.414 | INFO | } | 20260508 08:08:09.414 | INFO | Perf report for the last 5.13506 seconds: | 20260508 08:08:09.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:09.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:08:09.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 08:08:09.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:09.414 | INFO | Uploaded 36GiB to GPU in 2.49468s (14.4307GiB/s) | 20260508 08:08:09.414 | INFO | Downloaded 34GiB to GPU in 2.42256s (14.0347GiB/s) | 20260508 08:08:14.553 | INFO | Analyzing chunk frames = [6551552,6552064) | 20260508 08:08:14.553 | INFO | MemInfo: { | 20260508 08:08:14.553 | INFO | "gpu" : " 17 GB", | 20260508 08:08:14.553 | INFO | "huge" : "", | 20260508 08:08:14.553 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:14.553 | INFO | "malloc" : " 14 GB", | 20260508 08:08:14.553 | INFO | "pinned" : " 29 GB" | 20260508 08:08:14.553 | INFO | } | 20260508 08:08:14.553 | INFO | Perf report for the last 5.13913 seconds: | 20260508 08:08:14.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:14.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:08:14.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260508 08:08:14.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:14.553 | INFO | Uploaded 36GiB to GPU in 2.40725s (14.9548GiB/s) | 20260508 08:08:14.553 | INFO | Downloaded 34GiB to GPU in 2.52324s (13.4747GiB/s) | 20260508 08:08:19.693 | INFO | Analyzing chunk frames = [6552064,6552576) | 20260508 08:08:19.693 | INFO | MemInfo: { | 20260508 08:08:19.693 | INFO | "gpu" : " 17 GB", | 20260508 08:08:19.693 | INFO | "huge" : "", | 20260508 08:08:19.693 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:19.693 | INFO | "malloc" : " 14 GB", | 20260508 08:08:19.693 | INFO | "pinned" : " 29 GB" | 20260508 08:08:19.693 | INFO | } | 20260508 08:08:19.693 | INFO | Perf report for the last 5.14034 seconds: | 20260508 08:08:19.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:19.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.693 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:08:19.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.693 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:08:19.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:19.694 | INFO | Uploaded 36GiB to GPU in 2.38874s (15.0707GiB/s) | 20260508 08:08:19.694 | INFO | Downloaded 34GiB to GPU in 2.43757s (13.9483GiB/s) | 20260508 08:08:24.827 | INFO | Analyzing chunk frames = [6552576,6553088) | 20260508 08:08:24.827 | INFO | MemInfo: { | 20260508 08:08:24.827 | INFO | "gpu" : " 17 GB", | 20260508 08:08:24.827 | INFO | "huge" : "", | 20260508 08:08:24.827 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:24.827 | INFO | "malloc" : " 14 GB", | 20260508 08:08:24.827 | INFO | "pinned" : " 29 GB" | 20260508 08:08:24.827 | INFO | } | 20260508 08:08:24.827 | INFO | Perf report for the last 5.13405 seconds: | 20260508 08:08:24.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:24.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:08:24.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:08:24.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:24.828 | INFO | Uploaded 36GiB to GPU in 2.41247s (14.9224GiB/s) | 20260508 08:08:24.828 | INFO | Downloaded 34GiB to GPU in 2.44128s (13.9271GiB/s) | 20260508 08:08:29.971 | INFO | Analyzing chunk frames = [6553088,6553600) | 20260508 08:08:29.971 | INFO | MemInfo: { | 20260508 08:08:29.971 | INFO | "gpu" : " 17 GB", | 20260508 08:08:29.971 | INFO | "huge" : "", | 20260508 08:08:29.971 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:29.971 | INFO | "malloc" : " 14 GB", | 20260508 08:08:29.971 | INFO | "pinned" : " 29 GB" | 20260508 08:08:29.971 | INFO | } | 20260508 08:08:29.971 | INFO | Perf report for the last 5.14377 seconds: | 20260508 08:08:29.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:29.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.971 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 08:08:29.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.971 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:08:29.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:29.972 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:08:29.972 | INFO | Total mem usage: 16.7 GiB | 20260508 08:08:29.972 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:08:29.972 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:08:29.972 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:08:29.972 | INFO | High Water Breakdown: | 20260508 08:08:29.972 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:08:29.972 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:08:29.972 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:08:29.972 | INFO | ------------------------------------------ | 20260508 08:08:29.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:08:29.972 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:08:29.972 | INFO | Total mem usage: 26.5 GiB | 20260508 08:08:29.972 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:08:29.972 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:08:29.972 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:08:29.972 | INFO | High Water Breakdown: | 20260508 08:08:29.972 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:08:29.972 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:08:29.972 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:08:29.972 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:08:29.972 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:08:29.972 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:08:29.972 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:08:29.972 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:08:29.972 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:08:29.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:08:29.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:08:29.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:08:29.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:08:29.972 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:08:29.972 | INFO | ------------------------------------------ | 20260508 08:08:29.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:08:29.972 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:08:29.972 | INFO | Total mem usage: 27.8 GiB | 20260508 08:08:29.972 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:08:29.972 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:08:29.972 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:08:29.972 | INFO | High Water Breakdown: | 20260508 08:08:29.972 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:08:29.972 | INFO | ------------------------------------------ | 20260508 08:08:29.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:08:29.972 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:08:29.972 | INFO | Total mem usage: 15.7 GiB | 20260508 08:08:29.972 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:08:29.972 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:08:29.972 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:08:29.972 | INFO | High Water Breakdown: | 20260508 08:08:29.972 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:08:29.972 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:08:29.972 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:08:29.972 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:08:29.972 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:08:29.972 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:08:29.972 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:08:29.972 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:08:29.972 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:08:29.972 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:08:29.972 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:08:29.972 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:08:29.972 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:08:29.972 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:08:29.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:08:29.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:08:29.972 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:08:29.972 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:08:29.972 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:08:29.972 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:08:29.972 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:08:29.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:08:29.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:08:29.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:08:29.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:08:29.972 | INFO | ------------------------------------------ | 20260508 08:08:29.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:08:29.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:08:29.972 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:08:29.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:08:29.972 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.51e+01, 1.38e+01 / 6.94e-01 | 20260508 08:08:29.972 | INFO | PerfReport: 0.30%, AggregateStats -- 9.86e-02 / 1.06e-01, 1.38e-02 / 5.17e-03 | 20260508 08:08:29.972 | INFO | PerfReport: 12.14%, Baseline -- 3.99e+00 / 4.41e+00, 5.92e-01 / 2.70e-01 | 20260508 08:08:29.972 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.08e-01 / 1.38e-01 | 20260508 08:08:29.972 | INFO | PerfReport: 15.88%, DME -- 4.17e+01 / 4.47e+01, 3.41e+00 / 1.13e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.90e+00 / 1.40e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.34e+00 / 2.56e+00, 4.10e-01 / 2.21e-01 | 20260508 08:08:29.972 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.15e+00, 2.50e-01 / 1.31e-01 | 20260508 08:08:29.972 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.09e-01 / 3.42e-01, 4.92e-02 / 2.37e-02 | 20260508 08:08:29.972 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:08:29.972 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:08:29.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:08:29.972 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 24.42%, Upload -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 24.47%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 31.06%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 2.21%, LockUpload -- 2.34e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 2.40%, LockDownload -- 2.54e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 15.42%, LockCompute -- 1.63e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 08:08:29.972 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 3.79e-02 / 0.00e+00 | 20260508 08:08:29.972 | INFO | Uploaded 36GiB to GPU in 2.46842s (14.5842GiB/s) | 20260508 08:08:29.972 | INFO | Downloaded 34GiB to GPU in 2.44215s (13.9222GiB/s) | 20260508 08:08:30.069 | INFO | Starting a round of Baz IO | 20260508 08:08:33.482 | INFO | Populating RTMetrics report | 20260508 08:08:33.482 | INFO | RTMetrics report populated | 20260508 08:08:33.482 | INFO | Opening tmp RTMetrics json file | 20260508 08:08:33.493 | INFO | Serializing RTMetrics to json | 20260508 08:08:33.494 | INFO | Serialization done, swapping tmp file now... | 20260508 08:08:33.495 | INFO | RTMetrics done | 20260508 08:08:33.556 | INFO | Finished a round of Baz IO: Wrote 1.513 GiB in 3.487 seconds | 20260508 08:08:33.557 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.46% / 0.00% / 59.54% | 20260508 08:08:33.557 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.31002% / 0% / 5.5631e-05% | 20260508 08:08:33.557 | INFO | Baz IO overall: | 20260508 08:08:33.557 | INFO | BytesWritten : 2880709161372 | 20260508 08:08:33.557 | INFO | NumEvents : 1387185437716 | 20260508 08:08:35.114 | INFO | Analyzing chunk frames = [6553600,6554112) | 20260508 08:08:35.114 | INFO | MemInfo: { | 20260508 08:08:35.114 | INFO | "gpu" : " 17 GB", | 20260508 08:08:35.114 | INFO | "huge" : "", | 20260508 08:08:35.114 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:35.114 | INFO | "malloc" : " 14 GB", | 20260508 08:08:35.114 | INFO | "pinned" : " 29 GB" | 20260508 08:08:35.114 | INFO | } | 20260508 08:08:35.114 | INFO | Perf report for the last 5.14317 seconds: | 20260508 08:08:35.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:35.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:35.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:35.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:35.115 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:08:35.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:35.115 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:08:35.115 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 08:08:35.115 | INFO | BAZWRITER : 67.3%, 2.6, 89.7ms, 0 | 20260508 08:08:35.115 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:08:35.115 | INFO | Uploaded 36GiB to GPU in 2.564s (14.0406GiB/s) | 20260508 08:08:35.115 | INFO | Downloaded 38GiB to GPU in 2.71744s (13.9838GiB/s) | 20260508 08:08:40.252 | INFO | Analyzing chunk frames = [6554112,6554624) | 20260508 08:08:40.252 | INFO | MemInfo: { | 20260508 08:08:40.252 | INFO | "gpu" : " 17 GB", | 20260508 08:08:40.252 | INFO | "huge" : "", | 20260508 08:08:40.252 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:40.252 | INFO | "malloc" : " 14 GB", | 20260508 08:08:40.252 | INFO | "pinned" : " 29 GB" | 20260508 08:08:40.252 | INFO | } | 20260508 08:08:40.252 | INFO | Perf report for the last 5.13784 seconds: | 20260508 08:08:40.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:40.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.252 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:08:40.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.252 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:08:40.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:40.253 | INFO | Uploaded 36GiB to GPU in 2.52002s (14.2856GiB/s) | 20260508 08:08:40.253 | INFO | Downloaded 34GiB to GPU in 2.44395s (13.9119GiB/s) | 20260508 08:08:45.385 | INFO | Analyzing chunk frames = [6554624,6555136) | 20260508 08:08:45.385 | INFO | MemInfo: { | 20260508 08:08:45.385 | INFO | "gpu" : " 17 GB", | 20260508 08:08:45.385 | INFO | "huge" : "", | 20260508 08:08:45.385 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:45.385 | INFO | "malloc" : " 14 GB", | 20260508 08:08:45.385 | INFO | "pinned" : " 29 GB" | 20260508 08:08:45.385 | INFO | } | 20260508 08:08:45.385 | INFO | Perf report for the last 5.1327 seconds: | 20260508 08:08:45.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:45.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.385 | INFO | ANALYSIS : 67.3%, 3.0, 104.4ms, 0 | 20260508 08:08:45.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.385 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:08:45.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:45.386 | INFO | Uploaded 36GiB to GPU in 2.56223s (14.0503GiB/s) | 20260508 08:08:45.386 | INFO | Downloaded 34GiB to GPU in 2.43318s (13.9735GiB/s) | 20260508 08:08:50.528 | INFO | Analyzing chunk frames = [6555136,6555648) | 20260508 08:08:50.528 | INFO | MemInfo: { | 20260508 08:08:50.528 | INFO | "gpu" : " 17 GB", | 20260508 08:08:50.528 | INFO | "huge" : "", | 20260508 08:08:50.528 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:50.528 | INFO | "malloc" : " 14 GB", | 20260508 08:08:50.528 | INFO | "pinned" : " 29 GB" | 20260508 08:08:50.528 | INFO | } | 20260508 08:08:50.528 | INFO | Perf report for the last 5.14313 seconds: | 20260508 08:08:50.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:50.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:08:50.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260508 08:08:50.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:50.528 | INFO | Uploaded 36GiB to GPU in 2.52013s (14.285GiB/s) | 20260508 08:08:50.529 | INFO | Downloaded 34GiB to GPU in 2.4489s (13.8838GiB/s) | 20260508 08:08:55.672 | INFO | Analyzing chunk frames = [6555648,6556160) | 20260508 08:08:55.672 | INFO | MemInfo: { | 20260508 08:08:55.672 | INFO | "gpu" : " 17 GB", | 20260508 08:08:55.672 | INFO | "huge" : "", | 20260508 08:08:55.672 | INFO | "huge_shared" : " 29 GB", | 20260508 08:08:55.672 | INFO | "malloc" : " 14 GB", | 20260508 08:08:55.672 | INFO | "pinned" : " 29 GB" | 20260508 08:08:55.672 | INFO | } | 20260508 08:08:55.672 | INFO | Perf report for the last 5.14394 seconds: | 20260508 08:08:55.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:08:55.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 08:08:55.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | PRE_HQ : 30.4%, 1.0, 15.9ms, 0 | 20260508 08:08:55.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:08:55.672 | INFO | Uploaded 36GiB to GPU in 2.39912s (15.0055GiB/s) | 20260508 08:08:55.672 | INFO | Downloaded 34GiB to GPU in 2.50262s (13.5858GiB/s) | 20260508 08:09:00.813 | INFO | Analyzing chunk frames = [6556160,6556672) | 20260508 08:09:00.813 | INFO | MemInfo: { | 20260508 08:09:00.813 | INFO | "gpu" : " 17 GB", | 20260508 08:09:00.813 | INFO | "huge" : "", | 20260508 08:09:00.813 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:00.813 | INFO | "malloc" : " 14 GB", | 20260508 08:09:00.813 | INFO | "pinned" : " 29 GB" | 20260508 08:09:00.813 | INFO | } | 20260508 08:09:00.813 | INFO | Perf report for the last 5.14129 seconds: | 20260508 08:09:00.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:00.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.813 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 08:09:00.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.813 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:09:00.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:00.814 | INFO | Uploaded 36GiB to GPU in 2.40365s (14.9772GiB/s) | 20260508 08:09:00.814 | INFO | Downloaded 34GiB to GPU in 2.41346s (14.0877GiB/s) | 20260508 08:09:05.950 | INFO | Analyzing chunk frames = [6556672,6557184) | 20260508 08:09:05.950 | INFO | MemInfo: { | 20260508 08:09:05.950 | INFO | "gpu" : " 17 GB", | 20260508 08:09:05.950 | INFO | "huge" : "", | 20260508 08:09:05.950 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:05.950 | INFO | "malloc" : " 14 GB", | 20260508 08:09:05.950 | INFO | "pinned" : " 29 GB" | 20260508 08:09:05.950 | INFO | } | 20260508 08:09:05.950 | INFO | Perf report for the last 5.13653 seconds: | 20260508 08:09:05.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:05.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:09:05.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:09:05.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:05.950 | INFO | Uploaded 36GiB to GPU in 2.43933s (14.7581GiB/s) | 20260508 08:09:05.950 | INFO | Downloaded 34GiB to GPU in 2.4292s (13.9964GiB/s) | 20260508 08:09:11.084 | INFO | Analyzing chunk frames = [6557184,6557696) | 20260508 08:09:11.084 | INFO | MemInfo: { | 20260508 08:09:11.084 | INFO | "gpu" : " 17 GB", | 20260508 08:09:11.084 | INFO | "huge" : "", | 20260508 08:09:11.084 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:11.084 | INFO | "malloc" : " 14 GB", | 20260508 08:09:11.084 | INFO | "pinned" : " 29 GB" | 20260508 08:09:11.084 | INFO | } | 20260508 08:09:11.084 | INFO | Perf report for the last 5.13382 seconds: | 20260508 08:09:11.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:11.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 08:09:11.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:09:11.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:11.084 | INFO | Uploaded 36GiB to GPU in 2.44518s (14.7229GiB/s) | 20260508 08:09:11.084 | INFO | Downloaded 34GiB to GPU in 2.42463s (14.0228GiB/s) | 20260508 08:09:11.194 | INFO | Starting a round of Baz IO | 20260508 08:09:14.597 | INFO | Populating RTMetrics report | 20260508 08:09:14.598 | INFO | RTMetrics report populated | 20260508 08:09:14.598 | INFO | Opening tmp RTMetrics json file | 20260508 08:09:14.608 | INFO | Serializing RTMetrics to json | 20260508 08:09:14.610 | INFO | Serialization done, swapping tmp file now... | 20260508 08:09:14.611 | INFO | RTMetrics done | 20260508 08:09:14.672 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.478 seconds | 20260508 08:09:14.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.46% / 0.00% / 59.54% | 20260508 08:09:14.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.309499% / 0% / 3.57628e-05% | 20260508 08:09:14.672 | INFO | Baz IO overall: | 20260508 08:09:14.672 | INFO | BytesWritten : 2882332055971 | 20260508 08:09:14.672 | INFO | NumEvents : 1387954773469 | 20260508 08:09:16.231 | INFO | Analyzing chunk frames = [6557696,6558208) | 20260508 08:09:16.231 | INFO | MemInfo: { | 20260508 08:09:16.231 | INFO | "gpu" : " 17 GB", | 20260508 08:09:16.231 | INFO | "huge" : "", | 20260508 08:09:16.231 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:16.231 | INFO | "malloc" : " 14 GB", | 20260508 08:09:16.231 | INFO | "pinned" : " 29 GB" | 20260508 08:09:16.231 | INFO | } | 20260508 08:09:16.231 | INFO | Perf report for the last 5.14763 seconds: | 20260508 08:09:16.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:16.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:16.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:16.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:16.231 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 08:09:16.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:16.231 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:09:16.231 | INFO | PRE_HQ : 65.5%, 1.6, 55.0ms, 0 | 20260508 08:09:16.231 | INFO | BAZWRITER : 67.3%, 2.6, 90.2ms, 0 | 20260508 08:09:16.231 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:09:16.232 | INFO | Uploaded 36GiB to GPU in 2.59659s (13.8643GiB/s) | 20260508 08:09:16.232 | INFO | Downloaded 38GiB to GPU in 2.71754s (13.9832GiB/s) | 20260508 08:09:21.367 | INFO | Analyzing chunk frames = [6558208,6558720) | 20260508 08:09:21.367 | INFO | MemInfo: { | 20260508 08:09:21.367 | INFO | "gpu" : " 17 GB", | 20260508 08:09:21.367 | INFO | "huge" : "", | 20260508 08:09:21.367 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:21.367 | INFO | "malloc" : " 14 GB", | 20260508 08:09:21.367 | INFO | "pinned" : " 29 GB" | 20260508 08:09:21.367 | INFO | } | 20260508 08:09:21.367 | INFO | Perf report for the last 5.1357 seconds: | 20260508 08:09:21.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:21.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | ANALYSIS : 67.1%, 3.0, 104.5ms, 0 | 20260508 08:09:21.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:09:21.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:21.367 | INFO | Uploaded 36GiB to GPU in 2.53978s (14.1744GiB/s) | 20260508 08:09:21.367 | INFO | Downloaded 34GiB to GPU in 2.42488s (14.0213GiB/s) | 20260508 08:09:26.508 | INFO | Analyzing chunk frames = [6558720,6559232) | 20260508 08:09:26.508 | INFO | MemInfo: { | 20260508 08:09:26.508 | INFO | "gpu" : " 17 GB", | 20260508 08:09:26.508 | INFO | "huge" : "", | 20260508 08:09:26.508 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:26.508 | INFO | "malloc" : " 14 GB", | 20260508 08:09:26.508 | INFO | "pinned" : " 29 GB" | 20260508 08:09:26.508 | INFO | } | 20260508 08:09:26.508 | INFO | Perf report for the last 5.14079 seconds: | 20260508 08:09:26.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:26.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 08:09:26.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:09:26.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:26.508 | INFO | Uploaded 36GiB to GPU in 2.57999s (13.9535GiB/s) | 20260508 08:09:26.508 | INFO | Downloaded 34GiB to GPU in 2.43469s (13.9648GiB/s) | 20260508 08:09:31.640 | INFO | Analyzing chunk frames = [6559232,6559744) | 20260508 08:09:31.640 | INFO | MemInfo: { | 20260508 08:09:31.640 | INFO | "gpu" : " 17 GB", | 20260508 08:09:31.640 | INFO | "huge" : "", | 20260508 08:09:31.640 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:31.640 | INFO | "malloc" : " 14 GB", | 20260508 08:09:31.640 | INFO | "pinned" : " 29 GB" | 20260508 08:09:31.640 | INFO | } | 20260508 08:09:31.640 | INFO | Perf report for the last 5.13233 seconds: | 20260508 08:09:31.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:31.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 08:09:31.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260508 08:09:31.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:31.640 | INFO | Uploaded 36GiB to GPU in 2.53082s (14.2247GiB/s) | 20260508 08:09:31.641 | INFO | Downloaded 34GiB to GPU in 2.44653s (13.8972GiB/s) | 20260508 08:09:36.784 | INFO | Analyzing chunk frames = [6559744,6560256) | 20260508 08:09:36.784 | INFO | MemInfo: { | 20260508 08:09:36.784 | INFO | "gpu" : " 17 GB", | 20260508 08:09:36.784 | INFO | "huge" : "", | 20260508 08:09:36.784 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:36.784 | INFO | "malloc" : " 14 GB", | 20260508 08:09:36.784 | INFO | "pinned" : " 29 GB" | 20260508 08:09:36.784 | INFO | } | 20260508 08:09:36.784 | INFO | Perf report for the last 5.14388 seconds: | 20260508 08:09:36.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:36.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 08:09:36.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | PRE_HQ : 30.8%, 1.0, 16.1ms, 0 | 20260508 08:09:36.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:36.784 | INFO | Uploaded 36GiB to GPU in 2.3073s (15.6027GiB/s) | 20260508 08:09:36.784 | INFO | Downloaded 34GiB to GPU in 2.47719s (13.7252GiB/s) | 20260508 08:09:41.922 | INFO | Analyzing chunk frames = [6560256,6560768) | 20260508 08:09:41.922 | INFO | MemInfo: { | 20260508 08:09:41.922 | INFO | "gpu" : " 17 GB", | 20260508 08:09:41.922 | INFO | "huge" : "", | 20260508 08:09:41.922 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:41.922 | INFO | "malloc" : " 14 GB", | 20260508 08:09:41.922 | INFO | "pinned" : " 29 GB" | 20260508 08:09:41.922 | INFO | } | 20260508 08:09:41.922 | INFO | Perf report for the last 5.13777 seconds: | 20260508 08:09:41.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:41.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:09:41.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:09:41.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:41.922 | INFO | Uploaded 36GiB to GPU in 2.41415s (14.9121GiB/s) | 20260508 08:09:41.922 | INFO | Downloaded 34GiB to GPU in 2.43539s (13.9608GiB/s) | 20260508 08:09:47.061 | INFO | Analyzing chunk frames = [6560768,6561280) | 20260508 08:09:47.061 | INFO | MemInfo: { | 20260508 08:09:47.061 | INFO | "gpu" : " 17 GB", | 20260508 08:09:47.061 | INFO | "huge" : "", | 20260508 08:09:47.061 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:47.061 | INFO | "malloc" : " 14 GB", | 20260508 08:09:47.061 | INFO | "pinned" : " 29 GB" | 20260508 08:09:47.061 | INFO | } | 20260508 08:09:47.061 | INFO | Perf report for the last 5.13897 seconds: | 20260508 08:09:47.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:47.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.061 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:09:47.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.061 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:09:47.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:47.062 | INFO | Uploaded 36GiB to GPU in 2.45009s (14.6933GiB/s) | 20260508 08:09:47.062 | INFO | Downloaded 34GiB to GPU in 2.43561s (13.9596GiB/s) | 20260508 08:09:52.201 | INFO | Analyzing chunk frames = [6561280,6561792) | 20260508 08:09:52.201 | INFO | MemInfo: { | 20260508 08:09:52.201 | INFO | "gpu" : " 17 GB", | 20260508 08:09:52.201 | INFO | "huge" : "", | 20260508 08:09:52.201 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:52.201 | INFO | "malloc" : " 14 GB", | 20260508 08:09:52.201 | INFO | "pinned" : " 29 GB" | 20260508 08:09:52.201 | INFO | } | 20260508 08:09:52.201 | INFO | Perf report for the last 5.14005 seconds: | 20260508 08:09:52.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:52.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 08:09:52.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:09:52.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:52.201 | INFO | Uploaded 36GiB to GPU in 2.5048s (14.3724GiB/s) | 20260508 08:09:52.201 | INFO | Downloaded 34GiB to GPU in 2.44481s (13.907GiB/s) | 20260508 08:09:52.201 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:09:52.201 | INFO | Total mem usage: 16.7 GiB | 20260508 08:09:52.201 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:09:52.201 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:09:52.201 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:09:52.201 | INFO | High Water Breakdown: | 20260508 08:09:52.201 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:09:52.201 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:09:52.201 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:09:52.201 | INFO | ------------------------------------------ | 20260508 08:09:52.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:09:52.201 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:09:52.201 | INFO | Total mem usage: 26.5 GiB | 20260508 08:09:52.201 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:09:52.201 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:09:52.201 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:09:52.201 | INFO | High Water Breakdown: | 20260508 08:09:52.201 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:09:52.201 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:09:52.201 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:09:52.201 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:09:52.201 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:09:52.201 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:09:52.201 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:09:52.201 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:09:52.201 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:09:52.201 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:09:52.201 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:09:52.201 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:09:52.201 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:09:52.201 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:09:52.201 | INFO | ------------------------------------------ | 20260508 08:09:52.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:09:52.201 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:09:52.201 | INFO | Total mem usage: 27.8 GiB | 20260508 08:09:52.201 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:09:52.201 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:09:52.201 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:09:52.201 | INFO | High Water Breakdown: | 20260508 08:09:52.201 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:09:52.201 | INFO | ------------------------------------------ | 20260508 08:09:52.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:09:52.201 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:09:52.201 | INFO | Total mem usage: 15.7 GiB | 20260508 08:09:52.201 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:09:52.201 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:09:52.201 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:09:52.201 | INFO | High Water Breakdown: | 20260508 08:09:52.201 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:09:52.201 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:09:52.201 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:09:52.201 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:09:52.201 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:09:52.201 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:09:52.201 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:09:52.201 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:09:52.201 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:09:52.201 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:09:52.201 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:09:52.201 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:09:52.201 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:09:52.201 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:09:52.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:09:52.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:09:52.201 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:09:52.201 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:09:52.201 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:09:52.201 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:09:52.201 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:09:52.201 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:09:52.201 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:09:52.201 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:09:52.201 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:09:52.201 | INFO | ------------------------------------------ | 20260508 08:09:52.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:09:52.201 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:09:52.201 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:09:52.201 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:09:52.201 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 6.13e-01 | 20260508 08:09:52.201 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.11e-02 / 6.34e-03 | 20260508 08:09:52.201 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.46e+00, 5.91e-01 / 2.15e-01 | 20260508 08:09:52.201 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.37e-01 | 20260508 08:09:52.201 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.51e+01, 3.51e+00 / 9.06e-01 | 20260508 08:09:52.201 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.87e+00 / 1.31e+00 | 20260508 08:09:52.201 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.60e+00, 4.11e-01 / 2.24e-01 | 20260508 08:09:52.201 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.20e+00, 2.47e-01 / 1.10e-01 | 20260508 08:09:52.201 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.09e-01 / 3.45e-01, 4.91e-02 / 2.13e-02 | 20260508 08:09:52.201 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:09:52.201 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:09:52.201 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:09:52.201 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:09:52.201 | INFO | PerfReport: 24.48%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 08:09:52.202 | INFO | PerfReport: 24.39%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:09:52.202 | INFO | PerfReport: 30.98%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:09:52.202 | INFO | PerfReport: 2.34%, LockUpload -- 2.47e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 08:09:52.202 | INFO | PerfReport: 2.41%, LockDownload -- 2.54e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260508 08:09:52.202 | INFO | PerfReport: 15.40%, LockCompute -- 1.63e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:09:52.202 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.93e-02 / 0.00e+00 | 20260508 08:09:52.298 | INFO | Starting a round of Baz IO | 20260508 08:09:55.705 | INFO | Populating RTMetrics report | 20260508 08:09:55.705 | INFO | RTMetrics report populated | 20260508 08:09:55.705 | INFO | Opening tmp RTMetrics json file | 20260508 08:09:55.716 | INFO | Serializing RTMetrics to json | 20260508 08:09:55.718 | INFO | Serialization done, swapping tmp file now... | 20260508 08:09:55.718 | INFO | RTMetrics done | 20260508 08:09:55.779 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.481 seconds | 20260508 08:09:55.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.46% / 0.00% / 59.54% | 20260508 08:09:55.780 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307977% / 0% / 4.37101e-05% | 20260508 08:09:55.780 | INFO | Baz IO overall: | 20260508 08:09:55.780 | INFO | BytesWritten : 2883953993248 | 20260508 08:09:55.780 | INFO | NumEvents : 1388723433819 | 20260508 08:09:57.337 | INFO | Analyzing chunk frames = [6561792,6562304) | 20260508 08:09:57.337 | INFO | MemInfo: { | 20260508 08:09:57.337 | INFO | "gpu" : " 17 GB", | 20260508 08:09:57.337 | INFO | "huge" : "", | 20260508 08:09:57.337 | INFO | "huge_shared" : " 29 GB", | 20260508 08:09:57.337 | INFO | "malloc" : " 14 GB", | 20260508 08:09:57.337 | INFO | "pinned" : " 29 GB" | 20260508 08:09:57.337 | INFO | } | 20260508 08:09:57.337 | INFO | Perf report for the last 5.13579 seconds: | 20260508 08:09:57.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:09:57.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:57.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:57.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:57.337 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:09:57.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:09:57.337 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:09:57.337 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 08:09:57.337 | INFO | BAZWRITER : 67.1%, 2.6, 89.4ms, 0 | 20260508 08:09:57.337 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:09:57.337 | INFO | Uploaded 36GiB to GPU in 2.58349s (13.9346GiB/s) | 20260508 08:09:57.337 | INFO | Downloaded 38GiB to GPU in 2.70829s (14.031GiB/s) | 20260508 08:10:02.481 | INFO | Analyzing chunk frames = [6562304,6562816) | 20260508 08:10:02.481 | INFO | MemInfo: { | 20260508 08:10:02.481 | INFO | "gpu" : " 17 GB", | 20260508 08:10:02.481 | INFO | "huge" : "", | 20260508 08:10:02.481 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:02.481 | INFO | "malloc" : " 14 GB", | 20260508 08:10:02.481 | INFO | "pinned" : " 29 GB" | 20260508 08:10:02.481 | INFO | } | 20260508 08:10:02.481 | INFO | Perf report for the last 5.14399 seconds: | 20260508 08:10:02.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:02.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:10:02.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:10:02.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:02.481 | INFO | Uploaded 36GiB to GPU in 2.49271s (14.4421GiB/s) | 20260508 08:10:02.481 | INFO | Downloaded 34GiB to GPU in 2.4492s (13.8821GiB/s) | 20260508 08:10:07.619 | INFO | Analyzing chunk frames = [6562816,6563328) | 20260508 08:10:07.619 | INFO | MemInfo: { | 20260508 08:10:07.619 | INFO | "gpu" : " 17 GB", | 20260508 08:10:07.619 | INFO | "huge" : "", | 20260508 08:10:07.619 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:07.619 | INFO | "malloc" : " 14 GB", | 20260508 08:10:07.619 | INFO | "pinned" : " 29 GB" | 20260508 08:10:07.619 | INFO | } | 20260508 08:10:07.619 | INFO | Perf report for the last 5.13827 seconds: | 20260508 08:10:07.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:07.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 08:10:07.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:10:07.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:07.619 | INFO | Uploaded 36GiB to GPU in 2.5383s (14.1827GiB/s) | 20260508 08:10:07.620 | INFO | Downloaded 34GiB to GPU in 2.43746s (13.949GiB/s) | 20260508 08:10:12.762 | INFO | Analyzing chunk frames = [6563328,6563840) | 20260508 08:10:12.762 | INFO | MemInfo: { | 20260508 08:10:12.762 | INFO | "gpu" : " 17 GB", | 20260508 08:10:12.762 | INFO | "huge" : "", | 20260508 08:10:12.762 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:12.762 | INFO | "malloc" : " 14 GB", | 20260508 08:10:12.762 | INFO | "pinned" : " 29 GB" | 20260508 08:10:12.762 | INFO | } | 20260508 08:10:12.762 | INFO | Perf report for the last 5.14297 seconds: | 20260508 08:10:12.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:12.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | ANALYSIS : 67.0%, 2.9, 103.7ms, 0 | 20260508 08:10:12.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 08:10:12.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:12.762 | INFO | Uploaded 36GiB to GPU in 2.58389s (13.9325GiB/s) | 20260508 08:10:12.763 | INFO | Downloaded 34GiB to GPU in 2.44302s (13.9172GiB/s) | 20260508 08:10:17.898 | INFO | Analyzing chunk frames = [6563840,6564352) | 20260508 08:10:17.898 | INFO | MemInfo: { | 20260508 08:10:17.898 | INFO | "gpu" : " 17 GB", | 20260508 08:10:17.898 | INFO | "huge" : "", | 20260508 08:10:17.898 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:17.898 | INFO | "malloc" : " 14 GB", | 20260508 08:10:17.898 | INFO | "pinned" : " 29 GB" | 20260508 08:10:17.898 | INFO | } | 20260508 08:10:17.899 | INFO | Perf report for the last 5.13651 seconds: | 20260508 08:10:17.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:17.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:10:17.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260508 08:10:17.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:17.899 | INFO | Uploaded 36GiB to GPU in 2.38527s (15.0926GiB/s) | 20260508 08:10:17.899 | INFO | Downloaded 34GiB to GPU in 2.52374s (13.4721GiB/s) | 20260508 08:10:23.035 | INFO | Analyzing chunk frames = [6564352,6564864) | 20260508 08:10:23.035 | INFO | MemInfo: { | 20260508 08:10:23.035 | INFO | "gpu" : " 17 GB", | 20260508 08:10:23.035 | INFO | "huge" : "", | 20260508 08:10:23.035 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:23.035 | INFO | "malloc" : " 14 GB", | 20260508 08:10:23.035 | INFO | "pinned" : " 29 GB" | 20260508 08:10:23.035 | INFO | } | 20260508 08:10:23.035 | INFO | Perf report for the last 5.13638 seconds: | 20260508 08:10:23.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:23.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 08:10:23.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:10:23.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:23.035 | INFO | Uploaded 36GiB to GPU in 2.31306s (15.5638GiB/s) | 20260508 08:10:23.035 | INFO | Downloaded 34GiB to GPU in 2.45455s (13.8518GiB/s) | 20260508 08:10:28.178 | INFO | Analyzing chunk frames = [6564864,6565376) | 20260508 08:10:28.178 | INFO | MemInfo: { | 20260508 08:10:28.178 | INFO | "gpu" : " 17 GB", | 20260508 08:10:28.178 | INFO | "huge" : "", | 20260508 08:10:28.178 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:28.178 | INFO | "malloc" : " 14 GB", | 20260508 08:10:28.178 | INFO | "pinned" : " 29 GB" | 20260508 08:10:28.178 | INFO | } | 20260508 08:10:28.178 | INFO | Perf report for the last 5.14287 seconds: | 20260508 08:10:28.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:28.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:10:28.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:10:28.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:28.178 | INFO | Uploaded 36GiB to GPU in 2.38002s (15.1259GiB/s) | 20260508 08:10:28.178 | INFO | Downloaded 34GiB to GPU in 2.45431s (13.8532GiB/s) | 20260508 08:10:33.323 | INFO | Analyzing chunk frames = [6565376,6565888) | 20260508 08:10:33.323 | INFO | MemInfo: { | 20260508 08:10:33.323 | INFO | "gpu" : " 17 GB", | 20260508 08:10:33.323 | INFO | "huge" : "", | 20260508 08:10:33.323 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:33.323 | INFO | "malloc" : " 14 GB", | 20260508 08:10:33.323 | INFO | "pinned" : " 29 GB" | 20260508 08:10:33.323 | INFO | } | 20260508 08:10:33.323 | INFO | Perf report for the last 5.14553 seconds: | 20260508 08:10:33.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:33.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.323 | INFO | ANALYSIS : 65.7%, 3.0, 102.9ms, 0 | 20260508 08:10:33.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.323 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:10:33.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:33.324 | INFO | Uploaded 36GiB to GPU in 2.43326s (14.795GiB/s) | 20260508 08:10:33.324 | INFO | Downloaded 34GiB to GPU in 2.46363s (13.8008GiB/s) | 20260508 08:10:33.420 | INFO | Starting a round of Baz IO | 20260508 08:10:36.829 | INFO | Populating RTMetrics report | 20260508 08:10:36.829 | INFO | RTMetrics report populated | 20260508 08:10:36.830 | INFO | Opening tmp RTMetrics json file | 20260508 08:10:36.841 | INFO | Serializing RTMetrics to json | 20260508 08:10:36.842 | INFO | Serialization done, swapping tmp file now... | 20260508 08:10:36.842 | INFO | RTMetrics done | 20260508 08:10:36.903 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.482 seconds | 20260508 08:10:36.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.46% / 0.00% / 59.54% | 20260508 08:10:36.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307592% / 3.97364e-06% / 4.37101e-05% | 20260508 08:10:36.903 | INFO | Baz IO overall: | 20260508 08:10:36.903 | INFO | BytesWritten : 2885576417501 | 20260508 08:10:36.903 | INFO | NumEvents : 1389492685181 | 20260508 08:10:38.458 | INFO | Analyzing chunk frames = [6565888,6566400) | 20260508 08:10:38.458 | INFO | MemInfo: { | 20260508 08:10:38.458 | INFO | "gpu" : " 17 GB", | 20260508 08:10:38.458 | INFO | "huge" : "", | 20260508 08:10:38.458 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:38.458 | INFO | "malloc" : " 14 GB", | 20260508 08:10:38.458 | INFO | "pinned" : " 29 GB" | 20260508 08:10:38.458 | INFO | } | 20260508 08:10:38.458 | INFO | Perf report for the last 5.13506 seconds: | 20260508 08:10:38.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:38.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:38.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:38.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:38.458 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:10:38.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:38.459 | INFO | RT_METRICS : 39.2%, 1.0, 20.5ms, 0 | 20260508 08:10:38.459 | INFO | PRE_HQ : 65.6%, 1.6, 55.1ms, 0 | 20260508 08:10:38.459 | INFO | BAZWRITER : 67.1%, 2.6, 90.3ms, 0 | 20260508 08:10:38.459 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:10:38.459 | INFO | Uploaded 36GiB to GPU in 2.59501s (13.8728GiB/s) | 20260508 08:10:38.459 | INFO | Downloaded 38GiB to GPU in 2.70757s (14.0347GiB/s) | 20260508 08:10:43.592 | INFO | Analyzing chunk frames = [6566400,6566912) | 20260508 08:10:43.592 | INFO | MemInfo: { | 20260508 08:10:43.592 | INFO | "gpu" : " 17 GB", | 20260508 08:10:43.592 | INFO | "huge" : "", | 20260508 08:10:43.592 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:43.592 | INFO | "malloc" : " 14 GB", | 20260508 08:10:43.592 | INFO | "pinned" : " 29 GB" | 20260508 08:10:43.592 | INFO | } | 20260508 08:10:43.592 | INFO | Perf report for the last 5.1337 seconds: | 20260508 08:10:43.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:43.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.592 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:10:43.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.592 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:10:43.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:43.593 | INFO | Uploaded 36GiB to GPU in 2.49914s (14.4049GiB/s) | 20260508 08:10:43.593 | INFO | Downloaded 34GiB to GPU in 2.42843s (14.0008GiB/s) | 20260508 08:10:48.737 | INFO | Analyzing chunk frames = [6566912,6567424) | 20260508 08:10:48.737 | INFO | MemInfo: { | 20260508 08:10:48.737 | INFO | "gpu" : " 17 GB", | 20260508 08:10:48.737 | INFO | "huge" : "", | 20260508 08:10:48.737 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:48.737 | INFO | "malloc" : " 14 GB", | 20260508 08:10:48.737 | INFO | "pinned" : " 29 GB" | 20260508 08:10:48.737 | INFO | } | 20260508 08:10:48.737 | INFO | Perf report for the last 5.14517 seconds: | 20260508 08:10:48.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:48.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.737 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 08:10:48.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.737 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:10:48.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:48.738 | INFO | Uploaded 36GiB to GPU in 2.48549s (14.484GiB/s) | 20260508 08:10:48.738 | INFO | Downloaded 34GiB to GPU in 2.43619s (13.9562GiB/s) | 20260508 08:10:53.876 | INFO | Analyzing chunk frames = [6567424,6567936) | 20260508 08:10:53.876 | INFO | MemInfo: { | 20260508 08:10:53.876 | INFO | "gpu" : " 17 GB", | 20260508 08:10:53.876 | INFO | "huge" : "", | 20260508 08:10:53.876 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:53.876 | INFO | "malloc" : " 14 GB", | 20260508 08:10:53.876 | INFO | "pinned" : " 29 GB" | 20260508 08:10:53.877 | INFO | } | 20260508 08:10:53.877 | INFO | Perf report for the last 5.13982 seconds: | 20260508 08:10:53.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:53.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.877 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:10:53.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.877 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:10:53.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:53.878 | INFO | Uploaded 36GiB to GPU in 2.49654s (14.4199GiB/s) | 20260508 08:10:53.878 | INFO | Downloaded 34GiB to GPU in 2.44594s (13.9006GiB/s) | 20260508 08:10:59.011 | INFO | Analyzing chunk frames = [6567936,6568448) | 20260508 08:10:59.011 | INFO | MemInfo: { | 20260508 08:10:59.011 | INFO | "gpu" : " 17 GB", | 20260508 08:10:59.011 | INFO | "huge" : "", | 20260508 08:10:59.011 | INFO | "huge_shared" : " 29 GB", | 20260508 08:10:59.011 | INFO | "malloc" : " 14 GB", | 20260508 08:10:59.011 | INFO | "pinned" : " 29 GB" | 20260508 08:10:59.012 | INFO | } | 20260508 08:10:59.012 | INFO | Perf report for the last 5.13442 seconds: | 20260508 08:10:59.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:10:59.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 08:10:59.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 08:10:59.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:10:59.012 | INFO | Uploaded 36GiB to GPU in 2.43542s (14.7818GiB/s) | 20260508 08:10:59.012 | INFO | Downloaded 34GiB to GPU in 2.53558s (13.4091GiB/s) | 20260508 08:11:04.148 | INFO | Analyzing chunk frames = [6568448,6568960) | 20260508 08:11:04.148 | INFO | MemInfo: { | 20260508 08:11:04.148 | INFO | "gpu" : " 17 GB", | 20260508 08:11:04.148 | INFO | "huge" : "", | 20260508 08:11:04.148 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:04.148 | INFO | "malloc" : " 14 GB", | 20260508 08:11:04.148 | INFO | "pinned" : " 29 GB" | 20260508 08:11:04.148 | INFO | } | 20260508 08:11:04.148 | INFO | Perf report for the last 5.13643 seconds: | 20260508 08:11:04.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:04.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:11:04.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:11:04.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:04.148 | INFO | Uploaded 36GiB to GPU in 2.38675s (15.0833GiB/s) | 20260508 08:11:04.148 | INFO | Downloaded 34GiB to GPU in 2.44091s (13.9292GiB/s) | 20260508 08:11:09.289 | INFO | Analyzing chunk frames = [6568960,6569472) | 20260508 08:11:09.289 | INFO | MemInfo: { | 20260508 08:11:09.289 | INFO | "gpu" : " 17 GB", | 20260508 08:11:09.289 | INFO | "huge" : "", | 20260508 08:11:09.289 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:09.289 | INFO | "malloc" : " 14 GB", | 20260508 08:11:09.289 | INFO | "pinned" : " 29 GB" | 20260508 08:11:09.289 | INFO | } | 20260508 08:11:09.289 | INFO | Perf report for the last 5.14132 seconds: | 20260508 08:11:09.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:09.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.290 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:11:09.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.290 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:11:09.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:09.290 | INFO | Uploaded 36GiB to GPU in 2.46426s (14.6089GiB/s) | 20260508 08:11:09.290 | INFO | Downloaded 34GiB to GPU in 2.43123s (13.9847GiB/s) | 20260508 08:11:14.430 | INFO | Analyzing chunk frames = [6569472,6569984) | 20260508 08:11:14.430 | INFO | MemInfo: { | 20260508 08:11:14.430 | INFO | "gpu" : " 17 GB", | 20260508 08:11:14.430 | INFO | "huge" : "", | 20260508 08:11:14.430 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:14.430 | INFO | "malloc" : " 14 GB", | 20260508 08:11:14.430 | INFO | "pinned" : " 29 GB" | 20260508 08:11:14.430 | INFO | } | 20260508 08:11:14.430 | INFO | Perf report for the last 5.14086 seconds: | 20260508 08:11:14.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:14.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:11:14.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:11:14.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:14.430 | INFO | Uploaded 36GiB to GPU in 2.47335s (14.5552GiB/s) | 20260508 08:11:14.430 | INFO | Downloaded 34GiB to GPU in 2.44956s (13.8801GiB/s) | 20260508 08:11:14.431 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:11:14.431 | INFO | Total mem usage: 16.7 GiB | 20260508 08:11:14.431 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:11:14.431 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:11:14.431 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:11:14.431 | INFO | High Water Breakdown: | 20260508 08:11:14.431 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:11:14.431 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:11:14.431 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:11:14.431 | INFO | ------------------------------------------ | 20260508 08:11:14.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:11:14.431 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:11:14.431 | INFO | Total mem usage: 26.5 GiB | 20260508 08:11:14.431 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:11:14.431 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:11:14.431 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:11:14.431 | INFO | High Water Breakdown: | 20260508 08:11:14.431 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:11:14.431 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:11:14.431 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:11:14.431 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:11:14.431 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:11:14.431 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:11:14.431 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:11:14.431 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:11:14.431 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:11:14.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:11:14.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:11:14.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:11:14.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:11:14.431 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:11:14.431 | INFO | ------------------------------------------ | 20260508 08:11:14.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:11:14.431 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:11:14.431 | INFO | Total mem usage: 27.8 GiB | 20260508 08:11:14.431 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:11:14.431 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:11:14.431 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:11:14.431 | INFO | High Water Breakdown: | 20260508 08:11:14.431 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:11:14.431 | INFO | ------------------------------------------ | 20260508 08:11:14.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:11:14.431 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:11:14.431 | INFO | Total mem usage: 15.7 GiB | 20260508 08:11:14.431 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:11:14.431 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:11:14.431 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:11:14.431 | INFO | High Water Breakdown: | 20260508 08:11:14.431 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:11:14.431 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:11:14.431 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:11:14.431 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:11:14.431 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:11:14.431 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:11:14.431 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:11:14.431 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:11:14.431 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:11:14.431 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:11:14.431 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:11:14.431 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:11:14.431 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:11:14.431 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:11:14.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:11:14.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:11:14.431 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:11:14.431 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:11:14.431 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:11:14.432 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:11:14.432 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:11:14.432 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:11:14.432 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:11:14.432 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:11:14.432 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:11:14.432 | INFO | ------------------------------------------ | 20260508 08:11:14.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:11:14.432 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:11:14.432 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:11:14.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:11:14.432 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 7.46e-01 | 20260508 08:11:14.432 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.10e-02 / 7.25e-03 | 20260508 08:11:14.432 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.47e+00, 5.89e-01 / 2.19e-01 | 20260508 08:11:14.432 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.45e-01 | 20260508 08:11:14.432 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.48e+01, 3.67e+00 / 1.13e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.90e+00 / 1.46e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.11e-01 / 2.72e-01 | 20260508 08:11:14.432 | INFO | PerfReport: 3.07%, Metrics -- 1.00e+00 / 1.16e+00, 2.51e-01 / 1.45e-01 | 20260508 08:11:14.432 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.41e-01, 4.98e-02 / 3.31e-02 | 20260508 08:11:14.432 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:11:14.432 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:11:14.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:11:14.432 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 24.36%, Upload -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 24.53%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 30.99%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 2.14%, LockUpload -- 2.26e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 2.57%, LockDownload -- 2.71e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 15.39%, LockCompute -- 1.63e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 08:11:14.432 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 08:11:14.527 | INFO | Starting a round of Baz IO | 20260508 08:11:17.942 | INFO | Populating RTMetrics report | 20260508 08:11:17.942 | INFO | RTMetrics report populated | 20260508 08:11:17.942 | INFO | Opening tmp RTMetrics json file | 20260508 08:11:17.954 | INFO | Serializing RTMetrics to json | 20260508 08:11:17.955 | INFO | Serialization done, swapping tmp file now... | 20260508 08:11:17.956 | INFO | RTMetrics done | 20260508 08:11:18.019 | INFO | Finished a round of Baz IO: Wrote 1.51 GiB in 3.491 seconds | 20260508 08:11:18.019 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.54% | 20260508 08:11:18.019 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307767% / 0% / 6.75519e-05% | 20260508 08:11:18.019 | INFO | Baz IO overall: | 20260508 08:11:18.019 | INFO | BytesWritten : 2887198279379 | 20260508 08:11:18.019 | INFO | NumEvents : 1390261393954 | 20260508 08:11:19.573 | INFO | Analyzing chunk frames = [6569984,6570496) | 20260508 08:11:19.573 | INFO | MemInfo: { | 20260508 08:11:19.573 | INFO | "gpu" : " 17 GB", | 20260508 08:11:19.573 | INFO | "huge" : "", | 20260508 08:11:19.573 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:19.573 | INFO | "malloc" : " 14 GB", | 20260508 08:11:19.573 | INFO | "pinned" : " 29 GB" | 20260508 08:11:19.573 | INFO | } | 20260508 08:11:19.573 | INFO | Perf report for the last 5.14301 seconds: | 20260508 08:11:19.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:19.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:19.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:19.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:19.573 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:11:19.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:19.573 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 08:11:19.573 | INFO | PRE_HQ : 65.4%, 1.6, 55.3ms, 0 | 20260508 08:11:19.573 | INFO | BAZWRITER : 67.2%, 2.6, 89.1ms, 0 | 20260508 08:11:19.573 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:11:19.574 | INFO | Uploaded 36GiB to GPU in 2.58187s (13.9434GiB/s) | 20260508 08:11:19.574 | INFO | Downloaded 38GiB to GPU in 2.70741s (14.0355GiB/s) | 20260508 08:11:24.708 | INFO | Analyzing chunk frames = [6570496,6571008) | 20260508 08:11:24.708 | INFO | MemInfo: { | 20260508 08:11:24.708 | INFO | "gpu" : " 17 GB", | 20260508 08:11:24.708 | INFO | "huge" : "", | 20260508 08:11:24.708 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:24.708 | INFO | "malloc" : " 14 GB", | 20260508 08:11:24.708 | INFO | "pinned" : " 29 GB" | 20260508 08:11:24.708 | INFO | } | 20260508 08:11:24.708 | INFO | Perf report for the last 5.13505 seconds: | 20260508 08:11:24.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:24.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.708 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:11:24.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.708 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 08:11:24.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:24.709 | INFO | Uploaded 36GiB to GPU in 2.51809s (14.2965GiB/s) | 20260508 08:11:24.709 | INFO | Downloaded 34GiB to GPU in 2.4599s (13.8217GiB/s) | 20260508 08:11:29.847 | INFO | Analyzing chunk frames = [6571008,6571520) | 20260508 08:11:29.847 | INFO | MemInfo: { | 20260508 08:11:29.847 | INFO | "gpu" : " 17 GB", | 20260508 08:11:29.847 | INFO | "huge" : "", | 20260508 08:11:29.847 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:29.847 | INFO | "malloc" : " 14 GB", | 20260508 08:11:29.847 | INFO | "pinned" : " 29 GB" | 20260508 08:11:29.847 | INFO | } | 20260508 08:11:29.847 | INFO | Perf report for the last 5.13838 seconds: | 20260508 08:11:29.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:29.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:11:29.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:11:29.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:29.847 | INFO | Uploaded 36GiB to GPU in 2.47988s (14.5169GiB/s) | 20260508 08:11:29.847 | INFO | Downloaded 34GiB to GPU in 2.42883s (13.9985GiB/s) | 20260508 08:11:34.990 | INFO | Analyzing chunk frames = [6571520,6572032) | 20260508 08:11:34.990 | INFO | MemInfo: { | 20260508 08:11:34.990 | INFO | "gpu" : " 17 GB", | 20260508 08:11:34.990 | INFO | "huge" : "", | 20260508 08:11:34.990 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:34.990 | INFO | "malloc" : " 14 GB", | 20260508 08:11:34.990 | INFO | "pinned" : " 29 GB" | 20260508 08:11:34.990 | INFO | } | 20260508 08:11:34.990 | INFO | Perf report for the last 5.14293 seconds: | 20260508 08:11:34.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:34.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.990 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 08:11:34.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.990 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:11:34.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:34.991 | INFO | Uploaded 36GiB to GPU in 2.58654s (13.9182GiB/s) | 20260508 08:11:34.991 | INFO | Downloaded 34GiB to GPU in 2.43138s (13.9838GiB/s) | 20260508 08:11:40.137 | INFO | Analyzing chunk frames = [6572032,6572544) | 20260508 08:11:40.137 | INFO | MemInfo: { | 20260508 08:11:40.137 | INFO | "gpu" : " 17 GB", | 20260508 08:11:40.137 | INFO | "huge" : "", | 20260508 08:11:40.137 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:40.137 | INFO | "malloc" : " 14 GB", | 20260508 08:11:40.137 | INFO | "pinned" : " 29 GB" | 20260508 08:11:40.137 | INFO | } | 20260508 08:11:40.137 | INFO | Perf report for the last 5.14771 seconds: | 20260508 08:11:40.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:40.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.137 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 08:11:40.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.137 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:11:40.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:40.138 | INFO | Uploaded 36GiB to GPU in 2.36035s (15.252GiB/s) | 20260508 08:11:40.138 | INFO | Downloaded 34GiB to GPU in 2.52507s (13.465GiB/s) | 20260508 08:11:45.275 | INFO | Analyzing chunk frames = [6572544,6573056) | 20260508 08:11:45.275 | INFO | MemInfo: { | 20260508 08:11:45.275 | INFO | "gpu" : " 17 GB", | 20260508 08:11:45.275 | INFO | "huge" : "", | 20260508 08:11:45.275 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:45.275 | INFO | "malloc" : " 14 GB", | 20260508 08:11:45.275 | INFO | "pinned" : " 29 GB" | 20260508 08:11:45.275 | INFO | } | 20260508 08:11:45.275 | INFO | Perf report for the last 5.13761 seconds: | 20260508 08:11:45.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:45.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 08:11:45.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:11:45.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:45.275 | INFO | Uploaded 36GiB to GPU in 2.35675s (15.2753GiB/s) | 20260508 08:11:45.275 | INFO | Downloaded 34GiB to GPU in 2.43295s (13.9748GiB/s) | 20260508 08:11:50.406 | INFO | Analyzing chunk frames = [6573056,6573568) | 20260508 08:11:50.406 | INFO | MemInfo: { | 20260508 08:11:50.406 | INFO | "gpu" : " 17 GB", | 20260508 08:11:50.406 | INFO | "huge" : "", | 20260508 08:11:50.406 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:50.406 | INFO | "malloc" : " 14 GB", | 20260508 08:11:50.406 | INFO | "pinned" : " 29 GB" | 20260508 08:11:50.406 | INFO | } | 20260508 08:11:50.406 | INFO | Perf report for the last 5.13126 seconds: | 20260508 08:11:50.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:50.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:11:50.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:11:50.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:50.406 | INFO | Uploaded 36GiB to GPU in 2.41109s (14.931GiB/s) | 20260508 08:11:50.406 | INFO | Downloaded 34GiB to GPU in 2.44517s (13.905GiB/s) | 20260508 08:11:55.547 | INFO | Analyzing chunk frames = [6573568,6574080) | 20260508 08:11:55.547 | INFO | MemInfo: { | 20260508 08:11:55.547 | INFO | "gpu" : " 17 GB", | 20260508 08:11:55.547 | INFO | "huge" : "", | 20260508 08:11:55.547 | INFO | "huge_shared" : " 29 GB", | 20260508 08:11:55.547 | INFO | "malloc" : " 14 GB", | 20260508 08:11:55.547 | INFO | "pinned" : " 29 GB" | 20260508 08:11:55.547 | INFO | } | 20260508 08:11:55.547 | INFO | Perf report for the last 5.14084 seconds: | 20260508 08:11:55.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:11:55.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.547 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:11:55.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.547 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:11:55.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:11:55.548 | INFO | Uploaded 36GiB to GPU in 2.42554s (14.8421GiB/s) | 20260508 08:11:55.548 | INFO | Downloaded 34GiB to GPU in 2.44716s (13.8937GiB/s) | 20260508 08:11:55.661 | INFO | Starting a round of Baz IO | 20260508 08:11:59.075 | INFO | Populating RTMetrics report | 20260508 08:11:59.075 | INFO | RTMetrics report populated | 20260508 08:11:59.075 | INFO | Opening tmp RTMetrics json file | 20260508 08:11:59.086 | INFO | Serializing RTMetrics to json | 20260508 08:11:59.088 | INFO | Serialization done, swapping tmp file now... | 20260508 08:11:59.088 | INFO | RTMetrics done | 20260508 08:11:59.141 | INFO | Finished a round of Baz IO: Wrote 1.51 GiB in 3.48 seconds | 20260508 08:11:59.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.55% | 20260508 08:11:59.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.306972% / 3.97364e-06% / 4.37101e-05% | 20260508 08:11:59.142 | INFO | Baz IO overall: | 20260508 08:11:59.142 | INFO | BytesWritten : 2888819634966 | 20260508 08:11:59.142 | INFO | NumEvents : 1391029878342 | 20260508 08:12:00.687 | INFO | Analyzing chunk frames = [6574080,6574592) | 20260508 08:12:00.687 | INFO | MemInfo: { | 20260508 08:12:00.687 | INFO | "gpu" : " 17 GB", | 20260508 08:12:00.687 | INFO | "huge" : "", | 20260508 08:12:00.687 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:00.687 | INFO | "malloc" : " 14 GB", | 20260508 08:12:00.687 | INFO | "pinned" : " 29 GB" | 20260508 08:12:00.687 | INFO | } | 20260508 08:12:00.687 | INFO | Perf report for the last 5.14021 seconds: | 20260508 08:12:00.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:00.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:00.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:00.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:00.687 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 08:12:00.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:00.687 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 08:12:00.687 | INFO | PRE_HQ : 66.2%, 1.7, 59.5ms, 0 | 20260508 08:12:00.688 | INFO | BAZWRITER : 67.2%, 2.7, 93.3ms, 0 | 20260508 08:12:00.688 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:12:00.688 | INFO | Uploaded 36GiB to GPU in 2.5535s (14.0983GiB/s) | 20260508 08:12:00.688 | INFO | Downloaded 38GiB to GPU in 2.708s (14.0325GiB/s) | 20260508 08:12:05.824 | INFO | Analyzing chunk frames = [6574592,6575104) | 20260508 08:12:05.824 | INFO | MemInfo: { | 20260508 08:12:05.824 | INFO | "gpu" : " 17 GB", | 20260508 08:12:05.824 | INFO | "huge" : "", | 20260508 08:12:05.824 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:05.824 | INFO | "malloc" : " 14 GB", | 20260508 08:12:05.824 | INFO | "pinned" : " 29 GB" | 20260508 08:12:05.824 | INFO | } | 20260508 08:12:05.824 | INFO | Perf report for the last 5.13702 seconds: | 20260508 08:12:05.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:05.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.824 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:12:05.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.825 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 08:12:05.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:05.825 | INFO | Uploaded 36GiB to GPU in 2.50335s (14.3807GiB/s) | 20260508 08:12:05.825 | INFO | Downloaded 34GiB to GPU in 2.43852s (13.9429GiB/s) | 20260508 08:12:10.962 | INFO | Analyzing chunk frames = [6575104,6575616) | 20260508 08:12:10.962 | INFO | MemInfo: { | 20260508 08:12:10.962 | INFO | "gpu" : " 17 GB", | 20260508 08:12:10.962 | INFO | "huge" : "", | 20260508 08:12:10.962 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:10.962 | INFO | "malloc" : " 14 GB", | 20260508 08:12:10.963 | INFO | "pinned" : " 29 GB" | 20260508 08:12:10.963 | INFO | } | 20260508 08:12:10.963 | INFO | Perf report for the last 5.13824 seconds: | 20260508 08:12:10.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:10.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 08:12:10.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:12:10.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:10.963 | INFO | Uploaded 36GiB to GPU in 2.56593s (14.03GiB/s) | 20260508 08:12:10.963 | INFO | Downloaded 34GiB to GPU in 2.42967s (13.9937GiB/s) | 20260508 08:12:16.103 | INFO | Analyzing chunk frames = [6575616,6576128) | 20260508 08:12:16.103 | INFO | MemInfo: { | 20260508 08:12:16.103 | INFO | "gpu" : " 17 GB", | 20260508 08:12:16.103 | INFO | "huge" : "", | 20260508 08:12:16.103 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:16.103 | INFO | "malloc" : " 14 GB", | 20260508 08:12:16.103 | INFO | "pinned" : " 29 GB" | 20260508 08:12:16.103 | INFO | } | 20260508 08:12:16.103 | INFO | Perf report for the last 5.14017 seconds: | 20260508 08:12:16.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:16.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 08:12:16.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:12:16.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:16.103 | INFO | Uploaded 36GiB to GPU in 2.57966s (13.9553GiB/s) | 20260508 08:12:16.103 | INFO | Downloaded 34GiB to GPU in 2.43251s (13.9773GiB/s) | 20260508 08:12:21.244 | INFO | Analyzing chunk frames = [6576128,6576640) | 20260508 08:12:21.244 | INFO | MemInfo: { | 20260508 08:12:21.244 | INFO | "gpu" : " 17 GB", | 20260508 08:12:21.244 | INFO | "huge" : "", | 20260508 08:12:21.244 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:21.244 | INFO | "malloc" : " 14 GB", | 20260508 08:12:21.244 | INFO | "pinned" : " 29 GB" | 20260508 08:12:21.244 | INFO | } | 20260508 08:12:21.244 | INFO | Perf report for the last 5.14095 seconds: | 20260508 08:12:21.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:21.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:12:21.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:12:21.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:21.244 | INFO | Uploaded 36GiB to GPU in 2.45422s (14.6686GiB/s) | 20260508 08:12:21.244 | INFO | Downloaded 34GiB to GPU in 2.53279s (13.4239GiB/s) | 20260508 08:12:26.383 | INFO | Analyzing chunk frames = [6576640,6577152) | 20260508 08:12:26.383 | INFO | MemInfo: { | 20260508 08:12:26.383 | INFO | "gpu" : " 17 GB", | 20260508 08:12:26.383 | INFO | "huge" : "", | 20260508 08:12:26.383 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:26.383 | INFO | "malloc" : " 14 GB", | 20260508 08:12:26.383 | INFO | "pinned" : " 29 GB" | 20260508 08:12:26.383 | INFO | } | 20260508 08:12:26.383 | INFO | Perf report for the last 5.13948 seconds: | 20260508 08:12:26.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:26.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.383 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:12:26.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.383 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:12:26.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:26.384 | INFO | Uploaded 36GiB to GPU in 2.39897s (15.0065GiB/s) | 20260508 08:12:26.384 | INFO | Downloaded 34GiB to GPU in 2.41809s (14.0607GiB/s) | 20260508 08:12:31.523 | INFO | Analyzing chunk frames = [6577152,6577664) | 20260508 08:12:31.523 | INFO | MemInfo: { | 20260508 08:12:31.523 | INFO | "gpu" : " 17 GB", | 20260508 08:12:31.523 | INFO | "huge" : "", | 20260508 08:12:31.523 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:31.523 | INFO | "malloc" : " 14 GB", | 20260508 08:12:31.523 | INFO | "pinned" : " 29 GB" | 20260508 08:12:31.523 | INFO | } | 20260508 08:12:31.523 | INFO | Perf report for the last 5.13972 seconds: | 20260508 08:12:31.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:31.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.523 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 08:12:31.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.523 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 08:12:31.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:31.524 | INFO | Uploaded 36GiB to GPU in 2.45867s (14.6421GiB/s) | 20260508 08:12:31.524 | INFO | Downloaded 34GiB to GPU in 2.42898s (13.9976GiB/s) | 20260508 08:12:36.658 | INFO | Analyzing chunk frames = [6577664,6578176) | 20260508 08:12:36.658 | INFO | MemInfo: { | 20260508 08:12:36.658 | INFO | "gpu" : " 17 GB", | 20260508 08:12:36.658 | INFO | "huge" : "", | 20260508 08:12:36.659 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:36.659 | INFO | "malloc" : " 14 GB", | 20260508 08:12:36.659 | INFO | "pinned" : " 29 GB" | 20260508 08:12:36.659 | INFO | } | 20260508 08:12:36.659 | INFO | Perf report for the last 5.13584 seconds: | 20260508 08:12:36.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:36.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 08:12:36.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 08:12:36.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:36.659 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:12:36.659 | INFO | Total mem usage: 16.7 GiB | 20260508 08:12:36.659 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:12:36.659 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 08:12:36.659 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:12:36.659 | INFO | High Water Breakdown: | 20260508 08:12:36.659 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:12:36.659 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:12:36.659 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:12:36.659 | INFO | ------------------------------------------ | 20260508 08:12:36.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:12:36.659 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:12:36.659 | INFO | Total mem usage: 26.5 GiB | 20260508 08:12:36.659 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:12:36.659 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:12:36.659 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:12:36.659 | INFO | High Water Breakdown: | 20260508 08:12:36.659 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:12:36.659 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:12:36.659 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:12:36.659 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:12:36.659 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:12:36.659 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:12:36.659 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:12:36.659 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:12:36.659 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:12:36.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:12:36.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:12:36.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:12:36.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:12:36.659 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:12:36.659 | INFO | ------------------------------------------ | 20260508 08:12:36.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:12:36.659 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:12:36.659 | INFO | Total mem usage: 27.8 GiB | 20260508 08:12:36.659 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:12:36.659 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:12:36.659 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:12:36.659 | INFO | High Water Breakdown: | 20260508 08:12:36.659 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:12:36.659 | INFO | ------------------------------------------ | 20260508 08:12:36.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:12:36.659 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:12:36.659 | INFO | Total mem usage: 15.7 GiB | 20260508 08:12:36.659 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:12:36.659 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:12:36.659 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:12:36.659 | INFO | High Water Breakdown: | 20260508 08:12:36.659 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:12:36.659 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:12:36.660 | INFO | Uploaded 36GiB to GPU in 2.46369s (14.6122GiB/s) | 20260508 08:12:36.660 | INFO | Downloaded 34GiB to GPU in 2.43023s (13.9904GiB/s) | 20260508 08:12:36.659 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:12:36.660 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:12:36.660 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:12:36.660 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:12:36.660 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:12:36.660 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:12:36.660 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:12:36.660 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:12:36.660 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:12:36.660 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:12:36.660 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:12:36.660 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:12:36.660 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:12:36.660 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:12:36.660 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:12:36.660 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:12:36.660 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:12:36.660 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:12:36.660 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:12:36.660 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:12:36.660 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:12:36.660 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:12:36.660 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:12:36.660 | INFO | ------------------------------------------ | 20260508 08:12:36.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:12:36.660 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:12:36.660 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:12:36.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:12:36.660 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.38e-01 | 20260508 08:12:36.660 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.13e-02 / 6.12e-03 | 20260508 08:12:36.660 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.41e+00, 5.96e-01 / 2.14e-01 | 20260508 08:12:36.660 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.19e-01 | 20260508 08:12:36.660 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.51e+01, 3.79e+00 / 1.00e+00 | 20260508 08:12:36.660 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.87e+00 / 1.31e+00 | 20260508 08:12:36.660 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.57e+00, 4.11e-01 / 2.02e-01 | 20260508 08:12:36.660 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.17e+00, 2.49e-01 / 1.15e-01 | 20260508 08:12:36.660 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.39e-01, 4.91e-02 / 2.81e-02 | 20260508 08:12:36.660 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:12:36.660 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:12:36.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:12:36.661 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 24.46%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 24.46%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 30.96%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 2.23%, LockUpload -- 2.36e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 2.54%, LockDownload -- 2.68e+00 / 0.00e+00, 4.02e+00 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 15.33%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:12:36.661 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260508 08:12:36.767 | INFO | Starting a round of Baz IO | 20260508 08:12:40.167 | INFO | Populating RTMetrics report | 20260508 08:12:40.168 | INFO | RTMetrics report populated | 20260508 08:12:40.168 | INFO | Opening tmp RTMetrics json file | 20260508 08:12:40.179 | INFO | Serializing RTMetrics to json | 20260508 08:12:40.180 | INFO | Serialization done, swapping tmp file now... | 20260508 08:12:40.180 | INFO | RTMetrics done | 20260508 08:12:40.238 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.47 seconds | 20260508 08:12:40.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.55% | 20260508 08:12:40.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307119% / 7.94729e-06% / 5.16574e-05% | 20260508 08:12:40.238 | INFO | Baz IO overall: | 20260508 08:12:40.238 | INFO | BytesWritten : 2890442002185 | 20260508 08:12:40.238 | INFO | NumEvents : 1391799666074 | 20260508 08:12:41.800 | INFO | Analyzing chunk frames = [6578176,6578688) | 20260508 08:12:41.800 | INFO | MemInfo: { | 20260508 08:12:41.800 | INFO | "gpu" : " 17 GB", | 20260508 08:12:41.800 | INFO | "huge" : "", | 20260508 08:12:41.800 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:41.800 | INFO | "malloc" : " 14 GB", | 20260508 08:12:41.800 | INFO | "pinned" : " 29 GB" | 20260508 08:12:41.800 | INFO | } | 20260508 08:12:41.800 | INFO | Perf report for the last 5.14112 seconds: | 20260508 08:12:41.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:41.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:41.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:41.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:41.800 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 08:12:41.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:41.800 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260508 08:12:41.800 | INFO | PRE_HQ : 66.4%, 1.8, 62.0ms, 0 | 20260508 08:12:41.800 | INFO | BAZWRITER : 66.7%, 2.8, 95.7ms, 0 | 20260508 08:12:41.800 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:12:41.801 | INFO | Uploaded 36GiB to GPU in 2.58838s (13.9083GiB/s) | 20260508 08:12:41.801 | INFO | Downloaded 38GiB to GPU in 2.70444s (14.051GiB/s) | 20260508 08:12:46.940 | INFO | Analyzing chunk frames = [6578688,6579200) | 20260508 08:12:46.940 | INFO | MemInfo: { | 20260508 08:12:46.940 | INFO | "gpu" : " 17 GB", | 20260508 08:12:46.940 | INFO | "huge" : "", | 20260508 08:12:46.940 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:46.940 | INFO | "malloc" : " 14 GB", | 20260508 08:12:46.940 | INFO | "pinned" : " 29 GB" | 20260508 08:12:46.941 | INFO | } | 20260508 08:12:46.941 | INFO | Perf report for the last 5.14057 seconds: | 20260508 08:12:46.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:46.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:12:46.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:12:46.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:46.941 | INFO | Uploaded 36GiB to GPU in 2.54055s (14.1701GiB/s) | 20260508 08:12:46.941 | INFO | Downloaded 34GiB to GPU in 2.44715s (13.8937GiB/s) | 20260508 08:12:52.081 | INFO | Analyzing chunk frames = [6579200,6579712) | 20260508 08:12:52.081 | INFO | MemInfo: { | 20260508 08:12:52.081 | INFO | "gpu" : " 17 GB", | 20260508 08:12:52.081 | INFO | "huge" : "", | 20260508 08:12:52.081 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:52.081 | INFO | "malloc" : " 14 GB", | 20260508 08:12:52.081 | INFO | "pinned" : " 29 GB" | 20260508 08:12:52.081 | INFO | } | 20260508 08:12:52.081 | INFO | Perf report for the last 5.14013 seconds: | 20260508 08:12:52.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:52.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 08:12:52.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:12:52.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:52.081 | INFO | Uploaded 36GiB to GPU in 2.52275s (14.2701GiB/s) | 20260508 08:12:52.081 | INFO | Downloaded 34GiB to GPU in 2.43283s (13.9755GiB/s) | 20260508 08:12:57.218 | INFO | Analyzing chunk frames = [6579712,6580224) | 20260508 08:12:57.218 | INFO | MemInfo: { | 20260508 08:12:57.218 | INFO | "gpu" : " 17 GB", | 20260508 08:12:57.218 | INFO | "huge" : "", | 20260508 08:12:57.218 | INFO | "huge_shared" : " 29 GB", | 20260508 08:12:57.218 | INFO | "malloc" : " 14 GB", | 20260508 08:12:57.218 | INFO | "pinned" : " 29 GB" | 20260508 08:12:57.218 | INFO | } | 20260508 08:12:57.218 | INFO | Perf report for the last 5.13733 seconds: | 20260508 08:12:57.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:12:57.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | ANALYSIS : 67.1%, 3.0, 103.7ms, 0 | 20260508 08:12:57.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:12:57.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:12:57.218 | INFO | Uploaded 36GiB to GPU in 2.59464s (13.8747GiB/s) | 20260508 08:12:57.218 | INFO | Downloaded 34GiB to GPU in 2.42352s (14.0292GiB/s) | 20260508 08:13:02.353 | INFO | Analyzing chunk frames = [6580224,6580736) | 20260508 08:13:02.353 | INFO | MemInfo: { | 20260508 08:13:02.353 | INFO | "gpu" : " 17 GB", | 20260508 08:13:02.353 | INFO | "huge" : "", | 20260508 08:13:02.353 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:02.353 | INFO | "malloc" : " 14 GB", | 20260508 08:13:02.354 | INFO | "pinned" : " 29 GB" | 20260508 08:13:02.354 | INFO | } | 20260508 08:13:02.354 | INFO | Perf report for the last 5.13566 seconds: | 20260508 08:13:02.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:02.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 08:13:02.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:13:02.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:02.354 | INFO | Uploaded 36GiB to GPU in 2.4211s (14.8693GiB/s) | 20260508 08:13:02.354 | INFO | Downloaded 34GiB to GPU in 2.4991s (13.6049GiB/s) | 20260508 08:13:07.498 | INFO | Analyzing chunk frames = [6580736,6581248) | 20260508 08:13:07.498 | INFO | MemInfo: { | 20260508 08:13:07.498 | INFO | "gpu" : " 17 GB", | 20260508 08:13:07.498 | INFO | "huge" : "", | 20260508 08:13:07.498 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:07.499 | INFO | "malloc" : " 14 GB", | 20260508 08:13:07.499 | INFO | "pinned" : " 29 GB" | 20260508 08:13:07.499 | INFO | } | 20260508 08:13:07.499 | INFO | Perf report for the last 5.14492 seconds: | 20260508 08:13:07.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:07.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 08:13:07.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:13:07.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:07.499 | INFO | Uploaded 36GiB to GPU in 2.37654s (15.1481GiB/s) | 20260508 08:13:07.499 | INFO | Downloaded 34GiB to GPU in 2.43325s (13.9731GiB/s) | 20260508 08:13:12.637 | INFO | Analyzing chunk frames = [6581248,6581760) | 20260508 08:13:12.637 | INFO | MemInfo: { | 20260508 08:13:12.637 | INFO | "gpu" : " 17 GB", | 20260508 08:13:12.637 | INFO | "huge" : "", | 20260508 08:13:12.637 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:12.637 | INFO | "malloc" : " 14 GB", | 20260508 08:13:12.637 | INFO | "pinned" : " 29 GB" | 20260508 08:13:12.637 | INFO | } | 20260508 08:13:12.637 | INFO | Perf report for the last 5.13857 seconds: | 20260508 08:13:12.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:12.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.638 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:13:12.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.638 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:13:12.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:12.638 | INFO | Uploaded 36GiB to GPU in 2.42807s (14.8266GiB/s) | 20260508 08:13:12.638 | INFO | Downloaded 34GiB to GPU in 2.44734s (13.8926GiB/s) | 20260508 08:13:17.784 | INFO | Analyzing chunk frames = [6581760,6582272) | 20260508 08:13:17.784 | INFO | MemInfo: { | 20260508 08:13:17.784 | INFO | "gpu" : " 17 GB", | 20260508 08:13:17.784 | INFO | "huge" : "", | 20260508 08:13:17.784 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:17.784 | INFO | "malloc" : " 14 GB", | 20260508 08:13:17.784 | INFO | "pinned" : " 29 GB" | 20260508 08:13:17.784 | INFO | } | 20260508 08:13:17.784 | INFO | Perf report for the last 5.14684 seconds: | 20260508 08:13:17.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:17.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 08:13:17.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:13:17.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:17.784 | INFO | Uploaded 36GiB to GPU in 2.45469s (14.6658GiB/s) | 20260508 08:13:17.785 | INFO | Downloaded 34GiB to GPU in 2.44413s (13.9109GiB/s) | 20260508 08:13:17.882 | INFO | Starting a round of Baz IO | 20260508 08:13:21.289 | INFO | Populating RTMetrics report | 20260508 08:13:21.289 | INFO | RTMetrics report populated | 20260508 08:13:21.289 | INFO | Opening tmp RTMetrics json file | 20260508 08:13:21.297 | INFO | Serializing RTMetrics to json | 20260508 08:13:21.298 | INFO | Serialization done, swapping tmp file now... | 20260508 08:13:21.299 | INFO | RTMetrics done | 20260508 08:13:21.363 | INFO | Finished a round of Baz IO: Wrote 1.509 GiB in 3.48 seconds | 20260508 08:13:21.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.55% | 20260508 08:13:21.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304878% / 0% / 5.5631e-05% | 20260508 08:13:21.363 | INFO | Baz IO overall: | 20260508 08:13:21.363 | INFO | BytesWritten : 2892061773125 | 20260508 08:13:21.363 | INFO | NumEvents : 1392567563716 | 20260508 08:13:22.915 | INFO | Analyzing chunk frames = [6582272,6582784) | 20260508 08:13:22.915 | INFO | MemInfo: { | 20260508 08:13:22.915 | INFO | "gpu" : " 17 GB", | 20260508 08:13:22.915 | INFO | "huge" : "", | 20260508 08:13:22.915 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:22.915 | INFO | "malloc" : " 14 GB", | 20260508 08:13:22.915 | INFO | "pinned" : " 29 GB" | 20260508 08:13:22.915 | INFO | } | 20260508 08:13:22.915 | INFO | Perf report for the last 5.13076 seconds: | 20260508 08:13:22.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:22.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:22.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:22.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:22.915 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:13:22.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:22.915 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:13:22.915 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 08:13:22.915 | INFO | BAZWRITER : 67.2%, 2.6, 88.3ms, 0 | 20260508 08:13:22.915 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:13:22.915 | INFO | Uploaded 36GiB to GPU in 2.57424s (13.9847GiB/s) | 20260508 08:13:22.915 | INFO | Downloaded 38GiB to GPU in 2.7171s (13.9855GiB/s) | 20260508 08:13:28.055 | INFO | Analyzing chunk frames = [6582784,6583296) | 20260508 08:13:28.055 | INFO | MemInfo: { | 20260508 08:13:28.055 | INFO | "gpu" : " 17 GB", | 20260508 08:13:28.055 | INFO | "huge" : "", | 20260508 08:13:28.055 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:28.055 | INFO | "malloc" : " 14 GB", | 20260508 08:13:28.055 | INFO | "pinned" : " 29 GB" | 20260508 08:13:28.055 | INFO | } | 20260508 08:13:28.055 | INFO | Perf report for the last 5.14014 seconds: | 20260508 08:13:28.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:28.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:13:28.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:13:28.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:28.055 | INFO | Uploaded 36GiB to GPU in 2.50938s (14.3462GiB/s) | 20260508 08:13:28.055 | INFO | Downloaded 34GiB to GPU in 2.45337s (13.8585GiB/s) | 20260508 08:13:33.192 | INFO | Analyzing chunk frames = [6583296,6583808) | 20260508 08:13:33.192 | INFO | MemInfo: { | 20260508 08:13:33.192 | INFO | "gpu" : " 17 GB", | 20260508 08:13:33.192 | INFO | "huge" : "", | 20260508 08:13:33.192 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:33.192 | INFO | "malloc" : " 14 GB", | 20260508 08:13:33.192 | INFO | "pinned" : " 29 GB" | 20260508 08:13:33.192 | INFO | } | 20260508 08:13:33.192 | INFO | Perf report for the last 5.13714 seconds: | 20260508 08:13:33.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:33.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.192 | INFO | ANALYSIS : 67.1%, 3.0, 104.2ms, 0 | 20260508 08:13:33.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.192 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:13:33.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:33.193 | INFO | Uploaded 36GiB to GPU in 2.55641s (14.0823GiB/s) | 20260508 08:13:33.193 | INFO | Downloaded 34GiB to GPU in 2.443s (13.9173GiB/s) | 20260508 08:13:38.330 | INFO | Analyzing chunk frames = [6583808,6584320) | 20260508 08:13:38.330 | INFO | MemInfo: { | 20260508 08:13:38.330 | INFO | "gpu" : " 17 GB", | 20260508 08:13:38.330 | INFO | "huge" : "", | 20260508 08:13:38.330 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:38.330 | INFO | "malloc" : " 14 GB", | 20260508 08:13:38.330 | INFO | "pinned" : " 29 GB" | 20260508 08:13:38.330 | INFO | } | 20260508 08:13:38.330 | INFO | Perf report for the last 5.13804 seconds: | 20260508 08:13:38.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:38.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 08:13:38.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:13:38.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:38.331 | INFO | Uploaded 36GiB to GPU in 2.56825s (14.0173GiB/s) | 20260508 08:13:38.331 | INFO | Downloaded 34GiB to GPU in 2.43057s (13.9885GiB/s) | 20260508 08:13:43.471 | INFO | Analyzing chunk frames = [6584320,6584832) | 20260508 08:13:43.471 | INFO | MemInfo: { | 20260508 08:13:43.471 | INFO | "gpu" : " 17 GB", | 20260508 08:13:43.471 | INFO | "huge" : "", | 20260508 08:13:43.471 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:43.471 | INFO | "malloc" : " 14 GB", | 20260508 08:13:43.471 | INFO | "pinned" : " 29 GB" | 20260508 08:13:43.471 | INFO | } | 20260508 08:13:43.472 | INFO | Perf report for the last 5.14132 seconds: | 20260508 08:13:43.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:43.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:13:43.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 08:13:43.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:43.472 | INFO | Uploaded 36GiB to GPU in 2.37882s (15.1336GiB/s) | 20260508 08:13:43.472 | INFO | Downloaded 34GiB to GPU in 2.54453s (13.362GiB/s) | 20260508 08:13:48.611 | INFO | Analyzing chunk frames = [6584832,6585344) | 20260508 08:13:48.611 | INFO | MemInfo: { | 20260508 08:13:48.611 | INFO | "gpu" : " 17 GB", | 20260508 08:13:48.611 | INFO | "huge" : "", | 20260508 08:13:48.611 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:48.611 | INFO | "malloc" : " 14 GB", | 20260508 08:13:48.611 | INFO | "pinned" : " 29 GB" | 20260508 08:13:48.611 | INFO | } | 20260508 08:13:48.611 | INFO | Perf report for the last 5.1396 seconds: | 20260508 08:13:48.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:48.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:13:48.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:13:48.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:48.611 | INFO | Uploaded 36GiB to GPU in 2.38163s (15.1157GiB/s) | 20260508 08:13:48.611 | INFO | Downloaded 34GiB to GPU in 2.4404s (13.9322GiB/s) | 20260508 08:13:53.753 | INFO | Analyzing chunk frames = [6585344,6585856) | 20260508 08:13:53.753 | INFO | MemInfo: { | 20260508 08:13:53.753 | INFO | "gpu" : " 17 GB", | 20260508 08:13:53.753 | INFO | "huge" : "", | 20260508 08:13:53.753 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:53.753 | INFO | "malloc" : " 14 GB", | 20260508 08:13:53.753 | INFO | "pinned" : " 29 GB" | 20260508 08:13:53.753 | INFO | } | 20260508 08:13:53.753 | INFO | Perf report for the last 5.14189 seconds: | 20260508 08:13:53.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:53.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:13:53.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:13:53.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:53.753 | INFO | Uploaded 36GiB to GPU in 2.39443s (15.0349GiB/s) | 20260508 08:13:53.753 | INFO | Downloaded 34GiB to GPU in 2.41993s (14.05GiB/s) | 20260508 08:13:58.893 | INFO | Analyzing chunk frames = [6585856,6586368) | 20260508 08:13:58.893 | INFO | MemInfo: { | 20260508 08:13:58.893 | INFO | "gpu" : " 17 GB", | 20260508 08:13:58.893 | INFO | "huge" : "", | 20260508 08:13:58.893 | INFO | "huge_shared" : " 29 GB", | 20260508 08:13:58.893 | INFO | "malloc" : " 14 GB", | 20260508 08:13:58.893 | INFO | "pinned" : " 29 GB" | 20260508 08:13:58.893 | INFO | } | 20260508 08:13:58.893 | INFO | Perf report for the last 5.13973 seconds: | 20260508 08:13:58.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:13:58.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 08:13:58.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:13:58.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:13:58.893 | INFO | Uploaded 36GiB to GPU in 2.42436s (14.8493GiB/s) | 20260508 08:13:58.893 | INFO | Downloaded 34GiB to GPU in 2.44809s (13.8884GiB/s) | 20260508 08:13:58.893 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:13:58.893 | INFO | Total mem usage: 16.7 GiB | 20260508 08:13:58.893 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:13:58.893 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 08:13:58.893 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:13:58.893 | INFO | High Water Breakdown: | 20260508 08:13:58.893 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:13:58.893 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:13:58.893 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:13:58.893 | INFO | ------------------------------------------ | 20260508 08:13:58.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:13:58.893 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:13:58.893 | INFO | Total mem usage: 26.5 GiB | 20260508 08:13:58.893 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:13:58.893 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:13:58.893 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:13:58.893 | INFO | High Water Breakdown: | 20260508 08:13:58.893 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:13:58.893 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:13:58.893 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:13:58.893 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:13:58.893 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:13:58.893 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:13:58.893 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:13:58.893 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:13:58.894 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:13:58.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:13:58.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:13:58.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:13:58.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:13:58.894 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:13:58.894 | INFO | ------------------------------------------ | 20260508 08:13:58.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:13:58.894 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:13:58.894 | INFO | Total mem usage: 27.8 GiB | 20260508 08:13:58.894 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:13:58.894 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:13:58.894 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:13:58.894 | INFO | High Water Breakdown: | 20260508 08:13:58.894 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:13:58.894 | INFO | ------------------------------------------ | 20260508 08:13:58.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:13:58.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:13:58.894 | INFO | Total mem usage: 15.7 GiB | 20260508 08:13:58.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:13:58.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:13:58.894 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:13:58.894 | INFO | High Water Breakdown: | 20260508 08:13:58.894 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:13:58.894 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:13:58.894 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:13:58.894 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:13:58.894 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:13:58.894 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:13:58.894 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:13:58.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:13:58.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:13:58.894 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:13:58.894 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:13:58.894 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:13:58.894 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:13:58.894 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:13:58.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:13:58.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:13:58.894 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:13:58.894 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:13:58.894 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:13:58.894 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:13:58.894 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:13:58.894 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:13:58.894 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:13:58.894 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:13:58.894 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:13:58.894 | INFO | ------------------------------------------ | 20260508 08:13:58.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:13:58.894 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:13:58.894 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:13:58.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:13:58.894 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 4.87e-01 | 20260508 08:13:58.894 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.03e-01, 1.11e-02 / 6.06e-03 | 20260508 08:13:58.894 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.43e+00, 5.91e-01 / 2.10e-01 | 20260508 08:13:58.894 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 1.43e-01 | 20260508 08:13:58.894 | INFO | PerfReport: 15.91%, DME -- 4.16e+01 / 4.49e+01, 3.46e+00 / 1.06e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.83e+00 / 1.44e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.35e+00 / 2.45e+00, 4.10e-01 / 2.92e-01 | 20260508 08:13:58.894 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.12e+00, 2.51e-01 / 1.37e-01 | 20260508 08:13:58.894 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.06e-01 / 3.31e-01, 4.87e-02 / 2.81e-02 | 20260508 08:13:58.894 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:13:58.894 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:13:58.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:13:58.894 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 24.48%, Upload -- 2.59e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 24.49%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 30.91%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 2.28%, LockUpload -- 2.41e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 2.53%, LockDownload -- 2.67e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 15.29%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:13:58.894 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.22e-02 / 0.00e+00 | 20260508 08:13:58.998 | INFO | Starting a round of Baz IO | 20260508 08:14:02.398 | INFO | Populating RTMetrics report | 20260508 08:14:02.399 | INFO | RTMetrics report populated | 20260508 08:14:02.399 | INFO | Opening tmp RTMetrics json file | 20260508 08:14:02.409 | INFO | Serializing RTMetrics to json | 20260508 08:14:02.410 | INFO | Serialization done, swapping tmp file now... | 20260508 08:14:02.411 | INFO | RTMetrics done | 20260508 08:14:02.474 | INFO | Finished a round of Baz IO: Wrote 1.509 GiB in 3.475 seconds | 20260508 08:14:02.474 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.55% | 20260508 08:14:02.474 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.305518% / 0% / 5.96046e-05% | 20260508 08:14:02.474 | INFO | Baz IO overall: | 20260508 08:14:02.474 | INFO | BytesWritten : 2893682498278 | 20260508 08:14:02.474 | INFO | NumEvents : 1393336258771 | 20260508 08:14:04.029 | INFO | Analyzing chunk frames = [6586368,6586880) | 20260508 08:14:04.029 | INFO | MemInfo: { | 20260508 08:14:04.029 | INFO | "gpu" : " 17 GB", | 20260508 08:14:04.029 | INFO | "huge" : "", | 20260508 08:14:04.029 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:04.029 | INFO | "malloc" : " 14 GB", | 20260508 08:14:04.029 | INFO | "pinned" : " 29 GB" | 20260508 08:14:04.029 | INFO | } | 20260508 08:14:04.029 | INFO | Perf report for the last 5.13603 seconds: | 20260508 08:14:04.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:04.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:04.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:04.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:04.029 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:14:04.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:04.029 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:14:04.029 | INFO | PRE_HQ : 65.5%, 1.6, 55.0ms, 0 | 20260508 08:14:04.029 | INFO | BAZWRITER : 67.3%, 2.5, 88.1ms, 0 | 20260508 08:14:04.029 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:14:04.029 | INFO | Uploaded 36GiB to GPU in 2.56857s (14.0156GiB/s) | 20260508 08:14:04.029 | INFO | Downloaded 38GiB to GPU in 2.70573s (14.0443GiB/s) | 20260508 08:14:09.165 | INFO | Analyzing chunk frames = [6586880,6587392) | 20260508 08:14:09.165 | INFO | MemInfo: { | 20260508 08:14:09.165 | INFO | "gpu" : " 17 GB", | 20260508 08:14:09.165 | INFO | "huge" : "", | 20260508 08:14:09.165 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:09.165 | INFO | "malloc" : " 14 GB", | 20260508 08:14:09.165 | INFO | "pinned" : " 29 GB" | 20260508 08:14:09.165 | INFO | } | 20260508 08:14:09.166 | INFO | Perf report for the last 5.13669 seconds: | 20260508 08:14:09.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:09.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:14:09.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:14:09.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:09.166 | INFO | Uploaded 36GiB to GPU in 2.49833s (14.4096GiB/s) | 20260508 08:14:09.166 | INFO | Downloaded 34GiB to GPU in 2.43208s (13.9798GiB/s) | 20260508 08:14:14.305 | INFO | Analyzing chunk frames = [6587392,6587904) | 20260508 08:14:14.305 | INFO | MemInfo: { | 20260508 08:14:14.305 | INFO | "gpu" : " 17 GB", | 20260508 08:14:14.305 | INFO | "huge" : "", | 20260508 08:14:14.305 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:14.305 | INFO | "malloc" : " 14 GB", | 20260508 08:14:14.305 | INFO | "pinned" : " 29 GB" | 20260508 08:14:14.305 | INFO | } | 20260508 08:14:14.305 | INFO | Perf report for the last 5.13964 seconds: | 20260508 08:14:14.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:14.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 08:14:14.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:14:14.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:14.305 | INFO | Uploaded 36GiB to GPU in 2.5818s (13.9438GiB/s) | 20260508 08:14:14.306 | INFO | Downloaded 34GiB to GPU in 2.43537s (13.9609GiB/s) | 20260508 08:14:19.449 | INFO | Analyzing chunk frames = [6587904,6588416) | 20260508 08:14:19.449 | INFO | MemInfo: { | 20260508 08:14:19.449 | INFO | "gpu" : " 17 GB", | 20260508 08:14:19.449 | INFO | "huge" : "", | 20260508 08:14:19.449 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:19.449 | INFO | "malloc" : " 14 GB", | 20260508 08:14:19.449 | INFO | "pinned" : " 29 GB" | 20260508 08:14:19.449 | INFO | } | 20260508 08:14:19.449 | INFO | Perf report for the last 5.14418 seconds: | 20260508 08:14:19.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:19.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 08:14:19.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 08:14:19.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:19.450 | INFO | Uploaded 36GiB to GPU in 2.56204s (14.0513GiB/s) | 20260508 08:14:19.451 | INFO | Downloaded 34GiB to GPU in 2.40677s (14.1268GiB/s) | 20260508 08:14:24.598 | INFO | Analyzing chunk frames = [6588416,6588928) | 20260508 08:14:24.598 | INFO | MemInfo: { | 20260508 08:14:24.598 | INFO | "gpu" : " 17 GB", | 20260508 08:14:24.598 | INFO | "huge" : "", | 20260508 08:14:24.598 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:24.598 | INFO | "malloc" : " 14 GB", | 20260508 08:14:24.598 | INFO | "pinned" : " 29 GB" | 20260508 08:14:24.598 | INFO | } | 20260508 08:14:24.598 | INFO | Perf report for the last 5.1484 seconds: | 20260508 08:14:24.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:24.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 08:14:24.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | PRE_HQ : 30.6%, 1.0, 16.0ms, 0 | 20260508 08:14:24.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:24.598 | INFO | Uploaded 36GiB to GPU in 2.30111s (15.6446GiB/s) | 20260508 08:14:24.598 | INFO | Downloaded 34GiB to GPU in 2.47822s (13.7195GiB/s) | 20260508 08:14:29.729 | INFO | Analyzing chunk frames = [6588928,6589440) | 20260508 08:14:29.729 | INFO | MemInfo: { | 20260508 08:14:29.729 | INFO | "gpu" : " 17 GB", | 20260508 08:14:29.729 | INFO | "huge" : "", | 20260508 08:14:29.729 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:29.729 | INFO | "malloc" : " 14 GB", | 20260508 08:14:29.729 | INFO | "pinned" : " 29 GB" | 20260508 08:14:29.729 | INFO | } | 20260508 08:14:29.729 | INFO | Perf report for the last 5.1313 seconds: | 20260508 08:14:29.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:29.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.729 | INFO | ANALYSIS : 65.6%, 3.0, 101.8ms, 0 | 20260508 08:14:29.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.729 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:14:29.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:29.730 | INFO | Uploaded 36GiB to GPU in 2.36308s (15.2344GiB/s) | 20260508 08:14:29.730 | INFO | Downloaded 34GiB to GPU in 2.42463s (14.0228GiB/s) | 20260508 08:14:34.864 | INFO | Analyzing chunk frames = [6589440,6589952) | 20260508 08:14:34.864 | INFO | MemInfo: { | 20260508 08:14:34.864 | INFO | "gpu" : " 17 GB", | 20260508 08:14:34.864 | INFO | "huge" : "", | 20260508 08:14:34.864 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:34.864 | INFO | "malloc" : " 14 GB", | 20260508 08:14:34.864 | INFO | "pinned" : " 29 GB" | 20260508 08:14:34.864 | INFO | } | 20260508 08:14:34.864 | INFO | Perf report for the last 5.1352 seconds: | 20260508 08:14:34.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:34.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.864 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:14:34.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.865 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:14:34.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:34.865 | INFO | Uploaded 36GiB to GPU in 2.41358s (14.9156GiB/s) | 20260508 08:14:34.865 | INFO | Downloaded 34GiB to GPU in 2.43271s (13.9762GiB/s) | 20260508 08:14:40.002 | INFO | Analyzing chunk frames = [6589952,6590464) | 20260508 08:14:40.002 | INFO | MemInfo: { | 20260508 08:14:40.002 | INFO | "gpu" : " 17 GB", | 20260508 08:14:40.002 | INFO | "huge" : "", | 20260508 08:14:40.002 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:40.002 | INFO | "malloc" : " 14 GB", | 20260508 08:14:40.002 | INFO | "pinned" : " 29 GB" | 20260508 08:14:40.003 | INFO | } | 20260508 08:14:40.003 | INFO | Perf report for the last 5.1382 seconds: | 20260508 08:14:40.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:40.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:14:40.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:14:40.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:40.003 | INFO | Uploaded 36GiB to GPU in 2.48407s (14.4923GiB/s) | 20260508 08:14:40.003 | INFO | Downloaded 34GiB to GPU in 2.44767s (13.8908GiB/s) | 20260508 08:14:40.099 | INFO | Starting a round of Baz IO | 20260508 08:14:43.524 | INFO | Populating RTMetrics report | 20260508 08:14:43.525 | INFO | RTMetrics report populated | 20260508 08:14:43.525 | INFO | Opening tmp RTMetrics json file | 20260508 08:14:43.534 | INFO | Serializing RTMetrics to json | 20260508 08:14:43.536 | INFO | Serialization done, swapping tmp file now... | 20260508 08:14:43.536 | INFO | RTMetrics done | 20260508 08:14:43.591 | INFO | Finished a round of Baz IO: Wrote 1.507 GiB in 3.492 seconds | 20260508 08:14:43.591 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.55% | 20260508 08:14:43.591 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.303304% / 0% / 4.76837e-05% | 20260508 08:14:43.592 | INFO | Baz IO overall: | 20260508 08:14:43.592 | INFO | BytesWritten : 2895300732539 | 20260508 08:14:43.592 | INFO | NumEvents : 1394103144267 | 20260508 08:14:45.146 | INFO | Analyzing chunk frames = [6590464,6590976) | 20260508 08:14:45.146 | INFO | MemInfo: { | 20260508 08:14:45.146 | INFO | "gpu" : " 17 GB", | 20260508 08:14:45.146 | INFO | "huge" : "", | 20260508 08:14:45.146 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:45.146 | INFO | "malloc" : " 14 GB", | 20260508 08:14:45.146 | INFO | "pinned" : " 29 GB" | 20260508 08:14:45.146 | INFO | } | 20260508 08:14:45.146 | INFO | Perf report for the last 5.14362 seconds: | 20260508 08:14:45.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:45.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:45.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:45.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:45.146 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 08:14:45.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:45.146 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:14:45.146 | INFO | PRE_HQ : 65.3%, 1.6, 54.7ms, 0 | 20260508 08:14:45.146 | INFO | BAZWRITER : 67.2%, 2.5, 88.0ms, 0 | 20260508 08:14:45.146 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 08:14:45.146 | INFO | Uploaded 36GiB to GPU in 2.5645s (14.0378GiB/s) | 20260508 08:14:45.147 | INFO | Downloaded 38GiB to GPU in 2.70793s (14.0329GiB/s) | 20260508 08:14:50.286 | INFO | Analyzing chunk frames = [6590976,6591488) | 20260508 08:14:50.286 | INFO | MemInfo: { | 20260508 08:14:50.286 | INFO | "gpu" : " 17 GB", | 20260508 08:14:50.286 | INFO | "huge" : "", | 20260508 08:14:50.286 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:50.286 | INFO | "malloc" : " 14 GB", | 20260508 08:14:50.286 | INFO | "pinned" : " 29 GB" | 20260508 08:14:50.286 | INFO | } | 20260508 08:14:50.286 | INFO | Perf report for the last 5.13955 seconds: | 20260508 08:14:50.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:50.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:14:50.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:14:50.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:50.286 | INFO | Uploaded 36GiB to GPU in 2.46375s (14.6119GiB/s) | 20260508 08:14:50.286 | INFO | Downloaded 34GiB to GPU in 2.45388s (13.8556GiB/s) | 20260508 08:14:55.419 | INFO | Analyzing chunk frames = [6591488,6592000) | 20260508 08:14:55.419 | INFO | MemInfo: { | 20260508 08:14:55.419 | INFO | "gpu" : " 17 GB", | 20260508 08:14:55.419 | INFO | "huge" : "", | 20260508 08:14:55.419 | INFO | "huge_shared" : " 29 GB", | 20260508 08:14:55.419 | INFO | "malloc" : " 14 GB", | 20260508 08:14:55.419 | INFO | "pinned" : " 29 GB" | 20260508 08:14:55.419 | INFO | } | 20260508 08:14:55.419 | INFO | Perf report for the last 5.1333 seconds: | 20260508 08:14:55.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:14:55.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 08:14:55.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:14:55.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:14:55.419 | INFO | Uploaded 36GiB to GPU in 2.52472s (14.259GiB/s) | 20260508 08:14:55.419 | INFO | Downloaded 34GiB to GPU in 2.45308s (13.8601GiB/s) | 20260508 08:15:00.564 | INFO | Analyzing chunk frames = [6592000,6592512) | 20260508 08:15:00.564 | INFO | MemInfo: { | 20260508 08:15:00.564 | INFO | "gpu" : " 17 GB", | 20260508 08:15:00.564 | INFO | "huge" : "", | 20260508 08:15:00.564 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:00.564 | INFO | "malloc" : " 14 GB", | 20260508 08:15:00.564 | INFO | "pinned" : " 29 GB" | 20260508 08:15:00.564 | INFO | } | 20260508 08:15:00.564 | INFO | Perf report for the last 5.14499 seconds: | 20260508 08:15:00.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:00.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.564 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 08:15:00.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.564 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:15:00.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:00.565 | INFO | Uploaded 36GiB to GPU in 2.54556s (14.1423GiB/s) | 20260508 08:15:00.565 | INFO | Downloaded 34GiB to GPU in 2.4502s (13.8764GiB/s) | 20260508 08:15:05.703 | INFO | Analyzing chunk frames = [6592512,6593024) | 20260508 08:15:05.703 | INFO | MemInfo: { | 20260508 08:15:05.703 | INFO | "gpu" : " 17 GB", | 20260508 08:15:05.703 | INFO | "huge" : "", | 20260508 08:15:05.703 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:05.703 | INFO | "malloc" : " 14 GB", | 20260508 08:15:05.703 | INFO | "pinned" : " 29 GB" | 20260508 08:15:05.703 | INFO | } | 20260508 08:15:05.703 | INFO | Perf report for the last 5.13919 seconds: | 20260508 08:15:05.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:05.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.703 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:15:05.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.703 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:15:05.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:05.704 | INFO | Uploaded 36GiB to GPU in 2.33829s (15.3959GiB/s) | 20260508 08:15:05.704 | INFO | Downloaded 34GiB to GPU in 2.51462s (13.5209GiB/s) | 20260508 08:15:10.840 | INFO | Analyzing chunk frames = [6593024,6593536) | 20260508 08:15:10.840 | INFO | MemInfo: { | 20260508 08:15:10.840 | INFO | "gpu" : " 17 GB", | 20260508 08:15:10.840 | INFO | "huge" : "", | 20260508 08:15:10.840 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:10.840 | INFO | "malloc" : " 14 GB", | 20260508 08:15:10.840 | INFO | "pinned" : " 29 GB" | 20260508 08:15:10.840 | INFO | } | 20260508 08:15:10.840 | INFO | Perf report for the last 5.13696 seconds: | 20260508 08:15:10.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:10.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.840 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:15:10.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.840 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:15:10.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:10.841 | INFO | Uploaded 36GiB to GPU in 2.34886s (15.3266GiB/s) | 20260508 08:15:10.841 | INFO | Downloaded 34GiB to GPU in 2.43115s (13.9852GiB/s) | 20260508 08:15:15.978 | INFO | Analyzing chunk frames = [6593536,6594048) | 20260508 08:15:15.978 | INFO | MemInfo: { | 20260508 08:15:15.978 | INFO | "gpu" : " 17 GB", | 20260508 08:15:15.978 | INFO | "huge" : "", | 20260508 08:15:15.978 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:15.978 | INFO | "malloc" : " 14 GB", | 20260508 08:15:15.978 | INFO | "pinned" : " 29 GB" | 20260508 08:15:15.978 | INFO | } | 20260508 08:15:15.978 | INFO | Perf report for the last 5.13763 seconds: | 20260508 08:15:15.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:15.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.978 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 08:15:15.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.978 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:15:15.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:15.979 | INFO | Uploaded 36GiB to GPU in 2.4299s (14.8154GiB/s) | 20260508 08:15:15.979 | INFO | Downloaded 34GiB to GPU in 2.45559s (13.846GiB/s) | 20260508 08:15:21.117 | INFO | Analyzing chunk frames = [6594048,6594560) | 20260508 08:15:21.117 | INFO | MemInfo: { | 20260508 08:15:21.117 | INFO | "gpu" : " 17 GB", | 20260508 08:15:21.117 | INFO | "huge" : "", | 20260508 08:15:21.117 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:21.117 | INFO | "malloc" : " 14 GB", | 20260508 08:15:21.117 | INFO | "pinned" : " 29 GB" | 20260508 08:15:21.117 | INFO | } | 20260508 08:15:21.117 | INFO | Perf report for the last 5.13894 seconds: | 20260508 08:15:21.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:21.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:15:21.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:15:21.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:21.117 | INFO | Uploaded 36GiB to GPU in 2.41368s (14.915GiB/s) | 20260508 08:15:21.117 | INFO | Downloaded 34GiB to GPU in 2.44792s (13.8894GiB/s) | 20260508 08:15:21.117 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:15:21.117 | INFO | Total mem usage: 16.7 GiB | 20260508 08:15:21.117 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:15:21.117 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:15:21.117 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:15:21.117 | INFO | High Water Breakdown: | 20260508 08:15:21.117 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:15:21.117 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:15:21.117 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:15:21.117 | INFO | ------------------------------------------ | 20260508 08:15:21.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:15:21.117 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:15:21.117 | INFO | Total mem usage: 26.5 GiB | 20260508 08:15:21.117 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:15:21.117 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:15:21.117 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:15:21.117 | INFO | High Water Breakdown: | 20260508 08:15:21.117 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:15:21.117 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:15:21.117 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:15:21.117 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:15:21.117 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:15:21.117 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:15:21.117 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:15:21.117 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:15:21.117 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:15:21.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:15:21.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:15:21.117 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:15:21.117 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:15:21.117 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:15:21.117 | INFO | ------------------------------------------ | 20260508 08:15:21.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:15:21.117 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:15:21.117 | INFO | Total mem usage: 27.8 GiB | 20260508 08:15:21.117 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:15:21.117 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:15:21.117 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:15:21.117 | INFO | High Water Breakdown: | 20260508 08:15:21.117 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:15:21.117 | INFO | ------------------------------------------ | 20260508 08:15:21.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:15:21.117 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:15:21.117 | INFO | Total mem usage: 15.7 GiB | 20260508 08:15:21.117 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:15:21.117 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:15:21.117 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:15:21.117 | INFO | High Water Breakdown: | 20260508 08:15:21.117 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:15:21.117 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:15:21.117 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:15:21.117 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:15:21.117 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:15:21.117 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:15:21.117 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:15:21.117 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:15:21.117 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:15:21.117 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:15:21.117 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:15:21.117 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:15:21.117 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:15:21.117 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:15:21.117 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:15:21.117 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:15:21.117 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:15:21.117 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:15:21.117 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:15:21.117 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:15:21.117 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:15:21.117 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:15:21.117 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:15:21.117 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:15:21.117 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:15:21.117 | INFO | ------------------------------------------ | 20260508 08:15:21.117 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:15:21.117 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:15:21.117 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:15:21.117 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:15:21.117 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.38e+01 / 6.55e-01 | 20260508 08:15:21.117 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.11e-02 / 5.58e-03 | 20260508 08:15:21.117 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.48e+00, 5.95e-01 / 2.61e-01 | 20260508 08:15:21.117 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.13e-01 | 20260508 08:15:21.117 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.51e+01, 3.66e+00 / 1.08e+00 | 20260508 08:15:21.118 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.87e+00 / 1.30e+00 | 20260508 08:15:21.118 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.54e+00, 4.10e-01 / 2.52e-01 | 20260508 08:15:21.118 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.16e+00, 2.49e-01 / 1.33e-01 | 20260508 08:15:21.118 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.36e-01, 4.86e-02 / 2.43e-02 | 20260508 08:15:21.118 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:15:21.118 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:15:21.118 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:15:21.118 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 08:15:21.118 | INFO | PerfReport: 24.28%, Upload -- 2.57e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 08:15:21.118 | INFO | PerfReport: 24.45%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 08:15:21.118 | INFO | PerfReport: 30.98%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 08:15:21.118 | INFO | PerfReport: 2.14%, LockUpload -- 2.26e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 08:15:21.119 | INFO | PerfReport: 2.63%, LockDownload -- 2.78e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 08:15:21.119 | INFO | PerfReport: 15.52%, LockCompute -- 1.64e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:15:21.119 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 1.03e-02 / 0.00e+00 | 20260508 08:15:21.214 | INFO | Starting a round of Baz IO | 20260508 08:15:24.637 | INFO | Populating RTMetrics report | 20260508 08:15:24.638 | INFO | RTMetrics report populated | 20260508 08:15:24.638 | INFO | Opening tmp RTMetrics json file | 20260508 08:15:24.645 | INFO | Serializing RTMetrics to json | 20260508 08:15:24.647 | INFO | Serialization done, swapping tmp file now... | 20260508 08:15:24.647 | INFO | RTMetrics done | 20260508 08:15:24.708 | INFO | Finished a round of Baz IO: Wrote 1.506 GiB in 3.493 seconds | 20260508 08:15:24.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.45% / 0.00% / 59.55% | 20260508 08:15:24.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.3033% / 0% / 8.74201e-05% | 20260508 08:15:24.708 | INFO | Baz IO overall: | 20260508 08:15:24.708 | INFO | BytesWritten : 2896917993019 | 20260508 08:15:24.708 | INFO | NumEvents : 1394869255240 | 20260508 08:15:26.258 | INFO | Analyzing chunk frames = [6594560,6595072) | 20260508 08:15:26.258 | INFO | MemInfo: { | 20260508 08:15:26.258 | INFO | "gpu" : " 17 GB", | 20260508 08:15:26.258 | INFO | "huge" : "", | 20260508 08:15:26.258 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:26.258 | INFO | "malloc" : " 14 GB", | 20260508 08:15:26.258 | INFO | "pinned" : " 29 GB" | 20260508 08:15:26.258 | INFO | } | 20260508 08:15:26.258 | INFO | Perf report for the last 5.14101 seconds: | 20260508 08:15:26.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:26.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:26.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:26.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:26.258 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:15:26.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:26.258 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:15:26.258 | INFO | PRE_HQ : 65.7%, 1.6, 54.9ms, 0 | 20260508 08:15:26.258 | INFO | BAZWRITER : 67.3%, 2.6, 89.3ms, 0 | 20260508 08:15:26.258 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 08:15:26.259 | INFO | Uploaded 36GiB to GPU in 2.57045s (14.0053GiB/s) | 20260508 08:15:26.259 | INFO | Downloaded 38GiB to GPU in 2.70521s (14.047GiB/s) | 20260508 08:15:31.401 | INFO | Analyzing chunk frames = [6595072,6595584) | 20260508 08:15:31.401 | INFO | MemInfo: { | 20260508 08:15:31.401 | INFO | "gpu" : " 17 GB", | 20260508 08:15:31.401 | INFO | "huge" : "", | 20260508 08:15:31.401 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:31.401 | INFO | "malloc" : " 14 GB", | 20260508 08:15:31.401 | INFO | "pinned" : " 29 GB" | 20260508 08:15:31.401 | INFO | } | 20260508 08:15:31.401 | INFO | Perf report for the last 5.14328 seconds: | 20260508 08:15:31.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:31.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.401 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 08:15:31.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.401 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 08:15:31.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:31.402 | INFO | Uploaded 36GiB to GPU in 2.52602s (14.2516GiB/s) | 20260508 08:15:31.402 | INFO | Downloaded 34GiB to GPU in 2.44222s (13.9218GiB/s) | 20260508 08:15:36.541 | INFO | Analyzing chunk frames = [6595584,6596096) | 20260508 08:15:36.541 | INFO | MemInfo: { | 20260508 08:15:36.541 | INFO | "gpu" : " 17 GB", | 20260508 08:15:36.541 | INFO | "huge" : "", | 20260508 08:15:36.541 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:36.541 | INFO | "malloc" : " 14 GB", | 20260508 08:15:36.541 | INFO | "pinned" : " 29 GB" | 20260508 08:15:36.541 | INFO | } | 20260508 08:15:36.541 | INFO | Perf report for the last 5.14005 seconds: | 20260508 08:15:36.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:36.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.541 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 08:15:36.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.541 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:15:36.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:36.542 | INFO | Uploaded 36GiB to GPU in 2.51716s (14.3018GiB/s) | 20260508 08:15:36.542 | INFO | Downloaded 34GiB to GPU in 2.43339s (13.9723GiB/s) | 20260508 08:15:41.674 | INFO | Analyzing chunk frames = [6596096,6596608) | 20260508 08:15:41.674 | INFO | MemInfo: { | 20260508 08:15:41.674 | INFO | "gpu" : " 17 GB", | 20260508 08:15:41.674 | INFO | "huge" : "", | 20260508 08:15:41.674 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:41.675 | INFO | "malloc" : " 14 GB", | 20260508 08:15:41.675 | INFO | "pinned" : " 29 GB" | 20260508 08:15:41.675 | INFO | } | 20260508 08:15:41.675 | INFO | Perf report for the last 5.13323 seconds: | 20260508 08:15:41.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:41.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 08:15:41.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:15:41.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:41.675 | INFO | Uploaded 36GiB to GPU in 2.51839s (14.2949GiB/s) | 20260508 08:15:41.675 | INFO | Downloaded 34GiB to GPU in 2.43337s (13.9724GiB/s) | 20260508 08:15:46.819 | INFO | Analyzing chunk frames = [6596608,6597120) | 20260508 08:15:46.819 | INFO | MemInfo: { | 20260508 08:15:46.819 | INFO | "gpu" : " 17 GB", | 20260508 08:15:46.819 | INFO | "huge" : "", | 20260508 08:15:46.819 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:46.819 | INFO | "malloc" : " 14 GB", | 20260508 08:15:46.819 | INFO | "pinned" : " 29 GB" | 20260508 08:15:46.819 | INFO | } | 20260508 08:15:46.819 | INFO | Perf report for the last 5.14419 seconds: | 20260508 08:15:46.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:46.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 08:15:46.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 08:15:46.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:46.819 | INFO | Uploaded 36GiB to GPU in 2.4155s (14.9037GiB/s) | 20260508 08:15:46.819 | INFO | Downloaded 34GiB to GPU in 2.51629s (13.512GiB/s) | 20260508 08:15:51.956 | INFO | Analyzing chunk frames = [6597120,6597632) | 20260508 08:15:51.956 | INFO | MemInfo: { | 20260508 08:15:51.956 | INFO | "gpu" : " 17 GB", | 20260508 08:15:51.956 | INFO | "huge" : "", | 20260508 08:15:51.956 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:51.956 | INFO | "malloc" : " 14 GB", | 20260508 08:15:51.956 | INFO | "pinned" : " 29 GB" | 20260508 08:15:51.956 | INFO | } | 20260508 08:15:51.956 | INFO | Perf report for the last 5.13738 seconds: | 20260508 08:15:51.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:51.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:15:51.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:15:51.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:51.956 | INFO | Uploaded 36GiB to GPU in 2.39656s (15.0215GiB/s) | 20260508 08:15:51.956 | INFO | Downloaded 34GiB to GPU in 2.44154s (13.9256GiB/s) | 20260508 08:15:57.094 | INFO | Analyzing chunk frames = [6597632,6598144) | 20260508 08:15:57.094 | INFO | MemInfo: { | 20260508 08:15:57.094 | INFO | "gpu" : " 17 GB", | 20260508 08:15:57.094 | INFO | "huge" : "", | 20260508 08:15:57.094 | INFO | "huge_shared" : " 29 GB", | 20260508 08:15:57.094 | INFO | "malloc" : " 14 GB", | 20260508 08:15:57.094 | INFO | "pinned" : " 29 GB" | 20260508 08:15:57.094 | INFO | } | 20260508 08:15:57.094 | INFO | Perf report for the last 5.13812 seconds: | 20260508 08:15:57.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:15:57.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.094 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:15:57.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.094 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:15:57.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:15:57.095 | INFO | Uploaded 36GiB to GPU in 2.46487s (14.6052GiB/s) | 20260508 08:15:57.095 | INFO | Downloaded 34GiB to GPU in 2.43066s (13.988GiB/s) | 20260508 08:16:02.235 | INFO | Analyzing chunk frames = [6598144,6598656) | 20260508 08:16:02.235 | INFO | MemInfo: { | 20260508 08:16:02.235 | INFO | "gpu" : " 17 GB", | 20260508 08:16:02.235 | INFO | "huge" : "", | 20260508 08:16:02.235 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:02.235 | INFO | "malloc" : " 14 GB", | 20260508 08:16:02.235 | INFO | "pinned" : " 29 GB" | 20260508 08:16:02.235 | INFO | } | 20260508 08:16:02.235 | INFO | Perf report for the last 5.1409 seconds: | 20260508 08:16:02.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:02.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:16:02.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:16:02.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:02.235 | INFO | Uploaded 36GiB to GPU in 2.47476s (14.5469GiB/s) | 20260508 08:16:02.236 | INFO | Downloaded 34GiB to GPU in 2.449s (13.8832GiB/s) | 20260508 08:16:02.333 | INFO | Starting a round of Baz IO | 20260508 08:16:05.749 | INFO | Populating RTMetrics report | 20260508 08:16:05.749 | INFO | RTMetrics report populated | 20260508 08:16:05.749 | INFO | Opening tmp RTMetrics json file | 20260508 08:16:05.758 | INFO | Serializing RTMetrics to json | 20260508 08:16:05.760 | INFO | Serialization done, swapping tmp file now... | 20260508 08:16:05.760 | INFO | RTMetrics done | 20260508 08:16:05.816 | INFO | Finished a round of Baz IO: Wrote 1.507 GiB in 3.483 seconds | 20260508 08:16:05.816 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.55% | 20260508 08:16:05.816 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.302962% / 3.97364e-06% / 6.35783e-05% | 20260508 08:16:05.816 | INFO | Baz IO overall: | 20260508 08:16:05.816 | INFO | BytesWritten : 2898536132715 | 20260508 08:16:05.816 | INFO | NumEvents : 1395636027682 | 20260508 08:16:07.372 | INFO | Analyzing chunk frames = [6598656,6599168) | 20260508 08:16:07.372 | INFO | MemInfo: { | 20260508 08:16:07.372 | INFO | "gpu" : " 17 GB", | 20260508 08:16:07.372 | INFO | "huge" : "", | 20260508 08:16:07.372 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:07.372 | INFO | "malloc" : " 14 GB", | 20260508 08:16:07.372 | INFO | "pinned" : " 29 GB" | 20260508 08:16:07.372 | INFO | } | 20260508 08:16:07.372 | INFO | Perf report for the last 5.13682 seconds: | 20260508 08:16:07.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:07.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:07.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:07.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:07.372 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 08:16:07.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:07.372 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260508 08:16:07.372 | INFO | PRE_HQ : 65.7%, 1.7, 58.8ms, 0 | 20260508 08:16:07.372 | INFO | BAZWRITER : 67.3%, 3.0, 103.4ms, 0 | 20260508 08:16:07.372 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 08:16:07.372 | INFO | Uploaded 36GiB to GPU in 2.5809s (13.9486GiB/s) | 20260508 08:16:07.372 | INFO | Downloaded 38GiB to GPU in 2.70336s (14.0566GiB/s) | 20260508 08:16:12.516 | INFO | Analyzing chunk frames = [6599168,6599680) | 20260508 08:16:12.516 | INFO | MemInfo: { | 20260508 08:16:12.516 | INFO | "gpu" : " 17 GB", | 20260508 08:16:12.516 | INFO | "huge" : "", | 20260508 08:16:12.516 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:12.517 | INFO | "malloc" : " 14 GB", | 20260508 08:16:12.517 | INFO | "pinned" : " 29 GB" | 20260508 08:16:12.517 | INFO | } | 20260508 08:16:12.517 | INFO | Perf report for the last 5.14472 seconds: | 20260508 08:16:12.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:12.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 08:16:12.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:16:12.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:12.517 | INFO | Uploaded 36GiB to GPU in 2.48688s (14.476GiB/s) | 20260508 08:16:12.517 | INFO | Downloaded 34GiB to GPU in 2.45585s (13.8445GiB/s) | 20260508 08:16:17.656 | INFO | Analyzing chunk frames = [6599680,6600192) | 20260508 08:16:17.656 | INFO | MemInfo: { | 20260508 08:16:17.656 | INFO | "gpu" : " 17 GB", | 20260508 08:16:17.656 | INFO | "huge" : "", | 20260508 08:16:17.656 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:17.656 | INFO | "malloc" : " 14 GB", | 20260508 08:16:17.656 | INFO | "pinned" : " 29 GB" | 20260508 08:16:17.656 | INFO | } | 20260508 08:16:17.656 | INFO | Perf report for the last 5.13918 seconds: | 20260508 08:16:17.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:17.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.656 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 08:16:17.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.656 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:16:17.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:17.657 | INFO | Uploaded 36GiB to GPU in 2.46273s (14.6179GiB/s) | 20260508 08:16:17.657 | INFO | Downloaded 34GiB to GPU in 2.44717s (13.8936GiB/s) | 20260508 08:16:22.794 | INFO | Analyzing chunk frames = [6600192,6600704) | 20260508 08:16:22.794 | INFO | MemInfo: { | 20260508 08:16:22.794 | INFO | "gpu" : " 17 GB", | 20260508 08:16:22.794 | INFO | "huge" : "", | 20260508 08:16:22.794 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:22.794 | INFO | "malloc" : " 14 GB", | 20260508 08:16:22.794 | INFO | "pinned" : " 29 GB" | 20260508 08:16:22.794 | INFO | } | 20260508 08:16:22.794 | INFO | Perf report for the last 5.13787 seconds: | 20260508 08:16:22.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:22.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.794 | INFO | ANALYSIS : 66.8%, 2.9, 103.3ms, 0 | 20260508 08:16:22.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.794 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:16:22.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:22.795 | INFO | Uploaded 36GiB to GPU in 2.54057s (14.17GiB/s) | 20260508 08:16:22.795 | INFO | Downloaded 34GiB to GPU in 2.44759s (13.8912GiB/s) | 20260508 08:16:27.926 | INFO | Analyzing chunk frames = [6600704,6601216) | 20260508 08:16:27.926 | INFO | MemInfo: { | 20260508 08:16:27.926 | INFO | "gpu" : " 17 GB", | 20260508 08:16:27.926 | INFO | "huge" : "", | 20260508 08:16:27.926 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:27.926 | INFO | "malloc" : " 14 GB", | 20260508 08:16:27.926 | INFO | "pinned" : " 29 GB" | 20260508 08:16:27.926 | INFO | } | 20260508 08:16:27.926 | INFO | Perf report for the last 5.13268 seconds: | 20260508 08:16:27.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:27.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:16:27.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:16:27.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:27.927 | INFO | Uploaded 36GiB to GPU in 2.33632s (15.4088GiB/s) | 20260508 08:16:27.927 | INFO | Downloaded 34GiB to GPU in 2.52766s (13.4512GiB/s) | 20260508 08:16:33.068 | INFO | Analyzing chunk frames = [6601216,6601728) | 20260508 08:16:33.068 | INFO | MemInfo: { | 20260508 08:16:33.068 | INFO | "gpu" : " 17 GB", | 20260508 08:16:33.068 | INFO | "huge" : "", | 20260508 08:16:33.068 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:33.068 | INFO | "malloc" : " 14 GB", | 20260508 08:16:33.068 | INFO | "pinned" : " 29 GB" | 20260508 08:16:33.068 | INFO | } | 20260508 08:16:33.068 | INFO | Perf report for the last 5.14187 seconds: | 20260508 08:16:33.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:33.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.068 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 08:16:33.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.068 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:16:33.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:33.069 | INFO | Uploaded 36GiB to GPU in 2.44916s (14.6989GiB/s) | 20260508 08:16:33.069 | INFO | Downloaded 34GiB to GPU in 2.44525s (13.9045GiB/s) | 20260508 08:16:38.215 | INFO | Analyzing chunk frames = [6601728,6602240) | 20260508 08:16:38.215 | INFO | MemInfo: { | 20260508 08:16:38.215 | INFO | "gpu" : " 17 GB", | 20260508 08:16:38.215 | INFO | "huge" : "", | 20260508 08:16:38.215 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:38.215 | INFO | "malloc" : " 14 GB", | 20260508 08:16:38.215 | INFO | "pinned" : " 29 GB" | 20260508 08:16:38.215 | INFO | } | 20260508 08:16:38.215 | INFO | Perf report for the last 5.14685 seconds: | 20260508 08:16:38.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:38.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.215 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:16:38.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.215 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:16:38.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:38.216 | INFO | Uploaded 36GiB to GPU in 2.47761s (14.5302GiB/s) | 20260508 08:16:38.216 | INFO | Downloaded 34GiB to GPU in 2.44442s (13.9093GiB/s) | 20260508 08:16:43.359 | INFO | Analyzing chunk frames = [6602240,6602752) | 20260508 08:16:43.359 | INFO | MemInfo: { | 20260508 08:16:43.359 | INFO | "gpu" : " 17 GB", | 20260508 08:16:43.359 | INFO | "huge" : "", | 20260508 08:16:43.359 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:43.359 | INFO | "malloc" : " 14 GB", | 20260508 08:16:43.360 | INFO | "pinned" : " 29 GB" | 20260508 08:16:43.360 | INFO | } | 20260508 08:16:43.360 | INFO | Perf report for the last 5.14437 seconds: | 20260508 08:16:43.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:43.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | ANALYSIS : 65.7%, 3.0, 102.9ms, 0 | 20260508 08:16:43.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 08:16:43.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:43.360 | INFO | Uploaded 36GiB to GPU in 2.42721s (14.8318GiB/s) | 20260508 08:16:43.360 | INFO | Downloaded 34GiB to GPU in 2.43668s (13.9534GiB/s) | 20260508 08:16:43.360 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:16:43.360 | INFO | Total mem usage: 16.7 GiB | 20260508 08:16:43.360 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:16:43.360 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13%) | 20260508 08:16:43.360 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:16:43.360 | INFO | High Water Breakdown: | 20260508 08:16:43.360 | INFO | 60% : 8.86 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:16:43.360 | INFO | 39.8% : 5.88 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:16:43.360 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:16:43.360 | INFO | ------------------------------------------ | 20260508 08:16:43.360 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:16:43.360 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:16:43.360 | INFO | Total mem usage: 26.5 GiB | 20260508 08:16:43.360 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:16:43.360 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 08:16:43.360 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 08:16:43.360 | INFO | High Water Breakdown: | 20260508 08:16:43.360 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:16:43.360 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:16:43.360 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:16:43.360 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:16:43.360 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:16:43.360 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:16:43.360 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:16:43.360 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:16:43.360 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:16:43.360 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:16:43.360 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:16:43.360 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:16:43.360 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:16:43.360 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:16:43.360 | INFO | ------------------------------------------ | 20260508 08:16:43.360 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:16:43.360 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:16:43.360 | INFO | Total mem usage: 27.8 GiB | 20260508 08:16:43.360 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:16:43.360 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:16:43.360 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:16:43.360 | INFO | High Water Breakdown: | 20260508 08:16:43.360 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:16:43.360 | INFO | ------------------------------------------ | 20260508 08:16:43.360 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:16:43.360 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:16:43.360 | INFO | Total mem usage: 15.7 GiB | 20260508 08:16:43.360 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:16:43.360 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:16:43.360 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:16:43.360 | INFO | High Water Breakdown: | 20260508 08:16:43.360 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:16:43.360 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:16:43.360 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:16:43.360 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:16:43.360 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:16:43.360 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:16:43.360 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:16:43.360 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:16:43.360 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:16:43.360 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:16:43.360 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:16:43.360 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:16:43.360 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:16:43.360 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:16:43.360 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:16:43.360 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:16:43.360 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:16:43.360 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:16:43.360 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:16:43.360 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:16:43.360 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:16:43.360 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:16:43.360 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:16:43.360 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:16:43.360 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:16:43.360 | INFO | ------------------------------------------ | 20260508 08:16:43.360 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:16:43.360 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:16:43.360 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:16:43.360 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:16:43.360 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.38e+01 / 6.92e-01 | 20260508 08:16:43.360 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.15e-02 / 4.59e-03 | 20260508 08:16:43.360 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.38e+00, 5.91e-01 / 2.24e-01 | 20260508 08:16:43.360 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.42e+00, 2.07e-01 / 1.36e-01 | 20260508 08:16:43.360 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.49e+01, 3.60e+00 / 1.11e+00 | 20260508 08:16:43.360 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.85e+00 / 1.38e+00 | 20260508 08:16:43.360 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.47e+00, 4.12e-01 / 2.57e-01 | 20260508 08:16:43.360 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.22e+00, 2.52e-01 / 1.53e-01 | 20260508 08:16:43.361 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.39e-01, 4.99e-02 / 2.71e-02 | 20260508 08:16:43.361 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:16:43.361 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:16:43.361 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:16:43.361 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 24.41%, Upload -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 24.48%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 30.91%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 2.19%, LockUpload -- 2.32e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 2.56%, LockDownload -- 2.71e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 15.43%, LockCompute -- 1.63e+01 / 0.00e+00, 1.91e+01 / 0.00e+00 | 20260508 08:16:43.361 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260508 08:16:43.473 | INFO | Starting a round of Baz IO | 20260508 08:16:46.875 | INFO | Populating RTMetrics report | 20260508 08:16:46.876 | INFO | RTMetrics report populated | 20260508 08:16:46.876 | INFO | Opening tmp RTMetrics json file | 20260508 08:16:46.889 | INFO | Serializing RTMetrics to json | 20260508 08:16:46.890 | INFO | Serialization done, swapping tmp file now... | 20260508 08:16:46.890 | INFO | RTMetrics done | 20260508 08:16:46.949 | INFO | Finished a round of Baz IO: Wrote 1.507 GiB in 3.475 seconds | 20260508 08:16:46.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.56% | 20260508 08:16:46.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.302366% / 0% / 5.96046e-05% | 20260508 08:16:46.949 | INFO | Baz IO overall: | 20260508 08:16:46.949 | INFO | BytesWritten : 2900153781399 | 20260508 08:16:46.949 | INFO | NumEvents : 1396402506964 | 20260508 08:16:48.487 | INFO | Analyzing chunk frames = [6602752,6603264) | 20260508 08:16:48.487 | INFO | MemInfo: { | 20260508 08:16:48.487 | INFO | "gpu" : " 17 GB", | 20260508 08:16:48.487 | INFO | "huge" : "", | 20260508 08:16:48.488 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:48.488 | INFO | "malloc" : " 14 GB", | 20260508 08:16:48.488 | INFO | "pinned" : " 29 GB" | 20260508 08:16:48.488 | INFO | } | 20260508 08:16:48.488 | INFO | Perf report for the last 5.12789 seconds: | 20260508 08:16:48.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:48.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:48.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:48.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:48.488 | INFO | ANALYSIS : 67.9%, 3.0, 105.2ms, 0 | 20260508 08:16:48.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:48.488 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260508 08:16:48.488 | INFO | PRE_HQ : 66.8%, 1.8, 63.8ms, 0 | 20260508 08:16:48.488 | INFO | BAZWRITER : 67.1%, 2.8, 97.8ms, 0 | 20260508 08:16:48.488 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:16:48.488 | INFO | Uploaded 36GiB to GPU in 2.6308s (13.684GiB/s) | 20260508 08:16:48.488 | INFO | Downloaded 38GiB to GPU in 2.70501s (14.048GiB/s) | 20260508 08:16:53.629 | INFO | Analyzing chunk frames = [6603264,6603776) | 20260508 08:16:53.629 | INFO | MemInfo: { | 20260508 08:16:53.629 | INFO | "gpu" : " 17 GB", | 20260508 08:16:53.629 | INFO | "huge" : "", | 20260508 08:16:53.629 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:53.629 | INFO | "malloc" : " 14 GB", | 20260508 08:16:53.629 | INFO | "pinned" : " 29 GB" | 20260508 08:16:53.629 | INFO | } | 20260508 08:16:53.629 | INFO | Perf report for the last 5.14137 seconds: | 20260508 08:16:53.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:53.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:16:53.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:16:53.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:53.629 | INFO | Uploaded 36GiB to GPU in 2.52001s (14.2856GiB/s) | 20260508 08:16:53.629 | INFO | Downloaded 34GiB to GPU in 2.43332s (13.9727GiB/s) | 20260508 08:16:58.767 | INFO | Analyzing chunk frames = [6603776,6604288) | 20260508 08:16:58.767 | INFO | MemInfo: { | 20260508 08:16:58.767 | INFO | "gpu" : " 17 GB", | 20260508 08:16:58.767 | INFO | "huge" : "", | 20260508 08:16:58.767 | INFO | "huge_shared" : " 29 GB", | 20260508 08:16:58.767 | INFO | "malloc" : " 14 GB", | 20260508 08:16:58.767 | INFO | "pinned" : " 29 GB" | 20260508 08:16:58.767 | INFO | } | 20260508 08:16:58.767 | INFO | Perf report for the last 5.13824 seconds: | 20260508 08:16:58.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:16:58.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.767 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 08:16:58.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.767 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:16:58.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:16:58.768 | INFO | Uploaded 36GiB to GPU in 2.49595s (14.4234GiB/s) | 20260508 08:16:58.768 | INFO | Downloaded 34GiB to GPU in 2.43546s (13.9604GiB/s) | 20260508 08:17:03.909 | INFO | Analyzing chunk frames = [6604288,6604800) | 20260508 08:17:03.909 | INFO | MemInfo: { | 20260508 08:17:03.909 | INFO | "gpu" : " 17 GB", | 20260508 08:17:03.909 | INFO | "huge" : "", | 20260508 08:17:03.909 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:03.909 | INFO | "malloc" : " 14 GB", | 20260508 08:17:03.909 | INFO | "pinned" : " 29 GB" | 20260508 08:17:03.909 | INFO | } | 20260508 08:17:03.909 | INFO | Perf report for the last 5.14171 seconds: | 20260508 08:17:03.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:03.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 08:17:03.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:17:03.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:03.909 | INFO | Uploaded 36GiB to GPU in 2.58692s (13.9161GiB/s) | 20260508 08:17:03.909 | INFO | Downloaded 34GiB to GPU in 2.4113s (14.1003GiB/s) | 20260508 08:17:09.045 | INFO | Analyzing chunk frames = [6604800,6605312) | 20260508 08:17:09.045 | INFO | MemInfo: { | 20260508 08:17:09.045 | INFO | "gpu" : " 17 GB", | 20260508 08:17:09.045 | INFO | "huge" : "", | 20260508 08:17:09.045 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:09.045 | INFO | "malloc" : " 14 GB", | 20260508 08:17:09.045 | INFO | "pinned" : " 29 GB" | 20260508 08:17:09.045 | INFO | } | 20260508 08:17:09.045 | INFO | Perf report for the last 5.13658 seconds: | 20260508 08:17:09.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:09.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 08:17:09.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:17:09.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:09.046 | INFO | Uploaded 36GiB to GPU in 2.40297s (14.9814GiB/s) | 20260508 08:17:09.046 | INFO | Downloaded 34GiB to GPU in 2.51198s (13.5351GiB/s) | 20260508 08:17:14.184 | INFO | Analyzing chunk frames = [6605312,6605824) | 20260508 08:17:14.184 | INFO | MemInfo: { | 20260508 08:17:14.184 | INFO | "gpu" : " 17 GB", | 20260508 08:17:14.184 | INFO | "huge" : "", | 20260508 08:17:14.184 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:14.184 | INFO | "malloc" : " 14 GB", | 20260508 08:17:14.184 | INFO | "pinned" : " 29 GB" | 20260508 08:17:14.184 | INFO | } | 20260508 08:17:14.184 | INFO | Perf report for the last 5.13831 seconds: | 20260508 08:17:14.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:14.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 08:17:14.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:17:14.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:14.184 | INFO | Uploaded 36GiB to GPU in 2.4611s (14.6276GiB/s) | 20260508 08:17:14.184 | INFO | Downloaded 34GiB to GPU in 2.43499s (13.9631GiB/s) | 20260508 08:17:19.330 | INFO | Analyzing chunk frames = [6605824,6606336) | 20260508 08:17:19.330 | INFO | MemInfo: { | 20260508 08:17:19.330 | INFO | "gpu" : " 17 GB", | 20260508 08:17:19.330 | INFO | "huge" : "", | 20260508 08:17:19.330 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:19.330 | INFO | "malloc" : " 14 GB", | 20260508 08:17:19.330 | INFO | "pinned" : " 29 GB" | 20260508 08:17:19.330 | INFO | } | 20260508 08:17:19.330 | INFO | Perf report for the last 5.14654 seconds: | 20260508 08:17:19.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:19.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.330 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 08:17:19.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.331 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:17:19.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:19.331 | INFO | Uploaded 36GiB to GPU in 2.43918s (14.7591GiB/s) | 20260508 08:17:19.331 | INFO | Downloaded 34GiB to GPU in 2.43611s (13.9567GiB/s) | 20260508 08:17:24.465 | INFO | Analyzing chunk frames = [6606336,6606848) | 20260508 08:17:24.465 | INFO | MemInfo: { | 20260508 08:17:24.465 | INFO | "gpu" : " 17 GB", | 20260508 08:17:24.465 | INFO | "huge" : "", | 20260508 08:17:24.465 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:24.465 | INFO | "malloc" : " 14 GB", | 20260508 08:17:24.465 | INFO | "pinned" : " 29 GB" | 20260508 08:17:24.465 | INFO | } | 20260508 08:17:24.465 | INFO | Perf report for the last 5.13478 seconds: | 20260508 08:17:24.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:24.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 08:17:24.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:17:24.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:24.465 | INFO | Uploaded 36GiB to GPU in 2.44904s (14.6996GiB/s) | 20260508 08:17:24.465 | INFO | Downloaded 34GiB to GPU in 2.42586s (14.0156GiB/s) | 20260508 08:17:24.562 | INFO | Starting a round of Baz IO | 20260508 08:17:27.972 | INFO | Populating RTMetrics report | 20260508 08:17:27.972 | INFO | RTMetrics report populated | 20260508 08:17:27.972 | INFO | Opening tmp RTMetrics json file | 20260508 08:17:27.983 | INFO | Serializing RTMetrics to json | 20260508 08:17:27.984 | INFO | Serialization done, swapping tmp file now... | 20260508 08:17:27.985 | INFO | RTMetrics done | 20260508 08:17:28.044 | INFO | Finished a round of Baz IO: Wrote 1.507 GiB in 3.482 seconds | 20260508 08:17:28.044 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.56% | 20260508 08:17:28.044 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.302748% / 3.97364e-06% / 5.16574e-05% | 20260508 08:17:28.044 | INFO | Baz IO overall: | 20260508 08:17:28.044 | INFO | BytesWritten : 2901771988524 | 20260508 08:17:28.044 | INFO | NumEvents : 1397169564078 | 20260508 08:17:29.604 | INFO | Analyzing chunk frames = [6606848,6607360) | 20260508 08:17:29.604 | INFO | MemInfo: { | 20260508 08:17:29.604 | INFO | "gpu" : " 17 GB", | 20260508 08:17:29.604 | INFO | "huge" : "", | 20260508 08:17:29.604 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:29.604 | INFO | "malloc" : " 14 GB", | 20260508 08:17:29.604 | INFO | "pinned" : " 29 GB" | 20260508 08:17:29.604 | INFO | } | 20260508 08:17:29.604 | INFO | Perf report for the last 5.13886 seconds: | 20260508 08:17:29.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:29.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:29.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:29.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:29.604 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 08:17:29.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:29.604 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:17:29.604 | INFO | PRE_HQ : 65.5%, 1.6, 54.7ms, 0 | 20260508 08:17:29.604 | INFO | BAZWRITER : 67.1%, 2.6, 89.9ms, 0 | 20260508 08:17:29.604 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:17:29.604 | INFO | Uploaded 36GiB to GPU in 2.60548s (13.817GiB/s) | 20260508 08:17:29.604 | INFO | Downloaded 38GiB to GPU in 2.70175s (14.0649GiB/s) | 20260508 08:17:34.740 | INFO | Analyzing chunk frames = [6607360,6607872) | 20260508 08:17:34.740 | INFO | MemInfo: { | 20260508 08:17:34.740 | INFO | "gpu" : " 17 GB", | 20260508 08:17:34.740 | INFO | "huge" : "", | 20260508 08:17:34.740 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:34.740 | INFO | "malloc" : " 14 GB", | 20260508 08:17:34.740 | INFO | "pinned" : " 29 GB" | 20260508 08:17:34.740 | INFO | } | 20260508 08:17:34.740 | INFO | Perf report for the last 5.1359 seconds: | 20260508 08:17:34.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:34.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 08:17:34.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:17:34.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:34.740 | INFO | Uploaded 36GiB to GPU in 2.51942s (14.289GiB/s) | 20260508 08:17:34.740 | INFO | Downloaded 34GiB to GPU in 2.43173s (13.9818GiB/s) | 20260508 08:17:39.882 | INFO | Analyzing chunk frames = [6607872,6608384) | 20260508 08:17:39.882 | INFO | MemInfo: { | 20260508 08:17:39.882 | INFO | "gpu" : " 17 GB", | 20260508 08:17:39.882 | INFO | "huge" : "", | 20260508 08:17:39.882 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:39.882 | INFO | "malloc" : " 14 GB", | 20260508 08:17:39.882 | INFO | "pinned" : " 29 GB" | 20260508 08:17:39.882 | INFO | } | 20260508 08:17:39.883 | INFO | Perf report for the last 5.14261 seconds: | 20260508 08:17:39.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:39.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 08:17:39.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:17:39.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:39.883 | INFO | Uploaded 36GiB to GPU in 2.59535s (13.8709GiB/s) | 20260508 08:17:39.883 | INFO | Downloaded 34GiB to GPU in 2.43413s (13.968GiB/s) | 20260508 08:17:45.025 | INFO | Analyzing chunk frames = [6608384,6608896) | 20260508 08:17:45.025 | INFO | MemInfo: { | 20260508 08:17:45.025 | INFO | "gpu" : " 17 GB", | 20260508 08:17:45.025 | INFO | "huge" : "", | 20260508 08:17:45.025 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:45.025 | INFO | "malloc" : " 14 GB", | 20260508 08:17:45.025 | INFO | "pinned" : " 29 GB" | 20260508 08:17:45.025 | INFO | } | 20260508 08:17:45.025 | INFO | Perf report for the last 5.1429 seconds: | 20260508 08:17:45.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:45.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:17:45.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:17:45.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:45.026 | INFO | Uploaded 36GiB to GPU in 2.52772s (14.2421GiB/s) | 20260508 08:17:45.026 | INFO | Downloaded 34GiB to GPU in 2.42936s (13.9955GiB/s) | 20260508 08:17:50.166 | INFO | Analyzing chunk frames = [6608896,6609408) | 20260508 08:17:50.166 | INFO | MemInfo: { | 20260508 08:17:50.166 | INFO | "gpu" : " 17 GB", | 20260508 08:17:50.166 | INFO | "huge" : "", | 20260508 08:17:50.166 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:50.166 | INFO | "malloc" : " 14 GB", | 20260508 08:17:50.166 | INFO | "pinned" : " 29 GB" | 20260508 08:17:50.166 | INFO | } | 20260508 08:17:50.166 | INFO | Perf report for the last 5.14016 seconds: | 20260508 08:17:50.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:50.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:17:50.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:17:50.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:50.166 | INFO | Uploaded 36GiB to GPU in 2.35904s (15.2604GiB/s) | 20260508 08:17:50.166 | INFO | Downloaded 34GiB to GPU in 2.49936s (13.6035GiB/s) | 20260508 08:17:55.302 | INFO | Analyzing chunk frames = [6609408,6609920) | 20260508 08:17:55.302 | INFO | MemInfo: { | 20260508 08:17:55.302 | INFO | "gpu" : " 17 GB", | 20260508 08:17:55.302 | INFO | "huge" : "", | 20260508 08:17:55.302 | INFO | "huge_shared" : " 29 GB", | 20260508 08:17:55.302 | INFO | "malloc" : " 14 GB", | 20260508 08:17:55.302 | INFO | "pinned" : " 29 GB" | 20260508 08:17:55.302 | INFO | } | 20260508 08:17:55.302 | INFO | Perf report for the last 5.13641 seconds: | 20260508 08:17:55.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:17:55.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.302 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 08:17:55.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.302 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 08:17:55.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:17:55.303 | INFO | Uploaded 36GiB to GPU in 2.40831s (14.9482GiB/s) | 20260508 08:17:55.303 | INFO | Downloaded 34GiB to GPU in 2.42061s (14.046GiB/s) | 20260508 08:18:00.436 | INFO | Analyzing chunk frames = [6609920,6610432) | 20260508 08:18:00.436 | INFO | MemInfo: { | 20260508 08:18:00.436 | INFO | "gpu" : " 17 GB", | 20260508 08:18:00.436 | INFO | "huge" : "", | 20260508 08:18:00.436 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:00.436 | INFO | "malloc" : " 14 GB", | 20260508 08:18:00.436 | INFO | "pinned" : " 29 GB" | 20260508 08:18:00.436 | INFO | } | 20260508 08:18:00.436 | INFO | Perf report for the last 5.13398 seconds: | 20260508 08:18:00.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:00.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.436 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 08:18:00.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.436 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 08:18:00.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:00.437 | INFO | Uploaded 36GiB to GPU in 2.43841s (14.7637GiB/s) | 20260508 08:18:00.437 | INFO | Downloaded 34GiB to GPU in 2.42498s (14.0208GiB/s) | 20260508 08:18:05.577 | INFO | Analyzing chunk frames = [6610432,6610944) | 20260508 08:18:05.577 | INFO | MemInfo: { | 20260508 08:18:05.577 | INFO | "gpu" : " 17 GB", | 20260508 08:18:05.577 | INFO | "huge" : "", | 20260508 08:18:05.577 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:05.577 | INFO | "malloc" : " 14 GB", | 20260508 08:18:05.577 | INFO | "pinned" : " 29 GB" | 20260508 08:18:05.577 | INFO | } | 20260508 08:18:05.577 | INFO | Perf report for the last 5.14129 seconds: | 20260508 08:18:05.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:05.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.578 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:18:05.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.578 | INFO | PRE_HQ : 34.4%, 1.0, 17.8ms, 0 | 20260508 08:18:05.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:05.578 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:18:05.578 | INFO | Total mem usage: 16.7 GiB | 20260508 08:18:05.578 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:18:05.578 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.5%) | 20260508 08:18:05.578 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:18:05.578 | INFO | High Water Breakdown: | 20260508 08:18:05.578 | INFO | 59.7% : 8.62 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:18:05.578 | INFO | 40.2% : 5.8 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:18:05.578 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:18:05.578 | INFO | ------------------------------------------ | 20260508 08:18:05.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:18:05.578 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:18:05.578 | INFO | Total mem usage: 26.5 GiB | 20260508 08:18:05.578 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:18:05.578 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 08:18:05.578 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 08:18:05.578 | INFO | High Water Breakdown: | 20260508 08:18:05.578 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:18:05.578 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:18:05.578 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:18:05.578 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:18:05.578 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:18:05.578 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:18:05.578 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:18:05.578 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:18:05.578 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:18:05.578 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:18:05.578 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:18:05.578 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:18:05.578 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:18:05.578 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:18:05.578 | INFO | ------------------------------------------ | 20260508 08:18:05.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:18:05.578 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:18:05.578 | INFO | Total mem usage: 27.8 GiB | 20260508 08:18:05.578 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:18:05.578 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:18:05.578 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:18:05.578 | INFO | High Water Breakdown: | 20260508 08:18:05.578 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:18:05.578 | INFO | ------------------------------------------ | 20260508 08:18:05.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:18:05.578 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:18:05.578 | INFO | Total mem usage: 15.7 GiB | 20260508 08:18:05.578 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:18:05.578 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:18:05.578 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:18:05.578 | INFO | High Water Breakdown: | 20260508 08:18:05.578 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:18:05.578 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:18:05.578 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:18:05.578 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:18:05.578 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:18:05.578 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:18:05.578 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:18:05.578 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:18:05.578 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:18:05.578 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:18:05.578 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:18:05.578 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:18:05.578 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:18:05.578 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:18:05.578 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:18:05.578 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:18:05.578 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:18:05.578 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:18:05.578 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:18:05.578 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:18:05.578 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:18:05.578 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:18:05.578 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:18:05.578 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:18:05.578 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:18:05.578 | INFO | ------------------------------------------ | 20260508 08:18:05.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:18:05.578 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:18:05.578 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:18:05.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:18:05.578 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 7.81e-01 | 20260508 08:18:05.578 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.04e-01, 1.18e-02 / 6.75e-03 | 20260508 08:18:05.578 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.43e+00, 5.90e-01 / 2.25e-01 | 20260508 08:18:05.578 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.43e+00, 2.10e-01 / 1.49e-01 | 20260508 08:18:05.578 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.50e+01, 3.70e+00 / 1.16e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.83e+00 / 1.44e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.48e+00, 4.07e-01 / 2.76e-01 | 20260508 08:18:05.578 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.19e+00, 2.53e-01 / 1.51e-01 | 20260508 08:18:05.578 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.42e-01, 5.00e-02 / 2.72e-02 | 20260508 08:18:05.578 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:18:05.578 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:18:05.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:18:05.578 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 24.57%, Upload -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 24.37%, Download -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 30.96%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 2.42%, LockUpload -- 2.56e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 2.31%, LockDownload -- 2.44e+00 / 0.00e+00, 3.73e+00 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 08:18:05.578 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 9.15e-03 / 0.00e+00 | 20260508 08:18:05.578 | INFO | Uploaded 36GiB to GPU in 2.46644s (14.596GiB/s) | 20260508 08:18:05.578 | INFO | Downloaded 34GiB to GPU in 2.43601s (13.9573GiB/s) | 20260508 08:18:05.687 | INFO | Starting a round of Baz IO | 20260508 08:18:09.108 | INFO | Populating RTMetrics report | 20260508 08:18:09.108 | INFO | RTMetrics report populated | 20260508 08:18:09.108 | INFO | Opening tmp RTMetrics json file | 20260508 08:18:09.118 | INFO | Serializing RTMetrics to json | 20260508 08:18:09.119 | INFO | Serialization done, swapping tmp file now... | 20260508 08:18:09.120 | INFO | RTMetrics done | 20260508 08:18:09.174 | INFO | Finished a round of Baz IO: Wrote 1.507 GiB in 3.486 seconds | 20260508 08:18:09.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.56% | 20260508 08:18:09.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.3021% / 3.97364e-06% / 5.5631e-05% | 20260508 08:18:09.174 | INFO | Baz IO overall: | 20260508 08:18:09.175 | INFO | BytesWritten : 2903390310963 | 20260508 08:18:09.175 | INFO | NumEvents : 1397936261674 | 20260508 08:18:10.722 | INFO | Analyzing chunk frames = [6610944,6611456) | 20260508 08:18:10.722 | INFO | MemInfo: { | 20260508 08:18:10.722 | INFO | "gpu" : " 17 GB", | 20260508 08:18:10.722 | INFO | "huge" : "", | 20260508 08:18:10.722 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:10.722 | INFO | "malloc" : " 14 GB", | 20260508 08:18:10.722 | INFO | "pinned" : " 29 GB" | 20260508 08:18:10.722 | INFO | } | 20260508 08:18:10.722 | INFO | Perf report for the last 5.14417 seconds: | 20260508 08:18:10.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:10.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:10.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:10.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:10.722 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 08:18:10.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:10.722 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260508 08:18:10.722 | INFO | PRE_HQ : 66.4%, 1.8, 60.9ms, 0 | 20260508 08:18:10.722 | INFO | BAZWRITER : 67.0%, 2.7, 94.3ms, 0 | 20260508 08:18:10.722 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:18:10.722 | INFO | Uploaded 36GiB to GPU in 2.55706s (14.0787GiB/s) | 20260508 08:18:10.722 | INFO | Downloaded 38GiB to GPU in 2.7103s (14.0206GiB/s) | 20260508 08:18:15.861 | INFO | Analyzing chunk frames = [6611456,6611968) | 20260508 08:18:15.861 | INFO | MemInfo: { | 20260508 08:18:15.861 | INFO | "gpu" : " 17 GB", | 20260508 08:18:15.861 | INFO | "huge" : "", | 20260508 08:18:15.861 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:15.861 | INFO | "malloc" : " 14 GB", | 20260508 08:18:15.861 | INFO | "pinned" : " 29 GB" | 20260508 08:18:15.861 | INFO | } | 20260508 08:18:15.861 | INFO | Perf report for the last 5.13946 seconds: | 20260508 08:18:15.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:15.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:18:15.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:18:15.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:15.861 | INFO | Uploaded 36GiB to GPU in 2.53029s (14.2276GiB/s) | 20260508 08:18:15.861 | INFO | Downloaded 34GiB to GPU in 2.44149s (13.9259GiB/s) | 20260508 08:18:20.999 | INFO | Analyzing chunk frames = [6611968,6612480) | 20260508 08:18:20.999 | INFO | MemInfo: { | 20260508 08:18:20.999 | INFO | "gpu" : " 17 GB", | 20260508 08:18:20.999 | INFO | "huge" : "", | 20260508 08:18:20.999 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:20.999 | INFO | "malloc" : " 14 GB", | 20260508 08:18:20.999 | INFO | "pinned" : " 29 GB" | 20260508 08:18:20.999 | INFO | } | 20260508 08:18:20.999 | INFO | Perf report for the last 5.13833 seconds: | 20260508 08:18:21.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:21.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 08:18:21.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:18:21.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:21.000 | INFO | Uploaded 36GiB to GPU in 2.51482s (14.3151GiB/s) | 20260508 08:18:21.000 | INFO | Downloaded 34GiB to GPU in 2.43421s (13.9676GiB/s) | 20260508 08:18:26.137 | INFO | Analyzing chunk frames = [6612480,6612992) | 20260508 08:18:26.137 | INFO | MemInfo: { | 20260508 08:18:26.137 | INFO | "gpu" : " 17 GB", | 20260508 08:18:26.137 | INFO | "huge" : "", | 20260508 08:18:26.137 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:26.137 | INFO | "malloc" : " 14 GB", | 20260508 08:18:26.137 | INFO | "pinned" : " 29 GB" | 20260508 08:18:26.137 | INFO | } | 20260508 08:18:26.137 | INFO | Perf report for the last 5.13721 seconds: | 20260508 08:18:26.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:26.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 08:18:26.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:18:26.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:26.137 | INFO | Uploaded 36GiB to GPU in 2.52739s (14.2439GiB/s) | 20260508 08:18:26.137 | INFO | Downloaded 34GiB to GPU in 2.43391s (13.9693GiB/s) | 20260508 08:18:31.277 | INFO | Analyzing chunk frames = [6612992,6613504) | 20260508 08:18:31.277 | INFO | MemInfo: { | 20260508 08:18:31.277 | INFO | "gpu" : " 17 GB", | 20260508 08:18:31.277 | INFO | "huge" : "", | 20260508 08:18:31.277 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:31.277 | INFO | "malloc" : " 14 GB", | 20260508 08:18:31.277 | INFO | "pinned" : " 29 GB" | 20260508 08:18:31.277 | INFO | } | 20260508 08:18:31.277 | INFO | Perf report for the last 5.14002 seconds: | 20260508 08:18:31.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:31.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 08:18:31.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:18:31.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:31.277 | INFO | Uploaded 36GiB to GPU in 2.30855s (15.5942GiB/s) | 20260508 08:18:31.277 | INFO | Downloaded 34GiB to GPU in 2.50031s (13.5983GiB/s) | 20260508 08:18:36.413 | INFO | Analyzing chunk frames = [6613504,6614016) | 20260508 08:18:36.413 | INFO | MemInfo: { | 20260508 08:18:36.413 | INFO | "gpu" : " 17 GB", | 20260508 08:18:36.413 | INFO | "huge" : "", | 20260508 08:18:36.413 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:36.414 | INFO | "malloc" : " 14 GB", | 20260508 08:18:36.414 | INFO | "pinned" : " 29 GB" | 20260508 08:18:36.414 | INFO | } | 20260508 08:18:36.414 | INFO | Perf report for the last 5.13699 seconds: | 20260508 08:18:36.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:36.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 08:18:36.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:18:36.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:36.414 | INFO | Uploaded 36GiB to GPU in 2.40571s (14.9644GiB/s) | 20260508 08:18:36.414 | INFO | Downloaded 34GiB to GPU in 2.42282s (14.0332GiB/s) | 20260508 08:18:41.552 | INFO | Analyzing chunk frames = [6614016,6614528) | 20260508 08:18:41.552 | INFO | MemInfo: { | 20260508 08:18:41.552 | INFO | "gpu" : " 17 GB", | 20260508 08:18:41.552 | INFO | "huge" : "", | 20260508 08:18:41.553 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:41.553 | INFO | "malloc" : " 14 GB", | 20260508 08:18:41.553 | INFO | "pinned" : " 29 GB" | 20260508 08:18:41.553 | INFO | } | 20260508 08:18:41.553 | INFO | Perf report for the last 5.13906 seconds: | 20260508 08:18:41.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:41.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 08:18:41.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 08:18:41.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:41.553 | INFO | Uploaded 36GiB to GPU in 2.46406s (14.61GiB/s) | 20260508 08:18:41.553 | INFO | Downloaded 34GiB to GPU in 2.43942s (13.9377GiB/s) | 20260508 08:18:46.691 | INFO | Analyzing chunk frames = [6614528,6615040) | 20260508 08:18:46.691 | INFO | MemInfo: { | 20260508 08:18:46.691 | INFO | "gpu" : " 17 GB", | 20260508 08:18:46.691 | INFO | "huge" : "", | 20260508 08:18:46.692 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:46.692 | INFO | "malloc" : " 14 GB", | 20260508 08:18:46.692 | INFO | "pinned" : " 29 GB" | 20260508 08:18:46.692 | INFO | } | 20260508 08:18:46.692 | INFO | Perf report for the last 5.13899 seconds: | 20260508 08:18:46.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:46.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:18:46.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 08:18:46.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:46.692 | INFO | Uploaded 36GiB to GPU in 2.45732s (14.6501GiB/s) | 20260508 08:18:46.692 | INFO | Downloaded 34GiB to GPU in 2.44018s (13.9334GiB/s) | 20260508 08:18:46.804 | INFO | Starting a round of Baz IO | 20260508 08:18:50.261 | INFO | Populating RTMetrics report | 20260508 08:18:50.261 | INFO | RTMetrics report populated | 20260508 08:18:50.261 | INFO | Opening tmp RTMetrics json file | 20260508 08:18:50.262 | INFO | Serializing RTMetrics to json | 20260508 08:18:50.263 | INFO | Serialization done, swapping tmp file now... | 20260508 08:18:50.263 | INFO | RTMetrics done | 20260508 08:18:50.344 | INFO | Finished a round of Baz IO: Wrote 1.507 GiB in 3.54 seconds | 20260508 08:18:50.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.56% | 20260508 08:18:50.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.300992% / 0% / 4.37101e-05% | 20260508 08:18:50.344 | INFO | Baz IO overall: | 20260508 08:18:50.344 | INFO | BytesWritten : 2905008855559 | 20260508 08:18:50.344 | INFO | NumEvents : 1398703283677 | 20260508 08:18:51.834 | INFO | Analyzing chunk frames = [6615040,6615552) | 20260508 08:18:51.834 | INFO | MemInfo: { | 20260508 08:18:51.834 | INFO | "gpu" : " 17 GB", | 20260508 08:18:51.834 | INFO | "huge" : "", | 20260508 08:18:51.834 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:51.834 | INFO | "malloc" : " 14 GB", | 20260508 08:18:51.834 | INFO | "pinned" : " 29 GB" | 20260508 08:18:51.834 | INFO | } | 20260508 08:18:51.834 | INFO | Perf report for the last 5.14247 seconds: | 20260508 08:18:51.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:51.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:51.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:51.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:51.834 | INFO | ANALYSIS : 69.1%, 2.9, 106.2ms, 0 | 20260508 08:18:51.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:51.834 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260508 08:18:51.834 | INFO | PRE_HQ : 67.8%, 1.7, 61.1ms, 0 | 20260508 08:18:51.834 | INFO | BAZWRITER : 68.2%, 3.0, 105.8ms, 0 | 20260508 08:18:51.834 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:18:51.835 | INFO | Uploaded 36GiB to GPU in 2.55886s (14.0688GiB/s) | 20260508 08:18:51.835 | INFO | Downloaded 38GiB to GPU in 2.71714s (13.9853GiB/s) | 20260508 08:18:56.978 | INFO | Analyzing chunk frames = [6615552,6616064) | 20260508 08:18:56.978 | INFO | MemInfo: { | 20260508 08:18:56.978 | INFO | "gpu" : " 17 GB", | 20260508 08:18:56.978 | INFO | "huge" : "", | 20260508 08:18:56.978 | INFO | "huge_shared" : " 29 GB", | 20260508 08:18:56.978 | INFO | "malloc" : " 14 GB", | 20260508 08:18:56.978 | INFO | "pinned" : " 29 GB" | 20260508 08:18:56.978 | INFO | } | 20260508 08:18:56.978 | INFO | Perf report for the last 5.14419 seconds: | 20260508 08:18:56.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:18:56.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:18:56.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:18:56.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:18:56.979 | INFO | Uploaded 36GiB to GPU in 2.54316s (14.1556GiB/s) | 20260508 08:18:56.979 | INFO | Downloaded 34GiB to GPU in 2.45444s (13.8525GiB/s) | 20260508 08:19:02.119 | INFO | Analyzing chunk frames = [6616064,6616576) | 20260508 08:19:02.120 | INFO | MemInfo: { | 20260508 08:19:02.120 | INFO | "gpu" : " 17 GB", | 20260508 08:19:02.120 | INFO | "huge" : "", | 20260508 08:19:02.120 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:02.120 | INFO | "malloc" : " 14 GB", | 20260508 08:19:02.120 | INFO | "pinned" : " 29 GB" | 20260508 08:19:02.120 | INFO | } | 20260508 08:19:02.120 | INFO | Perf report for the last 5.14108 seconds: | 20260508 08:19:02.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:02.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 08:19:02.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:19:02.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:02.120 | INFO | Uploaded 36GiB to GPU in 2.59205s (13.8886GiB/s) | 20260508 08:19:02.120 | INFO | Downloaded 34GiB to GPU in 2.43671s (13.9533GiB/s) | 20260508 08:19:07.249 | INFO | Analyzing chunk frames = [6616576,6617088) | 20260508 08:19:07.249 | INFO | MemInfo: { | 20260508 08:19:07.249 | INFO | "gpu" : " 17 GB", | 20260508 08:19:07.249 | INFO | "huge" : "", | 20260508 08:19:07.249 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:07.249 | INFO | "malloc" : " 14 GB", | 20260508 08:19:07.249 | INFO | "pinned" : " 29 GB" | 20260508 08:19:07.249 | INFO | } | 20260508 08:19:07.249 | INFO | Perf report for the last 5.12922 seconds: | 20260508 08:19:07.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:07.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 08:19:07.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:19:07.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:07.249 | INFO | Uploaded 36GiB to GPU in 2.52023s (14.2844GiB/s) | 20260508 08:19:07.249 | INFO | Downloaded 34GiB to GPU in 2.42855s (14.0001GiB/s) | 20260508 08:19:12.390 | INFO | Analyzing chunk frames = [6617088,6617600) | 20260508 08:19:12.390 | INFO | MemInfo: { | 20260508 08:19:12.390 | INFO | "gpu" : " 17 GB", | 20260508 08:19:12.390 | INFO | "huge" : "", | 20260508 08:19:12.390 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:12.390 | INFO | "malloc" : " 14 GB", | 20260508 08:19:12.390 | INFO | "pinned" : " 29 GB" | 20260508 08:19:12.390 | INFO | } | 20260508 08:19:12.391 | INFO | Perf report for the last 5.14174 seconds: | 20260508 08:19:12.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:12.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 08:19:12.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:19:12.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:12.391 | INFO | Uploaded 36GiB to GPU in 2.45567s (14.66GiB/s) | 20260508 08:19:12.391 | INFO | Downloaded 34GiB to GPU in 2.52689s (13.4553GiB/s) | 20260508 08:19:17.530 | INFO | Analyzing chunk frames = [6617600,6618112) | 20260508 08:19:17.530 | INFO | MemInfo: { | 20260508 08:19:17.530 | INFO | "gpu" : " 17 GB", | 20260508 08:19:17.530 | INFO | "huge" : "", | 20260508 08:19:17.530 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:17.530 | INFO | "malloc" : " 14 GB", | 20260508 08:19:17.530 | INFO | "pinned" : " 29 GB" | 20260508 08:19:17.530 | INFO | } | 20260508 08:19:17.531 | INFO | Perf report for the last 5.13995 seconds: | 20260508 08:19:17.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:17.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:19:17.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:19:17.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:17.531 | INFO | Uploaded 36GiB to GPU in 2.37344s (15.1679GiB/s) | 20260508 08:19:17.531 | INFO | Downloaded 34GiB to GPU in 2.43759s (13.9482GiB/s) | 20260508 08:19:22.673 | INFO | Analyzing chunk frames = [6618112,6618624) | 20260508 08:19:22.673 | INFO | MemInfo: { | 20260508 08:19:22.673 | INFO | "gpu" : " 17 GB", | 20260508 08:19:22.673 | INFO | "huge" : "", | 20260508 08:19:22.673 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:22.673 | INFO | "malloc" : " 14 GB", | 20260508 08:19:22.673 | INFO | "pinned" : " 29 GB" | 20260508 08:19:22.673 | INFO | } | 20260508 08:19:22.673 | INFO | Perf report for the last 5.14284 seconds: | 20260508 08:19:22.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:22.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.673 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:19:22.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.674 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:19:22.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:22.674 | INFO | Uploaded 36GiB to GPU in 2.41109s (14.931GiB/s) | 20260508 08:19:22.674 | INFO | Downloaded 34GiB to GPU in 2.44248s (13.9203GiB/s) | 20260508 08:19:27.813 | INFO | Analyzing chunk frames = [6618624,6619136) | 20260508 08:19:27.813 | INFO | MemInfo: { | 20260508 08:19:27.813 | INFO | "gpu" : " 17 GB", | 20260508 08:19:27.813 | INFO | "huge" : "", | 20260508 08:19:27.813 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:27.813 | INFO | "malloc" : " 14 GB", | 20260508 08:19:27.813 | INFO | "pinned" : " 29 GB" | 20260508 08:19:27.813 | INFO | } | 20260508 08:19:27.813 | INFO | Perf report for the last 5.14002 seconds: | 20260508 08:19:27.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:27.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.813 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 08:19:27.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.814 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 08:19:27.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:27.814 | INFO | Uploaded 36GiB to GPU in 2.42359s (14.854GiB/s) | 20260508 08:19:27.814 | INFO | Downloaded 34GiB to GPU in 2.44403s (13.9114GiB/s) | 20260508 08:19:27.814 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:19:27.814 | INFO | Total mem usage: 16.7 GiB | 20260508 08:19:27.814 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:19:27.814 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260508 08:19:27.814 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:19:27.814 | INFO | High Water Breakdown: | 20260508 08:19:27.814 | INFO | 60.2% : 8.96 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:19:27.814 | INFO | 39.7% : 5.9 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:19:27.814 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:19:27.814 | INFO | ------------------------------------------ | 20260508 08:19:27.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:19:27.814 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:19:27.814 | INFO | Total mem usage: 26.5 GiB | 20260508 08:19:27.814 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:19:27.814 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:19:27.814 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:19:27.814 | INFO | High Water Breakdown: | 20260508 08:19:27.814 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:19:27.814 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:19:27.814 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:19:27.814 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:19:27.814 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:19:27.814 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:19:27.814 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:19:27.814 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:19:27.814 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:19:27.814 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:19:27.814 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:19:27.814 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:19:27.814 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:19:27.814 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:19:27.814 | INFO | ------------------------------------------ | 20260508 08:19:27.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:19:27.814 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:19:27.814 | INFO | Total mem usage: 27.8 GiB | 20260508 08:19:27.814 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:19:27.814 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:19:27.814 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:19:27.814 | INFO | High Water Breakdown: | 20260508 08:19:27.814 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:19:27.814 | INFO | ------------------------------------------ | 20260508 08:19:27.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:19:27.814 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:19:27.814 | INFO | Total mem usage: 15.7 GiB | 20260508 08:19:27.814 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:19:27.814 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:19:27.814 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:19:27.814 | INFO | High Water Breakdown: | 20260508 08:19:27.814 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:19:27.814 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:19:27.814 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:19:27.814 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:19:27.814 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:19:27.814 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:19:27.814 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:19:27.814 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:19:27.814 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:19:27.814 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:19:27.814 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:19:27.814 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:19:27.814 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:19:27.814 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:19:27.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:19:27.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:19:27.814 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:19:27.814 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:19:27.814 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:19:27.814 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:19:27.814 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:19:27.814 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:19:27.814 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:19:27.814 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:19:27.814 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:19:27.814 | INFO | ------------------------------------------ | 20260508 08:19:27.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:19:27.814 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:19:27.814 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:19:27.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:19:27.814 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.40e+01 / 7.97e-01 | 20260508 08:19:27.814 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.16e-02 / 4.94e-03 | 20260508 08:19:27.814 | INFO | PerfReport: 12.11%, Baseline -- 3.98e+00 / 4.49e+00, 5.91e-01 / 2.20e-01 | 20260508 08:19:27.814 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.44e+00, 2.08e-01 / 1.33e-01 | 20260508 08:19:27.814 | INFO | PerfReport: 15.92%, DME -- 4.18e+01 / 4.52e+01, 3.66e+00 / 9.77e-01 | 20260508 08:19:27.814 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.88e+01 / 2.01e+01, 1.92e+00 / 1.26e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.34e+00 / 2.56e+00, 4.05e-01 / 2.29e-01 | 20260508 08:19:27.814 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.19e+00, 2.51e-01 / 1.18e-01 | 20260508 08:19:27.814 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.07e-01 / 3.39e-01, 4.93e-02 / 2.12e-02 | 20260508 08:19:27.814 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:19:27.814 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:19:27.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:19:27.814 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 24.41%, Upload -- 2.58e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 24.45%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 31.05%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 2.34%, LockUpload -- 2.47e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 2.39%, LockDownload -- 2.53e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 08:19:27.814 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260508 08:19:27.910 | INFO | Starting a round of Baz IO | 20260508 08:19:31.321 | INFO | Populating RTMetrics report | 20260508 08:19:31.321 | INFO | RTMetrics report populated | 20260508 08:19:31.321 | INFO | Opening tmp RTMetrics json file | 20260508 08:19:31.329 | INFO | Serializing RTMetrics to json | 20260508 08:19:31.330 | INFO | Serialization done, swapping tmp file now... | 20260508 08:19:31.331 | INFO | RTMetrics done | 20260508 08:19:31.391 | INFO | Finished a round of Baz IO: Wrote 1.506 GiB in 3.48 seconds | 20260508 08:19:31.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.56% | 20260508 08:19:31.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.300972% / 0% / 8.34465e-05% | 20260508 08:19:31.391 | INFO | Baz IO overall: | 20260508 08:19:31.391 | INFO | BytesWritten : 2906626418169 | 20260508 08:19:31.391 | INFO | NumEvents : 1399469602538 | 20260508 08:19:32.943 | INFO | Analyzing chunk frames = [6619136,6619648) | 20260508 08:19:32.943 | INFO | MemInfo: { | 20260508 08:19:32.943 | INFO | "gpu" : " 17 GB", | 20260508 08:19:32.943 | INFO | "huge" : "", | 20260508 08:19:32.943 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:32.943 | INFO | "malloc" : " 14 GB", | 20260508 08:19:32.943 | INFO | "pinned" : " 29 GB" | 20260508 08:19:32.943 | INFO | } | 20260508 08:19:32.943 | INFO | Perf report for the last 5.12924 seconds: | 20260508 08:19:32.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:32.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:32.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:32.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:32.943 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 08:19:32.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:32.943 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:19:32.943 | INFO | PRE_HQ : 65.7%, 1.6, 55.1ms, 0 | 20260508 08:19:32.943 | INFO | BAZWRITER : 67.2%, 2.6, 89.6ms, 0 | 20260508 08:19:32.943 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:19:32.943 | INFO | Uploaded 36GiB to GPU in 2.57258s (13.9938GiB/s) | 20260508 08:19:32.943 | INFO | Downloaded 38GiB to GPU in 2.70741s (14.0355GiB/s) | 20260508 08:19:38.094 | INFO | Analyzing chunk frames = [6619648,6620160) | 20260508 08:19:38.094 | INFO | MemInfo: { | 20260508 08:19:38.094 | INFO | "gpu" : " 17 GB", | 20260508 08:19:38.094 | INFO | "huge" : "", | 20260508 08:19:38.094 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:38.094 | INFO | "malloc" : " 14 GB", | 20260508 08:19:38.094 | INFO | "pinned" : " 29 GB" | 20260508 08:19:38.094 | INFO | } | 20260508 08:19:38.094 | INFO | Perf report for the last 5.15181 seconds: | 20260508 08:19:38.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:38.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 08:19:38.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:19:38.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:38.095 | INFO | Uploaded 36GiB to GPU in 2.49726s (14.4158GiB/s) | 20260508 08:19:38.095 | INFO | Downloaded 34GiB to GPU in 2.42863s (13.9997GiB/s) | 20260508 08:19:43.231 | INFO | Analyzing chunk frames = [6620160,6620672) | 20260508 08:19:43.231 | INFO | MemInfo: { | 20260508 08:19:43.231 | INFO | "gpu" : " 17 GB", | 20260508 08:19:43.231 | INFO | "huge" : "", | 20260508 08:19:43.231 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:43.231 | INFO | "malloc" : " 14 GB", | 20260508 08:19:43.231 | INFO | "pinned" : " 29 GB" | 20260508 08:19:43.231 | INFO | } | 20260508 08:19:43.231 | INFO | Perf report for the last 5.1364 seconds: | 20260508 08:19:43.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:43.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 08:19:43.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:19:43.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:43.231 | INFO | Uploaded 36GiB to GPU in 2.45439s (14.6676GiB/s) | 20260508 08:19:43.231 | INFO | Downloaded 34GiB to GPU in 2.43021s (13.9905GiB/s) | 20260508 08:19:48.366 | INFO | Analyzing chunk frames = [6620672,6621184) | 20260508 08:19:48.366 | INFO | MemInfo: { | 20260508 08:19:48.366 | INFO | "gpu" : " 17 GB", | 20260508 08:19:48.366 | INFO | "huge" : "", | 20260508 08:19:48.366 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:48.366 | INFO | "malloc" : " 14 GB", | 20260508 08:19:48.366 | INFO | "pinned" : " 29 GB" | 20260508 08:19:48.366 | INFO | } | 20260508 08:19:48.366 | INFO | Perf report for the last 5.13507 seconds: | 20260508 08:19:48.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:48.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:19:48.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:19:48.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:48.366 | INFO | Uploaded 36GiB to GPU in 2.50555s (14.3681GiB/s) | 20260508 08:19:48.366 | INFO | Downloaded 34GiB to GPU in 2.43875s (13.9415GiB/s) | 20260508 08:19:53.500 | INFO | Analyzing chunk frames = [6621184,6621696) | 20260508 08:19:53.500 | INFO | MemInfo: { | 20260508 08:19:53.500 | INFO | "gpu" : " 17 GB", | 20260508 08:19:53.500 | INFO | "huge" : "", | 20260508 08:19:53.500 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:53.501 | INFO | "malloc" : " 14 GB", | 20260508 08:19:53.501 | INFO | "pinned" : " 29 GB" | 20260508 08:19:53.501 | INFO | } | 20260508 08:19:53.501 | INFO | Perf report for the last 5.13457 seconds: | 20260508 08:19:53.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:53.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:19:53.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:19:53.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:53.501 | INFO | Uploaded 36GiB to GPU in 2.42434s (14.8494GiB/s) | 20260508 08:19:53.501 | INFO | Downloaded 34GiB to GPU in 2.5144s (13.5221GiB/s) | 20260508 08:19:58.643 | INFO | Analyzing chunk frames = [6621696,6622208) | 20260508 08:19:58.643 | INFO | MemInfo: { | 20260508 08:19:58.643 | INFO | "gpu" : " 17 GB", | 20260508 08:19:58.643 | INFO | "huge" : "", | 20260508 08:19:58.643 | INFO | "huge_shared" : " 29 GB", | 20260508 08:19:58.643 | INFO | "malloc" : " 14 GB", | 20260508 08:19:58.643 | INFO | "pinned" : " 29 GB" | 20260508 08:19:58.643 | INFO | } | 20260508 08:19:58.643 | INFO | Perf report for the last 5.14279 seconds: | 20260508 08:19:58.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:19:58.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.643 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 08:19:58.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.643 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:19:58.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:19:58.644 | INFO | Uploaded 36GiB to GPU in 2.47187s (14.5639GiB/s) | 20260508 08:19:58.644 | INFO | Downloaded 34GiB to GPU in 2.42784s (14.0042GiB/s) | 20260508 08:20:03.784 | INFO | Analyzing chunk frames = [6622208,6622720) | 20260508 08:20:03.784 | INFO | MemInfo: { | 20260508 08:20:03.784 | INFO | "gpu" : " 17 GB", | 20260508 08:20:03.784 | INFO | "huge" : "", | 20260508 08:20:03.784 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:03.784 | INFO | "malloc" : " 14 GB", | 20260508 08:20:03.784 | INFO | "pinned" : " 29 GB" | 20260508 08:20:03.784 | INFO | } | 20260508 08:20:03.784 | INFO | Perf report for the last 5.14084 seconds: | 20260508 08:20:03.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:03.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.784 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 08:20:03.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.785 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:20:03.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:03.785 | INFO | Uploaded 36GiB to GPU in 2.46089s (14.6288GiB/s) | 20260508 08:20:03.785 | INFO | Downloaded 34GiB to GPU in 2.4336s (13.9711GiB/s) | 20260508 08:20:08.926 | INFO | Analyzing chunk frames = [6622720,6623232) | 20260508 08:20:08.926 | INFO | MemInfo: { | 20260508 08:20:08.926 | INFO | "gpu" : " 17 GB", | 20260508 08:20:08.926 | INFO | "huge" : "", | 20260508 08:20:08.926 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:08.926 | INFO | "malloc" : " 14 GB", | 20260508 08:20:08.926 | INFO | "pinned" : " 29 GB" | 20260508 08:20:08.926 | INFO | } | 20260508 08:20:08.926 | INFO | Perf report for the last 5.14192 seconds: | 20260508 08:20:08.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:08.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.926 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 08:20:08.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.926 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:20:08.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:08.927 | INFO | Uploaded 36GiB to GPU in 2.50403s (14.3768GiB/s) | 20260508 08:20:08.927 | INFO | Downloaded 34GiB to GPU in 2.44063s (13.9308GiB/s) | 20260508 08:20:09.023 | INFO | Starting a round of Baz IO | 20260508 08:20:12.436 | INFO | Populating RTMetrics report | 20260508 08:20:12.436 | INFO | RTMetrics report populated | 20260508 08:20:12.436 | INFO | Opening tmp RTMetrics json file | 20260508 08:20:12.447 | INFO | Serializing RTMetrics to json | 20260508 08:20:12.448 | INFO | Serialization done, swapping tmp file now... | 20260508 08:20:12.449 | INFO | RTMetrics done | 20260508 08:20:12.509 | INFO | Finished a round of Baz IO: Wrote 1.508 GiB in 3.485 seconds | 20260508 08:20:12.509 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.00% / 59.56% | 20260508 08:20:12.509 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.300586% / 0% / 5.96046e-05% | 20260508 08:20:12.509 | INFO | Baz IO overall: | 20260508 08:20:12.509 | INFO | BytesWritten : 2908245759156 | 20260508 08:20:12.509 | INFO | NumEvents : 1400237521448 | 20260508 08:20:14.061 | INFO | Analyzing chunk frames = [6623232,6623744) | 20260508 08:20:14.061 | INFO | MemInfo: { | 20260508 08:20:14.061 | INFO | "gpu" : " 17 GB", | 20260508 08:20:14.061 | INFO | "huge" : "", | 20260508 08:20:14.061 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:14.061 | INFO | "malloc" : " 14 GB", | 20260508 08:20:14.061 | INFO | "pinned" : " 29 GB" | 20260508 08:20:14.061 | INFO | } | 20260508 08:20:14.061 | INFO | Perf report for the last 5.13481 seconds: | 20260508 08:20:14.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:14.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:14.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:14.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:14.061 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 08:20:14.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:14.061 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:20:14.061 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:20:14.061 | INFO | BAZWRITER : 67.3%, 2.6, 89.2ms, 0 | 20260508 08:20:14.061 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:20:14.062 | INFO | Uploaded 36GiB to GPU in 2.58322s (13.9361GiB/s) | 20260508 08:20:14.062 | INFO | Downloaded 38GiB to GPU in 2.70845s (14.0302GiB/s) | 20260508 08:20:19.197 | INFO | Analyzing chunk frames = [6623744,6624256) | 20260508 08:20:19.197 | INFO | MemInfo: { | 20260508 08:20:19.197 | INFO | "gpu" : " 17 GB", | 20260508 08:20:19.197 | INFO | "huge" : "", | 20260508 08:20:19.197 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:19.197 | INFO | "malloc" : " 14 GB", | 20260508 08:20:19.197 | INFO | "pinned" : " 29 GB" | 20260508 08:20:19.197 | INFO | } | 20260508 08:20:19.197 | INFO | Perf report for the last 5.13605 seconds: | 20260508 08:20:19.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:19.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.197 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:20:19.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.197 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:20:19.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:19.198 | INFO | Uploaded 36GiB to GPU in 2.49912s (14.4051GiB/s) | 20260508 08:20:19.198 | INFO | Downloaded 34GiB to GPU in 2.43913s (13.9394GiB/s) | 20260508 08:20:24.341 | INFO | Analyzing chunk frames = [6624256,6624768) | 20260508 08:20:24.341 | INFO | MemInfo: { | 20260508 08:20:24.341 | INFO | "gpu" : " 17 GB", | 20260508 08:20:24.341 | INFO | "huge" : "", | 20260508 08:20:24.341 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:24.341 | INFO | "malloc" : " 14 GB", | 20260508 08:20:24.341 | INFO | "pinned" : " 29 GB" | 20260508 08:20:24.341 | INFO | } | 20260508 08:20:24.341 | INFO | Perf report for the last 5.14409 seconds: | 20260508 08:20:24.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:24.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.341 | INFO | ANALYSIS : 67.2%, 3.0, 104.6ms, 0 | 20260508 08:20:24.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.341 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:20:24.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:24.342 | INFO | Uploaded 36GiB to GPU in 2.58636s (13.9192GiB/s) | 20260508 08:20:24.342 | INFO | Downloaded 34GiB to GPU in 2.4265s (14.0119GiB/s) | 20260508 08:20:29.482 | INFO | Analyzing chunk frames = [6624768,6625280) | 20260508 08:20:29.482 | INFO | MemInfo: { | 20260508 08:20:29.482 | INFO | "gpu" : " 17 GB", | 20260508 08:20:29.482 | INFO | "huge" : "", | 20260508 08:20:29.482 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:29.482 | INFO | "malloc" : " 14 GB", | 20260508 08:20:29.482 | INFO | "pinned" : " 29 GB" | 20260508 08:20:29.482 | INFO | } | 20260508 08:20:29.482 | INFO | Perf report for the last 5.14088 seconds: | 20260508 08:20:29.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:29.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 08:20:29.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:20:29.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:29.482 | INFO | Uploaded 36GiB to GPU in 2.58858s (13.9072GiB/s) | 20260508 08:20:29.483 | INFO | Downloaded 34GiB to GPU in 2.42372s (14.028GiB/s) | 20260508 08:20:34.625 | INFO | Analyzing chunk frames = [6625280,6625792) | 20260508 08:20:34.625 | INFO | MemInfo: { | 20260508 08:20:34.625 | INFO | "gpu" : " 17 GB", | 20260508 08:20:34.625 | INFO | "huge" : "", | 20260508 08:20:34.625 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:34.625 | INFO | "malloc" : " 14 GB", | 20260508 08:20:34.625 | INFO | "pinned" : " 29 GB" | 20260508 08:20:34.625 | INFO | } | 20260508 08:20:34.625 | INFO | Perf report for the last 5.14343 seconds: | 20260508 08:20:34.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:34.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 08:20:34.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | PRE_HQ : 31.9%, 1.0, 16.7ms, 0 | 20260508 08:20:34.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:34.626 | INFO | Uploaded 36GiB to GPU in 2.3974s (15.0163GiB/s) | 20260508 08:20:34.626 | INFO | Downloaded 34GiB to GPU in 2.52012s (13.4914GiB/s) | 20260508 08:20:39.761 | INFO | Analyzing chunk frames = [6625792,6626304) | 20260508 08:20:39.761 | INFO | MemInfo: { | 20260508 08:20:39.761 | INFO | "gpu" : " 17 GB", | 20260508 08:20:39.761 | INFO | "huge" : "", | 20260508 08:20:39.761 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:39.761 | INFO | "malloc" : " 14 GB", | 20260508 08:20:39.761 | INFO | "pinned" : " 29 GB" | 20260508 08:20:39.761 | INFO | } | 20260508 08:20:39.761 | INFO | Perf report for the last 5.13519 seconds: | 20260508 08:20:39.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:39.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 08:20:39.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:20:39.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:39.761 | INFO | Uploaded 36GiB to GPU in 2.35674s (15.2753GiB/s) | 20260508 08:20:39.761 | INFO | Downloaded 34GiB to GPU in 2.43226s (13.9787GiB/s) | 20260508 08:20:44.904 | INFO | Analyzing chunk frames = [6626304,6626816) | 20260508 08:20:44.904 | INFO | MemInfo: { | 20260508 08:20:44.904 | INFO | "gpu" : " 17 GB", | 20260508 08:20:44.904 | INFO | "huge" : "", | 20260508 08:20:44.904 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:44.904 | INFO | "malloc" : " 14 GB", | 20260508 08:20:44.904 | INFO | "pinned" : " 29 GB" | 20260508 08:20:44.904 | INFO | } | 20260508 08:20:44.904 | INFO | Perf report for the last 5.14322 seconds: | 20260508 08:20:44.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:44.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 08:20:44.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:20:44.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:44.904 | INFO | Uploaded 36GiB to GPU in 2.41444s (14.9103GiB/s) | 20260508 08:20:44.904 | INFO | Downloaded 34GiB to GPU in 2.4325s (13.9774GiB/s) | 20260508 08:20:50.038 | INFO | Analyzing chunk frames = [6626816,6627328) | 20260508 08:20:50.038 | INFO | MemInfo: { | 20260508 08:20:50.038 | INFO | "gpu" : " 17 GB", | 20260508 08:20:50.038 | INFO | "huge" : "", | 20260508 08:20:50.038 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:50.038 | INFO | "malloc" : " 14 GB", | 20260508 08:20:50.038 | INFO | "pinned" : " 29 GB" | 20260508 08:20:50.038 | INFO | } | 20260508 08:20:50.038 | INFO | Perf report for the last 5.13421 seconds: | 20260508 08:20:50.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:50.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 08:20:50.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:20:50.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:50.038 | INFO | Uploaded 36GiB to GPU in 2.43389s (14.7912GiB/s) | 20260508 08:20:50.039 | INFO | Downloaded 34GiB to GPU in 2.45238s (13.8641GiB/s) | 20260508 08:20:50.039 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:20:50.039 | INFO | Total mem usage: 16.7 GiB | 20260508 08:20:50.039 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:20:50.039 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:20:50.039 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:20:50.039 | INFO | High Water Breakdown: | 20260508 08:20:50.039 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:20:50.039 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:20:50.039 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:20:50.039 | INFO | ------------------------------------------ | 20260508 08:20:50.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:20:50.039 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:20:50.039 | INFO | Total mem usage: 26.5 GiB | 20260508 08:20:50.039 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:20:50.039 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:20:50.039 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:20:50.039 | INFO | High Water Breakdown: | 20260508 08:20:50.039 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:20:50.039 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:20:50.039 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:20:50.039 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:20:50.039 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:20:50.039 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:20:50.039 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:20:50.039 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:20:50.039 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:20:50.039 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:20:50.039 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:20:50.039 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:20:50.039 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:20:50.039 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:20:50.039 | INFO | ------------------------------------------ | 20260508 08:20:50.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:20:50.039 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:20:50.039 | INFO | Total mem usage: 27.8 GiB | 20260508 08:20:50.039 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:20:50.039 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:20:50.039 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:20:50.039 | INFO | High Water Breakdown: | 20260508 08:20:50.039 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:20:50.039 | INFO | ------------------------------------------ | 20260508 08:20:50.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:20:50.039 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:20:50.039 | INFO | Total mem usage: 15.7 GiB | 20260508 08:20:50.039 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:20:50.039 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:20:50.039 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:20:50.039 | INFO | High Water Breakdown: | 20260508 08:20:50.039 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:20:50.039 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:20:50.039 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:20:50.039 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:20:50.039 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:20:50.039 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:20:50.039 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:20:50.039 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:20:50.039 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:20:50.039 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:20:50.039 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:20:50.039 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:20:50.039 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:20:50.039 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:20:50.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:20:50.039 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:20:50.039 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:20:50.039 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:20:50.039 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:20:50.039 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:20:50.039 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:20:50.039 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:20:50.039 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:20:50.039 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:20:50.039 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:20:50.039 | INFO | ------------------------------------------ | 20260508 08:20:50.039 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:20:50.039 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:20:50.039 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:20:50.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:20:50.039 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.95e-01 | 20260508 08:20:50.039 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.13e-02 / 7.05e-03 | 20260508 08:20:50.039 | INFO | PerfReport: 12.14%, Baseline -- 3.96e+00 / 4.34e+00, 5.87e-01 / 2.59e-01 | 20260508 08:20:50.039 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.41e+00, 2.07e-01 / 1.35e-01 | 20260508 08:20:50.039 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.49e+01, 3.69e+00 / 1.27e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.86e+01 / 2.05e+01, 1.79e+00 / 1.29e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.35e+00 / 2.46e+00, 4.10e-01 / 2.62e-01 | 20260508 08:20:50.039 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.18e+00, 2.49e-01 / 1.64e-01 | 20260508 08:20:50.039 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.06e-01 / 3.38e-01, 4.86e-02 / 2.86e-02 | 20260508 08:20:50.039 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:20:50.039 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:20:50.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:20:50.039 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 24.48%, Upload -- 2.59e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 24.42%, Download -- 2.58e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 30.87%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 2.31%, LockUpload -- 2.45e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 2.51%, LockDownload -- 2.65e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 15.40%, LockCompute -- 1.63e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 08:20:50.039 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 08:20:50.135 | INFO | Starting a round of Baz IO | 20260508 08:20:53.539 | INFO | Populating RTMetrics report | 20260508 08:20:53.539 | INFO | RTMetrics report populated | 20260508 08:20:53.539 | INFO | Opening tmp RTMetrics json file | 20260508 08:20:53.549 | INFO | Serializing RTMetrics to json | 20260508 08:20:53.550 | INFO | Serialization done, swapping tmp file now... | 20260508 08:20:53.551 | INFO | RTMetrics done | 20260508 08:20:53.612 | INFO | Finished a round of Baz IO: Wrote 1.506 GiB in 3.477 seconds | 20260508 08:20:53.612 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.56% | 20260508 08:20:53.612 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.299684% / 3.97364e-06% / 7.15256e-05% | 20260508 08:20:53.612 | INFO | Baz IO overall: | 20260508 08:20:53.612 | INFO | BytesWritten : 2909862475554 | 20260508 08:20:53.612 | INFO | NumEvents : 1401003455936 | 20260508 08:20:55.179 | INFO | Analyzing chunk frames = [6627328,6627840) | 20260508 08:20:55.179 | INFO | MemInfo: { | 20260508 08:20:55.179 | INFO | "gpu" : " 17 GB", | 20260508 08:20:55.179 | INFO | "huge" : "", | 20260508 08:20:55.179 | INFO | "huge_shared" : " 29 GB", | 20260508 08:20:55.179 | INFO | "malloc" : " 14 GB", | 20260508 08:20:55.179 | INFO | "pinned" : " 29 GB" | 20260508 08:20:55.179 | INFO | } | 20260508 08:20:55.179 | INFO | Perf report for the last 5.14081 seconds: | 20260508 08:20:55.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:20:55.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:55.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:55.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:55.179 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 08:20:55.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:20:55.179 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:20:55.179 | INFO | PRE_HQ : 65.3%, 1.6, 54.8ms, 0 | 20260508 08:20:55.179 | INFO | BAZWRITER : 67.0%, 2.6, 90.5ms, 0 | 20260508 08:20:55.179 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:20:55.180 | INFO | Uploaded 36GiB to GPU in 2.56473s (14.0366GiB/s) | 20260508 08:20:55.180 | INFO | Downloaded 38GiB to GPU in 2.69715s (14.089GiB/s) | 20260508 08:21:00.317 | INFO | Analyzing chunk frames = [6627840,6628352) | 20260508 08:21:00.317 | INFO | MemInfo: { | 20260508 08:21:00.317 | INFO | "gpu" : " 17 GB", | 20260508 08:21:00.317 | INFO | "huge" : "", | 20260508 08:21:00.317 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:00.317 | INFO | "malloc" : " 14 GB", | 20260508 08:21:00.317 | INFO | "pinned" : " 29 GB" | 20260508 08:21:00.317 | INFO | } | 20260508 08:21:00.317 | INFO | Perf report for the last 5.13786 seconds: | 20260508 08:21:00.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:00.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.317 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 08:21:00.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.317 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:21:00.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:00.318 | INFO | Uploaded 36GiB to GPU in 2.44464s (14.7261GiB/s) | 20260508 08:21:00.318 | INFO | Downloaded 34GiB to GPU in 2.46199s (13.81GiB/s) | 20260508 08:21:05.456 | INFO | Analyzing chunk frames = [6628352,6628864) | 20260508 08:21:05.456 | INFO | MemInfo: { | 20260508 08:21:05.456 | INFO | "gpu" : " 17 GB", | 20260508 08:21:05.456 | INFO | "huge" : "", | 20260508 08:21:05.456 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:05.456 | INFO | "malloc" : " 14 GB", | 20260508 08:21:05.457 | INFO | "pinned" : " 29 GB" | 20260508 08:21:05.457 | INFO | } | 20260508 08:21:05.457 | INFO | Perf report for the last 5.1398 seconds: | 20260508 08:21:05.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:05.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 08:21:05.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:21:05.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:05.457 | INFO | Uploaded 36GiB to GPU in 2.45952s (14.637GiB/s) | 20260508 08:21:05.457 | INFO | Downloaded 34GiB to GPU in 2.45971s (13.8228GiB/s) | 20260508 08:21:10.602 | INFO | Analyzing chunk frames = [6628864,6629376) | 20260508 08:21:10.602 | INFO | MemInfo: { | 20260508 08:21:10.602 | INFO | "gpu" : " 17 GB", | 20260508 08:21:10.602 | INFO | "huge" : "", | 20260508 08:21:10.603 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:10.603 | INFO | "malloc" : " 14 GB", | 20260508 08:21:10.603 | INFO | "pinned" : " 29 GB" | 20260508 08:21:10.603 | INFO | } | 20260508 08:21:10.603 | INFO | Perf report for the last 5.14611 seconds: | 20260508 08:21:10.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:10.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 08:21:10.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:21:10.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:10.603 | INFO | Uploaded 36GiB to GPU in 2.48215s (14.5036GiB/s) | 20260508 08:21:10.603 | INFO | Downloaded 34GiB to GPU in 2.44543s (13.9035GiB/s) | 20260508 08:21:15.738 | INFO | Analyzing chunk frames = [6629376,6629888) | 20260508 08:21:15.738 | INFO | MemInfo: { | 20260508 08:21:15.738 | INFO | "gpu" : " 17 GB", | 20260508 08:21:15.738 | INFO | "huge" : "", | 20260508 08:21:15.738 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:15.738 | INFO | "malloc" : " 14 GB", | 20260508 08:21:15.738 | INFO | "pinned" : " 29 GB" | 20260508 08:21:15.739 | INFO | } | 20260508 08:21:15.739 | INFO | Perf report for the last 5.13584 seconds: | 20260508 08:21:15.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:15.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 08:21:15.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:21:15.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:15.739 | INFO | Uploaded 36GiB to GPU in 2.35049s (15.316GiB/s) | 20260508 08:21:15.739 | INFO | Downloaded 34GiB to GPU in 2.51319s (13.5286GiB/s) | 20260508 08:21:20.872 | INFO | Analyzing chunk frames = [6629888,6630400) | 20260508 08:21:20.872 | INFO | MemInfo: { | 20260508 08:21:20.872 | INFO | "gpu" : " 17 GB", | 20260508 08:21:20.872 | INFO | "huge" : "", | 20260508 08:21:20.872 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:20.872 | INFO | "malloc" : " 14 GB", | 20260508 08:21:20.872 | INFO | "pinned" : " 29 GB" | 20260508 08:21:20.872 | INFO | } | 20260508 08:21:20.872 | INFO | Perf report for the last 5.13358 seconds: | 20260508 08:21:20.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:20.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 08:21:20.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:21:20.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:20.872 | INFO | Uploaded 36GiB to GPU in 2.3584s (15.2646GiB/s) | 20260508 08:21:20.873 | INFO | Downloaded 34GiB to GPU in 2.44058s (13.9311GiB/s) | 20260508 08:21:26.010 | INFO | Analyzing chunk frames = [6630400,6630912) | 20260508 08:21:26.010 | INFO | MemInfo: { | 20260508 08:21:26.010 | INFO | "gpu" : " 17 GB", | 20260508 08:21:26.010 | INFO | "huge" : "", | 20260508 08:21:26.010 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:26.010 | INFO | "malloc" : " 14 GB", | 20260508 08:21:26.010 | INFO | "pinned" : " 29 GB" | 20260508 08:21:26.010 | INFO | } | 20260508 08:21:26.010 | INFO | Perf report for the last 5.13806 seconds: | 20260508 08:21:26.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:26.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.011 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 08:21:26.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.011 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:21:26.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:26.011 | INFO | Uploaded 36GiB to GPU in 2.46122s (14.6269GiB/s) | 20260508 08:21:26.011 | INFO | Downloaded 34GiB to GPU in 2.44408s (13.9112GiB/s) | 20260508 08:21:31.151 | INFO | Analyzing chunk frames = [6630912,6631424) | 20260508 08:21:31.151 | INFO | MemInfo: { | 20260508 08:21:31.151 | INFO | "gpu" : " 17 GB", | 20260508 08:21:31.151 | INFO | "huge" : "", | 20260508 08:21:31.151 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:31.151 | INFO | "malloc" : " 14 GB", | 20260508 08:21:31.151 | INFO | "pinned" : " 29 GB" | 20260508 08:21:31.151 | INFO | } | 20260508 08:21:31.151 | INFO | Perf report for the last 5.14064 seconds: | 20260508 08:21:31.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:31.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.151 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:21:31.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.151 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:21:31.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:31.152 | INFO | Uploaded 36GiB to GPU in 2.47353s (14.5541GiB/s) | 20260508 08:21:31.152 | INFO | Downloaded 34GiB to GPU in 2.46163s (13.812GiB/s) | 20260508 08:21:31.249 | INFO | Starting a round of Baz IO | 20260508 08:21:34.672 | INFO | Populating RTMetrics report | 20260508 08:21:34.672 | INFO | RTMetrics report populated | 20260508 08:21:34.672 | INFO | Opening tmp RTMetrics json file | 20260508 08:21:34.681 | INFO | Serializing RTMetrics to json | 20260508 08:21:34.683 | INFO | Serialization done, swapping tmp file now... | 20260508 08:21:34.683 | INFO | RTMetrics done | 20260508 08:21:34.742 | INFO | Finished a round of Baz IO: Wrote 1.504 GiB in 3.493 seconds | 20260508 08:21:34.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.57% | 20260508 08:21:34.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.29846% / 0% / 5.5631e-05% | 20260508 08:21:34.742 | INFO | Baz IO overall: | 20260508 08:21:34.742 | INFO | BytesWritten : 2911477806689 | 20260508 08:21:34.742 | INFO | NumEvents : 1401768605121 | 20260508 08:21:36.295 | INFO | Analyzing chunk frames = [6631424,6631936) | 20260508 08:21:36.295 | INFO | MemInfo: { | 20260508 08:21:36.295 | INFO | "gpu" : " 17 GB", | 20260508 08:21:36.295 | INFO | "huge" : "", | 20260508 08:21:36.295 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:36.295 | INFO | "malloc" : " 14 GB", | 20260508 08:21:36.295 | INFO | "pinned" : " 29 GB" | 20260508 08:21:36.295 | INFO | } | 20260508 08:21:36.295 | INFO | Perf report for the last 5.14425 seconds: | 20260508 08:21:36.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:36.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:36.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:36.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:36.295 | INFO | ANALYSIS : 67.5%, 3.0, 105.7ms, 0 | 20260508 08:21:36.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:36.295 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:21:36.295 | INFO | PRE_HQ : 65.2%, 1.6, 54.6ms, 0 | 20260508 08:21:36.295 | INFO | BAZWRITER : 67.1%, 2.6, 89.9ms, 0 | 20260508 08:21:36.295 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:21:36.296 | INFO | Uploaded 36GiB to GPU in 2.5983s (13.8552GiB/s) | 20260508 08:21:36.296 | INFO | Downloaded 38GiB to GPU in 2.6932s (14.1096GiB/s) | 20260508 08:21:41.435 | INFO | Analyzing chunk frames = [6631936,6632448) | 20260508 08:21:41.435 | INFO | MemInfo: { | 20260508 08:21:41.435 | INFO | "gpu" : " 17 GB", | 20260508 08:21:41.435 | INFO | "huge" : "", | 20260508 08:21:41.435 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:41.435 | INFO | "malloc" : " 14 GB", | 20260508 08:21:41.435 | INFO | "pinned" : " 29 GB" | 20260508 08:21:41.435 | INFO | } | 20260508 08:21:41.435 | INFO | Perf report for the last 5.14022 seconds: | 20260508 08:21:41.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:41.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.436 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:21:41.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.436 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:21:41.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:41.436 | INFO | Uploaded 36GiB to GPU in 2.51373s (14.3214GiB/s) | 20260508 08:21:41.436 | INFO | Downloaded 34GiB to GPU in 2.45577s (13.8449GiB/s) | 20260508 08:21:46.570 | INFO | Analyzing chunk frames = [6632448,6632960) | 20260508 08:21:46.570 | INFO | MemInfo: { | 20260508 08:21:46.570 | INFO | "gpu" : " 17 GB", | 20260508 08:21:46.570 | INFO | "huge" : "", | 20260508 08:21:46.570 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:46.570 | INFO | "malloc" : " 14 GB", | 20260508 08:21:46.570 | INFO | "pinned" : " 29 GB" | 20260508 08:21:46.570 | INFO | } | 20260508 08:21:46.570 | INFO | Perf report for the last 5.13444 seconds: | 20260508 08:21:46.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:46.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.570 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:21:46.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.570 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:21:46.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:46.571 | INFO | Uploaded 36GiB to GPU in 2.47364s (14.5534GiB/s) | 20260508 08:21:46.571 | INFO | Downloaded 34GiB to GPU in 2.4367s (13.9533GiB/s) | 20260508 08:21:51.711 | INFO | Analyzing chunk frames = [6632960,6633472) | 20260508 08:21:51.711 | INFO | MemInfo: { | 20260508 08:21:51.711 | INFO | "gpu" : " 17 GB", | 20260508 08:21:51.711 | INFO | "huge" : "", | 20260508 08:21:51.711 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:51.711 | INFO | "malloc" : " 14 GB", | 20260508 08:21:51.711 | INFO | "pinned" : " 29 GB" | 20260508 08:21:51.711 | INFO | } | 20260508 08:21:51.711 | INFO | Perf report for the last 5.1409 seconds: | 20260508 08:21:51.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:51.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 08:21:51.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:21:51.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:51.711 | INFO | Uploaded 36GiB to GPU in 2.56627s (14.0281GiB/s) | 20260508 08:21:51.712 | INFO | Downloaded 34GiB to GPU in 2.43461s (13.9653GiB/s) | 20260508 08:21:56.850 | INFO | Analyzing chunk frames = [6633472,6633984) | 20260508 08:21:56.850 | INFO | MemInfo: { | 20260508 08:21:56.850 | INFO | "gpu" : " 17 GB", | 20260508 08:21:56.850 | INFO | "huge" : "", | 20260508 08:21:56.850 | INFO | "huge_shared" : " 29 GB", | 20260508 08:21:56.850 | INFO | "malloc" : " 14 GB", | 20260508 08:21:56.850 | INFO | "pinned" : " 29 GB" | 20260508 08:21:56.850 | INFO | } | 20260508 08:21:56.851 | INFO | Perf report for the last 5.13976 seconds: | 20260508 08:21:56.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:21:56.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 08:21:56.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:21:56.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:21:56.851 | INFO | Uploaded 36GiB to GPU in 2.41908s (14.8817GiB/s) | 20260508 08:21:56.851 | INFO | Downloaded 34GiB to GPU in 2.54025s (13.3845GiB/s) | 20260508 08:22:01.992 | INFO | Analyzing chunk frames = [6633984,6634496) | 20260508 08:22:01.992 | INFO | MemInfo: { | 20260508 08:22:01.992 | INFO | "gpu" : " 17 GB", | 20260508 08:22:01.992 | INFO | "huge" : "", | 20260508 08:22:01.992 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:01.992 | INFO | "malloc" : " 14 GB", | 20260508 08:22:01.992 | INFO | "pinned" : " 29 GB" | 20260508 08:22:01.992 | INFO | } | 20260508 08:22:01.992 | INFO | Perf report for the last 5.14164 seconds: | 20260508 08:22:01.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:01.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.992 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 08:22:01.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.992 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:22:01.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:01.993 | INFO | Uploaded 36GiB to GPU in 2.35358s (15.2959GiB/s) | 20260508 08:22:01.993 | INFO | Downloaded 34GiB to GPU in 2.44796s (13.8891GiB/s) | 20260508 08:22:07.136 | INFO | Analyzing chunk frames = [6634496,6635008) | 20260508 08:22:07.136 | INFO | MemInfo: { | 20260508 08:22:07.136 | INFO | "gpu" : " 17 GB", | 20260508 08:22:07.136 | INFO | "huge" : "", | 20260508 08:22:07.136 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:07.136 | INFO | "malloc" : " 14 GB", | 20260508 08:22:07.136 | INFO | "pinned" : " 29 GB" | 20260508 08:22:07.136 | INFO | } | 20260508 08:22:07.136 | INFO | Perf report for the last 5.14394 seconds: | 20260508 08:22:07.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:07.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.136 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:22:07.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.136 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:22:07.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:07.137 | INFO | Uploaded 36GiB to GPU in 2.44369s (14.7318GiB/s) | 20260508 08:22:07.137 | INFO | Downloaded 34GiB to GPU in 2.45091s (13.8724GiB/s) | 20260508 08:22:12.270 | INFO | Analyzing chunk frames = [6635008,6635520) | 20260508 08:22:12.270 | INFO | MemInfo: { | 20260508 08:22:12.270 | INFO | "gpu" : " 17 GB", | 20260508 08:22:12.270 | INFO | "huge" : "", | 20260508 08:22:12.270 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:12.270 | INFO | "malloc" : " 14 GB", | 20260508 08:22:12.270 | INFO | "pinned" : " 29 GB" | 20260508 08:22:12.270 | INFO | } | 20260508 08:22:12.270 | INFO | Perf report for the last 5.13405 seconds: | 20260508 08:22:12.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:12.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.270 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 08:22:12.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.271 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:22:12.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:12.271 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:22:12.271 | INFO | Total mem usage: 16.7 GiB | 20260508 08:22:12.271 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:22:12.271 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:22:12.271 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:22:12.271 | INFO | High Water Breakdown: | 20260508 08:22:12.271 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:22:12.271 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:22:12.271 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:22:12.271 | INFO | ------------------------------------------ | 20260508 08:22:12.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:22:12.271 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:22:12.271 | INFO | Total mem usage: 26.5 GiB | 20260508 08:22:12.271 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:22:12.271 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:22:12.271 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:22:12.271 | INFO | High Water Breakdown: | 20260508 08:22:12.271 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:22:12.271 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:22:12.271 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:22:12.271 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:22:12.271 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:22:12.271 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:22:12.271 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:22:12.271 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:22:12.271 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:22:12.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:22:12.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:22:12.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:22:12.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:22:12.271 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:22:12.271 | INFO | ------------------------------------------ | 20260508 08:22:12.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:22:12.271 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:22:12.271 | INFO | Total mem usage: 27.8 GiB | 20260508 08:22:12.271 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:22:12.271 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:22:12.271 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:22:12.271 | INFO | High Water Breakdown: | 20260508 08:22:12.271 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:22:12.271 | INFO | ------------------------------------------ | 20260508 08:22:12.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:22:12.271 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:22:12.271 | INFO | Total mem usage: 15.7 GiB | 20260508 08:22:12.271 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:22:12.271 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:22:12.271 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:22:12.271 | INFO | High Water Breakdown: | 20260508 08:22:12.271 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:22:12.271 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:22:12.271 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:22:12.271 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:22:12.271 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:22:12.271 | INFO | Uploaded 36GiB to GPU in 2.40937s (14.9417GiB/s) | 20260508 08:22:12.271 | INFO | Downloaded 34GiB to GPU in 2.4586s (13.829GiB/s) | 20260508 08:22:12.271 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:22:12.272 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:22:12.272 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:22:12.272 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:22:12.272 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:22:12.272 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:22:12.272 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:22:12.272 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:22:12.272 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:22:12.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:22:12.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:22:12.272 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:22:12.272 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:22:12.272 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:22:12.272 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:22:12.272 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:22:12.272 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:22:12.272 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:22:12.272 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:22:12.272 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:22:12.272 | INFO | ------------------------------------------ | 20260508 08:22:12.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:22:12.272 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:22:12.272 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:22:12.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:22:12.272 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.83e-01 | 20260508 08:22:12.272 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.22e-02 / 6.30e-03 | 20260508 08:22:12.272 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.46e+00, 5.87e-01 / 2.36e-01 | 20260508 08:22:12.272 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.29e-01 | 20260508 08:22:12.272 | INFO | PerfReport: 15.93%, DME -- 4.17e+01 / 4.50e+01, 3.58e+00 / 1.20e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.85e+00 / 1.43e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.12e-01 / 2.25e-01 | 20260508 08:22:12.272 | INFO | PerfReport: 3.07%, Metrics -- 1.00e+00 / 1.16e+00, 2.52e-01 / 1.46e-01 | 20260508 08:22:12.272 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.36e-01, 4.86e-02 / 2.36e-02 | 20260508 08:22:12.272 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:22:12.272 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:22:12.272 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:22:12.272 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 24.28%, Upload -- 2.56e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 24.57%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 30.98%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 2.06%, LockUpload -- 2.18e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 2.66%, LockDownload -- 2.81e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 15.45%, LockCompute -- 1.63e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 08:22:12.272 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260508 08:22:12.385 | INFO | Starting a round of Baz IO | 20260508 08:22:15.787 | INFO | Populating RTMetrics report | 20260508 08:22:15.787 | INFO | RTMetrics report populated | 20260508 08:22:15.787 | INFO | Opening tmp RTMetrics json file | 20260508 08:22:15.800 | INFO | Serializing RTMetrics to json | 20260508 08:22:15.801 | INFO | Serialization done, swapping tmp file now... | 20260508 08:22:15.802 | INFO | RTMetrics done | 20260508 08:22:15.854 | INFO | Finished a round of Baz IO: Wrote 1.508 GiB in 3.468 seconds | 20260508 08:22:15.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.57% | 20260508 08:22:15.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.299668% / 0% / 5.16574e-05% | 20260508 08:22:15.854 | INFO | Baz IO overall: | 20260508 08:22:15.854 | INFO | BytesWritten : 2913096477982 | 20260508 08:22:15.854 | INFO | NumEvents : 1402536308617 | 20260508 08:22:17.406 | INFO | Analyzing chunk frames = [6635520,6636032) | 20260508 08:22:17.406 | INFO | MemInfo: { | 20260508 08:22:17.406 | INFO | "gpu" : " 17 GB", | 20260508 08:22:17.406 | INFO | "huge" : "", | 20260508 08:22:17.406 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:17.406 | INFO | "malloc" : " 14 GB", | 20260508 08:22:17.406 | INFO | "pinned" : " 29 GB" | 20260508 08:22:17.406 | INFO | } | 20260508 08:22:17.406 | INFO | Perf report for the last 5.13601 seconds: | 20260508 08:22:17.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:17.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:17.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:17.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:17.406 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 08:22:17.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:17.407 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260508 08:22:17.407 | INFO | PRE_HQ : 66.2%, 1.8, 60.2ms, 0 | 20260508 08:22:17.407 | INFO | BAZWRITER : 67.0%, 2.7, 94.0ms, 0 | 20260508 08:22:17.407 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:22:17.407 | INFO | Uploaded 36GiB to GPU in 2.57675s (13.9711GiB/s) | 20260508 08:22:17.407 | INFO | Downloaded 38GiB to GPU in 2.70846s (14.0301GiB/s) | 20260508 08:22:22.547 | INFO | Analyzing chunk frames = [6636032,6636544) | 20260508 08:22:22.547 | INFO | MemInfo: { | 20260508 08:22:22.547 | INFO | "gpu" : " 17 GB", | 20260508 08:22:22.547 | INFO | "huge" : "", | 20260508 08:22:22.547 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:22.547 | INFO | "malloc" : " 14 GB", | 20260508 08:22:22.547 | INFO | "pinned" : " 29 GB" | 20260508 08:22:22.547 | INFO | } | 20260508 08:22:22.547 | INFO | Perf report for the last 5.14099 seconds: | 20260508 08:22:22.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:22.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.547 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:22:22.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.547 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:22:22.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:22.548 | INFO | Uploaded 36GiB to GPU in 2.48689s (14.4759GiB/s) | 20260508 08:22:22.548 | INFO | Downloaded 34GiB to GPU in 2.43224s (13.9789GiB/s) | 20260508 08:22:27.682 | INFO | Analyzing chunk frames = [6636544,6637056) | 20260508 08:22:27.682 | INFO | MemInfo: { | 20260508 08:22:27.682 | INFO | "gpu" : " 17 GB", | 20260508 08:22:27.682 | INFO | "huge" : "", | 20260508 08:22:27.682 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:27.682 | INFO | "malloc" : " 14 GB", | 20260508 08:22:27.682 | INFO | "pinned" : " 29 GB" | 20260508 08:22:27.682 | INFO | } | 20260508 08:22:27.682 | INFO | Perf report for the last 5.13478 seconds: | 20260508 08:22:27.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:27.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.682 | INFO | ANALYSIS : 67.1%, 3.0, 104.1ms, 0 | 20260508 08:22:27.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.682 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:22:27.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:27.683 | INFO | Uploaded 36GiB to GPU in 2.57463s (13.9826GiB/s) | 20260508 08:22:27.683 | INFO | Downloaded 34GiB to GPU in 2.42816s (14.0024GiB/s) | 20260508 08:22:32.823 | INFO | Analyzing chunk frames = [6637056,6637568) | 20260508 08:22:32.823 | INFO | MemInfo: { | 20260508 08:22:32.823 | INFO | "gpu" : " 17 GB", | 20260508 08:22:32.823 | INFO | "huge" : "", | 20260508 08:22:32.823 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:32.823 | INFO | "malloc" : " 14 GB", | 20260508 08:22:32.823 | INFO | "pinned" : " 29 GB" | 20260508 08:22:32.823 | INFO | } | 20260508 08:22:32.823 | INFO | Perf report for the last 5.14084 seconds: | 20260508 08:22:32.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:32.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 08:22:32.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:22:32.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:32.823 | INFO | Uploaded 36GiB to GPU in 2.57687s (13.9704GiB/s) | 20260508 08:22:32.823 | INFO | Downloaded 34GiB to GPU in 2.41643s (14.0704GiB/s) | 20260508 08:22:37.959 | INFO | Analyzing chunk frames = [6637568,6638080) | 20260508 08:22:37.959 | INFO | MemInfo: { | 20260508 08:22:37.959 | INFO | "gpu" : " 17 GB", | 20260508 08:22:37.959 | INFO | "huge" : "", | 20260508 08:22:37.959 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:37.959 | INFO | "malloc" : " 14 GB", | 20260508 08:22:37.959 | INFO | "pinned" : " 29 GB" | 20260508 08:22:37.959 | INFO | } | 20260508 08:22:37.959 | INFO | Perf report for the last 5.13651 seconds: | 20260508 08:22:37.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:37.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 08:22:37.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:22:37.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:37.960 | INFO | Uploaded 36GiB to GPU in 2.31617s (15.5429GiB/s) | 20260508 08:22:37.960 | INFO | Downloaded 34GiB to GPU in 2.48751s (13.6683GiB/s) | 20260508 08:22:43.106 | INFO | Analyzing chunk frames = [6638080,6638592) | 20260508 08:22:43.106 | INFO | MemInfo: { | 20260508 08:22:43.106 | INFO | "gpu" : " 17 GB", | 20260508 08:22:43.106 | INFO | "huge" : "", | 20260508 08:22:43.106 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:43.106 | INFO | "malloc" : " 14 GB", | 20260508 08:22:43.106 | INFO | "pinned" : " 29 GB" | 20260508 08:22:43.106 | INFO | } | 20260508 08:22:43.106 | INFO | Perf report for the last 5.14634 seconds: | 20260508 08:22:43.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:43.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:22:43.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:22:43.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:43.106 | INFO | Uploaded 36GiB to GPU in 2.39536s (15.0291GiB/s) | 20260508 08:22:43.106 | INFO | Downloaded 34GiB to GPU in 2.434s (13.9688GiB/s) | 20260508 08:22:48.247 | INFO | Analyzing chunk frames = [6638592,6639104) | 20260508 08:22:48.247 | INFO | MemInfo: { | 20260508 08:22:48.247 | INFO | "gpu" : " 17 GB", | 20260508 08:22:48.247 | INFO | "huge" : "", | 20260508 08:22:48.247 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:48.247 | INFO | "malloc" : " 14 GB", | 20260508 08:22:48.247 | INFO | "pinned" : " 29 GB" | 20260508 08:22:48.247 | INFO | } | 20260508 08:22:48.247 | INFO | Perf report for the last 5.14111 seconds: | 20260508 08:22:48.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:48.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.247 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:22:48.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.247 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:22:48.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:48.248 | INFO | Uploaded 36GiB to GPU in 2.42068s (14.8719GiB/s) | 20260508 08:22:48.248 | INFO | Downloaded 34GiB to GPU in 2.4426s (13.9196GiB/s) | 20260508 08:22:53.386 | INFO | Analyzing chunk frames = [6639104,6639616) | 20260508 08:22:53.386 | INFO | MemInfo: { | 20260508 08:22:53.386 | INFO | "gpu" : " 17 GB", | 20260508 08:22:53.386 | INFO | "huge" : "", | 20260508 08:22:53.386 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:53.386 | INFO | "malloc" : " 14 GB", | 20260508 08:22:53.386 | INFO | "pinned" : " 29 GB" | 20260508 08:22:53.386 | INFO | } | 20260508 08:22:53.386 | INFO | Perf report for the last 5.13951 seconds: | 20260508 08:22:53.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:53.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.386 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 08:22:53.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.387 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:22:53.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:53.387 | INFO | Uploaded 36GiB to GPU in 2.43986s (14.755GiB/s) | 20260508 08:22:53.387 | INFO | Downloaded 34GiB to GPU in 2.44726s (13.8931GiB/s) | 20260508 08:22:53.484 | INFO | Starting a round of Baz IO | 20260508 08:22:56.908 | INFO | Populating RTMetrics report | 20260508 08:22:56.908 | INFO | RTMetrics report populated | 20260508 08:22:56.908 | INFO | Opening tmp RTMetrics json file | 20260508 08:22:56.918 | INFO | Serializing RTMetrics to json | 20260508 08:22:56.920 | INFO | Serialization done, swapping tmp file now... | 20260508 08:22:56.921 | INFO | RTMetrics done | 20260508 08:22:56.981 | INFO | Finished a round of Baz IO: Wrote 1.508 GiB in 3.496 seconds | 20260508 08:22:56.981 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.57% | 20260508 08:22:56.981 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.299227% / 0% / 6.35783e-05% | 20260508 08:22:56.981 | INFO | Baz IO overall: | 20260508 08:22:56.981 | INFO | BytesWritten : 2914716022930 | 20260508 08:22:56.981 | INFO | NumEvents : 1403303188757 | 20260508 08:22:58.518 | INFO | Analyzing chunk frames = [6639616,6640128) | 20260508 08:22:58.518 | INFO | MemInfo: { | 20260508 08:22:58.518 | INFO | "gpu" : " 17 GB", | 20260508 08:22:58.518 | INFO | "huge" : "", | 20260508 08:22:58.518 | INFO | "huge_shared" : " 29 GB", | 20260508 08:22:58.518 | INFO | "malloc" : " 14 GB", | 20260508 08:22:58.518 | INFO | "pinned" : " 29 GB" | 20260508 08:22:58.518 | INFO | } | 20260508 08:22:58.518 | INFO | Perf report for the last 5.13134 seconds: | 20260508 08:22:58.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:22:58.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:58.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:58.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:58.518 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 08:22:58.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:22:58.518 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:22:58.518 | INFO | PRE_HQ : 65.7%, 1.6, 54.6ms, 0 | 20260508 08:22:58.518 | INFO | BAZWRITER : 67.7%, 2.6, 90.8ms, 0 | 20260508 08:22:58.518 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:22:58.518 | INFO | Uploaded 36GiB to GPU in 2.56993s (14.0082GiB/s) | 20260508 08:22:58.518 | INFO | Downloaded 38GiB to GPU in 2.71189s (14.0124GiB/s) | 20260508 08:23:03.659 | INFO | Analyzing chunk frames = [6640128,6640640) | 20260508 08:23:03.659 | INFO | MemInfo: { | 20260508 08:23:03.659 | INFO | "gpu" : " 17 GB", | 20260508 08:23:03.659 | INFO | "huge" : "", | 20260508 08:23:03.659 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:03.659 | INFO | "malloc" : " 14 GB", | 20260508 08:23:03.659 | INFO | "pinned" : " 29 GB" | 20260508 08:23:03.659 | INFO | } | 20260508 08:23:03.659 | INFO | Perf report for the last 5.14133 seconds: | 20260508 08:23:03.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:03.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.659 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:23:03.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.659 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:23:03.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:03.660 | INFO | Uploaded 36GiB to GPU in 2.51154s (14.3338GiB/s) | 20260508 08:23:03.660 | INFO | Downloaded 34GiB to GPU in 2.44138s (13.9266GiB/s) | 20260508 08:23:08.806 | INFO | Analyzing chunk frames = [6640640,6641152) | 20260508 08:23:08.806 | INFO | MemInfo: { | 20260508 08:23:08.806 | INFO | "gpu" : " 17 GB", | 20260508 08:23:08.806 | INFO | "huge" : "", | 20260508 08:23:08.806 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:08.806 | INFO | "malloc" : " 14 GB", | 20260508 08:23:08.806 | INFO | "pinned" : " 29 GB" | 20260508 08:23:08.806 | INFO | } | 20260508 08:23:08.807 | INFO | Perf report for the last 5.14734 seconds: | 20260508 08:23:08.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:08.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 08:23:08.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:23:08.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:08.807 | INFO | Uploaded 36GiB to GPU in 2.55954s (14.065GiB/s) | 20260508 08:23:08.807 | INFO | Downloaded 34GiB to GPU in 2.43657s (13.954GiB/s) | 20260508 08:23:13.944 | INFO | Analyzing chunk frames = [6641152,6641664) | 20260508 08:23:13.944 | INFO | MemInfo: { | 20260508 08:23:13.944 | INFO | "gpu" : " 17 GB", | 20260508 08:23:13.944 | INFO | "huge" : "", | 20260508 08:23:13.944 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:13.944 | INFO | "malloc" : " 14 GB", | 20260508 08:23:13.944 | INFO | "pinned" : " 29 GB" | 20260508 08:23:13.944 | INFO | } | 20260508 08:23:13.944 | INFO | Perf report for the last 5.13749 seconds: | 20260508 08:23:13.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:13.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 08:23:13.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:23:13.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:13.944 | INFO | Uploaded 36GiB to GPU in 2.49809s (14.411GiB/s) | 20260508 08:23:13.944 | INFO | Downloaded 34GiB to GPU in 2.42976s (13.9932GiB/s) | 20260508 08:23:19.083 | INFO | Analyzing chunk frames = [6641664,6642176) | 20260508 08:23:19.083 | INFO | MemInfo: { | 20260508 08:23:19.083 | INFO | "gpu" : " 17 GB", | 20260508 08:23:19.083 | INFO | "huge" : "", | 20260508 08:23:19.083 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:19.083 | INFO | "malloc" : " 14 GB", | 20260508 08:23:19.083 | INFO | "pinned" : " 29 GB" | 20260508 08:23:19.083 | INFO | } | 20260508 08:23:19.083 | INFO | Perf report for the last 5.13878 seconds: | 20260508 08:23:19.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:19.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 08:23:19.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:23:19.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:19.083 | INFO | Uploaded 36GiB to GPU in 2.40049s (14.9969GiB/s) | 20260508 08:23:19.083 | INFO | Downloaded 34GiB to GPU in 2.54416s (13.364GiB/s) | 20260508 08:23:24.218 | INFO | Analyzing chunk frames = [6642176,6642688) | 20260508 08:23:24.218 | INFO | MemInfo: { | 20260508 08:23:24.218 | INFO | "gpu" : " 17 GB", | 20260508 08:23:24.218 | INFO | "huge" : "", | 20260508 08:23:24.218 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:24.218 | INFO | "malloc" : " 14 GB", | 20260508 08:23:24.218 | INFO | "pinned" : " 29 GB" | 20260508 08:23:24.219 | INFO | } | 20260508 08:23:24.219 | INFO | Perf report for the last 5.13588 seconds: | 20260508 08:23:24.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:24.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 08:23:24.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:23:24.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:24.219 | INFO | Uploaded 36GiB to GPU in 2.4202s (14.8748GiB/s) | 20260508 08:23:24.219 | INFO | Downloaded 34GiB to GPU in 2.44176s (13.9244GiB/s) | 20260508 08:23:29.360 | INFO | Analyzing chunk frames = [6642688,6643200) | 20260508 08:23:29.360 | INFO | MemInfo: { | 20260508 08:23:29.360 | INFO | "gpu" : " 17 GB", | 20260508 08:23:29.360 | INFO | "huge" : "", | 20260508 08:23:29.361 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:29.361 | INFO | "malloc" : " 14 GB", | 20260508 08:23:29.361 | INFO | "pinned" : " 29 GB" | 20260508 08:23:29.361 | INFO | } | 20260508 08:23:29.361 | INFO | Perf report for the last 5.14201 seconds: | 20260508 08:23:29.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:29.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 08:23:29.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:23:29.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:29.361 | INFO | Uploaded 36GiB to GPU in 2.47666s (14.5357GiB/s) | 20260508 08:23:29.361 | INFO | Downloaded 34GiB to GPU in 2.43575s (13.9588GiB/s) | 20260508 08:23:34.499 | INFO | Analyzing chunk frames = [6643200,6643712) | 20260508 08:23:34.499 | INFO | MemInfo: { | 20260508 08:23:34.499 | INFO | "gpu" : " 17 GB", | 20260508 08:23:34.499 | INFO | "huge" : "", | 20260508 08:23:34.499 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:34.499 | INFO | "malloc" : " 14 GB", | 20260508 08:23:34.499 | INFO | "pinned" : " 29 GB" | 20260508 08:23:34.499 | INFO | } | 20260508 08:23:34.499 | INFO | Perf report for the last 5.13838 seconds: | 20260508 08:23:34.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:34.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:23:34.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:23:34.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:34.499 | INFO | Uploaded 36GiB to GPU in 2.48155s (14.5071GiB/s) | 20260508 08:23:34.499 | INFO | Downloaded 34GiB to GPU in 2.4519s (13.8668GiB/s) | 20260508 08:23:34.499 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:23:34.500 | INFO | Total mem usage: 16.7 GiB | 20260508 08:23:34.500 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:23:34.500 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 08:23:34.500 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:23:34.500 | INFO | High Water Breakdown: | 20260508 08:23:34.500 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:23:34.500 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:23:34.500 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:23:34.500 | INFO | ------------------------------------------ | 20260508 08:23:34.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:23:34.500 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:23:34.500 | INFO | Total mem usage: 26.5 GiB | 20260508 08:23:34.500 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:23:34.500 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:23:34.500 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:23:34.500 | INFO | High Water Breakdown: | 20260508 08:23:34.500 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:23:34.500 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:23:34.500 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:23:34.500 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:23:34.500 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:23:34.500 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:23:34.500 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:23:34.500 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:23:34.500 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:23:34.500 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:23:34.500 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:23:34.500 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:23:34.500 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:23:34.500 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:23:34.500 | INFO | ------------------------------------------ | 20260508 08:23:34.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:23:34.500 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:23:34.500 | INFO | Total mem usage: 27.8 GiB | 20260508 08:23:34.500 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:23:34.500 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:23:34.500 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:23:34.500 | INFO | High Water Breakdown: | 20260508 08:23:34.500 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:23:34.500 | INFO | ------------------------------------------ | 20260508 08:23:34.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:23:34.500 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:23:34.500 | INFO | Total mem usage: 15.7 GiB | 20260508 08:23:34.500 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:23:34.500 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:23:34.500 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 08:23:34.500 | INFO | High Water Breakdown: | 20260508 08:23:34.500 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:23:34.500 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:23:34.500 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:23:34.500 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:23:34.500 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:23:34.500 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:23:34.500 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:23:34.500 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:23:34.500 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:23:34.500 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:23:34.500 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:23:34.500 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:23:34.500 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:23:34.500 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:23:34.500 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:23:34.500 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:23:34.500 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:23:34.500 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:23:34.500 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:23:34.500 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:23:34.500 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:23:34.500 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:23:34.500 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:23:34.500 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:23:34.500 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:23:34.500 | INFO | ------------------------------------------ | 20260508 08:23:34.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:23:34.500 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:23:34.500 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:23:34.500 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:23:34.500 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.14e-01 | 20260508 08:23:34.500 | INFO | PerfReport: 0.29%, AggregateStats -- 9.61e-02 / 1.03e-01, 1.10e-02 / 4.55e-03 | 20260508 08:23:34.500 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.38e+00, 5.86e-01 / 2.50e-01 | 20260508 08:23:34.500 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.07e-01 / 1.18e-01 | 20260508 08:23:34.500 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.51e+01, 3.62e+00 / 1.03e+00 | 20260508 08:23:34.500 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.83e+00 / 1.31e+00 | 20260508 08:23:34.500 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.50e+00, 4.03e-01 / 2.63e-01 | 20260508 08:23:34.500 | INFO | PerfReport: 3.09%, Metrics -- 1.01e+00 / 1.17e+00, 2.49e-01 / 1.22e-01 | 20260508 08:23:34.500 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.06e-01 / 3.34e-01, 4.86e-02 / 2.33e-02 | 20260508 08:23:34.500 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:23:34.500 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:23:34.500 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:23:34.500 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 08:23:34.500 | INFO | PerfReport: 24.45%, Upload -- 2.58e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 08:23:34.501 | INFO | PerfReport: 24.44%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 08:23:34.501 | INFO | PerfReport: 30.91%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:23:34.501 | INFO | PerfReport: 2.25%, LockUpload -- 2.38e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 08:23:34.501 | INFO | PerfReport: 2.57%, LockDownload -- 2.72e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 08:23:34.501 | INFO | PerfReport: 15.37%, LockCompute -- 1.62e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260508 08:23:34.501 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260508 08:23:34.597 | INFO | Starting a round of Baz IO | 20260508 08:23:38.017 | INFO | Populating RTMetrics report | 20260508 08:23:38.017 | INFO | RTMetrics report populated | 20260508 08:23:38.017 | INFO | Opening tmp RTMetrics json file | 20260508 08:23:38.030 | INFO | Serializing RTMetrics to json | 20260508 08:23:38.031 | INFO | Serialization done, swapping tmp file now... | 20260508 08:23:38.032 | INFO | RTMetrics done | 20260508 08:23:38.087 | INFO | Finished a round of Baz IO: Wrote 1.511 GiB in 3.489 seconds | 20260508 08:23:38.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.57% | 20260508 08:23:38.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.298536% / 0% / 4.76837e-05% | 20260508 08:23:38.087 | INFO | Baz IO overall: | 20260508 08:23:38.087 | INFO | BytesWritten : 2916338267121 | 20260508 08:23:38.087 | INFO | NumEvents : 1404070857392 | 20260508 08:23:39.642 | INFO | Analyzing chunk frames = [6643712,6644224) | 20260508 08:23:39.642 | INFO | MemInfo: { | 20260508 08:23:39.642 | INFO | "gpu" : " 17 GB", | 20260508 08:23:39.642 | INFO | "huge" : "", | 20260508 08:23:39.642 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:39.642 | INFO | "malloc" : " 14 GB", | 20260508 08:23:39.642 | INFO | "pinned" : " 29 GB" | 20260508 08:23:39.642 | INFO | } | 20260508 08:23:39.642 | INFO | Perf report for the last 5.14323 seconds: | 20260508 08:23:39.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:39.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:39.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:39.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:39.643 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:23:39.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:39.643 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:23:39.643 | INFO | PRE_HQ : 65.4%, 1.6, 55.0ms, 0 | 20260508 08:23:39.643 | INFO | BAZWRITER : 67.2%, 2.6, 89.8ms, 0 | 20260508 08:23:39.643 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:23:39.643 | INFO | Uploaded 36GiB to GPU in 2.58206s (13.9424GiB/s) | 20260508 08:23:39.643 | INFO | Downloaded 38GiB to GPU in 2.71917s (13.9748GiB/s) | 20260508 08:23:44.772 | INFO | Analyzing chunk frames = [6644224,6644736) | 20260508 08:23:44.772 | INFO | MemInfo: { | 20260508 08:23:44.772 | INFO | "gpu" : " 17 GB", | 20260508 08:23:44.772 | INFO | "huge" : "", | 20260508 08:23:44.772 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:44.772 | INFO | "malloc" : " 14 GB", | 20260508 08:23:44.772 | INFO | "pinned" : " 29 GB" | 20260508 08:23:44.772 | INFO | } | 20260508 08:23:44.772 | INFO | Perf report for the last 5.12981 seconds: | 20260508 08:23:44.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:44.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.772 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 08:23:44.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.772 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:23:44.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:44.773 | INFO | Uploaded 36GiB to GPU in 2.47913s (14.5212GiB/s) | 20260508 08:23:44.773 | INFO | Downloaded 34GiB to GPU in 2.47074s (13.7611GiB/s) | 20260508 08:23:49.915 | INFO | Analyzing chunk frames = [6644736,6645248) | 20260508 08:23:49.915 | INFO | MemInfo: { | 20260508 08:23:49.915 | INFO | "gpu" : " 17 GB", | 20260508 08:23:49.915 | INFO | "huge" : "", | 20260508 08:23:49.915 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:49.915 | INFO | "malloc" : " 14 GB", | 20260508 08:23:49.915 | INFO | "pinned" : " 29 GB" | 20260508 08:23:49.915 | INFO | } | 20260508 08:23:49.915 | INFO | Perf report for the last 5.14293 seconds: | 20260508 08:23:49.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:49.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.915 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 08:23:49.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.915 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:23:49.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:49.916 | INFO | Uploaded 36GiB to GPU in 2.57077s (14.0036GiB/s) | 20260508 08:23:49.916 | INFO | Downloaded 34GiB to GPU in 2.43568s (13.9592GiB/s) | 20260508 08:23:55.055 | INFO | Analyzing chunk frames = [6645248,6645760) | 20260508 08:23:55.055 | INFO | MemInfo: { | 20260508 08:23:55.055 | INFO | "gpu" : " 17 GB", | 20260508 08:23:55.055 | INFO | "huge" : "", | 20260508 08:23:55.055 | INFO | "huge_shared" : " 29 GB", | 20260508 08:23:55.055 | INFO | "malloc" : " 14 GB", | 20260508 08:23:55.055 | INFO | "pinned" : " 29 GB" | 20260508 08:23:55.055 | INFO | } | 20260508 08:23:55.055 | INFO | Perf report for the last 5.13966 seconds: | 20260508 08:23:55.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:23:55.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.055 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 08:23:55.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.055 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 08:23:55.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:23:55.056 | INFO | Uploaded 36GiB to GPU in 2.56946s (14.0107GiB/s) | 20260508 08:23:55.056 | INFO | Downloaded 34GiB to GPU in 2.42584s (14.0158GiB/s) | 20260508 08:24:00.195 | INFO | Analyzing chunk frames = [6645760,6646272) | 20260508 08:24:00.195 | INFO | MemInfo: { | 20260508 08:24:00.195 | INFO | "gpu" : " 17 GB", | 20260508 08:24:00.195 | INFO | "huge" : "", | 20260508 08:24:00.195 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:00.195 | INFO | "malloc" : " 14 GB", | 20260508 08:24:00.195 | INFO | "pinned" : " 29 GB" | 20260508 08:24:00.195 | INFO | } | 20260508 08:24:00.195 | INFO | Perf report for the last 5.13999 seconds: | 20260508 08:24:00.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:00.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 08:24:00.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:24:00.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:00.195 | INFO | Uploaded 36GiB to GPU in 2.36987s (15.1907GiB/s) | 20260508 08:24:00.195 | INFO | Downloaded 34GiB to GPU in 2.49992s (13.6004GiB/s) | 20260508 08:24:05.331 | INFO | Analyzing chunk frames = [6646272,6646784) | 20260508 08:24:05.331 | INFO | MemInfo: { | 20260508 08:24:05.331 | INFO | "gpu" : " 17 GB", | 20260508 08:24:05.331 | INFO | "huge" : "", | 20260508 08:24:05.331 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:05.331 | INFO | "malloc" : " 14 GB", | 20260508 08:24:05.331 | INFO | "pinned" : " 29 GB" | 20260508 08:24:05.331 | INFO | } | 20260508 08:24:05.331 | INFO | Perf report for the last 5.13626 seconds: | 20260508 08:24:05.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:05.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.331 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:24:05.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.331 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:24:05.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:05.332 | INFO | Uploaded 36GiB to GPU in 2.37111s (15.1827GiB/s) | 20260508 08:24:05.332 | INFO | Downloaded 34GiB to GPU in 2.43368s (13.9706GiB/s) | 20260508 08:24:10.469 | INFO | Analyzing chunk frames = [6646784,6647296) | 20260508 08:24:10.469 | INFO | MemInfo: { | 20260508 08:24:10.469 | INFO | "gpu" : " 17 GB", | 20260508 08:24:10.469 | INFO | "huge" : "", | 20260508 08:24:10.469 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:10.469 | INFO | "malloc" : " 14 GB", | 20260508 08:24:10.469 | INFO | "pinned" : " 29 GB" | 20260508 08:24:10.469 | INFO | } | 20260508 08:24:10.469 | INFO | Perf report for the last 5.13759 seconds: | 20260508 08:24:10.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:10.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 08:24:10.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:24:10.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:10.469 | INFO | Uploaded 36GiB to GPU in 2.50303s (14.3826GiB/s) | 20260508 08:24:10.469 | INFO | Downloaded 34GiB to GPU in 2.45134s (13.87GiB/s) | 20260508 08:24:15.611 | INFO | Analyzing chunk frames = [6647296,6647808) | 20260508 08:24:15.611 | INFO | MemInfo: { | 20260508 08:24:15.611 | INFO | "gpu" : " 17 GB", | 20260508 08:24:15.611 | INFO | "huge" : "", | 20260508 08:24:15.611 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:15.611 | INFO | "malloc" : " 14 GB", | 20260508 08:24:15.611 | INFO | "pinned" : " 29 GB" | 20260508 08:24:15.611 | INFO | } | 20260508 08:24:15.611 | INFO | Perf report for the last 5.14236 seconds: | 20260508 08:24:15.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:15.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.611 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:24:15.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.611 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:24:15.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:15.612 | INFO | Uploaded 36GiB to GPU in 2.47148s (14.5662GiB/s) | 20260508 08:24:15.612 | INFO | Downloaded 34GiB to GPU in 2.44923s (13.8819GiB/s) | 20260508 08:24:15.709 | INFO | Starting a round of Baz IO | 20260508 08:24:19.119 | INFO | Populating RTMetrics report | 20260508 08:24:19.120 | INFO | RTMetrics report populated | 20260508 08:24:19.120 | INFO | Opening tmp RTMetrics json file | 20260508 08:24:19.131 | INFO | Serializing RTMetrics to json | 20260508 08:24:19.132 | INFO | Serialization done, swapping tmp file now... | 20260508 08:24:19.132 | INFO | RTMetrics done | 20260508 08:24:19.194 | INFO | Finished a round of Baz IO: Wrote 1.509 GiB in 3.485 seconds | 20260508 08:24:19.195 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.57% | 20260508 08:24:19.195 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.297288% / 7.94729e-06% / 5.16574e-05% | 20260508 08:24:19.195 | INFO | Baz IO overall: | 20260508 08:24:19.195 | INFO | BytesWritten : 2917958011395 | 20260508 08:24:19.195 | INFO | NumEvents : 1404837612945 | 20260508 08:24:20.758 | INFO | Analyzing chunk frames = [6647808,6648320) | 20260508 08:24:20.758 | INFO | MemInfo: { | 20260508 08:24:20.758 | INFO | "gpu" : " 17 GB", | 20260508 08:24:20.758 | INFO | "huge" : "", | 20260508 08:24:20.758 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:20.758 | INFO | "malloc" : " 14 GB", | 20260508 08:24:20.758 | INFO | "pinned" : " 29 GB" | 20260508 08:24:20.758 | INFO | } | 20260508 08:24:20.758 | INFO | Perf report for the last 5.14664 seconds: | 20260508 08:24:20.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:20.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:20.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:20.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:20.758 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 08:24:20.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:20.758 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:24:20.758 | INFO | PRE_HQ : 65.3%, 1.6, 54.8ms, 0 | 20260508 08:24:20.758 | INFO | BAZWRITER : 67.3%, 2.6, 90.7ms, 0 | 20260508 08:24:20.758 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:24:20.758 | INFO | Uploaded 36GiB to GPU in 2.57836s (13.9623GiB/s) | 20260508 08:24:20.759 | INFO | Downloaded 38GiB to GPU in 2.7069s (14.0382GiB/s) | 20260508 08:24:25.891 | INFO | Analyzing chunk frames = [6648320,6648832) | 20260508 08:24:25.891 | INFO | MemInfo: { | 20260508 08:24:25.891 | INFO | "gpu" : " 17 GB", | 20260508 08:24:25.891 | INFO | "huge" : "", | 20260508 08:24:25.891 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:25.891 | INFO | "malloc" : " 14 GB", | 20260508 08:24:25.891 | INFO | "pinned" : " 29 GB" | 20260508 08:24:25.891 | INFO | } | 20260508 08:24:25.891 | INFO | Perf report for the last 5.13297 seconds: | 20260508 08:24:25.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:25.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:24:25.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:24:25.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:25.891 | INFO | Uploaded 36GiB to GPU in 2.48079s (14.5115GiB/s) | 20260508 08:24:25.892 | INFO | Downloaded 34GiB to GPU in 2.47354s (13.7455GiB/s) | 20260508 08:24:31.029 | INFO | Analyzing chunk frames = [6648832,6649344) | 20260508 08:24:31.029 | INFO | MemInfo: { | 20260508 08:24:31.029 | INFO | "gpu" : " 17 GB", | 20260508 08:24:31.029 | INFO | "huge" : "", | 20260508 08:24:31.029 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:31.029 | INFO | "malloc" : " 14 GB", | 20260508 08:24:31.029 | INFO | "pinned" : " 29 GB" | 20260508 08:24:31.029 | INFO | } | 20260508 08:24:31.029 | INFO | Perf report for the last 5.13849 seconds: | 20260508 08:24:31.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:31.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.029 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 08:24:31.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.029 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 08:24:31.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:31.030 | INFO | Uploaded 36GiB to GPU in 2.44259s (14.7385GiB/s) | 20260508 08:24:31.030 | INFO | Downloaded 34GiB to GPU in 2.45141s (13.8696GiB/s) | 20260508 08:24:36.173 | INFO | Analyzing chunk frames = [6649344,6649856) | 20260508 08:24:36.173 | INFO | MemInfo: { | 20260508 08:24:36.173 | INFO | "gpu" : " 17 GB", | 20260508 08:24:36.173 | INFO | "huge" : "", | 20260508 08:24:36.173 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:36.173 | INFO | "malloc" : " 14 GB", | 20260508 08:24:36.173 | INFO | "pinned" : " 29 GB" | 20260508 08:24:36.173 | INFO | } | 20260508 08:24:36.173 | INFO | Perf report for the last 5.14404 seconds: | 20260508 08:24:36.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:36.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.173 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 08:24:36.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.173 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:24:36.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:36.174 | INFO | Uploaded 36GiB to GPU in 2.48521s (14.4857GiB/s) | 20260508 08:24:36.174 | INFO | Downloaded 34GiB to GPU in 2.45788s (13.8331GiB/s) | 20260508 08:24:41.310 | INFO | Analyzing chunk frames = [6649856,6650368) | 20260508 08:24:41.310 | INFO | MemInfo: { | 20260508 08:24:41.310 | INFO | "gpu" : " 17 GB", | 20260508 08:24:41.310 | INFO | "huge" : "", | 20260508 08:24:41.310 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:41.310 | INFO | "malloc" : " 14 GB", | 20260508 08:24:41.310 | INFO | "pinned" : " 29 GB" | 20260508 08:24:41.310 | INFO | } | 20260508 08:24:41.310 | INFO | Perf report for the last 5.13708 seconds: | 20260508 08:24:41.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:41.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.310 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:24:41.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.310 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:24:41.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:41.311 | INFO | Uploaded 36GiB to GPU in 2.3759s (15.1522GiB/s) | 20260508 08:24:41.311 | INFO | Downloaded 34GiB to GPU in 2.51073s (13.5419GiB/s) | 20260508 08:24:46.458 | INFO | Analyzing chunk frames = [6650368,6650880) | 20260508 08:24:46.458 | INFO | MemInfo: { | 20260508 08:24:46.458 | INFO | "gpu" : " 17 GB", | 20260508 08:24:46.458 | INFO | "huge" : "", | 20260508 08:24:46.458 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:46.458 | INFO | "malloc" : " 14 GB", | 20260508 08:24:46.458 | INFO | "pinned" : " 29 GB" | 20260508 08:24:46.458 | INFO | } | 20260508 08:24:46.458 | INFO | Perf report for the last 5.14787 seconds: | 20260508 08:24:46.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:46.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:24:46.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:24:46.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:46.458 | INFO | Uploaded 36GiB to GPU in 2.37791s (15.1393GiB/s) | 20260508 08:24:46.459 | INFO | Downloaded 34GiB to GPU in 2.44528s (13.9044GiB/s) | 20260508 08:24:51.589 | INFO | Analyzing chunk frames = [6650880,6651392) | 20260508 08:24:51.589 | INFO | MemInfo: { | 20260508 08:24:51.589 | INFO | "gpu" : " 17 GB", | 20260508 08:24:51.589 | INFO | "huge" : "", | 20260508 08:24:51.589 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:51.589 | INFO | "malloc" : " 14 GB", | 20260508 08:24:51.589 | INFO | "pinned" : " 29 GB" | 20260508 08:24:51.589 | INFO | } | 20260508 08:24:51.589 | INFO | Perf report for the last 5.13117 seconds: | 20260508 08:24:51.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:51.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.590 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:24:51.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.590 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:24:51.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:51.590 | INFO | Uploaded 36GiB to GPU in 2.39813s (15.0117GiB/s) | 20260508 08:24:51.590 | INFO | Downloaded 34GiB to GPU in 2.44309s (13.9168GiB/s) | 20260508 08:24:56.725 | INFO | Analyzing chunk frames = [6651392,6651904) | 20260508 08:24:56.725 | INFO | MemInfo: { | 20260508 08:24:56.725 | INFO | "gpu" : " 17 GB", | 20260508 08:24:56.725 | INFO | "huge" : "", | 20260508 08:24:56.725 | INFO | "huge_shared" : " 29 GB", | 20260508 08:24:56.725 | INFO | "malloc" : " 14 GB", | 20260508 08:24:56.725 | INFO | "pinned" : " 29 GB" | 20260508 08:24:56.725 | INFO | } | 20260508 08:24:56.725 | INFO | Perf report for the last 5.13604 seconds: | 20260508 08:24:56.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:24:56.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.726 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:24:56.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.726 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:24:56.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:24:56.726 | INFO | Uploaded 36GiB to GPU in 2.47208s (14.5626GiB/s) | 20260508 08:24:56.726 | INFO | Downloaded 34GiB to GPU in 2.46501s (13.7931GiB/s) | 20260508 08:24:56.726 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:24:56.726 | INFO | Total mem usage: 16.7 GiB | 20260508 08:24:56.726 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:24:56.726 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:24:56.726 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:24:56.726 | INFO | High Water Breakdown: | 20260508 08:24:56.726 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:24:56.726 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:24:56.726 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:24:56.726 | INFO | ------------------------------------------ | 20260508 08:24:56.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:24:56.726 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:24:56.726 | INFO | Total mem usage: 26.5 GiB | 20260508 08:24:56.726 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:24:56.726 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:24:56.726 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:24:56.726 | INFO | High Water Breakdown: | 20260508 08:24:56.726 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:24:56.726 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:24:56.726 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:24:56.726 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:24:56.726 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:24:56.726 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:24:56.726 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:24:56.726 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:24:56.726 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:24:56.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:24:56.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:24:56.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:24:56.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:24:56.726 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:24:56.726 | INFO | ------------------------------------------ | 20260508 08:24:56.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:24:56.726 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:24:56.726 | INFO | Total mem usage: 27.8 GiB | 20260508 08:24:56.726 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:24:56.726 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:24:56.726 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:24:56.726 | INFO | High Water Breakdown: | 20260508 08:24:56.726 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:24:56.726 | INFO | ------------------------------------------ | 20260508 08:24:56.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:24:56.726 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:24:56.726 | INFO | Total mem usage: 15.7 GiB | 20260508 08:24:56.726 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:24:56.726 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:24:56.726 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:24:56.726 | INFO | High Water Breakdown: | 20260508 08:24:56.726 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:24:56.726 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:24:56.726 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:24:56.726 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:24:56.726 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:24:56.726 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:24:56.726 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:24:56.726 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:24:56.726 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:24:56.726 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:24:56.726 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:24:56.726 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:24:56.726 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:24:56.726 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:24:56.726 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:24:56.726 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:24:56.726 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:24:56.726 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:24:56.726 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:24:56.726 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:24:56.726 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:24:56.726 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:24:56.726 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:24:56.726 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:24:56.726 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:24:56.726 | INFO | ------------------------------------------ | 20260508 08:24:56.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:24:56.726 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:24:56.726 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:24:56.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:24:56.726 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.62e-01 | 20260508 08:24:56.726 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.07e-02 / 5.51e-03 | 20260508 08:24:56.726 | INFO | PerfReport: 12.11%, Baseline -- 3.96e+00 / 4.45e+00, 5.90e-01 / 2.25e-01 | 20260508 08:24:56.726 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.41e-01 | 20260508 08:24:56.726 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.47e+01, 3.50e+00 / 1.05e+00 | 20260508 08:24:56.726 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.89e+00 / 1.39e+00 | 20260508 08:24:56.726 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.47e+00, 4.09e-01 / 2.38e-01 | 20260508 08:24:56.726 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.19e+00, 2.54e-01 / 1.43e-01 | 20260508 08:24:56.726 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.06e-01 / 3.36e-01, 4.90e-02 / 2.29e-02 | 20260508 08:24:56.727 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:24:56.727 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:24:56.727 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:24:56.727 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 24.35%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 24.54%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 30.97%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 2.10%, LockUpload -- 2.22e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 2.62%, LockDownload -- 2.77e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 15.41%, LockCompute -- 1.63e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:24:56.727 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 08:24:56.823 | INFO | Starting a round of Baz IO | 20260508 08:25:00.248 | INFO | Populating RTMetrics report | 20260508 08:25:00.249 | INFO | RTMetrics report populated | 20260508 08:25:00.249 | INFO | Opening tmp RTMetrics json file | 20260508 08:25:00.260 | INFO | Serializing RTMetrics to json | 20260508 08:25:00.262 | INFO | Serialization done, swapping tmp file now... | 20260508 08:25:00.262 | INFO | RTMetrics done | 20260508 08:25:00.318 | INFO | Finished a round of Baz IO: Wrote 1.509 GiB in 3.495 seconds | 20260508 08:25:00.318 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.43% / 0.00% / 59.57% | 20260508 08:25:00.318 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.297499% / 0% / 3.97364e-05% | 20260508 08:25:00.318 | INFO | Baz IO overall: | 20260508 08:25:00.318 | INFO | BytesWritten : 2919578108042 | 20260508 08:25:00.318 | INFO | NumEvents : 1405605677418 | 20260508 08:25:01.868 | INFO | Analyzing chunk frames = [6651904,6652416) | 20260508 08:25:01.868 | INFO | MemInfo: { | 20260508 08:25:01.868 | INFO | "gpu" : " 17 GB", | 20260508 08:25:01.868 | INFO | "huge" : "", | 20260508 08:25:01.868 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:01.868 | INFO | "malloc" : " 14 GB", | 20260508 08:25:01.868 | INFO | "pinned" : " 29 GB" | 20260508 08:25:01.868 | INFO | } | 20260508 08:25:01.868 | INFO | Perf report for the last 5.14231 seconds: | 20260508 08:25:01.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:01.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:01.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:01.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:01.868 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 08:25:01.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:01.868 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:25:01.868 | INFO | PRE_HQ : 65.7%, 1.6, 54.7ms, 0 | 20260508 08:25:01.868 | INFO | BAZWRITER : 67.5%, 2.6, 90.7ms, 0 | 20260508 08:25:01.868 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:25:01.868 | INFO | Uploaded 36GiB to GPU in 2.56402s (14.0405GiB/s) | 20260508 08:25:01.868 | INFO | Downloaded 38GiB to GPU in 2.70348s (14.056GiB/s) | 20260508 08:25:07.011 | INFO | Analyzing chunk frames = [6652416,6652928) | 20260508 08:25:07.011 | INFO | MemInfo: { | 20260508 08:25:07.011 | INFO | "gpu" : " 17 GB", | 20260508 08:25:07.011 | INFO | "huge" : "", | 20260508 08:25:07.011 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:07.011 | INFO | "malloc" : " 14 GB", | 20260508 08:25:07.012 | INFO | "pinned" : " 29 GB" | 20260508 08:25:07.012 | INFO | } | 20260508 08:25:07.012 | INFO | Perf report for the last 5.14376 seconds: | 20260508 08:25:07.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:07.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 08:25:07.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:25:07.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:07.012 | INFO | Uploaded 36GiB to GPU in 2.52941s (14.2326GiB/s) | 20260508 08:25:07.012 | INFO | Downloaded 34GiB to GPU in 2.45167s (13.8681GiB/s) | 20260508 08:25:12.144 | INFO | Analyzing chunk frames = [6652928,6653440) | 20260508 08:25:12.144 | INFO | MemInfo: { | 20260508 08:25:12.144 | INFO | "gpu" : " 17 GB", | 20260508 08:25:12.144 | INFO | "huge" : "", | 20260508 08:25:12.144 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:12.144 | INFO | "malloc" : " 14 GB", | 20260508 08:25:12.144 | INFO | "pinned" : " 29 GB" | 20260508 08:25:12.144 | INFO | } | 20260508 08:25:12.144 | INFO | Perf report for the last 5.13271 seconds: | 20260508 08:25:12.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:12.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.144 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260508 08:25:12.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.144 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:25:12.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:12.145 | INFO | Uploaded 36GiB to GPU in 2.5027s (14.3844GiB/s) | 20260508 08:25:12.145 | INFO | Downloaded 34GiB to GPU in 2.43414s (13.968GiB/s) | 20260508 08:25:17.282 | INFO | Analyzing chunk frames = [6653440,6653952) | 20260508 08:25:17.282 | INFO | MemInfo: { | 20260508 08:25:17.282 | INFO | "gpu" : " 17 GB", | 20260508 08:25:17.282 | INFO | "huge" : "", | 20260508 08:25:17.282 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:17.282 | INFO | "malloc" : " 14 GB", | 20260508 08:25:17.282 | INFO | "pinned" : " 29 GB" | 20260508 08:25:17.282 | INFO | } | 20260508 08:25:17.282 | INFO | Perf report for the last 5.13755 seconds: | 20260508 08:25:17.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:17.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 08:25:17.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:25:17.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:17.282 | INFO | Uploaded 36GiB to GPU in 2.51833s (14.2952GiB/s) | 20260508 08:25:17.282 | INFO | Downloaded 34GiB to GPU in 2.43659s (13.9539GiB/s) | 20260508 08:25:22.429 | INFO | Analyzing chunk frames = [6653952,6654464) | 20260508 08:25:22.429 | INFO | MemInfo: { | 20260508 08:25:22.429 | INFO | "gpu" : " 17 GB", | 20260508 08:25:22.429 | INFO | "huge" : "", | 20260508 08:25:22.429 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:22.429 | INFO | "malloc" : " 14 GB", | 20260508 08:25:22.430 | INFO | "pinned" : " 29 GB" | 20260508 08:25:22.430 | INFO | } | 20260508 08:25:22.430 | INFO | Perf report for the last 5.14778 seconds: | 20260508 08:25:22.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:22.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 08:25:22.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:25:22.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:22.430 | INFO | Uploaded 36GiB to GPU in 2.40027s (14.9983GiB/s) | 20260508 08:25:22.430 | INFO | Downloaded 34GiB to GPU in 2.51777s (13.504GiB/s) | 20260508 08:25:27.569 | INFO | Analyzing chunk frames = [6654464,6654976) | 20260508 08:25:27.569 | INFO | MemInfo: { | 20260508 08:25:27.569 | INFO | "gpu" : " 17 GB", | 20260508 08:25:27.569 | INFO | "huge" : "", | 20260508 08:25:27.569 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:27.569 | INFO | "malloc" : " 14 GB", | 20260508 08:25:27.569 | INFO | "pinned" : " 29 GB" | 20260508 08:25:27.569 | INFO | } | 20260508 08:25:27.569 | INFO | Perf report for the last 5.1395 seconds: | 20260508 08:25:27.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:27.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 08:25:27.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:25:27.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:27.569 | INFO | Uploaded 36GiB to GPU in 2.39578s (15.0264GiB/s) | 20260508 08:25:27.569 | INFO | Downloaded 34GiB to GPU in 2.42278s (14.0335GiB/s) | 20260508 08:25:32.705 | INFO | Analyzing chunk frames = [6654976,6655488) | 20260508 08:25:32.705 | INFO | MemInfo: { | 20260508 08:25:32.705 | INFO | "gpu" : " 17 GB", | 20260508 08:25:32.705 | INFO | "huge" : "", | 20260508 08:25:32.705 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:32.705 | INFO | "malloc" : " 14 GB", | 20260508 08:25:32.705 | INFO | "pinned" : " 29 GB" | 20260508 08:25:32.706 | INFO | } | 20260508 08:25:32.706 | INFO | Perf report for the last 5.13646 seconds: | 20260508 08:25:32.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:32.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:25:32.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:25:32.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:32.706 | INFO | Uploaded 36GiB to GPU in 2.46579s (14.5998GiB/s) | 20260508 08:25:32.706 | INFO | Downloaded 34GiB to GPU in 2.43877s (13.9414GiB/s) | 20260508 08:25:37.844 | INFO | Analyzing chunk frames = [6655488,6656000) | 20260508 08:25:37.844 | INFO | MemInfo: { | 20260508 08:25:37.844 | INFO | "gpu" : " 17 GB", | 20260508 08:25:37.844 | INFO | "huge" : "", | 20260508 08:25:37.845 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:37.845 | INFO | "malloc" : " 14 GB", | 20260508 08:25:37.845 | INFO | "pinned" : " 29 GB" | 20260508 08:25:37.845 | INFO | } | 20260508 08:25:37.845 | INFO | Perf report for the last 5.13928 seconds: | 20260508 08:25:37.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:37.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:25:37.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 08:25:37.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:37.845 | INFO | Uploaded 36GiB to GPU in 2.447s (14.7119GiB/s) | 20260508 08:25:37.845 | INFO | Downloaded 34GiB to GPU in 2.43589s (13.9579GiB/s) | 20260508 08:25:37.959 | INFO | Starting a round of Baz IO | 20260508 08:25:41.363 | INFO | Populating RTMetrics report | 20260508 08:25:41.363 | INFO | RTMetrics report populated | 20260508 08:25:41.363 | INFO | Opening tmp RTMetrics json file | 20260508 08:25:41.364 | INFO | Serializing RTMetrics to json | 20260508 08:25:41.365 | INFO | Serialization done, swapping tmp file now... | 20260508 08:25:41.366 | INFO | RTMetrics done | 20260508 08:25:41.434 | INFO | Finished a round of Baz IO: Wrote 1.506 GiB in 3.475 seconds | 20260508 08:25:41.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.57% | 20260508 08:25:41.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.296168% / 3.97364e-06% / 5.16574e-05% | 20260508 08:25:41.435 | INFO | Baz IO overall: | 20260508 08:25:41.435 | INFO | BytesWritten : 2921194846163 | 20260508 08:25:41.435 | INFO | NumEvents : 1406371576982 | 20260508 08:25:42.979 | INFO | Analyzing chunk frames = [6656000,6656512) | 20260508 08:25:42.979 | INFO | MemInfo: { | 20260508 08:25:42.979 | INFO | "gpu" : " 17 GB", | 20260508 08:25:42.979 | INFO | "huge" : "", | 20260508 08:25:42.979 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:42.980 | INFO | "malloc" : " 14 GB", | 20260508 08:25:42.980 | INFO | "pinned" : " 29 GB" | 20260508 08:25:42.980 | INFO | } | 20260508 08:25:42.980 | INFO | Perf report for the last 5.13461 seconds: | 20260508 08:25:42.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:42.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:42.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:42.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:42.980 | INFO | ANALYSIS : 68.1%, 3.0, 105.7ms, 0 | 20260508 08:25:42.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:42.980 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260508 08:25:42.980 | INFO | PRE_HQ : 66.7%, 1.8, 62.3ms, 0 | 20260508 08:25:42.980 | INFO | BAZWRITER : 67.0%, 2.8, 97.3ms, 0 | 20260508 08:25:42.980 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 08:25:42.980 | INFO | Uploaded 36GiB to GPU in 2.56548s (14.0325GiB/s) | 20260508 08:25:42.980 | INFO | Downloaded 38GiB to GPU in 2.70262s (14.0604GiB/s) | 20260508 08:25:48.119 | INFO | Analyzing chunk frames = [6656512,6657024) | 20260508 08:25:48.119 | INFO | MemInfo: { | 20260508 08:25:48.119 | INFO | "gpu" : " 17 GB", | 20260508 08:25:48.119 | INFO | "huge" : "", | 20260508 08:25:48.119 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:48.119 | INFO | "malloc" : " 14 GB", | 20260508 08:25:48.119 | INFO | "pinned" : " 29 GB" | 20260508 08:25:48.119 | INFO | } | 20260508 08:25:48.119 | INFO | Perf report for the last 5.1397 seconds: | 20260508 08:25:48.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:48.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.119 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:25:48.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.119 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:25:48.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:48.120 | INFO | Uploaded 36GiB to GPU in 2.50223s (14.3872GiB/s) | 20260508 08:25:48.120 | INFO | Downloaded 34GiB to GPU in 2.43939s (13.9379GiB/s) | 20260508 08:25:53.265 | INFO | Analyzing chunk frames = [6657024,6657536) | 20260508 08:25:53.265 | INFO | MemInfo: { | 20260508 08:25:53.265 | INFO | "gpu" : " 17 GB", | 20260508 08:25:53.265 | INFO | "huge" : "", | 20260508 08:25:53.265 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:53.265 | INFO | "malloc" : " 14 GB", | 20260508 08:25:53.265 | INFO | "pinned" : " 29 GB" | 20260508 08:25:53.265 | INFO | } | 20260508 08:25:53.265 | INFO | Perf report for the last 5.14585 seconds: | 20260508 08:25:53.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:53.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.265 | INFO | ANALYSIS : 67.3%, 3.0, 104.7ms, 0 | 20260508 08:25:53.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.265 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:25:53.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:53.266 | INFO | Uploaded 36GiB to GPU in 2.59052s (13.8968GiB/s) | 20260508 08:25:53.266 | INFO | Downloaded 34GiB to GPU in 2.42989s (13.9924GiB/s) | 20260508 08:25:58.405 | INFO | Analyzing chunk frames = [6657536,6658048) | 20260508 08:25:58.405 | INFO | MemInfo: { | 20260508 08:25:58.405 | INFO | "gpu" : " 17 GB", | 20260508 08:25:58.405 | INFO | "huge" : "", | 20260508 08:25:58.405 | INFO | "huge_shared" : " 29 GB", | 20260508 08:25:58.405 | INFO | "malloc" : " 14 GB", | 20260508 08:25:58.405 | INFO | "pinned" : " 29 GB" | 20260508 08:25:58.405 | INFO | } | 20260508 08:25:58.405 | INFO | Perf report for the last 5.14025 seconds: | 20260508 08:25:58.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:25:58.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 08:25:58.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:25:58.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:25:58.406 | INFO | Uploaded 36GiB to GPU in 2.52005s (14.2854GiB/s) | 20260508 08:25:58.406 | INFO | Downloaded 34GiB to GPU in 2.43049s (13.989GiB/s) | 20260508 08:26:03.544 | INFO | Analyzing chunk frames = [6658048,6658560) | 20260508 08:26:03.544 | INFO | MemInfo: { | 20260508 08:26:03.544 | INFO | "gpu" : " 17 GB", | 20260508 08:26:03.544 | INFO | "huge" : "", | 20260508 08:26:03.544 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:03.544 | INFO | "malloc" : " 14 GB", | 20260508 08:26:03.544 | INFO | "pinned" : " 29 GB" | 20260508 08:26:03.544 | INFO | } | 20260508 08:26:03.544 | INFO | Perf report for the last 5.13908 seconds: | 20260508 08:26:03.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:03.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.545 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:26:03.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.545 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:26:03.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:03.545 | INFO | Uploaded 36GiB to GPU in 2.33296s (15.431GiB/s) | 20260508 08:26:03.545 | INFO | Downloaded 34GiB to GPU in 2.50999s (13.5459GiB/s) | 20260508 08:26:08.684 | INFO | Analyzing chunk frames = [6658560,6659072) | 20260508 08:26:08.684 | INFO | MemInfo: { | 20260508 08:26:08.684 | INFO | "gpu" : " 17 GB", | 20260508 08:26:08.684 | INFO | "huge" : "", | 20260508 08:26:08.684 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:08.684 | INFO | "malloc" : " 14 GB", | 20260508 08:26:08.684 | INFO | "pinned" : " 29 GB" | 20260508 08:26:08.684 | INFO | } | 20260508 08:26:08.684 | INFO | Perf report for the last 5.13975 seconds: | 20260508 08:26:08.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:08.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.684 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:26:08.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.684 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:26:08.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:08.685 | INFO | Uploaded 36GiB to GPU in 2.38329s (15.1051GiB/s) | 20260508 08:26:08.685 | INFO | Downloaded 34GiB to GPU in 2.44208s (13.9226GiB/s) | 20260508 08:26:13.821 | INFO | Analyzing chunk frames = [6659072,6659584) | 20260508 08:26:13.821 | INFO | MemInfo: { | 20260508 08:26:13.821 | INFO | "gpu" : " 17 GB", | 20260508 08:26:13.821 | INFO | "huge" : "", | 20260508 08:26:13.821 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:13.821 | INFO | "malloc" : " 14 GB", | 20260508 08:26:13.821 | INFO | "pinned" : " 29 GB" | 20260508 08:26:13.821 | INFO | } | 20260508 08:26:13.821 | INFO | Perf report for the last 5.13716 seconds: | 20260508 08:26:13.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:13.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.821 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:26:13.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.822 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:26:13.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:13.822 | INFO | Uploaded 36GiB to GPU in 2.41867s (14.8842GiB/s) | 20260508 08:26:13.822 | INFO | Downloaded 34GiB to GPU in 2.43558s (13.9597GiB/s) | 20260508 08:26:18.956 | INFO | Analyzing chunk frames = [6659584,6660096) | 20260508 08:26:18.956 | INFO | MemInfo: { | 20260508 08:26:18.956 | INFO | "gpu" : " 17 GB", | 20260508 08:26:18.956 | INFO | "huge" : "", | 20260508 08:26:18.956 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:18.956 | INFO | "malloc" : " 14 GB", | 20260508 08:26:18.956 | INFO | "pinned" : " 29 GB" | 20260508 08:26:18.956 | INFO | } | 20260508 08:26:18.956 | INFO | Perf report for the last 5.13468 seconds: | 20260508 08:26:18.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:18.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:26:18.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:26:18.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:18.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:26:18.956 | INFO | Total mem usage: 16.7 GiB | 20260508 08:26:18.956 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:26:18.956 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:26:18.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:26:18.957 | INFO | High Water Breakdown: | 20260508 08:26:18.957 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:26:18.957 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:26:18.957 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:26:18.957 | INFO | ------------------------------------------ | 20260508 08:26:18.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:26:18.957 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:26:18.957 | INFO | Total mem usage: 26.5 GiB | 20260508 08:26:18.957 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:26:18.957 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:26:18.957 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:26:18.957 | INFO | High Water Breakdown: | 20260508 08:26:18.957 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:26:18.957 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:26:18.957 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:26:18.957 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:26:18.957 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:26:18.957 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:26:18.957 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:26:18.957 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:26:18.957 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:26:18.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:26:18.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:26:18.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:26:18.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:26:18.957 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:26:18.957 | INFO | ------------------------------------------ | 20260508 08:26:18.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:26:18.957 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:26:18.957 | INFO | Total mem usage: 27.8 GiB | 20260508 08:26:18.957 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:26:18.957 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:26:18.957 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:26:18.957 | INFO | High Water Breakdown: | 20260508 08:26:18.957 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:26:18.957 | INFO | ------------------------------------------ | 20260508 08:26:18.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:26:18.957 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:26:18.957 | INFO | Total mem usage: 15.7 GiB | 20260508 08:26:18.957 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:26:18.957 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:26:18.957 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:26:18.957 | INFO | High Water Breakdown: | 20260508 08:26:18.957 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:26:18.957 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:26:18.957 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:26:18.957 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:26:18.957 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:26:18.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:26:18.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:26:18.957 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:26:18.957 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:26:18.957 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:26:18.957 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:26:18.957 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:26:18.957 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:26:18.957 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:26:18.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:26:18.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:26:18.957 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:26:18.957 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:26:18.957 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:26:18.957 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:26:18.957 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:26:18.957 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:26:18.957 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:26:18.957 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:26:18.957 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:26:18.957 | INFO | ------------------------------------------ | 20260508 08:26:18.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:26:18.957 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:26:18.957 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:26:18.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:26:18.957 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.40e+01 / 7.03e-01 | 20260508 08:26:18.957 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.11e-02 / 5.68e-03 | 20260508 08:26:18.957 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.35e+00, 5.92e-01 / 2.50e-01 | 20260508 08:26:18.957 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.42e+00, 2.11e-01 / 1.34e-01 | 20260508 08:26:18.957 | INFO | PerfReport: 15.98%, DME -- 4.19e+01 / 4.54e+01, 3.67e+00 / 1.24e+00 | 20260508 08:26:18.957 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.86e+00 / 9.39e-01 | 20260508 08:26:18.957 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.08e-01 / 2.23e-01 | 20260508 08:26:18.957 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.16e+00, 2.48e-01 / 8.64e-02 | 20260508 08:26:18.957 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.44e-01, 4.92e-02 / 2.97e-02 | 20260508 08:26:18.957 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:26:18.957 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:26:18.957 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:26:18.957 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:26:18.957 | INFO | PerfReport: 24.41%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 08:26:18.957 | INFO | PerfReport: 24.45%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:26:18.957 | INFO | Uploaded 36GiB to GPU in 2.48234s (14.5024GiB/s) | 20260508 08:26:18.957 | INFO | PerfReport: 31.00%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:26:18.957 | INFO | PerfReport: 2.33%, LockUpload -- 2.46e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 08:26:18.957 | INFO | PerfReport: 2.42%, LockDownload -- 2.55e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 08:26:18.957 | INFO | PerfReport: 15.38%, LockCompute -- 1.63e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:26:18.957 | INFO | Downloaded 34GiB to GPU in 2.45436s (13.8529GiB/s) | 20260508 08:26:18.957 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 08:26:19.054 | INFO | Starting a round of Baz IO | 20260508 08:26:22.472 | INFO | Populating RTMetrics report | 20260508 08:26:22.473 | INFO | RTMetrics report populated | 20260508 08:26:22.473 | INFO | Opening tmp RTMetrics json file | 20260508 08:26:22.481 | INFO | Serializing RTMetrics to json | 20260508 08:26:22.483 | INFO | Serialization done, swapping tmp file now... | 20260508 08:26:22.483 | INFO | RTMetrics done | 20260508 08:26:22.541 | INFO | Finished a round of Baz IO: Wrote 1.505 GiB in 3.487 seconds | 20260508 08:26:22.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.58% | 20260508 08:26:22.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.296509% / 0% / 8.34465e-05% | 20260508 08:26:22.541 | INFO | Baz IO overall: | 20260508 08:26:22.541 | INFO | BytesWritten : 2922811002895 | 20260508 08:26:22.541 | INFO | NumEvents : 1407137119367 | 20260508 08:26:24.093 | INFO | Analyzing chunk frames = [6660096,6660608) | 20260508 08:26:24.093 | INFO | MemInfo: { | 20260508 08:26:24.093 | INFO | "gpu" : " 17 GB", | 20260508 08:26:24.093 | INFO | "huge" : "", | 20260508 08:26:24.093 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:24.093 | INFO | "malloc" : " 14 GB", | 20260508 08:26:24.093 | INFO | "pinned" : " 29 GB" | 20260508 08:26:24.093 | INFO | } | 20260508 08:26:24.094 | INFO | Perf report for the last 5.13739 seconds: | 20260508 08:26:24.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:24.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:24.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:24.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:24.094 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 08:26:24.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:24.094 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:26:24.094 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 08:26:24.094 | INFO | BAZWRITER : 67.2%, 2.6, 88.5ms, 0 | 20260508 08:26:24.094 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:26:24.094 | INFO | Uploaded 36GiB to GPU in 2.5781s (13.9638GiB/s) | 20260508 08:26:24.094 | INFO | Downloaded 38GiB to GPU in 2.70952s (14.0246GiB/s) | 20260508 08:26:29.233 | INFO | Analyzing chunk frames = [6660608,6661120) | 20260508 08:26:29.233 | INFO | MemInfo: { | 20260508 08:26:29.233 | INFO | "gpu" : " 17 GB", | 20260508 08:26:29.233 | INFO | "huge" : "", | 20260508 08:26:29.233 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:29.233 | INFO | "malloc" : " 14 GB", | 20260508 08:26:29.233 | INFO | "pinned" : " 29 GB" | 20260508 08:26:29.233 | INFO | } | 20260508 08:26:29.233 | INFO | Perf report for the last 5.13934 seconds: | 20260508 08:26:29.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:29.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:26:29.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:26:29.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:29.233 | INFO | Uploaded 36GiB to GPU in 2.46567s (14.6005GiB/s) | 20260508 08:26:29.233 | INFO | Downloaded 34GiB to GPU in 2.46648s (13.7848GiB/s) | 20260508 08:26:34.378 | INFO | Analyzing chunk frames = [6661120,6661632) | 20260508 08:26:34.378 | INFO | MemInfo: { | 20260508 08:26:34.378 | INFO | "gpu" : " 17 GB", | 20260508 08:26:34.378 | INFO | "huge" : "", | 20260508 08:26:34.378 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:34.378 | INFO | "malloc" : " 14 GB", | 20260508 08:26:34.378 | INFO | "pinned" : " 29 GB" | 20260508 08:26:34.378 | INFO | } | 20260508 08:26:34.378 | INFO | Perf report for the last 5.14498 seconds: | 20260508 08:26:34.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:34.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.378 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 08:26:34.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.378 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 08:26:34.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:34.379 | INFO | Uploaded 36GiB to GPU in 2.55403s (14.0954GiB/s) | 20260508 08:26:34.379 | INFO | Downloaded 34GiB to GPU in 2.44747s (13.8919GiB/s) | 20260508 08:26:39.521 | INFO | Analyzing chunk frames = [6661632,6662144) | 20260508 08:26:39.521 | INFO | MemInfo: { | 20260508 08:26:39.521 | INFO | "gpu" : " 17 GB", | 20260508 08:26:39.521 | INFO | "huge" : "", | 20260508 08:26:39.521 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:39.521 | INFO | "malloc" : " 14 GB", | 20260508 08:26:39.521 | INFO | "pinned" : " 29 GB" | 20260508 08:26:39.521 | INFO | } | 20260508 08:26:39.521 | INFO | Perf report for the last 5.14333 seconds: | 20260508 08:26:39.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:39.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 08:26:39.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:26:39.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:39.521 | INFO | Uploaded 36GiB to GPU in 2.55266s (14.1029GiB/s) | 20260508 08:26:39.522 | INFO | Downloaded 34GiB to GPU in 2.45296s (13.8608GiB/s) | 20260508 08:26:44.657 | INFO | Analyzing chunk frames = [6662144,6662656) | 20260508 08:26:44.657 | INFO | MemInfo: { | 20260508 08:26:44.657 | INFO | "gpu" : " 17 GB", | 20260508 08:26:44.657 | INFO | "huge" : "", | 20260508 08:26:44.657 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:44.658 | INFO | "malloc" : " 14 GB", | 20260508 08:26:44.658 | INFO | "pinned" : " 29 GB" | 20260508 08:26:44.658 | INFO | } | 20260508 08:26:44.658 | INFO | Perf report for the last 5.13654 seconds: | 20260508 08:26:44.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:44.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.658 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 08:26:44.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.658 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:26:44.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:44.659 | INFO | Uploaded 36GiB to GPU in 2.34165s (15.3737GiB/s) | 20260508 08:26:44.659 | INFO | Downloaded 34GiB to GPU in 2.51496s (13.5191GiB/s) | 20260508 08:26:49.787 | INFO | Analyzing chunk frames = [6662656,6663168) | 20260508 08:26:49.787 | INFO | MemInfo: { | 20260508 08:26:49.787 | INFO | "gpu" : " 17 GB", | 20260508 08:26:49.787 | INFO | "huge" : "", | 20260508 08:26:49.787 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:49.787 | INFO | "malloc" : " 14 GB", | 20260508 08:26:49.787 | INFO | "pinned" : " 29 GB" | 20260508 08:26:49.787 | INFO | } | 20260508 08:26:49.787 | INFO | Perf report for the last 5.12921 seconds: | 20260508 08:26:49.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:49.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 08:26:49.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:26:49.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:49.787 | INFO | Uploaded 36GiB to GPU in 2.35578s (15.2815GiB/s) | 20260508 08:26:49.787 | INFO | Downloaded 34GiB to GPU in 2.45634s (13.8417GiB/s) | 20260508 08:26:54.933 | INFO | Analyzing chunk frames = [6663168,6663680) | 20260508 08:26:54.933 | INFO | MemInfo: { | 20260508 08:26:54.933 | INFO | "gpu" : " 17 GB", | 20260508 08:26:54.933 | INFO | "huge" : "", | 20260508 08:26:54.933 | INFO | "huge_shared" : " 29 GB", | 20260508 08:26:54.933 | INFO | "malloc" : " 14 GB", | 20260508 08:26:54.933 | INFO | "pinned" : " 29 GB" | 20260508 08:26:54.933 | INFO | } | 20260508 08:26:54.933 | INFO | Perf report for the last 5.14624 seconds: | 20260508 08:26:54.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:26:54.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.934 | INFO | ANALYSIS : 65.8%, 3.0, 103.1ms, 0 | 20260508 08:26:54.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.934 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:26:54.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:26:54.934 | INFO | Uploaded 36GiB to GPU in 2.43786s (14.7671GiB/s) | 20260508 08:26:54.934 | INFO | Downloaded 34GiB to GPU in 2.45679s (13.8392GiB/s) | 20260508 08:27:00.075 | INFO | Analyzing chunk frames = [6663680,6664192) | 20260508 08:27:00.075 | INFO | MemInfo: { | 20260508 08:27:00.075 | INFO | "gpu" : " 17 GB", | 20260508 08:27:00.075 | INFO | "huge" : "", | 20260508 08:27:00.075 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:00.075 | INFO | "malloc" : " 14 GB", | 20260508 08:27:00.075 | INFO | "pinned" : " 29 GB" | 20260508 08:27:00.075 | INFO | } | 20260508 08:27:00.075 | INFO | Perf report for the last 5.14201 seconds: | 20260508 08:27:00.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:00.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.076 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 08:27:00.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.076 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:27:00.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:00.076 | INFO | Uploaded 36GiB to GPU in 2.41256s (14.9219GiB/s) | 20260508 08:27:00.076 | INFO | Downloaded 34GiB to GPU in 2.46667s (13.7837GiB/s) | 20260508 08:27:00.173 | INFO | Starting a round of Baz IO | 20260508 08:27:03.577 | INFO | Populating RTMetrics report | 20260508 08:27:03.577 | INFO | RTMetrics report populated | 20260508 08:27:03.577 | INFO | Opening tmp RTMetrics json file | 20260508 08:27:03.586 | INFO | Serializing RTMetrics to json | 20260508 08:27:03.590 | INFO | Serialization done, swapping tmp file now... | 20260508 08:27:03.591 | INFO | RTMetrics done | 20260508 08:27:03.649 | INFO | Finished a round of Baz IO: Wrote 1.506 GiB in 3.476 seconds | 20260508 08:27:03.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.58% | 20260508 08:27:03.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.29517% / 0% / 8.34465e-05% | 20260508 08:27:03.649 | INFO | Baz IO overall: | 20260508 08:27:03.649 | INFO | BytesWritten : 2924428103591 | 20260508 08:27:03.649 | INFO | NumEvents : 1407903584944 | 20260508 08:27:05.221 | INFO | Analyzing chunk frames = [6664192,6664704) | 20260508 08:27:05.222 | INFO | MemInfo: { | 20260508 08:27:05.222 | INFO | "gpu" : " 17 GB", | 20260508 08:27:05.222 | INFO | "huge" : "", | 20260508 08:27:05.222 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:05.222 | INFO | "malloc" : " 14 GB", | 20260508 08:27:05.222 | INFO | "pinned" : " 29 GB" | 20260508 08:27:05.222 | INFO | } | 20260508 08:27:05.222 | INFO | Perf report for the last 5.14647 seconds: | 20260508 08:27:05.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:05.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:05.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:05.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:05.222 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 08:27:05.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:05.222 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:27:05.222 | INFO | PRE_HQ : 65.2%, 1.6, 54.7ms, 0 | 20260508 08:27:05.222 | INFO | BAZWRITER : 67.0%, 2.6, 89.1ms, 0 | 20260508 08:27:05.222 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:27:05.222 | INFO | Uploaded 36GiB to GPU in 2.56912s (14.0126GiB/s) | 20260508 08:27:05.222 | INFO | Downloaded 38GiB to GPU in 2.72063s (13.9674GiB/s) | 20260508 08:27:10.352 | INFO | Analyzing chunk frames = [6664704,6665216) | 20260508 08:27:10.352 | INFO | MemInfo: { | 20260508 08:27:10.352 | INFO | "gpu" : " 17 GB", | 20260508 08:27:10.352 | INFO | "huge" : "", | 20260508 08:27:10.352 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:10.352 | INFO | "malloc" : " 14 GB", | 20260508 08:27:10.352 | INFO | "pinned" : " 29 GB" | 20260508 08:27:10.352 | INFO | } | 20260508 08:27:10.352 | INFO | Perf report for the last 5.13055 seconds: | 20260508 08:27:10.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:10.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.352 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:27:10.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.352 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:27:10.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:10.353 | INFO | Uploaded 36GiB to GPU in 2.51112s (14.3363GiB/s) | 20260508 08:27:10.353 | INFO | Downloaded 34GiB to GPU in 2.45889s (13.8274GiB/s) | 20260508 08:27:15.489 | INFO | Analyzing chunk frames = [6665216,6665728) | 20260508 08:27:15.489 | INFO | MemInfo: { | 20260508 08:27:15.489 | INFO | "gpu" : " 17 GB", | 20260508 08:27:15.489 | INFO | "huge" : "", | 20260508 08:27:15.489 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:15.489 | INFO | "malloc" : " 14 GB", | 20260508 08:27:15.490 | INFO | "pinned" : " 29 GB" | 20260508 08:27:15.490 | INFO | } | 20260508 08:27:15.490 | INFO | Perf report for the last 5.13736 seconds: | 20260508 08:27:15.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:15.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 08:27:15.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:27:15.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:15.490 | INFO | Uploaded 36GiB to GPU in 2.48737s (14.4731GiB/s) | 20260508 08:27:15.490 | INFO | Downloaded 34GiB to GPU in 2.44351s (13.9144GiB/s) | 20260508 08:27:20.632 | INFO | Analyzing chunk frames = [6665728,6666240) | 20260508 08:27:20.632 | INFO | MemInfo: { | 20260508 08:27:20.632 | INFO | "gpu" : " 17 GB", | 20260508 08:27:20.632 | INFO | "huge" : "", | 20260508 08:27:20.632 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:20.632 | INFO | "malloc" : " 14 GB", | 20260508 08:27:20.632 | INFO | "pinned" : " 29 GB" | 20260508 08:27:20.632 | INFO | } | 20260508 08:27:20.632 | INFO | Perf report for the last 5.14212 seconds: | 20260508 08:27:20.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:20.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 08:27:20.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 08:27:20.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:20.632 | INFO | Uploaded 36GiB to GPU in 2.57391s (13.9865GiB/s) | 20260508 08:27:20.632 | INFO | Downloaded 34GiB to GPU in 2.43355s (13.9714GiB/s) | 20260508 08:27:25.769 | INFO | Analyzing chunk frames = [6666240,6666752) | 20260508 08:27:25.769 | INFO | MemInfo: { | 20260508 08:27:25.769 | INFO | "gpu" : " 17 GB", | 20260508 08:27:25.769 | INFO | "huge" : "", | 20260508 08:27:25.769 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:25.769 | INFO | "malloc" : " 14 GB", | 20260508 08:27:25.769 | INFO | "pinned" : " 29 GB" | 20260508 08:27:25.769 | INFO | } | 20260508 08:27:25.769 | INFO | Perf report for the last 5.13759 seconds: | 20260508 08:27:25.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:25.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.769 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 08:27:25.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.770 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 08:27:25.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:25.770 | INFO | Uploaded 36GiB to GPU in 2.36819s (15.2015GiB/s) | 20260508 08:27:25.770 | INFO | Downloaded 34GiB to GPU in 2.4989s (13.606GiB/s) | 20260508 08:27:30.909 | INFO | Analyzing chunk frames = [6666752,6667264) | 20260508 08:27:30.909 | INFO | MemInfo: { | 20260508 08:27:30.909 | INFO | "gpu" : " 17 GB", | 20260508 08:27:30.909 | INFO | "huge" : "", | 20260508 08:27:30.909 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:30.909 | INFO | "malloc" : " 14 GB", | 20260508 08:27:30.909 | INFO | "pinned" : " 29 GB" | 20260508 08:27:30.909 | INFO | } | 20260508 08:27:30.909 | INFO | Perf report for the last 5.13996 seconds: | 20260508 08:27:30.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:30.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:27:30.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:27:30.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:30.910 | INFO | Uploaded 36GiB to GPU in 2.36735s (15.2069GiB/s) | 20260508 08:27:30.910 | INFO | Downloaded 34GiB to GPU in 2.44131s (13.9269GiB/s) | 20260508 08:27:36.044 | INFO | Analyzing chunk frames = [6667264,6667776) | 20260508 08:27:36.044 | INFO | MemInfo: { | 20260508 08:27:36.044 | INFO | "gpu" : " 17 GB", | 20260508 08:27:36.044 | INFO | "huge" : "", | 20260508 08:27:36.044 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:36.044 | INFO | "malloc" : " 14 GB", | 20260508 08:27:36.044 | INFO | "pinned" : " 29 GB" | 20260508 08:27:36.044 | INFO | } | 20260508 08:27:36.044 | INFO | Perf report for the last 5.13454 seconds: | 20260508 08:27:36.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:36.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.044 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:27:36.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.044 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:27:36.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:36.045 | INFO | Uploaded 36GiB to GPU in 2.42913s (14.8201GiB/s) | 20260508 08:27:36.045 | INFO | Downloaded 34GiB to GPU in 2.44624s (13.8989GiB/s) | 20260508 08:27:41.191 | INFO | Analyzing chunk frames = [6667776,6668288) | 20260508 08:27:41.191 | INFO | MemInfo: { | 20260508 08:27:41.191 | INFO | "gpu" : " 17 GB", | 20260508 08:27:41.191 | INFO | "huge" : "", | 20260508 08:27:41.191 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:41.191 | INFO | "malloc" : " 14 GB", | 20260508 08:27:41.191 | INFO | "pinned" : " 29 GB" | 20260508 08:27:41.191 | INFO | } | 20260508 08:27:41.191 | INFO | Perf report for the last 5.14718 seconds: | 20260508 08:27:41.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:41.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 08:27:41.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:27:41.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:41.191 | INFO | Uploaded 36GiB to GPU in 2.48017s (14.5151GiB/s) | 20260508 08:27:41.191 | INFO | Downloaded 34GiB to GPU in 2.45404s (13.8547GiB/s) | 20260508 08:27:41.191 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:27:41.191 | INFO | Total mem usage: 16.7 GiB | 20260508 08:27:41.191 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:27:41.191 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:27:41.191 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:27:41.191 | INFO | High Water Breakdown: | 20260508 08:27:41.191 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:27:41.191 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:27:41.191 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:27:41.192 | INFO | ------------------------------------------ | 20260508 08:27:41.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:27:41.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:27:41.192 | INFO | Total mem usage: 26.5 GiB | 20260508 08:27:41.192 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:27:41.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:27:41.192 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:27:41.192 | INFO | High Water Breakdown: | 20260508 08:27:41.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:27:41.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:27:41.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:27:41.192 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:27:41.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:27:41.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:27:41.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:27:41.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:27:41.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:27:41.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:27:41.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:27:41.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:27:41.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:27:41.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:27:41.192 | INFO | ------------------------------------------ | 20260508 08:27:41.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:27:41.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:27:41.192 | INFO | Total mem usage: 27.8 GiB | 20260508 08:27:41.192 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:27:41.192 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:27:41.192 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:27:41.192 | INFO | High Water Breakdown: | 20260508 08:27:41.192 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:27:41.192 | INFO | ------------------------------------------ | 20260508 08:27:41.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:27:41.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:27:41.192 | INFO | Total mem usage: 15.7 GiB | 20260508 08:27:41.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:27:41.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:27:41.192 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:27:41.192 | INFO | High Water Breakdown: | 20260508 08:27:41.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:27:41.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:27:41.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:27:41.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:27:41.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:27:41.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:27:41.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:27:41.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:27:41.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:27:41.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:27:41.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:27:41.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:27:41.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:27:41.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:27:41.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:27:41.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:27:41.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:27:41.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:27:41.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:27:41.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:27:41.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:27:41.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:27:41.192 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:27:41.192 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:27:41.192 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:27:41.192 | INFO | ------------------------------------------ | 20260508 08:27:41.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:27:41.192 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:27:41.192 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:27:41.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:27:41.192 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.79e-01 | 20260508 08:27:41.192 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.05e-01, 1.15e-02 / 5.34e-03 | 20260508 08:27:41.192 | INFO | PerfReport: 12.10%, Baseline -- 3.95e+00 / 4.36e+00, 5.86e-01 / 2.66e-01 | 20260508 08:27:41.192 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.08e-01 / 1.34e-01 | 20260508 08:27:41.192 | INFO | PerfReport: 15.97%, DME -- 4.17e+01 / 4.50e+01, 3.56e+00 / 1.25e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.82e+00 / 1.58e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.35e+00 / 2.49e+00, 4.08e-01 / 2.71e-01 | 20260508 08:27:41.192 | INFO | PerfReport: 3.08%, Metrics -- 1.01e+00 / 1.15e+00, 2.48e-01 / 1.52e-01 | 20260508 08:27:41.192 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.06e-01 / 3.37e-01, 4.83e-02 / 2.57e-02 | 20260508 08:27:41.192 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:27:41.192 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:27:41.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:27:41.192 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 24.34%, Upload -- 2.57e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 24.57%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 30.89%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 2.07%, LockUpload -- 2.19e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 2.74%, LockDownload -- 2.89e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 15.38%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:27:41.192 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260508 08:27:41.288 | INFO | Starting a round of Baz IO | 20260508 08:27:44.708 | INFO | Populating RTMetrics report | 20260508 08:27:44.708 | INFO | RTMetrics report populated | 20260508 08:27:44.708 | INFO | Opening tmp RTMetrics json file | 20260508 08:27:44.719 | INFO | Serializing RTMetrics to json | 20260508 08:27:44.720 | INFO | Serialization done, swapping tmp file now... | 20260508 08:27:44.721 | INFO | RTMetrics done | 20260508 08:27:44.777 | INFO | Finished a round of Baz IO: Wrote 1.505 GiB in 3.488 seconds | 20260508 08:27:44.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.58% | 20260508 08:27:44.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.294439% / 3.97364e-06% / 6.75519e-05% | 20260508 08:27:44.777 | INFO | Baz IO overall: | 20260508 08:27:44.777 | INFO | BytesWritten : 2926043706861 | 20260508 08:27:44.777 | INFO | NumEvents : 1408669033060 | 20260508 08:27:46.329 | INFO | Analyzing chunk frames = [6668288,6668800) | 20260508 08:27:46.329 | INFO | MemInfo: { | 20260508 08:27:46.329 | INFO | "gpu" : " 17 GB", | 20260508 08:27:46.329 | INFO | "huge" : "", | 20260508 08:27:46.329 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:46.329 | INFO | "malloc" : " 14 GB", | 20260508 08:27:46.329 | INFO | "pinned" : " 29 GB" | 20260508 08:27:46.329 | INFO | } | 20260508 08:27:46.329 | INFO | Perf report for the last 5.13835 seconds: | 20260508 08:27:46.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:46.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:46.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:46.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:46.330 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 08:27:46.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:46.330 | INFO | RT_METRICS : 39.1%, 1.0, 20.5ms, 0 | 20260508 08:27:46.330 | INFO | PRE_HQ : 65.2%, 1.6, 55.0ms, 0 | 20260508 08:27:46.330 | INFO | BAZWRITER : 67.4%, 2.6, 89.4ms, 0 | 20260508 08:27:46.330 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 08:27:46.330 | INFO | Uploaded 36GiB to GPU in 2.54489s (14.146GiB/s) | 20260508 08:27:46.330 | INFO | Downloaded 38GiB to GPU in 2.7012s (14.0678GiB/s) | 20260508 08:27:51.469 | INFO | Analyzing chunk frames = [6668800,6669312) | 20260508 08:27:51.469 | INFO | MemInfo: { | 20260508 08:27:51.469 | INFO | "gpu" : " 17 GB", | 20260508 08:27:51.469 | INFO | "huge" : "", | 20260508 08:27:51.469 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:51.469 | INFO | "malloc" : " 14 GB", | 20260508 08:27:51.469 | INFO | "pinned" : " 29 GB" | 20260508 08:27:51.470 | INFO | } | 20260508 08:27:51.470 | INFO | Perf report for the last 5.14008 seconds: | 20260508 08:27:51.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:51.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:27:51.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:27:51.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:51.470 | INFO | Uploaded 36GiB to GPU in 2.52325s (14.2673GiB/s) | 20260508 08:27:51.470 | INFO | Downloaded 34GiB to GPU in 2.45241s (13.8639GiB/s) | 20260508 08:27:56.601 | INFO | Analyzing chunk frames = [6669312,6669824) | 20260508 08:27:56.601 | INFO | MemInfo: { | 20260508 08:27:56.601 | INFO | "gpu" : " 17 GB", | 20260508 08:27:56.601 | INFO | "huge" : "", | 20260508 08:27:56.601 | INFO | "huge_shared" : " 29 GB", | 20260508 08:27:56.601 | INFO | "malloc" : " 14 GB", | 20260508 08:27:56.601 | INFO | "pinned" : " 29 GB" | 20260508 08:27:56.601 | INFO | } | 20260508 08:27:56.601 | INFO | Perf report for the last 5.13182 seconds: | 20260508 08:27:56.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:27:56.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.601 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:27:56.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.602 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 08:27:56.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:27:56.602 | INFO | Uploaded 36GiB to GPU in 2.48564s (14.4832GiB/s) | 20260508 08:27:56.602 | INFO | Downloaded 34GiB to GPU in 2.44284s (13.9182GiB/s) | 20260508 08:28:01.744 | INFO | Analyzing chunk frames = [6669824,6670336) | 20260508 08:28:01.744 | INFO | MemInfo: { | 20260508 08:28:01.744 | INFO | "gpu" : " 17 GB", | 20260508 08:28:01.744 | INFO | "huge" : "", | 20260508 08:28:01.744 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:01.744 | INFO | "malloc" : " 14 GB", | 20260508 08:28:01.744 | INFO | "pinned" : " 29 GB" | 20260508 08:28:01.744 | INFO | } | 20260508 08:28:01.744 | INFO | Perf report for the last 5.14258 seconds: | 20260508 08:28:01.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:01.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.744 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 08:28:01.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.744 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:28:01.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:01.745 | INFO | Uploaded 36GiB to GPU in 2.57703s (13.9696GiB/s) | 20260508 08:28:01.745 | INFO | Downloaded 34GiB to GPU in 2.43053s (13.9887GiB/s) | 20260508 08:28:06.886 | INFO | Analyzing chunk frames = [6670336,6670848) | 20260508 08:28:06.886 | INFO | MemInfo: { | 20260508 08:28:06.886 | INFO | "gpu" : " 17 GB", | 20260508 08:28:06.886 | INFO | "huge" : "", | 20260508 08:28:06.886 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:06.886 | INFO | "malloc" : " 14 GB", | 20260508 08:28:06.886 | INFO | "pinned" : " 29 GB" | 20260508 08:28:06.886 | INFO | } | 20260508 08:28:06.886 | INFO | Perf report for the last 5.14205 seconds: | 20260508 08:28:06.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:06.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.886 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 08:28:06.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.886 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 08:28:06.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:06.887 | INFO | Uploaded 36GiB to GPU in 2.38377s (15.1021GiB/s) | 20260508 08:28:06.887 | INFO | Downloaded 34GiB to GPU in 2.54298s (13.3701GiB/s) | 20260508 08:28:12.026 | INFO | Analyzing chunk frames = [6670848,6671360) | 20260508 08:28:12.026 | INFO | MemInfo: { | 20260508 08:28:12.026 | INFO | "gpu" : " 17 GB", | 20260508 08:28:12.026 | INFO | "huge" : "", | 20260508 08:28:12.026 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:12.026 | INFO | "malloc" : " 14 GB", | 20260508 08:28:12.026 | INFO | "pinned" : " 29 GB" | 20260508 08:28:12.026 | INFO | } | 20260508 08:28:12.026 | INFO | Perf report for the last 5.13983 seconds: | 20260508 08:28:12.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:12.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:28:12.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:28:12.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:12.026 | INFO | Uploaded 36GiB to GPU in 2.40207s (14.9871GiB/s) | 20260508 08:28:12.026 | INFO | Downloaded 34GiB to GPU in 2.42294s (14.0325GiB/s) | 20260508 08:28:17.167 | INFO | Analyzing chunk frames = [6671360,6671872) | 20260508 08:28:17.167 | INFO | MemInfo: { | 20260508 08:28:17.167 | INFO | "gpu" : " 17 GB", | 20260508 08:28:17.167 | INFO | "huge" : "", | 20260508 08:28:17.167 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:17.167 | INFO | "malloc" : " 14 GB", | 20260508 08:28:17.167 | INFO | "pinned" : " 29 GB" | 20260508 08:28:17.167 | INFO | } | 20260508 08:28:17.167 | INFO | Perf report for the last 5.14154 seconds: | 20260508 08:28:17.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:17.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.168 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:28:17.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.168 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:28:17.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:17.168 | INFO | Uploaded 36GiB to GPU in 2.45027s (14.6923GiB/s) | 20260508 08:28:17.168 | INFO | Downloaded 34GiB to GPU in 2.43305s (13.9742GiB/s) | 20260508 08:28:22.301 | INFO | Analyzing chunk frames = [6671872,6672384) | 20260508 08:28:22.301 | INFO | MemInfo: { | 20260508 08:28:22.301 | INFO | "gpu" : " 17 GB", | 20260508 08:28:22.301 | INFO | "huge" : "", | 20260508 08:28:22.301 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:22.301 | INFO | "malloc" : " 14 GB", | 20260508 08:28:22.302 | INFO | "pinned" : " 29 GB" | 20260508 08:28:22.302 | INFO | } | 20260508 08:28:22.302 | INFO | Perf report for the last 5.13419 seconds: | 20260508 08:28:22.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:22.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 08:28:22.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:28:22.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:22.302 | INFO | Uploaded 36GiB to GPU in 2.46565s (14.6006GiB/s) | 20260508 08:28:22.302 | INFO | Downloaded 34GiB to GPU in 2.4459s (13.9008GiB/s) | 20260508 08:28:22.399 | INFO | Starting a round of Baz IO | 20260508 08:28:25.818 | INFO | Populating RTMetrics report | 20260508 08:28:25.818 | INFO | RTMetrics report populated | 20260508 08:28:25.818 | INFO | Opening tmp RTMetrics json file | 20260508 08:28:25.829 | INFO | Serializing RTMetrics to json | 20260508 08:28:25.830 | INFO | Serialization done, swapping tmp file now... | 20260508 08:28:25.830 | INFO | RTMetrics done | 20260508 08:28:25.884 | INFO | Finished a round of Baz IO: Wrote 1.504 GiB in 3.485 seconds | 20260508 08:28:25.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.58% | 20260508 08:28:25.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.293744% / 0% / 7.15256e-05% | 20260508 08:28:25.884 | INFO | Baz IO overall: | 20260508 08:28:25.884 | INFO | BytesWritten : 2927658276896 | 20260508 08:28:25.884 | INFO | NumEvents : 1409433758449 | 20260508 08:28:27.445 | INFO | Analyzing chunk frames = [6672384,6672896) | 20260508 08:28:27.445 | INFO | MemInfo: { | 20260508 08:28:27.445 | INFO | "gpu" : " 17 GB", | 20260508 08:28:27.445 | INFO | "huge" : "", | 20260508 08:28:27.446 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:27.446 | INFO | "malloc" : " 14 GB", | 20260508 08:28:27.446 | INFO | "pinned" : " 29 GB" | 20260508 08:28:27.446 | INFO | } | 20260508 08:28:27.446 | INFO | Perf report for the last 5.14394 seconds: | 20260508 08:28:27.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:27.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:27.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:27.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:27.446 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 08:28:27.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:27.446 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:28:27.446 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 08:28:27.446 | INFO | BAZWRITER : 67.0%, 2.6, 89.3ms, 0 | 20260508 08:28:27.446 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:28:27.446 | INFO | Uploaded 36GiB to GPU in 2.57239s (13.9948GiB/s) | 20260508 08:28:27.446 | INFO | Downloaded 38GiB to GPU in 2.71591s (13.9916GiB/s) | 20260508 08:28:32.586 | INFO | Analyzing chunk frames = [6672896,6673408) | 20260508 08:28:32.586 | INFO | MemInfo: { | 20260508 08:28:32.586 | INFO | "gpu" : " 17 GB", | 20260508 08:28:32.586 | INFO | "huge" : "", | 20260508 08:28:32.586 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:32.586 | INFO | "malloc" : " 14 GB", | 20260508 08:28:32.586 | INFO | "pinned" : " 29 GB" | 20260508 08:28:32.586 | INFO | } | 20260508 08:28:32.586 | INFO | Perf report for the last 5.14064 seconds: | 20260508 08:28:32.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:32.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.586 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:28:32.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.586 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:28:32.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:32.587 | INFO | Uploaded 36GiB to GPU in 2.50344s (14.3802GiB/s) | 20260508 08:28:32.587 | INFO | Downloaded 34GiB to GPU in 2.46357s (13.8011GiB/s) | 20260508 08:28:37.723 | INFO | Analyzing chunk frames = [6673408,6673920) | 20260508 08:28:37.723 | INFO | MemInfo: { | 20260508 08:28:37.723 | INFO | "gpu" : " 17 GB", | 20260508 08:28:37.723 | INFO | "huge" : "", | 20260508 08:28:37.723 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:37.723 | INFO | "malloc" : " 14 GB", | 20260508 08:28:37.723 | INFO | "pinned" : " 29 GB" | 20260508 08:28:37.723 | INFO | } | 20260508 08:28:37.723 | INFO | Perf report for the last 5.13662 seconds: | 20260508 08:28:37.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:37.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.723 | INFO | ANALYSIS : 67.3%, 3.0, 104.5ms, 0 | 20260508 08:28:37.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.723 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:28:37.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:37.724 | INFO | Uploaded 36GiB to GPU in 2.56584s (14.0305GiB/s) | 20260508 08:28:37.724 | INFO | Downloaded 34GiB to GPU in 2.44982s (13.8786GiB/s) | 20260508 08:28:42.856 | INFO | Analyzing chunk frames = [6673920,6674432) | 20260508 08:28:42.856 | INFO | MemInfo: { | 20260508 08:28:42.856 | INFO | "gpu" : " 17 GB", | 20260508 08:28:42.856 | INFO | "huge" : "", | 20260508 08:28:42.856 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:42.856 | INFO | "malloc" : " 14 GB", | 20260508 08:28:42.856 | INFO | "pinned" : " 29 GB" | 20260508 08:28:42.856 | INFO | } | 20260508 08:28:42.857 | INFO | Perf report for the last 5.13374 seconds: | 20260508 08:28:42.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:42.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | ANALYSIS : 66.9%, 2.9, 103.4ms, 0 | 20260508 08:28:42.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:28:42.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:42.857 | INFO | Uploaded 36GiB to GPU in 2.55456s (14.0924GiB/s) | 20260508 08:28:42.857 | INFO | Downloaded 34GiB to GPU in 2.44788s (13.8895GiB/s) | 20260508 08:28:47.998 | INFO | Analyzing chunk frames = [6674432,6674944) | 20260508 08:28:47.998 | INFO | MemInfo: { | 20260508 08:28:47.998 | INFO | "gpu" : " 17 GB", | 20260508 08:28:47.998 | INFO | "huge" : "", | 20260508 08:28:47.998 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:47.998 | INFO | "malloc" : " 14 GB", | 20260508 08:28:47.998 | INFO | "pinned" : " 29 GB" | 20260508 08:28:47.998 | INFO | } | 20260508 08:28:47.998 | INFO | Perf report for the last 5.14096 seconds: | 20260508 08:28:47.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:47.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 08:28:47.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:28:47.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:47.998 | INFO | Uploaded 36GiB to GPU in 2.40502s (14.9687GiB/s) | 20260508 08:28:47.998 | INFO | Downloaded 34GiB to GPU in 2.5446s (13.3616GiB/s) | 20260508 08:28:53.138 | INFO | Analyzing chunk frames = [6674944,6675456) | 20260508 08:28:53.138 | INFO | MemInfo: { | 20260508 08:28:53.138 | INFO | "gpu" : " 17 GB", | 20260508 08:28:53.138 | INFO | "huge" : "", | 20260508 08:28:53.138 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:53.138 | INFO | "malloc" : " 14 GB", | 20260508 08:28:53.138 | INFO | "pinned" : " 29 GB" | 20260508 08:28:53.138 | INFO | } | 20260508 08:28:53.139 | INFO | Perf report for the last 5.14088 seconds: | 20260508 08:28:53.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:53.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 08:28:53.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:28:53.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:53.139 | INFO | Uploaded 36GiB to GPU in 2.4495s (14.6969GiB/s) | 20260508 08:28:53.139 | INFO | Downloaded 34GiB to GPU in 2.4572s (13.8369GiB/s) | 20260508 08:28:58.281 | INFO | Analyzing chunk frames = [6675456,6675968) | 20260508 08:28:58.281 | INFO | MemInfo: { | 20260508 08:28:58.281 | INFO | "gpu" : " 17 GB", | 20260508 08:28:58.281 | INFO | "huge" : "", | 20260508 08:28:58.281 | INFO | "huge_shared" : " 29 GB", | 20260508 08:28:58.281 | INFO | "malloc" : " 14 GB", | 20260508 08:28:58.281 | INFO | "pinned" : " 29 GB" | 20260508 08:28:58.281 | INFO | } | 20260508 08:28:58.281 | INFO | Perf report for the last 5.14205 seconds: | 20260508 08:28:58.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:28:58.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:28:58.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:28:58.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:28:58.281 | INFO | Uploaded 36GiB to GPU in 2.44355s (14.7327GiB/s) | 20260508 08:28:58.281 | INFO | Downloaded 34GiB to GPU in 2.45599s (13.8437GiB/s) | 20260508 08:29:03.418 | INFO | Analyzing chunk frames = [6675968,6676480) | 20260508 08:29:03.418 | INFO | MemInfo: { | 20260508 08:29:03.418 | INFO | "gpu" : " 17 GB", | 20260508 08:29:03.418 | INFO | "huge" : "", | 20260508 08:29:03.418 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:03.418 | INFO | "malloc" : " 14 GB", | 20260508 08:29:03.418 | INFO | "pinned" : " 29 GB" | 20260508 08:29:03.418 | INFO | } | 20260508 08:29:03.418 | INFO | Perf report for the last 5.13747 seconds: | 20260508 08:29:03.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:03.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.418 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 08:29:03.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.418 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 08:29:03.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:03.419 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:29:03.419 | INFO | Total mem usage: 16.7 GiB | 20260508 08:29:03.419 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:29:03.419 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 08:29:03.419 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:29:03.419 | INFO | High Water Breakdown: | 20260508 08:29:03.419 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:29:03.419 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:29:03.419 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:29:03.419 | INFO | ------------------------------------------ | 20260508 08:29:03.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:29:03.419 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:29:03.419 | INFO | Total mem usage: 26.5 GiB | 20260508 08:29:03.419 | INFO | Uploaded 36GiB to GPU in 2.38742s (15.079GiB/s) | 20260508 08:29:03.419 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:29:03.419 | INFO | Downloaded 34GiB to GPU in 2.4606s (13.8178GiB/s) | 20260508 08:29:03.419 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:29:03.419 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:29:03.419 | INFO | High Water Breakdown: | 20260508 08:29:03.419 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:29:03.419 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:29:03.419 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:29:03.419 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:29:03.419 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:29:03.419 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:29:03.419 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:29:03.419 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:29:03.419 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:29:03.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:29:03.419 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:29:03.419 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:29:03.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:29:03.419 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:29:03.419 | INFO | ------------------------------------------ | 20260508 08:29:03.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:29:03.419 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:29:03.419 | INFO | Total mem usage: 27.8 GiB | 20260508 08:29:03.419 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:29:03.419 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:29:03.419 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:29:03.419 | INFO | High Water Breakdown: | 20260508 08:29:03.419 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:29:03.419 | INFO | ------------------------------------------ | 20260508 08:29:03.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:29:03.419 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:29:03.419 | INFO | Total mem usage: 15.7 GiB | 20260508 08:29:03.419 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:29:03.419 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:29:03.419 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:29:03.419 | INFO | High Water Breakdown: | 20260508 08:29:03.419 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:29:03.420 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:29:03.420 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:29:03.420 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:29:03.420 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:29:03.420 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:29:03.420 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:29:03.420 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:29:03.420 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:29:03.420 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:29:03.420 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:29:03.420 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:29:03.420 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:29:03.420 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:29:03.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:29:03.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:29:03.420 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:29:03.420 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:29:03.420 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:29:03.420 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:29:03.420 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:29:03.420 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:29:03.420 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:29:03.420 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:29:03.420 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:29:03.420 | INFO | ------------------------------------------ | 20260508 08:29:03.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:29:03.420 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:29:03.420 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:29:03.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:29:03.420 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 5.20e-01 | 20260508 08:29:03.420 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.07e-01, 1.19e-02 / 1.77e-02 | 20260508 08:29:03.420 | INFO | PerfReport: 12.09%, Baseline -- 3.96e+00 / 4.45e+00, 5.89e-01 / 2.75e-01 | 20260508 08:29:03.420 | INFO | PerfReport: 3.35%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.27e-01 | 20260508 08:29:03.420 | INFO | PerfReport: 15.89%, DME -- 4.16e+01 / 4.49e+01, 3.77e+00 / 1.08e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.91e+00 / 1.32e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.52e+00, 4.07e-01 / 2.29e-01 | 20260508 08:29:03.420 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.18e+00, 2.51e-01 / 1.21e-01 | 20260508 08:29:03.420 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.48e-01, 4.99e-02 / 2.75e-02 | 20260508 08:29:03.420 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:29:03.420 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:29:03.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:29:03.420 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 24.44%, Upload -- 2.59e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 24.54%, Download -- 2.60e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 30.93%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 2.17%, LockUpload -- 2.29e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 2.60%, LockDownload -- 2.75e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 15.30%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:29:03.420 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260508 08:29:03.532 | INFO | Starting a round of Baz IO | 20260508 08:29:06.945 | INFO | Populating RTMetrics report | 20260508 08:29:06.945 | INFO | RTMetrics report populated | 20260508 08:29:06.945 | INFO | Opening tmp RTMetrics json file | 20260508 08:29:06.955 | INFO | Serializing RTMetrics to json | 20260508 08:29:06.956 | INFO | Serialization done, swapping tmp file now... | 20260508 08:29:06.957 | INFO | RTMetrics done | 20260508 08:29:07.011 | INFO | Finished a round of Baz IO: Wrote 1.503 GiB in 3.479 seconds | 20260508 08:29:07.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.58% | 20260508 08:29:07.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.29345% / 0% / 7.15256e-05% | 20260508 08:29:07.011 | INFO | Baz IO overall: | 20260508 08:29:07.011 | INFO | BytesWritten : 2929272494980 | 20260508 08:29:07.011 | INFO | NumEvents : 1410198332235 | 20260508 08:29:08.554 | INFO | Analyzing chunk frames = [6676480,6676992) | 20260508 08:29:08.554 | INFO | MemInfo: { | 20260508 08:29:08.554 | INFO | "gpu" : " 17 GB", | 20260508 08:29:08.554 | INFO | "huge" : "", | 20260508 08:29:08.554 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:08.554 | INFO | "malloc" : " 14 GB", | 20260508 08:29:08.554 | INFO | "pinned" : " 29 GB" | 20260508 08:29:08.554 | INFO | } | 20260508 08:29:08.554 | INFO | Perf report for the last 5.13577 seconds: | 20260508 08:29:08.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:08.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:08.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:08.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:08.554 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 08:29:08.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:08.554 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 08:29:08.554 | INFO | PRE_HQ : 66.5%, 1.8, 61.2ms, 0 | 20260508 08:29:08.554 | INFO | BAZWRITER : 67.2%, 2.8, 94.9ms, 0 | 20260508 08:29:08.554 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:29:08.554 | INFO | Uploaded 36GiB to GPU in 2.58079s (13.9492GiB/s) | 20260508 08:29:08.554 | INFO | Downloaded 38GiB to GPU in 2.71518s (13.9954GiB/s) | 20260508 08:29:13.693 | INFO | Analyzing chunk frames = [6676992,6677504) | 20260508 08:29:13.693 | INFO | MemInfo: { | 20260508 08:29:13.693 | INFO | "gpu" : " 17 GB", | 20260508 08:29:13.693 | INFO | "huge" : "", | 20260508 08:29:13.693 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:13.693 | INFO | "malloc" : " 14 GB", | 20260508 08:29:13.693 | INFO | "pinned" : " 29 GB" | 20260508 08:29:13.694 | INFO | } | 20260508 08:29:13.694 | INFO | Perf report for the last 5.13983 seconds: | 20260508 08:29:13.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:13.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:29:13.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:29:13.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:13.694 | INFO | Uploaded 36GiB to GPU in 2.47504s (14.5452GiB/s) | 20260508 08:29:13.694 | INFO | Downloaded 34GiB to GPU in 2.46335s (13.8024GiB/s) | 20260508 08:29:18.838 | INFO | Analyzing chunk frames = [6677504,6678016) | 20260508 08:29:18.838 | INFO | MemInfo: { | 20260508 08:29:18.838 | INFO | "gpu" : " 17 GB", | 20260508 08:29:18.838 | INFO | "huge" : "", | 20260508 08:29:18.838 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:18.838 | INFO | "malloc" : " 14 GB", | 20260508 08:29:18.838 | INFO | "pinned" : " 29 GB" | 20260508 08:29:18.838 | INFO | } | 20260508 08:29:18.838 | INFO | Perf report for the last 5.14408 seconds: | 20260508 08:29:18.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:18.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 08:29:18.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:29:18.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:18.838 | INFO | Uploaded 36GiB to GPU in 2.53463s (14.2033GiB/s) | 20260508 08:29:18.839 | INFO | Downloaded 34GiB to GPU in 2.44962s (13.8797GiB/s) | 20260508 08:29:23.980 | INFO | Analyzing chunk frames = [6678016,6678528) | 20260508 08:29:23.980 | INFO | MemInfo: { | 20260508 08:29:23.980 | INFO | "gpu" : " 17 GB", | 20260508 08:29:23.980 | INFO | "huge" : "", | 20260508 08:29:23.980 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:23.980 | INFO | "malloc" : " 14 GB", | 20260508 08:29:23.980 | INFO | "pinned" : " 29 GB" | 20260508 08:29:23.980 | INFO | } | 20260508 08:29:23.980 | INFO | Perf report for the last 5.14236 seconds: | 20260508 08:29:23.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:23.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.980 | INFO | ANALYSIS : 66.7%, 3.0, 103.2ms, 0 | 20260508 08:29:23.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.980 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:29:23.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:23.981 | INFO | Uploaded 36GiB to GPU in 2.54359s (14.1532GiB/s) | 20260508 08:29:23.981 | INFO | Downloaded 34GiB to GPU in 2.45036s (13.8755GiB/s) | 20260508 08:29:29.114 | INFO | Analyzing chunk frames = [6678528,6679040) | 20260508 08:29:29.114 | INFO | MemInfo: { | 20260508 08:29:29.114 | INFO | "gpu" : " 17 GB", | 20260508 08:29:29.114 | INFO | "huge" : "", | 20260508 08:29:29.114 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:29.114 | INFO | "malloc" : " 14 GB", | 20260508 08:29:29.114 | INFO | "pinned" : " 29 GB" | 20260508 08:29:29.115 | INFO | } | 20260508 08:29:29.115 | INFO | Perf report for the last 5.13481 seconds: | 20260508 08:29:29.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:29.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 08:29:29.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:29:29.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:29.115 | INFO | Uploaded 36GiB to GPU in 2.36073s (15.2495GiB/s) | 20260508 08:29:29.115 | INFO | Downloaded 34GiB to GPU in 2.57595s (13.199GiB/s) | 20260508 08:29:34.254 | INFO | Analyzing chunk frames = [6679040,6679552) | 20260508 08:29:34.254 | INFO | MemInfo: { | 20260508 08:29:34.254 | INFO | "gpu" : " 17 GB", | 20260508 08:29:34.254 | INFO | "huge" : "", | 20260508 08:29:34.254 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:34.254 | INFO | "malloc" : " 14 GB", | 20260508 08:29:34.255 | INFO | "pinned" : " 29 GB" | 20260508 08:29:34.255 | INFO | } | 20260508 08:29:34.255 | INFO | Perf report for the last 5.13972 seconds: | 20260508 08:29:34.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:34.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 08:29:34.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:29:34.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:34.255 | INFO | Uploaded 36GiB to GPU in 2.35715s (15.2727GiB/s) | 20260508 08:29:34.255 | INFO | Downloaded 34GiB to GPU in 2.43634s (13.9554GiB/s) | 20260508 08:29:39.394 | INFO | Analyzing chunk frames = [6679552,6680064) | 20260508 08:29:39.394 | INFO | MemInfo: { | 20260508 08:29:39.394 | INFO | "gpu" : " 17 GB", | 20260508 08:29:39.394 | INFO | "huge" : "", | 20260508 08:29:39.394 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:39.394 | INFO | "malloc" : " 14 GB", | 20260508 08:29:39.394 | INFO | "pinned" : " 29 GB" | 20260508 08:29:39.394 | INFO | } | 20260508 08:29:39.394 | INFO | Perf report for the last 5.13928 seconds: | 20260508 08:29:39.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:39.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 08:29:39.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 08:29:39.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:39.394 | INFO | Uploaded 36GiB to GPU in 2.40271s (14.9831GiB/s) | 20260508 08:29:39.395 | INFO | Downloaded 34GiB to GPU in 2.44146s (13.9261GiB/s) | 20260508 08:29:44.534 | INFO | Analyzing chunk frames = [6680064,6680576) | 20260508 08:29:44.534 | INFO | MemInfo: { | 20260508 08:29:44.534 | INFO | "gpu" : " 17 GB", | 20260508 08:29:44.534 | INFO | "huge" : "", | 20260508 08:29:44.534 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:44.534 | INFO | "malloc" : " 14 GB", | 20260508 08:29:44.534 | INFO | "pinned" : " 29 GB" | 20260508 08:29:44.534 | INFO | } | 20260508 08:29:44.534 | INFO | Perf report for the last 5.14027 seconds: | 20260508 08:29:44.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:44.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.535 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 08:29:44.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.535 | INFO | PRE_HQ : 33.8%, 1.0, 17.5ms, 0 | 20260508 08:29:44.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:44.535 | INFO | Uploaded 36GiB to GPU in 2.43217s (14.8016GiB/s) | 20260508 08:29:44.535 | INFO | Downloaded 34GiB to GPU in 2.45826s (13.8309GiB/s) | 20260508 08:29:44.649 | INFO | Starting a round of Baz IO | 20260508 08:29:48.051 | INFO | Populating RTMetrics report | 20260508 08:29:48.052 | INFO | RTMetrics report populated | 20260508 08:29:48.052 | INFO | Opening tmp RTMetrics json file | 20260508 08:29:48.059 | INFO | Serializing RTMetrics to json | 20260508 08:29:48.060 | INFO | Serialization done, swapping tmp file now... | 20260508 08:29:48.061 | INFO | RTMetrics done | 20260508 08:29:48.120 | INFO | Finished a round of Baz IO: Wrote 1.503 GiB in 3.471 seconds | 20260508 08:29:48.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.00% / 59.58% | 20260508 08:29:48.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.292683% / 3.97364e-06% / 9.13938e-05% | 20260508 08:29:48.120 | INFO | Baz IO overall: | 20260508 08:29:48.120 | INFO | BytesWritten : 2930886051172 | 20260508 08:29:48.121 | INFO | NumEvents : 1410962439419 | 20260508 08:29:49.674 | INFO | Analyzing chunk frames = [6680576,6681088) | 20260508 08:29:49.674 | INFO | MemInfo: { | 20260508 08:29:49.674 | INFO | "gpu" : " 17 GB", | 20260508 08:29:49.674 | INFO | "huge" : "", | 20260508 08:29:49.674 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:49.674 | INFO | "malloc" : " 14 GB", | 20260508 08:29:49.674 | INFO | "pinned" : " 29 GB" | 20260508 08:29:49.674 | INFO | } | 20260508 08:29:49.674 | INFO | Perf report for the last 5.13973 seconds: | 20260508 08:29:49.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:49.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:49.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:49.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:49.674 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 08:29:49.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:49.674 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 08:29:49.674 | INFO | PRE_HQ : 66.3%, 1.8, 61.8ms, 0 | 20260508 08:29:49.674 | INFO | BAZWRITER : 67.0%, 2.8, 96.1ms, 0 | 20260508 08:29:49.674 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:29:49.675 | INFO | Uploaded 36GiB to GPU in 2.60946s (13.7959GiB/s) | 20260508 08:29:49.675 | INFO | Downloaded 38GiB to GPU in 2.69816s (14.0836GiB/s) | 20260508 08:29:54.806 | INFO | Analyzing chunk frames = [6681088,6681600) | 20260508 08:29:54.806 | INFO | MemInfo: { | 20260508 08:29:54.806 | INFO | "gpu" : " 17 GB", | 20260508 08:29:54.806 | INFO | "huge" : "", | 20260508 08:29:54.807 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:54.807 | INFO | "malloc" : " 14 GB", | 20260508 08:29:54.807 | INFO | "pinned" : " 29 GB" | 20260508 08:29:54.807 | INFO | } | 20260508 08:29:54.807 | INFO | Perf report for the last 5.13269 seconds: | 20260508 08:29:54.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:54.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 08:29:54.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:29:54.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:54.807 | INFO | Uploaded 36GiB to GPU in 2.54964s (14.1196GiB/s) | 20260508 08:29:54.807 | INFO | Downloaded 34GiB to GPU in 2.4467s (13.8963GiB/s) | 20260508 08:29:59.958 | INFO | Analyzing chunk frames = [6681600,6682112) | 20260508 08:29:59.958 | INFO | MemInfo: { | 20260508 08:29:59.958 | INFO | "gpu" : " 17 GB", | 20260508 08:29:59.958 | INFO | "huge" : "", | 20260508 08:29:59.958 | INFO | "huge_shared" : " 29 GB", | 20260508 08:29:59.958 | INFO | "malloc" : " 14 GB", | 20260508 08:29:59.958 | INFO | "pinned" : " 29 GB" | 20260508 08:29:59.958 | INFO | } | 20260508 08:29:59.958 | INFO | Perf report for the last 5.15163 seconds: | 20260508 08:29:59.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:29:59.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.958 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 08:29:59.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.958 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:29:59.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:29:59.959 | INFO | Uploaded 36GiB to GPU in 2.5181s (14.2965GiB/s) | 20260508 08:29:59.959 | INFO | Downloaded 34GiB to GPU in 2.43018s (13.9907GiB/s) | 20260508 08:30:05.092 | INFO | Analyzing chunk frames = [6682112,6682624) | 20260508 08:30:05.092 | INFO | MemInfo: { | 20260508 08:30:05.092 | INFO | "gpu" : " 17 GB", | 20260508 08:30:05.092 | INFO | "huge" : "", | 20260508 08:30:05.092 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:05.092 | INFO | "malloc" : " 14 GB", | 20260508 08:30:05.092 | INFO | "pinned" : " 29 GB" | 20260508 08:30:05.092 | INFO | } | 20260508 08:30:05.092 | INFO | Perf report for the last 5.13379 seconds: | 20260508 08:30:05.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:05.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 08:30:05.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:30:05.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:05.092 | INFO | Uploaded 36GiB to GPU in 2.49516s (14.4279GiB/s) | 20260508 08:30:05.092 | INFO | Downloaded 34GiB to GPU in 2.43243s (13.9778GiB/s) | 20260508 08:30:10.226 | INFO | Analyzing chunk frames = [6682624,6683136) | 20260508 08:30:10.226 | INFO | MemInfo: { | 20260508 08:30:10.226 | INFO | "gpu" : " 17 GB", | 20260508 08:30:10.226 | INFO | "huge" : "", | 20260508 08:30:10.226 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:10.226 | INFO | "malloc" : " 14 GB", | 20260508 08:30:10.226 | INFO | "pinned" : " 29 GB" | 20260508 08:30:10.226 | INFO | } | 20260508 08:30:10.226 | INFO | Perf report for the last 5.13373 seconds: | 20260508 08:30:10.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:10.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 08:30:10.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:30:10.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:10.226 | INFO | Uploaded 36GiB to GPU in 2.30177s (15.6401GiB/s) | 20260508 08:30:10.226 | INFO | Downloaded 34GiB to GPU in 2.49837s (13.6089GiB/s) | 20260508 08:30:15.362 | INFO | Analyzing chunk frames = [6683136,6683648) | 20260508 08:30:15.362 | INFO | MemInfo: { | 20260508 08:30:15.362 | INFO | "gpu" : " 17 GB", | 20260508 08:30:15.362 | INFO | "huge" : "", | 20260508 08:30:15.362 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:15.362 | INFO | "malloc" : " 14 GB", | 20260508 08:30:15.362 | INFO | "pinned" : " 29 GB" | 20260508 08:30:15.362 | INFO | } | 20260508 08:30:15.362 | INFO | Perf report for the last 5.1364 seconds: | 20260508 08:30:15.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:15.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.362 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 08:30:15.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.363 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:30:15.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:15.363 | INFO | Uploaded 36GiB to GPU in 2.39092s (15.057GiB/s) | 20260508 08:30:15.363 | INFO | Downloaded 34GiB to GPU in 2.43427s (13.9672GiB/s) | 20260508 08:30:20.508 | INFO | Analyzing chunk frames = [6683648,6684160) | 20260508 08:30:20.508 | INFO | MemInfo: { | 20260508 08:30:20.508 | INFO | "gpu" : " 17 GB", | 20260508 08:30:20.508 | INFO | "huge" : "", | 20260508 08:30:20.508 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:20.508 | INFO | "malloc" : " 14 GB", | 20260508 08:30:20.508 | INFO | "pinned" : " 29 GB" | 20260508 08:30:20.508 | INFO | } | 20260508 08:30:20.509 | INFO | Perf report for the last 5.1461 seconds: | 20260508 08:30:20.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:20.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 08:30:20.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:30:20.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:20.509 | INFO | Uploaded 36GiB to GPU in 2.41874s (14.8838GiB/s) | 20260508 08:30:20.509 | INFO | Downloaded 34GiB to GPU in 2.44684s (13.8955GiB/s) | 20260508 08:30:25.647 | INFO | Analyzing chunk frames = [6684160,6684672) | 20260508 08:30:25.647 | INFO | MemInfo: { | 20260508 08:30:25.647 | INFO | "gpu" : " 17 GB", | 20260508 08:30:25.647 | INFO | "huge" : "", | 20260508 08:30:25.647 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:25.647 | INFO | "malloc" : " 14 GB", | 20260508 08:30:25.647 | INFO | "pinned" : " 29 GB" | 20260508 08:30:25.647 | INFO | } | 20260508 08:30:25.647 | INFO | Perf report for the last 5.13885 seconds: | 20260508 08:30:25.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:25.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.647 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 08:30:25.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.648 | INFO | PRE_HQ : 34.0%, 1.0, 17.6ms, 0 | 20260508 08:30:25.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:25.648 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:30:25.648 | INFO | Total mem usage: 16.7 GiB | 20260508 08:30:25.648 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:30:25.648 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 08:30:25.648 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:30:25.648 | INFO | High Water Breakdown: | 20260508 08:30:25.648 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:30:25.648 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:30:25.648 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:30:25.648 | INFO | ------------------------------------------ | 20260508 08:30:25.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:30:25.648 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:30:25.648 | INFO | Total mem usage: 26.5 GiB | 20260508 08:30:25.648 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:30:25.648 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 08:30:25.648 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 08:30:25.648 | INFO | High Water Breakdown: | 20260508 08:30:25.648 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:30:25.648 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:30:25.648 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:30:25.648 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:30:25.648 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:30:25.648 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:30:25.648 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:30:25.648 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:30:25.648 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:30:25.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:30:25.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:30:25.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:30:25.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:30:25.648 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:30:25.648 | INFO | ------------------------------------------ | 20260508 08:30:25.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:30:25.648 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:30:25.648 | INFO | Total mem usage: 27.8 GiB | 20260508 08:30:25.648 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:30:25.648 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:30:25.648 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:30:25.648 | INFO | High Water Breakdown: | 20260508 08:30:25.648 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:30:25.648 | INFO | ------------------------------------------ | 20260508 08:30:25.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:30:25.648 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:30:25.648 | INFO | Uploaded 36GiB to GPU in 2.43783s (14.7672GiB/s) | 20260508 08:30:25.648 | INFO | Total mem usage: 15.7 GiB | 20260508 08:30:25.648 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:30:25.648 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:30:25.648 | INFO | Downloaded 34GiB to GPU in 2.44819s (13.8878GiB/s) | 20260508 08:30:25.648 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:30:25.649 | INFO | High Water Breakdown: | 20260508 08:30:25.649 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:30:25.649 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:30:25.649 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:30:25.649 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:30:25.649 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:30:25.649 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:30:25.649 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:30:25.649 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:30:25.649 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:30:25.649 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:30:25.649 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:30:25.649 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:30:25.649 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:30:25.649 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:30:25.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:30:25.649 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:30:25.649 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:30:25.649 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:30:25.649 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:30:25.649 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:30:25.649 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:30:25.649 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:30:25.649 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:30:25.649 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:30:25.649 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:30:25.649 | INFO | ------------------------------------------ | 20260508 08:30:25.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:30:25.649 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:30:25.649 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:30:25.649 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:30:25.649 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.40e+01 / 8.91e-01 | 20260508 08:30:25.649 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.05e-01, 1.08e-02 / 4.50e-03 | 20260508 08:30:25.649 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.46e+00, 5.88e-01 / 2.23e-01 | 20260508 08:30:25.649 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.09e-01 | 20260508 08:30:25.649 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.53e+01, 3.87e+00 / 1.07e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.87e+00 / 1.25e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.55e+00, 4.10e-01 / 2.54e-01 | 20260508 08:30:25.650 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.18e+00, 2.50e-01 / 1.24e-01 | 20260508 08:30:25.650 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.44e-01, 4.85e-02 / 2.39e-02 | 20260508 08:30:25.650 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:30:25.650 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:30:25.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:30:25.650 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 24.31%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 24.56%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 31.03%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 2.59%, LockDownload -- 2.73e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 15.34%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:30:25.650 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260508 08:30:25.761 | INFO | Starting a round of Baz IO | 20260508 08:30:29.161 | INFO | Populating RTMetrics report | 20260508 08:30:29.162 | INFO | RTMetrics report populated | 20260508 08:30:29.162 | INFO | Opening tmp RTMetrics json file | 20260508 08:30:29.174 | INFO | Serializing RTMetrics to json | 20260508 08:30:29.175 | INFO | Serialization done, swapping tmp file now... | 20260508 08:30:29.176 | INFO | RTMetrics done | 20260508 08:30:29.232 | INFO | Finished a round of Baz IO: Wrote 1.504 GiB in 3.471 seconds | 20260508 08:30:29.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.58% | 20260508 08:30:29.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.293152% / 0% / 0.000111262% | 20260508 08:30:29.232 | INFO | Baz IO overall: | 20260508 08:30:29.232 | INFO | BytesWritten : 2932501401838 | 20260508 08:30:29.232 | INFO | NumEvents : 1411728006734 | 20260508 08:30:30.791 | INFO | Analyzing chunk frames = [6684672,6685184) | 20260508 08:30:30.791 | INFO | MemInfo: { | 20260508 08:30:30.791 | INFO | "gpu" : " 17 GB", | 20260508 08:30:30.791 | INFO | "huge" : "", | 20260508 08:30:30.791 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:30.791 | INFO | "malloc" : " 14 GB", | 20260508 08:30:30.791 | INFO | "pinned" : " 29 GB" | 20260508 08:30:30.792 | INFO | } | 20260508 08:30:30.792 | INFO | Perf report for the last 5.14426 seconds: | 20260508 08:30:30.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:30.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:30.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:30.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:30.792 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 08:30:30.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:30.792 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260508 08:30:30.792 | INFO | PRE_HQ : 66.4%, 1.8, 62.8ms, 0 | 20260508 08:30:30.792 | INFO | BAZWRITER : 66.9%, 2.8, 98.0ms, 0 | 20260508 08:30:30.792 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:30:30.792 | INFO | Uploaded 36GiB to GPU in 2.56378s (14.0418GiB/s) | 20260508 08:30:30.792 | INFO | Downloaded 38GiB to GPU in 2.71464s (13.9982GiB/s) | 20260508 08:30:35.933 | INFO | Analyzing chunk frames = [6685184,6685696) | 20260508 08:30:35.933 | INFO | MemInfo: { | 20260508 08:30:35.933 | INFO | "gpu" : " 17 GB", | 20260508 08:30:35.933 | INFO | "huge" : "", | 20260508 08:30:35.933 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:35.933 | INFO | "malloc" : " 14 GB", | 20260508 08:30:35.933 | INFO | "pinned" : " 29 GB" | 20260508 08:30:35.933 | INFO | } | 20260508 08:30:35.933 | INFO | Perf report for the last 5.14111 seconds: | 20260508 08:30:35.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:35.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:30:35.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:30:35.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:35.933 | INFO | Uploaded 36GiB to GPU in 2.51711s (14.3021GiB/s) | 20260508 08:30:35.933 | INFO | Downloaded 34GiB to GPU in 2.44926s (13.8818GiB/s) | 20260508 08:30:41.060 | INFO | Analyzing chunk frames = [6685696,6686208) | 20260508 08:30:41.060 | INFO | MemInfo: { | 20260508 08:30:41.060 | INFO | "gpu" : " 17 GB", | 20260508 08:30:41.060 | INFO | "huge" : "", | 20260508 08:30:41.060 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:41.060 | INFO | "malloc" : " 14 GB", | 20260508 08:30:41.060 | INFO | "pinned" : " 29 GB" | 20260508 08:30:41.060 | INFO | } | 20260508 08:30:41.060 | INFO | Perf report for the last 5.12744 seconds: | 20260508 08:30:41.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:41.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.060 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 08:30:41.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.061 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:30:41.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:41.061 | INFO | Uploaded 36GiB to GPU in 2.4568s (14.6532GiB/s) | 20260508 08:30:41.061 | INFO | Downloaded 34GiB to GPU in 2.42005s (14.0493GiB/s) | 20260508 08:30:46.205 | INFO | Analyzing chunk frames = [6686208,6686720) | 20260508 08:30:46.205 | INFO | MemInfo: { | 20260508 08:30:46.205 | INFO | "gpu" : " 17 GB", | 20260508 08:30:46.205 | INFO | "huge" : "", | 20260508 08:30:46.205 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:46.205 | INFO | "malloc" : " 14 GB", | 20260508 08:30:46.205 | INFO | "pinned" : " 29 GB" | 20260508 08:30:46.205 | INFO | } | 20260508 08:30:46.205 | INFO | Perf report for the last 5.14498 seconds: | 20260508 08:30:46.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:46.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.205 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 08:30:46.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.205 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:30:46.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:46.206 | INFO | Uploaded 36GiB to GPU in 2.5803s (13.9519GiB/s) | 20260508 08:30:46.206 | INFO | Downloaded 34GiB to GPU in 2.4239s (14.027GiB/s) | 20260508 08:30:51.342 | INFO | Analyzing chunk frames = [6686720,6687232) | 20260508 08:30:51.342 | INFO | MemInfo: { | 20260508 08:30:51.342 | INFO | "gpu" : " 17 GB", | 20260508 08:30:51.342 | INFO | "huge" : "", | 20260508 08:30:51.342 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:51.342 | INFO | "malloc" : " 14 GB", | 20260508 08:30:51.342 | INFO | "pinned" : " 29 GB" | 20260508 08:30:51.342 | INFO | } | 20260508 08:30:51.342 | INFO | Perf report for the last 5.1368 seconds: | 20260508 08:30:51.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:51.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 08:30:51.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:30:51.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:51.342 | INFO | Uploaded 36GiB to GPU in 2.26547s (15.8907GiB/s) | 20260508 08:30:51.342 | INFO | Downloaded 34GiB to GPU in 2.49289s (13.6388GiB/s) | 20260508 08:30:56.486 | INFO | Analyzing chunk frames = [6687232,6687744) | 20260508 08:30:56.486 | INFO | MemInfo: { | 20260508 08:30:56.486 | INFO | "gpu" : " 17 GB", | 20260508 08:30:56.486 | INFO | "huge" : "", | 20260508 08:30:56.486 | INFO | "huge_shared" : " 29 GB", | 20260508 08:30:56.486 | INFO | "malloc" : " 14 GB", | 20260508 08:30:56.486 | INFO | "pinned" : " 29 GB" | 20260508 08:30:56.486 | INFO | } | 20260508 08:30:56.486 | INFO | Perf report for the last 5.14394 seconds: | 20260508 08:30:56.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:30:56.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:30:56.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:30:56.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:30:56.486 | INFO | Uploaded 36GiB to GPU in 2.42289s (14.8583GiB/s) | 20260508 08:30:56.487 | INFO | Downloaded 34GiB to GPU in 2.43302s (13.9744GiB/s) | 20260508 08:31:01.623 | INFO | Analyzing chunk frames = [6687744,6688256) | 20260508 08:31:01.623 | INFO | MemInfo: { | 20260508 08:31:01.623 | INFO | "gpu" : " 17 GB", | 20260508 08:31:01.623 | INFO | "huge" : "", | 20260508 08:31:01.623 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:01.623 | INFO | "malloc" : " 14 GB", | 20260508 08:31:01.623 | INFO | "pinned" : " 29 GB" | 20260508 08:31:01.623 | INFO | } | 20260508 08:31:01.623 | INFO | Perf report for the last 5.13738 seconds: | 20260508 08:31:01.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:01.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.623 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 08:31:01.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.623 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:31:01.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:01.624 | INFO | Uploaded 36GiB to GPU in 2.46283s (14.6173GiB/s) | 20260508 08:31:01.624 | INFO | Downloaded 34GiB to GPU in 2.43292s (13.975GiB/s) | 20260508 08:31:06.763 | INFO | Analyzing chunk frames = [6688256,6688768) | 20260508 08:31:06.763 | INFO | MemInfo: { | 20260508 08:31:06.763 | INFO | "gpu" : " 17 GB", | 20260508 08:31:06.763 | INFO | "huge" : "", | 20260508 08:31:06.763 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:06.763 | INFO | "malloc" : " 14 GB", | 20260508 08:31:06.763 | INFO | "pinned" : " 29 GB" | 20260508 08:31:06.763 | INFO | } | 20260508 08:31:06.763 | INFO | Perf report for the last 5.13995 seconds: | 20260508 08:31:06.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:06.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 08:31:06.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:31:06.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:06.764 | INFO | Uploaded 36GiB to GPU in 2.45571s (14.6597GiB/s) | 20260508 08:31:06.764 | INFO | Downloaded 34GiB to GPU in 2.42688s (14.0097GiB/s) | 20260508 08:31:06.862 | INFO | Starting a round of Baz IO | 20260508 08:31:10.286 | INFO | Populating RTMetrics report | 20260508 08:31:10.287 | INFO | RTMetrics report populated | 20260508 08:31:10.287 | INFO | Opening tmp RTMetrics json file | 20260508 08:31:10.296 | INFO | Serializing RTMetrics to json | 20260508 08:31:10.298 | INFO | Serialization done, swapping tmp file now... | 20260508 08:31:10.298 | INFO | RTMetrics done | 20260508 08:31:10.358 | INFO | Finished a round of Baz IO: Wrote 1.502 GiB in 3.496 seconds | 20260508 08:31:10.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.59% | 20260508 08:31:10.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.291351% / 3.97364e-06% / 7.94729e-05% | 20260508 08:31:10.358 | INFO | Baz IO overall: | 20260508 08:31:10.358 | INFO | BytesWritten : 2934114615774 | 20260508 08:31:10.358 | INFO | NumEvents : 1412492138523 | 20260508 08:31:11.902 | INFO | Analyzing chunk frames = [6688768,6689280) | 20260508 08:31:11.902 | INFO | MemInfo: { | 20260508 08:31:11.902 | INFO | "gpu" : " 17 GB", | 20260508 08:31:11.902 | INFO | "huge" : "", | 20260508 08:31:11.902 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:11.902 | INFO | "malloc" : " 14 GB", | 20260508 08:31:11.902 | INFO | "pinned" : " 29 GB" | 20260508 08:31:11.902 | INFO | } | 20260508 08:31:11.902 | INFO | Perf report for the last 5.1389 seconds: | 20260508 08:31:11.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:11.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:11.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:11.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:11.902 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:31:11.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:11.902 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 08:31:11.902 | INFO | PRE_HQ : 65.5%, 1.6, 54.7ms, 0 | 20260508 08:31:11.902 | INFO | BAZWRITER : 67.3%, 2.6, 89.1ms, 0 | 20260508 08:31:11.902 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:31:11.903 | INFO | Uploaded 36GiB to GPU in 2.6015s (13.8382GiB/s) | 20260508 08:31:11.903 | INFO | Downloaded 38GiB to GPU in 2.70816s (14.0317GiB/s) | 20260508 08:31:17.041 | INFO | Analyzing chunk frames = [6689280,6689792) | 20260508 08:31:17.041 | INFO | MemInfo: { | 20260508 08:31:17.041 | INFO | "gpu" : " 17 GB", | 20260508 08:31:17.041 | INFO | "huge" : "", | 20260508 08:31:17.042 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:17.042 | INFO | "malloc" : " 14 GB", | 20260508 08:31:17.042 | INFO | "pinned" : " 29 GB" | 20260508 08:31:17.042 | INFO | } | 20260508 08:31:17.042 | INFO | Perf report for the last 5.13945 seconds: | 20260508 08:31:17.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:17.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:31:17.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:31:17.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:17.042 | INFO | Uploaded 36GiB to GPU in 2.50662s (14.362GiB/s) | 20260508 08:31:17.042 | INFO | Downloaded 34GiB to GPU in 2.45742s (13.8357GiB/s) | 20260508 08:31:22.177 | INFO | Analyzing chunk frames = [6689792,6690304) | 20260508 08:31:22.177 | INFO | MemInfo: { | 20260508 08:31:22.177 | INFO | "gpu" : " 17 GB", | 20260508 08:31:22.177 | INFO | "huge" : "", | 20260508 08:31:22.177 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:22.177 | INFO | "malloc" : " 14 GB", | 20260508 08:31:22.177 | INFO | "pinned" : " 29 GB" | 20260508 08:31:22.177 | INFO | } | 20260508 08:31:22.177 | INFO | Perf report for the last 5.13558 seconds: | 20260508 08:31:22.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:22.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.177 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:31:22.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.177 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:31:22.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:22.178 | INFO | Uploaded 36GiB to GPU in 2.46353s (14.6132GiB/s) | 20260508 08:31:22.178 | INFO | Downloaded 34GiB to GPU in 2.43667s (13.9535GiB/s) | 20260508 08:31:27.317 | INFO | Analyzing chunk frames = [6690304,6690816) | 20260508 08:31:27.317 | INFO | MemInfo: { | 20260508 08:31:27.317 | INFO | "gpu" : " 17 GB", | 20260508 08:31:27.317 | INFO | "huge" : "", | 20260508 08:31:27.317 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:27.317 | INFO | "malloc" : " 14 GB", | 20260508 08:31:27.317 | INFO | "pinned" : " 29 GB" | 20260508 08:31:27.317 | INFO | } | 20260508 08:31:27.317 | INFO | Perf report for the last 5.14001 seconds: | 20260508 08:31:27.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:27.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | ANALYSIS : 66.8%, 3.0, 103.4ms, 0 | 20260508 08:31:27.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:31:27.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:27.318 | INFO | Uploaded 36GiB to GPU in 2.55352s (14.0982GiB/s) | 20260508 08:31:27.318 | INFO | Downloaded 34GiB to GPU in 2.43696s (13.9518GiB/s) | 20260508 08:31:32.459 | INFO | Analyzing chunk frames = [6690816,6691328) | 20260508 08:31:32.459 | INFO | MemInfo: { | 20260508 08:31:32.459 | INFO | "gpu" : " 17 GB", | 20260508 08:31:32.459 | INFO | "huge" : "", | 20260508 08:31:32.459 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:32.459 | INFO | "malloc" : " 14 GB", | 20260508 08:31:32.459 | INFO | "pinned" : " 29 GB" | 20260508 08:31:32.459 | INFO | } | 20260508 08:31:32.459 | INFO | Perf report for the last 5.1419 seconds: | 20260508 08:31:32.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:32.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.459 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 08:31:32.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.460 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:31:32.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:32.460 | INFO | Uploaded 36GiB to GPU in 2.29137s (15.7111GiB/s) | 20260508 08:31:32.460 | INFO | Downloaded 34GiB to GPU in 2.50183s (13.59GiB/s) | 20260508 08:31:37.604 | INFO | Analyzing chunk frames = [6691328,6691840) | 20260508 08:31:37.604 | INFO | MemInfo: { | 20260508 08:31:37.604 | INFO | "gpu" : " 17 GB", | 20260508 08:31:37.604 | INFO | "huge" : "", | 20260508 08:31:37.604 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:37.604 | INFO | "malloc" : " 14 GB", | 20260508 08:31:37.604 | INFO | "pinned" : " 29 GB" | 20260508 08:31:37.604 | INFO | } | 20260508 08:31:37.604 | INFO | Perf report for the last 5.1448 seconds: | 20260508 08:31:37.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:37.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 08:31:37.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:31:37.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:37.604 | INFO | Uploaded 36GiB to GPU in 2.33503s (15.4174GiB/s) | 20260508 08:31:37.604 | INFO | Downloaded 34GiB to GPU in 2.44958s (13.8799GiB/s) | 20260508 08:31:42.739 | INFO | Analyzing chunk frames = [6691840,6692352) | 20260508 08:31:42.739 | INFO | MemInfo: { | 20260508 08:31:42.739 | INFO | "gpu" : " 17 GB", | 20260508 08:31:42.739 | INFO | "huge" : "", | 20260508 08:31:42.739 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:42.739 | INFO | "malloc" : " 14 GB", | 20260508 08:31:42.739 | INFO | "pinned" : " 29 GB" | 20260508 08:31:42.739 | INFO | } | 20260508 08:31:42.739 | INFO | Perf report for the last 5.13507 seconds: | 20260508 08:31:42.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:42.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.739 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 08:31:42.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.739 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:31:42.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:42.740 | INFO | Uploaded 36GiB to GPU in 2.36741s (15.2065GiB/s) | 20260508 08:31:42.740 | INFO | Downloaded 34GiB to GPU in 2.44711s (13.894GiB/s) | 20260508 08:31:47.871 | INFO | Analyzing chunk frames = [6692352,6692864) | 20260508 08:31:47.871 | INFO | MemInfo: { | 20260508 08:31:47.871 | INFO | "gpu" : " 17 GB", | 20260508 08:31:47.871 | INFO | "huge" : "", | 20260508 08:31:47.871 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:47.871 | INFO | "malloc" : " 14 GB", | 20260508 08:31:47.871 | INFO | "pinned" : " 29 GB" | 20260508 08:31:47.871 | INFO | } | 20260508 08:31:47.871 | INFO | Perf report for the last 5.13214 seconds: | 20260508 08:31:47.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:47.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.871 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 08:31:47.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.871 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:31:47.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:47.872 | INFO | Uploaded 36GiB to GPU in 2.48705s (14.475GiB/s) | 20260508 08:31:47.872 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:31:47.872 | INFO | Total mem usage: 16.7 GiB | 20260508 08:31:47.872 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:31:47.872 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:31:47.872 | INFO | Downloaded 34GiB to GPU in 2.49251s (13.6409GiB/s) | 20260508 08:31:47.872 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:31:47.872 | INFO | High Water Breakdown: | 20260508 08:31:47.872 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:31:47.872 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:31:47.872 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:31:47.872 | INFO | ------------------------------------------ | 20260508 08:31:47.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:31:47.872 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:31:47.872 | INFO | Total mem usage: 26.5 GiB | 20260508 08:31:47.872 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:31:47.872 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:31:47.872 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:31:47.872 | INFO | High Water Breakdown: | 20260508 08:31:47.872 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:31:47.872 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:31:47.872 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:31:47.872 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:31:47.872 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:31:47.872 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:31:47.872 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:31:47.872 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:31:47.872 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:31:47.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:31:47.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:31:47.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:31:47.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:31:47.872 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:31:47.872 | INFO | ------------------------------------------ | 20260508 08:31:47.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:31:47.872 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:31:47.872 | INFO | Total mem usage: 27.8 GiB | 20260508 08:31:47.872 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:31:47.872 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:31:47.872 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:31:47.872 | INFO | High Water Breakdown: | 20260508 08:31:47.872 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:31:47.872 | INFO | ------------------------------------------ | 20260508 08:31:47.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:31:47.872 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:31:47.872 | INFO | Total mem usage: 15.7 GiB | 20260508 08:31:47.872 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:31:47.872 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:31:47.872 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:31:47.872 | INFO | High Water Breakdown: | 20260508 08:31:47.872 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:31:47.872 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:31:47.872 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:31:47.872 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:31:47.872 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:31:47.872 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:31:47.872 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:31:47.872 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:31:47.872 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:31:47.872 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:31:47.872 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:31:47.872 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:31:47.872 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:31:47.872 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:31:47.872 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:31:47.872 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:31:47.872 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:31:47.872 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:31:47.872 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:31:47.872 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:31:47.872 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:31:47.872 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:31:47.872 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:31:47.872 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:31:47.872 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:31:47.872 | INFO | ------------------------------------------ | 20260508 08:31:47.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:31:47.872 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:31:47.872 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:31:47.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:31:47.872 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.42e+01 / 5.99e-01 | 20260508 08:31:47.872 | INFO | PerfReport: 0.29%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.06e-02 / 6.58e-03 | 20260508 08:31:47.872 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.38e+00, 5.93e-01 / 2.43e-01 | 20260508 08:31:47.872 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.42e+00, 2.11e-01 / 1.42e-01 | 20260508 08:31:47.872 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.53e+01, 3.79e+00 / 1.17e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.88e+00 / 1.46e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.10e-01 / 2.74e-01 | 20260508 08:31:47.872 | INFO | PerfReport: 3.07%, Metrics -- 1.01e+00 / 1.16e+00, 2.54e-01 / 1.40e-01 | 20260508 08:31:47.872 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.34e-01, 4.94e-02 / 2.67e-02 | 20260508 08:31:47.872 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:31:47.872 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:31:47.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:31:47.872 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 24.28%, Upload -- 2.56e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 24.52%, Download -- 2.59e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 31.08%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 2.56%, LockDownload -- 2.70e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 15.38%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 08:31:47.872 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 3.06e-02 / 0.00e+00 | 20260508 08:31:47.970 | INFO | Starting a round of Baz IO | 20260508 08:31:51.389 | INFO | Populating RTMetrics report | 20260508 08:31:51.390 | INFO | RTMetrics report populated | 20260508 08:31:51.390 | INFO | Opening tmp RTMetrics json file | 20260508 08:31:51.399 | INFO | Serializing RTMetrics to json | 20260508 08:31:51.401 | INFO | Serialization done, swapping tmp file now... | 20260508 08:31:51.401 | INFO | RTMetrics done | 20260508 08:31:51.459 | INFO | Finished a round of Baz IO: Wrote 1.502 GiB in 3.489 seconds | 20260508 08:31:51.459 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.59% | 20260508 08:31:51.459 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.291395% / 3.97364e-06% / 4.76837e-05% | 20260508 08:31:51.459 | INFO | Baz IO overall: | 20260508 08:31:51.459 | INFO | BytesWritten : 2935727636667 | 20260508 08:31:51.459 | INFO | NumEvents : 1413256150350 | 20260508 08:31:53.013 | INFO | Analyzing chunk frames = [6692864,6693376) | 20260508 08:31:53.013 | INFO | MemInfo: { | 20260508 08:31:53.013 | INFO | "gpu" : " 17 GB", | 20260508 08:31:53.013 | INFO | "huge" : "", | 20260508 08:31:53.013 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:53.013 | INFO | "malloc" : " 14 GB", | 20260508 08:31:53.013 | INFO | "pinned" : " 29 GB" | 20260508 08:31:53.013 | INFO | } | 20260508 08:31:53.013 | INFO | Perf report for the last 5.14162 seconds: | 20260508 08:31:53.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:53.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:53.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:53.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:53.013 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:31:53.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:53.013 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:31:53.013 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 08:31:53.013 | INFO | BAZWRITER : 67.1%, 2.6, 88.7ms, 0 | 20260508 08:31:53.013 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:31:53.014 | INFO | Uploaded 36GiB to GPU in 2.58532s (13.9248GiB/s) | 20260508 08:31:53.014 | INFO | Downloaded 38GiB to GPU in 2.70766s (14.0343GiB/s) | 20260508 08:31:58.157 | INFO | Analyzing chunk frames = [6693376,6693888) | 20260508 08:31:58.157 | INFO | MemInfo: { | 20260508 08:31:58.157 | INFO | "gpu" : " 17 GB", | 20260508 08:31:58.157 | INFO | "huge" : "", | 20260508 08:31:58.157 | INFO | "huge_shared" : " 29 GB", | 20260508 08:31:58.157 | INFO | "malloc" : " 14 GB", | 20260508 08:31:58.157 | INFO | "pinned" : " 29 GB" | 20260508 08:31:58.157 | INFO | } | 20260508 08:31:58.157 | INFO | Perf report for the last 5.14393 seconds: | 20260508 08:31:58.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:31:58.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:31:58.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:31:58.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:31:58.157 | INFO | Uploaded 36GiB to GPU in 2.50553s (14.3682GiB/s) | 20260508 08:31:58.157 | INFO | Downloaded 34GiB to GPU in 2.43952s (13.9372GiB/s) | 20260508 08:32:03.297 | INFO | Analyzing chunk frames = [6693888,6694400) | 20260508 08:32:03.297 | INFO | MemInfo: { | 20260508 08:32:03.297 | INFO | "gpu" : " 17 GB", | 20260508 08:32:03.297 | INFO | "huge" : "", | 20260508 08:32:03.297 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:03.297 | INFO | "malloc" : " 14 GB", | 20260508 08:32:03.297 | INFO | "pinned" : " 29 GB" | 20260508 08:32:03.297 | INFO | } | 20260508 08:32:03.297 | INFO | Perf report for the last 5.1404 seconds: | 20260508 08:32:03.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:03.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.297 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 08:32:03.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.297 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:32:03.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:03.298 | INFO | Uploaded 36GiB to GPU in 2.48043s (14.5136GiB/s) | 20260508 08:32:03.298 | INFO | Downloaded 34GiB to GPU in 2.429s (13.9975GiB/s) | 20260508 08:32:08.436 | INFO | Analyzing chunk frames = [6694400,6694912) | 20260508 08:32:08.436 | INFO | MemInfo: { | 20260508 08:32:08.436 | INFO | "gpu" : " 17 GB", | 20260508 08:32:08.436 | INFO | "huge" : "", | 20260508 08:32:08.436 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:08.436 | INFO | "malloc" : " 14 GB", | 20260508 08:32:08.436 | INFO | "pinned" : " 29 GB" | 20260508 08:32:08.436 | INFO | } | 20260508 08:32:08.436 | INFO | Perf report for the last 5.13906 seconds: | 20260508 08:32:08.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:08.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.436 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 08:32:08.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.437 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 08:32:08.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:08.437 | INFO | Uploaded 36GiB to GPU in 2.58392s (13.9323GiB/s) | 20260508 08:32:08.437 | INFO | Downloaded 34GiB to GPU in 2.4351s (13.9625GiB/s) | 20260508 08:32:13.572 | INFO | Analyzing chunk frames = [6694912,6695424) | 20260508 08:32:13.572 | INFO | MemInfo: { | 20260508 08:32:13.572 | INFO | "gpu" : " 17 GB", | 20260508 08:32:13.572 | INFO | "huge" : "", | 20260508 08:32:13.572 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:13.572 | INFO | "malloc" : " 14 GB", | 20260508 08:32:13.572 | INFO | "pinned" : " 29 GB" | 20260508 08:32:13.572 | INFO | } | 20260508 08:32:13.572 | INFO | Perf report for the last 5.13571 seconds: | 20260508 08:32:13.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:13.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.572 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:32:13.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.572 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:32:13.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:13.573 | INFO | Uploaded 36GiB to GPU in 2.38413s (15.0998GiB/s) | 20260508 08:32:13.573 | INFO | Downloaded 34GiB to GPU in 2.50787s (13.5573GiB/s) | 20260508 08:32:18.716 | INFO | Analyzing chunk frames = [6695424,6695936) | 20260508 08:32:18.716 | INFO | MemInfo: { | 20260508 08:32:18.716 | INFO | "gpu" : " 17 GB", | 20260508 08:32:18.716 | INFO | "huge" : "", | 20260508 08:32:18.716 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:18.716 | INFO | "malloc" : " 14 GB", | 20260508 08:32:18.717 | INFO | "pinned" : " 29 GB" | 20260508 08:32:18.717 | INFO | } | 20260508 08:32:18.717 | INFO | Perf report for the last 5.14455 seconds: | 20260508 08:32:18.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:18.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 08:32:18.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:32:18.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:18.717 | INFO | Uploaded 36GiB to GPU in 2.3702s (15.1886GiB/s) | 20260508 08:32:18.717 | INFO | Downloaded 34GiB to GPU in 2.42293s (14.0326GiB/s) | 20260508 08:32:23.851 | INFO | Analyzing chunk frames = [6695936,6696448) | 20260508 08:32:23.851 | INFO | MemInfo: { | 20260508 08:32:23.851 | INFO | "gpu" : " 17 GB", | 20260508 08:32:23.851 | INFO | "huge" : "", | 20260508 08:32:23.851 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:23.851 | INFO | "malloc" : " 14 GB", | 20260508 08:32:23.851 | INFO | "pinned" : " 29 GB" | 20260508 08:32:23.852 | INFO | } | 20260508 08:32:23.852 | INFO | Perf report for the last 5.13487 seconds: | 20260508 08:32:23.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:23.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260508 08:32:23.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:32:23.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:23.852 | INFO | Uploaded 36GiB to GPU in 2.43912s (14.7594GiB/s) | 20260508 08:32:23.852 | INFO | Downloaded 34GiB to GPU in 2.44849s (13.8861GiB/s) | 20260508 08:32:28.992 | INFO | Analyzing chunk frames = [6696448,6696960) | 20260508 08:32:28.992 | INFO | MemInfo: { | 20260508 08:32:28.992 | INFO | "gpu" : " 17 GB", | 20260508 08:32:28.992 | INFO | "huge" : "", | 20260508 08:32:28.992 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:28.992 | INFO | "malloc" : " 14 GB", | 20260508 08:32:28.992 | INFO | "pinned" : " 29 GB" | 20260508 08:32:28.992 | INFO | } | 20260508 08:32:28.992 | INFO | Perf report for the last 5.14082 seconds: | 20260508 08:32:28.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:28.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.992 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:32:28.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.992 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:32:28.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:28.993 | INFO | Uploaded 36GiB to GPU in 2.48096s (14.5105GiB/s) | 20260508 08:32:28.993 | INFO | Downloaded 34GiB to GPU in 2.44811s (13.8882GiB/s) | 20260508 08:32:29.090 | INFO | Starting a round of Baz IO | 20260508 08:32:32.513 | INFO | Populating RTMetrics report | 20260508 08:32:32.513 | INFO | RTMetrics report populated | 20260508 08:32:32.513 | INFO | Opening tmp RTMetrics json file | 20260508 08:32:32.523 | INFO | Serializing RTMetrics to json | 20260508 08:32:32.524 | INFO | Serialization done, swapping tmp file now... | 20260508 08:32:32.524 | INFO | RTMetrics done | 20260508 08:32:32.578 | INFO | Finished a round of Baz IO: Wrote 1.502 GiB in 3.487 seconds | 20260508 08:32:32.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.59% | 20260508 08:32:32.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290147% / 1.19209e-05% / 0.000111262% | 20260508 08:32:32.578 | INFO | Baz IO overall: | 20260508 08:32:32.578 | INFO | BytesWritten : 2937340460530 | 20260508 08:32:32.578 | INFO | NumEvents : 1414020244203 | 20260508 08:32:34.128 | INFO | Analyzing chunk frames = [6696960,6697472) | 20260508 08:32:34.128 | INFO | MemInfo: { | 20260508 08:32:34.128 | INFO | "gpu" : " 17 GB", | 20260508 08:32:34.128 | INFO | "huge" : "", | 20260508 08:32:34.128 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:34.128 | INFO | "malloc" : " 14 GB", | 20260508 08:32:34.128 | INFO | "pinned" : " 29 GB" | 20260508 08:32:34.128 | INFO | } | 20260508 08:32:34.128 | INFO | Perf report for the last 5.13558 seconds: | 20260508 08:32:34.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:34.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:34.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:34.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:34.128 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 08:32:34.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:34.128 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:32:34.128 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 08:32:34.128 | INFO | BAZWRITER : 67.2%, 2.6, 88.2ms, 0 | 20260508 08:32:34.128 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:32:34.128 | INFO | Uploaded 36GiB to GPU in 2.55093s (14.1125GiB/s) | 20260508 08:32:34.128 | INFO | Downloaded 38GiB to GPU in 2.71514s (13.9956GiB/s) | 20260508 08:32:39.269 | INFO | Analyzing chunk frames = [6697472,6697984) | 20260508 08:32:39.269 | INFO | MemInfo: { | 20260508 08:32:39.269 | INFO | "gpu" : " 17 GB", | 20260508 08:32:39.269 | INFO | "huge" : "", | 20260508 08:32:39.269 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:39.269 | INFO | "malloc" : " 14 GB", | 20260508 08:32:39.269 | INFO | "pinned" : " 29 GB" | 20260508 08:32:39.269 | INFO | } | 20260508 08:32:39.269 | INFO | Perf report for the last 5.14091 seconds: | 20260508 08:32:39.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:39.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.269 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 08:32:39.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.269 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:32:39.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:39.270 | INFO | Uploaded 36GiB to GPU in 2.52567s (14.2536GiB/s) | 20260508 08:32:39.270 | INFO | Downloaded 34GiB to GPU in 2.45537s (13.8472GiB/s) | 20260508 08:32:44.408 | INFO | Analyzing chunk frames = [6697984,6698496) | 20260508 08:32:44.408 | INFO | MemInfo: { | 20260508 08:32:44.408 | INFO | "gpu" : " 17 GB", | 20260508 08:32:44.408 | INFO | "huge" : "", | 20260508 08:32:44.408 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:44.408 | INFO | "malloc" : " 14 GB", | 20260508 08:32:44.408 | INFO | "pinned" : " 29 GB" | 20260508 08:32:44.408 | INFO | } | 20260508 08:32:44.408 | INFO | Perf report for the last 5.13915 seconds: | 20260508 08:32:44.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:44.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.408 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 08:32:44.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.408 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:32:44.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:44.409 | INFO | Uploaded 36GiB to GPU in 2.50272s (14.3843GiB/s) | 20260508 08:32:44.409 | INFO | Downloaded 34GiB to GPU in 2.44002s (13.9343GiB/s) | 20260508 08:32:49.551 | INFO | Analyzing chunk frames = [6698496,6699008) | 20260508 08:32:49.551 | INFO | MemInfo: { | 20260508 08:32:49.551 | INFO | "gpu" : " 17 GB", | 20260508 08:32:49.551 | INFO | "huge" : "", | 20260508 08:32:49.551 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:49.551 | INFO | "malloc" : " 14 GB", | 20260508 08:32:49.551 | INFO | "pinned" : " 29 GB" | 20260508 08:32:49.551 | INFO | } | 20260508 08:32:49.551 | INFO | Perf report for the last 5.14279 seconds: | 20260508 08:32:49.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:49.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 08:32:49.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:32:49.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:49.551 | INFO | Uploaded 36GiB to GPU in 2.49802s (14.4114GiB/s) | 20260508 08:32:49.551 | INFO | Downloaded 34GiB to GPU in 2.43258s (13.9769GiB/s) | 20260508 08:32:54.689 | INFO | Analyzing chunk frames = [6699008,6699520) | 20260508 08:32:54.689 | INFO | MemInfo: { | 20260508 08:32:54.689 | INFO | "gpu" : " 17 GB", | 20260508 08:32:54.689 | INFO | "huge" : "", | 20260508 08:32:54.689 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:54.689 | INFO | "malloc" : " 14 GB", | 20260508 08:32:54.689 | INFO | "pinned" : " 29 GB" | 20260508 08:32:54.689 | INFO | } | 20260508 08:32:54.689 | INFO | Perf report for the last 5.13848 seconds: | 20260508 08:32:54.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:54.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.689 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 08:32:54.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.689 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:32:54.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:54.690 | INFO | Uploaded 36GiB to GPU in 2.31612s (15.5432GiB/s) | 20260508 08:32:54.690 | INFO | Downloaded 34GiB to GPU in 2.49006s (13.6543GiB/s) | 20260508 08:32:59.824 | INFO | Analyzing chunk frames = [6699520,6700032) | 20260508 08:32:59.824 | INFO | MemInfo: { | 20260508 08:32:59.824 | INFO | "gpu" : " 17 GB", | 20260508 08:32:59.824 | INFO | "huge" : "", | 20260508 08:32:59.824 | INFO | "huge_shared" : " 29 GB", | 20260508 08:32:59.824 | INFO | "malloc" : " 14 GB", | 20260508 08:32:59.824 | INFO | "pinned" : " 29 GB" | 20260508 08:32:59.824 | INFO | } | 20260508 08:32:59.824 | INFO | Perf report for the last 5.13476 seconds: | 20260508 08:32:59.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:32:59.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.824 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260508 08:32:59.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.824 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:32:59.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:32:59.825 | INFO | Uploaded 36GiB to GPU in 2.38126s (15.118GiB/s) | 20260508 08:32:59.825 | INFO | Downloaded 34GiB to GPU in 2.43489s (13.9637GiB/s) | 20260508 08:33:04.966 | INFO | Analyzing chunk frames = [6700032,6700544) | 20260508 08:33:04.966 | INFO | MemInfo: { | 20260508 08:33:04.966 | INFO | "gpu" : " 17 GB", | 20260508 08:33:04.966 | INFO | "huge" : "", | 20260508 08:33:04.966 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:04.966 | INFO | "malloc" : " 14 GB", | 20260508 08:33:04.966 | INFO | "pinned" : " 29 GB" | 20260508 08:33:04.966 | INFO | } | 20260508 08:33:04.966 | INFO | Perf report for the last 5.14178 seconds: | 20260508 08:33:04.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:04.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.966 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 08:33:04.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.966 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:33:04.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:04.967 | INFO | Uploaded 36GiB to GPU in 2.396s (15.025GiB/s) | 20260508 08:33:04.967 | INFO | Downloaded 34GiB to GPU in 2.43392s (13.9692GiB/s) | 20260508 08:33:10.105 | INFO | Analyzing chunk frames = [6700544,6701056) | 20260508 08:33:10.105 | INFO | MemInfo: { | 20260508 08:33:10.105 | INFO | "gpu" : " 17 GB", | 20260508 08:33:10.105 | INFO | "huge" : "", | 20260508 08:33:10.105 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:10.106 | INFO | "malloc" : " 14 GB", | 20260508 08:33:10.106 | INFO | "pinned" : " 29 GB" | 20260508 08:33:10.106 | INFO | } | 20260508 08:33:10.106 | INFO | Perf report for the last 5.13962 seconds: | 20260508 08:33:10.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:10.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 08:33:10.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:33:10.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:10.106 | INFO | Uploaded 36GiB to GPU in 2.4625s (14.6193GiB/s) | 20260508 08:33:10.106 | INFO | Downloaded 34GiB to GPU in 2.44382s (13.9127GiB/s) | 20260508 08:33:10.106 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:33:10.106 | INFO | Total mem usage: 16.7 GiB | 20260508 08:33:10.106 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:33:10.106 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:33:10.106 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:33:10.106 | INFO | High Water Breakdown: | 20260508 08:33:10.106 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:33:10.106 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:33:10.106 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:33:10.106 | INFO | ------------------------------------------ | 20260508 08:33:10.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:33:10.107 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:33:10.107 | INFO | Total mem usage: 26.5 GiB | 20260508 08:33:10.107 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:33:10.107 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:33:10.107 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:33:10.107 | INFO | High Water Breakdown: | 20260508 08:33:10.107 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:33:10.107 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:33:10.107 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:33:10.107 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:33:10.107 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:33:10.107 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:33:10.107 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:33:10.107 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:33:10.107 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:33:10.107 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:33:10.107 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:33:10.107 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:33:10.107 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:33:10.107 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:33:10.107 | INFO | ------------------------------------------ | 20260508 08:33:10.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:33:10.107 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:33:10.107 | INFO | Total mem usage: 27.8 GiB | 20260508 08:33:10.107 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:33:10.107 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:33:10.107 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:33:10.107 | INFO | High Water Breakdown: | 20260508 08:33:10.107 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:33:10.107 | INFO | ------------------------------------------ | 20260508 08:33:10.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:33:10.107 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:33:10.107 | INFO | Total mem usage: 15.7 GiB | 20260508 08:33:10.107 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:33:10.107 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:33:10.107 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:33:10.107 | INFO | High Water Breakdown: | 20260508 08:33:10.107 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:33:10.107 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:33:10.107 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:33:10.107 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:33:10.107 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:33:10.107 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:33:10.107 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:33:10.107 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:33:10.107 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:33:10.107 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:33:10.107 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:33:10.107 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:33:10.107 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:33:10.107 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:33:10.107 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:33:10.107 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:33:10.107 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:33:10.107 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:33:10.107 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:33:10.107 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:33:10.107 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:33:10.107 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:33:10.107 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:33:10.107 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:33:10.107 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:33:10.107 | INFO | ------------------------------------------ | 20260508 08:33:10.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:33:10.107 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:33:10.107 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:33:10.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:33:10.107 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.41e+01 / 7.40e-01 | 20260508 08:33:10.107 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.01e-01, 1.13e-02 / 5.12e-03 | 20260508 08:33:10.107 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.39e+00, 5.98e-01 / 2.35e-01 | 20260508 08:33:10.107 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.42e+00, 2.10e-01 / 1.33e-01 | 20260508 08:33:10.107 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.53e+01, 3.62e+00 / 1.22e+00 | 20260508 08:33:10.107 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.87e+00 / 1.48e+00 | 20260508 08:33:10.107 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.50e+00, 4.08e-01 / 2.85e-01 | 20260508 08:33:10.107 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.15e+00, 2.53e-01 / 1.56e-01 | 20260508 08:33:10.107 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.36e-01, 4.93e-02 / 2.62e-02 | 20260508 08:33:10.107 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:33:10.107 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:33:10.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:33:10.107 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:33:10.107 | INFO | PerfReport: 24.35%, Upload -- 2.57e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 08:33:10.107 | INFO | PerfReport: 24.48%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 08:33:10.107 | INFO | PerfReport: 31.05%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:33:10.108 | INFO | PerfReport: 2.18%, LockUpload -- 2.30e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 08:33:10.108 | INFO | PerfReport: 2.51%, LockDownload -- 2.65e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 08:33:10.108 | INFO | PerfReport: 15.42%, LockCompute -- 1.63e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:33:10.108 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260508 08:33:10.203 | INFO | Starting a round of Baz IO | 20260508 08:33:13.625 | INFO | Populating RTMetrics report | 20260508 08:33:13.625 | INFO | RTMetrics report populated | 20260508 08:33:13.625 | INFO | Opening tmp RTMetrics json file | 20260508 08:33:13.637 | INFO | Serializing RTMetrics to json | 20260508 08:33:13.638 | INFO | Serialization done, swapping tmp file now... | 20260508 08:33:13.639 | INFO | RTMetrics done | 20260508 08:33:13.692 | INFO | Finished a round of Baz IO: Wrote 1.501 GiB in 3.488 seconds | 20260508 08:33:13.692 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.59% | 20260508 08:33:13.692 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290008% / 0% / 7.94729e-05% | 20260508 08:33:13.692 | INFO | Baz IO overall: | 20260508 08:33:13.692 | INFO | BytesWritten : 2938951700667 | 20260508 08:33:13.692 | INFO | NumEvents : 1414783301736 | 20260508 08:33:15.251 | INFO | Analyzing chunk frames = [6701056,6701568) | 20260508 08:33:15.251 | INFO | MemInfo: { | 20260508 08:33:15.251 | INFO | "gpu" : " 17 GB", | 20260508 08:33:15.251 | INFO | "huge" : "", | 20260508 08:33:15.251 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:15.251 | INFO | "malloc" : " 14 GB", | 20260508 08:33:15.251 | INFO | "pinned" : " 29 GB" | 20260508 08:33:15.251 | INFO | } | 20260508 08:33:15.251 | INFO | Perf report for the last 5.14534 seconds: | 20260508 08:33:15.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:15.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:15.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:15.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:15.251 | INFO | ANALYSIS : 67.5%, 3.0, 105.7ms, 0 | 20260508 08:33:15.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:15.251 | INFO | RT_METRICS : 41.8%, 1.0, 22.0ms, 0 | 20260508 08:33:15.251 | INFO | PRE_HQ : 65.7%, 1.7, 58.8ms, 0 | 20260508 08:33:15.251 | INFO | BAZWRITER : 67.2%, 2.7, 92.1ms, 0 | 20260508 08:33:15.251 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 08:33:15.251 | INFO | Uploaded 36GiB to GPU in 2.57133s (14.0005GiB/s) | 20260508 08:33:15.251 | INFO | Downloaded 38GiB to GPU in 2.70752s (14.035GiB/s) | 20260508 08:33:20.383 | INFO | Analyzing chunk frames = [6701568,6702080) | 20260508 08:33:20.383 | INFO | MemInfo: { | 20260508 08:33:20.383 | INFO | "gpu" : " 17 GB", | 20260508 08:33:20.383 | INFO | "huge" : "", | 20260508 08:33:20.383 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:20.383 | INFO | "malloc" : " 14 GB", | 20260508 08:33:20.384 | INFO | "pinned" : " 29 GB" | 20260508 08:33:20.384 | INFO | } | 20260508 08:33:20.384 | INFO | Perf report for the last 5.13266 seconds: | 20260508 08:33:20.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:20.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 08:33:20.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:33:20.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:20.384 | INFO | Uploaded 36GiB to GPU in 2.51952s (14.2884GiB/s) | 20260508 08:33:20.384 | INFO | Downloaded 34GiB to GPU in 2.45072s (13.8735GiB/s) | 20260508 08:33:25.524 | INFO | Analyzing chunk frames = [6702080,6702592) | 20260508 08:33:25.524 | INFO | MemInfo: { | 20260508 08:33:25.524 | INFO | "gpu" : " 17 GB", | 20260508 08:33:25.524 | INFO | "huge" : "", | 20260508 08:33:25.524 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:25.524 | INFO | "malloc" : " 14 GB", | 20260508 08:33:25.524 | INFO | "pinned" : " 29 GB" | 20260508 08:33:25.524 | INFO | } | 20260508 08:33:25.525 | INFO | Perf report for the last 5.14094 seconds: | 20260508 08:33:25.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:25.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 08:33:25.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:33:25.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:25.525 | INFO | Uploaded 36GiB to GPU in 2.48691s (14.4758GiB/s) | 20260508 08:33:25.525 | INFO | Downloaded 34GiB to GPU in 2.43916s (13.9393GiB/s) | 20260508 08:33:30.667 | INFO | Analyzing chunk frames = [6702592,6703104) | 20260508 08:33:30.668 | INFO | MemInfo: { | 20260508 08:33:30.668 | INFO | "gpu" : " 17 GB", | 20260508 08:33:30.668 | INFO | "huge" : "", | 20260508 08:33:30.668 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:30.668 | INFO | "malloc" : " 14 GB", | 20260508 08:33:30.668 | INFO | "pinned" : " 29 GB" | 20260508 08:33:30.668 | INFO | } | 20260508 08:33:30.668 | INFO | Perf report for the last 5.14364 seconds: | 20260508 08:33:30.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:30.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | ANALYSIS : 66.7%, 3.0, 103.4ms, 0 | 20260508 08:33:30.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:33:30.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:30.668 | INFO | Uploaded 36GiB to GPU in 2.53647s (14.193GiB/s) | 20260508 08:33:30.668 | INFO | Downloaded 34GiB to GPU in 2.43931s (13.9384GiB/s) | 20260508 08:33:35.806 | INFO | Analyzing chunk frames = [6703104,6703616) | 20260508 08:33:35.806 | INFO | MemInfo: { | 20260508 08:33:35.806 | INFO | "gpu" : " 17 GB", | 20260508 08:33:35.806 | INFO | "huge" : "", | 20260508 08:33:35.806 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:35.806 | INFO | "malloc" : " 14 GB", | 20260508 08:33:35.806 | INFO | "pinned" : " 29 GB" | 20260508 08:33:35.806 | INFO | } | 20260508 08:33:35.806 | INFO | Perf report for the last 5.13781 seconds: | 20260508 08:33:35.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:35.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 08:33:35.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:33:35.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:35.806 | INFO | Uploaded 36GiB to GPU in 2.39396s (15.0379GiB/s) | 20260508 08:33:35.807 | INFO | Downloaded 34GiB to GPU in 2.51617s (13.5126GiB/s) | 20260508 08:33:40.939 | INFO | Analyzing chunk frames = [6703616,6704128) | 20260508 08:33:40.939 | INFO | MemInfo: { | 20260508 08:33:40.939 | INFO | "gpu" : " 17 GB", | 20260508 08:33:40.939 | INFO | "huge" : "", | 20260508 08:33:40.939 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:40.939 | INFO | "malloc" : " 14 GB", | 20260508 08:33:40.939 | INFO | "pinned" : " 29 GB" | 20260508 08:33:40.939 | INFO | } | 20260508 08:33:40.939 | INFO | Perf report for the last 5.13295 seconds: | 20260508 08:33:40.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:40.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.939 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:33:40.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.939 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:33:40.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:40.940 | INFO | Uploaded 36GiB to GPU in 2.42768s (14.829GiB/s) | 20260508 08:33:40.940 | INFO | Downloaded 34GiB to GPU in 2.43884s (13.9411GiB/s) | 20260508 08:33:46.076 | INFO | Analyzing chunk frames = [6704128,6704640) | 20260508 08:33:46.076 | INFO | MemInfo: { | 20260508 08:33:46.076 | INFO | "gpu" : " 17 GB", | 20260508 08:33:46.076 | INFO | "huge" : "", | 20260508 08:33:46.076 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:46.076 | INFO | "malloc" : " 14 GB", | 20260508 08:33:46.076 | INFO | "pinned" : " 29 GB" | 20260508 08:33:46.076 | INFO | } | 20260508 08:33:46.076 | INFO | Perf report for the last 5.13711 seconds: | 20260508 08:33:46.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:46.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.076 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 08:33:46.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.076 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:33:46.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:46.077 | INFO | Uploaded 36GiB to GPU in 2.50456s (14.3738GiB/s) | 20260508 08:33:46.077 | INFO | Downloaded 34GiB to GPU in 2.42821s (14.0021GiB/s) | 20260508 08:33:51.222 | INFO | Analyzing chunk frames = [6704640,6705152) | 20260508 08:33:51.222 | INFO | MemInfo: { | 20260508 08:33:51.222 | INFO | "gpu" : " 17 GB", | 20260508 08:33:51.222 | INFO | "huge" : "", | 20260508 08:33:51.223 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:51.223 | INFO | "malloc" : " 14 GB", | 20260508 08:33:51.223 | INFO | "pinned" : " 29 GB" | 20260508 08:33:51.223 | INFO | } | 20260508 08:33:51.223 | INFO | Perf report for the last 5.1465 seconds: | 20260508 08:33:51.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:51.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | ANALYSIS : 65.9%, 3.0, 103.2ms, 0 | 20260508 08:33:51.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:33:51.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:51.223 | INFO | Uploaded 36GiB to GPU in 2.46672s (14.5943GiB/s) | 20260508 08:33:51.223 | INFO | Downloaded 34GiB to GPU in 2.43913s (13.9394GiB/s) | 20260508 08:33:51.320 | INFO | Starting a round of Baz IO | 20260508 08:33:54.729 | INFO | Populating RTMetrics report | 20260508 08:33:54.729 | INFO | RTMetrics report populated | 20260508 08:33:54.729 | INFO | Opening tmp RTMetrics json file | 20260508 08:33:54.740 | INFO | Serializing RTMetrics to json | 20260508 08:33:54.742 | INFO | Serialization done, swapping tmp file now... | 20260508 08:33:54.743 | INFO | RTMetrics done | 20260508 08:33:54.819 | INFO | Finished a round of Baz IO: Wrote 1.501 GiB in 3.499 seconds | 20260508 08:33:54.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.59% | 20260508 08:33:54.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290565% / 0% / 5.5631e-05% | 20260508 08:33:54.819 | INFO | Baz IO overall: | 20260508 08:33:54.819 | INFO | BytesWritten : 2940563828361 | 20260508 08:33:54.819 | INFO | NumEvents : 1415547079704 | 20260508 08:33:56.368 | INFO | Analyzing chunk frames = [6705152,6705664) | 20260508 08:33:56.368 | INFO | MemInfo: { | 20260508 08:33:56.368 | INFO | "gpu" : " 17 GB", | 20260508 08:33:56.368 | INFO | "huge" : "", | 20260508 08:33:56.369 | INFO | "huge_shared" : " 29 GB", | 20260508 08:33:56.369 | INFO | "malloc" : " 14 GB", | 20260508 08:33:56.369 | INFO | "pinned" : " 29 GB" | 20260508 08:33:56.369 | INFO | } | 20260508 08:33:56.369 | INFO | Perf report for the last 5.14596 seconds: | 20260508 08:33:56.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:33:56.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:56.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:56.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:56.369 | INFO | ANALYSIS : 67.5%, 3.0, 104.5ms, 0 | 20260508 08:33:56.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:33:56.369 | INFO | RT_METRICS : 38.9%, 1.0, 20.2ms, 0 | 20260508 08:33:56.369 | INFO | PRE_HQ : 65.5%, 1.6, 54.4ms, 0 | 20260508 08:33:56.369 | INFO | BAZWRITER : 67.4%, 2.6, 89.0ms, 0 | 20260508 08:33:56.369 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 08:33:56.369 | INFO | Uploaded 36GiB to GPU in 2.60888s (13.799GiB/s) | 20260508 08:33:56.369 | INFO | Downloaded 38GiB to GPU in 2.70814s (14.0318GiB/s) | 20260508 08:34:01.501 | INFO | Analyzing chunk frames = [6705664,6706176) | 20260508 08:34:01.501 | INFO | MemInfo: { | 20260508 08:34:01.501 | INFO | "gpu" : " 17 GB", | 20260508 08:34:01.501 | INFO | "huge" : "", | 20260508 08:34:01.501 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:01.501 | INFO | "malloc" : " 14 GB", | 20260508 08:34:01.501 | INFO | "pinned" : " 29 GB" | 20260508 08:34:01.501 | INFO | } | 20260508 08:34:01.501 | INFO | Perf report for the last 5.13248 seconds: | 20260508 08:34:01.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:01.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.501 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 08:34:01.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.501 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:34:01.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:01.502 | INFO | Uploaded 36GiB to GPU in 2.53114s (14.2228GiB/s) | 20260508 08:34:01.502 | INFO | Downloaded 34GiB to GPU in 2.44649s (13.8975GiB/s) | 20260508 08:34:06.634 | INFO | Analyzing chunk frames = [6706176,6706688) | 20260508 08:34:06.634 | INFO | MemInfo: { | 20260508 08:34:06.634 | INFO | "gpu" : " 17 GB", | 20260508 08:34:06.634 | INFO | "huge" : "", | 20260508 08:34:06.634 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:06.634 | INFO | "malloc" : " 14 GB", | 20260508 08:34:06.634 | INFO | "pinned" : " 29 GB" | 20260508 08:34:06.634 | INFO | } | 20260508 08:34:06.634 | INFO | Perf report for the last 5.1329 seconds: | 20260508 08:34:06.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:06.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 08:34:06.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:34:06.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:06.634 | INFO | Uploaded 36GiB to GPU in 2.58698s (13.9158GiB/s) | 20260508 08:34:06.634 | INFO | Downloaded 34GiB to GPU in 2.43002s (13.9916GiB/s) | 20260508 08:34:11.777 | INFO | Analyzing chunk frames = [6706688,6707200) | 20260508 08:34:11.777 | INFO | MemInfo: { | 20260508 08:34:11.777 | INFO | "gpu" : " 17 GB", | 20260508 08:34:11.777 | INFO | "huge" : "", | 20260508 08:34:11.777 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:11.777 | INFO | "malloc" : " 14 GB", | 20260508 08:34:11.777 | INFO | "pinned" : " 29 GB" | 20260508 08:34:11.777 | INFO | } | 20260508 08:34:11.778 | INFO | Perf report for the last 5.14351 seconds: | 20260508 08:34:11.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:11.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | ANALYSIS : 67.0%, 2.9, 103.8ms, 0 | 20260508 08:34:11.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:34:11.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:11.778 | INFO | Uploaded 36GiB to GPU in 2.60233s (13.8337GiB/s) | 20260508 08:34:11.778 | INFO | Downloaded 34GiB to GPU in 2.43044s (13.9893GiB/s) | 20260508 08:34:16.914 | INFO | Analyzing chunk frames = [6707200,6707712) | 20260508 08:34:16.914 | INFO | MemInfo: { | 20260508 08:34:16.914 | INFO | "gpu" : " 17 GB", | 20260508 08:34:16.914 | INFO | "huge" : "", | 20260508 08:34:16.914 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:16.914 | INFO | "malloc" : " 14 GB", | 20260508 08:34:16.914 | INFO | "pinned" : " 29 GB" | 20260508 08:34:16.914 | INFO | } | 20260508 08:34:16.914 | INFO | Perf report for the last 5.13651 seconds: | 20260508 08:34:16.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:16.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 08:34:16.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 08:34:16.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:16.914 | INFO | Uploaded 36GiB to GPU in 2.31259s (15.567GiB/s) | 20260508 08:34:16.914 | INFO | Downloaded 34GiB to GPU in 2.48286s (13.6939GiB/s) | 20260508 08:34:22.058 | INFO | Analyzing chunk frames = [6707712,6708224) | 20260508 08:34:22.058 | INFO | MemInfo: { | 20260508 08:34:22.058 | INFO | "gpu" : " 17 GB", | 20260508 08:34:22.058 | INFO | "huge" : "", | 20260508 08:34:22.058 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:22.058 | INFO | "malloc" : " 14 GB", | 20260508 08:34:22.058 | INFO | "pinned" : " 29 GB" | 20260508 08:34:22.058 | INFO | } | 20260508 08:34:22.058 | INFO | Perf report for the last 5.14408 seconds: | 20260508 08:34:22.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:22.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.058 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:34:22.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.058 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 08:34:22.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:22.059 | INFO | Uploaded 36GiB to GPU in 2.40322s (14.9799GiB/s) | 20260508 08:34:22.059 | INFO | Downloaded 34GiB to GPU in 2.43253s (13.9772GiB/s) | 20260508 08:34:27.199 | INFO | Analyzing chunk frames = [6708224,6708736) | 20260508 08:34:27.199 | INFO | MemInfo: { | 20260508 08:34:27.199 | INFO | "gpu" : " 17 GB", | 20260508 08:34:27.199 | INFO | "huge" : "", | 20260508 08:34:27.199 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:27.199 | INFO | "malloc" : " 14 GB", | 20260508 08:34:27.199 | INFO | "pinned" : " 29 GB" | 20260508 08:34:27.199 | INFO | } | 20260508 08:34:27.199 | INFO | Perf report for the last 5.14129 seconds: | 20260508 08:34:27.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:27.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.200 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 08:34:27.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.200 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:34:27.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:27.200 | INFO | Uploaded 36GiB to GPU in 2.44008s (14.7536GiB/s) | 20260508 08:34:27.200 | INFO | Downloaded 34GiB to GPU in 2.44202s (13.9229GiB/s) | 20260508 08:34:32.337 | INFO | Analyzing chunk frames = [6708736,6709248) | 20260508 08:34:32.337 | INFO | MemInfo: { | 20260508 08:34:32.337 | INFO | "gpu" : " 17 GB", | 20260508 08:34:32.337 | INFO | "huge" : "", | 20260508 08:34:32.337 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:32.337 | INFO | "malloc" : " 14 GB", | 20260508 08:34:32.337 | INFO | "pinned" : " 29 GB" | 20260508 08:34:32.337 | INFO | } | 20260508 08:34:32.337 | INFO | Perf report for the last 5.1379 seconds: | 20260508 08:34:32.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:32.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.337 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 08:34:32.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.338 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 08:34:32.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:32.338 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:34:32.338 | INFO | Total mem usage: 16.7 GiB | 20260508 08:34:32.338 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:34:32.338 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:34:32.338 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:34:32.338 | INFO | High Water Breakdown: | 20260508 08:34:32.338 | INFO | Uploaded 36GiB to GPU in 2.43386s (14.7913GiB/s) | 20260508 08:34:32.338 | INFO | Downloaded 34GiB to GPU in 2.44469s (13.9077GiB/s) | 20260508 08:34:32.338 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:34:32.338 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:34:32.338 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:34:32.338 | INFO | ------------------------------------------ | 20260508 08:34:32.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:34:32.338 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:34:32.338 | INFO | Total mem usage: 26.5 GiB | 20260508 08:34:32.338 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:34:32.338 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:34:32.338 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:34:32.338 | INFO | High Water Breakdown: | 20260508 08:34:32.338 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:34:32.338 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:34:32.338 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:34:32.338 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:34:32.338 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:34:32.338 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:34:32.338 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:34:32.338 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:34:32.338 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:34:32.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:34:32.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:34:32.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:34:32.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:34:32.338 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:34:32.339 | INFO | ------------------------------------------ | 20260508 08:34:32.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:34:32.339 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:34:32.339 | INFO | Total mem usage: 27.8 GiB | 20260508 08:34:32.339 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:34:32.339 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:34:32.339 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:34:32.339 | INFO | High Water Breakdown: | 20260508 08:34:32.339 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:34:32.339 | INFO | ------------------------------------------ | 20260508 08:34:32.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:34:32.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:34:32.339 | INFO | Total mem usage: 15.7 GiB | 20260508 08:34:32.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:34:32.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:34:32.339 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:34:32.339 | INFO | High Water Breakdown: | 20260508 08:34:32.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:34:32.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:34:32.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:34:32.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:34:32.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:34:32.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:34:32.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:34:32.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:34:32.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:34:32.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:34:32.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:34:32.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:34:32.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:34:32.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:34:32.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:34:32.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:34:32.339 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:34:32.339 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:34:32.339 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:34:32.339 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:34:32.339 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:34:32.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:34:32.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:34:32.339 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:34:32.339 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:34:32.339 | INFO | ------------------------------------------ | 20260508 08:34:32.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:34:32.339 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:34:32.339 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:34:32.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:34:32.339 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 5.22e-01 | 20260508 08:34:32.339 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.25e-02 / 5.79e-03 | 20260508 08:34:32.339 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.46e+00, 5.93e-01 / 2.15e-01 | 20260508 08:34:32.339 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.15e-01 | 20260508 08:34:32.339 | INFO | PerfReport: 15.97%, DME -- 4.18e+01 / 4.53e+01, 3.59e+00 / 1.11e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.87e+00 / 1.43e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.10e-01 / 2.62e-01 | 20260508 08:34:32.339 | INFO | PerfReport: 3.07%, Metrics -- 1.00e+00 / 1.15e+00, 2.50e-01 / 1.33e-01 | 20260508 08:34:32.339 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.34e-01, 4.93e-02 / 2.07e-02 | 20260508 08:34:32.339 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:34:32.339 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:34:32.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:34:32.339 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 24.52%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 24.43%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 30.95%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 2.33%, LockUpload -- 2.47e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 2.42%, LockDownload -- 2.56e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 15.33%, LockCompute -- 1.62e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:34:32.339 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 08:34:32.435 | INFO | Starting a round of Baz IO | 20260508 08:34:35.852 | INFO | Populating RTMetrics report | 20260508 08:34:35.853 | INFO | RTMetrics report populated | 20260508 08:34:35.853 | INFO | Opening tmp RTMetrics json file | 20260508 08:34:35.860 | INFO | Serializing RTMetrics to json | 20260508 08:34:35.861 | INFO | Serialization done, swapping tmp file now... | 20260508 08:34:35.862 | INFO | RTMetrics done | 20260508 08:34:35.922 | INFO | Finished a round of Baz IO: Wrote 1.5 GiB in 3.486 seconds | 20260508 08:34:35.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.41% / 0.00% / 59.59% | 20260508 08:34:35.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289134% / 0% / 8.34465e-05% | 20260508 08:34:35.922 | INFO | Baz IO overall: | 20260508 08:34:35.922 | INFO | BytesWritten : 2942174226150 | 20260508 08:34:35.922 | INFO | NumEvents : 1416309756270 | 20260508 08:34:37.479 | INFO | Analyzing chunk frames = [6709248,6709760) | 20260508 08:34:37.479 | INFO | MemInfo: { | 20260508 08:34:37.479 | INFO | "gpu" : " 17 GB", | 20260508 08:34:37.479 | INFO | "huge" : "", | 20260508 08:34:37.479 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:37.479 | INFO | "malloc" : " 14 GB", | 20260508 08:34:37.479 | INFO | "pinned" : " 29 GB" | 20260508 08:34:37.479 | INFO | } | 20260508 08:34:37.479 | INFO | Perf report for the last 5.14179 seconds: | 20260508 08:34:37.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:37.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:37.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:37.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:37.479 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:34:37.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:37.479 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:34:37.479 | INFO | PRE_HQ : 65.6%, 1.6, 54.8ms, 0 | 20260508 08:34:37.479 | INFO | BAZWRITER : 67.2%, 2.5, 87.7ms, 0 | 20260508 08:34:37.479 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:34:37.479 | INFO | Uploaded 36GiB to GPU in 2.59363s (13.8801GiB/s) | 20260508 08:34:37.480 | INFO | Downloaded 38GiB to GPU in 2.7231s (13.9547GiB/s) | 20260508 08:34:42.617 | INFO | Analyzing chunk frames = [6709760,6710272) | 20260508 08:34:42.617 | INFO | MemInfo: { | 20260508 08:34:42.617 | INFO | "gpu" : " 17 GB", | 20260508 08:34:42.617 | INFO | "huge" : "", | 20260508 08:34:42.617 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:42.617 | INFO | "malloc" : " 14 GB", | 20260508 08:34:42.617 | INFO | "pinned" : " 29 GB" | 20260508 08:34:42.617 | INFO | } | 20260508 08:34:42.617 | INFO | Perf report for the last 5.1383 seconds: | 20260508 08:34:42.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:42.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:34:42.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:34:42.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:42.618 | INFO | Uploaded 36GiB to GPU in 2.51442s (14.3174GiB/s) | 20260508 08:34:42.618 | INFO | Downloaded 34GiB to GPU in 2.45035s (13.8756GiB/s) | 20260508 08:34:47.755 | INFO | Analyzing chunk frames = [6710272,6710784) | 20260508 08:34:47.755 | INFO | MemInfo: { | 20260508 08:34:47.755 | INFO | "gpu" : " 17 GB", | 20260508 08:34:47.755 | INFO | "huge" : "", | 20260508 08:34:47.755 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:47.755 | INFO | "malloc" : " 14 GB", | 20260508 08:34:47.755 | INFO | "pinned" : " 29 GB" | 20260508 08:34:47.755 | INFO | } | 20260508 08:34:47.755 | INFO | Perf report for the last 5.13738 seconds: | 20260508 08:34:47.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:47.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:34:47.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:34:47.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:47.755 | INFO | Uploaded 36GiB to GPU in 2.48741s (14.4729GiB/s) | 20260508 08:34:47.755 | INFO | Downloaded 34GiB to GPU in 2.43517s (13.962GiB/s) | 20260508 08:34:52.889 | INFO | Analyzing chunk frames = [6710784,6711296) | 20260508 08:34:52.889 | INFO | MemInfo: { | 20260508 08:34:52.889 | INFO | "gpu" : " 17 GB", | 20260508 08:34:52.889 | INFO | "huge" : "", | 20260508 08:34:52.889 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:52.889 | INFO | "malloc" : " 14 GB", | 20260508 08:34:52.889 | INFO | "pinned" : " 29 GB" | 20260508 08:34:52.889 | INFO | } | 20260508 08:34:52.890 | INFO | Perf report for the last 5.13468 seconds: | 20260508 08:34:52.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:52.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 08:34:52.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:34:52.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:52.890 | INFO | Uploaded 36GiB to GPU in 2.58977s (13.9009GiB/s) | 20260508 08:34:52.890 | INFO | Downloaded 34GiB to GPU in 2.43426s (13.9673GiB/s) | 20260508 08:34:58.032 | INFO | Analyzing chunk frames = [6711296,6711808) | 20260508 08:34:58.032 | INFO | MemInfo: { | 20260508 08:34:58.032 | INFO | "gpu" : " 17 GB", | 20260508 08:34:58.032 | INFO | "huge" : "", | 20260508 08:34:58.032 | INFO | "huge_shared" : " 29 GB", | 20260508 08:34:58.032 | INFO | "malloc" : " 14 GB", | 20260508 08:34:58.032 | INFO | "pinned" : " 29 GB" | 20260508 08:34:58.032 | INFO | } | 20260508 08:34:58.032 | INFO | Perf report for the last 5.14234 seconds: | 20260508 08:34:58.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:34:58.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 08:34:58.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | PRE_HQ : 29.8%, 1.0, 15.6ms, 0 | 20260508 08:34:58.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:34:58.032 | INFO | Uploaded 36GiB to GPU in 2.3151s (15.5501GiB/s) | 20260508 08:34:58.032 | INFO | Downloaded 34GiB to GPU in 2.49988s (13.6006GiB/s) | 20260508 08:35:03.173 | INFO | Analyzing chunk frames = [6711808,6712320) | 20260508 08:35:03.173 | INFO | MemInfo: { | 20260508 08:35:03.173 | INFO | "gpu" : " 17 GB", | 20260508 08:35:03.173 | INFO | "huge" : "", | 20260508 08:35:03.173 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:03.173 | INFO | "malloc" : " 14 GB", | 20260508 08:35:03.173 | INFO | "pinned" : " 29 GB" | 20260508 08:35:03.173 | INFO | } | 20260508 08:35:03.173 | INFO | Perf report for the last 5.14129 seconds: | 20260508 08:35:03.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:03.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.173 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:35:03.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.173 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:35:03.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:03.174 | INFO | Uploaded 36GiB to GPU in 2.4076s (14.9527GiB/s) | 20260508 08:35:03.174 | INFO | Downloaded 34GiB to GPU in 2.41747s (14.0643GiB/s) | 20260508 08:35:08.315 | INFO | Analyzing chunk frames = [6712320,6712832) | 20260508 08:35:08.315 | INFO | MemInfo: { | 20260508 08:35:08.315 | INFO | "gpu" : " 17 GB", | 20260508 08:35:08.315 | INFO | "huge" : "", | 20260508 08:35:08.315 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:08.315 | INFO | "malloc" : " 14 GB", | 20260508 08:35:08.315 | INFO | "pinned" : " 29 GB" | 20260508 08:35:08.315 | INFO | } | 20260508 08:35:08.316 | INFO | Perf report for the last 5.14226 seconds: | 20260508 08:35:08.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:08.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:35:08.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:35:08.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:08.316 | INFO | Uploaded 36GiB to GPU in 2.45281s (14.677GiB/s) | 20260508 08:35:08.316 | INFO | Downloaded 34GiB to GPU in 2.43745s (13.949GiB/s) | 20260508 08:35:13.450 | INFO | Analyzing chunk frames = [6712832,6713344) | 20260508 08:35:13.450 | INFO | MemInfo: { | 20260508 08:35:13.450 | INFO | "gpu" : " 17 GB", | 20260508 08:35:13.450 | INFO | "huge" : "", | 20260508 08:35:13.450 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:13.450 | INFO | "malloc" : " 14 GB", | 20260508 08:35:13.450 | INFO | "pinned" : " 29 GB" | 20260508 08:35:13.450 | INFO | } | 20260508 08:35:13.450 | INFO | Perf report for the last 5.13441 seconds: | 20260508 08:35:13.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:13.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.450 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 08:35:13.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.450 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:35:13.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:13.451 | INFO | Uploaded 36GiB to GPU in 2.44565s (14.72GiB/s) | 20260508 08:35:13.451 | INFO | Downloaded 34GiB to GPU in 2.42592s (14.0153GiB/s) | 20260508 08:35:13.548 | INFO | Starting a round of Baz IO | 20260508 08:35:16.969 | INFO | Populating RTMetrics report | 20260508 08:35:16.969 | INFO | RTMetrics report populated | 20260508 08:35:16.969 | INFO | Opening tmp RTMetrics json file | 20260508 08:35:16.977 | INFO | Serializing RTMetrics to json | 20260508 08:35:16.978 | INFO | Serialization done, swapping tmp file now... | 20260508 08:35:16.979 | INFO | RTMetrics done | 20260508 08:35:17.036 | INFO | Finished a round of Baz IO: Wrote 1.502 GiB in 3.487 seconds | 20260508 08:35:17.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.59% | 20260508 08:35:17.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288733% / 0% / 6.35783e-05% | 20260508 08:35:17.036 | INFO | Baz IO overall: | 20260508 08:35:17.036 | INFO | BytesWritten : 2943786751311 | 20260508 08:35:17.036 | INFO | NumEvents : 1417073938140 | 20260508 08:35:18.587 | INFO | Analyzing chunk frames = [6713344,6713856) | 20260508 08:35:18.587 | INFO | MemInfo: { | 20260508 08:35:18.587 | INFO | "gpu" : " 17 GB", | 20260508 08:35:18.587 | INFO | "huge" : "", | 20260508 08:35:18.587 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:18.587 | INFO | "malloc" : " 14 GB", | 20260508 08:35:18.587 | INFO | "pinned" : " 29 GB" | 20260508 08:35:18.587 | INFO | } | 20260508 08:35:18.587 | INFO | Perf report for the last 5.13742 seconds: | 20260508 08:35:18.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:18.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:18.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:18.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:18.587 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:35:18.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:18.587 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:35:18.587 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 08:35:18.588 | INFO | BAZWRITER : 67.2%, 2.5, 87.9ms, 0 | 20260508 08:35:18.588 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:35:18.588 | INFO | Uploaded 36GiB to GPU in 2.61558s (13.7637GiB/s) | 20260508 08:35:18.588 | INFO | Downloaded 38GiB to GPU in 2.70411s (14.0527GiB/s) | 20260508 08:35:23.729 | INFO | Analyzing chunk frames = [6713856,6714368) | 20260508 08:35:23.729 | INFO | MemInfo: { | 20260508 08:35:23.729 | INFO | "gpu" : " 17 GB", | 20260508 08:35:23.729 | INFO | "huge" : "", | 20260508 08:35:23.729 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:23.729 | INFO | "malloc" : " 14 GB", | 20260508 08:35:23.729 | INFO | "pinned" : " 29 GB" | 20260508 08:35:23.729 | INFO | } | 20260508 08:35:23.729 | INFO | Perf report for the last 5.14167 seconds: | 20260508 08:35:23.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:23.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:35:23.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:35:23.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:23.729 | INFO | Uploaded 36GiB to GPU in 2.502s (14.3885GiB/s) | 20260508 08:35:23.729 | INFO | Downloaded 34GiB to GPU in 2.44393s (13.912GiB/s) | 20260508 08:35:28.867 | INFO | Analyzing chunk frames = [6714368,6714880) | 20260508 08:35:28.867 | INFO | MemInfo: { | 20260508 08:35:28.867 | INFO | "gpu" : " 17 GB", | 20260508 08:35:28.867 | INFO | "huge" : "", | 20260508 08:35:28.868 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:28.868 | INFO | "malloc" : " 14 GB", | 20260508 08:35:28.868 | INFO | "pinned" : " 29 GB" | 20260508 08:35:28.868 | INFO | } | 20260508 08:35:28.868 | INFO | Perf report for the last 5.13862 seconds: | 20260508 08:35:28.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:28.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 08:35:28.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:35:28.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:28.868 | INFO | Uploaded 36GiB to GPU in 2.48926s (14.4622GiB/s) | 20260508 08:35:28.868 | INFO | Downloaded 34GiB to GPU in 2.44776s (13.8902GiB/s) | 20260508 08:35:34.014 | INFO | Analyzing chunk frames = [6714880,6715392) | 20260508 08:35:34.014 | INFO | MemInfo: { | 20260508 08:35:34.014 | INFO | "gpu" : " 17 GB", | 20260508 08:35:34.014 | INFO | "huge" : "", | 20260508 08:35:34.014 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:34.014 | INFO | "malloc" : " 14 GB", | 20260508 08:35:34.014 | INFO | "pinned" : " 29 GB" | 20260508 08:35:34.014 | INFO | } | 20260508 08:35:34.014 | INFO | Perf report for the last 5.14669 seconds: | 20260508 08:35:34.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:34.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.014 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 08:35:34.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.014 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:35:34.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:34.015 | INFO | Uploaded 36GiB to GPU in 2.48235s (14.5024GiB/s) | 20260508 08:35:34.015 | INFO | Downloaded 34GiB to GPU in 2.43324s (13.9731GiB/s) | 20260508 08:35:39.146 | INFO | Analyzing chunk frames = [6715392,6715904) | 20260508 08:35:39.146 | INFO | MemInfo: { | 20260508 08:35:39.146 | INFO | "gpu" : " 17 GB", | 20260508 08:35:39.146 | INFO | "huge" : "", | 20260508 08:35:39.146 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:39.146 | INFO | "malloc" : " 14 GB", | 20260508 08:35:39.146 | INFO | "pinned" : " 29 GB" | 20260508 08:35:39.146 | INFO | } | 20260508 08:35:39.146 | INFO | Perf report for the last 5.13171 seconds: | 20260508 08:35:39.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:39.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 08:35:39.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:35:39.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:39.146 | INFO | Uploaded 36GiB to GPU in 2.3367s (15.4063GiB/s) | 20260508 08:35:39.146 | INFO | Downloaded 34GiB to GPU in 2.51074s (13.5418GiB/s) | 20260508 08:35:44.286 | INFO | Analyzing chunk frames = [6715904,6716416) | 20260508 08:35:44.286 | INFO | MemInfo: { | 20260508 08:35:44.286 | INFO | "gpu" : " 17 GB", | 20260508 08:35:44.286 | INFO | "huge" : "", | 20260508 08:35:44.286 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:44.286 | INFO | "malloc" : " 14 GB", | 20260508 08:35:44.286 | INFO | "pinned" : " 29 GB" | 20260508 08:35:44.286 | INFO | } | 20260508 08:35:44.286 | INFO | Perf report for the last 5.13967 seconds: | 20260508 08:35:44.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:44.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 08:35:44.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:35:44.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:44.286 | INFO | Uploaded 36GiB to GPU in 2.32727s (15.4687GiB/s) | 20260508 08:35:44.286 | INFO | Downloaded 34GiB to GPU in 2.45037s (13.8754GiB/s) | 20260508 08:35:49.428 | INFO | Analyzing chunk frames = [6716416,6716928) | 20260508 08:35:49.428 | INFO | MemInfo: { | 20260508 08:35:49.428 | INFO | "gpu" : " 17 GB", | 20260508 08:35:49.428 | INFO | "huge" : "", | 20260508 08:35:49.428 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:49.428 | INFO | "malloc" : " 14 GB", | 20260508 08:35:49.428 | INFO | "pinned" : " 29 GB" | 20260508 08:35:49.428 | INFO | } | 20260508 08:35:49.428 | INFO | Perf report for the last 5.14233 seconds: | 20260508 08:35:49.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:49.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.428 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 08:35:49.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.428 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:35:49.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:49.429 | INFO | Uploaded 36GiB to GPU in 2.39357s (15.0403GiB/s) | 20260508 08:35:49.429 | INFO | Downloaded 34GiB to GPU in 2.45094s (13.8722GiB/s) | 20260508 08:35:54.567 | INFO | Analyzing chunk frames = [6716928,6717440) | 20260508 08:35:54.567 | INFO | MemInfo: { | 20260508 08:35:54.567 | INFO | "gpu" : " 17 GB", | 20260508 08:35:54.567 | INFO | "huge" : "", | 20260508 08:35:54.567 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:54.567 | INFO | "malloc" : " 14 GB", | 20260508 08:35:54.567 | INFO | "pinned" : " 29 GB" | 20260508 08:35:54.567 | INFO | } | 20260508 08:35:54.567 | INFO | Perf report for the last 5.13875 seconds: | 20260508 08:35:54.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:54.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 08:35:54.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:35:54.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:54.567 | INFO | Uploaded 36GiB to GPU in 2.38292s (15.1075GiB/s) | 20260508 08:35:54.567 | INFO | Downloaded 34GiB to GPU in 2.45189s (13.8669GiB/s) | 20260508 08:35:54.567 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:35:54.567 | INFO | Total mem usage: 16.7 GiB | 20260508 08:35:54.567 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:35:54.567 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:35:54.567 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:35:54.567 | INFO | High Water Breakdown: | 20260508 08:35:54.567 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:35:54.567 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:35:54.567 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:35:54.567 | INFO | ------------------------------------------ | 20260508 08:35:54.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:35:54.567 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:35:54.567 | INFO | Total mem usage: 26.5 GiB | 20260508 08:35:54.567 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:35:54.568 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:35:54.568 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:35:54.568 | INFO | High Water Breakdown: | 20260508 08:35:54.568 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:35:54.568 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:35:54.568 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:35:54.568 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:35:54.568 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:35:54.568 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:35:54.568 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:35:54.568 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:35:54.568 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:35:54.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:35:54.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:35:54.568 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:35:54.568 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:35:54.568 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:35:54.568 | INFO | ------------------------------------------ | 20260508 08:35:54.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:35:54.568 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:35:54.568 | INFO | Total mem usage: 27.8 GiB | 20260508 08:35:54.568 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:35:54.568 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:35:54.568 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:35:54.568 | INFO | High Water Breakdown: | 20260508 08:35:54.568 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:35:54.568 | INFO | ------------------------------------------ | 20260508 08:35:54.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:35:54.568 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:35:54.568 | INFO | Total mem usage: 15.7 GiB | 20260508 08:35:54.568 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:35:54.568 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:35:54.568 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:35:54.568 | INFO | High Water Breakdown: | 20260508 08:35:54.568 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:35:54.568 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:35:54.568 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:35:54.568 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:35:54.568 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:35:54.568 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:35:54.568 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:35:54.568 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:35:54.568 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:35:54.568 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:35:54.568 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:35:54.568 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:35:54.568 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:35:54.568 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:35:54.568 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:35:54.568 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:35:54.568 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:35:54.568 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:35:54.568 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:35:54.568 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:35:54.568 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:35:54.568 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:35:54.568 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:35:54.568 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:35:54.568 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:35:54.568 | INFO | ------------------------------------------ | 20260508 08:35:54.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:35:54.568 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:35:54.568 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:35:54.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:35:54.568 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 7.44e-01 | 20260508 08:35:54.568 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.09e-02 / 4.70e-03 | 20260508 08:35:54.568 | INFO | PerfReport: 12.15%, Baseline -- 3.99e+00 / 4.40e+00, 5.95e-01 / 2.41e-01 | 20260508 08:35:54.568 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.11e-01 / 1.24e-01 | 20260508 08:35:54.568 | INFO | PerfReport: 15.95%, DME -- 4.19e+01 / 4.53e+01, 3.71e+00 / 1.19e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.86e+00 / 1.35e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.56e+00, 4.06e-01 / 2.46e-01 | 20260508 08:35:54.568 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.14e+00, 2.50e-01 / 1.44e-01 | 20260508 08:35:54.568 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.38e-01, 4.87e-02 / 2.90e-02 | 20260508 08:35:54.568 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:35:54.568 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:35:54.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:35:54.568 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 24.27%, Upload -- 2.56e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 24.51%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 31.09%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 2.18%, LockUpload -- 2.30e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 2.53%, LockDownload -- 2.67e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 15.41%, LockCompute -- 1.63e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:35:54.568 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260508 08:35:54.665 | INFO | Starting a round of Baz IO | 20260508 08:35:58.089 | INFO | Populating RTMetrics report | 20260508 08:35:58.089 | INFO | RTMetrics report populated | 20260508 08:35:58.089 | INFO | Opening tmp RTMetrics json file | 20260508 08:35:58.090 | INFO | Serializing RTMetrics to json | 20260508 08:35:58.091 | INFO | Serialization done, swapping tmp file now... | 20260508 08:35:58.092 | INFO | RTMetrics done | 20260508 08:35:58.155 | INFO | Finished a round of Baz IO: Wrote 1.502 GiB in 3.49 seconds | 20260508 08:35:58.155 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.60% | 20260508 08:35:58.155 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289313% / 0% / 8.34465e-05% | 20260508 08:35:58.155 | INFO | Baz IO overall: | 20260508 08:35:58.155 | INFO | BytesWritten : 2945399117780 | 20260508 08:35:58.155 | INFO | NumEvents : 1417838147028 | 20260508 08:35:59.705 | INFO | Analyzing chunk frames = [6717440,6717952) | 20260508 08:35:59.705 | INFO | MemInfo: { | 20260508 08:35:59.705 | INFO | "gpu" : " 17 GB", | 20260508 08:35:59.705 | INFO | "huge" : "", | 20260508 08:35:59.705 | INFO | "huge_shared" : " 29 GB", | 20260508 08:35:59.705 | INFO | "malloc" : " 14 GB", | 20260508 08:35:59.705 | INFO | "pinned" : " 29 GB" | 20260508 08:35:59.705 | INFO | } | 20260508 08:35:59.705 | INFO | Perf report for the last 5.1381 seconds: | 20260508 08:35:59.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:35:59.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:59.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:59.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:59.705 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 08:35:59.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:35:59.705 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:35:59.705 | INFO | PRE_HQ : 65.8%, 1.6, 54.9ms, 0 | 20260508 08:35:59.705 | INFO | BAZWRITER : 67.2%, 2.5, 87.3ms, 0 | 20260508 08:35:59.705 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 08:35:59.705 | INFO | Uploaded 36GiB to GPU in 2.52019s (14.2847GiB/s) | 20260508 08:35:59.705 | INFO | Downloaded 38GiB to GPU in 2.71651s (13.9885GiB/s) | 20260508 08:36:04.840 | INFO | Analyzing chunk frames = [6717952,6718464) | 20260508 08:36:04.840 | INFO | MemInfo: { | 20260508 08:36:04.840 | INFO | "gpu" : " 17 GB", | 20260508 08:36:04.840 | INFO | "huge" : "", | 20260508 08:36:04.840 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:04.840 | INFO | "malloc" : " 14 GB", | 20260508 08:36:04.840 | INFO | "pinned" : " 29 GB" | 20260508 08:36:04.840 | INFO | } | 20260508 08:36:04.840 | INFO | Perf report for the last 5.13484 seconds: | 20260508 08:36:04.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:04.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:36:04.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 08:36:04.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:04.840 | INFO | Uploaded 36GiB to GPU in 2.47395s (14.5516GiB/s) | 20260508 08:36:04.840 | INFO | Downloaded 34GiB to GPU in 2.45721s (13.8369GiB/s) | 20260508 08:36:09.983 | INFO | Analyzing chunk frames = [6718464,6718976) | 20260508 08:36:09.983 | INFO | MemInfo: { | 20260508 08:36:09.983 | INFO | "gpu" : " 17 GB", | 20260508 08:36:09.983 | INFO | "huge" : "", | 20260508 08:36:09.983 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:09.983 | INFO | "malloc" : " 14 GB", | 20260508 08:36:09.983 | INFO | "pinned" : " 29 GB" | 20260508 08:36:09.983 | INFO | } | 20260508 08:36:09.983 | INFO | Perf report for the last 5.14323 seconds: | 20260508 08:36:09.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:09.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | ANALYSIS : 66.9%, 3.0, 103.9ms, 0 | 20260508 08:36:09.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:36:09.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:09.983 | INFO | Uploaded 36GiB to GPU in 2.53902s (14.1787GiB/s) | 20260508 08:36:09.983 | INFO | Downloaded 34GiB to GPU in 2.44859s (13.8855GiB/s) | 20260508 08:36:15.127 | INFO | Analyzing chunk frames = [6718976,6719488) | 20260508 08:36:15.127 | INFO | MemInfo: { | 20260508 08:36:15.127 | INFO | "gpu" : " 17 GB", | 20260508 08:36:15.127 | INFO | "huge" : "", | 20260508 08:36:15.127 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:15.127 | INFO | "malloc" : " 14 GB", | 20260508 08:36:15.127 | INFO | "pinned" : " 29 GB" | 20260508 08:36:15.128 | INFO | } | 20260508 08:36:15.128 | INFO | Perf report for the last 5.14444 seconds: | 20260508 08:36:15.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:15.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 08:36:15.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:36:15.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:15.128 | INFO | Uploaded 36GiB to GPU in 2.54298s (14.1566GiB/s) | 20260508 08:36:15.128 | INFO | Downloaded 34GiB to GPU in 2.44863s (13.8853GiB/s) | 20260508 08:36:20.263 | INFO | Analyzing chunk frames = [6719488,6720000) | 20260508 08:36:20.263 | INFO | MemInfo: { | 20260508 08:36:20.263 | INFO | "gpu" : " 17 GB", | 20260508 08:36:20.263 | INFO | "huge" : "", | 20260508 08:36:20.263 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:20.263 | INFO | "malloc" : " 14 GB", | 20260508 08:36:20.263 | INFO | "pinned" : " 29 GB" | 20260508 08:36:20.263 | INFO | } | 20260508 08:36:20.263 | INFO | Perf report for the last 5.13589 seconds: | 20260508 08:36:20.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:20.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.264 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:36:20.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.264 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:36:20.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:20.264 | INFO | Uploaded 36GiB to GPU in 2.38504s (15.0941GiB/s) | 20260508 08:36:20.264 | INFO | Downloaded 34GiB to GPU in 2.52484s (13.4662GiB/s) | 20260508 08:36:25.398 | INFO | Analyzing chunk frames = [6720000,6720512) | 20260508 08:36:25.398 | INFO | MemInfo: { | 20260508 08:36:25.398 | INFO | "gpu" : " 17 GB", | 20260508 08:36:25.398 | INFO | "huge" : "", | 20260508 08:36:25.398 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:25.398 | INFO | "malloc" : " 14 GB", | 20260508 08:36:25.398 | INFO | "pinned" : " 29 GB" | 20260508 08:36:25.398 | INFO | } | 20260508 08:36:25.398 | INFO | Perf report for the last 5.13465 seconds: | 20260508 08:36:25.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:25.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:36:25.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:36:25.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:25.398 | INFO | Uploaded 36GiB to GPU in 2.36254s (15.2378GiB/s) | 20260508 08:36:25.398 | INFO | Downloaded 34GiB to GPU in 2.45033s (13.8757GiB/s) | 20260508 08:36:30.542 | INFO | Analyzing chunk frames = [6720512,6721024) | 20260508 08:36:30.542 | INFO | MemInfo: { | 20260508 08:36:30.542 | INFO | "gpu" : " 17 GB", | 20260508 08:36:30.542 | INFO | "huge" : "", | 20260508 08:36:30.542 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:30.543 | INFO | "malloc" : " 14 GB", | 20260508 08:36:30.543 | INFO | "pinned" : " 29 GB" | 20260508 08:36:30.543 | INFO | } | 20260508 08:36:30.543 | INFO | Perf report for the last 5.14452 seconds: | 20260508 08:36:30.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:30.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 08:36:30.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:36:30.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:30.543 | INFO | Uploaded 36GiB to GPU in 2.37407s (15.1639GiB/s) | 20260508 08:36:30.543 | INFO | Downloaded 34GiB to GPU in 2.44691s (13.8951GiB/s) | 20260508 08:36:35.680 | INFO | Analyzing chunk frames = [6721024,6721536) | 20260508 08:36:35.680 | INFO | MemInfo: { | 20260508 08:36:35.680 | INFO | "gpu" : " 17 GB", | 20260508 08:36:35.680 | INFO | "huge" : "", | 20260508 08:36:35.680 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:35.680 | INFO | "malloc" : " 14 GB", | 20260508 08:36:35.680 | INFO | "pinned" : " 29 GB" | 20260508 08:36:35.680 | INFO | } | 20260508 08:36:35.680 | INFO | Perf report for the last 5.13759 seconds: | 20260508 08:36:35.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:35.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.680 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 08:36:35.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.680 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:36:35.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:35.681 | INFO | Uploaded 36GiB to GPU in 2.39005s (15.0624GiB/s) | 20260508 08:36:35.681 | INFO | Downloaded 34GiB to GPU in 2.44497s (13.9061GiB/s) | 20260508 08:36:35.777 | INFO | Starting a round of Baz IO | 20260508 08:36:39.207 | INFO | Populating RTMetrics report | 20260508 08:36:39.207 | INFO | RTMetrics report populated | 20260508 08:36:39.207 | INFO | Opening tmp RTMetrics json file | 20260508 08:36:39.216 | INFO | Serializing RTMetrics to json | 20260508 08:36:39.218 | INFO | Serialization done, swapping tmp file now... | 20260508 08:36:39.218 | INFO | RTMetrics done | 20260508 08:36:39.272 | INFO | Finished a round of Baz IO: Wrote 1.501 GiB in 3.495 seconds | 20260508 08:36:39.272 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.60% | 20260508 08:36:39.272 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288832% / 0% / 9.13938e-05% | 20260508 08:36:39.272 | INFO | Baz IO overall: | 20260508 08:36:39.272 | INFO | BytesWritten : 2947010953698 | 20260508 08:36:39.273 | INFO | NumEvents : 1418601810859 | 20260508 08:36:40.819 | INFO | Analyzing chunk frames = [6721536,6722048) | 20260508 08:36:40.819 | INFO | MemInfo: { | 20260508 08:36:40.819 | INFO | "gpu" : " 17 GB", | 20260508 08:36:40.819 | INFO | "huge" : "", | 20260508 08:36:40.819 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:40.819 | INFO | "malloc" : " 14 GB", | 20260508 08:36:40.819 | INFO | "pinned" : " 29 GB" | 20260508 08:36:40.819 | INFO | } | 20260508 08:36:40.820 | INFO | Perf report for the last 5.13934 seconds: | 20260508 08:36:40.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:40.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:40.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:40.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:40.820 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:36:40.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:40.820 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260508 08:36:40.820 | INFO | PRE_HQ : 65.7%, 1.7, 57.7ms, 0 | 20260508 08:36:40.820 | INFO | BAZWRITER : 67.5%, 2.6, 90.9ms, 0 | 20260508 08:36:40.820 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:36:40.820 | INFO | Uploaded 36GiB to GPU in 2.58557s (13.9234GiB/s) | 20260508 08:36:40.820 | INFO | Downloaded 38GiB to GPU in 2.70578s (14.044GiB/s) | 20260508 08:36:45.961 | INFO | Analyzing chunk frames = [6722048,6722560) | 20260508 08:36:45.961 | INFO | MemInfo: { | 20260508 08:36:45.961 | INFO | "gpu" : " 17 GB", | 20260508 08:36:45.961 | INFO | "huge" : "", | 20260508 08:36:45.961 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:45.961 | INFO | "malloc" : " 14 GB", | 20260508 08:36:45.961 | INFO | "pinned" : " 29 GB" | 20260508 08:36:45.961 | INFO | } | 20260508 08:36:45.961 | INFO | Perf report for the last 5.14117 seconds: | 20260508 08:36:45.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:45.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 08:36:45.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:36:45.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:45.961 | INFO | Uploaded 36GiB to GPU in 2.47833s (14.5259GiB/s) | 20260508 08:36:45.961 | INFO | Downloaded 34GiB to GPU in 2.45225s (13.8648GiB/s) | 20260508 08:36:51.099 | INFO | Analyzing chunk frames = [6722560,6723072) | 20260508 08:36:51.099 | INFO | MemInfo: { | 20260508 08:36:51.099 | INFO | "gpu" : " 17 GB", | 20260508 08:36:51.099 | INFO | "huge" : "", | 20260508 08:36:51.099 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:51.099 | INFO | "malloc" : " 14 GB", | 20260508 08:36:51.099 | INFO | "pinned" : " 29 GB" | 20260508 08:36:51.099 | INFO | } | 20260508 08:36:51.099 | INFO | Perf report for the last 5.13842 seconds: | 20260508 08:36:51.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:51.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.099 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 08:36:51.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.099 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:36:51.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:51.100 | INFO | Uploaded 36GiB to GPU in 2.45451s (14.6669GiB/s) | 20260508 08:36:51.100 | INFO | Downloaded 34GiB to GPU in 2.45391s (13.8554GiB/s) | 20260508 08:36:56.239 | INFO | Analyzing chunk frames = [6723072,6723584) | 20260508 08:36:56.239 | INFO | MemInfo: { | 20260508 08:36:56.239 | INFO | "gpu" : " 17 GB", | 20260508 08:36:56.239 | INFO | "huge" : "", | 20260508 08:36:56.239 | INFO | "huge_shared" : " 29 GB", | 20260508 08:36:56.239 | INFO | "malloc" : " 14 GB", | 20260508 08:36:56.239 | INFO | "pinned" : " 29 GB" | 20260508 08:36:56.239 | INFO | } | 20260508 08:36:56.239 | INFO | Perf report for the last 5.14006 seconds: | 20260508 08:36:56.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:36:56.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.239 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 08:36:56.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.239 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:36:56.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:36:56.240 | INFO | Uploaded 36GiB to GPU in 2.47715s (14.5328GiB/s) | 20260508 08:36:56.240 | INFO | Downloaded 34GiB to GPU in 2.43427s (13.9672GiB/s) | 20260508 08:37:01.377 | INFO | Analyzing chunk frames = [6723584,6724096) | 20260508 08:37:01.377 | INFO | MemInfo: { | 20260508 08:37:01.377 | INFO | "gpu" : " 17 GB", | 20260508 08:37:01.377 | INFO | "huge" : "", | 20260508 08:37:01.377 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:01.377 | INFO | "malloc" : " 14 GB", | 20260508 08:37:01.377 | INFO | "pinned" : " 29 GB" | 20260508 08:37:01.377 | INFO | } | 20260508 08:37:01.378 | INFO | Perf report for the last 5.13822 seconds: | 20260508 08:37:01.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:01.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:37:01.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:37:01.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:01.378 | INFO | Uploaded 36GiB to GPU in 2.26579s (15.8885GiB/s) | 20260508 08:37:01.378 | INFO | Downloaded 34GiB to GPU in 2.51744s (13.5058GiB/s) | 20260508 08:37:06.513 | INFO | Analyzing chunk frames = [6724096,6724608) | 20260508 08:37:06.513 | INFO | MemInfo: { | 20260508 08:37:06.513 | INFO | "gpu" : " 17 GB", | 20260508 08:37:06.513 | INFO | "huge" : "", | 20260508 08:37:06.513 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:06.513 | INFO | "malloc" : " 14 GB", | 20260508 08:37:06.513 | INFO | "pinned" : " 29 GB" | 20260508 08:37:06.513 | INFO | } | 20260508 08:37:06.514 | INFO | Perf report for the last 5.13602 seconds: | 20260508 08:37:06.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:06.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:37:06.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:37:06.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:06.514 | INFO | Uploaded 36GiB to GPU in 2.39176s (15.0517GiB/s) | 20260508 08:37:06.514 | INFO | Downloaded 34GiB to GPU in 2.44097s (13.9289GiB/s) | 20260508 08:37:11.652 | INFO | Analyzing chunk frames = [6724608,6725120) | 20260508 08:37:11.652 | INFO | MemInfo: { | 20260508 08:37:11.652 | INFO | "gpu" : " 17 GB", | 20260508 08:37:11.652 | INFO | "huge" : "", | 20260508 08:37:11.652 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:11.652 | INFO | "malloc" : " 14 GB", | 20260508 08:37:11.652 | INFO | "pinned" : " 29 GB" | 20260508 08:37:11.652 | INFO | } | 20260508 08:37:11.652 | INFO | Perf report for the last 5.1383 seconds: | 20260508 08:37:11.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:11.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:37:11.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:37:11.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:11.652 | INFO | Uploaded 36GiB to GPU in 2.40793s (14.9506GiB/s) | 20260508 08:37:11.652 | INFO | Downloaded 34GiB to GPU in 2.44773s (13.8904GiB/s) | 20260508 08:37:16.796 | INFO | Analyzing chunk frames = [6725120,6725632) | 20260508 08:37:16.796 | INFO | MemInfo: { | 20260508 08:37:16.796 | INFO | "gpu" : " 17 GB", | 20260508 08:37:16.796 | INFO | "huge" : "", | 20260508 08:37:16.796 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:16.796 | INFO | "malloc" : " 14 GB", | 20260508 08:37:16.796 | INFO | "pinned" : " 29 GB" | 20260508 08:37:16.796 | INFO | } | 20260508 08:37:16.796 | INFO | Perf report for the last 5.14391 seconds: | 20260508 08:37:16.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:16.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 08:37:16.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 08:37:16.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:16.796 | INFO | Uploaded 36GiB to GPU in 2.4296s (14.8173GiB/s) | 20260508 08:37:16.796 | INFO | Downloaded 34GiB to GPU in 2.4628s (13.8054GiB/s) | 20260508 08:37:16.796 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:37:16.796 | INFO | Total mem usage: 16.7 GiB | 20260508 08:37:16.796 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:37:16.796 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:37:16.796 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:37:16.796 | INFO | High Water Breakdown: | 20260508 08:37:16.796 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:37:16.796 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:37:16.796 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:37:16.796 | INFO | ------------------------------------------ | 20260508 08:37:16.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:37:16.796 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:37:16.796 | INFO | Total mem usage: 26.5 GiB | 20260508 08:37:16.796 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:37:16.796 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:37:16.796 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:37:16.796 | INFO | High Water Breakdown: | 20260508 08:37:16.796 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:37:16.796 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:37:16.796 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:37:16.796 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:37:16.796 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:37:16.796 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:37:16.797 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:37:16.797 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:37:16.797 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:37:16.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:37:16.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:37:16.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:37:16.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:37:16.797 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:37:16.797 | INFO | ------------------------------------------ | 20260508 08:37:16.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:37:16.797 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:37:16.797 | INFO | Total mem usage: 27.8 GiB | 20260508 08:37:16.797 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:37:16.797 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:37:16.797 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:37:16.797 | INFO | High Water Breakdown: | 20260508 08:37:16.797 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:37:16.797 | INFO | ------------------------------------------ | 20260508 08:37:16.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:37:16.797 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:37:16.797 | INFO | Total mem usage: 15.7 GiB | 20260508 08:37:16.797 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:37:16.797 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:37:16.797 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:37:16.797 | INFO | High Water Breakdown: | 20260508 08:37:16.797 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:37:16.797 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:37:16.797 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:37:16.797 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:37:16.797 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:37:16.797 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:37:16.797 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:37:16.797 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:37:16.797 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:37:16.797 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:37:16.797 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:37:16.797 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:37:16.797 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:37:16.797 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:37:16.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:37:16.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:37:16.797 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:37:16.797 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:37:16.797 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:37:16.797 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:37:16.797 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:37:16.797 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:37:16.797 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:37:16.797 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:37:16.797 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:37:16.797 | INFO | ------------------------------------------ | 20260508 08:37:16.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:37:16.797 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:37:16.797 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:37:16.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:37:16.797 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.38e+01 / 7.21e-01 | 20260508 08:37:16.797 | INFO | PerfReport: 0.29%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.09e-02 / 5.40e-03 | 20260508 08:37:16.797 | INFO | PerfReport: 12.12%, Baseline -- 3.98e+00 / 4.34e+00, 5.94e-01 / 3.02e-01 | 20260508 08:37:16.797 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.43e+00, 2.09e-01 / 1.39e-01 | 20260508 08:37:16.797 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.57e+01, 3.85e+00 / 1.23e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.89e+00 / 1.03e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.34e+00 / 2.54e+00, 4.12e-01 / 2.00e-01 | 20260508 08:37:16.797 | INFO | PerfReport: 3.05%, Metrics -- 1.00e+00 / 1.16e+00, 2.51e-01 / 1.05e-01 | 20260508 08:37:16.797 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.42e-01, 4.92e-02 / 1.74e-02 | 20260508 08:37:16.797 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:37:16.797 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:37:16.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:37:16.797 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 24.12%, Upload -- 2.54e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 31.10%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 2.01%, LockUpload -- 2.12e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 2.72%, LockDownload -- 2.87e+00 / 0.00e+00, 4.35e+00 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 15.44%, LockCompute -- 1.63e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 08:37:16.797 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260508 08:37:16.895 | INFO | Starting a round of Baz IO | 20260508 08:37:20.304 | INFO | Populating RTMetrics report | 20260508 08:37:20.304 | INFO | RTMetrics report populated | 20260508 08:37:20.304 | INFO | Opening tmp RTMetrics json file | 20260508 08:37:20.316 | INFO | Serializing RTMetrics to json | 20260508 08:37:20.317 | INFO | Serialization done, swapping tmp file now... | 20260508 08:37:20.317 | INFO | RTMetrics done | 20260508 08:37:20.374 | INFO | Finished a round of Baz IO: Wrote 1.502 GiB in 3.479 seconds | 20260508 08:37:20.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.60% | 20260508 08:37:20.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288868% / 0% / 7.94729e-05% | 20260508 08:37:20.375 | INFO | Baz IO overall: | 20260508 08:37:20.375 | INFO | BytesWritten : 2948623204488 | 20260508 08:37:20.375 | INFO | NumEvents : 1419365822422 | 20260508 08:37:21.938 | INFO | Analyzing chunk frames = [6725632,6726144) | 20260508 08:37:21.938 | INFO | MemInfo: { | 20260508 08:37:21.938 | INFO | "gpu" : " 17 GB", | 20260508 08:37:21.938 | INFO | "huge" : "", | 20260508 08:37:21.938 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:21.938 | INFO | "malloc" : " 14 GB", | 20260508 08:37:21.938 | INFO | "pinned" : " 29 GB" | 20260508 08:37:21.938 | INFO | } | 20260508 08:37:21.938 | INFO | Perf report for the last 5.14252 seconds: | 20260508 08:37:21.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:21.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:21.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:21.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:21.938 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 08:37:21.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:21.938 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:37:21.938 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 08:37:21.938 | INFO | BAZWRITER : 66.9%, 2.5, 87.4ms, 0 | 20260508 08:37:21.938 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:37:21.939 | INFO | Uploaded 36GiB to GPU in 2.6068s (13.8101GiB/s) | 20260508 08:37:21.939 | INFO | Downloaded 38GiB to GPU in 2.69769s (14.0861GiB/s) | 20260508 08:37:27.078 | INFO | Analyzing chunk frames = [6726144,6726656) | 20260508 08:37:27.078 | INFO | MemInfo: { | 20260508 08:37:27.078 | INFO | "gpu" : " 17 GB", | 20260508 08:37:27.078 | INFO | "huge" : "", | 20260508 08:37:27.078 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:27.078 | INFO | "malloc" : " 14 GB", | 20260508 08:37:27.078 | INFO | "pinned" : " 29 GB" | 20260508 08:37:27.078 | INFO | } | 20260508 08:37:27.078 | INFO | Perf report for the last 5.14005 seconds: | 20260508 08:37:27.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:27.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.078 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:37:27.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.078 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:37:27.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:27.079 | INFO | Uploaded 36GiB to GPU in 2.54905s (14.1229GiB/s) | 20260508 08:37:27.079 | INFO | Downloaded 34GiB to GPU in 2.45108s (13.8714GiB/s) | 20260508 08:37:32.211 | INFO | Analyzing chunk frames = [6726656,6727168) | 20260508 08:37:32.211 | INFO | MemInfo: { | 20260508 08:37:32.211 | INFO | "gpu" : " 17 GB", | 20260508 08:37:32.211 | INFO | "huge" : "", | 20260508 08:37:32.211 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:32.211 | INFO | "malloc" : " 14 GB", | 20260508 08:37:32.211 | INFO | "pinned" : " 29 GB" | 20260508 08:37:32.211 | INFO | } | 20260508 08:37:32.211 | INFO | Perf report for the last 5.13278 seconds: | 20260508 08:37:32.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:32.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.211 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 08:37:32.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.211 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:37:32.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:32.212 | INFO | Uploaded 36GiB to GPU in 2.48237s (14.5023GiB/s) | 20260508 08:37:32.212 | INFO | Downloaded 34GiB to GPU in 2.40809s (14.1191GiB/s) | 20260508 08:37:37.350 | INFO | Analyzing chunk frames = [6727168,6727680) | 20260508 08:37:37.350 | INFO | MemInfo: { | 20260508 08:37:37.350 | INFO | "gpu" : " 17 GB", | 20260508 08:37:37.350 | INFO | "huge" : "", | 20260508 08:37:37.350 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:37.350 | INFO | "malloc" : " 14 GB", | 20260508 08:37:37.350 | INFO | "pinned" : " 29 GB" | 20260508 08:37:37.350 | INFO | } | 20260508 08:37:37.350 | INFO | Perf report for the last 5.13887 seconds: | 20260508 08:37:37.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:37.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 08:37:37.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:37:37.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:37.350 | INFO | Uploaded 36GiB to GPU in 2.60208s (13.8351GiB/s) | 20260508 08:37:37.350 | INFO | Downloaded 34GiB to GPU in 2.42288s (14.0329GiB/s) | 20260508 08:37:42.488 | INFO | Analyzing chunk frames = [6727680,6728192) | 20260508 08:37:42.488 | INFO | MemInfo: { | 20260508 08:37:42.488 | INFO | "gpu" : " 17 GB", | 20260508 08:37:42.488 | INFO | "huge" : "", | 20260508 08:37:42.488 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:42.488 | INFO | "malloc" : " 14 GB", | 20260508 08:37:42.488 | INFO | "pinned" : " 29 GB" | 20260508 08:37:42.488 | INFO | } | 20260508 08:37:42.489 | INFO | Perf report for the last 5.13845 seconds: | 20260508 08:37:42.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:42.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:37:42.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:37:42.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:42.489 | INFO | Uploaded 36GiB to GPU in 2.43252s (14.7995GiB/s) | 20260508 08:37:42.489 | INFO | Downloaded 34GiB to GPU in 2.50869s (13.5529GiB/s) | 20260508 08:37:47.632 | INFO | Analyzing chunk frames = [6728192,6728704) | 20260508 08:37:47.632 | INFO | MemInfo: { | 20260508 08:37:47.632 | INFO | "gpu" : " 17 GB", | 20260508 08:37:47.632 | INFO | "huge" : "", | 20260508 08:37:47.633 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:47.633 | INFO | "malloc" : " 14 GB", | 20260508 08:37:47.633 | INFO | "pinned" : " 29 GB" | 20260508 08:37:47.633 | INFO | } | 20260508 08:37:47.633 | INFO | Perf report for the last 5.14407 seconds: | 20260508 08:37:47.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:47.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 08:37:47.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:37:47.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:47.633 | INFO | Uploaded 36GiB to GPU in 2.47135s (14.567GiB/s) | 20260508 08:37:47.633 | INFO | Downloaded 34GiB to GPU in 2.43528s (13.9614GiB/s) | 20260508 08:37:52.778 | INFO | Analyzing chunk frames = [6728704,6729216) | 20260508 08:37:52.778 | INFO | MemInfo: { | 20260508 08:37:52.778 | INFO | "gpu" : " 17 GB", | 20260508 08:37:52.778 | INFO | "huge" : "", | 20260508 08:37:52.778 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:52.778 | INFO | "malloc" : " 14 GB", | 20260508 08:37:52.778 | INFO | "pinned" : " 29 GB" | 20260508 08:37:52.778 | INFO | } | 20260508 08:37:52.778 | INFO | Perf report for the last 5.14573 seconds: | 20260508 08:37:52.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:52.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.778 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 08:37:52.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.778 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:37:52.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:52.779 | INFO | Uploaded 36GiB to GPU in 2.45438s (14.6676GiB/s) | 20260508 08:37:52.779 | INFO | Downloaded 34GiB to GPU in 2.45096s (13.8721GiB/s) | 20260508 08:37:57.909 | INFO | Analyzing chunk frames = [6729216,6729728) | 20260508 08:37:57.909 | INFO | MemInfo: { | 20260508 08:37:57.909 | INFO | "gpu" : " 17 GB", | 20260508 08:37:57.909 | INFO | "huge" : "", | 20260508 08:37:57.909 | INFO | "huge_shared" : " 29 GB", | 20260508 08:37:57.909 | INFO | "malloc" : " 14 GB", | 20260508 08:37:57.909 | INFO | "pinned" : " 29 GB" | 20260508 08:37:57.909 | INFO | } | 20260508 08:37:57.909 | INFO | Perf report for the last 5.13078 seconds: | 20260508 08:37:57.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:37:57.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 08:37:57.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:37:57.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:37:57.909 | INFO | Uploaded 36GiB to GPU in 2.43984s (14.7551GiB/s) | 20260508 08:37:57.909 | INFO | Downloaded 34GiB to GPU in 2.44815s (13.888GiB/s) | 20260508 08:37:58.006 | INFO | Starting a round of Baz IO | 20260508 08:38:01.415 | INFO | Populating RTMetrics report | 20260508 08:38:01.415 | INFO | RTMetrics report populated | 20260508 08:38:01.415 | INFO | Opening tmp RTMetrics json file | 20260508 08:38:01.423 | INFO | Serializing RTMetrics to json | 20260508 08:38:01.429 | INFO | Serialization done, swapping tmp file now... | 20260508 08:38:01.429 | INFO | RTMetrics done | 20260508 08:38:01.487 | INFO | Finished a round of Baz IO: Wrote 1.5 GiB in 3.48 seconds | 20260508 08:38:01.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.60% | 20260508 08:38:01.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288161% / 3.97364e-06% / 5.5631e-05% | 20260508 08:38:01.487 | INFO | Baz IO overall: | 20260508 08:38:01.487 | INFO | BytesWritten : 2950233686017 | 20260508 08:38:01.487 | INFO | NumEvents : 1420128496030 | 20260508 08:38:03.053 | INFO | Analyzing chunk frames = [6729728,6730240) | 20260508 08:38:03.053 | INFO | MemInfo: { | 20260508 08:38:03.053 | INFO | "gpu" : " 17 GB", | 20260508 08:38:03.053 | INFO | "huge" : "", | 20260508 08:38:03.054 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:03.054 | INFO | "malloc" : " 14 GB", | 20260508 08:38:03.054 | INFO | "pinned" : " 29 GB" | 20260508 08:38:03.054 | INFO | } | 20260508 08:38:03.054 | INFO | Perf report for the last 5.14443 seconds: | 20260508 08:38:03.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:03.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:03.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:03.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:03.054 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 08:38:03.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:03.054 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 08:38:03.054 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 08:38:03.054 | INFO | BAZWRITER : 67.0%, 2.6, 88.0ms, 0 | 20260508 08:38:03.054 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:38:03.054 | INFO | Uploaded 36GiB to GPU in 2.5877s (13.9119GiB/s) | 20260508 08:38:03.054 | INFO | Downloaded 38GiB to GPU in 2.71471s (13.9978GiB/s) | 20260508 08:38:08.189 | INFO | Analyzing chunk frames = [6730240,6730752) | 20260508 08:38:08.189 | INFO | MemInfo: { | 20260508 08:38:08.189 | INFO | "gpu" : " 17 GB", | 20260508 08:38:08.189 | INFO | "huge" : "", | 20260508 08:38:08.189 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:08.189 | INFO | "malloc" : " 14 GB", | 20260508 08:38:08.189 | INFO | "pinned" : " 29 GB" | 20260508 08:38:08.189 | INFO | } | 20260508 08:38:08.189 | INFO | Perf report for the last 5.13576 seconds: | 20260508 08:38:08.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:08.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.189 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:38:08.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.190 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:38:08.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:08.190 | INFO | Uploaded 36GiB to GPU in 2.48056s (14.5128GiB/s) | 20260508 08:38:08.190 | INFO | Downloaded 34GiB to GPU in 2.47411s (13.7423GiB/s) | 20260508 08:38:13.329 | INFO | Analyzing chunk frames = [6730752,6731264) | 20260508 08:38:13.329 | INFO | MemInfo: { | 20260508 08:38:13.329 | INFO | "gpu" : " 17 GB", | 20260508 08:38:13.329 | INFO | "huge" : "", | 20260508 08:38:13.329 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:13.329 | INFO | "malloc" : " 14 GB", | 20260508 08:38:13.329 | INFO | "pinned" : " 29 GB" | 20260508 08:38:13.329 | INFO | } | 20260508 08:38:13.329 | INFO | Perf report for the last 5.1394 seconds: | 20260508 08:38:13.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:13.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 08:38:13.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:38:13.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:13.329 | INFO | Uploaded 36GiB to GPU in 2.45682s (14.6531GiB/s) | 20260508 08:38:13.329 | INFO | Downloaded 34GiB to GPU in 2.45002s (13.8774GiB/s) | 20260508 08:38:18.468 | INFO | Analyzing chunk frames = [6731264,6731776) | 20260508 08:38:18.468 | INFO | MemInfo: { | 20260508 08:38:18.468 | INFO | "gpu" : " 17 GB", | 20260508 08:38:18.468 | INFO | "huge" : "", | 20260508 08:38:18.468 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:18.468 | INFO | "malloc" : " 14 GB", | 20260508 08:38:18.468 | INFO | "pinned" : " 29 GB" | 20260508 08:38:18.468 | INFO | } | 20260508 08:38:18.468 | INFO | Perf report for the last 5.13957 seconds: | 20260508 08:38:18.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:18.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.468 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 08:38:18.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.468 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 08:38:18.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:18.469 | INFO | Uploaded 36GiB to GPU in 2.47262s (14.5595GiB/s) | 20260508 08:38:18.469 | INFO | Downloaded 34GiB to GPU in 2.43955s (13.937GiB/s) | 20260508 08:38:23.604 | INFO | Analyzing chunk frames = [6731776,6732288) | 20260508 08:38:23.604 | INFO | MemInfo: { | 20260508 08:38:23.604 | INFO | "gpu" : " 17 GB", | 20260508 08:38:23.604 | INFO | "huge" : "", | 20260508 08:38:23.605 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:23.605 | INFO | "malloc" : " 14 GB", | 20260508 08:38:23.605 | INFO | "pinned" : " 29 GB" | 20260508 08:38:23.605 | INFO | } | 20260508 08:38:23.605 | INFO | Perf report for the last 5.13674 seconds: | 20260508 08:38:23.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:23.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.605 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:38:23.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.605 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:38:23.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:23.606 | INFO | Uploaded 36GiB to GPU in 2.38196s (15.1136GiB/s) | 20260508 08:38:23.606 | INFO | Downloaded 34GiB to GPU in 2.51523s (13.5177GiB/s) | 20260508 08:38:28.748 | INFO | Analyzing chunk frames = [6732288,6732800) | 20260508 08:38:28.748 | INFO | MemInfo: { | 20260508 08:38:28.748 | INFO | "gpu" : " 17 GB", | 20260508 08:38:28.748 | INFO | "huge" : "", | 20260508 08:38:28.748 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:28.748 | INFO | "malloc" : " 14 GB", | 20260508 08:38:28.748 | INFO | "pinned" : " 29 GB" | 20260508 08:38:28.748 | INFO | } | 20260508 08:38:28.748 | INFO | Perf report for the last 5.14311 seconds: | 20260508 08:38:28.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:28.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.748 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 08:38:28.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.748 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:38:28.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:28.749 | INFO | Uploaded 36GiB to GPU in 2.36282s (15.2361GiB/s) | 20260508 08:38:28.749 | INFO | Downloaded 34GiB to GPU in 2.43659s (13.9539GiB/s) | 20260508 08:38:33.887 | INFO | Analyzing chunk frames = [6732800,6733312) | 20260508 08:38:33.887 | INFO | MemInfo: { | 20260508 08:38:33.887 | INFO | "gpu" : " 17 GB", | 20260508 08:38:33.887 | INFO | "huge" : "", | 20260508 08:38:33.887 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:33.887 | INFO | "malloc" : " 14 GB", | 20260508 08:38:33.887 | INFO | "pinned" : " 29 GB" | 20260508 08:38:33.887 | INFO | } | 20260508 08:38:33.887 | INFO | Perf report for the last 5.13908 seconds: | 20260508 08:38:33.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:33.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.887 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:38:33.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.887 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:38:33.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:33.888 | INFO | Uploaded 36GiB to GPU in 2.41225s (14.9238GiB/s) | 20260508 08:38:33.888 | INFO | Downloaded 34GiB to GPU in 2.44771s (13.8905GiB/s) | 20260508 08:38:39.024 | INFO | Analyzing chunk frames = [6733312,6733824) | 20260508 08:38:39.024 | INFO | MemInfo: { | 20260508 08:38:39.024 | INFO | "gpu" : " 17 GB", | 20260508 08:38:39.024 | INFO | "huge" : "", | 20260508 08:38:39.024 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:39.024 | INFO | "malloc" : " 14 GB", | 20260508 08:38:39.024 | INFO | "pinned" : " 29 GB" | 20260508 08:38:39.024 | INFO | } | 20260508 08:38:39.024 | INFO | Perf report for the last 5.13662 seconds: | 20260508 08:38:39.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:39.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 08:38:39.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:38:39.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:39.024 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:38:39.024 | INFO | Total mem usage: 16.7 GiB | 20260508 08:38:39.024 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:38:39.024 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:38:39.024 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:38:39.024 | INFO | High Water Breakdown: | 20260508 08:38:39.024 | INFO | Uploaded 36GiB to GPU in 2.41876s (14.8837GiB/s) | 20260508 08:38:39.024 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:38:39.025 | INFO | Downloaded 34GiB to GPU in 2.44827s (13.8874GiB/s) | 20260508 08:38:39.025 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:38:39.025 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:38:39.025 | INFO | ------------------------------------------ | 20260508 08:38:39.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:38:39.025 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:38:39.025 | INFO | Total mem usage: 26.5 GiB | 20260508 08:38:39.025 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:38:39.025 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:38:39.025 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:38:39.025 | INFO | High Water Breakdown: | 20260508 08:38:39.025 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:38:39.025 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:38:39.025 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:38:39.025 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:38:39.025 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:38:39.025 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:38:39.025 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:38:39.025 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:38:39.025 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:38:39.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:38:39.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:38:39.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:38:39.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:38:39.025 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:38:39.025 | INFO | ------------------------------------------ | 20260508 08:38:39.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:38:39.025 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:38:39.025 | INFO | Total mem usage: 27.8 GiB | 20260508 08:38:39.025 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:38:39.025 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:38:39.025 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:38:39.025 | INFO | High Water Breakdown: | 20260508 08:38:39.025 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:38:39.025 | INFO | ------------------------------------------ | 20260508 08:38:39.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:38:39.025 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:38:39.025 | INFO | Total mem usage: 15.7 GiB | 20260508 08:38:39.025 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:38:39.025 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:38:39.025 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:38:39.025 | INFO | High Water Breakdown: | 20260508 08:38:39.025 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:38:39.025 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:38:39.025 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:38:39.025 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:38:39.025 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:38:39.025 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:38:39.025 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:38:39.025 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:38:39.025 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:38:39.025 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:38:39.025 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:38:39.025 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:38:39.025 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:38:39.025 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:38:39.025 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:38:39.025 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:38:39.025 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:38:39.025 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:38:39.025 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:38:39.025 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:38:39.025 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:38:39.025 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:38:39.025 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:38:39.025 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:38:39.025 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:38:39.025 | INFO | ------------------------------------------ | 20260508 08:38:39.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:38:39.025 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:38:39.025 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:38:39.025 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:38:39.025 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.39e+01 / 5.85e-01 | 20260508 08:38:39.025 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.05e-01, 1.20e-02 / 7.38e-03 | 20260508 08:38:39.025 | INFO | PerfReport: 12.12%, Baseline -- 3.96e+00 / 4.38e+00, 5.89e-01 / 2.17e-01 | 20260508 08:38:39.025 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.17e-01 | 20260508 08:38:39.025 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.49e+01, 3.42e+00 / 9.04e-01 | 20260508 08:38:39.025 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.83e+00 / 1.18e+00 | 20260508 08:38:39.025 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.50e+00, 4.07e-01 / 2.60e-01 | 20260508 08:38:39.025 | INFO | PerfReport: 3.07%, Metrics -- 1.00e+00 / 1.13e+00, 2.50e-01 / 1.34e-01 | 20260508 08:38:39.025 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.39e-01, 4.89e-02 / 2.82e-02 | 20260508 08:38:39.026 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:38:39.026 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:38:39.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:38:39.026 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 24.43%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 24.52%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 30.96%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 2.21%, LockUpload -- 2.33e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 2.50%, LockDownload -- 2.64e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 15.38%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:38:39.026 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 1.12e-02 / 0.00e+00 | 20260508 08:38:39.122 | INFO | Starting a round of Baz IO | 20260508 08:38:42.551 | INFO | Populating RTMetrics report | 20260508 08:38:42.551 | INFO | RTMetrics report populated | 20260508 08:38:42.551 | INFO | Opening tmp RTMetrics json file | 20260508 08:38:42.561 | INFO | Serializing RTMetrics to json | 20260508 08:38:42.563 | INFO | Serialization done, swapping tmp file now... | 20260508 08:38:42.563 | INFO | RTMetrics done | 20260508 08:38:42.615 | INFO | Finished a round of Baz IO: Wrote 1.5 GiB in 3.493 seconds | 20260508 08:38:42.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.60% | 20260508 08:38:42.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.286043% / 0% / 7.94729e-05% | 20260508 08:38:42.615 | INFO | Baz IO overall: | 20260508 08:38:42.615 | INFO | BytesWritten : 2951844126728 | 20260508 08:38:42.615 | INFO | NumEvents : 1420891170159 | 20260508 08:38:44.158 | INFO | Analyzing chunk frames = [6733824,6734336) | 20260508 08:38:44.158 | INFO | MemInfo: { | 20260508 08:38:44.158 | INFO | "gpu" : " 17 GB", | 20260508 08:38:44.158 | INFO | "huge" : "", | 20260508 08:38:44.158 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:44.158 | INFO | "malloc" : " 14 GB", | 20260508 08:38:44.158 | INFO | "pinned" : " 29 GB" | 20260508 08:38:44.158 | INFO | } | 20260508 08:38:44.158 | INFO | Perf report for the last 5.13379 seconds: | 20260508 08:38:44.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:44.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:44.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:44.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:44.158 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 08:38:44.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:44.158 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:38:44.158 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:38:44.158 | INFO | BAZWRITER : 67.3%, 2.5, 88.0ms, 0 | 20260508 08:38:44.158 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:38:44.158 | INFO | Uploaded 36GiB to GPU in 2.59371s (13.8797GiB/s) | 20260508 08:38:44.158 | INFO | Downloaded 38GiB to GPU in 2.6998s (14.0751GiB/s) | 20260508 08:38:49.305 | INFO | Analyzing chunk frames = [6734336,6734848) | 20260508 08:38:49.305 | INFO | MemInfo: { | 20260508 08:38:49.305 | INFO | "gpu" : " 17 GB", | 20260508 08:38:49.305 | INFO | "huge" : "", | 20260508 08:38:49.305 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:49.305 | INFO | "malloc" : " 14 GB", | 20260508 08:38:49.306 | INFO | "pinned" : " 29 GB" | 20260508 08:38:49.306 | INFO | } | 20260508 08:38:49.306 | INFO | Perf report for the last 5.14791 seconds: | 20260508 08:38:49.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:49.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 08:38:49.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:38:49.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:49.306 | INFO | Uploaded 36GiB to GPU in 2.5272s (14.245GiB/s) | 20260508 08:38:49.306 | INFO | Downloaded 34GiB to GPU in 2.46005s (13.8209GiB/s) | 20260508 08:38:54.444 | INFO | Analyzing chunk frames = [6734848,6735360) | 20260508 08:38:54.444 | INFO | MemInfo: { | 20260508 08:38:54.444 | INFO | "gpu" : " 17 GB", | 20260508 08:38:54.444 | INFO | "huge" : "", | 20260508 08:38:54.444 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:54.444 | INFO | "malloc" : " 14 GB", | 20260508 08:38:54.444 | INFO | "pinned" : " 29 GB" | 20260508 08:38:54.444 | INFO | } | 20260508 08:38:54.444 | INFO | Perf report for the last 5.13844 seconds: | 20260508 08:38:54.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:54.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.444 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 08:38:54.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.444 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 08:38:54.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:54.445 | INFO | Uploaded 36GiB to GPU in 2.47145s (14.5664GiB/s) | 20260508 08:38:54.445 | INFO | Downloaded 34GiB to GPU in 2.44179s (13.9242GiB/s) | 20260508 08:38:59.582 | INFO | Analyzing chunk frames = [6735360,6735872) | 20260508 08:38:59.582 | INFO | MemInfo: { | 20260508 08:38:59.582 | INFO | "gpu" : " 17 GB", | 20260508 08:38:59.582 | INFO | "huge" : "", | 20260508 08:38:59.582 | INFO | "huge_shared" : " 29 GB", | 20260508 08:38:59.582 | INFO | "malloc" : " 14 GB", | 20260508 08:38:59.582 | INFO | "pinned" : " 29 GB" | 20260508 08:38:59.582 | INFO | } | 20260508 08:38:59.582 | INFO | Perf report for the last 5.13775 seconds: | 20260508 08:38:59.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:38:59.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | ANALYSIS : 66.9%, 2.9, 103.5ms, 0 | 20260508 08:38:59.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:38:59.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:38:59.582 | INFO | Uploaded 36GiB to GPU in 2.56482s (14.0361GiB/s) | 20260508 08:38:59.582 | INFO | Downloaded 34GiB to GPU in 2.43817s (13.9449GiB/s) | 20260508 08:39:04.721 | INFO | Analyzing chunk frames = [6735872,6736384) | 20260508 08:39:04.721 | INFO | MemInfo: { | 20260508 08:39:04.721 | INFO | "gpu" : " 17 GB", | 20260508 08:39:04.721 | INFO | "huge" : "", | 20260508 08:39:04.721 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:04.721 | INFO | "malloc" : " 14 GB", | 20260508 08:39:04.721 | INFO | "pinned" : " 29 GB" | 20260508 08:39:04.721 | INFO | } | 20260508 08:39:04.721 | INFO | Perf report for the last 5.13941 seconds: | 20260508 08:39:04.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:04.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.721 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 08:39:04.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.721 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:39:04.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:04.722 | INFO | Uploaded 36GiB to GPU in 2.36167s (15.2434GiB/s) | 20260508 08:39:04.722 | INFO | Downloaded 34GiB to GPU in 2.5388s (13.3922GiB/s) | 20260508 08:39:09.858 | INFO | Analyzing chunk frames = [6736384,6736896) | 20260508 08:39:09.858 | INFO | MemInfo: { | 20260508 08:39:09.858 | INFO | "gpu" : " 17 GB", | 20260508 08:39:09.858 | INFO | "huge" : "", | 20260508 08:39:09.858 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:09.858 | INFO | "malloc" : " 14 GB", | 20260508 08:39:09.858 | INFO | "pinned" : " 29 GB" | 20260508 08:39:09.858 | INFO | } | 20260508 08:39:09.858 | INFO | Perf report for the last 5.13713 seconds: | 20260508 08:39:09.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:09.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 08:39:09.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:39:09.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:09.859 | INFO | Uploaded 36GiB to GPU in 2.33315s (15.4298GiB/s) | 20260508 08:39:09.859 | INFO | Downloaded 34GiB to GPU in 2.45186s (13.867GiB/s) | 20260508 08:39:14.999 | INFO | Analyzing chunk frames = [6736896,6737408) | 20260508 08:39:14.999 | INFO | MemInfo: { | 20260508 08:39:14.999 | INFO | "gpu" : " 17 GB", | 20260508 08:39:14.999 | INFO | "huge" : "", | 20260508 08:39:14.999 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:14.999 | INFO | "malloc" : " 14 GB", | 20260508 08:39:14.999 | INFO | "pinned" : " 29 GB" | 20260508 08:39:14.999 | INFO | } | 20260508 08:39:14.999 | INFO | Perf report for the last 5.14099 seconds: | 20260508 08:39:14.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:14.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:14.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:15.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:15.000 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:39:15.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:15.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:15.000 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:39:15.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:15.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:15.000 | INFO | Uploaded 36GiB to GPU in 2.46638s (14.5963GiB/s) | 20260508 08:39:15.000 | INFO | Downloaded 34GiB to GPU in 2.46627s (13.786GiB/s) | 20260508 08:39:20.142 | INFO | Analyzing chunk frames = [6737408,6737920) | 20260508 08:39:20.142 | INFO | MemInfo: { | 20260508 08:39:20.142 | INFO | "gpu" : " 17 GB", | 20260508 08:39:20.142 | INFO | "huge" : "", | 20260508 08:39:20.142 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:20.142 | INFO | "malloc" : " 14 GB", | 20260508 08:39:20.142 | INFO | "pinned" : " 29 GB" | 20260508 08:39:20.142 | INFO | } | 20260508 08:39:20.142 | INFO | Perf report for the last 5.14229 seconds: | 20260508 08:39:20.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:20.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:39:20.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:39:20.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:20.142 | INFO | Uploaded 36GiB to GPU in 2.41987s (14.8769GiB/s) | 20260508 08:39:20.142 | INFO | Downloaded 34GiB to GPU in 2.46581s (13.7886GiB/s) | 20260508 08:39:20.239 | INFO | Starting a round of Baz IO | 20260508 08:39:23.664 | INFO | Populating RTMetrics report | 20260508 08:39:23.664 | INFO | RTMetrics report populated | 20260508 08:39:23.664 | INFO | Opening tmp RTMetrics json file | 20260508 08:39:23.672 | INFO | Serializing RTMetrics to json | 20260508 08:39:23.674 | INFO | Serialization done, swapping tmp file now... | 20260508 08:39:23.674 | INFO | RTMetrics done | 20260508 08:39:23.729 | INFO | Finished a round of Baz IO: Wrote 1.499 GiB in 3.489 seconds | 20260508 08:39:23.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.00% / 59.60% | 20260508 08:39:23.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.285411% / 0% / 7.15256e-05% | 20260508 08:39:23.729 | INFO | Baz IO overall: | 20260508 08:39:23.729 | INFO | BytesWritten : 2953453622899 | 20260508 08:39:23.729 | INFO | NumEvents : 1421653073244 | 20260508 08:39:25.279 | INFO | Analyzing chunk frames = [6737920,6738432) | 20260508 08:39:25.279 | INFO | MemInfo: { | 20260508 08:39:25.279 | INFO | "gpu" : " 17 GB", | 20260508 08:39:25.279 | INFO | "huge" : "", | 20260508 08:39:25.279 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:25.279 | INFO | "malloc" : " 14 GB", | 20260508 08:39:25.279 | INFO | "pinned" : " 29 GB" | 20260508 08:39:25.279 | INFO | } | 20260508 08:39:25.279 | INFO | Perf report for the last 5.13707 seconds: | 20260508 08:39:25.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:25.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:25.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:25.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:25.279 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:39:25.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:25.279 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 08:39:25.279 | INFO | PRE_HQ : 65.6%, 1.6, 55.8ms, 0 | 20260508 08:39:25.279 | INFO | BAZWRITER : 67.4%, 2.6, 88.7ms, 0 | 20260508 08:39:25.279 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:39:25.279 | INFO | Uploaded 36GiB to GPU in 2.57441s (13.9838GiB/s) | 20260508 08:39:25.279 | INFO | Downloaded 38GiB to GPU in 2.72131s (13.9639GiB/s) | 20260508 08:39:30.416 | INFO | Analyzing chunk frames = [6738432,6738944) | 20260508 08:39:30.416 | INFO | MemInfo: { | 20260508 08:39:30.416 | INFO | "gpu" : " 17 GB", | 20260508 08:39:30.416 | INFO | "huge" : "", | 20260508 08:39:30.416 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:30.416 | INFO | "malloc" : " 14 GB", | 20260508 08:39:30.416 | INFO | "pinned" : " 29 GB" | 20260508 08:39:30.416 | INFO | } | 20260508 08:39:30.416 | INFO | Perf report for the last 5.13707 seconds: | 20260508 08:39:30.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:30.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 08:39:30.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:39:30.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:30.416 | INFO | Uploaded 36GiB to GPU in 2.47617s (14.5386GiB/s) | 20260508 08:39:30.416 | INFO | Downloaded 34GiB to GPU in 2.46033s (13.8193GiB/s) | 20260508 08:39:35.559 | INFO | Analyzing chunk frames = [6738944,6739456) | 20260508 08:39:35.559 | INFO | MemInfo: { | 20260508 08:39:35.559 | INFO | "gpu" : " 17 GB", | 20260508 08:39:35.559 | INFO | "huge" : "", | 20260508 08:39:35.559 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:35.559 | INFO | "malloc" : " 14 GB", | 20260508 08:39:35.559 | INFO | "pinned" : " 29 GB" | 20260508 08:39:35.559 | INFO | } | 20260508 08:39:35.559 | INFO | Perf report for the last 5.14311 seconds: | 20260508 08:39:35.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:35.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.559 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 08:39:35.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.559 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:39:35.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:35.560 | INFO | Uploaded 36GiB to GPU in 2.53937s (14.1767GiB/s) | 20260508 08:39:35.560 | INFO | Downloaded 34GiB to GPU in 2.44758s (13.8913GiB/s) | 20260508 08:39:40.701 | INFO | Analyzing chunk frames = [6739456,6739968) | 20260508 08:39:40.701 | INFO | MemInfo: { | 20260508 08:39:40.701 | INFO | "gpu" : " 17 GB", | 20260508 08:39:40.701 | INFO | "huge" : "", | 20260508 08:39:40.701 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:40.701 | INFO | "malloc" : " 14 GB", | 20260508 08:39:40.701 | INFO | "pinned" : " 29 GB" | 20260508 08:39:40.701 | INFO | } | 20260508 08:39:40.701 | INFO | Perf report for the last 5.14211 seconds: | 20260508 08:39:40.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:40.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 08:39:40.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:39:40.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:40.701 | INFO | Uploaded 36GiB to GPU in 2.46152s (14.6251GiB/s) | 20260508 08:39:40.701 | INFO | Downloaded 34GiB to GPU in 2.44847s (13.8862GiB/s) | 20260508 08:39:45.837 | INFO | Analyzing chunk frames = [6739968,6740480) | 20260508 08:39:45.837 | INFO | MemInfo: { | 20260508 08:39:45.837 | INFO | "gpu" : " 17 GB", | 20260508 08:39:45.837 | INFO | "huge" : "", | 20260508 08:39:45.837 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:45.837 | INFO | "malloc" : " 14 GB", | 20260508 08:39:45.837 | INFO | "pinned" : " 29 GB" | 20260508 08:39:45.837 | INFO | } | 20260508 08:39:45.837 | INFO | Perf report for the last 5.13588 seconds: | 20260508 08:39:45.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:45.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 08:39:45.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:39:45.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:45.837 | INFO | Uploaded 36GiB to GPU in 2.41614s (14.8998GiB/s) | 20260508 08:39:45.837 | INFO | Downloaded 34GiB to GPU in 2.54275s (13.3713GiB/s) | 20260508 08:39:50.973 | INFO | Analyzing chunk frames = [6740480,6740992) | 20260508 08:39:50.973 | INFO | MemInfo: { | 20260508 08:39:50.973 | INFO | "gpu" : " 17 GB", | 20260508 08:39:50.973 | INFO | "huge" : "", | 20260508 08:39:50.973 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:50.973 | INFO | "malloc" : " 14 GB", | 20260508 08:39:50.973 | INFO | "pinned" : " 29 GB" | 20260508 08:39:50.973 | INFO | } | 20260508 08:39:50.973 | INFO | Perf report for the last 5.13613 seconds: | 20260508 08:39:50.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:50.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 08:39:50.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:39:50.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:50.973 | INFO | Uploaded 36GiB to GPU in 2.334s (15.4242GiB/s) | 20260508 08:39:50.974 | INFO | Downloaded 34GiB to GPU in 2.44474s (13.9074GiB/s) | 20260508 08:39:56.115 | INFO | Analyzing chunk frames = [6740992,6741504) | 20260508 08:39:56.115 | INFO | MemInfo: { | 20260508 08:39:56.115 | INFO | "gpu" : " 17 GB", | 20260508 08:39:56.115 | INFO | "huge" : "", | 20260508 08:39:56.115 | INFO | "huge_shared" : " 29 GB", | 20260508 08:39:56.115 | INFO | "malloc" : " 14 GB", | 20260508 08:39:56.115 | INFO | "pinned" : " 29 GB" | 20260508 08:39:56.115 | INFO | } | 20260508 08:39:56.115 | INFO | Perf report for the last 5.1416 seconds: | 20260508 08:39:56.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:39:56.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.115 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:39:56.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.115 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:39:56.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:39:56.116 | INFO | Uploaded 36GiB to GPU in 2.44441s (14.7275GiB/s) | 20260508 08:39:56.116 | INFO | Downloaded 34GiB to GPU in 2.46072s (13.8171GiB/s) | 20260508 08:40:01.254 | INFO | Analyzing chunk frames = [6741504,6742016) | 20260508 08:40:01.254 | INFO | MemInfo: { | 20260508 08:40:01.254 | INFO | "gpu" : " 17 GB", | 20260508 08:40:01.254 | INFO | "huge" : "", | 20260508 08:40:01.255 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:01.255 | INFO | "malloc" : " 14 GB", | 20260508 08:40:01.255 | INFO | "pinned" : " 29 GB" | 20260508 08:40:01.255 | INFO | } | 20260508 08:40:01.255 | INFO | Perf report for the last 5.13994 seconds: | 20260508 08:40:01.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:01.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:40:01.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260508 08:40:01.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:01.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:40:01.255 | INFO | Total mem usage: 16.7 GiB | 20260508 08:40:01.255 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:40:01.255 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:40:01.255 | INFO | Uploaded 36GiB to GPU in 2.39637s (15.0227GiB/s) | 20260508 08:40:01.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:40:01.256 | INFO | High Water Breakdown: | 20260508 08:40:01.256 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:40:01.256 | INFO | Downloaded 34GiB to GPU in 2.45953s (13.8238GiB/s) | 20260508 08:40:01.256 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:40:01.256 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:40:01.256 | INFO | ------------------------------------------ | 20260508 08:40:01.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:40:01.256 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:40:01.256 | INFO | Total mem usage: 26.5 GiB | 20260508 08:40:01.256 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:40:01.256 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:40:01.256 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:40:01.256 | INFO | High Water Breakdown: | 20260508 08:40:01.256 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:40:01.256 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:40:01.256 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:40:01.256 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:40:01.256 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:40:01.256 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:40:01.256 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:40:01.256 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:40:01.256 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:40:01.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:40:01.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:40:01.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:40:01.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:40:01.256 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:40:01.256 | INFO | ------------------------------------------ | 20260508 08:40:01.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:40:01.256 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:40:01.256 | INFO | Total mem usage: 27.8 GiB | 20260508 08:40:01.256 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:40:01.256 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:40:01.256 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:40:01.256 | INFO | High Water Breakdown: | 20260508 08:40:01.256 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:40:01.256 | INFO | ------------------------------------------ | 20260508 08:40:01.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:40:01.256 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:40:01.256 | INFO | Total mem usage: 15.7 GiB | 20260508 08:40:01.256 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:40:01.256 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:40:01.256 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:40:01.256 | INFO | High Water Breakdown: | 20260508 08:40:01.256 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:40:01.256 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:40:01.256 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:40:01.256 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:40:01.256 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:40:01.256 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:40:01.256 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:40:01.256 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:40:01.256 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:40:01.256 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:40:01.256 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:40:01.257 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:40:01.257 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:40:01.257 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:40:01.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:40:01.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:40:01.257 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:40:01.257 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:40:01.257 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:40:01.257 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:40:01.257 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:40:01.257 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:40:01.257 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:40:01.257 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:40:01.257 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:40:01.257 | INFO | ------------------------------------------ | 20260508 08:40:01.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:40:01.257 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:40:01.257 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:40:01.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:40:01.257 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.41e+01 / 5.86e-01 | 20260508 08:40:01.257 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.11e-02 / 5.90e-03 | 20260508 08:40:01.257 | INFO | PerfReport: 12.11%, Baseline -- 3.97e+00 / 4.40e+00, 5.89e-01 / 2.69e-01 | 20260508 08:40:01.257 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.25e-01 | 20260508 08:40:01.257 | INFO | PerfReport: 15.90%, DME -- 4.17e+01 / 4.50e+01, 3.75e+00 / 1.14e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.92e+00 / 1.31e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.32e+00 / 2.48e+00, 4.07e-01 / 2.50e-01 | 20260508 08:40:01.257 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.17e+00, 2.53e-01 / 1.42e-01 | 20260508 08:40:01.257 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.31e-01, 4.93e-02 / 2.22e-02 | 20260508 08:40:01.257 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:40:01.257 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:40:01.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:40:01.257 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 24.26%, Upload -- 2.56e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 24.61%, Download -- 2.60e+01 / 0.00e+00, 8.36e+00 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 31.02%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 2.03%, LockUpload -- 2.15e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 2.68%, LockDownload -- 2.83e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 15.38%, LockCompute -- 1.62e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 08:40:01.257 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 08:40:01.369 | INFO | Starting a round of Baz IO | 20260508 08:40:04.762 | INFO | Populating RTMetrics report | 20260508 08:40:04.763 | INFO | RTMetrics report populated | 20260508 08:40:04.763 | INFO | Opening tmp RTMetrics json file | 20260508 08:40:04.773 | INFO | Serializing RTMetrics to json | 20260508 08:40:04.774 | INFO | Serialization done, swapping tmp file now... | 20260508 08:40:04.774 | INFO | RTMetrics done | 20260508 08:40:04.833 | INFO | Finished a round of Baz IO: Wrote 1.5 GiB in 3.464 seconds | 20260508 08:40:04.833 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.60% | 20260508 08:40:04.833 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.285931% / 0% / 7.54992e-05% | 20260508 08:40:04.833 | INFO | Baz IO overall: | 20260508 08:40:04.833 | INFO | BytesWritten : 2955064173336 | 20260508 08:40:04.833 | INFO | NumEvents : 1422416019298 | 20260508 08:40:06.394 | INFO | Analyzing chunk frames = [6742016,6742528) | 20260508 08:40:06.394 | INFO | MemInfo: { | 20260508 08:40:06.394 | INFO | "gpu" : " 17 GB", | 20260508 08:40:06.394 | INFO | "huge" : "", | 20260508 08:40:06.394 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:06.394 | INFO | "malloc" : " 14 GB", | 20260508 08:40:06.394 | INFO | "pinned" : " 29 GB" | 20260508 08:40:06.394 | INFO | } | 20260508 08:40:06.394 | INFO | Perf report for the last 5.13957 seconds: | 20260508 08:40:06.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:06.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:06.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:06.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:06.395 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 08:40:06.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:06.395 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 08:40:06.395 | INFO | PRE_HQ : 66.4%, 1.8, 62.2ms, 0 | 20260508 08:40:06.395 | INFO | BAZWRITER : 66.7%, 2.7, 94.2ms, 0 | 20260508 08:40:06.395 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:40:06.395 | INFO | Uploaded 36GiB to GPU in 2.57655s (13.9722GiB/s) | 20260508 08:40:06.395 | INFO | Downloaded 38GiB to GPU in 2.7043s (14.0517GiB/s) | 20260508 08:40:11.535 | INFO | Analyzing chunk frames = [6742528,6743040) | 20260508 08:40:11.535 | INFO | MemInfo: { | 20260508 08:40:11.535 | INFO | "gpu" : " 17 GB", | 20260508 08:40:11.535 | INFO | "huge" : "", | 20260508 08:40:11.535 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:11.535 | INFO | "malloc" : " 14 GB", | 20260508 08:40:11.535 | INFO | "pinned" : " 29 GB" | 20260508 08:40:11.535 | INFO | } | 20260508 08:40:11.535 | INFO | Perf report for the last 5.14086 seconds: | 20260508 08:40:11.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:11.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.535 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:40:11.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.535 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:40:11.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:11.536 | INFO | Uploaded 36GiB to GPU in 2.50293s (14.3832GiB/s) | 20260508 08:40:11.536 | INFO | Downloaded 34GiB to GPU in 2.4371s (13.951GiB/s) | 20260508 08:40:16.666 | INFO | Analyzing chunk frames = [6743040,6743552) | 20260508 08:40:16.666 | INFO | MemInfo: { | 20260508 08:40:16.666 | INFO | "gpu" : " 17 GB", | 20260508 08:40:16.666 | INFO | "huge" : "", | 20260508 08:40:16.666 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:16.666 | INFO | "malloc" : " 14 GB", | 20260508 08:40:16.666 | INFO | "pinned" : " 29 GB" | 20260508 08:40:16.666 | INFO | } | 20260508 08:40:16.667 | INFO | Perf report for the last 5.13123 seconds: | 20260508 08:40:16.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:16.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 08:40:16.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:40:16.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:16.667 | INFO | Uploaded 36GiB to GPU in 2.46952s (14.5777GiB/s) | 20260508 08:40:16.667 | INFO | Downloaded 34GiB to GPU in 2.43858s (13.9425GiB/s) | 20260508 08:40:21.813 | INFO | Analyzing chunk frames = [6743552,6744064) | 20260508 08:40:21.813 | INFO | MemInfo: { | 20260508 08:40:21.813 | INFO | "gpu" : " 17 GB", | 20260508 08:40:21.813 | INFO | "huge" : "", | 20260508 08:40:21.813 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:21.813 | INFO | "malloc" : " 14 GB", | 20260508 08:40:21.813 | INFO | "pinned" : " 29 GB" | 20260508 08:40:21.813 | INFO | } | 20260508 08:40:21.813 | INFO | Perf report for the last 5.14637 seconds: | 20260508 08:40:21.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:21.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 08:40:21.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:40:21.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:21.813 | INFO | Uploaded 36GiB to GPU in 2.57857s (13.9612GiB/s) | 20260508 08:40:21.813 | INFO | Downloaded 34GiB to GPU in 2.42725s (14.0076GiB/s) | 20260508 08:40:26.950 | INFO | Analyzing chunk frames = [6744064,6744576) | 20260508 08:40:26.950 | INFO | MemInfo: { | 20260508 08:40:26.950 | INFO | "gpu" : " 17 GB", | 20260508 08:40:26.950 | INFO | "huge" : "", | 20260508 08:40:26.950 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:26.950 | INFO | "malloc" : " 14 GB", | 20260508 08:40:26.950 | INFO | "pinned" : " 29 GB" | 20260508 08:40:26.950 | INFO | } | 20260508 08:40:26.950 | INFO | Perf report for the last 5.13757 seconds: | 20260508 08:40:26.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:26.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.951 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:40:26.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.951 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:40:26.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:26.951 | INFO | Uploaded 36GiB to GPU in 2.3787s (15.1343GiB/s) | 20260508 08:40:26.951 | INFO | Downloaded 34GiB to GPU in 2.50954s (13.5483GiB/s) | 20260508 08:40:32.093 | INFO | Analyzing chunk frames = [6744576,6745088) | 20260508 08:40:32.093 | INFO | MemInfo: { | 20260508 08:40:32.093 | INFO | "gpu" : " 17 GB", | 20260508 08:40:32.093 | INFO | "huge" : "", | 20260508 08:40:32.093 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:32.093 | INFO | "malloc" : " 14 GB", | 20260508 08:40:32.093 | INFO | "pinned" : " 29 GB" | 20260508 08:40:32.093 | INFO | } | 20260508 08:40:32.093 | INFO | Perf report for the last 5.1428 seconds: | 20260508 08:40:32.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:32.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 08:40:32.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:40:32.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:32.093 | INFO | Uploaded 36GiB to GPU in 2.40161s (14.99GiB/s) | 20260508 08:40:32.094 | INFO | Downloaded 34GiB to GPU in 2.43409s (13.9682GiB/s) | 20260508 08:40:37.229 | INFO | Analyzing chunk frames = [6745088,6745600) | 20260508 08:40:37.229 | INFO | MemInfo: { | 20260508 08:40:37.229 | INFO | "gpu" : " 17 GB", | 20260508 08:40:37.229 | INFO | "huge" : "", | 20260508 08:40:37.230 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:37.230 | INFO | "malloc" : " 14 GB", | 20260508 08:40:37.230 | INFO | "pinned" : " 29 GB" | 20260508 08:40:37.230 | INFO | } | 20260508 08:40:37.230 | INFO | Perf report for the last 5.13632 seconds: | 20260508 08:40:37.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:37.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:40:37.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 08:40:37.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:37.230 | INFO | Uploaded 36GiB to GPU in 2.41269s (14.9211GiB/s) | 20260508 08:40:37.230 | INFO | Downloaded 34GiB to GPU in 2.44031s (13.9327GiB/s) | 20260508 08:40:42.370 | INFO | Analyzing chunk frames = [6745600,6746112) | 20260508 08:40:42.370 | INFO | MemInfo: { | 20260508 08:40:42.370 | INFO | "gpu" : " 17 GB", | 20260508 08:40:42.370 | INFO | "huge" : "", | 20260508 08:40:42.370 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:42.370 | INFO | "malloc" : " 14 GB", | 20260508 08:40:42.370 | INFO | "pinned" : " 29 GB" | 20260508 08:40:42.370 | INFO | } | 20260508 08:40:42.370 | INFO | Perf report for the last 5.14056 seconds: | 20260508 08:40:42.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:42.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 08:40:42.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:40:42.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:42.370 | INFO | Uploaded 36GiB to GPU in 2.425s (14.8453GiB/s) | 20260508 08:40:42.371 | INFO | Downloaded 34GiB to GPU in 2.44219s (13.9219GiB/s) | 20260508 08:40:42.468 | INFO | Starting a round of Baz IO | 20260508 08:40:45.892 | INFO | Populating RTMetrics report | 20260508 08:40:45.893 | INFO | RTMetrics report populated | 20260508 08:40:45.893 | INFO | Opening tmp RTMetrics json file | 20260508 08:40:45.900 | INFO | Serializing RTMetrics to json | 20260508 08:40:45.901 | INFO | Serialization done, swapping tmp file now... | 20260508 08:40:45.902 | INFO | RTMetrics done | 20260508 08:40:45.958 | INFO | Finished a round of Baz IO: Wrote 1.5 GiB in 3.49 seconds | 20260508 08:40:45.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:40:45.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.285645% / 0% / 8.74201e-05% | 20260508 08:40:45.958 | INFO | Baz IO overall: | 20260508 08:40:45.958 | INFO | BytesWritten : 2956674522932 | 20260508 08:40:45.958 | INFO | NumEvents : 1423178900076 | 20260508 08:40:47.510 | INFO | Analyzing chunk frames = [6746112,6746624) | 20260508 08:40:47.510 | INFO | MemInfo: { | 20260508 08:40:47.510 | INFO | "gpu" : " 17 GB", | 20260508 08:40:47.510 | INFO | "huge" : "", | 20260508 08:40:47.510 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:47.510 | INFO | "malloc" : " 14 GB", | 20260508 08:40:47.510 | INFO | "pinned" : " 29 GB" | 20260508 08:40:47.510 | INFO | } | 20260508 08:40:47.510 | INFO | Perf report for the last 5.13981 seconds: | 20260508 08:40:47.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:47.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:47.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:47.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:47.510 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:40:47.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:47.510 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 08:40:47.510 | INFO | PRE_HQ : 65.7%, 1.6, 55.3ms, 0 | 20260508 08:40:47.510 | INFO | BAZWRITER : 67.3%, 2.5, 88.1ms, 0 | 20260508 08:40:47.510 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:40:47.510 | INFO | Uploaded 36GiB to GPU in 2.5783s (13.9627GiB/s) | 20260508 08:40:47.510 | INFO | Downloaded 38GiB to GPU in 2.70161s (14.0657GiB/s) | 20260508 08:40:52.651 | INFO | Analyzing chunk frames = [6746624,6747136) | 20260508 08:40:52.651 | INFO | MemInfo: { | 20260508 08:40:52.651 | INFO | "gpu" : " 17 GB", | 20260508 08:40:52.651 | INFO | "huge" : "", | 20260508 08:40:52.651 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:52.651 | INFO | "malloc" : " 14 GB", | 20260508 08:40:52.651 | INFO | "pinned" : " 29 GB" | 20260508 08:40:52.651 | INFO | } | 20260508 08:40:52.651 | INFO | Perf report for the last 5.14094 seconds: | 20260508 08:40:52.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:52.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:40:52.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:40:52.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:52.651 | INFO | Uploaded 36GiB to GPU in 2.51995s (14.286GiB/s) | 20260508 08:40:52.651 | INFO | Downloaded 34GiB to GPU in 2.46036s (13.8191GiB/s) | 20260508 08:40:57.787 | INFO | Analyzing chunk frames = [6747136,6747648) | 20260508 08:40:57.787 | INFO | MemInfo: { | 20260508 08:40:57.787 | INFO | "gpu" : " 17 GB", | 20260508 08:40:57.787 | INFO | "huge" : "", | 20260508 08:40:57.787 | INFO | "huge_shared" : " 29 GB", | 20260508 08:40:57.787 | INFO | "malloc" : " 14 GB", | 20260508 08:40:57.787 | INFO | "pinned" : " 29 GB" | 20260508 08:40:57.787 | INFO | } | 20260508 08:40:57.787 | INFO | Perf report for the last 5.1359 seconds: | 20260508 08:40:57.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:40:57.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 08:40:57.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:40:57.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:40:57.787 | INFO | Uploaded 36GiB to GPU in 2.4726s (14.5596GiB/s) | 20260508 08:40:57.787 | INFO | Downloaded 34GiB to GPU in 2.44767s (13.8908GiB/s) | 20260508 08:41:02.923 | INFO | Analyzing chunk frames = [6747648,6748160) | 20260508 08:41:02.923 | INFO | MemInfo: { | 20260508 08:41:02.923 | INFO | "gpu" : " 17 GB", | 20260508 08:41:02.923 | INFO | "huge" : "", | 20260508 08:41:02.923 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:02.923 | INFO | "malloc" : " 14 GB", | 20260508 08:41:02.923 | INFO | "pinned" : " 29 GB" | 20260508 08:41:02.923 | INFO | } | 20260508 08:41:02.923 | INFO | Perf report for the last 5.13639 seconds: | 20260508 08:41:02.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:02.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.923 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:41:02.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.923 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:41:02.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:02.924 | INFO | Uploaded 36GiB to GPU in 2.50087s (14.395GiB/s) | 20260508 08:41:02.924 | INFO | Downloaded 34GiB to GPU in 2.44209s (13.9225GiB/s) | 20260508 08:41:08.067 | INFO | Analyzing chunk frames = [6748160,6748672) | 20260508 08:41:08.067 | INFO | MemInfo: { | 20260508 08:41:08.067 | INFO | "gpu" : " 17 GB", | 20260508 08:41:08.067 | INFO | "huge" : "", | 20260508 08:41:08.067 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:08.067 | INFO | "malloc" : " 14 GB", | 20260508 08:41:08.067 | INFO | "pinned" : " 29 GB" | 20260508 08:41:08.067 | INFO | } | 20260508 08:41:08.067 | INFO | Perf report for the last 5.14369 seconds: | 20260508 08:41:08.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:08.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 08:41:08.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:41:08.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:08.067 | INFO | Uploaded 36GiB to GPU in 2.39605s (15.0247GiB/s) | 20260508 08:41:08.068 | INFO | Downloaded 34GiB to GPU in 2.52864s (13.446GiB/s) | 20260508 08:41:13.210 | INFO | Analyzing chunk frames = [6748672,6749184) | 20260508 08:41:13.210 | INFO | MemInfo: { | 20260508 08:41:13.210 | INFO | "gpu" : " 17 GB", | 20260508 08:41:13.210 | INFO | "huge" : "", | 20260508 08:41:13.210 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:13.210 | INFO | "malloc" : " 14 GB", | 20260508 08:41:13.210 | INFO | "pinned" : " 29 GB" | 20260508 08:41:13.210 | INFO | } | 20260508 08:41:13.210 | INFO | Perf report for the last 5.14294 seconds: | 20260508 08:41:13.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:13.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 08:41:13.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 08:41:13.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:13.210 | INFO | Uploaded 36GiB to GPU in 2.37675s (15.1467GiB/s) | 20260508 08:41:13.210 | INFO | Downloaded 34GiB to GPU in 2.43656s (13.9541GiB/s) | 20260508 08:41:18.342 | INFO | Analyzing chunk frames = [6749184,6749696) | 20260508 08:41:18.342 | INFO | MemInfo: { | 20260508 08:41:18.342 | INFO | "gpu" : " 17 GB", | 20260508 08:41:18.342 | INFO | "huge" : "", | 20260508 08:41:18.342 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:18.342 | INFO | "malloc" : " 14 GB", | 20260508 08:41:18.342 | INFO | "pinned" : " 29 GB" | 20260508 08:41:18.342 | INFO | } | 20260508 08:41:18.342 | INFO | Perf report for the last 5.13191 seconds: | 20260508 08:41:18.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:18.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.342 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:41:18.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.342 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:41:18.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:18.343 | INFO | Uploaded 36GiB to GPU in 2.39414s (15.0367GiB/s) | 20260508 08:41:18.343 | INFO | Downloaded 34GiB to GPU in 2.4357s (13.959GiB/s) | 20260508 08:41:23.480 | INFO | Analyzing chunk frames = [6749696,6750208) | 20260508 08:41:23.480 | INFO | MemInfo: { | 20260508 08:41:23.480 | INFO | "gpu" : " 17 GB", | 20260508 08:41:23.480 | INFO | "huge" : "", | 20260508 08:41:23.480 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:23.480 | INFO | "malloc" : " 14 GB", | 20260508 08:41:23.480 | INFO | "pinned" : " 29 GB" | 20260508 08:41:23.480 | INFO | } | 20260508 08:41:23.480 | INFO | Perf report for the last 5.1383 seconds: | 20260508 08:41:23.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:23.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.480 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 08:41:23.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.480 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:41:23.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:23.481 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:41:23.481 | INFO | Total mem usage: 16.7 GiB | 20260508 08:41:23.481 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:41:23.481 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:41:23.481 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:41:23.481 | INFO | High Water Breakdown: | 20260508 08:41:23.481 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:41:23.481 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:41:23.481 | INFO | Uploaded 36GiB to GPU in 2.47069s (14.5709GiB/s) | 20260508 08:41:23.481 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:41:23.481 | INFO | ------------------------------------------ | 20260508 08:41:23.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:41:23.481 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:41:23.481 | INFO | Downloaded 34GiB to GPU in 2.44483s (13.9069GiB/s) | 20260508 08:41:23.481 | INFO | Total mem usage: 26.5 GiB | 20260508 08:41:23.481 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:41:23.481 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 08:41:23.481 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 08:41:23.481 | INFO | High Water Breakdown: | 20260508 08:41:23.481 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:41:23.481 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:41:23.481 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:41:23.481 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:41:23.481 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:41:23.481 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:41:23.481 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:41:23.481 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:41:23.481 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:41:23.481 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:41:23.481 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:41:23.481 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:41:23.481 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:41:23.481 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:41:23.481 | INFO | ------------------------------------------ | 20260508 08:41:23.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:41:23.481 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:41:23.481 | INFO | Total mem usage: 27.8 GiB | 20260508 08:41:23.481 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:41:23.481 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:41:23.481 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:41:23.481 | INFO | High Water Breakdown: | 20260508 08:41:23.481 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:41:23.481 | INFO | ------------------------------------------ | 20260508 08:41:23.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:41:23.481 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:41:23.481 | INFO | Total mem usage: 15.7 GiB | 20260508 08:41:23.481 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:41:23.481 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:41:23.481 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:41:23.481 | INFO | High Water Breakdown: | 20260508 08:41:23.481 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:41:23.481 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:41:23.481 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:41:23.481 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:41:23.481 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:41:23.481 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:41:23.481 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:41:23.481 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:41:23.481 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:41:23.481 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:41:23.481 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:41:23.481 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:41:23.481 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:41:23.481 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:41:23.481 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:41:23.481 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:41:23.481 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:41:23.481 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:41:23.481 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:41:23.481 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:41:23.481 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:41:23.481 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:41:23.481 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:41:23.481 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:41:23.482 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:41:23.482 | INFO | ------------------------------------------ | 20260508 08:41:23.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:41:23.482 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:41:23.482 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:41:23.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:41:23.482 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.77e-01 | 20260508 08:41:23.482 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.15e-02 / 6.83e-03 | 20260508 08:41:23.482 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.42e+00, 5.95e-01 / 2.10e-01 | 20260508 08:41:23.482 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.40e+00, 2.08e-01 / 1.23e-01 | 20260508 08:41:23.482 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.50e+01, 3.64e+00 / 1.24e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.88e+00 / 1.46e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.47e+00, 4.06e-01 / 2.68e-01 | 20260508 08:41:23.482 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.13e+00, 2.47e-01 / 1.41e-01 | 20260508 08:41:23.482 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.30e-01, 4.82e-02 / 2.61e-02 | 20260508 08:41:23.482 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:41:23.482 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:41:23.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:41:23.482 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 24.35%, Upload -- 2.57e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 24.52%, Download -- 2.59e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 2.15%, LockUpload -- 2.27e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 2.55%, LockDownload -- 2.69e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 15.37%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:41:23.482 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260508 08:41:23.577 | INFO | Starting a round of Baz IO | 20260508 08:41:26.984 | INFO | Populating RTMetrics report | 20260508 08:41:26.984 | INFO | RTMetrics report populated | 20260508 08:41:26.984 | INFO | Opening tmp RTMetrics json file | 20260508 08:41:26.995 | INFO | Serializing RTMetrics to json | 20260508 08:41:26.996 | INFO | Serialization done, swapping tmp file now... | 20260508 08:41:26.997 | INFO | RTMetrics done | 20260508 08:41:27.053 | INFO | Finished a round of Baz IO: Wrote 1.498 GiB in 3.476 seconds | 20260508 08:41:27.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:41:27.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.284715% / 7.94729e-06% / 6.35783e-05% | 20260508 08:41:27.054 | INFO | Baz IO overall: | 20260508 08:41:27.054 | INFO | BytesWritten : 2958283047387 | 20260508 08:41:27.054 | INFO | NumEvents : 1423940345799 | 20260508 08:41:28.623 | INFO | Analyzing chunk frames = [6750208,6750720) | 20260508 08:41:28.623 | INFO | MemInfo: { | 20260508 08:41:28.623 | INFO | "gpu" : " 17 GB", | 20260508 08:41:28.623 | INFO | "huge" : "", | 20260508 08:41:28.623 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:28.623 | INFO | "malloc" : " 14 GB", | 20260508 08:41:28.623 | INFO | "pinned" : " 29 GB" | 20260508 08:41:28.623 | INFO | } | 20260508 08:41:28.623 | INFO | Perf report for the last 5.14315 seconds: | 20260508 08:41:28.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:28.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:28.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:28.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:28.624 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 08:41:28.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:28.624 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:41:28.624 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:41:28.624 | INFO | BAZWRITER : 66.9%, 2.6, 87.9ms, 0 | 20260508 08:41:28.624 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:41:28.624 | INFO | Uploaded 36GiB to GPU in 2.5828s (13.9384GiB/s) | 20260508 08:41:28.624 | INFO | Downloaded 38GiB to GPU in 2.71683s (13.9869GiB/s) | 20260508 08:41:33.760 | INFO | Analyzing chunk frames = [6750720,6751232) | 20260508 08:41:33.760 | INFO | MemInfo: { | 20260508 08:41:33.760 | INFO | "gpu" : " 17 GB", | 20260508 08:41:33.760 | INFO | "huge" : "", | 20260508 08:41:33.761 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:33.761 | INFO | "malloc" : " 14 GB", | 20260508 08:41:33.761 | INFO | "pinned" : " 29 GB" | 20260508 08:41:33.761 | INFO | } | 20260508 08:41:33.761 | INFO | Perf report for the last 5.1372 seconds: | 20260508 08:41:33.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:33.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 08:41:33.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:41:33.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:33.761 | INFO | Uploaded 36GiB to GPU in 2.50984s (14.3435GiB/s) | 20260508 08:41:33.761 | INFO | Downloaded 34GiB to GPU in 2.42397s (14.0266GiB/s) | 20260508 08:41:38.907 | INFO | Analyzing chunk frames = [6751232,6751744) | 20260508 08:41:38.907 | INFO | MemInfo: { | 20260508 08:41:38.907 | INFO | "gpu" : " 17 GB", | 20260508 08:41:38.907 | INFO | "huge" : "", | 20260508 08:41:38.907 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:38.907 | INFO | "malloc" : " 14 GB", | 20260508 08:41:38.907 | INFO | "pinned" : " 29 GB" | 20260508 08:41:38.907 | INFO | } | 20260508 08:41:38.907 | INFO | Perf report for the last 5.14639 seconds: | 20260508 08:41:38.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:38.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 08:41:38.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:41:38.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:38.907 | INFO | Uploaded 36GiB to GPU in 2.57941s (13.9567GiB/s) | 20260508 08:41:38.907 | INFO | Downloaded 34GiB to GPU in 2.43933s (13.9382GiB/s) | 20260508 08:41:44.042 | INFO | Analyzing chunk frames = [6751744,6752256) | 20260508 08:41:44.042 | INFO | MemInfo: { | 20260508 08:41:44.042 | INFO | "gpu" : " 17 GB", | 20260508 08:41:44.042 | INFO | "huge" : "", | 20260508 08:41:44.042 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:44.042 | INFO | "malloc" : " 14 GB", | 20260508 08:41:44.042 | INFO | "pinned" : " 29 GB" | 20260508 08:41:44.042 | INFO | } | 20260508 08:41:44.042 | INFO | Perf report for the last 5.13529 seconds: | 20260508 08:41:44.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:44.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.043 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:41:44.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.043 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 08:41:44.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:44.043 | INFO | Uploaded 36GiB to GPU in 2.50107s (14.3939GiB/s) | 20260508 08:41:44.043 | INFO | Downloaded 34GiB to GPU in 2.43515s (13.9622GiB/s) | 20260508 08:41:49.179 | INFO | Analyzing chunk frames = [6752256,6752768) | 20260508 08:41:49.179 | INFO | MemInfo: { | 20260508 08:41:49.179 | INFO | "gpu" : " 17 GB", | 20260508 08:41:49.179 | INFO | "huge" : "", | 20260508 08:41:49.179 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:49.179 | INFO | "malloc" : " 14 GB", | 20260508 08:41:49.179 | INFO | "pinned" : " 29 GB" | 20260508 08:41:49.179 | INFO | } | 20260508 08:41:49.179 | INFO | Perf report for the last 5.13648 seconds: | 20260508 08:41:49.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:49.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:41:49.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:41:49.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:49.179 | INFO | Uploaded 36GiB to GPU in 2.43389s (14.7911GiB/s) | 20260508 08:41:49.179 | INFO | Downloaded 34GiB to GPU in 2.53075s (13.4348GiB/s) | 20260508 08:41:54.321 | INFO | Analyzing chunk frames = [6752768,6753280) | 20260508 08:41:54.321 | INFO | MemInfo: { | 20260508 08:41:54.321 | INFO | "gpu" : " 17 GB", | 20260508 08:41:54.321 | INFO | "huge" : "", | 20260508 08:41:54.321 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:54.321 | INFO | "malloc" : " 14 GB", | 20260508 08:41:54.321 | INFO | "pinned" : " 29 GB" | 20260508 08:41:54.321 | INFO | } | 20260508 08:41:54.321 | INFO | Perf report for the last 5.14243 seconds: | 20260508 08:41:54.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:54.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:41:54.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:41:54.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:54.321 | INFO | Uploaded 36GiB to GPU in 2.43053s (14.8116GiB/s) | 20260508 08:41:54.321 | INFO | Downloaded 34GiB to GPU in 2.42759s (14.0056GiB/s) | 20260508 08:41:59.461 | INFO | Analyzing chunk frames = [6753280,6753792) | 20260508 08:41:59.461 | INFO | MemInfo: { | 20260508 08:41:59.461 | INFO | "gpu" : " 17 GB", | 20260508 08:41:59.461 | INFO | "huge" : "", | 20260508 08:41:59.461 | INFO | "huge_shared" : " 29 GB", | 20260508 08:41:59.461 | INFO | "malloc" : " 14 GB", | 20260508 08:41:59.462 | INFO | "pinned" : " 29 GB" | 20260508 08:41:59.462 | INFO | } | 20260508 08:41:59.462 | INFO | Perf report for the last 5.14042 seconds: | 20260508 08:41:59.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:41:59.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:41:59.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:41:59.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:41:59.462 | INFO | Uploaded 36GiB to GPU in 2.46029s (14.6324GiB/s) | 20260508 08:41:59.462 | INFO | Downloaded 34GiB to GPU in 2.44381s (13.9127GiB/s) | 20260508 08:42:04.600 | INFO | Analyzing chunk frames = [6753792,6754304) | 20260508 08:42:04.600 | INFO | MemInfo: { | 20260508 08:42:04.600 | INFO | "gpu" : " 17 GB", | 20260508 08:42:04.600 | INFO | "huge" : "", | 20260508 08:42:04.600 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:04.600 | INFO | "malloc" : " 14 GB", | 20260508 08:42:04.600 | INFO | "pinned" : " 29 GB" | 20260508 08:42:04.600 | INFO | } | 20260508 08:42:04.600 | INFO | Perf report for the last 5.13824 seconds: | 20260508 08:42:04.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:04.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 08:42:04.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:42:04.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:04.600 | INFO | Uploaded 36GiB to GPU in 2.44684s (14.7129GiB/s) | 20260508 08:42:04.600 | INFO | Downloaded 34GiB to GPU in 2.42394s (14.0268GiB/s) | 20260508 08:42:04.697 | INFO | Starting a round of Baz IO | 20260508 08:42:08.123 | INFO | Populating RTMetrics report | 20260508 08:42:08.123 | INFO | RTMetrics report populated | 20260508 08:42:08.123 | INFO | Opening tmp RTMetrics json file | 20260508 08:42:08.133 | INFO | Serializing RTMetrics to json | 20260508 08:42:08.135 | INFO | Serialization done, swapping tmp file now... | 20260508 08:42:08.135 | INFO | RTMetrics done | 20260508 08:42:08.189 | INFO | Finished a round of Baz IO: Wrote 1.498 GiB in 3.491 seconds | 20260508 08:42:08.189 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:42:08.189 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283706% / 0% / 7.15256e-05% | 20260508 08:42:08.189 | INFO | Baz IO overall: | 20260508 08:42:08.189 | INFO | BytesWritten : 2959891316834 | 20260508 08:42:08.189 | INFO | NumEvents : 1424701593834 | 20260508 08:42:09.737 | INFO | Analyzing chunk frames = [6754304,6754816) | 20260508 08:42:09.737 | INFO | MemInfo: { | 20260508 08:42:09.737 | INFO | "gpu" : " 17 GB", | 20260508 08:42:09.737 | INFO | "huge" : "", | 20260508 08:42:09.737 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:09.737 | INFO | "malloc" : " 14 GB", | 20260508 08:42:09.737 | INFO | "pinned" : " 29 GB" | 20260508 08:42:09.737 | INFO | } | 20260508 08:42:09.737 | INFO | Perf report for the last 5.13738 seconds: | 20260508 08:42:09.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:09.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:09.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:09.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:09.737 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:42:09.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:09.737 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:42:09.737 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 08:42:09.737 | INFO | BAZWRITER : 67.2%, 2.5, 87.8ms, 0 | 20260508 08:42:09.737 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:42:09.738 | INFO | Uploaded 36GiB to GPU in 2.59978s (13.8473GiB/s) | 20260508 08:42:09.738 | INFO | Downloaded 38GiB to GPU in 2.70855s (14.0296GiB/s) | 20260508 08:42:14.879 | INFO | Analyzing chunk frames = [6754816,6755328) | 20260508 08:42:14.879 | INFO | MemInfo: { | 20260508 08:42:14.879 | INFO | "gpu" : " 17 GB", | 20260508 08:42:14.879 | INFO | "huge" : "", | 20260508 08:42:14.879 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:14.879 | INFO | "malloc" : " 14 GB", | 20260508 08:42:14.879 | INFO | "pinned" : " 29 GB" | 20260508 08:42:14.879 | INFO | } | 20260508 08:42:14.879 | INFO | Perf report for the last 5.14156 seconds: | 20260508 08:42:14.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:14.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 08:42:14.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:42:14.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:14.879 | INFO | Uploaded 36GiB to GPU in 2.51085s (14.3378GiB/s) | 20260508 08:42:14.879 | INFO | Downloaded 34GiB to GPU in 2.45976s (13.8225GiB/s) | 20260508 08:42:20.018 | INFO | Analyzing chunk frames = [6755328,6755840) | 20260508 08:42:20.018 | INFO | MemInfo: { | 20260508 08:42:20.018 | INFO | "gpu" : " 17 GB", | 20260508 08:42:20.018 | INFO | "huge" : "", | 20260508 08:42:20.018 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:20.018 | INFO | "malloc" : " 14 GB", | 20260508 08:42:20.018 | INFO | "pinned" : " 29 GB" | 20260508 08:42:20.018 | INFO | } | 20260508 08:42:20.018 | INFO | Perf report for the last 5.13934 seconds: | 20260508 08:42:20.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:20.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.018 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 08:42:20.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.018 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 08:42:20.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:20.019 | INFO | Uploaded 36GiB to GPU in 2.47928s (14.5203GiB/s) | 20260508 08:42:20.019 | INFO | Downloaded 34GiB to GPU in 2.44016s (13.9335GiB/s) | 20260508 08:42:25.156 | INFO | Analyzing chunk frames = [6755840,6756352) | 20260508 08:42:25.156 | INFO | MemInfo: { | 20260508 08:42:25.156 | INFO | "gpu" : " 17 GB", | 20260508 08:42:25.156 | INFO | "huge" : "", | 20260508 08:42:25.156 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:25.156 | INFO | "malloc" : " 14 GB", | 20260508 08:42:25.156 | INFO | "pinned" : " 29 GB" | 20260508 08:42:25.156 | INFO | } | 20260508 08:42:25.156 | INFO | Perf report for the last 5.13815 seconds: | 20260508 08:42:25.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:25.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.157 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 08:42:25.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.157 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:42:25.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:25.158 | INFO | Uploaded 36GiB to GPU in 2.49659s (14.4197GiB/s) | 20260508 08:42:25.158 | INFO | Downloaded 34GiB to GPU in 2.43434s (13.9668GiB/s) | 20260508 08:42:30.292 | INFO | Analyzing chunk frames = [6756352,6756864) | 20260508 08:42:30.292 | INFO | MemInfo: { | 20260508 08:42:30.292 | INFO | "gpu" : " 17 GB", | 20260508 08:42:30.292 | INFO | "huge" : "", | 20260508 08:42:30.292 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:30.292 | INFO | "malloc" : " 14 GB", | 20260508 08:42:30.292 | INFO | "pinned" : " 29 GB" | 20260508 08:42:30.292 | INFO | } | 20260508 08:42:30.292 | INFO | Perf report for the last 5.1357 seconds: | 20260508 08:42:30.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:30.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.292 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 08:42:30.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.292 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:42:30.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:30.293 | INFO | Uploaded 36GiB to GPU in 2.40392s (14.9755GiB/s) | 20260508 08:42:30.293 | INFO | Downloaded 34GiB to GPU in 2.52687s (13.4554GiB/s) | 20260508 08:42:35.432 | INFO | Analyzing chunk frames = [6756864,6757376) | 20260508 08:42:35.432 | INFO | MemInfo: { | 20260508 08:42:35.432 | INFO | "gpu" : " 17 GB", | 20260508 08:42:35.432 | INFO | "huge" : "", | 20260508 08:42:35.432 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:35.432 | INFO | "malloc" : " 14 GB", | 20260508 08:42:35.432 | INFO | "pinned" : " 29 GB" | 20260508 08:42:35.432 | INFO | } | 20260508 08:42:35.432 | INFO | Perf report for the last 5.13979 seconds: | 20260508 08:42:35.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:35.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 08:42:35.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:42:35.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:35.432 | INFO | Uploaded 36GiB to GPU in 2.48256s (14.5012GiB/s) | 20260508 08:42:35.432 | INFO | Downloaded 34GiB to GPU in 2.45539s (13.8471GiB/s) | 20260508 08:42:40.578 | INFO | Analyzing chunk frames = [6757376,6757888) | 20260508 08:42:40.578 | INFO | MemInfo: { | 20260508 08:42:40.578 | INFO | "gpu" : " 17 GB", | 20260508 08:42:40.578 | INFO | "huge" : "", | 20260508 08:42:40.578 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:40.578 | INFO | "malloc" : " 14 GB", | 20260508 08:42:40.578 | INFO | "pinned" : " 29 GB" | 20260508 08:42:40.578 | INFO | } | 20260508 08:42:40.578 | INFO | Perf report for the last 5.14616 seconds: | 20260508 08:42:40.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:40.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | ANALYSIS : 66.0%, 3.0, 103.3ms, 0 | 20260508 08:42:40.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:42:40.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:40.578 | INFO | Uploaded 36GiB to GPU in 2.51s (14.3427GiB/s) | 20260508 08:42:40.578 | INFO | Downloaded 34GiB to GPU in 2.45104s (13.8717GiB/s) | 20260508 08:42:45.714 | INFO | Analyzing chunk frames = [6757888,6758400) | 20260508 08:42:45.714 | INFO | MemInfo: { | 20260508 08:42:45.714 | INFO | "gpu" : " 17 GB", | 20260508 08:42:45.714 | INFO | "huge" : "", | 20260508 08:42:45.714 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:45.714 | INFO | "malloc" : " 14 GB", | 20260508 08:42:45.714 | INFO | "pinned" : " 29 GB" | 20260508 08:42:45.714 | INFO | } | 20260508 08:42:45.715 | INFO | Perf report for the last 5.13647 seconds: | 20260508 08:42:45.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:45.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:42:45.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:42:45.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:45.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:42:45.715 | INFO | Total mem usage: 16.7 GiB | 20260508 08:42:45.715 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:42:45.715 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:42:45.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:42:45.715 | INFO | High Water Breakdown: | 20260508 08:42:45.715 | INFO | 59.4% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:42:45.715 | INFO | 40.4% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:42:45.715 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:42:45.715 | INFO | ------------------------------------------ | 20260508 08:42:45.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:42:45.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:42:45.715 | INFO | Total mem usage: 26.5 GiB | 20260508 08:42:45.715 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:42:45.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:42:45.715 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:42:45.715 | INFO | High Water Breakdown: | 20260508 08:42:45.715 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:42:45.715 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:42:45.715 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:42:45.715 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:42:45.715 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:42:45.715 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:42:45.715 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:42:45.715 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:42:45.715 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:42:45.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:42:45.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:42:45.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:42:45.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:42:45.715 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:42:45.715 | INFO | ------------------------------------------ | 20260508 08:42:45.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:42:45.715 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:42:45.715 | INFO | Total mem usage: 27.8 GiB | 20260508 08:42:45.715 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:42:45.715 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:42:45.715 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:42:45.715 | INFO | High Water Breakdown: | 20260508 08:42:45.715 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:42:45.715 | INFO | ------------------------------------------ | 20260508 08:42:45.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:42:45.715 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:42:45.715 | INFO | Total mem usage: 15.7 GiB | 20260508 08:42:45.715 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:42:45.715 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:42:45.715 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:42:45.715 | INFO | High Water Breakdown: | 20260508 08:42:45.715 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:42:45.715 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:42:45.715 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:42:45.715 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:42:45.715 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:42:45.715 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:42:45.715 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:42:45.715 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:42:45.715 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:42:45.715 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:42:45.715 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:42:45.715 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:42:45.715 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:42:45.715 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:42:45.715 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:42:45.715 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:42:45.715 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:42:45.715 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:42:45.715 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:42:45.715 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:42:45.715 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:42:45.715 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:42:45.715 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:42:45.715 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:42:45.715 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:42:45.715 | INFO | ------------------------------------------ | 20260508 08:42:45.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:42:45.715 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:42:45.715 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:42:45.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:42:45.715 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.37e+01 / 6.02e-01 | 20260508 08:42:45.715 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.14e-02 / 6.72e-03 | 20260508 08:42:45.715 | INFO | PerfReport: 12.13%, Baseline -- 3.97e+00 / 4.39e+00, 5.85e-01 / 2.78e-01 | 20260508 08:42:45.715 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.07e-01 / 1.22e-01 | 20260508 08:42:45.715 | INFO | PerfReport: 15.90%, DME -- 4.16e+01 / 4.47e+01, 3.62e+00 / 1.06e+00 | 20260508 08:42:45.715 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.89e+00 / 1.31e+00 | 20260508 08:42:45.715 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.44e+00, 4.03e-01 / 2.36e-01 | 20260508 08:42:45.715 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.18e+00, 2.54e-01 / 1.26e-01 | 20260508 08:42:45.715 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.37e-01, 4.90e-02 / 2.35e-02 | 20260508 08:42:45.715 | INFO | Uploaded 36GiB to GPU in 2.42948s (14.818GiB/s) | 20260508 08:42:45.715 | INFO | Downloaded 34GiB to GPU in 2.4432s (13.9162GiB/s) | 20260508 08:42:45.715 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:42:45.716 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:42:45.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:42:45.716 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 24.54%, Upload -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 24.48%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 30.96%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 2.25%, LockUpload -- 2.38e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 2.45%, LockDownload -- 2.59e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 15.31%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:42:45.716 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 08:42:45.812 | INFO | Starting a round of Baz IO | 20260508 08:42:49.222 | INFO | Populating RTMetrics report | 20260508 08:42:49.223 | INFO | RTMetrics report populated | 20260508 08:42:49.223 | INFO | Opening tmp RTMetrics json file | 20260508 08:42:49.234 | INFO | Serializing RTMetrics to json | 20260508 08:42:49.236 | INFO | Serialization done, swapping tmp file now... | 20260508 08:42:49.236 | INFO | RTMetrics done | 20260508 08:42:49.296 | INFO | Finished a round of Baz IO: Wrote 1.498 GiB in 3.483 seconds | 20260508 08:42:49.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:42:49.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283726% / 0% / 5.96046e-05% | 20260508 08:42:49.296 | INFO | Baz IO overall: | 20260508 08:42:49.296 | INFO | BytesWritten : 2961499343741 | 20260508 08:42:49.296 | INFO | NumEvents : 1425462989201 | 20260508 08:42:50.854 | INFO | Analyzing chunk frames = [6758400,6758912) | 20260508 08:42:50.854 | INFO | MemInfo: { | 20260508 08:42:50.854 | INFO | "gpu" : " 17 GB", | 20260508 08:42:50.854 | INFO | "huge" : "", | 20260508 08:42:50.854 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:50.854 | INFO | "malloc" : " 14 GB", | 20260508 08:42:50.854 | INFO | "pinned" : " 29 GB" | 20260508 08:42:50.854 | INFO | } | 20260508 08:42:50.854 | INFO | Perf report for the last 5.13993 seconds: | 20260508 08:42:50.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:50.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:50.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:50.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:50.855 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 08:42:50.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:50.855 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:42:50.855 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:42:50.855 | INFO | BAZWRITER : 67.1%, 2.6, 88.1ms, 0 | 20260508 08:42:50.855 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:42:50.855 | INFO | Uploaded 36GiB to GPU in 2.59695s (13.8624GiB/s) | 20260508 08:42:50.855 | INFO | Downloaded 38GiB to GPU in 2.69726s (14.0884GiB/s) | 20260508 08:42:55.987 | INFO | Analyzing chunk frames = [6758912,6759424) | 20260508 08:42:55.987 | INFO | MemInfo: { | 20260508 08:42:55.987 | INFO | "gpu" : " 17 GB", | 20260508 08:42:55.987 | INFO | "huge" : "", | 20260508 08:42:55.987 | INFO | "huge_shared" : " 29 GB", | 20260508 08:42:55.987 | INFO | "malloc" : " 14 GB", | 20260508 08:42:55.987 | INFO | "pinned" : " 29 GB" | 20260508 08:42:55.988 | INFO | } | 20260508 08:42:55.988 | INFO | Perf report for the last 5.13319 seconds: | 20260508 08:42:55.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:42:55.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 08:42:55.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:42:55.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:42:55.988 | INFO | Uploaded 36GiB to GPU in 2.54273s (14.158GiB/s) | 20260508 08:42:55.988 | INFO | Downloaded 34GiB to GPU in 2.44889s (13.8839GiB/s) | 20260508 08:43:01.129 | INFO | Analyzing chunk frames = [6759424,6759936) | 20260508 08:43:01.129 | INFO | MemInfo: { | 20260508 08:43:01.129 | INFO | "gpu" : " 17 GB", | 20260508 08:43:01.129 | INFO | "huge" : "", | 20260508 08:43:01.129 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:01.129 | INFO | "malloc" : " 14 GB", | 20260508 08:43:01.129 | INFO | "pinned" : " 29 GB" | 20260508 08:43:01.129 | INFO | } | 20260508 08:43:01.129 | INFO | Perf report for the last 5.14147 seconds: | 20260508 08:43:01.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:01.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | ANALYSIS : 67.4%, 3.0, 104.7ms, 0 | 20260508 08:43:01.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:43:01.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:01.129 | INFO | Uploaded 36GiB to GPU in 2.59178s (13.8901GiB/s) | 20260508 08:43:01.130 | INFO | Downloaded 34GiB to GPU in 2.43197s (13.9804GiB/s) | 20260508 08:43:06.271 | INFO | Analyzing chunk frames = [6759936,6760448) | 20260508 08:43:06.271 | INFO | MemInfo: { | 20260508 08:43:06.271 | INFO | "gpu" : " 17 GB", | 20260508 08:43:06.271 | INFO | "huge" : "", | 20260508 08:43:06.271 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:06.271 | INFO | "malloc" : " 14 GB", | 20260508 08:43:06.271 | INFO | "pinned" : " 29 GB" | 20260508 08:43:06.271 | INFO | } | 20260508 08:43:06.271 | INFO | Perf report for the last 5.14194 seconds: | 20260508 08:43:06.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:06.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.271 | INFO | ANALYSIS : 66.9%, 3.0, 103.6ms, 0 | 20260508 08:43:06.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.271 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:43:06.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:06.272 | INFO | Uploaded 36GiB to GPU in 2.58705s (13.9155GiB/s) | 20260508 08:43:06.272 | INFO | Downloaded 34GiB to GPU in 2.41793s (14.0616GiB/s) | 20260508 08:43:11.415 | INFO | Analyzing chunk frames = [6760448,6760960) | 20260508 08:43:11.415 | INFO | MemInfo: { | 20260508 08:43:11.415 | INFO | "gpu" : " 17 GB", | 20260508 08:43:11.415 | INFO | "huge" : "", | 20260508 08:43:11.415 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:11.415 | INFO | "malloc" : " 14 GB", | 20260508 08:43:11.415 | INFO | "pinned" : " 29 GB" | 20260508 08:43:11.415 | INFO | } | 20260508 08:43:11.415 | INFO | Perf report for the last 5.14431 seconds: | 20260508 08:43:11.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:11.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.415 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 08:43:11.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.416 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:43:11.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:11.416 | INFO | Uploaded 36GiB to GPU in 2.31271s (15.5662GiB/s) | 20260508 08:43:11.416 | INFO | Downloaded 34GiB to GPU in 2.49175s (13.645GiB/s) | 20260508 08:43:16.556 | INFO | Analyzing chunk frames = [6760960,6761472) | 20260508 08:43:16.556 | INFO | MemInfo: { | 20260508 08:43:16.556 | INFO | "gpu" : " 17 GB", | 20260508 08:43:16.557 | INFO | "huge" : "", | 20260508 08:43:16.557 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:16.557 | INFO | "malloc" : " 14 GB", | 20260508 08:43:16.557 | INFO | "pinned" : " 29 GB" | 20260508 08:43:16.557 | INFO | } | 20260508 08:43:16.557 | INFO | Perf report for the last 5.14111 seconds: | 20260508 08:43:16.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:16.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 08:43:16.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:43:16.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:16.557 | INFO | Uploaded 36GiB to GPU in 2.3891s (15.0685GiB/s) | 20260508 08:43:16.557 | INFO | Downloaded 34GiB to GPU in 2.4396s (13.9367GiB/s) | 20260508 08:43:21.689 | INFO | Analyzing chunk frames = [6761472,6761984) | 20260508 08:43:21.689 | INFO | MemInfo: { | 20260508 08:43:21.689 | INFO | "gpu" : " 17 GB", | 20260508 08:43:21.689 | INFO | "huge" : "", | 20260508 08:43:21.689 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:21.689 | INFO | "malloc" : " 14 GB", | 20260508 08:43:21.689 | INFO | "pinned" : " 29 GB" | 20260508 08:43:21.690 | INFO | } | 20260508 08:43:21.690 | INFO | Perf report for the last 5.13304 seconds: | 20260508 08:43:21.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:21.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:43:21.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:43:21.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:21.690 | INFO | Uploaded 36GiB to GPU in 2.42495s (14.8457GiB/s) | 20260508 08:43:21.690 | INFO | Downloaded 34GiB to GPU in 2.43761s (13.9481GiB/s) | 20260508 08:43:26.832 | INFO | Analyzing chunk frames = [6761984,6762496) | 20260508 08:43:26.832 | INFO | MemInfo: { | 20260508 08:43:26.832 | INFO | "gpu" : " 17 GB", | 20260508 08:43:26.832 | INFO | "huge" : "", | 20260508 08:43:26.832 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:26.832 | INFO | "malloc" : " 14 GB", | 20260508 08:43:26.832 | INFO | "pinned" : " 29 GB" | 20260508 08:43:26.832 | INFO | } | 20260508 08:43:26.832 | INFO | Perf report for the last 5.14242 seconds: | 20260508 08:43:26.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:26.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 08:43:26.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 08:43:26.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:26.832 | INFO | Uploaded 36GiB to GPU in 2.42675s (14.8347GiB/s) | 20260508 08:43:26.832 | INFO | Downloaded 34GiB to GPU in 2.44945s (13.8807GiB/s) | 20260508 08:43:26.929 | INFO | Starting a round of Baz IO | 20260508 08:43:30.337 | INFO | Populating RTMetrics report | 20260508 08:43:30.338 | INFO | RTMetrics report populated | 20260508 08:43:30.338 | INFO | Opening tmp RTMetrics json file | 20260508 08:43:30.348 | INFO | Serializing RTMetrics to json | 20260508 08:43:30.349 | INFO | Serialization done, swapping tmp file now... | 20260508 08:43:30.350 | INFO | RTMetrics done | 20260508 08:43:30.409 | INFO | Finished a round of Baz IO: Wrote 1.498 GiB in 3.479 seconds | 20260508 08:43:30.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:43:30.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283539% / 0% / 5.16574e-05% | 20260508 08:43:30.409 | INFO | Baz IO overall: | 20260508 08:43:30.409 | INFO | BytesWritten : 2963107678315 | 20260508 08:43:30.409 | INFO | NumEvents : 1426224811390 | 20260508 08:43:31.968 | INFO | Analyzing chunk frames = [6762496,6763008) | 20260508 08:43:31.968 | INFO | MemInfo: { | 20260508 08:43:31.968 | INFO | "gpu" : " 17 GB", | 20260508 08:43:31.968 | INFO | "huge" : "", | 20260508 08:43:31.968 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:31.968 | INFO | "malloc" : " 14 GB", | 20260508 08:43:31.968 | INFO | "pinned" : " 29 GB" | 20260508 08:43:31.968 | INFO | } | 20260508 08:43:31.968 | INFO | Perf report for the last 5.1364 seconds: | 20260508 08:43:31.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:31.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:31.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:31.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:31.968 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:43:31.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:31.969 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:43:31.969 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 08:43:31.969 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 08:43:31.969 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:43:31.969 | INFO | Uploaded 36GiB to GPU in 2.58441s (13.9297GiB/s) | 20260508 08:43:31.969 | INFO | Downloaded 38GiB to GPU in 2.71108s (14.0165GiB/s) | 20260508 08:43:37.105 | INFO | Analyzing chunk frames = [6763008,6763520) | 20260508 08:43:37.105 | INFO | MemInfo: { | 20260508 08:43:37.105 | INFO | "gpu" : " 17 GB", | 20260508 08:43:37.105 | INFO | "huge" : "", | 20260508 08:43:37.105 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:37.105 | INFO | "malloc" : " 14 GB", | 20260508 08:43:37.105 | INFO | "pinned" : " 29 GB" | 20260508 08:43:37.106 | INFO | } | 20260508 08:43:37.106 | INFO | Perf report for the last 5.13719 seconds: | 20260508 08:43:37.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:37.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 08:43:37.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:43:37.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:37.106 | INFO | Uploaded 36GiB to GPU in 2.52479s (14.2586GiB/s) | 20260508 08:43:37.106 | INFO | Downloaded 34GiB to GPU in 2.42691s (14.0096GiB/s) | 20260508 08:43:42.243 | INFO | Analyzing chunk frames = [6763520,6764032) | 20260508 08:43:42.243 | INFO | MemInfo: { | 20260508 08:43:42.243 | INFO | "gpu" : " 17 GB", | 20260508 08:43:42.243 | INFO | "huge" : "", | 20260508 08:43:42.243 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:42.243 | INFO | "malloc" : " 14 GB", | 20260508 08:43:42.243 | INFO | "pinned" : " 29 GB" | 20260508 08:43:42.243 | INFO | } | 20260508 08:43:42.243 | INFO | Perf report for the last 5.13709 seconds: | 20260508 08:43:42.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:42.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 08:43:42.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:43:42.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:42.243 | INFO | Uploaded 36GiB to GPU in 2.50112s (14.3936GiB/s) | 20260508 08:43:42.243 | INFO | Downloaded 34GiB to GPU in 2.4385s (13.943GiB/s) | 20260508 08:43:47.387 | INFO | Analyzing chunk frames = [6764032,6764544) | 20260508 08:43:47.387 | INFO | MemInfo: { | 20260508 08:43:47.387 | INFO | "gpu" : " 17 GB", | 20260508 08:43:47.387 | INFO | "huge" : "", | 20260508 08:43:47.387 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:47.387 | INFO | "malloc" : " 14 GB", | 20260508 08:43:47.387 | INFO | "pinned" : " 29 GB" | 20260508 08:43:47.387 | INFO | } | 20260508 08:43:47.387 | INFO | Perf report for the last 5.14436 seconds: | 20260508 08:43:47.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:47.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 08:43:47.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:43:47.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:47.387 | INFO | Uploaded 36GiB to GPU in 2.5754s (13.9784GiB/s) | 20260508 08:43:47.388 | INFO | Downloaded 34GiB to GPU in 2.43627s (13.9558GiB/s) | 20260508 08:43:52.531 | INFO | Analyzing chunk frames = [6764544,6765056) | 20260508 08:43:52.531 | INFO | MemInfo: { | 20260508 08:43:52.531 | INFO | "gpu" : " 17 GB", | 20260508 08:43:52.531 | INFO | "huge" : "", | 20260508 08:43:52.531 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:52.531 | INFO | "malloc" : " 14 GB", | 20260508 08:43:52.531 | INFO | "pinned" : " 29 GB" | 20260508 08:43:52.531 | INFO | } | 20260508 08:43:52.531 | INFO | Perf report for the last 5.1441 seconds: | 20260508 08:43:52.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:52.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.531 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 08:43:52.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.531 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:43:52.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:52.532 | INFO | Uploaded 36GiB to GPU in 2.39555s (15.0279GiB/s) | 20260508 08:43:52.532 | INFO | Downloaded 34GiB to GPU in 2.50412s (13.5776GiB/s) | 20260508 08:43:57.669 | INFO | Analyzing chunk frames = [6765056,6765568) | 20260508 08:43:57.669 | INFO | MemInfo: { | 20260508 08:43:57.669 | INFO | "gpu" : " 17 GB", | 20260508 08:43:57.669 | INFO | "huge" : "", | 20260508 08:43:57.669 | INFO | "huge_shared" : " 29 GB", | 20260508 08:43:57.669 | INFO | "malloc" : " 14 GB", | 20260508 08:43:57.669 | INFO | "pinned" : " 29 GB" | 20260508 08:43:57.669 | INFO | } | 20260508 08:43:57.669 | INFO | Perf report for the last 5.13811 seconds: | 20260508 08:43:57.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:43:57.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.669 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:43:57.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.670 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:43:57.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:43:57.670 | INFO | Uploaded 36GiB to GPU in 2.38268s (15.109GiB/s) | 20260508 08:43:57.670 | INFO | Downloaded 34GiB to GPU in 2.43313s (13.9738GiB/s) | 20260508 08:44:02.804 | INFO | Analyzing chunk frames = [6765568,6766080) | 20260508 08:44:02.804 | INFO | MemInfo: { | 20260508 08:44:02.804 | INFO | "gpu" : " 17 GB", | 20260508 08:44:02.804 | INFO | "huge" : "", | 20260508 08:44:02.804 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:02.804 | INFO | "malloc" : " 14 GB", | 20260508 08:44:02.804 | INFO | "pinned" : " 29 GB" | 20260508 08:44:02.804 | INFO | } | 20260508 08:44:02.804 | INFO | Perf report for the last 5.13489 seconds: | 20260508 08:44:02.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:02.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.804 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:44:02.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.804 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 08:44:02.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:02.805 | INFO | Uploaded 36GiB to GPU in 2.441s (14.7481GiB/s) | 20260508 08:44:02.805 | INFO | Downloaded 34GiB to GPU in 2.43712s (13.9509GiB/s) | 20260508 08:44:07.939 | INFO | Analyzing chunk frames = [6766080,6766592) | 20260508 08:44:07.939 | INFO | MemInfo: { | 20260508 08:44:07.939 | INFO | "gpu" : " 17 GB", | 20260508 08:44:07.939 | INFO | "huge" : "", | 20260508 08:44:07.939 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:07.939 | INFO | "malloc" : " 14 GB", | 20260508 08:44:07.939 | INFO | "pinned" : " 29 GB" | 20260508 08:44:07.939 | INFO | } | 20260508 08:44:07.940 | INFO | Perf report for the last 5.13535 seconds: | 20260508 08:44:07.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:07.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 08:44:07.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260508 08:44:07.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:07.940 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:44:07.940 | INFO | Total mem usage: 16.7 GiB | 20260508 08:44:07.940 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:44:07.940 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:44:07.940 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:44:07.940 | INFO | High Water Breakdown: | 20260508 08:44:07.940 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:44:07.940 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:44:07.940 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:44:07.940 | INFO | ------------------------------------------ | 20260508 08:44:07.940 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:44:07.940 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:44:07.940 | INFO | Total mem usage: 26.5 GiB | 20260508 08:44:07.940 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:44:07.941 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:44:07.941 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:44:07.940 | INFO | Uploaded 36GiB to GPU in 2.45874s (14.6416GiB/s) | 20260508 08:44:07.941 | INFO | High Water Breakdown: | 20260508 08:44:07.941 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:44:07.941 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:44:07.941 | INFO | Downloaded 34GiB to GPU in 2.43181s (13.9814GiB/s) | 20260508 08:44:07.941 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:44:07.941 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:44:07.941 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:44:07.941 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:44:07.941 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:44:07.941 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:44:07.941 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:44:07.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:44:07.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:44:07.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:44:07.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:44:07.941 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:44:07.941 | INFO | ------------------------------------------ | 20260508 08:44:07.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:44:07.941 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:44:07.941 | INFO | Total mem usage: 27.8 GiB | 20260508 08:44:07.941 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:44:07.941 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:44:07.941 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:44:07.941 | INFO | High Water Breakdown: | 20260508 08:44:07.941 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:44:07.941 | INFO | ------------------------------------------ | 20260508 08:44:07.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:44:07.941 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:44:07.941 | INFO | Total mem usage: 15.7 GiB | 20260508 08:44:07.941 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:44:07.941 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:44:07.941 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:44:07.941 | INFO | High Water Breakdown: | 20260508 08:44:07.941 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:44:07.941 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:44:07.941 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:44:07.941 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:44:07.941 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:44:07.941 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:44:07.941 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:44:07.941 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:44:07.941 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:44:07.941 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:44:07.941 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:44:07.941 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:44:07.941 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:44:07.941 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:44:07.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:44:07.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:44:07.941 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:44:07.941 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:44:07.941 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:44:07.941 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:44:07.941 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:44:07.941 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:44:07.941 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:44:07.941 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:44:07.941 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:44:07.941 | INFO | ------------------------------------------ | 20260508 08:44:07.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:44:07.942 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:44:07.942 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:44:07.942 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:44:07.942 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.39e+01 / 6.43e-01 | 20260508 08:44:07.942 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.14e-02 / 8.51e-03 | 20260508 08:44:07.942 | INFO | PerfReport: 12.13%, Baseline -- 3.96e+00 / 4.41e+00, 5.93e-01 / 2.15e-01 | 20260508 08:44:07.942 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.10e-01 / 1.36e-01 | 20260508 08:44:07.942 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.50e+01, 3.64e+00 / 1.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.80e+00 / 1.27e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.35e+00 / 2.45e+00, 4.07e-01 / 2.67e-01 | 20260508 08:44:07.942 | INFO | PerfReport: 3.08%, Metrics -- 1.00e+00 / 1.12e+00, 2.46e-01 / 9.49e-02 | 20260508 08:44:07.942 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.06e-01 / 3.36e-01, 4.91e-02 / 2.45e-02 | 20260508 08:44:07.942 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:44:07.942 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:44:07.942 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:44:07.942 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 24.49%, Upload -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 24.43%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 30.90%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 2.34%, LockUpload -- 2.47e+00 / 0.00e+00, 5.80e+00 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 2.52%, LockDownload -- 2.66e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 15.32%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:44:07.942 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260508 08:44:08.053 | INFO | Starting a round of Baz IO | 20260508 08:44:11.463 | INFO | Populating RTMetrics report | 20260508 08:44:11.463 | INFO | RTMetrics report populated | 20260508 08:44:11.463 | INFO | Opening tmp RTMetrics json file | 20260508 08:44:11.471 | INFO | Serializing RTMetrics to json | 20260508 08:44:11.472 | INFO | Serialization done, swapping tmp file now... | 20260508 08:44:11.473 | INFO | RTMetrics done | 20260508 08:44:11.530 | INFO | Finished a round of Baz IO: Wrote 1.498 GiB in 3.477 seconds | 20260508 08:44:11.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:44:11.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.28348% / 3.97364e-06% / 7.15256e-05% | 20260508 08:44:11.531 | INFO | Baz IO overall: | 20260508 08:44:11.531 | INFO | BytesWritten : 2964716535810 | 20260508 08:44:11.531 | INFO | NumEvents : 1426987138855 | 20260508 08:44:13.083 | INFO | Analyzing chunk frames = [6766592,6767104) | 20260508 08:44:13.083 | INFO | MemInfo: { | 20260508 08:44:13.083 | INFO | "gpu" : " 17 GB", | 20260508 08:44:13.083 | INFO | "huge" : "", | 20260508 08:44:13.083 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:13.083 | INFO | "malloc" : " 14 GB", | 20260508 08:44:13.083 | INFO | "pinned" : " 29 GB" | 20260508 08:44:13.083 | INFO | } | 20260508 08:44:13.083 | INFO | Perf report for the last 5.14292 seconds: | 20260508 08:44:13.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:13.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:13.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:13.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:13.083 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 08:44:13.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:13.083 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 08:44:13.083 | INFO | PRE_HQ : 66.0%, 1.7, 59.6ms, 0 | 20260508 08:44:13.083 | INFO | BAZWRITER : 66.8%, 2.7, 92.5ms, 0 | 20260508 08:44:13.083 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:44:13.083 | INFO | Uploaded 36GiB to GPU in 2.6151s (13.7662GiB/s) | 20260508 08:44:13.083 | INFO | Downloaded 38GiB to GPU in 2.71298s (14.0067GiB/s) | 20260508 08:44:18.224 | INFO | Analyzing chunk frames = [6767104,6767616) | 20260508 08:44:18.224 | INFO | MemInfo: { | 20260508 08:44:18.224 | INFO | "gpu" : " 17 GB", | 20260508 08:44:18.224 | INFO | "huge" : "", | 20260508 08:44:18.224 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:18.224 | INFO | "malloc" : " 14 GB", | 20260508 08:44:18.224 | INFO | "pinned" : " 29 GB" | 20260508 08:44:18.224 | INFO | } | 20260508 08:44:18.224 | INFO | Perf report for the last 5.14176 seconds: | 20260508 08:44:18.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:18.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.224 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:44:18.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.224 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:44:18.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:18.225 | INFO | Uploaded 36GiB to GPU in 2.51947s (14.2887GiB/s) | 20260508 08:44:18.225 | INFO | Downloaded 34GiB to GPU in 2.44325s (13.9159GiB/s) | 20260508 08:44:23.365 | INFO | Analyzing chunk frames = [6767616,6768128) | 20260508 08:44:23.365 | INFO | MemInfo: { | 20260508 08:44:23.365 | INFO | "gpu" : " 17 GB", | 20260508 08:44:23.365 | INFO | "huge" : "", | 20260508 08:44:23.365 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:23.365 | INFO | "malloc" : " 14 GB", | 20260508 08:44:23.365 | INFO | "pinned" : " 29 GB" | 20260508 08:44:23.365 | INFO | } | 20260508 08:44:23.365 | INFO | Perf report for the last 5.14083 seconds: | 20260508 08:44:23.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:23.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.365 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 08:44:23.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.365 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:44:23.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:23.366 | INFO | Uploaded 36GiB to GPU in 2.48114s (14.5094GiB/s) | 20260508 08:44:23.366 | INFO | Downloaded 34GiB to GPU in 2.43808s (13.9454GiB/s) | 20260508 08:44:28.496 | INFO | Analyzing chunk frames = [6768128,6768640) | 20260508 08:44:28.496 | INFO | MemInfo: { | 20260508 08:44:28.496 | INFO | "gpu" : " 17 GB", | 20260508 08:44:28.496 | INFO | "huge" : "", | 20260508 08:44:28.496 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:28.496 | INFO | "malloc" : " 14 GB", | 20260508 08:44:28.496 | INFO | "pinned" : " 29 GB" | 20260508 08:44:28.496 | INFO | } | 20260508 08:44:28.496 | INFO | Perf report for the last 5.13082 seconds: | 20260508 08:44:28.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:28.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.496 | INFO | ANALYSIS : 66.4%, 3.0, 102.7ms, 0 | 20260508 08:44:28.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.496 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:44:28.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:28.497 | INFO | Uploaded 36GiB to GPU in 2.47696s (14.5339GiB/s) | 20260508 08:44:28.497 | INFO | Downloaded 34GiB to GPU in 2.42557s (14.0173GiB/s) | 20260508 08:44:33.641 | INFO | Analyzing chunk frames = [6768640,6769152) | 20260508 08:44:33.641 | INFO | MemInfo: { | 20260508 08:44:33.641 | INFO | "gpu" : " 17 GB", | 20260508 08:44:33.641 | INFO | "huge" : "", | 20260508 08:44:33.641 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:33.641 | INFO | "malloc" : " 14 GB", | 20260508 08:44:33.641 | INFO | "pinned" : " 29 GB" | 20260508 08:44:33.641 | INFO | } | 20260508 08:44:33.641 | INFO | Perf report for the last 5.14537 seconds: | 20260508 08:44:33.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:33.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.641 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 08:44:33.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.642 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:44:33.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:33.642 | INFO | Uploaded 36GiB to GPU in 2.33037s (15.4482GiB/s) | 20260508 08:44:33.642 | INFO | Downloaded 34GiB to GPU in 2.51516s (13.518GiB/s) | 20260508 08:44:38.778 | INFO | Analyzing chunk frames = [6769152,6769664) | 20260508 08:44:38.778 | INFO | MemInfo: { | 20260508 08:44:38.778 | INFO | "gpu" : " 17 GB", | 20260508 08:44:38.778 | INFO | "huge" : "", | 20260508 08:44:38.778 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:38.778 | INFO | "malloc" : " 14 GB", | 20260508 08:44:38.778 | INFO | "pinned" : " 29 GB" | 20260508 08:44:38.778 | INFO | } | 20260508 08:44:38.778 | INFO | Perf report for the last 5.13678 seconds: | 20260508 08:44:38.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:38.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.778 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 08:44:38.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.778 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:44:38.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:38.779 | INFO | Uploaded 36GiB to GPU in 2.35905s (15.2604GiB/s) | 20260508 08:44:38.779 | INFO | Downloaded 34GiB to GPU in 2.42911s (13.9969GiB/s) | 20260508 08:44:43.921 | INFO | Analyzing chunk frames = [6769664,6770176) | 20260508 08:44:43.921 | INFO | MemInfo: { | 20260508 08:44:43.921 | INFO | "gpu" : " 17 GB", | 20260508 08:44:43.921 | INFO | "huge" : "", | 20260508 08:44:43.921 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:43.923 | INFO | "malloc" : " 14 GB", | 20260508 08:44:43.923 | INFO | "pinned" : " 29 GB" | 20260508 08:44:43.923 | INFO | } | 20260508 08:44:43.923 | INFO | Perf report for the last 5.14505 seconds: | 20260508 08:44:43.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:43.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.923 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 08:44:43.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.923 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 08:44:43.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:43.924 | INFO | Uploaded 36GiB to GPU in 2.42044s (14.8734GiB/s) | 20260508 08:44:43.924 | INFO | Downloaded 34GiB to GPU in 2.44909s (13.8827GiB/s) | 20260508 08:44:49.061 | INFO | Analyzing chunk frames = [6770176,6770688) | 20260508 08:44:49.061 | INFO | MemInfo: { | 20260508 08:44:49.061 | INFO | "gpu" : " 17 GB", | 20260508 08:44:49.061 | INFO | "huge" : "", | 20260508 08:44:49.061 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:49.061 | INFO | "malloc" : " 14 GB", | 20260508 08:44:49.062 | INFO | "pinned" : " 29 GB" | 20260508 08:44:49.062 | INFO | } | 20260508 08:44:49.062 | INFO | Perf report for the last 5.13833 seconds: | 20260508 08:44:49.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:49.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.062 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 08:44:49.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.062 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 08:44:49.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:49.063 | INFO | Uploaded 36GiB to GPU in 2.37641s (15.1489GiB/s) | 20260508 08:44:49.063 | INFO | Downloaded 34GiB to GPU in 2.42941s (13.9952GiB/s) | 20260508 08:44:49.173 | INFO | Starting a round of Baz IO | 20260508 08:44:52.580 | INFO | Populating RTMetrics report | 20260508 08:44:52.581 | INFO | RTMetrics report populated | 20260508 08:44:52.581 | INFO | Opening tmp RTMetrics json file | 20260508 08:44:52.590 | INFO | Serializing RTMetrics to json | 20260508 08:44:52.591 | INFO | Serialization done, swapping tmp file now... | 20260508 08:44:52.592 | INFO | RTMetrics done | 20260508 08:44:52.657 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.483 seconds | 20260508 08:44:52.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.39% / 0.00% / 59.61% | 20260508 08:44:52.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.282141% / 3.97364e-06% / 7.15256e-05% | 20260508 08:44:52.657 | INFO | Baz IO overall: | 20260508 08:44:52.657 | INFO | BytesWritten : 2966322773220 | 20260508 08:44:52.657 | INFO | NumEvents : 1427747397748 | 20260508 08:44:54.200 | INFO | Analyzing chunk frames = [6770688,6771200) | 20260508 08:44:54.200 | INFO | MemInfo: { | 20260508 08:44:54.200 | INFO | "gpu" : " 17 GB", | 20260508 08:44:54.200 | INFO | "huge" : "", | 20260508 08:44:54.200 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:54.200 | INFO | "malloc" : " 14 GB", | 20260508 08:44:54.200 | INFO | "pinned" : " 29 GB" | 20260508 08:44:54.201 | INFO | } | 20260508 08:44:54.201 | INFO | Perf report for the last 5.13893 seconds: | 20260508 08:44:54.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:54.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:54.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:54.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:54.201 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 08:44:54.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:54.201 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260508 08:44:54.201 | INFO | PRE_HQ : 66.3%, 1.8, 60.7ms, 0 | 20260508 08:44:54.201 | INFO | BAZWRITER : 66.9%, 2.7, 93.5ms, 0 | 20260508 08:44:54.201 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:44:54.201 | INFO | Uploaded 36GiB to GPU in 2.60365s (13.8267GiB/s) | 20260508 08:44:54.201 | INFO | Downloaded 38GiB to GPU in 2.71273s (14.008GiB/s) | 20260508 08:44:59.341 | INFO | Analyzing chunk frames = [6771200,6771712) | 20260508 08:44:59.341 | INFO | MemInfo: { | 20260508 08:44:59.341 | INFO | "gpu" : " 17 GB", | 20260508 08:44:59.341 | INFO | "huge" : "", | 20260508 08:44:59.341 | INFO | "huge_shared" : " 29 GB", | 20260508 08:44:59.341 | INFO | "malloc" : " 14 GB", | 20260508 08:44:59.341 | INFO | "pinned" : " 29 GB" | 20260508 08:44:59.341 | INFO | } | 20260508 08:44:59.341 | INFO | Perf report for the last 5.14015 seconds: | 20260508 08:44:59.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:44:59.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:44:59.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:44:59.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:44:59.341 | INFO | Uploaded 36GiB to GPU in 2.53872s (14.1803GiB/s) | 20260508 08:44:59.341 | INFO | Downloaded 34GiB to GPU in 2.44884s (13.8841GiB/s) | 20260508 08:45:04.481 | INFO | Analyzing chunk frames = [6771712,6772224) | 20260508 08:45:04.481 | INFO | MemInfo: { | 20260508 08:45:04.481 | INFO | "gpu" : " 17 GB", | 20260508 08:45:04.481 | INFO | "huge" : "", | 20260508 08:45:04.481 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:04.481 | INFO | "malloc" : " 14 GB", | 20260508 08:45:04.481 | INFO | "pinned" : " 29 GB" | 20260508 08:45:04.481 | INFO | } | 20260508 08:45:04.481 | INFO | Perf report for the last 5.14005 seconds: | 20260508 08:45:04.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:04.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.481 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:45:04.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.481 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:45:04.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:04.482 | INFO | Uploaded 36GiB to GPU in 2.48667s (14.4772GiB/s) | 20260508 08:45:04.482 | INFO | Downloaded 34GiB to GPU in 2.42726s (14.0076GiB/s) | 20260508 08:45:09.617 | INFO | Analyzing chunk frames = [6772224,6772736) | 20260508 08:45:09.617 | INFO | MemInfo: { | 20260508 08:45:09.617 | INFO | "gpu" : " 17 GB", | 20260508 08:45:09.617 | INFO | "huge" : "", | 20260508 08:45:09.617 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:09.617 | INFO | "malloc" : " 14 GB", | 20260508 08:45:09.617 | INFO | "pinned" : " 29 GB" | 20260508 08:45:09.617 | INFO | } | 20260508 08:45:09.617 | INFO | Perf report for the last 5.13638 seconds: | 20260508 08:45:09.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:09.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.617 | INFO | ANALYSIS : 67.1%, 3.0, 103.7ms, 0 | 20260508 08:45:09.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.617 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:45:09.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:09.618 | INFO | Uploaded 36GiB to GPU in 2.5879s (13.9109GiB/s) | 20260508 08:45:09.618 | INFO | Downloaded 34GiB to GPU in 2.41673s (14.0686GiB/s) | 20260508 08:45:14.748 | INFO | Analyzing chunk frames = [6772736,6773248) | 20260508 08:45:14.748 | INFO | MemInfo: { | 20260508 08:45:14.748 | INFO | "gpu" : " 17 GB", | 20260508 08:45:14.748 | INFO | "huge" : "", | 20260508 08:45:14.748 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:14.748 | INFO | "malloc" : " 14 GB", | 20260508 08:45:14.748 | INFO | "pinned" : " 29 GB" | 20260508 08:45:14.749 | INFO | } | 20260508 08:45:14.749 | INFO | Perf report for the last 5.13139 seconds: | 20260508 08:45:14.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:14.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 08:45:14.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:45:14.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:14.749 | INFO | Uploaded 36GiB to GPU in 2.39724s (15.0173GiB/s) | 20260508 08:45:14.749 | INFO | Downloaded 34GiB to GPU in 2.52308s (13.4756GiB/s) | 20260508 08:45:19.894 | INFO | Analyzing chunk frames = [6773248,6773760) | 20260508 08:45:19.894 | INFO | MemInfo: { | 20260508 08:45:19.894 | INFO | "gpu" : " 17 GB", | 20260508 08:45:19.894 | INFO | "huge" : "", | 20260508 08:45:19.894 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:19.894 | INFO | "malloc" : " 14 GB", | 20260508 08:45:19.894 | INFO | "pinned" : " 29 GB" | 20260508 08:45:19.894 | INFO | } | 20260508 08:45:19.894 | INFO | Perf report for the last 5.14533 seconds: | 20260508 08:45:19.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:19.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.894 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:45:19.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.894 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:45:19.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:19.895 | INFO | Uploaded 36GiB to GPU in 2.37337s (15.1683GiB/s) | 20260508 08:45:19.895 | INFO | Downloaded 34GiB to GPU in 2.43664s (13.9536GiB/s) | 20260508 08:45:25.033 | INFO | Analyzing chunk frames = [6773760,6774272) | 20260508 08:45:25.033 | INFO | MemInfo: { | 20260508 08:45:25.033 | INFO | "gpu" : " 17 GB", | 20260508 08:45:25.033 | INFO | "huge" : "", | 20260508 08:45:25.033 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:25.033 | INFO | "malloc" : " 14 GB", | 20260508 08:45:25.033 | INFO | "pinned" : " 29 GB" | 20260508 08:45:25.033 | INFO | } | 20260508 08:45:25.033 | INFO | Perf report for the last 5.13905 seconds: | 20260508 08:45:25.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:25.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:45:25.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 08:45:25.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:25.033 | INFO | Uploaded 36GiB to GPU in 2.41772s (14.8901GiB/s) | 20260508 08:45:25.033 | INFO | Downloaded 34GiB to GPU in 2.43529s (13.9614GiB/s) | 20260508 08:45:30.176 | INFO | Analyzing chunk frames = [6774272,6774784) | 20260508 08:45:30.176 | INFO | MemInfo: { | 20260508 08:45:30.176 | INFO | "gpu" : " 17 GB", | 20260508 08:45:30.176 | INFO | "huge" : "", | 20260508 08:45:30.176 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:30.176 | INFO | "malloc" : " 14 GB", | 20260508 08:45:30.176 | INFO | "pinned" : " 29 GB" | 20260508 08:45:30.176 | INFO | } | 20260508 08:45:30.176 | INFO | Perf report for the last 5.14325 seconds: | 20260508 08:45:30.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:30.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.176 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 08:45:30.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.176 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:45:30.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:30.177 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:45:30.177 | INFO | Total mem usage: 16.7 GiB | 20260508 08:45:30.177 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:45:30.177 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:45:30.177 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:45:30.177 | INFO | High Water Breakdown: | 20260508 08:45:30.177 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:45:30.177 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:45:30.177 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:45:30.177 | INFO | ------------------------------------------ | 20260508 08:45:30.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:45:30.177 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:45:30.177 | INFO | Total mem usage: 26.5 GiB | 20260508 08:45:30.177 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:45:30.177 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 08:45:30.177 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 08:45:30.177 | INFO | High Water Breakdown: | 20260508 08:45:30.177 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:45:30.177 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:45:30.177 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:45:30.177 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:45:30.177 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:45:30.177 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:45:30.177 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:45:30.177 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:45:30.177 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:45:30.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:45:30.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:45:30.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:45:30.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:45:30.177 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:45:30.177 | INFO | ------------------------------------------ | 20260508 08:45:30.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:45:30.177 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:45:30.177 | INFO | Total mem usage: 27.8 GiB | 20260508 08:45:30.177 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:45:30.177 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:45:30.177 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:45:30.177 | INFO | High Water Breakdown: | 20260508 08:45:30.177 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:45:30.177 | INFO | ------------------------------------------ | 20260508 08:45:30.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:45:30.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:45:30.177 | INFO | Total mem usage: 15.7 GiB | 20260508 08:45:30.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:45:30.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:45:30.177 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:45:30.177 | INFO | High Water Breakdown: | 20260508 08:45:30.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:45:30.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:45:30.177 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:45:30.177 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:45:30.177 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:45:30.177 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:45:30.177 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:45:30.177 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:45:30.177 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:45:30.177 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:45:30.177 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:45:30.177 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:45:30.177 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:45:30.177 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:45:30.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:45:30.177 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:45:30.177 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:45:30.177 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:45:30.177 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:45:30.177 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:45:30.177 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:45:30.177 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:45:30.177 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:45:30.177 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:45:30.177 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:45:30.177 | INFO | ------------------------------------------ | 20260508 08:45:30.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:45:30.177 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:45:30.177 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:45:30.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:45:30.177 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.40e+01 / 6.88e-01 | 20260508 08:45:30.177 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.08e-01, 1.24e-02 / 8.86e-03 | 20260508 08:45:30.177 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.39e+00, 5.89e-01 / 1.81e-01 | 20260508 08:45:30.177 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.42e+00, 2.08e-01 / 1.13e-01 | 20260508 08:45:30.177 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.51e+01, 3.56e+00 / 1.12e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.83e+00 / 1.23e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.04e-01 / 2.33e-01 | 20260508 08:45:30.177 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.12e+00, 2.49e-01 / 1.25e-01 | 20260508 08:45:30.177 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.42e-01, 4.89e-02 / 2.56e-02 | 20260508 08:45:30.177 | INFO | Uploaded 36GiB to GPU in 2.46616s (14.5976GiB/s) | 20260508 08:45:30.177 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:45:30.177 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:45:30.177 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:45:30.177 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 08:45:30.177 | INFO | Downloaded 34GiB to GPU in 2.44367s (13.9135GiB/s) | 20260508 08:45:30.177 | INFO | PerfReport: 24.35%, Upload -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 24.49%, Download -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 31.01%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 2.52%, LockDownload -- 2.65e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 08:45:30.177 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 08:45:30.274 | INFO | Starting a round of Baz IO | 20260508 08:45:33.695 | INFO | Populating RTMetrics report | 20260508 08:45:33.696 | INFO | RTMetrics report populated | 20260508 08:45:33.696 | INFO | Opening tmp RTMetrics json file | 20260508 08:45:33.706 | INFO | Serializing RTMetrics to json | 20260508 08:45:33.707 | INFO | Serialization done, swapping tmp file now... | 20260508 08:45:33.708 | INFO | RTMetrics done | 20260508 08:45:33.761 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.487 seconds | 20260508 08:45:33.761 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.61% | 20260508 08:45:33.761 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.281421% / 0% / 5.96046e-05% | 20260508 08:45:33.761 | INFO | Baz IO overall: | 20260508 08:45:33.761 | INFO | BytesWritten : 2967929138355 | 20260508 08:45:33.761 | INFO | NumEvents : 1428507814510 | 20260508 08:45:35.309 | INFO | Analyzing chunk frames = [6774784,6775296) | 20260508 08:45:35.309 | INFO | MemInfo: { | 20260508 08:45:35.309 | INFO | "gpu" : " 17 GB", | 20260508 08:45:35.309 | INFO | "huge" : "", | 20260508 08:45:35.309 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:35.309 | INFO | "malloc" : " 14 GB", | 20260508 08:45:35.309 | INFO | "pinned" : " 29 GB" | 20260508 08:45:35.309 | INFO | } | 20260508 08:45:35.309 | INFO | Perf report for the last 5.13316 seconds: | 20260508 08:45:35.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:35.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:35.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:35.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:35.310 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 08:45:35.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:35.310 | INFO | RT_METRICS : 38.9%, 1.0, 20.3ms, 0 | 20260508 08:45:35.310 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 08:45:35.310 | INFO | BAZWRITER : 67.2%, 2.6, 88.4ms, 0 | 20260508 08:45:35.310 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:45:35.310 | INFO | Uploaded 36GiB to GPU in 2.57483s (13.9815GiB/s) | 20260508 08:45:35.310 | INFO | Downloaded 38GiB to GPU in 2.70527s (14.0467GiB/s) | 20260508 08:45:40.447 | INFO | Analyzing chunk frames = [6775296,6775808) | 20260508 08:45:40.447 | INFO | MemInfo: { | 20260508 08:45:40.447 | INFO | "gpu" : " 17 GB", | 20260508 08:45:40.447 | INFO | "huge" : "", | 20260508 08:45:40.447 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:40.447 | INFO | "malloc" : " 14 GB", | 20260508 08:45:40.447 | INFO | "pinned" : " 29 GB" | 20260508 08:45:40.447 | INFO | } | 20260508 08:45:40.447 | INFO | Perf report for the last 5.13766 seconds: | 20260508 08:45:40.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:40.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.447 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 08:45:40.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.447 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:45:40.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:40.448 | INFO | Uploaded 36GiB to GPU in 2.50749s (14.357GiB/s) | 20260508 08:45:40.448 | INFO | Downloaded 34GiB to GPU in 2.45068s (13.8737GiB/s) | 20260508 08:45:45.591 | INFO | Analyzing chunk frames = [6775808,6776320) | 20260508 08:45:45.591 | INFO | MemInfo: { | 20260508 08:45:45.591 | INFO | "gpu" : " 17 GB", | 20260508 08:45:45.591 | INFO | "huge" : "", | 20260508 08:45:45.591 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:45.591 | INFO | "malloc" : " 14 GB", | 20260508 08:45:45.591 | INFO | "pinned" : " 29 GB" | 20260508 08:45:45.591 | INFO | } | 20260508 08:45:45.592 | INFO | Perf report for the last 5.14442 seconds: | 20260508 08:45:45.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:45.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | ANALYSIS : 67.0%, 3.0, 104.1ms, 0 | 20260508 08:45:45.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 08:45:45.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:45.592 | INFO | Uploaded 36GiB to GPU in 2.57598s (13.9752GiB/s) | 20260508 08:45:45.592 | INFO | Downloaded 34GiB to GPU in 2.43281s (13.9756GiB/s) | 20260508 08:45:50.731 | INFO | Analyzing chunk frames = [6776320,6776832) | 20260508 08:45:50.731 | INFO | MemInfo: { | 20260508 08:45:50.731 | INFO | "gpu" : " 17 GB", | 20260508 08:45:50.731 | INFO | "huge" : "", | 20260508 08:45:50.731 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:50.731 | INFO | "malloc" : " 14 GB", | 20260508 08:45:50.731 | INFO | "pinned" : " 29 GB" | 20260508 08:45:50.731 | INFO | } | 20260508 08:45:50.731 | INFO | Perf report for the last 5.13984 seconds: | 20260508 08:45:50.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:50.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.732 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 08:45:50.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.732 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:45:50.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:50.732 | INFO | Uploaded 36GiB to GPU in 2.57378s (13.9872GiB/s) | 20260508 08:45:50.732 | INFO | Downloaded 34GiB to GPU in 2.43329s (13.9729GiB/s) | 20260508 08:45:55.873 | INFO | Analyzing chunk frames = [6776832,6777344) | 20260508 08:45:55.873 | INFO | MemInfo: { | 20260508 08:45:55.873 | INFO | "gpu" : " 17 GB", | 20260508 08:45:55.873 | INFO | "huge" : "", | 20260508 08:45:55.873 | INFO | "huge_shared" : " 29 GB", | 20260508 08:45:55.873 | INFO | "malloc" : " 14 GB", | 20260508 08:45:55.873 | INFO | "pinned" : " 29 GB" | 20260508 08:45:55.873 | INFO | } | 20260508 08:45:55.873 | INFO | Perf report for the last 5.1414 seconds: | 20260508 08:45:55.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:45:55.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 08:45:55.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:45:55.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:45:55.873 | INFO | Uploaded 36GiB to GPU in 2.29224s (15.7052GiB/s) | 20260508 08:45:55.873 | INFO | Downloaded 34GiB to GPU in 2.50512s (13.5722GiB/s) | 20260508 08:46:01.005 | INFO | Analyzing chunk frames = [6777344,6777856) | 20260508 08:46:01.005 | INFO | MemInfo: { | 20260508 08:46:01.005 | INFO | "gpu" : " 17 GB", | 20260508 08:46:01.005 | INFO | "huge" : "", | 20260508 08:46:01.005 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:01.005 | INFO | "malloc" : " 14 GB", | 20260508 08:46:01.005 | INFO | "pinned" : " 29 GB" | 20260508 08:46:01.005 | INFO | } | 20260508 08:46:01.005 | INFO | Perf report for the last 5.13253 seconds: | 20260508 08:46:01.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:01.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.005 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 08:46:01.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.005 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:46:01.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:01.006 | INFO | Uploaded 36GiB to GPU in 2.41094s (14.9319GiB/s) | 20260508 08:46:01.006 | INFO | Downloaded 34GiB to GPU in 2.4309s (13.9866GiB/s) | 20260508 08:46:06.152 | INFO | Analyzing chunk frames = [6777856,6778368) | 20260508 08:46:06.152 | INFO | MemInfo: { | 20260508 08:46:06.152 | INFO | "gpu" : " 17 GB", | 20260508 08:46:06.152 | INFO | "huge" : "", | 20260508 08:46:06.152 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:06.152 | INFO | "malloc" : " 14 GB", | 20260508 08:46:06.152 | INFO | "pinned" : " 29 GB" | 20260508 08:46:06.152 | INFO | } | 20260508 08:46:06.152 | INFO | Perf report for the last 5.1465 seconds: | 20260508 08:46:06.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:06.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 08:46:06.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:46:06.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:06.152 | INFO | Uploaded 36GiB to GPU in 2.53048s (14.2265GiB/s) | 20260508 08:46:06.152 | INFO | Downloaded 34GiB to GPU in 2.45098s (13.872GiB/s) | 20260508 08:46:11.293 | INFO | Analyzing chunk frames = [6778368,6778880) | 20260508 08:46:11.293 | INFO | MemInfo: { | 20260508 08:46:11.293 | INFO | "gpu" : " 17 GB", | 20260508 08:46:11.293 | INFO | "huge" : "", | 20260508 08:46:11.293 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:11.293 | INFO | "malloc" : " 14 GB", | 20260508 08:46:11.293 | INFO | "pinned" : " 29 GB" | 20260508 08:46:11.293 | INFO | } | 20260508 08:46:11.293 | INFO | Perf report for the last 5.14085 seconds: | 20260508 08:46:11.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:11.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:46:11.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:46:11.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:11.293 | INFO | Uploaded 36GiB to GPU in 2.44383s (14.731GiB/s) | 20260508 08:46:11.293 | INFO | Downloaded 34GiB to GPU in 2.43759s (13.9482GiB/s) | 20260508 08:46:11.390 | INFO | Starting a round of Baz IO | 20260508 08:46:14.812 | INFO | Populating RTMetrics report | 20260508 08:46:14.812 | INFO | RTMetrics report populated | 20260508 08:46:14.812 | INFO | Opening tmp RTMetrics json file | 20260508 08:46:14.823 | INFO | Serializing RTMetrics to json | 20260508 08:46:14.824 | INFO | Serialization done, swapping tmp file now... | 20260508 08:46:14.825 | INFO | RTMetrics done | 20260508 08:46:14.878 | INFO | Finished a round of Baz IO: Wrote 1.494 GiB in 3.488 seconds | 20260508 08:46:14.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.62% | 20260508 08:46:14.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.28009% / 0% / 6.75519e-05% | 20260508 08:46:14.879 | INFO | Baz IO overall: | 20260508 08:46:14.879 | INFO | BytesWritten : 2969533328938 | 20260508 08:46:14.879 | INFO | NumEvents : 1429266551514 | 20260508 08:46:16.427 | INFO | Analyzing chunk frames = [6778880,6779392) | 20260508 08:46:16.427 | INFO | MemInfo: { | 20260508 08:46:16.427 | INFO | "gpu" : " 17 GB", | 20260508 08:46:16.427 | INFO | "huge" : "", | 20260508 08:46:16.427 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:16.427 | INFO | "malloc" : " 14 GB", | 20260508 08:46:16.427 | INFO | "pinned" : " 29 GB" | 20260508 08:46:16.427 | INFO | } | 20260508 08:46:16.427 | INFO | Perf report for the last 5.13432 seconds: | 20260508 08:46:16.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:16.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:16.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:16.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:16.427 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 08:46:16.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:16.427 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:46:16.427 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 08:46:16.427 | INFO | BAZWRITER : 67.3%, 2.6, 88.8ms, 0 | 20260508 08:46:16.427 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:46:16.427 | INFO | Uploaded 36GiB to GPU in 2.57441s (13.9838GiB/s) | 20260508 08:46:16.427 | INFO | Downloaded 38GiB to GPU in 2.7146s (13.9984GiB/s) | 20260508 08:46:21.568 | INFO | Analyzing chunk frames = [6779392,6779904) | 20260508 08:46:21.568 | INFO | MemInfo: { | 20260508 08:46:21.568 | INFO | "gpu" : " 17 GB", | 20260508 08:46:21.568 | INFO | "huge" : "", | 20260508 08:46:21.568 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:21.568 | INFO | "malloc" : " 14 GB", | 20260508 08:46:21.568 | INFO | "pinned" : " 29 GB" | 20260508 08:46:21.568 | INFO | } | 20260508 08:46:21.568 | INFO | Perf report for the last 5.14129 seconds: | 20260508 08:46:21.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:21.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.568 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 08:46:21.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.569 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:46:21.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:21.569 | INFO | Uploaded 36GiB to GPU in 2.51299s (14.3256GiB/s) | 20260508 08:46:21.569 | INFO | Downloaded 34GiB to GPU in 2.44912s (13.8825GiB/s) | 20260508 08:46:26.701 | INFO | Analyzing chunk frames = [6779904,6780416) | 20260508 08:46:26.701 | INFO | MemInfo: { | 20260508 08:46:26.701 | INFO | "gpu" : " 17 GB", | 20260508 08:46:26.701 | INFO | "huge" : "", | 20260508 08:46:26.701 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:26.701 | INFO | "malloc" : " 14 GB", | 20260508 08:46:26.701 | INFO | "pinned" : " 29 GB" | 20260508 08:46:26.702 | INFO | } | 20260508 08:46:26.702 | INFO | Perf report for the last 5.13331 seconds: | 20260508 08:46:26.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:26.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | ANALYSIS : 66.9%, 3.0, 103.7ms, 0 | 20260508 08:46:26.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:46:26.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:26.702 | INFO | Uploaded 36GiB to GPU in 2.49s (14.4578GiB/s) | 20260508 08:46:26.702 | INFO | Downloaded 34GiB to GPU in 2.43835s (13.9438GiB/s) | 20260508 08:46:31.847 | INFO | Analyzing chunk frames = [6780416,6780928) | 20260508 08:46:31.847 | INFO | MemInfo: { | 20260508 08:46:31.847 | INFO | "gpu" : " 17 GB", | 20260508 08:46:31.847 | INFO | "huge" : "", | 20260508 08:46:31.847 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:31.847 | INFO | "malloc" : " 14 GB", | 20260508 08:46:31.847 | INFO | "pinned" : " 29 GB" | 20260508 08:46:31.847 | INFO | } | 20260508 08:46:31.847 | INFO | Perf report for the last 5.14562 seconds: | 20260508 08:46:31.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:31.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.847 | INFO | ANALYSIS : 66.7%, 3.0, 103.4ms, 0 | 20260508 08:46:31.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.847 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:46:31.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:31.848 | INFO | Uploaded 36GiB to GPU in 2.55823s (14.0722GiB/s) | 20260508 08:46:31.848 | INFO | Downloaded 34GiB to GPU in 2.43763s (13.948GiB/s) | 20260508 08:46:36.985 | INFO | Analyzing chunk frames = [6780928,6781440) | 20260508 08:46:36.985 | INFO | MemInfo: { | 20260508 08:46:36.985 | INFO | "gpu" : " 17 GB", | 20260508 08:46:36.985 | INFO | "huge" : "", | 20260508 08:46:36.985 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:36.985 | INFO | "malloc" : " 14 GB", | 20260508 08:46:36.985 | INFO | "pinned" : " 29 GB" | 20260508 08:46:36.985 | INFO | } | 20260508 08:46:36.985 | INFO | Perf report for the last 5.13771 seconds: | 20260508 08:46:36.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:36.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.985 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:46:36.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.985 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260508 08:46:36.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:36.986 | INFO | Uploaded 36GiB to GPU in 2.36127s (15.246GiB/s) | 20260508 08:46:36.986 | INFO | Downloaded 34GiB to GPU in 2.50567s (13.5692GiB/s) | 20260508 08:46:42.127 | INFO | Analyzing chunk frames = [6781440,6781952) | 20260508 08:46:42.127 | INFO | MemInfo: { | 20260508 08:46:42.127 | INFO | "gpu" : " 17 GB", | 20260508 08:46:42.127 | INFO | "huge" : "", | 20260508 08:46:42.127 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:42.127 | INFO | "malloc" : " 14 GB", | 20260508 08:46:42.127 | INFO | "pinned" : " 29 GB" | 20260508 08:46:42.127 | INFO | } | 20260508 08:46:42.128 | INFO | Perf report for the last 5.14243 seconds: | 20260508 08:46:42.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:42.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:46:42.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:46:42.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:42.128 | INFO | Uploaded 36GiB to GPU in 2.39327s (15.0422GiB/s) | 20260508 08:46:42.128 | INFO | Downloaded 34GiB to GPU in 2.45623s (13.8423GiB/s) | 20260508 08:46:47.260 | INFO | Analyzing chunk frames = [6781952,6782464) | 20260508 08:46:47.260 | INFO | MemInfo: { | 20260508 08:46:47.260 | INFO | "gpu" : " 17 GB", | 20260508 08:46:47.260 | INFO | "huge" : "", | 20260508 08:46:47.260 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:47.260 | INFO | "malloc" : " 14 GB", | 20260508 08:46:47.260 | INFO | "pinned" : " 29 GB" | 20260508 08:46:47.260 | INFO | } | 20260508 08:46:47.260 | INFO | Perf report for the last 5.13235 seconds: | 20260508 08:46:47.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:47.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 08:46:47.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:46:47.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:47.260 | INFO | Uploaded 36GiB to GPU in 2.38553s (15.091GiB/s) | 20260508 08:46:47.260 | INFO | Downloaded 34GiB to GPU in 2.44486s (13.9067GiB/s) | 20260508 08:46:52.398 | INFO | Analyzing chunk frames = [6782464,6782976) | 20260508 08:46:52.398 | INFO | MemInfo: { | 20260508 08:46:52.398 | INFO | "gpu" : " 17 GB", | 20260508 08:46:52.398 | INFO | "huge" : "", | 20260508 08:46:52.398 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:52.398 | INFO | "malloc" : " 14 GB", | 20260508 08:46:52.398 | INFO | "pinned" : " 29 GB" | 20260508 08:46:52.398 | INFO | } | 20260508 08:46:52.399 | INFO | Perf report for the last 5.13859 seconds: | 20260508 08:46:52.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:52.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 08:46:52.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:46:52.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:52.399 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:46:52.399 | INFO | Total mem usage: 16.7 GiB | 20260508 08:46:52.399 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:46:52.399 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:46:52.399 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:46:52.399 | INFO | High Water Breakdown: | 20260508 08:46:52.399 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:46:52.399 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:46:52.399 | INFO | Uploaded 36GiB to GPU in 2.39006s (15.0624GiB/s) | 20260508 08:46:52.399 | INFO | Downloaded 34GiB to GPU in 2.46076s (13.8169GiB/s) | 20260508 08:46:52.399 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:46:52.399 | INFO | ------------------------------------------ | 20260508 08:46:52.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:46:52.399 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:46:52.399 | INFO | Total mem usage: 26.5 GiB | 20260508 08:46:52.399 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:46:52.399 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:46:52.399 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:46:52.399 | INFO | High Water Breakdown: | 20260508 08:46:52.399 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:46:52.399 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:46:52.399 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:46:52.399 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:46:52.399 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:46:52.399 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:46:52.399 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:46:52.399 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:46:52.399 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:46:52.399 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:46:52.399 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:46:52.399 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:46:52.399 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:46:52.399 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:46:52.399 | INFO | ------------------------------------------ | 20260508 08:46:52.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:46:52.399 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:46:52.399 | INFO | Total mem usage: 27.8 GiB | 20260508 08:46:52.399 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:46:52.399 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:46:52.399 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:46:52.399 | INFO | High Water Breakdown: | 20260508 08:46:52.399 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:46:52.399 | INFO | ------------------------------------------ | 20260508 08:46:52.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:46:52.399 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:46:52.399 | INFO | Total mem usage: 15.7 GiB | 20260508 08:46:52.399 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:46:52.399 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:46:52.399 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:46:52.399 | INFO | High Water Breakdown: | 20260508 08:46:52.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:46:52.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:46:52.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:46:52.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:46:52.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:46:52.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:46:52.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:46:52.399 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:46:52.399 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:46:52.399 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:46:52.399 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:46:52.399 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:46:52.399 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:46:52.399 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:46:52.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:46:52.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:46:52.399 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:46:52.399 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:46:52.399 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:46:52.399 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:46:52.399 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:46:52.399 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:46:52.399 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:46:52.399 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:46:52.399 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:46:52.400 | INFO | ------------------------------------------ | 20260508 08:46:52.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:46:52.400 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:46:52.400 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:46:52.400 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:46:52.400 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.42e+01 / 7.22e-01 | 20260508 08:46:52.400 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.21e-02 / 5.97e-03 | 20260508 08:46:52.400 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.42e+00, 5.93e-01 / 2.48e-01 | 20260508 08:46:52.400 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.31e-01 | 20260508 08:46:52.400 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.51e+01, 3.87e+00 / 1.41e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.89e+00 / 1.42e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.46e+00, 4.12e-01 / 2.99e-01 | 20260508 08:46:52.400 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.16e+00, 2.52e-01 / 1.56e-01 | 20260508 08:46:52.400 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.35e-01, 4.98e-02 / 2.82e-02 | 20260508 08:46:52.400 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:46:52.400 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:46:52.400 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:46:52.400 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.49e+01 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 24.38%, Upload -- 2.58e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 24.49%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 30.98%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 2.21%, LockUpload -- 2.34e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 2.58%, LockDownload -- 2.72e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:46:52.400 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 08:46:52.496 | INFO | Starting a round of Baz IO | 20260508 08:46:55.928 | INFO | Populating RTMetrics report | 20260508 08:46:55.928 | INFO | RTMetrics report populated | 20260508 08:46:55.928 | INFO | Opening tmp RTMetrics json file | 20260508 08:46:55.937 | INFO | Serializing RTMetrics to json | 20260508 08:46:55.939 | INFO | Serialization done, swapping tmp file now... | 20260508 08:46:55.940 | INFO | RTMetrics done | 20260508 08:46:55.996 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.499 seconds | 20260508 08:46:55.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.62% | 20260508 08:46:55.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280996% / 0% / 5.5631e-05% | 20260508 08:46:55.996 | INFO | Baz IO overall: | 20260508 08:46:55.996 | INFO | BytesWritten : 2971139382564 | 20260508 08:46:55.996 | INFO | NumEvents : 1430026941163 | 20260508 08:46:57.540 | INFO | Analyzing chunk frames = [6782976,6783488) | 20260508 08:46:57.540 | INFO | MemInfo: { | 20260508 08:46:57.540 | INFO | "gpu" : " 17 GB", | 20260508 08:46:57.540 | INFO | "huge" : "", | 20260508 08:46:57.540 | INFO | "huge_shared" : " 29 GB", | 20260508 08:46:57.540 | INFO | "malloc" : " 14 GB", | 20260508 08:46:57.540 | INFO | "pinned" : " 29 GB" | 20260508 08:46:57.540 | INFO | } | 20260508 08:46:57.540 | INFO | Perf report for the last 5.14172 seconds: | 20260508 08:46:57.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:46:57.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:57.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:57.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:57.540 | INFO | ANALYSIS : 67.8%, 3.0, 106.0ms, 0 | 20260508 08:46:57.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:46:57.540 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:46:57.540 | INFO | PRE_HQ : 65.4%, 1.6, 54.5ms, 0 | 20260508 08:46:57.540 | INFO | BAZWRITER : 67.4%, 2.5, 87.7ms, 0 | 20260508 08:46:57.541 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 08:46:57.541 | INFO | Uploaded 36GiB to GPU in 2.5812s (13.947GiB/s) | 20260508 08:46:57.541 | INFO | Downloaded 38GiB to GPU in 2.71376s (14.0027GiB/s) | 20260508 08:47:02.681 | INFO | Analyzing chunk frames = [6783488,6784000) | 20260508 08:47:02.681 | INFO | MemInfo: { | 20260508 08:47:02.681 | INFO | "gpu" : " 17 GB", | 20260508 08:47:02.681 | INFO | "huge" : "", | 20260508 08:47:02.681 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:02.681 | INFO | "malloc" : " 14 GB", | 20260508 08:47:02.681 | INFO | "pinned" : " 29 GB" | 20260508 08:47:02.681 | INFO | } | 20260508 08:47:02.681 | INFO | Perf report for the last 5.14062 seconds: | 20260508 08:47:02.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:02.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:47:02.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:47:02.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:02.681 | INFO | Uploaded 36GiB to GPU in 2.45816s (14.6451GiB/s) | 20260508 08:47:02.681 | INFO | Downloaded 34GiB to GPU in 2.45399s (13.855GiB/s) | 20260508 08:47:07.823 | INFO | Analyzing chunk frames = [6784000,6784512) | 20260508 08:47:07.823 | INFO | MemInfo: { | 20260508 08:47:07.823 | INFO | "gpu" : " 17 GB", | 20260508 08:47:07.823 | INFO | "huge" : "", | 20260508 08:47:07.823 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:07.823 | INFO | "malloc" : " 14 GB", | 20260508 08:47:07.823 | INFO | "pinned" : " 29 GB" | 20260508 08:47:07.823 | INFO | } | 20260508 08:47:07.823 | INFO | Perf report for the last 5.14206 seconds: | 20260508 08:47:07.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:07.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 08:47:07.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:47:07.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:07.823 | INFO | Uploaded 36GiB to GPU in 2.47109s (14.5685GiB/s) | 20260508 08:47:07.823 | INFO | Downloaded 34GiB to GPU in 2.4581s (13.8318GiB/s) | 20260508 08:47:12.963 | INFO | Analyzing chunk frames = [6784512,6785024) | 20260508 08:47:12.963 | INFO | MemInfo: { | 20260508 08:47:12.963 | INFO | "gpu" : " 17 GB", | 20260508 08:47:12.963 | INFO | "huge" : "", | 20260508 08:47:12.963 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:12.963 | INFO | "malloc" : " 14 GB", | 20260508 08:47:12.963 | INFO | "pinned" : " 29 GB" | 20260508 08:47:12.963 | INFO | } | 20260508 08:47:12.963 | INFO | Perf report for the last 5.14001 seconds: | 20260508 08:47:12.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:12.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:47:12.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:47:12.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:12.963 | INFO | Uploaded 36GiB to GPU in 2.49289s (14.4411GiB/s) | 20260508 08:47:12.963 | INFO | Downloaded 34GiB to GPU in 2.44695s (13.8948GiB/s) | 20260508 08:47:18.106 | INFO | Analyzing chunk frames = [6785024,6785536) | 20260508 08:47:18.106 | INFO | MemInfo: { | 20260508 08:47:18.106 | INFO | "gpu" : " 17 GB", | 20260508 08:47:18.106 | INFO | "huge" : "", | 20260508 08:47:18.106 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:18.106 | INFO | "malloc" : " 14 GB", | 20260508 08:47:18.106 | INFO | "pinned" : " 29 GB" | 20260508 08:47:18.106 | INFO | } | 20260508 08:47:18.106 | INFO | Perf report for the last 5.14307 seconds: | 20260508 08:47:18.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:18.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.106 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 08:47:18.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.106 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:47:18.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:18.107 | INFO | Uploaded 36GiB to GPU in 2.37311s (15.17GiB/s) | 20260508 08:47:18.107 | INFO | Downloaded 34GiB to GPU in 2.55934s (13.2847GiB/s) | 20260508 08:47:23.239 | INFO | Analyzing chunk frames = [6785536,6786048) | 20260508 08:47:23.239 | INFO | MemInfo: { | 20260508 08:47:23.239 | INFO | "gpu" : " 17 GB", | 20260508 08:47:23.239 | INFO | "huge" : "", | 20260508 08:47:23.240 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:23.240 | INFO | "malloc" : " 14 GB", | 20260508 08:47:23.240 | INFO | "pinned" : " 29 GB" | 20260508 08:47:23.240 | INFO | } | 20260508 08:47:23.240 | INFO | Perf report for the last 5.13356 seconds: | 20260508 08:47:23.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:23.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 08:47:23.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:47:23.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:23.240 | INFO | Uploaded 36GiB to GPU in 2.36087s (15.2486GiB/s) | 20260508 08:47:23.240 | INFO | Downloaded 34GiB to GPU in 2.44496s (13.9062GiB/s) | 20260508 08:47:28.376 | INFO | Analyzing chunk frames = [6786048,6786560) | 20260508 08:47:28.376 | INFO | MemInfo: { | 20260508 08:47:28.376 | INFO | "gpu" : " 17 GB", | 20260508 08:47:28.376 | INFO | "huge" : "", | 20260508 08:47:28.376 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:28.376 | INFO | "malloc" : " 14 GB", | 20260508 08:47:28.376 | INFO | "pinned" : " 29 GB" | 20260508 08:47:28.376 | INFO | } | 20260508 08:47:28.376 | INFO | Perf report for the last 5.13643 seconds: | 20260508 08:47:28.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:28.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.376 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:47:28.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.376 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:47:28.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:28.377 | INFO | Uploaded 36GiB to GPU in 2.42496s (14.8456GiB/s) | 20260508 08:47:28.377 | INFO | Downloaded 34GiB to GPU in 2.43894s (13.9405GiB/s) | 20260508 08:47:33.514 | INFO | Analyzing chunk frames = [6786560,6787072) | 20260508 08:47:33.514 | INFO | MemInfo: { | 20260508 08:47:33.514 | INFO | "gpu" : " 17 GB", | 20260508 08:47:33.514 | INFO | "huge" : "", | 20260508 08:47:33.514 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:33.514 | INFO | "malloc" : " 14 GB", | 20260508 08:47:33.514 | INFO | "pinned" : " 29 GB" | 20260508 08:47:33.514 | INFO | } | 20260508 08:47:33.514 | INFO | Perf report for the last 5.13801 seconds: | 20260508 08:47:33.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:33.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 08:47:33.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:47:33.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:33.514 | INFO | Uploaded 36GiB to GPU in 2.42183s (14.8648GiB/s) | 20260508 08:47:33.514 | INFO | Downloaded 34GiB to GPU in 2.43844s (13.9433GiB/s) | 20260508 08:47:33.611 | INFO | Starting a round of Baz IO | 20260508 08:47:37.025 | INFO | Populating RTMetrics report | 20260508 08:47:37.025 | INFO | RTMetrics report populated | 20260508 08:47:37.025 | INFO | Opening tmp RTMetrics json file | 20260508 08:47:37.034 | INFO | Serializing RTMetrics to json | 20260508 08:47:37.035 | INFO | Serialization done, swapping tmp file now... | 20260508 08:47:37.036 | INFO | RTMetrics done | 20260508 08:47:37.099 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.487 seconds | 20260508 08:47:37.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.62% | 20260508 08:47:37.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280078% / 0% / 7.15256e-05% | 20260508 08:47:37.099 | INFO | Baz IO overall: | 20260508 08:47:37.099 | INFO | BytesWritten : 2972745234213 | 20260508 08:47:37.099 | INFO | NumEvents : 1430787114142 | 20260508 08:47:38.656 | INFO | Analyzing chunk frames = [6787072,6787584) | 20260508 08:47:38.656 | INFO | MemInfo: { | 20260508 08:47:38.656 | INFO | "gpu" : " 17 GB", | 20260508 08:47:38.656 | INFO | "huge" : "", | 20260508 08:47:38.656 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:38.656 | INFO | "malloc" : " 14 GB", | 20260508 08:47:38.656 | INFO | "pinned" : " 29 GB" | 20260508 08:47:38.656 | INFO | } | 20260508 08:47:38.656 | INFO | Perf report for the last 5.14173 seconds: | 20260508 08:47:38.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:38.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:38.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:38.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:38.656 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:47:38.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:38.656 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 08:47:38.656 | INFO | PRE_HQ : 65.3%, 1.6, 54.8ms, 0 | 20260508 08:47:38.656 | INFO | BAZWRITER : 67.1%, 2.6, 88.1ms, 0 | 20260508 08:47:38.656 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:47:38.656 | INFO | Uploaded 36GiB to GPU in 2.58461s (13.9286GiB/s) | 20260508 08:47:38.656 | INFO | Downloaded 38GiB to GPU in 2.70704s (14.0375GiB/s) | 20260508 08:47:43.800 | INFO | Analyzing chunk frames = [6787584,6788096) | 20260508 08:47:43.800 | INFO | MemInfo: { | 20260508 08:47:43.800 | INFO | "gpu" : " 17 GB", | 20260508 08:47:43.800 | INFO | "huge" : "", | 20260508 08:47:43.800 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:43.800 | INFO | "malloc" : " 14 GB", | 20260508 08:47:43.800 | INFO | "pinned" : " 29 GB" | 20260508 08:47:43.800 | INFO | } | 20260508 08:47:43.800 | INFO | Perf report for the last 5.14413 seconds: | 20260508 08:47:43.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:43.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 08:47:43.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260508 08:47:43.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:43.800 | INFO | Uploaded 36GiB to GPU in 2.53646s (14.193GiB/s) | 20260508 08:47:43.800 | INFO | Downloaded 34GiB to GPU in 2.42784s (14.0042GiB/s) | 20260508 08:47:48.936 | INFO | Analyzing chunk frames = [6788096,6788608) | 20260508 08:47:48.936 | INFO | MemInfo: { | 20260508 08:47:48.936 | INFO | "gpu" : " 17 GB", | 20260508 08:47:48.936 | INFO | "huge" : "", | 20260508 08:47:48.936 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:48.936 | INFO | "malloc" : " 14 GB", | 20260508 08:47:48.936 | INFO | "pinned" : " 29 GB" | 20260508 08:47:48.936 | INFO | } | 20260508 08:47:48.936 | INFO | Perf report for the last 5.13637 seconds: | 20260508 08:47:48.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:48.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.936 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 08:47:48.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.936 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:47:48.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:48.937 | INFO | Uploaded 36GiB to GPU in 2.5221s (14.2738GiB/s) | 20260508 08:47:48.937 | INFO | Downloaded 34GiB to GPU in 2.43404s (13.9685GiB/s) | 20260508 08:47:54.073 | INFO | Analyzing chunk frames = [6788608,6789120) | 20260508 08:47:54.073 | INFO | MemInfo: { | 20260508 08:47:54.073 | INFO | "gpu" : " 17 GB", | 20260508 08:47:54.073 | INFO | "huge" : "", | 20260508 08:47:54.073 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:54.073 | INFO | "malloc" : " 14 GB", | 20260508 08:47:54.073 | INFO | "pinned" : " 29 GB" | 20260508 08:47:54.073 | INFO | } | 20260508 08:47:54.073 | INFO | Perf report for the last 5.13709 seconds: | 20260508 08:47:54.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:54.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.073 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 08:47:54.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.074 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:47:54.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:54.074 | INFO | Uploaded 36GiB to GPU in 2.51981s (14.2868GiB/s) | 20260508 08:47:54.074 | INFO | Downloaded 34GiB to GPU in 2.43525s (13.9616GiB/s) | 20260508 08:47:59.210 | INFO | Analyzing chunk frames = [6789120,6789632) | 20260508 08:47:59.210 | INFO | MemInfo: { | 20260508 08:47:59.210 | INFO | "gpu" : " 17 GB", | 20260508 08:47:59.210 | INFO | "huge" : "", | 20260508 08:47:59.210 | INFO | "huge_shared" : " 29 GB", | 20260508 08:47:59.210 | INFO | "malloc" : " 14 GB", | 20260508 08:47:59.210 | INFO | "pinned" : " 29 GB" | 20260508 08:47:59.210 | INFO | } | 20260508 08:47:59.210 | INFO | Perf report for the last 5.13656 seconds: | 20260508 08:47:59.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:47:59.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:47:59.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260508 08:47:59.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:47:59.210 | INFO | Uploaded 36GiB to GPU in 2.44042s (14.7516GiB/s) | 20260508 08:47:59.210 | INFO | Downloaded 34GiB to GPU in 2.51314s (13.5289GiB/s) | 20260508 08:48:04.351 | INFO | Analyzing chunk frames = [6789632,6790144) | 20260508 08:48:04.351 | INFO | MemInfo: { | 20260508 08:48:04.351 | INFO | "gpu" : " 17 GB", | 20260508 08:48:04.351 | INFO | "huge" : "", | 20260508 08:48:04.351 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:04.351 | INFO | "malloc" : " 14 GB", | 20260508 08:48:04.351 | INFO | "pinned" : " 29 GB" | 20260508 08:48:04.351 | INFO | } | 20260508 08:48:04.351 | INFO | Perf report for the last 5.14117 seconds: | 20260508 08:48:04.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:04.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 08:48:04.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:48:04.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:04.351 | INFO | Uploaded 36GiB to GPU in 2.40163s (14.9898GiB/s) | 20260508 08:48:04.351 | INFO | Downloaded 34GiB to GPU in 2.4396s (13.9367GiB/s) | 20260508 08:48:09.494 | INFO | Analyzing chunk frames = [6790144,6790656) | 20260508 08:48:09.495 | INFO | MemInfo: { | 20260508 08:48:09.495 | INFO | "gpu" : " 17 GB", | 20260508 08:48:09.495 | INFO | "huge" : "", | 20260508 08:48:09.495 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:09.495 | INFO | "malloc" : " 14 GB", | 20260508 08:48:09.495 | INFO | "pinned" : " 29 GB" | 20260508 08:48:09.495 | INFO | } | 20260508 08:48:09.495 | INFO | Perf report for the last 5.14349 seconds: | 20260508 08:48:09.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:09.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260508 08:48:09.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 08:48:09.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:09.495 | INFO | Uploaded 36GiB to GPU in 2.44297s (14.7362GiB/s) | 20260508 08:48:09.495 | INFO | Downloaded 34GiB to GPU in 2.44606s (13.8999GiB/s) | 20260508 08:48:14.636 | INFO | Analyzing chunk frames = [6790656,6791168) | 20260508 08:48:14.636 | INFO | MemInfo: { | 20260508 08:48:14.636 | INFO | "gpu" : " 17 GB", | 20260508 08:48:14.636 | INFO | "huge" : "", | 20260508 08:48:14.636 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:14.636 | INFO | "malloc" : " 14 GB", | 20260508 08:48:14.636 | INFO | "pinned" : " 29 GB" | 20260508 08:48:14.636 | INFO | } | 20260508 08:48:14.636 | INFO | Perf report for the last 5.14155 seconds: | 20260508 08:48:14.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:14.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.636 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:48:14.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.636 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:48:14.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:14.637 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:48:14.637 | INFO | Total mem usage: 16.7 GiB | 20260508 08:48:14.637 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:48:14.637 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:48:14.637 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:48:14.637 | INFO | High Water Breakdown: | 20260508 08:48:14.637 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:48:14.637 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:48:14.637 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:48:14.637 | INFO | ------------------------------------------ | 20260508 08:48:14.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:48:14.637 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:48:14.637 | INFO | Total mem usage: 26.5 GiB | 20260508 08:48:14.637 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:48:14.637 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:48:14.637 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:48:14.637 | INFO | High Water Breakdown: | 20260508 08:48:14.637 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:48:14.637 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:48:14.637 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:48:14.637 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:48:14.637 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:48:14.637 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:48:14.637 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:48:14.637 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:48:14.637 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:48:14.637 | INFO | Uploaded 36GiB to GPU in 2.44383s (14.731GiB/s) | 20260508 08:48:14.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:48:14.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:48:14.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:48:14.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:48:14.637 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:48:14.637 | INFO | ------------------------------------------ | 20260508 08:48:14.637 | INFO | Downloaded 34GiB to GPU in 2.44467s (13.9078GiB/s) | 20260508 08:48:14.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:48:14.637 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:48:14.637 | INFO | Total mem usage: 27.8 GiB | 20260508 08:48:14.637 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:48:14.637 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:48:14.637 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:48:14.637 | INFO | High Water Breakdown: | 20260508 08:48:14.637 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:48:14.637 | INFO | ------------------------------------------ | 20260508 08:48:14.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:48:14.637 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:48:14.637 | INFO | Total mem usage: 15.7 GiB | 20260508 08:48:14.637 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:48:14.637 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:48:14.637 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:48:14.637 | INFO | High Water Breakdown: | 20260508 08:48:14.637 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:48:14.637 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:48:14.637 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:48:14.637 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:48:14.637 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:48:14.637 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:48:14.637 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:48:14.637 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:48:14.637 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:48:14.637 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:48:14.637 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:48:14.637 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:48:14.637 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:48:14.637 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:48:14.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:48:14.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:48:14.637 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:48:14.637 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:48:14.637 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:48:14.637 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:48:14.637 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:48:14.637 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:48:14.637 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:48:14.637 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:48:14.637 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:48:14.637 | INFO | ------------------------------------------ | 20260508 08:48:14.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:48:14.637 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:48:14.637 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:48:14.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:48:14.637 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.40e+01 / 7.13e-01 | 20260508 08:48:14.637 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.06e-01, 1.11e-02 / 5.98e-03 | 20260508 08:48:14.637 | INFO | PerfReport: 12.11%, Baseline -- 3.98e+00 / 4.43e+00, 5.91e-01 / 2.50e-01 | 20260508 08:48:14.637 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.09e-01 / 1.29e-01 | 20260508 08:48:14.638 | INFO | PerfReport: 15.92%, DME -- 4.18e+01 / 4.51e+01, 3.70e+00 / 1.20e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.88e+01 / 2.02e+01, 1.94e+00 / 1.36e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.32e+00 / 2.56e+00, 4.11e-01 / 2.32e-01 | 20260508 08:48:14.638 | INFO | PerfReport: 3.03%, Metrics -- 9.96e-01 / 1.17e+00, 2.51e-01 / 1.24e-01 | 20260508 08:48:14.638 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.42e-01, 4.99e-02 / 2.55e-02 | 20260508 08:48:14.638 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:48:14.638 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:48:14.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:48:14.638 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.48e+01 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 24.34%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 24.54%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 31.11%, Compute -- 3.29e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 2.17%, LockUpload -- 2.29e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 2.46%, LockDownload -- 2.60e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 08:48:14.638 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260508 08:48:14.734 | INFO | Starting a round of Baz IO | 20260508 08:48:18.141 | INFO | Populating RTMetrics report | 20260508 08:48:18.141 | INFO | RTMetrics report populated | 20260508 08:48:18.141 | INFO | Opening tmp RTMetrics json file | 20260508 08:48:18.154 | INFO | Serializing RTMetrics to json | 20260508 08:48:18.155 | INFO | Serialization done, swapping tmp file now... | 20260508 08:48:18.155 | INFO | RTMetrics done | 20260508 08:48:18.212 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.478 seconds | 20260508 08:48:18.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.62% | 20260508 08:48:18.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280603% / 0% / 7.15256e-05% | 20260508 08:48:18.213 | INFO | Baz IO overall: | 20260508 08:48:18.213 | INFO | BytesWritten : 2974351627531 | 20260508 08:48:18.213 | INFO | NumEvents : 1431547536123 | 20260508 08:48:19.768 | INFO | Analyzing chunk frames = [6791168,6791680) | 20260508 08:48:19.768 | INFO | MemInfo: { | 20260508 08:48:19.768 | INFO | "gpu" : " 17 GB", | 20260508 08:48:19.768 | INFO | "huge" : "", | 20260508 08:48:19.768 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:19.768 | INFO | "malloc" : " 14 GB", | 20260508 08:48:19.768 | INFO | "pinned" : " 29 GB" | 20260508 08:48:19.768 | INFO | } | 20260508 08:48:19.768 | INFO | Perf report for the last 5.13198 seconds: | 20260508 08:48:19.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:19.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:19.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:19.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:19.768 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 08:48:19.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:19.768 | INFO | RT_METRICS : 38.9%, 1.0, 20.3ms, 0 | 20260508 08:48:19.768 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 08:48:19.768 | INFO | BAZWRITER : 67.1%, 2.6, 88.4ms, 0 | 20260508 08:48:19.768 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:48:19.769 | INFO | Uploaded 36GiB to GPU in 2.56676s (14.0254GiB/s) | 20260508 08:48:19.769 | INFO | Downloaded 38GiB to GPU in 2.70798s (14.0326GiB/s) | 20260508 08:48:24.914 | INFO | Analyzing chunk frames = [6791680,6792192) | 20260508 08:48:24.914 | INFO | MemInfo: { | 20260508 08:48:24.914 | INFO | "gpu" : " 17 GB", | 20260508 08:48:24.914 | INFO | "huge" : "", | 20260508 08:48:24.914 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:24.914 | INFO | "malloc" : " 14 GB", | 20260508 08:48:24.914 | INFO | "pinned" : " 29 GB" | 20260508 08:48:24.914 | INFO | } | 20260508 08:48:24.914 | INFO | Perf report for the last 5.14578 seconds: | 20260508 08:48:24.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:24.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:48:24.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:48:24.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:24.914 | INFO | Uploaded 36GiB to GPU in 2.51528s (14.3125GiB/s) | 20260508 08:48:24.914 | INFO | Downloaded 34GiB to GPU in 2.44105s (13.9285GiB/s) | 20260508 08:48:30.051 | INFO | Analyzing chunk frames = [6792192,6792704) | 20260508 08:48:30.051 | INFO | MemInfo: { | 20260508 08:48:30.051 | INFO | "gpu" : " 17 GB", | 20260508 08:48:30.051 | INFO | "huge" : "", | 20260508 08:48:30.051 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:30.051 | INFO | "malloc" : " 14 GB", | 20260508 08:48:30.051 | INFO | "pinned" : " 29 GB" | 20260508 08:48:30.051 | INFO | } | 20260508 08:48:30.051 | INFO | Perf report for the last 5.13736 seconds: | 20260508 08:48:30.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:30.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.052 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:48:30.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.052 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:48:30.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:30.052 | INFO | Uploaded 36GiB to GPU in 2.4923s (14.4445GiB/s) | 20260508 08:48:30.052 | INFO | Downloaded 34GiB to GPU in 2.43573s (13.9589GiB/s) | 20260508 08:48:35.191 | INFO | Analyzing chunk frames = [6792704,6793216) | 20260508 08:48:35.191 | INFO | MemInfo: { | 20260508 08:48:35.191 | INFO | "gpu" : " 17 GB", | 20260508 08:48:35.191 | INFO | "huge" : "", | 20260508 08:48:35.191 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:35.191 | INFO | "malloc" : " 14 GB", | 20260508 08:48:35.191 | INFO | "pinned" : " 29 GB" | 20260508 08:48:35.191 | INFO | } | 20260508 08:48:35.191 | INFO | Perf report for the last 5.13999 seconds: | 20260508 08:48:35.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:35.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.192 | INFO | ANALYSIS : 66.9%, 3.0, 103.5ms, 0 | 20260508 08:48:35.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.192 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:48:35.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:35.192 | INFO | Uploaded 36GiB to GPU in 2.57439s (13.9839GiB/s) | 20260508 08:48:35.192 | INFO | Downloaded 34GiB to GPU in 2.43486s (13.9638GiB/s) | 20260508 08:48:40.328 | INFO | Analyzing chunk frames = [6793216,6793728) | 20260508 08:48:40.328 | INFO | MemInfo: { | 20260508 08:48:40.328 | INFO | "gpu" : " 17 GB", | 20260508 08:48:40.328 | INFO | "huge" : "", | 20260508 08:48:40.329 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:40.329 | INFO | "malloc" : " 14 GB", | 20260508 08:48:40.329 | INFO | "pinned" : " 29 GB" | 20260508 08:48:40.329 | INFO | } | 20260508 08:48:40.329 | INFO | Perf report for the last 5.13723 seconds: | 20260508 08:48:40.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:40.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:48:40.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 08:48:40.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:40.329 | INFO | Uploaded 36GiB to GPU in 2.41266s (14.9213GiB/s) | 20260508 08:48:40.329 | INFO | Downloaded 34GiB to GPU in 2.53454s (13.4147GiB/s) | 20260508 08:48:45.466 | INFO | Analyzing chunk frames = [6793728,6794240) | 20260508 08:48:45.466 | INFO | MemInfo: { | 20260508 08:48:45.466 | INFO | "gpu" : " 17 GB", | 20260508 08:48:45.466 | INFO | "huge" : "", | 20260508 08:48:45.466 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:45.466 | INFO | "malloc" : " 14 GB", | 20260508 08:48:45.466 | INFO | "pinned" : " 29 GB" | 20260508 08:48:45.466 | INFO | } | 20260508 08:48:45.466 | INFO | Perf report for the last 5.13772 seconds: | 20260508 08:48:45.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:45.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.466 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:48:45.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.467 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:48:45.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:45.467 | INFO | Uploaded 36GiB to GPU in 2.37053s (15.1865GiB/s) | 20260508 08:48:45.467 | INFO | Downloaded 34GiB to GPU in 2.43105s (13.9857GiB/s) | 20260508 08:48:50.608 | INFO | Analyzing chunk frames = [6794240,6794752) | 20260508 08:48:50.608 | INFO | MemInfo: { | 20260508 08:48:50.608 | INFO | "gpu" : " 17 GB", | 20260508 08:48:50.608 | INFO | "huge" : "", | 20260508 08:48:50.608 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:50.608 | INFO | "malloc" : " 14 GB", | 20260508 08:48:50.608 | INFO | "pinned" : " 29 GB" | 20260508 08:48:50.608 | INFO | } | 20260508 08:48:50.608 | INFO | Perf report for the last 5.14168 seconds: | 20260508 08:48:50.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:50.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:48:50.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 08:48:50.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:50.608 | INFO | Uploaded 36GiB to GPU in 2.42472s (14.8471GiB/s) | 20260508 08:48:50.608 | INFO | Downloaded 34GiB to GPU in 2.44826s (13.8874GiB/s) | 20260508 08:48:55.750 | INFO | Analyzing chunk frames = [6794752,6795264) | 20260508 08:48:55.750 | INFO | MemInfo: { | 20260508 08:48:55.750 | INFO | "gpu" : " 17 GB", | 20260508 08:48:55.751 | INFO | "huge" : "", | 20260508 08:48:55.751 | INFO | "huge_shared" : " 29 GB", | 20260508 08:48:55.751 | INFO | "malloc" : " 14 GB", | 20260508 08:48:55.751 | INFO | "pinned" : " 29 GB" | 20260508 08:48:55.751 | INFO | } | 20260508 08:48:55.751 | INFO | Perf report for the last 5.14249 seconds: | 20260508 08:48:55.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:48:55.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | ANALYSIS : 65.8%, 3.0, 103.0ms, 0 | 20260508 08:48:55.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:48:55.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:48:55.751 | INFO | Uploaded 36GiB to GPU in 2.44777s (14.7073GiB/s) | 20260508 08:48:55.751 | INFO | Downloaded 34GiB to GPU in 2.44997s (13.8777GiB/s) | 20260508 08:48:55.848 | INFO | Starting a round of Baz IO | 20260508 08:48:59.267 | INFO | Populating RTMetrics report | 20260508 08:48:59.268 | INFO | RTMetrics report populated | 20260508 08:48:59.268 | INFO | Opening tmp RTMetrics json file | 20260508 08:48:59.276 | INFO | Serializing RTMetrics to json | 20260508 08:48:59.277 | INFO | Serialization done, swapping tmp file now... | 20260508 08:48:59.278 | INFO | RTMetrics done | 20260508 08:48:59.333 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.485 seconds | 20260508 08:48:59.333 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.62% | 20260508 08:48:59.333 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280023% / 3.97364e-06% / 3.97364e-05% | 20260508 08:48:59.333 | INFO | Baz IO overall: | 20260508 08:48:59.333 | INFO | BytesWritten : 2975958302948 | 20260508 08:48:59.333 | INFO | NumEvents : 1432308346809 | 20260508 08:49:00.885 | INFO | Analyzing chunk frames = [6795264,6795776) | 20260508 08:49:00.885 | INFO | MemInfo: { | 20260508 08:49:00.885 | INFO | "gpu" : " 17 GB", | 20260508 08:49:00.885 | INFO | "huge" : "", | 20260508 08:49:00.885 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:00.885 | INFO | "malloc" : " 14 GB", | 20260508 08:49:00.885 | INFO | "pinned" : " 29 GB" | 20260508 08:49:00.885 | INFO | } | 20260508 08:49:00.885 | INFO | Perf report for the last 5.1348 seconds: | 20260508 08:49:00.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:00.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:00.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:00.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:00.885 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 08:49:00.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:00.885 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:49:00.885 | INFO | PRE_HQ : 65.3%, 1.6, 54.6ms, 0 | 20260508 08:49:00.886 | INFO | BAZWRITER : 67.3%, 2.5, 87.9ms, 0 | 20260508 08:49:00.886 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 08:49:00.886 | INFO | Uploaded 36GiB to GPU in 2.5757s (13.9768GiB/s) | 20260508 08:49:00.886 | INFO | Downloaded 38GiB to GPU in 2.71263s (14.0085GiB/s) | 20260508 08:49:06.019 | INFO | Analyzing chunk frames = [6795776,6796288) | 20260508 08:49:06.019 | INFO | MemInfo: { | 20260508 08:49:06.019 | INFO | "gpu" : " 17 GB", | 20260508 08:49:06.019 | INFO | "huge" : "", | 20260508 08:49:06.019 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:06.019 | INFO | "malloc" : " 14 GB", | 20260508 08:49:06.019 | INFO | "pinned" : " 29 GB" | 20260508 08:49:06.019 | INFO | } | 20260508 08:49:06.019 | INFO | Perf report for the last 5.13391 seconds: | 20260508 08:49:06.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:06.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.019 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 08:49:06.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.019 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:49:06.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:06.020 | INFO | Uploaded 36GiB to GPU in 2.5097s (14.3443GiB/s) | 20260508 08:49:06.020 | INFO | Downloaded 34GiB to GPU in 2.44129s (13.9271GiB/s) | 20260508 08:49:11.165 | INFO | Analyzing chunk frames = [6796288,6796800) | 20260508 08:49:11.165 | INFO | MemInfo: { | 20260508 08:49:11.165 | INFO | "gpu" : " 17 GB", | 20260508 08:49:11.165 | INFO | "huge" : "", | 20260508 08:49:11.166 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:11.166 | INFO | "malloc" : " 14 GB", | 20260508 08:49:11.166 | INFO | "pinned" : " 29 GB" | 20260508 08:49:11.166 | INFO | } | 20260508 08:49:11.166 | INFO | Perf report for the last 5.14627 seconds: | 20260508 08:49:11.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:11.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | ANALYSIS : 67.1%, 3.0, 104.4ms, 0 | 20260508 08:49:11.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:49:11.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:11.166 | INFO | Uploaded 36GiB to GPU in 2.55737s (14.077GiB/s) | 20260508 08:49:11.166 | INFO | Downloaded 34GiB to GPU in 2.43438s (13.9666GiB/s) | 20260508 08:49:16.303 | INFO | Analyzing chunk frames = [6796800,6797312) | 20260508 08:49:16.303 | INFO | MemInfo: { | 20260508 08:49:16.303 | INFO | "gpu" : " 17 GB", | 20260508 08:49:16.303 | INFO | "huge" : "", | 20260508 08:49:16.303 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:16.303 | INFO | "malloc" : " 14 GB", | 20260508 08:49:16.303 | INFO | "pinned" : " 29 GB" | 20260508 08:49:16.303 | INFO | } | 20260508 08:49:16.303 | INFO | Perf report for the last 5.13757 seconds: | 20260508 08:49:16.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:16.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | ANALYSIS : 67.0%, 2.9, 103.6ms, 0 | 20260508 08:49:16.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:49:16.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:16.303 | INFO | Uploaded 36GiB to GPU in 2.56789s (14.0193GiB/s) | 20260508 08:49:16.303 | INFO | Downloaded 34GiB to GPU in 2.43198s (13.9804GiB/s) | 20260508 08:49:21.446 | INFO | Analyzing chunk frames = [6797312,6797824) | 20260508 08:49:21.446 | INFO | MemInfo: { | 20260508 08:49:21.446 | INFO | "gpu" : " 17 GB", | 20260508 08:49:21.446 | INFO | "huge" : "", | 20260508 08:49:21.446 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:21.446 | INFO | "malloc" : " 14 GB", | 20260508 08:49:21.446 | INFO | "pinned" : " 29 GB" | 20260508 08:49:21.446 | INFO | } | 20260508 08:49:21.446 | INFO | Perf report for the last 5.14263 seconds: | 20260508 08:49:21.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:21.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 08:49:21.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:49:21.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:21.446 | INFO | Uploaded 36GiB to GPU in 2.37572s (15.1533GiB/s) | 20260508 08:49:21.446 | INFO | Downloaded 34GiB to GPU in 2.50227s (13.5876GiB/s) | 20260508 08:49:26.581 | INFO | Analyzing chunk frames = [6797824,6798336) | 20260508 08:49:26.581 | INFO | MemInfo: { | 20260508 08:49:26.581 | INFO | "gpu" : " 17 GB", | 20260508 08:49:26.581 | INFO | "huge" : "", | 20260508 08:49:26.581 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:26.581 | INFO | "malloc" : " 14 GB", | 20260508 08:49:26.581 | INFO | "pinned" : " 29 GB" | 20260508 08:49:26.581 | INFO | } | 20260508 08:49:26.581 | INFO | Perf report for the last 5.13493 seconds: | 20260508 08:49:26.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:26.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 08:49:26.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260508 08:49:26.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:26.581 | INFO | Uploaded 36GiB to GPU in 2.39945s (15.0034GiB/s) | 20260508 08:49:26.581 | INFO | Downloaded 34GiB to GPU in 2.43046s (13.9891GiB/s) | 20260508 08:49:31.722 | INFO | Analyzing chunk frames = [6798336,6798848) | 20260508 08:49:31.722 | INFO | MemInfo: { | 20260508 08:49:31.722 | INFO | "gpu" : " 17 GB", | 20260508 08:49:31.722 | INFO | "huge" : "", | 20260508 08:49:31.722 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:31.722 | INFO | "malloc" : " 14 GB", | 20260508 08:49:31.722 | INFO | "pinned" : " 29 GB" | 20260508 08:49:31.722 | INFO | } | 20260508 08:49:31.722 | INFO | Perf report for the last 5.14156 seconds: | 20260508 08:49:31.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:31.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.722 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 08:49:31.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.723 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260508 08:49:31.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:31.723 | INFO | Uploaded 36GiB to GPU in 2.45591s (14.6585GiB/s) | 20260508 08:49:31.723 | INFO | Downloaded 34GiB to GPU in 2.44174s (13.9245GiB/s) | 20260508 08:49:36.864 | INFO | Analyzing chunk frames = [6798848,6799360) | 20260508 08:49:36.864 | INFO | MemInfo: { | 20260508 08:49:36.864 | INFO | "gpu" : " 17 GB", | 20260508 08:49:36.864 | INFO | "huge" : "", | 20260508 08:49:36.864 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:36.864 | INFO | "malloc" : " 14 GB", | 20260508 08:49:36.864 | INFO | "pinned" : " 29 GB" | 20260508 08:49:36.864 | INFO | } | 20260508 08:49:36.864 | INFO | Perf report for the last 5.14142 seconds: | 20260508 08:49:36.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:36.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:49:36.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:49:36.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:36.864 | INFO | Uploaded 36GiB to GPU in 2.43532s (14.7825GiB/s) | 20260508 08:49:36.864 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:49:36.864 | INFO | Total mem usage: 16.7 GiB | 20260508 08:49:36.864 | INFO | Downloaded 34GiB to GPU in 2.42986s (13.9926GiB/s) | 20260508 08:49:36.864 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:49:36.864 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:49:36.864 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:49:36.864 | INFO | High Water Breakdown: | 20260508 08:49:36.864 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:49:36.864 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:49:36.864 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:49:36.864 | INFO | ------------------------------------------ | 20260508 08:49:36.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:49:36.864 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:49:36.864 | INFO | Total mem usage: 26.5 GiB | 20260508 08:49:36.864 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:49:36.864 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:49:36.864 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:49:36.864 | INFO | High Water Breakdown: | 20260508 08:49:36.864 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:49:36.864 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:49:36.864 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:49:36.864 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:49:36.864 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:49:36.864 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:49:36.864 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:49:36.864 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:49:36.864 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:49:36.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:49:36.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:49:36.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:49:36.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:49:36.864 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:49:36.864 | INFO | ------------------------------------------ | 20260508 08:49:36.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:49:36.864 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:49:36.864 | INFO | Total mem usage: 27.8 GiB | 20260508 08:49:36.864 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:49:36.865 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:49:36.865 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:49:36.865 | INFO | High Water Breakdown: | 20260508 08:49:36.865 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:49:36.865 | INFO | ------------------------------------------ | 20260508 08:49:36.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:49:36.865 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:49:36.865 | INFO | Total mem usage: 15.7 GiB | 20260508 08:49:36.865 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:49:36.865 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:49:36.865 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:49:36.865 | INFO | High Water Breakdown: | 20260508 08:49:36.865 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:49:36.865 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:49:36.865 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:49:36.865 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:49:36.865 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:49:36.865 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:49:36.865 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:49:36.865 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:49:36.865 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:49:36.865 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:49:36.865 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:49:36.865 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:49:36.865 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:49:36.865 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:49:36.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:49:36.865 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:49:36.865 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:49:36.865 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:49:36.865 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:49:36.865 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:49:36.865 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:49:36.865 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:49:36.865 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:49:36.865 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:49:36.865 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:49:36.865 | INFO | ------------------------------------------ | 20260508 08:49:36.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:49:36.865 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:49:36.865 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:49:36.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:49:36.865 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.85e-01 | 20260508 08:49:36.865 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.02e-01, 1.12e-02 / 4.74e-03 | 20260508 08:49:36.865 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.46e+00, 5.94e-01 / 1.91e-01 | 20260508 08:49:36.865 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.18e-01 | 20260508 08:49:36.865 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.51e+01, 3.69e+00 / 1.09e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 1.30e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.10e-01 / 2.52e-01 | 20260508 08:49:36.865 | INFO | PerfReport: 3.07%, Metrics -- 1.00e+00 / 1.15e+00, 2.50e-01 / 1.42e-01 | 20260508 08:49:36.865 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.43e-01, 4.88e-02 / 3.48e-02 | 20260508 08:49:36.865 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:49:36.865 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:49:36.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:49:36.865 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 24.45%, Upload -- 2.58e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 24.47%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 30.93%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 2.22%, LockUpload -- 2.34e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 2.58%, LockDownload -- 2.72e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 15.34%, LockCompute -- 1.62e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:49:36.865 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 08:49:36.960 | INFO | Starting a round of Baz IO | 20260508 08:49:40.369 | INFO | Populating RTMetrics report | 20260508 08:49:40.369 | INFO | RTMetrics report populated | 20260508 08:49:40.369 | INFO | Opening tmp RTMetrics json file | 20260508 08:49:40.379 | INFO | Serializing RTMetrics to json | 20260508 08:49:40.381 | INFO | Serialization done, swapping tmp file now... | 20260508 08:49:40.381 | INFO | RTMetrics done | 20260508 08:49:40.440 | INFO | Finished a round of Baz IO: Wrote 1.494 GiB in 3.48 seconds | 20260508 08:49:40.440 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.00% / 59.62% | 20260508 08:49:40.440 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.278842% / 0% / 5.5631e-05% | 20260508 08:49:40.441 | INFO | Baz IO overall: | 20260508 08:49:40.441 | INFO | BytesWritten : 2977562747193 | 20260508 08:49:40.441 | INFO | NumEvents : 1433067555908 | 20260508 08:49:42.006 | INFO | Analyzing chunk frames = [6799360,6799872) | 20260508 08:49:42.006 | INFO | MemInfo: { | 20260508 08:49:42.006 | INFO | "gpu" : " 17 GB", | 20260508 08:49:42.006 | INFO | "huge" : "", | 20260508 08:49:42.006 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:42.006 | INFO | "malloc" : " 14 GB", | 20260508 08:49:42.006 | INFO | "pinned" : " 29 GB" | 20260508 08:49:42.006 | INFO | } | 20260508 08:49:42.006 | INFO | Perf report for the last 5.14181 seconds: | 20260508 08:49:42.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:42.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:42.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:42.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:42.006 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 08:49:42.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:42.006 | INFO | RT_METRICS : 39.2%, 1.0, 20.6ms, 0 | 20260508 08:49:42.006 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 08:49:42.006 | INFO | BAZWRITER : 67.1%, 2.6, 88.4ms, 0 | 20260508 08:49:42.006 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:49:42.006 | INFO | Uploaded 36GiB to GPU in 2.59957s (13.8484GiB/s) | 20260508 08:49:42.006 | INFO | Downloaded 38GiB to GPU in 2.69681s (14.0907GiB/s) | 20260508 08:49:47.140 | INFO | Analyzing chunk frames = [6799872,6800384) | 20260508 08:49:47.140 | INFO | MemInfo: { | 20260508 08:49:47.140 | INFO | "gpu" : " 17 GB", | 20260508 08:49:47.140 | INFO | "huge" : "", | 20260508 08:49:47.140 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:47.140 | INFO | "malloc" : " 14 GB", | 20260508 08:49:47.140 | INFO | "pinned" : " 29 GB" | 20260508 08:49:47.140 | INFO | } | 20260508 08:49:47.140 | INFO | Perf report for the last 5.13473 seconds: | 20260508 08:49:47.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:47.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.140 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 08:49:47.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.140 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:49:47.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:47.141 | INFO | Uploaded 36GiB to GPU in 2.51163s (14.3333GiB/s) | 20260508 08:49:47.141 | INFO | Downloaded 34GiB to GPU in 2.45631s (13.8419GiB/s) | 20260508 08:49:52.277 | INFO | Analyzing chunk frames = [6800384,6800896) | 20260508 08:49:52.277 | INFO | MemInfo: { | 20260508 08:49:52.277 | INFO | "gpu" : " 17 GB", | 20260508 08:49:52.277 | INFO | "huge" : "", | 20260508 08:49:52.277 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:52.277 | INFO | "malloc" : " 14 GB", | 20260508 08:49:52.277 | INFO | "pinned" : " 29 GB" | 20260508 08:49:52.277 | INFO | } | 20260508 08:49:52.277 | INFO | Perf report for the last 5.13656 seconds: | 20260508 08:49:52.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:52.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | ANALYSIS : 67.2%, 3.0, 104.4ms, 0 | 20260508 08:49:52.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:49:52.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:52.277 | INFO | Uploaded 36GiB to GPU in 2.58968s (13.9013GiB/s) | 20260508 08:49:52.278 | INFO | Downloaded 34GiB to GPU in 2.45376s (13.8563GiB/s) | 20260508 08:49:57.424 | INFO | Analyzing chunk frames = [6800896,6801408) | 20260508 08:49:57.424 | INFO | MemInfo: { | 20260508 08:49:57.424 | INFO | "gpu" : " 17 GB", | 20260508 08:49:57.424 | INFO | "huge" : "", | 20260508 08:49:57.424 | INFO | "huge_shared" : " 29 GB", | 20260508 08:49:57.424 | INFO | "malloc" : " 14 GB", | 20260508 08:49:57.424 | INFO | "pinned" : " 29 GB" | 20260508 08:49:57.424 | INFO | } | 20260508 08:49:57.424 | INFO | Perf report for the last 5.14724 seconds: | 20260508 08:49:57.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:49:57.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 08:49:57.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:49:57.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:49:57.424 | INFO | Uploaded 36GiB to GPU in 2.49827s (14.41GiB/s) | 20260508 08:49:57.424 | INFO | Downloaded 34GiB to GPU in 2.43628s (13.9557GiB/s) | 20260508 08:50:02.562 | INFO | Analyzing chunk frames = [6801408,6801920) | 20260508 08:50:02.562 | INFO | MemInfo: { | 20260508 08:50:02.562 | INFO | "gpu" : " 17 GB", | 20260508 08:50:02.562 | INFO | "huge" : "", | 20260508 08:50:02.562 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:02.562 | INFO | "malloc" : " 14 GB", | 20260508 08:50:02.562 | INFO | "pinned" : " 29 GB" | 20260508 08:50:02.562 | INFO | } | 20260508 08:50:02.562 | INFO | Perf report for the last 5.1379 seconds: | 20260508 08:50:02.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:02.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.562 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 08:50:02.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.562 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:50:02.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:02.563 | INFO | Uploaded 36GiB to GPU in 2.36565s (15.2178GiB/s) | 20260508 08:50:02.563 | INFO | Downloaded 34GiB to GPU in 2.5175s (13.5055GiB/s) | 20260508 08:50:07.698 | INFO | Analyzing chunk frames = [6801920,6802432) | 20260508 08:50:07.698 | INFO | MemInfo: { | 20260508 08:50:07.698 | INFO | "gpu" : " 17 GB", | 20260508 08:50:07.698 | INFO | "huge" : "", | 20260508 08:50:07.698 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:07.698 | INFO | "malloc" : " 14 GB", | 20260508 08:50:07.698 | INFO | "pinned" : " 29 GB" | 20260508 08:50:07.698 | INFO | } | 20260508 08:50:07.698 | INFO | Perf report for the last 5.13607 seconds: | 20260508 08:50:07.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:07.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 08:50:07.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:50:07.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:07.698 | INFO | Uploaded 36GiB to GPU in 2.34487s (15.3527GiB/s) | 20260508 08:50:07.699 | INFO | Downloaded 34GiB to GPU in 2.45311s (13.8599GiB/s) | 20260508 08:50:12.836 | INFO | Analyzing chunk frames = [6802432,6802944) | 20260508 08:50:12.836 | INFO | MemInfo: { | 20260508 08:50:12.836 | INFO | "gpu" : " 17 GB", | 20260508 08:50:12.836 | INFO | "huge" : "", | 20260508 08:50:12.836 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:12.836 | INFO | "malloc" : " 14 GB", | 20260508 08:50:12.836 | INFO | "pinned" : " 29 GB" | 20260508 08:50:12.836 | INFO | } | 20260508 08:50:12.836 | INFO | Perf report for the last 5.13769 seconds: | 20260508 08:50:12.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:12.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:50:12.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:50:12.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:12.836 | INFO | Uploaded 36GiB to GPU in 2.35274s (15.3013GiB/s) | 20260508 08:50:12.836 | INFO | Downloaded 34GiB to GPU in 2.44798s (13.889GiB/s) | 20260508 08:50:17.975 | INFO | Analyzing chunk frames = [6802944,6803456) | 20260508 08:50:17.975 | INFO | MemInfo: { | 20260508 08:50:17.975 | INFO | "gpu" : " 17 GB", | 20260508 08:50:17.975 | INFO | "huge" : "", | 20260508 08:50:17.975 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:17.975 | INFO | "malloc" : " 14 GB", | 20260508 08:50:17.975 | INFO | "pinned" : " 29 GB" | 20260508 08:50:17.975 | INFO | } | 20260508 08:50:17.975 | INFO | Perf report for the last 5.13943 seconds: | 20260508 08:50:17.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:17.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.975 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 08:50:17.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.975 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:50:17.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:17.976 | INFO | Uploaded 36GiB to GPU in 2.3959s (15.0257GiB/s) | 20260508 08:50:17.976 | INFO | Downloaded 34GiB to GPU in 2.44964s (13.8796GiB/s) | 20260508 08:50:18.072 | INFO | Starting a round of Baz IO | 20260508 08:50:21.494 | INFO | Populating RTMetrics report | 20260508 08:50:21.494 | INFO | RTMetrics report populated | 20260508 08:50:21.494 | INFO | Opening tmp RTMetrics json file | 20260508 08:50:21.503 | INFO | Serializing RTMetrics to json | 20260508 08:50:21.505 | INFO | Serialization done, swapping tmp file now... | 20260508 08:50:21.505 | INFO | RTMetrics done | 20260508 08:50:21.563 | INFO | Finished a round of Baz IO: Wrote 1.495 GiB in 3.491 seconds | 20260508 08:50:21.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.62% | 20260508 08:50:21.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.279009% / 0% / 7.54992e-05% | 20260508 08:50:21.563 | INFO | Baz IO overall: | 20260508 08:50:21.563 | INFO | BytesWritten : 2979168439117 | 20260508 08:50:21.563 | INFO | NumEvents : 1433827787069 | 20260508 08:50:23.115 | INFO | Analyzing chunk frames = [6803456,6803968) | 20260508 08:50:23.115 | INFO | MemInfo: { | 20260508 08:50:23.115 | INFO | "gpu" : " 17 GB", | 20260508 08:50:23.115 | INFO | "huge" : "", | 20260508 08:50:23.115 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:23.115 | INFO | "malloc" : " 14 GB", | 20260508 08:50:23.115 | INFO | "pinned" : " 29 GB" | 20260508 08:50:23.115 | INFO | } | 20260508 08:50:23.115 | INFO | Perf report for the last 5.13962 seconds: | 20260508 08:50:23.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:23.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:23.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:23.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:23.115 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:50:23.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:23.115 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 08:50:23.115 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 08:50:23.115 | INFO | BAZWRITER : 67.1%, 2.6, 88.9ms, 0 | 20260508 08:50:23.115 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:50:23.115 | INFO | Uploaded 36GiB to GPU in 2.57301s (13.9914GiB/s) | 20260508 08:50:23.116 | INFO | Downloaded 38GiB to GPU in 2.71281s (14.0076GiB/s) | 20260508 08:50:28.256 | INFO | Analyzing chunk frames = [6803968,6804480) | 20260508 08:50:28.256 | INFO | MemInfo: { | 20260508 08:50:28.256 | INFO | "gpu" : " 17 GB", | 20260508 08:50:28.256 | INFO | "huge" : "", | 20260508 08:50:28.256 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:28.256 | INFO | "malloc" : " 14 GB", | 20260508 08:50:28.256 | INFO | "pinned" : " 29 GB" | 20260508 08:50:28.256 | INFO | } | 20260508 08:50:28.256 | INFO | Perf report for the last 5.14117 seconds: | 20260508 08:50:28.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:28.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:50:28.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:50:28.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:28.256 | INFO | Uploaded 36GiB to GPU in 2.48107s (14.5099GiB/s) | 20260508 08:50:28.256 | INFO | Downloaded 34GiB to GPU in 2.45787s (13.8331GiB/s) | 20260508 08:50:33.397 | INFO | Analyzing chunk frames = [6804480,6804992) | 20260508 08:50:33.397 | INFO | MemInfo: { | 20260508 08:50:33.397 | INFO | "gpu" : " 17 GB", | 20260508 08:50:33.397 | INFO | "huge" : "", | 20260508 08:50:33.397 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:33.397 | INFO | "malloc" : " 14 GB", | 20260508 08:50:33.397 | INFO | "pinned" : " 29 GB" | 20260508 08:50:33.397 | INFO | } | 20260508 08:50:33.397 | INFO | Perf report for the last 5.1406 seconds: | 20260508 08:50:33.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:33.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 08:50:33.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:50:33.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:33.397 | INFO | Uploaded 36GiB to GPU in 2.47259s (14.5596GiB/s) | 20260508 08:50:33.397 | INFO | Downloaded 34GiB to GPU in 2.46399s (13.7988GiB/s) | 20260508 08:50:38.534 | INFO | Analyzing chunk frames = [6804992,6805504) | 20260508 08:50:38.534 | INFO | MemInfo: { | 20260508 08:50:38.534 | INFO | "gpu" : " 17 GB", | 20260508 08:50:38.534 | INFO | "huge" : "", | 20260508 08:50:38.534 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:38.534 | INFO | "malloc" : " 14 GB", | 20260508 08:50:38.534 | INFO | "pinned" : " 29 GB" | 20260508 08:50:38.534 | INFO | } | 20260508 08:50:38.534 | INFO | Perf report for the last 5.13722 seconds: | 20260508 08:50:38.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:38.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.534 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 08:50:38.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.534 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260508 08:50:38.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:38.535 | INFO | Uploaded 36GiB to GPU in 2.48518s (14.4859GiB/s) | 20260508 08:50:38.535 | INFO | Downloaded 34GiB to GPU in 2.46127s (13.814GiB/s) | 20260508 08:50:43.671 | INFO | Analyzing chunk frames = [6805504,6806016) | 20260508 08:50:43.671 | INFO | MemInfo: { | 20260508 08:50:43.671 | INFO | "gpu" : " 17 GB", | 20260508 08:50:43.671 | INFO | "huge" : "", | 20260508 08:50:43.671 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:43.671 | INFO | "malloc" : " 14 GB", | 20260508 08:50:43.671 | INFO | "pinned" : " 29 GB" | 20260508 08:50:43.671 | INFO | } | 20260508 08:50:43.671 | INFO | Perf report for the last 5.13711 seconds: | 20260508 08:50:43.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:43.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.671 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:50:43.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.671 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:50:43.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:43.672 | INFO | Uploaded 36GiB to GPU in 2.38533s (15.0923GiB/s) | 20260508 08:50:43.672 | INFO | Downloaded 34GiB to GPU in 2.57195s (13.2195GiB/s) | 20260508 08:50:48.816 | INFO | Analyzing chunk frames = [6806016,6806528) | 20260508 08:50:48.816 | INFO | MemInfo: { | 20260508 08:50:48.816 | INFO | "gpu" : " 17 GB", | 20260508 08:50:48.816 | INFO | "huge" : "", | 20260508 08:50:48.816 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:48.816 | INFO | "malloc" : " 14 GB", | 20260508 08:50:48.816 | INFO | "pinned" : " 29 GB" | 20260508 08:50:48.816 | INFO | } | 20260508 08:50:48.816 | INFO | Perf report for the last 5.14454 seconds: | 20260508 08:50:48.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:48.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 08:50:48.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:50:48.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:48.816 | INFO | Uploaded 36GiB to GPU in 2.34583s (15.3464GiB/s) | 20260508 08:50:48.816 | INFO | Downloaded 34GiB to GPU in 2.45427s (13.8534GiB/s) | 20260508 08:50:53.956 | INFO | Analyzing chunk frames = [6806528,6807040) | 20260508 08:50:53.956 | INFO | MemInfo: { | 20260508 08:50:53.956 | INFO | "gpu" : " 17 GB", | 20260508 08:50:53.956 | INFO | "huge" : "", | 20260508 08:50:53.956 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:53.956 | INFO | "malloc" : " 14 GB", | 20260508 08:50:53.956 | INFO | "pinned" : " 29 GB" | 20260508 08:50:53.956 | INFO | } | 20260508 08:50:53.956 | INFO | Perf report for the last 5.14041 seconds: | 20260508 08:50:53.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:53.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 08:50:53.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:50:53.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:53.956 | INFO | Uploaded 36GiB to GPU in 2.37947s (15.1294GiB/s) | 20260508 08:50:53.956 | INFO | Downloaded 34GiB to GPU in 2.45974s (13.8226GiB/s) | 20260508 08:50:59.092 | INFO | Analyzing chunk frames = [6807040,6807552) | 20260508 08:50:59.092 | INFO | MemInfo: { | 20260508 08:50:59.092 | INFO | "gpu" : " 17 GB", | 20260508 08:50:59.092 | INFO | "huge" : "", | 20260508 08:50:59.092 | INFO | "huge_shared" : " 29 GB", | 20260508 08:50:59.092 | INFO | "malloc" : " 14 GB", | 20260508 08:50:59.092 | INFO | "pinned" : " 29 GB" | 20260508 08:50:59.092 | INFO | } | 20260508 08:50:59.092 | INFO | Perf report for the last 5.13603 seconds: | 20260508 08:50:59.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:50:59.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 08:50:59.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 08:50:59.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:50:59.092 | INFO | Uploaded 36GiB to GPU in 2.39716s (15.0178GiB/s) | 20260508 08:50:59.092 | INFO | Downloaded 34GiB to GPU in 2.46419s (13.7976GiB/s) | 20260508 08:50:59.092 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:50:59.092 | INFO | Total mem usage: 16.7 GiB | 20260508 08:50:59.092 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:50:59.092 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:50:59.092 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:50:59.093 | INFO | High Water Breakdown: | 20260508 08:50:59.093 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:50:59.093 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:50:59.093 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:50:59.093 | INFO | ------------------------------------------ | 20260508 08:50:59.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:50:59.093 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:50:59.093 | INFO | Total mem usage: 26.5 GiB | 20260508 08:50:59.093 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:50:59.093 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:50:59.093 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:50:59.093 | INFO | High Water Breakdown: | 20260508 08:50:59.093 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:50:59.093 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:50:59.093 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:50:59.093 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:50:59.093 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:50:59.093 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:50:59.093 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:50:59.093 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:50:59.093 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:50:59.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:50:59.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:50:59.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:50:59.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:50:59.093 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:50:59.093 | INFO | ------------------------------------------ | 20260508 08:50:59.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:50:59.093 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:50:59.093 | INFO | Total mem usage: 27.8 GiB | 20260508 08:50:59.093 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:50:59.093 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:50:59.093 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:50:59.093 | INFO | High Water Breakdown: | 20260508 08:50:59.093 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:50:59.093 | INFO | ------------------------------------------ | 20260508 08:50:59.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:50:59.093 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:50:59.093 | INFO | Total mem usage: 15.7 GiB | 20260508 08:50:59.093 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:50:59.093 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:50:59.093 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:50:59.093 | INFO | High Water Breakdown: | 20260508 08:50:59.093 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:50:59.093 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:50:59.093 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:50:59.093 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:50:59.093 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:50:59.093 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:50:59.093 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:50:59.093 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:50:59.093 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:50:59.093 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:50:59.093 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:50:59.093 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:50:59.093 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:50:59.093 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:50:59.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:50:59.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:50:59.093 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:50:59.093 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:50:59.093 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:50:59.093 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:50:59.093 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:50:59.093 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:50:59.093 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:50:59.093 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:50:59.093 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:50:59.093 | INFO | ------------------------------------------ | 20260508 08:50:59.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:50:59.093 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:50:59.093 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:50:59.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:50:59.093 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.39e+01 / 7.53e-01 | 20260508 08:50:59.093 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.13e-02 / 4.70e-03 | 20260508 08:50:59.093 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.46e+00, 5.92e-01 / 2.03e-01 | 20260508 08:50:59.093 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.10e-01 / 1.16e-01 | 20260508 08:50:59.093 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.49e+01, 3.62e+00 / 1.02e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.88e+00 / 1.33e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.06e-01 / 2.31e-01 | 20260508 08:50:59.093 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.15e+00, 2.49e-01 / 1.31e-01 | 20260508 08:50:59.093 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.44e-01, 4.95e-02 / 3.44e-02 | 20260508 08:50:59.093 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:50:59.093 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:50:59.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:50:59.093 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 24.18%, Upload -- 2.55e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 24.67%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 31.08%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 2.08%, LockUpload -- 2.20e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 2.61%, LockDownload -- 2.76e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 08:50:59.093 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260508 08:50:59.189 | INFO | Starting a round of Baz IO | 20260508 08:51:02.609 | INFO | Populating RTMetrics report | 20260508 08:51:02.609 | INFO | RTMetrics report populated | 20260508 08:51:02.609 | INFO | Opening tmp RTMetrics json file | 20260508 08:51:02.620 | INFO | Serializing RTMetrics to json | 20260508 08:51:02.621 | INFO | Serialization done, swapping tmp file now... | 20260508 08:51:02.622 | INFO | RTMetrics done | 20260508 08:51:02.675 | INFO | Finished a round of Baz IO: Wrote 1.496 GiB in 3.486 seconds | 20260508 08:51:02.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:51:02.675 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.279121% / 0% / 7.94729e-05% | 20260508 08:51:02.675 | INFO | Baz IO overall: | 20260508 08:51:02.675 | INFO | BytesWritten : 2980775083469 | 20260508 08:51:02.675 | INFO | NumEvents : 1434588578551 | 20260508 08:51:04.231 | INFO | Analyzing chunk frames = [6807552,6808064) | 20260508 08:51:04.231 | INFO | MemInfo: { | 20260508 08:51:04.231 | INFO | "gpu" : " 17 GB", | 20260508 08:51:04.231 | INFO | "huge" : "", | 20260508 08:51:04.231 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:04.231 | INFO | "malloc" : " 14 GB", | 20260508 08:51:04.231 | INFO | "pinned" : " 29 GB" | 20260508 08:51:04.231 | INFO | } | 20260508 08:51:04.231 | INFO | Perf report for the last 5.13855 seconds: | 20260508 08:51:04.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:04.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:04.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:04.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:04.231 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 08:51:04.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:04.231 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 08:51:04.231 | INFO | PRE_HQ : 65.2%, 1.6, 54.7ms, 0 | 20260508 08:51:04.231 | INFO | BAZWRITER : 67.1%, 2.6, 89.4ms, 0 | 20260508 08:51:04.231 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:51:04.231 | INFO | Uploaded 36GiB to GPU in 2.55002s (14.1175GiB/s) | 20260508 08:51:04.231 | INFO | Downloaded 38GiB to GPU in 2.7101s (14.0216GiB/s) | 20260508 08:51:09.374 | INFO | Analyzing chunk frames = [6808064,6808576) | 20260508 08:51:09.374 | INFO | MemInfo: { | 20260508 08:51:09.374 | INFO | "gpu" : " 17 GB", | 20260508 08:51:09.374 | INFO | "huge" : "", | 20260508 08:51:09.374 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:09.374 | INFO | "malloc" : " 14 GB", | 20260508 08:51:09.374 | INFO | "pinned" : " 29 GB" | 20260508 08:51:09.374 | INFO | } | 20260508 08:51:09.374 | INFO | Perf report for the last 5.14307 seconds: | 20260508 08:51:09.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:09.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 08:51:09.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:51:09.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:09.374 | INFO | Uploaded 36GiB to GPU in 2.47288s (14.5579GiB/s) | 20260508 08:51:09.374 | INFO | Downloaded 34GiB to GPU in 2.46611s (13.7869GiB/s) | 20260508 08:51:14.508 | INFO | Analyzing chunk frames = [6808576,6809088) | 20260508 08:51:14.508 | INFO | MemInfo: { | 20260508 08:51:14.508 | INFO | "gpu" : " 17 GB", | 20260508 08:51:14.508 | INFO | "huge" : "", | 20260508 08:51:14.508 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:14.508 | INFO | "malloc" : " 14 GB", | 20260508 08:51:14.508 | INFO | "pinned" : " 29 GB" | 20260508 08:51:14.508 | INFO | } | 20260508 08:51:14.508 | INFO | Perf report for the last 5.13451 seconds: | 20260508 08:51:14.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:14.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:51:14.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 08:51:14.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:14.508 | INFO | Uploaded 36GiB to GPU in 2.44226s (14.7405GiB/s) | 20260508 08:51:14.508 | INFO | Downloaded 34GiB to GPU in 2.4484s (13.8866GiB/s) | 20260508 08:51:19.650 | INFO | Analyzing chunk frames = [6809088,6809600) | 20260508 08:51:19.650 | INFO | MemInfo: { | 20260508 08:51:19.650 | INFO | "gpu" : " 17 GB", | 20260508 08:51:19.650 | INFO | "huge" : "", | 20260508 08:51:19.650 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:19.650 | INFO | "malloc" : " 14 GB", | 20260508 08:51:19.650 | INFO | "pinned" : " 29 GB" | 20260508 08:51:19.650 | INFO | } | 20260508 08:51:19.650 | INFO | Perf report for the last 5.1419 seconds: | 20260508 08:51:19.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:19.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 08:51:19.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:51:19.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:19.650 | INFO | Uploaded 36GiB to GPU in 2.48049s (14.5133GiB/s) | 20260508 08:51:19.651 | INFO | Downloaded 34GiB to GPU in 2.45287s (13.8613GiB/s) | 20260508 08:51:24.785 | INFO | Analyzing chunk frames = [6809600,6810112) | 20260508 08:51:24.785 | INFO | MemInfo: { | 20260508 08:51:24.785 | INFO | "gpu" : " 17 GB", | 20260508 08:51:24.785 | INFO | "huge" : "", | 20260508 08:51:24.785 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:24.786 | INFO | "malloc" : " 14 GB", | 20260508 08:51:24.786 | INFO | "pinned" : " 29 GB" | 20260508 08:51:24.786 | INFO | } | 20260508 08:51:24.786 | INFO | Perf report for the last 5.13539 seconds: | 20260508 08:51:24.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:24.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 08:51:24.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 08:51:24.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:24.786 | INFO | Uploaded 36GiB to GPU in 2.40541s (14.9662GiB/s) | 20260508 08:51:24.786 | INFO | Downloaded 34GiB to GPU in 2.53467s (13.414GiB/s) | 20260508 08:51:29.928 | INFO | Analyzing chunk frames = [6810112,6810624) | 20260508 08:51:29.928 | INFO | MemInfo: { | 20260508 08:51:29.928 | INFO | "gpu" : " 17 GB", | 20260508 08:51:29.928 | INFO | "huge" : "", | 20260508 08:51:29.928 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:29.928 | INFO | "malloc" : " 14 GB", | 20260508 08:51:29.928 | INFO | "pinned" : " 29 GB" | 20260508 08:51:29.928 | INFO | } | 20260508 08:51:29.928 | INFO | Perf report for the last 5.14202 seconds: | 20260508 08:51:29.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:29.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 08:51:29.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:51:29.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:29.928 | INFO | Uploaded 36GiB to GPU in 2.32826s (15.4622GiB/s) | 20260508 08:51:29.928 | INFO | Downloaded 34GiB to GPU in 2.44942s (13.8809GiB/s) | 20260508 08:51:35.069 | INFO | Analyzing chunk frames = [6810624,6811136) | 20260508 08:51:35.069 | INFO | MemInfo: { | 20260508 08:51:35.069 | INFO | "gpu" : " 17 GB", | 20260508 08:51:35.069 | INFO | "huge" : "", | 20260508 08:51:35.069 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:35.069 | INFO | "malloc" : " 14 GB", | 20260508 08:51:35.069 | INFO | "pinned" : " 29 GB" | 20260508 08:51:35.069 | INFO | } | 20260508 08:51:35.069 | INFO | Perf report for the last 5.14113 seconds: | 20260508 08:51:35.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:35.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:51:35.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:51:35.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:35.069 | INFO | Uploaded 36GiB to GPU in 2.3799s (15.1267GiB/s) | 20260508 08:51:35.069 | INFO | Downloaded 34GiB to GPU in 2.44575s (13.9017GiB/s) | 20260508 08:51:40.208 | INFO | Analyzing chunk frames = [6811136,6811648) | 20260508 08:51:40.208 | INFO | MemInfo: { | 20260508 08:51:40.208 | INFO | "gpu" : " 17 GB", | 20260508 08:51:40.208 | INFO | "huge" : "", | 20260508 08:51:40.208 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:40.208 | INFO | "malloc" : " 14 GB", | 20260508 08:51:40.208 | INFO | "pinned" : " 29 GB" | 20260508 08:51:40.208 | INFO | } | 20260508 08:51:40.208 | INFO | Perf report for the last 5.13951 seconds: | 20260508 08:51:40.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:40.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.208 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 08:51:40.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.208 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:51:40.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:40.209 | INFO | Uploaded 36GiB to GPU in 2.37011s (15.1892GiB/s) | 20260508 08:51:40.209 | INFO | Downloaded 34GiB to GPU in 2.45658s (13.8404GiB/s) | 20260508 08:51:40.307 | INFO | Starting a round of Baz IO | 20260508 08:51:43.731 | INFO | Populating RTMetrics report | 20260508 08:51:43.731 | INFO | RTMetrics report populated | 20260508 08:51:43.731 | INFO | Opening tmp RTMetrics json file | 20260508 08:51:43.741 | INFO | Serializing RTMetrics to json | 20260508 08:51:43.742 | INFO | Serialization done, swapping tmp file now... | 20260508 08:51:43.743 | INFO | RTMetrics done | 20260508 08:51:43.797 | INFO | Finished a round of Baz IO: Wrote 1.495 GiB in 3.49 seconds | 20260508 08:51:43.797 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:51:43.797 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277364% / 0% / 6.75519e-05% | 20260508 08:51:43.797 | INFO | Baz IO overall: | 20260508 08:51:43.797 | INFO | BytesWritten : 2982380133742 | 20260508 08:51:43.797 | INFO | NumEvents : 1435348179458 | 20260508 08:51:45.345 | INFO | Analyzing chunk frames = [6811648,6812160) | 20260508 08:51:45.345 | INFO | MemInfo: { | 20260508 08:51:45.345 | INFO | "gpu" : " 17 GB", | 20260508 08:51:45.345 | INFO | "huge" : "", | 20260508 08:51:45.345 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:45.345 | INFO | "malloc" : " 14 GB", | 20260508 08:51:45.345 | INFO | "pinned" : " 29 GB" | 20260508 08:51:45.345 | INFO | } | 20260508 08:51:45.345 | INFO | Perf report for the last 5.13704 seconds: | 20260508 08:51:45.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:45.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:45.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:45.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:45.345 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 08:51:45.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:45.346 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:51:45.346 | INFO | PRE_HQ : 65.4%, 1.6, 54.3ms, 0 | 20260508 08:51:45.346 | INFO | BAZWRITER : 67.3%, 2.6, 88.7ms, 0 | 20260508 08:51:45.346 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:51:45.346 | INFO | Uploaded 36GiB to GPU in 2.58904s (13.9047GiB/s) | 20260508 08:51:45.346 | INFO | Downloaded 38GiB to GPU in 2.70765s (14.0343GiB/s) | 20260508 08:51:50.481 | INFO | Analyzing chunk frames = [6812160,6812672) | 20260508 08:51:50.481 | INFO | MemInfo: { | 20260508 08:51:50.481 | INFO | "gpu" : " 17 GB", | 20260508 08:51:50.481 | INFO | "huge" : "", | 20260508 08:51:50.481 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:50.481 | INFO | "malloc" : " 14 GB", | 20260508 08:51:50.481 | INFO | "pinned" : " 29 GB" | 20260508 08:51:50.481 | INFO | } | 20260508 08:51:50.481 | INFO | Perf report for the last 5.13567 seconds: | 20260508 08:51:50.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:50.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.481 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:51:50.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.481 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 08:51:50.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:50.482 | INFO | Uploaded 36GiB to GPU in 2.49274s (14.4419GiB/s) | 20260508 08:51:50.482 | INFO | Downloaded 34GiB to GPU in 2.47407s (13.7425GiB/s) | 20260508 08:51:55.624 | INFO | Analyzing chunk frames = [6812672,6813184) | 20260508 08:51:55.624 | INFO | MemInfo: { | 20260508 08:51:55.624 | INFO | "gpu" : " 17 GB", | 20260508 08:51:55.624 | INFO | "huge" : "", | 20260508 08:51:55.624 | INFO | "huge_shared" : " 29 GB", | 20260508 08:51:55.624 | INFO | "malloc" : " 14 GB", | 20260508 08:51:55.624 | INFO | "pinned" : " 29 GB" | 20260508 08:51:55.624 | INFO | } | 20260508 08:51:55.624 | INFO | Perf report for the last 5.14264 seconds: | 20260508 08:51:55.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:51:55.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 08:51:55.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 08:51:55.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:51:55.624 | INFO | Uploaded 36GiB to GPU in 2.47279s (14.5585GiB/s) | 20260508 08:51:55.624 | INFO | Downloaded 34GiB to GPU in 2.46346s (13.8017GiB/s) | 20260508 08:52:00.767 | INFO | Analyzing chunk frames = [6813184,6813696) | 20260508 08:52:00.767 | INFO | MemInfo: { | 20260508 08:52:00.767 | INFO | "gpu" : " 17 GB", | 20260508 08:52:00.767 | INFO | "huge" : "", | 20260508 08:52:00.767 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:00.767 | INFO | "malloc" : " 14 GB", | 20260508 08:52:00.767 | INFO | "pinned" : " 29 GB" | 20260508 08:52:00.767 | INFO | } | 20260508 08:52:00.767 | INFO | Perf report for the last 5.14374 seconds: | 20260508 08:52:00.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:00.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.767 | INFO | ANALYSIS : 66.7%, 3.0, 103.3ms, 0 | 20260508 08:52:00.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.768 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 08:52:00.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:00.768 | INFO | Uploaded 36GiB to GPU in 2.52865s (14.2368GiB/s) | 20260508 08:52:00.768 | INFO | Downloaded 34GiB to GPU in 2.44319s (13.9162GiB/s) | 20260508 08:52:05.906 | INFO | Analyzing chunk frames = [6813696,6814208) | 20260508 08:52:05.906 | INFO | MemInfo: { | 20260508 08:52:05.906 | INFO | "gpu" : " 17 GB", | 20260508 08:52:05.906 | INFO | "huge" : "", | 20260508 08:52:05.906 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:05.906 | INFO | "malloc" : " 14 GB", | 20260508 08:52:05.906 | INFO | "pinned" : " 29 GB" | 20260508 08:52:05.906 | INFO | } | 20260508 08:52:05.906 | INFO | Perf report for the last 5.1385 seconds: | 20260508 08:52:05.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:05.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 08:52:05.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 08:52:05.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:05.906 | INFO | Uploaded 36GiB to GPU in 2.39605s (15.0248GiB/s) | 20260508 08:52:05.906 | INFO | Downloaded 34GiB to GPU in 2.53709s (13.4012GiB/s) | 20260508 08:52:11.043 | INFO | Analyzing chunk frames = [6814208,6814720) | 20260508 08:52:11.043 | INFO | MemInfo: { | 20260508 08:52:11.043 | INFO | "gpu" : " 17 GB", | 20260508 08:52:11.043 | INFO | "huge" : "", | 20260508 08:52:11.043 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:11.043 | INFO | "malloc" : " 14 GB", | 20260508 08:52:11.043 | INFO | "pinned" : " 29 GB" | 20260508 08:52:11.043 | INFO | } | 20260508 08:52:11.043 | INFO | Perf report for the last 5.13709 seconds: | 20260508 08:52:11.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:11.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 08:52:11.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:52:11.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:11.043 | INFO | Uploaded 36GiB to GPU in 2.33523s (15.4161GiB/s) | 20260508 08:52:11.043 | INFO | Downloaded 34GiB to GPU in 2.43331s (13.9727GiB/s) | 20260508 08:52:16.185 | INFO | Analyzing chunk frames = [6814720,6815232) | 20260508 08:52:16.185 | INFO | MemInfo: { | 20260508 08:52:16.185 | INFO | "gpu" : " 17 GB", | 20260508 08:52:16.185 | INFO | "huge" : "", | 20260508 08:52:16.185 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:16.185 | INFO | "malloc" : " 14 GB", | 20260508 08:52:16.185 | INFO | "pinned" : " 29 GB" | 20260508 08:52:16.185 | INFO | } | 20260508 08:52:16.185 | INFO | Perf report for the last 5.14155 seconds: | 20260508 08:52:16.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:16.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:52:16.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:52:16.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:16.185 | INFO | Uploaded 36GiB to GPU in 2.40176s (14.989GiB/s) | 20260508 08:52:16.185 | INFO | Downloaded 34GiB to GPU in 2.45049s (13.8748GiB/s) | 20260508 08:52:21.323 | INFO | Analyzing chunk frames = [6815232,6815744) | 20260508 08:52:21.323 | INFO | MemInfo: { | 20260508 08:52:21.323 | INFO | "gpu" : " 17 GB", | 20260508 08:52:21.323 | INFO | "huge" : "", | 20260508 08:52:21.323 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:21.323 | INFO | "malloc" : " 14 GB", | 20260508 08:52:21.323 | INFO | "pinned" : " 29 GB" | 20260508 08:52:21.323 | INFO | } | 20260508 08:52:21.323 | INFO | Perf report for the last 5.13842 seconds: | 20260508 08:52:21.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:21.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 08:52:21.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:52:21.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:21.323 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:52:21.323 | INFO | Total mem usage: 16.7 GiB | 20260508 08:52:21.323 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:52:21.323 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:52:21.323 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:52:21.323 | INFO | High Water Breakdown: | 20260508 08:52:21.323 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:52:21.323 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:52:21.323 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:52:21.323 | INFO | ------------------------------------------ | 20260508 08:52:21.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:52:21.323 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:52:21.323 | INFO | Total mem usage: 26.5 GiB | 20260508 08:52:21.323 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:52:21.323 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:52:21.323 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:52:21.323 | INFO | High Water Breakdown: | 20260508 08:52:21.323 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:52:21.323 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:52:21.323 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:52:21.323 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:52:21.323 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:52:21.323 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:52:21.323 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:52:21.323 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:52:21.323 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:52:21.323 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:52:21.323 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:52:21.323 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:52:21.323 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:52:21.323 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:52:21.323 | INFO | ------------------------------------------ | 20260508 08:52:21.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:52:21.323 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:52:21.323 | INFO | Total mem usage: 27.8 GiB | 20260508 08:52:21.323 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:52:21.323 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:52:21.323 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:52:21.323 | INFO | High Water Breakdown: | 20260508 08:52:21.323 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:52:21.323 | INFO | ------------------------------------------ | 20260508 08:52:21.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:52:21.323 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:52:21.323 | INFO | Total mem usage: 15.7 GiB | 20260508 08:52:21.323 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:52:21.323 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:52:21.323 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:52:21.323 | INFO | High Water Breakdown: | 20260508 08:52:21.323 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:52:21.323 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:52:21.323 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:52:21.324 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:52:21.324 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:52:21.324 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:52:21.324 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:52:21.324 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:52:21.324 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:52:21.324 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:52:21.324 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:52:21.324 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:52:21.324 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:52:21.324 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:52:21.324 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:52:21.324 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:52:21.324 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:52:21.324 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:52:21.324 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:52:21.324 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:52:21.324 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:52:21.324 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:52:21.324 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:52:21.324 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:52:21.324 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:52:21.324 | INFO | ------------------------------------------ | 20260508 08:52:21.324 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:52:21.324 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:52:21.324 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:52:21.324 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:52:21.324 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.38e+01 / 7.12e-01 | 20260508 08:52:21.324 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.02e-01, 1.16e-02 / 7.88e-03 | 20260508 08:52:21.324 | INFO | PerfReport: 12.14%, Baseline -- 3.99e+00 / 4.48e+00, 5.97e-01 / 2.64e-01 | 20260508 08:52:21.324 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.40e+00, 2.10e-01 / 1.33e-01 | 20260508 08:52:21.324 | INFO | PerfReport: 15.93%, DME -- 4.19e+01 / 4.50e+01, 3.78e+00 / 1.20e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.93e+00 / 1.38e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.32e+00 / 2.48e+00, 4.06e-01 / 2.70e-01 | 20260508 08:52:21.324 | INFO | PerfReport: 3.04%, Metrics -- 1.00e+00 / 1.18e+00, 2.50e-01 / 1.55e-01 | 20260508 08:52:21.324 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.42e-01, 4.95e-02 / 2.75e-02 | 20260508 08:52:21.324 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:52:21.324 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:52:21.324 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:52:21.324 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 24.11%, Upload -- 2.54e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 24.65%, Download -- 2.60e+01 / 0.00e+00, 8.31e+00 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 31.16%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 1.99%, LockUpload -- 2.10e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.30e+00 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 15.41%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 08:52:21.324 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260508 08:52:21.324 | INFO | Uploaded 36GiB to GPU in 2.38995s (15.0631GiB/s) | 20260508 08:52:21.324 | INFO | Downloaded 34GiB to GPU in 2.43881s (13.9412GiB/s) | 20260508 08:52:21.420 | INFO | Starting a round of Baz IO | 20260508 08:52:24.840 | INFO | Populating RTMetrics report | 20260508 08:52:24.840 | INFO | RTMetrics report populated | 20260508 08:52:24.840 | INFO | Opening tmp RTMetrics json file | 20260508 08:52:24.851 | INFO | Serializing RTMetrics to json | 20260508 08:52:24.852 | INFO | Serialization done, swapping tmp file now... | 20260508 08:52:24.853 | INFO | RTMetrics done | 20260508 08:52:24.909 | INFO | Finished a round of Baz IO: Wrote 1.494 GiB in 3.488 seconds | 20260508 08:52:24.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:52:24.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.278326% / 0% / 5.5631e-05% | 20260508 08:52:24.909 | INFO | Baz IO overall: | 20260508 08:52:24.909 | INFO | BytesWritten : 2983984325944 | 20260508 08:52:24.909 | INFO | NumEvents : 1436107403904 | 20260508 08:52:26.465 | INFO | Analyzing chunk frames = [6815744,6816256) | 20260508 08:52:26.465 | INFO | MemInfo: { | 20260508 08:52:26.465 | INFO | "gpu" : " 17 GB", | 20260508 08:52:26.465 | INFO | "huge" : "", | 20260508 08:52:26.465 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:26.465 | INFO | "malloc" : " 14 GB", | 20260508 08:52:26.465 | INFO | "pinned" : " 29 GB" | 20260508 08:52:26.465 | INFO | } | 20260508 08:52:26.465 | INFO | Perf report for the last 5.14203 seconds: | 20260508 08:52:26.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:26.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:26.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:26.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:26.465 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 08:52:26.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:26.465 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260508 08:52:26.465 | INFO | PRE_HQ : 65.7%, 1.7, 57.6ms, 0 | 20260508 08:52:26.465 | INFO | BAZWRITER : 67.1%, 2.6, 90.7ms, 0 | 20260508 08:52:26.465 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:52:26.466 | INFO | Uploaded 36GiB to GPU in 2.58892s (13.9054GiB/s) | 20260508 08:52:26.466 | INFO | Downloaded 38GiB to GPU in 2.70566s (14.0446GiB/s) | 20260508 08:52:31.599 | INFO | Analyzing chunk frames = [6816256,6816768) | 20260508 08:52:31.599 | INFO | MemInfo: { | 20260508 08:52:31.599 | INFO | "gpu" : " 17 GB", | 20260508 08:52:31.599 | INFO | "huge" : "", | 20260508 08:52:31.599 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:31.599 | INFO | "malloc" : " 14 GB", | 20260508 08:52:31.599 | INFO | "pinned" : " 29 GB" | 20260508 08:52:31.600 | INFO | } | 20260508 08:52:31.600 | INFO | Perf report for the last 5.13445 seconds: | 20260508 08:52:31.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:31.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 08:52:31.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:52:31.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:31.600 | INFO | Uploaded 36GiB to GPU in 2.50844s (14.3516GiB/s) | 20260508 08:52:31.600 | INFO | Downloaded 34GiB to GPU in 2.45327s (13.8591GiB/s) | 20260508 08:52:36.742 | INFO | Analyzing chunk frames = [6816768,6817280) | 20260508 08:52:36.742 | INFO | MemInfo: { | 20260508 08:52:36.742 | INFO | "gpu" : " 17 GB", | 20260508 08:52:36.742 | INFO | "huge" : "", | 20260508 08:52:36.742 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:36.742 | INFO | "malloc" : " 14 GB", | 20260508 08:52:36.742 | INFO | "pinned" : " 29 GB" | 20260508 08:52:36.742 | INFO | } | 20260508 08:52:36.742 | INFO | Perf report for the last 5.14257 seconds: | 20260508 08:52:36.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:36.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 08:52:36.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:52:36.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:36.742 | INFO | Uploaded 36GiB to GPU in 2.52428s (14.2615GiB/s) | 20260508 08:52:36.742 | INFO | Downloaded 34GiB to GPU in 2.45492s (13.8497GiB/s) | 20260508 08:52:41.880 | INFO | Analyzing chunk frames = [6817280,6817792) | 20260508 08:52:41.880 | INFO | MemInfo: { | 20260508 08:52:41.880 | INFO | "gpu" : " 17 GB", | 20260508 08:52:41.880 | INFO | "huge" : "", | 20260508 08:52:41.880 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:41.880 | INFO | "malloc" : " 14 GB", | 20260508 08:52:41.880 | INFO | "pinned" : " 29 GB" | 20260508 08:52:41.880 | INFO | } | 20260508 08:52:41.880 | INFO | Perf report for the last 5.13816 seconds: | 20260508 08:52:41.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:41.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.880 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 08:52:41.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.880 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 08:52:41.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:41.881 | INFO | Uploaded 36GiB to GPU in 2.49129s (14.4503GiB/s) | 20260508 08:52:41.881 | INFO | Downloaded 34GiB to GPU in 2.43016s (13.9908GiB/s) | 20260508 08:52:47.023 | INFO | Analyzing chunk frames = [6817792,6818304) | 20260508 08:52:47.023 | INFO | MemInfo: { | 20260508 08:52:47.023 | INFO | "gpu" : " 17 GB", | 20260508 08:52:47.023 | INFO | "huge" : "", | 20260508 08:52:47.023 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:47.023 | INFO | "malloc" : " 14 GB", | 20260508 08:52:47.023 | INFO | "pinned" : " 29 GB" | 20260508 08:52:47.023 | INFO | } | 20260508 08:52:47.023 | INFO | Perf report for the last 5.14271 seconds: | 20260508 08:52:47.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:47.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 08:52:47.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:52:47.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:47.023 | INFO | Uploaded 36GiB to GPU in 2.26166s (15.9175GiB/s) | 20260508 08:52:47.023 | INFO | Downloaded 34GiB to GPU in 2.50755s (13.559GiB/s) | 20260508 08:52:52.160 | INFO | Analyzing chunk frames = [6818304,6818816) | 20260508 08:52:52.160 | INFO | MemInfo: { | 20260508 08:52:52.160 | INFO | "gpu" : " 17 GB", | 20260508 08:52:52.160 | INFO | "huge" : "", | 20260508 08:52:52.160 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:52.160 | INFO | "malloc" : " 14 GB", | 20260508 08:52:52.160 | INFO | "pinned" : " 29 GB" | 20260508 08:52:52.160 | INFO | } | 20260508 08:52:52.160 | INFO | Perf report for the last 5.1369 seconds: | 20260508 08:52:52.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:52.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.160 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:52:52.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.160 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:52:52.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:52.161 | INFO | Uploaded 36GiB to GPU in 2.38766s (15.0775GiB/s) | 20260508 08:52:52.161 | INFO | Downloaded 34GiB to GPU in 2.43664s (13.9537GiB/s) | 20260508 08:52:57.291 | INFO | Analyzing chunk frames = [6818816,6819328) | 20260508 08:52:57.291 | INFO | MemInfo: { | 20260508 08:52:57.291 | INFO | "gpu" : " 17 GB", | 20260508 08:52:57.291 | INFO | "huge" : "", | 20260508 08:52:57.291 | INFO | "huge_shared" : " 29 GB", | 20260508 08:52:57.291 | INFO | "malloc" : " 14 GB", | 20260508 08:52:57.291 | INFO | "pinned" : " 29 GB" | 20260508 08:52:57.291 | INFO | } | 20260508 08:52:57.291 | INFO | Perf report for the last 5.13082 seconds: | 20260508 08:52:57.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:52:57.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 08:52:57.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:52:57.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:52:57.291 | INFO | Uploaded 36GiB to GPU in 2.41485s (14.9078GiB/s) | 20260508 08:52:57.291 | INFO | Downloaded 34GiB to GPU in 2.44446s (13.909GiB/s) | 20260508 08:53:02.433 | INFO | Analyzing chunk frames = [6819328,6819840) | 20260508 08:53:02.433 | INFO | MemInfo: { | 20260508 08:53:02.433 | INFO | "gpu" : " 17 GB", | 20260508 08:53:02.433 | INFO | "huge" : "", | 20260508 08:53:02.433 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:02.433 | INFO | "malloc" : " 14 GB", | 20260508 08:53:02.433 | INFO | "pinned" : " 29 GB" | 20260508 08:53:02.433 | INFO | } | 20260508 08:53:02.433 | INFO | Perf report for the last 5.14243 seconds: | 20260508 08:53:02.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:02.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.433 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 08:53:02.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.433 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 08:53:02.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:02.434 | INFO | Uploaded 36GiB to GPU in 2.46224s (14.6208GiB/s) | 20260508 08:53:02.434 | INFO | Downloaded 34GiB to GPU in 2.45853s (13.8294GiB/s) | 20260508 08:53:02.530 | INFO | Starting a round of Baz IO | 20260508 08:53:05.950 | INFO | Populating RTMetrics report | 20260508 08:53:05.950 | INFO | RTMetrics report populated | 20260508 08:53:05.950 | INFO | Opening tmp RTMetrics json file | 20260508 08:53:05.961 | INFO | Serializing RTMetrics to json | 20260508 08:53:05.962 | INFO | Serialization done, swapping tmp file now... | 20260508 08:53:05.963 | INFO | RTMetrics done | 20260508 08:53:06.017 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.487 seconds | 20260508 08:53:06.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:53:06.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.276077% / 0% / 7.94729e-05% | 20260508 08:53:06.017 | INFO | Baz IO overall: | 20260508 08:53:06.017 | INFO | BytesWritten : 2985586141757 | 20260508 08:53:06.017 | INFO | NumEvents : 1436864923022 | 20260508 08:53:07.578 | INFO | Analyzing chunk frames = [6819840,6820352) | 20260508 08:53:07.578 | INFO | MemInfo: { | 20260508 08:53:07.578 | INFO | "gpu" : " 17 GB", | 20260508 08:53:07.578 | INFO | "huge" : "", | 20260508 08:53:07.579 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:07.579 | INFO | "malloc" : " 14 GB", | 20260508 08:53:07.579 | INFO | "pinned" : " 29 GB" | 20260508 08:53:07.579 | INFO | } | 20260508 08:53:07.579 | INFO | Perf report for the last 5.14532 seconds: | 20260508 08:53:07.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:07.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:07.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:07.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:07.579 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 08:53:07.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:07.579 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 08:53:07.579 | INFO | PRE_HQ : 65.4%, 1.6, 54.4ms, 0 | 20260508 08:53:07.579 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 08:53:07.579 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:53:07.579 | INFO | Uploaded 36GiB to GPU in 2.57033s (14.006GiB/s) | 20260508 08:53:07.579 | INFO | Downloaded 38GiB to GPU in 2.71073s (14.0183GiB/s) | 20260508 08:53:12.719 | INFO | Analyzing chunk frames = [6820352,6820864) | 20260508 08:53:12.719 | INFO | MemInfo: { | 20260508 08:53:12.719 | INFO | "gpu" : " 17 GB", | 20260508 08:53:12.719 | INFO | "huge" : "", | 20260508 08:53:12.719 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:12.719 | INFO | "malloc" : " 14 GB", | 20260508 08:53:12.719 | INFO | "pinned" : " 29 GB" | 20260508 08:53:12.719 | INFO | } | 20260508 08:53:12.719 | INFO | Perf report for the last 5.14046 seconds: | 20260508 08:53:12.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:12.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 08:53:12.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:53:12.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:12.719 | INFO | Uploaded 36GiB to GPU in 2.50045s (14.3974GiB/s) | 20260508 08:53:12.719 | INFO | Downloaded 34GiB to GPU in 2.46125s (13.8141GiB/s) | 20260508 08:53:17.851 | INFO | Analyzing chunk frames = [6820864,6821376) | 20260508 08:53:17.851 | INFO | MemInfo: { | 20260508 08:53:17.851 | INFO | "gpu" : " 17 GB", | 20260508 08:53:17.851 | INFO | "huge" : "", | 20260508 08:53:17.851 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:17.851 | INFO | "malloc" : " 14 GB", | 20260508 08:53:17.851 | INFO | "pinned" : " 29 GB" | 20260508 08:53:17.851 | INFO | } | 20260508 08:53:17.851 | INFO | Perf report for the last 5.13239 seconds: | 20260508 08:53:17.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:17.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.852 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 08:53:17.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.852 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:53:17.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:17.852 | INFO | Uploaded 36GiB to GPU in 2.4679s (14.5873GiB/s) | 20260508 08:53:17.852 | INFO | Downloaded 34GiB to GPU in 2.44003s (13.9343GiB/s) | 20260508 08:53:22.994 | INFO | Analyzing chunk frames = [6821376,6821888) | 20260508 08:53:22.994 | INFO | MemInfo: { | 20260508 08:53:22.994 | INFO | "gpu" : " 17 GB", | 20260508 08:53:22.994 | INFO | "huge" : "", | 20260508 08:53:22.994 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:22.994 | INFO | "malloc" : " 14 GB", | 20260508 08:53:22.994 | INFO | "pinned" : " 29 GB" | 20260508 08:53:22.994 | INFO | } | 20260508 08:53:22.994 | INFO | Perf report for the last 5.14264 seconds: | 20260508 08:53:22.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:22.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 08:53:22.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 08:53:22.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:22.994 | INFO | Uploaded 36GiB to GPU in 2.47282s (14.5583GiB/s) | 20260508 08:53:22.994 | INFO | Downloaded 34GiB to GPU in 2.43232s (13.9784GiB/s) | 20260508 08:53:28.133 | INFO | Analyzing chunk frames = [6821888,6822400) | 20260508 08:53:28.133 | INFO | MemInfo: { | 20260508 08:53:28.133 | INFO | "gpu" : " 17 GB", | 20260508 08:53:28.133 | INFO | "huge" : "", | 20260508 08:53:28.133 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:28.133 | INFO | "malloc" : " 14 GB", | 20260508 08:53:28.133 | INFO | "pinned" : " 29 GB" | 20260508 08:53:28.133 | INFO | } | 20260508 08:53:28.133 | INFO | Perf report for the last 5.13889 seconds: | 20260508 08:53:28.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:28.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 08:53:28.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 08:53:28.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:28.133 | INFO | Uploaded 36GiB to GPU in 2.28299s (15.7688GiB/s) | 20260508 08:53:28.133 | INFO | Downloaded 34GiB to GPU in 2.51727s (13.5067GiB/s) | 20260508 08:53:33.273 | INFO | Analyzing chunk frames = [6822400,6822912) | 20260508 08:53:33.273 | INFO | MemInfo: { | 20260508 08:53:33.273 | INFO | "gpu" : " 17 GB", | 20260508 08:53:33.273 | INFO | "huge" : "", | 20260508 08:53:33.273 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:33.273 | INFO | "malloc" : " 14 GB", | 20260508 08:53:33.273 | INFO | "pinned" : " 29 GB" | 20260508 08:53:33.273 | INFO | } | 20260508 08:53:33.274 | INFO | Perf report for the last 5.14047 seconds: | 20260508 08:53:33.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:33.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 08:53:33.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:53:33.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:33.274 | INFO | Uploaded 36GiB to GPU in 2.31269s (15.5663GiB/s) | 20260508 08:53:33.274 | INFO | Downloaded 34GiB to GPU in 2.44729s (13.8929GiB/s) | 20260508 08:53:38.410 | INFO | Analyzing chunk frames = [6822912,6823424) | 20260508 08:53:38.410 | INFO | MemInfo: { | 20260508 08:53:38.410 | INFO | "gpu" : " 17 GB", | 20260508 08:53:38.410 | INFO | "huge" : "", | 20260508 08:53:38.410 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:38.410 | INFO | "malloc" : " 14 GB", | 20260508 08:53:38.410 | INFO | "pinned" : " 29 GB" | 20260508 08:53:38.410 | INFO | } | 20260508 08:53:38.410 | INFO | Perf report for the last 5.13665 seconds: | 20260508 08:53:38.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:38.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.410 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:53:38.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.410 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 08:53:38.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:38.411 | INFO | Uploaded 36GiB to GPU in 2.38512s (15.0936GiB/s) | 20260508 08:53:38.411 | INFO | Downloaded 34GiB to GPU in 2.46333s (13.8025GiB/s) | 20260508 08:53:43.551 | INFO | Analyzing chunk frames = [6823424,6823936) | 20260508 08:53:43.551 | INFO | MemInfo: { | 20260508 08:53:43.551 | INFO | "gpu" : " 17 GB", | 20260508 08:53:43.552 | INFO | "huge" : "", | 20260508 08:53:43.552 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:43.552 | INFO | "malloc" : " 14 GB", | 20260508 08:53:43.552 | INFO | "pinned" : " 29 GB" | 20260508 08:53:43.552 | INFO | } | 20260508 08:53:43.552 | INFO | Perf report for the last 5.14146 seconds: | 20260508 08:53:43.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:43.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:53:43.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 08:53:43.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:43.552 | INFO | Uploaded 36GiB to GPU in 2.40952s (14.9407GiB/s) | 20260508 08:53:43.552 | INFO | Downloaded 34GiB to GPU in 2.46705s (13.7816GiB/s) | 20260508 08:53:43.553 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:53:43.553 | INFO | Total mem usage: 16.7 GiB | 20260508 08:53:43.553 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:53:43.553 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:53:43.553 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:53:43.553 | INFO | High Water Breakdown: | 20260508 08:53:43.553 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:53:43.553 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:53:43.553 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:53:43.553 | INFO | ------------------------------------------ | 20260508 08:53:43.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:53:43.553 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:53:43.553 | INFO | Total mem usage: 26.5 GiB | 20260508 08:53:43.553 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:53:43.553 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:53:43.553 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:53:43.553 | INFO | High Water Breakdown: | 20260508 08:53:43.553 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:53:43.553 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:53:43.553 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:53:43.553 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:53:43.553 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:53:43.553 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:53:43.553 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:53:43.553 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:53:43.553 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:53:43.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:53:43.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:53:43.553 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:53:43.553 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:53:43.553 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:53:43.553 | INFO | ------------------------------------------ | 20260508 08:53:43.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:53:43.553 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:53:43.553 | INFO | Total mem usage: 27.8 GiB | 20260508 08:53:43.553 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:53:43.553 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:53:43.553 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:53:43.553 | INFO | High Water Breakdown: | 20260508 08:53:43.553 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:53:43.553 | INFO | ------------------------------------------ | 20260508 08:53:43.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:53:43.553 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:53:43.553 | INFO | Total mem usage: 15.7 GiB | 20260508 08:53:43.553 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:53:43.553 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:53:43.553 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:53:43.553 | INFO | High Water Breakdown: | 20260508 08:53:43.553 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:53:43.553 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:53:43.553 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:53:43.554 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:53:43.554 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:53:43.554 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:53:43.554 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:53:43.554 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:53:43.554 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:53:43.554 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:53:43.554 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:53:43.554 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:53:43.554 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:53:43.554 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:53:43.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:53:43.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:53:43.554 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:53:43.554 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:53:43.554 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:53:43.554 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:53:43.554 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:53:43.554 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:53:43.554 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:53:43.554 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:53:43.554 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:53:43.554 | INFO | ------------------------------------------ | 20260508 08:53:43.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:53:43.554 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:53:43.554 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:53:43.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:53:43.554 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.41e+01 / 7.13e-01 | 20260508 08:53:43.554 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.04e-01, 1.17e-02 / 5.43e-03 | 20260508 08:53:43.554 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.41e+00, 5.95e-01 / 2.59e-01 | 20260508 08:53:43.554 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.46e+00, 2.12e-01 / 1.50e-01 | 20260508 08:53:43.554 | INFO | PerfReport: 15.95%, DME -- 4.19e+01 / 4.54e+01, 3.82e+00 / 1.35e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.88e+01 / 1.99e+01, 1.91e+00 / 1.42e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.58e+00, 4.08e-01 / 2.56e-01 | 20260508 08:53:43.554 | INFO | PerfReport: 3.04%, Metrics -- 9.97e-01 / 1.14e+00, 2.50e-01 / 1.22e-01 | 20260508 08:53:43.554 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.43e-01, 4.96e-02 / 2.77e-02 | 20260508 08:53:43.554 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:53:43.554 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:53:43.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:53:43.554 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 24.11%, Upload -- 2.54e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 31.16%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 2.09%, LockUpload -- 2.20e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 2.55%, LockDownload -- 2.68e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 15.47%, LockCompute -- 1.63e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 08:53:43.554 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260508 08:53:43.663 | INFO | Starting a round of Baz IO | 20260508 08:53:47.062 | INFO | Populating RTMetrics report | 20260508 08:53:47.063 | INFO | RTMetrics report populated | 20260508 08:53:47.063 | INFO | Opening tmp RTMetrics json file | 20260508 08:53:47.074 | INFO | Serializing RTMetrics to json | 20260508 08:53:47.076 | INFO | Serialization done, swapping tmp file now... | 20260508 08:53:47.076 | INFO | RTMetrics done | 20260508 08:53:47.132 | INFO | Finished a round of Baz IO: Wrote 1.493 GiB in 3.468 seconds | 20260508 08:53:47.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:53:47.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.275779% / 0% / 9.13938e-05% | 20260508 08:53:47.132 | INFO | Baz IO overall: | 20260508 08:53:47.132 | INFO | BytesWritten : 2987189002322 | 20260508 08:53:47.132 | INFO | NumEvents : 1437623223992 | 20260508 08:53:48.694 | INFO | Analyzing chunk frames = [6823936,6824448) | 20260508 08:53:48.694 | INFO | MemInfo: { | 20260508 08:53:48.694 | INFO | "gpu" : " 17 GB", | 20260508 08:53:48.694 | INFO | "huge" : "", | 20260508 08:53:48.694 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:48.694 | INFO | "malloc" : " 14 GB", | 20260508 08:53:48.694 | INFO | "pinned" : " 29 GB" | 20260508 08:53:48.694 | INFO | } | 20260508 08:53:48.694 | INFO | Perf report for the last 5.14271 seconds: | 20260508 08:53:48.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:48.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:48.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:48.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:48.694 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 08:53:48.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:48.694 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 08:53:48.695 | INFO | PRE_HQ : 66.5%, 1.8, 61.9ms, 0 | 20260508 08:53:48.695 | INFO | BAZWRITER : 66.7%, 2.8, 94.3ms, 0 | 20260508 08:53:48.695 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:53:48.695 | INFO | Uploaded 36GiB to GPU in 2.59857s (13.8538GiB/s) | 20260508 08:53:48.695 | INFO | Downloaded 38GiB to GPU in 2.69847s (14.0821GiB/s) | 20260508 08:53:53.834 | INFO | Analyzing chunk frames = [6824448,6824960) | 20260508 08:53:53.835 | INFO | MemInfo: { | 20260508 08:53:53.835 | INFO | "gpu" : " 17 GB", | 20260508 08:53:53.835 | INFO | "huge" : "", | 20260508 08:53:53.835 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:53.835 | INFO | "malloc" : " 14 GB", | 20260508 08:53:53.835 | INFO | "pinned" : " 29 GB" | 20260508 08:53:53.835 | INFO | } | 20260508 08:53:53.835 | INFO | Perf report for the last 5.14022 seconds: | 20260508 08:53:53.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:53.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 08:53:53.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:53:53.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:53.835 | INFO | Uploaded 36GiB to GPU in 2.51984s (14.2866GiB/s) | 20260508 08:53:53.835 | INFO | Downloaded 34GiB to GPU in 2.45734s (13.8361GiB/s) | 20260508 08:53:58.969 | INFO | Analyzing chunk frames = [6824960,6825472) | 20260508 08:53:58.969 | INFO | MemInfo: { | 20260508 08:53:58.969 | INFO | "gpu" : " 17 GB", | 20260508 08:53:58.969 | INFO | "huge" : "", | 20260508 08:53:58.969 | INFO | "huge_shared" : " 29 GB", | 20260508 08:53:58.969 | INFO | "malloc" : " 14 GB", | 20260508 08:53:58.969 | INFO | "pinned" : " 29 GB" | 20260508 08:53:58.969 | INFO | } | 20260508 08:53:58.969 | INFO | Perf report for the last 5.13436 seconds: | 20260508 08:53:58.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:53:58.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 08:53:58.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:53:58.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:53:58.969 | INFO | Uploaded 36GiB to GPU in 2.44404s (14.7297GiB/s) | 20260508 08:53:58.969 | INFO | Downloaded 34GiB to GPU in 2.40569s (14.1332GiB/s) | 20260508 08:54:04.104 | INFO | Analyzing chunk frames = [6825472,6825984) | 20260508 08:54:04.104 | INFO | MemInfo: { | 20260508 08:54:04.104 | INFO | "gpu" : " 17 GB", | 20260508 08:54:04.104 | INFO | "huge" : "", | 20260508 08:54:04.104 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:04.104 | INFO | "malloc" : " 14 GB", | 20260508 08:54:04.104 | INFO | "pinned" : " 29 GB" | 20260508 08:54:04.104 | INFO | } | 20260508 08:54:04.104 | INFO | Perf report for the last 5.13477 seconds: | 20260508 08:54:04.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:04.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | ANALYSIS : 67.0%, 3.0, 103.6ms, 0 | 20260508 08:54:04.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:54:04.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:04.104 | INFO | Uploaded 36GiB to GPU in 2.57876s (13.9602GiB/s) | 20260508 08:54:04.104 | INFO | Downloaded 34GiB to GPU in 2.42649s (14.012GiB/s) | 20260508 08:54:09.246 | INFO | Analyzing chunk frames = [6825984,6826496) | 20260508 08:54:09.246 | INFO | MemInfo: { | 20260508 08:54:09.246 | INFO | "gpu" : " 17 GB", | 20260508 08:54:09.246 | INFO | "huge" : "", | 20260508 08:54:09.246 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:09.246 | INFO | "malloc" : " 14 GB", | 20260508 08:54:09.246 | INFO | "pinned" : " 29 GB" | 20260508 08:54:09.246 | INFO | } | 20260508 08:54:09.246 | INFO | Perf report for the last 5.14252 seconds: | 20260508 08:54:09.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:09.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 08:54:09.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 08:54:09.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:09.246 | INFO | Uploaded 36GiB to GPU in 2.42144s (14.8672GiB/s) | 20260508 08:54:09.246 | INFO | Downloaded 34GiB to GPU in 2.50384s (13.5792GiB/s) | 20260508 08:54:14.393 | INFO | Analyzing chunk frames = [6826496,6827008) | 20260508 08:54:14.393 | INFO | MemInfo: { | 20260508 08:54:14.393 | INFO | "gpu" : " 17 GB", | 20260508 08:54:14.393 | INFO | "huge" : "", | 20260508 08:54:14.393 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:14.393 | INFO | "malloc" : " 14 GB", | 20260508 08:54:14.393 | INFO | "pinned" : " 29 GB" | 20260508 08:54:14.393 | INFO | } | 20260508 08:54:14.394 | INFO | Perf report for the last 5.14718 seconds: | 20260508 08:54:14.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:14.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 08:54:14.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:54:14.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:14.394 | INFO | Uploaded 36GiB to GPU in 2.38734s (15.0795GiB/s) | 20260508 08:54:14.394 | INFO | Downloaded 34GiB to GPU in 2.43414s (13.968GiB/s) | 20260508 08:54:19.529 | INFO | Analyzing chunk frames = [6827008,6827520) | 20260508 08:54:19.529 | INFO | MemInfo: { | 20260508 08:54:19.529 | INFO | "gpu" : " 17 GB", | 20260508 08:54:19.529 | INFO | "huge" : "", | 20260508 08:54:19.529 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:19.529 | INFO | "malloc" : " 14 GB", | 20260508 08:54:19.529 | INFO | "pinned" : " 29 GB" | 20260508 08:54:19.529 | INFO | } | 20260508 08:54:19.529 | INFO | Perf report for the last 5.13534 seconds: | 20260508 08:54:19.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:19.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260508 08:54:19.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 08:54:19.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:19.529 | INFO | Uploaded 36GiB to GPU in 2.38511s (15.0936GiB/s) | 20260508 08:54:19.529 | INFO | Downloaded 34GiB to GPU in 2.4392s (13.939GiB/s) | 20260508 08:54:24.668 | INFO | Analyzing chunk frames = [6827520,6828032) | 20260508 08:54:24.668 | INFO | MemInfo: { | 20260508 08:54:24.668 | INFO | "gpu" : " 17 GB", | 20260508 08:54:24.668 | INFO | "huge" : "", | 20260508 08:54:24.668 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:24.668 | INFO | "malloc" : " 14 GB", | 20260508 08:54:24.668 | INFO | "pinned" : " 29 GB" | 20260508 08:54:24.668 | INFO | } | 20260508 08:54:24.668 | INFO | Perf report for the last 5.13902 seconds: | 20260508 08:54:24.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:24.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 08:54:24.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 08:54:24.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:24.668 | INFO | Uploaded 36GiB to GPU in 2.41589s (14.9013GiB/s) | 20260508 08:54:24.669 | INFO | Downloaded 34GiB to GPU in 2.44503s (13.9057GiB/s) | 20260508 08:54:24.781 | INFO | Starting a round of Baz IO | 20260508 08:54:28.186 | INFO | Populating RTMetrics report | 20260508 08:54:28.186 | INFO | RTMetrics report populated | 20260508 08:54:28.186 | INFO | Opening tmp RTMetrics json file | 20260508 08:54:28.187 | INFO | Serializing RTMetrics to json | 20260508 08:54:28.188 | INFO | Serialization done, swapping tmp file now... | 20260508 08:54:28.189 | INFO | RTMetrics done | 20260508 08:54:28.253 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.472 seconds | 20260508 08:54:28.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:54:28.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.275401% / 0% / 5.5631e-05% | 20260508 08:54:28.253 | INFO | Baz IO overall: | 20260508 08:54:28.253 | INFO | BytesWritten : 2988791198883 | 20260508 08:54:28.254 | INFO | NumEvents : 1438381346097 | 20260508 08:54:29.800 | INFO | Analyzing chunk frames = [6828032,6828544) | 20260508 08:54:29.800 | INFO | MemInfo: { | 20260508 08:54:29.800 | INFO | "gpu" : " 17 GB", | 20260508 08:54:29.800 | INFO | "huge" : "", | 20260508 08:54:29.800 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:29.800 | INFO | "malloc" : " 14 GB", | 20260508 08:54:29.801 | INFO | "pinned" : " 29 GB" | 20260508 08:54:29.801 | INFO | } | 20260508 08:54:29.801 | INFO | Perf report for the last 5.13263 seconds: | 20260508 08:54:29.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:29.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:29.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:29.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:29.801 | INFO | ANALYSIS : 68.2%, 3.0, 105.7ms, 0 | 20260508 08:54:29.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:29.801 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 08:54:29.801 | INFO | PRE_HQ : 66.9%, 1.8, 61.6ms, 0 | 20260508 08:54:29.801 | INFO | BAZWRITER : 66.9%, 2.8, 95.3ms, 0 | 20260508 08:54:29.801 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 08:54:29.801 | INFO | Uploaded 36GiB to GPU in 2.53651s (14.1928GiB/s) | 20260508 08:54:29.801 | INFO | Downloaded 38GiB to GPU in 2.72214s (13.9596GiB/s) | 20260508 08:54:34.948 | INFO | Analyzing chunk frames = [6828544,6829056) | 20260508 08:54:34.948 | INFO | MemInfo: { | 20260508 08:54:34.948 | INFO | "gpu" : " 17 GB", | 20260508 08:54:34.948 | INFO | "huge" : "", | 20260508 08:54:34.948 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:34.948 | INFO | "malloc" : " 14 GB", | 20260508 08:54:34.948 | INFO | "pinned" : " 29 GB" | 20260508 08:54:34.948 | INFO | } | 20260508 08:54:34.948 | INFO | Perf report for the last 5.1474 seconds: | 20260508 08:54:34.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:34.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 08:54:34.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:54:34.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:34.948 | INFO | Uploaded 36GiB to GPU in 2.52497s (14.2576GiB/s) | 20260508 08:54:34.948 | INFO | Downloaded 34GiB to GPU in 2.46209s (13.8094GiB/s) | 20260508 08:54:40.083 | INFO | Analyzing chunk frames = [6829056,6829568) | 20260508 08:54:40.083 | INFO | MemInfo: { | 20260508 08:54:40.083 | INFO | "gpu" : " 17 GB", | 20260508 08:54:40.083 | INFO | "huge" : "", | 20260508 08:54:40.083 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:40.083 | INFO | "malloc" : " 14 GB", | 20260508 08:54:40.083 | INFO | "pinned" : " 29 GB" | 20260508 08:54:40.083 | INFO | } | 20260508 08:54:40.083 | INFO | Perf report for the last 5.13527 seconds: | 20260508 08:54:40.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:40.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:54:40.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:54:40.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:40.083 | INFO | Uploaded 36GiB to GPU in 2.48906s (14.4633GiB/s) | 20260508 08:54:40.084 | INFO | Downloaded 34GiB to GPU in 2.43489s (13.9637GiB/s) | 20260508 08:54:45.229 | INFO | Analyzing chunk frames = [6829568,6830080) | 20260508 08:54:45.229 | INFO | MemInfo: { | 20260508 08:54:45.229 | INFO | "gpu" : " 17 GB", | 20260508 08:54:45.229 | INFO | "huge" : "", | 20260508 08:54:45.229 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:45.229 | INFO | "malloc" : " 14 GB", | 20260508 08:54:45.230 | INFO | "pinned" : " 29 GB" | 20260508 08:54:45.230 | INFO | } | 20260508 08:54:45.230 | INFO | Perf report for the last 5.14632 seconds: | 20260508 08:54:45.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:45.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | ANALYSIS : 66.8%, 3.0, 103.5ms, 0 | 20260508 08:54:45.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:54:45.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:45.230 | INFO | Uploaded 36GiB to GPU in 2.57899s (13.9589GiB/s) | 20260508 08:54:45.230 | INFO | Downloaded 34GiB to GPU in 2.42785s (14.0041GiB/s) | 20260508 08:54:50.363 | INFO | Analyzing chunk frames = [6830080,6830592) | 20260508 08:54:50.363 | INFO | MemInfo: { | 20260508 08:54:50.363 | INFO | "gpu" : " 17 GB", | 20260508 08:54:50.363 | INFO | "huge" : "", | 20260508 08:54:50.363 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:50.363 | INFO | "malloc" : " 14 GB", | 20260508 08:54:50.363 | INFO | "pinned" : " 29 GB" | 20260508 08:54:50.363 | INFO | } | 20260508 08:54:50.363 | INFO | Perf report for the last 5.13341 seconds: | 20260508 08:54:50.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:50.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 08:54:50.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 08:54:50.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:50.363 | INFO | Uploaded 36GiB to GPU in 2.31125s (15.576GiB/s) | 20260508 08:54:50.363 | INFO | Downloaded 34GiB to GPU in 2.50447s (13.5757GiB/s) | 20260508 08:54:55.501 | INFO | Analyzing chunk frames = [6830592,6831104) | 20260508 08:54:55.501 | INFO | MemInfo: { | 20260508 08:54:55.501 | INFO | "gpu" : " 17 GB", | 20260508 08:54:55.501 | INFO | "huge" : "", | 20260508 08:54:55.501 | INFO | "huge_shared" : " 29 GB", | 20260508 08:54:55.501 | INFO | "malloc" : " 14 GB", | 20260508 08:54:55.501 | INFO | "pinned" : " 29 GB" | 20260508 08:54:55.501 | INFO | } | 20260508 08:54:55.501 | INFO | Perf report for the last 5.13827 seconds: | 20260508 08:54:55.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:54:55.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.501 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:54:55.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.501 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 08:54:55.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:54:55.502 | INFO | Uploaded 36GiB to GPU in 2.40866s (14.9461GiB/s) | 20260508 08:54:55.502 | INFO | Downloaded 34GiB to GPU in 2.41505s (14.0784GiB/s) | 20260508 08:55:00.641 | INFO | Analyzing chunk frames = [6831104,6831616) | 20260508 08:55:00.641 | INFO | MemInfo: { | 20260508 08:55:00.641 | INFO | "gpu" : " 17 GB", | 20260508 08:55:00.641 | INFO | "huge" : "", | 20260508 08:55:00.641 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:00.641 | INFO | "malloc" : " 14 GB", | 20260508 08:55:00.641 | INFO | "pinned" : " 29 GB" | 20260508 08:55:00.641 | INFO | } | 20260508 08:55:00.641 | INFO | Perf report for the last 5.13962 seconds: | 20260508 08:55:00.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:00.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:55:00.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 08:55:00.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:00.641 | INFO | Uploaded 36GiB to GPU in 2.45566s (14.66GiB/s) | 20260508 08:55:00.641 | INFO | Downloaded 34GiB to GPU in 2.43481s (13.9641GiB/s) | 20260508 08:55:05.784 | INFO | Analyzing chunk frames = [6831616,6832128) | 20260508 08:55:05.784 | INFO | MemInfo: { | 20260508 08:55:05.784 | INFO | "gpu" : " 17 GB", | 20260508 08:55:05.784 | INFO | "huge" : "", | 20260508 08:55:05.784 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:05.784 | INFO | "malloc" : " 14 GB", | 20260508 08:55:05.784 | INFO | "pinned" : " 29 GB" | 20260508 08:55:05.784 | INFO | } | 20260508 08:55:05.784 | INFO | Perf report for the last 5.14304 seconds: | 20260508 08:55:05.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:05.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 08:55:05.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260508 08:55:05.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:05.784 | INFO | Uploaded 36GiB to GPU in 2.46231s (14.6204GiB/s) | 20260508 08:55:05.784 | INFO | Downloaded 34GiB to GPU in 2.43842s (13.9434GiB/s) | 20260508 08:55:05.785 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:55:05.785 | INFO | Total mem usage: 16.7 GiB | 20260508 08:55:05.785 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:55:05.785 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:55:05.785 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:55:05.785 | INFO | High Water Breakdown: | 20260508 08:55:05.785 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:55:05.785 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:55:05.785 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:55:05.785 | INFO | ------------------------------------------ | 20260508 08:55:05.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:55:05.785 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:55:05.785 | INFO | Total mem usage: 26.5 GiB | 20260508 08:55:05.785 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:55:05.785 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 08:55:05.785 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 08:55:05.785 | INFO | High Water Breakdown: | 20260508 08:55:05.785 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:55:05.785 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:55:05.785 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:55:05.785 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:55:05.785 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:55:05.785 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:55:05.785 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:55:05.785 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:55:05.785 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:55:05.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:55:05.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:55:05.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:55:05.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:55:05.785 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:55:05.785 | INFO | ------------------------------------------ | 20260508 08:55:05.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:55:05.785 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:55:05.785 | INFO | Total mem usage: 27.8 GiB | 20260508 08:55:05.785 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:55:05.785 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:55:05.785 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:55:05.785 | INFO | High Water Breakdown: | 20260508 08:55:05.785 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:55:05.785 | INFO | ------------------------------------------ | 20260508 08:55:05.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:55:05.785 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:55:05.785 | INFO | Total mem usage: 15.7 GiB | 20260508 08:55:05.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:55:05.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:55:05.785 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:55:05.785 | INFO | High Water Breakdown: | 20260508 08:55:05.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:55:05.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:55:05.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:55:05.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:55:05.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:55:05.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:55:05.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:55:05.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:55:05.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:55:05.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:55:05.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:55:05.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:55:05.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:55:05.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:55:05.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:55:05.786 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:55:05.786 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:55:05.786 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:55:05.786 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:55:05.786 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:55:05.786 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:55:05.786 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:55:05.786 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:55:05.786 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:55:05.786 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:55:05.786 | INFO | ------------------------------------------ | 20260508 08:55:05.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:55:05.786 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:55:05.786 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:55:05.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:55:05.786 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 7.92e-01 | 20260508 08:55:05.786 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.08e-01, 1.16e-02 / 7.89e-03 | 20260508 08:55:05.786 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.45e+00, 5.86e-01 / 2.47e-01 | 20260508 08:55:05.786 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.22e-01 | 20260508 08:55:05.786 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.55e+01, 3.85e+00 / 1.05e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.80e+00 / 1.13e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.57e+00, 4.06e-01 / 2.55e-01 | 20260508 08:55:05.786 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.14e+00, 2.48e-01 / 1.38e-01 | 20260508 08:55:05.786 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.40e-01, 4.93e-02 / 2.92e-02 | 20260508 08:55:05.786 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:55:05.786 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:55:05.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:55:05.786 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 24.37%, Upload -- 2.57e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 24.46%, Download -- 2.58e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 31.02%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 2.23%, LockUpload -- 2.36e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 2.52%, LockDownload -- 2.66e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 08:55:05.786 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260508 08:55:05.897 | INFO | Starting a round of Baz IO | 20260508 08:55:09.290 | INFO | Populating RTMetrics report | 20260508 08:55:09.290 | INFO | RTMetrics report populated | 20260508 08:55:09.290 | INFO | Opening tmp RTMetrics json file | 20260508 08:55:09.300 | INFO | Serializing RTMetrics to json | 20260508 08:55:09.301 | INFO | Serialization done, swapping tmp file now... | 20260508 08:55:09.302 | INFO | RTMetrics done | 20260508 08:55:09.364 | INFO | Finished a round of Baz IO: Wrote 1.493 GiB in 3.466 seconds | 20260508 08:55:09.364 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.37% / 0.00% / 59.63% | 20260508 08:55:09.364 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.275489% / 3.97364e-06% / 8.74201e-05% | 20260508 08:55:09.364 | INFO | Baz IO overall: | 20260508 08:55:09.364 | INFO | BytesWritten : 2990394157322 | 20260508 08:55:09.364 | INFO | NumEvents : 1439140387618 | 20260508 08:55:10.919 | INFO | Analyzing chunk frames = [6832128,6832640) | 20260508 08:55:10.919 | INFO | MemInfo: { | 20260508 08:55:10.919 | INFO | "gpu" : " 17 GB", | 20260508 08:55:10.919 | INFO | "huge" : "", | 20260508 08:55:10.919 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:10.919 | INFO | "malloc" : " 14 GB", | 20260508 08:55:10.919 | INFO | "pinned" : " 29 GB" | 20260508 08:55:10.919 | INFO | } | 20260508 08:55:10.919 | INFO | Perf report for the last 5.13462 seconds: | 20260508 08:55:10.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:10.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:10.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:10.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:10.919 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 08:55:10.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:10.919 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260508 08:55:10.919 | INFO | PRE_HQ : 66.2%, 1.8, 60.8ms, 0 | 20260508 08:55:10.919 | INFO | BAZWRITER : 66.8%, 2.7, 93.5ms, 0 | 20260508 08:55:10.919 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:55:10.919 | INFO | Uploaded 36GiB to GPU in 2.59994s (13.8465GiB/s) | 20260508 08:55:10.919 | INFO | Downloaded 38GiB to GPU in 2.69964s (14.0759GiB/s) | 20260508 08:55:16.058 | INFO | Analyzing chunk frames = [6832640,6833152) | 20260508 08:55:16.058 | INFO | MemInfo: { | 20260508 08:55:16.058 | INFO | "gpu" : " 17 GB", | 20260508 08:55:16.058 | INFO | "huge" : "", | 20260508 08:55:16.058 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:16.058 | INFO | "malloc" : " 14 GB", | 20260508 08:55:16.058 | INFO | "pinned" : " 29 GB" | 20260508 08:55:16.058 | INFO | } | 20260508 08:55:16.058 | INFO | Perf report for the last 5.1395 seconds: | 20260508 08:55:16.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:16.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 08:55:16.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:55:16.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:16.058 | INFO | Uploaded 36GiB to GPU in 2.54365s (14.1529GiB/s) | 20260508 08:55:16.058 | INFO | Downloaded 34GiB to GPU in 2.43684s (13.9525GiB/s) | 20260508 08:55:21.198 | INFO | Analyzing chunk frames = [6833152,6833664) | 20260508 08:55:21.198 | INFO | MemInfo: { | 20260508 08:55:21.198 | INFO | "gpu" : " 17 GB", | 20260508 08:55:21.198 | INFO | "huge" : "", | 20260508 08:55:21.198 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:21.199 | INFO | "malloc" : " 14 GB", | 20260508 08:55:21.199 | INFO | "pinned" : " 29 GB" | 20260508 08:55:21.199 | INFO | } | 20260508 08:55:21.199 | INFO | Perf report for the last 5.14047 seconds: | 20260508 08:55:21.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:21.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 08:55:21.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 08:55:21.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:21.199 | INFO | Uploaded 36GiB to GPU in 2.48333s (14.4967GiB/s) | 20260508 08:55:21.199 | INFO | Downloaded 34GiB to GPU in 2.42416s (14.0255GiB/s) | 20260508 08:55:26.343 | INFO | Analyzing chunk frames = [6833664,6834176) | 20260508 08:55:26.343 | INFO | MemInfo: { | 20260508 08:55:26.343 | INFO | "gpu" : " 17 GB", | 20260508 08:55:26.343 | INFO | "huge" : "", | 20260508 08:55:26.343 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:26.343 | INFO | "malloc" : " 14 GB", | 20260508 08:55:26.343 | INFO | "pinned" : " 29 GB" | 20260508 08:55:26.343 | INFO | } | 20260508 08:55:26.343 | INFO | Perf report for the last 5.14407 seconds: | 20260508 08:55:26.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:26.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | ANALYSIS : 66.9%, 2.9, 103.6ms, 0 | 20260508 08:55:26.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:55:26.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:26.343 | INFO | Uploaded 36GiB to GPU in 2.59019s (13.8986GiB/s) | 20260508 08:55:26.343 | INFO | Downloaded 34GiB to GPU in 2.43033s (13.9899GiB/s) | 20260508 08:55:31.481 | INFO | Analyzing chunk frames = [6834176,6834688) | 20260508 08:55:31.481 | INFO | MemInfo: { | 20260508 08:55:31.481 | INFO | "gpu" : " 17 GB", | 20260508 08:55:31.481 | INFO | "huge" : "", | 20260508 08:55:31.481 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:31.481 | INFO | "malloc" : " 14 GB", | 20260508 08:55:31.481 | INFO | "pinned" : " 29 GB" | 20260508 08:55:31.481 | INFO | } | 20260508 08:55:31.481 | INFO | Perf report for the last 5.13863 seconds: | 20260508 08:55:31.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:31.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.481 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:55:31.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.481 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:55:31.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:31.482 | INFO | Uploaded 36GiB to GPU in 2.41504s (14.9066GiB/s) | 20260508 08:55:31.482 | INFO | Downloaded 34GiB to GPU in 2.51417s (13.5233GiB/s) | 20260508 08:55:36.611 | INFO | Analyzing chunk frames = [6834688,6835200) | 20260508 08:55:36.611 | INFO | MemInfo: { | 20260508 08:55:36.611 | INFO | "gpu" : " 17 GB", | 20260508 08:55:36.611 | INFO | "huge" : "", | 20260508 08:55:36.611 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:36.611 | INFO | "malloc" : " 14 GB", | 20260508 08:55:36.611 | INFO | "pinned" : " 29 GB" | 20260508 08:55:36.611 | INFO | } | 20260508 08:55:36.611 | INFO | Perf report for the last 5.13011 seconds: | 20260508 08:55:36.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:36.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | ANALYSIS : 65.7%, 3.0, 102.0ms, 0 | 20260508 08:55:36.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:55:36.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:36.612 | INFO | Uploaded 36GiB to GPU in 2.38755s (15.0782GiB/s) | 20260508 08:55:36.612 | INFO | Downloaded 34GiB to GPU in 2.4213s (14.042GiB/s) | 20260508 08:55:41.761 | INFO | Analyzing chunk frames = [6835200,6835712) | 20260508 08:55:41.761 | INFO | MemInfo: { | 20260508 08:55:41.761 | INFO | "gpu" : " 17 GB", | 20260508 08:55:41.761 | INFO | "huge" : "", | 20260508 08:55:41.761 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:41.761 | INFO | "malloc" : " 14 GB", | 20260508 08:55:41.761 | INFO | "pinned" : " 29 GB" | 20260508 08:55:41.761 | INFO | } | 20260508 08:55:41.761 | INFO | Perf report for the last 5.14958 seconds: | 20260508 08:55:41.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:41.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | ANALYSIS : 66.1%, 3.0, 103.5ms, 0 | 20260508 08:55:41.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 08:55:41.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:41.761 | INFO | Uploaded 36GiB to GPU in 2.50086s (14.395GiB/s) | 20260508 08:55:41.762 | INFO | Downloaded 34GiB to GPU in 2.43648s (13.9546GiB/s) | 20260508 08:55:46.893 | INFO | Analyzing chunk frames = [6835712,6836224) | 20260508 08:55:46.894 | INFO | MemInfo: { | 20260508 08:55:46.894 | INFO | "gpu" : " 17 GB", | 20260508 08:55:46.894 | INFO | "huge" : "", | 20260508 08:55:46.894 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:46.894 | INFO | "malloc" : " 14 GB", | 20260508 08:55:46.894 | INFO | "pinned" : " 29 GB" | 20260508 08:55:46.894 | INFO | } | 20260508 08:55:46.894 | INFO | Perf report for the last 5.13263 seconds: | 20260508 08:55:46.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:46.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 08:55:46.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260508 08:55:46.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:46.894 | INFO | Uploaded 36GiB to GPU in 2.4631s (14.6157GiB/s) | 20260508 08:55:46.894 | INFO | Downloaded 34GiB to GPU in 2.43792s (13.9463GiB/s) | 20260508 08:55:46.991 | INFO | Starting a round of Baz IO | 20260508 08:55:50.415 | INFO | Populating RTMetrics report | 20260508 08:55:50.415 | INFO | RTMetrics report populated | 20260508 08:55:50.415 | INFO | Opening tmp RTMetrics json file | 20260508 08:55:50.422 | INFO | Serializing RTMetrics to json | 20260508 08:55:50.423 | INFO | Serialization done, swapping tmp file now... | 20260508 08:55:50.424 | INFO | RTMetrics done | 20260508 08:55:50.484 | INFO | Finished a round of Baz IO: Wrote 1.493 GiB in 3.493 seconds | 20260508 08:55:50.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.63% | 20260508 08:55:50.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.274881% / 0% / 7.54992e-05% | 20260508 08:55:50.484 | INFO | Baz IO overall: | 20260508 08:55:50.484 | INFO | BytesWritten : 2991997771112 | 20260508 08:55:50.484 | INFO | NumEvents : 1439899894388 | 20260508 08:55:52.038 | INFO | Analyzing chunk frames = [6836224,6836736) | 20260508 08:55:52.038 | INFO | MemInfo: { | 20260508 08:55:52.038 | INFO | "gpu" : " 17 GB", | 20260508 08:55:52.038 | INFO | "huge" : "", | 20260508 08:55:52.038 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:52.038 | INFO | "malloc" : " 14 GB", | 20260508 08:55:52.038 | INFO | "pinned" : " 29 GB" | 20260508 08:55:52.038 | INFO | } | 20260508 08:55:52.038 | INFO | Perf report for the last 5.14461 seconds: | 20260508 08:55:52.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:52.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:52.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:52.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:52.038 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 08:55:52.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:52.038 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 08:55:52.038 | INFO | PRE_HQ : 65.3%, 1.6, 54.6ms, 0 | 20260508 08:55:52.038 | INFO | BAZWRITER : 67.2%, 2.5, 88.0ms, 0 | 20260508 08:55:52.038 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 08:55:52.039 | INFO | Uploaded 36GiB to GPU in 2.59799s (13.8569GiB/s) | 20260508 08:55:52.039 | INFO | Downloaded 38GiB to GPU in 2.69926s (14.0779GiB/s) | 20260508 08:55:57.174 | INFO | Analyzing chunk frames = [6836736,6837248) | 20260508 08:55:57.174 | INFO | MemInfo: { | 20260508 08:55:57.174 | INFO | "gpu" : " 17 GB", | 20260508 08:55:57.174 | INFO | "huge" : "", | 20260508 08:55:57.174 | INFO | "huge_shared" : " 29 GB", | 20260508 08:55:57.174 | INFO | "malloc" : " 14 GB", | 20260508 08:55:57.174 | INFO | "pinned" : " 29 GB" | 20260508 08:55:57.174 | INFO | } | 20260508 08:55:57.174 | INFO | Perf report for the last 5.13558 seconds: | 20260508 08:55:57.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:55:57.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.174 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:55:57.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.174 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:55:57.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:55:57.175 | INFO | Uploaded 36GiB to GPU in 2.54085s (14.1685GiB/s) | 20260508 08:55:57.175 | INFO | Downloaded 34GiB to GPU in 2.45619s (13.8426GiB/s) | 20260508 08:56:02.311 | INFO | Analyzing chunk frames = [6837248,6837760) | 20260508 08:56:02.311 | INFO | MemInfo: { | 20260508 08:56:02.311 | INFO | "gpu" : " 17 GB", | 20260508 08:56:02.311 | INFO | "huge" : "", | 20260508 08:56:02.311 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:02.311 | INFO | "malloc" : " 14 GB", | 20260508 08:56:02.311 | INFO | "pinned" : " 29 GB" | 20260508 08:56:02.311 | INFO | } | 20260508 08:56:02.311 | INFO | Perf report for the last 5.13715 seconds: | 20260508 08:56:02.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:02.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 08:56:02.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:56:02.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:02.311 | INFO | Uploaded 36GiB to GPU in 2.51811s (14.2964GiB/s) | 20260508 08:56:02.311 | INFO | Downloaded 34GiB to GPU in 2.43804s (13.9456GiB/s) | 20260508 08:56:07.457 | INFO | Analyzing chunk frames = [6837760,6838272) | 20260508 08:56:07.457 | INFO | MemInfo: { | 20260508 08:56:07.457 | INFO | "gpu" : " 17 GB", | 20260508 08:56:07.457 | INFO | "huge" : "", | 20260508 08:56:07.457 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:07.457 | INFO | "malloc" : " 14 GB", | 20260508 08:56:07.457 | INFO | "pinned" : " 29 GB" | 20260508 08:56:07.457 | INFO | } | 20260508 08:56:07.457 | INFO | Perf report for the last 5.14573 seconds: | 20260508 08:56:07.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:07.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | ANALYSIS : 67.0%, 2.9, 103.7ms, 0 | 20260508 08:56:07.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:56:07.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:07.457 | INFO | Uploaded 36GiB to GPU in 2.60119s (13.8398GiB/s) | 20260508 08:56:07.457 | INFO | Downloaded 34GiB to GPU in 2.43102s (13.9859GiB/s) | 20260508 08:56:12.599 | INFO | Analyzing chunk frames = [6838272,6838784) | 20260508 08:56:12.599 | INFO | MemInfo: { | 20260508 08:56:12.599 | INFO | "gpu" : " 17 GB", | 20260508 08:56:12.599 | INFO | "huge" : "", | 20260508 08:56:12.599 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:12.599 | INFO | "malloc" : " 14 GB", | 20260508 08:56:12.599 | INFO | "pinned" : " 29 GB" | 20260508 08:56:12.599 | INFO | } | 20260508 08:56:12.599 | INFO | Perf report for the last 5.14215 seconds: | 20260508 08:56:12.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:12.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.599 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 08:56:12.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.599 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 08:56:12.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:12.600 | INFO | Uploaded 36GiB to GPU in 2.42109s (14.8694GiB/s) | 20260508 08:56:12.600 | INFO | Downloaded 34GiB to GPU in 2.51474s (13.5203GiB/s) | 20260508 08:56:17.732 | INFO | Analyzing chunk frames = [6838784,6839296) | 20260508 08:56:17.732 | INFO | MemInfo: { | 20260508 08:56:17.732 | INFO | "gpu" : " 17 GB", | 20260508 08:56:17.732 | INFO | "huge" : "", | 20260508 08:56:17.732 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:17.732 | INFO | "malloc" : " 14 GB", | 20260508 08:56:17.732 | INFO | "pinned" : " 29 GB" | 20260508 08:56:17.732 | INFO | } | 20260508 08:56:17.732 | INFO | Perf report for the last 5.13342 seconds: | 20260508 08:56:17.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:17.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.733 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:56:17.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.733 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 08:56:17.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:17.733 | INFO | Uploaded 36GiB to GPU in 2.37802s (15.1387GiB/s) | 20260508 08:56:17.733 | INFO | Downloaded 34GiB to GPU in 2.42569s (14.0166GiB/s) | 20260508 08:56:22.864 | INFO | Analyzing chunk frames = [6839296,6839808) | 20260508 08:56:22.864 | INFO | MemInfo: { | 20260508 08:56:22.864 | INFO | "gpu" : " 17 GB", | 20260508 08:56:22.864 | INFO | "huge" : "", | 20260508 08:56:22.864 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:22.864 | INFO | "malloc" : " 14 GB", | 20260508 08:56:22.864 | INFO | "pinned" : " 29 GB" | 20260508 08:56:22.864 | INFO | } | 20260508 08:56:22.864 | INFO | Perf report for the last 5.1317 seconds: | 20260508 08:56:22.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:22.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.864 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:56:22.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.864 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:56:22.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:22.865 | INFO | Uploaded 36GiB to GPU in 2.41625s (14.8991GiB/s) | 20260508 08:56:22.865 | INFO | Downloaded 34GiB to GPU in 2.43684s (13.9525GiB/s) | 20260508 08:56:28.009 | INFO | Analyzing chunk frames = [6839808,6840320) | 20260508 08:56:28.009 | INFO | MemInfo: { | 20260508 08:56:28.009 | INFO | "gpu" : " 17 GB", | 20260508 08:56:28.009 | INFO | "huge" : "", | 20260508 08:56:28.009 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:28.009 | INFO | "malloc" : " 14 GB", | 20260508 08:56:28.009 | INFO | "pinned" : " 29 GB" | 20260508 08:56:28.009 | INFO | } | 20260508 08:56:28.009 | INFO | Perf report for the last 5.14511 seconds: | 20260508 08:56:28.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:28.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.009 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 08:56:28.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.009 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:56:28.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:28.010 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:56:28.010 | INFO | Total mem usage: 16.7 GiB | 20260508 08:56:28.010 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:56:28.010 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 08:56:28.010 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:56:28.010 | INFO | High Water Breakdown: | 20260508 08:56:28.010 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:56:28.010 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:56:28.010 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:56:28.010 | INFO | ------------------------------------------ | 20260508 08:56:28.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:56:28.010 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:56:28.010 | INFO | Total mem usage: 26.5 GiB | 20260508 08:56:28.010 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:56:28.010 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 08:56:28.010 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 08:56:28.010 | INFO | High Water Breakdown: | 20260508 08:56:28.010 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:56:28.010 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:56:28.010 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:56:28.010 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:56:28.010 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:56:28.010 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:56:28.010 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:56:28.010 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:56:28.010 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:56:28.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:56:28.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:56:28.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:56:28.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:56:28.010 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:56:28.010 | INFO | ------------------------------------------ | 20260508 08:56:28.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:56:28.010 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:56:28.010 | INFO | Total mem usage: 27.8 GiB | 20260508 08:56:28.010 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:56:28.010 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:56:28.010 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:56:28.010 | INFO | High Water Breakdown: | 20260508 08:56:28.010 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:56:28.010 | INFO | ------------------------------------------ | 20260508 08:56:28.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:56:28.010 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:56:28.010 | INFO | Total mem usage: 15.7 GiB | 20260508 08:56:28.010 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:56:28.010 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:56:28.010 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:56:28.010 | INFO | High Water Breakdown: | 20260508 08:56:28.010 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:56:28.010 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:56:28.010 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:56:28.010 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:56:28.010 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:56:28.010 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:56:28.010 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:56:28.010 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:56:28.010 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:56:28.010 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:56:28.010 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:56:28.010 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:56:28.010 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:56:28.010 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:56:28.010 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:56:28.010 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:56:28.010 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:56:28.010 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:56:28.010 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:56:28.010 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:56:28.010 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:56:28.010 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:56:28.010 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:56:28.010 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:56:28.010 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:56:28.010 | INFO | ------------------------------------------ | 20260508 08:56:28.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:56:28.010 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:56:28.010 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:56:28.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:56:28.010 | INFO | Uploaded 36GiB to GPU in 2.48865s (14.4657GiB/s) | 20260508 08:56:28.010 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 7.29e-01 | 20260508 08:56:28.011 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.04e-01, 1.18e-02 / 3.47e-03 | 20260508 08:56:28.011 | INFO | PerfReport: 12.12%, Baseline -- 3.95e+00 / 4.46e+00, 5.91e-01 / 2.45e-01 | 20260508 08:56:28.011 | INFO | Downloaded 34GiB to GPU in 2.45131s (13.8701GiB/s) | 20260508 08:56:28.011 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.43e+00, 2.08e-01 / 1.20e-01 | 20260508 08:56:28.011 | INFO | PerfReport: 15.97%, DME -- 4.17e+01 / 4.50e+01, 3.65e+00 / 9.59e-01 | 20260508 08:56:28.011 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.30e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.35e+00 / 2.49e+00, 4.08e-01 / 2.86e-01 | 20260508 08:56:28.011 | INFO | PerfReport: 3.07%, Metrics -- 1.00e+00 / 1.13e+00, 2.45e-01 / 1.01e-01 | 20260508 08:56:28.011 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.33e-01, 4.81e-02 / 2.29e-02 | 20260508 08:56:28.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:56:28.011 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:56:28.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:56:28.011 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 24.59%, Upload -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 24.41%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 30.84%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 2.42%, LockUpload -- 2.56e+00 / 0.00e+00, 5.87e+00 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 2.42%, LockDownload -- 2.56e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 15.31%, LockCompute -- 1.62e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 08:56:28.011 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260508 08:56:28.108 | INFO | Starting a round of Baz IO | 20260508 08:56:31.530 | INFO | Populating RTMetrics report | 20260508 08:56:31.530 | INFO | RTMetrics report populated | 20260508 08:56:31.530 | INFO | Opening tmp RTMetrics json file | 20260508 08:56:31.536 | INFO | Serializing RTMetrics to json | 20260508 08:56:31.538 | INFO | Serialization done, swapping tmp file now... | 20260508 08:56:31.539 | INFO | RTMetrics done | 20260508 08:56:31.598 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.49 seconds | 20260508 08:56:31.598 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.64% | 20260508 08:56:31.598 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.274448% / 7.94729e-06% / 7.15256e-05% | 20260508 08:56:31.598 | INFO | Baz IO overall: | 20260508 08:56:31.598 | INFO | BytesWritten : 2993599434599 | 20260508 08:56:31.598 | INFO | NumEvents : 1440658088494 | 20260508 08:56:33.153 | INFO | Analyzing chunk frames = [6840320,6840832) | 20260508 08:56:33.153 | INFO | MemInfo: { | 20260508 08:56:33.153 | INFO | "gpu" : " 17 GB", | 20260508 08:56:33.153 | INFO | "huge" : "", | 20260508 08:56:33.153 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:33.153 | INFO | "malloc" : " 14 GB", | 20260508 08:56:33.153 | INFO | "pinned" : " 29 GB" | 20260508 08:56:33.153 | INFO | } | 20260508 08:56:33.153 | INFO | Perf report for the last 5.14396 seconds: | 20260508 08:56:33.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:33.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:33.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:33.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:33.153 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 08:56:33.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:33.153 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:56:33.153 | INFO | PRE_HQ : 65.3%, 1.6, 54.6ms, 0 | 20260508 08:56:33.153 | INFO | BAZWRITER : 67.2%, 2.5, 88.0ms, 0 | 20260508 08:56:33.153 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:56:33.154 | INFO | Uploaded 36GiB to GPU in 2.57803s (13.9642GiB/s) | 20260508 08:56:33.154 | INFO | Downloaded 38GiB to GPU in 2.70555s (14.0452GiB/s) | 20260508 08:56:38.291 | INFO | Analyzing chunk frames = [6840832,6841344) | 20260508 08:56:38.291 | INFO | MemInfo: { | 20260508 08:56:38.291 | INFO | "gpu" : " 17 GB", | 20260508 08:56:38.291 | INFO | "huge" : "", | 20260508 08:56:38.291 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:38.291 | INFO | "malloc" : " 14 GB", | 20260508 08:56:38.291 | INFO | "pinned" : " 29 GB" | 20260508 08:56:38.291 | INFO | } | 20260508 08:56:38.291 | INFO | Perf report for the last 5.13756 seconds: | 20260508 08:56:38.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:38.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 08:56:38.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:56:38.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:38.291 | INFO | Uploaded 36GiB to GPU in 2.50819s (14.353GiB/s) | 20260508 08:56:38.291 | INFO | Downloaded 34GiB to GPU in 2.43233s (13.9784GiB/s) | 20260508 08:56:43.432 | INFO | Analyzing chunk frames = [6841344,6841856) | 20260508 08:56:43.432 | INFO | MemInfo: { | 20260508 08:56:43.432 | INFO | "gpu" : " 17 GB", | 20260508 08:56:43.432 | INFO | "huge" : "", | 20260508 08:56:43.432 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:43.432 | INFO | "malloc" : " 14 GB", | 20260508 08:56:43.432 | INFO | "pinned" : " 29 GB" | 20260508 08:56:43.432 | INFO | } | 20260508 08:56:43.432 | INFO | Perf report for the last 5.14147 seconds: | 20260508 08:56:43.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:43.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.432 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 08:56:43.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.432 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:56:43.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:43.433 | INFO | Uploaded 36GiB to GPU in 2.49161s (14.4485GiB/s) | 20260508 08:56:43.433 | INFO | Downloaded 34GiB to GPU in 2.43866s (13.9421GiB/s) | 20260508 08:56:48.569 | INFO | Analyzing chunk frames = [6841856,6842368) | 20260508 08:56:48.569 | INFO | MemInfo: { | 20260508 08:56:48.569 | INFO | "gpu" : " 17 GB", | 20260508 08:56:48.569 | INFO | "huge" : "", | 20260508 08:56:48.569 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:48.569 | INFO | "malloc" : " 14 GB", | 20260508 08:56:48.569 | INFO | "pinned" : " 29 GB" | 20260508 08:56:48.569 | INFO | } | 20260508 08:56:48.569 | INFO | Perf report for the last 5.13657 seconds: | 20260508 08:56:48.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:48.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 08:56:48.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:56:48.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:48.569 | INFO | Uploaded 36GiB to GPU in 2.4745s (14.5484GiB/s) | 20260508 08:56:48.570 | INFO | Downloaded 34GiB to GPU in 2.43879s (13.9413GiB/s) | 20260508 08:56:53.703 | INFO | Analyzing chunk frames = [6842368,6842880) | 20260508 08:56:53.703 | INFO | MemInfo: { | 20260508 08:56:53.703 | INFO | "gpu" : " 17 GB", | 20260508 08:56:53.703 | INFO | "huge" : "", | 20260508 08:56:53.703 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:53.703 | INFO | "malloc" : " 14 GB", | 20260508 08:56:53.703 | INFO | "pinned" : " 29 GB" | 20260508 08:56:53.703 | INFO | } | 20260508 08:56:53.703 | INFO | Perf report for the last 5.13449 seconds: | 20260508 08:56:53.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:53.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.703 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 08:56:53.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.703 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260508 08:56:53.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:53.704 | INFO | Uploaded 36GiB to GPU in 2.38096s (15.12GiB/s) | 20260508 08:56:53.704 | INFO | Downloaded 34GiB to GPU in 2.51341s (13.5274GiB/s) | 20260508 08:56:58.847 | INFO | Analyzing chunk frames = [6842880,6843392) | 20260508 08:56:58.847 | INFO | MemInfo: { | 20260508 08:56:58.847 | INFO | "gpu" : " 17 GB", | 20260508 08:56:58.847 | INFO | "huge" : "", | 20260508 08:56:58.847 | INFO | "huge_shared" : " 29 GB", | 20260508 08:56:58.847 | INFO | "malloc" : " 14 GB", | 20260508 08:56:58.847 | INFO | "pinned" : " 29 GB" | 20260508 08:56:58.847 | INFO | } | 20260508 08:56:58.847 | INFO | Perf report for the last 5.14374 seconds: | 20260508 08:56:58.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:56:58.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 08:56:58.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 08:56:58.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:56:58.847 | INFO | Uploaded 36GiB to GPU in 2.43217s (14.8016GiB/s) | 20260508 08:56:58.847 | INFO | Downloaded 34GiB to GPU in 2.43411s (13.9681GiB/s) | 20260508 08:57:03.990 | INFO | Analyzing chunk frames = [6843392,6843904) | 20260508 08:57:03.990 | INFO | MemInfo: { | 20260508 08:57:03.990 | INFO | "gpu" : " 17 GB", | 20260508 08:57:03.990 | INFO | "huge" : "", | 20260508 08:57:03.990 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:03.990 | INFO | "malloc" : " 14 GB", | 20260508 08:57:03.990 | INFO | "pinned" : " 29 GB" | 20260508 08:57:03.990 | INFO | } | 20260508 08:57:03.990 | INFO | Perf report for the last 5.14276 seconds: | 20260508 08:57:03.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:03.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:57:03.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:57:03.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:03.990 | INFO | Uploaded 36GiB to GPU in 2.44984s (14.6948GiB/s) | 20260508 08:57:03.990 | INFO | Downloaded 34GiB to GPU in 2.43161s (13.9825GiB/s) | 20260508 08:57:09.123 | INFO | Analyzing chunk frames = [6843904,6844416) | 20260508 08:57:09.123 | INFO | MemInfo: { | 20260508 08:57:09.123 | INFO | "gpu" : " 17 GB", | 20260508 08:57:09.123 | INFO | "huge" : "", | 20260508 08:57:09.124 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:09.124 | INFO | "malloc" : " 14 GB", | 20260508 08:57:09.124 | INFO | "pinned" : " 29 GB" | 20260508 08:57:09.124 | INFO | } | 20260508 08:57:09.124 | INFO | Perf report for the last 5.13373 seconds: | 20260508 08:57:09.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:09.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 08:57:09.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:57:09.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:09.124 | INFO | Uploaded 36GiB to GPU in 2.459s (14.6401GiB/s) | 20260508 08:57:09.124 | INFO | Downloaded 34GiB to GPU in 2.43569s (13.9591GiB/s) | 20260508 08:57:09.221 | INFO | Starting a round of Baz IO | 20260508 08:57:12.629 | INFO | Populating RTMetrics report | 20260508 08:57:12.629 | INFO | RTMetrics report populated | 20260508 08:57:12.629 | INFO | Opening tmp RTMetrics json file | 20260508 08:57:12.639 | INFO | Serializing RTMetrics to json | 20260508 08:57:12.641 | INFO | Serialization done, swapping tmp file now... | 20260508 08:57:12.641 | INFO | RTMetrics done | 20260508 08:57:12.701 | INFO | Finished a round of Baz IO: Wrote 1.491 GiB in 3.48 seconds | 20260508 08:57:12.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.64% | 20260508 08:57:12.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272679% / 3.97364e-06% / 4.37101e-05% | 20260508 08:57:12.701 | INFO | Baz IO overall: | 20260508 08:57:12.701 | INFO | BytesWritten : 2995200690078 | 20260508 08:57:12.701 | INFO | NumEvents : 1441415800148 | 20260508 08:57:14.268 | INFO | Analyzing chunk frames = [6844416,6844928) | 20260508 08:57:14.268 | INFO | MemInfo: { | 20260508 08:57:14.268 | INFO | "gpu" : " 17 GB", | 20260508 08:57:14.268 | INFO | "huge" : "", | 20260508 08:57:14.268 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:14.268 | INFO | "malloc" : " 14 GB", | 20260508 08:57:14.268 | INFO | "pinned" : " 29 GB" | 20260508 08:57:14.268 | INFO | } | 20260508 08:57:14.268 | INFO | Perf report for the last 5.14427 seconds: | 20260508 08:57:14.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:14.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:14.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:14.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:14.268 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 08:57:14.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:14.268 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 08:57:14.268 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 08:57:14.268 | INFO | BAZWRITER : 67.0%, 2.6, 88.7ms, 0 | 20260508 08:57:14.268 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:57:14.269 | INFO | Uploaded 36GiB to GPU in 2.58975s (13.901GiB/s) | 20260508 08:57:14.269 | INFO | Downloaded 38GiB to GPU in 2.71213s (14.0111GiB/s) | 20260508 08:57:19.410 | INFO | Analyzing chunk frames = [6844928,6845440) | 20260508 08:57:19.410 | INFO | MemInfo: { | 20260508 08:57:19.410 | INFO | "gpu" : " 17 GB", | 20260508 08:57:19.410 | INFO | "huge" : "", | 20260508 08:57:19.410 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:19.410 | INFO | "malloc" : " 14 GB", | 20260508 08:57:19.410 | INFO | "pinned" : " 29 GB" | 20260508 08:57:19.410 | INFO | } | 20260508 08:57:19.410 | INFO | Perf report for the last 5.14199 seconds: | 20260508 08:57:19.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:19.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.410 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 08:57:19.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.410 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:57:19.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:19.411 | INFO | Uploaded 36GiB to GPU in 2.53537s (14.1991GiB/s) | 20260508 08:57:19.411 | INFO | Downloaded 34GiB to GPU in 2.44908s (13.8828GiB/s) | 20260508 08:57:24.544 | INFO | Analyzing chunk frames = [6845440,6845952) | 20260508 08:57:24.544 | INFO | MemInfo: { | 20260508 08:57:24.544 | INFO | "gpu" : " 17 GB", | 20260508 08:57:24.544 | INFO | "huge" : "", | 20260508 08:57:24.544 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:24.545 | INFO | "malloc" : " 14 GB", | 20260508 08:57:24.545 | INFO | "pinned" : " 29 GB" | 20260508 08:57:24.545 | INFO | } | 20260508 08:57:24.545 | INFO | Perf report for the last 5.13466 seconds: | 20260508 08:57:24.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:24.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 08:57:24.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 08:57:24.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:24.545 | INFO | Uploaded 36GiB to GPU in 2.49673s (14.4189GiB/s) | 20260508 08:57:24.545 | INFO | Downloaded 34GiB to GPU in 2.42787s (14.0041GiB/s) | 20260508 08:57:29.683 | INFO | Analyzing chunk frames = [6845952,6846464) | 20260508 08:57:29.683 | INFO | MemInfo: { | 20260508 08:57:29.683 | INFO | "gpu" : " 17 GB", | 20260508 08:57:29.683 | INFO | "huge" : "", | 20260508 08:57:29.683 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:29.683 | INFO | "malloc" : " 14 GB", | 20260508 08:57:29.683 | INFO | "pinned" : " 29 GB" | 20260508 08:57:29.683 | INFO | } | 20260508 08:57:29.683 | INFO | Perf report for the last 5.13861 seconds: | 20260508 08:57:29.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:29.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.683 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 08:57:29.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.683 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:57:29.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:29.684 | INFO | Uploaded 36GiB to GPU in 2.51533s (14.3122GiB/s) | 20260508 08:57:29.684 | INFO | Downloaded 34GiB to GPU in 2.43812s (13.9452GiB/s) | 20260508 08:57:34.818 | INFO | Analyzing chunk frames = [6846464,6846976) | 20260508 08:57:34.818 | INFO | MemInfo: { | 20260508 08:57:34.818 | INFO | "gpu" : " 17 GB", | 20260508 08:57:34.818 | INFO | "huge" : "", | 20260508 08:57:34.818 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:34.818 | INFO | "malloc" : " 14 GB", | 20260508 08:57:34.818 | INFO | "pinned" : " 29 GB" | 20260508 08:57:34.818 | INFO | } | 20260508 08:57:34.818 | INFO | Perf report for the last 5.13459 seconds: | 20260508 08:57:34.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:34.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 08:57:34.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260508 08:57:34.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:34.818 | INFO | Uploaded 36GiB to GPU in 2.43035s (14.8127GiB/s) | 20260508 08:57:34.818 | INFO | Downloaded 34GiB to GPU in 2.52123s (13.4855GiB/s) | 20260508 08:57:39.963 | INFO | Analyzing chunk frames = [6846976,6847488) | 20260508 08:57:39.963 | INFO | MemInfo: { | 20260508 08:57:39.963 | INFO | "gpu" : " 17 GB", | 20260508 08:57:39.963 | INFO | "huge" : "", | 20260508 08:57:39.963 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:39.963 | INFO | "malloc" : " 14 GB", | 20260508 08:57:39.963 | INFO | "pinned" : " 29 GB" | 20260508 08:57:39.963 | INFO | } | 20260508 08:57:39.964 | INFO | Perf report for the last 5.14565 seconds: | 20260508 08:57:39.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:39.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 08:57:39.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 08:57:39.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:39.964 | INFO | Uploaded 36GiB to GPU in 2.38576s (15.0896GiB/s) | 20260508 08:57:39.964 | INFO | Downloaded 34GiB to GPU in 2.43698s (13.9517GiB/s) | 20260508 08:57:45.106 | INFO | Analyzing chunk frames = [6847488,6848000) | 20260508 08:57:45.106 | INFO | MemInfo: { | 20260508 08:57:45.106 | INFO | "gpu" : " 17 GB", | 20260508 08:57:45.106 | INFO | "huge" : "", | 20260508 08:57:45.106 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:45.106 | INFO | "malloc" : " 14 GB", | 20260508 08:57:45.106 | INFO | "pinned" : " 29 GB" | 20260508 08:57:45.106 | INFO | } | 20260508 08:57:45.106 | INFO | Perf report for the last 5.14259 seconds: | 20260508 08:57:45.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:45.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 08:57:45.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:57:45.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:45.106 | INFO | Uploaded 36GiB to GPU in 2.42676s (14.8346GiB/s) | 20260508 08:57:45.107 | INFO | Downloaded 34GiB to GPU in 2.44708s (13.8941GiB/s) | 20260508 08:57:50.246 | INFO | Analyzing chunk frames = [6848000,6848512) | 20260508 08:57:50.246 | INFO | MemInfo: { | 20260508 08:57:50.246 | INFO | "gpu" : " 17 GB", | 20260508 08:57:50.246 | INFO | "huge" : "", | 20260508 08:57:50.246 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:50.246 | INFO | "malloc" : " 14 GB", | 20260508 08:57:50.246 | INFO | "pinned" : " 29 GB" | 20260508 08:57:50.246 | INFO | } | 20260508 08:57:50.246 | INFO | Perf report for the last 5.13986 seconds: | 20260508 08:57:50.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:50.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:57:50.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:57:50.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:50.246 | INFO | Uploaded 36GiB to GPU in 2.42461s (14.8477GiB/s) | 20260508 08:57:50.246 | INFO | Downloaded 34GiB to GPU in 2.43906s (13.9398GiB/s) | 20260508 08:57:50.246 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:57:50.246 | INFO | Total mem usage: 16.7 GiB | 20260508 08:57:50.246 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:57:50.246 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:57:50.246 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:57:50.246 | INFO | High Water Breakdown: | 20260508 08:57:50.246 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:57:50.246 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:57:50.246 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:57:50.246 | INFO | ------------------------------------------ | 20260508 08:57:50.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:57:50.246 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:57:50.246 | INFO | Total mem usage: 26.5 GiB | 20260508 08:57:50.246 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:57:50.246 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:57:50.246 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:57:50.246 | INFO | High Water Breakdown: | 20260508 08:57:50.246 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:57:50.246 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:57:50.246 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:57:50.246 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:57:50.247 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:57:50.247 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:57:50.247 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:57:50.247 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:57:50.247 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:57:50.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:57:50.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:57:50.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:57:50.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:57:50.247 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:57:50.247 | INFO | ------------------------------------------ | 20260508 08:57:50.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:57:50.247 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:57:50.247 | INFO | Total mem usage: 27.8 GiB | 20260508 08:57:50.247 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:57:50.247 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:57:50.247 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:57:50.247 | INFO | High Water Breakdown: | 20260508 08:57:50.247 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:57:50.247 | INFO | ------------------------------------------ | 20260508 08:57:50.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:57:50.247 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:57:50.247 | INFO | Total mem usage: 15.7 GiB | 20260508 08:57:50.247 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:57:50.247 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:57:50.247 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:57:50.247 | INFO | High Water Breakdown: | 20260508 08:57:50.247 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:57:50.247 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:57:50.247 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:57:50.247 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:57:50.247 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:57:50.247 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:57:50.247 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:57:50.247 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:57:50.247 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:57:50.247 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:57:50.247 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:57:50.247 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:57:50.247 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:57:50.247 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:57:50.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:57:50.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:57:50.247 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:57:50.247 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:57:50.247 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:57:50.247 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:57:50.247 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:57:50.247 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:57:50.247 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:57:50.247 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:57:50.247 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:57:50.247 | INFO | ------------------------------------------ | 20260508 08:57:50.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:57:50.247 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:57:50.247 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:57:50.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:57:50.247 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.90e-01 | 20260508 08:57:50.247 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.22e-02 / 5.81e-03 | 20260508 08:57:50.247 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.40e+00, 5.95e-01 / 2.66e-01 | 20260508 08:57:50.247 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.41e+00, 2.08e-01 / 1.41e-01 | 20260508 08:57:50.247 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.50e+01, 3.74e+00 / 1.29e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.83e+00 / 1.51e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.42e+00, 4.06e-01 / 2.92e-01 | 20260508 08:57:50.247 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.15e+00, 2.50e-01 / 1.34e-01 | 20260508 08:57:50.247 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.40e-01, 4.84e-02 / 2.80e-02 | 20260508 08:57:50.247 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:57:50.247 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:57:50.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:57:50.247 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 24.41%, Upload -- 2.58e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 24.49%, Download -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 31.03%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 2.43%, LockDownload -- 2.57e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 08:57:50.247 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 4.02e-02 / 0.00e+00 | 20260508 08:57:50.343 | INFO | Starting a round of Baz IO | 20260508 08:57:53.752 | INFO | Populating RTMetrics report | 20260508 08:57:53.752 | INFO | RTMetrics report populated | 20260508 08:57:53.752 | INFO | Opening tmp RTMetrics json file | 20260508 08:57:53.762 | INFO | Serializing RTMetrics to json | 20260508 08:57:53.763 | INFO | Serialization done, swapping tmp file now... | 20260508 08:57:53.764 | INFO | RTMetrics done | 20260508 08:57:53.826 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.483 seconds | 20260508 08:57:53.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.64% | 20260508 08:57:53.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272846% / 7.94729e-06% / 7.15256e-05% | 20260508 08:57:53.827 | INFO | Baz IO overall: | 20260508 08:57:53.827 | INFO | BytesWritten : 2996802785053 | 20260508 08:57:53.827 | INFO | NumEvents : 1442174053791 | 20260508 08:57:55.376 | INFO | Analyzing chunk frames = [6848512,6849024) | 20260508 08:57:55.376 | INFO | MemInfo: { | 20260508 08:57:55.376 | INFO | "gpu" : " 17 GB", | 20260508 08:57:55.376 | INFO | "huge" : "", | 20260508 08:57:55.376 | INFO | "huge_shared" : " 29 GB", | 20260508 08:57:55.376 | INFO | "malloc" : " 14 GB", | 20260508 08:57:55.376 | INFO | "pinned" : " 29 GB" | 20260508 08:57:55.376 | INFO | } | 20260508 08:57:55.376 | INFO | Perf report for the last 5.13035 seconds: | 20260508 08:57:55.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:57:55.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:55.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:55.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:55.376 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 08:57:55.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:57:55.376 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 08:57:55.376 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 08:57:55.376 | INFO | BAZWRITER : 67.2%, 2.5, 87.8ms, 0 | 20260508 08:57:55.377 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:57:55.377 | INFO | Uploaded 36GiB to GPU in 2.57697s (13.9699GiB/s) | 20260508 08:57:55.377 | INFO | Downloaded 38GiB to GPU in 2.72716s (13.9339GiB/s) | 20260508 08:58:00.519 | INFO | Analyzing chunk frames = [6849024,6849536) | 20260508 08:58:00.519 | INFO | MemInfo: { | 20260508 08:58:00.519 | INFO | "gpu" : " 17 GB", | 20260508 08:58:00.519 | INFO | "huge" : "", | 20260508 08:58:00.519 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:00.519 | INFO | "malloc" : " 14 GB", | 20260508 08:58:00.519 | INFO | "pinned" : " 29 GB" | 20260508 08:58:00.520 | INFO | } | 20260508 08:58:00.520 | INFO | Perf report for the last 5.14322 seconds: | 20260508 08:58:00.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:00.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 08:58:00.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260508 08:58:00.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:00.520 | INFO | Uploaded 36GiB to GPU in 2.49401s (14.4346GiB/s) | 20260508 08:58:00.520 | INFO | Downloaded 34GiB to GPU in 2.45692s (13.8385GiB/s) | 20260508 08:58:05.656 | INFO | Analyzing chunk frames = [6849536,6850048) | 20260508 08:58:05.656 | INFO | MemInfo: { | 20260508 08:58:05.656 | INFO | "gpu" : " 17 GB", | 20260508 08:58:05.656 | INFO | "huge" : "", | 20260508 08:58:05.656 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:05.656 | INFO | "malloc" : " 14 GB", | 20260508 08:58:05.656 | INFO | "pinned" : " 29 GB" | 20260508 08:58:05.656 | INFO | } | 20260508 08:58:05.656 | INFO | Perf report for the last 5.13679 seconds: | 20260508 08:58:05.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:05.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.656 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 08:58:05.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.657 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 08:58:05.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:05.657 | INFO | Uploaded 36GiB to GPU in 2.45054s (14.6906GiB/s) | 20260508 08:58:05.657 | INFO | Downloaded 34GiB to GPU in 2.45452s (13.852GiB/s) | 20260508 08:58:10.799 | INFO | Analyzing chunk frames = [6850048,6850560) | 20260508 08:58:10.799 | INFO | MemInfo: { | 20260508 08:58:10.799 | INFO | "gpu" : " 17 GB", | 20260508 08:58:10.799 | INFO | "huge" : "", | 20260508 08:58:10.799 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:10.800 | INFO | "malloc" : " 14 GB", | 20260508 08:58:10.800 | INFO | "pinned" : " 29 GB" | 20260508 08:58:10.800 | INFO | } | 20260508 08:58:10.800 | INFO | Perf report for the last 5.14361 seconds: | 20260508 08:58:10.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:10.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.800 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 08:58:10.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.800 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 08:58:10.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:10.801 | INFO | Uploaded 36GiB to GPU in 2.47063s (14.5712GiB/s) | 20260508 08:58:10.801 | INFO | Downloaded 34GiB to GPU in 2.45009s (13.877GiB/s) | 20260508 08:58:15.937 | INFO | Analyzing chunk frames = [6850560,6851072) | 20260508 08:58:15.937 | INFO | MemInfo: { | 20260508 08:58:15.937 | INFO | "gpu" : " 17 GB", | 20260508 08:58:15.937 | INFO | "huge" : "", | 20260508 08:58:15.937 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:15.937 | INFO | "malloc" : " 14 GB", | 20260508 08:58:15.937 | INFO | "pinned" : " 29 GB" | 20260508 08:58:15.937 | INFO | } | 20260508 08:58:15.937 | INFO | Perf report for the last 5.13719 seconds: | 20260508 08:58:15.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:15.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 08:58:15.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 08:58:15.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:15.937 | INFO | Uploaded 36GiB to GPU in 2.39257s (15.0466GiB/s) | 20260508 08:58:15.938 | INFO | Downloaded 34GiB to GPU in 2.53752s (13.3989GiB/s) | 20260508 08:58:21.072 | INFO | Analyzing chunk frames = [6851072,6851584) | 20260508 08:58:21.072 | INFO | MemInfo: { | 20260508 08:58:21.072 | INFO | "gpu" : " 17 GB", | 20260508 08:58:21.072 | INFO | "huge" : "", | 20260508 08:58:21.072 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:21.072 | INFO | "malloc" : " 14 GB", | 20260508 08:58:21.072 | INFO | "pinned" : " 29 GB" | 20260508 08:58:21.072 | INFO | } | 20260508 08:58:21.072 | INFO | Perf report for the last 5.13516 seconds: | 20260508 08:58:21.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:21.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.072 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 08:58:21.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.073 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:58:21.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:21.073 | INFO | Uploaded 36GiB to GPU in 2.39038s (15.0604GiB/s) | 20260508 08:58:21.073 | INFO | Downloaded 34GiB to GPU in 2.45342s (13.8582GiB/s) | 20260508 08:58:26.218 | INFO | Analyzing chunk frames = [6851584,6852096) | 20260508 08:58:26.218 | INFO | MemInfo: { | 20260508 08:58:26.218 | INFO | "gpu" : " 17 GB", | 20260508 08:58:26.218 | INFO | "huge" : "", | 20260508 08:58:26.218 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:26.218 | INFO | "malloc" : " 14 GB", | 20260508 08:58:26.218 | INFO | "pinned" : " 29 GB" | 20260508 08:58:26.218 | INFO | } | 20260508 08:58:26.218 | INFO | Perf report for the last 5.14594 seconds: | 20260508 08:58:26.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:26.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.218 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 08:58:26.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.218 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:58:26.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:26.219 | INFO | Uploaded 36GiB to GPU in 2.39848s (15.0095GiB/s) | 20260508 08:58:26.219 | INFO | Downloaded 34GiB to GPU in 2.45469s (13.8511GiB/s) | 20260508 08:58:31.358 | INFO | Analyzing chunk frames = [6852096,6852608) | 20260508 08:58:31.358 | INFO | MemInfo: { | 20260508 08:58:31.358 | INFO | "gpu" : " 17 GB", | 20260508 08:58:31.358 | INFO | "huge" : "", | 20260508 08:58:31.358 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:31.358 | INFO | "malloc" : " 14 GB", | 20260508 08:58:31.358 | INFO | "pinned" : " 29 GB" | 20260508 08:58:31.358 | INFO | } | 20260508 08:58:31.359 | INFO | Perf report for the last 5.14019 seconds: | 20260508 08:58:31.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:31.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 08:58:31.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 08:58:31.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:31.359 | INFO | Uploaded 36GiB to GPU in 2.41623s (14.8993GiB/s) | 20260508 08:58:31.359 | INFO | Downloaded 34GiB to GPU in 2.45638s (13.8415GiB/s) | 20260508 08:58:31.456 | INFO | Starting a round of Baz IO | 20260508 08:58:34.877 | INFO | Populating RTMetrics report | 20260508 08:58:34.877 | INFO | RTMetrics report populated | 20260508 08:58:34.877 | INFO | Opening tmp RTMetrics json file | 20260508 08:58:34.886 | INFO | Serializing RTMetrics to json | 20260508 08:58:34.887 | INFO | Serialization done, swapping tmp file now... | 20260508 08:58:34.888 | INFO | RTMetrics done | 20260508 08:58:34.943 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.487 seconds | 20260508 08:58:34.943 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.64% | 20260508 08:58:34.943 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.273212% / 0% / 5.5631e-05% | 20260508 08:58:34.943 | INFO | Baz IO overall: | 20260508 08:58:34.943 | INFO | BytesWritten : 2998404677442 | 20260508 08:58:34.943 | INFO | NumEvents : 1442932336238 | 20260508 08:58:36.495 | INFO | Analyzing chunk frames = [6852608,6853120) | 20260508 08:58:36.495 | INFO | MemInfo: { | 20260508 08:58:36.495 | INFO | "gpu" : " 17 GB", | 20260508 08:58:36.495 | INFO | "huge" : "", | 20260508 08:58:36.495 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:36.495 | INFO | "malloc" : " 14 GB", | 20260508 08:58:36.495 | INFO | "pinned" : " 29 GB" | 20260508 08:58:36.495 | INFO | } | 20260508 08:58:36.495 | INFO | Perf report for the last 5.13648 seconds: | 20260508 08:58:36.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:36.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:36.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:36.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:36.495 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 08:58:36.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:36.495 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 08:58:36.495 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 08:58:36.495 | INFO | BAZWRITER : 67.2%, 2.6, 88.2ms, 0 | 20260508 08:58:36.495 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 08:58:36.495 | INFO | Uploaded 36GiB to GPU in 2.58922s (13.9038GiB/s) | 20260508 08:58:36.495 | INFO | Downloaded 38GiB to GPU in 2.70142s (14.0667GiB/s) | 20260508 08:58:41.628 | INFO | Analyzing chunk frames = [6853120,6853632) | 20260508 08:58:41.628 | INFO | MemInfo: { | 20260508 08:58:41.628 | INFO | "gpu" : " 17 GB", | 20260508 08:58:41.628 | INFO | "huge" : "", | 20260508 08:58:41.628 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:41.629 | INFO | "malloc" : " 14 GB", | 20260508 08:58:41.629 | INFO | "pinned" : " 29 GB" | 20260508 08:58:41.629 | INFO | } | 20260508 08:58:41.629 | INFO | Perf report for the last 5.13356 seconds: | 20260508 08:58:41.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:41.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 08:58:41.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 08:58:41.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:41.629 | INFO | Uploaded 36GiB to GPU in 2.50304s (14.3825GiB/s) | 20260508 08:58:41.629 | INFO | Downloaded 34GiB to GPU in 2.42372s (14.028GiB/s) | 20260508 08:58:46.778 | INFO | Analyzing chunk frames = [6853632,6854144) | 20260508 08:58:46.778 | INFO | MemInfo: { | 20260508 08:58:46.778 | INFO | "gpu" : " 17 GB", | 20260508 08:58:46.778 | INFO | "huge" : "", | 20260508 08:58:46.778 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:46.778 | INFO | "malloc" : " 14 GB", | 20260508 08:58:46.778 | INFO | "pinned" : " 29 GB" | 20260508 08:58:46.778 | INFO | } | 20260508 08:58:46.778 | INFO | Perf report for the last 5.14945 seconds: | 20260508 08:58:46.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:46.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 08:58:46.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 08:58:46.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:46.778 | INFO | Uploaded 36GiB to GPU in 2.47241s (14.5607GiB/s) | 20260508 08:58:46.779 | INFO | Downloaded 34GiB to GPU in 2.43525s (13.9616GiB/s) | 20260508 08:58:51.914 | INFO | Analyzing chunk frames = [6854144,6854656) | 20260508 08:58:51.914 | INFO | MemInfo: { | 20260508 08:58:51.914 | INFO | "gpu" : " 17 GB", | 20260508 08:58:51.914 | INFO | "huge" : "", | 20260508 08:58:51.915 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:51.915 | INFO | "malloc" : " 14 GB", | 20260508 08:58:51.915 | INFO | "pinned" : " 29 GB" | 20260508 08:58:51.915 | INFO | } | 20260508 08:58:51.915 | INFO | Perf report for the last 5.13667 seconds: | 20260508 08:58:51.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:51.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.915 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 08:58:51.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.915 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 08:58:51.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:51.916 | INFO | Uploaded 36GiB to GPU in 2.45694s (14.6524GiB/s) | 20260508 08:58:51.916 | INFO | Downloaded 34GiB to GPU in 2.41506s (14.0783GiB/s) | 20260508 08:58:57.054 | INFO | Analyzing chunk frames = [6854656,6855168) | 20260508 08:58:57.054 | INFO | MemInfo: { | 20260508 08:58:57.054 | INFO | "gpu" : " 17 GB", | 20260508 08:58:57.054 | INFO | "huge" : "", | 20260508 08:58:57.054 | INFO | "huge_shared" : " 29 GB", | 20260508 08:58:57.054 | INFO | "malloc" : " 14 GB", | 20260508 08:58:57.054 | INFO | "pinned" : " 29 GB" | 20260508 08:58:57.054 | INFO | } | 20260508 08:58:57.054 | INFO | Perf report for the last 5.13953 seconds: | 20260508 08:58:57.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:58:57.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.054 | INFO | ANALYSIS : 66.3%, 3.0, 102.3ms, 0 | 20260508 08:58:57.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.054 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260508 08:58:57.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:58:57.055 | INFO | Uploaded 36GiB to GPU in 2.28512s (15.7541GiB/s) | 20260508 08:58:57.055 | INFO | Downloaded 34GiB to GPU in 2.4967s (13.618GiB/s) | 20260508 08:59:02.191 | INFO | Analyzing chunk frames = [6855168,6855680) | 20260508 08:59:02.191 | INFO | MemInfo: { | 20260508 08:59:02.191 | INFO | "gpu" : " 17 GB", | 20260508 08:59:02.191 | INFO | "huge" : "", | 20260508 08:59:02.191 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:02.191 | INFO | "malloc" : " 14 GB", | 20260508 08:59:02.191 | INFO | "pinned" : " 29 GB" | 20260508 08:59:02.191 | INFO | } | 20260508 08:59:02.191 | INFO | Perf report for the last 5.13714 seconds: | 20260508 08:59:02.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:02.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.192 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 08:59:02.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.192 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 08:59:02.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:02.192 | INFO | Uploaded 36GiB to GPU in 2.40101s (14.9937GiB/s) | 20260508 08:59:02.192 | INFO | Downloaded 34GiB to GPU in 2.42318s (14.0311GiB/s) | 20260508 08:59:07.328 | INFO | Analyzing chunk frames = [6855680,6856192) | 20260508 08:59:07.328 | INFO | MemInfo: { | 20260508 08:59:07.328 | INFO | "gpu" : " 17 GB", | 20260508 08:59:07.328 | INFO | "huge" : "", | 20260508 08:59:07.328 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:07.328 | INFO | "malloc" : " 14 GB", | 20260508 08:59:07.328 | INFO | "pinned" : " 29 GB" | 20260508 08:59:07.328 | INFO | } | 20260508 08:59:07.328 | INFO | Perf report for the last 5.1369 seconds: | 20260508 08:59:07.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:07.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.328 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 08:59:07.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.328 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:59:07.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:07.329 | INFO | Uploaded 36GiB to GPU in 2.46018s (14.633GiB/s) | 20260508 08:59:07.329 | INFO | Downloaded 34GiB to GPU in 2.43584s (13.9582GiB/s) | 20260508 08:59:12.470 | INFO | Analyzing chunk frames = [6856192,6856704) | 20260508 08:59:12.470 | INFO | MemInfo: { | 20260508 08:59:12.470 | INFO | "gpu" : " 17 GB", | 20260508 08:59:12.470 | INFO | "huge" : "", | 20260508 08:59:12.470 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:12.470 | INFO | "malloc" : " 14 GB", | 20260508 08:59:12.470 | INFO | "pinned" : " 29 GB" | 20260508 08:59:12.470 | INFO | } | 20260508 08:59:12.470 | INFO | Perf report for the last 5.14199 seconds: | 20260508 08:59:12.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:12.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.470 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 08:59:12.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.470 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 08:59:12.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:12.471 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 08:59:12.471 | INFO | Total mem usage: 16.7 GiB | 20260508 08:59:12.471 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 08:59:12.471 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 08:59:12.471 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 08:59:12.471 | INFO | High Water Breakdown: | 20260508 08:59:12.471 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 08:59:12.471 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 08:59:12.471 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 08:59:12.471 | INFO | ------------------------------------------ | 20260508 08:59:12.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:59:12.471 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 08:59:12.471 | INFO | Total mem usage: 26.5 GiB | 20260508 08:59:12.471 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 08:59:12.471 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 08:59:12.471 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 08:59:12.471 | INFO | High Water Breakdown: | 20260508 08:59:12.471 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:59:12.471 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:59:12.471 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:59:12.471 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:59:12.471 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:59:12.471 | INFO | Uploaded 36GiB to GPU in 2.49686s (14.4181GiB/s) | 20260508 08:59:12.471 | INFO | Downloaded 34GiB to GPU in 2.44351s (13.9144GiB/s) | 20260508 08:59:12.471 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:59:12.471 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 08:59:12.471 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 08:59:12.471 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:59:12.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 08:59:12.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 08:59:12.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 08:59:12.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 08:59:12.471 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 08:59:12.471 | INFO | ------------------------------------------ | 20260508 08:59:12.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:59:12.471 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 08:59:12.471 | INFO | Total mem usage: 27.8 GiB | 20260508 08:59:12.471 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 08:59:12.471 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 08:59:12.471 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 08:59:12.471 | INFO | High Water Breakdown: | 20260508 08:59:12.471 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 08:59:12.471 | INFO | ------------------------------------------ | 20260508 08:59:12.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:59:12.471 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 08:59:12.471 | INFO | Total mem usage: 15.7 GiB | 20260508 08:59:12.471 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 08:59:12.471 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 08:59:12.471 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 08:59:12.471 | INFO | High Water Breakdown: | 20260508 08:59:12.471 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 08:59:12.471 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 08:59:12.471 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 08:59:12.471 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 08:59:12.471 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 08:59:12.471 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 08:59:12.471 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 08:59:12.471 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 08:59:12.471 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 08:59:12.471 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 08:59:12.471 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 08:59:12.471 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 08:59:12.471 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 08:59:12.471 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 08:59:12.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 08:59:12.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 08:59:12.471 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 08:59:12.471 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 08:59:12.471 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 08:59:12.471 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 08:59:12.471 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 08:59:12.471 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 08:59:12.471 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 08:59:12.471 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 08:59:12.471 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 08:59:12.471 | INFO | ------------------------------------------ | 20260508 08:59:12.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 08:59:12.471 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 08:59:12.471 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:59:12.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:59:12.471 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.37e+01 / 7.42e-01 | 20260508 08:59:12.472 | INFO | PerfReport: 0.30%, AggregateStats -- 9.90e-02 / 1.06e-01, 1.34e-02 / 6.54e-03 | 20260508 08:59:12.472 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.45e+00, 5.97e-01 / 2.14e-01 | 20260508 08:59:12.472 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.23e-01 | 20260508 08:59:12.472 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.52e+01, 3.61e+00 / 1.19e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.86e+00 / 1.39e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.07e-01 / 2.59e-01 | 20260508 08:59:12.472 | INFO | PerfReport: 3.05%, Metrics -- 1.00e+00 / 1.14e+00, 2.47e-01 / 1.34e-01 | 20260508 08:59:12.472 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.08e-01 / 3.43e-01, 4.97e-02 / 3.09e-02 | 20260508 08:59:12.472 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 08:59:12.472 | INFO | PerfReport: Gathered 1536 samples. | 20260508 08:59:12.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 08:59:12.472 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 24.23%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 24.55%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 31.09%, Compute -- 3.28e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 2.61%, LockDownload -- 2.75e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 08:59:12.472 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260508 08:59:12.568 | INFO | Starting a round of Baz IO | 20260508 08:59:15.990 | INFO | Populating RTMetrics report | 20260508 08:59:15.990 | INFO | RTMetrics report populated | 20260508 08:59:15.990 | INFO | Opening tmp RTMetrics json file | 20260508 08:59:16.000 | INFO | Serializing RTMetrics to json | 20260508 08:59:16.001 | INFO | Serialization done, swapping tmp file now... | 20260508 08:59:16.001 | INFO | RTMetrics done | 20260508 08:59:16.059 | INFO | Finished a round of Baz IO: Wrote 1.491 GiB in 3.49 seconds | 20260508 08:59:16.059 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.64% | 20260508 08:59:16.059 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.270955% / 0% / 4.76837e-05% | 20260508 08:59:16.059 | INFO | Baz IO overall: | 20260508 08:59:16.059 | INFO | BytesWritten : 3000005219821 | 20260508 08:59:16.059 | INFO | NumEvents : 1443689883808 | 20260508 08:59:17.611 | INFO | Analyzing chunk frames = [6856704,6857216) | 20260508 08:59:17.611 | INFO | MemInfo: { | 20260508 08:59:17.611 | INFO | "gpu" : " 17 GB", | 20260508 08:59:17.611 | INFO | "huge" : "", | 20260508 08:59:17.611 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:17.611 | INFO | "malloc" : " 14 GB", | 20260508 08:59:17.611 | INFO | "pinned" : " 29 GB" | 20260508 08:59:17.611 | INFO | } | 20260508 08:59:17.611 | INFO | Perf report for the last 5.14095 seconds: | 20260508 08:59:17.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:17.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:17.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:17.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:17.611 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 08:59:17.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:17.611 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:59:17.611 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 08:59:17.611 | INFO | BAZWRITER : 67.3%, 2.6, 89.0ms, 0 | 20260508 08:59:17.611 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:59:17.612 | INFO | Uploaded 36GiB to GPU in 2.6003s (13.8446GiB/s) | 20260508 08:59:17.612 | INFO | Downloaded 38GiB to GPU in 2.70422s (14.0521GiB/s) | 20260508 08:59:22.745 | INFO | Analyzing chunk frames = [6857216,6857728) | 20260508 08:59:22.745 | INFO | MemInfo: { | 20260508 08:59:22.745 | INFO | "gpu" : " 17 GB", | 20260508 08:59:22.745 | INFO | "huge" : "", | 20260508 08:59:22.746 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:22.746 | INFO | "malloc" : " 14 GB", | 20260508 08:59:22.746 | INFO | "pinned" : " 29 GB" | 20260508 08:59:22.746 | INFO | } | 20260508 08:59:22.746 | INFO | Perf report for the last 5.1344 seconds: | 20260508 08:59:22.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:22.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 08:59:22.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 08:59:22.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:22.746 | INFO | Uploaded 36GiB to GPU in 2.50288s (14.3834GiB/s) | 20260508 08:59:22.746 | INFO | Downloaded 34GiB to GPU in 2.46416s (13.7978GiB/s) | 20260508 08:59:27.885 | INFO | Analyzing chunk frames = [6857728,6858240) | 20260508 08:59:27.885 | INFO | MemInfo: { | 20260508 08:59:27.885 | INFO | "gpu" : " 17 GB", | 20260508 08:59:27.886 | INFO | "huge" : "", | 20260508 08:59:27.886 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:27.886 | INFO | "malloc" : " 14 GB", | 20260508 08:59:27.886 | INFO | "pinned" : " 29 GB" | 20260508 08:59:27.886 | INFO | } | 20260508 08:59:27.886 | INFO | Perf report for the last 5.13998 seconds: | 20260508 08:59:27.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:27.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 08:59:27.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 08:59:27.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:27.886 | INFO | Uploaded 36GiB to GPU in 2.48622s (14.4798GiB/s) | 20260508 08:59:27.886 | INFO | Downloaded 34GiB to GPU in 2.44027s (13.9329GiB/s) | 20260508 08:59:33.030 | INFO | Analyzing chunk frames = [6858240,6858752) | 20260508 08:59:33.030 | INFO | MemInfo: { | 20260508 08:59:33.030 | INFO | "gpu" : " 17 GB", | 20260508 08:59:33.030 | INFO | "huge" : "", | 20260508 08:59:33.030 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:33.030 | INFO | "malloc" : " 14 GB", | 20260508 08:59:33.030 | INFO | "pinned" : " 29 GB" | 20260508 08:59:33.030 | INFO | } | 20260508 08:59:33.030 | INFO | Perf report for the last 5.14448 seconds: | 20260508 08:59:33.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:33.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 08:59:33.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 08:59:33.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:33.030 | INFO | Uploaded 36GiB to GPU in 2.48394s (14.4931GiB/s) | 20260508 08:59:33.031 | INFO | Downloaded 34GiB to GPU in 2.42504s (14.0204GiB/s) | 20260508 08:59:38.170 | INFO | Analyzing chunk frames = [6858752,6859264) | 20260508 08:59:38.170 | INFO | MemInfo: { | 20260508 08:59:38.170 | INFO | "gpu" : " 17 GB", | 20260508 08:59:38.170 | INFO | "huge" : "", | 20260508 08:59:38.170 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:38.170 | INFO | "malloc" : " 14 GB", | 20260508 08:59:38.170 | INFO | "pinned" : " 29 GB" | 20260508 08:59:38.170 | INFO | } | 20260508 08:59:38.170 | INFO | Perf report for the last 5.13989 seconds: | 20260508 08:59:38.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:38.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.170 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 08:59:38.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.170 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 08:59:38.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:38.171 | INFO | Uploaded 36GiB to GPU in 2.3785s (15.1356GiB/s) | 20260508 08:59:38.171 | INFO | Downloaded 34GiB to GPU in 2.52599s (13.4601GiB/s) | 20260508 08:59:43.307 | INFO | Analyzing chunk frames = [6859264,6859776) | 20260508 08:59:43.307 | INFO | MemInfo: { | 20260508 08:59:43.307 | INFO | "gpu" : " 17 GB", | 20260508 08:59:43.307 | INFO | "huge" : "", | 20260508 08:59:43.307 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:43.307 | INFO | "malloc" : " 14 GB", | 20260508 08:59:43.307 | INFO | "pinned" : " 29 GB" | 20260508 08:59:43.307 | INFO | } | 20260508 08:59:43.307 | INFO | Perf report for the last 5.13677 seconds: | 20260508 08:59:43.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:43.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 08:59:43.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:59:43.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:43.307 | INFO | Uploaded 36GiB to GPU in 2.36084s (15.2488GiB/s) | 20260508 08:59:43.307 | INFO | Downloaded 34GiB to GPU in 2.43506s (13.9627GiB/s) | 20260508 08:59:48.447 | INFO | Analyzing chunk frames = [6859776,6860288) | 20260508 08:59:48.447 | INFO | MemInfo: { | 20260508 08:59:48.447 | INFO | "gpu" : " 17 GB", | 20260508 08:59:48.447 | INFO | "huge" : "", | 20260508 08:59:48.447 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:48.447 | INFO | "malloc" : " 14 GB", | 20260508 08:59:48.447 | INFO | "pinned" : " 29 GB" | 20260508 08:59:48.447 | INFO | } | 20260508 08:59:48.447 | INFO | Perf report for the last 5.14041 seconds: | 20260508 08:59:48.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:48.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.447 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 08:59:48.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.447 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 08:59:48.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:48.448 | INFO | Uploaded 36GiB to GPU in 2.40278s (14.9826GiB/s) | 20260508 08:59:48.448 | INFO | Downloaded 34GiB to GPU in 2.44797s (13.8891GiB/s) | 20260508 08:59:53.583 | INFO | Analyzing chunk frames = [6860288,6860800) | 20260508 08:59:53.583 | INFO | MemInfo: { | 20260508 08:59:53.583 | INFO | "gpu" : " 17 GB", | 20260508 08:59:53.583 | INFO | "huge" : "", | 20260508 08:59:53.583 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:53.583 | INFO | "malloc" : " 14 GB", | 20260508 08:59:53.583 | INFO | "pinned" : " 29 GB" | 20260508 08:59:53.583 | INFO | } | 20260508 08:59:53.583 | INFO | Perf report for the last 5.13583 seconds: | 20260508 08:59:53.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:53.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.583 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 08:59:53.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.583 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 08:59:53.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:53.584 | INFO | Uploaded 36GiB to GPU in 2.40223s (14.9861GiB/s) | 20260508 08:59:53.584 | INFO | Downloaded 34GiB to GPU in 2.44785s (13.8897GiB/s) | 20260508 08:59:53.680 | INFO | Starting a round of Baz IO | 20260508 08:59:57.090 | INFO | Populating RTMetrics report | 20260508 08:59:57.091 | INFO | RTMetrics report populated | 20260508 08:59:57.091 | INFO | Opening tmp RTMetrics json file | 20260508 08:59:57.101 | INFO | Serializing RTMetrics to json | 20260508 08:59:57.102 | INFO | Serialization done, swapping tmp file now... | 20260508 08:59:57.103 | INFO | RTMetrics done | 20260508 08:59:57.164 | INFO | Finished a round of Baz IO: Wrote 1.491 GiB in 3.483 seconds | 20260508 08:59:57.164 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.00% / 59.64% | 20260508 08:59:57.164 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.270609% / 3.97364e-06% / 6.35783e-05% | 20260508 08:59:57.164 | INFO | Baz IO overall: | 20260508 08:59:57.164 | INFO | BytesWritten : 3001605921462 | 20260508 08:59:57.164 | INFO | NumEvents : 1444447400153 | 20260508 08:59:58.726 | INFO | Analyzing chunk frames = [6860800,6861312) | 20260508 08:59:58.726 | INFO | MemInfo: { | 20260508 08:59:58.726 | INFO | "gpu" : " 17 GB", | 20260508 08:59:58.726 | INFO | "huge" : "", | 20260508 08:59:58.726 | INFO | "huge_shared" : " 29 GB", | 20260508 08:59:58.726 | INFO | "malloc" : " 14 GB", | 20260508 08:59:58.726 | INFO | "pinned" : " 29 GB" | 20260508 08:59:58.726 | INFO | } | 20260508 08:59:58.726 | INFO | Perf report for the last 5.14264 seconds: | 20260508 08:59:58.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 08:59:58.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:58.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:58.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:58.726 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 08:59:58.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 08:59:58.726 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 08:59:58.726 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 08:59:58.726 | INFO | BAZWRITER : 67.0%, 2.6, 89.0ms, 0 | 20260508 08:59:58.726 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 08:59:58.726 | INFO | Uploaded 36GiB to GPU in 2.612s (13.7825GiB/s) | 20260508 08:59:58.726 | INFO | Downloaded 38GiB to GPU in 2.69904s (14.0791GiB/s) | 20260508 09:00:03.867 | INFO | Analyzing chunk frames = [6861312,6861824) | 20260508 09:00:03.867 | INFO | MemInfo: { | 20260508 09:00:03.867 | INFO | "gpu" : " 17 GB", | 20260508 09:00:03.867 | INFO | "huge" : "", | 20260508 09:00:03.867 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:03.867 | INFO | "malloc" : " 14 GB", | 20260508 09:00:03.867 | INFO | "pinned" : " 29 GB" | 20260508 09:00:03.867 | INFO | } | 20260508 09:00:03.867 | INFO | Perf report for the last 5.14137 seconds: | 20260508 09:00:03.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:03.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.867 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 09:00:03.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.867 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 09:00:03.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:03.868 | INFO | Uploaded 36GiB to GPU in 2.53887s (14.1795GiB/s) | 20260508 09:00:03.868 | INFO | Downloaded 34GiB to GPU in 2.45571s (13.8453GiB/s) | 20260508 09:00:09.005 | INFO | Analyzing chunk frames = [6861824,6862336) | 20260508 09:00:09.005 | INFO | MemInfo: { | 20260508 09:00:09.005 | INFO | "gpu" : " 17 GB", | 20260508 09:00:09.005 | INFO | "huge" : "", | 20260508 09:00:09.005 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:09.005 | INFO | "malloc" : " 14 GB", | 20260508 09:00:09.005 | INFO | "pinned" : " 29 GB" | 20260508 09:00:09.005 | INFO | } | 20260508 09:00:09.005 | INFO | Perf report for the last 5.13794 seconds: | 20260508 09:00:09.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:09.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.005 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:00:09.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.005 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:00:09.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:09.006 | INFO | Uploaded 36GiB to GPU in 2.49238s (14.444GiB/s) | 20260508 09:00:09.006 | INFO | Downloaded 34GiB to GPU in 2.4249s (14.0212GiB/s) | 20260508 09:00:14.138 | INFO | Analyzing chunk frames = [6862336,6862848) | 20260508 09:00:14.138 | INFO | MemInfo: { | 20260508 09:00:14.138 | INFO | "gpu" : " 17 GB", | 20260508 09:00:14.138 | INFO | "huge" : "", | 20260508 09:00:14.138 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:14.138 | INFO | "malloc" : " 14 GB", | 20260508 09:00:14.138 | INFO | "pinned" : " 29 GB" | 20260508 09:00:14.138 | INFO | } | 20260508 09:00:14.138 | INFO | Perf report for the last 5.13297 seconds: | 20260508 09:00:14.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:14.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | ANALYSIS : 67.1%, 2.9, 103.7ms, 0 | 20260508 09:00:14.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:00:14.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:14.138 | INFO | Uploaded 36GiB to GPU in 2.59431s (13.8765GiB/s) | 20260508 09:00:14.138 | INFO | Downloaded 34GiB to GPU in 2.42556s (14.0174GiB/s) | 20260508 09:00:19.283 | INFO | Analyzing chunk frames = [6862848,6863360) | 20260508 09:00:19.283 | INFO | MemInfo: { | 20260508 09:00:19.283 | INFO | "gpu" : " 17 GB", | 20260508 09:00:19.283 | INFO | "huge" : "", | 20260508 09:00:19.283 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:19.284 | INFO | "malloc" : " 14 GB", | 20260508 09:00:19.284 | INFO | "pinned" : " 29 GB" | 20260508 09:00:19.284 | INFO | } | 20260508 09:00:19.284 | INFO | Perf report for the last 5.14546 seconds: | 20260508 09:00:19.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:19.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 09:00:19.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:00:19.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:19.284 | INFO | Uploaded 36GiB to GPU in 2.32003s (15.5171GiB/s) | 20260508 09:00:19.284 | INFO | Downloaded 34GiB to GPU in 2.50023s (13.5987GiB/s) | 20260508 09:00:24.419 | INFO | Analyzing chunk frames = [6863360,6863872) | 20260508 09:00:24.419 | INFO | MemInfo: { | 20260508 09:00:24.419 | INFO | "gpu" : " 17 GB", | 20260508 09:00:24.419 | INFO | "huge" : "", | 20260508 09:00:24.419 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:24.419 | INFO | "malloc" : " 14 GB", | 20260508 09:00:24.419 | INFO | "pinned" : " 29 GB" | 20260508 09:00:24.419 | INFO | } | 20260508 09:00:24.419 | INFO | Perf report for the last 5.13569 seconds: | 20260508 09:00:24.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:24.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.419 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 09:00:24.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.419 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:00:24.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:24.420 | INFO | Uploaded 36GiB to GPU in 2.38791s (15.0759GiB/s) | 20260508 09:00:24.420 | INFO | Downloaded 34GiB to GPU in 2.44407s (13.9112GiB/s) | 20260508 09:00:29.560 | INFO | Analyzing chunk frames = [6863872,6864384) | 20260508 09:00:29.560 | INFO | MemInfo: { | 20260508 09:00:29.560 | INFO | "gpu" : " 17 GB", | 20260508 09:00:29.560 | INFO | "huge" : "", | 20260508 09:00:29.560 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:29.560 | INFO | "malloc" : " 14 GB", | 20260508 09:00:29.560 | INFO | "pinned" : " 29 GB" | 20260508 09:00:29.560 | INFO | } | 20260508 09:00:29.560 | INFO | Perf report for the last 5.14063 seconds: | 20260508 09:00:29.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:29.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 09:00:29.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:00:29.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:29.560 | INFO | Uploaded 36GiB to GPU in 2.45415s (14.669GiB/s) | 20260508 09:00:29.560 | INFO | Downloaded 34GiB to GPU in 2.45523s (13.848GiB/s) | 20260508 09:00:34.697 | INFO | Analyzing chunk frames = [6864384,6864896) | 20260508 09:00:34.697 | INFO | MemInfo: { | 20260508 09:00:34.697 | INFO | "gpu" : " 17 GB", | 20260508 09:00:34.697 | INFO | "huge" : "", | 20260508 09:00:34.697 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:34.697 | INFO | "malloc" : " 14 GB", | 20260508 09:00:34.697 | INFO | "pinned" : " 29 GB" | 20260508 09:00:34.697 | INFO | } | 20260508 09:00:34.697 | INFO | Perf report for the last 5.13712 seconds: | 20260508 09:00:34.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:34.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.697 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 09:00:34.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.697 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:00:34.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:34.698 | INFO | Uploaded 36GiB to GPU in 2.43518s (14.7833GiB/s) | 20260508 09:00:34.698 | INFO | Downloaded 34GiB to GPU in 2.45262s (13.8627GiB/s) | 20260508 09:00:34.698 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:00:34.698 | INFO | Total mem usage: 16.7 GiB | 20260508 09:00:34.698 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:00:34.698 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:00:34.698 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:00:34.698 | INFO | High Water Breakdown: | 20260508 09:00:34.698 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:00:34.698 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:00:34.698 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:00:34.698 | INFO | ------------------------------------------ | 20260508 09:00:34.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:00:34.699 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:00:34.699 | INFO | Total mem usage: 26.5 GiB | 20260508 09:00:34.699 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:00:34.699 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:00:34.699 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:00:34.699 | INFO | High Water Breakdown: | 20260508 09:00:34.699 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:00:34.699 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:00:34.699 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:00:34.699 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:00:34.699 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:00:34.699 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:00:34.699 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:00:34.699 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:00:34.699 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:00:34.699 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:00:34.699 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:00:34.699 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:00:34.699 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:00:34.699 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:00:34.699 | INFO | ------------------------------------------ | 20260508 09:00:34.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:00:34.699 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:00:34.699 | INFO | Total mem usage: 27.8 GiB | 20260508 09:00:34.699 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:00:34.699 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:00:34.699 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:00:34.699 | INFO | High Water Breakdown: | 20260508 09:00:34.699 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:00:34.699 | INFO | ------------------------------------------ | 20260508 09:00:34.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:00:34.699 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:00:34.699 | INFO | Total mem usage: 15.7 GiB | 20260508 09:00:34.699 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:00:34.699 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:00:34.699 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:00:34.699 | INFO | High Water Breakdown: | 20260508 09:00:34.699 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:00:34.699 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:00:34.699 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:00:34.699 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:00:34.699 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:00:34.699 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:00:34.699 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:00:34.699 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:00:34.699 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:00:34.699 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:00:34.699 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:00:34.699 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:00:34.699 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:00:34.699 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:00:34.699 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:00:34.699 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:00:34.699 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:00:34.699 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:00:34.699 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:00:34.699 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:00:34.699 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:00:34.699 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:00:34.699 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:00:34.699 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:00:34.699 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:00:34.699 | INFO | ------------------------------------------ | 20260508 09:00:34.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:00:34.699 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:00:34.699 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:00:34.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:00:34.700 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 6.45e-01 | 20260508 09:00:34.700 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.19e-02 / 5.73e-03 | 20260508 09:00:34.700 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.47e+00, 5.88e-01 / 2.01e-01 | 20260508 09:00:34.700 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.44e+00, 2.09e-01 / 1.34e-01 | 20260508 09:00:34.700 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.51e+01, 3.74e+00 / 1.12e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.86e+00 / 1.38e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.07e-01 / 2.43e-01 | 20260508 09:00:34.700 | INFO | PerfReport: 3.04%, Metrics -- 9.97e-01 / 1.11e+00, 2.51e-01 / 1.30e-01 | 20260508 09:00:34.700 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.33e-01, 4.92e-02 / 2.54e-02 | 20260508 09:00:34.700 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:00:34.700 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:00:34.700 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:00:34.700 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 24.33%, Upload -- 2.57e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 24.52%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 31.05%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 2.21%, LockUpload -- 2.33e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 2.45%, LockDownload -- 2.59e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 15.42%, LockCompute -- 1.63e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 09:00:34.700 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260508 09:00:34.808 | INFO | Starting a round of Baz IO | 20260508 09:00:38.213 | INFO | Populating RTMetrics report | 20260508 09:00:38.213 | INFO | RTMetrics report populated | 20260508 09:00:38.213 | INFO | Opening tmp RTMetrics json file | 20260508 09:00:38.220 | INFO | Serializing RTMetrics to json | 20260508 09:00:38.221 | INFO | Serialization done, swapping tmp file now... | 20260508 09:00:38.222 | INFO | RTMetrics done | 20260508 09:00:38.281 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.472 seconds | 20260508 09:00:38.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.64% | 20260508 09:00:38.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.271038% / 0% / 5.96046e-05% | 20260508 09:00:38.281 | INFO | Baz IO overall: | 20260508 09:00:38.281 | INFO | BytesWritten : 3003207413405 | 20260508 09:00:38.281 | INFO | NumEvents : 1445205432655 | 20260508 09:00:39.839 | INFO | Analyzing chunk frames = [6864896,6865408) | 20260508 09:00:39.839 | INFO | MemInfo: { | 20260508 09:00:39.839 | INFO | "gpu" : " 17 GB", | 20260508 09:00:39.839 | INFO | "huge" : "", | 20260508 09:00:39.839 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:39.839 | INFO | "malloc" : " 14 GB", | 20260508 09:00:39.839 | INFO | "pinned" : " 29 GB" | 20260508 09:00:39.839 | INFO | } | 20260508 09:00:39.839 | INFO | Perf report for the last 5.1423 seconds: | 20260508 09:00:39.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:39.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:39.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:39.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:39.840 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 09:00:39.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:39.840 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260508 09:00:39.840 | INFO | PRE_HQ : 66.0%, 1.8, 60.0ms, 0 | 20260508 09:00:39.840 | INFO | BAZWRITER : 66.9%, 2.7, 93.6ms, 0 | 20260508 09:00:39.840 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:00:39.840 | INFO | Uploaded 36GiB to GPU in 2.57158s (13.9992GiB/s) | 20260508 09:00:39.840 | INFO | Downloaded 38GiB to GPU in 2.70973s (14.0236GiB/s) | 20260508 09:00:44.979 | INFO | Analyzing chunk frames = [6865408,6865920) | 20260508 09:00:44.979 | INFO | MemInfo: { | 20260508 09:00:44.979 | INFO | "gpu" : " 17 GB", | 20260508 09:00:44.979 | INFO | "huge" : "", | 20260508 09:00:44.979 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:44.979 | INFO | "malloc" : " 14 GB", | 20260508 09:00:44.979 | INFO | "pinned" : " 29 GB" | 20260508 09:00:44.980 | INFO | } | 20260508 09:00:44.980 | INFO | Perf report for the last 5.1401 seconds: | 20260508 09:00:44.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:44.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:00:44.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:00:44.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:44.980 | INFO | Uploaded 36GiB to GPU in 2.51784s (14.298GiB/s) | 20260508 09:00:44.980 | INFO | Downloaded 34GiB to GPU in 2.45789s (13.833GiB/s) | 20260508 09:00:50.121 | INFO | Analyzing chunk frames = [6865920,6866432) | 20260508 09:00:50.121 | INFO | MemInfo: { | 20260508 09:00:50.121 | INFO | "gpu" : " 17 GB", | 20260508 09:00:50.121 | INFO | "huge" : "", | 20260508 09:00:50.121 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:50.121 | INFO | "malloc" : " 14 GB", | 20260508 09:00:50.121 | INFO | "pinned" : " 29 GB" | 20260508 09:00:50.121 | INFO | } | 20260508 09:00:50.121 | INFO | Perf report for the last 5.14172 seconds: | 20260508 09:00:50.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:50.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.121 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:00:50.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.121 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:00:50.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:50.122 | INFO | Uploaded 36GiB to GPU in 2.49362s (14.4369GiB/s) | 20260508 09:00:50.122 | INFO | Downloaded 34GiB to GPU in 2.43431s (13.967GiB/s) | 20260508 09:00:55.257 | INFO | Analyzing chunk frames = [6866432,6866944) | 20260508 09:00:55.257 | INFO | MemInfo: { | 20260508 09:00:55.257 | INFO | "gpu" : " 17 GB", | 20260508 09:00:55.257 | INFO | "huge" : "", | 20260508 09:00:55.257 | INFO | "huge_shared" : " 29 GB", | 20260508 09:00:55.257 | INFO | "malloc" : " 14 GB", | 20260508 09:00:55.257 | INFO | "pinned" : " 29 GB" | 20260508 09:00:55.257 | INFO | } | 20260508 09:00:55.257 | INFO | Perf report for the last 5.13613 seconds: | 20260508 09:00:55.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:00:55.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.257 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 09:00:55.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.257 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:00:55.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:00:55.258 | INFO | Uploaded 36GiB to GPU in 2.51174s (14.3327GiB/s) | 20260508 09:00:55.258 | INFO | Downloaded 34GiB to GPU in 2.44162s (13.9252GiB/s) | 20260508 09:01:00.390 | INFO | Analyzing chunk frames = [6866944,6867456) | 20260508 09:01:00.390 | INFO | MemInfo: { | 20260508 09:01:00.390 | INFO | "gpu" : " 17 GB", | 20260508 09:01:00.390 | INFO | "huge" : "", | 20260508 09:01:00.390 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:00.390 | INFO | "malloc" : " 14 GB", | 20260508 09:01:00.390 | INFO | "pinned" : " 29 GB" | 20260508 09:01:00.390 | INFO | } | 20260508 09:01:00.390 | INFO | Perf report for the last 5.13269 seconds: | 20260508 09:01:00.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:00.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.390 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 09:01:00.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.390 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:01:00.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:00.391 | INFO | Uploaded 36GiB to GPU in 2.29281s (15.7013GiB/s) | 20260508 09:01:00.391 | INFO | Downloaded 34GiB to GPU in 2.50361s (13.5804GiB/s) | 20260508 09:01:05.537 | INFO | Analyzing chunk frames = [6867456,6867968) | 20260508 09:01:05.537 | INFO | MemInfo: { | 20260508 09:01:05.537 | INFO | "gpu" : " 17 GB", | 20260508 09:01:05.537 | INFO | "huge" : "", | 20260508 09:01:05.537 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:05.537 | INFO | "malloc" : " 14 GB", | 20260508 09:01:05.537 | INFO | "pinned" : " 29 GB" | 20260508 09:01:05.537 | INFO | } | 20260508 09:01:05.538 | INFO | Perf report for the last 5.14736 seconds: | 20260508 09:01:05.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:05.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 09:01:05.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:01:05.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:05.538 | INFO | Uploaded 36GiB to GPU in 2.41409s (14.9124GiB/s) | 20260508 09:01:05.538 | INFO | Downloaded 34GiB to GPU in 2.43751s (13.9487GiB/s) | 20260508 09:01:10.679 | INFO | Analyzing chunk frames = [6867968,6868480) | 20260508 09:01:10.679 | INFO | MemInfo: { | 20260508 09:01:10.679 | INFO | "gpu" : " 17 GB", | 20260508 09:01:10.679 | INFO | "huge" : "", | 20260508 09:01:10.679 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:10.679 | INFO | "malloc" : " 14 GB", | 20260508 09:01:10.679 | INFO | "pinned" : " 29 GB" | 20260508 09:01:10.679 | INFO | } | 20260508 09:01:10.679 | INFO | Perf report for the last 5.14166 seconds: | 20260508 09:01:10.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:10.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.679 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 09:01:10.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.679 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:01:10.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:10.680 | INFO | Uploaded 36GiB to GPU in 2.44789s (14.7066GiB/s) | 20260508 09:01:10.680 | INFO | Downloaded 34GiB to GPU in 2.43302s (13.9744GiB/s) | 20260508 09:01:15.821 | INFO | Analyzing chunk frames = [6868480,6868992) | 20260508 09:01:15.821 | INFO | MemInfo: { | 20260508 09:01:15.821 | INFO | "gpu" : " 17 GB", | 20260508 09:01:15.821 | INFO | "huge" : "", | 20260508 09:01:15.821 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:15.821 | INFO | "malloc" : " 14 GB", | 20260508 09:01:15.821 | INFO | "pinned" : " 29 GB" | 20260508 09:01:15.821 | INFO | } | 20260508 09:01:15.821 | INFO | Perf report for the last 5.14219 seconds: | 20260508 09:01:15.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:15.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.821 | INFO | ANALYSIS : 65.8%, 3.0, 103.0ms, 0 | 20260508 09:01:15.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.822 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:01:15.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:15.822 | INFO | Uploaded 36GiB to GPU in 2.46122s (14.6269GiB/s) | 20260508 09:01:15.822 | INFO | Downloaded 34GiB to GPU in 2.43871s (13.9418GiB/s) | 20260508 09:01:15.919 | INFO | Starting a round of Baz IO | 20260508 09:01:19.325 | INFO | Populating RTMetrics report | 20260508 09:01:19.325 | INFO | RTMetrics report populated | 20260508 09:01:19.325 | INFO | Opening tmp RTMetrics json file | 20260508 09:01:19.335 | INFO | Serializing RTMetrics to json | 20260508 09:01:19.337 | INFO | Serialization done, swapping tmp file now... | 20260508 09:01:19.337 | INFO | RTMetrics done | 20260508 09:01:19.399 | INFO | Finished a round of Baz IO: Wrote 1.491 GiB in 3.48 seconds | 20260508 09:01:19.399 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:01:19.399 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.270903% / 3.97364e-06% / 6.35783e-05% | 20260508 09:01:19.399 | INFO | Baz IO overall: | 20260508 09:01:19.399 | INFO | BytesWritten : 3004808626681 | 20260508 09:01:19.399 | INFO | NumEvents : 1445963246520 | 20260508 09:01:20.951 | INFO | Analyzing chunk frames = [6868992,6869504) | 20260508 09:01:20.951 | INFO | MemInfo: { | 20260508 09:01:20.951 | INFO | "gpu" : " 17 GB", | 20260508 09:01:20.951 | INFO | "huge" : "", | 20260508 09:01:20.952 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:20.952 | INFO | "malloc" : " 14 GB", | 20260508 09:01:20.952 | INFO | "pinned" : " 29 GB" | 20260508 09:01:20.952 | INFO | } | 20260508 09:01:20.952 | INFO | Perf report for the last 5.1303 seconds: | 20260508 09:01:20.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:20.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:20.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:20.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:20.952 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 09:01:20.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:20.952 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:01:20.952 | INFO | PRE_HQ : 65.5%, 1.6, 54.7ms, 0 | 20260508 09:01:20.952 | INFO | BAZWRITER : 67.2%, 2.6, 88.4ms, 0 | 20260508 09:01:20.952 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:01:20.952 | INFO | Uploaded 36GiB to GPU in 2.60447s (13.8224GiB/s) | 20260508 09:01:20.952 | INFO | Downloaded 38GiB to GPU in 2.69852s (14.0818GiB/s) | 20260508 09:01:26.093 | INFO | Analyzing chunk frames = [6869504,6870016) | 20260508 09:01:26.093 | INFO | MemInfo: { | 20260508 09:01:26.093 | INFO | "gpu" : " 17 GB", | 20260508 09:01:26.093 | INFO | "huge" : "", | 20260508 09:01:26.093 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:26.093 | INFO | "malloc" : " 14 GB", | 20260508 09:01:26.093 | INFO | "pinned" : " 29 GB" | 20260508 09:01:26.093 | INFO | } | 20260508 09:01:26.093 | INFO | Perf report for the last 5.14147 seconds: | 20260508 09:01:26.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:26.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:01:26.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:01:26.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:26.093 | INFO | Uploaded 36GiB to GPU in 2.51707s (14.3023GiB/s) | 20260508 09:01:26.094 | INFO | Downloaded 34GiB to GPU in 2.46115s (13.8147GiB/s) | 20260508 09:01:31.231 | INFO | Analyzing chunk frames = [6870016,6870528) | 20260508 09:01:31.231 | INFO | MemInfo: { | 20260508 09:01:31.231 | INFO | "gpu" : " 17 GB", | 20260508 09:01:31.232 | INFO | "huge" : "", | 20260508 09:01:31.232 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:31.232 | INFO | "malloc" : " 14 GB", | 20260508 09:01:31.232 | INFO | "pinned" : " 29 GB" | 20260508 09:01:31.232 | INFO | } | 20260508 09:01:31.232 | INFO | Perf report for the last 5.13851 seconds: | 20260508 09:01:31.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:31.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 09:01:31.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:01:31.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:31.232 | INFO | Uploaded 36GiB to GPU in 2.4511s (14.6873GiB/s) | 20260508 09:01:31.232 | INFO | Downloaded 34GiB to GPU in 2.43127s (13.9845GiB/s) | 20260508 09:01:36.371 | INFO | Analyzing chunk frames = [6870528,6871040) | 20260508 09:01:36.371 | INFO | MemInfo: { | 20260508 09:01:36.371 | INFO | "gpu" : " 17 GB", | 20260508 09:01:36.371 | INFO | "huge" : "", | 20260508 09:01:36.372 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:36.372 | INFO | "malloc" : " 14 GB", | 20260508 09:01:36.372 | INFO | "pinned" : " 29 GB" | 20260508 09:01:36.372 | INFO | } | 20260508 09:01:36.372 | INFO | Perf report for the last 5.14 seconds: | 20260508 09:01:36.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:36.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.372 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 09:01:36.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.372 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:01:36.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:36.373 | INFO | Uploaded 36GiB to GPU in 2.56105s (14.0568GiB/s) | 20260508 09:01:36.373 | INFO | Downloaded 34GiB to GPU in 2.44015s (13.9336GiB/s) | 20260508 09:01:41.514 | INFO | Analyzing chunk frames = [6871040,6871552) | 20260508 09:01:41.514 | INFO | MemInfo: { | 20260508 09:01:41.514 | INFO | "gpu" : " 17 GB", | 20260508 09:01:41.514 | INFO | "huge" : "", | 20260508 09:01:41.514 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:41.514 | INFO | "malloc" : " 14 GB", | 20260508 09:01:41.514 | INFO | "pinned" : " 29 GB" | 20260508 09:01:41.514 | INFO | } | 20260508 09:01:41.514 | INFO | Perf report for the last 5.14234 seconds: | 20260508 09:01:41.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:41.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:01:41.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:01:41.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:41.514 | INFO | Uploaded 36GiB to GPU in 2.40128s (14.992GiB/s) | 20260508 09:01:41.515 | INFO | Downloaded 34GiB to GPU in 2.56201s (13.2708GiB/s) | 20260508 09:01:46.648 | INFO | Analyzing chunk frames = [6871552,6872064) | 20260508 09:01:46.648 | INFO | MemInfo: { | 20260508 09:01:46.648 | INFO | "gpu" : " 17 GB", | 20260508 09:01:46.648 | INFO | "huge" : "", | 20260508 09:01:46.648 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:46.648 | INFO | "malloc" : " 14 GB", | 20260508 09:01:46.648 | INFO | "pinned" : " 29 GB" | 20260508 09:01:46.648 | INFO | } | 20260508 09:01:46.648 | INFO | Perf report for the last 5.13437 seconds: | 20260508 09:01:46.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:46.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.649 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 09:01:46.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.649 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:01:46.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:46.649 | INFO | Uploaded 36GiB to GPU in 2.33214s (15.4364GiB/s) | 20260508 09:01:46.649 | INFO | Downloaded 34GiB to GPU in 2.45066s (13.8738GiB/s) | 20260508 09:01:51.793 | INFO | Analyzing chunk frames = [6872064,6872576) | 20260508 09:01:51.793 | INFO | MemInfo: { | 20260508 09:01:51.793 | INFO | "gpu" : " 17 GB", | 20260508 09:01:51.793 | INFO | "huge" : "", | 20260508 09:01:51.793 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:51.793 | INFO | "malloc" : " 14 GB", | 20260508 09:01:51.793 | INFO | "pinned" : " 29 GB" | 20260508 09:01:51.793 | INFO | } | 20260508 09:01:51.793 | INFO | Perf report for the last 5.14444 seconds: | 20260508 09:01:51.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:51.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:01:51.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:01:51.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:51.793 | INFO | Uploaded 36GiB to GPU in 2.3895s (15.0659GiB/s) | 20260508 09:01:51.793 | INFO | Downloaded 34GiB to GPU in 2.44825s (13.8875GiB/s) | 20260508 09:01:56.930 | INFO | Analyzing chunk frames = [6872576,6873088) | 20260508 09:01:56.930 | INFO | MemInfo: { | 20260508 09:01:56.930 | INFO | "gpu" : " 17 GB", | 20260508 09:01:56.930 | INFO | "huge" : "", | 20260508 09:01:56.930 | INFO | "huge_shared" : " 29 GB", | 20260508 09:01:56.930 | INFO | "malloc" : " 14 GB", | 20260508 09:01:56.930 | INFO | "pinned" : " 29 GB" | 20260508 09:01:56.930 | INFO | } | 20260508 09:01:56.930 | INFO | Perf report for the last 5.137 seconds: | 20260508 09:01:56.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:01:56.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:01:56.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:01:56.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:01:56.930 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:01:56.931 | INFO | Total mem usage: 16.7 GiB | 20260508 09:01:56.931 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:01:56.931 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:01:56.931 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:01:56.931 | INFO | High Water Breakdown: | 20260508 09:01:56.931 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:01:56.931 | INFO | Uploaded 36GiB to GPU in 2.41936s (14.88GiB/s) | 20260508 09:01:56.931 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:01:56.931 | INFO | Downloaded 34GiB to GPU in 2.46672s (13.7835GiB/s) | 20260508 09:01:56.931 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:01:56.931 | INFO | ------------------------------------------ | 20260508 09:01:56.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:01:56.931 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:01:56.931 | INFO | Total mem usage: 26.5 GiB | 20260508 09:01:56.931 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:01:56.931 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:01:56.931 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:01:56.931 | INFO | High Water Breakdown: | 20260508 09:01:56.931 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:01:56.931 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:01:56.931 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:01:56.931 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:01:56.931 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:01:56.931 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:01:56.931 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:01:56.931 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:01:56.931 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:01:56.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:01:56.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:01:56.931 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:01:56.931 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:01:56.931 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:01:56.931 | INFO | ------------------------------------------ | 20260508 09:01:56.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:01:56.931 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:01:56.931 | INFO | Total mem usage: 27.8 GiB | 20260508 09:01:56.931 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:01:56.931 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:01:56.931 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:01:56.931 | INFO | High Water Breakdown: | 20260508 09:01:56.931 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:01:56.931 | INFO | ------------------------------------------ | 20260508 09:01:56.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:01:56.931 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:01:56.931 | INFO | Total mem usage: 15.7 GiB | 20260508 09:01:56.931 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:01:56.931 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:01:56.931 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:01:56.931 | INFO | High Water Breakdown: | 20260508 09:01:56.931 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:01:56.931 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:01:56.931 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:01:56.931 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:01:56.931 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:01:56.931 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:01:56.931 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:01:56.931 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:01:56.931 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:01:56.931 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:01:56.931 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:01:56.931 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:01:56.931 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:01:56.931 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:01:56.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:01:56.931 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:01:56.931 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:01:56.931 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:01:56.931 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:01:56.931 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:01:56.931 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:01:56.931 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:01:56.931 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:01:56.931 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:01:56.931 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:01:56.931 | INFO | ------------------------------------------ | 20260508 09:01:56.931 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:01:56.931 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:01:56.931 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:01:56.931 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:01:56.931 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 5.93e-01 | 20260508 09:01:56.931 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.02e-01, 1.10e-02 / 4.90e-03 | 20260508 09:01:56.931 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.40e+00, 5.90e-01 / 2.55e-01 | 20260508 09:01:56.932 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.27e-01 | 20260508 09:01:56.932 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.51e+01, 3.79e+00 / 1.04e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.83e+00 / 1.20e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.50e+00, 4.08e-01 / 2.67e-01 | 20260508 09:01:56.932 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.14e+00, 2.49e-01 / 1.41e-01 | 20260508 09:01:56.932 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.32e-01, 4.88e-02 / 2.56e-02 | 20260508 09:01:56.932 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:01:56.932 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:01:56.932 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:01:56.932 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 24.32%, Upload -- 2.56e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 31.02%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 2.58%, LockDownload -- 2.72e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 15.33%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:01:56.932 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 09:01:57.027 | INFO | Starting a round of Baz IO | 20260508 09:02:00.440 | INFO | Populating RTMetrics report | 20260508 09:02:00.440 | INFO | RTMetrics report populated | 20260508 09:02:00.440 | INFO | Opening tmp RTMetrics json file | 20260508 09:02:00.441 | INFO | Serializing RTMetrics to json | 20260508 09:02:00.442 | INFO | Serialization done, swapping tmp file now... | 20260508 09:02:00.443 | INFO | RTMetrics done | 20260508 09:02:00.502 | INFO | Finished a round of Baz IO: Wrote 1.49 GiB in 3.475 seconds | 20260508 09:02:00.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:02:00.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.2701% / 0% / 5.96046e-05% | 20260508 09:02:00.503 | INFO | Baz IO overall: | 20260508 09:02:00.503 | INFO | BytesWritten : 3006408969582 | 20260508 09:02:00.503 | INFO | NumEvents : 1446720348815 | 20260508 09:02:02.070 | INFO | Analyzing chunk frames = [6873088,6873600) | 20260508 09:02:02.070 | INFO | MemInfo: { | 20260508 09:02:02.070 | INFO | "gpu" : " 17 GB", | 20260508 09:02:02.070 | INFO | "huge" : "", | 20260508 09:02:02.070 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:02.070 | INFO | "malloc" : " 14 GB", | 20260508 09:02:02.070 | INFO | "pinned" : " 29 GB" | 20260508 09:02:02.070 | INFO | } | 20260508 09:02:02.070 | INFO | Perf report for the last 5.1398 seconds: | 20260508 09:02:02.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:02.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:02.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:02.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:02.070 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 09:02:02.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:02.070 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:02:02.070 | INFO | PRE_HQ : 65.7%, 1.6, 54.7ms, 0 | 20260508 09:02:02.070 | INFO | BAZWRITER : 67.0%, 2.6, 88.7ms, 0 | 20260508 09:02:02.070 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 09:02:02.070 | INFO | Uploaded 36GiB to GPU in 2.51649s (14.3057GiB/s) | 20260508 09:02:02.071 | INFO | Downloaded 38GiB to GPU in 2.69924s (14.078GiB/s) | 20260508 09:02:07.205 | INFO | Analyzing chunk frames = [6873600,6874112) | 20260508 09:02:07.205 | INFO | MemInfo: { | 20260508 09:02:07.205 | INFO | "gpu" : " 17 GB", | 20260508 09:02:07.205 | INFO | "huge" : "", | 20260508 09:02:07.205 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:07.205 | INFO | "malloc" : " 14 GB", | 20260508 09:02:07.205 | INFO | "pinned" : " 29 GB" | 20260508 09:02:07.205 | INFO | } | 20260508 09:02:07.205 | INFO | Perf report for the last 5.13536 seconds: | 20260508 09:02:07.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:07.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 09:02:07.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:02:07.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:07.205 | INFO | Uploaded 36GiB to GPU in 2.50264s (14.3848GiB/s) | 20260508 09:02:07.205 | INFO | Downloaded 34GiB to GPU in 2.45093s (13.8723GiB/s) | 20260508 09:02:12.351 | INFO | Analyzing chunk frames = [6874112,6874624) | 20260508 09:02:12.351 | INFO | MemInfo: { | 20260508 09:02:12.351 | INFO | "gpu" : " 17 GB", | 20260508 09:02:12.351 | INFO | "huge" : "", | 20260508 09:02:12.351 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:12.351 | INFO | "malloc" : " 14 GB", | 20260508 09:02:12.351 | INFO | "pinned" : " 29 GB" | 20260508 09:02:12.351 | INFO | } | 20260508 09:02:12.351 | INFO | Perf report for the last 5.14581 seconds: | 20260508 09:02:12.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:12.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 09:02:12.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:02:12.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:12.351 | INFO | Uploaded 36GiB to GPU in 2.50309s (14.3822GiB/s) | 20260508 09:02:12.351 | INFO | Downloaded 34GiB to GPU in 2.43516s (13.9621GiB/s) | 20260508 09:02:17.488 | INFO | Analyzing chunk frames = [6874624,6875136) | 20260508 09:02:17.488 | INFO | MemInfo: { | 20260508 09:02:17.488 | INFO | "gpu" : " 17 GB", | 20260508 09:02:17.488 | INFO | "huge" : "", | 20260508 09:02:17.488 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:17.488 | INFO | "malloc" : " 14 GB", | 20260508 09:02:17.488 | INFO | "pinned" : " 29 GB" | 20260508 09:02:17.488 | INFO | } | 20260508 09:02:17.488 | INFO | Perf report for the last 5.13715 seconds: | 20260508 09:02:17.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:17.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.488 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 09:02:17.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.488 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 09:02:17.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:17.489 | INFO | Uploaded 36GiB to GPU in 2.4975s (14.4144GiB/s) | 20260508 09:02:17.489 | INFO | Downloaded 34GiB to GPU in 2.42549s (14.0178GiB/s) | 20260508 09:02:22.626 | INFO | Analyzing chunk frames = [6875136,6875648) | 20260508 09:02:22.626 | INFO | MemInfo: { | 20260508 09:02:22.626 | INFO | "gpu" : " 17 GB", | 20260508 09:02:22.626 | INFO | "huge" : "", | 20260508 09:02:22.626 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:22.626 | INFO | "malloc" : " 14 GB", | 20260508 09:02:22.626 | INFO | "pinned" : " 29 GB" | 20260508 09:02:22.626 | INFO | } | 20260508 09:02:22.626 | INFO | Perf report for the last 5.13784 seconds: | 20260508 09:02:22.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:22.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 09:02:22.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260508 09:02:22.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:22.626 | INFO | Uploaded 36GiB to GPU in 2.31397s (15.5577GiB/s) | 20260508 09:02:22.626 | INFO | Downloaded 34GiB to GPU in 2.50253s (13.5863GiB/s) | 20260508 09:02:27.765 | INFO | Analyzing chunk frames = [6875648,6876160) | 20260508 09:02:27.765 | INFO | MemInfo: { | 20260508 09:02:27.765 | INFO | "gpu" : " 17 GB", | 20260508 09:02:27.765 | INFO | "huge" : "", | 20260508 09:02:27.765 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:27.765 | INFO | "malloc" : " 14 GB", | 20260508 09:02:27.765 | INFO | "pinned" : " 29 GB" | 20260508 09:02:27.765 | INFO | } | 20260508 09:02:27.765 | INFO | Perf report for the last 5.13939 seconds: | 20260508 09:02:27.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:27.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.765 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 09:02:27.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.766 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:02:27.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:27.766 | INFO | Uploaded 36GiB to GPU in 2.3968s (15.02GiB/s) | 20260508 09:02:27.766 | INFO | Downloaded 34GiB to GPU in 2.42895s (13.9978GiB/s) | 20260508 09:02:32.899 | INFO | Analyzing chunk frames = [6876160,6876672) | 20260508 09:02:32.899 | INFO | MemInfo: { | 20260508 09:02:32.899 | INFO | "gpu" : " 17 GB", | 20260508 09:02:32.899 | INFO | "huge" : "", | 20260508 09:02:32.899 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:32.899 | INFO | "malloc" : " 14 GB", | 20260508 09:02:32.899 | INFO | "pinned" : " 29 GB" | 20260508 09:02:32.899 | INFO | } | 20260508 09:02:32.899 | INFO | Perf report for the last 5.13379 seconds: | 20260508 09:02:32.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:32.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 09:02:32.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:02:32.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:32.899 | INFO | Uploaded 36GiB to GPU in 2.46501s (14.6044GiB/s) | 20260508 09:02:32.900 | INFO | Downloaded 34GiB to GPU in 2.44323s (13.916GiB/s) | 20260508 09:02:38.041 | INFO | Analyzing chunk frames = [6876672,6877184) | 20260508 09:02:38.041 | INFO | MemInfo: { | 20260508 09:02:38.041 | INFO | "gpu" : " 17 GB", | 20260508 09:02:38.041 | INFO | "huge" : "", | 20260508 09:02:38.041 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:38.041 | INFO | "malloc" : " 14 GB", | 20260508 09:02:38.041 | INFO | "pinned" : " 29 GB" | 20260508 09:02:38.041 | INFO | } | 20260508 09:02:38.041 | INFO | Perf report for the last 5.14221 seconds: | 20260508 09:02:38.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:38.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.041 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 09:02:38.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.042 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 09:02:38.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:38.042 | INFO | Uploaded 36GiB to GPU in 2.52539s (14.2552GiB/s) | 20260508 09:02:38.042 | INFO | Downloaded 34GiB to GPU in 2.45105s (13.8716GiB/s) | 20260508 09:02:38.140 | INFO | Starting a round of Baz IO | 20260508 09:02:41.545 | INFO | Populating RTMetrics report | 20260508 09:02:41.546 | INFO | RTMetrics report populated | 20260508 09:02:41.546 | INFO | Opening tmp RTMetrics json file | 20260508 09:02:41.557 | INFO | Serializing RTMetrics to json | 20260508 09:02:41.558 | INFO | Serialization done, swapping tmp file now... | 20260508 09:02:41.559 | INFO | RTMetrics done | 20260508 09:02:41.616 | INFO | Finished a round of Baz IO: Wrote 1.49 GiB in 3.475 seconds | 20260508 09:02:41.616 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:02:41.616 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.270108% / 3.97364e-06% / 6.35783e-05% | 20260508 09:02:41.616 | INFO | Baz IO overall: | 20260508 09:02:41.616 | INFO | BytesWritten : 3008008984695 | 20260508 09:02:41.616 | INFO | NumEvents : 1447477334765 | 20260508 09:02:43.188 | INFO | Analyzing chunk frames = [6877184,6877696) | 20260508 09:02:43.188 | INFO | MemInfo: { | 20260508 09:02:43.188 | INFO | "gpu" : " 17 GB", | 20260508 09:02:43.188 | INFO | "huge" : "", | 20260508 09:02:43.188 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:43.188 | INFO | "malloc" : " 14 GB", | 20260508 09:02:43.188 | INFO | "pinned" : " 29 GB" | 20260508 09:02:43.188 | INFO | } | 20260508 09:02:43.188 | INFO | Perf report for the last 5.14683 seconds: | 20260508 09:02:43.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:43.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:43.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:43.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:43.188 | INFO | ANALYSIS : 67.4%, 3.0, 105.5ms, 0 | 20260508 09:02:43.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:43.188 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:02:43.188 | INFO | PRE_HQ : 65.2%, 1.6, 54.5ms, 0 | 20260508 09:02:43.188 | INFO | BAZWRITER : 66.9%, 2.5, 87.2ms, 0 | 20260508 09:02:43.188 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:02:43.188 | INFO | Uploaded 36GiB to GPU in 2.61626s (13.7601GiB/s) | 20260508 09:02:43.189 | INFO | Downloaded 38GiB to GPU in 2.70699s (14.0378GiB/s) | 20260508 09:02:48.329 | INFO | Analyzing chunk frames = [6877696,6878208) | 20260508 09:02:48.329 | INFO | MemInfo: { | 20260508 09:02:48.329 | INFO | "gpu" : " 17 GB", | 20260508 09:02:48.329 | INFO | "huge" : "", | 20260508 09:02:48.329 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:48.329 | INFO | "malloc" : " 14 GB", | 20260508 09:02:48.329 | INFO | "pinned" : " 29 GB" | 20260508 09:02:48.329 | INFO | } | 20260508 09:02:48.329 | INFO | Perf report for the last 5.14071 seconds: | 20260508 09:02:48.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:48.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:02:48.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:02:48.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:48.329 | INFO | Uploaded 36GiB to GPU in 2.57383s (13.9869GiB/s) | 20260508 09:02:48.329 | INFO | Downloaded 34GiB to GPU in 2.47789s (13.7214GiB/s) | 20260508 09:02:53.465 | INFO | Analyzing chunk frames = [6878208,6878720) | 20260508 09:02:53.465 | INFO | MemInfo: { | 20260508 09:02:53.465 | INFO | "gpu" : " 17 GB", | 20260508 09:02:53.465 | INFO | "huge" : "", | 20260508 09:02:53.465 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:53.465 | INFO | "malloc" : " 14 GB", | 20260508 09:02:53.465 | INFO | "pinned" : " 29 GB" | 20260508 09:02:53.465 | INFO | } | 20260508 09:02:53.465 | INFO | Perf report for the last 5.13609 seconds: | 20260508 09:02:53.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:53.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.465 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 09:02:53.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.465 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:02:53.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:53.466 | INFO | Uploaded 36GiB to GPU in 2.49631s (14.4213GiB/s) | 20260508 09:02:53.466 | INFO | Downloaded 34GiB to GPU in 2.43071s (13.9877GiB/s) | 20260508 09:02:58.602 | INFO | Analyzing chunk frames = [6878720,6879232) | 20260508 09:02:58.602 | INFO | MemInfo: { | 20260508 09:02:58.602 | INFO | "gpu" : " 17 GB", | 20260508 09:02:58.602 | INFO | "huge" : "", | 20260508 09:02:58.602 | INFO | "huge_shared" : " 29 GB", | 20260508 09:02:58.602 | INFO | "malloc" : " 14 GB", | 20260508 09:02:58.602 | INFO | "pinned" : " 29 GB" | 20260508 09:02:58.602 | INFO | } | 20260508 09:02:58.603 | INFO | Perf report for the last 5.13744 seconds: | 20260508 09:02:58.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:02:58.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 09:02:58.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 09:02:58.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:02:58.603 | INFO | Uploaded 36GiB to GPU in 2.49133s (14.4501GiB/s) | 20260508 09:02:58.603 | INFO | Downloaded 34GiB to GPU in 2.40025s (14.1652GiB/s) | 20260508 09:03:03.740 | INFO | Analyzing chunk frames = [6879232,6879744) | 20260508 09:03:03.740 | INFO | MemInfo: { | 20260508 09:03:03.740 | INFO | "gpu" : " 17 GB", | 20260508 09:03:03.740 | INFO | "huge" : "", | 20260508 09:03:03.740 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:03.740 | INFO | "malloc" : " 14 GB", | 20260508 09:03:03.740 | INFO | "pinned" : " 29 GB" | 20260508 09:03:03.740 | INFO | } | 20260508 09:03:03.740 | INFO | Perf report for the last 5.13745 seconds: | 20260508 09:03:03.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:03.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.740 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:03:03.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.740 | INFO | PRE_HQ : 31.9%, 1.0, 16.7ms, 0 | 20260508 09:03:03.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:03.741 | INFO | Uploaded 36GiB to GPU in 2.41772s (14.8901GiB/s) | 20260508 09:03:03.741 | INFO | Downloaded 34GiB to GPU in 2.51135s (13.5385GiB/s) | 20260508 09:03:08.881 | INFO | Analyzing chunk frames = [6879744,6880256) | 20260508 09:03:08.881 | INFO | MemInfo: { | 20260508 09:03:08.881 | INFO | "gpu" : " 17 GB", | 20260508 09:03:08.881 | INFO | "huge" : "", | 20260508 09:03:08.881 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:08.881 | INFO | "malloc" : " 14 GB", | 20260508 09:03:08.881 | INFO | "pinned" : " 29 GB" | 20260508 09:03:08.881 | INFO | } | 20260508 09:03:08.881 | INFO | Perf report for the last 5.14106 seconds: | 20260508 09:03:08.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:08.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 09:03:08.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:03:08.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:08.881 | INFO | Uploaded 36GiB to GPU in 2.40258s (14.9839GiB/s) | 20260508 09:03:08.881 | INFO | Downloaded 34GiB to GPU in 2.42386s (14.0272GiB/s) | 20260508 09:03:14.018 | INFO | Analyzing chunk frames = [6880256,6880768) | 20260508 09:03:14.018 | INFO | MemInfo: { | 20260508 09:03:14.018 | INFO | "gpu" : " 17 GB", | 20260508 09:03:14.018 | INFO | "huge" : "", | 20260508 09:03:14.018 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:14.018 | INFO | "malloc" : " 14 GB", | 20260508 09:03:14.018 | INFO | "pinned" : " 29 GB" | 20260508 09:03:14.018 | INFO | } | 20260508 09:03:14.018 | INFO | Perf report for the last 5.13672 seconds: | 20260508 09:03:14.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:14.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 09:03:14.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:03:14.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:14.018 | INFO | Uploaded 36GiB to GPU in 2.44524s (14.7225GiB/s) | 20260508 09:03:14.018 | INFO | Downloaded 34GiB to GPU in 2.42849s (14.0005GiB/s) | 20260508 09:03:19.154 | INFO | Analyzing chunk frames = [6880768,6881280) | 20260508 09:03:19.154 | INFO | MemInfo: { | 20260508 09:03:19.154 | INFO | "gpu" : " 17 GB", | 20260508 09:03:19.154 | INFO | "huge" : "", | 20260508 09:03:19.155 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:19.155 | INFO | "malloc" : " 14 GB", | 20260508 09:03:19.155 | INFO | "pinned" : " 29 GB" | 20260508 09:03:19.155 | INFO | } | 20260508 09:03:19.155 | INFO | Perf report for the last 5.13696 seconds: | 20260508 09:03:19.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:19.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 09:03:19.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:03:19.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:19.155 | INFO | Uploaded 36GiB to GPU in 2.44847s (14.7031GiB/s) | 20260508 09:03:19.155 | INFO | Downloaded 34GiB to GPU in 2.45157s (13.8687GiB/s) | 20260508 09:03:19.156 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:03:19.156 | INFO | Total mem usage: 16.7 GiB | 20260508 09:03:19.156 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:03:19.156 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:03:19.156 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:03:19.156 | INFO | High Water Breakdown: | 20260508 09:03:19.156 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:03:19.156 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:03:19.156 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:03:19.156 | INFO | ------------------------------------------ | 20260508 09:03:19.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:03:19.156 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:03:19.156 | INFO | Total mem usage: 26.5 GiB | 20260508 09:03:19.156 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:03:19.156 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:03:19.156 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:03:19.156 | INFO | High Water Breakdown: | 20260508 09:03:19.156 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:03:19.156 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:03:19.156 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:03:19.156 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:03:19.156 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:03:19.156 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:03:19.156 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:03:19.156 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:03:19.156 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:03:19.156 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:03:19.156 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:03:19.156 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:03:19.156 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:03:19.156 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:03:19.156 | INFO | ------------------------------------------ | 20260508 09:03:19.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:03:19.156 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:03:19.156 | INFO | Total mem usage: 27.8 GiB | 20260508 09:03:19.156 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:03:19.156 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:03:19.156 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:03:19.156 | INFO | High Water Breakdown: | 20260508 09:03:19.156 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:03:19.156 | INFO | ------------------------------------------ | 20260508 09:03:19.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:03:19.156 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:03:19.156 | INFO | Total mem usage: 15.7 GiB | 20260508 09:03:19.156 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:03:19.156 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:03:19.156 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:03:19.156 | INFO | High Water Breakdown: | 20260508 09:03:19.156 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:03:19.156 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:03:19.156 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:03:19.156 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:03:19.156 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:03:19.156 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:03:19.156 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:03:19.156 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:03:19.156 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:03:19.156 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:03:19.156 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:03:19.156 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:03:19.156 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:03:19.156 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:03:19.156 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:03:19.156 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:03:19.156 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:03:19.156 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:03:19.156 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:03:19.156 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:03:19.156 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:03:19.156 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:03:19.156 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:03:19.156 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:03:19.156 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:03:19.156 | INFO | ------------------------------------------ | 20260508 09:03:19.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:03:19.156 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:03:19.156 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:03:19.156 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:03:19.156 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 7.92e-01 | 20260508 09:03:19.156 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.05e-01, 1.22e-02 / 7.44e-03 | 20260508 09:03:19.156 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.39e+00, 6.00e-01 / 2.76e-01 | 20260508 09:03:19.156 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.44e+00, 2.11e-01 / 1.38e-01 | 20260508 09:03:19.156 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.51e+01, 3.65e+00 / 1.32e+00 | 20260508 09:03:19.156 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.88e+00 / 1.59e+00 | 20260508 09:03:19.156 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.08e-01 / 2.86e-01 | 20260508 09:03:19.156 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.18e+00, 2.47e-01 / 1.37e-01 | 20260508 09:03:19.156 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.06e-01 / 3.39e-01, 4.87e-02 / 3.02e-02 | 20260508 09:03:19.157 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:03:19.157 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:03:19.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:03:19.157 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 24.43%, Upload -- 2.58e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 24.46%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 31.07%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 2.28%, LockUpload -- 2.41e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 2.42%, LockDownload -- 2.56e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 15.32%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:03:19.157 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 09:03:19.252 | INFO | Starting a round of Baz IO | 20260508 09:03:22.663 | INFO | Populating RTMetrics report | 20260508 09:03:22.663 | INFO | RTMetrics report populated | 20260508 09:03:22.663 | INFO | Opening tmp RTMetrics json file | 20260508 09:03:22.674 | INFO | Serializing RTMetrics to json | 20260508 09:03:22.675 | INFO | Serialization done, swapping tmp file now... | 20260508 09:03:22.675 | INFO | RTMetrics done | 20260508 09:03:22.734 | INFO | Finished a round of Baz IO: Wrote 1.49 GiB in 3.481 seconds | 20260508 09:03:22.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:03:22.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268972% / 3.97364e-06% / 7.15256e-05% | 20260508 09:03:22.734 | INFO | Baz IO overall: | 20260508 09:03:22.734 | INFO | BytesWritten : 3009608876641 | 20260508 09:03:22.734 | INFO | NumEvents : 1448234204309 | 20260508 09:03:24.299 | INFO | Analyzing chunk frames = [6881280,6881792) | 20260508 09:03:24.299 | INFO | MemInfo: { | 20260508 09:03:24.299 | INFO | "gpu" : " 17 GB", | 20260508 09:03:24.299 | INFO | "huge" : "", | 20260508 09:03:24.299 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:24.299 | INFO | "malloc" : " 14 GB", | 20260508 09:03:24.299 | INFO | "pinned" : " 29 GB" | 20260508 09:03:24.299 | INFO | } | 20260508 09:03:24.299 | INFO | Perf report for the last 5.14409 seconds: | 20260508 09:03:24.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:24.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:24.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:24.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:24.299 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:03:24.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:24.299 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:03:24.299 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 09:03:24.299 | INFO | BAZWRITER : 67.0%, 2.6, 89.7ms, 0 | 20260508 09:03:24.299 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:03:24.300 | INFO | Uploaded 36GiB to GPU in 2.603s (13.8302GiB/s) | 20260508 09:03:24.300 | INFO | Downloaded 38GiB to GPU in 2.7037s (14.0548GiB/s) | 20260508 09:03:29.441 | INFO | Analyzing chunk frames = [6881792,6882304) | 20260508 09:03:29.441 | INFO | MemInfo: { | 20260508 09:03:29.441 | INFO | "gpu" : " 17 GB", | 20260508 09:03:29.441 | INFO | "huge" : "", | 20260508 09:03:29.441 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:29.441 | INFO | "malloc" : " 14 GB", | 20260508 09:03:29.441 | INFO | "pinned" : " 29 GB" | 20260508 09:03:29.441 | INFO | } | 20260508 09:03:29.441 | INFO | Perf report for the last 5.14214 seconds: | 20260508 09:03:29.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:29.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.442 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 09:03:29.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.442 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:03:29.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:29.443 | INFO | Uploaded 36GiB to GPU in 2.54292s (14.1569GiB/s) | 20260508 09:03:29.443 | INFO | Downloaded 34GiB to GPU in 2.45605s (13.8434GiB/s) | 20260508 09:03:34.577 | INFO | Analyzing chunk frames = [6882304,6882816) | 20260508 09:03:34.577 | INFO | MemInfo: { | 20260508 09:03:34.577 | INFO | "gpu" : " 17 GB", | 20260508 09:03:34.577 | INFO | "huge" : "", | 20260508 09:03:34.577 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:34.577 | INFO | "malloc" : " 14 GB", | 20260508 09:03:34.577 | INFO | "pinned" : " 29 GB" | 20260508 09:03:34.577 | INFO | } | 20260508 09:03:34.577 | INFO | Perf report for the last 5.13585 seconds: | 20260508 09:03:34.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:34.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.577 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 09:03:34.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.577 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:03:34.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:34.578 | INFO | Uploaded 36GiB to GPU in 2.49474s (14.4303GiB/s) | 20260508 09:03:34.578 | INFO | Downloaded 34GiB to GPU in 2.4324s (13.978GiB/s) | 20260508 09:03:39.712 | INFO | Analyzing chunk frames = [6882816,6883328) | 20260508 09:03:39.712 | INFO | MemInfo: { | 20260508 09:03:39.712 | INFO | "gpu" : " 17 GB", | 20260508 09:03:39.712 | INFO | "huge" : "", | 20260508 09:03:39.712 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:39.712 | INFO | "malloc" : " 14 GB", | 20260508 09:03:39.712 | INFO | "pinned" : " 29 GB" | 20260508 09:03:39.712 | INFO | } | 20260508 09:03:39.713 | INFO | Perf report for the last 5.13563 seconds: | 20260508 09:03:39.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:39.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 09:03:39.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:03:39.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:39.713 | INFO | Uploaded 36GiB to GPU in 2.52699s (14.2462GiB/s) | 20260508 09:03:39.713 | INFO | Downloaded 34GiB to GPU in 2.42771s (14.005GiB/s) | 20260508 09:03:44.858 | INFO | Analyzing chunk frames = [6883328,6883840) | 20260508 09:03:44.858 | INFO | MemInfo: { | 20260508 09:03:44.858 | INFO | "gpu" : " 17 GB", | 20260508 09:03:44.858 | INFO | "huge" : "", | 20260508 09:03:44.858 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:44.858 | INFO | "malloc" : " 14 GB", | 20260508 09:03:44.858 | INFO | "pinned" : " 29 GB" | 20260508 09:03:44.858 | INFO | } | 20260508 09:03:44.858 | INFO | Perf report for the last 5.14555 seconds: | 20260508 09:03:44.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:44.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 09:03:44.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:03:44.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:44.858 | INFO | Uploaded 36GiB to GPU in 2.46963s (14.5771GiB/s) | 20260508 09:03:44.858 | INFO | Downloaded 34GiB to GPU in 2.53121s (13.4323GiB/s) | 20260508 09:03:49.995 | INFO | Analyzing chunk frames = [6883840,6884352) | 20260508 09:03:49.995 | INFO | MemInfo: { | 20260508 09:03:49.995 | INFO | "gpu" : " 17 GB", | 20260508 09:03:49.995 | INFO | "huge" : "", | 20260508 09:03:49.995 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:49.995 | INFO | "malloc" : " 14 GB", | 20260508 09:03:49.995 | INFO | "pinned" : " 29 GB" | 20260508 09:03:49.995 | INFO | } | 20260508 09:03:49.995 | INFO | Perf report for the last 5.13737 seconds: | 20260508 09:03:49.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:49.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.996 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:03:49.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.996 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:03:49.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:49.996 | INFO | Uploaded 36GiB to GPU in 2.39591s (15.0256GiB/s) | 20260508 09:03:49.996 | INFO | Downloaded 34GiB to GPU in 2.43841s (13.9435GiB/s) | 20260508 09:03:55.136 | INFO | Analyzing chunk frames = [6884352,6884864) | 20260508 09:03:55.136 | INFO | MemInfo: { | 20260508 09:03:55.136 | INFO | "gpu" : " 17 GB", | 20260508 09:03:55.136 | INFO | "huge" : "", | 20260508 09:03:55.136 | INFO | "huge_shared" : " 29 GB", | 20260508 09:03:55.136 | INFO | "malloc" : " 14 GB", | 20260508 09:03:55.137 | INFO | "pinned" : " 29 GB" | 20260508 09:03:55.137 | INFO | } | 20260508 09:03:55.137 | INFO | Perf report for the last 5.14109 seconds: | 20260508 09:03:55.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:03:55.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 09:03:55.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:03:55.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:03:55.137 | INFO | Uploaded 36GiB to GPU in 2.44971s (14.6956GiB/s) | 20260508 09:03:55.137 | INFO | Downloaded 34GiB to GPU in 2.44196s (13.9232GiB/s) | 20260508 09:04:00.278 | INFO | Analyzing chunk frames = [6884864,6885376) | 20260508 09:04:00.278 | INFO | MemInfo: { | 20260508 09:04:00.278 | INFO | "gpu" : " 17 GB", | 20260508 09:04:00.278 | INFO | "huge" : "", | 20260508 09:04:00.278 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:00.278 | INFO | "malloc" : " 14 GB", | 20260508 09:04:00.278 | INFO | "pinned" : " 29 GB" | 20260508 09:04:00.278 | INFO | } | 20260508 09:04:00.278 | INFO | Perf report for the last 5.14158 seconds: | 20260508 09:04:00.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:00.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:04:00.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:04:00.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:00.278 | INFO | Uploaded 36GiB to GPU in 2.40872s (14.9457GiB/s) | 20260508 09:04:00.278 | INFO | Downloaded 34GiB to GPU in 2.43102s (13.9859GiB/s) | 20260508 09:04:00.377 | INFO | Starting a round of Baz IO | 20260508 09:04:03.780 | INFO | Populating RTMetrics report | 20260508 09:04:03.781 | INFO | RTMetrics report populated | 20260508 09:04:03.781 | INFO | Opening tmp RTMetrics json file | 20260508 09:04:03.791 | INFO | Serializing RTMetrics to json | 20260508 09:04:03.792 | INFO | Serialization done, swapping tmp file now... | 20260508 09:04:03.793 | INFO | RTMetrics done | 20260508 09:04:03.852 | INFO | Finished a round of Baz IO: Wrote 1.489 GiB in 3.474 seconds | 20260508 09:04:03.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:04:03.852 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268149% / 3.97364e-06% / 6.75519e-05% | 20260508 09:04:03.852 | INFO | Baz IO overall: | 20260508 09:04:03.852 | INFO | BytesWritten : 3011207576675 | 20260508 09:04:03.852 | INFO | NumEvents : 1448990357066 | 20260508 09:04:05.409 | INFO | Analyzing chunk frames = [6885376,6885888) | 20260508 09:04:05.409 | INFO | MemInfo: { | 20260508 09:04:05.409 | INFO | "gpu" : " 17 GB", | 20260508 09:04:05.409 | INFO | "huge" : "", | 20260508 09:04:05.409 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:05.409 | INFO | "malloc" : " 14 GB", | 20260508 09:04:05.409 | INFO | "pinned" : " 29 GB" | 20260508 09:04:05.409 | INFO | } | 20260508 09:04:05.409 | INFO | Perf report for the last 5.13081 seconds: | 20260508 09:04:05.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:05.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:05.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:05.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:05.409 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 09:04:05.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:05.409 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:04:05.409 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 09:04:05.409 | INFO | BAZWRITER : 67.1%, 2.6, 88.2ms, 0 | 20260508 09:04:05.409 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:04:05.409 | INFO | Uploaded 36GiB to GPU in 2.56237s (14.0495GiB/s) | 20260508 09:04:05.410 | INFO | Downloaded 38GiB to GPU in 2.71085s (14.0178GiB/s) | 20260508 09:04:10.557 | INFO | Analyzing chunk frames = [6885888,6886400) | 20260508 09:04:10.557 | INFO | MemInfo: { | 20260508 09:04:10.557 | INFO | "gpu" : " 17 GB", | 20260508 09:04:10.557 | INFO | "huge" : "", | 20260508 09:04:10.557 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:10.558 | INFO | "malloc" : " 14 GB", | 20260508 09:04:10.558 | INFO | "pinned" : " 29 GB" | 20260508 09:04:10.558 | INFO | } | 20260508 09:04:10.558 | INFO | Perf report for the last 5.14872 seconds: | 20260508 09:04:10.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:10.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:04:10.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 09:04:10.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:10.558 | INFO | Uploaded 36GiB to GPU in 2.46318s (14.6152GiB/s) | 20260508 09:04:10.558 | INFO | Downloaded 34GiB to GPU in 2.45005s (13.8773GiB/s) | 20260508 09:04:15.691 | INFO | Analyzing chunk frames = [6886400,6886912) | 20260508 09:04:15.691 | INFO | MemInfo: { | 20260508 09:04:15.691 | INFO | "gpu" : " 17 GB", | 20260508 09:04:15.691 | INFO | "huge" : "", | 20260508 09:04:15.691 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:15.692 | INFO | "malloc" : " 14 GB", | 20260508 09:04:15.692 | INFO | "pinned" : " 29 GB" | 20260508 09:04:15.692 | INFO | } | 20260508 09:04:15.692 | INFO | Perf report for the last 5.1339 seconds: | 20260508 09:04:15.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:15.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:04:15.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:04:15.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:15.692 | INFO | Uploaded 36GiB to GPU in 2.43982s (14.7552GiB/s) | 20260508 09:04:15.692 | INFO | Downloaded 34GiB to GPU in 2.44288s (13.918GiB/s) | 20260508 09:04:20.833 | INFO | Analyzing chunk frames = [6886912,6887424) | 20260508 09:04:20.833 | INFO | MemInfo: { | 20260508 09:04:20.833 | INFO | "gpu" : " 17 GB", | 20260508 09:04:20.833 | INFO | "huge" : "", | 20260508 09:04:20.833 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:20.833 | INFO | "malloc" : " 14 GB", | 20260508 09:04:20.833 | INFO | "pinned" : " 29 GB" | 20260508 09:04:20.833 | INFO | } | 20260508 09:04:20.833 | INFO | Perf report for the last 5.14154 seconds: | 20260508 09:04:20.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:20.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.833 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:04:20.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.833 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:04:20.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:20.834 | INFO | Uploaded 36GiB to GPU in 2.49436s (14.4326GiB/s) | 20260508 09:04:20.834 | INFO | Downloaded 34GiB to GPU in 2.44127s (13.9272GiB/s) | 20260508 09:04:25.967 | INFO | Analyzing chunk frames = [6887424,6887936) | 20260508 09:04:25.967 | INFO | MemInfo: { | 20260508 09:04:25.967 | INFO | "gpu" : " 17 GB", | 20260508 09:04:25.967 | INFO | "huge" : "", | 20260508 09:04:25.967 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:25.967 | INFO | "malloc" : " 14 GB", | 20260508 09:04:25.967 | INFO | "pinned" : " 29 GB" | 20260508 09:04:25.967 | INFO | } | 20260508 09:04:25.967 | INFO | Perf report for the last 5.13392 seconds: | 20260508 09:04:25.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:25.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.967 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:04:25.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.967 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 09:04:25.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:25.968 | INFO | Uploaded 36GiB to GPU in 2.41684s (14.8955GiB/s) | 20260508 09:04:25.968 | INFO | Downloaded 34GiB to GPU in 2.54857s (13.3408GiB/s) | 20260508 09:04:31.111 | INFO | Analyzing chunk frames = [6887936,6888448) | 20260508 09:04:31.111 | INFO | MemInfo: { | 20260508 09:04:31.111 | INFO | "gpu" : " 17 GB", | 20260508 09:04:31.111 | INFO | "huge" : "", | 20260508 09:04:31.111 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:31.111 | INFO | "malloc" : " 14 GB", | 20260508 09:04:31.111 | INFO | "pinned" : " 29 GB" | 20260508 09:04:31.111 | INFO | } | 20260508 09:04:31.112 | INFO | Perf report for the last 5.14441 seconds: | 20260508 09:04:31.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:31.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 09:04:31.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:04:31.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:31.112 | INFO | Uploaded 36GiB to GPU in 2.38192s (15.1139GiB/s) | 20260508 09:04:31.112 | INFO | Downloaded 34GiB to GPU in 2.43948s (13.9374GiB/s) | 20260508 09:04:36.247 | INFO | Analyzing chunk frames = [6888448,6888960) | 20260508 09:04:36.247 | INFO | MemInfo: { | 20260508 09:04:36.247 | INFO | "gpu" : " 17 GB", | 20260508 09:04:36.247 | INFO | "huge" : "", | 20260508 09:04:36.247 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:36.247 | INFO | "malloc" : " 14 GB", | 20260508 09:04:36.247 | INFO | "pinned" : " 29 GB" | 20260508 09:04:36.247 | INFO | } | 20260508 09:04:36.247 | INFO | Perf report for the last 5.13589 seconds: | 20260508 09:04:36.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:36.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.248 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:04:36.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.248 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:04:36.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:36.248 | INFO | Uploaded 36GiB to GPU in 2.38902s (15.0689GiB/s) | 20260508 09:04:36.248 | INFO | Downloaded 34GiB to GPU in 2.43349s (13.9717GiB/s) | 20260508 09:04:41.390 | INFO | Analyzing chunk frames = [6888960,6889472) | 20260508 09:04:41.390 | INFO | MemInfo: { | 20260508 09:04:41.390 | INFO | "gpu" : " 17 GB", | 20260508 09:04:41.390 | INFO | "huge" : "", | 20260508 09:04:41.390 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:41.390 | INFO | "malloc" : " 14 GB", | 20260508 09:04:41.390 | INFO | "pinned" : " 29 GB" | 20260508 09:04:41.390 | INFO | } | 20260508 09:04:41.390 | INFO | Perf report for the last 5.14269 seconds: | 20260508 09:04:41.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:41.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.390 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 09:04:41.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.390 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:04:41.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:41.391 | INFO | Uploaded 36GiB to GPU in 2.44898s (14.7GiB/s) | 20260508 09:04:41.391 | INFO | Downloaded 34GiB to GPU in 2.45098s (13.872GiB/s) | 20260508 09:04:41.391 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:04:41.391 | INFO | Total mem usage: 16.7 GiB | 20260508 09:04:41.391 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:04:41.391 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:04:41.391 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:04:41.391 | INFO | High Water Breakdown: | 20260508 09:04:41.391 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:04:41.391 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:04:41.391 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:04:41.391 | INFO | ------------------------------------------ | 20260508 09:04:41.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:04:41.391 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:04:41.391 | INFO | Total mem usage: 26.5 GiB | 20260508 09:04:41.391 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:04:41.391 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:04:41.391 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:04:41.391 | INFO | High Water Breakdown: | 20260508 09:04:41.391 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:04:41.391 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:04:41.391 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:04:41.391 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:04:41.391 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:04:41.391 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:04:41.391 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:04:41.391 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:04:41.391 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:04:41.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:04:41.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:04:41.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:04:41.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:04:41.391 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:04:41.391 | INFO | ------------------------------------------ | 20260508 09:04:41.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:04:41.391 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:04:41.391 | INFO | Total mem usage: 27.8 GiB | 20260508 09:04:41.391 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:04:41.391 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:04:41.391 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:04:41.391 | INFO | High Water Breakdown: | 20260508 09:04:41.391 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:04:41.391 | INFO | ------------------------------------------ | 20260508 09:04:41.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:04:41.391 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:04:41.391 | INFO | Total mem usage: 15.7 GiB | 20260508 09:04:41.391 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:04:41.391 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:04:41.391 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:04:41.391 | INFO | High Water Breakdown: | 20260508 09:04:41.391 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:04:41.391 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:04:41.391 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:04:41.391 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:04:41.391 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:04:41.391 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:04:41.391 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:04:41.391 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:04:41.391 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:04:41.391 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:04:41.391 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:04:41.391 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:04:41.391 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:04:41.391 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:04:41.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:04:41.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:04:41.391 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:04:41.391 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:04:41.391 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:04:41.391 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:04:41.391 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:04:41.391 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:04:41.391 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:04:41.391 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:04:41.391 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:04:41.391 | INFO | ------------------------------------------ | 20260508 09:04:41.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:04:41.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:04:41.391 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:04:41.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:04:41.391 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 6.83e-01 | 20260508 09:04:41.391 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.05e-01, 1.37e-02 / 5.03e-03 | 20260508 09:04:41.391 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.36e+00, 5.98e-01 / 3.16e-01 | 20260508 09:04:41.391 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.45e+00, 2.11e-01 / 1.25e-01 | 20260508 09:04:41.391 | INFO | PerfReport: 15.91%, DME -- 4.17e+01 / 4.48e+01, 3.71e+00 / 1.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.91e+00 / 1.41e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.32e+00 / 2.44e+00, 4.07e-01 / 2.74e-01 | 20260508 09:04:41.391 | INFO | PerfReport: 3.03%, Metrics -- 9.95e-01 / 1.12e+00, 2.50e-01 / 1.20e-01 | 20260508 09:04:41.391 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.30e-01, 4.83e-02 / 2.73e-02 | 20260508 09:04:41.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:04:41.391 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:04:41.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:04:41.391 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 24.37%, Upload -- 2.57e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 24.55%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 31.09%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 2.51%, LockDownload -- 2.64e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 15.31%, LockCompute -- 1.61e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:04:41.391 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 09:04:41.488 | INFO | Starting a round of Baz IO | 20260508 09:04:44.906 | INFO | Populating RTMetrics report | 20260508 09:04:44.907 | INFO | RTMetrics report populated | 20260508 09:04:44.907 | INFO | Opening tmp RTMetrics json file | 20260508 09:04:44.916 | INFO | Serializing RTMetrics to json | 20260508 09:04:44.918 | INFO | Serialization done, swapping tmp file now... | 20260508 09:04:44.918 | INFO | RTMetrics done | 20260508 09:04:44.979 | INFO | Finished a round of Baz IO: Wrote 1.49 GiB in 3.491 seconds | 20260508 09:04:44.979 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:04:44.979 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.269302% / 0% / 6.35783e-05% | 20260508 09:04:44.979 | INFO | Baz IO overall: | 20260508 09:04:44.979 | INFO | BytesWritten : 3012807699709 | 20260508 09:04:44.979 | INFO | NumEvents : 1449747524545 | 20260508 09:04:46.524 | INFO | Analyzing chunk frames = [6889472,6889984) | 20260508 09:04:46.524 | INFO | MemInfo: { | 20260508 09:04:46.524 | INFO | "gpu" : " 17 GB", | 20260508 09:04:46.524 | INFO | "huge" : "", | 20260508 09:04:46.524 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:46.524 | INFO | "malloc" : " 14 GB", | 20260508 09:04:46.524 | INFO | "pinned" : " 29 GB" | 20260508 09:04:46.524 | INFO | } | 20260508 09:04:46.525 | INFO | Perf report for the last 5.13435 seconds: | 20260508 09:04:46.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:46.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:46.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:46.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:46.525 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 09:04:46.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:46.525 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 09:04:46.525 | INFO | PRE_HQ : 65.4%, 1.6, 54.8ms, 0 | 20260508 09:04:46.525 | INFO | BAZWRITER : 67.3%, 2.6, 88.4ms, 0 | 20260508 09:04:46.525 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:04:46.525 | INFO | Uploaded 36GiB to GPU in 2.62727s (13.7024GiB/s) | 20260508 09:04:46.525 | INFO | Downloaded 38GiB to GPU in 2.69379s (14.1065GiB/s) | 20260508 09:04:51.663 | INFO | Analyzing chunk frames = [6889984,6890496) | 20260508 09:04:51.663 | INFO | MemInfo: { | 20260508 09:04:51.663 | INFO | "gpu" : " 17 GB", | 20260508 09:04:51.663 | INFO | "huge" : "", | 20260508 09:04:51.663 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:51.663 | INFO | "malloc" : " 14 GB", | 20260508 09:04:51.663 | INFO | "pinned" : " 29 GB" | 20260508 09:04:51.663 | INFO | } | 20260508 09:04:51.663 | INFO | Perf report for the last 5.13841 seconds: | 20260508 09:04:51.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:51.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 09:04:51.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:04:51.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:51.663 | INFO | Uploaded 36GiB to GPU in 2.54171s (14.1637GiB/s) | 20260508 09:04:51.663 | INFO | Downloaded 34GiB to GPU in 2.44594s (13.9006GiB/s) | 20260508 09:04:56.807 | INFO | Analyzing chunk frames = [6890496,6891008) | 20260508 09:04:56.808 | INFO | MemInfo: { | 20260508 09:04:56.808 | INFO | "gpu" : " 17 GB", | 20260508 09:04:56.808 | INFO | "huge" : "", | 20260508 09:04:56.808 | INFO | "huge_shared" : " 29 GB", | 20260508 09:04:56.808 | INFO | "malloc" : " 14 GB", | 20260508 09:04:56.808 | INFO | "pinned" : " 29 GB" | 20260508 09:04:56.808 | INFO | } | 20260508 09:04:56.808 | INFO | Perf report for the last 5.14462 seconds: | 20260508 09:04:56.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:04:56.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 09:04:56.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:04:56.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:04:56.808 | INFO | Uploaded 36GiB to GPU in 2.52647s (14.2491GiB/s) | 20260508 09:04:56.808 | INFO | Downloaded 34GiB to GPU in 2.44135s (13.9267GiB/s) | 20260508 09:05:01.951 | INFO | Analyzing chunk frames = [6891008,6891520) | 20260508 09:05:01.951 | INFO | MemInfo: { | 20260508 09:05:01.951 | INFO | "gpu" : " 17 GB", | 20260508 09:05:01.951 | INFO | "huge" : "", | 20260508 09:05:01.951 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:01.951 | INFO | "malloc" : " 14 GB", | 20260508 09:05:01.951 | INFO | "pinned" : " 29 GB" | 20260508 09:05:01.951 | INFO | } | 20260508 09:05:01.951 | INFO | Perf report for the last 5.14354 seconds: | 20260508 09:05:01.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:01.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.951 | INFO | ANALYSIS : 67.0%, 3.0, 103.7ms, 0 | 20260508 09:05:01.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.951 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:05:01.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:01.952 | INFO | Uploaded 36GiB to GPU in 2.5937s (13.8798GiB/s) | 20260508 09:05:01.952 | INFO | Downloaded 34GiB to GPU in 2.42286s (14.033GiB/s) | 20260508 09:05:07.091 | INFO | Analyzing chunk frames = [6891520,6892032) | 20260508 09:05:07.091 | INFO | MemInfo: { | 20260508 09:05:07.091 | INFO | "gpu" : " 17 GB", | 20260508 09:05:07.091 | INFO | "huge" : "", | 20260508 09:05:07.091 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:07.091 | INFO | "malloc" : " 14 GB", | 20260508 09:05:07.091 | INFO | "pinned" : " 29 GB" | 20260508 09:05:07.091 | INFO | } | 20260508 09:05:07.091 | INFO | Perf report for the last 5.14017 seconds: | 20260508 09:05:07.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:07.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.091 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 09:05:07.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.091 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:05:07.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:07.092 | INFO | Uploaded 36GiB to GPU in 2.47504s (14.5452GiB/s) | 20260508 09:05:07.092 | INFO | Downloaded 34GiB to GPU in 2.54772s (13.3453GiB/s) | 20260508 09:05:12.225 | INFO | Analyzing chunk frames = [6892032,6892544) | 20260508 09:05:12.225 | INFO | MemInfo: { | 20260508 09:05:12.225 | INFO | "gpu" : " 17 GB", | 20260508 09:05:12.225 | INFO | "huge" : "", | 20260508 09:05:12.225 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:12.226 | INFO | "malloc" : " 14 GB", | 20260508 09:05:12.226 | INFO | "pinned" : " 29 GB" | 20260508 09:05:12.226 | INFO | } | 20260508 09:05:12.226 | INFO | Perf report for the last 5.13431 seconds: | 20260508 09:05:12.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:12.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260508 09:05:12.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:05:12.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:12.226 | INFO | Uploaded 36GiB to GPU in 2.39065s (15.0587GiB/s) | 20260508 09:05:12.226 | INFO | Downloaded 34GiB to GPU in 2.44332s (13.9155GiB/s) | 20260508 09:05:17.366 | INFO | Analyzing chunk frames = [6892544,6893056) | 20260508 09:05:17.366 | INFO | MemInfo: { | 20260508 09:05:17.366 | INFO | "gpu" : " 17 GB", | 20260508 09:05:17.366 | INFO | "huge" : "", | 20260508 09:05:17.366 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:17.366 | INFO | "malloc" : " 14 GB", | 20260508 09:05:17.366 | INFO | "pinned" : " 29 GB" | 20260508 09:05:17.366 | INFO | } | 20260508 09:05:17.366 | INFO | Perf report for the last 5.14013 seconds: | 20260508 09:05:17.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:17.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:05:17.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 09:05:17.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:17.366 | INFO | Uploaded 36GiB to GPU in 2.4087s (14.9458GiB/s) | 20260508 09:05:17.366 | INFO | Downloaded 34GiB to GPU in 2.44401s (13.9116GiB/s) | 20260508 09:05:22.503 | INFO | Analyzing chunk frames = [6893056,6893568) | 20260508 09:05:22.503 | INFO | MemInfo: { | 20260508 09:05:22.503 | INFO | "gpu" : " 17 GB", | 20260508 09:05:22.503 | INFO | "huge" : "", | 20260508 09:05:22.503 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:22.503 | INFO | "malloc" : " 14 GB", | 20260508 09:05:22.503 | INFO | "pinned" : " 29 GB" | 20260508 09:05:22.503 | INFO | } | 20260508 09:05:22.503 | INFO | Perf report for the last 5.1369 seconds: | 20260508 09:05:22.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:22.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:05:22.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:05:22.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:22.503 | INFO | Uploaded 36GiB to GPU in 2.40988s (14.9385GiB/s) | 20260508 09:05:22.503 | INFO | Downloaded 34GiB to GPU in 2.42638s (14.0126GiB/s) | 20260508 09:05:22.600 | INFO | Starting a round of Baz IO | 20260508 09:05:26.021 | INFO | Populating RTMetrics report | 20260508 09:05:26.022 | INFO | RTMetrics report populated | 20260508 09:05:26.022 | INFO | Opening tmp RTMetrics json file | 20260508 09:05:26.031 | INFO | Serializing RTMetrics to json | 20260508 09:05:26.032 | INFO | Serialization done, swapping tmp file now... | 20260508 09:05:26.033 | INFO | RTMetrics done | 20260508 09:05:26.089 | INFO | Finished a round of Baz IO: Wrote 1.488 GiB in 3.489 seconds | 20260508 09:05:26.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.35% / 0.00% / 59.65% | 20260508 09:05:26.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.267661% / 3.97364e-06% / 6.75519e-05% | 20260508 09:05:26.089 | INFO | Baz IO overall: | 20260508 09:05:26.089 | INFO | BytesWritten : 3014405878179 | 20260508 09:05:26.089 | INFO | NumEvents : 1450503582517 | 20260508 09:05:27.651 | INFO | Analyzing chunk frames = [6893568,6894080) | 20260508 09:05:27.651 | INFO | MemInfo: { | 20260508 09:05:27.651 | INFO | "gpu" : " 17 GB", | 20260508 09:05:27.651 | INFO | "huge" : "", | 20260508 09:05:27.651 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:27.651 | INFO | "malloc" : " 14 GB", | 20260508 09:05:27.651 | INFO | "pinned" : " 29 GB" | 20260508 09:05:27.651 | INFO | } | 20260508 09:05:27.651 | INFO | Perf report for the last 5.14844 seconds: | 20260508 09:05:27.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:27.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:27.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:27.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:27.651 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 09:05:27.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:27.651 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:05:27.651 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 09:05:27.651 | INFO | BAZWRITER : 67.3%, 2.6, 89.1ms, 0 | 20260508 09:05:27.651 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 09:05:27.652 | INFO | Uploaded 36GiB to GPU in 2.52358s (14.2655GiB/s) | 20260508 09:05:27.652 | INFO | Downloaded 38GiB to GPU in 2.71485s (13.9971GiB/s) | 20260508 09:05:32.781 | INFO | Analyzing chunk frames = [6894080,6894592) | 20260508 09:05:32.781 | INFO | MemInfo: { | 20260508 09:05:32.781 | INFO | "gpu" : " 17 GB", | 20260508 09:05:32.781 | INFO | "huge" : "", | 20260508 09:05:32.781 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:32.781 | INFO | "malloc" : " 14 GB", | 20260508 09:05:32.782 | INFO | "pinned" : " 29 GB" | 20260508 09:05:32.782 | INFO | } | 20260508 09:05:32.782 | INFO | Perf report for the last 5.1305 seconds: | 20260508 09:05:32.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:32.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | ANALYSIS : 66.9%, 3.0, 104.0ms, 0 | 20260508 09:05:32.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:05:32.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:32.782 | INFO | Uploaded 36GiB to GPU in 2.51634s (14.3065GiB/s) | 20260508 09:05:32.782 | INFO | Downloaded 34GiB to GPU in 2.46274s (13.8058GiB/s) | 20260508 09:05:37.921 | INFO | Analyzing chunk frames = [6894592,6895104) | 20260508 09:05:37.921 | INFO | MemInfo: { | 20260508 09:05:37.921 | INFO | "gpu" : " 17 GB", | 20260508 09:05:37.921 | INFO | "huge" : "", | 20260508 09:05:37.921 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:37.921 | INFO | "malloc" : " 14 GB", | 20260508 09:05:37.921 | INFO | "pinned" : " 29 GB" | 20260508 09:05:37.921 | INFO | } | 20260508 09:05:37.921 | INFO | Perf report for the last 5.13925 seconds: | 20260508 09:05:37.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:37.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:05:37.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:05:37.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:37.921 | INFO | Uploaded 36GiB to GPU in 2.49441s (14.4323GiB/s) | 20260508 09:05:37.921 | INFO | Downloaded 34GiB to GPU in 2.44443s (13.9092GiB/s) | 20260508 09:05:43.063 | INFO | Analyzing chunk frames = [6895104,6895616) | 20260508 09:05:43.063 | INFO | MemInfo: { | 20260508 09:05:43.063 | INFO | "gpu" : " 17 GB", | 20260508 09:05:43.063 | INFO | "huge" : "", | 20260508 09:05:43.063 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:43.063 | INFO | "malloc" : " 14 GB", | 20260508 09:05:43.063 | INFO | "pinned" : " 29 GB" | 20260508 09:05:43.063 | INFO | } | 20260508 09:05:43.063 | INFO | Perf report for the last 5.14177 seconds: | 20260508 09:05:43.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:43.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:05:43.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:05:43.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:43.063 | INFO | Uploaded 36GiB to GPU in 2.49659s (14.4197GiB/s) | 20260508 09:05:43.063 | INFO | Downloaded 34GiB to GPU in 2.44363s (13.9137GiB/s) | 20260508 09:05:48.201 | INFO | Analyzing chunk frames = [6895616,6896128) | 20260508 09:05:48.201 | INFO | MemInfo: { | 20260508 09:05:48.201 | INFO | "gpu" : " 17 GB", | 20260508 09:05:48.201 | INFO | "huge" : "", | 20260508 09:05:48.201 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:48.201 | INFO | "malloc" : " 14 GB", | 20260508 09:05:48.201 | INFO | "pinned" : " 29 GB" | 20260508 09:05:48.201 | INFO | } | 20260508 09:05:48.201 | INFO | Perf report for the last 5.13849 seconds: | 20260508 09:05:48.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:48.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.201 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 09:05:48.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.201 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:05:48.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:48.202 | INFO | Uploaded 36GiB to GPU in 2.40968s (14.9397GiB/s) | 20260508 09:05:48.202 | INFO | Downloaded 34GiB to GPU in 2.53129s (13.4319GiB/s) | 20260508 09:05:53.338 | INFO | Analyzing chunk frames = [6896128,6896640) | 20260508 09:05:53.338 | INFO | MemInfo: { | 20260508 09:05:53.338 | INFO | "gpu" : " 17 GB", | 20260508 09:05:53.338 | INFO | "huge" : "", | 20260508 09:05:53.338 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:53.338 | INFO | "malloc" : " 14 GB", | 20260508 09:05:53.339 | INFO | "pinned" : " 29 GB" | 20260508 09:05:53.339 | INFO | } | 20260508 09:05:53.339 | INFO | Perf report for the last 5.13769 seconds: | 20260508 09:05:53.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:53.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | ANALYSIS : 65.6%, 3.0, 100.9ms, 0 | 20260508 09:05:53.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | PRE_HQ : 31.4%, 1.0, 16.1ms, 0 | 20260508 09:05:53.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:53.339 | INFO | Uploaded 36GiB to GPU in 2.39278s (15.0453GiB/s) | 20260508 09:05:53.340 | INFO | Downloaded 34GiB to GPU in 2.44021s (13.9332GiB/s) | 20260508 09:05:58.475 | INFO | Analyzing chunk frames = [6896640,6897152) | 20260508 09:05:58.475 | INFO | MemInfo: { | 20260508 09:05:58.475 | INFO | "gpu" : " 17 GB", | 20260508 09:05:58.475 | INFO | "huge" : "", | 20260508 09:05:58.475 | INFO | "huge_shared" : " 29 GB", | 20260508 09:05:58.475 | INFO | "malloc" : " 14 GB", | 20260508 09:05:58.476 | INFO | "pinned" : " 29 GB" | 20260508 09:05:58.476 | INFO | } | 20260508 09:05:58.476 | INFO | Perf report for the last 5.13667 seconds: | 20260508 09:05:58.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:05:58.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 09:05:58.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 09:05:58.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:05:58.476 | INFO | Uploaded 36GiB to GPU in 2.3736s (15.1669GiB/s) | 20260508 09:05:58.476 | INFO | Downloaded 34GiB to GPU in 2.43751s (13.9487GiB/s) | 20260508 09:06:03.620 | INFO | Analyzing chunk frames = [6897152,6897664) | 20260508 09:06:03.620 | INFO | MemInfo: { | 20260508 09:06:03.620 | INFO | "gpu" : " 17 GB", | 20260508 09:06:03.620 | INFO | "huge" : "", | 20260508 09:06:03.620 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:03.620 | INFO | "malloc" : " 14 GB", | 20260508 09:06:03.620 | INFO | "pinned" : " 29 GB" | 20260508 09:06:03.620 | INFO | } | 20260508 09:06:03.620 | INFO | Perf report for the last 5.14446 seconds: | 20260508 09:06:03.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:03.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.620 | INFO | ANALYSIS : 65.8%, 3.0, 103.0ms, 0 | 20260508 09:06:03.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.620 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260508 09:06:03.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:03.621 | INFO | Uploaded 36GiB to GPU in 2.43941s (14.7576GiB/s) | 20260508 09:06:03.621 | INFO | Downloaded 34GiB to GPU in 2.4456s (13.9025GiB/s) | 20260508 09:06:03.621 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:06:03.621 | INFO | Total mem usage: 16.7 GiB | 20260508 09:06:03.621 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:06:03.621 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:06:03.621 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:06:03.621 | INFO | High Water Breakdown: | 20260508 09:06:03.621 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:06:03.621 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:06:03.621 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:06:03.621 | INFO | ------------------------------------------ | 20260508 09:06:03.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:06:03.621 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:06:03.621 | INFO | Total mem usage: 26.5 GiB | 20260508 09:06:03.621 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:06:03.621 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:06:03.621 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:06:03.621 | INFO | High Water Breakdown: | 20260508 09:06:03.621 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:06:03.621 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:06:03.621 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:06:03.621 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:06:03.621 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:06:03.621 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:06:03.621 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:06:03.621 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:06:03.621 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:06:03.621 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:06:03.621 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:06:03.621 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:06:03.621 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:06:03.621 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:06:03.621 | INFO | ------------------------------------------ | 20260508 09:06:03.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:06:03.621 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:06:03.621 | INFO | Total mem usage: 27.8 GiB | 20260508 09:06:03.622 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:06:03.622 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:06:03.622 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:06:03.622 | INFO | High Water Breakdown: | 20260508 09:06:03.622 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:06:03.622 | INFO | ------------------------------------------ | 20260508 09:06:03.622 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:06:03.622 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:06:03.622 | INFO | Total mem usage: 15.7 GiB | 20260508 09:06:03.622 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:06:03.622 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:06:03.622 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:06:03.622 | INFO | High Water Breakdown: | 20260508 09:06:03.622 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:06:03.622 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:06:03.622 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:06:03.622 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:06:03.622 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:06:03.622 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:06:03.622 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:06:03.622 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:06:03.622 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:06:03.622 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:06:03.622 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:06:03.622 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:06:03.622 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:06:03.622 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:06:03.622 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:06:03.622 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:06:03.622 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:06:03.622 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:06:03.622 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:06:03.622 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:06:03.622 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:06:03.622 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:06:03.622 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:06:03.622 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:06:03.622 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:06:03.622 | INFO | ------------------------------------------ | 20260508 09:06:03.622 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:06:03.622 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:06:03.622 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:06:03.622 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:06:03.622 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.42e+01 / 6.63e-01 | 20260508 09:06:03.622 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.06e-01, 1.15e-02 / 5.73e-03 | 20260508 09:06:03.622 | INFO | PerfReport: 12.13%, Baseline -- 3.98e+00 / 4.49e+00, 5.92e-01 / 2.49e-01 | 20260508 09:06:03.622 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.20e-01 | 20260508 09:06:03.622 | INFO | PerfReport: 15.92%, DME -- 4.17e+01 / 4.49e+01, 3.69e+00 / 1.01e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.89e+00 / 1.36e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.10e-01 / 2.94e-01 | 20260508 09:06:03.622 | INFO | PerfReport: 3.04%, Metrics -- 9.95e-01 / 1.13e+00, 2.47e-01 / 1.08e-01 | 20260508 09:06:03.622 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.31e-01, 4.88e-02 / 2.51e-02 | 20260508 09:06:03.622 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:06:03.622 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:06:03.622 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:06:03.622 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.47e+01 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 24.42%, Upload -- 2.58e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 24.52%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 31.02%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 2.26%, LockUpload -- 2.39e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 2.47%, LockDownload -- 2.61e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 15.30%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 09:06:03.622 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 09:06:03.728 | INFO | Starting a round of Baz IO | 20260508 09:06:07.140 | INFO | Populating RTMetrics report | 20260508 09:06:07.140 | INFO | RTMetrics report populated | 20260508 09:06:07.140 | INFO | Opening tmp RTMetrics json file | 20260508 09:06:07.141 | INFO | Serializing RTMetrics to json | 20260508 09:06:07.143 | INFO | Serialization done, swapping tmp file now... | 20260508 09:06:07.143 | INFO | RTMetrics done | 20260508 09:06:07.207 | INFO | Finished a round of Baz IO: Wrote 1.488 GiB in 3.478 seconds | 20260508 09:06:07.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.65% | 20260508 09:06:07.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266385% / 3.97364e-06% / 7.54992e-05% | 20260508 09:06:07.207 | INFO | Baz IO overall: | 20260508 09:06:07.207 | INFO | BytesWritten : 3016003449679 | 20260508 09:06:07.207 | INFO | NumEvents : 1451259308502 | 20260508 09:06:08.765 | INFO | Analyzing chunk frames = [6897664,6898176) | 20260508 09:06:08.765 | INFO | MemInfo: { | 20260508 09:06:08.765 | INFO | "gpu" : " 17 GB", | 20260508 09:06:08.765 | INFO | "huge" : "", | 20260508 09:06:08.765 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:08.765 | INFO | "malloc" : " 14 GB", | 20260508 09:06:08.765 | INFO | "pinned" : " 29 GB" | 20260508 09:06:08.765 | INFO | } | 20260508 09:06:08.765 | INFO | Perf report for the last 5.14515 seconds: | 20260508 09:06:08.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:08.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:08.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:08.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:08.765 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 09:06:08.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:08.765 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260508 09:06:08.765 | INFO | PRE_HQ : 66.2%, 1.7, 58.9ms, 0 | 20260508 09:06:08.765 | INFO | BAZWRITER : 66.9%, 2.7, 93.5ms, 0 | 20260508 09:06:08.765 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 09:06:08.766 | INFO | Uploaded 36GiB to GPU in 2.56138s (14.0549GiB/s) | 20260508 09:06:08.766 | INFO | Downloaded 38GiB to GPU in 2.71779s (13.9819GiB/s) | 20260508 09:06:13.899 | INFO | Analyzing chunk frames = [6898176,6898688) | 20260508 09:06:13.899 | INFO | MemInfo: { | 20260508 09:06:13.899 | INFO | "gpu" : " 17 GB", | 20260508 09:06:13.899 | INFO | "huge" : "", | 20260508 09:06:13.899 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:13.899 | INFO | "malloc" : " 14 GB", | 20260508 09:06:13.899 | INFO | "pinned" : " 29 GB" | 20260508 09:06:13.899 | INFO | } | 20260508 09:06:13.899 | INFO | Perf report for the last 5.13416 seconds: | 20260508 09:06:13.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:13.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.899 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 09:06:13.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.899 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:06:13.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:13.900 | INFO | Uploaded 36GiB to GPU in 2.4747s (14.5472GiB/s) | 20260508 09:06:13.900 | INFO | Downloaded 34GiB to GPU in 2.4701s (13.7646GiB/s) | 20260508 09:06:19.034 | INFO | Analyzing chunk frames = [6898688,6899200) | 20260508 09:06:19.034 | INFO | MemInfo: { | 20260508 09:06:19.034 | INFO | "gpu" : " 17 GB", | 20260508 09:06:19.034 | INFO | "huge" : "", | 20260508 09:06:19.034 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:19.034 | INFO | "malloc" : " 14 GB", | 20260508 09:06:19.034 | INFO | "pinned" : " 29 GB" | 20260508 09:06:19.034 | INFO | } | 20260508 09:06:19.035 | INFO | Perf report for the last 5.13509 seconds: | 20260508 09:06:19.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:19.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:06:19.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:06:19.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:19.035 | INFO | Uploaded 36GiB to GPU in 2.46254s (14.619GiB/s) | 20260508 09:06:19.035 | INFO | Downloaded 34GiB to GPU in 2.43264s (13.9766GiB/s) | 20260508 09:06:24.171 | INFO | Analyzing chunk frames = [6899200,6899712) | 20260508 09:06:24.171 | INFO | MemInfo: { | 20260508 09:06:24.171 | INFO | "gpu" : " 17 GB", | 20260508 09:06:24.171 | INFO | "huge" : "", | 20260508 09:06:24.171 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:24.171 | INFO | "malloc" : " 14 GB", | 20260508 09:06:24.172 | INFO | "pinned" : " 29 GB" | 20260508 09:06:24.172 | INFO | } | 20260508 09:06:24.172 | INFO | Perf report for the last 5.13703 seconds: | 20260508 09:06:24.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:24.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 09:06:24.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:06:24.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:24.172 | INFO | Uploaded 36GiB to GPU in 2.49542s (14.4264GiB/s) | 20260508 09:06:24.172 | INFO | Downloaded 34GiB to GPU in 2.43441s (13.9664GiB/s) | 20260508 09:06:29.313 | INFO | Analyzing chunk frames = [6899712,6900224) | 20260508 09:06:29.313 | INFO | MemInfo: { | 20260508 09:06:29.313 | INFO | "gpu" : " 17 GB", | 20260508 09:06:29.313 | INFO | "huge" : "", | 20260508 09:06:29.313 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:29.313 | INFO | "malloc" : " 14 GB", | 20260508 09:06:29.313 | INFO | "pinned" : " 29 GB" | 20260508 09:06:29.313 | INFO | } | 20260508 09:06:29.313 | INFO | Perf report for the last 5.14123 seconds: | 20260508 09:06:29.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:29.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 09:06:29.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:06:29.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:29.313 | INFO | Uploaded 36GiB to GPU in 2.38336s (15.1048GiB/s) | 20260508 09:06:29.313 | INFO | Downloaded 34GiB to GPU in 2.49858s (13.6077GiB/s) | 20260508 09:06:34.457 | INFO | Analyzing chunk frames = [6900224,6900736) | 20260508 09:06:34.457 | INFO | MemInfo: { | 20260508 09:06:34.457 | INFO | "gpu" : " 17 GB", | 20260508 09:06:34.457 | INFO | "huge" : "", | 20260508 09:06:34.457 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:34.457 | INFO | "malloc" : " 14 GB", | 20260508 09:06:34.457 | INFO | "pinned" : " 29 GB" | 20260508 09:06:34.457 | INFO | } | 20260508 09:06:34.457 | INFO | Perf report for the last 5.14423 seconds: | 20260508 09:06:34.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:34.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 09:06:34.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:06:34.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:34.457 | INFO | Uploaded 36GiB to GPU in 2.44553s (14.7207GiB/s) | 20260508 09:06:34.457 | INFO | Downloaded 34GiB to GPU in 2.43006s (13.9914GiB/s) | 20260508 09:06:39.600 | INFO | Analyzing chunk frames = [6900736,6901248) | 20260508 09:06:39.600 | INFO | MemInfo: { | 20260508 09:06:39.600 | INFO | "gpu" : " 17 GB", | 20260508 09:06:39.600 | INFO | "huge" : "", | 20260508 09:06:39.600 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:39.600 | INFO | "malloc" : " 14 GB", | 20260508 09:06:39.600 | INFO | "pinned" : " 29 GB" | 20260508 09:06:39.600 | INFO | } | 20260508 09:06:39.601 | INFO | Perf report for the last 5.14348 seconds: | 20260508 09:06:39.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:39.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:06:39.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 09:06:39.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:39.601 | INFO | Uploaded 36GiB to GPU in 2.4634s (14.6139GiB/s) | 20260508 09:06:39.601 | INFO | Downloaded 34GiB to GPU in 2.4331s (13.974GiB/s) | 20260508 09:06:44.735 | INFO | Analyzing chunk frames = [6901248,6901760) | 20260508 09:06:44.735 | INFO | MemInfo: { | 20260508 09:06:44.735 | INFO | "gpu" : " 17 GB", | 20260508 09:06:44.735 | INFO | "huge" : "", | 20260508 09:06:44.735 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:44.735 | INFO | "malloc" : " 14 GB", | 20260508 09:06:44.735 | INFO | "pinned" : " 29 GB" | 20260508 09:06:44.735 | INFO | } | 20260508 09:06:44.735 | INFO | Perf report for the last 5.13475 seconds: | 20260508 09:06:44.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:44.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.735 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 09:06:44.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.736 | INFO | PRE_HQ : 32.9%, 1.0, 17.0ms, 0 | 20260508 09:06:44.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:44.736 | INFO | Uploaded 36GiB to GPU in 2.46837s (14.5846GiB/s) | 20260508 09:06:44.736 | INFO | Downloaded 34GiB to GPU in 2.44583s (13.9012GiB/s) | 20260508 09:06:44.848 | INFO | Starting a round of Baz IO | 20260508 09:06:48.246 | INFO | Populating RTMetrics report | 20260508 09:06:48.246 | INFO | RTMetrics report populated | 20260508 09:06:48.246 | INFO | Opening tmp RTMetrics json file | 20260508 09:06:48.253 | INFO | Serializing RTMetrics to json | 20260508 09:06:48.265 | INFO | Serialization done, swapping tmp file now... | 20260508 09:06:48.265 | INFO | RTMetrics done | 20260508 09:06:48.328 | INFO | Finished a round of Baz IO: Wrote 1.488 GiB in 3.479 seconds | 20260508 09:06:48.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:06:48.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266735% / 0% / 7.94729e-05% | 20260508 09:06:48.328 | INFO | Baz IO overall: | 20260508 09:06:48.328 | INFO | BytesWritten : 3017600999590 | 20260508 09:06:48.328 | INFO | NumEvents : 1452014991671 | 20260508 09:06:49.875 | INFO | Analyzing chunk frames = [6901760,6902272) | 20260508 09:06:49.875 | INFO | MemInfo: { | 20260508 09:06:49.875 | INFO | "gpu" : " 17 GB", | 20260508 09:06:49.875 | INFO | "huge" : "", | 20260508 09:06:49.875 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:49.875 | INFO | "malloc" : " 14 GB", | 20260508 09:06:49.875 | INFO | "pinned" : " 29 GB" | 20260508 09:06:49.875 | INFO | } | 20260508 09:06:49.875 | INFO | Perf report for the last 5.14008 seconds: | 20260508 09:06:49.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:49.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:49.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:49.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:49.875 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 09:06:49.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:49.876 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260508 09:06:49.876 | INFO | PRE_HQ : 66.4%, 1.8, 61.9ms, 0 | 20260508 09:06:49.876 | INFO | BAZWRITER : 66.9%, 2.8, 96.2ms, 0 | 20260508 09:06:49.876 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:06:49.876 | INFO | Uploaded 36GiB to GPU in 2.62526s (13.7129GiB/s) | 20260508 09:06:49.876 | INFO | Downloaded 38GiB to GPU in 2.69868s (14.081GiB/s) | 20260508 09:06:55.010 | INFO | Analyzing chunk frames = [6902272,6902784) | 20260508 09:06:55.010 | INFO | MemInfo: { | 20260508 09:06:55.010 | INFO | "gpu" : " 17 GB", | 20260508 09:06:55.010 | INFO | "huge" : "", | 20260508 09:06:55.010 | INFO | "huge_shared" : " 29 GB", | 20260508 09:06:55.010 | INFO | "malloc" : " 14 GB", | 20260508 09:06:55.010 | INFO | "pinned" : " 29 GB" | 20260508 09:06:55.010 | INFO | } | 20260508 09:06:55.010 | INFO | Perf report for the last 5.13506 seconds: | 20260508 09:06:55.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:06:55.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.011 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:06:55.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.011 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:06:55.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:06:55.011 | INFO | Uploaded 36GiB to GPU in 2.54652s (14.1369GiB/s) | 20260508 09:06:55.011 | INFO | Downloaded 34GiB to GPU in 2.45401s (13.8548GiB/s) | 20260508 09:07:00.149 | INFO | Analyzing chunk frames = [6902784,6903296) | 20260508 09:07:00.149 | INFO | MemInfo: { | 20260508 09:07:00.149 | INFO | "gpu" : " 17 GB", | 20260508 09:07:00.149 | INFO | "huge" : "", | 20260508 09:07:00.149 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:00.149 | INFO | "malloc" : " 14 GB", | 20260508 09:07:00.149 | INFO | "pinned" : " 29 GB" | 20260508 09:07:00.149 | INFO | } | 20260508 09:07:00.150 | INFO | Perf report for the last 5.13906 seconds: | 20260508 09:07:00.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:00.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 09:07:00.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:07:00.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:00.150 | INFO | Uploaded 36GiB to GPU in 2.51759s (14.2994GiB/s) | 20260508 09:07:00.150 | INFO | Downloaded 34GiB to GPU in 2.44448s (13.9089GiB/s) | 20260508 09:07:05.292 | INFO | Analyzing chunk frames = [6903296,6903808) | 20260508 09:07:05.292 | INFO | MemInfo: { | 20260508 09:07:05.293 | INFO | "gpu" : " 17 GB", | 20260508 09:07:05.293 | INFO | "huge" : "", | 20260508 09:07:05.293 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:05.293 | INFO | "malloc" : " 14 GB", | 20260508 09:07:05.293 | INFO | "pinned" : " 29 GB" | 20260508 09:07:05.293 | INFO | } | 20260508 09:07:05.293 | INFO | Perf report for the last 5.14313 seconds: | 20260508 09:07:05.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:05.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | ANALYSIS : 67.0%, 2.9, 103.8ms, 0 | 20260508 09:07:05.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:07:05.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:05.293 | INFO | Uploaded 36GiB to GPU in 2.59235s (13.887GiB/s) | 20260508 09:07:05.293 | INFO | Downloaded 34GiB to GPU in 2.4299s (13.9923GiB/s) | 20260508 09:07:10.429 | INFO | Analyzing chunk frames = [6903808,6904320) | 20260508 09:07:10.429 | INFO | MemInfo: { | 20260508 09:07:10.429 | INFO | "gpu" : " 17 GB", | 20260508 09:07:10.429 | INFO | "huge" : "", | 20260508 09:07:10.429 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:10.429 | INFO | "malloc" : " 14 GB", | 20260508 09:07:10.429 | INFO | "pinned" : " 29 GB" | 20260508 09:07:10.429 | INFO | } | 20260508 09:07:10.429 | INFO | Perf report for the last 5.13641 seconds: | 20260508 09:07:10.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:10.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 09:07:10.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:07:10.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:10.429 | INFO | Uploaded 36GiB to GPU in 2.36981s (15.1911GiB/s) | 20260508 09:07:10.429 | INFO | Downloaded 34GiB to GPU in 2.50877s (13.5525GiB/s) | 20260508 09:07:15.574 | INFO | Analyzing chunk frames = [6904320,6904832) | 20260508 09:07:15.574 | INFO | MemInfo: { | 20260508 09:07:15.574 | INFO | "gpu" : " 17 GB", | 20260508 09:07:15.574 | INFO | "huge" : "", | 20260508 09:07:15.574 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:15.574 | INFO | "malloc" : " 14 GB", | 20260508 09:07:15.574 | INFO | "pinned" : " 29 GB" | 20260508 09:07:15.574 | INFO | } | 20260508 09:07:15.575 | INFO | Perf report for the last 5.1454 seconds: | 20260508 09:07:15.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:15.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:07:15.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:07:15.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:15.575 | INFO | Uploaded 36GiB to GPU in 2.38559s (15.0906GiB/s) | 20260508 09:07:15.575 | INFO | Downloaded 34GiB to GPU in 2.4197s (14.0514GiB/s) | 20260508 09:07:20.711 | INFO | Analyzing chunk frames = [6904832,6905344) | 20260508 09:07:20.711 | INFO | MemInfo: { | 20260508 09:07:20.711 | INFO | "gpu" : " 17 GB", | 20260508 09:07:20.711 | INFO | "huge" : "", | 20260508 09:07:20.711 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:20.712 | INFO | "malloc" : " 14 GB", | 20260508 09:07:20.712 | INFO | "pinned" : " 29 GB" | 20260508 09:07:20.712 | INFO | } | 20260508 09:07:20.712 | INFO | Perf report for the last 5.13716 seconds: | 20260508 09:07:20.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:20.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:07:20.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:07:20.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:20.712 | INFO | Uploaded 36GiB to GPU in 2.42562s (14.8416GiB/s) | 20260508 09:07:20.712 | INFO | Downloaded 34GiB to GPU in 2.43224s (13.9789GiB/s) | 20260508 09:07:25.848 | INFO | Analyzing chunk frames = [6905344,6905856) | 20260508 09:07:25.848 | INFO | MemInfo: { | 20260508 09:07:25.848 | INFO | "gpu" : " 17 GB", | 20260508 09:07:25.848 | INFO | "huge" : "", | 20260508 09:07:25.848 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:25.848 | INFO | "malloc" : " 14 GB", | 20260508 09:07:25.848 | INFO | "pinned" : " 29 GB" | 20260508 09:07:25.848 | INFO | } | 20260508 09:07:25.848 | INFO | Perf report for the last 5.13678 seconds: | 20260508 09:07:25.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:25.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:07:25.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:07:25.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:25.849 | INFO | Uploaded 36GiB to GPU in 2.43121s (14.8075GiB/s) | 20260508 09:07:25.849 | INFO | Downloaded 34GiB to GPU in 2.42309s (14.0317GiB/s) | 20260508 09:07:25.849 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:07:25.849 | INFO | Total mem usage: 16.7 GiB | 20260508 09:07:25.849 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:07:25.849 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:07:25.849 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:07:25.849 | INFO | High Water Breakdown: | 20260508 09:07:25.849 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:07:25.849 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:07:25.849 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:07:25.849 | INFO | ------------------------------------------ | 20260508 09:07:25.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:07:25.849 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:07:25.849 | INFO | Total mem usage: 26.5 GiB | 20260508 09:07:25.849 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:07:25.849 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:07:25.849 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:07:25.849 | INFO | High Water Breakdown: | 20260508 09:07:25.849 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:07:25.849 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:07:25.849 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:07:25.849 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:07:25.849 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:07:25.849 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:07:25.849 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:07:25.849 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:07:25.849 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:07:25.850 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:07:25.850 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:07:25.850 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:07:25.850 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:07:25.850 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:07:25.850 | INFO | ------------------------------------------ | 20260508 09:07:25.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:07:25.850 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:07:25.850 | INFO | Total mem usage: 27.8 GiB | 20260508 09:07:25.850 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:07:25.850 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:07:25.850 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:07:25.850 | INFO | High Water Breakdown: | 20260508 09:07:25.850 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:07:25.850 | INFO | ------------------------------------------ | 20260508 09:07:25.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:07:25.850 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:07:25.850 | INFO | Total mem usage: 15.7 GiB | 20260508 09:07:25.850 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:07:25.850 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:07:25.850 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:07:25.850 | INFO | High Water Breakdown: | 20260508 09:07:25.850 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:07:25.850 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:07:25.850 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:07:25.850 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:07:25.850 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:07:25.850 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:07:25.850 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:07:25.850 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:07:25.850 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:07:25.850 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:07:25.850 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:07:25.850 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:07:25.850 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:07:25.850 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:07:25.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:07:25.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:07:25.850 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:07:25.850 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:07:25.850 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:07:25.850 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:07:25.850 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:07:25.850 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:07:25.850 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:07:25.850 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:07:25.850 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:07:25.850 | INFO | ------------------------------------------ | 20260508 09:07:25.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:07:25.850 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:07:25.850 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:07:25.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:07:25.850 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.50e+01, 1.40e+01 / 6.40e-01 | 20260508 09:07:25.850 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.04e-01, 1.14e-02 / 5.84e-03 | 20260508 09:07:25.850 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.39e+00, 5.95e-01 / 2.64e-01 | 20260508 09:07:25.850 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.11e-01 / 1.40e-01 | 20260508 09:07:25.850 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.48e+01, 3.39e+00 / 1.03e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.88e+00 / 1.41e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.05e-01 / 2.64e-01 | 20260508 09:07:25.850 | INFO | PerfReport: 3.05%, Metrics -- 1.00e+00 / 1.18e+00, 2.49e-01 / 1.45e-01 | 20260508 09:07:25.850 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.34e-01, 4.90e-02 / 2.35e-02 | 20260508 09:07:25.850 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:07:25.850 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:07:25.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:07:25.850 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 24.47%, Upload -- 2.58e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 24.48%, Download -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 31.07%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 2.31%, LockUpload -- 2.43e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 2.41%, LockDownload -- 2.54e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 15.25%, LockCompute -- 1.61e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:07:25.850 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260508 09:07:25.946 | INFO | Starting a round of Baz IO | 20260508 09:07:29.352 | INFO | Populating RTMetrics report | 20260508 09:07:29.353 | INFO | RTMetrics report populated | 20260508 09:07:29.353 | INFO | Opening tmp RTMetrics json file | 20260508 09:07:29.491 | INFO | Serializing RTMetrics to json | 20260508 09:07:29.493 | INFO | Serialization done, swapping tmp file now... | 20260508 09:07:29.495 | INFO | RTMetrics done | 20260508 09:07:29.523 | INFO | Finished a round of Baz IO: Wrote 1.488 GiB in 3.576 seconds | 20260508 09:07:29.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:07:29.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266687% / 0% / 8.34465e-05% | 20260508 09:07:29.523 | INFO | Baz IO overall: | 20260508 09:07:29.523 | INFO | BytesWritten : 3019198720280 | 20260508 09:07:29.523 | INFO | NumEvents : 1452770868326 | 20260508 09:07:30.981 | INFO | Analyzing chunk frames = [6905856,6906368) | 20260508 09:07:30.981 | INFO | MemInfo: { | 20260508 09:07:30.981 | INFO | "gpu" : " 17 GB", | 20260508 09:07:30.981 | INFO | "huge" : "", | 20260508 09:07:30.981 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:30.981 | INFO | "malloc" : " 14 GB", | 20260508 09:07:30.981 | INFO | "pinned" : " 29 GB" | 20260508 09:07:30.981 | INFO | } | 20260508 09:07:30.981 | INFO | Perf report for the last 5.13299 seconds: | 20260508 09:07:30.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:30.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:30.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:30.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:30.982 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 09:07:30.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:30.982 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260508 09:07:30.982 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 09:07:30.982 | INFO | BAZWRITER : 68.9%, 2.7, 96.0ms, 0 | 20260508 09:07:30.982 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260508 09:07:30.982 | INFO | Uploaded 36GiB to GPU in 2.60543s (13.8173GiB/s) | 20260508 09:07:30.982 | INFO | Downloaded 38GiB to GPU in 2.71099s (14.017GiB/s) | 20260508 09:07:36.129 | INFO | Analyzing chunk frames = [6906368,6906880) | 20260508 09:07:36.129 | INFO | MemInfo: { | 20260508 09:07:36.129 | INFO | "gpu" : " 17 GB", | 20260508 09:07:36.129 | INFO | "huge" : "", | 20260508 09:07:36.129 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:36.129 | INFO | "malloc" : " 14 GB", | 20260508 09:07:36.129 | INFO | "pinned" : " 29 GB" | 20260508 09:07:36.129 | INFO | } | 20260508 09:07:36.129 | INFO | Perf report for the last 5.1476 seconds: | 20260508 09:07:36.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:36.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:07:36.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:07:36.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:36.129 | INFO | Uploaded 36GiB to GPU in 2.47134s (14.567GiB/s) | 20260508 09:07:36.129 | INFO | Downloaded 34GiB to GPU in 2.44603s (13.9001GiB/s) | 20260508 09:07:41.270 | INFO | Analyzing chunk frames = [6906880,6907392) | 20260508 09:07:41.270 | INFO | MemInfo: { | 20260508 09:07:41.270 | INFO | "gpu" : " 17 GB", | 20260508 09:07:41.270 | INFO | "huge" : "", | 20260508 09:07:41.270 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:41.270 | INFO | "malloc" : " 14 GB", | 20260508 09:07:41.270 | INFO | "pinned" : " 29 GB" | 20260508 09:07:41.270 | INFO | } | 20260508 09:07:41.271 | INFO | Perf report for the last 5.14141 seconds: | 20260508 09:07:41.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:41.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:07:41.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:07:41.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:41.271 | INFO | Uploaded 36GiB to GPU in 2.49237s (14.4441GiB/s) | 20260508 09:07:41.271 | INFO | Downloaded 34GiB to GPU in 2.43785s (13.9467GiB/s) | 20260508 09:07:46.407 | INFO | Analyzing chunk frames = [6907392,6907904) | 20260508 09:07:46.407 | INFO | MemInfo: { | 20260508 09:07:46.407 | INFO | "gpu" : " 17 GB", | 20260508 09:07:46.407 | INFO | "huge" : "", | 20260508 09:07:46.407 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:46.408 | INFO | "malloc" : " 14 GB", | 20260508 09:07:46.408 | INFO | "pinned" : " 29 GB" | 20260508 09:07:46.408 | INFO | } | 20260508 09:07:46.408 | INFO | Perf report for the last 5.1371 seconds: | 20260508 09:07:46.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:46.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:07:46.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:07:46.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:46.408 | INFO | Uploaded 36GiB to GPU in 2.4834s (14.4963GiB/s) | 20260508 09:07:46.408 | INFO | Downloaded 34GiB to GPU in 2.42275s (14.0336GiB/s) | 20260508 09:07:51.546 | INFO | Analyzing chunk frames = [6907904,6908416) | 20260508 09:07:51.546 | INFO | MemInfo: { | 20260508 09:07:51.546 | INFO | "gpu" : " 17 GB", | 20260508 09:07:51.546 | INFO | "huge" : "", | 20260508 09:07:51.546 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:51.546 | INFO | "malloc" : " 14 GB", | 20260508 09:07:51.546 | INFO | "pinned" : " 29 GB" | 20260508 09:07:51.546 | INFO | } | 20260508 09:07:51.546 | INFO | Perf report for the last 5.13837 seconds: | 20260508 09:07:51.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:51.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 09:07:51.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:07:51.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:51.546 | INFO | Uploaded 36GiB to GPU in 2.40896s (14.9442GiB/s) | 20260508 09:07:51.546 | INFO | Downloaded 34GiB to GPU in 2.51483s (13.5198GiB/s) | 20260508 09:07:56.683 | INFO | Analyzing chunk frames = [6908416,6908928) | 20260508 09:07:56.683 | INFO | MemInfo: { | 20260508 09:07:56.683 | INFO | "gpu" : " 17 GB", | 20260508 09:07:56.683 | INFO | "huge" : "", | 20260508 09:07:56.683 | INFO | "huge_shared" : " 29 GB", | 20260508 09:07:56.684 | INFO | "malloc" : " 14 GB", | 20260508 09:07:56.684 | INFO | "pinned" : " 29 GB" | 20260508 09:07:56.684 | INFO | } | 20260508 09:07:56.684 | INFO | Perf report for the last 5.13777 seconds: | 20260508 09:07:56.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:07:56.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:07:56.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 09:07:56.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:07:56.684 | INFO | Uploaded 36GiB to GPU in 2.35725s (15.272GiB/s) | 20260508 09:07:56.684 | INFO | Downloaded 34GiB to GPU in 2.45339s (13.8584GiB/s) | 20260508 09:08:01.824 | INFO | Analyzing chunk frames = [6908928,6909440) | 20260508 09:08:01.824 | INFO | MemInfo: { | 20260508 09:08:01.824 | INFO | "gpu" : " 17 GB", | 20260508 09:08:01.824 | INFO | "huge" : "", | 20260508 09:08:01.824 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:01.824 | INFO | "malloc" : " 14 GB", | 20260508 09:08:01.824 | INFO | "pinned" : " 29 GB" | 20260508 09:08:01.825 | INFO | } | 20260508 09:08:01.825 | INFO | Perf report for the last 5.14073 seconds: | 20260508 09:08:01.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:01.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 09:08:01.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 09:08:01.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:01.825 | INFO | Uploaded 36GiB to GPU in 2.38191s (15.1139GiB/s) | 20260508 09:08:01.825 | INFO | Downloaded 34GiB to GPU in 2.45059s (13.8742GiB/s) | 20260508 09:08:06.968 | INFO | Analyzing chunk frames = [6909440,6909952) | 20260508 09:08:06.968 | INFO | MemInfo: { | 20260508 09:08:06.968 | INFO | "gpu" : " 17 GB", | 20260508 09:08:06.968 | INFO | "huge" : "", | 20260508 09:08:06.968 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:06.968 | INFO | "malloc" : " 14 GB", | 20260508 09:08:06.968 | INFO | "pinned" : " 29 GB" | 20260508 09:08:06.971 | INFO | } | 20260508 09:08:06.971 | INFO | Perf report for the last 5.14662 seconds: | 20260508 09:08:06.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:06.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | ANALYSIS : 65.5%, 3.0, 102.6ms, 0 | 20260508 09:08:06.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:08:06.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:06.971 | INFO | Uploaded 36GiB to GPU in 2.38254s (15.1099GiB/s) | 20260508 09:08:06.972 | INFO | Downloaded 34GiB to GPU in 2.4513s (13.8702GiB/s) | 20260508 09:08:07.068 | INFO | Starting a round of Baz IO | 20260508 09:08:10.475 | INFO | Populating RTMetrics report | 20260508 09:08:10.475 | INFO | RTMetrics report populated | 20260508 09:08:10.475 | INFO | Opening tmp RTMetrics json file | 20260508 09:08:10.486 | INFO | Serializing RTMetrics to json | 20260508 09:08:10.487 | INFO | Serialization done, swapping tmp file now... | 20260508 09:08:10.488 | INFO | RTMetrics done | 20260508 09:08:10.547 | INFO | Finished a round of Baz IO: Wrote 1.488 GiB in 3.478 seconds | 20260508 09:08:10.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:08:10.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266846% / 0% / 7.54992e-05% | 20260508 09:08:10.547 | INFO | Baz IO overall: | 20260508 09:08:10.547 | INFO | BytesWritten : 3020796101657 | 20260508 09:08:10.547 | INFO | NumEvents : 1453526523601 | 20260508 09:08:12.108 | INFO | Analyzing chunk frames = [6909952,6910464) | 20260508 09:08:12.108 | INFO | MemInfo: { | 20260508 09:08:12.108 | INFO | "gpu" : " 17 GB", | 20260508 09:08:12.108 | INFO | "huge" : "", | 20260508 09:08:12.108 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:12.108 | INFO | "malloc" : " 14 GB", | 20260508 09:08:12.108 | INFO | "pinned" : " 29 GB" | 20260508 09:08:12.108 | INFO | } | 20260508 09:08:12.108 | INFO | Perf report for the last 5.1371 seconds: | 20260508 09:08:12.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:12.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:12.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:12.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:12.108 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 09:08:12.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:12.108 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:08:12.108 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 09:08:12.108 | INFO | BAZWRITER : 67.0%, 2.5, 87.7ms, 0 | 20260508 09:08:12.108 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:08:12.109 | INFO | Uploaded 36GiB to GPU in 2.59612s (13.8668GiB/s) | 20260508 09:08:12.109 | INFO | Downloaded 38GiB to GPU in 2.69995s (14.0743GiB/s) | 20260508 09:08:17.237 | INFO | Analyzing chunk frames = [6910464,6910976) | 20260508 09:08:17.237 | INFO | MemInfo: { | 20260508 09:08:17.237 | INFO | "gpu" : " 17 GB", | 20260508 09:08:17.237 | INFO | "huge" : "", | 20260508 09:08:17.237 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:17.237 | INFO | "malloc" : " 14 GB", | 20260508 09:08:17.237 | INFO | "pinned" : " 29 GB" | 20260508 09:08:17.237 | INFO | } | 20260508 09:08:17.237 | INFO | Perf report for the last 5.12887 seconds: | 20260508 09:08:17.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:17.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.237 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 09:08:17.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.237 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:08:17.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:17.238 | INFO | Uploaded 36GiB to GPU in 2.45339s (14.6736GiB/s) | 20260508 09:08:17.238 | INFO | Downloaded 34GiB to GPU in 2.46343s (13.8019GiB/s) | 20260508 09:08:22.380 | INFO | Analyzing chunk frames = [6910976,6911488) | 20260508 09:08:22.380 | INFO | MemInfo: { | 20260508 09:08:22.380 | INFO | "gpu" : " 17 GB", | 20260508 09:08:22.380 | INFO | "huge" : "", | 20260508 09:08:22.380 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:22.380 | INFO | "malloc" : " 14 GB", | 20260508 09:08:22.380 | INFO | "pinned" : " 29 GB" | 20260508 09:08:22.380 | INFO | } | 20260508 09:08:22.380 | INFO | Perf report for the last 5.14317 seconds: | 20260508 09:08:22.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:22.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.380 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 09:08:22.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.381 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:08:22.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:22.381 | INFO | Uploaded 36GiB to GPU in 2.54012s (14.1726GiB/s) | 20260508 09:08:22.381 | INFO | Downloaded 34GiB to GPU in 2.43315s (13.9737GiB/s) | 20260508 09:08:27.517 | INFO | Analyzing chunk frames = [6911488,6912000) | 20260508 09:08:27.518 | INFO | MemInfo: { | 20260508 09:08:27.518 | INFO | "gpu" : " 17 GB", | 20260508 09:08:27.518 | INFO | "huge" : "", | 20260508 09:08:27.518 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:27.518 | INFO | "malloc" : " 14 GB", | 20260508 09:08:27.518 | INFO | "pinned" : " 29 GB" | 20260508 09:08:27.518 | INFO | } | 20260508 09:08:27.518 | INFO | Perf report for the last 5.13759 seconds: | 20260508 09:08:27.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:27.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 09:08:27.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:08:27.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:27.518 | INFO | Uploaded 36GiB to GPU in 2.51566s (14.3104GiB/s) | 20260508 09:08:27.519 | INFO | Downloaded 34GiB to GPU in 2.44269s (13.9191GiB/s) | 20260508 09:08:32.656 | INFO | Analyzing chunk frames = [6912000,6912512) | 20260508 09:08:32.656 | INFO | MemInfo: { | 20260508 09:08:32.656 | INFO | "gpu" : " 17 GB", | 20260508 09:08:32.656 | INFO | "huge" : "", | 20260508 09:08:32.656 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:32.656 | INFO | "malloc" : " 14 GB", | 20260508 09:08:32.656 | INFO | "pinned" : " 29 GB" | 20260508 09:08:32.656 | INFO | } | 20260508 09:08:32.656 | INFO | Perf report for the last 5.13843 seconds: | 20260508 09:08:32.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:32.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.657 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 09:08:32.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.657 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 09:08:32.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:32.657 | INFO | Uploaded 36GiB to GPU in 2.30731s (15.6026GiB/s) | 20260508 09:08:32.657 | INFO | Downloaded 34GiB to GPU in 2.49772s (13.6124GiB/s) | 20260508 09:08:37.795 | INFO | Analyzing chunk frames = [6912512,6913024) | 20260508 09:08:37.795 | INFO | MemInfo: { | 20260508 09:08:37.795 | INFO | "gpu" : " 17 GB", | 20260508 09:08:37.795 | INFO | "huge" : "", | 20260508 09:08:37.795 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:37.795 | INFO | "malloc" : " 14 GB", | 20260508 09:08:37.795 | INFO | "pinned" : " 29 GB" | 20260508 09:08:37.795 | INFO | } | 20260508 09:08:37.795 | INFO | Perf report for the last 5.13899 seconds: | 20260508 09:08:37.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:37.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.795 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:08:37.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.796 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:08:37.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:37.796 | INFO | Uploaded 36GiB to GPU in 2.348s (15.3322GiB/s) | 20260508 09:08:37.796 | INFO | Downloaded 34GiB to GPU in 2.42694s (14.0094GiB/s) | 20260508 09:08:42.936 | INFO | Analyzing chunk frames = [6913024,6913536) | 20260508 09:08:42.936 | INFO | MemInfo: { | 20260508 09:08:42.936 | INFO | "gpu" : " 17 GB", | 20260508 09:08:42.936 | INFO | "huge" : "", | 20260508 09:08:42.936 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:42.936 | INFO | "malloc" : " 14 GB", | 20260508 09:08:42.937 | INFO | "pinned" : " 29 GB" | 20260508 09:08:42.937 | INFO | } | 20260508 09:08:42.937 | INFO | Perf report for the last 5.14115 seconds: | 20260508 09:08:42.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:42.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:08:42.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:08:42.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:42.937 | INFO | Uploaded 36GiB to GPU in 2.42579s (14.8405GiB/s) | 20260508 09:08:42.937 | INFO | Downloaded 34GiB to GPU in 2.43564s (13.9594GiB/s) | 20260508 09:08:48.081 | INFO | Analyzing chunk frames = [6913536,6914048) | 20260508 09:08:48.081 | INFO | MemInfo: { | 20260508 09:08:48.081 | INFO | "gpu" : " 17 GB", | 20260508 09:08:48.081 | INFO | "huge" : "", | 20260508 09:08:48.081 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:48.081 | INFO | "malloc" : " 14 GB", | 20260508 09:08:48.082 | INFO | "pinned" : " 29 GB" | 20260508 09:08:48.082 | INFO | } | 20260508 09:08:48.082 | INFO | Perf report for the last 5.14503 seconds: | 20260508 09:08:48.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:48.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 09:08:48.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:08:48.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:48.082 | INFO | Uploaded 36GiB to GPU in 2.4396s (14.7565GiB/s) | 20260508 09:08:48.082 | INFO | Downloaded 34GiB to GPU in 2.43728s (13.95GiB/s) | 20260508 09:08:48.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:08:48.082 | INFO | Total mem usage: 16.7 GiB | 20260508 09:08:48.082 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:08:48.082 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.8%) | 20260508 09:08:48.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:08:48.082 | INFO | High Water Breakdown: | 20260508 09:08:48.082 | INFO | 59.6% : 8.59 GiB ( 83.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:08:48.082 | INFO | 40.3% : 5.8 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:08:48.082 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:08:48.082 | INFO | ------------------------------------------ | 20260508 09:08:48.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:08:48.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:08:48.082 | INFO | Total mem usage: 26.5 GiB | 20260508 09:08:48.082 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:08:48.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:08:48.082 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:08:48.082 | INFO | High Water Breakdown: | 20260508 09:08:48.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:08:48.083 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:08:48.083 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:08:48.083 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:08:48.083 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:08:48.083 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:08:48.083 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:08:48.083 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:08:48.083 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:08:48.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:08:48.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:08:48.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:08:48.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:08:48.083 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:08:48.083 | INFO | ------------------------------------------ | 20260508 09:08:48.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:08:48.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:08:48.083 | INFO | Total mem usage: 27.8 GiB | 20260508 09:08:48.083 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:08:48.083 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:08:48.083 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:08:48.083 | INFO | High Water Breakdown: | 20260508 09:08:48.083 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:08:48.083 | INFO | ------------------------------------------ | 20260508 09:08:48.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:08:48.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:08:48.083 | INFO | Total mem usage: 15.7 GiB | 20260508 09:08:48.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:08:48.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:08:48.083 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:08:48.083 | INFO | High Water Breakdown: | 20260508 09:08:48.083 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:08:48.083 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:08:48.083 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:08:48.083 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:08:48.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:08:48.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:08:48.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:08:48.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:08:48.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:08:48.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:08:48.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:08:48.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:08:48.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:08:48.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:08:48.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:08:48.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:08:48.083 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:08:48.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:08:48.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:08:48.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:08:48.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:08:48.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:08:48.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:08:48.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:08:48.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:08:48.083 | INFO | ------------------------------------------ | 20260508 09:08:48.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:08:48.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:08:48.083 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:08:48.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:08:48.083 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 5.04e-01 | 20260508 09:08:48.083 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.15e-02 / 6.05e-03 | 20260508 09:08:48.083 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.42e+00, 5.94e-01 / 2.17e-01 | 20260508 09:08:48.083 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.10e-01 / 1.24e-01 | 20260508 09:08:48.083 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.50e+01, 3.67e+00 / 1.05e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.88e+00 / 1.19e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.33e+00 / 2.46e+00, 4.07e-01 / 2.06e-01 | 20260508 09:08:48.083 | INFO | PerfReport: 3.03%, Metrics -- 9.94e-01 / 1.15e+00, 2.48e-01 / 1.43e-01 | 20260508 09:08:48.083 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.30e-01, 4.83e-02 / 2.55e-02 | 20260508 09:08:48.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:08:48.083 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:08:48.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:08:48.083 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 24.54%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 31.12%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 2.54%, LockDownload -- 2.68e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 15.40%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:08:48.083 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260508 09:08:48.179 | INFO | Starting a round of Baz IO | 20260508 09:08:51.582 | INFO | Populating RTMetrics report | 20260508 09:08:51.583 | INFO | RTMetrics report populated | 20260508 09:08:51.583 | INFO | Opening tmp RTMetrics json file | 20260508 09:08:51.594 | INFO | Serializing RTMetrics to json | 20260508 09:08:51.596 | INFO | Serialization done, swapping tmp file now... | 20260508 09:08:51.596 | INFO | RTMetrics done | 20260508 09:08:51.654 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.475 seconds | 20260508 09:08:51.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:08:51.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.265137% / 0% / 4.76837e-05% | 20260508 09:08:51.654 | INFO | Baz IO overall: | 20260508 09:08:51.654 | INFO | BytesWritten : 3022390922981 | 20260508 09:08:51.654 | INFO | NumEvents : 1454280355281 | 20260508 09:08:53.218 | INFO | Analyzing chunk frames = [6914048,6914560) | 20260508 09:08:53.218 | INFO | MemInfo: { | 20260508 09:08:53.218 | INFO | "gpu" : " 17 GB", | 20260508 09:08:53.218 | INFO | "huge" : "", | 20260508 09:08:53.218 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:53.218 | INFO | "malloc" : " 14 GB", | 20260508 09:08:53.218 | INFO | "pinned" : " 29 GB" | 20260508 09:08:53.218 | INFO | } | 20260508 09:08:53.218 | INFO | Perf report for the last 5.1361 seconds: | 20260508 09:08:53.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:53.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:53.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:53.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:53.218 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 09:08:53.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:53.218 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 09:08:53.218 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 09:08:53.218 | INFO | BAZWRITER : 67.0%, 2.5, 87.7ms, 0 | 20260508 09:08:53.218 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:08:53.218 | INFO | Uploaded 36GiB to GPU in 2.57968s (13.9552GiB/s) | 20260508 09:08:53.218 | INFO | Downloaded 38GiB to GPU in 2.70879s (14.0284GiB/s) | 20260508 09:08:58.358 | INFO | Analyzing chunk frames = [6914560,6915072) | 20260508 09:08:58.358 | INFO | MemInfo: { | 20260508 09:08:58.358 | INFO | "gpu" : " 17 GB", | 20260508 09:08:58.358 | INFO | "huge" : "", | 20260508 09:08:58.358 | INFO | "huge_shared" : " 29 GB", | 20260508 09:08:58.358 | INFO | "malloc" : " 14 GB", | 20260508 09:08:58.358 | INFO | "pinned" : " 29 GB" | 20260508 09:08:58.358 | INFO | } | 20260508 09:08:58.358 | INFO | Perf report for the last 5.14005 seconds: | 20260508 09:08:58.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:08:58.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 09:08:58.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:08:58.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:08:58.358 | INFO | Uploaded 36GiB to GPU in 2.46662s (14.5949GiB/s) | 20260508 09:08:58.358 | INFO | Downloaded 34GiB to GPU in 2.45988s (13.8218GiB/s) | 20260508 09:09:03.492 | INFO | Analyzing chunk frames = [6915072,6915584) | 20260508 09:09:03.492 | INFO | MemInfo: { | 20260508 09:09:03.492 | INFO | "gpu" : " 17 GB", | 20260508 09:09:03.492 | INFO | "huge" : "", | 20260508 09:09:03.492 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:03.492 | INFO | "malloc" : " 14 GB", | 20260508 09:09:03.492 | INFO | "pinned" : " 29 GB" | 20260508 09:09:03.492 | INFO | } | 20260508 09:09:03.493 | INFO | Perf report for the last 5.13465 seconds: | 20260508 09:09:03.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:03.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.493 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:09:03.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.493 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 09:09:03.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:03.494 | INFO | Uploaded 36GiB to GPU in 2.42251s (14.8606GiB/s) | 20260508 09:09:03.494 | INFO | Downloaded 34GiB to GPU in 2.44369s (13.9134GiB/s) | 20260508 09:09:08.639 | INFO | Analyzing chunk frames = [6915584,6916096) | 20260508 09:09:08.640 | INFO | MemInfo: { | 20260508 09:09:08.640 | INFO | "gpu" : " 17 GB", | 20260508 09:09:08.640 | INFO | "huge" : "", | 20260508 09:09:08.640 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:08.640 | INFO | "malloc" : " 14 GB", | 20260508 09:09:08.640 | INFO | "pinned" : " 29 GB" | 20260508 09:09:08.640 | INFO | } | 20260508 09:09:08.640 | INFO | Perf report for the last 5.14741 seconds: | 20260508 09:09:08.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:08.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.640 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:09:08.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.640 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 09:09:08.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:08.641 | INFO | Uploaded 36GiB to GPU in 2.46458s (14.607GiB/s) | 20260508 09:09:08.641 | INFO | Downloaded 34GiB to GPU in 2.44215s (13.9221GiB/s) | 20260508 09:09:13.780 | INFO | Analyzing chunk frames = [6916096,6916608) | 20260508 09:09:13.780 | INFO | MemInfo: { | 20260508 09:09:13.780 | INFO | "gpu" : " 17 GB", | 20260508 09:09:13.780 | INFO | "huge" : "", | 20260508 09:09:13.780 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:13.780 | INFO | "malloc" : " 14 GB", | 20260508 09:09:13.780 | INFO | "pinned" : " 29 GB" | 20260508 09:09:13.780 | INFO | } | 20260508 09:09:13.780 | INFO | Perf report for the last 5.13991 seconds: | 20260508 09:09:13.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:13.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 09:09:13.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260508 09:09:13.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:13.780 | INFO | Uploaded 36GiB to GPU in 2.37671s (15.147GiB/s) | 20260508 09:09:13.780 | INFO | Downloaded 34GiB to GPU in 2.53665s (13.4035GiB/s) | 20260508 09:09:18.919 | INFO | Analyzing chunk frames = [6916608,6917120) | 20260508 09:09:18.919 | INFO | MemInfo: { | 20260508 09:09:18.919 | INFO | "gpu" : " 17 GB", | 20260508 09:09:18.919 | INFO | "huge" : "", | 20260508 09:09:18.919 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:18.919 | INFO | "malloc" : " 14 GB", | 20260508 09:09:18.919 | INFO | "pinned" : " 29 GB" | 20260508 09:09:18.919 | INFO | } | 20260508 09:09:18.919 | INFO | Perf report for the last 5.13928 seconds: | 20260508 09:09:18.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:18.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 09:09:18.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:09:18.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:18.919 | INFO | Uploaded 36GiB to GPU in 2.31296s (15.5645GiB/s) | 20260508 09:09:18.920 | INFO | Downloaded 34GiB to GPU in 2.45662s (13.8401GiB/s) | 20260508 09:09:24.056 | INFO | Analyzing chunk frames = [6917120,6917632) | 20260508 09:09:24.056 | INFO | MemInfo: { | 20260508 09:09:24.056 | INFO | "gpu" : " 17 GB", | 20260508 09:09:24.056 | INFO | "huge" : "", | 20260508 09:09:24.056 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:24.056 | INFO | "malloc" : " 14 GB", | 20260508 09:09:24.056 | INFO | "pinned" : " 29 GB" | 20260508 09:09:24.056 | INFO | } | 20260508 09:09:24.056 | INFO | Perf report for the last 5.13672 seconds: | 20260508 09:09:24.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:24.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:09:24.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:09:24.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:24.056 | INFO | Uploaded 36GiB to GPU in 2.39537s (15.029GiB/s) | 20260508 09:09:24.056 | INFO | Downloaded 34GiB to GPU in 2.45984s (13.8221GiB/s) | 20260508 09:09:29.189 | INFO | Analyzing chunk frames = [6917632,6918144) | 20260508 09:09:29.189 | INFO | MemInfo: { | 20260508 09:09:29.189 | INFO | "gpu" : " 17 GB", | 20260508 09:09:29.189 | INFO | "huge" : "", | 20260508 09:09:29.189 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:29.189 | INFO | "malloc" : " 14 GB", | 20260508 09:09:29.189 | INFO | "pinned" : " 29 GB" | 20260508 09:09:29.189 | INFO | } | 20260508 09:09:29.189 | INFO | Perf report for the last 5.13306 seconds: | 20260508 09:09:29.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:29.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 09:09:29.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:09:29.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:29.189 | INFO | Uploaded 36GiB to GPU in 2.36784s (15.2037GiB/s) | 20260508 09:09:29.189 | INFO | Downloaded 34GiB to GPU in 2.44235s (13.921GiB/s) | 20260508 09:09:29.286 | INFO | Starting a round of Baz IO | 20260508 09:09:32.706 | INFO | Populating RTMetrics report | 20260508 09:09:32.706 | INFO | RTMetrics report populated | 20260508 09:09:32.706 | INFO | Opening tmp RTMetrics json file | 20260508 09:09:32.717 | INFO | Serializing RTMetrics to json | 20260508 09:09:32.719 | INFO | Serialization done, swapping tmp file now... | 20260508 09:09:32.719 | INFO | RTMetrics done | 20260508 09:09:32.778 | INFO | Finished a round of Baz IO: Wrote 1.486 GiB in 3.492 seconds | 20260508 09:09:32.778 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:09:32.778 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.265078% / 0% / 5.16574e-05% | 20260508 09:09:32.778 | INFO | Baz IO overall: | 20260508 09:09:32.778 | INFO | BytesWritten : 3023986611871 | 20260508 09:09:32.778 | INFO | NumEvents : 1455034858362 | 20260508 09:09:34.333 | INFO | Analyzing chunk frames = [6918144,6918656) | 20260508 09:09:34.333 | INFO | MemInfo: { | 20260508 09:09:34.333 | INFO | "gpu" : " 17 GB", | 20260508 09:09:34.333 | INFO | "huge" : "", | 20260508 09:09:34.333 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:34.333 | INFO | "malloc" : " 14 GB", | 20260508 09:09:34.333 | INFO | "pinned" : " 29 GB" | 20260508 09:09:34.333 | INFO | } | 20260508 09:09:34.333 | INFO | Perf report for the last 5.1439 seconds: | 20260508 09:09:34.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:34.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:34.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:34.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:34.333 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 09:09:34.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:34.333 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:09:34.333 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 09:09:34.333 | INFO | BAZWRITER : 67.4%, 2.6, 90.3ms, 0 | 20260508 09:09:34.333 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:09:34.333 | INFO | Uploaded 36GiB to GPU in 2.56064s (14.059GiB/s) | 20260508 09:09:34.333 | INFO | Downloaded 38GiB to GPU in 2.71641s (13.989GiB/s) | 20260508 09:09:39.477 | INFO | Analyzing chunk frames = [6918656,6919168) | 20260508 09:09:39.477 | INFO | MemInfo: { | 20260508 09:09:39.477 | INFO | "gpu" : " 17 GB", | 20260508 09:09:39.477 | INFO | "huge" : "", | 20260508 09:09:39.477 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:39.477 | INFO | "malloc" : " 14 GB", | 20260508 09:09:39.477 | INFO | "pinned" : " 29 GB" | 20260508 09:09:39.477 | INFO | } | 20260508 09:09:39.477 | INFO | Perf report for the last 5.14397 seconds: | 20260508 09:09:39.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:39.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:09:39.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:09:39.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:39.477 | INFO | Uploaded 36GiB to GPU in 2.51472s (14.3157GiB/s) | 20260508 09:09:39.477 | INFO | Downloaded 34GiB to GPU in 2.44589s (13.9009GiB/s) | 20260508 09:09:44.613 | INFO | Analyzing chunk frames = [6919168,6919680) | 20260508 09:09:44.613 | INFO | MemInfo: { | 20260508 09:09:44.613 | INFO | "gpu" : " 17 GB", | 20260508 09:09:44.613 | INFO | "huge" : "", | 20260508 09:09:44.613 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:44.613 | INFO | "malloc" : " 14 GB", | 20260508 09:09:44.613 | INFO | "pinned" : " 29 GB" | 20260508 09:09:44.613 | INFO | } | 20260508 09:09:44.613 | INFO | Perf report for the last 5.1363 seconds: | 20260508 09:09:44.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:44.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.613 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:09:44.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.613 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:09:44.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:44.614 | INFO | Uploaded 36GiB to GPU in 2.43448s (14.7875GiB/s) | 20260508 09:09:44.614 | INFO | Downloaded 34GiB to GPU in 2.42696s (14.0093GiB/s) | 20260508 09:09:49.745 | INFO | Analyzing chunk frames = [6919680,6920192) | 20260508 09:09:49.745 | INFO | MemInfo: { | 20260508 09:09:49.745 | INFO | "gpu" : " 17 GB", | 20260508 09:09:49.745 | INFO | "huge" : "", | 20260508 09:09:49.745 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:49.745 | INFO | "malloc" : " 14 GB", | 20260508 09:09:49.746 | INFO | "pinned" : " 29 GB" | 20260508 09:09:49.746 | INFO | } | 20260508 09:09:49.746 | INFO | Perf report for the last 5.13276 seconds: | 20260508 09:09:49.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:49.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 09:09:49.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 09:09:49.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:49.746 | INFO | Uploaded 36GiB to GPU in 2.50548s (14.3685GiB/s) | 20260508 09:09:49.747 | INFO | Downloaded 34GiB to GPU in 2.44298s (13.9174GiB/s) | 20260508 09:09:54.891 | INFO | Analyzing chunk frames = [6920192,6920704) | 20260508 09:09:54.891 | INFO | MemInfo: { | 20260508 09:09:54.891 | INFO | "gpu" : " 17 GB", | 20260508 09:09:54.891 | INFO | "huge" : "", | 20260508 09:09:54.891 | INFO | "huge_shared" : " 29 GB", | 20260508 09:09:54.891 | INFO | "malloc" : " 14 GB", | 20260508 09:09:54.891 | INFO | "pinned" : " 29 GB" | 20260508 09:09:54.891 | INFO | } | 20260508 09:09:54.891 | INFO | Perf report for the last 5.14494 seconds: | 20260508 09:09:54.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:09:54.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 09:09:54.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | PRE_HQ : 30.6%, 1.0, 16.0ms, 0 | 20260508 09:09:54.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:09:54.891 | INFO | Uploaded 36GiB to GPU in 2.30161s (15.6412GiB/s) | 20260508 09:09:54.891 | INFO | Downloaded 34GiB to GPU in 2.49134s (13.6473GiB/s) | 20260508 09:10:00.030 | INFO | Analyzing chunk frames = [6920704,6921216) | 20260508 09:10:00.030 | INFO | MemInfo: { | 20260508 09:10:00.030 | INFO | "gpu" : " 17 GB", | 20260508 09:10:00.030 | INFO | "huge" : "", | 20260508 09:10:00.030 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:00.030 | INFO | "malloc" : " 14 GB", | 20260508 09:10:00.030 | INFO | "pinned" : " 29 GB" | 20260508 09:10:00.030 | INFO | } | 20260508 09:10:00.030 | INFO | Perf report for the last 5.13911 seconds: | 20260508 09:10:00.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:00.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 09:10:00.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:10:00.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:00.030 | INFO | Uploaded 36GiB to GPU in 2.39544s (15.0286GiB/s) | 20260508 09:10:00.030 | INFO | Downloaded 34GiB to GPU in 2.43476s (13.9644GiB/s) | 20260508 09:10:05.167 | INFO | Analyzing chunk frames = [6921216,6921728) | 20260508 09:10:05.167 | INFO | MemInfo: { | 20260508 09:10:05.167 | INFO | "gpu" : " 17 GB", | 20260508 09:10:05.167 | INFO | "huge" : "", | 20260508 09:10:05.167 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:05.168 | INFO | "malloc" : " 14 GB", | 20260508 09:10:05.168 | INFO | "pinned" : " 29 GB" | 20260508 09:10:05.168 | INFO | } | 20260508 09:10:05.168 | INFO | Perf report for the last 5.1376 seconds: | 20260508 09:10:05.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:05.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:10:05.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:10:05.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:05.168 | INFO | Uploaded 36GiB to GPU in 2.41899s (14.8822GiB/s) | 20260508 09:10:05.168 | INFO | Downloaded 34GiB to GPU in 2.4381s (13.9453GiB/s) | 20260508 09:10:10.305 | INFO | Analyzing chunk frames = [6921728,6922240) | 20260508 09:10:10.305 | INFO | MemInfo: { | 20260508 09:10:10.305 | INFO | "gpu" : " 17 GB", | 20260508 09:10:10.305 | INFO | "huge" : "", | 20260508 09:10:10.305 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:10.305 | INFO | "malloc" : " 14 GB", | 20260508 09:10:10.305 | INFO | "pinned" : " 29 GB" | 20260508 09:10:10.305 | INFO | } | 20260508 09:10:10.305 | INFO | Perf report for the last 5.13767 seconds: | 20260508 09:10:10.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:10.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.305 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 09:10:10.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.305 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:10:10.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:10.306 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:10:10.306 | INFO | Total mem usage: 16.7 GiB | 20260508 09:10:10.306 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:10:10.306 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:10:10.306 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:10:10.306 | INFO | High Water Breakdown: | 20260508 09:10:10.306 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:10:10.306 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:10:10.306 | INFO | Uploaded 36GiB to GPU in 2.40577s (14.964GiB/s) | 20260508 09:10:10.306 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:10:10.306 | INFO | ------------------------------------------ | 20260508 09:10:10.306 | INFO | Downloaded 34GiB to GPU in 2.43539s (13.9608GiB/s) | 20260508 09:10:10.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:10:10.306 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:10:10.306 | INFO | Total mem usage: 26.5 GiB | 20260508 09:10:10.306 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:10:10.306 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:10:10.306 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:10:10.306 | INFO | High Water Breakdown: | 20260508 09:10:10.306 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:10:10.306 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:10:10.306 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:10:10.306 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:10:10.306 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:10:10.306 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:10:10.306 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:10:10.306 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:10:10.306 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:10:10.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:10:10.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:10:10.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:10:10.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:10:10.306 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:10:10.306 | INFO | ------------------------------------------ | 20260508 09:10:10.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:10:10.306 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:10:10.306 | INFO | Total mem usage: 27.8 GiB | 20260508 09:10:10.306 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:10:10.306 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:10:10.306 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:10:10.306 | INFO | High Water Breakdown: | 20260508 09:10:10.306 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:10:10.306 | INFO | ------------------------------------------ | 20260508 09:10:10.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:10:10.306 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:10:10.306 | INFO | Total mem usage: 15.7 GiB | 20260508 09:10:10.306 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:10:10.307 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:10:10.307 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:10:10.307 | INFO | High Water Breakdown: | 20260508 09:10:10.307 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:10:10.307 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:10:10.307 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:10:10.307 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:10:10.307 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:10:10.307 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:10:10.307 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:10:10.307 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:10:10.307 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:10:10.307 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:10:10.307 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:10:10.307 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:10:10.307 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:10:10.307 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:10:10.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:10:10.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:10:10.307 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:10:10.307 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:10:10.307 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:10:10.307 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:10:10.307 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:10:10.307 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:10:10.307 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:10:10.307 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:10:10.307 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:10:10.307 | INFO | ------------------------------------------ | 20260508 09:10:10.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:10:10.307 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:10:10.307 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:10:10.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:10:10.307 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.42e+01 / 7.05e-01 | 20260508 09:10:10.307 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.11e-02 / 5.99e-03 | 20260508 09:10:10.307 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.43e+00, 5.94e-01 / 2.24e-01 | 20260508 09:10:10.307 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.11e-01 / 1.33e-01 | 20260508 09:10:10.307 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.52e+01, 3.69e+00 / 1.18e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.88e+00 / 1.39e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.07e-01 / 2.94e-01 | 20260508 09:10:10.307 | INFO | PerfReport: 3.04%, Metrics -- 9.97e-01 / 1.14e+00, 2.47e-01 / 1.44e-01 | 20260508 09:10:10.307 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.33e-01, 4.77e-02 / 2.70e-02 | 20260508 09:10:10.307 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:10:10.307 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:10:10.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:10:10.307 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 24.08%, Upload -- 2.53e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 24.61%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 31.17%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 2.02%, LockUpload -- 2.13e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 15.45%, LockCompute -- 1.63e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:10:10.307 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260508 09:10:10.403 | INFO | Starting a round of Baz IO | 20260508 09:10:13.819 | INFO | Populating RTMetrics report | 20260508 09:10:13.819 | INFO | RTMetrics report populated | 20260508 09:10:13.819 | INFO | Opening tmp RTMetrics json file | 20260508 09:10:13.832 | INFO | Serializing RTMetrics to json | 20260508 09:10:13.833 | INFO | Serialization done, swapping tmp file now... | 20260508 09:10:13.834 | INFO | RTMetrics done | 20260508 09:10:13.888 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.484 seconds | 20260508 09:10:13.888 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:10:13.888 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.263492% / 0% / 7.54992e-05% | 20260508 09:10:13.888 | INFO | Baz IO overall: | 20260508 09:10:13.888 | INFO | BytesWritten : 3025580945022 | 20260508 09:10:13.888 | INFO | NumEvents : 1455788226926 | 20260508 09:10:15.447 | INFO | Analyzing chunk frames = [6922240,6922752) | 20260508 09:10:15.447 | INFO | MemInfo: { | 20260508 09:10:15.447 | INFO | "gpu" : " 17 GB", | 20260508 09:10:15.447 | INFO | "huge" : "", | 20260508 09:10:15.447 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:15.447 | INFO | "malloc" : " 14 GB", | 20260508 09:10:15.447 | INFO | "pinned" : " 29 GB" | 20260508 09:10:15.447 | INFO | } | 20260508 09:10:15.447 | INFO | Perf report for the last 5.14198 seconds: | 20260508 09:10:15.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:15.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:15.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:15.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:15.447 | INFO | ANALYSIS : 67.4%, 3.0, 105.5ms, 0 | 20260508 09:10:15.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:15.447 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 09:10:15.447 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 09:10:15.447 | INFO | BAZWRITER : 67.1%, 2.5, 87.9ms, 0 | 20260508 09:10:15.447 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 09:10:15.448 | INFO | Uploaded 36GiB to GPU in 2.58195s (13.9429GiB/s) | 20260508 09:10:15.448 | INFO | Downloaded 38GiB to GPU in 2.70699s (14.0377GiB/s) | 20260508 09:10:20.588 | INFO | Analyzing chunk frames = [6922752,6923264) | 20260508 09:10:20.588 | INFO | MemInfo: { | 20260508 09:10:20.588 | INFO | "gpu" : " 17 GB", | 20260508 09:10:20.588 | INFO | "huge" : "", | 20260508 09:10:20.588 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:20.588 | INFO | "malloc" : " 14 GB", | 20260508 09:10:20.588 | INFO | "pinned" : " 29 GB" | 20260508 09:10:20.588 | INFO | } | 20260508 09:10:20.588 | INFO | Perf report for the last 5.14044 seconds: | 20260508 09:10:20.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:20.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:10:20.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:10:20.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:20.588 | INFO | Uploaded 36GiB to GPU in 2.48934s (14.4617GiB/s) | 20260508 09:10:20.588 | INFO | Downloaded 34GiB to GPU in 2.46663s (13.784GiB/s) | 20260508 09:10:25.733 | INFO | Analyzing chunk frames = [6923264,6923776) | 20260508 09:10:25.733 | INFO | MemInfo: { | 20260508 09:10:25.733 | INFO | "gpu" : " 17 GB", | 20260508 09:10:25.733 | INFO | "huge" : "", | 20260508 09:10:25.733 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:25.733 | INFO | "malloc" : " 14 GB", | 20260508 09:10:25.733 | INFO | "pinned" : " 29 GB" | 20260508 09:10:25.733 | INFO | } | 20260508 09:10:25.733 | INFO | Perf report for the last 5.14571 seconds: | 20260508 09:10:25.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:25.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 09:10:25.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:10:25.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:25.734 | INFO | Uploaded 36GiB to GPU in 2.46295s (14.6166GiB/s) | 20260508 09:10:25.734 | INFO | Downloaded 34GiB to GPU in 2.45415s (13.8541GiB/s) | 20260508 09:10:30.866 | INFO | Analyzing chunk frames = [6923776,6924288) | 20260508 09:10:30.866 | INFO | MemInfo: { | 20260508 09:10:30.866 | INFO | "gpu" : " 17 GB", | 20260508 09:10:30.866 | INFO | "huge" : "", | 20260508 09:10:30.866 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:30.866 | INFO | "malloc" : " 14 GB", | 20260508 09:10:30.866 | INFO | "pinned" : " 29 GB" | 20260508 09:10:30.866 | INFO | } | 20260508 09:10:30.866 | INFO | Perf report for the last 5.13296 seconds: | 20260508 09:10:30.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:30.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.866 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 09:10:30.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.866 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:10:30.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:30.867 | INFO | Uploaded 36GiB to GPU in 2.42224s (14.8623GiB/s) | 20260508 09:10:30.867 | INFO | Downloaded 34GiB to GPU in 2.42533s (14.0187GiB/s) | 20260508 09:10:36.003 | INFO | Analyzing chunk frames = [6924288,6924800) | 20260508 09:10:36.003 | INFO | MemInfo: { | 20260508 09:10:36.003 | INFO | "gpu" : " 17 GB", | 20260508 09:10:36.003 | INFO | "huge" : "", | 20260508 09:10:36.003 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:36.003 | INFO | "malloc" : " 14 GB", | 20260508 09:10:36.003 | INFO | "pinned" : " 29 GB" | 20260508 09:10:36.004 | INFO | } | 20260508 09:10:36.004 | INFO | Perf report for the last 5.13726 seconds: | 20260508 09:10:36.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:36.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 09:10:36.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:10:36.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:36.004 | INFO | Uploaded 36GiB to GPU in 2.34498s (15.352GiB/s) | 20260508 09:10:36.004 | INFO | Downloaded 34GiB to GPU in 2.52613s (13.4593GiB/s) | 20260508 09:10:41.144 | INFO | Analyzing chunk frames = [6924800,6925312) | 20260508 09:10:41.144 | INFO | MemInfo: { | 20260508 09:10:41.144 | INFO | "gpu" : " 17 GB", | 20260508 09:10:41.144 | INFO | "huge" : "", | 20260508 09:10:41.144 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:41.144 | INFO | "malloc" : " 14 GB", | 20260508 09:10:41.144 | INFO | "pinned" : " 29 GB" | 20260508 09:10:41.144 | INFO | } | 20260508 09:10:41.144 | INFO | Perf report for the last 5.1402 seconds: | 20260508 09:10:41.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:41.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:10:41.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:10:41.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:41.144 | INFO | Uploaded 36GiB to GPU in 2.38446s (15.0978GiB/s) | 20260508 09:10:41.144 | INFO | Downloaded 34GiB to GPU in 2.44578s (13.9015GiB/s) | 20260508 09:10:46.290 | INFO | Analyzing chunk frames = [6925312,6925824) | 20260508 09:10:46.290 | INFO | MemInfo: { | 20260508 09:10:46.290 | INFO | "gpu" : " 17 GB", | 20260508 09:10:46.290 | INFO | "huge" : "", | 20260508 09:10:46.290 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:46.290 | INFO | "malloc" : " 14 GB", | 20260508 09:10:46.290 | INFO | "pinned" : " 29 GB" | 20260508 09:10:46.290 | INFO | } | 20260508 09:10:46.290 | INFO | Perf report for the last 5.14591 seconds: | 20260508 09:10:46.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:46.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:10:46.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:10:46.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:46.290 | INFO | Uploaded 36GiB to GPU in 2.43105s (14.8084GiB/s) | 20260508 09:10:46.290 | INFO | Downloaded 34GiB to GPU in 2.44601s (13.9002GiB/s) | 20260508 09:10:51.425 | INFO | Analyzing chunk frames = [6925824,6926336) | 20260508 09:10:51.425 | INFO | MemInfo: { | 20260508 09:10:51.425 | INFO | "gpu" : " 17 GB", | 20260508 09:10:51.425 | INFO | "huge" : "", | 20260508 09:10:51.425 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:51.425 | INFO | "malloc" : " 14 GB", | 20260508 09:10:51.425 | INFO | "pinned" : " 29 GB" | 20260508 09:10:51.425 | INFO | } | 20260508 09:10:51.425 | INFO | Perf report for the last 5.13505 seconds: | 20260508 09:10:51.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:51.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 09:10:51.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:10:51.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:51.425 | INFO | Uploaded 36GiB to GPU in 2.41227s (14.9237GiB/s) | 20260508 09:10:51.425 | INFO | Downloaded 34GiB to GPU in 2.43842s (13.9434GiB/s) | 20260508 09:10:51.522 | INFO | Starting a round of Baz IO | 20260508 09:10:54.944 | INFO | Populating RTMetrics report | 20260508 09:10:54.944 | INFO | RTMetrics report populated | 20260508 09:10:54.944 | INFO | Opening tmp RTMetrics json file | 20260508 09:10:54.953 | INFO | Serializing RTMetrics to json | 20260508 09:10:54.955 | INFO | Serialization done, swapping tmp file now... | 20260508 09:10:54.955 | INFO | RTMetrics done | 20260508 09:10:55.014 | INFO | Finished a round of Baz IO: Wrote 1.486 GiB in 3.491 seconds | 20260508 09:10:55.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.00% / 59.66% | 20260508 09:10:55.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.264335% / 3.97364e-06% / 6.35783e-05% | 20260508 09:10:55.014 | INFO | Baz IO overall: | 20260508 09:10:55.014 | INFO | BytesWritten : 3027176674962 | 20260508 09:10:55.014 | INFO | NumEvents : 1456542930876 | 20260508 09:10:56.559 | INFO | Analyzing chunk frames = [6926336,6926848) | 20260508 09:10:56.559 | INFO | MemInfo: { | 20260508 09:10:56.559 | INFO | "gpu" : " 17 GB", | 20260508 09:10:56.559 | INFO | "huge" : "", | 20260508 09:10:56.559 | INFO | "huge_shared" : " 29 GB", | 20260508 09:10:56.559 | INFO | "malloc" : " 14 GB", | 20260508 09:10:56.559 | INFO | "pinned" : " 29 GB" | 20260508 09:10:56.560 | INFO | } | 20260508 09:10:56.560 | INFO | Perf report for the last 5.13466 seconds: | 20260508 09:10:56.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:10:56.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:56.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:56.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:56.560 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:10:56.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:10:56.560 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:10:56.560 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 09:10:56.560 | INFO | BAZWRITER : 67.2%, 2.6, 89.3ms, 0 | 20260508 09:10:56.560 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:10:56.560 | INFO | Uploaded 36GiB to GPU in 2.59587s (13.8682GiB/s) | 20260508 09:10:56.560 | INFO | Downloaded 38GiB to GPU in 2.71147s (14.0145GiB/s) | 20260508 09:11:01.699 | INFO | Analyzing chunk frames = [6926848,6927360) | 20260508 09:11:01.699 | INFO | MemInfo: { | 20260508 09:11:01.699 | INFO | "gpu" : " 17 GB", | 20260508 09:11:01.699 | INFO | "huge" : "", | 20260508 09:11:01.699 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:01.699 | INFO | "malloc" : " 14 GB", | 20260508 09:11:01.699 | INFO | "pinned" : " 29 GB" | 20260508 09:11:01.699 | INFO | } | 20260508 09:11:01.699 | INFO | Perf report for the last 5.13963 seconds: | 20260508 09:11:01.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:01.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.699 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:11:01.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.699 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:11:01.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:01.700 | INFO | Uploaded 36GiB to GPU in 2.52772s (14.2421GiB/s) | 20260508 09:11:01.700 | INFO | Downloaded 34GiB to GPU in 2.47203s (13.7539GiB/s) | 20260508 09:11:06.842 | INFO | Analyzing chunk frames = [6927360,6927872) | 20260508 09:11:06.842 | INFO | MemInfo: { | 20260508 09:11:06.842 | INFO | "gpu" : " 17 GB", | 20260508 09:11:06.842 | INFO | "huge" : "", | 20260508 09:11:06.842 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:06.842 | INFO | "malloc" : " 14 GB", | 20260508 09:11:06.842 | INFO | "pinned" : " 29 GB" | 20260508 09:11:06.842 | INFO | } | 20260508 09:11:06.842 | INFO | Perf report for the last 5.1431 seconds: | 20260508 09:11:06.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:06.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 09:11:06.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:11:06.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:06.843 | INFO | Uploaded 36GiB to GPU in 2.46407s (14.6099GiB/s) | 20260508 09:11:06.843 | INFO | Downloaded 34GiB to GPU in 2.43401s (13.9687GiB/s) | 20260508 09:11:11.977 | INFO | Analyzing chunk frames = [6927872,6928384) | 20260508 09:11:11.977 | INFO | MemInfo: { | 20260508 09:11:11.977 | INFO | "gpu" : " 17 GB", | 20260508 09:11:11.977 | INFO | "huge" : "", | 20260508 09:11:11.977 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:11.977 | INFO | "malloc" : " 14 GB", | 20260508 09:11:11.977 | INFO | "pinned" : " 29 GB" | 20260508 09:11:11.977 | INFO | } | 20260508 09:11:11.977 | INFO | Perf report for the last 5.13498 seconds: | 20260508 09:11:11.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:11.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.978 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 09:11:11.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.978 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:11:11.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:11.978 | INFO | Uploaded 36GiB to GPU in 2.48345s (14.496GiB/s) | 20260508 09:11:11.978 | INFO | Downloaded 34GiB to GPU in 2.43905s (13.9398GiB/s) | 20260508 09:11:17.120 | INFO | Analyzing chunk frames = [6928384,6928896) | 20260508 09:11:17.120 | INFO | MemInfo: { | 20260508 09:11:17.120 | INFO | "gpu" : " 17 GB", | 20260508 09:11:17.120 | INFO | "huge" : "", | 20260508 09:11:17.120 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:17.120 | INFO | "malloc" : " 14 GB", | 20260508 09:11:17.120 | INFO | "pinned" : " 29 GB" | 20260508 09:11:17.120 | INFO | } | 20260508 09:11:17.120 | INFO | Perf report for the last 5.14287 seconds: | 20260508 09:11:17.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:17.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.120 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 09:11:17.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.120 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:11:17.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:17.121 | INFO | Uploaded 36GiB to GPU in 2.38722s (15.0803GiB/s) | 20260508 09:11:17.121 | INFO | Downloaded 34GiB to GPU in 2.51951s (13.4947GiB/s) | 20260508 09:11:22.257 | INFO | Analyzing chunk frames = [6928896,6929408) | 20260508 09:11:22.257 | INFO | MemInfo: { | 20260508 09:11:22.257 | INFO | "gpu" : " 17 GB", | 20260508 09:11:22.257 | INFO | "huge" : "", | 20260508 09:11:22.257 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:22.257 | INFO | "malloc" : " 14 GB", | 20260508 09:11:22.257 | INFO | "pinned" : " 29 GB" | 20260508 09:11:22.257 | INFO | } | 20260508 09:11:22.257 | INFO | Perf report for the last 5.13714 seconds: | 20260508 09:11:22.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:22.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.257 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:11:22.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.257 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:11:22.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:22.258 | INFO | Uploaded 36GiB to GPU in 2.33058s (15.4468GiB/s) | 20260508 09:11:22.258 | INFO | Downloaded 34GiB to GPU in 2.44794s (13.8892GiB/s) | 20260508 09:11:27.402 | INFO | Analyzing chunk frames = [6929408,6929920) | 20260508 09:11:27.402 | INFO | MemInfo: { | 20260508 09:11:27.402 | INFO | "gpu" : " 17 GB", | 20260508 09:11:27.402 | INFO | "huge" : "", | 20260508 09:11:27.402 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:27.402 | INFO | "malloc" : " 14 GB", | 20260508 09:11:27.402 | INFO | "pinned" : " 29 GB" | 20260508 09:11:27.402 | INFO | } | 20260508 09:11:27.402 | INFO | Perf report for the last 5.14453 seconds: | 20260508 09:11:27.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:27.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 09:11:27.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:11:27.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:27.402 | INFO | Uploaded 36GiB to GPU in 2.37088s (15.1843GiB/s) | 20260508 09:11:27.402 | INFO | Downloaded 34GiB to GPU in 2.44711s (13.8939GiB/s) | 20260508 09:11:32.540 | INFO | Analyzing chunk frames = [6929920,6930432) | 20260508 09:11:32.540 | INFO | MemInfo: { | 20260508 09:11:32.540 | INFO | "gpu" : " 17 GB", | 20260508 09:11:32.540 | INFO | "huge" : "", | 20260508 09:11:32.540 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:32.540 | INFO | "malloc" : " 14 GB", | 20260508 09:11:32.540 | INFO | "pinned" : " 29 GB" | 20260508 09:11:32.540 | INFO | } | 20260508 09:11:32.540 | INFO | Perf report for the last 5.13829 seconds: | 20260508 09:11:32.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:32.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.540 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 09:11:32.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.540 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 09:11:32.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:32.541 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:11:32.541 | INFO | Total mem usage: 16.7 GiB | 20260508 09:11:32.541 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:11:32.541 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:11:32.541 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:11:32.541 | INFO | High Water Breakdown: | 20260508 09:11:32.541 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:11:32.541 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:11:32.541 | INFO | Uploaded 36GiB to GPU in 2.39107s (15.0561GiB/s) | 20260508 09:11:32.541 | INFO | Downloaded 34GiB to GPU in 2.45862s (13.8289GiB/s) | 20260508 09:11:32.541 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:11:32.541 | INFO | ------------------------------------------ | 20260508 09:11:32.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:11:32.541 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:11:32.541 | INFO | Total mem usage: 26.5 GiB | 20260508 09:11:32.541 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:11:32.541 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:11:32.541 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:11:32.541 | INFO | High Water Breakdown: | 20260508 09:11:32.541 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:11:32.541 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:11:32.541 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:11:32.541 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:11:32.541 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:11:32.541 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:11:32.541 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:11:32.541 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:11:32.541 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:11:32.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:11:32.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:11:32.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:11:32.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:11:32.541 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:11:32.541 | INFO | ------------------------------------------ | 20260508 09:11:32.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:11:32.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:11:32.541 | INFO | Total mem usage: 27.8 GiB | 20260508 09:11:32.541 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:11:32.541 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:11:32.541 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:11:32.541 | INFO | High Water Breakdown: | 20260508 09:11:32.541 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:11:32.541 | INFO | ------------------------------------------ | 20260508 09:11:32.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:11:32.542 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:11:32.542 | INFO | Total mem usage: 15.7 GiB | 20260508 09:11:32.542 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:11:32.542 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:11:32.542 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:11:32.542 | INFO | High Water Breakdown: | 20260508 09:11:32.542 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:11:32.542 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:11:32.542 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:11:32.542 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:11:32.542 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:11:32.542 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:11:32.542 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:11:32.542 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:11:32.542 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:11:32.542 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:11:32.542 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:11:32.542 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:11:32.542 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:11:32.542 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:11:32.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:11:32.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:11:32.542 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:11:32.542 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:11:32.542 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:11:32.542 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:11:32.542 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:11:32.542 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:11:32.542 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:11:32.542 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:11:32.542 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:11:32.542 | INFO | ------------------------------------------ | 20260508 09:11:32.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:11:32.542 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:11:32.542 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:11:32.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:11:32.542 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.41e+01 / 5.59e-01 | 20260508 09:11:32.542 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.06e-01, 1.08e-02 / 6.41e-03 | 20260508 09:11:32.542 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.42e+00, 5.96e-01 / 2.51e-01 | 20260508 09:11:32.542 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.11e-01 / 1.15e-01 | 20260508 09:11:32.542 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.49e+01, 3.37e+00 / 9.95e-01 | 20260508 09:11:32.542 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.87e+00 / 1.35e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.06e-01 / 2.91e-01 | 20260508 09:11:32.542 | INFO | PerfReport: 3.04%, Metrics -- 9.97e-01 / 1.15e+00, 2.49e-01 / 1.43e-01 | 20260508 09:11:32.542 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.38e-01, 4.82e-02 / 2.88e-02 | 20260508 09:11:32.542 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:11:32.542 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:11:32.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:11:32.542 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 24.16%, Upload -- 2.54e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 24.63%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 31.12%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 2.03%, LockUpload -- 2.14e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 2.63%, LockDownload -- 2.77e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 15.42%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:11:32.542 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 09:11:32.638 | INFO | Starting a round of Baz IO | 20260508 09:11:36.055 | INFO | Populating RTMetrics report | 20260508 09:11:36.055 | INFO | RTMetrics report populated | 20260508 09:11:36.055 | INFO | Opening tmp RTMetrics json file | 20260508 09:11:36.064 | INFO | Serializing RTMetrics to json | 20260508 09:11:36.065 | INFO | Serialization done, swapping tmp file now... | 20260508 09:11:36.066 | INFO | RTMetrics done | 20260508 09:11:36.121 | INFO | Finished a round of Baz IO: Wrote 1.486 GiB in 3.482 seconds | 20260508 09:11:36.121 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.66% | 20260508 09:11:36.121 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262793% / 0% / 5.5631e-05% | 20260508 09:11:36.121 | INFO | Baz IO overall: | 20260508 09:11:36.121 | INFO | BytesWritten : 3028772112900 | 20260508 09:11:36.121 | INFO | NumEvents : 1457297481012 | 20260508 09:11:37.679 | INFO | Analyzing chunk frames = [6930432,6930944) | 20260508 09:11:37.679 | INFO | MemInfo: { | 20260508 09:11:37.679 | INFO | "gpu" : " 17 GB", | 20260508 09:11:37.679 | INFO | "huge" : "", | 20260508 09:11:37.679 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:37.679 | INFO | "malloc" : " 14 GB", | 20260508 09:11:37.679 | INFO | "pinned" : " 29 GB" | 20260508 09:11:37.679 | INFO | } | 20260508 09:11:37.679 | INFO | Perf report for the last 5.13886 seconds: | 20260508 09:11:37.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:37.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:37.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:37.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:37.679 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 09:11:37.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:37.679 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:11:37.679 | INFO | PRE_HQ : 65.6%, 1.6, 54.4ms, 0 | 20260508 09:11:37.679 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 09:11:37.679 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:11:37.679 | INFO | Uploaded 36GiB to GPU in 2.57065s (14.0042GiB/s) | 20260508 09:11:37.679 | INFO | Downloaded 38GiB to GPU in 2.71959s (13.9727GiB/s) | 20260508 09:11:42.814 | INFO | Analyzing chunk frames = [6930944,6931456) | 20260508 09:11:42.814 | INFO | MemInfo: { | 20260508 09:11:42.814 | INFO | "gpu" : " 17 GB", | 20260508 09:11:42.814 | INFO | "huge" : "", | 20260508 09:11:42.814 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:42.814 | INFO | "malloc" : " 14 GB", | 20260508 09:11:42.814 | INFO | "pinned" : " 29 GB" | 20260508 09:11:42.814 | INFO | } | 20260508 09:11:42.814 | INFO | Perf report for the last 5.13522 seconds: | 20260508 09:11:42.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:42.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.814 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 09:11:42.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.814 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:11:42.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:42.815 | INFO | Uploaded 36GiB to GPU in 2.51249s (14.3284GiB/s) | 20260508 09:11:42.815 | INFO | Downloaded 34GiB to GPU in 2.45s (13.8776GiB/s) | 20260508 09:11:47.953 | INFO | Analyzing chunk frames = [6931456,6931968) | 20260508 09:11:47.953 | INFO | MemInfo: { | 20260508 09:11:47.953 | INFO | "gpu" : " 17 GB", | 20260508 09:11:47.953 | INFO | "huge" : "", | 20260508 09:11:47.953 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:47.953 | INFO | "malloc" : " 14 GB", | 20260508 09:11:47.953 | INFO | "pinned" : " 29 GB" | 20260508 09:11:47.953 | INFO | } | 20260508 09:11:47.953 | INFO | Perf report for the last 5.13873 seconds: | 20260508 09:11:47.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:47.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:11:47.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:11:47.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:47.953 | INFO | Uploaded 36GiB to GPU in 2.49539s (14.4266GiB/s) | 20260508 09:11:47.953 | INFO | Downloaded 34GiB to GPU in 2.44466s (13.9079GiB/s) | 20260508 09:11:53.093 | INFO | Analyzing chunk frames = [6931968,6932480) | 20260508 09:11:53.093 | INFO | MemInfo: { | 20260508 09:11:53.093 | INFO | "gpu" : " 17 GB", | 20260508 09:11:53.093 | INFO | "huge" : "", | 20260508 09:11:53.093 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:53.093 | INFO | "malloc" : " 14 GB", | 20260508 09:11:53.093 | INFO | "pinned" : " 29 GB" | 20260508 09:11:53.093 | INFO | } | 20260508 09:11:53.093 | INFO | Perf report for the last 5.13976 seconds: | 20260508 09:11:53.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:53.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.093 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:11:53.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.093 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:11:53.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:53.094 | INFO | Uploaded 36GiB to GPU in 2.49695s (14.4176GiB/s) | 20260508 09:11:53.094 | INFO | Downloaded 34GiB to GPU in 2.44187s (13.9237GiB/s) | 20260508 09:11:58.240 | INFO | Analyzing chunk frames = [6932480,6932992) | 20260508 09:11:58.240 | INFO | MemInfo: { | 20260508 09:11:58.240 | INFO | "gpu" : " 17 GB", | 20260508 09:11:58.240 | INFO | "huge" : "", | 20260508 09:11:58.240 | INFO | "huge_shared" : " 29 GB", | 20260508 09:11:58.240 | INFO | "malloc" : " 14 GB", | 20260508 09:11:58.240 | INFO | "pinned" : " 29 GB" | 20260508 09:11:58.240 | INFO | } | 20260508 09:11:58.240 | INFO | Perf report for the last 5.1472 seconds: | 20260508 09:11:58.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:11:58.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | ANALYSIS : 66.3%, 3.0, 103.8ms, 0 | 20260508 09:11:58.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 09:11:58.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:11:58.240 | INFO | Uploaded 36GiB to GPU in 2.40999s (14.9378GiB/s) | 20260508 09:11:58.240 | INFO | Downloaded 34GiB to GPU in 2.52937s (13.4421GiB/s) | 20260508 09:12:03.379 | INFO | Analyzing chunk frames = [6932992,6933504) | 20260508 09:12:03.379 | INFO | MemInfo: { | 20260508 09:12:03.379 | INFO | "gpu" : " 17 GB", | 20260508 09:12:03.379 | INFO | "huge" : "", | 20260508 09:12:03.379 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:03.379 | INFO | "malloc" : " 14 GB", | 20260508 09:12:03.379 | INFO | "pinned" : " 29 GB" | 20260508 09:12:03.379 | INFO | } | 20260508 09:12:03.379 | INFO | Perf report for the last 5.13912 seconds: | 20260508 09:12:03.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:03.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 09:12:03.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:12:03.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:03.379 | INFO | Uploaded 36GiB to GPU in 2.39686s (15.0196GiB/s) | 20260508 09:12:03.379 | INFO | Downloaded 34GiB to GPU in 2.42794s (14.0036GiB/s) | 20260508 09:12:08.510 | INFO | Analyzing chunk frames = [6933504,6934016) | 20260508 09:12:08.510 | INFO | MemInfo: { | 20260508 09:12:08.510 | INFO | "gpu" : " 17 GB", | 20260508 09:12:08.510 | INFO | "huge" : "", | 20260508 09:12:08.510 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:08.510 | INFO | "malloc" : " 14 GB", | 20260508 09:12:08.513 | INFO | "pinned" : " 29 GB" | 20260508 09:12:08.513 | INFO | } | 20260508 09:12:08.513 | INFO | Perf report for the last 5.1335 seconds: | 20260508 09:12:08.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:08.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:12:08.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260508 09:12:08.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:08.513 | INFO | Uploaded 36GiB to GPU in 2.42545s (14.8426GiB/s) | 20260508 09:12:08.513 | INFO | Downloaded 34GiB to GPU in 2.42592s (14.0153GiB/s) | 20260508 09:12:13.655 | INFO | Analyzing chunk frames = [6934016,6934528) | 20260508 09:12:13.655 | INFO | MemInfo: { | 20260508 09:12:13.655 | INFO | "gpu" : " 17 GB", | 20260508 09:12:13.655 | INFO | "huge" : "", | 20260508 09:12:13.655 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:13.655 | INFO | "malloc" : " 14 GB", | 20260508 09:12:13.656 | INFO | "pinned" : " 29 GB" | 20260508 09:12:13.656 | INFO | } | 20260508 09:12:13.656 | INFO | Perf report for the last 5.14291 seconds: | 20260508 09:12:13.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:13.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:12:13.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 09:12:13.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:13.656 | INFO | Uploaded 36GiB to GPU in 2.439s (14.7602GiB/s) | 20260508 09:12:13.656 | INFO | Downloaded 34GiB to GPU in 2.42891s (13.9981GiB/s) | 20260508 09:12:13.767 | INFO | Starting a round of Baz IO | 20260508 09:12:17.158 | INFO | Populating RTMetrics report | 20260508 09:12:17.159 | INFO | RTMetrics report populated | 20260508 09:12:17.159 | INFO | Opening tmp RTMetrics json file | 20260508 09:12:17.169 | INFO | Serializing RTMetrics to json | 20260508 09:12:17.170 | INFO | Serialization done, swapping tmp file now... | 20260508 09:12:17.171 | INFO | RTMetrics done | 20260508 09:12:17.230 | INFO | Finished a round of Baz IO: Wrote 1.487 GiB in 3.462 seconds | 20260508 09:12:17.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.67% | 20260508 09:12:17.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262614% / 1.19209e-05% / 7.54992e-05% | 20260508 09:12:17.230 | INFO | Baz IO overall: | 20260508 09:12:17.230 | INFO | BytesWritten : 3030368718815 | 20260508 09:12:17.230 | INFO | NumEvents : 1458052977367 | 20260508 09:12:18.789 | INFO | Analyzing chunk frames = [6934528,6935040) | 20260508 09:12:18.789 | INFO | MemInfo: { | 20260508 09:12:18.789 | INFO | "gpu" : " 17 GB", | 20260508 09:12:18.789 | INFO | "huge" : "", | 20260508 09:12:18.789 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:18.790 | INFO | "malloc" : " 14 GB", | 20260508 09:12:18.790 | INFO | "pinned" : " 29 GB" | 20260508 09:12:18.790 | INFO | } | 20260508 09:12:18.790 | INFO | Perf report for the last 5.13463 seconds: | 20260508 09:12:18.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:18.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:18.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:18.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:18.790 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 09:12:18.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:18.790 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260508 09:12:18.790 | INFO | PRE_HQ : 66.3%, 1.8, 61.0ms, 0 | 20260508 09:12:18.790 | INFO | BAZWRITER : 66.7%, 2.7, 93.3ms, 0 | 20260508 09:12:18.790 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:12:18.791 | INFO | Uploaded 36GiB to GPU in 2.59935s (13.8496GiB/s) | 20260508 09:12:18.791 | INFO | Downloaded 38GiB to GPU in 2.71453s (13.9987GiB/s) | 20260508 09:12:23.931 | INFO | Analyzing chunk frames = [6935040,6935552) | 20260508 09:12:23.931 | INFO | MemInfo: { | 20260508 09:12:23.931 | INFO | "gpu" : " 17 GB", | 20260508 09:12:23.931 | INFO | "huge" : "", | 20260508 09:12:23.931 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:23.931 | INFO | "malloc" : " 14 GB", | 20260508 09:12:23.931 | INFO | "pinned" : " 29 GB" | 20260508 09:12:23.931 | INFO | } | 20260508 09:12:23.931 | INFO | Perf report for the last 5.14116 seconds: | 20260508 09:12:23.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:23.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.931 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:12:23.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.931 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:12:23.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:23.932 | INFO | Uploaded 36GiB to GPU in 2.51051s (14.3397GiB/s) | 20260508 09:12:23.932 | INFO | Downloaded 34GiB to GPU in 2.45533s (13.8474GiB/s) | 20260508 09:12:29.068 | INFO | Analyzing chunk frames = [6935552,6936064) | 20260508 09:12:29.068 | INFO | MemInfo: { | 20260508 09:12:29.068 | INFO | "gpu" : " 17 GB", | 20260508 09:12:29.068 | INFO | "huge" : "", | 20260508 09:12:29.068 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:29.068 | INFO | "malloc" : " 14 GB", | 20260508 09:12:29.068 | INFO | "pinned" : " 29 GB" | 20260508 09:12:29.068 | INFO | } | 20260508 09:12:29.068 | INFO | Perf report for the last 5.13653 seconds: | 20260508 09:12:29.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:29.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 09:12:29.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:12:29.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:29.068 | INFO | Uploaded 36GiB to GPU in 2.48247s (14.5017GiB/s) | 20260508 09:12:29.068 | INFO | Downloaded 34GiB to GPU in 2.44675s (13.896GiB/s) | 20260508 09:12:34.208 | INFO | Analyzing chunk frames = [6936064,6936576) | 20260508 09:12:34.208 | INFO | MemInfo: { | 20260508 09:12:34.208 | INFO | "gpu" : " 17 GB", | 20260508 09:12:34.208 | INFO | "huge" : "", | 20260508 09:12:34.208 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:34.208 | INFO | "malloc" : " 14 GB", | 20260508 09:12:34.208 | INFO | "pinned" : " 29 GB" | 20260508 09:12:34.208 | INFO | } | 20260508 09:12:34.208 | INFO | Perf report for the last 5.14048 seconds: | 20260508 09:12:34.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:34.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.209 | INFO | ANALYSIS : 66.6%, 3.0, 103.2ms, 0 | 20260508 09:12:34.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.209 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:12:34.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:34.209 | INFO | Uploaded 36GiB to GPU in 2.50593s (14.3659GiB/s) | 20260508 09:12:34.209 | INFO | Downloaded 34GiB to GPU in 2.44284s (13.9182GiB/s) | 20260508 09:12:39.352 | INFO | Analyzing chunk frames = [6936576,6937088) | 20260508 09:12:39.352 | INFO | MemInfo: { | 20260508 09:12:39.352 | INFO | "gpu" : " 17 GB", | 20260508 09:12:39.352 | INFO | "huge" : "", | 20260508 09:12:39.352 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:39.352 | INFO | "malloc" : " 14 GB", | 20260508 09:12:39.352 | INFO | "pinned" : " 29 GB" | 20260508 09:12:39.352 | INFO | } | 20260508 09:12:39.352 | INFO | Perf report for the last 5.14374 seconds: | 20260508 09:12:39.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:39.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.352 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 09:12:39.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.352 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 09:12:39.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:39.353 | INFO | Uploaded 36GiB to GPU in 2.3649s (15.2226GiB/s) | 20260508 09:12:39.353 | INFO | Downloaded 34GiB to GPU in 2.53971s (13.3874GiB/s) | 20260508 09:12:44.495 | INFO | Analyzing chunk frames = [6937088,6937600) | 20260508 09:12:44.495 | INFO | MemInfo: { | 20260508 09:12:44.495 | INFO | "gpu" : " 17 GB", | 20260508 09:12:44.495 | INFO | "huge" : "", | 20260508 09:12:44.495 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:44.495 | INFO | "malloc" : " 14 GB", | 20260508 09:12:44.495 | INFO | "pinned" : " 29 GB" | 20260508 09:12:44.495 | INFO | } | 20260508 09:12:44.495 | INFO | Perf report for the last 5.14294 seconds: | 20260508 09:12:44.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:44.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 09:12:44.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:12:44.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:44.495 | INFO | Uploaded 36GiB to GPU in 2.35589s (15.2809GiB/s) | 20260508 09:12:44.496 | INFO | Downloaded 34GiB to GPU in 2.45482s (13.8503GiB/s) | 20260508 09:12:49.628 | INFO | Analyzing chunk frames = [6937600,6938112) | 20260508 09:12:49.628 | INFO | MemInfo: { | 20260508 09:12:49.628 | INFO | "gpu" : " 17 GB", | 20260508 09:12:49.628 | INFO | "huge" : "", | 20260508 09:12:49.628 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:49.628 | INFO | "malloc" : " 14 GB", | 20260508 09:12:49.628 | INFO | "pinned" : " 29 GB" | 20260508 09:12:49.628 | INFO | } | 20260508 09:12:49.628 | INFO | Perf report for the last 5.13312 seconds: | 20260508 09:12:49.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:49.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.628 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:12:49.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.629 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 09:12:49.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:49.629 | INFO | Uploaded 36GiB to GPU in 2.37985s (15.127GiB/s) | 20260508 09:12:49.629 | INFO | Downloaded 34GiB to GPU in 2.45017s (13.8766GiB/s) | 20260508 09:12:54.761 | INFO | Analyzing chunk frames = [6938112,6938624) | 20260508 09:12:54.761 | INFO | MemInfo: { | 20260508 09:12:54.761 | INFO | "gpu" : " 17 GB", | 20260508 09:12:54.761 | INFO | "huge" : "", | 20260508 09:12:54.761 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:54.762 | INFO | "malloc" : " 14 GB", | 20260508 09:12:54.762 | INFO | "pinned" : " 29 GB" | 20260508 09:12:54.762 | INFO | } | 20260508 09:12:54.762 | INFO | Perf report for the last 5.1335 seconds: | 20260508 09:12:54.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:54.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:12:54.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260508 09:12:54.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:54.762 | INFO | Uploaded 36GiB to GPU in 2.39008s (15.0622GiB/s) | 20260508 09:12:54.763 | INFO | Downloaded 34GiB to GPU in 2.4454s (13.9036GiB/s) | 20260508 09:12:54.763 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:12:54.763 | INFO | Total mem usage: 16.7 GiB | 20260508 09:12:54.763 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:12:54.763 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:12:54.763 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:12:54.763 | INFO | High Water Breakdown: | 20260508 09:12:54.763 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:12:54.763 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:12:54.763 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:12:54.763 | INFO | ------------------------------------------ | 20260508 09:12:54.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:12:54.763 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:12:54.763 | INFO | Total mem usage: 26.5 GiB | 20260508 09:12:54.763 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:12:54.763 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 09:12:54.763 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 09:12:54.763 | INFO | High Water Breakdown: | 20260508 09:12:54.763 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:12:54.763 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:12:54.763 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:12:54.763 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:12:54.763 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:12:54.763 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:12:54.763 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:12:54.763 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:12:54.763 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:12:54.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:12:54.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:12:54.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:12:54.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:12:54.763 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:12:54.763 | INFO | ------------------------------------------ | 20260508 09:12:54.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:12:54.763 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:12:54.763 | INFO | Total mem usage: 27.8 GiB | 20260508 09:12:54.763 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:12:54.763 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:12:54.763 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:12:54.763 | INFO | High Water Breakdown: | 20260508 09:12:54.763 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:12:54.763 | INFO | ------------------------------------------ | 20260508 09:12:54.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:12:54.763 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:12:54.763 | INFO | Total mem usage: 15.7 GiB | 20260508 09:12:54.763 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:12:54.763 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:12:54.763 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:12:54.763 | INFO | High Water Breakdown: | 20260508 09:12:54.763 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:12:54.763 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:12:54.763 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:12:54.763 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:12:54.763 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:12:54.763 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:12:54.763 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:12:54.763 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:12:54.763 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:12:54.763 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:12:54.763 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:12:54.763 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:12:54.763 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:12:54.763 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:12:54.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:12:54.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:12:54.763 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:12:54.763 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:12:54.764 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:12:54.764 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:12:54.764 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:12:54.764 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:12:54.764 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:12:54.764 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:12:54.764 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:12:54.764 | INFO | ------------------------------------------ | 20260508 09:12:54.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:12:54.764 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:12:54.764 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:12:54.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:12:54.764 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.39e+01 / 5.42e-01 | 20260508 09:12:54.764 | INFO | PerfReport: 0.30%, AggregateStats -- 9.82e-02 / 1.07e-01, 1.18e-02 / 5.85e-03 | 20260508 09:12:54.764 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.46e+00, 5.93e-01 / 2.20e-01 | 20260508 09:12:54.764 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.11e-01 / 1.22e-01 | 20260508 09:12:54.764 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.48e+01, 3.61e+00 / 1.08e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.81e+00 / 1.36e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.46e+00, 4.08e-01 / 2.71e-01 | 20260508 09:12:54.764 | INFO | PerfReport: 3.04%, Metrics -- 9.97e-01 / 1.12e+00, 2.51e-01 / 1.03e-01 | 20260508 09:12:54.764 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.37e-01, 4.95e-02 / 2.93e-02 | 20260508 09:12:54.764 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:12:54.764 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:12:54.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:12:54.764 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 2.54%, LockDownload -- 2.68e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 15.34%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:12:54.764 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260508 09:12:54.874 | INFO | Starting a round of Baz IO | 20260508 09:12:58.288 | INFO | Populating RTMetrics report | 20260508 09:12:58.289 | INFO | RTMetrics report populated | 20260508 09:12:58.289 | INFO | Opening tmp RTMetrics json file | 20260508 09:12:58.298 | INFO | Serializing RTMetrics to json | 20260508 09:12:58.300 | INFO | Serialization done, swapping tmp file now... | 20260508 09:12:58.300 | INFO | RTMetrics done | 20260508 09:12:58.355 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.48 seconds | 20260508 09:12:58.355 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.67% | 20260508 09:12:58.355 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.261815% / 0% / 5.96046e-05% | 20260508 09:12:58.355 | INFO | Baz IO overall: | 20260508 09:12:58.355 | INFO | BytesWritten : 3031963158371 | 20260508 09:12:58.355 | INFO | NumEvents : 1458806641363 | 20260508 09:12:59.904 | INFO | Analyzing chunk frames = [6938624,6939136) | 20260508 09:12:59.904 | INFO | MemInfo: { | 20260508 09:12:59.904 | INFO | "gpu" : " 17 GB", | 20260508 09:12:59.904 | INFO | "huge" : "", | 20260508 09:12:59.904 | INFO | "huge_shared" : " 29 GB", | 20260508 09:12:59.904 | INFO | "malloc" : " 14 GB", | 20260508 09:12:59.904 | INFO | "pinned" : " 29 GB" | 20260508 09:12:59.904 | INFO | } | 20260508 09:12:59.904 | INFO | Perf report for the last 5.14244 seconds: | 20260508 09:12:59.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:12:59.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:59.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:59.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:59.904 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 09:12:59.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:12:59.904 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260508 09:12:59.904 | INFO | PRE_HQ : 66.4%, 1.8, 60.8ms, 0 | 20260508 09:12:59.904 | INFO | BAZWRITER : 67.0%, 2.7, 94.4ms, 0 | 20260508 09:12:59.905 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:12:59.905 | INFO | Uploaded 36GiB to GPU in 2.54556s (14.1422GiB/s) | 20260508 09:12:59.905 | INFO | Downloaded 38GiB to GPU in 2.72556s (13.9421GiB/s) | 20260508 09:13:05.058 | INFO | Analyzing chunk frames = [6939136,6939648) | 20260508 09:13:05.058 | INFO | MemInfo: { | 20260508 09:13:05.058 | INFO | "gpu" : " 17 GB", | 20260508 09:13:05.058 | INFO | "huge" : "", | 20260508 09:13:05.058 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:05.058 | INFO | "malloc" : " 14 GB", | 20260508 09:13:05.058 | INFO | "pinned" : " 29 GB" | 20260508 09:13:05.058 | INFO | } | 20260508 09:13:05.058 | INFO | Perf report for the last 5.15352 seconds: | 20260508 09:13:05.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:05.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:13:05.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:13:05.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:05.058 | INFO | Uploaded 36GiB to GPU in 2.51706s (14.3024GiB/s) | 20260508 09:13:05.058 | INFO | Downloaded 34GiB to GPU in 2.44086s (13.9295GiB/s) | 20260508 09:13:10.187 | INFO | Analyzing chunk frames = [6939648,6940160) | 20260508 09:13:10.187 | INFO | MemInfo: { | 20260508 09:13:10.187 | INFO | "gpu" : " 17 GB", | 20260508 09:13:10.187 | INFO | "huge" : "", | 20260508 09:13:10.187 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:10.187 | INFO | "malloc" : " 14 GB", | 20260508 09:13:10.187 | INFO | "pinned" : " 29 GB" | 20260508 09:13:10.187 | INFO | } | 20260508 09:13:10.187 | INFO | Perf report for the last 5.12957 seconds: | 20260508 09:13:10.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:10.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.187 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 09:13:10.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.188 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:13:10.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:10.188 | INFO | Uploaded 36GiB to GPU in 2.4886s (14.466GiB/s) | 20260508 09:13:10.188 | INFO | Downloaded 34GiB to GPU in 2.43804s (13.9456GiB/s) | 20260508 09:13:15.323 | INFO | Analyzing chunk frames = [6940160,6940672) | 20260508 09:13:15.323 | INFO | MemInfo: { | 20260508 09:13:15.323 | INFO | "gpu" : " 17 GB", | 20260508 09:13:15.323 | INFO | "huge" : "", | 20260508 09:13:15.323 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:15.323 | INFO | "malloc" : " 14 GB", | 20260508 09:13:15.323 | INFO | "pinned" : " 29 GB" | 20260508 09:13:15.323 | INFO | } | 20260508 09:13:15.323 | INFO | Perf report for the last 5.13589 seconds: | 20260508 09:13:15.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:15.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.323 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 09:13:15.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.323 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:13:15.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:15.324 | INFO | Uploaded 36GiB to GPU in 2.49724s (14.4159GiB/s) | 20260508 09:13:15.324 | INFO | Downloaded 34GiB to GPU in 2.4266s (14.0114GiB/s) | 20260508 09:13:20.462 | INFO | Analyzing chunk frames = [6940672,6941184) | 20260508 09:13:20.462 | INFO | MemInfo: { | 20260508 09:13:20.462 | INFO | "gpu" : " 17 GB", | 20260508 09:13:20.462 | INFO | "huge" : "", | 20260508 09:13:20.462 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:20.462 | INFO | "malloc" : " 14 GB", | 20260508 09:13:20.462 | INFO | "pinned" : " 29 GB" | 20260508 09:13:20.462 | INFO | } | 20260508 09:13:20.462 | INFO | Perf report for the last 5.13877 seconds: | 20260508 09:13:20.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:20.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 09:13:20.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:13:20.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:20.462 | INFO | Uploaded 36GiB to GPU in 2.32238s (15.5014GiB/s) | 20260508 09:13:20.462 | INFO | Downloaded 34GiB to GPU in 2.50294s (13.584GiB/s) | 20260508 09:13:25.599 | INFO | Analyzing chunk frames = [6941184,6941696) | 20260508 09:13:25.599 | INFO | MemInfo: { | 20260508 09:13:25.599 | INFO | "gpu" : " 17 GB", | 20260508 09:13:25.599 | INFO | "huge" : "", | 20260508 09:13:25.599 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:25.599 | INFO | "malloc" : " 14 GB", | 20260508 09:13:25.599 | INFO | "pinned" : " 29 GB" | 20260508 09:13:25.599 | INFO | } | 20260508 09:13:25.599 | INFO | Perf report for the last 5.13712 seconds: | 20260508 09:13:25.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:25.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.599 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 09:13:25.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.599 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:13:25.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:25.600 | INFO | Uploaded 36GiB to GPU in 2.43796s (14.7665GiB/s) | 20260508 09:13:25.600 | INFO | Downloaded 34GiB to GPU in 2.43353s (13.9715GiB/s) | 20260508 09:13:30.740 | INFO | Analyzing chunk frames = [6941696,6942208) | 20260508 09:13:30.740 | INFO | MemInfo: { | 20260508 09:13:30.740 | INFO | "gpu" : " 17 GB", | 20260508 09:13:30.740 | INFO | "huge" : "", | 20260508 09:13:30.740 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:30.740 | INFO | "malloc" : " 14 GB", | 20260508 09:13:30.740 | INFO | "pinned" : " 29 GB" | 20260508 09:13:30.740 | INFO | } | 20260508 09:13:30.740 | INFO | Perf report for the last 5.14108 seconds: | 20260508 09:13:30.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:30.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 09:13:30.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 09:13:30.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:30.741 | INFO | Uploaded 36GiB to GPU in 2.49472s (14.4305GiB/s) | 20260508 09:13:30.741 | INFO | Downloaded 34GiB to GPU in 2.42972s (13.9934GiB/s) | 20260508 09:13:35.884 | INFO | Analyzing chunk frames = [6942208,6942720) | 20260508 09:13:35.884 | INFO | MemInfo: { | 20260508 09:13:35.884 | INFO | "gpu" : " 17 GB", | 20260508 09:13:35.884 | INFO | "huge" : "", | 20260508 09:13:35.884 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:35.884 | INFO | "malloc" : " 14 GB", | 20260508 09:13:35.884 | INFO | "pinned" : " 29 GB" | 20260508 09:13:35.884 | INFO | } | 20260508 09:13:35.884 | INFO | Perf report for the last 5.1441 seconds: | 20260508 09:13:35.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:35.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.885 | INFO | ANALYSIS : 65.9%, 3.0, 103.2ms, 0 | 20260508 09:13:35.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.885 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 09:13:35.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:35.885 | INFO | Uploaded 36GiB to GPU in 2.46152s (14.6251GiB/s) | 20260508 09:13:35.885 | INFO | Downloaded 34GiB to GPU in 2.42723s (14.0077GiB/s) | 20260508 09:13:36.014 | INFO | Starting a round of Baz IO | 20260508 09:13:39.400 | INFO | Populating RTMetrics report | 20260508 09:13:39.401 | INFO | RTMetrics report populated | 20260508 09:13:39.401 | INFO | Opening tmp RTMetrics json file | 20260508 09:13:39.409 | INFO | Serializing RTMetrics to json | 20260508 09:13:39.410 | INFO | Serialization done, swapping tmp file now... | 20260508 09:13:39.411 | INFO | RTMetrics done | 20260508 09:13:39.471 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.457 seconds | 20260508 09:13:39.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.67% | 20260508 09:13:39.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262618% / 0% / 8.34465e-05% | 20260508 09:13:39.471 | INFO | Baz IO overall: | 20260508 09:13:39.471 | INFO | BytesWritten : 3033558073594 | 20260508 09:13:39.471 | INFO | NumEvents : 1459560757163 | 20260508 09:13:41.020 | INFO | Analyzing chunk frames = [6942720,6943232) | 20260508 09:13:41.020 | INFO | MemInfo: { | 20260508 09:13:41.020 | INFO | "gpu" : " 17 GB", | 20260508 09:13:41.020 | INFO | "huge" : "", | 20260508 09:13:41.020 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:41.020 | INFO | "malloc" : " 14 GB", | 20260508 09:13:41.020 | INFO | "pinned" : " 29 GB" | 20260508 09:13:41.020 | INFO | } | 20260508 09:13:41.020 | INFO | Perf report for the last 5.13596 seconds: | 20260508 09:13:41.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:41.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:41.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:41.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:41.021 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 09:13:41.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:41.021 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260508 09:13:41.021 | INFO | PRE_HQ : 66.5%, 1.8, 61.8ms, 0 | 20260508 09:13:41.021 | INFO | BAZWRITER : 67.1%, 2.7, 94.2ms, 0 | 20260508 09:13:41.021 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:13:41.021 | INFO | Uploaded 36GiB to GPU in 2.60556s (13.8166GiB/s) | 20260508 09:13:41.021 | INFO | Downloaded 38GiB to GPU in 2.70371s (14.0547GiB/s) | 20260508 09:13:46.164 | INFO | Analyzing chunk frames = [6943232,6943744) | 20260508 09:13:46.164 | INFO | MemInfo: { | 20260508 09:13:46.164 | INFO | "gpu" : " 17 GB", | 20260508 09:13:46.164 | INFO | "huge" : "", | 20260508 09:13:46.164 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:46.164 | INFO | "malloc" : " 14 GB", | 20260508 09:13:46.164 | INFO | "pinned" : " 29 GB" | 20260508 09:13:46.164 | INFO | } | 20260508 09:13:46.164 | INFO | Perf report for the last 5.14379 seconds: | 20260508 09:13:46.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:46.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.164 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:13:46.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.164 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:13:46.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:46.165 | INFO | Uploaded 36GiB to GPU in 2.50909s (14.3478GiB/s) | 20260508 09:13:46.165 | INFO | Downloaded 34GiB to GPU in 2.45619s (13.8426GiB/s) | 20260508 09:13:51.299 | INFO | Analyzing chunk frames = [6943744,6944256) | 20260508 09:13:51.300 | INFO | MemInfo: { | 20260508 09:13:51.300 | INFO | "gpu" : " 17 GB", | 20260508 09:13:51.300 | INFO | "huge" : "", | 20260508 09:13:51.300 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:51.300 | INFO | "malloc" : " 14 GB", | 20260508 09:13:51.300 | INFO | "pinned" : " 29 GB" | 20260508 09:13:51.300 | INFO | } | 20260508 09:13:51.300 | INFO | Perf report for the last 5.13551 seconds: | 20260508 09:13:51.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:51.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.300 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 09:13:51.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.300 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:13:51.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:51.301 | INFO | Uploaded 36GiB to GPU in 2.43819s (14.765GiB/s) | 20260508 09:13:51.301 | INFO | Downloaded 34GiB to GPU in 2.41913s (14.0547GiB/s) | 20260508 09:13:56.440 | INFO | Analyzing chunk frames = [6944256,6944768) | 20260508 09:13:56.440 | INFO | MemInfo: { | 20260508 09:13:56.440 | INFO | "gpu" : " 17 GB", | 20260508 09:13:56.440 | INFO | "huge" : "", | 20260508 09:13:56.440 | INFO | "huge_shared" : " 29 GB", | 20260508 09:13:56.440 | INFO | "malloc" : " 14 GB", | 20260508 09:13:56.440 | INFO | "pinned" : " 29 GB" | 20260508 09:13:56.440 | INFO | } | 20260508 09:13:56.441 | INFO | Perf report for the last 5.14094 seconds: | 20260508 09:13:56.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:13:56.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 09:13:56.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:13:56.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:13:56.441 | INFO | Uploaded 36GiB to GPU in 2.56581s (14.0306GiB/s) | 20260508 09:13:56.441 | INFO | Downloaded 34GiB to GPU in 2.43693s (13.952GiB/s) | 20260508 09:14:01.580 | INFO | Analyzing chunk frames = [6944768,6945280) | 20260508 09:14:01.580 | INFO | MemInfo: { | 20260508 09:14:01.580 | INFO | "gpu" : " 17 GB", | 20260508 09:14:01.580 | INFO | "huge" : "", | 20260508 09:14:01.580 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:01.580 | INFO | "malloc" : " 14 GB", | 20260508 09:14:01.580 | INFO | "pinned" : " 29 GB" | 20260508 09:14:01.580 | INFO | } | 20260508 09:14:01.580 | INFO | Perf report for the last 5.13944 seconds: | 20260508 09:14:01.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:01.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 09:14:01.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:14:01.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:01.580 | INFO | Uploaded 36GiB to GPU in 2.30053s (15.6486GiB/s) | 20260508 09:14:01.580 | INFO | Downloaded 34GiB to GPU in 2.50294s (13.5841GiB/s) | 20260508 09:14:06.718 | INFO | Analyzing chunk frames = [6945280,6945792) | 20260508 09:14:06.718 | INFO | MemInfo: { | 20260508 09:14:06.718 | INFO | "gpu" : " 17 GB", | 20260508 09:14:06.718 | INFO | "huge" : "", | 20260508 09:14:06.718 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:06.718 | INFO | "malloc" : " 14 GB", | 20260508 09:14:06.718 | INFO | "pinned" : " 29 GB" | 20260508 09:14:06.718 | INFO | } | 20260508 09:14:06.718 | INFO | Perf report for the last 5.13809 seconds: | 20260508 09:14:06.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:06.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.718 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:14:06.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.718 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:14:06.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:06.719 | INFO | Uploaded 36GiB to GPU in 2.33876s (15.3928GiB/s) | 20260508 09:14:06.719 | INFO | Downloaded 34GiB to GPU in 2.44661s (13.8968GiB/s) | 20260508 09:14:11.859 | INFO | Analyzing chunk frames = [6945792,6946304) | 20260508 09:14:11.859 | INFO | MemInfo: { | 20260508 09:14:11.859 | INFO | "gpu" : " 17 GB", | 20260508 09:14:11.859 | INFO | "huge" : "", | 20260508 09:14:11.859 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:11.859 | INFO | "malloc" : " 14 GB", | 20260508 09:14:11.859 | INFO | "pinned" : " 29 GB" | 20260508 09:14:11.859 | INFO | } | 20260508 09:14:11.859 | INFO | Perf report for the last 5.14055 seconds: | 20260508 09:14:11.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:11.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:14:11.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:14:11.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:11.859 | INFO | Uploaded 36GiB to GPU in 2.38642s (15.0854GiB/s) | 20260508 09:14:11.859 | INFO | Downloaded 34GiB to GPU in 2.45495s (13.8496GiB/s) | 20260508 09:14:16.998 | INFO | Analyzing chunk frames = [6946304,6946816) | 20260508 09:14:16.998 | INFO | MemInfo: { | 20260508 09:14:16.998 | INFO | "gpu" : " 17 GB", | 20260508 09:14:16.998 | INFO | "huge" : "", | 20260508 09:14:16.998 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:16.998 | INFO | "malloc" : " 14 GB", | 20260508 09:14:16.998 | INFO | "pinned" : " 29 GB" | 20260508 09:14:16.998 | INFO | } | 20260508 09:14:16.998 | INFO | Perf report for the last 5.13934 seconds: | 20260508 09:14:16.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:16.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.998 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 09:14:16.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.998 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:14:16.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:16.999 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:14:16.999 | INFO | Total mem usage: 16.7 GiB | 20260508 09:14:16.999 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:14:16.999 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:14:16.999 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:14:16.999 | INFO | High Water Breakdown: | 20260508 09:14:16.999 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:14:16.999 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:14:16.999 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:14:16.999 | INFO | ------------------------------------------ | 20260508 09:14:16.999 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:14:16.999 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:14:16.999 | INFO | Total mem usage: 26.5 GiB | 20260508 09:14:16.999 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:14:16.999 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:14:16.999 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:14:16.999 | INFO | High Water Breakdown: | 20260508 09:14:16.999 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:14:16.999 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:14:16.999 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:14:16.999 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:14:16.999 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:14:16.999 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:14:16.999 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:14:16.999 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:14:16.999 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:14:16.999 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:14:16.999 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:14:16.999 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:14:16.999 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:14:16.999 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:14:16.999 | INFO | ------------------------------------------ | 20260508 09:14:16.999 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:14:16.999 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:14:16.999 | INFO | Total mem usage: 27.8 GiB | 20260508 09:14:16.999 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:14:16.999 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:14:16.999 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:14:16.999 | INFO | High Water Breakdown: | 20260508 09:14:16.999 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:14:16.999 | INFO | ------------------------------------------ | 20260508 09:14:16.999 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:14:16.999 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:14:16.999 | INFO | Total mem usage: 15.7 GiB | 20260508 09:14:16.999 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:14:16.999 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:14:16.999 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:14:16.999 | INFO | High Water Breakdown: | 20260508 09:14:16.999 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:14:16.999 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:14:16.999 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:14:16.999 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:14:16.999 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:14:16.999 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:14:16.999 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:14:16.999 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:14:16.999 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:14:16.999 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:14:16.999 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:14:16.999 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:14:16.999 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:14:16.999 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:14:16.999 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:14:16.999 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:14:16.999 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:14:16.999 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:14:16.999 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:14:16.999 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:14:16.999 | INFO | Uploaded 36GiB to GPU in 2.40395s (14.9754GiB/s) | 20260508 09:14:16.999 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:14:16.999 | INFO | Downloaded 34GiB to GPU in 2.46252s (13.807GiB/s) | 20260508 09:14:16.999 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:14:16.999 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:14:16.999 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:14:16.999 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:14:16.999 | INFO | ------------------------------------------ | 20260508 09:14:16.999 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:14:16.999 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:14:17.000 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:14:17.000 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:14:17.000 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.16e-01 | 20260508 09:14:17.000 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.14e-02 / 6.61e-03 | 20260508 09:14:17.000 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.42e+00, 5.94e-01 / 2.23e-01 | 20260508 09:14:17.000 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.43e-01 | 20260508 09:14:17.000 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.57e+01, 3.89e+00 / 1.05e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.81e+00 / 1.23e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.54e+00, 4.06e-01 / 2.63e-01 | 20260508 09:14:17.000 | INFO | PerfReport: 3.05%, Metrics -- 9.98e-01 / 1.14e+00, 2.47e-01 / 1.14e-01 | 20260508 09:14:17.000 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.35e-01, 4.82e-02 / 2.12e-02 | 20260508 09:14:17.000 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:14:17.000 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:14:17.000 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:14:17.000 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 24.27%, Upload -- 2.56e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 24.51%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 31.01%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 2.18%, LockUpload -- 2.30e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 2.61%, LockDownload -- 2.76e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 15.40%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 09:14:17.000 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260508 09:14:17.097 | INFO | Starting a round of Baz IO | 20260508 09:14:20.517 | INFO | Populating RTMetrics report | 20260508 09:14:20.518 | INFO | RTMetrics report populated | 20260508 09:14:20.518 | INFO | Opening tmp RTMetrics json file | 20260508 09:14:20.527 | INFO | Serializing RTMetrics to json | 20260508 09:14:20.528 | INFO | Serialization done, swapping tmp file now... | 20260508 09:14:20.529 | INFO | RTMetrics done | 20260508 09:14:20.585 | INFO | Finished a round of Baz IO: Wrote 1.486 GiB in 3.487 seconds | 20260508 09:14:20.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.67% | 20260508 09:14:20.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.261831% / 0% / 6.35783e-05% | 20260508 09:14:20.585 | INFO | Baz IO overall: | 20260508 09:14:20.585 | INFO | BytesWritten : 3035153984881 | 20260508 09:14:20.585 | INFO | NumEvents : 1460315456122 | 20260508 09:14:22.136 | INFO | Analyzing chunk frames = [6946816,6947328) | 20260508 09:14:22.136 | INFO | MemInfo: { | 20260508 09:14:22.136 | INFO | "gpu" : " 17 GB", | 20260508 09:14:22.136 | INFO | "huge" : "", | 20260508 09:14:22.136 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:22.136 | INFO | "malloc" : " 14 GB", | 20260508 09:14:22.136 | INFO | "pinned" : " 29 GB" | 20260508 09:14:22.136 | INFO | } | 20260508 09:14:22.136 | INFO | Perf report for the last 5.13785 seconds: | 20260508 09:14:22.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:22.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:22.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:22.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:22.136 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:14:22.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:22.136 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:14:22.136 | INFO | PRE_HQ : 65.5%, 1.6, 54.4ms, 0 | 20260508 09:14:22.136 | INFO | BAZWRITER : 67.2%, 2.6, 88.2ms, 0 | 20260508 09:14:22.136 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:14:22.137 | INFO | Uploaded 36GiB to GPU in 2.58014s (13.9528GiB/s) | 20260508 09:14:22.137 | INFO | Downloaded 38GiB to GPU in 2.70848s (14.03GiB/s) | 20260508 09:14:27.273 | INFO | Analyzing chunk frames = [6947328,6947840) | 20260508 09:14:27.273 | INFO | MemInfo: { | 20260508 09:14:27.273 | INFO | "gpu" : " 17 GB", | 20260508 09:14:27.273 | INFO | "huge" : "", | 20260508 09:14:27.273 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:27.273 | INFO | "malloc" : " 14 GB", | 20260508 09:14:27.273 | INFO | "pinned" : " 29 GB" | 20260508 09:14:27.273 | INFO | } | 20260508 09:14:27.273 | INFO | Perf report for the last 5.13706 seconds: | 20260508 09:14:27.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:27.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:14:27.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:14:27.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:27.273 | INFO | Uploaded 36GiB to GPU in 2.49901s (14.4057GiB/s) | 20260508 09:14:27.273 | INFO | Downloaded 34GiB to GPU in 2.50116s (13.5937GiB/s) | 20260508 09:14:32.419 | INFO | Analyzing chunk frames = [6947840,6948352) | 20260508 09:14:32.419 | INFO | MemInfo: { | 20260508 09:14:32.419 | INFO | "gpu" : " 17 GB", | 20260508 09:14:32.419 | INFO | "huge" : "", | 20260508 09:14:32.419 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:32.419 | INFO | "malloc" : " 14 GB", | 20260508 09:14:32.419 | INFO | "pinned" : " 29 GB" | 20260508 09:14:32.420 | INFO | } | 20260508 09:14:32.420 | INFO | Perf report for the last 5.14644 seconds: | 20260508 09:14:32.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:32.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 09:14:32.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:14:32.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:32.420 | INFO | Uploaded 36GiB to GPU in 2.44491s (14.7245GiB/s) | 20260508 09:14:32.420 | INFO | Downloaded 34GiB to GPU in 2.45279s (13.8618GiB/s) | 20260508 09:14:37.556 | INFO | Analyzing chunk frames = [6948352,6948864) | 20260508 09:14:37.556 | INFO | MemInfo: { | 20260508 09:14:37.556 | INFO | "gpu" : " 17 GB", | 20260508 09:14:37.556 | INFO | "huge" : "", | 20260508 09:14:37.556 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:37.556 | INFO | "malloc" : " 14 GB", | 20260508 09:14:37.556 | INFO | "pinned" : " 29 GB" | 20260508 09:14:37.556 | INFO | } | 20260508 09:14:37.556 | INFO | Perf report for the last 5.13625 seconds: | 20260508 09:14:37.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:37.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 09:14:37.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:14:37.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:37.556 | INFO | Uploaded 36GiB to GPU in 2.4194s (14.8797GiB/s) | 20260508 09:14:37.556 | INFO | Downloaded 34GiB to GPU in 2.43494s (13.9634GiB/s) | 20260508 09:14:42.702 | INFO | Analyzing chunk frames = [6948864,6949376) | 20260508 09:14:42.702 | INFO | MemInfo: { | 20260508 09:14:42.702 | INFO | "gpu" : " 17 GB", | 20260508 09:14:42.702 | INFO | "huge" : "", | 20260508 09:14:42.702 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:42.702 | INFO | "malloc" : " 14 GB", | 20260508 09:14:42.702 | INFO | "pinned" : " 29 GB" | 20260508 09:14:42.702 | INFO | } | 20260508 09:14:42.702 | INFO | Perf report for the last 5.14665 seconds: | 20260508 09:14:42.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:42.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.703 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 09:14:42.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.703 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 09:14:42.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:42.703 | INFO | Uploaded 36GiB to GPU in 2.37439s (15.1618GiB/s) | 20260508 09:14:42.703 | INFO | Downloaded 34GiB to GPU in 2.54053s (13.383GiB/s) | 20260508 09:14:47.831 | INFO | Analyzing chunk frames = [6949376,6949888) | 20260508 09:14:47.831 | INFO | MemInfo: { | 20260508 09:14:47.831 | INFO | "gpu" : " 17 GB", | 20260508 09:14:47.831 | INFO | "huge" : "", | 20260508 09:14:47.831 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:47.831 | INFO | "malloc" : " 14 GB", | 20260508 09:14:47.831 | INFO | "pinned" : " 29 GB" | 20260508 09:14:47.831 | INFO | } | 20260508 09:14:47.831 | INFO | Perf report for the last 5.1287 seconds: | 20260508 09:14:47.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:47.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.831 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 09:14:47.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.831 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 09:14:47.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:47.832 | INFO | Uploaded 36GiB to GPU in 2.36779s (15.204GiB/s) | 20260508 09:14:47.832 | INFO | Downloaded 34GiB to GPU in 2.4566s (13.8403GiB/s) | 20260508 09:14:52.968 | INFO | Analyzing chunk frames = [6949888,6950400) | 20260508 09:14:52.968 | INFO | MemInfo: { | 20260508 09:14:52.968 | INFO | "gpu" : " 17 GB", | 20260508 09:14:52.968 | INFO | "huge" : "", | 20260508 09:14:52.968 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:52.968 | INFO | "malloc" : " 14 GB", | 20260508 09:14:52.968 | INFO | "pinned" : " 29 GB" | 20260508 09:14:52.968 | INFO | } | 20260508 09:14:52.968 | INFO | Perf report for the last 5.13723 seconds: | 20260508 09:14:52.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:52.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.969 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:14:52.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.969 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:14:52.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:52.969 | INFO | Uploaded 36GiB to GPU in 2.40391s (14.9756GiB/s) | 20260508 09:14:52.969 | INFO | Downloaded 34GiB to GPU in 2.45357s (13.8574GiB/s) | 20260508 09:14:58.109 | INFO | Analyzing chunk frames = [6950400,6950912) | 20260508 09:14:58.109 | INFO | MemInfo: { | 20260508 09:14:58.109 | INFO | "gpu" : " 17 GB", | 20260508 09:14:58.109 | INFO | "huge" : "", | 20260508 09:14:58.109 | INFO | "huge_shared" : " 29 GB", | 20260508 09:14:58.109 | INFO | "malloc" : " 14 GB", | 20260508 09:14:58.109 | INFO | "pinned" : " 29 GB" | 20260508 09:14:58.109 | INFO | } | 20260508 09:14:58.109 | INFO | Perf report for the last 5.1407 seconds: | 20260508 09:14:58.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:14:58.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:14:58.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:14:58.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:14:58.109 | INFO | Uploaded 36GiB to GPU in 2.39671s (15.0206GiB/s) | 20260508 09:14:58.109 | INFO | Downloaded 34GiB to GPU in 2.44344s (13.9148GiB/s) | 20260508 09:14:58.207 | INFO | Starting a round of Baz IO | 20260508 09:15:01.630 | INFO | Populating RTMetrics report | 20260508 09:15:01.630 | INFO | RTMetrics report populated | 20260508 09:15:01.630 | INFO | Opening tmp RTMetrics json file | 20260508 09:15:01.640 | INFO | Serializing RTMetrics to json | 20260508 09:15:01.642 | INFO | Serialization done, swapping tmp file now... | 20260508 09:15:01.643 | INFO | RTMetrics done | 20260508 09:15:01.707 | INFO | Finished a round of Baz IO: Wrote 1.487 GiB in 3.5 seconds | 20260508 09:15:01.707 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.67% | 20260508 09:15:01.707 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262535% / 3.97364e-06% / 6.35783e-05% | 20260508 09:15:01.707 | INFO | Baz IO overall: | 20260508 09:15:01.707 | INFO | BytesWritten : 3036750599429 | 20260508 09:15:01.707 | INFO | NumEvents : 1461070842894 | 20260508 09:15:03.250 | INFO | Analyzing chunk frames = [6950912,6951424) | 20260508 09:15:03.251 | INFO | MemInfo: { | 20260508 09:15:03.251 | INFO | "gpu" : " 17 GB", | 20260508 09:15:03.251 | INFO | "huge" : "", | 20260508 09:15:03.251 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:03.251 | INFO | "malloc" : " 14 GB", | 20260508 09:15:03.251 | INFO | "pinned" : " 29 GB" | 20260508 09:15:03.251 | INFO | } | 20260508 09:15:03.251 | INFO | Perf report for the last 5.14143 seconds: | 20260508 09:15:03.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:03.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:03.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:03.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:03.251 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 09:15:03.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:03.251 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 09:15:03.251 | INFO | PRE_HQ : 65.2%, 1.6, 54.4ms, 0 | 20260508 09:15:03.251 | INFO | BAZWRITER : 67.6%, 2.6, 90.7ms, 0 | 20260508 09:15:03.251 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 09:15:03.251 | INFO | Uploaded 36GiB to GPU in 2.58752s (13.9129GiB/s) | 20260508 09:15:03.251 | INFO | Downloaded 38GiB to GPU in 2.71336s (14.0048GiB/s) | 20260508 09:15:08.390 | INFO | Analyzing chunk frames = [6951424,6951936) | 20260508 09:15:08.390 | INFO | MemInfo: { | 20260508 09:15:08.390 | INFO | "gpu" : " 17 GB", | 20260508 09:15:08.390 | INFO | "huge" : "", | 20260508 09:15:08.390 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:08.391 | INFO | "malloc" : " 14 GB", | 20260508 09:15:08.391 | INFO | "pinned" : " 29 GB" | 20260508 09:15:08.391 | INFO | } | 20260508 09:15:08.391 | INFO | Perf report for the last 5.13999 seconds: | 20260508 09:15:08.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:08.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:15:08.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:15:08.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:08.391 | INFO | Uploaded 36GiB to GPU in 2.51215s (14.3304GiB/s) | 20260508 09:15:08.391 | INFO | Downloaded 34GiB to GPU in 2.44165s (13.925GiB/s) | 20260508 09:15:13.526 | INFO | Analyzing chunk frames = [6951936,6952448) | 20260508 09:15:13.526 | INFO | MemInfo: { | 20260508 09:15:13.526 | INFO | "gpu" : " 17 GB", | 20260508 09:15:13.526 | INFO | "huge" : "", | 20260508 09:15:13.526 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:13.526 | INFO | "malloc" : " 14 GB", | 20260508 09:15:13.526 | INFO | "pinned" : " 29 GB" | 20260508 09:15:13.527 | INFO | } | 20260508 09:15:13.527 | INFO | Perf report for the last 5.13611 seconds: | 20260508 09:15:13.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:13.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:15:13.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:15:13.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:13.527 | INFO | Uploaded 36GiB to GPU in 2.45898s (14.6402GiB/s) | 20260508 09:15:13.527 | INFO | Downloaded 34GiB to GPU in 2.42496s (14.0209GiB/s) | 20260508 09:15:18.676 | INFO | Analyzing chunk frames = [6952448,6952960) | 20260508 09:15:18.676 | INFO | MemInfo: { | 20260508 09:15:18.676 | INFO | "gpu" : " 17 GB", | 20260508 09:15:18.676 | INFO | "huge" : "", | 20260508 09:15:18.676 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:18.676 | INFO | "malloc" : " 14 GB", | 20260508 09:15:18.676 | INFO | "pinned" : " 29 GB" | 20260508 09:15:18.676 | INFO | } | 20260508 09:15:18.676 | INFO | Perf report for the last 5.14973 seconds: | 20260508 09:15:18.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:18.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 09:15:18.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:15:18.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:18.677 | INFO | Uploaded 36GiB to GPU in 2.49597s (14.4233GiB/s) | 20260508 09:15:18.677 | INFO | Downloaded 34GiB to GPU in 2.42691s (14.0096GiB/s) | 20260508 09:15:23.812 | INFO | Analyzing chunk frames = [6952960,6953472) | 20260508 09:15:23.812 | INFO | MemInfo: { | 20260508 09:15:23.812 | INFO | "gpu" : " 17 GB", | 20260508 09:15:23.812 | INFO | "huge" : "", | 20260508 09:15:23.812 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:23.812 | INFO | "malloc" : " 14 GB", | 20260508 09:15:23.812 | INFO | "pinned" : " 29 GB" | 20260508 09:15:23.812 | INFO | } | 20260508 09:15:23.812 | INFO | Perf report for the last 5.13567 seconds: | 20260508 09:15:23.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:23.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:15:23.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 09:15:23.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:23.812 | INFO | Uploaded 36GiB to GPU in 2.42681s (14.8343GiB/s) | 20260508 09:15:23.812 | INFO | Downloaded 34GiB to GPU in 2.52487s (13.466GiB/s) | 20260508 09:15:28.949 | INFO | Analyzing chunk frames = [6953472,6953984) | 20260508 09:15:28.949 | INFO | MemInfo: { | 20260508 09:15:28.949 | INFO | "gpu" : " 17 GB", | 20260508 09:15:28.949 | INFO | "huge" : "", | 20260508 09:15:28.949 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:28.949 | INFO | "malloc" : " 14 GB", | 20260508 09:15:28.949 | INFO | "pinned" : " 29 GB" | 20260508 09:15:28.949 | INFO | } | 20260508 09:15:28.949 | INFO | Perf report for the last 5.13689 seconds: | 20260508 09:15:28.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:28.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:15:28.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:15:28.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:28.949 | INFO | Uploaded 36GiB to GPU in 2.37989s (15.1268GiB/s) | 20260508 09:15:28.949 | INFO | Downloaded 34GiB to GPU in 2.41826s (14.0597GiB/s) | 20260508 09:15:34.084 | INFO | Analyzing chunk frames = [6953984,6954496) | 20260508 09:15:34.084 | INFO | MemInfo: { | 20260508 09:15:34.084 | INFO | "gpu" : " 17 GB", | 20260508 09:15:34.084 | INFO | "huge" : "", | 20260508 09:15:34.084 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:34.084 | INFO | "malloc" : " 14 GB", | 20260508 09:15:34.085 | INFO | "pinned" : " 29 GB" | 20260508 09:15:34.085 | INFO | } | 20260508 09:15:34.085 | INFO | Perf report for the last 5.13567 seconds: | 20260508 09:15:34.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:34.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 09:15:34.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:15:34.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:34.085 | INFO | Uploaded 36GiB to GPU in 2.46051s (14.6311GiB/s) | 20260508 09:15:34.085 | INFO | Downloaded 34GiB to GPU in 2.43196s (13.9805GiB/s) | 20260508 09:15:39.226 | INFO | Analyzing chunk frames = [6954496,6955008) | 20260508 09:15:39.226 | INFO | MemInfo: { | 20260508 09:15:39.226 | INFO | "gpu" : " 17 GB", | 20260508 09:15:39.226 | INFO | "huge" : "", | 20260508 09:15:39.226 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:39.226 | INFO | "malloc" : " 14 GB", | 20260508 09:15:39.227 | INFO | "pinned" : " 29 GB" | 20260508 09:15:39.227 | INFO | } | 20260508 09:15:39.227 | INFO | Perf report for the last 5.14195 seconds: | 20260508 09:15:39.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:39.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:15:39.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:15:39.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:39.227 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:15:39.227 | INFO | Total mem usage: 16.7 GiB | 20260508 09:15:39.227 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:15:39.227 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:15:39.227 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:15:39.227 | INFO | High Water Breakdown: | 20260508 09:15:39.227 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:15:39.227 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:15:39.227 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:15:39.227 | INFO | ------------------------------------------ | 20260508 09:15:39.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:15:39.227 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:15:39.227 | INFO | Total mem usage: 26.5 GiB | 20260508 09:15:39.227 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:15:39.227 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:15:39.227 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:15:39.227 | INFO | High Water Breakdown: | 20260508 09:15:39.227 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:15:39.227 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:15:39.227 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:15:39.227 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:15:39.227 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:15:39.227 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:15:39.227 | INFO | Uploaded 36GiB to GPU in 2.43185s (14.8035GiB/s) | 20260508 09:15:39.227 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:15:39.227 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:15:39.227 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:15:39.227 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:15:39.227 | INFO | Downloaded 34GiB to GPU in 2.42568s (14.0167GiB/s) | 20260508 09:15:39.227 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:15:39.228 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:15:39.228 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:15:39.228 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:15:39.228 | INFO | ------------------------------------------ | 20260508 09:15:39.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:15:39.228 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:15:39.228 | INFO | Total mem usage: 27.8 GiB | 20260508 09:15:39.228 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:15:39.228 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:15:39.228 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:15:39.228 | INFO | High Water Breakdown: | 20260508 09:15:39.228 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:15:39.228 | INFO | ------------------------------------------ | 20260508 09:15:39.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:15:39.228 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:15:39.228 | INFO | Total mem usage: 15.7 GiB | 20260508 09:15:39.228 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:15:39.228 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:15:39.228 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:15:39.228 | INFO | High Water Breakdown: | 20260508 09:15:39.228 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:15:39.228 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:15:39.228 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:15:39.228 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:15:39.228 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:15:39.228 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:15:39.228 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:15:39.228 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:15:39.228 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:15:39.228 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:15:39.228 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:15:39.228 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:15:39.228 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:15:39.228 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:15:39.228 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:15:39.228 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:15:39.228 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:15:39.228 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:15:39.228 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:15:39.228 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:15:39.228 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:15:39.228 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:15:39.228 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:15:39.228 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:15:39.228 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:15:39.228 | INFO | ------------------------------------------ | 20260508 09:15:39.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:15:39.228 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:15:39.228 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:15:39.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:15:39.228 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 5.42e-01 | 20260508 09:15:39.228 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.10e-02 / 6.83e-03 | 20260508 09:15:39.228 | INFO | PerfReport: 12.14%, Baseline -- 3.98e+00 / 4.38e+00, 5.93e-01 / 2.39e-01 | 20260508 09:15:39.228 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.43e+00, 2.09e-01 / 1.40e-01 | 20260508 09:15:39.228 | INFO | PerfReport: 15.93%, DME -- 4.18e+01 / 4.51e+01, 3.73e+00 / 1.23e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.88e+00 / 1.33e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.43e+00, 4.07e-01 / 2.97e-01 | 20260508 09:15:39.228 | INFO | PerfReport: 3.02%, Metrics -- 9.91e-01 / 1.13e+00, 2.43e-01 / 1.10e-01 | 20260508 09:15:39.228 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.29e-01, 4.79e-02 / 2.36e-02 | 20260508 09:15:39.228 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:15:39.228 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:15:39.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:15:39.228 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 24.25%, Upload -- 2.55e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 31.10%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 2.10%, LockUpload -- 2.21e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 2.62%, LockDownload -- 2.76e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 15.32%, LockCompute -- 1.61e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:15:39.228 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 3.58e-02 / 0.00e+00 | 20260508 09:15:39.324 | INFO | Starting a round of Baz IO | 20260508 09:15:42.732 | INFO | Populating RTMetrics report | 20260508 09:15:42.733 | INFO | RTMetrics report populated | 20260508 09:15:42.733 | INFO | Opening tmp RTMetrics json file | 20260508 09:15:42.744 | INFO | Serializing RTMetrics to json | 20260508 09:15:42.745 | INFO | Serialization done, swapping tmp file now... | 20260508 09:15:42.746 | INFO | RTMetrics done | 20260508 09:15:42.803 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.479 seconds | 20260508 09:15:42.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.33% / 0.00% / 59.67% | 20260508 09:15:42.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.260735% / 0% / 6.35783e-05% | 20260508 09:15:42.803 | INFO | Baz IO overall: | 20260508 09:15:42.803 | INFO | BytesWritten : 3038344646761 | 20260508 09:15:42.804 | INFO | NumEvents : 1461824558342 | 20260508 09:15:44.367 | INFO | Analyzing chunk frames = [6955008,6955520) | 20260508 09:15:44.368 | INFO | MemInfo: { | 20260508 09:15:44.368 | INFO | "gpu" : " 17 GB", | 20260508 09:15:44.368 | INFO | "huge" : "", | 20260508 09:15:44.368 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:44.368 | INFO | "malloc" : " 14 GB", | 20260508 09:15:44.368 | INFO | "pinned" : " 29 GB" | 20260508 09:15:44.368 | INFO | } | 20260508 09:15:44.368 | INFO | Perf report for the last 5.14102 seconds: | 20260508 09:15:44.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:44.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:44.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:44.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:44.368 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 09:15:44.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:44.368 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:15:44.368 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 09:15:44.368 | INFO | BAZWRITER : 67.0%, 2.6, 88.0ms, 0 | 20260508 09:15:44.368 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:15:44.368 | INFO | Uploaded 36GiB to GPU in 2.61076s (13.7891GiB/s) | 20260508 09:15:44.368 | INFO | Downloaded 38GiB to GPU in 2.70796s (14.0327GiB/s) | 20260508 09:15:49.513 | INFO | Analyzing chunk frames = [6955520,6956032) | 20260508 09:15:49.513 | INFO | MemInfo: { | 20260508 09:15:49.513 | INFO | "gpu" : " 17 GB", | 20260508 09:15:49.513 | INFO | "huge" : "", | 20260508 09:15:49.513 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:49.514 | INFO | "malloc" : " 14 GB", | 20260508 09:15:49.514 | INFO | "pinned" : " 29 GB" | 20260508 09:15:49.514 | INFO | } | 20260508 09:15:49.514 | INFO | Perf report for the last 5.14588 seconds: | 20260508 09:15:49.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:49.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:15:49.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:15:49.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:49.514 | INFO | Uploaded 36GiB to GPU in 2.50537s (14.3691GiB/s) | 20260508 09:15:49.514 | INFO | Downloaded 34GiB to GPU in 2.44942s (13.8809GiB/s) | 20260508 09:15:54.642 | INFO | Analyzing chunk frames = [6956032,6956544) | 20260508 09:15:54.642 | INFO | MemInfo: { | 20260508 09:15:54.642 | INFO | "gpu" : " 17 GB", | 20260508 09:15:54.642 | INFO | "huge" : "", | 20260508 09:15:54.642 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:54.642 | INFO | "malloc" : " 14 GB", | 20260508 09:15:54.642 | INFO | "pinned" : " 29 GB" | 20260508 09:15:54.642 | INFO | } | 20260508 09:15:54.642 | INFO | Perf report for the last 5.12863 seconds: | 20260508 09:15:54.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:54.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.642 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:15:54.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.642 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 09:15:54.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:54.643 | INFO | Uploaded 36GiB to GPU in 2.4781s (14.5272GiB/s) | 20260508 09:15:54.643 | INFO | Downloaded 34GiB to GPU in 2.44064s (13.9308GiB/s) | 20260508 09:15:59.784 | INFO | Analyzing chunk frames = [6956544,6957056) | 20260508 09:15:59.784 | INFO | MemInfo: { | 20260508 09:15:59.784 | INFO | "gpu" : " 17 GB", | 20260508 09:15:59.784 | INFO | "huge" : "", | 20260508 09:15:59.784 | INFO | "huge_shared" : " 29 GB", | 20260508 09:15:59.784 | INFO | "malloc" : " 14 GB", | 20260508 09:15:59.784 | INFO | "pinned" : " 29 GB" | 20260508 09:15:59.784 | INFO | } | 20260508 09:15:59.784 | INFO | Perf report for the last 5.14196 seconds: | 20260508 09:15:59.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:15:59.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.784 | INFO | ANALYSIS : 66.8%, 2.9, 103.5ms, 0 | 20260508 09:15:59.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.784 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:15:59.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:15:59.785 | INFO | Uploaded 36GiB to GPU in 2.56824s (14.0174GiB/s) | 20260508 09:15:59.785 | INFO | Downloaded 34GiB to GPU in 2.44227s (13.9215GiB/s) | 20260508 09:16:04.924 | INFO | Analyzing chunk frames = [6957056,6957568) | 20260508 09:16:04.924 | INFO | MemInfo: { | 20260508 09:16:04.924 | INFO | "gpu" : " 17 GB", | 20260508 09:16:04.924 | INFO | "huge" : "", | 20260508 09:16:04.924 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:04.924 | INFO | "malloc" : " 14 GB", | 20260508 09:16:04.924 | INFO | "pinned" : " 29 GB" | 20260508 09:16:04.924 | INFO | } | 20260508 09:16:04.924 | INFO | Perf report for the last 5.13991 seconds: | 20260508 09:16:04.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:04.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.924 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 09:16:04.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.924 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:16:04.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:04.925 | INFO | Uploaded 36GiB to GPU in 2.36777s (15.2042GiB/s) | 20260508 09:16:04.925 | INFO | Downloaded 34GiB to GPU in 2.51818s (13.5018GiB/s) | 20260508 09:16:10.065 | INFO | Analyzing chunk frames = [6957568,6958080) | 20260508 09:16:10.065 | INFO | MemInfo: { | 20260508 09:16:10.065 | INFO | "gpu" : " 17 GB", | 20260508 09:16:10.065 | INFO | "huge" : "", | 20260508 09:16:10.065 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:10.066 | INFO | "malloc" : " 14 GB", | 20260508 09:16:10.066 | INFO | "pinned" : " 29 GB" | 20260508 09:16:10.066 | INFO | } | 20260508 09:16:10.066 | INFO | Perf report for the last 5.14152 seconds: | 20260508 09:16:10.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:10.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:16:10.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:16:10.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:10.066 | INFO | Uploaded 36GiB to GPU in 2.38361s (15.1032GiB/s) | 20260508 09:16:10.066 | INFO | Downloaded 34GiB to GPU in 2.43979s (13.9356GiB/s) | 20260508 09:16:15.201 | INFO | Analyzing chunk frames = [6958080,6958592) | 20260508 09:16:15.201 | INFO | MemInfo: { | 20260508 09:16:15.201 | INFO | "gpu" : " 17 GB", | 20260508 09:16:15.201 | INFO | "huge" : "", | 20260508 09:16:15.201 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:15.201 | INFO | "malloc" : " 14 GB", | 20260508 09:16:15.201 | INFO | "pinned" : " 29 GB" | 20260508 09:16:15.201 | INFO | } | 20260508 09:16:15.201 | INFO | Perf report for the last 5.13524 seconds: | 20260508 09:16:15.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:15.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:16:15.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:16:15.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:15.201 | INFO | Uploaded 36GiB to GPU in 2.39047s (15.0598GiB/s) | 20260508 09:16:15.201 | INFO | Downloaded 34GiB to GPU in 2.44288s (13.918GiB/s) | 20260508 09:16:20.341 | INFO | Analyzing chunk frames = [6958592,6959104) | 20260508 09:16:20.341 | INFO | MemInfo: { | 20260508 09:16:20.341 | INFO | "gpu" : " 17 GB", | 20260508 09:16:20.341 | INFO | "huge" : "", | 20260508 09:16:20.341 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:20.341 | INFO | "malloc" : " 14 GB", | 20260508 09:16:20.341 | INFO | "pinned" : " 29 GB" | 20260508 09:16:20.341 | INFO | } | 20260508 09:16:20.341 | INFO | Perf report for the last 5.14042 seconds: | 20260508 09:16:20.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:20.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.341 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:16:20.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.342 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:16:20.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:20.342 | INFO | Uploaded 36GiB to GPU in 2.43013s (14.814GiB/s) | 20260508 09:16:20.342 | INFO | Downloaded 34GiB to GPU in 2.45078s (13.8731GiB/s) | 20260508 09:16:20.439 | INFO | Starting a round of Baz IO | 20260508 09:16:23.850 | INFO | Populating RTMetrics report | 20260508 09:16:23.850 | INFO | RTMetrics report populated | 20260508 09:16:23.850 | INFO | Opening tmp RTMetrics json file | 20260508 09:16:23.859 | INFO | Serializing RTMetrics to json | 20260508 09:16:23.860 | INFO | Serialization done, swapping tmp file now... | 20260508 09:16:23.861 | INFO | RTMetrics done | 20260508 09:16:23.921 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.481 seconds | 20260508 09:16:23.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.67% | 20260508 09:16:23.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.260842% / 7.94729e-06% / 5.5631e-05% | 20260508 09:16:23.921 | INFO | Baz IO overall: | 20260508 09:16:23.921 | INFO | BytesWritten : 3039939257085 | 20260508 09:16:23.921 | INFO | NumEvents : 1462578883634 | 20260508 09:16:25.482 | INFO | Analyzing chunk frames = [6959104,6959616) | 20260508 09:16:25.482 | INFO | MemInfo: { | 20260508 09:16:25.482 | INFO | "gpu" : " 17 GB", | 20260508 09:16:25.482 | INFO | "huge" : "", | 20260508 09:16:25.482 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:25.482 | INFO | "malloc" : " 14 GB", | 20260508 09:16:25.482 | INFO | "pinned" : " 29 GB" | 20260508 09:16:25.482 | INFO | } | 20260508 09:16:25.482 | INFO | Perf report for the last 5.14091 seconds: | 20260508 09:16:25.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:25.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:25.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:25.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:25.482 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:16:25.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:25.483 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:16:25.483 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 09:16:25.483 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 09:16:25.483 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:16:25.483 | INFO | Uploaded 36GiB to GPU in 2.60777s (13.8049GiB/s) | 20260508 09:16:25.483 | INFO | Downloaded 38GiB to GPU in 2.70886s (14.0281GiB/s) | 20260508 09:16:30.619 | INFO | Analyzing chunk frames = [6959616,6960128) | 20260508 09:16:30.619 | INFO | MemInfo: { | 20260508 09:16:30.619 | INFO | "gpu" : " 17 GB", | 20260508 09:16:30.619 | INFO | "huge" : "", | 20260508 09:16:30.619 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:30.619 | INFO | "malloc" : " 14 GB", | 20260508 09:16:30.619 | INFO | "pinned" : " 29 GB" | 20260508 09:16:30.619 | INFO | } | 20260508 09:16:30.619 | INFO | Perf report for the last 5.1371 seconds: | 20260508 09:16:30.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:30.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.619 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:16:30.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.620 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 09:16:30.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:30.620 | INFO | Uploaded 36GiB to GPU in 2.51444s (14.3173GiB/s) | 20260508 09:16:30.620 | INFO | Downloaded 34GiB to GPU in 2.45789s (13.833GiB/s) | 20260508 09:16:35.759 | INFO | Analyzing chunk frames = [6960128,6960640) | 20260508 09:16:35.759 | INFO | MemInfo: { | 20260508 09:16:35.759 | INFO | "gpu" : " 17 GB", | 20260508 09:16:35.759 | INFO | "huge" : "", | 20260508 09:16:35.759 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:35.759 | INFO | "malloc" : " 14 GB", | 20260508 09:16:35.759 | INFO | "pinned" : " 29 GB" | 20260508 09:16:35.759 | INFO | } | 20260508 09:16:35.759 | INFO | Perf report for the last 5.13937 seconds: | 20260508 09:16:35.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:35.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:16:35.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:16:35.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:35.759 | INFO | Uploaded 36GiB to GPU in 2.46938s (14.5785GiB/s) | 20260508 09:16:35.759 | INFO | Downloaded 34GiB to GPU in 2.43752s (13.9486GiB/s) | 20260508 09:16:40.896 | INFO | Analyzing chunk frames = [6960640,6961152) | 20260508 09:16:40.896 | INFO | MemInfo: { | 20260508 09:16:40.896 | INFO | "gpu" : " 17 GB", | 20260508 09:16:40.896 | INFO | "huge" : "", | 20260508 09:16:40.896 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:40.896 | INFO | "malloc" : " 14 GB", | 20260508 09:16:40.896 | INFO | "pinned" : " 29 GB" | 20260508 09:16:40.896 | INFO | } | 20260508 09:16:40.896 | INFO | Perf report for the last 5.13754 seconds: | 20260508 09:16:40.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:40.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.896 | INFO | ANALYSIS : 66.6%, 2.9, 103.0ms, 0 | 20260508 09:16:40.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.896 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:16:40.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:40.897 | INFO | Uploaded 36GiB to GPU in 2.52173s (14.2759GiB/s) | 20260508 09:16:40.897 | INFO | Downloaded 34GiB to GPU in 2.4251s (14.0201GiB/s) | 20260508 09:16:46.035 | INFO | Analyzing chunk frames = [6961152,6961664) | 20260508 09:16:46.035 | INFO | MemInfo: { | 20260508 09:16:46.035 | INFO | "gpu" : " 17 GB", | 20260508 09:16:46.035 | INFO | "huge" : "", | 20260508 09:16:46.035 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:46.035 | INFO | "malloc" : " 14 GB", | 20260508 09:16:46.035 | INFO | "pinned" : " 29 GB" | 20260508 09:16:46.035 | INFO | } | 20260508 09:16:46.035 | INFO | Perf report for the last 5.13851 seconds: | 20260508 09:16:46.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:46.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 09:16:46.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:16:46.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:46.035 | INFO | Uploaded 36GiB to GPU in 2.36138s (15.2453GiB/s) | 20260508 09:16:46.035 | INFO | Downloaded 34GiB to GPU in 2.51053s (13.5429GiB/s) | 20260508 09:16:51.178 | INFO | Analyzing chunk frames = [6961664,6962176) | 20260508 09:16:51.178 | INFO | MemInfo: { | 20260508 09:16:51.178 | INFO | "gpu" : " 17 GB", | 20260508 09:16:51.178 | INFO | "huge" : "", | 20260508 09:16:51.178 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:51.178 | INFO | "malloc" : " 14 GB", | 20260508 09:16:51.178 | INFO | "pinned" : " 29 GB" | 20260508 09:16:51.178 | INFO | } | 20260508 09:16:51.178 | INFO | Perf report for the last 5.14342 seconds: | 20260508 09:16:51.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:51.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.178 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 09:16:51.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.178 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:16:51.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:51.179 | INFO | Uploaded 36GiB to GPU in 2.52905s (14.2346GiB/s) | 20260508 09:16:51.179 | INFO | Downloaded 34GiB to GPU in 2.47861s (13.7173GiB/s) | 20260508 09:16:56.320 | INFO | Analyzing chunk frames = [6962176,6962688) | 20260508 09:16:56.320 | INFO | MemInfo: { | 20260508 09:16:56.320 | INFO | "gpu" : " 17 GB", | 20260508 09:16:56.320 | INFO | "huge" : "", | 20260508 09:16:56.320 | INFO | "huge_shared" : " 29 GB", | 20260508 09:16:56.320 | INFO | "malloc" : " 14 GB", | 20260508 09:16:56.320 | INFO | "pinned" : " 29 GB" | 20260508 09:16:56.320 | INFO | } | 20260508 09:16:56.320 | INFO | Perf report for the last 5.14166 seconds: | 20260508 09:16:56.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:16:56.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.320 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:16:56.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.320 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:16:56.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:16:56.321 | INFO | Uploaded 36GiB to GPU in 2.41568s (14.9026GiB/s) | 20260508 09:16:56.321 | INFO | Downloaded 34GiB to GPU in 2.44117s (13.9277GiB/s) | 20260508 09:17:01.461 | INFO | Analyzing chunk frames = [6962688,6963200) | 20260508 09:17:01.461 | INFO | MemInfo: { | 20260508 09:17:01.461 | INFO | "gpu" : " 17 GB", | 20260508 09:17:01.461 | INFO | "huge" : "", | 20260508 09:17:01.461 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:01.461 | INFO | "malloc" : " 14 GB", | 20260508 09:17:01.461 | INFO | "pinned" : " 29 GB" | 20260508 09:17:01.461 | INFO | } | 20260508 09:17:01.461 | INFO | Perf report for the last 5.14105 seconds: | 20260508 09:17:01.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:01.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:17:01.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 09:17:01.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:01.461 | INFO | Uploaded 36GiB to GPU in 2.42227s (14.8621GiB/s) | 20260508 09:17:01.461 | INFO | Downloaded 34GiB to GPU in 2.43944s (13.9376GiB/s) | 20260508 09:17:01.462 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:17:01.462 | INFO | Total mem usage: 16.7 GiB | 20260508 09:17:01.462 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:17:01.462 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.1%) | 20260508 09:17:01.462 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:17:01.462 | INFO | High Water Breakdown: | 20260508 09:17:01.462 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:17:01.462 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:17:01.462 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:17:01.462 | INFO | ------------------------------------------ | 20260508 09:17:01.462 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:17:01.462 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:17:01.462 | INFO | Total mem usage: 26.5 GiB | 20260508 09:17:01.462 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:17:01.462 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:17:01.462 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:17:01.462 | INFO | High Water Breakdown: | 20260508 09:17:01.462 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:17:01.462 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:17:01.462 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:17:01.462 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:17:01.462 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:17:01.462 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:17:01.462 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:17:01.462 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:17:01.462 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:17:01.462 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:17:01.462 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:17:01.462 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:17:01.462 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:17:01.462 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:17:01.462 | INFO | ------------------------------------------ | 20260508 09:17:01.462 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:17:01.462 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:17:01.462 | INFO | Total mem usage: 27.8 GiB | 20260508 09:17:01.462 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:17:01.462 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:17:01.462 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:17:01.462 | INFO | High Water Breakdown: | 20260508 09:17:01.462 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:17:01.462 | INFO | ------------------------------------------ | 20260508 09:17:01.462 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:17:01.462 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:17:01.462 | INFO | Total mem usage: 15.7 GiB | 20260508 09:17:01.462 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:17:01.462 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:17:01.462 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:17:01.462 | INFO | High Water Breakdown: | 20260508 09:17:01.462 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:17:01.463 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:17:01.463 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:17:01.463 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:17:01.463 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:17:01.463 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:17:01.463 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:17:01.463 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:17:01.463 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:17:01.463 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:17:01.463 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:17:01.463 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:17:01.463 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:17:01.463 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:17:01.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:17:01.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:17:01.463 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:17:01.463 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:17:01.463 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:17:01.463 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:17:01.463 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:17:01.463 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:17:01.463 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:17:01.463 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:17:01.463 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:17:01.463 | INFO | ------------------------------------------ | 20260508 09:17:01.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:17:01.463 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:17:01.463 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:17:01.463 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:17:01.463 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.96e-01 | 20260508 09:17:01.463 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.07e-01, 1.15e-02 / 9.52e-03 | 20260508 09:17:01.463 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.38e+00, 5.89e-01 / 2.54e-01 | 20260508 09:17:01.463 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.27e-01 | 20260508 09:17:01.463 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.49e+01, 3.65e+00 / 1.06e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.86e+00 / 1.33e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.44e+00, 4.11e-01 / 2.81e-01 | 20260508 09:17:01.463 | INFO | PerfReport: 3.04%, Metrics -- 9.95e-01 / 1.18e+00, 2.51e-01 / 1.40e-01 | 20260508 09:17:01.463 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.33e-01, 4.82e-02 / 2.98e-02 | 20260508 09:17:01.463 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:17:01.463 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:17:01.463 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:17:01.463 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 24.42%, Upload -- 2.58e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 24.56%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 31.00%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 2.14%, LockUpload -- 2.26e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 2.55%, LockDownload -- 2.69e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 15.31%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:17:01.463 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 09:17:01.575 | INFO | Starting a round of Baz IO | 20260508 09:17:04.971 | INFO | Populating RTMetrics report | 20260508 09:17:04.971 | INFO | RTMetrics report populated | 20260508 09:17:04.971 | INFO | Opening tmp RTMetrics json file | 20260508 09:17:04.981 | INFO | Serializing RTMetrics to json | 20260508 09:17:04.983 | INFO | Serialization done, swapping tmp file now... | 20260508 09:17:04.983 | INFO | RTMetrics done | 20260508 09:17:05.043 | INFO | Finished a round of Baz IO: Wrote 1.484 GiB in 3.468 seconds | 20260508 09:17:05.043 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:17:05.043 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259801% / 3.97364e-06% / 6.35783e-05% | 20260508 09:17:05.043 | INFO | Baz IO overall: | 20260508 09:17:05.043 | INFO | BytesWritten : 3041532216442 | 20260508 09:17:05.043 | INFO | NumEvents : 1463332053714 | 20260508 09:17:06.587 | INFO | Analyzing chunk frames = [6963200,6963712) | 20260508 09:17:06.587 | INFO | MemInfo: { | 20260508 09:17:06.587 | INFO | "gpu" : " 17 GB", | 20260508 09:17:06.587 | INFO | "huge" : "", | 20260508 09:17:06.587 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:06.587 | INFO | "malloc" : " 14 GB", | 20260508 09:17:06.587 | INFO | "pinned" : " 29 GB" | 20260508 09:17:06.587 | INFO | } | 20260508 09:17:06.587 | INFO | Perf report for the last 5.12605 seconds: | 20260508 09:17:06.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:06.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:06.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:06.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:06.587 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260508 09:17:06.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:06.587 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260508 09:17:06.587 | INFO | PRE_HQ : 66.5%, 1.8, 62.3ms, 0 | 20260508 09:17:06.587 | INFO | BAZWRITER : 67.0%, 2.7, 94.4ms, 0 | 20260508 09:17:06.587 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:17:06.588 | INFO | Uploaded 36GiB to GPU in 2.64245s (13.6237GiB/s) | 20260508 09:17:06.588 | INFO | Downloaded 38GiB to GPU in 2.7122s (14.0108GiB/s) | 20260508 09:17:11.736 | INFO | Analyzing chunk frames = [6963712,6964224) | 20260508 09:17:11.736 | INFO | MemInfo: { | 20260508 09:17:11.736 | INFO | "gpu" : " 17 GB", | 20260508 09:17:11.736 | INFO | "huge" : "", | 20260508 09:17:11.736 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:11.736 | INFO | "malloc" : " 14 GB", | 20260508 09:17:11.736 | INFO | "pinned" : " 29 GB" | 20260508 09:17:11.736 | INFO | } | 20260508 09:17:11.736 | INFO | Perf report for the last 5.14879 seconds: | 20260508 09:17:11.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:11.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | ANALYSIS : 66.8%, 3.0, 104.3ms, 0 | 20260508 09:17:11.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:17:11.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:11.736 | INFO | Uploaded 36GiB to GPU in 2.54238s (14.1599GiB/s) | 20260508 09:17:11.737 | INFO | Downloaded 34GiB to GPU in 2.44573s (13.9018GiB/s) | 20260508 09:17:16.879 | INFO | Analyzing chunk frames = [6964224,6964736) | 20260508 09:17:16.879 | INFO | MemInfo: { | 20260508 09:17:16.879 | INFO | "gpu" : " 17 GB", | 20260508 09:17:16.879 | INFO | "huge" : "", | 20260508 09:17:16.879 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:16.879 | INFO | "malloc" : " 14 GB", | 20260508 09:17:16.879 | INFO | "pinned" : " 29 GB" | 20260508 09:17:16.879 | INFO | } | 20260508 09:17:16.879 | INFO | Perf report for the last 5.14333 seconds: | 20260508 09:17:16.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:16.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.879 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:17:16.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.879 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:17:16.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:16.880 | INFO | Uploaded 36GiB to GPU in 2.50385s (14.3779GiB/s) | 20260508 09:17:16.880 | INFO | Downloaded 34GiB to GPU in 2.43602s (13.9572GiB/s) | 20260508 09:17:22.017 | INFO | Analyzing chunk frames = [6964736,6965248) | 20260508 09:17:22.017 | INFO | MemInfo: { | 20260508 09:17:22.017 | INFO | "gpu" : " 17 GB", | 20260508 09:17:22.017 | INFO | "huge" : "", | 20260508 09:17:22.017 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:22.017 | INFO | "malloc" : " 14 GB", | 20260508 09:17:22.018 | INFO | "pinned" : " 29 GB" | 20260508 09:17:22.018 | INFO | } | 20260508 09:17:22.018 | INFO | Perf report for the last 5.13838 seconds: | 20260508 09:17:22.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:22.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 09:17:22.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:17:22.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:22.018 | INFO | Uploaded 36GiB to GPU in 2.51488s (14.3148GiB/s) | 20260508 09:17:22.018 | INFO | Downloaded 34GiB to GPU in 2.41168s (14.098GiB/s) | 20260508 09:17:27.155 | INFO | Analyzing chunk frames = [6965248,6965760) | 20260508 09:17:27.155 | INFO | MemInfo: { | 20260508 09:17:27.155 | INFO | "gpu" : " 17 GB", | 20260508 09:17:27.155 | INFO | "huge" : "", | 20260508 09:17:27.155 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:27.155 | INFO | "malloc" : " 14 GB", | 20260508 09:17:27.155 | INFO | "pinned" : " 29 GB" | 20260508 09:17:27.155 | INFO | } | 20260508 09:17:27.155 | INFO | Perf report for the last 5.13712 seconds: | 20260508 09:17:27.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:27.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 09:17:27.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 09:17:27.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:27.155 | INFO | Uploaded 36GiB to GPU in 2.40187s (14.9883GiB/s) | 20260508 09:17:27.155 | INFO | Downloaded 34GiB to GPU in 2.50458s (13.5751GiB/s) | 20260508 09:17:32.290 | INFO | Analyzing chunk frames = [6965760,6966272) | 20260508 09:17:32.290 | INFO | MemInfo: { | 20260508 09:17:32.290 | INFO | "gpu" : " 17 GB", | 20260508 09:17:32.290 | INFO | "huge" : "", | 20260508 09:17:32.290 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:32.290 | INFO | "malloc" : " 14 GB", | 20260508 09:17:32.290 | INFO | "pinned" : " 29 GB" | 20260508 09:17:32.290 | INFO | } | 20260508 09:17:32.290 | INFO | Perf report for the last 5.1355 seconds: | 20260508 09:17:32.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:32.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.290 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:17:32.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.290 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:17:32.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:32.291 | INFO | Uploaded 36GiB to GPU in 2.39174s (15.0518GiB/s) | 20260508 09:17:32.291 | INFO | Downloaded 34GiB to GPU in 2.41815s (14.0603GiB/s) | 20260508 09:17:37.430 | INFO | Analyzing chunk frames = [6966272,6966784) | 20260508 09:17:37.430 | INFO | MemInfo: { | 20260508 09:17:37.430 | INFO | "gpu" : " 17 GB", | 20260508 09:17:37.430 | INFO | "huge" : "", | 20260508 09:17:37.430 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:37.430 | INFO | "malloc" : " 14 GB", | 20260508 09:17:37.430 | INFO | "pinned" : " 29 GB" | 20260508 09:17:37.430 | INFO | } | 20260508 09:17:37.430 | INFO | Perf report for the last 5.13968 seconds: | 20260508 09:17:37.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:37.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:17:37.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 09:17:37.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:37.430 | INFO | Uploaded 36GiB to GPU in 2.44049s (14.7512GiB/s) | 20260508 09:17:37.430 | INFO | Downloaded 34GiB to GPU in 2.41856s (14.058GiB/s) | 20260508 09:17:42.572 | INFO | Analyzing chunk frames = [6966784,6967296) | 20260508 09:17:42.572 | INFO | MemInfo: { | 20260508 09:17:42.572 | INFO | "gpu" : " 17 GB", | 20260508 09:17:42.572 | INFO | "huge" : "", | 20260508 09:17:42.572 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:42.572 | INFO | "malloc" : " 14 GB", | 20260508 09:17:42.572 | INFO | "pinned" : " 29 GB" | 20260508 09:17:42.572 | INFO | } | 20260508 09:17:42.572 | INFO | Perf report for the last 5.14224 seconds: | 20260508 09:17:42.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:42.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 09:17:42.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:17:42.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:42.572 | INFO | Uploaded 36GiB to GPU in 2.44839s (14.7035GiB/s) | 20260508 09:17:42.572 | INFO | Downloaded 34GiB to GPU in 2.43162s (13.9825GiB/s) | 20260508 09:17:42.673 | INFO | Starting a round of Baz IO | 20260508 09:17:46.080 | INFO | Populating RTMetrics report | 20260508 09:17:46.080 | INFO | RTMetrics report populated | 20260508 09:17:46.080 | INFO | Opening tmp RTMetrics json file | 20260508 09:17:46.090 | INFO | Serializing RTMetrics to json | 20260508 09:17:46.093 | INFO | Serialization done, swapping tmp file now... | 20260508 09:17:46.093 | INFO | RTMetrics done | 20260508 09:17:46.151 | INFO | Finished a round of Baz IO: Wrote 1.485 GiB in 3.478 seconds | 20260508 09:17:46.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:17:46.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.260254% / 0% / 5.5631e-05% | 20260508 09:17:46.151 | INFO | Baz IO overall: | 20260508 09:17:46.151 | INFO | BytesWritten : 3043126685635 | 20260508 09:17:46.151 | INFO | NumEvents : 1464086483564 | 20260508 09:17:47.714 | INFO | Analyzing chunk frames = [6967296,6967808) | 20260508 09:17:47.714 | INFO | MemInfo: { | 20260508 09:17:47.714 | INFO | "gpu" : " 17 GB", | 20260508 09:17:47.714 | INFO | "huge" : "", | 20260508 09:17:47.714 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:47.714 | INFO | "malloc" : " 14 GB", | 20260508 09:17:47.714 | INFO | "pinned" : " 29 GB" | 20260508 09:17:47.714 | INFO | } | 20260508 09:17:47.714 | INFO | Perf report for the last 5.14172 seconds: | 20260508 09:17:47.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:47.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:47.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:47.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:47.714 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:17:47.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:47.714 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:17:47.714 | INFO | PRE_HQ : 65.5%, 1.6, 54.9ms, 0 | 20260508 09:17:47.714 | INFO | BAZWRITER : 67.0%, 2.5, 87.8ms, 0 | 20260508 09:17:47.714 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:17:47.714 | INFO | Uploaded 36GiB to GPU in 2.6019s (13.836GiB/s) | 20260508 09:17:47.714 | INFO | Downloaded 38GiB to GPU in 2.69813s (14.0839GiB/s) | 20260508 09:17:52.844 | INFO | Analyzing chunk frames = [6967808,6968320) | 20260508 09:17:52.844 | INFO | MemInfo: { | 20260508 09:17:52.844 | INFO | "gpu" : " 17 GB", | 20260508 09:17:52.844 | INFO | "huge" : "", | 20260508 09:17:52.844 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:52.844 | INFO | "malloc" : " 14 GB", | 20260508 09:17:52.844 | INFO | "pinned" : " 29 GB" | 20260508 09:17:52.844 | INFO | } | 20260508 09:17:52.844 | INFO | Perf report for the last 5.13042 seconds: | 20260508 09:17:52.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:52.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.844 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:17:52.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.845 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:17:52.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:52.845 | INFO | Uploaded 36GiB to GPU in 2.48449s (14.4899GiB/s) | 20260508 09:17:52.845 | INFO | Downloaded 34GiB to GPU in 2.45724s (13.8367GiB/s) | 20260508 09:17:57.993 | INFO | Analyzing chunk frames = [6968320,6968832) | 20260508 09:17:57.993 | INFO | MemInfo: { | 20260508 09:17:57.993 | INFO | "gpu" : " 17 GB", | 20260508 09:17:57.993 | INFO | "huge" : "", | 20260508 09:17:57.993 | INFO | "huge_shared" : " 29 GB", | 20260508 09:17:57.993 | INFO | "malloc" : " 14 GB", | 20260508 09:17:57.993 | INFO | "pinned" : " 29 GB" | 20260508 09:17:57.993 | INFO | } | 20260508 09:17:57.993 | INFO | Perf report for the last 5.14846 seconds: | 20260508 09:17:57.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:17:57.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 09:17:57.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:17:57.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:17:57.993 | INFO | Uploaded 36GiB to GPU in 2.5224s (14.2721GiB/s) | 20260508 09:17:57.993 | INFO | Downloaded 34GiB to GPU in 2.44278s (13.9186GiB/s) | 20260508 09:18:03.130 | INFO | Analyzing chunk frames = [6968832,6969344) | 20260508 09:18:03.130 | INFO | MemInfo: { | 20260508 09:18:03.130 | INFO | "gpu" : " 17 GB", | 20260508 09:18:03.130 | INFO | "huge" : "", | 20260508 09:18:03.130 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:03.130 | INFO | "malloc" : " 14 GB", | 20260508 09:18:03.130 | INFO | "pinned" : " 29 GB" | 20260508 09:18:03.130 | INFO | } | 20260508 09:18:03.130 | INFO | Perf report for the last 5.1375 seconds: | 20260508 09:18:03.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:03.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.130 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 09:18:03.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.130 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:18:03.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:03.131 | INFO | Uploaded 36GiB to GPU in 2.51355s (14.3224GiB/s) | 20260508 09:18:03.131 | INFO | Downloaded 34GiB to GPU in 2.41444s (14.0819GiB/s) | 20260508 09:18:08.269 | INFO | Analyzing chunk frames = [6969344,6969856) | 20260508 09:18:08.269 | INFO | MemInfo: { | 20260508 09:18:08.269 | INFO | "gpu" : " 17 GB", | 20260508 09:18:08.269 | INFO | "huge" : "", | 20260508 09:18:08.269 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:08.269 | INFO | "malloc" : " 14 GB", | 20260508 09:18:08.270 | INFO | "pinned" : " 29 GB" | 20260508 09:18:08.270 | INFO | } | 20260508 09:18:08.270 | INFO | Perf report for the last 5.1393 seconds: | 20260508 09:18:08.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:08.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:18:08.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 09:18:08.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:08.270 | INFO | Uploaded 36GiB to GPU in 2.44065s (14.7501GiB/s) | 20260508 09:18:08.270 | INFO | Downloaded 34GiB to GPU in 2.51667s (13.5099GiB/s) | 20260508 09:18:13.408 | INFO | Analyzing chunk frames = [6969856,6970368) | 20260508 09:18:13.408 | INFO | MemInfo: { | 20260508 09:18:13.408 | INFO | "gpu" : " 17 GB", | 20260508 09:18:13.408 | INFO | "huge" : "", | 20260508 09:18:13.408 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:13.408 | INFO | "malloc" : " 14 GB", | 20260508 09:18:13.408 | INFO | "pinned" : " 29 GB" | 20260508 09:18:13.408 | INFO | } | 20260508 09:18:13.408 | INFO | Perf report for the last 5.13811 seconds: | 20260508 09:18:13.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:13.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 09:18:13.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 09:18:13.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:13.408 | INFO | Uploaded 36GiB to GPU in 2.39036s (15.0605GiB/s) | 20260508 09:18:13.408 | INFO | Downloaded 34GiB to GPU in 2.41938s (14.0532GiB/s) | 20260508 09:18:18.542 | INFO | Analyzing chunk frames = [6970368,6970880) | 20260508 09:18:18.542 | INFO | MemInfo: { | 20260508 09:18:18.542 | INFO | "gpu" : " 17 GB", | 20260508 09:18:18.542 | INFO | "huge" : "", | 20260508 09:18:18.542 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:18.542 | INFO | "malloc" : " 14 GB", | 20260508 09:18:18.542 | INFO | "pinned" : " 29 GB" | 20260508 09:18:18.542 | INFO | } | 20260508 09:18:18.542 | INFO | Perf report for the last 5.13396 seconds: | 20260508 09:18:18.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:18.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:18:18.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:18:18.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:18.542 | INFO | Uploaded 36GiB to GPU in 2.44407s (14.7295GiB/s) | 20260508 09:18:18.542 | INFO | Downloaded 34GiB to GPU in 2.41035s (14.1058GiB/s) | 20260508 09:18:23.685 | INFO | Analyzing chunk frames = [6970880,6971392) | 20260508 09:18:23.685 | INFO | MemInfo: { | 20260508 09:18:23.685 | INFO | "gpu" : " 17 GB", | 20260508 09:18:23.685 | INFO | "huge" : "", | 20260508 09:18:23.685 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:23.685 | INFO | "malloc" : " 14 GB", | 20260508 09:18:23.685 | INFO | "pinned" : " 29 GB" | 20260508 09:18:23.685 | INFO | } | 20260508 09:18:23.685 | INFO | Perf report for the last 5.14332 seconds: | 20260508 09:18:23.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:23.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.685 | INFO | ANALYSIS : 65.8%, 3.0, 103.0ms, 0 | 20260508 09:18:23.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.685 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:18:23.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:23.686 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:18:23.686 | INFO | Total mem usage: 16.7 GiB | 20260508 09:18:23.686 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:18:23.686 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:18:23.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:18:23.686 | INFO | High Water Breakdown: | 20260508 09:18:23.686 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:18:23.686 | INFO | Uploaded 36GiB to GPU in 2.44769s (14.7077GiB/s) | 20260508 09:18:23.686 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:18:23.686 | INFO | Downloaded 34GiB to GPU in 2.44125s (13.9273GiB/s) | 20260508 09:18:23.686 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:18:23.686 | INFO | ------------------------------------------ | 20260508 09:18:23.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:18:23.686 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:18:23.686 | INFO | Total mem usage: 26.5 GiB | 20260508 09:18:23.686 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:18:23.686 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:18:23.686 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:18:23.686 | INFO | High Water Breakdown: | 20260508 09:18:23.686 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:18:23.686 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:18:23.686 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:18:23.686 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:18:23.686 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:18:23.686 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:18:23.686 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:18:23.686 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:18:23.686 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:18:23.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:18:23.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:18:23.686 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:18:23.686 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:18:23.686 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:18:23.686 | INFO | ------------------------------------------ | 20260508 09:18:23.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:18:23.686 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:18:23.686 | INFO | Total mem usage: 27.8 GiB | 20260508 09:18:23.686 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:18:23.686 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:18:23.686 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:18:23.686 | INFO | High Water Breakdown: | 20260508 09:18:23.686 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:18:23.686 | INFO | ------------------------------------------ | 20260508 09:18:23.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:18:23.686 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:18:23.686 | INFO | Total mem usage: 15.7 GiB | 20260508 09:18:23.686 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:18:23.686 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:18:23.686 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:18:23.686 | INFO | High Water Breakdown: | 20260508 09:18:23.686 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:18:23.686 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:18:23.686 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:18:23.686 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:18:23.686 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:18:23.686 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:18:23.686 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:18:23.686 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:18:23.686 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:18:23.686 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:18:23.686 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:18:23.686 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:18:23.686 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:18:23.686 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:18:23.687 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:18:23.687 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:18:23.687 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:18:23.687 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:18:23.687 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:18:23.687 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:18:23.687 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:18:23.687 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:18:23.687 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:18:23.687 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:18:23.687 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:18:23.687 | INFO | ------------------------------------------ | 20260508 09:18:23.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:18:23.687 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:18:23.687 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:18:23.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:18:23.687 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.40e+01 / 6.84e-01 | 20260508 09:18:23.687 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.06e-01, 1.18e-02 / 7.26e-03 | 20260508 09:18:23.687 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.43e+00, 5.96e-01 / 2.47e-01 | 20260508 09:18:23.687 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.45e+00, 2.10e-01 / 1.41e-01 | 20260508 09:18:23.687 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.48e+01, 3.44e+00 / 1.07e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.88e+00 / 1.31e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.33e+00 / 2.47e+00, 4.06e-01 / 2.50e-01 | 20260508 09:18:23.687 | INFO | PerfReport: 3.03%, Metrics -- 9.93e-01 / 1.15e+00, 2.50e-01 / 1.31e-01 | 20260508 09:18:23.687 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.34e-01, 4.81e-02 / 2.68e-02 | 20260508 09:18:23.687 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:18:23.687 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:18:23.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:18:23.687 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 24.53%, Upload -- 2.59e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 24.43%, Download -- 2.58e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 31.07%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 2.36%, LockUpload -- 2.49e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 2.34%, LockDownload -- 2.47e+00 / 0.00e+00, 3.85e+00 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 15.25%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:18:23.687 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.10e-02 / 0.00e+00 | 20260508 09:18:23.783 | INFO | Starting a round of Baz IO | 20260508 09:18:27.186 | INFO | Populating RTMetrics report | 20260508 09:18:27.186 | INFO | RTMetrics report populated | 20260508 09:18:27.186 | INFO | Opening tmp RTMetrics json file | 20260508 09:18:27.196 | INFO | Serializing RTMetrics to json | 20260508 09:18:27.197 | INFO | Serialization done, swapping tmp file now... | 20260508 09:18:27.197 | INFO | RTMetrics done | 20260508 09:18:27.258 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.474 seconds | 20260508 09:18:27.258 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:18:27.258 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258625% / 0% / 4.76837e-05% | 20260508 09:18:27.258 | INFO | Baz IO overall: | 20260508 09:18:27.258 | INFO | BytesWritten : 3044719516307 | 20260508 09:18:27.258 | INFO | NumEvents : 1464839920615 | 20260508 09:18:28.830 | INFO | Analyzing chunk frames = [6971392,6971904) | 20260508 09:18:28.830 | INFO | MemInfo: { | 20260508 09:18:28.830 | INFO | "gpu" : " 17 GB", | 20260508 09:18:28.830 | INFO | "huge" : "", | 20260508 09:18:28.830 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:28.830 | INFO | "malloc" : " 14 GB", | 20260508 09:18:28.830 | INFO | "pinned" : " 29 GB" | 20260508 09:18:28.830 | INFO | } | 20260508 09:18:28.830 | INFO | Perf report for the last 5.14488 seconds: | 20260508 09:18:28.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:28.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:28.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:28.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:28.830 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 09:18:28.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:28.830 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:18:28.830 | INFO | PRE_HQ : 65.1%, 1.6, 54.5ms, 0 | 20260508 09:18:28.830 | INFO | BAZWRITER : 66.7%, 2.5, 87.4ms, 0 | 20260508 09:18:28.830 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:18:28.830 | INFO | Uploaded 36GiB to GPU in 2.60992s (13.7935GiB/s) | 20260508 09:18:28.830 | INFO | Downloaded 38GiB to GPU in 2.69664s (14.0916GiB/s) | 20260508 09:18:33.967 | INFO | Analyzing chunk frames = [6971904,6972416) | 20260508 09:18:33.967 | INFO | MemInfo: { | 20260508 09:18:33.967 | INFO | "gpu" : " 17 GB", | 20260508 09:18:33.967 | INFO | "huge" : "", | 20260508 09:18:33.967 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:33.967 | INFO | "malloc" : " 14 GB", | 20260508 09:18:33.967 | INFO | "pinned" : " 29 GB" | 20260508 09:18:33.967 | INFO | } | 20260508 09:18:33.967 | INFO | Perf report for the last 5.1371 seconds: | 20260508 09:18:33.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:33.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 09:18:33.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:18:33.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:33.967 | INFO | Uploaded 36GiB to GPU in 2.50885s (14.3492GiB/s) | 20260508 09:18:33.967 | INFO | Downloaded 34GiB to GPU in 2.46729s (13.7803GiB/s) | 20260508 09:18:39.098 | INFO | Analyzing chunk frames = [6972416,6972928) | 20260508 09:18:39.098 | INFO | MemInfo: { | 20260508 09:18:39.098 | INFO | "gpu" : " 17 GB", | 20260508 09:18:39.098 | INFO | "huge" : "", | 20260508 09:18:39.098 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:39.098 | INFO | "malloc" : " 14 GB", | 20260508 09:18:39.098 | INFO | "pinned" : " 29 GB" | 20260508 09:18:39.098 | INFO | } | 20260508 09:18:39.098 | INFO | Perf report for the last 5.13091 seconds: | 20260508 09:18:39.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:39.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260508 09:18:39.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:18:39.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:39.098 | INFO | Uploaded 36GiB to GPU in 2.52053s (14.2827GiB/s) | 20260508 09:18:39.098 | INFO | Downloaded 34GiB to GPU in 2.44311s (13.9167GiB/s) | 20260508 09:18:44.242 | INFO | Analyzing chunk frames = [6972928,6973440) | 20260508 09:18:44.242 | INFO | MemInfo: { | 20260508 09:18:44.242 | INFO | "gpu" : " 17 GB", | 20260508 09:18:44.242 | INFO | "huge" : "", | 20260508 09:18:44.242 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:44.242 | INFO | "malloc" : " 14 GB", | 20260508 09:18:44.242 | INFO | "pinned" : " 29 GB" | 20260508 09:18:44.242 | INFO | } | 20260508 09:18:44.242 | INFO | Perf report for the last 5.14384 seconds: | 20260508 09:18:44.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:44.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.242 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 09:18:44.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.242 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:18:44.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:44.243 | INFO | Uploaded 36GiB to GPU in 2.59887s (13.8522GiB/s) | 20260508 09:18:44.243 | INFO | Downloaded 34GiB to GPU in 2.41931s (14.0536GiB/s) | 20260508 09:18:49.382 | INFO | Analyzing chunk frames = [6973440,6973952) | 20260508 09:18:49.382 | INFO | MemInfo: { | 20260508 09:18:49.382 | INFO | "gpu" : " 17 GB", | 20260508 09:18:49.382 | INFO | "huge" : "", | 20260508 09:18:49.382 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:49.382 | INFO | "malloc" : " 14 GB", | 20260508 09:18:49.382 | INFO | "pinned" : " 29 GB" | 20260508 09:18:49.382 | INFO | } | 20260508 09:18:49.382 | INFO | Perf report for the last 5.14043 seconds: | 20260508 09:18:49.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:49.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.382 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:18:49.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.382 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:18:49.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:49.383 | INFO | Uploaded 36GiB to GPU in 2.44077s (14.7495GiB/s) | 20260508 09:18:49.383 | INFO | Downloaded 34GiB to GPU in 2.51722s (13.5069GiB/s) | 20260508 09:18:54.524 | INFO | Analyzing chunk frames = [6973952,6974464) | 20260508 09:18:54.524 | INFO | MemInfo: { | 20260508 09:18:54.524 | INFO | "gpu" : " 17 GB", | 20260508 09:18:54.524 | INFO | "huge" : "", | 20260508 09:18:54.524 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:54.524 | INFO | "malloc" : " 14 GB", | 20260508 09:18:54.524 | INFO | "pinned" : " 29 GB" | 20260508 09:18:54.524 | INFO | } | 20260508 09:18:54.524 | INFO | Perf report for the last 5.1419 seconds: | 20260508 09:18:54.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:54.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.524 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:18:54.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.524 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:18:54.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:54.525 | INFO | Uploaded 36GiB to GPU in 2.40115s (14.9928GiB/s) | 20260508 09:18:54.525 | INFO | Downloaded 34GiB to GPU in 2.42639s (14.0126GiB/s) | 20260508 09:18:59.663 | INFO | Analyzing chunk frames = [6974464,6974976) | 20260508 09:18:59.663 | INFO | MemInfo: { | 20260508 09:18:59.663 | INFO | "gpu" : " 17 GB", | 20260508 09:18:59.663 | INFO | "huge" : "", | 20260508 09:18:59.663 | INFO | "huge_shared" : " 29 GB", | 20260508 09:18:59.664 | INFO | "malloc" : " 14 GB", | 20260508 09:18:59.664 | INFO | "pinned" : " 29 GB" | 20260508 09:18:59.664 | INFO | } | 20260508 09:18:59.664 | INFO | Perf report for the last 5.13942 seconds: | 20260508 09:18:59.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:18:59.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:18:59.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:18:59.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:18:59.664 | INFO | Uploaded 36GiB to GPU in 2.44229s (14.7403GiB/s) | 20260508 09:18:59.664 | INFO | Downloaded 34GiB to GPU in 2.43238s (13.9781GiB/s) | 20260508 09:19:04.801 | INFO | Analyzing chunk frames = [6974976,6975488) | 20260508 09:19:04.801 | INFO | MemInfo: { | 20260508 09:19:04.801 | INFO | "gpu" : " 17 GB", | 20260508 09:19:04.801 | INFO | "huge" : "", | 20260508 09:19:04.801 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:04.801 | INFO | "malloc" : " 14 GB", | 20260508 09:19:04.801 | INFO | "pinned" : " 29 GB" | 20260508 09:19:04.801 | INFO | } | 20260508 09:19:04.801 | INFO | Perf report for the last 5.13727 seconds: | 20260508 09:19:04.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:04.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:19:04.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:19:04.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:04.801 | INFO | Uploaded 36GiB to GPU in 2.43474s (14.786GiB/s) | 20260508 09:19:04.801 | INFO | Downloaded 34GiB to GPU in 2.41671s (14.0687GiB/s) | 20260508 09:19:04.898 | INFO | Starting a round of Baz IO | 20260508 09:19:08.309 | INFO | Populating RTMetrics report | 20260508 09:19:08.309 | INFO | RTMetrics report populated | 20260508 09:19:08.309 | INFO | Opening tmp RTMetrics json file | 20260508 09:19:08.319 | INFO | Serializing RTMetrics to json | 20260508 09:19:08.320 | INFO | Serialization done, swapping tmp file now... | 20260508 09:19:08.321 | INFO | RTMetrics done | 20260508 09:19:08.379 | INFO | Finished a round of Baz IO: Wrote 1.484 GiB in 3.48 seconds | 20260508 09:19:08.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:19:08.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259105% / 3.97364e-06% / 5.96046e-05% | 20260508 09:19:08.379 | INFO | Baz IO overall: | 20260508 09:19:08.379 | INFO | BytesWritten : 3046312502787 | 20260508 09:19:08.379 | INFO | NumEvents : 1465593357721 | 20260508 09:19:09.941 | INFO | Analyzing chunk frames = [6975488,6976000) | 20260508 09:19:09.941 | INFO | MemInfo: { | 20260508 09:19:09.941 | INFO | "gpu" : " 17 GB", | 20260508 09:19:09.941 | INFO | "huge" : "", | 20260508 09:19:09.941 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:09.941 | INFO | "malloc" : " 14 GB", | 20260508 09:19:09.941 | INFO | "pinned" : " 29 GB" | 20260508 09:19:09.941 | INFO | } | 20260508 09:19:09.941 | INFO | Perf report for the last 5.14023 seconds: | 20260508 09:19:09.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:09.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:09.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:09.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:09.941 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:19:09.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:09.941 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:19:09.941 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 09:19:09.941 | INFO | BAZWRITER : 66.9%, 2.5, 87.5ms, 0 | 20260508 09:19:09.941 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:19:09.942 | INFO | Uploaded 36GiB to GPU in 2.611s (13.7878GiB/s) | 20260508 09:19:09.942 | INFO | Downloaded 38GiB to GPU in 2.70804s (14.0323GiB/s) | 20260508 09:19:15.080 | INFO | Analyzing chunk frames = [6976000,6976512) | 20260508 09:19:15.080 | INFO | MemInfo: { | 20260508 09:19:15.080 | INFO | "gpu" : " 17 GB", | 20260508 09:19:15.080 | INFO | "huge" : "", | 20260508 09:19:15.080 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:15.080 | INFO | "malloc" : " 14 GB", | 20260508 09:19:15.080 | INFO | "pinned" : " 29 GB" | 20260508 09:19:15.080 | INFO | } | 20260508 09:19:15.080 | INFO | Perf report for the last 5.13921 seconds: | 20260508 09:19:15.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:15.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.081 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:19:15.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.081 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:19:15.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:15.081 | INFO | Uploaded 36GiB to GPU in 2.53029s (14.2276GiB/s) | 20260508 09:19:15.081 | INFO | Downloaded 34GiB to GPU in 2.45267s (13.8624GiB/s) | 20260508 09:19:20.217 | INFO | Analyzing chunk frames = [6976512,6977024) | 20260508 09:19:20.217 | INFO | MemInfo: { | 20260508 09:19:20.217 | INFO | "gpu" : " 17 GB", | 20260508 09:19:20.217 | INFO | "huge" : "", | 20260508 09:19:20.217 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:20.217 | INFO | "malloc" : " 14 GB", | 20260508 09:19:20.217 | INFO | "pinned" : " 29 GB" | 20260508 09:19:20.217 | INFO | } | 20260508 09:19:20.217 | INFO | Perf report for the last 5.13666 seconds: | 20260508 09:19:20.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:20.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.217 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:19:20.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.217 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 09:19:20.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:20.218 | INFO | Uploaded 36GiB to GPU in 2.50423s (14.3757GiB/s) | 20260508 09:19:20.218 | INFO | Downloaded 34GiB to GPU in 2.42777s (14.0046GiB/s) | 20260508 09:19:25.355 | INFO | Analyzing chunk frames = [6977024,6977536) | 20260508 09:19:25.356 | INFO | MemInfo: { | 20260508 09:19:25.356 | INFO | "gpu" : " 17 GB", | 20260508 09:19:25.356 | INFO | "huge" : "", | 20260508 09:19:25.356 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:25.356 | INFO | "malloc" : " 14 GB", | 20260508 09:19:25.356 | INFO | "pinned" : " 29 GB" | 20260508 09:19:25.356 | INFO | } | 20260508 09:19:25.356 | INFO | Perf report for the last 5.13844 seconds: | 20260508 09:19:25.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:25.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 09:19:25.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:19:25.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:25.356 | INFO | Uploaded 36GiB to GPU in 2.60056s (13.8432GiB/s) | 20260508 09:19:25.356 | INFO | Downloaded 34GiB to GPU in 2.43093s (13.9864GiB/s) | 20260508 09:19:30.499 | INFO | Analyzing chunk frames = [6977536,6978048) | 20260508 09:19:30.499 | INFO | MemInfo: { | 20260508 09:19:30.499 | INFO | "gpu" : " 17 GB", | 20260508 09:19:30.499 | INFO | "huge" : "", | 20260508 09:19:30.499 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:30.499 | INFO | "malloc" : " 14 GB", | 20260508 09:19:30.499 | INFO | "pinned" : " 29 GB" | 20260508 09:19:30.500 | INFO | } | 20260508 09:19:30.500 | INFO | Perf report for the last 5.1441 seconds: | 20260508 09:19:30.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:30.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:19:30.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:19:30.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:30.500 | INFO | Uploaded 36GiB to GPU in 2.4209s (14.8705GiB/s) | 20260508 09:19:30.500 | INFO | Downloaded 34GiB to GPU in 2.51861s (13.4995GiB/s) | 20260508 09:19:35.640 | INFO | Analyzing chunk frames = [6978048,6978560) | 20260508 09:19:35.640 | INFO | MemInfo: { | 20260508 09:19:35.640 | INFO | "gpu" : " 17 GB", | 20260508 09:19:35.640 | INFO | "huge" : "", | 20260508 09:19:35.640 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:35.640 | INFO | "malloc" : " 14 GB", | 20260508 09:19:35.640 | INFO | "pinned" : " 29 GB" | 20260508 09:19:35.640 | INFO | } | 20260508 09:19:35.640 | INFO | Perf report for the last 5.14046 seconds: | 20260508 09:19:35.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:35.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:19:35.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:19:35.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:35.640 | INFO | Uploaded 36GiB to GPU in 2.41044s (14.935GiB/s) | 20260508 09:19:35.640 | INFO | Downloaded 34GiB to GPU in 2.41807s (14.0608GiB/s) | 20260508 09:19:40.780 | INFO | Analyzing chunk frames = [6978560,6979072) | 20260508 09:19:40.780 | INFO | MemInfo: { | 20260508 09:19:40.780 | INFO | "gpu" : " 17 GB", | 20260508 09:19:40.780 | INFO | "huge" : "", | 20260508 09:19:40.780 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:40.780 | INFO | "malloc" : " 14 GB", | 20260508 09:19:40.780 | INFO | "pinned" : " 29 GB" | 20260508 09:19:40.780 | INFO | } | 20260508 09:19:40.780 | INFO | Perf report for the last 5.13977 seconds: | 20260508 09:19:40.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:40.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 09:19:40.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:19:40.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:40.780 | INFO | Uploaded 36GiB to GPU in 2.43725s (14.7707GiB/s) | 20260508 09:19:40.780 | INFO | Downloaded 34GiB to GPU in 2.42918s (13.9965GiB/s) | 20260508 09:19:45.912 | INFO | Analyzing chunk frames = [6979072,6979584) | 20260508 09:19:45.912 | INFO | MemInfo: { | 20260508 09:19:45.912 | INFO | "gpu" : " 17 GB", | 20260508 09:19:45.912 | INFO | "huge" : "", | 20260508 09:19:45.912 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:45.912 | INFO | "malloc" : " 14 GB", | 20260508 09:19:45.912 | INFO | "pinned" : " 29 GB" | 20260508 09:19:45.912 | INFO | } | 20260508 09:19:45.912 | INFO | Perf report for the last 5.13189 seconds: | 20260508 09:19:45.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:45.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 09:19:45.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 09:19:45.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:45.912 | INFO | Uploaded 36GiB to GPU in 2.43635s (14.7762GiB/s) | 20260508 09:19:45.912 | INFO | Downloaded 34GiB to GPU in 2.41035s (14.1059GiB/s) | 20260508 09:19:45.912 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:19:45.912 | INFO | Total mem usage: 16.7 GiB | 20260508 09:19:45.912 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:19:45.912 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:19:45.912 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:19:45.912 | INFO | High Water Breakdown: | 20260508 09:19:45.912 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:19:45.912 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:19:45.912 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:19:45.912 | INFO | ------------------------------------------ | 20260508 09:19:45.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:19:45.913 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:19:45.913 | INFO | Total mem usage: 26.5 GiB | 20260508 09:19:45.913 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:19:45.913 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:19:45.913 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:19:45.913 | INFO | High Water Breakdown: | 20260508 09:19:45.913 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:19:45.913 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:19:45.913 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:19:45.913 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:19:45.913 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:19:45.913 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:19:45.913 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:19:45.913 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:19:45.913 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:19:45.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:19:45.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:19:45.913 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:19:45.913 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:19:45.913 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:19:45.913 | INFO | ------------------------------------------ | 20260508 09:19:45.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:19:45.913 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:19:45.913 | INFO | Total mem usage: 27.8 GiB | 20260508 09:19:45.913 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:19:45.913 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:19:45.913 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:19:45.913 | INFO | High Water Breakdown: | 20260508 09:19:45.913 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:19:45.913 | INFO | ------------------------------------------ | 20260508 09:19:45.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:19:45.913 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:19:45.913 | INFO | Total mem usage: 15.7 GiB | 20260508 09:19:45.913 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:19:45.913 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:19:45.913 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:19:45.913 | INFO | High Water Breakdown: | 20260508 09:19:45.913 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:19:45.913 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:19:45.913 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:19:45.913 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:19:45.913 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:19:45.913 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:19:45.913 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:19:45.913 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:19:45.913 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:19:45.913 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:19:45.913 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:19:45.913 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:19:45.913 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:19:45.913 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:19:45.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:19:45.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:19:45.913 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:19:45.913 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:19:45.913 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:19:45.913 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:19:45.913 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:19:45.913 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:19:45.913 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:19:45.913 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:19:45.913 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:19:45.913 | INFO | ------------------------------------------ | 20260508 09:19:45.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:19:45.913 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:19:45.913 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:19:45.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:19:45.913 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 5.47e-01 | 20260508 09:19:45.913 | INFO | PerfReport: 0.29%, AggregateStats -- 9.63e-02 / 1.04e-01, 1.11e-02 / 5.93e-03 | 20260508 09:19:45.913 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.42e+00, 5.98e-01 / 2.67e-01 | 20260508 09:19:45.913 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.34e-01 | 20260508 09:19:45.913 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.49e+01, 3.67e+00 / 1.27e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.82e+00 / 1.60e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.45e+00, 4.07e-01 / 2.82e-01 | 20260508 09:19:45.913 | INFO | PerfReport: 3.05%, Metrics -- 9.95e-01 / 1.18e+00, 2.49e-01 / 1.42e-01 | 20260508 09:19:45.913 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.32e-01, 4.83e-02 / 2.65e-02 | 20260508 09:19:45.913 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:19:45.913 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:19:45.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:19:45.913 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 24.62%, Upload -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 24.44%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 30.97%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 2.40%, LockUpload -- 2.53e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 2.35%, LockDownload -- 2.48e+00 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 15.22%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:19:45.913 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260508 09:19:46.011 | INFO | Starting a round of Baz IO | 20260508 09:19:49.428 | INFO | Populating RTMetrics report | 20260508 09:19:49.428 | INFO | RTMetrics report populated | 20260508 09:19:49.428 | INFO | Opening tmp RTMetrics json file | 20260508 09:19:49.438 | INFO | Serializing RTMetrics to json | 20260508 09:19:49.439 | INFO | Serialization done, swapping tmp file now... | 20260508 09:19:49.439 | INFO | RTMetrics done | 20260508 09:19:49.496 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.484 seconds | 20260508 09:19:49.496 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:19:49.496 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258001% / 0% / 6.35783e-05% | 20260508 09:19:49.496 | INFO | Baz IO overall: | 20260508 09:19:49.496 | INFO | BytesWritten : 3047904450572 | 20260508 09:19:49.496 | INFO | NumEvents : 1466345943892 | 20260508 09:19:51.051 | INFO | Analyzing chunk frames = [6979584,6980096) | 20260508 09:19:51.051 | INFO | MemInfo: { | 20260508 09:19:51.051 | INFO | "gpu" : " 17 GB", | 20260508 09:19:51.051 | INFO | "huge" : "", | 20260508 09:19:51.051 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:51.052 | INFO | "malloc" : " 14 GB", | 20260508 09:19:51.052 | INFO | "pinned" : " 29 GB" | 20260508 09:19:51.052 | INFO | } | 20260508 09:19:51.052 | INFO | Perf report for the last 5.13988 seconds: | 20260508 09:19:51.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:51.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:51.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:51.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:51.052 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:19:51.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:51.052 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:19:51.052 | INFO | PRE_HQ : 65.3%, 1.6, 54.9ms, 0 | 20260508 09:19:51.052 | INFO | BAZWRITER : 67.2%, 2.6, 88.2ms, 0 | 20260508 09:19:51.052 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:19:51.052 | INFO | Uploaded 36GiB to GPU in 2.59933s (13.8497GiB/s) | 20260508 09:19:51.053 | INFO | Downloaded 38GiB to GPU in 2.70163s (14.0656GiB/s) | 20260508 09:19:56.195 | INFO | Analyzing chunk frames = [6980096,6980608) | 20260508 09:19:56.195 | INFO | MemInfo: { | 20260508 09:19:56.195 | INFO | "gpu" : " 17 GB", | 20260508 09:19:56.195 | INFO | "huge" : "", | 20260508 09:19:56.195 | INFO | "huge_shared" : " 29 GB", | 20260508 09:19:56.195 | INFO | "malloc" : " 14 GB", | 20260508 09:19:56.195 | INFO | "pinned" : " 29 GB" | 20260508 09:19:56.195 | INFO | } | 20260508 09:19:56.195 | INFO | Perf report for the last 5.14329 seconds: | 20260508 09:19:56.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:19:56.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.195 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 09:19:56.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.195 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 09:19:56.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:19:56.196 | INFO | Uploaded 36GiB to GPU in 2.52167s (14.2763GiB/s) | 20260508 09:19:56.196 | INFO | Downloaded 34GiB to GPU in 2.46803s (13.7762GiB/s) | 20260508 09:20:01.335 | INFO | Analyzing chunk frames = [6980608,6981120) | 20260508 09:20:01.335 | INFO | MemInfo: { | 20260508 09:20:01.335 | INFO | "gpu" : " 17 GB", | 20260508 09:20:01.335 | INFO | "huge" : "", | 20260508 09:20:01.335 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:01.335 | INFO | "malloc" : " 14 GB", | 20260508 09:20:01.335 | INFO | "pinned" : " 29 GB" | 20260508 09:20:01.335 | INFO | } | 20260508 09:20:01.335 | INFO | Perf report for the last 5.14029 seconds: | 20260508 09:20:01.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:01.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.335 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 09:20:01.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.335 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:20:01.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:01.336 | INFO | Uploaded 36GiB to GPU in 2.47513s (14.5447GiB/s) | 20260508 09:20:01.336 | INFO | Downloaded 34GiB to GPU in 2.45089s (13.8725GiB/s) | 20260508 09:20:06.475 | INFO | Analyzing chunk frames = [6981120,6981632) | 20260508 09:20:06.475 | INFO | MemInfo: { | 20260508 09:20:06.475 | INFO | "gpu" : " 17 GB", | 20260508 09:20:06.475 | INFO | "huge" : "", | 20260508 09:20:06.475 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:06.475 | INFO | "malloc" : " 14 GB", | 20260508 09:20:06.475 | INFO | "pinned" : " 29 GB" | 20260508 09:20:06.476 | INFO | } | 20260508 09:20:06.476 | INFO | Perf report for the last 5.14021 seconds: | 20260508 09:20:06.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:06.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:20:06.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:20:06.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:06.476 | INFO | Uploaded 36GiB to GPU in 2.49449s (14.4318GiB/s) | 20260508 09:20:06.476 | INFO | Downloaded 34GiB to GPU in 2.44644s (13.8977GiB/s) | 20260508 09:20:11.607 | INFO | Analyzing chunk frames = [6981632,6982144) | 20260508 09:20:11.607 | INFO | MemInfo: { | 20260508 09:20:11.607 | INFO | "gpu" : " 17 GB", | 20260508 09:20:11.607 | INFO | "huge" : "", | 20260508 09:20:11.607 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:11.607 | INFO | "malloc" : " 14 GB", | 20260508 09:20:11.607 | INFO | "pinned" : " 29 GB" | 20260508 09:20:11.607 | INFO | } | 20260508 09:20:11.607 | INFO | Perf report for the last 5.13181 seconds: | 20260508 09:20:11.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:11.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.607 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 09:20:11.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.607 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:20:11.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:11.608 | INFO | Uploaded 36GiB to GPU in 2.38078s (15.1211GiB/s) | 20260508 09:20:11.608 | INFO | Downloaded 34GiB to GPU in 2.51431s (13.5226GiB/s) | 20260508 09:20:16.751 | INFO | Analyzing chunk frames = [6982144,6982656) | 20260508 09:20:16.751 | INFO | MemInfo: { | 20260508 09:20:16.751 | INFO | "gpu" : " 17 GB", | 20260508 09:20:16.751 | INFO | "huge" : "", | 20260508 09:20:16.751 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:16.751 | INFO | "malloc" : " 14 GB", | 20260508 09:20:16.751 | INFO | "pinned" : " 29 GB" | 20260508 09:20:16.751 | INFO | } | 20260508 09:20:16.751 | INFO | Perf report for the last 5.144 seconds: | 20260508 09:20:16.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:16.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.752 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:20:16.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.752 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:20:16.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:16.752 | INFO | Uploaded 36GiB to GPU in 2.37994s (15.1264GiB/s) | 20260508 09:20:16.752 | INFO | Downloaded 34GiB to GPU in 2.43387s (13.9695GiB/s) | 20260508 09:20:21.892 | INFO | Analyzing chunk frames = [6982656,6983168) | 20260508 09:20:21.892 | INFO | MemInfo: { | 20260508 09:20:21.892 | INFO | "gpu" : " 17 GB", | 20260508 09:20:21.892 | INFO | "huge" : "", | 20260508 09:20:21.892 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:21.892 | INFO | "malloc" : " 14 GB", | 20260508 09:20:21.892 | INFO | "pinned" : " 29 GB" | 20260508 09:20:21.892 | INFO | } | 20260508 09:20:21.892 | INFO | Perf report for the last 5.14066 seconds: | 20260508 09:20:21.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:21.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:20:21.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:20:21.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:21.892 | INFO | Uploaded 36GiB to GPU in 2.42222s (14.8624GiB/s) | 20260508 09:20:21.892 | INFO | Downloaded 34GiB to GPU in 2.44599s (13.9003GiB/s) | 20260508 09:20:27.031 | INFO | Analyzing chunk frames = [6983168,6983680) | 20260508 09:20:27.031 | INFO | MemInfo: { | 20260508 09:20:27.031 | INFO | "gpu" : " 17 GB", | 20260508 09:20:27.031 | INFO | "huge" : "", | 20260508 09:20:27.031 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:27.031 | INFO | "malloc" : " 14 GB", | 20260508 09:20:27.031 | INFO | "pinned" : " 29 GB" | 20260508 09:20:27.031 | INFO | } | 20260508 09:20:27.032 | INFO | Perf report for the last 5.13945 seconds: | 20260508 09:20:27.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:27.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:20:27.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:20:27.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:27.032 | INFO | Uploaded 36GiB to GPU in 2.40513s (14.968GiB/s) | 20260508 09:20:27.032 | INFO | Downloaded 34GiB to GPU in 2.43701s (13.9515GiB/s) | 20260508 09:20:27.129 | INFO | Starting a round of Baz IO | 20260508 09:20:30.542 | INFO | Populating RTMetrics report | 20260508 09:20:30.543 | INFO | RTMetrics report populated | 20260508 09:20:30.543 | INFO | Opening tmp RTMetrics json file | 20260508 09:20:30.552 | INFO | Serializing RTMetrics to json | 20260508 09:20:30.553 | INFO | Serialization done, swapping tmp file now... | 20260508 09:20:30.554 | INFO | RTMetrics done | 20260508 09:20:30.615 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.485 seconds | 20260508 09:20:30.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:20:30.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258148% / 0% / 4.37101e-05% | 20260508 09:20:30.615 | INFO | Baz IO overall: | 20260508 09:20:30.615 | INFO | BytesWritten : 3049496663289 | 20260508 09:20:30.615 | INFO | NumEvents : 1467098800506 | 20260508 09:20:32.175 | INFO | Analyzing chunk frames = [6983680,6984192) | 20260508 09:20:32.175 | INFO | MemInfo: { | 20260508 09:20:32.175 | INFO | "gpu" : " 17 GB", | 20260508 09:20:32.175 | INFO | "huge" : "", | 20260508 09:20:32.175 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:32.175 | INFO | "malloc" : " 14 GB", | 20260508 09:20:32.175 | INFO | "pinned" : " 29 GB" | 20260508 09:20:32.175 | INFO | } | 20260508 09:20:32.175 | INFO | Perf report for the last 5.14327 seconds: | 20260508 09:20:32.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:32.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:32.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:32.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:32.175 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:20:32.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:32.175 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 09:20:32.175 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 09:20:32.175 | INFO | BAZWRITER : 67.1%, 2.5, 87.7ms, 0 | 20260508 09:20:32.175 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:20:32.176 | INFO | Uploaded 36GiB to GPU in 2.60776s (13.805GiB/s) | 20260508 09:20:32.176 | INFO | Downloaded 38GiB to GPU in 2.71044s (14.0199GiB/s) | 20260508 09:20:37.309 | INFO | Analyzing chunk frames = [6984192,6984704) | 20260508 09:20:37.309 | INFO | MemInfo: { | 20260508 09:20:37.309 | INFO | "gpu" : " 17 GB", | 20260508 09:20:37.309 | INFO | "huge" : "", | 20260508 09:20:37.309 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:37.309 | INFO | "malloc" : " 14 GB", | 20260508 09:20:37.309 | INFO | "pinned" : " 29 GB" | 20260508 09:20:37.309 | INFO | } | 20260508 09:20:37.309 | INFO | Perf report for the last 5.13429 seconds: | 20260508 09:20:37.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:37.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.309 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 09:20:37.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.309 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:20:37.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:37.310 | INFO | Uploaded 36GiB to GPU in 2.48493s (14.4873GiB/s) | 20260508 09:20:37.310 | INFO | Downloaded 34GiB to GPU in 2.48057s (13.7065GiB/s) | 20260508 09:20:42.452 | INFO | Analyzing chunk frames = [6984704,6985216) | 20260508 09:20:42.452 | INFO | MemInfo: { | 20260508 09:20:42.452 | INFO | "gpu" : " 17 GB", | 20260508 09:20:42.452 | INFO | "huge" : "", | 20260508 09:20:42.452 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:42.453 | INFO | "malloc" : " 14 GB", | 20260508 09:20:42.453 | INFO | "pinned" : " 29 GB" | 20260508 09:20:42.453 | INFO | } | 20260508 09:20:42.453 | INFO | Perf report for the last 5.14351 seconds: | 20260508 09:20:42.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:42.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:20:42.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:20:42.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:42.453 | INFO | Uploaded 36GiB to GPU in 2.47327s (14.5556GiB/s) | 20260508 09:20:42.453 | INFO | Downloaded 34GiB to GPU in 2.44357s (13.914GiB/s) | 20260508 09:20:47.591 | INFO | Analyzing chunk frames = [6985216,6985728) | 20260508 09:20:47.591 | INFO | MemInfo: { | 20260508 09:20:47.591 | INFO | "gpu" : " 17 GB", | 20260508 09:20:47.591 | INFO | "huge" : "", | 20260508 09:20:47.591 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:47.591 | INFO | "malloc" : " 14 GB", | 20260508 09:20:47.591 | INFO | "pinned" : " 29 GB" | 20260508 09:20:47.591 | INFO | } | 20260508 09:20:47.591 | INFO | Perf report for the last 5.13816 seconds: | 20260508 09:20:47.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:47.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:20:47.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:20:47.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:47.591 | INFO | Uploaded 36GiB to GPU in 2.49298s (14.4405GiB/s) | 20260508 09:20:47.591 | INFO | Downloaded 34GiB to GPU in 2.43673s (13.9531GiB/s) | 20260508 09:20:52.726 | INFO | Analyzing chunk frames = [6985728,6986240) | 20260508 09:20:52.726 | INFO | MemInfo: { | 20260508 09:20:52.726 | INFO | "gpu" : " 17 GB", | 20260508 09:20:52.726 | INFO | "huge" : "", | 20260508 09:20:52.726 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:52.726 | INFO | "malloc" : " 14 GB", | 20260508 09:20:52.726 | INFO | "pinned" : " 29 GB" | 20260508 09:20:52.726 | INFO | } | 20260508 09:20:52.726 | INFO | Perf report for the last 5.13519 seconds: | 20260508 09:20:52.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:52.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.726 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 09:20:52.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.726 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 09:20:52.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:52.727 | INFO | Uploaded 36GiB to GPU in 2.35704s (15.2734GiB/s) | 20260508 09:20:52.727 | INFO | Downloaded 34GiB to GPU in 2.52527s (13.4639GiB/s) | 20260508 09:20:57.862 | INFO | Analyzing chunk frames = [6986240,6986752) | 20260508 09:20:57.862 | INFO | MemInfo: { | 20260508 09:20:57.862 | INFO | "gpu" : " 17 GB", | 20260508 09:20:57.862 | INFO | "huge" : "", | 20260508 09:20:57.862 | INFO | "huge_shared" : " 29 GB", | 20260508 09:20:57.862 | INFO | "malloc" : " 14 GB", | 20260508 09:20:57.862 | INFO | "pinned" : " 29 GB" | 20260508 09:20:57.862 | INFO | } | 20260508 09:20:57.862 | INFO | Perf report for the last 5.13605 seconds: | 20260508 09:20:57.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:20:57.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.862 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:20:57.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.862 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:20:57.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:20:57.863 | INFO | Uploaded 36GiB to GPU in 2.37037s (15.1875GiB/s) | 20260508 09:20:57.863 | INFO | Downloaded 34GiB to GPU in 2.43621s (13.9561GiB/s) | 20260508 09:21:03.003 | INFO | Analyzing chunk frames = [6986752,6987264) | 20260508 09:21:03.004 | INFO | MemInfo: { | 20260508 09:21:03.004 | INFO | "gpu" : " 17 GB", | 20260508 09:21:03.004 | INFO | "huge" : "", | 20260508 09:21:03.004 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:03.004 | INFO | "malloc" : " 14 GB", | 20260508 09:21:03.004 | INFO | "pinned" : " 29 GB" | 20260508 09:21:03.004 | INFO | } | 20260508 09:21:03.004 | INFO | Perf report for the last 5.14165 seconds: | 20260508 09:21:03.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:03.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:21:03.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 09:21:03.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:03.004 | INFO | Uploaded 36GiB to GPU in 2.41576s (14.9021GiB/s) | 20260508 09:21:03.004 | INFO | Downloaded 34GiB to GPU in 2.4526s (13.8628GiB/s) | 20260508 09:21:08.152 | INFO | Analyzing chunk frames = [6987264,6987776) | 20260508 09:21:08.152 | INFO | MemInfo: { | 20260508 09:21:08.152 | INFO | "gpu" : " 17 GB", | 20260508 09:21:08.152 | INFO | "huge" : "", | 20260508 09:21:08.152 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:08.152 | INFO | "malloc" : " 14 GB", | 20260508 09:21:08.152 | INFO | "pinned" : " 29 GB" | 20260508 09:21:08.152 | INFO | } | 20260508 09:21:08.152 | INFO | Perf report for the last 5.14826 seconds: | 20260508 09:21:08.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:08.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | ANALYSIS : 65.6%, 3.0, 102.8ms, 0 | 20260508 09:21:08.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:21:08.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:08.152 | INFO | Uploaded 36GiB to GPU in 2.43221s (14.8013GiB/s) | 20260508 09:21:08.152 | INFO | Downloaded 34GiB to GPU in 2.44537s (13.9038GiB/s) | 20260508 09:21:08.153 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:21:08.153 | INFO | Total mem usage: 16.7 GiB | 20260508 09:21:08.153 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:21:08.153 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:21:08.153 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:21:08.153 | INFO | High Water Breakdown: | 20260508 09:21:08.153 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:21:08.153 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:21:08.153 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:21:08.153 | INFO | ------------------------------------------ | 20260508 09:21:08.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:21:08.153 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:21:08.153 | INFO | Total mem usage: 26.5 GiB | 20260508 09:21:08.153 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:21:08.153 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:21:08.153 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:21:08.153 | INFO | High Water Breakdown: | 20260508 09:21:08.153 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:21:08.153 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:21:08.153 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:21:08.153 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:21:08.153 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:21:08.153 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:21:08.153 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:21:08.153 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:21:08.153 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:21:08.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:21:08.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:21:08.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:21:08.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:21:08.153 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:21:08.153 | INFO | ------------------------------------------ | 20260508 09:21:08.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:21:08.153 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:21:08.153 | INFO | Total mem usage: 27.8 GiB | 20260508 09:21:08.153 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:21:08.153 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:21:08.153 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:21:08.153 | INFO | High Water Breakdown: | 20260508 09:21:08.153 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:21:08.153 | INFO | ------------------------------------------ | 20260508 09:21:08.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:21:08.153 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:21:08.153 | INFO | Total mem usage: 15.7 GiB | 20260508 09:21:08.153 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:21:08.153 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:21:08.153 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:21:08.153 | INFO | High Water Breakdown: | 20260508 09:21:08.153 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:21:08.153 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:21:08.153 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:21:08.153 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:21:08.153 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:21:08.153 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:21:08.153 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:21:08.153 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:21:08.153 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:21:08.153 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:21:08.153 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:21:08.153 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:21:08.153 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:21:08.153 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:21:08.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:21:08.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:21:08.153 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:21:08.153 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:21:08.153 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:21:08.153 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:21:08.153 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:21:08.153 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:21:08.153 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:21:08.153 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:21:08.153 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:21:08.153 | INFO | ------------------------------------------ | 20260508 09:21:08.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:21:08.154 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:21:08.154 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:21:08.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:21:08.154 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.42e+01 / 5.52e-01 | 20260508 09:21:08.154 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.05e-01, 1.13e-02 / 6.19e-03 | 20260508 09:21:08.154 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.48e+00, 5.82e-01 / 2.07e-01 | 20260508 09:21:08.154 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.45e+00, 2.08e-01 / 1.29e-01 | 20260508 09:21:08.154 | INFO | PerfReport: 15.97%, DME -- 4.19e+01 / 4.51e+01, 3.58e+00 / 9.38e-01 | 20260508 09:21:08.154 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.29e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.32e+00 / 2.55e+00, 4.04e-01 / 2.54e-01 | 20260508 09:21:08.154 | INFO | PerfReport: 3.02%, Metrics -- 9.91e-01 / 1.13e+00, 2.45e-01 / 9.99e-02 | 20260508 09:21:08.154 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.39e-01, 4.89e-02 / 3.00e-02 | 20260508 09:21:08.154 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:21:08.154 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:21:08.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:21:08.154 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 24.61%, Download -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 31.11%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 2.14%, LockUpload -- 2.26e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 2.54%, LockDownload -- 2.68e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 15.30%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:21:08.154 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260508 09:21:08.249 | INFO | Starting a round of Baz IO | 20260508 09:21:11.660 | INFO | Populating RTMetrics report | 20260508 09:21:11.660 | INFO | RTMetrics report populated | 20260508 09:21:11.660 | INFO | Opening tmp RTMetrics json file | 20260508 09:21:11.671 | INFO | Serializing RTMetrics to json | 20260508 09:21:11.672 | INFO | Serialization done, swapping tmp file now... | 20260508 09:21:11.672 | INFO | RTMetrics done | 20260508 09:21:11.733 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.483 seconds | 20260508 09:21:11.733 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.00% / 59.68% | 20260508 09:21:11.733 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257496% / 3.97364e-06% / 5.16574e-05% | 20260508 09:21:11.733 | INFO | Baz IO overall: | 20260508 09:21:11.733 | INFO | BytesWritten : 3051089122253 | 20260508 09:21:11.733 | INFO | NumEvents : 1467852161825 | 20260508 09:21:13.290 | INFO | Analyzing chunk frames = [6987776,6988288) | 20260508 09:21:13.290 | INFO | MemInfo: { | 20260508 09:21:13.290 | INFO | "gpu" : " 17 GB", | 20260508 09:21:13.290 | INFO | "huge" : "", | 20260508 09:21:13.290 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:13.290 | INFO | "malloc" : " 14 GB", | 20260508 09:21:13.290 | INFO | "pinned" : " 29 GB" | 20260508 09:21:13.290 | INFO | } | 20260508 09:21:13.290 | INFO | Perf report for the last 5.13808 seconds: | 20260508 09:21:13.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:13.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:13.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:13.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:13.290 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:21:13.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:13.290 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:21:13.290 | INFO | PRE_HQ : 65.5%, 1.6, 54.5ms, 0 | 20260508 09:21:13.290 | INFO | BAZWRITER : 67.1%, 2.6, 88.4ms, 0 | 20260508 09:21:13.290 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:21:13.291 | INFO | Uploaded 36GiB to GPU in 2.62371s (13.721GiB/s) | 20260508 09:21:13.291 | INFO | Downloaded 38GiB to GPU in 2.71266s (14.0084GiB/s) | 20260508 09:21:18.422 | INFO | Analyzing chunk frames = [6988288,6988800) | 20260508 09:21:18.423 | INFO | MemInfo: { | 20260508 09:21:18.423 | INFO | "gpu" : " 17 GB", | 20260508 09:21:18.423 | INFO | "huge" : "", | 20260508 09:21:18.423 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:18.423 | INFO | "malloc" : " 14 GB", | 20260508 09:21:18.423 | INFO | "pinned" : " 29 GB" | 20260508 09:21:18.423 | INFO | } | 20260508 09:21:18.423 | INFO | Perf report for the last 5.13266 seconds: | 20260508 09:21:18.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:18.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.423 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 09:21:18.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.423 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:21:18.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:18.424 | INFO | Uploaded 36GiB to GPU in 2.45605s (14.6577GiB/s) | 20260508 09:21:18.424 | INFO | Downloaded 34GiB to GPU in 2.47333s (13.7467GiB/s) | 20260508 09:21:23.561 | INFO | Analyzing chunk frames = [6988800,6989312) | 20260508 09:21:23.561 | INFO | MemInfo: { | 20260508 09:21:23.561 | INFO | "gpu" : " 17 GB", | 20260508 09:21:23.561 | INFO | "huge" : "", | 20260508 09:21:23.561 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:23.561 | INFO | "malloc" : " 14 GB", | 20260508 09:21:23.561 | INFO | "pinned" : " 29 GB" | 20260508 09:21:23.562 | INFO | } | 20260508 09:21:23.562 | INFO | Perf report for the last 5.13876 seconds: | 20260508 09:21:23.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:23.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:21:23.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:21:23.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:23.562 | INFO | Uploaded 36GiB to GPU in 2.47994s (14.5165GiB/s) | 20260508 09:21:23.562 | INFO | Downloaded 34GiB to GPU in 2.43961s (13.9367GiB/s) | 20260508 09:21:28.703 | INFO | Analyzing chunk frames = [6989312,6989824) | 20260508 09:21:28.703 | INFO | MemInfo: { | 20260508 09:21:28.703 | INFO | "gpu" : " 17 GB", | 20260508 09:21:28.703 | INFO | "huge" : "", | 20260508 09:21:28.703 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:28.703 | INFO | "malloc" : " 14 GB", | 20260508 09:21:28.703 | INFO | "pinned" : " 29 GB" | 20260508 09:21:28.703 | INFO | } | 20260508 09:21:28.703 | INFO | Perf report for the last 5.14122 seconds: | 20260508 09:21:28.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:28.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:21:28.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:21:28.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:28.703 | INFO | Uploaded 36GiB to GPU in 2.48871s (14.4653GiB/s) | 20260508 09:21:28.703 | INFO | Downloaded 34GiB to GPU in 2.43378s (13.97GiB/s) | 20260508 09:21:33.848 | INFO | Analyzing chunk frames = [6989824,6990336) | 20260508 09:21:33.848 | INFO | MemInfo: { | 20260508 09:21:33.848 | INFO | "gpu" : " 17 GB", | 20260508 09:21:33.848 | INFO | "huge" : "", | 20260508 09:21:33.848 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:33.848 | INFO | "malloc" : " 14 GB", | 20260508 09:21:33.848 | INFO | "pinned" : " 29 GB" | 20260508 09:21:33.848 | INFO | } | 20260508 09:21:33.848 | INFO | Perf report for the last 5.14518 seconds: | 20260508 09:21:33.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:33.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.848 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 09:21:33.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.848 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:21:33.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:33.849 | INFO | Uploaded 36GiB to GPU in 2.29906s (15.6586GiB/s) | 20260508 09:21:33.849 | INFO | Downloaded 34GiB to GPU in 2.51433s (13.5225GiB/s) | 20260508 09:21:38.981 | INFO | Analyzing chunk frames = [6990336,6990848) | 20260508 09:21:38.981 | INFO | MemInfo: { | 20260508 09:21:38.981 | INFO | "gpu" : " 17 GB", | 20260508 09:21:38.981 | INFO | "huge" : "", | 20260508 09:21:38.981 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:38.981 | INFO | "malloc" : " 14 GB", | 20260508 09:21:38.981 | INFO | "pinned" : " 29 GB" | 20260508 09:21:38.981 | INFO | } | 20260508 09:21:38.981 | INFO | Perf report for the last 5.13309 seconds: | 20260508 09:21:38.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:38.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.981 | INFO | ANALYSIS : 65.4%, 3.0, 101.5ms, 0 | 20260508 09:21:38.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.981 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:21:38.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:38.982 | INFO | Uploaded 36GiB to GPU in 2.35414s (15.2922GiB/s) | 20260508 09:21:38.982 | INFO | Downloaded 34GiB to GPU in 2.43263s (13.9766GiB/s) | 20260508 09:21:44.116 | INFO | Analyzing chunk frames = [6990848,6991360) | 20260508 09:21:44.116 | INFO | MemInfo: { | 20260508 09:21:44.116 | INFO | "gpu" : " 17 GB", | 20260508 09:21:44.116 | INFO | "huge" : "", | 20260508 09:21:44.116 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:44.116 | INFO | "malloc" : " 14 GB", | 20260508 09:21:44.116 | INFO | "pinned" : " 29 GB" | 20260508 09:21:44.116 | INFO | } | 20260508 09:21:44.116 | INFO | Perf report for the last 5.1347 seconds: | 20260508 09:21:44.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:44.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:21:44.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:21:44.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:44.116 | INFO | Uploaded 36GiB to GPU in 2.40349s (14.9782GiB/s) | 20260508 09:21:44.116 | INFO | Downloaded 34GiB to GPU in 2.44695s (13.8948GiB/s) | 20260508 09:21:49.259 | INFO | Analyzing chunk frames = [6991360,6991872) | 20260508 09:21:49.259 | INFO | MemInfo: { | 20260508 09:21:49.259 | INFO | "gpu" : " 17 GB", | 20260508 09:21:49.259 | INFO | "huge" : "", | 20260508 09:21:49.259 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:49.259 | INFO | "malloc" : " 14 GB", | 20260508 09:21:49.259 | INFO | "pinned" : " 29 GB" | 20260508 09:21:49.259 | INFO | } | 20260508 09:21:49.259 | INFO | Perf report for the last 5.14328 seconds: | 20260508 09:21:49.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:49.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.259 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 09:21:49.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.259 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:21:49.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:49.260 | INFO | Uploaded 36GiB to GPU in 2.42408s (14.851GiB/s) | 20260508 09:21:49.260 | INFO | Downloaded 34GiB to GPU in 2.43888s (13.9408GiB/s) | 20260508 09:21:49.356 | INFO | Starting a round of Baz IO | 20260508 09:21:52.782 | INFO | Populating RTMetrics report | 20260508 09:21:52.783 | INFO | RTMetrics report populated | 20260508 09:21:52.783 | INFO | Opening tmp RTMetrics json file | 20260508 09:21:52.790 | INFO | Serializing RTMetrics to json | 20260508 09:21:52.793 | INFO | Serialization done, swapping tmp file now... | 20260508 09:21:52.793 | INFO | RTMetrics done | 20260508 09:21:52.852 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.495 seconds | 20260508 09:21:52.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.68% | 20260508 09:21:52.852 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.25779% / 0% / 6.75519e-05% | 20260508 09:21:52.852 | INFO | Baz IO overall: | 20260508 09:21:52.852 | INFO | BytesWritten : 3052681655506 | 20260508 09:21:52.852 | INFO | NumEvents : 1468605330913 | 20260508 09:21:54.399 | INFO | Analyzing chunk frames = [6991872,6992384) | 20260508 09:21:54.399 | INFO | MemInfo: { | 20260508 09:21:54.399 | INFO | "gpu" : " 17 GB", | 20260508 09:21:54.399 | INFO | "huge" : "", | 20260508 09:21:54.399 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:54.399 | INFO | "malloc" : " 14 GB", | 20260508 09:21:54.399 | INFO | "pinned" : " 29 GB" | 20260508 09:21:54.399 | INFO | } | 20260508 09:21:54.400 | INFO | Perf report for the last 5.14036 seconds: | 20260508 09:21:54.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:54.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:54.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:54.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:54.400 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 09:21:54.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:54.400 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:21:54.400 | INFO | PRE_HQ : 65.1%, 1.6, 54.5ms, 0 | 20260508 09:21:54.400 | INFO | BAZWRITER : 67.3%, 2.5, 87.9ms, 0 | 20260508 09:21:54.400 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:21:54.400 | INFO | Uploaded 36GiB to GPU in 2.62734s (13.7021GiB/s) | 20260508 09:21:54.400 | INFO | Downloaded 38GiB to GPU in 2.69186s (14.1166GiB/s) | 20260508 09:21:59.538 | INFO | Analyzing chunk frames = [6992384,6992896) | 20260508 09:21:59.538 | INFO | MemInfo: { | 20260508 09:21:59.538 | INFO | "gpu" : " 17 GB", | 20260508 09:21:59.538 | INFO | "huge" : "", | 20260508 09:21:59.538 | INFO | "huge_shared" : " 29 GB", | 20260508 09:21:59.538 | INFO | "malloc" : " 14 GB", | 20260508 09:21:59.539 | INFO | "pinned" : " 29 GB" | 20260508 09:21:59.539 | INFO | } | 20260508 09:21:59.539 | INFO | Perf report for the last 5.13905 seconds: | 20260508 09:21:59.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:21:59.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:21:59.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:21:59.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:21:59.539 | INFO | Uploaded 36GiB to GPU in 2.5402s (14.1721GiB/s) | 20260508 09:21:59.539 | INFO | Downloaded 34GiB to GPU in 2.44368s (13.9134GiB/s) | 20260508 09:22:04.677 | INFO | Analyzing chunk frames = [6992896,6993408) | 20260508 09:22:04.677 | INFO | MemInfo: { | 20260508 09:22:04.677 | INFO | "gpu" : " 17 GB", | 20260508 09:22:04.677 | INFO | "huge" : "", | 20260508 09:22:04.677 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:04.677 | INFO | "malloc" : " 14 GB", | 20260508 09:22:04.677 | INFO | "pinned" : " 29 GB" | 20260508 09:22:04.677 | INFO | } | 20260508 09:22:04.677 | INFO | Perf report for the last 5.13863 seconds: | 20260508 09:22:04.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:04.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 09:22:04.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:22:04.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:04.677 | INFO | Uploaded 36GiB to GPU in 2.51151s (14.334GiB/s) | 20260508 09:22:04.678 | INFO | Downloaded 34GiB to GPU in 2.43048s (13.989GiB/s) | 20260508 09:22:09.817 | INFO | Analyzing chunk frames = [6993408,6993920) | 20260508 09:22:09.817 | INFO | MemInfo: { | 20260508 09:22:09.817 | INFO | "gpu" : " 17 GB", | 20260508 09:22:09.817 | INFO | "huge" : "", | 20260508 09:22:09.817 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:09.817 | INFO | "malloc" : " 14 GB", | 20260508 09:22:09.817 | INFO | "pinned" : " 29 GB" | 20260508 09:22:09.817 | INFO | } | 20260508 09:22:09.818 | INFO | Perf report for the last 5.1405 seconds: | 20260508 09:22:09.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:09.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 09:22:09.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 09:22:09.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:09.818 | INFO | Uploaded 36GiB to GPU in 2.51807s (14.2967GiB/s) | 20260508 09:22:09.818 | INFO | Downloaded 34GiB to GPU in 2.41753s (14.064GiB/s) | 20260508 09:22:14.961 | INFO | Analyzing chunk frames = [6993920,6994432) | 20260508 09:22:14.961 | INFO | MemInfo: { | 20260508 09:22:14.961 | INFO | "gpu" : " 17 GB", | 20260508 09:22:14.961 | INFO | "huge" : "", | 20260508 09:22:14.961 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:14.961 | INFO | "malloc" : " 14 GB", | 20260508 09:22:14.961 | INFO | "pinned" : " 29 GB" | 20260508 09:22:14.961 | INFO | } | 20260508 09:22:14.961 | INFO | Perf report for the last 5.1437 seconds: | 20260508 09:22:14.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:14.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.962 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 09:22:14.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.962 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:22:14.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:14.962 | INFO | Uploaded 36GiB to GPU in 2.30476s (15.6198GiB/s) | 20260508 09:22:14.962 | INFO | Downloaded 34GiB to GPU in 2.49229s (13.6421GiB/s) | 20260508 09:22:20.103 | INFO | Analyzing chunk frames = [6994432,6994944) | 20260508 09:22:20.103 | INFO | MemInfo: { | 20260508 09:22:20.103 | INFO | "gpu" : " 17 GB", | 20260508 09:22:20.103 | INFO | "huge" : "", | 20260508 09:22:20.103 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:20.103 | INFO | "malloc" : " 14 GB", | 20260508 09:22:20.103 | INFO | "pinned" : " 29 GB" | 20260508 09:22:20.103 | INFO | } | 20260508 09:22:20.103 | INFO | Perf report for the last 5.14157 seconds: | 20260508 09:22:20.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:20.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:22:20.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:22:20.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:20.103 | INFO | Uploaded 36GiB to GPU in 2.40463s (14.9711GiB/s) | 20260508 09:22:20.103 | INFO | Downloaded 34GiB to GPU in 2.43393s (13.9692GiB/s) | 20260508 09:22:25.237 | INFO | Analyzing chunk frames = [6994944,6995456) | 20260508 09:22:25.237 | INFO | MemInfo: { | 20260508 09:22:25.237 | INFO | "gpu" : " 17 GB", | 20260508 09:22:25.237 | INFO | "huge" : "", | 20260508 09:22:25.237 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:25.238 | INFO | "malloc" : " 14 GB", | 20260508 09:22:25.238 | INFO | "pinned" : " 29 GB" | 20260508 09:22:25.238 | INFO | } | 20260508 09:22:25.238 | INFO | Perf report for the last 5.13469 seconds: | 20260508 09:22:25.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:25.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:22:25.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:22:25.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:25.238 | INFO | Uploaded 36GiB to GPU in 2.44946s (14.6971GiB/s) | 20260508 09:22:25.238 | INFO | Downloaded 34GiB to GPU in 2.43529s (13.9614GiB/s) | 20260508 09:22:30.374 | INFO | Analyzing chunk frames = [6995456,6995968) | 20260508 09:22:30.374 | INFO | MemInfo: { | 20260508 09:22:30.374 | INFO | "gpu" : " 17 GB", | 20260508 09:22:30.374 | INFO | "huge" : "", | 20260508 09:22:30.374 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:30.374 | INFO | "malloc" : " 14 GB", | 20260508 09:22:30.374 | INFO | "pinned" : " 29 GB" | 20260508 09:22:30.374 | INFO | } | 20260508 09:22:30.374 | INFO | Perf report for the last 5.1361 seconds: | 20260508 09:22:30.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:30.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 09:22:30.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:22:30.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:30.374 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:22:30.374 | INFO | Total mem usage: 16.7 GiB | 20260508 09:22:30.374 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:22:30.374 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:22:30.374 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:22:30.374 | INFO | High Water Breakdown: | 20260508 09:22:30.374 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:22:30.374 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:22:30.374 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:22:30.375 | INFO | ------------------------------------------ | 20260508 09:22:30.374 | INFO | Uploaded 36GiB to GPU in 2.44006s (14.7538GiB/s) | 20260508 09:22:30.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:22:30.375 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:22:30.375 | INFO | Downloaded 34GiB to GPU in 2.43583s (13.9583GiB/s) | 20260508 09:22:30.375 | INFO | Total mem usage: 26.5 GiB | 20260508 09:22:30.375 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:22:30.375 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:22:30.375 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:22:30.375 | INFO | High Water Breakdown: | 20260508 09:22:30.375 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:22:30.375 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:22:30.375 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:22:30.375 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:22:30.375 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:22:30.375 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:22:30.375 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:22:30.375 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:22:30.375 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:22:30.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:22:30.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:22:30.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:22:30.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:22:30.375 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:22:30.375 | INFO | ------------------------------------------ | 20260508 09:22:30.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:22:30.375 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:22:30.375 | INFO | Total mem usage: 27.8 GiB | 20260508 09:22:30.375 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:22:30.375 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:22:30.375 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:22:30.375 | INFO | High Water Breakdown: | 20260508 09:22:30.375 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:22:30.375 | INFO | ------------------------------------------ | 20260508 09:22:30.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:22:30.375 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:22:30.375 | INFO | Total mem usage: 15.7 GiB | 20260508 09:22:30.375 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:22:30.375 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:22:30.375 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:22:30.375 | INFO | High Water Breakdown: | 20260508 09:22:30.375 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:22:30.375 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:22:30.375 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:22:30.375 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:22:30.375 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:22:30.375 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:22:30.375 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:22:30.375 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:22:30.375 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:22:30.375 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:22:30.375 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:22:30.375 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:22:30.375 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:22:30.375 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:22:30.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:22:30.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:22:30.375 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:22:30.375 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:22:30.375 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:22:30.375 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:22:30.375 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:22:30.375 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:22:30.375 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:22:30.375 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:22:30.375 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:22:30.375 | INFO | ------------------------------------------ | 20260508 09:22:30.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:22:30.375 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:22:30.375 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:22:30.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:22:30.375 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 5.99e-01 | 20260508 09:22:30.375 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.07e-02 / 4.60e-03 | 20260508 09:22:30.376 | INFO | PerfReport: 12.21%, Baseline -- 4.00e+00 / 4.46e+00, 5.96e-01 / 2.00e-01 | 20260508 09:22:30.376 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.45e+00, 2.11e-01 / 1.25e-01 | 20260508 09:22:30.376 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.54e+01, 3.72e+00 / 9.11e-01 | 20260508 09:22:30.376 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.84e+00 / 1.22e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.52e+00, 4.04e-01 / 2.47e-01 | 20260508 09:22:30.376 | INFO | PerfReport: 3.04%, Metrics -- 9.96e-01 / 1.12e+00, 2.48e-01 / 1.00e-01 | 20260508 09:22:30.376 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.29e-01, 4.86e-02 / 2.69e-02 | 20260508 09:22:30.376 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:22:30.376 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:22:30.376 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:22:30.376 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 24.30%, Upload -- 2.56e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 24.52%, Download -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 31.09%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 2.26%, LockUpload -- 2.38e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 2.45%, LockDownload -- 2.59e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 15.37%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:22:30.376 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 09:22:30.472 | INFO | Starting a round of Baz IO | 20260508 09:22:33.887 | INFO | Populating RTMetrics report | 20260508 09:22:33.888 | INFO | RTMetrics report populated | 20260508 09:22:33.888 | INFO | Opening tmp RTMetrics json file | 20260508 09:22:33.897 | INFO | Serializing RTMetrics to json | 20260508 09:22:33.898 | INFO | Serialization done, swapping tmp file now... | 20260508 09:22:33.899 | INFO | RTMetrics done | 20260508 09:22:33.955 | INFO | Finished a round of Baz IO: Wrote 1.481 GiB in 3.483 seconds | 20260508 09:22:33.955 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.69% | 20260508 09:22:33.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255791% / 3.97364e-06% / 5.16574e-05% | 20260508 09:22:33.955 | INFO | Baz IO overall: | 20260508 09:22:33.955 | INFO | BytesWritten : 3054272246455 | 20260508 09:22:33.955 | INFO | NumEvents : 1469357164621 | 20260508 09:22:35.515 | INFO | Analyzing chunk frames = [6995968,6996480) | 20260508 09:22:35.515 | INFO | MemInfo: { | 20260508 09:22:35.515 | INFO | "gpu" : " 17 GB", | 20260508 09:22:35.515 | INFO | "huge" : "", | 20260508 09:22:35.515 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:35.515 | INFO | "malloc" : " 14 GB", | 20260508 09:22:35.515 | INFO | "pinned" : " 29 GB" | 20260508 09:22:35.515 | INFO | } | 20260508 09:22:35.515 | INFO | Perf report for the last 5.14157 seconds: | 20260508 09:22:35.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:35.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:35.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:35.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:35.516 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 09:22:35.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:35.516 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:22:35.516 | INFO | PRE_HQ : 65.2%, 1.6, 54.5ms, 0 | 20260508 09:22:35.516 | INFO | BAZWRITER : 67.0%, 2.5, 87.1ms, 0 | 20260508 09:22:35.516 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:22:35.516 | INFO | Uploaded 36GiB to GPU in 2.60976s (13.7944GiB/s) | 20260508 09:22:35.516 | INFO | Downloaded 38GiB to GPU in 2.70931s (14.0257GiB/s) | 20260508 09:22:40.655 | INFO | Analyzing chunk frames = [6996480,6996992) | 20260508 09:22:40.655 | INFO | MemInfo: { | 20260508 09:22:40.655 | INFO | "gpu" : " 17 GB", | 20260508 09:22:40.655 | INFO | "huge" : "", | 20260508 09:22:40.655 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:40.655 | INFO | "malloc" : " 14 GB", | 20260508 09:22:40.655 | INFO | "pinned" : " 29 GB" | 20260508 09:22:40.655 | INFO | } | 20260508 09:22:40.655 | INFO | Perf report for the last 5.13979 seconds: | 20260508 09:22:40.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:40.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.655 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:22:40.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.656 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:22:40.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:40.656 | INFO | Uploaded 36GiB to GPU in 2.49944s (14.4032GiB/s) | 20260508 09:22:40.656 | INFO | Downloaded 34GiB to GPU in 2.46145s (13.813GiB/s) | 20260508 09:22:45.794 | INFO | Analyzing chunk frames = [6996992,6997504) | 20260508 09:22:45.794 | INFO | MemInfo: { | 20260508 09:22:45.794 | INFO | "gpu" : " 17 GB", | 20260508 09:22:45.794 | INFO | "huge" : "", | 20260508 09:22:45.794 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:45.794 | INFO | "malloc" : " 14 GB", | 20260508 09:22:45.794 | INFO | "pinned" : " 29 GB" | 20260508 09:22:45.794 | INFO | } | 20260508 09:22:45.794 | INFO | Perf report for the last 5.13888 seconds: | 20260508 09:22:45.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:45.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.794 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:22:45.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.794 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 09:22:45.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:45.795 | INFO | Uploaded 36GiB to GPU in 2.4658s (14.5997GiB/s) | 20260508 09:22:45.795 | INFO | Downloaded 34GiB to GPU in 2.43776s (13.9473GiB/s) | 20260508 09:22:50.933 | INFO | Analyzing chunk frames = [6997504,6998016) | 20260508 09:22:50.933 | INFO | MemInfo: { | 20260508 09:22:50.933 | INFO | "gpu" : " 17 GB", | 20260508 09:22:50.933 | INFO | "huge" : "", | 20260508 09:22:50.933 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:50.933 | INFO | "malloc" : " 14 GB", | 20260508 09:22:50.933 | INFO | "pinned" : " 29 GB" | 20260508 09:22:50.933 | INFO | } | 20260508 09:22:50.933 | INFO | Perf report for the last 5.13914 seconds: | 20260508 09:22:50.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:50.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 09:22:50.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:22:50.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:50.933 | INFO | Uploaded 36GiB to GPU in 2.48035s (14.5141GiB/s) | 20260508 09:22:50.934 | INFO | Downloaded 34GiB to GPU in 2.4128s (14.0915GiB/s) | 20260508 09:22:56.073 | INFO | Analyzing chunk frames = [6998016,6998528) | 20260508 09:22:56.073 | INFO | MemInfo: { | 20260508 09:22:56.073 | INFO | "gpu" : " 17 GB", | 20260508 09:22:56.073 | INFO | "huge" : "", | 20260508 09:22:56.073 | INFO | "huge_shared" : " 29 GB", | 20260508 09:22:56.073 | INFO | "malloc" : " 14 GB", | 20260508 09:22:56.073 | INFO | "pinned" : " 29 GB" | 20260508 09:22:56.073 | INFO | } | 20260508 09:22:56.073 | INFO | Perf report for the last 5.1396 seconds: | 20260508 09:22:56.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:22:56.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.073 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 09:22:56.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.073 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:22:56.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:22:56.074 | INFO | Uploaded 36GiB to GPU in 2.30069s (15.6475GiB/s) | 20260508 09:22:56.074 | INFO | Downloaded 34GiB to GPU in 2.49886s (13.6062GiB/s) | 20260508 09:23:01.214 | INFO | Analyzing chunk frames = [6998528,6999040) | 20260508 09:23:01.214 | INFO | MemInfo: { | 20260508 09:23:01.214 | INFO | "gpu" : " 17 GB", | 20260508 09:23:01.214 | INFO | "huge" : "", | 20260508 09:23:01.214 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:01.214 | INFO | "malloc" : " 14 GB", | 20260508 09:23:01.215 | INFO | "pinned" : " 29 GB" | 20260508 09:23:01.215 | INFO | } | 20260508 09:23:01.215 | INFO | Perf report for the last 5.14187 seconds: | 20260508 09:23:01.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:01.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 09:23:01.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:23:01.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:01.215 | INFO | Uploaded 36GiB to GPU in 2.3569s (15.2743GiB/s) | 20260508 09:23:01.215 | INFO | Downloaded 34GiB to GPU in 2.45169s (13.868GiB/s) | 20260508 09:23:06.348 | INFO | Analyzing chunk frames = [6999040,6999552) | 20260508 09:23:06.348 | INFO | MemInfo: { | 20260508 09:23:06.348 | INFO | "gpu" : " 17 GB", | 20260508 09:23:06.348 | INFO | "huge" : "", | 20260508 09:23:06.348 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:06.348 | INFO | "malloc" : " 14 GB", | 20260508 09:23:06.349 | INFO | "pinned" : " 29 GB" | 20260508 09:23:06.349 | INFO | } | 20260508 09:23:06.349 | INFO | Perf report for the last 5.13385 seconds: | 20260508 09:23:06.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:06.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 09:23:06.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:23:06.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:06.349 | INFO | Uploaded 36GiB to GPU in 2.38819s (15.0742GiB/s) | 20260508 09:23:06.349 | INFO | Downloaded 34GiB to GPU in 2.45484s (13.8502GiB/s) | 20260508 09:23:11.492 | INFO | Analyzing chunk frames = [6999552,7000064) | 20260508 09:23:11.492 | INFO | MemInfo: { | 20260508 09:23:11.492 | INFO | "gpu" : " 17 GB", | 20260508 09:23:11.492 | INFO | "huge" : "", | 20260508 09:23:11.492 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:11.492 | INFO | "malloc" : " 14 GB", | 20260508 09:23:11.493 | INFO | "pinned" : " 29 GB" | 20260508 09:23:11.493 | INFO | } | 20260508 09:23:11.493 | INFO | Perf report for the last 5.144 seconds: | 20260508 09:23:11.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:11.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 09:23:11.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:23:11.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:11.493 | INFO | Uploaded 36GiB to GPU in 2.38946s (15.0661GiB/s) | 20260508 09:23:11.493 | INFO | Downloaded 34GiB to GPU in 2.45462s (13.8514GiB/s) | 20260508 09:23:11.590 | INFO | Starting a round of Baz IO | 20260508 09:23:15.013 | INFO | Populating RTMetrics report | 20260508 09:23:15.013 | INFO | RTMetrics report populated | 20260508 09:23:15.013 | INFO | Opening tmp RTMetrics json file | 20260508 09:23:15.024 | INFO | Serializing RTMetrics to json | 20260508 09:23:15.025 | INFO | Serialization done, swapping tmp file now... | 20260508 09:23:15.026 | INFO | RTMetrics done | 20260508 09:23:15.080 | INFO | Finished a round of Baz IO: Wrote 1.492 GiB in 3.489 seconds | 20260508 09:23:15.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.69% | 20260508 09:23:15.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259125% / 3.97364e-06% / 5.5631e-05% | 20260508 09:23:15.080 | INFO | Baz IO overall: | 20260508 09:23:15.080 | INFO | BytesWritten : 3055874332391 | 20260508 09:23:15.080 | INFO | NumEvents : 1470113155931 | 20260508 09:23:16.624 | INFO | Analyzing chunk frames = [7000064,7000576) | 20260508 09:23:16.624 | INFO | MemInfo: { | 20260508 09:23:16.624 | INFO | "gpu" : " 17 GB", | 20260508 09:23:16.624 | INFO | "huge" : "", | 20260508 09:23:16.624 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:16.624 | INFO | "malloc" : " 14 GB", | 20260508 09:23:16.624 | INFO | "pinned" : " 29 GB" | 20260508 09:23:16.624 | INFO | } | 20260508 09:23:16.624 | INFO | Perf report for the last 5.13109 seconds: | 20260508 09:23:16.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:16.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:16.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:16.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:16.624 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 09:23:16.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:16.624 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:23:16.624 | INFO | PRE_HQ : 65.8%, 1.6, 54.8ms, 0 | 20260508 09:23:16.624 | INFO | BAZWRITER : 67.4%, 2.5, 88.1ms, 0 | 20260508 09:23:16.624 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:23:16.624 | INFO | Uploaded 36GiB to GPU in 2.59653s (13.8647GiB/s) | 20260508 09:23:16.624 | INFO | Downloaded 38GiB to GPU in 2.71174s (14.0131GiB/s) | 20260508 09:23:21.767 | INFO | Analyzing chunk frames = [7000576,7001088) | 20260508 09:23:21.767 | INFO | MemInfo: { | 20260508 09:23:21.767 | INFO | "gpu" : " 17 GB", | 20260508 09:23:21.768 | INFO | "huge" : "", | 20260508 09:23:21.768 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:21.768 | INFO | "malloc" : " 14 GB", | 20260508 09:23:21.768 | INFO | "pinned" : " 29 GB" | 20260508 09:23:21.768 | INFO | } | 20260508 09:23:21.768 | INFO | Perf report for the last 5.14404 seconds: | 20260508 09:23:21.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:21.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 09:23:21.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260508 09:23:21.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:21.768 | INFO | Uploaded 36GiB to GPU in 2.51248s (14.3285GiB/s) | 20260508 09:23:21.768 | INFO | Downloaded 34GiB to GPU in 2.43863s (13.9423GiB/s) | 20260508 09:23:26.911 | INFO | Analyzing chunk frames = [7001088,7001600) | 20260508 09:23:26.911 | INFO | MemInfo: { | 20260508 09:23:26.911 | INFO | "gpu" : " 17 GB", | 20260508 09:23:26.911 | INFO | "huge" : "", | 20260508 09:23:26.911 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:26.911 | INFO | "malloc" : " 14 GB", | 20260508 09:23:26.911 | INFO | "pinned" : " 29 GB" | 20260508 09:23:26.911 | INFO | } | 20260508 09:23:26.912 | INFO | Perf report for the last 5.14371 seconds: | 20260508 09:23:26.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:26.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 09:23:26.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 09:23:26.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:26.912 | INFO | Uploaded 36GiB to GPU in 2.47839s (14.5255GiB/s) | 20260508 09:23:26.912 | INFO | Downloaded 34GiB to GPU in 2.42932s (13.9957GiB/s) | 20260508 09:23:32.051 | INFO | Analyzing chunk frames = [7001600,7002112) | 20260508 09:23:32.051 | INFO | MemInfo: { | 20260508 09:23:32.051 | INFO | "gpu" : " 17 GB", | 20260508 09:23:32.051 | INFO | "huge" : "", | 20260508 09:23:32.051 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:32.051 | INFO | "malloc" : " 14 GB", | 20260508 09:23:32.051 | INFO | "pinned" : " 29 GB" | 20260508 09:23:32.051 | INFO | } | 20260508 09:23:32.052 | INFO | Perf report for the last 5.14003 seconds: | 20260508 09:23:32.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:32.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 09:23:32.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 09:23:32.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:32.052 | INFO | Uploaded 36GiB to GPU in 2.46717s (14.5916GiB/s) | 20260508 09:23:32.052 | INFO | Downloaded 34GiB to GPU in 2.41475s (14.0801GiB/s) | 20260508 09:23:37.182 | INFO | Analyzing chunk frames = [7002112,7002624) | 20260508 09:23:37.182 | INFO | MemInfo: { | 20260508 09:23:37.182 | INFO | "gpu" : " 17 GB", | 20260508 09:23:37.182 | INFO | "huge" : "", | 20260508 09:23:37.182 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:37.182 | INFO | "malloc" : " 14 GB", | 20260508 09:23:37.182 | INFO | "pinned" : " 29 GB" | 20260508 09:23:37.182 | INFO | } | 20260508 09:23:37.182 | INFO | Perf report for the last 5.13053 seconds: | 20260508 09:23:37.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:37.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:23:37.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 09:23:37.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:37.182 | INFO | Uploaded 36GiB to GPU in 2.41843s (14.8857GiB/s) | 20260508 09:23:37.182 | INFO | Downloaded 34GiB to GPU in 2.51976s (13.4933GiB/s) | 20260508 09:23:42.324 | INFO | Analyzing chunk frames = [7002624,7003136) | 20260508 09:23:42.324 | INFO | MemInfo: { | 20260508 09:23:42.324 | INFO | "gpu" : " 17 GB", | 20260508 09:23:42.324 | INFO | "huge" : "", | 20260508 09:23:42.324 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:42.324 | INFO | "malloc" : " 14 GB", | 20260508 09:23:42.324 | INFO | "pinned" : " 29 GB" | 20260508 09:23:42.324 | INFO | } | 20260508 09:23:42.324 | INFO | Perf report for the last 5.14207 seconds: | 20260508 09:23:42.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:42.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:23:42.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:23:42.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:42.324 | INFO | Uploaded 36GiB to GPU in 2.39834s (15.0104GiB/s) | 20260508 09:23:42.324 | INFO | Downloaded 34GiB to GPU in 2.41892s (14.0559GiB/s) | 20260508 09:23:47.466 | INFO | Analyzing chunk frames = [7003136,7003648) | 20260508 09:23:47.466 | INFO | MemInfo: { | 20260508 09:23:47.466 | INFO | "gpu" : " 17 GB", | 20260508 09:23:47.466 | INFO | "huge" : "", | 20260508 09:23:47.466 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:47.466 | INFO | "malloc" : " 14 GB", | 20260508 09:23:47.466 | INFO | "pinned" : " 29 GB" | 20260508 09:23:47.466 | INFO | } | 20260508 09:23:47.466 | INFO | Perf report for the last 5.1419 seconds: | 20260508 09:23:47.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:47.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.466 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:23:47.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.466 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:23:47.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:47.467 | INFO | Uploaded 36GiB to GPU in 2.4543s (14.6681GiB/s) | 20260508 09:23:47.467 | INFO | Downloaded 34GiB to GPU in 2.41665s (14.0691GiB/s) | 20260508 09:23:52.608 | INFO | Analyzing chunk frames = [7003648,7004160) | 20260508 09:23:52.608 | INFO | MemInfo: { | 20260508 09:23:52.608 | INFO | "gpu" : " 17 GB", | 20260508 09:23:52.608 | INFO | "huge" : "", | 20260508 09:23:52.608 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:52.608 | INFO | "malloc" : " 14 GB", | 20260508 09:23:52.608 | INFO | "pinned" : " 29 GB" | 20260508 09:23:52.608 | INFO | } | 20260508 09:23:52.608 | INFO | Perf report for the last 5.14173 seconds: | 20260508 09:23:52.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:52.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 09:23:52.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:23:52.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:52.608 | INFO | Uploaded 36GiB to GPU in 2.47089s (14.5696GiB/s) | 20260508 09:23:52.608 | INFO | Downloaded 34GiB to GPU in 2.45527s (13.8478GiB/s) | 20260508 09:23:52.608 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:23:52.608 | INFO | Total mem usage: 16.7 GiB | 20260508 09:23:52.608 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:23:52.608 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:23:52.608 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:23:52.608 | INFO | High Water Breakdown: | 20260508 09:23:52.608 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:23:52.608 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:23:52.608 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:23:52.608 | INFO | ------------------------------------------ | 20260508 09:23:52.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:23:52.608 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:23:52.608 | INFO | Total mem usage: 26.5 GiB | 20260508 09:23:52.608 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:23:52.608 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:23:52.608 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:23:52.608 | INFO | High Water Breakdown: | 20260508 09:23:52.608 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:23:52.608 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:23:52.608 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:23:52.608 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:23:52.608 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:23:52.609 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:23:52.609 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:23:52.609 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:23:52.609 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:23:52.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:23:52.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:23:52.609 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:23:52.609 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:23:52.609 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:23:52.609 | INFO | ------------------------------------------ | 20260508 09:23:52.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:23:52.609 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:23:52.609 | INFO | Total mem usage: 27.8 GiB | 20260508 09:23:52.609 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:23:52.609 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:23:52.609 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:23:52.609 | INFO | High Water Breakdown: | 20260508 09:23:52.609 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:23:52.609 | INFO | ------------------------------------------ | 20260508 09:23:52.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:23:52.609 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:23:52.609 | INFO | Total mem usage: 15.7 GiB | 20260508 09:23:52.609 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:23:52.609 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:23:52.609 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:23:52.609 | INFO | High Water Breakdown: | 20260508 09:23:52.609 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:23:52.609 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:23:52.609 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:23:52.609 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:23:52.609 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:23:52.609 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:23:52.609 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:23:52.609 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:23:52.609 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:23:52.609 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:23:52.609 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:23:52.609 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:23:52.609 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:23:52.609 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:23:52.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:23:52.609 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:23:52.609 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:23:52.609 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:23:52.609 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:23:52.609 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:23:52.609 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:23:52.609 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:23:52.609 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:23:52.609 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:23:52.609 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:23:52.609 | INFO | ------------------------------------------ | 20260508 09:23:52.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:23:52.609 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:23:52.609 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:23:52.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:23:52.609 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.40e+01 / 6.26e-01 | 20260508 09:23:52.609 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.06e-01, 1.16e-02 / 5.34e-03 | 20260508 09:23:52.609 | INFO | PerfReport: 12.21%, Baseline -- 4.00e+00 / 4.36e+00, 5.94e-01 / 2.16e-01 | 20260508 09:23:52.609 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.43e+00, 2.11e-01 / 1.17e-01 | 20260508 09:23:52.609 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.50e+01, 3.61e+00 / 1.31e+00 | 20260508 09:23:52.609 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.82e+00 / 1.27e+00 | 20260508 09:23:52.609 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.47e+00, 4.02e-01 / 2.60e-01 | 20260508 09:23:52.609 | INFO | PerfReport: 3.06%, Metrics -- 1.00e+00 / 1.13e+00, 2.48e-01 / 8.82e-02 | 20260508 09:23:52.610 | INFO | PerfReport: 0.94%, PulseSerializing -- 3.07e-01 / 3.36e-01, 4.84e-02 / 2.61e-02 | 20260508 09:23:52.610 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:23:52.610 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:23:52.610 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:23:52.610 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 24.28%, Upload -- 2.56e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 24.53%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 31.09%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 2.54%, LockDownload -- 2.67e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:23:52.610 | INFO | PerfReport: 0.01%, Misc -- 1.51e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260508 09:23:52.705 | INFO | Starting a round of Baz IO | 20260508 09:23:56.112 | INFO | Populating RTMetrics report | 20260508 09:23:56.112 | INFO | RTMetrics report populated | 20260508 09:23:56.112 | INFO | Opening tmp RTMetrics json file | 20260508 09:23:56.124 | INFO | Serializing RTMetrics to json | 20260508 09:23:56.125 | INFO | Serialization done, swapping tmp file now... | 20260508 09:23:56.125 | INFO | RTMetrics done | 20260508 09:23:56.187 | INFO | Finished a round of Baz IO: Wrote 1.491 GiB in 3.481 seconds | 20260508 09:23:56.187 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.69% | 20260508 09:23:56.187 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258883% / 3.97364e-06% / 6.35783e-05% | 20260508 09:23:56.187 | INFO | Baz IO overall: | 20260508 09:23:56.187 | INFO | BytesWritten : 3057474927734 | 20260508 09:23:56.187 | INFO | NumEvents : 1470868099948 | 20260508 09:23:57.746 | INFO | Analyzing chunk frames = [7004160,7004672) | 20260508 09:23:57.746 | INFO | MemInfo: { | 20260508 09:23:57.746 | INFO | "gpu" : " 17 GB", | 20260508 09:23:57.746 | INFO | "huge" : "", | 20260508 09:23:57.746 | INFO | "huge_shared" : " 29 GB", | 20260508 09:23:57.746 | INFO | "malloc" : " 14 GB", | 20260508 09:23:57.746 | INFO | "pinned" : " 29 GB" | 20260508 09:23:57.746 | INFO | } | 20260508 09:23:57.746 | INFO | Perf report for the last 5.13819 seconds: | 20260508 09:23:57.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:23:57.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:57.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:57.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:57.746 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 09:23:57.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:23:57.746 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:23:57.746 | INFO | PRE_HQ : 65.3%, 1.6, 54.6ms, 0 | 20260508 09:23:57.746 | INFO | BAZWRITER : 67.1%, 2.6, 88.4ms, 0 | 20260508 09:23:57.746 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:23:57.747 | INFO | Uploaded 36GiB to GPU in 2.61217s (13.7816GiB/s) | 20260508 09:23:57.747 | INFO | Downloaded 38GiB to GPU in 2.70218s (14.0627GiB/s) | 20260508 09:24:02.880 | INFO | Analyzing chunk frames = [7004672,7005184) | 20260508 09:24:02.880 | INFO | MemInfo: { | 20260508 09:24:02.880 | INFO | "gpu" : " 17 GB", | 20260508 09:24:02.880 | INFO | "huge" : "", | 20260508 09:24:02.880 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:02.880 | INFO | "malloc" : " 14 GB", | 20260508 09:24:02.880 | INFO | "pinned" : " 29 GB" | 20260508 09:24:02.881 | INFO | } | 20260508 09:24:02.881 | INFO | Perf report for the last 5.13461 seconds: | 20260508 09:24:02.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:02.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 09:24:02.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:24:02.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:02.881 | INFO | Uploaded 36GiB to GPU in 2.51804s (14.2968GiB/s) | 20260508 09:24:02.881 | INFO | Downloaded 34GiB to GPU in 2.45864s (13.8288GiB/s) | 20260508 09:24:08.024 | INFO | Analyzing chunk frames = [7005184,7005696) | 20260508 09:24:08.024 | INFO | MemInfo: { | 20260508 09:24:08.024 | INFO | "gpu" : " 17 GB", | 20260508 09:24:08.024 | INFO | "huge" : "", | 20260508 09:24:08.024 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:08.024 | INFO | "malloc" : " 14 GB", | 20260508 09:24:08.024 | INFO | "pinned" : " 29 GB" | 20260508 09:24:08.024 | INFO | } | 20260508 09:24:08.024 | INFO | Perf report for the last 5.14355 seconds: | 20260508 09:24:08.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:08.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.024 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 09:24:08.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.024 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:24:08.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:08.025 | INFO | Uploaded 36GiB to GPU in 2.47917s (14.521GiB/s) | 20260508 09:24:08.025 | INFO | Downloaded 34GiB to GPU in 2.4341s (13.9682GiB/s) | 20260508 09:24:13.161 | INFO | Analyzing chunk frames = [7005696,7006208) | 20260508 09:24:13.161 | INFO | MemInfo: { | 20260508 09:24:13.161 | INFO | "gpu" : " 17 GB", | 20260508 09:24:13.161 | INFO | "huge" : "", | 20260508 09:24:13.161 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:13.161 | INFO | "malloc" : " 14 GB", | 20260508 09:24:13.161 | INFO | "pinned" : " 29 GB" | 20260508 09:24:13.161 | INFO | } | 20260508 09:24:13.161 | INFO | Perf report for the last 5.13694 seconds: | 20260508 09:24:13.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:13.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.161 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:24:13.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.162 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:24:13.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:13.162 | INFO | Uploaded 36GiB to GPU in 2.49415s (14.4338GiB/s) | 20260508 09:24:13.162 | INFO | Downloaded 34GiB to GPU in 2.42869s (13.9993GiB/s) | 20260508 09:24:18.301 | INFO | Analyzing chunk frames = [7006208,7006720) | 20260508 09:24:18.301 | INFO | MemInfo: { | 20260508 09:24:18.301 | INFO | "gpu" : " 17 GB", | 20260508 09:24:18.301 | INFO | "huge" : "", | 20260508 09:24:18.301 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:18.301 | INFO | "malloc" : " 14 GB", | 20260508 09:24:18.301 | INFO | "pinned" : " 29 GB" | 20260508 09:24:18.301 | INFO | } | 20260508 09:24:18.302 | INFO | Perf report for the last 5.14054 seconds: | 20260508 09:24:18.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:18.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 09:24:18.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 09:24:18.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:18.302 | INFO | Uploaded 36GiB to GPU in 2.27259s (15.841GiB/s) | 20260508 09:24:18.302 | INFO | Downloaded 34GiB to GPU in 2.49485s (13.6281GiB/s) | 20260508 09:24:23.438 | INFO | Analyzing chunk frames = [7006720,7007232) | 20260508 09:24:23.438 | INFO | MemInfo: { | 20260508 09:24:23.438 | INFO | "gpu" : " 17 GB", | 20260508 09:24:23.438 | INFO | "huge" : "", | 20260508 09:24:23.438 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:23.438 | INFO | "malloc" : " 14 GB", | 20260508 09:24:23.438 | INFO | "pinned" : " 29 GB" | 20260508 09:24:23.438 | INFO | } | 20260508 09:24:23.438 | INFO | Perf report for the last 5.13608 seconds: | 20260508 09:24:23.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:23.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 09:24:23.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:24:23.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:23.438 | INFO | Uploaded 36GiB to GPU in 2.37012s (15.1891GiB/s) | 20260508 09:24:23.438 | INFO | Downloaded 34GiB to GPU in 2.45068s (13.8737GiB/s) | 20260508 09:24:28.584 | INFO | Analyzing chunk frames = [7007232,7007744) | 20260508 09:24:28.584 | INFO | MemInfo: { | 20260508 09:24:28.584 | INFO | "gpu" : " 17 GB", | 20260508 09:24:28.584 | INFO | "huge" : "", | 20260508 09:24:28.584 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:28.585 | INFO | "malloc" : " 14 GB", | 20260508 09:24:28.585 | INFO | "pinned" : " 29 GB" | 20260508 09:24:28.585 | INFO | } | 20260508 09:24:28.585 | INFO | Perf report for the last 5.1468 seconds: | 20260508 09:24:28.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:28.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 09:24:28.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:24:28.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:28.585 | INFO | Uploaded 36GiB to GPU in 2.40691s (14.9569GiB/s) | 20260508 09:24:28.585 | INFO | Downloaded 34GiB to GPU in 2.44759s (13.8912GiB/s) | 20260508 09:24:33.721 | INFO | Analyzing chunk frames = [7007744,7008256) | 20260508 09:24:33.721 | INFO | MemInfo: { | 20260508 09:24:33.721 | INFO | "gpu" : " 17 GB", | 20260508 09:24:33.721 | INFO | "huge" : "", | 20260508 09:24:33.721 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:33.721 | INFO | "malloc" : " 14 GB", | 20260508 09:24:33.721 | INFO | "pinned" : " 29 GB" | 20260508 09:24:33.721 | INFO | } | 20260508 09:24:33.721 | INFO | Perf report for the last 5.13676 seconds: | 20260508 09:24:33.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:33.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.721 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 09:24:33.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.722 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:24:33.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:33.722 | INFO | Uploaded 36GiB to GPU in 2.41727s (14.8928GiB/s) | 20260508 09:24:33.722 | INFO | Downloaded 34GiB to GPU in 2.45702s (13.8379GiB/s) | 20260508 09:24:33.819 | INFO | Starting a round of Baz IO | 20260508 09:24:37.230 | INFO | Populating RTMetrics report | 20260508 09:24:37.230 | INFO | RTMetrics report populated | 20260508 09:24:37.230 | INFO | Opening tmp RTMetrics json file | 20260508 09:24:37.239 | INFO | Serializing RTMetrics to json | 20260508 09:24:37.240 | INFO | Serialization done, swapping tmp file now... | 20260508 09:24:37.241 | INFO | RTMetrics done | 20260508 09:24:37.300 | INFO | Finished a round of Baz IO: Wrote 1.489 GiB in 3.481 seconds | 20260508 09:24:37.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.69% | 20260508 09:24:37.301 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259968% / 3.97364e-06% / 3.57628e-05% | 20260508 09:24:37.301 | INFO | Baz IO overall: | 20260508 09:24:37.301 | INFO | BytesWritten : 3059074123357 | 20260508 09:24:37.301 | INFO | NumEvents : 1471623849059 | 20260508 09:24:38.859 | INFO | Analyzing chunk frames = [7008256,7008768) | 20260508 09:24:38.859 | INFO | MemInfo: { | 20260508 09:24:38.859 | INFO | "gpu" : " 17 GB", | 20260508 09:24:38.859 | INFO | "huge" : "", | 20260508 09:24:38.859 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:38.859 | INFO | "malloc" : " 14 GB", | 20260508 09:24:38.859 | INFO | "pinned" : " 29 GB" | 20260508 09:24:38.859 | INFO | } | 20260508 09:24:38.859 | INFO | Perf report for the last 5.1374 seconds: | 20260508 09:24:38.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:38.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:38.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:38.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:38.859 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:24:38.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:38.859 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:24:38.859 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 09:24:38.859 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 09:24:38.859 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:24:38.860 | INFO | Uploaded 36GiB to GPU in 2.58032s (13.9517GiB/s) | 20260508 09:24:38.860 | INFO | Downloaded 38GiB to GPU in 2.71387s (14.0021GiB/s) | 20260508 09:24:43.998 | INFO | Analyzing chunk frames = [7008768,7009280) | 20260508 09:24:43.998 | INFO | MemInfo: { | 20260508 09:24:43.998 | INFO | "gpu" : " 17 GB", | 20260508 09:24:43.998 | INFO | "huge" : "", | 20260508 09:24:43.998 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:43.998 | INFO | "malloc" : " 14 GB", | 20260508 09:24:43.998 | INFO | "pinned" : " 29 GB" | 20260508 09:24:43.998 | INFO | } | 20260508 09:24:43.999 | INFO | Perf report for the last 5.13974 seconds: | 20260508 09:24:43.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:43.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:24:43.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 09:24:43.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:43.999 | INFO | Uploaded 36GiB to GPU in 2.48682s (14.4763GiB/s) | 20260508 09:24:43.999 | INFO | Downloaded 34GiB to GPU in 2.47077s (13.7609GiB/s) | 20260508 09:24:49.132 | INFO | Analyzing chunk frames = [7009280,7009792) | 20260508 09:24:49.132 | INFO | MemInfo: { | 20260508 09:24:49.132 | INFO | "gpu" : " 17 GB", | 20260508 09:24:49.132 | INFO | "huge" : "", | 20260508 09:24:49.132 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:49.132 | INFO | "malloc" : " 14 GB", | 20260508 09:24:49.132 | INFO | "pinned" : " 29 GB" | 20260508 09:24:49.132 | INFO | } | 20260508 09:24:49.132 | INFO | Perf report for the last 5.13371 seconds: | 20260508 09:24:49.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:49.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.132 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 09:24:49.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.132 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:24:49.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:49.133 | INFO | Uploaded 36GiB to GPU in 2.4263s (14.8374GiB/s) | 20260508 09:24:49.133 | INFO | Downloaded 34GiB to GPU in 2.44274s (13.9188GiB/s) | 20260508 09:24:54.278 | INFO | Analyzing chunk frames = [7009792,7010304) | 20260508 09:24:54.278 | INFO | MemInfo: { | 20260508 09:24:54.278 | INFO | "gpu" : " 17 GB", | 20260508 09:24:54.278 | INFO | "huge" : "", | 20260508 09:24:54.278 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:54.278 | INFO | "malloc" : " 14 GB", | 20260508 09:24:54.279 | INFO | "pinned" : " 29 GB" | 20260508 09:24:54.279 | INFO | } | 20260508 09:24:54.279 | INFO | Perf report for the last 5.14626 seconds: | 20260508 09:24:54.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:54.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 09:24:54.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:24:54.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:54.279 | INFO | Uploaded 36GiB to GPU in 2.46871s (14.5825GiB/s) | 20260508 09:24:54.279 | INFO | Downloaded 34GiB to GPU in 2.4488s (13.8844GiB/s) | 20260508 09:24:59.416 | INFO | Analyzing chunk frames = [7010304,7010816) | 20260508 09:24:59.416 | INFO | MemInfo: { | 20260508 09:24:59.416 | INFO | "gpu" : " 17 GB", | 20260508 09:24:59.416 | INFO | "huge" : "", | 20260508 09:24:59.416 | INFO | "huge_shared" : " 29 GB", | 20260508 09:24:59.416 | INFO | "malloc" : " 14 GB", | 20260508 09:24:59.417 | INFO | "pinned" : " 29 GB" | 20260508 09:24:59.417 | INFO | } | 20260508 09:24:59.417 | INFO | Perf report for the last 5.13808 seconds: | 20260508 09:24:59.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:24:59.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:24:59.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 09:24:59.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:24:59.417 | INFO | Uploaded 36GiB to GPU in 2.4247s (14.8472GiB/s) | 20260508 09:24:59.417 | INFO | Downloaded 34GiB to GPU in 2.55138s (13.3261GiB/s) | 20260508 09:25:04.566 | INFO | Analyzing chunk frames = [7010816,7011328) | 20260508 09:25:04.566 | INFO | MemInfo: { | 20260508 09:25:04.566 | INFO | "gpu" : " 17 GB", | 20260508 09:25:04.566 | INFO | "huge" : "", | 20260508 09:25:04.566 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:04.566 | INFO | "malloc" : " 14 GB", | 20260508 09:25:04.566 | INFO | "pinned" : " 29 GB" | 20260508 09:25:04.566 | INFO | } | 20260508 09:25:04.566 | INFO | Perf report for the last 5.14933 seconds: | 20260508 09:25:04.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:04.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.566 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 09:25:04.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.566 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:25:04.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:04.567 | INFO | Uploaded 36GiB to GPU in 2.3215s (15.5072GiB/s) | 20260508 09:25:04.567 | INFO | Downloaded 34GiB to GPU in 2.44605s (13.9GiB/s) | 20260508 09:25:09.697 | INFO | Analyzing chunk frames = [7011328,7011840) | 20260508 09:25:09.697 | INFO | MemInfo: { | 20260508 09:25:09.697 | INFO | "gpu" : " 17 GB", | 20260508 09:25:09.697 | INFO | "huge" : "", | 20260508 09:25:09.697 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:09.697 | INFO | "malloc" : " 14 GB", | 20260508 09:25:09.697 | INFO | "pinned" : " 29 GB" | 20260508 09:25:09.697 | INFO | } | 20260508 09:25:09.697 | INFO | Perf report for the last 5.13094 seconds: | 20260508 09:25:09.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:09.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:25:09.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:25:09.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:09.697 | INFO | Uploaded 36GiB to GPU in 2.40436s (14.9728GiB/s) | 20260508 09:25:09.697 | INFO | Downloaded 34GiB to GPU in 2.46583s (13.7884GiB/s) | 20260508 09:25:14.834 | INFO | Analyzing chunk frames = [7011840,7012352) | 20260508 09:25:14.834 | INFO | MemInfo: { | 20260508 09:25:14.834 | INFO | "gpu" : " 17 GB", | 20260508 09:25:14.834 | INFO | "huge" : "", | 20260508 09:25:14.834 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:14.834 | INFO | "malloc" : " 14 GB", | 20260508 09:25:14.834 | INFO | "pinned" : " 29 GB" | 20260508 09:25:14.834 | INFO | } | 20260508 09:25:14.835 | INFO | Perf report for the last 5.13761 seconds: | 20260508 09:25:14.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:14.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:25:14.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 09:25:14.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:14.835 | INFO | Uploaded 36GiB to GPU in 2.41323s (14.9178GiB/s) | 20260508 09:25:14.835 | INFO | Downloaded 34GiB to GPU in 2.45027s (13.876GiB/s) | 20260508 09:25:14.835 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:25:14.835 | INFO | Total mem usage: 16.7 GiB | 20260508 09:25:14.835 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:25:14.835 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:25:14.835 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:25:14.835 | INFO | High Water Breakdown: | 20260508 09:25:14.835 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:25:14.835 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:25:14.835 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:25:14.835 | INFO | ------------------------------------------ | 20260508 09:25:14.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:25:14.835 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:25:14.835 | INFO | Total mem usage: 26.5 GiB | 20260508 09:25:14.835 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:25:14.835 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:25:14.835 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:25:14.835 | INFO | High Water Breakdown: | 20260508 09:25:14.835 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:25:14.835 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:25:14.835 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:25:14.835 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:25:14.835 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:25:14.835 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:25:14.835 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:25:14.835 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:25:14.835 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:25:14.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:25:14.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:25:14.836 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:25:14.836 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:25:14.836 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:25:14.836 | INFO | ------------------------------------------ | 20260508 09:25:14.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:25:14.836 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:25:14.836 | INFO | Total mem usage: 27.8 GiB | 20260508 09:25:14.836 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:25:14.836 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:25:14.836 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:25:14.836 | INFO | High Water Breakdown: | 20260508 09:25:14.836 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:25:14.836 | INFO | ------------------------------------------ | 20260508 09:25:14.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:25:14.836 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:25:14.836 | INFO | Total mem usage: 15.7 GiB | 20260508 09:25:14.836 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:25:14.836 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:25:14.836 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:25:14.836 | INFO | High Water Breakdown: | 20260508 09:25:14.836 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:25:14.836 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:25:14.836 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:25:14.836 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:25:14.836 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:25:14.836 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:25:14.836 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:25:14.836 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:25:14.836 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:25:14.836 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:25:14.836 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:25:14.836 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:25:14.836 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:25:14.836 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:25:14.836 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:25:14.836 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:25:14.836 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:25:14.836 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:25:14.836 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:25:14.836 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:25:14.836 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:25:14.836 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:25:14.836 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:25:14.836 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:25:14.837 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:25:14.837 | INFO | ------------------------------------------ | 20260508 09:25:14.837 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:25:14.837 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:25:14.837 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:25:14.837 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:25:14.837 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 7.72e-01 | 20260508 09:25:14.837 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.06e-01, 1.08e-02 / 4.38e-03 | 20260508 09:25:14.837 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.40e+00, 5.90e-01 / 2.82e-01 | 20260508 09:25:14.837 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.10e-01 / 1.07e-01 | 20260508 09:25:14.837 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.49e+01, 3.66e+00 / 8.31e-01 | 20260508 09:25:14.837 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.79e+00 / 1.38e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.46e+00, 4.11e-01 / 2.95e-01 | 20260508 09:25:14.837 | INFO | PerfReport: 3.05%, Metrics -- 9.98e-01 / 1.14e+00, 2.49e-01 / 1.37e-01 | 20260508 09:25:14.837 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.35e-01, 4.83e-02 / 3.16e-02 | 20260508 09:25:14.837 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:25:14.837 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:25:14.837 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:25:14.837 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 24.18%, Upload -- 2.55e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 24.66%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 2.03%, LockUpload -- 2.14e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 2.68%, LockDownload -- 2.82e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:25:14.837 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.38e-02 / 0.00e+00 | 20260508 09:25:14.948 | INFO | Starting a round of Baz IO | 20260508 09:25:18.359 | INFO | Populating RTMetrics report | 20260508 09:25:18.360 | INFO | RTMetrics report populated | 20260508 09:25:18.360 | INFO | Opening tmp RTMetrics json file | 20260508 09:25:18.369 | INFO | Serializing RTMetrics to json | 20260508 09:25:18.371 | INFO | Serialization done, swapping tmp file now... | 20260508 09:25:18.371 | INFO | RTMetrics done | 20260508 09:25:18.427 | INFO | Finished a round of Baz IO: Wrote 1.486 GiB in 3.478 seconds | 20260508 09:25:18.427 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.69% | 20260508 09:25:18.427 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258267% / 0% / 7.54992e-05% | 20260508 09:25:18.427 | INFO | Baz IO overall: | 20260508 09:25:18.427 | INFO | BytesWritten : 3060670080270 | 20260508 09:25:18.427 | INFO | NumEvents : 1472377844046 | 20260508 09:25:19.974 | INFO | Analyzing chunk frames = [7012352,7012864) | 20260508 09:25:19.974 | INFO | MemInfo: { | 20260508 09:25:19.974 | INFO | "gpu" : " 17 GB", | 20260508 09:25:19.974 | INFO | "huge" : "", | 20260508 09:25:19.974 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:19.974 | INFO | "malloc" : " 14 GB", | 20260508 09:25:19.974 | INFO | "pinned" : " 29 GB" | 20260508 09:25:19.974 | INFO | } | 20260508 09:25:19.974 | INFO | Perf report for the last 5.13967 seconds: | 20260508 09:25:19.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:19.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:19.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:19.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:19.974 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 09:25:19.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:19.974 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260508 09:25:19.974 | INFO | PRE_HQ : 66.4%, 1.8, 61.2ms, 0 | 20260508 09:25:19.974 | INFO | BAZWRITER : 67.0%, 2.7, 93.9ms, 0 | 20260508 09:25:19.975 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:25:19.975 | INFO | Uploaded 36GiB to GPU in 2.58707s (13.9154GiB/s) | 20260508 09:25:19.975 | INFO | Downloaded 38GiB to GPU in 2.71172s (14.0133GiB/s) | 20260508 09:25:25.118 | INFO | Analyzing chunk frames = [7012864,7013376) | 20260508 09:25:25.118 | INFO | MemInfo: { | 20260508 09:25:25.118 | INFO | "gpu" : " 17 GB", | 20260508 09:25:25.118 | INFO | "huge" : "", | 20260508 09:25:25.118 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:25.118 | INFO | "malloc" : " 14 GB", | 20260508 09:25:25.118 | INFO | "pinned" : " 29 GB" | 20260508 09:25:25.118 | INFO | } | 20260508 09:25:25.118 | INFO | Perf report for the last 5.14411 seconds: | 20260508 09:25:25.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:25.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.118 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:25:25.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.119 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:25:25.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:25.119 | INFO | Uploaded 36GiB to GPU in 2.50215s (14.3876GiB/s) | 20260508 09:25:25.119 | INFO | Downloaded 34GiB to GPU in 2.47109s (13.7591GiB/s) | 20260508 09:25:30.255 | INFO | Analyzing chunk frames = [7013376,7013888) | 20260508 09:25:30.255 | INFO | MemInfo: { | 20260508 09:25:30.255 | INFO | "gpu" : " 17 GB", | 20260508 09:25:30.255 | INFO | "huge" : "", | 20260508 09:25:30.255 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:30.255 | INFO | "malloc" : " 14 GB", | 20260508 09:25:30.255 | INFO | "pinned" : " 29 GB" | 20260508 09:25:30.255 | INFO | } | 20260508 09:25:30.255 | INFO | Perf report for the last 5.13678 seconds: | 20260508 09:25:30.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:30.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 09:25:30.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:25:30.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:30.255 | INFO | Uploaded 36GiB to GPU in 2.43609s (14.7778GiB/s) | 20260508 09:25:30.255 | INFO | Downloaded 34GiB to GPU in 2.45046s (13.8749GiB/s) | 20260508 09:25:35.389 | INFO | Analyzing chunk frames = [7013888,7014400) | 20260508 09:25:35.389 | INFO | MemInfo: { | 20260508 09:25:35.389 | INFO | "gpu" : " 17 GB", | 20260508 09:25:35.389 | INFO | "huge" : "", | 20260508 09:25:35.389 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:35.389 | INFO | "malloc" : " 14 GB", | 20260508 09:25:35.389 | INFO | "pinned" : " 29 GB" | 20260508 09:25:35.389 | INFO | } | 20260508 09:25:35.389 | INFO | Perf report for the last 5.13426 seconds: | 20260508 09:25:35.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:35.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.389 | INFO | ANALYSIS : 66.7%, 2.9, 103.0ms, 0 | 20260508 09:25:35.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.390 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:25:35.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:35.390 | INFO | Uploaded 36GiB to GPU in 2.53564s (14.1976GiB/s) | 20260508 09:25:35.390 | INFO | Downloaded 34GiB to GPU in 2.43526s (13.9615GiB/s) | 20260508 09:25:40.537 | INFO | Analyzing chunk frames = [7014400,7014912) | 20260508 09:25:40.537 | INFO | MemInfo: { | 20260508 09:25:40.537 | INFO | "gpu" : " 17 GB", | 20260508 09:25:40.537 | INFO | "huge" : "", | 20260508 09:25:40.537 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:40.537 | INFO | "malloc" : " 14 GB", | 20260508 09:25:40.537 | INFO | "pinned" : " 29 GB" | 20260508 09:25:40.537 | INFO | } | 20260508 09:25:40.537 | INFO | Perf report for the last 5.1478 seconds: | 20260508 09:25:40.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:40.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.537 | INFO | ANALYSIS : 66.0%, 3.0, 103.4ms, 0 | 20260508 09:25:40.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.537 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:25:40.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:40.538 | INFO | Uploaded 36GiB to GPU in 2.35069s (15.3146GiB/s) | 20260508 09:25:40.538 | INFO | Downloaded 34GiB to GPU in 2.50238s (13.5871GiB/s) | 20260508 09:25:45.671 | INFO | Analyzing chunk frames = [7014912,7015424) | 20260508 09:25:45.671 | INFO | MemInfo: { | 20260508 09:25:45.671 | INFO | "gpu" : " 17 GB", | 20260508 09:25:45.671 | INFO | "huge" : "", | 20260508 09:25:45.671 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:45.671 | INFO | "malloc" : " 14 GB", | 20260508 09:25:45.671 | INFO | "pinned" : " 29 GB" | 20260508 09:25:45.671 | INFO | } | 20260508 09:25:45.671 | INFO | Perf report for the last 5.13417 seconds: | 20260508 09:25:45.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:45.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.672 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 09:25:45.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.672 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:25:45.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:45.672 | INFO | Uploaded 36GiB to GPU in 2.34659s (15.3414GiB/s) | 20260508 09:25:45.672 | INFO | Downloaded 34GiB to GPU in 2.45055s (13.8744GiB/s) | 20260508 09:25:50.810 | INFO | Analyzing chunk frames = [7015424,7015936) | 20260508 09:25:50.810 | INFO | MemInfo: { | 20260508 09:25:50.810 | INFO | "gpu" : " 17 GB", | 20260508 09:25:50.810 | INFO | "huge" : "", | 20260508 09:25:50.810 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:50.810 | INFO | "malloc" : " 14 GB", | 20260508 09:25:50.810 | INFO | "pinned" : " 29 GB" | 20260508 09:25:50.810 | INFO | } | 20260508 09:25:50.810 | INFO | Perf report for the last 5.13878 seconds: | 20260508 09:25:50.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:50.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.810 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:25:50.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.811 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 09:25:50.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:50.811 | INFO | Uploaded 36GiB to GPU in 2.36655s (15.212GiB/s) | 20260508 09:25:50.811 | INFO | Downloaded 34GiB to GPU in 2.45358s (13.8573GiB/s) | 20260508 09:25:55.947 | INFO | Analyzing chunk frames = [7015936,7016448) | 20260508 09:25:55.947 | INFO | MemInfo: { | 20260508 09:25:55.947 | INFO | "gpu" : " 17 GB", | 20260508 09:25:55.947 | INFO | "huge" : "", | 20260508 09:25:55.947 | INFO | "huge_shared" : " 29 GB", | 20260508 09:25:55.947 | INFO | "malloc" : " 14 GB", | 20260508 09:25:55.947 | INFO | "pinned" : " 29 GB" | 20260508 09:25:55.947 | INFO | } | 20260508 09:25:55.947 | INFO | Perf report for the last 5.13701 seconds: | 20260508 09:25:55.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:25:55.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.947 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:25:55.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.948 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 09:25:55.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:25:55.948 | INFO | Uploaded 36GiB to GPU in 2.40664s (14.9586GiB/s) | 20260508 09:25:55.948 | INFO | Downloaded 34GiB to GPU in 2.45859s (13.8291GiB/s) | 20260508 09:25:56.062 | INFO | Starting a round of Baz IO | 20260508 09:25:59.463 | INFO | Populating RTMetrics report | 20260508 09:25:59.463 | INFO | RTMetrics report populated | 20260508 09:25:59.463 | INFO | Opening tmp RTMetrics json file | 20260508 09:25:59.473 | INFO | Serializing RTMetrics to json | 20260508 09:25:59.474 | INFO | Serialization done, swapping tmp file now... | 20260508 09:25:59.474 | INFO | RTMetrics done | 20260508 09:25:59.533 | INFO | Finished a round of Baz IO: Wrote 1.484 GiB in 3.471 seconds | 20260508 09:25:59.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.31% / 0.00% / 59.69% | 20260508 09:25:59.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257067% / 0% / 5.96046e-05% | 20260508 09:25:59.533 | INFO | Baz IO overall: | 20260508 09:25:59.533 | INFO | BytesWritten : 3062263515643 | 20260508 09:25:59.533 | INFO | NumEvents : 1473130605175 | 20260508 09:26:01.086 | INFO | Analyzing chunk frames = [7016448,7016960) | 20260508 09:26:01.086 | INFO | MemInfo: { | 20260508 09:26:01.086 | INFO | "gpu" : " 17 GB", | 20260508 09:26:01.086 | INFO | "huge" : "", | 20260508 09:26:01.086 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:01.086 | INFO | "malloc" : " 14 GB", | 20260508 09:26:01.086 | INFO | "pinned" : " 29 GB" | 20260508 09:26:01.086 | INFO | } | 20260508 09:26:01.086 | INFO | Perf report for the last 5.13863 seconds: | 20260508 09:26:01.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:01.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:01.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:01.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:01.086 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 09:26:01.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:01.086 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 09:26:01.086 | INFO | PRE_HQ : 66.6%, 1.8, 61.5ms, 0 | 20260508 09:26:01.086 | INFO | BAZWRITER : 66.9%, 2.7, 93.9ms, 0 | 20260508 09:26:01.086 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:26:01.086 | INFO | Uploaded 36GiB to GPU in 2.59976s (13.8474GiB/s) | 20260508 09:26:01.086 | INFO | Downloaded 38GiB to GPU in 2.71908s (13.9753GiB/s) | 20260508 09:26:06.232 | INFO | Analyzing chunk frames = [7016960,7017472) | 20260508 09:26:06.232 | INFO | MemInfo: { | 20260508 09:26:06.232 | INFO | "gpu" : " 17 GB", | 20260508 09:26:06.232 | INFO | "huge" : "", | 20260508 09:26:06.232 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:06.232 | INFO | "malloc" : " 14 GB", | 20260508 09:26:06.232 | INFO | "pinned" : " 29 GB" | 20260508 09:26:06.232 | INFO | } | 20260508 09:26:06.232 | INFO | Perf report for the last 5.14642 seconds: | 20260508 09:26:06.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:06.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.232 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:26:06.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.232 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:26:06.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:06.233 | INFO | Uploaded 36GiB to GPU in 2.49127s (14.4504GiB/s) | 20260508 09:26:06.233 | INFO | Downloaded 34GiB to GPU in 2.47164s (13.756GiB/s) | 20260508 09:26:11.372 | INFO | Analyzing chunk frames = [7017472,7017984) | 20260508 09:26:11.372 | INFO | MemInfo: { | 20260508 09:26:11.372 | INFO | "gpu" : " 17 GB", | 20260508 09:26:11.372 | INFO | "huge" : "", | 20260508 09:26:11.372 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:11.372 | INFO | "malloc" : " 14 GB", | 20260508 09:26:11.372 | INFO | "pinned" : " 29 GB" | 20260508 09:26:11.372 | INFO | } | 20260508 09:26:11.372 | INFO | Perf report for the last 5.13997 seconds: | 20260508 09:26:11.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:11.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.372 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 09:26:11.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.372 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:26:11.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:11.373 | INFO | Uploaded 36GiB to GPU in 2.43654s (14.7751GiB/s) | 20260508 09:26:11.373 | INFO | Downloaded 34GiB to GPU in 2.43351s (13.9716GiB/s) | 20260508 09:26:16.505 | INFO | Analyzing chunk frames = [7017984,7018496) | 20260508 09:26:16.505 | INFO | MemInfo: { | 20260508 09:26:16.505 | INFO | "gpu" : " 17 GB", | 20260508 09:26:16.505 | INFO | "huge" : "", | 20260508 09:26:16.505 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:16.505 | INFO | "malloc" : " 14 GB", | 20260508 09:26:16.505 | INFO | "pinned" : " 29 GB" | 20260508 09:26:16.505 | INFO | } | 20260508 09:26:16.505 | INFO | Perf report for the last 5.13315 seconds: | 20260508 09:26:16.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:16.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 09:26:16.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:26:16.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:16.506 | INFO | Uploaded 36GiB to GPU in 2.43771s (14.768GiB/s) | 20260508 09:26:16.506 | INFO | Downloaded 34GiB to GPU in 2.43039s (13.9895GiB/s) | 20260508 09:26:21.643 | INFO | Analyzing chunk frames = [7018496,7019008) | 20260508 09:26:21.643 | INFO | MemInfo: { | 20260508 09:26:21.643 | INFO | "gpu" : " 17 GB", | 20260508 09:26:21.643 | INFO | "huge" : "", | 20260508 09:26:21.643 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:21.643 | INFO | "malloc" : " 14 GB", | 20260508 09:26:21.643 | INFO | "pinned" : " 29 GB" | 20260508 09:26:21.644 | INFO | } | 20260508 09:26:21.644 | INFO | Perf report for the last 5.13814 seconds: | 20260508 09:26:21.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:21.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 09:26:21.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 09:26:21.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:21.644 | INFO | Uploaded 36GiB to GPU in 2.30193s (15.639GiB/s) | 20260508 09:26:21.644 | INFO | Downloaded 34GiB to GPU in 2.51218s (13.5341GiB/s) | 20260508 09:26:26.788 | INFO | Analyzing chunk frames = [7019008,7019520) | 20260508 09:26:26.788 | INFO | MemInfo: { | 20260508 09:26:26.788 | INFO | "gpu" : " 17 GB", | 20260508 09:26:26.788 | INFO | "huge" : "", | 20260508 09:26:26.788 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:26.788 | INFO | "malloc" : " 14 GB", | 20260508 09:26:26.788 | INFO | "pinned" : " 29 GB" | 20260508 09:26:26.788 | INFO | } | 20260508 09:26:26.788 | INFO | Perf report for the last 5.14461 seconds: | 20260508 09:26:26.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:26.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.789 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:26:26.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.789 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:26:26.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:26.789 | INFO | Uploaded 36GiB to GPU in 2.37387s (15.1651GiB/s) | 20260508 09:26:26.789 | INFO | Downloaded 34GiB to GPU in 2.44121s (13.9275GiB/s) | 20260508 09:26:31.929 | INFO | Analyzing chunk frames = [7019520,7020032) | 20260508 09:26:31.929 | INFO | MemInfo: { | 20260508 09:26:31.929 | INFO | "gpu" : " 17 GB", | 20260508 09:26:31.929 | INFO | "huge" : "", | 20260508 09:26:31.929 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:31.929 | INFO | "malloc" : " 14 GB", | 20260508 09:26:31.929 | INFO | "pinned" : " 29 GB" | 20260508 09:26:31.929 | INFO | } | 20260508 09:26:31.929 | INFO | Perf report for the last 5.14049 seconds: | 20260508 09:26:31.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:31.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:26:31.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 09:26:31.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:31.929 | INFO | Uploaded 36GiB to GPU in 2.41073s (14.9332GiB/s) | 20260508 09:26:31.929 | INFO | Downloaded 34GiB to GPU in 2.44111s (13.9281GiB/s) | 20260508 09:26:37.063 | INFO | Analyzing chunk frames = [7020032,7020544) | 20260508 09:26:37.063 | INFO | MemInfo: { | 20260508 09:26:37.063 | INFO | "gpu" : " 17 GB", | 20260508 09:26:37.063 | INFO | "huge" : "", | 20260508 09:26:37.063 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:37.063 | INFO | "malloc" : " 14 GB", | 20260508 09:26:37.063 | INFO | "pinned" : " 29 GB" | 20260508 09:26:37.063 | INFO | } | 20260508 09:26:37.063 | INFO | Perf report for the last 5.1341 seconds: | 20260508 09:26:37.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:37.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 09:26:37.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:26:37.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:37.063 | INFO | Uploaded 36GiB to GPU in 2.45243s (14.6793GiB/s) | 20260508 09:26:37.063 | INFO | Downloaded 34GiB to GPU in 2.4489s (13.8838GiB/s) | 20260508 09:26:37.063 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:26:37.063 | INFO | Total mem usage: 16.7 GiB | 20260508 09:26:37.063 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:26:37.063 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:26:37.063 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:26:37.063 | INFO | High Water Breakdown: | 20260508 09:26:37.063 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:26:37.063 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:26:37.063 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:26:37.063 | INFO | ------------------------------------------ | 20260508 09:26:37.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:26:37.063 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:26:37.063 | INFO | Total mem usage: 26.5 GiB | 20260508 09:26:37.063 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:26:37.063 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:26:37.063 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:26:37.063 | INFO | High Water Breakdown: | 20260508 09:26:37.063 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:26:37.063 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:26:37.063 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:26:37.063 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:26:37.063 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:26:37.063 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:26:37.063 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:26:37.063 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:26:37.063 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:26:37.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:26:37.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:26:37.063 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:26:37.063 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:26:37.063 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:26:37.063 | INFO | ------------------------------------------ | 20260508 09:26:37.063 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:26:37.063 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:26:37.063 | INFO | Total mem usage: 27.8 GiB | 20260508 09:26:37.063 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:26:37.063 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:26:37.064 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:26:37.064 | INFO | High Water Breakdown: | 20260508 09:26:37.064 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:26:37.064 | INFO | ------------------------------------------ | 20260508 09:26:37.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:26:37.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:26:37.064 | INFO | Total mem usage: 15.7 GiB | 20260508 09:26:37.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:26:37.064 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:26:37.064 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:26:37.064 | INFO | High Water Breakdown: | 20260508 09:26:37.064 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:26:37.064 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:26:37.064 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:26:37.064 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:26:37.064 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:26:37.064 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:26:37.064 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:26:37.064 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:26:37.064 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:26:37.064 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:26:37.064 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:26:37.064 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:26:37.064 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:26:37.064 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:26:37.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:26:37.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:26:37.064 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:26:37.064 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:26:37.064 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:26:37.064 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:26:37.064 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:26:37.064 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:26:37.064 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:26:37.064 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:26:37.064 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:26:37.064 | INFO | ------------------------------------------ | 20260508 09:26:37.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:26:37.064 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:26:37.064 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:26:37.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:26:37.064 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 6.77e-01 | 20260508 09:26:37.064 | INFO | PerfReport: 0.30%, AggregateStats -- 9.79e-02 / 1.05e-01, 1.14e-02 / 4.78e-03 | 20260508 09:26:37.064 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.41e+00, 5.91e-01 / 2.32e-01 | 20260508 09:26:37.064 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.11e-01 / 1.33e-01 | 20260508 09:26:37.064 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.51e+01, 3.75e+00 / 1.12e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.85e+00 / 1.38e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.07e-01 / 2.70e-01 | 20260508 09:26:37.064 | INFO | PerfReport: 3.05%, Metrics -- 9.98e-01 / 1.14e+00, 2.47e-01 / 1.30e-01 | 20260508 09:26:37.064 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.05e-01 / 3.33e-01, 4.92e-02 / 2.38e-02 | 20260508 09:26:37.064 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:26:37.064 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:26:37.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:26:37.064 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 24.15%, Upload -- 2.54e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 24.65%, Download -- 2.59e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 2.00%, LockUpload -- 2.10e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 2.75%, LockDownload -- 2.89e+00 / 0.00e+00, 4.40e+00 / 0.00e+00 | 20260508 09:26:37.064 | INFO | PerfReport: 15.32%, LockCompute -- 1.61e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:26:37.065 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260508 09:26:37.160 | INFO | Starting a round of Baz IO | 20260508 09:26:40.572 | INFO | Populating RTMetrics report | 20260508 09:26:40.572 | INFO | RTMetrics report populated | 20260508 09:26:40.573 | INFO | Opening tmp RTMetrics json file | 20260508 09:26:40.583 | INFO | Serializing RTMetrics to json | 20260508 09:26:40.584 | INFO | Serialization done, swapping tmp file now... | 20260508 09:26:40.585 | INFO | RTMetrics done | 20260508 09:26:40.647 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.487 seconds | 20260508 09:26:40.647 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.69% | 20260508 09:26:40.647 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.256828% / 3.97364e-06% / 5.16574e-05% | 20260508 09:26:40.647 | INFO | Baz IO overall: | 20260508 09:26:40.647 | INFO | BytesWritten : 3063856243152 | 20260508 09:26:40.647 | INFO | NumEvents : 1473883039575 | 20260508 09:26:42.202 | INFO | Analyzing chunk frames = [7020544,7021056) | 20260508 09:26:42.202 | INFO | MemInfo: { | 20260508 09:26:42.202 | INFO | "gpu" : " 17 GB", | 20260508 09:26:42.202 | INFO | "huge" : "", | 20260508 09:26:42.202 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:42.202 | INFO | "malloc" : " 14 GB", | 20260508 09:26:42.202 | INFO | "pinned" : " 29 GB" | 20260508 09:26:42.202 | INFO | } | 20260508 09:26:42.202 | INFO | Perf report for the last 5.13934 seconds: | 20260508 09:26:42.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:42.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:42.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:42.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:42.202 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:26:42.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:42.203 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:26:42.203 | INFO | PRE_HQ : 65.5%, 1.6, 54.7ms, 0 | 20260508 09:26:42.203 | INFO | BAZWRITER : 67.3%, 2.5, 88.0ms, 0 | 20260508 09:26:42.203 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:26:42.203 | INFO | Uploaded 36GiB to GPU in 2.60344s (13.8279GiB/s) | 20260508 09:26:42.203 | INFO | Downloaded 38GiB to GPU in 2.71552s (13.9936GiB/s) | 20260508 09:26:47.343 | INFO | Analyzing chunk frames = [7021056,7021568) | 20260508 09:26:47.343 | INFO | MemInfo: { | 20260508 09:26:47.343 | INFO | "gpu" : " 17 GB", | 20260508 09:26:47.343 | INFO | "huge" : "", | 20260508 09:26:47.343 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:47.343 | INFO | "malloc" : " 14 GB", | 20260508 09:26:47.343 | INFO | "pinned" : " 29 GB" | 20260508 09:26:47.344 | INFO | } | 20260508 09:26:47.344 | INFO | Perf report for the last 5.1413 seconds: | 20260508 09:26:47.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:47.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 09:26:47.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:26:47.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:47.344 | INFO | Uploaded 36GiB to GPU in 2.54488s (14.146GiB/s) | 20260508 09:26:47.344 | INFO | Downloaded 34GiB to GPU in 2.4479s (13.8895GiB/s) | 20260508 09:26:52.483 | INFO | Analyzing chunk frames = [7021568,7022080) | 20260508 09:26:52.483 | INFO | MemInfo: { | 20260508 09:26:52.483 | INFO | "gpu" : " 17 GB", | 20260508 09:26:52.483 | INFO | "huge" : "", | 20260508 09:26:52.483 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:52.483 | INFO | "malloc" : " 14 GB", | 20260508 09:26:52.483 | INFO | "pinned" : " 29 GB" | 20260508 09:26:52.483 | INFO | } | 20260508 09:26:52.483 | INFO | Perf report for the last 5.13964 seconds: | 20260508 09:26:52.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:52.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:26:52.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:26:52.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:52.483 | INFO | Uploaded 36GiB to GPU in 2.49589s (14.4237GiB/s) | 20260508 09:26:52.484 | INFO | Downloaded 34GiB to GPU in 2.42371s (14.0281GiB/s) | 20260508 09:26:57.623 | INFO | Analyzing chunk frames = [7022080,7022592) | 20260508 09:26:57.623 | INFO | MemInfo: { | 20260508 09:26:57.623 | INFO | "gpu" : " 17 GB", | 20260508 09:26:57.623 | INFO | "huge" : "", | 20260508 09:26:57.623 | INFO | "huge_shared" : " 29 GB", | 20260508 09:26:57.623 | INFO | "malloc" : " 14 GB", | 20260508 09:26:57.623 | INFO | "pinned" : " 29 GB" | 20260508 09:26:57.624 | INFO | } | 20260508 09:26:57.624 | INFO | Perf report for the last 5.14037 seconds: | 20260508 09:26:57.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:26:57.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 09:26:57.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:26:57.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:26:57.624 | INFO | Uploaded 36GiB to GPU in 2.52395s (14.2634GiB/s) | 20260508 09:26:57.624 | INFO | Downloaded 34GiB to GPU in 2.42469s (14.0224GiB/s) | 20260508 09:27:02.759 | INFO | Analyzing chunk frames = [7022592,7023104) | 20260508 09:27:02.759 | INFO | MemInfo: { | 20260508 09:27:02.759 | INFO | "gpu" : " 17 GB", | 20260508 09:27:02.759 | INFO | "huge" : "", | 20260508 09:27:02.759 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:02.759 | INFO | "malloc" : " 14 GB", | 20260508 09:27:02.759 | INFO | "pinned" : " 29 GB" | 20260508 09:27:02.759 | INFO | } | 20260508 09:27:02.759 | INFO | Perf report for the last 5.13562 seconds: | 20260508 09:27:02.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:02.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 09:27:02.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 09:27:02.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:02.759 | INFO | Uploaded 36GiB to GPU in 2.45825s (14.6445GiB/s) | 20260508 09:27:02.760 | INFO | Downloaded 34GiB to GPU in 2.5308s (13.4345GiB/s) | 20260508 09:27:07.897 | INFO | Analyzing chunk frames = [7023104,7023616) | 20260508 09:27:07.897 | INFO | MemInfo: { | 20260508 09:27:07.897 | INFO | "gpu" : " 17 GB", | 20260508 09:27:07.897 | INFO | "huge" : "", | 20260508 09:27:07.897 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:07.898 | INFO | "malloc" : " 14 GB", | 20260508 09:27:07.898 | INFO | "pinned" : " 29 GB" | 20260508 09:27:07.898 | INFO | } | 20260508 09:27:07.898 | INFO | Perf report for the last 5.13842 seconds: | 20260508 09:27:07.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:07.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:27:07.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:27:07.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:07.898 | INFO | Uploaded 36GiB to GPU in 2.38849s (15.0723GiB/s) | 20260508 09:27:07.898 | INFO | Downloaded 34GiB to GPU in 2.4332s (13.9733GiB/s) | 20260508 09:27:13.041 | INFO | Analyzing chunk frames = [7023616,7024128) | 20260508 09:27:13.041 | INFO | MemInfo: { | 20260508 09:27:13.041 | INFO | "gpu" : " 17 GB", | 20260508 09:27:13.041 | INFO | "huge" : "", | 20260508 09:27:13.041 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:13.041 | INFO | "malloc" : " 14 GB", | 20260508 09:27:13.041 | INFO | "pinned" : " 29 GB" | 20260508 09:27:13.041 | INFO | } | 20260508 09:27:13.041 | INFO | Perf report for the last 5.1437 seconds: | 20260508 09:27:13.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:13.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.041 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:27:13.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.041 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:27:13.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:13.042 | INFO | Uploaded 36GiB to GPU in 2.41073s (14.9332GiB/s) | 20260508 09:27:13.042 | INFO | Downloaded 34GiB to GPU in 2.43924s (13.9388GiB/s) | 20260508 09:27:18.185 | INFO | Analyzing chunk frames = [7024128,7024640) | 20260508 09:27:18.185 | INFO | MemInfo: { | 20260508 09:27:18.185 | INFO | "gpu" : " 17 GB", | 20260508 09:27:18.185 | INFO | "huge" : "", | 20260508 09:27:18.185 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:18.185 | INFO | "malloc" : " 14 GB", | 20260508 09:27:18.185 | INFO | "pinned" : " 29 GB" | 20260508 09:27:18.185 | INFO | } | 20260508 09:27:18.185 | INFO | Perf report for the last 5.14407 seconds: | 20260508 09:27:18.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:18.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.186 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:27:18.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.186 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:27:18.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:18.186 | INFO | Uploaded 36GiB to GPU in 2.43015s (14.8139GiB/s) | 20260508 09:27:18.186 | INFO | Downloaded 34GiB to GPU in 2.4527s (13.8623GiB/s) | 20260508 09:27:18.282 | INFO | Starting a round of Baz IO | 20260508 09:27:21.692 | INFO | Populating RTMetrics report | 20260508 09:27:21.692 | INFO | RTMetrics report populated | 20260508 09:27:21.692 | INFO | Opening tmp RTMetrics json file | 20260508 09:27:21.701 | INFO | Serializing RTMetrics to json | 20260508 09:27:21.702 | INFO | Serialization done, swapping tmp file now... | 20260508 09:27:21.702 | INFO | RTMetrics done | 20260508 09:27:21.764 | INFO | Finished a round of Baz IO: Wrote 1.482 GiB in 3.482 seconds | 20260508 09:27:21.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:27:21.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.256336% / 0% / 5.96046e-05% | 20260508 09:27:21.765 | INFO | Baz IO overall: | 20260508 09:27:21.765 | INFO | BytesWritten : 3065448058061 | 20260508 09:27:21.765 | INFO | NumEvents : 1474635134461 | 20260508 09:27:23.324 | INFO | Analyzing chunk frames = [7024640,7025152) | 20260508 09:27:23.324 | INFO | MemInfo: { | 20260508 09:27:23.324 | INFO | "gpu" : " 17 GB", | 20260508 09:27:23.324 | INFO | "huge" : "", | 20260508 09:27:23.324 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:23.324 | INFO | "malloc" : " 14 GB", | 20260508 09:27:23.324 | INFO | "pinned" : " 29 GB" | 20260508 09:27:23.324 | INFO | } | 20260508 09:27:23.324 | INFO | Perf report for the last 5.13877 seconds: | 20260508 09:27:23.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:23.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:23.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:23.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:23.324 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:27:23.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:23.324 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:27:23.324 | INFO | PRE_HQ : 65.4%, 1.6, 54.4ms, 0 | 20260508 09:27:23.324 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 09:27:23.324 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:27:23.325 | INFO | Uploaded 36GiB to GPU in 2.58383s (13.9328GiB/s) | 20260508 09:27:23.325 | INFO | Downloaded 38GiB to GPU in 2.71515s (13.9956GiB/s) | 20260508 09:27:28.454 | INFO | Analyzing chunk frames = [7025152,7025664) | 20260508 09:27:28.454 | INFO | MemInfo: { | 20260508 09:27:28.454 | INFO | "gpu" : " 17 GB", | 20260508 09:27:28.454 | INFO | "huge" : "", | 20260508 09:27:28.454 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:28.454 | INFO | "malloc" : " 14 GB", | 20260508 09:27:28.454 | INFO | "pinned" : " 29 GB" | 20260508 09:27:28.454 | INFO | } | 20260508 09:27:28.454 | INFO | Perf report for the last 5.13007 seconds: | 20260508 09:27:28.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:28.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.454 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 09:27:28.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.454 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:27:28.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:28.455 | INFO | Uploaded 36GiB to GPU in 2.40429s (14.9733GiB/s) | 20260508 09:27:28.455 | INFO | Downloaded 34GiB to GPU in 2.47289s (13.7491GiB/s) | 20260508 09:27:33.594 | INFO | Analyzing chunk frames = [7025664,7026176) | 20260508 09:27:33.594 | INFO | MemInfo: { | 20260508 09:27:33.594 | INFO | "gpu" : " 17 GB", | 20260508 09:27:33.594 | INFO | "huge" : "", | 20260508 09:27:33.594 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:33.594 | INFO | "malloc" : " 14 GB", | 20260508 09:27:33.594 | INFO | "pinned" : " 29 GB" | 20260508 09:27:33.594 | INFO | } | 20260508 09:27:33.595 | INFO | Perf report for the last 5.14022 seconds: | 20260508 09:27:33.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:33.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 09:27:33.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:27:33.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:33.595 | INFO | Uploaded 36GiB to GPU in 2.45686s (14.6528GiB/s) | 20260508 09:27:33.595 | INFO | Downloaded 34GiB to GPU in 2.4555s (13.8465GiB/s) | 20260508 09:27:38.735 | INFO | Analyzing chunk frames = [7026176,7026688) | 20260508 09:27:38.735 | INFO | MemInfo: { | 20260508 09:27:38.735 | INFO | "gpu" : " 17 GB", | 20260508 09:27:38.735 | INFO | "huge" : "", | 20260508 09:27:38.735 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:38.735 | INFO | "malloc" : " 14 GB", | 20260508 09:27:38.735 | INFO | "pinned" : " 29 GB" | 20260508 09:27:38.735 | INFO | } | 20260508 09:27:38.735 | INFO | Perf report for the last 5.14028 seconds: | 20260508 09:27:38.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:38.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | ANALYSIS : 66.8%, 2.9, 103.4ms, 0 | 20260508 09:27:38.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:27:38.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:38.735 | INFO | Uploaded 36GiB to GPU in 2.54439s (14.1488GiB/s) | 20260508 09:27:38.735 | INFO | Downloaded 34GiB to GPU in 2.43764s (13.9479GiB/s) | 20260508 09:27:43.874 | INFO | Analyzing chunk frames = [7026688,7027200) | 20260508 09:27:43.874 | INFO | MemInfo: { | 20260508 09:27:43.874 | INFO | "gpu" : " 17 GB", | 20260508 09:27:43.874 | INFO | "huge" : "", | 20260508 09:27:43.875 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:43.875 | INFO | "malloc" : " 14 GB", | 20260508 09:27:43.875 | INFO | "pinned" : " 29 GB" | 20260508 09:27:43.875 | INFO | } | 20260508 09:27:43.875 | INFO | Perf report for the last 5.13983 seconds: | 20260508 09:27:43.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:43.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 09:27:43.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:27:43.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:43.875 | INFO | Uploaded 36GiB to GPU in 2.26274s (15.9099GiB/s) | 20260508 09:27:43.875 | INFO | Downloaded 34GiB to GPU in 2.49527s (13.6258GiB/s) | 20260508 09:27:49.015 | INFO | Analyzing chunk frames = [7027200,7027712) | 20260508 09:27:49.015 | INFO | MemInfo: { | 20260508 09:27:49.015 | INFO | "gpu" : " 17 GB", | 20260508 09:27:49.015 | INFO | "huge" : "", | 20260508 09:27:49.015 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:49.015 | INFO | "malloc" : " 14 GB", | 20260508 09:27:49.015 | INFO | "pinned" : " 29 GB" | 20260508 09:27:49.015 | INFO | } | 20260508 09:27:49.015 | INFO | Perf report for the last 5.14051 seconds: | 20260508 09:27:49.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:49.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.015 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 09:27:49.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.015 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:27:49.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:49.016 | INFO | Uploaded 36GiB to GPU in 2.40099s (14.9938GiB/s) | 20260508 09:27:49.016 | INFO | Downloaded 34GiB to GPU in 2.43635s (13.9553GiB/s) | 20260508 09:27:54.153 | INFO | Analyzing chunk frames = [7027712,7028224) | 20260508 09:27:54.153 | INFO | MemInfo: { | 20260508 09:27:54.153 | INFO | "gpu" : " 17 GB", | 20260508 09:27:54.153 | INFO | "huge" : "", | 20260508 09:27:54.153 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:54.154 | INFO | "malloc" : " 14 GB", | 20260508 09:27:54.154 | INFO | "pinned" : " 29 GB" | 20260508 09:27:54.154 | INFO | } | 20260508 09:27:54.154 | INFO | Perf report for the last 5.13842 seconds: | 20260508 09:27:54.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:54.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:27:54.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:27:54.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:54.154 | INFO | Uploaded 36GiB to GPU in 2.42072s (14.8716GiB/s) | 20260508 09:27:54.154 | INFO | Downloaded 34GiB to GPU in 2.44018s (13.9334GiB/s) | 20260508 09:27:59.296 | INFO | Analyzing chunk frames = [7028224,7028736) | 20260508 09:27:59.296 | INFO | MemInfo: { | 20260508 09:27:59.296 | INFO | "gpu" : " 17 GB", | 20260508 09:27:59.296 | INFO | "huge" : "", | 20260508 09:27:59.296 | INFO | "huge_shared" : " 29 GB", | 20260508 09:27:59.296 | INFO | "malloc" : " 14 GB", | 20260508 09:27:59.296 | INFO | "pinned" : " 29 GB" | 20260508 09:27:59.296 | INFO | } | 20260508 09:27:59.296 | INFO | Perf report for the last 5.14221 seconds: | 20260508 09:27:59.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:27:59.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 09:27:59.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:27:59.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:27:59.296 | INFO | Uploaded 36GiB to GPU in 2.40934s (14.9418GiB/s) | 20260508 09:27:59.296 | INFO | Downloaded 34GiB to GPU in 2.43394s (13.9691GiB/s) | 20260508 09:27:59.296 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:27:59.296 | INFO | Total mem usage: 16.7 GiB | 20260508 09:27:59.296 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:27:59.296 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:27:59.296 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:27:59.296 | INFO | High Water Breakdown: | 20260508 09:27:59.296 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:27:59.296 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:27:59.296 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:27:59.296 | INFO | ------------------------------------------ | 20260508 09:27:59.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:27:59.296 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:27:59.296 | INFO | Total mem usage: 26.5 GiB | 20260508 09:27:59.296 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:27:59.296 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:27:59.297 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:27:59.297 | INFO | High Water Breakdown: | 20260508 09:27:59.297 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:27:59.297 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:27:59.297 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:27:59.297 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:27:59.297 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:27:59.297 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:27:59.297 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:27:59.297 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:27:59.297 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:27:59.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:27:59.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:27:59.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:27:59.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:27:59.297 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:27:59.297 | INFO | ------------------------------------------ | 20260508 09:27:59.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:27:59.297 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:27:59.297 | INFO | Total mem usage: 27.8 GiB | 20260508 09:27:59.297 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:27:59.297 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:27:59.297 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:27:59.297 | INFO | High Water Breakdown: | 20260508 09:27:59.297 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:27:59.297 | INFO | ------------------------------------------ | 20260508 09:27:59.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:27:59.297 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:27:59.297 | INFO | Total mem usage: 15.7 GiB | 20260508 09:27:59.297 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:27:59.297 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:27:59.297 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:27:59.297 | INFO | High Water Breakdown: | 20260508 09:27:59.297 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:27:59.297 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:27:59.297 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:27:59.297 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:27:59.297 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:27:59.298 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:27:59.298 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:27:59.298 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:27:59.298 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:27:59.298 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:27:59.298 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:27:59.298 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:27:59.298 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:27:59.298 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:27:59.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:27:59.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:27:59.298 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:27:59.298 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:27:59.298 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:27:59.298 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:27:59.298 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:27:59.298 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:27:59.298 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:27:59.298 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:27:59.298 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:27:59.298 | INFO | ------------------------------------------ | 20260508 09:27:59.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:27:59.298 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:27:59.298 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:27:59.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:27:59.298 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.17e-01 | 20260508 09:27:59.298 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.15e-02 / 5.87e-03 | 20260508 09:27:59.298 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.41e+00, 5.90e-01 / 2.72e-01 | 20260508 09:27:59.298 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.32e-01 | 20260508 09:27:59.298 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.53e+01, 3.79e+00 / 1.16e+00 | 20260508 09:27:59.298 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.84e+00 / 1.39e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.53e+00, 4.05e-01 / 2.66e-01 | 20260508 09:27:59.299 | INFO | PerfReport: 3.04%, Metrics -- 9.96e-01 / 1.15e+00, 2.47e-01 / 1.13e-01 | 20260508 09:27:59.299 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.36e-01, 4.98e-02 / 2.29e-02 | 20260508 09:27:59.299 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:27:59.299 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:27:59.299 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:27:59.299 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 24.30%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 24.56%, Download -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 2.17%, LockUpload -- 2.28e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 2.56%, LockDownload -- 2.70e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:27:59.299 | INFO | PerfReport: 0.01%, Misc -- 1.19e-02 / 0.00e+00, 1.06e-02 / 0.00e+00 | 20260508 09:27:59.393 | INFO | Starting a round of Baz IO | 20260508 09:28:02.805 | INFO | Populating RTMetrics report | 20260508 09:28:02.805 | INFO | RTMetrics report populated | 20260508 09:28:02.805 | INFO | Opening tmp RTMetrics json file | 20260508 09:28:02.814 | INFO | Serializing RTMetrics to json | 20260508 09:28:02.816 | INFO | Serialization done, swapping tmp file now... | 20260508 09:28:02.816 | INFO | RTMetrics done | 20260508 09:28:02.877 | INFO | Finished a round of Baz IO: Wrote 1.484 GiB in 3.483 seconds | 20260508 09:28:02.877 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:28:02.877 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257031% / 0% / 7.15256e-05% | 20260508 09:28:02.877 | INFO | Baz IO overall: | 20260508 09:28:02.877 | INFO | BytesWritten : 3067041545778 | 20260508 09:28:02.877 | INFO | NumEvents : 1475388767317 | 20260508 09:28:04.431 | INFO | Analyzing chunk frames = [7028736,7029248) | 20260508 09:28:04.431 | INFO | MemInfo: { | 20260508 09:28:04.431 | INFO | "gpu" : " 17 GB", | 20260508 09:28:04.431 | INFO | "huge" : "", | 20260508 09:28:04.431 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:04.431 | INFO | "malloc" : " 14 GB", | 20260508 09:28:04.431 | INFO | "pinned" : " 29 GB" | 20260508 09:28:04.431 | INFO | } | 20260508 09:28:04.431 | INFO | Perf report for the last 5.1348 seconds: | 20260508 09:28:04.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:04.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:04.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:04.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:04.431 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 09:28:04.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:04.431 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:28:04.431 | INFO | PRE_HQ : 65.4%, 1.6, 54.3ms, 0 | 20260508 09:28:04.431 | INFO | BAZWRITER : 67.2%, 2.6, 88.0ms, 0 | 20260508 09:28:04.431 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:28:04.431 | INFO | Uploaded 36GiB to GPU in 2.5983s (13.8552GiB/s) | 20260508 09:28:04.431 | INFO | Downloaded 38GiB to GPU in 2.70894s (14.0276GiB/s) | 20260508 09:28:09.571 | INFO | Analyzing chunk frames = [7029248,7029760) | 20260508 09:28:09.571 | INFO | MemInfo: { | 20260508 09:28:09.571 | INFO | "gpu" : " 17 GB", | 20260508 09:28:09.571 | INFO | "huge" : "", | 20260508 09:28:09.571 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:09.571 | INFO | "malloc" : " 14 GB", | 20260508 09:28:09.571 | INFO | "pinned" : " 29 GB" | 20260508 09:28:09.571 | INFO | } | 20260508 09:28:09.571 | INFO | Perf report for the last 5.14048 seconds: | 20260508 09:28:09.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:09.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:28:09.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:28:09.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:09.571 | INFO | Uploaded 36GiB to GPU in 2.53657s (14.1924GiB/s) | 20260508 09:28:09.572 | INFO | Downloaded 34GiB to GPU in 2.43475s (13.9644GiB/s) | 20260508 09:28:14.707 | INFO | Analyzing chunk frames = [7029760,7030272) | 20260508 09:28:14.707 | INFO | MemInfo: { | 20260508 09:28:14.707 | INFO | "gpu" : " 17 GB", | 20260508 09:28:14.707 | INFO | "huge" : "", | 20260508 09:28:14.707 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:14.707 | INFO | "malloc" : " 14 GB", | 20260508 09:28:14.707 | INFO | "pinned" : " 29 GB" | 20260508 09:28:14.707 | INFO | } | 20260508 09:28:14.707 | INFO | Perf report for the last 5.13619 seconds: | 20260508 09:28:14.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:14.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.707 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 09:28:14.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.708 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:28:14.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:14.708 | INFO | Uploaded 36GiB to GPU in 2.51757s (14.2995GiB/s) | 20260508 09:28:14.708 | INFO | Downloaded 34GiB to GPU in 2.43117s (13.985GiB/s) | 20260508 09:28:19.849 | INFO | Analyzing chunk frames = [7030272,7030784) | 20260508 09:28:19.849 | INFO | MemInfo: { | 20260508 09:28:19.849 | INFO | "gpu" : " 17 GB", | 20260508 09:28:19.849 | INFO | "huge" : "", | 20260508 09:28:19.849 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:19.849 | INFO | "malloc" : " 14 GB", | 20260508 09:28:19.849 | INFO | "pinned" : " 29 GB" | 20260508 09:28:19.849 | INFO | } | 20260508 09:28:19.849 | INFO | Perf report for the last 5.14175 seconds: | 20260508 09:28:19.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:19.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.849 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 09:28:19.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.849 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:28:19.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:19.850 | INFO | Uploaded 36GiB to GPU in 2.52974s (14.2307GiB/s) | 20260508 09:28:19.850 | INFO | Downloaded 34GiB to GPU in 2.43403s (13.9686GiB/s) | 20260508 09:28:24.994 | INFO | Analyzing chunk frames = [7030784,7031296) | 20260508 09:28:24.994 | INFO | MemInfo: { | 20260508 09:28:24.994 | INFO | "gpu" : " 17 GB", | 20260508 09:28:24.994 | INFO | "huge" : "", | 20260508 09:28:24.994 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:24.994 | INFO | "malloc" : " 14 GB", | 20260508 09:28:24.994 | INFO | "pinned" : " 29 GB" | 20260508 09:28:24.995 | INFO | } | 20260508 09:28:24.995 | INFO | Perf report for the last 5.14552 seconds: | 20260508 09:28:24.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:24.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:28:24.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:28:24.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:24.995 | INFO | Uploaded 36GiB to GPU in 2.42265s (14.8597GiB/s) | 20260508 09:28:24.995 | INFO | Downloaded 34GiB to GPU in 2.52525s (13.464GiB/s) | 20260508 09:28:30.136 | INFO | Analyzing chunk frames = [7031296,7031808) | 20260508 09:28:30.136 | INFO | MemInfo: { | 20260508 09:28:30.136 | INFO | "gpu" : " 17 GB", | 20260508 09:28:30.136 | INFO | "huge" : "", | 20260508 09:28:30.136 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:30.136 | INFO | "malloc" : " 14 GB", | 20260508 09:28:30.136 | INFO | "pinned" : " 29 GB" | 20260508 09:28:30.136 | INFO | } | 20260508 09:28:30.136 | INFO | Perf report for the last 5.14151 seconds: | 20260508 09:28:30.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:30.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:28:30.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:28:30.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:30.136 | INFO | Uploaded 36GiB to GPU in 2.37633s (15.1494GiB/s) | 20260508 09:28:30.136 | INFO | Downloaded 34GiB to GPU in 2.42065s (14.0458GiB/s) | 20260508 09:28:35.269 | INFO | Analyzing chunk frames = [7031808,7032320) | 20260508 09:28:35.269 | INFO | MemInfo: { | 20260508 09:28:35.269 | INFO | "gpu" : " 17 GB", | 20260508 09:28:35.269 | INFO | "huge" : "", | 20260508 09:28:35.269 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:35.269 | INFO | "malloc" : " 14 GB", | 20260508 09:28:35.269 | INFO | "pinned" : " 29 GB" | 20260508 09:28:35.269 | INFO | } | 20260508 09:28:35.269 | INFO | Perf report for the last 5.13305 seconds: | 20260508 09:28:35.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:35.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:28:35.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:28:35.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:35.269 | INFO | Uploaded 36GiB to GPU in 2.44876s (14.7013GiB/s) | 20260508 09:28:35.270 | INFO | Downloaded 34GiB to GPU in 2.42484s (14.0215GiB/s) | 20260508 09:28:40.403 | INFO | Analyzing chunk frames = [7032320,7032832) | 20260508 09:28:40.403 | INFO | MemInfo: { | 20260508 09:28:40.403 | INFO | "gpu" : " 17 GB", | 20260508 09:28:40.403 | INFO | "huge" : "", | 20260508 09:28:40.403 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:40.403 | INFO | "malloc" : " 14 GB", | 20260508 09:28:40.403 | INFO | "pinned" : " 29 GB" | 20260508 09:28:40.403 | INFO | } | 20260508 09:28:40.403 | INFO | Perf report for the last 5.13367 seconds: | 20260508 09:28:40.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:40.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 09:28:40.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 09:28:40.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:40.403 | INFO | Uploaded 36GiB to GPU in 2.44627s (14.7163GiB/s) | 20260508 09:28:40.404 | INFO | Downloaded 34GiB to GPU in 2.42213s (14.0372GiB/s) | 20260508 09:28:40.500 | INFO | Starting a round of Baz IO | 20260508 09:28:43.911 | INFO | Populating RTMetrics report | 20260508 09:28:43.911 | INFO | RTMetrics report populated | 20260508 09:28:43.911 | INFO | Opening tmp RTMetrics json file | 20260508 09:28:43.912 | INFO | Serializing RTMetrics to json | 20260508 09:28:43.913 | INFO | Serialization done, swapping tmp file now... | 20260508 09:28:43.914 | INFO | RTMetrics done | 20260508 09:28:43.992 | INFO | Finished a round of Baz IO: Wrote 1.481 GiB in 3.492 seconds | 20260508 09:28:43.992 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:28:43.992 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.256073% / 3.97364e-06% / 6.35783e-05% | 20260508 09:28:43.992 | INFO | Baz IO overall: | 20260508 09:28:43.992 | INFO | BytesWritten : 3068632250188 | 20260508 09:28:43.992 | INFO | NumEvents : 1476140258717 | 20260508 09:28:45.549 | INFO | Analyzing chunk frames = [7032832,7033344) | 20260508 09:28:45.549 | INFO | MemInfo: { | 20260508 09:28:45.549 | INFO | "gpu" : " 17 GB", | 20260508 09:28:45.549 | INFO | "huge" : "", | 20260508 09:28:45.549 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:45.550 | INFO | "malloc" : " 14 GB", | 20260508 09:28:45.550 | INFO | "pinned" : " 29 GB" | 20260508 09:28:45.550 | INFO | } | 20260508 09:28:45.550 | INFO | Perf report for the last 5.14667 seconds: | 20260508 09:28:45.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:45.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:45.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:45.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:45.550 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 09:28:45.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:45.550 | INFO | RT_METRICS : 41.4%, 1.0, 21.8ms, 0 | 20260508 09:28:45.550 | INFO | PRE_HQ : 65.8%, 1.7, 58.7ms, 0 | 20260508 09:28:45.550 | INFO | BAZWRITER : 67.4%, 2.6, 91.8ms, 0 | 20260508 09:28:45.550 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:28:45.550 | INFO | Uploaded 36GiB to GPU in 2.61162s (13.7846GiB/s) | 20260508 09:28:45.550 | INFO | Downloaded 38GiB to GPU in 2.71477s (13.9975GiB/s) | 20260508 09:28:50.693 | INFO | Analyzing chunk frames = [7033344,7033856) | 20260508 09:28:50.693 | INFO | MemInfo: { | 20260508 09:28:50.693 | INFO | "gpu" : " 17 GB", | 20260508 09:28:50.693 | INFO | "huge" : "", | 20260508 09:28:50.693 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:50.693 | INFO | "malloc" : " 14 GB", | 20260508 09:28:50.693 | INFO | "pinned" : " 29 GB" | 20260508 09:28:50.693 | INFO | } | 20260508 09:28:50.693 | INFO | Perf report for the last 5.14377 seconds: | 20260508 09:28:50.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:50.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.693 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:28:50.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.694 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:28:50.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:50.694 | INFO | Uploaded 36GiB to GPU in 2.54389s (14.1516GiB/s) | 20260508 09:28:50.694 | INFO | Downloaded 34GiB to GPU in 2.45544s (13.8468GiB/s) | 20260508 09:28:55.831 | INFO | Analyzing chunk frames = [7033856,7034368) | 20260508 09:28:55.831 | INFO | MemInfo: { | 20260508 09:28:55.831 | INFO | "gpu" : " 17 GB", | 20260508 09:28:55.831 | INFO | "huge" : "", | 20260508 09:28:55.831 | INFO | "huge_shared" : " 29 GB", | 20260508 09:28:55.831 | INFO | "malloc" : " 14 GB", | 20260508 09:28:55.831 | INFO | "pinned" : " 29 GB" | 20260508 09:28:55.831 | INFO | } | 20260508 09:28:55.831 | INFO | Perf report for the last 5.13803 seconds: | 20260508 09:28:55.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:28:55.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.831 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:28:55.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.832 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:28:55.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:28:55.832 | INFO | Uploaded 36GiB to GPU in 2.50007s (14.3996GiB/s) | 20260508 09:28:55.832 | INFO | Downloaded 34GiB to GPU in 2.43286s (13.9753GiB/s) | 20260508 09:29:00.968 | INFO | Analyzing chunk frames = [7034368,7034880) | 20260508 09:29:00.968 | INFO | MemInfo: { | 20260508 09:29:00.968 | INFO | "gpu" : " 17 GB", | 20260508 09:29:00.968 | INFO | "huge" : "", | 20260508 09:29:00.968 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:00.968 | INFO | "malloc" : " 14 GB", | 20260508 09:29:00.968 | INFO | "pinned" : " 29 GB" | 20260508 09:29:00.968 | INFO | } | 20260508 09:29:00.968 | INFO | Perf report for the last 5.13654 seconds: | 20260508 09:29:00.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:00.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.968 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 09:29:00.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.968 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:29:00.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:00.969 | INFO | Uploaded 36GiB to GPU in 2.4831s (14.498GiB/s) | 20260508 09:29:00.969 | INFO | Downloaded 34GiB to GPU in 2.40839s (14.1173GiB/s) | 20260508 09:29:06.106 | INFO | Analyzing chunk frames = [7034880,7035392) | 20260508 09:29:06.106 | INFO | MemInfo: { | 20260508 09:29:06.106 | INFO | "gpu" : " 17 GB", | 20260508 09:29:06.106 | INFO | "huge" : "", | 20260508 09:29:06.106 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:06.106 | INFO | "malloc" : " 14 GB", | 20260508 09:29:06.106 | INFO | "pinned" : " 29 GB" | 20260508 09:29:06.106 | INFO | } | 20260508 09:29:06.106 | INFO | Perf report for the last 5.13778 seconds: | 20260508 09:29:06.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:06.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 09:29:06.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:29:06.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:06.106 | INFO | Uploaded 36GiB to GPU in 2.31069s (15.5798GiB/s) | 20260508 09:29:06.106 | INFO | Downloaded 34GiB to GPU in 2.48779s (13.6668GiB/s) | 20260508 09:29:11.243 | INFO | Analyzing chunk frames = [7035392,7035904) | 20260508 09:29:11.243 | INFO | MemInfo: { | 20260508 09:29:11.243 | INFO | "gpu" : " 17 GB", | 20260508 09:29:11.243 | INFO | "huge" : "", | 20260508 09:29:11.243 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:11.243 | INFO | "malloc" : " 14 GB", | 20260508 09:29:11.243 | INFO | "pinned" : " 29 GB" | 20260508 09:29:11.243 | INFO | } | 20260508 09:29:11.243 | INFO | Perf report for the last 5.13732 seconds: | 20260508 09:29:11.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:11.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 09:29:11.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:29:11.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:11.243 | INFO | Uploaded 36GiB to GPU in 2.40103s (14.9935GiB/s) | 20260508 09:29:11.243 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0224GiB/s) | 20260508 09:29:16.382 | INFO | Analyzing chunk frames = [7035904,7036416) | 20260508 09:29:16.382 | INFO | MemInfo: { | 20260508 09:29:16.382 | INFO | "gpu" : " 17 GB", | 20260508 09:29:16.382 | INFO | "huge" : "", | 20260508 09:29:16.382 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:16.383 | INFO | "malloc" : " 14 GB", | 20260508 09:29:16.383 | INFO | "pinned" : " 29 GB" | 20260508 09:29:16.383 | INFO | } | 20260508 09:29:16.383 | INFO | Perf report for the last 5.13957 seconds: | 20260508 09:29:16.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:16.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:29:16.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:29:16.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:16.383 | INFO | Uploaded 36GiB to GPU in 2.44839s (14.7035GiB/s) | 20260508 09:29:16.383 | INFO | Downloaded 34GiB to GPU in 2.41339s (14.0881GiB/s) | 20260508 09:29:21.525 | INFO | Analyzing chunk frames = [7036416,7036928) | 20260508 09:29:21.525 | INFO | MemInfo: { | 20260508 09:29:21.525 | INFO | "gpu" : " 17 GB", | 20260508 09:29:21.525 | INFO | "huge" : "", | 20260508 09:29:21.525 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:21.525 | INFO | "malloc" : " 14 GB", | 20260508 09:29:21.525 | INFO | "pinned" : " 29 GB" | 20260508 09:29:21.525 | INFO | } | 20260508 09:29:21.525 | INFO | Perf report for the last 5.14253 seconds: | 20260508 09:29:21.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:21.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.525 | INFO | ANALYSIS : 65.8%, 3.0, 103.0ms, 0 | 20260508 09:29:21.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.525 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 09:29:21.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:21.526 | INFO | Uploaded 36GiB to GPU in 2.44659s (14.7144GiB/s) | 20260508 09:29:21.526 | INFO | Downloaded 34GiB to GPU in 2.42822s (14.002GiB/s) | 20260508 09:29:21.526 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:29:21.526 | INFO | Total mem usage: 16.7 GiB | 20260508 09:29:21.526 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:29:21.526 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:29:21.526 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:29:21.526 | INFO | High Water Breakdown: | 20260508 09:29:21.526 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:29:21.526 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:29:21.526 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:29:21.526 | INFO | ------------------------------------------ | 20260508 09:29:21.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:29:21.526 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:29:21.526 | INFO | Total mem usage: 26.5 GiB | 20260508 09:29:21.526 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:29:21.526 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:29:21.526 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:29:21.526 | INFO | High Water Breakdown: | 20260508 09:29:21.526 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:29:21.526 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:29:21.526 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:29:21.526 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:29:21.526 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:29:21.526 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:29:21.526 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:29:21.526 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:29:21.526 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:29:21.526 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:29:21.526 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:29:21.526 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:29:21.526 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:29:21.526 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:29:21.526 | INFO | ------------------------------------------ | 20260508 09:29:21.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:29:21.526 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:29:21.526 | INFO | Total mem usage: 27.8 GiB | 20260508 09:29:21.526 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:29:21.526 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:29:21.526 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:29:21.526 | INFO | High Water Breakdown: | 20260508 09:29:21.526 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:29:21.526 | INFO | ------------------------------------------ | 20260508 09:29:21.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:29:21.526 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:29:21.526 | INFO | Total mem usage: 15.7 GiB | 20260508 09:29:21.526 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:29:21.526 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:29:21.526 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:29:21.526 | INFO | High Water Breakdown: | 20260508 09:29:21.526 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:29:21.526 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:29:21.526 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:29:21.526 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:29:21.526 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:29:21.526 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:29:21.526 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:29:21.526 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:29:21.526 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:29:21.526 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:29:21.526 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:29:21.526 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:29:21.526 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:29:21.526 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:29:21.526 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:29:21.526 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:29:21.526 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:29:21.526 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:29:21.526 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:29:21.526 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:29:21.526 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:29:21.526 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:29:21.526 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:29:21.527 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:29:21.527 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:29:21.527 | INFO | ------------------------------------------ | 20260508 09:29:21.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:29:21.527 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:29:21.527 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:29:21.527 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:29:21.527 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 7.45e-01 | 20260508 09:29:21.527 | INFO | PerfReport: 0.29%, AggregateStats -- 9.65e-02 / 1.05e-01, 1.16e-02 / 6.47e-03 | 20260508 09:29:21.527 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.36e+00, 5.96e-01 / 2.54e-01 | 20260508 09:29:21.527 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.42e+00, 2.10e-01 / 1.44e-01 | 20260508 09:29:21.527 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.52e+01, 3.66e+00 / 1.26e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.82e+00 / 1.39e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.42e+00, 4.05e-01 / 2.81e-01 | 20260508 09:29:21.527 | INFO | PerfReport: 3.05%, Metrics -- 9.97e-01 / 1.12e+00, 2.46e-01 / 1.18e-01 | 20260508 09:29:21.527 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.37e-01, 4.85e-02 / 2.99e-02 | 20260508 09:29:21.527 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:29:21.527 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:29:21.527 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:29:21.527 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 24.46%, Upload -- 2.58e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 24.42%, Download -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 2.37%, LockUpload -- 2.50e+00 / 0.00e+00, 5.81e+00 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 2.36%, LockDownload -- 2.49e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 15.33%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:29:21.527 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.15e-02 / 0.00e+00 | 20260508 09:29:21.622 | INFO | Starting a round of Baz IO | 20260508 09:29:25.041 | INFO | Populating RTMetrics report | 20260508 09:29:25.041 | INFO | RTMetrics report populated | 20260508 09:29:25.041 | INFO | Opening tmp RTMetrics json file | 20260508 09:29:25.048 | INFO | Serializing RTMetrics to json | 20260508 09:29:25.050 | INFO | Serialization done, swapping tmp file now... | 20260508 09:29:25.050 | INFO | RTMetrics done | 20260508 09:29:25.106 | INFO | Finished a round of Baz IO: Wrote 1.484 GiB in 3.484 seconds | 20260508 09:29:25.106 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:29:25.106 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.256236% / 0% / 9.13938e-05% | 20260508 09:29:25.106 | INFO | Baz IO overall: | 20260508 09:29:25.106 | INFO | BytesWritten : 3070225203699 | 20260508 09:29:25.106 | INFO | NumEvents : 1476893477513 | 20260508 09:29:26.662 | INFO | Analyzing chunk frames = [7036928,7037440) | 20260508 09:29:26.662 | INFO | MemInfo: { | 20260508 09:29:26.662 | INFO | "gpu" : " 17 GB", | 20260508 09:29:26.662 | INFO | "huge" : "", | 20260508 09:29:26.662 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:26.662 | INFO | "malloc" : " 14 GB", | 20260508 09:29:26.662 | INFO | "pinned" : " 29 GB" | 20260508 09:29:26.662 | INFO | } | 20260508 09:29:26.662 | INFO | Perf report for the last 5.13693 seconds: | 20260508 09:29:26.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:26.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:26.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:26.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:26.662 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 09:29:26.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:26.662 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:29:26.662 | INFO | PRE_HQ : 65.2%, 1.6, 54.5ms, 0 | 20260508 09:29:26.662 | INFO | BAZWRITER : 67.1%, 2.5, 87.0ms, 0 | 20260508 09:29:26.662 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:29:26.663 | INFO | Uploaded 36GiB to GPU in 2.6057s (13.8159GiB/s) | 20260508 09:29:26.663 | INFO | Downloaded 38GiB to GPU in 2.71076s (14.0182GiB/s) | 20260508 09:29:31.807 | INFO | Analyzing chunk frames = [7037440,7037952) | 20260508 09:29:31.807 | INFO | MemInfo: { | 20260508 09:29:31.807 | INFO | "gpu" : " 17 GB", | 20260508 09:29:31.808 | INFO | "huge" : "", | 20260508 09:29:31.808 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:31.808 | INFO | "malloc" : " 14 GB", | 20260508 09:29:31.808 | INFO | "pinned" : " 29 GB" | 20260508 09:29:31.808 | INFO | } | 20260508 09:29:31.808 | INFO | Perf report for the last 5.14548 seconds: | 20260508 09:29:31.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:31.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:29:31.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:29:31.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:31.808 | INFO | Uploaded 36GiB to GPU in 2.52542s (14.2551GiB/s) | 20260508 09:29:31.808 | INFO | Downloaded 34GiB to GPU in 2.46874s (13.7722GiB/s) | 20260508 09:29:36.945 | INFO | Analyzing chunk frames = [7037952,7038464) | 20260508 09:29:36.945 | INFO | MemInfo: { | 20260508 09:29:36.945 | INFO | "gpu" : " 17 GB", | 20260508 09:29:36.945 | INFO | "huge" : "", | 20260508 09:29:36.945 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:36.945 | INFO | "malloc" : " 14 GB", | 20260508 09:29:36.945 | INFO | "pinned" : " 29 GB" | 20260508 09:29:36.945 | INFO | } | 20260508 09:29:36.945 | INFO | Perf report for the last 5.13734 seconds: | 20260508 09:29:36.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:36.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.945 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 09:29:36.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.945 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:29:36.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:36.946 | INFO | Uploaded 36GiB to GPU in 2.44453s (14.7267GiB/s) | 20260508 09:29:36.946 | INFO | Downloaded 34GiB to GPU in 2.4308s (13.9872GiB/s) | 20260508 09:29:42.082 | INFO | Analyzing chunk frames = [7038464,7038976) | 20260508 09:29:42.082 | INFO | MemInfo: { | 20260508 09:29:42.082 | INFO | "gpu" : " 17 GB", | 20260508 09:29:42.082 | INFO | "huge" : "", | 20260508 09:29:42.082 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:42.082 | INFO | "malloc" : " 14 GB", | 20260508 09:29:42.082 | INFO | "pinned" : " 29 GB" | 20260508 09:29:42.082 | INFO | } | 20260508 09:29:42.082 | INFO | Perf report for the last 5.137 seconds: | 20260508 09:29:42.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:42.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 09:29:42.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:29:42.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:42.082 | INFO | Uploaded 36GiB to GPU in 2.45613s (14.6572GiB/s) | 20260508 09:29:42.082 | INFO | Downloaded 34GiB to GPU in 2.41556s (14.0754GiB/s) | 20260508 09:29:47.215 | INFO | Analyzing chunk frames = [7038976,7039488) | 20260508 09:29:47.215 | INFO | MemInfo: { | 20260508 09:29:47.215 | INFO | "gpu" : " 17 GB", | 20260508 09:29:47.215 | INFO | "huge" : "", | 20260508 09:29:47.215 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:47.215 | INFO | "malloc" : " 14 GB", | 20260508 09:29:47.215 | INFO | "pinned" : " 29 GB" | 20260508 09:29:47.215 | INFO | } | 20260508 09:29:47.215 | INFO | Perf report for the last 5.13316 seconds: | 20260508 09:29:47.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:47.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.215 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 09:29:47.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.215 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:29:47.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:47.216 | INFO | Uploaded 36GiB to GPU in 2.38104s (15.1194GiB/s) | 20260508 09:29:47.216 | INFO | Downloaded 34GiB to GPU in 2.54539s (13.3575GiB/s) | 20260508 09:29:52.360 | INFO | Analyzing chunk frames = [7039488,7040000) | 20260508 09:29:52.360 | INFO | MemInfo: { | 20260508 09:29:52.360 | INFO | "gpu" : " 17 GB", | 20260508 09:29:52.360 | INFO | "huge" : "", | 20260508 09:29:52.360 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:52.360 | INFO | "malloc" : " 14 GB", | 20260508 09:29:52.360 | INFO | "pinned" : " 29 GB" | 20260508 09:29:52.360 | INFO | } | 20260508 09:29:52.360 | INFO | Perf report for the last 5.14453 seconds: | 20260508 09:29:52.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:52.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 09:29:52.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:29:52.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:52.360 | INFO | Uploaded 36GiB to GPU in 2.32205s (15.5035GiB/s) | 20260508 09:29:52.360 | INFO | Downloaded 34GiB to GPU in 2.45062s (13.8741GiB/s) | 20260508 09:29:57.500 | INFO | Analyzing chunk frames = [7040000,7040512) | 20260508 09:29:57.500 | INFO | MemInfo: { | 20260508 09:29:57.500 | INFO | "gpu" : " 17 GB", | 20260508 09:29:57.501 | INFO | "huge" : "", | 20260508 09:29:57.501 | INFO | "huge_shared" : " 29 GB", | 20260508 09:29:57.501 | INFO | "malloc" : " 14 GB", | 20260508 09:29:57.501 | INFO | "pinned" : " 29 GB" | 20260508 09:29:57.501 | INFO | } | 20260508 09:29:57.501 | INFO | Perf report for the last 5.14091 seconds: | 20260508 09:29:57.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:29:57.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:29:57.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 09:29:57.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:29:57.501 | INFO | Uploaded 36GiB to GPU in 2.39152s (15.0532GiB/s) | 20260508 09:29:57.501 | INFO | Downloaded 34GiB to GPU in 2.45908s (13.8263GiB/s) | 20260508 09:30:02.641 | INFO | Analyzing chunk frames = [7040512,7041024) | 20260508 09:30:02.641 | INFO | MemInfo: { | 20260508 09:30:02.641 | INFO | "gpu" : " 17 GB", | 20260508 09:30:02.641 | INFO | "huge" : "", | 20260508 09:30:02.641 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:02.641 | INFO | "malloc" : " 14 GB", | 20260508 09:30:02.641 | INFO | "pinned" : " 29 GB" | 20260508 09:30:02.641 | INFO | } | 20260508 09:30:02.641 | INFO | Perf report for the last 5.14049 seconds: | 20260508 09:30:02.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:02.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:30:02.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:30:02.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:02.641 | INFO | Uploaded 36GiB to GPU in 2.40891s (14.9445GiB/s) | 20260508 09:30:02.641 | INFO | Downloaded 34GiB to GPU in 2.46104s (13.8153GiB/s) | 20260508 09:30:02.738 | INFO | Starting a round of Baz IO | 20260508 09:30:06.147 | INFO | Populating RTMetrics report | 20260508 09:30:06.147 | INFO | RTMetrics report populated | 20260508 09:30:06.147 | INFO | Opening tmp RTMetrics json file | 20260508 09:30:06.158 | INFO | Serializing RTMetrics to json | 20260508 09:30:06.160 | INFO | Serialization done, swapping tmp file now... | 20260508 09:30:06.160 | INFO | RTMetrics done | 20260508 09:30:06.219 | INFO | Finished a round of Baz IO: Wrote 1.482 GiB in 3.481 seconds | 20260508 09:30:06.219 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:30:06.219 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255287% / 7.94729e-06% / 5.16574e-05% | 20260508 09:30:06.219 | INFO | Baz IO overall: | 20260508 09:30:06.219 | INFO | BytesWritten : 3071816604722 | 20260508 09:30:06.219 | INFO | NumEvents : 1477645794370 | 20260508 09:30:07.779 | INFO | Analyzing chunk frames = [7041024,7041536) | 20260508 09:30:07.779 | INFO | MemInfo: { | 20260508 09:30:07.779 | INFO | "gpu" : " 17 GB", | 20260508 09:30:07.779 | INFO | "huge" : "", | 20260508 09:30:07.780 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:07.780 | INFO | "malloc" : " 14 GB", | 20260508 09:30:07.780 | INFO | "pinned" : " 29 GB" | 20260508 09:30:07.780 | INFO | } | 20260508 09:30:07.780 | INFO | Perf report for the last 5.13843 seconds: | 20260508 09:30:07.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:07.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:07.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:07.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:07.780 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 09:30:07.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:07.780 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 09:30:07.780 | INFO | PRE_HQ : 65.3%, 1.6, 54.6ms, 0 | 20260508 09:30:07.780 | INFO | BAZWRITER : 67.1%, 2.5, 87.4ms, 0 | 20260508 09:30:07.780 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:30:07.780 | INFO | Uploaded 36GiB to GPU in 2.56824s (14.0174GiB/s) | 20260508 09:30:07.780 | INFO | Downloaded 38GiB to GPU in 2.70478s (14.0492GiB/s) | 20260508 09:30:12.914 | INFO | Analyzing chunk frames = [7041536,7042048) | 20260508 09:30:12.914 | INFO | MemInfo: { | 20260508 09:30:12.914 | INFO | "gpu" : " 17 GB", | 20260508 09:30:12.914 | INFO | "huge" : "", | 20260508 09:30:12.914 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:12.914 | INFO | "malloc" : " 14 GB", | 20260508 09:30:12.914 | INFO | "pinned" : " 29 GB" | 20260508 09:30:12.914 | INFO | } | 20260508 09:30:12.914 | INFO | Perf report for the last 5.1342 seconds: | 20260508 09:30:12.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:12.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:30:12.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:30:12.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:12.914 | INFO | Uploaded 36GiB to GPU in 2.5271s (14.2456GiB/s) | 20260508 09:30:12.914 | INFO | Downloaded 34GiB to GPU in 2.46554s (13.7901GiB/s) | 20260508 09:30:18.056 | INFO | Analyzing chunk frames = [7042048,7042560) | 20260508 09:30:18.056 | INFO | MemInfo: { | 20260508 09:30:18.056 | INFO | "gpu" : " 17 GB", | 20260508 09:30:18.056 | INFO | "huge" : "", | 20260508 09:30:18.056 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:18.057 | INFO | "malloc" : " 14 GB", | 20260508 09:30:18.057 | INFO | "pinned" : " 29 GB" | 20260508 09:30:18.057 | INFO | } | 20260508 09:30:18.057 | INFO | Perf report for the last 5.14288 seconds: | 20260508 09:30:18.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:18.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 09:30:18.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:30:18.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:18.057 | INFO | Uploaded 36GiB to GPU in 2.49799s (14.4116GiB/s) | 20260508 09:30:18.057 | INFO | Downloaded 34GiB to GPU in 2.43272s (13.9761GiB/s) | 20260508 09:30:23.196 | INFO | Analyzing chunk frames = [7042560,7043072) | 20260508 09:30:23.196 | INFO | MemInfo: { | 20260508 09:30:23.196 | INFO | "gpu" : " 17 GB", | 20260508 09:30:23.196 | INFO | "huge" : "", | 20260508 09:30:23.196 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:23.196 | INFO | "malloc" : " 14 GB", | 20260508 09:30:23.196 | INFO | "pinned" : " 29 GB" | 20260508 09:30:23.196 | INFO | } | 20260508 09:30:23.196 | INFO | Perf report for the last 5.13969 seconds: | 20260508 09:30:23.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:23.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:30:23.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:30:23.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:23.197 | INFO | Uploaded 36GiB to GPU in 2.49533s (14.427GiB/s) | 20260508 09:30:23.197 | INFO | Downloaded 34GiB to GPU in 2.43321s (13.9733GiB/s) | 20260508 09:30:28.339 | INFO | Analyzing chunk frames = [7043072,7043584) | 20260508 09:30:28.339 | INFO | MemInfo: { | 20260508 09:30:28.339 | INFO | "gpu" : " 17 GB", | 20260508 09:30:28.339 | INFO | "huge" : "", | 20260508 09:30:28.339 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:28.339 | INFO | "malloc" : " 14 GB", | 20260508 09:30:28.339 | INFO | "pinned" : " 29 GB" | 20260508 09:30:28.339 | INFO | } | 20260508 09:30:28.339 | INFO | Perf report for the last 5.14302 seconds: | 20260508 09:30:28.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:28.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.340 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 09:30:28.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.340 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:30:28.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:28.340 | INFO | Uploaded 36GiB to GPU in 2.4325s (14.7996GiB/s) | 20260508 09:30:28.340 | INFO | Downloaded 34GiB to GPU in 2.52977s (13.44GiB/s) | 20260508 09:30:33.471 | INFO | Analyzing chunk frames = [7043584,7044096) | 20260508 09:30:33.471 | INFO | MemInfo: { | 20260508 09:30:33.471 | INFO | "gpu" : " 17 GB", | 20260508 09:30:33.471 | INFO | "huge" : "", | 20260508 09:30:33.471 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:33.471 | INFO | "malloc" : " 14 GB", | 20260508 09:30:33.471 | INFO | "pinned" : " 29 GB" | 20260508 09:30:33.472 | INFO | } | 20260508 09:30:33.472 | INFO | Perf report for the last 5.13221 seconds: | 20260508 09:30:33.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:33.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 09:30:33.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:30:33.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:33.472 | INFO | Uploaded 36GiB to GPU in 2.37052s (15.1865GiB/s) | 20260508 09:30:33.472 | INFO | Downloaded 34GiB to GPU in 2.41262s (14.0926GiB/s) | 20260508 09:30:38.615 | INFO | Analyzing chunk frames = [7044096,7044608) | 20260508 09:30:38.615 | INFO | MemInfo: { | 20260508 09:30:38.615 | INFO | "gpu" : " 17 GB", | 20260508 09:30:38.615 | INFO | "huge" : "", | 20260508 09:30:38.615 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:38.615 | INFO | "malloc" : " 14 GB", | 20260508 09:30:38.615 | INFO | "pinned" : " 29 GB" | 20260508 09:30:38.615 | INFO | } | 20260508 09:30:38.615 | INFO | Perf report for the last 5.14305 seconds: | 20260508 09:30:38.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:38.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 09:30:38.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:30:38.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:38.615 | INFO | Uploaded 36GiB to GPU in 2.45149s (14.685GiB/s) | 20260508 09:30:38.615 | INFO | Downloaded 34GiB to GPU in 2.4268s (14.0102GiB/s) | 20260508 09:30:43.753 | INFO | Analyzing chunk frames = [7044608,7045120) | 20260508 09:30:43.753 | INFO | MemInfo: { | 20260508 09:30:43.753 | INFO | "gpu" : " 17 GB", | 20260508 09:30:43.753 | INFO | "huge" : "", | 20260508 09:30:43.753 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:43.753 | INFO | "malloc" : " 14 GB", | 20260508 09:30:43.753 | INFO | "pinned" : " 29 GB" | 20260508 09:30:43.753 | INFO | } | 20260508 09:30:43.753 | INFO | Perf report for the last 5.13831 seconds: | 20260508 09:30:43.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:43.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.753 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:30:43.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.753 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:30:43.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:43.754 | INFO | Uploaded 36GiB to GPU in 2.44513s (14.7231GiB/s) | 20260508 09:30:43.754 | INFO | Downloaded 34GiB to GPU in 2.43244s (13.9777GiB/s) | 20260508 09:30:43.754 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:30:43.754 | INFO | Total mem usage: 16.7 GiB | 20260508 09:30:43.754 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:30:43.754 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:30:43.754 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:30:43.754 | INFO | High Water Breakdown: | 20260508 09:30:43.754 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:30:43.754 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:30:43.754 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:30:43.754 | INFO | ------------------------------------------ | 20260508 09:30:43.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:30:43.754 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:30:43.754 | INFO | Total mem usage: 26.5 GiB | 20260508 09:30:43.754 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:30:43.754 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:30:43.754 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:30:43.754 | INFO | High Water Breakdown: | 20260508 09:30:43.754 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:30:43.754 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:30:43.754 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:30:43.754 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:30:43.754 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:30:43.754 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:30:43.754 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:30:43.754 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:30:43.754 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:30:43.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:30:43.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:30:43.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:30:43.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:30:43.754 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:30:43.754 | INFO | ------------------------------------------ | 20260508 09:30:43.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:30:43.754 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:30:43.754 | INFO | Total mem usage: 27.8 GiB | 20260508 09:30:43.754 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:30:43.754 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:30:43.754 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:30:43.754 | INFO | High Water Breakdown: | 20260508 09:30:43.754 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:30:43.754 | INFO | ------------------------------------------ | 20260508 09:30:43.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:30:43.754 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:30:43.754 | INFO | Total mem usage: 15.7 GiB | 20260508 09:30:43.754 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:30:43.754 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:30:43.754 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:30:43.754 | INFO | High Water Breakdown: | 20260508 09:30:43.755 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:30:43.755 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:30:43.755 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:30:43.755 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:30:43.755 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:30:43.755 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:30:43.755 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:30:43.755 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:30:43.755 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:30:43.755 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:30:43.755 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:30:43.755 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:30:43.755 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:30:43.755 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:30:43.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:30:43.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:30:43.755 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:30:43.755 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:30:43.755 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:30:43.755 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:30:43.755 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:30:43.755 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:30:43.755 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:30:43.755 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:30:43.755 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:30:43.755 | INFO | ------------------------------------------ | 20260508 09:30:43.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:30:43.755 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:30:43.755 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:30:43.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:30:43.755 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 5.58e-01 | 20260508 09:30:43.755 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.02e-01, 1.10e-02 / 5.62e-03 | 20260508 09:30:43.755 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.91e-01 / 2.04e-01 | 20260508 09:30:43.755 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.43e+00, 2.09e-01 / 1.18e-01 | 20260508 09:30:43.755 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.50e+01, 3.68e+00 / 1.08e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.85e+00 / 1.24e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.49e+00, 4.06e-01 / 2.75e-01 | 20260508 09:30:43.755 | INFO | PerfReport: 3.04%, Metrics -- 9.94e-01 / 1.14e+00, 2.46e-01 / 1.19e-01 | 20260508 09:30:43.755 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.30e-01, 4.86e-02 / 2.41e-02 | 20260508 09:30:43.755 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:30:43.755 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:30:43.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:30:43.755 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 24.30%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 2.14%, LockUpload -- 2.25e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 2.56%, LockDownload -- 2.70e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 09:30:43.755 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 09:30:43.854 | INFO | Starting a round of Baz IO | 20260508 09:30:47.263 | INFO | Populating RTMetrics report | 20260508 09:30:47.263 | INFO | RTMetrics report populated | 20260508 09:30:47.263 | INFO | Opening tmp RTMetrics json file | 20260508 09:30:47.274 | INFO | Serializing RTMetrics to json | 20260508 09:30:47.275 | INFO | Serialization done, swapping tmp file now... | 20260508 09:30:47.275 | INFO | RTMetrics done | 20260508 09:30:47.334 | INFO | Finished a round of Baz IO: Wrote 1.482 GiB in 3.48 seconds | 20260508 09:30:47.334 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:30:47.334 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.254957% / 0% / 8.34465e-05% | 20260508 09:30:47.334 | INFO | Baz IO overall: | 20260508 09:30:47.335 | INFO | BytesWritten : 3073408133766 | 20260508 09:30:47.335 | INFO | NumEvents : 1478398248091 | 20260508 09:30:48.893 | INFO | Analyzing chunk frames = [7045120,7045632) | 20260508 09:30:48.893 | INFO | MemInfo: { | 20260508 09:30:48.893 | INFO | "gpu" : " 17 GB", | 20260508 09:30:48.893 | INFO | "huge" : "", | 20260508 09:30:48.893 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:48.893 | INFO | "malloc" : " 14 GB", | 20260508 09:30:48.893 | INFO | "pinned" : " 29 GB" | 20260508 09:30:48.893 | INFO | } | 20260508 09:30:48.893 | INFO | Perf report for the last 5.13979 seconds: | 20260508 09:30:48.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:48.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:48.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:48.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:48.893 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:30:48.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:48.893 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:30:48.893 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 09:30:48.893 | INFO | BAZWRITER : 67.2%, 2.5, 87.2ms, 0 | 20260508 09:30:48.893 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:30:48.893 | INFO | Uploaded 36GiB to GPU in 2.58766s (13.9122GiB/s) | 20260508 09:30:48.893 | INFO | Downloaded 38GiB to GPU in 2.70865s (14.0291GiB/s) | 20260508 09:30:54.030 | INFO | Analyzing chunk frames = [7045632,7046144) | 20260508 09:30:54.030 | INFO | MemInfo: { | 20260508 09:30:54.030 | INFO | "gpu" : " 17 GB", | 20260508 09:30:54.030 | INFO | "huge" : "", | 20260508 09:30:54.030 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:54.030 | INFO | "malloc" : " 14 GB", | 20260508 09:30:54.030 | INFO | "pinned" : " 29 GB" | 20260508 09:30:54.030 | INFO | } | 20260508 09:30:54.030 | INFO | Perf report for the last 5.13753 seconds: | 20260508 09:30:54.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:54.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.031 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:30:54.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.031 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:30:54.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:54.031 | INFO | Uploaded 36GiB to GPU in 2.55341s (14.0988GiB/s) | 20260508 09:30:54.031 | INFO | Downloaded 34GiB to GPU in 2.4585s (13.8295GiB/s) | 20260508 09:30:59.167 | INFO | Analyzing chunk frames = [7046144,7046656) | 20260508 09:30:59.167 | INFO | MemInfo: { | 20260508 09:30:59.167 | INFO | "gpu" : " 17 GB", | 20260508 09:30:59.167 | INFO | "huge" : "", | 20260508 09:30:59.167 | INFO | "huge_shared" : " 29 GB", | 20260508 09:30:59.167 | INFO | "malloc" : " 14 GB", | 20260508 09:30:59.167 | INFO | "pinned" : " 29 GB" | 20260508 09:30:59.167 | INFO | } | 20260508 09:30:59.168 | INFO | Perf report for the last 5.13707 seconds: | 20260508 09:30:59.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:30:59.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 09:30:59.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:30:59.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:30:59.168 | INFO | Uploaded 36GiB to GPU in 2.5015s (14.3914GiB/s) | 20260508 09:30:59.168 | INFO | Downloaded 34GiB to GPU in 2.43295s (13.9748GiB/s) | 20260508 09:31:04.309 | INFO | Analyzing chunk frames = [7046656,7047168) | 20260508 09:31:04.309 | INFO | MemInfo: { | 20260508 09:31:04.309 | INFO | "gpu" : " 17 GB", | 20260508 09:31:04.309 | INFO | "huge" : "", | 20260508 09:31:04.309 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:04.309 | INFO | "malloc" : " 14 GB", | 20260508 09:31:04.309 | INFO | "pinned" : " 29 GB" | 20260508 09:31:04.309 | INFO | } | 20260508 09:31:04.310 | INFO | Perf report for the last 5.14193 seconds: | 20260508 09:31:04.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:04.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | ANALYSIS : 67.1%, 2.9, 103.8ms, 0 | 20260508 09:31:04.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 09:31:04.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:04.310 | INFO | Uploaded 36GiB to GPU in 2.59765s (13.8587GiB/s) | 20260508 09:31:04.310 | INFO | Downloaded 34GiB to GPU in 2.43176s (13.9816GiB/s) | 20260508 09:31:09.450 | INFO | Analyzing chunk frames = [7047168,7047680) | 20260508 09:31:09.450 | INFO | MemInfo: { | 20260508 09:31:09.450 | INFO | "gpu" : " 17 GB", | 20260508 09:31:09.450 | INFO | "huge" : "", | 20260508 09:31:09.450 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:09.450 | INFO | "malloc" : " 14 GB", | 20260508 09:31:09.450 | INFO | "pinned" : " 29 GB" | 20260508 09:31:09.450 | INFO | } | 20260508 09:31:09.450 | INFO | Perf report for the last 5.14075 seconds: | 20260508 09:31:09.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:09.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.450 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 09:31:09.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.450 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 09:31:09.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:09.451 | INFO | Uploaded 36GiB to GPU in 2.49761s (14.4138GiB/s) | 20260508 09:31:09.451 | INFO | Downloaded 34GiB to GPU in 2.53847s (13.3939GiB/s) | 20260508 09:31:14.594 | INFO | Analyzing chunk frames = [7047680,7048192) | 20260508 09:31:14.594 | INFO | MemInfo: { | 20260508 09:31:14.594 | INFO | "gpu" : " 17 GB", | 20260508 09:31:14.594 | INFO | "huge" : "", | 20260508 09:31:14.594 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:14.594 | INFO | "malloc" : " 14 GB", | 20260508 09:31:14.594 | INFO | "pinned" : " 29 GB" | 20260508 09:31:14.594 | INFO | } | 20260508 09:31:14.594 | INFO | Perf report for the last 5.14384 seconds: | 20260508 09:31:14.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:14.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.594 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 09:31:14.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.594 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:31:14.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:14.595 | INFO | Uploaded 36GiB to GPU in 2.3872s (15.0805GiB/s) | 20260508 09:31:14.595 | INFO | Downloaded 34GiB to GPU in 2.4382s (13.9447GiB/s) | 20260508 09:31:19.726 | INFO | Analyzing chunk frames = [7048192,7048704) | 20260508 09:31:19.726 | INFO | MemInfo: { | 20260508 09:31:19.726 | INFO | "gpu" : " 17 GB", | 20260508 09:31:19.726 | INFO | "huge" : "", | 20260508 09:31:19.726 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:19.726 | INFO | "malloc" : " 14 GB", | 20260508 09:31:19.726 | INFO | "pinned" : " 29 GB" | 20260508 09:31:19.726 | INFO | } | 20260508 09:31:19.726 | INFO | Perf report for the last 5.1322 seconds: | 20260508 09:31:19.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:19.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.726 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:31:19.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.726 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:31:19.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:19.727 | INFO | Uploaded 36GiB to GPU in 2.42506s (14.845GiB/s) | 20260508 09:31:19.727 | INFO | Downloaded 34GiB to GPU in 2.43245s (13.9777GiB/s) | 20260508 09:31:24.868 | INFO | Analyzing chunk frames = [7048704,7049216) | 20260508 09:31:24.868 | INFO | MemInfo: { | 20260508 09:31:24.868 | INFO | "gpu" : " 17 GB", | 20260508 09:31:24.868 | INFO | "huge" : "", | 20260508 09:31:24.868 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:24.868 | INFO | "malloc" : " 14 GB", | 20260508 09:31:24.868 | INFO | "pinned" : " 29 GB" | 20260508 09:31:24.868 | INFO | } | 20260508 09:31:24.868 | INFO | Perf report for the last 5.14184 seconds: | 20260508 09:31:24.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:24.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.868 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 09:31:24.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.868 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:31:24.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:24.869 | INFO | Uploaded 36GiB to GPU in 2.43342s (14.794GiB/s) | 20260508 09:31:24.869 | INFO | Downloaded 34GiB to GPU in 2.4399s (13.935GiB/s) | 20260508 09:31:24.966 | INFO | Starting a round of Baz IO | 20260508 09:31:28.383 | INFO | Populating RTMetrics report | 20260508 09:31:28.383 | INFO | RTMetrics report populated | 20260508 09:31:28.383 | INFO | Opening tmp RTMetrics json file | 20260508 09:31:28.389 | INFO | Serializing RTMetrics to json | 20260508 09:31:28.391 | INFO | Serialization done, swapping tmp file now... | 20260508 09:31:28.392 | INFO | RTMetrics done | 20260508 09:31:28.450 | INFO | Finished a round of Baz IO: Wrote 1.484 GiB in 3.484 seconds | 20260508 09:31:28.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.00% / 59.70% | 20260508 09:31:28.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255199% / 0% / 5.16574e-05% | 20260508 09:31:28.450 | INFO | Baz IO overall: | 20260508 09:31:28.450 | INFO | BytesWritten : 3075001598525 | 20260508 09:31:28.450 | INFO | NumEvents : 1479152200089 | 20260508 09:31:30.007 | INFO | Analyzing chunk frames = [7049216,7049728) | 20260508 09:31:30.007 | INFO | MemInfo: { | 20260508 09:31:30.007 | INFO | "gpu" : " 17 GB", | 20260508 09:31:30.007 | INFO | "huge" : "", | 20260508 09:31:30.007 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:30.007 | INFO | "malloc" : " 14 GB", | 20260508 09:31:30.007 | INFO | "pinned" : " 29 GB" | 20260508 09:31:30.007 | INFO | } | 20260508 09:31:30.007 | INFO | Perf report for the last 5.13903 seconds: | 20260508 09:31:30.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:30.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:30.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:30.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:30.007 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 09:31:30.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:30.007 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:31:30.007 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 09:31:30.007 | INFO | BAZWRITER : 66.9%, 2.5, 87.2ms, 0 | 20260508 09:31:30.007 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:31:30.007 | INFO | Uploaded 36GiB to GPU in 2.58281s (13.9383GiB/s) | 20260508 09:31:30.007 | INFO | Downloaded 38GiB to GPU in 2.70126s (14.0675GiB/s) | 20260508 09:31:35.147 | INFO | Analyzing chunk frames = [7049728,7050240) | 20260508 09:31:35.147 | INFO | MemInfo: { | 20260508 09:31:35.147 | INFO | "gpu" : " 17 GB", | 20260508 09:31:35.147 | INFO | "huge" : "", | 20260508 09:31:35.147 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:35.147 | INFO | "malloc" : " 14 GB", | 20260508 09:31:35.147 | INFO | "pinned" : " 29 GB" | 20260508 09:31:35.147 | INFO | } | 20260508 09:31:35.147 | INFO | Perf report for the last 5.14003 seconds: | 20260508 09:31:35.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:35.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.147 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:31:35.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.147 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:31:35.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:35.148 | INFO | Uploaded 36GiB to GPU in 2.51147s (14.3342GiB/s) | 20260508 09:31:35.148 | INFO | Downloaded 34GiB to GPU in 2.44796s (13.8891GiB/s) | 20260508 09:31:40.285 | INFO | Analyzing chunk frames = [7050240,7050752) | 20260508 09:31:40.285 | INFO | MemInfo: { | 20260508 09:31:40.285 | INFO | "gpu" : " 17 GB", | 20260508 09:31:40.285 | INFO | "huge" : "", | 20260508 09:31:40.285 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:40.285 | INFO | "malloc" : " 14 GB", | 20260508 09:31:40.285 | INFO | "pinned" : " 29 GB" | 20260508 09:31:40.285 | INFO | } | 20260508 09:31:40.285 | INFO | Perf report for the last 5.13817 seconds: | 20260508 09:31:40.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:40.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.286 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 09:31:40.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.286 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 09:31:40.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:40.286 | INFO | Uploaded 36GiB to GPU in 2.49069s (14.4538GiB/s) | 20260508 09:31:40.286 | INFO | Downloaded 34GiB to GPU in 2.44022s (13.9332GiB/s) | 20260508 09:31:45.424 | INFO | Analyzing chunk frames = [7050752,7051264) | 20260508 09:31:45.424 | INFO | MemInfo: { | 20260508 09:31:45.424 | INFO | "gpu" : " 17 GB", | 20260508 09:31:45.424 | INFO | "huge" : "", | 20260508 09:31:45.424 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:45.424 | INFO | "malloc" : " 14 GB", | 20260508 09:31:45.424 | INFO | "pinned" : " 29 GB" | 20260508 09:31:45.424 | INFO | } | 20260508 09:31:45.425 | INFO | Perf report for the last 5.1391 seconds: | 20260508 09:31:45.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:45.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 09:31:45.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260508 09:31:45.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:45.425 | INFO | Uploaded 36GiB to GPU in 2.47954s (14.5188GiB/s) | 20260508 09:31:45.425 | INFO | Downloaded 34GiB to GPU in 2.40899s (14.1138GiB/s) | 20260508 09:31:50.567 | INFO | Analyzing chunk frames = [7051264,7051776) | 20260508 09:31:50.567 | INFO | MemInfo: { | 20260508 09:31:50.567 | INFO | "gpu" : " 17 GB", | 20260508 09:31:50.567 | INFO | "huge" : "", | 20260508 09:31:50.567 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:50.567 | INFO | "malloc" : " 14 GB", | 20260508 09:31:50.567 | INFO | "pinned" : " 29 GB" | 20260508 09:31:50.567 | INFO | } | 20260508 09:31:50.567 | INFO | Perf report for the last 5.14252 seconds: | 20260508 09:31:50.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:50.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 09:31:50.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | PRE_HQ : 31.7%, 1.0, 16.6ms, 0 | 20260508 09:31:50.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:50.567 | INFO | Uploaded 36GiB to GPU in 2.39991s (15.0006GiB/s) | 20260508 09:31:50.567 | INFO | Downloaded 34GiB to GPU in 2.50356s (13.5807GiB/s) | 20260508 09:31:55.710 | INFO | Analyzing chunk frames = [7051776,7052288) | 20260508 09:31:55.710 | INFO | MemInfo: { | 20260508 09:31:55.710 | INFO | "gpu" : " 17 GB", | 20260508 09:31:55.711 | INFO | "huge" : "", | 20260508 09:31:55.711 | INFO | "huge_shared" : " 29 GB", | 20260508 09:31:55.711 | INFO | "malloc" : " 14 GB", | 20260508 09:31:55.711 | INFO | "pinned" : " 29 GB" | 20260508 09:31:55.711 | INFO | } | 20260508 09:31:55.711 | INFO | Perf report for the last 5.14359 seconds: | 20260508 09:31:55.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:31:55.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:31:55.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:31:55.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:31:55.711 | INFO | Uploaded 36GiB to GPU in 2.38868s (15.0711GiB/s) | 20260508 09:31:55.711 | INFO | Downloaded 34GiB to GPU in 2.4164s (14.0705GiB/s) | 20260508 09:32:00.841 | INFO | Analyzing chunk frames = [7052288,7052800) | 20260508 09:32:00.841 | INFO | MemInfo: { | 20260508 09:32:00.841 | INFO | "gpu" : " 17 GB", | 20260508 09:32:00.841 | INFO | "huge" : "", | 20260508 09:32:00.841 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:00.841 | INFO | "malloc" : " 14 GB", | 20260508 09:32:00.841 | INFO | "pinned" : " 29 GB" | 20260508 09:32:00.841 | INFO | } | 20260508 09:32:00.841 | INFO | Perf report for the last 5.13052 seconds: | 20260508 09:32:00.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:00.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.841 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 09:32:00.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.841 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:32:00.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:00.842 | INFO | Uploaded 36GiB to GPU in 2.43519s (14.7832GiB/s) | 20260508 09:32:00.842 | INFO | Downloaded 34GiB to GPU in 2.43586s (13.9581GiB/s) | 20260508 09:32:05.977 | INFO | Analyzing chunk frames = [7052800,7053312) | 20260508 09:32:05.977 | INFO | MemInfo: { | 20260508 09:32:05.977 | INFO | "gpu" : " 17 GB", | 20260508 09:32:05.977 | INFO | "huge" : "", | 20260508 09:32:05.977 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:05.977 | INFO | "malloc" : " 14 GB", | 20260508 09:32:05.977 | INFO | "pinned" : " 29 GB" | 20260508 09:32:05.977 | INFO | } | 20260508 09:32:05.977 | INFO | Perf report for the last 5.13584 seconds: | 20260508 09:32:05.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:05.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 09:32:05.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:32:05.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:05.977 | INFO | Uploaded 36GiB to GPU in 2.44838s (14.7036GiB/s) | 20260508 09:32:05.977 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:32:05.977 | INFO | Total mem usage: 16.7 GiB | 20260508 09:32:05.977 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:32:05.977 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:32:05.977 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:32:05.977 | INFO | High Water Breakdown: | 20260508 09:32:05.977 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:32:05.977 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:32:05.978 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:32:05.978 | INFO | ------------------------------------------ | 20260508 09:32:05.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:32:05.978 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:32:05.978 | INFO | Total mem usage: 26.5 GiB | 20260508 09:32:05.978 | INFO | Downloaded 34GiB to GPU in 2.42613s (14.0141GiB/s) | 20260508 09:32:05.978 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:32:05.978 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:32:05.978 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:32:05.978 | INFO | High Water Breakdown: | 20260508 09:32:05.978 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:32:05.978 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:32:05.978 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:32:05.978 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:32:05.978 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:32:05.978 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:32:05.978 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:32:05.978 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:32:05.978 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:32:05.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:32:05.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:32:05.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:32:05.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:32:05.978 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:32:05.978 | INFO | ------------------------------------------ | 20260508 09:32:05.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:32:05.978 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:32:05.978 | INFO | Total mem usage: 27.8 GiB | 20260508 09:32:05.978 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:32:05.978 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:32:05.978 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:32:05.978 | INFO | High Water Breakdown: | 20260508 09:32:05.978 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:32:05.978 | INFO | ------------------------------------------ | 20260508 09:32:05.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:32:05.978 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:32:05.978 | INFO | Total mem usage: 15.7 GiB | 20260508 09:32:05.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:32:05.978 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:32:05.978 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:32:05.978 | INFO | High Water Breakdown: | 20260508 09:32:05.978 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:32:05.978 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:32:05.978 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:32:05.978 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:32:05.978 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:32:05.978 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:32:05.978 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:32:05.978 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:32:05.978 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:32:05.978 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:32:05.978 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:32:05.978 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:32:05.978 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:32:05.978 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:32:05.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:32:05.978 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:32:05.978 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:32:05.978 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:32:05.978 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:32:05.978 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:32:05.978 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:32:05.978 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:32:05.978 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:32:05.978 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:32:05.978 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:32:05.978 | INFO | ------------------------------------------ | 20260508 09:32:05.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:32:05.979 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:32:05.979 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:32:05.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:32:05.979 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.92e-01 | 20260508 09:32:05.979 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.08e-01, 1.20e-02 / 2.06e-02 | 20260508 09:32:05.979 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.41e+00, 5.95e-01 / 3.07e-01 | 20260508 09:32:05.979 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.16e-01 | 20260508 09:32:05.979 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.48e+01, 3.57e+00 / 8.13e-01 | 20260508 09:32:05.979 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.85e+00 / 1.04e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.46e+00, 4.07e-01 / 2.54e-01 | 20260508 09:32:05.979 | INFO | PerfReport: 3.05%, Metrics -- 9.99e-01 / 1.14e+00, 2.47e-01 / 1.19e-01 | 20260508 09:32:05.979 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.34e-01, 4.91e-02 / 3.05e-02 | 20260508 09:32:05.979 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:32:05.979 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:32:05.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:32:05.979 | INFO | PerfReport: Full Execution -- 1.06e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 24.49%, Upload -- 2.59e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 24.46%, Download -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 31.00%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 2.28%, LockUpload -- 2.40e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 2.42%, LockDownload -- 2.56e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 15.33%, LockCompute -- 1.62e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260508 09:32:05.979 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260508 09:32:06.074 | INFO | Starting a round of Baz IO | 20260508 09:32:09.497 | INFO | Populating RTMetrics report | 20260508 09:32:09.497 | INFO | RTMetrics report populated | 20260508 09:32:09.497 | INFO | Opening tmp RTMetrics json file | 20260508 09:32:09.507 | INFO | Serializing RTMetrics to json | 20260508 09:32:09.509 | INFO | Serialization done, swapping tmp file now... | 20260508 09:32:09.509 | INFO | RTMetrics done | 20260508 09:32:09.564 | INFO | Finished a round of Baz IO: Wrote 1.481 GiB in 3.489 seconds | 20260508 09:32:09.564 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.70% | 20260508 09:32:09.564 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.253475% / 0% / 4.76837e-05% | 20260508 09:32:09.564 | INFO | Baz IO overall: | 20260508 09:32:09.564 | INFO | BytesWritten : 3076591788131 | 20260508 09:32:09.564 | INFO | NumEvents : 1479903707208 | 20260508 09:32:11.119 | INFO | Analyzing chunk frames = [7053312,7053824) | 20260508 09:32:11.119 | INFO | MemInfo: { | 20260508 09:32:11.119 | INFO | "gpu" : " 17 GB", | 20260508 09:32:11.119 | INFO | "huge" : "", | 20260508 09:32:11.119 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:11.119 | INFO | "malloc" : " 14 GB", | 20260508 09:32:11.119 | INFO | "pinned" : " 29 GB" | 20260508 09:32:11.119 | INFO | } | 20260508 09:32:11.119 | INFO | Perf report for the last 5.14158 seconds: | 20260508 09:32:11.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:11.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:11.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:11.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:11.119 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 09:32:11.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:11.119 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:32:11.119 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 09:32:11.119 | INFO | BAZWRITER : 67.2%, 2.5, 87.2ms, 0 | 20260508 09:32:11.119 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:32:11.119 | INFO | Uploaded 36GiB to GPU in 2.58665s (13.9176GiB/s) | 20260508 09:32:11.119 | INFO | Downloaded 38GiB to GPU in 2.70337s (14.0565GiB/s) | 20260508 09:32:16.266 | INFO | Analyzing chunk frames = [7053824,7054336) | 20260508 09:32:16.266 | INFO | MemInfo: { | 20260508 09:32:16.266 | INFO | "gpu" : " 17 GB", | 20260508 09:32:16.266 | INFO | "huge" : "", | 20260508 09:32:16.266 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:16.266 | INFO | "malloc" : " 14 GB", | 20260508 09:32:16.266 | INFO | "pinned" : " 29 GB" | 20260508 09:32:16.266 | INFO | } | 20260508 09:32:16.266 | INFO | Perf report for the last 5.14761 seconds: | 20260508 09:32:16.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:16.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.266 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 09:32:16.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.266 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:32:16.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:16.267 | INFO | Uploaded 36GiB to GPU in 2.5386s (14.181GiB/s) | 20260508 09:32:16.267 | INFO | Downloaded 34GiB to GPU in 2.43893s (13.9405GiB/s) | 20260508 09:32:21.405 | INFO | Analyzing chunk frames = [7054336,7054848) | 20260508 09:32:21.405 | INFO | MemInfo: { | 20260508 09:32:21.405 | INFO | "gpu" : " 17 GB", | 20260508 09:32:21.405 | INFO | "huge" : "", | 20260508 09:32:21.405 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:21.405 | INFO | "malloc" : " 14 GB", | 20260508 09:32:21.405 | INFO | "pinned" : " 29 GB" | 20260508 09:32:21.405 | INFO | } | 20260508 09:32:21.405 | INFO | Perf report for the last 5.13881 seconds: | 20260508 09:32:21.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:21.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 09:32:21.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:32:21.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:21.405 | INFO | Uploaded 36GiB to GPU in 2.50398s (14.3771GiB/s) | 20260508 09:32:21.405 | INFO | Downloaded 34GiB to GPU in 2.42324s (14.0308GiB/s) | 20260508 09:32:26.542 | INFO | Analyzing chunk frames = [7054848,7055360) | 20260508 09:32:26.542 | INFO | MemInfo: { | 20260508 09:32:26.542 | INFO | "gpu" : " 17 GB", | 20260508 09:32:26.542 | INFO | "huge" : "", | 20260508 09:32:26.542 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:26.542 | INFO | "malloc" : " 14 GB", | 20260508 09:32:26.542 | INFO | "pinned" : " 29 GB" | 20260508 09:32:26.542 | INFO | } | 20260508 09:32:26.542 | INFO | Perf report for the last 5.13673 seconds: | 20260508 09:32:26.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:26.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 09:32:26.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:32:26.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:26.542 | INFO | Uploaded 36GiB to GPU in 2.48115s (14.5094GiB/s) | 20260508 09:32:26.542 | INFO | Downloaded 34GiB to GPU in 2.39777s (14.1798GiB/s) | 20260508 09:32:31.678 | INFO | Analyzing chunk frames = [7055360,7055872) | 20260508 09:32:31.678 | INFO | MemInfo: { | 20260508 09:32:31.678 | INFO | "gpu" : " 17 GB", | 20260508 09:32:31.678 | INFO | "huge" : "", | 20260508 09:32:31.678 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:31.678 | INFO | "malloc" : " 14 GB", | 20260508 09:32:31.678 | INFO | "pinned" : " 29 GB" | 20260508 09:32:31.678 | INFO | } | 20260508 09:32:31.678 | INFO | Perf report for the last 5.13615 seconds: | 20260508 09:32:31.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:31.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 09:32:31.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 09:32:31.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:31.678 | INFO | Uploaded 36GiB to GPU in 2.42455s (14.8481GiB/s) | 20260508 09:32:31.678 | INFO | Downloaded 34GiB to GPU in 2.53388s (13.4181GiB/s) | 20260508 09:32:36.820 | INFO | Analyzing chunk frames = [7055872,7056384) | 20260508 09:32:36.820 | INFO | MemInfo: { | 20260508 09:32:36.820 | INFO | "gpu" : " 17 GB", | 20260508 09:32:36.820 | INFO | "huge" : "", | 20260508 09:32:36.820 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:36.820 | INFO | "malloc" : " 14 GB", | 20260508 09:32:36.820 | INFO | "pinned" : " 29 GB" | 20260508 09:32:36.820 | INFO | } | 20260508 09:32:36.820 | INFO | Perf report for the last 5.14176 seconds: | 20260508 09:32:36.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:36.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:32:36.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:32:36.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:36.820 | INFO | Uploaded 36GiB to GPU in 2.37492s (15.1584GiB/s) | 20260508 09:32:36.820 | INFO | Downloaded 34GiB to GPU in 2.43847s (13.9432GiB/s) | 20260508 09:32:41.956 | INFO | Analyzing chunk frames = [7056384,7056896) | 20260508 09:32:41.956 | INFO | MemInfo: { | 20260508 09:32:41.956 | INFO | "gpu" : " 17 GB", | 20260508 09:32:41.956 | INFO | "huge" : "", | 20260508 09:32:41.956 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:41.956 | INFO | "malloc" : " 14 GB", | 20260508 09:32:41.956 | INFO | "pinned" : " 29 GB" | 20260508 09:32:41.956 | INFO | } | 20260508 09:32:41.956 | INFO | Perf report for the last 5.13639 seconds: | 20260508 09:32:41.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:41.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.956 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:32:41.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.956 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:32:41.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:41.957 | INFO | Uploaded 36GiB to GPU in 2.41719s (14.8933GiB/s) | 20260508 09:32:41.957 | INFO | Downloaded 34GiB to GPU in 2.43714s (13.9508GiB/s) | 20260508 09:32:47.101 | INFO | Analyzing chunk frames = [7056896,7057408) | 20260508 09:32:47.101 | INFO | MemInfo: { | 20260508 09:32:47.101 | INFO | "gpu" : " 17 GB", | 20260508 09:32:47.101 | INFO | "huge" : "", | 20260508 09:32:47.101 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:47.101 | INFO | "malloc" : " 14 GB", | 20260508 09:32:47.101 | INFO | "pinned" : " 29 GB" | 20260508 09:32:47.101 | INFO | } | 20260508 09:32:47.101 | INFO | Perf report for the last 5.14497 seconds: | 20260508 09:32:47.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:47.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.101 | INFO | ANALYSIS : 65.8%, 3.0, 103.1ms, 0 | 20260508 09:32:47.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.101 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 09:32:47.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:47.102 | INFO | Uploaded 36GiB to GPU in 2.43293s (14.797GiB/s) | 20260508 09:32:47.102 | INFO | Downloaded 34GiB to GPU in 2.44829s (13.8872GiB/s) | 20260508 09:32:47.202 | INFO | Starting a round of Baz IO | 20260508 09:32:50.627 | INFO | Populating RTMetrics report | 20260508 09:32:50.627 | INFO | RTMetrics report populated | 20260508 09:32:50.627 | INFO | Opening tmp RTMetrics json file | 20260508 09:32:50.635 | INFO | Serializing RTMetrics to json | 20260508 09:32:50.636 | INFO | Serialization done, swapping tmp file now... | 20260508 09:32:50.637 | INFO | RTMetrics done | 20260508 09:32:50.704 | INFO | Finished a round of Baz IO: Wrote 1.483 GiB in 3.502 seconds | 20260508 09:32:50.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:32:50.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.254071% / 0% / 7.15256e-05% | 20260508 09:32:50.704 | INFO | Baz IO overall: | 20260508 09:32:50.704 | INFO | BytesWritten : 3078183986043 | 20260508 09:32:50.704 | INFO | NumEvents : 1480656620964 | 20260508 09:32:52.235 | INFO | Analyzing chunk frames = [7057408,7057920) | 20260508 09:32:52.235 | INFO | MemInfo: { | 20260508 09:32:52.235 | INFO | "gpu" : " 17 GB", | 20260508 09:32:52.235 | INFO | "huge" : "", | 20260508 09:32:52.235 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:52.235 | INFO | "malloc" : " 14 GB", | 20260508 09:32:52.235 | INFO | "pinned" : " 29 GB" | 20260508 09:32:52.235 | INFO | } | 20260508 09:32:52.235 | INFO | Perf report for the last 5.13405 seconds: | 20260508 09:32:52.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:52.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:52.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:52.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:52.235 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 09:32:52.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:52.235 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:32:52.235 | INFO | PRE_HQ : 66.1%, 1.6, 55.1ms, 0 | 20260508 09:32:52.235 | INFO | BAZWRITER : 67.5%, 2.5, 87.8ms, 0 | 20260508 09:32:52.235 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 09:32:52.236 | INFO | Uploaded 36GiB to GPU in 2.58832s (13.9087GiB/s) | 20260508 09:32:52.236 | INFO | Downloaded 38GiB to GPU in 2.73938s (13.8717GiB/s) | 20260508 09:32:57.380 | INFO | Analyzing chunk frames = [7057920,7058432) | 20260508 09:32:57.380 | INFO | MemInfo: { | 20260508 09:32:57.380 | INFO | "gpu" : " 17 GB", | 20260508 09:32:57.381 | INFO | "huge" : "", | 20260508 09:32:57.381 | INFO | "huge_shared" : " 29 GB", | 20260508 09:32:57.381 | INFO | "malloc" : " 14 GB", | 20260508 09:32:57.381 | INFO | "pinned" : " 29 GB" | 20260508 09:32:57.381 | INFO | } | 20260508 09:32:57.381 | INFO | Perf report for the last 5.14532 seconds: | 20260508 09:32:57.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:32:57.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 09:32:57.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 09:32:57.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:32:57.381 | INFO | Uploaded 36GiB to GPU in 2.52091s (14.2806GiB/s) | 20260508 09:32:57.381 | INFO | Downloaded 34GiB to GPU in 2.45767s (13.8342GiB/s) | 20260508 09:33:02.517 | INFO | Analyzing chunk frames = [7058432,7058944) | 20260508 09:33:02.517 | INFO | MemInfo: { | 20260508 09:33:02.517 | INFO | "gpu" : " 17 GB", | 20260508 09:33:02.517 | INFO | "huge" : "", | 20260508 09:33:02.517 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:02.517 | INFO | "malloc" : " 14 GB", | 20260508 09:33:02.517 | INFO | "pinned" : " 29 GB" | 20260508 09:33:02.517 | INFO | } | 20260508 09:33:02.517 | INFO | Perf report for the last 5.13616 seconds: | 20260508 09:33:02.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:02.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.517 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 09:33:02.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.517 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:33:02.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:02.518 | INFO | Uploaded 36GiB to GPU in 2.46109s (14.6277GiB/s) | 20260508 09:33:02.518 | INFO | Downloaded 34GiB to GPU in 2.42658s (14.0115GiB/s) | 20260508 09:33:07.652 | INFO | Analyzing chunk frames = [7058944,7059456) | 20260508 09:33:07.652 | INFO | MemInfo: { | 20260508 09:33:07.652 | INFO | "gpu" : " 17 GB", | 20260508 09:33:07.652 | INFO | "huge" : "", | 20260508 09:33:07.652 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:07.652 | INFO | "malloc" : " 14 GB", | 20260508 09:33:07.652 | INFO | "pinned" : " 29 GB" | 20260508 09:33:07.652 | INFO | } | 20260508 09:33:07.652 | INFO | Perf report for the last 5.13515 seconds: | 20260508 09:33:07.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:07.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 09:33:07.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:33:07.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:07.652 | INFO | Uploaded 36GiB to GPU in 2.45626s (14.6564GiB/s) | 20260508 09:33:07.652 | INFO | Downloaded 34GiB to GPU in 2.39884s (14.1735GiB/s) | 20260508 09:33:12.792 | INFO | Analyzing chunk frames = [7059456,7059968) | 20260508 09:33:12.792 | INFO | MemInfo: { | 20260508 09:33:12.792 | INFO | "gpu" : " 17 GB", | 20260508 09:33:12.792 | INFO | "huge" : "", | 20260508 09:33:12.792 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:12.792 | INFO | "malloc" : " 14 GB", | 20260508 09:33:12.792 | INFO | "pinned" : " 29 GB" | 20260508 09:33:12.792 | INFO | } | 20260508 09:33:12.792 | INFO | Perf report for the last 5.14039 seconds: | 20260508 09:33:12.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:12.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.792 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 09:33:12.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.792 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:33:12.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:12.793 | INFO | Uploaded 36GiB to GPU in 2.39556s (15.0278GiB/s) | 20260508 09:33:12.793 | INFO | Downloaded 34GiB to GPU in 2.51822s (13.5016GiB/s) | 20260508 09:33:17.930 | INFO | Analyzing chunk frames = [7059968,7060480) | 20260508 09:33:17.930 | INFO | MemInfo: { | 20260508 09:33:17.930 | INFO | "gpu" : " 17 GB", | 20260508 09:33:17.930 | INFO | "huge" : "", | 20260508 09:33:17.930 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:17.930 | INFO | "malloc" : " 14 GB", | 20260508 09:33:17.930 | INFO | "pinned" : " 29 GB" | 20260508 09:33:17.930 | INFO | } | 20260508 09:33:17.930 | INFO | Perf report for the last 5.1376 seconds: | 20260508 09:33:17.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:17.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.930 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:33:17.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.930 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:33:17.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:17.931 | INFO | Uploaded 36GiB to GPU in 2.36848s (15.1996GiB/s) | 20260508 09:33:17.931 | INFO | Downloaded 34GiB to GPU in 2.43293s (13.9749GiB/s) | 20260508 09:33:23.075 | INFO | Analyzing chunk frames = [7060480,7060992) | 20260508 09:33:23.075 | INFO | MemInfo: { | 20260508 09:33:23.075 | INFO | "gpu" : " 17 GB", | 20260508 09:33:23.075 | INFO | "huge" : "", | 20260508 09:33:23.075 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:23.075 | INFO | "malloc" : " 14 GB", | 20260508 09:33:23.075 | INFO | "pinned" : " 29 GB" | 20260508 09:33:23.075 | INFO | } | 20260508 09:33:23.075 | INFO | Perf report for the last 5.14541 seconds: | 20260508 09:33:23.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:23.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.075 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 09:33:23.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.075 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:33:23.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:23.076 | INFO | Uploaded 36GiB to GPU in 2.41771s (14.8901GiB/s) | 20260508 09:33:23.076 | INFO | Downloaded 34GiB to GPU in 2.44076s (13.9301GiB/s) | 20260508 09:33:28.216 | INFO | Analyzing chunk frames = [7060992,7061504) | 20260508 09:33:28.216 | INFO | MemInfo: { | 20260508 09:33:28.216 | INFO | "gpu" : " 17 GB", | 20260508 09:33:28.216 | INFO | "huge" : "", | 20260508 09:33:28.216 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:28.216 | INFO | "malloc" : " 14 GB", | 20260508 09:33:28.216 | INFO | "pinned" : " 29 GB" | 20260508 09:33:28.216 | INFO | } | 20260508 09:33:28.216 | INFO | Perf report for the last 5.14074 seconds: | 20260508 09:33:28.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:28.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:33:28.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 09:33:28.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:28.216 | INFO | Uploaded 36GiB to GPU in 2.43834s (14.7641GiB/s) | 20260508 09:33:28.216 | INFO | Downloaded 34GiB to GPU in 2.44615s (13.8994GiB/s) | 20260508 09:33:28.217 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:33:28.217 | INFO | Total mem usage: 16.7 GiB | 20260508 09:33:28.217 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:33:28.217 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:33:28.217 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:33:28.217 | INFO | High Water Breakdown: | 20260508 09:33:28.217 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:33:28.217 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:33:28.217 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:33:28.217 | INFO | ------------------------------------------ | 20260508 09:33:28.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:33:28.217 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:33:28.217 | INFO | Total mem usage: 26.5 GiB | 20260508 09:33:28.217 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:33:28.217 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:33:28.217 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:33:28.217 | INFO | High Water Breakdown: | 20260508 09:33:28.217 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:33:28.217 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:33:28.217 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:33:28.217 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:33:28.217 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:33:28.217 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:33:28.217 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:33:28.217 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:33:28.217 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:33:28.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:33:28.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:33:28.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:33:28.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:33:28.217 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:33:28.217 | INFO | ------------------------------------------ | 20260508 09:33:28.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:33:28.217 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:33:28.217 | INFO | Total mem usage: 27.8 GiB | 20260508 09:33:28.217 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:33:28.217 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:33:28.217 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:33:28.217 | INFO | High Water Breakdown: | 20260508 09:33:28.217 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:33:28.217 | INFO | ------------------------------------------ | 20260508 09:33:28.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:33:28.217 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:33:28.217 | INFO | Total mem usage: 15.7 GiB | 20260508 09:33:28.217 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:33:28.217 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:33:28.217 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:33:28.217 | INFO | High Water Breakdown: | 20260508 09:33:28.217 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:33:28.217 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:33:28.217 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:33:28.217 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:33:28.217 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:33:28.217 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:33:28.217 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:33:28.217 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:33:28.218 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:33:28.218 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:33:28.218 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:33:28.218 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:33:28.218 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:33:28.218 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:33:28.218 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:33:28.218 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:33:28.218 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:33:28.218 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:33:28.218 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:33:28.218 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:33:28.218 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:33:28.218 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:33:28.218 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:33:28.218 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:33:28.218 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:33:28.218 | INFO | ------------------------------------------ | 20260508 09:33:28.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:33:28.218 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:33:28.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:33:28.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:33:28.218 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 5.84e-01 | 20260508 09:33:28.218 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.15e-02 / 5.60e-03 | 20260508 09:33:28.218 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.48e+00, 6.00e-01 / 2.33e-01 | 20260508 09:33:28.218 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.12e-01 / 1.21e-01 | 20260508 09:33:28.218 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.51e+01, 3.80e+00 / 1.13e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.36e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.51e+00, 4.09e-01 / 2.80e-01 | 20260508 09:33:28.218 | INFO | PerfReport: 3.04%, Metrics -- 9.96e-01 / 1.14e+00, 2.47e-01 / 1.25e-01 | 20260508 09:33:28.218 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.32e-01, 4.89e-02 / 2.57e-02 | 20260508 09:33:28.218 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:33:28.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:33:28.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:33:28.218 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 24.32%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 24.49%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 31.03%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 2.23%, LockUpload -- 2.36e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 2.57%, LockDownload -- 2.71e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 09:33:28.218 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 09:33:28.313 | INFO | Starting a round of Baz IO | 20260508 09:33:31.739 | INFO | Populating RTMetrics report | 20260508 09:33:31.739 | INFO | RTMetrics report populated | 20260508 09:33:31.739 | INFO | Opening tmp RTMetrics json file | 20260508 09:33:31.748 | INFO | Serializing RTMetrics to json | 20260508 09:33:31.750 | INFO | Serialization done, swapping tmp file now... | 20260508 09:33:31.751 | INFO | RTMetrics done | 20260508 09:33:31.807 | INFO | Finished a round of Baz IO: Wrote 1.481 GiB in 3.493 seconds | 20260508 09:33:31.807 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:33:31.807 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252287% / 0% / 5.16574e-05% | 20260508 09:33:31.807 | INFO | Baz IO overall: | 20260508 09:33:31.807 | INFO | BytesWritten : 3079773729472 | 20260508 09:33:31.807 | INFO | NumEvents : 1481407629127 | 20260508 09:33:33.357 | INFO | Analyzing chunk frames = [7061504,7062016) | 20260508 09:33:33.357 | INFO | MemInfo: { | 20260508 09:33:33.357 | INFO | "gpu" : " 17 GB", | 20260508 09:33:33.357 | INFO | "huge" : "", | 20260508 09:33:33.357 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:33.357 | INFO | "malloc" : " 14 GB", | 20260508 09:33:33.357 | INFO | "pinned" : " 29 GB" | 20260508 09:33:33.357 | INFO | } | 20260508 09:33:33.357 | INFO | Perf report for the last 5.14071 seconds: | 20260508 09:33:33.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:33.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:33.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:33.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:33.357 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 09:33:33.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:33.357 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 09:33:33.357 | INFO | PRE_HQ : 66.0%, 1.7, 59.0ms, 0 | 20260508 09:33:33.357 | INFO | BAZWRITER : 67.3%, 2.7, 92.6ms, 0 | 20260508 09:33:33.357 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:33:33.358 | INFO | Uploaded 36GiB to GPU in 2.57728s (13.9682GiB/s) | 20260508 09:33:33.358 | INFO | Downloaded 38GiB to GPU in 2.71089s (14.0175GiB/s) | 20260508 09:33:38.486 | INFO | Analyzing chunk frames = [7062016,7062528) | 20260508 09:33:38.486 | INFO | MemInfo: { | 20260508 09:33:38.486 | INFO | "gpu" : " 17 GB", | 20260508 09:33:38.486 | INFO | "huge" : "", | 20260508 09:33:38.486 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:38.486 | INFO | "malloc" : " 14 GB", | 20260508 09:33:38.486 | INFO | "pinned" : " 29 GB" | 20260508 09:33:38.486 | INFO | } | 20260508 09:33:38.486 | INFO | Perf report for the last 5.12951 seconds: | 20260508 09:33:38.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:38.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.486 | INFO | ANALYSIS : 66.8%, 3.0, 103.9ms, 0 | 20260508 09:33:38.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.486 | INFO | PRE_HQ : 22.9%, 1.0, 11.8ms, 0 | 20260508 09:33:38.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:38.487 | INFO | Uploaded 36GiB to GPU in 2.48551s (14.4839GiB/s) | 20260508 09:33:38.487 | INFO | Downloaded 34GiB to GPU in 2.49146s (13.6466GiB/s) | 20260508 09:33:43.631 | INFO | Analyzing chunk frames = [7062528,7063040) | 20260508 09:33:43.631 | INFO | MemInfo: { | 20260508 09:33:43.631 | INFO | "gpu" : " 17 GB", | 20260508 09:33:43.631 | INFO | "huge" : "", | 20260508 09:33:43.631 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:43.631 | INFO | "malloc" : " 14 GB", | 20260508 09:33:43.631 | INFO | "pinned" : " 29 GB" | 20260508 09:33:43.631 | INFO | } | 20260508 09:33:43.631 | INFO | Perf report for the last 5.14474 seconds: | 20260508 09:33:43.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:43.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 09:33:43.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:33:43.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:43.631 | INFO | Uploaded 36GiB to GPU in 2.44164s (14.7442GiB/s) | 20260508 09:33:43.631 | INFO | Downloaded 34GiB to GPU in 2.45012s (13.8769GiB/s) | 20260508 09:33:48.774 | INFO | Analyzing chunk frames = [7063040,7063552) | 20260508 09:33:48.774 | INFO | MemInfo: { | 20260508 09:33:48.774 | INFO | "gpu" : " 17 GB", | 20260508 09:33:48.774 | INFO | "huge" : "", | 20260508 09:33:48.774 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:48.774 | INFO | "malloc" : " 14 GB", | 20260508 09:33:48.774 | INFO | "pinned" : " 29 GB" | 20260508 09:33:48.774 | INFO | } | 20260508 09:33:48.774 | INFO | Perf report for the last 5.14306 seconds: | 20260508 09:33:48.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:48.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 09:33:48.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:33:48.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:48.774 | INFO | Uploaded 36GiB to GPU in 2.43421s (14.7892GiB/s) | 20260508 09:33:48.774 | INFO | Downloaded 34GiB to GPU in 2.42625s (14.0134GiB/s) | 20260508 09:33:53.914 | INFO | Analyzing chunk frames = [7063552,7064064) | 20260508 09:33:53.914 | INFO | MemInfo: { | 20260508 09:33:53.914 | INFO | "gpu" : " 17 GB", | 20260508 09:33:53.914 | INFO | "huge" : "", | 20260508 09:33:53.914 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:53.914 | INFO | "malloc" : " 14 GB", | 20260508 09:33:53.914 | INFO | "pinned" : " 29 GB" | 20260508 09:33:53.914 | INFO | } | 20260508 09:33:53.914 | INFO | Perf report for the last 5.13979 seconds: | 20260508 09:33:53.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:53.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 09:33:53.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:33:53.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:53.914 | INFO | Uploaded 36GiB to GPU in 2.38654s (15.0846GiB/s) | 20260508 09:33:53.914 | INFO | Downloaded 34GiB to GPU in 2.52905s (13.4438GiB/s) | 20260508 09:33:59.051 | INFO | Analyzing chunk frames = [7064064,7064576) | 20260508 09:33:59.051 | INFO | MemInfo: { | 20260508 09:33:59.051 | INFO | "gpu" : " 17 GB", | 20260508 09:33:59.051 | INFO | "huge" : "", | 20260508 09:33:59.051 | INFO | "huge_shared" : " 29 GB", | 20260508 09:33:59.051 | INFO | "malloc" : " 14 GB", | 20260508 09:33:59.051 | INFO | "pinned" : " 29 GB" | 20260508 09:33:59.051 | INFO | } | 20260508 09:33:59.051 | INFO | Perf report for the last 5.13706 seconds: | 20260508 09:33:59.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:33:59.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:33:59.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:33:59.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:33:59.051 | INFO | Uploaded 36GiB to GPU in 2.34873s (15.3274GiB/s) | 20260508 09:33:59.051 | INFO | Downloaded 34GiB to GPU in 2.43266s (13.9765GiB/s) | 20260508 09:34:04.186 | INFO | Analyzing chunk frames = [7064576,7065088) | 20260508 09:34:04.186 | INFO | MemInfo: { | 20260508 09:34:04.186 | INFO | "gpu" : " 17 GB", | 20260508 09:34:04.186 | INFO | "huge" : "", | 20260508 09:34:04.186 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:04.186 | INFO | "malloc" : " 14 GB", | 20260508 09:34:04.186 | INFO | "pinned" : " 29 GB" | 20260508 09:34:04.186 | INFO | } | 20260508 09:34:04.186 | INFO | Perf report for the last 5.13495 seconds: | 20260508 09:34:04.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:04.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:34:04.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:34:04.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:04.186 | INFO | Uploaded 36GiB to GPU in 2.39976s (15.0015GiB/s) | 20260508 09:34:04.186 | INFO | Downloaded 34GiB to GPU in 2.43476s (13.9644GiB/s) | 20260508 09:34:09.329 | INFO | Analyzing chunk frames = [7065088,7065600) | 20260508 09:34:09.329 | INFO | MemInfo: { | 20260508 09:34:09.329 | INFO | "gpu" : " 17 GB", | 20260508 09:34:09.329 | INFO | "huge" : "", | 20260508 09:34:09.329 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:09.329 | INFO | "malloc" : " 14 GB", | 20260508 09:34:09.329 | INFO | "pinned" : " 29 GB" | 20260508 09:34:09.329 | INFO | } | 20260508 09:34:09.329 | INFO | Perf report for the last 5.14309 seconds: | 20260508 09:34:09.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:09.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.329 | INFO | ANALYSIS : 65.7%, 3.0, 102.9ms, 0 | 20260508 09:34:09.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.329 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:34:09.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:09.330 | INFO | Uploaded 36GiB to GPU in 2.41685s (14.8954GiB/s) | 20260508 09:34:09.330 | INFO | Downloaded 34GiB to GPU in 2.44591s (13.9008GiB/s) | 20260508 09:34:09.427 | INFO | Starting a round of Baz IO | 20260508 09:34:12.840 | INFO | Populating RTMetrics report | 20260508 09:34:12.840 | INFO | RTMetrics report populated | 20260508 09:34:12.840 | INFO | Opening tmp RTMetrics json file | 20260508 09:34:12.850 | INFO | Serializing RTMetrics to json | 20260508 09:34:12.852 | INFO | Serialization done, swapping tmp file now... | 20260508 09:34:12.852 | INFO | RTMetrics done | 20260508 09:34:12.912 | INFO | Finished a round of Baz IO: Wrote 1.481 GiB in 3.485 seconds | 20260508 09:34:12.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:34:12.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252275% / 0% / 5.16574e-05% | 20260508 09:34:12.913 | INFO | Baz IO overall: | 20260508 09:34:12.913 | INFO | BytesWritten : 3081364090678 | 20260508 09:34:12.913 | INFO | NumEvents : 1482158975832 | 20260508 09:34:14.468 | INFO | Analyzing chunk frames = [7065600,7066112) | 20260508 09:34:14.468 | INFO | MemInfo: { | 20260508 09:34:14.468 | INFO | "gpu" : " 17 GB", | 20260508 09:34:14.468 | INFO | "huge" : "", | 20260508 09:34:14.468 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:14.468 | INFO | "malloc" : " 14 GB", | 20260508 09:34:14.468 | INFO | "pinned" : " 29 GB" | 20260508 09:34:14.468 | INFO | } | 20260508 09:34:14.468 | INFO | Perf report for the last 5.13895 seconds: | 20260508 09:34:14.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:14.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:14.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:14.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:14.468 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:34:14.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:14.468 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 09:34:14.468 | INFO | PRE_HQ : 65.4%, 1.6, 54.4ms, 0 | 20260508 09:34:14.468 | INFO | BAZWRITER : 67.1%, 2.5, 87.8ms, 0 | 20260508 09:34:14.469 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:34:14.469 | INFO | Uploaded 36GiB to GPU in 2.60979s (13.7942GiB/s) | 20260508 09:34:14.469 | INFO | Downloaded 38GiB to GPU in 2.70223s (14.0624GiB/s) | 20260508 09:34:19.604 | INFO | Analyzing chunk frames = [7066112,7066624) | 20260508 09:34:19.604 | INFO | MemInfo: { | 20260508 09:34:19.604 | INFO | "gpu" : " 17 GB", | 20260508 09:34:19.604 | INFO | "huge" : "", | 20260508 09:34:19.604 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:19.605 | INFO | "malloc" : " 14 GB", | 20260508 09:34:19.605 | INFO | "pinned" : " 29 GB" | 20260508 09:34:19.605 | INFO | } | 20260508 09:34:19.605 | INFO | Perf report for the last 5.13654 seconds: | 20260508 09:34:19.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:19.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 09:34:19.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:34:19.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:19.605 | INFO | Uploaded 36GiB to GPU in 2.54194s (14.1624GiB/s) | 20260508 09:34:19.605 | INFO | Downloaded 34GiB to GPU in 2.45806s (13.8321GiB/s) | 20260508 09:34:24.742 | INFO | Analyzing chunk frames = [7066624,7067136) | 20260508 09:34:24.742 | INFO | MemInfo: { | 20260508 09:34:24.742 | INFO | "gpu" : " 17 GB", | 20260508 09:34:24.742 | INFO | "huge" : "", | 20260508 09:34:24.742 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:24.742 | INFO | "malloc" : " 14 GB", | 20260508 09:34:24.742 | INFO | "pinned" : " 29 GB" | 20260508 09:34:24.742 | INFO | } | 20260508 09:34:24.742 | INFO | Perf report for the last 5.13769 seconds: | 20260508 09:34:24.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:24.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.742 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 09:34:24.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.742 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:34:24.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:24.743 | INFO | Uploaded 36GiB to GPU in 2.52063s (14.2822GiB/s) | 20260508 09:34:24.743 | INFO | Downloaded 34GiB to GPU in 2.43813s (13.9451GiB/s) | 20260508 09:34:29.884 | INFO | Analyzing chunk frames = [7067136,7067648) | 20260508 09:34:29.884 | INFO | MemInfo: { | 20260508 09:34:29.884 | INFO | "gpu" : " 17 GB", | 20260508 09:34:29.884 | INFO | "huge" : "", | 20260508 09:34:29.884 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:29.884 | INFO | "malloc" : " 14 GB", | 20260508 09:34:29.884 | INFO | "pinned" : " 29 GB" | 20260508 09:34:29.884 | INFO | } | 20260508 09:34:29.885 | INFO | Perf report for the last 5.14239 seconds: | 20260508 09:34:29.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:29.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 09:34:29.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:34:29.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:29.885 | INFO | Uploaded 36GiB to GPU in 2.53111s (14.223GiB/s) | 20260508 09:34:29.885 | INFO | Downloaded 34GiB to GPU in 2.42851s (14.0003GiB/s) | 20260508 09:34:35.032 | INFO | Analyzing chunk frames = [7067648,7068160) | 20260508 09:34:35.032 | INFO | MemInfo: { | 20260508 09:34:35.032 | INFO | "gpu" : " 17 GB", | 20260508 09:34:35.032 | INFO | "huge" : "", | 20260508 09:34:35.032 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:35.032 | INFO | "malloc" : " 14 GB", | 20260508 09:34:35.032 | INFO | "pinned" : " 29 GB" | 20260508 09:34:35.032 | INFO | } | 20260508 09:34:35.033 | INFO | Perf report for the last 5.14775 seconds: | 20260508 09:34:35.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:35.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 09:34:35.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:34:35.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:35.033 | INFO | Uploaded 36GiB to GPU in 2.41025s (14.9362GiB/s) | 20260508 09:34:35.033 | INFO | Downloaded 34GiB to GPU in 2.50718s (13.5611GiB/s) | 20260508 09:34:40.165 | INFO | Analyzing chunk frames = [7068160,7068672) | 20260508 09:34:40.165 | INFO | MemInfo: { | 20260508 09:34:40.165 | INFO | "gpu" : " 17 GB", | 20260508 09:34:40.165 | INFO | "huge" : "", | 20260508 09:34:40.165 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:40.165 | INFO | "malloc" : " 14 GB", | 20260508 09:34:40.165 | INFO | "pinned" : " 29 GB" | 20260508 09:34:40.165 | INFO | } | 20260508 09:34:40.165 | INFO | Perf report for the last 5.13237 seconds: | 20260508 09:34:40.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:40.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 09:34:40.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 09:34:40.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:40.165 | INFO | Uploaded 36GiB to GPU in 2.37586s (15.1524GiB/s) | 20260508 09:34:40.165 | INFO | Downloaded 34GiB to GPU in 2.43034s (13.9898GiB/s) | 20260508 09:34:45.299 | INFO | Analyzing chunk frames = [7068672,7069184) | 20260508 09:34:45.299 | INFO | MemInfo: { | 20260508 09:34:45.299 | INFO | "gpu" : " 17 GB", | 20260508 09:34:45.299 | INFO | "huge" : "", | 20260508 09:34:45.299 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:45.299 | INFO | "malloc" : " 14 GB", | 20260508 09:34:45.299 | INFO | "pinned" : " 29 GB" | 20260508 09:34:45.299 | INFO | } | 20260508 09:34:45.299 | INFO | Perf report for the last 5.13423 seconds: | 20260508 09:34:45.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:45.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.299 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:34:45.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.299 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:34:45.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:45.300 | INFO | Uploaded 36GiB to GPU in 2.39117s (15.0554GiB/s) | 20260508 09:34:45.300 | INFO | Downloaded 34GiB to GPU in 2.43482s (13.9641GiB/s) | 20260508 09:34:50.445 | INFO | Analyzing chunk frames = [7069184,7069696) | 20260508 09:34:50.445 | INFO | MemInfo: { | 20260508 09:34:50.445 | INFO | "gpu" : " 17 GB", | 20260508 09:34:50.445 | INFO | "huge" : "", | 20260508 09:34:50.445 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:50.445 | INFO | "malloc" : " 14 GB", | 20260508 09:34:50.445 | INFO | "pinned" : " 29 GB" | 20260508 09:34:50.445 | INFO | } | 20260508 09:34:50.445 | INFO | Perf report for the last 5.14592 seconds: | 20260508 09:34:50.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:50.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 09:34:50.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:34:50.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:50.445 | INFO | Uploaded 36GiB to GPU in 2.4061s (14.962GiB/s) | 20260508 09:34:50.445 | INFO | Downloaded 34GiB to GPU in 2.43757s (13.9483GiB/s) | 20260508 09:34:50.446 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:34:50.446 | INFO | Total mem usage: 16.7 GiB | 20260508 09:34:50.446 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:34:50.446 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.5%) | 20260508 09:34:50.446 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:34:50.446 | INFO | High Water Breakdown: | 20260508 09:34:50.446 | INFO | 59.7% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:34:50.446 | INFO | 40.2% : 5.76 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:34:50.446 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:34:50.446 | INFO | ------------------------------------------ | 20260508 09:34:50.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:34:50.446 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:34:50.446 | INFO | Total mem usage: 26.5 GiB | 20260508 09:34:50.446 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:34:50.446 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:34:50.446 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:34:50.446 | INFO | High Water Breakdown: | 20260508 09:34:50.446 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:34:50.446 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:34:50.446 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:34:50.446 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:34:50.446 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:34:50.446 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:34:50.446 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:34:50.446 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:34:50.446 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:34:50.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:34:50.446 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:34:50.446 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:34:50.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:34:50.446 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:34:50.446 | INFO | ------------------------------------------ | 20260508 09:34:50.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:34:50.446 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:34:50.446 | INFO | Total mem usage: 27.8 GiB | 20260508 09:34:50.446 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:34:50.446 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:34:50.446 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:34:50.446 | INFO | High Water Breakdown: | 20260508 09:34:50.446 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:34:50.446 | INFO | ------------------------------------------ | 20260508 09:34:50.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:34:50.446 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:34:50.446 | INFO | Total mem usage: 15.7 GiB | 20260508 09:34:50.446 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:34:50.446 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:34:50.446 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:34:50.446 | INFO | High Water Breakdown: | 20260508 09:34:50.446 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:34:50.446 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:34:50.446 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:34:50.446 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:34:50.446 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:34:50.446 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:34:50.446 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:34:50.446 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:34:50.446 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:34:50.446 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:34:50.446 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:34:50.446 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:34:50.446 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:34:50.446 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:34:50.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:34:50.446 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:34:50.446 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:34:50.446 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:34:50.446 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:34:50.446 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:34:50.446 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:34:50.446 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:34:50.446 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:34:50.446 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:34:50.446 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:34:50.446 | INFO | ------------------------------------------ | 20260508 09:34:50.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:34:50.447 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:34:50.447 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:34:50.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:34:50.447 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.42e+01 / 6.55e-01 | 20260508 09:34:50.447 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.19e-02 / 6.58e-03 | 20260508 09:34:50.447 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.47e+00, 5.95e-01 / 2.23e-01 | 20260508 09:34:50.447 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.45e+00, 2.08e-01 / 1.24e-01 | 20260508 09:34:50.447 | INFO | PerfReport: 15.97%, DME -- 4.19e+01 / 4.50e+01, 3.77e+00 / 1.05e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.88e+01 / 2.05e+01, 1.90e+00 / 1.45e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.32e+00 / 2.47e+00, 4.07e-01 / 2.63e-01 | 20260508 09:34:50.447 | INFO | PerfReport: 3.02%, Metrics -- 9.91e-01 / 1.14e+00, 2.46e-01 / 1.17e-01 | 20260508 09:34:50.447 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.31e-01, 4.95e-02 / 2.28e-02 | 20260508 09:34:50.447 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:34:50.447 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:34:50.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:34:50.447 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 24.26%, Upload -- 2.56e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 24.56%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 31.11%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 2.19%, LockUpload -- 2.31e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 2.50%, LockDownload -- 2.63e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 15.37%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:34:50.447 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 09:34:50.542 | INFO | Starting a round of Baz IO | 20260508 09:34:53.965 | INFO | Populating RTMetrics report | 20260508 09:34:53.966 | INFO | RTMetrics report populated | 20260508 09:34:53.966 | INFO | Opening tmp RTMetrics json file | 20260508 09:34:53.974 | INFO | Serializing RTMetrics to json | 20260508 09:34:53.975 | INFO | Serialization done, swapping tmp file now... | 20260508 09:34:53.976 | INFO | RTMetrics done | 20260508 09:34:54.032 | INFO | Finished a round of Baz IO: Wrote 1.48 GiB in 3.49 seconds | 20260508 09:34:54.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:34:54.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.251424% / 0% / 7.94729e-05% | 20260508 09:34:54.032 | INFO | Baz IO overall: | 20260508 09:34:54.032 | INFO | BytesWritten : 3082953037235 | 20260508 09:34:54.032 | INFO | NumEvents : 1482909463309 | 20260508 09:34:55.581 | INFO | Analyzing chunk frames = [7069696,7070208) | 20260508 09:34:55.581 | INFO | MemInfo: { | 20260508 09:34:55.581 | INFO | "gpu" : " 17 GB", | 20260508 09:34:55.581 | INFO | "huge" : "", | 20260508 09:34:55.581 | INFO | "huge_shared" : " 29 GB", | 20260508 09:34:55.581 | INFO | "malloc" : " 14 GB", | 20260508 09:34:55.581 | INFO | "pinned" : " 29 GB" | 20260508 09:34:55.581 | INFO | } | 20260508 09:34:55.581 | INFO | Perf report for the last 5.13605 seconds: | 20260508 09:34:55.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:34:55.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:55.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:55.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:55.581 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:34:55.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:34:55.581 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:34:55.581 | INFO | PRE_HQ : 65.5%, 1.6, 54.7ms, 0 | 20260508 09:34:55.581 | INFO | BAZWRITER : 67.3%, 2.5, 88.1ms, 0 | 20260508 09:34:55.581 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:34:55.581 | INFO | Uploaded 36GiB to GPU in 2.5887s (13.9066GiB/s) | 20260508 09:34:55.582 | INFO | Downloaded 38GiB to GPU in 2.70402s (14.0532GiB/s) | 20260508 09:35:00.725 | INFO | Analyzing chunk frames = [7070208,7070720) | 20260508 09:35:00.725 | INFO | MemInfo: { | 20260508 09:35:00.725 | INFO | "gpu" : " 17 GB", | 20260508 09:35:00.725 | INFO | "huge" : "", | 20260508 09:35:00.725 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:00.725 | INFO | "malloc" : " 14 GB", | 20260508 09:35:00.725 | INFO | "pinned" : " 29 GB" | 20260508 09:35:00.725 | INFO | } | 20260508 09:35:00.725 | INFO | Perf report for the last 5.14384 seconds: | 20260508 09:35:00.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:00.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:35:00.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:35:00.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:00.725 | INFO | Uploaded 36GiB to GPU in 2.49202s (14.4461GiB/s) | 20260508 09:35:00.725 | INFO | Downloaded 34GiB to GPU in 2.46079s (13.8167GiB/s) | 20260508 09:35:05.862 | INFO | Analyzing chunk frames = [7070720,7071232) | 20260508 09:35:05.862 | INFO | MemInfo: { | 20260508 09:35:05.862 | INFO | "gpu" : " 17 GB", | 20260508 09:35:05.862 | INFO | "huge" : "", | 20260508 09:35:05.862 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:05.862 | INFO | "malloc" : " 14 GB", | 20260508 09:35:05.862 | INFO | "pinned" : " 29 GB" | 20260508 09:35:05.862 | INFO | } | 20260508 09:35:05.862 | INFO | Perf report for the last 5.13729 seconds: | 20260508 09:35:05.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:05.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.862 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 09:35:05.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.862 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:35:05.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:05.863 | INFO | Uploaded 36GiB to GPU in 2.44098s (14.7482GiB/s) | 20260508 09:35:05.863 | INFO | Downloaded 34GiB to GPU in 2.43847s (13.9432GiB/s) | 20260508 09:35:10.994 | INFO | Analyzing chunk frames = [7071232,7071744) | 20260508 09:35:10.994 | INFO | MemInfo: { | 20260508 09:35:10.994 | INFO | "gpu" : " 17 GB", | 20260508 09:35:10.994 | INFO | "huge" : "", | 20260508 09:35:10.994 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:10.994 | INFO | "malloc" : " 14 GB", | 20260508 09:35:10.994 | INFO | "pinned" : " 29 GB" | 20260508 09:35:10.994 | INFO | } | 20260508 09:35:10.994 | INFO | Perf report for the last 5.13188 seconds: | 20260508 09:35:10.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:10.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 09:35:10.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:35:10.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:10.994 | INFO | Uploaded 36GiB to GPU in 2.4605s (14.6312GiB/s) | 20260508 09:35:10.995 | INFO | Downloaded 34GiB to GPU in 2.43265s (13.9766GiB/s) | 20260508 09:35:16.138 | INFO | Analyzing chunk frames = [7071744,7072256) | 20260508 09:35:16.138 | INFO | MemInfo: { | 20260508 09:35:16.138 | INFO | "gpu" : " 17 GB", | 20260508 09:35:16.138 | INFO | "huge" : "", | 20260508 09:35:16.138 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:16.138 | INFO | "malloc" : " 14 GB", | 20260508 09:35:16.138 | INFO | "pinned" : " 29 GB" | 20260508 09:35:16.138 | INFO | } | 20260508 09:35:16.138 | INFO | Perf report for the last 5.14368 seconds: | 20260508 09:35:16.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:16.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 09:35:16.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260508 09:35:16.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:16.138 | INFO | Uploaded 36GiB to GPU in 2.37598s (15.1516GiB/s) | 20260508 09:35:16.138 | INFO | Downloaded 34GiB to GPU in 2.53486s (13.413GiB/s) | 20260508 09:35:21.277 | INFO | Analyzing chunk frames = [7072256,7072768) | 20260508 09:35:21.277 | INFO | MemInfo: { | 20260508 09:35:21.277 | INFO | "gpu" : " 17 GB", | 20260508 09:35:21.277 | INFO | "huge" : "", | 20260508 09:35:21.277 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:21.277 | INFO | "malloc" : " 14 GB", | 20260508 09:35:21.277 | INFO | "pinned" : " 29 GB" | 20260508 09:35:21.278 | INFO | } | 20260508 09:35:21.278 | INFO | Perf report for the last 5.13984 seconds: | 20260508 09:35:21.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:21.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:35:21.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:35:21.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:21.278 | INFO | Uploaded 36GiB to GPU in 2.36313s (15.2341GiB/s) | 20260508 09:35:21.278 | INFO | Downloaded 34GiB to GPU in 2.42506s (14.0203GiB/s) | 20260508 09:35:26.419 | INFO | Analyzing chunk frames = [7072768,7073280) | 20260508 09:35:26.419 | INFO | MemInfo: { | 20260508 09:35:26.419 | INFO | "gpu" : " 17 GB", | 20260508 09:35:26.419 | INFO | "huge" : "", | 20260508 09:35:26.419 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:26.419 | INFO | "malloc" : " 14 GB", | 20260508 09:35:26.419 | INFO | "pinned" : " 29 GB" | 20260508 09:35:26.419 | INFO | } | 20260508 09:35:26.419 | INFO | Perf report for the last 5.1415 seconds: | 20260508 09:35:26.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:26.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.419 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:35:26.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.419 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:35:26.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:26.420 | INFO | Uploaded 36GiB to GPU in 2.41507s (14.9064GiB/s) | 20260508 09:35:26.420 | INFO | Downloaded 34GiB to GPU in 2.44114s (13.9279GiB/s) | 20260508 09:35:31.553 | INFO | Analyzing chunk frames = [7073280,7073792) | 20260508 09:35:31.553 | INFO | MemInfo: { | 20260508 09:35:31.553 | INFO | "gpu" : " 17 GB", | 20260508 09:35:31.553 | INFO | "huge" : "", | 20260508 09:35:31.553 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:31.554 | INFO | "malloc" : " 14 GB", | 20260508 09:35:31.554 | INFO | "pinned" : " 29 GB" | 20260508 09:35:31.554 | INFO | } | 20260508 09:35:31.554 | INFO | Perf report for the last 5.13449 seconds: | 20260508 09:35:31.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:31.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 09:35:31.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 09:35:31.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:31.554 | INFO | Uploaded 36GiB to GPU in 2.4648s (14.6057GiB/s) | 20260508 09:35:31.555 | INFO | Downloaded 34GiB to GPU in 2.45121s (13.8707GiB/s) | 20260508 09:35:31.661 | INFO | Starting a round of Baz IO | 20260508 09:35:35.062 | INFO | Populating RTMetrics report | 20260508 09:35:35.062 | INFO | RTMetrics report populated | 20260508 09:35:35.062 | INFO | Opening tmp RTMetrics json file | 20260508 09:35:35.068 | INFO | Serializing RTMetrics to json | 20260508 09:35:35.077 | INFO | Serialization done, swapping tmp file now... | 20260508 09:35:35.077 | INFO | RTMetrics done | 20260508 09:35:35.135 | INFO | Finished a round of Baz IO: Wrote 1.48 GiB in 3.473 seconds | 20260508 09:35:35.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:35:35.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.250995% / 3.97364e-06% / 6.75519e-05% | 20260508 09:35:35.135 | INFO | Baz IO overall: | 20260508 09:35:35.135 | INFO | BytesWritten : 3084541855590 | 20260508 09:35:35.135 | INFO | NumEvents : 1483659919826 | 20260508 09:35:36.695 | INFO | Analyzing chunk frames = [7073792,7074304) | 20260508 09:35:36.695 | INFO | MemInfo: { | 20260508 09:35:36.695 | INFO | "gpu" : " 17 GB", | 20260508 09:35:36.695 | INFO | "huge" : "", | 20260508 09:35:36.695 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:36.695 | INFO | "malloc" : " 14 GB", | 20260508 09:35:36.695 | INFO | "pinned" : " 29 GB" | 20260508 09:35:36.695 | INFO | } | 20260508 09:35:36.695 | INFO | Perf report for the last 5.14108 seconds: | 20260508 09:35:36.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:36.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:36.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:36.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:36.695 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:35:36.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:36.695 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:35:36.695 | INFO | PRE_HQ : 65.2%, 1.6, 54.7ms, 0 | 20260508 09:35:36.695 | INFO | BAZWRITER : 67.1%, 2.5, 87.5ms, 0 | 20260508 09:35:36.695 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:35:36.695 | INFO | Uploaded 36GiB to GPU in 2.60607s (13.8139GiB/s) | 20260508 09:35:36.695 | INFO | Downloaded 38GiB to GPU in 2.71009s (14.0217GiB/s) | 20260508 09:35:41.835 | INFO | Analyzing chunk frames = [7074304,7074816) | 20260508 09:35:41.835 | INFO | MemInfo: { | 20260508 09:35:41.835 | INFO | "gpu" : " 17 GB", | 20260508 09:35:41.835 | INFO | "huge" : "", | 20260508 09:35:41.835 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:41.835 | INFO | "malloc" : " 14 GB", | 20260508 09:35:41.835 | INFO | "pinned" : " 29 GB" | 20260508 09:35:41.835 | INFO | } | 20260508 09:35:41.835 | INFO | Perf report for the last 5.14002 seconds: | 20260508 09:35:41.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:41.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:35:41.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:35:41.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:41.835 | INFO | Uploaded 36GiB to GPU in 2.51612s (14.3077GiB/s) | 20260508 09:35:41.835 | INFO | Downloaded 34GiB to GPU in 2.46072s (13.8171GiB/s) | 20260508 09:35:46.975 | INFO | Analyzing chunk frames = [7074816,7075328) | 20260508 09:35:46.975 | INFO | MemInfo: { | 20260508 09:35:46.976 | INFO | "gpu" : " 17 GB", | 20260508 09:35:46.976 | INFO | "huge" : "", | 20260508 09:35:46.976 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:46.976 | INFO | "malloc" : " 14 GB", | 20260508 09:35:46.976 | INFO | "pinned" : " 29 GB" | 20260508 09:35:46.976 | INFO | } | 20260508 09:35:46.976 | INFO | Perf report for the last 5.14092 seconds: | 20260508 09:35:46.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:46.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:35:46.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260508 09:35:46.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:46.976 | INFO | Uploaded 36GiB to GPU in 2.46512s (14.6037GiB/s) | 20260508 09:35:46.976 | INFO | Downloaded 34GiB to GPU in 2.43635s (13.9553GiB/s) | 20260508 09:35:52.119 | INFO | Analyzing chunk frames = [7075328,7075840) | 20260508 09:35:52.119 | INFO | MemInfo: { | 20260508 09:35:52.119 | INFO | "gpu" : " 17 GB", | 20260508 09:35:52.119 | INFO | "huge" : "", | 20260508 09:35:52.119 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:52.119 | INFO | "malloc" : " 14 GB", | 20260508 09:35:52.119 | INFO | "pinned" : " 29 GB" | 20260508 09:35:52.119 | INFO | } | 20260508 09:35:52.119 | INFO | Perf report for the last 5.14299 seconds: | 20260508 09:35:52.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:52.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:35:52.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:35:52.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:52.119 | INFO | Uploaded 36GiB to GPU in 2.48585s (14.4819GiB/s) | 20260508 09:35:52.119 | INFO | Downloaded 34GiB to GPU in 2.42847s (14.0006GiB/s) | 20260508 09:35:57.251 | INFO | Analyzing chunk frames = [7075840,7076352) | 20260508 09:35:57.251 | INFO | MemInfo: { | 20260508 09:35:57.251 | INFO | "gpu" : " 17 GB", | 20260508 09:35:57.251 | INFO | "huge" : "", | 20260508 09:35:57.251 | INFO | "huge_shared" : " 29 GB", | 20260508 09:35:57.251 | INFO | "malloc" : " 14 GB", | 20260508 09:35:57.251 | INFO | "pinned" : " 29 GB" | 20260508 09:35:57.251 | INFO | } | 20260508 09:35:57.251 | INFO | Perf report for the last 5.13198 seconds: | 20260508 09:35:57.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:35:57.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 09:35:57.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:35:57.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:35:57.251 | INFO | Uploaded 36GiB to GPU in 2.302s (15.6386GiB/s) | 20260508 09:35:57.251 | INFO | Downloaded 34GiB to GPU in 2.51034s (13.544GiB/s) | 20260508 09:36:02.396 | INFO | Analyzing chunk frames = [7076352,7076864) | 20260508 09:36:02.396 | INFO | MemInfo: { | 20260508 09:36:02.396 | INFO | "gpu" : " 17 GB", | 20260508 09:36:02.396 | INFO | "huge" : "", | 20260508 09:36:02.396 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:02.396 | INFO | "malloc" : " 14 GB", | 20260508 09:36:02.396 | INFO | "pinned" : " 29 GB" | 20260508 09:36:02.396 | INFO | } | 20260508 09:36:02.396 | INFO | Perf report for the last 5.14515 seconds: | 20260508 09:36:02.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:02.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.396 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:36:02.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.396 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:36:02.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:02.397 | INFO | Uploaded 36GiB to GPU in 2.38667s (15.0838GiB/s) | 20260508 09:36:02.397 | INFO | Downloaded 34GiB to GPU in 2.44364s (13.9137GiB/s) | 20260508 09:36:07.535 | INFO | Analyzing chunk frames = [7076864,7077376) | 20260508 09:36:07.535 | INFO | MemInfo: { | 20260508 09:36:07.535 | INFO | "gpu" : " 17 GB", | 20260508 09:36:07.535 | INFO | "huge" : "", | 20260508 09:36:07.535 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:07.535 | INFO | "malloc" : " 14 GB", | 20260508 09:36:07.535 | INFO | "pinned" : " 29 GB" | 20260508 09:36:07.535 | INFO | } | 20260508 09:36:07.535 | INFO | Perf report for the last 5.13882 seconds: | 20260508 09:36:07.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:07.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:36:07.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:36:07.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:07.535 | INFO | Uploaded 36GiB to GPU in 2.39869s (15.0082GiB/s) | 20260508 09:36:07.535 | INFO | Downloaded 34GiB to GPU in 2.44398s (13.9117GiB/s) | 20260508 09:36:12.676 | INFO | Analyzing chunk frames = [7077376,7077888) | 20260508 09:36:12.676 | INFO | MemInfo: { | 20260508 09:36:12.676 | INFO | "gpu" : " 17 GB", | 20260508 09:36:12.676 | INFO | "huge" : "", | 20260508 09:36:12.676 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:12.676 | INFO | "malloc" : " 14 GB", | 20260508 09:36:12.676 | INFO | "pinned" : " 29 GB" | 20260508 09:36:12.676 | INFO | } | 20260508 09:36:12.676 | INFO | Perf report for the last 5.14122 seconds: | 20260508 09:36:12.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:12.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 09:36:12.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:36:12.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:12.676 | INFO | Uploaded 36GiB to GPU in 2.44158s (14.7445GiB/s) | 20260508 09:36:12.676 | INFO | Downloaded 34GiB to GPU in 2.45227s (13.8647GiB/s) | 20260508 09:36:12.677 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:36:12.677 | INFO | Total mem usage: 16.7 GiB | 20260508 09:36:12.677 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:36:12.677 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:36:12.677 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:36:12.677 | INFO | High Water Breakdown: | 20260508 09:36:12.677 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:36:12.677 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:36:12.677 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:36:12.677 | INFO | ------------------------------------------ | 20260508 09:36:12.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:36:12.677 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:36:12.677 | INFO | Total mem usage: 26.5 GiB | 20260508 09:36:12.677 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:36:12.677 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:36:12.677 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:36:12.677 | INFO | High Water Breakdown: | 20260508 09:36:12.677 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:36:12.677 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:36:12.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:36:12.677 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:36:12.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:36:12.677 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:36:12.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:36:12.677 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:36:12.677 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:36:12.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:36:12.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:36:12.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:36:12.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:36:12.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:36:12.677 | INFO | ------------------------------------------ | 20260508 09:36:12.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:36:12.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:36:12.677 | INFO | Total mem usage: 27.8 GiB | 20260508 09:36:12.677 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:36:12.677 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:36:12.677 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:36:12.677 | INFO | High Water Breakdown: | 20260508 09:36:12.677 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:36:12.677 | INFO | ------------------------------------------ | 20260508 09:36:12.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:36:12.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:36:12.677 | INFO | Total mem usage: 15.7 GiB | 20260508 09:36:12.677 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:36:12.677 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:36:12.677 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:36:12.677 | INFO | High Water Breakdown: | 20260508 09:36:12.677 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:36:12.677 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:36:12.677 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:36:12.677 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:36:12.677 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:36:12.677 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:36:12.677 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:36:12.677 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:36:12.677 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:36:12.677 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:36:12.677 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:36:12.677 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:36:12.677 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:36:12.677 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:36:12.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:36:12.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:36:12.677 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:36:12.677 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:36:12.677 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:36:12.677 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:36:12.677 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:36:12.677 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:36:12.677 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:36:12.677 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:36:12.677 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:36:12.677 | INFO | ------------------------------------------ | 20260508 09:36:12.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:36:12.678 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:36:12.678 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:36:12.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:36:12.678 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.42e+01 / 6.77e-01 | 20260508 09:36:12.678 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.15e-02 / 4.60e-03 | 20260508 09:36:12.678 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.35e+00, 5.91e-01 / 2.77e-01 | 20260508 09:36:12.678 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.30e-01 | 20260508 09:36:12.678 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.53e+01, 3.70e+00 / 1.14e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.17e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.10e-01 / 2.48e-01 | 20260508 09:36:12.678 | INFO | PerfReport: 3.04%, Metrics -- 9.94e-01 / 1.14e+00, 2.46e-01 / 1.30e-01 | 20260508 09:36:12.678 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.34e-01, 4.86e-02 / 2.53e-02 | 20260508 09:36:12.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:36:12.678 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:36:12.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:36:12.678 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 24.57%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 31.01%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 2.66%, LockDownload -- 2.80e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 15.41%, LockCompute -- 1.62e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260508 09:36:12.678 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260508 09:36:12.773 | INFO | Starting a round of Baz IO | 20260508 09:36:16.184 | INFO | Populating RTMetrics report | 20260508 09:36:16.185 | INFO | RTMetrics report populated | 20260508 09:36:16.185 | INFO | Opening tmp RTMetrics json file | 20260508 09:36:16.193 | INFO | Serializing RTMetrics to json | 20260508 09:36:16.194 | INFO | Serialization done, swapping tmp file now... | 20260508 09:36:16.195 | INFO | RTMetrics done | 20260508 09:36:16.255 | INFO | Finished a round of Baz IO: Wrote 1.48 GiB in 3.481 seconds | 20260508 09:36:16.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:36:16.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.250268% / 0% / 7.54992e-05% | 20260508 09:36:16.255 | INFO | Baz IO overall: | 20260508 09:36:16.255 | INFO | BytesWritten : 3086131169303 | 20260508 09:36:16.255 | INFO | NumEvents : 1484410857140 | 20260508 09:36:17.812 | INFO | Analyzing chunk frames = [7077888,7078400) | 20260508 09:36:17.812 | INFO | MemInfo: { | 20260508 09:36:17.812 | INFO | "gpu" : " 17 GB", | 20260508 09:36:17.812 | INFO | "huge" : "", | 20260508 09:36:17.812 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:17.812 | INFO | "malloc" : " 14 GB", | 20260508 09:36:17.812 | INFO | "pinned" : " 29 GB" | 20260508 09:36:17.812 | INFO | } | 20260508 09:36:17.812 | INFO | Perf report for the last 5.13607 seconds: | 20260508 09:36:17.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:17.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:17.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:17.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:17.812 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 09:36:17.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:17.812 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:36:17.812 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 09:36:17.812 | INFO | BAZWRITER : 67.4%, 2.8, 97.1ms, 0 | 20260508 09:36:17.812 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:36:17.813 | INFO | Uploaded 36GiB to GPU in 2.60213s (13.8348GiB/s) | 20260508 09:36:17.813 | INFO | Downloaded 38GiB to GPU in 2.71449s (13.9989GiB/s) | 20260508 09:36:22.949 | INFO | Analyzing chunk frames = [7078400,7078912) | 20260508 09:36:22.949 | INFO | MemInfo: { | 20260508 09:36:22.949 | INFO | "gpu" : " 17 GB", | 20260508 09:36:22.949 | INFO | "huge" : "", | 20260508 09:36:22.949 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:22.949 | INFO | "malloc" : " 14 GB", | 20260508 09:36:22.949 | INFO | "pinned" : " 29 GB" | 20260508 09:36:22.950 | INFO | } | 20260508 09:36:22.950 | INFO | Perf report for the last 5.13761 seconds: | 20260508 09:36:22.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:22.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 09:36:22.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:36:22.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:22.950 | INFO | Uploaded 36GiB to GPU in 2.47634s (14.5376GiB/s) | 20260508 09:36:22.950 | INFO | Downloaded 34GiB to GPU in 2.48221s (13.6975GiB/s) | 20260508 09:36:28.092 | INFO | Analyzing chunk frames = [7078912,7079424) | 20260508 09:36:28.092 | INFO | MemInfo: { | 20260508 09:36:28.092 | INFO | "gpu" : " 17 GB", | 20260508 09:36:28.092 | INFO | "huge" : "", | 20260508 09:36:28.092 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:28.093 | INFO | "malloc" : " 14 GB", | 20260508 09:36:28.093 | INFO | "pinned" : " 29 GB" | 20260508 09:36:28.093 | INFO | } | 20260508 09:36:28.093 | INFO | Perf report for the last 5.14284 seconds: | 20260508 09:36:28.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:28.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 09:36:28.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:36:28.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:28.093 | INFO | Uploaded 36GiB to GPU in 2.45762s (14.6483GiB/s) | 20260508 09:36:28.093 | INFO | Downloaded 34GiB to GPU in 2.42551s (14.0176GiB/s) | 20260508 09:36:33.232 | INFO | Analyzing chunk frames = [7079424,7079936) | 20260508 09:36:33.232 | INFO | MemInfo: { | 20260508 09:36:33.232 | INFO | "gpu" : " 17 GB", | 20260508 09:36:33.232 | INFO | "huge" : "", | 20260508 09:36:33.232 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:33.232 | INFO | "malloc" : " 14 GB", | 20260508 09:36:33.233 | INFO | "pinned" : " 29 GB" | 20260508 09:36:33.233 | INFO | } | 20260508 09:36:33.233 | INFO | Perf report for the last 5.14011 seconds: | 20260508 09:36:33.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:33.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.233 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 09:36:33.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.233 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:36:33.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:33.234 | INFO | Uploaded 36GiB to GPU in 2.48455s (14.4896GiB/s) | 20260508 09:36:33.234 | INFO | Downloaded 34GiB to GPU in 2.43009s (13.9913GiB/s) | 20260508 09:36:38.367 | INFO | Analyzing chunk frames = [7079936,7080448) | 20260508 09:36:38.367 | INFO | MemInfo: { | 20260508 09:36:38.367 | INFO | "gpu" : " 17 GB", | 20260508 09:36:38.367 | INFO | "huge" : "", | 20260508 09:36:38.368 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:38.368 | INFO | "malloc" : " 14 GB", | 20260508 09:36:38.368 | INFO | "pinned" : " 29 GB" | 20260508 09:36:38.368 | INFO | } | 20260508 09:36:38.368 | INFO | Perf report for the last 5.13503 seconds: | 20260508 09:36:38.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:38.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:36:38.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:36:38.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:38.368 | INFO | Uploaded 36GiB to GPU in 2.45778s (14.6474GiB/s) | 20260508 09:36:38.368 | INFO | Downloaded 34GiB to GPU in 2.54408s (13.3644GiB/s) | 20260508 09:36:43.500 | INFO | Analyzing chunk frames = [7080448,7080960) | 20260508 09:36:43.500 | INFO | MemInfo: { | 20260508 09:36:43.500 | INFO | "gpu" : " 17 GB", | 20260508 09:36:43.500 | INFO | "huge" : "", | 20260508 09:36:43.500 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:43.500 | INFO | "malloc" : " 14 GB", | 20260508 09:36:43.500 | INFO | "pinned" : " 29 GB" | 20260508 09:36:43.500 | INFO | } | 20260508 09:36:43.500 | INFO | Perf report for the last 5.1324 seconds: | 20260508 09:36:43.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:43.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.500 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 09:36:43.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.500 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:36:43.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:43.501 | INFO | Uploaded 36GiB to GPU in 2.37605s (15.1512GiB/s) | 20260508 09:36:43.501 | INFO | Downloaded 34GiB to GPU in 2.43398s (13.9689GiB/s) | 20260508 09:36:48.649 | INFO | Analyzing chunk frames = [7080960,7081472) | 20260508 09:36:48.649 | INFO | MemInfo: { | 20260508 09:36:48.649 | INFO | "gpu" : " 17 GB", | 20260508 09:36:48.649 | INFO | "huge" : "", | 20260508 09:36:48.649 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:48.649 | INFO | "malloc" : " 14 GB", | 20260508 09:36:48.649 | INFO | "pinned" : " 29 GB" | 20260508 09:36:48.649 | INFO | } | 20260508 09:36:48.649 | INFO | Perf report for the last 5.14908 seconds: | 20260508 09:36:48.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:48.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.649 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 09:36:48.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.649 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:36:48.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:48.650 | INFO | Uploaded 36GiB to GPU in 2.42678s (14.8345GiB/s) | 20260508 09:36:48.650 | INFO | Downloaded 34GiB to GPU in 2.43702s (13.9515GiB/s) | 20260508 09:36:53.789 | INFO | Analyzing chunk frames = [7081472,7081984) | 20260508 09:36:53.789 | INFO | MemInfo: { | 20260508 09:36:53.789 | INFO | "gpu" : " 17 GB", | 20260508 09:36:53.789 | INFO | "huge" : "", | 20260508 09:36:53.789 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:53.789 | INFO | "malloc" : " 14 GB", | 20260508 09:36:53.789 | INFO | "pinned" : " 29 GB" | 20260508 09:36:53.789 | INFO | } | 20260508 09:36:53.789 | INFO | Perf report for the last 5.1393 seconds: | 20260508 09:36:53.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:53.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:36:53.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 09:36:53.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:53.789 | INFO | Uploaded 36GiB to GPU in 2.41453s (14.9098GiB/s) | 20260508 09:36:53.789 | INFO | Downloaded 34GiB to GPU in 2.44123s (13.9274GiB/s) | 20260508 09:36:53.886 | INFO | Starting a round of Baz IO | 20260508 09:36:57.295 | INFO | Populating RTMetrics report | 20260508 09:36:57.296 | INFO | RTMetrics report populated | 20260508 09:36:57.296 | INFO | Opening tmp RTMetrics json file | 20260508 09:36:57.306 | INFO | Serializing RTMetrics to json | 20260508 09:36:57.307 | INFO | Serialization done, swapping tmp file now... | 20260508 09:36:57.307 | INFO | RTMetrics done | 20260508 09:36:57.365 | INFO | Finished a round of Baz IO: Wrote 1.48 GiB in 3.479 seconds | 20260508 09:36:57.366 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:36:57.366 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.251218% / 3.97364e-06% / 7.54992e-05% | 20260508 09:36:57.366 | INFO | Baz IO overall: | 20260508 09:36:57.366 | INFO | BytesWritten : 3087720622311 | 20260508 09:36:57.366 | INFO | NumEvents : 1485162012736 | 20260508 09:36:58.923 | INFO | Analyzing chunk frames = [7081984,7082496) | 20260508 09:36:58.923 | INFO | MemInfo: { | 20260508 09:36:58.923 | INFO | "gpu" : " 17 GB", | 20260508 09:36:58.923 | INFO | "huge" : "", | 20260508 09:36:58.923 | INFO | "huge_shared" : " 29 GB", | 20260508 09:36:58.923 | INFO | "malloc" : " 14 GB", | 20260508 09:36:58.923 | INFO | "pinned" : " 29 GB" | 20260508 09:36:58.923 | INFO | } | 20260508 09:36:58.923 | INFO | Perf report for the last 5.13442 seconds: | 20260508 09:36:58.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:36:58.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:58.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:58.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:58.923 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:36:58.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:36:58.923 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:36:58.923 | INFO | PRE_HQ : 65.5%, 1.6, 54.4ms, 0 | 20260508 09:36:58.923 | INFO | BAZWRITER : 67.1%, 2.5, 87.4ms, 0 | 20260508 09:36:58.923 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:36:58.923 | INFO | Uploaded 36GiB to GPU in 2.62033s (13.7387GiB/s) | 20260508 09:36:58.923 | INFO | Downloaded 38GiB to GPU in 2.70556s (14.0451GiB/s) | 20260508 09:37:04.060 | INFO | Analyzing chunk frames = [7082496,7083008) | 20260508 09:37:04.060 | INFO | MemInfo: { | 20260508 09:37:04.060 | INFO | "gpu" : " 17 GB", | 20260508 09:37:04.060 | INFO | "huge" : "", | 20260508 09:37:04.060 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:04.060 | INFO | "malloc" : " 14 GB", | 20260508 09:37:04.061 | INFO | "pinned" : " 29 GB" | 20260508 09:37:04.061 | INFO | } | 20260508 09:37:04.061 | INFO | Perf report for the last 5.13763 seconds: | 20260508 09:37:04.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:04.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:37:04.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:37:04.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:04.061 | INFO | Uploaded 36GiB to GPU in 2.54327s (14.155GiB/s) | 20260508 09:37:04.061 | INFO | Downloaded 34GiB to GPU in 2.45698s (13.8381GiB/s) | 20260508 09:37:09.205 | INFO | Analyzing chunk frames = [7083008,7083520) | 20260508 09:37:09.205 | INFO | MemInfo: { | 20260508 09:37:09.205 | INFO | "gpu" : " 17 GB", | 20260508 09:37:09.205 | INFO | "huge" : "", | 20260508 09:37:09.205 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:09.205 | INFO | "malloc" : " 14 GB", | 20260508 09:37:09.205 | INFO | "pinned" : " 29 GB" | 20260508 09:37:09.205 | INFO | } | 20260508 09:37:09.205 | INFO | Perf report for the last 5.14465 seconds: | 20260508 09:37:09.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:09.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.205 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 09:37:09.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.205 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:37:09.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:09.206 | INFO | Uploaded 36GiB to GPU in 2.50052s (14.397GiB/s) | 20260508 09:37:09.206 | INFO | Downloaded 34GiB to GPU in 2.42516s (14.0197GiB/s) | 20260508 09:37:14.344 | INFO | Analyzing chunk frames = [7083520,7084032) | 20260508 09:37:14.344 | INFO | MemInfo: { | 20260508 09:37:14.344 | INFO | "gpu" : " 17 GB", | 20260508 09:37:14.344 | INFO | "huge" : "", | 20260508 09:37:14.344 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:14.344 | INFO | "malloc" : " 14 GB", | 20260508 09:37:14.344 | INFO | "pinned" : " 29 GB" | 20260508 09:37:14.344 | INFO | } | 20260508 09:37:14.344 | INFO | Perf report for the last 5.13859 seconds: | 20260508 09:37:14.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:14.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 09:37:14.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:37:14.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:14.345 | INFO | Uploaded 36GiB to GPU in 2.5168s (14.3039GiB/s) | 20260508 09:37:14.345 | INFO | Downloaded 34GiB to GPU in 2.42111s (14.0432GiB/s) | 20260508 09:37:19.485 | INFO | Analyzing chunk frames = [7084032,7084544) | 20260508 09:37:19.485 | INFO | MemInfo: { | 20260508 09:37:19.485 | INFO | "gpu" : " 17 GB", | 20260508 09:37:19.485 | INFO | "huge" : "", | 20260508 09:37:19.485 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:19.485 | INFO | "malloc" : " 14 GB", | 20260508 09:37:19.485 | INFO | "pinned" : " 29 GB" | 20260508 09:37:19.485 | INFO | } | 20260508 09:37:19.485 | INFO | Perf report for the last 5.14132 seconds: | 20260508 09:37:19.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:19.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.485 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 09:37:19.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.485 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:37:19.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:19.486 | INFO | Uploaded 36GiB to GPU in 2.48263s (14.5007GiB/s) | 20260508 09:37:19.486 | INFO | Downloaded 34GiB to GPU in 2.5365s (13.4043GiB/s) | 20260508 09:37:24.624 | INFO | Analyzing chunk frames = [7084544,7085056) | 20260508 09:37:24.624 | INFO | MemInfo: { | 20260508 09:37:24.624 | INFO | "gpu" : " 17 GB", | 20260508 09:37:24.624 | INFO | "huge" : "", | 20260508 09:37:24.624 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:24.624 | INFO | "malloc" : " 14 GB", | 20260508 09:37:24.624 | INFO | "pinned" : " 29 GB" | 20260508 09:37:24.624 | INFO | } | 20260508 09:37:24.624 | INFO | Perf report for the last 5.13883 seconds: | 20260508 09:37:24.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:24.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.624 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:37:24.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.624 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:37:24.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:24.625 | INFO | Uploaded 36GiB to GPU in 2.36539s (15.2195GiB/s) | 20260508 09:37:24.625 | INFO | Downloaded 34GiB to GPU in 2.42737s (14.0069GiB/s) | 20260508 09:37:29.759 | INFO | Analyzing chunk frames = [7085056,7085568) | 20260508 09:37:29.759 | INFO | MemInfo: { | 20260508 09:37:29.759 | INFO | "gpu" : " 17 GB", | 20260508 09:37:29.759 | INFO | "huge" : "", | 20260508 09:37:29.759 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:29.759 | INFO | "malloc" : " 14 GB", | 20260508 09:37:29.760 | INFO | "pinned" : " 29 GB" | 20260508 09:37:29.760 | INFO | } | 20260508 09:37:29.760 | INFO | Perf report for the last 5.13546 seconds: | 20260508 09:37:29.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:29.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | ANALYSIS : 65.8%, 3.0, 102.5ms, 0 | 20260508 09:37:29.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | PRE_HQ : 32.5%, 1.0, 16.8ms, 0 | 20260508 09:37:29.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:29.760 | INFO | Uploaded 36GiB to GPU in 2.46573s (14.6001GiB/s) | 20260508 09:37:29.760 | INFO | Downloaded 34GiB to GPU in 2.44886s (13.884GiB/s) | 20260508 09:37:34.906 | INFO | Analyzing chunk frames = [7085568,7086080) | 20260508 09:37:34.906 | INFO | MemInfo: { | 20260508 09:37:34.906 | INFO | "gpu" : " 17 GB", | 20260508 09:37:34.906 | INFO | "huge" : "", | 20260508 09:37:34.906 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:34.906 | INFO | "malloc" : " 14 GB", | 20260508 09:37:34.907 | INFO | "pinned" : " 29 GB" | 20260508 09:37:34.907 | INFO | } | 20260508 09:37:34.907 | INFO | Perf report for the last 5.14711 seconds: | 20260508 09:37:34.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:34.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 09:37:34.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260508 09:37:34.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:34.907 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:37:34.907 | INFO | Total mem usage: 16.7 GiB | 20260508 09:37:34.907 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:37:34.907 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -13.8%) | 20260508 09:37:34.907 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:37:34.907 | INFO | High Water Breakdown: | 20260508 09:37:34.907 | INFO | 59.9% : 8.79 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:37:34.907 | INFO | 39.9% : 5.85 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:37:34.907 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:37:34.907 | INFO | ------------------------------------------ | 20260508 09:37:34.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:37:34.907 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:37:34.907 | INFO | Total mem usage: 26.5 GiB | 20260508 09:37:34.907 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:37:34.907 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:37:34.907 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:37:34.907 | INFO | High Water Breakdown: | 20260508 09:37:34.908 | INFO | Uploaded 36GiB to GPU in 2.40871s (14.9457GiB/s) | 20260508 09:37:34.908 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:37:34.908 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:37:34.908 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:37:34.908 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:37:34.908 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:37:34.908 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:37:34.908 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:37:34.908 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:37:34.908 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:37:34.908 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:37:34.908 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:37:34.908 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:37:34.908 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:37:34.908 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:37:34.908 | INFO | ------------------------------------------ | 20260508 09:37:34.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:37:34.908 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:37:34.908 | INFO | Total mem usage: 27.8 GiB | 20260508 09:37:34.908 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:37:34.908 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:37:34.908 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:37:34.908 | INFO | High Water Breakdown: | 20260508 09:37:34.908 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:37:34.908 | INFO | ------------------------------------------ | 20260508 09:37:34.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:37:34.908 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:37:34.908 | INFO | Total mem usage: 15.7 GiB | 20260508 09:37:34.908 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:37:34.908 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:37:34.908 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:37:34.908 | INFO | High Water Breakdown: | 20260508 09:37:34.908 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:37:34.908 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:37:34.908 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:37:34.908 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:37:34.908 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:37:34.908 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:37:34.908 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:37:34.908 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:37:34.908 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:37:34.908 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:37:34.908 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:37:34.908 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:37:34.908 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:37:34.908 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:37:34.908 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:37:34.908 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:37:34.908 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:37:34.908 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:37:34.908 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:37:34.908 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:37:34.908 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:37:34.908 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:37:34.908 | INFO | Downloaded 34GiB to GPU in 2.43258s (13.9769GiB/s) | 20260508 09:37:34.908 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:37:34.908 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:37:34.908 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:37:34.908 | INFO | ------------------------------------------ | 20260508 09:37:34.908 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:37:34.908 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:37:34.908 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:37:34.908 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:37:34.909 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 8.71e-01 | 20260508 09:37:34.909 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.06e-01, 1.10e-02 / 6.89e-03 | 20260508 09:37:34.909 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.48e+00, 5.95e-01 / 1.99e-01 | 20260508 09:37:34.909 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.42e+00, 2.08e-01 / 1.32e-01 | 20260508 09:37:34.909 | INFO | PerfReport: 15.95%, DME -- 4.18e+01 / 4.48e+01, 3.61e+00 / 9.08e-01 | 20260508 09:37:34.909 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.85e+00 / 1.21e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.45e+00, 4.07e-01 / 1.83e-01 | 20260508 09:37:34.909 | INFO | PerfReport: 3.04%, Metrics -- 9.94e-01 / 1.15e+00, 2.50e-01 / 1.11e-01 | 20260508 09:37:34.909 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.04e-01 / 3.38e-01, 4.95e-02 / 2.39e-02 | 20260508 09:37:34.909 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:37:34.909 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:37:34.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:37:34.909 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 24.44%, Upload -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 24.54%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 31.04%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 2.25%, LockUpload -- 2.37e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 2.43%, LockDownload -- 2.56e+00 / 0.00e+00, 3.91e+00 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 15.29%, LockCompute -- 1.61e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:37:34.909 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 09:37:35.020 | INFO | Starting a round of Baz IO | 20260508 09:37:38.433 | INFO | Populating RTMetrics report | 20260508 09:37:38.434 | INFO | RTMetrics report populated | 20260508 09:37:38.434 | INFO | Opening tmp RTMetrics json file | 20260508 09:37:38.444 | INFO | Serializing RTMetrics to json | 20260508 09:37:38.445 | INFO | Serialization done, swapping tmp file now... | 20260508 09:37:38.446 | INFO | RTMetrics done | 20260508 09:37:38.502 | INFO | Finished a round of Baz IO: Wrote 1.48 GiB in 3.481 seconds | 20260508 09:37:38.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.29% / 0.00% / 59.71% | 20260508 09:37:38.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.250598% / 3.97364e-06% / 8.74201e-05% | 20260508 09:37:38.502 | INFO | Baz IO overall: | 20260508 09:37:38.502 | INFO | BytesWritten : 3089309265857 | 20260508 09:37:38.502 | INFO | NumEvents : 1485912605396 | 20260508 09:37:40.043 | INFO | Analyzing chunk frames = [7086080,7086592) | 20260508 09:37:40.043 | INFO | MemInfo: { | 20260508 09:37:40.043 | INFO | "gpu" : " 17 GB", | 20260508 09:37:40.043 | INFO | "huge" : "", | 20260508 09:37:40.043 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:40.043 | INFO | "malloc" : " 14 GB", | 20260508 09:37:40.043 | INFO | "pinned" : " 29 GB" | 20260508 09:37:40.043 | INFO | } | 20260508 09:37:40.043 | INFO | Perf report for the last 5.13618 seconds: | 20260508 09:37:40.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:40.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:40.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:40.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:40.043 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 09:37:40.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:40.043 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260508 09:37:40.043 | INFO | PRE_HQ : 66.6%, 1.8, 61.0ms, 0 | 20260508 09:37:40.043 | INFO | BAZWRITER : 67.2%, 2.7, 93.9ms, 0 | 20260508 09:37:40.043 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:37:40.043 | INFO | Uploaded 36GiB to GPU in 2.60507s (13.8192GiB/s) | 20260508 09:37:40.043 | INFO | Downloaded 38GiB to GPU in 2.70413s (14.0526GiB/s) | 20260508 09:37:45.181 | INFO | Analyzing chunk frames = [7086592,7087104) | 20260508 09:37:45.181 | INFO | MemInfo: { | 20260508 09:37:45.181 | INFO | "gpu" : " 17 GB", | 20260508 09:37:45.181 | INFO | "huge" : "", | 20260508 09:37:45.181 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:45.181 | INFO | "malloc" : " 14 GB", | 20260508 09:37:45.181 | INFO | "pinned" : " 29 GB" | 20260508 09:37:45.181 | INFO | } | 20260508 09:37:45.181 | INFO | Perf report for the last 5.13803 seconds: | 20260508 09:37:45.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:45.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 09:37:45.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:37:45.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:45.181 | INFO | Uploaded 36GiB to GPU in 2.48481s (14.488GiB/s) | 20260508 09:37:45.181 | INFO | Downloaded 34GiB to GPU in 2.46094s (13.8159GiB/s) | 20260508 09:37:50.319 | INFO | Analyzing chunk frames = [7087104,7087616) | 20260508 09:37:50.319 | INFO | MemInfo: { | 20260508 09:37:50.319 | INFO | "gpu" : " 17 GB", | 20260508 09:37:50.319 | INFO | "huge" : "", | 20260508 09:37:50.319 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:50.319 | INFO | "malloc" : " 14 GB", | 20260508 09:37:50.319 | INFO | "pinned" : " 29 GB" | 20260508 09:37:50.319 | INFO | } | 20260508 09:37:50.319 | INFO | Perf report for the last 5.13833 seconds: | 20260508 09:37:50.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:50.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.319 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:37:50.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.319 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:37:50.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:50.320 | INFO | Uploaded 36GiB to GPU in 2.426s (14.8393GiB/s) | 20260508 09:37:50.320 | INFO | Downloaded 34GiB to GPU in 2.4211s (14.0432GiB/s) | 20260508 09:37:55.460 | INFO | Analyzing chunk frames = [7087616,7088128) | 20260508 09:37:55.460 | INFO | MemInfo: { | 20260508 09:37:55.460 | INFO | "gpu" : " 17 GB", | 20260508 09:37:55.460 | INFO | "huge" : "", | 20260508 09:37:55.460 | INFO | "huge_shared" : " 29 GB", | 20260508 09:37:55.460 | INFO | "malloc" : " 14 GB", | 20260508 09:37:55.460 | INFO | "pinned" : " 29 GB" | 20260508 09:37:55.460 | INFO | } | 20260508 09:37:55.460 | INFO | Perf report for the last 5.14069 seconds: | 20260508 09:37:55.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:37:55.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 09:37:55.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:37:55.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:37:55.460 | INFO | Uploaded 36GiB to GPU in 2.42435s (14.8493GiB/s) | 20260508 09:37:55.460 | INFO | Downloaded 34GiB to GPU in 2.42227s (14.0364GiB/s) | 20260508 09:38:00.597 | INFO | Analyzing chunk frames = [7088128,7088640) | 20260508 09:38:00.597 | INFO | MemInfo: { | 20260508 09:38:00.597 | INFO | "gpu" : " 17 GB", | 20260508 09:38:00.597 | INFO | "huge" : "", | 20260508 09:38:00.597 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:00.597 | INFO | "malloc" : " 14 GB", | 20260508 09:38:00.597 | INFO | "pinned" : " 29 GB" | 20260508 09:38:00.597 | INFO | } | 20260508 09:38:00.597 | INFO | Perf report for the last 5.13724 seconds: | 20260508 09:38:00.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:00.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.597 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 09:38:00.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.597 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 09:38:00.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:00.598 | INFO | Uploaded 36GiB to GPU in 2.3618s (15.2426GiB/s) | 20260508 09:38:00.598 | INFO | Downloaded 34GiB to GPU in 2.51071s (13.542GiB/s) | 20260508 09:38:05.737 | INFO | Analyzing chunk frames = [7088640,7089152) | 20260508 09:38:05.737 | INFO | MemInfo: { | 20260508 09:38:05.737 | INFO | "gpu" : " 17 GB", | 20260508 09:38:05.737 | INFO | "huge" : "", | 20260508 09:38:05.737 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:05.737 | INFO | "malloc" : " 14 GB", | 20260508 09:38:05.737 | INFO | "pinned" : " 29 GB" | 20260508 09:38:05.737 | INFO | } | 20260508 09:38:05.737 | INFO | Perf report for the last 5.13951 seconds: | 20260508 09:38:05.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:05.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 09:38:05.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:38:05.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:05.737 | INFO | Uploaded 36GiB to GPU in 2.30446s (15.6219GiB/s) | 20260508 09:38:05.737 | INFO | Downloaded 34GiB to GPU in 2.42915s (13.9967GiB/s) | 20260508 09:38:10.874 | INFO | Analyzing chunk frames = [7089152,7089664) | 20260508 09:38:10.874 | INFO | MemInfo: { | 20260508 09:38:10.874 | INFO | "gpu" : " 17 GB", | 20260508 09:38:10.874 | INFO | "huge" : "", | 20260508 09:38:10.874 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:10.874 | INFO | "malloc" : " 14 GB", | 20260508 09:38:10.874 | INFO | "pinned" : " 29 GB" | 20260508 09:38:10.874 | INFO | } | 20260508 09:38:10.874 | INFO | Perf report for the last 5.13763 seconds: | 20260508 09:38:10.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:10.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.874 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:38:10.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.875 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 09:38:10.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:10.875 | INFO | Uploaded 36GiB to GPU in 2.39409s (15.037GiB/s) | 20260508 09:38:10.875 | INFO | Downloaded 34GiB to GPU in 2.45541s (13.847GiB/s) | 20260508 09:38:16.012 | INFO | Analyzing chunk frames = [7089664,7090176) | 20260508 09:38:16.012 | INFO | MemInfo: { | 20260508 09:38:16.012 | INFO | "gpu" : " 17 GB", | 20260508 09:38:16.012 | INFO | "huge" : "", | 20260508 09:38:16.012 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:16.012 | INFO | "malloc" : " 14 GB", | 20260508 09:38:16.012 | INFO | "pinned" : " 29 GB" | 20260508 09:38:16.012 | INFO | } | 20260508 09:38:16.012 | INFO | Perf report for the last 5.13805 seconds: | 20260508 09:38:16.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:16.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 09:38:16.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:38:16.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:16.013 | INFO | Uploaded 36GiB to GPU in 2.3905s (15.0596GiB/s) | 20260508 09:38:16.013 | INFO | Downloaded 34GiB to GPU in 2.45808s (13.8319GiB/s) | 20260508 09:38:16.110 | INFO | Starting a round of Baz IO | 20260508 09:38:19.523 | INFO | Populating RTMetrics report | 20260508 09:38:19.523 | INFO | RTMetrics report populated | 20260508 09:38:19.523 | INFO | Opening tmp RTMetrics json file | 20260508 09:38:19.524 | INFO | Serializing RTMetrics to json | 20260508 09:38:19.525 | INFO | Serialization done, swapping tmp file now... | 20260508 09:38:19.525 | INFO | RTMetrics done | 20260508 09:38:19.586 | INFO | Finished a round of Baz IO: Wrote 1.478 GiB in 3.476 seconds | 20260508 09:38:19.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.71% | 20260508 09:38:19.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.249294% / 0% / 5.96046e-05% | 20260508 09:38:19.586 | INFO | Baz IO overall: | 20260508 09:38:19.586 | INFO | BytesWritten : 3090896789276 | 20260508 09:38:19.586 | INFO | NumEvents : 1486662796426 | 20260508 09:38:21.158 | INFO | Analyzing chunk frames = [7090176,7090688) | 20260508 09:38:21.158 | INFO | MemInfo: { | 20260508 09:38:21.158 | INFO | "gpu" : " 17 GB", | 20260508 09:38:21.158 | INFO | "huge" : "", | 20260508 09:38:21.158 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:21.158 | INFO | "malloc" : " 14 GB", | 20260508 09:38:21.158 | INFO | "pinned" : " 29 GB" | 20260508 09:38:21.158 | INFO | } | 20260508 09:38:21.158 | INFO | Perf report for the last 5.14596 seconds: | 20260508 09:38:21.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:21.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:21.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:21.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:21.159 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 09:38:21.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:21.159 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 09:38:21.159 | INFO | PRE_HQ : 65.6%, 1.6, 54.4ms, 0 | 20260508 09:38:21.159 | INFO | BAZWRITER : 66.8%, 2.6, 87.8ms, 0 | 20260508 09:38:21.159 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 09:38:21.159 | INFO | Uploaded 36GiB to GPU in 2.53421s (14.2056GiB/s) | 20260508 09:38:21.159 | INFO | Downloaded 38GiB to GPU in 2.71099s (14.017GiB/s) | 20260508 09:38:26.301 | INFO | Analyzing chunk frames = [7090688,7091200) | 20260508 09:38:26.301 | INFO | MemInfo: { | 20260508 09:38:26.301 | INFO | "gpu" : " 17 GB", | 20260508 09:38:26.301 | INFO | "huge" : "", | 20260508 09:38:26.301 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:26.301 | INFO | "malloc" : " 14 GB", | 20260508 09:38:26.301 | INFO | "pinned" : " 29 GB" | 20260508 09:38:26.301 | INFO | } | 20260508 09:38:26.302 | INFO | Perf report for the last 5.14312 seconds: | 20260508 09:38:26.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:26.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 09:38:26.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:38:26.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:26.302 | INFO | Uploaded 36GiB to GPU in 2.52947s (14.2322GiB/s) | 20260508 09:38:26.302 | INFO | Downloaded 34GiB to GPU in 2.45342s (13.8582GiB/s) | 20260508 09:38:31.439 | INFO | Analyzing chunk frames = [7091200,7091712) | 20260508 09:38:31.439 | INFO | MemInfo: { | 20260508 09:38:31.439 | INFO | "gpu" : " 17 GB", | 20260508 09:38:31.439 | INFO | "huge" : "", | 20260508 09:38:31.439 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:31.439 | INFO | "malloc" : " 14 GB", | 20260508 09:38:31.439 | INFO | "pinned" : " 29 GB" | 20260508 09:38:31.439 | INFO | } | 20260508 09:38:31.439 | INFO | Perf report for the last 5.13733 seconds: | 20260508 09:38:31.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:31.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:38:31.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:38:31.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:31.439 | INFO | Uploaded 36GiB to GPU in 2.4995s (14.4029GiB/s) | 20260508 09:38:31.439 | INFO | Downloaded 34GiB to GPU in 2.42987s (13.9925GiB/s) | 20260508 09:38:36.568 | INFO | Analyzing chunk frames = [7091712,7092224) | 20260508 09:38:36.568 | INFO | MemInfo: { | 20260508 09:38:36.568 | INFO | "gpu" : " 17 GB", | 20260508 09:38:36.568 | INFO | "huge" : "", | 20260508 09:38:36.568 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:36.568 | INFO | "malloc" : " 14 GB", | 20260508 09:38:36.568 | INFO | "pinned" : " 29 GB" | 20260508 09:38:36.568 | INFO | } | 20260508 09:38:36.568 | INFO | Perf report for the last 5.12893 seconds: | 20260508 09:38:36.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:36.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | ANALYSIS : 66.3%, 3.0, 102.5ms, 0 | 20260508 09:38:36.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:38:36.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:36.568 | INFO | Uploaded 36GiB to GPU in 2.45459s (14.6664GiB/s) | 20260508 09:38:36.568 | INFO | Downloaded 34GiB to GPU in 2.39393s (14.2026GiB/s) | 20260508 09:38:41.716 | INFO | Analyzing chunk frames = [7092224,7092736) | 20260508 09:38:41.716 | INFO | MemInfo: { | 20260508 09:38:41.716 | INFO | "gpu" : " 17 GB", | 20260508 09:38:41.716 | INFO | "huge" : "", | 20260508 09:38:41.716 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:41.716 | INFO | "malloc" : " 14 GB", | 20260508 09:38:41.716 | INFO | "pinned" : " 29 GB" | 20260508 09:38:41.716 | INFO | } | 20260508 09:38:41.716 | INFO | Perf report for the last 5.14804 seconds: | 20260508 09:38:41.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:41.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 09:38:41.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 09:38:41.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:41.716 | INFO | Uploaded 36GiB to GPU in 2.30562s (15.614GiB/s) | 20260508 09:38:41.716 | INFO | Downloaded 34GiB to GPU in 2.50668s (13.5638GiB/s) | 20260508 09:38:46.854 | INFO | Analyzing chunk frames = [7092736,7093248) | 20260508 09:38:46.854 | INFO | MemInfo: { | 20260508 09:38:46.854 | INFO | "gpu" : " 17 GB", | 20260508 09:38:46.854 | INFO | "huge" : "", | 20260508 09:38:46.854 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:46.854 | INFO | "malloc" : " 14 GB", | 20260508 09:38:46.854 | INFO | "pinned" : " 29 GB" | 20260508 09:38:46.854 | INFO | } | 20260508 09:38:46.854 | INFO | Perf report for the last 5.13837 seconds: | 20260508 09:38:46.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:46.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.854 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:38:46.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.854 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260508 09:38:46.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:46.855 | INFO | Uploaded 36GiB to GPU in 2.3996s (15.0025GiB/s) | 20260508 09:38:46.855 | INFO | Downloaded 34GiB to GPU in 2.42968s (13.9936GiB/s) | 20260508 09:38:51.994 | INFO | Analyzing chunk frames = [7093248,7093760) | 20260508 09:38:51.994 | INFO | MemInfo: { | 20260508 09:38:51.994 | INFO | "gpu" : " 17 GB", | 20260508 09:38:51.994 | INFO | "huge" : "", | 20260508 09:38:51.994 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:51.994 | INFO | "malloc" : " 14 GB", | 20260508 09:38:51.994 | INFO | "pinned" : " 29 GB" | 20260508 09:38:51.994 | INFO | } | 20260508 09:38:51.994 | INFO | Perf report for the last 5.13963 seconds: | 20260508 09:38:51.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:51.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:38:51.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:38:51.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:51.994 | INFO | Uploaded 36GiB to GPU in 2.45577s (14.6593GiB/s) | 20260508 09:38:51.994 | INFO | Downloaded 34GiB to GPU in 2.43055s (13.9886GiB/s) | 20260508 09:38:57.136 | INFO | Analyzing chunk frames = [7093760,7094272) | 20260508 09:38:57.136 | INFO | MemInfo: { | 20260508 09:38:57.136 | INFO | "gpu" : " 17 GB", | 20260508 09:38:57.136 | INFO | "huge" : "", | 20260508 09:38:57.136 | INFO | "huge_shared" : " 29 GB", | 20260508 09:38:57.136 | INFO | "malloc" : " 14 GB", | 20260508 09:38:57.136 | INFO | "pinned" : " 29 GB" | 20260508 09:38:57.136 | INFO | } | 20260508 09:38:57.136 | INFO | Perf report for the last 5.14239 seconds: | 20260508 09:38:57.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:38:57.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.136 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 09:38:57.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.137 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:38:57.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:38:57.137 | INFO | Uploaded 36GiB to GPU in 2.42456s (14.848GiB/s) | 20260508 09:38:57.137 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:38:57.137 | INFO | Total mem usage: 16.7 GiB | 20260508 09:38:57.137 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:38:57.137 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:38:57.137 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:38:57.137 | INFO | High Water Breakdown: | 20260508 09:38:57.137 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:38:57.137 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:38:57.137 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:38:57.137 | INFO | ------------------------------------------ | 20260508 09:38:57.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:38:57.137 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:38:57.137 | INFO | Total mem usage: 26.5 GiB | 20260508 09:38:57.137 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:38:57.137 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:38:57.137 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:38:57.137 | INFO | High Water Breakdown: | 20260508 09:38:57.137 | INFO | Downloaded 34GiB to GPU in 2.42305s (14.0319GiB/s) | 20260508 09:38:57.137 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:38:57.137 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:38:57.137 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:38:57.137 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:38:57.137 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:38:57.137 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:38:57.137 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:38:57.137 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:38:57.137 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:38:57.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:38:57.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:38:57.137 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:38:57.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:38:57.137 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:38:57.137 | INFO | ------------------------------------------ | 20260508 09:38:57.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:38:57.137 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:38:57.137 | INFO | Total mem usage: 27.8 GiB | 20260508 09:38:57.137 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:38:57.137 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:38:57.137 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:38:57.137 | INFO | High Water Breakdown: | 20260508 09:38:57.137 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:38:57.137 | INFO | ------------------------------------------ | 20260508 09:38:57.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:38:57.137 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:38:57.137 | INFO | Total mem usage: 15.7 GiB | 20260508 09:38:57.137 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:38:57.137 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:38:57.137 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:38:57.137 | INFO | High Water Breakdown: | 20260508 09:38:57.137 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:38:57.137 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:38:57.137 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:38:57.137 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:38:57.137 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:38:57.137 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:38:57.137 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:38:57.137 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:38:57.137 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:38:57.137 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:38:57.137 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:38:57.137 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:38:57.137 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:38:57.137 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:38:57.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:38:57.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:38:57.137 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:38:57.137 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:38:57.137 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:38:57.137 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:38:57.137 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:38:57.137 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:38:57.137 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:38:57.137 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:38:57.137 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:38:57.137 | INFO | ------------------------------------------ | 20260508 09:38:57.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:38:57.137 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:38:57.137 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:38:57.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:38:57.137 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 7.10e-01 | 20260508 09:38:57.137 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.05e-01, 1.09e-02 / 4.46e-03 | 20260508 09:38:57.137 | INFO | PerfReport: 12.19%, Baseline -- 4.00e+00 / 4.40e+00, 5.98e-01 / 2.13e-01 | 20260508 09:38:57.138 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.12e-01 / 1.20e-01 | 20260508 09:38:57.138 | INFO | PerfReport: 16.01%, DME -- 4.20e+01 / 4.55e+01, 3.85e+00 / 1.13e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.85e+00 / 1.26e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.57e+00, 4.05e-01 / 2.61e-01 | 20260508 09:38:57.138 | INFO | PerfReport: 3.04%, Metrics -- 9.96e-01 / 1.14e+00, 2.48e-01 / 1.22e-01 | 20260508 09:38:57.138 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.44e-01, 4.91e-02 / 2.90e-02 | 20260508 09:38:57.138 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:38:57.138 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:38:57.138 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:38:57.138 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 24.11%, Upload -- 2.54e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 24.51%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 31.16%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 2.61%, LockDownload -- 2.75e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 15.47%, LockCompute -- 1.63e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:38:57.138 | INFO | PerfReport: 0.01%, Misc -- 1.50e-02 / 0.00e+00, 3.25e-02 / 0.00e+00 | 20260508 09:38:57.234 | INFO | Starting a round of Baz IO | 20260508 09:39:00.661 | INFO | Populating RTMetrics report | 20260508 09:39:00.661 | INFO | RTMetrics report populated | 20260508 09:39:00.661 | INFO | Opening tmp RTMetrics json file | 20260508 09:39:00.670 | INFO | Serializing RTMetrics to json | 20260508 09:39:00.672 | INFO | Serialization done, swapping tmp file now... | 20260508 09:39:00.672 | INFO | RTMetrics done | 20260508 09:39:00.728 | INFO | Finished a round of Baz IO: Wrote 1.477 GiB in 3.494 seconds | 20260508 09:39:00.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.72% | 20260508 09:39:00.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.248675% / 3.97364e-06% / 3.97364e-05% | 20260508 09:39:00.728 | INFO | Baz IO overall: | 20260508 09:39:00.728 | INFO | BytesWritten : 3092482926076 | 20260508 09:39:00.728 | INFO | NumEvents : 1487412389572 | 20260508 09:39:02.271 | INFO | Analyzing chunk frames = [7094272,7094784) | 20260508 09:39:02.271 | INFO | MemInfo: { | 20260508 09:39:02.271 | INFO | "gpu" : " 17 GB", | 20260508 09:39:02.271 | INFO | "huge" : "", | 20260508 09:39:02.271 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:02.271 | INFO | "malloc" : " 14 GB", | 20260508 09:39:02.271 | INFO | "pinned" : " 29 GB" | 20260508 09:39:02.271 | INFO | } | 20260508 09:39:02.271 | INFO | Perf report for the last 5.13439 seconds: | 20260508 09:39:02.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:02.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:02.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:02.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:02.271 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 09:39:02.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:02.271 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:39:02.271 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 09:39:02.271 | INFO | BAZWRITER : 67.2%, 2.5, 86.9ms, 0 | 20260508 09:39:02.271 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:39:02.271 | INFO | Uploaded 36GiB to GPU in 2.60921s (13.7973GiB/s) | 20260508 09:39:02.271 | INFO | Downloaded 38GiB to GPU in 2.71053s (14.0194GiB/s) | 20260508 09:39:07.410 | INFO | Analyzing chunk frames = [7094784,7095296) | 20260508 09:39:07.410 | INFO | MemInfo: { | 20260508 09:39:07.410 | INFO | "gpu" : " 17 GB", | 20260508 09:39:07.410 | INFO | "huge" : "", | 20260508 09:39:07.410 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:07.410 | INFO | "malloc" : " 14 GB", | 20260508 09:39:07.410 | INFO | "pinned" : " 29 GB" | 20260508 09:39:07.410 | INFO | } | 20260508 09:39:07.410 | INFO | Perf report for the last 5.13919 seconds: | 20260508 09:39:07.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:07.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 09:39:07.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:39:07.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:07.410 | INFO | Uploaded 36GiB to GPU in 2.45336s (14.6738GiB/s) | 20260508 09:39:07.410 | INFO | Downloaded 34GiB to GPU in 2.4733s (13.7468GiB/s) | 20260508 09:39:12.552 | INFO | Analyzing chunk frames = [7095296,7095808) | 20260508 09:39:12.552 | INFO | MemInfo: { | 20260508 09:39:12.552 | INFO | "gpu" : " 17 GB", | 20260508 09:39:12.552 | INFO | "huge" : "", | 20260508 09:39:12.552 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:12.552 | INFO | "malloc" : " 14 GB", | 20260508 09:39:12.552 | INFO | "pinned" : " 29 GB" | 20260508 09:39:12.552 | INFO | } | 20260508 09:39:12.552 | INFO | Perf report for the last 5.14233 seconds: | 20260508 09:39:12.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:12.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.552 | INFO | ANALYSIS : 67.1%, 3.0, 104.3ms, 0 | 20260508 09:39:12.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.552 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:39:12.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:12.553 | INFO | Uploaded 36GiB to GPU in 2.57246s (13.9944GiB/s) | 20260508 09:39:12.553 | INFO | Downloaded 34GiB to GPU in 2.45199s (13.8663GiB/s) | 20260508 09:39:17.687 | INFO | Analyzing chunk frames = [7095808,7096320) | 20260508 09:39:17.687 | INFO | MemInfo: { | 20260508 09:39:17.687 | INFO | "gpu" : " 17 GB", | 20260508 09:39:17.687 | INFO | "huge" : "", | 20260508 09:39:17.687 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:17.687 | INFO | "malloc" : " 14 GB", | 20260508 09:39:17.687 | INFO | "pinned" : " 29 GB" | 20260508 09:39:17.687 | INFO | } | 20260508 09:39:17.687 | INFO | Perf report for the last 5.13472 seconds: | 20260508 09:39:17.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:17.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.687 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:39:17.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.687 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:39:17.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:17.688 | INFO | Uploaded 36GiB to GPU in 2.48238s (14.5022GiB/s) | 20260508 09:39:17.688 | INFO | Downloaded 34GiB to GPU in 2.43253s (13.9772GiB/s) | 20260508 09:39:22.823 | INFO | Analyzing chunk frames = [7096320,7096832) | 20260508 09:39:22.823 | INFO | MemInfo: { | 20260508 09:39:22.823 | INFO | "gpu" : " 17 GB", | 20260508 09:39:22.823 | INFO | "huge" : "", | 20260508 09:39:22.823 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:22.823 | INFO | "malloc" : " 14 GB", | 20260508 09:39:22.823 | INFO | "pinned" : " 29 GB" | 20260508 09:39:22.823 | INFO | } | 20260508 09:39:22.823 | INFO | Perf report for the last 5.13577 seconds: | 20260508 09:39:22.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:22.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 09:39:22.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 09:39:22.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:22.823 | INFO | Uploaded 36GiB to GPU in 2.37899s (15.1325GiB/s) | 20260508 09:39:22.823 | INFO | Downloaded 34GiB to GPU in 2.53208s (13.4277GiB/s) | 20260508 09:39:27.965 | INFO | Analyzing chunk frames = [7096832,7097344) | 20260508 09:39:27.965 | INFO | MemInfo: { | 20260508 09:39:27.965 | INFO | "gpu" : " 17 GB", | 20260508 09:39:27.965 | INFO | "huge" : "", | 20260508 09:39:27.965 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:27.965 | INFO | "malloc" : " 14 GB", | 20260508 09:39:27.965 | INFO | "pinned" : " 29 GB" | 20260508 09:39:27.965 | INFO | } | 20260508 09:39:27.965 | INFO | Perf report for the last 5.14256 seconds: | 20260508 09:39:27.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:27.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.965 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 09:39:27.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.966 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:39:27.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:27.966 | INFO | Uploaded 36GiB to GPU in 2.33016s (15.4496GiB/s) | 20260508 09:39:27.966 | INFO | Downloaded 34GiB to GPU in 2.44634s (13.8983GiB/s) | 20260508 09:39:33.108 | INFO | Analyzing chunk frames = [7097344,7097856) | 20260508 09:39:33.108 | INFO | MemInfo: { | 20260508 09:39:33.108 | INFO | "gpu" : " 17 GB", | 20260508 09:39:33.108 | INFO | "huge" : "", | 20260508 09:39:33.108 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:33.108 | INFO | "malloc" : " 14 GB", | 20260508 09:39:33.108 | INFO | "pinned" : " 29 GB" | 20260508 09:39:33.108 | INFO | } | 20260508 09:39:33.108 | INFO | Perf report for the last 5.14302 seconds: | 20260508 09:39:33.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:33.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.109 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:39:33.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.109 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:39:33.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:33.109 | INFO | Uploaded 36GiB to GPU in 2.37522s (15.1565GiB/s) | 20260508 09:39:33.109 | INFO | Downloaded 34GiB to GPU in 2.45785s (13.8333GiB/s) | 20260508 09:39:38.249 | INFO | Analyzing chunk frames = [7097856,7098368) | 20260508 09:39:38.249 | INFO | MemInfo: { | 20260508 09:39:38.249 | INFO | "gpu" : " 17 GB", | 20260508 09:39:38.249 | INFO | "huge" : "", | 20260508 09:39:38.249 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:38.250 | INFO | "malloc" : " 14 GB", | 20260508 09:39:38.250 | INFO | "pinned" : " 29 GB" | 20260508 09:39:38.250 | INFO | } | 20260508 09:39:38.250 | INFO | Perf report for the last 5.14124 seconds: | 20260508 09:39:38.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:38.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:39:38.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:39:38.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:38.250 | INFO | Uploaded 36GiB to GPU in 2.38859s (15.0717GiB/s) | 20260508 09:39:38.250 | INFO | Downloaded 34GiB to GPU in 2.46131s (13.8138GiB/s) | 20260508 09:39:38.347 | INFO | Starting a round of Baz IO | 20260508 09:39:41.777 | INFO | Populating RTMetrics report | 20260508 09:39:41.777 | INFO | RTMetrics report populated | 20260508 09:39:41.777 | INFO | Opening tmp RTMetrics json file | 20260508 09:39:41.784 | INFO | Serializing RTMetrics to json | 20260508 09:39:41.785 | INFO | Serialization done, swapping tmp file now... | 20260508 09:39:41.786 | INFO | RTMetrics done | 20260508 09:39:41.850 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.502 seconds | 20260508 09:39:41.850 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.72% | 20260508 09:39:41.850 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247614% / 3.97364e-06% / 7.94729e-05% | 20260508 09:39:41.850 | INFO | Baz IO overall: | 20260508 09:39:41.850 | INFO | BytesWritten : 3094067842719 | 20260508 09:39:41.850 | INFO | NumEvents : 1488160926644 | 20260508 09:39:43.384 | INFO | Analyzing chunk frames = [7098368,7098880) | 20260508 09:39:43.384 | INFO | MemInfo: { | 20260508 09:39:43.384 | INFO | "gpu" : " 17 GB", | 20260508 09:39:43.384 | INFO | "huge" : "", | 20260508 09:39:43.384 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:43.384 | INFO | "malloc" : " 14 GB", | 20260508 09:39:43.384 | INFO | "pinned" : " 29 GB" | 20260508 09:39:43.384 | INFO | } | 20260508 09:39:43.385 | INFO | Perf report for the last 5.13485 seconds: | 20260508 09:39:43.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:43.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:43.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:43.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:43.385 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 09:39:43.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:43.385 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:39:43.385 | INFO | PRE_HQ : 65.5%, 1.6, 54.5ms, 0 | 20260508 09:39:43.385 | INFO | BAZWRITER : 67.6%, 2.5, 87.4ms, 0 | 20260508 09:39:43.385 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:39:43.385 | INFO | Uploaded 36GiB to GPU in 2.59407s (13.8778GiB/s) | 20260508 09:39:43.385 | INFO | Downloaded 38GiB to GPU in 2.71814s (13.9802GiB/s) | 20260508 09:39:48.521 | INFO | Analyzing chunk frames = [7098880,7099392) | 20260508 09:39:48.521 | INFO | MemInfo: { | 20260508 09:39:48.521 | INFO | "gpu" : " 17 GB", | 20260508 09:39:48.521 | INFO | "huge" : "", | 20260508 09:39:48.522 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:48.522 | INFO | "malloc" : " 14 GB", | 20260508 09:39:48.522 | INFO | "pinned" : " 29 GB" | 20260508 09:39:48.522 | INFO | } | 20260508 09:39:48.522 | INFO | Perf report for the last 5.13719 seconds: | 20260508 09:39:48.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:48.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:39:48.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:39:48.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:48.522 | INFO | Uploaded 36GiB to GPU in 2.49135s (14.45GiB/s) | 20260508 09:39:48.522 | INFO | Downloaded 34GiB to GPU in 2.47715s (13.7255GiB/s) | 20260508 09:39:53.663 | INFO | Analyzing chunk frames = [7099392,7099904) | 20260508 09:39:53.663 | INFO | MemInfo: { | 20260508 09:39:53.663 | INFO | "gpu" : " 17 GB", | 20260508 09:39:53.663 | INFO | "huge" : "", | 20260508 09:39:53.663 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:53.663 | INFO | "malloc" : " 14 GB", | 20260508 09:39:53.663 | INFO | "pinned" : " 29 GB" | 20260508 09:39:53.663 | INFO | } | 20260508 09:39:53.663 | INFO | Perf report for the last 5.14149 seconds: | 20260508 09:39:53.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:53.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.663 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:39:53.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.663 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 09:39:53.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:53.664 | INFO | Uploaded 36GiB to GPU in 2.43707s (14.7718GiB/s) | 20260508 09:39:53.664 | INFO | Downloaded 34GiB to GPU in 2.4404s (13.9321GiB/s) | 20260508 09:39:58.807 | INFO | Analyzing chunk frames = [7099904,7100416) | 20260508 09:39:58.807 | INFO | MemInfo: { | 20260508 09:39:58.807 | INFO | "gpu" : " 17 GB", | 20260508 09:39:58.807 | INFO | "huge" : "", | 20260508 09:39:58.807 | INFO | "huge_shared" : " 29 GB", | 20260508 09:39:58.807 | INFO | "malloc" : " 14 GB", | 20260508 09:39:58.807 | INFO | "pinned" : " 29 GB" | 20260508 09:39:58.807 | INFO | } | 20260508 09:39:58.807 | INFO | Perf report for the last 5.14384 seconds: | 20260508 09:39:58.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:39:58.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 09:39:58.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:39:58.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:39:58.807 | INFO | Uploaded 36GiB to GPU in 2.46259s (14.6188GiB/s) | 20260508 09:39:58.807 | INFO | Downloaded 34GiB to GPU in 2.43387s (13.9695GiB/s) | 20260508 09:40:03.948 | INFO | Analyzing chunk frames = [7100416,7100928) | 20260508 09:40:03.948 | INFO | MemInfo: { | 20260508 09:40:03.948 | INFO | "gpu" : " 17 GB", | 20260508 09:40:03.948 | INFO | "huge" : "", | 20260508 09:40:03.948 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:03.948 | INFO | "malloc" : " 14 GB", | 20260508 09:40:03.948 | INFO | "pinned" : " 29 GB" | 20260508 09:40:03.948 | INFO | } | 20260508 09:40:03.948 | INFO | Perf report for the last 5.14072 seconds: | 20260508 09:40:03.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:03.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 09:40:03.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:40:03.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:03.948 | INFO | Uploaded 36GiB to GPU in 2.36563s (15.2179GiB/s) | 20260508 09:40:03.948 | INFO | Downloaded 34GiB to GPU in 2.53003s (13.4386GiB/s) | 20260508 09:40:09.080 | INFO | Analyzing chunk frames = [7100928,7101440) | 20260508 09:40:09.080 | INFO | MemInfo: { | 20260508 09:40:09.080 | INFO | "gpu" : " 17 GB", | 20260508 09:40:09.080 | INFO | "huge" : "", | 20260508 09:40:09.080 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:09.080 | INFO | "malloc" : " 14 GB", | 20260508 09:40:09.080 | INFO | "pinned" : " 29 GB" | 20260508 09:40:09.080 | INFO | } | 20260508 09:40:09.080 | INFO | Perf report for the last 5.13244 seconds: | 20260508 09:40:09.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:09.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.080 | INFO | ANALYSIS : 65.2%, 3.0, 101.2ms, 0 | 20260508 09:40:09.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.080 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:40:09.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:09.081 | INFO | Uploaded 36GiB to GPU in 2.31298s (15.5643GiB/s) | 20260508 09:40:09.081 | INFO | Downloaded 34GiB to GPU in 2.4406s (13.931GiB/s) | 20260508 09:40:14.224 | INFO | Analyzing chunk frames = [7101440,7101952) | 20260508 09:40:14.225 | INFO | MemInfo: { | 20260508 09:40:14.225 | INFO | "gpu" : " 17 GB", | 20260508 09:40:14.225 | INFO | "huge" : "", | 20260508 09:40:14.225 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:14.225 | INFO | "malloc" : " 14 GB", | 20260508 09:40:14.225 | INFO | "pinned" : " 29 GB" | 20260508 09:40:14.225 | INFO | } | 20260508 09:40:14.225 | INFO | Perf report for the last 5.14446 seconds: | 20260508 09:40:14.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:14.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:40:14.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:40:14.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:14.225 | INFO | Uploaded 36GiB to GPU in 2.39342s (15.0412GiB/s) | 20260508 09:40:14.225 | INFO | Downloaded 34GiB to GPU in 2.45215s (13.8654GiB/s) | 20260508 09:40:19.361 | INFO | Analyzing chunk frames = [7101952,7102464) | 20260508 09:40:19.362 | INFO | MemInfo: { | 20260508 09:40:19.362 | INFO | "gpu" : " 17 GB", | 20260508 09:40:19.362 | INFO | "huge" : "", | 20260508 09:40:19.362 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:19.362 | INFO | "malloc" : " 14 GB", | 20260508 09:40:19.362 | INFO | "pinned" : " 29 GB" | 20260508 09:40:19.362 | INFO | } | 20260508 09:40:19.362 | INFO | Perf report for the last 5.137 seconds: | 20260508 09:40:19.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:19.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:40:19.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:40:19.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:19.362 | INFO | Uploaded 36GiB to GPU in 2.40141s (14.9912GiB/s) | 20260508 09:40:19.362 | INFO | Downloaded 34GiB to GPU in 2.45219s (13.8652GiB/s) | 20260508 09:40:19.362 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:40:19.362 | INFO | Total mem usage: 16.7 GiB | 20260508 09:40:19.362 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:40:19.362 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:40:19.362 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:40:19.362 | INFO | High Water Breakdown: | 20260508 09:40:19.362 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:40:19.362 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:40:19.362 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:40:19.362 | INFO | ------------------------------------------ | 20260508 09:40:19.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:40:19.362 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:40:19.362 | INFO | Total mem usage: 26.5 GiB | 20260508 09:40:19.362 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:40:19.362 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:40:19.362 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:40:19.362 | INFO | High Water Breakdown: | 20260508 09:40:19.362 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:40:19.362 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:40:19.362 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:40:19.362 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:40:19.362 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:40:19.362 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:40:19.362 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:40:19.362 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:40:19.362 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:40:19.362 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:40:19.362 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:40:19.362 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:40:19.362 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:40:19.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:40:19.363 | INFO | ------------------------------------------ | 20260508 09:40:19.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:40:19.363 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:40:19.363 | INFO | Total mem usage: 27.8 GiB | 20260508 09:40:19.363 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:40:19.363 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:40:19.363 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:40:19.363 | INFO | High Water Breakdown: | 20260508 09:40:19.363 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:40:19.363 | INFO | ------------------------------------------ | 20260508 09:40:19.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:40:19.363 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:40:19.363 | INFO | Total mem usage: 15.7 GiB | 20260508 09:40:19.363 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:40:19.363 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:40:19.363 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:40:19.363 | INFO | High Water Breakdown: | 20260508 09:40:19.363 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:40:19.363 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:40:19.363 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:40:19.363 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:40:19.363 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:40:19.363 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:40:19.363 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:40:19.363 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:40:19.363 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:40:19.363 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:40:19.363 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:40:19.363 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:40:19.363 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:40:19.363 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:40:19.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:40:19.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:40:19.363 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:40:19.363 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:40:19.363 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:40:19.363 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:40:19.363 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:40:19.363 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:40:19.363 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:40:19.363 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:40:19.363 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:40:19.363 | INFO | ------------------------------------------ | 20260508 09:40:19.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:40:19.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:40:19.364 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:40:19.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:40:19.364 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.62e-01 | 20260508 09:40:19.364 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.08e-02 / 6.08e-03 | 20260508 09:40:19.364 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.41e+00, 5.88e-01 / 2.51e-01 | 20260508 09:40:19.364 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.42e+00, 2.10e-01 / 1.50e-01 | 20260508 09:40:19.364 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.52e+01, 3.83e+00 / 1.12e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 1.32e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.52e+00, 4.06e-01 / 2.70e-01 | 20260508 09:40:19.364 | INFO | PerfReport: 3.02%, Metrics -- 9.89e-01 / 1.13e+00, 2.48e-01 / 1.22e-01 | 20260508 09:40:19.364 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.37e-01, 4.84e-02 / 2.58e-02 | 20260508 09:40:19.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:40:19.364 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:40:19.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:40:19.364 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 24.13%, Upload -- 2.54e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 24.67%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 31.09%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 2.09%, LockUpload -- 2.20e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 2.69%, LockDownload -- 2.83e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:40:19.364 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260508 09:40:19.459 | INFO | Starting a round of Baz IO | 20260508 09:40:22.874 | INFO | Populating RTMetrics report | 20260508 09:40:22.874 | INFO | RTMetrics report populated | 20260508 09:40:22.875 | INFO | Opening tmp RTMetrics json file | 20260508 09:40:22.883 | INFO | Serializing RTMetrics to json | 20260508 09:40:22.885 | INFO | Serialization done, swapping tmp file now... | 20260508 09:40:22.885 | INFO | RTMetrics done | 20260508 09:40:22.940 | INFO | Finished a round of Baz IO: Wrote 1.478 GiB in 3.481 seconds | 20260508 09:40:22.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.72% | 20260508 09:40:22.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247689% / 7.94729e-06% / 5.5631e-05% | 20260508 09:40:22.940 | INFO | Baz IO overall: | 20260508 09:40:22.940 | INFO | BytesWritten : 3095654482796 | 20260508 09:40:22.940 | INFO | NumEvents : 1488910873659 | 20260508 09:40:24.497 | INFO | Analyzing chunk frames = [7102464,7102976) | 20260508 09:40:24.497 | INFO | MemInfo: { | 20260508 09:40:24.497 | INFO | "gpu" : " 17 GB", | 20260508 09:40:24.497 | INFO | "huge" : "", | 20260508 09:40:24.497 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:24.497 | INFO | "malloc" : " 14 GB", | 20260508 09:40:24.497 | INFO | "pinned" : " 29 GB" | 20260508 09:40:24.497 | INFO | } | 20260508 09:40:24.497 | INFO | Perf report for the last 5.13549 seconds: | 20260508 09:40:24.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:24.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:24.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:24.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:24.497 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 09:40:24.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:24.497 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:40:24.498 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 09:40:24.498 | INFO | BAZWRITER : 67.1%, 2.5, 86.8ms, 0 | 20260508 09:40:24.498 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:40:24.498 | INFO | Uploaded 36GiB to GPU in 2.59463s (13.8748GiB/s) | 20260508 09:40:24.498 | INFO | Downloaded 38GiB to GPU in 2.70538s (14.0461GiB/s) | 20260508 09:40:29.638 | INFO | Analyzing chunk frames = [7102976,7103488) | 20260508 09:40:29.638 | INFO | MemInfo: { | 20260508 09:40:29.638 | INFO | "gpu" : " 17 GB", | 20260508 09:40:29.638 | INFO | "huge" : "", | 20260508 09:40:29.638 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:29.638 | INFO | "malloc" : " 14 GB", | 20260508 09:40:29.638 | INFO | "pinned" : " 29 GB" | 20260508 09:40:29.638 | INFO | } | 20260508 09:40:29.638 | INFO | Perf report for the last 5.1411 seconds: | 20260508 09:40:29.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:29.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.638 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:40:29.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.639 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:40:29.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:29.639 | INFO | Uploaded 36GiB to GPU in 2.52973s (14.2308GiB/s) | 20260508 09:40:29.639 | INFO | Downloaded 34GiB to GPU in 2.46038s (13.819GiB/s) | 20260508 09:40:34.774 | INFO | Analyzing chunk frames = [7103488,7104000) | 20260508 09:40:34.774 | INFO | MemInfo: { | 20260508 09:40:34.774 | INFO | "gpu" : " 17 GB", | 20260508 09:40:34.774 | INFO | "huge" : "", | 20260508 09:40:34.774 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:34.775 | INFO | "malloc" : " 14 GB", | 20260508 09:40:34.775 | INFO | "pinned" : " 29 GB" | 20260508 09:40:34.775 | INFO | } | 20260508 09:40:34.775 | INFO | Perf report for the last 5.13621 seconds: | 20260508 09:40:34.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:34.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.775 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:40:34.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.775 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:40:34.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:34.776 | INFO | Uploaded 36GiB to GPU in 2.46669s (14.5944GiB/s) | 20260508 09:40:34.776 | INFO | Downloaded 34GiB to GPU in 2.41563s (14.075GiB/s) | 20260508 09:40:39.919 | INFO | Analyzing chunk frames = [7104000,7104512) | 20260508 09:40:39.919 | INFO | MemInfo: { | 20260508 09:40:39.919 | INFO | "gpu" : " 17 GB", | 20260508 09:40:39.919 | INFO | "huge" : "", | 20260508 09:40:39.919 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:39.919 | INFO | "malloc" : " 14 GB", | 20260508 09:40:39.919 | INFO | "pinned" : " 29 GB" | 20260508 09:40:39.919 | INFO | } | 20260508 09:40:39.919 | INFO | Perf report for the last 5.14422 seconds: | 20260508 09:40:39.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:39.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:40:39.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:40:39.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:39.919 | INFO | Uploaded 36GiB to GPU in 2.50712s (14.3591GiB/s) | 20260508 09:40:39.919 | INFO | Downloaded 34GiB to GPU in 2.42816s (14.0023GiB/s) | 20260508 09:40:45.060 | INFO | Analyzing chunk frames = [7104512,7105024) | 20260508 09:40:45.060 | INFO | MemInfo: { | 20260508 09:40:45.060 | INFO | "gpu" : " 17 GB", | 20260508 09:40:45.060 | INFO | "huge" : "", | 20260508 09:40:45.060 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:45.060 | INFO | "malloc" : " 14 GB", | 20260508 09:40:45.060 | INFO | "pinned" : " 29 GB" | 20260508 09:40:45.060 | INFO | } | 20260508 09:40:45.061 | INFO | Perf report for the last 5.14169 seconds: | 20260508 09:40:45.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:45.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:40:45.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:40:45.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:45.061 | INFO | Uploaded 36GiB to GPU in 2.42648s (14.8363GiB/s) | 20260508 09:40:45.061 | INFO | Downloaded 34GiB to GPU in 2.52141s (13.4845GiB/s) | 20260508 09:40:50.199 | INFO | Analyzing chunk frames = [7105024,7105536) | 20260508 09:40:50.199 | INFO | MemInfo: { | 20260508 09:40:50.199 | INFO | "gpu" : " 17 GB", | 20260508 09:40:50.199 | INFO | "huge" : "", | 20260508 09:40:50.199 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:50.199 | INFO | "malloc" : " 14 GB", | 20260508 09:40:50.199 | INFO | "pinned" : " 29 GB" | 20260508 09:40:50.199 | INFO | } | 20260508 09:40:50.199 | INFO | Perf report for the last 5.13872 seconds: | 20260508 09:40:50.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:50.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.199 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:40:50.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.199 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:40:50.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:50.200 | INFO | Uploaded 36GiB to GPU in 2.38284s (15.108GiB/s) | 20260508 09:40:50.200 | INFO | Downloaded 34GiB to GPU in 2.43841s (13.9435GiB/s) | 20260508 09:40:55.331 | INFO | Analyzing chunk frames = [7105536,7106048) | 20260508 09:40:55.331 | INFO | MemInfo: { | 20260508 09:40:55.331 | INFO | "gpu" : " 17 GB", | 20260508 09:40:55.331 | INFO | "huge" : "", | 20260508 09:40:55.331 | INFO | "huge_shared" : " 29 GB", | 20260508 09:40:55.331 | INFO | "malloc" : " 14 GB", | 20260508 09:40:55.331 | INFO | "pinned" : " 29 GB" | 20260508 09:40:55.331 | INFO | } | 20260508 09:40:55.331 | INFO | Perf report for the last 5.13154 seconds: | 20260508 09:40:55.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:40:55.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 09:40:55.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:40:55.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:40:55.331 | INFO | Uploaded 36GiB to GPU in 2.41963s (14.8783GiB/s) | 20260508 09:40:55.331 | INFO | Downloaded 34GiB to GPU in 2.44256s (13.9198GiB/s) | 20260508 09:41:00.476 | INFO | Analyzing chunk frames = [7106048,7106560) | 20260508 09:41:00.476 | INFO | MemInfo: { | 20260508 09:41:00.476 | INFO | "gpu" : " 17 GB", | 20260508 09:41:00.476 | INFO | "huge" : "", | 20260508 09:41:00.476 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:00.476 | INFO | "malloc" : " 14 GB", | 20260508 09:41:00.476 | INFO | "pinned" : " 29 GB" | 20260508 09:41:00.476 | INFO | } | 20260508 09:41:00.476 | INFO | Perf report for the last 5.14555 seconds: | 20260508 09:41:00.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:00.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 09:41:00.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:41:00.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:00.477 | INFO | Uploaded 36GiB to GPU in 2.44534s (14.7219GiB/s) | 20260508 09:41:00.478 | INFO | Downloaded 34GiB to GPU in 2.43943s (13.9377GiB/s) | 20260508 09:41:00.592 | INFO | Starting a round of Baz IO | 20260508 09:41:03.996 | INFO | Populating RTMetrics report | 20260508 09:41:03.996 | INFO | RTMetrics report populated | 20260508 09:41:03.996 | INFO | Opening tmp RTMetrics json file | 20260508 09:41:04.004 | INFO | Serializing RTMetrics to json | 20260508 09:41:04.005 | INFO | Serialization done, swapping tmp file now... | 20260508 09:41:04.005 | INFO | RTMetrics done | 20260508 09:41:04.064 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.471 seconds | 20260508 09:41:04.064 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.72% | 20260508 09:41:04.064 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247236% / 0% / 6.35783e-05% | 20260508 09:41:04.064 | INFO | Baz IO overall: | 20260508 09:41:04.064 | INFO | BytesWritten : 3097239495908 | 20260508 09:41:04.064 | INFO | NumEvents : 1489659682531 | 20260508 09:41:05.614 | INFO | Analyzing chunk frames = [7106560,7107072) | 20260508 09:41:05.614 | INFO | MemInfo: { | 20260508 09:41:05.614 | INFO | "gpu" : " 17 GB", | 20260508 09:41:05.614 | INFO | "huge" : "", | 20260508 09:41:05.614 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:05.614 | INFO | "malloc" : " 14 GB", | 20260508 09:41:05.614 | INFO | "pinned" : " 29 GB" | 20260508 09:41:05.615 | INFO | } | 20260508 09:41:05.615 | INFO | Perf report for the last 5.13821 seconds: | 20260508 09:41:05.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:05.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:05.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:05.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:05.615 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:41:05.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:05.615 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260508 09:41:05.615 | INFO | PRE_HQ : 65.9%, 1.7, 57.8ms, 0 | 20260508 09:41:05.615 | INFO | BAZWRITER : 66.9%, 2.6, 90.0ms, 0 | 20260508 09:41:05.615 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:41:05.615 | INFO | Uploaded 36GiB to GPU in 2.58554s (13.9236GiB/s) | 20260508 09:41:05.615 | INFO | Downloaded 38GiB to GPU in 2.70551s (14.0454GiB/s) | 20260508 09:41:10.757 | INFO | Analyzing chunk frames = [7107072,7107584) | 20260508 09:41:10.757 | INFO | MemInfo: { | 20260508 09:41:10.757 | INFO | "gpu" : " 17 GB", | 20260508 09:41:10.757 | INFO | "huge" : "", | 20260508 09:41:10.757 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:10.757 | INFO | "malloc" : " 14 GB", | 20260508 09:41:10.757 | INFO | "pinned" : " 29 GB" | 20260508 09:41:10.757 | INFO | } | 20260508 09:41:10.757 | INFO | Perf report for the last 5.14283 seconds: | 20260508 09:41:10.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:10.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.758 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:41:10.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.758 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:41:10.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:10.758 | INFO | Uploaded 36GiB to GPU in 2.52471s (14.2591GiB/s) | 20260508 09:41:10.758 | INFO | Downloaded 34GiB to GPU in 2.45095s (13.8722GiB/s) | 20260508 09:41:15.896 | INFO | Analyzing chunk frames = [7107584,7108096) | 20260508 09:41:15.896 | INFO | MemInfo: { | 20260508 09:41:15.896 | INFO | "gpu" : " 17 GB", | 20260508 09:41:15.896 | INFO | "huge" : "", | 20260508 09:41:15.897 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:15.897 | INFO | "malloc" : " 14 GB", | 20260508 09:41:15.897 | INFO | "pinned" : " 29 GB" | 20260508 09:41:15.897 | INFO | } | 20260508 09:41:15.897 | INFO | Perf report for the last 5.13925 seconds: | 20260508 09:41:15.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:15.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:41:15.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:41:15.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:15.897 | INFO | Uploaded 36GiB to GPU in 2.4674s (14.5903GiB/s) | 20260508 09:41:15.897 | INFO | Downloaded 34GiB to GPU in 2.43689s (13.9522GiB/s) | 20260508 09:41:21.031 | INFO | Analyzing chunk frames = [7108096,7108608) | 20260508 09:41:21.031 | INFO | MemInfo: { | 20260508 09:41:21.031 | INFO | "gpu" : " 17 GB", | 20260508 09:41:21.031 | INFO | "huge" : "", | 20260508 09:41:21.031 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:21.031 | INFO | "malloc" : " 14 GB", | 20260508 09:41:21.032 | INFO | "pinned" : " 29 GB" | 20260508 09:41:21.032 | INFO | } | 20260508 09:41:21.032 | INFO | Perf report for the last 5.13499 seconds: | 20260508 09:41:21.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:21.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 09:41:21.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:41:21.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:21.032 | INFO | Uploaded 36GiB to GPU in 2.45018s (14.6928GiB/s) | 20260508 09:41:21.032 | INFO | Downloaded 34GiB to GPU in 2.40913s (14.113GiB/s) | 20260508 09:41:26.176 | INFO | Analyzing chunk frames = [7108608,7109120) | 20260508 09:41:26.176 | INFO | MemInfo: { | 20260508 09:41:26.176 | INFO | "gpu" : " 17 GB", | 20260508 09:41:26.176 | INFO | "huge" : "", | 20260508 09:41:26.176 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:26.176 | INFO | "malloc" : " 14 GB", | 20260508 09:41:26.176 | INFO | "pinned" : " 29 GB" | 20260508 09:41:26.176 | INFO | } | 20260508 09:41:26.176 | INFO | Perf report for the last 5.14418 seconds: | 20260508 09:41:26.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:26.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 09:41:26.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:41:26.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:26.176 | INFO | Uploaded 36GiB to GPU in 2.31864s (15.5264GiB/s) | 20260508 09:41:26.176 | INFO | Downloaded 34GiB to GPU in 2.48269s (13.6948GiB/s) | 20260508 09:41:31.313 | INFO | Analyzing chunk frames = [7109120,7109632) | 20260508 09:41:31.313 | INFO | MemInfo: { | 20260508 09:41:31.313 | INFO | "gpu" : " 17 GB", | 20260508 09:41:31.313 | INFO | "huge" : "", | 20260508 09:41:31.313 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:31.313 | INFO | "malloc" : " 14 GB", | 20260508 09:41:31.313 | INFO | "pinned" : " 29 GB" | 20260508 09:41:31.313 | INFO | } | 20260508 09:41:31.313 | INFO | Perf report for the last 5.13707 seconds: | 20260508 09:41:31.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:31.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.313 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:41:31.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.313 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:41:31.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:31.314 | INFO | Uploaded 36GiB to GPU in 2.38514s (15.0934GiB/s) | 20260508 09:41:31.314 | INFO | Downloaded 34GiB to GPU in 2.43061s (13.9883GiB/s) | 20260508 09:41:36.454 | INFO | Analyzing chunk frames = [7109632,7110144) | 20260508 09:41:36.454 | INFO | MemInfo: { | 20260508 09:41:36.454 | INFO | "gpu" : " 17 GB", | 20260508 09:41:36.454 | INFO | "huge" : "", | 20260508 09:41:36.454 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:36.454 | INFO | "malloc" : " 14 GB", | 20260508 09:41:36.454 | INFO | "pinned" : " 29 GB" | 20260508 09:41:36.454 | INFO | } | 20260508 09:41:36.454 | INFO | Perf report for the last 5.14138 seconds: | 20260508 09:41:36.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:36.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.454 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:41:36.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.455 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 09:41:36.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:36.455 | INFO | Uploaded 36GiB to GPU in 2.42997s (14.815GiB/s) | 20260508 09:41:36.455 | INFO | Downloaded 34GiB to GPU in 2.44067s (13.9306GiB/s) | 20260508 09:41:41.586 | INFO | Analyzing chunk frames = [7110144,7110656) | 20260508 09:41:41.586 | INFO | MemInfo: { | 20260508 09:41:41.586 | INFO | "gpu" : " 17 GB", | 20260508 09:41:41.586 | INFO | "huge" : "", | 20260508 09:41:41.586 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:41.586 | INFO | "malloc" : " 14 GB", | 20260508 09:41:41.586 | INFO | "pinned" : " 29 GB" | 20260508 09:41:41.586 | INFO | } | 20260508 09:41:41.586 | INFO | Perf report for the last 5.13164 seconds: | 20260508 09:41:41.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:41.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.586 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 09:41:41.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.586 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 09:41:41.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:41.587 | INFO | Uploaded 36GiB to GPU in 2.40752s (14.9531GiB/s) | 20260508 09:41:41.587 | INFO | Downloaded 34GiB to GPU in 2.42587s (14.0156GiB/s) | 20260508 09:41:41.587 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:41:41.587 | INFO | Total mem usage: 16.7 GiB | 20260508 09:41:41.587 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:41:41.587 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -15.8%) | 20260508 09:41:41.587 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:41:41.587 | INFO | High Water Breakdown: | 20260508 09:41:41.587 | INFO | 59.7% : 8.61 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:41:41.587 | INFO | 40.1% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:41:41.587 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:41:41.587 | INFO | ------------------------------------------ | 20260508 09:41:41.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:41:41.587 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:41:41.587 | INFO | Total mem usage: 26.5 GiB | 20260508 09:41:41.587 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:41:41.587 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:41:41.587 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:41:41.587 | INFO | High Water Breakdown: | 20260508 09:41:41.587 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:41:41.587 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:41:41.587 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:41:41.587 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:41:41.587 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:41:41.587 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:41:41.587 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:41:41.587 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:41:41.587 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:41:41.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:41:41.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:41:41.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:41:41.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:41:41.587 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:41:41.587 | INFO | ------------------------------------------ | 20260508 09:41:41.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:41:41.587 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:41:41.587 | INFO | Total mem usage: 27.8 GiB | 20260508 09:41:41.587 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:41:41.587 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:41:41.587 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:41:41.588 | INFO | High Water Breakdown: | 20260508 09:41:41.588 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:41:41.588 | INFO | ------------------------------------------ | 20260508 09:41:41.588 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:41:41.588 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:41:41.588 | INFO | Total mem usage: 15.7 GiB | 20260508 09:41:41.588 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:41:41.588 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:41:41.588 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:41:41.588 | INFO | High Water Breakdown: | 20260508 09:41:41.588 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:41:41.588 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:41:41.588 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:41:41.588 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:41:41.588 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:41:41.588 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:41:41.588 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:41:41.588 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:41:41.588 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:41:41.588 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:41:41.588 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:41:41.588 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:41:41.588 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:41:41.588 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:41:41.588 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:41:41.588 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:41:41.588 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:41:41.588 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:41:41.588 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:41:41.588 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:41:41.588 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:41:41.588 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:41:41.588 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:41:41.588 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:41:41.588 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:41:41.588 | INFO | ------------------------------------------ | 20260508 09:41:41.588 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:41:41.588 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:41:41.588 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:41:41.588 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:41:41.588 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.41e+01 / 7.05e-01 | 20260508 09:41:41.588 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.09e-01, 1.19e-02 / 1.88e-02 | 20260508 09:41:41.588 | INFO | PerfReport: 12.16%, Baseline -- 3.99e+00 / 4.40e+00, 5.96e-01 / 2.47e-01 | 20260508 09:41:41.588 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.47e+00, 2.13e-01 / 1.27e-01 | 20260508 09:41:41.588 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.52e+01, 3.85e+00 / 1.18e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.86e+00 / 1.30e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.49e+00, 4.06e-01 / 2.55e-01 | 20260508 09:41:41.588 | INFO | PerfReport: 3.03%, Metrics -- 9.94e-01 / 1.13e+00, 2.49e-01 / 1.03e-01 | 20260508 09:41:41.588 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.03e-01 / 3.38e-01, 4.92e-02 / 3.02e-02 | 20260508 09:41:41.588 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:41:41.588 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:41:41.588 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:41:41.588 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.46e+01 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 24.31%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 24.49%, Download -- 2.58e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 31.11%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 2.17%, LockUpload -- 2.28e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 2.51%, LockDownload -- 2.64e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 15.41%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:41:41.588 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260508 09:41:41.699 | INFO | Starting a round of Baz IO | 20260508 09:41:45.112 | INFO | Populating RTMetrics report | 20260508 09:41:45.112 | INFO | RTMetrics report populated | 20260508 09:41:45.112 | INFO | Opening tmp RTMetrics json file | 20260508 09:41:45.119 | INFO | Serializing RTMetrics to json | 20260508 09:41:45.121 | INFO | Serialization done, swapping tmp file now... | 20260508 09:41:45.122 | INFO | RTMetrics done | 20260508 09:41:45.174 | INFO | Finished a round of Baz IO: Wrote 1.477 GiB in 3.475 seconds | 20260508 09:41:45.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.72% | 20260508 09:41:45.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247467% / 0% / 5.96046e-05% | 20260508 09:41:45.175 | INFO | Baz IO overall: | 20260508 09:41:45.175 | INFO | BytesWritten : 3098825328947 | 20260508 09:41:45.175 | INFO | NumEvents : 1490409323996 | 20260508 09:41:46.732 | INFO | Analyzing chunk frames = [7110656,7111168) | 20260508 09:41:46.732 | INFO | MemInfo: { | 20260508 09:41:46.732 | INFO | "gpu" : " 17 GB", | 20260508 09:41:46.732 | INFO | "huge" : "", | 20260508 09:41:46.732 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:46.732 | INFO | "malloc" : " 14 GB", | 20260508 09:41:46.732 | INFO | "pinned" : " 29 GB" | 20260508 09:41:46.732 | INFO | } | 20260508 09:41:46.732 | INFO | Perf report for the last 5.1456 seconds: | 20260508 09:41:46.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:46.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:46.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:46.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:46.732 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:41:46.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:46.732 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260508 09:41:46.732 | INFO | PRE_HQ : 66.4%, 1.8, 63.0ms, 0 | 20260508 09:41:46.732 | INFO | BAZWRITER : 66.8%, 2.7, 94.3ms, 0 | 20260508 09:41:46.732 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:41:46.732 | INFO | Uploaded 36GiB to GPU in 2.58964s (13.9016GiB/s) | 20260508 09:41:46.733 | INFO | Downloaded 38GiB to GPU in 2.71374s (14.0028GiB/s) | 20260508 09:41:51.872 | INFO | Analyzing chunk frames = [7111168,7111680) | 20260508 09:41:51.872 | INFO | MemInfo: { | 20260508 09:41:51.872 | INFO | "gpu" : " 17 GB", | 20260508 09:41:51.872 | INFO | "huge" : "", | 20260508 09:41:51.872 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:51.872 | INFO | "malloc" : " 14 GB", | 20260508 09:41:51.872 | INFO | "pinned" : " 29 GB" | 20260508 09:41:51.872 | INFO | } | 20260508 09:41:51.872 | INFO | Perf report for the last 5.14058 seconds: | 20260508 09:41:51.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:51.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.872 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:41:51.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.872 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:41:51.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:51.873 | INFO | Uploaded 36GiB to GPU in 2.5429s (14.1571GiB/s) | 20260508 09:41:51.873 | INFO | Downloaded 34GiB to GPU in 2.4701s (13.7646GiB/s) | 20260508 09:41:57.011 | INFO | Analyzing chunk frames = [7111680,7112192) | 20260508 09:41:57.011 | INFO | MemInfo: { | 20260508 09:41:57.011 | INFO | "gpu" : " 17 GB", | 20260508 09:41:57.011 | INFO | "huge" : "", | 20260508 09:41:57.011 | INFO | "huge_shared" : " 29 GB", | 20260508 09:41:57.011 | INFO | "malloc" : " 14 GB", | 20260508 09:41:57.011 | INFO | "pinned" : " 29 GB" | 20260508 09:41:57.011 | INFO | } | 20260508 09:41:57.011 | INFO | Perf report for the last 5.13872 seconds: | 20260508 09:41:57.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:41:57.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:41:57.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:41:57.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:41:57.011 | INFO | Uploaded 36GiB to GPU in 2.46924s (14.5794GiB/s) | 20260508 09:41:57.012 | INFO | Downloaded 34GiB to GPU in 2.42157s (14.0405GiB/s) | 20260508 09:42:02.145 | INFO | Analyzing chunk frames = [7112192,7112704) | 20260508 09:42:02.145 | INFO | MemInfo: { | 20260508 09:42:02.145 | INFO | "gpu" : " 17 GB", | 20260508 09:42:02.145 | INFO | "huge" : "", | 20260508 09:42:02.145 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:02.145 | INFO | "malloc" : " 14 GB", | 20260508 09:42:02.145 | INFO | "pinned" : " 29 GB" | 20260508 09:42:02.145 | INFO | } | 20260508 09:42:02.145 | INFO | Perf report for the last 5.13442 seconds: | 20260508 09:42:02.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:02.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.146 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 09:42:02.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.146 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:42:02.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:02.146 | INFO | Uploaded 36GiB to GPU in 2.46826s (14.5852GiB/s) | 20260508 09:42:02.146 | INFO | Downloaded 34GiB to GPU in 2.44595s (13.9006GiB/s) | 20260508 09:42:07.282 | INFO | Analyzing chunk frames = [7112704,7113216) | 20260508 09:42:07.282 | INFO | MemInfo: { | 20260508 09:42:07.282 | INFO | "gpu" : " 17 GB", | 20260508 09:42:07.282 | INFO | "huge" : "", | 20260508 09:42:07.282 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:07.282 | INFO | "malloc" : " 14 GB", | 20260508 09:42:07.283 | INFO | "pinned" : " 29 GB" | 20260508 09:42:07.283 | INFO | } | 20260508 09:42:07.283 | INFO | Perf report for the last 5.13722 seconds: | 20260508 09:42:07.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:07.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 09:42:07.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:42:07.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:07.283 | INFO | Uploaded 36GiB to GPU in 2.30047s (15.649GiB/s) | 20260508 09:42:07.283 | INFO | Downloaded 34GiB to GPU in 2.49288s (13.6388GiB/s) | 20260508 09:42:12.426 | INFO | Analyzing chunk frames = [7113216,7113728) | 20260508 09:42:12.426 | INFO | MemInfo: { | 20260508 09:42:12.426 | INFO | "gpu" : " 17 GB", | 20260508 09:42:12.426 | INFO | "huge" : "", | 20260508 09:42:12.426 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:12.426 | INFO | "malloc" : " 14 GB", | 20260508 09:42:12.426 | INFO | "pinned" : " 29 GB" | 20260508 09:42:12.426 | INFO | } | 20260508 09:42:12.426 | INFO | Perf report for the last 5.14369 seconds: | 20260508 09:42:12.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:12.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.426 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:42:12.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.426 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:42:12.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:12.427 | INFO | Uploaded 36GiB to GPU in 2.39751s (15.0156GiB/s) | 20260508 09:42:12.427 | INFO | Downloaded 34GiB to GPU in 2.41694s (14.0674GiB/s) | 20260508 09:42:17.572 | INFO | Analyzing chunk frames = [7113728,7114240) | 20260508 09:42:17.572 | INFO | MemInfo: { | 20260508 09:42:17.572 | INFO | "gpu" : " 17 GB", | 20260508 09:42:17.572 | INFO | "huge" : "", | 20260508 09:42:17.572 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:17.573 | INFO | "malloc" : " 14 GB", | 20260508 09:42:17.573 | INFO | "pinned" : " 29 GB" | 20260508 09:42:17.573 | INFO | } | 20260508 09:42:17.573 | INFO | Perf report for the last 5.14631 seconds: | 20260508 09:42:17.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:17.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 09:42:17.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:42:17.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:17.573 | INFO | Uploaded 36GiB to GPU in 2.44865s (14.702GiB/s) | 20260508 09:42:17.573 | INFO | Downloaded 34GiB to GPU in 2.43225s (13.9788GiB/s) | 20260508 09:42:22.711 | INFO | Analyzing chunk frames = [7114240,7114752) | 20260508 09:42:22.711 | INFO | MemInfo: { | 20260508 09:42:22.711 | INFO | "gpu" : " 17 GB", | 20260508 09:42:22.711 | INFO | "huge" : "", | 20260508 09:42:22.711 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:22.711 | INFO | "malloc" : " 14 GB", | 20260508 09:42:22.711 | INFO | "pinned" : " 29 GB" | 20260508 09:42:22.711 | INFO | } | 20260508 09:42:22.711 | INFO | Perf report for the last 5.13832 seconds: | 20260508 09:42:22.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:22.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:42:22.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:42:22.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:22.711 | INFO | Uploaded 36GiB to GPU in 2.43499s (14.7845GiB/s) | 20260508 09:42:22.711 | INFO | Downloaded 34GiB to GPU in 2.43015s (13.9909GiB/s) | 20260508 09:42:22.812 | INFO | Starting a round of Baz IO | 20260508 09:42:26.221 | INFO | Populating RTMetrics report | 20260508 09:42:26.222 | INFO | RTMetrics report populated | 20260508 09:42:26.222 | INFO | Opening tmp RTMetrics json file | 20260508 09:42:26.232 | INFO | Serializing RTMetrics to json | 20260508 09:42:26.233 | INFO | Serialization done, swapping tmp file now... | 20260508 09:42:26.233 | INFO | RTMetrics done | 20260508 09:42:26.292 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.48 seconds | 20260508 09:42:26.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.00% / 59.72% | 20260508 09:42:26.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.247165% / 0% / 6.35783e-05% | 20260508 09:42:26.292 | INFO | Baz IO overall: | 20260508 09:42:26.292 | INFO | BytesWritten : 3100410647363 | 20260508 09:42:26.292 | INFO | NumEvents : 1491158631936 | 20260508 09:42:27.841 | INFO | Analyzing chunk frames = [7114752,7115264) | 20260508 09:42:27.841 | INFO | MemInfo: { | 20260508 09:42:27.841 | INFO | "gpu" : " 17 GB", | 20260508 09:42:27.841 | INFO | "huge" : "", | 20260508 09:42:27.841 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:27.841 | INFO | "malloc" : " 14 GB", | 20260508 09:42:27.841 | INFO | "pinned" : " 29 GB" | 20260508 09:42:27.841 | INFO | } | 20260508 09:42:27.841 | INFO | Perf report for the last 5.13001 seconds: | 20260508 09:42:27.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:27.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:27.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:27.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:27.841 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 09:42:27.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:27.841 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:42:27.841 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 09:42:27.841 | INFO | BAZWRITER : 67.2%, 2.5, 86.4ms, 0 | 20260508 09:42:27.841 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:42:27.841 | INFO | Uploaded 36GiB to GPU in 2.60152s (13.8381GiB/s) | 20260508 09:42:27.841 | INFO | Downloaded 38GiB to GPU in 2.70775s (14.0338GiB/s) | 20260508 09:42:32.983 | INFO | Analyzing chunk frames = [7115264,7115776) | 20260508 09:42:32.983 | INFO | MemInfo: { | 20260508 09:42:32.983 | INFO | "gpu" : " 17 GB", | 20260508 09:42:32.983 | INFO | "huge" : "", | 20260508 09:42:32.983 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:32.983 | INFO | "malloc" : " 14 GB", | 20260508 09:42:32.983 | INFO | "pinned" : " 29 GB" | 20260508 09:42:32.983 | INFO | } | 20260508 09:42:32.983 | INFO | Perf report for the last 5.1418 seconds: | 20260508 09:42:32.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:32.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 09:42:32.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:42:32.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:32.983 | INFO | Uploaded 36GiB to GPU in 2.53664s (14.192GiB/s) | 20260508 09:42:32.983 | INFO | Downloaded 34GiB to GPU in 2.44117s (13.9277GiB/s) | 20260508 09:42:38.122 | INFO | Analyzing chunk frames = [7115776,7116288) | 20260508 09:42:38.122 | INFO | MemInfo: { | 20260508 09:42:38.122 | INFO | "gpu" : " 17 GB", | 20260508 09:42:38.122 | INFO | "huge" : "", | 20260508 09:42:38.122 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:38.122 | INFO | "malloc" : " 14 GB", | 20260508 09:42:38.122 | INFO | "pinned" : " 29 GB" | 20260508 09:42:38.122 | INFO | } | 20260508 09:42:38.122 | INFO | Perf report for the last 5.13896 seconds: | 20260508 09:42:38.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:38.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:42:38.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:42:38.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:38.122 | INFO | Uploaded 36GiB to GPU in 2.51975s (14.2871GiB/s) | 20260508 09:42:38.122 | INFO | Downloaded 34GiB to GPU in 2.43802s (13.9457GiB/s) | 20260508 09:42:43.262 | INFO | Analyzing chunk frames = [7116288,7116800) | 20260508 09:42:43.262 | INFO | MemInfo: { | 20260508 09:42:43.262 | INFO | "gpu" : " 17 GB", | 20260508 09:42:43.262 | INFO | "huge" : "", | 20260508 09:42:43.263 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:43.263 | INFO | "malloc" : " 14 GB", | 20260508 09:42:43.263 | INFO | "pinned" : " 29 GB" | 20260508 09:42:43.263 | INFO | } | 20260508 09:42:43.263 | INFO | Perf report for the last 5.1408 seconds: | 20260508 09:42:43.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:43.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 09:42:43.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:42:43.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:43.263 | INFO | Uploaded 36GiB to GPU in 2.52796s (14.2407GiB/s) | 20260508 09:42:43.263 | INFO | Downloaded 34GiB to GPU in 2.42578s (14.0161GiB/s) | 20260508 09:42:48.404 | INFO | Analyzing chunk frames = [7116800,7117312) | 20260508 09:42:48.404 | INFO | MemInfo: { | 20260508 09:42:48.404 | INFO | "gpu" : " 17 GB", | 20260508 09:42:48.404 | INFO | "huge" : "", | 20260508 09:42:48.404 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:48.404 | INFO | "malloc" : " 14 GB", | 20260508 09:42:48.404 | INFO | "pinned" : " 29 GB" | 20260508 09:42:48.404 | INFO | } | 20260508 09:42:48.404 | INFO | Perf report for the last 5.14177 seconds: | 20260508 09:42:48.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:48.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.405 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 09:42:48.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.405 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:42:48.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:48.405 | INFO | Uploaded 36GiB to GPU in 2.38469s (15.0963GiB/s) | 20260508 09:42:48.405 | INFO | Downloaded 34GiB to GPU in 2.50892s (13.5517GiB/s) | 20260508 09:42:53.545 | INFO | Analyzing chunk frames = [7117312,7117824) | 20260508 09:42:53.545 | INFO | MemInfo: { | 20260508 09:42:53.545 | INFO | "gpu" : " 17 GB", | 20260508 09:42:53.545 | INFO | "huge" : "", | 20260508 09:42:53.545 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:53.546 | INFO | "malloc" : " 14 GB", | 20260508 09:42:53.546 | INFO | "pinned" : " 29 GB" | 20260508 09:42:53.546 | INFO | } | 20260508 09:42:53.546 | INFO | Perf report for the last 5.14118 seconds: | 20260508 09:42:53.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:53.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 09:42:53.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260508 09:42:53.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:53.546 | INFO | Uploaded 36GiB to GPU in 2.42169s (14.8657GiB/s) | 20260508 09:42:53.546 | INFO | Downloaded 34GiB to GPU in 2.43255s (13.9771GiB/s) | 20260508 09:42:58.681 | INFO | Analyzing chunk frames = [7117824,7118336) | 20260508 09:42:58.681 | INFO | MemInfo: { | 20260508 09:42:58.681 | INFO | "gpu" : " 17 GB", | 20260508 09:42:58.681 | INFO | "huge" : "", | 20260508 09:42:58.681 | INFO | "huge_shared" : " 29 GB", | 20260508 09:42:58.681 | INFO | "malloc" : " 14 GB", | 20260508 09:42:58.681 | INFO | "pinned" : " 29 GB" | 20260508 09:42:58.681 | INFO | } | 20260508 09:42:58.681 | INFO | Perf report for the last 5.13562 seconds: | 20260508 09:42:58.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:42:58.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.681 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:42:58.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.681 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:42:58.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:42:58.682 | INFO | Uploaded 36GiB to GPU in 2.46563s (14.6008GiB/s) | 20260508 09:42:58.682 | INFO | Downloaded 34GiB to GPU in 2.43365s (13.9708GiB/s) | 20260508 09:43:03.819 | INFO | Analyzing chunk frames = [7118336,7118848) | 20260508 09:43:03.819 | INFO | MemInfo: { | 20260508 09:43:03.819 | INFO | "gpu" : " 17 GB", | 20260508 09:43:03.819 | INFO | "huge" : "", | 20260508 09:43:03.819 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:03.819 | INFO | "malloc" : " 14 GB", | 20260508 09:43:03.819 | INFO | "pinned" : " 29 GB" | 20260508 09:43:03.820 | INFO | } | 20260508 09:43:03.820 | INFO | Perf report for the last 5.13849 seconds: | 20260508 09:43:03.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:03.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 09:43:03.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 09:43:03.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:03.820 | INFO | Uploaded 36GiB to GPU in 2.43786s (14.7671GiB/s) | 20260508 09:43:03.820 | INFO | Downloaded 34GiB to GPU in 2.43596s (13.9575GiB/s) | 20260508 09:43:03.820 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:43:03.820 | INFO | Total mem usage: 16.7 GiB | 20260508 09:43:03.820 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:43:03.820 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:43:03.820 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:43:03.820 | INFO | High Water Breakdown: | 20260508 09:43:03.820 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:43:03.820 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:43:03.820 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:43:03.820 | INFO | ------------------------------------------ | 20260508 09:43:03.820 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:43:03.820 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:43:03.820 | INFO | Total mem usage: 26.5 GiB | 20260508 09:43:03.820 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:43:03.820 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:43:03.820 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:43:03.820 | INFO | High Water Breakdown: | 20260508 09:43:03.820 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:43:03.820 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:43:03.821 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:43:03.821 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:43:03.821 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:43:03.821 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:43:03.821 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:43:03.821 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:43:03.821 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:43:03.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:43:03.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:43:03.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:43:03.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:43:03.821 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:43:03.821 | INFO | ------------------------------------------ | 20260508 09:43:03.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:43:03.821 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:43:03.821 | INFO | Total mem usage: 27.8 GiB | 20260508 09:43:03.821 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:43:03.821 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:43:03.821 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:43:03.821 | INFO | High Water Breakdown: | 20260508 09:43:03.821 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:43:03.821 | INFO | ------------------------------------------ | 20260508 09:43:03.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:43:03.821 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:43:03.821 | INFO | Total mem usage: 15.7 GiB | 20260508 09:43:03.821 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:43:03.821 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:43:03.821 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:43:03.821 | INFO | High Water Breakdown: | 20260508 09:43:03.821 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:43:03.821 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:43:03.821 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:43:03.821 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:43:03.821 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:43:03.821 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:43:03.821 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:43:03.821 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:43:03.821 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:43:03.821 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:43:03.821 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:43:03.821 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:43:03.821 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:43:03.821 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:43:03.821 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:43:03.821 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:43:03.821 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:43:03.821 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:43:03.821 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:43:03.821 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:43:03.821 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:43:03.821 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:43:03.821 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:43:03.821 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:43:03.821 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:43:03.821 | INFO | ------------------------------------------ | 20260508 09:43:03.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:43:03.821 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:43:03.821 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:43:03.821 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:43:03.821 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 7.30e-01 | 20260508 09:43:03.821 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.06e-01, 1.14e-02 / 4.74e-03 | 20260508 09:43:03.821 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.38e+00, 5.92e-01 / 2.33e-01 | 20260508 09:43:03.821 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.19e-01 | 20260508 09:43:03.821 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.53e+01, 3.75e+00 / 1.10e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.83e+00 / 1.12e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.47e+00, 4.05e-01 / 2.56e-01 | 20260508 09:43:03.821 | INFO | PerfReport: 3.04%, Metrics -- 9.94e-01 / 1.12e+00, 2.47e-01 / 1.00e-01 | 20260508 09:43:03.821 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.29e-01, 4.82e-02 / 2.35e-02 | 20260508 09:43:03.821 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:43:03.821 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:43:03.821 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:43:03.821 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 24.42%, Upload -- 2.57e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 24.49%, Download -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 31.03%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 2.30%, LockUpload -- 2.42e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 2.42%, LockDownload -- 2.55e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 15.33%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:43:03.821 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260508 09:43:03.917 | INFO | Starting a round of Baz IO | 20260508 09:43:07.327 | INFO | Populating RTMetrics report | 20260508 09:43:07.327 | INFO | RTMetrics report populated | 20260508 09:43:07.327 | INFO | Opening tmp RTMetrics json file | 20260508 09:43:07.339 | INFO | Serializing RTMetrics to json | 20260508 09:43:07.340 | INFO | Serialization done, swapping tmp file now... | 20260508 09:43:07.340 | INFO | RTMetrics done | 20260508 09:43:07.398 | INFO | Finished a round of Baz IO: Wrote 1.478 GiB in 3.481 seconds | 20260508 09:43:07.398 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.72% | 20260508 09:43:07.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.246362% / 0% / 7.54992e-05% | 20260508 09:43:07.398 | INFO | Baz IO overall: | 20260508 09:43:07.398 | INFO | BytesWritten : 3101997770754 | 20260508 09:43:07.398 | INFO | NumEvents : 1491909263669 | 20260508 09:43:08.957 | INFO | Analyzing chunk frames = [7118848,7119360) | 20260508 09:43:08.957 | INFO | MemInfo: { | 20260508 09:43:08.957 | INFO | "gpu" : " 17 GB", | 20260508 09:43:08.957 | INFO | "huge" : "", | 20260508 09:43:08.957 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:08.957 | INFO | "malloc" : " 14 GB", | 20260508 09:43:08.957 | INFO | "pinned" : " 29 GB" | 20260508 09:43:08.957 | INFO | } | 20260508 09:43:08.957 | INFO | Perf report for the last 5.13688 seconds: | 20260508 09:43:08.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:08.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:08.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:08.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:08.957 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:43:08.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:08.957 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:43:08.957 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 09:43:08.957 | INFO | BAZWRITER : 67.2%, 2.5, 87.3ms, 0 | 20260508 09:43:08.957 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:43:08.957 | INFO | Uploaded 36GiB to GPU in 2.60317s (13.8293GiB/s) | 20260508 09:43:08.957 | INFO | Downloaded 38GiB to GPU in 2.71631s (13.9896GiB/s) | 20260508 09:43:14.093 | INFO | Analyzing chunk frames = [7119360,7119872) | 20260508 09:43:14.093 | INFO | MemInfo: { | 20260508 09:43:14.093 | INFO | "gpu" : " 17 GB", | 20260508 09:43:14.093 | INFO | "huge" : "", | 20260508 09:43:14.093 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:14.093 | INFO | "malloc" : " 14 GB", | 20260508 09:43:14.093 | INFO | "pinned" : " 29 GB" | 20260508 09:43:14.093 | INFO | } | 20260508 09:43:14.094 | INFO | Perf report for the last 5.13686 seconds: | 20260508 09:43:14.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:14.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:43:14.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:43:14.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:14.094 | INFO | Uploaded 36GiB to GPU in 2.54783s (14.1297GiB/s) | 20260508 09:43:14.094 | INFO | Downloaded 34GiB to GPU in 2.44783s (13.8898GiB/s) | 20260508 09:43:19.238 | INFO | Analyzing chunk frames = [7119872,7120384) | 20260508 09:43:19.238 | INFO | MemInfo: { | 20260508 09:43:19.238 | INFO | "gpu" : " 17 GB", | 20260508 09:43:19.238 | INFO | "huge" : "", | 20260508 09:43:19.238 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:19.238 | INFO | "malloc" : " 14 GB", | 20260508 09:43:19.238 | INFO | "pinned" : " 29 GB" | 20260508 09:43:19.238 | INFO | } | 20260508 09:43:19.238 | INFO | Perf report for the last 5.14493 seconds: | 20260508 09:43:19.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:19.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.239 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 09:43:19.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.239 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:43:19.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:19.239 | INFO | Uploaded 36GiB to GPU in 2.51878s (14.2927GiB/s) | 20260508 09:43:19.239 | INFO | Downloaded 34GiB to GPU in 2.44239s (13.9208GiB/s) | 20260508 09:43:24.381 | INFO | Analyzing chunk frames = [7120384,7120896) | 20260508 09:43:24.381 | INFO | MemInfo: { | 20260508 09:43:24.381 | INFO | "gpu" : " 17 GB", | 20260508 09:43:24.381 | INFO | "huge" : "", | 20260508 09:43:24.381 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:24.381 | INFO | "malloc" : " 14 GB", | 20260508 09:43:24.381 | INFO | "pinned" : " 29 GB" | 20260508 09:43:24.381 | INFO | } | 20260508 09:43:24.381 | INFO | Perf report for the last 5.14237 seconds: | 20260508 09:43:24.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:24.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:43:24.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:43:24.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:24.381 | INFO | Uploaded 36GiB to GPU in 2.52147s (14.2774GiB/s) | 20260508 09:43:24.381 | INFO | Downloaded 34GiB to GPU in 2.42836s (14.0012GiB/s) | 20260508 09:43:29.524 | INFO | Analyzing chunk frames = [7120896,7121408) | 20260508 09:43:29.524 | INFO | MemInfo: { | 20260508 09:43:29.524 | INFO | "gpu" : " 17 GB", | 20260508 09:43:29.524 | INFO | "huge" : "", | 20260508 09:43:29.524 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:29.524 | INFO | "malloc" : " 14 GB", | 20260508 09:43:29.524 | INFO | "pinned" : " 29 GB" | 20260508 09:43:29.524 | INFO | } | 20260508 09:43:29.524 | INFO | Perf report for the last 5.143 seconds: | 20260508 09:43:29.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:29.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 09:43:29.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 09:43:29.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:29.524 | INFO | Uploaded 36GiB to GPU in 2.33361s (15.4268GiB/s) | 20260508 09:43:29.524 | INFO | Downloaded 34GiB to GPU in 2.49194s (13.644GiB/s) | 20260508 09:43:34.656 | INFO | Analyzing chunk frames = [7121408,7121920) | 20260508 09:43:34.656 | INFO | MemInfo: { | 20260508 09:43:34.656 | INFO | "gpu" : " 17 GB", | 20260508 09:43:34.656 | INFO | "huge" : "", | 20260508 09:43:34.656 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:34.656 | INFO | "malloc" : " 14 GB", | 20260508 09:43:34.656 | INFO | "pinned" : " 29 GB" | 20260508 09:43:34.656 | INFO | } | 20260508 09:43:34.656 | INFO | Perf report for the last 5.13251 seconds: | 20260508 09:43:34.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:34.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.656 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 09:43:34.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.657 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:43:34.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:34.657 | INFO | Uploaded 36GiB to GPU in 2.39929s (15.0045GiB/s) | 20260508 09:43:34.657 | INFO | Downloaded 34GiB to GPU in 2.42354s (14.029GiB/s) | 20260508 09:43:39.795 | INFO | Analyzing chunk frames = [7121920,7122432) | 20260508 09:43:39.795 | INFO | MemInfo: { | 20260508 09:43:39.795 | INFO | "gpu" : " 17 GB", | 20260508 09:43:39.795 | INFO | "huge" : "", | 20260508 09:43:39.795 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:39.795 | INFO | "malloc" : " 14 GB", | 20260508 09:43:39.795 | INFO | "pinned" : " 29 GB" | 20260508 09:43:39.795 | INFO | } | 20260508 09:43:39.795 | INFO | Perf report for the last 5.13875 seconds: | 20260508 09:43:39.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:39.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:43:39.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:43:39.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:39.795 | INFO | Uploaded 36GiB to GPU in 2.4461s (14.7173GiB/s) | 20260508 09:43:39.796 | INFO | Downloaded 34GiB to GPU in 2.43646s (13.9546GiB/s) | 20260508 09:43:44.935 | INFO | Analyzing chunk frames = [7122432,7122944) | 20260508 09:43:44.935 | INFO | MemInfo: { | 20260508 09:43:44.935 | INFO | "gpu" : " 17 GB", | 20260508 09:43:44.935 | INFO | "huge" : "", | 20260508 09:43:44.935 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:44.935 | INFO | "malloc" : " 14 GB", | 20260508 09:43:44.935 | INFO | "pinned" : " 29 GB" | 20260508 09:43:44.935 | INFO | } | 20260508 09:43:44.935 | INFO | Perf report for the last 5.13979 seconds: | 20260508 09:43:44.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:44.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 09:43:44.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:43:44.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:44.935 | INFO | Uploaded 36GiB to GPU in 2.42131s (14.868GiB/s) | 20260508 09:43:44.935 | INFO | Downloaded 34GiB to GPU in 2.41933s (14.0535GiB/s) | 20260508 09:43:45.032 | INFO | Starting a round of Baz IO | 20260508 09:43:48.444 | INFO | Populating RTMetrics report | 20260508 09:43:48.444 | INFO | RTMetrics report populated | 20260508 09:43:48.444 | INFO | Opening tmp RTMetrics json file | 20260508 09:43:48.455 | INFO | Serializing RTMetrics to json | 20260508 09:43:48.457 | INFO | Serialization done, swapping tmp file now... | 20260508 09:43:48.457 | INFO | RTMetrics done | 20260508 09:43:48.513 | INFO | Finished a round of Baz IO: Wrote 1.477 GiB in 3.48 seconds | 20260508 09:43:48.513 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:43:48.513 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.246414% / 1.58946e-05% / 3.97364e-05% | 20260508 09:43:48.513 | INFO | Baz IO overall: | 20260508 09:43:48.513 | INFO | BytesWritten : 3103583308596 | 20260508 09:43:48.513 | INFO | NumEvents : 1492658593814 | 20260508 09:43:50.078 | INFO | Analyzing chunk frames = [7122944,7123456) | 20260508 09:43:50.078 | INFO | MemInfo: { | 20260508 09:43:50.078 | INFO | "gpu" : " 17 GB", | 20260508 09:43:50.078 | INFO | "huge" : "", | 20260508 09:43:50.078 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:50.078 | INFO | "malloc" : " 14 GB", | 20260508 09:43:50.078 | INFO | "pinned" : " 29 GB" | 20260508 09:43:50.078 | INFO | } | 20260508 09:43:50.078 | INFO | Perf report for the last 5.14335 seconds: | 20260508 09:43:50.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:50.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:50.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:50.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:50.078 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:43:50.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:50.078 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:43:50.078 | INFO | PRE_HQ : 65.2%, 1.6, 54.3ms, 0 | 20260508 09:43:50.078 | INFO | BAZWRITER : 67.0%, 2.5, 87.1ms, 0 | 20260508 09:43:50.078 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:43:50.078 | INFO | Uploaded 36GiB to GPU in 2.59737s (13.8602GiB/s) | 20260508 09:43:50.079 | INFO | Downloaded 38GiB to GPU in 2.71028s (14.0207GiB/s) | 20260508 09:43:55.218 | INFO | Analyzing chunk frames = [7123456,7123968) | 20260508 09:43:55.218 | INFO | MemInfo: { | 20260508 09:43:55.218 | INFO | "gpu" : " 17 GB", | 20260508 09:43:55.218 | INFO | "huge" : "", | 20260508 09:43:55.218 | INFO | "huge_shared" : " 29 GB", | 20260508 09:43:55.218 | INFO | "malloc" : " 14 GB", | 20260508 09:43:55.218 | INFO | "pinned" : " 29 GB" | 20260508 09:43:55.218 | INFO | } | 20260508 09:43:55.218 | INFO | Perf report for the last 5.13981 seconds: | 20260508 09:43:55.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:43:55.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.218 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 09:43:55.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.218 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:43:55.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:43:55.219 | INFO | Uploaded 36GiB to GPU in 2.46551s (14.6014GiB/s) | 20260508 09:43:55.219 | INFO | Downloaded 34GiB to GPU in 2.46588s (13.7882GiB/s) | 20260508 09:44:00.352 | INFO | Analyzing chunk frames = [7123968,7124480) | 20260508 09:44:00.352 | INFO | MemInfo: { | 20260508 09:44:00.352 | INFO | "gpu" : " 17 GB", | 20260508 09:44:00.352 | INFO | "huge" : "", | 20260508 09:44:00.352 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:00.352 | INFO | "malloc" : " 14 GB", | 20260508 09:44:00.352 | INFO | "pinned" : " 29 GB" | 20260508 09:44:00.352 | INFO | } | 20260508 09:44:00.352 | INFO | Perf report for the last 5.13416 seconds: | 20260508 09:44:00.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:00.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.352 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 09:44:00.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.352 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:44:00.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:00.353 | INFO | Uploaded 36GiB to GPU in 2.46695s (14.5929GiB/s) | 20260508 09:44:00.353 | INFO | Downloaded 34GiB to GPU in 2.43329s (13.9728GiB/s) | 20260508 09:44:05.495 | INFO | Analyzing chunk frames = [7124480,7124992) | 20260508 09:44:05.495 | INFO | MemInfo: { | 20260508 09:44:05.495 | INFO | "gpu" : " 17 GB", | 20260508 09:44:05.495 | INFO | "huge" : "", | 20260508 09:44:05.495 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:05.495 | INFO | "malloc" : " 14 GB", | 20260508 09:44:05.495 | INFO | "pinned" : " 29 GB" | 20260508 09:44:05.495 | INFO | } | 20260508 09:44:05.495 | INFO | Perf report for the last 5.14294 seconds: | 20260508 09:44:05.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:05.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.495 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 09:44:05.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.495 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:44:05.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:05.496 | INFO | Uploaded 36GiB to GPU in 2.4786s (14.5243GiB/s) | 20260508 09:44:05.496 | INFO | Downloaded 34GiB to GPU in 2.42068s (14.0456GiB/s) | 20260508 09:44:10.636 | INFO | Analyzing chunk frames = [7124992,7125504) | 20260508 09:44:10.636 | INFO | MemInfo: { | 20260508 09:44:10.636 | INFO | "gpu" : " 17 GB", | 20260508 09:44:10.636 | INFO | "huge" : "", | 20260508 09:44:10.636 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:10.636 | INFO | "malloc" : " 14 GB", | 20260508 09:44:10.636 | INFO | "pinned" : " 29 GB" | 20260508 09:44:10.636 | INFO | } | 20260508 09:44:10.636 | INFO | Perf report for the last 5.14106 seconds: | 20260508 09:44:10.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:10.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.636 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 09:44:10.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.636 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:44:10.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:10.637 | INFO | Uploaded 36GiB to GPU in 2.37037s (15.1875GiB/s) | 20260508 09:44:10.637 | INFO | Downloaded 34GiB to GPU in 2.52231s (13.4797GiB/s) | 20260508 09:44:15.771 | INFO | Analyzing chunk frames = [7125504,7126016) | 20260508 09:44:15.771 | INFO | MemInfo: { | 20260508 09:44:15.771 | INFO | "gpu" : " 17 GB", | 20260508 09:44:15.771 | INFO | "huge" : "", | 20260508 09:44:15.771 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:15.771 | INFO | "malloc" : " 14 GB", | 20260508 09:44:15.771 | INFO | "pinned" : " 29 GB" | 20260508 09:44:15.772 | INFO | } | 20260508 09:44:15.772 | INFO | Perf report for the last 5.1352 seconds: | 20260508 09:44:15.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:15.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 09:44:15.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:44:15.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:15.772 | INFO | Uploaded 36GiB to GPU in 2.3467s (15.3407GiB/s) | 20260508 09:44:15.772 | INFO | Downloaded 34GiB to GPU in 2.45711s (13.8374GiB/s) | 20260508 09:44:20.906 | INFO | Analyzing chunk frames = [7126016,7126528) | 20260508 09:44:20.906 | INFO | MemInfo: { | 20260508 09:44:20.906 | INFO | "gpu" : " 17 GB", | 20260508 09:44:20.906 | INFO | "huge" : "", | 20260508 09:44:20.906 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:20.906 | INFO | "malloc" : " 14 GB", | 20260508 09:44:20.906 | INFO | "pinned" : " 29 GB" | 20260508 09:44:20.906 | INFO | } | 20260508 09:44:20.906 | INFO | Perf report for the last 5.13447 seconds: | 20260508 09:44:20.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:20.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:44:20.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:44:20.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:20.906 | INFO | Uploaded 36GiB to GPU in 2.36453s (15.225GiB/s) | 20260508 09:44:20.906 | INFO | Downloaded 34GiB to GPU in 2.44508s (13.9055GiB/s) | 20260508 09:44:26.044 | INFO | Analyzing chunk frames = [7126528,7127040) | 20260508 09:44:26.044 | INFO | MemInfo: { | 20260508 09:44:26.044 | INFO | "gpu" : " 17 GB", | 20260508 09:44:26.044 | INFO | "huge" : "", | 20260508 09:44:26.044 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:26.044 | INFO | "malloc" : " 14 GB", | 20260508 09:44:26.044 | INFO | "pinned" : " 29 GB" | 20260508 09:44:26.044 | INFO | } | 20260508 09:44:26.044 | INFO | Perf report for the last 5.13836 seconds: | 20260508 09:44:26.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:26.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.044 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 09:44:26.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.045 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:44:26.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:26.045 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:44:26.045 | INFO | Total mem usage: 16.7 GiB | 20260508 09:44:26.045 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:44:26.045 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:44:26.045 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:44:26.045 | INFO | High Water Breakdown: | 20260508 09:44:26.045 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:44:26.045 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:44:26.045 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:44:26.045 | INFO | ------------------------------------------ | 20260508 09:44:26.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:44:26.045 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:44:26.045 | INFO | Uploaded 36GiB to GPU in 2.41699s (14.8946GiB/s) | 20260508 09:44:26.045 | INFO | Total mem usage: 26.5 GiB | 20260508 09:44:26.045 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:44:26.045 | INFO | Downloaded 34GiB to GPU in 2.45643s (13.8412GiB/s) | 20260508 09:44:26.045 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:44:26.045 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:44:26.045 | INFO | High Water Breakdown: | 20260508 09:44:26.045 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:44:26.045 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:44:26.045 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:44:26.045 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:44:26.045 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:44:26.045 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:44:26.045 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:44:26.045 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:44:26.045 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:44:26.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:44:26.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:44:26.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:44:26.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:44:26.046 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:44:26.046 | INFO | ------------------------------------------ | 20260508 09:44:26.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:44:26.046 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:44:26.046 | INFO | Total mem usage: 27.8 GiB | 20260508 09:44:26.046 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:44:26.046 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:44:26.046 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:44:26.046 | INFO | High Water Breakdown: | 20260508 09:44:26.046 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:44:26.046 | INFO | ------------------------------------------ | 20260508 09:44:26.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:44:26.046 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:44:26.046 | INFO | Total mem usage: 15.7 GiB | 20260508 09:44:26.046 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:44:26.046 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:44:26.046 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:44:26.046 | INFO | High Water Breakdown: | 20260508 09:44:26.046 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:44:26.046 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:44:26.046 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:44:26.046 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:44:26.046 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:44:26.046 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:44:26.046 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:44:26.046 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:44:26.046 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:44:26.046 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:44:26.046 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:44:26.046 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:44:26.046 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:44:26.046 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:44:26.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:44:26.046 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:44:26.046 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:44:26.046 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:44:26.046 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:44:26.046 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:44:26.046 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:44:26.046 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:44:26.046 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:44:26.046 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:44:26.046 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:44:26.046 | INFO | ------------------------------------------ | 20260508 09:44:26.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:44:26.046 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:44:26.046 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:44:26.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:44:26.046 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.42e+01 / 7.09e-01 | 20260508 09:44:26.046 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.08e-02 / 5.99e-03 | 20260508 09:44:26.046 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.39e+00, 5.91e-01 / 2.41e-01 | 20260508 09:44:26.046 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.41e+00, 2.10e-01 / 1.41e-01 | 20260508 09:44:26.046 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.53e+01, 3.80e+00 / 1.23e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.82e+00 / 1.24e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.46e+00, 4.06e-01 / 2.56e-01 | 20260508 09:44:26.046 | INFO | PerfReport: 3.03%, Metrics -- 9.93e-01 / 1.12e+00, 2.47e-01 / 1.31e-01 | 20260508 09:44:26.046 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.32e-01, 4.83e-02 / 2.63e-02 | 20260508 09:44:26.046 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:44:26.046 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:44:26.046 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:44:26.046 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 24.55%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 2.23%, LockUpload -- 2.35e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 2.52%, LockDownload -- 2.65e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:44:26.046 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 09:44:26.142 | INFO | Starting a round of Baz IO | 20260508 09:44:29.568 | INFO | Populating RTMetrics report | 20260508 09:44:29.568 | INFO | RTMetrics report populated | 20260508 09:44:29.568 | INFO | Opening tmp RTMetrics json file | 20260508 09:44:29.578 | INFO | Serializing RTMetrics to json | 20260508 09:44:29.579 | INFO | Serialization done, swapping tmp file now... | 20260508 09:44:29.580 | INFO | RTMetrics done | 20260508 09:44:29.637 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.495 seconds | 20260508 09:44:29.637 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:44:29.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.245416% / 0% / 4.37101e-05% | 20260508 09:44:29.638 | INFO | Baz IO overall: | 20260508 09:44:29.638 | INFO | BytesWritten : 3105168293160 | 20260508 09:44:29.638 | INFO | NumEvents : 1493407578898 | 20260508 09:44:31.189 | INFO | Analyzing chunk frames = [7127040,7127552) | 20260508 09:44:31.189 | INFO | MemInfo: { | 20260508 09:44:31.189 | INFO | "gpu" : " 17 GB", | 20260508 09:44:31.189 | INFO | "huge" : "", | 20260508 09:44:31.189 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:31.189 | INFO | "malloc" : " 14 GB", | 20260508 09:44:31.189 | INFO | "pinned" : " 29 GB" | 20260508 09:44:31.189 | INFO | } | 20260508 09:44:31.189 | INFO | Perf report for the last 5.14469 seconds: | 20260508 09:44:31.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:31.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:31.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:31.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:31.189 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 09:44:31.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:31.189 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 09:44:31.189 | INFO | PRE_HQ : 65.5%, 1.6, 54.3ms, 0 | 20260508 09:44:31.189 | INFO | BAZWRITER : 67.3%, 2.5, 87.4ms, 0 | 20260508 09:44:31.189 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:44:31.190 | INFO | Uploaded 36GiB to GPU in 2.59448s (13.8756GiB/s) | 20260508 09:44:31.190 | INFO | Downloaded 38GiB to GPU in 2.71803s (13.9807GiB/s) | 20260508 09:44:36.330 | INFO | Analyzing chunk frames = [7127552,7128064) | 20260508 09:44:36.330 | INFO | MemInfo: { | 20260508 09:44:36.330 | INFO | "gpu" : " 17 GB", | 20260508 09:44:36.330 | INFO | "huge" : "", | 20260508 09:44:36.330 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:36.330 | INFO | "malloc" : " 14 GB", | 20260508 09:44:36.330 | INFO | "pinned" : " 29 GB" | 20260508 09:44:36.330 | INFO | } | 20260508 09:44:36.330 | INFO | Perf report for the last 5.14122 seconds: | 20260508 09:44:36.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:36.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.330 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:44:36.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.331 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 09:44:36.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:36.331 | INFO | Uploaded 36GiB to GPU in 2.49414s (14.4338GiB/s) | 20260508 09:44:36.331 | INFO | Downloaded 34GiB to GPU in 2.46891s (13.7713GiB/s) | 20260508 09:44:41.468 | INFO | Analyzing chunk frames = [7128064,7128576) | 20260508 09:44:41.468 | INFO | MemInfo: { | 20260508 09:44:41.468 | INFO | "gpu" : " 17 GB", | 20260508 09:44:41.468 | INFO | "huge" : "", | 20260508 09:44:41.468 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:41.468 | INFO | "malloc" : " 14 GB", | 20260508 09:44:41.468 | INFO | "pinned" : " 29 GB" | 20260508 09:44:41.468 | INFO | } | 20260508 09:44:41.469 | INFO | Perf report for the last 5.13813 seconds: | 20260508 09:44:41.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:41.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 09:44:41.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260508 09:44:41.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:41.469 | INFO | Uploaded 36GiB to GPU in 2.45415s (14.669GiB/s) | 20260508 09:44:41.469 | INFO | Downloaded 34GiB to GPU in 2.45341s (13.8583GiB/s) | 20260508 09:44:46.606 | INFO | Analyzing chunk frames = [7128576,7129088) | 20260508 09:44:46.606 | INFO | MemInfo: { | 20260508 09:44:46.606 | INFO | "gpu" : " 17 GB", | 20260508 09:44:46.606 | INFO | "huge" : "", | 20260508 09:44:46.606 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:46.606 | INFO | "malloc" : " 14 GB", | 20260508 09:44:46.606 | INFO | "pinned" : " 29 GB" | 20260508 09:44:46.606 | INFO | } | 20260508 09:44:46.607 | INFO | Perf report for the last 5.13806 seconds: | 20260508 09:44:46.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:46.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:44:46.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 09:44:46.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:46.607 | INFO | Uploaded 36GiB to GPU in 2.49308s (14.4399GiB/s) | 20260508 09:44:46.607 | INFO | Downloaded 34GiB to GPU in 2.44881s (13.8843GiB/s) | 20260508 09:44:51.748 | INFO | Analyzing chunk frames = [7129088,7129600) | 20260508 09:44:51.748 | INFO | MemInfo: { | 20260508 09:44:51.748 | INFO | "gpu" : " 17 GB", | 20260508 09:44:51.748 | INFO | "huge" : "", | 20260508 09:44:51.748 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:51.748 | INFO | "malloc" : " 14 GB", | 20260508 09:44:51.748 | INFO | "pinned" : " 29 GB" | 20260508 09:44:51.748 | INFO | } | 20260508 09:44:51.748 | INFO | Perf report for the last 5.14145 seconds: | 20260508 09:44:51.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:51.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 09:44:51.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 09:44:51.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:51.748 | INFO | Uploaded 36GiB to GPU in 2.29477s (15.6878GiB/s) | 20260508 09:44:51.748 | INFO | Downloaded 34GiB to GPU in 2.51955s (13.4945GiB/s) | 20260508 09:44:56.886 | INFO | Analyzing chunk frames = [7129600,7130112) | 20260508 09:44:56.886 | INFO | MemInfo: { | 20260508 09:44:56.886 | INFO | "gpu" : " 17 GB", | 20260508 09:44:56.886 | INFO | "huge" : "", | 20260508 09:44:56.886 | INFO | "huge_shared" : " 29 GB", | 20260508 09:44:56.886 | INFO | "malloc" : " 14 GB", | 20260508 09:44:56.886 | INFO | "pinned" : " 29 GB" | 20260508 09:44:56.886 | INFO | } | 20260508 09:44:56.886 | INFO | Perf report for the last 5.13803 seconds: | 20260508 09:44:56.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:44:56.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.886 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:44:56.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.886 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:44:56.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:44:56.887 | INFO | Uploaded 36GiB to GPU in 2.35682s (15.2748GiB/s) | 20260508 09:44:56.887 | INFO | Downloaded 34GiB to GPU in 2.44803s (13.8887GiB/s) | 20260508 09:45:02.026 | INFO | Analyzing chunk frames = [7130112,7130624) | 20260508 09:45:02.026 | INFO | MemInfo: { | 20260508 09:45:02.026 | INFO | "gpu" : " 17 GB", | 20260508 09:45:02.026 | INFO | "huge" : "", | 20260508 09:45:02.026 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:02.026 | INFO | "malloc" : " 14 GB", | 20260508 09:45:02.026 | INFO | "pinned" : " 29 GB" | 20260508 09:45:02.026 | INFO | } | 20260508 09:45:02.026 | INFO | Perf report for the last 5.13986 seconds: | 20260508 09:45:02.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:02.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:45:02.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 09:45:02.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:02.026 | INFO | Uploaded 36GiB to GPU in 2.39342s (15.0412GiB/s) | 20260508 09:45:02.026 | INFO | Downloaded 34GiB to GPU in 2.45373s (13.8565GiB/s) | 20260508 09:45:07.166 | INFO | Analyzing chunk frames = [7130624,7131136) | 20260508 09:45:07.166 | INFO | MemInfo: { | 20260508 09:45:07.166 | INFO | "gpu" : " 17 GB", | 20260508 09:45:07.166 | INFO | "huge" : "", | 20260508 09:45:07.167 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:07.167 | INFO | "malloc" : " 14 GB", | 20260508 09:45:07.167 | INFO | "pinned" : " 29 GB" | 20260508 09:45:07.167 | INFO | } | 20260508 09:45:07.167 | INFO | Perf report for the last 5.14067 seconds: | 20260508 09:45:07.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:07.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 09:45:07.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 09:45:07.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:07.167 | INFO | Uploaded 36GiB to GPU in 2.3774s (15.1426GiB/s) | 20260508 09:45:07.167 | INFO | Downloaded 34GiB to GPU in 2.44399s (13.9117GiB/s) | 20260508 09:45:07.265 | INFO | Starting a round of Baz IO | 20260508 09:45:10.689 | INFO | Populating RTMetrics report | 20260508 09:45:10.690 | INFO | RTMetrics report populated | 20260508 09:45:10.690 | INFO | Opening tmp RTMetrics json file | 20260508 09:45:10.698 | INFO | Serializing RTMetrics to json | 20260508 09:45:10.699 | INFO | Serialization done, swapping tmp file now... | 20260508 09:45:10.700 | INFO | RTMetrics done | 20260508 09:45:10.757 | INFO | Finished a round of Baz IO: Wrote 1.475 GiB in 3.492 seconds | 20260508 09:45:10.757 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:45:10.757 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.244908% / 7.94729e-06% / 3.97364e-05% | 20260508 09:45:10.757 | INFO | Baz IO overall: | 20260508 09:45:10.757 | INFO | BytesWritten : 3106752084744 | 20260508 09:45:10.757 | INFO | NumEvents : 1494155441675 | 20260508 09:45:12.307 | INFO | Analyzing chunk frames = [7131136,7131648) | 20260508 09:45:12.307 | INFO | MemInfo: { | 20260508 09:45:12.307 | INFO | "gpu" : " 17 GB", | 20260508 09:45:12.307 | INFO | "huge" : "", | 20260508 09:45:12.307 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:12.307 | INFO | "malloc" : " 14 GB", | 20260508 09:45:12.307 | INFO | "pinned" : " 29 GB" | 20260508 09:45:12.307 | INFO | } | 20260508 09:45:12.307 | INFO | Perf report for the last 5.1405 seconds: | 20260508 09:45:12.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:12.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:12.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:12.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:12.307 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 09:45:12.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:12.307 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:45:12.307 | INFO | PRE_HQ : 65.6%, 1.6, 54.8ms, 0 | 20260508 09:45:12.307 | INFO | BAZWRITER : 67.3%, 2.5, 87.8ms, 0 | 20260508 09:45:12.307 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:45:12.307 | INFO | Uploaded 36GiB to GPU in 2.60158s (13.8378GiB/s) | 20260508 09:45:12.308 | INFO | Downloaded 38GiB to GPU in 2.71055s (14.0193GiB/s) | 20260508 09:45:17.444 | INFO | Analyzing chunk frames = [7131648,7132160) | 20260508 09:45:17.444 | INFO | MemInfo: { | 20260508 09:45:17.444 | INFO | "gpu" : " 17 GB", | 20260508 09:45:17.444 | INFO | "huge" : "", | 20260508 09:45:17.444 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:17.444 | INFO | "malloc" : " 14 GB", | 20260508 09:45:17.444 | INFO | "pinned" : " 29 GB" | 20260508 09:45:17.444 | INFO | } | 20260508 09:45:17.444 | INFO | Perf report for the last 5.13684 seconds: | 20260508 09:45:17.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:17.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 09:45:17.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:45:17.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:17.444 | INFO | Uploaded 36GiB to GPU in 2.49922s (14.4045GiB/s) | 20260508 09:45:17.444 | INFO | Downloaded 34GiB to GPU in 2.48269s (13.6948GiB/s) | 20260508 09:45:22.584 | INFO | Analyzing chunk frames = [7132160,7132672) | 20260508 09:45:22.584 | INFO | MemInfo: { | 20260508 09:45:22.584 | INFO | "gpu" : " 17 GB", | 20260508 09:45:22.584 | INFO | "huge" : "", | 20260508 09:45:22.584 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:22.584 | INFO | "malloc" : " 14 GB", | 20260508 09:45:22.584 | INFO | "pinned" : " 29 GB" | 20260508 09:45:22.585 | INFO | } | 20260508 09:45:22.585 | INFO | Perf report for the last 5.14059 seconds: | 20260508 09:45:22.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:22.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.585 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 09:45:22.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.585 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:45:22.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:22.586 | INFO | Uploaded 36GiB to GPU in 2.44481s (14.7251GiB/s) | 20260508 09:45:22.586 | INFO | Downloaded 34GiB to GPU in 2.42566s (14.0168GiB/s) | 20260508 09:45:27.723 | INFO | Analyzing chunk frames = [7132672,7133184) | 20260508 09:45:27.723 | INFO | MemInfo: { | 20260508 09:45:27.723 | INFO | "gpu" : " 17 GB", | 20260508 09:45:27.723 | INFO | "huge" : "", | 20260508 09:45:27.723 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:27.723 | INFO | "malloc" : " 14 GB", | 20260508 09:45:27.723 | INFO | "pinned" : " 29 GB" | 20260508 09:45:27.723 | INFO | } | 20260508 09:45:27.724 | INFO | Perf report for the last 5.13899 seconds: | 20260508 09:45:27.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:27.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 09:45:27.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 09:45:27.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:27.724 | INFO | Uploaded 36GiB to GPU in 2.41843s (14.8857GiB/s) | 20260508 09:45:27.724 | INFO | Downloaded 34GiB to GPU in 2.43706s (13.9512GiB/s) | 20260508 09:45:32.857 | INFO | Analyzing chunk frames = [7133184,7133696) | 20260508 09:45:32.857 | INFO | MemInfo: { | 20260508 09:45:32.857 | INFO | "gpu" : " 17 GB", | 20260508 09:45:32.857 | INFO | "huge" : "", | 20260508 09:45:32.857 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:32.857 | INFO | "malloc" : " 14 GB", | 20260508 09:45:32.857 | INFO | "pinned" : " 29 GB" | 20260508 09:45:32.857 | INFO | } | 20260508 09:45:32.857 | INFO | Perf report for the last 5.13348 seconds: | 20260508 09:45:32.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:32.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 09:45:32.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 09:45:32.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:32.857 | INFO | Uploaded 36GiB to GPU in 2.37231s (15.1751GiB/s) | 20260508 09:45:32.857 | INFO | Downloaded 34GiB to GPU in 2.52156s (13.4837GiB/s) | 20260508 09:45:38.005 | INFO | Analyzing chunk frames = [7133696,7134208) | 20260508 09:45:38.005 | INFO | MemInfo: { | 20260508 09:45:38.005 | INFO | "gpu" : " 17 GB", | 20260508 09:45:38.005 | INFO | "huge" : "", | 20260508 09:45:38.005 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:38.005 | INFO | "malloc" : " 14 GB", | 20260508 09:45:38.005 | INFO | "pinned" : " 29 GB" | 20260508 09:45:38.005 | INFO | } | 20260508 09:45:38.005 | INFO | Perf report for the last 5.14812 seconds: | 20260508 09:45:38.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:38.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.005 | INFO | ANALYSIS : 65.1%, 3.0, 101.5ms, 0 | 20260508 09:45:38.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.005 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:45:38.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:38.006 | INFO | Uploaded 36GiB to GPU in 2.33505s (15.4172GiB/s) | 20260508 09:45:38.006 | INFO | Downloaded 34GiB to GPU in 2.45045s (13.875GiB/s) | 20260508 09:45:43.147 | INFO | Analyzing chunk frames = [7134208,7134720) | 20260508 09:45:43.147 | INFO | MemInfo: { | 20260508 09:45:43.147 | INFO | "gpu" : " 17 GB", | 20260508 09:45:43.147 | INFO | "huge" : "", | 20260508 09:45:43.147 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:43.147 | INFO | "malloc" : " 14 GB", | 20260508 09:45:43.148 | INFO | "pinned" : " 29 GB" | 20260508 09:45:43.148 | INFO | } | 20260508 09:45:43.148 | INFO | Perf report for the last 5.14239 seconds: | 20260508 09:45:43.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:43.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:45:43.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:45:43.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:43.148 | INFO | Uploaded 36GiB to GPU in 2.39571s (15.0269GiB/s) | 20260508 09:45:43.148 | INFO | Downloaded 34GiB to GPU in 2.45263s (13.8627GiB/s) | 20260508 09:45:48.284 | INFO | Analyzing chunk frames = [7134720,7135232) | 20260508 09:45:48.284 | INFO | MemInfo: { | 20260508 09:45:48.284 | INFO | "gpu" : " 17 GB", | 20260508 09:45:48.284 | INFO | "huge" : "", | 20260508 09:45:48.284 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:48.284 | INFO | "malloc" : " 14 GB", | 20260508 09:45:48.284 | INFO | "pinned" : " 29 GB" | 20260508 09:45:48.284 | INFO | } | 20260508 09:45:48.284 | INFO | Perf report for the last 5.13639 seconds: | 20260508 09:45:48.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:48.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.284 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 09:45:48.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.284 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:45:48.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:48.285 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:45:48.285 | INFO | Total mem usage: 16.7 GiB | 20260508 09:45:48.285 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:45:48.285 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:45:48.285 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:45:48.285 | INFO | High Water Breakdown: | 20260508 09:45:48.285 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:45:48.285 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:45:48.285 | INFO | Uploaded 36GiB to GPU in 2.40398s (14.9752GiB/s) | 20260508 09:45:48.285 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:45:48.285 | INFO | ------------------------------------------ | 20260508 09:45:48.285 | INFO | Downloaded 34GiB to GPU in 2.45552s (13.8463GiB/s) | 20260508 09:45:48.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:45:48.285 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:45:48.285 | INFO | Total mem usage: 26.5 GiB | 20260508 09:45:48.285 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:45:48.285 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:45:48.285 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:45:48.285 | INFO | High Water Breakdown: | 20260508 09:45:48.285 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:45:48.285 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:45:48.285 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:45:48.285 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:45:48.285 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:45:48.285 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:45:48.285 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:45:48.285 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:45:48.285 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:45:48.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:45:48.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:45:48.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:45:48.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:45:48.285 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:45:48.285 | INFO | ------------------------------------------ | 20260508 09:45:48.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:45:48.285 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:45:48.285 | INFO | Total mem usage: 27.8 GiB | 20260508 09:45:48.285 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:45:48.285 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:45:48.285 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:45:48.285 | INFO | High Water Breakdown: | 20260508 09:45:48.285 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:45:48.285 | INFO | ------------------------------------------ | 20260508 09:45:48.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:45:48.285 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:45:48.285 | INFO | Total mem usage: 15.7 GiB | 20260508 09:45:48.285 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:45:48.285 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:45:48.285 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:45:48.285 | INFO | High Water Breakdown: | 20260508 09:45:48.285 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:45:48.285 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:45:48.285 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:45:48.285 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:45:48.285 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:45:48.285 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:45:48.286 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:45:48.286 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:45:48.286 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:45:48.286 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:45:48.286 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:45:48.286 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:45:48.286 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:45:48.286 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:45:48.286 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:45:48.286 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:45:48.286 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:45:48.286 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:45:48.286 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:45:48.286 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:45:48.286 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:45:48.286 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:45:48.286 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:45:48.286 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:45:48.286 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:45:48.286 | INFO | ------------------------------------------ | 20260508 09:45:48.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:45:48.286 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:45:48.286 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:45:48.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:45:48.286 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.56e+01, 1.39e+01 / 7.73e-01 | 20260508 09:45:48.286 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.06e-01, 1.20e-02 / 4.88e-03 | 20260508 09:45:48.286 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.43e+00, 5.91e-01 / 2.00e-01 | 20260508 09:45:48.286 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.43e+00, 2.09e-01 / 1.32e-01 | 20260508 09:45:48.286 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.54e+01, 3.80e+00 / 1.33e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.83e+00 / 1.37e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.54e+00, 4.06e-01 / 2.32e-01 | 20260508 09:45:48.286 | INFO | PerfReport: 3.03%, Metrics -- 9.94e-01 / 1.16e+00, 2.45e-01 / 1.51e-01 | 20260508 09:45:48.286 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.35e-01, 4.81e-02 / 2.15e-02 | 20260508 09:45:48.286 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:45:48.286 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:45:48.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:45:48.286 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 24.08%, Upload -- 2.53e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 24.67%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 31.12%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 1.98%, LockUpload -- 2.09e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 2.73%, LockDownload -- 2.88e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 15.40%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:45:48.286 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 09:45:48.382 | INFO | Starting a round of Baz IO | 20260508 09:45:51.795 | INFO | Populating RTMetrics report | 20260508 09:45:51.795 | INFO | RTMetrics report populated | 20260508 09:45:51.795 | INFO | Opening tmp RTMetrics json file | 20260508 09:45:51.809 | INFO | Serializing RTMetrics to json | 20260508 09:45:51.810 | INFO | Serialization done, swapping tmp file now... | 20260508 09:45:51.810 | INFO | RTMetrics done | 20260508 09:45:51.861 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.479 seconds | 20260508 09:45:51.861 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:45:51.861 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.245726% / 3.97364e-06% / 4.76837e-05% | 20260508 09:45:51.861 | INFO | Baz IO overall: | 20260508 09:45:51.861 | INFO | BytesWritten : 3108337016397 | 20260508 09:45:51.861 | INFO | NumEvents : 1494904162513 | 20260508 09:45:53.417 | INFO | Analyzing chunk frames = [7135232,7135744) | 20260508 09:45:53.417 | INFO | MemInfo: { | 20260508 09:45:53.417 | INFO | "gpu" : " 17 GB", | 20260508 09:45:53.417 | INFO | "huge" : "", | 20260508 09:45:53.417 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:53.417 | INFO | "malloc" : " 14 GB", | 20260508 09:45:53.417 | INFO | "pinned" : " 29 GB" | 20260508 09:45:53.417 | INFO | } | 20260508 09:45:53.417 | INFO | Perf report for the last 5.13297 seconds: | 20260508 09:45:53.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:53.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:53.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:53.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:53.417 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260508 09:45:53.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:53.417 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:45:53.417 | INFO | PRE_HQ : 65.8%, 1.6, 54.4ms, 0 | 20260508 09:45:53.417 | INFO | BAZWRITER : 67.0%, 2.6, 89.1ms, 0 | 20260508 09:45:53.417 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:45:53.417 | INFO | Uploaded 36GiB to GPU in 2.56026s (14.0611GiB/s) | 20260508 09:45:53.417 | INFO | Downloaded 38GiB to GPU in 2.71237s (14.0099GiB/s) | 20260508 09:45:58.557 | INFO | Analyzing chunk frames = [7135744,7136256) | 20260508 09:45:58.557 | INFO | MemInfo: { | 20260508 09:45:58.557 | INFO | "gpu" : " 17 GB", | 20260508 09:45:58.557 | INFO | "huge" : "", | 20260508 09:45:58.557 | INFO | "huge_shared" : " 29 GB", | 20260508 09:45:58.557 | INFO | "malloc" : " 14 GB", | 20260508 09:45:58.557 | INFO | "pinned" : " 29 GB" | 20260508 09:45:58.557 | INFO | } | 20260508 09:45:58.557 | INFO | Perf report for the last 5.14035 seconds: | 20260508 09:45:58.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:45:58.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.557 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 09:45:58.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.558 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 09:45:58.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:45:58.558 | INFO | Uploaded 36GiB to GPU in 2.42319s (14.8565GiB/s) | 20260508 09:45:58.558 | INFO | Downloaded 34GiB to GPU in 2.47601s (13.7318GiB/s) | 20260508 09:46:03.694 | INFO | Analyzing chunk frames = [7136256,7136768) | 20260508 09:46:03.694 | INFO | MemInfo: { | 20260508 09:46:03.694 | INFO | "gpu" : " 17 GB", | 20260508 09:46:03.694 | INFO | "huge" : "", | 20260508 09:46:03.694 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:03.694 | INFO | "malloc" : " 14 GB", | 20260508 09:46:03.694 | INFO | "pinned" : " 29 GB" | 20260508 09:46:03.695 | INFO | } | 20260508 09:46:03.695 | INFO | Perf report for the last 5.13729 seconds: | 20260508 09:46:03.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:03.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:46:03.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:46:03.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:03.695 | INFO | Uploaded 36GiB to GPU in 2.44465s (14.7261GiB/s) | 20260508 09:46:03.695 | INFO | Downloaded 34GiB to GPU in 2.45467s (13.8511GiB/s) | 20260508 09:46:08.838 | INFO | Analyzing chunk frames = [7136768,7137280) | 20260508 09:46:08.838 | INFO | MemInfo: { | 20260508 09:46:08.838 | INFO | "gpu" : " 17 GB", | 20260508 09:46:08.838 | INFO | "huge" : "", | 20260508 09:46:08.838 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:08.838 | INFO | "malloc" : " 14 GB", | 20260508 09:46:08.838 | INFO | "pinned" : " 29 GB" | 20260508 09:46:08.838 | INFO | } | 20260508 09:46:08.838 | INFO | Perf report for the last 5.14371 seconds: | 20260508 09:46:08.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:08.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.839 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:46:08.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.839 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:46:08.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:08.839 | INFO | Uploaded 36GiB to GPU in 2.48032s (14.5143GiB/s) | 20260508 09:46:08.839 | INFO | Downloaded 34GiB to GPU in 2.45412s (13.8542GiB/s) | 20260508 09:46:13.984 | INFO | Analyzing chunk frames = [7137280,7137792) | 20260508 09:46:13.984 | INFO | MemInfo: { | 20260508 09:46:13.984 | INFO | "gpu" : " 17 GB", | 20260508 09:46:13.984 | INFO | "huge" : "", | 20260508 09:46:13.984 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:13.984 | INFO | "malloc" : " 14 GB", | 20260508 09:46:13.984 | INFO | "pinned" : " 29 GB" | 20260508 09:46:13.984 | INFO | } | 20260508 09:46:13.984 | INFO | Perf report for the last 5.14557 seconds: | 20260508 09:46:13.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:13.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | ANALYSIS : 66.2%, 3.0, 103.7ms, 0 | 20260508 09:46:13.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 09:46:13.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:13.984 | INFO | Uploaded 36GiB to GPU in 2.36035s (15.252GiB/s) | 20260508 09:46:13.984 | INFO | Downloaded 34GiB to GPU in 2.52759s (13.4516GiB/s) | 20260508 09:46:19.112 | INFO | Analyzing chunk frames = [7137792,7138304) | 20260508 09:46:19.112 | INFO | MemInfo: { | 20260508 09:46:19.112 | INFO | "gpu" : " 17 GB", | 20260508 09:46:19.112 | INFO | "huge" : "", | 20260508 09:46:19.112 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:19.112 | INFO | "malloc" : " 14 GB", | 20260508 09:46:19.112 | INFO | "pinned" : " 29 GB" | 20260508 09:46:19.112 | INFO | } | 20260508 09:46:19.112 | INFO | Perf report for the last 5.12838 seconds: | 20260508 09:46:19.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:19.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.112 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 09:46:19.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.113 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:46:19.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:19.113 | INFO | Uploaded 36GiB to GPU in 2.37449s (15.1612GiB/s) | 20260508 09:46:19.113 | INFO | Downloaded 34GiB to GPU in 2.43112s (13.9853GiB/s) | 20260508 09:46:24.257 | INFO | Analyzing chunk frames = [7138304,7138816) | 20260508 09:46:24.257 | INFO | MemInfo: { | 20260508 09:46:24.257 | INFO | "gpu" : " 17 GB", | 20260508 09:46:24.257 | INFO | "huge" : "", | 20260508 09:46:24.257 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:24.258 | INFO | "malloc" : " 14 GB", | 20260508 09:46:24.258 | INFO | "pinned" : " 29 GB" | 20260508 09:46:24.258 | INFO | } | 20260508 09:46:24.258 | INFO | Perf report for the last 5.14526 seconds: | 20260508 09:46:24.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:24.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 09:46:24.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:46:24.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:24.258 | INFO | Uploaded 36GiB to GPU in 2.40904s (14.9437GiB/s) | 20260508 09:46:24.258 | INFO | Downloaded 34GiB to GPU in 2.44194s (13.9234GiB/s) | 20260508 09:46:29.399 | INFO | Analyzing chunk frames = [7138816,7139328) | 20260508 09:46:29.399 | INFO | MemInfo: { | 20260508 09:46:29.399 | INFO | "gpu" : " 17 GB", | 20260508 09:46:29.399 | INFO | "huge" : "", | 20260508 09:46:29.399 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:29.399 | INFO | "malloc" : " 14 GB", | 20260508 09:46:29.399 | INFO | "pinned" : " 29 GB" | 20260508 09:46:29.399 | INFO | } | 20260508 09:46:29.399 | INFO | Perf report for the last 5.14121 seconds: | 20260508 09:46:29.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:29.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:46:29.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:46:29.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:29.399 | INFO | Uploaded 36GiB to GPU in 2.40808s (14.9497GiB/s) | 20260508 09:46:29.399 | INFO | Downloaded 34GiB to GPU in 2.43829s (13.9442GiB/s) | 20260508 09:46:29.514 | INFO | Starting a round of Baz IO | 20260508 09:46:32.908 | INFO | Populating RTMetrics report | 20260508 09:46:32.908 | INFO | RTMetrics report populated | 20260508 09:46:32.908 | INFO | Opening tmp RTMetrics json file | 20260508 09:46:32.918 | INFO | Serializing RTMetrics to json | 20260508 09:46:32.920 | INFO | Serialization done, swapping tmp file now... | 20260508 09:46:32.920 | INFO | RTMetrics done | 20260508 09:46:32.977 | INFO | Finished a round of Baz IO: Wrote 1.477 GiB in 3.463 seconds | 20260508 09:46:32.977 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:46:32.977 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.245647% / 0% / 8.74201e-05% | 20260508 09:46:32.977 | INFO | Baz IO overall: | 20260508 09:46:32.977 | INFO | BytesWritten : 3109922769945 | 20260508 09:46:32.977 | INFO | NumEvents : 1495653417282 | 20260508 09:46:34.536 | INFO | Analyzing chunk frames = [7139328,7139840) | 20260508 09:46:34.536 | INFO | MemInfo: { | 20260508 09:46:34.536 | INFO | "gpu" : " 17 GB", | 20260508 09:46:34.536 | INFO | "huge" : "", | 20260508 09:46:34.536 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:34.536 | INFO | "malloc" : " 14 GB", | 20260508 09:46:34.536 | INFO | "pinned" : " 29 GB" | 20260508 09:46:34.536 | INFO | } | 20260508 09:46:34.536 | INFO | Perf report for the last 5.13705 seconds: | 20260508 09:46:34.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:34.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:34.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:34.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:34.536 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 09:46:34.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:34.536 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 09:46:34.536 | INFO | PRE_HQ : 66.3%, 1.7, 59.9ms, 0 | 20260508 09:46:34.536 | INFO | BAZWRITER : 66.7%, 2.7, 91.8ms, 0 | 20260508 09:46:34.536 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:46:34.536 | INFO | Uploaded 36GiB to GPU in 2.62377s (13.7207GiB/s) | 20260508 09:46:34.536 | INFO | Downloaded 38GiB to GPU in 2.70247s (14.0612GiB/s) | 20260508 09:46:39.676 | INFO | Analyzing chunk frames = [7139840,7140352) | 20260508 09:46:39.676 | INFO | MemInfo: { | 20260508 09:46:39.676 | INFO | "gpu" : " 17 GB", | 20260508 09:46:39.676 | INFO | "huge" : "", | 20260508 09:46:39.677 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:39.677 | INFO | "malloc" : " 14 GB", | 20260508 09:46:39.677 | INFO | "pinned" : " 29 GB" | 20260508 09:46:39.677 | INFO | } | 20260508 09:46:39.677 | INFO | Perf report for the last 5.14066 seconds: | 20260508 09:46:39.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:39.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 09:46:39.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:46:39.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:39.677 | INFO | Uploaded 36GiB to GPU in 2.5406s (14.1699GiB/s) | 20260508 09:46:39.677 | INFO | Downloaded 34GiB to GPU in 2.4563s (13.842GiB/s) | 20260508 09:46:44.813 | INFO | Analyzing chunk frames = [7140352,7140864) | 20260508 09:46:44.813 | INFO | MemInfo: { | 20260508 09:46:44.813 | INFO | "gpu" : " 17 GB", | 20260508 09:46:44.813 | INFO | "huge" : "", | 20260508 09:46:44.813 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:44.813 | INFO | "malloc" : " 14 GB", | 20260508 09:46:44.813 | INFO | "pinned" : " 29 GB" | 20260508 09:46:44.813 | INFO | } | 20260508 09:46:44.813 | INFO | Perf report for the last 5.13643 seconds: | 20260508 09:46:44.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:44.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 09:46:44.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:46:44.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:44.813 | INFO | Uploaded 36GiB to GPU in 2.51866s (14.2933GiB/s) | 20260508 09:46:44.813 | INFO | Downloaded 34GiB to GPU in 2.43672s (13.9532GiB/s) | 20260508 09:46:49.951 | INFO | Analyzing chunk frames = [7140864,7141376) | 20260508 09:46:49.951 | INFO | MemInfo: { | 20260508 09:46:49.951 | INFO | "gpu" : " 17 GB", | 20260508 09:46:49.951 | INFO | "huge" : "", | 20260508 09:46:49.951 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:49.952 | INFO | "malloc" : " 14 GB", | 20260508 09:46:49.952 | INFO | "pinned" : " 29 GB" | 20260508 09:46:49.952 | INFO | } | 20260508 09:46:49.952 | INFO | Perf report for the last 5.13864 seconds: | 20260508 09:46:49.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:49.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 09:46:49.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:46:49.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:49.952 | INFO | Uploaded 36GiB to GPU in 2.49481s (14.43GiB/s) | 20260508 09:46:49.952 | INFO | Downloaded 34GiB to GPU in 2.39926s (14.171GiB/s) | 20260508 09:46:55.092 | INFO | Analyzing chunk frames = [7141376,7141888) | 20260508 09:46:55.092 | INFO | MemInfo: { | 20260508 09:46:55.092 | INFO | "gpu" : " 17 GB", | 20260508 09:46:55.092 | INFO | "huge" : "", | 20260508 09:46:55.092 | INFO | "huge_shared" : " 29 GB", | 20260508 09:46:55.092 | INFO | "malloc" : " 14 GB", | 20260508 09:46:55.092 | INFO | "pinned" : " 29 GB" | 20260508 09:46:55.092 | INFO | } | 20260508 09:46:55.092 | INFO | Perf report for the last 5.14031 seconds: | 20260508 09:46:55.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:46:55.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.092 | INFO | ANALYSIS : 66.6%, 3.0, 104.1ms, 0 | 20260508 09:46:55.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.092 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:46:55.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:46:55.093 | INFO | Uploaded 36GiB to GPU in 2.48009s (14.5156GiB/s) | 20260508 09:46:55.093 | INFO | Downloaded 34GiB to GPU in 2.5401s (13.3853GiB/s) | 20260508 09:47:00.229 | INFO | Analyzing chunk frames = [7141888,7142400) | 20260508 09:47:00.229 | INFO | MemInfo: { | 20260508 09:47:00.229 | INFO | "gpu" : " 17 GB", | 20260508 09:47:00.229 | INFO | "huge" : "", | 20260508 09:47:00.229 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:00.229 | INFO | "malloc" : " 14 GB", | 20260508 09:47:00.229 | INFO | "pinned" : " 29 GB" | 20260508 09:47:00.229 | INFO | } | 20260508 09:47:00.229 | INFO | Perf report for the last 5.13725 seconds: | 20260508 09:47:00.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:00.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.229 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:47:00.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.229 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:47:00.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:00.230 | INFO | Uploaded 36GiB to GPU in 2.37904s (15.1321GiB/s) | 20260508 09:47:00.230 | INFO | Downloaded 34GiB to GPU in 2.43539s (13.9608GiB/s) | 20260508 09:47:05.364 | INFO | Analyzing chunk frames = [7142400,7142912) | 20260508 09:47:05.364 | INFO | MemInfo: { | 20260508 09:47:05.364 | INFO | "gpu" : " 17 GB", | 20260508 09:47:05.364 | INFO | "huge" : "", | 20260508 09:47:05.364 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:05.364 | INFO | "malloc" : " 14 GB", | 20260508 09:47:05.364 | INFO | "pinned" : " 29 GB" | 20260508 09:47:05.364 | INFO | } | 20260508 09:47:05.364 | INFO | Perf report for the last 5.13475 seconds: | 20260508 09:47:05.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:05.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 09:47:05.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:47:05.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:05.364 | INFO | Uploaded 36GiB to GPU in 2.43697s (14.7725GiB/s) | 20260508 09:47:05.364 | INFO | Downloaded 34GiB to GPU in 2.43929s (13.9385GiB/s) | 20260508 09:47:10.510 | INFO | Analyzing chunk frames = [7142912,7143424) | 20260508 09:47:10.510 | INFO | MemInfo: { | 20260508 09:47:10.510 | INFO | "gpu" : " 17 GB", | 20260508 09:47:10.510 | INFO | "huge" : "", | 20260508 09:47:10.510 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:10.510 | INFO | "malloc" : " 14 GB", | 20260508 09:47:10.510 | INFO | "pinned" : " 29 GB" | 20260508 09:47:10.510 | INFO | } | 20260508 09:47:10.510 | INFO | Perf report for the last 5.14594 seconds: | 20260508 09:47:10.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:10.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | ANALYSIS : 65.7%, 3.0, 101.7ms, 0 | 20260508 09:47:10.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | PRE_HQ : 23.5%, 1.0, 12.1ms, 0 | 20260508 09:47:10.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:10.510 | INFO | Uploaded 36GiB to GPU in 2.44988s (14.6946GiB/s) | 20260508 09:47:10.510 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:47:10.510 | INFO | Total mem usage: 16.7 GiB | 20260508 09:47:10.510 | INFO | Downloaded 34GiB to GPU in 2.4603s (13.8195GiB/s) | 20260508 09:47:10.510 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:47:10.510 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:47:10.510 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:47:10.510 | INFO | High Water Breakdown: | 20260508 09:47:10.510 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:47:10.510 | INFO | 40.3% : 5.78 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:47:10.510 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:47:10.510 | INFO | ------------------------------------------ | 20260508 09:47:10.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:47:10.510 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:47:10.510 | INFO | Total mem usage: 26.5 GiB | 20260508 09:47:10.510 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:47:10.510 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:47:10.510 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:47:10.510 | INFO | High Water Breakdown: | 20260508 09:47:10.510 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:47:10.510 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:47:10.510 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:47:10.510 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:47:10.510 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:47:10.510 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:47:10.510 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:47:10.510 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:47:10.510 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:47:10.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:47:10.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:47:10.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:47:10.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:47:10.510 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:47:10.510 | INFO | ------------------------------------------ | 20260508 09:47:10.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:47:10.510 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:47:10.510 | INFO | Total mem usage: 27.8 GiB | 20260508 09:47:10.510 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:47:10.510 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:47:10.510 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:47:10.510 | INFO | High Water Breakdown: | 20260508 09:47:10.510 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:47:10.510 | INFO | ------------------------------------------ | 20260508 09:47:10.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:47:10.510 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:47:10.510 | INFO | Total mem usage: 15.7 GiB | 20260508 09:47:10.510 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:47:10.510 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:47:10.510 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:47:10.510 | INFO | High Water Breakdown: | 20260508 09:47:10.510 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:47:10.510 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:47:10.510 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:47:10.510 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:47:10.510 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:47:10.510 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:47:10.510 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:47:10.510 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:47:10.510 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:47:10.510 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:47:10.510 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:47:10.510 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:47:10.510 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:47:10.510 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:47:10.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:47:10.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:47:10.510 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:47:10.510 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:47:10.510 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:47:10.510 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:47:10.510 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:47:10.510 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:47:10.510 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:47:10.510 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:47:10.511 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:47:10.511 | INFO | ------------------------------------------ | 20260508 09:47:10.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:47:10.511 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:47:10.511 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:47:10.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:47:10.511 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 5.59e-01 | 20260508 09:47:10.511 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.04e-01, 1.10e-02 / 5.94e-03 | 20260508 09:47:10.511 | INFO | PerfReport: 12.14%, Baseline -- 3.97e+00 / 4.42e+00, 5.94e-01 / 2.24e-01 | 20260508 09:47:10.511 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.49e+00, 2.10e-01 / 1.20e-01 | 20260508 09:47:10.511 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.50e+01, 3.74e+00 / 9.63e-01 | 20260508 09:47:10.511 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.85e+00 / 1.25e+00 | 20260508 09:47:10.511 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.53e+00, 4.09e-01 / 2.07e-01 | 20260508 09:47:10.511 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.14e+00, 2.47e-01 / 1.45e-01 | 20260508 09:47:10.511 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.34e-01, 4.90e-02 / 2.69e-02 | 20260508 09:47:10.511 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:47:10.511 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:47:10.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:47:10.512 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 24.34%, Upload -- 2.56e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 31.05%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 2.21%, LockUpload -- 2.33e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 2.54%, LockDownload -- 2.68e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 15.25%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:47:10.512 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 09:47:10.608 | INFO | Starting a round of Baz IO | 20260508 09:47:14.030 | INFO | Populating RTMetrics report | 20260508 09:47:14.031 | INFO | RTMetrics report populated | 20260508 09:47:14.031 | INFO | Opening tmp RTMetrics json file | 20260508 09:47:14.041 | INFO | Serializing RTMetrics to json | 20260508 09:47:14.042 | INFO | Serialization done, swapping tmp file now... | 20260508 09:47:14.043 | INFO | RTMetrics done | 20260508 09:47:14.098 | INFO | Finished a round of Baz IO: Wrote 1.478 GiB in 3.49 seconds | 20260508 09:47:14.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:47:14.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.245273% / 0% / 3.57628e-05% | 20260508 09:47:14.098 | INFO | Baz IO overall: | 20260508 09:47:14.098 | INFO | BytesWritten : 3111509303082 | 20260508 09:47:14.098 | INFO | NumEvents : 1496403564526 | 20260508 09:47:15.651 | INFO | Analyzing chunk frames = [7143424,7143936) | 20260508 09:47:15.651 | INFO | MemInfo: { | 20260508 09:47:15.651 | INFO | "gpu" : " 17 GB", | 20260508 09:47:15.651 | INFO | "huge" : "", | 20260508 09:47:15.651 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:15.651 | INFO | "malloc" : " 14 GB", | 20260508 09:47:15.651 | INFO | "pinned" : " 29 GB" | 20260508 09:47:15.651 | INFO | } | 20260508 09:47:15.651 | INFO | Perf report for the last 5.14111 seconds: | 20260508 09:47:15.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:15.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:15.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:15.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:15.651 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:47:15.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:15.651 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:47:15.651 | INFO | PRE_HQ : 65.5%, 1.6, 54.8ms, 0 | 20260508 09:47:15.651 | INFO | BAZWRITER : 67.2%, 2.5, 87.6ms, 0 | 20260508 09:47:15.651 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 09:47:15.652 | INFO | Uploaded 36GiB to GPU in 2.57344s (13.989GiB/s) | 20260508 09:47:15.652 | INFO | Downloaded 38GiB to GPU in 2.70771s (14.034GiB/s) | 20260508 09:47:20.791 | INFO | Analyzing chunk frames = [7143936,7144448) | 20260508 09:47:20.791 | INFO | MemInfo: { | 20260508 09:47:20.791 | INFO | "gpu" : " 17 GB", | 20260508 09:47:20.791 | INFO | "huge" : "", | 20260508 09:47:20.791 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:20.791 | INFO | "malloc" : " 14 GB", | 20260508 09:47:20.791 | INFO | "pinned" : " 29 GB" | 20260508 09:47:20.791 | INFO | } | 20260508 09:47:20.791 | INFO | Perf report for the last 5.14029 seconds: | 20260508 09:47:20.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:20.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.791 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:47:20.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.792 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:47:20.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:20.792 | INFO | Uploaded 36GiB to GPU in 2.48152s (14.5072GiB/s) | 20260508 09:47:20.792 | INFO | Downloaded 34GiB to GPU in 2.47021s (13.764GiB/s) | 20260508 09:47:25.925 | INFO | Analyzing chunk frames = [7144448,7144960) | 20260508 09:47:25.925 | INFO | MemInfo: { | 20260508 09:47:25.925 | INFO | "gpu" : " 17 GB", | 20260508 09:47:25.925 | INFO | "huge" : "", | 20260508 09:47:25.925 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:25.925 | INFO | "malloc" : " 14 GB", | 20260508 09:47:25.925 | INFO | "pinned" : " 29 GB" | 20260508 09:47:25.925 | INFO | } | 20260508 09:47:25.925 | INFO | Perf report for the last 5.13369 seconds: | 20260508 09:47:25.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:25.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.925 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 09:47:25.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.925 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:47:25.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:25.926 | INFO | Uploaded 36GiB to GPU in 2.44346s (14.7332GiB/s) | 20260508 09:47:25.926 | INFO | Downloaded 34GiB to GPU in 2.42497s (14.0208GiB/s) | 20260508 09:47:31.067 | INFO | Analyzing chunk frames = [7144960,7145472) | 20260508 09:47:31.067 | INFO | MemInfo: { | 20260508 09:47:31.067 | INFO | "gpu" : " 17 GB", | 20260508 09:47:31.067 | INFO | "huge" : "", | 20260508 09:47:31.067 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:31.067 | INFO | "malloc" : " 14 GB", | 20260508 09:47:31.068 | INFO | "pinned" : " 29 GB" | 20260508 09:47:31.068 | INFO | } | 20260508 09:47:31.068 | INFO | Perf report for the last 5.14256 seconds: | 20260508 09:47:31.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:31.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 09:47:31.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:47:31.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:31.068 | INFO | Uploaded 36GiB to GPU in 2.45529s (14.6622GiB/s) | 20260508 09:47:31.068 | INFO | Downloaded 34GiB to GPU in 2.4368s (13.9527GiB/s) | 20260508 09:47:36.205 | INFO | Analyzing chunk frames = [7145472,7145984) | 20260508 09:47:36.205 | INFO | MemInfo: { | 20260508 09:47:36.205 | INFO | "gpu" : " 17 GB", | 20260508 09:47:36.205 | INFO | "huge" : "", | 20260508 09:47:36.205 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:36.205 | INFO | "malloc" : " 14 GB", | 20260508 09:47:36.205 | INFO | "pinned" : " 29 GB" | 20260508 09:47:36.205 | INFO | } | 20260508 09:47:36.205 | INFO | Perf report for the last 5.13755 seconds: | 20260508 09:47:36.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:36.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.205 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 09:47:36.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.205 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:47:36.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:36.206 | INFO | Uploaded 36GiB to GPU in 2.39494s (15.0317GiB/s) | 20260508 09:47:36.206 | INFO | Downloaded 34GiB to GPU in 2.53174s (13.4295GiB/s) | 20260508 09:47:41.346 | INFO | Analyzing chunk frames = [7145984,7146496) | 20260508 09:47:41.346 | INFO | MemInfo: { | 20260508 09:47:41.346 | INFO | "gpu" : " 17 GB", | 20260508 09:47:41.346 | INFO | "huge" : "", | 20260508 09:47:41.346 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:41.346 | INFO | "malloc" : " 14 GB", | 20260508 09:47:41.346 | INFO | "pinned" : " 29 GB" | 20260508 09:47:41.346 | INFO | } | 20260508 09:47:41.347 | INFO | Perf report for the last 5.1413 seconds: | 20260508 09:47:41.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:41.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 09:47:41.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:47:41.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:41.347 | INFO | Uploaded 36GiB to GPU in 2.36646s (15.2126GiB/s) | 20260508 09:47:41.347 | INFO | Downloaded 34GiB to GPU in 2.45214s (13.8654GiB/s) | 20260508 09:47:46.484 | INFO | Analyzing chunk frames = [7146496,7147008) | 20260508 09:47:46.484 | INFO | MemInfo: { | 20260508 09:47:46.484 | INFO | "gpu" : " 17 GB", | 20260508 09:47:46.484 | INFO | "huge" : "", | 20260508 09:47:46.484 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:46.484 | INFO | "malloc" : " 14 GB", | 20260508 09:47:46.484 | INFO | "pinned" : " 29 GB" | 20260508 09:47:46.484 | INFO | } | 20260508 09:47:46.484 | INFO | Perf report for the last 5.1374 seconds: | 20260508 09:47:46.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:46.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:47:46.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:47:46.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:46.484 | INFO | Uploaded 36GiB to GPU in 2.37262s (15.1731GiB/s) | 20260508 09:47:46.484 | INFO | Downloaded 34GiB to GPU in 2.44813s (13.8882GiB/s) | 20260508 09:47:51.623 | INFO | Analyzing chunk frames = [7147008,7147520) | 20260508 09:47:51.623 | INFO | MemInfo: { | 20260508 09:47:51.623 | INFO | "gpu" : " 17 GB", | 20260508 09:47:51.623 | INFO | "huge" : "", | 20260508 09:47:51.623 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:51.623 | INFO | "malloc" : " 14 GB", | 20260508 09:47:51.623 | INFO | "pinned" : " 29 GB" | 20260508 09:47:51.623 | INFO | } | 20260508 09:47:51.623 | INFO | Perf report for the last 5.13897 seconds: | 20260508 09:47:51.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:51.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | ANALYSIS : 65.3%, 3.0, 102.1ms, 0 | 20260508 09:47:51.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:47:51.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:51.623 | INFO | Uploaded 36GiB to GPU in 2.37121s (15.1821GiB/s) | 20260508 09:47:51.623 | INFO | Downloaded 34GiB to GPU in 2.45765s (13.8343GiB/s) | 20260508 09:47:51.720 | INFO | Starting a round of Baz IO | 20260508 09:47:55.134 | INFO | Populating RTMetrics report | 20260508 09:47:55.134 | INFO | RTMetrics report populated | 20260508 09:47:55.134 | INFO | Opening tmp RTMetrics json file | 20260508 09:47:55.145 | INFO | Serializing RTMetrics to json | 20260508 09:47:55.147 | INFO | Serialization done, swapping tmp file now... | 20260508 09:47:55.147 | INFO | RTMetrics done | 20260508 09:47:55.208 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.488 seconds | 20260508 09:47:55.208 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:47:55.208 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.243974% / 0% / 5.5631e-05% | 20260508 09:47:55.208 | INFO | Baz IO overall: | 20260508 09:47:55.208 | INFO | BytesWritten : 3113094490954 | 20260508 09:47:55.208 | INFO | NumEvents : 1497152743184 | 20260508 09:47:56.770 | INFO | Analyzing chunk frames = [7147520,7148032) | 20260508 09:47:56.770 | INFO | MemInfo: { | 20260508 09:47:56.770 | INFO | "gpu" : " 17 GB", | 20260508 09:47:56.770 | INFO | "huge" : "", | 20260508 09:47:56.770 | INFO | "huge_shared" : " 29 GB", | 20260508 09:47:56.770 | INFO | "malloc" : " 14 GB", | 20260508 09:47:56.770 | INFO | "pinned" : " 29 GB" | 20260508 09:47:56.770 | INFO | } | 20260508 09:47:56.770 | INFO | Perf report for the last 5.14736 seconds: | 20260508 09:47:56.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:47:56.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:56.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:56.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:56.770 | INFO | ANALYSIS : 67.5%, 3.0, 105.7ms, 0 | 20260508 09:47:56.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:47:56.770 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:47:56.770 | INFO | PRE_HQ : 65.3%, 1.6, 54.3ms, 0 | 20260508 09:47:56.770 | INFO | BAZWRITER : 67.2%, 2.5, 87.8ms, 0 | 20260508 09:47:56.771 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:47:56.771 | INFO | Uploaded 36GiB to GPU in 2.5868s (13.9168GiB/s) | 20260508 09:47:56.771 | INFO | Downloaded 38GiB to GPU in 2.71556s (13.9934GiB/s) | 20260508 09:48:01.904 | INFO | Analyzing chunk frames = [7148032,7148544) | 20260508 09:48:01.904 | INFO | MemInfo: { | 20260508 09:48:01.904 | INFO | "gpu" : " 17 GB", | 20260508 09:48:01.904 | INFO | "huge" : "", | 20260508 09:48:01.904 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:01.904 | INFO | "malloc" : " 14 GB", | 20260508 09:48:01.904 | INFO | "pinned" : " 29 GB" | 20260508 09:48:01.904 | INFO | } | 20260508 09:48:01.904 | INFO | Perf report for the last 5.1337 seconds: | 20260508 09:48:01.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:01.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 09:48:01.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:48:01.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:01.904 | INFO | Uploaded 36GiB to GPU in 2.47145s (14.5663GiB/s) | 20260508 09:48:01.904 | INFO | Downloaded 34GiB to GPU in 2.4716s (13.7563GiB/s) | 20260508 09:48:07.042 | INFO | Analyzing chunk frames = [7148544,7149056) | 20260508 09:48:07.042 | INFO | MemInfo: { | 20260508 09:48:07.042 | INFO | "gpu" : " 17 GB", | 20260508 09:48:07.042 | INFO | "huge" : "", | 20260508 09:48:07.042 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:07.042 | INFO | "malloc" : " 14 GB", | 20260508 09:48:07.042 | INFO | "pinned" : " 29 GB" | 20260508 09:48:07.042 | INFO | } | 20260508 09:48:07.042 | INFO | Perf report for the last 5.13807 seconds: | 20260508 09:48:07.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:07.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:48:07.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:48:07.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:07.042 | INFO | Uploaded 36GiB to GPU in 2.46624s (14.5971GiB/s) | 20260508 09:48:07.043 | INFO | Downloaded 34GiB to GPU in 2.44113s (13.928GiB/s) | 20260508 09:48:12.179 | INFO | Analyzing chunk frames = [7149056,7149568) | 20260508 09:48:12.179 | INFO | MemInfo: { | 20260508 09:48:12.179 | INFO | "gpu" : " 17 GB", | 20260508 09:48:12.179 | INFO | "huge" : "", | 20260508 09:48:12.179 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:12.179 | INFO | "malloc" : " 14 GB", | 20260508 09:48:12.179 | INFO | "pinned" : " 29 GB" | 20260508 09:48:12.179 | INFO | } | 20260508 09:48:12.179 | INFO | Perf report for the last 5.13722 seconds: | 20260508 09:48:12.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:12.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.179 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:48:12.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.180 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 09:48:12.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:12.180 | INFO | Uploaded 36GiB to GPU in 2.50088s (14.3949GiB/s) | 20260508 09:48:12.180 | INFO | Downloaded 34GiB to GPU in 2.42544s (14.0181GiB/s) | 20260508 09:48:17.321 | INFO | Analyzing chunk frames = [7149568,7150080) | 20260508 09:48:17.321 | INFO | MemInfo: { | 20260508 09:48:17.321 | INFO | "gpu" : " 17 GB", | 20260508 09:48:17.321 | INFO | "huge" : "", | 20260508 09:48:17.321 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:17.321 | INFO | "malloc" : " 14 GB", | 20260508 09:48:17.321 | INFO | "pinned" : " 29 GB" | 20260508 09:48:17.321 | INFO | } | 20260508 09:48:17.321 | INFO | Perf report for the last 5.14197 seconds: | 20260508 09:48:17.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:17.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.321 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 09:48:17.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.321 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:48:17.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:17.322 | INFO | Uploaded 36GiB to GPU in 2.40419s (14.9738GiB/s) | 20260508 09:48:17.322 | INFO | Downloaded 34GiB to GPU in 2.52278s (13.4772GiB/s) | 20260508 09:48:22.459 | INFO | Analyzing chunk frames = [7150080,7150592) | 20260508 09:48:22.459 | INFO | MemInfo: { | 20260508 09:48:22.459 | INFO | "gpu" : " 17 GB", | 20260508 09:48:22.459 | INFO | "huge" : "", | 20260508 09:48:22.459 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:22.459 | INFO | "malloc" : " 14 GB", | 20260508 09:48:22.459 | INFO | "pinned" : " 29 GB" | 20260508 09:48:22.459 | INFO | } | 20260508 09:48:22.459 | INFO | Perf report for the last 5.13785 seconds: | 20260508 09:48:22.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:22.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.459 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:48:22.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.459 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:48:22.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:22.460 | INFO | Uploaded 36GiB to GPU in 2.3931s (15.0432GiB/s) | 20260508 09:48:22.460 | INFO | Downloaded 34GiB to GPU in 2.42669s (14.0109GiB/s) | 20260508 09:48:27.599 | INFO | Analyzing chunk frames = [7150592,7151104) | 20260508 09:48:27.599 | INFO | MemInfo: { | 20260508 09:48:27.599 | INFO | "gpu" : " 17 GB", | 20260508 09:48:27.599 | INFO | "huge" : "", | 20260508 09:48:27.599 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:27.599 | INFO | "malloc" : " 14 GB", | 20260508 09:48:27.599 | INFO | "pinned" : " 29 GB" | 20260508 09:48:27.599 | INFO | } | 20260508 09:48:27.599 | INFO | Perf report for the last 5.14012 seconds: | 20260508 09:48:27.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:27.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.599 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:48:27.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.599 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:48:27.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:27.600 | INFO | Uploaded 36GiB to GPU in 2.44049s (14.7511GiB/s) | 20260508 09:48:27.600 | INFO | Downloaded 34GiB to GPU in 2.41497s (14.0788GiB/s) | 20260508 09:48:32.739 | INFO | Analyzing chunk frames = [7151104,7151616) | 20260508 09:48:32.739 | INFO | MemInfo: { | 20260508 09:48:32.739 | INFO | "gpu" : " 17 GB", | 20260508 09:48:32.739 | INFO | "huge" : "", | 20260508 09:48:32.739 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:32.739 | INFO | "malloc" : " 14 GB", | 20260508 09:48:32.739 | INFO | "pinned" : " 29 GB" | 20260508 09:48:32.739 | INFO | } | 20260508 09:48:32.739 | INFO | Perf report for the last 5.13967 seconds: | 20260508 09:48:32.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:32.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:48:32.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:48:32.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:32.739 | INFO | Uploaded 36GiB to GPU in 2.42768s (14.829GiB/s) | 20260508 09:48:32.739 | INFO | Downloaded 34GiB to GPU in 2.42127s (14.0422GiB/s) | 20260508 09:48:32.739 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:48:32.740 | INFO | Total mem usage: 16.7 GiB | 20260508 09:48:32.740 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:48:32.740 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:48:32.740 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:48:32.740 | INFO | High Water Breakdown: | 20260508 09:48:32.740 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:48:32.740 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:48:32.740 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:48:32.740 | INFO | ------------------------------------------ | 20260508 09:48:32.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:48:32.740 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:48:32.740 | INFO | Total mem usage: 26.5 GiB | 20260508 09:48:32.740 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:48:32.740 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:48:32.740 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:48:32.740 | INFO | High Water Breakdown: | 20260508 09:48:32.740 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:48:32.740 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:48:32.740 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:48:32.740 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:48:32.740 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:48:32.740 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:48:32.740 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:48:32.740 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:48:32.740 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:48:32.740 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:48:32.740 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:48:32.740 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:48:32.740 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:48:32.740 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:48:32.740 | INFO | ------------------------------------------ | 20260508 09:48:32.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:48:32.740 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:48:32.740 | INFO | Total mem usage: 27.8 GiB | 20260508 09:48:32.740 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:48:32.740 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:48:32.740 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:48:32.740 | INFO | High Water Breakdown: | 20260508 09:48:32.740 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:48:32.740 | INFO | ------------------------------------------ | 20260508 09:48:32.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:48:32.740 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:48:32.740 | INFO | Total mem usage: 15.7 GiB | 20260508 09:48:32.740 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:48:32.741 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:48:32.741 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:48:32.741 | INFO | High Water Breakdown: | 20260508 09:48:32.741 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:48:32.741 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:48:32.741 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:48:32.741 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:48:32.741 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:48:32.741 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:48:32.741 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:48:32.741 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:48:32.741 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:48:32.741 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:48:32.741 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:48:32.741 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:48:32.741 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:48:32.741 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:48:32.741 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:48:32.741 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:48:32.741 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:48:32.741 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:48:32.741 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:48:32.741 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:48:32.741 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:48:32.741 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:48:32.741 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:48:32.741 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:48:32.741 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:48:32.741 | INFO | ------------------------------------------ | 20260508 09:48:32.741 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:48:32.741 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:48:32.741 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:48:32.741 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:48:32.741 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 6.76e-01 | 20260508 09:48:32.741 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.11e-02 / 5.47e-03 | 20260508 09:48:32.741 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.51e+00, 5.92e-01 / 1.80e-01 | 20260508 09:48:32.741 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.22e-01 | 20260508 09:48:32.741 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.50e+01, 3.67e+00 / 1.05e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 1.38e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.07e-01 / 2.51e-01 | 20260508 09:48:32.741 | INFO | PerfReport: 3.03%, Metrics -- 9.93e-01 / 1.15e+00, 2.48e-01 / 1.57e-01 | 20260508 09:48:32.741 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.27e-01, 4.84e-02 / 2.64e-02 | 20260508 09:48:32.741 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:48:32.741 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:48:32.741 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:48:32.741 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 2.11%, LockUpload -- 2.22e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 2.65%, LockDownload -- 2.79e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 15.30%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:48:32.741 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260508 09:48:32.837 | INFO | Starting a round of Baz IO | 20260508 09:48:36.249 | INFO | Populating RTMetrics report | 20260508 09:48:36.249 | INFO | RTMetrics report populated | 20260508 09:48:36.249 | INFO | Opening tmp RTMetrics json file | 20260508 09:48:36.259 | INFO | Serializing RTMetrics to json | 20260508 09:48:36.261 | INFO | Serialization done, swapping tmp file now... | 20260508 09:48:36.261 | INFO | RTMetrics done | 20260508 09:48:36.324 | INFO | Finished a round of Baz IO: Wrote 1.475 GiB in 3.486 seconds | 20260508 09:48:36.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.27% / 0.00% / 59.73% | 20260508 09:48:36.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.24387% / 0% / 4.76837e-05% | 20260508 09:48:36.324 | INFO | Baz IO overall: | 20260508 09:48:36.324 | INFO | BytesWritten : 3114678626921 | 20260508 09:48:36.324 | INFO | NumEvents : 1497901094781 | 20260508 09:48:37.879 | INFO | Analyzing chunk frames = [7151616,7152128) | 20260508 09:48:37.879 | INFO | MemInfo: { | 20260508 09:48:37.879 | INFO | "gpu" : " 17 GB", | 20260508 09:48:37.879 | INFO | "huge" : "", | 20260508 09:48:37.879 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:37.879 | INFO | "malloc" : " 14 GB", | 20260508 09:48:37.879 | INFO | "pinned" : " 29 GB" | 20260508 09:48:37.879 | INFO | } | 20260508 09:48:37.879 | INFO | Perf report for the last 5.14008 seconds: | 20260508 09:48:37.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:37.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:37.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:37.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:37.879 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:48:37.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:37.879 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 09:48:37.879 | INFO | PRE_HQ : 65.6%, 1.6, 54.8ms, 0 | 20260508 09:48:37.879 | INFO | BAZWRITER : 67.2%, 2.6, 88.4ms, 0 | 20260508 09:48:37.879 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:48:37.879 | INFO | Uploaded 36GiB to GPU in 2.59752s (13.8594GiB/s) | 20260508 09:48:37.880 | INFO | Downloaded 38GiB to GPU in 2.70629s (14.0414GiB/s) | 20260508 09:48:43.020 | INFO | Analyzing chunk frames = [7152128,7152640) | 20260508 09:48:43.020 | INFO | MemInfo: { | 20260508 09:48:43.020 | INFO | "gpu" : " 17 GB", | 20260508 09:48:43.020 | INFO | "huge" : "", | 20260508 09:48:43.020 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:43.020 | INFO | "malloc" : " 14 GB", | 20260508 09:48:43.020 | INFO | "pinned" : " 29 GB" | 20260508 09:48:43.020 | INFO | } | 20260508 09:48:43.020 | INFO | Perf report for the last 5.14112 seconds: | 20260508 09:48:43.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:43.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.020 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:48:43.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.020 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:48:43.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:43.021 | INFO | Uploaded 36GiB to GPU in 2.51459s (14.3165GiB/s) | 20260508 09:48:43.021 | INFO | Downloaded 34GiB to GPU in 2.46681s (13.783GiB/s) | 20260508 09:48:48.158 | INFO | Analyzing chunk frames = [7152640,7153152) | 20260508 09:48:48.158 | INFO | MemInfo: { | 20260508 09:48:48.158 | INFO | "gpu" : " 17 GB", | 20260508 09:48:48.158 | INFO | "huge" : "", | 20260508 09:48:48.158 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:48.158 | INFO | "malloc" : " 14 GB", | 20260508 09:48:48.158 | INFO | "pinned" : " 29 GB" | 20260508 09:48:48.158 | INFO | } | 20260508 09:48:48.158 | INFO | Perf report for the last 5.13761 seconds: | 20260508 09:48:48.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:48.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:48:48.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 09:48:48.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:48.158 | INFO | Uploaded 36GiB to GPU in 2.45777s (14.6474GiB/s) | 20260508 09:48:48.158 | INFO | Downloaded 34GiB to GPU in 2.43505s (13.9627GiB/s) | 20260508 09:48:53.302 | INFO | Analyzing chunk frames = [7153152,7153664) | 20260508 09:48:53.302 | INFO | MemInfo: { | 20260508 09:48:53.302 | INFO | "gpu" : " 17 GB", | 20260508 09:48:53.302 | INFO | "huge" : "", | 20260508 09:48:53.302 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:53.302 | INFO | "malloc" : " 14 GB", | 20260508 09:48:53.302 | INFO | "pinned" : " 29 GB" | 20260508 09:48:53.302 | INFO | } | 20260508 09:48:53.302 | INFO | Perf report for the last 5.14387 seconds: | 20260508 09:48:53.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:53.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 09:48:53.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:48:53.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:53.302 | INFO | Uploaded 36GiB to GPU in 2.45185s (14.6828GiB/s) | 20260508 09:48:53.302 | INFO | Downloaded 34GiB to GPU in 2.40916s (14.1128GiB/s) | 20260508 09:48:58.436 | INFO | Analyzing chunk frames = [7153664,7154176) | 20260508 09:48:58.436 | INFO | MemInfo: { | 20260508 09:48:58.436 | INFO | "gpu" : " 17 GB", | 20260508 09:48:58.436 | INFO | "huge" : "", | 20260508 09:48:58.436 | INFO | "huge_shared" : " 29 GB", | 20260508 09:48:58.436 | INFO | "malloc" : " 14 GB", | 20260508 09:48:58.436 | INFO | "pinned" : " 29 GB" | 20260508 09:48:58.436 | INFO | } | 20260508 09:48:58.436 | INFO | Perf report for the last 5.13459 seconds: | 20260508 09:48:58.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:48:58.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.436 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 09:48:58.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.436 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:48:58.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:48:58.437 | INFO | Uploaded 36GiB to GPU in 2.29819s (15.6645GiB/s) | 20260508 09:48:58.437 | INFO | Downloaded 34GiB to GPU in 2.49571s (13.6234GiB/s) | 20260508 09:49:03.576 | INFO | Analyzing chunk frames = [7154176,7154688) | 20260508 09:49:03.576 | INFO | MemInfo: { | 20260508 09:49:03.576 | INFO | "gpu" : " 17 GB", | 20260508 09:49:03.576 | INFO | "huge" : "", | 20260508 09:49:03.576 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:03.576 | INFO | "malloc" : " 14 GB", | 20260508 09:49:03.576 | INFO | "pinned" : " 29 GB" | 20260508 09:49:03.576 | INFO | } | 20260508 09:49:03.576 | INFO | Perf report for the last 5.13997 seconds: | 20260508 09:49:03.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:03.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.576 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 09:49:03.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.576 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:49:03.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:03.577 | INFO | Uploaded 36GiB to GPU in 2.36062s (15.2502GiB/s) | 20260508 09:49:03.577 | INFO | Downloaded 34GiB to GPU in 2.44775s (13.8903GiB/s) | 20260508 09:49:08.714 | INFO | Analyzing chunk frames = [7154688,7155200) | 20260508 09:49:08.714 | INFO | MemInfo: { | 20260508 09:49:08.714 | INFO | "gpu" : " 17 GB", | 20260508 09:49:08.714 | INFO | "huge" : "", | 20260508 09:49:08.715 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:08.715 | INFO | "malloc" : " 14 GB", | 20260508 09:49:08.715 | INFO | "pinned" : " 29 GB" | 20260508 09:49:08.715 | INFO | } | 20260508 09:49:08.715 | INFO | Perf report for the last 5.13834 seconds: | 20260508 09:49:08.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:08.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 09:49:08.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:49:08.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:08.715 | INFO | Uploaded 36GiB to GPU in 2.35763s (15.2696GiB/s) | 20260508 09:49:08.715 | INFO | Downloaded 34GiB to GPU in 2.44763s (13.891GiB/s) | 20260508 09:49:13.855 | INFO | Analyzing chunk frames = [7155200,7155712) | 20260508 09:49:13.855 | INFO | MemInfo: { | 20260508 09:49:13.855 | INFO | "gpu" : " 17 GB", | 20260508 09:49:13.855 | INFO | "huge" : "", | 20260508 09:49:13.855 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:13.855 | INFO | "malloc" : " 14 GB", | 20260508 09:49:13.855 | INFO | "pinned" : " 29 GB" | 20260508 09:49:13.855 | INFO | } | 20260508 09:49:13.855 | INFO | Perf report for the last 5.14007 seconds: | 20260508 09:49:13.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:13.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 09:49:13.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:49:13.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:13.855 | INFO | Uploaded 36GiB to GPU in 2.4056s (14.9651GiB/s) | 20260508 09:49:13.855 | INFO | Downloaded 34GiB to GPU in 2.45037s (13.8754GiB/s) | 20260508 09:49:13.953 | INFO | Starting a round of Baz IO | 20260508 09:49:17.363 | INFO | Populating RTMetrics report | 20260508 09:49:17.364 | INFO | RTMetrics report populated | 20260508 09:49:17.364 | INFO | Opening tmp RTMetrics json file | 20260508 09:49:17.372 | INFO | Serializing RTMetrics to json | 20260508 09:49:17.373 | INFO | Serialization done, swapping tmp file now... | 20260508 09:49:17.374 | INFO | RTMetrics done | 20260508 09:49:17.430 | INFO | Finished a round of Baz IO: Wrote 1.476 GiB in 3.477 seconds | 20260508 09:49:17.430 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.73% | 20260508 09:49:17.430 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.243211% / 0% / 5.5631e-05% | 20260508 09:49:17.431 | INFO | Baz IO overall: | 20260508 09:49:17.431 | INFO | BytesWritten : 3116263321987 | 20260508 09:49:17.431 | INFO | NumEvents : 1498650329887 | 20260508 09:49:18.993 | INFO | Analyzing chunk frames = [7155712,7156224) | 20260508 09:49:18.993 | INFO | MemInfo: { | 20260508 09:49:18.993 | INFO | "gpu" : " 17 GB", | 20260508 09:49:18.993 | INFO | "huge" : "", | 20260508 09:49:18.993 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:18.993 | INFO | "malloc" : " 14 GB", | 20260508 09:49:18.993 | INFO | "pinned" : " 29 GB" | 20260508 09:49:18.993 | INFO | } | 20260508 09:49:18.993 | INFO | Perf report for the last 5.13834 seconds: | 20260508 09:49:18.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:18.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:18.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:18.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:18.993 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 09:49:18.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:18.993 | INFO | RT_METRICS : 40.0%, 1.0, 20.9ms, 0 | 20260508 09:49:18.993 | INFO | PRE_HQ : 65.6%, 1.7, 56.4ms, 0 | 20260508 09:49:18.993 | INFO | BAZWRITER : 67.0%, 2.9, 98.7ms, 0 | 20260508 09:49:18.993 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:49:18.993 | INFO | Uploaded 36GiB to GPU in 2.57694s (13.9701GiB/s) | 20260508 09:49:18.993 | INFO | Downloaded 38GiB to GPU in 2.70312s (14.0578GiB/s) | 20260508 09:49:24.128 | INFO | Analyzing chunk frames = [7156224,7156736) | 20260508 09:49:24.128 | INFO | MemInfo: { | 20260508 09:49:24.128 | INFO | "gpu" : " 17 GB", | 20260508 09:49:24.128 | INFO | "huge" : "", | 20260508 09:49:24.128 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:24.128 | INFO | "malloc" : " 14 GB", | 20260508 09:49:24.129 | INFO | "pinned" : " 29 GB" | 20260508 09:49:24.129 | INFO | } | 20260508 09:49:24.129 | INFO | Perf report for the last 5.13553 seconds: | 20260508 09:49:24.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:24.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 09:49:24.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:49:24.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:24.129 | INFO | Uploaded 36GiB to GPU in 2.4965s (14.4202GiB/s) | 20260508 09:49:24.129 | INFO | Downloaded 34GiB to GPU in 2.47862s (13.7173GiB/s) | 20260508 09:49:29.273 | INFO | Analyzing chunk frames = [7156736,7157248) | 20260508 09:49:29.273 | INFO | MemInfo: { | 20260508 09:49:29.273 | INFO | "gpu" : " 17 GB", | 20260508 09:49:29.273 | INFO | "huge" : "", | 20260508 09:49:29.273 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:29.273 | INFO | "malloc" : " 14 GB", | 20260508 09:49:29.273 | INFO | "pinned" : " 29 GB" | 20260508 09:49:29.273 | INFO | } | 20260508 09:49:29.273 | INFO | Perf report for the last 5.14459 seconds: | 20260508 09:49:29.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:29.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.273 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 09:49:29.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.273 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:49:29.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:29.274 | INFO | Uploaded 36GiB to GPU in 2.44809s (14.7053GiB/s) | 20260508 09:49:29.274 | INFO | Downloaded 34GiB to GPU in 2.4421s (13.9225GiB/s) | 20260508 09:49:34.415 | INFO | Analyzing chunk frames = [7157248,7157760) | 20260508 09:49:34.415 | INFO | MemInfo: { | 20260508 09:49:34.415 | INFO | "gpu" : " 17 GB", | 20260508 09:49:34.415 | INFO | "huge" : "", | 20260508 09:49:34.415 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:34.415 | INFO | "malloc" : " 14 GB", | 20260508 09:49:34.416 | INFO | "pinned" : " 29 GB" | 20260508 09:49:34.416 | INFO | } | 20260508 09:49:34.416 | INFO | Perf report for the last 5.14247 seconds: | 20260508 09:49:34.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:34.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 09:49:34.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:49:34.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:34.416 | INFO | Uploaded 36GiB to GPU in 2.42049s (14.873GiB/s) | 20260508 09:49:34.416 | INFO | Downloaded 34GiB to GPU in 2.40632s (14.1295GiB/s) | 20260508 09:49:39.551 | INFO | Analyzing chunk frames = [7157760,7158272) | 20260508 09:49:39.551 | INFO | MemInfo: { | 20260508 09:49:39.551 | INFO | "gpu" : " 17 GB", | 20260508 09:49:39.551 | INFO | "huge" : "", | 20260508 09:49:39.551 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:39.551 | INFO | "malloc" : " 14 GB", | 20260508 09:49:39.551 | INFO | "pinned" : " 29 GB" | 20260508 09:49:39.551 | INFO | } | 20260508 09:49:39.551 | INFO | Perf report for the last 5.13536 seconds: | 20260508 09:49:39.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:39.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 09:49:39.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:49:39.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:39.551 | INFO | Uploaded 36GiB to GPU in 2.36772s (15.2045GiB/s) | 20260508 09:49:39.551 | INFO | Downloaded 34GiB to GPU in 2.51295s (13.5299GiB/s) | 20260508 09:49:44.687 | INFO | Analyzing chunk frames = [7158272,7158784) | 20260508 09:49:44.687 | INFO | MemInfo: { | 20260508 09:49:44.687 | INFO | "gpu" : " 17 GB", | 20260508 09:49:44.687 | INFO | "huge" : "", | 20260508 09:49:44.687 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:44.687 | INFO | "malloc" : " 14 GB", | 20260508 09:49:44.687 | INFO | "pinned" : " 29 GB" | 20260508 09:49:44.687 | INFO | } | 20260508 09:49:44.687 | INFO | Perf report for the last 5.13584 seconds: | 20260508 09:49:44.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:44.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:49:44.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:49:44.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:44.687 | INFO | Uploaded 36GiB to GPU in 2.37049s (15.1867GiB/s) | 20260508 09:49:44.687 | INFO | Downloaded 34GiB to GPU in 2.44222s (13.9218GiB/s) | 20260508 09:49:49.830 | INFO | Analyzing chunk frames = [7158784,7159296) | 20260508 09:49:49.830 | INFO | MemInfo: { | 20260508 09:49:49.830 | INFO | "gpu" : " 17 GB", | 20260508 09:49:49.830 | INFO | "huge" : "", | 20260508 09:49:49.830 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:49.830 | INFO | "malloc" : " 14 GB", | 20260508 09:49:49.830 | INFO | "pinned" : " 29 GB" | 20260508 09:49:49.830 | INFO | } | 20260508 09:49:49.830 | INFO | Perf report for the last 5.14289 seconds: | 20260508 09:49:49.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:49.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:49:49.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:49:49.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:49.830 | INFO | Uploaded 36GiB to GPU in 2.40642s (14.96GiB/s) | 20260508 09:49:49.830 | INFO | Downloaded 34GiB to GPU in 2.44308s (13.9169GiB/s) | 20260508 09:49:54.973 | INFO | Analyzing chunk frames = [7159296,7159808) | 20260508 09:49:54.973 | INFO | MemInfo: { | 20260508 09:49:54.973 | INFO | "gpu" : " 17 GB", | 20260508 09:49:54.973 | INFO | "huge" : "", | 20260508 09:49:54.973 | INFO | "huge_shared" : " 29 GB", | 20260508 09:49:54.973 | INFO | "malloc" : " 14 GB", | 20260508 09:49:54.973 | INFO | "pinned" : " 29 GB" | 20260508 09:49:54.973 | INFO | } | 20260508 09:49:54.973 | INFO | Perf report for the last 5.14328 seconds: | 20260508 09:49:54.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:49:54.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 09:49:54.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:49:54.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:49:54.973 | INFO | Uploaded 36GiB to GPU in 2.41006s (14.9374GiB/s) | 20260508 09:49:54.973 | INFO | Downloaded 34GiB to GPU in 2.44057s (13.9312GiB/s) | 20260508 09:49:54.974 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:49:54.974 | INFO | Total mem usage: 16.7 GiB | 20260508 09:49:54.974 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:49:54.974 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.6%) | 20260508 09:49:54.974 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:49:54.974 | INFO | High Water Breakdown: | 20260508 09:49:54.974 | INFO | 59.8% : 8.71 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:49:54.974 | INFO | 40% : 5.83 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:49:54.974 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:49:54.974 | INFO | ------------------------------------------ | 20260508 09:49:54.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:49:54.974 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:49:54.974 | INFO | Total mem usage: 26.5 GiB | 20260508 09:49:54.974 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:49:54.974 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:49:54.974 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:49:54.974 | INFO | High Water Breakdown: | 20260508 09:49:54.974 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:49:54.974 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:49:54.974 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:49:54.974 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:49:54.974 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:49:54.974 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:49:54.974 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:49:54.974 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:49:54.974 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:49:54.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:49:54.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:49:54.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:49:54.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:49:54.974 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:49:54.974 | INFO | ------------------------------------------ | 20260508 09:49:54.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:49:54.974 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:49:54.974 | INFO | Total mem usage: 27.8 GiB | 20260508 09:49:54.974 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:49:54.974 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:49:54.974 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:49:54.974 | INFO | High Water Breakdown: | 20260508 09:49:54.974 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:49:54.974 | INFO | ------------------------------------------ | 20260508 09:49:54.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:49:54.974 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:49:54.974 | INFO | Total mem usage: 15.7 GiB | 20260508 09:49:54.974 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:49:54.974 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:49:54.974 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:49:54.974 | INFO | High Water Breakdown: | 20260508 09:49:54.974 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:49:54.974 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:49:54.974 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:49:54.974 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:49:54.974 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:49:54.974 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:49:54.974 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:49:54.974 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:49:54.975 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:49:54.975 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:49:54.975 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:49:54.975 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:49:54.975 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:49:54.975 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:49:54.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:49:54.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:49:54.975 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:49:54.975 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:49:54.975 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:49:54.975 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:49:54.975 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:49:54.975 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:49:54.975 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:49:54.975 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:49:54.975 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:49:54.975 | INFO | ------------------------------------------ | 20260508 09:49:54.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:49:54.975 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:49:54.975 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:49:54.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:49:54.975 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 6.28e-01 | 20260508 09:49:54.975 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.05e-01, 1.12e-02 / 5.58e-03 | 20260508 09:49:54.975 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.46e+00, 5.97e-01 / 2.10e-01 | 20260508 09:49:54.975 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.11e-01 / 1.24e-01 | 20260508 09:49:54.975 | INFO | PerfReport: 16.02%, DME -- 4.20e+01 / 4.52e+01, 3.59e+00 / 1.17e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.83e+00 / 1.40e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.06e-01 / 2.88e-01 | 20260508 09:49:54.975 | INFO | PerfReport: 3.04%, Metrics -- 9.94e-01 / 1.13e+00, 2.45e-01 / 1.21e-01 | 20260508 09:49:54.975 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.32e-01, 4.90e-02 / 2.74e-02 | 20260508 09:49:54.975 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:49:54.975 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:49:54.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:49:54.975 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 24.10%, Upload -- 2.54e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 2.09%, LockUpload -- 2.20e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 2.65%, LockDownload -- 2.79e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 15.43%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 09:49:54.975 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 3.59e-02 / 0.00e+00 | 20260508 09:49:55.070 | INFO | Starting a round of Baz IO | 20260508 09:49:58.484 | INFO | Populating RTMetrics report | 20260508 09:49:58.485 | INFO | RTMetrics report populated | 20260508 09:49:58.485 | INFO | Opening tmp RTMetrics json file | 20260508 09:49:58.496 | INFO | Serializing RTMetrics to json | 20260508 09:49:58.497 | INFO | Serialization done, swapping tmp file now... | 20260508 09:49:58.498 | INFO | RTMetrics done | 20260508 09:49:58.554 | INFO | Finished a round of Baz IO: Wrote 1.475 GiB in 3.484 seconds | 20260508 09:49:58.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:49:58.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.24273% / 0% / 6.35783e-05% | 20260508 09:49:58.554 | INFO | Baz IO overall: | 20260508 09:49:58.554 | INFO | BytesWritten : 3117847075525 | 20260508 09:49:58.554 | INFO | NumEvents : 1499398752236 | 20260508 09:50:00.111 | INFO | Analyzing chunk frames = [7159808,7160320) | 20260508 09:50:00.111 | INFO | MemInfo: { | 20260508 09:50:00.111 | INFO | "gpu" : " 17 GB", | 20260508 09:50:00.111 | INFO | "huge" : "", | 20260508 09:50:00.111 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:00.111 | INFO | "malloc" : " 14 GB", | 20260508 09:50:00.111 | INFO | "pinned" : " 29 GB" | 20260508 09:50:00.111 | INFO | } | 20260508 09:50:00.111 | INFO | Perf report for the last 5.13798 seconds: | 20260508 09:50:00.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:00.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:00.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:00.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:00.111 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:50:00.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:00.111 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:50:00.111 | INFO | PRE_HQ : 65.4%, 1.6, 54.2ms, 0 | 20260508 09:50:00.111 | INFO | BAZWRITER : 67.1%, 2.5, 87.0ms, 0 | 20260508 09:50:00.111 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:50:00.112 | INFO | Uploaded 36GiB to GPU in 2.60143s (13.8386GiB/s) | 20260508 09:50:00.112 | INFO | Downloaded 38GiB to GPU in 2.70908s (14.0269GiB/s) | 20260508 09:50:05.248 | INFO | Analyzing chunk frames = [7160320,7160832) | 20260508 09:50:05.248 | INFO | MemInfo: { | 20260508 09:50:05.248 | INFO | "gpu" : " 17 GB", | 20260508 09:50:05.248 | INFO | "huge" : "", | 20260508 09:50:05.248 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:05.248 | INFO | "malloc" : " 14 GB", | 20260508 09:50:05.248 | INFO | "pinned" : " 29 GB" | 20260508 09:50:05.248 | INFO | } | 20260508 09:50:05.248 | INFO | Perf report for the last 5.13723 seconds: | 20260508 09:50:05.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:05.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.248 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 09:50:05.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.249 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:50:05.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:05.249 | INFO | Uploaded 36GiB to GPU in 2.5131s (14.325GiB/s) | 20260508 09:50:05.249 | INFO | Downloaded 34GiB to GPU in 2.46893s (13.7712GiB/s) | 20260508 09:50:10.384 | INFO | Analyzing chunk frames = [7160832,7161344) | 20260508 09:50:10.384 | INFO | MemInfo: { | 20260508 09:50:10.384 | INFO | "gpu" : " 17 GB", | 20260508 09:50:10.384 | INFO | "huge" : "", | 20260508 09:50:10.384 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:10.384 | INFO | "malloc" : " 14 GB", | 20260508 09:50:10.384 | INFO | "pinned" : " 29 GB" | 20260508 09:50:10.384 | INFO | } | 20260508 09:50:10.384 | INFO | Perf report for the last 5.1358 seconds: | 20260508 09:50:10.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:10.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.384 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 09:50:10.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.384 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:50:10.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:10.385 | INFO | Uploaded 36GiB to GPU in 2.44854s (14.7027GiB/s) | 20260508 09:50:10.385 | INFO | Downloaded 34GiB to GPU in 2.42486s (14.0215GiB/s) | 20260508 09:50:15.526 | INFO | Analyzing chunk frames = [7161344,7161856) | 20260508 09:50:15.526 | INFO | MemInfo: { | 20260508 09:50:15.526 | INFO | "gpu" : " 17 GB", | 20260508 09:50:15.526 | INFO | "huge" : "", | 20260508 09:50:15.526 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:15.526 | INFO | "malloc" : " 14 GB", | 20260508 09:50:15.526 | INFO | "pinned" : " 29 GB" | 20260508 09:50:15.526 | INFO | } | 20260508 09:50:15.526 | INFO | Perf report for the last 5.1419 seconds: | 20260508 09:50:15.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:15.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 09:50:15.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:50:15.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:15.526 | INFO | Uploaded 36GiB to GPU in 2.45357s (14.6725GiB/s) | 20260508 09:50:15.527 | INFO | Downloaded 34GiB to GPU in 2.40369s (14.1449GiB/s) | 20260508 09:50:20.665 | INFO | Analyzing chunk frames = [7161856,7162368) | 20260508 09:50:20.665 | INFO | MemInfo: { | 20260508 09:50:20.665 | INFO | "gpu" : " 17 GB", | 20260508 09:50:20.665 | INFO | "huge" : "", | 20260508 09:50:20.665 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:20.665 | INFO | "malloc" : " 14 GB", | 20260508 09:50:20.665 | INFO | "pinned" : " 29 GB" | 20260508 09:50:20.665 | INFO | } | 20260508 09:50:20.665 | INFO | Perf report for the last 5.13935 seconds: | 20260508 09:50:20.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:20.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.666 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 09:50:20.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.666 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:50:20.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:20.666 | INFO | Uploaded 36GiB to GPU in 2.3994s (15.0038GiB/s) | 20260508 09:50:20.666 | INFO | Downloaded 34GiB to GPU in 2.5368s (13.4027GiB/s) | 20260508 09:50:25.801 | INFO | Analyzing chunk frames = [7162368,7162880) | 20260508 09:50:25.801 | INFO | MemInfo: { | 20260508 09:50:25.801 | INFO | "gpu" : " 17 GB", | 20260508 09:50:25.801 | INFO | "huge" : "", | 20260508 09:50:25.801 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:25.801 | INFO | "malloc" : " 14 GB", | 20260508 09:50:25.801 | INFO | "pinned" : " 29 GB" | 20260508 09:50:25.801 | INFO | } | 20260508 09:50:25.801 | INFO | Perf report for the last 5.13599 seconds: | 20260508 09:50:25.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:25.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:50:25.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 09:50:25.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:25.802 | INFO | Uploaded 36GiB to GPU in 2.34624s (15.3437GiB/s) | 20260508 09:50:25.802 | INFO | Downloaded 34GiB to GPU in 2.44151s (13.9258GiB/s) | 20260508 09:50:30.944 | INFO | Analyzing chunk frames = [7162880,7163392) | 20260508 09:50:30.944 | INFO | MemInfo: { | 20260508 09:50:30.944 | INFO | "gpu" : " 17 GB", | 20260508 09:50:30.944 | INFO | "huge" : "", | 20260508 09:50:30.944 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:30.944 | INFO | "malloc" : " 14 GB", | 20260508 09:50:30.944 | INFO | "pinned" : " 29 GB" | 20260508 09:50:30.944 | INFO | } | 20260508 09:50:30.944 | INFO | Perf report for the last 5.14291 seconds: | 20260508 09:50:30.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:30.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 09:50:30.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:50:30.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:30.945 | INFO | Uploaded 36GiB to GPU in 2.37226s (15.1754GiB/s) | 20260508 09:50:30.945 | INFO | Downloaded 34GiB to GPU in 2.45021s (13.8764GiB/s) | 20260508 09:50:36.085 | INFO | Analyzing chunk frames = [7163392,7163904) | 20260508 09:50:36.085 | INFO | MemInfo: { | 20260508 09:50:36.085 | INFO | "gpu" : " 17 GB", | 20260508 09:50:36.085 | INFO | "huge" : "", | 20260508 09:50:36.085 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:36.085 | INFO | "malloc" : " 14 GB", | 20260508 09:50:36.085 | INFO | "pinned" : " 29 GB" | 20260508 09:50:36.085 | INFO | } | 20260508 09:50:36.085 | INFO | Perf report for the last 5.14051 seconds: | 20260508 09:50:36.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:36.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.085 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 09:50:36.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.085 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:50:36.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:36.086 | INFO | Uploaded 36GiB to GPU in 2.37667s (15.1473GiB/s) | 20260508 09:50:36.086 | INFO | Downloaded 34GiB to GPU in 2.44741s (13.8922GiB/s) | 20260508 09:50:36.183 | INFO | Starting a round of Baz IO | 20260508 09:50:39.591 | INFO | Populating RTMetrics report | 20260508 09:50:39.592 | INFO | RTMetrics report populated | 20260508 09:50:39.592 | INFO | Opening tmp RTMetrics json file | 20260508 09:50:39.602 | INFO | Serializing RTMetrics to json | 20260508 09:50:39.603 | INFO | Serialization done, swapping tmp file now... | 20260508 09:50:39.604 | INFO | RTMetrics done | 20260508 09:50:39.664 | INFO | Finished a round of Baz IO: Wrote 1.475 GiB in 3.481 seconds | 20260508 09:50:39.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:50:39.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.243628% / 0% / 5.5631e-05% | 20260508 09:50:39.664 | INFO | Baz IO overall: | 20260508 09:50:39.664 | INFO | BytesWritten : 3119431050746 | 20260508 09:50:39.664 | INFO | NumEvents : 1500147366493 | 20260508 09:50:41.225 | INFO | Analyzing chunk frames = [7163904,7164416) | 20260508 09:50:41.225 | INFO | MemInfo: { | 20260508 09:50:41.225 | INFO | "gpu" : " 17 GB", | 20260508 09:50:41.225 | INFO | "huge" : "", | 20260508 09:50:41.225 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:41.225 | INFO | "malloc" : " 14 GB", | 20260508 09:50:41.225 | INFO | "pinned" : " 29 GB" | 20260508 09:50:41.225 | INFO | } | 20260508 09:50:41.225 | INFO | Perf report for the last 5.14027 seconds: | 20260508 09:50:41.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:41.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:41.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:41.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:41.225 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:50:41.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:41.225 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 09:50:41.225 | INFO | PRE_HQ : 65.4%, 1.6, 54.4ms, 0 | 20260508 09:50:41.225 | INFO | BAZWRITER : 67.0%, 2.5, 87.5ms, 0 | 20260508 09:50:41.225 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:50:41.226 | INFO | Uploaded 36GiB to GPU in 2.58686s (13.9165GiB/s) | 20260508 09:50:41.226 | INFO | Downloaded 38GiB to GPU in 2.70972s (14.0236GiB/s) | 20260508 09:50:46.366 | INFO | Analyzing chunk frames = [7164416,7164928) | 20260508 09:50:46.366 | INFO | MemInfo: { | 20260508 09:50:46.366 | INFO | "gpu" : " 17 GB", | 20260508 09:50:46.366 | INFO | "huge" : "", | 20260508 09:50:46.366 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:46.366 | INFO | "malloc" : " 14 GB", | 20260508 09:50:46.366 | INFO | "pinned" : " 29 GB" | 20260508 09:50:46.366 | INFO | } | 20260508 09:50:46.366 | INFO | Perf report for the last 5.14109 seconds: | 20260508 09:50:46.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:46.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.366 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 09:50:46.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.366 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 09:50:46.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:46.367 | INFO | Uploaded 36GiB to GPU in 2.50209s (14.388GiB/s) | 20260508 09:50:46.367 | INFO | Downloaded 34GiB to GPU in 2.47317s (13.7476GiB/s) | 20260508 09:50:51.500 | INFO | Analyzing chunk frames = [7164928,7165440) | 20260508 09:50:51.500 | INFO | MemInfo: { | 20260508 09:50:51.500 | INFO | "gpu" : " 17 GB", | 20260508 09:50:51.500 | INFO | "huge" : "", | 20260508 09:50:51.500 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:51.500 | INFO | "malloc" : " 14 GB", | 20260508 09:50:51.500 | INFO | "pinned" : " 29 GB" | 20260508 09:50:51.500 | INFO | } | 20260508 09:50:51.500 | INFO | Perf report for the last 5.13403 seconds: | 20260508 09:50:51.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:51.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.500 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 09:50:51.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.500 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:50:51.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:51.501 | INFO | Uploaded 36GiB to GPU in 2.47337s (14.555GiB/s) | 20260508 09:50:51.501 | INFO | Downloaded 34GiB to GPU in 2.42788s (14.004GiB/s) | 20260508 09:50:56.636 | INFO | Analyzing chunk frames = [7165440,7165952) | 20260508 09:50:56.636 | INFO | MemInfo: { | 20260508 09:50:56.636 | INFO | "gpu" : " 17 GB", | 20260508 09:50:56.636 | INFO | "huge" : "", | 20260508 09:50:56.636 | INFO | "huge_shared" : " 29 GB", | 20260508 09:50:56.636 | INFO | "malloc" : " 14 GB", | 20260508 09:50:56.636 | INFO | "pinned" : " 29 GB" | 20260508 09:50:56.636 | INFO | } | 20260508 09:50:56.636 | INFO | Perf report for the last 5.13554 seconds: | 20260508 09:50:56.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:50:56.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.636 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:50:56.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.636 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:50:56.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:50:56.637 | INFO | Uploaded 36GiB to GPU in 2.49959s (14.4024GiB/s) | 20260508 09:50:56.637 | INFO | Downloaded 34GiB to GPU in 2.41684s (14.0679GiB/s) | 20260508 09:51:01.779 | INFO | Analyzing chunk frames = [7165952,7166464) | 20260508 09:51:01.779 | INFO | MemInfo: { | 20260508 09:51:01.779 | INFO | "gpu" : " 17 GB", | 20260508 09:51:01.779 | INFO | "huge" : "", | 20260508 09:51:01.779 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:01.779 | INFO | "malloc" : " 14 GB", | 20260508 09:51:01.779 | INFO | "pinned" : " 29 GB" | 20260508 09:51:01.779 | INFO | } | 20260508 09:51:01.779 | INFO | Perf report for the last 5.14316 seconds: | 20260508 09:51:01.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:01.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.779 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 09:51:01.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.779 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 09:51:01.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:01.780 | INFO | Uploaded 36GiB to GPU in 2.45656s (14.6547GiB/s) | 20260508 09:51:01.780 | INFO | Downloaded 34GiB to GPU in 2.53849s (13.3938GiB/s) | 20260508 09:51:06.923 | INFO | Analyzing chunk frames = [7166464,7166976) | 20260508 09:51:06.923 | INFO | MemInfo: { | 20260508 09:51:06.923 | INFO | "gpu" : " 17 GB", | 20260508 09:51:06.923 | INFO | "huge" : "", | 20260508 09:51:06.923 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:06.923 | INFO | "malloc" : " 14 GB", | 20260508 09:51:06.924 | INFO | "pinned" : " 29 GB" | 20260508 09:51:06.924 | INFO | } | 20260508 09:51:06.924 | INFO | Perf report for the last 5.14457 seconds: | 20260508 09:51:06.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:06.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 09:51:06.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:51:06.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:06.924 | INFO | Uploaded 36GiB to GPU in 2.40557s (14.9652GiB/s) | 20260508 09:51:06.924 | INFO | Downloaded 34GiB to GPU in 2.41739s (14.0647GiB/s) | 20260508 09:51:12.068 | INFO | Analyzing chunk frames = [7166976,7167488) | 20260508 09:51:12.068 | INFO | MemInfo: { | 20260508 09:51:12.068 | INFO | "gpu" : " 17 GB", | 20260508 09:51:12.068 | INFO | "huge" : "", | 20260508 09:51:12.068 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:12.068 | INFO | "malloc" : " 14 GB", | 20260508 09:51:12.068 | INFO | "pinned" : " 29 GB" | 20260508 09:51:12.068 | INFO | } | 20260508 09:51:12.069 | INFO | Perf report for the last 5.14486 seconds: | 20260508 09:51:12.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:12.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 09:51:12.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:51:12.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:12.069 | INFO | Uploaded 36GiB to GPU in 2.46546s (14.6017GiB/s) | 20260508 09:51:12.069 | INFO | Downloaded 34GiB to GPU in 2.44201s (13.923GiB/s) | 20260508 09:51:17.194 | INFO | Analyzing chunk frames = [7167488,7168000) | 20260508 09:51:17.194 | INFO | MemInfo: { | 20260508 09:51:17.194 | INFO | "gpu" : " 17 GB", | 20260508 09:51:17.194 | INFO | "huge" : "", | 20260508 09:51:17.194 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:17.194 | INFO | "malloc" : " 14 GB", | 20260508 09:51:17.194 | INFO | "pinned" : " 29 GB" | 20260508 09:51:17.195 | INFO | } | 20260508 09:51:17.195 | INFO | Perf report for the last 5.12602 seconds: | 20260508 09:51:17.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:17.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 09:51:17.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 09:51:17.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:17.195 | INFO | Uploaded 36GiB to GPU in 2.47698s (14.5338GiB/s) | 20260508 09:51:17.195 | INFO | Downloaded 34GiB to GPU in 2.44238s (13.9209GiB/s) | 20260508 09:51:17.195 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:51:17.195 | INFO | Total mem usage: 16.7 GiB | 20260508 09:51:17.195 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:51:17.195 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:51:17.195 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:51:17.195 | INFO | High Water Breakdown: | 20260508 09:51:17.195 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:51:17.195 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:51:17.195 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:51:17.195 | INFO | ------------------------------------------ | 20260508 09:51:17.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:51:17.195 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:51:17.195 | INFO | Total mem usage: 26.5 GiB | 20260508 09:51:17.195 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:51:17.195 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:51:17.195 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:51:17.195 | INFO | High Water Breakdown: | 20260508 09:51:17.195 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:51:17.195 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:51:17.195 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:51:17.195 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:51:17.195 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:51:17.195 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:51:17.195 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:51:17.195 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:51:17.195 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:51:17.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:51:17.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:51:17.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:51:17.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:51:17.195 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:51:17.195 | INFO | ------------------------------------------ | 20260508 09:51:17.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:51:17.195 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:51:17.195 | INFO | Total mem usage: 27.8 GiB | 20260508 09:51:17.195 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:51:17.195 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:51:17.195 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:51:17.195 | INFO | High Water Breakdown: | 20260508 09:51:17.195 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:51:17.195 | INFO | ------------------------------------------ | 20260508 09:51:17.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:51:17.195 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:51:17.195 | INFO | Total mem usage: 15.7 GiB | 20260508 09:51:17.195 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:51:17.195 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:51:17.195 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:51:17.195 | INFO | High Water Breakdown: | 20260508 09:51:17.195 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:51:17.195 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:51:17.195 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:51:17.195 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:51:17.195 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:51:17.195 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:51:17.195 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:51:17.195 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:51:17.195 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:51:17.195 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:51:17.195 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:51:17.195 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:51:17.195 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:51:17.195 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:51:17.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:51:17.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:51:17.195 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:51:17.195 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:51:17.195 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:51:17.195 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:51:17.195 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:51:17.195 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:51:17.195 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:51:17.195 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:51:17.195 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:51:17.195 | INFO | ------------------------------------------ | 20260508 09:51:17.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:51:17.195 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:51:17.195 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:51:17.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:51:17.195 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 7.00e-01 | 20260508 09:51:17.195 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.07e-01, 1.24e-02 / 1.05e-02 | 20260508 09:51:17.195 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.40e+00, 5.90e-01 / 2.37e-01 | 20260508 09:51:17.195 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.01e-01 | 20260508 09:51:17.195 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.50e+01, 3.65e+00 / 9.98e-01 | 20260508 09:51:17.195 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.87e+00 / 1.36e+00 | 20260508 09:51:17.195 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.43e+00, 4.07e-01 / 2.68e-01 | 20260508 09:51:17.195 | INFO | PerfReport: 3.02%, Metrics -- 9.89e-01 / 1.16e+00, 2.47e-01 / 1.46e-01 | 20260508 09:51:17.195 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.38e-01, 4.91e-02 / 2.98e-02 | 20260508 09:51:17.195 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:51:17.196 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:51:17.196 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:51:17.196 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 24.33%, Upload -- 2.56e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 24.57%, Download -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 31.09%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 2.51%, LockDownload -- 2.65e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:51:17.196 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260508 09:51:17.304 | INFO | Starting a round of Baz IO | 20260508 09:51:20.725 | INFO | Populating RTMetrics report | 20260508 09:51:20.725 | INFO | RTMetrics report populated | 20260508 09:51:20.725 | INFO | Opening tmp RTMetrics json file | 20260508 09:51:20.734 | INFO | Serializing RTMetrics to json | 20260508 09:51:20.735 | INFO | Serialization done, swapping tmp file now... | 20260508 09:51:20.736 | INFO | RTMetrics done | 20260508 09:51:20.793 | INFO | Finished a round of Baz IO: Wrote 1.473 GiB in 3.489 seconds | 20260508 09:51:20.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:51:20.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.24213% / 0% / 4.76837e-05% | 20260508 09:51:20.793 | INFO | Baz IO overall: | 20260508 09:51:20.793 | INFO | BytesWritten : 3121013166404 | 20260508 09:51:20.793 | INFO | NumEvents : 1500894695097 | 20260508 09:51:22.338 | INFO | Analyzing chunk frames = [7168000,7168512) | 20260508 09:51:22.338 | INFO | MemInfo: { | 20260508 09:51:22.338 | INFO | "gpu" : " 17 GB", | 20260508 09:51:22.338 | INFO | "huge" : "", | 20260508 09:51:22.338 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:22.338 | INFO | "malloc" : " 14 GB", | 20260508 09:51:22.338 | INFO | "pinned" : " 29 GB" | 20260508 09:51:22.338 | INFO | } | 20260508 09:51:22.339 | INFO | Perf report for the last 5.14394 seconds: | 20260508 09:51:22.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:22.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:22.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:22.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:22.339 | INFO | ANALYSIS : 67.7%, 3.0, 106.0ms, 0 | 20260508 09:51:22.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:22.339 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:51:22.339 | INFO | PRE_HQ : 65.4%, 1.6, 54.6ms, 0 | 20260508 09:51:22.339 | INFO | BAZWRITER : 67.3%, 2.5, 87.3ms, 0 | 20260508 09:51:22.339 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:51:22.339 | INFO | Uploaded 36GiB to GPU in 2.61072s (13.7893GiB/s) | 20260508 09:51:22.339 | INFO | Downloaded 38GiB to GPU in 2.6965s (14.0923GiB/s) | 20260508 09:51:27.478 | INFO | Analyzing chunk frames = [7168512,7169024) | 20260508 09:51:27.478 | INFO | MemInfo: { | 20260508 09:51:27.478 | INFO | "gpu" : " 17 GB", | 20260508 09:51:27.478 | INFO | "huge" : "", | 20260508 09:51:27.478 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:27.478 | INFO | "malloc" : " 14 GB", | 20260508 09:51:27.478 | INFO | "pinned" : " 29 GB" | 20260508 09:51:27.478 | INFO | } | 20260508 09:51:27.478 | INFO | Perf report for the last 5.13957 seconds: | 20260508 09:51:27.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:27.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.478 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:51:27.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.478 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 09:51:27.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:27.479 | INFO | Uploaded 36GiB to GPU in 2.53377s (14.2081GiB/s) | 20260508 09:51:27.479 | INFO | Downloaded 34GiB to GPU in 2.47253s (13.7511GiB/s) | 20260508 09:51:32.618 | INFO | Analyzing chunk frames = [7169024,7169536) | 20260508 09:51:32.618 | INFO | MemInfo: { | 20260508 09:51:32.618 | INFO | "gpu" : " 17 GB", | 20260508 09:51:32.618 | INFO | "huge" : "", | 20260508 09:51:32.618 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:32.618 | INFO | "malloc" : " 14 GB", | 20260508 09:51:32.618 | INFO | "pinned" : " 29 GB" | 20260508 09:51:32.618 | INFO | } | 20260508 09:51:32.618 | INFO | Perf report for the last 5.1401 seconds: | 20260508 09:51:32.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:32.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.618 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 09:51:32.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.619 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 09:51:32.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:32.619 | INFO | Uploaded 36GiB to GPU in 2.4617s (14.624GiB/s) | 20260508 09:51:32.619 | INFO | Downloaded 34GiB to GPU in 2.4344s (13.9665GiB/s) | 20260508 09:51:37.759 | INFO | Analyzing chunk frames = [7169536,7170048) | 20260508 09:51:37.759 | INFO | MemInfo: { | 20260508 09:51:37.759 | INFO | "gpu" : " 17 GB", | 20260508 09:51:37.759 | INFO | "huge" : "", | 20260508 09:51:37.759 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:37.759 | INFO | "malloc" : " 14 GB", | 20260508 09:51:37.759 | INFO | "pinned" : " 29 GB" | 20260508 09:51:37.759 | INFO | } | 20260508 09:51:37.759 | INFO | Perf report for the last 5.1408 seconds: | 20260508 09:51:37.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:37.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.759 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 09:51:37.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.759 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 09:51:37.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:37.760 | INFO | Uploaded 36GiB to GPU in 2.4582s (14.6449GiB/s) | 20260508 09:51:37.760 | INFO | Downloaded 34GiB to GPU in 2.41707s (14.0666GiB/s) | 20260508 09:51:42.892 | INFO | Analyzing chunk frames = [7170048,7170560) | 20260508 09:51:42.892 | INFO | MemInfo: { | 20260508 09:51:42.892 | INFO | "gpu" : " 17 GB", | 20260508 09:51:42.892 | INFO | "huge" : "", | 20260508 09:51:42.892 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:42.892 | INFO | "malloc" : " 14 GB", | 20260508 09:51:42.892 | INFO | "pinned" : " 29 GB" | 20260508 09:51:42.892 | INFO | } | 20260508 09:51:42.892 | INFO | Perf report for the last 5.13293 seconds: | 20260508 09:51:42.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:42.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 09:51:42.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | PRE_HQ : 30.4%, 1.0, 15.9ms, 0 | 20260508 09:51:42.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:42.892 | INFO | Uploaded 36GiB to GPU in 2.39774s (15.0142GiB/s) | 20260508 09:51:42.892 | INFO | Downloaded 34GiB to GPU in 2.53393s (13.4179GiB/s) | 20260508 09:51:48.042 | INFO | Analyzing chunk frames = [7170560,7171072) | 20260508 09:51:48.042 | INFO | MemInfo: { | 20260508 09:51:48.042 | INFO | "gpu" : " 17 GB", | 20260508 09:51:48.042 | INFO | "huge" : "", | 20260508 09:51:48.042 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:48.042 | INFO | "malloc" : " 14 GB", | 20260508 09:51:48.042 | INFO | "pinned" : " 29 GB" | 20260508 09:51:48.042 | INFO | } | 20260508 09:51:48.042 | INFO | Perf report for the last 5.15018 seconds: | 20260508 09:51:48.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:48.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 09:51:48.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:51:48.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:48.042 | INFO | Uploaded 36GiB to GPU in 2.38983s (15.0638GiB/s) | 20260508 09:51:48.042 | INFO | Downloaded 34GiB to GPU in 2.4348s (13.9642GiB/s) | 20260508 09:51:53.180 | INFO | Analyzing chunk frames = [7171072,7171584) | 20260508 09:51:53.180 | INFO | MemInfo: { | 20260508 09:51:53.180 | INFO | "gpu" : " 17 GB", | 20260508 09:51:53.180 | INFO | "huge" : "", | 20260508 09:51:53.180 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:53.180 | INFO | "malloc" : " 14 GB", | 20260508 09:51:53.180 | INFO | "pinned" : " 29 GB" | 20260508 09:51:53.180 | INFO | } | 20260508 09:51:53.180 | INFO | Perf report for the last 5.13764 seconds: | 20260508 09:51:53.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:53.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 09:51:53.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:51:53.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:53.180 | INFO | Uploaded 36GiB to GPU in 2.43267s (14.7986GiB/s) | 20260508 09:51:53.180 | INFO | Downloaded 34GiB to GPU in 2.43992s (13.9349GiB/s) | 20260508 09:51:58.310 | INFO | Analyzing chunk frames = [7171584,7172096) | 20260508 09:51:58.310 | INFO | MemInfo: { | 20260508 09:51:58.310 | INFO | "gpu" : " 17 GB", | 20260508 09:51:58.310 | INFO | "huge" : "", | 20260508 09:51:58.310 | INFO | "huge_shared" : " 29 GB", | 20260508 09:51:58.310 | INFO | "malloc" : " 14 GB", | 20260508 09:51:58.310 | INFO | "pinned" : " 29 GB" | 20260508 09:51:58.310 | INFO | } | 20260508 09:51:58.310 | INFO | Perf report for the last 5.13037 seconds: | 20260508 09:51:58.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:51:58.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.310 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 09:51:58.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.310 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:51:58.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:51:58.311 | INFO | Uploaded 36GiB to GPU in 2.44086s (14.7489GiB/s) | 20260508 09:51:58.311 | INFO | Downloaded 34GiB to GPU in 2.44587s (13.901GiB/s) | 20260508 09:51:58.407 | INFO | Starting a round of Baz IO | 20260508 09:52:01.815 | INFO | Populating RTMetrics report | 20260508 09:52:01.815 | INFO | RTMetrics report populated | 20260508 09:52:01.815 | INFO | Opening tmp RTMetrics json file | 20260508 09:52:01.825 | INFO | Serializing RTMetrics to json | 20260508 09:52:01.827 | INFO | Serialization done, swapping tmp file now... | 20260508 09:52:01.828 | INFO | RTMetrics done | 20260508 09:52:01.889 | INFO | Finished a round of Baz IO: Wrote 1.471 GiB in 3.481 seconds | 20260508 09:52:01.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:52:01.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.240425% / 0% / 5.16574e-05% | 20260508 09:52:01.889 | INFO | Baz IO overall: | 20260508 09:52:01.889 | INFO | BytesWritten : 3122592961597 | 20260508 09:52:01.889 | INFO | NumEvents : 1501640467154 | 20260508 09:52:03.445 | INFO | Analyzing chunk frames = [7172096,7172608) | 20260508 09:52:03.445 | INFO | MemInfo: { | 20260508 09:52:03.445 | INFO | "gpu" : " 17 GB", | 20260508 09:52:03.445 | INFO | "huge" : "", | 20260508 09:52:03.445 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:03.445 | INFO | "malloc" : " 14 GB", | 20260508 09:52:03.445 | INFO | "pinned" : " 29 GB" | 20260508 09:52:03.445 | INFO | } | 20260508 09:52:03.445 | INFO | Perf report for the last 5.13515 seconds: | 20260508 09:52:03.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:03.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:03.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:03.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:03.445 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 09:52:03.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:03.446 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:52:03.446 | INFO | PRE_HQ : 65.2%, 1.6, 54.4ms, 0 | 20260508 09:52:03.446 | INFO | BAZWRITER : 67.1%, 2.7, 94.3ms, 0 | 20260508 09:52:03.446 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:52:03.446 | INFO | Uploaded 36GiB to GPU in 2.60856s (13.8007GiB/s) | 20260508 09:52:03.446 | INFO | Downloaded 38GiB to GPU in 2.70051s (14.0714GiB/s) | 20260508 09:52:08.592 | INFO | Analyzing chunk frames = [7172608,7173120) | 20260508 09:52:08.592 | INFO | MemInfo: { | 20260508 09:52:08.592 | INFO | "gpu" : " 17 GB", | 20260508 09:52:08.592 | INFO | "huge" : "", | 20260508 09:52:08.592 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:08.592 | INFO | "malloc" : " 14 GB", | 20260508 09:52:08.592 | INFO | "pinned" : " 29 GB" | 20260508 09:52:08.593 | INFO | } | 20260508 09:52:08.593 | INFO | Perf report for the last 5.14721 seconds: | 20260508 09:52:08.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:08.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 09:52:08.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 09:52:08.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:08.593 | INFO | Uploaded 36GiB to GPU in 2.54447s (14.1483GiB/s) | 20260508 09:52:08.593 | INFO | Downloaded 34GiB to GPU in 2.45789s (13.833GiB/s) | 20260508 09:52:13.736 | INFO | Analyzing chunk frames = [7173120,7173632) | 20260508 09:52:13.736 | INFO | MemInfo: { | 20260508 09:52:13.736 | INFO | "gpu" : " 17 GB", | 20260508 09:52:13.736 | INFO | "huge" : "", | 20260508 09:52:13.736 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:13.736 | INFO | "malloc" : " 14 GB", | 20260508 09:52:13.736 | INFO | "pinned" : " 29 GB" | 20260508 09:52:13.736 | INFO | } | 20260508 09:52:13.736 | INFO | Perf report for the last 5.14352 seconds: | 20260508 09:52:13.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:13.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.736 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:52:13.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.736 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 09:52:13.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:13.737 | INFO | Uploaded 36GiB to GPU in 2.50523s (14.3699GiB/s) | 20260508 09:52:13.737 | INFO | Downloaded 34GiB to GPU in 2.42106s (14.0435GiB/s) | 20260508 09:52:18.872 | INFO | Analyzing chunk frames = [7173632,7174144) | 20260508 09:52:18.873 | INFO | MemInfo: { | 20260508 09:52:18.873 | INFO | "gpu" : " 17 GB", | 20260508 09:52:18.873 | INFO | "huge" : "", | 20260508 09:52:18.873 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:18.873 | INFO | "malloc" : " 14 GB", | 20260508 09:52:18.873 | INFO | "pinned" : " 29 GB" | 20260508 09:52:18.873 | INFO | } | 20260508 09:52:18.873 | INFO | Perf report for the last 5.13683 seconds: | 20260508 09:52:18.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:18.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.873 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:52:18.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.873 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260508 09:52:18.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:18.874 | INFO | Uploaded 36GiB to GPU in 2.50164s (14.3905GiB/s) | 20260508 09:52:18.874 | INFO | Downloaded 34GiB to GPU in 2.43032s (13.9899GiB/s) | 20260508 09:52:24.009 | INFO | Analyzing chunk frames = [7174144,7174656) | 20260508 09:52:24.009 | INFO | MemInfo: { | 20260508 09:52:24.009 | INFO | "gpu" : " 17 GB", | 20260508 09:52:24.009 | INFO | "huge" : "", | 20260508 09:52:24.009 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:24.009 | INFO | "malloc" : " 14 GB", | 20260508 09:52:24.009 | INFO | "pinned" : " 29 GB" | 20260508 09:52:24.009 | INFO | } | 20260508 09:52:24.009 | INFO | Perf report for the last 5.13622 seconds: | 20260508 09:52:24.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:24.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | ANALYSIS : 67.0%, 3.0, 104.4ms, 0 | 20260508 09:52:24.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | PRE_HQ : 31.6%, 1.0, 16.5ms, 0 | 20260508 09:52:24.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:24.009 | INFO | Uploaded 36GiB to GPU in 2.5126s (14.3278GiB/s) | 20260508 09:52:24.009 | INFO | Downloaded 34GiB to GPU in 2.53575s (13.4083GiB/s) | 20260508 09:52:29.147 | INFO | Analyzing chunk frames = [7174656,7175168) | 20260508 09:52:29.147 | INFO | MemInfo: { | 20260508 09:52:29.147 | INFO | "gpu" : " 17 GB", | 20260508 09:52:29.147 | INFO | "huge" : "", | 20260508 09:52:29.147 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:29.147 | INFO | "malloc" : " 14 GB", | 20260508 09:52:29.147 | INFO | "pinned" : " 29 GB" | 20260508 09:52:29.147 | INFO | } | 20260508 09:52:29.147 | INFO | Perf report for the last 5.13782 seconds: | 20260508 09:52:29.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:29.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 09:52:29.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:52:29.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:29.147 | INFO | Uploaded 36GiB to GPU in 2.3539s (15.2938GiB/s) | 20260508 09:52:29.147 | INFO | Downloaded 34GiB to GPU in 2.41382s (14.0855GiB/s) | 20260508 09:52:34.287 | INFO | Analyzing chunk frames = [7175168,7175680) | 20260508 09:52:34.287 | INFO | MemInfo: { | 20260508 09:52:34.287 | INFO | "gpu" : " 17 GB", | 20260508 09:52:34.287 | INFO | "huge" : "", | 20260508 09:52:34.287 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:34.287 | INFO | "malloc" : " 14 GB", | 20260508 09:52:34.287 | INFO | "pinned" : " 29 GB" | 20260508 09:52:34.287 | INFO | } | 20260508 09:52:34.287 | INFO | Perf report for the last 5.13974 seconds: | 20260508 09:52:34.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:34.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:52:34.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:52:34.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:34.287 | INFO | Uploaded 36GiB to GPU in 2.42305s (14.8573GiB/s) | 20260508 09:52:34.287 | INFO | Downloaded 34GiB to GPU in 2.42508s (14.0202GiB/s) | 20260508 09:52:39.428 | INFO | Analyzing chunk frames = [7175680,7176192) | 20260508 09:52:39.428 | INFO | MemInfo: { | 20260508 09:52:39.428 | INFO | "gpu" : " 17 GB", | 20260508 09:52:39.428 | INFO | "huge" : "", | 20260508 09:52:39.428 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:39.428 | INFO | "malloc" : " 14 GB", | 20260508 09:52:39.428 | INFO | "pinned" : " 29 GB" | 20260508 09:52:39.428 | INFO | } | 20260508 09:52:39.428 | INFO | Perf report for the last 5.14124 seconds: | 20260508 09:52:39.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:39.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:52:39.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 09:52:39.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:39.428 | INFO | Uploaded 36GiB to GPU in 2.41581s (14.9018GiB/s) | 20260508 09:52:39.428 | INFO | Downloaded 34GiB to GPU in 2.43517s (13.9621GiB/s) | 20260508 09:52:39.429 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:52:39.429 | INFO | Total mem usage: 16.7 GiB | 20260508 09:52:39.429 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:52:39.429 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.7%) | 20260508 09:52:39.429 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:52:39.429 | INFO | High Water Breakdown: | 20260508 09:52:39.429 | INFO | 59.8% : 8.7 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:52:39.429 | INFO | 40% : 5.83 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:52:39.429 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:52:39.429 | INFO | ------------------------------------------ | 20260508 09:52:39.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:52:39.429 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:52:39.429 | INFO | Total mem usage: 26.5 GiB | 20260508 09:52:39.429 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:52:39.429 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:52:39.429 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:52:39.429 | INFO | High Water Breakdown: | 20260508 09:52:39.429 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:52:39.429 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:52:39.429 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:52:39.429 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:52:39.429 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:52:39.429 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:52:39.429 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:52:39.429 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:52:39.429 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:52:39.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:52:39.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:52:39.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:52:39.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:52:39.429 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:52:39.429 | INFO | ------------------------------------------ | 20260508 09:52:39.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:52:39.429 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:52:39.429 | INFO | Total mem usage: 27.8 GiB | 20260508 09:52:39.429 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:52:39.429 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:52:39.429 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:52:39.429 | INFO | High Water Breakdown: | 20260508 09:52:39.429 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:52:39.429 | INFO | ------------------------------------------ | 20260508 09:52:39.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:52:39.429 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:52:39.429 | INFO | Total mem usage: 15.7 GiB | 20260508 09:52:39.429 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:52:39.429 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:52:39.429 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:52:39.429 | INFO | High Water Breakdown: | 20260508 09:52:39.429 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:52:39.429 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:52:39.429 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:52:39.429 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:52:39.429 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:52:39.429 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:52:39.429 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:52:39.429 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:52:39.429 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:52:39.429 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:52:39.429 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:52:39.429 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:52:39.429 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:52:39.429 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:52:39.429 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:52:39.429 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:52:39.429 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:52:39.429 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:52:39.429 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:52:39.430 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:52:39.430 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:52:39.430 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:52:39.430 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:52:39.430 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:52:39.430 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:52:39.430 | INFO | ------------------------------------------ | 20260508 09:52:39.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:52:39.430 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:52:39.430 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:52:39.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:52:39.430 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 5.73e-01 | 20260508 09:52:39.430 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.14e-02 / 8.26e-03 | 20260508 09:52:39.430 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.39e+00, 5.71e-01 / 2.67e-01 | 20260508 09:52:39.430 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.43e+00, 2.05e-01 / 1.31e-01 | 20260508 09:52:39.430 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.49e+01, 3.72e+00 / 1.10e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.81e+00 / 1.41e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.46e+00, 4.00e-01 / 2.61e-01 | 20260508 09:52:39.430 | INFO | PerfReport: 3.04%, Metrics -- 9.93e-01 / 1.18e+00, 2.45e-01 / 1.39e-01 | 20260508 09:52:39.430 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.38e-01, 4.83e-02 / 2.51e-02 | 20260508 09:52:39.430 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:52:39.430 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:52:39.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:52:39.430 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 24.45%, Upload -- 2.58e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 24.52%, Download -- 2.58e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 2.22%, LockUpload -- 2.34e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 2.45%, LockDownload -- 2.58e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 15.31%, LockCompute -- 1.61e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:52:39.430 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260508 09:52:39.525 | INFO | Starting a round of Baz IO | 20260508 09:52:42.933 | INFO | Populating RTMetrics report | 20260508 09:52:42.933 | INFO | RTMetrics report populated | 20260508 09:52:42.933 | INFO | Opening tmp RTMetrics json file | 20260508 09:52:42.946 | INFO | Serializing RTMetrics to json | 20260508 09:52:42.947 | INFO | Serialization done, swapping tmp file now... | 20260508 09:52:42.948 | INFO | RTMetrics done | 20260508 09:52:43.003 | INFO | Finished a round of Baz IO: Wrote 1.472 GiB in 3.478 seconds | 20260508 09:52:43.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:52:43.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.240898% / 0% / 6.75519e-05% | 20260508 09:52:43.003 | INFO | Baz IO overall: | 20260508 09:52:43.003 | INFO | BytesWritten : 3124173880693 | 20260508 09:52:43.003 | INFO | NumEvents : 1502387235116 | 20260508 09:52:44.571 | INFO | Analyzing chunk frames = [7176192,7176704) | 20260508 09:52:44.571 | INFO | MemInfo: { | 20260508 09:52:44.571 | INFO | "gpu" : " 17 GB", | 20260508 09:52:44.571 | INFO | "huge" : "", | 20260508 09:52:44.571 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:44.571 | INFO | "malloc" : " 14 GB", | 20260508 09:52:44.571 | INFO | "pinned" : " 29 GB" | 20260508 09:52:44.571 | INFO | } | 20260508 09:52:44.571 | INFO | Perf report for the last 5.14287 seconds: | 20260508 09:52:44.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:44.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:44.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:44.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:44.571 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 09:52:44.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:44.571 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 09:52:44.571 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 09:52:44.571 | INFO | BAZWRITER : 66.9%, 2.5, 86.4ms, 0 | 20260508 09:52:44.571 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:52:44.571 | INFO | Uploaded 36GiB to GPU in 2.5921s (13.8883GiB/s) | 20260508 09:52:44.571 | INFO | Downloaded 38GiB to GPU in 2.71161s (14.0138GiB/s) | 20260508 09:52:49.708 | INFO | Analyzing chunk frames = [7176704,7177216) | 20260508 09:52:49.708 | INFO | MemInfo: { | 20260508 09:52:49.708 | INFO | "gpu" : " 17 GB", | 20260508 09:52:49.708 | INFO | "huge" : "", | 20260508 09:52:49.708 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:49.708 | INFO | "malloc" : " 14 GB", | 20260508 09:52:49.708 | INFO | "pinned" : " 29 GB" | 20260508 09:52:49.708 | INFO | } | 20260508 09:52:49.708 | INFO | Perf report for the last 5.13702 seconds: | 20260508 09:52:49.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:49.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.708 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 09:52:49.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.708 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 09:52:49.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:49.709 | INFO | Uploaded 36GiB to GPU in 2.49292s (14.4409GiB/s) | 20260508 09:52:49.709 | INFO | Downloaded 34GiB to GPU in 2.46902s (13.7706GiB/s) | 20260508 09:52:54.850 | INFO | Analyzing chunk frames = [7177216,7177728) | 20260508 09:52:54.850 | INFO | MemInfo: { | 20260508 09:52:54.850 | INFO | "gpu" : " 17 GB", | 20260508 09:52:54.850 | INFO | "huge" : "", | 20260508 09:52:54.850 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:54.850 | INFO | "malloc" : " 14 GB", | 20260508 09:52:54.850 | INFO | "pinned" : " 29 GB" | 20260508 09:52:54.850 | INFO | } | 20260508 09:52:54.850 | INFO | Perf report for the last 5.14224 seconds: | 20260508 09:52:54.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:54.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 09:52:54.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:52:54.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:54.850 | INFO | Uploaded 36GiB to GPU in 2.45638s (14.6557GiB/s) | 20260508 09:52:54.851 | INFO | Downloaded 34GiB to GPU in 2.41276s (14.0917GiB/s) | 20260508 09:52:59.986 | INFO | Analyzing chunk frames = [7177728,7178240) | 20260508 09:52:59.986 | INFO | MemInfo: { | 20260508 09:52:59.986 | INFO | "gpu" : " 17 GB", | 20260508 09:52:59.986 | INFO | "huge" : "", | 20260508 09:52:59.986 | INFO | "huge_shared" : " 29 GB", | 20260508 09:52:59.987 | INFO | "malloc" : " 14 GB", | 20260508 09:52:59.987 | INFO | "pinned" : " 29 GB" | 20260508 09:52:59.987 | INFO | } | 20260508 09:52:59.987 | INFO | Perf report for the last 5.13639 seconds: | 20260508 09:52:59.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:52:59.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 09:52:59.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:52:59.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:52:59.987 | INFO | Uploaded 36GiB to GPU in 2.47054s (14.5717GiB/s) | 20260508 09:52:59.987 | INFO | Downloaded 34GiB to GPU in 2.43676s (13.9529GiB/s) | 20260508 09:53:05.128 | INFO | Analyzing chunk frames = [7178240,7178752) | 20260508 09:53:05.128 | INFO | MemInfo: { | 20260508 09:53:05.128 | INFO | "gpu" : " 17 GB", | 20260508 09:53:05.128 | INFO | "huge" : "", | 20260508 09:53:05.128 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:05.128 | INFO | "malloc" : " 14 GB", | 20260508 09:53:05.128 | INFO | "pinned" : " 29 GB" | 20260508 09:53:05.128 | INFO | } | 20260508 09:53:05.128 | INFO | Perf report for the last 5.14149 seconds: | 20260508 09:53:05.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:05.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 09:53:05.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 09:53:05.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:05.128 | INFO | Uploaded 36GiB to GPU in 2.38324s (15.1055GiB/s) | 20260508 09:53:05.129 | INFO | Downloaded 34GiB to GPU in 2.50443s (13.5759GiB/s) | 20260508 09:53:10.265 | INFO | Analyzing chunk frames = [7178752,7179264) | 20260508 09:53:10.265 | INFO | MemInfo: { | 20260508 09:53:10.265 | INFO | "gpu" : " 17 GB", | 20260508 09:53:10.265 | INFO | "huge" : "", | 20260508 09:53:10.265 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:10.265 | INFO | "malloc" : " 14 GB", | 20260508 09:53:10.265 | INFO | "pinned" : " 29 GB" | 20260508 09:53:10.265 | INFO | } | 20260508 09:53:10.265 | INFO | Perf report for the last 5.13726 seconds: | 20260508 09:53:10.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:10.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.266 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:53:10.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.266 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:53:10.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:10.266 | INFO | Uploaded 36GiB to GPU in 2.36707s (15.2087GiB/s) | 20260508 09:53:10.266 | INFO | Downloaded 34GiB to GPU in 2.43437s (13.9666GiB/s) | 20260508 09:53:15.405 | INFO | Analyzing chunk frames = [7179264,7179776) | 20260508 09:53:15.405 | INFO | MemInfo: { | 20260508 09:53:15.405 | INFO | "gpu" : " 17 GB", | 20260508 09:53:15.405 | INFO | "huge" : "", | 20260508 09:53:15.405 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:15.405 | INFO | "malloc" : " 14 GB", | 20260508 09:53:15.405 | INFO | "pinned" : " 29 GB" | 20260508 09:53:15.405 | INFO | } | 20260508 09:53:15.405 | INFO | Perf report for the last 5.1399 seconds: | 20260508 09:53:15.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:15.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.405 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:53:15.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.405 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:53:15.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:15.406 | INFO | Uploaded 36GiB to GPU in 2.40025s (14.9985GiB/s) | 20260508 09:53:15.406 | INFO | Downloaded 34GiB to GPU in 2.4342s (13.9676GiB/s) | 20260508 09:53:20.541 | INFO | Analyzing chunk frames = [7179776,7180288) | 20260508 09:53:20.541 | INFO | MemInfo: { | 20260508 09:53:20.541 | INFO | "gpu" : " 17 GB", | 20260508 09:53:20.541 | INFO | "huge" : "", | 20260508 09:53:20.541 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:20.541 | INFO | "malloc" : " 14 GB", | 20260508 09:53:20.541 | INFO | "pinned" : " 29 GB" | 20260508 09:53:20.541 | INFO | } | 20260508 09:53:20.541 | INFO | Perf report for the last 5.13598 seconds: | 20260508 09:53:20.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:20.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.541 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 09:53:20.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.541 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 09:53:20.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:20.542 | INFO | Uploaded 36GiB to GPU in 2.43604s (14.7781GiB/s) | 20260508 09:53:20.542 | INFO | Downloaded 34GiB to GPU in 2.43802s (13.9458GiB/s) | 20260508 09:53:20.639 | INFO | Starting a round of Baz IO | 20260508 09:53:24.065 | INFO | Populating RTMetrics report | 20260508 09:53:24.065 | INFO | RTMetrics report populated | 20260508 09:53:24.065 | INFO | Opening tmp RTMetrics json file | 20260508 09:53:24.068 | INFO | Serializing RTMetrics to json | 20260508 09:53:24.080 | INFO | Serialization done, swapping tmp file now... | 20260508 09:53:24.080 | INFO | RTMetrics done | 20260508 09:53:24.129 | INFO | Finished a round of Baz IO: Wrote 1.471 GiB in 3.49 seconds | 20260508 09:53:24.129 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:53:24.129 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239468% / 0% / 6.75519e-05% | 20260508 09:53:24.129 | INFO | Baz IO overall: | 20260508 09:53:24.129 | INFO | BytesWritten : 3125753436607 | 20260508 09:53:24.129 | INFO | NumEvents : 1503133190584 | 20260508 09:53:25.685 | INFO | Analyzing chunk frames = [7180288,7180800) | 20260508 09:53:25.685 | INFO | MemInfo: { | 20260508 09:53:25.685 | INFO | "gpu" : " 17 GB", | 20260508 09:53:25.685 | INFO | "huge" : "", | 20260508 09:53:25.685 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:25.685 | INFO | "malloc" : " 14 GB", | 20260508 09:53:25.685 | INFO | "pinned" : " 29 GB" | 20260508 09:53:25.685 | INFO | } | 20260508 09:53:25.685 | INFO | Perf report for the last 5.14403 seconds: | 20260508 09:53:25.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:25.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:25.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:25.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:25.685 | INFO | ANALYSIS : 67.9%, 3.0, 106.0ms, 0 | 20260508 09:53:25.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:25.686 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:53:25.686 | INFO | PRE_HQ : 65.7%, 1.6, 54.6ms, 0 | 20260508 09:53:25.686 | INFO | BAZWRITER : 67.1%, 2.5, 86.9ms, 0 | 20260508 09:53:25.686 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 09:53:25.686 | INFO | Uploaded 36GiB to GPU in 2.55346s (14.0985GiB/s) | 20260508 09:53:25.686 | INFO | Downloaded 38GiB to GPU in 2.72135s (13.9636GiB/s) | 20260508 09:53:30.817 | INFO | Analyzing chunk frames = [7180800,7181312) | 20260508 09:53:30.817 | INFO | MemInfo: { | 20260508 09:53:30.817 | INFO | "gpu" : " 17 GB", | 20260508 09:53:30.817 | INFO | "huge" : "", | 20260508 09:53:30.817 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:30.817 | INFO | "malloc" : " 14 GB", | 20260508 09:53:30.817 | INFO | "pinned" : " 29 GB" | 20260508 09:53:30.817 | INFO | } | 20260508 09:53:30.817 | INFO | Perf report for the last 5.132 seconds: | 20260508 09:53:30.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:30.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.817 | INFO | ANALYSIS : 67.0%, 3.0, 104.2ms, 0 | 20260508 09:53:30.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.818 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260508 09:53:30.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:30.818 | INFO | Uploaded 36GiB to GPU in 2.53848s (14.1817GiB/s) | 20260508 09:53:30.818 | INFO | Downloaded 34GiB to GPU in 2.4664s (13.7852GiB/s) | 20260508 09:53:35.954 | INFO | Analyzing chunk frames = [7181312,7181824) | 20260508 09:53:35.955 | INFO | MemInfo: { | 20260508 09:53:35.955 | INFO | "gpu" : " 17 GB", | 20260508 09:53:35.955 | INFO | "huge" : "", | 20260508 09:53:35.955 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:35.955 | INFO | "malloc" : " 14 GB", | 20260508 09:53:35.955 | INFO | "pinned" : " 29 GB" | 20260508 09:53:35.955 | INFO | } | 20260508 09:53:35.955 | INFO | Perf report for the last 5.13734 seconds: | 20260508 09:53:35.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:35.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:53:35.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:53:35.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:35.955 | INFO | Uploaded 36GiB to GPU in 2.45716s (14.6511GiB/s) | 20260508 09:53:35.955 | INFO | Downloaded 34GiB to GPU in 2.41748s (14.0642GiB/s) | 20260508 09:53:41.096 | INFO | Analyzing chunk frames = [7181824,7182336) | 20260508 09:53:41.096 | INFO | MemInfo: { | 20260508 09:53:41.096 | INFO | "gpu" : " 17 GB", | 20260508 09:53:41.096 | INFO | "huge" : "", | 20260508 09:53:41.096 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:41.097 | INFO | "malloc" : " 14 GB", | 20260508 09:53:41.097 | INFO | "pinned" : " 29 GB" | 20260508 09:53:41.097 | INFO | } | 20260508 09:53:41.097 | INFO | Perf report for the last 5.14182 seconds: | 20260508 09:53:41.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:41.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:53:41.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:53:41.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:41.097 | INFO | Uploaded 36GiB to GPU in 2.50805s (14.3538GiB/s) | 20260508 09:53:41.097 | INFO | Downloaded 34GiB to GPU in 2.43065s (13.988GiB/s) | 20260508 09:53:46.244 | INFO | Analyzing chunk frames = [7182336,7182848) | 20260508 09:53:46.244 | INFO | MemInfo: { | 20260508 09:53:46.244 | INFO | "gpu" : " 17 GB", | 20260508 09:53:46.244 | INFO | "huge" : "", | 20260508 09:53:46.244 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:46.244 | INFO | "malloc" : " 14 GB", | 20260508 09:53:46.245 | INFO | "pinned" : " 29 GB" | 20260508 09:53:46.245 | INFO | } | 20260508 09:53:46.245 | INFO | Perf report for the last 5.14796 seconds: | 20260508 09:53:46.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:46.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:53:46.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 09:53:46.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:46.245 | INFO | Uploaded 36GiB to GPU in 2.46749s (14.5897GiB/s) | 20260508 09:53:46.245 | INFO | Downloaded 34GiB to GPU in 2.5237s (13.4723GiB/s) | 20260508 09:53:51.385 | INFO | Analyzing chunk frames = [7182848,7183360) | 20260508 09:53:51.385 | INFO | MemInfo: { | 20260508 09:53:51.385 | INFO | "gpu" : " 17 GB", | 20260508 09:53:51.385 | INFO | "huge" : "", | 20260508 09:53:51.385 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:51.385 | INFO | "malloc" : " 14 GB", | 20260508 09:53:51.385 | INFO | "pinned" : " 29 GB" | 20260508 09:53:51.385 | INFO | } | 20260508 09:53:51.385 | INFO | Perf report for the last 5.14063 seconds: | 20260508 09:53:51.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:51.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.385 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 09:53:51.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.385 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 09:53:51.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:51.386 | INFO | Uploaded 36GiB to GPU in 2.35816s (15.2661GiB/s) | 20260508 09:53:51.386 | INFO | Downloaded 34GiB to GPU in 2.42741s (14.0067GiB/s) | 20260508 09:53:56.519 | INFO | Analyzing chunk frames = [7183360,7183872) | 20260508 09:53:56.519 | INFO | MemInfo: { | 20260508 09:53:56.519 | INFO | "gpu" : " 17 GB", | 20260508 09:53:56.519 | INFO | "huge" : "", | 20260508 09:53:56.519 | INFO | "huge_shared" : " 29 GB", | 20260508 09:53:56.519 | INFO | "malloc" : " 14 GB", | 20260508 09:53:56.519 | INFO | "pinned" : " 29 GB" | 20260508 09:53:56.519 | INFO | } | 20260508 09:53:56.519 | INFO | Perf report for the last 5.13372 seconds: | 20260508 09:53:56.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:53:56.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:53:56.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:53:56.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:53:56.519 | INFO | Uploaded 36GiB to GPU in 2.41585s (14.9016GiB/s) | 20260508 09:53:56.519 | INFO | Downloaded 34GiB to GPU in 2.43846s (13.9432GiB/s) | 20260508 09:54:01.658 | INFO | Analyzing chunk frames = [7183872,7184384) | 20260508 09:54:01.658 | INFO | MemInfo: { | 20260508 09:54:01.658 | INFO | "gpu" : " 17 GB", | 20260508 09:54:01.658 | INFO | "huge" : "", | 20260508 09:54:01.658 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:01.658 | INFO | "malloc" : " 14 GB", | 20260508 09:54:01.658 | INFO | "pinned" : " 29 GB" | 20260508 09:54:01.658 | INFO | } | 20260508 09:54:01.658 | INFO | Perf report for the last 5.13915 seconds: | 20260508 09:54:01.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:01.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 09:54:01.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:54:01.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:01.658 | INFO | Uploaded 36GiB to GPU in 2.40634s (14.9605GiB/s) | 20260508 09:54:01.658 | INFO | Downloaded 34GiB to GPU in 2.42909s (13.997GiB/s) | 20260508 09:54:01.659 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:54:01.659 | INFO | Total mem usage: 16.7 GiB | 20260508 09:54:01.659 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:54:01.659 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:54:01.659 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:54:01.659 | INFO | High Water Breakdown: | 20260508 09:54:01.659 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:54:01.659 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:54:01.659 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:54:01.659 | INFO | ------------------------------------------ | 20260508 09:54:01.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:54:01.659 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:54:01.659 | INFO | Total mem usage: 26.5 GiB | 20260508 09:54:01.659 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:54:01.659 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:54:01.659 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:54:01.659 | INFO | High Water Breakdown: | 20260508 09:54:01.659 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:54:01.659 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:54:01.659 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:54:01.659 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:54:01.659 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:54:01.659 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:54:01.659 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:54:01.659 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:54:01.659 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:54:01.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:54:01.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:54:01.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:54:01.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:54:01.659 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:54:01.659 | INFO | ------------------------------------------ | 20260508 09:54:01.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:54:01.659 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:54:01.659 | INFO | Total mem usage: 27.8 GiB | 20260508 09:54:01.659 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:54:01.659 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:54:01.659 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:54:01.659 | INFO | High Water Breakdown: | 20260508 09:54:01.660 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:54:01.660 | INFO | ------------------------------------------ | 20260508 09:54:01.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:54:01.660 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:54:01.660 | INFO | Total mem usage: 15.7 GiB | 20260508 09:54:01.660 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:54:01.660 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:54:01.660 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:54:01.660 | INFO | High Water Breakdown: | 20260508 09:54:01.660 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:54:01.660 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:54:01.660 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:54:01.660 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:54:01.660 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:54:01.660 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:54:01.660 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:54:01.660 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:54:01.660 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:54:01.660 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:54:01.660 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:54:01.660 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:54:01.660 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:54:01.660 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:54:01.660 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:54:01.660 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:54:01.660 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:54:01.660 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:54:01.660 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:54:01.660 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:54:01.660 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:54:01.660 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:54:01.660 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:54:01.660 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:54:01.660 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:54:01.660 | INFO | ------------------------------------------ | 20260508 09:54:01.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:54:01.660 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:54:01.660 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:54:01.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:54:01.660 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.66e-01 | 20260508 09:54:01.660 | INFO | PerfReport: 0.29%, AggregateStats -- 9.65e-02 / 1.04e-01, 1.10e-02 / 5.60e-03 | 20260508 09:54:01.660 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.45e+00, 5.99e-01 / 2.01e-01 | 20260508 09:54:01.660 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.41e+00, 2.10e-01 / 1.53e-01 | 20260508 09:54:01.660 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.49e+01, 3.68e+00 / 1.13e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.81e+00 / 1.37e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.07e-01 / 2.69e-01 | 20260508 09:54:01.660 | INFO | PerfReport: 3.03%, Metrics -- 9.90e-01 / 1.14e+00, 2.47e-01 / 1.47e-01 | 20260508 09:54:01.660 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.30e-01, 4.81e-02 / 2.87e-02 | 20260508 09:54:01.660 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:54:01.660 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:54:01.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:54:01.660 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 24.30%, Upload -- 2.56e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 24.54%, Download -- 2.58e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 2.18%, LockUpload -- 2.30e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 2.57%, LockDownload -- 2.70e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 15.34%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 09:54:01.660 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.20e-02 / 0.00e+00 | 20260508 09:54:01.787 | INFO | Starting a round of Baz IO | 20260508 09:54:05.188 | INFO | Populating RTMetrics report | 20260508 09:54:05.188 | INFO | RTMetrics report populated | 20260508 09:54:05.188 | INFO | Opening tmp RTMetrics json file | 20260508 09:54:05.196 | INFO | Serializing RTMetrics to json | 20260508 09:54:05.198 | INFO | Serialization done, swapping tmp file now... | 20260508 09:54:05.198 | INFO | RTMetrics done | 20260508 09:54:05.253 | INFO | Finished a round of Baz IO: Wrote 1.472 GiB in 3.466 seconds | 20260508 09:54:05.254 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.00% / 59.74% | 20260508 09:54:05.254 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239003% / 7.94729e-06% / 3.97364e-05% | 20260508 09:54:05.254 | INFO | Baz IO overall: | 20260508 09:54:05.254 | INFO | BytesWritten : 3127334055970 | 20260508 09:54:05.254 | INFO | NumEvents : 1503879989888 | 20260508 09:54:06.800 | INFO | Analyzing chunk frames = [7184384,7184896) | 20260508 09:54:06.800 | INFO | MemInfo: { | 20260508 09:54:06.800 | INFO | "gpu" : " 17 GB", | 20260508 09:54:06.800 | INFO | "huge" : "", | 20260508 09:54:06.800 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:06.800 | INFO | "malloc" : " 14 GB", | 20260508 09:54:06.800 | INFO | "pinned" : " 29 GB" | 20260508 09:54:06.800 | INFO | } | 20260508 09:54:06.800 | INFO | Perf report for the last 5.14227 seconds: | 20260508 09:54:06.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:06.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:06.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:06.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:06.800 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 09:54:06.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:06.801 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260508 09:54:06.801 | INFO | PRE_HQ : 66.4%, 1.8, 61.7ms, 0 | 20260508 09:54:06.801 | INFO | BAZWRITER : 67.1%, 2.7, 92.8ms, 0 | 20260508 09:54:06.801 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:54:06.801 | INFO | Uploaded 36GiB to GPU in 2.58756s (13.9127GiB/s) | 20260508 09:54:06.801 | INFO | Downloaded 38GiB to GPU in 2.72066s (13.9672GiB/s) | 20260508 09:54:11.938 | INFO | Analyzing chunk frames = [7184896,7185408) | 20260508 09:54:11.938 | INFO | MemInfo: { | 20260508 09:54:11.938 | INFO | "gpu" : " 17 GB", | 20260508 09:54:11.938 | INFO | "huge" : "", | 20260508 09:54:11.938 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:11.938 | INFO | "malloc" : " 14 GB", | 20260508 09:54:11.938 | INFO | "pinned" : " 29 GB" | 20260508 09:54:11.938 | INFO | } | 20260508 09:54:11.938 | INFO | Perf report for the last 5.13794 seconds: | 20260508 09:54:11.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:11.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.938 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 09:54:11.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.939 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 09:54:11.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:11.939 | INFO | Uploaded 36GiB to GPU in 2.47113s (14.5682GiB/s) | 20260508 09:54:11.939 | INFO | Downloaded 34GiB to GPU in 2.49479s (13.6284GiB/s) | 20260508 09:54:17.079 | INFO | Analyzing chunk frames = [7185408,7185920) | 20260508 09:54:17.079 | INFO | MemInfo: { | 20260508 09:54:17.079 | INFO | "gpu" : " 17 GB", | 20260508 09:54:17.079 | INFO | "huge" : "", | 20260508 09:54:17.079 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:17.079 | INFO | "malloc" : " 14 GB", | 20260508 09:54:17.079 | INFO | "pinned" : " 29 GB" | 20260508 09:54:17.079 | INFO | } | 20260508 09:54:17.079 | INFO | Perf report for the last 5.14061 seconds: | 20260508 09:54:17.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:17.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.079 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 09:54:17.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.079 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:54:17.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:17.080 | INFO | Uploaded 36GiB to GPU in 2.43127s (14.8071GiB/s) | 20260508 09:54:17.080 | INFO | Downloaded 34GiB to GPU in 2.4358s (13.9584GiB/s) | 20260508 09:54:22.213 | INFO | Analyzing chunk frames = [7185920,7186432) | 20260508 09:54:22.213 | INFO | MemInfo: { | 20260508 09:54:22.213 | INFO | "gpu" : " 17 GB", | 20260508 09:54:22.213 | INFO | "huge" : "", | 20260508 09:54:22.213 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:22.213 | INFO | "malloc" : " 14 GB", | 20260508 09:54:22.213 | INFO | "pinned" : " 29 GB" | 20260508 09:54:22.213 | INFO | } | 20260508 09:54:22.213 | INFO | Perf report for the last 5.13421 seconds: | 20260508 09:54:22.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:22.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 09:54:22.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:54:22.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:22.213 | INFO | Uploaded 36GiB to GPU in 2.43583s (14.7794GiB/s) | 20260508 09:54:22.214 | INFO | Downloaded 34GiB to GPU in 2.45517s (13.8483GiB/s) | 20260508 09:54:27.356 | INFO | Analyzing chunk frames = [7186432,7186944) | 20260508 09:54:27.356 | INFO | MemInfo: { | 20260508 09:54:27.356 | INFO | "gpu" : " 17 GB", | 20260508 09:54:27.356 | INFO | "huge" : "", | 20260508 09:54:27.356 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:27.356 | INFO | "malloc" : " 14 GB", | 20260508 09:54:27.356 | INFO | "pinned" : " 29 GB" | 20260508 09:54:27.356 | INFO | } | 20260508 09:54:27.356 | INFO | Perf report for the last 5.14312 seconds: | 20260508 09:54:27.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:27.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.356 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 09:54:27.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.356 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:54:27.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:27.357 | INFO | Uploaded 36GiB to GPU in 2.37715s (15.1442GiB/s) | 20260508 09:54:27.357 | INFO | Downloaded 34GiB to GPU in 2.52943s (13.4418GiB/s) | 20260508 09:54:32.496 | INFO | Analyzing chunk frames = [7186944,7187456) | 20260508 09:54:32.496 | INFO | MemInfo: { | 20260508 09:54:32.496 | INFO | "gpu" : " 17 GB", | 20260508 09:54:32.496 | INFO | "huge" : "", | 20260508 09:54:32.496 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:32.496 | INFO | "malloc" : " 14 GB", | 20260508 09:54:32.496 | INFO | "pinned" : " 29 GB" | 20260508 09:54:32.496 | INFO | } | 20260508 09:54:32.496 | INFO | Perf report for the last 5.13977 seconds: | 20260508 09:54:32.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:32.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.496 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 09:54:32.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.496 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:54:32.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:32.497 | INFO | Uploaded 36GiB to GPU in 2.3922s (15.0489GiB/s) | 20260508 09:54:32.497 | INFO | Downloaded 34GiB to GPU in 2.43524s (13.9617GiB/s) | 20260508 09:54:37.631 | INFO | Analyzing chunk frames = [7187456,7187968) | 20260508 09:54:37.632 | INFO | MemInfo: { | 20260508 09:54:37.632 | INFO | "gpu" : " 17 GB", | 20260508 09:54:37.632 | INFO | "huge" : "", | 20260508 09:54:37.632 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:37.632 | INFO | "malloc" : " 14 GB", | 20260508 09:54:37.632 | INFO | "pinned" : " 29 GB" | 20260508 09:54:37.632 | INFO | } | 20260508 09:54:37.632 | INFO | Perf report for the last 5.13565 seconds: | 20260508 09:54:37.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:37.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:54:37.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:54:37.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:37.632 | INFO | Uploaded 36GiB to GPU in 2.42528s (14.8436GiB/s) | 20260508 09:54:37.632 | INFO | Downloaded 34GiB to GPU in 2.44796s (13.8891GiB/s) | 20260508 09:54:42.773 | INFO | Analyzing chunk frames = [7187968,7188480) | 20260508 09:54:42.773 | INFO | MemInfo: { | 20260508 09:54:42.773 | INFO | "gpu" : " 17 GB", | 20260508 09:54:42.773 | INFO | "huge" : "", | 20260508 09:54:42.773 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:42.773 | INFO | "malloc" : " 14 GB", | 20260508 09:54:42.773 | INFO | "pinned" : " 29 GB" | 20260508 09:54:42.773 | INFO | } | 20260508 09:54:42.773 | INFO | Perf report for the last 5.1412 seconds: | 20260508 09:54:42.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:42.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 09:54:42.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:54:42.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:42.773 | INFO | Uploaded 36GiB to GPU in 2.41332s (14.9172GiB/s) | 20260508 09:54:42.773 | INFO | Downloaded 34GiB to GPU in 2.45248s (13.8635GiB/s) | 20260508 09:54:42.870 | INFO | Starting a round of Baz IO | 20260508 09:54:46.287 | INFO | Populating RTMetrics report | 20260508 09:54:46.287 | INFO | RTMetrics report populated | 20260508 09:54:46.287 | INFO | Opening tmp RTMetrics json file | 20260508 09:54:46.295 | INFO | Serializing RTMetrics to json | 20260508 09:54:46.296 | INFO | Serialization done, swapping tmp file now... | 20260508 09:54:46.296 | INFO | RTMetrics done | 20260508 09:54:46.356 | INFO | Finished a round of Baz IO: Wrote 1.471 GiB in 3.486 seconds | 20260508 09:54:46.356 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.74% | 20260508 09:54:46.356 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238844% / 0% / 5.96046e-05% | 20260508 09:54:46.356 | INFO | Baz IO overall: | 20260508 09:54:46.356 | INFO | BytesWritten : 3128913153921 | 20260508 09:54:46.356 | INFO | NumEvents : 1504625639949 | 20260508 09:54:47.909 | INFO | Analyzing chunk frames = [7188480,7188992) | 20260508 09:54:47.909 | INFO | MemInfo: { | 20260508 09:54:47.909 | INFO | "gpu" : " 17 GB", | 20260508 09:54:47.909 | INFO | "huge" : "", | 20260508 09:54:47.909 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:47.909 | INFO | "malloc" : " 14 GB", | 20260508 09:54:47.909 | INFO | "pinned" : " 29 GB" | 20260508 09:54:47.909 | INFO | } | 20260508 09:54:47.909 | INFO | Perf report for the last 5.13612 seconds: | 20260508 09:54:47.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:47.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:47.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:47.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:47.909 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:54:47.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:47.909 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:54:47.909 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 09:54:47.909 | INFO | BAZWRITER : 67.2%, 2.5, 86.9ms, 0 | 20260508 09:54:47.909 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:54:47.910 | INFO | Uploaded 36GiB to GPU in 2.59434s (13.8764GiB/s) | 20260508 09:54:47.910 | INFO | Downloaded 38GiB to GPU in 2.70734s (14.0359GiB/s) | 20260508 09:54:53.051 | INFO | Analyzing chunk frames = [7188992,7189504) | 20260508 09:54:53.051 | INFO | MemInfo: { | 20260508 09:54:53.051 | INFO | "gpu" : " 17 GB", | 20260508 09:54:53.051 | INFO | "huge" : "", | 20260508 09:54:53.051 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:53.051 | INFO | "malloc" : " 14 GB", | 20260508 09:54:53.051 | INFO | "pinned" : " 29 GB" | 20260508 09:54:53.051 | INFO | } | 20260508 09:54:53.051 | INFO | Perf report for the last 5.1418 seconds: | 20260508 09:54:53.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:53.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 09:54:53.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 09:54:53.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:53.051 | INFO | Uploaded 36GiB to GPU in 2.50354s (14.3797GiB/s) | 20260508 09:54:53.051 | INFO | Downloaded 34GiB to GPU in 2.45332s (13.8587GiB/s) | 20260508 09:54:58.191 | INFO | Analyzing chunk frames = [7189504,7190016) | 20260508 09:54:58.192 | INFO | MemInfo: { | 20260508 09:54:58.192 | INFO | "gpu" : " 17 GB", | 20260508 09:54:58.192 | INFO | "huge" : "", | 20260508 09:54:58.192 | INFO | "huge_shared" : " 29 GB", | 20260508 09:54:58.192 | INFO | "malloc" : " 14 GB", | 20260508 09:54:58.192 | INFO | "pinned" : " 29 GB" | 20260508 09:54:58.192 | INFO | } | 20260508 09:54:58.192 | INFO | Perf report for the last 5.14075 seconds: | 20260508 09:54:58.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:54:58.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 09:54:58.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:54:58.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:54:58.192 | INFO | Uploaded 36GiB to GPU in 2.46868s (14.5827GiB/s) | 20260508 09:54:58.192 | INFO | Downloaded 34GiB to GPU in 2.43281s (13.9756GiB/s) | 20260508 09:55:03.330 | INFO | Analyzing chunk frames = [7190016,7190528) | 20260508 09:55:03.330 | INFO | MemInfo: { | 20260508 09:55:03.330 | INFO | "gpu" : " 17 GB", | 20260508 09:55:03.330 | INFO | "huge" : "", | 20260508 09:55:03.330 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:03.330 | INFO | "malloc" : " 14 GB", | 20260508 09:55:03.330 | INFO | "pinned" : " 29 GB" | 20260508 09:55:03.330 | INFO | } | 20260508 09:55:03.330 | INFO | Perf report for the last 5.13864 seconds: | 20260508 09:55:03.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:03.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.330 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 09:55:03.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.330 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:55:03.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:03.331 | INFO | Uploaded 36GiB to GPU in 2.49214s (14.4454GiB/s) | 20260508 09:55:03.331 | INFO | Downloaded 34GiB to GPU in 2.42638s (14.0126GiB/s) | 20260508 09:55:08.468 | INFO | Analyzing chunk frames = [7190528,7191040) | 20260508 09:55:08.468 | INFO | MemInfo: { | 20260508 09:55:08.468 | INFO | "gpu" : " 17 GB", | 20260508 09:55:08.468 | INFO | "huge" : "", | 20260508 09:55:08.468 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:08.468 | INFO | "malloc" : " 14 GB", | 20260508 09:55:08.468 | INFO | "pinned" : " 29 GB" | 20260508 09:55:08.468 | INFO | } | 20260508 09:55:08.468 | INFO | Perf report for the last 5.13776 seconds: | 20260508 09:55:08.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:08.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.468 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 09:55:08.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.468 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:55:08.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:08.469 | INFO | Uploaded 36GiB to GPU in 2.29983s (15.6534GiB/s) | 20260508 09:55:08.469 | INFO | Downloaded 34GiB to GPU in 2.50903s (13.5511GiB/s) | 20260508 09:55:13.607 | INFO | Analyzing chunk frames = [7191040,7191552) | 20260508 09:55:13.607 | INFO | MemInfo: { | 20260508 09:55:13.607 | INFO | "gpu" : " 17 GB", | 20260508 09:55:13.607 | INFO | "huge" : "", | 20260508 09:55:13.607 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:13.607 | INFO | "malloc" : " 14 GB", | 20260508 09:55:13.607 | INFO | "pinned" : " 29 GB" | 20260508 09:55:13.607 | INFO | } | 20260508 09:55:13.607 | INFO | Perf report for the last 5.1393 seconds: | 20260508 09:55:13.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:13.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.608 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 09:55:13.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.608 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 09:55:13.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:13.608 | INFO | Uploaded 36GiB to GPU in 2.35305s (15.2993GiB/s) | 20260508 09:55:13.608 | INFO | Downloaded 34GiB to GPU in 2.45249s (13.8634GiB/s) | 20260508 09:55:18.751 | INFO | Analyzing chunk frames = [7191552,7192064) | 20260508 09:55:18.751 | INFO | MemInfo: { | 20260508 09:55:18.751 | INFO | "gpu" : " 17 GB", | 20260508 09:55:18.751 | INFO | "huge" : "", | 20260508 09:55:18.751 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:18.751 | INFO | "malloc" : " 14 GB", | 20260508 09:55:18.751 | INFO | "pinned" : " 29 GB" | 20260508 09:55:18.751 | INFO | } | 20260508 09:55:18.751 | INFO | Perf report for the last 5.14356 seconds: | 20260508 09:55:18.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:18.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.751 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 09:55:18.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.751 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:55:18.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:18.752 | INFO | Uploaded 36GiB to GPU in 2.41049s (14.9347GiB/s) | 20260508 09:55:18.752 | INFO | Downloaded 34GiB to GPU in 2.45648s (13.8409GiB/s) | 20260508 09:55:23.888 | INFO | Analyzing chunk frames = [7192064,7192576) | 20260508 09:55:23.888 | INFO | MemInfo: { | 20260508 09:55:23.888 | INFO | "gpu" : " 17 GB", | 20260508 09:55:23.888 | INFO | "huge" : "", | 20260508 09:55:23.888 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:23.888 | INFO | "malloc" : " 14 GB", | 20260508 09:55:23.888 | INFO | "pinned" : " 29 GB" | 20260508 09:55:23.888 | INFO | } | 20260508 09:55:23.888 | INFO | Perf report for the last 5.13716 seconds: | 20260508 09:55:23.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:23.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.888 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 09:55:23.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.888 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:55:23.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:23.889 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:55:23.889 | INFO | Total mem usage: 16.7 GiB | 20260508 09:55:23.889 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:55:23.889 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 09:55:23.889 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:55:23.889 | INFO | High Water Breakdown: | 20260508 09:55:23.889 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:55:23.889 | INFO | 40.3% : 5.78 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:55:23.889 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:55:23.889 | INFO | ------------------------------------------ | 20260508 09:55:23.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:55:23.889 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:55:23.889 | INFO | Total mem usage: 26.5 GiB | 20260508 09:55:23.889 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:55:23.889 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:55:23.889 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:55:23.889 | INFO | High Water Breakdown: | 20260508 09:55:23.889 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:55:23.889 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:55:23.889 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:55:23.889 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:55:23.889 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:55:23.889 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:55:23.889 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:55:23.889 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:55:23.889 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:55:23.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:55:23.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:55:23.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:55:23.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:55:23.889 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:55:23.889 | INFO | ------------------------------------------ | 20260508 09:55:23.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:55:23.889 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:55:23.889 | INFO | Uploaded 36GiB to GPU in 2.38089s (15.1204GiB/s) | 20260508 09:55:23.889 | INFO | Total mem usage: 27.8 GiB | 20260508 09:55:23.889 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:55:23.889 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:55:23.889 | INFO | Downloaded 34GiB to GPU in 2.46129s (13.8139GiB/s) | 20260508 09:55:23.889 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:55:23.889 | INFO | High Water Breakdown: | 20260508 09:55:23.889 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:55:23.889 | INFO | ------------------------------------------ | 20260508 09:55:23.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:55:23.889 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:55:23.889 | INFO | Total mem usage: 15.7 GiB | 20260508 09:55:23.889 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:55:23.889 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:55:23.889 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:55:23.889 | INFO | High Water Breakdown: | 20260508 09:55:23.889 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:55:23.889 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:55:23.889 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:55:23.889 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:55:23.889 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:55:23.889 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:55:23.889 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:55:23.889 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:55:23.889 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:55:23.889 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:55:23.889 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:55:23.889 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:55:23.889 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:55:23.889 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:55:23.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:55:23.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:55:23.889 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:55:23.889 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:55:23.889 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:55:23.889 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:55:23.889 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:55:23.889 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:55:23.889 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:55:23.889 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:55:23.889 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:55:23.889 | INFO | ------------------------------------------ | 20260508 09:55:23.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:55:23.890 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:55:23.890 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:55:23.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:55:23.890 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.50e-01 | 20260508 09:55:23.890 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.06e-01, 1.14e-02 / 3.39e-03 | 20260508 09:55:23.890 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.42e+00, 5.85e-01 / 2.23e-01 | 20260508 09:55:23.890 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.15e-01 | 20260508 09:55:23.890 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.52e+01, 3.82e+00 / 1.21e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.83e+00 / 1.17e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.50e+00, 4.10e-01 / 2.50e-01 | 20260508 09:55:23.890 | INFO | PerfReport: 3.02%, Metrics -- 9.89e-01 / 1.12e+00, 2.44e-01 / 1.17e-01 | 20260508 09:55:23.890 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.31e-01, 4.83e-02 / 2.77e-02 | 20260508 09:55:23.890 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:55:23.890 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:55:23.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:55:23.890 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 24.15%, Upload -- 2.54e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 24.66%, Download -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 2.05%, LockUpload -- 2.16e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 2.63%, LockDownload -- 2.77e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 09:55:23.890 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260508 09:55:23.986 | INFO | Starting a round of Baz IO | 20260508 09:55:27.398 | INFO | Populating RTMetrics report | 20260508 09:55:27.399 | INFO | RTMetrics report populated | 20260508 09:55:27.399 | INFO | Opening tmp RTMetrics json file | 20260508 09:55:27.411 | INFO | Serializing RTMetrics to json | 20260508 09:55:27.412 | INFO | Serialization done, swapping tmp file now... | 20260508 09:55:27.413 | INFO | RTMetrics done | 20260508 09:55:27.469 | INFO | Finished a round of Baz IO: Wrote 1.472 GiB in 3.483 seconds | 20260508 09:55:27.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:55:27.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23915% / 3.97364e-06% / 6.35783e-05% | 20260508 09:55:27.470 | INFO | Baz IO overall: | 20260508 09:55:27.470 | INFO | BytesWritten : 3130493537676 | 20260508 09:55:27.470 | INFO | NumEvents : 1505372319751 | 20260508 09:55:29.027 | INFO | Analyzing chunk frames = [7192576,7193088) | 20260508 09:55:29.027 | INFO | MemInfo: { | 20260508 09:55:29.027 | INFO | "gpu" : " 17 GB", | 20260508 09:55:29.027 | INFO | "huge" : "", | 20260508 09:55:29.027 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:29.027 | INFO | "malloc" : " 14 GB", | 20260508 09:55:29.027 | INFO | "pinned" : " 29 GB" | 20260508 09:55:29.027 | INFO | } | 20260508 09:55:29.027 | INFO | Perf report for the last 5.13872 seconds: | 20260508 09:55:29.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:29.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:29.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:29.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:29.027 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:55:29.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:29.027 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:55:29.027 | INFO | PRE_HQ : 65.6%, 1.6, 54.5ms, 0 | 20260508 09:55:29.027 | INFO | BAZWRITER : 67.1%, 2.5, 86.8ms, 0 | 20260508 09:55:29.027 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:55:29.027 | INFO | Uploaded 36GiB to GPU in 2.59871s (13.853GiB/s) | 20260508 09:55:29.027 | INFO | Downloaded 38GiB to GPU in 2.71815s (13.9801GiB/s) | 20260508 09:55:34.167 | INFO | Analyzing chunk frames = [7193088,7193600) | 20260508 09:55:34.167 | INFO | MemInfo: { | 20260508 09:55:34.167 | INFO | "gpu" : " 17 GB", | 20260508 09:55:34.167 | INFO | "huge" : "", | 20260508 09:55:34.167 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:34.167 | INFO | "malloc" : " 14 GB", | 20260508 09:55:34.167 | INFO | "pinned" : " 29 GB" | 20260508 09:55:34.167 | INFO | } | 20260508 09:55:34.167 | INFO | Perf report for the last 5.1403 seconds: | 20260508 09:55:34.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:34.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.167 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 09:55:34.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.167 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:55:34.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:34.168 | INFO | Uploaded 36GiB to GPU in 2.48763s (14.4716GiB/s) | 20260508 09:55:34.168 | INFO | Downloaded 34GiB to GPU in 2.47182s (13.7551GiB/s) | 20260508 09:55:39.304 | INFO | Analyzing chunk frames = [7193600,7194112) | 20260508 09:55:39.304 | INFO | MemInfo: { | 20260508 09:55:39.304 | INFO | "gpu" : " 17 GB", | 20260508 09:55:39.304 | INFO | "huge" : "", | 20260508 09:55:39.304 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:39.305 | INFO | "malloc" : " 14 GB", | 20260508 09:55:39.305 | INFO | "pinned" : " 29 GB" | 20260508 09:55:39.305 | INFO | } | 20260508 09:55:39.305 | INFO | Perf report for the last 5.13737 seconds: | 20260508 09:55:39.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:39.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 09:55:39.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:55:39.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:39.305 | INFO | Uploaded 36GiB to GPU in 2.45624s (14.6565GiB/s) | 20260508 09:55:39.305 | INFO | Downloaded 34GiB to GPU in 2.41523s (14.0773GiB/s) | 20260508 09:55:44.447 | INFO | Analyzing chunk frames = [7194112,7194624) | 20260508 09:55:44.447 | INFO | MemInfo: { | 20260508 09:55:44.447 | INFO | "gpu" : " 17 GB", | 20260508 09:55:44.447 | INFO | "huge" : "", | 20260508 09:55:44.447 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:44.447 | INFO | "malloc" : " 14 GB", | 20260508 09:55:44.448 | INFO | "pinned" : " 29 GB" | 20260508 09:55:44.448 | INFO | } | 20260508 09:55:44.448 | INFO | Perf report for the last 5.14303 seconds: | 20260508 09:55:44.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:44.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 09:55:44.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:55:44.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:44.448 | INFO | Uploaded 36GiB to GPU in 2.49509s (14.4284GiB/s) | 20260508 09:55:44.448 | INFO | Downloaded 34GiB to GPU in 2.42055s (14.0464GiB/s) | 20260508 09:55:49.587 | INFO | Analyzing chunk frames = [7194624,7195136) | 20260508 09:55:49.587 | INFO | MemInfo: { | 20260508 09:55:49.587 | INFO | "gpu" : " 17 GB", | 20260508 09:55:49.587 | INFO | "huge" : "", | 20260508 09:55:49.587 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:49.587 | INFO | "malloc" : " 14 GB", | 20260508 09:55:49.587 | INFO | "pinned" : " 29 GB" | 20260508 09:55:49.587 | INFO | } | 20260508 09:55:49.587 | INFO | Perf report for the last 5.1397 seconds: | 20260508 09:55:49.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:49.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.588 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 09:55:49.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.588 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:55:49.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:49.588 | INFO | Uploaded 36GiB to GPU in 2.43218s (14.8015GiB/s) | 20260508 09:55:49.588 | INFO | Downloaded 34GiB to GPU in 2.52081s (13.4877GiB/s) | 20260508 09:55:54.717 | INFO | Analyzing chunk frames = [7195136,7195648) | 20260508 09:55:54.717 | INFO | MemInfo: { | 20260508 09:55:54.717 | INFO | "gpu" : " 17 GB", | 20260508 09:55:54.717 | INFO | "huge" : "", | 20260508 09:55:54.717 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:54.717 | INFO | "malloc" : " 14 GB", | 20260508 09:55:54.717 | INFO | "pinned" : " 29 GB" | 20260508 09:55:54.717 | INFO | } | 20260508 09:55:54.717 | INFO | Perf report for the last 5.13004 seconds: | 20260508 09:55:54.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:54.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.718 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 09:55:54.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.718 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:55:54.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:54.718 | INFO | Uploaded 36GiB to GPU in 2.41805s (14.8881GiB/s) | 20260508 09:55:54.718 | INFO | Downloaded 34GiB to GPU in 2.43353s (13.9714GiB/s) | 20260508 09:55:59.863 | INFO | Analyzing chunk frames = [7195648,7196160) | 20260508 09:55:59.863 | INFO | MemInfo: { | 20260508 09:55:59.863 | INFO | "gpu" : " 17 GB", | 20260508 09:55:59.863 | INFO | "huge" : "", | 20260508 09:55:59.863 | INFO | "huge_shared" : " 29 GB", | 20260508 09:55:59.863 | INFO | "malloc" : " 14 GB", | 20260508 09:55:59.863 | INFO | "pinned" : " 29 GB" | 20260508 09:55:59.863 | INFO | } | 20260508 09:55:59.863 | INFO | Perf report for the last 5.14557 seconds: | 20260508 09:55:59.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:55:59.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.863 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 09:55:59.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.863 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:55:59.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:55:59.864 | INFO | Uploaded 36GiB to GPU in 2.44671s (14.7137GiB/s) | 20260508 09:55:59.864 | INFO | Downloaded 34GiB to GPU in 2.43155s (13.9829GiB/s) | 20260508 09:56:05.009 | INFO | Analyzing chunk frames = [7196160,7196672) | 20260508 09:56:05.009 | INFO | MemInfo: { | 20260508 09:56:05.009 | INFO | "gpu" : " 17 GB", | 20260508 09:56:05.009 | INFO | "huge" : "", | 20260508 09:56:05.009 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:05.009 | INFO | "malloc" : " 14 GB", | 20260508 09:56:05.009 | INFO | "pinned" : " 29 GB" | 20260508 09:56:05.009 | INFO | } | 20260508 09:56:05.009 | INFO | Perf report for the last 5.14595 seconds: | 20260508 09:56:05.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:05.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 09:56:05.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:56:05.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:05.009 | INFO | Uploaded 36GiB to GPU in 2.44406s (14.7296GiB/s) | 20260508 09:56:05.009 | INFO | Downloaded 34GiB to GPU in 2.43017s (13.9908GiB/s) | 20260508 09:56:05.107 | INFO | Starting a round of Baz IO | 20260508 09:56:08.518 | INFO | Populating RTMetrics report | 20260508 09:56:08.518 | INFO | RTMetrics report populated | 20260508 09:56:08.518 | INFO | Opening tmp RTMetrics json file | 20260508 09:56:08.528 | INFO | Serializing RTMetrics to json | 20260508 09:56:08.529 | INFO | Serialization done, swapping tmp file now... | 20260508 09:56:08.530 | INFO | RTMetrics done | 20260508 09:56:08.589 | INFO | Finished a round of Baz IO: Wrote 1.471 GiB in 3.482 seconds | 20260508 09:56:08.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:56:08.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238482% / 0% / 6.75519e-05% | 20260508 09:56:08.589 | INFO | Baz IO overall: | 20260508 09:56:08.589 | INFO | BytesWritten : 3132073072570 | 20260508 09:56:08.589 | INFO | NumEvents : 1506118061917 | 20260508 09:56:10.147 | INFO | Analyzing chunk frames = [7196672,7197184) | 20260508 09:56:10.147 | INFO | MemInfo: { | 20260508 09:56:10.147 | INFO | "gpu" : " 17 GB", | 20260508 09:56:10.147 | INFO | "huge" : "", | 20260508 09:56:10.147 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:10.147 | INFO | "malloc" : " 14 GB", | 20260508 09:56:10.147 | INFO | "pinned" : " 29 GB" | 20260508 09:56:10.147 | INFO | } | 20260508 09:56:10.147 | INFO | Perf report for the last 5.13802 seconds: | 20260508 09:56:10.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:10.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:10.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:10.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:10.147 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 09:56:10.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:10.147 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:56:10.147 | INFO | PRE_HQ : 65.5%, 1.6, 54.4ms, 0 | 20260508 09:56:10.147 | INFO | BAZWRITER : 67.2%, 2.5, 87.3ms, 0 | 20260508 09:56:10.147 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:56:10.147 | INFO | Uploaded 36GiB to GPU in 2.63891s (13.642GiB/s) | 20260508 09:56:10.147 | INFO | Downloaded 38GiB to GPU in 2.71045s (14.0198GiB/s) | 20260508 09:56:15.279 | INFO | Analyzing chunk frames = [7197184,7197696) | 20260508 09:56:15.279 | INFO | MemInfo: { | 20260508 09:56:15.279 | INFO | "gpu" : " 17 GB", | 20260508 09:56:15.279 | INFO | "huge" : "", | 20260508 09:56:15.279 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:15.280 | INFO | "malloc" : " 14 GB", | 20260508 09:56:15.280 | INFO | "pinned" : " 29 GB" | 20260508 09:56:15.280 | INFO | } | 20260508 09:56:15.280 | INFO | Perf report for the last 5.13254 seconds: | 20260508 09:56:15.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:15.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 09:56:15.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 09:56:15.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:15.280 | INFO | Uploaded 36GiB to GPU in 2.48439s (14.4905GiB/s) | 20260508 09:56:15.280 | INFO | Downloaded 34GiB to GPU in 2.45605s (13.8434GiB/s) | 20260508 09:56:20.424 | INFO | Analyzing chunk frames = [7197696,7198208) | 20260508 09:56:20.424 | INFO | MemInfo: { | 20260508 09:56:20.424 | INFO | "gpu" : " 17 GB", | 20260508 09:56:20.424 | INFO | "huge" : "", | 20260508 09:56:20.424 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:20.424 | INFO | "malloc" : " 14 GB", | 20260508 09:56:20.424 | INFO | "pinned" : " 29 GB" | 20260508 09:56:20.424 | INFO | } | 20260508 09:56:20.424 | INFO | Perf report for the last 5.14485 seconds: | 20260508 09:56:20.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:20.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 09:56:20.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 09:56:20.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:20.425 | INFO | Uploaded 36GiB to GPU in 2.4732s (14.5561GiB/s) | 20260508 09:56:20.425 | INFO | Downloaded 34GiB to GPU in 2.40955s (14.1105GiB/s) | 20260508 09:56:25.558 | INFO | Analyzing chunk frames = [7198208,7198720) | 20260508 09:56:25.558 | INFO | MemInfo: { | 20260508 09:56:25.558 | INFO | "gpu" : " 17 GB", | 20260508 09:56:25.558 | INFO | "huge" : "", | 20260508 09:56:25.558 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:25.558 | INFO | "malloc" : " 14 GB", | 20260508 09:56:25.558 | INFO | "pinned" : " 29 GB" | 20260508 09:56:25.558 | INFO | } | 20260508 09:56:25.558 | INFO | Perf report for the last 5.13371 seconds: | 20260508 09:56:25.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:25.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.558 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 09:56:25.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.558 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 09:56:25.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:25.559 | INFO | Uploaded 36GiB to GPU in 2.49816s (14.4106GiB/s) | 20260508 09:56:25.559 | INFO | Downloaded 34GiB to GPU in 2.45063s (13.874GiB/s) | 20260508 09:56:30.699 | INFO | Analyzing chunk frames = [7198720,7199232) | 20260508 09:56:30.699 | INFO | MemInfo: { | 20260508 09:56:30.699 | INFO | "gpu" : " 17 GB", | 20260508 09:56:30.699 | INFO | "huge" : "", | 20260508 09:56:30.699 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:30.699 | INFO | "malloc" : " 14 GB", | 20260508 09:56:30.699 | INFO | "pinned" : " 29 GB" | 20260508 09:56:30.699 | INFO | } | 20260508 09:56:30.699 | INFO | Perf report for the last 5.141 seconds: | 20260508 09:56:30.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:30.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.699 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 09:56:30.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.699 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:56:30.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:30.700 | INFO | Uploaded 36GiB to GPU in 2.39531s (15.0294GiB/s) | 20260508 09:56:30.700 | INFO | Downloaded 34GiB to GPU in 2.52278s (13.4772GiB/s) | 20260508 09:56:35.837 | INFO | Analyzing chunk frames = [7199232,7199744) | 20260508 09:56:35.837 | INFO | MemInfo: { | 20260508 09:56:35.837 | INFO | "gpu" : " 17 GB", | 20260508 09:56:35.837 | INFO | "huge" : "", | 20260508 09:56:35.837 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:35.837 | INFO | "malloc" : " 14 GB", | 20260508 09:56:35.837 | INFO | "pinned" : " 29 GB" | 20260508 09:56:35.837 | INFO | } | 20260508 09:56:35.837 | INFO | Perf report for the last 5.13821 seconds: | 20260508 09:56:35.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:35.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.837 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 09:56:35.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.838 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:56:35.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:35.838 | INFO | Uploaded 36GiB to GPU in 2.40202s (14.9874GiB/s) | 20260508 09:56:35.838 | INFO | Downloaded 34GiB to GPU in 2.42755s (14.0059GiB/s) | 20260508 09:56:40.977 | INFO | Analyzing chunk frames = [7199744,7200256) | 20260508 09:56:40.977 | INFO | MemInfo: { | 20260508 09:56:40.977 | INFO | "gpu" : " 17 GB", | 20260508 09:56:40.977 | INFO | "huge" : "", | 20260508 09:56:40.977 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:40.977 | INFO | "malloc" : " 14 GB", | 20260508 09:56:40.977 | INFO | "pinned" : " 29 GB" | 20260508 09:56:40.977 | INFO | } | 20260508 09:56:40.977 | INFO | Perf report for the last 5.13959 seconds: | 20260508 09:56:40.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:40.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 09:56:40.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 09:56:40.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:40.977 | INFO | Uploaded 36GiB to GPU in 2.44964s (14.696GiB/s) | 20260508 09:56:40.977 | INFO | Downloaded 34GiB to GPU in 2.42931s (13.9957GiB/s) | 20260508 09:56:46.120 | INFO | Analyzing chunk frames = [7200256,7200768) | 20260508 09:56:46.120 | INFO | MemInfo: { | 20260508 09:56:46.120 | INFO | "gpu" : " 17 GB", | 20260508 09:56:46.120 | INFO | "huge" : "", | 20260508 09:56:46.120 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:46.120 | INFO | "malloc" : " 14 GB", | 20260508 09:56:46.120 | INFO | "pinned" : " 29 GB" | 20260508 09:56:46.120 | INFO | } | 20260508 09:56:46.120 | INFO | Perf report for the last 5.14282 seconds: | 20260508 09:56:46.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:46.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 09:56:46.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:56:46.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:46.120 | INFO | Uploaded 36GiB to GPU in 2.45834s (14.644GiB/s) | 20260508 09:56:46.120 | INFO | Downloaded 34GiB to GPU in 2.43357s (13.9713GiB/s) | 20260508 09:56:46.120 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:56:46.120 | INFO | Total mem usage: 16.7 GiB | 20260508 09:56:46.120 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:56:46.120 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:56:46.120 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:56:46.120 | INFO | High Water Breakdown: | 20260508 09:56:46.120 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:56:46.120 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:56:46.120 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:56:46.120 | INFO | ------------------------------------------ | 20260508 09:56:46.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:56:46.121 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:56:46.121 | INFO | Total mem usage: 26.5 GiB | 20260508 09:56:46.121 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:56:46.121 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:56:46.121 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:56:46.121 | INFO | High Water Breakdown: | 20260508 09:56:46.121 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:56:46.121 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:56:46.121 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:56:46.121 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:56:46.121 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:56:46.121 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:56:46.121 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:56:46.121 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:56:46.121 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:56:46.121 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:56:46.121 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:56:46.121 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:56:46.121 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:56:46.121 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:56:46.121 | INFO | ------------------------------------------ | 20260508 09:56:46.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:56:46.121 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:56:46.121 | INFO | Total mem usage: 27.8 GiB | 20260508 09:56:46.121 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:56:46.121 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:56:46.121 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:56:46.121 | INFO | High Water Breakdown: | 20260508 09:56:46.121 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:56:46.121 | INFO | ------------------------------------------ | 20260508 09:56:46.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:56:46.121 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:56:46.121 | INFO | Total mem usage: 15.7 GiB | 20260508 09:56:46.121 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:56:46.121 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:56:46.121 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:56:46.121 | INFO | High Water Breakdown: | 20260508 09:56:46.121 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:56:46.121 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:56:46.121 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:56:46.121 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:56:46.121 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:56:46.121 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:56:46.121 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:56:46.121 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:56:46.121 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:56:46.121 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:56:46.121 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:56:46.121 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:56:46.121 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:56:46.121 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:56:46.121 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:56:46.121 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:56:46.121 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:56:46.121 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:56:46.121 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:56:46.121 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:56:46.121 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:56:46.121 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:56:46.121 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:56:46.121 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:56:46.121 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:56:46.121 | INFO | ------------------------------------------ | 20260508 09:56:46.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:56:46.121 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:56:46.121 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:56:46.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:56:46.122 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.39e+01 / 5.38e-01 | 20260508 09:56:46.122 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.23e-02 / 7.31e-03 | 20260508 09:56:46.122 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.45e+00, 5.95e-01 / 2.29e-01 | 20260508 09:56:46.122 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.27e-01 | 20260508 09:56:46.122 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.51e+01, 3.78e+00 / 1.18e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.81e+00 / 1.28e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.53e+00, 4.07e-01 / 2.53e-01 | 20260508 09:56:46.122 | INFO | PerfReport: 3.03%, Metrics -- 9.90e-01 / 1.13e+00, 2.49e-01 / 1.12e-01 | 20260508 09:56:46.122 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.33e-01, 4.88e-02 / 2.67e-02 | 20260508 09:56:46.122 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:56:46.122 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:56:46.122 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:56:46.122 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 24.45%, Upload -- 2.58e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 24.52%, Download -- 2.58e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 2.27%, LockUpload -- 2.39e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 2.48%, LockDownload -- 2.61e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 15.24%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:56:46.122 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260508 09:56:46.220 | INFO | Starting a round of Baz IO | 20260508 09:56:49.660 | INFO | Populating RTMetrics report | 20260508 09:56:49.660 | INFO | RTMetrics report populated | 20260508 09:56:49.660 | INFO | Opening tmp RTMetrics json file | 20260508 09:56:49.661 | INFO | Serializing RTMetrics to json | 20260508 09:56:49.663 | INFO | Serialization done, swapping tmp file now... | 20260508 09:56:49.663 | INFO | RTMetrics done | 20260508 09:56:49.746 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.525 seconds | 20260508 09:56:49.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:56:49.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.237926% / 0% / 5.96046e-05% | 20260508 09:56:49.746 | INFO | Baz IO overall: | 20260508 09:56:49.746 | INFO | BytesWritten : 3133651587241 | 20260508 09:56:49.746 | INFO | NumEvents : 1506863104487 | 20260508 09:56:51.257 | INFO | Analyzing chunk frames = [7200768,7201280) | 20260508 09:56:51.257 | INFO | MemInfo: { | 20260508 09:56:51.257 | INFO | "gpu" : " 17 GB", | 20260508 09:56:51.257 | INFO | "huge" : "", | 20260508 09:56:51.257 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:51.257 | INFO | "malloc" : " 14 GB", | 20260508 09:56:51.257 | INFO | "pinned" : " 29 GB" | 20260508 09:56:51.257 | INFO | } | 20260508 09:56:51.257 | INFO | Perf report for the last 5.13733 seconds: | 20260508 09:56:51.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:51.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:51.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:51.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:51.257 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 09:56:51.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:51.257 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 09:56:51.257 | INFO | PRE_HQ : 66.1%, 1.6, 55.8ms, 0 | 20260508 09:56:51.258 | INFO | BAZWRITER : 68.0%, 2.5, 88.4ms, 0 | 20260508 09:56:51.258 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 09:56:51.258 | INFO | Uploaded 36GiB to GPU in 2.6018s (13.8366GiB/s) | 20260508 09:56:51.258 | INFO | Downloaded 38GiB to GPU in 2.70389s (14.0538GiB/s) | 20260508 09:56:56.397 | INFO | Analyzing chunk frames = [7201280,7201792) | 20260508 09:56:56.397 | INFO | MemInfo: { | 20260508 09:56:56.397 | INFO | "gpu" : " 17 GB", | 20260508 09:56:56.397 | INFO | "huge" : "", | 20260508 09:56:56.397 | INFO | "huge_shared" : " 29 GB", | 20260508 09:56:56.397 | INFO | "malloc" : " 14 GB", | 20260508 09:56:56.397 | INFO | "pinned" : " 29 GB" | 20260508 09:56:56.397 | INFO | } | 20260508 09:56:56.397 | INFO | Perf report for the last 5.13994 seconds: | 20260508 09:56:56.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:56:56.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.397 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 09:56:56.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.397 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:56:56.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:56:56.398 | INFO | Uploaded 36GiB to GPU in 2.52821s (14.2393GiB/s) | 20260508 09:56:56.398 | INFO | Downloaded 34GiB to GPU in 2.47269s (13.7502GiB/s) | 20260508 09:57:01.532 | INFO | Analyzing chunk frames = [7201792,7202304) | 20260508 09:57:01.532 | INFO | MemInfo: { | 20260508 09:57:01.532 | INFO | "gpu" : " 17 GB", | 20260508 09:57:01.532 | INFO | "huge" : "", | 20260508 09:57:01.532 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:01.532 | INFO | "malloc" : " 14 GB", | 20260508 09:57:01.532 | INFO | "pinned" : " 29 GB" | 20260508 09:57:01.532 | INFO | } | 20260508 09:57:01.532 | INFO | Perf report for the last 5.1349 seconds: | 20260508 09:57:01.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:01.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.532 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 09:57:01.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.532 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:57:01.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:01.533 | INFO | Uploaded 36GiB to GPU in 2.47575s (14.541GiB/s) | 20260508 09:57:01.533 | INFO | Downloaded 34GiB to GPU in 2.44049s (13.9316GiB/s) | 20260508 09:57:06.674 | INFO | Analyzing chunk frames = [7202304,7202816) | 20260508 09:57:06.674 | INFO | MemInfo: { | 20260508 09:57:06.674 | INFO | "gpu" : " 17 GB", | 20260508 09:57:06.674 | INFO | "huge" : "", | 20260508 09:57:06.674 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:06.674 | INFO | "malloc" : " 14 GB", | 20260508 09:57:06.674 | INFO | "pinned" : " 29 GB" | 20260508 09:57:06.674 | INFO | } | 20260508 09:57:06.674 | INFO | Perf report for the last 5.1422 seconds: | 20260508 09:57:06.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:06.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.674 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 09:57:06.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.674 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:57:06.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:06.675 | INFO | Uploaded 36GiB to GPU in 2.47937s (14.5198GiB/s) | 20260508 09:57:06.675 | INFO | Downloaded 34GiB to GPU in 2.42178s (14.0393GiB/s) | 20260508 09:57:11.815 | INFO | Analyzing chunk frames = [7202816,7203328) | 20260508 09:57:11.815 | INFO | MemInfo: { | 20260508 09:57:11.815 | INFO | "gpu" : " 17 GB", | 20260508 09:57:11.815 | INFO | "huge" : "", | 20260508 09:57:11.815 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:11.816 | INFO | "malloc" : " 14 GB", | 20260508 09:57:11.816 | INFO | "pinned" : " 29 GB" | 20260508 09:57:11.816 | INFO | } | 20260508 09:57:11.816 | INFO | Perf report for the last 5.14137 seconds: | 20260508 09:57:11.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:11.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 09:57:11.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:57:11.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:11.816 | INFO | Uploaded 36GiB to GPU in 2.31897s (15.5241GiB/s) | 20260508 09:57:11.816 | INFO | Downloaded 34GiB to GPU in 2.51675s (13.5095GiB/s) | 20260508 09:57:16.957 | INFO | Analyzing chunk frames = [7203328,7203840) | 20260508 09:57:16.957 | INFO | MemInfo: { | 20260508 09:57:16.957 | INFO | "gpu" : " 17 GB", | 20260508 09:57:16.957 | INFO | "huge" : "", | 20260508 09:57:16.957 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:16.957 | INFO | "malloc" : " 14 GB", | 20260508 09:57:16.957 | INFO | "pinned" : " 29 GB" | 20260508 09:57:16.957 | INFO | } | 20260508 09:57:16.958 | INFO | Perf report for the last 5.14182 seconds: | 20260508 09:57:16.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:16.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 09:57:16.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:57:16.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:16.958 | INFO | Uploaded 36GiB to GPU in 2.34128s (15.3762GiB/s) | 20260508 09:57:16.958 | INFO | Downloaded 34GiB to GPU in 2.45371s (13.8566GiB/s) | 20260508 09:57:22.091 | INFO | Analyzing chunk frames = [7203840,7204352) | 20260508 09:57:22.091 | INFO | MemInfo: { | 20260508 09:57:22.091 | INFO | "gpu" : " 17 GB", | 20260508 09:57:22.091 | INFO | "huge" : "", | 20260508 09:57:22.092 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:22.092 | INFO | "malloc" : " 14 GB", | 20260508 09:57:22.092 | INFO | "pinned" : " 29 GB" | 20260508 09:57:22.092 | INFO | } | 20260508 09:57:22.092 | INFO | Perf report for the last 5.13414 seconds: | 20260508 09:57:22.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:22.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 09:57:22.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:57:22.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:22.092 | INFO | Uploaded 36GiB to GPU in 2.38634s (15.0859GiB/s) | 20260508 09:57:22.092 | INFO | Downloaded 34GiB to GPU in 2.44438s (13.9095GiB/s) | 20260508 09:57:27.228 | INFO | Analyzing chunk frames = [7204352,7204864) | 20260508 09:57:27.228 | INFO | MemInfo: { | 20260508 09:57:27.228 | INFO | "gpu" : " 17 GB", | 20260508 09:57:27.228 | INFO | "huge" : "", | 20260508 09:57:27.229 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:27.229 | INFO | "malloc" : " 14 GB", | 20260508 09:57:27.229 | INFO | "pinned" : " 29 GB" | 20260508 09:57:27.229 | INFO | } | 20260508 09:57:27.229 | INFO | Perf report for the last 5.13711 seconds: | 20260508 09:57:27.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:27.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 09:57:27.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 09:57:27.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:27.229 | INFO | Uploaded 36GiB to GPU in 2.39025s (15.0612GiB/s) | 20260508 09:57:27.230 | INFO | Downloaded 34GiB to GPU in 2.43796s (13.9461GiB/s) | 20260508 09:57:27.342 | INFO | Starting a round of Baz IO | 20260508 09:57:30.759 | INFO | Populating RTMetrics report | 20260508 09:57:30.760 | INFO | RTMetrics report populated | 20260508 09:57:30.760 | INFO | Opening tmp RTMetrics json file | 20260508 09:57:30.766 | INFO | Serializing RTMetrics to json | 20260508 09:57:30.768 | INFO | Serialization done, swapping tmp file now... | 20260508 09:57:30.769 | INFO | RTMetrics done | 20260508 09:57:30.825 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.482 seconds | 20260508 09:57:30.825 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:57:30.825 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.237246% / 0% / 8.34465e-05% | 20260508 09:57:30.825 | INFO | Baz IO overall: | 20260508 09:57:30.825 | INFO | BytesWritten : 3135229669340 | 20260508 09:57:30.825 | INFO | NumEvents : 1507607809374 | 20260508 09:57:32.372 | INFO | Analyzing chunk frames = [7204864,7205376) | 20260508 09:57:32.372 | INFO | MemInfo: { | 20260508 09:57:32.372 | INFO | "gpu" : " 17 GB", | 20260508 09:57:32.373 | INFO | "huge" : "", | 20260508 09:57:32.373 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:32.373 | INFO | "malloc" : " 14 GB", | 20260508 09:57:32.373 | INFO | "pinned" : " 29 GB" | 20260508 09:57:32.373 | INFO | } | 20260508 09:57:32.373 | INFO | Perf report for the last 5.14401 seconds: | 20260508 09:57:32.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:32.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:32.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:32.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:32.373 | INFO | ANALYSIS : 67.8%, 3.0, 105.9ms, 0 | 20260508 09:57:32.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:32.373 | INFO | RT_METRICS : 43.5%, 1.0, 22.9ms, 0 | 20260508 09:57:32.373 | INFO | PRE_HQ : 66.5%, 1.8, 62.4ms, 0 | 20260508 09:57:32.373 | INFO | BAZWRITER : 67.0%, 2.7, 93.2ms, 0 | 20260508 09:57:32.373 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:57:32.373 | INFO | Uploaded 36GiB to GPU in 2.60376s (13.8262GiB/s) | 20260508 09:57:32.373 | INFO | Downloaded 38GiB to GPU in 2.71508s (13.9959GiB/s) | 20260508 09:57:37.514 | INFO | Analyzing chunk frames = [7205376,7205888) | 20260508 09:57:37.514 | INFO | MemInfo: { | 20260508 09:57:37.514 | INFO | "gpu" : " 17 GB", | 20260508 09:57:37.514 | INFO | "huge" : "", | 20260508 09:57:37.514 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:37.514 | INFO | "malloc" : " 14 GB", | 20260508 09:57:37.514 | INFO | "pinned" : " 29 GB" | 20260508 09:57:37.514 | INFO | } | 20260508 09:57:37.514 | INFO | Perf report for the last 5.14088 seconds: | 20260508 09:57:37.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:37.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 09:57:37.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:57:37.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:37.514 | INFO | Uploaded 36GiB to GPU in 2.52285s (14.2696GiB/s) | 20260508 09:57:37.515 | INFO | Downloaded 34GiB to GPU in 2.4384s (13.9436GiB/s) | 20260508 09:57:42.653 | INFO | Analyzing chunk frames = [7205888,7206400) | 20260508 09:57:42.653 | INFO | MemInfo: { | 20260508 09:57:42.653 | INFO | "gpu" : " 17 GB", | 20260508 09:57:42.653 | INFO | "huge" : "", | 20260508 09:57:42.653 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:42.653 | INFO | "malloc" : " 14 GB", | 20260508 09:57:42.653 | INFO | "pinned" : " 29 GB" | 20260508 09:57:42.653 | INFO | } | 20260508 09:57:42.653 | INFO | Perf report for the last 5.13949 seconds: | 20260508 09:57:42.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:42.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.653 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:57:42.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.654 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:57:42.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:42.654 | INFO | Uploaded 36GiB to GPU in 2.4542s (14.6687GiB/s) | 20260508 09:57:42.654 | INFO | Downloaded 34GiB to GPU in 2.43836s (13.9438GiB/s) | 20260508 09:57:47.791 | INFO | Analyzing chunk frames = [7206400,7206912) | 20260508 09:57:47.791 | INFO | MemInfo: { | 20260508 09:57:47.791 | INFO | "gpu" : " 17 GB", | 20260508 09:57:47.791 | INFO | "huge" : "", | 20260508 09:57:47.791 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:47.791 | INFO | "malloc" : " 14 GB", | 20260508 09:57:47.791 | INFO | "pinned" : " 29 GB" | 20260508 09:57:47.792 | INFO | } | 20260508 09:57:47.792 | INFO | Perf report for the last 5.13837 seconds: | 20260508 09:57:47.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:47.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 09:57:47.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 09:57:47.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:47.792 | INFO | Uploaded 36GiB to GPU in 2.49311s (14.4398GiB/s) | 20260508 09:57:47.792 | INFO | Downloaded 34GiB to GPU in 2.42593s (14.0153GiB/s) | 20260508 09:57:52.929 | INFO | Analyzing chunk frames = [7206912,7207424) | 20260508 09:57:52.929 | INFO | MemInfo: { | 20260508 09:57:52.929 | INFO | "gpu" : " 17 GB", | 20260508 09:57:52.929 | INFO | "huge" : "", | 20260508 09:57:52.929 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:52.929 | INFO | "malloc" : " 14 GB", | 20260508 09:57:52.929 | INFO | "pinned" : " 29 GB" | 20260508 09:57:52.929 | INFO | } | 20260508 09:57:52.929 | INFO | Perf report for the last 5.13785 seconds: | 20260508 09:57:52.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:52.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.930 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 09:57:52.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.930 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 09:57:52.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:52.930 | INFO | Uploaded 36GiB to GPU in 2.30352s (15.6282GiB/s) | 20260508 09:57:52.930 | INFO | Downloaded 34GiB to GPU in 2.50375s (13.5796GiB/s) | 20260508 09:57:58.070 | INFO | Analyzing chunk frames = [7207424,7207936) | 20260508 09:57:58.070 | INFO | MemInfo: { | 20260508 09:57:58.070 | INFO | "gpu" : " 17 GB", | 20260508 09:57:58.070 | INFO | "huge" : "", | 20260508 09:57:58.070 | INFO | "huge_shared" : " 29 GB", | 20260508 09:57:58.070 | INFO | "malloc" : " 14 GB", | 20260508 09:57:58.070 | INFO | "pinned" : " 29 GB" | 20260508 09:57:58.070 | INFO | } | 20260508 09:57:58.070 | INFO | Perf report for the last 5.1407 seconds: | 20260508 09:57:58.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:57:58.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 09:57:58.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 09:57:58.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:57:58.070 | INFO | Uploaded 36GiB to GPU in 2.40217s (14.9864GiB/s) | 20260508 09:57:58.070 | INFO | Downloaded 34GiB to GPU in 2.4234s (14.0299GiB/s) | 20260508 09:58:03.207 | INFO | Analyzing chunk frames = [7207936,7208448) | 20260508 09:58:03.207 | INFO | MemInfo: { | 20260508 09:58:03.207 | INFO | "gpu" : " 17 GB", | 20260508 09:58:03.207 | INFO | "huge" : "", | 20260508 09:58:03.207 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:03.207 | INFO | "malloc" : " 14 GB", | 20260508 09:58:03.207 | INFO | "pinned" : " 29 GB" | 20260508 09:58:03.207 | INFO | } | 20260508 09:58:03.207 | INFO | Perf report for the last 5.13655 seconds: | 20260508 09:58:03.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:03.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 09:58:03.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 09:58:03.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:03.207 | INFO | Uploaded 36GiB to GPU in 2.46388s (14.6111GiB/s) | 20260508 09:58:03.207 | INFO | Downloaded 34GiB to GPU in 2.43958s (13.9368GiB/s) | 20260508 09:58:08.351 | INFO | Analyzing chunk frames = [7208448,7208960) | 20260508 09:58:08.351 | INFO | MemInfo: { | 20260508 09:58:08.351 | INFO | "gpu" : " 17 GB", | 20260508 09:58:08.351 | INFO | "huge" : "", | 20260508 09:58:08.351 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:08.351 | INFO | "malloc" : " 14 GB", | 20260508 09:58:08.351 | INFO | "pinned" : " 29 GB" | 20260508 09:58:08.351 | INFO | } | 20260508 09:58:08.351 | INFO | Perf report for the last 5.14428 seconds: | 20260508 09:58:08.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:08.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | ANALYSIS : 65.5%, 3.0, 102.6ms, 0 | 20260508 09:58:08.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:58:08.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:08.351 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:58:08.351 | INFO | Total mem usage: 16.7 GiB | 20260508 09:58:08.351 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:58:08.351 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 09:58:08.351 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:58:08.352 | INFO | High Water Breakdown: | 20260508 09:58:08.352 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:58:08.352 | INFO | 40.3% : 5.78 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:58:08.352 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:58:08.352 | INFO | ------------------------------------------ | 20260508 09:58:08.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:58:08.352 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:58:08.352 | INFO | Total mem usage: 26.5 GiB | 20260508 09:58:08.352 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:58:08.352 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 09:58:08.352 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 09:58:08.352 | INFO | High Water Breakdown: | 20260508 09:58:08.352 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:58:08.352 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:58:08.352 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:58:08.352 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:58:08.352 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:58:08.352 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:58:08.352 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:58:08.352 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:58:08.352 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:58:08.352 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:58:08.352 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:58:08.352 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:58:08.352 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:58:08.352 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:58:08.352 | INFO | ------------------------------------------ | 20260508 09:58:08.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:58:08.352 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:58:08.352 | INFO | Total mem usage: 27.8 GiB | 20260508 09:58:08.352 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:58:08.352 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:58:08.352 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:58:08.352 | INFO | High Water Breakdown: | 20260508 09:58:08.352 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:58:08.352 | INFO | ------------------------------------------ | 20260508 09:58:08.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:58:08.352 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:58:08.352 | INFO | Total mem usage: 15.7 GiB | 20260508 09:58:08.352 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:58:08.352 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:58:08.352 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:58:08.352 | INFO | High Water Breakdown: | 20260508 09:58:08.352 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:58:08.352 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:58:08.352 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:58:08.352 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:58:08.352 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:58:08.352 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:58:08.352 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:58:08.352 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:58:08.352 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:58:08.352 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:58:08.352 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:58:08.352 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:58:08.352 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:58:08.352 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:58:08.352 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:58:08.352 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:58:08.352 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:58:08.352 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:58:08.352 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:58:08.352 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:58:08.352 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:58:08.352 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:58:08.352 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:58:08.352 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:58:08.352 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:58:08.352 | INFO | ------------------------------------------ | 20260508 09:58:08.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:58:08.352 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:58:08.352 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:58:08.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:58:08.352 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.39e+01 / 7.53e-01 | 20260508 09:58:08.352 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.16e-02 / 4.10e-03 | 20260508 09:58:08.352 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.39e+00, 5.93e-01 / 2.66e-01 | 20260508 09:58:08.352 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.42e-01 | 20260508 09:58:08.352 | INFO | PerfReport: 16.00%, DME -- 4.20e+01 / 4.55e+01, 3.85e+00 / 1.37e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.84e+00 / 1.30e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.34e+00 / 2.55e+00, 4.08e-01 / 2.47e-01 | 20260508 09:58:08.352 | INFO | PerfReport: 3.02%, Metrics -- 9.90e-01 / 1.10e+00, 2.47e-01 / 1.14e-01 | 20260508 09:58:08.352 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.31e-01, 4.88e-02 / 2.92e-02 | 20260508 09:58:08.352 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:58:08.352 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:58:08.352 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:58:08.352 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.45e+01 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 24.53%, Download -- 2.58e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 31.13%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 2.53%, LockDownload -- 2.67e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 15.42%, LockCompute -- 1.62e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:58:08.352 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 09:58:08.352 | INFO | Uploaded 36GiB to GPU in 2.41987s (14.8768GiB/s) | 20260508 09:58:08.352 | INFO | Downloaded 34GiB to GPU in 2.41863s (14.0576GiB/s) | 20260508 09:58:08.449 | INFO | Starting a round of Baz IO | 20260508 09:58:11.866 | INFO | Populating RTMetrics report | 20260508 09:58:11.866 | INFO | RTMetrics report populated | 20260508 09:58:11.866 | INFO | Opening tmp RTMetrics json file | 20260508 09:58:11.874 | INFO | Serializing RTMetrics to json | 20260508 09:58:11.876 | INFO | Serialization done, swapping tmp file now... | 20260508 09:58:11.877 | INFO | RTMetrics done | 20260508 09:58:11.933 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.484 seconds | 20260508 09:58:11.934 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:58:11.934 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238287% / 7.94729e-06% / 5.5631e-05% | 20260508 09:58:11.934 | INFO | Baz IO overall: | 20260508 09:58:11.934 | INFO | BytesWritten : 3136808437846 | 20260508 09:58:11.934 | INFO | NumEvents : 1508352992274 | 20260508 09:58:13.487 | INFO | Analyzing chunk frames = [7208960,7209472) | 20260508 09:58:13.487 | INFO | MemInfo: { | 20260508 09:58:13.487 | INFO | "gpu" : " 17 GB", | 20260508 09:58:13.487 | INFO | "huge" : "", | 20260508 09:58:13.487 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:13.487 | INFO | "malloc" : " 14 GB", | 20260508 09:58:13.487 | INFO | "pinned" : " 29 GB" | 20260508 09:58:13.487 | INFO | } | 20260508 09:58:13.487 | INFO | Perf report for the last 5.13598 seconds: | 20260508 09:58:13.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:13.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:13.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:13.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:13.487 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 09:58:13.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:13.487 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 09:58:13.487 | INFO | PRE_HQ : 65.3%, 1.6, 54.3ms, 0 | 20260508 09:58:13.487 | INFO | BAZWRITER : 67.2%, 2.5, 87.4ms, 0 | 20260508 09:58:13.487 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 09:58:13.488 | INFO | Uploaded 36GiB to GPU in 2.59611s (13.8669GiB/s) | 20260508 09:58:13.488 | INFO | Downloaded 38GiB to GPU in 2.70613s (14.0422GiB/s) | 20260508 09:58:18.630 | INFO | Analyzing chunk frames = [7209472,7209984) | 20260508 09:58:18.630 | INFO | MemInfo: { | 20260508 09:58:18.630 | INFO | "gpu" : " 17 GB", | 20260508 09:58:18.630 | INFO | "huge" : "", | 20260508 09:58:18.630 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:18.630 | INFO | "malloc" : " 14 GB", | 20260508 09:58:18.630 | INFO | "pinned" : " 29 GB" | 20260508 09:58:18.630 | INFO | } | 20260508 09:58:18.630 | INFO | Perf report for the last 5.1434 seconds: | 20260508 09:58:18.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:18.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.630 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 09:58:18.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.631 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:58:18.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:18.631 | INFO | Uploaded 36GiB to GPU in 2.49994s (14.4004GiB/s) | 20260508 09:58:18.631 | INFO | Downloaded 34GiB to GPU in 2.4626s (13.8066GiB/s) | 20260508 09:58:23.769 | INFO | Analyzing chunk frames = [7209984,7210496) | 20260508 09:58:23.769 | INFO | MemInfo: { | 20260508 09:58:23.769 | INFO | "gpu" : " 17 GB", | 20260508 09:58:23.769 | INFO | "huge" : "", | 20260508 09:58:23.769 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:23.769 | INFO | "malloc" : " 14 GB", | 20260508 09:58:23.769 | INFO | "pinned" : " 29 GB" | 20260508 09:58:23.769 | INFO | } | 20260508 09:58:23.769 | INFO | Perf report for the last 5.13895 seconds: | 20260508 09:58:23.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:23.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.769 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:58:23.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.769 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 09:58:23.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:23.770 | INFO | Uploaded 36GiB to GPU in 2.49892s (14.4062GiB/s) | 20260508 09:58:23.770 | INFO | Downloaded 34GiB to GPU in 2.42842s (14.0009GiB/s) | 20260508 09:58:28.902 | INFO | Analyzing chunk frames = [7210496,7211008) | 20260508 09:58:28.902 | INFO | MemInfo: { | 20260508 09:58:28.902 | INFO | "gpu" : " 17 GB", | 20260508 09:58:28.902 | INFO | "huge" : "", | 20260508 09:58:28.902 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:28.902 | INFO | "malloc" : " 14 GB", | 20260508 09:58:28.902 | INFO | "pinned" : " 29 GB" | 20260508 09:58:28.902 | INFO | } | 20260508 09:58:28.902 | INFO | Perf report for the last 5.13239 seconds: | 20260508 09:58:28.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:28.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | ANALYSIS : 66.6%, 3.0, 103.1ms, 0 | 20260508 09:58:28.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 09:58:28.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:28.902 | INFO | Uploaded 36GiB to GPU in 2.52828s (14.239GiB/s) | 20260508 09:58:28.902 | INFO | Downloaded 34GiB to GPU in 2.42043s (14.0471GiB/s) | 20260508 09:58:34.039 | INFO | Analyzing chunk frames = [7211008,7211520) | 20260508 09:58:34.039 | INFO | MemInfo: { | 20260508 09:58:34.039 | INFO | "gpu" : " 17 GB", | 20260508 09:58:34.039 | INFO | "huge" : "", | 20260508 09:58:34.039 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:34.039 | INFO | "malloc" : " 14 GB", | 20260508 09:58:34.039 | INFO | "pinned" : " 29 GB" | 20260508 09:58:34.039 | INFO | } | 20260508 09:58:34.039 | INFO | Perf report for the last 5.13704 seconds: | 20260508 09:58:34.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:34.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 09:58:34.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 09:58:34.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:34.039 | INFO | Uploaded 36GiB to GPU in 2.50444s (14.3745GiB/s) | 20260508 09:58:34.040 | INFO | Downloaded 34GiB to GPU in 2.53226s (13.4268GiB/s) | 20260508 09:58:39.184 | INFO | Analyzing chunk frames = [7211520,7212032) | 20260508 09:58:39.184 | INFO | MemInfo: { | 20260508 09:58:39.184 | INFO | "gpu" : " 17 GB", | 20260508 09:58:39.184 | INFO | "huge" : "", | 20260508 09:58:39.184 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:39.184 | INFO | "malloc" : " 14 GB", | 20260508 09:58:39.184 | INFO | "pinned" : " 29 GB" | 20260508 09:58:39.184 | INFO | } | 20260508 09:58:39.184 | INFO | Perf report for the last 5.14512 seconds: | 20260508 09:58:39.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:39.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.184 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 09:58:39.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.184 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 09:58:39.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:39.185 | INFO | Uploaded 36GiB to GPU in 2.38597s (15.0882GiB/s) | 20260508 09:58:39.185 | INFO | Downloaded 34GiB to GPU in 2.43836s (13.9438GiB/s) | 20260508 09:58:44.322 | INFO | Analyzing chunk frames = [7212032,7212544) | 20260508 09:58:44.322 | INFO | MemInfo: { | 20260508 09:58:44.322 | INFO | "gpu" : " 17 GB", | 20260508 09:58:44.322 | INFO | "huge" : "", | 20260508 09:58:44.322 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:44.322 | INFO | "malloc" : " 14 GB", | 20260508 09:58:44.322 | INFO | "pinned" : " 29 GB" | 20260508 09:58:44.322 | INFO | } | 20260508 09:58:44.322 | INFO | Perf report for the last 5.13837 seconds: | 20260508 09:58:44.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:44.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.322 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 09:58:44.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.322 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:58:44.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:44.323 | INFO | Uploaded 36GiB to GPU in 2.44451s (14.7269GiB/s) | 20260508 09:58:44.323 | INFO | Downloaded 34GiB to GPU in 2.42541s (14.0182GiB/s) | 20260508 09:58:49.464 | INFO | Analyzing chunk frames = [7212544,7213056) | 20260508 09:58:49.464 | INFO | MemInfo: { | 20260508 09:58:49.464 | INFO | "gpu" : " 17 GB", | 20260508 09:58:49.464 | INFO | "huge" : "", | 20260508 09:58:49.464 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:49.464 | INFO | "malloc" : " 14 GB", | 20260508 09:58:49.464 | INFO | "pinned" : " 29 GB" | 20260508 09:58:49.464 | INFO | } | 20260508 09:58:49.464 | INFO | Perf report for the last 5.14184 seconds: | 20260508 09:58:49.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:49.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.464 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 09:58:49.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.464 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:58:49.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:49.465 | INFO | Uploaded 36GiB to GPU in 2.42914s (14.82GiB/s) | 20260508 09:58:49.465 | INFO | Downloaded 34GiB to GPU in 2.43357s (13.9712GiB/s) | 20260508 09:58:49.562 | INFO | Starting a round of Baz IO | 20260508 09:58:52.984 | INFO | Populating RTMetrics report | 20260508 09:58:52.984 | INFO | RTMetrics report populated | 20260508 09:58:52.984 | INFO | Opening tmp RTMetrics json file | 20260508 09:58:52.994 | INFO | Serializing RTMetrics to json | 20260508 09:58:52.996 | INFO | Serialization done, swapping tmp file now... | 20260508 09:58:52.996 | INFO | RTMetrics done | 20260508 09:58:53.054 | INFO | Finished a round of Baz IO: Wrote 1.472 GiB in 3.492 seconds | 20260508 09:58:53.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:58:53.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.237672% / 0% / 5.16574e-05% | 20260508 09:58:53.054 | INFO | Baz IO overall: | 20260508 09:58:53.054 | INFO | BytesWritten : 3138388740486 | 20260508 09:58:53.054 | INFO | NumEvents : 1509099542690 | 20260508 09:58:54.603 | INFO | Analyzing chunk frames = [7213056,7213568) | 20260508 09:58:54.603 | INFO | MemInfo: { | 20260508 09:58:54.603 | INFO | "gpu" : " 17 GB", | 20260508 09:58:54.603 | INFO | "huge" : "", | 20260508 09:58:54.603 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:54.603 | INFO | "malloc" : " 14 GB", | 20260508 09:58:54.603 | INFO | "pinned" : " 29 GB" | 20260508 09:58:54.603 | INFO | } | 20260508 09:58:54.603 | INFO | Perf report for the last 5.13885 seconds: | 20260508 09:58:54.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:54.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:54.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:54.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:54.603 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 09:58:54.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:54.603 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 09:58:54.603 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 09:58:54.603 | INFO | BAZWRITER : 67.4%, 2.5, 88.0ms, 0 | 20260508 09:58:54.603 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:58:54.604 | INFO | Uploaded 36GiB to GPU in 2.57894s (13.9592GiB/s) | 20260508 09:58:54.604 | INFO | Downloaded 38GiB to GPU in 2.71544s (13.994GiB/s) | 20260508 09:58:59.736 | INFO | Analyzing chunk frames = [7213568,7214080) | 20260508 09:58:59.736 | INFO | MemInfo: { | 20260508 09:58:59.736 | INFO | "gpu" : " 17 GB", | 20260508 09:58:59.736 | INFO | "huge" : "", | 20260508 09:58:59.736 | INFO | "huge_shared" : " 29 GB", | 20260508 09:58:59.736 | INFO | "malloc" : " 14 GB", | 20260508 09:58:59.736 | INFO | "pinned" : " 29 GB" | 20260508 09:58:59.736 | INFO | } | 20260508 09:58:59.736 | INFO | Perf report for the last 5.1332 seconds: | 20260508 09:58:59.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:58:59.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 09:58:59.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 09:58:59.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:58:59.736 | INFO | Uploaded 36GiB to GPU in 2.40637s (14.9603GiB/s) | 20260508 09:58:59.736 | INFO | Downloaded 34GiB to GPU in 2.47206s (13.7537GiB/s) | 20260508 09:59:04.880 | INFO | Analyzing chunk frames = [7214080,7214592) | 20260508 09:59:04.880 | INFO | MemInfo: { | 20260508 09:59:04.880 | INFO | "gpu" : " 17 GB", | 20260508 09:59:04.880 | INFO | "huge" : "", | 20260508 09:59:04.880 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:04.880 | INFO | "malloc" : " 14 GB", | 20260508 09:59:04.880 | INFO | "pinned" : " 29 GB" | 20260508 09:59:04.880 | INFO | } | 20260508 09:59:04.880 | INFO | Perf report for the last 5.14361 seconds: | 20260508 09:59:04.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:04.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 09:59:04.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 09:59:04.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:04.880 | INFO | Uploaded 36GiB to GPU in 2.42803s (14.8268GiB/s) | 20260508 09:59:04.880 | INFO | Downloaded 34GiB to GPU in 2.42998s (13.9919GiB/s) | 20260508 09:59:10.023 | INFO | Analyzing chunk frames = [7214592,7215104) | 20260508 09:59:10.023 | INFO | MemInfo: { | 20260508 09:59:10.023 | INFO | "gpu" : " 17 GB", | 20260508 09:59:10.023 | INFO | "huge" : "", | 20260508 09:59:10.023 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:10.024 | INFO | "malloc" : " 14 GB", | 20260508 09:59:10.024 | INFO | "pinned" : " 29 GB" | 20260508 09:59:10.024 | INFO | } | 20260508 09:59:10.024 | INFO | Perf report for the last 5.14387 seconds: | 20260508 09:59:10.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:10.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.024 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 09:59:10.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.024 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 09:59:10.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:10.025 | INFO | Uploaded 36GiB to GPU in 2.42618s (14.8382GiB/s) | 20260508 09:59:10.025 | INFO | Downloaded 34GiB to GPU in 2.43235s (13.9783GiB/s) | 20260508 09:59:15.167 | INFO | Analyzing chunk frames = [7215104,7215616) | 20260508 09:59:15.167 | INFO | MemInfo: { | 20260508 09:59:15.167 | INFO | "gpu" : " 17 GB", | 20260508 09:59:15.167 | INFO | "huge" : "", | 20260508 09:59:15.167 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:15.167 | INFO | "malloc" : " 14 GB", | 20260508 09:59:15.167 | INFO | "pinned" : " 29 GB" | 20260508 09:59:15.167 | INFO | } | 20260508 09:59:15.167 | INFO | Perf report for the last 5.14321 seconds: | 20260508 09:59:15.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:15.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 09:59:15.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 09:59:15.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:15.167 | INFO | Uploaded 36GiB to GPU in 2.32994s (15.451GiB/s) | 20260508 09:59:15.167 | INFO | Downloaded 34GiB to GPU in 2.50377s (13.5795GiB/s) | 20260508 09:59:20.298 | INFO | Analyzing chunk frames = [7215616,7216128) | 20260508 09:59:20.298 | INFO | MemInfo: { | 20260508 09:59:20.298 | INFO | "gpu" : " 17 GB", | 20260508 09:59:20.298 | INFO | "huge" : "", | 20260508 09:59:20.298 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:20.298 | INFO | "malloc" : " 14 GB", | 20260508 09:59:20.298 | INFO | "pinned" : " 29 GB" | 20260508 09:59:20.299 | INFO | } | 20260508 09:59:20.299 | INFO | Perf report for the last 5.13158 seconds: | 20260508 09:59:20.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:20.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 09:59:20.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 09:59:20.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:20.299 | INFO | Uploaded 36GiB to GPU in 2.3802s (15.1247GiB/s) | 20260508 09:59:20.299 | INFO | Downloaded 34GiB to GPU in 2.44124s (13.9273GiB/s) | 20260508 09:59:25.440 | INFO | Analyzing chunk frames = [7216128,7216640) | 20260508 09:59:25.440 | INFO | MemInfo: { | 20260508 09:59:25.440 | INFO | "gpu" : " 17 GB", | 20260508 09:59:25.440 | INFO | "huge" : "", | 20260508 09:59:25.440 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:25.441 | INFO | "malloc" : " 14 GB", | 20260508 09:59:25.441 | INFO | "pinned" : " 29 GB" | 20260508 09:59:25.441 | INFO | } | 20260508 09:59:25.441 | INFO | Perf report for the last 5.1422 seconds: | 20260508 09:59:25.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:25.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 09:59:25.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 09:59:25.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:25.441 | INFO | Uploaded 36GiB to GPU in 2.40756s (14.9529GiB/s) | 20260508 09:59:25.441 | INFO | Downloaded 34GiB to GPU in 2.44646s (13.8977GiB/s) | 20260508 09:59:30.579 | INFO | Analyzing chunk frames = [7216640,7217152) | 20260508 09:59:30.579 | INFO | MemInfo: { | 20260508 09:59:30.579 | INFO | "gpu" : " 17 GB", | 20260508 09:59:30.579 | INFO | "huge" : "", | 20260508 09:59:30.579 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:30.579 | INFO | "malloc" : " 14 GB", | 20260508 09:59:30.580 | INFO | "pinned" : " 29 GB" | 20260508 09:59:30.580 | INFO | } | 20260508 09:59:30.580 | INFO | Perf report for the last 5.13877 seconds: | 20260508 09:59:30.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:30.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 09:59:30.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 09:59:30.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:30.580 | INFO | Uploaded 36GiB to GPU in 2.43881s (14.7613GiB/s) | 20260508 09:59:30.580 | INFO | Downloaded 34GiB to GPU in 2.4506s (13.8742GiB/s) | 20260508 09:59:30.580 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 09:59:30.580 | INFO | Total mem usage: 16.7 GiB | 20260508 09:59:30.580 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 09:59:30.580 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 09:59:30.580 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 09:59:30.580 | INFO | High Water Breakdown: | 20260508 09:59:30.580 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 09:59:30.580 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 09:59:30.580 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 09:59:30.580 | INFO | ------------------------------------------ | 20260508 09:59:30.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:59:30.580 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 09:59:30.580 | INFO | Total mem usage: 26.5 GiB | 20260508 09:59:30.580 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 09:59:30.580 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 09:59:30.580 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 09:59:30.580 | INFO | High Water Breakdown: | 20260508 09:59:30.580 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:59:30.580 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:59:30.580 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:59:30.580 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:59:30.580 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:59:30.580 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:59:30.580 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 09:59:30.580 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 09:59:30.581 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:59:30.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 09:59:30.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 09:59:30.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 09:59:30.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 09:59:30.581 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 09:59:30.581 | INFO | ------------------------------------------ | 20260508 09:59:30.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:59:30.581 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 09:59:30.581 | INFO | Total mem usage: 27.8 GiB | 20260508 09:59:30.581 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 09:59:30.581 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 09:59:30.581 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 09:59:30.581 | INFO | High Water Breakdown: | 20260508 09:59:30.581 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 09:59:30.581 | INFO | ------------------------------------------ | 20260508 09:59:30.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:59:30.581 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 09:59:30.581 | INFO | Total mem usage: 15.7 GiB | 20260508 09:59:30.581 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 09:59:30.581 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 09:59:30.581 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 09:59:30.581 | INFO | High Water Breakdown: | 20260508 09:59:30.581 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 09:59:30.581 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 09:59:30.581 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 09:59:30.581 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 09:59:30.581 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 09:59:30.581 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 09:59:30.581 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 09:59:30.581 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 09:59:30.581 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 09:59:30.581 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 09:59:30.581 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 09:59:30.581 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 09:59:30.581 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 09:59:30.581 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 09:59:30.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 09:59:30.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 09:59:30.581 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 09:59:30.581 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 09:59:30.581 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 09:59:30.581 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 09:59:30.581 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 09:59:30.581 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 09:59:30.581 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 09:59:30.581 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 09:59:30.581 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 09:59:30.581 | INFO | ------------------------------------------ | 20260508 09:59:30.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 09:59:30.581 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 09:59:30.581 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:59:30.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:59:30.581 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.94e-01 | 20260508 09:59:30.581 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.15e-02 / 6.52e-03 | 20260508 09:59:30.581 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.50e+00, 5.88e-01 / 1.94e-01 | 20260508 09:59:30.581 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.27e-01 | 20260508 09:59:30.581 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.47e+01, 3.63e+00 / 8.01e-01 | 20260508 09:59:30.581 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.85e+00 / 1.38e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.45e+00, 4.01e-01 / 2.69e-01 | 20260508 09:59:30.581 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.16e+00, 2.44e-01 / 1.46e-01 | 20260508 09:59:30.581 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.44e-01, 4.78e-02 / 3.75e-02 | 20260508 09:59:30.581 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 09:59:30.581 | INFO | PerfReport: Gathered 1536 samples. | 20260508 09:59:30.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 09:59:30.581 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 24.57%, Download -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 31.11%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 2.20%, LockUpload -- 2.31e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 2.52%, LockDownload -- 2.66e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 15.29%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 09:59:30.581 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260508 09:59:30.677 | INFO | Starting a round of Baz IO | 20260508 09:59:34.088 | INFO | Populating RTMetrics report | 20260508 09:59:34.088 | INFO | RTMetrics report populated | 20260508 09:59:34.088 | INFO | Opening tmp RTMetrics json file | 20260508 09:59:34.099 | INFO | Serializing RTMetrics to json | 20260508 09:59:34.100 | INFO | Serialization done, swapping tmp file now... | 20260508 09:59:34.100 | INFO | RTMetrics done | 20260508 09:59:34.158 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.481 seconds | 20260508 09:59:34.158 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.25% / 0.00% / 59.75% | 20260508 09:59:34.158 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.237266% / 0% / 4.76837e-05% | 20260508 09:59:34.158 | INFO | Baz IO overall: | 20260508 09:59:34.158 | INFO | BytesWritten : 3139967451448 | 20260508 09:59:34.158 | INFO | NumEvents : 1509844975187 | 20260508 09:59:35.715 | INFO | Analyzing chunk frames = [7217152,7217664) | 20260508 09:59:35.715 | INFO | MemInfo: { | 20260508 09:59:35.715 | INFO | "gpu" : " 17 GB", | 20260508 09:59:35.715 | INFO | "huge" : "", | 20260508 09:59:35.715 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:35.715 | INFO | "malloc" : " 14 GB", | 20260508 09:59:35.715 | INFO | "pinned" : " 29 GB" | 20260508 09:59:35.715 | INFO | } | 20260508 09:59:35.715 | INFO | Perf report for the last 5.13561 seconds: | 20260508 09:59:35.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:35.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:35.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:35.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:35.715 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 09:59:35.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:35.715 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 09:59:35.715 | INFO | PRE_HQ : 65.5%, 1.6, 54.3ms, 0 | 20260508 09:59:35.715 | INFO | BAZWRITER : 67.2%, 2.5, 86.7ms, 0 | 20260508 09:59:35.715 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 09:59:35.715 | INFO | Uploaded 36GiB to GPU in 2.608s (13.8037GiB/s) | 20260508 09:59:35.715 | INFO | Downloaded 38GiB to GPU in 2.71791s (13.9813GiB/s) | 20260508 09:59:40.853 | INFO | Analyzing chunk frames = [7217664,7218176) | 20260508 09:59:40.853 | INFO | MemInfo: { | 20260508 09:59:40.853 | INFO | "gpu" : " 17 GB", | 20260508 09:59:40.853 | INFO | "huge" : "", | 20260508 09:59:40.853 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:40.853 | INFO | "malloc" : " 14 GB", | 20260508 09:59:40.853 | INFO | "pinned" : " 29 GB" | 20260508 09:59:40.853 | INFO | } | 20260508 09:59:40.853 | INFO | Perf report for the last 5.13806 seconds: | 20260508 09:59:40.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:40.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.853 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 09:59:40.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.853 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 09:59:40.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:40.854 | INFO | Uploaded 36GiB to GPU in 2.50881s (14.3495GiB/s) | 20260508 09:59:40.854 | INFO | Downloaded 34GiB to GPU in 2.45703s (13.8379GiB/s) | 20260508 09:59:45.988 | INFO | Analyzing chunk frames = [7218176,7218688) | 20260508 09:59:45.988 | INFO | MemInfo: { | 20260508 09:59:45.988 | INFO | "gpu" : " 17 GB", | 20260508 09:59:45.988 | INFO | "huge" : "", | 20260508 09:59:45.988 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:45.988 | INFO | "malloc" : " 14 GB", | 20260508 09:59:45.988 | INFO | "pinned" : " 29 GB" | 20260508 09:59:45.988 | INFO | } | 20260508 09:59:45.988 | INFO | Perf report for the last 5.1351 seconds: | 20260508 09:59:45.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:45.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.988 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 09:59:45.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.989 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 09:59:45.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:45.989 | INFO | Uploaded 36GiB to GPU in 2.45905s (14.6398GiB/s) | 20260508 09:59:45.989 | INFO | Downloaded 34GiB to GPU in 2.42022s (14.0483GiB/s) | 20260508 09:59:51.131 | INFO | Analyzing chunk frames = [7218688,7219200) | 20260508 09:59:51.131 | INFO | MemInfo: { | 20260508 09:59:51.131 | INFO | "gpu" : " 17 GB", | 20260508 09:59:51.131 | INFO | "huge" : "", | 20260508 09:59:51.131 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:51.131 | INFO | "malloc" : " 14 GB", | 20260508 09:59:51.131 | INFO | "pinned" : " 29 GB" | 20260508 09:59:51.131 | INFO | } | 20260508 09:59:51.131 | INFO | Perf report for the last 5.14283 seconds: | 20260508 09:59:51.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:51.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.131 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 09:59:51.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.131 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 09:59:51.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:51.132 | INFO | Uploaded 36GiB to GPU in 2.48679s (14.4765GiB/s) | 20260508 09:59:51.132 | INFO | Downloaded 34GiB to GPU in 2.4254s (14.0183GiB/s) | 20260508 09:59:56.275 | INFO | Analyzing chunk frames = [7219200,7219712) | 20260508 09:59:56.275 | INFO | MemInfo: { | 20260508 09:59:56.275 | INFO | "gpu" : " 17 GB", | 20260508 09:59:56.275 | INFO | "huge" : "", | 20260508 09:59:56.275 | INFO | "huge_shared" : " 29 GB", | 20260508 09:59:56.275 | INFO | "malloc" : " 14 GB", | 20260508 09:59:56.275 | INFO | "pinned" : " 29 GB" | 20260508 09:59:56.275 | INFO | } | 20260508 09:59:56.275 | INFO | Perf report for the last 5.14374 seconds: | 20260508 09:59:56.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 09:59:56.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 09:59:56.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 09:59:56.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 09:59:56.275 | INFO | Uploaded 36GiB to GPU in 2.34152s (15.3746GiB/s) | 20260508 09:59:56.275 | INFO | Downloaded 34GiB to GPU in 2.52745s (13.4523GiB/s) | 20260508 10:00:01.417 | INFO | Analyzing chunk frames = [7219712,7220224) | 20260508 10:00:01.417 | INFO | MemInfo: { | 20260508 10:00:01.417 | INFO | "gpu" : " 17 GB", | 20260508 10:00:01.417 | INFO | "huge" : "", | 20260508 10:00:01.417 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:01.417 | INFO | "malloc" : " 14 GB", | 20260508 10:00:01.417 | INFO | "pinned" : " 29 GB" | 20260508 10:00:01.417 | INFO | } | 20260508 10:00:01.417 | INFO | Perf report for the last 5.14206 seconds: | 20260508 10:00:01.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:01.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.418 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 10:00:01.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.418 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:00:01.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:01.418 | INFO | Uploaded 36GiB to GPU in 2.33929s (15.3893GiB/s) | 20260508 10:00:01.418 | INFO | Downloaded 34GiB to GPU in 2.44681s (13.8956GiB/s) | 20260508 10:00:06.548 | INFO | Analyzing chunk frames = [7220224,7220736) | 20260508 10:00:06.548 | INFO | MemInfo: { | 20260508 10:00:06.548 | INFO | "gpu" : " 17 GB", | 20260508 10:00:06.548 | INFO | "huge" : "", | 20260508 10:00:06.548 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:06.548 | INFO | "malloc" : " 14 GB", | 20260508 10:00:06.548 | INFO | "pinned" : " 29 GB" | 20260508 10:00:06.548 | INFO | } | 20260508 10:00:06.548 | INFO | Perf report for the last 5.13106 seconds: | 20260508 10:00:06.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:06.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.548 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 10:00:06.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.548 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:00:06.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:06.549 | INFO | Uploaded 36GiB to GPU in 2.41169s (14.9273GiB/s) | 20260508 10:00:06.549 | INFO | Downloaded 34GiB to GPU in 2.44758s (13.8913GiB/s) | 20260508 10:00:11.692 | INFO | Analyzing chunk frames = [7220736,7221248) | 20260508 10:00:11.692 | INFO | MemInfo: { | 20260508 10:00:11.692 | INFO | "gpu" : " 17 GB", | 20260508 10:00:11.692 | INFO | "huge" : "", | 20260508 10:00:11.692 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:11.692 | INFO | "malloc" : " 14 GB", | 20260508 10:00:11.692 | INFO | "pinned" : " 29 GB" | 20260508 10:00:11.692 | INFO | } | 20260508 10:00:11.692 | INFO | Perf report for the last 5.14372 seconds: | 20260508 10:00:11.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:11.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:00:11.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 10:00:11.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:11.692 | INFO | Uploaded 36GiB to GPU in 2.40406s (14.9747GiB/s) | 20260508 10:00:11.693 | INFO | Downloaded 34GiB to GPU in 2.4494s (13.881GiB/s) | 20260508 10:00:11.789 | INFO | Starting a round of Baz IO | 20260508 10:00:15.198 | INFO | Populating RTMetrics report | 20260508 10:00:15.199 | INFO | RTMetrics report populated | 20260508 10:00:15.199 | INFO | Opening tmp RTMetrics json file | 20260508 10:00:15.209 | INFO | Serializing RTMetrics to json | 20260508 10:00:15.210 | INFO | Serialization done, swapping tmp file now... | 20260508 10:00:15.211 | INFO | RTMetrics done | 20260508 10:00:15.271 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.481 seconds | 20260508 10:00:15.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.75% | 20260508 10:00:15.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.237147% / 0% / 5.96046e-05% | 20260508 10:00:15.271 | INFO | Baz IO overall: | 20260508 10:00:15.271 | INFO | BytesWritten : 3141546238430 | 20260508 10:00:15.271 | INFO | NumEvents : 1510590522580 | 20260508 10:00:16.831 | INFO | Analyzing chunk frames = [7221248,7221760) | 20260508 10:00:16.831 | INFO | MemInfo: { | 20260508 10:00:16.831 | INFO | "gpu" : " 17 GB", | 20260508 10:00:16.831 | INFO | "huge" : "", | 20260508 10:00:16.831 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:16.832 | INFO | "malloc" : " 14 GB", | 20260508 10:00:16.832 | INFO | "pinned" : " 29 GB" | 20260508 10:00:16.832 | INFO | } | 20260508 10:00:16.832 | INFO | Perf report for the last 5.13984 seconds: | 20260508 10:00:16.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:16.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:16.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:16.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:16.832 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:00:16.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:16.832 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 10:00:16.832 | INFO | PRE_HQ : 65.4%, 1.6, 54.5ms, 0 | 20260508 10:00:16.832 | INFO | BAZWRITER : 67.1%, 2.5, 86.8ms, 0 | 20260508 10:00:16.832 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:00:16.832 | INFO | Uploaded 36GiB to GPU in 2.58584s (13.922GiB/s) | 20260508 10:00:16.833 | INFO | Downloaded 38GiB to GPU in 2.71564s (13.993GiB/s) | 20260508 10:00:21.975 | INFO | Analyzing chunk frames = [7221760,7222272) | 20260508 10:00:21.975 | INFO | MemInfo: { | 20260508 10:00:21.975 | INFO | "gpu" : " 17 GB", | 20260508 10:00:21.975 | INFO | "huge" : "", | 20260508 10:00:21.976 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:21.976 | INFO | "malloc" : " 14 GB", | 20260508 10:00:21.976 | INFO | "pinned" : " 29 GB" | 20260508 10:00:21.976 | INFO | } | 20260508 10:00:21.976 | INFO | Perf report for the last 5.14395 seconds: | 20260508 10:00:21.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:21.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 10:00:21.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260508 10:00:21.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:21.976 | INFO | Uploaded 36GiB to GPU in 2.49247s (14.4435GiB/s) | 20260508 10:00:21.976 | INFO | Downloaded 34GiB to GPU in 2.4726s (13.7507GiB/s) | 20260508 10:00:27.115 | INFO | Analyzing chunk frames = [7222272,7222784) | 20260508 10:00:27.115 | INFO | MemInfo: { | 20260508 10:00:27.115 | INFO | "gpu" : " 17 GB", | 20260508 10:00:27.115 | INFO | "huge" : "", | 20260508 10:00:27.115 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:27.115 | INFO | "malloc" : " 14 GB", | 20260508 10:00:27.115 | INFO | "pinned" : " 29 GB" | 20260508 10:00:27.115 | INFO | } | 20260508 10:00:27.115 | INFO | Perf report for the last 5.13954 seconds: | 20260508 10:00:27.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:27.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.115 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 10:00:27.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.115 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:00:27.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:27.116 | INFO | Uploaded 36GiB to GPU in 2.38844s (15.0726GiB/s) | 20260508 10:00:27.116 | INFO | Downloaded 34GiB to GPU in 2.43709s (13.951GiB/s) | 20260508 10:00:32.248 | INFO | Analyzing chunk frames = [7222784,7223296) | 20260508 10:00:32.248 | INFO | MemInfo: { | 20260508 10:00:32.248 | INFO | "gpu" : " 17 GB", | 20260508 10:00:32.248 | INFO | "huge" : "", | 20260508 10:00:32.248 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:32.248 | INFO | "malloc" : " 14 GB", | 20260508 10:00:32.248 | INFO | "pinned" : " 29 GB" | 20260508 10:00:32.248 | INFO | } | 20260508 10:00:32.248 | INFO | Perf report for the last 5.13312 seconds: | 20260508 10:00:32.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:32.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.248 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 10:00:32.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.248 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:00:32.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:32.249 | INFO | Uploaded 36GiB to GPU in 2.442s (14.742GiB/s) | 20260508 10:00:32.249 | INFO | Downloaded 34GiB to GPU in 2.45652s (13.8407GiB/s) | 20260508 10:00:37.388 | INFO | Analyzing chunk frames = [7223296,7223808) | 20260508 10:00:37.388 | INFO | MemInfo: { | 20260508 10:00:37.388 | INFO | "gpu" : " 17 GB", | 20260508 10:00:37.388 | INFO | "huge" : "", | 20260508 10:00:37.388 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:37.388 | INFO | "malloc" : " 14 GB", | 20260508 10:00:37.388 | INFO | "pinned" : " 29 GB" | 20260508 10:00:37.388 | INFO | } | 20260508 10:00:37.388 | INFO | Perf report for the last 5.14004 seconds: | 20260508 10:00:37.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:37.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.388 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 10:00:37.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.389 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:00:37.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:37.389 | INFO | Uploaded 36GiB to GPU in 2.3766s (15.1477GiB/s) | 20260508 10:00:37.389 | INFO | Downloaded 34GiB to GPU in 2.52726s (13.4533GiB/s) | 20260508 10:00:42.528 | INFO | Analyzing chunk frames = [7223808,7224320) | 20260508 10:00:42.528 | INFO | MemInfo: { | 20260508 10:00:42.528 | INFO | "gpu" : " 17 GB", | 20260508 10:00:42.528 | INFO | "huge" : "", | 20260508 10:00:42.528 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:42.528 | INFO | "malloc" : " 14 GB", | 20260508 10:00:42.529 | INFO | "pinned" : " 29 GB" | 20260508 10:00:42.529 | INFO | } | 20260508 10:00:42.529 | INFO | Perf report for the last 5.14059 seconds: | 20260508 10:00:42.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:42.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 10:00:42.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:00:42.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:42.529 | INFO | Uploaded 36GiB to GPU in 2.32574s (15.4789GiB/s) | 20260508 10:00:42.529 | INFO | Downloaded 34GiB to GPU in 2.44585s (13.9011GiB/s) | 20260508 10:00:47.668 | INFO | Analyzing chunk frames = [7224320,7224832) | 20260508 10:00:47.668 | INFO | MemInfo: { | 20260508 10:00:47.668 | INFO | "gpu" : " 17 GB", | 20260508 10:00:47.668 | INFO | "huge" : "", | 20260508 10:00:47.668 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:47.668 | INFO | "malloc" : " 14 GB", | 20260508 10:00:47.668 | INFO | "pinned" : " 29 GB" | 20260508 10:00:47.668 | INFO | } | 20260508 10:00:47.668 | INFO | Perf report for the last 5.13888 seconds: | 20260508 10:00:47.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:47.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:00:47.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:00:47.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:47.668 | INFO | Uploaded 36GiB to GPU in 2.4083s (14.9483GiB/s) | 20260508 10:00:47.668 | INFO | Downloaded 34GiB to GPU in 2.44822s (13.8876GiB/s) | 20260508 10:00:52.804 | INFO | Analyzing chunk frames = [7224832,7225344) | 20260508 10:00:52.804 | INFO | MemInfo: { | 20260508 10:00:52.804 | INFO | "gpu" : " 17 GB", | 20260508 10:00:52.804 | INFO | "huge" : "", | 20260508 10:00:52.805 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:52.805 | INFO | "malloc" : " 14 GB", | 20260508 10:00:52.805 | INFO | "pinned" : " 29 GB" | 20260508 10:00:52.805 | INFO | } | 20260508 10:00:52.805 | INFO | Perf report for the last 5.13689 seconds: | 20260508 10:00:52.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:52.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:00:52.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:00:52.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:52.805 | INFO | Uploaded 36GiB to GPU in 2.38922s (15.0677GiB/s) | 20260508 10:00:52.805 | INFO | Downloaded 34GiB to GPU in 2.44731s (13.8928GiB/s) | 20260508 10:00:52.805 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:00:52.805 | INFO | Total mem usage: 16.7 GiB | 20260508 10:00:52.805 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:00:52.805 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:00:52.805 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:00:52.805 | INFO | High Water Breakdown: | 20260508 10:00:52.805 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:00:52.805 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:00:52.805 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:00:52.805 | INFO | ------------------------------------------ | 20260508 10:00:52.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:00:52.805 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:00:52.805 | INFO | Total mem usage: 26.5 GiB | 20260508 10:00:52.805 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:00:52.806 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:00:52.806 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:00:52.806 | INFO | High Water Breakdown: | 20260508 10:00:52.806 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:00:52.806 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:00:52.806 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:00:52.806 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:00:52.806 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:00:52.806 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:00:52.806 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:00:52.806 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:00:52.806 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:00:52.806 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:00:52.806 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:00:52.806 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:00:52.806 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:00:52.806 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:00:52.806 | INFO | ------------------------------------------ | 20260508 10:00:52.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:00:52.806 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:00:52.806 | INFO | Total mem usage: 27.8 GiB | 20260508 10:00:52.806 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:00:52.806 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:00:52.806 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:00:52.806 | INFO | High Water Breakdown: | 20260508 10:00:52.806 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:00:52.806 | INFO | ------------------------------------------ | 20260508 10:00:52.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:00:52.806 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:00:52.806 | INFO | Total mem usage: 15.7 GiB | 20260508 10:00:52.806 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:00:52.806 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:00:52.806 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:00:52.806 | INFO | High Water Breakdown: | 20260508 10:00:52.806 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:00:52.806 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:00:52.806 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:00:52.806 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:00:52.806 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:00:52.806 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:00:52.806 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:00:52.806 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:00:52.806 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:00:52.806 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:00:52.806 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:00:52.806 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:00:52.806 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:00:52.806 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:00:52.806 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:00:52.806 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:00:52.806 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:00:52.806 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:00:52.806 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:00:52.806 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:00:52.806 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:00:52.806 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:00:52.806 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:00:52.806 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:00:52.806 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:00:52.806 | INFO | ------------------------------------------ | 20260508 10:00:52.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:00:52.806 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:00:52.806 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:00:52.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:00:52.806 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.58e-01 | 20260508 10:00:52.806 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.17e-02 / 7.52e-03 | 20260508 10:00:52.806 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.44e+00, 5.91e-01 / 2.13e-01 | 20260508 10:00:52.806 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.42e+00, 2.10e-01 / 1.52e-01 | 20260508 10:00:52.806 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.54e+01, 3.80e+00 / 9.38e-01 | 20260508 10:00:52.806 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.36e+00 | 20260508 10:00:52.806 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.50e+00, 4.07e-01 / 2.92e-01 | 20260508 10:00:52.806 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.08e+00, 2.44e-01 / 1.03e-01 | 20260508 10:00:52.806 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.32e-01, 4.83e-02 / 2.81e-02 | 20260508 10:00:52.807 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:00:52.807 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:00:52.807 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:00:52.807 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 24.12%, Upload -- 2.54e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 24.66%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 31.05%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 2.02%, LockUpload -- 2.13e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 2.75%, LockDownload -- 2.90e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:00:52.807 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.21e-02 / 0.00e+00 | 20260508 10:00:52.903 | INFO | Starting a round of Baz IO | 20260508 10:00:56.324 | INFO | Populating RTMetrics report | 20260508 10:00:56.325 | INFO | RTMetrics report populated | 20260508 10:00:56.325 | INFO | Opening tmp RTMetrics json file | 20260508 10:00:56.326 | INFO | Serializing RTMetrics to json | 20260508 10:00:56.327 | INFO | Serialization done, swapping tmp file now... | 20260508 10:00:56.327 | INFO | RTMetrics done | 20260508 10:00:56.389 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.486 seconds | 20260508 10:00:56.389 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:00:56.389 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.236595% / 3.97364e-06% / 2.78155e-05% | 20260508 10:00:56.389 | INFO | Baz IO overall: | 20260508 10:00:56.389 | INFO | BytesWritten : 3143124152332 | 20260508 10:00:56.389 | INFO | NumEvents : 1511335792746 | 20260508 10:00:57.947 | INFO | Analyzing chunk frames = [7225344,7225856) | 20260508 10:00:57.947 | INFO | MemInfo: { | 20260508 10:00:57.947 | INFO | "gpu" : " 17 GB", | 20260508 10:00:57.947 | INFO | "huge" : "", | 20260508 10:00:57.947 | INFO | "huge_shared" : " 29 GB", | 20260508 10:00:57.947 | INFO | "malloc" : " 14 GB", | 20260508 10:00:57.947 | INFO | "pinned" : " 29 GB" | 20260508 10:00:57.947 | INFO | } | 20260508 10:00:57.947 | INFO | Perf report for the last 5.14182 seconds: | 20260508 10:00:57.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:00:57.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:57.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:57.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:57.947 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 10:00:57.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:00:57.947 | INFO | RT_METRICS : 40.1%, 1.0, 21.1ms, 0 | 20260508 10:00:57.947 | INFO | PRE_HQ : 66.0%, 1.7, 56.7ms, 0 | 20260508 10:00:57.947 | INFO | BAZWRITER : 67.4%, 2.6, 88.4ms, 0 | 20260508 10:00:57.947 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 10:00:57.947 | INFO | Uploaded 36GiB to GPU in 2.53976s (14.1746GiB/s) | 20260508 10:00:57.947 | INFO | Downloaded 38GiB to GPU in 2.73087s (13.915GiB/s) | 20260508 10:01:03.086 | INFO | Analyzing chunk frames = [7225856,7226368) | 20260508 10:01:03.086 | INFO | MemInfo: { | 20260508 10:01:03.086 | INFO | "gpu" : " 17 GB", | 20260508 10:01:03.086 | INFO | "huge" : "", | 20260508 10:01:03.086 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:03.086 | INFO | "malloc" : " 14 GB", | 20260508 10:01:03.086 | INFO | "pinned" : " 29 GB" | 20260508 10:01:03.086 | INFO | } | 20260508 10:01:03.086 | INFO | Perf report for the last 5.1397 seconds: | 20260508 10:01:03.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:03.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.086 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 10:01:03.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.086 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:01:03.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:03.087 | INFO | Uploaded 36GiB to GPU in 2.50461s (14.3735GiB/s) | 20260508 10:01:03.087 | INFO | Downloaded 34GiB to GPU in 2.46818s (13.7753GiB/s) | 20260508 10:01:08.226 | INFO | Analyzing chunk frames = [7226368,7226880) | 20260508 10:01:08.226 | INFO | MemInfo: { | 20260508 10:01:08.226 | INFO | "gpu" : " 17 GB", | 20260508 10:01:08.226 | INFO | "huge" : "", | 20260508 10:01:08.226 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:08.226 | INFO | "malloc" : " 14 GB", | 20260508 10:01:08.226 | INFO | "pinned" : " 29 GB" | 20260508 10:01:08.226 | INFO | } | 20260508 10:01:08.226 | INFO | Perf report for the last 5.13995 seconds: | 20260508 10:01:08.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:08.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.226 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:01:08.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.226 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:01:08.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:08.227 | INFO | Uploaded 36GiB to GPU in 2.48427s (14.4912GiB/s) | 20260508 10:01:08.227 | INFO | Downloaded 34GiB to GPU in 2.42386s (14.0272GiB/s) | 20260508 10:01:13.365 | INFO | Analyzing chunk frames = [7226880,7227392) | 20260508 10:01:13.365 | INFO | MemInfo: { | 20260508 10:01:13.365 | INFO | "gpu" : " 17 GB", | 20260508 10:01:13.365 | INFO | "huge" : "", | 20260508 10:01:13.365 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:13.365 | INFO | "malloc" : " 14 GB", | 20260508 10:01:13.365 | INFO | "pinned" : " 29 GB" | 20260508 10:01:13.365 | INFO | } | 20260508 10:01:13.365 | INFO | Perf report for the last 5.13915 seconds: | 20260508 10:01:13.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:13.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.366 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:01:13.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.366 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:01:13.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:13.366 | INFO | Uploaded 36GiB to GPU in 2.44747s (14.7091GiB/s) | 20260508 10:01:13.366 | INFO | Downloaded 34GiB to GPU in 2.41443s (14.082GiB/s) | 20260508 10:01:18.501 | INFO | Analyzing chunk frames = [7227392,7227904) | 20260508 10:01:18.501 | INFO | MemInfo: { | 20260508 10:01:18.501 | INFO | "gpu" : " 17 GB", | 20260508 10:01:18.501 | INFO | "huge" : "", | 20260508 10:01:18.501 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:18.501 | INFO | "malloc" : " 14 GB", | 20260508 10:01:18.501 | INFO | "pinned" : " 29 GB" | 20260508 10:01:18.501 | INFO | } | 20260508 10:01:18.501 | INFO | Perf report for the last 5.13574 seconds: | 20260508 10:01:18.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:18.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:01:18.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:01:18.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:18.501 | INFO | Uploaded 36GiB to GPU in 2.30636s (15.609GiB/s) | 20260508 10:01:18.502 | INFO | Downloaded 34GiB to GPU in 2.50419s (13.5773GiB/s) | 20260508 10:01:23.640 | INFO | Analyzing chunk frames = [7227904,7228416) | 20260508 10:01:23.640 | INFO | MemInfo: { | 20260508 10:01:23.640 | INFO | "gpu" : " 17 GB", | 20260508 10:01:23.640 | INFO | "huge" : "", | 20260508 10:01:23.640 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:23.640 | INFO | "malloc" : " 14 GB", | 20260508 10:01:23.640 | INFO | "pinned" : " 29 GB" | 20260508 10:01:23.640 | INFO | } | 20260508 10:01:23.640 | INFO | Perf report for the last 5.13875 seconds: | 20260508 10:01:23.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:23.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 10:01:23.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:01:23.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:23.640 | INFO | Uploaded 36GiB to GPU in 2.37943s (15.1297GiB/s) | 20260508 10:01:23.640 | INFO | Downloaded 34GiB to GPU in 2.41743s (14.0645GiB/s) | 20260508 10:01:28.784 | INFO | Analyzing chunk frames = [7228416,7228928) | 20260508 10:01:28.784 | INFO | MemInfo: { | 20260508 10:01:28.784 | INFO | "gpu" : " 17 GB", | 20260508 10:01:28.784 | INFO | "huge" : "", | 20260508 10:01:28.785 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:28.785 | INFO | "malloc" : " 14 GB", | 20260508 10:01:28.785 | INFO | "pinned" : " 29 GB" | 20260508 10:01:28.785 | INFO | } | 20260508 10:01:28.785 | INFO | Perf report for the last 5.14494 seconds: | 20260508 10:01:28.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:28.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.785 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 10:01:28.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.786 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:01:28.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:28.786 | INFO | Uploaded 36GiB to GPU in 2.4451s (14.7233GiB/s) | 20260508 10:01:28.786 | INFO | Downloaded 34GiB to GPU in 2.4269s (14.0097GiB/s) | 20260508 10:01:33.927 | INFO | Analyzing chunk frames = [7228928,7229440) | 20260508 10:01:33.928 | INFO | MemInfo: { | 20260508 10:01:33.928 | INFO | "gpu" : " 17 GB", | 20260508 10:01:33.928 | INFO | "huge" : "", | 20260508 10:01:33.928 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:33.928 | INFO | "malloc" : " 14 GB", | 20260508 10:01:33.928 | INFO | "pinned" : " 29 GB" | 20260508 10:01:33.928 | INFO | } | 20260508 10:01:33.928 | INFO | Perf report for the last 5.1429 seconds: | 20260508 10:01:33.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:33.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 10:01:33.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260508 10:01:33.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:33.928 | INFO | Uploaded 36GiB to GPU in 2.45539s (14.6616GiB/s) | 20260508 10:01:33.928 | INFO | Downloaded 34GiB to GPU in 2.42744s (14.0065GiB/s) | 20260508 10:01:34.025 | INFO | Starting a round of Baz IO | 20260508 10:01:37.443 | INFO | Populating RTMetrics report | 20260508 10:01:37.443 | INFO | RTMetrics report populated | 20260508 10:01:37.443 | INFO | Opening tmp RTMetrics json file | 20260508 10:01:37.452 | INFO | Serializing RTMetrics to json | 20260508 10:01:37.453 | INFO | Serialization done, swapping tmp file now... | 20260508 10:01:37.454 | INFO | RTMetrics done | 20260508 10:01:37.508 | INFO | Finished a round of Baz IO: Wrote 1.469 GiB in 3.482 seconds | 20260508 10:01:37.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:01:37.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.235669% / 7.94729e-06% / 7.15256e-05% | 20260508 10:01:37.508 | INFO | Baz IO overall: | 20260508 10:01:37.508 | INFO | BytesWritten : 3144701422377 | 20260508 10:01:37.508 | INFO | NumEvents : 1512080783071 | 20260508 10:01:39.061 | INFO | Analyzing chunk frames = [7229440,7229952) | 20260508 10:01:39.061 | INFO | MemInfo: { | 20260508 10:01:39.061 | INFO | "gpu" : " 17 GB", | 20260508 10:01:39.061 | INFO | "huge" : "", | 20260508 10:01:39.061 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:39.061 | INFO | "malloc" : " 14 GB", | 20260508 10:01:39.061 | INFO | "pinned" : " 29 GB" | 20260508 10:01:39.061 | INFO | } | 20260508 10:01:39.061 | INFO | Perf report for the last 5.13358 seconds: | 20260508 10:01:39.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:39.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:39.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:39.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:39.062 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 10:01:39.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:39.062 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:01:39.062 | INFO | PRE_HQ : 65.6%, 1.6, 54.1ms, 0 | 20260508 10:01:39.062 | INFO | BAZWRITER : 67.2%, 2.5, 85.3ms, 0 | 20260508 10:01:39.062 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:01:39.062 | INFO | Uploaded 36GiB to GPU in 2.59226s (13.8875GiB/s) | 20260508 10:01:39.062 | INFO | Downloaded 38GiB to GPU in 2.69835s (14.0827GiB/s) | 20260508 10:01:44.203 | INFO | Analyzing chunk frames = [7229952,7230464) | 20260508 10:01:44.203 | INFO | MemInfo: { | 20260508 10:01:44.203 | INFO | "gpu" : " 17 GB", | 20260508 10:01:44.203 | INFO | "huge" : "", | 20260508 10:01:44.203 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:44.203 | INFO | "malloc" : " 14 GB", | 20260508 10:01:44.203 | INFO | "pinned" : " 29 GB" | 20260508 10:01:44.203 | INFO | } | 20260508 10:01:44.203 | INFO | Perf report for the last 5.14143 seconds: | 20260508 10:01:44.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:44.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 10:01:44.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:01:44.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:44.203 | INFO | Uploaded 36GiB to GPU in 2.44356s (14.7326GiB/s) | 20260508 10:01:44.203 | INFO | Downloaded 34GiB to GPU in 2.46997s (13.7653GiB/s) | 20260508 10:01:49.338 | INFO | Analyzing chunk frames = [7230464,7230976) | 20260508 10:01:49.338 | INFO | MemInfo: { | 20260508 10:01:49.338 | INFO | "gpu" : " 17 GB", | 20260508 10:01:49.338 | INFO | "huge" : "", | 20260508 10:01:49.338 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:49.338 | INFO | "malloc" : " 14 GB", | 20260508 10:01:49.338 | INFO | "pinned" : " 29 GB" | 20260508 10:01:49.339 | INFO | } | 20260508 10:01:49.339 | INFO | Perf report for the last 5.13571 seconds: | 20260508 10:01:49.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:49.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:01:49.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 10:01:49.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:49.339 | INFO | Uploaded 36GiB to GPU in 2.46308s (14.6159GiB/s) | 20260508 10:01:49.339 | INFO | Downloaded 34GiB to GPU in 2.4514s (13.8696GiB/s) | 20260508 10:01:54.478 | INFO | Analyzing chunk frames = [7230976,7231488) | 20260508 10:01:54.478 | INFO | MemInfo: { | 20260508 10:01:54.478 | INFO | "gpu" : " 17 GB", | 20260508 10:01:54.478 | INFO | "huge" : "", | 20260508 10:01:54.478 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:54.478 | INFO | "malloc" : " 14 GB", | 20260508 10:01:54.479 | INFO | "pinned" : " 29 GB" | 20260508 10:01:54.479 | INFO | } | 20260508 10:01:54.479 | INFO | Perf report for the last 5.13999 seconds: | 20260508 10:01:54.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:54.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 10:01:54.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:01:54.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:54.479 | INFO | Uploaded 36GiB to GPU in 2.5072s (14.3587GiB/s) | 20260508 10:01:54.479 | INFO | Downloaded 34GiB to GPU in 2.42488s (14.0213GiB/s) | 20260508 10:01:59.621 | INFO | Analyzing chunk frames = [7231488,7232000) | 20260508 10:01:59.621 | INFO | MemInfo: { | 20260508 10:01:59.621 | INFO | "gpu" : " 17 GB", | 20260508 10:01:59.621 | INFO | "huge" : "", | 20260508 10:01:59.621 | INFO | "huge_shared" : " 29 GB", | 20260508 10:01:59.621 | INFO | "malloc" : " 14 GB", | 20260508 10:01:59.621 | INFO | "pinned" : " 29 GB" | 20260508 10:01:59.621 | INFO | } | 20260508 10:01:59.621 | INFO | Perf report for the last 5.14264 seconds: | 20260508 10:01:59.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:01:59.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.621 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 10:01:59.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.621 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:01:59.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:01:59.622 | INFO | Uploaded 36GiB to GPU in 2.38698s (15.0818GiB/s) | 20260508 10:01:59.622 | INFO | Downloaded 34GiB to GPU in 2.51005s (13.5455GiB/s) | 20260508 10:02:04.754 | INFO | Analyzing chunk frames = [7232000,7232512) | 20260508 10:02:04.754 | INFO | MemInfo: { | 20260508 10:02:04.754 | INFO | "gpu" : " 17 GB", | 20260508 10:02:04.754 | INFO | "huge" : "", | 20260508 10:02:04.754 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:04.754 | INFO | "malloc" : " 14 GB", | 20260508 10:02:04.754 | INFO | "pinned" : " 29 GB" | 20260508 10:02:04.754 | INFO | } | 20260508 10:02:04.754 | INFO | Perf report for the last 5.13309 seconds: | 20260508 10:02:04.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:04.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.754 | INFO | ANALYSIS : 65.3%, 3.0, 101.4ms, 0 | 20260508 10:02:04.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.755 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 10:02:04.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:04.755 | INFO | Uploaded 36GiB to GPU in 2.32731s (15.4685GiB/s) | 20260508 10:02:04.755 | INFO | Downloaded 34GiB to GPU in 2.44965s (13.8795GiB/s) | 20260508 10:02:09.903 | INFO | Analyzing chunk frames = [7232512,7233024) | 20260508 10:02:09.903 | INFO | MemInfo: { | 20260508 10:02:09.903 | INFO | "gpu" : " 17 GB", | 20260508 10:02:09.903 | INFO | "huge" : "", | 20260508 10:02:09.903 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:09.903 | INFO | "malloc" : " 14 GB", | 20260508 10:02:09.903 | INFO | "pinned" : " 29 GB" | 20260508 10:02:09.903 | INFO | } | 20260508 10:02:09.903 | INFO | Perf report for the last 5.14878 seconds: | 20260508 10:02:09.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:09.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.903 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 10:02:09.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.903 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260508 10:02:09.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:09.904 | INFO | Uploaded 36GiB to GPU in 2.40254s (14.9841GiB/s) | 20260508 10:02:09.904 | INFO | Downloaded 34GiB to GPU in 2.4563s (13.8419GiB/s) | 20260508 10:02:15.045 | INFO | Analyzing chunk frames = [7233024,7233536) | 20260508 10:02:15.045 | INFO | MemInfo: { | 20260508 10:02:15.045 | INFO | "gpu" : " 17 GB", | 20260508 10:02:15.045 | INFO | "huge" : "", | 20260508 10:02:15.045 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:15.045 | INFO | "malloc" : " 14 GB", | 20260508 10:02:15.045 | INFO | "pinned" : " 29 GB" | 20260508 10:02:15.045 | INFO | } | 20260508 10:02:15.046 | INFO | Perf report for the last 5.14231 seconds: | 20260508 10:02:15.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:15.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:02:15.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260508 10:02:15.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:15.046 | INFO | Uploaded 36GiB to GPU in 2.45362s (14.6722GiB/s) | 20260508 10:02:15.046 | INFO | Downloaded 34GiB to GPU in 2.46812s (13.7757GiB/s) | 20260508 10:02:15.046 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:02:15.046 | INFO | Total mem usage: 16.7 GiB | 20260508 10:02:15.046 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:02:15.046 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:02:15.046 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:02:15.046 | INFO | High Water Breakdown: | 20260508 10:02:15.046 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:02:15.046 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:02:15.046 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:02:15.046 | INFO | ------------------------------------------ | 20260508 10:02:15.046 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:02:15.046 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:02:15.046 | INFO | Total mem usage: 26.5 GiB | 20260508 10:02:15.046 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:02:15.046 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:02:15.046 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:02:15.046 | INFO | High Water Breakdown: | 20260508 10:02:15.046 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:02:15.046 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:02:15.046 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:02:15.046 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:02:15.046 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:02:15.046 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:02:15.046 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:02:15.046 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:02:15.047 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:02:15.047 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:02:15.047 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:02:15.047 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:02:15.047 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:02:15.047 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:02:15.047 | INFO | ------------------------------------------ | 20260508 10:02:15.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:02:15.047 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:02:15.047 | INFO | Total mem usage: 27.8 GiB | 20260508 10:02:15.047 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:02:15.047 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:02:15.047 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:02:15.047 | INFO | High Water Breakdown: | 20260508 10:02:15.047 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:02:15.047 | INFO | ------------------------------------------ | 20260508 10:02:15.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:02:15.047 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:02:15.047 | INFO | Total mem usage: 15.7 GiB | 20260508 10:02:15.047 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:02:15.047 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:02:15.047 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:02:15.047 | INFO | High Water Breakdown: | 20260508 10:02:15.047 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:02:15.047 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:02:15.047 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:02:15.047 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:02:15.047 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:02:15.047 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:02:15.047 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:02:15.047 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:02:15.047 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:02:15.047 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:02:15.047 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:02:15.047 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:02:15.047 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:02:15.047 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:02:15.047 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:02:15.047 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:02:15.047 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:02:15.047 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:02:15.047 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:02:15.047 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:02:15.047 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:02:15.047 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:02:15.047 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:02:15.047 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:02:15.047 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:02:15.047 | INFO | ------------------------------------------ | 20260508 10:02:15.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:02:15.047 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:02:15.047 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:02:15.047 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:02:15.047 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.42e+01 / 7.42e-01 | 20260508 10:02:15.047 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.06e-01, 1.13e-02 / 6.22e-03 | 20260508 10:02:15.047 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.38e+00, 5.94e-01 / 2.31e-01 | 20260508 10:02:15.047 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.47e+00, 2.09e-01 / 1.17e-01 | 20260508 10:02:15.047 | INFO | PerfReport: 15.98%, DME -- 4.20e+01 / 4.54e+01, 3.74e+00 / 1.28e+00 | 20260508 10:02:15.047 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.88e+01 / 2.00e+01, 1.89e+00 / 1.37e+00 | 20260508 10:02:15.047 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.09e-01 / 2.77e-01 | 20260508 10:02:15.047 | INFO | PerfReport: 3.02%, Metrics -- 9.91e-01 / 1.13e+00, 2.45e-01 / 1.28e-01 | 20260508 10:02:15.048 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.33e-01, 4.87e-02 / 2.53e-02 | 20260508 10:02:15.048 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:02:15.048 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:02:15.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:02:15.048 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 24.19%, Upload -- 2.55e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 24.57%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 31.18%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 2.15%, LockUpload -- 2.26e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 2.53%, LockDownload -- 2.67e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:02:15.048 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260508 10:02:15.159 | INFO | Starting a round of Baz IO | 20260508 10:02:18.565 | INFO | Populating RTMetrics report | 20260508 10:02:18.565 | INFO | RTMetrics report populated | 20260508 10:02:18.565 | INFO | Opening tmp RTMetrics json file | 20260508 10:02:18.572 | INFO | Serializing RTMetrics to json | 20260508 10:02:18.573 | INFO | Serialization done, swapping tmp file now... | 20260508 10:02:18.573 | INFO | RTMetrics done | 20260508 10:02:18.630 | INFO | Finished a round of Baz IO: Wrote 1.467 GiB in 3.471 seconds | 20260508 10:02:18.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:02:18.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.235136% / 3.97364e-06% / 6.75519e-05% | 20260508 10:02:18.630 | INFO | Baz IO overall: | 20260508 10:02:18.630 | INFO | BytesWritten : 3146276601902 | 20260508 10:02:18.630 | INFO | NumEvents : 1512824364134 | 20260508 10:02:20.176 | INFO | Analyzing chunk frames = [7233536,7234048) | 20260508 10:02:20.176 | INFO | MemInfo: { | 20260508 10:02:20.176 | INFO | "gpu" : " 17 GB", | 20260508 10:02:20.176 | INFO | "huge" : "", | 20260508 10:02:20.176 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:20.176 | INFO | "malloc" : " 14 GB", | 20260508 10:02:20.176 | INFO | "pinned" : " 29 GB" | 20260508 10:02:20.176 | INFO | } | 20260508 10:02:20.176 | INFO | Perf report for the last 5.13082 seconds: | 20260508 10:02:20.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:20.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:20.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:20.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:20.176 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 10:02:20.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:20.177 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260508 10:02:20.177 | INFO | PRE_HQ : 66.4%, 1.8, 60.5ms, 0 | 20260508 10:02:20.177 | INFO | BAZWRITER : 67.0%, 2.7, 92.9ms, 0 | 20260508 10:02:20.177 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:02:20.177 | INFO | Uploaded 36GiB to GPU in 2.60352s (13.8274GiB/s) | 20260508 10:02:20.177 | INFO | Downloaded 38GiB to GPU in 2.72394s (13.9504GiB/s) | 20260508 10:02:25.308 | INFO | Analyzing chunk frames = [7234048,7234560) | 20260508 10:02:25.308 | INFO | MemInfo: { | 20260508 10:02:25.308 | INFO | "gpu" : " 17 GB", | 20260508 10:02:25.308 | INFO | "huge" : "", | 20260508 10:02:25.308 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:25.308 | INFO | "malloc" : " 14 GB", | 20260508 10:02:25.308 | INFO | "pinned" : " 29 GB" | 20260508 10:02:25.308 | INFO | } | 20260508 10:02:25.308 | INFO | Perf report for the last 5.13216 seconds: | 20260508 10:02:25.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:25.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | ANALYSIS : 66.7%, 3.0, 103.8ms, 0 | 20260508 10:02:25.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:02:25.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:25.309 | INFO | Uploaded 36GiB to GPU in 2.48283s (14.4996GiB/s) | 20260508 10:02:25.309 | INFO | Downloaded 34GiB to GPU in 2.47423s (13.7416GiB/s) | 20260508 10:02:30.450 | INFO | Analyzing chunk frames = [7234560,7235072) | 20260508 10:02:30.450 | INFO | MemInfo: { | 20260508 10:02:30.450 | INFO | "gpu" : " 17 GB", | 20260508 10:02:30.450 | INFO | "huge" : "", | 20260508 10:02:30.450 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:30.450 | INFO | "malloc" : " 14 GB", | 20260508 10:02:30.450 | INFO | "pinned" : " 29 GB" | 20260508 10:02:30.450 | INFO | } | 20260508 10:02:30.450 | INFO | Perf report for the last 5.14127 seconds: | 20260508 10:02:30.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:30.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:02:30.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:02:30.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:30.450 | INFO | Uploaded 36GiB to GPU in 2.45257s (14.6785GiB/s) | 20260508 10:02:30.450 | INFO | Downloaded 34GiB to GPU in 2.4492s (13.8821GiB/s) | 20260508 10:02:35.595 | INFO | Analyzing chunk frames = [7235072,7235584) | 20260508 10:02:35.595 | INFO | MemInfo: { | 20260508 10:02:35.595 | INFO | "gpu" : " 17 GB", | 20260508 10:02:35.595 | INFO | "huge" : "", | 20260508 10:02:35.595 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:35.595 | INFO | "malloc" : " 14 GB", | 20260508 10:02:35.595 | INFO | "pinned" : " 29 GB" | 20260508 10:02:35.595 | INFO | } | 20260508 10:02:35.595 | INFO | Perf report for the last 5.14526 seconds: | 20260508 10:02:35.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:35.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | ANALYSIS : 66.6%, 2.9, 103.1ms, 0 | 20260508 10:02:35.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:02:35.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:35.595 | INFO | Uploaded 36GiB to GPU in 2.52819s (14.2394GiB/s) | 20260508 10:02:35.595 | INFO | Downloaded 34GiB to GPU in 2.44964s (13.8796GiB/s) | 20260508 10:02:40.734 | INFO | Analyzing chunk frames = [7235584,7236096) | 20260508 10:02:40.734 | INFO | MemInfo: { | 20260508 10:02:40.734 | INFO | "gpu" : " 17 GB", | 20260508 10:02:40.734 | INFO | "huge" : "", | 20260508 10:02:40.734 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:40.734 | INFO | "malloc" : " 14 GB", | 20260508 10:02:40.734 | INFO | "pinned" : " 29 GB" | 20260508 10:02:40.734 | INFO | } | 20260508 10:02:40.734 | INFO | Perf report for the last 5.13879 seconds: | 20260508 10:02:40.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:40.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 10:02:40.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:02:40.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:40.734 | INFO | Uploaded 36GiB to GPU in 2.37245s (15.1742GiB/s) | 20260508 10:02:40.734 | INFO | Downloaded 34GiB to GPU in 2.53145s (13.431GiB/s) | 20260508 10:02:45.874 | INFO | Analyzing chunk frames = [7236096,7236608) | 20260508 10:02:45.874 | INFO | MemInfo: { | 20260508 10:02:45.874 | INFO | "gpu" : " 17 GB", | 20260508 10:02:45.874 | INFO | "huge" : "", | 20260508 10:02:45.874 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:45.874 | INFO | "malloc" : " 14 GB", | 20260508 10:02:45.874 | INFO | "pinned" : " 29 GB" | 20260508 10:02:45.874 | INFO | } | 20260508 10:02:45.874 | INFO | Perf report for the last 5.14027 seconds: | 20260508 10:02:45.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:45.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 10:02:45.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:02:45.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:45.874 | INFO | Uploaded 36GiB to GPU in 2.35382s (15.2943GiB/s) | 20260508 10:02:45.874 | INFO | Downloaded 34GiB to GPU in 2.4487s (13.8849GiB/s) | 20260508 10:02:51.008 | INFO | Analyzing chunk frames = [7236608,7237120) | 20260508 10:02:51.008 | INFO | MemInfo: { | 20260508 10:02:51.008 | INFO | "gpu" : " 17 GB", | 20260508 10:02:51.008 | INFO | "huge" : "", | 20260508 10:02:51.008 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:51.008 | INFO | "malloc" : " 14 GB", | 20260508 10:02:51.008 | INFO | "pinned" : " 29 GB" | 20260508 10:02:51.008 | INFO | } | 20260508 10:02:51.009 | INFO | Perf report for the last 5.13441 seconds: | 20260508 10:02:51.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:51.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:02:51.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:02:51.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:51.009 | INFO | Uploaded 36GiB to GPU in 2.42717s (14.8321GiB/s) | 20260508 10:02:51.009 | INFO | Downloaded 34GiB to GPU in 2.45422s (13.8537GiB/s) | 20260508 10:02:56.148 | INFO | Analyzing chunk frames = [7237120,7237632) | 20260508 10:02:56.148 | INFO | MemInfo: { | 20260508 10:02:56.148 | INFO | "gpu" : " 17 GB", | 20260508 10:02:56.148 | INFO | "huge" : "", | 20260508 10:02:56.148 | INFO | "huge_shared" : " 29 GB", | 20260508 10:02:56.148 | INFO | "malloc" : " 14 GB", | 20260508 10:02:56.148 | INFO | "pinned" : " 29 GB" | 20260508 10:02:56.149 | INFO | } | 20260508 10:02:56.149 | INFO | Perf report for the last 5.13999 seconds: | 20260508 10:02:56.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:02:56.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 10:02:56.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 10:02:56.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:02:56.149 | INFO | Uploaded 36GiB to GPU in 2.44207s (14.7416GiB/s) | 20260508 10:02:56.149 | INFO | Downloaded 34GiB to GPU in 2.45774s (13.8339GiB/s) | 20260508 10:02:56.246 | INFO | Starting a round of Baz IO | 20260508 10:02:59.665 | INFO | Populating RTMetrics report | 20260508 10:02:59.665 | INFO | RTMetrics report populated | 20260508 10:02:59.665 | INFO | Opening tmp RTMetrics json file | 20260508 10:02:59.666 | INFO | Serializing RTMetrics to json | 20260508 10:02:59.667 | INFO | Serialization done, swapping tmp file now... | 20260508 10:02:59.667 | INFO | RTMetrics done | 20260508 10:02:59.731 | INFO | Finished a round of Baz IO: Wrote 1.47 GiB in 3.485 seconds | 20260508 10:02:59.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:02:59.732 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234445% / 0% / 6.35783e-05% | 20260508 10:02:59.732 | INFO | Baz IO overall: | 20260508 10:02:59.732 | INFO | BytesWritten : 3147854484608 | 20260508 10:02:59.732 | INFO | NumEvents : 1513569969842 | 20260508 10:03:01.288 | INFO | Analyzing chunk frames = [7237632,7238144) | 20260508 10:03:01.288 | INFO | MemInfo: { | 20260508 10:03:01.288 | INFO | "gpu" : " 17 GB", | 20260508 10:03:01.288 | INFO | "huge" : "", | 20260508 10:03:01.288 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:01.288 | INFO | "malloc" : " 14 GB", | 20260508 10:03:01.288 | INFO | "pinned" : " 29 GB" | 20260508 10:03:01.288 | INFO | } | 20260508 10:03:01.289 | INFO | Perf report for the last 5.13997 seconds: | 20260508 10:03:01.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:01.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:01.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:01.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:01.289 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 10:03:01.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:01.289 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:03:01.289 | INFO | PRE_HQ : 65.7%, 1.6, 54.6ms, 0 | 20260508 10:03:01.289 | INFO | BAZWRITER : 67.1%, 2.5, 86.8ms, 0 | 20260508 10:03:01.289 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:03:01.289 | INFO | Uploaded 36GiB to GPU in 2.55592s (14.0849GiB/s) | 20260508 10:03:01.289 | INFO | Downloaded 38GiB to GPU in 2.71286s (14.0074GiB/s) | 20260508 10:03:06.432 | INFO | Analyzing chunk frames = [7238144,7238656) | 20260508 10:03:06.432 | INFO | MemInfo: { | 20260508 10:03:06.432 | INFO | "gpu" : " 17 GB", | 20260508 10:03:06.432 | INFO | "huge" : "", | 20260508 10:03:06.432 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:06.432 | INFO | "malloc" : " 14 GB", | 20260508 10:03:06.432 | INFO | "pinned" : " 29 GB" | 20260508 10:03:06.432 | INFO | } | 20260508 10:03:06.432 | INFO | Perf report for the last 5.14385 seconds: | 20260508 10:03:06.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:06.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:03:06.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:03:06.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:06.433 | INFO | Uploaded 36GiB to GPU in 2.50222s (14.3872GiB/s) | 20260508 10:03:06.433 | INFO | Downloaded 34GiB to GPU in 2.45345s (13.8581GiB/s) | 20260508 10:03:11.572 | INFO | Analyzing chunk frames = [7238656,7239168) | 20260508 10:03:11.572 | INFO | MemInfo: { | 20260508 10:03:11.572 | INFO | "gpu" : " 17 GB", | 20260508 10:03:11.572 | INFO | "huge" : "", | 20260508 10:03:11.572 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:11.572 | INFO | "malloc" : " 14 GB", | 20260508 10:03:11.572 | INFO | "pinned" : " 29 GB" | 20260508 10:03:11.572 | INFO | } | 20260508 10:03:11.572 | INFO | Perf report for the last 5.13944 seconds: | 20260508 10:03:11.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:11.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:03:11.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:03:11.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:11.572 | INFO | Uploaded 36GiB to GPU in 2.45415s (14.669GiB/s) | 20260508 10:03:11.572 | INFO | Downloaded 34GiB to GPU in 2.43421s (13.9676GiB/s) | 20260508 10:03:16.710 | INFO | Analyzing chunk frames = [7239168,7239680) | 20260508 10:03:16.710 | INFO | MemInfo: { | 20260508 10:03:16.710 | INFO | "gpu" : " 17 GB", | 20260508 10:03:16.710 | INFO | "huge" : "", | 20260508 10:03:16.710 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:16.710 | INFO | "malloc" : " 14 GB", | 20260508 10:03:16.710 | INFO | "pinned" : " 29 GB" | 20260508 10:03:16.710 | INFO | } | 20260508 10:03:16.711 | INFO | Perf report for the last 5.13869 seconds: | 20260508 10:03:16.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:16.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:03:16.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:03:16.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:16.711 | INFO | Uploaded 36GiB to GPU in 2.46651s (14.5955GiB/s) | 20260508 10:03:16.711 | INFO | Downloaded 34GiB to GPU in 2.4481s (13.8883GiB/s) | 20260508 10:03:21.848 | INFO | Analyzing chunk frames = [7239680,7240192) | 20260508 10:03:21.848 | INFO | MemInfo: { | 20260508 10:03:21.848 | INFO | "gpu" : " 17 GB", | 20260508 10:03:21.848 | INFO | "huge" : "", | 20260508 10:03:21.848 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:21.848 | INFO | "malloc" : " 14 GB", | 20260508 10:03:21.848 | INFO | "pinned" : " 29 GB" | 20260508 10:03:21.848 | INFO | } | 20260508 10:03:21.848 | INFO | Perf report for the last 5.13783 seconds: | 20260508 10:03:21.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:21.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.848 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 10:03:21.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.849 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:03:21.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:21.849 | INFO | Uploaded 36GiB to GPU in 2.37228s (15.1753GiB/s) | 20260508 10:03:21.849 | INFO | Downloaded 34GiB to GPU in 2.51251s (13.5323GiB/s) | 20260508 10:03:26.988 | INFO | Analyzing chunk frames = [7240192,7240704) | 20260508 10:03:26.988 | INFO | MemInfo: { | 20260508 10:03:26.988 | INFO | "gpu" : " 17 GB", | 20260508 10:03:26.988 | INFO | "huge" : "", | 20260508 10:03:26.988 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:26.988 | INFO | "malloc" : " 14 GB", | 20260508 10:03:26.988 | INFO | "pinned" : " 29 GB" | 20260508 10:03:26.988 | INFO | } | 20260508 10:03:26.988 | INFO | Perf report for the last 5.13976 seconds: | 20260508 10:03:26.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:26.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 10:03:26.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:03:26.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:26.988 | INFO | Uploaded 36GiB to GPU in 2.34254s (15.3679GiB/s) | 20260508 10:03:26.988 | INFO | Downloaded 34GiB to GPU in 2.4378s (13.947GiB/s) | 20260508 10:03:32.123 | INFO | Analyzing chunk frames = [7240704,7241216) | 20260508 10:03:32.123 | INFO | MemInfo: { | 20260508 10:03:32.123 | INFO | "gpu" : " 17 GB", | 20260508 10:03:32.123 | INFO | "huge" : "", | 20260508 10:03:32.123 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:32.123 | INFO | "malloc" : " 14 GB", | 20260508 10:03:32.123 | INFO | "pinned" : " 29 GB" | 20260508 10:03:32.123 | INFO | } | 20260508 10:03:32.123 | INFO | Perf report for the last 5.13483 seconds: | 20260508 10:03:32.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:32.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 10:03:32.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 10:03:32.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:32.123 | INFO | Uploaded 36GiB to GPU in 2.38507s (15.0939GiB/s) | 20260508 10:03:32.123 | INFO | Downloaded 34GiB to GPU in 2.45666s (13.8399GiB/s) | 20260508 10:03:37.262 | INFO | Analyzing chunk frames = [7241216,7241728) | 20260508 10:03:37.262 | INFO | MemInfo: { | 20260508 10:03:37.262 | INFO | "gpu" : " 17 GB", | 20260508 10:03:37.262 | INFO | "huge" : "", | 20260508 10:03:37.262 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:37.262 | INFO | "malloc" : " 14 GB", | 20260508 10:03:37.262 | INFO | "pinned" : " 29 GB" | 20260508 10:03:37.262 | INFO | } | 20260508 10:03:37.262 | INFO | Perf report for the last 5.13933 seconds: | 20260508 10:03:37.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:37.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 10:03:37.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:03:37.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:37.262 | INFO | Uploaded 36GiB to GPU in 2.4315s (14.8057GiB/s) | 20260508 10:03:37.263 | INFO | Downloaded 34GiB to GPU in 2.44965s (13.8796GiB/s) | 20260508 10:03:37.263 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:03:37.263 | INFO | Total mem usage: 16.7 GiB | 20260508 10:03:37.263 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:03:37.263 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 10:03:37.263 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:03:37.263 | INFO | High Water Breakdown: | 20260508 10:03:37.263 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:03:37.263 | INFO | 40.3% : 5.78 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:03:37.263 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:03:37.263 | INFO | ------------------------------------------ | 20260508 10:03:37.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:03:37.263 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:03:37.263 | INFO | Total mem usage: 26.5 GiB | 20260508 10:03:37.263 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:03:37.263 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 10:03:37.263 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 10:03:37.263 | INFO | High Water Breakdown: | 20260508 10:03:37.263 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:03:37.263 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:03:37.263 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:03:37.263 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:03:37.263 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:03:37.263 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:03:37.263 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:03:37.263 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:03:37.263 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:03:37.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:03:37.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:03:37.263 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:03:37.263 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:03:37.263 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:03:37.263 | INFO | ------------------------------------------ | 20260508 10:03:37.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:03:37.263 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:03:37.263 | INFO | Total mem usage: 27.8 GiB | 20260508 10:03:37.263 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:03:37.263 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:03:37.263 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:03:37.263 | INFO | High Water Breakdown: | 20260508 10:03:37.263 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:03:37.263 | INFO | ------------------------------------------ | 20260508 10:03:37.263 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:03:37.263 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:03:37.263 | INFO | Total mem usage: 15.7 GiB | 20260508 10:03:37.263 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:03:37.263 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:03:37.263 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:03:37.264 | INFO | High Water Breakdown: | 20260508 10:03:37.264 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:03:37.264 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:03:37.264 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:03:37.264 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:03:37.264 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:03:37.264 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:03:37.264 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:03:37.264 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:03:37.264 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:03:37.264 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:03:37.264 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:03:37.264 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:03:37.264 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:03:37.264 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:03:37.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:03:37.264 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:03:37.264 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:03:37.264 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:03:37.264 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:03:37.264 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:03:37.264 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:03:37.264 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:03:37.264 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:03:37.264 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:03:37.264 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:03:37.264 | INFO | ------------------------------------------ | 20260508 10:03:37.264 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:03:37.264 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:03:37.264 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:03:37.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:03:37.264 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 6.02e-01 | 20260508 10:03:37.264 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.03e-01, 1.12e-02 / 4.89e-03 | 20260508 10:03:37.264 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.42e+00, 5.89e-01 / 2.20e-01 | 20260508 10:03:37.264 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 9.70e-02 | 20260508 10:03:37.264 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.53e+01, 3.68e+00 / 1.08e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.84e+00 / 1.23e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.47e+00, 4.07e-01 / 2.58e-01 | 20260508 10:03:37.264 | INFO | PerfReport: 3.03%, Metrics -- 9.92e-01 / 1.16e+00, 2.46e-01 / 1.31e-01 | 20260508 10:03:37.264 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.02e-01 / 3.34e-01, 4.81e-02 / 2.51e-02 | 20260508 10:03:37.264 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:03:37.264 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:03:37.264 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:03:37.264 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 24.21%, Upload -- 2.55e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 24.66%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 2.02%, LockUpload -- 2.13e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 2.68%, LockDownload -- 2.82e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 10:03:37.264 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260508 10:03:37.359 | INFO | Starting a round of Baz IO | 20260508 10:03:40.780 | INFO | Populating RTMetrics report | 20260508 10:03:40.780 | INFO | RTMetrics report populated | 20260508 10:03:40.780 | INFO | Opening tmp RTMetrics json file | 20260508 10:03:40.788 | INFO | Serializing RTMetrics to json | 20260508 10:03:40.790 | INFO | Serialization done, swapping tmp file now... | 20260508 10:03:40.790 | INFO | RTMetrics done | 20260508 10:03:40.847 | INFO | Finished a round of Baz IO: Wrote 1.469 GiB in 3.488 seconds | 20260508 10:03:40.847 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:03:40.847 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.233881% / 1.19209e-05% / 5.96046e-05% | 20260508 10:03:40.847 | INFO | Baz IO overall: | 20260508 10:03:40.847 | INFO | BytesWritten : 3149431458757 | 20260508 10:03:40.847 | INFO | NumEvents : 1514314860417 | 20260508 10:03:42.405 | INFO | Analyzing chunk frames = [7241728,7242240) | 20260508 10:03:42.405 | INFO | MemInfo: { | 20260508 10:03:42.405 | INFO | "gpu" : " 17 GB", | 20260508 10:03:42.405 | INFO | "huge" : "", | 20260508 10:03:42.405 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:42.405 | INFO | "malloc" : " 14 GB", | 20260508 10:03:42.405 | INFO | "pinned" : " 29 GB" | 20260508 10:03:42.405 | INFO | } | 20260508 10:03:42.405 | INFO | Perf report for the last 5.14279 seconds: | 20260508 10:03:42.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:42.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:42.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:42.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:42.405 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:03:42.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:42.405 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:03:42.405 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 10:03:42.405 | INFO | BAZWRITER : 67.1%, 2.5, 86.3ms, 0 | 20260508 10:03:42.405 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:03:42.406 | INFO | Uploaded 36GiB to GPU in 2.5816s (13.9448GiB/s) | 20260508 10:03:42.406 | INFO | Downloaded 38GiB to GPU in 2.71484s (13.9971GiB/s) | 20260508 10:03:47.547 | INFO | Analyzing chunk frames = [7242240,7242752) | 20260508 10:03:47.547 | INFO | MemInfo: { | 20260508 10:03:47.547 | INFO | "gpu" : " 17 GB", | 20260508 10:03:47.547 | INFO | "huge" : "", | 20260508 10:03:47.547 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:47.547 | INFO | "malloc" : " 14 GB", | 20260508 10:03:47.547 | INFO | "pinned" : " 29 GB" | 20260508 10:03:47.547 | INFO | } | 20260508 10:03:47.547 | INFO | Perf report for the last 5.14155 seconds: | 20260508 10:03:47.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:47.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:03:47.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:03:47.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:47.547 | INFO | Uploaded 36GiB to GPU in 2.41166s (14.9275GiB/s) | 20260508 10:03:47.547 | INFO | Downloaded 34GiB to GPU in 2.47564s (13.7338GiB/s) | 20260508 10:03:52.686 | INFO | Analyzing chunk frames = [7242752,7243264) | 20260508 10:03:52.686 | INFO | MemInfo: { | 20260508 10:03:52.686 | INFO | "gpu" : " 17 GB", | 20260508 10:03:52.686 | INFO | "huge" : "", | 20260508 10:03:52.686 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:52.686 | INFO | "malloc" : " 14 GB", | 20260508 10:03:52.686 | INFO | "pinned" : " 29 GB" | 20260508 10:03:52.686 | INFO | } | 20260508 10:03:52.686 | INFO | Perf report for the last 5.13921 seconds: | 20260508 10:03:52.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:52.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:03:52.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:03:52.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:52.686 | INFO | Uploaded 36GiB to GPU in 2.41952s (14.879GiB/s) | 20260508 10:03:52.686 | INFO | Downloaded 34GiB to GPU in 2.4359s (13.9579GiB/s) | 20260508 10:03:57.819 | INFO | Analyzing chunk frames = [7243264,7243776) | 20260508 10:03:57.819 | INFO | MemInfo: { | 20260508 10:03:57.819 | INFO | "gpu" : " 17 GB", | 20260508 10:03:57.819 | INFO | "huge" : "", | 20260508 10:03:57.819 | INFO | "huge_shared" : " 29 GB", | 20260508 10:03:57.819 | INFO | "malloc" : " 14 GB", | 20260508 10:03:57.819 | INFO | "pinned" : " 29 GB" | 20260508 10:03:57.819 | INFO | } | 20260508 10:03:57.819 | INFO | Perf report for the last 5.13342 seconds: | 20260508 10:03:57.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:03:57.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 10:03:57.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:03:57.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:03:57.820 | INFO | Uploaded 36GiB to GPU in 2.43655s (14.775GiB/s) | 20260508 10:03:57.820 | INFO | Downloaded 34GiB to GPU in 2.4733s (13.7468GiB/s) | 20260508 10:04:02.961 | INFO | Analyzing chunk frames = [7243776,7244288) | 20260508 10:04:02.961 | INFO | MemInfo: { | 20260508 10:04:02.961 | INFO | "gpu" : " 17 GB", | 20260508 10:04:02.961 | INFO | "huge" : "", | 20260508 10:04:02.961 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:02.961 | INFO | "malloc" : " 14 GB", | 20260508 10:04:02.961 | INFO | "pinned" : " 29 GB" | 20260508 10:04:02.961 | INFO | } | 20260508 10:04:02.961 | INFO | Perf report for the last 5.14188 seconds: | 20260508 10:04:02.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:02.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.962 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 10:04:02.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.962 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 10:04:02.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:02.962 | INFO | Uploaded 36GiB to GPU in 2.36934s (15.1941GiB/s) | 20260508 10:04:02.962 | INFO | Downloaded 34GiB to GPU in 2.52019s (13.4911GiB/s) | 20260508 10:04:08.100 | INFO | Analyzing chunk frames = [7244288,7244800) | 20260508 10:04:08.100 | INFO | MemInfo: { | 20260508 10:04:08.100 | INFO | "gpu" : " 17 GB", | 20260508 10:04:08.100 | INFO | "huge" : "", | 20260508 10:04:08.100 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:08.100 | INFO | "malloc" : " 14 GB", | 20260508 10:04:08.100 | INFO | "pinned" : " 29 GB" | 20260508 10:04:08.100 | INFO | } | 20260508 10:04:08.100 | INFO | Perf report for the last 5.13913 seconds: | 20260508 10:04:08.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:08.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.101 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:04:08.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.101 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:04:08.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:08.101 | INFO | Uploaded 36GiB to GPU in 2.33933s (15.389GiB/s) | 20260508 10:04:08.101 | INFO | Downloaded 34GiB to GPU in 2.45059s (13.8742GiB/s) | 20260508 10:04:13.246 | INFO | Analyzing chunk frames = [7244800,7245312) | 20260508 10:04:13.246 | INFO | MemInfo: { | 20260508 10:04:13.246 | INFO | "gpu" : " 17 GB", | 20260508 10:04:13.246 | INFO | "huge" : "", | 20260508 10:04:13.246 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:13.246 | INFO | "malloc" : " 14 GB", | 20260508 10:04:13.246 | INFO | "pinned" : " 29 GB" | 20260508 10:04:13.246 | INFO | } | 20260508 10:04:13.246 | INFO | Perf report for the last 5.14578 seconds: | 20260508 10:04:13.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:13.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.247 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 10:04:13.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.247 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:04:13.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:13.247 | INFO | Uploaded 36GiB to GPU in 2.37949s (15.1293GiB/s) | 20260508 10:04:13.247 | INFO | Downloaded 34GiB to GPU in 2.4509s (13.8724GiB/s) | 20260508 10:04:18.382 | INFO | Analyzing chunk frames = [7245312,7245824) | 20260508 10:04:18.382 | INFO | MemInfo: { | 20260508 10:04:18.382 | INFO | "gpu" : " 17 GB", | 20260508 10:04:18.382 | INFO | "huge" : "", | 20260508 10:04:18.382 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:18.382 | INFO | "malloc" : " 14 GB", | 20260508 10:04:18.382 | INFO | "pinned" : " 29 GB" | 20260508 10:04:18.382 | INFO | } | 20260508 10:04:18.382 | INFO | Perf report for the last 5.13612 seconds: | 20260508 10:04:18.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:18.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 10:04:18.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:04:18.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:18.383 | INFO | Uploaded 36GiB to GPU in 2.37899s (15.1324GiB/s) | 20260508 10:04:18.383 | INFO | Downloaded 34GiB to GPU in 2.44345s (13.9148GiB/s) | 20260508 10:04:18.480 | INFO | Starting a round of Baz IO | 20260508 10:04:21.905 | INFO | Populating RTMetrics report | 20260508 10:04:21.905 | INFO | RTMetrics report populated | 20260508 10:04:21.905 | INFO | Opening tmp RTMetrics json file | 20260508 10:04:21.912 | INFO | Serializing RTMetrics to json | 20260508 10:04:21.913 | INFO | Serialization done, swapping tmp file now... | 20260508 10:04:21.914 | INFO | RTMetrics done | 20260508 10:04:21.970 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.489 seconds | 20260508 10:04:21.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:04:21.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.233006% / 0% / 6.35783e-05% | 20260508 10:04:21.970 | INFO | Baz IO overall: | 20260508 10:04:21.970 | INFO | BytesWritten : 3151006044521 | 20260508 10:04:21.970 | INFO | NumEvents : 1515058278126 | 20260508 10:04:23.520 | INFO | Analyzing chunk frames = [7245824,7246336) | 20260508 10:04:23.520 | INFO | MemInfo: { | 20260508 10:04:23.520 | INFO | "gpu" : " 17 GB", | 20260508 10:04:23.520 | INFO | "huge" : "", | 20260508 10:04:23.520 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:23.520 | INFO | "malloc" : " 14 GB", | 20260508 10:04:23.520 | INFO | "pinned" : " 29 GB" | 20260508 10:04:23.520 | INFO | } | 20260508 10:04:23.521 | INFO | Perf report for the last 5.13828 seconds: | 20260508 10:04:23.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:23.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:23.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:23.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:23.521 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 10:04:23.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:23.521 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:04:23.521 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 10:04:23.521 | INFO | BAZWRITER : 67.2%, 2.5, 86.9ms, 0 | 20260508 10:04:23.521 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:04:23.521 | INFO | Uploaded 36GiB to GPU in 2.57192s (13.9973GiB/s) | 20260508 10:04:23.521 | INFO | Downloaded 38GiB to GPU in 2.71037s (14.0203GiB/s) | 20260508 10:04:28.658 | INFO | Analyzing chunk frames = [7246336,7246848) | 20260508 10:04:28.658 | INFO | MemInfo: { | 20260508 10:04:28.658 | INFO | "gpu" : " 17 GB", | 20260508 10:04:28.658 | INFO | "huge" : "", | 20260508 10:04:28.659 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:28.659 | INFO | "malloc" : " 14 GB", | 20260508 10:04:28.659 | INFO | "pinned" : " 29 GB" | 20260508 10:04:28.659 | INFO | } | 20260508 10:04:28.659 | INFO | Perf report for the last 5.138 seconds: | 20260508 10:04:28.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:28.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:04:28.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:04:28.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:28.659 | INFO | Uploaded 36GiB to GPU in 2.46445s (14.6077GiB/s) | 20260508 10:04:28.659 | INFO | Downloaded 34GiB to GPU in 2.47668s (13.7281GiB/s) | 20260508 10:04:33.798 | INFO | Analyzing chunk frames = [7246848,7247360) | 20260508 10:04:33.798 | INFO | MemInfo: { | 20260508 10:04:33.798 | INFO | "gpu" : " 17 GB", | 20260508 10:04:33.798 | INFO | "huge" : "", | 20260508 10:04:33.798 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:33.798 | INFO | "malloc" : " 14 GB", | 20260508 10:04:33.798 | INFO | "pinned" : " 29 GB" | 20260508 10:04:33.798 | INFO | } | 20260508 10:04:33.798 | INFO | Perf report for the last 5.13894 seconds: | 20260508 10:04:33.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:33.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 10:04:33.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:04:33.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:33.798 | INFO | Uploaded 36GiB to GPU in 2.45302s (14.6758GiB/s) | 20260508 10:04:33.798 | INFO | Downloaded 34GiB to GPU in 2.42269s (14.034GiB/s) | 20260508 10:04:38.942 | INFO | Analyzing chunk frames = [7247360,7247872) | 20260508 10:04:38.942 | INFO | MemInfo: { | 20260508 10:04:38.942 | INFO | "gpu" : " 17 GB", | 20260508 10:04:38.942 | INFO | "huge" : "", | 20260508 10:04:38.942 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:38.942 | INFO | "malloc" : " 14 GB", | 20260508 10:04:38.942 | INFO | "pinned" : " 29 GB" | 20260508 10:04:38.942 | INFO | } | 20260508 10:04:38.942 | INFO | Perf report for the last 5.14465 seconds: | 20260508 10:04:38.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:38.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.942 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 10:04:38.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.942 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:04:38.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:38.943 | INFO | Uploaded 36GiB to GPU in 2.49673s (14.4188GiB/s) | 20260508 10:04:38.943 | INFO | Downloaded 34GiB to GPU in 2.42948s (13.9948GiB/s) | 20260508 10:04:44.076 | INFO | Analyzing chunk frames = [7247872,7248384) | 20260508 10:04:44.076 | INFO | MemInfo: { | 20260508 10:04:44.076 | INFO | "gpu" : " 17 GB", | 20260508 10:04:44.076 | INFO | "huge" : "", | 20260508 10:04:44.076 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:44.076 | INFO | "malloc" : " 14 GB", | 20260508 10:04:44.076 | INFO | "pinned" : " 29 GB" | 20260508 10:04:44.076 | INFO | } | 20260508 10:04:44.076 | INFO | Perf report for the last 5.13381 seconds: | 20260508 10:04:44.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:44.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.076 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:04:44.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.076 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:04:44.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:44.077 | INFO | Uploaded 36GiB to GPU in 2.42983s (14.8158GiB/s) | 20260508 10:04:44.077 | INFO | Downloaded 34GiB to GPU in 2.52875s (13.4454GiB/s) | 20260508 10:04:49.220 | INFO | Analyzing chunk frames = [7248384,7248896) | 20260508 10:04:49.220 | INFO | MemInfo: { | 20260508 10:04:49.220 | INFO | "gpu" : " 17 GB", | 20260508 10:04:49.220 | INFO | "huge" : "", | 20260508 10:04:49.220 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:49.220 | INFO | "malloc" : " 14 GB", | 20260508 10:04:49.220 | INFO | "pinned" : " 29 GB" | 20260508 10:04:49.220 | INFO | } | 20260508 10:04:49.220 | INFO | Perf report for the last 5.14374 seconds: | 20260508 10:04:49.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:49.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:04:49.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:04:49.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:49.220 | INFO | Uploaded 36GiB to GPU in 2.38642s (15.0853GiB/s) | 20260508 10:04:49.220 | INFO | Downloaded 34GiB to GPU in 2.43464s (13.9651GiB/s) | 20260508 10:04:54.358 | INFO | Analyzing chunk frames = [7248896,7249408) | 20260508 10:04:54.358 | INFO | MemInfo: { | 20260508 10:04:54.358 | INFO | "gpu" : " 17 GB", | 20260508 10:04:54.358 | INFO | "huge" : "", | 20260508 10:04:54.358 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:54.358 | INFO | "malloc" : " 14 GB", | 20260508 10:04:54.358 | INFO | "pinned" : " 29 GB" | 20260508 10:04:54.358 | INFO | } | 20260508 10:04:54.358 | INFO | Perf report for the last 5.13836 seconds: | 20260508 10:04:54.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:54.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.358 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 10:04:54.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.358 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:04:54.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:54.359 | INFO | Uploaded 36GiB to GPU in 2.43861s (14.7625GiB/s) | 20260508 10:04:54.359 | INFO | Downloaded 34GiB to GPU in 2.4387s (13.9418GiB/s) | 20260508 10:04:59.495 | INFO | Analyzing chunk frames = [7249408,7249920) | 20260508 10:04:59.495 | INFO | MemInfo: { | 20260508 10:04:59.495 | INFO | "gpu" : " 17 GB", | 20260508 10:04:59.495 | INFO | "huge" : "", | 20260508 10:04:59.495 | INFO | "huge_shared" : " 29 GB", | 20260508 10:04:59.495 | INFO | "malloc" : " 14 GB", | 20260508 10:04:59.495 | INFO | "pinned" : " 29 GB" | 20260508 10:04:59.495 | INFO | } | 20260508 10:04:59.495 | INFO | Perf report for the last 5.13721 seconds: | 20260508 10:04:59.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:04:59.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.496 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:04:59.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.496 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:04:59.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:04:59.496 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:04:59.496 | INFO | Total mem usage: 16.7 GiB | 20260508 10:04:59.496 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:04:59.496 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:04:59.496 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:04:59.496 | INFO | High Water Breakdown: | 20260508 10:04:59.496 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:04:59.496 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:04:59.496 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:04:59.496 | INFO | ------------------------------------------ | 20260508 10:04:59.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:04:59.496 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:04:59.496 | INFO | Total mem usage: 26.5 GiB | 20260508 10:04:59.496 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:04:59.496 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:04:59.496 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:04:59.496 | INFO | High Water Breakdown: | 20260508 10:04:59.496 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:04:59.496 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:04:59.496 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:04:59.496 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:04:59.496 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:04:59.496 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:04:59.496 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:04:59.496 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:04:59.496 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:04:59.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:04:59.496 | INFO | Uploaded 36GiB to GPU in 2.42729s (14.8313GiB/s) | 20260508 10:04:59.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:04:59.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:04:59.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:04:59.497 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:04:59.497 | INFO | ------------------------------------------ | 20260508 10:04:59.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:04:59.497 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:04:59.497 | INFO | Total mem usage: 27.8 GiB | 20260508 10:04:59.497 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:04:59.497 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:04:59.497 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:04:59.497 | INFO | High Water Breakdown: | 20260508 10:04:59.497 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:04:59.497 | INFO | ------------------------------------------ | 20260508 10:04:59.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:04:59.497 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:04:59.497 | INFO | Total mem usage: 15.7 GiB | 20260508 10:04:59.497 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:04:59.497 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:04:59.497 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:04:59.497 | INFO | High Water Breakdown: | 20260508 10:04:59.497 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:04:59.497 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:04:59.497 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:04:59.497 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:04:59.497 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:04:59.497 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:04:59.497 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:04:59.497 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:04:59.497 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:04:59.497 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:04:59.497 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:04:59.497 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:04:59.497 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:04:59.497 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:04:59.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:04:59.497 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:04:59.497 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:04:59.497 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:04:59.497 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:04:59.497 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:04:59.497 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:04:59.497 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:04:59.497 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:04:59.497 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:04:59.497 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:04:59.497 | INFO | ------------------------------------------ | 20260508 10:04:59.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:04:59.496 | INFO | Downloaded 34GiB to GPU in 2.42318s (14.0312GiB/s) | 20260508 10:04:59.497 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:04:59.497 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:04:59.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:04:59.497 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.42e+01 / 6.61e-01 | 20260508 10:04:59.497 | INFO | PerfReport: 0.29%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.13e-02 / 5.71e-03 | 20260508 10:04:59.497 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.44e+00, 5.90e-01 / 2.44e-01 | 20260508 10:04:59.497 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.47e+00, 2.09e-01 / 1.26e-01 | 20260508 10:04:59.497 | INFO | PerfReport: 15.98%, DME -- 4.19e+01 / 4.49e+01, 3.58e+00 / 9.34e-01 | 20260508 10:04:59.497 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.87e+00 / 1.36e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.32e+00 / 2.41e+00, 4.05e-01 / 2.93e-01 | 20260508 10:04:59.497 | INFO | PerfReport: 3.01%, Metrics -- 9.88e-01 / 1.16e+00, 2.46e-01 / 1.35e-01 | 20260508 10:04:59.497 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.33e-01, 4.79e-02 / 2.95e-02 | 20260508 10:04:59.497 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:04:59.497 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:04:59.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:04:59.497 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 24.12%, Upload -- 2.54e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 24.65%, Download -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 31.16%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 2.07%, LockUpload -- 2.18e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 2.62%, LockDownload -- 2.76e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:04:59.497 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 10:04:59.594 | INFO | Starting a round of Baz IO | 20260508 10:05:03.010 | INFO | Populating RTMetrics report | 20260508 10:05:03.010 | INFO | RTMetrics report populated | 20260508 10:05:03.011 | INFO | Opening tmp RTMetrics json file | 20260508 10:05:03.019 | INFO | Serializing RTMetrics to json | 20260508 10:05:03.020 | INFO | Serialization done, swapping tmp file now... | 20260508 10:05:03.021 | INFO | RTMetrics done | 20260508 10:05:03.079 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.485 seconds | 20260508 10:05:03.079 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.00% / 59.76% | 20260508 10:05:03.079 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232816% / 0% / 4.76837e-05% | 20260508 10:05:03.079 | INFO | Baz IO overall: | 20260508 10:05:03.080 | INFO | BytesWritten : 3152579893467 | 20260508 10:05:03.080 | INFO | NumEvents : 1515801138932 | 20260508 10:05:04.633 | INFO | Analyzing chunk frames = [7249920,7250432) | 20260508 10:05:04.633 | INFO | MemInfo: { | 20260508 10:05:04.633 | INFO | "gpu" : " 17 GB", | 20260508 10:05:04.633 | INFO | "huge" : "", | 20260508 10:05:04.633 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:04.633 | INFO | "malloc" : " 14 GB", | 20260508 10:05:04.633 | INFO | "pinned" : " 29 GB" | 20260508 10:05:04.633 | INFO | } | 20260508 10:05:04.633 | INFO | Perf report for the last 5.13721 seconds: | 20260508 10:05:04.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:04.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:04.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:04.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:04.633 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 10:05:04.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:04.633 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:05:04.633 | INFO | PRE_HQ : 65.5%, 1.6, 54.5ms, 0 | 20260508 10:05:04.633 | INFO | BAZWRITER : 67.1%, 2.5, 86.6ms, 0 | 20260508 10:05:04.633 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:05:04.633 | INFO | Uploaded 36GiB to GPU in 2.60505s (13.8193GiB/s) | 20260508 10:05:04.633 | INFO | Downloaded 38GiB to GPU in 2.71063s (14.0189GiB/s) | 20260508 10:05:09.771 | INFO | Analyzing chunk frames = [7250432,7250944) | 20260508 10:05:09.771 | INFO | MemInfo: { | 20260508 10:05:09.771 | INFO | "gpu" : " 17 GB", | 20260508 10:05:09.771 | INFO | "huge" : "", | 20260508 10:05:09.771 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:09.771 | INFO | "malloc" : " 14 GB", | 20260508 10:05:09.771 | INFO | "pinned" : " 29 GB" | 20260508 10:05:09.771 | INFO | } | 20260508 10:05:09.771 | INFO | Perf report for the last 5.13872 seconds: | 20260508 10:05:09.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:09.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.771 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:05:09.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.772 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 10:05:09.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:09.772 | INFO | Uploaded 36GiB to GPU in 2.48662s (14.4775GiB/s) | 20260508 10:05:09.772 | INFO | Downloaded 34GiB to GPU in 2.45834s (13.8305GiB/s) | 20260508 10:05:14.915 | INFO | Analyzing chunk frames = [7250944,7251456) | 20260508 10:05:14.915 | INFO | MemInfo: { | 20260508 10:05:14.915 | INFO | "gpu" : " 17 GB", | 20260508 10:05:14.915 | INFO | "huge" : "", | 20260508 10:05:14.915 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:14.915 | INFO | "malloc" : " 14 GB", | 20260508 10:05:14.915 | INFO | "pinned" : " 29 GB" | 20260508 10:05:14.915 | INFO | } | 20260508 10:05:14.915 | INFO | Perf report for the last 5.14349 seconds: | 20260508 10:05:14.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:14.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:05:14.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:05:14.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:14.915 | INFO | Uploaded 36GiB to GPU in 2.47587s (14.5404GiB/s) | 20260508 10:05:14.915 | INFO | Downloaded 34GiB to GPU in 2.41764s (14.0633GiB/s) | 20260508 10:05:20.053 | INFO | Analyzing chunk frames = [7251456,7251968) | 20260508 10:05:20.053 | INFO | MemInfo: { | 20260508 10:05:20.053 | INFO | "gpu" : " 17 GB", | 20260508 10:05:20.053 | INFO | "huge" : "", | 20260508 10:05:20.053 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:20.053 | INFO | "malloc" : " 14 GB", | 20260508 10:05:20.053 | INFO | "pinned" : " 29 GB" | 20260508 10:05:20.053 | INFO | } | 20260508 10:05:20.053 | INFO | Perf report for the last 5.13792 seconds: | 20260508 10:05:20.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:20.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:05:20.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:05:20.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:20.053 | INFO | Uploaded 36GiB to GPU in 2.48531s (14.4851GiB/s) | 20260508 10:05:20.053 | INFO | Downloaded 34GiB to GPU in 2.40295s (14.1492GiB/s) | 20260508 10:05:25.198 | INFO | Analyzing chunk frames = [7251968,7252480) | 20260508 10:05:25.198 | INFO | MemInfo: { | 20260508 10:05:25.198 | INFO | "gpu" : " 17 GB", | 20260508 10:05:25.198 | INFO | "huge" : "", | 20260508 10:05:25.198 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:25.198 | INFO | "malloc" : " 14 GB", | 20260508 10:05:25.198 | INFO | "pinned" : " 29 GB" | 20260508 10:05:25.198 | INFO | } | 20260508 10:05:25.198 | INFO | Perf report for the last 5.14552 seconds: | 20260508 10:05:25.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:25.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.198 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 10:05:25.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.199 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:05:25.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:25.199 | INFO | Uploaded 36GiB to GPU in 2.47904s (14.5218GiB/s) | 20260508 10:05:25.199 | INFO | Downloaded 34GiB to GPU in 2.51916s (13.4966GiB/s) | 20260508 10:05:30.329 | INFO | Analyzing chunk frames = [7252480,7252992) | 20260508 10:05:30.329 | INFO | MemInfo: { | 20260508 10:05:30.329 | INFO | "gpu" : " 17 GB", | 20260508 10:05:30.329 | INFO | "huge" : "", | 20260508 10:05:30.329 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:30.329 | INFO | "malloc" : " 14 GB", | 20260508 10:05:30.329 | INFO | "pinned" : " 29 GB" | 20260508 10:05:30.329 | INFO | } | 20260508 10:05:30.329 | INFO | Perf report for the last 5.13056 seconds: | 20260508 10:05:30.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:30.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.329 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 10:05:30.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.329 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 10:05:30.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:30.330 | INFO | Uploaded 36GiB to GPU in 2.36122s (15.2463GiB/s) | 20260508 10:05:30.330 | INFO | Downloaded 34GiB to GPU in 2.43473s (13.9646GiB/s) | 20260508 10:05:35.474 | INFO | Analyzing chunk frames = [7252992,7253504) | 20260508 10:05:35.474 | INFO | MemInfo: { | 20260508 10:05:35.474 | INFO | "gpu" : " 17 GB", | 20260508 10:05:35.474 | INFO | "huge" : "", | 20260508 10:05:35.474 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:35.474 | INFO | "malloc" : " 14 GB", | 20260508 10:05:35.475 | INFO | "pinned" : " 29 GB" | 20260508 10:05:35.475 | INFO | } | 20260508 10:05:35.475 | INFO | Perf report for the last 5.14575 seconds: | 20260508 10:05:35.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:35.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 10:05:35.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:05:35.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:35.475 | INFO | Uploaded 36GiB to GPU in 2.39534s (15.0292GiB/s) | 20260508 10:05:35.475 | INFO | Downloaded 34GiB to GPU in 2.44527s (13.9044GiB/s) | 20260508 10:05:40.616 | INFO | Analyzing chunk frames = [7253504,7254016) | 20260508 10:05:40.616 | INFO | MemInfo: { | 20260508 10:05:40.616 | INFO | "gpu" : " 17 GB", | 20260508 10:05:40.616 | INFO | "huge" : "", | 20260508 10:05:40.616 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:40.616 | INFO | "malloc" : " 14 GB", | 20260508 10:05:40.617 | INFO | "pinned" : " 29 GB" | 20260508 10:05:40.617 | INFO | } | 20260508 10:05:40.617 | INFO | Perf report for the last 5.14191 seconds: | 20260508 10:05:40.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:40.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:05:40.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:05:40.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:40.617 | INFO | Uploaded 36GiB to GPU in 2.44637s (14.7157GiB/s) | 20260508 10:05:40.617 | INFO | Downloaded 34GiB to GPU in 2.45284s (13.8615GiB/s) | 20260508 10:05:40.729 | INFO | Starting a round of Baz IO | 20260508 10:05:44.138 | INFO | Populating RTMetrics report | 20260508 10:05:44.138 | INFO | RTMetrics report populated | 20260508 10:05:44.138 | INFO | Opening tmp RTMetrics json file | 20260508 10:05:44.144 | INFO | Serializing RTMetrics to json | 20260508 10:05:44.146 | INFO | Serialization done, swapping tmp file now... | 20260508 10:05:44.146 | INFO | RTMetrics done | 20260508 10:05:44.203 | INFO | Finished a round of Baz IO: Wrote 1.465 GiB in 3.473 seconds | 20260508 10:05:44.203 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.76% | 20260508 10:05:44.203 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232434% / 3.97364e-06% / 8.34465e-05% | 20260508 10:05:44.203 | INFO | Baz IO overall: | 20260508 10:05:44.204 | INFO | BytesWritten : 3154153190475 | 20260508 10:05:44.204 | INFO | NumEvents : 1516543538044 | 20260508 10:05:45.748 | INFO | Analyzing chunk frames = [7254016,7254528) | 20260508 10:05:45.748 | INFO | MemInfo: { | 20260508 10:05:45.748 | INFO | "gpu" : " 17 GB", | 20260508 10:05:45.748 | INFO | "huge" : "", | 20260508 10:05:45.748 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:45.748 | INFO | "malloc" : " 14 GB", | 20260508 10:05:45.748 | INFO | "pinned" : " 29 GB" | 20260508 10:05:45.748 | INFO | } | 20260508 10:05:45.748 | INFO | Perf report for the last 5.13149 seconds: | 20260508 10:05:45.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:45.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:45.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:45.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:45.748 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:05:45.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:45.748 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260508 10:05:45.748 | INFO | PRE_HQ : 66.2%, 1.7, 58.3ms, 0 | 20260508 10:05:45.748 | INFO | BAZWRITER : 67.3%, 2.9, 100.6ms, 0 | 20260508 10:05:45.748 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:05:45.748 | INFO | Uploaded 36GiB to GPU in 2.62482s (13.7152GiB/s) | 20260508 10:05:45.748 | INFO | Downloaded 38GiB to GPU in 2.7134s (14.0046GiB/s) | 20260508 10:05:50.889 | INFO | Analyzing chunk frames = [7254528,7255040) | 20260508 10:05:50.889 | INFO | MemInfo: { | 20260508 10:05:50.889 | INFO | "gpu" : " 17 GB", | 20260508 10:05:50.889 | INFO | "huge" : "", | 20260508 10:05:50.889 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:50.889 | INFO | "malloc" : " 14 GB", | 20260508 10:05:50.889 | INFO | "pinned" : " 29 GB" | 20260508 10:05:50.889 | INFO | } | 20260508 10:05:50.889 | INFO | Perf report for the last 5.14124 seconds: | 20260508 10:05:50.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:50.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.889 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 10:05:50.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.890 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:05:50.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:50.890 | INFO | Uploaded 36GiB to GPU in 2.49271s (14.4421GiB/s) | 20260508 10:05:50.890 | INFO | Downloaded 34GiB to GPU in 2.46204s (13.8097GiB/s) | 20260508 10:05:56.026 | INFO | Analyzing chunk frames = [7255040,7255552) | 20260508 10:05:56.026 | INFO | MemInfo: { | 20260508 10:05:56.026 | INFO | "gpu" : " 17 GB", | 20260508 10:05:56.026 | INFO | "huge" : "", | 20260508 10:05:56.026 | INFO | "huge_shared" : " 29 GB", | 20260508 10:05:56.026 | INFO | "malloc" : " 14 GB", | 20260508 10:05:56.026 | INFO | "pinned" : " 29 GB" | 20260508 10:05:56.026 | INFO | } | 20260508 10:05:56.026 | INFO | Perf report for the last 5.13666 seconds: | 20260508 10:05:56.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:05:56.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.026 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:05:56.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.026 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:05:56.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:05:56.027 | INFO | Uploaded 36GiB to GPU in 2.44795s (14.7062GiB/s) | 20260508 10:05:56.027 | INFO | Downloaded 34GiB to GPU in 2.45693s (13.8384GiB/s) | 20260508 10:06:01.168 | INFO | Analyzing chunk frames = [7255552,7256064) | 20260508 10:06:01.168 | INFO | MemInfo: { | 20260508 10:06:01.168 | INFO | "gpu" : " 17 GB", | 20260508 10:06:01.168 | INFO | "huge" : "", | 20260508 10:06:01.168 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:01.169 | INFO | "malloc" : " 14 GB", | 20260508 10:06:01.169 | INFO | "pinned" : " 29 GB" | 20260508 10:06:01.169 | INFO | } | 20260508 10:06:01.169 | INFO | Perf report for the last 5.14251 seconds: | 20260508 10:06:01.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:01.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:06:01.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:06:01.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:01.169 | INFO | Uploaded 36GiB to GPU in 2.42901s (14.8209GiB/s) | 20260508 10:06:01.169 | INFO | Downloaded 34GiB to GPU in 2.41866s (14.0574GiB/s) | 20260508 10:06:06.310 | INFO | Analyzing chunk frames = [7256064,7256576) | 20260508 10:06:06.310 | INFO | MemInfo: { | 20260508 10:06:06.310 | INFO | "gpu" : " 17 GB", | 20260508 10:06:06.310 | INFO | "huge" : "", | 20260508 10:06:06.310 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:06.310 | INFO | "malloc" : " 14 GB", | 20260508 10:06:06.310 | INFO | "pinned" : " 29 GB" | 20260508 10:06:06.310 | INFO | } | 20260508 10:06:06.310 | INFO | Perf report for the last 5.14157 seconds: | 20260508 10:06:06.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:06.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.310 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 10:06:06.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.310 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:06:06.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:06.311 | INFO | Uploaded 36GiB to GPU in 2.33451s (15.4208GiB/s) | 20260508 10:06:06.311 | INFO | Downloaded 34GiB to GPU in 2.51688s (13.5088GiB/s) | 20260508 10:06:11.447 | INFO | Analyzing chunk frames = [7256576,7257088) | 20260508 10:06:11.447 | INFO | MemInfo: { | 20260508 10:06:11.447 | INFO | "gpu" : " 17 GB", | 20260508 10:06:11.447 | INFO | "huge" : "", | 20260508 10:06:11.447 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:11.447 | INFO | "malloc" : " 14 GB", | 20260508 10:06:11.447 | INFO | "pinned" : " 29 GB" | 20260508 10:06:11.447 | INFO | } | 20260508 10:06:11.448 | INFO | Perf report for the last 5.13734 seconds: | 20260508 10:06:11.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:11.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | ANALYSIS : 65.1%, 3.0, 101.3ms, 0 | 20260508 10:06:11.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:06:11.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:11.448 | INFO | Uploaded 36GiB to GPU in 2.31922s (15.5224GiB/s) | 20260508 10:06:11.448 | INFO | Downloaded 34GiB to GPU in 2.45055s (13.8744GiB/s) | 20260508 10:06:16.580 | INFO | Analyzing chunk frames = [7257088,7257600) | 20260508 10:06:16.580 | INFO | MemInfo: { | 20260508 10:06:16.580 | INFO | "gpu" : " 17 GB", | 20260508 10:06:16.580 | INFO | "huge" : "", | 20260508 10:06:16.580 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:16.580 | INFO | "malloc" : " 14 GB", | 20260508 10:06:16.580 | INFO | "pinned" : " 29 GB" | 20260508 10:06:16.580 | INFO | } | 20260508 10:06:16.580 | INFO | Perf report for the last 5.13254 seconds: | 20260508 10:06:16.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:16.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 10:06:16.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:06:16.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:16.580 | INFO | Uploaded 36GiB to GPU in 2.39094s (15.0568GiB/s) | 20260508 10:06:16.580 | INFO | Downloaded 34GiB to GPU in 2.44645s (13.8977GiB/s) | 20260508 10:06:21.723 | INFO | Analyzing chunk frames = [7257600,7258112) | 20260508 10:06:21.723 | INFO | MemInfo: { | 20260508 10:06:21.723 | INFO | "gpu" : " 17 GB", | 20260508 10:06:21.723 | INFO | "huge" : "", | 20260508 10:06:21.723 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:21.723 | INFO | "malloc" : " 14 GB", | 20260508 10:06:21.724 | INFO | "pinned" : " 29 GB" | 20260508 10:06:21.724 | INFO | } | 20260508 10:06:21.724 | INFO | Perf report for the last 5.14344 seconds: | 20260508 10:06:21.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:21.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:06:21.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:06:21.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:21.724 | INFO | Uploaded 36GiB to GPU in 2.40619s (14.9614GiB/s) | 20260508 10:06:21.724 | INFO | Downloaded 34GiB to GPU in 2.46155s (13.8124GiB/s) | 20260508 10:06:21.724 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:06:21.724 | INFO | Total mem usage: 16.7 GiB | 20260508 10:06:21.724 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:06:21.724 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.7%) | 20260508 10:06:21.724 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:06:21.724 | INFO | High Water Breakdown: | 20260508 10:06:21.724 | INFO | 59.8% : 8.7 GiB ( 84.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:06:21.724 | INFO | 40.1% : 5.83 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:06:21.724 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:06:21.724 | INFO | ------------------------------------------ | 20260508 10:06:21.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:06:21.724 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:06:21.724 | INFO | Total mem usage: 26.5 GiB | 20260508 10:06:21.724 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:06:21.724 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:06:21.724 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:06:21.724 | INFO | High Water Breakdown: | 20260508 10:06:21.724 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:06:21.724 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:06:21.724 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:06:21.724 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:06:21.724 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:06:21.724 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:06:21.724 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:06:21.724 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:06:21.724 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:06:21.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:06:21.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:06:21.724 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:06:21.724 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:06:21.724 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:06:21.724 | INFO | ------------------------------------------ | 20260508 10:06:21.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:06:21.724 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:06:21.724 | INFO | Total mem usage: 27.8 GiB | 20260508 10:06:21.724 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:06:21.724 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:06:21.724 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:06:21.724 | INFO | High Water Breakdown: | 20260508 10:06:21.724 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:06:21.724 | INFO | ------------------------------------------ | 20260508 10:06:21.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:06:21.724 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:06:21.724 | INFO | Total mem usage: 15.7 GiB | 20260508 10:06:21.724 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:06:21.724 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:06:21.724 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:06:21.724 | INFO | High Water Breakdown: | 20260508 10:06:21.724 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:06:21.724 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:06:21.724 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:06:21.724 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:06:21.724 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:06:21.724 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:06:21.724 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:06:21.724 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:06:21.724 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:06:21.724 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:06:21.724 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:06:21.724 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:06:21.724 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:06:21.724 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:06:21.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:06:21.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:06:21.724 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:06:21.724 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:06:21.724 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:06:21.724 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:06:21.724 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:06:21.724 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:06:21.724 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:06:21.724 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:06:21.724 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:06:21.724 | INFO | ------------------------------------------ | 20260508 10:06:21.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:06:21.724 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:06:21.724 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:06:21.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:06:21.724 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.59e-01 | 20260508 10:06:21.724 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.09e-01, 1.12e-02 / 1.00e-02 | 20260508 10:06:21.724 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.42e+00, 5.90e-01 / 2.46e-01 | 20260508 10:06:21.724 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.49e+00, 2.11e-01 / 1.06e-01 | 20260508 10:06:21.724 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.49e+01, 3.59e+00 / 9.76e-01 | 20260508 10:06:21.724 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.82e+00 / 1.46e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.47e+00, 4.08e-01 / 2.80e-01 | 20260508 10:06:21.724 | INFO | PerfReport: 3.02%, Metrics -- 9.88e-01 / 1.12e+00, 2.45e-01 / 1.35e-01 | 20260508 10:06:21.724 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.31e-01, 4.86e-02 / 3.65e-02 | 20260508 10:06:21.724 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:06:21.724 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:06:21.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:06:21.724 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 24.25%, Upload -- 2.55e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 24.61%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 31.10%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 2.10%, LockUpload -- 2.21e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 2.62%, LockDownload -- 2.76e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 15.30%, LockCompute -- 1.61e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 10:06:21.724 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 3.16e-02 / 0.00e+00 | 20260508 10:06:21.821 | INFO | Starting a round of Baz IO | 20260508 10:06:25.234 | INFO | Populating RTMetrics report | 20260508 10:06:25.235 | INFO | RTMetrics report populated | 20260508 10:06:25.235 | INFO | Opening tmp RTMetrics json file | 20260508 10:06:25.246 | INFO | Serializing RTMetrics to json | 20260508 10:06:25.247 | INFO | Serialization done, swapping tmp file now... | 20260508 10:06:25.248 | INFO | RTMetrics done | 20260508 10:06:25.299 | INFO | Finished a round of Baz IO: Wrote 1.467 GiB in 3.478 seconds | 20260508 10:06:25.299 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:06:25.299 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232255% / 3.97364e-06% / 5.96046e-05% | 20260508 10:06:25.299 | INFO | Baz IO overall: | 20260508 10:06:25.299 | INFO | BytesWritten : 3155727916281 | 20260508 10:06:25.299 | INFO | NumEvents : 1517287087886 | 20260508 10:06:26.868 | INFO | Analyzing chunk frames = [7258112,7258624) | 20260508 10:06:26.868 | INFO | MemInfo: { | 20260508 10:06:26.868 | INFO | "gpu" : " 17 GB", | 20260508 10:06:26.868 | INFO | "huge" : "", | 20260508 10:06:26.868 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:26.868 | INFO | "malloc" : " 14 GB", | 20260508 10:06:26.868 | INFO | "pinned" : " 29 GB" | 20260508 10:06:26.868 | INFO | } | 20260508 10:06:26.868 | INFO | Perf report for the last 5.1444 seconds: | 20260508 10:06:26.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:26.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:26.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:26.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:26.868 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:06:26.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:26.868 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:06:26.868 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 10:06:26.868 | INFO | BAZWRITER : 67.0%, 2.6, 88.3ms, 0 | 20260508 10:06:26.868 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:06:26.869 | INFO | Uploaded 36GiB to GPU in 2.54103s (14.1675GiB/s) | 20260508 10:06:26.869 | INFO | Downloaded 38GiB to GPU in 2.71164s (14.0137GiB/s) | 20260508 10:06:32.004 | INFO | Analyzing chunk frames = [7258624,7259136) | 20260508 10:06:32.004 | INFO | MemInfo: { | 20260508 10:06:32.004 | INFO | "gpu" : " 17 GB", | 20260508 10:06:32.004 | INFO | "huge" : "", | 20260508 10:06:32.004 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:32.004 | INFO | "malloc" : " 14 GB", | 20260508 10:06:32.004 | INFO | "pinned" : " 29 GB" | 20260508 10:06:32.004 | INFO | } | 20260508 10:06:32.004 | INFO | Perf report for the last 5.13619 seconds: | 20260508 10:06:32.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:32.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.004 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 10:06:32.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.004 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:06:32.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:32.005 | INFO | Uploaded 36GiB to GPU in 2.49647s (14.4204GiB/s) | 20260508 10:06:32.005 | INFO | Downloaded 34GiB to GPU in 2.47581s (13.7329GiB/s) | 20260508 10:06:37.143 | INFO | Analyzing chunk frames = [7259136,7259648) | 20260508 10:06:37.143 | INFO | MemInfo: { | 20260508 10:06:37.143 | INFO | "gpu" : " 17 GB", | 20260508 10:06:37.143 | INFO | "huge" : "", | 20260508 10:06:37.143 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:37.143 | INFO | "malloc" : " 14 GB", | 20260508 10:06:37.143 | INFO | "pinned" : " 29 GB" | 20260508 10:06:37.143 | INFO | } | 20260508 10:06:37.143 | INFO | Perf report for the last 5.13882 seconds: | 20260508 10:06:37.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:37.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:06:37.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:06:37.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:37.143 | INFO | Uploaded 36GiB to GPU in 2.44374s (14.7315GiB/s) | 20260508 10:06:37.143 | INFO | Downloaded 34GiB to GPU in 2.42755s (14.0059GiB/s) | 20260508 10:06:42.285 | INFO | Analyzing chunk frames = [7259648,7260160) | 20260508 10:06:42.285 | INFO | MemInfo: { | 20260508 10:06:42.285 | INFO | "gpu" : " 17 GB", | 20260508 10:06:42.285 | INFO | "huge" : "", | 20260508 10:06:42.286 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:42.286 | INFO | "malloc" : " 14 GB", | 20260508 10:06:42.286 | INFO | "pinned" : " 29 GB" | 20260508 10:06:42.286 | INFO | } | 20260508 10:06:42.286 | INFO | Perf report for the last 5.14275 seconds: | 20260508 10:06:42.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:42.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:06:42.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:06:42.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:42.286 | INFO | Uploaded 36GiB to GPU in 2.43116s (14.8077GiB/s) | 20260508 10:06:42.286 | INFO | Downloaded 34GiB to GPU in 2.43025s (13.9903GiB/s) | 20260508 10:06:47.422 | INFO | Analyzing chunk frames = [7260160,7260672) | 20260508 10:06:47.422 | INFO | MemInfo: { | 20260508 10:06:47.422 | INFO | "gpu" : " 17 GB", | 20260508 10:06:47.422 | INFO | "huge" : "", | 20260508 10:06:47.423 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:47.423 | INFO | "malloc" : " 14 GB", | 20260508 10:06:47.423 | INFO | "pinned" : " 29 GB" | 20260508 10:06:47.423 | INFO | } | 20260508 10:06:47.423 | INFO | Perf report for the last 5.13703 seconds: | 20260508 10:06:47.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:47.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:06:47.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:06:47.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:47.423 | INFO | Uploaded 36GiB to GPU in 2.26946s (15.8628GiB/s) | 20260508 10:06:47.423 | INFO | Downloaded 34GiB to GPU in 2.5015s (13.5919GiB/s) | 20260508 10:06:52.561 | INFO | Analyzing chunk frames = [7260672,7261184) | 20260508 10:06:52.561 | INFO | MemInfo: { | 20260508 10:06:52.561 | INFO | "gpu" : " 17 GB", | 20260508 10:06:52.561 | INFO | "huge" : "", | 20260508 10:06:52.561 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:52.561 | INFO | "malloc" : " 14 GB", | 20260508 10:06:52.561 | INFO | "pinned" : " 29 GB" | 20260508 10:06:52.561 | INFO | } | 20260508 10:06:52.561 | INFO | Perf report for the last 5.13814 seconds: | 20260508 10:06:52.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:52.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.561 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:06:52.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.561 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:06:52.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:52.562 | INFO | Uploaded 36GiB to GPU in 2.35303s (15.2994GiB/s) | 20260508 10:06:52.562 | INFO | Downloaded 34GiB to GPU in 2.44924s (13.8819GiB/s) | 20260508 10:06:57.697 | INFO | Analyzing chunk frames = [7261184,7261696) | 20260508 10:06:57.697 | INFO | MemInfo: { | 20260508 10:06:57.697 | INFO | "gpu" : " 17 GB", | 20260508 10:06:57.697 | INFO | "huge" : "", | 20260508 10:06:57.697 | INFO | "huge_shared" : " 29 GB", | 20260508 10:06:57.697 | INFO | "malloc" : " 14 GB", | 20260508 10:06:57.697 | INFO | "pinned" : " 29 GB" | 20260508 10:06:57.697 | INFO | } | 20260508 10:06:57.697 | INFO | Perf report for the last 5.13632 seconds: | 20260508 10:06:57.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:06:57.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.697 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:06:57.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.697 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:06:57.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:06:57.698 | INFO | Uploaded 36GiB to GPU in 2.39271s (15.0457GiB/s) | 20260508 10:06:57.698 | INFO | Downloaded 34GiB to GPU in 2.44857s (13.8856GiB/s) | 20260508 10:07:02.835 | INFO | Analyzing chunk frames = [7261696,7262208) | 20260508 10:07:02.835 | INFO | MemInfo: { | 20260508 10:07:02.835 | INFO | "gpu" : " 17 GB", | 20260508 10:07:02.835 | INFO | "huge" : "", | 20260508 10:07:02.835 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:02.835 | INFO | "malloc" : " 14 GB", | 20260508 10:07:02.835 | INFO | "pinned" : " 29 GB" | 20260508 10:07:02.835 | INFO | } | 20260508 10:07:02.835 | INFO | Perf report for the last 5.13787 seconds: | 20260508 10:07:02.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:02.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:07:02.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 10:07:02.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:02.835 | INFO | Uploaded 36GiB to GPU in 2.40042s (14.9973GiB/s) | 20260508 10:07:02.835 | INFO | Downloaded 34GiB to GPU in 2.45191s (13.8668GiB/s) | 20260508 10:07:02.932 | INFO | Starting a round of Baz IO | 20260508 10:07:06.333 | INFO | Populating RTMetrics report | 20260508 10:07:06.333 | INFO | RTMetrics report populated | 20260508 10:07:06.333 | INFO | Opening tmp RTMetrics json file | 20260508 10:07:06.344 | INFO | Serializing RTMetrics to json | 20260508 10:07:06.346 | INFO | Serialization done, swapping tmp file now... | 20260508 10:07:06.346 | INFO | RTMetrics done | 20260508 10:07:06.407 | INFO | Finished a round of Baz IO: Wrote 1.465 GiB in 3.475 seconds | 20260508 10:07:06.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:07:06.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23187% / 0% / 6.75519e-05% | 20260508 10:07:06.407 | INFO | Baz IO overall: | 20260508 10:07:06.407 | INFO | BytesWritten : 3157301211096 | 20260508 10:07:06.407 | INFO | NumEvents : 1518029648951 | 20260508 10:07:07.980 | INFO | Analyzing chunk frames = [7262208,7262720) | 20260508 10:07:07.980 | INFO | MemInfo: { | 20260508 10:07:07.980 | INFO | "gpu" : " 17 GB", | 20260508 10:07:07.980 | INFO | "huge" : "", | 20260508 10:07:07.980 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:07.980 | INFO | "malloc" : " 14 GB", | 20260508 10:07:07.980 | INFO | "pinned" : " 29 GB" | 20260508 10:07:07.980 | INFO | } | 20260508 10:07:07.980 | INFO | Perf report for the last 5.14498 seconds: | 20260508 10:07:07.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:07.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:07.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:07.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:07.980 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 10:07:07.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:07.980 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:07:07.980 | INFO | PRE_HQ : 65.1%, 1.6, 54.1ms, 0 | 20260508 10:07:07.980 | INFO | BAZWRITER : 66.8%, 2.5, 86.6ms, 0 | 20260508 10:07:07.980 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:07:07.980 | INFO | Uploaded 36GiB to GPU in 2.57474s (13.982GiB/s) | 20260508 10:07:07.980 | INFO | Downloaded 38GiB to GPU in 2.71613s (13.9905GiB/s) | 20260508 10:07:13.120 | INFO | Analyzing chunk frames = [7262720,7263232) | 20260508 10:07:13.120 | INFO | MemInfo: { | 20260508 10:07:13.120 | INFO | "gpu" : " 17 GB", | 20260508 10:07:13.120 | INFO | "huge" : "", | 20260508 10:07:13.120 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:13.120 | INFO | "malloc" : " 14 GB", | 20260508 10:07:13.120 | INFO | "pinned" : " 29 GB" | 20260508 10:07:13.120 | INFO | } | 20260508 10:07:13.120 | INFO | Perf report for the last 5.13999 seconds: | 20260508 10:07:13.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:13.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 10:07:13.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:07:13.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:13.120 | INFO | Uploaded 36GiB to GPU in 2.42644s (14.8365GiB/s) | 20260508 10:07:13.120 | INFO | Downloaded 34GiB to GPU in 2.48148s (13.7015GiB/s) | 20260508 10:07:18.263 | INFO | Analyzing chunk frames = [7263232,7263744) | 20260508 10:07:18.263 | INFO | MemInfo: { | 20260508 10:07:18.263 | INFO | "gpu" : " 17 GB", | 20260508 10:07:18.263 | INFO | "huge" : "", | 20260508 10:07:18.263 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:18.263 | INFO | "malloc" : " 14 GB", | 20260508 10:07:18.263 | INFO | "pinned" : " 29 GB" | 20260508 10:07:18.263 | INFO | } | 20260508 10:07:18.263 | INFO | Perf report for the last 5.14304 seconds: | 20260508 10:07:18.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:18.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.263 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 10:07:18.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.263 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:07:18.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:18.264 | INFO | Uploaded 36GiB to GPU in 2.42073s (14.8715GiB/s) | 20260508 10:07:18.264 | INFO | Downloaded 34GiB to GPU in 2.44412s (13.9109GiB/s) | 20260508 10:07:23.393 | INFO | Analyzing chunk frames = [7263744,7264256) | 20260508 10:07:23.393 | INFO | MemInfo: { | 20260508 10:07:23.393 | INFO | "gpu" : " 17 GB", | 20260508 10:07:23.393 | INFO | "huge" : "", | 20260508 10:07:23.393 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:23.393 | INFO | "malloc" : " 14 GB", | 20260508 10:07:23.393 | INFO | "pinned" : " 29 GB" | 20260508 10:07:23.394 | INFO | } | 20260508 10:07:23.394 | INFO | Perf report for the last 5.13034 seconds: | 20260508 10:07:23.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:23.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 10:07:23.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 10:07:23.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:23.394 | INFO | Uploaded 36GiB to GPU in 2.42455s (14.8481GiB/s) | 20260508 10:07:23.394 | INFO | Downloaded 34GiB to GPU in 2.45639s (13.8415GiB/s) | 20260508 10:07:28.534 | INFO | Analyzing chunk frames = [7264256,7264768) | 20260508 10:07:28.534 | INFO | MemInfo: { | 20260508 10:07:28.534 | INFO | "gpu" : " 17 GB", | 20260508 10:07:28.534 | INFO | "huge" : "", | 20260508 10:07:28.534 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:28.534 | INFO | "malloc" : " 14 GB", | 20260508 10:07:28.534 | INFO | "pinned" : " 29 GB" | 20260508 10:07:28.534 | INFO | } | 20260508 10:07:28.534 | INFO | Perf report for the last 5.14079 seconds: | 20260508 10:07:28.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:28.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 10:07:28.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:07:28.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:28.535 | INFO | Uploaded 36GiB to GPU in 2.35416s (15.2921GiB/s) | 20260508 10:07:28.535 | INFO | Downloaded 34GiB to GPU in 2.53019s (13.4377GiB/s) | 20260508 10:07:33.674 | INFO | Analyzing chunk frames = [7264768,7265280) | 20260508 10:07:33.674 | INFO | MemInfo: { | 20260508 10:07:33.674 | INFO | "gpu" : " 17 GB", | 20260508 10:07:33.674 | INFO | "huge" : "", | 20260508 10:07:33.674 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:33.674 | INFO | "malloc" : " 14 GB", | 20260508 10:07:33.674 | INFO | "pinned" : " 29 GB" | 20260508 10:07:33.674 | INFO | } | 20260508 10:07:33.674 | INFO | Perf report for the last 5.13942 seconds: | 20260508 10:07:33.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:33.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 10:07:33.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:07:33.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:33.674 | INFO | Uploaded 36GiB to GPU in 2.31913s (15.5231GiB/s) | 20260508 10:07:33.674 | INFO | Downloaded 34GiB to GPU in 2.45109s (13.8714GiB/s) | 20260508 10:07:38.818 | INFO | Analyzing chunk frames = [7265280,7265792) | 20260508 10:07:38.818 | INFO | MemInfo: { | 20260508 10:07:38.818 | INFO | "gpu" : " 17 GB", | 20260508 10:07:38.818 | INFO | "huge" : "", | 20260508 10:07:38.818 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:38.818 | INFO | "malloc" : " 14 GB", | 20260508 10:07:38.818 | INFO | "pinned" : " 29 GB" | 20260508 10:07:38.818 | INFO | } | 20260508 10:07:38.818 | INFO | Perf report for the last 5.14418 seconds: | 20260508 10:07:38.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:38.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 10:07:38.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:07:38.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:38.818 | INFO | Uploaded 36GiB to GPU in 2.36863s (15.1987GiB/s) | 20260508 10:07:38.818 | INFO | Downloaded 34GiB to GPU in 2.44492s (13.9064GiB/s) | 20260508 10:07:43.955 | INFO | Analyzing chunk frames = [7265792,7266304) | 20260508 10:07:43.955 | INFO | MemInfo: { | 20260508 10:07:43.955 | INFO | "gpu" : " 17 GB", | 20260508 10:07:43.955 | INFO | "huge" : "", | 20260508 10:07:43.955 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:43.955 | INFO | "malloc" : " 14 GB", | 20260508 10:07:43.955 | INFO | "pinned" : " 29 GB" | 20260508 10:07:43.955 | INFO | } | 20260508 10:07:43.955 | INFO | Perf report for the last 5.13727 seconds: | 20260508 10:07:43.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:43.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.955 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 10:07:43.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.955 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:07:43.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:43.956 | INFO | Uploaded 36GiB to GPU in 2.4024s (14.985GiB/s) | 20260508 10:07:43.956 | INFO | Downloaded 34GiB to GPU in 2.46443s (13.7963GiB/s) | 20260508 10:07:43.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:07:43.956 | INFO | Total mem usage: 16.7 GiB | 20260508 10:07:43.956 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:07:43.956 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:07:43.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:07:43.956 | INFO | High Water Breakdown: | 20260508 10:07:43.956 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:07:43.956 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:07:43.956 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:07:43.956 | INFO | ------------------------------------------ | 20260508 10:07:43.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:07:43.956 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:07:43.956 | INFO | Total mem usage: 26.5 GiB | 20260508 10:07:43.956 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:07:43.956 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:07:43.956 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:07:43.956 | INFO | High Water Breakdown: | 20260508 10:07:43.956 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:07:43.956 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:07:43.956 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:07:43.956 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:07:43.956 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:07:43.956 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:07:43.956 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:07:43.956 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:07:43.956 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:07:43.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:07:43.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:07:43.956 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:07:43.956 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:07:43.956 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:07:43.956 | INFO | ------------------------------------------ | 20260508 10:07:43.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:07:43.956 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:07:43.956 | INFO | Total mem usage: 27.8 GiB | 20260508 10:07:43.956 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:07:43.956 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:07:43.956 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:07:43.956 | INFO | High Water Breakdown: | 20260508 10:07:43.956 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:07:43.956 | INFO | ------------------------------------------ | 20260508 10:07:43.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:07:43.956 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:07:43.956 | INFO | Total mem usage: 15.7 GiB | 20260508 10:07:43.956 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:07:43.956 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:07:43.956 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:07:43.956 | INFO | High Water Breakdown: | 20260508 10:07:43.956 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:07:43.956 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:07:43.956 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:07:43.956 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:07:43.956 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:07:43.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:07:43.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:07:43.958 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:07:43.958 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:07:43.958 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:07:43.958 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:07:43.958 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:07:43.958 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:07:43.958 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:07:43.958 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:07:43.958 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:07:43.958 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:07:43.958 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:07:43.958 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:07:43.958 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:07:43.958 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:07:43.958 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:07:43.958 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:07:43.958 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:07:43.958 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:07:43.958 | INFO | ------------------------------------------ | 20260508 10:07:43.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:07:43.958 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:07:43.958 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:07:43.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:07:43.958 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 6.80e-01 | 20260508 10:07:43.958 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.08e-02 / 4.25e-03 | 20260508 10:07:43.958 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.49e+00, 6.01e-01 / 2.01e-01 | 20260508 10:07:43.958 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.43e+00, 2.09e-01 / 1.49e-01 | 20260508 10:07:43.958 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.54e+01, 3.85e+00 / 1.05e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.82e+00 / 1.31e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.03e-01 / 2.52e-01 | 20260508 10:07:43.958 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.13e+00, 2.45e-01 / 1.34e-01 | 20260508 10:07:43.958 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.35e-01, 4.72e-02 / 2.39e-02 | 20260508 10:07:43.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:07:43.958 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:07:43.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:07:43.958 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 23.93%, Upload -- 2.51e+01 / 0.00e+00, 7.73e+00 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 24.71%, Download -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 31.17%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 1.94%, LockUpload -- 2.04e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 2.81%, LockDownload -- 2.95e+00 / 0.00e+00, 4.46e+00 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 15.43%, LockCompute -- 1.62e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:07:43.958 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 1.05e-02 / 0.00e+00 | 20260508 10:07:44.060 | INFO | Starting a round of Baz IO | 20260508 10:07:47.472 | INFO | Populating RTMetrics report | 20260508 10:07:47.472 | INFO | RTMetrics report populated | 20260508 10:07:47.472 | INFO | Opening tmp RTMetrics json file | 20260508 10:07:47.481 | INFO | Serializing RTMetrics to json | 20260508 10:07:47.482 | INFO | Serialization done, swapping tmp file now... | 20260508 10:07:47.482 | INFO | RTMetrics done | 20260508 10:07:47.541 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.48 seconds | 20260508 10:07:47.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:07:47.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.231433% / 0% / 5.16574e-05% | 20260508 10:07:47.541 | INFO | Baz IO overall: | 20260508 10:07:47.541 | INFO | BytesWritten : 3158875418948 | 20260508 10:07:47.541 | INFO | NumEvents : 1518772608393 | 20260508 10:07:49.092 | INFO | Analyzing chunk frames = [7266304,7266816) | 20260508 10:07:49.092 | INFO | MemInfo: { | 20260508 10:07:49.092 | INFO | "gpu" : " 17 GB", | 20260508 10:07:49.092 | INFO | "huge" : "", | 20260508 10:07:49.092 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:49.092 | INFO | "malloc" : " 14 GB", | 20260508 10:07:49.092 | INFO | "pinned" : " 29 GB" | 20260508 10:07:49.092 | INFO | } | 20260508 10:07:49.092 | INFO | Perf report for the last 5.13646 seconds: | 20260508 10:07:49.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:49.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:49.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:49.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:49.092 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:07:49.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:49.092 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:07:49.092 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 10:07:49.092 | INFO | BAZWRITER : 67.3%, 2.5, 87.9ms, 0 | 20260508 10:07:49.092 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:07:49.092 | INFO | Uploaded 36GiB to GPU in 2.60952s (13.7956GiB/s) | 20260508 10:07:49.092 | INFO | Downloaded 38GiB to GPU in 2.70955s (14.0245GiB/s) | 20260508 10:07:54.235 | INFO | Analyzing chunk frames = [7266816,7267328) | 20260508 10:07:54.235 | INFO | MemInfo: { | 20260508 10:07:54.235 | INFO | "gpu" : " 17 GB", | 20260508 10:07:54.235 | INFO | "huge" : "", | 20260508 10:07:54.235 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:54.235 | INFO | "malloc" : " 14 GB", | 20260508 10:07:54.235 | INFO | "pinned" : " 29 GB" | 20260508 10:07:54.235 | INFO | } | 20260508 10:07:54.235 | INFO | Perf report for the last 5.14344 seconds: | 20260508 10:07:54.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:54.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.235 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 10:07:54.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.235 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:07:54.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:54.236 | INFO | Uploaded 36GiB to GPU in 2.53629s (14.194GiB/s) | 20260508 10:07:54.236 | INFO | Downloaded 34GiB to GPU in 2.44629s (13.8986GiB/s) | 20260508 10:07:59.371 | INFO | Analyzing chunk frames = [7267328,7267840) | 20260508 10:07:59.371 | INFO | MemInfo: { | 20260508 10:07:59.371 | INFO | "gpu" : " 17 GB", | 20260508 10:07:59.371 | INFO | "huge" : "", | 20260508 10:07:59.371 | INFO | "huge_shared" : " 29 GB", | 20260508 10:07:59.372 | INFO | "malloc" : " 14 GB", | 20260508 10:07:59.372 | INFO | "pinned" : " 29 GB" | 20260508 10:07:59.372 | INFO | } | 20260508 10:07:59.372 | INFO | Perf report for the last 5.13641 seconds: | 20260508 10:07:59.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:07:59.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:07:59.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 10:07:59.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:07:59.372 | INFO | Uploaded 36GiB to GPU in 2.48222s (14.5032GiB/s) | 20260508 10:07:59.372 | INFO | Downloaded 34GiB to GPU in 2.43365s (13.9708GiB/s) | 20260508 10:08:04.515 | INFO | Analyzing chunk frames = [7267840,7268352) | 20260508 10:08:04.515 | INFO | MemInfo: { | 20260508 10:08:04.515 | INFO | "gpu" : " 17 GB", | 20260508 10:08:04.515 | INFO | "huge" : "", | 20260508 10:08:04.515 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:04.515 | INFO | "malloc" : " 14 GB", | 20260508 10:08:04.515 | INFO | "pinned" : " 29 GB" | 20260508 10:08:04.515 | INFO | } | 20260508 10:08:04.515 | INFO | Perf report for the last 5.14371 seconds: | 20260508 10:08:04.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:04.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.515 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 10:08:04.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.516 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:08:04.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:04.516 | INFO | Uploaded 36GiB to GPU in 2.50205s (14.3882GiB/s) | 20260508 10:08:04.516 | INFO | Downloaded 34GiB to GPU in 2.42896s (13.9978GiB/s) | 20260508 10:08:09.650 | INFO | Analyzing chunk frames = [7268352,7268864) | 20260508 10:08:09.650 | INFO | MemInfo: { | 20260508 10:08:09.650 | INFO | "gpu" : " 17 GB", | 20260508 10:08:09.650 | INFO | "huge" : "", | 20260508 10:08:09.651 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:09.651 | INFO | "malloc" : " 14 GB", | 20260508 10:08:09.651 | INFO | "pinned" : " 29 GB" | 20260508 10:08:09.651 | INFO | } | 20260508 10:08:09.651 | INFO | Perf report for the last 5.13522 seconds: | 20260508 10:08:09.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:09.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:08:09.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:08:09.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:09.651 | INFO | Uploaded 36GiB to GPU in 2.33372s (15.426GiB/s) | 20260508 10:08:09.651 | INFO | Downloaded 34GiB to GPU in 2.50028s (13.5985GiB/s) | 20260508 10:08:14.794 | INFO | Analyzing chunk frames = [7268864,7269376) | 20260508 10:08:14.794 | INFO | MemInfo: { | 20260508 10:08:14.794 | INFO | "gpu" : " 17 GB", | 20260508 10:08:14.794 | INFO | "huge" : "", | 20260508 10:08:14.794 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:14.794 | INFO | "malloc" : " 14 GB", | 20260508 10:08:14.794 | INFO | "pinned" : " 29 GB" | 20260508 10:08:14.794 | INFO | } | 20260508 10:08:14.795 | INFO | Perf report for the last 5.14385 seconds: | 20260508 10:08:14.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:14.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:08:14.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:08:14.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:14.795 | INFO | Uploaded 36GiB to GPU in 2.37761s (15.1412GiB/s) | 20260508 10:08:14.795 | INFO | Downloaded 34GiB to GPU in 2.4091s (14.1131GiB/s) | 20260508 10:08:19.933 | INFO | Analyzing chunk frames = [7269376,7269888) | 20260508 10:08:19.933 | INFO | MemInfo: { | 20260508 10:08:19.933 | INFO | "gpu" : " 17 GB", | 20260508 10:08:19.933 | INFO | "huge" : "", | 20260508 10:08:19.933 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:19.933 | INFO | "malloc" : " 14 GB", | 20260508 10:08:19.933 | INFO | "pinned" : " 29 GB" | 20260508 10:08:19.933 | INFO | } | 20260508 10:08:19.933 | INFO | Perf report for the last 5.13877 seconds: | 20260508 10:08:19.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:19.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.933 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:08:19.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.933 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:08:19.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:19.934 | INFO | Uploaded 36GiB to GPU in 2.4742s (14.5501GiB/s) | 20260508 10:08:19.934 | INFO | Downloaded 34GiB to GPU in 2.43152s (13.983GiB/s) | 20260508 10:08:25.074 | INFO | Analyzing chunk frames = [7269888,7270400) | 20260508 10:08:25.074 | INFO | MemInfo: { | 20260508 10:08:25.074 | INFO | "gpu" : " 17 GB", | 20260508 10:08:25.074 | INFO | "huge" : "", | 20260508 10:08:25.074 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:25.074 | INFO | "malloc" : " 14 GB", | 20260508 10:08:25.074 | INFO | "pinned" : " 29 GB" | 20260508 10:08:25.074 | INFO | } | 20260508 10:08:25.075 | INFO | Perf report for the last 5.14124 seconds: | 20260508 10:08:25.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:25.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:08:25.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:08:25.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:25.075 | INFO | Uploaded 36GiB to GPU in 2.4444s (14.7275GiB/s) | 20260508 10:08:25.075 | INFO | Downloaded 34GiB to GPU in 2.42358s (14.0289GiB/s) | 20260508 10:08:25.172 | INFO | Starting a round of Baz IO | 20260508 10:08:28.582 | INFO | Populating RTMetrics report | 20260508 10:08:28.583 | INFO | RTMetrics report populated | 20260508 10:08:28.583 | INFO | Opening tmp RTMetrics json file | 20260508 10:08:28.594 | INFO | Serializing RTMetrics to json | 20260508 10:08:28.595 | INFO | Serialization done, swapping tmp file now... | 20260508 10:08:28.596 | INFO | RTMetrics done | 20260508 10:08:28.652 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.479 seconds | 20260508 10:08:28.652 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:08:28.652 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.231171% / 3.97364e-06% / 6.35783e-05% | 20260508 10:08:28.652 | INFO | Baz IO overall: | 20260508 10:08:28.652 | INFO | BytesWritten : 3160449481506 | 20260508 10:08:28.652 | INFO | NumEvents : 1519515453662 | 20260508 10:08:30.206 | INFO | Analyzing chunk frames = [7270400,7270912) | 20260508 10:08:30.206 | INFO | MemInfo: { | 20260508 10:08:30.206 | INFO | "gpu" : " 17 GB", | 20260508 10:08:30.206 | INFO | "huge" : "", | 20260508 10:08:30.206 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:30.206 | INFO | "malloc" : " 14 GB", | 20260508 10:08:30.206 | INFO | "pinned" : " 29 GB" | 20260508 10:08:30.206 | INFO | } | 20260508 10:08:30.206 | INFO | Perf report for the last 5.13169 seconds: | 20260508 10:08:30.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:30.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:30.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:30.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:30.206 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:08:30.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:30.207 | INFO | RT_METRICS : 38.7%, 1.0, 20.2ms, 0 | 20260508 10:08:30.207 | INFO | PRE_HQ : 65.4%, 1.6, 54.2ms, 0 | 20260508 10:08:30.207 | INFO | BAZWRITER : 67.1%, 2.5, 86.3ms, 0 | 20260508 10:08:30.207 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:08:30.207 | INFO | Uploaded 36GiB to GPU in 2.61597s (13.7616GiB/s) | 20260508 10:08:30.207 | INFO | Downloaded 38GiB to GPU in 2.70248s (14.0612GiB/s) | 20260508 10:08:35.348 | INFO | Analyzing chunk frames = [7270912,7271424) | 20260508 10:08:35.348 | INFO | MemInfo: { | 20260508 10:08:35.348 | INFO | "gpu" : " 17 GB", | 20260508 10:08:35.348 | INFO | "huge" : "", | 20260508 10:08:35.348 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:35.348 | INFO | "malloc" : " 14 GB", | 20260508 10:08:35.348 | INFO | "pinned" : " 29 GB" | 20260508 10:08:35.348 | INFO | } | 20260508 10:08:35.348 | INFO | Perf report for the last 5.14194 seconds: | 20260508 10:08:35.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:35.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:08:35.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:08:35.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:35.348 | INFO | Uploaded 36GiB to GPU in 2.54471s (14.147GiB/s) | 20260508 10:08:35.349 | INFO | Downloaded 34GiB to GPU in 2.45579s (13.8448GiB/s) | 20260508 10:08:40.489 | INFO | Analyzing chunk frames = [7271424,7271936) | 20260508 10:08:40.489 | INFO | MemInfo: { | 20260508 10:08:40.489 | INFO | "gpu" : " 17 GB", | 20260508 10:08:40.489 | INFO | "huge" : "", | 20260508 10:08:40.489 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:40.489 | INFO | "malloc" : " 14 GB", | 20260508 10:08:40.489 | INFO | "pinned" : " 29 GB" | 20260508 10:08:40.489 | INFO | } | 20260508 10:08:40.489 | INFO | Perf report for the last 5.14091 seconds: | 20260508 10:08:40.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:40.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:08:40.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:08:40.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:40.489 | INFO | Uploaded 36GiB to GPU in 2.49529s (14.4272GiB/s) | 20260508 10:08:40.489 | INFO | Downloaded 34GiB to GPU in 2.4251s (14.02GiB/s) | 20260508 10:08:45.628 | INFO | Analyzing chunk frames = [7271936,7272448) | 20260508 10:08:45.628 | INFO | MemInfo: { | 20260508 10:08:45.628 | INFO | "gpu" : " 17 GB", | 20260508 10:08:45.628 | INFO | "huge" : "", | 20260508 10:08:45.628 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:45.628 | INFO | "malloc" : " 14 GB", | 20260508 10:08:45.628 | INFO | "pinned" : " 29 GB" | 20260508 10:08:45.628 | INFO | } | 20260508 10:08:45.628 | INFO | Perf report for the last 5.13905 seconds: | 20260508 10:08:45.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:45.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.628 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:08:45.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.628 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:08:45.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:45.629 | INFO | Uploaded 36GiB to GPU in 2.4858s (14.4822GiB/s) | 20260508 10:08:45.629 | INFO | Downloaded 34GiB to GPU in 2.39416s (14.2012GiB/s) | 20260508 10:08:50.766 | INFO | Analyzing chunk frames = [7272448,7272960) | 20260508 10:08:50.766 | INFO | MemInfo: { | 20260508 10:08:50.766 | INFO | "gpu" : " 17 GB", | 20260508 10:08:50.766 | INFO | "huge" : "", | 20260508 10:08:50.766 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:50.766 | INFO | "malloc" : " 14 GB", | 20260508 10:08:50.766 | INFO | "pinned" : " 29 GB" | 20260508 10:08:50.766 | INFO | } | 20260508 10:08:50.766 | INFO | Perf report for the last 5.13785 seconds: | 20260508 10:08:50.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:50.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 10:08:50.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:08:50.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:50.766 | INFO | Uploaded 36GiB to GPU in 2.50551s (14.3684GiB/s) | 20260508 10:08:50.767 | INFO | Downloaded 34GiB to GPU in 2.53146s (13.431GiB/s) | 20260508 10:08:55.904 | INFO | Analyzing chunk frames = [7272960,7273472) | 20260508 10:08:55.904 | INFO | MemInfo: { | 20260508 10:08:55.904 | INFO | "gpu" : " 17 GB", | 20260508 10:08:55.904 | INFO | "huge" : "", | 20260508 10:08:55.904 | INFO | "huge_shared" : " 29 GB", | 20260508 10:08:55.904 | INFO | "malloc" : " 14 GB", | 20260508 10:08:55.904 | INFO | "pinned" : " 29 GB" | 20260508 10:08:55.904 | INFO | } | 20260508 10:08:55.904 | INFO | Perf report for the last 5.13838 seconds: | 20260508 10:08:55.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:08:55.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.905 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:08:55.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.905 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:08:55.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:08:55.905 | INFO | Uploaded 36GiB to GPU in 2.39349s (15.0408GiB/s) | 20260508 10:08:55.905 | INFO | Downloaded 34GiB to GPU in 2.42461s (14.0229GiB/s) | 20260508 10:09:01.044 | INFO | Analyzing chunk frames = [7273472,7273984) | 20260508 10:09:01.044 | INFO | MemInfo: { | 20260508 10:09:01.044 | INFO | "gpu" : " 17 GB", | 20260508 10:09:01.044 | INFO | "huge" : "", | 20260508 10:09:01.044 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:01.044 | INFO | "malloc" : " 14 GB", | 20260508 10:09:01.044 | INFO | "pinned" : " 29 GB" | 20260508 10:09:01.044 | INFO | } | 20260508 10:09:01.044 | INFO | Perf report for the last 5.13952 seconds: | 20260508 10:09:01.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:01.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 10:09:01.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:09:01.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:01.044 | INFO | Uploaded 36GiB to GPU in 2.46137s (14.626GiB/s) | 20260508 10:09:01.044 | INFO | Downloaded 34GiB to GPU in 2.43304s (13.9743GiB/s) | 20260508 10:09:06.184 | INFO | Analyzing chunk frames = [7273984,7274496) | 20260508 10:09:06.184 | INFO | MemInfo: { | 20260508 10:09:06.184 | INFO | "gpu" : " 17 GB", | 20260508 10:09:06.184 | INFO | "huge" : "", | 20260508 10:09:06.184 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:06.184 | INFO | "malloc" : " 14 GB", | 20260508 10:09:06.184 | INFO | "pinned" : " 29 GB" | 20260508 10:09:06.184 | INFO | } | 20260508 10:09:06.184 | INFO | Perf report for the last 5.14043 seconds: | 20260508 10:09:06.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:06.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.185 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 10:09:06.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.185 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 10:09:06.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:06.185 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:09:06.185 | INFO | Total mem usage: 16.7 GiB | 20260508 10:09:06.185 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:09:06.185 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:09:06.185 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:09:06.185 | INFO | High Water Breakdown: | 20260508 10:09:06.185 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:09:06.185 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:09:06.185 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:09:06.185 | INFO | ------------------------------------------ | 20260508 10:09:06.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:09:06.185 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:09:06.185 | INFO | Total mem usage: 26.5 GiB | 20260508 10:09:06.185 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:09:06.185 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:09:06.185 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:09:06.185 | INFO | High Water Breakdown: | 20260508 10:09:06.185 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:09:06.185 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:09:06.185 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:09:06.185 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:09:06.185 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:09:06.185 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:09:06.185 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:09:06.185 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:09:06.185 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:09:06.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:09:06.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:09:06.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:09:06.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:09:06.185 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:09:06.185 | INFO | ------------------------------------------ | 20260508 10:09:06.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:09:06.185 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:09:06.185 | INFO | Total mem usage: 27.8 GiB | 20260508 10:09:06.185 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:09:06.185 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:09:06.185 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:09:06.185 | INFO | High Water Breakdown: | 20260508 10:09:06.185 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:09:06.185 | INFO | ------------------------------------------ | 20260508 10:09:06.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:09:06.185 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:09:06.185 | INFO | Total mem usage: 15.7 GiB | 20260508 10:09:06.185 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:09:06.185 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:09:06.185 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:09:06.185 | INFO | High Water Breakdown: | 20260508 10:09:06.185 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:09:06.185 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:09:06.185 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:09:06.185 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:09:06.185 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:09:06.185 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:09:06.185 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:09:06.185 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:09:06.185 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:09:06.185 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:09:06.185 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:09:06.185 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:09:06.185 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:09:06.185 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:09:06.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:09:06.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:09:06.185 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:09:06.185 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:09:06.185 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:09:06.185 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:09:06.185 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:09:06.185 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:09:06.185 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:09:06.185 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:09:06.185 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:09:06.185 | INFO | ------------------------------------------ | 20260508 10:09:06.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:09:06.185 | INFO | Uploaded 36GiB to GPU in 2.45075s (14.6894GiB/s) | 20260508 10:09:06.185 | INFO | Downloaded 34GiB to GPU in 2.4228s (14.0333GiB/s) | 20260508 10:09:06.185 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:09:06.185 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:09:06.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:09:06.185 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 7.25e-01 | 20260508 10:09:06.185 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.04e-01, 1.09e-02 / 4.11e-03 | 20260508 10:09:06.185 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.40e+00, 5.93e-01 / 2.22e-01 | 20260508 10:09:06.185 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.11e-01 / 1.19e-01 | 20260508 10:09:06.185 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.50e+01, 3.77e+00 / 1.09e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.83e+00 / 1.18e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.45e+00, 4.08e-01 / 2.58e-01 | 20260508 10:09:06.185 | INFO | PerfReport: 3.02%, Metrics -- 9.88e-01 / 1.11e+00, 2.47e-01 / 1.11e-01 | 20260508 10:09:06.185 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.32e-01, 4.83e-02 / 2.77e-02 | 20260508 10:09:06.185 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:09:06.185 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:09:06.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:09:06.185 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 24.52%, Upload -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 24.43%, Download -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 31.02%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 2.33%, LockUpload -- 2.46e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 2.38%, LockDownload -- 2.51e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 15.31%, LockCompute -- 1.61e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260508 10:09:06.185 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260508 10:09:06.282 | INFO | Starting a round of Baz IO | 20260508 10:09:09.702 | INFO | Populating RTMetrics report | 20260508 10:09:09.702 | INFO | RTMetrics report populated | 20260508 10:09:09.702 | INFO | Opening tmp RTMetrics json file | 20260508 10:09:09.703 | INFO | Serializing RTMetrics to json | 20260508 10:09:09.704 | INFO | Serialization done, swapping tmp file now... | 20260508 10:09:09.704 | INFO | RTMetrics done | 20260508 10:09:09.770 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.487 seconds | 20260508 10:09:09.770 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:09:09.770 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230877% / 3.97364e-06% / 4.76837e-05% | 20260508 10:09:09.770 | INFO | Baz IO overall: | 20260508 10:09:09.770 | INFO | BytesWritten : 3162023428761 | 20260508 10:09:09.770 | INFO | NumEvents : 1520258200667 | 20260508 10:09:11.330 | INFO | Analyzing chunk frames = [7274496,7275008) | 20260508 10:09:11.330 | INFO | MemInfo: { | 20260508 10:09:11.330 | INFO | "gpu" : " 17 GB", | 20260508 10:09:11.330 | INFO | "huge" : "", | 20260508 10:09:11.330 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:11.330 | INFO | "malloc" : " 14 GB", | 20260508 10:09:11.330 | INFO | "pinned" : " 29 GB" | 20260508 10:09:11.330 | INFO | } | 20260508 10:09:11.330 | INFO | Perf report for the last 5.14563 seconds: | 20260508 10:09:11.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:11.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:11.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:11.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:11.330 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 10:09:11.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:11.330 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:09:11.330 | INFO | PRE_HQ : 65.6%, 1.6, 54.4ms, 0 | 20260508 10:09:11.330 | INFO | BAZWRITER : 67.0%, 2.5, 86.9ms, 0 | 20260508 10:09:11.330 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:09:11.331 | INFO | Uploaded 36GiB to GPU in 2.56651s (14.0268GiB/s) | 20260508 10:09:11.331 | INFO | Downloaded 38GiB to GPU in 2.72094s (13.9657GiB/s) | 20260508 10:09:16.463 | INFO | Analyzing chunk frames = [7275008,7275520) | 20260508 10:09:16.463 | INFO | MemInfo: { | 20260508 10:09:16.463 | INFO | "gpu" : " 17 GB", | 20260508 10:09:16.463 | INFO | "huge" : "", | 20260508 10:09:16.464 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:16.464 | INFO | "malloc" : " 14 GB", | 20260508 10:09:16.464 | INFO | "pinned" : " 29 GB" | 20260508 10:09:16.464 | INFO | } | 20260508 10:09:16.464 | INFO | Perf report for the last 5.13365 seconds: | 20260508 10:09:16.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:16.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:09:16.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:09:16.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:16.464 | INFO | Uploaded 36GiB to GPU in 2.44058s (14.7506GiB/s) | 20260508 10:09:16.464 | INFO | Downloaded 34GiB to GPU in 2.47823s (13.7195GiB/s) | 20260508 10:09:21.600 | INFO | Analyzing chunk frames = [7275520,7276032) | 20260508 10:09:21.600 | INFO | MemInfo: { | 20260508 10:09:21.600 | INFO | "gpu" : " 17 GB", | 20260508 10:09:21.600 | INFO | "huge" : "", | 20260508 10:09:21.600 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:21.600 | INFO | "malloc" : " 14 GB", | 20260508 10:09:21.600 | INFO | "pinned" : " 29 GB" | 20260508 10:09:21.600 | INFO | } | 20260508 10:09:21.600 | INFO | Perf report for the last 5.13647 seconds: | 20260508 10:09:21.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:21.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.600 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:09:21.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.600 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:09:21.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:21.601 | INFO | Uploaded 36GiB to GPU in 2.45965s (14.6362GiB/s) | 20260508 10:09:21.601 | INFO | Downloaded 34GiB to GPU in 2.42718s (14.008GiB/s) | 20260508 10:09:26.746 | INFO | Analyzing chunk frames = [7276032,7276544) | 20260508 10:09:26.746 | INFO | MemInfo: { | 20260508 10:09:26.746 | INFO | "gpu" : " 17 GB", | 20260508 10:09:26.746 | INFO | "huge" : "", | 20260508 10:09:26.746 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:26.746 | INFO | "malloc" : " 14 GB", | 20260508 10:09:26.746 | INFO | "pinned" : " 29 GB" | 20260508 10:09:26.746 | INFO | } | 20260508 10:09:26.746 | INFO | Perf report for the last 5.14584 seconds: | 20260508 10:09:26.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:26.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 10:09:26.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:09:26.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:26.746 | INFO | Uploaded 36GiB to GPU in 2.45524s (14.6625GiB/s) | 20260508 10:09:26.747 | INFO | Downloaded 34GiB to GPU in 2.40876s (14.1151GiB/s) | 20260508 10:09:31.884 | INFO | Analyzing chunk frames = [7276544,7277056) | 20260508 10:09:31.884 | INFO | MemInfo: { | 20260508 10:09:31.884 | INFO | "gpu" : " 17 GB", | 20260508 10:09:31.884 | INFO | "huge" : "", | 20260508 10:09:31.884 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:31.884 | INFO | "malloc" : " 14 GB", | 20260508 10:09:31.884 | INFO | "pinned" : " 29 GB" | 20260508 10:09:31.884 | INFO | } | 20260508 10:09:31.884 | INFO | Perf report for the last 5.13804 seconds: | 20260508 10:09:31.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:31.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 10:09:31.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:09:31.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:31.884 | INFO | Uploaded 36GiB to GPU in 2.36835s (15.2005GiB/s) | 20260508 10:09:31.884 | INFO | Downloaded 34GiB to GPU in 2.52882s (13.445GiB/s) | 20260508 10:09:37.019 | INFO | Analyzing chunk frames = [7277056,7277568) | 20260508 10:09:37.019 | INFO | MemInfo: { | 20260508 10:09:37.019 | INFO | "gpu" : " 17 GB", | 20260508 10:09:37.019 | INFO | "huge" : "", | 20260508 10:09:37.019 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:37.019 | INFO | "malloc" : " 14 GB", | 20260508 10:09:37.019 | INFO | "pinned" : " 29 GB" | 20260508 10:09:37.019 | INFO | } | 20260508 10:09:37.019 | INFO | Perf report for the last 5.1351 seconds: | 20260508 10:09:37.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:37.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.019 | INFO | ANALYSIS : 65.1%, 3.0, 101.2ms, 0 | 20260508 10:09:37.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.020 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:09:37.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:37.020 | INFO | Uploaded 36GiB to GPU in 2.32776s (15.4655GiB/s) | 20260508 10:09:37.020 | INFO | Downloaded 34GiB to GPU in 2.45245s (13.8637GiB/s) | 20260508 10:09:42.157 | INFO | Analyzing chunk frames = [7277568,7278080) | 20260508 10:09:42.157 | INFO | MemInfo: { | 20260508 10:09:42.157 | INFO | "gpu" : " 17 GB", | 20260508 10:09:42.157 | INFO | "huge" : "", | 20260508 10:09:42.158 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:42.158 | INFO | "malloc" : " 14 GB", | 20260508 10:09:42.158 | INFO | "pinned" : " 29 GB" | 20260508 10:09:42.158 | INFO | } | 20260508 10:09:42.158 | INFO | Perf report for the last 5.13841 seconds: | 20260508 10:09:42.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:42.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 10:09:42.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:09:42.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:42.158 | INFO | Uploaded 36GiB to GPU in 2.38256s (15.1098GiB/s) | 20260508 10:09:42.158 | INFO | Downloaded 34GiB to GPU in 2.44903s (13.8831GiB/s) | 20260508 10:09:47.300 | INFO | Analyzing chunk frames = [7278080,7278592) | 20260508 10:09:47.300 | INFO | MemInfo: { | 20260508 10:09:47.300 | INFO | "gpu" : " 17 GB", | 20260508 10:09:47.301 | INFO | "huge" : "", | 20260508 10:09:47.301 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:47.301 | INFO | "malloc" : " 14 GB", | 20260508 10:09:47.301 | INFO | "pinned" : " 29 GB" | 20260508 10:09:47.301 | INFO | } | 20260508 10:09:47.301 | INFO | Perf report for the last 5.14308 seconds: | 20260508 10:09:47.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:47.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 10:09:47.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:09:47.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:47.301 | INFO | Uploaded 36GiB to GPU in 2.3828s (15.1083GiB/s) | 20260508 10:09:47.301 | INFO | Downloaded 34GiB to GPU in 2.45192s (13.8667GiB/s) | 20260508 10:09:47.399 | INFO | Starting a round of Baz IO | 20260508 10:09:50.815 | INFO | Populating RTMetrics report | 20260508 10:09:50.815 | INFO | RTMetrics report populated | 20260508 10:09:50.815 | INFO | Opening tmp RTMetrics json file | 20260508 10:09:50.823 | INFO | Serializing RTMetrics to json | 20260508 10:09:50.824 | INFO | Serialization done, swapping tmp file now... | 20260508 10:09:50.824 | INFO | RTMetrics done | 20260508 10:09:50.882 | INFO | Finished a round of Baz IO: Wrote 1.467 GiB in 3.483 seconds | 20260508 10:09:50.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:09:50.882 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.231349% / 7.94729e-06% / 5.5631e-05% | 20260508 10:09:50.882 | INFO | Baz IO overall: | 20260508 10:09:50.882 | INFO | BytesWritten : 3163598951479 | 20260508 10:09:50.882 | INFO | NumEvents : 1521002434385 | 20260508 10:09:52.442 | INFO | Analyzing chunk frames = [7278592,7279104) | 20260508 10:09:52.442 | INFO | MemInfo: { | 20260508 10:09:52.442 | INFO | "gpu" : " 17 GB", | 20260508 10:09:52.442 | INFO | "huge" : "", | 20260508 10:09:52.442 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:52.442 | INFO | "malloc" : " 14 GB", | 20260508 10:09:52.442 | INFO | "pinned" : " 29 GB" | 20260508 10:09:52.442 | INFO | } | 20260508 10:09:52.442 | INFO | Perf report for the last 5.14107 seconds: | 20260508 10:09:52.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:52.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:52.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:52.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:52.442 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:09:52.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:52.442 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:09:52.442 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 10:09:52.442 | INFO | BAZWRITER : 67.1%, 2.5, 86.8ms, 0 | 20260508 10:09:52.442 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:09:52.442 | INFO | Uploaded 36GiB to GPU in 2.58154s (13.9452GiB/s) | 20260508 10:09:52.442 | INFO | Downloaded 38GiB to GPU in 2.71582s (13.9921GiB/s) | 20260508 10:09:57.578 | INFO | Analyzing chunk frames = [7279104,7279616) | 20260508 10:09:57.578 | INFO | MemInfo: { | 20260508 10:09:57.578 | INFO | "gpu" : " 17 GB", | 20260508 10:09:57.578 | INFO | "huge" : "", | 20260508 10:09:57.578 | INFO | "huge_shared" : " 29 GB", | 20260508 10:09:57.578 | INFO | "malloc" : " 14 GB", | 20260508 10:09:57.578 | INFO | "pinned" : " 29 GB" | 20260508 10:09:57.578 | INFO | } | 20260508 10:09:57.578 | INFO | Perf report for the last 5.13599 seconds: | 20260508 10:09:57.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:09:57.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:09:57.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:09:57.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:09:57.578 | INFO | Uploaded 36GiB to GPU in 2.42043s (14.8734GiB/s) | 20260508 10:09:57.578 | INFO | Downloaded 34GiB to GPU in 2.48455s (13.6846GiB/s) | 20260508 10:10:02.721 | INFO | Analyzing chunk frames = [7279616,7280128) | 20260508 10:10:02.721 | INFO | MemInfo: { | 20260508 10:10:02.721 | INFO | "gpu" : " 17 GB", | 20260508 10:10:02.721 | INFO | "huge" : "", | 20260508 10:10:02.722 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:02.722 | INFO | "malloc" : " 14 GB", | 20260508 10:10:02.722 | INFO | "pinned" : " 29 GB" | 20260508 10:10:02.722 | INFO | } | 20260508 10:10:02.722 | INFO | Perf report for the last 5.14374 seconds: | 20260508 10:10:02.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:02.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 10:10:02.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:10:02.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:02.722 | INFO | Uploaded 36GiB to GPU in 2.4488s (14.7011GiB/s) | 20260508 10:10:02.722 | INFO | Downloaded 34GiB to GPU in 2.44666s (13.8965GiB/s) | 20260508 10:10:07.853 | INFO | Analyzing chunk frames = [7280128,7280640) | 20260508 10:10:07.853 | INFO | MemInfo: { | 20260508 10:10:07.853 | INFO | "gpu" : " 17 GB", | 20260508 10:10:07.853 | INFO | "huge" : "", | 20260508 10:10:07.853 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:07.853 | INFO | "malloc" : " 14 GB", | 20260508 10:10:07.853 | INFO | "pinned" : " 29 GB" | 20260508 10:10:07.853 | INFO | } | 20260508 10:10:07.853 | INFO | Perf report for the last 5.13148 seconds: | 20260508 10:10:07.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:07.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.853 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 10:10:07.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.853 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:10:07.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:07.854 | INFO | Uploaded 36GiB to GPU in 2.43922s (14.7588GiB/s) | 20260508 10:10:07.854 | INFO | Downloaded 34GiB to GPU in 2.4602s (13.82GiB/s) | 20260508 10:10:12.998 | INFO | Analyzing chunk frames = [7280640,7281152) | 20260508 10:10:12.998 | INFO | MemInfo: { | 20260508 10:10:12.998 | INFO | "gpu" : " 17 GB", | 20260508 10:10:12.998 | INFO | "huge" : "", | 20260508 10:10:12.998 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:12.998 | INFO | "malloc" : " 14 GB", | 20260508 10:10:12.998 | INFO | "pinned" : " 29 GB" | 20260508 10:10:12.998 | INFO | } | 20260508 10:10:12.998 | INFO | Perf report for the last 5.14478 seconds: | 20260508 10:10:12.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:12.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 10:10:12.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | PRE_HQ : 26.2%, 1.0, 13.7ms, 0 | 20260508 10:10:12.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:12.998 | INFO | Uploaded 36GiB to GPU in 2.32409s (15.4899GiB/s) | 20260508 10:10:12.998 | INFO | Downloaded 34GiB to GPU in 2.51724s (13.5069GiB/s) | 20260508 10:10:18.135 | INFO | Analyzing chunk frames = [7281152,7281664) | 20260508 10:10:18.135 | INFO | MemInfo: { | 20260508 10:10:18.135 | INFO | "gpu" : " 17 GB", | 20260508 10:10:18.135 | INFO | "huge" : "", | 20260508 10:10:18.135 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:18.135 | INFO | "malloc" : " 14 GB", | 20260508 10:10:18.135 | INFO | "pinned" : " 29 GB" | 20260508 10:10:18.135 | INFO | } | 20260508 10:10:18.135 | INFO | Perf report for the last 5.13703 seconds: | 20260508 10:10:18.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:18.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:10:18.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:10:18.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:18.135 | INFO | Uploaded 36GiB to GPU in 2.35859s (15.2633GiB/s) | 20260508 10:10:18.136 | INFO | Downloaded 34GiB to GPU in 2.44545s (13.9033GiB/s) | 20260508 10:10:23.275 | INFO | Analyzing chunk frames = [7281664,7282176) | 20260508 10:10:23.275 | INFO | MemInfo: { | 20260508 10:10:23.275 | INFO | "gpu" : " 17 GB", | 20260508 10:10:23.275 | INFO | "huge" : "", | 20260508 10:10:23.275 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:23.275 | INFO | "malloc" : " 14 GB", | 20260508 10:10:23.275 | INFO | "pinned" : " 29 GB" | 20260508 10:10:23.275 | INFO | } | 20260508 10:10:23.275 | INFO | Perf report for the last 5.14018 seconds: | 20260508 10:10:23.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:23.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:10:23.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:10:23.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:23.275 | INFO | Uploaded 36GiB to GPU in 2.40204s (14.9873GiB/s) | 20260508 10:10:23.275 | INFO | Downloaded 34GiB to GPU in 2.45414s (13.8542GiB/s) | 20260508 10:10:28.413 | INFO | Analyzing chunk frames = [7282176,7282688) | 20260508 10:10:28.413 | INFO | MemInfo: { | 20260508 10:10:28.413 | INFO | "gpu" : " 17 GB", | 20260508 10:10:28.413 | INFO | "huge" : "", | 20260508 10:10:28.413 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:28.413 | INFO | "malloc" : " 14 GB", | 20260508 10:10:28.413 | INFO | "pinned" : " 29 GB" | 20260508 10:10:28.413 | INFO | } | 20260508 10:10:28.413 | INFO | Perf report for the last 5.13773 seconds: | 20260508 10:10:28.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:28.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:10:28.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:10:28.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:28.413 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:10:28.413 | INFO | Total mem usage: 16.7 GiB | 20260508 10:10:28.413 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:10:28.413 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:10:28.413 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:10:28.413 | INFO | High Water Breakdown: | 20260508 10:10:28.413 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:10:28.413 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:10:28.413 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:10:28.413 | INFO | ------------------------------------------ | 20260508 10:10:28.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:10:28.413 | INFO | Uploaded 36GiB to GPU in 2.40237s (14.9852GiB/s) | 20260508 10:10:28.413 | INFO | Downloaded 34GiB to GPU in 2.45129s (13.8703GiB/s) | 20260508 10:10:28.413 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:10:28.414 | INFO | Total mem usage: 26.5 GiB | 20260508 10:10:28.414 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:10:28.414 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:10:28.414 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:10:28.414 | INFO | High Water Breakdown: | 20260508 10:10:28.414 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:10:28.414 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:10:28.414 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:10:28.414 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:10:28.414 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:10:28.414 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:10:28.414 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:10:28.414 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:10:28.414 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:10:28.414 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:10:28.414 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:10:28.414 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:10:28.414 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:10:28.414 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:10:28.414 | INFO | ------------------------------------------ | 20260508 10:10:28.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:10:28.414 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:10:28.414 | INFO | Total mem usage: 27.8 GiB | 20260508 10:10:28.414 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:10:28.414 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:10:28.414 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:10:28.414 | INFO | High Water Breakdown: | 20260508 10:10:28.414 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:10:28.414 | INFO | ------------------------------------------ | 20260508 10:10:28.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:10:28.414 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:10:28.414 | INFO | Total mem usage: 15.7 GiB | 20260508 10:10:28.414 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:10:28.414 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:10:28.414 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:10:28.414 | INFO | High Water Breakdown: | 20260508 10:10:28.414 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:10:28.414 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:10:28.414 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:10:28.414 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:10:28.414 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:10:28.414 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:10:28.414 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:10:28.414 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:10:28.414 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:10:28.414 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:10:28.414 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:10:28.414 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:10:28.414 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:10:28.414 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:10:28.414 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:10:28.414 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:10:28.414 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:10:28.414 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:10:28.414 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:10:28.414 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:10:28.414 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:10:28.414 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:10:28.414 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:10:28.414 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:10:28.414 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:10:28.414 | INFO | ------------------------------------------ | 20260508 10:10:28.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:10:28.414 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:10:28.414 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:10:28.414 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:10:28.414 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.42e+01 / 6.56e-01 | 20260508 10:10:28.414 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.11e-02 / 6.72e-03 | 20260508 10:10:28.414 | INFO | PerfReport: 12.21%, Baseline -- 4.00e+00 / 4.50e+00, 6.03e-01 / 2.00e-01 | 20260508 10:10:28.414 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.49e+00, 2.13e-01 / 9.79e-02 | 20260508 10:10:28.414 | INFO | PerfReport: 16.00%, DME -- 4.20e+01 / 4.53e+01, 3.75e+00 / 8.32e-01 | 20260508 10:10:28.414 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.83e+00 / 9.95e-01 | 20260508 10:10:28.414 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.58e+00, 4.08e-01 / 2.51e-01 | 20260508 10:10:28.414 | INFO | PerfReport: 3.03%, Metrics -- 9.93e-01 / 1.18e+00, 2.48e-01 / 1.30e-01 | 20260508 10:10:28.414 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.40e-01, 4.88e-02 / 2.98e-02 | 20260508 10:10:28.415 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:10:28.415 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:10:28.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:10:28.415 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 24.01%, Upload -- 2.52e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 24.71%, Download -- 2.60e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 31.18%, Compute -- 3.28e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 2.02%, LockUpload -- 2.12e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 2.73%, LockDownload -- 2.87e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 15.35%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:10:28.415 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 10:10:28.510 | INFO | Starting a round of Baz IO | 20260508 10:10:31.910 | INFO | Populating RTMetrics report | 20260508 10:10:31.910 | INFO | RTMetrics report populated | 20260508 10:10:31.910 | INFO | Opening tmp RTMetrics json file | 20260508 10:10:31.921 | INFO | Serializing RTMetrics to json | 20260508 10:10:31.922 | INFO | Serialization done, swapping tmp file now... | 20260508 10:10:31.922 | INFO | RTMetrics done | 20260508 10:10:31.980 | INFO | Finished a round of Baz IO: Wrote 1.465 GiB in 3.47 seconds | 20260508 10:10:31.980 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:10:31.980 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230531% / 0% / 7.54992e-05% | 20260508 10:10:31.980 | INFO | Baz IO overall: | 20260508 10:10:31.980 | INFO | BytesWritten : 3165172361877 | 20260508 10:10:31.980 | INFO | NumEvents : 1521744991415 | 20260508 10:10:33.556 | INFO | Analyzing chunk frames = [7282688,7283200) | 20260508 10:10:33.556 | INFO | MemInfo: { | 20260508 10:10:33.556 | INFO | "gpu" : " 17 GB", | 20260508 10:10:33.556 | INFO | "huge" : "", | 20260508 10:10:33.556 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:33.556 | INFO | "malloc" : " 14 GB", | 20260508 10:10:33.556 | INFO | "pinned" : " 29 GB" | 20260508 10:10:33.556 | INFO | } | 20260508 10:10:33.556 | INFO | Perf report for the last 5.14287 seconds: | 20260508 10:10:33.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:33.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:33.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:33.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:33.556 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 10:10:33.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:33.556 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:10:33.556 | INFO | PRE_HQ : 65.1%, 1.6, 54.3ms, 0 | 20260508 10:10:33.556 | INFO | BAZWRITER : 66.8%, 2.5, 86.8ms, 0 | 20260508 10:10:33.556 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:10:33.556 | INFO | Uploaded 36GiB to GPU in 2.5821s (13.9421GiB/s) | 20260508 10:10:33.556 | INFO | Downloaded 38GiB to GPU in 2.71409s (14.001GiB/s) | 20260508 10:10:38.695 | INFO | Analyzing chunk frames = [7283200,7283712) | 20260508 10:10:38.695 | INFO | MemInfo: { | 20260508 10:10:38.695 | INFO | "gpu" : " 17 GB", | 20260508 10:10:38.695 | INFO | "huge" : "", | 20260508 10:10:38.695 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:38.695 | INFO | "malloc" : " 14 GB", | 20260508 10:10:38.695 | INFO | "pinned" : " 29 GB" | 20260508 10:10:38.695 | INFO | } | 20260508 10:10:38.695 | INFO | Perf report for the last 5.13906 seconds: | 20260508 10:10:38.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:38.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:10:38.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:10:38.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:38.695 | INFO | Uploaded 36GiB to GPU in 2.48536s (14.4848GiB/s) | 20260508 10:10:38.695 | INFO | Downloaded 34GiB to GPU in 2.47279s (13.7497GiB/s) | 20260508 10:10:43.830 | INFO | Analyzing chunk frames = [7283712,7284224) | 20260508 10:10:43.830 | INFO | MemInfo: { | 20260508 10:10:43.830 | INFO | "gpu" : " 17 GB", | 20260508 10:10:43.830 | INFO | "huge" : "", | 20260508 10:10:43.830 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:43.830 | INFO | "malloc" : " 14 GB", | 20260508 10:10:43.830 | INFO | "pinned" : " 29 GB" | 20260508 10:10:43.830 | INFO | } | 20260508 10:10:43.830 | INFO | Perf report for the last 5.13557 seconds: | 20260508 10:10:43.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:43.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.830 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:10:43.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.830 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:10:43.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:43.831 | INFO | Uploaded 36GiB to GPU in 2.47726s (14.5322GiB/s) | 20260508 10:10:43.831 | INFO | Downloaded 34GiB to GPU in 2.42126s (14.0423GiB/s) | 20260508 10:10:48.973 | INFO | Analyzing chunk frames = [7284224,7284736) | 20260508 10:10:48.973 | INFO | MemInfo: { | 20260508 10:10:48.973 | INFO | "gpu" : " 17 GB", | 20260508 10:10:48.973 | INFO | "huge" : "", | 20260508 10:10:48.973 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:48.973 | INFO | "malloc" : " 14 GB", | 20260508 10:10:48.973 | INFO | "pinned" : " 29 GB" | 20260508 10:10:48.973 | INFO | } | 20260508 10:10:48.974 | INFO | Perf report for the last 5.14309 seconds: | 20260508 10:10:48.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:48.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 10:10:48.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:10:48.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:48.974 | INFO | Uploaded 36GiB to GPU in 2.49819s (14.4104GiB/s) | 20260508 10:10:48.974 | INFO | Downloaded 34GiB to GPU in 2.42859s (13.9999GiB/s) | 20260508 10:10:54.106 | INFO | Analyzing chunk frames = [7284736,7285248) | 20260508 10:10:54.106 | INFO | MemInfo: { | 20260508 10:10:54.106 | INFO | "gpu" : " 17 GB", | 20260508 10:10:54.106 | INFO | "huge" : "", | 20260508 10:10:54.106 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:54.106 | INFO | "malloc" : " 14 GB", | 20260508 10:10:54.106 | INFO | "pinned" : " 29 GB" | 20260508 10:10:54.106 | INFO | } | 20260508 10:10:54.106 | INFO | Perf report for the last 5.1329 seconds: | 20260508 10:10:54.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:54.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.107 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:10:54.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.107 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 10:10:54.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:54.107 | INFO | Uploaded 36GiB to GPU in 2.29681s (15.6739GiB/s) | 20260508 10:10:54.107 | INFO | Downloaded 34GiB to GPU in 2.48663s (13.6731GiB/s) | 20260508 10:10:59.253 | INFO | Analyzing chunk frames = [7285248,7285760) | 20260508 10:10:59.253 | INFO | MemInfo: { | 20260508 10:10:59.253 | INFO | "gpu" : " 17 GB", | 20260508 10:10:59.253 | INFO | "huge" : "", | 20260508 10:10:59.253 | INFO | "huge_shared" : " 29 GB", | 20260508 10:10:59.254 | INFO | "malloc" : " 14 GB", | 20260508 10:10:59.254 | INFO | "pinned" : " 29 GB" | 20260508 10:10:59.254 | INFO | } | 20260508 10:10:59.254 | INFO | Perf report for the last 5.14728 seconds: | 20260508 10:10:59.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:10:59.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 10:10:59.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:10:59.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:10:59.254 | INFO | Uploaded 36GiB to GPU in 2.3745s (15.1611GiB/s) | 20260508 10:10:59.254 | INFO | Downloaded 34GiB to GPU in 2.43337s (13.9724GiB/s) | 20260508 10:11:04.393 | INFO | Analyzing chunk frames = [7285760,7286272) | 20260508 10:11:04.393 | INFO | MemInfo: { | 20260508 10:11:04.393 | INFO | "gpu" : " 17 GB", | 20260508 10:11:04.393 | INFO | "huge" : "", | 20260508 10:11:04.393 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:04.393 | INFO | "malloc" : " 14 GB", | 20260508 10:11:04.393 | INFO | "pinned" : " 29 GB" | 20260508 10:11:04.393 | INFO | } | 20260508 10:11:04.393 | INFO | Perf report for the last 5.1396 seconds: | 20260508 10:11:04.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:04.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.393 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 10:11:04.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.393 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:11:04.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:04.394 | INFO | Uploaded 36GiB to GPU in 2.43068s (14.8107GiB/s) | 20260508 10:11:04.394 | INFO | Downloaded 34GiB to GPU in 2.442s (13.923GiB/s) | 20260508 10:11:09.530 | INFO | Analyzing chunk frames = [7286272,7286784) | 20260508 10:11:09.530 | INFO | MemInfo: { | 20260508 10:11:09.530 | INFO | "gpu" : " 17 GB", | 20260508 10:11:09.530 | INFO | "huge" : "", | 20260508 10:11:09.530 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:09.531 | INFO | "malloc" : " 14 GB", | 20260508 10:11:09.531 | INFO | "pinned" : " 29 GB" | 20260508 10:11:09.531 | INFO | } | 20260508 10:11:09.531 | INFO | Perf report for the last 5.13738 seconds: | 20260508 10:11:09.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:09.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 10:11:09.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:11:09.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:09.531 | INFO | Uploaded 36GiB to GPU in 2.41645s (14.8979GiB/s) | 20260508 10:11:09.531 | INFO | Downloaded 34GiB to GPU in 2.4461s (13.8997GiB/s) | 20260508 10:11:09.628 | INFO | Starting a round of Baz IO | 20260508 10:11:13.049 | INFO | Populating RTMetrics report | 20260508 10:11:13.050 | INFO | RTMetrics report populated | 20260508 10:11:13.050 | INFO | Opening tmp RTMetrics json file | 20260508 10:11:13.054 | INFO | Serializing RTMetrics to json | 20260508 10:11:13.056 | INFO | Serialization done, swapping tmp file now... | 20260508 10:11:13.056 | INFO | RTMetrics done | 20260508 10:11:13.112 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.483 seconds | 20260508 10:11:13.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.23% / 0.00% / 59.77% | 20260508 10:11:13.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230618% / 0% / 5.96046e-05% | 20260508 10:11:13.112 | INFO | Baz IO overall: | 20260508 10:11:13.112 | INFO | BytesWritten : 3166746627897 | 20260508 10:11:13.112 | INFO | NumEvents : 1522488393182 | 20260508 10:11:14.662 | INFO | Analyzing chunk frames = [7286784,7287296) | 20260508 10:11:14.662 | INFO | MemInfo: { | 20260508 10:11:14.662 | INFO | "gpu" : " 17 GB", | 20260508 10:11:14.662 | INFO | "huge" : "", | 20260508 10:11:14.662 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:14.662 | INFO | "malloc" : " 14 GB", | 20260508 10:11:14.662 | INFO | "pinned" : " 29 GB" | 20260508 10:11:14.662 | INFO | } | 20260508 10:11:14.662 | INFO | Perf report for the last 5.13171 seconds: | 20260508 10:11:14.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:14.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:14.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:14.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:14.663 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:11:14.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:14.663 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 10:11:14.663 | INFO | PRE_HQ : 65.7%, 1.6, 54.8ms, 0 | 20260508 10:11:14.663 | INFO | BAZWRITER : 67.3%, 2.5, 86.8ms, 0 | 20260508 10:11:14.663 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:11:14.663 | INFO | Uploaded 36GiB to GPU in 2.58449s (13.9292GiB/s) | 20260508 10:11:14.663 | INFO | Downloaded 38GiB to GPU in 2.71078s (14.0181GiB/s) | 20260508 10:11:19.805 | INFO | Analyzing chunk frames = [7287296,7287808) | 20260508 10:11:19.805 | INFO | MemInfo: { | 20260508 10:11:19.805 | INFO | "gpu" : " 17 GB", | 20260508 10:11:19.805 | INFO | "huge" : "", | 20260508 10:11:19.805 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:19.805 | INFO | "malloc" : " 14 GB", | 20260508 10:11:19.805 | INFO | "pinned" : " 29 GB" | 20260508 10:11:19.805 | INFO | } | 20260508 10:11:19.805 | INFO | Perf report for the last 5.14273 seconds: | 20260508 10:11:19.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:19.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.805 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:11:19.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.805 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 10:11:19.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:19.806 | INFO | Uploaded 36GiB to GPU in 2.51554s (14.311GiB/s) | 20260508 10:11:19.806 | INFO | Downloaded 34GiB to GPU in 2.44265s (13.9193GiB/s) | 20260508 10:11:24.947 | INFO | Analyzing chunk frames = [7287808,7288320) | 20260508 10:11:24.947 | INFO | MemInfo: { | 20260508 10:11:24.947 | INFO | "gpu" : " 17 GB", | 20260508 10:11:24.947 | INFO | "huge" : "", | 20260508 10:11:24.947 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:24.947 | INFO | "malloc" : " 14 GB", | 20260508 10:11:24.947 | INFO | "pinned" : " 29 GB" | 20260508 10:11:24.947 | INFO | } | 20260508 10:11:24.947 | INFO | Perf report for the last 5.14174 seconds: | 20260508 10:11:24.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:24.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 10:11:24.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:11:24.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:24.947 | INFO | Uploaded 36GiB to GPU in 2.45109s (14.6874GiB/s) | 20260508 10:11:24.947 | INFO | Downloaded 34GiB to GPU in 2.42108s (14.0433GiB/s) | 20260508 10:11:30.087 | INFO | Analyzing chunk frames = [7288320,7288832) | 20260508 10:11:30.087 | INFO | MemInfo: { | 20260508 10:11:30.087 | INFO | "gpu" : " 17 GB", | 20260508 10:11:30.087 | INFO | "huge" : "", | 20260508 10:11:30.087 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:30.087 | INFO | "malloc" : " 14 GB", | 20260508 10:11:30.087 | INFO | "pinned" : " 29 GB" | 20260508 10:11:30.087 | INFO | } | 20260508 10:11:30.087 | INFO | Perf report for the last 5.14004 seconds: | 20260508 10:11:30.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:30.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 10:11:30.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:11:30.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:30.087 | INFO | Uploaded 36GiB to GPU in 2.48996s (14.4581GiB/s) | 20260508 10:11:30.087 | INFO | Downloaded 34GiB to GPU in 2.42676s (14.0104GiB/s) | 20260508 10:11:35.227 | INFO | Analyzing chunk frames = [7288832,7289344) | 20260508 10:11:35.227 | INFO | MemInfo: { | 20260508 10:11:35.227 | INFO | "gpu" : " 17 GB", | 20260508 10:11:35.227 | INFO | "huge" : "", | 20260508 10:11:35.227 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:35.227 | INFO | "malloc" : " 14 GB", | 20260508 10:11:35.227 | INFO | "pinned" : " 29 GB" | 20260508 10:11:35.227 | INFO | } | 20260508 10:11:35.227 | INFO | Perf report for the last 5.14025 seconds: | 20260508 10:11:35.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:35.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.227 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:11:35.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.227 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:11:35.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:35.228 | INFO | Uploaded 36GiB to GPU in 2.30522s (15.6167GiB/s) | 20260508 10:11:35.228 | INFO | Downloaded 34GiB to GPU in 2.49313s (13.6375GiB/s) | 20260508 10:11:40.361 | INFO | Analyzing chunk frames = [7289344,7289856) | 20260508 10:11:40.361 | INFO | MemInfo: { | 20260508 10:11:40.361 | INFO | "gpu" : " 17 GB", | 20260508 10:11:40.361 | INFO | "huge" : "", | 20260508 10:11:40.361 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:40.361 | INFO | "malloc" : " 14 GB", | 20260508 10:11:40.361 | INFO | "pinned" : " 29 GB" | 20260508 10:11:40.361 | INFO | } | 20260508 10:11:40.361 | INFO | Perf report for the last 5.13365 seconds: | 20260508 10:11:40.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:40.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 10:11:40.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:11:40.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:40.361 | INFO | Uploaded 36GiB to GPU in 2.35753s (15.2702GiB/s) | 20260508 10:11:40.361 | INFO | Downloaded 34GiB to GPU in 2.42313s (14.0315GiB/s) | 20260508 10:11:45.503 | INFO | Analyzing chunk frames = [7289856,7290368) | 20260508 10:11:45.503 | INFO | MemInfo: { | 20260508 10:11:45.503 | INFO | "gpu" : " 17 GB", | 20260508 10:11:45.503 | INFO | "huge" : "", | 20260508 10:11:45.503 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:45.503 | INFO | "malloc" : " 14 GB", | 20260508 10:11:45.503 | INFO | "pinned" : " 29 GB" | 20260508 10:11:45.503 | INFO | } | 20260508 10:11:45.503 | INFO | Perf report for the last 5.14233 seconds: | 20260508 10:11:45.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:45.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.503 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 10:11:45.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.503 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:11:45.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:45.504 | INFO | Uploaded 36GiB to GPU in 2.3987s (15.0081GiB/s) | 20260508 10:11:45.504 | INFO | Downloaded 34GiB to GPU in 2.44808s (13.8884GiB/s) | 20260508 10:11:50.643 | INFO | Analyzing chunk frames = [7290368,7290880) | 20260508 10:11:50.643 | INFO | MemInfo: { | 20260508 10:11:50.643 | INFO | "gpu" : " 17 GB", | 20260508 10:11:50.643 | INFO | "huge" : "", | 20260508 10:11:50.643 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:50.643 | INFO | "malloc" : " 14 GB", | 20260508 10:11:50.643 | INFO | "pinned" : " 29 GB" | 20260508 10:11:50.643 | INFO | } | 20260508 10:11:50.644 | INFO | Perf report for the last 5.14024 seconds: | 20260508 10:11:50.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:50.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:11:50.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:11:50.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:50.644 | INFO | Uploaded 36GiB to GPU in 2.40107s (14.9933GiB/s) | 20260508 10:11:50.644 | INFO | Downloaded 34GiB to GPU in 2.43463s (13.9652GiB/s) | 20260508 10:11:50.644 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:11:50.644 | INFO | Total mem usage: 16.7 GiB | 20260508 10:11:50.644 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:11:50.644 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:11:50.644 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:11:50.644 | INFO | High Water Breakdown: | 20260508 10:11:50.644 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:11:50.644 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:11:50.644 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:11:50.644 | INFO | ------------------------------------------ | 20260508 10:11:50.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:11:50.644 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:11:50.644 | INFO | Total mem usage: 26.5 GiB | 20260508 10:11:50.644 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:11:50.644 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:11:50.644 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:11:50.644 | INFO | High Water Breakdown: | 20260508 10:11:50.644 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:11:50.644 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:11:50.644 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:11:50.644 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:11:50.644 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:11:50.644 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:11:50.644 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:11:50.644 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:11:50.644 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:11:50.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:11:50.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:11:50.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:11:50.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:11:50.645 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:11:50.645 | INFO | ------------------------------------------ | 20260508 10:11:50.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:11:50.645 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:11:50.645 | INFO | Total mem usage: 27.8 GiB | 20260508 10:11:50.645 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:11:50.645 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:11:50.645 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:11:50.645 | INFO | High Water Breakdown: | 20260508 10:11:50.645 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:11:50.645 | INFO | ------------------------------------------ | 20260508 10:11:50.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:11:50.645 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:11:50.645 | INFO | Total mem usage: 15.7 GiB | 20260508 10:11:50.645 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:11:50.645 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:11:50.645 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:11:50.645 | INFO | High Water Breakdown: | 20260508 10:11:50.645 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:11:50.645 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:11:50.645 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:11:50.645 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:11:50.645 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:11:50.645 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:11:50.645 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:11:50.645 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:11:50.645 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:11:50.645 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:11:50.645 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:11:50.645 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:11:50.645 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:11:50.645 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:11:50.645 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:11:50.645 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:11:50.645 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:11:50.645 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:11:50.645 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:11:50.645 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:11:50.645 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:11:50.645 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:11:50.645 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:11:50.645 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:11:50.645 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:11:50.645 | INFO | ------------------------------------------ | 20260508 10:11:50.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:11:50.645 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:11:50.645 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:11:50.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:11:50.645 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 6.82e-01 | 20260508 10:11:50.645 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.11e-02 / 4.10e-03 | 20260508 10:11:50.645 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.49e+00, 6.04e-01 / 1.87e-01 | 20260508 10:11:50.645 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.48e+00, 2.14e-01 / 1.15e-01 | 20260508 10:11:50.645 | INFO | PerfReport: 16.03%, DME -- 4.20e+01 / 4.53e+01, 3.70e+00 / 1.09e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.84e+00 / 1.32e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.08e-01 / 2.89e-01 | 20260508 10:11:50.646 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.15e+00, 2.46e-01 / 1.67e-01 | 20260508 10:11:50.646 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.35e-01, 4.81e-02 / 2.96e-02 | 20260508 10:11:50.646 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:11:50.646 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:11:50.646 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:11:50.646 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 24.16%, Upload -- 2.54e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 24.52%, Download -- 2.58e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 31.13%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 2.55%, LockDownload -- 2.69e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 15.46%, LockCompute -- 1.63e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:11:50.646 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 9.11e-03 / 0.00e+00 | 20260508 10:11:50.756 | INFO | Starting a round of Baz IO | 20260508 10:11:54.152 | INFO | Populating RTMetrics report | 20260508 10:11:54.152 | INFO | RTMetrics report populated | 20260508 10:11:54.152 | INFO | Opening tmp RTMetrics json file | 20260508 10:11:54.164 | INFO | Serializing RTMetrics to json | 20260508 10:11:54.165 | INFO | Serialization done, swapping tmp file now... | 20260508 10:11:54.165 | INFO | RTMetrics done | 20260508 10:11:54.226 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.47 seconds | 20260508 10:11:54.226 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.77% | 20260508 10:11:54.226 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230205% / 3.97364e-06% / 8.74201e-05% | 20260508 10:11:54.226 | INFO | Baz IO overall: | 20260508 10:11:54.226 | INFO | BytesWritten : 3168320741601 | 20260508 10:11:54.226 | INFO | NumEvents : 1523232017930 | 20260508 10:11:55.785 | INFO | Analyzing chunk frames = [7290880,7291392) | 20260508 10:11:55.785 | INFO | MemInfo: { | 20260508 10:11:55.785 | INFO | "gpu" : " 17 GB", | 20260508 10:11:55.785 | INFO | "huge" : "", | 20260508 10:11:55.785 | INFO | "huge_shared" : " 29 GB", | 20260508 10:11:55.785 | INFO | "malloc" : " 14 GB", | 20260508 10:11:55.785 | INFO | "pinned" : " 29 GB" | 20260508 10:11:55.785 | INFO | } | 20260508 10:11:55.785 | INFO | Perf report for the last 5.14121 seconds: | 20260508 10:11:55.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:11:55.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:55.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:55.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:55.785 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 10:11:55.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:11:55.785 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260508 10:11:55.785 | INFO | PRE_HQ : 66.1%, 1.7, 59.7ms, 0 | 20260508 10:11:55.785 | INFO | BAZWRITER : 66.8%, 2.7, 92.1ms, 0 | 20260508 10:11:55.785 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:11:55.785 | INFO | Uploaded 36GiB to GPU in 2.57295s (13.9917GiB/s) | 20260508 10:11:55.785 | INFO | Downloaded 38GiB to GPU in 2.70602s (14.0428GiB/s) | 20260508 10:12:00.922 | INFO | Analyzing chunk frames = [7291392,7291904) | 20260508 10:12:00.922 | INFO | MemInfo: { | 20260508 10:12:00.922 | INFO | "gpu" : " 17 GB", | 20260508 10:12:00.922 | INFO | "huge" : "", | 20260508 10:12:00.922 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:00.922 | INFO | "malloc" : " 14 GB", | 20260508 10:12:00.923 | INFO | "pinned" : " 29 GB" | 20260508 10:12:00.923 | INFO | } | 20260508 10:12:00.923 | INFO | Perf report for the last 5.13789 seconds: | 20260508 10:12:00.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:00.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:12:00.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:12:00.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:00.923 | INFO | Uploaded 36GiB to GPU in 2.48966s (14.4598GiB/s) | 20260508 10:12:00.923 | INFO | Downloaded 34GiB to GPU in 2.46108s (13.8151GiB/s) | 20260508 10:12:06.071 | INFO | Analyzing chunk frames = [7291904,7292416) | 20260508 10:12:06.071 | INFO | MemInfo: { | 20260508 10:12:06.071 | INFO | "gpu" : " 17 GB", | 20260508 10:12:06.071 | INFO | "huge" : "", | 20260508 10:12:06.071 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:06.071 | INFO | "malloc" : " 14 GB", | 20260508 10:12:06.071 | INFO | "pinned" : " 29 GB" | 20260508 10:12:06.071 | INFO | } | 20260508 10:12:06.071 | INFO | Perf report for the last 5.14841 seconds: | 20260508 10:12:06.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:06.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 10:12:06.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:12:06.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:06.071 | INFO | Uploaded 36GiB to GPU in 2.45637s (14.6558GiB/s) | 20260508 10:12:06.071 | INFO | Downloaded 34GiB to GPU in 2.44825s (13.8875GiB/s) | 20260508 10:12:11.203 | INFO | Analyzing chunk frames = [7292416,7292928) | 20260508 10:12:11.203 | INFO | MemInfo: { | 20260508 10:12:11.203 | INFO | "gpu" : " 17 GB", | 20260508 10:12:11.203 | INFO | "huge" : "", | 20260508 10:12:11.203 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:11.203 | INFO | "malloc" : " 14 GB", | 20260508 10:12:11.204 | INFO | "pinned" : " 29 GB" | 20260508 10:12:11.204 | INFO | } | 20260508 10:12:11.204 | INFO | Perf report for the last 5.13252 seconds: | 20260508 10:12:11.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:11.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | ANALYSIS : 66.0%, 3.0, 102.1ms, 0 | 20260508 10:12:11.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:12:11.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:11.204 | INFO | Uploaded 36GiB to GPU in 2.47062s (14.5712GiB/s) | 20260508 10:12:11.204 | INFO | Downloaded 34GiB to GPU in 2.46747s (13.7793GiB/s) | 20260508 10:12:16.342 | INFO | Analyzing chunk frames = [7292928,7293440) | 20260508 10:12:16.342 | INFO | MemInfo: { | 20260508 10:12:16.342 | INFO | "gpu" : " 17 GB", | 20260508 10:12:16.342 | INFO | "huge" : "", | 20260508 10:12:16.342 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:16.342 | INFO | "malloc" : " 14 GB", | 20260508 10:12:16.342 | INFO | "pinned" : " 29 GB" | 20260508 10:12:16.342 | INFO | } | 20260508 10:12:16.342 | INFO | Perf report for the last 5.13854 seconds: | 20260508 10:12:16.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:16.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | ANALYSIS : 66.5%, 3.0, 103.9ms, 0 | 20260508 10:12:16.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:12:16.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:16.342 | INFO | Uploaded 36GiB to GPU in 2.39146s (15.0535GiB/s) | 20260508 10:12:16.343 | INFO | Downloaded 34GiB to GPU in 2.5517s (13.3244GiB/s) | 20260508 10:12:21.481 | INFO | Analyzing chunk frames = [7293440,7293952) | 20260508 10:12:21.481 | INFO | MemInfo: { | 20260508 10:12:21.481 | INFO | "gpu" : " 17 GB", | 20260508 10:12:21.481 | INFO | "huge" : "", | 20260508 10:12:21.481 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:21.481 | INFO | "malloc" : " 14 GB", | 20260508 10:12:21.481 | INFO | "pinned" : " 29 GB" | 20260508 10:12:21.481 | INFO | } | 20260508 10:12:21.481 | INFO | Perf report for the last 5.13867 seconds: | 20260508 10:12:21.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:21.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:12:21.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:12:21.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:21.481 | INFO | Uploaded 36GiB to GPU in 2.36154s (15.2443GiB/s) | 20260508 10:12:21.481 | INFO | Downloaded 34GiB to GPU in 2.43911s (13.9395GiB/s) | 20260508 10:12:26.615 | INFO | Analyzing chunk frames = [7293952,7294464) | 20260508 10:12:26.615 | INFO | MemInfo: { | 20260508 10:12:26.615 | INFO | "gpu" : " 17 GB", | 20260508 10:12:26.615 | INFO | "huge" : "", | 20260508 10:12:26.615 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:26.615 | INFO | "malloc" : " 14 GB", | 20260508 10:12:26.616 | INFO | "pinned" : " 29 GB" | 20260508 10:12:26.616 | INFO | } | 20260508 10:12:26.616 | INFO | Perf report for the last 5.13474 seconds: | 20260508 10:12:26.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:26.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 10:12:26.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:12:26.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:26.616 | INFO | Uploaded 36GiB to GPU in 2.38645s (15.0852GiB/s) | 20260508 10:12:26.616 | INFO | Downloaded 34GiB to GPU in 2.4513s (13.8702GiB/s) | 20260508 10:12:31.757 | INFO | Analyzing chunk frames = [7294464,7294976) | 20260508 10:12:31.757 | INFO | MemInfo: { | 20260508 10:12:31.757 | INFO | "gpu" : " 17 GB", | 20260508 10:12:31.757 | INFO | "huge" : "", | 20260508 10:12:31.757 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:31.757 | INFO | "malloc" : " 14 GB", | 20260508 10:12:31.757 | INFO | "pinned" : " 29 GB" | 20260508 10:12:31.757 | INFO | } | 20260508 10:12:31.757 | INFO | Perf report for the last 5.14185 seconds: | 20260508 10:12:31.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:31.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.758 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 10:12:31.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.758 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:12:31.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:31.758 | INFO | Uploaded 36GiB to GPU in 2.40594s (14.963GiB/s) | 20260508 10:12:31.758 | INFO | Downloaded 34GiB to GPU in 2.43866s (13.9421GiB/s) | 20260508 10:12:31.855 | INFO | Starting a round of Baz IO | 20260508 10:12:35.262 | INFO | Populating RTMetrics report | 20260508 10:12:35.263 | INFO | RTMetrics report populated | 20260508 10:12:35.263 | INFO | Opening tmp RTMetrics json file | 20260508 10:12:35.272 | INFO | Serializing RTMetrics to json | 20260508 10:12:35.274 | INFO | Serialization done, swapping tmp file now... | 20260508 10:12:35.274 | INFO | RTMetrics done | 20260508 10:12:35.336 | INFO | Finished a round of Baz IO: Wrote 1.466 GiB in 3.48 seconds | 20260508 10:12:35.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:12:35.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230432% / 7.94729e-06% / 3.97364e-05% | 20260508 10:12:35.336 | INFO | Baz IO overall: | 20260508 10:12:35.336 | INFO | BytesWritten : 3169894792277 | 20260508 10:12:35.336 | INFO | NumEvents : 1523975411243 | 20260508 10:12:36.904 | INFO | Analyzing chunk frames = [7294976,7295488) | 20260508 10:12:36.904 | INFO | MemInfo: { | 20260508 10:12:36.904 | INFO | "gpu" : " 17 GB", | 20260508 10:12:36.904 | INFO | "huge" : "", | 20260508 10:12:36.904 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:36.904 | INFO | "malloc" : " 14 GB", | 20260508 10:12:36.904 | INFO | "pinned" : " 29 GB" | 20260508 10:12:36.904 | INFO | } | 20260508 10:12:36.904 | INFO | Perf report for the last 5.14656 seconds: | 20260508 10:12:36.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:36.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:36.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:36.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:36.904 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:12:36.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:36.904 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:12:36.904 | INFO | PRE_HQ : 65.2%, 1.6, 54.3ms, 0 | 20260508 10:12:36.904 | INFO | BAZWRITER : 67.0%, 2.5, 87.2ms, 0 | 20260508 10:12:36.904 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:12:36.905 | INFO | Uploaded 36GiB to GPU in 2.59585s (13.8683GiB/s) | 20260508 10:12:36.905 | INFO | Downloaded 38GiB to GPU in 2.71254s (14.009GiB/s) | 20260508 10:12:42.039 | INFO | Analyzing chunk frames = [7295488,7296000) | 20260508 10:12:42.039 | INFO | MemInfo: { | 20260508 10:12:42.039 | INFO | "gpu" : " 17 GB", | 20260508 10:12:42.039 | INFO | "huge" : "", | 20260508 10:12:42.039 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:42.040 | INFO | "malloc" : " 14 GB", | 20260508 10:12:42.040 | INFO | "pinned" : " 29 GB" | 20260508 10:12:42.040 | INFO | } | 20260508 10:12:42.040 | INFO | Perf report for the last 5.13567 seconds: | 20260508 10:12:42.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:42.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 10:12:42.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:12:42.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:42.040 | INFO | Uploaded 36GiB to GPU in 2.41059s (14.9341GiB/s) | 20260508 10:12:42.040 | INFO | Downloaded 34GiB to GPU in 2.46495s (13.7934GiB/s) | 20260508 10:12:47.174 | INFO | Analyzing chunk frames = [7296000,7296512) | 20260508 10:12:47.174 | INFO | MemInfo: { | 20260508 10:12:47.174 | INFO | "gpu" : " 17 GB", | 20260508 10:12:47.174 | INFO | "huge" : "", | 20260508 10:12:47.174 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:47.174 | INFO | "malloc" : " 14 GB", | 20260508 10:12:47.174 | INFO | "pinned" : " 29 GB" | 20260508 10:12:47.174 | INFO | } | 20260508 10:12:47.174 | INFO | Perf report for the last 5.13449 seconds: | 20260508 10:12:47.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:47.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.174 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 10:12:47.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.174 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:12:47.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:47.175 | INFO | Uploaded 36GiB to GPU in 2.44942s (14.6973GiB/s) | 20260508 10:12:47.175 | INFO | Downloaded 34GiB to GPU in 2.42993s (13.9922GiB/s) | 20260508 10:12:52.315 | INFO | Analyzing chunk frames = [7296512,7297024) | 20260508 10:12:52.315 | INFO | MemInfo: { | 20260508 10:12:52.316 | INFO | "gpu" : " 17 GB", | 20260508 10:12:52.316 | INFO | "huge" : "", | 20260508 10:12:52.316 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:52.316 | INFO | "malloc" : " 14 GB", | 20260508 10:12:52.316 | INFO | "pinned" : " 29 GB" | 20260508 10:12:52.316 | INFO | } | 20260508 10:12:52.316 | INFO | Perf report for the last 5.14146 seconds: | 20260508 10:12:52.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:52.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 10:12:52.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:12:52.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:52.316 | INFO | Uploaded 36GiB to GPU in 2.42971s (14.8166GiB/s) | 20260508 10:12:52.317 | INFO | Downloaded 34GiB to GPU in 2.41388s (14.0852GiB/s) | 20260508 10:12:57.454 | INFO | Analyzing chunk frames = [7297024,7297536) | 20260508 10:12:57.454 | INFO | MemInfo: { | 20260508 10:12:57.454 | INFO | "gpu" : " 17 GB", | 20260508 10:12:57.454 | INFO | "huge" : "", | 20260508 10:12:57.454 | INFO | "huge_shared" : " 29 GB", | 20260508 10:12:57.454 | INFO | "malloc" : " 14 GB", | 20260508 10:12:57.454 | INFO | "pinned" : " 29 GB" | 20260508 10:12:57.454 | INFO | } | 20260508 10:12:57.454 | INFO | Perf report for the last 5.13865 seconds: | 20260508 10:12:57.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:12:57.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.454 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 10:12:57.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.454 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:12:57.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:12:57.455 | INFO | Uploaded 36GiB to GPU in 2.40829s (14.9484GiB/s) | 20260508 10:12:57.455 | INFO | Downloaded 34GiB to GPU in 2.54006s (13.3855GiB/s) | 20260508 10:13:02.598 | INFO | Analyzing chunk frames = [7297536,7298048) | 20260508 10:13:02.598 | INFO | MemInfo: { | 20260508 10:13:02.598 | INFO | "gpu" : " 17 GB", | 20260508 10:13:02.598 | INFO | "huge" : "", | 20260508 10:13:02.598 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:02.598 | INFO | "malloc" : " 14 GB", | 20260508 10:13:02.598 | INFO | "pinned" : " 29 GB" | 20260508 10:13:02.598 | INFO | } | 20260508 10:13:02.598 | INFO | Perf report for the last 5.14409 seconds: | 20260508 10:13:02.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:02.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.599 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 10:13:02.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.599 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260508 10:13:02.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:02.599 | INFO | Uploaded 36GiB to GPU in 2.31855s (15.5269GiB/s) | 20260508 10:13:02.599 | INFO | Downloaded 34GiB to GPU in 2.45321s (13.8594GiB/s) | 20260508 10:13:07.738 | INFO | Analyzing chunk frames = [7298048,7298560) | 20260508 10:13:07.738 | INFO | MemInfo: { | 20260508 10:13:07.738 | INFO | "gpu" : " 17 GB", | 20260508 10:13:07.738 | INFO | "huge" : "", | 20260508 10:13:07.738 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:07.738 | INFO | "malloc" : " 14 GB", | 20260508 10:13:07.738 | INFO | "pinned" : " 29 GB" | 20260508 10:13:07.738 | INFO | } | 20260508 10:13:07.738 | INFO | Perf report for the last 5.13945 seconds: | 20260508 10:13:07.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:07.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.738 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:13:07.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.738 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:13:07.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:07.739 | INFO | Uploaded 36GiB to GPU in 2.39822s (15.0112GiB/s) | 20260508 10:13:07.739 | INFO | Downloaded 34GiB to GPU in 2.45223s (13.865GiB/s) | 20260508 10:13:12.871 | INFO | Analyzing chunk frames = [7298560,7299072) | 20260508 10:13:12.871 | INFO | MemInfo: { | 20260508 10:13:12.871 | INFO | "gpu" : " 17 GB", | 20260508 10:13:12.871 | INFO | "huge" : "", | 20260508 10:13:12.871 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:12.871 | INFO | "malloc" : " 14 GB", | 20260508 10:13:12.871 | INFO | "pinned" : " 29 GB" | 20260508 10:13:12.871 | INFO | } | 20260508 10:13:12.871 | INFO | Perf report for the last 5.13342 seconds: | 20260508 10:13:12.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:12.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.871 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 10:13:12.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.871 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:13:12.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:12.872 | INFO | Uploaded 36GiB to GPU in 2.39609s (15.0245GiB/s) | 20260508 10:13:12.872 | INFO | Downloaded 34GiB to GPU in 2.44454s (13.9086GiB/s) | 20260508 10:13:12.872 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:13:12.872 | INFO | Total mem usage: 16.7 GiB | 20260508 10:13:12.872 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:13:12.872 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 10:13:12.872 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:13:12.872 | INFO | High Water Breakdown: | 20260508 10:13:12.872 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:13:12.872 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:13:12.872 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:13:12.872 | INFO | ------------------------------------------ | 20260508 10:13:12.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:13:12.872 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:13:12.872 | INFO | Total mem usage: 26.5 GiB | 20260508 10:13:12.872 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:13:12.872 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:13:12.872 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:13:12.872 | INFO | High Water Breakdown: | 20260508 10:13:12.872 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:13:12.872 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:13:12.872 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:13:12.872 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:13:12.872 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:13:12.872 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:13:12.872 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:13:12.872 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:13:12.872 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:13:12.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:13:12.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:13:12.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:13:12.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:13:12.872 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:13:12.872 | INFO | ------------------------------------------ | 20260508 10:13:12.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:13:12.872 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:13:12.872 | INFO | Total mem usage: 27.8 GiB | 20260508 10:13:12.872 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:13:12.872 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:13:12.872 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:13:12.872 | INFO | High Water Breakdown: | 20260508 10:13:12.872 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:13:12.872 | INFO | ------------------------------------------ | 20260508 10:13:12.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:13:12.872 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:13:12.872 | INFO | Total mem usage: 15.7 GiB | 20260508 10:13:12.872 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:13:12.872 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:13:12.872 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:13:12.872 | INFO | High Water Breakdown: | 20260508 10:13:12.872 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:13:12.872 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:13:12.872 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:13:12.872 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:13:12.872 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:13:12.872 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:13:12.872 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:13:12.872 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:13:12.872 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:13:12.872 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:13:12.872 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:13:12.872 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:13:12.872 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:13:12.872 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:13:12.872 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:13:12.872 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:13:12.872 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:13:12.872 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:13:12.872 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:13:12.872 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:13:12.872 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:13:12.872 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:13:12.872 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:13:12.872 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:13:12.872 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:13:12.872 | INFO | ------------------------------------------ | 20260508 10:13:12.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:13:12.872 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:13:12.872 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:13:12.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:13:12.872 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 8.05e-01 | 20260508 10:13:12.872 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.09e-02 / 6.19e-03 | 20260508 10:13:12.872 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.44e+00, 5.92e-01 / 1.82e-01 | 20260508 10:13:12.872 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.37e-01 | 20260508 10:13:12.872 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.51e+01, 3.75e+00 / 9.96e-01 | 20260508 10:13:12.872 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.83e+00 / 1.30e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.05e-01 / 2.88e-01 | 20260508 10:13:12.872 | INFO | PerfReport: 3.02%, Metrics -- 9.88e-01 / 1.14e+00, 2.43e-01 / 1.49e-01 | 20260508 10:13:12.872 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.34e-01, 4.79e-02 / 3.56e-02 | 20260508 10:13:12.872 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:13:12.872 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:13:12.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:13:12.872 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 24.10%, Upload -- 2.54e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 24.68%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 2.00%, LockUpload -- 2.11e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 2.70%, LockDownload -- 2.84e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:13:12.872 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 10:13:12.969 | INFO | Starting a round of Baz IO | 20260508 10:13:16.383 | INFO | Populating RTMetrics report | 20260508 10:13:16.383 | INFO | RTMetrics report populated | 20260508 10:13:16.383 | INFO | Opening tmp RTMetrics json file | 20260508 10:13:16.384 | INFO | Serializing RTMetrics to json | 20260508 10:13:16.385 | INFO | Serialization done, swapping tmp file now... | 20260508 10:13:16.385 | INFO | RTMetrics done | 20260508 10:13:16.446 | INFO | Finished a round of Baz IO: Wrote 1.464 GiB in 3.477 seconds | 20260508 10:13:16.447 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:13:16.447 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.229212% / 3.97364e-06% / 5.96046e-05% | 20260508 10:13:16.447 | INFO | Baz IO overall: | 20260508 10:13:16.447 | INFO | BytesWritten : 3171467148643 | 20260508 10:13:16.447 | INFO | NumEvents : 1524717717210 | 20260508 10:13:18.017 | INFO | Analyzing chunk frames = [7299072,7299584) | 20260508 10:13:18.017 | INFO | MemInfo: { | 20260508 10:13:18.017 | INFO | "gpu" : " 17 GB", | 20260508 10:13:18.017 | INFO | "huge" : "", | 20260508 10:13:18.017 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:18.017 | INFO | "malloc" : " 14 GB", | 20260508 10:13:18.017 | INFO | "pinned" : " 29 GB" | 20260508 10:13:18.017 | INFO | } | 20260508 10:13:18.017 | INFO | Perf report for the last 5.14526 seconds: | 20260508 10:13:18.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:18.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:18.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:18.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:18.017 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 10:13:18.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:18.017 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:13:18.017 | INFO | PRE_HQ : 65.7%, 1.6, 54.7ms, 0 | 20260508 10:13:18.017 | INFO | BAZWRITER : 67.0%, 2.6, 88.0ms, 0 | 20260508 10:13:18.017 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:13:18.017 | INFO | Uploaded 36GiB to GPU in 2.50433s (14.3751GiB/s) | 20260508 10:13:18.017 | INFO | Downloaded 38GiB to GPU in 2.67689s (14.1956GiB/s) | 20260508 10:13:23.153 | INFO | Analyzing chunk frames = [7299584,7300096) | 20260508 10:13:23.153 | INFO | MemInfo: { | 20260508 10:13:23.153 | INFO | "gpu" : " 17 GB", | 20260508 10:13:23.153 | INFO | "huge" : "", | 20260508 10:13:23.153 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:23.153 | INFO | "malloc" : " 14 GB", | 20260508 10:13:23.153 | INFO | "pinned" : " 29 GB" | 20260508 10:13:23.153 | INFO | } | 20260508 10:13:23.153 | INFO | Perf report for the last 5.13683 seconds: | 20260508 10:13:23.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:23.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.154 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:13:23.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.154 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:13:23.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:23.154 | INFO | Uploaded 36GiB to GPU in 2.46113s (14.6274GiB/s) | 20260508 10:13:23.154 | INFO | Downloaded 34GiB to GPU in 2.46709s (13.7814GiB/s) | 20260508 10:13:28.288 | INFO | Analyzing chunk frames = [7300096,7300608) | 20260508 10:13:28.288 | INFO | MemInfo: { | 20260508 10:13:28.288 | INFO | "gpu" : " 17 GB", | 20260508 10:13:28.288 | INFO | "huge" : "", | 20260508 10:13:28.289 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:28.289 | INFO | "malloc" : " 14 GB", | 20260508 10:13:28.289 | INFO | "pinned" : " 29 GB" | 20260508 10:13:28.289 | INFO | } | 20260508 10:13:28.289 | INFO | Perf report for the last 5.13521 seconds: | 20260508 10:13:28.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:28.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 10:13:28.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:13:28.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:28.289 | INFO | Uploaded 36GiB to GPU in 2.46447s (14.6076GiB/s) | 20260508 10:13:28.289 | INFO | Downloaded 34GiB to GPU in 2.42452s (14.0234GiB/s) | 20260508 10:13:33.426 | INFO | Analyzing chunk frames = [7300608,7301120) | 20260508 10:13:33.426 | INFO | MemInfo: { | 20260508 10:13:33.427 | INFO | "gpu" : " 17 GB", | 20260508 10:13:33.427 | INFO | "huge" : "", | 20260508 10:13:33.427 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:33.427 | INFO | "malloc" : " 14 GB", | 20260508 10:13:33.427 | INFO | "pinned" : " 29 GB" | 20260508 10:13:33.427 | INFO | } | 20260508 10:13:33.427 | INFO | Perf report for the last 5.13809 seconds: | 20260508 10:13:33.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:33.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 10:13:33.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:13:33.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:33.427 | INFO | Uploaded 36GiB to GPU in 2.48588s (14.4818GiB/s) | 20260508 10:13:33.427 | INFO | Downloaded 34GiB to GPU in 2.4316s (13.9826GiB/s) | 20260508 10:13:38.570 | INFO | Analyzing chunk frames = [7301120,7301632) | 20260508 10:13:38.570 | INFO | MemInfo: { | 20260508 10:13:38.570 | INFO | "gpu" : " 17 GB", | 20260508 10:13:38.570 | INFO | "huge" : "", | 20260508 10:13:38.570 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:38.570 | INFO | "malloc" : " 14 GB", | 20260508 10:13:38.570 | INFO | "pinned" : " 29 GB" | 20260508 10:13:38.570 | INFO | } | 20260508 10:13:38.570 | INFO | Perf report for the last 5.14325 seconds: | 20260508 10:13:38.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:38.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 10:13:38.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:13:38.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:38.570 | INFO | Uploaded 36GiB to GPU in 2.35178s (15.3076GiB/s) | 20260508 10:13:38.571 | INFO | Downloaded 34GiB to GPU in 2.50231s (13.5875GiB/s) | 20260508 10:13:43.720 | INFO | Analyzing chunk frames = [7301632,7302144) | 20260508 10:13:43.720 | INFO | MemInfo: { | 20260508 10:13:43.720 | INFO | "gpu" : " 17 GB", | 20260508 10:13:43.720 | INFO | "huge" : "", | 20260508 10:13:43.720 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:43.720 | INFO | "malloc" : " 14 GB", | 20260508 10:13:43.720 | INFO | "pinned" : " 29 GB" | 20260508 10:13:43.720 | INFO | } | 20260508 10:13:43.720 | INFO | Perf report for the last 5.14981 seconds: | 20260508 10:13:43.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:43.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:13:43.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:13:43.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:43.720 | INFO | Uploaded 36GiB to GPU in 2.37868s (15.1344GiB/s) | 20260508 10:13:43.720 | INFO | Downloaded 34GiB to GPU in 2.42802s (14.0032GiB/s) | 20260508 10:13:48.849 | INFO | Analyzing chunk frames = [7302144,7302656) | 20260508 10:13:48.849 | INFO | MemInfo: { | 20260508 10:13:48.849 | INFO | "gpu" : " 17 GB", | 20260508 10:13:48.849 | INFO | "huge" : "", | 20260508 10:13:48.849 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:48.849 | INFO | "malloc" : " 14 GB", | 20260508 10:13:48.849 | INFO | "pinned" : " 29 GB" | 20260508 10:13:48.849 | INFO | } | 20260508 10:13:48.849 | INFO | Perf report for the last 5.12936 seconds: | 20260508 10:13:48.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:48.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.849 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 10:13:48.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.849 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:13:48.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:48.850 | INFO | Uploaded 36GiB to GPU in 2.45739s (14.6497GiB/s) | 20260508 10:13:48.850 | INFO | Downloaded 34GiB to GPU in 2.44768s (13.8907GiB/s) | 20260508 10:13:53.988 | INFO | Analyzing chunk frames = [7302656,7303168) | 20260508 10:13:53.988 | INFO | MemInfo: { | 20260508 10:13:53.988 | INFO | "gpu" : " 17 GB", | 20260508 10:13:53.988 | INFO | "huge" : "", | 20260508 10:13:53.988 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:53.988 | INFO | "malloc" : " 14 GB", | 20260508 10:13:53.988 | INFO | "pinned" : " 29 GB" | 20260508 10:13:53.988 | INFO | } | 20260508 10:13:53.988 | INFO | Perf report for the last 5.1392 seconds: | 20260508 10:13:53.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:53.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.989 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 10:13:53.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.989 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:13:53.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:53.989 | INFO | Uploaded 36GiB to GPU in 2.43712s (14.7715GiB/s) | 20260508 10:13:53.989 | INFO | Downloaded 34GiB to GPU in 2.42844s (14.0008GiB/s) | 20260508 10:13:54.085 | INFO | Starting a round of Baz IO | 20260508 10:13:57.507 | INFO | Populating RTMetrics report | 20260508 10:13:57.508 | INFO | RTMetrics report populated | 20260508 10:13:57.508 | INFO | Opening tmp RTMetrics json file | 20260508 10:13:57.508 | INFO | Serializing RTMetrics to json | 20260508 10:13:57.509 | INFO | Serialization done, swapping tmp file now... | 20260508 10:13:57.510 | INFO | RTMetrics done | 20260508 10:13:57.574 | INFO | Finished a round of Baz IO: Wrote 1.465 GiB in 3.488 seconds | 20260508 10:13:57.574 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:13:57.574 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.228699% / 0% / 4.37101e-05% | 20260508 10:13:57.574 | INFO | Baz IO overall: | 20260508 10:13:57.574 | INFO | BytesWritten : 3173040130362 | 20260508 10:13:57.574 | INFO | NumEvents : 1525460582428 | 20260508 10:13:59.124 | INFO | Analyzing chunk frames = [7303168,7303680) | 20260508 10:13:59.124 | INFO | MemInfo: { | 20260508 10:13:59.124 | INFO | "gpu" : " 17 GB", | 20260508 10:13:59.124 | INFO | "huge" : "", | 20260508 10:13:59.124 | INFO | "huge_shared" : " 29 GB", | 20260508 10:13:59.124 | INFO | "malloc" : " 14 GB", | 20260508 10:13:59.124 | INFO | "pinned" : " 29 GB" | 20260508 10:13:59.124 | INFO | } | 20260508 10:13:59.124 | INFO | Perf report for the last 5.13589 seconds: | 20260508 10:13:59.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:13:59.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:59.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:59.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:59.124 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 10:13:59.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:13:59.125 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 10:13:59.125 | INFO | PRE_HQ : 65.8%, 1.6, 54.1ms, 0 | 20260508 10:13:59.125 | INFO | BAZWRITER : 67.2%, 2.5, 87.2ms, 0 | 20260508 10:13:59.125 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:13:59.125 | INFO | Uploaded 36GiB to GPU in 2.55631s (14.0828GiB/s) | 20260508 10:13:59.125 | INFO | Downloaded 38GiB to GPU in 2.71616s (13.9903GiB/s) | 20260508 10:14:04.268 | INFO | Analyzing chunk frames = [7303680,7304192) | 20260508 10:14:04.268 | INFO | MemInfo: { | 20260508 10:14:04.268 | INFO | "gpu" : " 17 GB", | 20260508 10:14:04.268 | INFO | "huge" : "", | 20260508 10:14:04.268 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:04.268 | INFO | "malloc" : " 14 GB", | 20260508 10:14:04.268 | INFO | "pinned" : " 29 GB" | 20260508 10:14:04.268 | INFO | } | 20260508 10:14:04.268 | INFO | Perf report for the last 5.14394 seconds: | 20260508 10:14:04.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:04.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.268 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 10:14:04.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.268 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:14:04.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:04.269 | INFO | Uploaded 36GiB to GPU in 2.48896s (14.4639GiB/s) | 20260508 10:14:04.269 | INFO | Downloaded 34GiB to GPU in 2.45211s (13.8656GiB/s) | 20260508 10:14:09.407 | INFO | Analyzing chunk frames = [7304192,7304704) | 20260508 10:14:09.407 | INFO | MemInfo: { | 20260508 10:14:09.407 | INFO | "gpu" : " 17 GB", | 20260508 10:14:09.407 | INFO | "huge" : "", | 20260508 10:14:09.407 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:09.407 | INFO | "malloc" : " 14 GB", | 20260508 10:14:09.407 | INFO | "pinned" : " 29 GB" | 20260508 10:14:09.407 | INFO | } | 20260508 10:14:09.407 | INFO | Perf report for the last 5.13906 seconds: | 20260508 10:14:09.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:09.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.407 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:14:09.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.407 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 10:14:09.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:09.408 | INFO | Uploaded 36GiB to GPU in 2.49091s (14.4525GiB/s) | 20260508 10:14:09.408 | INFO | Downloaded 34GiB to GPU in 2.41978s (14.0509GiB/s) | 20260508 10:14:14.550 | INFO | Analyzing chunk frames = [7304704,7305216) | 20260508 10:14:14.550 | INFO | MemInfo: { | 20260508 10:14:14.550 | INFO | "gpu" : " 17 GB", | 20260508 10:14:14.550 | INFO | "huge" : "", | 20260508 10:14:14.550 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:14.550 | INFO | "malloc" : " 14 GB", | 20260508 10:14:14.550 | INFO | "pinned" : " 29 GB" | 20260508 10:14:14.550 | INFO | } | 20260508 10:14:14.551 | INFO | Perf report for the last 5.14311 seconds: | 20260508 10:14:14.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:14.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 10:14:14.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:14:14.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:14.551 | INFO | Uploaded 36GiB to GPU in 2.5172s (14.3016GiB/s) | 20260508 10:14:14.551 | INFO | Downloaded 34GiB to GPU in 2.42121s (14.0426GiB/s) | 20260508 10:14:19.687 | INFO | Analyzing chunk frames = [7305216,7305728) | 20260508 10:14:19.687 | INFO | MemInfo: { | 20260508 10:14:19.687 | INFO | "gpu" : " 17 GB", | 20260508 10:14:19.687 | INFO | "huge" : "", | 20260508 10:14:19.687 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:19.687 | INFO | "malloc" : " 14 GB", | 20260508 10:14:19.687 | INFO | "pinned" : " 29 GB" | 20260508 10:14:19.687 | INFO | } | 20260508 10:14:19.687 | INFO | Perf report for the last 5.1365 seconds: | 20260508 10:14:19.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:19.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.687 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 10:14:19.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.687 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:14:19.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:19.688 | INFO | Uploaded 36GiB to GPU in 2.47699s (14.5337GiB/s) | 20260508 10:14:19.688 | INFO | Downloaded 34GiB to GPU in 2.54059s (13.3827GiB/s) | 20260508 10:14:24.826 | INFO | Analyzing chunk frames = [7305728,7306240) | 20260508 10:14:24.826 | INFO | MemInfo: { | 20260508 10:14:24.826 | INFO | "gpu" : " 17 GB", | 20260508 10:14:24.826 | INFO | "huge" : "", | 20260508 10:14:24.826 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:24.826 | INFO | "malloc" : " 14 GB", | 20260508 10:14:24.826 | INFO | "pinned" : " 29 GB" | 20260508 10:14:24.826 | INFO | } | 20260508 10:14:24.826 | INFO | Perf report for the last 5.13918 seconds: | 20260508 10:14:24.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:24.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.826 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:14:24.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.826 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:14:24.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:24.827 | INFO | Uploaded 36GiB to GPU in 2.38237s (15.111GiB/s) | 20260508 10:14:24.827 | INFO | Downloaded 34GiB to GPU in 2.43713s (13.9508GiB/s) | 20260508 10:14:29.964 | INFO | Analyzing chunk frames = [7306240,7306752) | 20260508 10:14:29.964 | INFO | MemInfo: { | 20260508 10:14:29.964 | INFO | "gpu" : " 17 GB", | 20260508 10:14:29.964 | INFO | "huge" : "", | 20260508 10:14:29.964 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:29.964 | INFO | "malloc" : " 14 GB", | 20260508 10:14:29.964 | INFO | "pinned" : " 29 GB" | 20260508 10:14:29.964 | INFO | } | 20260508 10:14:29.964 | INFO | Perf report for the last 5.13804 seconds: | 20260508 10:14:29.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:29.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.964 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:14:29.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.964 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:14:29.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:29.965 | INFO | Uploaded 36GiB to GPU in 2.42836s (14.8248GiB/s) | 20260508 10:14:29.965 | INFO | Downloaded 34GiB to GPU in 2.43711s (13.951GiB/s) | 20260508 10:14:35.106 | INFO | Analyzing chunk frames = [7306752,7307264) | 20260508 10:14:35.106 | INFO | MemInfo: { | 20260508 10:14:35.106 | INFO | "gpu" : " 17 GB", | 20260508 10:14:35.106 | INFO | "huge" : "", | 20260508 10:14:35.106 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:35.106 | INFO | "malloc" : " 14 GB", | 20260508 10:14:35.106 | INFO | "pinned" : " 29 GB" | 20260508 10:14:35.106 | INFO | } | 20260508 10:14:35.106 | INFO | Perf report for the last 5.14206 seconds: | 20260508 10:14:35.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:35.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:14:35.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 10:14:35.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:35.107 | INFO | Uploaded 36GiB to GPU in 2.42212s (14.863GiB/s) | 20260508 10:14:35.107 | INFO | Downloaded 34GiB to GPU in 2.43026s (13.9903GiB/s) | 20260508 10:14:35.107 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:14:35.107 | INFO | Total mem usage: 16.7 GiB | 20260508 10:14:35.107 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:14:35.107 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:14:35.107 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:14:35.107 | INFO | High Water Breakdown: | 20260508 10:14:35.107 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:14:35.107 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:14:35.107 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:14:35.107 | INFO | ------------------------------------------ | 20260508 10:14:35.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:14:35.107 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:14:35.107 | INFO | Total mem usage: 26.5 GiB | 20260508 10:14:35.107 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:14:35.107 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:14:35.107 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:14:35.107 | INFO | High Water Breakdown: | 20260508 10:14:35.107 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:14:35.107 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:14:35.107 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:14:35.107 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:14:35.107 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:14:35.107 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:14:35.107 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:14:35.107 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:14:35.107 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:14:35.107 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:14:35.107 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:14:35.107 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:14:35.107 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:14:35.107 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:14:35.107 | INFO | ------------------------------------------ | 20260508 10:14:35.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:14:35.107 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:14:35.107 | INFO | Total mem usage: 27.8 GiB | 20260508 10:14:35.108 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:14:35.108 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:14:35.108 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:14:35.108 | INFO | High Water Breakdown: | 20260508 10:14:35.108 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:14:35.108 | INFO | ------------------------------------------ | 20260508 10:14:35.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:14:35.108 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:14:35.108 | INFO | Total mem usage: 15.7 GiB | 20260508 10:14:35.108 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:14:35.108 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:14:35.108 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:14:35.108 | INFO | High Water Breakdown: | 20260508 10:14:35.108 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:14:35.108 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:14:35.108 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:14:35.108 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:14:35.108 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:14:35.108 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:14:35.108 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:14:35.108 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:14:35.108 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:14:35.108 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:14:35.108 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:14:35.108 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:14:35.108 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:14:35.108 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:14:35.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:14:35.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:14:35.108 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:14:35.108 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:14:35.108 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:14:35.108 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:14:35.108 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:14:35.108 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:14:35.108 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:14:35.108 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:14:35.108 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:14:35.108 | INFO | ------------------------------------------ | 20260508 10:14:35.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:14:35.108 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:14:35.108 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:14:35.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:14:35.108 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.41e+01 / 7.00e-01 | 20260508 10:14:35.108 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.05e-01, 1.08e-02 / 4.77e-03 | 20260508 10:14:35.108 | INFO | PerfReport: 12.17%, Baseline -- 3.99e+00 / 4.52e+00, 5.97e-01 / 1.66e-01 | 20260508 10:14:35.108 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.11e-01 / 1.12e-01 | 20260508 10:14:35.108 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.51e+01, 3.66e+00 / 7.70e-01 | 20260508 10:14:35.108 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.86e+00 / 1.08e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.58e+00, 4.04e-01 / 1.84e-01 | 20260508 10:14:35.108 | INFO | PerfReport: 3.01%, Metrics -- 9.87e-01 / 1.22e+00, 2.47e-01 / 1.28e-01 | 20260508 10:14:35.108 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.43e-01, 4.85e-02 / 2.62e-02 | 20260508 10:14:35.108 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:14:35.108 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:14:35.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:14:35.108 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 24.51%, Download -- 2.58e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 31.10%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 2.23%, LockUpload -- 2.35e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 2.53%, LockDownload -- 2.67e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 15.32%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:14:35.108 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.67e-02 / 0.00e+00 | 20260508 10:14:35.206 | INFO | Starting a round of Baz IO | 20260508 10:14:38.631 | INFO | Populating RTMetrics report | 20260508 10:14:38.632 | INFO | RTMetrics report populated | 20260508 10:14:38.632 | INFO | Opening tmp RTMetrics json file | 20260508 10:14:38.639 | INFO | Serializing RTMetrics to json | 20260508 10:14:38.640 | INFO | Serialization done, swapping tmp file now... | 20260508 10:14:38.640 | INFO | RTMetrics done | 20260508 10:14:38.696 | INFO | Finished a round of Baz IO: Wrote 1.463 GiB in 3.49 seconds | 20260508 10:14:38.696 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:14:38.696 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227221% / 3.97364e-06% / 3.97364e-05% | 20260508 10:14:38.696 | INFO | Baz IO overall: | 20260508 10:14:38.696 | INFO | BytesWritten : 3174611377184 | 20260508 10:14:38.696 | INFO | NumEvents : 1526202494911 | 20260508 10:14:40.243 | INFO | Analyzing chunk frames = [7307264,7307776) | 20260508 10:14:40.243 | INFO | MemInfo: { | 20260508 10:14:40.243 | INFO | "gpu" : " 17 GB", | 20260508 10:14:40.243 | INFO | "huge" : "", | 20260508 10:14:40.243 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:40.243 | INFO | "malloc" : " 14 GB", | 20260508 10:14:40.243 | INFO | "pinned" : " 29 GB" | 20260508 10:14:40.243 | INFO | } | 20260508 10:14:40.243 | INFO | Perf report for the last 5.13677 seconds: | 20260508 10:14:40.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:40.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:40.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:40.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:40.243 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 10:14:40.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:40.243 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:14:40.243 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 10:14:40.243 | INFO | BAZWRITER : 67.4%, 2.5, 85.8ms, 0 | 20260508 10:14:40.244 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:14:40.244 | INFO | Uploaded 36GiB to GPU in 2.5827s (13.9389GiB/s) | 20260508 10:14:40.244 | INFO | Downloaded 38GiB to GPU in 2.70352s (14.0557GiB/s) | 20260508 10:14:45.379 | INFO | Analyzing chunk frames = [7307776,7308288) | 20260508 10:14:45.379 | INFO | MemInfo: { | 20260508 10:14:45.379 | INFO | "gpu" : " 17 GB", | 20260508 10:14:45.379 | INFO | "huge" : "", | 20260508 10:14:45.379 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:45.379 | INFO | "malloc" : " 14 GB", | 20260508 10:14:45.379 | INFO | "pinned" : " 29 GB" | 20260508 10:14:45.379 | INFO | } | 20260508 10:14:45.379 | INFO | Perf report for the last 5.13589 seconds: | 20260508 10:14:45.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:45.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 10:14:45.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 10:14:45.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:45.379 | INFO | Uploaded 36GiB to GPU in 2.43676s (14.7737GiB/s) | 20260508 10:14:45.379 | INFO | Downloaded 34GiB to GPU in 2.47369s (13.7447GiB/s) | 20260508 10:14:50.522 | INFO | Analyzing chunk frames = [7308288,7308800) | 20260508 10:14:50.522 | INFO | MemInfo: { | 20260508 10:14:50.522 | INFO | "gpu" : " 17 GB", | 20260508 10:14:50.522 | INFO | "huge" : "", | 20260508 10:14:50.522 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:50.522 | INFO | "malloc" : " 14 GB", | 20260508 10:14:50.522 | INFO | "pinned" : " 29 GB" | 20260508 10:14:50.522 | INFO | } | 20260508 10:14:50.522 | INFO | Perf report for the last 5.14311 seconds: | 20260508 10:14:50.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:50.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 10:14:50.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:14:50.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:50.522 | INFO | Uploaded 36GiB to GPU in 2.43394s (14.7908GiB/s) | 20260508 10:14:50.523 | INFO | Downloaded 34GiB to GPU in 2.44246s (13.9204GiB/s) | 20260508 10:14:55.664 | INFO | Analyzing chunk frames = [7308800,7309312) | 20260508 10:14:55.664 | INFO | MemInfo: { | 20260508 10:14:55.664 | INFO | "gpu" : " 17 GB", | 20260508 10:14:55.664 | INFO | "huge" : "", | 20260508 10:14:55.664 | INFO | "huge_shared" : " 29 GB", | 20260508 10:14:55.664 | INFO | "malloc" : " 14 GB", | 20260508 10:14:55.664 | INFO | "pinned" : " 29 GB" | 20260508 10:14:55.664 | INFO | } | 20260508 10:14:55.664 | INFO | Perf report for the last 5.1418 seconds: | 20260508 10:14:55.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:14:55.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:14:55.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:14:55.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:14:55.664 | INFO | Uploaded 36GiB to GPU in 2.43643s (14.7757GiB/s) | 20260508 10:14:55.664 | INFO | Downloaded 34GiB to GPU in 2.43696s (13.9518GiB/s) | 20260508 10:15:00.803 | INFO | Analyzing chunk frames = [7309312,7309824) | 20260508 10:15:00.803 | INFO | MemInfo: { | 20260508 10:15:00.803 | INFO | "gpu" : " 17 GB", | 20260508 10:15:00.803 | INFO | "huge" : "", | 20260508 10:15:00.803 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:00.803 | INFO | "malloc" : " 14 GB", | 20260508 10:15:00.803 | INFO | "pinned" : " 29 GB" | 20260508 10:15:00.803 | INFO | } | 20260508 10:15:00.803 | INFO | Perf report for the last 5.139 seconds: | 20260508 10:15:00.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:00.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.803 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:15:00.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.803 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:15:00.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:00.804 | INFO | Uploaded 36GiB to GPU in 2.28378s (15.7633GiB/s) | 20260508 10:15:00.804 | INFO | Downloaded 34GiB to GPU in 2.50742s (13.5598GiB/s) | 20260508 10:15:05.934 | INFO | Analyzing chunk frames = [7309824,7310336) | 20260508 10:15:05.934 | INFO | MemInfo: { | 20260508 10:15:05.934 | INFO | "gpu" : " 17 GB", | 20260508 10:15:05.934 | INFO | "huge" : "", | 20260508 10:15:05.934 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:05.934 | INFO | "malloc" : " 14 GB", | 20260508 10:15:05.934 | INFO | "pinned" : " 29 GB" | 20260508 10:15:05.934 | INFO | } | 20260508 10:15:05.934 | INFO | Perf report for the last 5.13101 seconds: | 20260508 10:15:05.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:05.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 10:15:05.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:15:05.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:05.934 | INFO | Uploaded 36GiB to GPU in 2.38459s (15.0969GiB/s) | 20260508 10:15:05.934 | INFO | Downloaded 34GiB to GPU in 2.43249s (13.9774GiB/s) | 20260508 10:15:11.079 | INFO | Analyzing chunk frames = [7310336,7310848) | 20260508 10:15:11.079 | INFO | MemInfo: { | 20260508 10:15:11.079 | INFO | "gpu" : " 17 GB", | 20260508 10:15:11.079 | INFO | "huge" : "", | 20260508 10:15:11.079 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:11.079 | INFO | "malloc" : " 14 GB", | 20260508 10:15:11.079 | INFO | "pinned" : " 29 GB" | 20260508 10:15:11.079 | INFO | } | 20260508 10:15:11.079 | INFO | Perf report for the last 5.14491 seconds: | 20260508 10:15:11.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:11.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:15:11.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:15:11.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:11.079 | INFO | Uploaded 36GiB to GPU in 2.40777s (14.9516GiB/s) | 20260508 10:15:11.079 | INFO | Downloaded 34GiB to GPU in 2.44203s (13.9229GiB/s) | 20260508 10:15:16.218 | INFO | Analyzing chunk frames = [7310848,7311360) | 20260508 10:15:16.218 | INFO | MemInfo: { | 20260508 10:15:16.218 | INFO | "gpu" : " 17 GB", | 20260508 10:15:16.218 | INFO | "huge" : "", | 20260508 10:15:16.218 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:16.219 | INFO | "malloc" : " 14 GB", | 20260508 10:15:16.219 | INFO | "pinned" : " 29 GB" | 20260508 10:15:16.219 | INFO | } | 20260508 10:15:16.219 | INFO | Perf report for the last 5.13979 seconds: | 20260508 10:15:16.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:16.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:15:16.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:15:16.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:16.219 | INFO | Uploaded 36GiB to GPU in 2.42623s (14.8378GiB/s) | 20260508 10:15:16.219 | INFO | Downloaded 34GiB to GPU in 2.44507s (13.9056GiB/s) | 20260508 10:15:16.317 | INFO | Starting a round of Baz IO | 20260508 10:15:19.742 | INFO | Populating RTMetrics report | 20260508 10:15:19.742 | INFO | RTMetrics report populated | 20260508 10:15:19.742 | INFO | Opening tmp RTMetrics json file | 20260508 10:15:19.743 | INFO | Serializing RTMetrics to json | 20260508 10:15:19.744 | INFO | Serialization done, swapping tmp file now... | 20260508 10:15:19.745 | INFO | RTMetrics done | 20260508 10:15:19.804 | INFO | Finished a round of Baz IO: Wrote 1.463 GiB in 3.486 seconds | 20260508 10:15:19.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:15:19.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227388% / 3.97364e-06% / 5.5631e-05% | 20260508 10:15:19.804 | INFO | Baz IO overall: | 20260508 10:15:19.804 | INFO | BytesWritten : 3176182268201 | 20260508 10:15:19.804 | INFO | NumEvents : 1526944197360 | 20260508 10:15:21.361 | INFO | Analyzing chunk frames = [7311360,7311872) | 20260508 10:15:21.361 | INFO | MemInfo: { | 20260508 10:15:21.361 | INFO | "gpu" : " 17 GB", | 20260508 10:15:21.361 | INFO | "huge" : "", | 20260508 10:15:21.361 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:21.361 | INFO | "malloc" : " 14 GB", | 20260508 10:15:21.361 | INFO | "pinned" : " 29 GB" | 20260508 10:15:21.361 | INFO | } | 20260508 10:15:21.361 | INFO | Perf report for the last 5.14266 seconds: | 20260508 10:15:21.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:21.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:21.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:21.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:21.361 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 10:15:21.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:21.362 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 10:15:21.362 | INFO | PRE_HQ : 65.7%, 1.6, 54.4ms, 0 | 20260508 10:15:21.362 | INFO | BAZWRITER : 67.1%, 2.5, 85.8ms, 0 | 20260508 10:15:21.362 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:15:21.362 | INFO | Uploaded 36GiB to GPU in 2.53008s (14.2288GiB/s) | 20260508 10:15:21.362 | INFO | Downloaded 38GiB to GPU in 2.70972s (14.0236GiB/s) | 20260508 10:15:26.495 | INFO | Analyzing chunk frames = [7311872,7312384) | 20260508 10:15:26.495 | INFO | MemInfo: { | 20260508 10:15:26.495 | INFO | "gpu" : " 17 GB", | 20260508 10:15:26.495 | INFO | "huge" : "", | 20260508 10:15:26.495 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:26.495 | INFO | "malloc" : " 14 GB", | 20260508 10:15:26.495 | INFO | "pinned" : " 29 GB" | 20260508 10:15:26.495 | INFO | } | 20260508 10:15:26.495 | INFO | Perf report for the last 5.134 seconds: | 20260508 10:15:26.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:26.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.496 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:15:26.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.496 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:15:26.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:26.496 | INFO | Uploaded 36GiB to GPU in 2.47048s (14.5721GiB/s) | 20260508 10:15:26.496 | INFO | Downloaded 34GiB to GPU in 2.47665s (13.7282GiB/s) | 20260508 10:15:31.634 | INFO | Analyzing chunk frames = [7312384,7312896) | 20260508 10:15:31.634 | INFO | MemInfo: { | 20260508 10:15:31.634 | INFO | "gpu" : " 17 GB", | 20260508 10:15:31.634 | INFO | "huge" : "", | 20260508 10:15:31.634 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:31.634 | INFO | "malloc" : " 14 GB", | 20260508 10:15:31.635 | INFO | "pinned" : " 29 GB" | 20260508 10:15:31.635 | INFO | } | 20260508 10:15:31.635 | INFO | Perf report for the last 5.13914 seconds: | 20260508 10:15:31.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:31.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:15:31.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:15:31.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:31.635 | INFO | Uploaded 36GiB to GPU in 2.45621s (14.6567GiB/s) | 20260508 10:15:31.635 | INFO | Downloaded 34GiB to GPU in 2.41921s (14.0542GiB/s) | 20260508 10:15:36.778 | INFO | Analyzing chunk frames = [7312896,7313408) | 20260508 10:15:36.778 | INFO | MemInfo: { | 20260508 10:15:36.778 | INFO | "gpu" : " 17 GB", | 20260508 10:15:36.778 | INFO | "huge" : "", | 20260508 10:15:36.778 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:36.778 | INFO | "malloc" : " 14 GB", | 20260508 10:15:36.778 | INFO | "pinned" : " 29 GB" | 20260508 10:15:36.778 | INFO | } | 20260508 10:15:36.778 | INFO | Perf report for the last 5.14341 seconds: | 20260508 10:15:36.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:36.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.778 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 10:15:36.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.778 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:15:36.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:36.779 | INFO | Uploaded 36GiB to GPU in 2.45508s (14.6635GiB/s) | 20260508 10:15:36.779 | INFO | Downloaded 34GiB to GPU in 2.41449s (14.0816GiB/s) | 20260508 10:15:41.913 | INFO | Analyzing chunk frames = [7313408,7313920) | 20260508 10:15:41.913 | INFO | MemInfo: { | 20260508 10:15:41.913 | INFO | "gpu" : " 17 GB", | 20260508 10:15:41.913 | INFO | "huge" : "", | 20260508 10:15:41.913 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:41.913 | INFO | "malloc" : " 14 GB", | 20260508 10:15:41.913 | INFO | "pinned" : " 29 GB" | 20260508 10:15:41.913 | INFO | } | 20260508 10:15:41.913 | INFO | Perf report for the last 5.13485 seconds: | 20260508 10:15:41.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:41.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 10:15:41.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:15:41.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:41.913 | INFO | Uploaded 36GiB to GPU in 2.36126s (15.2461GiB/s) | 20260508 10:15:41.913 | INFO | Downloaded 34GiB to GPU in 2.53783s (13.3973GiB/s) | 20260508 10:15:47.055 | INFO | Analyzing chunk frames = [7313920,7314432) | 20260508 10:15:47.055 | INFO | MemInfo: { | 20260508 10:15:47.055 | INFO | "gpu" : " 17 GB", | 20260508 10:15:47.055 | INFO | "huge" : "", | 20260508 10:15:47.055 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:47.055 | INFO | "malloc" : " 14 GB", | 20260508 10:15:47.055 | INFO | "pinned" : " 29 GB" | 20260508 10:15:47.055 | INFO | } | 20260508 10:15:47.055 | INFO | Perf report for the last 5.14258 seconds: | 20260508 10:15:47.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:47.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.056 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:15:47.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.056 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:15:47.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:47.056 | INFO | Uploaded 36GiB to GPU in 2.37375s (15.1659GiB/s) | 20260508 10:15:47.056 | INFO | Downloaded 34GiB to GPU in 2.44453s (13.9086GiB/s) | 20260508 10:15:52.190 | INFO | Analyzing chunk frames = [7314432,7314944) | 20260508 10:15:52.190 | INFO | MemInfo: { | 20260508 10:15:52.190 | INFO | "gpu" : " 17 GB", | 20260508 10:15:52.190 | INFO | "huge" : "", | 20260508 10:15:52.190 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:52.190 | INFO | "malloc" : " 14 GB", | 20260508 10:15:52.190 | INFO | "pinned" : " 29 GB" | 20260508 10:15:52.190 | INFO | } | 20260508 10:15:52.190 | INFO | Perf report for the last 5.13438 seconds: | 20260508 10:15:52.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:52.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 10:15:52.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260508 10:15:52.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:52.190 | INFO | Uploaded 36GiB to GPU in 2.44326s (14.7344GiB/s) | 20260508 10:15:52.191 | INFO | Downloaded 34GiB to GPU in 2.46007s (13.8208GiB/s) | 20260508 10:15:57.334 | INFO | Analyzing chunk frames = [7314944,7315456) | 20260508 10:15:57.334 | INFO | MemInfo: { | 20260508 10:15:57.334 | INFO | "gpu" : " 17 GB", | 20260508 10:15:57.334 | INFO | "huge" : "", | 20260508 10:15:57.334 | INFO | "huge_shared" : " 29 GB", | 20260508 10:15:57.334 | INFO | "malloc" : " 14 GB", | 20260508 10:15:57.334 | INFO | "pinned" : " 29 GB" | 20260508 10:15:57.334 | INFO | } | 20260508 10:15:57.334 | INFO | Perf report for the last 5.14425 seconds: | 20260508 10:15:57.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:15:57.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:15:57.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:15:57.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:15:57.334 | INFO | Uploaded 36GiB to GPU in 2.40439s (14.9726GiB/s) | 20260508 10:15:57.334 | INFO | Downloaded 34GiB to GPU in 2.44631s (13.8985GiB/s) | 20260508 10:15:57.335 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:15:57.335 | INFO | Total mem usage: 16.7 GiB | 20260508 10:15:57.335 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:15:57.335 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:15:57.335 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:15:57.335 | INFO | High Water Breakdown: | 20260508 10:15:57.335 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:15:57.335 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:15:57.335 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:15:57.335 | INFO | ------------------------------------------ | 20260508 10:15:57.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:15:57.335 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:15:57.335 | INFO | Total mem usage: 26.5 GiB | 20260508 10:15:57.335 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:15:57.335 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:15:57.335 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:15:57.335 | INFO | High Water Breakdown: | 20260508 10:15:57.335 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:15:57.335 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:15:57.335 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:15:57.335 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:15:57.335 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:15:57.335 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:15:57.335 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:15:57.335 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:15:57.335 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:15:57.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:15:57.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:15:57.335 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:15:57.335 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:15:57.335 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:15:57.335 | INFO | ------------------------------------------ | 20260508 10:15:57.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:15:57.335 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:15:57.335 | INFO | Total mem usage: 27.8 GiB | 20260508 10:15:57.335 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:15:57.335 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:15:57.335 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:15:57.335 | INFO | High Water Breakdown: | 20260508 10:15:57.335 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:15:57.335 | INFO | ------------------------------------------ | 20260508 10:15:57.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:15:57.335 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:15:57.335 | INFO | Total mem usage: 15.7 GiB | 20260508 10:15:57.335 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:15:57.335 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:15:57.335 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:15:57.335 | INFO | High Water Breakdown: | 20260508 10:15:57.335 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:15:57.335 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:15:57.335 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:15:57.335 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:15:57.335 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:15:57.335 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:15:57.335 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:15:57.335 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:15:57.335 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:15:57.335 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:15:57.335 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:15:57.335 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:15:57.335 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:15:57.335 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:15:57.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:15:57.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:15:57.335 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:15:57.335 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:15:57.335 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:15:57.335 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:15:57.335 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:15:57.335 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:15:57.335 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:15:57.335 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:15:57.335 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:15:57.335 | INFO | ------------------------------------------ | 20260508 10:15:57.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:15:57.335 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:15:57.335 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:15:57.335 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:15:57.335 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.37e-01 | 20260508 10:15:57.335 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.06e-01, 1.22e-02 / 3.98e-03 | 20260508 10:15:57.335 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.44e+00, 6.00e-01 / 2.52e-01 | 20260508 10:15:57.335 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.12e-01 / 1.12e-01 | 20260508 10:15:57.335 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.57e+01, 3.75e+00 / 9.19e-01 | 20260508 10:15:57.335 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 1.95e+01, 1.82e+00 / 8.76e-01 | 20260508 10:15:57.335 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.58e+00, 4.09e-01 / 2.00e-01 | 20260508 10:15:57.335 | INFO | PerfReport: 3.02%, Metrics -- 9.90e-01 / 1.17e+00, 2.45e-01 / 9.74e-02 | 20260508 10:15:57.335 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.37e-01, 4.77e-02 / 2.07e-02 | 20260508 10:15:57.335 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:15:57.335 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:15:57.335 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:15:57.335 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 24.07%, Upload -- 2.53e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 24.63%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 2.10%, LockUpload -- 2.21e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 2.70%, LockDownload -- 2.84e+00 / 0.00e+00, 4.36e+00 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 15.37%, LockCompute -- 1.62e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:15:57.335 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260508 10:15:57.432 | INFO | Starting a round of Baz IO | 20260508 10:16:00.841 | INFO | Populating RTMetrics report | 20260508 10:16:00.842 | INFO | RTMetrics report populated | 20260508 10:16:00.842 | INFO | Opening tmp RTMetrics json file | 20260508 10:16:00.854 | INFO | Serializing RTMetrics to json | 20260508 10:16:00.856 | INFO | Serialization done, swapping tmp file now... | 20260508 10:16:00.856 | INFO | RTMetrics done | 20260508 10:16:00.913 | INFO | Finished a round of Baz IO: Wrote 1.462 GiB in 3.481 seconds | 20260508 10:16:00.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:16:00.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226255% / 0% / 5.96046e-05% | 20260508 10:16:00.913 | INFO | Baz IO overall: | 20260508 10:16:00.913 | INFO | BytesWritten : 3177751636076 | 20260508 10:16:00.913 | INFO | NumEvents : 1527684717769 | 20260508 10:16:02.479 | INFO | Analyzing chunk frames = [7315456,7315968) | 20260508 10:16:02.480 | INFO | MemInfo: { | 20260508 10:16:02.480 | INFO | "gpu" : " 17 GB", | 20260508 10:16:02.480 | INFO | "huge" : "", | 20260508 10:16:02.480 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:02.480 | INFO | "malloc" : " 14 GB", | 20260508 10:16:02.480 | INFO | "pinned" : " 29 GB" | 20260508 10:16:02.480 | INFO | } | 20260508 10:16:02.480 | INFO | Perf report for the last 5.14544 seconds: | 20260508 10:16:02.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:02.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:02.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:02.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:02.480 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:16:02.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:02.480 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 10:16:02.480 | INFO | PRE_HQ : 65.4%, 1.6, 54.9ms, 0 | 20260508 10:16:02.480 | INFO | BAZWRITER : 67.0%, 2.5, 86.7ms, 0 | 20260508 10:16:02.480 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:16:02.480 | INFO | Uploaded 36GiB to GPU in 2.59539s (13.8707GiB/s) | 20260508 10:16:02.480 | INFO | Downloaded 38GiB to GPU in 2.70217s (14.0628GiB/s) | 20260508 10:16:07.611 | INFO | Analyzing chunk frames = [7315968,7316480) | 20260508 10:16:07.611 | INFO | MemInfo: { | 20260508 10:16:07.611 | INFO | "gpu" : " 17 GB", | 20260508 10:16:07.611 | INFO | "huge" : "", | 20260508 10:16:07.611 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:07.611 | INFO | "malloc" : " 14 GB", | 20260508 10:16:07.611 | INFO | "pinned" : " 29 GB" | 20260508 10:16:07.611 | INFO | } | 20260508 10:16:07.611 | INFO | Perf report for the last 5.13159 seconds: | 20260508 10:16:07.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:07.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.611 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:16:07.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.611 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:16:07.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:07.612 | INFO | Uploaded 36GiB to GPU in 2.4868s (14.4764GiB/s) | 20260508 10:16:07.612 | INFO | Downloaded 34GiB to GPU in 2.4598s (13.8222GiB/s) | 20260508 10:16:12.751 | INFO | Analyzing chunk frames = [7316480,7316992) | 20260508 10:16:12.751 | INFO | MemInfo: { | 20260508 10:16:12.751 | INFO | "gpu" : " 17 GB", | 20260508 10:16:12.751 | INFO | "huge" : "", | 20260508 10:16:12.751 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:12.751 | INFO | "malloc" : " 14 GB", | 20260508 10:16:12.751 | INFO | "pinned" : " 29 GB" | 20260508 10:16:12.751 | INFO | } | 20260508 10:16:12.751 | INFO | Perf report for the last 5.13989 seconds: | 20260508 10:16:12.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:12.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 10:16:12.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:16:12.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:12.751 | INFO | Uploaded 36GiB to GPU in 2.49536s (14.4268GiB/s) | 20260508 10:16:12.751 | INFO | Downloaded 34GiB to GPU in 2.43072s (13.9876GiB/s) | 20260508 10:16:17.888 | INFO | Analyzing chunk frames = [7316992,7317504) | 20260508 10:16:17.888 | INFO | MemInfo: { | 20260508 10:16:17.888 | INFO | "gpu" : " 17 GB", | 20260508 10:16:17.888 | INFO | "huge" : "", | 20260508 10:16:17.888 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:17.888 | INFO | "malloc" : " 14 GB", | 20260508 10:16:17.888 | INFO | "pinned" : " 29 GB" | 20260508 10:16:17.888 | INFO | } | 20260508 10:16:17.888 | INFO | Perf report for the last 5.13689 seconds: | 20260508 10:16:17.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:17.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:16:17.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:16:17.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:17.888 | INFO | Uploaded 36GiB to GPU in 2.4784s (14.5255GiB/s) | 20260508 10:16:17.888 | INFO | Downloaded 34GiB to GPU in 2.39698s (14.1845GiB/s) | 20260508 10:16:23.026 | INFO | Analyzing chunk frames = [7317504,7318016) | 20260508 10:16:23.026 | INFO | MemInfo: { | 20260508 10:16:23.026 | INFO | "gpu" : " 17 GB", | 20260508 10:16:23.026 | INFO | "huge" : "", | 20260508 10:16:23.026 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:23.026 | INFO | "malloc" : " 14 GB", | 20260508 10:16:23.026 | INFO | "pinned" : " 29 GB" | 20260508 10:16:23.026 | INFO | } | 20260508 10:16:23.026 | INFO | Perf report for the last 5.13805 seconds: | 20260508 10:16:23.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:23.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.026 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:16:23.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.026 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 10:16:23.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:23.027 | INFO | Uploaded 36GiB to GPU in 2.41254s (14.922GiB/s) | 20260508 10:16:23.027 | INFO | Downloaded 34GiB to GPU in 2.52057s (13.489GiB/s) | 20260508 10:16:28.173 | INFO | Analyzing chunk frames = [7318016,7318528) | 20260508 10:16:28.173 | INFO | MemInfo: { | 20260508 10:16:28.173 | INFO | "gpu" : " 17 GB", | 20260508 10:16:28.173 | INFO | "huge" : "", | 20260508 10:16:28.173 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:28.173 | INFO | "malloc" : " 14 GB", | 20260508 10:16:28.173 | INFO | "pinned" : " 29 GB" | 20260508 10:16:28.173 | INFO | } | 20260508 10:16:28.173 | INFO | Perf report for the last 5.14674 seconds: | 20260508 10:16:28.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:28.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:16:28.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 10:16:28.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:28.173 | INFO | Uploaded 36GiB to GPU in 2.37422s (15.1629GiB/s) | 20260508 10:16:28.173 | INFO | Downloaded 34GiB to GPU in 2.42661s (14.0113GiB/s) | 20260508 10:16:33.311 | INFO | Analyzing chunk frames = [7318528,7319040) | 20260508 10:16:33.311 | INFO | MemInfo: { | 20260508 10:16:33.311 | INFO | "gpu" : " 17 GB", | 20260508 10:16:33.311 | INFO | "huge" : "", | 20260508 10:16:33.311 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:33.311 | INFO | "malloc" : " 14 GB", | 20260508 10:16:33.311 | INFO | "pinned" : " 29 GB" | 20260508 10:16:33.311 | INFO | } | 20260508 10:16:33.311 | INFO | Perf report for the last 5.1383 seconds: | 20260508 10:16:33.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:33.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:16:33.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:16:33.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:33.311 | INFO | Uploaded 36GiB to GPU in 2.43923s (14.7587GiB/s) | 20260508 10:16:33.311 | INFO | Downloaded 34GiB to GPU in 2.44793s (13.8893GiB/s) | 20260508 10:16:38.445 | INFO | Analyzing chunk frames = [7319040,7319552) | 20260508 10:16:38.445 | INFO | MemInfo: { | 20260508 10:16:38.445 | INFO | "gpu" : " 17 GB", | 20260508 10:16:38.445 | INFO | "huge" : "", | 20260508 10:16:38.445 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:38.445 | INFO | "malloc" : " 14 GB", | 20260508 10:16:38.445 | INFO | "pinned" : " 29 GB" | 20260508 10:16:38.445 | INFO | } | 20260508 10:16:38.445 | INFO | Perf report for the last 5.13373 seconds: | 20260508 10:16:38.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:38.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 10:16:38.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:16:38.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:38.445 | INFO | Uploaded 36GiB to GPU in 2.40335s (14.9791GiB/s) | 20260508 10:16:38.445 | INFO | Downloaded 34GiB to GPU in 2.43276s (13.9759GiB/s) | 20260508 10:16:38.551 | INFO | Starting a round of Baz IO | 20260508 10:16:41.959 | INFO | Populating RTMetrics report | 20260508 10:16:41.959 | INFO | RTMetrics report populated | 20260508 10:16:41.959 | INFO | Opening tmp RTMetrics json file | 20260508 10:16:41.966 | INFO | Serializing RTMetrics to json | 20260508 10:16:41.968 | INFO | Serialization done, swapping tmp file now... | 20260508 10:16:41.968 | INFO | RTMetrics done | 20260508 10:16:42.024 | INFO | Finished a round of Baz IO: Wrote 1.462 GiB in 3.472 seconds | 20260508 10:16:42.024 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:16:42.024 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226506% / 0% / 6.75519e-05% | 20260508 10:16:42.024 | INFO | Baz IO overall: | 20260508 10:16:42.024 | INFO | BytesWritten : 3179321133856 | 20260508 10:16:42.024 | INFO | NumEvents : 1528425211203 | 20260508 10:16:43.587 | INFO | Analyzing chunk frames = [7319552,7320064) | 20260508 10:16:43.587 | INFO | MemInfo: { | 20260508 10:16:43.587 | INFO | "gpu" : " 17 GB", | 20260508 10:16:43.587 | INFO | "huge" : "", | 20260508 10:16:43.587 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:43.587 | INFO | "malloc" : " 14 GB", | 20260508 10:16:43.587 | INFO | "pinned" : " 29 GB" | 20260508 10:16:43.587 | INFO | } | 20260508 10:16:43.587 | INFO | Perf report for the last 5.14242 seconds: | 20260508 10:16:43.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:43.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:43.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:43.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:43.587 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:16:43.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:43.587 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:16:43.587 | INFO | PRE_HQ : 65.2%, 1.6, 54.5ms, 0 | 20260508 10:16:43.587 | INFO | BAZWRITER : 67.0%, 2.5, 86.1ms, 0 | 20260508 10:16:43.587 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:16:43.588 | INFO | Uploaded 36GiB to GPU in 2.58023s (13.9522GiB/s) | 20260508 10:16:43.588 | INFO | Downloaded 38GiB to GPU in 2.708s (14.0325GiB/s) | 20260508 10:16:48.728 | INFO | Analyzing chunk frames = [7320064,7320576) | 20260508 10:16:48.728 | INFO | MemInfo: { | 20260508 10:16:48.728 | INFO | "gpu" : " 17 GB", | 20260508 10:16:48.728 | INFO | "huge" : "", | 20260508 10:16:48.728 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:48.728 | INFO | "malloc" : " 14 GB", | 20260508 10:16:48.728 | INFO | "pinned" : " 29 GB" | 20260508 10:16:48.728 | INFO | } | 20260508 10:16:48.728 | INFO | Perf report for the last 5.14087 seconds: | 20260508 10:16:48.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:48.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.728 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:16:48.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.728 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:16:48.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:48.729 | INFO | Uploaded 36GiB to GPU in 2.45173s (14.6835GiB/s) | 20260508 10:16:48.729 | INFO | Downloaded 34GiB to GPU in 2.45825s (13.831GiB/s) | 20260508 10:16:53.864 | INFO | Analyzing chunk frames = [7320576,7321088) | 20260508 10:16:53.864 | INFO | MemInfo: { | 20260508 10:16:53.864 | INFO | "gpu" : " 17 GB", | 20260508 10:16:53.864 | INFO | "huge" : "", | 20260508 10:16:53.864 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:53.865 | INFO | "malloc" : " 14 GB", | 20260508 10:16:53.865 | INFO | "pinned" : " 29 GB" | 20260508 10:16:53.865 | INFO | } | 20260508 10:16:53.865 | INFO | Perf report for the last 5.13642 seconds: | 20260508 10:16:53.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:53.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:16:53.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:16:53.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:53.865 | INFO | Uploaded 36GiB to GPU in 2.45238s (14.6796GiB/s) | 20260508 10:16:53.865 | INFO | Downloaded 34GiB to GPU in 2.41334s (14.0883GiB/s) | 20260508 10:16:59.003 | INFO | Analyzing chunk frames = [7321088,7321600) | 20260508 10:16:59.003 | INFO | MemInfo: { | 20260508 10:16:59.003 | INFO | "gpu" : " 17 GB", | 20260508 10:16:59.003 | INFO | "huge" : "", | 20260508 10:16:59.003 | INFO | "huge_shared" : " 29 GB", | 20260508 10:16:59.003 | INFO | "malloc" : " 14 GB", | 20260508 10:16:59.003 | INFO | "pinned" : " 29 GB" | 20260508 10:16:59.003 | INFO | } | 20260508 10:16:59.003 | INFO | Perf report for the last 5.13829 seconds: | 20260508 10:16:59.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:16:59.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:16:59.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:16:59.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:16:59.003 | INFO | Uploaded 36GiB to GPU in 2.47668s (14.5356GiB/s) | 20260508 10:16:59.003 | INFO | Downloaded 34GiB to GPU in 2.41802s (14.0611GiB/s) | 20260508 10:17:04.146 | INFO | Analyzing chunk frames = [7321600,7322112) | 20260508 10:17:04.146 | INFO | MemInfo: { | 20260508 10:17:04.146 | INFO | "gpu" : " 17 GB", | 20260508 10:17:04.146 | INFO | "huge" : "", | 20260508 10:17:04.146 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:04.146 | INFO | "malloc" : " 14 GB", | 20260508 10:17:04.146 | INFO | "pinned" : " 29 GB" | 20260508 10:17:04.146 | INFO | } | 20260508 10:17:04.146 | INFO | Perf report for the last 5.14302 seconds: | 20260508 10:17:04.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:04.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.146 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 10:17:04.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.146 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:17:04.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:04.147 | INFO | Uploaded 36GiB to GPU in 2.41285s (14.9201GiB/s) | 20260508 10:17:04.147 | INFO | Downloaded 34GiB to GPU in 2.4916s (13.6459GiB/s) | 20260508 10:17:09.287 | INFO | Analyzing chunk frames = [7322112,7322624) | 20260508 10:17:09.287 | INFO | MemInfo: { | 20260508 10:17:09.287 | INFO | "gpu" : " 17 GB", | 20260508 10:17:09.287 | INFO | "huge" : "", | 20260508 10:17:09.287 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:09.287 | INFO | "malloc" : " 14 GB", | 20260508 10:17:09.287 | INFO | "pinned" : " 29 GB" | 20260508 10:17:09.287 | INFO | } | 20260508 10:17:09.287 | INFO | Perf report for the last 5.14142 seconds: | 20260508 10:17:09.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:09.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.287 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 10:17:09.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.287 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:17:09.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:09.288 | INFO | Uploaded 36GiB to GPU in 2.40077s (14.9952GiB/s) | 20260508 10:17:09.288 | INFO | Downloaded 34GiB to GPU in 2.42092s (14.0442GiB/s) | 20260508 10:17:14.431 | INFO | Analyzing chunk frames = [7322624,7323136) | 20260508 10:17:14.431 | INFO | MemInfo: { | 20260508 10:17:14.431 | INFO | "gpu" : " 17 GB", | 20260508 10:17:14.431 | INFO | "huge" : "", | 20260508 10:17:14.431 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:14.431 | INFO | "malloc" : " 14 GB", | 20260508 10:17:14.431 | INFO | "pinned" : " 29 GB" | 20260508 10:17:14.431 | INFO | } | 20260508 10:17:14.431 | INFO | Perf report for the last 5.14359 seconds: | 20260508 10:17:14.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:14.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:17:14.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:17:14.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:14.431 | INFO | Uploaded 36GiB to GPU in 2.44851s (14.7028GiB/s) | 20260508 10:17:14.431 | INFO | Downloaded 34GiB to GPU in 2.43236s (13.9782GiB/s) | 20260508 10:17:19.563 | INFO | Analyzing chunk frames = [7323136,7323648) | 20260508 10:17:19.563 | INFO | MemInfo: { | 20260508 10:17:19.563 | INFO | "gpu" : " 17 GB", | 20260508 10:17:19.563 | INFO | "huge" : "", | 20260508 10:17:19.563 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:19.563 | INFO | "malloc" : " 14 GB", | 20260508 10:17:19.563 | INFO | "pinned" : " 29 GB" | 20260508 10:17:19.563 | INFO | } | 20260508 10:17:19.563 | INFO | Perf report for the last 5.13191 seconds: | 20260508 10:17:19.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:19.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 10:17:19.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 10:17:19.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:19.563 | INFO | Uploaded 36GiB to GPU in 2.48002s (14.516GiB/s) | 20260508 10:17:19.563 | INFO | Downloaded 34GiB to GPU in 2.44551s (13.903GiB/s) | 20260508 10:17:19.563 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:17:19.563 | INFO | Total mem usage: 16.7 GiB | 20260508 10:17:19.563 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:17:19.563 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:17:19.563 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:17:19.563 | INFO | High Water Breakdown: | 20260508 10:17:19.563 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:17:19.563 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:17:19.563 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:17:19.563 | INFO | ------------------------------------------ | 20260508 10:17:19.563 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:17:19.563 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:17:19.563 | INFO | Total mem usage: 26.5 GiB | 20260508 10:17:19.563 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:17:19.563 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:17:19.563 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:17:19.563 | INFO | High Water Breakdown: | 20260508 10:17:19.563 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:17:19.563 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:17:19.563 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:17:19.563 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:17:19.563 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:17:19.563 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:17:19.564 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:17:19.564 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:17:19.564 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:17:19.564 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:17:19.564 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:17:19.564 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:17:19.564 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:17:19.564 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:17:19.564 | INFO | ------------------------------------------ | 20260508 10:17:19.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:17:19.564 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:17:19.564 | INFO | Total mem usage: 27.8 GiB | 20260508 10:17:19.564 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:17:19.564 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:17:19.564 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:17:19.564 | INFO | High Water Breakdown: | 20260508 10:17:19.564 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:17:19.564 | INFO | ------------------------------------------ | 20260508 10:17:19.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:17:19.564 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:17:19.564 | INFO | Total mem usage: 15.7 GiB | 20260508 10:17:19.564 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:17:19.564 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:17:19.564 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:17:19.564 | INFO | High Water Breakdown: | 20260508 10:17:19.564 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:17:19.564 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:17:19.564 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:17:19.564 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:17:19.564 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:17:19.564 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:17:19.564 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:17:19.564 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:17:19.564 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:17:19.564 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:17:19.564 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:17:19.564 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:17:19.564 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:17:19.564 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:17:19.564 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:17:19.564 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:17:19.564 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:17:19.564 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:17:19.564 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:17:19.564 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:17:19.564 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:17:19.564 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:17:19.564 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:17:19.564 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:17:19.564 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:17:19.564 | INFO | ------------------------------------------ | 20260508 10:17:19.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:17:19.564 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:17:19.564 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:17:19.564 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:17:19.564 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 7.24e-01 | 20260508 10:17:19.564 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.07e-02 / 5.01e-03 | 20260508 10:17:19.564 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.45e+00, 5.93e-01 / 2.37e-01 | 20260508 10:17:19.564 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.12e-01 / 1.10e-01 | 20260508 10:17:19.564 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.52e+01, 3.66e+00 / 1.09e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.79e+00 / 1.32e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.07e-01 / 2.91e-01 | 20260508 10:17:19.564 | INFO | PerfReport: 3.02%, Metrics -- 9.88e-01 / 1.12e+00, 2.44e-01 / 1.09e-01 | 20260508 10:17:19.564 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.35e-01, 4.81e-02 / 3.13e-02 | 20260508 10:17:19.564 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:17:19.564 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:17:19.564 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:17:19.564 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 24.36%, Upload -- 2.56e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 24.49%, Download -- 2.58e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.43e+01 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 2.26%, LockUpload -- 2.38e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 2.45%, LockDownload -- 2.58e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:17:19.564 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260508 10:17:19.661 | INFO | Starting a round of Baz IO | 20260508 10:17:23.067 | INFO | Populating RTMetrics report | 20260508 10:17:23.067 | INFO | RTMetrics report populated | 20260508 10:17:23.067 | INFO | Opening tmp RTMetrics json file | 20260508 10:17:23.077 | INFO | Serializing RTMetrics to json | 20260508 10:17:23.078 | INFO | Serialization done, swapping tmp file now... | 20260508 10:17:23.079 | INFO | RTMetrics done | 20260508 10:17:23.138 | INFO | Finished a round of Baz IO: Wrote 1.462 GiB in 3.477 seconds | 20260508 10:17:23.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.00% / 59.78% | 20260508 10:17:23.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226061% / 0% / 7.54992e-05% | 20260508 10:17:23.138 | INFO | Baz IO overall: | 20260508 10:17:23.138 | INFO | BytesWritten : 3180890912792 | 20260508 10:17:23.138 | INFO | NumEvents : 1529166091486 | 20260508 10:17:24.696 | INFO | Analyzing chunk frames = [7323648,7324160) | 20260508 10:17:24.696 | INFO | MemInfo: { | 20260508 10:17:24.696 | INFO | "gpu" : " 17 GB", | 20260508 10:17:24.696 | INFO | "huge" : "", | 20260508 10:17:24.696 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:24.696 | INFO | "malloc" : " 14 GB", | 20260508 10:17:24.696 | INFO | "pinned" : " 29 GB" | 20260508 10:17:24.696 | INFO | } | 20260508 10:17:24.696 | INFO | Perf report for the last 5.13284 seconds: | 20260508 10:17:24.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:24.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:24.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:24.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:24.696 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 10:17:24.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:24.696 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:17:24.696 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 10:17:24.696 | INFO | BAZWRITER : 66.9%, 2.5, 87.0ms, 0 | 20260508 10:17:24.696 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:17:24.696 | INFO | Uploaded 36GiB to GPU in 2.62102s (13.7351GiB/s) | 20260508 10:17:24.696 | INFO | Downloaded 38GiB to GPU in 2.70283s (14.0593GiB/s) | 20260508 10:17:29.840 | INFO | Analyzing chunk frames = [7324160,7324672) | 20260508 10:17:29.840 | INFO | MemInfo: { | 20260508 10:17:29.840 | INFO | "gpu" : " 17 GB", | 20260508 10:17:29.840 | INFO | "huge" : "", | 20260508 10:17:29.840 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:29.840 | INFO | "malloc" : " 14 GB", | 20260508 10:17:29.841 | INFO | "pinned" : " 29 GB" | 20260508 10:17:29.841 | INFO | } | 20260508 10:17:29.841 | INFO | Perf report for the last 5.14483 seconds: | 20260508 10:17:29.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:29.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 10:17:29.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:17:29.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:29.841 | INFO | Uploaded 36GiB to GPU in 2.519s (14.2914GiB/s) | 20260508 10:17:29.841 | INFO | Downloaded 34GiB to GPU in 2.4571s (13.8375GiB/s) | 20260508 10:17:34.985 | INFO | Analyzing chunk frames = [7324672,7325184) | 20260508 10:17:34.985 | INFO | MemInfo: { | 20260508 10:17:34.985 | INFO | "gpu" : " 17 GB", | 20260508 10:17:34.985 | INFO | "huge" : "", | 20260508 10:17:34.985 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:34.985 | INFO | "malloc" : " 14 GB", | 20260508 10:17:34.985 | INFO | "pinned" : " 29 GB" | 20260508 10:17:34.985 | INFO | } | 20260508 10:17:34.986 | INFO | Perf report for the last 5.14492 seconds: | 20260508 10:17:34.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:34.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:17:34.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260508 10:17:34.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:34.986 | INFO | Uploaded 36GiB to GPU in 2.46815s (14.5858GiB/s) | 20260508 10:17:34.986 | INFO | Downloaded 34GiB to GPU in 2.4314s (13.9837GiB/s) | 20260508 10:17:40.122 | INFO | Analyzing chunk frames = [7325184,7325696) | 20260508 10:17:40.122 | INFO | MemInfo: { | 20260508 10:17:40.122 | INFO | "gpu" : " 17 GB", | 20260508 10:17:40.122 | INFO | "huge" : "", | 20260508 10:17:40.122 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:40.122 | INFO | "malloc" : " 14 GB", | 20260508 10:17:40.122 | INFO | "pinned" : " 29 GB" | 20260508 10:17:40.122 | INFO | } | 20260508 10:17:40.122 | INFO | Perf report for the last 5.13679 seconds: | 20260508 10:17:40.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:40.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.122 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:17:40.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.123 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 10:17:40.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:40.123 | INFO | Uploaded 36GiB to GPU in 2.46532s (14.6025GiB/s) | 20260508 10:17:40.123 | INFO | Downloaded 34GiB to GPU in 2.46006s (13.8208GiB/s) | 20260508 10:17:45.258 | INFO | Analyzing chunk frames = [7325696,7326208) | 20260508 10:17:45.258 | INFO | MemInfo: { | 20260508 10:17:45.258 | INFO | "gpu" : " 17 GB", | 20260508 10:17:45.258 | INFO | "huge" : "", | 20260508 10:17:45.258 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:45.258 | INFO | "malloc" : " 14 GB", | 20260508 10:17:45.258 | INFO | "pinned" : " 29 GB" | 20260508 10:17:45.258 | INFO | } | 20260508 10:17:45.258 | INFO | Perf report for the last 5.13601 seconds: | 20260508 10:17:45.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:45.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.258 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 10:17:45.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.258 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260508 10:17:45.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:45.259 | INFO | Uploaded 36GiB to GPU in 2.4025s (14.9844GiB/s) | 20260508 10:17:45.259 | INFO | Downloaded 34GiB to GPU in 2.5153s (13.5173GiB/s) | 20260508 10:17:50.397 | INFO | Analyzing chunk frames = [7326208,7326720) | 20260508 10:17:50.397 | INFO | MemInfo: { | 20260508 10:17:50.397 | INFO | "gpu" : " 17 GB", | 20260508 10:17:50.397 | INFO | "huge" : "", | 20260508 10:17:50.397 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:50.397 | INFO | "malloc" : " 14 GB", | 20260508 10:17:50.398 | INFO | "pinned" : " 29 GB" | 20260508 10:17:50.398 | INFO | } | 20260508 10:17:50.398 | INFO | Perf report for the last 5.13928 seconds: | 20260508 10:17:50.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:50.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:17:50.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 10:17:50.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:50.398 | INFO | Uploaded 36GiB to GPU in 2.34039s (15.3821GiB/s) | 20260508 10:17:50.398 | INFO | Downloaded 34GiB to GPU in 2.45155s (13.8688GiB/s) | 20260508 10:17:55.536 | INFO | Analyzing chunk frames = [7326720,7327232) | 20260508 10:17:55.536 | INFO | MemInfo: { | 20260508 10:17:55.536 | INFO | "gpu" : " 17 GB", | 20260508 10:17:55.536 | INFO | "huge" : "", | 20260508 10:17:55.536 | INFO | "huge_shared" : " 29 GB", | 20260508 10:17:55.536 | INFO | "malloc" : " 14 GB", | 20260508 10:17:55.536 | INFO | "pinned" : " 29 GB" | 20260508 10:17:55.536 | INFO | } | 20260508 10:17:55.536 | INFO | Perf report for the last 5.13862 seconds: | 20260508 10:17:55.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:17:55.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.537 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 10:17:55.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.537 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:17:55.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:17:55.537 | INFO | Uploaded 36GiB to GPU in 2.4019s (14.9881GiB/s) | 20260508 10:17:55.537 | INFO | Downloaded 34GiB to GPU in 2.46037s (13.819GiB/s) | 20260508 10:18:00.676 | INFO | Analyzing chunk frames = [7327232,7327744) | 20260508 10:18:00.676 | INFO | MemInfo: { | 20260508 10:18:00.676 | INFO | "gpu" : " 17 GB", | 20260508 10:18:00.676 | INFO | "huge" : "", | 20260508 10:18:00.676 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:00.676 | INFO | "malloc" : " 14 GB", | 20260508 10:18:00.676 | INFO | "pinned" : " 29 GB" | 20260508 10:18:00.676 | INFO | } | 20260508 10:18:00.676 | INFO | Perf report for the last 5.13998 seconds: | 20260508 10:18:00.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:00.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.676 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 10:18:00.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.676 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:18:00.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:00.677 | INFO | Uploaded 36GiB to GPU in 2.42324s (14.8562GiB/s) | 20260508 10:18:00.677 | INFO | Downloaded 34GiB to GPU in 2.4674s (13.7797GiB/s) | 20260508 10:18:00.776 | INFO | Starting a round of Baz IO | 20260508 10:18:04.179 | INFO | Populating RTMetrics report | 20260508 10:18:04.179 | INFO | RTMetrics report populated | 20260508 10:18:04.179 | INFO | Opening tmp RTMetrics json file | 20260508 10:18:04.188 | INFO | Serializing RTMetrics to json | 20260508 10:18:04.190 | INFO | Serialization done, swapping tmp file now... | 20260508 10:18:04.190 | INFO | RTMetrics done | 20260508 10:18:04.249 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.473 seconds | 20260508 10:18:04.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.78% | 20260508 10:18:04.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225401% / 3.97364e-06% / 5.16574e-05% | 20260508 10:18:04.249 | INFO | Baz IO overall: | 20260508 10:18:04.249 | INFO | BytesWritten : 3182459484478 | 20260508 10:18:04.249 | INFO | NumEvents : 1529905989867 | 20260508 10:18:05.819 | INFO | Analyzing chunk frames = [7327744,7328256) | 20260508 10:18:05.819 | INFO | MemInfo: { | 20260508 10:18:05.819 | INFO | "gpu" : " 17 GB", | 20260508 10:18:05.819 | INFO | "huge" : "", | 20260508 10:18:05.819 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:05.819 | INFO | "malloc" : " 14 GB", | 20260508 10:18:05.819 | INFO | "pinned" : " 29 GB" | 20260508 10:18:05.819 | INFO | } | 20260508 10:18:05.819 | INFO | Perf report for the last 5.14278 seconds: | 20260508 10:18:05.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:05.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:05.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:05.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:05.819 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 10:18:05.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:05.819 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:18:05.819 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 10:18:05.819 | INFO | BAZWRITER : 66.9%, 2.5, 86.3ms, 0 | 20260508 10:18:05.819 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:18:05.819 | INFO | Uploaded 36GiB to GPU in 2.57989s (13.9541GiB/s) | 20260508 10:18:05.819 | INFO | Downloaded 38GiB to GPU in 2.7162s (13.9901GiB/s) | 20260508 10:18:10.953 | INFO | Analyzing chunk frames = [7328256,7328768) | 20260508 10:18:10.953 | INFO | MemInfo: { | 20260508 10:18:10.953 | INFO | "gpu" : " 17 GB", | 20260508 10:18:10.953 | INFO | "huge" : "", | 20260508 10:18:10.953 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:10.953 | INFO | "malloc" : " 14 GB", | 20260508 10:18:10.953 | INFO | "pinned" : " 29 GB" | 20260508 10:18:10.953 | INFO | } | 20260508 10:18:10.953 | INFO | Perf report for the last 5.13393 seconds: | 20260508 10:18:10.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:10.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 10:18:10.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:18:10.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:10.953 | INFO | Uploaded 36GiB to GPU in 2.50501s (14.3712GiB/s) | 20260508 10:18:10.953 | INFO | Downloaded 34GiB to GPU in 2.48172s (13.7002GiB/s) | 20260508 10:18:16.098 | INFO | Analyzing chunk frames = [7328768,7329280) | 20260508 10:18:16.098 | INFO | MemInfo: { | 20260508 10:18:16.098 | INFO | "gpu" : " 17 GB", | 20260508 10:18:16.098 | INFO | "huge" : "", | 20260508 10:18:16.098 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:16.098 | INFO | "malloc" : " 14 GB", | 20260508 10:18:16.099 | INFO | "pinned" : " 29 GB" | 20260508 10:18:16.099 | INFO | } | 20260508 10:18:16.099 | INFO | Perf report for the last 5.14556 seconds: | 20260508 10:18:16.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:16.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 10:18:16.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:18:16.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:16.099 | INFO | Uploaded 36GiB to GPU in 2.43153s (14.8055GiB/s) | 20260508 10:18:16.099 | INFO | Downloaded 34GiB to GPU in 2.44787s (13.8896GiB/s) | 20260508 10:18:21.239 | INFO | Analyzing chunk frames = [7329280,7329792) | 20260508 10:18:21.239 | INFO | MemInfo: { | 20260508 10:18:21.239 | INFO | "gpu" : " 17 GB", | 20260508 10:18:21.239 | INFO | "huge" : "", | 20260508 10:18:21.239 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:21.239 | INFO | "malloc" : " 14 GB", | 20260508 10:18:21.239 | INFO | "pinned" : " 29 GB" | 20260508 10:18:21.239 | INFO | } | 20260508 10:18:21.239 | INFO | Perf report for the last 5.1403 seconds: | 20260508 10:18:21.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:21.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:18:21.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260508 10:18:21.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:21.239 | INFO | Uploaded 36GiB to GPU in 2.44874s (14.7014GiB/s) | 20260508 10:18:21.239 | INFO | Downloaded 34GiB to GPU in 2.46117s (13.8146GiB/s) | 20260508 10:18:26.378 | INFO | Analyzing chunk frames = [7329792,7330304) | 20260508 10:18:26.378 | INFO | MemInfo: { | 20260508 10:18:26.378 | INFO | "gpu" : " 17 GB", | 20260508 10:18:26.378 | INFO | "huge" : "", | 20260508 10:18:26.378 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:26.378 | INFO | "malloc" : " 14 GB", | 20260508 10:18:26.378 | INFO | "pinned" : " 29 GB" | 20260508 10:18:26.378 | INFO | } | 20260508 10:18:26.378 | INFO | Perf report for the last 5.13886 seconds: | 20260508 10:18:26.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:26.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 10:18:26.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | PRE_HQ : 31.3%, 1.0, 16.4ms, 0 | 20260508 10:18:26.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:26.378 | INFO | Uploaded 36GiB to GPU in 2.34134s (15.3758GiB/s) | 20260508 10:18:26.378 | INFO | Downloaded 34GiB to GPU in 2.51602s (13.5134GiB/s) | 20260508 10:18:31.511 | INFO | Analyzing chunk frames = [7330304,7330816) | 20260508 10:18:31.511 | INFO | MemInfo: { | 20260508 10:18:31.511 | INFO | "gpu" : " 17 GB", | 20260508 10:18:31.511 | INFO | "huge" : "", | 20260508 10:18:31.511 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:31.511 | INFO | "malloc" : " 14 GB", | 20260508 10:18:31.511 | INFO | "pinned" : " 29 GB" | 20260508 10:18:31.511 | INFO | } | 20260508 10:18:31.511 | INFO | Perf report for the last 5.13328 seconds: | 20260508 10:18:31.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:31.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 10:18:31.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:18:31.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:31.511 | INFO | Uploaded 36GiB to GPU in 2.38523s (15.0929GiB/s) | 20260508 10:18:31.511 | INFO | Downloaded 34GiB to GPU in 2.43246s (13.9776GiB/s) | 20260508 10:18:36.652 | INFO | Analyzing chunk frames = [7330816,7331328) | 20260508 10:18:36.652 | INFO | MemInfo: { | 20260508 10:18:36.652 | INFO | "gpu" : " 17 GB", | 20260508 10:18:36.652 | INFO | "huge" : "", | 20260508 10:18:36.652 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:36.652 | INFO | "malloc" : " 14 GB", | 20260508 10:18:36.652 | INFO | "pinned" : " 29 GB" | 20260508 10:18:36.652 | INFO | } | 20260508 10:18:36.652 | INFO | Perf report for the last 5.14092 seconds: | 20260508 10:18:36.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:36.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:18:36.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:18:36.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:36.652 | INFO | Uploaded 36GiB to GPU in 2.42644s (14.8366GiB/s) | 20260508 10:18:36.652 | INFO | Downloaded 34GiB to GPU in 2.4525s (13.8634GiB/s) | 20260508 10:18:41.796 | INFO | Analyzing chunk frames = [7331328,7331840) | 20260508 10:18:41.796 | INFO | MemInfo: { | 20260508 10:18:41.796 | INFO | "gpu" : " 17 GB", | 20260508 10:18:41.796 | INFO | "huge" : "", | 20260508 10:18:41.796 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:41.796 | INFO | "malloc" : " 14 GB", | 20260508 10:18:41.796 | INFO | "pinned" : " 29 GB" | 20260508 10:18:41.796 | INFO | } | 20260508 10:18:41.796 | INFO | Perf report for the last 5.14426 seconds: | 20260508 10:18:41.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:41.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.796 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:18:41.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.796 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:18:41.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:41.797 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:18:41.797 | INFO | Total mem usage: 16.7 GiB | 20260508 10:18:41.797 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:18:41.797 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:18:41.797 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:18:41.797 | INFO | High Water Breakdown: | 20260508 10:18:41.797 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:18:41.797 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:18:41.797 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:18:41.797 | INFO | ------------------------------------------ | 20260508 10:18:41.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:18:41.797 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:18:41.797 | INFO | Total mem usage: 26.5 GiB | 20260508 10:18:41.797 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:18:41.797 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:18:41.797 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:18:41.797 | INFO | High Water Breakdown: | 20260508 10:18:41.797 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:18:41.797 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:18:41.797 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:18:41.797 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:18:41.797 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:18:41.797 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:18:41.797 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:18:41.797 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:18:41.797 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:18:41.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:18:41.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:18:41.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:18:41.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:18:41.797 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:18:41.797 | INFO | ------------------------------------------ | 20260508 10:18:41.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:18:41.797 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:18:41.797 | INFO | Total mem usage: 27.8 GiB | 20260508 10:18:41.797 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:18:41.797 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:18:41.797 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:18:41.797 | INFO | High Water Breakdown: | 20260508 10:18:41.797 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:18:41.797 | INFO | ------------------------------------------ | 20260508 10:18:41.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:18:41.797 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:18:41.797 | INFO | Total mem usage: 15.7 GiB | 20260508 10:18:41.797 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:18:41.797 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:18:41.797 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:18:41.797 | INFO | High Water Breakdown: | 20260508 10:18:41.797 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:18:41.797 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:18:41.797 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:18:41.797 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:18:41.797 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:18:41.797 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:18:41.797 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:18:41.797 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:18:41.797 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:18:41.797 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:18:41.797 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:18:41.797 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:18:41.797 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:18:41.797 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:18:41.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:18:41.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:18:41.797 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:18:41.797 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:18:41.797 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:18:41.797 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:18:41.797 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:18:41.797 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:18:41.797 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:18:41.797 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:18:41.797 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:18:41.797 | INFO | ------------------------------------------ | 20260508 10:18:41.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:18:41.797 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:18:41.797 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:18:41.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:18:41.797 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.38e-01 | 20260508 10:18:41.797 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.06e-01, 1.17e-02 / 3.24e-03 | 20260508 10:18:41.797 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.43e+00, 5.91e-01 / 2.33e-01 | 20260508 10:18:41.797 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.23e-01 | 20260508 10:18:41.797 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.50e+01, 3.50e+00 / 8.74e-01 | 20260508 10:18:41.797 | INFO | Uploaded 36GiB to GPU in 2.41509s (14.9063GiB/s) | 20260508 10:18:41.798 | INFO | Downloaded 34GiB to GPU in 2.44222s (13.9217GiB/s) | 20260508 10:18:41.797 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.25e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.35e+00 / 2.51e+00, 4.08e-01 / 1.93e-01 | 20260508 10:18:41.798 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.08e+00, 2.44e-01 / 1.13e-01 | 20260508 10:18:41.798 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.03e-01 / 3.25e-01, 4.83e-02 / 2.39e-02 | 20260508 10:18:41.798 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:18:41.798 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:18:41.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:18:41.798 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 24.23%, Upload -- 2.55e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 24.68%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 31.03%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 2.03%, LockUpload -- 2.14e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 10:18:41.798 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 10:18:41.895 | INFO | Starting a round of Baz IO | 20260508 10:18:45.306 | INFO | Populating RTMetrics report | 20260508 10:18:45.307 | INFO | RTMetrics report populated | 20260508 10:18:45.307 | INFO | Opening tmp RTMetrics json file | 20260508 10:18:45.315 | INFO | Serializing RTMetrics to json | 20260508 10:18:45.316 | INFO | Serialization done, swapping tmp file now... | 20260508 10:18:45.317 | INFO | RTMetrics done | 20260508 10:18:45.371 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.475 seconds | 20260508 10:18:45.371 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:18:45.371 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225111% / 0% / 4.76837e-05% | 20260508 10:18:45.371 | INFO | Baz IO overall: | 20260508 10:18:45.371 | INFO | BytesWritten : 3184028240821 | 20260508 10:18:45.371 | INFO | NumEvents : 1530645870122 | 20260508 10:18:46.933 | INFO | Analyzing chunk frames = [7331840,7332352) | 20260508 10:18:46.933 | INFO | MemInfo: { | 20260508 10:18:46.933 | INFO | "gpu" : " 17 GB", | 20260508 10:18:46.933 | INFO | "huge" : "", | 20260508 10:18:46.933 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:46.933 | INFO | "malloc" : " 14 GB", | 20260508 10:18:46.933 | INFO | "pinned" : " 29 GB" | 20260508 10:18:46.933 | INFO | } | 20260508 10:18:46.933 | INFO | Perf report for the last 5.13651 seconds: | 20260508 10:18:46.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:46.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:46.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:46.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:46.933 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 10:18:46.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:46.933 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 10:18:46.933 | INFO | PRE_HQ : 65.3%, 1.6, 54.6ms, 0 | 20260508 10:18:46.933 | INFO | BAZWRITER : 67.0%, 2.5, 86.6ms, 0 | 20260508 10:18:46.933 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:18:46.933 | INFO | Uploaded 36GiB to GPU in 2.5949s (13.8734GiB/s) | 20260508 10:18:46.933 | INFO | Downloaded 38GiB to GPU in 2.69714s (14.089GiB/s) | 20260508 10:18:52.072 | INFO | Analyzing chunk frames = [7332352,7332864) | 20260508 10:18:52.072 | INFO | MemInfo: { | 20260508 10:18:52.072 | INFO | "gpu" : " 17 GB", | 20260508 10:18:52.072 | INFO | "huge" : "", | 20260508 10:18:52.072 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:52.072 | INFO | "malloc" : " 14 GB", | 20260508 10:18:52.072 | INFO | "pinned" : " 29 GB" | 20260508 10:18:52.072 | INFO | } | 20260508 10:18:52.072 | INFO | Perf report for the last 5.13889 seconds: | 20260508 10:18:52.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:52.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:18:52.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:18:52.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:52.072 | INFO | Uploaded 36GiB to GPU in 2.5461s (14.1393GiB/s) | 20260508 10:18:52.072 | INFO | Downloaded 34GiB to GPU in 2.46893s (13.7712GiB/s) | 20260508 10:18:57.204 | INFO | Analyzing chunk frames = [7332864,7333376) | 20260508 10:18:57.204 | INFO | MemInfo: { | 20260508 10:18:57.204 | INFO | "gpu" : " 17 GB", | 20260508 10:18:57.204 | INFO | "huge" : "", | 20260508 10:18:57.204 | INFO | "huge_shared" : " 29 GB", | 20260508 10:18:57.204 | INFO | "malloc" : " 14 GB", | 20260508 10:18:57.204 | INFO | "pinned" : " 29 GB" | 20260508 10:18:57.204 | INFO | } | 20260508 10:18:57.204 | INFO | Perf report for the last 5.1321 seconds: | 20260508 10:18:57.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:18:57.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 10:18:57.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:18:57.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:18:57.204 | INFO | Uploaded 36GiB to GPU in 2.49571s (14.4247GiB/s) | 20260508 10:18:57.204 | INFO | Downloaded 34GiB to GPU in 2.42331s (14.0304GiB/s) | 20260508 10:19:02.352 | INFO | Analyzing chunk frames = [7333376,7333888) | 20260508 10:19:02.352 | INFO | MemInfo: { | 20260508 10:19:02.352 | INFO | "gpu" : " 17 GB", | 20260508 10:19:02.352 | INFO | "huge" : "", | 20260508 10:19:02.352 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:02.352 | INFO | "malloc" : " 14 GB", | 20260508 10:19:02.352 | INFO | "pinned" : " 29 GB" | 20260508 10:19:02.352 | INFO | } | 20260508 10:19:02.352 | INFO | Perf report for the last 5.14799 seconds: | 20260508 10:19:02.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:02.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:19:02.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:19:02.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:02.352 | INFO | Uploaded 36GiB to GPU in 2.51516s (14.3132GiB/s) | 20260508 10:19:02.352 | INFO | Downloaded 34GiB to GPU in 2.41582s (14.0739GiB/s) | 20260508 10:19:07.488 | INFO | Analyzing chunk frames = [7333888,7334400) | 20260508 10:19:07.488 | INFO | MemInfo: { | 20260508 10:19:07.488 | INFO | "gpu" : " 17 GB", | 20260508 10:19:07.488 | INFO | "huge" : "", | 20260508 10:19:07.488 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:07.488 | INFO | "malloc" : " 14 GB", | 20260508 10:19:07.488 | INFO | "pinned" : " 29 GB" | 20260508 10:19:07.488 | INFO | } | 20260508 10:19:07.488 | INFO | Perf report for the last 5.13612 seconds: | 20260508 10:19:07.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:07.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.488 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:19:07.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.488 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:19:07.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:07.489 | INFO | Uploaded 36GiB to GPU in 2.32618s (15.476GiB/s) | 20260508 10:19:07.489 | INFO | Downloaded 34GiB to GPU in 2.50352s (13.5809GiB/s) | 20260508 10:19:12.626 | INFO | Analyzing chunk frames = [7334400,7334912) | 20260508 10:19:12.626 | INFO | MemInfo: { | 20260508 10:19:12.626 | INFO | "gpu" : " 17 GB", | 20260508 10:19:12.626 | INFO | "huge" : "", | 20260508 10:19:12.626 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:12.626 | INFO | "malloc" : " 14 GB", | 20260508 10:19:12.626 | INFO | "pinned" : " 29 GB" | 20260508 10:19:12.626 | INFO | } | 20260508 10:19:12.627 | INFO | Perf report for the last 5.13852 seconds: | 20260508 10:19:12.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:12.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:19:12.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:19:12.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:12.627 | INFO | Uploaded 36GiB to GPU in 2.3797s (15.128GiB/s) | 20260508 10:19:12.627 | INFO | Downloaded 34GiB to GPU in 2.4247s (14.0223GiB/s) | 20260508 10:19:17.768 | INFO | Analyzing chunk frames = [7334912,7335424) | 20260508 10:19:17.768 | INFO | MemInfo: { | 20260508 10:19:17.768 | INFO | "gpu" : " 17 GB", | 20260508 10:19:17.768 | INFO | "huge" : "", | 20260508 10:19:17.768 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:17.768 | INFO | "malloc" : " 14 GB", | 20260508 10:19:17.768 | INFO | "pinned" : " 29 GB" | 20260508 10:19:17.768 | INFO | } | 20260508 10:19:17.768 | INFO | Perf report for the last 5.14193 seconds: | 20260508 10:19:17.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:17.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.769 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 10:19:17.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.769 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:19:17.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:17.769 | INFO | Uploaded 36GiB to GPU in 2.45489s (14.6646GiB/s) | 20260508 10:19:17.769 | INFO | Downloaded 34GiB to GPU in 2.44134s (13.9268GiB/s) | 20260508 10:19:22.907 | INFO | Analyzing chunk frames = [7335424,7335936) | 20260508 10:19:22.907 | INFO | MemInfo: { | 20260508 10:19:22.907 | INFO | "gpu" : " 17 GB", | 20260508 10:19:22.907 | INFO | "huge" : "", | 20260508 10:19:22.907 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:22.907 | INFO | "malloc" : " 14 GB", | 20260508 10:19:22.907 | INFO | "pinned" : " 29 GB" | 20260508 10:19:22.907 | INFO | } | 20260508 10:19:22.908 | INFO | Perf report for the last 5.13905 seconds: | 20260508 10:19:22.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:22.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:19:22.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:19:22.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:22.908 | INFO | Uploaded 36GiB to GPU in 2.42809s (14.8265GiB/s) | 20260508 10:19:22.908 | INFO | Downloaded 34GiB to GPU in 2.41665s (14.069GiB/s) | 20260508 10:19:23.005 | INFO | Starting a round of Baz IO | 20260508 10:19:26.417 | INFO | Populating RTMetrics report | 20260508 10:19:26.417 | INFO | RTMetrics report populated | 20260508 10:19:26.417 | INFO | Opening tmp RTMetrics json file | 20260508 10:19:26.426 | INFO | Serializing RTMetrics to json | 20260508 10:19:26.427 | INFO | Serialization done, swapping tmp file now... | 20260508 10:19:26.428 | INFO | RTMetrics done | 20260508 10:19:26.486 | INFO | Finished a round of Baz IO: Wrote 1.462 GiB in 3.481 seconds | 20260508 10:19:26.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:19:26.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225572% / 0% / 4.76837e-05% | 20260508 10:19:26.486 | INFO | Baz IO overall: | 20260508 10:19:26.486 | INFO | BytesWritten : 3185598063787 | 20260508 10:19:26.486 | INFO | NumEvents : 1531386668180 | 20260508 10:19:28.048 | INFO | Analyzing chunk frames = [7335936,7336448) | 20260508 10:19:28.048 | INFO | MemInfo: { | 20260508 10:19:28.048 | INFO | "gpu" : " 17 GB", | 20260508 10:19:28.048 | INFO | "huge" : "", | 20260508 10:19:28.048 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:28.048 | INFO | "malloc" : " 14 GB", | 20260508 10:19:28.048 | INFO | "pinned" : " 29 GB" | 20260508 10:19:28.048 | INFO | } | 20260508 10:19:28.048 | INFO | Perf report for the last 5.14093 seconds: | 20260508 10:19:28.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:28.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:28.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:28.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:28.049 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:19:28.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:28.049 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:19:28.049 | INFO | PRE_HQ : 65.5%, 1.6, 54.1ms, 0 | 20260508 10:19:28.049 | INFO | BAZWRITER : 67.0%, 2.5, 86.1ms, 0 | 20260508 10:19:28.049 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:19:28.049 | INFO | Uploaded 36GiB to GPU in 2.60427s (13.8235GiB/s) | 20260508 10:19:28.049 | INFO | Downloaded 38GiB to GPU in 2.71946s (13.9734GiB/s) | 20260508 10:19:33.188 | INFO | Analyzing chunk frames = [7336448,7336960) | 20260508 10:19:33.188 | INFO | MemInfo: { | 20260508 10:19:33.188 | INFO | "gpu" : " 17 GB", | 20260508 10:19:33.188 | INFO | "huge" : "", | 20260508 10:19:33.188 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:33.189 | INFO | "malloc" : " 14 GB", | 20260508 10:19:33.189 | INFO | "pinned" : " 29 GB" | 20260508 10:19:33.189 | INFO | } | 20260508 10:19:33.189 | INFO | Perf report for the last 5.14014 seconds: | 20260508 10:19:33.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:33.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 10:19:33.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 10:19:33.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:33.189 | INFO | Uploaded 36GiB to GPU in 2.50549s (14.3684GiB/s) | 20260508 10:19:33.189 | INFO | Downloaded 34GiB to GPU in 2.46218s (13.8089GiB/s) | 20260508 10:19:38.328 | INFO | Analyzing chunk frames = [7336960,7337472) | 20260508 10:19:38.328 | INFO | MemInfo: { | 20260508 10:19:38.328 | INFO | "gpu" : " 17 GB", | 20260508 10:19:38.328 | INFO | "huge" : "", | 20260508 10:19:38.328 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:38.328 | INFO | "malloc" : " 14 GB", | 20260508 10:19:38.328 | INFO | "pinned" : " 29 GB" | 20260508 10:19:38.328 | INFO | } | 20260508 10:19:38.328 | INFO | Perf report for the last 5.13966 seconds: | 20260508 10:19:38.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:38.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.328 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:19:38.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.328 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 10:19:38.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:38.329 | INFO | Uploaded 36GiB to GPU in 2.46999s (14.5749GiB/s) | 20260508 10:19:38.329 | INFO | Downloaded 34GiB to GPU in 2.44246s (13.9204GiB/s) | 20260508 10:19:43.459 | INFO | Analyzing chunk frames = [7337472,7337984) | 20260508 10:19:43.459 | INFO | MemInfo: { | 20260508 10:19:43.459 | INFO | "gpu" : " 17 GB", | 20260508 10:19:43.459 | INFO | "huge" : "", | 20260508 10:19:43.459 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:43.459 | INFO | "malloc" : " 14 GB", | 20260508 10:19:43.459 | INFO | "pinned" : " 29 GB" | 20260508 10:19:43.459 | INFO | } | 20260508 10:19:43.459 | INFO | Perf report for the last 5.13076 seconds: | 20260508 10:19:43.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:43.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.459 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 10:19:43.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.459 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:19:43.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:43.460 | INFO | Uploaded 36GiB to GPU in 2.45647s (14.6552GiB/s) | 20260508 10:19:43.460 | INFO | Downloaded 34GiB to GPU in 2.43336s (13.9725GiB/s) | 20260508 10:19:48.603 | INFO | Analyzing chunk frames = [7337984,7338496) | 20260508 10:19:48.603 | INFO | MemInfo: { | 20260508 10:19:48.603 | INFO | "gpu" : " 17 GB", | 20260508 10:19:48.603 | INFO | "huge" : "", | 20260508 10:19:48.603 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:48.603 | INFO | "malloc" : " 14 GB", | 20260508 10:19:48.603 | INFO | "pinned" : " 29 GB" | 20260508 10:19:48.603 | INFO | } | 20260508 10:19:48.604 | INFO | Perf report for the last 5.14443 seconds: | 20260508 10:19:48.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:48.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 10:19:48.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:19:48.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:48.604 | INFO | Uploaded 36GiB to GPU in 2.40642s (14.96GiB/s) | 20260508 10:19:48.604 | INFO | Downloaded 34GiB to GPU in 2.54383s (13.3657GiB/s) | 20260508 10:19:53.747 | INFO | Analyzing chunk frames = [7338496,7339008) | 20260508 10:19:53.747 | INFO | MemInfo: { | 20260508 10:19:53.747 | INFO | "gpu" : " 17 GB", | 20260508 10:19:53.747 | INFO | "huge" : "", | 20260508 10:19:53.747 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:53.747 | INFO | "malloc" : " 14 GB", | 20260508 10:19:53.747 | INFO | "pinned" : " 29 GB" | 20260508 10:19:53.747 | INFO | } | 20260508 10:19:53.747 | INFO | Perf report for the last 5.14343 seconds: | 20260508 10:19:53.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:53.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.747 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 10:19:53.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.747 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:19:53.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:53.748 | INFO | Uploaded 36GiB to GPU in 2.32865s (15.4596GiB/s) | 20260508 10:19:53.748 | INFO | Downloaded 34GiB to GPU in 2.4517s (13.8679GiB/s) | 20260508 10:19:58.887 | INFO | Analyzing chunk frames = [7339008,7339520) | 20260508 10:19:58.887 | INFO | MemInfo: { | 20260508 10:19:58.887 | INFO | "gpu" : " 17 GB", | 20260508 10:19:58.887 | INFO | "huge" : "", | 20260508 10:19:58.887 | INFO | "huge_shared" : " 29 GB", | 20260508 10:19:58.887 | INFO | "malloc" : " 14 GB", | 20260508 10:19:58.887 | INFO | "pinned" : " 29 GB" | 20260508 10:19:58.887 | INFO | } | 20260508 10:19:58.887 | INFO | Perf report for the last 5.13993 seconds: | 20260508 10:19:58.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:19:58.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 10:19:58.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:19:58.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:19:58.887 | INFO | Uploaded 36GiB to GPU in 2.39297s (15.044GiB/s) | 20260508 10:19:58.887 | INFO | Downloaded 34GiB to GPU in 2.44586s (13.901GiB/s) | 20260508 10:20:04.022 | INFO | Analyzing chunk frames = [7339520,7340032) | 20260508 10:20:04.022 | INFO | MemInfo: { | 20260508 10:20:04.022 | INFO | "gpu" : " 17 GB", | 20260508 10:20:04.022 | INFO | "huge" : "", | 20260508 10:20:04.022 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:04.022 | INFO | "malloc" : " 14 GB", | 20260508 10:20:04.022 | INFO | "pinned" : " 29 GB" | 20260508 10:20:04.022 | INFO | } | 20260508 10:20:04.022 | INFO | Perf report for the last 5.13545 seconds: | 20260508 10:20:04.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:04.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.022 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 10:20:04.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.022 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:20:04.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:04.023 | INFO | Uploaded 36GiB to GPU in 2.43708s (14.7718GiB/s) | 20260508 10:20:04.023 | INFO | Downloaded 34GiB to GPU in 2.46156s (13.8124GiB/s) | 20260508 10:20:04.023 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:20:04.023 | INFO | Total mem usage: 16.7 GiB | 20260508 10:20:04.023 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:20:04.023 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:20:04.023 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:20:04.023 | INFO | High Water Breakdown: | 20260508 10:20:04.023 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:20:04.023 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:20:04.023 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:20:04.023 | INFO | ------------------------------------------ | 20260508 10:20:04.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:20:04.023 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:20:04.023 | INFO | Total mem usage: 26.5 GiB | 20260508 10:20:04.023 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:20:04.023 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:20:04.023 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:20:04.023 | INFO | High Water Breakdown: | 20260508 10:20:04.023 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:20:04.023 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:20:04.023 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:20:04.023 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:20:04.023 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:20:04.023 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:20:04.023 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:20:04.023 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:20:04.023 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:20:04.023 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:20:04.023 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:20:04.023 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:20:04.023 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:20:04.023 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:20:04.023 | INFO | ------------------------------------------ | 20260508 10:20:04.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:20:04.023 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:20:04.023 | INFO | Total mem usage: 27.8 GiB | 20260508 10:20:04.023 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:20:04.023 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:20:04.023 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:20:04.023 | INFO | High Water Breakdown: | 20260508 10:20:04.023 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:20:04.024 | INFO | ------------------------------------------ | 20260508 10:20:04.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:20:04.024 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:20:04.024 | INFO | Total mem usage: 15.7 GiB | 20260508 10:20:04.024 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:20:04.024 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:20:04.024 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:20:04.024 | INFO | High Water Breakdown: | 20260508 10:20:04.024 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:20:04.024 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:20:04.024 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:20:04.024 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:20:04.024 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:20:04.024 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:20:04.024 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:20:04.024 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:20:04.024 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:20:04.024 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:20:04.024 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:20:04.024 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:20:04.024 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:20:04.024 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:20:04.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:20:04.024 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:20:04.024 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:20:04.024 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:20:04.024 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:20:04.024 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:20:04.024 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:20:04.024 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:20:04.024 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:20:04.024 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:20:04.024 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:20:04.024 | INFO | ------------------------------------------ | 20260508 10:20:04.024 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:20:04.024 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:20:04.024 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:20:04.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:20:04.024 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.42e+01 / 7.34e-01 | 20260508 10:20:04.024 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.09e-02 / 5.07e-03 | 20260508 10:20:04.024 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.44e+00, 5.97e-01 / 2.07e-01 | 20260508 10:20:04.024 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.09e-01 | 20260508 10:20:04.024 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.53e+01, 3.74e+00 / 9.79e-01 | 20260508 10:20:04.024 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 1.36e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.47e+00, 4.08e-01 / 2.82e-01 | 20260508 10:20:04.024 | INFO | PerfReport: 3.01%, Metrics -- 9.85e-01 / 1.10e+00, 2.47e-01 / 1.19e-01 | 20260508 10:20:04.024 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.25e-01, 4.79e-02 / 2.74e-02 | 20260508 10:20:04.024 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:20:04.024 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:20:04.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:20:04.024 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 24.33%, Upload -- 2.56e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 2.20%, LockUpload -- 2.31e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 2.49%, LockDownload -- 2.62e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:20:04.024 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 3.29e-02 / 0.00e+00 | 20260508 10:20:04.120 | INFO | Starting a round of Baz IO | 20260508 10:20:07.542 | INFO | Populating RTMetrics report | 20260508 10:20:07.542 | INFO | RTMetrics report populated | 20260508 10:20:07.542 | INFO | Opening tmp RTMetrics json file | 20260508 10:20:07.551 | INFO | Serializing RTMetrics to json | 20260508 10:20:07.552 | INFO | Serialization done, swapping tmp file now... | 20260508 10:20:07.553 | INFO | RTMetrics done | 20260508 10:20:07.611 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.491 seconds | 20260508 10:20:07.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:20:07.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225004% / 0% / 5.16574e-05% | 20260508 10:20:07.611 | INFO | Baz IO overall: | 20260508 10:20:07.611 | INFO | BytesWritten : 3187166816806 | 20260508 10:20:07.611 | INFO | NumEvents : 1532126324160 | 20260508 10:20:09.164 | INFO | Analyzing chunk frames = [7340032,7340544) | 20260508 10:20:09.164 | INFO | MemInfo: { | 20260508 10:20:09.164 | INFO | "gpu" : " 17 GB", | 20260508 10:20:09.164 | INFO | "huge" : "", | 20260508 10:20:09.164 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:09.164 | INFO | "malloc" : " 14 GB", | 20260508 10:20:09.164 | INFO | "pinned" : " 29 GB" | 20260508 10:20:09.164 | INFO | } | 20260508 10:20:09.164 | INFO | Perf report for the last 5.14201 seconds: | 20260508 10:20:09.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:09.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:09.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:09.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:09.165 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 10:20:09.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:09.165 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:20:09.165 | INFO | PRE_HQ : 65.7%, 1.6, 54.6ms, 0 | 20260508 10:20:09.165 | INFO | BAZWRITER : 67.3%, 2.5, 86.4ms, 0 | 20260508 10:20:09.165 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:20:09.165 | INFO | Uploaded 36GiB to GPU in 2.60613s (13.8136GiB/s) | 20260508 10:20:09.165 | INFO | Downloaded 38GiB to GPU in 2.7202s (13.9695GiB/s) | 20260508 10:20:14.300 | INFO | Analyzing chunk frames = [7340544,7341056) | 20260508 10:20:14.300 | INFO | MemInfo: { | 20260508 10:20:14.300 | INFO | "gpu" : " 17 GB", | 20260508 10:20:14.300 | INFO | "huge" : "", | 20260508 10:20:14.301 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:14.301 | INFO | "malloc" : " 14 GB", | 20260508 10:20:14.301 | INFO | "pinned" : " 29 GB" | 20260508 10:20:14.301 | INFO | } | 20260508 10:20:14.301 | INFO | Perf report for the last 5.13631 seconds: | 20260508 10:20:14.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:14.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 10:20:14.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:20:14.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:14.301 | INFO | Uploaded 36GiB to GPU in 2.41543s (14.9042GiB/s) | 20260508 10:20:14.301 | INFO | Downloaded 34GiB to GPU in 2.47137s (13.7575GiB/s) | 20260508 10:20:19.439 | INFO | Analyzing chunk frames = [7341056,7341568) | 20260508 10:20:19.439 | INFO | MemInfo: { | 20260508 10:20:19.439 | INFO | "gpu" : " 17 GB", | 20260508 10:20:19.439 | INFO | "huge" : "", | 20260508 10:20:19.439 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:19.439 | INFO | "malloc" : " 14 GB", | 20260508 10:20:19.439 | INFO | "pinned" : " 29 GB" | 20260508 10:20:19.439 | INFO | } | 20260508 10:20:19.439 | INFO | Perf report for the last 5.13827 seconds: | 20260508 10:20:19.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:19.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 10:20:19.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:20:19.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:19.439 | INFO | Uploaded 36GiB to GPU in 2.43621s (14.777GiB/s) | 20260508 10:20:19.439 | INFO | Downloaded 34GiB to GPU in 2.43569s (13.9591GiB/s) | 20260508 10:20:24.578 | INFO | Analyzing chunk frames = [7341568,7342080) | 20260508 10:20:24.578 | INFO | MemInfo: { | 20260508 10:20:24.578 | INFO | "gpu" : " 17 GB", | 20260508 10:20:24.578 | INFO | "huge" : "", | 20260508 10:20:24.578 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:24.578 | INFO | "malloc" : " 14 GB", | 20260508 10:20:24.578 | INFO | "pinned" : " 29 GB" | 20260508 10:20:24.578 | INFO | } | 20260508 10:20:24.578 | INFO | Perf report for the last 5.13936 seconds: | 20260508 10:20:24.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:24.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.578 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:20:24.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.578 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:20:24.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:24.579 | INFO | Uploaded 36GiB to GPU in 2.42714s (14.8323GiB/s) | 20260508 10:20:24.579 | INFO | Downloaded 34GiB to GPU in 2.42536s (14.0185GiB/s) | 20260508 10:20:29.715 | INFO | Analyzing chunk frames = [7342080,7342592) | 20260508 10:20:29.715 | INFO | MemInfo: { | 20260508 10:20:29.715 | INFO | "gpu" : " 17 GB", | 20260508 10:20:29.715 | INFO | "huge" : "", | 20260508 10:20:29.715 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:29.715 | INFO | "malloc" : " 14 GB", | 20260508 10:20:29.715 | INFO | "pinned" : " 29 GB" | 20260508 10:20:29.715 | INFO | } | 20260508 10:20:29.715 | INFO | Perf report for the last 5.13707 seconds: | 20260508 10:20:29.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:29.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.716 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:20:29.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.716 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:20:29.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:29.716 | INFO | Uploaded 36GiB to GPU in 2.37464s (15.1602GiB/s) | 20260508 10:20:29.716 | INFO | Downloaded 34GiB to GPU in 2.52705s (13.4544GiB/s) | 20260508 10:20:34.864 | INFO | Analyzing chunk frames = [7342592,7343104) | 20260508 10:20:34.864 | INFO | MemInfo: { | 20260508 10:20:34.864 | INFO | "gpu" : " 17 GB", | 20260508 10:20:34.864 | INFO | "huge" : "", | 20260508 10:20:34.864 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:34.864 | INFO | "malloc" : " 14 GB", | 20260508 10:20:34.864 | INFO | "pinned" : " 29 GB" | 20260508 10:20:34.865 | INFO | } | 20260508 10:20:34.865 | INFO | Perf report for the last 5.14908 seconds: | 20260508 10:20:34.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:34.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 10:20:34.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:20:34.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:34.865 | INFO | Uploaded 36GiB to GPU in 2.35523s (15.2851GiB/s) | 20260508 10:20:34.865 | INFO | Downloaded 34GiB to GPU in 2.44911s (13.8826GiB/s) | 20260508 10:20:40.003 | INFO | Analyzing chunk frames = [7343104,7343616) | 20260508 10:20:40.003 | INFO | MemInfo: { | 20260508 10:20:40.003 | INFO | "gpu" : " 17 GB", | 20260508 10:20:40.003 | INFO | "huge" : "", | 20260508 10:20:40.003 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:40.003 | INFO | "malloc" : " 14 GB", | 20260508 10:20:40.003 | INFO | "pinned" : " 29 GB" | 20260508 10:20:40.003 | INFO | } | 20260508 10:20:40.004 | INFO | Perf report for the last 5.13896 seconds: | 20260508 10:20:40.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:40.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:20:40.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:20:40.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:40.004 | INFO | Uploaded 36GiB to GPU in 2.42874s (14.8225GiB/s) | 20260508 10:20:40.004 | INFO | Downloaded 34GiB to GPU in 2.46334s (13.8024GiB/s) | 20260508 10:20:45.138 | INFO | Analyzing chunk frames = [7343616,7344128) | 20260508 10:20:45.138 | INFO | MemInfo: { | 20260508 10:20:45.138 | INFO | "gpu" : " 17 GB", | 20260508 10:20:45.138 | INFO | "huge" : "", | 20260508 10:20:45.138 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:45.138 | INFO | "malloc" : " 14 GB", | 20260508 10:20:45.138 | INFO | "pinned" : " 29 GB" | 20260508 10:20:45.138 | INFO | } | 20260508 10:20:45.138 | INFO | Perf report for the last 5.13447 seconds: | 20260508 10:20:45.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:45.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:20:45.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:20:45.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:45.138 | INFO | Uploaded 36GiB to GPU in 2.41846s (14.8855GiB/s) | 20260508 10:20:45.138 | INFO | Downloaded 34GiB to GPU in 2.46403s (13.7985GiB/s) | 20260508 10:20:45.235 | INFO | Starting a round of Baz IO | 20260508 10:20:48.660 | INFO | Populating RTMetrics report | 20260508 10:20:48.660 | INFO | RTMetrics report populated | 20260508 10:20:48.660 | INFO | Opening tmp RTMetrics json file | 20260508 10:20:48.665 | INFO | Serializing RTMetrics to json | 20260508 10:20:48.667 | INFO | Serialization done, swapping tmp file now... | 20260508 10:20:48.667 | INFO | RTMetrics done | 20260508 10:20:48.723 | INFO | Finished a round of Baz IO: Wrote 1.462 GiB in 3.487 seconds | 20260508 10:20:48.723 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:20:48.723 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225091% / 0% / 5.96046e-05% | 20260508 10:20:48.723 | INFO | Baz IO overall: | 20260508 10:20:48.723 | INFO | BytesWritten : 3188736121110 | 20260508 10:20:48.723 | INFO | NumEvents : 1532866645065 | 20260508 10:20:50.274 | INFO | Analyzing chunk frames = [7344128,7344640) | 20260508 10:20:50.274 | INFO | MemInfo: { | 20260508 10:20:50.274 | INFO | "gpu" : " 17 GB", | 20260508 10:20:50.274 | INFO | "huge" : "", | 20260508 10:20:50.274 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:50.274 | INFO | "malloc" : " 14 GB", | 20260508 10:20:50.274 | INFO | "pinned" : " 29 GB" | 20260508 10:20:50.274 | INFO | } | 20260508 10:20:50.274 | INFO | Perf report for the last 5.13629 seconds: | 20260508 10:20:50.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:50.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:50.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:50.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:50.274 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 10:20:50.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:50.274 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:20:50.274 | INFO | PRE_HQ : 65.4%, 1.6, 54.2ms, 0 | 20260508 10:20:50.274 | INFO | BAZWRITER : 67.2%, 2.5, 86.7ms, 0 | 20260508 10:20:50.274 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:20:50.275 | INFO | Uploaded 36GiB to GPU in 2.58407s (13.9315GiB/s) | 20260508 10:20:50.275 | INFO | Downloaded 38GiB to GPU in 2.70195s (14.0639GiB/s) | 20260508 10:20:55.415 | INFO | Analyzing chunk frames = [7344640,7345152) | 20260508 10:20:55.415 | INFO | MemInfo: { | 20260508 10:20:55.415 | INFO | "gpu" : " 17 GB", | 20260508 10:20:55.415 | INFO | "huge" : "", | 20260508 10:20:55.415 | INFO | "huge_shared" : " 29 GB", | 20260508 10:20:55.415 | INFO | "malloc" : " 14 GB", | 20260508 10:20:55.415 | INFO | "pinned" : " 29 GB" | 20260508 10:20:55.415 | INFO | } | 20260508 10:20:55.415 | INFO | Perf report for the last 5.14108 seconds: | 20260508 10:20:55.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:20:55.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.415 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 10:20:55.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.416 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:20:55.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:20:55.416 | INFO | Uploaded 36GiB to GPU in 2.52321s (14.2675GiB/s) | 20260508 10:20:55.416 | INFO | Downloaded 34GiB to GPU in 2.48412s (13.6869GiB/s) | 20260508 10:21:00.557 | INFO | Analyzing chunk frames = [7345152,7345664) | 20260508 10:21:00.557 | INFO | MemInfo: { | 20260508 10:21:00.557 | INFO | "gpu" : " 17 GB", | 20260508 10:21:00.557 | INFO | "huge" : "", | 20260508 10:21:00.557 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:00.557 | INFO | "malloc" : " 14 GB", | 20260508 10:21:00.557 | INFO | "pinned" : " 29 GB" | 20260508 10:21:00.557 | INFO | } | 20260508 10:21:00.557 | INFO | Perf report for the last 5.14203 seconds: | 20260508 10:21:00.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:00.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.558 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 10:21:00.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.558 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:21:00.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:00.558 | INFO | Uploaded 36GiB to GPU in 2.42716s (14.8321GiB/s) | 20260508 10:21:00.558 | INFO | Downloaded 34GiB to GPU in 2.45246s (13.8636GiB/s) | 20260508 10:21:05.698 | INFO | Analyzing chunk frames = [7345664,7346176) | 20260508 10:21:05.698 | INFO | MemInfo: { | 20260508 10:21:05.698 | INFO | "gpu" : " 17 GB", | 20260508 10:21:05.698 | INFO | "huge" : "", | 20260508 10:21:05.698 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:05.698 | INFO | "malloc" : " 14 GB", | 20260508 10:21:05.698 | INFO | "pinned" : " 29 GB" | 20260508 10:21:05.698 | INFO | } | 20260508 10:21:05.698 | INFO | Perf report for the last 5.14027 seconds: | 20260508 10:21:05.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:05.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 10:21:05.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:21:05.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:05.698 | INFO | Uploaded 36GiB to GPU in 2.41166s (14.9275GiB/s) | 20260508 10:21:05.698 | INFO | Downloaded 34GiB to GPU in 2.43499s (13.9631GiB/s) | 20260508 10:21:10.835 | INFO | Analyzing chunk frames = [7346176,7346688) | 20260508 10:21:10.835 | INFO | MemInfo: { | 20260508 10:21:10.835 | INFO | "gpu" : " 17 GB", | 20260508 10:21:10.835 | INFO | "huge" : "", | 20260508 10:21:10.836 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:10.836 | INFO | "malloc" : " 14 GB", | 20260508 10:21:10.836 | INFO | "pinned" : " 29 GB" | 20260508 10:21:10.836 | INFO | } | 20260508 10:21:10.836 | INFO | Perf report for the last 5.13797 seconds: | 20260508 10:21:10.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:10.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 10:21:10.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 10:21:10.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:10.836 | INFO | Uploaded 36GiB to GPU in 2.4019s (14.9882GiB/s) | 20260508 10:21:10.836 | INFO | Downloaded 34GiB to GPU in 2.54614s (13.3536GiB/s) | 20260508 10:21:15.970 | INFO | Analyzing chunk frames = [7346688,7347200) | 20260508 10:21:15.970 | INFO | MemInfo: { | 20260508 10:21:15.970 | INFO | "gpu" : " 17 GB", | 20260508 10:21:15.970 | INFO | "huge" : "", | 20260508 10:21:15.970 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:15.970 | INFO | "malloc" : " 14 GB", | 20260508 10:21:15.970 | INFO | "pinned" : " 29 GB" | 20260508 10:21:15.970 | INFO | } | 20260508 10:21:15.970 | INFO | Perf report for the last 5.13436 seconds: | 20260508 10:21:15.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:15.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | ANALYSIS : 65.1%, 3.0, 101.3ms, 0 | 20260508 10:21:15.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:21:15.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:15.970 | INFO | Uploaded 36GiB to GPU in 2.31929s (15.522GiB/s) | 20260508 10:21:15.970 | INFO | Downloaded 34GiB to GPU in 2.44447s (13.9089GiB/s) | 20260508 10:21:21.110 | INFO | Analyzing chunk frames = [7347200,7347712) | 20260508 10:21:21.110 | INFO | MemInfo: { | 20260508 10:21:21.110 | INFO | "gpu" : " 17 GB", | 20260508 10:21:21.111 | INFO | "huge" : "", | 20260508 10:21:21.111 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:21.111 | INFO | "malloc" : " 14 GB", | 20260508 10:21:21.111 | INFO | "pinned" : " 29 GB" | 20260508 10:21:21.111 | INFO | } | 20260508 10:21:21.111 | INFO | Perf report for the last 5.14053 seconds: | 20260508 10:21:21.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:21.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 10:21:21.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:21:21.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:21.111 | INFO | Uploaded 36GiB to GPU in 2.35816s (15.2662GiB/s) | 20260508 10:21:21.111 | INFO | Downloaded 34GiB to GPU in 2.45604s (13.8434GiB/s) | 20260508 10:21:26.254 | INFO | Analyzing chunk frames = [7347712,7348224) | 20260508 10:21:26.254 | INFO | MemInfo: { | 20260508 10:21:26.254 | INFO | "gpu" : " 17 GB", | 20260508 10:21:26.254 | INFO | "huge" : "", | 20260508 10:21:26.254 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:26.254 | INFO | "malloc" : " 14 GB", | 20260508 10:21:26.254 | INFO | "pinned" : " 29 GB" | 20260508 10:21:26.254 | INFO | } | 20260508 10:21:26.254 | INFO | Perf report for the last 5.14338 seconds: | 20260508 10:21:26.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:26.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:21:26.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:21:26.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:26.254 | INFO | Uploaded 36GiB to GPU in 2.38526s (15.0927GiB/s) | 20260508 10:21:26.254 | INFO | Downloaded 34GiB to GPU in 2.44651s (13.8973GiB/s) | 20260508 10:21:26.254 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:21:26.254 | INFO | Total mem usage: 16.7 GiB | 20260508 10:21:26.254 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:21:26.254 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:21:26.254 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:21:26.254 | INFO | High Water Breakdown: | 20260508 10:21:26.254 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:21:26.254 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:21:26.254 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:21:26.254 | INFO | ------------------------------------------ | 20260508 10:21:26.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:21:26.254 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:21:26.254 | INFO | Total mem usage: 26.5 GiB | 20260508 10:21:26.254 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:21:26.254 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:21:26.254 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:21:26.254 | INFO | High Water Breakdown: | 20260508 10:21:26.254 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:21:26.254 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:21:26.254 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:21:26.254 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:21:26.254 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:21:26.254 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:21:26.254 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:21:26.254 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:21:26.254 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:21:26.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:21:26.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:21:26.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:21:26.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:21:26.255 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:21:26.255 | INFO | ------------------------------------------ | 20260508 10:21:26.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:21:26.255 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:21:26.255 | INFO | Total mem usage: 27.8 GiB | 20260508 10:21:26.255 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:21:26.255 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:21:26.255 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:21:26.255 | INFO | High Water Breakdown: | 20260508 10:21:26.255 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:21:26.255 | INFO | ------------------------------------------ | 20260508 10:21:26.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:21:26.255 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:21:26.255 | INFO | Total mem usage: 15.7 GiB | 20260508 10:21:26.255 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:21:26.255 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:21:26.255 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:21:26.255 | INFO | High Water Breakdown: | 20260508 10:21:26.255 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:21:26.255 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:21:26.255 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:21:26.255 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:21:26.255 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:21:26.255 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:21:26.255 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:21:26.255 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:21:26.255 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:21:26.255 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:21:26.255 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:21:26.255 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:21:26.255 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:21:26.255 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:21:26.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:21:26.255 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:21:26.255 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:21:26.255 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:21:26.255 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:21:26.255 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:21:26.255 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:21:26.255 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:21:26.255 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:21:26.255 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:21:26.255 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:21:26.255 | INFO | ------------------------------------------ | 20260508 10:21:26.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:21:26.255 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:21:26.255 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:21:26.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:21:26.255 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.41e+01 / 7.49e-01 | 20260508 10:21:26.255 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.06e-01, 1.15e-02 / 9.30e-03 | 20260508 10:21:26.255 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.49e+00, 5.91e-01 / 2.15e-01 | 20260508 10:21:26.255 | INFO | PerfReport: 3.37%, Histogram -- 1.11e+00 / 1.47e+00, 2.08e-01 / 1.27e-01 | 20260508 10:21:26.255 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.52e+01, 3.86e+00 / 1.12e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.31e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.49e+00, 4.08e-01 / 3.00e-01 | 20260508 10:21:26.255 | INFO | PerfReport: 3.01%, Metrics -- 9.86e-01 / 1.13e+00, 2.42e-01 / 1.40e-01 | 20260508 10:21:26.255 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.34e-01, 4.76e-02 / 3.07e-02 | 20260508 10:21:26.255 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:21:26.255 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:21:26.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:21:26.255 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 24.06%, Upload -- 2.53e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 24.72%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 31.17%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 1.98%, LockUpload -- 2.08e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 2.71%, LockDownload -- 2.85e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 15.35%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:21:26.255 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260508 10:21:26.351 | INFO | Starting a round of Baz IO | 20260508 10:21:29.774 | INFO | Populating RTMetrics report | 20260508 10:21:29.774 | INFO | RTMetrics report populated | 20260508 10:21:29.774 | INFO | Opening tmp RTMetrics json file | 20260508 10:21:29.785 | INFO | Serializing RTMetrics to json | 20260508 10:21:29.787 | INFO | Serialization done, swapping tmp file now... | 20260508 10:21:29.788 | INFO | RTMetrics done | 20260508 10:21:29.845 | INFO | Finished a round of Baz IO: Wrote 1.463 GiB in 3.493 seconds | 20260508 10:21:29.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:21:29.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225373% / 0% / 5.96046e-05% | 20260508 10:21:29.845 | INFO | Baz IO overall: | 20260508 10:21:29.845 | INFO | BytesWritten : 3190306605952 | 20260508 10:21:29.845 | INFO | NumEvents : 1533607902584 | 20260508 10:21:31.389 | INFO | Analyzing chunk frames = [7348224,7348736) | 20260508 10:21:31.389 | INFO | MemInfo: { | 20260508 10:21:31.389 | INFO | "gpu" : " 17 GB", | 20260508 10:21:31.389 | INFO | "huge" : "", | 20260508 10:21:31.389 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:31.389 | INFO | "malloc" : " 14 GB", | 20260508 10:21:31.389 | INFO | "pinned" : " 29 GB" | 20260508 10:21:31.389 | INFO | } | 20260508 10:21:31.390 | INFO | Perf report for the last 5.1355 seconds: | 20260508 10:21:31.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:31.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:31.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:31.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:31.390 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 10:21:31.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:31.390 | INFO | RT_METRICS : 38.9%, 1.0, 20.4ms, 0 | 20260508 10:21:31.390 | INFO | PRE_HQ : 65.6%, 1.6, 54.6ms, 0 | 20260508 10:21:31.390 | INFO | BAZWRITER : 67.4%, 2.5, 86.9ms, 0 | 20260508 10:21:31.390 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:21:31.390 | INFO | Uploaded 36GiB to GPU in 2.63118s (13.6821GiB/s) | 20260508 10:21:31.390 | INFO | Downloaded 38GiB to GPU in 2.72099s (13.9655GiB/s) | 20260508 10:21:36.531 | INFO | Analyzing chunk frames = [7348736,7349248) | 20260508 10:21:36.531 | INFO | MemInfo: { | 20260508 10:21:36.531 | INFO | "gpu" : " 17 GB", | 20260508 10:21:36.531 | INFO | "huge" : "", | 20260508 10:21:36.531 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:36.531 | INFO | "malloc" : " 14 GB", | 20260508 10:21:36.531 | INFO | "pinned" : " 29 GB" | 20260508 10:21:36.531 | INFO | } | 20260508 10:21:36.531 | INFO | Perf report for the last 5.14143 seconds: | 20260508 10:21:36.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:36.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.531 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:21:36.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.531 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:21:36.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:36.532 | INFO | Uploaded 36GiB to GPU in 2.42008s (14.8755GiB/s) | 20260508 10:21:36.532 | INFO | Downloaded 34GiB to GPU in 2.4695s (13.768GiB/s) | 20260508 10:21:41.668 | INFO | Analyzing chunk frames = [7349248,7349760) | 20260508 10:21:41.668 | INFO | MemInfo: { | 20260508 10:21:41.668 | INFO | "gpu" : " 17 GB", | 20260508 10:21:41.668 | INFO | "huge" : "", | 20260508 10:21:41.668 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:41.668 | INFO | "malloc" : " 14 GB", | 20260508 10:21:41.668 | INFO | "pinned" : " 29 GB" | 20260508 10:21:41.668 | INFO | } | 20260508 10:21:41.668 | INFO | Perf report for the last 5.13706 seconds: | 20260508 10:21:41.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:41.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 10:21:41.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:21:41.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:41.668 | INFO | Uploaded 36GiB to GPU in 2.46423s (14.6091GiB/s) | 20260508 10:21:41.668 | INFO | Downloaded 34GiB to GPU in 2.44987s (13.8783GiB/s) | 20260508 10:21:46.810 | INFO | Analyzing chunk frames = [7349760,7350272) | 20260508 10:21:46.810 | INFO | MemInfo: { | 20260508 10:21:46.810 | INFO | "gpu" : " 17 GB", | 20260508 10:21:46.810 | INFO | "huge" : "", | 20260508 10:21:46.810 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:46.811 | INFO | "malloc" : " 14 GB", | 20260508 10:21:46.811 | INFO | "pinned" : " 29 GB" | 20260508 10:21:46.811 | INFO | } | 20260508 10:21:46.811 | INFO | Perf report for the last 5.14251 seconds: | 20260508 10:21:46.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:46.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 10:21:46.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 10:21:46.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:46.811 | INFO | Uploaded 36GiB to GPU in 2.44516s (14.723GiB/s) | 20260508 10:21:46.811 | INFO | Downloaded 34GiB to GPU in 2.46072s (13.8171GiB/s) | 20260508 10:21:51.950 | INFO | Analyzing chunk frames = [7350272,7350784) | 20260508 10:21:51.950 | INFO | MemInfo: { | 20260508 10:21:51.950 | INFO | "gpu" : " 17 GB", | 20260508 10:21:51.950 | INFO | "huge" : "", | 20260508 10:21:51.950 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:51.950 | INFO | "malloc" : " 14 GB", | 20260508 10:21:51.950 | INFO | "pinned" : " 29 GB" | 20260508 10:21:51.950 | INFO | } | 20260508 10:21:51.951 | INFO | Perf report for the last 5.13991 seconds: | 20260508 10:21:51.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:51.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 10:21:51.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:21:51.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:51.951 | INFO | Uploaded 36GiB to GPU in 2.36471s (15.2238GiB/s) | 20260508 10:21:51.951 | INFO | Downloaded 34GiB to GPU in 2.52914s (13.4433GiB/s) | 20260508 10:21:57.086 | INFO | Analyzing chunk frames = [7350784,7351296) | 20260508 10:21:57.086 | INFO | MemInfo: { | 20260508 10:21:57.086 | INFO | "gpu" : " 17 GB", | 20260508 10:21:57.086 | INFO | "huge" : "", | 20260508 10:21:57.086 | INFO | "huge_shared" : " 29 GB", | 20260508 10:21:57.086 | INFO | "malloc" : " 14 GB", | 20260508 10:21:57.086 | INFO | "pinned" : " 29 GB" | 20260508 10:21:57.086 | INFO | } | 20260508 10:21:57.086 | INFO | Perf report for the last 5.13562 seconds: | 20260508 10:21:57.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:21:57.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:21:57.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:21:57.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:21:57.086 | INFO | Uploaded 36GiB to GPU in 2.33537s (15.4151GiB/s) | 20260508 10:21:57.086 | INFO | Downloaded 34GiB to GPU in 2.44733s (13.8927GiB/s) | 20260508 10:22:02.222 | INFO | Analyzing chunk frames = [7351296,7351808) | 20260508 10:22:02.222 | INFO | MemInfo: { | 20260508 10:22:02.222 | INFO | "gpu" : " 17 GB", | 20260508 10:22:02.222 | INFO | "huge" : "", | 20260508 10:22:02.222 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:02.222 | INFO | "malloc" : " 14 GB", | 20260508 10:22:02.222 | INFO | "pinned" : " 29 GB" | 20260508 10:22:02.222 | INFO | } | 20260508 10:22:02.222 | INFO | Perf report for the last 5.13568 seconds: | 20260508 10:22:02.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:02.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:22:02.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:22:02.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:02.222 | INFO | Uploaded 36GiB to GPU in 2.38663s (15.0841GiB/s) | 20260508 10:22:02.222 | INFO | Downloaded 34GiB to GPU in 2.44745s (13.892GiB/s) | 20260508 10:22:07.368 | INFO | Analyzing chunk frames = [7351808,7352320) | 20260508 10:22:07.368 | INFO | MemInfo: { | 20260508 10:22:07.368 | INFO | "gpu" : " 17 GB", | 20260508 10:22:07.368 | INFO | "huge" : "", | 20260508 10:22:07.368 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:07.368 | INFO | "malloc" : " 14 GB", | 20260508 10:22:07.368 | INFO | "pinned" : " 29 GB" | 20260508 10:22:07.368 | INFO | } | 20260508 10:22:07.369 | INFO | Perf report for the last 5.14667 seconds: | 20260508 10:22:07.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:07.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 10:22:07.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:22:07.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:07.369 | INFO | Uploaded 36GiB to GPU in 2.37491s (15.1584GiB/s) | 20260508 10:22:07.369 | INFO | Downloaded 34GiB to GPU in 2.44106s (13.9284GiB/s) | 20260508 10:22:07.466 | INFO | Starting a round of Baz IO | 20260508 10:22:10.875 | INFO | Populating RTMetrics report | 20260508 10:22:10.876 | INFO | RTMetrics report populated | 20260508 10:22:10.876 | INFO | Opening tmp RTMetrics json file | 20260508 10:22:10.885 | INFO | Serializing RTMetrics to json | 20260508 10:22:10.888 | INFO | Serialization done, swapping tmp file now... | 20260508 10:22:10.888 | INFO | RTMetrics done | 20260508 10:22:10.945 | INFO | Finished a round of Baz IO: Wrote 1.46 GiB in 3.478 seconds | 20260508 10:22:10.945 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:22:10.945 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224086% / 3.97364e-06% / 5.96046e-05% | 20260508 10:22:10.945 | INFO | Baz IO overall: | 20260508 10:22:10.945 | INFO | BytesWritten : 3191874428905 | 20260508 10:22:10.945 | INFO | NumEvents : 1534347553465 | 20260508 10:22:12.512 | INFO | Analyzing chunk frames = [7352320,7352832) | 20260508 10:22:12.512 | INFO | MemInfo: { | 20260508 10:22:12.512 | INFO | "gpu" : " 17 GB", | 20260508 10:22:12.512 | INFO | "huge" : "", | 20260508 10:22:12.513 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:12.513 | INFO | "malloc" : " 14 GB", | 20260508 10:22:12.513 | INFO | "pinned" : " 29 GB" | 20260508 10:22:12.513 | INFO | } | 20260508 10:22:12.513 | INFO | Perf report for the last 5.14421 seconds: | 20260508 10:22:12.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:12.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:12.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:12.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:12.513 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:22:12.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:12.513 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 10:22:12.513 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 10:22:12.513 | INFO | BAZWRITER : 66.8%, 2.5, 86.6ms, 0 | 20260508 10:22:12.513 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:22:12.513 | INFO | Uploaded 36GiB to GPU in 2.58157s (13.945GiB/s) | 20260508 10:22:12.513 | INFO | Downloaded 38GiB to GPU in 2.71492s (13.9967GiB/s) | 20260508 10:22:17.645 | INFO | Analyzing chunk frames = [7352832,7353344) | 20260508 10:22:17.645 | INFO | MemInfo: { | 20260508 10:22:17.645 | INFO | "gpu" : " 17 GB", | 20260508 10:22:17.645 | INFO | "huge" : "", | 20260508 10:22:17.645 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:17.645 | INFO | "malloc" : " 14 GB", | 20260508 10:22:17.645 | INFO | "pinned" : " 29 GB" | 20260508 10:22:17.645 | INFO | } | 20260508 10:22:17.645 | INFO | Perf report for the last 5.13262 seconds: | 20260508 10:22:17.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:17.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.645 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:22:17.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.645 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:22:17.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:17.646 | INFO | Uploaded 36GiB to GPU in 2.41158s (14.928GiB/s) | 20260508 10:22:17.646 | INFO | Downloaded 34GiB to GPU in 2.48029s (13.7081GiB/s) | 20260508 10:22:22.781 | INFO | Analyzing chunk frames = [7353344,7353856) | 20260508 10:22:22.781 | INFO | MemInfo: { | 20260508 10:22:22.781 | INFO | "gpu" : " 17 GB", | 20260508 10:22:22.781 | INFO | "huge" : "", | 20260508 10:22:22.782 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:22.782 | INFO | "malloc" : " 14 GB", | 20260508 10:22:22.782 | INFO | "pinned" : " 29 GB" | 20260508 10:22:22.782 | INFO | } | 20260508 10:22:22.782 | INFO | Perf report for the last 5.13632 seconds: | 20260508 10:22:22.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:22.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:22:22.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:22:22.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:22.782 | INFO | Uploaded 36GiB to GPU in 2.46226s (14.6207GiB/s) | 20260508 10:22:22.782 | INFO | Downloaded 34GiB to GPU in 2.44914s (13.8824GiB/s) | 20260508 10:22:27.926 | INFO | Analyzing chunk frames = [7353856,7354368) | 20260508 10:22:27.926 | INFO | MemInfo: { | 20260508 10:22:27.926 | INFO | "gpu" : " 17 GB", | 20260508 10:22:27.926 | INFO | "huge" : "", | 20260508 10:22:27.926 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:27.926 | INFO | "malloc" : " 14 GB", | 20260508 10:22:27.926 | INFO | "pinned" : " 29 GB" | 20260508 10:22:27.926 | INFO | } | 20260508 10:22:27.926 | INFO | Perf report for the last 5.14441 seconds: | 20260508 10:22:27.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:27.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 10:22:27.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:22:27.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:27.926 | INFO | Uploaded 36GiB to GPU in 2.41422s (14.9117GiB/s) | 20260508 10:22:27.926 | INFO | Downloaded 34GiB to GPU in 2.43412s (13.9681GiB/s) | 20260508 10:22:33.062 | INFO | Analyzing chunk frames = [7354368,7354880) | 20260508 10:22:33.062 | INFO | MemInfo: { | 20260508 10:22:33.062 | INFO | "gpu" : " 17 GB", | 20260508 10:22:33.062 | INFO | "huge" : "", | 20260508 10:22:33.063 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:33.063 | INFO | "malloc" : " 14 GB", | 20260508 10:22:33.063 | INFO | "pinned" : " 29 GB" | 20260508 10:22:33.063 | INFO | } | 20260508 10:22:33.063 | INFO | Perf report for the last 5.1366 seconds: | 20260508 10:22:33.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:33.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 10:22:33.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:22:33.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:33.063 | INFO | Uploaded 36GiB to GPU in 2.42575s (14.8408GiB/s) | 20260508 10:22:33.063 | INFO | Downloaded 34GiB to GPU in 2.53183s (13.429GiB/s) | 20260508 10:22:38.207 | INFO | Analyzing chunk frames = [7354880,7355392) | 20260508 10:22:38.207 | INFO | MemInfo: { | 20260508 10:22:38.207 | INFO | "gpu" : " 17 GB", | 20260508 10:22:38.207 | INFO | "huge" : "", | 20260508 10:22:38.207 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:38.207 | INFO | "malloc" : " 14 GB", | 20260508 10:22:38.207 | INFO | "pinned" : " 29 GB" | 20260508 10:22:38.207 | INFO | } | 20260508 10:22:38.207 | INFO | Perf report for the last 5.14456 seconds: | 20260508 10:22:38.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:38.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.207 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:22:38.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.207 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:22:38.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:38.208 | INFO | Uploaded 36GiB to GPU in 2.3597s (15.2562GiB/s) | 20260508 10:22:38.208 | INFO | Downloaded 34GiB to GPU in 2.44545s (13.9034GiB/s) | 20260508 10:22:43.345 | INFO | Analyzing chunk frames = [7355392,7355904) | 20260508 10:22:43.345 | INFO | MemInfo: { | 20260508 10:22:43.345 | INFO | "gpu" : " 17 GB", | 20260508 10:22:43.345 | INFO | "huge" : "", | 20260508 10:22:43.345 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:43.345 | INFO | "malloc" : " 14 GB", | 20260508 10:22:43.345 | INFO | "pinned" : " 29 GB" | 20260508 10:22:43.345 | INFO | } | 20260508 10:22:43.345 | INFO | Perf report for the last 5.13811 seconds: | 20260508 10:22:43.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:43.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.346 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:22:43.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.346 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:22:43.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:43.346 | INFO | Uploaded 36GiB to GPU in 2.43137s (14.8065GiB/s) | 20260508 10:22:43.346 | INFO | Downloaded 34GiB to GPU in 2.4517s (13.8679GiB/s) | 20260508 10:22:48.476 | INFO | Analyzing chunk frames = [7355904,7356416) | 20260508 10:22:48.476 | INFO | MemInfo: { | 20260508 10:22:48.476 | INFO | "gpu" : " 17 GB", | 20260508 10:22:48.476 | INFO | "huge" : "", | 20260508 10:22:48.476 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:48.476 | INFO | "malloc" : " 14 GB", | 20260508 10:22:48.476 | INFO | "pinned" : " 29 GB" | 20260508 10:22:48.476 | INFO | } | 20260508 10:22:48.476 | INFO | Perf report for the last 5.13082 seconds: | 20260508 10:22:48.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:48.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.476 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 10:22:48.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.476 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:22:48.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:48.477 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:22:48.477 | INFO | Total mem usage: 16.7 GiB | 20260508 10:22:48.477 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:22:48.477 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:22:48.477 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:22:48.477 | INFO | High Water Breakdown: | 20260508 10:22:48.477 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:22:48.477 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:22:48.477 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:22:48.477 | INFO | ------------------------------------------ | 20260508 10:22:48.477 | INFO | Uploaded 36GiB to GPU in 2.4515s (14.6849GiB/s) | 20260508 10:22:48.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:22:48.477 | INFO | Downloaded 34GiB to GPU in 2.45111s (13.8713GiB/s) | 20260508 10:22:48.477 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:22:48.477 | INFO | Total mem usage: 26.5 GiB | 20260508 10:22:48.477 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:22:48.477 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:22:48.477 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:22:48.477 | INFO | High Water Breakdown: | 20260508 10:22:48.477 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:22:48.477 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:22:48.477 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:22:48.477 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:22:48.477 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:22:48.477 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:22:48.477 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:22:48.477 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:22:48.477 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:22:48.477 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:22:48.477 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:22:48.477 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:22:48.477 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:22:48.477 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:22:48.477 | INFO | ------------------------------------------ | 20260508 10:22:48.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:22:48.477 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:22:48.477 | INFO | Total mem usage: 27.8 GiB | 20260508 10:22:48.477 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:22:48.477 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:22:48.477 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:22:48.477 | INFO | High Water Breakdown: | 20260508 10:22:48.477 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:22:48.477 | INFO | ------------------------------------------ | 20260508 10:22:48.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:22:48.477 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:22:48.477 | INFO | Total mem usage: 15.7 GiB | 20260508 10:22:48.477 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:22:48.478 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:22:48.478 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:22:48.478 | INFO | High Water Breakdown: | 20260508 10:22:48.478 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:22:48.478 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:22:48.478 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:22:48.478 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:22:48.478 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:22:48.478 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:22:48.478 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:22:48.478 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:22:48.478 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:22:48.478 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:22:48.478 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:22:48.478 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:22:48.478 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:22:48.478 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:22:48.478 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:22:48.478 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:22:48.478 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:22:48.478 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:22:48.478 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:22:48.478 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:22:48.478 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:22:48.478 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:22:48.478 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:22:48.478 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:22:48.478 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:22:48.478 | INFO | ------------------------------------------ | 20260508 10:22:48.478 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:22:48.478 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:22:48.478 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:22:48.478 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:22:48.478 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.41e+01 / 7.95e-01 | 20260508 10:22:48.478 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.02e-01, 1.20e-02 / 5.54e-03 | 20260508 10:22:48.478 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.44e+00, 5.96e-01 / 2.26e-01 | 20260508 10:22:48.478 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.40e+00, 2.08e-01 / 1.48e-01 | 20260508 10:22:48.478 | INFO | PerfReport: 15.97%, DME -- 4.19e+01 / 4.51e+01, 3.78e+00 / 9.83e-01 | 20260508 10:22:48.478 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.84e+00 / 1.32e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.47e+00, 4.06e-01 / 2.59e-01 | 20260508 10:22:48.478 | INFO | PerfReport: 3.01%, Metrics -- 9.85e-01 / 1.15e+00, 2.43e-01 / 1.20e-01 | 20260508 10:22:48.478 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.34e-01, 4.76e-02 / 2.69e-02 | 20260508 10:22:48.478 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:22:48.478 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:22:48.478 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:22:48.478 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 24.13%, Upload -- 2.54e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 31.16%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 1.98%, LockUpload -- 2.09e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 2.66%, LockDownload -- 2.79e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:22:48.478 | INFO | PerfReport: 0.01%, Misc -- 1.18e-02 / 0.00e+00, 1.00e-02 / 0.00e+00 | 20260508 10:22:48.574 | INFO | Starting a round of Baz IO | 20260508 10:22:51.981 | INFO | Populating RTMetrics report | 20260508 10:22:51.981 | INFO | RTMetrics report populated | 20260508 10:22:51.981 | INFO | Opening tmp RTMetrics json file | 20260508 10:22:51.993 | INFO | Serializing RTMetrics to json | 20260508 10:22:51.995 | INFO | Serialization done, swapping tmp file now... | 20260508 10:22:51.995 | INFO | RTMetrics done | 20260508 10:22:52.053 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.479 seconds | 20260508 10:22:52.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.21% / 0.00% / 59.79% | 20260508 10:22:52.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.223831% / 3.97364e-06% / 7.15256e-05% | 20260508 10:22:52.053 | INFO | Baz IO overall: | 20260508 10:22:52.053 | INFO | BytesWritten : 3193443618387 | 20260508 10:22:52.054 | INFO | NumEvents : 1535087405508 | 20260508 10:22:53.619 | INFO | Analyzing chunk frames = [7356416,7356928) | 20260508 10:22:53.619 | INFO | MemInfo: { | 20260508 10:22:53.619 | INFO | "gpu" : " 17 GB", | 20260508 10:22:53.619 | INFO | "huge" : "", | 20260508 10:22:53.619 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:53.619 | INFO | "malloc" : " 14 GB", | 20260508 10:22:53.619 | INFO | "pinned" : " 29 GB" | 20260508 10:22:53.619 | INFO | } | 20260508 10:22:53.619 | INFO | Perf report for the last 5.14303 seconds: | 20260508 10:22:53.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:53.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:53.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:53.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:53.619 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:22:53.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:53.619 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 10:22:53.620 | INFO | PRE_HQ : 65.8%, 1.7, 57.5ms, 0 | 20260508 10:22:53.620 | INFO | BAZWRITER : 67.2%, 2.6, 89.1ms, 0 | 20260508 10:22:53.620 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:22:53.620 | INFO | Uploaded 36GiB to GPU in 2.58462s (13.9285GiB/s) | 20260508 10:22:53.620 | INFO | Downloaded 38GiB to GPU in 2.69481s (14.1012GiB/s) | 20260508 10:22:58.758 | INFO | Analyzing chunk frames = [7356928,7357440) | 20260508 10:22:58.758 | INFO | MemInfo: { | 20260508 10:22:58.758 | INFO | "gpu" : " 17 GB", | 20260508 10:22:58.758 | INFO | "huge" : "", | 20260508 10:22:58.758 | INFO | "huge_shared" : " 29 GB", | 20260508 10:22:58.759 | INFO | "malloc" : " 14 GB", | 20260508 10:22:58.759 | INFO | "pinned" : " 29 GB" | 20260508 10:22:58.759 | INFO | } | 20260508 10:22:58.759 | INFO | Perf report for the last 5.13936 seconds: | 20260508 10:22:58.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:22:58.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 10:22:58.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:22:58.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:22:58.759 | INFO | Uploaded 36GiB to GPU in 2.50258s (14.3852GiB/s) | 20260508 10:22:58.759 | INFO | Downloaded 34GiB to GPU in 2.46771s (13.778GiB/s) | 20260508 10:23:03.901 | INFO | Analyzing chunk frames = [7357440,7357952) | 20260508 10:23:03.901 | INFO | MemInfo: { | 20260508 10:23:03.901 | INFO | "gpu" : " 17 GB", | 20260508 10:23:03.901 | INFO | "huge" : "", | 20260508 10:23:03.901 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:03.901 | INFO | "malloc" : " 14 GB", | 20260508 10:23:03.901 | INFO | "pinned" : " 29 GB" | 20260508 10:23:03.901 | INFO | } | 20260508 10:23:03.901 | INFO | Perf report for the last 5.14248 seconds: | 20260508 10:23:03.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:03.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 10:23:03.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:23:03.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:03.901 | INFO | Uploaded 36GiB to GPU in 2.5038s (14.3781GiB/s) | 20260508 10:23:03.902 | INFO | Downloaded 34GiB to GPU in 2.42018s (14.0486GiB/s) | 20260508 10:23:09.036 | INFO | Analyzing chunk frames = [7357952,7358464) | 20260508 10:23:09.036 | INFO | MemInfo: { | 20260508 10:23:09.036 | INFO | "gpu" : " 17 GB", | 20260508 10:23:09.036 | INFO | "huge" : "", | 20260508 10:23:09.036 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:09.036 | INFO | "malloc" : " 14 GB", | 20260508 10:23:09.036 | INFO | "pinned" : " 29 GB" | 20260508 10:23:09.036 | INFO | } | 20260508 10:23:09.036 | INFO | Perf report for the last 5.1351 seconds: | 20260508 10:23:09.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:09.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.036 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 10:23:09.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.036 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:23:09.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:09.037 | INFO | Uploaded 36GiB to GPU in 2.48569s (14.4829GiB/s) | 20260508 10:23:09.037 | INFO | Downloaded 34GiB to GPU in 2.40734s (14.1235GiB/s) | 20260508 10:23:14.181 | INFO | Analyzing chunk frames = [7358464,7358976) | 20260508 10:23:14.181 | INFO | MemInfo: { | 20260508 10:23:14.181 | INFO | "gpu" : " 17 GB", | 20260508 10:23:14.181 | INFO | "huge" : "", | 20260508 10:23:14.181 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:14.181 | INFO | "malloc" : " 14 GB", | 20260508 10:23:14.181 | INFO | "pinned" : " 29 GB" | 20260508 10:23:14.181 | INFO | } | 20260508 10:23:14.182 | INFO | Perf report for the last 5.14523 seconds: | 20260508 10:23:14.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:14.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 10:23:14.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:23:14.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:14.182 | INFO | Uploaded 36GiB to GPU in 2.40986s (14.9386GiB/s) | 20260508 10:23:14.182 | INFO | Downloaded 34GiB to GPU in 2.51939s (13.4953GiB/s) | 20260508 10:23:19.317 | INFO | Analyzing chunk frames = [7358976,7359488) | 20260508 10:23:19.317 | INFO | MemInfo: { | 20260508 10:23:19.317 | INFO | "gpu" : " 17 GB", | 20260508 10:23:19.317 | INFO | "huge" : "", | 20260508 10:23:19.317 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:19.317 | INFO | "malloc" : " 14 GB", | 20260508 10:23:19.317 | INFO | "pinned" : " 29 GB" | 20260508 10:23:19.317 | INFO | } | 20260508 10:23:19.317 | INFO | Perf report for the last 5.13572 seconds: | 20260508 10:23:19.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:19.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.317 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:23:19.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.317 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:23:19.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:19.318 | INFO | Uploaded 36GiB to GPU in 2.35026s (15.3174GiB/s) | 20260508 10:23:19.318 | INFO | Downloaded 34GiB to GPU in 2.4277s (14.005GiB/s) | 20260508 10:23:24.458 | INFO | Analyzing chunk frames = [7359488,7360000) | 20260508 10:23:24.458 | INFO | MemInfo: { | 20260508 10:23:24.458 | INFO | "gpu" : " 17 GB", | 20260508 10:23:24.458 | INFO | "huge" : "", | 20260508 10:23:24.458 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:24.458 | INFO | "malloc" : " 14 GB", | 20260508 10:23:24.458 | INFO | "pinned" : " 29 GB" | 20260508 10:23:24.458 | INFO | } | 20260508 10:23:24.458 | INFO | Perf report for the last 5.14072 seconds: | 20260508 10:23:24.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:24.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:23:24.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:23:24.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:24.458 | INFO | Uploaded 36GiB to GPU in 2.41857s (14.8848GiB/s) | 20260508 10:23:24.458 | INFO | Downloaded 34GiB to GPU in 2.43535s (13.961GiB/s) | 20260508 10:23:29.594 | INFO | Analyzing chunk frames = [7360000,7360512) | 20260508 10:23:29.594 | INFO | MemInfo: { | 20260508 10:23:29.594 | INFO | "gpu" : " 17 GB", | 20260508 10:23:29.594 | INFO | "huge" : "", | 20260508 10:23:29.594 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:29.594 | INFO | "malloc" : " 14 GB", | 20260508 10:23:29.594 | INFO | "pinned" : " 29 GB" | 20260508 10:23:29.594 | INFO | } | 20260508 10:23:29.594 | INFO | Perf report for the last 5.13592 seconds: | 20260508 10:23:29.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:29.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:23:29.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:23:29.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:29.594 | INFO | Uploaded 36GiB to GPU in 2.43888s (14.7609GiB/s) | 20260508 10:23:29.594 | INFO | Downloaded 34GiB to GPU in 2.44238s (13.9209GiB/s) | 20260508 10:23:29.691 | INFO | Starting a round of Baz IO | 20260508 10:23:33.103 | INFO | Populating RTMetrics report | 20260508 10:23:33.103 | INFO | RTMetrics report populated | 20260508 10:23:33.103 | INFO | Opening tmp RTMetrics json file | 20260508 10:23:33.114 | INFO | Serializing RTMetrics to json | 20260508 10:23:33.115 | INFO | Serialization done, swapping tmp file now... | 20260508 10:23:33.115 | INFO | RTMetrics done | 20260508 10:23:33.173 | INFO | Finished a round of Baz IO: Wrote 1.468 GiB in 3.482 seconds | 20260508 10:23:33.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.79% | 20260508 10:23:33.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227002% / 0% / 7.54992e-05% | 20260508 10:23:33.173 | INFO | Baz IO overall: | 20260508 10:23:33.173 | INFO | BytesWritten : 3195019437769 | 20260508 10:23:33.173 | INFO | NumEvents : 1535830638966 | 20260508 10:23:34.734 | INFO | Analyzing chunk frames = [7360512,7361024) | 20260508 10:23:34.734 | INFO | MemInfo: { | 20260508 10:23:34.734 | INFO | "gpu" : " 17 GB", | 20260508 10:23:34.734 | INFO | "huge" : "", | 20260508 10:23:34.734 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:34.734 | INFO | "malloc" : " 14 GB", | 20260508 10:23:34.734 | INFO | "pinned" : " 29 GB" | 20260508 10:23:34.734 | INFO | } | 20260508 10:23:34.734 | INFO | Perf report for the last 5.14047 seconds: | 20260508 10:23:34.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:34.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:34.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:34.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:34.734 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:23:34.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:34.735 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:23:34.735 | INFO | PRE_HQ : 65.5%, 1.6, 54.6ms, 0 | 20260508 10:23:34.735 | INFO | BAZWRITER : 67.0%, 2.5, 87.2ms, 0 | 20260508 10:23:34.735 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:23:34.735 | INFO | Uploaded 36GiB to GPU in 2.59534s (13.871GiB/s) | 20260508 10:23:34.735 | INFO | Downloaded 38GiB to GPU in 2.71905s (13.9755GiB/s) | 20260508 10:23:39.878 | INFO | Analyzing chunk frames = [7361024,7361536) | 20260508 10:23:39.878 | INFO | MemInfo: { | 20260508 10:23:39.878 | INFO | "gpu" : " 17 GB", | 20260508 10:23:39.878 | INFO | "huge" : "", | 20260508 10:23:39.878 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:39.878 | INFO | "malloc" : " 14 GB", | 20260508 10:23:39.878 | INFO | "pinned" : " 29 GB" | 20260508 10:23:39.878 | INFO | } | 20260508 10:23:39.878 | INFO | Perf report for the last 5.14407 seconds: | 20260508 10:23:39.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:39.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 10:23:39.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:23:39.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:39.879 | INFO | Uploaded 36GiB to GPU in 2.41613s (14.8999GiB/s) | 20260508 10:23:39.879 | INFO | Downloaded 34GiB to GPU in 2.47971s (13.7113GiB/s) | 20260508 10:23:45.016 | INFO | Analyzing chunk frames = [7361536,7362048) | 20260508 10:23:45.016 | INFO | MemInfo: { | 20260508 10:23:45.016 | INFO | "gpu" : " 17 GB", | 20260508 10:23:45.016 | INFO | "huge" : "", | 20260508 10:23:45.016 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:45.016 | INFO | "malloc" : " 14 GB", | 20260508 10:23:45.016 | INFO | "pinned" : " 29 GB" | 20260508 10:23:45.016 | INFO | } | 20260508 10:23:45.016 | INFO | Perf report for the last 5.13755 seconds: | 20260508 10:23:45.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:45.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:23:45.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:23:45.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:45.016 | INFO | Uploaded 36GiB to GPU in 2.453s (14.6759GiB/s) | 20260508 10:23:45.016 | INFO | Downloaded 34GiB to GPU in 2.4617s (13.8116GiB/s) | 20260508 10:23:50.159 | INFO | Analyzing chunk frames = [7362048,7362560) | 20260508 10:23:50.159 | INFO | MemInfo: { | 20260508 10:23:50.159 | INFO | "gpu" : " 17 GB", | 20260508 10:23:50.159 | INFO | "huge" : "", | 20260508 10:23:50.159 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:50.160 | INFO | "malloc" : " 14 GB", | 20260508 10:23:50.160 | INFO | "pinned" : " 29 GB" | 20260508 10:23:50.160 | INFO | } | 20260508 10:23:50.160 | INFO | Perf report for the last 5.14366 seconds: | 20260508 10:23:50.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:50.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 10:23:50.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:23:50.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:50.160 | INFO | Uploaded 36GiB to GPU in 2.42461s (14.8478GiB/s) | 20260508 10:23:50.160 | INFO | Downloaded 34GiB to GPU in 2.42305s (14.0319GiB/s) | 20260508 10:23:55.291 | INFO | Analyzing chunk frames = [7362560,7363072) | 20260508 10:23:55.291 | INFO | MemInfo: { | 20260508 10:23:55.291 | INFO | "gpu" : " 17 GB", | 20260508 10:23:55.291 | INFO | "huge" : "", | 20260508 10:23:55.291 | INFO | "huge_shared" : " 29 GB", | 20260508 10:23:55.291 | INFO | "malloc" : " 14 GB", | 20260508 10:23:55.291 | INFO | "pinned" : " 29 GB" | 20260508 10:23:55.291 | INFO | } | 20260508 10:23:55.291 | INFO | Perf report for the last 5.13156 seconds: | 20260508 10:23:55.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:23:55.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.291 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:23:55.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.291 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:23:55.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:23:55.292 | INFO | Uploaded 36GiB to GPU in 2.37465s (15.1601GiB/s) | 20260508 10:23:55.292 | INFO | Downloaded 34GiB to GPU in 2.54619s (13.3533GiB/s) | 20260508 10:24:00.435 | INFO | Analyzing chunk frames = [7363072,7363584) | 20260508 10:24:00.435 | INFO | MemInfo: { | 20260508 10:24:00.435 | INFO | "gpu" : " 17 GB", | 20260508 10:24:00.435 | INFO | "huge" : "", | 20260508 10:24:00.435 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:00.435 | INFO | "malloc" : " 14 GB", | 20260508 10:24:00.435 | INFO | "pinned" : " 29 GB" | 20260508 10:24:00.435 | INFO | } | 20260508 10:24:00.435 | INFO | Perf report for the last 5.14372 seconds: | 20260508 10:24:00.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:00.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:24:00.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 10:24:00.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:00.435 | INFO | Uploaded 36GiB to GPU in 2.36683s (15.2102GiB/s) | 20260508 10:24:00.435 | INFO | Downloaded 34GiB to GPU in 2.43671s (13.9532GiB/s) | 20260508 10:24:05.572 | INFO | Analyzing chunk frames = [7363584,7364096) | 20260508 10:24:05.572 | INFO | MemInfo: { | 20260508 10:24:05.572 | INFO | "gpu" : " 17 GB", | 20260508 10:24:05.572 | INFO | "huge" : "", | 20260508 10:24:05.572 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:05.572 | INFO | "malloc" : " 14 GB", | 20260508 10:24:05.572 | INFO | "pinned" : " 29 GB" | 20260508 10:24:05.572 | INFO | } | 20260508 10:24:05.572 | INFO | Perf report for the last 5.13733 seconds: | 20260508 10:24:05.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:05.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.572 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 10:24:05.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.572 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:24:05.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:05.573 | INFO | Uploaded 36GiB to GPU in 2.39857s (15.0089GiB/s) | 20260508 10:24:05.573 | INFO | Downloaded 34GiB to GPU in 2.44085s (13.9296GiB/s) | 20260508 10:24:10.711 | INFO | Analyzing chunk frames = [7364096,7364608) | 20260508 10:24:10.711 | INFO | MemInfo: { | 20260508 10:24:10.711 | INFO | "gpu" : " 17 GB", | 20260508 10:24:10.711 | INFO | "huge" : "", | 20260508 10:24:10.711 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:10.711 | INFO | "malloc" : " 14 GB", | 20260508 10:24:10.711 | INFO | "pinned" : " 29 GB" | 20260508 10:24:10.711 | INFO | } | 20260508 10:24:10.712 | INFO | Perf report for the last 5.13912 seconds: | 20260508 10:24:10.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:10.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:24:10.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:24:10.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:10.712 | INFO | Uploaded 36GiB to GPU in 2.44046s (14.7513GiB/s) | 20260508 10:24:10.712 | INFO | Downloaded 34GiB to GPU in 2.45863s (13.8288GiB/s) | 20260508 10:24:10.712 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:24:10.712 | INFO | Total mem usage: 16.7 GiB | 20260508 10:24:10.712 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:24:10.712 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:24:10.712 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:24:10.712 | INFO | High Water Breakdown: | 20260508 10:24:10.712 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:24:10.712 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:24:10.712 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:24:10.712 | INFO | ------------------------------------------ | 20260508 10:24:10.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:24:10.712 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:24:10.712 | INFO | Total mem usage: 26.5 GiB | 20260508 10:24:10.712 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:24:10.712 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:24:10.712 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:24:10.712 | INFO | High Water Breakdown: | 20260508 10:24:10.712 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:24:10.712 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:24:10.712 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:24:10.712 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:24:10.712 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:24:10.712 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:24:10.712 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:24:10.712 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:24:10.712 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:24:10.712 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:24:10.712 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:24:10.712 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:24:10.712 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:24:10.712 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:24:10.712 | INFO | ------------------------------------------ | 20260508 10:24:10.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:24:10.712 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:24:10.712 | INFO | Total mem usage: 27.8 GiB | 20260508 10:24:10.712 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:24:10.712 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:24:10.712 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:24:10.712 | INFO | High Water Breakdown: | 20260508 10:24:10.712 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:24:10.712 | INFO | ------------------------------------------ | 20260508 10:24:10.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:24:10.712 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:24:10.712 | INFO | Total mem usage: 15.7 GiB | 20260508 10:24:10.712 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:24:10.712 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:24:10.712 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:24:10.712 | INFO | High Water Breakdown: | 20260508 10:24:10.712 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:24:10.712 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:24:10.712 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:24:10.712 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:24:10.712 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:24:10.712 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:24:10.712 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:24:10.712 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:24:10.712 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:24:10.712 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:24:10.712 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:24:10.712 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:24:10.712 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:24:10.712 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:24:10.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:24:10.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:24:10.712 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:24:10.712 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:24:10.712 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:24:10.712 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:24:10.712 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:24:10.712 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:24:10.712 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:24:10.712 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:24:10.712 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:24:10.712 | INFO | ------------------------------------------ | 20260508 10:24:10.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:24:10.712 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:24:10.712 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:24:10.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:24:10.712 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 6.24e-01 | 20260508 10:24:10.712 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.14e-02 / 4.83e-03 | 20260508 10:24:10.712 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.47e+00, 5.92e-01 / 2.31e-01 | 20260508 10:24:10.712 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.47e+00, 2.11e-01 / 1.30e-01 | 20260508 10:24:10.712 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.49e+01, 3.73e+00 / 1.05e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.80e+00 / 1.41e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.49e+00, 4.03e-01 / 2.95e-01 | 20260508 10:24:10.712 | INFO | PerfReport: 3.03%, Metrics -- 9.91e-01 / 1.15e+00, 2.48e-01 / 1.33e-01 | 20260508 10:24:10.712 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.39e-01, 4.81e-02 / 3.10e-02 | 20260508 10:24:10.712 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:24:10.712 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:24:10.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:24:10.712 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 24.25%, Upload -- 2.55e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 24.63%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 31.09%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 2.14%, LockUpload -- 2.25e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 2.62%, LockDownload -- 2.76e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 15.26%, LockCompute -- 1.60e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 10:24:10.712 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260508 10:24:10.810 | INFO | Starting a round of Baz IO | 20260508 10:24:14.229 | INFO | Populating RTMetrics report | 20260508 10:24:14.229 | INFO | RTMetrics report populated | 20260508 10:24:14.229 | INFO | Opening tmp RTMetrics json file | 20260508 10:24:14.238 | INFO | Serializing RTMetrics to json | 20260508 10:24:14.240 | INFO | Serialization done, swapping tmp file now... | 20260508 10:24:14.240 | INFO | RTMetrics done | 20260508 10:24:14.300 | INFO | Finished a round of Baz IO: Wrote 1.465 GiB in 3.49 seconds | 20260508 10:24:14.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:24:14.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22678% / 3.97364e-06% / 4.76837e-05% | 20260508 10:24:14.300 | INFO | Baz IO overall: | 20260508 10:24:14.300 | INFO | BytesWritten : 3196592371665 | 20260508 10:24:14.300 | INFO | NumEvents : 1536572464188 | 20260508 10:24:15.851 | INFO | Analyzing chunk frames = [7364608,7365120) | 20260508 10:24:15.851 | INFO | MemInfo: { | 20260508 10:24:15.851 | INFO | "gpu" : " 17 GB", | 20260508 10:24:15.851 | INFO | "huge" : "", | 20260508 10:24:15.851 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:15.851 | INFO | "malloc" : " 14 GB", | 20260508 10:24:15.851 | INFO | "pinned" : " 29 GB" | 20260508 10:24:15.851 | INFO | } | 20260508 10:24:15.851 | INFO | Perf report for the last 5.13955 seconds: | 20260508 10:24:15.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:15.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:15.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:15.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:15.851 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:24:15.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:15.851 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:24:15.851 | INFO | PRE_HQ : 65.5%, 1.6, 54.3ms, 0 | 20260508 10:24:15.851 | INFO | BAZWRITER : 67.2%, 2.5, 86.4ms, 0 | 20260508 10:24:15.851 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:24:15.851 | INFO | Uploaded 36GiB to GPU in 2.60537s (13.8176GiB/s) | 20260508 10:24:15.852 | INFO | Downloaded 38GiB to GPU in 2.71695s (13.9863GiB/s) | 20260508 10:24:20.988 | INFO | Analyzing chunk frames = [7365120,7365632) | 20260508 10:24:20.988 | INFO | MemInfo: { | 20260508 10:24:20.988 | INFO | "gpu" : " 17 GB", | 20260508 10:24:20.988 | INFO | "huge" : "", | 20260508 10:24:20.988 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:20.988 | INFO | "malloc" : " 14 GB", | 20260508 10:24:20.988 | INFO | "pinned" : " 29 GB" | 20260508 10:24:20.988 | INFO | } | 20260508 10:24:20.988 | INFO | Perf report for the last 5.1367 seconds: | 20260508 10:24:20.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:20.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | ANALYSIS : 66.9%, 3.0, 104.1ms, 0 | 20260508 10:24:20.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:24:20.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:20.988 | INFO | Uploaded 36GiB to GPU in 2.52254s (14.2713GiB/s) | 20260508 10:24:20.988 | INFO | Downloaded 34GiB to GPU in 2.45524s (13.8479GiB/s) | 20260508 10:24:26.128 | INFO | Analyzing chunk frames = [7365632,7366144) | 20260508 10:24:26.128 | INFO | MemInfo: { | 20260508 10:24:26.128 | INFO | "gpu" : " 17 GB", | 20260508 10:24:26.128 | INFO | "huge" : "", | 20260508 10:24:26.128 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:26.128 | INFO | "malloc" : " 14 GB", | 20260508 10:24:26.128 | INFO | "pinned" : " 29 GB" | 20260508 10:24:26.128 | INFO | } | 20260508 10:24:26.128 | INFO | Perf report for the last 5.14008 seconds: | 20260508 10:24:26.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:26.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:24:26.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:24:26.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:26.128 | INFO | Uploaded 36GiB to GPU in 2.43679s (14.7735GiB/s) | 20260508 10:24:26.128 | INFO | Downloaded 34GiB to GPU in 2.429s (13.9976GiB/s) | 20260508 10:24:31.269 | INFO | Analyzing chunk frames = [7366144,7366656) | 20260508 10:24:31.269 | INFO | MemInfo: { | 20260508 10:24:31.269 | INFO | "gpu" : " 17 GB", | 20260508 10:24:31.269 | INFO | "huge" : "", | 20260508 10:24:31.269 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:31.269 | INFO | "malloc" : " 14 GB", | 20260508 10:24:31.269 | INFO | "pinned" : " 29 GB" | 20260508 10:24:31.269 | INFO | } | 20260508 10:24:31.269 | INFO | Perf report for the last 5.14113 seconds: | 20260508 10:24:31.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:31.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 10:24:31.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:24:31.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:31.269 | INFO | Uploaded 36GiB to GPU in 2.48827s (14.4679GiB/s) | 20260508 10:24:31.269 | INFO | Downloaded 34GiB to GPU in 2.42801s (14.0033GiB/s) | 20260508 10:24:36.406 | INFO | Analyzing chunk frames = [7366656,7367168) | 20260508 10:24:36.406 | INFO | MemInfo: { | 20260508 10:24:36.406 | INFO | "gpu" : " 17 GB", | 20260508 10:24:36.406 | INFO | "huge" : "", | 20260508 10:24:36.406 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:36.406 | INFO | "malloc" : " 14 GB", | 20260508 10:24:36.406 | INFO | "pinned" : " 29 GB" | 20260508 10:24:36.406 | INFO | } | 20260508 10:24:36.406 | INFO | Perf report for the last 5.137 seconds: | 20260508 10:24:36.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:36.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 10:24:36.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:24:36.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:36.406 | INFO | Uploaded 36GiB to GPU in 2.37251s (15.1738GiB/s) | 20260508 10:24:36.406 | INFO | Downloaded 34GiB to GPU in 2.54602s (13.3542GiB/s) | 20260508 10:24:41.547 | INFO | Analyzing chunk frames = [7367168,7367680) | 20260508 10:24:41.547 | INFO | MemInfo: { | 20260508 10:24:41.547 | INFO | "gpu" : " 17 GB", | 20260508 10:24:41.547 | INFO | "huge" : "", | 20260508 10:24:41.547 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:41.547 | INFO | "malloc" : " 14 GB", | 20260508 10:24:41.547 | INFO | "pinned" : " 29 GB" | 20260508 10:24:41.547 | INFO | } | 20260508 10:24:41.547 | INFO | Perf report for the last 5.14081 seconds: | 20260508 10:24:41.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:41.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 10:24:41.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:24:41.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:41.547 | INFO | Uploaded 36GiB to GPU in 2.32087s (15.5114GiB/s) | 20260508 10:24:41.548 | INFO | Downloaded 34GiB to GPU in 2.45092s (13.8724GiB/s) | 20260508 10:24:46.689 | INFO | Analyzing chunk frames = [7367680,7368192) | 20260508 10:24:46.689 | INFO | MemInfo: { | 20260508 10:24:46.689 | INFO | "gpu" : " 17 GB", | 20260508 10:24:46.689 | INFO | "huge" : "", | 20260508 10:24:46.689 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:46.689 | INFO | "malloc" : " 14 GB", | 20260508 10:24:46.689 | INFO | "pinned" : " 29 GB" | 20260508 10:24:46.689 | INFO | } | 20260508 10:24:46.689 | INFO | Perf report for the last 5.14199 seconds: | 20260508 10:24:46.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:46.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.689 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:24:46.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.689 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:24:46.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:46.690 | INFO | Uploaded 36GiB to GPU in 2.41493s (14.9073GiB/s) | 20260508 10:24:46.690 | INFO | Downloaded 34GiB to GPU in 2.45819s (13.8313GiB/s) | 20260508 10:24:51.833 | INFO | Analyzing chunk frames = [7368192,7368704) | 20260508 10:24:51.833 | INFO | MemInfo: { | 20260508 10:24:51.833 | INFO | "gpu" : " 17 GB", | 20260508 10:24:51.833 | INFO | "huge" : "", | 20260508 10:24:51.833 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:51.833 | INFO | "malloc" : " 14 GB", | 20260508 10:24:51.833 | INFO | "pinned" : " 29 GB" | 20260508 10:24:51.833 | INFO | } | 20260508 10:24:51.833 | INFO | Perf report for the last 5.14436 seconds: | 20260508 10:24:51.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:51.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.833 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 10:24:51.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.833 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:24:51.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:51.834 | INFO | Uploaded 36GiB to GPU in 2.39191s (15.0508GiB/s) | 20260508 10:24:51.834 | INFO | Downloaded 34GiB to GPU in 2.43667s (13.9535GiB/s) | 20260508 10:24:51.931 | INFO | Starting a round of Baz IO | 20260508 10:24:55.330 | INFO | Populating RTMetrics report | 20260508 10:24:55.331 | INFO | RTMetrics report populated | 20260508 10:24:55.331 | INFO | Opening tmp RTMetrics json file | 20260508 10:24:55.341 | INFO | Serializing RTMetrics to json | 20260508 10:24:55.342 | INFO | Serialization done, swapping tmp file now... | 20260508 10:24:55.342 | INFO | RTMetrics done | 20260508 10:24:55.402 | INFO | Finished a round of Baz IO: Wrote 1.463 GiB in 3.47 seconds | 20260508 10:24:55.402 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:24:55.402 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225743% / 0% / 4.37101e-05% | 20260508 10:24:55.402 | INFO | Baz IO overall: | 20260508 10:24:55.402 | INFO | BytesWritten : 3198163784571 | 20260508 10:24:55.402 | INFO | NumEvents : 1537313564235 | 20260508 10:24:56.968 | INFO | Analyzing chunk frames = [7368704,7369216) | 20260508 10:24:56.968 | INFO | MemInfo: { | 20260508 10:24:56.968 | INFO | "gpu" : " 17 GB", | 20260508 10:24:56.968 | INFO | "huge" : "", | 20260508 10:24:56.968 | INFO | "huge_shared" : " 29 GB", | 20260508 10:24:56.968 | INFO | "malloc" : " 14 GB", | 20260508 10:24:56.968 | INFO | "pinned" : " 29 GB" | 20260508 10:24:56.968 | INFO | } | 20260508 10:24:56.968 | INFO | Perf report for the last 5.13456 seconds: | 20260508 10:24:56.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:24:56.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:56.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:56.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:56.968 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 10:24:56.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:24:56.968 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:24:56.968 | INFO | PRE_HQ : 65.5%, 1.6, 54.2ms, 0 | 20260508 10:24:56.968 | INFO | BAZWRITER : 66.9%, 2.5, 86.6ms, 0 | 20260508 10:24:56.968 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:24:56.968 | INFO | Uploaded 36GiB to GPU in 2.61181s (13.7835GiB/s) | 20260508 10:24:56.968 | INFO | Downloaded 38GiB to GPU in 2.72536s (13.9431GiB/s) | 20260508 10:25:02.099 | INFO | Analyzing chunk frames = [7369216,7369728) | 20260508 10:25:02.099 | INFO | MemInfo: { | 20260508 10:25:02.099 | INFO | "gpu" : " 17 GB", | 20260508 10:25:02.099 | INFO | "huge" : "", | 20260508 10:25:02.099 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:02.099 | INFO | "malloc" : " 14 GB", | 20260508 10:25:02.099 | INFO | "pinned" : " 29 GB" | 20260508 10:25:02.099 | INFO | } | 20260508 10:25:02.099 | INFO | Perf report for the last 5.13164 seconds: | 20260508 10:25:02.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:02.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.100 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 10:25:02.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.100 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:25:02.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:02.100 | INFO | Uploaded 36GiB to GPU in 2.43849s (14.7632GiB/s) | 20260508 10:25:02.100 | INFO | Downloaded 34GiB to GPU in 2.48374s (13.689GiB/s) | 20260508 10:25:07.241 | INFO | Analyzing chunk frames = [7369728,7370240) | 20260508 10:25:07.241 | INFO | MemInfo: { | 20260508 10:25:07.241 | INFO | "gpu" : " 17 GB", | 20260508 10:25:07.241 | INFO | "huge" : "", | 20260508 10:25:07.241 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:07.241 | INFO | "malloc" : " 14 GB", | 20260508 10:25:07.242 | INFO | "pinned" : " 29 GB" | 20260508 10:25:07.242 | INFO | } | 20260508 10:25:07.242 | INFO | Perf report for the last 5.1421 seconds: | 20260508 10:25:07.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:07.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:25:07.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:25:07.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:07.242 | INFO | Uploaded 36GiB to GPU in 2.44713s (14.7111GiB/s) | 20260508 10:25:07.242 | INFO | Downloaded 34GiB to GPU in 2.44843s (13.8864GiB/s) | 20260508 10:25:12.381 | INFO | Analyzing chunk frames = [7370240,7370752) | 20260508 10:25:12.381 | INFO | MemInfo: { | 20260508 10:25:12.381 | INFO | "gpu" : " 17 GB", | 20260508 10:25:12.381 | INFO | "huge" : "", | 20260508 10:25:12.381 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:12.381 | INFO | "malloc" : " 14 GB", | 20260508 10:25:12.381 | INFO | "pinned" : " 29 GB" | 20260508 10:25:12.381 | INFO | } | 20260508 10:25:12.381 | INFO | Perf report for the last 5.13935 seconds: | 20260508 10:25:12.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:12.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:25:12.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:25:12.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:12.381 | INFO | Uploaded 36GiB to GPU in 2.42317s (14.8565GiB/s) | 20260508 10:25:12.381 | INFO | Downloaded 34GiB to GPU in 2.42487s (14.0214GiB/s) | 20260508 10:25:17.524 | INFO | Analyzing chunk frames = [7370752,7371264) | 20260508 10:25:17.524 | INFO | MemInfo: { | 20260508 10:25:17.524 | INFO | "gpu" : " 17 GB", | 20260508 10:25:17.524 | INFO | "huge" : "", | 20260508 10:25:17.524 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:17.524 | INFO | "malloc" : " 14 GB", | 20260508 10:25:17.525 | INFO | "pinned" : " 29 GB" | 20260508 10:25:17.525 | INFO | } | 20260508 10:25:17.525 | INFO | Perf report for the last 5.14381 seconds: | 20260508 10:25:17.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:17.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 10:25:17.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:25:17.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:17.525 | INFO | Uploaded 36GiB to GPU in 2.37463s (15.1603GiB/s) | 20260508 10:25:17.525 | INFO | Downloaded 34GiB to GPU in 2.51636s (13.5116GiB/s) | 20260508 10:25:22.661 | INFO | Analyzing chunk frames = [7371264,7371776) | 20260508 10:25:22.661 | INFO | MemInfo: { | 20260508 10:25:22.661 | INFO | "gpu" : " 17 GB", | 20260508 10:25:22.661 | INFO | "huge" : "", | 20260508 10:25:22.661 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:22.661 | INFO | "malloc" : " 14 GB", | 20260508 10:25:22.661 | INFO | "pinned" : " 29 GB" | 20260508 10:25:22.661 | INFO | } | 20260508 10:25:22.661 | INFO | Perf report for the last 5.13656 seconds: | 20260508 10:25:22.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:22.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.661 | INFO | ANALYSIS : 65.1%, 3.0, 101.2ms, 0 | 20260508 10:25:22.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.661 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:25:22.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:22.662 | INFO | Uploaded 36GiB to GPU in 2.30785s (15.5989GiB/s) | 20260508 10:25:22.662 | INFO | Downloaded 34GiB to GPU in 2.44805s (13.8886GiB/s) | 20260508 10:25:27.798 | INFO | Analyzing chunk frames = [7371776,7372288) | 20260508 10:25:27.798 | INFO | MemInfo: { | 20260508 10:25:27.798 | INFO | "gpu" : " 17 GB", | 20260508 10:25:27.798 | INFO | "huge" : "", | 20260508 10:25:27.798 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:27.798 | INFO | "malloc" : " 14 GB", | 20260508 10:25:27.798 | INFO | "pinned" : " 29 GB" | 20260508 10:25:27.798 | INFO | } | 20260508 10:25:27.798 | INFO | Perf report for the last 5.13691 seconds: | 20260508 10:25:27.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:27.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.798 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 10:25:27.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.798 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:25:27.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:27.799 | INFO | Uploaded 36GiB to GPU in 2.38073s (15.1214GiB/s) | 20260508 10:25:27.799 | INFO | Downloaded 34GiB to GPU in 2.44128s (13.9271GiB/s) | 20260508 10:25:32.943 | INFO | Analyzing chunk frames = [7372288,7372800) | 20260508 10:25:32.943 | INFO | MemInfo: { | 20260508 10:25:32.943 | INFO | "gpu" : " 17 GB", | 20260508 10:25:32.943 | INFO | "huge" : "", | 20260508 10:25:32.943 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:32.943 | INFO | "malloc" : " 14 GB", | 20260508 10:25:32.943 | INFO | "pinned" : " 29 GB" | 20260508 10:25:32.943 | INFO | } | 20260508 10:25:32.943 | INFO | Perf report for the last 5.14458 seconds: | 20260508 10:25:32.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:32.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 10:25:32.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:25:32.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:32.943 | INFO | Uploaded 36GiB to GPU in 2.40552s (14.9656GiB/s) | 20260508 10:25:32.943 | INFO | Downloaded 34GiB to GPU in 2.45395s (13.8552GiB/s) | 20260508 10:25:32.943 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:25:32.943 | INFO | Total mem usage: 16.7 GiB | 20260508 10:25:32.943 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:25:32.943 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:25:32.943 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:25:32.943 | INFO | High Water Breakdown: | 20260508 10:25:32.943 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:25:32.943 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:25:32.943 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:25:32.943 | INFO | ------------------------------------------ | 20260508 10:25:32.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:25:32.943 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:25:32.944 | INFO | Total mem usage: 26.5 GiB | 20260508 10:25:32.944 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:25:32.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:25:32.944 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:25:32.944 | INFO | High Water Breakdown: | 20260508 10:25:32.944 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:25:32.944 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:25:32.944 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:25:32.944 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:25:32.944 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:25:32.944 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:25:32.944 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:25:32.944 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:25:32.944 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:25:32.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:25:32.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:25:32.944 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:25:32.944 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:25:32.944 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:25:32.944 | INFO | ------------------------------------------ | 20260508 10:25:32.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:25:32.944 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:25:32.944 | INFO | Total mem usage: 27.8 GiB | 20260508 10:25:32.944 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:25:32.944 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:25:32.944 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:25:32.944 | INFO | High Water Breakdown: | 20260508 10:25:32.944 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:25:32.944 | INFO | ------------------------------------------ | 20260508 10:25:32.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:25:32.944 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:25:32.944 | INFO | Total mem usage: 15.7 GiB | 20260508 10:25:32.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:25:32.944 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:25:32.944 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:25:32.944 | INFO | High Water Breakdown: | 20260508 10:25:32.944 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:25:32.944 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:25:32.944 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:25:32.944 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:25:32.944 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:25:32.944 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:25:32.944 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:25:32.944 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:25:32.944 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:25:32.944 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:25:32.944 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:25:32.944 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:25:32.944 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:25:32.944 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:25:32.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:25:32.944 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:25:32.944 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:25:32.944 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:25:32.944 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:25:32.944 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:25:32.944 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:25:32.944 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:25:32.944 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:25:32.944 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:25:32.944 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:25:32.944 | INFO | ------------------------------------------ | 20260508 10:25:32.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:25:32.944 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:25:32.944 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:25:32.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:25:32.944 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.38e+01 / 6.22e-01 | 20260508 10:25:32.944 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.21e-02 / 4.64e-03 | 20260508 10:25:32.944 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.46e+00, 5.96e-01 / 2.24e-01 | 20260508 10:25:32.944 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.47e+00, 2.09e-01 / 1.19e-01 | 20260508 10:25:32.944 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.50e+01, 3.68e+00 / 1.14e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 1.33e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.49e+00, 4.05e-01 / 2.79e-01 | 20260508 10:25:32.944 | INFO | PerfReport: 3.02%, Metrics -- 9.88e-01 / 1.12e+00, 2.47e-01 / 1.40e-01 | 20260508 10:25:32.944 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.34e-01, 4.91e-02 / 2.97e-02 | 20260508 10:25:32.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:25:32.944 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:25:32.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:25:32.944 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 24.12%, Upload -- 2.54e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 24.69%, Download -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 31.14%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 2.05%, LockUpload -- 2.15e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 10:25:32.944 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:25:32.945 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 1.07e-02 / 0.00e+00 | 20260508 10:25:33.040 | INFO | Starting a round of Baz IO | 20260508 10:25:36.460 | INFO | Populating RTMetrics report | 20260508 10:25:36.460 | INFO | RTMetrics report populated | 20260508 10:25:36.460 | INFO | Opening tmp RTMetrics json file | 20260508 10:25:36.470 | INFO | Serializing RTMetrics to json | 20260508 10:25:36.471 | INFO | Serialization done, swapping tmp file now... | 20260508 10:25:36.472 | INFO | RTMetrics done | 20260508 10:25:36.526 | INFO | Finished a round of Baz IO: Wrote 1.462 GiB in 3.486 seconds | 20260508 10:25:36.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:25:36.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224447% / 0% / 9.53674e-05% | 20260508 10:25:36.527 | INFO | Baz IO overall: | 20260508 10:25:36.527 | INFO | BytesWritten : 3199733553817 | 20260508 10:25:36.527 | INFO | NumEvents : 1538054015991 | 20260508 10:25:38.081 | INFO | Analyzing chunk frames = [7372800,7373312) | 20260508 10:25:38.081 | INFO | MemInfo: { | 20260508 10:25:38.081 | INFO | "gpu" : " 17 GB", | 20260508 10:25:38.081 | INFO | "huge" : "", | 20260508 10:25:38.081 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:38.081 | INFO | "malloc" : " 14 GB", | 20260508 10:25:38.081 | INFO | "pinned" : " 29 GB" | 20260508 10:25:38.081 | INFO | } | 20260508 10:25:38.081 | INFO | Perf report for the last 5.13839 seconds: | 20260508 10:25:38.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:38.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:38.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:38.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:38.081 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:25:38.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:38.081 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:25:38.082 | INFO | PRE_HQ : 65.5%, 1.6, 54.3ms, 0 | 20260508 10:25:38.082 | INFO | BAZWRITER : 67.1%, 2.5, 86.2ms, 0 | 20260508 10:25:38.082 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:25:38.082 | INFO | Uploaded 36GiB to GPU in 2.59681s (13.8631GiB/s) | 20260508 10:25:38.082 | INFO | Downloaded 38GiB to GPU in 2.69748s (14.0872GiB/s) | 20260508 10:25:43.217 | INFO | Analyzing chunk frames = [7373312,7373824) | 20260508 10:25:43.217 | INFO | MemInfo: { | 20260508 10:25:43.217 | INFO | "gpu" : " 17 GB", | 20260508 10:25:43.217 | INFO | "huge" : "", | 20260508 10:25:43.217 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:43.217 | INFO | "malloc" : " 14 GB", | 20260508 10:25:43.217 | INFO | "pinned" : " 29 GB" | 20260508 10:25:43.217 | INFO | } | 20260508 10:25:43.217 | INFO | Perf report for the last 5.13576 seconds: | 20260508 10:25:43.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:43.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.217 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 10:25:43.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.217 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:25:43.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:43.218 | INFO | Uploaded 36GiB to GPU in 2.46787s (14.5875GiB/s) | 20260508 10:25:43.218 | INFO | Downloaded 34GiB to GPU in 2.46968s (13.767GiB/s) | 20260508 10:25:48.358 | INFO | Analyzing chunk frames = [7373824,7374336) | 20260508 10:25:48.358 | INFO | MemInfo: { | 20260508 10:25:48.358 | INFO | "gpu" : " 17 GB", | 20260508 10:25:48.358 | INFO | "huge" : "", | 20260508 10:25:48.358 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:48.358 | INFO | "malloc" : " 14 GB", | 20260508 10:25:48.358 | INFO | "pinned" : " 29 GB" | 20260508 10:25:48.358 | INFO | } | 20260508 10:25:48.358 | INFO | Perf report for the last 5.14118 seconds: | 20260508 10:25:48.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:48.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.358 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:25:48.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.358 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:25:48.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:48.359 | INFO | Uploaded 36GiB to GPU in 2.45417s (14.6689GiB/s) | 20260508 10:25:48.359 | INFO | Downloaded 34GiB to GPU in 2.42903s (13.9973GiB/s) | 20260508 10:25:53.498 | INFO | Analyzing chunk frames = [7374336,7374848) | 20260508 10:25:53.498 | INFO | MemInfo: { | 20260508 10:25:53.498 | INFO | "gpu" : " 17 GB", | 20260508 10:25:53.498 | INFO | "huge" : "", | 20260508 10:25:53.498 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:53.498 | INFO | "malloc" : " 14 GB", | 20260508 10:25:53.498 | INFO | "pinned" : " 29 GB" | 20260508 10:25:53.498 | INFO | } | 20260508 10:25:53.498 | INFO | Perf report for the last 5.1397 seconds: | 20260508 10:25:53.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:53.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.498 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 10:25:53.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.498 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:25:53.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:53.499 | INFO | Uploaded 36GiB to GPU in 2.46149s (14.6253GiB/s) | 20260508 10:25:53.499 | INFO | Downloaded 34GiB to GPU in 2.41942s (14.0529GiB/s) | 20260508 10:25:58.637 | INFO | Analyzing chunk frames = [7374848,7375360) | 20260508 10:25:58.637 | INFO | MemInfo: { | 20260508 10:25:58.637 | INFO | "gpu" : " 17 GB", | 20260508 10:25:58.637 | INFO | "huge" : "", | 20260508 10:25:58.637 | INFO | "huge_shared" : " 29 GB", | 20260508 10:25:58.637 | INFO | "malloc" : " 14 GB", | 20260508 10:25:58.638 | INFO | "pinned" : " 29 GB" | 20260508 10:25:58.638 | INFO | } | 20260508 10:25:58.638 | INFO | Perf report for the last 5.13965 seconds: | 20260508 10:25:58.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:25:58.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 10:25:58.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:25:58.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:25:58.638 | INFO | Uploaded 36GiB to GPU in 2.41616s (14.8997GiB/s) | 20260508 10:25:58.638 | INFO | Downloaded 34GiB to GPU in 2.52463s (13.4673GiB/s) | 20260508 10:26:03.778 | INFO | Analyzing chunk frames = [7375360,7375872) | 20260508 10:26:03.778 | INFO | MemInfo: { | 20260508 10:26:03.778 | INFO | "gpu" : " 17 GB", | 20260508 10:26:03.778 | INFO | "huge" : "", | 20260508 10:26:03.778 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:03.779 | INFO | "malloc" : " 14 GB", | 20260508 10:26:03.779 | INFO | "pinned" : " 29 GB" | 20260508 10:26:03.779 | INFO | } | 20260508 10:26:03.779 | INFO | Perf report for the last 5.14103 seconds: | 20260508 10:26:03.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:03.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 10:26:03.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:26:03.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:03.779 | INFO | Uploaded 36GiB to GPU in 2.3954s (15.0288GiB/s) | 20260508 10:26:03.779 | INFO | Downloaded 34GiB to GPU in 2.42954s (13.9944GiB/s) | 20260508 10:26:08.916 | INFO | Analyzing chunk frames = [7375872,7376384) | 20260508 10:26:08.916 | INFO | MemInfo: { | 20260508 10:26:08.916 | INFO | "gpu" : " 17 GB", | 20260508 10:26:08.916 | INFO | "huge" : "", | 20260508 10:26:08.916 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:08.917 | INFO | "malloc" : " 14 GB", | 20260508 10:26:08.917 | INFO | "pinned" : " 29 GB" | 20260508 10:26:08.917 | INFO | } | 20260508 10:26:08.917 | INFO | Perf report for the last 5.13811 seconds: | 20260508 10:26:08.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:08.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:26:08.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:26:08.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:08.917 | INFO | Uploaded 36GiB to GPU in 2.44625s (14.7164GiB/s) | 20260508 10:26:08.917 | INFO | Downloaded 34GiB to GPU in 2.44126s (13.9272GiB/s) | 20260508 10:26:14.048 | INFO | Analyzing chunk frames = [7376384,7376896) | 20260508 10:26:14.048 | INFO | MemInfo: { | 20260508 10:26:14.048 | INFO | "gpu" : " 17 GB", | 20260508 10:26:14.048 | INFO | "huge" : "", | 20260508 10:26:14.048 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:14.048 | INFO | "malloc" : " 14 GB", | 20260508 10:26:14.048 | INFO | "pinned" : " 29 GB" | 20260508 10:26:14.049 | INFO | } | 20260508 10:26:14.049 | INFO | Perf report for the last 5.13179 seconds: | 20260508 10:26:14.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:14.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 10:26:14.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 10:26:14.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:14.049 | INFO | Uploaded 36GiB to GPU in 2.4397s (14.7559GiB/s) | 20260508 10:26:14.049 | INFO | Downloaded 34GiB to GPU in 2.42551s (14.0177GiB/s) | 20260508 10:26:14.162 | INFO | Starting a round of Baz IO | 20260508 10:26:17.570 | INFO | Populating RTMetrics report | 20260508 10:26:17.570 | INFO | RTMetrics report populated | 20260508 10:26:17.570 | INFO | Opening tmp RTMetrics json file | 20260508 10:26:17.571 | INFO | Serializing RTMetrics to json | 20260508 10:26:17.572 | INFO | Serialization done, swapping tmp file now... | 20260508 10:26:17.572 | INFO | RTMetrics done | 20260508 10:26:17.634 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.472 seconds | 20260508 10:26:17.634 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:26:17.634 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22409% / 7.94729e-06% / 6.35783e-05% | 20260508 10:26:17.634 | INFO | Baz IO overall: | 20260508 10:26:17.634 | INFO | BytesWritten : 3201302509592 | 20260508 10:26:17.635 | INFO | NumEvents : 1538794161042 | 20260508 10:26:19.195 | INFO | Analyzing chunk frames = [7376896,7377408) | 20260508 10:26:19.195 | INFO | MemInfo: { | 20260508 10:26:19.195 | INFO | "gpu" : " 17 GB", | 20260508 10:26:19.195 | INFO | "huge" : "", | 20260508 10:26:19.195 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:19.195 | INFO | "malloc" : " 14 GB", | 20260508 10:26:19.195 | INFO | "pinned" : " 29 GB" | 20260508 10:26:19.195 | INFO | } | 20260508 10:26:19.195 | INFO | Perf report for the last 5.14629 seconds: | 20260508 10:26:19.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:19.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:19.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:19.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:19.195 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 10:26:19.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:19.195 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260508 10:26:19.195 | INFO | PRE_HQ : 66.7%, 1.8, 61.9ms, 0 | 20260508 10:26:19.195 | INFO | BAZWRITER : 66.8%, 2.7, 93.8ms, 0 | 20260508 10:26:19.195 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 10:26:19.195 | INFO | Uploaded 36GiB to GPU in 2.56656s (14.0266GiB/s) | 20260508 10:26:19.196 | INFO | Downloaded 38GiB to GPU in 2.70598s (14.043GiB/s) | 20260508 10:26:24.340 | INFO | Analyzing chunk frames = [7377408,7377920) | 20260508 10:26:24.340 | INFO | MemInfo: { | 20260508 10:26:24.340 | INFO | "gpu" : " 17 GB", | 20260508 10:26:24.340 | INFO | "huge" : "", | 20260508 10:26:24.340 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:24.340 | INFO | "malloc" : " 14 GB", | 20260508 10:26:24.340 | INFO | "pinned" : " 29 GB" | 20260508 10:26:24.340 | INFO | } | 20260508 10:26:24.340 | INFO | Perf report for the last 5.14497 seconds: | 20260508 10:26:24.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:24.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 10:26:24.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:26:24.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:24.340 | INFO | Uploaded 36GiB to GPU in 2.54391s (14.1514GiB/s) | 20260508 10:26:24.341 | INFO | Downloaded 34GiB to GPU in 2.4503s (13.8759GiB/s) | 20260508 10:26:29.477 | INFO | Analyzing chunk frames = [7377920,7378432) | 20260508 10:26:29.477 | INFO | MemInfo: { | 20260508 10:26:29.477 | INFO | "gpu" : " 17 GB", | 20260508 10:26:29.477 | INFO | "huge" : "", | 20260508 10:26:29.477 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:29.477 | INFO | "malloc" : " 14 GB", | 20260508 10:26:29.477 | INFO | "pinned" : " 29 GB" | 20260508 10:26:29.477 | INFO | } | 20260508 10:26:29.477 | INFO | Perf report for the last 5.13753 seconds: | 20260508 10:26:29.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:29.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.477 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 10:26:29.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.478 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:26:29.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:29.478 | INFO | Uploaded 36GiB to GPU in 2.50708s (14.3593GiB/s) | 20260508 10:26:29.478 | INFO | Downloaded 34GiB to GPU in 2.43477s (13.9644GiB/s) | 20260508 10:26:34.616 | INFO | Analyzing chunk frames = [7378432,7378944) | 20260508 10:26:34.616 | INFO | MemInfo: { | 20260508 10:26:34.616 | INFO | "gpu" : " 17 GB", | 20260508 10:26:34.617 | INFO | "huge" : "", | 20260508 10:26:34.617 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:34.617 | INFO | "malloc" : " 14 GB", | 20260508 10:26:34.617 | INFO | "pinned" : " 29 GB" | 20260508 10:26:34.617 | INFO | } | 20260508 10:26:34.617 | INFO | Perf report for the last 5.13928 seconds: | 20260508 10:26:34.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:34.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:26:34.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:26:34.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:34.617 | INFO | Uploaded 36GiB to GPU in 2.48407s (14.4923GiB/s) | 20260508 10:26:34.617 | INFO | Downloaded 34GiB to GPU in 2.43155s (13.9829GiB/s) | 20260508 10:26:39.754 | INFO | Analyzing chunk frames = [7378944,7379456) | 20260508 10:26:39.754 | INFO | MemInfo: { | 20260508 10:26:39.754 | INFO | "gpu" : " 17 GB", | 20260508 10:26:39.754 | INFO | "huge" : "", | 20260508 10:26:39.754 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:39.754 | INFO | "malloc" : " 14 GB", | 20260508 10:26:39.754 | INFO | "pinned" : " 29 GB" | 20260508 10:26:39.754 | INFO | } | 20260508 10:26:39.754 | INFO | Perf report for the last 5.13724 seconds: | 20260508 10:26:39.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:39.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:26:39.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:26:39.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:39.754 | INFO | Uploaded 36GiB to GPU in 2.37688s (15.1459GiB/s) | 20260508 10:26:39.754 | INFO | Downloaded 34GiB to GPU in 2.51604s (13.5133GiB/s) | 20260508 10:26:44.894 | INFO | Analyzing chunk frames = [7379456,7379968) | 20260508 10:26:44.894 | INFO | MemInfo: { | 20260508 10:26:44.894 | INFO | "gpu" : " 17 GB", | 20260508 10:26:44.894 | INFO | "huge" : "", | 20260508 10:26:44.894 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:44.894 | INFO | "malloc" : " 14 GB", | 20260508 10:26:44.894 | INFO | "pinned" : " 29 GB" | 20260508 10:26:44.894 | INFO | } | 20260508 10:26:44.895 | INFO | Perf report for the last 5.14055 seconds: | 20260508 10:26:44.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:44.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:26:44.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:26:44.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:44.895 | INFO | Uploaded 36GiB to GPU in 2.37385s (15.1652GiB/s) | 20260508 10:26:44.895 | INFO | Downloaded 34GiB to GPU in 2.42832s (14.0014GiB/s) | 20260508 10:26:50.029 | INFO | Analyzing chunk frames = [7379968,7380480) | 20260508 10:26:50.029 | INFO | MemInfo: { | 20260508 10:26:50.029 | INFO | "gpu" : " 17 GB", | 20260508 10:26:50.029 | INFO | "huge" : "", | 20260508 10:26:50.030 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:50.030 | INFO | "malloc" : " 14 GB", | 20260508 10:26:50.030 | INFO | "pinned" : " 29 GB" | 20260508 10:26:50.030 | INFO | } | 20260508 10:26:50.030 | INFO | Perf report for the last 5.13516 seconds: | 20260508 10:26:50.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:50.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:26:50.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:26:50.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:50.030 | INFO | Uploaded 36GiB to GPU in 2.42835s (14.8249GiB/s) | 20260508 10:26:50.030 | INFO | Downloaded 34GiB to GPU in 2.44227s (13.9214GiB/s) | 20260508 10:26:55.171 | INFO | Analyzing chunk frames = [7380480,7380992) | 20260508 10:26:55.171 | INFO | MemInfo: { | 20260508 10:26:55.171 | INFO | "gpu" : " 17 GB", | 20260508 10:26:55.171 | INFO | "huge" : "", | 20260508 10:26:55.172 | INFO | "huge_shared" : " 29 GB", | 20260508 10:26:55.172 | INFO | "malloc" : " 14 GB", | 20260508 10:26:55.172 | INFO | "pinned" : " 29 GB" | 20260508 10:26:55.172 | INFO | } | 20260508 10:26:55.172 | INFO | Perf report for the last 5.14201 seconds: | 20260508 10:26:55.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:26:55.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:26:55.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:26:55.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:26:55.172 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:26:55.172 | INFO | Total mem usage: 16.7 GiB | 20260508 10:26:55.172 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:26:55.172 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 10:26:55.172 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:26:55.172 | INFO | High Water Breakdown: | 20260508 10:26:55.172 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:26:55.172 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:26:55.172 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:26:55.172 | INFO | ------------------------------------------ | 20260508 10:26:55.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:26:55.172 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:26:55.172 | INFO | Total mem usage: 26.5 GiB | 20260508 10:26:55.172 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:26:55.172 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:26:55.172 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:26:55.172 | INFO | High Water Breakdown: | 20260508 10:26:55.172 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:26:55.172 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:26:55.172 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:26:55.172 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:26:55.172 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:26:55.172 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:26:55.172 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:26:55.172 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:26:55.172 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:26:55.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:26:55.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:26:55.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:26:55.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:26:55.172 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:26:55.172 | INFO | ------------------------------------------ | 20260508 10:26:55.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:26:55.172 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:26:55.172 | INFO | Total mem usage: 27.8 GiB | 20260508 10:26:55.172 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:26:55.172 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:26:55.172 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:26:55.172 | INFO | High Water Breakdown: | 20260508 10:26:55.172 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:26:55.172 | INFO | ------------------------------------------ | 20260508 10:26:55.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:26:55.172 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:26:55.172 | INFO | Total mem usage: 15.7 GiB | 20260508 10:26:55.172 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:26:55.172 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:26:55.172 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:26:55.172 | INFO | High Water Breakdown: | 20260508 10:26:55.172 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:26:55.172 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:26:55.172 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:26:55.172 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:26:55.172 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:26:55.172 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:26:55.172 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:26:55.172 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:26:55.172 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:26:55.172 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:26:55.172 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:26:55.172 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:26:55.172 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:26:55.172 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:26:55.172 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:26:55.172 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:26:55.172 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:26:55.172 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:26:55.172 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:26:55.172 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:26:55.172 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:26:55.172 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:26:55.172 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:26:55.172 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:26:55.172 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:26:55.172 | INFO | ------------------------------------------ | 20260508 10:26:55.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:26:55.172 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:26:55.172 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:26:55.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:26:55.172 | INFO | Uploaded 36GiB to GPU in 2.431s (14.8087GiB/s) | 20260508 10:26:55.173 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.08e-01 | 20260508 10:26:55.173 | INFO | Downloaded 34GiB to GPU in 2.4279s (14.0039GiB/s) | 20260508 10:26:55.173 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.07e-02 / 6.81e-03 | 20260508 10:26:55.173 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.41e+00, 5.94e-01 / 2.30e-01 | 20260508 10:26:55.173 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.45e+00, 2.13e-01 / 1.35e-01 | 20260508 10:26:55.173 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.52e+01, 3.69e+00 / 1.17e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.84e+00 / 1.39e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.05e-01 / 2.74e-01 | 20260508 10:26:55.173 | INFO | PerfReport: 3.02%, Metrics -- 9.89e-01 / 1.14e+00, 2.46e-01 / 1.17e-01 | 20260508 10:26:55.173 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.32e-01, 4.92e-02 / 2.78e-02 | 20260508 10:26:55.173 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:26:55.173 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:26:55.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:26:55.173 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 24.35%, Upload -- 2.56e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 24.52%, Download -- 2.58e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 31.09%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 2.44%, LockDownload -- 2.57e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 15.34%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 10:26:55.173 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.34e-02 / 0.00e+00 | 20260508 10:26:55.270 | INFO | Starting a round of Baz IO | 20260508 10:26:58.686 | INFO | Populating RTMetrics report | 20260508 10:26:58.687 | INFO | RTMetrics report populated | 20260508 10:26:58.687 | INFO | Opening tmp RTMetrics json file | 20260508 10:26:58.696 | INFO | Serializing RTMetrics to json | 20260508 10:26:58.697 | INFO | Serialization done, swapping tmp file now... | 20260508 10:26:58.698 | INFO | RTMetrics done | 20260508 10:26:58.756 | INFO | Finished a round of Baz IO: Wrote 1.463 GiB in 3.485 seconds | 20260508 10:26:58.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:26:58.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224495% / 0% / 5.16574e-05% | 20260508 10:26:58.756 | INFO | Baz IO overall: | 20260508 10:26:58.756 | INFO | BytesWritten : 3202873082775 | 20260508 10:26:58.756 | INFO | NumEvents : 1539535707063 | 20260508 10:27:00.304 | INFO | Analyzing chunk frames = [7380992,7381504) | 20260508 10:27:00.304 | INFO | MemInfo: { | 20260508 10:27:00.304 | INFO | "gpu" : " 17 GB", | 20260508 10:27:00.304 | INFO | "huge" : "", | 20260508 10:27:00.304 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:00.304 | INFO | "malloc" : " 14 GB", | 20260508 10:27:00.304 | INFO | "pinned" : " 29 GB" | 20260508 10:27:00.304 | INFO | } | 20260508 10:27:00.304 | INFO | Perf report for the last 5.13242 seconds: | 20260508 10:27:00.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:00.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:00.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:00.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:00.304 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:27:00.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:00.304 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:27:00.304 | INFO | PRE_HQ : 65.6%, 1.6, 54.5ms, 0 | 20260508 10:27:00.304 | INFO | BAZWRITER : 67.2%, 2.5, 87.8ms, 0 | 20260508 10:27:00.304 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:27:00.305 | INFO | Uploaded 36GiB to GPU in 2.58686s (13.9165GiB/s) | 20260508 10:27:00.305 | INFO | Downloaded 38GiB to GPU in 2.72052s (13.9679GiB/s) | 20260508 10:27:05.453 | INFO | Analyzing chunk frames = [7381504,7382016) | 20260508 10:27:05.453 | INFO | MemInfo: { | 20260508 10:27:05.453 | INFO | "gpu" : " 17 GB", | 20260508 10:27:05.453 | INFO | "huge" : "", | 20260508 10:27:05.453 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:05.453 | INFO | "malloc" : " 14 GB", | 20260508 10:27:05.453 | INFO | "pinned" : " 29 GB" | 20260508 10:27:05.453 | INFO | } | 20260508 10:27:05.453 | INFO | Perf report for the last 5.14872 seconds: | 20260508 10:27:05.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:05.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 10:27:05.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 10:27:05.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:05.453 | INFO | Uploaded 36GiB to GPU in 2.4875s (14.4724GiB/s) | 20260508 10:27:05.453 | INFO | Downloaded 34GiB to GPU in 2.46525s (13.7917GiB/s) | 20260508 10:27:10.594 | INFO | Analyzing chunk frames = [7382016,7382528) | 20260508 10:27:10.594 | INFO | MemInfo: { | 20260508 10:27:10.594 | INFO | "gpu" : " 17 GB", | 20260508 10:27:10.594 | INFO | "huge" : "", | 20260508 10:27:10.594 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:10.594 | INFO | "malloc" : " 14 GB", | 20260508 10:27:10.594 | INFO | "pinned" : " 29 GB" | 20260508 10:27:10.594 | INFO | } | 20260508 10:27:10.594 | INFO | Perf report for the last 5.14141 seconds: | 20260508 10:27:10.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:10.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.594 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:27:10.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.594 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:27:10.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:10.595 | INFO | Uploaded 36GiB to GPU in 2.46651s (14.5955GiB/s) | 20260508 10:27:10.595 | INFO | Downloaded 34GiB to GPU in 2.45734s (13.8361GiB/s) | 20260508 10:27:15.727 | INFO | Analyzing chunk frames = [7382528,7383040) | 20260508 10:27:15.727 | INFO | MemInfo: { | 20260508 10:27:15.727 | INFO | "gpu" : " 17 GB", | 20260508 10:27:15.727 | INFO | "huge" : "", | 20260508 10:27:15.727 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:15.727 | INFO | "malloc" : " 14 GB", | 20260508 10:27:15.727 | INFO | "pinned" : " 29 GB" | 20260508 10:27:15.727 | INFO | } | 20260508 10:27:15.727 | INFO | Perf report for the last 5.13305 seconds: | 20260508 10:27:15.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:15.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.727 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 10:27:15.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.727 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:27:15.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:15.728 | INFO | Uploaded 36GiB to GPU in 2.43577s (14.7797GiB/s) | 20260508 10:27:15.728 | INFO | Downloaded 34GiB to GPU in 2.46247s (13.8073GiB/s) | 20260508 10:27:20.862 | INFO | Analyzing chunk frames = [7383040,7383552) | 20260508 10:27:20.862 | INFO | MemInfo: { | 20260508 10:27:20.862 | INFO | "gpu" : " 17 GB", | 20260508 10:27:20.862 | INFO | "huge" : "", | 20260508 10:27:20.862 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:20.862 | INFO | "malloc" : " 14 GB", | 20260508 10:27:20.862 | INFO | "pinned" : " 29 GB" | 20260508 10:27:20.862 | INFO | } | 20260508 10:27:20.862 | INFO | Perf report for the last 5.13433 seconds: | 20260508 10:27:20.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:20.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 10:27:20.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:27:20.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:20.862 | INFO | Uploaded 36GiB to GPU in 2.39925s (15.0047GiB/s) | 20260508 10:27:20.862 | INFO | Downloaded 34GiB to GPU in 2.53772s (13.3979GiB/s) | 20260508 10:27:26.004 | INFO | Analyzing chunk frames = [7383552,7384064) | 20260508 10:27:26.004 | INFO | MemInfo: { | 20260508 10:27:26.004 | INFO | "gpu" : " 17 GB", | 20260508 10:27:26.005 | INFO | "huge" : "", | 20260508 10:27:26.005 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:26.005 | INFO | "malloc" : " 14 GB", | 20260508 10:27:26.005 | INFO | "pinned" : " 29 GB" | 20260508 10:27:26.005 | INFO | } | 20260508 10:27:26.005 | INFO | Perf report for the last 5.14308 seconds: | 20260508 10:27:26.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:26.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:27:26.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260508 10:27:26.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:26.005 | INFO | Uploaded 36GiB to GPU in 2.36841s (15.2GiB/s) | 20260508 10:27:26.005 | INFO | Downloaded 34GiB to GPU in 2.43928s (13.9385GiB/s) | 20260508 10:27:31.146 | INFO | Analyzing chunk frames = [7384064,7384576) | 20260508 10:27:31.146 | INFO | MemInfo: { | 20260508 10:27:31.146 | INFO | "gpu" : " 17 GB", | 20260508 10:27:31.146 | INFO | "huge" : "", | 20260508 10:27:31.146 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:31.146 | INFO | "malloc" : " 14 GB", | 20260508 10:27:31.146 | INFO | "pinned" : " 29 GB" | 20260508 10:27:31.146 | INFO | } | 20260508 10:27:31.146 | INFO | Perf report for the last 5.14126 seconds: | 20260508 10:27:31.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:31.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:27:31.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:27:31.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:31.146 | INFO | Uploaded 36GiB to GPU in 2.42618s (14.8381GiB/s) | 20260508 10:27:31.146 | INFO | Downloaded 34GiB to GPU in 2.44103s (13.9285GiB/s) | 20260508 10:27:36.293 | INFO | Analyzing chunk frames = [7384576,7385088) | 20260508 10:27:36.293 | INFO | MemInfo: { | 20260508 10:27:36.293 | INFO | "gpu" : " 17 GB", | 20260508 10:27:36.293 | INFO | "huge" : "", | 20260508 10:27:36.293 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:36.293 | INFO | "malloc" : " 14 GB", | 20260508 10:27:36.293 | INFO | "pinned" : " 29 GB" | 20260508 10:27:36.293 | INFO | } | 20260508 10:27:36.293 | INFO | Perf report for the last 5.1473 seconds: | 20260508 10:27:36.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:36.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.293 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:27:36.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.293 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:27:36.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:36.294 | INFO | Uploaded 36GiB to GPU in 2.40676s (14.9579GiB/s) | 20260508 10:27:36.294 | INFO | Downloaded 34GiB to GPU in 2.43441s (13.9664GiB/s) | 20260508 10:27:36.391 | INFO | Starting a round of Baz IO | 20260508 10:27:39.809 | INFO | Populating RTMetrics report | 20260508 10:27:39.809 | INFO | RTMetrics report populated | 20260508 10:27:39.809 | INFO | Opening tmp RTMetrics json file | 20260508 10:27:39.816 | INFO | Serializing RTMetrics to json | 20260508 10:27:39.817 | INFO | Serialization done, swapping tmp file now... | 20260508 10:27:39.818 | INFO | RTMetrics done | 20260508 10:27:39.879 | INFO | Finished a round of Baz IO: Wrote 1.46 GiB in 3.487 seconds | 20260508 10:27:39.879 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:27:39.879 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222405% / 3.97364e-06% / 5.16574e-05% | 20260508 10:27:39.879 | INFO | Baz IO overall: | 20260508 10:27:39.879 | INFO | BytesWritten : 3204441118084 | 20260508 10:27:39.879 | INFO | NumEvents : 1540275179261 | 20260508 10:27:41.429 | INFO | Analyzing chunk frames = [7385088,7385600) | 20260508 10:27:41.429 | INFO | MemInfo: { | 20260508 10:27:41.429 | INFO | "gpu" : " 17 GB", | 20260508 10:27:41.429 | INFO | "huge" : "", | 20260508 10:27:41.429 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:41.429 | INFO | "malloc" : " 14 GB", | 20260508 10:27:41.429 | INFO | "pinned" : " 29 GB" | 20260508 10:27:41.429 | INFO | } | 20260508 10:27:41.429 | INFO | Perf report for the last 5.13575 seconds: | 20260508 10:27:41.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:41.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:41.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:41.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:41.429 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:27:41.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:41.429 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:27:41.429 | INFO | PRE_HQ : 65.7%, 1.6, 54.6ms, 0 | 20260508 10:27:41.429 | INFO | BAZWRITER : 66.9%, 2.5, 87.1ms, 0 | 20260508 10:27:41.429 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:27:41.429 | INFO | Uploaded 36GiB to GPU in 2.6289s (13.6939GiB/s) | 20260508 10:27:41.429 | INFO | Downloaded 38GiB to GPU in 2.7225s (13.9577GiB/s) | 20260508 10:27:46.561 | INFO | Analyzing chunk frames = [7385600,7386112) | 20260508 10:27:46.561 | INFO | MemInfo: { | 20260508 10:27:46.561 | INFO | "gpu" : " 17 GB", | 20260508 10:27:46.561 | INFO | "huge" : "", | 20260508 10:27:46.561 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:46.561 | INFO | "malloc" : " 14 GB", | 20260508 10:27:46.561 | INFO | "pinned" : " 29 GB" | 20260508 10:27:46.561 | INFO | } | 20260508 10:27:46.561 | INFO | Perf report for the last 5.13184 seconds: | 20260508 10:27:46.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:46.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 10:27:46.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:27:46.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:46.561 | INFO | Uploaded 36GiB to GPU in 2.44248s (14.7391GiB/s) | 20260508 10:27:46.561 | INFO | Downloaded 34GiB to GPU in 2.47004s (13.765GiB/s) | 20260508 10:27:51.702 | INFO | Analyzing chunk frames = [7386112,7386624) | 20260508 10:27:51.702 | INFO | MemInfo: { | 20260508 10:27:51.702 | INFO | "gpu" : " 17 GB", | 20260508 10:27:51.702 | INFO | "huge" : "", | 20260508 10:27:51.702 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:51.702 | INFO | "malloc" : " 14 GB", | 20260508 10:27:51.702 | INFO | "pinned" : " 29 GB" | 20260508 10:27:51.702 | INFO | } | 20260508 10:27:51.702 | INFO | Perf report for the last 5.14091 seconds: | 20260508 10:27:51.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:51.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 10:27:51.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:27:51.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:51.702 | INFO | Uploaded 36GiB to GPU in 2.46167s (14.6242GiB/s) | 20260508 10:27:51.702 | INFO | Downloaded 34GiB to GPU in 2.43275s (13.9759GiB/s) | 20260508 10:27:56.841 | INFO | Analyzing chunk frames = [7386624,7387136) | 20260508 10:27:56.841 | INFO | MemInfo: { | 20260508 10:27:56.841 | INFO | "gpu" : " 17 GB", | 20260508 10:27:56.841 | INFO | "huge" : "", | 20260508 10:27:56.842 | INFO | "huge_shared" : " 29 GB", | 20260508 10:27:56.842 | INFO | "malloc" : " 14 GB", | 20260508 10:27:56.842 | INFO | "pinned" : " 29 GB" | 20260508 10:27:56.842 | INFO | } | 20260508 10:27:56.842 | INFO | Perf report for the last 5.13979 seconds: | 20260508 10:27:56.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:27:56.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 10:27:56.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:27:56.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:27:56.842 | INFO | Uploaded 36GiB to GPU in 2.44052s (14.7509GiB/s) | 20260508 10:27:56.842 | INFO | Downloaded 34GiB to GPU in 2.40932s (14.1119GiB/s) | 20260508 10:28:01.979 | INFO | Analyzing chunk frames = [7387136,7387648) | 20260508 10:28:01.979 | INFO | MemInfo: { | 20260508 10:28:01.979 | INFO | "gpu" : " 17 GB", | 20260508 10:28:01.979 | INFO | "huge" : "", | 20260508 10:28:01.979 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:01.979 | INFO | "malloc" : " 14 GB", | 20260508 10:28:01.979 | INFO | "pinned" : " 29 GB" | 20260508 10:28:01.979 | INFO | } | 20260508 10:28:01.979 | INFO | Perf report for the last 5.1374 seconds: | 20260508 10:28:01.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:01.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:28:01.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 10:28:01.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:01.979 | INFO | Uploaded 36GiB to GPU in 2.30634s (15.6091GiB/s) | 20260508 10:28:01.979 | INFO | Downloaded 34GiB to GPU in 2.51467s (13.5207GiB/s) | 20260508 10:28:07.122 | INFO | Analyzing chunk frames = [7387648,7388160) | 20260508 10:28:07.122 | INFO | MemInfo: { | 20260508 10:28:07.122 | INFO | "gpu" : " 17 GB", | 20260508 10:28:07.122 | INFO | "huge" : "", | 20260508 10:28:07.122 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:07.122 | INFO | "malloc" : " 14 GB", | 20260508 10:28:07.122 | INFO | "pinned" : " 29 GB" | 20260508 10:28:07.122 | INFO | } | 20260508 10:28:07.122 | INFO | Perf report for the last 5.14297 seconds: | 20260508 10:28:07.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:07.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.122 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 10:28:07.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.122 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:28:07.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:07.123 | INFO | Uploaded 36GiB to GPU in 2.33099s (15.444GiB/s) | 20260508 10:28:07.123 | INFO | Downloaded 34GiB to GPU in 2.4516s (13.8685GiB/s) | 20260508 10:28:12.262 | INFO | Analyzing chunk frames = [7388160,7388672) | 20260508 10:28:12.262 | INFO | MemInfo: { | 20260508 10:28:12.262 | INFO | "gpu" : " 17 GB", | 20260508 10:28:12.262 | INFO | "huge" : "", | 20260508 10:28:12.262 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:12.263 | INFO | "malloc" : " 14 GB", | 20260508 10:28:12.263 | INFO | "pinned" : " 29 GB" | 20260508 10:28:12.263 | INFO | } | 20260508 10:28:12.263 | INFO | Perf report for the last 5.14059 seconds: | 20260508 10:28:12.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:12.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 10:28:12.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:28:12.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:12.263 | INFO | Uploaded 36GiB to GPU in 2.40256s (14.984GiB/s) | 20260508 10:28:12.263 | INFO | Downloaded 34GiB to GPU in 2.454s (13.855GiB/s) | 20260508 10:28:17.401 | INFO | Analyzing chunk frames = [7388672,7389184) | 20260508 10:28:17.401 | INFO | MemInfo: { | 20260508 10:28:17.401 | INFO | "gpu" : " 17 GB", | 20260508 10:28:17.401 | INFO | "huge" : "", | 20260508 10:28:17.401 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:17.401 | INFO | "malloc" : " 14 GB", | 20260508 10:28:17.401 | INFO | "pinned" : " 29 GB" | 20260508 10:28:17.401 | INFO | } | 20260508 10:28:17.401 | INFO | Perf report for the last 5.13828 seconds: | 20260508 10:28:17.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:17.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 10:28:17.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:28:17.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:17.401 | INFO | Uploaded 36GiB to GPU in 2.39815s (15.0116GiB/s) | 20260508 10:28:17.401 | INFO | Downloaded 34GiB to GPU in 2.44835s (13.8869GiB/s) | 20260508 10:28:17.402 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:28:17.402 | INFO | Total mem usage: 16.7 GiB | 20260508 10:28:17.402 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:28:17.402 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:28:17.402 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:28:17.402 | INFO | High Water Breakdown: | 20260508 10:28:17.402 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:28:17.402 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:28:17.402 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:28:17.402 | INFO | ------------------------------------------ | 20260508 10:28:17.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:28:17.402 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:28:17.402 | INFO | Total mem usage: 26.5 GiB | 20260508 10:28:17.402 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:28:17.402 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:28:17.402 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:28:17.402 | INFO | High Water Breakdown: | 20260508 10:28:17.402 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:28:17.402 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:28:17.402 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:28:17.402 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:28:17.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:28:17.402 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:28:17.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:28:17.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:28:17.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:28:17.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:28:17.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:28:17.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:28:17.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:28:17.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:28:17.402 | INFO | ------------------------------------------ | 20260508 10:28:17.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:28:17.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:28:17.402 | INFO | Total mem usage: 27.8 GiB | 20260508 10:28:17.402 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:28:17.402 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:28:17.402 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:28:17.402 | INFO | High Water Breakdown: | 20260508 10:28:17.402 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:28:17.402 | INFO | ------------------------------------------ | 20260508 10:28:17.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:28:17.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:28:17.402 | INFO | Total mem usage: 15.7 GiB | 20260508 10:28:17.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:28:17.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:28:17.402 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:28:17.402 | INFO | High Water Breakdown: | 20260508 10:28:17.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:28:17.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:28:17.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:28:17.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:28:17.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:28:17.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:28:17.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:28:17.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:28:17.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:28:17.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:28:17.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:28:17.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:28:17.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:28:17.402 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:28:17.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:28:17.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:28:17.402 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:28:17.402 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:28:17.402 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:28:17.402 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:28:17.402 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:28:17.402 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:28:17.402 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:28:17.402 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:28:17.402 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:28:17.402 | INFO | ------------------------------------------ | 20260508 10:28:17.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:28:17.402 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:28:17.402 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:28:17.402 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:28:17.402 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 7.79e-01 | 20260508 10:28:17.402 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.20e-02 / 9.02e-03 | 20260508 10:28:17.402 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.48e+00, 5.96e-01 / 2.25e-01 | 20260508 10:28:17.402 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.15e-01 | 20260508 10:28:17.402 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.52e+01, 3.73e+00 / 1.12e+00 | 20260508 10:28:17.402 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.81e+00 / 1.12e+00 | 20260508 10:28:17.402 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.09e-01 / 2.37e-01 | 20260508 10:28:17.402 | INFO | PerfReport: 3.02%, Metrics -- 9.88e-01 / 1.11e+00, 2.44e-01 / 1.38e-01 | 20260508 10:28:17.402 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.30e-01, 4.87e-02 / 2.47e-02 | 20260508 10:28:17.403 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:28:17.403 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:28:17.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:28:17.403 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 24.14%, Upload -- 2.54e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 24.68%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 31.10%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 2.02%, LockUpload -- 2.13e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 2.71%, LockDownload -- 2.85e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 15.34%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:28:17.403 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 3.84e-02 / 0.00e+00 | 20260508 10:28:17.499 | INFO | Starting a round of Baz IO | 20260508 10:28:20.921 | INFO | Populating RTMetrics report | 20260508 10:28:20.921 | INFO | RTMetrics report populated | 20260508 10:28:20.921 | INFO | Opening tmp RTMetrics json file | 20260508 10:28:20.930 | INFO | Serializing RTMetrics to json | 20260508 10:28:20.932 | INFO | Serialization done, swapping tmp file now... | 20260508 10:28:20.932 | INFO | RTMetrics done | 20260508 10:28:20.986 | INFO | Finished a round of Baz IO: Wrote 1.459 GiB in 3.487 seconds | 20260508 10:28:20.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:28:20.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.223188% / 0% / 7.54992e-05% | 20260508 10:28:20.986 | INFO | Baz IO overall: | 20260508 10:28:20.986 | INFO | BytesWritten : 3206007290110 | 20260508 10:28:20.986 | INFO | NumEvents : 1541013540799 | 20260508 10:28:22.541 | INFO | Analyzing chunk frames = [7389184,7389696) | 20260508 10:28:22.541 | INFO | MemInfo: { | 20260508 10:28:22.541 | INFO | "gpu" : " 17 GB", | 20260508 10:28:22.541 | INFO | "huge" : "", | 20260508 10:28:22.541 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:22.541 | INFO | "malloc" : " 14 GB", | 20260508 10:28:22.541 | INFO | "pinned" : " 29 GB" | 20260508 10:28:22.541 | INFO | } | 20260508 10:28:22.542 | INFO | Perf report for the last 5.14051 seconds: | 20260508 10:28:22.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:22.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:22.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:22.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:22.542 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:28:22.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:22.542 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:28:22.542 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 10:28:22.542 | INFO | BAZWRITER : 67.2%, 2.5, 86.4ms, 0 | 20260508 10:28:22.542 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:28:22.542 | INFO | Uploaded 36GiB to GPU in 2.57709s (13.9693GiB/s) | 20260508 10:28:22.542 | INFO | Downloaded 38GiB to GPU in 2.7216s (13.9624GiB/s) | 20260508 10:28:27.676 | INFO | Analyzing chunk frames = [7389696,7390208) | 20260508 10:28:27.676 | INFO | MemInfo: { | 20260508 10:28:27.676 | INFO | "gpu" : " 17 GB", | 20260508 10:28:27.676 | INFO | "huge" : "", | 20260508 10:28:27.676 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:27.676 | INFO | "malloc" : " 14 GB", | 20260508 10:28:27.676 | INFO | "pinned" : " 29 GB" | 20260508 10:28:27.676 | INFO | } | 20260508 10:28:27.676 | INFO | Perf report for the last 5.13485 seconds: | 20260508 10:28:27.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:27.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.676 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:28:27.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.677 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:28:27.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:27.677 | INFO | Uploaded 36GiB to GPU in 2.42972s (14.8165GiB/s) | 20260508 10:28:27.677 | INFO | Downloaded 34GiB to GPU in 2.47033s (13.7633GiB/s) | 20260508 10:28:32.814 | INFO | Analyzing chunk frames = [7390208,7390720) | 20260508 10:28:32.814 | INFO | MemInfo: { | 20260508 10:28:32.814 | INFO | "gpu" : " 17 GB", | 20260508 10:28:32.814 | INFO | "huge" : "", | 20260508 10:28:32.814 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:32.815 | INFO | "malloc" : " 14 GB", | 20260508 10:28:32.815 | INFO | "pinned" : " 29 GB" | 20260508 10:28:32.815 | INFO | } | 20260508 10:28:32.815 | INFO | Perf report for the last 5.13841 seconds: | 20260508 10:28:32.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:32.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 10:28:32.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:28:32.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:32.815 | INFO | Uploaded 36GiB to GPU in 2.42469s (14.8473GiB/s) | 20260508 10:28:32.815 | INFO | Downloaded 34GiB to GPU in 2.4559s (13.8442GiB/s) | 20260508 10:28:37.958 | INFO | Analyzing chunk frames = [7390720,7391232) | 20260508 10:28:37.958 | INFO | MemInfo: { | 20260508 10:28:37.958 | INFO | "gpu" : " 17 GB", | 20260508 10:28:37.958 | INFO | "huge" : "", | 20260508 10:28:37.958 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:37.959 | INFO | "malloc" : " 14 GB", | 20260508 10:28:37.959 | INFO | "pinned" : " 29 GB" | 20260508 10:28:37.959 | INFO | } | 20260508 10:28:37.959 | INFO | Perf report for the last 5.14387 seconds: | 20260508 10:28:37.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:37.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 10:28:37.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260508 10:28:37.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:37.959 | INFO | Uploaded 36GiB to GPU in 2.43731s (14.7704GiB/s) | 20260508 10:28:37.959 | INFO | Downloaded 34GiB to GPU in 2.4334s (13.9722GiB/s) | 20260508 10:28:43.102 | INFO | Analyzing chunk frames = [7391232,7391744) | 20260508 10:28:43.102 | INFO | MemInfo: { | 20260508 10:28:43.102 | INFO | "gpu" : " 17 GB", | 20260508 10:28:43.102 | INFO | "huge" : "", | 20260508 10:28:43.102 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:43.102 | INFO | "malloc" : " 14 GB", | 20260508 10:28:43.103 | INFO | "pinned" : " 29 GB" | 20260508 10:28:43.103 | INFO | } | 20260508 10:28:43.103 | INFO | Perf report for the last 5.14386 seconds: | 20260508 10:28:43.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:43.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 10:28:43.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:28:43.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:43.103 | INFO | Uploaded 36GiB to GPU in 2.38852s (15.0721GiB/s) | 20260508 10:28:43.103 | INFO | Downloaded 34GiB to GPU in 2.5171s (13.5076GiB/s) | 20260508 10:28:48.239 | INFO | Analyzing chunk frames = [7391744,7392256) | 20260508 10:28:48.239 | INFO | MemInfo: { | 20260508 10:28:48.239 | INFO | "gpu" : " 17 GB", | 20260508 10:28:48.239 | INFO | "huge" : "", | 20260508 10:28:48.239 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:48.239 | INFO | "malloc" : " 14 GB", | 20260508 10:28:48.239 | INFO | "pinned" : " 29 GB" | 20260508 10:28:48.239 | INFO | } | 20260508 10:28:48.239 | INFO | Perf report for the last 5.13687 seconds: | 20260508 10:28:48.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:48.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.239 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:28:48.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.239 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:28:48.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:48.240 | INFO | Uploaded 36GiB to GPU in 2.38941s (15.0665GiB/s) | 20260508 10:28:48.240 | INFO | Downloaded 34GiB to GPU in 2.43794s (13.9462GiB/s) | 20260508 10:28:53.377 | INFO | Analyzing chunk frames = [7392256,7392768) | 20260508 10:28:53.377 | INFO | MemInfo: { | 20260508 10:28:53.377 | INFO | "gpu" : " 17 GB", | 20260508 10:28:53.377 | INFO | "huge" : "", | 20260508 10:28:53.377 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:53.377 | INFO | "malloc" : " 14 GB", | 20260508 10:28:53.377 | INFO | "pinned" : " 29 GB" | 20260508 10:28:53.377 | INFO | } | 20260508 10:28:53.377 | INFO | Perf report for the last 5.13767 seconds: | 20260508 10:28:53.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:53.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.377 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:28:53.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.378 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:28:53.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:53.378 | INFO | Uploaded 36GiB to GPU in 2.41736s (14.8923GiB/s) | 20260508 10:28:53.378 | INFO | Downloaded 34GiB to GPU in 2.44006s (13.9341GiB/s) | 20260508 10:28:58.513 | INFO | Analyzing chunk frames = [7392768,7393280) | 20260508 10:28:58.513 | INFO | MemInfo: { | 20260508 10:28:58.513 | INFO | "gpu" : " 17 GB", | 20260508 10:28:58.513 | INFO | "huge" : "", | 20260508 10:28:58.513 | INFO | "huge_shared" : " 29 GB", | 20260508 10:28:58.513 | INFO | "malloc" : " 14 GB", | 20260508 10:28:58.513 | INFO | "pinned" : " 29 GB" | 20260508 10:28:58.513 | INFO | } | 20260508 10:28:58.514 | INFO | Perf report for the last 5.13639 seconds: | 20260508 10:28:58.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:28:58.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 10:28:58.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:28:58.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:28:58.514 | INFO | Uploaded 36GiB to GPU in 2.44503s (14.7238GiB/s) | 20260508 10:28:58.514 | INFO | Downloaded 34GiB to GPU in 2.44776s (13.8903GiB/s) | 20260508 10:28:58.611 | INFO | Starting a round of Baz IO | 20260508 10:29:02.021 | INFO | Populating RTMetrics report | 20260508 10:29:02.021 | INFO | RTMetrics report populated | 20260508 10:29:02.021 | INFO | Opening tmp RTMetrics json file | 20260508 10:29:02.031 | INFO | Serializing RTMetrics to json | 20260508 10:29:02.032 | INFO | Serialization done, swapping tmp file now... | 20260508 10:29:02.033 | INFO | RTMetrics done | 20260508 10:29:02.094 | INFO | Finished a round of Baz IO: Wrote 1.46 GiB in 3.482 seconds | 20260508 10:29:02.094 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.00% / 59.80% | 20260508 10:29:02.094 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222218% / 0% / 7.15256e-05% | 20260508 10:29:02.094 | INFO | Baz IO overall: | 20260508 10:29:02.094 | INFO | BytesWritten : 3207574987815 | 20260508 10:29:02.094 | INFO | NumEvents : 1541752892153 | 20260508 10:29:03.654 | INFO | Analyzing chunk frames = [7393280,7393792) | 20260508 10:29:03.654 | INFO | MemInfo: { | 20260508 10:29:03.654 | INFO | "gpu" : " 17 GB", | 20260508 10:29:03.654 | INFO | "huge" : "", | 20260508 10:29:03.654 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:03.654 | INFO | "malloc" : " 14 GB", | 20260508 10:29:03.654 | INFO | "pinned" : " 29 GB" | 20260508 10:29:03.654 | INFO | } | 20260508 10:29:03.654 | INFO | Perf report for the last 5.14082 seconds: | 20260508 10:29:03.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:03.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:03.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:03.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:03.654 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:29:03.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:03.654 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:29:03.654 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 10:29:03.655 | INFO | BAZWRITER : 67.1%, 2.5, 86.4ms, 0 | 20260508 10:29:03.655 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:29:03.655 | INFO | Uploaded 36GiB to GPU in 2.59818s (13.8558GiB/s) | 20260508 10:29:03.655 | INFO | Downloaded 38GiB to GPU in 2.70792s (14.0329GiB/s) | 20260508 10:29:08.794 | INFO | Analyzing chunk frames = [7393792,7394304) | 20260508 10:29:08.794 | INFO | MemInfo: { | 20260508 10:29:08.794 | INFO | "gpu" : " 17 GB", | 20260508 10:29:08.794 | INFO | "huge" : "", | 20260508 10:29:08.794 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:08.794 | INFO | "malloc" : " 14 GB", | 20260508 10:29:08.794 | INFO | "pinned" : " 29 GB" | 20260508 10:29:08.794 | INFO | } | 20260508 10:29:08.795 | INFO | Perf report for the last 5.14009 seconds: | 20260508 10:29:08.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:08.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 10:29:08.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:29:08.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:08.795 | INFO | Uploaded 36GiB to GPU in 2.47057s (14.5716GiB/s) | 20260508 10:29:08.795 | INFO | Downloaded 34GiB to GPU in 2.46766s (13.7782GiB/s) | 20260508 10:29:13.932 | INFO | Analyzing chunk frames = [7394304,7394816) | 20260508 10:29:13.932 | INFO | MemInfo: { | 20260508 10:29:13.932 | INFO | "gpu" : " 17 GB", | 20260508 10:29:13.932 | INFO | "huge" : "", | 20260508 10:29:13.932 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:13.933 | INFO | "malloc" : " 14 GB", | 20260508 10:29:13.933 | INFO | "pinned" : " 29 GB" | 20260508 10:29:13.933 | INFO | } | 20260508 10:29:13.933 | INFO | Perf report for the last 5.13808 seconds: | 20260508 10:29:13.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:13.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:29:13.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 10:29:13.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:13.933 | INFO | Uploaded 36GiB to GPU in 2.47168s (14.565GiB/s) | 20260508 10:29:13.933 | INFO | Downloaded 34GiB to GPU in 2.45115s (13.871GiB/s) | 20260508 10:29:19.068 | INFO | Analyzing chunk frames = [7394816,7395328) | 20260508 10:29:19.068 | INFO | MemInfo: { | 20260508 10:29:19.068 | INFO | "gpu" : " 17 GB", | 20260508 10:29:19.068 | INFO | "huge" : "", | 20260508 10:29:19.068 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:19.068 | INFO | "malloc" : " 14 GB", | 20260508 10:29:19.068 | INFO | "pinned" : " 29 GB" | 20260508 10:29:19.068 | INFO | } | 20260508 10:29:19.068 | INFO | Perf report for the last 5.1357 seconds: | 20260508 10:29:19.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:19.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.068 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 10:29:19.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.068 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:29:19.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:19.069 | INFO | Uploaded 36GiB to GPU in 2.48066s (14.5123GiB/s) | 20260508 10:29:19.069 | INFO | Downloaded 34GiB to GPU in 2.42783s (14.0043GiB/s) | 20260508 10:29:24.215 | INFO | Analyzing chunk frames = [7395328,7395840) | 20260508 10:29:24.215 | INFO | MemInfo: { | 20260508 10:29:24.215 | INFO | "gpu" : " 17 GB", | 20260508 10:29:24.215 | INFO | "huge" : "", | 20260508 10:29:24.215 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:24.215 | INFO | "malloc" : " 14 GB", | 20260508 10:29:24.215 | INFO | "pinned" : " 29 GB" | 20260508 10:29:24.215 | INFO | } | 20260508 10:29:24.215 | INFO | Perf report for the last 5.14718 seconds: | 20260508 10:29:24.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:24.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 10:29:24.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:29:24.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:24.216 | INFO | Uploaded 36GiB to GPU in 2.34828s (15.3304GiB/s) | 20260508 10:29:24.216 | INFO | Downloaded 34GiB to GPU in 2.51317s (13.5287GiB/s) | 20260508 10:29:29.350 | INFO | Analyzing chunk frames = [7395840,7396352) | 20260508 10:29:29.350 | INFO | MemInfo: { | 20260508 10:29:29.350 | INFO | "gpu" : " 17 GB", | 20260508 10:29:29.350 | INFO | "huge" : "", | 20260508 10:29:29.350 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:29.350 | INFO | "malloc" : " 14 GB", | 20260508 10:29:29.350 | INFO | "pinned" : " 29 GB" | 20260508 10:29:29.350 | INFO | } | 20260508 10:29:29.350 | INFO | Perf report for the last 5.13457 seconds: | 20260508 10:29:29.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:29.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.350 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 10:29:29.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.350 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:29:29.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:29.351 | INFO | Uploaded 36GiB to GPU in 2.3408s (15.3794GiB/s) | 20260508 10:29:29.351 | INFO | Downloaded 34GiB to GPU in 2.43633s (13.9554GiB/s) | 20260508 10:29:34.490 | INFO | Analyzing chunk frames = [7396352,7396864) | 20260508 10:29:34.490 | INFO | MemInfo: { | 20260508 10:29:34.490 | INFO | "gpu" : " 17 GB", | 20260508 10:29:34.490 | INFO | "huge" : "", | 20260508 10:29:34.490 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:34.490 | INFO | "malloc" : " 14 GB", | 20260508 10:29:34.490 | INFO | "pinned" : " 29 GB" | 20260508 10:29:34.490 | INFO | } | 20260508 10:29:34.490 | INFO | Perf report for the last 5.13976 seconds: | 20260508 10:29:34.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:34.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:29:34.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:29:34.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:34.490 | INFO | Uploaded 36GiB to GPU in 2.4217s (14.8656GiB/s) | 20260508 10:29:34.490 | INFO | Downloaded 34GiB to GPU in 2.44495s (13.9062GiB/s) | 20260508 10:29:39.626 | INFO | Analyzing chunk frames = [7396864,7397376) | 20260508 10:29:39.626 | INFO | MemInfo: { | 20260508 10:29:39.626 | INFO | "gpu" : " 17 GB", | 20260508 10:29:39.626 | INFO | "huge" : "", | 20260508 10:29:39.626 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:39.626 | INFO | "malloc" : " 14 GB", | 20260508 10:29:39.626 | INFO | "pinned" : " 29 GB" | 20260508 10:29:39.626 | INFO | } | 20260508 10:29:39.627 | INFO | Perf report for the last 5.13664 seconds: | 20260508 10:29:39.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:39.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 10:29:39.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:29:39.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:39.627 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:29:39.627 | INFO | Total mem usage: 16.7 GiB | 20260508 10:29:39.627 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:29:39.627 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:29:39.627 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:29:39.627 | INFO | High Water Breakdown: | 20260508 10:29:39.627 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:29:39.627 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:29:39.627 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:29:39.627 | INFO | ------------------------------------------ | 20260508 10:29:39.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:29:39.627 | INFO | Uploaded 36GiB to GPU in 2.44659s (14.7143GiB/s) | 20260508 10:29:39.627 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:29:39.627 | INFO | Total mem usage: 26.5 GiB | 20260508 10:29:39.627 | INFO | Downloaded 34GiB to GPU in 2.45307s (13.8602GiB/s) | 20260508 10:29:39.627 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:29:39.627 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:29:39.627 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:29:39.627 | INFO | High Water Breakdown: | 20260508 10:29:39.627 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:29:39.627 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:29:39.627 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:29:39.627 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:29:39.627 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:29:39.627 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:29:39.627 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:29:39.627 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:29:39.627 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:29:39.627 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:29:39.627 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:29:39.627 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:29:39.627 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:29:39.627 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:29:39.627 | INFO | ------------------------------------------ | 20260508 10:29:39.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:29:39.627 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:29:39.627 | INFO | Total mem usage: 27.8 GiB | 20260508 10:29:39.627 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:29:39.627 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:29:39.628 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:29:39.628 | INFO | High Water Breakdown: | 20260508 10:29:39.628 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:29:39.628 | INFO | ------------------------------------------ | 20260508 10:29:39.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:29:39.628 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:29:39.628 | INFO | Total mem usage: 15.7 GiB | 20260508 10:29:39.628 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:29:39.628 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:29:39.628 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:29:39.628 | INFO | High Water Breakdown: | 20260508 10:29:39.628 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:29:39.628 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:29:39.628 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:29:39.628 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:29:39.628 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:29:39.628 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:29:39.628 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:29:39.628 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:29:39.628 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:29:39.628 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:29:39.628 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:29:39.628 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:29:39.628 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:29:39.628 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:29:39.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:29:39.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:29:39.628 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:29:39.628 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:29:39.628 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:29:39.628 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:29:39.628 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:29:39.628 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:29:39.628 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:29:39.628 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:29:39.628 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:29:39.628 | INFO | ------------------------------------------ | 20260508 10:29:39.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:29:39.628 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:29:39.628 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:29:39.628 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:29:39.628 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.39e+01 / 5.57e-01 | 20260508 10:29:39.628 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.10e-02 / 4.59e-03 | 20260508 10:29:39.628 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.40e+00, 5.93e-01 / 2.45e-01 | 20260508 10:29:39.628 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.44e+00, 2.10e-01 / 1.35e-01 | 20260508 10:29:39.628 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.49e+01, 3.55e+00 / 1.01e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.83e+00 / 1.31e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.33e+00 / 2.44e+00, 4.05e-01 / 2.85e-01 | 20260508 10:29:39.628 | INFO | PerfReport: 3.01%, Metrics -- 9.86e-01 / 1.14e+00, 2.45e-01 / 1.26e-01 | 20260508 10:29:39.628 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.40e-01, 4.90e-02 / 3.33e-02 | 20260508 10:29:39.628 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:29:39.628 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:29:39.628 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:29:39.628 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 24.18%, Upload -- 2.54e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 24.64%, Download -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 2.09%, LockUpload -- 2.20e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 2.63%, LockDownload -- 2.77e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:29:39.628 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.34e-02 / 0.00e+00 | 20260508 10:29:39.724 | INFO | Starting a round of Baz IO | 20260508 10:29:43.140 | INFO | Populating RTMetrics report | 20260508 10:29:43.141 | INFO | RTMetrics report populated | 20260508 10:29:43.141 | INFO | Opening tmp RTMetrics json file | 20260508 10:29:43.151 | INFO | Serializing RTMetrics to json | 20260508 10:29:43.153 | INFO | Serialization done, swapping tmp file now... | 20260508 10:29:43.153 | INFO | RTMetrics done | 20260508 10:29:43.212 | INFO | Finished a round of Baz IO: Wrote 1.461 GiB in 3.487 seconds | 20260508 10:29:43.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.80% | 20260508 10:29:43.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222544% / 0% / 6.35783e-05% | 20260508 10:29:43.212 | INFO | Baz IO overall: | 20260508 10:29:43.212 | INFO | BytesWritten : 3209143303792 | 20260508 10:29:43.212 | INFO | NumEvents : 1542493119158 | 20260508 10:29:44.768 | INFO | Analyzing chunk frames = [7397376,7397888) | 20260508 10:29:44.768 | INFO | MemInfo: { | 20260508 10:29:44.768 | INFO | "gpu" : " 17 GB", | 20260508 10:29:44.768 | INFO | "huge" : "", | 20260508 10:29:44.768 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:44.768 | INFO | "malloc" : " 14 GB", | 20260508 10:29:44.768 | INFO | "pinned" : " 29 GB" | 20260508 10:29:44.768 | INFO | } | 20260508 10:29:44.768 | INFO | Perf report for the last 5.14115 seconds: | 20260508 10:29:44.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:44.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:44.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:44.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:44.768 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 10:29:44.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:44.768 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260508 10:29:44.768 | INFO | PRE_HQ : 65.8%, 1.7, 57.5ms, 0 | 20260508 10:29:44.768 | INFO | BAZWRITER : 67.1%, 2.6, 89.9ms, 0 | 20260508 10:29:44.768 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:29:44.768 | INFO | Uploaded 36GiB to GPU in 2.59769s (13.8585GiB/s) | 20260508 10:29:44.768 | INFO | Downloaded 38GiB to GPU in 2.7097s (14.0237GiB/s) | 20260508 10:29:49.911 | INFO | Analyzing chunk frames = [7397888,7398400) | 20260508 10:29:49.911 | INFO | MemInfo: { | 20260508 10:29:49.911 | INFO | "gpu" : " 17 GB", | 20260508 10:29:49.911 | INFO | "huge" : "", | 20260508 10:29:49.911 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:49.911 | INFO | "malloc" : " 14 GB", | 20260508 10:29:49.911 | INFO | "pinned" : " 29 GB" | 20260508 10:29:49.911 | INFO | } | 20260508 10:29:49.911 | INFO | Perf report for the last 5.14355 seconds: | 20260508 10:29:49.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:49.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.911 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 10:29:49.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.911 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:29:49.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:49.912 | INFO | Uploaded 36GiB to GPU in 2.44689s (14.7125GiB/s) | 20260508 10:29:49.912 | INFO | Downloaded 34GiB to GPU in 2.47078s (13.7608GiB/s) | 20260508 10:29:55.052 | INFO | Analyzing chunk frames = [7398400,7398912) | 20260508 10:29:55.052 | INFO | MemInfo: { | 20260508 10:29:55.052 | INFO | "gpu" : " 17 GB", | 20260508 10:29:55.052 | INFO | "huge" : "", | 20260508 10:29:55.052 | INFO | "huge_shared" : " 29 GB", | 20260508 10:29:55.052 | INFO | "malloc" : " 14 GB", | 20260508 10:29:55.052 | INFO | "pinned" : " 29 GB" | 20260508 10:29:55.052 | INFO | } | 20260508 10:29:55.052 | INFO | Perf report for the last 5.14054 seconds: | 20260508 10:29:55.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:29:55.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:29:55.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:29:55.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:29:55.052 | INFO | Uploaded 36GiB to GPU in 2.47549s (14.5426GiB/s) | 20260508 10:29:55.052 | INFO | Downloaded 34GiB to GPU in 2.43894s (13.9405GiB/s) | 20260508 10:30:00.189 | INFO | Analyzing chunk frames = [7398912,7399424) | 20260508 10:30:00.189 | INFO | MemInfo: { | 20260508 10:30:00.189 | INFO | "gpu" : " 17 GB", | 20260508 10:30:00.189 | INFO | "huge" : "", | 20260508 10:30:00.189 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:00.189 | INFO | "malloc" : " 14 GB", | 20260508 10:30:00.189 | INFO | "pinned" : " 29 GB" | 20260508 10:30:00.189 | INFO | } | 20260508 10:30:00.189 | INFO | Perf report for the last 5.13764 seconds: | 20260508 10:30:00.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:00.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.189 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 10:30:00.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.190 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:30:00.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:00.190 | INFO | Uploaded 36GiB to GPU in 2.46727s (14.5911GiB/s) | 20260508 10:30:00.190 | INFO | Downloaded 34GiB to GPU in 2.44591s (13.9008GiB/s) | 20260508 10:30:05.323 | INFO | Analyzing chunk frames = [7399424,7399936) | 20260508 10:30:05.323 | INFO | MemInfo: { | 20260508 10:30:05.323 | INFO | "gpu" : " 17 GB", | 20260508 10:30:05.323 | INFO | "huge" : "", | 20260508 10:30:05.323 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:05.323 | INFO | "malloc" : " 14 GB", | 20260508 10:30:05.323 | INFO | "pinned" : " 29 GB" | 20260508 10:30:05.323 | INFO | } | 20260508 10:30:05.323 | INFO | Perf report for the last 5.13346 seconds: | 20260508 10:30:05.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:05.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:30:05.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:30:05.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:05.323 | INFO | Uploaded 36GiB to GPU in 2.42432s (14.8495GiB/s) | 20260508 10:30:05.323 | INFO | Downloaded 34GiB to GPU in 2.53353s (13.42GiB/s) | 20260508 10:30:10.466 | INFO | Analyzing chunk frames = [7399936,7400448) | 20260508 10:30:10.466 | INFO | MemInfo: { | 20260508 10:30:10.466 | INFO | "gpu" : " 17 GB", | 20260508 10:30:10.466 | INFO | "huge" : "", | 20260508 10:30:10.466 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:10.466 | INFO | "malloc" : " 14 GB", | 20260508 10:30:10.466 | INFO | "pinned" : " 29 GB" | 20260508 10:30:10.466 | INFO | } | 20260508 10:30:10.467 | INFO | Perf report for the last 5.14367 seconds: | 20260508 10:30:10.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:10.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | ANALYSIS : 65.1%, 3.0, 101.3ms, 0 | 20260508 10:30:10.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:30:10.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:10.467 | INFO | Uploaded 36GiB to GPU in 2.33165s (15.4397GiB/s) | 20260508 10:30:10.467 | INFO | Downloaded 34GiB to GPU in 2.44788s (13.8895GiB/s) | 20260508 10:30:15.608 | INFO | Analyzing chunk frames = [7400448,7400960) | 20260508 10:30:15.608 | INFO | MemInfo: { | 20260508 10:30:15.608 | INFO | "gpu" : " 17 GB", | 20260508 10:30:15.608 | INFO | "huge" : "", | 20260508 10:30:15.609 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:15.609 | INFO | "malloc" : " 14 GB", | 20260508 10:30:15.609 | INFO | "pinned" : " 29 GB" | 20260508 10:30:15.609 | INFO | } | 20260508 10:30:15.609 | INFO | Perf report for the last 5.14203 seconds: | 20260508 10:30:15.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:15.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 10:30:15.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:30:15.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:15.609 | INFO | Uploaded 36GiB to GPU in 2.39108s (15.056GiB/s) | 20260508 10:30:15.609 | INFO | Downloaded 34GiB to GPU in 2.45042s (13.8752GiB/s) | 20260508 10:30:20.745 | INFO | Analyzing chunk frames = [7400960,7401472) | 20260508 10:30:20.745 | INFO | MemInfo: { | 20260508 10:30:20.745 | INFO | "gpu" : " 17 GB", | 20260508 10:30:20.745 | INFO | "huge" : "", | 20260508 10:30:20.745 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:20.745 | INFO | "malloc" : " 14 GB", | 20260508 10:30:20.745 | INFO | "pinned" : " 29 GB" | 20260508 10:30:20.745 | INFO | } | 20260508 10:30:20.745 | INFO | Perf report for the last 5.13639 seconds: | 20260508 10:30:20.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:20.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:30:20.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 10:30:20.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:20.745 | INFO | Uploaded 36GiB to GPU in 2.38548s (15.0913GiB/s) | 20260508 10:30:20.745 | INFO | Downloaded 34GiB to GPU in 2.44725s (13.8931GiB/s) | 20260508 10:30:20.845 | INFO | Starting a round of Baz IO | 20260508 10:30:24.267 | INFO | Populating RTMetrics report | 20260508 10:30:24.267 | INFO | RTMetrics report populated | 20260508 10:30:24.267 | INFO | Opening tmp RTMetrics json file | 20260508 10:30:24.275 | INFO | Serializing RTMetrics to json | 20260508 10:30:24.276 | INFO | Serialization done, swapping tmp file now... | 20260508 10:30:24.277 | INFO | RTMetrics done | 20260508 10:30:24.336 | INFO | Finished a round of Baz IO: Wrote 1.459 GiB in 3.49 seconds | 20260508 10:30:24.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:30:24.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.221038% / 0% / 6.35783e-05% | 20260508 10:30:24.336 | INFO | Baz IO overall: | 20260508 10:30:24.336 | INFO | BytesWritten : 3210709676801 | 20260508 10:30:24.336 | INFO | NumEvents : 1543231745655 | 20260508 10:30:25.881 | INFO | Analyzing chunk frames = [7401472,7401984) | 20260508 10:30:25.881 | INFO | MemInfo: { | 20260508 10:30:25.881 | INFO | "gpu" : " 17 GB", | 20260508 10:30:25.881 | INFO | "huge" : "", | 20260508 10:30:25.881 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:25.881 | INFO | "malloc" : " 14 GB", | 20260508 10:30:25.881 | INFO | "pinned" : " 29 GB" | 20260508 10:30:25.882 | INFO | } | 20260508 10:30:25.882 | INFO | Perf report for the last 5.13652 seconds: | 20260508 10:30:25.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:25.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:25.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:25.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:25.882 | INFO | ANALYSIS : 67.9%, 3.0, 105.5ms, 0 | 20260508 10:30:25.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:25.882 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:30:25.882 | INFO | PRE_HQ : 65.8%, 1.6, 54.9ms, 0 | 20260508 10:30:25.882 | INFO | BAZWRITER : 67.3%, 2.9, 98.8ms, 0 | 20260508 10:30:25.882 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:30:25.882 | INFO | Uploaded 36GiB to GPU in 2.58291s (13.9378GiB/s) | 20260508 10:30:25.882 | INFO | Downloaded 38GiB to GPU in 2.71927s (13.9743GiB/s) | 20260508 10:30:31.026 | INFO | Analyzing chunk frames = [7401984,7402496) | 20260508 10:30:31.026 | INFO | MemInfo: { | 20260508 10:30:31.026 | INFO | "gpu" : " 17 GB", | 20260508 10:30:31.026 | INFO | "huge" : "", | 20260508 10:30:31.026 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:31.026 | INFO | "malloc" : " 14 GB", | 20260508 10:30:31.026 | INFO | "pinned" : " 29 GB" | 20260508 10:30:31.026 | INFO | } | 20260508 10:30:31.026 | INFO | Perf report for the last 5.14423 seconds: | 20260508 10:30:31.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:31.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 10:30:31.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:30:31.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:31.026 | INFO | Uploaded 36GiB to GPU in 2.51702s (14.3026GiB/s) | 20260508 10:30:31.026 | INFO | Downloaded 34GiB to GPU in 2.48562s (13.6787GiB/s) | 20260508 10:30:36.164 | INFO | Analyzing chunk frames = [7402496,7403008) | 20260508 10:30:36.164 | INFO | MemInfo: { | 20260508 10:30:36.164 | INFO | "gpu" : " 17 GB", | 20260508 10:30:36.164 | INFO | "huge" : "", | 20260508 10:30:36.164 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:36.164 | INFO | "malloc" : " 14 GB", | 20260508 10:30:36.164 | INFO | "pinned" : " 29 GB" | 20260508 10:30:36.164 | INFO | } | 20260508 10:30:36.164 | INFO | Perf report for the last 5.13821 seconds: | 20260508 10:30:36.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:36.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 10:30:36.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:30:36.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:36.164 | INFO | Uploaded 36GiB to GPU in 2.43703s (14.7721GiB/s) | 20260508 10:30:36.164 | INFO | Downloaded 34GiB to GPU in 2.44939s (13.881GiB/s) | 20260508 10:30:41.299 | INFO | Analyzing chunk frames = [7403008,7403520) | 20260508 10:30:41.299 | INFO | MemInfo: { | 20260508 10:30:41.299 | INFO | "gpu" : " 17 GB", | 20260508 10:30:41.299 | INFO | "huge" : "", | 20260508 10:30:41.299 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:41.299 | INFO | "malloc" : " 14 GB", | 20260508 10:30:41.299 | INFO | "pinned" : " 29 GB" | 20260508 10:30:41.299 | INFO | } | 20260508 10:30:41.299 | INFO | Perf report for the last 5.13519 seconds: | 20260508 10:30:41.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:41.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 10:30:41.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:30:41.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:41.299 | INFO | Uploaded 36GiB to GPU in 2.43068s (14.8107GiB/s) | 20260508 10:30:41.300 | INFO | Downloaded 34GiB to GPU in 2.41845s (14.0586GiB/s) | 20260508 10:30:46.441 | INFO | Analyzing chunk frames = [7403520,7404032) | 20260508 10:30:46.441 | INFO | MemInfo: { | 20260508 10:30:46.441 | INFO | "gpu" : " 17 GB", | 20260508 10:30:46.441 | INFO | "huge" : "", | 20260508 10:30:46.441 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:46.441 | INFO | "malloc" : " 14 GB", | 20260508 10:30:46.441 | INFO | "pinned" : " 29 GB" | 20260508 10:30:46.441 | INFO | } | 20260508 10:30:46.441 | INFO | Perf report for the last 5.14163 seconds: | 20260508 10:30:46.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:46.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 10:30:46.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 10:30:46.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:46.441 | INFO | Uploaded 36GiB to GPU in 2.36523s (15.2205GiB/s) | 20260508 10:30:46.441 | INFO | Downloaded 34GiB to GPU in 2.52505s (13.4651GiB/s) | 20260508 10:30:51.578 | INFO | Analyzing chunk frames = [7404032,7404544) | 20260508 10:30:51.578 | INFO | MemInfo: { | 20260508 10:30:51.578 | INFO | "gpu" : " 17 GB", | 20260508 10:30:51.578 | INFO | "huge" : "", | 20260508 10:30:51.578 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:51.578 | INFO | "malloc" : " 14 GB", | 20260508 10:30:51.578 | INFO | "pinned" : " 29 GB" | 20260508 10:30:51.578 | INFO | } | 20260508 10:30:51.578 | INFO | Perf report for the last 5.1371 seconds: | 20260508 10:30:51.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:51.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:30:51.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 10:30:51.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:51.578 | INFO | Uploaded 36GiB to GPU in 2.36123s (15.2463GiB/s) | 20260508 10:30:51.578 | INFO | Downloaded 34GiB to GPU in 2.44248s (13.9203GiB/s) | 20260508 10:30:56.719 | INFO | Analyzing chunk frames = [7404544,7405056) | 20260508 10:30:56.719 | INFO | MemInfo: { | 20260508 10:30:56.719 | INFO | "gpu" : " 17 GB", | 20260508 10:30:56.719 | INFO | "huge" : "", | 20260508 10:30:56.719 | INFO | "huge_shared" : " 29 GB", | 20260508 10:30:56.719 | INFO | "malloc" : " 14 GB", | 20260508 10:30:56.719 | INFO | "pinned" : " 29 GB" | 20260508 10:30:56.719 | INFO | } | 20260508 10:30:56.719 | INFO | Perf report for the last 5.14134 seconds: | 20260508 10:30:56.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:30:56.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.719 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:30:56.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.719 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:30:56.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:30:56.720 | INFO | Uploaded 36GiB to GPU in 2.40715s (14.9554GiB/s) | 20260508 10:30:56.720 | INFO | Downloaded 34GiB to GPU in 2.44666s (13.8965GiB/s) | 20260508 10:31:01.866 | INFO | Analyzing chunk frames = [7405056,7405568) | 20260508 10:31:01.866 | INFO | MemInfo: { | 20260508 10:31:01.866 | INFO | "gpu" : " 17 GB", | 20260508 10:31:01.866 | INFO | "huge" : "", | 20260508 10:31:01.866 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:01.866 | INFO | "malloc" : " 14 GB", | 20260508 10:31:01.866 | INFO | "pinned" : " 29 GB" | 20260508 10:31:01.866 | INFO | } | 20260508 10:31:01.866 | INFO | Perf report for the last 5.14666 seconds: | 20260508 10:31:01.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:01.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 10:31:01.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 10:31:01.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:01.866 | INFO | Uploaded 36GiB to GPU in 2.42742s (14.8305GiB/s) | 20260508 10:31:01.866 | INFO | Downloaded 34GiB to GPU in 2.44571s (13.9019GiB/s) | 20260508 10:31:01.867 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:31:01.867 | INFO | Total mem usage: 16.7 GiB | 20260508 10:31:01.867 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:31:01.867 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.2%) | 20260508 10:31:01.867 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:31:01.867 | INFO | High Water Breakdown: | 20260508 10:31:01.867 | INFO | 60.4% : 9.05 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:31:01.867 | INFO | 39.5% : 5.93 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:31:01.867 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:31:01.867 | INFO | ------------------------------------------ | 20260508 10:31:01.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:31:01.867 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:31:01.867 | INFO | Total mem usage: 26.5 GiB | 20260508 10:31:01.867 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:31:01.867 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:31:01.867 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:31:01.867 | INFO | High Water Breakdown: | 20260508 10:31:01.867 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:31:01.867 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:31:01.867 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:31:01.867 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:31:01.867 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:31:01.867 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:31:01.867 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:31:01.867 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:31:01.867 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:31:01.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:31:01.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:31:01.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:31:01.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:31:01.867 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:31:01.867 | INFO | ------------------------------------------ | 20260508 10:31:01.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:31:01.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:31:01.867 | INFO | Total mem usage: 27.8 GiB | 20260508 10:31:01.867 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:31:01.867 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:31:01.867 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:31:01.867 | INFO | High Water Breakdown: | 20260508 10:31:01.867 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:31:01.867 | INFO | ------------------------------------------ | 20260508 10:31:01.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:31:01.867 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:31:01.867 | INFO | Total mem usage: 15.7 GiB | 20260508 10:31:01.867 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:31:01.867 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:31:01.867 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:31:01.867 | INFO | High Water Breakdown: | 20260508 10:31:01.867 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:31:01.867 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:31:01.867 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:31:01.867 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:31:01.867 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:31:01.867 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:31:01.867 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:31:01.867 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:31:01.867 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:31:01.867 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:31:01.867 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:31:01.867 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:31:01.867 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:31:01.867 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:31:01.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:31:01.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:31:01.867 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:31:01.867 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:31:01.867 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:31:01.867 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:31:01.867 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:31:01.867 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:31:01.867 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:31:01.867 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:31:01.867 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:31:01.867 | INFO | ------------------------------------------ | 20260508 10:31:01.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:31:01.867 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:31:01.867 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:31:01.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:31:01.867 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 5.79e-01 | 20260508 10:31:01.867 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.06e-01, 1.19e-02 / 5.98e-03 | 20260508 10:31:01.867 | INFO | PerfReport: 12.15%, Baseline -- 3.98e+00 / 4.44e+00, 5.90e-01 / 2.37e-01 | 20260508 10:31:01.867 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.10e-01 / 1.14e-01 | 20260508 10:31:01.867 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.51e+01, 3.66e+00 / 9.66e-01 | 20260508 10:31:01.867 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.23e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.07e-01 / 2.88e-01 | 20260508 10:31:01.867 | INFO | PerfReport: 3.01%, Metrics -- 9.84e-01 / 1.14e+00, 2.45e-01 / 1.43e-01 | 20260508 10:31:01.867 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.30e-01, 4.94e-02 / 2.67e-02 | 20260508 10:31:01.867 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:31:01.867 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:31:01.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:31:01.867 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 24.18%, Upload -- 2.54e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 24.69%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 2.04%, LockUpload -- 2.15e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 15.28%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:31:01.867 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260508 10:31:01.963 | INFO | Starting a round of Baz IO | 20260508 10:31:05.378 | INFO | Populating RTMetrics report | 20260508 10:31:05.378 | INFO | RTMetrics report populated | 20260508 10:31:05.378 | INFO | Opening tmp RTMetrics json file | 20260508 10:31:05.387 | INFO | Serializing RTMetrics to json | 20260508 10:31:05.389 | INFO | Serialization done, swapping tmp file now... | 20260508 10:31:05.389 | INFO | RTMetrics done | 20260508 10:31:05.449 | INFO | Finished a round of Baz IO: Wrote 1.459 GiB in 3.485 seconds | 20260508 10:31:05.449 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:31:05.449 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22107% / 0% / 5.96046e-05% | 20260508 10:31:05.449 | INFO | Baz IO overall: | 20260508 10:31:05.449 | INFO | BytesWritten : 3212276272798 | 20260508 10:31:05.449 | INFO | NumEvents : 1543970476961 | 20260508 10:31:07.002 | INFO | Analyzing chunk frames = [7405568,7406080) | 20260508 10:31:07.002 | INFO | MemInfo: { | 20260508 10:31:07.002 | INFO | "gpu" : " 17 GB", | 20260508 10:31:07.002 | INFO | "huge" : "", | 20260508 10:31:07.002 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:07.002 | INFO | "malloc" : " 14 GB", | 20260508 10:31:07.002 | INFO | "pinned" : " 29 GB" | 20260508 10:31:07.002 | INFO | } | 20260508 10:31:07.002 | INFO | Perf report for the last 5.13625 seconds: | 20260508 10:31:07.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:07.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:07.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:07.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:07.002 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 10:31:07.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:07.002 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:31:07.002 | INFO | PRE_HQ : 65.7%, 1.6, 54.5ms, 0 | 20260508 10:31:07.002 | INFO | BAZWRITER : 67.2%, 2.5, 86.4ms, 0 | 20260508 10:31:07.002 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:31:07.003 | INFO | Uploaded 36GiB to GPU in 2.64007s (13.636GiB/s) | 20260508 10:31:07.003 | INFO | Downloaded 38GiB to GPU in 2.70549s (14.0455GiB/s) | 20260508 10:31:12.136 | INFO | Analyzing chunk frames = [7406080,7406592) | 20260508 10:31:12.136 | INFO | MemInfo: { | 20260508 10:31:12.136 | INFO | "gpu" : " 17 GB", | 20260508 10:31:12.136 | INFO | "huge" : "", | 20260508 10:31:12.136 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:12.136 | INFO | "malloc" : " 14 GB", | 20260508 10:31:12.136 | INFO | "pinned" : " 29 GB" | 20260508 10:31:12.136 | INFO | } | 20260508 10:31:12.136 | INFO | Perf report for the last 5.13368 seconds: | 20260508 10:31:12.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:12.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.136 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 10:31:12.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.136 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:31:12.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:12.137 | INFO | Uploaded 36GiB to GPU in 2.45456s (14.6666GiB/s) | 20260508 10:31:12.137 | INFO | Downloaded 34GiB to GPU in 2.47456s (13.7398GiB/s) | 20260508 10:31:17.276 | INFO | Analyzing chunk frames = [7406592,7407104) | 20260508 10:31:17.276 | INFO | MemInfo: { | 20260508 10:31:17.276 | INFO | "gpu" : " 17 GB", | 20260508 10:31:17.276 | INFO | "huge" : "", | 20260508 10:31:17.276 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:17.276 | INFO | "malloc" : " 14 GB", | 20260508 10:31:17.276 | INFO | "pinned" : " 29 GB" | 20260508 10:31:17.276 | INFO | } | 20260508 10:31:17.276 | INFO | Perf report for the last 5.14044 seconds: | 20260508 10:31:17.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:17.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.277 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:31:17.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.277 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:31:17.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:17.277 | INFO | Uploaded 36GiB to GPU in 2.46009s (14.6336GiB/s) | 20260508 10:31:17.277 | INFO | Downloaded 34GiB to GPU in 2.42713s (14.0083GiB/s) | 20260508 10:31:22.416 | INFO | Analyzing chunk frames = [7407104,7407616) | 20260508 10:31:22.416 | INFO | MemInfo: { | 20260508 10:31:22.416 | INFO | "gpu" : " 17 GB", | 20260508 10:31:22.416 | INFO | "huge" : "", | 20260508 10:31:22.416 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:22.416 | INFO | "malloc" : " 14 GB", | 20260508 10:31:22.416 | INFO | "pinned" : " 29 GB" | 20260508 10:31:22.416 | INFO | } | 20260508 10:31:22.416 | INFO | Perf report for the last 5.13993 seconds: | 20260508 10:31:22.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:22.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.416 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 10:31:22.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.417 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:31:22.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:22.417 | INFO | Uploaded 36GiB to GPU in 2.46312s (14.6156GiB/s) | 20260508 10:31:22.417 | INFO | Downloaded 34GiB to GPU in 2.42431s (14.0246GiB/s) | 20260508 10:31:27.561 | INFO | Analyzing chunk frames = [7407616,7408128) | 20260508 10:31:27.561 | INFO | MemInfo: { | 20260508 10:31:27.562 | INFO | "gpu" : " 17 GB", | 20260508 10:31:27.562 | INFO | "huge" : "", | 20260508 10:31:27.562 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:27.562 | INFO | "malloc" : " 14 GB", | 20260508 10:31:27.562 | INFO | "pinned" : " 29 GB" | 20260508 10:31:27.562 | INFO | } | 20260508 10:31:27.562 | INFO | Perf report for the last 5.14537 seconds: | 20260508 10:31:27.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:27.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 10:31:27.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:31:27.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:27.562 | INFO | Uploaded 36GiB to GPU in 2.3301s (15.45GiB/s) | 20260508 10:31:27.562 | INFO | Downloaded 34GiB to GPU in 2.5194s (13.4953GiB/s) | 20260508 10:31:32.703 | INFO | Analyzing chunk frames = [7408128,7408640) | 20260508 10:31:32.703 | INFO | MemInfo: { | 20260508 10:31:32.703 | INFO | "gpu" : " 17 GB", | 20260508 10:31:32.703 | INFO | "huge" : "", | 20260508 10:31:32.703 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:32.703 | INFO | "malloc" : " 14 GB", | 20260508 10:31:32.703 | INFO | "pinned" : " 29 GB" | 20260508 10:31:32.703 | INFO | } | 20260508 10:31:32.703 | INFO | Perf report for the last 5.1416 seconds: | 20260508 10:31:32.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:32.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.703 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:31:32.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.704 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:31:32.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:32.704 | INFO | Uploaded 36GiB to GPU in 2.36943s (15.1935GiB/s) | 20260508 10:31:32.704 | INFO | Downloaded 34GiB to GPU in 2.45701s (13.838GiB/s) | 20260508 10:31:37.836 | INFO | Analyzing chunk frames = [7408640,7409152) | 20260508 10:31:37.836 | INFO | MemInfo: { | 20260508 10:31:37.836 | INFO | "gpu" : " 17 GB", | 20260508 10:31:37.836 | INFO | "huge" : "", | 20260508 10:31:37.836 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:37.836 | INFO | "malloc" : " 14 GB", | 20260508 10:31:37.836 | INFO | "pinned" : " 29 GB" | 20260508 10:31:37.836 | INFO | } | 20260508 10:31:37.837 | INFO | Perf report for the last 5.13313 seconds: | 20260508 10:31:37.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:37.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 10:31:37.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:31:37.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:37.837 | INFO | Uploaded 36GiB to GPU in 2.4233s (14.8558GiB/s) | 20260508 10:31:37.837 | INFO | Downloaded 34GiB to GPU in 2.47481s (13.7384GiB/s) | 20260508 10:31:42.973 | INFO | Analyzing chunk frames = [7409152,7409664) | 20260508 10:31:42.973 | INFO | MemInfo: { | 20260508 10:31:42.973 | INFO | "gpu" : " 17 GB", | 20260508 10:31:42.973 | INFO | "huge" : "", | 20260508 10:31:42.973 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:42.973 | INFO | "malloc" : " 14 GB", | 20260508 10:31:42.973 | INFO | "pinned" : " 29 GB" | 20260508 10:31:42.973 | INFO | } | 20260508 10:31:42.973 | INFO | Perf report for the last 5.13648 seconds: | 20260508 10:31:42.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:42.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:31:42.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:31:42.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:42.973 | INFO | Uploaded 36GiB to GPU in 2.44529s (14.7222GiB/s) | 20260508 10:31:42.973 | INFO | Downloaded 34GiB to GPU in 2.46616s (13.7866GiB/s) | 20260508 10:31:43.070 | INFO | Starting a round of Baz IO | 20260508 10:31:46.488 | INFO | Populating RTMetrics report | 20260508 10:31:46.489 | INFO | RTMetrics report populated | 20260508 10:31:46.489 | INFO | Opening tmp RTMetrics json file | 20260508 10:31:46.496 | INFO | Serializing RTMetrics to json | 20260508 10:31:46.498 | INFO | Serialization done, swapping tmp file now... | 20260508 10:31:46.498 | INFO | RTMetrics done | 20260508 10:31:46.563 | INFO | Finished a round of Baz IO: Wrote 1.459 GiB in 3.492 seconds | 20260508 10:31:46.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:31:46.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.220656% / 0% / 9.53674e-05% | 20260508 10:31:46.563 | INFO | Baz IO overall: | 20260508 10:31:46.563 | INFO | BytesWritten : 3213842397386 | 20260508 10:31:46.563 | INFO | NumEvents : 1544709033865 | 20260508 10:31:48.117 | INFO | Analyzing chunk frames = [7409664,7410176) | 20260508 10:31:48.117 | INFO | MemInfo: { | 20260508 10:31:48.117 | INFO | "gpu" : " 17 GB", | 20260508 10:31:48.117 | INFO | "huge" : "", | 20260508 10:31:48.117 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:48.117 | INFO | "malloc" : " 14 GB", | 20260508 10:31:48.117 | INFO | "pinned" : " 29 GB" | 20260508 10:31:48.117 | INFO | } | 20260508 10:31:48.117 | INFO | Perf report for the last 5.14436 seconds: | 20260508 10:31:48.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:48.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:48.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:48.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:48.117 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 10:31:48.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:48.117 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:31:48.118 | INFO | PRE_HQ : 65.0%, 1.6, 54.4ms, 0 | 20260508 10:31:48.118 | INFO | BAZWRITER : 67.1%, 2.5, 86.3ms, 0 | 20260508 10:31:48.118 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:31:48.118 | INFO | Uploaded 36GiB to GPU in 2.5991s (13.8509GiB/s) | 20260508 10:31:48.118 | INFO | Downloaded 38GiB to GPU in 2.70261s (14.0605GiB/s) | 20260508 10:31:53.251 | INFO | Analyzing chunk frames = [7410176,7410688) | 20260508 10:31:53.251 | INFO | MemInfo: { | 20260508 10:31:53.251 | INFO | "gpu" : " 17 GB", | 20260508 10:31:53.251 | INFO | "huge" : "", | 20260508 10:31:53.251 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:53.251 | INFO | "malloc" : " 14 GB", | 20260508 10:31:53.252 | INFO | "pinned" : " 29 GB" | 20260508 10:31:53.252 | INFO | } | 20260508 10:31:53.252 | INFO | Perf report for the last 5.1342 seconds: | 20260508 10:31:53.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:53.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:31:53.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:31:53.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:53.252 | INFO | Uploaded 36GiB to GPU in 2.43918s (14.759GiB/s) | 20260508 10:31:53.252 | INFO | Downloaded 34GiB to GPU in 2.46239s (13.8077GiB/s) | 20260508 10:31:58.387 | INFO | Analyzing chunk frames = [7410688,7411200) | 20260508 10:31:58.387 | INFO | MemInfo: { | 20260508 10:31:58.387 | INFO | "gpu" : " 17 GB", | 20260508 10:31:58.387 | INFO | "huge" : "", | 20260508 10:31:58.387 | INFO | "huge_shared" : " 29 GB", | 20260508 10:31:58.387 | INFO | "malloc" : " 14 GB", | 20260508 10:31:58.387 | INFO | "pinned" : " 29 GB" | 20260508 10:31:58.387 | INFO | } | 20260508 10:31:58.388 | INFO | Perf report for the last 5.13591 seconds: | 20260508 10:31:58.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:31:58.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 10:31:58.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 10:31:58.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:31:58.388 | INFO | Uploaded 36GiB to GPU in 2.44514s (14.7231GiB/s) | 20260508 10:31:58.388 | INFO | Downloaded 34GiB to GPU in 2.43746s (13.949GiB/s) | 20260508 10:32:03.532 | INFO | Analyzing chunk frames = [7411200,7411712) | 20260508 10:32:03.532 | INFO | MemInfo: { | 20260508 10:32:03.532 | INFO | "gpu" : " 17 GB", | 20260508 10:32:03.532 | INFO | "huge" : "", | 20260508 10:32:03.532 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:03.532 | INFO | "malloc" : " 14 GB", | 20260508 10:32:03.532 | INFO | "pinned" : " 29 GB" | 20260508 10:32:03.532 | INFO | } | 20260508 10:32:03.532 | INFO | Perf report for the last 5.14453 seconds: | 20260508 10:32:03.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:03.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 10:32:03.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:32:03.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:03.532 | INFO | Uploaded 36GiB to GPU in 2.47978s (14.5174GiB/s) | 20260508 10:32:03.532 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0225GiB/s) | 20260508 10:32:08.679 | INFO | Analyzing chunk frames = [7411712,7412224) | 20260508 10:32:08.679 | INFO | MemInfo: { | 20260508 10:32:08.679 | INFO | "gpu" : " 17 GB", | 20260508 10:32:08.679 | INFO | "huge" : "", | 20260508 10:32:08.679 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:08.679 | INFO | "malloc" : " 14 GB", | 20260508 10:32:08.679 | INFO | "pinned" : " 29 GB" | 20260508 10:32:08.679 | INFO | } | 20260508 10:32:08.679 | INFO | Perf report for the last 5.1474 seconds: | 20260508 10:32:08.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:08.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.680 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 10:32:08.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.680 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:32:08.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:08.680 | INFO | Uploaded 36GiB to GPU in 2.33595s (15.4113GiB/s) | 20260508 10:32:08.680 | INFO | Downloaded 34GiB to GPU in 2.50066s (13.5964GiB/s) | 20260508 10:32:13.813 | INFO | Analyzing chunk frames = [7412224,7412736) | 20260508 10:32:13.813 | INFO | MemInfo: { | 20260508 10:32:13.813 | INFO | "gpu" : " 17 GB", | 20260508 10:32:13.813 | INFO | "huge" : "", | 20260508 10:32:13.813 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:13.813 | INFO | "malloc" : " 14 GB", | 20260508 10:32:13.813 | INFO | "pinned" : " 29 GB" | 20260508 10:32:13.813 | INFO | } | 20260508 10:32:13.813 | INFO | Perf report for the last 5.13391 seconds: | 20260508 10:32:13.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:13.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.813 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:32:13.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.814 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:32:13.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:13.814 | INFO | Uploaded 36GiB to GPU in 2.39546s (15.0284GiB/s) | 20260508 10:32:13.814 | INFO | Downloaded 34GiB to GPU in 2.43265s (13.9765GiB/s) | 20260508 10:32:18.947 | INFO | Analyzing chunk frames = [7412736,7413248) | 20260508 10:32:18.947 | INFO | MemInfo: { | 20260508 10:32:18.947 | INFO | "gpu" : " 17 GB", | 20260508 10:32:18.947 | INFO | "huge" : "", | 20260508 10:32:18.947 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:18.947 | INFO | "malloc" : " 14 GB", | 20260508 10:32:18.947 | INFO | "pinned" : " 29 GB" | 20260508 10:32:18.947 | INFO | } | 20260508 10:32:18.947 | INFO | Perf report for the last 5.13398 seconds: | 20260508 10:32:18.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:18.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.947 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:32:18.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.948 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:32:18.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:18.948 | INFO | Uploaded 36GiB to GPU in 2.44147s (14.7452GiB/s) | 20260508 10:32:18.948 | INFO | Downloaded 34GiB to GPU in 2.44147s (13.926GiB/s) | 20260508 10:32:24.083 | INFO | Analyzing chunk frames = [7413248,7413760) | 20260508 10:32:24.083 | INFO | MemInfo: { | 20260508 10:32:24.083 | INFO | "gpu" : " 17 GB", | 20260508 10:32:24.083 | INFO | "huge" : "", | 20260508 10:32:24.083 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:24.083 | INFO | "malloc" : " 14 GB", | 20260508 10:32:24.084 | INFO | "pinned" : " 29 GB" | 20260508 10:32:24.084 | INFO | } | 20260508 10:32:24.084 | INFO | Perf report for the last 5.1363 seconds: | 20260508 10:32:24.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:24.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:32:24.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:32:24.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:24.084 | INFO | Uploaded 36GiB to GPU in 2.44053s (14.7509GiB/s) | 20260508 10:32:24.084 | INFO | Downloaded 34GiB to GPU in 2.43911s (13.9395GiB/s) | 20260508 10:32:24.084 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:32:24.084 | INFO | Total mem usage: 16.7 GiB | 20260508 10:32:24.084 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:32:24.084 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:32:24.084 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:32:24.084 | INFO | High Water Breakdown: | 20260508 10:32:24.084 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:32:24.084 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:32:24.084 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:32:24.084 | INFO | ------------------------------------------ | 20260508 10:32:24.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:32:24.084 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:32:24.085 | INFO | Total mem usage: 26.5 GiB | 20260508 10:32:24.085 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:32:24.085 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:32:24.085 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:32:24.085 | INFO | High Water Breakdown: | 20260508 10:32:24.085 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:32:24.085 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:32:24.085 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:32:24.085 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:32:24.085 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:32:24.085 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:32:24.085 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:32:24.085 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:32:24.085 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:32:24.085 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:32:24.085 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:32:24.085 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:32:24.085 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:32:24.085 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:32:24.085 | INFO | ------------------------------------------ | 20260508 10:32:24.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:32:24.085 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:32:24.085 | INFO | Total mem usage: 27.8 GiB | 20260508 10:32:24.085 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:32:24.085 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:32:24.085 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:32:24.085 | INFO | High Water Breakdown: | 20260508 10:32:24.085 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:32:24.085 | INFO | ------------------------------------------ | 20260508 10:32:24.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:32:24.085 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:32:24.085 | INFO | Total mem usage: 15.7 GiB | 20260508 10:32:24.085 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:32:24.085 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:32:24.085 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:32:24.085 | INFO | High Water Breakdown: | 20260508 10:32:24.085 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:32:24.085 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:32:24.085 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:32:24.085 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:32:24.085 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:32:24.085 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:32:24.085 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:32:24.085 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:32:24.085 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:32:24.085 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:32:24.085 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:32:24.085 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:32:24.085 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:32:24.085 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:32:24.085 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:32:24.085 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:32:24.085 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:32:24.085 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:32:24.085 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:32:24.085 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:32:24.085 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:32:24.085 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:32:24.085 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:32:24.085 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:32:24.085 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:32:24.085 | INFO | ------------------------------------------ | 20260508 10:32:24.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:32:24.085 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:32:24.085 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:32:24.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:32:24.085 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 7.78e-01 | 20260508 10:32:24.085 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.10e-02 / 5.06e-03 | 20260508 10:32:24.086 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.37e+00, 5.96e-01 / 2.66e-01 | 20260508 10:32:24.086 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.28e-01 | 20260508 10:32:24.086 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.55e+01, 3.74e+00 / 1.25e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.84e+00 / 1.29e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.09e-01 / 2.81e-01 | 20260508 10:32:24.086 | INFO | PerfReport: 3.01%, Metrics -- 9.85e-01 / 1.13e+00, 2.46e-01 / 1.27e-01 | 20260508 10:32:24.086 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.29e-01, 4.83e-02 / 2.29e-02 | 20260508 10:32:24.086 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:32:24.086 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:32:24.086 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:32:24.086 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 24.23%, Upload -- 2.55e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 24.61%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 31.10%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 2.12%, LockUpload -- 2.24e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 15.34%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:32:24.086 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 9.17e-03 / 0.00e+00 | 20260508 10:32:24.182 | INFO | Starting a round of Baz IO | 20260508 10:32:27.604 | INFO | Populating RTMetrics report | 20260508 10:32:27.605 | INFO | RTMetrics report populated | 20260508 10:32:27.605 | INFO | Opening tmp RTMetrics json file | 20260508 10:32:27.614 | INFO | Serializing RTMetrics to json | 20260508 10:32:27.616 | INFO | Serialization done, swapping tmp file now... | 20260508 10:32:27.616 | INFO | RTMetrics done | 20260508 10:32:27.673 | INFO | Finished a round of Baz IO: Wrote 1.458 GiB in 3.491 seconds | 20260508 10:32:27.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:32:27.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.219591% / 0% / 8.34465e-05% | 20260508 10:32:27.673 | INFO | Baz IO overall: | 20260508 10:32:27.673 | INFO | BytesWritten : 3215408136350 | 20260508 10:32:27.673 | INFO | NumEvents : 1545447345195 | 20260508 10:32:29.227 | INFO | Analyzing chunk frames = [7413760,7414272) | 20260508 10:32:29.227 | INFO | MemInfo: { | 20260508 10:32:29.227 | INFO | "gpu" : " 17 GB", | 20260508 10:32:29.227 | INFO | "huge" : "", | 20260508 10:32:29.227 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:29.227 | INFO | "malloc" : " 14 GB", | 20260508 10:32:29.227 | INFO | "pinned" : " 29 GB" | 20260508 10:32:29.227 | INFO | } | 20260508 10:32:29.227 | INFO | Perf report for the last 5.14353 seconds: | 20260508 10:32:29.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:29.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:29.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:29.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:29.227 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 10:32:29.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:29.227 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:32:29.227 | INFO | PRE_HQ : 65.1%, 1.6, 54.1ms, 0 | 20260508 10:32:29.227 | INFO | BAZWRITER : 67.2%, 2.5, 86.3ms, 0 | 20260508 10:32:29.227 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:32:29.228 | INFO | Uploaded 36GiB to GPU in 2.58807s (13.91GiB/s) | 20260508 10:32:29.228 | INFO | Downloaded 38GiB to GPU in 2.71912s (13.9751GiB/s) | 20260508 10:32:34.373 | INFO | Analyzing chunk frames = [7414272,7414784) | 20260508 10:32:34.373 | INFO | MemInfo: { | 20260508 10:32:34.373 | INFO | "gpu" : " 17 GB", | 20260508 10:32:34.373 | INFO | "huge" : "", | 20260508 10:32:34.373 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:34.373 | INFO | "malloc" : " 14 GB", | 20260508 10:32:34.373 | INFO | "pinned" : " 29 GB" | 20260508 10:32:34.373 | INFO | } | 20260508 10:32:34.373 | INFO | Perf report for the last 5.14573 seconds: | 20260508 10:32:34.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:34.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 10:32:34.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 10:32:34.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:34.373 | INFO | Uploaded 36GiB to GPU in 2.48549s (14.4841GiB/s) | 20260508 10:32:34.373 | INFO | Downloaded 34GiB to GPU in 2.49875s (13.6068GiB/s) | 20260508 10:32:39.509 | INFO | Analyzing chunk frames = [7414784,7415296) | 20260508 10:32:39.509 | INFO | MemInfo: { | 20260508 10:32:39.509 | INFO | "gpu" : " 17 GB", | 20260508 10:32:39.509 | INFO | "huge" : "", | 20260508 10:32:39.509 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:39.509 | INFO | "malloc" : " 14 GB", | 20260508 10:32:39.509 | INFO | "pinned" : " 29 GB" | 20260508 10:32:39.509 | INFO | } | 20260508 10:32:39.509 | INFO | Perf report for the last 5.13634 seconds: | 20260508 10:32:39.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:39.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.509 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:32:39.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.509 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:32:39.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:39.510 | INFO | Uploaded 36GiB to GPU in 2.45225s (14.6804GiB/s) | 20260508 10:32:39.510 | INFO | Downloaded 34GiB to GPU in 2.45709s (13.8375GiB/s) | 20260508 10:32:44.647 | INFO | Analyzing chunk frames = [7415296,7415808) | 20260508 10:32:44.647 | INFO | MemInfo: { | 20260508 10:32:44.647 | INFO | "gpu" : " 17 GB", | 20260508 10:32:44.647 | INFO | "huge" : "", | 20260508 10:32:44.647 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:44.647 | INFO | "malloc" : " 14 GB", | 20260508 10:32:44.647 | INFO | "pinned" : " 29 GB" | 20260508 10:32:44.647 | INFO | } | 20260508 10:32:44.647 | INFO | Perf report for the last 5.13764 seconds: | 20260508 10:32:44.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:44.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.647 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 10:32:44.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.647 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:32:44.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:44.648 | INFO | Uploaded 36GiB to GPU in 2.41581s (14.9018GiB/s) | 20260508 10:32:44.648 | INFO | Downloaded 34GiB to GPU in 2.42468s (14.0225GiB/s) | 20260508 10:32:49.789 | INFO | Analyzing chunk frames = [7415808,7416320) | 20260508 10:32:49.789 | INFO | MemInfo: { | 20260508 10:32:49.789 | INFO | "gpu" : " 17 GB", | 20260508 10:32:49.789 | INFO | "huge" : "", | 20260508 10:32:49.789 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:49.789 | INFO | "malloc" : " 14 GB", | 20260508 10:32:49.789 | INFO | "pinned" : " 29 GB" | 20260508 10:32:49.789 | INFO | } | 20260508 10:32:49.789 | INFO | Perf report for the last 5.14241 seconds: | 20260508 10:32:49.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:49.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.789 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 10:32:49.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.790 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:32:49.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:49.790 | INFO | Uploaded 36GiB to GPU in 2.28889s (15.7282GiB/s) | 20260508 10:32:49.790 | INFO | Downloaded 34GiB to GPU in 2.50441s (13.5761GiB/s) | 20260508 10:32:54.923 | INFO | Analyzing chunk frames = [7416320,7416832) | 20260508 10:32:54.923 | INFO | MemInfo: { | 20260508 10:32:54.923 | INFO | "gpu" : " 17 GB", | 20260508 10:32:54.923 | INFO | "huge" : "", | 20260508 10:32:54.923 | INFO | "huge_shared" : " 29 GB", | 20260508 10:32:54.923 | INFO | "malloc" : " 14 GB", | 20260508 10:32:54.923 | INFO | "pinned" : " 29 GB" | 20260508 10:32:54.923 | INFO | } | 20260508 10:32:54.923 | INFO | Perf report for the last 5.13364 seconds: | 20260508 10:32:54.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:32:54.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:32:54.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:32:54.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:32:54.923 | INFO | Uploaded 36GiB to GPU in 2.35641s (15.2774GiB/s) | 20260508 10:32:54.923 | INFO | Downloaded 34GiB to GPU in 2.44084s (13.9296GiB/s) | 20260508 10:33:00.065 | INFO | Analyzing chunk frames = [7416832,7417344) | 20260508 10:33:00.065 | INFO | MemInfo: { | 20260508 10:33:00.065 | INFO | "gpu" : " 17 GB", | 20260508 10:33:00.065 | INFO | "huge" : "", | 20260508 10:33:00.065 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:00.065 | INFO | "malloc" : " 14 GB", | 20260508 10:33:00.065 | INFO | "pinned" : " 29 GB" | 20260508 10:33:00.065 | INFO | } | 20260508 10:33:00.065 | INFO | Perf report for the last 5.14194 seconds: | 20260508 10:33:00.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:00.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:33:00.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:33:00.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:00.065 | INFO | Uploaded 36GiB to GPU in 2.42812s (14.8263GiB/s) | 20260508 10:33:00.065 | INFO | Downloaded 34GiB to GPU in 2.45481s (13.8504GiB/s) | 20260508 10:33:05.204 | INFO | Analyzing chunk frames = [7417344,7417856) | 20260508 10:33:05.204 | INFO | MemInfo: { | 20260508 10:33:05.204 | INFO | "gpu" : " 17 GB", | 20260508 10:33:05.204 | INFO | "huge" : "", | 20260508 10:33:05.204 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:05.204 | INFO | "malloc" : " 14 GB", | 20260508 10:33:05.204 | INFO | "pinned" : " 29 GB" | 20260508 10:33:05.204 | INFO | } | 20260508 10:33:05.204 | INFO | Perf report for the last 5.13884 seconds: | 20260508 10:33:05.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:05.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:33:05.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 10:33:05.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:05.204 | INFO | Uploaded 36GiB to GPU in 2.4175s (14.8914GiB/s) | 20260508 10:33:05.204 | INFO | Downloaded 34GiB to GPU in 2.45214s (13.8655GiB/s) | 20260508 10:33:05.301 | INFO | Starting a round of Baz IO | 20260508 10:33:08.713 | INFO | Populating RTMetrics report | 20260508 10:33:08.713 | INFO | RTMetrics report populated | 20260508 10:33:08.713 | INFO | Opening tmp RTMetrics json file | 20260508 10:33:08.723 | INFO | Serializing RTMetrics to json | 20260508 10:33:08.725 | INFO | Serialization done, swapping tmp file now... | 20260508 10:33:08.726 | INFO | RTMetrics done | 20260508 10:33:08.786 | INFO | Finished a round of Baz IO: Wrote 1.458 GiB in 3.484 seconds | 20260508 10:33:08.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:33:08.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22004% / 0% / 9.13938e-05% | 20260508 10:33:08.786 | INFO | Baz IO overall: | 20260508 10:33:08.786 | INFO | BytesWritten : 3216973649437 | 20260508 10:33:08.786 | INFO | NumEvents : 1546185787362 | 20260508 10:33:10.341 | INFO | Analyzing chunk frames = [7417856,7418368) | 20260508 10:33:10.341 | INFO | MemInfo: { | 20260508 10:33:10.341 | INFO | "gpu" : " 17 GB", | 20260508 10:33:10.341 | INFO | "huge" : "", | 20260508 10:33:10.341 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:10.342 | INFO | "malloc" : " 14 GB", | 20260508 10:33:10.342 | INFO | "pinned" : " 29 GB" | 20260508 10:33:10.342 | INFO | } | 20260508 10:33:10.342 | INFO | Perf report for the last 5.13779 seconds: | 20260508 10:33:10.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:10.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:10.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:10.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:10.342 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:33:10.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:10.342 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:33:10.342 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 10:33:10.342 | INFO | BAZWRITER : 67.1%, 2.5, 86.5ms, 0 | 20260508 10:33:10.342 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:33:10.342 | INFO | Uploaded 36GiB to GPU in 2.61144s (13.7855GiB/s) | 20260508 10:33:10.342 | INFO | Downloaded 38GiB to GPU in 2.69558s (14.0971GiB/s) | 20260508 10:33:15.486 | INFO | Analyzing chunk frames = [7418368,7418880) | 20260508 10:33:15.486 | INFO | MemInfo: { | 20260508 10:33:15.486 | INFO | "gpu" : " 17 GB", | 20260508 10:33:15.486 | INFO | "huge" : "", | 20260508 10:33:15.486 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:15.486 | INFO | "malloc" : " 14 GB", | 20260508 10:33:15.487 | INFO | "pinned" : " 29 GB" | 20260508 10:33:15.487 | INFO | } | 20260508 10:33:15.487 | INFO | Perf report for the last 5.14494 seconds: | 20260508 10:33:15.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:15.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:33:15.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:33:15.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:15.487 | INFO | Uploaded 36GiB to GPU in 2.55327s (14.0995GiB/s) | 20260508 10:33:15.487 | INFO | Downloaded 34GiB to GPU in 2.45661s (13.8402GiB/s) | 20260508 10:33:20.624 | INFO | Analyzing chunk frames = [7418880,7419392) | 20260508 10:33:20.624 | INFO | MemInfo: { | 20260508 10:33:20.624 | INFO | "gpu" : " 17 GB", | 20260508 10:33:20.624 | INFO | "huge" : "", | 20260508 10:33:20.624 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:20.624 | INFO | "malloc" : " 14 GB", | 20260508 10:33:20.624 | INFO | "pinned" : " 29 GB" | 20260508 10:33:20.624 | INFO | } | 20260508 10:33:20.624 | INFO | Perf report for the last 5.1374 seconds: | 20260508 10:33:20.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:20.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.624 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:33:20.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.624 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:33:20.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:20.625 | INFO | Uploaded 36GiB to GPU in 2.49367s (14.4365GiB/s) | 20260508 10:33:20.625 | INFO | Downloaded 34GiB to GPU in 2.42625s (14.0134GiB/s) | 20260508 10:33:25.764 | INFO | Analyzing chunk frames = [7419392,7419904) | 20260508 10:33:25.764 | INFO | MemInfo: { | 20260508 10:33:25.764 | INFO | "gpu" : " 17 GB", | 20260508 10:33:25.764 | INFO | "huge" : "", | 20260508 10:33:25.764 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:25.764 | INFO | "malloc" : " 14 GB", | 20260508 10:33:25.764 | INFO | "pinned" : " 29 GB" | 20260508 10:33:25.764 | INFO | } | 20260508 10:33:25.764 | INFO | Perf report for the last 5.1398 seconds: | 20260508 10:33:25.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:25.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:33:25.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:33:25.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:25.764 | INFO | Uploaded 36GiB to GPU in 2.48277s (14.4999GiB/s) | 20260508 10:33:25.764 | INFO | Downloaded 34GiB to GPU in 2.40254s (14.1517GiB/s) | 20260508 10:33:30.896 | INFO | Analyzing chunk frames = [7419904,7420416) | 20260508 10:33:30.896 | INFO | MemInfo: { | 20260508 10:33:30.896 | INFO | "gpu" : " 17 GB", | 20260508 10:33:30.896 | INFO | "huge" : "", | 20260508 10:33:30.896 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:30.896 | INFO | "malloc" : " 14 GB", | 20260508 10:33:30.896 | INFO | "pinned" : " 29 GB" | 20260508 10:33:30.896 | INFO | } | 20260508 10:33:30.896 | INFO | Perf report for the last 5.13197 seconds: | 20260508 10:33:30.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:30.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | ANALYSIS : 67.0%, 3.0, 104.3ms, 0 | 20260508 10:33:30.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:33:30.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:30.896 | INFO | Uploaded 36GiB to GPU in 2.5141s (14.3192GiB/s) | 20260508 10:33:30.896 | INFO | Downloaded 34GiB to GPU in 2.53383s (13.4184GiB/s) | 20260508 10:33:36.042 | INFO | Analyzing chunk frames = [7420416,7420928) | 20260508 10:33:36.042 | INFO | MemInfo: { | 20260508 10:33:36.042 | INFO | "gpu" : " 17 GB", | 20260508 10:33:36.042 | INFO | "huge" : "", | 20260508 10:33:36.042 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:36.042 | INFO | "malloc" : " 14 GB", | 20260508 10:33:36.042 | INFO | "pinned" : " 29 GB" | 20260508 10:33:36.042 | INFO | } | 20260508 10:33:36.042 | INFO | Perf report for the last 5.1465 seconds: | 20260508 10:33:36.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:36.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.042 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 10:33:36.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.042 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:33:36.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:36.043 | INFO | Uploaded 36GiB to GPU in 2.34748s (15.3356GiB/s) | 20260508 10:33:36.043 | INFO | Downloaded 34GiB to GPU in 2.43056s (13.9885GiB/s) | 20260508 10:33:41.178 | INFO | Analyzing chunk frames = [7420928,7421440) | 20260508 10:33:41.178 | INFO | MemInfo: { | 20260508 10:33:41.178 | INFO | "gpu" : " 17 GB", | 20260508 10:33:41.178 | INFO | "huge" : "", | 20260508 10:33:41.178 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:41.178 | INFO | "malloc" : " 14 GB", | 20260508 10:33:41.178 | INFO | "pinned" : " 29 GB" | 20260508 10:33:41.179 | INFO | } | 20260508 10:33:41.179 | INFO | Perf report for the last 5.13625 seconds: | 20260508 10:33:41.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:41.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:33:41.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260508 10:33:41.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:41.179 | INFO | Uploaded 36GiB to GPU in 2.4339s (14.7911GiB/s) | 20260508 10:33:41.179 | INFO | Downloaded 34GiB to GPU in 2.44077s (13.9301GiB/s) | 20260508 10:33:46.328 | INFO | Analyzing chunk frames = [7421440,7421952) | 20260508 10:33:46.328 | INFO | MemInfo: { | 20260508 10:33:46.328 | INFO | "gpu" : " 17 GB", | 20260508 10:33:46.328 | INFO | "huge" : "", | 20260508 10:33:46.328 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:46.329 | INFO | "malloc" : " 14 GB", | 20260508 10:33:46.329 | INFO | "pinned" : " 29 GB" | 20260508 10:33:46.329 | INFO | } | 20260508 10:33:46.329 | INFO | Perf report for the last 5.15015 seconds: | 20260508 10:33:46.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:46.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | ANALYSIS : 65.5%, 3.0, 102.6ms, 0 | 20260508 10:33:46.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:33:46.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:46.329 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:33:46.329 | INFO | Total mem usage: 16.7 GiB | 20260508 10:33:46.329 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:33:46.329 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:33:46.329 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:33:46.329 | INFO | High Water Breakdown: | 20260508 10:33:46.329 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:33:46.330 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:33:46.330 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:33:46.330 | INFO | ------------------------------------------ | 20260508 10:33:46.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:33:46.330 | INFO | Uploaded 36GiB to GPU in 2.414s (14.913GiB/s) | 20260508 10:33:46.330 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:33:46.330 | INFO | Total mem usage: 26.5 GiB | 20260508 10:33:46.330 | INFO | Downloaded 34GiB to GPU in 2.43232s (13.9784GiB/s) | 20260508 10:33:46.330 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:33:46.330 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:33:46.330 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:33:46.330 | INFO | High Water Breakdown: | 20260508 10:33:46.330 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:33:46.330 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:33:46.330 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:33:46.330 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:33:46.330 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:33:46.330 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:33:46.330 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:33:46.330 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:33:46.330 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:33:46.330 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:33:46.330 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:33:46.330 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:33:46.330 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:33:46.330 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:33:46.330 | INFO | ------------------------------------------ | 20260508 10:33:46.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:33:46.330 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:33:46.330 | INFO | Total mem usage: 27.8 GiB | 20260508 10:33:46.330 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:33:46.330 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:33:46.330 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:33:46.330 | INFO | High Water Breakdown: | 20260508 10:33:46.330 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:33:46.330 | INFO | ------------------------------------------ | 20260508 10:33:46.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:33:46.330 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:33:46.330 | INFO | Total mem usage: 15.7 GiB | 20260508 10:33:46.330 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:33:46.330 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:33:46.330 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:33:46.330 | INFO | High Water Breakdown: | 20260508 10:33:46.330 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:33:46.330 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:33:46.330 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:33:46.330 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:33:46.330 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:33:46.330 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:33:46.330 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:33:46.330 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:33:46.330 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:33:46.330 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:33:46.330 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:33:46.330 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:33:46.330 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:33:46.330 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:33:46.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:33:46.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:33:46.330 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:33:46.330 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:33:46.330 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:33:46.330 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:33:46.330 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:33:46.330 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:33:46.330 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:33:46.330 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:33:46.330 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:33:46.330 | INFO | ------------------------------------------ | 20260508 10:33:46.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:33:46.330 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:33:46.330 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:33:46.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:33:46.330 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 6.40e-01 | 20260508 10:33:46.330 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.20e-02 / 5.17e-03 | 20260508 10:33:46.330 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.41e+00, 5.90e-01 / 2.39e-01 | 20260508 10:33:46.330 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.11e-01 / 1.26e-01 | 20260508 10:33:46.330 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.50e+01, 3.77e+00 / 1.22e+00 | 20260508 10:33:46.330 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.84e+00 / 1.42e+00 | 20260508 10:33:46.330 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.47e+00, 4.07e-01 / 2.87e-01 | 20260508 10:33:46.330 | INFO | PerfReport: 3.00%, Metrics -- 9.83e-01 / 1.15e+00, 2.43e-01 / 9.44e-02 | 20260508 10:33:46.330 | INFO | PerfReport: 0.92%, PulseSerializing -- 2.99e-01 / 3.32e-01, 4.92e-02 / 2.86e-02 | 20260508 10:33:46.330 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:33:46.330 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:33:46.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:33:46.330 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.44e+01 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 2.19%, LockUpload -- 2.31e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 2.54%, LockDownload -- 2.67e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 15.29%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:33:46.331 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260508 10:33:46.427 | INFO | Starting a round of Baz IO | 20260508 10:33:49.829 | INFO | Populating RTMetrics report | 20260508 10:33:49.829 | INFO | RTMetrics report populated | 20260508 10:33:49.829 | INFO | Opening tmp RTMetrics json file | 20260508 10:33:49.840 | INFO | Serializing RTMetrics to json | 20260508 10:33:49.841 | INFO | Serialization done, swapping tmp file now... | 20260508 10:33:49.841 | INFO | RTMetrics done | 20260508 10:33:49.905 | INFO | Finished a round of Baz IO: Wrote 1.458 GiB in 3.477 seconds | 20260508 10:33:49.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:33:49.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.219576% / 0% / 7.15256e-05% | 20260508 10:33:49.905 | INFO | Baz IO overall: | 20260508 10:33:49.905 | INFO | BytesWritten : 3218538807600 | 20260508 10:33:49.905 | INFO | NumEvents : 1546923849616 | 20260508 10:33:51.461 | INFO | Analyzing chunk frames = [7421952,7422464) | 20260508 10:33:51.461 | INFO | MemInfo: { | 20260508 10:33:51.461 | INFO | "gpu" : " 17 GB", | 20260508 10:33:51.461 | INFO | "huge" : "", | 20260508 10:33:51.461 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:51.461 | INFO | "malloc" : " 14 GB", | 20260508 10:33:51.461 | INFO | "pinned" : " 29 GB" | 20260508 10:33:51.461 | INFO | } | 20260508 10:33:51.461 | INFO | Perf report for the last 5.13233 seconds: | 20260508 10:33:51.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:51.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:51.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:51.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:51.461 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 10:33:51.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:51.461 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:33:51.461 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 10:33:51.461 | INFO | BAZWRITER : 67.1%, 2.5, 86.7ms, 0 | 20260508 10:33:51.461 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:33:51.462 | INFO | Uploaded 36GiB to GPU in 2.61003s (13.793GiB/s) | 20260508 10:33:51.462 | INFO | Downloaded 38GiB to GPU in 2.71802s (13.9808GiB/s) | 20260508 10:33:56.597 | INFO | Analyzing chunk frames = [7422464,7422976) | 20260508 10:33:56.597 | INFO | MemInfo: { | 20260508 10:33:56.597 | INFO | "gpu" : " 17 GB", | 20260508 10:33:56.597 | INFO | "huge" : "", | 20260508 10:33:56.597 | INFO | "huge_shared" : " 29 GB", | 20260508 10:33:56.597 | INFO | "malloc" : " 14 GB", | 20260508 10:33:56.597 | INFO | "pinned" : " 29 GB" | 20260508 10:33:56.597 | INFO | } | 20260508 10:33:56.598 | INFO | Perf report for the last 5.13651 seconds: | 20260508 10:33:56.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:33:56.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:33:56.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:33:56.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:33:56.598 | INFO | Uploaded 36GiB to GPU in 2.40485s (14.9697GiB/s) | 20260508 10:33:56.598 | INFO | Downloaded 34GiB to GPU in 2.47676s (13.7276GiB/s) | 20260508 10:34:01.741 | INFO | Analyzing chunk frames = [7422976,7423488) | 20260508 10:34:01.741 | INFO | MemInfo: { | 20260508 10:34:01.741 | INFO | "gpu" : " 17 GB", | 20260508 10:34:01.741 | INFO | "huge" : "", | 20260508 10:34:01.741 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:01.741 | INFO | "malloc" : " 14 GB", | 20260508 10:34:01.741 | INFO | "pinned" : " 29 GB" | 20260508 10:34:01.741 | INFO | } | 20260508 10:34:01.741 | INFO | Perf report for the last 5.14353 seconds: | 20260508 10:34:01.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:01.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 10:34:01.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:34:01.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:01.741 | INFO | Uploaded 36GiB to GPU in 2.45515s (14.6631GiB/s) | 20260508 10:34:01.741 | INFO | Downloaded 34GiB to GPU in 2.44886s (13.884GiB/s) | 20260508 10:34:06.876 | INFO | Analyzing chunk frames = [7423488,7424000) | 20260508 10:34:06.876 | INFO | MemInfo: { | 20260508 10:34:06.876 | INFO | "gpu" : " 17 GB", | 20260508 10:34:06.876 | INFO | "huge" : "", | 20260508 10:34:06.876 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:06.876 | INFO | "malloc" : " 14 GB", | 20260508 10:34:06.876 | INFO | "pinned" : " 29 GB" | 20260508 10:34:06.876 | INFO | } | 20260508 10:34:06.876 | INFO | Perf report for the last 5.13507 seconds: | 20260508 10:34:06.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:06.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.876 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 10:34:06.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.876 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:34:06.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:06.877 | INFO | Uploaded 36GiB to GPU in 2.44064s (14.7502GiB/s) | 20260508 10:34:06.877 | INFO | Downloaded 34GiB to GPU in 2.44455s (13.9085GiB/s) | 20260508 10:34:12.013 | INFO | Analyzing chunk frames = [7424000,7424512) | 20260508 10:34:12.013 | INFO | MemInfo: { | 20260508 10:34:12.013 | INFO | "gpu" : " 17 GB", | 20260508 10:34:12.013 | INFO | "huge" : "", | 20260508 10:34:12.013 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:12.013 | INFO | "malloc" : " 14 GB", | 20260508 10:34:12.013 | INFO | "pinned" : " 29 GB" | 20260508 10:34:12.013 | INFO | } | 20260508 10:34:12.013 | INFO | Perf report for the last 5.13636 seconds: | 20260508 10:34:12.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:12.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:34:12.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:34:12.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:12.013 | INFO | Uploaded 36GiB to GPU in 2.39789s (15.0132GiB/s) | 20260508 10:34:12.013 | INFO | Downloaded 34GiB to GPU in 2.54798s (13.3439GiB/s) | 20260508 10:34:17.150 | INFO | Analyzing chunk frames = [7424512,7425024) | 20260508 10:34:17.150 | INFO | MemInfo: { | 20260508 10:34:17.150 | INFO | "gpu" : " 17 GB", | 20260508 10:34:17.150 | INFO | "huge" : "", | 20260508 10:34:17.150 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:17.150 | INFO | "malloc" : " 14 GB", | 20260508 10:34:17.150 | INFO | "pinned" : " 29 GB" | 20260508 10:34:17.150 | INFO | } | 20260508 10:34:17.150 | INFO | Perf report for the last 5.13776 seconds: | 20260508 10:34:17.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:17.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.150 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:34:17.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.150 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 10:34:17.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:17.151 | INFO | Uploaded 36GiB to GPU in 2.36971s (15.1917GiB/s) | 20260508 10:34:17.151 | INFO | Downloaded 34GiB to GPU in 2.45042s (13.8752GiB/s) | 20260508 10:34:22.294 | INFO | Analyzing chunk frames = [7425024,7425536) | 20260508 10:34:22.294 | INFO | MemInfo: { | 20260508 10:34:22.294 | INFO | "gpu" : " 17 GB", | 20260508 10:34:22.294 | INFO | "huge" : "", | 20260508 10:34:22.294 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:22.294 | INFO | "malloc" : " 14 GB", | 20260508 10:34:22.294 | INFO | "pinned" : " 29 GB" | 20260508 10:34:22.294 | INFO | } | 20260508 10:34:22.295 | INFO | Perf report for the last 5.14426 seconds: | 20260508 10:34:22.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:22.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 10:34:22.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:34:22.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:22.295 | INFO | Uploaded 36GiB to GPU in 2.40756s (14.9529GiB/s) | 20260508 10:34:22.295 | INFO | Downloaded 34GiB to GPU in 2.43928s (13.9385GiB/s) | 20260508 10:34:27.432 | INFO | Analyzing chunk frames = [7425536,7426048) | 20260508 10:34:27.432 | INFO | MemInfo: { | 20260508 10:34:27.432 | INFO | "gpu" : " 17 GB", | 20260508 10:34:27.432 | INFO | "huge" : "", | 20260508 10:34:27.432 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:27.432 | INFO | "malloc" : " 14 GB", | 20260508 10:34:27.433 | INFO | "pinned" : " 29 GB" | 20260508 10:34:27.433 | INFO | } | 20260508 10:34:27.433 | INFO | Perf report for the last 5.13791 seconds: | 20260508 10:34:27.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:27.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:34:27.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:34:27.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:27.433 | INFO | Uploaded 36GiB to GPU in 2.43042s (14.8122GiB/s) | 20260508 10:34:27.433 | INFO | Downloaded 34GiB to GPU in 2.44322s (13.9161GiB/s) | 20260508 10:34:27.530 | INFO | Starting a round of Baz IO | 20260508 10:34:30.941 | INFO | Populating RTMetrics report | 20260508 10:34:30.941 | INFO | RTMetrics report populated | 20260508 10:34:30.941 | INFO | Opening tmp RTMetrics json file | 20260508 10:34:30.950 | INFO | Serializing RTMetrics to json | 20260508 10:34:30.951 | INFO | Serialization done, swapping tmp file now... | 20260508 10:34:30.952 | INFO | RTMetrics done | 20260508 10:34:31.009 | INFO | Finished a round of Baz IO: Wrote 1.455 GiB in 3.478 seconds | 20260508 10:34:31.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:34:31.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218376% / 0% / 6.35783e-05% | 20260508 10:34:31.009 | INFO | Baz IO overall: | 20260508 10:34:31.009 | INFO | BytesWritten : 3220101569335 | 20260508 10:34:31.009 | INFO | NumEvents : 1547660395759 | 20260508 10:34:32.573 | INFO | Analyzing chunk frames = [7426048,7426560) | 20260508 10:34:32.573 | INFO | MemInfo: { | 20260508 10:34:32.573 | INFO | "gpu" : " 17 GB", | 20260508 10:34:32.573 | INFO | "huge" : "", | 20260508 10:34:32.573 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:32.573 | INFO | "malloc" : " 14 GB", | 20260508 10:34:32.574 | INFO | "pinned" : " 29 GB" | 20260508 10:34:32.574 | INFO | } | 20260508 10:34:32.574 | INFO | Perf report for the last 5.14103 seconds: | 20260508 10:34:32.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:32.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:32.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:32.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:32.574 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:34:32.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:32.574 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:34:32.574 | INFO | PRE_HQ : 65.3%, 1.6, 54.7ms, 0 | 20260508 10:34:32.574 | INFO | BAZWRITER : 66.9%, 2.5, 86.9ms, 0 | 20260508 10:34:32.574 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:34:32.574 | INFO | Uploaded 36GiB to GPU in 2.59982s (13.8471GiB/s) | 20260508 10:34:32.574 | INFO | Downloaded 38GiB to GPU in 2.69821s (14.0834GiB/s) | 20260508 10:34:37.711 | INFO | Analyzing chunk frames = [7426560,7427072) | 20260508 10:34:37.711 | INFO | MemInfo: { | 20260508 10:34:37.711 | INFO | "gpu" : " 17 GB", | 20260508 10:34:37.711 | INFO | "huge" : "", | 20260508 10:34:37.711 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:37.711 | INFO | "malloc" : " 14 GB", | 20260508 10:34:37.711 | INFO | "pinned" : " 29 GB" | 20260508 10:34:37.711 | INFO | } | 20260508 10:34:37.711 | INFO | Perf report for the last 5.13783 seconds: | 20260508 10:34:37.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:37.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 10:34:37.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:34:37.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:37.712 | INFO | Uploaded 36GiB to GPU in 2.54107s (14.1673GiB/s) | 20260508 10:34:37.712 | INFO | Downloaded 34GiB to GPU in 2.47716s (13.7254GiB/s) | 20260508 10:34:42.848 | INFO | Analyzing chunk frames = [7427072,7427584) | 20260508 10:34:42.848 | INFO | MemInfo: { | 20260508 10:34:42.848 | INFO | "gpu" : " 17 GB", | 20260508 10:34:42.848 | INFO | "huge" : "", | 20260508 10:34:42.848 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:42.848 | INFO | "malloc" : " 14 GB", | 20260508 10:34:42.848 | INFO | "pinned" : " 29 GB" | 20260508 10:34:42.848 | INFO | } | 20260508 10:34:42.848 | INFO | Perf report for the last 5.13635 seconds: | 20260508 10:34:42.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:42.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:34:42.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:34:42.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:42.848 | INFO | Uploaded 36GiB to GPU in 2.47423s (14.55GiB/s) | 20260508 10:34:42.848 | INFO | Downloaded 34GiB to GPU in 2.43712s (13.9509GiB/s) | 20260508 10:34:47.992 | INFO | Analyzing chunk frames = [7427584,7428096) | 20260508 10:34:47.992 | INFO | MemInfo: { | 20260508 10:34:47.992 | INFO | "gpu" : " 17 GB", | 20260508 10:34:47.992 | INFO | "huge" : "", | 20260508 10:34:47.992 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:47.992 | INFO | "malloc" : " 14 GB", | 20260508 10:34:47.992 | INFO | "pinned" : " 29 GB" | 20260508 10:34:47.992 | INFO | } | 20260508 10:34:47.992 | INFO | Perf report for the last 5.1445 seconds: | 20260508 10:34:47.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:47.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.992 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 10:34:47.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.992 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:34:47.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:47.993 | INFO | Uploaded 36GiB to GPU in 2.45721s (14.6508GiB/s) | 20260508 10:34:47.993 | INFO | Downloaded 34GiB to GPU in 2.41464s (14.0807GiB/s) | 20260508 10:34:53.137 | INFO | Analyzing chunk frames = [7428096,7428608) | 20260508 10:34:53.137 | INFO | MemInfo: { | 20260508 10:34:53.137 | INFO | "gpu" : " 17 GB", | 20260508 10:34:53.137 | INFO | "huge" : "", | 20260508 10:34:53.137 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:53.137 | INFO | "malloc" : " 14 GB", | 20260508 10:34:53.137 | INFO | "pinned" : " 29 GB" | 20260508 10:34:53.137 | INFO | } | 20260508 10:34:53.137 | INFO | Perf report for the last 5.14505 seconds: | 20260508 10:34:53.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:53.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.137 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 10:34:53.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.138 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:34:53.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:53.138 | INFO | Uploaded 36GiB to GPU in 2.39386s (15.0385GiB/s) | 20260508 10:34:53.138 | INFO | Downloaded 34GiB to GPU in 2.54561s (13.3563GiB/s) | 20260508 10:34:58.272 | INFO | Analyzing chunk frames = [7428608,7429120) | 20260508 10:34:58.272 | INFO | MemInfo: { | 20260508 10:34:58.272 | INFO | "gpu" : " 17 GB", | 20260508 10:34:58.272 | INFO | "huge" : "", | 20260508 10:34:58.272 | INFO | "huge_shared" : " 29 GB", | 20260508 10:34:58.272 | INFO | "malloc" : " 14 GB", | 20260508 10:34:58.272 | INFO | "pinned" : " 29 GB" | 20260508 10:34:58.272 | INFO | } | 20260508 10:34:58.272 | INFO | Perf report for the last 5.13437 seconds: | 20260508 10:34:58.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:34:58.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 10:34:58.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:34:58.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:34:58.272 | INFO | Uploaded 36GiB to GPU in 2.3777s (15.1407GiB/s) | 20260508 10:34:58.272 | INFO | Downloaded 34GiB to GPU in 2.43603s (13.9572GiB/s) | 20260508 10:35:03.407 | INFO | Analyzing chunk frames = [7429120,7429632) | 20260508 10:35:03.407 | INFO | MemInfo: { | 20260508 10:35:03.407 | INFO | "gpu" : " 17 GB", | 20260508 10:35:03.407 | INFO | "huge" : "", | 20260508 10:35:03.408 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:03.408 | INFO | "malloc" : " 14 GB", | 20260508 10:35:03.408 | INFO | "pinned" : " 29 GB" | 20260508 10:35:03.408 | INFO | } | 20260508 10:35:03.408 | INFO | Perf report for the last 5.13593 seconds: | 20260508 10:35:03.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:03.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:35:03.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:35:03.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:03.408 | INFO | Uploaded 36GiB to GPU in 2.39765s (15.0147GiB/s) | 20260508 10:35:03.408 | INFO | Downloaded 34GiB to GPU in 2.44576s (13.9016GiB/s) | 20260508 10:35:08.547 | INFO | Analyzing chunk frames = [7429632,7430144) | 20260508 10:35:08.547 | INFO | MemInfo: { | 20260508 10:35:08.547 | INFO | "gpu" : " 17 GB", | 20260508 10:35:08.547 | INFO | "huge" : "", | 20260508 10:35:08.547 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:08.547 | INFO | "malloc" : " 14 GB", | 20260508 10:35:08.547 | INFO | "pinned" : " 29 GB" | 20260508 10:35:08.547 | INFO | } | 20260508 10:35:08.547 | INFO | Perf report for the last 5.1397 seconds: | 20260508 10:35:08.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:08.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.548 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:35:08.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.548 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:35:08.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:08.548 | INFO | Uploaded 36GiB to GPU in 2.43956s (14.7568GiB/s) | 20260508 10:35:08.548 | INFO | Downloaded 34GiB to GPU in 2.44906s (13.8829GiB/s) | 20260508 10:35:08.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:35:08.548 | INFO | Total mem usage: 16.7 GiB | 20260508 10:35:08.548 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:35:08.548 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:35:08.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:35:08.548 | INFO | High Water Breakdown: | 20260508 10:35:08.548 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:35:08.548 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:35:08.548 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:35:08.548 | INFO | ------------------------------------------ | 20260508 10:35:08.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:35:08.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:35:08.548 | INFO | Total mem usage: 26.5 GiB | 20260508 10:35:08.548 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:35:08.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:35:08.548 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:35:08.548 | INFO | High Water Breakdown: | 20260508 10:35:08.548 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:35:08.548 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:35:08.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:35:08.548 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:35:08.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:35:08.548 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:35:08.549 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:35:08.549 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:35:08.549 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:35:08.549 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:35:08.549 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:35:08.549 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:35:08.549 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:35:08.549 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:35:08.549 | INFO | ------------------------------------------ | 20260508 10:35:08.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:35:08.549 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:35:08.549 | INFO | Total mem usage: 27.8 GiB | 20260508 10:35:08.549 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:35:08.549 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:35:08.549 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:35:08.549 | INFO | High Water Breakdown: | 20260508 10:35:08.549 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:35:08.549 | INFO | ------------------------------------------ | 20260508 10:35:08.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:35:08.549 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:35:08.549 | INFO | Total mem usage: 15.7 GiB | 20260508 10:35:08.549 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:35:08.549 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:35:08.549 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:35:08.549 | INFO | High Water Breakdown: | 20260508 10:35:08.549 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:35:08.549 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:35:08.549 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:35:08.549 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:35:08.549 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:35:08.549 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:35:08.549 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:35:08.549 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:35:08.549 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:35:08.549 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:35:08.549 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:35:08.549 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:35:08.549 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:35:08.549 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:35:08.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:35:08.549 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:35:08.549 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:35:08.549 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:35:08.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:35:08.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:35:08.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:35:08.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:35:08.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:35:08.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:35:08.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:35:08.549 | INFO | ------------------------------------------ | 20260508 10:35:08.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:35:08.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:35:08.549 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:35:08.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:35:08.549 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.42e+01 / 7.58e-01 | 20260508 10:35:08.549 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.04e-01, 1.10e-02 / 6.04e-03 | 20260508 10:35:08.549 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.47e+00, 5.86e-01 / 1.79e-01 | 20260508 10:35:08.549 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.07e-01 / 1.27e-01 | 20260508 10:35:08.549 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.51e+01, 3.80e+00 / 9.71e-01 | 20260508 10:35:08.549 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.84e+00 / 1.28e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.02e-01 / 2.37e-01 | 20260508 10:35:08.549 | INFO | PerfReport: 3.00%, Metrics -- 9.82e-01 / 1.11e+00, 2.44e-01 / 1.30e-01 | 20260508 10:35:08.549 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.24e-01, 4.87e-02 / 2.76e-02 | 20260508 10:35:08.549 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:35:08.549 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:35:08.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:35:08.549 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 24.24%, Upload -- 2.55e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 24.66%, Download -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 2.09%, LockUpload -- 2.20e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 2.61%, LockDownload -- 2.75e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 15.31%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:35:08.549 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 10:35:08.644 | INFO | Starting a round of Baz IO | 20260508 10:35:12.053 | INFO | Populating RTMetrics report | 20260508 10:35:12.054 | INFO | RTMetrics report populated | 20260508 10:35:12.054 | INFO | Opening tmp RTMetrics json file | 20260508 10:35:12.064 | INFO | Serializing RTMetrics to json | 20260508 10:35:12.065 | INFO | Serialization done, swapping tmp file now... | 20260508 10:35:12.066 | INFO | RTMetrics done | 20260508 10:35:12.126 | INFO | Finished a round of Baz IO: Wrote 1.455 GiB in 3.481 seconds | 20260508 10:35:12.126 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.19% / 0.00% / 59.81% | 20260508 10:35:12.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218538% / 0% / 6.35783e-05% | 20260508 10:35:12.126 | INFO | Baz IO overall: | 20260508 10:35:12.126 | INFO | BytesWritten : 3221664267236 | 20260508 10:35:12.126 | INFO | NumEvents : 1548396792943 | 20260508 10:35:13.688 | INFO | Analyzing chunk frames = [7430144,7430656) | 20260508 10:35:13.688 | INFO | MemInfo: { | 20260508 10:35:13.688 | INFO | "gpu" : " 17 GB", | 20260508 10:35:13.688 | INFO | "huge" : "", | 20260508 10:35:13.688 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:13.688 | INFO | "malloc" : " 14 GB", | 20260508 10:35:13.688 | INFO | "pinned" : " 29 GB" | 20260508 10:35:13.688 | INFO | } | 20260508 10:35:13.688 | INFO | Perf report for the last 5.14069 seconds: | 20260508 10:35:13.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:13.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:13.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:13.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:13.688 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:35:13.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:13.688 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:35:13.688 | INFO | PRE_HQ : 65.2%, 1.6, 54.4ms, 0 | 20260508 10:35:13.688 | INFO | BAZWRITER : 67.1%, 2.5, 86.4ms, 0 | 20260508 10:35:13.688 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:35:13.689 | INFO | Uploaded 36GiB to GPU in 2.59945s (13.8491GiB/s) | 20260508 10:35:13.689 | INFO | Downloaded 38GiB to GPU in 2.69835s (14.0827GiB/s) | 20260508 10:35:18.828 | INFO | Analyzing chunk frames = [7430656,7431168) | 20260508 10:35:18.828 | INFO | MemInfo: { | 20260508 10:35:18.828 | INFO | "gpu" : " 17 GB", | 20260508 10:35:18.828 | INFO | "huge" : "", | 20260508 10:35:18.828 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:18.828 | INFO | "malloc" : " 14 GB", | 20260508 10:35:18.828 | INFO | "pinned" : " 29 GB" | 20260508 10:35:18.828 | INFO | } | 20260508 10:35:18.828 | INFO | Perf report for the last 5.13989 seconds: | 20260508 10:35:18.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:18.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:35:18.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:35:18.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:18.828 | INFO | Uploaded 36GiB to GPU in 2.52236s (14.2724GiB/s) | 20260508 10:35:18.828 | INFO | Downloaded 34GiB to GPU in 2.45789s (13.833GiB/s) | 20260508 10:35:23.967 | INFO | Analyzing chunk frames = [7431168,7431680) | 20260508 10:35:23.967 | INFO | MemInfo: { | 20260508 10:35:23.967 | INFO | "gpu" : " 17 GB", | 20260508 10:35:23.967 | INFO | "huge" : "", | 20260508 10:35:23.967 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:23.967 | INFO | "malloc" : " 14 GB", | 20260508 10:35:23.967 | INFO | "pinned" : " 29 GB" | 20260508 10:35:23.967 | INFO | } | 20260508 10:35:23.967 | INFO | Perf report for the last 5.13905 seconds: | 20260508 10:35:23.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:23.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:35:23.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260508 10:35:23.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:23.967 | INFO | Uploaded 36GiB to GPU in 2.45857s (14.6426GiB/s) | 20260508 10:35:23.967 | INFO | Downloaded 34GiB to GPU in 2.44005s (13.9341GiB/s) | 20260508 10:35:29.104 | INFO | Analyzing chunk frames = [7431680,7432192) | 20260508 10:35:29.104 | INFO | MemInfo: { | 20260508 10:35:29.104 | INFO | "gpu" : " 17 GB", | 20260508 10:35:29.104 | INFO | "huge" : "", | 20260508 10:35:29.104 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:29.104 | INFO | "malloc" : " 14 GB", | 20260508 10:35:29.104 | INFO | "pinned" : " 29 GB" | 20260508 10:35:29.104 | INFO | } | 20260508 10:35:29.104 | INFO | Perf report for the last 5.13706 seconds: | 20260508 10:35:29.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:29.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.104 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 10:35:29.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.104 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:35:29.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:29.105 | INFO | Uploaded 36GiB to GPU in 2.48656s (14.4778GiB/s) | 20260508 10:35:29.105 | INFO | Downloaded 34GiB to GPU in 2.42986s (13.9926GiB/s) | 20260508 10:35:34.249 | INFO | Analyzing chunk frames = [7432192,7432704) | 20260508 10:35:34.249 | INFO | MemInfo: { | 20260508 10:35:34.249 | INFO | "gpu" : " 17 GB", | 20260508 10:35:34.249 | INFO | "huge" : "", | 20260508 10:35:34.249 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:34.249 | INFO | "malloc" : " 14 GB", | 20260508 10:35:34.249 | INFO | "pinned" : " 29 GB" | 20260508 10:35:34.249 | INFO | } | 20260508 10:35:34.249 | INFO | Perf report for the last 5.1446 seconds: | 20260508 10:35:34.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:34.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 10:35:34.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:35:34.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:34.249 | INFO | Uploaded 36GiB to GPU in 2.34693s (15.3392GiB/s) | 20260508 10:35:34.249 | INFO | Downloaded 34GiB to GPU in 2.52606s (13.4597GiB/s) | 20260508 10:35:39.389 | INFO | Analyzing chunk frames = [7432704,7433216) | 20260508 10:35:39.389 | INFO | MemInfo: { | 20260508 10:35:39.389 | INFO | "gpu" : " 17 GB", | 20260508 10:35:39.389 | INFO | "huge" : "", | 20260508 10:35:39.389 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:39.389 | INFO | "malloc" : " 14 GB", | 20260508 10:35:39.389 | INFO | "pinned" : " 29 GB" | 20260508 10:35:39.389 | INFO | } | 20260508 10:35:39.389 | INFO | Perf report for the last 5.14042 seconds: | 20260508 10:35:39.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:39.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 10:35:39.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:35:39.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:39.389 | INFO | Uploaded 36GiB to GPU in 2.3697s (15.1918GiB/s) | 20260508 10:35:39.390 | INFO | Downloaded 34GiB to GPU in 2.43797s (13.946GiB/s) | 20260508 10:35:44.520 | INFO | Analyzing chunk frames = [7433216,7433728) | 20260508 10:35:44.520 | INFO | MemInfo: { | 20260508 10:35:44.520 | INFO | "gpu" : " 17 GB", | 20260508 10:35:44.520 | INFO | "huge" : "", | 20260508 10:35:44.520 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:44.520 | INFO | "malloc" : " 14 GB", | 20260508 10:35:44.520 | INFO | "pinned" : " 29 GB" | 20260508 10:35:44.520 | INFO | } | 20260508 10:35:44.520 | INFO | Perf report for the last 5.13097 seconds: | 20260508 10:35:44.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:44.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.520 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:35:44.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.520 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:35:44.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:44.521 | INFO | Uploaded 36GiB to GPU in 2.42941s (14.8184GiB/s) | 20260508 10:35:44.521 | INFO | Downloaded 34GiB to GPU in 2.44265s (13.9193GiB/s) | 20260508 10:35:49.657 | INFO | Analyzing chunk frames = [7433728,7434240) | 20260508 10:35:49.657 | INFO | MemInfo: { | 20260508 10:35:49.657 | INFO | "gpu" : " 17 GB", | 20260508 10:35:49.657 | INFO | "huge" : "", | 20260508 10:35:49.657 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:49.657 | INFO | "malloc" : " 14 GB", | 20260508 10:35:49.657 | INFO | "pinned" : " 29 GB" | 20260508 10:35:49.657 | INFO | } | 20260508 10:35:49.657 | INFO | Perf report for the last 5.13689 seconds: | 20260508 10:35:49.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:49.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 10:35:49.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 10:35:49.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:49.658 | INFO | Uploaded 36GiB to GPU in 2.44879s (14.7011GiB/s) | 20260508 10:35:49.658 | INFO | Downloaded 34GiB to GPU in 2.45875s (13.8281GiB/s) | 20260508 10:35:49.755 | INFO | Starting a round of Baz IO | 20260508 10:35:53.174 | INFO | Populating RTMetrics report | 20260508 10:35:53.175 | INFO | RTMetrics report populated | 20260508 10:35:53.175 | INFO | Opening tmp RTMetrics json file | 20260508 10:35:53.180 | INFO | Serializing RTMetrics to json | 20260508 10:35:53.181 | INFO | Serialization done, swapping tmp file now... | 20260508 10:35:53.182 | INFO | RTMetrics done | 20260508 10:35:53.243 | INFO | Finished a round of Baz IO: Wrote 1.456 GiB in 3.488 seconds | 20260508 10:35:53.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.81% | 20260508 10:35:53.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218149% / 3.97364e-06% / 7.15256e-05% | 20260508 10:35:53.243 | INFO | Baz IO overall: | 20260508 10:35:53.243 | INFO | BytesWritten : 3223227160079 | 20260508 10:35:53.243 | INFO | NumEvents : 1549133379552 | 20260508 10:35:54.803 | INFO | Analyzing chunk frames = [7434240,7434752) | 20260508 10:35:54.803 | INFO | MemInfo: { | 20260508 10:35:54.803 | INFO | "gpu" : " 17 GB", | 20260508 10:35:54.803 | INFO | "huge" : "", | 20260508 10:35:54.803 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:54.803 | INFO | "malloc" : " 14 GB", | 20260508 10:35:54.803 | INFO | "pinned" : " 29 GB" | 20260508 10:35:54.803 | INFO | } | 20260508 10:35:54.803 | INFO | Perf report for the last 5.14585 seconds: | 20260508 10:35:54.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:54.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:54.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:54.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:54.803 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 10:35:54.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:54.803 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:35:54.803 | INFO | PRE_HQ : 65.2%, 1.6, 54.4ms, 0 | 20260508 10:35:54.803 | INFO | BAZWRITER : 67.1%, 2.5, 87.1ms, 0 | 20260508 10:35:54.803 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:35:54.803 | INFO | Uploaded 36GiB to GPU in 2.61037s (13.7912GiB/s) | 20260508 10:35:54.803 | INFO | Downloaded 38GiB to GPU in 2.71036s (14.0203GiB/s) | 20260508 10:35:59.944 | INFO | Analyzing chunk frames = [7434752,7435264) | 20260508 10:35:59.944 | INFO | MemInfo: { | 20260508 10:35:59.944 | INFO | "gpu" : " 17 GB", | 20260508 10:35:59.944 | INFO | "huge" : "", | 20260508 10:35:59.944 | INFO | "huge_shared" : " 29 GB", | 20260508 10:35:59.945 | INFO | "malloc" : " 14 GB", | 20260508 10:35:59.945 | INFO | "pinned" : " 29 GB" | 20260508 10:35:59.945 | INFO | } | 20260508 10:35:59.945 | INFO | Perf report for the last 5.14166 seconds: | 20260508 10:35:59.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:35:59.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | ANALYSIS : 66.8%, 3.0, 104.2ms, 0 | 20260508 10:35:59.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260508 10:35:59.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:35:59.945 | INFO | Uploaded 36GiB to GPU in 2.53815s (14.1836GiB/s) | 20260508 10:35:59.945 | INFO | Downloaded 34GiB to GPU in 2.44769s (13.8906GiB/s) | 20260508 10:36:05.088 | INFO | Analyzing chunk frames = [7435264,7435776) | 20260508 10:36:05.088 | INFO | MemInfo: { | 20260508 10:36:05.088 | INFO | "gpu" : " 17 GB", | 20260508 10:36:05.088 | INFO | "huge" : "", | 20260508 10:36:05.088 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:05.088 | INFO | "malloc" : " 14 GB", | 20260508 10:36:05.088 | INFO | "pinned" : " 29 GB" | 20260508 10:36:05.088 | INFO | } | 20260508 10:36:05.088 | INFO | Perf report for the last 5.14374 seconds: | 20260508 10:36:05.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:05.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.088 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:36:05.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.089 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260508 10:36:05.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:05.089 | INFO | Uploaded 36GiB to GPU in 2.5286s (14.2371GiB/s) | 20260508 10:36:05.089 | INFO | Downloaded 34GiB to GPU in 2.44318s (13.9163GiB/s) | 20260508 10:36:10.223 | INFO | Analyzing chunk frames = [7435776,7436288) | 20260508 10:36:10.223 | INFO | MemInfo: { | 20260508 10:36:10.223 | INFO | "gpu" : " 17 GB", | 20260508 10:36:10.223 | INFO | "huge" : "", | 20260508 10:36:10.223 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:10.223 | INFO | "malloc" : " 14 GB", | 20260508 10:36:10.223 | INFO | "pinned" : " 29 GB" | 20260508 10:36:10.223 | INFO | } | 20260508 10:36:10.223 | INFO | Perf report for the last 5.13491 seconds: | 20260508 10:36:10.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:10.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.223 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 10:36:10.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.224 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 10:36:10.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:10.224 | INFO | Uploaded 36GiB to GPU in 2.54874s (14.1246GiB/s) | 20260508 10:36:10.224 | INFO | Downloaded 34GiB to GPU in 2.45841s (13.8301GiB/s) | 20260508 10:36:15.360 | INFO | Analyzing chunk frames = [7436288,7436800) | 20260508 10:36:15.360 | INFO | MemInfo: { | 20260508 10:36:15.360 | INFO | "gpu" : " 17 GB", | 20260508 10:36:15.360 | INFO | "huge" : "", | 20260508 10:36:15.360 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:15.360 | INFO | "malloc" : " 14 GB", | 20260508 10:36:15.360 | INFO | "pinned" : " 29 GB" | 20260508 10:36:15.360 | INFO | } | 20260508 10:36:15.360 | INFO | Perf report for the last 5.13666 seconds: | 20260508 10:36:15.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:15.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.360 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 10:36:15.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.360 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260508 10:36:15.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:15.361 | INFO | Uploaded 36GiB to GPU in 2.39911s (15.0056GiB/s) | 20260508 10:36:15.361 | INFO | Downloaded 34GiB to GPU in 2.4949s (13.6278GiB/s) | 20260508 10:36:20.503 | INFO | Analyzing chunk frames = [7436800,7437312) | 20260508 10:36:20.503 | INFO | MemInfo: { | 20260508 10:36:20.503 | INFO | "gpu" : " 17 GB", | 20260508 10:36:20.503 | INFO | "huge" : "", | 20260508 10:36:20.503 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:20.503 | INFO | "malloc" : " 14 GB", | 20260508 10:36:20.503 | INFO | "pinned" : " 29 GB" | 20260508 10:36:20.503 | INFO | } | 20260508 10:36:20.503 | INFO | Perf report for the last 5.14282 seconds: | 20260508 10:36:20.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:20.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.503 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:36:20.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.503 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:36:20.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:20.504 | INFO | Uploaded 36GiB to GPU in 2.38952s (15.0658GiB/s) | 20260508 10:36:20.504 | INFO | Downloaded 34GiB to GPU in 2.42513s (14.0199GiB/s) | 20260508 10:36:25.640 | INFO | Analyzing chunk frames = [7437312,7437824) | 20260508 10:36:25.640 | INFO | MemInfo: { | 20260508 10:36:25.640 | INFO | "gpu" : " 17 GB", | 20260508 10:36:25.640 | INFO | "huge" : "", | 20260508 10:36:25.640 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:25.640 | INFO | "malloc" : " 14 GB", | 20260508 10:36:25.640 | INFO | "pinned" : " 29 GB" | 20260508 10:36:25.640 | INFO | } | 20260508 10:36:25.640 | INFO | Perf report for the last 5.13737 seconds: | 20260508 10:36:25.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:25.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.640 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 10:36:25.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.640 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:36:25.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:25.641 | INFO | Uploaded 36GiB to GPU in 2.47705s (14.5334GiB/s) | 20260508 10:36:25.641 | INFO | Downloaded 34GiB to GPU in 2.4407s (13.9304GiB/s) | 20260508 10:36:30.780 | INFO | Analyzing chunk frames = [7437824,7438336) | 20260508 10:36:30.780 | INFO | MemInfo: { | 20260508 10:36:30.780 | INFO | "gpu" : " 17 GB", | 20260508 10:36:30.780 | INFO | "huge" : "", | 20260508 10:36:30.780 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:30.780 | INFO | "malloc" : " 14 GB", | 20260508 10:36:30.780 | INFO | "pinned" : " 29 GB" | 20260508 10:36:30.780 | INFO | } | 20260508 10:36:30.780 | INFO | Perf report for the last 5.13961 seconds: | 20260508 10:36:30.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:30.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 10:36:30.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:36:30.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:30.780 | INFO | Uploaded 36GiB to GPU in 2.46921s (14.5795GiB/s) | 20260508 10:36:30.780 | INFO | Downloaded 34GiB to GPU in 2.42352s (14.0292GiB/s) | 20260508 10:36:30.780 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:36:30.780 | INFO | Total mem usage: 16.7 GiB | 20260508 10:36:30.780 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:36:30.780 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:36:30.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:36:30.780 | INFO | High Water Breakdown: | 20260508 10:36:30.780 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:36:30.780 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:36:30.780 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:36:30.781 | INFO | ------------------------------------------ | 20260508 10:36:30.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:36:30.781 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:36:30.781 | INFO | Total mem usage: 26.5 GiB | 20260508 10:36:30.781 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:36:30.781 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:36:30.781 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:36:30.781 | INFO | High Water Breakdown: | 20260508 10:36:30.781 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:36:30.781 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:36:30.781 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:36:30.781 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:36:30.781 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:36:30.781 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:36:30.781 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:36:30.781 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:36:30.781 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:36:30.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:36:30.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:36:30.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:36:30.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:36:30.781 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:36:30.781 | INFO | ------------------------------------------ | 20260508 10:36:30.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:36:30.781 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:36:30.781 | INFO | Total mem usage: 27.8 GiB | 20260508 10:36:30.781 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:36:30.781 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:36:30.781 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:36:30.781 | INFO | High Water Breakdown: | 20260508 10:36:30.781 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:36:30.781 | INFO | ------------------------------------------ | 20260508 10:36:30.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:36:30.781 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:36:30.781 | INFO | Total mem usage: 15.7 GiB | 20260508 10:36:30.781 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:36:30.781 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:36:30.781 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:36:30.781 | INFO | High Water Breakdown: | 20260508 10:36:30.781 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:36:30.781 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:36:30.781 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:36:30.781 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:36:30.781 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:36:30.781 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:36:30.781 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:36:30.781 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:36:30.781 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:36:30.781 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:36:30.781 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:36:30.781 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:36:30.781 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:36:30.781 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:36:30.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:36:30.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:36:30.782 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:36:30.782 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:36:30.782 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:36:30.782 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:36:30.782 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:36:30.782 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:36:30.782 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:36:30.782 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:36:30.782 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:36:30.782 | INFO | ------------------------------------------ | 20260508 10:36:30.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:36:30.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:36:30.782 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:36:30.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:36:30.782 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 6.99e-01 | 20260508 10:36:30.782 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.06e-01, 1.25e-02 / 5.10e-03 | 20260508 10:36:30.782 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.40e+00, 5.89e-01 / 1.77e-01 | 20260508 10:36:30.782 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.09e-01 / 1.29e-01 | 20260508 10:36:30.782 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.52e+01, 3.72e+00 / 1.10e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.27e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.09e-01 / 2.24e-01 | 20260508 10:36:30.782 | INFO | PerfReport: 3.02%, Metrics -- 9.86e-01 / 1.13e+00, 2.44e-01 / 1.07e-01 | 20260508 10:36:30.782 | INFO | PerfReport: 0.93%, PulseSerializing -- 3.02e-01 / 3.35e-01, 4.91e-02 / 2.17e-02 | 20260508 10:36:30.782 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:36:30.782 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:36:30.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:36:30.782 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 24.47%, Upload -- 2.58e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 24.54%, Download -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 31.00%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 2.47%, LockDownload -- 2.60e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 15.27%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:36:30.782 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 10:36:30.876 | INFO | Starting a round of Baz IO | 20260508 10:36:34.301 | INFO | Populating RTMetrics report | 20260508 10:36:34.302 | INFO | RTMetrics report populated | 20260508 10:36:34.302 | INFO | Opening tmp RTMetrics json file | 20260508 10:36:34.313 | INFO | Serializing RTMetrics to json | 20260508 10:36:34.314 | INFO | Serialization done, swapping tmp file now... | 20260508 10:36:34.314 | INFO | RTMetrics done | 20260508 10:36:34.370 | INFO | Finished a round of Baz IO: Wrote 1.456 GiB in 3.493 seconds | 20260508 10:36:34.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:36:34.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218463% / 0% / 6.35783e-05% | 20260508 10:36:34.370 | INFO | Baz IO overall: | 20260508 10:36:34.370 | INFO | BytesWritten : 3224790629205 | 20260508 10:36:34.370 | INFO | NumEvents : 1549870512102 | 20260508 10:36:35.917 | INFO | Analyzing chunk frames = [7438336,7438848) | 20260508 10:36:35.917 | INFO | MemInfo: { | 20260508 10:36:35.917 | INFO | "gpu" : " 17 GB", | 20260508 10:36:35.917 | INFO | "huge" : "", | 20260508 10:36:35.917 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:35.917 | INFO | "malloc" : " 14 GB", | 20260508 10:36:35.917 | INFO | "pinned" : " 29 GB" | 20260508 10:36:35.917 | INFO | } | 20260508 10:36:35.917 | INFO | Perf report for the last 5.13688 seconds: | 20260508 10:36:35.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:35.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:35.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:35.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:35.917 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 10:36:35.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:35.917 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:36:35.917 | INFO | PRE_HQ : 65.7%, 1.6, 54.2ms, 0 | 20260508 10:36:35.917 | INFO | BAZWRITER : 67.2%, 2.5, 87.0ms, 0 | 20260508 10:36:35.917 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:36:35.918 | INFO | Uploaded 36GiB to GPU in 2.59802s (13.8567GiB/s) | 20260508 10:36:35.918 | INFO | Downloaded 38GiB to GPU in 2.70866s (14.0291GiB/s) | 20260508 10:36:41.063 | INFO | Analyzing chunk frames = [7438848,7439360) | 20260508 10:36:41.063 | INFO | MemInfo: { | 20260508 10:36:41.063 | INFO | "gpu" : " 17 GB", | 20260508 10:36:41.063 | INFO | "huge" : "", | 20260508 10:36:41.063 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:41.063 | INFO | "malloc" : " 14 GB", | 20260508 10:36:41.063 | INFO | "pinned" : " 29 GB" | 20260508 10:36:41.063 | INFO | } | 20260508 10:36:41.063 | INFO | Perf report for the last 5.14633 seconds: | 20260508 10:36:41.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:41.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 10:36:41.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 10:36:41.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:41.063 | INFO | Uploaded 36GiB to GPU in 2.49333s (14.4385GiB/s) | 20260508 10:36:41.063 | INFO | Downloaded 34GiB to GPU in 2.46539s (13.7909GiB/s) | 20260508 10:36:46.199 | INFO | Analyzing chunk frames = [7439360,7439872) | 20260508 10:36:46.199 | INFO | MemInfo: { | 20260508 10:36:46.199 | INFO | "gpu" : " 17 GB", | 20260508 10:36:46.199 | INFO | "huge" : "", | 20260508 10:36:46.199 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:46.199 | INFO | "malloc" : " 14 GB", | 20260508 10:36:46.199 | INFO | "pinned" : " 29 GB" | 20260508 10:36:46.199 | INFO | } | 20260508 10:36:46.199 | INFO | Perf report for the last 5.13621 seconds: | 20260508 10:36:46.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:46.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.199 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:36:46.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.200 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 10:36:46.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:46.200 | INFO | Uploaded 36GiB to GPU in 2.50614s (14.3647GiB/s) | 20260508 10:36:46.200 | INFO | Downloaded 34GiB to GPU in 2.44952s (13.8802GiB/s) | 20260508 10:36:51.334 | INFO | Analyzing chunk frames = [7439872,7440384) | 20260508 10:36:51.334 | INFO | MemInfo: { | 20260508 10:36:51.334 | INFO | "gpu" : " 17 GB", | 20260508 10:36:51.334 | INFO | "huge" : "", | 20260508 10:36:51.334 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:51.334 | INFO | "malloc" : " 14 GB", | 20260508 10:36:51.334 | INFO | "pinned" : " 29 GB" | 20260508 10:36:51.334 | INFO | } | 20260508 10:36:51.334 | INFO | Perf report for the last 5.13491 seconds: | 20260508 10:36:51.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:51.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.334 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 10:36:51.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.334 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 10:36:51.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:51.335 | INFO | Uploaded 36GiB to GPU in 2.51265s (14.3275GiB/s) | 20260508 10:36:51.335 | INFO | Downloaded 34GiB to GPU in 2.44611s (13.8996GiB/s) | 20260508 10:36:56.471 | INFO | Analyzing chunk frames = [7440384,7440896) | 20260508 10:36:56.471 | INFO | MemInfo: { | 20260508 10:36:56.471 | INFO | "gpu" : " 17 GB", | 20260508 10:36:56.471 | INFO | "huge" : "", | 20260508 10:36:56.471 | INFO | "huge_shared" : " 29 GB", | 20260508 10:36:56.471 | INFO | "malloc" : " 14 GB", | 20260508 10:36:56.471 | INFO | "pinned" : " 29 GB" | 20260508 10:36:56.471 | INFO | } | 20260508 10:36:56.471 | INFO | Perf report for the last 5.1372 seconds: | 20260508 10:36:56.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:36:56.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.472 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 10:36:56.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.472 | INFO | PRE_HQ : 31.0%, 1.0, 16.2ms, 0 | 20260508 10:36:56.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:36:56.472 | INFO | Uploaded 36GiB to GPU in 2.46672s (14.5943GiB/s) | 20260508 10:36:56.472 | INFO | Downloaded 34GiB to GPU in 2.53025s (13.4374GiB/s) | 20260508 10:37:01.612 | INFO | Analyzing chunk frames = [7440896,7441408) | 20260508 10:37:01.612 | INFO | MemInfo: { | 20260508 10:37:01.612 | INFO | "gpu" : " 17 GB", | 20260508 10:37:01.612 | INFO | "huge" : "", | 20260508 10:37:01.612 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:01.612 | INFO | "malloc" : " 14 GB", | 20260508 10:37:01.612 | INFO | "pinned" : " 29 GB" | 20260508 10:37:01.612 | INFO | } | 20260508 10:37:01.612 | INFO | Perf report for the last 5.14068 seconds: | 20260508 10:37:01.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:01.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:37:01.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:37:01.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:01.612 | INFO | Uploaded 36GiB to GPU in 2.39779s (15.0138GiB/s) | 20260508 10:37:01.613 | INFO | Downloaded 34GiB to GPU in 2.42498s (14.0207GiB/s) | 20260508 10:37:06.758 | INFO | Analyzing chunk frames = [7441408,7441920) | 20260508 10:37:06.758 | INFO | MemInfo: { | 20260508 10:37:06.758 | INFO | "gpu" : " 17 GB", | 20260508 10:37:06.758 | INFO | "huge" : "", | 20260508 10:37:06.758 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:06.758 | INFO | "malloc" : " 14 GB", | 20260508 10:37:06.758 | INFO | "pinned" : " 29 GB" | 20260508 10:37:06.758 | INFO | } | 20260508 10:37:06.758 | INFO | Perf report for the last 5.14586 seconds: | 20260508 10:37:06.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:06.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.758 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:37:06.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.758 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:37:06.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:06.759 | INFO | Uploaded 36GiB to GPU in 2.43287s (14.7973GiB/s) | 20260508 10:37:06.759 | INFO | Downloaded 34GiB to GPU in 2.43316s (13.9736GiB/s) | 20260508 10:37:11.898 | INFO | Analyzing chunk frames = [7441920,7442432) | 20260508 10:37:11.898 | INFO | MemInfo: { | 20260508 10:37:11.898 | INFO | "gpu" : " 17 GB", | 20260508 10:37:11.898 | INFO | "huge" : "", | 20260508 10:37:11.898 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:11.898 | INFO | "malloc" : " 14 GB", | 20260508 10:37:11.898 | INFO | "pinned" : " 29 GB" | 20260508 10:37:11.898 | INFO | } | 20260508 10:37:11.898 | INFO | Perf report for the last 5.14022 seconds: | 20260508 10:37:11.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:11.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.898 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:37:11.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.898 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:37:11.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:11.899 | INFO | Uploaded 36GiB to GPU in 2.46041s (14.6317GiB/s) | 20260508 10:37:11.899 | INFO | Downloaded 34GiB to GPU in 2.43409s (13.9683GiB/s) | 20260508 10:37:11.996 | INFO | Starting a round of Baz IO | 20260508 10:37:15.399 | INFO | Populating RTMetrics report | 20260508 10:37:15.399 | INFO | RTMetrics report populated | 20260508 10:37:15.399 | INFO | Opening tmp RTMetrics json file | 20260508 10:37:15.409 | INFO | Serializing RTMetrics to json | 20260508 10:37:15.410 | INFO | Serialization done, swapping tmp file now... | 20260508 10:37:15.411 | INFO | RTMetrics done | 20260508 10:37:15.469 | INFO | Finished a round of Baz IO: Wrote 1.455 GiB in 3.473 seconds | 20260508 10:37:15.469 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:37:15.469 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.217315% / 0% / 6.35783e-05% | 20260508 10:37:15.469 | INFO | Baz IO overall: | 20260508 10:37:15.469 | INFO | BytesWritten : 3226353050398 | 20260508 10:37:15.469 | INFO | NumEvents : 1550606855404 | 20260508 10:37:17.038 | INFO | Analyzing chunk frames = [7442432,7442944) | 20260508 10:37:17.038 | INFO | MemInfo: { | 20260508 10:37:17.038 | INFO | "gpu" : " 17 GB", | 20260508 10:37:17.038 | INFO | "huge" : "", | 20260508 10:37:17.038 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:17.039 | INFO | "malloc" : " 14 GB", | 20260508 10:37:17.039 | INFO | "pinned" : " 29 GB" | 20260508 10:37:17.039 | INFO | } | 20260508 10:37:17.039 | INFO | Perf report for the last 5.14034 seconds: | 20260508 10:37:17.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:17.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:17.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:17.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:17.039 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 10:37:17.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:17.039 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:37:17.039 | INFO | PRE_HQ : 65.4%, 1.6, 54.3ms, 0 | 20260508 10:37:17.039 | INFO | BAZWRITER : 66.9%, 2.5, 86.1ms, 0 | 20260508 10:37:17.039 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:37:17.039 | INFO | Uploaded 36GiB to GPU in 2.63503s (13.6621GiB/s) | 20260508 10:37:17.039 | INFO | Downloaded 38GiB to GPU in 2.70807s (14.0321GiB/s) | 20260508 10:37:22.166 | INFO | Analyzing chunk frames = [7442944,7443456) | 20260508 10:37:22.166 | INFO | MemInfo: { | 20260508 10:37:22.166 | INFO | "gpu" : " 17 GB", | 20260508 10:37:22.166 | INFO | "huge" : "", | 20260508 10:37:22.166 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:22.167 | INFO | "malloc" : " 14 GB", | 20260508 10:37:22.167 | INFO | "pinned" : " 29 GB" | 20260508 10:37:22.167 | INFO | } | 20260508 10:37:22.167 | INFO | Perf report for the last 5.12802 seconds: | 20260508 10:37:22.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:22.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:37:22.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:37:22.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:22.167 | INFO | Uploaded 36GiB to GPU in 2.46478s (14.6058GiB/s) | 20260508 10:37:22.167 | INFO | Downloaded 34GiB to GPU in 2.45779s (13.8335GiB/s) | 20260508 10:37:27.310 | INFO | Analyzing chunk frames = [7443456,7443968) | 20260508 10:37:27.310 | INFO | MemInfo: { | 20260508 10:37:27.310 | INFO | "gpu" : " 17 GB", | 20260508 10:37:27.310 | INFO | "huge" : "", | 20260508 10:37:27.310 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:27.310 | INFO | "malloc" : " 14 GB", | 20260508 10:37:27.310 | INFO | "pinned" : " 29 GB" | 20260508 10:37:27.310 | INFO | } | 20260508 10:37:27.310 | INFO | Perf report for the last 5.14317 seconds: | 20260508 10:37:27.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:27.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 10:37:27.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:37:27.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:27.310 | INFO | Uploaded 36GiB to GPU in 2.48601s (14.481GiB/s) | 20260508 10:37:27.310 | INFO | Downloaded 34GiB to GPU in 2.4213s (14.0421GiB/s) | 20260508 10:37:32.453 | INFO | Analyzing chunk frames = [7443968,7444480) | 20260508 10:37:32.453 | INFO | MemInfo: { | 20260508 10:37:32.453 | INFO | "gpu" : " 17 GB", | 20260508 10:37:32.453 | INFO | "huge" : "", | 20260508 10:37:32.453 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:32.453 | INFO | "malloc" : " 14 GB", | 20260508 10:37:32.454 | INFO | "pinned" : " 29 GB" | 20260508 10:37:32.454 | INFO | } | 20260508 10:37:32.454 | INFO | Perf report for the last 5.1438 seconds: | 20260508 10:37:32.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:32.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 10:37:32.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:37:32.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:32.454 | INFO | Uploaded 36GiB to GPU in 2.48674s (14.4768GiB/s) | 20260508 10:37:32.454 | INFO | Downloaded 34GiB to GPU in 2.40942s (14.1113GiB/s) | 20260508 10:37:37.592 | INFO | Analyzing chunk frames = [7444480,7444992) | 20260508 10:37:37.592 | INFO | MemInfo: { | 20260508 10:37:37.592 | INFO | "gpu" : " 17 GB", | 20260508 10:37:37.592 | INFO | "huge" : "", | 20260508 10:37:37.592 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:37.592 | INFO | "malloc" : " 14 GB", | 20260508 10:37:37.592 | INFO | "pinned" : " 29 GB" | 20260508 10:37:37.593 | INFO | } | 20260508 10:37:37.593 | INFO | Perf report for the last 5.13891 seconds: | 20260508 10:37:37.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:37.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 10:37:37.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:37:37.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:37.593 | INFO | Uploaded 36GiB to GPU in 2.29719s (15.6713GiB/s) | 20260508 10:37:37.593 | INFO | Downloaded 34GiB to GPU in 2.4976s (13.6131GiB/s) | 20260508 10:37:42.730 | INFO | Analyzing chunk frames = [7444992,7445504) | 20260508 10:37:42.730 | INFO | MemInfo: { | 20260508 10:37:42.730 | INFO | "gpu" : " 17 GB", | 20260508 10:37:42.730 | INFO | "huge" : "", | 20260508 10:37:42.730 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:42.730 | INFO | "malloc" : " 14 GB", | 20260508 10:37:42.730 | INFO | "pinned" : " 29 GB" | 20260508 10:37:42.730 | INFO | } | 20260508 10:37:42.730 | INFO | Perf report for the last 5.13763 seconds: | 20260508 10:37:42.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:42.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.730 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 10:37:42.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.730 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:37:42.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:42.731 | INFO | Uploaded 36GiB to GPU in 2.39938s (15.0039GiB/s) | 20260508 10:37:42.731 | INFO | Downloaded 34GiB to GPU in 2.42321s (14.031GiB/s) | 20260508 10:37:47.870 | INFO | Analyzing chunk frames = [7445504,7446016) | 20260508 10:37:47.870 | INFO | MemInfo: { | 20260508 10:37:47.870 | INFO | "gpu" : " 17 GB", | 20260508 10:37:47.870 | INFO | "huge" : "", | 20260508 10:37:47.870 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:47.870 | INFO | "malloc" : " 14 GB", | 20260508 10:37:47.870 | INFO | "pinned" : " 29 GB" | 20260508 10:37:47.870 | INFO | } | 20260508 10:37:47.870 | INFO | Perf report for the last 5.13961 seconds: | 20260508 10:37:47.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:47.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:37:47.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 10:37:47.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:47.870 | INFO | Uploaded 36GiB to GPU in 2.42172s (14.8655GiB/s) | 20260508 10:37:47.870 | INFO | Downloaded 34GiB to GPU in 2.43508s (13.9626GiB/s) | 20260508 10:37:53.010 | INFO | Analyzing chunk frames = [7446016,7446528) | 20260508 10:37:53.010 | INFO | MemInfo: { | 20260508 10:37:53.010 | INFO | "gpu" : " 17 GB", | 20260508 10:37:53.010 | INFO | "huge" : "", | 20260508 10:37:53.010 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:53.010 | INFO | "malloc" : " 14 GB", | 20260508 10:37:53.010 | INFO | "pinned" : " 29 GB" | 20260508 10:37:53.010 | INFO | } | 20260508 10:37:53.010 | INFO | Perf report for the last 5.14044 seconds: | 20260508 10:37:53.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:53.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.010 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 10:37:53.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.010 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:37:53.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:53.011 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:37:53.011 | INFO | Total mem usage: 16.7 GiB | 20260508 10:37:53.011 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:37:53.011 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:37:53.011 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:37:53.011 | INFO | High Water Breakdown: | 20260508 10:37:53.011 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:37:53.011 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:37:53.011 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:37:53.011 | INFO | ------------------------------------------ | 20260508 10:37:53.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:37:53.011 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:37:53.011 | INFO | Total mem usage: 26.5 GiB | 20260508 10:37:53.011 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:37:53.011 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:37:53.011 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:37:53.011 | INFO | High Water Breakdown: | 20260508 10:37:53.011 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:37:53.011 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:37:53.011 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:37:53.011 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:37:53.011 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:37:53.011 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:37:53.011 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:37:53.011 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:37:53.011 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:37:53.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:37:53.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:37:53.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:37:53.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:37:53.011 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:37:53.011 | INFO | ------------------------------------------ | 20260508 10:37:53.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:37:53.011 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:37:53.011 | INFO | Total mem usage: 27.8 GiB | 20260508 10:37:53.011 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:37:53.011 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:37:53.011 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:37:53.011 | INFO | High Water Breakdown: | 20260508 10:37:53.011 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:37:53.011 | INFO | ------------------------------------------ | 20260508 10:37:53.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:37:53.011 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:37:53.011 | INFO | Total mem usage: 15.7 GiB | 20260508 10:37:53.011 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:37:53.011 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:37:53.011 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:37:53.011 | INFO | High Water Breakdown: | 20260508 10:37:53.011 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:37:53.011 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:37:53.011 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:37:53.011 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:37:53.011 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:37:53.011 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:37:53.011 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:37:53.011 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:37:53.011 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:37:53.011 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:37:53.011 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:37:53.011 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:37:53.011 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:37:53.011 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:37:53.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:37:53.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:37:53.011 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:37:53.011 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:37:53.011 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:37:53.011 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:37:53.011 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:37:53.011 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:37:53.011 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:37:53.011 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:37:53.011 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:37:53.011 | INFO | ------------------------------------------ | 20260508 10:37:53.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:37:53.011 | INFO | Uploaded 36GiB to GPU in 2.4564s (14.6556GiB/s) | 20260508 10:37:53.011 | INFO | Downloaded 34GiB to GPU in 2.44312s (13.9167GiB/s) | 20260508 10:37:53.011 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:37:53.011 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:37:53.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:37:53.011 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 5.31e-01 | 20260508 10:37:53.011 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.15e-02 / 5.08e-03 | 20260508 10:37:53.011 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.41e+00, 5.96e-01 / 2.21e-01 | 20260508 10:37:53.011 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.44e+00, 2.10e-01 / 1.19e-01 | 20260508 10:37:53.011 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.55e+01, 3.78e+00 / 1.11e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.21e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.04e-01 / 2.05e-01 | 20260508 10:37:53.011 | INFO | PerfReport: 3.03%, Metrics -- 9.89e-01 / 1.14e+00, 2.48e-01 / 8.00e-02 | 20260508 10:37:53.011 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.34e-01, 4.90e-02 / 2.10e-02 | 20260508 10:37:53.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:37:53.011 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:37:53.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:37:53.011 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 24.43%, Upload -- 2.57e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 24.53%, Download -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 2.30%, LockUpload -- 2.42e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 2.45%, LockDownload -- 2.58e+00 / 0.00e+00, 3.94e+00 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 15.23%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:37:53.011 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 10:37:53.108 | INFO | Starting a round of Baz IO | 20260508 10:37:56.519 | INFO | Populating RTMetrics report | 20260508 10:37:56.520 | INFO | RTMetrics report populated | 20260508 10:37:56.520 | INFO | Opening tmp RTMetrics json file | 20260508 10:37:56.521 | INFO | Serializing RTMetrics to json | 20260508 10:37:56.522 | INFO | Serialization done, swapping tmp file now... | 20260508 10:37:56.523 | INFO | RTMetrics done | 20260508 10:37:56.584 | INFO | Finished a round of Baz IO: Wrote 1.455 GiB in 3.475 seconds | 20260508 10:37:56.584 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:37:56.584 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216488% / 0% / 6.35783e-05% | 20260508 10:37:56.584 | INFO | Baz IO overall: | 20260508 10:37:56.584 | INFO | BytesWritten : 3227915716669 | 20260508 10:37:56.584 | INFO | NumEvents : 1551343377647 | 20260508 10:37:58.151 | INFO | Analyzing chunk frames = [7446528,7447040) | 20260508 10:37:58.151 | INFO | MemInfo: { | 20260508 10:37:58.151 | INFO | "gpu" : " 17 GB", | 20260508 10:37:58.151 | INFO | "huge" : "", | 20260508 10:37:58.151 | INFO | "huge_shared" : " 29 GB", | 20260508 10:37:58.151 | INFO | "malloc" : " 14 GB", | 20260508 10:37:58.151 | INFO | "pinned" : " 29 GB" | 20260508 10:37:58.151 | INFO | } | 20260508 10:37:58.151 | INFO | Perf report for the last 5.14101 seconds: | 20260508 10:37:58.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:37:58.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:58.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:58.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:58.152 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 10:37:58.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:37:58.152 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 10:37:58.152 | INFO | PRE_HQ : 65.7%, 1.6, 53.9ms, 0 | 20260508 10:37:58.152 | INFO | BAZWRITER : 67.0%, 2.5, 85.6ms, 0 | 20260508 10:37:58.152 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:37:58.152 | INFO | Uploaded 36GiB to GPU in 2.51179s (14.3324GiB/s) | 20260508 10:37:58.152 | INFO | Downloaded 38GiB to GPU in 2.69326s (14.1093GiB/s) | 20260508 10:38:03.284 | INFO | Analyzing chunk frames = [7447040,7447552) | 20260508 10:38:03.284 | INFO | MemInfo: { | 20260508 10:38:03.284 | INFO | "gpu" : " 17 GB", | 20260508 10:38:03.284 | INFO | "huge" : "", | 20260508 10:38:03.284 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:03.284 | INFO | "malloc" : " 14 GB", | 20260508 10:38:03.284 | INFO | "pinned" : " 29 GB" | 20260508 10:38:03.284 | INFO | } | 20260508 10:38:03.284 | INFO | Perf report for the last 5.13256 seconds: | 20260508 10:38:03.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:03.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 10:38:03.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 10:38:03.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:03.284 | INFO | Uploaded 36GiB to GPU in 2.4686s (14.5832GiB/s) | 20260508 10:38:03.284 | INFO | Downloaded 34GiB to GPU in 2.4639s (13.7992GiB/s) | 20260508 10:38:08.422 | INFO | Analyzing chunk frames = [7447552,7448064) | 20260508 10:38:08.422 | INFO | MemInfo: { | 20260508 10:38:08.422 | INFO | "gpu" : " 17 GB", | 20260508 10:38:08.422 | INFO | "huge" : "", | 20260508 10:38:08.422 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:08.423 | INFO | "malloc" : " 14 GB", | 20260508 10:38:08.423 | INFO | "pinned" : " 29 GB" | 20260508 10:38:08.423 | INFO | } | 20260508 10:38:08.423 | INFO | Perf report for the last 5.13868 seconds: | 20260508 10:38:08.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:08.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 10:38:08.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:38:08.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:08.423 | INFO | Uploaded 36GiB to GPU in 2.50554s (14.3682GiB/s) | 20260508 10:38:08.423 | INFO | Downloaded 34GiB to GPU in 2.42907s (13.9971GiB/s) | 20260508 10:38:13.564 | INFO | Analyzing chunk frames = [7448064,7448576) | 20260508 10:38:13.564 | INFO | MemInfo: { | 20260508 10:38:13.564 | INFO | "gpu" : " 17 GB", | 20260508 10:38:13.564 | INFO | "huge" : "", | 20260508 10:38:13.564 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:13.564 | INFO | "malloc" : " 14 GB", | 20260508 10:38:13.564 | INFO | "pinned" : " 29 GB" | 20260508 10:38:13.564 | INFO | } | 20260508 10:38:13.564 | INFO | Perf report for the last 5.14188 seconds: | 20260508 10:38:13.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:13.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.565 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:38:13.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.565 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:38:13.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:13.565 | INFO | Uploaded 36GiB to GPU in 2.47845s (14.5252GiB/s) | 20260508 10:38:13.565 | INFO | Downloaded 34GiB to GPU in 2.41447s (14.0818GiB/s) | 20260508 10:38:18.708 | INFO | Analyzing chunk frames = [7448576,7449088) | 20260508 10:38:18.708 | INFO | MemInfo: { | 20260508 10:38:18.708 | INFO | "gpu" : " 17 GB", | 20260508 10:38:18.708 | INFO | "huge" : "", | 20260508 10:38:18.708 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:18.708 | INFO | "malloc" : " 14 GB", | 20260508 10:38:18.708 | INFO | "pinned" : " 29 GB" | 20260508 10:38:18.708 | INFO | } | 20260508 10:38:18.709 | INFO | Perf report for the last 5.14405 seconds: | 20260508 10:38:18.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:18.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 10:38:18.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:38:18.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:18.709 | INFO | Uploaded 36GiB to GPU in 2.43346s (14.7937GiB/s) | 20260508 10:38:18.709 | INFO | Downloaded 34GiB to GPU in 2.54433s (13.3631GiB/s) | 20260508 10:38:23.844 | INFO | Analyzing chunk frames = [7449088,7449600) | 20260508 10:38:23.844 | INFO | MemInfo: { | 20260508 10:38:23.844 | INFO | "gpu" : " 17 GB", | 20260508 10:38:23.844 | INFO | "huge" : "", | 20260508 10:38:23.844 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:23.844 | INFO | "malloc" : " 14 GB", | 20260508 10:38:23.844 | INFO | "pinned" : " 29 GB" | 20260508 10:38:23.844 | INFO | } | 20260508 10:38:23.844 | INFO | Perf report for the last 5.13568 seconds: | 20260508 10:38:23.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:23.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 10:38:23.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:38:23.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:23.844 | INFO | Uploaded 36GiB to GPU in 2.37731s (15.1432GiB/s) | 20260508 10:38:23.845 | INFO | Downloaded 34GiB to GPU in 2.41431s (14.0827GiB/s) | 20260508 10:38:28.982 | INFO | Analyzing chunk frames = [7449600,7450112) | 20260508 10:38:28.982 | INFO | MemInfo: { | 20260508 10:38:28.982 | INFO | "gpu" : " 17 GB", | 20260508 10:38:28.982 | INFO | "huge" : "", | 20260508 10:38:28.982 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:28.982 | INFO | "malloc" : " 14 GB", | 20260508 10:38:28.982 | INFO | "pinned" : " 29 GB" | 20260508 10:38:28.982 | INFO | } | 20260508 10:38:28.982 | INFO | Perf report for the last 5.13825 seconds: | 20260508 10:38:28.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:28.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:38:28.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:38:28.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:28.983 | INFO | Uploaded 36GiB to GPU in 2.43273s (14.7982GiB/s) | 20260508 10:38:28.983 | INFO | Downloaded 34GiB to GPU in 2.42638s (14.0127GiB/s) | 20260508 10:38:34.121 | INFO | Analyzing chunk frames = [7450112,7450624) | 20260508 10:38:34.121 | INFO | MemInfo: { | 20260508 10:38:34.121 | INFO | "gpu" : " 17 GB", | 20260508 10:38:34.121 | INFO | "huge" : "", | 20260508 10:38:34.121 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:34.121 | INFO | "malloc" : " 14 GB", | 20260508 10:38:34.121 | INFO | "pinned" : " 29 GB" | 20260508 10:38:34.121 | INFO | } | 20260508 10:38:34.121 | INFO | Perf report for the last 5.13862 seconds: | 20260508 10:38:34.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:34.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:38:34.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 10:38:34.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:34.121 | INFO | Uploaded 36GiB to GPU in 2.44686s (14.7128GiB/s) | 20260508 10:38:34.121 | INFO | Downloaded 34GiB to GPU in 2.43415s (13.9679GiB/s) | 20260508 10:38:34.219 | INFO | Starting a round of Baz IO | 20260508 10:38:37.626 | INFO | Populating RTMetrics report | 20260508 10:38:37.627 | INFO | RTMetrics report populated | 20260508 10:38:37.627 | INFO | Opening tmp RTMetrics json file | 20260508 10:38:37.635 | INFO | Serializing RTMetrics to json | 20260508 10:38:37.636 | INFO | Serialization done, swapping tmp file now... | 20260508 10:38:37.637 | INFO | RTMetrics done | 20260508 10:38:37.698 | INFO | Finished a round of Baz IO: Wrote 1.456 GiB in 3.479 seconds | 20260508 10:38:37.698 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:38:37.698 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216953% / 0% / 5.5631e-05% | 20260508 10:38:37.698 | INFO | Baz IO overall: | 20260508 10:38:37.698 | INFO | BytesWritten : 3229478725284 | 20260508 10:38:37.698 | INFO | NumEvents : 1552080255539 | 20260508 10:38:39.263 | INFO | Analyzing chunk frames = [7450624,7451136) | 20260508 10:38:39.263 | INFO | MemInfo: { | 20260508 10:38:39.263 | INFO | "gpu" : " 17 GB", | 20260508 10:38:39.263 | INFO | "huge" : "", | 20260508 10:38:39.263 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:39.263 | INFO | "malloc" : " 14 GB", | 20260508 10:38:39.263 | INFO | "pinned" : " 29 GB" | 20260508 10:38:39.263 | INFO | } | 20260508 10:38:39.263 | INFO | Perf report for the last 5.142 seconds: | 20260508 10:38:39.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:39.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:39.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:39.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:39.263 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:38:39.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:39.263 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:38:39.263 | INFO | PRE_HQ : 65.4%, 1.6, 54.7ms, 0 | 20260508 10:38:39.263 | INFO | BAZWRITER : 67.0%, 2.5, 86.3ms, 0 | 20260508 10:38:39.263 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:38:39.264 | INFO | Uploaded 36GiB to GPU in 2.59938s (13.8495GiB/s) | 20260508 10:38:39.264 | INFO | Downloaded 38GiB to GPU in 2.72275s (13.9565GiB/s) | 20260508 10:38:44.407 | INFO | Analyzing chunk frames = [7451136,7451648) | 20260508 10:38:44.407 | INFO | MemInfo: { | 20260508 10:38:44.407 | INFO | "gpu" : " 17 GB", | 20260508 10:38:44.407 | INFO | "huge" : "", | 20260508 10:38:44.407 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:44.407 | INFO | "malloc" : " 14 GB", | 20260508 10:38:44.407 | INFO | "pinned" : " 29 GB" | 20260508 10:38:44.407 | INFO | } | 20260508 10:38:44.407 | INFO | Perf report for the last 5.14421 seconds: | 20260508 10:38:44.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:44.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.407 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 10:38:44.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.407 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:38:44.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:44.408 | INFO | Uploaded 36GiB to GPU in 2.45786s (14.6469GiB/s) | 20260508 10:38:44.408 | INFO | Downloaded 34GiB to GPU in 2.47147s (13.757GiB/s) | 20260508 10:38:49.546 | INFO | Analyzing chunk frames = [7451648,7452160) | 20260508 10:38:49.546 | INFO | MemInfo: { | 20260508 10:38:49.546 | INFO | "gpu" : " 17 GB", | 20260508 10:38:49.546 | INFO | "huge" : "", | 20260508 10:38:49.546 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:49.546 | INFO | "malloc" : " 14 GB", | 20260508 10:38:49.546 | INFO | "pinned" : " 29 GB" | 20260508 10:38:49.546 | INFO | } | 20260508 10:38:49.546 | INFO | Perf report for the last 5.13911 seconds: | 20260508 10:38:49.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:49.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.547 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:38:49.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.547 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:38:49.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:49.547 | INFO | Uploaded 36GiB to GPU in 2.45529s (14.6622GiB/s) | 20260508 10:38:49.547 | INFO | Downloaded 34GiB to GPU in 2.4445s (13.9088GiB/s) | 20260508 10:38:54.677 | INFO | Analyzing chunk frames = [7452160,7452672) | 20260508 10:38:54.677 | INFO | MemInfo: { | 20260508 10:38:54.677 | INFO | "gpu" : " 17 GB", | 20260508 10:38:54.677 | INFO | "huge" : "", | 20260508 10:38:54.677 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:54.677 | INFO | "malloc" : " 14 GB", | 20260508 10:38:54.677 | INFO | "pinned" : " 29 GB" | 20260508 10:38:54.677 | INFO | } | 20260508 10:38:54.678 | INFO | Perf report for the last 5.13109 seconds: | 20260508 10:38:54.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:54.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 10:38:54.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:38:54.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:54.678 | INFO | Uploaded 36GiB to GPU in 2.50776s (14.3554GiB/s) | 20260508 10:38:54.678 | INFO | Downloaded 34GiB to GPU in 2.46641s (13.7852GiB/s) | 20260508 10:38:59.822 | INFO | Analyzing chunk frames = [7452672,7453184) | 20260508 10:38:59.822 | INFO | MemInfo: { | 20260508 10:38:59.822 | INFO | "gpu" : " 17 GB", | 20260508 10:38:59.822 | INFO | "huge" : "", | 20260508 10:38:59.822 | INFO | "huge_shared" : " 29 GB", | 20260508 10:38:59.822 | INFO | "malloc" : " 14 GB", | 20260508 10:38:59.822 | INFO | "pinned" : " 29 GB" | 20260508 10:38:59.822 | INFO | } | 20260508 10:38:59.822 | INFO | Perf report for the last 5.14426 seconds: | 20260508 10:38:59.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:38:59.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 10:38:59.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:38:59.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:38:59.822 | INFO | Uploaded 36GiB to GPU in 2.29086s (15.7146GiB/s) | 20260508 10:38:59.822 | INFO | Downloaded 34GiB to GPU in 2.52092s (13.4871GiB/s) | 20260508 10:39:04.962 | INFO | Analyzing chunk frames = [7453184,7453696) | 20260508 10:39:04.962 | INFO | MemInfo: { | 20260508 10:39:04.962 | INFO | "gpu" : " 17 GB", | 20260508 10:39:04.962 | INFO | "huge" : "", | 20260508 10:39:04.962 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:04.962 | INFO | "malloc" : " 14 GB", | 20260508 10:39:04.962 | INFO | "pinned" : " 29 GB" | 20260508 10:39:04.962 | INFO | } | 20260508 10:39:04.962 | INFO | Perf report for the last 5.14059 seconds: | 20260508 10:39:04.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:04.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.963 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:39:04.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.963 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:39:04.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:04.963 | INFO | Uploaded 36GiB to GPU in 2.36397s (15.2286GiB/s) | 20260508 10:39:04.963 | INFO | Downloaded 34GiB to GPU in 2.43545s (13.9605GiB/s) | 20260508 10:39:10.099 | INFO | Analyzing chunk frames = [7453696,7454208) | 20260508 10:39:10.100 | INFO | MemInfo: { | 20260508 10:39:10.100 | INFO | "gpu" : " 17 GB", | 20260508 10:39:10.100 | INFO | "huge" : "", | 20260508 10:39:10.100 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:10.100 | INFO | "malloc" : " 14 GB", | 20260508 10:39:10.100 | INFO | "pinned" : " 29 GB" | 20260508 10:39:10.100 | INFO | } | 20260508 10:39:10.100 | INFO | Perf report for the last 5.13733 seconds: | 20260508 10:39:10.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:10.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.100 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:39:10.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.100 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 10:39:10.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:10.101 | INFO | Uploaded 36GiB to GPU in 2.40157s (14.9902GiB/s) | 20260508 10:39:10.101 | INFO | Downloaded 34GiB to GPU in 2.43829s (13.9442GiB/s) | 20260508 10:39:15.235 | INFO | Analyzing chunk frames = [7454208,7454720) | 20260508 10:39:15.235 | INFO | MemInfo: { | 20260508 10:39:15.235 | INFO | "gpu" : " 17 GB", | 20260508 10:39:15.235 | INFO | "huge" : "", | 20260508 10:39:15.235 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:15.235 | INFO | "malloc" : " 14 GB", | 20260508 10:39:15.236 | INFO | "pinned" : " 29 GB" | 20260508 10:39:15.236 | INFO | } | 20260508 10:39:15.236 | INFO | Perf report for the last 5.13576 seconds: | 20260508 10:39:15.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:15.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:39:15.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:39:15.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:15.236 | INFO | Uploaded 36GiB to GPU in 2.41526s (14.9052GiB/s) | 20260508 10:39:15.236 | INFO | Downloaded 34GiB to GPU in 2.44416s (13.9107GiB/s) | 20260508 10:39:15.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:39:15.236 | INFO | Total mem usage: 16.7 GiB | 20260508 10:39:15.236 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:39:15.236 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:39:15.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:39:15.236 | INFO | High Water Breakdown: | 20260508 10:39:15.236 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:39:15.236 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:39:15.236 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:39:15.236 | INFO | ------------------------------------------ | 20260508 10:39:15.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:39:15.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:39:15.236 | INFO | Total mem usage: 26.5 GiB | 20260508 10:39:15.236 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:39:15.236 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:39:15.236 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:39:15.236 | INFO | High Water Breakdown: | 20260508 10:39:15.236 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:39:15.236 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:39:15.236 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:39:15.236 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:39:15.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:39:15.236 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:39:15.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:39:15.236 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:39:15.237 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:39:15.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:39:15.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:39:15.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:39:15.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:39:15.237 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:39:15.237 | INFO | ------------------------------------------ | 20260508 10:39:15.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:39:15.237 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:39:15.237 | INFO | Total mem usage: 27.8 GiB | 20260508 10:39:15.237 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:39:15.237 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:39:15.237 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:39:15.237 | INFO | High Water Breakdown: | 20260508 10:39:15.237 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:39:15.237 | INFO | ------------------------------------------ | 20260508 10:39:15.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:39:15.237 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:39:15.237 | INFO | Total mem usage: 15.7 GiB | 20260508 10:39:15.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:39:15.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:39:15.237 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:39:15.237 | INFO | High Water Breakdown: | 20260508 10:39:15.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:39:15.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:39:15.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:39:15.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:39:15.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:39:15.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:39:15.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:39:15.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:39:15.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:39:15.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:39:15.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:39:15.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:39:15.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:39:15.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:39:15.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:39:15.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:39:15.237 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:39:15.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:39:15.237 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:39:15.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:39:15.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:39:15.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:39:15.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:39:15.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:39:15.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:39:15.237 | INFO | ------------------------------------------ | 20260508 10:39:15.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:39:15.237 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:39:15.237 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:39:15.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:39:15.237 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 5.70e-01 | 20260508 10:39:15.237 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.12e-02 / 3.17e-03 | 20260508 10:39:15.237 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.49e+00, 5.89e-01 / 2.08e-01 | 20260508 10:39:15.237 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 8.66e-02 | 20260508 10:39:15.237 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.54e+01, 3.78e+00 / 8.79e-01 | 20260508 10:39:15.237 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.72e+00 / 1.03e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.35e+00 / 2.60e+00, 3.99e-01 / 2.16e-01 | 20260508 10:39:15.237 | INFO | PerfReport: 3.03%, Metrics -- 9.90e-01 / 1.15e+00, 2.46e-01 / 1.25e-01 | 20260508 10:39:15.237 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.39e-01, 4.79e-02 / 2.32e-02 | 20260508 10:39:15.237 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:39:15.237 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:39:15.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:39:15.237 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 24.23%, Upload -- 2.55e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 24.61%, Download -- 2.59e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 31.04%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 2.19%, LockUpload -- 2.31e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 2.61%, LockDownload -- 2.74e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 15.30%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:39:15.237 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 10:39:15.333 | INFO | Starting a round of Baz IO | 20260508 10:39:18.752 | INFO | Populating RTMetrics report | 20260508 10:39:18.752 | INFO | RTMetrics report populated | 20260508 10:39:18.752 | INFO | Opening tmp RTMetrics json file | 20260508 10:39:18.753 | INFO | Serializing RTMetrics to json | 20260508 10:39:18.754 | INFO | Serialization done, swapping tmp file now... | 20260508 10:39:18.755 | INFO | RTMetrics done | 20260508 10:39:18.816 | INFO | Finished a round of Baz IO: Wrote 1.455 GiB in 3.483 seconds | 20260508 10:39:18.816 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:39:18.816 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216134% / 0% / 6.35783e-05% | 20260508 10:39:18.816 | INFO | Baz IO overall: | 20260508 10:39:18.816 | INFO | BytesWritten : 3231041349957 | 20260508 10:39:18.816 | INFO | NumEvents : 1552816839916 | 20260508 10:39:20.373 | INFO | Analyzing chunk frames = [7454720,7455232) | 20260508 10:39:20.373 | INFO | MemInfo: { | 20260508 10:39:20.373 | INFO | "gpu" : " 17 GB", | 20260508 10:39:20.373 | INFO | "huge" : "", | 20260508 10:39:20.373 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:20.373 | INFO | "malloc" : " 14 GB", | 20260508 10:39:20.373 | INFO | "pinned" : " 29 GB" | 20260508 10:39:20.373 | INFO | } | 20260508 10:39:20.373 | INFO | Perf report for the last 5.13772 seconds: | 20260508 10:39:20.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:20.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:20.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:20.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:20.373 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 10:39:20.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:20.373 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:39:20.373 | INFO | PRE_HQ : 65.7%, 1.6, 54.3ms, 0 | 20260508 10:39:20.373 | INFO | BAZWRITER : 67.1%, 2.5, 86.5ms, 0 | 20260508 10:39:20.373 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:39:20.373 | INFO | Uploaded 36GiB to GPU in 2.51059s (14.3392GiB/s) | 20260508 10:39:20.374 | INFO | Downloaded 38GiB to GPU in 2.6925s (14.1133GiB/s) | 20260508 10:39:25.518 | INFO | Analyzing chunk frames = [7455232,7455744) | 20260508 10:39:25.518 | INFO | MemInfo: { | 20260508 10:39:25.518 | INFO | "gpu" : " 17 GB", | 20260508 10:39:25.518 | INFO | "huge" : "", | 20260508 10:39:25.518 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:25.518 | INFO | "malloc" : " 14 GB", | 20260508 10:39:25.518 | INFO | "pinned" : " 29 GB" | 20260508 10:39:25.518 | INFO | } | 20260508 10:39:25.518 | INFO | Perf report for the last 5.14475 seconds: | 20260508 10:39:25.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:25.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | ANALYSIS : 66.7%, 3.0, 104.1ms, 0 | 20260508 10:39:25.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:39:25.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:25.518 | INFO | Uploaded 36GiB to GPU in 2.5286s (14.2371GiB/s) | 20260508 10:39:25.518 | INFO | Downloaded 34GiB to GPU in 2.46619s (13.7864GiB/s) | 20260508 10:39:30.662 | INFO | Analyzing chunk frames = [7455744,7456256) | 20260508 10:39:30.662 | INFO | MemInfo: { | 20260508 10:39:30.662 | INFO | "gpu" : " 17 GB", | 20260508 10:39:30.662 | INFO | "huge" : "", | 20260508 10:39:30.662 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:30.662 | INFO | "malloc" : " 14 GB", | 20260508 10:39:30.662 | INFO | "pinned" : " 29 GB" | 20260508 10:39:30.662 | INFO | } | 20260508 10:39:30.662 | INFO | Perf report for the last 5.1443 seconds: | 20260508 10:39:30.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:30.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.662 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 10:39:30.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.663 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:39:30.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:30.663 | INFO | Uploaded 36GiB to GPU in 2.46995s (14.5752GiB/s) | 20260508 10:39:30.663 | INFO | Downloaded 34GiB to GPU in 2.45021s (13.8763GiB/s) | 20260508 10:39:35.799 | INFO | Analyzing chunk frames = [7456256,7456768) | 20260508 10:39:35.799 | INFO | MemInfo: { | 20260508 10:39:35.799 | INFO | "gpu" : " 17 GB", | 20260508 10:39:35.799 | INFO | "huge" : "", | 20260508 10:39:35.799 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:35.799 | INFO | "malloc" : " 14 GB", | 20260508 10:39:35.799 | INFO | "pinned" : " 29 GB" | 20260508 10:39:35.799 | INFO | } | 20260508 10:39:35.799 | INFO | Perf report for the last 5.1365 seconds: | 20260508 10:39:35.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:35.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 10:39:35.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:39:35.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:35.799 | INFO | Uploaded 36GiB to GPU in 2.53744s (14.1875GiB/s) | 20260508 10:39:35.799 | INFO | Downloaded 34GiB to GPU in 2.45938s (13.8246GiB/s) | 20260508 10:39:40.931 | INFO | Analyzing chunk frames = [7456768,7457280) | 20260508 10:39:40.931 | INFO | MemInfo: { | 20260508 10:39:40.931 | INFO | "gpu" : " 17 GB", | 20260508 10:39:40.931 | INFO | "huge" : "", | 20260508 10:39:40.931 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:40.931 | INFO | "malloc" : " 14 GB", | 20260508 10:39:40.931 | INFO | "pinned" : " 29 GB" | 20260508 10:39:40.931 | INFO | } | 20260508 10:39:40.931 | INFO | Perf report for the last 5.13209 seconds: | 20260508 10:39:40.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:40.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:39:40.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 10:39:40.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:40.931 | INFO | Uploaded 36GiB to GPU in 2.39026s (15.0611GiB/s) | 20260508 10:39:40.931 | INFO | Downloaded 34GiB to GPU in 2.52468s (13.4671GiB/s) | 20260508 10:39:46.071 | INFO | Analyzing chunk frames = [7457280,7457792) | 20260508 10:39:46.071 | INFO | MemInfo: { | 20260508 10:39:46.071 | INFO | "gpu" : " 17 GB", | 20260508 10:39:46.071 | INFO | "huge" : "", | 20260508 10:39:46.071 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:46.072 | INFO | "malloc" : " 14 GB", | 20260508 10:39:46.072 | INFO | "pinned" : " 29 GB" | 20260508 10:39:46.072 | INFO | } | 20260508 10:39:46.072 | INFO | Perf report for the last 5.14066 seconds: | 20260508 10:39:46.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:46.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:39:46.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:39:46.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:46.072 | INFO | Uploaded 36GiB to GPU in 2.35681s (15.2749GiB/s) | 20260508 10:39:46.072 | INFO | Downloaded 34GiB to GPU in 2.43272s (13.9761GiB/s) | 20260508 10:39:51.213 | INFO | Analyzing chunk frames = [7457792,7458304) | 20260508 10:39:51.213 | INFO | MemInfo: { | 20260508 10:39:51.213 | INFO | "gpu" : " 17 GB", | 20260508 10:39:51.213 | INFO | "huge" : "", | 20260508 10:39:51.213 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:51.213 | INFO | "malloc" : " 14 GB", | 20260508 10:39:51.213 | INFO | "pinned" : " 29 GB" | 20260508 10:39:51.213 | INFO | } | 20260508 10:39:51.214 | INFO | Perf report for the last 5.14186 seconds: | 20260508 10:39:51.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:51.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 10:39:51.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:39:51.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:51.214 | INFO | Uploaded 36GiB to GPU in 2.42113s (14.8691GiB/s) | 20260508 10:39:51.214 | INFO | Downloaded 34GiB to GPU in 2.45199s (13.8663GiB/s) | 20260508 10:39:56.356 | INFO | Analyzing chunk frames = [7458304,7458816) | 20260508 10:39:56.356 | INFO | MemInfo: { | 20260508 10:39:56.356 | INFO | "gpu" : " 17 GB", | 20260508 10:39:56.356 | INFO | "huge" : "", | 20260508 10:39:56.356 | INFO | "huge_shared" : " 29 GB", | 20260508 10:39:56.356 | INFO | "malloc" : " 14 GB", | 20260508 10:39:56.356 | INFO | "pinned" : " 29 GB" | 20260508 10:39:56.356 | INFO | } | 20260508 10:39:56.356 | INFO | Perf report for the last 5.1426 seconds: | 20260508 10:39:56.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:39:56.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:39:56.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:39:56.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:39:56.356 | INFO | Uploaded 36GiB to GPU in 2.4266s (14.8356GiB/s) | 20260508 10:39:56.357 | INFO | Downloaded 34GiB to GPU in 2.4503s (13.8759GiB/s) | 20260508 10:39:56.453 | INFO | Starting a round of Baz IO | 20260508 10:39:59.868 | INFO | Populating RTMetrics report | 20260508 10:39:59.869 | INFO | RTMetrics report populated | 20260508 10:39:59.869 | INFO | Opening tmp RTMetrics json file | 20260508 10:39:59.869 | INFO | Serializing RTMetrics to json | 20260508 10:39:59.870 | INFO | Serialization done, swapping tmp file now... | 20260508 10:39:59.871 | INFO | RTMetrics done | 20260508 10:39:59.936 | INFO | Finished a round of Baz IO: Wrote 1.455 GiB in 3.483 seconds | 20260508 10:39:59.936 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:39:59.936 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.215542% / 7.94729e-06% / 6.75519e-05% | 20260508 10:39:59.936 | INFO | Baz IO overall: | 20260508 10:39:59.936 | INFO | BytesWritten : 3232603458394 | 20260508 10:39:59.936 | INFO | NumEvents : 1553552928677 | 20260508 10:40:01.494 | INFO | Analyzing chunk frames = [7458816,7459328) | 20260508 10:40:01.494 | INFO | MemInfo: { | 20260508 10:40:01.494 | INFO | "gpu" : " 17 GB", | 20260508 10:40:01.494 | INFO | "huge" : "", | 20260508 10:40:01.494 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:01.494 | INFO | "malloc" : " 14 GB", | 20260508 10:40:01.494 | INFO | "pinned" : " 29 GB" | 20260508 10:40:01.494 | INFO | } | 20260508 10:40:01.494 | INFO | Perf report for the last 5.13826 seconds: | 20260508 10:40:01.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:01.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:01.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:01.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:01.494 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260508 10:40:01.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:01.495 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:40:01.495 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 10:40:01.495 | INFO | BAZWRITER : 67.0%, 2.5, 87.0ms, 0 | 20260508 10:40:01.495 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 10:40:01.495 | INFO | Uploaded 36GiB to GPU in 2.57219s (13.9959GiB/s) | 20260508 10:40:01.495 | INFO | Downloaded 38GiB to GPU in 2.70844s (14.0302GiB/s) | 20260508 10:40:06.632 | INFO | Analyzing chunk frames = [7459328,7459840) | 20260508 10:40:06.632 | INFO | MemInfo: { | 20260508 10:40:06.632 | INFO | "gpu" : " 17 GB", | 20260508 10:40:06.632 | INFO | "huge" : "", | 20260508 10:40:06.632 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:06.632 | INFO | "malloc" : " 14 GB", | 20260508 10:40:06.632 | INFO | "pinned" : " 29 GB" | 20260508 10:40:06.632 | INFO | } | 20260508 10:40:06.632 | INFO | Perf report for the last 5.13794 seconds: | 20260508 10:40:06.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:06.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.632 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:40:06.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.632 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:40:06.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:06.633 | INFO | Uploaded 36GiB to GPU in 2.45813s (14.6453GiB/s) | 20260508 10:40:06.633 | INFO | Downloaded 34GiB to GPU in 2.47007s (13.7648GiB/s) | 20260508 10:40:11.772 | INFO | Analyzing chunk frames = [7459840,7460352) | 20260508 10:40:11.772 | INFO | MemInfo: { | 20260508 10:40:11.772 | INFO | "gpu" : " 17 GB", | 20260508 10:40:11.772 | INFO | "huge" : "", | 20260508 10:40:11.772 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:11.772 | INFO | "malloc" : " 14 GB", | 20260508 10:40:11.772 | INFO | "pinned" : " 29 GB" | 20260508 10:40:11.772 | INFO | } | 20260508 10:40:11.772 | INFO | Perf report for the last 5.13989 seconds: | 20260508 10:40:11.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:11.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.772 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 10:40:11.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.772 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:40:11.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:11.773 | INFO | Uploaded 36GiB to GPU in 2.46791s (14.5872GiB/s) | 20260508 10:40:11.773 | INFO | Downloaded 34GiB to GPU in 2.44879s (13.8844GiB/s) | 20260508 10:40:16.910 | INFO | Analyzing chunk frames = [7460352,7460864) | 20260508 10:40:16.910 | INFO | MemInfo: { | 20260508 10:40:16.910 | INFO | "gpu" : " 17 GB", | 20260508 10:40:16.910 | INFO | "huge" : "", | 20260508 10:40:16.910 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:16.910 | INFO | "malloc" : " 14 GB", | 20260508 10:40:16.910 | INFO | "pinned" : " 29 GB" | 20260508 10:40:16.910 | INFO | } | 20260508 10:40:16.910 | INFO | Perf report for the last 5.13785 seconds: | 20260508 10:40:16.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:16.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.910 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:40:16.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.910 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 10:40:16.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:16.911 | INFO | Uploaded 36GiB to GPU in 2.46152s (14.6251GiB/s) | 20260508 10:40:16.911 | INFO | Downloaded 34GiB to GPU in 2.45253s (13.8632GiB/s) | 20260508 10:40:22.048 | INFO | Analyzing chunk frames = [7460864,7461376) | 20260508 10:40:22.048 | INFO | MemInfo: { | 20260508 10:40:22.048 | INFO | "gpu" : " 17 GB", | 20260508 10:40:22.048 | INFO | "huge" : "", | 20260508 10:40:22.048 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:22.048 | INFO | "malloc" : " 14 GB", | 20260508 10:40:22.048 | INFO | "pinned" : " 29 GB" | 20260508 10:40:22.048 | INFO | } | 20260508 10:40:22.048 | INFO | Perf report for the last 5.13815 seconds: | 20260508 10:40:22.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:22.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.048 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 10:40:22.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.048 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:40:22.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:22.049 | INFO | Uploaded 36GiB to GPU in 2.3938s (15.0389GiB/s) | 20260508 10:40:22.049 | INFO | Downloaded 34GiB to GPU in 2.53799s (13.3964GiB/s) | 20260508 10:40:27.186 | INFO | Analyzing chunk frames = [7461376,7461888) | 20260508 10:40:27.186 | INFO | MemInfo: { | 20260508 10:40:27.186 | INFO | "gpu" : " 17 GB", | 20260508 10:40:27.186 | INFO | "huge" : "", | 20260508 10:40:27.186 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:27.186 | INFO | "malloc" : " 14 GB", | 20260508 10:40:27.186 | INFO | "pinned" : " 29 GB" | 20260508 10:40:27.186 | INFO | } | 20260508 10:40:27.186 | INFO | Perf report for the last 5.13774 seconds: | 20260508 10:40:27.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:27.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.186 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:40:27.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.186 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:40:27.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:27.187 | INFO | Uploaded 36GiB to GPU in 2.35675s (15.2753GiB/s) | 20260508 10:40:27.187 | INFO | Downloaded 34GiB to GPU in 2.43684s (13.9525GiB/s) | 20260508 10:40:32.332 | INFO | Analyzing chunk frames = [7461888,7462400) | 20260508 10:40:32.332 | INFO | MemInfo: { | 20260508 10:40:32.332 | INFO | "gpu" : " 17 GB", | 20260508 10:40:32.332 | INFO | "huge" : "", | 20260508 10:40:32.332 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:32.332 | INFO | "malloc" : " 14 GB", | 20260508 10:40:32.332 | INFO | "pinned" : " 29 GB" | 20260508 10:40:32.332 | INFO | } | 20260508 10:40:32.332 | INFO | Perf report for the last 5.14573 seconds: | 20260508 10:40:32.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:32.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:40:32.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:40:32.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:32.332 | INFO | Uploaded 36GiB to GPU in 2.4001s (14.9994GiB/s) | 20260508 10:40:32.332 | INFO | Downloaded 34GiB to GPU in 2.44173s (13.9246GiB/s) | 20260508 10:40:37.471 | INFO | Analyzing chunk frames = [7462400,7462912) | 20260508 10:40:37.471 | INFO | MemInfo: { | 20260508 10:40:37.471 | INFO | "gpu" : " 17 GB", | 20260508 10:40:37.471 | INFO | "huge" : "", | 20260508 10:40:37.471 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:37.471 | INFO | "malloc" : " 14 GB", | 20260508 10:40:37.471 | INFO | "pinned" : " 29 GB" | 20260508 10:40:37.471 | INFO | } | 20260508 10:40:37.471 | INFO | Perf report for the last 5.13917 seconds: | 20260508 10:40:37.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:37.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 10:40:37.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:40:37.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:37.471 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:40:37.471 | INFO | Total mem usage: 16.7 GiB | 20260508 10:40:37.471 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:40:37.471 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:40:37.471 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:40:37.471 | INFO | High Water Breakdown: | 20260508 10:40:37.471 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:40:37.471 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:40:37.471 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:40:37.471 | INFO | ------------------------------------------ | 20260508 10:40:37.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:40:37.471 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:40:37.471 | INFO | Total mem usage: 26.5 GiB | 20260508 10:40:37.471 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:40:37.471 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:40:37.471 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:40:37.471 | INFO | High Water Breakdown: | 20260508 10:40:37.471 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:40:37.471 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:40:37.471 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:40:37.471 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:40:37.471 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:40:37.471 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:40:37.471 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:40:37.471 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:40:37.471 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:40:37.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:40:37.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:40:37.471 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:40:37.471 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:40:37.471 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:40:37.471 | INFO | ------------------------------------------ | 20260508 10:40:37.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:40:37.471 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:40:37.471 | INFO | Total mem usage: 27.8 GiB | 20260508 10:40:37.471 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:40:37.471 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:40:37.471 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:40:37.471 | INFO | High Water Breakdown: | 20260508 10:40:37.471 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:40:37.471 | INFO | ------------------------------------------ | 20260508 10:40:37.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:40:37.471 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:40:37.471 | INFO | Total mem usage: 15.7 GiB | 20260508 10:40:37.471 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:40:37.471 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:40:37.471 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:40:37.471 | INFO | High Water Breakdown: | 20260508 10:40:37.471 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:40:37.471 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:40:37.471 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:40:37.471 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:40:37.471 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:40:37.471 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:40:37.471 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:40:37.471 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:40:37.471 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:40:37.471 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:40:37.471 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:40:37.471 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:40:37.471 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:40:37.471 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:40:37.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:40:37.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:40:37.471 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:40:37.471 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:40:37.471 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:40:37.471 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:40:37.471 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:40:37.471 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:40:37.471 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:40:37.471 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:40:37.471 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:40:37.471 | INFO | ------------------------------------------ | 20260508 10:40:37.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:40:37.472 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:40:37.472 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:40:37.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:40:37.472 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.42e+01 / 5.67e-01 | 20260508 10:40:37.472 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.11e-02 / 5.70e-03 | 20260508 10:40:37.472 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.43e+00, 5.87e-01 / 2.15e-01 | 20260508 10:40:37.472 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.13e-01 | 20260508 10:40:37.472 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.49e+01, 3.61e+00 / 9.78e-01 | 20260508 10:40:37.472 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 1.24e+00 | 20260508 10:40:37.472 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.06e-01 / 2.40e-01 | 20260508 10:40:37.472 | INFO | PerfReport: 3.02%, Metrics -- 9.86e-01 / 1.14e+00, 2.50e-01 / 1.21e-01 | 20260508 10:40:37.472 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.29e-01, 4.81e-02 / 2.77e-02 | 20260508 10:40:37.472 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:40:37.472 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:40:37.472 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:40:37.472 | INFO | Uploaded 36GiB to GPU in 2.46916s (14.5799GiB/s) | 20260508 10:40:37.472 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:40:37.473 | INFO | PerfReport: 24.24%, Upload -- 2.55e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 10:40:37.473 | INFO | Downloaded 34GiB to GPU in 2.4613s (13.8138GiB/s) | 20260508 10:40:37.473 | INFO | PerfReport: 24.66%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:40:37.473 | INFO | PerfReport: 31.01%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:40:37.473 | INFO | PerfReport: 2.13%, LockUpload -- 2.25e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260508 10:40:37.473 | INFO | PerfReport: 2.61%, LockDownload -- 2.75e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 10:40:37.473 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:40:37.473 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 1.03e-02 / 0.00e+00 | 20260508 10:40:37.569 | INFO | Starting a round of Baz IO | 20260508 10:40:40.980 | INFO | Populating RTMetrics report | 20260508 10:40:40.981 | INFO | RTMetrics report populated | 20260508 10:40:40.981 | INFO | Opening tmp RTMetrics json file | 20260508 10:40:40.988 | INFO | Serializing RTMetrics to json | 20260508 10:40:40.989 | INFO | Serialization done, swapping tmp file now... | 20260508 10:40:40.991 | INFO | RTMetrics done | 20260508 10:40:41.050 | INFO | Finished a round of Baz IO: Wrote 1.456 GiB in 3.48 seconds | 20260508 10:40:41.050 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:40:41.050 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21497% / 0% / 5.5631e-05% | 20260508 10:40:41.050 | INFO | Baz IO overall: | 20260508 10:40:41.050 | INFO | BytesWritten : 3234166405643 | 20260508 10:40:41.050 | INFO | NumEvents : 1554289820495 | 20260508 10:40:42.606 | INFO | Analyzing chunk frames = [7462912,7463424) | 20260508 10:40:42.606 | INFO | MemInfo: { | 20260508 10:40:42.606 | INFO | "gpu" : " 17 GB", | 20260508 10:40:42.606 | INFO | "huge" : "", | 20260508 10:40:42.606 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:42.606 | INFO | "malloc" : " 14 GB", | 20260508 10:40:42.606 | INFO | "pinned" : " 29 GB" | 20260508 10:40:42.606 | INFO | } | 20260508 10:40:42.607 | INFO | Perf report for the last 5.13557 seconds: | 20260508 10:40:42.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:42.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:42.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:42.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:42.607 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:40:42.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:42.607 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:40:42.607 | INFO | PRE_HQ : 65.3%, 1.6, 54.3ms, 0 | 20260508 10:40:42.607 | INFO | BAZWRITER : 67.0%, 2.5, 86.5ms, 0 | 20260508 10:40:42.607 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:40:42.607 | INFO | Uploaded 36GiB to GPU in 2.59929s (13.8499GiB/s) | 20260508 10:40:42.607 | INFO | Downloaded 38GiB to GPU in 2.71402s (14.0014GiB/s) | 20260508 10:40:47.740 | INFO | Analyzing chunk frames = [7463424,7463936) | 20260508 10:40:47.740 | INFO | MemInfo: { | 20260508 10:40:47.740 | INFO | "gpu" : " 17 GB", | 20260508 10:40:47.740 | INFO | "huge" : "", | 20260508 10:40:47.740 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:47.740 | INFO | "malloc" : " 14 GB", | 20260508 10:40:47.740 | INFO | "pinned" : " 29 GB" | 20260508 10:40:47.740 | INFO | } | 20260508 10:40:47.740 | INFO | Perf report for the last 5.13358 seconds: | 20260508 10:40:47.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:47.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.740 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 10:40:47.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.740 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:40:47.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:47.741 | INFO | Uploaded 36GiB to GPU in 2.4637s (14.6121GiB/s) | 20260508 10:40:47.741 | INFO | Downloaded 34GiB to GPU in 2.47172s (13.7556GiB/s) | 20260508 10:40:52.882 | INFO | Analyzing chunk frames = [7463936,7464448) | 20260508 10:40:52.882 | INFO | MemInfo: { | 20260508 10:40:52.882 | INFO | "gpu" : " 17 GB", | 20260508 10:40:52.882 | INFO | "huge" : "", | 20260508 10:40:52.882 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:52.882 | INFO | "malloc" : " 14 GB", | 20260508 10:40:52.882 | INFO | "pinned" : " 29 GB" | 20260508 10:40:52.882 | INFO | } | 20260508 10:40:52.882 | INFO | Perf report for the last 5.14193 seconds: | 20260508 10:40:52.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:52.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.882 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 10:40:52.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.882 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 10:40:52.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:52.883 | INFO | Uploaded 36GiB to GPU in 2.46498s (14.6046GiB/s) | 20260508 10:40:52.883 | INFO | Downloaded 34GiB to GPU in 2.4566s (13.8403GiB/s) | 20260508 10:40:58.023 | INFO | Analyzing chunk frames = [7464448,7464960) | 20260508 10:40:58.023 | INFO | MemInfo: { | 20260508 10:40:58.023 | INFO | "gpu" : " 17 GB", | 20260508 10:40:58.024 | INFO | "huge" : "", | 20260508 10:40:58.024 | INFO | "huge_shared" : " 29 GB", | 20260508 10:40:58.024 | INFO | "malloc" : " 14 GB", | 20260508 10:40:58.024 | INFO | "pinned" : " 29 GB" | 20260508 10:40:58.024 | INFO | } | 20260508 10:40:58.024 | INFO | Perf report for the last 5.14168 seconds: | 20260508 10:40:58.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:40:58.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:40:58.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 10:40:58.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:40:58.024 | INFO | Uploaded 36GiB to GPU in 2.45278s (14.6773GiB/s) | 20260508 10:40:58.024 | INFO | Downloaded 34GiB to GPU in 2.42101s (14.0437GiB/s) | 20260508 10:41:03.166 | INFO | Analyzing chunk frames = [7464960,7465472) | 20260508 10:41:03.166 | INFO | MemInfo: { | 20260508 10:41:03.166 | INFO | "gpu" : " 17 GB", | 20260508 10:41:03.166 | INFO | "huge" : "", | 20260508 10:41:03.166 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:03.166 | INFO | "malloc" : " 14 GB", | 20260508 10:41:03.166 | INFO | "pinned" : " 29 GB" | 20260508 10:41:03.166 | INFO | } | 20260508 10:41:03.166 | INFO | Perf report for the last 5.14249 seconds: | 20260508 10:41:03.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:03.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.166 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 10:41:03.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.166 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260508 10:41:03.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:03.167 | INFO | Uploaded 36GiB to GPU in 2.30549s (15.6149GiB/s) | 20260508 10:41:03.167 | INFO | Downloaded 34GiB to GPU in 2.51561s (13.5156GiB/s) | 20260508 10:41:08.302 | INFO | Analyzing chunk frames = [7465472,7465984) | 20260508 10:41:08.302 | INFO | MemInfo: { | 20260508 10:41:08.302 | INFO | "gpu" : " 17 GB", | 20260508 10:41:08.302 | INFO | "huge" : "", | 20260508 10:41:08.302 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:08.302 | INFO | "malloc" : " 14 GB", | 20260508 10:41:08.303 | INFO | "pinned" : " 29 GB" | 20260508 10:41:08.303 | INFO | } | 20260508 10:41:08.303 | INFO | Perf report for the last 5.13634 seconds: | 20260508 10:41:08.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:08.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:41:08.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:41:08.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:08.303 | INFO | Uploaded 36GiB to GPU in 2.3504s (15.3165GiB/s) | 20260508 10:41:08.303 | INFO | Downloaded 34GiB to GPU in 2.42812s (14.0026GiB/s) | 20260508 10:41:13.440 | INFO | Analyzing chunk frames = [7465984,7466496) | 20260508 10:41:13.440 | INFO | MemInfo: { | 20260508 10:41:13.440 | INFO | "gpu" : " 17 GB", | 20260508 10:41:13.440 | INFO | "huge" : "", | 20260508 10:41:13.440 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:13.440 | INFO | "malloc" : " 14 GB", | 20260508 10:41:13.440 | INFO | "pinned" : " 29 GB" | 20260508 10:41:13.440 | INFO | } | 20260508 10:41:13.440 | INFO | Perf report for the last 5.13737 seconds: | 20260508 10:41:13.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:13.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:41:13.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:41:13.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:13.440 | INFO | Uploaded 36GiB to GPU in 2.40912s (14.9432GiB/s) | 20260508 10:41:13.440 | INFO | Downloaded 34GiB to GPU in 2.43844s (13.9433GiB/s) | 20260508 10:41:18.587 | INFO | Analyzing chunk frames = [7466496,7467008) | 20260508 10:41:18.587 | INFO | MemInfo: { | 20260508 10:41:18.587 | INFO | "gpu" : " 17 GB", | 20260508 10:41:18.587 | INFO | "huge" : "", | 20260508 10:41:18.587 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:18.587 | INFO | "malloc" : " 14 GB", | 20260508 10:41:18.587 | INFO | "pinned" : " 29 GB" | 20260508 10:41:18.587 | INFO | } | 20260508 10:41:18.587 | INFO | Perf report for the last 5.14709 seconds: | 20260508 10:41:18.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:18.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 10:41:18.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:41:18.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:18.587 | INFO | Uploaded 36GiB to GPU in 2.41917s (14.8812GiB/s) | 20260508 10:41:18.587 | INFO | Downloaded 34GiB to GPU in 2.44348s (13.9146GiB/s) | 20260508 10:41:18.684 | INFO | Starting a round of Baz IO | 20260508 10:41:22.086 | INFO | Populating RTMetrics report | 20260508 10:41:22.086 | INFO | RTMetrics report populated | 20260508 10:41:22.086 | INFO | Opening tmp RTMetrics json file | 20260508 10:41:22.095 | INFO | Serializing RTMetrics to json | 20260508 10:41:22.097 | INFO | Serialization done, swapping tmp file now... | 20260508 10:41:22.097 | INFO | RTMetrics done | 20260508 10:41:22.157 | INFO | Finished a round of Baz IO: Wrote 1.454 GiB in 3.472 seconds | 20260508 10:41:22.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.00% / 59.82% | 20260508 10:41:22.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.214799% / 3.97364e-06% / 5.5631e-05% | 20260508 10:41:22.157 | INFO | Baz IO overall: | 20260508 10:41:22.157 | INFO | BytesWritten : 3235727494718 | 20260508 10:41:22.157 | INFO | NumEvents : 1555025267631 | 20260508 10:41:23.721 | INFO | Analyzing chunk frames = [7467008,7467520) | 20260508 10:41:23.721 | INFO | MemInfo: { | 20260508 10:41:23.721 | INFO | "gpu" : " 17 GB", | 20260508 10:41:23.721 | INFO | "huge" : "", | 20260508 10:41:23.721 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:23.721 | INFO | "malloc" : " 14 GB", | 20260508 10:41:23.721 | INFO | "pinned" : " 29 GB" | 20260508 10:41:23.721 | INFO | } | 20260508 10:41:23.721 | INFO | Perf report for the last 5.13432 seconds: | 20260508 10:41:23.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:23.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:23.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:23.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:23.722 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 10:41:23.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:23.722 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:41:23.722 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 10:41:23.722 | INFO | BAZWRITER : 66.9%, 2.5, 86.7ms, 0 | 20260508 10:41:23.722 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:41:23.722 | INFO | Uploaded 36GiB to GPU in 2.62825s (13.6973GiB/s) | 20260508 10:41:23.722 | INFO | Downloaded 38GiB to GPU in 2.70471s (14.0496GiB/s) | 20260508 10:41:28.865 | INFO | Analyzing chunk frames = [7467520,7468032) | 20260508 10:41:28.865 | INFO | MemInfo: { | 20260508 10:41:28.865 | INFO | "gpu" : " 17 GB", | 20260508 10:41:28.865 | INFO | "huge" : "", | 20260508 10:41:28.865 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:28.865 | INFO | "malloc" : " 14 GB", | 20260508 10:41:28.865 | INFO | "pinned" : " 29 GB" | 20260508 10:41:28.865 | INFO | } | 20260508 10:41:28.865 | INFO | Perf report for the last 5.14392 seconds: | 20260508 10:41:28.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:28.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.865 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:41:28.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.865 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:41:28.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:28.866 | INFO | Uploaded 36GiB to GPU in 2.44841s (14.7034GiB/s) | 20260508 10:41:28.866 | INFO | Downloaded 34GiB to GPU in 2.46421s (13.7976GiB/s) | 20260508 10:41:34.001 | INFO | Analyzing chunk frames = [7468032,7468544) | 20260508 10:41:34.001 | INFO | MemInfo: { | 20260508 10:41:34.001 | INFO | "gpu" : " 17 GB", | 20260508 10:41:34.001 | INFO | "huge" : "", | 20260508 10:41:34.001 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:34.001 | INFO | "malloc" : " 14 GB", | 20260508 10:41:34.001 | INFO | "pinned" : " 29 GB" | 20260508 10:41:34.001 | INFO | } | 20260508 10:41:34.001 | INFO | Perf report for the last 5.13569 seconds: | 20260508 10:41:34.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:34.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.001 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 10:41:34.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.001 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:41:34.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:34.002 | INFO | Uploaded 36GiB to GPU in 2.46459s (14.6069GiB/s) | 20260508 10:41:34.002 | INFO | Downloaded 34GiB to GPU in 2.44638s (13.8981GiB/s) | 20260508 10:41:39.143 | INFO | Analyzing chunk frames = [7468544,7469056) | 20260508 10:41:39.143 | INFO | MemInfo: { | 20260508 10:41:39.143 | INFO | "gpu" : " 17 GB", | 20260508 10:41:39.143 | INFO | "huge" : "", | 20260508 10:41:39.143 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:39.143 | INFO | "malloc" : " 14 GB", | 20260508 10:41:39.143 | INFO | "pinned" : " 29 GB" | 20260508 10:41:39.143 | INFO | } | 20260508 10:41:39.143 | INFO | Perf report for the last 5.1418 seconds: | 20260508 10:41:39.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:39.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.143 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:41:39.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.143 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:41:39.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:39.144 | INFO | Uploaded 36GiB to GPU in 2.46874s (14.5823GiB/s) | 20260508 10:41:39.144 | INFO | Downloaded 34GiB to GPU in 2.45485s (13.8501GiB/s) | 20260508 10:41:44.281 | INFO | Analyzing chunk frames = [7469056,7469568) | 20260508 10:41:44.281 | INFO | MemInfo: { | 20260508 10:41:44.281 | INFO | "gpu" : " 17 GB", | 20260508 10:41:44.281 | INFO | "huge" : "", | 20260508 10:41:44.283 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:44.283 | INFO | "malloc" : " 14 GB", | 20260508 10:41:44.283 | INFO | "pinned" : " 29 GB" | 20260508 10:41:44.283 | INFO | } | 20260508 10:41:44.283 | INFO | Perf report for the last 5.13998 seconds: | 20260508 10:41:44.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:44.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:41:44.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:41:44.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:44.283 | INFO | Uploaded 36GiB to GPU in 2.31292s (15.5647GiB/s) | 20260508 10:41:44.283 | INFO | Downloaded 34GiB to GPU in 2.51277s (13.5309GiB/s) | 20260508 10:41:49.421 | INFO | Analyzing chunk frames = [7469568,7470080) | 20260508 10:41:49.421 | INFO | MemInfo: { | 20260508 10:41:49.421 | INFO | "gpu" : " 17 GB", | 20260508 10:41:49.421 | INFO | "huge" : "", | 20260508 10:41:49.421 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:49.421 | INFO | "malloc" : " 14 GB", | 20260508 10:41:49.421 | INFO | "pinned" : " 29 GB" | 20260508 10:41:49.421 | INFO | } | 20260508 10:41:49.421 | INFO | Perf report for the last 5.13833 seconds: | 20260508 10:41:49.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:49.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.421 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 10:41:49.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.421 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:41:49.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:49.422 | INFO | Uploaded 36GiB to GPU in 2.3316s (15.4401GiB/s) | 20260508 10:41:49.422 | INFO | Downloaded 34GiB to GPU in 2.44139s (13.9265GiB/s) | 20260508 10:41:54.556 | INFO | Analyzing chunk frames = [7470080,7470592) | 20260508 10:41:54.556 | INFO | MemInfo: { | 20260508 10:41:54.556 | INFO | "gpu" : " 17 GB", | 20260508 10:41:54.556 | INFO | "huge" : "", | 20260508 10:41:54.556 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:54.556 | INFO | "malloc" : " 14 GB", | 20260508 10:41:54.556 | INFO | "pinned" : " 29 GB" | 20260508 10:41:54.556 | INFO | } | 20260508 10:41:54.557 | INFO | Perf report for the last 5.13525 seconds: | 20260508 10:41:54.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:54.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:41:54.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:41:54.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:54.557 | INFO | Uploaded 36GiB to GPU in 2.41564s (14.9029GiB/s) | 20260508 10:41:54.557 | INFO | Downloaded 34GiB to GPU in 2.46244s (13.8075GiB/s) | 20260508 10:41:59.692 | INFO | Analyzing chunk frames = [7470592,7471104) | 20260508 10:41:59.692 | INFO | MemInfo: { | 20260508 10:41:59.692 | INFO | "gpu" : " 17 GB", | 20260508 10:41:59.692 | INFO | "huge" : "", | 20260508 10:41:59.692 | INFO | "huge_shared" : " 29 GB", | 20260508 10:41:59.692 | INFO | "malloc" : " 14 GB", | 20260508 10:41:59.692 | INFO | "pinned" : " 29 GB" | 20260508 10:41:59.692 | INFO | } | 20260508 10:41:59.692 | INFO | Perf report for the last 5.13539 seconds: | 20260508 10:41:59.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:41:59.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 10:41:59.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:41:59.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:41:59.692 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:41:59.692 | INFO | Total mem usage: 16.7 GiB | 20260508 10:41:59.692 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:41:59.692 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:41:59.692 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:41:59.692 | INFO | High Water Breakdown: | 20260508 10:41:59.692 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:41:59.692 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:41:59.692 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:41:59.692 | INFO | ------------------------------------------ | 20260508 10:41:59.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:41:59.692 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:41:59.692 | INFO | Total mem usage: 26.5 GiB | 20260508 10:41:59.692 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:41:59.692 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:41:59.692 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:41:59.692 | INFO | High Water Breakdown: | 20260508 10:41:59.692 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:41:59.692 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:41:59.692 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:41:59.692 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:41:59.692 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:41:59.692 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:41:59.692 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:41:59.692 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:41:59.692 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:41:59.692 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:41:59.692 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:41:59.692 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:41:59.692 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:41:59.692 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:41:59.692 | INFO | ------------------------------------------ | 20260508 10:41:59.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:41:59.692 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:41:59.692 | INFO | Total mem usage: 27.8 GiB | 20260508 10:41:59.692 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:41:59.692 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:41:59.692 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:41:59.692 | INFO | High Water Breakdown: | 20260508 10:41:59.692 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:41:59.692 | INFO | ------------------------------------------ | 20260508 10:41:59.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:41:59.692 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:41:59.692 | INFO | Total mem usage: 15.7 GiB | 20260508 10:41:59.692 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:41:59.692 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:41:59.692 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:41:59.692 | INFO | High Water Breakdown: | 20260508 10:41:59.692 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:41:59.692 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:41:59.692 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:41:59.692 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:41:59.692 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:41:59.692 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:41:59.692 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:41:59.692 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:41:59.692 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:41:59.692 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:41:59.692 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:41:59.692 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:41:59.692 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:41:59.692 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:41:59.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:41:59.692 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:41:59.692 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:41:59.692 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:41:59.693 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:41:59.693 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:41:59.693 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:41:59.693 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:41:59.693 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:41:59.693 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:41:59.693 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:41:59.693 | INFO | ------------------------------------------ | 20260508 10:41:59.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:41:59.693 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:41:59.693 | INFO | Uploaded 36GiB to GPU in 2.37711s (15.1444GiB/s) | 20260508 10:41:59.693 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:41:59.693 | INFO | Downloaded 34GiB to GPU in 2.44286s (13.9181GiB/s) | 20260508 10:41:59.693 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:41:59.693 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 7.41e-01 | 20260508 10:41:59.693 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.04e-01, 1.18e-02 / 4.42e-03 | 20260508 10:41:59.693 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.49e+00, 6.02e-01 / 1.77e-01 | 20260508 10:41:59.693 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.47e+00, 2.12e-01 / 1.35e-01 | 20260508 10:41:59.693 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.56e+01, 3.73e+00 / 1.02e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.79e+00 / 1.26e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.58e+00, 4.08e-01 / 2.55e-01 | 20260508 10:41:59.693 | INFO | PerfReport: 3.02%, Metrics -- 9.89e-01 / 1.11e+00, 2.44e-01 / 1.04e-01 | 20260508 10:41:59.693 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.39e-01, 4.83e-02 / 3.02e-02 | 20260508 10:41:59.693 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:41:59.693 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:41:59.693 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:41:59.693 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 24.10%, Upload -- 2.53e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 24.67%, Download -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 2.10%, LockUpload -- 2.21e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 2.62%, LockDownload -- 2.75e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 15.37%, LockCompute -- 1.62e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:41:59.693 | INFO | PerfReport: 0.02%, Misc -- 1.64e-02 / 0.00e+00, 4.87e-02 / 0.00e+00 | 20260508 10:41:59.789 | INFO | Starting a round of Baz IO | 20260508 10:42:03.210 | INFO | Populating RTMetrics report | 20260508 10:42:03.211 | INFO | RTMetrics report populated | 20260508 10:42:03.211 | INFO | Opening tmp RTMetrics json file | 20260508 10:42:03.220 | INFO | Serializing RTMetrics to json | 20260508 10:42:03.221 | INFO | Serialization done, swapping tmp file now... | 20260508 10:42:03.222 | INFO | RTMetrics done | 20260508 10:42:03.277 | INFO | Finished a round of Baz IO: Wrote 1.454 GiB in 3.488 seconds | 20260508 10:42:03.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.82% | 20260508 10:42:03.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.214573% / 3.97364e-06% / 4.76837e-05% | 20260508 10:42:03.277 | INFO | Baz IO overall: | 20260508 10:42:03.277 | INFO | BytesWritten : 3237288736195 | 20260508 10:42:03.277 | INFO | NumEvents : 1555760796742 | 20260508 10:42:04.840 | INFO | Analyzing chunk frames = [7471104,7471616) | 20260508 10:42:04.840 | INFO | MemInfo: { | 20260508 10:42:04.840 | INFO | "gpu" : " 17 GB", | 20260508 10:42:04.840 | INFO | "huge" : "", | 20260508 10:42:04.840 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:04.840 | INFO | "malloc" : " 14 GB", | 20260508 10:42:04.840 | INFO | "pinned" : " 29 GB" | 20260508 10:42:04.840 | INFO | } | 20260508 10:42:04.841 | INFO | Perf report for the last 5.14872 seconds: | 20260508 10:42:04.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:04.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:04.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:04.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:04.841 | INFO | ANALYSIS : 67.5%, 3.0, 105.7ms, 0 | 20260508 10:42:04.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:04.841 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:42:04.841 | INFO | PRE_HQ : 65.2%, 1.6, 54.1ms, 0 | 20260508 10:42:04.841 | INFO | BAZWRITER : 67.0%, 2.5, 86.5ms, 0 | 20260508 10:42:04.841 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:42:04.841 | INFO | Uploaded 36GiB to GPU in 2.56928s (14.0117GiB/s) | 20260508 10:42:04.841 | INFO | Downloaded 38GiB to GPU in 2.71076s (14.0182GiB/s) | 20260508 10:42:09.984 | INFO | Analyzing chunk frames = [7471616,7472128) | 20260508 10:42:09.984 | INFO | MemInfo: { | 20260508 10:42:09.984 | INFO | "gpu" : " 17 GB", | 20260508 10:42:09.984 | INFO | "huge" : "", | 20260508 10:42:09.984 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:09.984 | INFO | "malloc" : " 14 GB", | 20260508 10:42:09.984 | INFO | "pinned" : " 29 GB" | 20260508 10:42:09.984 | INFO | } | 20260508 10:42:09.984 | INFO | Perf report for the last 5.14307 seconds: | 20260508 10:42:09.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:09.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 10:42:09.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:42:09.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:09.984 | INFO | Uploaded 36GiB to GPU in 2.40329s (14.9795GiB/s) | 20260508 10:42:09.984 | INFO | Downloaded 34GiB to GPU in 2.47662s (13.7284GiB/s) | 20260508 10:42:15.116 | INFO | Analyzing chunk frames = [7472128,7472640) | 20260508 10:42:15.116 | INFO | MemInfo: { | 20260508 10:42:15.116 | INFO | "gpu" : " 17 GB", | 20260508 10:42:15.116 | INFO | "huge" : "", | 20260508 10:42:15.116 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:15.117 | INFO | "malloc" : " 14 GB", | 20260508 10:42:15.117 | INFO | "pinned" : " 29 GB" | 20260508 10:42:15.117 | INFO | } | 20260508 10:42:15.117 | INFO | Perf report for the last 5.13285 seconds: | 20260508 10:42:15.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:15.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:42:15.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:42:15.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:15.117 | INFO | Uploaded 36GiB to GPU in 2.45204s (14.6816GiB/s) | 20260508 10:42:15.117 | INFO | Downloaded 34GiB to GPU in 2.45354s (13.8575GiB/s) | 20260508 10:42:20.251 | INFO | Analyzing chunk frames = [7472640,7473152) | 20260508 10:42:20.251 | INFO | MemInfo: { | 20260508 10:42:20.251 | INFO | "gpu" : " 17 GB", | 20260508 10:42:20.251 | INFO | "huge" : "", | 20260508 10:42:20.251 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:20.251 | INFO | "malloc" : " 14 GB", | 20260508 10:42:20.251 | INFO | "pinned" : " 29 GB" | 20260508 10:42:20.251 | INFO | } | 20260508 10:42:20.251 | INFO | Perf report for the last 5.13452 seconds: | 20260508 10:42:20.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:20.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 10:42:20.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:42:20.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:20.251 | INFO | Uploaded 36GiB to GPU in 2.426s (14.8392GiB/s) | 20260508 10:42:20.251 | INFO | Downloaded 34GiB to GPU in 2.43826s (13.9444GiB/s) | 20260508 10:42:25.393 | INFO | Analyzing chunk frames = [7473152,7473664) | 20260508 10:42:25.393 | INFO | MemInfo: { | 20260508 10:42:25.393 | INFO | "gpu" : " 17 GB", | 20260508 10:42:25.393 | INFO | "huge" : "", | 20260508 10:42:25.393 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:25.393 | INFO | "malloc" : " 14 GB", | 20260508 10:42:25.393 | INFO | "pinned" : " 29 GB" | 20260508 10:42:25.393 | INFO | } | 20260508 10:42:25.394 | INFO | Perf report for the last 5.1424 seconds: | 20260508 10:42:25.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:25.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 10:42:25.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 10:42:25.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:25.394 | INFO | Uploaded 36GiB to GPU in 2.39722s (15.0174GiB/s) | 20260508 10:42:25.394 | INFO | Downloaded 34GiB to GPU in 2.55902s (13.2863GiB/s) | 20260508 10:42:30.535 | INFO | Analyzing chunk frames = [7473664,7474176) | 20260508 10:42:30.535 | INFO | MemInfo: { | 20260508 10:42:30.535 | INFO | "gpu" : " 17 GB", | 20260508 10:42:30.535 | INFO | "huge" : "", | 20260508 10:42:30.535 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:30.535 | INFO | "malloc" : " 14 GB", | 20260508 10:42:30.535 | INFO | "pinned" : " 29 GB" | 20260508 10:42:30.535 | INFO | } | 20260508 10:42:30.535 | INFO | Perf report for the last 5.14159 seconds: | 20260508 10:42:30.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:30.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.535 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:42:30.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.535 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:42:30.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:30.536 | INFO | Uploaded 36GiB to GPU in 2.35269s (15.3016GiB/s) | 20260508 10:42:30.536 | INFO | Downloaded 34GiB to GPU in 2.43243s (13.9778GiB/s) | 20260508 10:42:35.677 | INFO | Analyzing chunk frames = [7474176,7474688) | 20260508 10:42:35.677 | INFO | MemInfo: { | 20260508 10:42:35.677 | INFO | "gpu" : " 17 GB", | 20260508 10:42:35.677 | INFO | "huge" : "", | 20260508 10:42:35.677 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:35.677 | INFO | "malloc" : " 14 GB", | 20260508 10:42:35.677 | INFO | "pinned" : " 29 GB" | 20260508 10:42:35.678 | INFO | } | 20260508 10:42:35.678 | INFO | Perf report for the last 5.14252 seconds: | 20260508 10:42:35.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:35.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:42:35.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:42:35.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:35.678 | INFO | Uploaded 36GiB to GPU in 2.4171s (14.8939GiB/s) | 20260508 10:42:35.678 | INFO | Downloaded 34GiB to GPU in 2.44839s (13.8867GiB/s) | 20260508 10:42:40.815 | INFO | Analyzing chunk frames = [7474688,7475200) | 20260508 10:42:40.815 | INFO | MemInfo: { | 20260508 10:42:40.815 | INFO | "gpu" : " 17 GB", | 20260508 10:42:40.815 | INFO | "huge" : "", | 20260508 10:42:40.815 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:40.815 | INFO | "malloc" : " 14 GB", | 20260508 10:42:40.815 | INFO | "pinned" : " 29 GB" | 20260508 10:42:40.815 | INFO | } | 20260508 10:42:40.815 | INFO | Perf report for the last 5.13732 seconds: | 20260508 10:42:40.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:40.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:42:40.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 10:42:40.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:40.815 | INFO | Uploaded 36GiB to GPU in 2.40769s (14.9521GiB/s) | 20260508 10:42:40.816 | INFO | Downloaded 34GiB to GPU in 2.44719s (13.8935GiB/s) | 20260508 10:42:40.929 | INFO | Starting a round of Baz IO | 20260508 10:42:44.322 | INFO | Populating RTMetrics report | 20260508 10:42:44.322 | INFO | RTMetrics report populated | 20260508 10:42:44.322 | INFO | Opening tmp RTMetrics json file | 20260508 10:42:44.332 | INFO | Serializing RTMetrics to json | 20260508 10:42:44.333 | INFO | Serialization done, swapping tmp file now... | 20260508 10:42:44.333 | INFO | RTMetrics done | 20260508 10:42:44.394 | INFO | Finished a round of Baz IO: Wrote 1.454 GiB in 3.465 seconds | 20260508 10:42:44.395 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:42:44.395 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21404% / 7.94729e-06% / 7.15256e-05% | 20260508 10:42:44.395 | INFO | Baz IO overall: | 20260508 10:42:44.395 | INFO | BytesWritten : 3238849691415 | 20260508 10:42:44.395 | INFO | NumEvents : 1556496207493 | 20260508 10:42:45.948 | INFO | Analyzing chunk frames = [7475200,7475712) | 20260508 10:42:45.948 | INFO | MemInfo: { | 20260508 10:42:45.948 | INFO | "gpu" : " 17 GB", | 20260508 10:42:45.948 | INFO | "huge" : "", | 20260508 10:42:45.948 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:45.948 | INFO | "malloc" : " 14 GB", | 20260508 10:42:45.948 | INFO | "pinned" : " 29 GB" | 20260508 10:42:45.948 | INFO | } | 20260508 10:42:45.948 | INFO | Perf report for the last 5.13341 seconds: | 20260508 10:42:45.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:45.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:45.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:45.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:45.948 | INFO | ANALYSIS : 67.7%, 3.0, 105.3ms, 0 | 20260508 10:42:45.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:45.949 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260508 10:42:45.949 | INFO | PRE_HQ : 66.5%, 1.8, 60.8ms, 0 | 20260508 10:42:45.949 | INFO | BAZWRITER : 66.8%, 2.7, 92.9ms, 0 | 20260508 10:42:45.949 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:42:45.949 | INFO | Uploaded 36GiB to GPU in 2.63558s (13.6592GiB/s) | 20260508 10:42:45.949 | INFO | Downloaded 38GiB to GPU in 2.72329s (13.9537GiB/s) | 20260508 10:42:51.093 | INFO | Analyzing chunk frames = [7475712,7476224) | 20260508 10:42:51.093 | INFO | MemInfo: { | 20260508 10:42:51.093 | INFO | "gpu" : " 17 GB", | 20260508 10:42:51.093 | INFO | "huge" : "", | 20260508 10:42:51.093 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:51.093 | INFO | "malloc" : " 14 GB", | 20260508 10:42:51.093 | INFO | "pinned" : " 29 GB" | 20260508 10:42:51.093 | INFO | } | 20260508 10:42:51.093 | INFO | Perf report for the last 5.14467 seconds: | 20260508 10:42:51.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:51.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 10:42:51.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 10:42:51.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:51.093 | INFO | Uploaded 36GiB to GPU in 2.45287s (14.6767GiB/s) | 20260508 10:42:51.093 | INFO | Downloaded 34GiB to GPU in 2.46831s (13.7746GiB/s) | 20260508 10:42:56.232 | INFO | Analyzing chunk frames = [7476224,7476736) | 20260508 10:42:56.232 | INFO | MemInfo: { | 20260508 10:42:56.232 | INFO | "gpu" : " 17 GB", | 20260508 10:42:56.232 | INFO | "huge" : "", | 20260508 10:42:56.232 | INFO | "huge_shared" : " 29 GB", | 20260508 10:42:56.232 | INFO | "malloc" : " 14 GB", | 20260508 10:42:56.232 | INFO | "pinned" : " 29 GB" | 20260508 10:42:56.232 | INFO | } | 20260508 10:42:56.232 | INFO | Perf report for the last 5.13932 seconds: | 20260508 10:42:56.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:42:56.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:42:56.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:42:56.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:42:56.233 | INFO | Uploaded 36GiB to GPU in 2.46463s (14.6067GiB/s) | 20260508 10:42:56.233 | INFO | Downloaded 34GiB to GPU in 2.44212s (13.9223GiB/s) | 20260508 10:43:01.367 | INFO | Analyzing chunk frames = [7476736,7477248) | 20260508 10:43:01.367 | INFO | MemInfo: { | 20260508 10:43:01.367 | INFO | "gpu" : " 17 GB", | 20260508 10:43:01.367 | INFO | "huge" : "", | 20260508 10:43:01.367 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:01.367 | INFO | "malloc" : " 14 GB", | 20260508 10:43:01.367 | INFO | "pinned" : " 29 GB" | 20260508 10:43:01.367 | INFO | } | 20260508 10:43:01.367 | INFO | Perf report for the last 5.13481 seconds: | 20260508 10:43:01.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:01.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.367 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 10:43:01.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.368 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:43:01.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:01.368 | INFO | Uploaded 36GiB to GPU in 2.46584s (14.5995GiB/s) | 20260508 10:43:01.368 | INFO | Downloaded 34GiB to GPU in 2.44821s (13.8877GiB/s) | 20260508 10:43:06.508 | INFO | Analyzing chunk frames = [7477248,7477760) | 20260508 10:43:06.508 | INFO | MemInfo: { | 20260508 10:43:06.508 | INFO | "gpu" : " 17 GB", | 20260508 10:43:06.508 | INFO | "huge" : "", | 20260508 10:43:06.508 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:06.508 | INFO | "malloc" : " 14 GB", | 20260508 10:43:06.508 | INFO | "pinned" : " 29 GB" | 20260508 10:43:06.508 | INFO | } | 20260508 10:43:06.508 | INFO | Perf report for the last 5.14036 seconds: | 20260508 10:43:06.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:06.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 10:43:06.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:43:06.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:06.508 | INFO | Uploaded 36GiB to GPU in 2.39202s (15.05GiB/s) | 20260508 10:43:06.508 | INFO | Downloaded 34GiB to GPU in 2.52722s (13.4535GiB/s) | 20260508 10:43:11.646 | INFO | Analyzing chunk frames = [7477760,7478272) | 20260508 10:43:11.646 | INFO | MemInfo: { | 20260508 10:43:11.646 | INFO | "gpu" : " 17 GB", | 20260508 10:43:11.646 | INFO | "huge" : "", | 20260508 10:43:11.646 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:11.646 | INFO | "malloc" : " 14 GB", | 20260508 10:43:11.646 | INFO | "pinned" : " 29 GB" | 20260508 10:43:11.646 | INFO | } | 20260508 10:43:11.646 | INFO | Perf report for the last 5.13821 seconds: | 20260508 10:43:11.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:11.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:43:11.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:43:11.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:11.646 | INFO | Uploaded 36GiB to GPU in 2.36217s (15.2402GiB/s) | 20260508 10:43:11.646 | INFO | Downloaded 34GiB to GPU in 2.43672s (13.9532GiB/s) | 20260508 10:43:16.789 | INFO | Analyzing chunk frames = [7478272,7478784) | 20260508 10:43:16.789 | INFO | MemInfo: { | 20260508 10:43:16.789 | INFO | "gpu" : " 17 GB", | 20260508 10:43:16.789 | INFO | "huge" : "", | 20260508 10:43:16.789 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:16.789 | INFO | "malloc" : " 14 GB", | 20260508 10:43:16.789 | INFO | "pinned" : " 29 GB" | 20260508 10:43:16.789 | INFO | } | 20260508 10:43:16.789 | INFO | Perf report for the last 5.14339 seconds: | 20260508 10:43:16.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:16.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.789 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:43:16.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.789 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:43:16.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:16.790 | INFO | Uploaded 36GiB to GPU in 2.38899s (15.0692GiB/s) | 20260508 10:43:16.790 | INFO | Downloaded 34GiB to GPU in 2.44049s (13.9317GiB/s) | 20260508 10:43:21.929 | INFO | Analyzing chunk frames = [7478784,7479296) | 20260508 10:43:21.929 | INFO | MemInfo: { | 20260508 10:43:21.929 | INFO | "gpu" : " 17 GB", | 20260508 10:43:21.929 | INFO | "huge" : "", | 20260508 10:43:21.929 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:21.929 | INFO | "malloc" : " 14 GB", | 20260508 10:43:21.929 | INFO | "pinned" : " 29 GB" | 20260508 10:43:21.929 | INFO | } | 20260508 10:43:21.929 | INFO | Perf report for the last 5.13948 seconds: | 20260508 10:43:21.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:21.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:43:21.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:43:21.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:21.929 | INFO | Uploaded 36GiB to GPU in 2.40426s (14.9734GiB/s) | 20260508 10:43:21.929 | INFO | Downloaded 34GiB to GPU in 2.41829s (14.0595GiB/s) | 20260508 10:43:21.929 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:43:21.929 | INFO | Total mem usage: 16.7 GiB | 20260508 10:43:21.929 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:43:21.929 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 10:43:21.929 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:43:21.929 | INFO | High Water Breakdown: | 20260508 10:43:21.929 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:43:21.929 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:43:21.929 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:43:21.929 | INFO | ------------------------------------------ | 20260508 10:43:21.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:43:21.929 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:43:21.929 | INFO | Total mem usage: 26.5 GiB | 20260508 10:43:21.929 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:43:21.929 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:43:21.929 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:43:21.929 | INFO | High Water Breakdown: | 20260508 10:43:21.929 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:43:21.929 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:43:21.929 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:43:21.929 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:43:21.929 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:43:21.929 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:43:21.929 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:43:21.929 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:43:21.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:43:21.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:43:21.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:43:21.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:43:21.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:43:21.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:43:21.929 | INFO | ------------------------------------------ | 20260508 10:43:21.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:43:21.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:43:21.929 | INFO | Total mem usage: 27.8 GiB | 20260508 10:43:21.929 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:43:21.929 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:43:21.929 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:43:21.929 | INFO | High Water Breakdown: | 20260508 10:43:21.929 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:43:21.929 | INFO | ------------------------------------------ | 20260508 10:43:21.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:43:21.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:43:21.929 | INFO | Total mem usage: 15.7 GiB | 20260508 10:43:21.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:43:21.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:43:21.929 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:43:21.929 | INFO | High Water Breakdown: | 20260508 10:43:21.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:43:21.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:43:21.929 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:43:21.929 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:43:21.929 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:43:21.929 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:43:21.929 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:43:21.929 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:43:21.929 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:43:21.929 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:43:21.929 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:43:21.929 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:43:21.929 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:43:21.929 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:43:21.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:43:21.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:43:21.929 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:43:21.929 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:43:21.929 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:43:21.929 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:43:21.929 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:43:21.929 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:43:21.929 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:43:21.929 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:43:21.929 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:43:21.929 | INFO | ------------------------------------------ | 20260508 10:43:21.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:43:21.929 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:43:21.929 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:43:21.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:43:21.929 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.42e+01 / 7.81e-01 | 20260508 10:43:21.929 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.13e-02 / 6.55e-03 | 20260508 10:43:21.929 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.49e+00, 5.88e-01 / 1.78e-01 | 20260508 10:43:21.929 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.49e+00, 2.10e-01 / 1.28e-01 | 20260508 10:43:21.929 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.52e+01, 3.69e+00 / 1.06e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.80e+00 / 1.22e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.60e+00, 4.04e-01 / 2.06e-01 | 20260508 10:43:21.929 | INFO | PerfReport: 3.01%, Metrics -- 9.85e-01 / 1.14e+00, 2.43e-01 / 1.05e-01 | 20260508 10:43:21.929 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.39e-01, 4.67e-02 / 2.42e-02 | 20260508 10:43:21.929 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:43:21.929 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:43:21.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:43:21.929 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 24.14%, Upload -- 2.54e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 24.69%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 31.09%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 2.07%, LockUpload -- 2.18e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 15.33%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:43:21.929 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 10:43:22.026 | INFO | Starting a round of Baz IO | 20260508 10:43:25.431 | INFO | Populating RTMetrics report | 20260508 10:43:25.432 | INFO | RTMetrics report populated | 20260508 10:43:25.432 | INFO | Opening tmp RTMetrics json file | 20260508 10:43:25.443 | INFO | Serializing RTMetrics to json | 20260508 10:43:25.444 | INFO | Serialization done, swapping tmp file now... | 20260508 10:43:25.444 | INFO | RTMetrics done | 20260508 10:43:25.504 | INFO | Finished a round of Baz IO: Wrote 1.453 GiB in 3.478 seconds | 20260508 10:43:25.504 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:43:25.504 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.213877% / 0% / 5.16574e-05% | 20260508 10:43:25.504 | INFO | Baz IO overall: | 20260508 10:43:25.504 | INFO | BytesWritten : 3240410184010 | 20260508 10:43:25.504 | INFO | NumEvents : 1557231447305 | 20260508 10:43:27.068 | INFO | Analyzing chunk frames = [7479296,7479808) | 20260508 10:43:27.068 | INFO | MemInfo: { | 20260508 10:43:27.068 | INFO | "gpu" : " 17 GB", | 20260508 10:43:27.068 | INFO | "huge" : "", | 20260508 10:43:27.068 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:27.068 | INFO | "malloc" : " 14 GB", | 20260508 10:43:27.068 | INFO | "pinned" : " 29 GB" | 20260508 10:43:27.068 | INFO | } | 20260508 10:43:27.068 | INFO | Perf report for the last 5.13946 seconds: | 20260508 10:43:27.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:27.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:27.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:27.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:27.068 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:43:27.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:27.068 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 10:43:27.068 | INFO | PRE_HQ : 65.5%, 1.6, 54.4ms, 0 | 20260508 10:43:27.069 | INFO | BAZWRITER : 67.0%, 2.5, 86.3ms, 0 | 20260508 10:43:27.069 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:43:27.069 | INFO | Uploaded 36GiB to GPU in 2.60282s (13.8311GiB/s) | 20260508 10:43:27.069 | INFO | Downloaded 38GiB to GPU in 2.70562s (14.0449GiB/s) | 20260508 10:43:32.203 | INFO | Analyzing chunk frames = [7479808,7480320) | 20260508 10:43:32.203 | INFO | MemInfo: { | 20260508 10:43:32.203 | INFO | "gpu" : " 17 GB", | 20260508 10:43:32.203 | INFO | "huge" : "", | 20260508 10:43:32.203 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:32.203 | INFO | "malloc" : " 14 GB", | 20260508 10:43:32.203 | INFO | "pinned" : " 29 GB" | 20260508 10:43:32.203 | INFO | } | 20260508 10:43:32.203 | INFO | Perf report for the last 5.13504 seconds: | 20260508 10:43:32.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:32.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.203 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 10:43:32.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.203 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:43:32.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:32.204 | INFO | Uploaded 36GiB to GPU in 2.48945s (14.461GiB/s) | 20260508 10:43:32.204 | INFO | Downloaded 34GiB to GPU in 2.46314s (13.8035GiB/s) | 20260508 10:43:37.344 | INFO | Analyzing chunk frames = [7480320,7480832) | 20260508 10:43:37.344 | INFO | MemInfo: { | 20260508 10:43:37.344 | INFO | "gpu" : " 17 GB", | 20260508 10:43:37.344 | INFO | "huge" : "", | 20260508 10:43:37.344 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:37.344 | INFO | "malloc" : " 14 GB", | 20260508 10:43:37.344 | INFO | "pinned" : " 29 GB" | 20260508 10:43:37.344 | INFO | } | 20260508 10:43:37.344 | INFO | Perf report for the last 5.14072 seconds: | 20260508 10:43:37.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:37.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 10:43:37.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:43:37.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:37.344 | INFO | Uploaded 36GiB to GPU in 2.49887s (14.4065GiB/s) | 20260508 10:43:37.344 | INFO | Downloaded 34GiB to GPU in 2.43526s (13.9615GiB/s) | 20260508 10:43:42.485 | INFO | Analyzing chunk frames = [7480832,7481344) | 20260508 10:43:42.485 | INFO | MemInfo: { | 20260508 10:43:42.485 | INFO | "gpu" : " 17 GB", | 20260508 10:43:42.485 | INFO | "huge" : "", | 20260508 10:43:42.485 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:42.485 | INFO | "malloc" : " 14 GB", | 20260508 10:43:42.485 | INFO | "pinned" : " 29 GB" | 20260508 10:43:42.485 | INFO | } | 20260508 10:43:42.485 | INFO | Perf report for the last 5.1409 seconds: | 20260508 10:43:42.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:42.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.485 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 10:43:42.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.485 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:43:42.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:42.486 | INFO | Uploaded 36GiB to GPU in 2.48502s (14.4868GiB/s) | 20260508 10:43:42.486 | INFO | Downloaded 34GiB to GPU in 2.41002s (14.1078GiB/s) | 20260508 10:43:47.627 | INFO | Analyzing chunk frames = [7481344,7481856) | 20260508 10:43:47.627 | INFO | MemInfo: { | 20260508 10:43:47.627 | INFO | "gpu" : " 17 GB", | 20260508 10:43:47.627 | INFO | "huge" : "", | 20260508 10:43:47.627 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:47.627 | INFO | "malloc" : " 14 GB", | 20260508 10:43:47.627 | INFO | "pinned" : " 29 GB" | 20260508 10:43:47.627 | INFO | } | 20260508 10:43:47.627 | INFO | Perf report for the last 5.14212 seconds: | 20260508 10:43:47.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:47.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:43:47.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:43:47.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:47.627 | INFO | Uploaded 36GiB to GPU in 2.29206s (15.7064GiB/s) | 20260508 10:43:47.628 | INFO | Downloaded 34GiB to GPU in 2.50935s (13.5493GiB/s) | 20260508 10:43:52.763 | INFO | Analyzing chunk frames = [7481856,7482368) | 20260508 10:43:52.763 | INFO | MemInfo: { | 20260508 10:43:52.763 | INFO | "gpu" : " 17 GB", | 20260508 10:43:52.763 | INFO | "huge" : "", | 20260508 10:43:52.763 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:52.763 | INFO | "malloc" : " 14 GB", | 20260508 10:43:52.763 | INFO | "pinned" : " 29 GB" | 20260508 10:43:52.763 | INFO | } | 20260508 10:43:52.763 | INFO | Perf report for the last 5.13608 seconds: | 20260508 10:43:52.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:52.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:43:52.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:43:52.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:52.763 | INFO | Uploaded 36GiB to GPU in 2.38655s (15.0845GiB/s) | 20260508 10:43:52.763 | INFO | Downloaded 34GiB to GPU in 2.41625s (14.0714GiB/s) | 20260508 10:43:57.907 | INFO | Analyzing chunk frames = [7482368,7482880) | 20260508 10:43:57.907 | INFO | MemInfo: { | 20260508 10:43:57.907 | INFO | "gpu" : " 17 GB", | 20260508 10:43:57.907 | INFO | "huge" : "", | 20260508 10:43:57.907 | INFO | "huge_shared" : " 29 GB", | 20260508 10:43:57.908 | INFO | "malloc" : " 14 GB", | 20260508 10:43:57.908 | INFO | "pinned" : " 29 GB" | 20260508 10:43:57.908 | INFO | } | 20260508 10:43:57.908 | INFO | Perf report for the last 5.14447 seconds: | 20260508 10:43:57.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:43:57.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 10:43:57.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:43:57.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:43:57.908 | INFO | Uploaded 36GiB to GPU in 2.44797s (14.7061GiB/s) | 20260508 10:43:57.908 | INFO | Downloaded 34GiB to GPU in 2.43898s (13.9402GiB/s) | 20260508 10:44:03.041 | INFO | Analyzing chunk frames = [7482880,7483392) | 20260508 10:44:03.041 | INFO | MemInfo: { | 20260508 10:44:03.041 | INFO | "gpu" : " 17 GB", | 20260508 10:44:03.041 | INFO | "huge" : "", | 20260508 10:44:03.041 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:03.041 | INFO | "malloc" : " 14 GB", | 20260508 10:44:03.041 | INFO | "pinned" : " 29 GB" | 20260508 10:44:03.041 | INFO | } | 20260508 10:44:03.041 | INFO | Perf report for the last 5.1331 seconds: | 20260508 10:44:03.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:03.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 10:44:03.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:44:03.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:03.041 | INFO | Uploaded 36GiB to GPU in 2.48097s (14.5104GiB/s) | 20260508 10:44:03.041 | INFO | Downloaded 34GiB to GPU in 2.44944s (13.8807GiB/s) | 20260508 10:44:03.138 | INFO | Starting a round of Baz IO | 20260508 10:44:06.547 | INFO | Populating RTMetrics report | 20260508 10:44:06.547 | INFO | RTMetrics report populated | 20260508 10:44:06.547 | INFO | Opening tmp RTMetrics json file | 20260508 10:44:06.556 | INFO | Serializing RTMetrics to json | 20260508 10:44:06.557 | INFO | Serialization done, swapping tmp file now... | 20260508 10:44:06.558 | INFO | RTMetrics done | 20260508 10:44:06.616 | INFO | Finished a round of Baz IO: Wrote 1.453 GiB in 3.478 seconds | 20260508 10:44:06.616 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:44:06.616 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212558% / 0% / 5.5631e-05% | 20260508 10:44:06.616 | INFO | Baz IO overall: | 20260508 10:44:06.616 | INFO | BytesWritten : 3241969860718 | 20260508 10:44:06.616 | INFO | NumEvents : 1557966107190 | 20260508 10:44:08.186 | INFO | Analyzing chunk frames = [7483392,7483904) | 20260508 10:44:08.186 | INFO | MemInfo: { | 20260508 10:44:08.186 | INFO | "gpu" : " 17 GB", | 20260508 10:44:08.187 | INFO | "huge" : "", | 20260508 10:44:08.187 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:08.187 | INFO | "malloc" : " 14 GB", | 20260508 10:44:08.187 | INFO | "pinned" : " 29 GB" | 20260508 10:44:08.187 | INFO | } | 20260508 10:44:08.187 | INFO | Perf report for the last 5.14587 seconds: | 20260508 10:44:08.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:08.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:08.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:08.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:08.187 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:44:08.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:08.187 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 10:44:08.187 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 10:44:08.187 | INFO | BAZWRITER : 66.9%, 2.5, 85.8ms, 0 | 20260508 10:44:08.187 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:44:08.187 | INFO | Uploaded 36GiB to GPU in 2.58684s (13.9166GiB/s) | 20260508 10:44:08.187 | INFO | Downloaded 38GiB to GPU in 2.71446s (13.9991GiB/s) | 20260508 10:44:13.322 | INFO | Analyzing chunk frames = [7483904,7484416) | 20260508 10:44:13.322 | INFO | MemInfo: { | 20260508 10:44:13.322 | INFO | "gpu" : " 17 GB", | 20260508 10:44:13.322 | INFO | "huge" : "", | 20260508 10:44:13.322 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:13.322 | INFO | "malloc" : " 14 GB", | 20260508 10:44:13.322 | INFO | "pinned" : " 29 GB" | 20260508 10:44:13.322 | INFO | } | 20260508 10:44:13.322 | INFO | Perf report for the last 5.13576 seconds: | 20260508 10:44:13.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:13.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 10:44:13.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:44:13.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:13.323 | INFO | Uploaded 36GiB to GPU in 2.42922s (14.8196GiB/s) | 20260508 10:44:13.323 | INFO | Downloaded 34GiB to GPU in 2.46696s (13.7821GiB/s) | 20260508 10:44:18.454 | INFO | Analyzing chunk frames = [7484416,7484928) | 20260508 10:44:18.454 | INFO | MemInfo: { | 20260508 10:44:18.454 | INFO | "gpu" : " 17 GB", | 20260508 10:44:18.454 | INFO | "huge" : "", | 20260508 10:44:18.454 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:18.454 | INFO | "malloc" : " 14 GB", | 20260508 10:44:18.454 | INFO | "pinned" : " 29 GB" | 20260508 10:44:18.454 | INFO | } | 20260508 10:44:18.454 | INFO | Perf report for the last 5.13158 seconds: | 20260508 10:44:18.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:18.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.454 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 10:44:18.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.454 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:44:18.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:18.455 | INFO | Uploaded 36GiB to GPU in 2.46784s (14.5876GiB/s) | 20260508 10:44:18.455 | INFO | Downloaded 34GiB to GPU in 2.44527s (13.9044GiB/s) | 20260508 10:44:23.601 | INFO | Analyzing chunk frames = [7484928,7485440) | 20260508 10:44:23.601 | INFO | MemInfo: { | 20260508 10:44:23.601 | INFO | "gpu" : " 17 GB", | 20260508 10:44:23.601 | INFO | "huge" : "", | 20260508 10:44:23.601 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:23.601 | INFO | "malloc" : " 14 GB", | 20260508 10:44:23.601 | INFO | "pinned" : " 29 GB" | 20260508 10:44:23.601 | INFO | } | 20260508 10:44:23.601 | INFO | Perf report for the last 5.14702 seconds: | 20260508 10:44:23.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:23.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.601 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 10:44:23.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.601 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:44:23.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:23.602 | INFO | Uploaded 36GiB to GPU in 2.48788s (14.4702GiB/s) | 20260508 10:44:23.602 | INFO | Downloaded 34GiB to GPU in 2.42167s (14.0399GiB/s) | 20260508 10:44:28.745 | INFO | Analyzing chunk frames = [7485440,7485952) | 20260508 10:44:28.745 | INFO | MemInfo: { | 20260508 10:44:28.745 | INFO | "gpu" : " 17 GB", | 20260508 10:44:28.745 | INFO | "huge" : "", | 20260508 10:44:28.745 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:28.745 | INFO | "malloc" : " 14 GB", | 20260508 10:44:28.745 | INFO | "pinned" : " 29 GB" | 20260508 10:44:28.745 | INFO | } | 20260508 10:44:28.745 | INFO | Perf report for the last 5.144 seconds: | 20260508 10:44:28.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:28.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 10:44:28.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:44:28.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:28.745 | INFO | Uploaded 36GiB to GPU in 2.27026s (15.8572GiB/s) | 20260508 10:44:28.745 | INFO | Downloaded 34GiB to GPU in 2.49969s (13.6017GiB/s) | 20260508 10:44:33.879 | INFO | Analyzing chunk frames = [7485952,7486464) | 20260508 10:44:33.879 | INFO | MemInfo: { | 20260508 10:44:33.879 | INFO | "gpu" : " 17 GB", | 20260508 10:44:33.879 | INFO | "huge" : "", | 20260508 10:44:33.879 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:33.879 | INFO | "malloc" : " 14 GB", | 20260508 10:44:33.879 | INFO | "pinned" : " 29 GB" | 20260508 10:44:33.879 | INFO | } | 20260508 10:44:33.879 | INFO | Perf report for the last 5.13406 seconds: | 20260508 10:44:33.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:33.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 10:44:33.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:44:33.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:33.879 | INFO | Uploaded 36GiB to GPU in 2.39048s (15.0598GiB/s) | 20260508 10:44:33.879 | INFO | Downloaded 34GiB to GPU in 2.44029s (13.9328GiB/s) | 20260508 10:44:39.011 | INFO | Analyzing chunk frames = [7486464,7486976) | 20260508 10:44:39.011 | INFO | MemInfo: { | 20260508 10:44:39.011 | INFO | "gpu" : " 17 GB", | 20260508 10:44:39.011 | INFO | "huge" : "", | 20260508 10:44:39.011 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:39.011 | INFO | "malloc" : " 14 GB", | 20260508 10:44:39.011 | INFO | "pinned" : " 29 GB" | 20260508 10:44:39.011 | INFO | } | 20260508 10:44:39.011 | INFO | Perf report for the last 5.13223 seconds: | 20260508 10:44:39.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:39.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.011 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:44:39.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.012 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:44:39.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:39.012 | INFO | Uploaded 36GiB to GPU in 2.39435s (15.0354GiB/s) | 20260508 10:44:39.012 | INFO | Downloaded 34GiB to GPU in 2.45034s (13.8756GiB/s) | 20260508 10:44:44.159 | INFO | Analyzing chunk frames = [7486976,7487488) | 20260508 10:44:44.159 | INFO | MemInfo: { | 20260508 10:44:44.159 | INFO | "gpu" : " 17 GB", | 20260508 10:44:44.159 | INFO | "huge" : "", | 20260508 10:44:44.159 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:44.159 | INFO | "malloc" : " 14 GB", | 20260508 10:44:44.159 | INFO | "pinned" : " 29 GB" | 20260508 10:44:44.159 | INFO | } | 20260508 10:44:44.159 | INFO | Perf report for the last 5.14745 seconds: | 20260508 10:44:44.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:44.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 10:44:44.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:44:44.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:44.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:44:44.159 | INFO | Total mem usage: 16.7 GiB | 20260508 10:44:44.159 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:44:44.159 | INFO | Uploaded 36GiB to GPU in 2.4148s (14.9081GiB/s) | 20260508 10:44:44.159 | INFO | Downloaded 34GiB to GPU in 2.44141s (13.9264GiB/s) | 20260508 10:44:44.159 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 10:44:44.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:44:44.159 | INFO | High Water Breakdown: | 20260508 10:44:44.159 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:44:44.160 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:44:44.160 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:44:44.160 | INFO | ------------------------------------------ | 20260508 10:44:44.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:44:44.160 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:44:44.160 | INFO | Total mem usage: 26.5 GiB | 20260508 10:44:44.160 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:44:44.160 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:44:44.160 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:44:44.160 | INFO | High Water Breakdown: | 20260508 10:44:44.160 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:44:44.160 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:44:44.160 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:44:44.160 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:44:44.160 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:44:44.160 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:44:44.160 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:44:44.160 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:44:44.160 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:44:44.160 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:44:44.160 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:44:44.160 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:44:44.160 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:44:44.160 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:44:44.160 | INFO | ------------------------------------------ | 20260508 10:44:44.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:44:44.160 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:44:44.160 | INFO | Total mem usage: 27.8 GiB | 20260508 10:44:44.160 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:44:44.160 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:44:44.160 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:44:44.160 | INFO | High Water Breakdown: | 20260508 10:44:44.160 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:44:44.160 | INFO | ------------------------------------------ | 20260508 10:44:44.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:44:44.160 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:44:44.160 | INFO | Total mem usage: 15.7 GiB | 20260508 10:44:44.160 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:44:44.160 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:44:44.160 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:44:44.160 | INFO | High Water Breakdown: | 20260508 10:44:44.160 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:44:44.160 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:44:44.160 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:44:44.160 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:44:44.160 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:44:44.160 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:44:44.160 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:44:44.160 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:44:44.160 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:44:44.160 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:44:44.160 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:44:44.160 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:44:44.160 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:44:44.160 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:44:44.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:44:44.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:44:44.160 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:44:44.160 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:44:44.160 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:44:44.160 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:44:44.160 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:44:44.160 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:44:44.160 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:44:44.160 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:44:44.160 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:44:44.160 | INFO | ------------------------------------------ | 20260508 10:44:44.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:44:44.160 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:44:44.160 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:44:44.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:44:44.160 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 6.26e-01 | 20260508 10:44:44.160 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.30e-02 / 4.52e-03 | 20260508 10:44:44.160 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.45e+00, 5.96e-01 / 2.16e-01 | 20260508 10:44:44.160 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.28e-01 | 20260508 10:44:44.160 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.54e+01, 3.75e+00 / 1.07e+00 | 20260508 10:44:44.160 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.75e+00 / 1.23e+00 | 20260508 10:44:44.160 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.55e+00, 4.08e-01 / 2.46e-01 | 20260508 10:44:44.160 | INFO | PerfReport: 3.02%, Metrics -- 9.85e-01 / 1.14e+00, 2.46e-01 / 1.57e-01 | 20260508 10:44:44.160 | INFO | PerfReport: 0.92%, PulseSerializing -- 2.99e-01 / 3.33e-01, 4.91e-02 / 2.94e-02 | 20260508 10:44:44.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:44:44.160 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:44:44.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:44:44.161 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 2.23%, LockUpload -- 2.34e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 2.56%, LockDownload -- 2.69e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 15.35%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:44:44.161 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260508 10:44:44.256 | INFO | Starting a round of Baz IO | 20260508 10:44:47.659 | INFO | Populating RTMetrics report | 20260508 10:44:47.659 | INFO | RTMetrics report populated | 20260508 10:44:47.659 | INFO | Opening tmp RTMetrics json file | 20260508 10:44:47.669 | INFO | Serializing RTMetrics to json | 20260508 10:44:47.670 | INFO | Serialization done, swapping tmp file now... | 20260508 10:44:47.671 | INFO | RTMetrics done | 20260508 10:44:47.730 | INFO | Finished a round of Baz IO: Wrote 1.453 GiB in 3.474 seconds | 20260508 10:44:47.730 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:44:47.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212753% / 0% / 6.75519e-05% | 20260508 10:44:47.731 | INFO | Baz IO overall: | 20260508 10:44:47.731 | INFO | BytesWritten : 3243530098076 | 20260508 10:44:47.731 | INFO | NumEvents : 1558701564330 | 20260508 10:44:49.296 | INFO | Analyzing chunk frames = [7487488,7488000) | 20260508 10:44:49.296 | INFO | MemInfo: { | 20260508 10:44:49.296 | INFO | "gpu" : " 17 GB", | 20260508 10:44:49.296 | INFO | "huge" : "", | 20260508 10:44:49.296 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:49.296 | INFO | "malloc" : " 14 GB", | 20260508 10:44:49.296 | INFO | "pinned" : " 29 GB" | 20260508 10:44:49.296 | INFO | } | 20260508 10:44:49.297 | INFO | Perf report for the last 5.13766 seconds: | 20260508 10:44:49.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:49.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:49.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:49.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:49.297 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 10:44:49.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:49.297 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:44:49.297 | INFO | PRE_HQ : 65.4%, 1.6, 54.3ms, 0 | 20260508 10:44:49.297 | INFO | BAZWRITER : 66.8%, 2.5, 87.4ms, 0 | 20260508 10:44:49.297 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:44:49.297 | INFO | Uploaded 36GiB to GPU in 2.62351s (13.7221GiB/s) | 20260508 10:44:49.297 | INFO | Downloaded 38GiB to GPU in 2.7121s (14.0113GiB/s) | 20260508 10:44:54.436 | INFO | Analyzing chunk frames = [7488000,7488512) | 20260508 10:44:54.436 | INFO | MemInfo: { | 20260508 10:44:54.436 | INFO | "gpu" : " 17 GB", | 20260508 10:44:54.436 | INFO | "huge" : "", | 20260508 10:44:54.436 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:54.436 | INFO | "malloc" : " 14 GB", | 20260508 10:44:54.436 | INFO | "pinned" : " 29 GB" | 20260508 10:44:54.436 | INFO | } | 20260508 10:44:54.436 | INFO | Perf report for the last 5.13966 seconds: | 20260508 10:44:54.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:54.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | ANALYSIS : 66.9%, 3.0, 104.3ms, 0 | 20260508 10:44:54.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:44:54.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:54.436 | INFO | Uploaded 36GiB to GPU in 2.52092s (14.2805GiB/s) | 20260508 10:44:54.437 | INFO | Downloaded 34GiB to GPU in 2.46504s (13.7929GiB/s) | 20260508 10:44:59.576 | INFO | Analyzing chunk frames = [7488512,7489024) | 20260508 10:44:59.576 | INFO | MemInfo: { | 20260508 10:44:59.576 | INFO | "gpu" : " 17 GB", | 20260508 10:44:59.576 | INFO | "huge" : "", | 20260508 10:44:59.576 | INFO | "huge_shared" : " 29 GB", | 20260508 10:44:59.576 | INFO | "malloc" : " 14 GB", | 20260508 10:44:59.576 | INFO | "pinned" : " 29 GB" | 20260508 10:44:59.576 | INFO | } | 20260508 10:44:59.576 | INFO | Perf report for the last 5.13984 seconds: | 20260508 10:44:59.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:44:59.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.576 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:44:59.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.576 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 10:44:59.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:44:59.577 | INFO | Uploaded 36GiB to GPU in 2.4659s (14.5991GiB/s) | 20260508 10:44:59.577 | INFO | Downloaded 34GiB to GPU in 2.45s (13.8776GiB/s) | 20260508 10:45:04.710 | INFO | Analyzing chunk frames = [7489024,7489536) | 20260508 10:45:04.710 | INFO | MemInfo: { | 20260508 10:45:04.710 | INFO | "gpu" : " 17 GB", | 20260508 10:45:04.710 | INFO | "huge" : "", | 20260508 10:45:04.710 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:04.710 | INFO | "malloc" : " 14 GB", | 20260508 10:45:04.710 | INFO | "pinned" : " 29 GB" | 20260508 10:45:04.711 | INFO | } | 20260508 10:45:04.711 | INFO | Perf report for the last 5.13447 seconds: | 20260508 10:45:04.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:04.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:45:04.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:45:04.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:04.711 | INFO | Uploaded 36GiB to GPU in 2.46677s (14.594GiB/s) | 20260508 10:45:04.711 | INFO | Downloaded 34GiB to GPU in 2.4652s (13.792GiB/s) | 20260508 10:45:09.854 | INFO | Analyzing chunk frames = [7489536,7490048) | 20260508 10:45:09.854 | INFO | MemInfo: { | 20260508 10:45:09.854 | INFO | "gpu" : " 17 GB", | 20260508 10:45:09.855 | INFO | "huge" : "", | 20260508 10:45:09.855 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:09.855 | INFO | "malloc" : " 14 GB", | 20260508 10:45:09.855 | INFO | "pinned" : " 29 GB" | 20260508 10:45:09.855 | INFO | } | 20260508 10:45:09.855 | INFO | Perf report for the last 5.14408 seconds: | 20260508 10:45:09.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:09.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 10:45:09.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:45:09.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:09.855 | INFO | Uploaded 36GiB to GPU in 2.39754s (15.0154GiB/s) | 20260508 10:45:09.855 | INFO | Downloaded 34GiB to GPU in 2.53721s (13.4006GiB/s) | 20260508 10:45:14.992 | INFO | Analyzing chunk frames = [7490048,7490560) | 20260508 10:45:14.992 | INFO | MemInfo: { | 20260508 10:45:14.992 | INFO | "gpu" : " 17 GB", | 20260508 10:45:14.992 | INFO | "huge" : "", | 20260508 10:45:14.992 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:14.992 | INFO | "malloc" : " 14 GB", | 20260508 10:45:14.992 | INFO | "pinned" : " 29 GB" | 20260508 10:45:14.992 | INFO | } | 20260508 10:45:14.992 | INFO | Perf report for the last 5.13774 seconds: | 20260508 10:45:14.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:14.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.992 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:45:14.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.992 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:45:14.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:14.993 | INFO | Uploaded 36GiB to GPU in 2.35215s (15.3052GiB/s) | 20260508 10:45:14.993 | INFO | Downloaded 34GiB to GPU in 2.45847s (13.8298GiB/s) | 20260508 10:45:20.130 | INFO | Analyzing chunk frames = [7490560,7491072) | 20260508 10:45:20.130 | INFO | MemInfo: { | 20260508 10:45:20.130 | INFO | "gpu" : " 17 GB", | 20260508 10:45:20.130 | INFO | "huge" : "", | 20260508 10:45:20.130 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:20.130 | INFO | "malloc" : " 14 GB", | 20260508 10:45:20.131 | INFO | "pinned" : " 29 GB" | 20260508 10:45:20.131 | INFO | } | 20260508 10:45:20.131 | INFO | Perf report for the last 5.13818 seconds: | 20260508 10:45:20.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:20.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 10:45:20.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:45:20.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:20.131 | INFO | Uploaded 36GiB to GPU in 2.34072s (15.3799GiB/s) | 20260508 10:45:20.131 | INFO | Downloaded 34GiB to GPU in 2.45509s (13.8488GiB/s) | 20260508 10:45:25.271 | INFO | Analyzing chunk frames = [7491072,7491584) | 20260508 10:45:25.271 | INFO | MemInfo: { | 20260508 10:45:25.271 | INFO | "gpu" : " 17 GB", | 20260508 10:45:25.271 | INFO | "huge" : "", | 20260508 10:45:25.271 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:25.271 | INFO | "malloc" : " 14 GB", | 20260508 10:45:25.271 | INFO | "pinned" : " 29 GB" | 20260508 10:45:25.271 | INFO | } | 20260508 10:45:25.271 | INFO | Perf report for the last 5.14039 seconds: | 20260508 10:45:25.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:25.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.271 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 10:45:25.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.271 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:45:25.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:25.272 | INFO | Uploaded 36GiB to GPU in 2.37289s (15.1714GiB/s) | 20260508 10:45:25.272 | INFO | Downloaded 34GiB to GPU in 2.44516s (13.905GiB/s) | 20260508 10:45:25.369 | INFO | Starting a round of Baz IO | 20260508 10:45:28.774 | INFO | Populating RTMetrics report | 20260508 10:45:28.774 | INFO | RTMetrics report populated | 20260508 10:45:28.774 | INFO | Opening tmp RTMetrics json file | 20260508 10:45:28.785 | INFO | Serializing RTMetrics to json | 20260508 10:45:28.786 | INFO | Serialization done, swapping tmp file now... | 20260508 10:45:28.786 | INFO | RTMetrics done | 20260508 10:45:28.845 | INFO | Finished a round of Baz IO: Wrote 1.452 GiB in 3.476 seconds | 20260508 10:45:28.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:45:28.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212932% / 0% / 5.5631e-05% | 20260508 10:45:28.845 | INFO | Baz IO overall: | 20260508 10:45:28.845 | INFO | BytesWritten : 3245089195272 | 20260508 10:45:28.845 | INFO | NumEvents : 1559436274663 | 20260508 10:45:30.413 | INFO | Analyzing chunk frames = [7491584,7492096) | 20260508 10:45:30.413 | INFO | MemInfo: { | 20260508 10:45:30.413 | INFO | "gpu" : " 17 GB", | 20260508 10:45:30.413 | INFO | "huge" : "", | 20260508 10:45:30.413 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:30.413 | INFO | "malloc" : " 14 GB", | 20260508 10:45:30.413 | INFO | "pinned" : " 29 GB" | 20260508 10:45:30.413 | INFO | } | 20260508 10:45:30.413 | INFO | Perf report for the last 5.14222 seconds: | 20260508 10:45:30.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:30.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:30.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:30.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:30.413 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:45:30.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:30.413 | INFO | RT_METRICS : 40.0%, 1.0, 21.0ms, 0 | 20260508 10:45:30.413 | INFO | PRE_HQ : 65.7%, 1.7, 56.6ms, 0 | 20260508 10:45:30.413 | INFO | BAZWRITER : 66.9%, 2.6, 88.2ms, 0 | 20260508 10:45:30.413 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:45:30.413 | INFO | Uploaded 36GiB to GPU in 2.59544s (13.8705GiB/s) | 20260508 10:45:30.413 | INFO | Downloaded 38GiB to GPU in 2.71302s (14.0065GiB/s) | 20260508 10:45:35.553 | INFO | Analyzing chunk frames = [7492096,7492608) | 20260508 10:45:35.553 | INFO | MemInfo: { | 20260508 10:45:35.553 | INFO | "gpu" : " 17 GB", | 20260508 10:45:35.553 | INFO | "huge" : "", | 20260508 10:45:35.553 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:35.553 | INFO | "malloc" : " 14 GB", | 20260508 10:45:35.553 | INFO | "pinned" : " 29 GB" | 20260508 10:45:35.553 | INFO | } | 20260508 10:45:35.554 | INFO | Perf report for the last 5.14031 seconds: | 20260508 10:45:35.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:35.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 10:45:35.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:45:35.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:35.554 | INFO | Uploaded 36GiB to GPU in 2.39034s (15.0606GiB/s) | 20260508 10:45:35.554 | INFO | Downloaded 34GiB to GPU in 2.48668s (13.6729GiB/s) | 20260508 10:45:40.690 | INFO | Analyzing chunk frames = [7492608,7493120) | 20260508 10:45:40.690 | INFO | MemInfo: { | 20260508 10:45:40.690 | INFO | "gpu" : " 17 GB", | 20260508 10:45:40.690 | INFO | "huge" : "", | 20260508 10:45:40.690 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:40.691 | INFO | "malloc" : " 14 GB", | 20260508 10:45:40.691 | INFO | "pinned" : " 29 GB" | 20260508 10:45:40.691 | INFO | } | 20260508 10:45:40.691 | INFO | Perf report for the last 5.13703 seconds: | 20260508 10:45:40.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:40.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 10:45:40.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:45:40.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:40.691 | INFO | Uploaded 36GiB to GPU in 2.43217s (14.8016GiB/s) | 20260508 10:45:40.691 | INFO | Downloaded 34GiB to GPU in 2.45037s (13.8755GiB/s) | 20260508 10:45:45.826 | INFO | Analyzing chunk frames = [7493120,7493632) | 20260508 10:45:45.826 | INFO | MemInfo: { | 20260508 10:45:45.826 | INFO | "gpu" : " 17 GB", | 20260508 10:45:45.826 | INFO | "huge" : "", | 20260508 10:45:45.826 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:45.827 | INFO | "malloc" : " 14 GB", | 20260508 10:45:45.827 | INFO | "pinned" : " 29 GB" | 20260508 10:45:45.827 | INFO | } | 20260508 10:45:45.827 | INFO | Perf report for the last 5.13606 seconds: | 20260508 10:45:45.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:45.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 10:45:45.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:45:45.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:45.827 | INFO | Uploaded 36GiB to GPU in 2.43429s (14.7887GiB/s) | 20260508 10:45:45.827 | INFO | Downloaded 34GiB to GPU in 2.43634s (13.9554GiB/s) | 20260508 10:45:50.964 | INFO | Analyzing chunk frames = [7493632,7494144) | 20260508 10:45:50.964 | INFO | MemInfo: { | 20260508 10:45:50.964 | INFO | "gpu" : " 17 GB", | 20260508 10:45:50.964 | INFO | "huge" : "", | 20260508 10:45:50.964 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:50.964 | INFO | "malloc" : " 14 GB", | 20260508 10:45:50.964 | INFO | "pinned" : " 29 GB" | 20260508 10:45:50.964 | INFO | } | 20260508 10:45:50.964 | INFO | Perf report for the last 5.1371 seconds: | 20260508 10:45:50.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:50.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:45:50.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 10:45:50.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:50.964 | INFO | Uploaded 36GiB to GPU in 2.29247s (15.7036GiB/s) | 20260508 10:45:50.964 | INFO | Downloaded 34GiB to GPU in 2.51459s (13.5211GiB/s) | 20260508 10:45:56.112 | INFO | Analyzing chunk frames = [7494144,7494656) | 20260508 10:45:56.112 | INFO | MemInfo: { | 20260508 10:45:56.112 | INFO | "gpu" : " 17 GB", | 20260508 10:45:56.112 | INFO | "huge" : "", | 20260508 10:45:56.112 | INFO | "huge_shared" : " 29 GB", | 20260508 10:45:56.112 | INFO | "malloc" : " 14 GB", | 20260508 10:45:56.112 | INFO | "pinned" : " 29 GB" | 20260508 10:45:56.112 | INFO | } | 20260508 10:45:56.112 | INFO | Perf report for the last 5.1486 seconds: | 20260508 10:45:56.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:45:56.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.112 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 10:45:56.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.112 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:45:56.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:45:56.113 | INFO | Uploaded 36GiB to GPU in 2.32655s (15.4736GiB/s) | 20260508 10:45:56.113 | INFO | Downloaded 34GiB to GPU in 2.43415s (13.9679GiB/s) | 20260508 10:46:01.256 | INFO | Analyzing chunk frames = [7494656,7495168) | 20260508 10:46:01.256 | INFO | MemInfo: { | 20260508 10:46:01.256 | INFO | "gpu" : " 17 GB", | 20260508 10:46:01.256 | INFO | "huge" : "", | 20260508 10:46:01.256 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:01.256 | INFO | "malloc" : " 14 GB", | 20260508 10:46:01.256 | INFO | "pinned" : " 29 GB" | 20260508 10:46:01.256 | INFO | } | 20260508 10:46:01.256 | INFO | Perf report for the last 5.14337 seconds: | 20260508 10:46:01.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:01.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 10:46:01.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:46:01.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:01.256 | INFO | Uploaded 36GiB to GPU in 2.37177s (15.1785GiB/s) | 20260508 10:46:01.256 | INFO | Downloaded 34GiB to GPU in 2.45794s (13.8327GiB/s) | 20260508 10:46:06.389 | INFO | Analyzing chunk frames = [7495168,7495680) | 20260508 10:46:06.389 | INFO | MemInfo: { | 20260508 10:46:06.389 | INFO | "gpu" : " 17 GB", | 20260508 10:46:06.389 | INFO | "huge" : "", | 20260508 10:46:06.389 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:06.389 | INFO | "malloc" : " 14 GB", | 20260508 10:46:06.389 | INFO | "pinned" : " 29 GB" | 20260508 10:46:06.389 | INFO | } | 20260508 10:46:06.389 | INFO | Perf report for the last 5.13302 seconds: | 20260508 10:46:06.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:06.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:46:06.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:46:06.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:06.389 | INFO | Uploaded 36GiB to GPU in 2.37032s (15.1878GiB/s) | 20260508 10:46:06.389 | INFO | Downloaded 34GiB to GPU in 2.46138s (13.8134GiB/s) | 20260508 10:46:06.389 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:46:06.389 | INFO | Total mem usage: 16.7 GiB | 20260508 10:46:06.389 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:46:06.389 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:46:06.389 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:46:06.389 | INFO | High Water Breakdown: | 20260508 10:46:06.389 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:46:06.389 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:46:06.389 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:46:06.389 | INFO | ------------------------------------------ | 20260508 10:46:06.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:46:06.389 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:46:06.389 | INFO | Total mem usage: 26.5 GiB | 20260508 10:46:06.389 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:46:06.389 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:46:06.389 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:46:06.389 | INFO | High Water Breakdown: | 20260508 10:46:06.389 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:46:06.389 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:46:06.389 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:46:06.389 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:46:06.389 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:46:06.389 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:46:06.389 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:46:06.389 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:46:06.389 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:46:06.389 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:46:06.389 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:46:06.389 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:46:06.389 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:46:06.389 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:46:06.389 | INFO | ------------------------------------------ | 20260508 10:46:06.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:46:06.389 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:46:06.389 | INFO | Total mem usage: 27.8 GiB | 20260508 10:46:06.389 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:46:06.389 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:46:06.389 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:46:06.389 | INFO | High Water Breakdown: | 20260508 10:46:06.389 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:46:06.389 | INFO | ------------------------------------------ | 20260508 10:46:06.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:46:06.390 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:46:06.390 | INFO | Total mem usage: 15.7 GiB | 20260508 10:46:06.390 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:46:06.390 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:46:06.390 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:46:06.390 | INFO | High Water Breakdown: | 20260508 10:46:06.390 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:46:06.390 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:46:06.390 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:46:06.390 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:46:06.390 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:46:06.390 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:46:06.390 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:46:06.390 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:46:06.390 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:46:06.390 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:46:06.390 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:46:06.390 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:46:06.390 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:46:06.390 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:46:06.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:46:06.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:46:06.390 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:46:06.390 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:46:06.390 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:46:06.390 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:46:06.390 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:46:06.390 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:46:06.390 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:46:06.390 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:46:06.390 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:46:06.390 | INFO | ------------------------------------------ | 20260508 10:46:06.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:46:06.390 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:46:06.390 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:46:06.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:46:06.390 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.38e+01 / 6.09e-01 | 20260508 10:46:06.390 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.15e-02 / 5.44e-03 | 20260508 10:46:06.390 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.36e+00, 5.92e-01 / 2.66e-01 | 20260508 10:46:06.390 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.46e+00, 2.09e-01 / 1.31e-01 | 20260508 10:46:06.390 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.55e+01, 3.84e+00 / 1.10e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.78e+00 / 1.30e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.06e-01 / 2.95e-01 | 20260508 10:46:06.390 | INFO | PerfReport: 3.02%, Metrics -- 9.86e-01 / 1.09e+00, 2.44e-01 / 9.09e-02 | 20260508 10:46:06.390 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.25e-01, 4.82e-02 / 2.72e-02 | 20260508 10:46:06.390 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:46:06.390 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:46:06.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:46:06.390 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 24.02%, Upload -- 2.52e+01 / 0.00e+00, 7.83e+00 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 2.06%, LockUpload -- 2.16e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 2.73%, LockDownload -- 2.87e+00 / 0.00e+00, 4.30e+00 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 15.30%, LockCompute -- 1.61e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 10:46:06.390 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 10:46:06.486 | INFO | Starting a round of Baz IO | 20260508 10:46:09.898 | INFO | Populating RTMetrics report | 20260508 10:46:09.898 | INFO | RTMetrics report populated | 20260508 10:46:09.898 | INFO | Opening tmp RTMetrics json file | 20260508 10:46:09.907 | INFO | Serializing RTMetrics to json | 20260508 10:46:09.908 | INFO | Serialization done, swapping tmp file now... | 20260508 10:46:09.908 | INFO | RTMetrics done | 20260508 10:46:09.969 | INFO | Finished a round of Baz IO: Wrote 1.453 GiB in 3.482 seconds | 20260508 10:46:09.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:46:09.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212828% / 0% / 6.75519e-05% | 20260508 10:46:09.969 | INFO | Baz IO overall: | 20260508 10:46:09.969 | INFO | BytesWritten : 3246648857412 | 20260508 10:46:09.969 | INFO | NumEvents : 1560171411459 | 20260508 10:46:11.522 | INFO | Analyzing chunk frames = [7495680,7496192) | 20260508 10:46:11.522 | INFO | MemInfo: { | 20260508 10:46:11.522 | INFO | "gpu" : " 17 GB", | 20260508 10:46:11.522 | INFO | "huge" : "", | 20260508 10:46:11.522 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:11.522 | INFO | "malloc" : " 14 GB", | 20260508 10:46:11.522 | INFO | "pinned" : " 29 GB" | 20260508 10:46:11.522 | INFO | } | 20260508 10:46:11.523 | INFO | Perf report for the last 5.13373 seconds: | 20260508 10:46:11.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:11.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:11.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:11.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:11.523 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:46:11.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:11.523 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:46:11.523 | INFO | PRE_HQ : 65.6%, 1.6, 54.4ms, 0 | 20260508 10:46:11.523 | INFO | BAZWRITER : 67.1%, 2.5, 86.6ms, 0 | 20260508 10:46:11.523 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:46:11.523 | INFO | Uploaded 36GiB to GPU in 2.60273s (13.8316GiB/s) | 20260508 10:46:11.523 | INFO | Downloaded 38GiB to GPU in 2.71645s (13.9889GiB/s) | 20260508 10:46:16.664 | INFO | Analyzing chunk frames = [7496192,7496704) | 20260508 10:46:16.664 | INFO | MemInfo: { | 20260508 10:46:16.664 | INFO | "gpu" : " 17 GB", | 20260508 10:46:16.664 | INFO | "huge" : "", | 20260508 10:46:16.664 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:16.664 | INFO | "malloc" : " 14 GB", | 20260508 10:46:16.664 | INFO | "pinned" : " 29 GB" | 20260508 10:46:16.664 | INFO | } | 20260508 10:46:16.664 | INFO | Perf report for the last 5.14163 seconds: | 20260508 10:46:16.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:16.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.664 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:46:16.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.664 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:46:16.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:16.665 | INFO | Uploaded 36GiB to GPU in 2.4511s (14.6873GiB/s) | 20260508 10:46:16.665 | INFO | Downloaded 34GiB to GPU in 2.46342s (13.8019GiB/s) | 20260508 10:46:21.805 | INFO | Analyzing chunk frames = [7496704,7497216) | 20260508 10:46:21.805 | INFO | MemInfo: { | 20260508 10:46:21.805 | INFO | "gpu" : " 17 GB", | 20260508 10:46:21.805 | INFO | "huge" : "", | 20260508 10:46:21.805 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:21.805 | INFO | "malloc" : " 14 GB", | 20260508 10:46:21.805 | INFO | "pinned" : " 29 GB" | 20260508 10:46:21.805 | INFO | } | 20260508 10:46:21.805 | INFO | Perf report for the last 5.14089 seconds: | 20260508 10:46:21.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:21.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.805 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 10:46:21.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.805 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:46:21.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:21.806 | INFO | Uploaded 36GiB to GPU in 2.46515s (14.6036GiB/s) | 20260508 10:46:21.806 | INFO | Downloaded 34GiB to GPU in 2.42319s (14.0311GiB/s) | 20260508 10:46:26.944 | INFO | Analyzing chunk frames = [7497216,7497728) | 20260508 10:46:26.944 | INFO | MemInfo: { | 20260508 10:46:26.944 | INFO | "gpu" : " 17 GB", | 20260508 10:46:26.944 | INFO | "huge" : "", | 20260508 10:46:26.944 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:26.944 | INFO | "malloc" : " 14 GB", | 20260508 10:46:26.944 | INFO | "pinned" : " 29 GB" | 20260508 10:46:26.944 | INFO | } | 20260508 10:46:26.944 | INFO | Perf report for the last 5.13917 seconds: | 20260508 10:46:26.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:26.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 10:46:26.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:46:26.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:26.944 | INFO | Uploaded 36GiB to GPU in 2.48308s (14.4981GiB/s) | 20260508 10:46:26.944 | INFO | Downloaded 34GiB to GPU in 2.44683s (13.8955GiB/s) | 20260508 10:46:32.089 | INFO | Analyzing chunk frames = [7497728,7498240) | 20260508 10:46:32.089 | INFO | MemInfo: { | 20260508 10:46:32.089 | INFO | "gpu" : " 17 GB", | 20260508 10:46:32.089 | INFO | "huge" : "", | 20260508 10:46:32.089 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:32.089 | INFO | "malloc" : " 14 GB", | 20260508 10:46:32.089 | INFO | "pinned" : " 29 GB" | 20260508 10:46:32.089 | INFO | } | 20260508 10:46:32.089 | INFO | Perf report for the last 5.14441 seconds: | 20260508 10:46:32.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:32.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:46:32.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:46:32.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:32.089 | INFO | Uploaded 36GiB to GPU in 2.29578s (15.681GiB/s) | 20260508 10:46:32.089 | INFO | Downloaded 34GiB to GPU in 2.4976s (13.6131GiB/s) | 20260508 10:46:37.221 | INFO | Analyzing chunk frames = [7498240,7498752) | 20260508 10:46:37.221 | INFO | MemInfo: { | 20260508 10:46:37.221 | INFO | "gpu" : " 17 GB", | 20260508 10:46:37.221 | INFO | "huge" : "", | 20260508 10:46:37.221 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:37.221 | INFO | "malloc" : " 14 GB", | 20260508 10:46:37.221 | INFO | "pinned" : " 29 GB" | 20260508 10:46:37.221 | INFO | } | 20260508 10:46:37.222 | INFO | Perf report for the last 5.13279 seconds: | 20260508 10:46:37.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:37.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 10:46:37.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 10:46:37.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:37.222 | INFO | Uploaded 36GiB to GPU in 2.37167s (15.1792GiB/s) | 20260508 10:46:37.222 | INFO | Downloaded 34GiB to GPU in 2.42983s (13.9927GiB/s) | 20260508 10:46:42.363 | INFO | Analyzing chunk frames = [7498752,7499264) | 20260508 10:46:42.363 | INFO | MemInfo: { | 20260508 10:46:42.363 | INFO | "gpu" : " 17 GB", | 20260508 10:46:42.363 | INFO | "huge" : "", | 20260508 10:46:42.363 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:42.363 | INFO | "malloc" : " 14 GB", | 20260508 10:46:42.363 | INFO | "pinned" : " 29 GB" | 20260508 10:46:42.363 | INFO | } | 20260508 10:46:42.363 | INFO | Perf report for the last 5.14142 seconds: | 20260508 10:46:42.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:42.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 10:46:42.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 10:46:42.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:42.363 | INFO | Uploaded 36GiB to GPU in 2.46102s (14.6281GiB/s) | 20260508 10:46:42.363 | INFO | Downloaded 34GiB to GPU in 2.4526s (13.8628GiB/s) | 20260508 10:46:47.503 | INFO | Analyzing chunk frames = [7499264,7499776) | 20260508 10:46:47.503 | INFO | MemInfo: { | 20260508 10:46:47.503 | INFO | "gpu" : " 17 GB", | 20260508 10:46:47.503 | INFO | "huge" : "", | 20260508 10:46:47.503 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:47.503 | INFO | "malloc" : " 14 GB", | 20260508 10:46:47.503 | INFO | "pinned" : " 29 GB" | 20260508 10:46:47.503 | INFO | } | 20260508 10:46:47.503 | INFO | Perf report for the last 5.14027 seconds: | 20260508 10:46:47.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:47.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.503 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 10:46:47.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.503 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:46:47.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:47.504 | INFO | Uploaded 36GiB to GPU in 2.42746s (14.8303GiB/s) | 20260508 10:46:47.504 | INFO | Downloaded 34GiB to GPU in 2.42238s (14.0358GiB/s) | 20260508 10:46:47.601 | INFO | Starting a round of Baz IO | 20260508 10:46:51.012 | INFO | Populating RTMetrics report | 20260508 10:46:51.012 | INFO | RTMetrics report populated | 20260508 10:46:51.012 | INFO | Opening tmp RTMetrics json file | 20260508 10:46:51.022 | INFO | Serializing RTMetrics to json | 20260508 10:46:51.023 | INFO | Serialization done, swapping tmp file now... | 20260508 10:46:51.024 | INFO | RTMetrics done | 20260508 10:46:51.083 | INFO | Finished a round of Baz IO: Wrote 1.452 GiB in 3.482 seconds | 20260508 10:46:51.083 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:46:51.083 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212288% / 0% / 7.54992e-05% | 20260508 10:46:51.084 | INFO | Baz IO overall: | 20260508 10:46:51.084 | INFO | BytesWritten : 3248207715679 | 20260508 10:46:51.084 | INFO | NumEvents : 1560906206632 | 20260508 10:46:52.636 | INFO | Analyzing chunk frames = [7499776,7500288) | 20260508 10:46:52.636 | INFO | MemInfo: { | 20260508 10:46:52.636 | INFO | "gpu" : " 17 GB", | 20260508 10:46:52.636 | INFO | "huge" : "", | 20260508 10:46:52.636 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:52.636 | INFO | "malloc" : " 14 GB", | 20260508 10:46:52.636 | INFO | "pinned" : " 29 GB" | 20260508 10:46:52.636 | INFO | } | 20260508 10:46:52.636 | INFO | Perf report for the last 5.13322 seconds: | 20260508 10:46:52.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:52.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:52.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:52.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:52.637 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 10:46:52.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:52.637 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:46:52.637 | INFO | PRE_HQ : 65.4%, 1.6, 54.2ms, 0 | 20260508 10:46:52.637 | INFO | BAZWRITER : 67.1%, 2.5, 86.5ms, 0 | 20260508 10:46:52.637 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:46:52.637 | INFO | Uploaded 36GiB to GPU in 2.58681s (13.9168GiB/s) | 20260508 10:46:52.637 | INFO | Downloaded 38GiB to GPU in 2.7225s (13.9578GiB/s) | 20260508 10:46:57.772 | INFO | Analyzing chunk frames = [7500288,7500800) | 20260508 10:46:57.772 | INFO | MemInfo: { | 20260508 10:46:57.772 | INFO | "gpu" : " 17 GB", | 20260508 10:46:57.772 | INFO | "huge" : "", | 20260508 10:46:57.773 | INFO | "huge_shared" : " 29 GB", | 20260508 10:46:57.773 | INFO | "malloc" : " 14 GB", | 20260508 10:46:57.773 | INFO | "pinned" : " 29 GB" | 20260508 10:46:57.773 | INFO | } | 20260508 10:46:57.773 | INFO | Perf report for the last 5.13615 seconds: | 20260508 10:46:57.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:46:57.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:46:57.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:46:57.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:46:57.773 | INFO | Uploaded 36GiB to GPU in 2.51783s (14.298GiB/s) | 20260508 10:46:57.773 | INFO | Downloaded 34GiB to GPU in 2.46597s (13.7877GiB/s) | 20260508 10:47:02.920 | INFO | Analyzing chunk frames = [7500800,7501312) | 20260508 10:47:02.920 | INFO | MemInfo: { | 20260508 10:47:02.920 | INFO | "gpu" : " 17 GB", | 20260508 10:47:02.920 | INFO | "huge" : "", | 20260508 10:47:02.920 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:02.920 | INFO | "malloc" : " 14 GB", | 20260508 10:47:02.920 | INFO | "pinned" : " 29 GB" | 20260508 10:47:02.920 | INFO | } | 20260508 10:47:02.920 | INFO | Perf report for the last 5.14746 seconds: | 20260508 10:47:02.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:02.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 10:47:02.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:47:02.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:02.920 | INFO | Uploaded 36GiB to GPU in 2.43718s (14.7712GiB/s) | 20260508 10:47:02.920 | INFO | Downloaded 34GiB to GPU in 2.44483s (13.9069GiB/s) | 20260508 10:47:08.065 | INFO | Analyzing chunk frames = [7501312,7501824) | 20260508 10:47:08.065 | INFO | MemInfo: { | 20260508 10:47:08.065 | INFO | "gpu" : " 17 GB", | 20260508 10:47:08.065 | INFO | "huge" : "", | 20260508 10:47:08.065 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:08.065 | INFO | "malloc" : " 14 GB", | 20260508 10:47:08.065 | INFO | "pinned" : " 29 GB" | 20260508 10:47:08.065 | INFO | } | 20260508 10:47:08.065 | INFO | Perf report for the last 5.14516 seconds: | 20260508 10:47:08.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:08.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.065 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 10:47:08.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.065 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:47:08.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:08.066 | INFO | Uploaded 36GiB to GPU in 2.4311s (14.8081GiB/s) | 20260508 10:47:08.066 | INFO | Downloaded 34GiB to GPU in 2.46063s (13.8176GiB/s) | 20260508 10:47:13.199 | INFO | Analyzing chunk frames = [7501824,7502336) | 20260508 10:47:13.199 | INFO | MemInfo: { | 20260508 10:47:13.199 | INFO | "gpu" : " 17 GB", | 20260508 10:47:13.199 | INFO | "huge" : "", | 20260508 10:47:13.199 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:13.199 | INFO | "malloc" : " 14 GB", | 20260508 10:47:13.199 | INFO | "pinned" : " 29 GB" | 20260508 10:47:13.199 | INFO | } | 20260508 10:47:13.199 | INFO | Perf report for the last 5.13348 seconds: | 20260508 10:47:13.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:13.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 10:47:13.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:47:13.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:13.199 | INFO | Uploaded 36GiB to GPU in 2.27098s (15.8522GiB/s) | 20260508 10:47:13.199 | INFO | Downloaded 34GiB to GPU in 2.51034s (13.544GiB/s) | 20260508 10:47:18.336 | INFO | Analyzing chunk frames = [7502336,7502848) | 20260508 10:47:18.336 | INFO | MemInfo: { | 20260508 10:47:18.336 | INFO | "gpu" : " 17 GB", | 20260508 10:47:18.336 | INFO | "huge" : "", | 20260508 10:47:18.336 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:18.336 | INFO | "malloc" : " 14 GB", | 20260508 10:47:18.336 | INFO | "pinned" : " 29 GB" | 20260508 10:47:18.336 | INFO | } | 20260508 10:47:18.336 | INFO | Perf report for the last 5.13729 seconds: | 20260508 10:47:18.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:18.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:47:18.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 10:47:18.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:18.336 | INFO | Uploaded 36GiB to GPU in 2.3666s (15.2117GiB/s) | 20260508 10:47:18.336 | INFO | Downloaded 34GiB to GPU in 2.4565s (13.8408GiB/s) | 20260508 10:47:23.474 | INFO | Analyzing chunk frames = [7502848,7503360) | 20260508 10:47:23.474 | INFO | MemInfo: { | 20260508 10:47:23.474 | INFO | "gpu" : " 17 GB", | 20260508 10:47:23.474 | INFO | "huge" : "", | 20260508 10:47:23.474 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:23.474 | INFO | "malloc" : " 14 GB", | 20260508 10:47:23.474 | INFO | "pinned" : " 29 GB" | 20260508 10:47:23.474 | INFO | } | 20260508 10:47:23.474 | INFO | Perf report for the last 5.13825 seconds: | 20260508 10:47:23.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:23.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.474 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:47:23.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.474 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:47:23.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:23.475 | INFO | Uploaded 36GiB to GPU in 2.42337s (14.8553GiB/s) | 20260508 10:47:23.475 | INFO | Downloaded 34GiB to GPU in 2.45974s (13.8226GiB/s) | 20260508 10:47:28.616 | INFO | Analyzing chunk frames = [7503360,7503872) | 20260508 10:47:28.616 | INFO | MemInfo: { | 20260508 10:47:28.616 | INFO | "gpu" : " 17 GB", | 20260508 10:47:28.616 | INFO | "huge" : "", | 20260508 10:47:28.616 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:28.616 | INFO | "malloc" : " 14 GB", | 20260508 10:47:28.616 | INFO | "pinned" : " 29 GB" | 20260508 10:47:28.616 | INFO | } | 20260508 10:47:28.616 | INFO | Perf report for the last 5.14166 seconds: | 20260508 10:47:28.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:28.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.616 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 10:47:28.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.616 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:47:28.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:28.617 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:47:28.617 | INFO | Total mem usage: 16.7 GiB | 20260508 10:47:28.617 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:47:28.617 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:47:28.617 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:47:28.617 | INFO | High Water Breakdown: | 20260508 10:47:28.617 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:47:28.617 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:47:28.617 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:47:28.617 | INFO | ------------------------------------------ | 20260508 10:47:28.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:47:28.617 | INFO | Uploaded 36GiB to GPU in 2.38248s (15.1103GiB/s) | 20260508 10:47:28.617 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:47:28.617 | INFO | Downloaded 34GiB to GPU in 2.44361s (13.9139GiB/s) | 20260508 10:47:28.617 | INFO | Total mem usage: 26.5 GiB | 20260508 10:47:28.617 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:47:28.617 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:47:28.617 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:47:28.617 | INFO | High Water Breakdown: | 20260508 10:47:28.617 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:47:28.617 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:47:28.617 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:47:28.617 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:47:28.617 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:47:28.617 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:47:28.617 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:47:28.617 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:47:28.617 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:47:28.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:47:28.617 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:47:28.617 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:47:28.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:47:28.617 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:47:28.617 | INFO | ------------------------------------------ | 20260508 10:47:28.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:47:28.617 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:47:28.617 | INFO | Total mem usage: 27.8 GiB | 20260508 10:47:28.617 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:47:28.617 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 10:47:28.617 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 10:47:28.617 | INFO | High Water Breakdown: | 20260508 10:47:28.617 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 10:47:28.617 | INFO | ------------------------------------------ | 20260508 10:47:28.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:47:28.617 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:47:28.617 | INFO | Total mem usage: 15.7 GiB | 20260508 10:47:28.617 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:47:28.617 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:47:28.617 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:47:28.617 | INFO | High Water Breakdown: | 20260508 10:47:28.617 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:47:28.617 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:47:28.617 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:47:28.617 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:47:28.617 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:47:28.617 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:47:28.617 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:47:28.617 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:47:28.617 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:47:28.617 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:47:28.617 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:47:28.617 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:47:28.617 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:47:28.617 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:47:28.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:47:28.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:47:28.617 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:47:28.617 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:47:28.617 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:47:28.617 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:47:28.617 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:47:28.617 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:47:28.617 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:47:28.617 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:47:28.617 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:47:28.617 | INFO | ------------------------------------------ | 20260508 10:47:28.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:47:28.618 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:47:28.618 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:47:28.618 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:47:28.618 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 4.83e-01 | 20260508 10:47:28.618 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.05e-01, 1.14e-02 / 2.77e-03 | 20260508 10:47:28.618 | INFO | PerfReport: 12.21%, Baseline -- 3.98e+00 / 4.38e+00, 5.97e-01 / 1.90e-01 | 20260508 10:47:28.618 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.46e+00, 2.11e-01 / 1.08e-01 | 20260508 10:47:28.618 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.54e+01, 3.76e+00 / 1.04e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.75e+00 / 1.21e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.35e+00 / 2.51e+00, 4.06e-01 / 2.71e-01 | 20260508 10:47:28.618 | INFO | PerfReport: 3.03%, Metrics -- 9.88e-01 / 1.15e+00, 2.42e-01 / 1.29e-01 | 20260508 10:47:28.618 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.00e-01 / 3.28e-01, 4.82e-02 / 2.69e-02 | 20260508 10:47:28.618 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:47:28.618 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:47:28.618 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:47:28.618 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.43e+01 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 24.13%, Upload -- 2.54e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 24.65%, Download -- 2.59e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 31.03%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 2.09%, LockUpload -- 2.19e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 2.70%, LockDownload -- 2.84e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 15.38%, LockCompute -- 1.62e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260508 10:47:28.618 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 8.95e-03 / 0.00e+00 | 20260508 10:47:28.715 | INFO | Starting a round of Baz IO | 20260508 10:47:32.127 | INFO | Populating RTMetrics report | 20260508 10:47:32.127 | INFO | RTMetrics report populated | 20260508 10:47:32.128 | INFO | Opening tmp RTMetrics json file | 20260508 10:47:32.137 | INFO | Serializing RTMetrics to json | 20260508 10:47:32.138 | INFO | Serialization done, swapping tmp file now... | 20260508 10:47:32.138 | INFO | RTMetrics done | 20260508 10:47:32.196 | INFO | Finished a round of Baz IO: Wrote 1.452 GiB in 3.481 seconds | 20260508 10:47:32.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.17% / 0.00% / 59.83% | 20260508 10:47:32.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.211819% / 0% / 6.75519e-05% | 20260508 10:47:32.196 | INFO | Baz IO overall: | 20260508 10:47:32.196 | INFO | BytesWritten : 3249766404501 | 20260508 10:47:32.196 | INFO | NumEvents : 1561640972632 | 20260508 10:47:33.759 | INFO | Analyzing chunk frames = [7503872,7504384) | 20260508 10:47:33.759 | INFO | MemInfo: { | 20260508 10:47:33.759 | INFO | "gpu" : " 17 GB", | 20260508 10:47:33.759 | INFO | "huge" : "", | 20260508 10:47:33.759 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:33.759 | INFO | "malloc" : " 14 GB", | 20260508 10:47:33.759 | INFO | "pinned" : " 29 GB" | 20260508 10:47:33.759 | INFO | } | 20260508 10:47:33.759 | INFO | Perf report for the last 5.14294 seconds: | 20260508 10:47:33.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:33.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:33.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:33.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:33.759 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:47:33.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:33.759 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 10:47:33.759 | INFO | PRE_HQ : 65.4%, 1.6, 54.5ms, 0 | 20260508 10:47:33.759 | INFO | BAZWRITER : 67.0%, 2.5, 86.4ms, 0 | 20260508 10:47:33.759 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:47:33.759 | INFO | Uploaded 36GiB to GPU in 2.59676s (13.8635GiB/s) | 20260508 10:47:33.759 | INFO | Downloaded 38GiB to GPU in 2.72316s (13.9544GiB/s) | 20260508 10:47:38.900 | INFO | Analyzing chunk frames = [7504384,7504896) | 20260508 10:47:38.900 | INFO | MemInfo: { | 20260508 10:47:38.900 | INFO | "gpu" : " 17 GB", | 20260508 10:47:38.900 | INFO | "huge" : "", | 20260508 10:47:38.900 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:38.900 | INFO | "malloc" : " 14 GB", | 20260508 10:47:38.900 | INFO | "pinned" : " 29 GB" | 20260508 10:47:38.900 | INFO | } | 20260508 10:47:38.900 | INFO | Perf report for the last 5.14076 seconds: | 20260508 10:47:38.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:38.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 10:47:38.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:47:38.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:38.900 | INFO | Uploaded 36GiB to GPU in 2.43375s (14.792GiB/s) | 20260508 10:47:38.900 | INFO | Downloaded 34GiB to GPU in 2.4736s (13.7451GiB/s) | 20260508 10:47:44.033 | INFO | Analyzing chunk frames = [7504896,7505408) | 20260508 10:47:44.033 | INFO | MemInfo: { | 20260508 10:47:44.033 | INFO | "gpu" : " 17 GB", | 20260508 10:47:44.033 | INFO | "huge" : "", | 20260508 10:47:44.033 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:44.033 | INFO | "malloc" : " 14 GB", | 20260508 10:47:44.033 | INFO | "pinned" : " 29 GB" | 20260508 10:47:44.033 | INFO | } | 20260508 10:47:44.033 | INFO | Perf report for the last 5.13322 seconds: | 20260508 10:47:44.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:44.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 10:47:44.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:47:44.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:44.033 | INFO | Uploaded 36GiB to GPU in 2.46525s (14.603GiB/s) | 20260508 10:47:44.033 | INFO | Downloaded 34GiB to GPU in 2.46091s (13.816GiB/s) | 20260508 10:47:49.174 | INFO | Analyzing chunk frames = [7505408,7505920) | 20260508 10:47:49.174 | INFO | MemInfo: { | 20260508 10:47:49.174 | INFO | "gpu" : " 17 GB", | 20260508 10:47:49.174 | INFO | "huge" : "", | 20260508 10:47:49.174 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:49.174 | INFO | "malloc" : " 14 GB", | 20260508 10:47:49.174 | INFO | "pinned" : " 29 GB" | 20260508 10:47:49.175 | INFO | } | 20260508 10:47:49.175 | INFO | Perf report for the last 5.14161 seconds: | 20260508 10:47:49.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:49.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 10:47:49.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 10:47:49.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:49.175 | INFO | Uploaded 36GiB to GPU in 2.43089s (14.8094GiB/s) | 20260508 10:47:49.175 | INFO | Downloaded 34GiB to GPU in 2.41491s (14.0792GiB/s) | 20260508 10:47:54.310 | INFO | Analyzing chunk frames = [7505920,7506432) | 20260508 10:47:54.311 | INFO | MemInfo: { | 20260508 10:47:54.311 | INFO | "gpu" : " 17 GB", | 20260508 10:47:54.311 | INFO | "huge" : "", | 20260508 10:47:54.311 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:54.311 | INFO | "malloc" : " 14 GB", | 20260508 10:47:54.311 | INFO | "pinned" : " 29 GB" | 20260508 10:47:54.311 | INFO | } | 20260508 10:47:54.311 | INFO | Perf report for the last 5.13609 seconds: | 20260508 10:47:54.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:54.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 10:47:54.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 10:47:54.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:54.311 | INFO | Uploaded 36GiB to GPU in 2.45512s (14.6632GiB/s) | 20260508 10:47:54.311 | INFO | Downloaded 34GiB to GPU in 2.54252s (13.3725GiB/s) | 20260508 10:47:59.453 | INFO | Analyzing chunk frames = [7506432,7506944) | 20260508 10:47:59.453 | INFO | MemInfo: { | 20260508 10:47:59.453 | INFO | "gpu" : " 17 GB", | 20260508 10:47:59.453 | INFO | "huge" : "", | 20260508 10:47:59.453 | INFO | "huge_shared" : " 29 GB", | 20260508 10:47:59.453 | INFO | "malloc" : " 14 GB", | 20260508 10:47:59.453 | INFO | "pinned" : " 29 GB" | 20260508 10:47:59.453 | INFO | } | 20260508 10:47:59.454 | INFO | Perf report for the last 5.14284 seconds: | 20260508 10:47:59.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:47:59.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:47:59.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:47:59.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:47:59.454 | INFO | Uploaded 36GiB to GPU in 2.36152s (15.2444GiB/s) | 20260508 10:47:59.454 | INFO | Downloaded 34GiB to GPU in 2.43742s (13.9491GiB/s) | 20260508 10:48:04.596 | INFO | Analyzing chunk frames = [7506944,7507456) | 20260508 10:48:04.596 | INFO | MemInfo: { | 20260508 10:48:04.596 | INFO | "gpu" : " 17 GB", | 20260508 10:48:04.596 | INFO | "huge" : "", | 20260508 10:48:04.596 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:04.596 | INFO | "malloc" : " 14 GB", | 20260508 10:48:04.596 | INFO | "pinned" : " 29 GB" | 20260508 10:48:04.596 | INFO | } | 20260508 10:48:04.596 | INFO | Perf report for the last 5.14238 seconds: | 20260508 10:48:04.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:04.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 10:48:04.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:48:04.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:04.596 | INFO | Uploaded 36GiB to GPU in 2.42535s (14.8432GiB/s) | 20260508 10:48:04.596 | INFO | Downloaded 34GiB to GPU in 2.45258s (13.8629GiB/s) | 20260508 10:48:09.728 | INFO | Analyzing chunk frames = [7507456,7507968) | 20260508 10:48:09.728 | INFO | MemInfo: { | 20260508 10:48:09.728 | INFO | "gpu" : " 17 GB", | 20260508 10:48:09.728 | INFO | "huge" : "", | 20260508 10:48:09.728 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:09.728 | INFO | "malloc" : " 14 GB", | 20260508 10:48:09.728 | INFO | "pinned" : " 29 GB" | 20260508 10:48:09.728 | INFO | } | 20260508 10:48:09.728 | INFO | Perf report for the last 5.13246 seconds: | 20260508 10:48:09.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:09.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.728 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 10:48:09.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.729 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:48:09.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:09.729 | INFO | Uploaded 36GiB to GPU in 2.44919s (14.6987GiB/s) | 20260508 10:48:09.729 | INFO | Downloaded 34GiB to GPU in 2.45378s (13.8562GiB/s) | 20260508 10:48:09.826 | INFO | Starting a round of Baz IO | 20260508 10:48:13.252 | INFO | Populating RTMetrics report | 20260508 10:48:13.252 | INFO | RTMetrics report populated | 20260508 10:48:13.252 | INFO | Opening tmp RTMetrics json file | 20260508 10:48:13.258 | INFO | Serializing RTMetrics to json | 20260508 10:48:13.259 | INFO | Serialization done, swapping tmp file now... | 20260508 10:48:13.261 | INFO | RTMetrics done | 20260508 10:48:13.320 | INFO | Finished a round of Baz IO: Wrote 1.452 GiB in 3.493 seconds | 20260508 10:48:13.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.83% | 20260508 10:48:13.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.211517% / 3.97364e-06% / 9.13938e-05% | 20260508 10:48:13.320 | INFO | Baz IO overall: | 20260508 10:48:13.320 | INFO | BytesWritten : 3251325149867 | 20260508 10:48:13.320 | INFO | NumEvents : 1562375923440 | 20260508 10:48:14.870 | INFO | Analyzing chunk frames = [7507968,7508480) | 20260508 10:48:14.870 | INFO | MemInfo: { | 20260508 10:48:14.870 | INFO | "gpu" : " 17 GB", | 20260508 10:48:14.870 | INFO | "huge" : "", | 20260508 10:48:14.870 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:14.870 | INFO | "malloc" : " 14 GB", | 20260508 10:48:14.870 | INFO | "pinned" : " 29 GB" | 20260508 10:48:14.870 | INFO | } | 20260508 10:48:14.870 | INFO | Perf report for the last 5.14193 seconds: | 20260508 10:48:14.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:14.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:14.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:14.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:14.870 | INFO | ANALYSIS : 67.7%, 3.0, 105.9ms, 0 | 20260508 10:48:14.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:14.870 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:48:14.870 | INFO | PRE_HQ : 65.1%, 1.6, 54.3ms, 0 | 20260508 10:48:14.870 | INFO | BAZWRITER : 67.2%, 2.5, 86.0ms, 0 | 20260508 10:48:14.870 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:48:14.871 | INFO | Uploaded 36GiB to GPU in 2.60821s (13.8026GiB/s) | 20260508 10:48:14.871 | INFO | Downloaded 38GiB to GPU in 2.69837s (14.0826GiB/s) | 20260508 10:48:20.011 | INFO | Analyzing chunk frames = [7508480,7508992) | 20260508 10:48:20.011 | INFO | MemInfo: { | 20260508 10:48:20.011 | INFO | "gpu" : " 17 GB", | 20260508 10:48:20.011 | INFO | "huge" : "", | 20260508 10:48:20.011 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:20.011 | INFO | "malloc" : " 14 GB", | 20260508 10:48:20.011 | INFO | "pinned" : " 29 GB" | 20260508 10:48:20.011 | INFO | } | 20260508 10:48:20.011 | INFO | Perf report for the last 5.14048 seconds: | 20260508 10:48:20.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:20.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 10:48:20.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:48:20.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:20.011 | INFO | Uploaded 36GiB to GPU in 2.48646s (14.4784GiB/s) | 20260508 10:48:20.011 | INFO | Downloaded 34GiB to GPU in 2.45775s (13.8338GiB/s) | 20260508 10:48:25.146 | INFO | Analyzing chunk frames = [7508992,7509504) | 20260508 10:48:25.146 | INFO | MemInfo: { | 20260508 10:48:25.146 | INFO | "gpu" : " 17 GB", | 20260508 10:48:25.146 | INFO | "huge" : "", | 20260508 10:48:25.146 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:25.146 | INFO | "malloc" : " 14 GB", | 20260508 10:48:25.146 | INFO | "pinned" : " 29 GB" | 20260508 10:48:25.146 | INFO | } | 20260508 10:48:25.146 | INFO | Perf report for the last 5.1356 seconds: | 20260508 10:48:25.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:25.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.147 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:48:25.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.147 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:48:25.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:25.147 | INFO | Uploaded 36GiB to GPU in 2.47314s (14.5564GiB/s) | 20260508 10:48:25.147 | INFO | Downloaded 34GiB to GPU in 2.4218s (14.0391GiB/s) | 20260508 10:48:30.281 | INFO | Analyzing chunk frames = [7509504,7510016) | 20260508 10:48:30.281 | INFO | MemInfo: { | 20260508 10:48:30.281 | INFO | "gpu" : " 17 GB", | 20260508 10:48:30.281 | INFO | "huge" : "", | 20260508 10:48:30.282 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:30.282 | INFO | "malloc" : " 14 GB", | 20260508 10:48:30.282 | INFO | "pinned" : " 29 GB" | 20260508 10:48:30.282 | INFO | } | 20260508 10:48:30.282 | INFO | Perf report for the last 5.13514 seconds: | 20260508 10:48:30.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:30.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 10:48:30.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:48:30.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:30.282 | INFO | Uploaded 36GiB to GPU in 2.49259s (14.4428GiB/s) | 20260508 10:48:30.282 | INFO | Downloaded 34GiB to GPU in 2.44841s (13.8866GiB/s) | 20260508 10:48:35.426 | INFO | Analyzing chunk frames = [7510016,7510528) | 20260508 10:48:35.426 | INFO | MemInfo: { | 20260508 10:48:35.426 | INFO | "gpu" : " 17 GB", | 20260508 10:48:35.426 | INFO | "huge" : "", | 20260508 10:48:35.426 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:35.426 | INFO | "malloc" : " 14 GB", | 20260508 10:48:35.426 | INFO | "pinned" : " 29 GB" | 20260508 10:48:35.426 | INFO | } | 20260508 10:48:35.426 | INFO | Perf report for the last 5.14474 seconds: | 20260508 10:48:35.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:35.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.426 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 10:48:35.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.426 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:48:35.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:35.427 | INFO | Uploaded 36GiB to GPU in 2.41098s (14.9317GiB/s) | 20260508 10:48:35.427 | INFO | Downloaded 34GiB to GPU in 2.50668s (13.5638GiB/s) | 20260508 10:48:40.573 | INFO | Analyzing chunk frames = [7510528,7511040) | 20260508 10:48:40.573 | INFO | MemInfo: { | 20260508 10:48:40.573 | INFO | "gpu" : " 17 GB", | 20260508 10:48:40.573 | INFO | "huge" : "", | 20260508 10:48:40.573 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:40.573 | INFO | "malloc" : " 14 GB", | 20260508 10:48:40.573 | INFO | "pinned" : " 29 GB" | 20260508 10:48:40.573 | INFO | } | 20260508 10:48:40.573 | INFO | Perf report for the last 5.14681 seconds: | 20260508 10:48:40.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:40.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:48:40.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:48:40.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:40.573 | INFO | Uploaded 36GiB to GPU in 2.38399s (15.1007GiB/s) | 20260508 10:48:40.573 | INFO | Downloaded 34GiB to GPU in 2.44205s (13.9227GiB/s) | 20260508 10:48:45.711 | INFO | Analyzing chunk frames = [7511040,7511552) | 20260508 10:48:45.711 | INFO | MemInfo: { | 20260508 10:48:45.711 | INFO | "gpu" : " 17 GB", | 20260508 10:48:45.711 | INFO | "huge" : "", | 20260508 10:48:45.711 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:45.711 | INFO | "malloc" : " 14 GB", | 20260508 10:48:45.711 | INFO | "pinned" : " 29 GB" | 20260508 10:48:45.711 | INFO | } | 20260508 10:48:45.711 | INFO | Perf report for the last 5.13808 seconds: | 20260508 10:48:45.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:45.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.711 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:48:45.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.711 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 10:48:45.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:45.712 | INFO | Uploaded 36GiB to GPU in 2.44879s (14.7012GiB/s) | 20260508 10:48:45.712 | INFO | Downloaded 34GiB to GPU in 2.45231s (13.8645GiB/s) | 20260508 10:48:50.847 | INFO | Analyzing chunk frames = [7511552,7512064) | 20260508 10:48:50.847 | INFO | MemInfo: { | 20260508 10:48:50.847 | INFO | "gpu" : " 17 GB", | 20260508 10:48:50.847 | INFO | "huge" : "", | 20260508 10:48:50.847 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:50.847 | INFO | "malloc" : " 14 GB", | 20260508 10:48:50.847 | INFO | "pinned" : " 29 GB" | 20260508 10:48:50.847 | INFO | } | 20260508 10:48:50.847 | INFO | Perf report for the last 5.13548 seconds: | 20260508 10:48:50.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:50.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 10:48:50.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:48:50.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:50.847 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:48:50.847 | INFO | Total mem usage: 16.7 GiB | 20260508 10:48:50.847 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:48:50.847 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:48:50.847 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:48:50.847 | INFO | High Water Breakdown: | 20260508 10:48:50.847 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:48:50.847 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:48:50.847 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:48:50.847 | INFO | ------------------------------------------ | 20260508 10:48:50.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:48:50.847 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:48:50.847 | INFO | Total mem usage: 26.5 GiB | 20260508 10:48:50.847 | INFO | Uploaded 36GiB to GPU in 2.4272s (14.8319GiB/s) | 20260508 10:48:50.847 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:48:50.847 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:48:50.847 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:48:50.847 | INFO | Downloaded 34GiB to GPU in 2.44244s (13.9205GiB/s) | 20260508 10:48:50.847 | INFO | High Water Breakdown: | 20260508 10:48:50.847 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:48:50.847 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:48:50.847 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:48:50.847 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:48:50.847 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:48:50.847 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:48:50.847 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:48:50.847 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:48:50.847 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:48:50.847 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:48:50.847 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:48:50.847 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:48:50.847 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:48:50.847 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:48:50.847 | INFO | ------------------------------------------ | 20260508 10:48:50.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:48:50.847 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:48:50.847 | INFO | Total mem usage: 27.8 GiB | 20260508 10:48:50.847 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:48:50.847 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:48:50.847 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:48:50.847 | INFO | High Water Breakdown: | 20260508 10:48:50.847 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:48:50.847 | INFO | ------------------------------------------ | 20260508 10:48:50.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:48:50.847 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:48:50.847 | INFO | Total mem usage: 15.7 GiB | 20260508 10:48:50.847 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:48:50.847 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:48:50.847 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:48:50.847 | INFO | High Water Breakdown: | 20260508 10:48:50.847 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:48:50.847 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:48:50.847 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:48:50.847 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:48:50.847 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:48:50.847 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:48:50.847 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:48:50.847 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:48:50.847 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:48:50.847 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:48:50.847 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:48:50.847 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:48:50.847 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:48:50.847 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:48:50.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:48:50.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:48:50.847 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:48:50.847 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:48:50.847 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:48:50.847 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:48:50.847 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:48:50.847 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:48:50.847 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:48:50.847 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:48:50.847 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:48:50.847 | INFO | ------------------------------------------ | 20260508 10:48:50.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:48:50.848 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:48:50.849 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:48:50.849 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:48:50.849 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.60e-01 | 20260508 10:48:50.849 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.08e-02 / 4.94e-03 | 20260508 10:48:50.849 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.47e+00, 5.89e-01 / 1.99e-01 | 20260508 10:48:50.849 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.14e-01 | 20260508 10:48:50.849 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.48e+01, 3.48e+00 / 9.20e-01 | 20260508 10:48:50.849 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.79e+00 / 1.37e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.05e-01 / 2.31e-01 | 20260508 10:48:50.849 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.16e+00, 2.44e-01 / 1.21e-01 | 20260508 10:48:50.849 | INFO | PerfReport: 0.92%, PulseSerializing -- 2.99e-01 / 3.32e-01, 4.82e-02 / 2.06e-02 | 20260508 10:48:50.849 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:48:50.849 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:48:50.849 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:48:50.849 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 24.34%, Upload -- 2.56e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 24.64%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 2.15%, LockUpload -- 2.26e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 2.56%, LockDownload -- 2.70e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 15.25%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:48:50.849 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260508 10:48:50.944 | INFO | Starting a round of Baz IO | 20260508 10:48:54.366 | INFO | Populating RTMetrics report | 20260508 10:48:54.366 | INFO | RTMetrics report populated | 20260508 10:48:54.366 | INFO | Opening tmp RTMetrics json file | 20260508 10:48:54.379 | INFO | Serializing RTMetrics to json | 20260508 10:48:54.380 | INFO | Serialization done, swapping tmp file now... | 20260508 10:48:54.381 | INFO | RTMetrics done | 20260508 10:48:54.431 | INFO | Finished a round of Baz IO: Wrote 1.451 GiB in 3.487 seconds | 20260508 10:48:54.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:48:54.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.211108% / 0% / 5.96046e-05% | 20260508 10:48:54.431 | INFO | Baz IO overall: | 20260508 10:48:54.431 | INFO | BytesWritten : 3252883176527 | 20260508 10:48:54.431 | INFO | NumEvents : 1563110200065 | 20260508 10:48:55.986 | INFO | Analyzing chunk frames = [7512064,7512576) | 20260508 10:48:55.986 | INFO | MemInfo: { | 20260508 10:48:55.986 | INFO | "gpu" : " 17 GB", | 20260508 10:48:55.986 | INFO | "huge" : "", | 20260508 10:48:55.986 | INFO | "huge_shared" : " 29 GB", | 20260508 10:48:55.986 | INFO | "malloc" : " 14 GB", | 20260508 10:48:55.986 | INFO | "pinned" : " 29 GB" | 20260508 10:48:55.986 | INFO | } | 20260508 10:48:55.986 | INFO | Perf report for the last 5.13922 seconds: | 20260508 10:48:55.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:48:55.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:55.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:55.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:55.986 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:48:55.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:48:55.986 | INFO | RT_METRICS : 38.8%, 1.0, 20.4ms, 0 | 20260508 10:48:55.986 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 10:48:55.986 | INFO | BAZWRITER : 67.1%, 2.5, 86.2ms, 0 | 20260508 10:48:55.986 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:48:55.986 | INFO | Uploaded 36GiB to GPU in 2.57472s (13.9821GiB/s) | 20260508 10:48:55.986 | INFO | Downloaded 38GiB to GPU in 2.70379s (14.0544GiB/s) | 20260508 10:49:01.125 | INFO | Analyzing chunk frames = [7512576,7513088) | 20260508 10:49:01.125 | INFO | MemInfo: { | 20260508 10:49:01.125 | INFO | "gpu" : " 17 GB", | 20260508 10:49:01.125 | INFO | "huge" : "", | 20260508 10:49:01.125 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:01.125 | INFO | "malloc" : " 14 GB", | 20260508 10:49:01.125 | INFO | "pinned" : " 29 GB" | 20260508 10:49:01.125 | INFO | } | 20260508 10:49:01.125 | INFO | Perf report for the last 5.13903 seconds: | 20260508 10:49:01.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:01.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:49:01.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 10:49:01.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:01.125 | INFO | Uploaded 36GiB to GPU in 2.46114s (14.6274GiB/s) | 20260508 10:49:01.126 | INFO | Downloaded 34GiB to GPU in 2.47336s (13.7465GiB/s) | 20260508 10:49:06.265 | INFO | Analyzing chunk frames = [7513088,7513600) | 20260508 10:49:06.265 | INFO | MemInfo: { | 20260508 10:49:06.265 | INFO | "gpu" : " 17 GB", | 20260508 10:49:06.265 | INFO | "huge" : "", | 20260508 10:49:06.265 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:06.265 | INFO | "malloc" : " 14 GB", | 20260508 10:49:06.265 | INFO | "pinned" : " 29 GB" | 20260508 10:49:06.266 | INFO | } | 20260508 10:49:06.266 | INFO | Perf report for the last 5.14058 seconds: | 20260508 10:49:06.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:06.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:49:06.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:49:06.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:06.266 | INFO | Uploaded 36GiB to GPU in 2.46467s (14.6064GiB/s) | 20260508 10:49:06.266 | INFO | Downloaded 34GiB to GPU in 2.42257s (14.0347GiB/s) | 20260508 10:49:11.398 | INFO | Analyzing chunk frames = [7513600,7514112) | 20260508 10:49:11.398 | INFO | MemInfo: { | 20260508 10:49:11.398 | INFO | "gpu" : " 17 GB", | 20260508 10:49:11.398 | INFO | "huge" : "", | 20260508 10:49:11.398 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:11.398 | INFO | "malloc" : " 14 GB", | 20260508 10:49:11.398 | INFO | "pinned" : " 29 GB" | 20260508 10:49:11.398 | INFO | } | 20260508 10:49:11.398 | INFO | Perf report for the last 5.13276 seconds: | 20260508 10:49:11.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:11.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.398 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 10:49:11.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.398 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:49:11.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:11.399 | INFO | Uploaded 36GiB to GPU in 2.47141s (14.5666GiB/s) | 20260508 10:49:11.399 | INFO | Downloaded 34GiB to GPU in 2.44629s (13.8986GiB/s) | 20260508 10:49:16.541 | INFO | Analyzing chunk frames = [7514112,7514624) | 20260508 10:49:16.541 | INFO | MemInfo: { | 20260508 10:49:16.541 | INFO | "gpu" : " 17 GB", | 20260508 10:49:16.541 | INFO | "huge" : "", | 20260508 10:49:16.541 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:16.541 | INFO | "malloc" : " 14 GB", | 20260508 10:49:16.541 | INFO | "pinned" : " 29 GB" | 20260508 10:49:16.541 | INFO | } | 20260508 10:49:16.541 | INFO | Perf report for the last 5.14259 seconds: | 20260508 10:49:16.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:16.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 10:49:16.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:49:16.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:16.541 | INFO | Uploaded 36GiB to GPU in 2.4375s (14.7692GiB/s) | 20260508 10:49:16.541 | INFO | Downloaded 34GiB to GPU in 2.5278s (13.4504GiB/s) | 20260508 10:49:21.683 | INFO | Analyzing chunk frames = [7514624,7515136) | 20260508 10:49:21.683 | INFO | MemInfo: { | 20260508 10:49:21.683 | INFO | "gpu" : " 17 GB", | 20260508 10:49:21.683 | INFO | "huge" : "", | 20260508 10:49:21.683 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:21.683 | INFO | "malloc" : " 14 GB", | 20260508 10:49:21.683 | INFO | "pinned" : " 29 GB" | 20260508 10:49:21.683 | INFO | } | 20260508 10:49:21.683 | INFO | Perf report for the last 5.14174 seconds: | 20260508 10:49:21.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:21.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 10:49:21.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:49:21.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:21.683 | INFO | Uploaded 36GiB to GPU in 2.34272s (15.3667GiB/s) | 20260508 10:49:21.683 | INFO | Downloaded 34GiB to GPU in 2.42294s (14.0325GiB/s) | 20260508 10:49:26.823 | INFO | Analyzing chunk frames = [7515136,7515648) | 20260508 10:49:26.823 | INFO | MemInfo: { | 20260508 10:49:26.823 | INFO | "gpu" : " 17 GB", | 20260508 10:49:26.823 | INFO | "huge" : "", | 20260508 10:49:26.824 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:26.824 | INFO | "malloc" : " 14 GB", | 20260508 10:49:26.824 | INFO | "pinned" : " 29 GB" | 20260508 10:49:26.824 | INFO | } | 20260508 10:49:26.824 | INFO | Perf report for the last 5.1408 seconds: | 20260508 10:49:26.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:26.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 10:49:26.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:49:26.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:26.824 | INFO | Uploaded 36GiB to GPU in 2.46388s (14.6111GiB/s) | 20260508 10:49:26.824 | INFO | Downloaded 34GiB to GPU in 2.43417s (13.9678GiB/s) | 20260508 10:49:31.961 | INFO | Analyzing chunk frames = [7515648,7516160) | 20260508 10:49:31.961 | INFO | MemInfo: { | 20260508 10:49:31.961 | INFO | "gpu" : " 17 GB", | 20260508 10:49:31.961 | INFO | "huge" : "", | 20260508 10:49:31.961 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:31.961 | INFO | "malloc" : " 14 GB", | 20260508 10:49:31.961 | INFO | "pinned" : " 29 GB" | 20260508 10:49:31.961 | INFO | } | 20260508 10:49:31.961 | INFO | Perf report for the last 5.13716 seconds: | 20260508 10:49:31.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:31.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 10:49:31.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 10:49:31.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:31.961 | INFO | Uploaded 36GiB to GPU in 2.51228s (14.3296GiB/s) | 20260508 10:49:31.961 | INFO | Downloaded 34GiB to GPU in 2.45818s (13.8313GiB/s) | 20260508 10:49:32.058 | INFO | Starting a round of Baz IO | 20260508 10:49:35.477 | INFO | Populating RTMetrics report | 20260508 10:49:35.477 | INFO | RTMetrics report populated | 20260508 10:49:35.477 | INFO | Opening tmp RTMetrics json file | 20260508 10:49:35.487 | INFO | Serializing RTMetrics to json | 20260508 10:49:35.488 | INFO | Serialization done, swapping tmp file now... | 20260508 10:49:35.489 | INFO | RTMetrics done | 20260508 10:49:35.545 | INFO | Finished a round of Baz IO: Wrote 1.452 GiB in 3.487 seconds | 20260508 10:49:35.545 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:49:35.545 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.211092% / 0% / 6.35783e-05% | 20260508 10:49:35.545 | INFO | Baz IO overall: | 20260508 10:49:35.545 | INFO | BytesWritten : 3254442191693 | 20260508 10:49:35.545 | INFO | NumEvents : 1563845329573 | 20260508 10:49:37.097 | INFO | Analyzing chunk frames = [7516160,7516672) | 20260508 10:49:37.097 | INFO | MemInfo: { | 20260508 10:49:37.097 | INFO | "gpu" : " 17 GB", | 20260508 10:49:37.097 | INFO | "huge" : "", | 20260508 10:49:37.097 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:37.097 | INFO | "malloc" : " 14 GB", | 20260508 10:49:37.097 | INFO | "pinned" : " 29 GB" | 20260508 10:49:37.097 | INFO | } | 20260508 10:49:37.097 | INFO | Perf report for the last 5.13608 seconds: | 20260508 10:49:37.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:37.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:37.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:37.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:37.097 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 10:49:37.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:37.097 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:49:37.097 | INFO | PRE_HQ : 65.2%, 1.6, 54.1ms, 0 | 20260508 10:49:37.097 | INFO | BAZWRITER : 67.2%, 2.5, 86.2ms, 0 | 20260508 10:49:37.097 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:49:37.097 | INFO | Uploaded 36GiB to GPU in 2.60871s (13.7999GiB/s) | 20260508 10:49:37.098 | INFO | Downloaded 38GiB to GPU in 2.70533s (14.0463GiB/s) | 20260508 10:49:42.236 | INFO | Analyzing chunk frames = [7516672,7517184) | 20260508 10:49:42.236 | INFO | MemInfo: { | 20260508 10:49:42.236 | INFO | "gpu" : " 17 GB", | 20260508 10:49:42.236 | INFO | "huge" : "", | 20260508 10:49:42.236 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:42.236 | INFO | "malloc" : " 14 GB", | 20260508 10:49:42.236 | INFO | "pinned" : " 29 GB" | 20260508 10:49:42.236 | INFO | } | 20260508 10:49:42.236 | INFO | Perf report for the last 5.13932 seconds: | 20260508 10:49:42.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:42.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.236 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 10:49:42.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.236 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:49:42.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:42.237 | INFO | Uploaded 36GiB to GPU in 2.46888s (14.5815GiB/s) | 20260508 10:49:42.237 | INFO | Downloaded 34GiB to GPU in 2.48029s (13.7081GiB/s) | 20260508 10:49:47.378 | INFO | Analyzing chunk frames = [7517184,7517696) | 20260508 10:49:47.378 | INFO | MemInfo: { | 20260508 10:49:47.378 | INFO | "gpu" : " 17 GB", | 20260508 10:49:47.378 | INFO | "huge" : "", | 20260508 10:49:47.378 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:47.378 | INFO | "malloc" : " 14 GB", | 20260508 10:49:47.378 | INFO | "pinned" : " 29 GB" | 20260508 10:49:47.378 | INFO | } | 20260508 10:49:47.378 | INFO | Perf report for the last 5.14167 seconds: | 20260508 10:49:47.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:47.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:49:47.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:49:47.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:47.378 | INFO | Uploaded 36GiB to GPU in 2.46202s (14.6221GiB/s) | 20260508 10:49:47.378 | INFO | Downloaded 34GiB to GPU in 2.43634s (13.9553GiB/s) | 20260508 10:49:52.521 | INFO | Analyzing chunk frames = [7517696,7518208) | 20260508 10:49:52.521 | INFO | MemInfo: { | 20260508 10:49:52.521 | INFO | "gpu" : " 17 GB", | 20260508 10:49:52.521 | INFO | "huge" : "", | 20260508 10:49:52.521 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:52.521 | INFO | "malloc" : " 14 GB", | 20260508 10:49:52.521 | INFO | "pinned" : " 29 GB" | 20260508 10:49:52.521 | INFO | } | 20260508 10:49:52.522 | INFO | Perf report for the last 5.14369 seconds: | 20260508 10:49:52.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:52.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 10:49:52.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:49:52.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:52.522 | INFO | Uploaded 36GiB to GPU in 2.44928s (14.6982GiB/s) | 20260508 10:49:52.522 | INFO | Downloaded 34GiB to GPU in 2.41264s (14.0925GiB/s) | 20260508 10:49:57.662 | INFO | Analyzing chunk frames = [7518208,7518720) | 20260508 10:49:57.662 | INFO | MemInfo: { | 20260508 10:49:57.662 | INFO | "gpu" : " 17 GB", | 20260508 10:49:57.662 | INFO | "huge" : "", | 20260508 10:49:57.662 | INFO | "huge_shared" : " 29 GB", | 20260508 10:49:57.662 | INFO | "malloc" : " 14 GB", | 20260508 10:49:57.662 | INFO | "pinned" : " 29 GB" | 20260508 10:49:57.662 | INFO | } | 20260508 10:49:57.662 | INFO | Perf report for the last 5.14079 seconds: | 20260508 10:49:57.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:49:57.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.662 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 10:49:57.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.662 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 10:49:57.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:49:57.663 | INFO | Uploaded 36GiB to GPU in 2.42677s (14.8345GiB/s) | 20260508 10:49:57.663 | INFO | Downloaded 34GiB to GPU in 2.5453s (13.3579GiB/s) | 20260508 10:50:02.796 | INFO | Analyzing chunk frames = [7518720,7519232) | 20260508 10:50:02.796 | INFO | MemInfo: { | 20260508 10:50:02.796 | INFO | "gpu" : " 17 GB", | 20260508 10:50:02.796 | INFO | "huge" : "", | 20260508 10:50:02.796 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:02.796 | INFO | "malloc" : " 14 GB", | 20260508 10:50:02.796 | INFO | "pinned" : " 29 GB" | 20260508 10:50:02.797 | INFO | } | 20260508 10:50:02.797 | INFO | Perf report for the last 5.13419 seconds: | 20260508 10:50:02.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:02.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:50:02.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:50:02.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:02.797 | INFO | Uploaded 36GiB to GPU in 2.366s (15.2155GiB/s) | 20260508 10:50:02.797 | INFO | Downloaded 34GiB to GPU in 2.44865s (13.8852GiB/s) | 20260508 10:50:07.934 | INFO | Analyzing chunk frames = [7519232,7519744) | 20260508 10:50:07.934 | INFO | MemInfo: { | 20260508 10:50:07.935 | INFO | "gpu" : " 17 GB", | 20260508 10:50:07.935 | INFO | "huge" : "", | 20260508 10:50:07.935 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:07.935 | INFO | "malloc" : " 14 GB", | 20260508 10:50:07.935 | INFO | "pinned" : " 29 GB" | 20260508 10:50:07.935 | INFO | } | 20260508 10:50:07.935 | INFO | Perf report for the last 5.13811 seconds: | 20260508 10:50:07.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:07.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 10:50:07.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 10:50:07.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:07.935 | INFO | Uploaded 36GiB to GPU in 2.43187s (14.8034GiB/s) | 20260508 10:50:07.935 | INFO | Downloaded 34GiB to GPU in 2.46877s (13.772GiB/s) | 20260508 10:50:13.075 | INFO | Analyzing chunk frames = [7519744,7520256) | 20260508 10:50:13.075 | INFO | MemInfo: { | 20260508 10:50:13.075 | INFO | "gpu" : " 17 GB", | 20260508 10:50:13.075 | INFO | "huge" : "", | 20260508 10:50:13.075 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:13.075 | INFO | "malloc" : " 14 GB", | 20260508 10:50:13.075 | INFO | "pinned" : " 29 GB" | 20260508 10:50:13.075 | INFO | } | 20260508 10:50:13.075 | INFO | Perf report for the last 5.14022 seconds: | 20260508 10:50:13.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:13.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 10:50:13.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:50:13.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:13.075 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:50:13.075 | INFO | Total mem usage: 16.7 GiB | 20260508 10:50:13.075 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:50:13.075 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:50:13.075 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:50:13.075 | INFO | High Water Breakdown: | 20260508 10:50:13.076 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:50:13.076 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:50:13.076 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:50:13.076 | INFO | ------------------------------------------ | 20260508 10:50:13.076 | INFO | Uploaded 36GiB to GPU in 2.37752s (15.1418GiB/s) | 20260508 10:50:13.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:50:13.076 | INFO | Downloaded 34GiB to GPU in 2.44436s (13.9095GiB/s) | 20260508 10:50:13.076 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:50:13.076 | INFO | Total mem usage: 26.5 GiB | 20260508 10:50:13.076 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:50:13.076 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:50:13.076 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:50:13.076 | INFO | High Water Breakdown: | 20260508 10:50:13.076 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:50:13.076 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:50:13.076 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:50:13.076 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:50:13.076 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:50:13.076 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:50:13.076 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:50:13.076 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:50:13.076 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:50:13.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:50:13.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:50:13.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:50:13.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:50:13.076 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:50:13.076 | INFO | ------------------------------------------ | 20260508 10:50:13.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:50:13.076 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:50:13.076 | INFO | Total mem usage: 27.8 GiB | 20260508 10:50:13.076 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:50:13.076 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:50:13.076 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:50:13.076 | INFO | High Water Breakdown: | 20260508 10:50:13.076 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:50:13.076 | INFO | ------------------------------------------ | 20260508 10:50:13.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:50:13.076 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:50:13.076 | INFO | Total mem usage: 15.7 GiB | 20260508 10:50:13.076 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:50:13.076 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:50:13.076 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:50:13.076 | INFO | High Water Breakdown: | 20260508 10:50:13.076 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:50:13.076 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:50:13.076 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:50:13.076 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:50:13.076 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:50:13.076 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:50:13.076 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:50:13.076 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:50:13.076 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:50:13.076 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:50:13.076 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:50:13.076 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:50:13.076 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:50:13.076 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:50:13.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:50:13.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:50:13.077 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:50:13.077 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:50:13.077 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:50:13.077 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:50:13.077 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:50:13.077 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:50:13.077 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:50:13.077 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:50:13.077 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:50:13.077 | INFO | ------------------------------------------ | 20260508 10:50:13.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:50:13.077 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:50:13.077 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:50:13.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:50:13.077 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.40e+01 / 5.34e-01 | 20260508 10:50:13.077 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.06e-01, 1.16e-02 / 7.72e-03 | 20260508 10:50:13.077 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.50e+00, 5.88e-01 / 1.68e-01 | 20260508 10:50:13.077 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.49e+00, 2.10e-01 / 1.22e-01 | 20260508 10:50:13.077 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.51e+01, 3.70e+00 / 9.24e-01 | 20260508 10:50:13.077 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.82e+00 / 1.16e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.04e-01 / 2.09e-01 | 20260508 10:50:13.077 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.15e+00, 2.43e-01 / 1.04e-01 | 20260508 10:50:13.077 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.37e-01, 4.88e-02 / 2.51e-02 | 20260508 10:50:13.077 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:50:13.077 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:50:13.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:50:13.077 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 24.32%, Upload -- 2.56e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 24.64%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 2.13%, LockUpload -- 2.24e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 15.27%, LockCompute -- 1.61e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:50:13.077 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260508 10:50:13.172 | INFO | Starting a round of Baz IO | 20260508 10:50:16.592 | INFO | Populating RTMetrics report | 20260508 10:50:16.593 | INFO | RTMetrics report populated | 20260508 10:50:16.593 | INFO | Opening tmp RTMetrics json file | 20260508 10:50:16.602 | INFO | Serializing RTMetrics to json | 20260508 10:50:16.603 | INFO | Serialization done, swapping tmp file now... | 20260508 10:50:16.604 | INFO | RTMetrics done | 20260508 10:50:16.658 | INFO | Finished a round of Baz IO: Wrote 1.45 GiB in 3.486 seconds | 20260508 10:50:16.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:50:16.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.210134% / 0% / 7.54992e-05% | 20260508 10:50:16.659 | INFO | Baz IO overall: | 20260508 10:50:16.659 | INFO | BytesWritten : 3255999424156 | 20260508 10:50:16.659 | INFO | NumEvents : 1564579045121 | 20260508 10:50:18.219 | INFO | Analyzing chunk frames = [7520256,7520768) | 20260508 10:50:18.219 | INFO | MemInfo: { | 20260508 10:50:18.219 | INFO | "gpu" : " 17 GB", | 20260508 10:50:18.219 | INFO | "huge" : "", | 20260508 10:50:18.219 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:18.219 | INFO | "malloc" : " 14 GB", | 20260508 10:50:18.219 | INFO | "pinned" : " 29 GB" | 20260508 10:50:18.219 | INFO | } | 20260508 10:50:18.219 | INFO | Perf report for the last 5.14371 seconds: | 20260508 10:50:18.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:18.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:18.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:18.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:18.219 | INFO | ANALYSIS : 67.7%, 3.0, 105.8ms, 0 | 20260508 10:50:18.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:18.219 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 10:50:18.219 | INFO | PRE_HQ : 65.5%, 1.6, 54.5ms, 0 | 20260508 10:50:18.219 | INFO | BAZWRITER : 67.1%, 2.5, 86.9ms, 0 | 20260508 10:50:18.219 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:50:18.219 | INFO | Uploaded 36GiB to GPU in 2.57748s (13.9671GiB/s) | 20260508 10:50:18.219 | INFO | Downloaded 38GiB to GPU in 2.71671s (13.9875GiB/s) | 20260508 10:50:23.356 | INFO | Analyzing chunk frames = [7520768,7521280) | 20260508 10:50:23.356 | INFO | MemInfo: { | 20260508 10:50:23.356 | INFO | "gpu" : " 17 GB", | 20260508 10:50:23.356 | INFO | "huge" : "", | 20260508 10:50:23.356 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:23.356 | INFO | "malloc" : " 14 GB", | 20260508 10:50:23.356 | INFO | "pinned" : " 29 GB" | 20260508 10:50:23.356 | INFO | } | 20260508 10:50:23.356 | INFO | Perf report for the last 5.13732 seconds: | 20260508 10:50:23.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:23.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:50:23.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:50:23.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:23.356 | INFO | Uploaded 36GiB to GPU in 2.47536s (14.5433GiB/s) | 20260508 10:50:23.357 | INFO | Downloaded 34GiB to GPU in 2.47368s (13.7447GiB/s) | 20260508 10:50:28.494 | INFO | Analyzing chunk frames = [7521280,7521792) | 20260508 10:50:28.494 | INFO | MemInfo: { | 20260508 10:50:28.494 | INFO | "gpu" : " 17 GB", | 20260508 10:50:28.494 | INFO | "huge" : "", | 20260508 10:50:28.494 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:28.494 | INFO | "malloc" : " 14 GB", | 20260508 10:50:28.494 | INFO | "pinned" : " 29 GB" | 20260508 10:50:28.494 | INFO | } | 20260508 10:50:28.494 | INFO | Perf report for the last 5.13807 seconds: | 20260508 10:50:28.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:28.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 10:50:28.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:50:28.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:28.494 | INFO | Uploaded 36GiB to GPU in 2.49042s (14.4554GiB/s) | 20260508 10:50:28.494 | INFO | Downloaded 34GiB to GPU in 2.43858s (13.9425GiB/s) | 20260508 10:50:33.638 | INFO | Analyzing chunk frames = [7521792,7522304) | 20260508 10:50:33.638 | INFO | MemInfo: { | 20260508 10:50:33.638 | INFO | "gpu" : " 17 GB", | 20260508 10:50:33.638 | INFO | "huge" : "", | 20260508 10:50:33.638 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:33.638 | INFO | "malloc" : " 14 GB", | 20260508 10:50:33.638 | INFO | "pinned" : " 29 GB" | 20260508 10:50:33.638 | INFO | } | 20260508 10:50:33.638 | INFO | Perf report for the last 5.14412 seconds: | 20260508 10:50:33.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:33.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.638 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 10:50:33.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.638 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:50:33.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:33.639 | INFO | Uploaded 36GiB to GPU in 2.46126s (14.6267GiB/s) | 20260508 10:50:33.639 | INFO | Downloaded 34GiB to GPU in 2.44342s (13.9149GiB/s) | 20260508 10:50:38.776 | INFO | Analyzing chunk frames = [7522304,7522816) | 20260508 10:50:38.776 | INFO | MemInfo: { | 20260508 10:50:38.776 | INFO | "gpu" : " 17 GB", | 20260508 10:50:38.776 | INFO | "huge" : "", | 20260508 10:50:38.776 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:38.776 | INFO | "malloc" : " 14 GB", | 20260508 10:50:38.776 | INFO | "pinned" : " 29 GB" | 20260508 10:50:38.776 | INFO | } | 20260508 10:50:38.776 | INFO | Perf report for the last 5.13776 seconds: | 20260508 10:50:38.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:38.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 10:50:38.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:50:38.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:38.776 | INFO | Uploaded 36GiB to GPU in 2.26228s (15.9132GiB/s) | 20260508 10:50:38.776 | INFO | Downloaded 34GiB to GPU in 2.4924s (13.6415GiB/s) | 20260508 10:50:43.911 | INFO | Analyzing chunk frames = [7522816,7523328) | 20260508 10:50:43.911 | INFO | MemInfo: { | 20260508 10:50:43.911 | INFO | "gpu" : " 17 GB", | 20260508 10:50:43.911 | INFO | "huge" : "", | 20260508 10:50:43.911 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:43.911 | INFO | "malloc" : " 14 GB", | 20260508 10:50:43.911 | INFO | "pinned" : " 29 GB" | 20260508 10:50:43.911 | INFO | } | 20260508 10:50:43.911 | INFO | Perf report for the last 5.13535 seconds: | 20260508 10:50:43.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:43.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.911 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 10:50:43.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.911 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 10:50:43.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:43.912 | INFO | Uploaded 36GiB to GPU in 2.36152s (15.2444GiB/s) | 20260508 10:50:43.912 | INFO | Downloaded 34GiB to GPU in 2.43428s (13.9672GiB/s) | 20260508 10:50:49.043 | INFO | Analyzing chunk frames = [7523328,7523840) | 20260508 10:50:49.043 | INFO | MemInfo: { | 20260508 10:50:49.043 | INFO | "gpu" : " 17 GB", | 20260508 10:50:49.043 | INFO | "huge" : "", | 20260508 10:50:49.043 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:49.043 | INFO | "malloc" : " 14 GB", | 20260508 10:50:49.043 | INFO | "pinned" : " 29 GB" | 20260508 10:50:49.043 | INFO | } | 20260508 10:50:49.043 | INFO | Perf report for the last 5.13186 seconds: | 20260508 10:50:49.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:49.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:50:49.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:50:49.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:49.043 | INFO | Uploaded 36GiB to GPU in 2.42278s (14.859GiB/s) | 20260508 10:50:49.043 | INFO | Downloaded 34GiB to GPU in 2.43722s (13.9503GiB/s) | 20260508 10:50:54.190 | INFO | Analyzing chunk frames = [7523840,7524352) | 20260508 10:50:54.190 | INFO | MemInfo: { | 20260508 10:50:54.190 | INFO | "gpu" : " 17 GB", | 20260508 10:50:54.190 | INFO | "huge" : "", | 20260508 10:50:54.190 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:54.190 | INFO | "malloc" : " 14 GB", | 20260508 10:50:54.190 | INFO | "pinned" : " 29 GB" | 20260508 10:50:54.190 | INFO | } | 20260508 10:50:54.190 | INFO | Perf report for the last 5.14683 seconds: | 20260508 10:50:54.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:54.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.190 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 10:50:54.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.190 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:50:54.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:54.191 | INFO | Uploaded 36GiB to GPU in 2.39462s (15.0337GiB/s) | 20260508 10:50:54.191 | INFO | Downloaded 34GiB to GPU in 2.42319s (14.0311GiB/s) | 20260508 10:50:54.286 | INFO | Starting a round of Baz IO | 20260508 10:50:57.694 | INFO | Populating RTMetrics report | 20260508 10:50:57.694 | INFO | RTMetrics report populated | 20260508 10:50:57.695 | INFO | Opening tmp RTMetrics json file | 20260508 10:50:57.704 | INFO | Serializing RTMetrics to json | 20260508 10:50:57.705 | INFO | Serialization done, swapping tmp file now... | 20260508 10:50:57.706 | INFO | RTMetrics done | 20260508 10:50:57.765 | INFO | Finished a round of Baz IO: Wrote 1.45 GiB in 3.478 seconds | 20260508 10:50:57.765 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:50:57.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209641% / 0% / 5.5631e-05% | 20260508 10:50:57.765 | INFO | Baz IO overall: | 20260508 10:50:57.765 | INFO | BytesWritten : 3257555910662 | 20260508 10:50:57.765 | INFO | NumEvents : 1565312136738 | 20260508 10:50:59.331 | INFO | Analyzing chunk frames = [7524352,7524864) | 20260508 10:50:59.331 | INFO | MemInfo: { | 20260508 10:50:59.331 | INFO | "gpu" : " 17 GB", | 20260508 10:50:59.331 | INFO | "huge" : "", | 20260508 10:50:59.331 | INFO | "huge_shared" : " 29 GB", | 20260508 10:50:59.331 | INFO | "malloc" : " 14 GB", | 20260508 10:50:59.331 | INFO | "pinned" : " 29 GB" | 20260508 10:50:59.331 | INFO | } | 20260508 10:50:59.331 | INFO | Perf report for the last 5.14098 seconds: | 20260508 10:50:59.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:50:59.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:59.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:59.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:59.331 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:50:59.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:50:59.331 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:50:59.331 | INFO | PRE_HQ : 65.5%, 1.6, 54.3ms, 0 | 20260508 10:50:59.331 | INFO | BAZWRITER : 66.9%, 2.5, 86.5ms, 0 | 20260508 10:50:59.331 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:50:59.331 | INFO | Uploaded 36GiB to GPU in 2.5927s (13.8851GiB/s) | 20260508 10:50:59.331 | INFO | Downloaded 38GiB to GPU in 2.71769s (13.9824GiB/s) | 20260508 10:51:04.470 | INFO | Analyzing chunk frames = [7524864,7525376) | 20260508 10:51:04.470 | INFO | MemInfo: { | 20260508 10:51:04.470 | INFO | "gpu" : " 17 GB", | 20260508 10:51:04.470 | INFO | "huge" : "", | 20260508 10:51:04.470 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:04.470 | INFO | "malloc" : " 14 GB", | 20260508 10:51:04.470 | INFO | "pinned" : " 29 GB" | 20260508 10:51:04.470 | INFO | } | 20260508 10:51:04.470 | INFO | Perf report for the last 5.13937 seconds: | 20260508 10:51:04.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:04.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.471 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:51:04.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.471 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:51:04.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:04.471 | INFO | Uploaded 36GiB to GPU in 2.43431s (14.7886GiB/s) | 20260508 10:51:04.471 | INFO | Downloaded 34GiB to GPU in 2.47578s (13.733GiB/s) | 20260508 10:51:09.605 | INFO | Analyzing chunk frames = [7525376,7525888) | 20260508 10:51:09.605 | INFO | MemInfo: { | 20260508 10:51:09.605 | INFO | "gpu" : " 17 GB", | 20260508 10:51:09.605 | INFO | "huge" : "", | 20260508 10:51:09.605 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:09.605 | INFO | "malloc" : " 14 GB", | 20260508 10:51:09.605 | INFO | "pinned" : " 29 GB" | 20260508 10:51:09.605 | INFO | } | 20260508 10:51:09.605 | INFO | Perf report for the last 5.13494 seconds: | 20260508 10:51:09.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:09.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.605 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 10:51:09.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.605 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:51:09.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:09.606 | INFO | Uploaded 36GiB to GPU in 2.46913s (14.5801GiB/s) | 20260508 10:51:09.606 | INFO | Downloaded 34GiB to GPU in 2.4621s (13.8093GiB/s) | 20260508 10:51:14.745 | INFO | Analyzing chunk frames = [7525888,7526400) | 20260508 10:51:14.745 | INFO | MemInfo: { | 20260508 10:51:14.745 | INFO | "gpu" : " 17 GB", | 20260508 10:51:14.745 | INFO | "huge" : "", | 20260508 10:51:14.745 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:14.745 | INFO | "malloc" : " 14 GB", | 20260508 10:51:14.745 | INFO | "pinned" : " 29 GB" | 20260508 10:51:14.745 | INFO | } | 20260508 10:51:14.745 | INFO | Perf report for the last 5.14007 seconds: | 20260508 10:51:14.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:14.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.746 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 10:51:14.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.746 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:51:14.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:14.746 | INFO | Uploaded 36GiB to GPU in 2.43525s (14.7829GiB/s) | 20260508 10:51:14.746 | INFO | Downloaded 34GiB to GPU in 2.44952s (13.8803GiB/s) | 20260508 10:51:19.886 | INFO | Analyzing chunk frames = [7526400,7526912) | 20260508 10:51:19.886 | INFO | MemInfo: { | 20260508 10:51:19.886 | INFO | "gpu" : " 17 GB", | 20260508 10:51:19.886 | INFO | "huge" : "", | 20260508 10:51:19.886 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:19.886 | INFO | "malloc" : " 14 GB", | 20260508 10:51:19.886 | INFO | "pinned" : " 29 GB" | 20260508 10:51:19.886 | INFO | } | 20260508 10:51:19.886 | INFO | Perf report for the last 5.1402 seconds: | 20260508 10:51:19.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:19.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 10:51:19.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:51:19.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:19.886 | INFO | Uploaded 36GiB to GPU in 2.41928s (14.8805GiB/s) | 20260508 10:51:19.886 | INFO | Downloaded 34GiB to GPU in 2.5412s (13.3795GiB/s) | 20260508 10:51:25.025 | INFO | Analyzing chunk frames = [7526912,7527424) | 20260508 10:51:25.025 | INFO | MemInfo: { | 20260508 10:51:25.025 | INFO | "gpu" : " 17 GB", | 20260508 10:51:25.025 | INFO | "huge" : "", | 20260508 10:51:25.025 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:25.025 | INFO | "malloc" : " 14 GB", | 20260508 10:51:25.025 | INFO | "pinned" : " 29 GB" | 20260508 10:51:25.025 | INFO | } | 20260508 10:51:25.025 | INFO | Perf report for the last 5.1394 seconds: | 20260508 10:51:25.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:25.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.025 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:51:25.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.025 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:51:25.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:25.026 | INFO | Uploaded 36GiB to GPU in 2.3683s (15.2008GiB/s) | 20260508 10:51:25.026 | INFO | Downloaded 34GiB to GPU in 2.43875s (13.9415GiB/s) | 20260508 10:51:30.170 | INFO | Analyzing chunk frames = [7527424,7527936) | 20260508 10:51:30.170 | INFO | MemInfo: { | 20260508 10:51:30.170 | INFO | "gpu" : " 17 GB", | 20260508 10:51:30.170 | INFO | "huge" : "", | 20260508 10:51:30.170 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:30.170 | INFO | "malloc" : " 14 GB", | 20260508 10:51:30.170 | INFO | "pinned" : " 29 GB" | 20260508 10:51:30.170 | INFO | } | 20260508 10:51:30.170 | INFO | Perf report for the last 5.14526 seconds: | 20260508 10:51:30.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:30.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.170 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 10:51:30.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.171 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:51:30.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:30.171 | INFO | Uploaded 36GiB to GPU in 2.42182s (14.8649GiB/s) | 20260508 10:51:30.171 | INFO | Downloaded 34GiB to GPU in 2.45521s (13.8481GiB/s) | 20260508 10:51:35.305 | INFO | Analyzing chunk frames = [7527936,7528448) | 20260508 10:51:35.305 | INFO | MemInfo: { | 20260508 10:51:35.305 | INFO | "gpu" : " 17 GB", | 20260508 10:51:35.305 | INFO | "huge" : "", | 20260508 10:51:35.305 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:35.305 | INFO | "malloc" : " 14 GB", | 20260508 10:51:35.305 | INFO | "pinned" : " 29 GB" | 20260508 10:51:35.305 | INFO | } | 20260508 10:51:35.305 | INFO | Perf report for the last 5.13431 seconds: | 20260508 10:51:35.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:35.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 10:51:35.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:51:35.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:35.305 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:51:35.305 | INFO | Total mem usage: 16.7 GiB | 20260508 10:51:35.305 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:51:35.305 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 10:51:35.305 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:51:35.305 | INFO | High Water Breakdown: | 20260508 10:51:35.305 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:51:35.305 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:51:35.305 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:51:35.305 | INFO | ------------------------------------------ | 20260508 10:51:35.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:51:35.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:51:35.305 | INFO | Total mem usage: 26.5 GiB | 20260508 10:51:35.305 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:51:35.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:51:35.305 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:51:35.305 | INFO | High Water Breakdown: | 20260508 10:51:35.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:51:35.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:51:35.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:51:35.305 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:51:35.305 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:51:35.305 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:51:35.305 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:51:35.305 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:51:35.305 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:51:35.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:51:35.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:51:35.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:51:35.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:51:35.305 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:51:35.305 | INFO | ------------------------------------------ | 20260508 10:51:35.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:51:35.305 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:51:35.305 | INFO | Total mem usage: 27.8 GiB | 20260508 10:51:35.305 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:51:35.305 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:51:35.305 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:51:35.305 | INFO | High Water Breakdown: | 20260508 10:51:35.305 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:51:35.305 | INFO | ------------------------------------------ | 20260508 10:51:35.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:51:35.305 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:51:35.305 | INFO | Total mem usage: 15.7 GiB | 20260508 10:51:35.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:51:35.305 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:51:35.305 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:51:35.305 | INFO | Uploaded 36GiB to GPU in 2.45757s (14.6486GiB/s) | 20260508 10:51:35.305 | INFO | High Water Breakdown: | 20260508 10:51:35.306 | INFO | Downloaded 34GiB to GPU in 2.46143s (13.8131GiB/s) | 20260508 10:51:35.306 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:51:35.306 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:51:35.306 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:51:35.306 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:51:35.306 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:51:35.306 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:51:35.306 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:51:35.306 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:51:35.306 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:51:35.306 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:51:35.306 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:51:35.306 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:51:35.306 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:51:35.306 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:51:35.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:51:35.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:51:35.306 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:51:35.306 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:51:35.306 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:51:35.306 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:51:35.306 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:51:35.306 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:51:35.306 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:51:35.306 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:51:35.306 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:51:35.306 | INFO | ------------------------------------------ | 20260508 10:51:35.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:51:35.306 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:51:35.306 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:51:35.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:51:35.306 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 7.19e-01 | 20260508 10:51:35.306 | INFO | PerfReport: 0.29%, AggregateStats -- 9.64e-02 / 1.04e-01, 1.14e-02 / 5.04e-03 | 20260508 10:51:35.306 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.46e+00, 5.92e-01 / 2.20e-01 | 20260508 10:51:35.306 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.42e-01 | 20260508 10:51:35.306 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.51e+01, 3.80e+00 / 9.13e-01 | 20260508 10:51:35.306 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.81e+00 / 1.37e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.45e+00, 4.07e-01 / 2.94e-01 | 20260508 10:51:35.306 | INFO | PerfReport: 3.00%, Metrics -- 9.79e-01 / 1.10e+00, 2.38e-01 / 1.16e-01 | 20260508 10:51:35.306 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.31e-01, 4.79e-02 / 3.31e-02 | 20260508 10:51:35.306 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:51:35.306 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:51:35.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:51:35.306 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 24.16%, Upload -- 2.54e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 24.67%, Download -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 2.08%, LockUpload -- 2.18e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 2.65%, LockDownload -- 2.78e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 15.35%, LockCompute -- 1.62e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 10:51:35.306 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260508 10:51:35.403 | INFO | Starting a round of Baz IO | 20260508 10:51:38.815 | INFO | Populating RTMetrics report | 20260508 10:51:38.816 | INFO | RTMetrics report populated | 20260508 10:51:38.816 | INFO | Opening tmp RTMetrics json file | 20260508 10:51:38.817 | INFO | Serializing RTMetrics to json | 20260508 10:51:38.818 | INFO | Serialization done, swapping tmp file now... | 20260508 10:51:38.818 | INFO | RTMetrics done | 20260508 10:51:38.878 | INFO | Finished a round of Baz IO: Wrote 1.448 GiB in 3.474 seconds | 20260508 10:51:38.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:51:38.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.208135% / 0% / 7.94729e-05% | 20260508 10:51:38.878 | INFO | Baz IO overall: | 20260508 10:51:38.878 | INFO | BytesWritten : 3259111151990 | 20260508 10:51:38.878 | INFO | NumEvents : 1566044523796 | 20260508 10:51:40.448 | INFO | Analyzing chunk frames = [7528448,7528960) | 20260508 10:51:40.448 | INFO | MemInfo: { | 20260508 10:51:40.448 | INFO | "gpu" : " 17 GB", | 20260508 10:51:40.448 | INFO | "huge" : "", | 20260508 10:51:40.448 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:40.448 | INFO | "malloc" : " 14 GB", | 20260508 10:51:40.448 | INFO | "pinned" : " 29 GB" | 20260508 10:51:40.448 | INFO | } | 20260508 10:51:40.448 | INFO | Perf report for the last 5.1433 seconds: | 20260508 10:51:40.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:40.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:40.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:40.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:40.448 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 10:51:40.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:40.448 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 10:51:40.448 | INFO | PRE_HQ : 65.6%, 1.6, 54.3ms, 0 | 20260508 10:51:40.448 | INFO | BAZWRITER : 66.8%, 2.5, 86.7ms, 0 | 20260508 10:51:40.448 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 10:51:40.448 | INFO | Uploaded 36GiB to GPU in 2.50574s (14.367GiB/s) | 20260508 10:51:40.448 | INFO | Downloaded 38GiB to GPU in 2.68083s (14.1747GiB/s) | 20260508 10:51:45.584 | INFO | Analyzing chunk frames = [7528960,7529472) | 20260508 10:51:45.584 | INFO | MemInfo: { | 20260508 10:51:45.584 | INFO | "gpu" : " 17 GB", | 20260508 10:51:45.584 | INFO | "huge" : "", | 20260508 10:51:45.584 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:45.584 | INFO | "malloc" : " 14 GB", | 20260508 10:51:45.584 | INFO | "pinned" : " 29 GB" | 20260508 10:51:45.584 | INFO | } | 20260508 10:51:45.584 | INFO | Perf report for the last 5.1363 seconds: | 20260508 10:51:45.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:45.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.585 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:51:45.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.585 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:51:45.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:45.585 | INFO | Uploaded 36GiB to GPU in 2.46268s (14.6182GiB/s) | 20260508 10:51:45.585 | INFO | Downloaded 34GiB to GPU in 2.47402s (13.7428GiB/s) | 20260508 10:51:50.728 | INFO | Analyzing chunk frames = [7529472,7529984) | 20260508 10:51:50.728 | INFO | MemInfo: { | 20260508 10:51:50.728 | INFO | "gpu" : " 17 GB", | 20260508 10:51:50.728 | INFO | "huge" : "", | 20260508 10:51:50.728 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:50.728 | INFO | "malloc" : " 14 GB", | 20260508 10:51:50.728 | INFO | "pinned" : " 29 GB" | 20260508 10:51:50.728 | INFO | } | 20260508 10:51:50.728 | INFO | Perf report for the last 5.14397 seconds: | 20260508 10:51:50.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:50.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 10:51:50.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260508 10:51:50.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:50.728 | INFO | Uploaded 36GiB to GPU in 2.4544s (14.6675GiB/s) | 20260508 10:51:50.728 | INFO | Downloaded 34GiB to GPU in 2.43556s (13.9598GiB/s) | 20260508 10:51:55.858 | INFO | Analyzing chunk frames = [7529984,7530496) | 20260508 10:51:55.858 | INFO | MemInfo: { | 20260508 10:51:55.858 | INFO | "gpu" : " 17 GB", | 20260508 10:51:55.859 | INFO | "huge" : "", | 20260508 10:51:55.859 | INFO | "huge_shared" : " 29 GB", | 20260508 10:51:55.859 | INFO | "malloc" : " 14 GB", | 20260508 10:51:55.859 | INFO | "pinned" : " 29 GB" | 20260508 10:51:55.859 | INFO | } | 20260508 10:51:55.859 | INFO | Perf report for the last 5.13042 seconds: | 20260508 10:51:55.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:51:55.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 10:51:55.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:51:55.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:51:55.859 | INFO | Uploaded 36GiB to GPU in 2.46408s (14.6099GiB/s) | 20260508 10:51:55.859 | INFO | Downloaded 34GiB to GPU in 2.44194s (13.9233GiB/s) | 20260508 10:52:01.003 | INFO | Analyzing chunk frames = [7530496,7531008) | 20260508 10:52:01.003 | INFO | MemInfo: { | 20260508 10:52:01.003 | INFO | "gpu" : " 17 GB", | 20260508 10:52:01.003 | INFO | "huge" : "", | 20260508 10:52:01.003 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:01.003 | INFO | "malloc" : " 14 GB", | 20260508 10:52:01.004 | INFO | "pinned" : " 29 GB" | 20260508 10:52:01.004 | INFO | } | 20260508 10:52:01.004 | INFO | Perf report for the last 5.14493 seconds: | 20260508 10:52:01.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:01.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 10:52:01.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:52:01.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:01.004 | INFO | Uploaded 36GiB to GPU in 2.39342s (15.0412GiB/s) | 20260508 10:52:01.004 | INFO | Downloaded 34GiB to GPU in 2.52481s (13.4663GiB/s) | 20260508 10:52:06.142 | INFO | Analyzing chunk frames = [7531008,7531520) | 20260508 10:52:06.142 | INFO | MemInfo: { | 20260508 10:52:06.142 | INFO | "gpu" : " 17 GB", | 20260508 10:52:06.142 | INFO | "huge" : "", | 20260508 10:52:06.142 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:06.142 | INFO | "malloc" : " 14 GB", | 20260508 10:52:06.142 | INFO | "pinned" : " 29 GB" | 20260508 10:52:06.142 | INFO | } | 20260508 10:52:06.142 | INFO | Perf report for the last 5.13814 seconds: | 20260508 10:52:06.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:06.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:52:06.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:52:06.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:06.142 | INFO | Uploaded 36GiB to GPU in 2.37001s (15.1898GiB/s) | 20260508 10:52:06.142 | INFO | Downloaded 34GiB to GPU in 2.4663s (13.7858GiB/s) | 20260508 10:52:11.281 | INFO | Analyzing chunk frames = [7531520,7532032) | 20260508 10:52:11.281 | INFO | MemInfo: { | 20260508 10:52:11.281 | INFO | "gpu" : " 17 GB", | 20260508 10:52:11.281 | INFO | "huge" : "", | 20260508 10:52:11.281 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:11.281 | INFO | "malloc" : " 14 GB", | 20260508 10:52:11.281 | INFO | "pinned" : " 29 GB" | 20260508 10:52:11.281 | INFO | } | 20260508 10:52:11.281 | INFO | Perf report for the last 5.13924 seconds: | 20260508 10:52:11.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:11.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.281 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 10:52:11.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.281 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:52:11.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:11.282 | INFO | Uploaded 36GiB to GPU in 2.3808s (15.1209GiB/s) | 20260508 10:52:11.282 | INFO | Downloaded 34GiB to GPU in 2.45736s (13.836GiB/s) | 20260508 10:52:16.417 | INFO | Analyzing chunk frames = [7532032,7532544) | 20260508 10:52:16.417 | INFO | MemInfo: { | 20260508 10:52:16.417 | INFO | "gpu" : " 17 GB", | 20260508 10:52:16.417 | INFO | "huge" : "", | 20260508 10:52:16.417 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:16.417 | INFO | "malloc" : " 14 GB", | 20260508 10:52:16.417 | INFO | "pinned" : " 29 GB" | 20260508 10:52:16.417 | INFO | } | 20260508 10:52:16.417 | INFO | Perf report for the last 5.13631 seconds: | 20260508 10:52:16.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:16.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.417 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:52:16.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.417 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:52:16.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:16.418 | INFO | Uploaded 36GiB to GPU in 2.39228s (15.0484GiB/s) | 20260508 10:52:16.418 | INFO | Downloaded 34GiB to GPU in 2.44631s (13.8985GiB/s) | 20260508 10:52:16.515 | INFO | Starting a round of Baz IO | 20260508 10:52:19.933 | INFO | Populating RTMetrics report | 20260508 10:52:19.933 | INFO | RTMetrics report populated | 20260508 10:52:19.933 | INFO | Opening tmp RTMetrics json file | 20260508 10:52:19.941 | INFO | Serializing RTMetrics to json | 20260508 10:52:19.943 | INFO | Serialization done, swapping tmp file now... | 20260508 10:52:19.943 | INFO | RTMetrics done | 20260508 10:52:19.999 | INFO | Finished a round of Baz IO: Wrote 1.448 GiB in 3.483 seconds | 20260508 10:52:19.999 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:52:19.999 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207229% / 0% / 7.94729e-05% | 20260508 10:52:19.999 | INFO | Baz IO overall: | 20260508 10:52:19.999 | INFO | BytesWritten : 3260666361748 | 20260508 10:52:19.999 | INFO | NumEvents : 1566776807270 | 20260508 10:52:21.554 | INFO | Analyzing chunk frames = [7532544,7533056) | 20260508 10:52:21.554 | INFO | MemInfo: { | 20260508 10:52:21.554 | INFO | "gpu" : " 17 GB", | 20260508 10:52:21.554 | INFO | "huge" : "", | 20260508 10:52:21.554 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:21.554 | INFO | "malloc" : " 14 GB", | 20260508 10:52:21.554 | INFO | "pinned" : " 29 GB" | 20260508 10:52:21.554 | INFO | } | 20260508 10:52:21.554 | INFO | Perf report for the last 5.13659 seconds: | 20260508 10:52:21.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:21.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:21.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:21.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:21.554 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:52:21.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:21.554 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:52:21.554 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 10:52:21.554 | INFO | BAZWRITER : 67.4%, 3.1, 107.4ms, 0 | 20260508 10:52:21.554 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:52:21.555 | INFO | Uploaded 36GiB to GPU in 2.5819s (13.9432GiB/s) | 20260508 10:52:21.555 | INFO | Downloaded 38GiB to GPU in 2.7129s (14.0071GiB/s) | 20260508 10:52:26.693 | INFO | Analyzing chunk frames = [7533056,7533568) | 20260508 10:52:26.693 | INFO | MemInfo: { | 20260508 10:52:26.693 | INFO | "gpu" : " 17 GB", | 20260508 10:52:26.693 | INFO | "huge" : "", | 20260508 10:52:26.693 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:26.693 | INFO | "malloc" : " 14 GB", | 20260508 10:52:26.693 | INFO | "pinned" : " 29 GB" | 20260508 10:52:26.693 | INFO | } | 20260508 10:52:26.693 | INFO | Perf report for the last 5.13909 seconds: | 20260508 10:52:26.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:26.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.693 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:52:26.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.693 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 10:52:26.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:26.694 | INFO | Uploaded 36GiB to GPU in 2.41838s (14.886GiB/s) | 20260508 10:52:26.694 | INFO | Downloaded 34GiB to GPU in 2.47287s (13.7492GiB/s) | 20260508 10:52:31.842 | INFO | Analyzing chunk frames = [7533568,7534080) | 20260508 10:52:31.842 | INFO | MemInfo: { | 20260508 10:52:31.842 | INFO | "gpu" : " 17 GB", | 20260508 10:52:31.842 | INFO | "huge" : "", | 20260508 10:52:31.842 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:31.842 | INFO | "malloc" : " 14 GB", | 20260508 10:52:31.843 | INFO | "pinned" : " 29 GB" | 20260508 10:52:31.843 | INFO | } | 20260508 10:52:31.843 | INFO | Perf report for the last 5.14943 seconds: | 20260508 10:52:31.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:31.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 10:52:31.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260508 10:52:31.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:31.843 | INFO | Uploaded 36GiB to GPU in 2.47072s (14.5707GiB/s) | 20260508 10:52:31.843 | INFO | Downloaded 34GiB to GPU in 2.46307s (13.8039GiB/s) | 20260508 10:52:36.981 | INFO | Analyzing chunk frames = [7534080,7534592) | 20260508 10:52:36.981 | INFO | MemInfo: { | 20260508 10:52:36.981 | INFO | "gpu" : " 17 GB", | 20260508 10:52:36.981 | INFO | "huge" : "", | 20260508 10:52:36.981 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:36.981 | INFO | "malloc" : " 14 GB", | 20260508 10:52:36.982 | INFO | "pinned" : " 29 GB" | 20260508 10:52:36.982 | INFO | } | 20260508 10:52:36.982 | INFO | Perf report for the last 5.1392 seconds: | 20260508 10:52:36.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:36.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 10:52:36.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 10:52:36.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:36.982 | INFO | Uploaded 36GiB to GPU in 2.45247s (14.6791GiB/s) | 20260508 10:52:36.982 | INFO | Downloaded 34GiB to GPU in 2.46829s (13.7747GiB/s) | 20260508 10:52:42.117 | INFO | Analyzing chunk frames = [7534592,7535104) | 20260508 10:52:42.117 | INFO | MemInfo: { | 20260508 10:52:42.117 | INFO | "gpu" : " 17 GB", | 20260508 10:52:42.117 | INFO | "huge" : "", | 20260508 10:52:42.117 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:42.117 | INFO | "malloc" : " 14 GB", | 20260508 10:52:42.117 | INFO | "pinned" : " 29 GB" | 20260508 10:52:42.117 | INFO | } | 20260508 10:52:42.117 | INFO | Perf report for the last 5.13483 seconds: | 20260508 10:52:42.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:42.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 10:52:42.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260508 10:52:42.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:42.117 | INFO | Uploaded 36GiB to GPU in 2.37148s (15.1804GiB/s) | 20260508 10:52:42.117 | INFO | Downloaded 34GiB to GPU in 2.52698s (13.4548GiB/s) | 20260508 10:52:47.256 | INFO | Analyzing chunk frames = [7535104,7535616) | 20260508 10:52:47.256 | INFO | MemInfo: { | 20260508 10:52:47.256 | INFO | "gpu" : " 17 GB", | 20260508 10:52:47.256 | INFO | "huge" : "", | 20260508 10:52:47.256 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:47.256 | INFO | "malloc" : " 14 GB", | 20260508 10:52:47.256 | INFO | "pinned" : " 29 GB" | 20260508 10:52:47.256 | INFO | } | 20260508 10:52:47.257 | INFO | Perf report for the last 5.13999 seconds: | 20260508 10:52:47.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:47.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 10:52:47.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:52:47.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:47.257 | INFO | Uploaded 36GiB to GPU in 2.38788s (15.0761GiB/s) | 20260508 10:52:47.257 | INFO | Downloaded 34GiB to GPU in 2.44224s (13.9217GiB/s) | 20260508 10:52:52.399 | INFO | Analyzing chunk frames = [7535616,7536128) | 20260508 10:52:52.399 | INFO | MemInfo: { | 20260508 10:52:52.399 | INFO | "gpu" : " 17 GB", | 20260508 10:52:52.399 | INFO | "huge" : "", | 20260508 10:52:52.399 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:52.399 | INFO | "malloc" : " 14 GB", | 20260508 10:52:52.399 | INFO | "pinned" : " 29 GB" | 20260508 10:52:52.399 | INFO | } | 20260508 10:52:52.399 | INFO | Perf report for the last 5.14205 seconds: | 20260508 10:52:52.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:52.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 10:52:52.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:52:52.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:52.399 | INFO | Uploaded 36GiB to GPU in 2.40972s (14.9395GiB/s) | 20260508 10:52:52.399 | INFO | Downloaded 34GiB to GPU in 2.43461s (13.9653GiB/s) | 20260508 10:52:57.537 | INFO | Analyzing chunk frames = [7536128,7536640) | 20260508 10:52:57.537 | INFO | MemInfo: { | 20260508 10:52:57.537 | INFO | "gpu" : " 17 GB", | 20260508 10:52:57.537 | INFO | "huge" : "", | 20260508 10:52:57.537 | INFO | "huge_shared" : " 29 GB", | 20260508 10:52:57.537 | INFO | "malloc" : " 14 GB", | 20260508 10:52:57.537 | INFO | "pinned" : " 29 GB" | 20260508 10:52:57.537 | INFO | } | 20260508 10:52:57.537 | INFO | Perf report for the last 5.13792 seconds: | 20260508 10:52:57.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:52:57.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:52:57.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:52:57.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:52:57.537 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:52:57.537 | INFO | Total mem usage: 16.7 GiB | 20260508 10:52:57.537 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:52:57.537 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -10.5%) | 20260508 10:52:57.537 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:52:57.537 | INFO | High Water Breakdown: | 20260508 10:52:57.537 | INFO | 60.4% : 9.12 GiB ( 88.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:52:57.537 | INFO | 39.4% : 5.95 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:52:57.537 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:52:57.537 | INFO | ------------------------------------------ | 20260508 10:52:57.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:52:57.537 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:52:57.537 | INFO | Total mem usage: 26.5 GiB | 20260508 10:52:57.537 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:52:57.537 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:52:57.537 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:52:57.537 | INFO | High Water Breakdown: | 20260508 10:52:57.537 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:52:57.537 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:52:57.537 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:52:57.537 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:52:57.537 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:52:57.537 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:52:57.537 | INFO | Uploaded 36GiB to GPU in 2.48548s (14.4841GiB/s) | 20260508 10:52:57.537 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:52:57.537 | INFO | Downloaded 34GiB to GPU in 2.47187s (13.7548GiB/s) | 20260508 10:52:57.537 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:52:57.538 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:52:57.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:52:57.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:52:57.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:52:57.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:52:57.538 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:52:57.538 | INFO | ------------------------------------------ | 20260508 10:52:57.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:52:57.538 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:52:57.538 | INFO | Total mem usage: 27.8 GiB | 20260508 10:52:57.538 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:52:57.538 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:52:57.538 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:52:57.538 | INFO | High Water Breakdown: | 20260508 10:52:57.538 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:52:57.538 | INFO | ------------------------------------------ | 20260508 10:52:57.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:52:57.538 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:52:57.538 | INFO | Total mem usage: 15.7 GiB | 20260508 10:52:57.538 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:52:57.538 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:52:57.538 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:52:57.538 | INFO | High Water Breakdown: | 20260508 10:52:57.538 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:52:57.538 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:52:57.538 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:52:57.538 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:52:57.538 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:52:57.538 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:52:57.538 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:52:57.538 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:52:57.538 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:52:57.538 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:52:57.538 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:52:57.538 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:52:57.538 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:52:57.538 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:52:57.538 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:52:57.538 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:52:57.538 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:52:57.538 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:52:57.538 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:52:57.538 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:52:57.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:52:57.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:52:57.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:52:57.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:52:57.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:52:57.539 | INFO | ------------------------------------------ | 20260508 10:52:57.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:52:57.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:52:57.539 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:52:57.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:52:57.539 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.51e+01, 1.38e+01 / 6.37e-01 | 20260508 10:52:57.539 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.05e-01, 1.15e-02 / 9.01e-03 | 20260508 10:52:57.539 | INFO | PerfReport: 12.19%, Baseline -- 3.99e+00 / 4.42e+00, 5.95e-01 / 2.62e-01 | 20260508 10:52:57.539 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.47e+00, 2.10e-01 / 1.14e-01 | 20260508 10:52:57.539 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.49e+01, 3.67e+00 / 1.01e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.84e+00 / 1.36e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.06e-01 / 2.73e-01 | 20260508 10:52:57.539 | INFO | PerfReport: 3.01%, Metrics -- 9.85e-01 / 1.17e+00, 2.43e-01 / 1.78e-01 | 20260508 10:52:57.539 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.32e-01, 4.85e-02 / 2.51e-02 | 20260508 10:52:57.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:52:57.539 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:52:57.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:52:57.539 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 24.14%, Upload -- 2.54e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 24.71%, Download -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 2.02%, LockUpload -- 2.13e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 2.67%, LockDownload -- 2.81e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 15.31%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 10:52:57.539 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 3.59e-02 / 0.00e+00 | 20260508 10:52:57.634 | INFO | Starting a round of Baz IO | 20260508 10:53:01.042 | INFO | Populating RTMetrics report | 20260508 10:53:01.042 | INFO | RTMetrics report populated | 20260508 10:53:01.042 | INFO | Opening tmp RTMetrics json file | 20260508 10:53:01.052 | INFO | Serializing RTMetrics to json | 20260508 10:53:01.053 | INFO | Serialization done, swapping tmp file now... | 20260508 10:53:01.053 | INFO | RTMetrics done | 20260508 10:53:01.111 | INFO | Finished a round of Baz IO: Wrote 1.45 GiB in 3.477 seconds | 20260508 10:53:01.111 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:53:01.111 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209169% / 3.97364e-06% / 9.53674e-05% | 20260508 10:53:01.111 | INFO | Baz IO overall: | 20260508 10:53:01.111 | INFO | BytesWritten : 3262223675585 | 20260508 10:53:01.111 | INFO | NumEvents : 1567510550700 | 20260508 10:53:02.678 | INFO | Analyzing chunk frames = [7536640,7537152) | 20260508 10:53:02.678 | INFO | MemInfo: { | 20260508 10:53:02.678 | INFO | "gpu" : " 17 GB", | 20260508 10:53:02.678 | INFO | "huge" : "", | 20260508 10:53:02.678 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:02.678 | INFO | "malloc" : " 14 GB", | 20260508 10:53:02.678 | INFO | "pinned" : " 29 GB" | 20260508 10:53:02.678 | INFO | } | 20260508 10:53:02.678 | INFO | Perf report for the last 5.14166 seconds: | 20260508 10:53:02.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:02.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:02.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:02.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:02.678 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:53:02.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:02.678 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:53:02.679 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 10:53:02.679 | INFO | BAZWRITER : 66.8%, 2.5, 86.1ms, 0 | 20260508 10:53:02.679 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:53:02.679 | INFO | Uploaded 36GiB to GPU in 2.59405s (13.8779GiB/s) | 20260508 10:53:02.679 | INFO | Downloaded 38GiB to GPU in 2.70197s (14.0638GiB/s) | 20260508 10:53:07.811 | INFO | Analyzing chunk frames = [7537152,7537664) | 20260508 10:53:07.811 | INFO | MemInfo: { | 20260508 10:53:07.811 | INFO | "gpu" : " 17 GB", | 20260508 10:53:07.811 | INFO | "huge" : "", | 20260508 10:53:07.811 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:07.811 | INFO | "malloc" : " 14 GB", | 20260508 10:53:07.811 | INFO | "pinned" : " 29 GB" | 20260508 10:53:07.811 | INFO | } | 20260508 10:53:07.811 | INFO | Perf report for the last 5.13287 seconds: | 20260508 10:53:07.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:07.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.811 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 10:53:07.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.811 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:53:07.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:07.812 | INFO | Uploaded 36GiB to GPU in 2.45664s (14.6542GiB/s) | 20260508 10:53:07.812 | INFO | Downloaded 34GiB to GPU in 2.46216s (13.809GiB/s) | 20260508 10:53:12.950 | INFO | Analyzing chunk frames = [7537664,7538176) | 20260508 10:53:12.950 | INFO | MemInfo: { | 20260508 10:53:12.950 | INFO | "gpu" : " 17 GB", | 20260508 10:53:12.950 | INFO | "huge" : "", | 20260508 10:53:12.950 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:12.950 | INFO | "malloc" : " 14 GB", | 20260508 10:53:12.950 | INFO | "pinned" : " 29 GB" | 20260508 10:53:12.950 | INFO | } | 20260508 10:53:12.950 | INFO | Perf report for the last 5.13916 seconds: | 20260508 10:53:12.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:12.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 10:53:12.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:53:12.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:12.951 | INFO | Uploaded 36GiB to GPU in 2.50187s (14.3892GiB/s) | 20260508 10:53:12.951 | INFO | Downloaded 34GiB to GPU in 2.43441s (13.9664GiB/s) | 20260508 10:53:18.094 | INFO | Analyzing chunk frames = [7538176,7538688) | 20260508 10:53:18.094 | INFO | MemInfo: { | 20260508 10:53:18.094 | INFO | "gpu" : " 17 GB", | 20260508 10:53:18.094 | INFO | "huge" : "", | 20260508 10:53:18.094 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:18.094 | INFO | "malloc" : " 14 GB", | 20260508 10:53:18.094 | INFO | "pinned" : " 29 GB" | 20260508 10:53:18.095 | INFO | } | 20260508 10:53:18.095 | INFO | Perf report for the last 5.14432 seconds: | 20260508 10:53:18.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:18.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:53:18.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:53:18.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:18.095 | INFO | Uploaded 36GiB to GPU in 2.50992s (14.3431GiB/s) | 20260508 10:53:18.095 | INFO | Downloaded 34GiB to GPU in 2.43102s (13.9859GiB/s) | 20260508 10:53:23.231 | INFO | Analyzing chunk frames = [7538688,7539200) | 20260508 10:53:23.231 | INFO | MemInfo: { | 20260508 10:53:23.231 | INFO | "gpu" : " 17 GB", | 20260508 10:53:23.231 | INFO | "huge" : "", | 20260508 10:53:23.231 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:23.231 | INFO | "malloc" : " 14 GB", | 20260508 10:53:23.231 | INFO | "pinned" : " 29 GB" | 20260508 10:53:23.231 | INFO | } | 20260508 10:53:23.231 | INFO | Perf report for the last 5.13602 seconds: | 20260508 10:53:23.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:23.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:53:23.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:53:23.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:23.231 | INFO | Uploaded 36GiB to GPU in 2.31733s (15.5351GiB/s) | 20260508 10:53:23.231 | INFO | Downloaded 34GiB to GPU in 2.5019s (13.5897GiB/s) | 20260508 10:53:28.370 | INFO | Analyzing chunk frames = [7539200,7539712) | 20260508 10:53:28.370 | INFO | MemInfo: { | 20260508 10:53:28.370 | INFO | "gpu" : " 17 GB", | 20260508 10:53:28.370 | INFO | "huge" : "", | 20260508 10:53:28.370 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:28.370 | INFO | "malloc" : " 14 GB", | 20260508 10:53:28.370 | INFO | "pinned" : " 29 GB" | 20260508 10:53:28.370 | INFO | } | 20260508 10:53:28.370 | INFO | Perf report for the last 5.13932 seconds: | 20260508 10:53:28.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:28.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:53:28.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:53:28.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:28.370 | INFO | Uploaded 36GiB to GPU in 2.38464s (15.0966GiB/s) | 20260508 10:53:28.370 | INFO | Downloaded 34GiB to GPU in 2.44408s (13.9112GiB/s) | 20260508 10:53:33.509 | INFO | Analyzing chunk frames = [7539712,7540224) | 20260508 10:53:33.509 | INFO | MemInfo: { | 20260508 10:53:33.509 | INFO | "gpu" : " 17 GB", | 20260508 10:53:33.509 | INFO | "huge" : "", | 20260508 10:53:33.509 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:33.509 | INFO | "malloc" : " 14 GB", | 20260508 10:53:33.509 | INFO | "pinned" : " 29 GB" | 20260508 10:53:33.509 | INFO | } | 20260508 10:53:33.509 | INFO | Perf report for the last 5.1388 seconds: | 20260508 10:53:33.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:33.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:53:33.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:53:33.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:33.509 | INFO | Uploaded 36GiB to GPU in 2.42619s (14.8381GiB/s) | 20260508 10:53:33.509 | INFO | Downloaded 34GiB to GPU in 2.44255s (13.9199GiB/s) | 20260508 10:53:38.650 | INFO | Analyzing chunk frames = [7540224,7540736) | 20260508 10:53:38.650 | INFO | MemInfo: { | 20260508 10:53:38.650 | INFO | "gpu" : " 17 GB", | 20260508 10:53:38.650 | INFO | "huge" : "", | 20260508 10:53:38.650 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:38.650 | INFO | "malloc" : " 14 GB", | 20260508 10:53:38.650 | INFO | "pinned" : " 29 GB" | 20260508 10:53:38.650 | INFO | } | 20260508 10:53:38.650 | INFO | Perf report for the last 5.14084 seconds: | 20260508 10:53:38.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:38.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 10:53:38.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:53:38.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:38.650 | INFO | Uploaded 36GiB to GPU in 2.40995s (14.9381GiB/s) | 20260508 10:53:38.651 | INFO | Downloaded 34GiB to GPU in 2.4343s (13.9671GiB/s) | 20260508 10:53:38.747 | INFO | Starting a round of Baz IO | 20260508 10:53:42.154 | INFO | Populating RTMetrics report | 20260508 10:53:42.154 | INFO | RTMetrics report populated | 20260508 10:53:42.154 | INFO | Opening tmp RTMetrics json file | 20260508 10:53:42.162 | INFO | Serializing RTMetrics to json | 20260508 10:53:42.163 | INFO | Serialization done, swapping tmp file now... | 20260508 10:53:42.164 | INFO | RTMetrics done | 20260508 10:53:42.224 | INFO | Finished a round of Baz IO: Wrote 1.45 GiB in 3.477 seconds | 20260508 10:53:42.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.00% / 59.84% | 20260508 10:53:42.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.208986% / 3.97364e-06% / 5.16574e-05% | 20260508 10:53:42.224 | INFO | Baz IO overall: | 20260508 10:53:42.224 | INFO | BytesWritten : 3263780303765 | 20260508 10:53:42.224 | INFO | NumEvents : 1568244029487 | 20260508 10:53:43.791 | INFO | Analyzing chunk frames = [7540736,7541248) | 20260508 10:53:43.791 | INFO | MemInfo: { | 20260508 10:53:43.791 | INFO | "gpu" : " 17 GB", | 20260508 10:53:43.791 | INFO | "huge" : "", | 20260508 10:53:43.791 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:43.791 | INFO | "malloc" : " 14 GB", | 20260508 10:53:43.791 | INFO | "pinned" : " 29 GB" | 20260508 10:53:43.791 | INFO | } | 20260508 10:53:43.791 | INFO | Perf report for the last 5.14151 seconds: | 20260508 10:53:43.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:43.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:43.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:43.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:43.791 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 10:53:43.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:43.791 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:53:43.791 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 10:53:43.791 | INFO | BAZWRITER : 66.8%, 2.8, 97.4ms, 0 | 20260508 10:53:43.791 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:53:43.792 | INFO | Uploaded 36GiB to GPU in 2.5735s (13.9887GiB/s) | 20260508 10:53:43.792 | INFO | Downloaded 38GiB to GPU in 2.7138s (14.0025GiB/s) | 20260508 10:53:48.931 | INFO | Analyzing chunk frames = [7541248,7541760) | 20260508 10:53:48.931 | INFO | MemInfo: { | 20260508 10:53:48.931 | INFO | "gpu" : " 17 GB", | 20260508 10:53:48.931 | INFO | "huge" : "", | 20260508 10:53:48.931 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:48.932 | INFO | "malloc" : " 14 GB", | 20260508 10:53:48.932 | INFO | "pinned" : " 29 GB" | 20260508 10:53:48.932 | INFO | } | 20260508 10:53:48.932 | INFO | Perf report for the last 5.14063 seconds: | 20260508 10:53:48.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:48.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.932 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:53:48.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.932 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 10:53:48.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:48.933 | INFO | Uploaded 36GiB to GPU in 2.44736s (14.7097GiB/s) | 20260508 10:53:48.933 | INFO | Downloaded 34GiB to GPU in 2.4655s (13.7903GiB/s) | 20260508 10:53:54.063 | INFO | Analyzing chunk frames = [7541760,7542272) | 20260508 10:53:54.063 | INFO | MemInfo: { | 20260508 10:53:54.063 | INFO | "gpu" : " 17 GB", | 20260508 10:53:54.063 | INFO | "huge" : "", | 20260508 10:53:54.063 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:54.063 | INFO | "malloc" : " 14 GB", | 20260508 10:53:54.063 | INFO | "pinned" : " 29 GB" | 20260508 10:53:54.063 | INFO | } | 20260508 10:53:54.063 | INFO | Perf report for the last 5.13118 seconds: | 20260508 10:53:54.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:54.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 10:53:54.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:53:54.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:54.063 | INFO | Uploaded 36GiB to GPU in 2.47694s (14.5341GiB/s) | 20260508 10:53:54.064 | INFO | Downloaded 34GiB to GPU in 2.45566s (13.8456GiB/s) | 20260508 10:53:59.208 | INFO | Analyzing chunk frames = [7542272,7542784) | 20260508 10:53:59.208 | INFO | MemInfo: { | 20260508 10:53:59.208 | INFO | "gpu" : " 17 GB", | 20260508 10:53:59.208 | INFO | "huge" : "", | 20260508 10:53:59.208 | INFO | "huge_shared" : " 29 GB", | 20260508 10:53:59.208 | INFO | "malloc" : " 14 GB", | 20260508 10:53:59.208 | INFO | "pinned" : " 29 GB" | 20260508 10:53:59.209 | INFO | } | 20260508 10:53:59.209 | INFO | Perf report for the last 5.1455 seconds: | 20260508 10:53:59.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:53:59.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 10:53:59.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:53:59.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:53:59.209 | INFO | Uploaded 36GiB to GPU in 2.47804s (14.5276GiB/s) | 20260508 10:53:59.209 | INFO | Downloaded 34GiB to GPU in 2.44307s (13.9169GiB/s) | 20260508 10:54:04.352 | INFO | Analyzing chunk frames = [7542784,7543296) | 20260508 10:54:04.352 | INFO | MemInfo: { | 20260508 10:54:04.352 | INFO | "gpu" : " 17 GB", | 20260508 10:54:04.352 | INFO | "huge" : "", | 20260508 10:54:04.352 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:04.352 | INFO | "malloc" : " 14 GB", | 20260508 10:54:04.352 | INFO | "pinned" : " 29 GB" | 20260508 10:54:04.352 | INFO | } | 20260508 10:54:04.352 | INFO | Perf report for the last 5.14376 seconds: | 20260508 10:54:04.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:04.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.352 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 10:54:04.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.352 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:54:04.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:04.353 | INFO | Uploaded 36GiB to GPU in 2.27425s (15.8294GiB/s) | 20260508 10:54:04.353 | INFO | Downloaded 34GiB to GPU in 2.4818s (13.6997GiB/s) | 20260508 10:54:09.486 | INFO | Analyzing chunk frames = [7543296,7543808) | 20260508 10:54:09.486 | INFO | MemInfo: { | 20260508 10:54:09.486 | INFO | "gpu" : " 17 GB", | 20260508 10:54:09.486 | INFO | "huge" : "", | 20260508 10:54:09.486 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:09.486 | INFO | "malloc" : " 14 GB", | 20260508 10:54:09.486 | INFO | "pinned" : " 29 GB" | 20260508 10:54:09.486 | INFO | } | 20260508 10:54:09.486 | INFO | Perf report for the last 5.13405 seconds: | 20260508 10:54:09.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:09.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.486 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 10:54:09.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.487 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:54:09.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:09.487 | INFO | Uploaded 36GiB to GPU in 2.39023s (15.0613GiB/s) | 20260508 10:54:09.487 | INFO | Downloaded 34GiB to GPU in 2.43136s (13.984GiB/s) | 20260508 10:54:14.624 | INFO | Analyzing chunk frames = [7543808,7544320) | 20260508 10:54:14.624 | INFO | MemInfo: { | 20260508 10:54:14.624 | INFO | "gpu" : " 17 GB", | 20260508 10:54:14.624 | INFO | "huge" : "", | 20260508 10:54:14.624 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:14.624 | INFO | "malloc" : " 14 GB", | 20260508 10:54:14.624 | INFO | "pinned" : " 29 GB" | 20260508 10:54:14.625 | INFO | } | 20260508 10:54:14.625 | INFO | Perf report for the last 5.13815 seconds: | 20260508 10:54:14.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:14.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 10:54:14.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:54:14.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:14.625 | INFO | Uploaded 36GiB to GPU in 2.42633s (14.8372GiB/s) | 20260508 10:54:14.625 | INFO | Downloaded 34GiB to GPU in 2.43489s (13.9636GiB/s) | 20260508 10:54:19.765 | INFO | Analyzing chunk frames = [7544320,7544832) | 20260508 10:54:19.765 | INFO | MemInfo: { | 20260508 10:54:19.765 | INFO | "gpu" : " 17 GB", | 20260508 10:54:19.765 | INFO | "huge" : "", | 20260508 10:54:19.765 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:19.765 | INFO | "malloc" : " 14 GB", | 20260508 10:54:19.765 | INFO | "pinned" : " 29 GB" | 20260508 10:54:19.765 | INFO | } | 20260508 10:54:19.765 | INFO | Perf report for the last 5.14074 seconds: | 20260508 10:54:19.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:19.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.765 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 10:54:19.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.765 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:54:19.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:19.766 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:54:19.766 | INFO | Total mem usage: 16.7 GiB | 20260508 10:54:19.766 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:54:19.766 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260508 10:54:19.766 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:54:19.766 | INFO | High Water Breakdown: | 20260508 10:54:19.766 | INFO | 60.2% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:54:19.766 | INFO | 39.6% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:54:19.766 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:54:19.766 | INFO | ------------------------------------------ | 20260508 10:54:19.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:54:19.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:54:19.766 | INFO | Total mem usage: 26.5 GiB | 20260508 10:54:19.766 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:54:19.766 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:54:19.766 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:54:19.766 | INFO | High Water Breakdown: | 20260508 10:54:19.766 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:54:19.766 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:54:19.766 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:54:19.766 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:54:19.766 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:54:19.766 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:54:19.766 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:54:19.766 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:54:19.766 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:54:19.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:54:19.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:54:19.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:54:19.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:54:19.766 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:54:19.766 | INFO | ------------------------------------------ | 20260508 10:54:19.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:54:19.766 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:54:19.766 | INFO | Total mem usage: 27.8 GiB | 20260508 10:54:19.766 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:54:19.766 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:54:19.766 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:54:19.766 | INFO | High Water Breakdown: | 20260508 10:54:19.766 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:54:19.766 | INFO | ------------------------------------------ | 20260508 10:54:19.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:54:19.766 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:54:19.766 | INFO | Total mem usage: 15.7 GiB | 20260508 10:54:19.766 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:54:19.766 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:54:19.766 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:54:19.766 | INFO | High Water Breakdown: | 20260508 10:54:19.766 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:54:19.766 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:54:19.766 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:54:19.766 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:54:19.766 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:54:19.766 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:54:19.766 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:54:19.766 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:54:19.766 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:54:19.766 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:54:19.766 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:54:19.766 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:54:19.766 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:54:19.766 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:54:19.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:54:19.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:54:19.766 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:54:19.766 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:54:19.766 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:54:19.766 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:54:19.766 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:54:19.766 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:54:19.766 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:54:19.766 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:54:19.766 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:54:19.766 | INFO | ------------------------------------------ | 20260508 10:54:19.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:54:19.766 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:54:19.766 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:54:19.766 | INFO | Uploaded 36GiB to GPU in 2.43466s (14.7864GiB/s) | 20260508 10:54:19.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:54:19.767 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.86e-01 | 20260508 10:54:19.767 | INFO | Downloaded 34GiB to GPU in 2.44914s (13.8824GiB/s) | 20260508 10:54:19.767 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.02e-01, 1.09e-02 / 5.82e-03 | 20260508 10:54:19.767 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.46e+00, 5.93e-01 / 2.26e-01 | 20260508 10:54:19.767 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.22e-01 | 20260508 10:54:19.767 | INFO | PerfReport: 16.07%, DME -- 4.20e+01 / 4.54e+01, 3.76e+00 / 1.15e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.76e+00 / 1.38e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.07e-01 / 2.52e-01 | 20260508 10:54:19.767 | INFO | PerfReport: 3.02%, Metrics -- 9.86e-01 / 1.13e+00, 2.44e-01 / 1.19e-01 | 20260508 10:54:19.767 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.31e-01, 4.89e-02 / 3.64e-02 | 20260508 10:54:19.767 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:54:19.767 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:54:19.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:54:19.767 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 24.61%, Download -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 2.19%, LockUpload -- 2.30e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 2.56%, LockDownload -- 2.70e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 15.31%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:54:19.767 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260508 10:54:19.864 | INFO | Starting a round of Baz IO | 20260508 10:54:23.267 | INFO | Populating RTMetrics report | 20260508 10:54:23.267 | INFO | RTMetrics report populated | 20260508 10:54:23.267 | INFO | Opening tmp RTMetrics json file | 20260508 10:54:23.276 | INFO | Serializing RTMetrics to json | 20260508 10:54:23.278 | INFO | Serialization done, swapping tmp file now... | 20260508 10:54:23.278 | INFO | RTMetrics done | 20260508 10:54:23.337 | INFO | Finished a round of Baz IO: Wrote 1.45 GiB in 3.472 seconds | 20260508 10:54:23.337 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.84% | 20260508 10:54:23.337 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209212% / 0% / 5.96046e-05% | 20260508 10:54:23.337 | INFO | Baz IO overall: | 20260508 10:54:23.337 | INFO | BytesWritten : 3265337646082 | 20260508 10:54:23.337 | INFO | NumEvents : 1568977979574 | 20260508 10:54:24.906 | INFO | Analyzing chunk frames = [7544832,7545344) | 20260508 10:54:24.906 | INFO | MemInfo: { | 20260508 10:54:24.906 | INFO | "gpu" : " 17 GB", | 20260508 10:54:24.906 | INFO | "huge" : "", | 20260508 10:54:24.906 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:24.906 | INFO | "malloc" : " 14 GB", | 20260508 10:54:24.906 | INFO | "pinned" : " 29 GB" | 20260508 10:54:24.906 | INFO | } | 20260508 10:54:24.906 | INFO | Perf report for the last 5.14043 seconds: | 20260508 10:54:24.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:24.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:24.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:24.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:24.906 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 10:54:24.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:24.906 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:54:24.906 | INFO | PRE_HQ : 65.5%, 1.6, 54.2ms, 0 | 20260508 10:54:24.906 | INFO | BAZWRITER : 66.9%, 2.5, 86.1ms, 0 | 20260508 10:54:24.906 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:54:24.906 | INFO | Uploaded 36GiB to GPU in 2.60185s (13.8363GiB/s) | 20260508 10:54:24.906 | INFO | Downloaded 38GiB to GPU in 2.72821s (13.9285GiB/s) | 20260508 10:54:30.046 | INFO | Analyzing chunk frames = [7545344,7545856) | 20260508 10:54:30.046 | INFO | MemInfo: { | 20260508 10:54:30.046 | INFO | "gpu" : " 17 GB", | 20260508 10:54:30.046 | INFO | "huge" : "", | 20260508 10:54:30.046 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:30.046 | INFO | "malloc" : " 14 GB", | 20260508 10:54:30.046 | INFO | "pinned" : " 29 GB" | 20260508 10:54:30.046 | INFO | } | 20260508 10:54:30.046 | INFO | Perf report for the last 5.14043 seconds: | 20260508 10:54:30.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:30.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.046 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:54:30.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.046 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 10:54:30.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:30.047 | INFO | Uploaded 36GiB to GPU in 2.44013s (14.7533GiB/s) | 20260508 10:54:30.047 | INFO | Downloaded 34GiB to GPU in 2.46257s (13.8067GiB/s) | 20260508 10:54:35.181 | INFO | Analyzing chunk frames = [7545856,7546368) | 20260508 10:54:35.181 | INFO | MemInfo: { | 20260508 10:54:35.181 | INFO | "gpu" : " 17 GB", | 20260508 10:54:35.181 | INFO | "huge" : "", | 20260508 10:54:35.181 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:35.181 | INFO | "malloc" : " 14 GB", | 20260508 10:54:35.181 | INFO | "pinned" : " 29 GB" | 20260508 10:54:35.181 | INFO | } | 20260508 10:54:35.181 | INFO | Perf report for the last 5.1346 seconds: | 20260508 10:54:35.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:35.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 10:54:35.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:54:35.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:35.181 | INFO | Uploaded 36GiB to GPU in 2.47389s (14.552GiB/s) | 20260508 10:54:35.181 | INFO | Downloaded 34GiB to GPU in 2.42974s (13.9933GiB/s) | 20260508 10:54:40.318 | INFO | Analyzing chunk frames = [7546368,7546880) | 20260508 10:54:40.318 | INFO | MemInfo: { | 20260508 10:54:40.318 | INFO | "gpu" : " 17 GB", | 20260508 10:54:40.318 | INFO | "huge" : "", | 20260508 10:54:40.318 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:40.318 | INFO | "malloc" : " 14 GB", | 20260508 10:54:40.318 | INFO | "pinned" : " 29 GB" | 20260508 10:54:40.318 | INFO | } | 20260508 10:54:40.318 | INFO | Perf report for the last 5.13739 seconds: | 20260508 10:54:40.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:40.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.318 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 10:54:40.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.318 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 10:54:40.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:40.319 | INFO | Uploaded 36GiB to GPU in 2.46312s (14.6156GiB/s) | 20260508 10:54:40.319 | INFO | Downloaded 34GiB to GPU in 2.41347s (14.0876GiB/s) | 20260508 10:54:45.463 | INFO | Analyzing chunk frames = [7546880,7547392) | 20260508 10:54:45.463 | INFO | MemInfo: { | 20260508 10:54:45.463 | INFO | "gpu" : " 17 GB", | 20260508 10:54:45.463 | INFO | "huge" : "", | 20260508 10:54:45.463 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:45.463 | INFO | "malloc" : " 14 GB", | 20260508 10:54:45.463 | INFO | "pinned" : " 29 GB" | 20260508 10:54:45.463 | INFO | } | 20260508 10:54:45.463 | INFO | Perf report for the last 5.14449 seconds: | 20260508 10:54:45.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:45.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 10:54:45.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 10:54:45.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:45.463 | INFO | Uploaded 36GiB to GPU in 2.25883s (15.9375GiB/s) | 20260508 10:54:45.463 | INFO | Downloaded 34GiB to GPU in 2.48518s (13.6811GiB/s) | 20260508 10:54:50.606 | INFO | Analyzing chunk frames = [7547392,7547904) | 20260508 10:54:50.606 | INFO | MemInfo: { | 20260508 10:54:50.606 | INFO | "gpu" : " 17 GB", | 20260508 10:54:50.606 | INFO | "huge" : "", | 20260508 10:54:50.606 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:50.606 | INFO | "malloc" : " 14 GB", | 20260508 10:54:50.606 | INFO | "pinned" : " 29 GB" | 20260508 10:54:50.606 | INFO | } | 20260508 10:54:50.606 | INFO | Perf report for the last 5.1434 seconds: | 20260508 10:54:50.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:50.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 10:54:50.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:54:50.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:50.606 | INFO | Uploaded 36GiB to GPU in 2.38123s (15.1182GiB/s) | 20260508 10:54:50.606 | INFO | Downloaded 34GiB to GPU in 2.41875s (14.0568GiB/s) | 20260508 10:54:55.742 | INFO | Analyzing chunk frames = [7547904,7548416) | 20260508 10:54:55.742 | INFO | MemInfo: { | 20260508 10:54:55.742 | INFO | "gpu" : " 17 GB", | 20260508 10:54:55.742 | INFO | "huge" : "", | 20260508 10:54:55.742 | INFO | "huge_shared" : " 29 GB", | 20260508 10:54:55.742 | INFO | "malloc" : " 14 GB", | 20260508 10:54:55.742 | INFO | "pinned" : " 29 GB" | 20260508 10:54:55.743 | INFO | } | 20260508 10:54:55.743 | INFO | Perf report for the last 5.13642 seconds: | 20260508 10:54:55.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:54:55.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 10:54:55.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 10:54:55.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:54:55.743 | INFO | Uploaded 36GiB to GPU in 2.44947s (14.6971GiB/s) | 20260508 10:54:55.743 | INFO | Downloaded 34GiB to GPU in 2.43818s (13.9448GiB/s) | 20260508 10:55:00.876 | INFO | Analyzing chunk frames = [7548416,7548928) | 20260508 10:55:00.876 | INFO | MemInfo: { | 20260508 10:55:00.876 | INFO | "gpu" : " 17 GB", | 20260508 10:55:00.876 | INFO | "huge" : "", | 20260508 10:55:00.876 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:00.876 | INFO | "malloc" : " 14 GB", | 20260508 10:55:00.876 | INFO | "pinned" : " 29 GB" | 20260508 10:55:00.876 | INFO | } | 20260508 10:55:00.877 | INFO | Perf report for the last 5.13389 seconds: | 20260508 10:55:00.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:00.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 10:55:00.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:55:00.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:00.877 | INFO | Uploaded 36GiB to GPU in 2.44115s (14.7471GiB/s) | 20260508 10:55:00.877 | INFO | Downloaded 34GiB to GPU in 2.42111s (14.0432GiB/s) | 20260508 10:55:00.974 | INFO | Starting a round of Baz IO | 20260508 10:55:04.380 | INFO | Populating RTMetrics report | 20260508 10:55:04.380 | INFO | RTMetrics report populated | 20260508 10:55:04.380 | INFO | Opening tmp RTMetrics json file | 20260508 10:55:04.390 | INFO | Serializing RTMetrics to json | 20260508 10:55:04.391 | INFO | Serialization done, swapping tmp file now... | 20260508 10:55:04.392 | INFO | RTMetrics done | 20260508 10:55:04.451 | INFO | Finished a round of Baz IO: Wrote 1.449 GiB in 3.477 seconds | 20260508 10:55:04.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:55:04.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207913% / 7.94729e-06% / 7.15256e-05% | 20260508 10:55:04.451 | INFO | Baz IO overall: | 20260508 10:55:04.451 | INFO | BytesWritten : 3266893628020 | 20260508 10:55:04.451 | INFO | NumEvents : 1569711152256 | 20260508 10:55:06.021 | INFO | Analyzing chunk frames = [7548928,7549440) | 20260508 10:55:06.021 | INFO | MemInfo: { | 20260508 10:55:06.021 | INFO | "gpu" : " 17 GB", | 20260508 10:55:06.021 | INFO | "huge" : "", | 20260508 10:55:06.021 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:06.021 | INFO | "malloc" : " 14 GB", | 20260508 10:55:06.021 | INFO | "pinned" : " 29 GB" | 20260508 10:55:06.021 | INFO | } | 20260508 10:55:06.021 | INFO | Perf report for the last 5.14456 seconds: | 20260508 10:55:06.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:06.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:06.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:06.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:06.021 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 10:55:06.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:06.021 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:55:06.021 | INFO | PRE_HQ : 65.0%, 1.6, 54.0ms, 0 | 20260508 10:55:06.021 | INFO | BAZWRITER : 66.9%, 2.5, 85.7ms, 0 | 20260508 10:55:06.021 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:55:06.022 | INFO | Uploaded 36GiB to GPU in 2.58395s (13.9322GiB/s) | 20260508 10:55:06.022 | INFO | Downloaded 38GiB to GPU in 2.70182s (14.0646GiB/s) | 20260508 10:55:11.155 | INFO | Analyzing chunk frames = [7549440,7549952) | 20260508 10:55:11.155 | INFO | MemInfo: { | 20260508 10:55:11.155 | INFO | "gpu" : " 17 GB", | 20260508 10:55:11.155 | INFO | "huge" : "", | 20260508 10:55:11.156 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:11.156 | INFO | "malloc" : " 14 GB", | 20260508 10:55:11.156 | INFO | "pinned" : " 29 GB" | 20260508 10:55:11.156 | INFO | } | 20260508 10:55:11.156 | INFO | Perf report for the last 5.13463 seconds: | 20260508 10:55:11.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:11.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:55:11.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:55:11.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:11.156 | INFO | Uploaded 36GiB to GPU in 2.45227s (14.6803GiB/s) | 20260508 10:55:11.156 | INFO | Downloaded 34GiB to GPU in 2.45595s (13.8439GiB/s) | 20260508 10:55:16.297 | INFO | Analyzing chunk frames = [7549952,7550464) | 20260508 10:55:16.297 | INFO | MemInfo: { | 20260508 10:55:16.297 | INFO | "gpu" : " 17 GB", | 20260508 10:55:16.297 | INFO | "huge" : "", | 20260508 10:55:16.297 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:16.297 | INFO | "malloc" : " 14 GB", | 20260508 10:55:16.297 | INFO | "pinned" : " 29 GB" | 20260508 10:55:16.297 | INFO | } | 20260508 10:55:16.297 | INFO | Perf report for the last 5.14102 seconds: | 20260508 10:55:16.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:16.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 10:55:16.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:55:16.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:16.297 | INFO | Uploaded 36GiB to GPU in 2.50628s (14.3639GiB/s) | 20260508 10:55:16.297 | INFO | Downloaded 34GiB to GPU in 2.4399s (13.935GiB/s) | 20260508 10:55:21.436 | INFO | Analyzing chunk frames = [7550464,7550976) | 20260508 10:55:21.436 | INFO | MemInfo: { | 20260508 10:55:21.436 | INFO | "gpu" : " 17 GB", | 20260508 10:55:21.436 | INFO | "huge" : "", | 20260508 10:55:21.436 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:21.436 | INFO | "malloc" : " 14 GB", | 20260508 10:55:21.436 | INFO | "pinned" : " 29 GB" | 20260508 10:55:21.436 | INFO | } | 20260508 10:55:21.436 | INFO | Perf report for the last 5.13939 seconds: | 20260508 10:55:21.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:21.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.436 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:55:21.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.436 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:55:21.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:21.437 | INFO | Uploaded 36GiB to GPU in 2.48971s (14.4595GiB/s) | 20260508 10:55:21.437 | INFO | Downloaded 34GiB to GPU in 2.4198s (14.0508GiB/s) | 20260508 10:55:26.577 | INFO | Analyzing chunk frames = [7550976,7551488) | 20260508 10:55:26.577 | INFO | MemInfo: { | 20260508 10:55:26.577 | INFO | "gpu" : " 17 GB", | 20260508 10:55:26.577 | INFO | "huge" : "", | 20260508 10:55:26.577 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:26.577 | INFO | "malloc" : " 14 GB", | 20260508 10:55:26.577 | INFO | "pinned" : " 29 GB" | 20260508 10:55:26.577 | INFO | } | 20260508 10:55:26.577 | INFO | Perf report for the last 5.14063 seconds: | 20260508 10:55:26.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:26.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:55:26.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:55:26.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:26.577 | INFO | Uploaded 36GiB to GPU in 2.30234s (15.6363GiB/s) | 20260508 10:55:26.577 | INFO | Downloaded 34GiB to GPU in 2.4954s (13.6251GiB/s) | 20260508 10:55:31.716 | INFO | Analyzing chunk frames = [7551488,7552000) | 20260508 10:55:31.716 | INFO | MemInfo: { | 20260508 10:55:31.716 | INFO | "gpu" : " 17 GB", | 20260508 10:55:31.716 | INFO | "huge" : "", | 20260508 10:55:31.716 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:31.716 | INFO | "malloc" : " 14 GB", | 20260508 10:55:31.716 | INFO | "pinned" : " 29 GB" | 20260508 10:55:31.716 | INFO | } | 20260508 10:55:31.716 | INFO | Perf report for the last 5.13885 seconds: | 20260508 10:55:31.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:31.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 10:55:31.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:55:31.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:31.716 | INFO | Uploaded 36GiB to GPU in 2.40581s (14.9637GiB/s) | 20260508 10:55:31.716 | INFO | Downloaded 34GiB to GPU in 2.42239s (14.0357GiB/s) | 20260508 10:55:36.855 | INFO | Analyzing chunk frames = [7552000,7552512) | 20260508 10:55:36.855 | INFO | MemInfo: { | 20260508 10:55:36.855 | INFO | "gpu" : " 17 GB", | 20260508 10:55:36.855 | INFO | "huge" : "", | 20260508 10:55:36.855 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:36.855 | INFO | "malloc" : " 14 GB", | 20260508 10:55:36.855 | INFO | "pinned" : " 29 GB" | 20260508 10:55:36.855 | INFO | } | 20260508 10:55:36.855 | INFO | Perf report for the last 5.13914 seconds: | 20260508 10:55:36.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:36.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 10:55:36.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:55:36.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:36.855 | INFO | Uploaded 36GiB to GPU in 2.43541s (14.7819GiB/s) | 20260508 10:55:36.856 | INFO | Downloaded 34GiB to GPU in 2.43919s (13.9391GiB/s) | 20260508 10:55:41.996 | INFO | Analyzing chunk frames = [7552512,7553024) | 20260508 10:55:41.996 | INFO | MemInfo: { | 20260508 10:55:41.996 | INFO | "gpu" : " 17 GB", | 20260508 10:55:41.996 | INFO | "huge" : "", | 20260508 10:55:41.996 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:41.996 | INFO | "malloc" : " 14 GB", | 20260508 10:55:41.996 | INFO | "pinned" : " 29 GB" | 20260508 10:55:41.996 | INFO | } | 20260508 10:55:41.996 | INFO | Perf report for the last 5.14099 seconds: | 20260508 10:55:41.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:41.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:55:41.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:55:41.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:41.996 | INFO | Uploaded 36GiB to GPU in 2.46522s (14.6031GiB/s) | 20260508 10:55:41.996 | INFO | Downloaded 34GiB to GPU in 2.45382s (13.856GiB/s) | 20260508 10:55:41.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:55:41.996 | INFO | Total mem usage: 16.7 GiB | 20260508 10:55:41.996 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:55:41.996 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:55:41.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:55:41.996 | INFO | High Water Breakdown: | 20260508 10:55:41.996 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:55:41.996 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:55:41.996 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:55:41.996 | INFO | ------------------------------------------ | 20260508 10:55:41.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:55:41.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:55:41.996 | INFO | Total mem usage: 26.5 GiB | 20260508 10:55:41.996 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:55:41.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:55:41.996 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:55:41.996 | INFO | High Water Breakdown: | 20260508 10:55:41.996 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:55:41.996 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:55:41.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:55:41.996 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:55:41.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:55:41.996 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:55:41.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:55:41.996 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:55:41.996 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:55:41.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:55:41.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:55:41.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:55:41.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:55:41.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:55:41.997 | INFO | ------------------------------------------ | 20260508 10:55:41.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:55:41.997 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:55:41.997 | INFO | Total mem usage: 27.8 GiB | 20260508 10:55:41.997 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:55:41.997 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 10:55:41.997 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 10:55:41.997 | INFO | High Water Breakdown: | 20260508 10:55:41.997 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 10:55:41.997 | INFO | ------------------------------------------ | 20260508 10:55:41.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:55:41.997 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:55:41.997 | INFO | Total mem usage: 15.7 GiB | 20260508 10:55:41.997 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:55:41.997 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:55:41.997 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:55:41.997 | INFO | High Water Breakdown: | 20260508 10:55:41.997 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:55:41.997 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:55:41.997 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:55:41.997 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:55:41.997 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:55:41.997 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:55:41.997 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:55:41.997 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:55:41.997 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:55:41.997 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:55:41.997 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:55:41.997 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:55:41.997 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:55:41.997 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:55:41.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:55:41.997 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:55:41.997 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:55:41.997 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:55:41.997 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:55:41.997 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:55:41.997 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:55:41.997 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:55:41.997 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:55:41.997 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:55:41.997 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:55:41.997 | INFO | ------------------------------------------ | 20260508 10:55:41.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:55:41.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:55:41.997 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:55:41.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:55:41.997 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.77e-01 | 20260508 10:55:41.997 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.15e-02 / 3.95e-03 | 20260508 10:55:41.997 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.48e+00, 6.00e-01 / 1.90e-01 | 20260508 10:55:41.997 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.50e+00, 2.11e-01 / 8.67e-02 | 20260508 10:55:41.997 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.54e+01, 3.77e+00 / 8.83e-01 | 20260508 10:55:41.997 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.76e+00 / 1.27e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.35e+00 / 2.55e+00, 4.09e-01 / 2.62e-01 | 20260508 10:55:41.997 | INFO | PerfReport: 3.02%, Metrics -- 9.86e-01 / 1.14e+00, 2.45e-01 / 1.25e-01 | 20260508 10:55:41.997 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.30e-01, 4.86e-02 / 2.78e-02 | 20260508 10:55:41.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:55:41.997 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:55:41.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:55:41.997 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 24.24%, Upload -- 2.55e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 24.54%, Download -- 2.58e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 2.21%, LockUpload -- 2.33e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 2.55%, LockDownload -- 2.69e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 10:55:41.997 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 4.10e-02 / 0.00e+00 | 20260508 10:55:42.093 | INFO | Starting a round of Baz IO | 20260508 10:55:45.506 | INFO | Populating RTMetrics report | 20260508 10:55:45.506 | INFO | RTMetrics report populated | 20260508 10:55:45.506 | INFO | Opening tmp RTMetrics json file | 20260508 10:55:45.514 | INFO | Serializing RTMetrics to json | 20260508 10:55:45.516 | INFO | Serialization done, swapping tmp file now... | 20260508 10:55:45.516 | INFO | RTMetrics done | 20260508 10:55:45.576 | INFO | Finished a round of Baz IO: Wrote 1.449 GiB in 3.482 seconds | 20260508 10:55:45.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:55:45.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.208342% / 0% / 6.35783e-05% | 20260508 10:55:45.576 | INFO | Baz IO overall: | 20260508 10:55:45.576 | INFO | BytesWritten : 3268449474465 | 20260508 10:55:45.576 | INFO | NumEvents : 1570444243165 | 20260508 10:55:47.130 | INFO | Analyzing chunk frames = [7553024,7553536) | 20260508 10:55:47.130 | INFO | MemInfo: { | 20260508 10:55:47.130 | INFO | "gpu" : " 17 GB", | 20260508 10:55:47.130 | INFO | "huge" : "", | 20260508 10:55:47.130 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:47.130 | INFO | "malloc" : " 14 GB", | 20260508 10:55:47.130 | INFO | "pinned" : " 29 GB" | 20260508 10:55:47.130 | INFO | } | 20260508 10:55:47.130 | INFO | Perf report for the last 5.13397 seconds: | 20260508 10:55:47.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:47.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:47.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:47.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:47.130 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 10:55:47.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:47.130 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:55:47.130 | INFO | PRE_HQ : 65.6%, 1.6, 54.2ms, 0 | 20260508 10:55:47.130 | INFO | BAZWRITER : 67.1%, 2.5, 86.3ms, 0 | 20260508 10:55:47.130 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 10:55:47.130 | INFO | Uploaded 36GiB to GPU in 2.59271s (13.8851GiB/s) | 20260508 10:55:47.130 | INFO | Downloaded 38GiB to GPU in 2.71296s (14.0069GiB/s) | 20260508 10:55:52.272 | INFO | Analyzing chunk frames = [7553536,7554048) | 20260508 10:55:52.272 | INFO | MemInfo: { | 20260508 10:55:52.272 | INFO | "gpu" : " 17 GB", | 20260508 10:55:52.272 | INFO | "huge" : "", | 20260508 10:55:52.272 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:52.272 | INFO | "malloc" : " 14 GB", | 20260508 10:55:52.272 | INFO | "pinned" : " 29 GB" | 20260508 10:55:52.272 | INFO | } | 20260508 10:55:52.272 | INFO | Perf report for the last 5.14229 seconds: | 20260508 10:55:52.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:52.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.272 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:55:52.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.272 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:55:52.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:52.273 | INFO | Uploaded 36GiB to GPU in 2.451s (14.6879GiB/s) | 20260508 10:55:52.273 | INFO | Downloaded 34GiB to GPU in 2.47296s (13.7487GiB/s) | 20260508 10:55:57.416 | INFO | Analyzing chunk frames = [7554048,7554560) | 20260508 10:55:57.416 | INFO | MemInfo: { | 20260508 10:55:57.416 | INFO | "gpu" : " 17 GB", | 20260508 10:55:57.416 | INFO | "huge" : "", | 20260508 10:55:57.416 | INFO | "huge_shared" : " 29 GB", | 20260508 10:55:57.416 | INFO | "malloc" : " 14 GB", | 20260508 10:55:57.416 | INFO | "pinned" : " 29 GB" | 20260508 10:55:57.416 | INFO | } | 20260508 10:55:57.416 | INFO | Perf report for the last 5.1439 seconds: | 20260508 10:55:57.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:55:57.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 10:55:57.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:55:57.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:55:57.416 | INFO | Uploaded 36GiB to GPU in 2.46507s (14.6041GiB/s) | 20260508 10:55:57.416 | INFO | Downloaded 34GiB to GPU in 2.42641s (14.0125GiB/s) | 20260508 10:56:02.557 | INFO | Analyzing chunk frames = [7554560,7555072) | 20260508 10:56:02.557 | INFO | MemInfo: { | 20260508 10:56:02.557 | INFO | "gpu" : " 17 GB", | 20260508 10:56:02.557 | INFO | "huge" : "", | 20260508 10:56:02.557 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:02.557 | INFO | "malloc" : " 14 GB", | 20260508 10:56:02.557 | INFO | "pinned" : " 29 GB" | 20260508 10:56:02.557 | INFO | } | 20260508 10:56:02.557 | INFO | Perf report for the last 5.14087 seconds: | 20260508 10:56:02.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:02.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 10:56:02.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:56:02.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:02.557 | INFO | Uploaded 36GiB to GPU in 2.51821s (14.2958GiB/s) | 20260508 10:56:02.557 | INFO | Downloaded 34GiB to GPU in 2.45298s (13.8607GiB/s) | 20260508 10:56:07.692 | INFO | Analyzing chunk frames = [7555072,7555584) | 20260508 10:56:07.692 | INFO | MemInfo: { | 20260508 10:56:07.692 | INFO | "gpu" : " 17 GB", | 20260508 10:56:07.692 | INFO | "huge" : "", | 20260508 10:56:07.692 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:07.692 | INFO | "malloc" : " 14 GB", | 20260508 10:56:07.692 | INFO | "pinned" : " 29 GB" | 20260508 10:56:07.693 | INFO | } | 20260508 10:56:07.693 | INFO | Perf report for the last 5.13564 seconds: | 20260508 10:56:07.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:07.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:56:07.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 10:56:07.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:07.693 | INFO | Uploaded 36GiB to GPU in 2.27896s (15.7967GiB/s) | 20260508 10:56:07.693 | INFO | Downloaded 34GiB to GPU in 2.51218s (13.5341GiB/s) | 20260508 10:56:12.826 | INFO | Analyzing chunk frames = [7555584,7556096) | 20260508 10:56:12.826 | INFO | MemInfo: { | 20260508 10:56:12.826 | INFO | "gpu" : " 17 GB", | 20260508 10:56:12.826 | INFO | "huge" : "", | 20260508 10:56:12.826 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:12.826 | INFO | "malloc" : " 14 GB", | 20260508 10:56:12.826 | INFO | "pinned" : " 29 GB" | 20260508 10:56:12.826 | INFO | } | 20260508 10:56:12.826 | INFO | Perf report for the last 5.13309 seconds: | 20260508 10:56:12.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:12.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 10:56:12.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:56:12.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:12.826 | INFO | Uploaded 36GiB to GPU in 2.3886s (15.0716GiB/s) | 20260508 10:56:12.826 | INFO | Downloaded 34GiB to GPU in 2.44496s (13.9062GiB/s) | 20260508 10:56:17.971 | INFO | Analyzing chunk frames = [7556096,7556608) | 20260508 10:56:17.971 | INFO | MemInfo: { | 20260508 10:56:17.971 | INFO | "gpu" : " 17 GB", | 20260508 10:56:17.971 | INFO | "huge" : "", | 20260508 10:56:17.971 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:17.971 | INFO | "malloc" : " 14 GB", | 20260508 10:56:17.971 | INFO | "pinned" : " 29 GB" | 20260508 10:56:17.971 | INFO | } | 20260508 10:56:17.971 | INFO | Perf report for the last 5.14516 seconds: | 20260508 10:56:17.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:17.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.971 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 10:56:17.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.971 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:56:17.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:17.972 | INFO | Uploaded 36GiB to GPU in 2.41237s (14.9231GiB/s) | 20260508 10:56:17.972 | INFO | Downloaded 34GiB to GPU in 2.4361s (13.9568GiB/s) | 20260508 10:56:23.108 | INFO | Analyzing chunk frames = [7556608,7557120) | 20260508 10:56:23.108 | INFO | MemInfo: { | 20260508 10:56:23.108 | INFO | "gpu" : " 17 GB", | 20260508 10:56:23.108 | INFO | "huge" : "", | 20260508 10:56:23.108 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:23.108 | INFO | "malloc" : " 14 GB", | 20260508 10:56:23.108 | INFO | "pinned" : " 29 GB" | 20260508 10:56:23.108 | INFO | } | 20260508 10:56:23.108 | INFO | Perf report for the last 5.13691 seconds: | 20260508 10:56:23.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:23.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 10:56:23.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 10:56:23.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:23.108 | INFO | Uploaded 36GiB to GPU in 2.39698s (15.0189GiB/s) | 20260508 10:56:23.109 | INFO | Downloaded 34GiB to GPU in 2.42755s (14.0059GiB/s) | 20260508 10:56:23.205 | INFO | Starting a round of Baz IO | 20260508 10:56:26.630 | INFO | Populating RTMetrics report | 20260508 10:56:26.630 | INFO | RTMetrics report populated | 20260508 10:56:26.630 | INFO | Opening tmp RTMetrics json file | 20260508 10:56:26.637 | INFO | Serializing RTMetrics to json | 20260508 10:56:26.639 | INFO | Serialization done, swapping tmp file now... | 20260508 10:56:26.639 | INFO | RTMetrics done | 20260508 10:56:26.697 | INFO | Finished a round of Baz IO: Wrote 1.451 GiB in 3.491 seconds | 20260508 10:56:26.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:56:26.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.208763% / 3.97364e-06% / 5.5631e-05% | 20260508 10:56:26.697 | INFO | Baz IO overall: | 20260508 10:56:26.697 | INFO | BytesWritten : 3270007257087 | 20260508 10:56:26.697 | INFO | NumEvents : 1571178829431 | 20260508 10:56:28.254 | INFO | Analyzing chunk frames = [7557120,7557632) | 20260508 10:56:28.254 | INFO | MemInfo: { | 20260508 10:56:28.254 | INFO | "gpu" : " 17 GB", | 20260508 10:56:28.254 | INFO | "huge" : "", | 20260508 10:56:28.254 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:28.254 | INFO | "malloc" : " 14 GB", | 20260508 10:56:28.254 | INFO | "pinned" : " 29 GB" | 20260508 10:56:28.254 | INFO | } | 20260508 10:56:28.254 | INFO | Perf report for the last 5.14597 seconds: | 20260508 10:56:28.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:28.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:28.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:28.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:28.254 | INFO | ANALYSIS : 67.9%, 3.0, 105.9ms, 0 | 20260508 10:56:28.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:28.254 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 10:56:28.254 | INFO | PRE_HQ : 65.4%, 1.6, 54.3ms, 0 | 20260508 10:56:28.254 | INFO | BAZWRITER : 67.1%, 2.5, 86.3ms, 0 | 20260508 10:56:28.254 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:56:28.254 | INFO | Uploaded 36GiB to GPU in 2.53703s (14.1898GiB/s) | 20260508 10:56:28.254 | INFO | Downloaded 38GiB to GPU in 2.71598s (13.9912GiB/s) | 20260508 10:56:33.389 | INFO | Analyzing chunk frames = [7557632,7558144) | 20260508 10:56:33.389 | INFO | MemInfo: { | 20260508 10:56:33.389 | INFO | "gpu" : " 17 GB", | 20260508 10:56:33.389 | INFO | "huge" : "", | 20260508 10:56:33.389 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:33.389 | INFO | "malloc" : " 14 GB", | 20260508 10:56:33.390 | INFO | "pinned" : " 29 GB" | 20260508 10:56:33.390 | INFO | } | 20260508 10:56:33.390 | INFO | Perf report for the last 5.13591 seconds: | 20260508 10:56:33.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:33.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 10:56:33.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:56:33.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:33.390 | INFO | Uploaded 36GiB to GPU in 2.42023s (14.8746GiB/s) | 20260508 10:56:33.390 | INFO | Downloaded 34GiB to GPU in 2.458s (13.8324GiB/s) | 20260508 10:56:38.530 | INFO | Analyzing chunk frames = [7558144,7558656) | 20260508 10:56:38.530 | INFO | MemInfo: { | 20260508 10:56:38.530 | INFO | "gpu" : " 17 GB", | 20260508 10:56:38.530 | INFO | "huge" : "", | 20260508 10:56:38.530 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:38.530 | INFO | "malloc" : " 14 GB", | 20260508 10:56:38.531 | INFO | "pinned" : " 29 GB" | 20260508 10:56:38.531 | INFO | } | 20260508 10:56:38.531 | INFO | Perf report for the last 5.14085 seconds: | 20260508 10:56:38.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:38.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:56:38.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:56:38.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:38.531 | INFO | Uploaded 36GiB to GPU in 2.48144s (14.5077GiB/s) | 20260508 10:56:38.531 | INFO | Downloaded 34GiB to GPU in 2.44168s (13.9248GiB/s) | 20260508 10:56:43.669 | INFO | Analyzing chunk frames = [7558656,7559168) | 20260508 10:56:43.669 | INFO | MemInfo: { | 20260508 10:56:43.669 | INFO | "gpu" : " 17 GB", | 20260508 10:56:43.669 | INFO | "huge" : "", | 20260508 10:56:43.669 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:43.669 | INFO | "malloc" : " 14 GB", | 20260508 10:56:43.670 | INFO | "pinned" : " 29 GB" | 20260508 10:56:43.670 | INFO | } | 20260508 10:56:43.670 | INFO | Perf report for the last 5.13903 seconds: | 20260508 10:56:43.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:43.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.670 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:56:43.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.670 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:56:43.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:43.671 | INFO | Uploaded 36GiB to GPU in 2.47153s (14.5659GiB/s) | 20260508 10:56:43.671 | INFO | Downloaded 34GiB to GPU in 2.46549s (13.7904GiB/s) | 20260508 10:56:48.804 | INFO | Analyzing chunk frames = [7559168,7559680) | 20260508 10:56:48.804 | INFO | MemInfo: { | 20260508 10:56:48.804 | INFO | "gpu" : " 17 GB", | 20260508 10:56:48.804 | INFO | "huge" : "", | 20260508 10:56:48.804 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:48.804 | INFO | "malloc" : " 14 GB", | 20260508 10:56:48.804 | INFO | "pinned" : " 29 GB" | 20260508 10:56:48.804 | INFO | } | 20260508 10:56:48.804 | INFO | Perf report for the last 5.13474 seconds: | 20260508 10:56:48.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:48.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.804 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 10:56:48.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.805 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 10:56:48.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:48.805 | INFO | Uploaded 36GiB to GPU in 2.27553s (15.8205GiB/s) | 20260508 10:56:48.805 | INFO | Downloaded 34GiB to GPU in 2.51335s (13.5278GiB/s) | 20260508 10:56:53.943 | INFO | Analyzing chunk frames = [7559680,7560192) | 20260508 10:56:53.943 | INFO | MemInfo: { | 20260508 10:56:53.943 | INFO | "gpu" : " 17 GB", | 20260508 10:56:53.943 | INFO | "huge" : "", | 20260508 10:56:53.943 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:53.943 | INFO | "malloc" : " 14 GB", | 20260508 10:56:53.943 | INFO | "pinned" : " 29 GB" | 20260508 10:56:53.943 | INFO | } | 20260508 10:56:53.943 | INFO | Perf report for the last 5.13865 seconds: | 20260508 10:56:53.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:53.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 10:56:53.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:56:53.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:53.943 | INFO | Uploaded 36GiB to GPU in 2.28395s (15.7622GiB/s) | 20260508 10:56:53.943 | INFO | Downloaded 34GiB to GPU in 2.41062s (14.1042GiB/s) | 20260508 10:56:59.085 | INFO | Analyzing chunk frames = [7560192,7560704) | 20260508 10:56:59.085 | INFO | MemInfo: { | 20260508 10:56:59.085 | INFO | "gpu" : " 17 GB", | 20260508 10:56:59.085 | INFO | "huge" : "", | 20260508 10:56:59.085 | INFO | "huge_shared" : " 29 GB", | 20260508 10:56:59.085 | INFO | "malloc" : " 14 GB", | 20260508 10:56:59.085 | INFO | "pinned" : " 29 GB" | 20260508 10:56:59.085 | INFO | } | 20260508 10:56:59.085 | INFO | Perf report for the last 5.14237 seconds: | 20260508 10:56:59.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:56:59.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.086 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 10:56:59.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.086 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 10:56:59.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:56:59.086 | INFO | Uploaded 36GiB to GPU in 2.41733s (14.8925GiB/s) | 20260508 10:56:59.086 | INFO | Downloaded 34GiB to GPU in 2.44351s (13.9144GiB/s) | 20260508 10:57:04.227 | INFO | Analyzing chunk frames = [7560704,7561216) | 20260508 10:57:04.227 | INFO | MemInfo: { | 20260508 10:57:04.227 | INFO | "gpu" : " 17 GB", | 20260508 10:57:04.227 | INFO | "huge" : "", | 20260508 10:57:04.227 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:04.227 | INFO | "malloc" : " 14 GB", | 20260508 10:57:04.227 | INFO | "pinned" : " 29 GB" | 20260508 10:57:04.227 | INFO | } | 20260508 10:57:04.227 | INFO | Perf report for the last 5.14151 seconds: | 20260508 10:57:04.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:04.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:57:04.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:57:04.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:04.227 | INFO | Uploaded 36GiB to GPU in 2.43614s (14.7775GiB/s) | 20260508 10:57:04.227 | INFO | Downloaded 34GiB to GPU in 2.44974s (13.879GiB/s) | 20260508 10:57:04.227 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:57:04.227 | INFO | Total mem usage: 16.7 GiB | 20260508 10:57:04.227 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:57:04.227 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 10:57:04.227 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:57:04.227 | INFO | High Water Breakdown: | 20260508 10:57:04.227 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:57:04.227 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:57:04.228 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:57:04.228 | INFO | ------------------------------------------ | 20260508 10:57:04.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:57:04.228 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:57:04.228 | INFO | Total mem usage: 26.5 GiB | 20260508 10:57:04.228 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:57:04.228 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:57:04.228 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:57:04.228 | INFO | High Water Breakdown: | 20260508 10:57:04.228 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:57:04.228 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:57:04.228 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:57:04.228 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:57:04.228 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:57:04.228 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:57:04.228 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:57:04.228 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:57:04.228 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:57:04.228 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:57:04.228 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:57:04.228 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:57:04.228 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:57:04.228 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:57:04.228 | INFO | ------------------------------------------ | 20260508 10:57:04.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:57:04.228 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:57:04.228 | INFO | Total mem usage: 27.8 GiB | 20260508 10:57:04.228 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:57:04.228 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:57:04.228 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:57:04.228 | INFO | High Water Breakdown: | 20260508 10:57:04.228 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:57:04.228 | INFO | ------------------------------------------ | 20260508 10:57:04.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:57:04.228 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:57:04.228 | INFO | Total mem usage: 15.7 GiB | 20260508 10:57:04.228 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:57:04.228 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:57:04.228 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:57:04.228 | INFO | High Water Breakdown: | 20260508 10:57:04.228 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:57:04.228 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:57:04.228 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:57:04.228 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:57:04.228 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:57:04.228 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:57:04.228 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:57:04.228 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:57:04.228 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:57:04.228 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:57:04.228 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:57:04.228 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:57:04.228 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:57:04.228 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:57:04.228 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:57:04.228 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:57:04.228 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:57:04.228 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:57:04.228 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:57:04.228 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:57:04.228 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:57:04.228 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:57:04.228 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:57:04.228 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:57:04.228 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:57:04.228 | INFO | ------------------------------------------ | 20260508 10:57:04.228 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:57:04.228 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:57:04.228 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:57:04.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:57:04.228 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.50e-01 | 20260508 10:57:04.228 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.09e-02 / 4.62e-03 | 20260508 10:57:04.228 | INFO | PerfReport: 12.22%, Baseline -- 4.00e+00 / 4.48e+00, 5.97e-01 / 2.27e-01 | 20260508 10:57:04.228 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.46e+00, 2.12e-01 / 1.23e-01 | 20260508 10:57:04.228 | INFO | PerfReport: 16.04%, DME -- 4.20e+01 / 4.57e+01, 3.89e+00 / 1.05e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.80e+00 / 9.43e-01 | 20260508 10:57:04.228 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.35e+00 / 2.56e+00, 4.09e-01 / 2.36e-01 | 20260508 10:57:04.228 | INFO | PerfReport: 3.01%, Metrics -- 9.86e-01 / 1.13e+00, 2.47e-01 / 1.30e-01 | 20260508 10:57:04.228 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.31e-01, 4.91e-02 / 2.79e-02 | 20260508 10:57:04.228 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:57:04.228 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:57:04.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:57:04.228 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 24.05%, Upload -- 2.53e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 24.65%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 31.15%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 2.63%, LockDownload -- 2.76e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 15.39%, LockCompute -- 1.62e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 10:57:04.228 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260508 10:57:04.325 | INFO | Starting a round of Baz IO | 20260508 10:57:07.729 | INFO | Populating RTMetrics report | 20260508 10:57:07.729 | INFO | RTMetrics report populated | 20260508 10:57:07.729 | INFO | Opening tmp RTMetrics json file | 20260508 10:57:07.740 | INFO | Serializing RTMetrics to json | 20260508 10:57:07.741 | INFO | Serialization done, swapping tmp file now... | 20260508 10:57:07.742 | INFO | RTMetrics done | 20260508 10:57:07.798 | INFO | Finished a round of Baz IO: Wrote 1.449 GiB in 3.473 seconds | 20260508 10:57:07.798 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:57:07.798 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207345% / 3.97364e-06% / 4.76837e-05% | 20260508 10:57:07.798 | INFO | Baz IO overall: | 20260508 10:57:07.798 | INFO | BytesWritten : 3271563426649 | 20260508 10:57:07.798 | INFO | NumEvents : 1571912242461 | 20260508 10:57:09.364 | INFO | Analyzing chunk frames = [7561216,7561728) | 20260508 10:57:09.364 | INFO | MemInfo: { | 20260508 10:57:09.364 | INFO | "gpu" : " 17 GB", | 20260508 10:57:09.364 | INFO | "huge" : "", | 20260508 10:57:09.364 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:09.364 | INFO | "malloc" : " 14 GB", | 20260508 10:57:09.364 | INFO | "pinned" : " 29 GB" | 20260508 10:57:09.364 | INFO | } | 20260508 10:57:09.364 | INFO | Perf report for the last 5.13681 seconds: | 20260508 10:57:09.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:09.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:09.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:09.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:09.364 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 10:57:09.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:09.364 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260508 10:57:09.364 | INFO | PRE_HQ : 65.9%, 1.7, 57.7ms, 0 | 20260508 10:57:09.364 | INFO | BAZWRITER : 67.2%, 2.6, 88.5ms, 0 | 20260508 10:57:09.364 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:57:09.364 | INFO | Uploaded 36GiB to GPU in 2.62631s (13.7075GiB/s) | 20260508 10:57:09.364 | INFO | Downloaded 38GiB to GPU in 2.70544s (14.0458GiB/s) | 20260508 10:57:14.502 | INFO | Analyzing chunk frames = [7561728,7562240) | 20260508 10:57:14.502 | INFO | MemInfo: { | 20260508 10:57:14.502 | INFO | "gpu" : " 17 GB", | 20260508 10:57:14.502 | INFO | "huge" : "", | 20260508 10:57:14.502 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:14.502 | INFO | "malloc" : " 14 GB", | 20260508 10:57:14.502 | INFO | "pinned" : " 29 GB" | 20260508 10:57:14.502 | INFO | } | 20260508 10:57:14.502 | INFO | Perf report for the last 5.13823 seconds: | 20260508 10:57:14.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:14.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.502 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 10:57:14.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.502 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:57:14.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:14.503 | INFO | Uploaded 36GiB to GPU in 2.48942s (14.4612GiB/s) | 20260508 10:57:14.503 | INFO | Downloaded 34GiB to GPU in 2.4586s (13.829GiB/s) | 20260508 10:57:19.634 | INFO | Analyzing chunk frames = [7562240,7562752) | 20260508 10:57:19.634 | INFO | MemInfo: { | 20260508 10:57:19.634 | INFO | "gpu" : " 17 GB", | 20260508 10:57:19.634 | INFO | "huge" : "", | 20260508 10:57:19.634 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:19.634 | INFO | "malloc" : " 14 GB", | 20260508 10:57:19.634 | INFO | "pinned" : " 29 GB" | 20260508 10:57:19.634 | INFO | } | 20260508 10:57:19.635 | INFO | Perf report for the last 5.13249 seconds: | 20260508 10:57:19.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:19.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 10:57:19.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:57:19.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:19.635 | INFO | Uploaded 36GiB to GPU in 2.50274s (14.3842GiB/s) | 20260508 10:57:19.635 | INFO | Downloaded 34GiB to GPU in 2.42715s (14.0082GiB/s) | 20260508 10:57:24.781 | INFO | Analyzing chunk frames = [7562752,7563264) | 20260508 10:57:24.781 | INFO | MemInfo: { | 20260508 10:57:24.781 | INFO | "gpu" : " 17 GB", | 20260508 10:57:24.781 | INFO | "huge" : "", | 20260508 10:57:24.781 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:24.781 | INFO | "malloc" : " 14 GB", | 20260508 10:57:24.781 | INFO | "pinned" : " 29 GB" | 20260508 10:57:24.781 | INFO | } | 20260508 10:57:24.781 | INFO | Perf report for the last 5.14677 seconds: | 20260508 10:57:24.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:24.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.781 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 10:57:24.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.781 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:57:24.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:24.782 | INFO | Uploaded 36GiB to GPU in 2.51001s (14.3426GiB/s) | 20260508 10:57:24.782 | INFO | Downloaded 34GiB to GPU in 2.42128s (14.0421GiB/s) | 20260508 10:57:29.922 | INFO | Analyzing chunk frames = [7563264,7563776) | 20260508 10:57:29.922 | INFO | MemInfo: { | 20260508 10:57:29.922 | INFO | "gpu" : " 17 GB", | 20260508 10:57:29.922 | INFO | "huge" : "", | 20260508 10:57:29.922 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:29.922 | INFO | "malloc" : " 14 GB", | 20260508 10:57:29.922 | INFO | "pinned" : " 29 GB" | 20260508 10:57:29.922 | INFO | } | 20260508 10:57:29.922 | INFO | Perf report for the last 5.14068 seconds: | 20260508 10:57:29.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:29.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 10:57:29.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:57:29.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:29.922 | INFO | Uploaded 36GiB to GPU in 2.29499s (15.6864GiB/s) | 20260508 10:57:29.922 | INFO | Downloaded 34GiB to GPU in 2.491s (13.6491GiB/s) | 20260508 10:57:35.062 | INFO | Analyzing chunk frames = [7563776,7564288) | 20260508 10:57:35.062 | INFO | MemInfo: { | 20260508 10:57:35.062 | INFO | "gpu" : " 17 GB", | 20260508 10:57:35.062 | INFO | "huge" : "", | 20260508 10:57:35.062 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:35.062 | INFO | "malloc" : " 14 GB", | 20260508 10:57:35.062 | INFO | "pinned" : " 29 GB" | 20260508 10:57:35.062 | INFO | } | 20260508 10:57:35.062 | INFO | Perf report for the last 5.14 seconds: | 20260508 10:57:35.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:35.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 10:57:35.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:57:35.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:35.062 | INFO | Uploaded 36GiB to GPU in 2.35721s (15.2723GiB/s) | 20260508 10:57:35.062 | INFO | Downloaded 34GiB to GPU in 2.42873s (13.9991GiB/s) | 20260508 10:57:40.200 | INFO | Analyzing chunk frames = [7564288,7564800) | 20260508 10:57:40.200 | INFO | MemInfo: { | 20260508 10:57:40.200 | INFO | "gpu" : " 17 GB", | 20260508 10:57:40.200 | INFO | "huge" : "", | 20260508 10:57:40.200 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:40.200 | INFO | "malloc" : " 14 GB", | 20260508 10:57:40.200 | INFO | "pinned" : " 29 GB" | 20260508 10:57:40.200 | INFO | } | 20260508 10:57:40.200 | INFO | Perf report for the last 5.13826 seconds: | 20260508 10:57:40.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:40.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.200 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 10:57:40.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.201 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 10:57:40.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:40.201 | INFO | Uploaded 36GiB to GPU in 2.42138s (14.8675GiB/s) | 20260508 10:57:40.201 | INFO | Downloaded 34GiB to GPU in 2.44195s (13.9233GiB/s) | 20260508 10:57:45.340 | INFO | Analyzing chunk frames = [7564800,7565312) | 20260508 10:57:45.340 | INFO | MemInfo: { | 20260508 10:57:45.340 | INFO | "gpu" : " 17 GB", | 20260508 10:57:45.341 | INFO | "huge" : "", | 20260508 10:57:45.341 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:45.341 | INFO | "malloc" : " 14 GB", | 20260508 10:57:45.341 | INFO | "pinned" : " 29 GB" | 20260508 10:57:45.341 | INFO | } | 20260508 10:57:45.341 | INFO | Perf report for the last 5.14035 seconds: | 20260508 10:57:45.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:45.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 10:57:45.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260508 10:57:45.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:45.341 | INFO | Uploaded 36GiB to GPU in 2.44984s (14.6948GiB/s) | 20260508 10:57:45.341 | INFO | Downloaded 34GiB to GPU in 2.45715s (13.8371GiB/s) | 20260508 10:57:45.445 | INFO | Starting a round of Baz IO | 20260508 10:57:48.865 | INFO | Populating RTMetrics report | 20260508 10:57:48.865 | INFO | RTMetrics report populated | 20260508 10:57:48.865 | INFO | Opening tmp RTMetrics json file | 20260508 10:57:48.872 | INFO | Serializing RTMetrics to json | 20260508 10:57:48.878 | INFO | Serialization done, swapping tmp file now... | 20260508 10:57:48.879 | INFO | RTMetrics done | 20260508 10:57:48.940 | INFO | Finished a round of Baz IO: Wrote 1.447 GiB in 3.495 seconds | 20260508 10:57:48.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:57:48.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207186% / 0% / 4.76837e-05% | 20260508 10:57:48.941 | INFO | Baz IO overall: | 20260508 10:57:48.941 | INFO | BytesWritten : 3273117499631 | 20260508 10:57:48.941 | INFO | NumEvents : 1572644379207 | 20260508 10:57:50.479 | INFO | Analyzing chunk frames = [7565312,7565824) | 20260508 10:57:50.479 | INFO | MemInfo: { | 20260508 10:57:50.479 | INFO | "gpu" : " 17 GB", | 20260508 10:57:50.479 | INFO | "huge" : "", | 20260508 10:57:50.479 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:50.479 | INFO | "malloc" : " 14 GB", | 20260508 10:57:50.479 | INFO | "pinned" : " 29 GB" | 20260508 10:57:50.479 | INFO | } | 20260508 10:57:50.479 | INFO | Perf report for the last 5.13805 seconds: | 20260508 10:57:50.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:50.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:50.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:50.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:50.479 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 10:57:50.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:50.479 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 10:57:50.479 | INFO | PRE_HQ : 66.1%, 1.6, 54.3ms, 0 | 20260508 10:57:50.479 | INFO | BAZWRITER : 67.6%, 2.5, 85.5ms, 0 | 20260508 10:57:50.479 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:57:50.479 | INFO | Uploaded 36GiB to GPU in 2.49985s (14.4009GiB/s) | 20260508 10:57:50.479 | INFO | Downloaded 38GiB to GPU in 2.72514s (13.9442GiB/s) | 20260508 10:57:55.618 | INFO | Analyzing chunk frames = [7565824,7566336) | 20260508 10:57:55.618 | INFO | MemInfo: { | 20260508 10:57:55.618 | INFO | "gpu" : " 17 GB", | 20260508 10:57:55.618 | INFO | "huge" : "", | 20260508 10:57:55.618 | INFO | "huge_shared" : " 29 GB", | 20260508 10:57:55.618 | INFO | "malloc" : " 14 GB", | 20260508 10:57:55.618 | INFO | "pinned" : " 29 GB" | 20260508 10:57:55.618 | INFO | } | 20260508 10:57:55.618 | INFO | Perf report for the last 5.1392 seconds: | 20260508 10:57:55.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:57:55.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 10:57:55.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 10:57:55.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:57:55.618 | INFO | Uploaded 36GiB to GPU in 2.43394s (14.7908GiB/s) | 20260508 10:57:55.619 | INFO | Downloaded 34GiB to GPU in 2.48494s (13.6824GiB/s) | 20260508 10:58:00.762 | INFO | Analyzing chunk frames = [7566336,7566848) | 20260508 10:58:00.762 | INFO | MemInfo: { | 20260508 10:58:00.762 | INFO | "gpu" : " 17 GB", | 20260508 10:58:00.762 | INFO | "huge" : "", | 20260508 10:58:00.762 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:00.762 | INFO | "malloc" : " 14 GB", | 20260508 10:58:00.762 | INFO | "pinned" : " 29 GB" | 20260508 10:58:00.762 | INFO | } | 20260508 10:58:00.762 | INFO | Perf report for the last 5.14452 seconds: | 20260508 10:58:00.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:00.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 10:58:00.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 10:58:00.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:00.763 | INFO | Uploaded 36GiB to GPU in 2.48449s (14.4899GiB/s) | 20260508 10:58:00.763 | INFO | Downloaded 34GiB to GPU in 2.46532s (13.7913GiB/s) | 20260508 10:58:05.894 | INFO | Analyzing chunk frames = [7566848,7567360) | 20260508 10:58:05.894 | INFO | MemInfo: { | 20260508 10:58:05.894 | INFO | "gpu" : " 17 GB", | 20260508 10:58:05.894 | INFO | "huge" : "", | 20260508 10:58:05.894 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:05.894 | INFO | "malloc" : " 14 GB", | 20260508 10:58:05.894 | INFO | "pinned" : " 29 GB" | 20260508 10:58:05.894 | INFO | } | 20260508 10:58:05.894 | INFO | Perf report for the last 5.13197 seconds: | 20260508 10:58:05.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:05.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.895 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 10:58:05.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.895 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 10:58:05.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:05.895 | INFO | Uploaded 36GiB to GPU in 2.44133s (14.7461GiB/s) | 20260508 10:58:05.895 | INFO | Downloaded 34GiB to GPU in 2.47326s (13.747GiB/s) | 20260508 10:58:11.039 | INFO | Analyzing chunk frames = [7567360,7567872) | 20260508 10:58:11.039 | INFO | MemInfo: { | 20260508 10:58:11.039 | INFO | "gpu" : " 17 GB", | 20260508 10:58:11.039 | INFO | "huge" : "", | 20260508 10:58:11.039 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:11.039 | INFO | "malloc" : " 14 GB", | 20260508 10:58:11.039 | INFO | "pinned" : " 29 GB" | 20260508 10:58:11.039 | INFO | } | 20260508 10:58:11.039 | INFO | Perf report for the last 5.14488 seconds: | 20260508 10:58:11.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:11.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.039 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 10:58:11.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.039 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 10:58:11.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:11.040 | INFO | Uploaded 36GiB to GPU in 2.44047s (14.7512GiB/s) | 20260508 10:58:11.040 | INFO | Downloaded 34GiB to GPU in 2.54109s (13.3801GiB/s) | 20260508 10:58:16.175 | INFO | Analyzing chunk frames = [7567872,7568384) | 20260508 10:58:16.175 | INFO | MemInfo: { | 20260508 10:58:16.175 | INFO | "gpu" : " 17 GB", | 20260508 10:58:16.175 | INFO | "huge" : "", | 20260508 10:58:16.175 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:16.175 | INFO | "malloc" : " 14 GB", | 20260508 10:58:16.175 | INFO | "pinned" : " 29 GB" | 20260508 10:58:16.175 | INFO | } | 20260508 10:58:16.175 | INFO | Perf report for the last 5.1358 seconds: | 20260508 10:58:16.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:16.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.175 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 10:58:16.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.175 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:58:16.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:16.176 | INFO | Uploaded 36GiB to GPU in 2.35456s (15.2895GiB/s) | 20260508 10:58:16.176 | INFO | Downloaded 34GiB to GPU in 2.45126s (13.8704GiB/s) | 20260508 10:58:21.312 | INFO | Analyzing chunk frames = [7568384,7568896) | 20260508 10:58:21.312 | INFO | MemInfo: { | 20260508 10:58:21.312 | INFO | "gpu" : " 17 GB", | 20260508 10:58:21.312 | INFO | "huge" : "", | 20260508 10:58:21.312 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:21.312 | INFO | "malloc" : " 14 GB", | 20260508 10:58:21.312 | INFO | "pinned" : " 29 GB" | 20260508 10:58:21.312 | INFO | } | 20260508 10:58:21.312 | INFO | Perf report for the last 5.13706 seconds: | 20260508 10:58:21.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:21.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.312 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 10:58:21.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.312 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 10:58:21.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:21.313 | INFO | Uploaded 36GiB to GPU in 2.36004s (15.254GiB/s) | 20260508 10:58:21.313 | INFO | Downloaded 34GiB to GPU in 2.45846s (13.8298GiB/s) | 20260508 10:58:26.452 | INFO | Analyzing chunk frames = [7568896,7569408) | 20260508 10:58:26.452 | INFO | MemInfo: { | 20260508 10:58:26.452 | INFO | "gpu" : " 17 GB", | 20260508 10:58:26.452 | INFO | "huge" : "", | 20260508 10:58:26.452 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:26.452 | INFO | "malloc" : " 14 GB", | 20260508 10:58:26.452 | INFO | "pinned" : " 29 GB" | 20260508 10:58:26.452 | INFO | } | 20260508 10:58:26.452 | INFO | Perf report for the last 5.14003 seconds: | 20260508 10:58:26.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:26.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.452 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 10:58:26.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.453 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 10:58:26.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:26.453 | INFO | Uploaded 36GiB to GPU in 2.3913s (15.0546GiB/s) | 20260508 10:58:26.453 | INFO | Downloaded 34GiB to GPU in 2.47316s (13.7476GiB/s) | 20260508 10:58:26.453 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:58:26.453 | INFO | Total mem usage: 16.7 GiB | 20260508 10:58:26.453 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:58:26.453 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:58:26.453 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:58:26.453 | INFO | High Water Breakdown: | 20260508 10:58:26.453 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:58:26.453 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:58:26.453 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:58:26.453 | INFO | ------------------------------------------ | 20260508 10:58:26.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:58:26.453 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:58:26.453 | INFO | Total mem usage: 26.5 GiB | 20260508 10:58:26.453 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:58:26.453 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 10:58:26.453 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 10:58:26.453 | INFO | High Water Breakdown: | 20260508 10:58:26.453 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:58:26.453 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:58:26.453 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:58:26.453 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:58:26.453 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:58:26.453 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:58:26.453 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:58:26.453 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:58:26.453 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:58:26.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:58:26.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:58:26.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:58:26.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:58:26.453 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:58:26.453 | INFO | ------------------------------------------ | 20260508 10:58:26.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:58:26.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:58:26.454 | INFO | Total mem usage: 27.8 GiB | 20260508 10:58:26.454 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:58:26.454 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:58:26.454 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:58:26.454 | INFO | High Water Breakdown: | 20260508 10:58:26.454 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:58:26.454 | INFO | ------------------------------------------ | 20260508 10:58:26.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:58:26.454 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:58:26.454 | INFO | Total mem usage: 15.7 GiB | 20260508 10:58:26.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:58:26.454 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:58:26.454 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:58:26.454 | INFO | High Water Breakdown: | 20260508 10:58:26.454 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:58:26.454 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:58:26.454 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:58:26.454 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:58:26.454 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:58:26.454 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:58:26.454 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:58:26.454 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:58:26.454 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:58:26.454 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:58:26.454 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:58:26.454 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:58:26.454 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:58:26.454 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:58:26.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:58:26.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:58:26.454 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:58:26.454 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:58:26.454 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:58:26.454 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:58:26.454 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:58:26.454 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:58:26.454 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:58:26.454 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:58:26.454 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:58:26.454 | INFO | ------------------------------------------ | 20260508 10:58:26.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:58:26.454 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:58:26.454 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:58:26.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:58:26.454 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 8.34e-01 | 20260508 10:58:26.454 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.06e-01, 1.09e-02 / 6.58e-03 | 20260508 10:58:26.454 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.47e+00, 5.98e-01 / 1.86e-01 | 20260508 10:58:26.454 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.11e-01 / 1.23e-01 | 20260508 10:58:26.454 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.53e+01, 3.54e+00 / 9.74e-01 | 20260508 10:58:26.454 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.82e+00 / 1.14e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.54e+00, 4.08e-01 / 2.27e-01 | 20260508 10:58:26.454 | INFO | PerfReport: 3.01%, Metrics -- 9.84e-01 / 1.15e+00, 2.46e-01 / 9.61e-02 | 20260508 10:58:26.454 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.38e-01, 4.85e-02 / 2.62e-02 | 20260508 10:58:26.454 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:58:26.454 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:58:26.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:58:26.454 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 24.17%, Upload -- 2.54e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 24.69%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 2.08%, LockUpload -- 2.19e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 2.68%, LockDownload -- 2.81e+00 / 0.00e+00, 4.28e+00 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 15.29%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 10:58:26.454 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 10:58:26.550 | INFO | Starting a round of Baz IO | 20260508 10:58:29.969 | INFO | Populating RTMetrics report | 20260508 10:58:29.970 | INFO | RTMetrics report populated | 20260508 10:58:29.970 | INFO | Opening tmp RTMetrics json file | 20260508 10:58:29.979 | INFO | Serializing RTMetrics to json | 20260508 10:58:29.981 | INFO | Serialization done, swapping tmp file now... | 20260508 10:58:29.981 | INFO | RTMetrics done | 20260508 10:58:30.035 | INFO | Finished a round of Baz IO: Wrote 1.447 GiB in 3.485 seconds | 20260508 10:58:30.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:58:30.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205672% / 3.97364e-06% / 5.16574e-05% | 20260508 10:58:30.035 | INFO | Baz IO overall: | 20260508 10:58:30.035 | INFO | BytesWritten : 3274670872389 | 20260508 10:58:30.035 | INFO | NumEvents : 1573375946521 | 20260508 10:58:31.588 | INFO | Analyzing chunk frames = [7569408,7569920) | 20260508 10:58:31.588 | INFO | MemInfo: { | 20260508 10:58:31.588 | INFO | "gpu" : " 17 GB", | 20260508 10:58:31.588 | INFO | "huge" : "", | 20260508 10:58:31.588 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:31.588 | INFO | "malloc" : " 14 GB", | 20260508 10:58:31.588 | INFO | "pinned" : " 29 GB" | 20260508 10:58:31.588 | INFO | } | 20260508 10:58:31.589 | INFO | Perf report for the last 5.13622 seconds: | 20260508 10:58:31.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:31.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:31.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:31.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:31.589 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:58:31.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:31.589 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:58:31.589 | INFO | PRE_HQ : 65.7%, 1.6, 54.3ms, 0 | 20260508 10:58:31.589 | INFO | BAZWRITER : 67.2%, 2.5, 86.2ms, 0 | 20260508 10:58:31.589 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:58:31.589 | INFO | Uploaded 36GiB to GPU in 2.5848s (13.9276GiB/s) | 20260508 10:58:31.589 | INFO | Downloaded 38GiB to GPU in 2.71159s (14.0139GiB/s) | 20260508 10:58:36.735 | INFO | Analyzing chunk frames = [7569920,7570432) | 20260508 10:58:36.735 | INFO | MemInfo: { | 20260508 10:58:36.735 | INFO | "gpu" : " 17 GB", | 20260508 10:58:36.735 | INFO | "huge" : "", | 20260508 10:58:36.735 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:36.735 | INFO | "malloc" : " 14 GB", | 20260508 10:58:36.735 | INFO | "pinned" : " 29 GB" | 20260508 10:58:36.735 | INFO | } | 20260508 10:58:36.735 | INFO | Perf report for the last 5.1467 seconds: | 20260508 10:58:36.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:36.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 10:58:36.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 10:58:36.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:36.735 | INFO | Uploaded 36GiB to GPU in 2.47977s (14.5175GiB/s) | 20260508 10:58:36.735 | INFO | Downloaded 34GiB to GPU in 2.46855s (13.7732GiB/s) | 20260508 10:58:41.873 | INFO | Analyzing chunk frames = [7570432,7570944) | 20260508 10:58:41.873 | INFO | MemInfo: { | 20260508 10:58:41.873 | INFO | "gpu" : " 17 GB", | 20260508 10:58:41.873 | INFO | "huge" : "", | 20260508 10:58:41.874 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:41.874 | INFO | "malloc" : " 14 GB", | 20260508 10:58:41.874 | INFO | "pinned" : " 29 GB" | 20260508 10:58:41.874 | INFO | } | 20260508 10:58:41.874 | INFO | Perf report for the last 5.13868 seconds: | 20260508 10:58:41.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:41.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 10:58:41.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 10:58:41.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:41.874 | INFO | Uploaded 36GiB to GPU in 2.49978s (14.4013GiB/s) | 20260508 10:58:41.874 | INFO | Downloaded 34GiB to GPU in 2.44675s (13.896GiB/s) | 20260508 10:58:47.012 | INFO | Analyzing chunk frames = [7570944,7571456) | 20260508 10:58:47.012 | INFO | MemInfo: { | 20260508 10:58:47.012 | INFO | "gpu" : " 17 GB", | 20260508 10:58:47.012 | INFO | "huge" : "", | 20260508 10:58:47.012 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:47.012 | INFO | "malloc" : " 14 GB", | 20260508 10:58:47.012 | INFO | "pinned" : " 29 GB" | 20260508 10:58:47.012 | INFO | } | 20260508 10:58:47.012 | INFO | Perf report for the last 5.13779 seconds: | 20260508 10:58:47.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:47.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:58:47.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 10:58:47.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:47.012 | INFO | Uploaded 36GiB to GPU in 2.47561s (14.5419GiB/s) | 20260508 10:58:47.012 | INFO | Downloaded 34GiB to GPU in 2.4475s (13.8917GiB/s) | 20260508 10:58:52.143 | INFO | Analyzing chunk frames = [7571456,7571968) | 20260508 10:58:52.143 | INFO | MemInfo: { | 20260508 10:58:52.143 | INFO | "gpu" : " 17 GB", | 20260508 10:58:52.143 | INFO | "huge" : "", | 20260508 10:58:52.143 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:52.143 | INFO | "malloc" : " 14 GB", | 20260508 10:58:52.143 | INFO | "pinned" : " 29 GB" | 20260508 10:58:52.143 | INFO | } | 20260508 10:58:52.143 | INFO | Perf report for the last 5.13163 seconds: | 20260508 10:58:52.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:52.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.143 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 10:58:52.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.143 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 10:58:52.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:52.144 | INFO | Uploaded 36GiB to GPU in 2.45416s (14.669GiB/s) | 20260508 10:58:52.144 | INFO | Downloaded 34GiB to GPU in 2.54303s (13.3699GiB/s) | 20260508 10:58:57.287 | INFO | Analyzing chunk frames = [7571968,7572480) | 20260508 10:58:57.287 | INFO | MemInfo: { | 20260508 10:58:57.287 | INFO | "gpu" : " 17 GB", | 20260508 10:58:57.287 | INFO | "huge" : "", | 20260508 10:58:57.287 | INFO | "huge_shared" : " 29 GB", | 20260508 10:58:57.287 | INFO | "malloc" : " 14 GB", | 20260508 10:58:57.287 | INFO | "pinned" : " 29 GB" | 20260508 10:58:57.287 | INFO | } | 20260508 10:58:57.288 | INFO | Perf report for the last 5.14418 seconds: | 20260508 10:58:57.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:58:57.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 10:58:57.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 10:58:57.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:58:57.288 | INFO | Uploaded 36GiB to GPU in 2.39181s (15.0514GiB/s) | 20260508 10:58:57.288 | INFO | Downloaded 34GiB to GPU in 2.41975s (14.0511GiB/s) | 20260508 10:59:02.435 | INFO | Analyzing chunk frames = [7572480,7572992) | 20260508 10:59:02.435 | INFO | MemInfo: { | 20260508 10:59:02.435 | INFO | "gpu" : " 17 GB", | 20260508 10:59:02.435 | INFO | "huge" : "", | 20260508 10:59:02.435 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:02.435 | INFO | "malloc" : " 14 GB", | 20260508 10:59:02.435 | INFO | "pinned" : " 29 GB" | 20260508 10:59:02.435 | INFO | } | 20260508 10:59:02.435 | INFO | Perf report for the last 5.14737 seconds: | 20260508 10:59:02.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:02.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.435 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 10:59:02.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.435 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 10:59:02.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:02.436 | INFO | Uploaded 36GiB to GPU in 2.44178s (14.7433GiB/s) | 20260508 10:59:02.436 | INFO | Downloaded 34GiB to GPU in 2.4285s (14.0004GiB/s) | 20260508 10:59:07.573 | INFO | Analyzing chunk frames = [7572992,7573504) | 20260508 10:59:07.573 | INFO | MemInfo: { | 20260508 10:59:07.573 | INFO | "gpu" : " 17 GB", | 20260508 10:59:07.573 | INFO | "huge" : "", | 20260508 10:59:07.573 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:07.573 | INFO | "malloc" : " 14 GB", | 20260508 10:59:07.573 | INFO | "pinned" : " 29 GB" | 20260508 10:59:07.573 | INFO | } | 20260508 10:59:07.573 | INFO | Perf report for the last 5.13822 seconds: | 20260508 10:59:07.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:07.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.573 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 10:59:07.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.573 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:59:07.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:07.574 | INFO | Uploaded 36GiB to GPU in 2.47342s (14.5548GiB/s) | 20260508 10:59:07.574 | INFO | Downloaded 34GiB to GPU in 2.44396s (13.9118GiB/s) | 20260508 10:59:07.670 | INFO | Starting a round of Baz IO | 20260508 10:59:11.090 | INFO | Populating RTMetrics report | 20260508 10:59:11.090 | INFO | RTMetrics report populated | 20260508 10:59:11.090 | INFO | Opening tmp RTMetrics json file | 20260508 10:59:11.097 | INFO | Serializing RTMetrics to json | 20260508 10:59:11.099 | INFO | Serialization done, swapping tmp file now... | 20260508 10:59:11.099 | INFO | RTMetrics done | 20260508 10:59:11.154 | INFO | Finished a round of Baz IO: Wrote 1.447 GiB in 3.483 seconds | 20260508 10:59:11.154 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:59:11.154 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.206192% / 3.97364e-06% / 7.54992e-05% | 20260508 10:59:11.154 | INFO | Baz IO overall: | 20260508 10:59:11.154 | INFO | BytesWritten : 3276224724076 | 20260508 10:59:11.154 | INFO | NumEvents : 1574107956481 | 20260508 10:59:12.709 | INFO | Analyzing chunk frames = [7573504,7574016) | 20260508 10:59:12.709 | INFO | MemInfo: { | 20260508 10:59:12.709 | INFO | "gpu" : " 17 GB", | 20260508 10:59:12.709 | INFO | "huge" : "", | 20260508 10:59:12.709 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:12.709 | INFO | "malloc" : " 14 GB", | 20260508 10:59:12.709 | INFO | "pinned" : " 29 GB" | 20260508 10:59:12.709 | INFO | } | 20260508 10:59:12.709 | INFO | Perf report for the last 5.13603 seconds: | 20260508 10:59:12.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:12.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:12.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:12.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:12.709 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 10:59:12.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:12.709 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 10:59:12.709 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 10:59:12.709 | INFO | BAZWRITER : 67.2%, 2.5, 85.9ms, 0 | 20260508 10:59:12.709 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:59:12.709 | INFO | Uploaded 36GiB to GPU in 2.62983s (13.6891GiB/s) | 20260508 10:59:12.710 | INFO | Downloaded 38GiB to GPU in 2.71978s (13.9717GiB/s) | 20260508 10:59:17.847 | INFO | Analyzing chunk frames = [7574016,7574528) | 20260508 10:59:17.847 | INFO | MemInfo: { | 20260508 10:59:17.847 | INFO | "gpu" : " 17 GB", | 20260508 10:59:17.847 | INFO | "huge" : "", | 20260508 10:59:17.847 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:17.847 | INFO | "malloc" : " 14 GB", | 20260508 10:59:17.847 | INFO | "pinned" : " 29 GB" | 20260508 10:59:17.847 | INFO | } | 20260508 10:59:17.847 | INFO | Perf report for the last 5.13775 seconds: | 20260508 10:59:17.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:17.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.847 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 10:59:17.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.847 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:59:17.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:17.848 | INFO | Uploaded 36GiB to GPU in 2.49492s (14.4293GiB/s) | 20260508 10:59:17.848 | INFO | Downloaded 34GiB to GPU in 2.46725s (13.7805GiB/s) | 20260508 10:59:22.988 | INFO | Analyzing chunk frames = [7574528,7575040) | 20260508 10:59:22.988 | INFO | MemInfo: { | 20260508 10:59:22.988 | INFO | "gpu" : " 17 GB", | 20260508 10:59:22.988 | INFO | "huge" : "", | 20260508 10:59:22.988 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:22.988 | INFO | "malloc" : " 14 GB", | 20260508 10:59:22.988 | INFO | "pinned" : " 29 GB" | 20260508 10:59:22.988 | INFO | } | 20260508 10:59:22.988 | INFO | Perf report for the last 5.1415 seconds: | 20260508 10:59:22.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:22.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 10:59:22.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 10:59:22.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:22.989 | INFO | Uploaded 36GiB to GPU in 2.51384s (14.3207GiB/s) | 20260508 10:59:22.989 | INFO | Downloaded 34GiB to GPU in 2.44466s (13.9079GiB/s) | 20260508 10:59:28.125 | INFO | Analyzing chunk frames = [7575040,7575552) | 20260508 10:59:28.125 | INFO | MemInfo: { | 20260508 10:59:28.125 | INFO | "gpu" : " 17 GB", | 20260508 10:59:28.125 | INFO | "huge" : "", | 20260508 10:59:28.125 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:28.125 | INFO | "malloc" : " 14 GB", | 20260508 10:59:28.125 | INFO | "pinned" : " 29 GB" | 20260508 10:59:28.125 | INFO | } | 20260508 10:59:28.125 | INFO | Perf report for the last 5.13653 seconds: | 20260508 10:59:28.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:28.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.125 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 10:59:28.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.125 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 10:59:28.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:28.126 | INFO | Uploaded 36GiB to GPU in 2.49161s (14.4485GiB/s) | 20260508 10:59:28.126 | INFO | Downloaded 34GiB to GPU in 2.4492s (13.8821GiB/s) | 20260508 10:59:33.265 | INFO | Analyzing chunk frames = [7575552,7576064) | 20260508 10:59:33.265 | INFO | MemInfo: { | 20260508 10:59:33.265 | INFO | "gpu" : " 17 GB", | 20260508 10:59:33.265 | INFO | "huge" : "", | 20260508 10:59:33.265 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:33.265 | INFO | "malloc" : " 14 GB", | 20260508 10:59:33.265 | INFO | "pinned" : " 29 GB" | 20260508 10:59:33.265 | INFO | } | 20260508 10:59:33.265 | INFO | Perf report for the last 5.14043 seconds: | 20260508 10:59:33.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:33.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 10:59:33.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 10:59:33.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:33.266 | INFO | Uploaded 36GiB to GPU in 2.42373s (14.8531GiB/s) | 20260508 10:59:33.266 | INFO | Downloaded 34GiB to GPU in 2.50404s (13.5781GiB/s) | 20260508 10:59:38.407 | INFO | Analyzing chunk frames = [7576064,7576576) | 20260508 10:59:38.407 | INFO | MemInfo: { | 20260508 10:59:38.407 | INFO | "gpu" : " 17 GB", | 20260508 10:59:38.407 | INFO | "huge" : "", | 20260508 10:59:38.407 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:38.407 | INFO | "malloc" : " 14 GB", | 20260508 10:59:38.407 | INFO | "pinned" : " 29 GB" | 20260508 10:59:38.407 | INFO | } | 20260508 10:59:38.407 | INFO | Perf report for the last 5.14191 seconds: | 20260508 10:59:38.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:38.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.407 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 10:59:38.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.407 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:59:38.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:38.408 | INFO | Uploaded 36GiB to GPU in 2.37909s (15.1318GiB/s) | 20260508 10:59:38.408 | INFO | Downloaded 34GiB to GPU in 2.43648s (13.9546GiB/s) | 20260508 10:59:43.546 | INFO | Analyzing chunk frames = [7576576,7577088) | 20260508 10:59:43.546 | INFO | MemInfo: { | 20260508 10:59:43.546 | INFO | "gpu" : " 17 GB", | 20260508 10:59:43.546 | INFO | "huge" : "", | 20260508 10:59:43.546 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:43.546 | INFO | "malloc" : " 14 GB", | 20260508 10:59:43.546 | INFO | "pinned" : " 29 GB" | 20260508 10:59:43.546 | INFO | } | 20260508 10:59:43.546 | INFO | Perf report for the last 5.13869 seconds: | 20260508 10:59:43.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:43.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 10:59:43.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 10:59:43.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:43.546 | INFO | Uploaded 36GiB to GPU in 2.44565s (14.72GiB/s) | 20260508 10:59:43.546 | INFO | Downloaded 34GiB to GPU in 2.45333s (13.8587GiB/s) | 20260508 10:59:48.686 | INFO | Analyzing chunk frames = [7577088,7577600) | 20260508 10:59:48.686 | INFO | MemInfo: { | 20260508 10:59:48.686 | INFO | "gpu" : " 17 GB", | 20260508 10:59:48.686 | INFO | "huge" : "", | 20260508 10:59:48.686 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:48.686 | INFO | "malloc" : " 14 GB", | 20260508 10:59:48.686 | INFO | "pinned" : " 29 GB" | 20260508 10:59:48.686 | INFO | } | 20260508 10:59:48.686 | INFO | Perf report for the last 5.13968 seconds: | 20260508 10:59:48.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:48.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 10:59:48.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 10:59:48.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:48.686 | INFO | Uploaded 36GiB to GPU in 2.46204s (14.622GiB/s) | 20260508 10:59:48.686 | INFO | Downloaded 34GiB to GPU in 2.44485s (13.9068GiB/s) | 20260508 10:59:48.686 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 10:59:48.686 | INFO | Total mem usage: 16.7 GiB | 20260508 10:59:48.686 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 10:59:48.686 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 10:59:48.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 10:59:48.686 | INFO | High Water Breakdown: | 20260508 10:59:48.686 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 10:59:48.686 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 10:59:48.687 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 10:59:48.687 | INFO | ------------------------------------------ | 20260508 10:59:48.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:59:48.687 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 10:59:48.687 | INFO | Total mem usage: 26.5 GiB | 20260508 10:59:48.687 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 10:59:48.687 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 10:59:48.687 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 10:59:48.687 | INFO | High Water Breakdown: | 20260508 10:59:48.687 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:59:48.687 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:59:48.687 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:59:48.687 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:59:48.687 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:59:48.687 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:59:48.687 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 10:59:48.687 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 10:59:48.687 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:59:48.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 10:59:48.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 10:59:48.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 10:59:48.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 10:59:48.687 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 10:59:48.687 | INFO | ------------------------------------------ | 20260508 10:59:48.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:59:48.687 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 10:59:48.687 | INFO | Total mem usage: 27.8 GiB | 20260508 10:59:48.687 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 10:59:48.687 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 10:59:48.687 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 10:59:48.687 | INFO | High Water Breakdown: | 20260508 10:59:48.687 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 10:59:48.687 | INFO | ------------------------------------------ | 20260508 10:59:48.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:59:48.687 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 10:59:48.687 | INFO | Total mem usage: 15.7 GiB | 20260508 10:59:48.687 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 10:59:48.687 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 10:59:48.687 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 10:59:48.687 | INFO | High Water Breakdown: | 20260508 10:59:48.687 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 10:59:48.687 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 10:59:48.687 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 10:59:48.688 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 10:59:48.688 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 10:59:48.688 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 10:59:48.688 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 10:59:48.688 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 10:59:48.688 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 10:59:48.688 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 10:59:48.688 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 10:59:48.688 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 10:59:48.688 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 10:59:48.688 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 10:59:48.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 10:59:48.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 10:59:48.688 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 10:59:48.688 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 10:59:48.688 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 10:59:48.688 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 10:59:48.688 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 10:59:48.688 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 10:59:48.688 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 10:59:48.688 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 10:59:48.688 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 10:59:48.688 | INFO | ------------------------------------------ | 20260508 10:59:48.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 10:59:48.688 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 10:59:48.688 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:59:48.688 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:59:48.688 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.40e+01 / 5.28e-01 | 20260508 10:59:48.688 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.05e-01, 1.09e-02 / 5.08e-03 | 20260508 10:59:48.688 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.44e+00, 5.89e-01 / 2.28e-01 | 20260508 10:59:48.688 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.50e+00, 2.10e-01 / 1.11e-01 | 20260508 10:59:48.688 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.48e+01, 3.62e+00 / 8.75e-01 | 20260508 10:59:48.688 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.78e+00 / 1.32e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.45e+00, 4.06e-01 / 2.67e-01 | 20260508 10:59:48.688 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.13e+00, 2.43e-01 / 1.18e-01 | 20260508 10:59:48.688 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.29e-01, 4.75e-02 / 2.91e-02 | 20260508 10:59:48.688 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 10:59:48.688 | INFO | PerfReport: Gathered 1536 samples. | 20260508 10:59:48.688 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 10:59:48.688 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 24.50%, Upload -- 2.58e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 24.62%, Download -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 30.97%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 2.23%, LockUpload -- 2.35e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 2.52%, LockDownload -- 2.66e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 15.15%, LockCompute -- 1.60e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 10:59:48.688 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 10:59:48.783 | INFO | Starting a round of Baz IO | 20260508 10:59:52.204 | INFO | Populating RTMetrics report | 20260508 10:59:52.204 | INFO | RTMetrics report populated | 20260508 10:59:52.204 | INFO | Opening tmp RTMetrics json file | 20260508 10:59:52.213 | INFO | Serializing RTMetrics to json | 20260508 10:59:52.214 | INFO | Serialization done, swapping tmp file now... | 20260508 10:59:52.215 | INFO | RTMetrics done | 20260508 10:59:52.267 | INFO | Finished a round of Baz IO: Wrote 1.449 GiB in 3.484 seconds | 20260508 10:59:52.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.15% / 0.00% / 59.85% | 20260508 10:59:52.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205934% / 7.94729e-06% / 8.74201e-05% | 20260508 10:59:52.267 | INFO | Baz IO overall: | 20260508 10:59:52.268 | INFO | BytesWritten : 3277780040309 | 20260508 10:59:52.268 | INFO | NumEvents : 1574841195715 | 20260508 10:59:53.823 | INFO | Analyzing chunk frames = [7577600,7578112) | 20260508 10:59:53.823 | INFO | MemInfo: { | 20260508 10:59:53.823 | INFO | "gpu" : " 17 GB", | 20260508 10:59:53.823 | INFO | "huge" : "", | 20260508 10:59:53.823 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:53.823 | INFO | "malloc" : " 14 GB", | 20260508 10:59:53.823 | INFO | "pinned" : " 29 GB" | 20260508 10:59:53.823 | INFO | } | 20260508 10:59:53.823 | INFO | Perf report for the last 5.13746 seconds: | 20260508 10:59:53.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:53.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:53.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:53.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:53.823 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 10:59:53.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:53.823 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 10:59:53.823 | INFO | PRE_HQ : 65.4%, 1.6, 54.2ms, 0 | 20260508 10:59:53.823 | INFO | BAZWRITER : 67.0%, 2.5, 86.4ms, 0 | 20260508 10:59:53.823 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 10:59:53.824 | INFO | Uploaded 36GiB to GPU in 2.59212s (13.8882GiB/s) | 20260508 10:59:53.824 | INFO | Downloaded 38GiB to GPU in 2.70984s (14.023GiB/s) | 20260508 10:59:58.959 | INFO | Analyzing chunk frames = [7578112,7578624) | 20260508 10:59:58.959 | INFO | MemInfo: { | 20260508 10:59:58.960 | INFO | "gpu" : " 17 GB", | 20260508 10:59:58.960 | INFO | "huge" : "", | 20260508 10:59:58.960 | INFO | "huge_shared" : " 29 GB", | 20260508 10:59:58.960 | INFO | "malloc" : " 14 GB", | 20260508 10:59:58.960 | INFO | "pinned" : " 29 GB" | 20260508 10:59:58.960 | INFO | } | 20260508 10:59:58.960 | INFO | Perf report for the last 5.13642 seconds: | 20260508 10:59:58.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 10:59:58.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 10:59:58.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 10:59:58.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 10:59:58.960 | INFO | Uploaded 36GiB to GPU in 2.45498s (14.6641GiB/s) | 20260508 10:59:58.960 | INFO | Downloaded 34GiB to GPU in 2.47165s (13.756GiB/s) | 20260508 11:00:04.098 | INFO | Analyzing chunk frames = [7578624,7579136) | 20260508 11:00:04.099 | INFO | MemInfo: { | 20260508 11:00:04.099 | INFO | "gpu" : " 17 GB", | 20260508 11:00:04.099 | INFO | "huge" : "", | 20260508 11:00:04.099 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:04.099 | INFO | "malloc" : " 14 GB", | 20260508 11:00:04.099 | INFO | "pinned" : " 29 GB" | 20260508 11:00:04.099 | INFO | } | 20260508 11:00:04.099 | INFO | Perf report for the last 5.13899 seconds: | 20260508 11:00:04.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:04.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:00:04.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:00:04.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:04.099 | INFO | Uploaded 36GiB to GPU in 2.47086s (14.5698GiB/s) | 20260508 11:00:04.099 | INFO | Downloaded 34GiB to GPU in 2.43534s (13.9611GiB/s) | 20260508 11:00:09.241 | INFO | Analyzing chunk frames = [7579136,7579648) | 20260508 11:00:09.241 | INFO | MemInfo: { | 20260508 11:00:09.241 | INFO | "gpu" : " 17 GB", | 20260508 11:00:09.241 | INFO | "huge" : "", | 20260508 11:00:09.241 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:09.241 | INFO | "malloc" : " 14 GB", | 20260508 11:00:09.241 | INFO | "pinned" : " 29 GB" | 20260508 11:00:09.241 | INFO | } | 20260508 11:00:09.242 | INFO | Perf report for the last 5.14283 seconds: | 20260508 11:00:09.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:09.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:00:09.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:00:09.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:09.242 | INFO | Uploaded 36GiB to GPU in 2.47414s (14.5505GiB/s) | 20260508 11:00:09.242 | INFO | Downloaded 34GiB to GPU in 2.4465s (13.8974GiB/s) | 20260508 11:00:14.388 | INFO | Analyzing chunk frames = [7579648,7580160) | 20260508 11:00:14.388 | INFO | MemInfo: { | 20260508 11:00:14.388 | INFO | "gpu" : " 17 GB", | 20260508 11:00:14.388 | INFO | "huge" : "", | 20260508 11:00:14.388 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:14.388 | INFO | "malloc" : " 14 GB", | 20260508 11:00:14.388 | INFO | "pinned" : " 29 GB" | 20260508 11:00:14.388 | INFO | } | 20260508 11:00:14.388 | INFO | Perf report for the last 5.14657 seconds: | 20260508 11:00:14.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:14.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 11:00:14.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:00:14.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:14.388 | INFO | Uploaded 36GiB to GPU in 2.30118s (15.6442GiB/s) | 20260508 11:00:14.388 | INFO | Downloaded 34GiB to GPU in 2.4966s (13.6185GiB/s) | 20260508 11:00:19.520 | INFO | Analyzing chunk frames = [7580160,7580672) | 20260508 11:00:19.520 | INFO | MemInfo: { | 20260508 11:00:19.520 | INFO | "gpu" : " 17 GB", | 20260508 11:00:19.520 | INFO | "huge" : "", | 20260508 11:00:19.520 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:19.520 | INFO | "malloc" : " 14 GB", | 20260508 11:00:19.520 | INFO | "pinned" : " 29 GB" | 20260508 11:00:19.520 | INFO | } | 20260508 11:00:19.521 | INFO | Perf report for the last 5.13241 seconds: | 20260508 11:00:19.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:19.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:00:19.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:00:19.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:19.521 | INFO | Uploaded 36GiB to GPU in 2.39305s (15.0436GiB/s) | 20260508 11:00:19.521 | INFO | Downloaded 34GiB to GPU in 2.43543s (13.9606GiB/s) | 20260508 11:00:24.655 | INFO | Analyzing chunk frames = [7580672,7581184) | 20260508 11:00:24.655 | INFO | MemInfo: { | 20260508 11:00:24.655 | INFO | "gpu" : " 17 GB", | 20260508 11:00:24.655 | INFO | "huge" : "", | 20260508 11:00:24.655 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:24.656 | INFO | "malloc" : " 14 GB", | 20260508 11:00:24.656 | INFO | "pinned" : " 29 GB" | 20260508 11:00:24.656 | INFO | } | 20260508 11:00:24.656 | INFO | Perf report for the last 5.13505 seconds: | 20260508 11:00:24.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:24.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 11:00:24.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:00:24.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:24.656 | INFO | Uploaded 36GiB to GPU in 2.4149s (14.9075GiB/s) | 20260508 11:00:24.656 | INFO | Downloaded 34GiB to GPU in 2.43214s (13.9795GiB/s) | 20260508 11:00:29.796 | INFO | Analyzing chunk frames = [7581184,7581696) | 20260508 11:00:29.796 | INFO | MemInfo: { | 20260508 11:00:29.796 | INFO | "gpu" : " 17 GB", | 20260508 11:00:29.796 | INFO | "huge" : "", | 20260508 11:00:29.796 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:29.796 | INFO | "malloc" : " 14 GB", | 20260508 11:00:29.796 | INFO | "pinned" : " 29 GB" | 20260508 11:00:29.796 | INFO | } | 20260508 11:00:29.796 | INFO | Perf report for the last 5.14072 seconds: | 20260508 11:00:29.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:29.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.797 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 11:00:29.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.797 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:00:29.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:29.797 | INFO | Uploaded 36GiB to GPU in 2.40761s (14.9526GiB/s) | 20260508 11:00:29.797 | INFO | Downloaded 34GiB to GPU in 2.42014s (14.0488GiB/s) | 20260508 11:00:29.904 | INFO | Starting a round of Baz IO | 20260508 11:00:33.300 | INFO | Populating RTMetrics report | 20260508 11:00:33.300 | INFO | RTMetrics report populated | 20260508 11:00:33.300 | INFO | Opening tmp RTMetrics json file | 20260508 11:00:33.313 | INFO | Serializing RTMetrics to json | 20260508 11:00:33.314 | INFO | Serialization done, swapping tmp file now... | 20260508 11:00:33.315 | INFO | RTMetrics done | 20260508 11:00:33.374 | INFO | Finished a round of Baz IO: Wrote 1.447 GiB in 3.469 seconds | 20260508 11:00:33.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.85% | 20260508 11:00:33.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.204702% / 0% / 6.75519e-05% | 20260508 11:00:33.374 | INFO | Baz IO overall: | 20260508 11:00:33.374 | INFO | BytesWritten : 3279334000055 | 20260508 11:00:33.374 | INFO | NumEvents : 1575573518724 | 20260508 11:00:34.937 | INFO | Analyzing chunk frames = [7581696,7582208) | 20260508 11:00:34.937 | INFO | MemInfo: { | 20260508 11:00:34.937 | INFO | "gpu" : " 17 GB", | 20260508 11:00:34.937 | INFO | "huge" : "", | 20260508 11:00:34.937 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:34.937 | INFO | "malloc" : " 14 GB", | 20260508 11:00:34.937 | INFO | "pinned" : " 29 GB" | 20260508 11:00:34.937 | INFO | } | 20260508 11:00:34.937 | INFO | Perf report for the last 5.14097 seconds: | 20260508 11:00:34.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:34.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:34.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:34.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:34.937 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:00:34.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:34.937 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 11:00:34.937 | INFO | PRE_HQ : 66.3%, 1.7, 58.8ms, 0 | 20260508 11:00:34.937 | INFO | BAZWRITER : 66.8%, 2.7, 91.5ms, 0 | 20260508 11:00:34.937 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:00:34.938 | INFO | Uploaded 36GiB to GPU in 2.5805s (13.9508GiB/s) | 20260508 11:00:34.938 | INFO | Downloaded 38GiB to GPU in 2.713s (14.0066GiB/s) | 20260508 11:00:40.075 | INFO | Analyzing chunk frames = [7582208,7582720) | 20260508 11:00:40.075 | INFO | MemInfo: { | 20260508 11:00:40.075 | INFO | "gpu" : " 17 GB", | 20260508 11:00:40.075 | INFO | "huge" : "", | 20260508 11:00:40.075 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:40.075 | INFO | "malloc" : " 14 GB", | 20260508 11:00:40.075 | INFO | "pinned" : " 29 GB" | 20260508 11:00:40.075 | INFO | } | 20260508 11:00:40.075 | INFO | Perf report for the last 5.13779 seconds: | 20260508 11:00:40.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:40.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.075 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:00:40.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.075 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:00:40.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:40.076 | INFO | Uploaded 36GiB to GPU in 2.45239s (14.6796GiB/s) | 20260508 11:00:40.076 | INFO | Downloaded 34GiB to GPU in 2.45171s (13.8679GiB/s) | 20260508 11:00:45.216 | INFO | Analyzing chunk frames = [7582720,7583232) | 20260508 11:00:45.216 | INFO | MemInfo: { | 20260508 11:00:45.216 | INFO | "gpu" : " 17 GB", | 20260508 11:00:45.216 | INFO | "huge" : "", | 20260508 11:00:45.216 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:45.216 | INFO | "malloc" : " 14 GB", | 20260508 11:00:45.216 | INFO | "pinned" : " 29 GB" | 20260508 11:00:45.216 | INFO | } | 20260508 11:00:45.216 | INFO | Perf report for the last 5.14121 seconds: | 20260508 11:00:45.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:45.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.216 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 11:00:45.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.216 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:00:45.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:45.217 | INFO | Uploaded 36GiB to GPU in 2.47799s (14.5279GiB/s) | 20260508 11:00:45.217 | INFO | Downloaded 34GiB to GPU in 2.42495s (14.0209GiB/s) | 20260508 11:00:50.353 | INFO | Analyzing chunk frames = [7583232,7583744) | 20260508 11:00:50.353 | INFO | MemInfo: { | 20260508 11:00:50.353 | INFO | "gpu" : " 17 GB", | 20260508 11:00:50.353 | INFO | "huge" : "", | 20260508 11:00:50.353 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:50.353 | INFO | "malloc" : " 14 GB", | 20260508 11:00:50.353 | INFO | "pinned" : " 29 GB" | 20260508 11:00:50.353 | INFO | } | 20260508 11:00:50.353 | INFO | Perf report for the last 5.13665 seconds: | 20260508 11:00:50.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:50.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:00:50.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:00:50.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:50.353 | INFO | Uploaded 36GiB to GPU in 2.47794s (14.5282GiB/s) | 20260508 11:00:50.353 | INFO | Downloaded 34GiB to GPU in 2.4466s (13.8968GiB/s) | 20260508 11:00:55.501 | INFO | Analyzing chunk frames = [7583744,7584256) | 20260508 11:00:55.501 | INFO | MemInfo: { | 20260508 11:00:55.501 | INFO | "gpu" : " 17 GB", | 20260508 11:00:55.502 | INFO | "huge" : "", | 20260508 11:00:55.502 | INFO | "huge_shared" : " 29 GB", | 20260508 11:00:55.502 | INFO | "malloc" : " 14 GB", | 20260508 11:00:55.502 | INFO | "pinned" : " 29 GB" | 20260508 11:00:55.502 | INFO | } | 20260508 11:00:55.502 | INFO | Perf report for the last 5.14862 seconds: | 20260508 11:00:55.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:00:55.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 11:00:55.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:00:55.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:00:55.502 | INFO | Uploaded 36GiB to GPU in 2.2992s (15.6576GiB/s) | 20260508 11:00:55.502 | INFO | Downloaded 34GiB to GPU in 2.50051s (13.5972GiB/s) | 20260508 11:01:00.638 | INFO | Analyzing chunk frames = [7584256,7584768) | 20260508 11:01:00.638 | INFO | MemInfo: { | 20260508 11:01:00.638 | INFO | "gpu" : " 17 GB", | 20260508 11:01:00.638 | INFO | "huge" : "", | 20260508 11:01:00.638 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:00.638 | INFO | "malloc" : " 14 GB", | 20260508 11:01:00.638 | INFO | "pinned" : " 29 GB" | 20260508 11:01:00.638 | INFO | } | 20260508 11:01:00.638 | INFO | Perf report for the last 5.13682 seconds: | 20260508 11:01:00.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:00.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 11:01:00.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:01:00.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:00.639 | INFO | Uploaded 36GiB to GPU in 2.39765s (15.0147GiB/s) | 20260508 11:01:00.639 | INFO | Downloaded 34GiB to GPU in 2.42068s (14.0457GiB/s) | 20260508 11:01:05.774 | INFO | Analyzing chunk frames = [7584768,7585280) | 20260508 11:01:05.774 | INFO | MemInfo: { | 20260508 11:01:05.774 | INFO | "gpu" : " 17 GB", | 20260508 11:01:05.774 | INFO | "huge" : "", | 20260508 11:01:05.774 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:05.774 | INFO | "malloc" : " 14 GB", | 20260508 11:01:05.774 | INFO | "pinned" : " 29 GB" | 20260508 11:01:05.774 | INFO | } | 20260508 11:01:05.774 | INFO | Perf report for the last 5.13591 seconds: | 20260508 11:01:05.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:05.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.774 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:01:05.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.775 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 11:01:05.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:05.775 | INFO | Uploaded 36GiB to GPU in 2.48375s (14.4942GiB/s) | 20260508 11:01:05.775 | INFO | Downloaded 34GiB to GPU in 2.4499s (13.8781GiB/s) | 20260508 11:01:10.906 | INFO | Analyzing chunk frames = [7585280,7585792) | 20260508 11:01:10.906 | INFO | MemInfo: { | 20260508 11:01:10.906 | INFO | "gpu" : " 17 GB", | 20260508 11:01:10.906 | INFO | "huge" : "", | 20260508 11:01:10.906 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:10.906 | INFO | "malloc" : " 14 GB", | 20260508 11:01:10.906 | INFO | "pinned" : " 29 GB" | 20260508 11:01:10.906 | INFO | } | 20260508 11:01:10.906 | INFO | Perf report for the last 5.13153 seconds: | 20260508 11:01:10.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:10.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 11:01:10.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:01:10.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:10.906 | INFO | Uploaded 36GiB to GPU in 2.46208s (14.6218GiB/s) | 20260508 11:01:10.906 | INFO | Downloaded 34GiB to GPU in 2.43872s (13.9417GiB/s) | 20260508 11:01:10.906 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:01:10.906 | INFO | Total mem usage: 16.7 GiB | 20260508 11:01:10.906 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:01:10.906 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 11:01:10.906 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:01:10.906 | INFO | High Water Breakdown: | 20260508 11:01:10.906 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:01:10.906 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:01:10.906 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:01:10.906 | INFO | ------------------------------------------ | 20260508 11:01:10.906 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:01:10.906 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:01:10.906 | INFO | Total mem usage: 26.5 GiB | 20260508 11:01:10.906 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:01:10.906 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260508 11:01:10.906 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260508 11:01:10.906 | INFO | High Water Breakdown: | 20260508 11:01:10.906 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:01:10.906 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:01:10.906 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:01:10.906 | INFO | 2% : 525 MiB ( 83.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:01:10.907 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:01:10.907 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:01:10.907 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:01:10.907 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:01:10.907 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:01:10.907 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:01:10.907 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:01:10.907 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:01:10.907 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:01:10.907 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:01:10.907 | INFO | ------------------------------------------ | 20260508 11:01:10.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:01:10.907 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:01:10.907 | INFO | Total mem usage: 27.8 GiB | 20260508 11:01:10.907 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:01:10.907 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:01:10.907 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:01:10.907 | INFO | High Water Breakdown: | 20260508 11:01:10.907 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:01:10.907 | INFO | ------------------------------------------ | 20260508 11:01:10.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:01:10.907 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:01:10.907 | INFO | Total mem usage: 15.7 GiB | 20260508 11:01:10.907 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:01:10.907 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:01:10.907 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:01:10.907 | INFO | High Water Breakdown: | 20260508 11:01:10.907 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:01:10.907 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:01:10.907 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:01:10.907 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:01:10.907 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:01:10.907 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:01:10.907 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:01:10.907 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:01:10.907 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:01:10.907 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:01:10.907 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:01:10.907 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:01:10.907 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:01:10.907 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:01:10.907 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:01:10.907 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:01:10.907 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:01:10.907 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:01:10.907 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:01:10.907 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:01:10.907 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:01:10.907 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:01:10.907 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:01:10.907 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:01:10.907 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:01:10.907 | INFO | ------------------------------------------ | 20260508 11:01:10.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:01:10.907 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:01:10.907 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:01:10.907 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:01:10.907 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.72e-01 | 20260508 11:01:10.907 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.15e-02 / 4.95e-03 | 20260508 11:01:10.907 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.42e+00, 5.95e-01 / 2.31e-01 | 20260508 11:01:10.907 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.21e-01 | 20260508 11:01:10.907 | INFO | PerfReport: 16.09%, DME -- 4.20e+01 / 4.56e+01, 3.70e+00 / 1.05e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.75e+00 / 1.20e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.53e+00, 4.06e-01 / 2.04e-01 | 20260508 11:01:10.907 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.12e+00, 2.41e-01 / 1.20e-01 | 20260508 11:01:10.907 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.26e-01, 4.83e-02 / 2.73e-02 | 20260508 11:01:10.907 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:01:10.907 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:01:10.907 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:01:10.907 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 24.24%, Upload -- 2.55e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 24.58%, Download -- 2.58e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 2.18%, LockUpload -- 2.29e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 11:01:10.907 | INFO | PerfReport: 2.61%, LockDownload -- 2.74e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 11:01:10.908 | INFO | PerfReport: 15.32%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 11:01:10.908 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 11:01:11.003 | INFO | Starting a round of Baz IO | 20260508 11:01:14.420 | INFO | Populating RTMetrics report | 20260508 11:01:14.421 | INFO | RTMetrics report populated | 20260508 11:01:14.421 | INFO | Opening tmp RTMetrics json file | 20260508 11:01:14.431 | INFO | Serializing RTMetrics to json | 20260508 11:01:14.432 | INFO | Serialization done, swapping tmp file now... | 20260508 11:01:14.432 | INFO | RTMetrics done | 20260508 11:01:14.492 | INFO | Finished a round of Baz IO: Wrote 1.446 GiB in 3.488 seconds | 20260508 11:01:14.492 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:01:14.492 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.204539% / 0% / 3.57628e-05% | 20260508 11:01:14.492 | INFO | Baz IO overall: | 20260508 11:01:14.492 | INFO | BytesWritten : 3280886594104 | 20260508 11:01:14.492 | INFO | NumEvents : 1576304818319 | 20260508 11:01:16.053 | INFO | Analyzing chunk frames = [7585792,7586304) | 20260508 11:01:16.053 | INFO | MemInfo: { | 20260508 11:01:16.053 | INFO | "gpu" : " 17 GB", | 20260508 11:01:16.053 | INFO | "huge" : "", | 20260508 11:01:16.053 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:16.053 | INFO | "malloc" : " 14 GB", | 20260508 11:01:16.053 | INFO | "pinned" : " 29 GB" | 20260508 11:01:16.053 | INFO | } | 20260508 11:01:16.053 | INFO | Perf report for the last 5.14669 seconds: | 20260508 11:01:16.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:16.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:16.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:16.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:16.053 | INFO | ANALYSIS : 67.6%, 3.0, 105.8ms, 0 | 20260508 11:01:16.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:16.053 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:01:16.053 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 11:01:16.053 | INFO | BAZWRITER : 67.1%, 2.5, 86.7ms, 0 | 20260508 11:01:16.053 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:01:16.054 | INFO | Uploaded 36GiB to GPU in 2.60034s (13.8444GiB/s) | 20260508 11:01:16.054 | INFO | Downloaded 38GiB to GPU in 2.71521s (13.9952GiB/s) | 20260508 11:01:21.192 | INFO | Analyzing chunk frames = [7586304,7586816) | 20260508 11:01:21.192 | INFO | MemInfo: { | 20260508 11:01:21.192 | INFO | "gpu" : " 17 GB", | 20260508 11:01:21.192 | INFO | "huge" : "", | 20260508 11:01:21.192 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:21.192 | INFO | "malloc" : " 14 GB", | 20260508 11:01:21.192 | INFO | "pinned" : " 29 GB" | 20260508 11:01:21.192 | INFO | } | 20260508 11:01:21.192 | INFO | Perf report for the last 5.1396 seconds: | 20260508 11:01:21.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:21.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.192 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 11:01:21.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.192 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:01:21.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:21.193 | INFO | Uploaded 36GiB to GPU in 2.46668s (14.5945GiB/s) | 20260508 11:01:21.193 | INFO | Downloaded 34GiB to GPU in 2.46304s (13.8041GiB/s) | 20260508 11:01:26.332 | INFO | Analyzing chunk frames = [7586816,7587328) | 20260508 11:01:26.332 | INFO | MemInfo: { | 20260508 11:01:26.332 | INFO | "gpu" : " 17 GB", | 20260508 11:01:26.332 | INFO | "huge" : "", | 20260508 11:01:26.332 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:26.332 | INFO | "malloc" : " 14 GB", | 20260508 11:01:26.332 | INFO | "pinned" : " 29 GB" | 20260508 11:01:26.332 | INFO | } | 20260508 11:01:26.332 | INFO | Perf report for the last 5.13978 seconds: | 20260508 11:01:26.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:26.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:01:26.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:01:26.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:26.332 | INFO | Uploaded 36GiB to GPU in 2.46752s (14.5896GiB/s) | 20260508 11:01:26.332 | INFO | Downloaded 34GiB to GPU in 2.45185s (13.8671GiB/s) | 20260508 11:01:31.474 | INFO | Analyzing chunk frames = [7587328,7587840) | 20260508 11:01:31.474 | INFO | MemInfo: { | 20260508 11:01:31.474 | INFO | "gpu" : " 17 GB", | 20260508 11:01:31.474 | INFO | "huge" : "", | 20260508 11:01:31.474 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:31.474 | INFO | "malloc" : " 14 GB", | 20260508 11:01:31.474 | INFO | "pinned" : " 29 GB" | 20260508 11:01:31.474 | INFO | } | 20260508 11:01:31.474 | INFO | Perf report for the last 5.14238 seconds: | 20260508 11:01:31.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:31.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 11:01:31.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:01:31.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:31.475 | INFO | Uploaded 36GiB to GPU in 2.45029s (14.6921GiB/s) | 20260508 11:01:31.475 | INFO | Downloaded 34GiB to GPU in 2.42399s (14.0264GiB/s) | 20260508 11:01:36.608 | INFO | Analyzing chunk frames = [7587840,7588352) | 20260508 11:01:36.608 | INFO | MemInfo: { | 20260508 11:01:36.608 | INFO | "gpu" : " 17 GB", | 20260508 11:01:36.608 | INFO | "huge" : "", | 20260508 11:01:36.608 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:36.608 | INFO | "malloc" : " 14 GB", | 20260508 11:01:36.608 | INFO | "pinned" : " 29 GB" | 20260508 11:01:36.608 | INFO | } | 20260508 11:01:36.608 | INFO | Perf report for the last 5.13354 seconds: | 20260508 11:01:36.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:36.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.608 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 11:01:36.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.608 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:01:36.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:36.609 | INFO | Uploaded 36GiB to GPU in 2.40365s (14.9772GiB/s) | 20260508 11:01:36.609 | INFO | Downloaded 34GiB to GPU in 2.51452s (13.5215GiB/s) | 20260508 11:01:41.750 | INFO | Analyzing chunk frames = [7588352,7588864) | 20260508 11:01:41.750 | INFO | MemInfo: { | 20260508 11:01:41.750 | INFO | "gpu" : " 17 GB", | 20260508 11:01:41.750 | INFO | "huge" : "", | 20260508 11:01:41.750 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:41.750 | INFO | "malloc" : " 14 GB", | 20260508 11:01:41.750 | INFO | "pinned" : " 29 GB" | 20260508 11:01:41.750 | INFO | } | 20260508 11:01:41.750 | INFO | Perf report for the last 5.14234 seconds: | 20260508 11:01:41.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:41.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.750 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:01:41.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.750 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260508 11:01:41.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:41.751 | INFO | Uploaded 36GiB to GPU in 2.37672s (15.1469GiB/s) | 20260508 11:01:41.751 | INFO | Downloaded 34GiB to GPU in 2.44685s (13.8954GiB/s) | 20260508 11:01:46.886 | INFO | Analyzing chunk frames = [7588864,7589376) | 20260508 11:01:46.886 | INFO | MemInfo: { | 20260508 11:01:46.886 | INFO | "gpu" : " 17 GB", | 20260508 11:01:46.886 | INFO | "huge" : "", | 20260508 11:01:46.886 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:46.886 | INFO | "malloc" : " 14 GB", | 20260508 11:01:46.886 | INFO | "pinned" : " 29 GB" | 20260508 11:01:46.886 | INFO | } | 20260508 11:01:46.886 | INFO | Perf report for the last 5.13534 seconds: | 20260508 11:01:46.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:46.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:01:46.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:01:46.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:46.886 | INFO | Uploaded 36GiB to GPU in 2.40368s (14.9771GiB/s) | 20260508 11:01:46.886 | INFO | Downloaded 34GiB to GPU in 2.46785s (13.7772GiB/s) | 20260508 11:01:52.031 | INFO | Analyzing chunk frames = [7589376,7589888) | 20260508 11:01:52.031 | INFO | MemInfo: { | 20260508 11:01:52.031 | INFO | "gpu" : " 17 GB", | 20260508 11:01:52.031 | INFO | "huge" : "", | 20260508 11:01:52.031 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:52.031 | INFO | "malloc" : " 14 GB", | 20260508 11:01:52.031 | INFO | "pinned" : " 29 GB" | 20260508 11:01:52.031 | INFO | } | 20260508 11:01:52.031 | INFO | Perf report for the last 5.14507 seconds: | 20260508 11:01:52.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:52.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 11:01:52.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:01:52.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:52.031 | INFO | Uploaded 36GiB to GPU in 2.40907s (14.9436GiB/s) | 20260508 11:01:52.031 | INFO | Downloaded 34GiB to GPU in 2.45421s (13.8538GiB/s) | 20260508 11:01:52.128 | INFO | Starting a round of Baz IO | 20260508 11:01:55.523 | INFO | Populating RTMetrics report | 20260508 11:01:55.523 | INFO | RTMetrics report populated | 20260508 11:01:55.524 | INFO | Opening tmp RTMetrics json file | 20260508 11:01:55.537 | INFO | Serializing RTMetrics to json | 20260508 11:01:55.538 | INFO | Serialization done, swapping tmp file now... | 20260508 11:01:55.539 | INFO | RTMetrics done | 20260508 11:01:55.600 | INFO | Finished a round of Baz IO: Wrote 1.446 GiB in 3.471 seconds | 20260508 11:01:55.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:01:55.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.204551% / 0% / 6.35783e-05% | 20260508 11:01:55.600 | INFO | Baz IO overall: | 20260508 11:01:55.600 | INFO | BytesWritten : 3282439633676 | 20260508 11:01:55.600 | INFO | NumEvents : 1577036583481 | 20260508 11:01:57.165 | INFO | Analyzing chunk frames = [7589888,7590400) | 20260508 11:01:57.165 | INFO | MemInfo: { | 20260508 11:01:57.165 | INFO | "gpu" : " 17 GB", | 20260508 11:01:57.165 | INFO | "huge" : "", | 20260508 11:01:57.165 | INFO | "huge_shared" : " 29 GB", | 20260508 11:01:57.165 | INFO | "malloc" : " 14 GB", | 20260508 11:01:57.165 | INFO | "pinned" : " 29 GB" | 20260508 11:01:57.165 | INFO | } | 20260508 11:01:57.165 | INFO | Perf report for the last 5.13453 seconds: | 20260508 11:01:57.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:01:57.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:57.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:57.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:57.165 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 11:01:57.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:01:57.165 | INFO | RT_METRICS : 39.0%, 1.0, 20.4ms, 0 | 20260508 11:01:57.165 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 11:01:57.165 | INFO | BAZWRITER : 66.9%, 2.5, 86.6ms, 0 | 20260508 11:01:57.165 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:01:57.166 | INFO | Uploaded 36GiB to GPU in 2.59537s (13.8708GiB/s) | 20260508 11:01:57.166 | INFO | Downloaded 38GiB to GPU in 2.73133s (13.9127GiB/s) | 20260508 11:02:02.311 | INFO | Analyzing chunk frames = [7590400,7590912) | 20260508 11:02:02.311 | INFO | MemInfo: { | 20260508 11:02:02.311 | INFO | "gpu" : " 17 GB", | 20260508 11:02:02.311 | INFO | "huge" : "", | 20260508 11:02:02.311 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:02.311 | INFO | "malloc" : " 14 GB", | 20260508 11:02:02.311 | INFO | "pinned" : " 29 GB" | 20260508 11:02:02.311 | INFO | } | 20260508 11:02:02.312 | INFO | Perf report for the last 5.14613 seconds: | 20260508 11:02:02.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:02.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:02:02.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:02:02.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:02.312 | INFO | Uploaded 36GiB to GPU in 2.42806s (14.8267GiB/s) | 20260508 11:02:02.312 | INFO | Downloaded 34GiB to GPU in 2.47491s (13.7379GiB/s) | 20260508 11:02:07.450 | INFO | Analyzing chunk frames = [7590912,7591424) | 20260508 11:02:07.450 | INFO | MemInfo: { | 20260508 11:02:07.450 | INFO | "gpu" : " 17 GB", | 20260508 11:02:07.450 | INFO | "huge" : "", | 20260508 11:02:07.450 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:07.450 | INFO | "malloc" : " 14 GB", | 20260508 11:02:07.450 | INFO | "pinned" : " 29 GB" | 20260508 11:02:07.450 | INFO | } | 20260508 11:02:07.450 | INFO | Perf report for the last 5.13893 seconds: | 20260508 11:02:07.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:07.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.451 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:02:07.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.451 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:02:07.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:07.451 | INFO | Uploaded 36GiB to GPU in 2.45355s (14.6726GiB/s) | 20260508 11:02:07.451 | INFO | Downloaded 34GiB to GPU in 2.46689s (13.7825GiB/s) | 20260508 11:02:12.590 | INFO | Analyzing chunk frames = [7591424,7591936) | 20260508 11:02:12.590 | INFO | MemInfo: { | 20260508 11:02:12.590 | INFO | "gpu" : " 17 GB", | 20260508 11:02:12.590 | INFO | "huge" : "", | 20260508 11:02:12.590 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:12.590 | INFO | "malloc" : " 14 GB", | 20260508 11:02:12.590 | INFO | "pinned" : " 29 GB" | 20260508 11:02:12.590 | INFO | } | 20260508 11:02:12.590 | INFO | Perf report for the last 5.13985 seconds: | 20260508 11:02:12.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:12.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.590 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 11:02:12.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.590 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:02:12.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:12.591 | INFO | Uploaded 36GiB to GPU in 2.44732s (14.71GiB/s) | 20260508 11:02:12.591 | INFO | Downloaded 34GiB to GPU in 2.46727s (13.7804GiB/s) | 20260508 11:02:17.719 | INFO | Analyzing chunk frames = [7591936,7592448) | 20260508 11:02:17.719 | INFO | MemInfo: { | 20260508 11:02:17.719 | INFO | "gpu" : " 17 GB", | 20260508 11:02:17.719 | INFO | "huge" : "", | 20260508 11:02:17.719 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:17.719 | INFO | "malloc" : " 14 GB", | 20260508 11:02:17.719 | INFO | "pinned" : " 29 GB" | 20260508 11:02:17.719 | INFO | } | 20260508 11:02:17.719 | INFO | Perf report for the last 5.12839 seconds: | 20260508 11:02:17.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:17.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:02:17.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:02:17.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:17.719 | INFO | Uploaded 36GiB to GPU in 2.2776s (15.8061GiB/s) | 20260508 11:02:17.719 | INFO | Downloaded 34GiB to GPU in 2.51611s (13.5129GiB/s) | 20260508 11:02:22.861 | INFO | Analyzing chunk frames = [7592448,7592960) | 20260508 11:02:22.861 | INFO | MemInfo: { | 20260508 11:02:22.861 | INFO | "gpu" : " 17 GB", | 20260508 11:02:22.861 | INFO | "huge" : "", | 20260508 11:02:22.861 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:22.861 | INFO | "malloc" : " 14 GB", | 20260508 11:02:22.861 | INFO | "pinned" : " 29 GB" | 20260508 11:02:22.861 | INFO | } | 20260508 11:02:22.861 | INFO | Perf report for the last 5.14254 seconds: | 20260508 11:02:22.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:22.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.861 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:02:22.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.861 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:02:22.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:22.862 | INFO | Uploaded 36GiB to GPU in 2.35381s (15.2944GiB/s) | 20260508 11:02:22.862 | INFO | Downloaded 34GiB to GPU in 2.44033s (13.9326GiB/s) | 20260508 11:02:28.002 | INFO | Analyzing chunk frames = [7592960,7593472) | 20260508 11:02:28.002 | INFO | MemInfo: { | 20260508 11:02:28.002 | INFO | "gpu" : " 17 GB", | 20260508 11:02:28.002 | INFO | "huge" : "", | 20260508 11:02:28.002 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:28.002 | INFO | "malloc" : " 14 GB", | 20260508 11:02:28.002 | INFO | "pinned" : " 29 GB" | 20260508 11:02:28.002 | INFO | } | 20260508 11:02:28.002 | INFO | Perf report for the last 5.14095 seconds: | 20260508 11:02:28.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:28.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.002 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:02:28.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.002 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:02:28.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:28.003 | INFO | Uploaded 36GiB to GPU in 2.41869s (14.8841GiB/s) | 20260508 11:02:28.003 | INFO | Downloaded 34GiB to GPU in 2.44615s (13.8994GiB/s) | 20260508 11:02:33.145 | INFO | Analyzing chunk frames = [7593472,7593984) | 20260508 11:02:33.145 | INFO | MemInfo: { | 20260508 11:02:33.145 | INFO | "gpu" : " 17 GB", | 20260508 11:02:33.145 | INFO | "huge" : "", | 20260508 11:02:33.145 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:33.145 | INFO | "malloc" : " 14 GB", | 20260508 11:02:33.145 | INFO | "pinned" : " 29 GB" | 20260508 11:02:33.145 | INFO | } | 20260508 11:02:33.145 | INFO | Perf report for the last 5.14251 seconds: | 20260508 11:02:33.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:33.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 11:02:33.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 11:02:33.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:33.145 | INFO | Uploaded 36GiB to GPU in 2.43079s (14.81GiB/s) | 20260508 11:02:33.145 | INFO | Downloaded 34GiB to GPU in 2.44759s (13.8912GiB/s) | 20260508 11:02:33.145 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:02:33.145 | INFO | Total mem usage: 16.7 GiB | 20260508 11:02:33.145 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:02:33.145 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:02:33.145 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:02:33.145 | INFO | High Water Breakdown: | 20260508 11:02:33.145 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:02:33.145 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:02:33.145 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:02:33.145 | INFO | ------------------------------------------ | 20260508 11:02:33.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:02:33.145 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:02:33.145 | INFO | Total mem usage: 26.5 GiB | 20260508 11:02:33.145 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:02:33.145 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:02:33.145 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:02:33.145 | INFO | High Water Breakdown: | 20260508 11:02:33.145 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:02:33.145 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:02:33.145 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:02:33.145 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:02:33.145 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:02:33.145 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:02:33.145 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:02:33.145 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:02:33.145 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:02:33.145 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:02:33.145 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:02:33.145 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:02:33.146 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:02:33.146 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:02:33.146 | INFO | ------------------------------------------ | 20260508 11:02:33.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:02:33.146 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:02:33.146 | INFO | Total mem usage: 27.8 GiB | 20260508 11:02:33.146 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:02:33.146 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:02:33.146 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:02:33.146 | INFO | High Water Breakdown: | 20260508 11:02:33.146 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:02:33.146 | INFO | ------------------------------------------ | 20260508 11:02:33.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:02:33.146 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:02:33.146 | INFO | Total mem usage: 15.7 GiB | 20260508 11:02:33.146 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:02:33.146 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:02:33.146 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:02:33.146 | INFO | High Water Breakdown: | 20260508 11:02:33.146 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:02:33.146 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:02:33.146 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:02:33.146 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:02:33.146 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:02:33.146 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:02:33.146 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:02:33.146 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:02:33.146 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:02:33.146 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:02:33.146 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:02:33.146 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:02:33.146 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:02:33.146 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:02:33.146 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:02:33.146 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:02:33.146 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:02:33.146 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:02:33.146 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:02:33.146 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:02:33.146 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:02:33.146 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:02:33.146 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:02:33.146 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:02:33.146 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:02:33.146 | INFO | ------------------------------------------ | 20260508 11:02:33.146 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:02:33.146 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:02:33.146 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:02:33.146 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:02:33.146 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.42e-01 | 20260508 11:02:33.146 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.01e-01, 1.17e-02 / 4.89e-03 | 20260508 11:02:33.146 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.38e+00, 5.91e-01 / 2.49e-01 | 20260508 11:02:33.146 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.12e-01 | 20260508 11:02:33.146 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.52e+01, 3.78e+00 / 9.46e-01 | 20260508 11:02:33.146 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.77e+00 / 1.25e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.06e-01 / 2.48e-01 | 20260508 11:02:33.146 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.11e+00, 2.43e-01 / 1.10e-01 | 20260508 11:02:33.146 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.31e-01, 4.86e-02 / 2.48e-02 | 20260508 11:02:33.146 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:02:33.146 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:02:33.146 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:02:33.146 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 24.14%, Upload -- 2.54e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 2.03%, LockUpload -- 2.14e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 2.71%, LockDownload -- 2.85e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 15.30%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:02:33.146 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260508 11:02:33.242 | INFO | Starting a round of Baz IO | 20260508 11:02:36.664 | INFO | Populating RTMetrics report | 20260508 11:02:36.664 | INFO | RTMetrics report populated | 20260508 11:02:36.664 | INFO | Opening tmp RTMetrics json file | 20260508 11:02:36.673 | INFO | Serializing RTMetrics to json | 20260508 11:02:36.674 | INFO | Serialization done, swapping tmp file now... | 20260508 11:02:36.675 | INFO | RTMetrics done | 20260508 11:02:36.727 | INFO | Finished a round of Baz IO: Wrote 1.446 GiB in 3.485 seconds | 20260508 11:02:36.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:02:36.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203705% / 3.97364e-06% / 5.16574e-05% | 20260508 11:02:36.727 | INFO | Baz IO overall: | 20260508 11:02:36.727 | INFO | BytesWritten : 3283992496195 | 20260508 11:02:36.727 | INFO | NumEvents : 1577768053757 | 20260508 11:02:38.285 | INFO | Analyzing chunk frames = [7593984,7594496) | 20260508 11:02:38.285 | INFO | MemInfo: { | 20260508 11:02:38.285 | INFO | "gpu" : " 17 GB", | 20260508 11:02:38.285 | INFO | "huge" : "", | 20260508 11:02:38.285 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:38.285 | INFO | "malloc" : " 14 GB", | 20260508 11:02:38.285 | INFO | "pinned" : " 29 GB" | 20260508 11:02:38.285 | INFO | } | 20260508 11:02:38.285 | INFO | Perf report for the last 5.1399 seconds: | 20260508 11:02:38.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:38.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:38.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:38.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:38.285 | INFO | ANALYSIS : 67.8%, 3.0, 105.8ms, 0 | 20260508 11:02:38.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:38.285 | INFO | RT_METRICS : 39.0%, 1.0, 20.5ms, 0 | 20260508 11:02:38.285 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 11:02:38.285 | INFO | BAZWRITER : 67.1%, 2.5, 87.1ms, 0 | 20260508 11:02:38.285 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:02:38.285 | INFO | Uploaded 36GiB to GPU in 2.5777s (13.9659GiB/s) | 20260508 11:02:38.285 | INFO | Downloaded 38GiB to GPU in 2.70918s (14.0264GiB/s) | 20260508 11:02:43.415 | INFO | Analyzing chunk frames = [7594496,7595008) | 20260508 11:02:43.415 | INFO | MemInfo: { | 20260508 11:02:43.415 | INFO | "gpu" : " 17 GB", | 20260508 11:02:43.415 | INFO | "huge" : "", | 20260508 11:02:43.415 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:43.415 | INFO | "malloc" : " 14 GB", | 20260508 11:02:43.415 | INFO | "pinned" : " 29 GB" | 20260508 11:02:43.415 | INFO | } | 20260508 11:02:43.415 | INFO | Perf report for the last 5.13047 seconds: | 20260508 11:02:43.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:43.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:02:43.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | PRE_HQ : 22.7%, 1.0, 11.7ms, 0 | 20260508 11:02:43.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:43.415 | INFO | Uploaded 36GiB to GPU in 2.4694s (14.5785GiB/s) | 20260508 11:02:43.415 | INFO | Downloaded 34GiB to GPU in 2.45891s (13.8272GiB/s) | 20260508 11:02:48.564 | INFO | Analyzing chunk frames = [7595008,7595520) | 20260508 11:02:48.564 | INFO | MemInfo: { | 20260508 11:02:48.564 | INFO | "gpu" : " 17 GB", | 20260508 11:02:48.564 | INFO | "huge" : "", | 20260508 11:02:48.564 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:48.564 | INFO | "malloc" : " 14 GB", | 20260508 11:02:48.564 | INFO | "pinned" : " 29 GB" | 20260508 11:02:48.564 | INFO | } | 20260508 11:02:48.564 | INFO | Perf report for the last 5.14893 seconds: | 20260508 11:02:48.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:48.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:02:48.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 11:02:48.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:48.564 | INFO | Uploaded 36GiB to GPU in 2.50526s (14.3698GiB/s) | 20260508 11:02:48.564 | INFO | Downloaded 34GiB to GPU in 2.42622s (14.0136GiB/s) | 20260508 11:02:53.699 | INFO | Analyzing chunk frames = [7595520,7596032) | 20260508 11:02:53.699 | INFO | MemInfo: { | 20260508 11:02:53.699 | INFO | "gpu" : " 17 GB", | 20260508 11:02:53.699 | INFO | "huge" : "", | 20260508 11:02:53.699 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:53.699 | INFO | "malloc" : " 14 GB", | 20260508 11:02:53.699 | INFO | "pinned" : " 29 GB" | 20260508 11:02:53.699 | INFO | } | 20260508 11:02:53.699 | INFO | Perf report for the last 5.13462 seconds: | 20260508 11:02:53.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:53.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:02:53.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:02:53.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:53.699 | INFO | Uploaded 36GiB to GPU in 2.4947s (14.4306GiB/s) | 20260508 11:02:53.699 | INFO | Downloaded 34GiB to GPU in 2.44363s (13.9137GiB/s) | 20260508 11:02:58.842 | INFO | Analyzing chunk frames = [7596032,7596544) | 20260508 11:02:58.842 | INFO | MemInfo: { | 20260508 11:02:58.842 | INFO | "gpu" : " 17 GB", | 20260508 11:02:58.842 | INFO | "huge" : "", | 20260508 11:02:58.842 | INFO | "huge_shared" : " 29 GB", | 20260508 11:02:58.842 | INFO | "malloc" : " 14 GB", | 20260508 11:02:58.842 | INFO | "pinned" : " 29 GB" | 20260508 11:02:58.842 | INFO | } | 20260508 11:02:58.842 | INFO | Perf report for the last 5.14347 seconds: | 20260508 11:02:58.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:02:58.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.842 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 11:02:58.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.842 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:02:58.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:02:58.843 | INFO | Uploaded 36GiB to GPU in 2.42544s (14.8427GiB/s) | 20260508 11:02:58.843 | INFO | Downloaded 34GiB to GPU in 2.53037s (13.4368GiB/s) | 20260508 11:03:03.978 | INFO | Analyzing chunk frames = [7596544,7597056) | 20260508 11:03:03.978 | INFO | MemInfo: { | 20260508 11:03:03.978 | INFO | "gpu" : " 17 GB", | 20260508 11:03:03.978 | INFO | "huge" : "", | 20260508 11:03:03.978 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:03.978 | INFO | "malloc" : " 14 GB", | 20260508 11:03:03.978 | INFO | "pinned" : " 29 GB" | 20260508 11:03:03.978 | INFO | } | 20260508 11:03:03.978 | INFO | Perf report for the last 5.13571 seconds: | 20260508 11:03:03.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:03.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:03:03.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:03:03.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:03.978 | INFO | Uploaded 36GiB to GPU in 2.39946s (15.0034GiB/s) | 20260508 11:03:03.978 | INFO | Downloaded 34GiB to GPU in 2.43096s (13.9863GiB/s) | 20260508 11:03:09.125 | INFO | Analyzing chunk frames = [7597056,7597568) | 20260508 11:03:09.125 | INFO | MemInfo: { | 20260508 11:03:09.125 | INFO | "gpu" : " 17 GB", | 20260508 11:03:09.125 | INFO | "huge" : "", | 20260508 11:03:09.125 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:09.125 | INFO | "malloc" : " 14 GB", | 20260508 11:03:09.125 | INFO | "pinned" : " 29 GB" | 20260508 11:03:09.125 | INFO | } | 20260508 11:03:09.125 | INFO | Perf report for the last 5.14694 seconds: | 20260508 11:03:09.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:09.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 11:03:09.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:03:09.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:09.125 | INFO | Uploaded 36GiB to GPU in 2.44107s (14.7476GiB/s) | 20260508 11:03:09.125 | INFO | Downloaded 34GiB to GPU in 2.44488s (13.9066GiB/s) | 20260508 11:03:14.259 | INFO | Analyzing chunk frames = [7597568,7598080) | 20260508 11:03:14.259 | INFO | MemInfo: { | 20260508 11:03:14.259 | INFO | "gpu" : " 17 GB", | 20260508 11:03:14.259 | INFO | "huge" : "", | 20260508 11:03:14.259 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:14.259 | INFO | "malloc" : " 14 GB", | 20260508 11:03:14.259 | INFO | "pinned" : " 29 GB" | 20260508 11:03:14.259 | INFO | } | 20260508 11:03:14.260 | INFO | Perf report for the last 5.13465 seconds: | 20260508 11:03:14.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:14.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 11:03:14.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:03:14.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:14.260 | INFO | Uploaded 36GiB to GPU in 2.38688s (15.0824GiB/s) | 20260508 11:03:14.260 | INFO | Downloaded 34GiB to GPU in 2.45012s (13.8769GiB/s) | 20260508 11:03:14.357 | INFO | Starting a round of Baz IO | 20260508 11:03:17.759 | INFO | Populating RTMetrics report | 20260508 11:03:17.759 | INFO | RTMetrics report populated | 20260508 11:03:17.759 | INFO | Opening tmp RTMetrics json file | 20260508 11:03:17.761 | INFO | Serializing RTMetrics to json | 20260508 11:03:17.762 | INFO | Serialization done, swapping tmp file now... | 20260508 11:03:17.763 | INFO | RTMetrics done | 20260508 11:03:17.844 | INFO | Finished a round of Baz IO: Wrote 1.445 GiB in 3.486 seconds | 20260508 11:03:17.844 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:03:17.844 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203339% / 0% / 5.16574e-05% | 20260508 11:03:17.844 | INFO | Baz IO overall: | 20260508 11:03:17.844 | INFO | BytesWritten : 3285544155715 | 20260508 11:03:17.844 | INFO | NumEvents : 1578498352149 | 20260508 11:03:19.396 | INFO | Analyzing chunk frames = [7598080,7598592) | 20260508 11:03:19.396 | INFO | MemInfo: { | 20260508 11:03:19.396 | INFO | "gpu" : " 17 GB", | 20260508 11:03:19.396 | INFO | "huge" : "", | 20260508 11:03:19.396 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:19.396 | INFO | "malloc" : " 14 GB", | 20260508 11:03:19.396 | INFO | "pinned" : " 29 GB" | 20260508 11:03:19.396 | INFO | } | 20260508 11:03:19.396 | INFO | Perf report for the last 5.13647 seconds: | 20260508 11:03:19.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:19.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:19.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:19.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:19.396 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:03:19.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:19.396 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:03:19.396 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 11:03:19.396 | INFO | BAZWRITER : 67.2%, 2.8, 97.2ms, 0 | 20260508 11:03:19.396 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:03:19.397 | INFO | Uploaded 36GiB to GPU in 2.60918s (13.7974GiB/s) | 20260508 11:03:19.397 | INFO | Downloaded 38GiB to GPU in 2.71469s (13.9979GiB/s) | 20260508 11:03:24.534 | INFO | Analyzing chunk frames = [7598592,7599104) | 20260508 11:03:24.534 | INFO | MemInfo: { | 20260508 11:03:24.534 | INFO | "gpu" : " 17 GB", | 20260508 11:03:24.534 | INFO | "huge" : "", | 20260508 11:03:24.534 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:24.534 | INFO | "malloc" : " 14 GB", | 20260508 11:03:24.534 | INFO | "pinned" : " 29 GB" | 20260508 11:03:24.534 | INFO | } | 20260508 11:03:24.534 | INFO | Perf report for the last 5.13824 seconds: | 20260508 11:03:24.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:24.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.534 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 11:03:24.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.534 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 11:03:24.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:24.535 | INFO | Uploaded 36GiB to GPU in 2.46723s (14.5912GiB/s) | 20260508 11:03:24.535 | INFO | Downloaded 34GiB to GPU in 2.46515s (13.7923GiB/s) | 20260508 11:03:29.670 | INFO | Analyzing chunk frames = [7599104,7599616) | 20260508 11:03:29.670 | INFO | MemInfo: { | 20260508 11:03:29.670 | INFO | "gpu" : " 17 GB", | 20260508 11:03:29.670 | INFO | "huge" : "", | 20260508 11:03:29.671 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:29.671 | INFO | "malloc" : " 14 GB", | 20260508 11:03:29.671 | INFO | "pinned" : " 29 GB" | 20260508 11:03:29.671 | INFO | } | 20260508 11:03:29.671 | INFO | Perf report for the last 5.13639 seconds: | 20260508 11:03:29.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:29.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:03:29.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:03:29.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:29.671 | INFO | Uploaded 36GiB to GPU in 2.47764s (14.5299GiB/s) | 20260508 11:03:29.671 | INFO | Downloaded 34GiB to GPU in 2.43024s (13.9904GiB/s) | 20260508 11:03:34.811 | INFO | Analyzing chunk frames = [7599616,7600128) | 20260508 11:03:34.811 | INFO | MemInfo: { | 20260508 11:03:34.811 | INFO | "gpu" : " 17 GB", | 20260508 11:03:34.811 | INFO | "huge" : "", | 20260508 11:03:34.811 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:34.811 | INFO | "malloc" : " 14 GB", | 20260508 11:03:34.811 | INFO | "pinned" : " 29 GB" | 20260508 11:03:34.811 | INFO | } | 20260508 11:03:34.812 | INFO | Perf report for the last 5.14085 seconds: | 20260508 11:03:34.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:34.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 11:03:34.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:03:34.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:34.812 | INFO | Uploaded 36GiB to GPU in 2.47321s (14.556GiB/s) | 20260508 11:03:34.812 | INFO | Downloaded 34GiB to GPU in 2.42802s (14.0032GiB/s) | 20260508 11:03:39.955 | INFO | Analyzing chunk frames = [7600128,7600640) | 20260508 11:03:39.955 | INFO | MemInfo: { | 20260508 11:03:39.955 | INFO | "gpu" : " 17 GB", | 20260508 11:03:39.955 | INFO | "huge" : "", | 20260508 11:03:39.955 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:39.955 | INFO | "malloc" : " 14 GB", | 20260508 11:03:39.955 | INFO | "pinned" : " 29 GB" | 20260508 11:03:39.955 | INFO | } | 20260508 11:03:39.955 | INFO | Perf report for the last 5.14394 seconds: | 20260508 11:03:39.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:39.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 11:03:39.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:03:39.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:39.956 | INFO | Uploaded 36GiB to GPU in 2.41525s (14.9053GiB/s) | 20260508 11:03:39.956 | INFO | Downloaded 34GiB to GPU in 2.52058s (13.489GiB/s) | 20260508 11:03:45.093 | INFO | Analyzing chunk frames = [7600640,7601152) | 20260508 11:03:45.093 | INFO | MemInfo: { | 20260508 11:03:45.093 | INFO | "gpu" : " 17 GB", | 20260508 11:03:45.093 | INFO | "huge" : "", | 20260508 11:03:45.093 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:45.093 | INFO | "malloc" : " 14 GB", | 20260508 11:03:45.093 | INFO | "pinned" : " 29 GB" | 20260508 11:03:45.093 | INFO | } | 20260508 11:03:45.093 | INFO | Perf report for the last 5.13743 seconds: | 20260508 11:03:45.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:45.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:03:45.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:03:45.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:45.093 | INFO | Uploaded 36GiB to GPU in 2.38247s (15.1104GiB/s) | 20260508 11:03:45.093 | INFO | Downloaded 34GiB to GPU in 2.41564s (14.075GiB/s) | 20260508 11:03:50.230 | INFO | Analyzing chunk frames = [7601152,7601664) | 20260508 11:03:50.230 | INFO | MemInfo: { | 20260508 11:03:50.230 | INFO | "gpu" : " 17 GB", | 20260508 11:03:50.230 | INFO | "huge" : "", | 20260508 11:03:50.230 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:50.230 | INFO | "malloc" : " 14 GB", | 20260508 11:03:50.230 | INFO | "pinned" : " 29 GB" | 20260508 11:03:50.231 | INFO | } | 20260508 11:03:50.231 | INFO | Perf report for the last 5.13768 seconds: | 20260508 11:03:50.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:50.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:03:50.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:03:50.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:50.231 | INFO | Uploaded 36GiB to GPU in 2.45417s (14.6689GiB/s) | 20260508 11:03:50.231 | INFO | Downloaded 34GiB to GPU in 2.43485s (13.9639GiB/s) | 20260508 11:03:55.371 | INFO | Analyzing chunk frames = [7601664,7602176) | 20260508 11:03:55.371 | INFO | MemInfo: { | 20260508 11:03:55.371 | INFO | "gpu" : " 17 GB", | 20260508 11:03:55.371 | INFO | "huge" : "", | 20260508 11:03:55.371 | INFO | "huge_shared" : " 29 GB", | 20260508 11:03:55.371 | INFO | "malloc" : " 14 GB", | 20260508 11:03:55.372 | INFO | "pinned" : " 29 GB" | 20260508 11:03:55.372 | INFO | } | 20260508 11:03:55.372 | INFO | Perf report for the last 5.14101 seconds: | 20260508 11:03:55.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:03:55.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 11:03:55.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 11:03:55.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:03:55.372 | INFO | Uploaded 36GiB to GPU in 2.45923s (14.6387GiB/s) | 20260508 11:03:55.372 | INFO | Downloaded 34GiB to GPU in 2.43264s (13.9766GiB/s) | 20260508 11:03:55.372 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:03:55.372 | INFO | Total mem usage: 16.7 GiB | 20260508 11:03:55.372 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:03:55.372 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260508 11:03:55.372 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:03:55.372 | INFO | High Water Breakdown: | 20260508 11:03:55.372 | INFO | 59.9% : 8.76 GiB ( 85% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:03:55.372 | INFO | 40% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:03:55.372 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:03:55.372 | INFO | ------------------------------------------ | 20260508 11:03:55.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:03:55.372 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:03:55.372 | INFO | Total mem usage: 26.5 GiB | 20260508 11:03:55.372 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:03:55.372 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:03:55.372 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:03:55.372 | INFO | High Water Breakdown: | 20260508 11:03:55.372 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:03:55.372 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:03:55.372 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:03:55.372 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:03:55.372 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:03:55.372 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:03:55.372 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:03:55.372 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:03:55.372 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:03:55.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:03:55.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:03:55.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:03:55.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:03:55.372 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:03:55.372 | INFO | ------------------------------------------ | 20260508 11:03:55.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:03:55.372 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:03:55.372 | INFO | Total mem usage: 27.8 GiB | 20260508 11:03:55.372 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:03:55.372 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:03:55.372 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:03:55.372 | INFO | High Water Breakdown: | 20260508 11:03:55.372 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:03:55.372 | INFO | ------------------------------------------ | 20260508 11:03:55.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:03:55.372 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:03:55.373 | INFO | Total mem usage: 15.7 GiB | 20260508 11:03:55.373 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:03:55.373 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:03:55.373 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:03:55.373 | INFO | High Water Breakdown: | 20260508 11:03:55.373 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:03:55.373 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:03:55.373 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:03:55.373 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:03:55.373 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:03:55.373 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:03:55.373 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:03:55.373 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:03:55.373 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:03:55.373 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:03:55.373 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:03:55.373 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:03:55.373 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:03:55.373 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:03:55.373 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:03:55.373 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:03:55.373 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:03:55.373 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:03:55.373 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:03:55.373 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:03:55.373 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:03:55.373 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:03:55.373 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:03:55.373 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:03:55.373 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:03:55.373 | INFO | ------------------------------------------ | 20260508 11:03:55.373 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:03:55.373 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:03:55.373 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:03:55.373 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:03:55.373 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 6.24e-01 | 20260508 11:03:55.373 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.11e-02 / 5.37e-03 | 20260508 11:03:55.373 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.34e+00, 5.92e-01 / 2.52e-01 | 20260508 11:03:55.373 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.10e-01 | 20260508 11:03:55.373 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.50e+01, 3.81e+00 / 1.17e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.79e+00 / 1.30e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.39e+00, 4.03e-01 / 2.25e-01 | 20260508 11:03:55.373 | INFO | PerfReport: 3.00%, Metrics -- 9.81e-01 / 1.13e+00, 2.42e-01 / 1.07e-01 | 20260508 11:03:55.373 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.21e-01, 4.77e-02 / 2.77e-02 | 20260508 11:03:55.373 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:03:55.373 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:03:55.373 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:03:55.373 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 24.41%, Upload -- 2.57e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 2.50%, LockDownload -- 2.63e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 15.18%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:03:55.373 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260508 11:03:55.469 | INFO | Starting a round of Baz IO | 20260508 11:03:58.878 | INFO | Populating RTMetrics report | 20260508 11:03:58.878 | INFO | RTMetrics report populated | 20260508 11:03:58.878 | INFO | Opening tmp RTMetrics json file | 20260508 11:03:58.879 | INFO | Serializing RTMetrics to json | 20260508 11:03:58.880 | INFO | Serialization done, swapping tmp file now... | 20260508 11:03:58.880 | INFO | RTMetrics done | 20260508 11:03:58.951 | INFO | Finished a round of Baz IO: Wrote 1.445 GiB in 3.482 seconds | 20260508 11:03:58.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:03:58.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203005% / 0% / 5.16574e-05% | 20260508 11:03:58.951 | INFO | Baz IO overall: | 20260508 11:03:58.951 | INFO | BytesWritten : 3287095975965 | 20260508 11:03:58.951 | INFO | NumEvents : 1579229087232 | 20260508 11:04:00.510 | INFO | Analyzing chunk frames = [7602176,7602688) | 20260508 11:04:00.510 | INFO | MemInfo: { | 20260508 11:04:00.510 | INFO | "gpu" : " 17 GB", | 20260508 11:04:00.510 | INFO | "huge" : "", | 20260508 11:04:00.510 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:00.510 | INFO | "malloc" : " 14 GB", | 20260508 11:04:00.510 | INFO | "pinned" : " 29 GB" | 20260508 11:04:00.510 | INFO | } | 20260508 11:04:00.510 | INFO | Perf report for the last 5.13817 seconds: | 20260508 11:04:00.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:00.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:00.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:00.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:00.510 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:04:00.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:00.510 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:04:00.510 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 11:04:00.510 | INFO | BAZWRITER : 67.1%, 2.5, 86.6ms, 0 | 20260508 11:04:00.510 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:04:00.510 | INFO | Uploaded 36GiB to GPU in 2.63847s (13.6443GiB/s) | 20260508 11:04:00.510 | INFO | Downloaded 38GiB to GPU in 2.71429s (14GiB/s) | 20260508 11:04:05.655 | INFO | Analyzing chunk frames = [7602688,7603200) | 20260508 11:04:05.655 | INFO | MemInfo: { | 20260508 11:04:05.655 | INFO | "gpu" : " 17 GB", | 20260508 11:04:05.655 | INFO | "huge" : "", | 20260508 11:04:05.655 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:05.656 | INFO | "malloc" : " 14 GB", | 20260508 11:04:05.656 | INFO | "pinned" : " 29 GB" | 20260508 11:04:05.656 | INFO | } | 20260508 11:04:05.656 | INFO | Perf report for the last 5.14577 seconds: | 20260508 11:04:05.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:05.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:04:05.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:04:05.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:05.656 | INFO | Uploaded 36GiB to GPU in 2.48786s (14.4703GiB/s) | 20260508 11:04:05.656 | INFO | Downloaded 34GiB to GPU in 2.45639s (13.8414GiB/s) | 20260508 11:04:10.792 | INFO | Analyzing chunk frames = [7603200,7603712) | 20260508 11:04:10.792 | INFO | MemInfo: { | 20260508 11:04:10.792 | INFO | "gpu" : " 17 GB", | 20260508 11:04:10.792 | INFO | "huge" : "", | 20260508 11:04:10.792 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:10.792 | INFO | "malloc" : " 14 GB", | 20260508 11:04:10.792 | INFO | "pinned" : " 29 GB" | 20260508 11:04:10.792 | INFO | } | 20260508 11:04:10.793 | INFO | Perf report for the last 5.13691 seconds: | 20260508 11:04:10.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:10.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 11:04:10.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:04:10.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:10.793 | INFO | Uploaded 36GiB to GPU in 2.49429s (14.433GiB/s) | 20260508 11:04:10.793 | INFO | Downloaded 34GiB to GPU in 2.42694s (14.0094GiB/s) | 20260508 11:04:15.926 | INFO | Analyzing chunk frames = [7603712,7604224) | 20260508 11:04:15.926 | INFO | MemInfo: { | 20260508 11:04:15.926 | INFO | "gpu" : " 17 GB", | 20260508 11:04:15.926 | INFO | "huge" : "", | 20260508 11:04:15.926 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:15.926 | INFO | "malloc" : " 14 GB", | 20260508 11:04:15.926 | INFO | "pinned" : " 29 GB" | 20260508 11:04:15.926 | INFO | } | 20260508 11:04:15.926 | INFO | Perf report for the last 5.1335 seconds: | 20260508 11:04:15.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:15.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 11:04:15.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:04:15.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:15.926 | INFO | Uploaded 36GiB to GPU in 2.50675s (14.3612GiB/s) | 20260508 11:04:15.926 | INFO | Downloaded 34GiB to GPU in 2.40646s (14.1287GiB/s) | 20260508 11:04:21.069 | INFO | Analyzing chunk frames = [7604224,7604736) | 20260508 11:04:21.069 | INFO | MemInfo: { | 20260508 11:04:21.069 | INFO | "gpu" : " 17 GB", | 20260508 11:04:21.069 | INFO | "huge" : "", | 20260508 11:04:21.069 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:21.069 | INFO | "malloc" : " 14 GB", | 20260508 11:04:21.069 | INFO | "pinned" : " 29 GB" | 20260508 11:04:21.069 | INFO | } | 20260508 11:04:21.069 | INFO | Perf report for the last 5.14278 seconds: | 20260508 11:04:21.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:21.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:04:21.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 11:04:21.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:21.069 | INFO | Uploaded 36GiB to GPU in 2.32549s (15.4806GiB/s) | 20260508 11:04:21.070 | INFO | Downloaded 34GiB to GPU in 2.48834s (13.6637GiB/s) | 20260508 11:04:26.211 | INFO | Analyzing chunk frames = [7604736,7605248) | 20260508 11:04:26.211 | INFO | MemInfo: { | 20260508 11:04:26.211 | INFO | "gpu" : " 17 GB", | 20260508 11:04:26.211 | INFO | "huge" : "", | 20260508 11:04:26.211 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:26.211 | INFO | "malloc" : " 14 GB", | 20260508 11:04:26.211 | INFO | "pinned" : " 29 GB" | 20260508 11:04:26.211 | INFO | } | 20260508 11:04:26.212 | INFO | Perf report for the last 5.14262 seconds: | 20260508 11:04:26.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:26.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.212 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:04:26.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.212 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:04:26.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:26.213 | INFO | Uploaded 36GiB to GPU in 2.37414s (15.1634GiB/s) | 20260508 11:04:26.213 | INFO | Downloaded 34GiB to GPU in 2.42687s (14.0098GiB/s) | 20260508 11:04:31.344 | INFO | Analyzing chunk frames = [7605248,7605760) | 20260508 11:04:31.344 | INFO | MemInfo: { | 20260508 11:04:31.344 | INFO | "gpu" : " 17 GB", | 20260508 11:04:31.344 | INFO | "huge" : "", | 20260508 11:04:31.344 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:31.344 | INFO | "malloc" : " 14 GB", | 20260508 11:04:31.345 | INFO | "pinned" : " 29 GB" | 20260508 11:04:31.345 | INFO | } | 20260508 11:04:31.345 | INFO | Perf report for the last 5.13321 seconds: | 20260508 11:04:31.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:31.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 11:04:31.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260508 11:04:31.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:31.345 | INFO | Uploaded 36GiB to GPU in 2.46816s (14.5858GiB/s) | 20260508 11:04:31.345 | INFO | Downloaded 34GiB to GPU in 2.46124s (13.8142GiB/s) | 20260508 11:04:36.485 | INFO | Analyzing chunk frames = [7605760,7606272) | 20260508 11:04:36.485 | INFO | MemInfo: { | 20260508 11:04:36.485 | INFO | "gpu" : " 17 GB", | 20260508 11:04:36.485 | INFO | "huge" : "", | 20260508 11:04:36.485 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:36.486 | INFO | "malloc" : " 14 GB", | 20260508 11:04:36.486 | INFO | "pinned" : " 29 GB" | 20260508 11:04:36.486 | INFO | } | 20260508 11:04:36.486 | INFO | Perf report for the last 5.1411 seconds: | 20260508 11:04:36.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:36.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.486 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:04:36.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.486 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 11:04:36.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:36.487 | INFO | Uploaded 36GiB to GPU in 2.43724s (14.7708GiB/s) | 20260508 11:04:36.487 | INFO | Downloaded 34GiB to GPU in 2.44758s (13.8912GiB/s) | 20260508 11:04:36.602 | INFO | Starting a round of Baz IO | 20260508 11:04:39.995 | INFO | Populating RTMetrics report | 20260508 11:04:39.995 | INFO | RTMetrics report populated | 20260508 11:04:39.995 | INFO | Opening tmp RTMetrics json file | 20260508 11:04:39.998 | INFO | Serializing RTMetrics to json | 20260508 11:04:40.000 | INFO | Serialization done, swapping tmp file now... | 20260508 11:04:40.000 | INFO | RTMetrics done | 20260508 11:04:40.056 | INFO | Finished a round of Baz IO: Wrote 1.446 GiB in 3.454 seconds | 20260508 11:04:40.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:04:40.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202851% / 0% / 8.74201e-05% | 20260508 11:04:40.056 | INFO | Baz IO overall: | 20260508 11:04:40.056 | INFO | BytesWritten : 3288648735732 | 20260508 11:04:40.056 | INFO | NumEvents : 1579960570761 | 20260508 11:04:41.628 | INFO | Analyzing chunk frames = [7606272,7606784) | 20260508 11:04:41.628 | INFO | MemInfo: { | 20260508 11:04:41.628 | INFO | "gpu" : " 17 GB", | 20260508 11:04:41.628 | INFO | "huge" : "", | 20260508 11:04:41.628 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:41.628 | INFO | "malloc" : " 14 GB", | 20260508 11:04:41.628 | INFO | "pinned" : " 29 GB" | 20260508 11:04:41.628 | INFO | } | 20260508 11:04:41.628 | INFO | Perf report for the last 5.14241 seconds: | 20260508 11:04:41.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:41.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:41.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:41.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:41.628 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 11:04:41.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:41.628 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260508 11:04:41.628 | INFO | PRE_HQ : 66.3%, 1.8, 61.8ms, 0 | 20260508 11:04:41.628 | INFO | BAZWRITER : 66.5%, 2.7, 93.2ms, 0 | 20260508 11:04:41.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:04:41.629 | INFO | Uploaded 36GiB to GPU in 2.58623s (13.9198GiB/s) | 20260508 11:04:41.629 | INFO | Downloaded 38GiB to GPU in 2.7243s (13.9485GiB/s) | 20260508 11:04:46.765 | INFO | Analyzing chunk frames = [7606784,7607296) | 20260508 11:04:46.765 | INFO | MemInfo: { | 20260508 11:04:46.765 | INFO | "gpu" : " 17 GB", | 20260508 11:04:46.765 | INFO | "huge" : "", | 20260508 11:04:46.765 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:46.765 | INFO | "malloc" : " 14 GB", | 20260508 11:04:46.765 | INFO | "pinned" : " 29 GB" | 20260508 11:04:46.765 | INFO | } | 20260508 11:04:46.765 | INFO | Perf report for the last 5.13691 seconds: | 20260508 11:04:46.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:46.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.765 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:04:46.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.765 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:04:46.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:46.766 | INFO | Uploaded 36GiB to GPU in 2.45107s (14.6875GiB/s) | 20260508 11:04:46.766 | INFO | Downloaded 34GiB to GPU in 2.48721s (13.67GiB/s) | 20260508 11:04:51.905 | INFO | Analyzing chunk frames = [7607296,7607808) | 20260508 11:04:51.905 | INFO | MemInfo: { | 20260508 11:04:51.905 | INFO | "gpu" : " 17 GB", | 20260508 11:04:51.905 | INFO | "huge" : "", | 20260508 11:04:51.905 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:51.905 | INFO | "malloc" : " 14 GB", | 20260508 11:04:51.905 | INFO | "pinned" : " 29 GB" | 20260508 11:04:51.905 | INFO | } | 20260508 11:04:51.905 | INFO | Perf report for the last 5.14009 seconds: | 20260508 11:04:51.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:51.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.905 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:04:51.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.905 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:04:51.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:51.906 | INFO | Uploaded 36GiB to GPU in 2.46685s (14.5935GiB/s) | 20260508 11:04:51.906 | INFO | Downloaded 34GiB to GPU in 2.45981s (13.8222GiB/s) | 20260508 11:04:57.046 | INFO | Analyzing chunk frames = [7607808,7608320) | 20260508 11:04:57.046 | INFO | MemInfo: { | 20260508 11:04:57.046 | INFO | "gpu" : " 17 GB", | 20260508 11:04:57.046 | INFO | "huge" : "", | 20260508 11:04:57.046 | INFO | "huge_shared" : " 29 GB", | 20260508 11:04:57.046 | INFO | "malloc" : " 14 GB", | 20260508 11:04:57.046 | INFO | "pinned" : " 29 GB" | 20260508 11:04:57.046 | INFO | } | 20260508 11:04:57.047 | INFO | Perf report for the last 5.14128 seconds: | 20260508 11:04:57.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:04:57.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:04:57.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:04:57.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:04:57.047 | INFO | Uploaded 36GiB to GPU in 2.44447s (14.7271GiB/s) | 20260508 11:04:57.047 | INFO | Downloaded 34GiB to GPU in 2.46262s (13.8064GiB/s) | 20260508 11:05:02.179 | INFO | Analyzing chunk frames = [7608320,7608832) | 20260508 11:05:02.179 | INFO | MemInfo: { | 20260508 11:05:02.179 | INFO | "gpu" : " 17 GB", | 20260508 11:05:02.179 | INFO | "huge" : "", | 20260508 11:05:02.179 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:02.179 | INFO | "malloc" : " 14 GB", | 20260508 11:05:02.179 | INFO | "pinned" : " 29 GB" | 20260508 11:05:02.179 | INFO | } | 20260508 11:05:02.179 | INFO | Perf report for the last 5.13221 seconds: | 20260508 11:05:02.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:02.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:05:02.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 11:05:02.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:02.179 | INFO | Uploaded 36GiB to GPU in 2.28529s (15.7529GiB/s) | 20260508 11:05:02.179 | INFO | Downloaded 34GiB to GPU in 2.49921s (13.6043GiB/s) | 20260508 11:05:07.324 | INFO | Analyzing chunk frames = [7608832,7609344) | 20260508 11:05:07.324 | INFO | MemInfo: { | 20260508 11:05:07.324 | INFO | "gpu" : " 17 GB", | 20260508 11:05:07.324 | INFO | "huge" : "", | 20260508 11:05:07.324 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:07.325 | INFO | "malloc" : " 14 GB", | 20260508 11:05:07.325 | INFO | "pinned" : " 29 GB" | 20260508 11:05:07.325 | INFO | } | 20260508 11:05:07.325 | INFO | Perf report for the last 5.14593 seconds: | 20260508 11:05:07.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:07.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 11:05:07.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:05:07.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:07.325 | INFO | Uploaded 36GiB to GPU in 2.34077s (15.3796GiB/s) | 20260508 11:05:07.325 | INFO | Downloaded 34GiB to GPU in 2.45297s (13.8607GiB/s) | 20260508 11:05:12.463 | INFO | Analyzing chunk frames = [7609344,7609856) | 20260508 11:05:12.463 | INFO | MemInfo: { | 20260508 11:05:12.463 | INFO | "gpu" : " 17 GB", | 20260508 11:05:12.463 | INFO | "huge" : "", | 20260508 11:05:12.463 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:12.464 | INFO | "malloc" : " 14 GB", | 20260508 11:05:12.464 | INFO | "pinned" : " 29 GB" | 20260508 11:05:12.464 | INFO | } | 20260508 11:05:12.464 | INFO | Perf report for the last 5.13887 seconds: | 20260508 11:05:12.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:12.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:05:12.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 11:05:12.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:12.464 | INFO | Uploaded 36GiB to GPU in 2.40189s (14.9882GiB/s) | 20260508 11:05:12.464 | INFO | Downloaded 34GiB to GPU in 2.4551s (13.8487GiB/s) | 20260508 11:05:17.603 | INFO | Analyzing chunk frames = [7609856,7610368) | 20260508 11:05:17.603 | INFO | MemInfo: { | 20260508 11:05:17.603 | INFO | "gpu" : " 17 GB", | 20260508 11:05:17.603 | INFO | "huge" : "", | 20260508 11:05:17.603 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:17.603 | INFO | "malloc" : " 14 GB", | 20260508 11:05:17.603 | INFO | "pinned" : " 29 GB" | 20260508 11:05:17.603 | INFO | } | 20260508 11:05:17.603 | INFO | Perf report for the last 5.13975 seconds: | 20260508 11:05:17.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:17.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.603 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 11:05:17.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.603 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:05:17.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:17.604 | INFO | Uploaded 36GiB to GPU in 2.39519s (15.0301GiB/s) | 20260508 11:05:17.604 | INFO | Downloaded 34GiB to GPU in 2.45193s (13.8666GiB/s) | 20260508 11:05:17.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:05:17.604 | INFO | Total mem usage: 16.7 GiB | 20260508 11:05:17.604 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:05:17.604 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 11:05:17.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:05:17.604 | INFO | High Water Breakdown: | 20260508 11:05:17.604 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:05:17.604 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:05:17.604 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:05:17.604 | INFO | ------------------------------------------ | 20260508 11:05:17.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:05:17.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:05:17.604 | INFO | Total mem usage: 26.5 GiB | 20260508 11:05:17.604 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:05:17.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:05:17.604 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:05:17.604 | INFO | High Water Breakdown: | 20260508 11:05:17.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:05:17.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:05:17.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:05:17.604 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:05:17.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:05:17.604 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:05:17.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:05:17.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:05:17.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:05:17.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:05:17.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:05:17.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:05:17.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:05:17.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:05:17.604 | INFO | ------------------------------------------ | 20260508 11:05:17.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:05:17.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:05:17.604 | INFO | Total mem usage: 27.8 GiB | 20260508 11:05:17.604 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:05:17.604 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:05:17.604 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:05:17.604 | INFO | High Water Breakdown: | 20260508 11:05:17.604 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:05:17.604 | INFO | ------------------------------------------ | 20260508 11:05:17.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:05:17.604 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:05:17.604 | INFO | Total mem usage: 15.7 GiB | 20260508 11:05:17.604 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:05:17.604 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:05:17.604 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:05:17.604 | INFO | High Water Breakdown: | 20260508 11:05:17.604 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:05:17.604 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:05:17.604 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:05:17.604 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:05:17.604 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:05:17.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:05:17.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:05:17.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:05:17.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:05:17.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:05:17.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:05:17.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:05:17.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:05:17.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:05:17.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:05:17.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:05:17.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:05:17.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:05:17.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:05:17.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:05:17.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:05:17.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:05:17.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:05:17.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:05:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:05:17.605 | INFO | ------------------------------------------ | 20260508 11:05:17.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:05:17.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:05:17.605 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:05:17.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:05:17.605 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 6.04e-01 | 20260508 11:05:17.605 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.10e-02 / 7.55e-03 | 20260508 11:05:17.605 | INFO | PerfReport: 12.22%, Baseline -- 4.00e+00 / 4.49e+00, 6.01e-01 / 2.16e-01 | 20260508 11:05:17.605 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.23e-01 | 20260508 11:05:17.605 | INFO | PerfReport: 16.03%, DME -- 4.20e+01 / 4.53e+01, 3.61e+00 / 9.85e-01 | 20260508 11:05:17.605 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.12e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.54e+00, 4.06e-01 / 2.42e-01 | 20260508 11:05:17.605 | INFO | PerfReport: 3.00%, Metrics -- 9.82e-01 / 1.10e+00, 2.46e-01 / 7.81e-02 | 20260508 11:05:17.605 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.99e-01 / 3.32e-01, 4.90e-02 / 2.74e-02 | 20260508 11:05:17.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:05:17.605 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:05:17.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:05:17.605 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 24.22%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 24.66%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 2.11%, LockUpload -- 2.22e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 15.28%, LockCompute -- 1.61e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:05:17.605 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 11:05:17.700 | INFO | Starting a round of Baz IO | 20260508 11:05:21.104 | INFO | Populating RTMetrics report | 20260508 11:05:21.105 | INFO | RTMetrics report populated | 20260508 11:05:21.105 | INFO | Opening tmp RTMetrics json file | 20260508 11:05:21.116 | INFO | Serializing RTMetrics to json | 20260508 11:05:21.117 | INFO | Serialization done, swapping tmp file now... | 20260508 11:05:21.117 | INFO | RTMetrics done | 20260508 11:05:21.175 | INFO | Finished a round of Baz IO: Wrote 1.444 GiB in 3.474 seconds | 20260508 11:05:21.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:05:21.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202529% / 3.97364e-06% / 7.54992e-05% | 20260508 11:05:21.175 | INFO | Baz IO overall: | 20260508 11:05:21.175 | INFO | BytesWritten : 3290199720403 | 20260508 11:05:21.175 | INFO | NumEvents : 1580690965908 | 20260508 11:05:22.742 | INFO | Analyzing chunk frames = [7610368,7610880) | 20260508 11:05:22.742 | INFO | MemInfo: { | 20260508 11:05:22.742 | INFO | "gpu" : " 17 GB", | 20260508 11:05:22.742 | INFO | "huge" : "", | 20260508 11:05:22.742 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:22.742 | INFO | "malloc" : " 14 GB", | 20260508 11:05:22.742 | INFO | "pinned" : " 29 GB" | 20260508 11:05:22.742 | INFO | } | 20260508 11:05:22.742 | INFO | Perf report for the last 5.13882 seconds: | 20260508 11:05:22.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:22.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:22.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:22.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:22.742 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:05:22.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:22.742 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:05:22.742 | INFO | PRE_HQ : 65.4%, 1.6, 54.0ms, 0 | 20260508 11:05:22.742 | INFO | BAZWRITER : 66.9%, 2.5, 85.7ms, 0 | 20260508 11:05:22.742 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:05:22.742 | INFO | Uploaded 36GiB to GPU in 2.59149s (13.8916GiB/s) | 20260508 11:05:22.743 | INFO | Downloaded 38GiB to GPU in 2.71557s (13.9934GiB/s) | 20260508 11:05:27.884 | INFO | Analyzing chunk frames = [7610880,7611392) | 20260508 11:05:27.884 | INFO | MemInfo: { | 20260508 11:05:27.884 | INFO | "gpu" : " 17 GB", | 20260508 11:05:27.884 | INFO | "huge" : "", | 20260508 11:05:27.884 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:27.884 | INFO | "malloc" : " 14 GB", | 20260508 11:05:27.884 | INFO | "pinned" : " 29 GB" | 20260508 11:05:27.884 | INFO | } | 20260508 11:05:27.884 | INFO | Perf report for the last 5.14152 seconds: | 20260508 11:05:27.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:27.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:05:27.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:05:27.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:27.884 | INFO | Uploaded 36GiB to GPU in 2.41354s (14.9159GiB/s) | 20260508 11:05:27.884 | INFO | Downloaded 34GiB to GPU in 2.45786s (13.8332GiB/s) | 20260508 11:05:33.021 | INFO | Analyzing chunk frames = [7611392,7611904) | 20260508 11:05:33.021 | INFO | MemInfo: { | 20260508 11:05:33.021 | INFO | "gpu" : " 17 GB", | 20260508 11:05:33.021 | INFO | "huge" : "", | 20260508 11:05:33.021 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:33.021 | INFO | "malloc" : " 14 GB", | 20260508 11:05:33.021 | INFO | "pinned" : " 29 GB" | 20260508 11:05:33.021 | INFO | } | 20260508 11:05:33.021 | INFO | Perf report for the last 5.13742 seconds: | 20260508 11:05:33.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:33.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:05:33.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:05:33.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:33.021 | INFO | Uploaded 36GiB to GPU in 2.515s (14.3141GiB/s) | 20260508 11:05:33.022 | INFO | Downloaded 34GiB to GPU in 2.45086s (13.8727GiB/s) | 20260508 11:05:38.157 | INFO | Analyzing chunk frames = [7611904,7612416) | 20260508 11:05:38.157 | INFO | MemInfo: { | 20260508 11:05:38.157 | INFO | "gpu" : " 17 GB", | 20260508 11:05:38.157 | INFO | "huge" : "", | 20260508 11:05:38.157 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:38.157 | INFO | "malloc" : " 14 GB", | 20260508 11:05:38.157 | INFO | "pinned" : " 29 GB" | 20260508 11:05:38.157 | INFO | } | 20260508 11:05:38.157 | INFO | Perf report for the last 5.13615 seconds: | 20260508 11:05:38.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:38.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.157 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:05:38.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.157 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:05:38.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:38.158 | INFO | Uploaded 36GiB to GPU in 2.4675s (14.5897GiB/s) | 20260508 11:05:38.158 | INFO | Downloaded 34GiB to GPU in 2.44562s (13.9024GiB/s) | 20260508 11:05:43.297 | INFO | Analyzing chunk frames = [7612416,7612928) | 20260508 11:05:43.297 | INFO | MemInfo: { | 20260508 11:05:43.297 | INFO | "gpu" : " 17 GB", | 20260508 11:05:43.297 | INFO | "huge" : "", | 20260508 11:05:43.297 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:43.297 | INFO | "malloc" : " 14 GB", | 20260508 11:05:43.297 | INFO | "pinned" : " 29 GB" | 20260508 11:05:43.297 | INFO | } | 20260508 11:05:43.297 | INFO | Perf report for the last 5.1399 seconds: | 20260508 11:05:43.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:43.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 11:05:43.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:05:43.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:43.297 | INFO | Uploaded 36GiB to GPU in 2.43826s (14.7646GiB/s) | 20260508 11:05:43.298 | INFO | Downloaded 34GiB to GPU in 2.52688s (13.4553GiB/s) | 20260508 11:05:48.431 | INFO | Analyzing chunk frames = [7612928,7613440) | 20260508 11:05:48.431 | INFO | MemInfo: { | 20260508 11:05:48.431 | INFO | "gpu" : " 17 GB", | 20260508 11:05:48.431 | INFO | "huge" : "", | 20260508 11:05:48.431 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:48.431 | INFO | "malloc" : " 14 GB", | 20260508 11:05:48.431 | INFO | "pinned" : " 29 GB" | 20260508 11:05:48.431 | INFO | } | 20260508 11:05:48.431 | INFO | Perf report for the last 5.13386 seconds: | 20260508 11:05:48.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:48.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260508 11:05:48.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260508 11:05:48.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:48.431 | INFO | Uploaded 36GiB to GPU in 2.42054s (14.8727GiB/s) | 20260508 11:05:48.431 | INFO | Downloaded 34GiB to GPU in 2.42088s (14.0445GiB/s) | 20260508 11:05:53.581 | INFO | Analyzing chunk frames = [7613440,7613952) | 20260508 11:05:53.581 | INFO | MemInfo: { | 20260508 11:05:53.581 | INFO | "gpu" : " 17 GB", | 20260508 11:05:53.581 | INFO | "huge" : "", | 20260508 11:05:53.581 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:53.581 | INFO | "malloc" : " 14 GB", | 20260508 11:05:53.581 | INFO | "pinned" : " 29 GB" | 20260508 11:05:53.581 | INFO | } | 20260508 11:05:53.581 | INFO | Perf report for the last 5.14966 seconds: | 20260508 11:05:53.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:53.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | ANALYSIS : 65.3%, 3.0, 102.2ms, 0 | 20260508 11:05:53.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:05:53.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:53.581 | INFO | Uploaded 36GiB to GPU in 2.44158s (14.7445GiB/s) | 20260508 11:05:53.581 | INFO | Downloaded 34GiB to GPU in 2.42805s (14.003GiB/s) | 20260508 11:05:58.717 | INFO | Analyzing chunk frames = [7613952,7614464) | 20260508 11:05:58.717 | INFO | MemInfo: { | 20260508 11:05:58.717 | INFO | "gpu" : " 17 GB", | 20260508 11:05:58.717 | INFO | "huge" : "", | 20260508 11:05:58.717 | INFO | "huge_shared" : " 29 GB", | 20260508 11:05:58.717 | INFO | "malloc" : " 14 GB", | 20260508 11:05:58.717 | INFO | "pinned" : " 29 GB" | 20260508 11:05:58.717 | INFO | } | 20260508 11:05:58.717 | INFO | Perf report for the last 5.13595 seconds: | 20260508 11:05:58.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:05:58.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 11:05:58.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:05:58.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:05:58.717 | INFO | Uploaded 36GiB to GPU in 2.49651s (14.4201GiB/s) | 20260508 11:05:58.717 | INFO | Downloaded 34GiB to GPU in 2.45006s (13.8772GiB/s) | 20260508 11:05:58.814 | INFO | Starting a round of Baz IO | 20260508 11:06:02.234 | INFO | Populating RTMetrics report | 20260508 11:06:02.235 | INFO | RTMetrics report populated | 20260508 11:06:02.235 | INFO | Opening tmp RTMetrics json file | 20260508 11:06:02.244 | INFO | Serializing RTMetrics to json | 20260508 11:06:02.245 | INFO | Serialization done, swapping tmp file now... | 20260508 11:06:02.246 | INFO | RTMetrics done | 20260508 11:06:02.303 | INFO | Finished a round of Baz IO: Wrote 1.445 GiB in 3.489 seconds | 20260508 11:06:02.303 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:06:02.303 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202819% / 0% / 4.76837e-05% | 20260508 11:06:02.304 | INFO | Baz IO overall: | 20260508 11:06:02.304 | INFO | BytesWritten : 3291750764664 | 20260508 11:06:02.304 | INFO | NumEvents : 1581421347226 | 20260508 11:06:03.857 | INFO | Analyzing chunk frames = [7614464,7614976) | 20260508 11:06:03.857 | INFO | MemInfo: { | 20260508 11:06:03.857 | INFO | "gpu" : " 17 GB", | 20260508 11:06:03.857 | INFO | "huge" : "", | 20260508 11:06:03.857 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:03.857 | INFO | "malloc" : " 14 GB", | 20260508 11:06:03.857 | INFO | "pinned" : " 29 GB" | 20260508 11:06:03.857 | INFO | } | 20260508 11:06:03.857 | INFO | Perf report for the last 5.14024 seconds: | 20260508 11:06:03.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:03.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:03.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:03.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:03.857 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 11:06:03.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:03.857 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:06:03.857 | INFO | PRE_HQ : 65.4%, 1.6, 54.4ms, 0 | 20260508 11:06:03.857 | INFO | BAZWRITER : 67.2%, 2.5, 86.5ms, 0 | 20260508 11:06:03.857 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 11:06:03.857 | INFO | Uploaded 36GiB to GPU in 2.59295s (13.8838GiB/s) | 20260508 11:06:03.857 | INFO | Downloaded 38GiB to GPU in 2.70229s (14.0621GiB/s) | 20260508 11:06:08.987 | INFO | Analyzing chunk frames = [7614976,7615488) | 20260508 11:06:08.987 | INFO | MemInfo: { | 20260508 11:06:08.987 | INFO | "gpu" : " 17 GB", | 20260508 11:06:08.987 | INFO | "huge" : "", | 20260508 11:06:08.988 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:08.988 | INFO | "malloc" : " 14 GB", | 20260508 11:06:08.988 | INFO | "pinned" : " 29 GB" | 20260508 11:06:08.988 | INFO | } | 20260508 11:06:08.988 | INFO | Perf report for the last 5.13076 seconds: | 20260508 11:06:08.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:08.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:06:08.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:06:08.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:08.988 | INFO | Uploaded 36GiB to GPU in 2.47866s (14.524GiB/s) | 20260508 11:06:08.988 | INFO | Downloaded 34GiB to GPU in 2.46246s (13.8074GiB/s) | 20260508 11:06:14.135 | INFO | Analyzing chunk frames = [7615488,7616000) | 20260508 11:06:14.135 | INFO | MemInfo: { | 20260508 11:06:14.135 | INFO | "gpu" : " 17 GB", | 20260508 11:06:14.135 | INFO | "huge" : "", | 20260508 11:06:14.135 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:14.135 | INFO | "malloc" : " 14 GB", | 20260508 11:06:14.135 | INFO | "pinned" : " 29 GB" | 20260508 11:06:14.135 | INFO | } | 20260508 11:06:14.135 | INFO | Perf report for the last 5.14751 seconds: | 20260508 11:06:14.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:14.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.135 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:06:14.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.135 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:06:14.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:14.136 | INFO | Uploaded 36GiB to GPU in 2.49599s (14.4231GiB/s) | 20260508 11:06:14.136 | INFO | Downloaded 34GiB to GPU in 2.42145s (14.0412GiB/s) | 20260508 11:06:19.271 | INFO | Analyzing chunk frames = [7616000,7616512) | 20260508 11:06:19.271 | INFO | MemInfo: { | 20260508 11:06:19.271 | INFO | "gpu" : " 17 GB", | 20260508 11:06:19.272 | INFO | "huge" : "", | 20260508 11:06:19.272 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:19.272 | INFO | "malloc" : " 14 GB", | 20260508 11:06:19.272 | INFO | "pinned" : " 29 GB" | 20260508 11:06:19.272 | INFO | } | 20260508 11:06:19.272 | INFO | Perf report for the last 5.13643 seconds: | 20260508 11:06:19.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:19.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.272 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:06:19.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.272 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:06:19.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:19.273 | INFO | Uploaded 36GiB to GPU in 2.49861s (14.408GiB/s) | 20260508 11:06:19.273 | INFO | Downloaded 34GiB to GPU in 2.4402s (13.9333GiB/s) | 20260508 11:06:24.416 | INFO | Analyzing chunk frames = [7616512,7617024) | 20260508 11:06:24.416 | INFO | MemInfo: { | 20260508 11:06:24.416 | INFO | "gpu" : " 17 GB", | 20260508 11:06:24.416 | INFO | "huge" : "", | 20260508 11:06:24.416 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:24.416 | INFO | "malloc" : " 14 GB", | 20260508 11:06:24.416 | INFO | "pinned" : " 29 GB" | 20260508 11:06:24.416 | INFO | } | 20260508 11:06:24.416 | INFO | Perf report for the last 5.14446 seconds: | 20260508 11:06:24.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:24.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.416 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:06:24.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.416 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260508 11:06:24.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:24.417 | INFO | Uploaded 36GiB to GPU in 2.32381s (15.4918GiB/s) | 20260508 11:06:24.417 | INFO | Downloaded 34GiB to GPU in 2.50122s (13.5934GiB/s) | 20260508 11:06:29.552 | INFO | Analyzing chunk frames = [7617024,7617536) | 20260508 11:06:29.552 | INFO | MemInfo: { | 20260508 11:06:29.552 | INFO | "gpu" : " 17 GB", | 20260508 11:06:29.552 | INFO | "huge" : "", | 20260508 11:06:29.552 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:29.552 | INFO | "malloc" : " 14 GB", | 20260508 11:06:29.552 | INFO | "pinned" : " 29 GB" | 20260508 11:06:29.552 | INFO | } | 20260508 11:06:29.552 | INFO | Perf report for the last 5.13614 seconds: | 20260508 11:06:29.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:29.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.552 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:06:29.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.553 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:06:29.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:29.553 | INFO | Uploaded 36GiB to GPU in 2.38827s (15.0737GiB/s) | 20260508 11:06:29.553 | INFO | Downloaded 34GiB to GPU in 2.43975s (13.9359GiB/s) | 20260508 11:06:34.689 | INFO | Analyzing chunk frames = [7617536,7618048) | 20260508 11:06:34.689 | INFO | MemInfo: { | 20260508 11:06:34.689 | INFO | "gpu" : " 17 GB", | 20260508 11:06:34.689 | INFO | "huge" : "", | 20260508 11:06:34.689 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:34.689 | INFO | "malloc" : " 14 GB", | 20260508 11:06:34.690 | INFO | "pinned" : " 29 GB" | 20260508 11:06:34.690 | INFO | } | 20260508 11:06:34.690 | INFO | Perf report for the last 5.13718 seconds: | 20260508 11:06:34.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:34.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:06:34.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:06:34.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:34.690 | INFO | Uploaded 36GiB to GPU in 2.44298s (14.7361GiB/s) | 20260508 11:06:34.690 | INFO | Downloaded 34GiB to GPU in 2.43872s (13.9417GiB/s) | 20260508 11:06:39.834 | INFO | Analyzing chunk frames = [7618048,7618560) | 20260508 11:06:39.834 | INFO | MemInfo: { | 20260508 11:06:39.834 | INFO | "gpu" : " 17 GB", | 20260508 11:06:39.834 | INFO | "huge" : "", | 20260508 11:06:39.834 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:39.834 | INFO | "malloc" : " 14 GB", | 20260508 11:06:39.834 | INFO | "pinned" : " 29 GB" | 20260508 11:06:39.834 | INFO | } | 20260508 11:06:39.834 | INFO | Perf report for the last 5.14434 seconds: | 20260508 11:06:39.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:39.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | ANALYSIS : 65.7%, 3.0, 102.9ms, 0 | 20260508 11:06:39.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:06:39.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:39.834 | INFO | Uploaded 36GiB to GPU in 2.47909s (14.5214GiB/s) | 20260508 11:06:39.834 | INFO | Downloaded 34GiB to GPU in 2.45643s (13.8412GiB/s) | 20260508 11:06:39.834 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:06:39.834 | INFO | Total mem usage: 16.7 GiB | 20260508 11:06:39.834 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:06:39.834 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:06:39.834 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:06:39.834 | INFO | High Water Breakdown: | 20260508 11:06:39.834 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:06:39.834 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:06:39.834 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:06:39.834 | INFO | ------------------------------------------ | 20260508 11:06:39.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:06:39.834 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:06:39.834 | INFO | Total mem usage: 26.5 GiB | 20260508 11:06:39.834 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:06:39.835 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:06:39.835 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:06:39.835 | INFO | High Water Breakdown: | 20260508 11:06:39.835 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:06:39.835 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:06:39.835 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:06:39.835 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:06:39.835 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:06:39.835 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:06:39.835 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:06:39.835 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:06:39.835 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:06:39.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:06:39.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:06:39.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:06:39.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:06:39.835 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:06:39.835 | INFO | ------------------------------------------ | 20260508 11:06:39.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:06:39.835 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:06:39.835 | INFO | Total mem usage: 27.8 GiB | 20260508 11:06:39.835 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:06:39.835 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:06:39.835 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:06:39.835 | INFO | High Water Breakdown: | 20260508 11:06:39.835 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:06:39.835 | INFO | ------------------------------------------ | 20260508 11:06:39.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:06:39.835 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:06:39.835 | INFO | Total mem usage: 15.7 GiB | 20260508 11:06:39.835 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:06:39.835 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:06:39.835 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:06:39.835 | INFO | High Water Breakdown: | 20260508 11:06:39.835 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:06:39.835 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:06:39.835 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:06:39.835 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:06:39.835 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:06:39.835 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:06:39.835 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:06:39.835 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:06:39.835 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:06:39.835 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:06:39.835 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:06:39.835 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:06:39.835 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:06:39.835 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:06:39.835 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:06:39.835 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:06:39.835 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:06:39.835 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:06:39.835 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:06:39.835 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:06:39.835 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:06:39.835 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:06:39.835 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:06:39.835 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:06:39.835 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:06:39.835 | INFO | ------------------------------------------ | 20260508 11:06:39.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:06:39.835 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:06:39.835 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:06:39.835 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:06:39.835 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.42e+01 / 6.91e-01 | 20260508 11:06:39.835 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.06e-01, 1.15e-02 / 1.31e-02 | 20260508 11:06:39.835 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.45e+00, 5.93e-01 / 2.07e-01 | 20260508 11:06:39.835 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.11e-01 | 20260508 11:06:39.835 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.53e+01, 3.72e+00 / 1.12e+00 | 20260508 11:06:39.835 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.76e+00 / 1.28e+00 | 20260508 11:06:39.835 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.35e+00 / 2.54e+00, 4.09e-01 / 2.23e-01 | 20260508 11:06:39.835 | INFO | PerfReport: 3.01%, Metrics -- 9.82e-01 / 1.13e+00, 2.44e-01 / 1.23e-01 | 20260508 11:06:39.835 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.27e-01, 4.81e-02 / 2.39e-02 | 20260508 11:06:39.836 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:06:39.836 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:06:39.836 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:06:39.836 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 24.42%, Upload -- 2.57e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 30.99%, Compute -- 3.26e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 2.53%, LockDownload -- 2.66e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 15.21%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:06:39.836 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260508 11:06:39.931 | INFO | Starting a round of Baz IO | 20260508 11:06:43.337 | INFO | Populating RTMetrics report | 20260508 11:06:43.337 | INFO | RTMetrics report populated | 20260508 11:06:43.337 | INFO | Opening tmp RTMetrics json file | 20260508 11:06:43.345 | INFO | Serializing RTMetrics to json | 20260508 11:06:43.346 | INFO | Serialization done, swapping tmp file now... | 20260508 11:06:43.347 | INFO | RTMetrics done | 20260508 11:06:43.408 | INFO | Finished a round of Baz IO: Wrote 1.445 GiB in 3.477 seconds | 20260508 11:06:43.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.00% / 59.86% | 20260508 11:06:43.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202711% / 0% / 7.54992e-05% | 20260508 11:06:43.408 | INFO | Baz IO overall: | 20260508 11:06:43.408 | INFO | BytesWritten : 3293301897582 | 20260508 11:06:43.408 | INFO | NumEvents : 1582152157793 | 20260508 11:06:44.968 | INFO | Analyzing chunk frames = [7618560,7619072) | 20260508 11:06:44.968 | INFO | MemInfo: { | 20260508 11:06:44.968 | INFO | "gpu" : " 17 GB", | 20260508 11:06:44.968 | INFO | "huge" : "", | 20260508 11:06:44.968 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:44.968 | INFO | "malloc" : " 14 GB", | 20260508 11:06:44.968 | INFO | "pinned" : " 29 GB" | 20260508 11:06:44.968 | INFO | } | 20260508 11:06:44.968 | INFO | Perf report for the last 5.13421 seconds: | 20260508 11:06:44.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:44.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:44.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:44.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:44.968 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 11:06:44.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:44.968 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:06:44.968 | INFO | PRE_HQ : 65.5%, 1.6, 54.2ms, 0 | 20260508 11:06:44.968 | INFO | BAZWRITER : 67.0%, 2.5, 86.0ms, 0 | 20260508 11:06:44.968 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:06:44.969 | INFO | Uploaded 36GiB to GPU in 2.59153s (13.8914GiB/s) | 20260508 11:06:44.969 | INFO | Downloaded 38GiB to GPU in 2.71291s (14.0071GiB/s) | 20260508 11:06:50.110 | INFO | Analyzing chunk frames = [7619072,7619584) | 20260508 11:06:50.110 | INFO | MemInfo: { | 20260508 11:06:50.110 | INFO | "gpu" : " 17 GB", | 20260508 11:06:50.110 | INFO | "huge" : "", | 20260508 11:06:50.110 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:50.110 | INFO | "malloc" : " 14 GB", | 20260508 11:06:50.110 | INFO | "pinned" : " 29 GB" | 20260508 11:06:50.110 | INFO | } | 20260508 11:06:50.111 | INFO | Perf report for the last 5.14234 seconds: | 20260508 11:06:50.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:50.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 11:06:50.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:06:50.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:50.111 | INFO | Uploaded 36GiB to GPU in 2.49724s (14.4159GiB/s) | 20260508 11:06:50.111 | INFO | Downloaded 34GiB to GPU in 2.47456s (13.7398GiB/s) | 20260508 11:06:55.249 | INFO | Analyzing chunk frames = [7619584,7620096) | 20260508 11:06:55.249 | INFO | MemInfo: { | 20260508 11:06:55.249 | INFO | "gpu" : " 17 GB", | 20260508 11:06:55.249 | INFO | "huge" : "", | 20260508 11:06:55.249 | INFO | "huge_shared" : " 29 GB", | 20260508 11:06:55.249 | INFO | "malloc" : " 14 GB", | 20260508 11:06:55.249 | INFO | "pinned" : " 29 GB" | 20260508 11:06:55.249 | INFO | } | 20260508 11:06:55.249 | INFO | Perf report for the last 5.13823 seconds: | 20260508 11:06:55.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:06:55.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:06:55.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:06:55.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:06:55.249 | INFO | Uploaded 36GiB to GPU in 2.50032s (14.3982GiB/s) | 20260508 11:06:55.249 | INFO | Downloaded 34GiB to GPU in 2.43962s (13.9366GiB/s) | 20260508 11:07:00.392 | INFO | Analyzing chunk frames = [7620096,7620608) | 20260508 11:07:00.392 | INFO | MemInfo: { | 20260508 11:07:00.393 | INFO | "gpu" : " 17 GB", | 20260508 11:07:00.393 | INFO | "huge" : "", | 20260508 11:07:00.393 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:00.393 | INFO | "malloc" : " 14 GB", | 20260508 11:07:00.393 | INFO | "pinned" : " 29 GB" | 20260508 11:07:00.393 | INFO | } | 20260508 11:07:00.393 | INFO | Perf report for the last 5.14423 seconds: | 20260508 11:07:00.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:00.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.393 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 11:07:00.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.393 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 11:07:00.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:00.394 | INFO | Uploaded 36GiB to GPU in 2.45099s (14.6879GiB/s) | 20260508 11:07:00.394 | INFO | Downloaded 34GiB to GPU in 2.4572s (13.8369GiB/s) | 20260508 11:07:05.531 | INFO | Analyzing chunk frames = [7620608,7621120) | 20260508 11:07:05.531 | INFO | MemInfo: { | 20260508 11:07:05.531 | INFO | "gpu" : " 17 GB", | 20260508 11:07:05.531 | INFO | "huge" : "", | 20260508 11:07:05.531 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:05.531 | INFO | "malloc" : " 14 GB", | 20260508 11:07:05.531 | INFO | "pinned" : " 29 GB" | 20260508 11:07:05.531 | INFO | } | 20260508 11:07:05.531 | INFO | Perf report for the last 5.13786 seconds: | 20260508 11:07:05.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:05.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:07:05.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | PRE_HQ : 31.9%, 1.0, 16.7ms, 0 | 20260508 11:07:05.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:05.531 | INFO | Uploaded 36GiB to GPU in 2.40467s (14.9709GiB/s) | 20260508 11:07:05.531 | INFO | Downloaded 34GiB to GPU in 2.51304s (13.5294GiB/s) | 20260508 11:07:10.669 | INFO | Analyzing chunk frames = [7621120,7621632) | 20260508 11:07:10.669 | INFO | MemInfo: { | 20260508 11:07:10.669 | INFO | "gpu" : " 17 GB", | 20260508 11:07:10.669 | INFO | "huge" : "", | 20260508 11:07:10.669 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:10.669 | INFO | "malloc" : " 14 GB", | 20260508 11:07:10.669 | INFO | "pinned" : " 29 GB" | 20260508 11:07:10.669 | INFO | } | 20260508 11:07:10.669 | INFO | Perf report for the last 5.13817 seconds: | 20260508 11:07:10.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:10.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 11:07:10.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:07:10.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:10.669 | INFO | Uploaded 36GiB to GPU in 2.39293s (15.0443GiB/s) | 20260508 11:07:10.670 | INFO | Downloaded 34GiB to GPU in 2.41982s (14.0506GiB/s) | 20260508 11:07:15.804 | INFO | Analyzing chunk frames = [7621632,7622144) | 20260508 11:07:15.804 | INFO | MemInfo: { | 20260508 11:07:15.804 | INFO | "gpu" : " 17 GB", | 20260508 11:07:15.804 | INFO | "huge" : "", | 20260508 11:07:15.804 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:15.804 | INFO | "malloc" : " 14 GB", | 20260508 11:07:15.804 | INFO | "pinned" : " 29 GB" | 20260508 11:07:15.804 | INFO | } | 20260508 11:07:15.804 | INFO | Perf report for the last 5.13476 seconds: | 20260508 11:07:15.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:15.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:07:15.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:07:15.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:15.804 | INFO | Uploaded 36GiB to GPU in 2.45772s (14.6477GiB/s) | 20260508 11:07:15.804 | INFO | Downloaded 34GiB to GPU in 2.44052s (13.9314GiB/s) | 20260508 11:07:20.946 | INFO | Analyzing chunk frames = [7622144,7622656) | 20260508 11:07:20.946 | INFO | MemInfo: { | 20260508 11:07:20.946 | INFO | "gpu" : " 17 GB", | 20260508 11:07:20.946 | INFO | "huge" : "", | 20260508 11:07:20.946 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:20.946 | INFO | "malloc" : " 14 GB", | 20260508 11:07:20.946 | INFO | "pinned" : " 29 GB" | 20260508 11:07:20.946 | INFO | } | 20260508 11:07:20.946 | INFO | Perf report for the last 5.14245 seconds: | 20260508 11:07:20.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:20.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.946 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 11:07:20.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.946 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:07:20.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:20.947 | INFO | Uploaded 36GiB to GPU in 2.45399s (14.67GiB/s) | 20260508 11:07:20.947 | INFO | Downloaded 34GiB to GPU in 2.41044s (14.1053GiB/s) | 20260508 11:07:21.044 | INFO | Starting a round of Baz IO | 20260508 11:07:24.452 | INFO | Populating RTMetrics report | 20260508 11:07:24.453 | INFO | RTMetrics report populated | 20260508 11:07:24.453 | INFO | Opening tmp RTMetrics json file | 20260508 11:07:24.463 | INFO | Serializing RTMetrics to json | 20260508 11:07:24.464 | INFO | Serialization done, swapping tmp file now... | 20260508 11:07:24.464 | INFO | RTMetrics done | 20260508 11:07:24.524 | INFO | Finished a round of Baz IO: Wrote 1.442 GiB in 3.479 seconds | 20260508 11:07:24.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.86% | 20260508 11:07:24.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201241% / 0% / 5.16574e-05% | 20260508 11:07:24.524 | INFO | Baz IO overall: | 20260508 11:07:24.524 | INFO | BytesWritten : 3294850432617 | 20260508 11:07:24.524 | INFO | NumEvents : 1582880900356 | 20260508 11:07:26.088 | INFO | Analyzing chunk frames = [7622656,7623168) | 20260508 11:07:26.088 | INFO | MemInfo: { | 20260508 11:07:26.088 | INFO | "gpu" : " 17 GB", | 20260508 11:07:26.088 | INFO | "huge" : "", | 20260508 11:07:26.088 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:26.088 | INFO | "malloc" : " 14 GB", | 20260508 11:07:26.088 | INFO | "pinned" : " 29 GB" | 20260508 11:07:26.088 | INFO | } | 20260508 11:07:26.088 | INFO | Perf report for the last 5.14146 seconds: | 20260508 11:07:26.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:26.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:26.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:26.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:26.088 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 11:07:26.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:26.088 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 11:07:26.088 | INFO | PRE_HQ : 65.4%, 1.6, 54.0ms, 0 | 20260508 11:07:26.088 | INFO | BAZWRITER : 67.0%, 2.5, 85.7ms, 0 | 20260508 11:07:26.088 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:07:26.088 | INFO | Uploaded 36GiB to GPU in 2.61836s (13.749GiB/s) | 20260508 11:07:26.088 | INFO | Downloaded 38GiB to GPU in 2.70803s (14.0323GiB/s) | 20260508 11:07:31.226 | INFO | Analyzing chunk frames = [7623168,7623680) | 20260508 11:07:31.226 | INFO | MemInfo: { | 20260508 11:07:31.226 | INFO | "gpu" : " 17 GB", | 20260508 11:07:31.226 | INFO | "huge" : "", | 20260508 11:07:31.226 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:31.226 | INFO | "malloc" : " 14 GB", | 20260508 11:07:31.226 | INFO | "pinned" : " 29 GB" | 20260508 11:07:31.226 | INFO | } | 20260508 11:07:31.226 | INFO | Perf report for the last 5.13799 seconds: | 20260508 11:07:31.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:31.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 11:07:31.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:07:31.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:31.226 | INFO | Uploaded 36GiB to GPU in 2.50606s (14.3652GiB/s) | 20260508 11:07:31.226 | INFO | Downloaded 34GiB to GPU in 2.4629s (13.8049GiB/s) | 20260508 11:07:36.363 | INFO | Analyzing chunk frames = [7623680,7624192) | 20260508 11:07:36.363 | INFO | MemInfo: { | 20260508 11:07:36.363 | INFO | "gpu" : " 17 GB", | 20260508 11:07:36.363 | INFO | "huge" : "", | 20260508 11:07:36.363 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:36.363 | INFO | "malloc" : " 14 GB", | 20260508 11:07:36.363 | INFO | "pinned" : " 29 GB" | 20260508 11:07:36.363 | INFO | } | 20260508 11:07:36.363 | INFO | Perf report for the last 5.13737 seconds: | 20260508 11:07:36.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:36.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 11:07:36.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:07:36.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:36.363 | INFO | Uploaded 36GiB to GPU in 2.49766s (14.4135GiB/s) | 20260508 11:07:36.364 | INFO | Downloaded 34GiB to GPU in 2.42307s (14.0318GiB/s) | 20260508 11:07:41.499 | INFO | Analyzing chunk frames = [7624192,7624704) | 20260508 11:07:41.499 | INFO | MemInfo: { | 20260508 11:07:41.499 | INFO | "gpu" : " 17 GB", | 20260508 11:07:41.499 | INFO | "huge" : "", | 20260508 11:07:41.499 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:41.499 | INFO | "malloc" : " 14 GB", | 20260508 11:07:41.499 | INFO | "pinned" : " 29 GB" | 20260508 11:07:41.499 | INFO | } | 20260508 11:07:41.499 | INFO | Perf report for the last 5.13597 seconds: | 20260508 11:07:41.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:41.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.499 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:07:41.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.499 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:07:41.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:41.500 | INFO | Uploaded 36GiB to GPU in 2.49811s (14.4109GiB/s) | 20260508 11:07:41.500 | INFO | Downloaded 34GiB to GPU in 2.44942s (13.8808GiB/s) | 20260508 11:07:46.643 | INFO | Analyzing chunk frames = [7624704,7625216) | 20260508 11:07:46.643 | INFO | MemInfo: { | 20260508 11:07:46.643 | INFO | "gpu" : " 17 GB", | 20260508 11:07:46.643 | INFO | "huge" : "", | 20260508 11:07:46.643 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:46.643 | INFO | "malloc" : " 14 GB", | 20260508 11:07:46.643 | INFO | "pinned" : " 29 GB" | 20260508 11:07:46.644 | INFO | } | 20260508 11:07:46.644 | INFO | Perf report for the last 5.14456 seconds: | 20260508 11:07:46.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:46.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 11:07:46.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:07:46.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:46.644 | INFO | Uploaded 36GiB to GPU in 2.31596s (15.5443GiB/s) | 20260508 11:07:46.644 | INFO | Downloaded 34GiB to GPU in 2.50224s (13.5878GiB/s) | 20260508 11:07:51.781 | INFO | Analyzing chunk frames = [7625216,7625728) | 20260508 11:07:51.781 | INFO | MemInfo: { | 20260508 11:07:51.781 | INFO | "gpu" : " 17 GB", | 20260508 11:07:51.781 | INFO | "huge" : "", | 20260508 11:07:51.781 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:51.781 | INFO | "malloc" : " 14 GB", | 20260508 11:07:51.781 | INFO | "pinned" : " 29 GB" | 20260508 11:07:51.781 | INFO | } | 20260508 11:07:51.782 | INFO | Perf report for the last 5.13781 seconds: | 20260508 11:07:51.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:51.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:07:51.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:07:51.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:51.782 | INFO | Uploaded 36GiB to GPU in 2.38864s (15.0713GiB/s) | 20260508 11:07:51.782 | INFO | Downloaded 34GiB to GPU in 2.42067s (14.0457GiB/s) | 20260508 11:07:56.926 | INFO | Analyzing chunk frames = [7625728,7626240) | 20260508 11:07:56.926 | INFO | MemInfo: { | 20260508 11:07:56.926 | INFO | "gpu" : " 17 GB", | 20260508 11:07:56.926 | INFO | "huge" : "", | 20260508 11:07:56.926 | INFO | "huge_shared" : " 29 GB", | 20260508 11:07:56.926 | INFO | "malloc" : " 14 GB", | 20260508 11:07:56.926 | INFO | "pinned" : " 29 GB" | 20260508 11:07:56.926 | INFO | } | 20260508 11:07:56.926 | INFO | Perf report for the last 5.14481 seconds: | 20260508 11:07:56.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:07:56.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.926 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 11:07:56.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.927 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:07:56.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:07:56.927 | INFO | Uploaded 36GiB to GPU in 2.44863s (14.7021GiB/s) | 20260508 11:07:56.927 | INFO | Downloaded 34GiB to GPU in 2.43957s (13.9369GiB/s) | 20260508 11:08:02.064 | INFO | Analyzing chunk frames = [7626240,7626752) | 20260508 11:08:02.064 | INFO | MemInfo: { | 20260508 11:08:02.064 | INFO | "gpu" : " 17 GB", | 20260508 11:08:02.064 | INFO | "huge" : "", | 20260508 11:08:02.064 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:02.064 | INFO | "malloc" : " 14 GB", | 20260508 11:08:02.064 | INFO | "pinned" : " 29 GB" | 20260508 11:08:02.064 | INFO | } | 20260508 11:08:02.064 | INFO | Perf report for the last 5.13747 seconds: | 20260508 11:08:02.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:02.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 11:08:02.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 11:08:02.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:02.064 | INFO | Uploaded 36GiB to GPU in 2.47872s (14.5236GiB/s) | 20260508 11:08:02.064 | INFO | Downloaded 34GiB to GPU in 2.44695s (13.8948GiB/s) | 20260508 11:08:02.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:08:02.064 | INFO | Total mem usage: 16.7 GiB | 20260508 11:08:02.064 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:08:02.064 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:08:02.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:08:02.064 | INFO | High Water Breakdown: | 20260508 11:08:02.065 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:08:02.065 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:08:02.065 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:08:02.065 | INFO | ------------------------------------------ | 20260508 11:08:02.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:08:02.065 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:08:02.065 | INFO | Total mem usage: 26.5 GiB | 20260508 11:08:02.065 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:08:02.065 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:08:02.065 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:08:02.065 | INFO | High Water Breakdown: | 20260508 11:08:02.065 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:08:02.065 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:08:02.065 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:08:02.065 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:08:02.065 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:08:02.065 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:08:02.065 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:08:02.065 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:08:02.065 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:08:02.065 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:08:02.065 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:08:02.065 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:08:02.065 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:08:02.065 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:08:02.065 | INFO | ------------------------------------------ | 20260508 11:08:02.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:08:02.065 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:08:02.065 | INFO | Total mem usage: 27.8 GiB | 20260508 11:08:02.065 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:08:02.065 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:08:02.065 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:08:02.065 | INFO | High Water Breakdown: | 20260508 11:08:02.065 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:08:02.065 | INFO | ------------------------------------------ | 20260508 11:08:02.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:08:02.065 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:08:02.065 | INFO | Total mem usage: 15.7 GiB | 20260508 11:08:02.065 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:08:02.065 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:08:02.065 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:08:02.065 | INFO | High Water Breakdown: | 20260508 11:08:02.065 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:08:02.065 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:08:02.065 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:08:02.065 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:08:02.065 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:08:02.065 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:08:02.065 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:08:02.065 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:08:02.065 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:08:02.065 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:08:02.065 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:08:02.065 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:08:02.065 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:08:02.065 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:08:02.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:08:02.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:08:02.065 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:08:02.065 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:08:02.065 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:08:02.065 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:08:02.065 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:08:02.065 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:08:02.065 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:08:02.065 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:08:02.065 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:08:02.065 | INFO | ------------------------------------------ | 20260508 11:08:02.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:08:02.065 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:08:02.065 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:08:02.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:08:02.065 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 6.04e-01 | 20260508 11:08:02.065 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.06e-01, 1.22e-02 / 5.15e-03 | 20260508 11:08:02.065 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.51e+00, 6.00e-01 / 1.82e-01 | 20260508 11:08:02.065 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.49e+00, 2.10e-01 / 1.12e-01 | 20260508 11:08:02.065 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.53e+01, 3.66e+00 / 9.19e-01 | 20260508 11:08:02.065 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.80e+00 / 1.39e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.52e+00, 4.07e-01 / 2.73e-01 | 20260508 11:08:02.066 | INFO | PerfReport: 3.00%, Metrics -- 9.81e-01 / 1.10e+00, 2.40e-01 / 1.04e-01 | 20260508 11:08:02.066 | INFO | PerfReport: 0.92%, PulseSerializing -- 2.99e-01 / 3.24e-01, 4.84e-02 / 3.02e-02 | 20260508 11:08:02.066 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:08:02.066 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:08:02.066 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:08:02.066 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 24.44%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 31.05%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 2.27%, LockUpload -- 2.39e+00 / 0.00e+00, 5.80e+00 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 2.45%, LockDownload -- 2.58e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 15.19%, LockCompute -- 1.60e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 11:08:02.066 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260508 11:08:02.164 | INFO | Starting a round of Baz IO | 20260508 11:08:05.582 | INFO | Populating RTMetrics report | 20260508 11:08:05.583 | INFO | RTMetrics report populated | 20260508 11:08:05.583 | INFO | Opening tmp RTMetrics json file | 20260508 11:08:05.593 | INFO | Serializing RTMetrics to json | 20260508 11:08:05.595 | INFO | Serialization done, swapping tmp file now... | 20260508 11:08:05.595 | INFO | RTMetrics done | 20260508 11:08:05.647 | INFO | Finished a round of Baz IO: Wrote 1.443 GiB in 3.482 seconds | 20260508 11:08:05.647 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:08:05.647 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201039% / 0% / 7.94729e-05% | 20260508 11:08:05.647 | INFO | Baz IO overall: | 20260508 11:08:05.647 | INFO | BytesWritten : 3296399792462 | 20260508 11:08:05.647 | INFO | NumEvents : 1583610258275 | 20260508 11:08:07.197 | INFO | Analyzing chunk frames = [7626752,7627264) | 20260508 11:08:07.197 | INFO | MemInfo: { | 20260508 11:08:07.197 | INFO | "gpu" : " 17 GB", | 20260508 11:08:07.197 | INFO | "huge" : "", | 20260508 11:08:07.197 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:07.197 | INFO | "malloc" : " 14 GB", | 20260508 11:08:07.197 | INFO | "pinned" : " 29 GB" | 20260508 11:08:07.197 | INFO | } | 20260508 11:08:07.197 | INFO | Perf report for the last 5.13331 seconds: | 20260508 11:08:07.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:07.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:07.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:07.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:07.197 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 11:08:07.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:07.197 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 11:08:07.198 | INFO | PRE_HQ : 65.7%, 1.6, 54.5ms, 0 | 20260508 11:08:07.198 | INFO | BAZWRITER : 67.3%, 2.5, 86.2ms, 0 | 20260508 11:08:07.198 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:08:07.198 | INFO | Uploaded 36GiB to GPU in 2.60389s (13.8255GiB/s) | 20260508 11:08:07.198 | INFO | Downloaded 38GiB to GPU in 2.70479s (14.0492GiB/s) | 20260508 11:08:12.336 | INFO | Analyzing chunk frames = [7627264,7627776) | 20260508 11:08:12.336 | INFO | MemInfo: { | 20260508 11:08:12.336 | INFO | "gpu" : " 17 GB", | 20260508 11:08:12.336 | INFO | "huge" : "", | 20260508 11:08:12.336 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:12.336 | INFO | "malloc" : " 14 GB", | 20260508 11:08:12.336 | INFO | "pinned" : " 29 GB" | 20260508 11:08:12.337 | INFO | } | 20260508 11:08:12.337 | INFO | Perf report for the last 5.13971 seconds: | 20260508 11:08:12.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:12.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:08:12.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 11:08:12.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:12.337 | INFO | Uploaded 36GiB to GPU in 2.45726s (14.6505GiB/s) | 20260508 11:08:12.337 | INFO | Downloaded 34GiB to GPU in 2.46433s (13.7969GiB/s) | 20260508 11:08:17.480 | INFO | Analyzing chunk frames = [7627776,7628288) | 20260508 11:08:17.480 | INFO | MemInfo: { | 20260508 11:08:17.480 | INFO | "gpu" : " 17 GB", | 20260508 11:08:17.480 | INFO | "huge" : "", | 20260508 11:08:17.480 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:17.481 | INFO | "malloc" : " 14 GB", | 20260508 11:08:17.481 | INFO | "pinned" : " 29 GB" | 20260508 11:08:17.481 | INFO | } | 20260508 11:08:17.481 | INFO | Perf report for the last 5.14391 seconds: | 20260508 11:08:17.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:17.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 11:08:17.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 11:08:17.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:17.481 | INFO | Uploaded 36GiB to GPU in 2.50801s (14.354GiB/s) | 20260508 11:08:17.481 | INFO | Downloaded 34GiB to GPU in 2.45416s (13.854GiB/s) | 20260508 11:08:22.618 | INFO | Analyzing chunk frames = [7628288,7628800) | 20260508 11:08:22.618 | INFO | MemInfo: { | 20260508 11:08:22.618 | INFO | "gpu" : " 17 GB", | 20260508 11:08:22.618 | INFO | "huge" : "", | 20260508 11:08:22.618 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:22.618 | INFO | "malloc" : " 14 GB", | 20260508 11:08:22.618 | INFO | "pinned" : " 29 GB" | 20260508 11:08:22.618 | INFO | } | 20260508 11:08:22.618 | INFO | Perf report for the last 5.13722 seconds: | 20260508 11:08:22.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:22.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 11:08:22.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260508 11:08:22.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:22.619 | INFO | Uploaded 36GiB to GPU in 2.47829s (14.5262GiB/s) | 20260508 11:08:22.619 | INFO | Downloaded 34GiB to GPU in 2.45864s (13.8288GiB/s) | 20260508 11:08:27.752 | INFO | Analyzing chunk frames = [7628800,7629312) | 20260508 11:08:27.752 | INFO | MemInfo: { | 20260508 11:08:27.752 | INFO | "gpu" : " 17 GB", | 20260508 11:08:27.752 | INFO | "huge" : "", | 20260508 11:08:27.752 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:27.752 | INFO | "malloc" : " 14 GB", | 20260508 11:08:27.752 | INFO | "pinned" : " 29 GB" | 20260508 11:08:27.753 | INFO | } | 20260508 11:08:27.753 | INFO | Perf report for the last 5.13447 seconds: | 20260508 11:08:27.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:27.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:08:27.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260508 11:08:27.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:27.753 | INFO | Uploaded 36GiB to GPU in 2.27789s (15.8041GiB/s) | 20260508 11:08:27.753 | INFO | Downloaded 34GiB to GPU in 2.50591s (13.5679GiB/s) | 20260508 11:08:32.902 | INFO | Analyzing chunk frames = [7629312,7629824) | 20260508 11:08:32.902 | INFO | MemInfo: { | 20260508 11:08:32.902 | INFO | "gpu" : " 17 GB", | 20260508 11:08:32.902 | INFO | "huge" : "", | 20260508 11:08:32.902 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:32.902 | INFO | "malloc" : " 14 GB", | 20260508 11:08:32.902 | INFO | "pinned" : " 29 GB" | 20260508 11:08:32.902 | INFO | } | 20260508 11:08:32.902 | INFO | Perf report for the last 5.14975 seconds: | 20260508 11:08:32.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:32.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.902 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 11:08:32.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.902 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:08:32.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:32.903 | INFO | Uploaded 36GiB to GPU in 2.33571s (15.4129GiB/s) | 20260508 11:08:32.903 | INFO | Downloaded 34GiB to GPU in 2.43033s (13.9899GiB/s) | 20260508 11:08:38.043 | INFO | Analyzing chunk frames = [7629824,7630336) | 20260508 11:08:38.043 | INFO | MemInfo: { | 20260508 11:08:38.043 | INFO | "gpu" : " 17 GB", | 20260508 11:08:38.043 | INFO | "huge" : "", | 20260508 11:08:38.043 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:38.043 | INFO | "malloc" : " 14 GB", | 20260508 11:08:38.043 | INFO | "pinned" : " 29 GB" | 20260508 11:08:38.043 | INFO | } | 20260508 11:08:38.043 | INFO | Perf report for the last 5.14104 seconds: | 20260508 11:08:38.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:38.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.043 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 11:08:38.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.043 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:08:38.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:38.044 | INFO | Uploaded 36GiB to GPU in 2.44107s (14.7476GiB/s) | 20260508 11:08:38.044 | INFO | Downloaded 34GiB to GPU in 2.45552s (13.8463GiB/s) | 20260508 11:08:43.176 | INFO | Analyzing chunk frames = [7630336,7630848) | 20260508 11:08:43.176 | INFO | MemInfo: { | 20260508 11:08:43.176 | INFO | "gpu" : " 17 GB", | 20260508 11:08:43.176 | INFO | "huge" : "", | 20260508 11:08:43.176 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:43.176 | INFO | "malloc" : " 14 GB", | 20260508 11:08:43.176 | INFO | "pinned" : " 29 GB" | 20260508 11:08:43.176 | INFO | } | 20260508 11:08:43.177 | INFO | Perf report for the last 5.13327 seconds: | 20260508 11:08:43.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:43.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 11:08:43.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:08:43.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:43.177 | INFO | Uploaded 36GiB to GPU in 2.46798s (14.5868GiB/s) | 20260508 11:08:43.177 | INFO | Downloaded 34GiB to GPU in 2.46178s (13.8111GiB/s) | 20260508 11:08:43.274 | INFO | Starting a round of Baz IO | 20260508 11:08:46.697 | INFO | Populating RTMetrics report | 20260508 11:08:46.697 | INFO | RTMetrics report populated | 20260508 11:08:46.697 | INFO | Opening tmp RTMetrics json file | 20260508 11:08:46.701 | INFO | Serializing RTMetrics to json | 20260508 11:08:46.712 | INFO | Serialization done, swapping tmp file now... | 20260508 11:08:46.712 | INFO | RTMetrics done | 20260508 11:08:46.764 | INFO | Finished a round of Baz IO: Wrote 1.443 GiB in 3.489 seconds | 20260508 11:08:46.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:08:46.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20136% / 0% / 7.15256e-05% | 20260508 11:08:46.764 | INFO | Baz IO overall: | 20260508 11:08:46.764 | INFO | BytesWritten : 3297949421304 | 20260508 11:08:46.764 | INFO | NumEvents : 1584340063256 | 20260508 11:08:48.314 | INFO | Analyzing chunk frames = [7630848,7631360) | 20260508 11:08:48.314 | INFO | MemInfo: { | 20260508 11:08:48.314 | INFO | "gpu" : " 17 GB", | 20260508 11:08:48.314 | INFO | "huge" : "", | 20260508 11:08:48.314 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:48.314 | INFO | "malloc" : " 14 GB", | 20260508 11:08:48.314 | INFO | "pinned" : " 29 GB" | 20260508 11:08:48.314 | INFO | } | 20260508 11:08:48.314 | INFO | Perf report for the last 5.13749 seconds: | 20260508 11:08:48.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:48.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:48.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:48.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:48.314 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 11:08:48.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:48.314 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 11:08:48.314 | INFO | PRE_HQ : 65.5%, 1.6, 54.3ms, 0 | 20260508 11:08:48.314 | INFO | BAZWRITER : 67.2%, 2.5, 85.5ms, 0 | 20260508 11:08:48.314 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 11:08:48.315 | INFO | Uploaded 36GiB to GPU in 2.63364s (13.6693GiB/s) | 20260508 11:08:48.315 | INFO | Downloaded 38GiB to GPU in 2.7037s (14.0548GiB/s) | 20260508 11:08:53.452 | INFO | Analyzing chunk frames = [7631360,7631872) | 20260508 11:08:53.452 | INFO | MemInfo: { | 20260508 11:08:53.452 | INFO | "gpu" : " 17 GB", | 20260508 11:08:53.452 | INFO | "huge" : "", | 20260508 11:08:53.452 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:53.452 | INFO | "malloc" : " 14 GB", | 20260508 11:08:53.452 | INFO | "pinned" : " 29 GB" | 20260508 11:08:53.452 | INFO | } | 20260508 11:08:53.452 | INFO | Perf report for the last 5.13825 seconds: | 20260508 11:08:53.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:53.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.452 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 11:08:53.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.453 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:08:53.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:53.453 | INFO | Uploaded 36GiB to GPU in 2.50059s (14.3966GiB/s) | 20260508 11:08:53.453 | INFO | Downloaded 34GiB to GPU in 2.45895s (13.827GiB/s) | 20260508 11:08:58.596 | INFO | Analyzing chunk frames = [7631872,7632384) | 20260508 11:08:58.596 | INFO | MemInfo: { | 20260508 11:08:58.596 | INFO | "gpu" : " 17 GB", | 20260508 11:08:58.596 | INFO | "huge" : "", | 20260508 11:08:58.596 | INFO | "huge_shared" : " 29 GB", | 20260508 11:08:58.596 | INFO | "malloc" : " 14 GB", | 20260508 11:08:58.596 | INFO | "pinned" : " 29 GB" | 20260508 11:08:58.596 | INFO | } | 20260508 11:08:58.596 | INFO | Perf report for the last 5.1438 seconds: | 20260508 11:08:58.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:08:58.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.596 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:08:58.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.596 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:08:58.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:08:58.597 | INFO | Uploaded 36GiB to GPU in 2.51585s (14.3093GiB/s) | 20260508 11:08:58.597 | INFO | Downloaded 34GiB to GPU in 2.43362s (13.9709GiB/s) | 20260508 11:09:03.733 | INFO | Analyzing chunk frames = [7632384,7632896) | 20260508 11:09:03.733 | INFO | MemInfo: { | 20260508 11:09:03.733 | INFO | "gpu" : " 17 GB", | 20260508 11:09:03.733 | INFO | "huge" : "", | 20260508 11:09:03.733 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:03.733 | INFO | "malloc" : " 14 GB", | 20260508 11:09:03.733 | INFO | "pinned" : " 29 GB" | 20260508 11:09:03.733 | INFO | } | 20260508 11:09:03.734 | INFO | Perf report for the last 5.13734 seconds: | 20260508 11:09:03.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:03.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 11:09:03.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:09:03.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:03.734 | INFO | Uploaded 36GiB to GPU in 2.50692s (14.3602GiB/s) | 20260508 11:09:03.734 | INFO | Downloaded 34GiB to GPU in 2.43441s (13.9664GiB/s) | 20260508 11:09:08.876 | INFO | Analyzing chunk frames = [7632896,7633408) | 20260508 11:09:08.876 | INFO | MemInfo: { | 20260508 11:09:08.876 | INFO | "gpu" : " 17 GB", | 20260508 11:09:08.876 | INFO | "huge" : "", | 20260508 11:09:08.876 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:08.876 | INFO | "malloc" : " 14 GB", | 20260508 11:09:08.876 | INFO | "pinned" : " 29 GB" | 20260508 11:09:08.876 | INFO | } | 20260508 11:09:08.876 | INFO | Perf report for the last 5.14237 seconds: | 20260508 11:09:08.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:08.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 11:09:08.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:09:08.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:08.877 | INFO | Uploaded 36GiB to GPU in 2.40932s (14.942GiB/s) | 20260508 11:09:08.877 | INFO | Downloaded 34GiB to GPU in 2.51159s (13.5372GiB/s) | 20260508 11:09:14.007 | INFO | Analyzing chunk frames = [7633408,7633920) | 20260508 11:09:14.007 | INFO | MemInfo: { | 20260508 11:09:14.007 | INFO | "gpu" : " 17 GB", | 20260508 11:09:14.007 | INFO | "huge" : "", | 20260508 11:09:14.007 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:14.007 | INFO | "malloc" : " 14 GB", | 20260508 11:09:14.007 | INFO | "pinned" : " 29 GB" | 20260508 11:09:14.007 | INFO | } | 20260508 11:09:14.007 | INFO | Perf report for the last 5.13129 seconds: | 20260508 11:09:14.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:14.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.007 | INFO | ANALYSIS : 65.6%, 3.0, 101.8ms, 0 | 20260508 11:09:14.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.007 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:09:14.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:14.008 | INFO | Uploaded 36GiB to GPU in 2.41965s (14.8782GiB/s) | 20260508 11:09:14.008 | INFO | Downloaded 34GiB to GPU in 2.42077s (14.0451GiB/s) | 20260508 11:09:19.154 | INFO | Analyzing chunk frames = [7633920,7634432) | 20260508 11:09:19.154 | INFO | MemInfo: { | 20260508 11:09:19.154 | INFO | "gpu" : " 17 GB", | 20260508 11:09:19.154 | INFO | "huge" : "", | 20260508 11:09:19.154 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:19.154 | INFO | "malloc" : " 14 GB", | 20260508 11:09:19.154 | INFO | "pinned" : " 29 GB" | 20260508 11:09:19.154 | INFO | } | 20260508 11:09:19.154 | INFO | Perf report for the last 5.14702 seconds: | 20260508 11:09:19.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:19.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.154 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 11:09:19.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.154 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:09:19.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:19.155 | INFO | Uploaded 36GiB to GPU in 2.43917s (14.7591GiB/s) | 20260508 11:09:19.155 | INFO | Downloaded 34GiB to GPU in 2.43074s (13.9875GiB/s) | 20260508 11:09:24.289 | INFO | Analyzing chunk frames = [7634432,7634944) | 20260508 11:09:24.289 | INFO | MemInfo: { | 20260508 11:09:24.289 | INFO | "gpu" : " 17 GB", | 20260508 11:09:24.289 | INFO | "huge" : "", | 20260508 11:09:24.289 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:24.289 | INFO | "malloc" : " 14 GB", | 20260508 11:09:24.289 | INFO | "pinned" : " 29 GB" | 20260508 11:09:24.289 | INFO | } | 20260508 11:09:24.289 | INFO | Perf report for the last 5.13476 seconds: | 20260508 11:09:24.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:24.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.289 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 11:09:24.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.289 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:09:24.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:24.290 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:09:24.290 | INFO | Total mem usage: 16.7 GiB | 20260508 11:09:24.290 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:09:24.290 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:09:24.290 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:09:24.290 | INFO | High Water Breakdown: | 20260508 11:09:24.290 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:09:24.290 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:09:24.290 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:09:24.290 | INFO | ------------------------------------------ | 20260508 11:09:24.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:09:24.290 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:09:24.290 | INFO | Total mem usage: 26.5 GiB | 20260508 11:09:24.290 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:09:24.290 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:09:24.290 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:09:24.290 | INFO | High Water Breakdown: | 20260508 11:09:24.290 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:09:24.290 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:09:24.290 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:09:24.290 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:09:24.290 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:09:24.290 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:09:24.290 | INFO | Uploaded 36GiB to GPU in 2.48602s (14.481GiB/s) | 20260508 11:09:24.290 | INFO | Downloaded 34GiB to GPU in 2.44863s (13.8853GiB/s) | 20260508 11:09:24.290 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:09:24.290 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:09:24.290 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:09:24.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:09:24.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:09:24.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:09:24.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:09:24.290 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:09:24.290 | INFO | ------------------------------------------ | 20260508 11:09:24.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:09:24.290 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:09:24.290 | INFO | Total mem usage: 27.8 GiB | 20260508 11:09:24.290 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:09:24.290 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:09:24.290 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:09:24.290 | INFO | High Water Breakdown: | 20260508 11:09:24.290 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:09:24.290 | INFO | ------------------------------------------ | 20260508 11:09:24.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:09:24.290 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:09:24.290 | INFO | Total mem usage: 15.7 GiB | 20260508 11:09:24.290 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:09:24.290 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:09:24.290 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:09:24.290 | INFO | High Water Breakdown: | 20260508 11:09:24.290 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:09:24.290 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:09:24.290 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:09:24.290 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:09:24.290 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:09:24.290 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:09:24.290 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:09:24.290 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:09:24.290 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:09:24.290 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:09:24.290 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:09:24.290 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:09:24.290 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:09:24.290 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:09:24.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:09:24.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:09:24.290 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:09:24.290 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:09:24.290 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:09:24.290 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:09:24.290 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:09:24.291 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:09:24.291 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:09:24.291 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:09:24.291 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:09:24.291 | INFO | ------------------------------------------ | 20260508 11:09:24.291 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:09:24.291 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:09:24.291 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:09:24.291 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:09:24.291 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.50e+01, 1.40e+01 / 4.34e-01 | 20260508 11:09:24.291 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.06e-01, 1.12e-02 / 4.28e-03 | 20260508 11:09:24.291 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.44e+00, 5.95e-01 / 2.28e-01 | 20260508 11:09:24.291 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.50e+00, 2.11e-01 / 1.13e-01 | 20260508 11:09:24.291 | INFO | PerfReport: 16.02%, DME -- 4.18e+01 / 4.51e+01, 3.59e+00 / 9.43e-01 | 20260508 11:09:24.291 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.79e+00 / 1.27e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.34e+00 / 2.56e+00, 4.06e-01 / 2.53e-01 | 20260508 11:09:24.291 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.11e+00, 2.42e-01 / 1.22e-01 | 20260508 11:09:24.291 | INFO | PerfReport: 0.92%, PulseSerializing -- 3.01e-01 / 3.39e-01, 4.82e-02 / 2.88e-02 | 20260508 11:09:24.291 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:09:24.291 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:09:24.291 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:09:24.291 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 24.40%, Upload -- 2.57e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 24.59%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 30.99%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 2.27%, LockUpload -- 2.39e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 2.50%, LockDownload -- 2.64e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 15.23%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:09:24.291 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 11:09:24.394 | INFO | Starting a round of Baz IO | 20260508 11:09:27.797 | INFO | Populating RTMetrics report | 20260508 11:09:27.798 | INFO | RTMetrics report populated | 20260508 11:09:27.798 | INFO | Opening tmp RTMetrics json file | 20260508 11:09:27.799 | INFO | Serializing RTMetrics to json | 20260508 11:09:27.800 | INFO | Serialization done, swapping tmp file now... | 20260508 11:09:27.801 | INFO | RTMetrics done | 20260508 11:09:27.859 | INFO | Finished a round of Baz IO: Wrote 1.444 GiB in 3.464 seconds | 20260508 11:09:27.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:09:27.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201658% / 3.97364e-06% / 8.34465e-05% | 20260508 11:09:27.859 | INFO | Baz IO overall: | 20260508 11:09:27.859 | INFO | BytesWritten : 3299500322560 | 20260508 11:09:27.859 | INFO | NumEvents : 1585070735896 | 20260508 11:09:29.428 | INFO | Analyzing chunk frames = [7634944,7635456) | 20260508 11:09:29.428 | INFO | MemInfo: { | 20260508 11:09:29.428 | INFO | "gpu" : " 17 GB", | 20260508 11:09:29.428 | INFO | "huge" : "", | 20260508 11:09:29.428 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:29.428 | INFO | "malloc" : " 14 GB", | 20260508 11:09:29.428 | INFO | "pinned" : " 29 GB" | 20260508 11:09:29.428 | INFO | } | 20260508 11:09:29.428 | INFO | Perf report for the last 5.13868 seconds: | 20260508 11:09:29.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:29.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:29.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:29.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:29.428 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 11:09:29.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:29.428 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:09:29.428 | INFO | PRE_HQ : 65.5%, 1.6, 53.8ms, 0 | 20260508 11:09:29.428 | INFO | BAZWRITER : 67.0%, 2.5, 85.1ms, 0 | 20260508 11:09:29.428 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 11:09:29.429 | INFO | Uploaded 36GiB to GPU in 2.52127s (14.2785GiB/s) | 20260508 11:09:29.429 | INFO | Downloaded 38GiB to GPU in 2.68335s (14.1614GiB/s) | 20260508 11:09:34.573 | INFO | Analyzing chunk frames = [7635456,7635968) | 20260508 11:09:34.573 | INFO | MemInfo: { | 20260508 11:09:34.573 | INFO | "gpu" : " 17 GB", | 20260508 11:09:34.573 | INFO | "huge" : "", | 20260508 11:09:34.573 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:34.573 | INFO | "malloc" : " 14 GB", | 20260508 11:09:34.573 | INFO | "pinned" : " 29 GB" | 20260508 11:09:34.573 | INFO | } | 20260508 11:09:34.573 | INFO | Perf report for the last 5.1451 seconds: | 20260508 11:09:34.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:34.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 11:09:34.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260508 11:09:34.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:34.573 | INFO | Uploaded 36GiB to GPU in 2.49766s (14.4135GiB/s) | 20260508 11:09:34.573 | INFO | Downloaded 34GiB to GPU in 2.46499s (13.7932GiB/s) | 20260508 11:09:39.710 | INFO | Analyzing chunk frames = [7635968,7636480) | 20260508 11:09:39.710 | INFO | MemInfo: { | 20260508 11:09:39.710 | INFO | "gpu" : " 17 GB", | 20260508 11:09:39.710 | INFO | "huge" : "", | 20260508 11:09:39.710 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:39.710 | INFO | "malloc" : " 14 GB", | 20260508 11:09:39.710 | INFO | "pinned" : " 29 GB" | 20260508 11:09:39.710 | INFO | } | 20260508 11:09:39.711 | INFO | Perf report for the last 5.13766 seconds: | 20260508 11:09:39.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:39.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:09:39.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:09:39.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:39.711 | INFO | Uploaded 36GiB to GPU in 2.50216s (14.3875GiB/s) | 20260508 11:09:39.711 | INFO | Downloaded 34GiB to GPU in 2.444s (13.9116GiB/s) | 20260508 11:09:44.849 | INFO | Analyzing chunk frames = [7636480,7636992) | 20260508 11:09:44.849 | INFO | MemInfo: { | 20260508 11:09:44.849 | INFO | "gpu" : " 17 GB", | 20260508 11:09:44.849 | INFO | "huge" : "", | 20260508 11:09:44.849 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:44.849 | INFO | "malloc" : " 14 GB", | 20260508 11:09:44.849 | INFO | "pinned" : " 29 GB" | 20260508 11:09:44.849 | INFO | } | 20260508 11:09:44.849 | INFO | Perf report for the last 5.13829 seconds: | 20260508 11:09:44.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:44.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:09:44.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:09:44.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:44.849 | INFO | Uploaded 36GiB to GPU in 2.50927s (14.3468GiB/s) | 20260508 11:09:44.849 | INFO | Downloaded 34GiB to GPU in 2.43961s (13.9367GiB/s) | 20260508 11:09:49.989 | INFO | Analyzing chunk frames = [7636992,7637504) | 20260508 11:09:49.989 | INFO | MemInfo: { | 20260508 11:09:49.989 | INFO | "gpu" : " 17 GB", | 20260508 11:09:49.989 | INFO | "huge" : "", | 20260508 11:09:49.989 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:49.989 | INFO | "malloc" : " 14 GB", | 20260508 11:09:49.989 | INFO | "pinned" : " 29 GB" | 20260508 11:09:49.989 | INFO | } | 20260508 11:09:49.989 | INFO | Perf report for the last 5.13998 seconds: | 20260508 11:09:49.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:49.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 11:09:49.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:09:49.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:49.989 | INFO | Uploaded 36GiB to GPU in 2.43248s (14.7997GiB/s) | 20260508 11:09:49.989 | INFO | Downloaded 34GiB to GPU in 2.51958s (13.4943GiB/s) | 20260508 11:09:55.127 | INFO | Analyzing chunk frames = [7637504,7638016) | 20260508 11:09:55.127 | INFO | MemInfo: { | 20260508 11:09:55.127 | INFO | "gpu" : " 17 GB", | 20260508 11:09:55.127 | INFO | "huge" : "", | 20260508 11:09:55.127 | INFO | "huge_shared" : " 29 GB", | 20260508 11:09:55.127 | INFO | "malloc" : " 14 GB", | 20260508 11:09:55.128 | INFO | "pinned" : " 29 GB" | 20260508 11:09:55.128 | INFO | } | 20260508 11:09:55.128 | INFO | Perf report for the last 5.13876 seconds: | 20260508 11:09:55.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:09:55.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:09:55.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:09:55.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:09:55.128 | INFO | Uploaded 36GiB to GPU in 2.40523s (14.9674GiB/s) | 20260508 11:09:55.128 | INFO | Downloaded 34GiB to GPU in 2.42084s (14.0447GiB/s) | 20260508 11:10:00.259 | INFO | Analyzing chunk frames = [7638016,7638528) | 20260508 11:10:00.259 | INFO | MemInfo: { | 20260508 11:10:00.259 | INFO | "gpu" : " 17 GB", | 20260508 11:10:00.259 | INFO | "huge" : "", | 20260508 11:10:00.259 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:00.259 | INFO | "malloc" : " 14 GB", | 20260508 11:10:00.259 | INFO | "pinned" : " 29 GB" | 20260508 11:10:00.259 | INFO | } | 20260508 11:10:00.259 | INFO | Perf report for the last 5.13183 seconds: | 20260508 11:10:00.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:00.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.260 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 11:10:00.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.260 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:10:00.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:00.260 | INFO | Uploaded 36GiB to GPU in 2.45434s (14.6679GiB/s) | 20260508 11:10:00.260 | INFO | Downloaded 34GiB to GPU in 2.43254s (13.9771GiB/s) | 20260508 11:10:05.400 | INFO | Analyzing chunk frames = [7638528,7639040) | 20260508 11:10:05.400 | INFO | MemInfo: { | 20260508 11:10:05.400 | INFO | "gpu" : " 17 GB", | 20260508 11:10:05.400 | INFO | "huge" : "", | 20260508 11:10:05.400 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:05.400 | INFO | "malloc" : " 14 GB", | 20260508 11:10:05.400 | INFO | "pinned" : " 29 GB" | 20260508 11:10:05.400 | INFO | } | 20260508 11:10:05.400 | INFO | Perf report for the last 5.14074 seconds: | 20260508 11:10:05.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:05.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.400 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 11:10:05.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.400 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:10:05.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:05.401 | INFO | Uploaded 36GiB to GPU in 2.4372s (14.7711GiB/s) | 20260508 11:10:05.401 | INFO | Downloaded 34GiB to GPU in 2.4237s (14.0281GiB/s) | 20260508 11:10:05.507 | INFO | Starting a round of Baz IO | 20260508 11:10:08.920 | INFO | Populating RTMetrics report | 20260508 11:10:08.920 | INFO | RTMetrics report populated | 20260508 11:10:08.920 | INFO | Opening tmp RTMetrics json file | 20260508 11:10:08.927 | INFO | Serializing RTMetrics to json | 20260508 11:10:08.928 | INFO | Serialization done, swapping tmp file now... | 20260508 11:10:08.929 | INFO | RTMetrics done | 20260508 11:10:08.985 | INFO | Finished a round of Baz IO: Wrote 1.444 GiB in 3.477 seconds | 20260508 11:10:08.985 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:10:08.985 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201448% / 0% / 7.15256e-05% | 20260508 11:10:08.985 | INFO | Baz IO overall: | 20260508 11:10:08.985 | INFO | BytesWritten : 3301050982074 | 20260508 11:10:08.985 | INFO | NumEvents : 1585801492909 | 20260508 11:10:10.546 | INFO | Analyzing chunk frames = [7639040,7639552) | 20260508 11:10:10.546 | INFO | MemInfo: { | 20260508 11:10:10.546 | INFO | "gpu" : " 17 GB", | 20260508 11:10:10.546 | INFO | "huge" : "", | 20260508 11:10:10.546 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:10.546 | INFO | "malloc" : " 14 GB", | 20260508 11:10:10.546 | INFO | "pinned" : " 29 GB" | 20260508 11:10:10.546 | INFO | } | 20260508 11:10:10.546 | INFO | Perf report for the last 5.14583 seconds: | 20260508 11:10:10.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:10.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:10.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:10.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:10.546 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:10:10.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:10.546 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260508 11:10:10.546 | INFO | PRE_HQ : 66.2%, 1.7, 60.0ms, 0 | 20260508 11:10:10.546 | INFO | BAZWRITER : 66.8%, 2.7, 91.2ms, 0 | 20260508 11:10:10.546 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:10:10.546 | INFO | Uploaded 36GiB to GPU in 2.61226s (13.7812GiB/s) | 20260508 11:10:10.546 | INFO | Downloaded 38GiB to GPU in 2.68203s (14.1683GiB/s) | 20260508 11:10:15.688 | INFO | Analyzing chunk frames = [7639552,7640064) | 20260508 11:10:15.688 | INFO | MemInfo: { | 20260508 11:10:15.688 | INFO | "gpu" : " 17 GB", | 20260508 11:10:15.688 | INFO | "huge" : "", | 20260508 11:10:15.688 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:15.688 | INFO | "malloc" : " 14 GB", | 20260508 11:10:15.688 | INFO | "pinned" : " 29 GB" | 20260508 11:10:15.688 | INFO | } | 20260508 11:10:15.688 | INFO | Perf report for the last 5.14229 seconds: | 20260508 11:10:15.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:15.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.688 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 11:10:15.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.689 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:10:15.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:15.689 | INFO | Uploaded 36GiB to GPU in 2.50534s (14.3693GiB/s) | 20260508 11:10:15.689 | INFO | Downloaded 34GiB to GPU in 2.46709s (13.7814GiB/s) | 20260508 11:10:20.823 | INFO | Analyzing chunk frames = [7640064,7640576) | 20260508 11:10:20.823 | INFO | MemInfo: { | 20260508 11:10:20.823 | INFO | "gpu" : " 17 GB", | 20260508 11:10:20.823 | INFO | "huge" : "", | 20260508 11:10:20.823 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:20.823 | INFO | "malloc" : " 14 GB", | 20260508 11:10:20.823 | INFO | "pinned" : " 29 GB" | 20260508 11:10:20.823 | INFO | } | 20260508 11:10:20.823 | INFO | Perf report for the last 5.13493 seconds: | 20260508 11:10:20.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:20.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.823 | INFO | ANALYSIS : 66.7%, 3.0, 103.5ms, 0 | 20260508 11:10:20.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.823 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:10:20.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:20.824 | INFO | Uploaded 36GiB to GPU in 2.51204s (14.331GiB/s) | 20260508 11:10:20.824 | INFO | Downloaded 34GiB to GPU in 2.44761s (13.8911GiB/s) | 20260508 11:10:25.959 | INFO | Analyzing chunk frames = [7640576,7641088) | 20260508 11:10:25.959 | INFO | MemInfo: { | 20260508 11:10:25.959 | INFO | "gpu" : " 17 GB", | 20260508 11:10:25.959 | INFO | "huge" : "", | 20260508 11:10:25.959 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:25.959 | INFO | "malloc" : " 14 GB", | 20260508 11:10:25.959 | INFO | "pinned" : " 29 GB" | 20260508 11:10:25.959 | INFO | } | 20260508 11:10:25.959 | INFO | Perf report for the last 5.13583 seconds: | 20260508 11:10:25.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:25.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.959 | INFO | ANALYSIS : 66.4%, 3.0, 102.9ms, 0 | 20260508 11:10:25.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.959 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:10:25.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:25.960 | INFO | Uploaded 36GiB to GPU in 2.54385s (14.1518GiB/s) | 20260508 11:10:25.960 | INFO | Downloaded 34GiB to GPU in 2.43784s (13.9468GiB/s) | 20260508 11:10:31.102 | INFO | Analyzing chunk frames = [7641088,7641600) | 20260508 11:10:31.102 | INFO | MemInfo: { | 20260508 11:10:31.102 | INFO | "gpu" : " 17 GB", | 20260508 11:10:31.102 | INFO | "huge" : "", | 20260508 11:10:31.102 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:31.102 | INFO | "malloc" : " 14 GB", | 20260508 11:10:31.102 | INFO | "pinned" : " 29 GB" | 20260508 11:10:31.102 | INFO | } | 20260508 11:10:31.102 | INFO | Perf report for the last 5.14305 seconds: | 20260508 11:10:31.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:31.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.102 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 11:10:31.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.102 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:10:31.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:31.103 | INFO | Uploaded 36GiB to GPU in 2.41782s (14.8895GiB/s) | 20260508 11:10:31.103 | INFO | Downloaded 34GiB to GPU in 2.52733s (13.4529GiB/s) | 20260508 11:10:36.248 | INFO | Analyzing chunk frames = [7641600,7642112) | 20260508 11:10:36.248 | INFO | MemInfo: { | 20260508 11:10:36.248 | INFO | "gpu" : " 17 GB", | 20260508 11:10:36.248 | INFO | "huge" : "", | 20260508 11:10:36.248 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:36.248 | INFO | "malloc" : " 14 GB", | 20260508 11:10:36.248 | INFO | "pinned" : " 29 GB" | 20260508 11:10:36.248 | INFO | } | 20260508 11:10:36.248 | INFO | Perf report for the last 5.14604 seconds: | 20260508 11:10:36.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:36.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.248 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:10:36.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.248 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:10:36.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:36.249 | INFO | Uploaded 36GiB to GPU in 2.41147s (14.9286GiB/s) | 20260508 11:10:36.249 | INFO | Downloaded 34GiB to GPU in 2.42803s (14.0031GiB/s) | 20260508 11:10:41.378 | INFO | Analyzing chunk frames = [7642112,7642624) | 20260508 11:10:41.378 | INFO | MemInfo: { | 20260508 11:10:41.378 | INFO | "gpu" : " 17 GB", | 20260508 11:10:41.378 | INFO | "huge" : "", | 20260508 11:10:41.378 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:41.379 | INFO | "malloc" : " 14 GB", | 20260508 11:10:41.379 | INFO | "pinned" : " 29 GB" | 20260508 11:10:41.379 | INFO | } | 20260508 11:10:41.379 | INFO | Perf report for the last 5.13038 seconds: | 20260508 11:10:41.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:41.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.379 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 11:10:41.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.379 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 11:10:41.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:41.380 | INFO | Uploaded 36GiB to GPU in 2.46143s (14.6256GiB/s) | 20260508 11:10:41.380 | INFO | Downloaded 34GiB to GPU in 2.4459s (13.9008GiB/s) | 20260508 11:10:46.518 | INFO | Analyzing chunk frames = [7642624,7643136) | 20260508 11:10:46.518 | INFO | MemInfo: { | 20260508 11:10:46.518 | INFO | "gpu" : " 17 GB", | 20260508 11:10:46.518 | INFO | "huge" : "", | 20260508 11:10:46.518 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:46.518 | INFO | "malloc" : " 14 GB", | 20260508 11:10:46.518 | INFO | "pinned" : " 29 GB" | 20260508 11:10:46.518 | INFO | } | 20260508 11:10:46.518 | INFO | Perf report for the last 5.13933 seconds: | 20260508 11:10:46.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:46.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:10:46.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:10:46.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:46.518 | INFO | Uploaded 36GiB to GPU in 2.46375s (14.6118GiB/s) | 20260508 11:10:46.518 | INFO | Downloaded 34GiB to GPU in 2.42804s (14.0031GiB/s) | 20260508 11:10:46.519 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:10:46.519 | INFO | Total mem usage: 16.7 GiB | 20260508 11:10:46.519 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:10:46.519 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:10:46.519 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:10:46.519 | INFO | High Water Breakdown: | 20260508 11:10:46.519 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:10:46.519 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:10:46.519 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:10:46.519 | INFO | ------------------------------------------ | 20260508 11:10:46.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:10:46.519 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:10:46.519 | INFO | Total mem usage: 26.5 GiB | 20260508 11:10:46.519 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:10:46.519 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:10:46.519 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:10:46.519 | INFO | High Water Breakdown: | 20260508 11:10:46.519 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:10:46.519 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:10:46.519 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:10:46.519 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:10:46.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:10:46.519 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:10:46.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:10:46.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:10:46.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:10:46.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:10:46.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:10:46.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:10:46.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:10:46.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:10:46.519 | INFO | ------------------------------------------ | 20260508 11:10:46.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:10:46.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:10:46.519 | INFO | Total mem usage: 27.8 GiB | 20260508 11:10:46.519 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:10:46.519 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:10:46.519 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:10:46.519 | INFO | High Water Breakdown: | 20260508 11:10:46.519 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:10:46.519 | INFO | ------------------------------------------ | 20260508 11:10:46.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:10:46.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:10:46.519 | INFO | Total mem usage: 15.7 GiB | 20260508 11:10:46.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:10:46.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:10:46.519 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:10:46.519 | INFO | High Water Breakdown: | 20260508 11:10:46.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:10:46.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:10:46.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:10:46.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:10:46.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:10:46.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:10:46.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:10:46.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:10:46.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:10:46.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:10:46.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:10:46.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:10:46.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:10:46.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:10:46.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:10:46.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:10:46.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:10:46.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:10:46.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:10:46.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:10:46.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:10:46.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:10:46.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:10:46.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:10:46.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:10:46.520 | INFO | ------------------------------------------ | 20260508 11:10:46.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:10:46.520 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:10:46.520 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:10:46.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:10:46.520 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.12e-01 | 20260508 11:10:46.520 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.20e-02 / 3.27e-03 | 20260508 11:10:46.520 | INFO | PerfReport: 12.15%, Baseline -- 3.97e+00 / 4.43e+00, 5.95e-01 / 2.17e-01 | 20260508 11:10:46.520 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.30e-01 | 20260508 11:10:46.520 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.50e+01, 3.65e+00 / 1.15e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.82e+00 / 1.40e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.06e-01 / 2.24e-01 | 20260508 11:10:46.520 | INFO | PerfReport: 3.00%, Metrics -- 9.80e-01 / 1.16e+00, 2.45e-01 / 1.21e-01 | 20260508 11:10:46.520 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.31e-01, 4.74e-02 / 2.56e-02 | 20260508 11:10:46.520 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:10:46.520 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:10:46.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:10:46.520 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 24.52%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 24.53%, Download -- 2.58e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 31.00%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 2.38%, LockUpload -- 2.50e+00 / 0.00e+00, 5.93e+00 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 2.41%, LockDownload -- 2.53e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 15.15%, LockCompute -- 1.60e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:10:46.520 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260508 11:10:46.615 | INFO | Starting a round of Baz IO | 20260508 11:10:50.036 | INFO | Populating RTMetrics report | 20260508 11:10:50.036 | INFO | RTMetrics report populated | 20260508 11:10:50.036 | INFO | Opening tmp RTMetrics json file | 20260508 11:10:50.037 | INFO | Serializing RTMetrics to json | 20260508 11:10:50.039 | INFO | Serialization done, swapping tmp file now... | 20260508 11:10:50.039 | INFO | RTMetrics done | 20260508 11:10:50.102 | INFO | Finished a round of Baz IO: Wrote 1.442 GiB in 3.486 seconds | 20260508 11:10:50.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:10:50.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200717% / 0% / 9.13938e-05% | 20260508 11:10:50.102 | INFO | Baz IO overall: | 20260508 11:10:50.102 | INFO | BytesWritten : 3302599791930 | 20260508 11:10:50.102 | INFO | NumEvents : 1586530908418 | 20260508 11:10:51.663 | INFO | Analyzing chunk frames = [7643136,7643648) | 20260508 11:10:51.663 | INFO | MemInfo: { | 20260508 11:10:51.663 | INFO | "gpu" : " 17 GB", | 20260508 11:10:51.663 | INFO | "huge" : "", | 20260508 11:10:51.663 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:51.663 | INFO | "malloc" : " 14 GB", | 20260508 11:10:51.663 | INFO | "pinned" : " 29 GB" | 20260508 11:10:51.663 | INFO | } | 20260508 11:10:51.663 | INFO | Perf report for the last 5.14464 seconds: | 20260508 11:10:51.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:51.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:51.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:51.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:51.663 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 11:10:51.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:51.663 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:10:51.663 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 11:10:51.663 | INFO | BAZWRITER : 67.0%, 2.5, 85.6ms, 0 | 20260508 11:10:51.663 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 11:10:51.663 | INFO | Uploaded 36GiB to GPU in 2.59075s (13.8956GiB/s) | 20260508 11:10:51.663 | INFO | Downloaded 38GiB to GPU in 2.7118s (14.0128GiB/s) | 20260508 11:10:56.801 | INFO | Analyzing chunk frames = [7643648,7644160) | 20260508 11:10:56.801 | INFO | MemInfo: { | 20260508 11:10:56.801 | INFO | "gpu" : " 17 GB", | 20260508 11:10:56.801 | INFO | "huge" : "", | 20260508 11:10:56.801 | INFO | "huge_shared" : " 29 GB", | 20260508 11:10:56.801 | INFO | "malloc" : " 14 GB", | 20260508 11:10:56.801 | INFO | "pinned" : " 29 GB" | 20260508 11:10:56.801 | INFO | } | 20260508 11:10:56.801 | INFO | Perf report for the last 5.13846 seconds: | 20260508 11:10:56.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:10:56.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.801 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:10:56.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.801 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:10:56.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:10:56.802 | INFO | Uploaded 36GiB to GPU in 2.43266s (14.7986GiB/s) | 20260508 11:10:56.802 | INFO | Downloaded 34GiB to GPU in 2.47206s (13.7537GiB/s) | 20260508 11:11:01.935 | INFO | Analyzing chunk frames = [7644160,7644672) | 20260508 11:11:01.935 | INFO | MemInfo: { | 20260508 11:11:01.935 | INFO | "gpu" : " 17 GB", | 20260508 11:11:01.936 | INFO | "huge" : "", | 20260508 11:11:01.936 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:01.936 | INFO | "malloc" : " 14 GB", | 20260508 11:11:01.936 | INFO | "pinned" : " 29 GB" | 20260508 11:11:01.936 | INFO | } | 20260508 11:11:01.936 | INFO | Perf report for the last 5.1344 seconds: | 20260508 11:11:01.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:01.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 11:11:01.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:11:01.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:01.936 | INFO | Uploaded 36GiB to GPU in 2.46073s (14.6298GiB/s) | 20260508 11:11:01.936 | INFO | Downloaded 34GiB to GPU in 2.446s (13.9003GiB/s) | 20260508 11:11:07.075 | INFO | Analyzing chunk frames = [7644672,7645184) | 20260508 11:11:07.075 | INFO | MemInfo: { | 20260508 11:11:07.075 | INFO | "gpu" : " 17 GB", | 20260508 11:11:07.075 | INFO | "huge" : "", | 20260508 11:11:07.075 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:07.075 | INFO | "malloc" : " 14 GB", | 20260508 11:11:07.075 | INFO | "pinned" : " 29 GB" | 20260508 11:11:07.075 | INFO | } | 20260508 11:11:07.076 | INFO | Perf report for the last 5.13991 seconds: | 20260508 11:11:07.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:07.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 11:11:07.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:11:07.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:07.076 | INFO | Uploaded 36GiB to GPU in 2.45935s (14.638GiB/s) | 20260508 11:11:07.076 | INFO | Downloaded 34GiB to GPU in 2.45031s (13.8758GiB/s) | 20260508 11:11:12.215 | INFO | Analyzing chunk frames = [7645184,7645696) | 20260508 11:11:12.215 | INFO | MemInfo: { | 20260508 11:11:12.215 | INFO | "gpu" : " 17 GB", | 20260508 11:11:12.215 | INFO | "huge" : "", | 20260508 11:11:12.215 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:12.215 | INFO | "malloc" : " 14 GB", | 20260508 11:11:12.215 | INFO | "pinned" : " 29 GB" | 20260508 11:11:12.215 | INFO | } | 20260508 11:11:12.215 | INFO | Perf report for the last 5.1398 seconds: | 20260508 11:11:12.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:12.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.215 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:11:12.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.215 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:11:12.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:12.216 | INFO | Uploaded 36GiB to GPU in 2.27599s (15.8173GiB/s) | 20260508 11:11:12.216 | INFO | Downloaded 34GiB to GPU in 2.5102s (13.5447GiB/s) | 20260508 11:11:17.357 | INFO | Analyzing chunk frames = [7645696,7646208) | 20260508 11:11:17.357 | INFO | MemInfo: { | 20260508 11:11:17.357 | INFO | "gpu" : " 17 GB", | 20260508 11:11:17.357 | INFO | "huge" : "", | 20260508 11:11:17.357 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:17.357 | INFO | "malloc" : " 14 GB", | 20260508 11:11:17.357 | INFO | "pinned" : " 29 GB" | 20260508 11:11:17.357 | INFO | } | 20260508 11:11:17.357 | INFO | Perf report for the last 5.14188 seconds: | 20260508 11:11:17.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:17.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:11:17.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260508 11:11:17.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:17.357 | INFO | Uploaded 36GiB to GPU in 2.36431s (15.2265GiB/s) | 20260508 11:11:17.358 | INFO | Downloaded 34GiB to GPU in 2.45714s (13.8372GiB/s) | 20260508 11:11:22.500 | INFO | Analyzing chunk frames = [7646208,7646720) | 20260508 11:11:22.500 | INFO | MemInfo: { | 20260508 11:11:22.500 | INFO | "gpu" : " 17 GB", | 20260508 11:11:22.500 | INFO | "huge" : "", | 20260508 11:11:22.500 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:22.500 | INFO | "malloc" : " 14 GB", | 20260508 11:11:22.500 | INFO | "pinned" : " 29 GB" | 20260508 11:11:22.500 | INFO | } | 20260508 11:11:22.500 | INFO | Perf report for the last 5.14288 seconds: | 20260508 11:11:22.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:22.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 11:11:22.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:11:22.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:22.500 | INFO | Uploaded 36GiB to GPU in 2.41203s (14.9252GiB/s) | 20260508 11:11:22.500 | INFO | Downloaded 34GiB to GPU in 2.46064s (13.8175GiB/s) | 20260508 11:11:27.635 | INFO | Analyzing chunk frames = [7646720,7647232) | 20260508 11:11:27.636 | INFO | MemInfo: { | 20260508 11:11:27.636 | INFO | "gpu" : " 17 GB", | 20260508 11:11:27.636 | INFO | "huge" : "", | 20260508 11:11:27.636 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:27.636 | INFO | "malloc" : " 14 GB", | 20260508 11:11:27.636 | INFO | "pinned" : " 29 GB" | 20260508 11:11:27.636 | INFO | } | 20260508 11:11:27.636 | INFO | Perf report for the last 5.1356 seconds: | 20260508 11:11:27.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:27.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:11:27.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:11:27.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:27.636 | INFO | Uploaded 36GiB to GPU in 2.46173s (14.6239GiB/s) | 20260508 11:11:27.636 | INFO | Downloaded 34GiB to GPU in 2.48175s (13.7GiB/s) | 20260508 11:11:27.733 | INFO | Starting a round of Baz IO | 20260508 11:11:31.151 | INFO | Populating RTMetrics report | 20260508 11:11:31.151 | INFO | RTMetrics report populated | 20260508 11:11:31.151 | INFO | Opening tmp RTMetrics json file | 20260508 11:11:31.152 | INFO | Serializing RTMetrics to json | 20260508 11:11:31.154 | INFO | Serialization done, swapping tmp file now... | 20260508 11:11:31.154 | INFO | RTMetrics done | 20260508 11:11:31.216 | INFO | Finished a round of Baz IO: Wrote 1.442 GiB in 3.483 seconds | 20260508 11:11:31.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:11:31.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200089% / 0% / 9.13938e-05% | 20260508 11:11:31.216 | INFO | Baz IO overall: | 20260508 11:11:31.216 | INFO | BytesWritten : 3304147905581 | 20260508 11:11:31.216 | INFO | NumEvents : 1587260001644 | 20260508 11:11:32.766 | INFO | Analyzing chunk frames = [7647232,7647744) | 20260508 11:11:32.766 | INFO | MemInfo: { | 20260508 11:11:32.766 | INFO | "gpu" : " 17 GB", | 20260508 11:11:32.766 | INFO | "huge" : "", | 20260508 11:11:32.766 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:32.766 | INFO | "malloc" : " 14 GB", | 20260508 11:11:32.766 | INFO | "pinned" : " 29 GB" | 20260508 11:11:32.767 | INFO | } | 20260508 11:11:32.767 | INFO | Perf report for the last 5.13083 seconds: | 20260508 11:11:32.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:32.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:32.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:32.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:32.767 | INFO | ANALYSIS : 68.2%, 3.0, 105.8ms, 0 | 20260508 11:11:32.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:32.767 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:11:32.767 | INFO | PRE_HQ : 65.9%, 1.6, 54.1ms, 0 | 20260508 11:11:32.767 | INFO | BAZWRITER : 67.2%, 2.5, 85.7ms, 0 | 20260508 11:11:32.767 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 11:11:32.767 | INFO | Uploaded 36GiB to GPU in 2.55972s (14.0641GiB/s) | 20260508 11:11:32.767 | INFO | Downloaded 38GiB to GPU in 2.71642s (13.989GiB/s) | 20260508 11:11:37.913 | INFO | Analyzing chunk frames = [7647744,7648256) | 20260508 11:11:37.914 | INFO | MemInfo: { | 20260508 11:11:37.914 | INFO | "gpu" : " 17 GB", | 20260508 11:11:37.914 | INFO | "huge" : "", | 20260508 11:11:37.914 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:37.914 | INFO | "malloc" : " 14 GB", | 20260508 11:11:37.914 | INFO | "pinned" : " 29 GB" | 20260508 11:11:37.914 | INFO | } | 20260508 11:11:37.914 | INFO | Perf report for the last 5.14721 seconds: | 20260508 11:11:37.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:37.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | ANALYSIS : 66.9%, 3.0, 104.4ms, 0 | 20260508 11:11:37.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:11:37.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:37.914 | INFO | Uploaded 36GiB to GPU in 2.53203s (14.2178GiB/s) | 20260508 11:11:37.914 | INFO | Downloaded 34GiB to GPU in 2.44854s (13.8858GiB/s) | 20260508 11:11:43.051 | INFO | Analyzing chunk frames = [7648256,7648768) | 20260508 11:11:43.051 | INFO | MemInfo: { | 20260508 11:11:43.051 | INFO | "gpu" : " 17 GB", | 20260508 11:11:43.051 | INFO | "huge" : "", | 20260508 11:11:43.051 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:43.051 | INFO | "malloc" : " 14 GB", | 20260508 11:11:43.051 | INFO | "pinned" : " 29 GB" | 20260508 11:11:43.051 | INFO | } | 20260508 11:11:43.051 | INFO | Perf report for the last 5.1374 seconds: | 20260508 11:11:43.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:43.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.051 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:11:43.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.051 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:11:43.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:43.052 | INFO | Uploaded 36GiB to GPU in 2.48039s (14.5139GiB/s) | 20260508 11:11:43.052 | INFO | Downloaded 34GiB to GPU in 2.43063s (13.9881GiB/s) | 20260508 11:11:48.192 | INFO | Analyzing chunk frames = [7648768,7649280) | 20260508 11:11:48.192 | INFO | MemInfo: { | 20260508 11:11:48.192 | INFO | "gpu" : " 17 GB", | 20260508 11:11:48.192 | INFO | "huge" : "", | 20260508 11:11:48.192 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:48.192 | INFO | "malloc" : " 14 GB", | 20260508 11:11:48.192 | INFO | "pinned" : " 29 GB" | 20260508 11:11:48.192 | INFO | } | 20260508 11:11:48.193 | INFO | Perf report for the last 5.14145 seconds: | 20260508 11:11:48.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:48.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 11:11:48.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:11:48.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:48.193 | INFO | Uploaded 36GiB to GPU in 2.45672s (14.6537GiB/s) | 20260508 11:11:48.193 | INFO | Downloaded 34GiB to GPU in 2.42424s (14.025GiB/s) | 20260508 11:11:53.336 | INFO | Analyzing chunk frames = [7649280,7649792) | 20260508 11:11:53.336 | INFO | MemInfo: { | 20260508 11:11:53.336 | INFO | "gpu" : " 17 GB", | 20260508 11:11:53.336 | INFO | "huge" : "", | 20260508 11:11:53.336 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:53.336 | INFO | "malloc" : " 14 GB", | 20260508 11:11:53.336 | INFO | "pinned" : " 29 GB" | 20260508 11:11:53.336 | INFO | } | 20260508 11:11:53.336 | INFO | Perf report for the last 5.14374 seconds: | 20260508 11:11:53.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:53.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.336 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260508 11:11:53.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.337 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260508 11:11:53.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:53.337 | INFO | Uploaded 36GiB to GPU in 2.4861s (14.4805GiB/s) | 20260508 11:11:53.337 | INFO | Downloaded 34GiB to GPU in 2.53492s (13.4127GiB/s) | 20260508 11:11:58.471 | INFO | Analyzing chunk frames = [7649792,7650304) | 20260508 11:11:58.471 | INFO | MemInfo: { | 20260508 11:11:58.471 | INFO | "gpu" : " 17 GB", | 20260508 11:11:58.471 | INFO | "huge" : "", | 20260508 11:11:58.471 | INFO | "huge_shared" : " 29 GB", | 20260508 11:11:58.471 | INFO | "malloc" : " 14 GB", | 20260508 11:11:58.471 | INFO | "pinned" : " 29 GB" | 20260508 11:11:58.471 | INFO | } | 20260508 11:11:58.471 | INFO | Perf report for the last 5.13471 seconds: | 20260508 11:11:58.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:11:58.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.471 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:11:58.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.471 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:11:58.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:11:58.472 | INFO | Uploaded 36GiB to GPU in 2.39743s (15.016GiB/s) | 20260508 11:11:58.472 | INFO | Downloaded 34GiB to GPU in 2.43205s (13.98GiB/s) | 20260508 11:12:03.615 | INFO | Analyzing chunk frames = [7650304,7650816) | 20260508 11:12:03.615 | INFO | MemInfo: { | 20260508 11:12:03.615 | INFO | "gpu" : " 17 GB", | 20260508 11:12:03.615 | INFO | "huge" : "", | 20260508 11:12:03.615 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:03.615 | INFO | "malloc" : " 14 GB", | 20260508 11:12:03.615 | INFO | "pinned" : " 29 GB" | 20260508 11:12:03.615 | INFO | } | 20260508 11:12:03.615 | INFO | Perf report for the last 5.14372 seconds: | 20260508 11:12:03.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:03.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 11:12:03.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:12:03.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:03.615 | INFO | Uploaded 36GiB to GPU in 2.46468s (14.6063GiB/s) | 20260508 11:12:03.615 | INFO | Downloaded 34GiB to GPU in 2.45809s (13.8319GiB/s) | 20260508 11:12:08.751 | INFO | Analyzing chunk frames = [7650816,7651328) | 20260508 11:12:08.751 | INFO | MemInfo: { | 20260508 11:12:08.751 | INFO | "gpu" : " 17 GB", | 20260508 11:12:08.751 | INFO | "huge" : "", | 20260508 11:12:08.751 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:08.751 | INFO | "malloc" : " 14 GB", | 20260508 11:12:08.751 | INFO | "pinned" : " 29 GB" | 20260508 11:12:08.751 | INFO | } | 20260508 11:12:08.751 | INFO | Perf report for the last 5.13636 seconds: | 20260508 11:12:08.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:08.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.751 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 11:12:08.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.751 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:12:08.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:08.752 | INFO | Uploaded 36GiB to GPU in 2.45539s (14.6616GiB/s) | 20260508 11:12:08.752 | INFO | Downloaded 34GiB to GPU in 2.44891s (13.8837GiB/s) | 20260508 11:12:08.752 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:12:08.752 | INFO | Total mem usage: 16.7 GiB | 20260508 11:12:08.752 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:12:08.752 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:12:08.752 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:12:08.752 | INFO | High Water Breakdown: | 20260508 11:12:08.752 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:12:08.752 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:12:08.752 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:12:08.752 | INFO | ------------------------------------------ | 20260508 11:12:08.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:12:08.752 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:12:08.752 | INFO | Total mem usage: 26.5 GiB | 20260508 11:12:08.752 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:12:08.752 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:12:08.752 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:12:08.752 | INFO | High Water Breakdown: | 20260508 11:12:08.752 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:12:08.752 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:12:08.752 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:12:08.752 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:12:08.752 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:12:08.752 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:12:08.752 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:12:08.752 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:12:08.752 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:12:08.752 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:12:08.752 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:12:08.752 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:12:08.752 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:12:08.752 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:12:08.752 | INFO | ------------------------------------------ | 20260508 11:12:08.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:12:08.752 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:12:08.752 | INFO | Total mem usage: 27.8 GiB | 20260508 11:12:08.752 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:12:08.752 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:12:08.752 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:12:08.752 | INFO | High Water Breakdown: | 20260508 11:12:08.752 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:12:08.752 | INFO | ------------------------------------------ | 20260508 11:12:08.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:12:08.752 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:12:08.752 | INFO | Total mem usage: 15.7 GiB | 20260508 11:12:08.752 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:12:08.752 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:12:08.752 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:12:08.752 | INFO | High Water Breakdown: | 20260508 11:12:08.752 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:12:08.752 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:12:08.752 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:12:08.752 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:12:08.752 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:12:08.752 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:12:08.752 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:12:08.752 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:12:08.752 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:12:08.752 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:12:08.752 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:12:08.752 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:12:08.752 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:12:08.752 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:12:08.752 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:12:08.752 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:12:08.752 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:12:08.752 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:12:08.752 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:12:08.752 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:12:08.752 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:12:08.752 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:12:08.752 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:12:08.752 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:12:08.752 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:12:08.752 | INFO | ------------------------------------------ | 20260508 11:12:08.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:12:08.752 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:12:08.753 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:12:08.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:12:08.753 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.40e+01 / 7.30e-01 | 20260508 11:12:08.753 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.09e-02 / 6.29e-03 | 20260508 11:12:08.753 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.45e+00, 5.93e-01 / 2.14e-01 | 20260508 11:12:08.753 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.27e-01 | 20260508 11:12:08.753 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.53e+01, 3.71e+00 / 9.02e-01 | 20260508 11:12:08.753 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.79e+00 / 1.30e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.34e+00 / 2.56e+00, 4.06e-01 / 2.47e-01 | 20260508 11:12:08.753 | INFO | PerfReport: 3.01%, Metrics -- 9.81e-01 / 1.15e+00, 2.44e-01 / 1.06e-01 | 20260508 11:12:08.753 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.34e-01, 4.74e-02 / 2.73e-02 | 20260508 11:12:08.753 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:12:08.753 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:12:08.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:12:08.753 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.42e+01 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 24.29%, Upload -- 2.56e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 24.66%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 31.01%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 2.10%, LockUpload -- 2.21e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 2.65%, LockDownload -- 2.80e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 15.27%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 11:12:08.753 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 11:12:08.848 | INFO | Starting a round of Baz IO | 20260508 11:12:12.266 | INFO | Populating RTMetrics report | 20260508 11:12:12.266 | INFO | RTMetrics report populated | 20260508 11:12:12.266 | INFO | Opening tmp RTMetrics json file | 20260508 11:12:12.267 | INFO | Serializing RTMetrics to json | 20260508 11:12:12.269 | INFO | Serialization done, swapping tmp file now... | 20260508 11:12:12.269 | INFO | RTMetrics done | 20260508 11:12:12.329 | INFO | Finished a round of Baz IO: Wrote 1.441 GiB in 3.481 seconds | 20260508 11:12:12.329 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:12:12.329 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198913% / 0% / 7.94729e-05% | 20260508 11:12:12.329 | INFO | Baz IO overall: | 20260508 11:12:12.329 | INFO | BytesWritten : 3305694713285 | 20260508 11:12:12.329 | INFO | NumEvents : 1587988169979 | 20260508 11:12:13.886 | INFO | Analyzing chunk frames = [7651328,7651840) | 20260508 11:12:13.886 | INFO | MemInfo: { | 20260508 11:12:13.886 | INFO | "gpu" : " 17 GB", | 20260508 11:12:13.886 | INFO | "huge" : "", | 20260508 11:12:13.886 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:13.886 | INFO | "malloc" : " 14 GB", | 20260508 11:12:13.886 | INFO | "pinned" : " 29 GB" | 20260508 11:12:13.886 | INFO | } | 20260508 11:12:13.886 | INFO | Perf report for the last 5.13477 seconds: | 20260508 11:12:13.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:13.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:13.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:13.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:13.886 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 11:12:13.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:13.886 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:12:13.886 | INFO | PRE_HQ : 65.8%, 1.6, 54.0ms, 0 | 20260508 11:12:13.886 | INFO | BAZWRITER : 67.0%, 2.5, 86.2ms, 0 | 20260508 11:12:13.886 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 11:12:13.886 | INFO | Uploaded 36GiB to GPU in 2.56478s (14.0363GiB/s) | 20260508 11:12:13.886 | INFO | Downloaded 38GiB to GPU in 2.70031s (14.0725GiB/s) | 20260508 11:12:19.023 | INFO | Analyzing chunk frames = [7651840,7652352) | 20260508 11:12:19.023 | INFO | MemInfo: { | 20260508 11:12:19.023 | INFO | "gpu" : " 17 GB", | 20260508 11:12:19.023 | INFO | "huge" : "", | 20260508 11:12:19.023 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:19.023 | INFO | "malloc" : " 14 GB", | 20260508 11:12:19.023 | INFO | "pinned" : " 29 GB" | 20260508 11:12:19.023 | INFO | } | 20260508 11:12:19.023 | INFO | Perf report for the last 5.13683 seconds: | 20260508 11:12:19.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:19.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:12:19.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:12:19.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:19.023 | INFO | Uploaded 36GiB to GPU in 2.41695s (14.8948GiB/s) | 20260508 11:12:19.024 | INFO | Downloaded 34GiB to GPU in 2.47704s (13.7261GiB/s) | 20260508 11:12:24.167 | INFO | Analyzing chunk frames = [7652352,7652864) | 20260508 11:12:24.167 | INFO | MemInfo: { | 20260508 11:12:24.167 | INFO | "gpu" : " 17 GB", | 20260508 11:12:24.167 | INFO | "huge" : "", | 20260508 11:12:24.167 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:24.167 | INFO | "malloc" : " 14 GB", | 20260508 11:12:24.167 | INFO | "pinned" : " 29 GB" | 20260508 11:12:24.167 | INFO | } | 20260508 11:12:24.167 | INFO | Perf report for the last 5.14432 seconds: | 20260508 11:12:24.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:24.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:12:24.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:12:24.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:24.167 | INFO | Uploaded 36GiB to GPU in 2.44751s (14.7088GiB/s) | 20260508 11:12:24.168 | INFO | Downloaded 34GiB to GPU in 2.45746s (13.8354GiB/s) | 20260508 11:12:29.313 | INFO | Analyzing chunk frames = [7652864,7653376) | 20260508 11:12:29.313 | INFO | MemInfo: { | 20260508 11:12:29.313 | INFO | "gpu" : " 17 GB", | 20260508 11:12:29.313 | INFO | "huge" : "", | 20260508 11:12:29.313 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:29.313 | INFO | "malloc" : " 14 GB", | 20260508 11:12:29.313 | INFO | "pinned" : " 29 GB" | 20260508 11:12:29.313 | INFO | } | 20260508 11:12:29.313 | INFO | Perf report for the last 5.14592 seconds: | 20260508 11:12:29.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:29.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.313 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 11:12:29.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.313 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:12:29.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:29.314 | INFO | Uploaded 36GiB to GPU in 2.45379s (14.6712GiB/s) | 20260508 11:12:29.314 | INFO | Downloaded 34GiB to GPU in 2.45792s (13.8328GiB/s) | 20260508 11:12:34.450 | INFO | Analyzing chunk frames = [7653376,7653888) | 20260508 11:12:34.450 | INFO | MemInfo: { | 20260508 11:12:34.450 | INFO | "gpu" : " 17 GB", | 20260508 11:12:34.450 | INFO | "huge" : "", | 20260508 11:12:34.450 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:34.450 | INFO | "malloc" : " 14 GB", | 20260508 11:12:34.450 | INFO | "pinned" : " 29 GB" | 20260508 11:12:34.450 | INFO | } | 20260508 11:12:34.450 | INFO | Perf report for the last 5.13671 seconds: | 20260508 11:12:34.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:34.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:12:34.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:12:34.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:34.450 | INFO | Uploaded 36GiB to GPU in 2.31061s (15.5803GiB/s) | 20260508 11:12:34.450 | INFO | Downloaded 34GiB to GPU in 2.50454s (13.5754GiB/s) | 20260508 11:12:39.581 | INFO | Analyzing chunk frames = [7653888,7654400) | 20260508 11:12:39.581 | INFO | MemInfo: { | 20260508 11:12:39.581 | INFO | "gpu" : " 17 GB", | 20260508 11:12:39.581 | INFO | "huge" : "", | 20260508 11:12:39.581 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:39.581 | INFO | "malloc" : " 14 GB", | 20260508 11:12:39.581 | INFO | "pinned" : " 29 GB" | 20260508 11:12:39.581 | INFO | } | 20260508 11:12:39.581 | INFO | Perf report for the last 5.1311 seconds: | 20260508 11:12:39.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:39.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | ANALYSIS : 65.4%, 3.0, 101.5ms, 0 | 20260508 11:12:39.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:12:39.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:39.581 | INFO | Uploaded 36GiB to GPU in 2.37311s (15.17GiB/s) | 20260508 11:12:39.581 | INFO | Downloaded 34GiB to GPU in 2.45231s (13.8645GiB/s) | 20260508 11:12:44.724 | INFO | Analyzing chunk frames = [7654400,7654912) | 20260508 11:12:44.724 | INFO | MemInfo: { | 20260508 11:12:44.724 | INFO | "gpu" : " 17 GB", | 20260508 11:12:44.724 | INFO | "huge" : "", | 20260508 11:12:44.724 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:44.724 | INFO | "malloc" : " 14 GB", | 20260508 11:12:44.725 | INFO | "pinned" : " 29 GB" | 20260508 11:12:44.725 | INFO | } | 20260508 11:12:44.725 | INFO | Perf report for the last 5.14363 seconds: | 20260508 11:12:44.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:44.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 11:12:44.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:12:44.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:44.725 | INFO | Uploaded 36GiB to GPU in 2.37497s (15.1581GiB/s) | 20260508 11:12:44.725 | INFO | Downloaded 34GiB to GPU in 2.45778s (13.8336GiB/s) | 20260508 11:12:49.863 | INFO | Analyzing chunk frames = [7654912,7655424) | 20260508 11:12:49.863 | INFO | MemInfo: { | 20260508 11:12:49.863 | INFO | "gpu" : " 17 GB", | 20260508 11:12:49.863 | INFO | "huge" : "", | 20260508 11:12:49.863 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:49.863 | INFO | "malloc" : " 14 GB", | 20260508 11:12:49.863 | INFO | "pinned" : " 29 GB" | 20260508 11:12:49.863 | INFO | } | 20260508 11:12:49.863 | INFO | Perf report for the last 5.13879 seconds: | 20260508 11:12:49.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:49.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.863 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 11:12:49.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.864 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:12:49.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:49.864 | INFO | Uploaded 36GiB to GPU in 2.40094s (14.9942GiB/s) | 20260508 11:12:49.864 | INFO | Downloaded 34GiB to GPU in 2.4624s (13.8077GiB/s) | 20260508 11:12:49.961 | INFO | Starting a round of Baz IO | 20260508 11:12:53.377 | INFO | Populating RTMetrics report | 20260508 11:12:53.377 | INFO | RTMetrics report populated | 20260508 11:12:53.377 | INFO | Opening tmp RTMetrics json file | 20260508 11:12:53.387 | INFO | Serializing RTMetrics to json | 20260508 11:12:53.388 | INFO | Serialization done, swapping tmp file now... | 20260508 11:12:53.389 | INFO | RTMetrics done | 20260508 11:12:53.442 | INFO | Finished a round of Baz IO: Wrote 1.44 GiB in 3.48 seconds | 20260508 11:12:53.442 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:12:53.442 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198146% / 3.97364e-06% / 7.94729e-05% | 20260508 11:12:53.442 | INFO | Baz IO overall: | 20260508 11:12:53.442 | INFO | BytesWritten : 3307241191413 | 20260508 11:12:53.442 | INFO | NumEvents : 1588716349697 | 20260508 11:12:55.009 | INFO | Analyzing chunk frames = [7655424,7655936) | 20260508 11:12:55.009 | INFO | MemInfo: { | 20260508 11:12:55.009 | INFO | "gpu" : " 17 GB", | 20260508 11:12:55.010 | INFO | "huge" : "", | 20260508 11:12:55.010 | INFO | "huge_shared" : " 29 GB", | 20260508 11:12:55.010 | INFO | "malloc" : " 14 GB", | 20260508 11:12:55.010 | INFO | "pinned" : " 29 GB" | 20260508 11:12:55.010 | INFO | } | 20260508 11:12:55.010 | INFO | Perf report for the last 5.14618 seconds: | 20260508 11:12:55.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:12:55.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:55.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:55.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:55.010 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 11:12:55.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:12:55.010 | INFO | RT_METRICS : 38.6%, 1.0, 20.3ms, 0 | 20260508 11:12:55.010 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 11:12:55.010 | INFO | BAZWRITER : 67.0%, 2.5, 85.4ms, 0 | 20260508 11:12:55.010 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 11:12:55.010 | INFO | Uploaded 36GiB to GPU in 2.55305s (14.1008GiB/s) | 20260508 11:12:55.010 | INFO | Downloaded 38GiB to GPU in 2.7263s (13.9383GiB/s) | 20260508 11:13:00.144 | INFO | Analyzing chunk frames = [7655936,7656448) | 20260508 11:13:00.144 | INFO | MemInfo: { | 20260508 11:13:00.144 | INFO | "gpu" : " 17 GB", | 20260508 11:13:00.144 | INFO | "huge" : "", | 20260508 11:13:00.144 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:00.144 | INFO | "malloc" : " 14 GB", | 20260508 11:13:00.144 | INFO | "pinned" : " 29 GB" | 20260508 11:13:00.144 | INFO | } | 20260508 11:13:00.145 | INFO | Perf report for the last 5.1349 seconds: | 20260508 11:13:00.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:00.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:13:00.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260508 11:13:00.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:00.145 | INFO | Uploaded 36GiB to GPU in 2.45388s (14.6706GiB/s) | 20260508 11:13:00.145 | INFO | Downloaded 34GiB to GPU in 2.4836s (13.6898GiB/s) | 20260508 11:13:05.278 | INFO | Analyzing chunk frames = [7656448,7656960) | 20260508 11:13:05.278 | INFO | MemInfo: { | 20260508 11:13:05.278 | INFO | "gpu" : " 17 GB", | 20260508 11:13:05.278 | INFO | "huge" : "", | 20260508 11:13:05.279 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:05.279 | INFO | "malloc" : " 14 GB", | 20260508 11:13:05.279 | INFO | "pinned" : " 29 GB" | 20260508 11:13:05.279 | INFO | } | 20260508 11:13:05.279 | INFO | Perf report for the last 5.13416 seconds: | 20260508 11:13:05.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:05.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:13:05.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:13:05.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:05.279 | INFO | Uploaded 36GiB to GPU in 2.44615s (14.717GiB/s) | 20260508 11:13:05.279 | INFO | Downloaded 34GiB to GPU in 2.44709s (13.894GiB/s) | 20260508 11:13:10.425 | INFO | Analyzing chunk frames = [7656960,7657472) | 20260508 11:13:10.425 | INFO | MemInfo: { | 20260508 11:13:10.425 | INFO | "gpu" : " 17 GB", | 20260508 11:13:10.425 | INFO | "huge" : "", | 20260508 11:13:10.425 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:10.425 | INFO | "malloc" : " 14 GB", | 20260508 11:13:10.425 | INFO | "pinned" : " 29 GB" | 20260508 11:13:10.425 | INFO | } | 20260508 11:13:10.425 | INFO | Perf report for the last 5.14676 seconds: | 20260508 11:13:10.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:10.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 11:13:10.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:13:10.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:10.426 | INFO | Uploaded 36GiB to GPU in 2.45363s (14.6722GiB/s) | 20260508 11:13:10.426 | INFO | Downloaded 34GiB to GPU in 2.45043s (13.8751GiB/s) | 20260508 11:13:15.560 | INFO | Analyzing chunk frames = [7657472,7657984) | 20260508 11:13:15.560 | INFO | MemInfo: { | 20260508 11:13:15.560 | INFO | "gpu" : " 17 GB", | 20260508 11:13:15.560 | INFO | "huge" : "", | 20260508 11:13:15.560 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:15.560 | INFO | "malloc" : " 14 GB", | 20260508 11:13:15.560 | INFO | "pinned" : " 29 GB" | 20260508 11:13:15.560 | INFO | } | 20260508 11:13:15.560 | INFO | Perf report for the last 5.13485 seconds: | 20260508 11:13:15.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:15.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.560 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 11:13:15.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.560 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 11:13:15.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:15.561 | INFO | Uploaded 36GiB to GPU in 2.39199s (15.0502GiB/s) | 20260508 11:13:15.561 | INFO | Downloaded 34GiB to GPU in 2.55185s (13.3236GiB/s) | 20260508 11:13:20.701 | INFO | Analyzing chunk frames = [7657984,7658496) | 20260508 11:13:20.701 | INFO | MemInfo: { | 20260508 11:13:20.701 | INFO | "gpu" : " 17 GB", | 20260508 11:13:20.701 | INFO | "huge" : "", | 20260508 11:13:20.701 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:20.701 | INFO | "malloc" : " 14 GB", | 20260508 11:13:20.701 | INFO | "pinned" : " 29 GB" | 20260508 11:13:20.701 | INFO | } | 20260508 11:13:20.701 | INFO | Perf report for the last 5.14061 seconds: | 20260508 11:13:20.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:20.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:13:20.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:13:20.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:20.701 | INFO | Uploaded 36GiB to GPU in 2.36674s (15.2108GiB/s) | 20260508 11:13:20.701 | INFO | Downloaded 34GiB to GPU in 2.43517s (13.962GiB/s) | 20260508 11:13:25.839 | INFO | Analyzing chunk frames = [7658496,7659008) | 20260508 11:13:25.839 | INFO | MemInfo: { | 20260508 11:13:25.839 | INFO | "gpu" : " 17 GB", | 20260508 11:13:25.839 | INFO | "huge" : "", | 20260508 11:13:25.839 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:25.839 | INFO | "malloc" : " 14 GB", | 20260508 11:13:25.839 | INFO | "pinned" : " 29 GB" | 20260508 11:13:25.839 | INFO | } | 20260508 11:13:25.839 | INFO | Perf report for the last 5.13791 seconds: | 20260508 11:13:25.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:25.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 11:13:25.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:13:25.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:25.839 | INFO | Uploaded 36GiB to GPU in 2.38231s (15.1114GiB/s) | 20260508 11:13:25.839 | INFO | Downloaded 34GiB to GPU in 2.4322s (13.9791GiB/s) | 20260508 11:13:30.980 | INFO | Analyzing chunk frames = [7659008,7659520) | 20260508 11:13:30.980 | INFO | MemInfo: { | 20260508 11:13:30.980 | INFO | "gpu" : " 17 GB", | 20260508 11:13:30.980 | INFO | "huge" : "", | 20260508 11:13:30.981 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:30.981 | INFO | "malloc" : " 14 GB", | 20260508 11:13:30.981 | INFO | "pinned" : " 29 GB" | 20260508 11:13:30.981 | INFO | } | 20260508 11:13:30.981 | INFO | Perf report for the last 5.14171 seconds: | 20260508 11:13:30.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:30.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 11:13:30.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:13:30.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:30.981 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:13:30.981 | INFO | Total mem usage: 16.7 GiB | 20260508 11:13:30.981 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:13:30.981 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:13:30.981 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:13:30.981 | INFO | High Water Breakdown: | 20260508 11:13:30.981 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:13:30.981 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:13:30.981 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:13:30.981 | INFO | ------------------------------------------ | 20260508 11:13:30.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:13:30.981 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:13:30.981 | INFO | Total mem usage: 26.5 GiB | 20260508 11:13:30.981 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:13:30.981 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:13:30.981 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:13:30.981 | INFO | High Water Breakdown: | 20260508 11:13:30.981 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:13:30.981 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:13:30.981 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:13:30.981 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:13:30.981 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:13:30.981 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:13:30.981 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:13:30.981 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:13:30.981 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:13:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:13:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:13:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:13:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:13:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:13:30.981 | INFO | ------------------------------------------ | 20260508 11:13:30.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:13:30.981 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:13:30.981 | INFO | Total mem usage: 27.8 GiB | 20260508 11:13:30.981 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:13:30.981 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:13:30.981 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:13:30.981 | INFO | High Water Breakdown: | 20260508 11:13:30.981 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:13:30.981 | INFO | ------------------------------------------ | 20260508 11:13:30.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:13:30.981 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:13:30.981 | INFO | Total mem usage: 15.7 GiB | 20260508 11:13:30.981 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:13:30.981 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:13:30.982 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:13:30.981 | INFO | Uploaded 36GiB to GPU in 2.42092s (14.8704GiB/s) | 20260508 11:13:30.982 | INFO | High Water Breakdown: | 20260508 11:13:30.982 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:13:30.982 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:13:30.982 | INFO | Downloaded 34GiB to GPU in 2.44378s (13.9129GiB/s) | 20260508 11:13:30.982 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:13:30.982 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:13:30.982 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:13:30.982 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:13:30.982 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:13:30.982 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:13:30.982 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:13:30.982 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:13:30.982 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:13:30.982 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:13:30.982 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:13:30.982 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:13:30.982 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:13:30.982 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:13:30.982 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:13:30.982 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:13:30.982 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:13:30.982 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:13:30.982 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:13:30.982 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:13:30.982 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:13:30.982 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:13:30.982 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:13:30.982 | INFO | ------------------------------------------ | 20260508 11:13:30.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:13:30.982 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:13:30.982 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:13:30.982 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:13:30.982 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 6.56e-01 | 20260508 11:13:30.982 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.12e-02 / 4.13e-03 | 20260508 11:13:30.982 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.52e+00, 5.97e-01 / 1.70e-01 | 20260508 11:13:30.982 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.51e+00, 2.10e-01 / 8.94e-02 | 20260508 11:13:30.982 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.51e+01, 3.69e+00 / 1.04e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.79e+00 / 1.28e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.54e+00, 4.05e-01 / 2.29e-01 | 20260508 11:13:30.982 | INFO | PerfReport: 3.01%, Metrics -- 9.81e-01 / 1.19e+00, 2.42e-01 / 1.10e-01 | 20260508 11:13:30.982 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.33e-01, 4.73e-02 / 2.54e-02 | 20260508 11:13:30.982 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:13:30.982 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:13:30.982 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:13:30.982 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 24.06%, Upload -- 2.53e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 31.10%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 1.99%, LockUpload -- 2.09e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 2.79%, LockDownload -- 2.93e+00 / 0.00e+00, 4.48e+00 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 15.28%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:13:30.982 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 9.77e-03 / 0.00e+00 | 20260508 11:13:31.079 | INFO | Starting a round of Baz IO | 20260508 11:13:34.473 | INFO | Populating RTMetrics report | 20260508 11:13:34.473 | INFO | RTMetrics report populated | 20260508 11:13:34.473 | INFO | Opening tmp RTMetrics json file | 20260508 11:13:34.490 | INFO | Serializing RTMetrics to json | 20260508 11:13:34.493 | INFO | Serialization done, swapping tmp file now... | 20260508 11:13:34.494 | INFO | RTMetrics done | 20260508 11:13:34.553 | INFO | Finished a round of Baz IO: Wrote 1.441 GiB in 3.473 seconds | 20260508 11:13:34.553 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.13% / 0.00% / 59.87% | 20260508 11:13:34.553 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198166% / 3.97364e-06% / 5.16574e-05% | 20260508 11:13:34.553 | INFO | Baz IO overall: | 20260508 11:13:34.553 | INFO | BytesWritten : 3308788192409 | 20260508 11:13:34.553 | INFO | NumEvents : 1589444668114 | 20260508 11:13:36.122 | INFO | Analyzing chunk frames = [7659520,7660032) | 20260508 11:13:36.122 | INFO | MemInfo: { | 20260508 11:13:36.122 | INFO | "gpu" : " 17 GB", | 20260508 11:13:36.122 | INFO | "huge" : "", | 20260508 11:13:36.122 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:36.122 | INFO | "malloc" : " 14 GB", | 20260508 11:13:36.122 | INFO | "pinned" : " 29 GB" | 20260508 11:13:36.122 | INFO | } | 20260508 11:13:36.122 | INFO | Perf report for the last 5.1411 seconds: | 20260508 11:13:36.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:36.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:36.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:36.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:36.122 | INFO | ANALYSIS : 67.3%, 3.0, 104.6ms, 0 | 20260508 11:13:36.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:36.122 | INFO | RT_METRICS : 40.4%, 1.0, 21.2ms, 0 | 20260508 11:13:36.122 | INFO | PRE_HQ : 65.5%, 1.7, 57.0ms, 0 | 20260508 11:13:36.122 | INFO | BAZWRITER : 67.1%, 3.1, 105.4ms, 0 | 20260508 11:13:36.122 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:13:36.122 | INFO | Uploaded 36GiB to GPU in 2.61457s (13.769GiB/s) | 20260508 11:13:36.122 | INFO | Downloaded 38GiB to GPU in 2.71914s (13.975GiB/s) | 20260508 11:13:41.259 | INFO | Analyzing chunk frames = [7660032,7660544) | 20260508 11:13:41.259 | INFO | MemInfo: { | 20260508 11:13:41.259 | INFO | "gpu" : " 17 GB", | 20260508 11:13:41.259 | INFO | "huge" : "", | 20260508 11:13:41.259 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:41.259 | INFO | "malloc" : " 14 GB", | 20260508 11:13:41.259 | INFO | "pinned" : " 29 GB" | 20260508 11:13:41.259 | INFO | } | 20260508 11:13:41.260 | INFO | Perf report for the last 5.13777 seconds: | 20260508 11:13:41.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:41.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:13:41.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:13:41.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:41.260 | INFO | Uploaded 36GiB to GPU in 2.43987s (14.7549GiB/s) | 20260508 11:13:41.260 | INFO | Downloaded 34GiB to GPU in 2.46897s (13.771GiB/s) | 20260508 11:13:46.396 | INFO | Analyzing chunk frames = [7660544,7661056) | 20260508 11:13:46.396 | INFO | MemInfo: { | 20260508 11:13:46.396 | INFO | "gpu" : " 17 GB", | 20260508 11:13:46.396 | INFO | "huge" : "", | 20260508 11:13:46.396 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:46.396 | INFO | "malloc" : " 14 GB", | 20260508 11:13:46.396 | INFO | "pinned" : " 29 GB" | 20260508 11:13:46.396 | INFO | } | 20260508 11:13:46.397 | INFO | Perf report for the last 5.13718 seconds: | 20260508 11:13:46.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:46.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 11:13:46.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:13:46.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:46.397 | INFO | Uploaded 36GiB to GPU in 2.46839s (14.5844GiB/s) | 20260508 11:13:46.397 | INFO | Downloaded 34GiB to GPU in 2.44539s (13.9037GiB/s) | 20260508 11:13:51.534 | INFO | Analyzing chunk frames = [7661056,7661568) | 20260508 11:13:51.534 | INFO | MemInfo: { | 20260508 11:13:51.534 | INFO | "gpu" : " 17 GB", | 20260508 11:13:51.534 | INFO | "huge" : "", | 20260508 11:13:51.534 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:51.534 | INFO | "malloc" : " 14 GB", | 20260508 11:13:51.534 | INFO | "pinned" : " 29 GB" | 20260508 11:13:51.534 | INFO | } | 20260508 11:13:51.534 | INFO | Perf report for the last 5.13717 seconds: | 20260508 11:13:51.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:51.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.534 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:13:51.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.534 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:13:51.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:51.535 | INFO | Uploaded 36GiB to GPU in 2.45391s (14.6705GiB/s) | 20260508 11:13:51.535 | INFO | Downloaded 34GiB to GPU in 2.4271s (14.0085GiB/s) | 20260508 11:13:56.673 | INFO | Analyzing chunk frames = [7661568,7662080) | 20260508 11:13:56.673 | INFO | MemInfo: { | 20260508 11:13:56.673 | INFO | "gpu" : " 17 GB", | 20260508 11:13:56.673 | INFO | "huge" : "", | 20260508 11:13:56.673 | INFO | "huge_shared" : " 29 GB", | 20260508 11:13:56.673 | INFO | "malloc" : " 14 GB", | 20260508 11:13:56.673 | INFO | "pinned" : " 29 GB" | 20260508 11:13:56.673 | INFO | } | 20260508 11:13:56.673 | INFO | Perf report for the last 5.13923 seconds: | 20260508 11:13:56.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:13:56.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 11:13:56.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:13:56.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:13:56.673 | INFO | Uploaded 36GiB to GPU in 2.45s (14.6939GiB/s) | 20260508 11:13:56.673 | INFO | Downloaded 34GiB to GPU in 2.54702s (13.3489GiB/s) | 20260508 11:14:01.817 | INFO | Analyzing chunk frames = [7662080,7662592) | 20260508 11:14:01.817 | INFO | MemInfo: { | 20260508 11:14:01.817 | INFO | "gpu" : " 17 GB", | 20260508 11:14:01.817 | INFO | "huge" : "", | 20260508 11:14:01.817 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:01.817 | INFO | "malloc" : " 14 GB", | 20260508 11:14:01.817 | INFO | "pinned" : " 29 GB" | 20260508 11:14:01.817 | INFO | } | 20260508 11:14:01.817 | INFO | Perf report for the last 5.14422 seconds: | 20260508 11:14:01.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:01.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.817 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 11:14:01.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.817 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:14:01.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:01.818 | INFO | Uploaded 36GiB to GPU in 2.33574s (15.4127GiB/s) | 20260508 11:14:01.818 | INFO | Downloaded 34GiB to GPU in 2.4436s (13.9139GiB/s) | 20260508 11:14:06.959 | INFO | Analyzing chunk frames = [7662592,7663104) | 20260508 11:14:06.959 | INFO | MemInfo: { | 20260508 11:14:06.959 | INFO | "gpu" : " 17 GB", | 20260508 11:14:06.959 | INFO | "huge" : "", | 20260508 11:14:06.959 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:06.959 | INFO | "malloc" : " 14 GB", | 20260508 11:14:06.960 | INFO | "pinned" : " 29 GB" | 20260508 11:14:06.960 | INFO | } | 20260508 11:14:06.960 | INFO | Perf report for the last 5.14232 seconds: | 20260508 11:14:06.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:06.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 11:14:06.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 11:14:06.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:06.960 | INFO | Uploaded 36GiB to GPU in 2.3979s (15.0131GiB/s) | 20260508 11:14:06.960 | INFO | Downloaded 34GiB to GPU in 2.45684s (13.8389GiB/s) | 20260508 11:14:12.092 | INFO | Analyzing chunk frames = [7663104,7663616) | 20260508 11:14:12.092 | INFO | MemInfo: { | 20260508 11:14:12.092 | INFO | "gpu" : " 17 GB", | 20260508 11:14:12.092 | INFO | "huge" : "", | 20260508 11:14:12.092 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:12.092 | INFO | "malloc" : " 14 GB", | 20260508 11:14:12.092 | INFO | "pinned" : " 29 GB" | 20260508 11:14:12.093 | INFO | } | 20260508 11:14:12.093 | INFO | Perf report for the last 5.13285 seconds: | 20260508 11:14:12.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:12.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:14:12.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:14:12.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:12.093 | INFO | Uploaded 36GiB to GPU in 2.45432s (14.668GiB/s) | 20260508 11:14:12.093 | INFO | Downloaded 34GiB to GPU in 2.47358s (13.7453GiB/s) | 20260508 11:14:12.190 | INFO | Starting a round of Baz IO | 20260508 11:14:15.600 | INFO | Populating RTMetrics report | 20260508 11:14:15.600 | INFO | RTMetrics report populated | 20260508 11:14:15.600 | INFO | Opening tmp RTMetrics json file | 20260508 11:14:15.611 | INFO | Serializing RTMetrics to json | 20260508 11:14:15.612 | INFO | Serialization done, swapping tmp file now... | 20260508 11:14:15.613 | INFO | RTMetrics done | 20260508 11:14:15.667 | INFO | Finished a round of Baz IO: Wrote 1.44 GiB in 3.477 seconds | 20260508 11:14:15.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.87% | 20260508 11:14:15.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.19834% / 0% / 5.5631e-05% | 20260508 11:14:15.668 | INFO | Baz IO overall: | 20260508 11:14:15.668 | INFO | BytesWritten : 3310334439254 | 20260508 11:14:15.668 | INFO | NumEvents : 1590172437615 | 20260508 11:14:17.234 | INFO | Analyzing chunk frames = [7663616,7664128) | 20260508 11:14:17.234 | INFO | MemInfo: { | 20260508 11:14:17.234 | INFO | "gpu" : " 17 GB", | 20260508 11:14:17.234 | INFO | "huge" : "", | 20260508 11:14:17.234 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:17.234 | INFO | "malloc" : " 14 GB", | 20260508 11:14:17.234 | INFO | "pinned" : " 29 GB" | 20260508 11:14:17.234 | INFO | } | 20260508 11:14:17.234 | INFO | Perf report for the last 5.14143 seconds: | 20260508 11:14:17.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:17.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:17.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:17.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:17.234 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 11:14:17.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:17.234 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 11:14:17.234 | INFO | PRE_HQ : 65.6%, 1.6, 54.7ms, 0 | 20260508 11:14:17.234 | INFO | BAZWRITER : 66.8%, 2.5, 86.0ms, 0 | 20260508 11:14:17.234 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:14:17.235 | INFO | Uploaded 36GiB to GPU in 2.59279s (13.8846GiB/s) | 20260508 11:14:17.235 | INFO | Downloaded 38GiB to GPU in 2.72695s (13.935GiB/s) | 20260508 11:14:22.373 | INFO | Analyzing chunk frames = [7664128,7664640) | 20260508 11:14:22.373 | INFO | MemInfo: { | 20260508 11:14:22.373 | INFO | "gpu" : " 17 GB", | 20260508 11:14:22.373 | INFO | "huge" : "", | 20260508 11:14:22.373 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:22.373 | INFO | "malloc" : " 14 GB", | 20260508 11:14:22.373 | INFO | "pinned" : " 29 GB" | 20260508 11:14:22.373 | INFO | } | 20260508 11:14:22.373 | INFO | Perf report for the last 5.13929 seconds: | 20260508 11:14:22.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:22.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.373 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:14:22.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.373 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:14:22.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:22.374 | INFO | Uploaded 36GiB to GPU in 2.42617s (14.8382GiB/s) | 20260508 11:14:22.374 | INFO | Downloaded 34GiB to GPU in 2.47811s (13.7201GiB/s) | 20260508 11:14:27.511 | INFO | Analyzing chunk frames = [7664640,7665152) | 20260508 11:14:27.511 | INFO | MemInfo: { | 20260508 11:14:27.511 | INFO | "gpu" : " 17 GB", | 20260508 11:14:27.511 | INFO | "huge" : "", | 20260508 11:14:27.511 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:27.511 | INFO | "malloc" : " 14 GB", | 20260508 11:14:27.511 | INFO | "pinned" : " 29 GB" | 20260508 11:14:27.511 | INFO | } | 20260508 11:14:27.511 | INFO | Perf report for the last 5.13812 seconds: | 20260508 11:14:27.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:27.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.511 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:14:27.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.512 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:14:27.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:27.512 | INFO | Uploaded 36GiB to GPU in 2.45276s (14.6773GiB/s) | 20260508 11:14:27.512 | INFO | Downloaded 34GiB to GPU in 2.45503s (13.8491GiB/s) | 20260508 11:14:32.654 | INFO | Analyzing chunk frames = [7665152,7665664) | 20260508 11:14:32.654 | INFO | MemInfo: { | 20260508 11:14:32.654 | INFO | "gpu" : " 17 GB", | 20260508 11:14:32.654 | INFO | "huge" : "", | 20260508 11:14:32.655 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:32.655 | INFO | "malloc" : " 14 GB", | 20260508 11:14:32.655 | INFO | "pinned" : " 29 GB" | 20260508 11:14:32.655 | INFO | } | 20260508 11:14:32.655 | INFO | Perf report for the last 5.14335 seconds: | 20260508 11:14:32.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:32.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 11:14:32.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:14:32.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:32.655 | INFO | Uploaded 36GiB to GPU in 2.44413s (14.7292GiB/s) | 20260508 11:14:32.655 | INFO | Downloaded 34GiB to GPU in 2.45386s (13.8557GiB/s) | 20260508 11:14:37.786 | INFO | Analyzing chunk frames = [7665664,7666176) | 20260508 11:14:37.786 | INFO | MemInfo: { | 20260508 11:14:37.786 | INFO | "gpu" : " 17 GB", | 20260508 11:14:37.786 | INFO | "huge" : "", | 20260508 11:14:37.786 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:37.786 | INFO | "malloc" : " 14 GB", | 20260508 11:14:37.786 | INFO | "pinned" : " 29 GB" | 20260508 11:14:37.786 | INFO | } | 20260508 11:14:37.786 | INFO | Perf report for the last 5.1314 seconds: | 20260508 11:14:37.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:37.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:14:37.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 11:14:37.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:37.786 | INFO | Uploaded 36GiB to GPU in 2.27924s (15.7948GiB/s) | 20260508 11:14:37.787 | INFO | Downloaded 34GiB to GPU in 2.50598s (13.5676GiB/s) | 20260508 11:14:42.934 | INFO | Analyzing chunk frames = [7666176,7666688) | 20260508 11:14:42.934 | INFO | MemInfo: { | 20260508 11:14:42.934 | INFO | "gpu" : " 17 GB", | 20260508 11:14:42.934 | INFO | "huge" : "", | 20260508 11:14:42.934 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:42.934 | INFO | "malloc" : " 14 GB", | 20260508 11:14:42.934 | INFO | "pinned" : " 29 GB" | 20260508 11:14:42.934 | INFO | } | 20260508 11:14:42.934 | INFO | Perf report for the last 5.1478 seconds: | 20260508 11:14:42.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:42.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.934 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 11:14:42.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.934 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 11:14:42.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:42.935 | INFO | Uploaded 36GiB to GPU in 2.36079s (15.2491GiB/s) | 20260508 11:14:42.935 | INFO | Downloaded 34GiB to GPU in 2.43322s (13.9732GiB/s) | 20260508 11:14:48.071 | INFO | Analyzing chunk frames = [7666688,7667200) | 20260508 11:14:48.071 | INFO | MemInfo: { | 20260508 11:14:48.071 | INFO | "gpu" : " 17 GB", | 20260508 11:14:48.071 | INFO | "huge" : "", | 20260508 11:14:48.071 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:48.072 | INFO | "malloc" : " 14 GB", | 20260508 11:14:48.072 | INFO | "pinned" : " 29 GB" | 20260508 11:14:48.072 | INFO | } | 20260508 11:14:48.072 | INFO | Perf report for the last 5.13767 seconds: | 20260508 11:14:48.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:48.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.072 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 11:14:48.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.072 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 11:14:48.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:48.073 | INFO | Uploaded 36GiB to GPU in 2.44534s (14.7219GiB/s) | 20260508 11:14:48.073 | INFO | Downloaded 34GiB to GPU in 2.45956s (13.8236GiB/s) | 20260508 11:14:53.207 | INFO | Analyzing chunk frames = [7667200,7667712) | 20260508 11:14:53.207 | INFO | MemInfo: { | 20260508 11:14:53.207 | INFO | "gpu" : " 17 GB", | 20260508 11:14:53.207 | INFO | "huge" : "", | 20260508 11:14:53.207 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:53.207 | INFO | "malloc" : " 14 GB", | 20260508 11:14:53.207 | INFO | "pinned" : " 29 GB" | 20260508 11:14:53.207 | INFO | } | 20260508 11:14:53.207 | INFO | Perf report for the last 5.1357 seconds: | 20260508 11:14:53.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:53.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.207 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 11:14:53.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.208 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:14:53.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:53.208 | INFO | Uploaded 36GiB to GPU in 2.45224s (14.6805GiB/s) | 20260508 11:14:53.208 | INFO | Downloaded 34GiB to GPU in 2.4589s (13.8273GiB/s) | 20260508 11:14:53.208 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:14:53.208 | INFO | Total mem usage: 16.7 GiB | 20260508 11:14:53.208 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:14:53.208 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -11.4%) | 20260508 11:14:53.208 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:14:53.208 | INFO | High Water Breakdown: | 20260508 11:14:53.208 | INFO | 60.4% : 9.05 GiB ( 87.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:14:53.208 | INFO | 39.4% : 5.91 GiB ( 87.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:14:53.208 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:14:53.208 | INFO | ------------------------------------------ | 20260508 11:14:53.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:14:53.208 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:14:53.208 | INFO | Total mem usage: 26.5 GiB | 20260508 11:14:53.208 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:14:53.208 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:14:53.208 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:14:53.208 | INFO | High Water Breakdown: | 20260508 11:14:53.208 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:14:53.208 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:14:53.208 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:14:53.208 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:14:53.208 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:14:53.208 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:14:53.208 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:14:53.208 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:14:53.208 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:14:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:14:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:14:53.209 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:14:53.209 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:14:53.209 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:14:53.209 | INFO | ------------------------------------------ | 20260508 11:14:53.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:14:53.209 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:14:53.209 | INFO | Total mem usage: 27.8 GiB | 20260508 11:14:53.209 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:14:53.209 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:14:53.209 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:14:53.209 | INFO | High Water Breakdown: | 20260508 11:14:53.209 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:14:53.209 | INFO | ------------------------------------------ | 20260508 11:14:53.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:14:53.209 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:14:53.209 | INFO | Total mem usage: 15.7 GiB | 20260508 11:14:53.209 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:14:53.209 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:14:53.209 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:14:53.209 | INFO | High Water Breakdown: | 20260508 11:14:53.209 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:14:53.209 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:14:53.209 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:14:53.209 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:14:53.209 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:14:53.209 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:14:53.209 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:14:53.209 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:14:53.209 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:14:53.209 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:14:53.209 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:14:53.209 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:14:53.209 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:14:53.209 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:14:53.209 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:14:53.209 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:14:53.209 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:14:53.209 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:14:53.209 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:14:53.209 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:14:53.209 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:14:53.209 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:14:53.209 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:14:53.209 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:14:53.209 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:14:53.209 | INFO | ------------------------------------------ | 20260508 11:14:53.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:14:53.209 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:14:53.209 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:14:53.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:14:53.209 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.40e+01 / 5.56e-01 | 20260508 11:14:53.209 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.03e-01, 1.18e-02 / 6.82e-03 | 20260508 11:14:53.209 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.42e+00, 5.91e-01 / 2.18e-01 | 20260508 11:14:53.209 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.26e-01 | 20260508 11:14:53.209 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.51e+01, 3.74e+00 / 9.14e-01 | 20260508 11:14:53.209 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.76e+00 / 1.14e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.35e+00 / 2.51e+00, 4.05e-01 / 2.19e-01 | 20260508 11:14:53.209 | INFO | PerfReport: 3.01%, Metrics -- 9.83e-01 / 1.10e+00, 2.42e-01 / 9.84e-02 | 20260508 11:14:53.209 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.29e-01, 4.80e-02 / 2.59e-02 | 20260508 11:14:53.209 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:14:53.209 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:14:53.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:14:53.209 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 24.21%, Upload -- 2.54e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 31.07%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 2.02%, LockUpload -- 2.12e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 2.70%, LockDownload -- 2.84e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 15.22%, LockCompute -- 1.60e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260508 11:14:53.209 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.91e-02 / 0.00e+00 | 20260508 11:14:53.312 | INFO | Starting a round of Baz IO | 20260508 11:14:56.728 | INFO | Populating RTMetrics report | 20260508 11:14:56.728 | INFO | RTMetrics report populated | 20260508 11:14:56.728 | INFO | Opening tmp RTMetrics json file | 20260508 11:14:56.729 | INFO | Serializing RTMetrics to json | 20260508 11:14:56.730 | INFO | Serialization done, swapping tmp file now... | 20260508 11:14:56.731 | INFO | RTMetrics done | 20260508 11:14:56.803 | INFO | Finished a round of Baz IO: Wrote 1.439 GiB in 3.491 seconds | 20260508 11:14:56.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:14:56.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.197319% / 3.97364e-06% / 7.15256e-05% | 20260508 11:14:56.803 | INFO | Baz IO overall: | 20260508 11:14:56.803 | INFO | BytesWritten : 3311879597185 | 20260508 11:14:56.803 | INFO | NumEvents : 1590899456005 | 20260508 11:14:58.351 | INFO | Analyzing chunk frames = [7667712,7668224) | 20260508 11:14:58.351 | INFO | MemInfo: { | 20260508 11:14:58.351 | INFO | "gpu" : " 17 GB", | 20260508 11:14:58.351 | INFO | "huge" : "", | 20260508 11:14:58.351 | INFO | "huge_shared" : " 29 GB", | 20260508 11:14:58.351 | INFO | "malloc" : " 14 GB", | 20260508 11:14:58.351 | INFO | "pinned" : " 29 GB" | 20260508 11:14:58.351 | INFO | } | 20260508 11:14:58.351 | INFO | Perf report for the last 5.14394 seconds: | 20260508 11:14:58.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:14:58.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:58.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:58.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:58.352 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 11:14:58.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:14:58.352 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 11:14:58.352 | INFO | PRE_HQ : 65.8%, 1.6, 54.0ms, 0 | 20260508 11:14:58.352 | INFO | BAZWRITER : 67.4%, 2.5, 86.9ms, 0 | 20260508 11:14:58.352 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:14:58.352 | INFO | Uploaded 36GiB to GPU in 2.55284s (14.1019GiB/s) | 20260508 11:14:58.352 | INFO | Downloaded 38GiB to GPU in 2.7078s (14.0335GiB/s) | 20260508 11:15:03.486 | INFO | Analyzing chunk frames = [7668224,7668736) | 20260508 11:15:03.487 | INFO | MemInfo: { | 20260508 11:15:03.487 | INFO | "gpu" : " 17 GB", | 20260508 11:15:03.487 | INFO | "huge" : "", | 20260508 11:15:03.487 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:03.487 | INFO | "malloc" : " 14 GB", | 20260508 11:15:03.487 | INFO | "pinned" : " 29 GB" | 20260508 11:15:03.487 | INFO | } | 20260508 11:15:03.487 | INFO | Perf report for the last 5.13535 seconds: | 20260508 11:15:03.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:03.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:15:03.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:15:03.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:03.487 | INFO | Uploaded 36GiB to GPU in 2.4621s (14.6217GiB/s) | 20260508 11:15:03.487 | INFO | Downloaded 34GiB to GPU in 2.45688s (13.8387GiB/s) | 20260508 11:15:08.624 | INFO | Analyzing chunk frames = [7668736,7669248) | 20260508 11:15:08.624 | INFO | MemInfo: { | 20260508 11:15:08.624 | INFO | "gpu" : " 17 GB", | 20260508 11:15:08.624 | INFO | "huge" : "", | 20260508 11:15:08.624 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:08.624 | INFO | "malloc" : " 14 GB", | 20260508 11:15:08.624 | INFO | "pinned" : " 29 GB" | 20260508 11:15:08.624 | INFO | } | 20260508 11:15:08.624 | INFO | Perf report for the last 5.13746 seconds: | 20260508 11:15:08.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:08.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.624 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:15:08.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.624 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:15:08.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:08.625 | INFO | Uploaded 36GiB to GPU in 2.48972s (14.4594GiB/s) | 20260508 11:15:08.625 | INFO | Downloaded 34GiB to GPU in 2.43292s (13.975GiB/s) | 20260508 11:15:13.770 | INFO | Analyzing chunk frames = [7669248,7669760) | 20260508 11:15:13.770 | INFO | MemInfo: { | 20260508 11:15:13.770 | INFO | "gpu" : " 17 GB", | 20260508 11:15:13.770 | INFO | "huge" : "", | 20260508 11:15:13.770 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:13.770 | INFO | "malloc" : " 14 GB", | 20260508 11:15:13.770 | INFO | "pinned" : " 29 GB" | 20260508 11:15:13.770 | INFO | } | 20260508 11:15:13.770 | INFO | Perf report for the last 5.14547 seconds: | 20260508 11:15:13.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:13.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:15:13.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:15:13.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:13.770 | INFO | Uploaded 36GiB to GPU in 2.47631s (14.5378GiB/s) | 20260508 11:15:13.770 | INFO | Downloaded 34GiB to GPU in 2.41605s (14.0726GiB/s) | 20260508 11:15:18.910 | INFO | Analyzing chunk frames = [7669760,7670272) | 20260508 11:15:18.910 | INFO | MemInfo: { | 20260508 11:15:18.910 | INFO | "gpu" : " 17 GB", | 20260508 11:15:18.910 | INFO | "huge" : "", | 20260508 11:15:18.910 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:18.910 | INFO | "malloc" : " 14 GB", | 20260508 11:15:18.910 | INFO | "pinned" : " 29 GB" | 20260508 11:15:18.910 | INFO | } | 20260508 11:15:18.910 | INFO | Perf report for the last 5.14004 seconds: | 20260508 11:15:18.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:18.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:15:18.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:15:18.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:18.910 | INFO | Uploaded 36GiB to GPU in 2.31321s (15.5628GiB/s) | 20260508 11:15:18.911 | INFO | Downloaded 34GiB to GPU in 2.48026s (13.7082GiB/s) | 20260508 11:15:24.042 | INFO | Analyzing chunk frames = [7670272,7670784) | 20260508 11:15:24.042 | INFO | MemInfo: { | 20260508 11:15:24.042 | INFO | "gpu" : " 17 GB", | 20260508 11:15:24.042 | INFO | "huge" : "", | 20260508 11:15:24.042 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:24.042 | INFO | "malloc" : " 14 GB", | 20260508 11:15:24.042 | INFO | "pinned" : " 29 GB" | 20260508 11:15:24.042 | INFO | } | 20260508 11:15:24.043 | INFO | Perf report for the last 5.13278 seconds: | 20260508 11:15:24.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:24.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | ANALYSIS : 65.6%, 3.0, 101.8ms, 0 | 20260508 11:15:24.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260508 11:15:24.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:24.043 | INFO | Uploaded 36GiB to GPU in 2.41694s (14.8948GiB/s) | 20260508 11:15:24.043 | INFO | Downloaded 34GiB to GPU in 2.43626s (13.9558GiB/s) | 20260508 11:15:29.185 | INFO | Analyzing chunk frames = [7670784,7671296) | 20260508 11:15:29.185 | INFO | MemInfo: { | 20260508 11:15:29.185 | INFO | "gpu" : " 17 GB", | 20260508 11:15:29.185 | INFO | "huge" : "", | 20260508 11:15:29.185 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:29.185 | INFO | "malloc" : " 14 GB", | 20260508 11:15:29.185 | INFO | "pinned" : " 29 GB" | 20260508 11:15:29.185 | INFO | } | 20260508 11:15:29.185 | INFO | Perf report for the last 5.14281 seconds: | 20260508 11:15:29.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:29.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.186 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:15:29.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.186 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260508 11:15:29.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:29.186 | INFO | Uploaded 36GiB to GPU in 2.46395s (14.6107GiB/s) | 20260508 11:15:29.186 | INFO | Downloaded 34GiB to GPU in 2.45663s (13.8401GiB/s) | 20260508 11:15:34.324 | INFO | Analyzing chunk frames = [7671296,7671808) | 20260508 11:15:34.324 | INFO | MemInfo: { | 20260508 11:15:34.324 | INFO | "gpu" : " 17 GB", | 20260508 11:15:34.324 | INFO | "huge" : "", | 20260508 11:15:34.324 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:34.324 | INFO | "malloc" : " 14 GB", | 20260508 11:15:34.324 | INFO | "pinned" : " 29 GB" | 20260508 11:15:34.324 | INFO | } | 20260508 11:15:34.324 | INFO | Perf report for the last 5.13875 seconds: | 20260508 11:15:34.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:34.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.324 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 11:15:34.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.325 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 11:15:34.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:34.325 | INFO | Uploaded 36GiB to GPU in 2.46657s (14.5952GiB/s) | 20260508 11:15:34.325 | INFO | Downloaded 34GiB to GPU in 2.4504s (13.8753GiB/s) | 20260508 11:15:34.439 | INFO | Starting a round of Baz IO | 20260508 11:15:37.834 | INFO | Populating RTMetrics report | 20260508 11:15:37.835 | INFO | RTMetrics report populated | 20260508 11:15:37.835 | INFO | Opening tmp RTMetrics json file | 20260508 11:15:37.835 | INFO | Serializing RTMetrics to json | 20260508 11:15:37.837 | INFO | Serialization done, swapping tmp file now... | 20260508 11:15:37.837 | INFO | RTMetrics done | 20260508 11:15:37.898 | INFO | Finished a round of Baz IO: Wrote 1.44 GiB in 3.458 seconds | 20260508 11:15:37.898 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:15:37.898 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.197323% / 0% / 5.5631e-05% | 20260508 11:15:37.898 | INFO | Baz IO overall: | 20260508 11:15:37.898 | INFO | BytesWritten : 3313425730843 | 20260508 11:15:37.898 | INFO | NumEvents : 1591627053414 | 20260508 11:15:39.462 | INFO | Analyzing chunk frames = [7671808,7672320) | 20260508 11:15:39.462 | INFO | MemInfo: { | 20260508 11:15:39.462 | INFO | "gpu" : " 17 GB", | 20260508 11:15:39.462 | INFO | "huge" : "", | 20260508 11:15:39.463 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:39.463 | INFO | "malloc" : " 14 GB", | 20260508 11:15:39.463 | INFO | "pinned" : " 29 GB" | 20260508 11:15:39.463 | INFO | } | 20260508 11:15:39.463 | INFO | Perf report for the last 5.13846 seconds: | 20260508 11:15:39.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:39.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:39.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:39.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:39.463 | INFO | ANALYSIS : 68.0%, 3.0, 105.4ms, 0 | 20260508 11:15:39.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:39.463 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260508 11:15:39.463 | INFO | PRE_HQ : 66.9%, 1.8, 61.8ms, 0 | 20260508 11:15:39.463 | INFO | BAZWRITER : 66.7%, 2.7, 94.0ms, 0 | 20260508 11:15:39.463 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 11:15:39.463 | INFO | Uploaded 36GiB to GPU in 2.5427s (14.1582GiB/s) | 20260508 11:15:39.463 | INFO | Downloaded 38GiB to GPU in 2.72714s (13.934GiB/s) | 20260508 11:15:44.598 | INFO | Analyzing chunk frames = [7672320,7672832) | 20260508 11:15:44.598 | INFO | MemInfo: { | 20260508 11:15:44.598 | INFO | "gpu" : " 17 GB", | 20260508 11:15:44.598 | INFO | "huge" : "", | 20260508 11:15:44.599 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:44.599 | INFO | "malloc" : " 14 GB", | 20260508 11:15:44.599 | INFO | "pinned" : " 29 GB" | 20260508 11:15:44.599 | INFO | } | 20260508 11:15:44.599 | INFO | Perf report for the last 5.13605 seconds: | 20260508 11:15:44.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:44.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | ANALYSIS : 66.9%, 3.0, 104.2ms, 0 | 20260508 11:15:44.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:15:44.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:44.599 | INFO | Uploaded 36GiB to GPU in 2.52359s (14.2654GiB/s) | 20260508 11:15:44.599 | INFO | Downloaded 34GiB to GPU in 2.4592s (13.8256GiB/s) | 20260508 11:15:49.742 | INFO | Analyzing chunk frames = [7672832,7673344) | 20260508 11:15:49.742 | INFO | MemInfo: { | 20260508 11:15:49.742 | INFO | "gpu" : " 17 GB", | 20260508 11:15:49.742 | INFO | "huge" : "", | 20260508 11:15:49.742 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:49.742 | INFO | "malloc" : " 14 GB", | 20260508 11:15:49.742 | INFO | "pinned" : " 29 GB" | 20260508 11:15:49.742 | INFO | } | 20260508 11:15:49.742 | INFO | Perf report for the last 5.14329 seconds: | 20260508 11:15:49.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:49.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:15:49.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:15:49.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:49.742 | INFO | Uploaded 36GiB to GPU in 2.44924s (14.6985GiB/s) | 20260508 11:15:49.742 | INFO | Downloaded 34GiB to GPU in 2.45912s (13.8261GiB/s) | 20260508 11:15:54.881 | INFO | Analyzing chunk frames = [7673344,7673856) | 20260508 11:15:54.881 | INFO | MemInfo: { | 20260508 11:15:54.881 | INFO | "gpu" : " 17 GB", | 20260508 11:15:54.881 | INFO | "huge" : "", | 20260508 11:15:54.881 | INFO | "huge_shared" : " 29 GB", | 20260508 11:15:54.881 | INFO | "malloc" : " 14 GB", | 20260508 11:15:54.881 | INFO | "pinned" : " 29 GB" | 20260508 11:15:54.881 | INFO | } | 20260508 11:15:54.881 | INFO | Perf report for the last 5.13945 seconds: | 20260508 11:15:54.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:15:54.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.882 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:15:54.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.882 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:15:54.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:15:54.882 | INFO | Uploaded 36GiB to GPU in 2.44795s (14.7062GiB/s) | 20260508 11:15:54.882 | INFO | Downloaded 34GiB to GPU in 2.46693s (13.7823GiB/s) | 20260508 11:16:00.023 | INFO | Analyzing chunk frames = [7673856,7674368) | 20260508 11:16:00.023 | INFO | MemInfo: { | 20260508 11:16:00.023 | INFO | "gpu" : " 17 GB", | 20260508 11:16:00.023 | INFO | "huge" : "", | 20260508 11:16:00.023 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:00.023 | INFO | "malloc" : " 14 GB", | 20260508 11:16:00.023 | INFO | "pinned" : " 29 GB" | 20260508 11:16:00.023 | INFO | } | 20260508 11:16:00.023 | INFO | Perf report for the last 5.14157 seconds: | 20260508 11:16:00.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:00.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:16:00.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:16:00.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:00.023 | INFO | Uploaded 36GiB to GPU in 2.28806s (15.7338GiB/s) | 20260508 11:16:00.023 | INFO | Downloaded 34GiB to GPU in 2.51715s (13.5074GiB/s) | 20260508 11:16:05.158 | INFO | Analyzing chunk frames = [7674368,7674880) | 20260508 11:16:05.158 | INFO | MemInfo: { | 20260508 11:16:05.158 | INFO | "gpu" : " 17 GB", | 20260508 11:16:05.158 | INFO | "huge" : "", | 20260508 11:16:05.158 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:05.158 | INFO | "malloc" : " 14 GB", | 20260508 11:16:05.158 | INFO | "pinned" : " 29 GB" | 20260508 11:16:05.158 | INFO | } | 20260508 11:16:05.158 | INFO | Perf report for the last 5.13496 seconds: | 20260508 11:16:05.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:05.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:16:05.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:16:05.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:05.158 | INFO | Uploaded 36GiB to GPU in 2.38209s (15.1128GiB/s) | 20260508 11:16:05.158 | INFO | Downloaded 34GiB to GPU in 2.43211s (13.9796GiB/s) | 20260508 11:16:10.295 | INFO | Analyzing chunk frames = [7674880,7675392) | 20260508 11:16:10.295 | INFO | MemInfo: { | 20260508 11:16:10.295 | INFO | "gpu" : " 17 GB", | 20260508 11:16:10.295 | INFO | "huge" : "", | 20260508 11:16:10.295 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:10.295 | INFO | "malloc" : " 14 GB", | 20260508 11:16:10.295 | INFO | "pinned" : " 29 GB" | 20260508 11:16:10.295 | INFO | } | 20260508 11:16:10.295 | INFO | Perf report for the last 5.13685 seconds: | 20260508 11:16:10.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:10.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.295 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:16:10.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.295 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:16:10.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:10.296 | INFO | Uploaded 36GiB to GPU in 2.41629s (14.8989GiB/s) | 20260508 11:16:10.296 | INFO | Downloaded 34GiB to GPU in 2.43333s (13.9726GiB/s) | 20260508 11:16:15.439 | INFO | Analyzing chunk frames = [7675392,7675904) | 20260508 11:16:15.439 | INFO | MemInfo: { | 20260508 11:16:15.439 | INFO | "gpu" : " 17 GB", | 20260508 11:16:15.439 | INFO | "huge" : "", | 20260508 11:16:15.439 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:15.439 | INFO | "malloc" : " 14 GB", | 20260508 11:16:15.439 | INFO | "pinned" : " 29 GB" | 20260508 11:16:15.439 | INFO | } | 20260508 11:16:15.439 | INFO | Perf report for the last 5.14408 seconds: | 20260508 11:16:15.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:15.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 11:16:15.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:16:15.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:15.439 | INFO | Uploaded 36GiB to GPU in 2.41935s (14.88GiB/s) | 20260508 11:16:15.439 | INFO | Downloaded 34GiB to GPU in 2.43049s (13.9889GiB/s) | 20260508 11:16:15.439 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:16:15.439 | INFO | Total mem usage: 16.7 GiB | 20260508 11:16:15.439 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:16:15.439 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 11:16:15.439 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:16:15.439 | INFO | High Water Breakdown: | 20260508 11:16:15.439 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:16:15.439 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:16:15.439 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:16:15.439 | INFO | ------------------------------------------ | 20260508 11:16:15.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:16:15.439 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:16:15.439 | INFO | Total mem usage: 26.5 GiB | 20260508 11:16:15.439 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:16:15.439 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:16:15.439 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:16:15.439 | INFO | High Water Breakdown: | 20260508 11:16:15.439 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:16:15.439 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:16:15.439 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:16:15.439 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:16:15.439 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:16:15.439 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:16:15.439 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:16:15.439 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:16:15.439 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:16:15.439 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:16:15.439 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:16:15.439 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:16:15.439 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:16:15.439 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:16:15.439 | INFO | ------------------------------------------ | 20260508 11:16:15.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:16:15.439 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:16:15.439 | INFO | Total mem usage: 27.8 GiB | 20260508 11:16:15.439 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:16:15.439 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:16:15.439 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:16:15.439 | INFO | High Water Breakdown: | 20260508 11:16:15.439 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:16:15.439 | INFO | ------------------------------------------ | 20260508 11:16:15.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:16:15.439 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:16:15.439 | INFO | Total mem usage: 15.7 GiB | 20260508 11:16:15.439 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:16:15.439 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:16:15.439 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:16:15.439 | INFO | High Water Breakdown: | 20260508 11:16:15.439 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:16:15.439 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:16:15.439 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:16:15.439 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:16:15.439 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:16:15.439 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:16:15.439 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:16:15.439 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:16:15.439 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:16:15.439 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:16:15.439 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:16:15.439 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:16:15.439 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:16:15.439 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:16:15.439 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:16:15.439 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:16:15.439 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:16:15.439 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:16:15.439 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:16:15.439 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:16:15.439 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:16:15.439 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:16:15.439 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:16:15.439 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:16:15.439 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:16:15.439 | INFO | ------------------------------------------ | 20260508 11:16:15.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:16:15.439 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:16:15.439 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:16:15.439 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:16:15.439 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 5.87e-01 | 20260508 11:16:15.439 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.14e-02 / 2.53e-03 | 20260508 11:16:15.439 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.47e+00, 6.00e-01 / 2.18e-01 | 20260508 11:16:15.439 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.11e-01 / 8.37e-02 | 20260508 11:16:15.439 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.58e+01, 3.83e+00 / 8.61e-01 | 20260508 11:16:15.439 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.86e+01 / 1.95e+01, 1.80e+00 / 8.98e-01 | 20260508 11:16:15.440 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.61e+00, 4.07e-01 / 1.81e-01 | 20260508 11:16:15.440 | INFO | PerfReport: 3.00%, Metrics -- 9.82e-01 / 1.16e+00, 2.44e-01 / 1.12e-01 | 20260508 11:16:15.440 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.37e-01, 4.78e-02 / 2.43e-02 | 20260508 11:16:15.440 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:16:15.440 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:16:15.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:16:15.440 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 24.19%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 31.07%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 2.14%, LockUpload -- 2.25e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 2.62%, LockDownload -- 2.76e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 15.36%, LockCompute -- 1.62e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:16:15.440 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260508 11:16:15.539 | INFO | Starting a round of Baz IO | 20260508 11:16:18.938 | INFO | Populating RTMetrics report | 20260508 11:16:18.938 | INFO | RTMetrics report populated | 20260508 11:16:18.938 | INFO | Opening tmp RTMetrics json file | 20260508 11:16:18.952 | INFO | Serializing RTMetrics to json | 20260508 11:16:18.953 | INFO | Serialization done, swapping tmp file now... | 20260508 11:16:18.953 | INFO | RTMetrics done | 20260508 11:16:19.012 | INFO | Finished a round of Baz IO: Wrote 1.44 GiB in 3.473 seconds | 20260508 11:16:19.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:16:19.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196735% / 0% / 6.35783e-05% | 20260508 11:16:19.013 | INFO | Baz IO overall: | 20260508 11:16:19.013 | INFO | BytesWritten : 3314971600733 | 20260508 11:16:19.013 | INFO | NumEvents : 1592354523088 | 20260508 11:16:20.583 | INFO | Analyzing chunk frames = [7675904,7676416) | 20260508 11:16:20.583 | INFO | MemInfo: { | 20260508 11:16:20.583 | INFO | "gpu" : " 17 GB", | 20260508 11:16:20.583 | INFO | "huge" : "", | 20260508 11:16:20.583 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:20.583 | INFO | "malloc" : " 14 GB", | 20260508 11:16:20.583 | INFO | "pinned" : " 29 GB" | 20260508 11:16:20.583 | INFO | } | 20260508 11:16:20.583 | INFO | Perf report for the last 5.14415 seconds: | 20260508 11:16:20.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:20.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:20.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:20.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:20.583 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 11:16:20.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:20.583 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:16:20.583 | INFO | PRE_HQ : 65.3%, 1.6, 54.3ms, 0 | 20260508 11:16:20.583 | INFO | BAZWRITER : 67.0%, 2.5, 87.6ms, 0 | 20260508 11:16:20.583 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 11:16:20.584 | INFO | Uploaded 36GiB to GPU in 2.60222s (13.8344GiB/s) | 20260508 11:16:20.584 | INFO | Downloaded 38GiB to GPU in 2.7161s (13.9907GiB/s) | 20260508 11:16:25.721 | INFO | Analyzing chunk frames = [7676416,7676928) | 20260508 11:16:25.721 | INFO | MemInfo: { | 20260508 11:16:25.721 | INFO | "gpu" : " 17 GB", | 20260508 11:16:25.721 | INFO | "huge" : "", | 20260508 11:16:25.721 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:25.721 | INFO | "malloc" : " 14 GB", | 20260508 11:16:25.721 | INFO | "pinned" : " 29 GB" | 20260508 11:16:25.721 | INFO | } | 20260508 11:16:25.721 | INFO | Perf report for the last 5.13789 seconds: | 20260508 11:16:25.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:25.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:16:25.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:16:25.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:25.721 | INFO | Uploaded 36GiB to GPU in 2.45613s (14.6572GiB/s) | 20260508 11:16:25.721 | INFO | Downloaded 34GiB to GPU in 2.46667s (13.7838GiB/s) | 20260508 11:16:30.851 | INFO | Analyzing chunk frames = [7676928,7677440) | 20260508 11:16:30.851 | INFO | MemInfo: { | 20260508 11:16:30.851 | INFO | "gpu" : " 17 GB", | 20260508 11:16:30.851 | INFO | "huge" : "", | 20260508 11:16:30.851 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:30.851 | INFO | "malloc" : " 14 GB", | 20260508 11:16:30.851 | INFO | "pinned" : " 29 GB" | 20260508 11:16:30.851 | INFO | } | 20260508 11:16:30.851 | INFO | Perf report for the last 5.13001 seconds: | 20260508 11:16:30.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:30.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 11:16:30.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:16:30.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:30.851 | INFO | Uploaded 36GiB to GPU in 2.47628s (14.5379GiB/s) | 20260508 11:16:30.851 | INFO | Downloaded 34GiB to GPU in 2.45838s (13.8302GiB/s) | 20260508 11:16:35.994 | INFO | Analyzing chunk frames = [7677440,7677952) | 20260508 11:16:35.994 | INFO | MemInfo: { | 20260508 11:16:35.994 | INFO | "gpu" : " 17 GB", | 20260508 11:16:35.994 | INFO | "huge" : "", | 20260508 11:16:35.994 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:35.994 | INFO | "malloc" : " 14 GB", | 20260508 11:16:35.994 | INFO | "pinned" : " 29 GB" | 20260508 11:16:35.994 | INFO | } | 20260508 11:16:35.994 | INFO | Perf report for the last 5.14277 seconds: | 20260508 11:16:35.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:35.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:16:35.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:16:35.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:35.994 | INFO | Uploaded 36GiB to GPU in 2.46502s (14.6043GiB/s) | 20260508 11:16:35.995 | INFO | Downloaded 34GiB to GPU in 2.43832s (13.944GiB/s) | 20260508 11:16:41.131 | INFO | Analyzing chunk frames = [7677952,7678464) | 20260508 11:16:41.131 | INFO | MemInfo: { | 20260508 11:16:41.131 | INFO | "gpu" : " 17 GB", | 20260508 11:16:41.131 | INFO | "huge" : "", | 20260508 11:16:41.131 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:41.132 | INFO | "malloc" : " 14 GB", | 20260508 11:16:41.132 | INFO | "pinned" : " 29 GB" | 20260508 11:16:41.132 | INFO | } | 20260508 11:16:41.132 | INFO | Perf report for the last 5.13775 seconds: | 20260508 11:16:41.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:41.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 11:16:41.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 11:16:41.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:41.132 | INFO | Uploaded 36GiB to GPU in 2.36755s (15.2056GiB/s) | 20260508 11:16:41.132 | INFO | Downloaded 34GiB to GPU in 2.52413s (13.47GiB/s) | 20260508 11:16:46.274 | INFO | Analyzing chunk frames = [7678464,7678976) | 20260508 11:16:46.274 | INFO | MemInfo: { | 20260508 11:16:46.274 | INFO | "gpu" : " 17 GB", | 20260508 11:16:46.274 | INFO | "huge" : "", | 20260508 11:16:46.274 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:46.274 | INFO | "malloc" : " 14 GB", | 20260508 11:16:46.274 | INFO | "pinned" : " 29 GB" | 20260508 11:16:46.274 | INFO | } | 20260508 11:16:46.274 | INFO | Perf report for the last 5.1422 seconds: | 20260508 11:16:46.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:46.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:16:46.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260508 11:16:46.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:46.274 | INFO | Uploaded 36GiB to GPU in 2.36592s (15.216GiB/s) | 20260508 11:16:46.274 | INFO | Downloaded 34GiB to GPU in 2.43621s (13.9561GiB/s) | 20260508 11:16:51.414 | INFO | Analyzing chunk frames = [7678976,7679488) | 20260508 11:16:51.415 | INFO | MemInfo: { | 20260508 11:16:51.415 | INFO | "gpu" : " 17 GB", | 20260508 11:16:51.415 | INFO | "huge" : "", | 20260508 11:16:51.415 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:51.415 | INFO | "malloc" : " 14 GB", | 20260508 11:16:51.415 | INFO | "pinned" : " 29 GB" | 20260508 11:16:51.415 | INFO | } | 20260508 11:16:51.415 | INFO | Perf report for the last 5.14094 seconds: | 20260508 11:16:51.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:51.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 11:16:51.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:16:51.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:51.415 | INFO | Uploaded 36GiB to GPU in 2.40172s (14.9893GiB/s) | 20260508 11:16:51.415 | INFO | Downloaded 34GiB to GPU in 2.43138s (13.9838GiB/s) | 20260508 11:16:56.554 | INFO | Analyzing chunk frames = [7679488,7680000) | 20260508 11:16:56.554 | INFO | MemInfo: { | 20260508 11:16:56.554 | INFO | "gpu" : " 17 GB", | 20260508 11:16:56.554 | INFO | "huge" : "", | 20260508 11:16:56.554 | INFO | "huge_shared" : " 29 GB", | 20260508 11:16:56.554 | INFO | "malloc" : " 14 GB", | 20260508 11:16:56.554 | INFO | "pinned" : " 29 GB" | 20260508 11:16:56.554 | INFO | } | 20260508 11:16:56.554 | INFO | Perf report for the last 5.13961 seconds: | 20260508 11:16:56.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:16:56.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.554 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 11:16:56.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.555 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 11:16:56.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:16:56.555 | INFO | Uploaded 36GiB to GPU in 2.46923s (14.5794GiB/s) | 20260508 11:16:56.555 | INFO | Downloaded 34GiB to GPU in 2.46933s (13.7689GiB/s) | 20260508 11:16:56.652 | INFO | Starting a round of Baz IO | 20260508 11:17:00.051 | INFO | Populating RTMetrics report | 20260508 11:17:00.052 | INFO | RTMetrics report populated | 20260508 11:17:00.052 | INFO | Opening tmp RTMetrics json file | 20260508 11:17:00.061 | INFO | Serializing RTMetrics to json | 20260508 11:17:00.062 | INFO | Serialization done, swapping tmp file now... | 20260508 11:17:00.063 | INFO | RTMetrics done | 20260508 11:17:00.123 | INFO | Finished a round of Baz IO: Wrote 1.441 GiB in 3.47 seconds | 20260508 11:17:00.123 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:17:00.123 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.197426% / 0% / 9.53674e-05% | 20260508 11:17:00.123 | INFO | Baz IO overall: | 20260508 11:17:00.123 | INFO | BytesWritten : 3316518750278 | 20260508 11:17:00.123 | INFO | NumEvents : 1593083174131 | 20260508 11:17:01.696 | INFO | Analyzing chunk frames = [7680000,7680512) | 20260508 11:17:01.696 | INFO | MemInfo: { | 20260508 11:17:01.696 | INFO | "gpu" : " 17 GB", | 20260508 11:17:01.696 | INFO | "huge" : "", | 20260508 11:17:01.696 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:01.696 | INFO | "malloc" : " 14 GB", | 20260508 11:17:01.696 | INFO | "pinned" : " 29 GB" | 20260508 11:17:01.696 | INFO | } | 20260508 11:17:01.696 | INFO | Perf report for the last 5.14179 seconds: | 20260508 11:17:01.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:01.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:01.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:01.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:01.696 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 11:17:01.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:01.696 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 11:17:01.696 | INFO | PRE_HQ : 65.2%, 1.6, 54.7ms, 0 | 20260508 11:17:01.696 | INFO | BAZWRITER : 66.8%, 2.5, 85.0ms, 0 | 20260508 11:17:01.696 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:17:01.696 | INFO | Uploaded 36GiB to GPU in 2.60499s (13.8196GiB/s) | 20260508 11:17:01.696 | INFO | Downloaded 38GiB to GPU in 2.70978s (14.0233GiB/s) | 20260508 11:17:06.831 | INFO | Analyzing chunk frames = [7680512,7681024) | 20260508 11:17:06.831 | INFO | MemInfo: { | 20260508 11:17:06.831 | INFO | "gpu" : " 17 GB", | 20260508 11:17:06.831 | INFO | "huge" : "", | 20260508 11:17:06.831 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:06.831 | INFO | "malloc" : " 14 GB", | 20260508 11:17:06.831 | INFO | "pinned" : " 29 GB" | 20260508 11:17:06.831 | INFO | } | 20260508 11:17:06.831 | INFO | Perf report for the last 5.13439 seconds: | 20260508 11:17:06.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:06.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:17:06.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:17:06.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:06.831 | INFO | Uploaded 36GiB to GPU in 2.42039s (14.8736GiB/s) | 20260508 11:17:06.831 | INFO | Downloaded 34GiB to GPU in 2.46613s (13.7868GiB/s) | 20260508 11:17:11.969 | INFO | Analyzing chunk frames = [7681024,7681536) | 20260508 11:17:11.969 | INFO | MemInfo: { | 20260508 11:17:11.969 | INFO | "gpu" : " 17 GB", | 20260508 11:17:11.969 | INFO | "huge" : "", | 20260508 11:17:11.969 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:11.969 | INFO | "malloc" : " 14 GB", | 20260508 11:17:11.970 | INFO | "pinned" : " 29 GB" | 20260508 11:17:11.970 | INFO | } | 20260508 11:17:11.970 | INFO | Perf report for the last 5.13894 seconds: | 20260508 11:17:11.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:11.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:17:11.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:17:11.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:11.970 | INFO | Uploaded 36GiB to GPU in 2.4945s (14.4317GiB/s) | 20260508 11:17:11.970 | INFO | Downloaded 34GiB to GPU in 2.44759s (13.8912GiB/s) | 20260508 11:17:17.112 | INFO | Analyzing chunk frames = [7681536,7682048) | 20260508 11:17:17.112 | INFO | MemInfo: { | 20260508 11:17:17.112 | INFO | "gpu" : " 17 GB", | 20260508 11:17:17.112 | INFO | "huge" : "", | 20260508 11:17:17.112 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:17.112 | INFO | "malloc" : " 14 GB", | 20260508 11:17:17.112 | INFO | "pinned" : " 29 GB" | 20260508 11:17:17.112 | INFO | } | 20260508 11:17:17.112 | INFO | Perf report for the last 5.14213 seconds: | 20260508 11:17:17.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:17.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:17:17.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:17:17.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:17.112 | INFO | Uploaded 36GiB to GPU in 2.46449s (14.6075GiB/s) | 20260508 11:17:17.112 | INFO | Downloaded 34GiB to GPU in 2.45703s (13.8378GiB/s) | 20260508 11:17:22.251 | INFO | Analyzing chunk frames = [7682048,7682560) | 20260508 11:17:22.251 | INFO | MemInfo: { | 20260508 11:17:22.251 | INFO | "gpu" : " 17 GB", | 20260508 11:17:22.251 | INFO | "huge" : "", | 20260508 11:17:22.251 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:22.251 | INFO | "malloc" : " 14 GB", | 20260508 11:17:22.251 | INFO | "pinned" : " 29 GB" | 20260508 11:17:22.251 | INFO | } | 20260508 11:17:22.251 | INFO | Perf report for the last 5.1398 seconds: | 20260508 11:17:22.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:22.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 11:17:22.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:17:22.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:22.252 | INFO | Uploaded 36GiB to GPU in 2.43887s (14.7609GiB/s) | 20260508 11:17:22.252 | INFO | Downloaded 34GiB to GPU in 2.54426s (13.3634GiB/s) | 20260508 11:17:27.391 | INFO | Analyzing chunk frames = [7682560,7683072) | 20260508 11:17:27.391 | INFO | MemInfo: { | 20260508 11:17:27.391 | INFO | "gpu" : " 17 GB", | 20260508 11:17:27.391 | INFO | "huge" : "", | 20260508 11:17:27.391 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:27.391 | INFO | "malloc" : " 14 GB", | 20260508 11:17:27.391 | INFO | "pinned" : " 29 GB" | 20260508 11:17:27.391 | INFO | } | 20260508 11:17:27.391 | INFO | Perf report for the last 5.13964 seconds: | 20260508 11:17:27.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:27.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 11:17:27.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:17:27.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:27.391 | INFO | Uploaded 36GiB to GPU in 2.36367s (15.2306GiB/s) | 20260508 11:17:27.391 | INFO | Downloaded 34GiB to GPU in 2.45192s (13.8667GiB/s) | 20260508 11:17:32.533 | INFO | Analyzing chunk frames = [7683072,7683584) | 20260508 11:17:32.533 | INFO | MemInfo: { | 20260508 11:17:32.533 | INFO | "gpu" : " 17 GB", | 20260508 11:17:32.533 | INFO | "huge" : "", | 20260508 11:17:32.533 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:32.533 | INFO | "malloc" : " 14 GB", | 20260508 11:17:32.533 | INFO | "pinned" : " 29 GB" | 20260508 11:17:32.533 | INFO | } | 20260508 11:17:32.533 | INFO | Perf report for the last 5.14189 seconds: | 20260508 11:17:32.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:32.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | ANALYSIS : 65.1%, 3.0, 101.6ms, 0 | 20260508 11:17:32.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:17:32.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:32.533 | INFO | Uploaded 36GiB to GPU in 2.38954s (15.0657GiB/s) | 20260508 11:17:32.534 | INFO | Downloaded 34GiB to GPU in 2.45621s (13.8425GiB/s) | 20260508 11:17:37.668 | INFO | Analyzing chunk frames = [7683584,7684096) | 20260508 11:17:37.668 | INFO | MemInfo: { | 20260508 11:17:37.668 | INFO | "gpu" : " 17 GB", | 20260508 11:17:37.668 | INFO | "huge" : "", | 20260508 11:17:37.668 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:37.668 | INFO | "malloc" : " 14 GB", | 20260508 11:17:37.668 | INFO | "pinned" : " 29 GB" | 20260508 11:17:37.668 | INFO | } | 20260508 11:17:37.668 | INFO | Perf report for the last 5.13511 seconds: | 20260508 11:17:37.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:37.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.668 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:17:37.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.668 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:17:37.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:37.669 | INFO | Uploaded 36GiB to GPU in 2.42288s (14.8583GiB/s) | 20260508 11:17:37.669 | INFO | Downloaded 34GiB to GPU in 2.46983s (13.7661GiB/s) | 20260508 11:17:37.669 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:17:37.669 | INFO | Total mem usage: 16.7 GiB | 20260508 11:17:37.669 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:17:37.669 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:17:37.669 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:17:37.669 | INFO | High Water Breakdown: | 20260508 11:17:37.669 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:17:37.669 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:17:37.669 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:17:37.669 | INFO | ------------------------------------------ | 20260508 11:17:37.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:17:37.669 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:17:37.669 | INFO | Total mem usage: 26.5 GiB | 20260508 11:17:37.669 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:17:37.669 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:17:37.669 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:17:37.669 | INFO | High Water Breakdown: | 20260508 11:17:37.669 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:17:37.669 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:17:37.669 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:17:37.669 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:17:37.669 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:17:37.669 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:17:37.669 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:17:37.669 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:17:37.669 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:17:37.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:17:37.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:17:37.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:17:37.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:17:37.669 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:17:37.669 | INFO | ------------------------------------------ | 20260508 11:17:37.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:17:37.669 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:17:37.669 | INFO | Total mem usage: 27.8 GiB | 20260508 11:17:37.669 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:17:37.669 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:17:37.669 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:17:37.669 | INFO | High Water Breakdown: | 20260508 11:17:37.669 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:17:37.669 | INFO | ------------------------------------------ | 20260508 11:17:37.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:17:37.669 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:17:37.669 | INFO | Total mem usage: 15.7 GiB | 20260508 11:17:37.669 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:17:37.669 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:17:37.669 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:17:37.669 | INFO | High Water Breakdown: | 20260508 11:17:37.669 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:17:37.669 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:17:37.670 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:17:37.670 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:17:37.670 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:17:37.670 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:17:37.670 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:17:37.670 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:17:37.670 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:17:37.670 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:17:37.670 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:17:37.670 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:17:37.670 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:17:37.670 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:17:37.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:17:37.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:17:37.670 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:17:37.670 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:17:37.670 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:17:37.670 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:17:37.670 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:17:37.670 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:17:37.670 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:17:37.670 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:17:37.670 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:17:37.670 | INFO | ------------------------------------------ | 20260508 11:17:37.670 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:17:37.670 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:17:37.670 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:17:37.670 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:17:37.670 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 6.35e-01 | 20260508 11:17:37.670 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.15e-02 / 7.89e-03 | 20260508 11:17:37.670 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.48e+00, 5.87e-01 / 2.02e-01 | 20260508 11:17:37.670 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.31e-01 | 20260508 11:17:37.670 | INFO | PerfReport: 16.02%, DME -- 4.18e+01 / 4.51e+01, 3.68e+00 / 9.95e-01 | 20260508 11:17:37.670 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.76e+00 / 1.35e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.34e+00 / 2.48e+00, 4.07e-01 / 2.79e-01 | 20260508 11:17:37.670 | INFO | PerfReport: 3.01%, Metrics -- 9.81e-01 / 1.09e+00, 2.41e-01 / 1.42e-01 | 20260508 11:17:37.670 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.30e-01, 4.76e-02 / 3.63e-02 | 20260508 11:17:37.670 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:17:37.670 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:17:37.670 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:17:37.670 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 24.28%, Upload -- 2.55e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 31.04%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 2.07%, LockUpload -- 2.17e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 2.64%, LockDownload -- 2.78e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 15.21%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:17:37.670 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260508 11:17:37.766 | INFO | Starting a round of Baz IO | 20260508 11:17:41.184 | INFO | Populating RTMetrics report | 20260508 11:17:41.184 | INFO | RTMetrics report populated | 20260508 11:17:41.184 | INFO | Opening tmp RTMetrics json file | 20260508 11:17:41.191 | INFO | Serializing RTMetrics to json | 20260508 11:17:41.192 | INFO | Serialization done, swapping tmp file now... | 20260508 11:17:41.192 | INFO | RTMetrics done | 20260508 11:17:41.251 | INFO | Finished a round of Baz IO: Wrote 1.44 GiB in 3.485 seconds | 20260508 11:17:41.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:17:41.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.197192% / 0% / 4.37101e-05% | 20260508 11:17:41.251 | INFO | Baz IO overall: | 20260508 11:17:41.251 | INFO | BytesWritten : 3318065073693 | 20260508 11:17:41.251 | INFO | NumEvents : 1593810978586 | 20260508 11:17:42.802 | INFO | Analyzing chunk frames = [7684096,7684608) | 20260508 11:17:42.802 | INFO | MemInfo: { | 20260508 11:17:42.802 | INFO | "gpu" : " 17 GB", | 20260508 11:17:42.802 | INFO | "huge" : "", | 20260508 11:17:42.802 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:42.802 | INFO | "malloc" : " 14 GB", | 20260508 11:17:42.802 | INFO | "pinned" : " 29 GB" | 20260508 11:17:42.802 | INFO | } | 20260508 11:17:42.803 | INFO | Perf report for the last 5.13435 seconds: | 20260508 11:17:42.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:42.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:42.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:42.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:42.803 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 11:17:42.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:42.803 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:17:42.803 | INFO | PRE_HQ : 65.6%, 1.6, 54.2ms, 0 | 20260508 11:17:42.803 | INFO | BAZWRITER : 67.4%, 2.7, 94.5ms, 0 | 20260508 11:17:42.803 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:17:42.803 | INFO | Uploaded 36GiB to GPU in 2.56588s (14.0302GiB/s) | 20260508 11:17:42.803 | INFO | Downloaded 38GiB to GPU in 2.71792s (13.9813GiB/s) | 20260508 11:17:47.942 | INFO | Analyzing chunk frames = [7684608,7685120) | 20260508 11:17:47.942 | INFO | MemInfo: { | 20260508 11:17:47.942 | INFO | "gpu" : " 17 GB", | 20260508 11:17:47.942 | INFO | "huge" : "", | 20260508 11:17:47.942 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:47.942 | INFO | "malloc" : " 14 GB", | 20260508 11:17:47.942 | INFO | "pinned" : " 29 GB" | 20260508 11:17:47.942 | INFO | } | 20260508 11:17:47.942 | INFO | Perf report for the last 5.13959 seconds: | 20260508 11:17:47.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:47.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:17:47.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:17:47.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:47.942 | INFO | Uploaded 36GiB to GPU in 2.42979s (14.8161GiB/s) | 20260508 11:17:47.942 | INFO | Downloaded 34GiB to GPU in 2.48411s (13.687GiB/s) | 20260508 11:17:53.091 | INFO | Analyzing chunk frames = [7685120,7685632) | 20260508 11:17:53.091 | INFO | MemInfo: { | 20260508 11:17:53.091 | INFO | "gpu" : " 17 GB", | 20260508 11:17:53.091 | INFO | "huge" : "", | 20260508 11:17:53.091 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:53.091 | INFO | "malloc" : " 14 GB", | 20260508 11:17:53.091 | INFO | "pinned" : " 29 GB" | 20260508 11:17:53.091 | INFO | } | 20260508 11:17:53.091 | INFO | Perf report for the last 5.14925 seconds: | 20260508 11:17:53.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:53.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.091 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 11:17:53.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.092 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:17:53.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:53.092 | INFO | Uploaded 36GiB to GPU in 2.45415s (14.669GiB/s) | 20260508 11:17:53.092 | INFO | Downloaded 34GiB to GPU in 2.43537s (13.9609GiB/s) | 20260508 11:17:58.225 | INFO | Analyzing chunk frames = [7685632,7686144) | 20260508 11:17:58.225 | INFO | MemInfo: { | 20260508 11:17:58.225 | INFO | "gpu" : " 17 GB", | 20260508 11:17:58.225 | INFO | "huge" : "", | 20260508 11:17:58.225 | INFO | "huge_shared" : " 29 GB", | 20260508 11:17:58.225 | INFO | "malloc" : " 14 GB", | 20260508 11:17:58.225 | INFO | "pinned" : " 29 GB" | 20260508 11:17:58.225 | INFO | } | 20260508 11:17:58.225 | INFO | Perf report for the last 5.13353 seconds: | 20260508 11:17:58.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:17:58.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 11:17:58.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:17:58.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:17:58.225 | INFO | Uploaded 36GiB to GPU in 2.49808s (14.4111GiB/s) | 20260508 11:17:58.225 | INFO | Downloaded 34GiB to GPU in 2.45963s (13.8232GiB/s) | 20260508 11:18:03.362 | INFO | Analyzing chunk frames = [7686144,7686656) | 20260508 11:18:03.362 | INFO | MemInfo: { | 20260508 11:18:03.362 | INFO | "gpu" : " 17 GB", | 20260508 11:18:03.362 | INFO | "huge" : "", | 20260508 11:18:03.362 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:03.362 | INFO | "malloc" : " 14 GB", | 20260508 11:18:03.362 | INFO | "pinned" : " 29 GB" | 20260508 11:18:03.362 | INFO | } | 20260508 11:18:03.362 | INFO | Perf report for the last 5.13731 seconds: | 20260508 11:18:03.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:03.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.362 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 11:18:03.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.362 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:18:03.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:03.363 | INFO | Uploaded 36GiB to GPU in 2.40482s (14.9699GiB/s) | 20260508 11:18:03.363 | INFO | Downloaded 34GiB to GPU in 2.54869s (13.3402GiB/s) | 20260508 11:18:08.504 | INFO | Analyzing chunk frames = [7686656,7687168) | 20260508 11:18:08.504 | INFO | MemInfo: { | 20260508 11:18:08.504 | INFO | "gpu" : " 17 GB", | 20260508 11:18:08.504 | INFO | "huge" : "", | 20260508 11:18:08.504 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:08.504 | INFO | "malloc" : " 14 GB", | 20260508 11:18:08.504 | INFO | "pinned" : " 29 GB" | 20260508 11:18:08.504 | INFO | } | 20260508 11:18:08.504 | INFO | Perf report for the last 5.14207 seconds: | 20260508 11:18:08.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:08.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.504 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:18:08.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.505 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:18:08.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:08.505 | INFO | Uploaded 36GiB to GPU in 2.35459s (15.2893GiB/s) | 20260508 11:18:08.505 | INFO | Downloaded 34GiB to GPU in 2.43261s (13.9767GiB/s) | 20260508 11:18:13.646 | INFO | Analyzing chunk frames = [7687168,7687680) | 20260508 11:18:13.646 | INFO | MemInfo: { | 20260508 11:18:13.646 | INFO | "gpu" : " 17 GB", | 20260508 11:18:13.646 | INFO | "huge" : "", | 20260508 11:18:13.646 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:13.646 | INFO | "malloc" : " 14 GB", | 20260508 11:18:13.646 | INFO | "pinned" : " 29 GB" | 20260508 11:18:13.646 | INFO | } | 20260508 11:18:13.646 | INFO | Perf report for the last 5.14187 seconds: | 20260508 11:18:13.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:13.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.646 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 11:18:13.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.646 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:18:13.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:13.647 | INFO | Uploaded 36GiB to GPU in 2.40211s (14.9868GiB/s) | 20260508 11:18:13.647 | INFO | Downloaded 34GiB to GPU in 2.43985s (13.9353GiB/s) | 20260508 11:18:18.785 | INFO | Analyzing chunk frames = [7687680,7688192) | 20260508 11:18:18.785 | INFO | MemInfo: { | 20260508 11:18:18.785 | INFO | "gpu" : " 17 GB", | 20260508 11:18:18.785 | INFO | "huge" : "", | 20260508 11:18:18.785 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:18.785 | INFO | "malloc" : " 14 GB", | 20260508 11:18:18.785 | INFO | "pinned" : " 29 GB" | 20260508 11:18:18.785 | INFO | } | 20260508 11:18:18.785 | INFO | Perf report for the last 5.1387 seconds: | 20260508 11:18:18.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:18.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 11:18:18.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:18:18.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:18.785 | INFO | Uploaded 36GiB to GPU in 2.44948s (14.697GiB/s) | 20260508 11:18:18.785 | INFO | Downloaded 34GiB to GPU in 2.45626s (13.8422GiB/s) | 20260508 11:18:18.882 | INFO | Starting a round of Baz IO | 20260508 11:18:22.285 | INFO | Populating RTMetrics report | 20260508 11:18:22.286 | INFO | RTMetrics report populated | 20260508 11:18:22.286 | INFO | Opening tmp RTMetrics json file | 20260508 11:18:22.295 | INFO | Serializing RTMetrics to json | 20260508 11:18:22.296 | INFO | Serialization done, swapping tmp file now... | 20260508 11:18:22.297 | INFO | RTMetrics done | 20260508 11:18:22.359 | INFO | Finished a round of Baz IO: Wrote 1.439 GiB in 3.476 seconds | 20260508 11:18:22.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:18:22.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196791% / 3.97364e-06% / 4.76837e-05% | 20260508 11:18:22.359 | INFO | Baz IO overall: | 20260508 11:18:22.359 | INFO | BytesWritten : 3319610002793 | 20260508 11:18:22.359 | INFO | NumEvents : 1594538306502 | 20260508 11:18:23.923 | INFO | Analyzing chunk frames = [7688192,7688704) | 20260508 11:18:23.923 | INFO | MemInfo: { | 20260508 11:18:23.923 | INFO | "gpu" : " 17 GB", | 20260508 11:18:23.923 | INFO | "huge" : "", | 20260508 11:18:23.923 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:23.923 | INFO | "malloc" : " 14 GB", | 20260508 11:18:23.923 | INFO | "pinned" : " 29 GB" | 20260508 11:18:23.924 | INFO | } | 20260508 11:18:23.924 | INFO | Perf report for the last 5.13866 seconds: | 20260508 11:18:23.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:23.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:23.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:23.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:23.924 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:18:23.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:23.924 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:18:23.924 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 11:18:23.924 | INFO | BAZWRITER : 67.1%, 2.7, 94.3ms, 0 | 20260508 11:18:23.924 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:18:23.924 | INFO | Uploaded 36GiB to GPU in 2.62944s (13.6911GiB/s) | 20260508 11:18:23.924 | INFO | Downloaded 38GiB to GPU in 2.71206s (14.0115GiB/s) | 20260508 11:18:29.059 | INFO | Analyzing chunk frames = [7688704,7689216) | 20260508 11:18:29.059 | INFO | MemInfo: { | 20260508 11:18:29.059 | INFO | "gpu" : " 17 GB", | 20260508 11:18:29.059 | INFO | "huge" : "", | 20260508 11:18:29.059 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:29.059 | INFO | "malloc" : " 14 GB", | 20260508 11:18:29.059 | INFO | "pinned" : " 29 GB" | 20260508 11:18:29.059 | INFO | } | 20260508 11:18:29.059 | INFO | Perf report for the last 5.13519 seconds: | 20260508 11:18:29.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:29.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:18:29.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260508 11:18:29.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:29.059 | INFO | Uploaded 36GiB to GPU in 2.43883s (14.7612GiB/s) | 20260508 11:18:29.060 | INFO | Downloaded 34GiB to GPU in 2.47108s (13.7591GiB/s) | 20260508 11:18:34.204 | INFO | Analyzing chunk frames = [7689216,7689728) | 20260508 11:18:34.204 | INFO | MemInfo: { | 20260508 11:18:34.204 | INFO | "gpu" : " 17 GB", | 20260508 11:18:34.204 | INFO | "huge" : "", | 20260508 11:18:34.204 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:34.204 | INFO | "malloc" : " 14 GB", | 20260508 11:18:34.204 | INFO | "pinned" : " 29 GB" | 20260508 11:18:34.205 | INFO | } | 20260508 11:18:34.205 | INFO | Perf report for the last 5.1457 seconds: | 20260508 11:18:34.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:34.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:18:34.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:18:34.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:34.205 | INFO | Uploaded 36GiB to GPU in 2.47078s (14.5703GiB/s) | 20260508 11:18:34.205 | INFO | Downloaded 34GiB to GPU in 2.44346s (13.9147GiB/s) | 20260508 11:18:39.345 | INFO | Analyzing chunk frames = [7689728,7690240) | 20260508 11:18:39.345 | INFO | MemInfo: { | 20260508 11:18:39.345 | INFO | "gpu" : " 17 GB", | 20260508 11:18:39.345 | INFO | "huge" : "", | 20260508 11:18:39.345 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:39.345 | INFO | "malloc" : " 14 GB", | 20260508 11:18:39.345 | INFO | "pinned" : " 29 GB" | 20260508 11:18:39.345 | INFO | } | 20260508 11:18:39.345 | INFO | Perf report for the last 5.1407 seconds: | 20260508 11:18:39.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:39.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.345 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:18:39.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.345 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:18:39.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:39.346 | INFO | Uploaded 36GiB to GPU in 2.47681s (14.5348GiB/s) | 20260508 11:18:39.346 | INFO | Downloaded 34GiB to GPU in 2.45051s (13.8747GiB/s) | 20260508 11:18:44.479 | INFO | Analyzing chunk frames = [7690240,7690752) | 20260508 11:18:44.479 | INFO | MemInfo: { | 20260508 11:18:44.479 | INFO | "gpu" : " 17 GB", | 20260508 11:18:44.479 | INFO | "huge" : "", | 20260508 11:18:44.479 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:44.479 | INFO | "malloc" : " 14 GB", | 20260508 11:18:44.479 | INFO | "pinned" : " 29 GB" | 20260508 11:18:44.479 | INFO | } | 20260508 11:18:44.479 | INFO | Perf report for the last 5.13395 seconds: | 20260508 11:18:44.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:44.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.479 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:18:44.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.479 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 11:18:44.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:44.480 | INFO | Uploaded 36GiB to GPU in 2.30745s (15.6016GiB/s) | 20260508 11:18:44.480 | INFO | Downloaded 34GiB to GPU in 2.50082s (13.5955GiB/s) | 20260508 11:18:49.615 | INFO | Analyzing chunk frames = [7690752,7691264) | 20260508 11:18:49.615 | INFO | MemInfo: { | 20260508 11:18:49.615 | INFO | "gpu" : " 17 GB", | 20260508 11:18:49.615 | INFO | "huge" : "", | 20260508 11:18:49.615 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:49.615 | INFO | "malloc" : " 14 GB", | 20260508 11:18:49.615 | INFO | "pinned" : " 29 GB" | 20260508 11:18:49.615 | INFO | } | 20260508 11:18:49.615 | INFO | Perf report for the last 5.13613 seconds: | 20260508 11:18:49.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:49.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.615 | INFO | ANALYSIS : 65.1%, 3.0, 101.2ms, 0 | 20260508 11:18:49.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.616 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:18:49.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:49.616 | INFO | Uploaded 36GiB to GPU in 2.30737s (15.6022GiB/s) | 20260508 11:18:49.616 | INFO | Downloaded 34GiB to GPU in 2.44367s (13.9135GiB/s) | 20260508 11:18:54.758 | INFO | Analyzing chunk frames = [7691264,7691776) | 20260508 11:18:54.758 | INFO | MemInfo: { | 20260508 11:18:54.758 | INFO | "gpu" : " 17 GB", | 20260508 11:18:54.758 | INFO | "huge" : "", | 20260508 11:18:54.758 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:54.758 | INFO | "malloc" : " 14 GB", | 20260508 11:18:54.758 | INFO | "pinned" : " 29 GB" | 20260508 11:18:54.758 | INFO | } | 20260508 11:18:54.758 | INFO | Perf report for the last 5.14265 seconds: | 20260508 11:18:54.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:54.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | ANALYSIS : 65.1%, 3.0, 101.6ms, 0 | 20260508 11:18:54.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:18:54.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:54.758 | INFO | Uploaded 36GiB to GPU in 2.38453s (15.0973GiB/s) | 20260508 11:18:54.758 | INFO | Downloaded 34GiB to GPU in 2.44165s (13.925GiB/s) | 20260508 11:18:59.897 | INFO | Analyzing chunk frames = [7691776,7692288) | 20260508 11:18:59.897 | INFO | MemInfo: { | 20260508 11:18:59.897 | INFO | "gpu" : " 17 GB", | 20260508 11:18:59.897 | INFO | "huge" : "", | 20260508 11:18:59.897 | INFO | "huge_shared" : " 29 GB", | 20260508 11:18:59.897 | INFO | "malloc" : " 14 GB", | 20260508 11:18:59.897 | INFO | "pinned" : " 29 GB" | 20260508 11:18:59.897 | INFO | } | 20260508 11:18:59.897 | INFO | Perf report for the last 5.13915 seconds: | 20260508 11:18:59.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:18:59.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.897 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 11:18:59.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.897 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 11:18:59.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:18:59.898 | INFO | Uploaded 36GiB to GPU in 2.37992s (15.1265GiB/s) | 20260508 11:18:59.898 | INFO | Downloaded 34GiB to GPU in 2.44882s (13.8842GiB/s) | 20260508 11:18:59.898 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:18:59.898 | INFO | Total mem usage: 16.7 GiB | 20260508 11:18:59.898 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:18:59.898 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -13.9%) | 20260508 11:18:59.898 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:18:59.898 | INFO | High Water Breakdown: | 20260508 11:18:59.898 | INFO | 59.9% : 8.78 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:18:59.898 | INFO | 39.9% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:18:59.898 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:18:59.898 | INFO | ------------------------------------------ | 20260508 11:18:59.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:18:59.898 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:18:59.898 | INFO | Total mem usage: 26.5 GiB | 20260508 11:18:59.898 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:18:59.898 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:18:59.898 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:18:59.898 | INFO | High Water Breakdown: | 20260508 11:18:59.898 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:18:59.898 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:18:59.898 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:18:59.898 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:18:59.898 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:18:59.898 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:18:59.898 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:18:59.898 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:18:59.898 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:18:59.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:18:59.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:18:59.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:18:59.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:18:59.898 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:18:59.898 | INFO | ------------------------------------------ | 20260508 11:18:59.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:18:59.898 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:18:59.898 | INFO | Total mem usage: 27.8 GiB | 20260508 11:18:59.898 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:18:59.898 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:18:59.898 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:18:59.898 | INFO | High Water Breakdown: | 20260508 11:18:59.898 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:18:59.898 | INFO | ------------------------------------------ | 20260508 11:18:59.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:18:59.898 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:18:59.898 | INFO | Total mem usage: 15.7 GiB | 20260508 11:18:59.898 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:18:59.898 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:18:59.898 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:18:59.898 | INFO | High Water Breakdown: | 20260508 11:18:59.898 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:18:59.898 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:18:59.898 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:18:59.898 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:18:59.898 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:18:59.898 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:18:59.898 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:18:59.898 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:18:59.898 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:18:59.898 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:18:59.898 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:18:59.898 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:18:59.898 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:18:59.898 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:18:59.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:18:59.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:18:59.899 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:18:59.899 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:18:59.899 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:18:59.899 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:18:59.899 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:18:59.899 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:18:59.899 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:18:59.899 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:18:59.899 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:18:59.899 | INFO | ------------------------------------------ | 20260508 11:18:59.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:18:59.899 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:18:59.899 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:18:59.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:18:59.899 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 5.41e-01 | 20260508 11:18:59.899 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.11e-02 / 5.55e-03 | 20260508 11:18:59.899 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.49e+00, 5.92e-01 / 1.49e-01 | 20260508 11:18:59.899 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.48e+00, 2.20e-01 / 1.13e-01 | 20260508 11:18:59.899 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.53e+01, 3.80e+00 / 8.88e-01 | 20260508 11:18:59.899 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.79e+00 / 1.26e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.08e-01 / 2.58e-01 | 20260508 11:18:59.899 | INFO | PerfReport: 3.00%, Metrics -- 9.80e-01 / 1.12e+00, 2.44e-01 / 1.16e-01 | 20260508 11:18:59.899 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.30e-01, 4.82e-02 / 2.95e-02 | 20260508 11:18:59.899 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:18:59.899 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:18:59.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:18:59.899 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 24.15%, Upload -- 2.54e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 2.04%, LockUpload -- 2.14e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 2.69%, LockDownload -- 2.83e+00 / 0.00e+00, 4.29e+00 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 15.26%, LockCompute -- 1.60e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:18:59.899 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260508 11:19:00.001 | INFO | Starting a round of Baz IO | 20260508 11:19:03.404 | INFO | Populating RTMetrics report | 20260508 11:19:03.405 | INFO | RTMetrics report populated | 20260508 11:19:03.405 | INFO | Opening tmp RTMetrics json file | 20260508 11:19:03.412 | INFO | Serializing RTMetrics to json | 20260508 11:19:03.413 | INFO | Serialization done, swapping tmp file now... | 20260508 11:19:03.414 | INFO | RTMetrics done | 20260508 11:19:03.471 | INFO | Finished a round of Baz IO: Wrote 1.439 GiB in 3.469 seconds | 20260508 11:19:03.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:19:03.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196604% / 0% / 6.35783e-05% | 20260508 11:19:03.471 | INFO | Baz IO overall: | 20260508 11:19:03.471 | INFO | BytesWritten : 3321155444288 | 20260508 11:19:03.471 | INFO | NumEvents : 1595265774806 | 20260508 11:19:05.038 | INFO | Analyzing chunk frames = [7692288,7692800) | 20260508 11:19:05.038 | INFO | MemInfo: { | 20260508 11:19:05.038 | INFO | "gpu" : " 17 GB", | 20260508 11:19:05.038 | INFO | "huge" : "", | 20260508 11:19:05.038 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:05.038 | INFO | "malloc" : " 14 GB", | 20260508 11:19:05.038 | INFO | "pinned" : " 29 GB" | 20260508 11:19:05.038 | INFO | } | 20260508 11:19:05.038 | INFO | Perf report for the last 5.14079 seconds: | 20260508 11:19:05.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:05.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:05.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:05.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:05.038 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:19:05.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:05.038 | INFO | RT_METRICS : 39.8%, 1.0, 20.9ms, 0 | 20260508 11:19:05.038 | INFO | PRE_HQ : 65.8%, 1.7, 56.5ms, 0 | 20260508 11:19:05.038 | INFO | BAZWRITER : 66.8%, 2.5, 86.3ms, 0 | 20260508 11:19:05.038 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:19:05.039 | INFO | Uploaded 36GiB to GPU in 2.6117s (13.7841GiB/s) | 20260508 11:19:05.039 | INFO | Downloaded 38GiB to GPU in 2.71444s (13.9992GiB/s) | 20260508 11:19:10.177 | INFO | Analyzing chunk frames = [7692800,7693312) | 20260508 11:19:10.177 | INFO | MemInfo: { | 20260508 11:19:10.177 | INFO | "gpu" : " 17 GB", | 20260508 11:19:10.177 | INFO | "huge" : "", | 20260508 11:19:10.177 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:10.177 | INFO | "malloc" : " 14 GB", | 20260508 11:19:10.177 | INFO | "pinned" : " 29 GB" | 20260508 11:19:10.177 | INFO | } | 20260508 11:19:10.177 | INFO | Perf report for the last 5.13898 seconds: | 20260508 11:19:10.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:10.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.177 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 11:19:10.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.177 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:19:10.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:10.178 | INFO | Uploaded 36GiB to GPU in 2.48626s (14.4796GiB/s) | 20260508 11:19:10.178 | INFO | Downloaded 34GiB to GPU in 2.46008s (13.8207GiB/s) | 20260508 11:19:15.315 | INFO | Analyzing chunk frames = [7693312,7693824) | 20260508 11:19:15.315 | INFO | MemInfo: { | 20260508 11:19:15.315 | INFO | "gpu" : " 17 GB", | 20260508 11:19:15.315 | INFO | "huge" : "", | 20260508 11:19:15.315 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:15.315 | INFO | "malloc" : " 14 GB", | 20260508 11:19:15.315 | INFO | "pinned" : " 29 GB" | 20260508 11:19:15.315 | INFO | } | 20260508 11:19:15.315 | INFO | Perf report for the last 5.13813 seconds: | 20260508 11:19:15.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:15.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.315 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:19:15.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.315 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:19:15.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:15.316 | INFO | Uploaded 36GiB to GPU in 2.4937s (14.4364GiB/s) | 20260508 11:19:15.316 | INFO | Downloaded 34GiB to GPU in 2.44716s (13.8936GiB/s) | 20260508 11:19:20.457 | INFO | Analyzing chunk frames = [7693824,7694336) | 20260508 11:19:20.457 | INFO | MemInfo: { | 20260508 11:19:20.457 | INFO | "gpu" : " 17 GB", | 20260508 11:19:20.457 | INFO | "huge" : "", | 20260508 11:19:20.457 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:20.457 | INFO | "malloc" : " 14 GB", | 20260508 11:19:20.457 | INFO | "pinned" : " 29 GB" | 20260508 11:19:20.457 | INFO | } | 20260508 11:19:20.457 | INFO | Perf report for the last 5.14181 seconds: | 20260508 11:19:20.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:20.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:19:20.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:19:20.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:20.457 | INFO | Uploaded 36GiB to GPU in 2.47677s (14.5351GiB/s) | 20260508 11:19:20.458 | INFO | Downloaded 34GiB to GPU in 2.44672s (13.8962GiB/s) | 20260508 11:19:25.592 | INFO | Analyzing chunk frames = [7694336,7694848) | 20260508 11:19:25.592 | INFO | MemInfo: { | 20260508 11:19:25.592 | INFO | "gpu" : " 17 GB", | 20260508 11:19:25.592 | INFO | "huge" : "", | 20260508 11:19:25.592 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:25.592 | INFO | "malloc" : " 14 GB", | 20260508 11:19:25.592 | INFO | "pinned" : " 29 GB" | 20260508 11:19:25.592 | INFO | } | 20260508 11:19:25.592 | INFO | Perf report for the last 5.13539 seconds: | 20260508 11:19:25.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:25.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.592 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:19:25.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.592 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 11:19:25.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:25.593 | INFO | Uploaded 36GiB to GPU in 2.30223s (15.637GiB/s) | 20260508 11:19:25.593 | INFO | Downloaded 34GiB to GPU in 2.49352s (13.6353GiB/s) | 20260508 11:19:30.737 | INFO | Analyzing chunk frames = [7694848,7695360) | 20260508 11:19:30.737 | INFO | MemInfo: { | 20260508 11:19:30.737 | INFO | "gpu" : " 17 GB", | 20260508 11:19:30.737 | INFO | "huge" : "", | 20260508 11:19:30.737 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:30.737 | INFO | "malloc" : " 14 GB", | 20260508 11:19:30.737 | INFO | "pinned" : " 29 GB" | 20260508 11:19:30.737 | INFO | } | 20260508 11:19:30.737 | INFO | Perf report for the last 5.14498 seconds: | 20260508 11:19:30.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:30.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.737 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:19:30.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.738 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:19:30.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:30.738 | INFO | Uploaded 36GiB to GPU in 2.39081s (15.0577GiB/s) | 20260508 11:19:30.738 | INFO | Downloaded 34GiB to GPU in 2.43178s (13.9815GiB/s) | 20260508 11:19:35.869 | INFO | Analyzing chunk frames = [7695360,7695872) | 20260508 11:19:35.869 | INFO | MemInfo: { | 20260508 11:19:35.869 | INFO | "gpu" : " 17 GB", | 20260508 11:19:35.869 | INFO | "huge" : "", | 20260508 11:19:35.869 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:35.869 | INFO | "malloc" : " 14 GB", | 20260508 11:19:35.869 | INFO | "pinned" : " 29 GB" | 20260508 11:19:35.869 | INFO | } | 20260508 11:19:35.869 | INFO | Perf report for the last 5.13194 seconds: | 20260508 11:19:35.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:35.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.869 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 11:19:35.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.870 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260508 11:19:35.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:35.870 | INFO | Uploaded 36GiB to GPU in 2.46521s (14.6032GiB/s) | 20260508 11:19:35.870 | INFO | Downloaded 34GiB to GPU in 2.44509s (13.9054GiB/s) | 20260508 11:19:41.014 | INFO | Analyzing chunk frames = [7695872,7696384) | 20260508 11:19:41.014 | INFO | MemInfo: { | 20260508 11:19:41.014 | INFO | "gpu" : " 17 GB", | 20260508 11:19:41.014 | INFO | "huge" : "", | 20260508 11:19:41.014 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:41.014 | INFO | "malloc" : " 14 GB", | 20260508 11:19:41.014 | INFO | "pinned" : " 29 GB" | 20260508 11:19:41.014 | INFO | } | 20260508 11:19:41.014 | INFO | Perf report for the last 5.14484 seconds: | 20260508 11:19:41.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:41.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.014 | INFO | ANALYSIS : 65.7%, 3.0, 102.9ms, 0 | 20260508 11:19:41.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.015 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260508 11:19:41.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:41.015 | INFO | Uploaded 36GiB to GPU in 2.45668s (14.6539GiB/s) | 20260508 11:19:41.015 | INFO | Downloaded 34GiB to GPU in 2.43209s (13.9797GiB/s) | 20260508 11:19:41.127 | INFO | Starting a round of Baz IO | 20260508 11:19:44.518 | INFO | Populating RTMetrics report | 20260508 11:19:44.519 | INFO | RTMetrics report populated | 20260508 11:19:44.519 | INFO | Opening tmp RTMetrics json file | 20260508 11:19:44.526 | INFO | Serializing RTMetrics to json | 20260508 11:19:44.528 | INFO | Serialization done, swapping tmp file now... | 20260508 11:19:44.528 | INFO | RTMetrics done | 20260508 11:19:44.587 | INFO | Finished a round of Baz IO: Wrote 1.438 GiB in 3.46 seconds | 20260508 11:19:44.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:19:44.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195456% / 0% / 4.76837e-05% | 20260508 11:19:44.587 | INFO | Baz IO overall: | 20260508 11:19:44.587 | INFO | BytesWritten : 3322699751769 | 20260508 11:19:44.587 | INFO | NumEvents : 1595992538815 | 20260508 11:19:46.153 | INFO | Analyzing chunk frames = [7696384,7696896) | 20260508 11:19:46.153 | INFO | MemInfo: { | 20260508 11:19:46.153 | INFO | "gpu" : " 17 GB", | 20260508 11:19:46.153 | INFO | "huge" : "", | 20260508 11:19:46.153 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:46.153 | INFO | "malloc" : " 14 GB", | 20260508 11:19:46.153 | INFO | "pinned" : " 29 GB" | 20260508 11:19:46.153 | INFO | } | 20260508 11:19:46.154 | INFO | Perf report for the last 5.13927 seconds: | 20260508 11:19:46.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:46.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:46.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:46.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:46.154 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 11:19:46.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:46.154 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260508 11:19:46.154 | INFO | PRE_HQ : 66.3%, 1.8, 61.7ms, 0 | 20260508 11:19:46.154 | INFO | BAZWRITER : 66.7%, 2.7, 92.0ms, 0 | 20260508 11:19:46.154 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:19:46.154 | INFO | Uploaded 36GiB to GPU in 2.63355s (13.6697GiB/s) | 20260508 11:19:46.154 | INFO | Downloaded 38GiB to GPU in 2.70896s (14.0275GiB/s) | 20260508 11:19:51.291 | INFO | Analyzing chunk frames = [7696896,7697408) | 20260508 11:19:51.291 | INFO | MemInfo: { | 20260508 11:19:51.291 | INFO | "gpu" : " 17 GB", | 20260508 11:19:51.291 | INFO | "huge" : "", | 20260508 11:19:51.291 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:51.291 | INFO | "malloc" : " 14 GB", | 20260508 11:19:51.291 | INFO | "pinned" : " 29 GB" | 20260508 11:19:51.291 | INFO | } | 20260508 11:19:51.291 | INFO | Perf report for the last 5.13793 seconds: | 20260508 11:19:51.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:51.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.292 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 11:19:51.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.292 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:19:51.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:51.292 | INFO | Uploaded 36GiB to GPU in 2.46871s (14.5825GiB/s) | 20260508 11:19:51.292 | INFO | Downloaded 34GiB to GPU in 2.47224s (13.7527GiB/s) | 20260508 11:19:56.429 | INFO | Analyzing chunk frames = [7697408,7697920) | 20260508 11:19:56.429 | INFO | MemInfo: { | 20260508 11:19:56.429 | INFO | "gpu" : " 17 GB", | 20260508 11:19:56.429 | INFO | "huge" : "", | 20260508 11:19:56.429 | INFO | "huge_shared" : " 29 GB", | 20260508 11:19:56.429 | INFO | "malloc" : " 14 GB", | 20260508 11:19:56.429 | INFO | "pinned" : " 29 GB" | 20260508 11:19:56.429 | INFO | } | 20260508 11:19:56.429 | INFO | Perf report for the last 5.13781 seconds: | 20260508 11:19:56.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:19:56.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.429 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:19:56.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.429 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:19:56.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:19:56.430 | INFO | Uploaded 36GiB to GPU in 2.45991s (14.6347GiB/s) | 20260508 11:19:56.430 | INFO | Downloaded 34GiB to GPU in 2.44875s (13.8846GiB/s) | 20260508 11:20:01.567 | INFO | Analyzing chunk frames = [7697920,7698432) | 20260508 11:20:01.567 | INFO | MemInfo: { | 20260508 11:20:01.567 | INFO | "gpu" : " 17 GB", | 20260508 11:20:01.567 | INFO | "huge" : "", | 20260508 11:20:01.567 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:01.567 | INFO | "malloc" : " 14 GB", | 20260508 11:20:01.567 | INFO | "pinned" : " 29 GB" | 20260508 11:20:01.567 | INFO | } | 20260508 11:20:01.567 | INFO | Perf report for the last 5.13792 seconds: | 20260508 11:20:01.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:01.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.567 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:20:01.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.568 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:20:01.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:01.568 | INFO | Uploaded 36GiB to GPU in 2.47782s (14.5289GiB/s) | 20260508 11:20:01.568 | INFO | Downloaded 34GiB to GPU in 2.45784s (13.8333GiB/s) | 20260508 11:20:06.711 | INFO | Analyzing chunk frames = [7698432,7698944) | 20260508 11:20:06.711 | INFO | MemInfo: { | 20260508 11:20:06.711 | INFO | "gpu" : " 17 GB", | 20260508 11:20:06.711 | INFO | "huge" : "", | 20260508 11:20:06.711 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:06.711 | INFO | "malloc" : " 14 GB", | 20260508 11:20:06.711 | INFO | "pinned" : " 29 GB" | 20260508 11:20:06.711 | INFO | } | 20260508 11:20:06.711 | INFO | Perf report for the last 5.14381 seconds: | 20260508 11:20:06.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:06.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 11:20:06.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:20:06.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:06.711 | INFO | Uploaded 36GiB to GPU in 2.27795s (15.8037GiB/s) | 20260508 11:20:06.712 | INFO | Downloaded 34GiB to GPU in 2.50406s (13.5779GiB/s) | 20260508 11:20:11.855 | INFO | Analyzing chunk frames = [7698944,7699456) | 20260508 11:20:11.855 | INFO | MemInfo: { | 20260508 11:20:11.855 | INFO | "gpu" : " 17 GB", | 20260508 11:20:11.855 | INFO | "huge" : "", | 20260508 11:20:11.855 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:11.855 | INFO | "malloc" : " 14 GB", | 20260508 11:20:11.855 | INFO | "pinned" : " 29 GB" | 20260508 11:20:11.855 | INFO | } | 20260508 11:20:11.855 | INFO | Perf report for the last 5.1444 seconds: | 20260508 11:20:11.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:11.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.856 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 11:20:11.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.856 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:20:11.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:11.856 | INFO | Uploaded 36GiB to GPU in 2.34913s (15.3248GiB/s) | 20260508 11:20:11.856 | INFO | Downloaded 34GiB to GPU in 2.45034s (13.8756GiB/s) | 20260508 11:20:16.988 | INFO | Analyzing chunk frames = [7699456,7699968) | 20260508 11:20:16.988 | INFO | MemInfo: { | 20260508 11:20:16.988 | INFO | "gpu" : " 17 GB", | 20260508 11:20:16.988 | INFO | "huge" : "", | 20260508 11:20:16.988 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:16.988 | INFO | "malloc" : " 14 GB", | 20260508 11:20:16.988 | INFO | "pinned" : " 29 GB" | 20260508 11:20:16.988 | INFO | } | 20260508 11:20:16.988 | INFO | Perf report for the last 5.13295 seconds: | 20260508 11:20:16.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:16.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.988 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:20:16.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.989 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:20:16.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:16.989 | INFO | Uploaded 36GiB to GPU in 2.41769s (14.8902GiB/s) | 20260508 11:20:16.989 | INFO | Downloaded 34GiB to GPU in 2.47386s (13.7437GiB/s) | 20260508 11:20:22.124 | INFO | Analyzing chunk frames = [7699968,7700480) | 20260508 11:20:22.124 | INFO | MemInfo: { | 20260508 11:20:22.124 | INFO | "gpu" : " 17 GB", | 20260508 11:20:22.124 | INFO | "huge" : "", | 20260508 11:20:22.124 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:22.124 | INFO | "malloc" : " 14 GB", | 20260508 11:20:22.124 | INFO | "pinned" : " 29 GB" | 20260508 11:20:22.124 | INFO | } | 20260508 11:20:22.124 | INFO | Perf report for the last 5.13593 seconds: | 20260508 11:20:22.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:22.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.124 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:20:22.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.124 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:20:22.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:22.125 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:20:22.125 | INFO | Total mem usage: 16.7 GiB | 20260508 11:20:22.125 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:20:22.125 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 11:20:22.125 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:20:22.125 | INFO | High Water Breakdown: | 20260508 11:20:22.125 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:20:22.125 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:20:22.125 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:20:22.125 | INFO | ------------------------------------------ | 20260508 11:20:22.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:20:22.125 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:20:22.125 | INFO | Total mem usage: 26.5 GiB | 20260508 11:20:22.125 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:20:22.125 | INFO | Uploaded 36GiB to GPU in 2.43304s (14.7963GiB/s) | 20260508 11:20:22.125 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:20:22.125 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:20:22.125 | INFO | High Water Breakdown: | 20260508 11:20:22.125 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:20:22.125 | INFO | Downloaded 34GiB to GPU in 2.47794s (13.7211GiB/s) | 20260508 11:20:22.125 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:20:22.125 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:20:22.125 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:20:22.125 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:20:22.125 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:20:22.125 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:20:22.125 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:20:22.125 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:20:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:20:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:20:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:20:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:20:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:20:22.125 | INFO | ------------------------------------------ | 20260508 11:20:22.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:20:22.125 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:20:22.125 | INFO | Total mem usage: 27.8 GiB | 20260508 11:20:22.125 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:20:22.126 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:20:22.126 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:20:22.126 | INFO | High Water Breakdown: | 20260508 11:20:22.126 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:20:22.126 | INFO | ------------------------------------------ | 20260508 11:20:22.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:20:22.126 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:20:22.126 | INFO | Total mem usage: 15.7 GiB | 20260508 11:20:22.126 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:20:22.126 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:20:22.126 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:20:22.126 | INFO | High Water Breakdown: | 20260508 11:20:22.126 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:20:22.126 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:20:22.126 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:20:22.126 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:20:22.126 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:20:22.126 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:20:22.126 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:20:22.126 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:20:22.126 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:20:22.126 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:20:22.126 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:20:22.126 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:20:22.126 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:20:22.126 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:20:22.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:20:22.126 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:20:22.126 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:20:22.126 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:20:22.126 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:20:22.126 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:20:22.126 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:20:22.126 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:20:22.126 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:20:22.126 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:20:22.126 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:20:22.126 | INFO | ------------------------------------------ | 20260508 11:20:22.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:20:22.126 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:20:22.126 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:20:22.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:20:22.126 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 5.36e-01 | 20260508 11:20:22.126 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.02e-01, 1.16e-02 / 6.66e-03 | 20260508 11:20:22.126 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.34e+00, 5.97e-01 / 2.73e-01 | 20260508 11:20:22.126 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 1.13e-01 | 20260508 11:20:22.126 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.54e+01, 3.84e+00 / 1.19e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.74e+00 / 1.29e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.09e-01 / 2.89e-01 | 20260508 11:20:22.126 | INFO | PerfReport: 3.00%, Metrics -- 9.79e-01 / 1.12e+00, 2.41e-01 / 1.06e-01 | 20260508 11:20:22.126 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.31e-01, 4.81e-02 / 3.02e-02 | 20260508 11:20:22.126 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:20:22.126 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:20:22.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:20:22.126 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 24.27%, Upload -- 2.55e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 24.68%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 31.02%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 2.16%, LockUpload -- 2.28e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 2.61%, LockDownload -- 2.75e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 15.23%, LockCompute -- 1.60e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:20:22.126 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 11:20:22.222 | INFO | Starting a round of Baz IO | 20260508 11:20:25.638 | INFO | Populating RTMetrics report | 20260508 11:20:25.638 | INFO | RTMetrics report populated | 20260508 11:20:25.638 | INFO | Opening tmp RTMetrics json file | 20260508 11:20:25.646 | INFO | Serializing RTMetrics to json | 20260508 11:20:25.647 | INFO | Serialization done, swapping tmp file now... | 20260508 11:20:25.648 | INFO | RTMetrics done | 20260508 11:20:25.703 | INFO | Finished a round of Baz IO: Wrote 1.437 GiB in 3.481 seconds | 20260508 11:20:25.703 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.00% / 59.88% | 20260508 11:20:25.703 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194883% / 0% / 3.17891e-05% | 20260508 11:20:25.703 | INFO | Baz IO overall: | 20260508 11:20:25.703 | INFO | BytesWritten : 3324242533242 | 20260508 11:20:25.703 | INFO | NumEvents : 1596718093880 | 20260508 11:20:27.269 | INFO | Analyzing chunk frames = [7700480,7700992) | 20260508 11:20:27.269 | INFO | MemInfo: { | 20260508 11:20:27.269 | INFO | "gpu" : " 17 GB", | 20260508 11:20:27.269 | INFO | "huge" : "", | 20260508 11:20:27.269 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:27.269 | INFO | "malloc" : " 14 GB", | 20260508 11:20:27.269 | INFO | "pinned" : " 29 GB" | 20260508 11:20:27.269 | INFO | } | 20260508 11:20:27.269 | INFO | Perf report for the last 5.14473 seconds: | 20260508 11:20:27.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:27.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:27.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:27.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:27.269 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 11:20:27.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:27.269 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:20:27.269 | INFO | PRE_HQ : 65.4%, 1.6, 54.0ms, 0 | 20260508 11:20:27.269 | INFO | BAZWRITER : 66.9%, 2.7, 94.7ms, 0 | 20260508 11:20:27.269 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:20:27.269 | INFO | Uploaded 36GiB to GPU in 2.58994s (13.9GiB/s) | 20260508 11:20:27.269 | INFO | Downloaded 38GiB to GPU in 2.71572s (13.9926GiB/s) | 20260508 11:20:32.404 | INFO | Analyzing chunk frames = [7700992,7701504) | 20260508 11:20:32.404 | INFO | MemInfo: { | 20260508 11:20:32.404 | INFO | "gpu" : " 17 GB", | 20260508 11:20:32.404 | INFO | "huge" : "", | 20260508 11:20:32.404 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:32.404 | INFO | "malloc" : " 14 GB", | 20260508 11:20:32.404 | INFO | "pinned" : " 29 GB" | 20260508 11:20:32.404 | INFO | } | 20260508 11:20:32.404 | INFO | Perf report for the last 5.13531 seconds: | 20260508 11:20:32.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:32.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.404 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:20:32.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.405 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:20:32.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:32.405 | INFO | Uploaded 36GiB to GPU in 2.45327s (14.6743GiB/s) | 20260508 11:20:32.405 | INFO | Downloaded 34GiB to GPU in 2.46916s (13.7699GiB/s) | 20260508 11:20:37.543 | INFO | Analyzing chunk frames = [7701504,7702016) | 20260508 11:20:37.543 | INFO | MemInfo: { | 20260508 11:20:37.543 | INFO | "gpu" : " 17 GB", | 20260508 11:20:37.543 | INFO | "huge" : "", | 20260508 11:20:37.543 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:37.543 | INFO | "malloc" : " 14 GB", | 20260508 11:20:37.543 | INFO | "pinned" : " 29 GB" | 20260508 11:20:37.543 | INFO | } | 20260508 11:20:37.543 | INFO | Perf report for the last 5.13902 seconds: | 20260508 11:20:37.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:37.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:20:37.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:20:37.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:37.544 | INFO | Uploaded 36GiB to GPU in 2.4827s (14.5004GiB/s) | 20260508 11:20:37.544 | INFO | Downloaded 34GiB to GPU in 2.44484s (13.9068GiB/s) | 20260508 11:20:42.684 | INFO | Analyzing chunk frames = [7702016,7702528) | 20260508 11:20:42.684 | INFO | MemInfo: { | 20260508 11:20:42.684 | INFO | "gpu" : " 17 GB", | 20260508 11:20:42.684 | INFO | "huge" : "", | 20260508 11:20:42.684 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:42.684 | INFO | "malloc" : " 14 GB", | 20260508 11:20:42.684 | INFO | "pinned" : " 29 GB" | 20260508 11:20:42.684 | INFO | } | 20260508 11:20:42.684 | INFO | Perf report for the last 5.14076 seconds: | 20260508 11:20:42.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:42.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.684 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 11:20:42.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.684 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:20:42.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:42.685 | INFO | Uploaded 36GiB to GPU in 2.4656s (14.6009GiB/s) | 20260508 11:20:42.685 | INFO | Downloaded 34GiB to GPU in 2.42927s (13.996GiB/s) | 20260508 11:20:47.824 | INFO | Analyzing chunk frames = [7702528,7703040) | 20260508 11:20:47.824 | INFO | MemInfo: { | 20260508 11:20:47.824 | INFO | "gpu" : " 17 GB", | 20260508 11:20:47.824 | INFO | "huge" : "", | 20260508 11:20:47.824 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:47.824 | INFO | "malloc" : " 14 GB", | 20260508 11:20:47.824 | INFO | "pinned" : " 29 GB" | 20260508 11:20:47.824 | INFO | } | 20260508 11:20:47.824 | INFO | Perf report for the last 5.14026 seconds: | 20260508 11:20:47.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:47.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 11:20:47.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:20:47.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:47.825 | INFO | Uploaded 36GiB to GPU in 2.4332s (14.7953GiB/s) | 20260508 11:20:47.825 | INFO | Downloaded 34GiB to GPU in 2.52814s (13.4486GiB/s) | 20260508 11:20:52.969 | INFO | Analyzing chunk frames = [7703040,7703552) | 20260508 11:20:52.969 | INFO | MemInfo: { | 20260508 11:20:52.969 | INFO | "gpu" : " 17 GB", | 20260508 11:20:52.969 | INFO | "huge" : "", | 20260508 11:20:52.969 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:52.969 | INFO | "malloc" : " 14 GB", | 20260508 11:20:52.970 | INFO | "pinned" : " 29 GB" | 20260508 11:20:52.970 | INFO | } | 20260508 11:20:52.970 | INFO | Perf report for the last 5.14516 seconds: | 20260508 11:20:52.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:52.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 11:20:52.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 11:20:52.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:52.970 | INFO | Uploaded 36GiB to GPU in 2.41342s (14.9166GiB/s) | 20260508 11:20:52.970 | INFO | Downloaded 34GiB to GPU in 2.42299s (14.0322GiB/s) | 20260508 11:20:58.106 | INFO | Analyzing chunk frames = [7703552,7704064) | 20260508 11:20:58.106 | INFO | MemInfo: { | 20260508 11:20:58.106 | INFO | "gpu" : " 17 GB", | 20260508 11:20:58.106 | INFO | "huge" : "", | 20260508 11:20:58.106 | INFO | "huge_shared" : " 29 GB", | 20260508 11:20:58.106 | INFO | "malloc" : " 14 GB", | 20260508 11:20:58.106 | INFO | "pinned" : " 29 GB" | 20260508 11:20:58.106 | INFO | } | 20260508 11:20:58.106 | INFO | Perf report for the last 5.13644 seconds: | 20260508 11:20:58.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:20:58.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.106 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:20:58.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.106 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:20:58.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:20:58.107 | INFO | Uploaded 36GiB to GPU in 2.47745s (14.531GiB/s) | 20260508 11:20:58.107 | INFO | Downloaded 34GiB to GPU in 2.4509s (13.8724GiB/s) | 20260508 11:21:03.240 | INFO | Analyzing chunk frames = [7704064,7704576) | 20260508 11:21:03.240 | INFO | MemInfo: { | 20260508 11:21:03.240 | INFO | "gpu" : " 17 GB", | 20260508 11:21:03.240 | INFO | "huge" : "", | 20260508 11:21:03.240 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:03.240 | INFO | "malloc" : " 14 GB", | 20260508 11:21:03.240 | INFO | "pinned" : " 29 GB" | 20260508 11:21:03.240 | INFO | } | 20260508 11:21:03.241 | INFO | Perf report for the last 5.1344 seconds: | 20260508 11:21:03.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:03.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 11:21:03.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:21:03.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:03.241 | INFO | Uploaded 36GiB to GPU in 2.48401s (14.4927GiB/s) | 20260508 11:21:03.241 | INFO | Downloaded 34GiB to GPU in 2.45078s (13.8731GiB/s) | 20260508 11:21:03.338 | INFO | Starting a round of Baz IO | 20260508 11:21:06.749 | INFO | Populating RTMetrics report | 20260508 11:21:06.749 | INFO | RTMetrics report populated | 20260508 11:21:06.749 | INFO | Opening tmp RTMetrics json file | 20260508 11:21:06.755 | INFO | Serializing RTMetrics to json | 20260508 11:21:06.757 | INFO | Serialization done, swapping tmp file now... | 20260508 11:21:06.757 | INFO | RTMetrics done | 20260508 11:21:06.812 | INFO | Finished a round of Baz IO: Wrote 1.438 GiB in 3.474 seconds | 20260508 11:21:06.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.88% | 20260508 11:21:06.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195694% / 7.94729e-06% / 6.35783e-05% | 20260508 11:21:06.812 | INFO | Baz IO overall: | 20260508 11:21:06.812 | INFO | BytesWritten : 3325786254845 | 20260508 11:21:06.812 | INFO | NumEvents : 1597444398228 | 20260508 11:21:08.375 | INFO | Analyzing chunk frames = [7704576,7705088) | 20260508 11:21:08.375 | INFO | MemInfo: { | 20260508 11:21:08.375 | INFO | "gpu" : " 17 GB", | 20260508 11:21:08.375 | INFO | "huge" : "", | 20260508 11:21:08.375 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:08.375 | INFO | "malloc" : " 14 GB", | 20260508 11:21:08.375 | INFO | "pinned" : " 29 GB" | 20260508 11:21:08.375 | INFO | } | 20260508 11:21:08.375 | INFO | Perf report for the last 5.13444 seconds: | 20260508 11:21:08.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:08.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:08.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:08.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:08.375 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 11:21:08.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:08.375 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:21:08.375 | INFO | PRE_HQ : 65.3%, 1.6, 54.0ms, 0 | 20260508 11:21:08.375 | INFO | BAZWRITER : 67.0%, 2.5, 85.1ms, 0 | 20260508 11:21:08.375 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:21:08.375 | INFO | Uploaded 36GiB to GPU in 2.59236s (13.8869GiB/s) | 20260508 11:21:08.375 | INFO | Downloaded 38GiB to GPU in 2.69852s (14.0818GiB/s) | 20260508 11:21:13.520 | INFO | Analyzing chunk frames = [7705088,7705600) | 20260508 11:21:13.520 | INFO | MemInfo: { | 20260508 11:21:13.520 | INFO | "gpu" : " 17 GB", | 20260508 11:21:13.520 | INFO | "huge" : "", | 20260508 11:21:13.520 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:13.520 | INFO | "malloc" : " 14 GB", | 20260508 11:21:13.520 | INFO | "pinned" : " 29 GB" | 20260508 11:21:13.520 | INFO | } | 20260508 11:21:13.520 | INFO | Perf report for the last 5.14519 seconds: | 20260508 11:21:13.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:13.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.520 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:21:13.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.520 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:21:13.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:13.521 | INFO | Uploaded 36GiB to GPU in 2.47077s (14.5703GiB/s) | 20260508 11:21:13.521 | INFO | Downloaded 34GiB to GPU in 2.46036s (13.8191GiB/s) | 20260508 11:21:18.660 | INFO | Analyzing chunk frames = [7705600,7706112) | 20260508 11:21:18.660 | INFO | MemInfo: { | 20260508 11:21:18.660 | INFO | "gpu" : " 17 GB", | 20260508 11:21:18.660 | INFO | "huge" : "", | 20260508 11:21:18.660 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:18.660 | INFO | "malloc" : " 14 GB", | 20260508 11:21:18.660 | INFO | "pinned" : " 29 GB" | 20260508 11:21:18.660 | INFO | } | 20260508 11:21:18.660 | INFO | Perf report for the last 5.14019 seconds: | 20260508 11:21:18.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:18.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.660 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 11:21:18.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.661 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:21:18.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:18.661 | INFO | Uploaded 36GiB to GPU in 2.51817s (14.2961GiB/s) | 20260508 11:21:18.661 | INFO | Downloaded 34GiB to GPU in 2.43024s (13.9904GiB/s) | 20260508 11:21:23.805 | INFO | Analyzing chunk frames = [7706112,7706624) | 20260508 11:21:23.805 | INFO | MemInfo: { | 20260508 11:21:23.805 | INFO | "gpu" : " 17 GB", | 20260508 11:21:23.805 | INFO | "huge" : "", | 20260508 11:21:23.805 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:23.805 | INFO | "malloc" : " 14 GB", | 20260508 11:21:23.805 | INFO | "pinned" : " 29 GB" | 20260508 11:21:23.805 | INFO | } | 20260508 11:21:23.806 | INFO | Perf report for the last 5.14517 seconds: | 20260508 11:21:23.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:23.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:21:23.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:21:23.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:23.806 | INFO | Uploaded 36GiB to GPU in 2.49667s (14.4192GiB/s) | 20260508 11:21:23.806 | INFO | Downloaded 34GiB to GPU in 2.44943s (13.8808GiB/s) | 20260508 11:21:28.941 | INFO | Analyzing chunk frames = [7706624,7707136) | 20260508 11:21:28.941 | INFO | MemInfo: { | 20260508 11:21:28.941 | INFO | "gpu" : " 17 GB", | 20260508 11:21:28.941 | INFO | "huge" : "", | 20260508 11:21:28.941 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:28.941 | INFO | "malloc" : " 14 GB", | 20260508 11:21:28.941 | INFO | "pinned" : " 29 GB" | 20260508 11:21:28.941 | INFO | } | 20260508 11:21:28.941 | INFO | Perf report for the last 5.13575 seconds: | 20260508 11:21:28.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:28.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.941 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:21:28.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.941 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 11:21:28.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:28.942 | INFO | Uploaded 36GiB to GPU in 2.43211s (14.802GiB/s) | 20260508 11:21:28.942 | INFO | Downloaded 34GiB to GPU in 2.51141s (13.5382GiB/s) | 20260508 11:21:34.076 | INFO | Analyzing chunk frames = [7707136,7707648) | 20260508 11:21:34.076 | INFO | MemInfo: { | 20260508 11:21:34.076 | INFO | "gpu" : " 17 GB", | 20260508 11:21:34.076 | INFO | "huge" : "", | 20260508 11:21:34.076 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:34.076 | INFO | "malloc" : " 14 GB", | 20260508 11:21:34.076 | INFO | "pinned" : " 29 GB" | 20260508 11:21:34.076 | INFO | } | 20260508 11:21:34.076 | INFO | Perf report for the last 5.1346 seconds: | 20260508 11:21:34.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:34.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:21:34.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:21:34.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:34.076 | INFO | Uploaded 36GiB to GPU in 2.38912s (15.0683GiB/s) | 20260508 11:21:34.076 | INFO | Downloaded 34GiB to GPU in 2.42553s (14.0175GiB/s) | 20260508 11:21:39.214 | INFO | Analyzing chunk frames = [7707648,7708160) | 20260508 11:21:39.214 | INFO | MemInfo: { | 20260508 11:21:39.214 | INFO | "gpu" : " 17 GB", | 20260508 11:21:39.214 | INFO | "huge" : "", | 20260508 11:21:39.214 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:39.214 | INFO | "malloc" : " 14 GB", | 20260508 11:21:39.214 | INFO | "pinned" : " 29 GB" | 20260508 11:21:39.214 | INFO | } | 20260508 11:21:39.214 | INFO | Perf report for the last 5.13826 seconds: | 20260508 11:21:39.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:39.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.214 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:21:39.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.214 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:21:39.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:39.215 | INFO | Uploaded 36GiB to GPU in 2.42811s (14.8264GiB/s) | 20260508 11:21:39.215 | INFO | Downloaded 34GiB to GPU in 2.44176s (13.9244GiB/s) | 20260508 11:21:44.357 | INFO | Analyzing chunk frames = [7708160,7708672) | 20260508 11:21:44.357 | INFO | MemInfo: { | 20260508 11:21:44.357 | INFO | "gpu" : " 17 GB", | 20260508 11:21:44.357 | INFO | "huge" : "", | 20260508 11:21:44.357 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:44.357 | INFO | "malloc" : " 14 GB", | 20260508 11:21:44.357 | INFO | "pinned" : " 29 GB" | 20260508 11:21:44.357 | INFO | } | 20260508 11:21:44.357 | INFO | Perf report for the last 5.14296 seconds: | 20260508 11:21:44.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:44.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 11:21:44.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:21:44.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:44.357 | INFO | Uploaded 36GiB to GPU in 2.42214s (14.8629GiB/s) | 20260508 11:21:44.357 | INFO | Downloaded 34GiB to GPU in 2.43374s (13.9703GiB/s) | 20260508 11:21:44.358 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:21:44.358 | INFO | Total mem usage: 16.7 GiB | 20260508 11:21:44.358 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:21:44.358 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14%) | 20260508 11:21:44.358 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:21:44.358 | INFO | High Water Breakdown: | 20260508 11:21:44.358 | INFO | 59.9% : 8.77 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:21:44.358 | INFO | 40% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:21:44.358 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:21:44.358 | INFO | ------------------------------------------ | 20260508 11:21:44.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:21:44.358 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:21:44.358 | INFO | Total mem usage: 26.5 GiB | 20260508 11:21:44.358 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:21:44.358 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:21:44.358 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:21:44.358 | INFO | High Water Breakdown: | 20260508 11:21:44.358 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:21:44.358 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:21:44.358 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:21:44.358 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:21:44.358 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:21:44.358 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:21:44.358 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:21:44.358 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:21:44.358 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:21:44.358 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:21:44.358 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:21:44.358 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:21:44.358 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:21:44.358 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:21:44.358 | INFO | ------------------------------------------ | 20260508 11:21:44.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:21:44.358 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:21:44.358 | INFO | Total mem usage: 27.8 GiB | 20260508 11:21:44.358 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:21:44.358 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:21:44.358 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:21:44.358 | INFO | High Water Breakdown: | 20260508 11:21:44.358 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:21:44.358 | INFO | ------------------------------------------ | 20260508 11:21:44.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:21:44.358 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:21:44.358 | INFO | Total mem usage: 15.7 GiB | 20260508 11:21:44.358 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:21:44.358 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:21:44.358 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:21:44.358 | INFO | High Water Breakdown: | 20260508 11:21:44.358 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:21:44.358 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:21:44.358 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:21:44.358 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:21:44.358 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:21:44.358 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:21:44.358 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:21:44.359 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:21:44.359 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:21:44.359 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:21:44.359 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:21:44.359 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:21:44.359 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:21:44.359 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:21:44.359 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:21:44.359 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:21:44.359 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:21:44.359 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:21:44.359 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:21:44.359 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:21:44.359 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:21:44.359 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:21:44.359 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:21:44.359 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:21:44.359 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:21:44.359 | INFO | ------------------------------------------ | 20260508 11:21:44.359 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:21:44.359 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:21:44.359 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:21:44.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:21:44.359 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 7.14e-01 | 20260508 11:21:44.359 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.05e-01, 1.12e-02 / 5.46e-03 | 20260508 11:21:44.359 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.47e+00, 5.85e-01 / 2.28e-01 | 20260508 11:21:44.359 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.27e-01 | 20260508 11:21:44.359 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.52e+01, 3.70e+00 / 9.89e-01 | 20260508 11:21:44.359 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.33e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.51e+00, 4.05e-01 / 2.37e-01 | 20260508 11:21:44.359 | INFO | PerfReport: 3.00%, Metrics -- 9.79e-01 / 1.09e+00, 2.43e-01 / 1.08e-01 | 20260508 11:21:44.359 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.24e-01, 4.67e-02 / 2.71e-02 | 20260508 11:21:44.359 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:21:44.359 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:21:44.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:21:44.359 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 24.47%, Upload -- 2.57e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 31.01%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 2.23%, LockUpload -- 2.34e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 2.52%, LockDownload -- 2.65e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 15.15%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:21:44.359 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260508 11:21:44.454 | INFO | Starting a round of Baz IO | 20260508 11:21:47.865 | INFO | Populating RTMetrics report | 20260508 11:21:47.865 | INFO | RTMetrics report populated | 20260508 11:21:47.865 | INFO | Opening tmp RTMetrics json file | 20260508 11:21:47.873 | INFO | Serializing RTMetrics to json | 20260508 11:21:47.875 | INFO | Serialization done, swapping tmp file now... | 20260508 11:21:47.875 | INFO | RTMetrics done | 20260508 11:21:47.935 | INFO | Finished a round of Baz IO: Wrote 1.438 GiB in 3.48 seconds | 20260508 11:21:47.935 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:21:47.935 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195368% / 7.94729e-06% / 7.15256e-05% | 20260508 11:21:47.935 | INFO | Baz IO overall: | 20260508 11:21:47.935 | INFO | BytesWritten : 3327330741580 | 20260508 11:21:47.935 | INFO | NumEvents : 1598171321698 | 20260508 11:21:49.500 | INFO | Analyzing chunk frames = [7708672,7709184) | 20260508 11:21:49.500 | INFO | MemInfo: { | 20260508 11:21:49.500 | INFO | "gpu" : " 17 GB", | 20260508 11:21:49.500 | INFO | "huge" : "", | 20260508 11:21:49.500 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:49.500 | INFO | "malloc" : " 14 GB", | 20260508 11:21:49.500 | INFO | "pinned" : " 29 GB" | 20260508 11:21:49.500 | INFO | } | 20260508 11:21:49.500 | INFO | Perf report for the last 5.1426 seconds: | 20260508 11:21:49.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:49.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:49.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:49.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:49.500 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 11:21:49.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:49.500 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:21:49.500 | INFO | PRE_HQ : 65.5%, 1.6, 54.2ms, 0 | 20260508 11:21:49.500 | INFO | BAZWRITER : 67.0%, 2.5, 84.8ms, 0 | 20260508 11:21:49.500 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:21:49.500 | INFO | Uploaded 36GiB to GPU in 2.62514s (13.7136GiB/s) | 20260508 11:21:49.500 | INFO | Downloaded 38GiB to GPU in 2.72462s (13.9469GiB/s) | 20260508 11:21:54.634 | INFO | Analyzing chunk frames = [7709184,7709696) | 20260508 11:21:54.634 | INFO | MemInfo: { | 20260508 11:21:54.634 | INFO | "gpu" : " 17 GB", | 20260508 11:21:54.634 | INFO | "huge" : "", | 20260508 11:21:54.634 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:54.634 | INFO | "malloc" : " 14 GB", | 20260508 11:21:54.634 | INFO | "pinned" : " 29 GB" | 20260508 11:21:54.635 | INFO | } | 20260508 11:21:54.635 | INFO | Perf report for the last 5.13477 seconds: | 20260508 11:21:54.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:54.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 11:21:54.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:21:54.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:54.635 | INFO | Uploaded 36GiB to GPU in 2.45827s (14.6445GiB/s) | 20260508 11:21:54.635 | INFO | Downloaded 34GiB to GPU in 2.4859s (13.6771GiB/s) | 20260508 11:21:59.777 | INFO | Analyzing chunk frames = [7709696,7710208) | 20260508 11:21:59.777 | INFO | MemInfo: { | 20260508 11:21:59.777 | INFO | "gpu" : " 17 GB", | 20260508 11:21:59.777 | INFO | "huge" : "", | 20260508 11:21:59.777 | INFO | "huge_shared" : " 29 GB", | 20260508 11:21:59.777 | INFO | "malloc" : " 14 GB", | 20260508 11:21:59.777 | INFO | "pinned" : " 29 GB" | 20260508 11:21:59.777 | INFO | } | 20260508 11:21:59.777 | INFO | Perf report for the last 5.14241 seconds: | 20260508 11:21:59.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:21:59.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.777 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:21:59.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.777 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:21:59.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:21:59.778 | INFO | Uploaded 36GiB to GPU in 2.45693s (14.6524GiB/s) | 20260508 11:21:59.778 | INFO | Downloaded 34GiB to GPU in 2.45738s (13.8359GiB/s) | 20260508 11:22:04.915 | INFO | Analyzing chunk frames = [7710208,7710720) | 20260508 11:22:04.915 | INFO | MemInfo: { | 20260508 11:22:04.915 | INFO | "gpu" : " 17 GB", | 20260508 11:22:04.915 | INFO | "huge" : "", | 20260508 11:22:04.915 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:04.915 | INFO | "malloc" : " 14 GB", | 20260508 11:22:04.915 | INFO | "pinned" : " 29 GB" | 20260508 11:22:04.915 | INFO | } | 20260508 11:22:04.915 | INFO | Perf report for the last 5.13821 seconds: | 20260508 11:22:04.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:04.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.915 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:22:04.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.915 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260508 11:22:04.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:04.916 | INFO | Uploaded 36GiB to GPU in 2.50268s (14.3846GiB/s) | 20260508 11:22:04.916 | INFO | Downloaded 34GiB to GPU in 2.47311s (13.7479GiB/s) | 20260508 11:22:10.054 | INFO | Analyzing chunk frames = [7710720,7711232) | 20260508 11:22:10.054 | INFO | MemInfo: { | 20260508 11:22:10.054 | INFO | "gpu" : " 17 GB", | 20260508 11:22:10.054 | INFO | "huge" : "", | 20260508 11:22:10.054 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:10.054 | INFO | "malloc" : " 14 GB", | 20260508 11:22:10.054 | INFO | "pinned" : " 29 GB" | 20260508 11:22:10.054 | INFO | } | 20260508 11:22:10.054 | INFO | Perf report for the last 5.13871 seconds: | 20260508 11:22:10.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:10.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 11:22:10.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:22:10.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:10.054 | INFO | Uploaded 36GiB to GPU in 2.40936s (14.9417GiB/s) | 20260508 11:22:10.054 | INFO | Downloaded 34GiB to GPU in 2.51376s (13.5256GiB/s) | 20260508 11:22:15.190 | INFO | Analyzing chunk frames = [7711232,7711744) | 20260508 11:22:15.191 | INFO | MemInfo: { | 20260508 11:22:15.191 | INFO | "gpu" : " 17 GB", | 20260508 11:22:15.191 | INFO | "huge" : "", | 20260508 11:22:15.191 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:15.191 | INFO | "malloc" : " 14 GB", | 20260508 11:22:15.191 | INFO | "pinned" : " 29 GB" | 20260508 11:22:15.191 | INFO | } | 20260508 11:22:15.191 | INFO | Perf report for the last 5.13685 seconds: | 20260508 11:22:15.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:15.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:22:15.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:22:15.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:15.191 | INFO | Uploaded 36GiB to GPU in 2.38101s (15.1196GiB/s) | 20260508 11:22:15.191 | INFO | Downloaded 34GiB to GPU in 2.4432s (13.9162GiB/s) | 20260508 11:22:20.330 | INFO | Analyzing chunk frames = [7711744,7712256) | 20260508 11:22:20.330 | INFO | MemInfo: { | 20260508 11:22:20.330 | INFO | "gpu" : " 17 GB", | 20260508 11:22:20.330 | INFO | "huge" : "", | 20260508 11:22:20.330 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:20.331 | INFO | "malloc" : " 14 GB", | 20260508 11:22:20.331 | INFO | "pinned" : " 29 GB" | 20260508 11:22:20.331 | INFO | } | 20260508 11:22:20.331 | INFO | Perf report for the last 5.13993 seconds: | 20260508 11:22:20.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:20.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:22:20.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:22:20.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:20.331 | INFO | Uploaded 36GiB to GPU in 2.43068s (14.8107GiB/s) | 20260508 11:22:20.331 | INFO | Downloaded 34GiB to GPU in 2.45693s (13.8384GiB/s) | 20260508 11:22:25.477 | INFO | Analyzing chunk frames = [7712256,7712768) | 20260508 11:22:25.477 | INFO | MemInfo: { | 20260508 11:22:25.477 | INFO | "gpu" : " 17 GB", | 20260508 11:22:25.477 | INFO | "huge" : "", | 20260508 11:22:25.477 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:25.477 | INFO | "malloc" : " 14 GB", | 20260508 11:22:25.477 | INFO | "pinned" : " 29 GB" | 20260508 11:22:25.477 | INFO | } | 20260508 11:22:25.477 | INFO | Perf report for the last 5.1465 seconds: | 20260508 11:22:25.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:25.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.477 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 11:22:25.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.477 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:22:25.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:25.478 | INFO | Uploaded 36GiB to GPU in 2.42434s (14.8494GiB/s) | 20260508 11:22:25.478 | INFO | Downloaded 34GiB to GPU in 2.43892s (13.9406GiB/s) | 20260508 11:22:25.575 | INFO | Starting a round of Baz IO | 20260508 11:22:28.990 | INFO | Populating RTMetrics report | 20260508 11:22:28.991 | INFO | RTMetrics report populated | 20260508 11:22:28.991 | INFO | Opening tmp RTMetrics json file | 20260508 11:22:29.000 | INFO | Serializing RTMetrics to json | 20260508 11:22:29.002 | INFO | Serialization done, swapping tmp file now... | 20260508 11:22:29.002 | INFO | RTMetrics done | 20260508 11:22:29.058 | INFO | Finished a round of Baz IO: Wrote 1.438 GiB in 3.483 seconds | 20260508 11:22:29.058 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:22:29.058 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194434% / 0% / 5.96046e-05% | 20260508 11:22:29.058 | INFO | Baz IO overall: | 20260508 11:22:29.059 | INFO | BytesWritten : 3328874328480 | 20260508 11:22:29.059 | INFO | NumEvents : 1598897731059 | 20260508 11:22:30.618 | INFO | Analyzing chunk frames = [7712768,7713280) | 20260508 11:22:30.618 | INFO | MemInfo: { | 20260508 11:22:30.618 | INFO | "gpu" : " 17 GB", | 20260508 11:22:30.618 | INFO | "huge" : "", | 20260508 11:22:30.618 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:30.618 | INFO | "malloc" : " 14 GB", | 20260508 11:22:30.619 | INFO | "pinned" : " 29 GB" | 20260508 11:22:30.619 | INFO | } | 20260508 11:22:30.619 | INFO | Perf report for the last 5.14134 seconds: | 20260508 11:22:30.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:30.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:30.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:30.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:30.619 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 11:22:30.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:30.619 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:22:30.619 | INFO | PRE_HQ : 65.7%, 1.6, 54.2ms, 0 | 20260508 11:22:30.619 | INFO | BAZWRITER : 67.1%, 2.5, 84.5ms, 0 | 20260508 11:22:30.619 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:22:30.619 | INFO | Uploaded 36GiB to GPU in 2.62123s (13.734GiB/s) | 20260508 11:22:30.619 | INFO | Downloaded 38GiB to GPU in 2.72061s (13.9675GiB/s) | 20260508 11:22:35.753 | INFO | Analyzing chunk frames = [7713280,7713792) | 20260508 11:22:35.753 | INFO | MemInfo: { | 20260508 11:22:35.753 | INFO | "gpu" : " 17 GB", | 20260508 11:22:35.753 | INFO | "huge" : "", | 20260508 11:22:35.753 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:35.753 | INFO | "malloc" : " 14 GB", | 20260508 11:22:35.753 | INFO | "pinned" : " 29 GB" | 20260508 11:22:35.753 | INFO | } | 20260508 11:22:35.753 | INFO | Perf report for the last 5.13419 seconds: | 20260508 11:22:35.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:35.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:22:35.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:22:35.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:35.753 | INFO | Uploaded 36GiB to GPU in 2.49489s (14.4295GiB/s) | 20260508 11:22:35.753 | INFO | Downloaded 34GiB to GPU in 2.46156s (13.8124GiB/s) | 20260508 11:22:40.885 | INFO | Analyzing chunk frames = [7713792,7714304) | 20260508 11:22:40.885 | INFO | MemInfo: { | 20260508 11:22:40.885 | INFO | "gpu" : " 17 GB", | 20260508 11:22:40.885 | INFO | "huge" : "", | 20260508 11:22:40.885 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:40.886 | INFO | "malloc" : " 14 GB", | 20260508 11:22:40.886 | INFO | "pinned" : " 29 GB" | 20260508 11:22:40.886 | INFO | } | 20260508 11:22:40.886 | INFO | Perf report for the last 5.13283 seconds: | 20260508 11:22:40.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:40.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 11:22:40.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 11:22:40.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:40.886 | INFO | Uploaded 36GiB to GPU in 2.55727s (14.0775GiB/s) | 20260508 11:22:40.886 | INFO | Downloaded 34GiB to GPU in 2.46471s (13.7948GiB/s) | 20260508 11:22:46.026 | INFO | Analyzing chunk frames = [7714304,7714816) | 20260508 11:22:46.026 | INFO | MemInfo: { | 20260508 11:22:46.026 | INFO | "gpu" : " 17 GB", | 20260508 11:22:46.026 | INFO | "huge" : "", | 20260508 11:22:46.026 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:46.027 | INFO | "malloc" : " 14 GB", | 20260508 11:22:46.027 | INFO | "pinned" : " 29 GB" | 20260508 11:22:46.027 | INFO | } | 20260508 11:22:46.027 | INFO | Perf report for the last 5.14103 seconds: | 20260508 11:22:46.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:46.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 11:22:46.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 11:22:46.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:46.027 | INFO | Uploaded 36GiB to GPU in 2.48616s (14.4802GiB/s) | 20260508 11:22:46.027 | INFO | Downloaded 34GiB to GPU in 2.42149s (14.041GiB/s) | 20260508 11:22:51.168 | INFO | Analyzing chunk frames = [7714816,7715328) | 20260508 11:22:51.168 | INFO | MemInfo: { | 20260508 11:22:51.168 | INFO | "gpu" : " 17 GB", | 20260508 11:22:51.168 | INFO | "huge" : "", | 20260508 11:22:51.169 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:51.169 | INFO | "malloc" : " 14 GB", | 20260508 11:22:51.169 | INFO | "pinned" : " 29 GB" | 20260508 11:22:51.169 | INFO | } | 20260508 11:22:51.169 | INFO | Perf report for the last 5.14205 seconds: | 20260508 11:22:51.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:51.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:22:51.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | PRE_HQ : 31.1%, 1.0, 16.3ms, 0 | 20260508 11:22:51.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:51.169 | INFO | Uploaded 36GiB to GPU in 2.31723s (15.5358GiB/s) | 20260508 11:22:51.169 | INFO | Downloaded 34GiB to GPU in 2.49467s (13.6291GiB/s) | 20260508 11:22:56.307 | INFO | Analyzing chunk frames = [7715328,7715840) | 20260508 11:22:56.307 | INFO | MemInfo: { | 20260508 11:22:56.307 | INFO | "gpu" : " 17 GB", | 20260508 11:22:56.307 | INFO | "huge" : "", | 20260508 11:22:56.307 | INFO | "huge_shared" : " 29 GB", | 20260508 11:22:56.307 | INFO | "malloc" : " 14 GB", | 20260508 11:22:56.307 | INFO | "pinned" : " 29 GB" | 20260508 11:22:56.308 | INFO | } | 20260508 11:22:56.308 | INFO | Perf report for the last 5.13876 seconds: | 20260508 11:22:56.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:22:56.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:22:56.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:22:56.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:22:56.308 | INFO | Uploaded 36GiB to GPU in 2.39275s (15.0455GiB/s) | 20260508 11:22:56.308 | INFO | Downloaded 34GiB to GPU in 2.42822s (14.002GiB/s) | 20260508 11:23:01.448 | INFO | Analyzing chunk frames = [7715840,7716352) | 20260508 11:23:01.448 | INFO | MemInfo: { | 20260508 11:23:01.448 | INFO | "gpu" : " 17 GB", | 20260508 11:23:01.448 | INFO | "huge" : "", | 20260508 11:23:01.448 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:01.449 | INFO | "malloc" : " 14 GB", | 20260508 11:23:01.449 | INFO | "pinned" : " 29 GB" | 20260508 11:23:01.449 | INFO | } | 20260508 11:23:01.449 | INFO | Perf report for the last 5.14121 seconds: | 20260508 11:23:01.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:01.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:23:01.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:23:01.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:01.449 | INFO | Uploaded 36GiB to GPU in 2.44465s (14.726GiB/s) | 20260508 11:23:01.449 | INFO | Downloaded 34GiB to GPU in 2.45341s (13.8583GiB/s) | 20260508 11:23:06.585 | INFO | Analyzing chunk frames = [7716352,7716864) | 20260508 11:23:06.585 | INFO | MemInfo: { | 20260508 11:23:06.585 | INFO | "gpu" : " 17 GB", | 20260508 11:23:06.585 | INFO | "huge" : "", | 20260508 11:23:06.585 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:06.585 | INFO | "malloc" : " 14 GB", | 20260508 11:23:06.585 | INFO | "pinned" : " 29 GB" | 20260508 11:23:06.585 | INFO | } | 20260508 11:23:06.586 | INFO | Perf report for the last 5.13672 seconds: | 20260508 11:23:06.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:06.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 11:23:06.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:23:06.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:06.586 | INFO | Uploaded 36GiB to GPU in 2.47828s (14.5262GiB/s) | 20260508 11:23:06.586 | INFO | Downloaded 34GiB to GPU in 2.45897s (13.8269GiB/s) | 20260508 11:23:06.586 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:23:06.586 | INFO | Total mem usage: 16.7 GiB | 20260508 11:23:06.586 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:23:06.586 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:23:06.586 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:23:06.586 | INFO | High Water Breakdown: | 20260508 11:23:06.586 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:23:06.586 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:23:06.586 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:23:06.586 | INFO | ------------------------------------------ | 20260508 11:23:06.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:23:06.586 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:23:06.586 | INFO | Total mem usage: 26.5 GiB | 20260508 11:23:06.586 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:23:06.586 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:23:06.586 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:23:06.586 | INFO | High Water Breakdown: | 20260508 11:23:06.586 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:23:06.586 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:23:06.586 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:23:06.586 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:23:06.586 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:23:06.586 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:23:06.586 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:23:06.586 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:23:06.586 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:23:06.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:23:06.586 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:23:06.586 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:23:06.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:23:06.586 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:23:06.586 | INFO | ------------------------------------------ | 20260508 11:23:06.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:23:06.586 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:23:06.586 | INFO | Total mem usage: 27.8 GiB | 20260508 11:23:06.586 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:23:06.586 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:23:06.586 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:23:06.586 | INFO | High Water Breakdown: | 20260508 11:23:06.586 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:23:06.586 | INFO | ------------------------------------------ | 20260508 11:23:06.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:23:06.586 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:23:06.586 | INFO | Total mem usage: 15.7 GiB | 20260508 11:23:06.586 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:23:06.586 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:23:06.586 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:23:06.586 | INFO | High Water Breakdown: | 20260508 11:23:06.586 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:23:06.586 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:23:06.586 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:23:06.586 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:23:06.586 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:23:06.586 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:23:06.586 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:23:06.586 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:23:06.586 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:23:06.586 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:23:06.586 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:23:06.586 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:23:06.586 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:23:06.586 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:23:06.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:23:06.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:23:06.586 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:23:06.586 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:23:06.586 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:23:06.586 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:23:06.586 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:23:06.586 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:23:06.586 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:23:06.586 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:23:06.586 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:23:06.586 | INFO | ------------------------------------------ | 20260508 11:23:06.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:23:06.586 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:23:06.586 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:23:06.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:23:06.586 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 6.01e-01 | 20260508 11:23:06.586 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.04e-01, 1.11e-02 / 6.86e-03 | 20260508 11:23:06.586 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.40e+00, 5.91e-01 / 2.66e-01 | 20260508 11:23:06.586 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.07e-01 / 1.15e-01 | 20260508 11:23:06.586 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.52e+01, 3.70e+00 / 1.06e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.76e+00 / 1.30e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.35e+00 / 2.48e+00, 4.05e-01 / 2.68e-01 | 20260508 11:23:06.586 | INFO | PerfReport: 3.00%, Metrics -- 9.79e-01 / 1.10e+00, 2.41e-01 / 1.00e-01 | 20260508 11:23:06.586 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.36e-01, 4.78e-02 / 3.33e-02 | 20260508 11:23:06.586 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:23:06.586 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:23:06.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:23:06.586 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 24.44%, Upload -- 2.57e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 24.69%, Download -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 31.02%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 2.60%, LockDownload -- 2.74e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 15.11%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:23:06.586 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260508 11:23:06.682 | INFO | Starting a round of Baz IO | 20260508 11:23:10.097 | INFO | Populating RTMetrics report | 20260508 11:23:10.097 | INFO | RTMetrics report populated | 20260508 11:23:10.097 | INFO | Opening tmp RTMetrics json file | 20260508 11:23:10.104 | INFO | Serializing RTMetrics to json | 20260508 11:23:10.105 | INFO | Serialization done, swapping tmp file now... | 20260508 11:23:10.105 | INFO | RTMetrics done | 20260508 11:23:10.163 | INFO | Finished a round of Baz IO: Wrote 1.433 GiB in 3.48 seconds | 20260508 11:23:10.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:23:10.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189153% / 0% / 5.96046e-05% | 20260508 11:23:10.163 | INFO | Baz IO overall: | 20260508 11:23:10.163 | INFO | BytesWritten : 3330412733135 | 20260508 11:23:10.163 | INFO | NumEvents : 1599620308829 | 20260508 11:23:11.725 | INFO | Analyzing chunk frames = [7716864,7717376) | 20260508 11:23:11.725 | INFO | MemInfo: { | 20260508 11:23:11.725 | INFO | "gpu" : " 17 GB", | 20260508 11:23:11.725 | INFO | "huge" : "", | 20260508 11:23:11.725 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:11.725 | INFO | "malloc" : " 14 GB", | 20260508 11:23:11.725 | INFO | "pinned" : " 29 GB" | 20260508 11:23:11.725 | INFO | } | 20260508 11:23:11.725 | INFO | Perf report for the last 5.13967 seconds: | 20260508 11:23:11.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:11.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:11.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:11.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:11.725 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 11:23:11.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:11.725 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 11:23:11.725 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 11:23:11.725 | INFO | BAZWRITER : 67.0%, 2.5, 84.8ms, 0 | 20260508 11:23:11.725 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:23:11.726 | INFO | Uploaded 36GiB to GPU in 2.55625s (14.0831GiB/s) | 20260508 11:23:11.726 | INFO | Downloaded 38GiB to GPU in 2.7195s (13.9731GiB/s) | 20260508 11:23:16.863 | INFO | Analyzing chunk frames = [7717376,7717888) | 20260508 11:23:16.863 | INFO | MemInfo: { | 20260508 11:23:16.863 | INFO | "gpu" : " 17 GB", | 20260508 11:23:16.863 | INFO | "huge" : "", | 20260508 11:23:16.863 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:16.863 | INFO | "malloc" : " 14 GB", | 20260508 11:23:16.863 | INFO | "pinned" : " 29 GB" | 20260508 11:23:16.863 | INFO | } | 20260508 11:23:16.864 | INFO | Perf report for the last 5.13832 seconds: | 20260508 11:23:16.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:16.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:23:16.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:23:16.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:16.864 | INFO | Uploaded 36GiB to GPU in 2.4472s (14.7107GiB/s) | 20260508 11:23:16.864 | INFO | Downloaded 34GiB to GPU in 2.48911s (13.6595GiB/s) | 20260508 11:23:22.002 | INFO | Analyzing chunk frames = [7717888,7718400) | 20260508 11:23:22.002 | INFO | MemInfo: { | 20260508 11:23:22.002 | INFO | "gpu" : " 17 GB", | 20260508 11:23:22.002 | INFO | "huge" : "", | 20260508 11:23:22.002 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:22.002 | INFO | "malloc" : " 14 GB", | 20260508 11:23:22.002 | INFO | "pinned" : " 29 GB" | 20260508 11:23:22.002 | INFO | } | 20260508 11:23:22.002 | INFO | Perf report for the last 5.13854 seconds: | 20260508 11:23:22.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:22.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.002 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:23:22.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.002 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:23:22.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:22.003 | INFO | Uploaded 36GiB to GPU in 2.44853s (14.7027GiB/s) | 20260508 11:23:22.003 | INFO | Downloaded 34GiB to GPU in 2.45536s (13.8473GiB/s) | 20260508 11:23:27.140 | INFO | Analyzing chunk frames = [7718400,7718912) | 20260508 11:23:27.140 | INFO | MemInfo: { | 20260508 11:23:27.140 | INFO | "gpu" : " 17 GB", | 20260508 11:23:27.140 | INFO | "huge" : "", | 20260508 11:23:27.140 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:27.140 | INFO | "malloc" : " 14 GB", | 20260508 11:23:27.140 | INFO | "pinned" : " 29 GB" | 20260508 11:23:27.140 | INFO | } | 20260508 11:23:27.140 | INFO | Perf report for the last 5.1378 seconds: | 20260508 11:23:27.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:27.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:23:27.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:23:27.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:27.140 | INFO | Uploaded 36GiB to GPU in 2.45927s (14.6385GiB/s) | 20260508 11:23:27.140 | INFO | Downloaded 34GiB to GPU in 2.46925s (13.7694GiB/s) | 20260508 11:23:32.284 | INFO | Analyzing chunk frames = [7718912,7719424) | 20260508 11:23:32.284 | INFO | MemInfo: { | 20260508 11:23:32.284 | INFO | "gpu" : " 17 GB", | 20260508 11:23:32.284 | INFO | "huge" : "", | 20260508 11:23:32.284 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:32.285 | INFO | "malloc" : " 14 GB", | 20260508 11:23:32.285 | INFO | "pinned" : " 29 GB" | 20260508 11:23:32.285 | INFO | } | 20260508 11:23:32.285 | INFO | Perf report for the last 5.14475 seconds: | 20260508 11:23:32.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:32.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:23:32.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:23:32.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:32.285 | INFO | Uploaded 36GiB to GPU in 2.28789s (15.735GiB/s) | 20260508 11:23:32.285 | INFO | Downloaded 34GiB to GPU in 2.51632s (13.5118GiB/s) | 20260508 11:23:37.431 | INFO | Analyzing chunk frames = [7719424,7719936) | 20260508 11:23:37.431 | INFO | MemInfo: { | 20260508 11:23:37.431 | INFO | "gpu" : " 17 GB", | 20260508 11:23:37.431 | INFO | "huge" : "", | 20260508 11:23:37.431 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:37.431 | INFO | "malloc" : " 14 GB", | 20260508 11:23:37.431 | INFO | "pinned" : " 29 GB" | 20260508 11:23:37.431 | INFO | } | 20260508 11:23:37.431 | INFO | Perf report for the last 5.14634 seconds: | 20260508 11:23:37.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:37.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.431 | INFO | ANALYSIS : 65.1%, 3.0, 101.5ms, 0 | 20260508 11:23:37.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.431 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:23:37.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:37.432 | INFO | Uploaded 36GiB to GPU in 2.36574s (15.2172GiB/s) | 20260508 11:23:37.432 | INFO | Downloaded 34GiB to GPU in 2.44344s (13.9148GiB/s) | 20260508 11:23:42.565 | INFO | Analyzing chunk frames = [7719936,7720448) | 20260508 11:23:42.565 | INFO | MemInfo: { | 20260508 11:23:42.565 | INFO | "gpu" : " 17 GB", | 20260508 11:23:42.565 | INFO | "huge" : "", | 20260508 11:23:42.565 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:42.565 | INFO | "malloc" : " 14 GB", | 20260508 11:23:42.565 | INFO | "pinned" : " 29 GB" | 20260508 11:23:42.565 | INFO | } | 20260508 11:23:42.565 | INFO | Perf report for the last 5.13406 seconds: | 20260508 11:23:42.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:42.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:23:42.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:23:42.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:42.565 | INFO | Uploaded 36GiB to GPU in 2.39024s (15.0613GiB/s) | 20260508 11:23:42.565 | INFO | Downloaded 34GiB to GPU in 2.46838s (13.7742GiB/s) | 20260508 11:23:47.696 | INFO | Analyzing chunk frames = [7720448,7720960) | 20260508 11:23:47.696 | INFO | MemInfo: { | 20260508 11:23:47.696 | INFO | "gpu" : " 17 GB", | 20260508 11:23:47.696 | INFO | "huge" : "", | 20260508 11:23:47.696 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:47.696 | INFO | "malloc" : " 14 GB", | 20260508 11:23:47.696 | INFO | "pinned" : " 29 GB" | 20260508 11:23:47.697 | INFO | } | 20260508 11:23:47.697 | INFO | Perf report for the last 5.13152 seconds: | 20260508 11:23:47.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:47.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:23:47.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:23:47.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:47.697 | INFO | Uploaded 36GiB to GPU in 2.42296s (14.8579GiB/s) | 20260508 11:23:47.697 | INFO | Downloaded 34GiB to GPU in 2.47108s (13.7592GiB/s) | 20260508 11:23:47.793 | INFO | Starting a round of Baz IO | 20260508 11:23:51.213 | INFO | Populating RTMetrics report | 20260508 11:23:51.213 | INFO | RTMetrics report populated | 20260508 11:23:51.213 | INFO | Opening tmp RTMetrics json file | 20260508 11:23:51.214 | INFO | Serializing RTMetrics to json | 20260508 11:23:51.215 | INFO | Serialization done, swapping tmp file now... | 20260508 11:23:51.215 | INFO | RTMetrics done | 20260508 11:23:51.276 | INFO | Finished a round of Baz IO: Wrote 1.433 GiB in 3.482 seconds | 20260508 11:23:51.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:23:51.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188816% / 0% / 5.16574e-05% | 20260508 11:23:51.276 | INFO | Baz IO overall: | 20260508 11:23:51.276 | INFO | BytesWritten : 3331951653021 | 20260508 11:23:51.276 | INFO | NumEvents : 1600343285458 | 20260508 11:23:52.841 | INFO | Analyzing chunk frames = [7720960,7721472) | 20260508 11:23:52.841 | INFO | MemInfo: { | 20260508 11:23:52.841 | INFO | "gpu" : " 17 GB", | 20260508 11:23:52.841 | INFO | "huge" : "", | 20260508 11:23:52.841 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:52.841 | INFO | "malloc" : " 14 GB", | 20260508 11:23:52.841 | INFO | "pinned" : " 29 GB" | 20260508 11:23:52.841 | INFO | } | 20260508 11:23:52.841 | INFO | Perf report for the last 5.1446 seconds: | 20260508 11:23:52.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:52.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:52.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:52.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:52.841 | INFO | ANALYSIS : 68.0%, 3.0, 105.9ms, 0 | 20260508 11:23:52.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:52.841 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 11:23:52.841 | INFO | PRE_HQ : 65.6%, 1.6, 54.0ms, 0 | 20260508 11:23:52.841 | INFO | BAZWRITER : 67.0%, 2.4, 84.5ms, 0 | 20260508 11:23:52.841 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:23:52.842 | INFO | Uploaded 36GiB to GPU in 2.52161s (14.2766GiB/s) | 20260508 11:23:52.842 | INFO | Downloaded 38GiB to GPU in 2.71826s (13.9795GiB/s) | 20260508 11:23:57.979 | INFO | Analyzing chunk frames = [7721472,7721984) | 20260508 11:23:57.979 | INFO | MemInfo: { | 20260508 11:23:57.979 | INFO | "gpu" : " 17 GB", | 20260508 11:23:57.979 | INFO | "huge" : "", | 20260508 11:23:57.979 | INFO | "huge_shared" : " 29 GB", | 20260508 11:23:57.979 | INFO | "malloc" : " 14 GB", | 20260508 11:23:57.979 | INFO | "pinned" : " 29 GB" | 20260508 11:23:57.979 | INFO | } | 20260508 11:23:57.979 | INFO | Perf report for the last 5.13791 seconds: | 20260508 11:23:57.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:23:57.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.979 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:23:57.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.979 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 11:23:57.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:23:57.980 | INFO | Uploaded 36GiB to GPU in 2.41992s (14.8765GiB/s) | 20260508 11:23:57.980 | INFO | Downloaded 34GiB to GPU in 2.47718s (13.7253GiB/s) | 20260508 11:24:03.119 | INFO | Analyzing chunk frames = [7721984,7722496) | 20260508 11:24:03.119 | INFO | MemInfo: { | 20260508 11:24:03.119 | INFO | "gpu" : " 17 GB", | 20260508 11:24:03.119 | INFO | "huge" : "", | 20260508 11:24:03.119 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:03.119 | INFO | "malloc" : " 14 GB", | 20260508 11:24:03.119 | INFO | "pinned" : " 29 GB" | 20260508 11:24:03.119 | INFO | } | 20260508 11:24:03.119 | INFO | Perf report for the last 5.1401 seconds: | 20260508 11:24:03.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:03.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.119 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:24:03.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.119 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:24:03.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:03.120 | INFO | Uploaded 36GiB to GPU in 2.45477s (14.6653GiB/s) | 20260508 11:24:03.120 | INFO | Downloaded 34GiB to GPU in 2.4556s (13.8459GiB/s) | 20260508 11:24:08.259 | INFO | Analyzing chunk frames = [7722496,7723008) | 20260508 11:24:08.259 | INFO | MemInfo: { | 20260508 11:24:08.259 | INFO | "gpu" : " 17 GB", | 20260508 11:24:08.259 | INFO | "huge" : "", | 20260508 11:24:08.259 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:08.259 | INFO | "malloc" : " 14 GB", | 20260508 11:24:08.259 | INFO | "pinned" : " 29 GB" | 20260508 11:24:08.259 | INFO | } | 20260508 11:24:08.259 | INFO | Perf report for the last 5.13983 seconds: | 20260508 11:24:08.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:08.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.259 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 11:24:08.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.259 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:24:08.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:08.260 | INFO | Uploaded 36GiB to GPU in 2.45402s (14.6698GiB/s) | 20260508 11:24:08.260 | INFO | Downloaded 34GiB to GPU in 2.45566s (13.8456GiB/s) | 20260508 11:24:13.395 | INFO | Analyzing chunk frames = [7723008,7723520) | 20260508 11:24:13.395 | INFO | MemInfo: { | 20260508 11:24:13.395 | INFO | "gpu" : " 17 GB", | 20260508 11:24:13.395 | INFO | "huge" : "", | 20260508 11:24:13.395 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:13.395 | INFO | "malloc" : " 14 GB", | 20260508 11:24:13.395 | INFO | "pinned" : " 29 GB" | 20260508 11:24:13.395 | INFO | } | 20260508 11:24:13.395 | INFO | Perf report for the last 5.13564 seconds: | 20260508 11:24:13.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:13.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 11:24:13.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:24:13.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:13.395 | INFO | Uploaded 36GiB to GPU in 2.39068s (15.0585GiB/s) | 20260508 11:24:13.395 | INFO | Downloaded 34GiB to GPU in 2.53026s (13.4374GiB/s) | 20260508 11:24:18.538 | INFO | Analyzing chunk frames = [7723520,7724032) | 20260508 11:24:18.538 | INFO | MemInfo: { | 20260508 11:24:18.538 | INFO | "gpu" : " 17 GB", | 20260508 11:24:18.538 | INFO | "huge" : "", | 20260508 11:24:18.538 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:18.538 | INFO | "malloc" : " 14 GB", | 20260508 11:24:18.538 | INFO | "pinned" : " 29 GB" | 20260508 11:24:18.538 | INFO | } | 20260508 11:24:18.538 | INFO | Perf report for the last 5.14319 seconds: | 20260508 11:24:18.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:18.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:24:18.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:24:18.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:18.538 | INFO | Uploaded 36GiB to GPU in 2.38982s (15.0639GiB/s) | 20260508 11:24:18.538 | INFO | Downloaded 34GiB to GPU in 2.44191s (13.9235GiB/s) | 20260508 11:24:23.677 | INFO | Analyzing chunk frames = [7724032,7724544) | 20260508 11:24:23.677 | INFO | MemInfo: { | 20260508 11:24:23.677 | INFO | "gpu" : " 17 GB", | 20260508 11:24:23.677 | INFO | "huge" : "", | 20260508 11:24:23.677 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:23.677 | INFO | "malloc" : " 14 GB", | 20260508 11:24:23.677 | INFO | "pinned" : " 29 GB" | 20260508 11:24:23.677 | INFO | } | 20260508 11:24:23.677 | INFO | Perf report for the last 5.13933 seconds: | 20260508 11:24:23.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:23.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.677 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 11:24:23.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.677 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:24:23.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:23.678 | INFO | Uploaded 36GiB to GPU in 2.429s (14.8209GiB/s) | 20260508 11:24:23.678 | INFO | Downloaded 34GiB to GPU in 2.44423s (13.9103GiB/s) | 20260508 11:24:28.816 | INFO | Analyzing chunk frames = [7724544,7725056) | 20260508 11:24:28.816 | INFO | MemInfo: { | 20260508 11:24:28.816 | INFO | "gpu" : " 17 GB", | 20260508 11:24:28.816 | INFO | "huge" : "", | 20260508 11:24:28.816 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:28.817 | INFO | "malloc" : " 14 GB", | 20260508 11:24:28.817 | INFO | "pinned" : " 29 GB" | 20260508 11:24:28.817 | INFO | } | 20260508 11:24:28.817 | INFO | Perf report for the last 5.13944 seconds: | 20260508 11:24:28.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:28.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:24:28.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:24:28.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:28.817 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:24:28.817 | INFO | Total mem usage: 16.7 GiB | 20260508 11:24:28.817 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:24:28.817 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 11:24:28.817 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:24:28.817 | INFO | High Water Breakdown: | 20260508 11:24:28.817 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:24:28.817 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:24:28.817 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:24:28.817 | INFO | ------------------------------------------ | 20260508 11:24:28.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:24:28.817 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:24:28.817 | INFO | Total mem usage: 26.5 GiB | 20260508 11:24:28.817 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:24:28.817 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:24:28.817 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:24:28.817 | INFO | High Water Breakdown: | 20260508 11:24:28.817 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:24:28.817 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:24:28.817 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:24:28.817 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:24:28.817 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:24:28.817 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:24:28.817 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:24:28.817 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:24:28.817 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:24:28.817 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:24:28.817 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:24:28.817 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:24:28.817 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:24:28.817 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:24:28.817 | INFO | ------------------------------------------ | 20260508 11:24:28.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:24:28.817 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:24:28.817 | INFO | Total mem usage: 27.8 GiB | 20260508 11:24:28.817 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:24:28.817 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:24:28.817 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:24:28.817 | INFO | High Water Breakdown: | 20260508 11:24:28.817 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:24:28.817 | INFO | ------------------------------------------ | 20260508 11:24:28.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:24:28.817 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:24:28.817 | INFO | Total mem usage: 15.7 GiB | 20260508 11:24:28.817 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:24:28.817 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:24:28.817 | INFO | Unused Cache: 869 MiB (Waste: 5.41%) | 20260508 11:24:28.817 | INFO | High Water Breakdown: | 20260508 11:24:28.817 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:24:28.817 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:24:28.817 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:24:28.817 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:24:28.817 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:24:28.817 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:24:28.817 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:24:28.817 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:24:28.817 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:24:28.817 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:24:28.817 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:24:28.817 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:24:28.817 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:24:28.817 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:24:28.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:24:28.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:24:28.817 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:24:28.817 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:24:28.817 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:24:28.817 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:24:28.817 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:24:28.817 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:24:28.817 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:24:28.817 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:24:28.817 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:24:28.817 | INFO | ------------------------------------------ | 20260508 11:24:28.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:24:28.817 | INFO | Uploaded 36GiB to GPU in 2.44071s (14.7498GiB/s) | 20260508 11:24:28.817 | INFO | Downloaded 34GiB to GPU in 2.46138s (13.8134GiB/s) | 20260508 11:24:28.817 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:24:28.817 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:24:28.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:24:28.817 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.40e+01 / 6.63e-01 | 20260508 11:24:28.817 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.18e-02 / 4.88e-03 | 20260508 11:24:28.817 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.31e+00, 5.93e-01 / 2.03e-01 | 20260508 11:24:28.817 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.11e-01 / 1.09e-01 | 20260508 11:24:28.817 | INFO | PerfReport: 16.07%, DME -- 4.19e+01 / 4.57e+01, 3.72e+00 / 1.05e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.76e+00 / 1.19e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.45e+00, 4.03e-01 / 2.88e-01 | 20260508 11:24:28.817 | INFO | PerfReport: 2.99%, Metrics -- 9.75e-01 / 1.13e+00, 2.40e-01 / 1.27e-01 | 20260508 11:24:28.817 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.29e-01, 4.76e-02 / 3.17e-02 | 20260508 11:24:28.817 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:24:28.817 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:24:28.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:24:28.817 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 24.09%, Upload -- 2.53e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 24.79%, Download -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 31.04%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 1.99%, LockUpload -- 2.09e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 11:24:28.817 | INFO | PerfReport: 2.80%, LockDownload -- 2.95e+00 / 0.00e+00, 4.43e+00 / 0.00e+00 | 20260508 11:24:28.818 | INFO | PerfReport: 15.28%, LockCompute -- 1.61e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 11:24:28.818 | INFO | PerfReport: 0.01%, Misc -- 1.19e-02 / 0.00e+00, 9.98e-03 / 0.00e+00 | 20260508 11:24:28.915 | INFO | Starting a round of Baz IO | 20260508 11:24:32.328 | INFO | Populating RTMetrics report | 20260508 11:24:32.329 | INFO | RTMetrics report populated | 20260508 11:24:32.329 | INFO | Opening tmp RTMetrics json file | 20260508 11:24:32.337 | INFO | Serializing RTMetrics to json | 20260508 11:24:32.338 | INFO | Serialization done, swapping tmp file now... | 20260508 11:24:32.339 | INFO | RTMetrics done | 20260508 11:24:32.395 | INFO | Finished a round of Baz IO: Wrote 1.432 GiB in 3.48 seconds | 20260508 11:24:32.395 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:24:32.395 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189296% / 0% / 4.37101e-05% | 20260508 11:24:32.395 | INFO | Baz IO overall: | 20260508 11:24:32.395 | INFO | BytesWritten : 3333489719859 | 20260508 11:24:32.395 | INFO | NumEvents : 1601065794582 | 20260508 11:24:33.952 | INFO | Analyzing chunk frames = [7725056,7725568) | 20260508 11:24:33.952 | INFO | MemInfo: { | 20260508 11:24:33.952 | INFO | "gpu" : " 17 GB", | 20260508 11:24:33.952 | INFO | "huge" : "", | 20260508 11:24:33.952 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:33.952 | INFO | "malloc" : " 14 GB", | 20260508 11:24:33.952 | INFO | "pinned" : " 29 GB" | 20260508 11:24:33.952 | INFO | } | 20260508 11:24:33.952 | INFO | Perf report for the last 5.13567 seconds: | 20260508 11:24:33.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:33.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:33.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:33.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:33.953 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:24:33.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:33.953 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:24:33.953 | INFO | PRE_HQ : 65.2%, 1.6, 54.0ms, 0 | 20260508 11:24:33.953 | INFO | BAZWRITER : 67.1%, 2.5, 84.7ms, 0 | 20260508 11:24:33.953 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:24:33.953 | INFO | Uploaded 36GiB to GPU in 2.62022s (13.7393GiB/s) | 20260508 11:24:33.953 | INFO | Downloaded 38GiB to GPU in 2.69551s (14.0975GiB/s) | 20260508 11:24:39.094 | INFO | Analyzing chunk frames = [7725568,7726080) | 20260508 11:24:39.094 | INFO | MemInfo: { | 20260508 11:24:39.094 | INFO | "gpu" : " 17 GB", | 20260508 11:24:39.094 | INFO | "huge" : "", | 20260508 11:24:39.094 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:39.095 | INFO | "malloc" : " 14 GB", | 20260508 11:24:39.095 | INFO | "pinned" : " 29 GB" | 20260508 11:24:39.095 | INFO | } | 20260508 11:24:39.095 | INFO | Perf report for the last 5.14222 seconds: | 20260508 11:24:39.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:39.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:24:39.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:24:39.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:39.095 | INFO | Uploaded 36GiB to GPU in 2.45497s (14.6641GiB/s) | 20260508 11:24:39.095 | INFO | Downloaded 34GiB to GPU in 2.47184s (13.7549GiB/s) | 20260508 11:24:44.235 | INFO | Analyzing chunk frames = [7726080,7726592) | 20260508 11:24:44.235 | INFO | MemInfo: { | 20260508 11:24:44.235 | INFO | "gpu" : " 17 GB", | 20260508 11:24:44.235 | INFO | "huge" : "", | 20260508 11:24:44.235 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:44.235 | INFO | "malloc" : " 14 GB", | 20260508 11:24:44.235 | INFO | "pinned" : " 29 GB" | 20260508 11:24:44.235 | INFO | } | 20260508 11:24:44.235 | INFO | Perf report for the last 5.14042 seconds: | 20260508 11:24:44.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:44.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.235 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:24:44.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.235 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:24:44.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:44.236 | INFO | Uploaded 36GiB to GPU in 2.4891s (14.4631GiB/s) | 20260508 11:24:44.236 | INFO | Downloaded 34GiB to GPU in 2.44425s (13.9102GiB/s) | 20260508 11:24:49.379 | INFO | Analyzing chunk frames = [7726592,7727104) | 20260508 11:24:49.379 | INFO | MemInfo: { | 20260508 11:24:49.379 | INFO | "gpu" : " 17 GB", | 20260508 11:24:49.379 | INFO | "huge" : "", | 20260508 11:24:49.379 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:49.379 | INFO | "malloc" : " 14 GB", | 20260508 11:24:49.379 | INFO | "pinned" : " 29 GB" | 20260508 11:24:49.379 | INFO | } | 20260508 11:24:49.379 | INFO | Perf report for the last 5.14381 seconds: | 20260508 11:24:49.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:49.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.379 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:24:49.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.379 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:24:49.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:49.380 | INFO | Uploaded 36GiB to GPU in 2.46956s (14.5775GiB/s) | 20260508 11:24:49.380 | INFO | Downloaded 34GiB to GPU in 2.46659s (13.7842GiB/s) | 20260508 11:24:54.515 | INFO | Analyzing chunk frames = [7727104,7727616) | 20260508 11:24:54.515 | INFO | MemInfo: { | 20260508 11:24:54.515 | INFO | "gpu" : " 17 GB", | 20260508 11:24:54.515 | INFO | "huge" : "", | 20260508 11:24:54.515 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:54.515 | INFO | "malloc" : " 14 GB", | 20260508 11:24:54.515 | INFO | "pinned" : " 29 GB" | 20260508 11:24:54.515 | INFO | } | 20260508 11:24:54.515 | INFO | Perf report for the last 5.13607 seconds: | 20260508 11:24:54.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:54.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 11:24:54.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 11:24:54.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:54.515 | INFO | Uploaded 36GiB to GPU in 2.45099s (14.6879GiB/s) | 20260508 11:24:54.515 | INFO | Downloaded 34GiB to GPU in 2.5261s (13.4595GiB/s) | 20260508 11:24:59.649 | INFO | Analyzing chunk frames = [7727616,7728128) | 20260508 11:24:59.649 | INFO | MemInfo: { | 20260508 11:24:59.649 | INFO | "gpu" : " 17 GB", | 20260508 11:24:59.649 | INFO | "huge" : "", | 20260508 11:24:59.649 | INFO | "huge_shared" : " 29 GB", | 20260508 11:24:59.649 | INFO | "malloc" : " 14 GB", | 20260508 11:24:59.649 | INFO | "pinned" : " 29 GB" | 20260508 11:24:59.649 | INFO | } | 20260508 11:24:59.649 | INFO | Perf report for the last 5.13429 seconds: | 20260508 11:24:59.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:24:59.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.649 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:24:59.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.649 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:24:59.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:24:59.650 | INFO | Uploaded 36GiB to GPU in 2.36352s (15.2315GiB/s) | 20260508 11:24:59.650 | INFO | Downloaded 34GiB to GPU in 2.43363s (13.9709GiB/s) | 20260508 11:25:04.789 | INFO | Analyzing chunk frames = [7728128,7728640) | 20260508 11:25:04.789 | INFO | MemInfo: { | 20260508 11:25:04.789 | INFO | "gpu" : " 17 GB", | 20260508 11:25:04.789 | INFO | "huge" : "", | 20260508 11:25:04.789 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:04.789 | INFO | "malloc" : " 14 GB", | 20260508 11:25:04.789 | INFO | "pinned" : " 29 GB" | 20260508 11:25:04.789 | INFO | } | 20260508 11:25:04.789 | INFO | Perf report for the last 5.14009 seconds: | 20260508 11:25:04.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:04.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.789 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 11:25:04.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.790 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 11:25:04.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:04.790 | INFO | Uploaded 36GiB to GPU in 2.45082s (14.6889GiB/s) | 20260508 11:25:04.790 | INFO | Downloaded 34GiB to GPU in 2.4545s (13.8521GiB/s) | 20260508 11:25:09.938 | INFO | Analyzing chunk frames = [7728640,7729152) | 20260508 11:25:09.938 | INFO | MemInfo: { | 20260508 11:25:09.938 | INFO | "gpu" : " 17 GB", | 20260508 11:25:09.938 | INFO | "huge" : "", | 20260508 11:25:09.938 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:09.938 | INFO | "malloc" : " 14 GB", | 20260508 11:25:09.938 | INFO | "pinned" : " 29 GB" | 20260508 11:25:09.938 | INFO | } | 20260508 11:25:09.938 | INFO | Perf report for the last 5.14834 seconds: | 20260508 11:25:09.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:09.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 11:25:09.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:25:09.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:09.938 | INFO | Uploaded 36GiB to GPU in 2.4219s (14.8644GiB/s) | 20260508 11:25:09.938 | INFO | Downloaded 34GiB to GPU in 2.43895s (13.9404GiB/s) | 20260508 11:25:10.043 | INFO | Starting a round of Baz IO | 20260508 11:25:13.449 | INFO | Populating RTMetrics report | 20260508 11:25:13.449 | INFO | RTMetrics report populated | 20260508 11:25:13.449 | INFO | Opening tmp RTMetrics json file | 20260508 11:25:13.457 | INFO | Serializing RTMetrics to json | 20260508 11:25:13.458 | INFO | Serialization done, swapping tmp file now... | 20260508 11:25:13.459 | INFO | RTMetrics done | 20260508 11:25:13.513 | INFO | Finished a round of Baz IO: Wrote 1.431 GiB in 3.469 seconds | 20260508 11:25:13.513 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:25:13.513 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188303% / 3.97364e-06% / 6.75519e-05% | 20260508 11:25:13.513 | INFO | Baz IO overall: | 20260508 11:25:13.513 | INFO | BytesWritten : 3335026471131 | 20260508 11:25:13.513 | INFO | NumEvents : 1601787703868 | 20260508 11:25:15.070 | INFO | Analyzing chunk frames = [7729152,7729664) | 20260508 11:25:15.070 | INFO | MemInfo: { | 20260508 11:25:15.070 | INFO | "gpu" : " 17 GB", | 20260508 11:25:15.070 | INFO | "huge" : "", | 20260508 11:25:15.070 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:15.070 | INFO | "malloc" : " 14 GB", | 20260508 11:25:15.070 | INFO | "pinned" : " 29 GB" | 20260508 11:25:15.070 | INFO | } | 20260508 11:25:15.070 | INFO | Perf report for the last 5.13227 seconds: | 20260508 11:25:15.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:15.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:15.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:15.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:15.070 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:25:15.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:15.070 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:25:15.070 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 11:25:15.070 | INFO | BAZWRITER : 67.0%, 2.5, 84.7ms, 0 | 20260508 11:25:15.070 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:25:15.070 | INFO | Uploaded 36GiB to GPU in 2.61801s (13.7509GiB/s) | 20260508 11:25:15.070 | INFO | Downloaded 38GiB to GPU in 2.71479s (13.9974GiB/s) | 20260508 11:25:20.209 | INFO | Analyzing chunk frames = [7729664,7730176) | 20260508 11:25:20.210 | INFO | MemInfo: { | 20260508 11:25:20.210 | INFO | "gpu" : " 17 GB", | 20260508 11:25:20.210 | INFO | "huge" : "", | 20260508 11:25:20.210 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:20.210 | INFO | "malloc" : " 14 GB", | 20260508 11:25:20.210 | INFO | "pinned" : " 29 GB" | 20260508 11:25:20.210 | INFO | } | 20260508 11:25:20.210 | INFO | Perf report for the last 5.1397 seconds: | 20260508 11:25:20.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:20.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:25:20.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:25:20.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:20.210 | INFO | Uploaded 36GiB to GPU in 2.44896s (14.7001GiB/s) | 20260508 11:25:20.210 | INFO | Downloaded 34GiB to GPU in 2.46764s (13.7784GiB/s) | 20260508 11:25:25.351 | INFO | Analyzing chunk frames = [7730176,7730688) | 20260508 11:25:25.351 | INFO | MemInfo: { | 20260508 11:25:25.351 | INFO | "gpu" : " 17 GB", | 20260508 11:25:25.351 | INFO | "huge" : "", | 20260508 11:25:25.351 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:25.351 | INFO | "malloc" : " 14 GB", | 20260508 11:25:25.351 | INFO | "pinned" : " 29 GB" | 20260508 11:25:25.351 | INFO | } | 20260508 11:25:25.351 | INFO | Perf report for the last 5.14135 seconds: | 20260508 11:25:25.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:25.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.351 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:25:25.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.351 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:25:25.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:25.352 | INFO | Uploaded 36GiB to GPU in 2.47711s (14.5331GiB/s) | 20260508 11:25:25.352 | INFO | Downloaded 34GiB to GPU in 2.45457s (13.8517GiB/s) | 20260508 11:25:30.488 | INFO | Analyzing chunk frames = [7730688,7731200) | 20260508 11:25:30.488 | INFO | MemInfo: { | 20260508 11:25:30.488 | INFO | "gpu" : " 17 GB", | 20260508 11:25:30.488 | INFO | "huge" : "", | 20260508 11:25:30.488 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:30.488 | INFO | "malloc" : " 14 GB", | 20260508 11:25:30.488 | INFO | "pinned" : " 29 GB" | 20260508 11:25:30.488 | INFO | } | 20260508 11:25:30.488 | INFO | Perf report for the last 5.13696 seconds: | 20260508 11:25:30.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:30.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.488 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 11:25:30.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.488 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:25:30.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:30.489 | INFO | Uploaded 36GiB to GPU in 2.52301s (14.2687GiB/s) | 20260508 11:25:30.489 | INFO | Downloaded 34GiB to GPU in 2.45127s (13.8704GiB/s) | 20260508 11:25:35.626 | INFO | Analyzing chunk frames = [7731200,7731712) | 20260508 11:25:35.626 | INFO | MemInfo: { | 20260508 11:25:35.626 | INFO | "gpu" : " 17 GB", | 20260508 11:25:35.626 | INFO | "huge" : "", | 20260508 11:25:35.626 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:35.626 | INFO | "malloc" : " 14 GB", | 20260508 11:25:35.626 | INFO | "pinned" : " 29 GB" | 20260508 11:25:35.626 | INFO | } | 20260508 11:25:35.627 | INFO | Perf report for the last 5.13837 seconds: | 20260508 11:25:35.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:35.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 11:25:35.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 11:25:35.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:35.627 | INFO | Uploaded 36GiB to GPU in 2.39719s (15.0176GiB/s) | 20260508 11:25:35.627 | INFO | Downloaded 34GiB to GPU in 2.53302s (13.4227GiB/s) | 20260508 11:25:40.769 | INFO | Analyzing chunk frames = [7731712,7732224) | 20260508 11:25:40.769 | INFO | MemInfo: { | 20260508 11:25:40.769 | INFO | "gpu" : " 17 GB", | 20260508 11:25:40.769 | INFO | "huge" : "", | 20260508 11:25:40.769 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:40.769 | INFO | "malloc" : " 14 GB", | 20260508 11:25:40.769 | INFO | "pinned" : " 29 GB" | 20260508 11:25:40.769 | INFO | } | 20260508 11:25:40.769 | INFO | Perf report for the last 5.14297 seconds: | 20260508 11:25:40.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:40.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:25:40.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260508 11:25:40.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:40.770 | INFO | Uploaded 36GiB to GPU in 2.36645s (15.2126GiB/s) | 20260508 11:25:40.770 | INFO | Downloaded 34GiB to GPU in 2.45549s (13.8465GiB/s) | 20260508 11:25:45.902 | INFO | Analyzing chunk frames = [7732224,7732736) | 20260508 11:25:45.902 | INFO | MemInfo: { | 20260508 11:25:45.902 | INFO | "gpu" : " 17 GB", | 20260508 11:25:45.902 | INFO | "huge" : "", | 20260508 11:25:45.902 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:45.902 | INFO | "malloc" : " 14 GB", | 20260508 11:25:45.902 | INFO | "pinned" : " 29 GB" | 20260508 11:25:45.902 | INFO | } | 20260508 11:25:45.902 | INFO | Perf report for the last 5.13296 seconds: | 20260508 11:25:45.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:45.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.902 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:25:45.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.902 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260508 11:25:45.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:45.903 | INFO | Uploaded 36GiB to GPU in 2.44088s (14.7488GiB/s) | 20260508 11:25:45.903 | INFO | Downloaded 34GiB to GPU in 2.47483s (13.7383GiB/s) | 20260508 11:25:51.053 | INFO | Analyzing chunk frames = [7732736,7733248) | 20260508 11:25:51.053 | INFO | MemInfo: { | 20260508 11:25:51.053 | INFO | "gpu" : " 17 GB", | 20260508 11:25:51.053 | INFO | "huge" : "", | 20260508 11:25:51.053 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:51.053 | INFO | "malloc" : " 14 GB", | 20260508 11:25:51.054 | INFO | "pinned" : " 29 GB" | 20260508 11:25:51.054 | INFO | } | 20260508 11:25:51.054 | INFO | Perf report for the last 5.15111 seconds: | 20260508 11:25:51.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:51.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | ANALYSIS : 65.3%, 3.0, 102.4ms, 0 | 20260508 11:25:51.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:25:51.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:51.054 | INFO | Uploaded 36GiB to GPU in 2.42133s (14.8678GiB/s) | 20260508 11:25:51.054 | INFO | Downloaded 34GiB to GPU in 2.47594s (13.7322GiB/s) | 20260508 11:25:51.054 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:25:51.054 | INFO | Total mem usage: 16.7 GiB | 20260508 11:25:51.054 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:25:51.054 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 11:25:51.054 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:25:51.054 | INFO | High Water Breakdown: | 20260508 11:25:51.054 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:25:51.054 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:25:51.054 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:25:51.054 | INFO | ------------------------------------------ | 20260508 11:25:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:25:51.054 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:25:51.054 | INFO | Total mem usage: 26.5 GiB | 20260508 11:25:51.054 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:25:51.054 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:25:51.054 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:25:51.054 | INFO | High Water Breakdown: | 20260508 11:25:51.054 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:25:51.054 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:25:51.054 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:25:51.054 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:25:51.054 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:25:51.054 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:25:51.054 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:25:51.054 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:25:51.054 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:25:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:25:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:25:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:25:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:25:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:25:51.054 | INFO | ------------------------------------------ | 20260508 11:25:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:25:51.054 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:25:51.054 | INFO | Total mem usage: 27.8 GiB | 20260508 11:25:51.054 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:25:51.054 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:25:51.054 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:25:51.054 | INFO | High Water Breakdown: | 20260508 11:25:51.054 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:25:51.054 | INFO | ------------------------------------------ | 20260508 11:25:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:25:51.054 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:25:51.054 | INFO | Total mem usage: 15.7 GiB | 20260508 11:25:51.054 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:25:51.054 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:25:51.054 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:25:51.054 | INFO | High Water Breakdown: | 20260508 11:25:51.054 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:25:51.054 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:25:51.054 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:25:51.054 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:25:51.054 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:25:51.054 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:25:51.054 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:25:51.054 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:25:51.054 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:25:51.054 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:25:51.054 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:25:51.054 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:25:51.054 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:25:51.054 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:25:51.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:25:51.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:25:51.054 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:25:51.054 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:25:51.054 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:25:51.054 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:25:51.054 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:25:51.054 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:25:51.054 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:25:51.054 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:25:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:25:51.054 | INFO | ------------------------------------------ | 20260508 11:25:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:25:51.054 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:25:51.054 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:25:51.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:25:51.054 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 5.63e-01 | 20260508 11:25:51.054 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.05e-01, 1.17e-02 / 6.92e-03 | 20260508 11:25:51.054 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.44e+00, 5.88e-01 / 2.34e-01 | 20260508 11:25:51.054 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.20e-01 | 20260508 11:25:51.054 | INFO | PerfReport: 15.99%, DME -- 4.17e+01 / 4.46e+01, 3.61e+00 / 9.16e-01 | 20260508 11:25:51.054 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.86e+01 / 2.08e+01, 1.80e+00 / 1.30e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.42e+00, 4.08e-01 / 2.21e-01 | 20260508 11:25:51.054 | INFO | PerfReport: 2.98%, Metrics -- 9.70e-01 / 1.15e+00, 2.40e-01 / 1.22e-01 | 20260508 11:25:51.054 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.23e-01, 4.73e-02 / 2.89e-02 | 20260508 11:25:51.054 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:25:51.054 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:25:51.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:25:51.054 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 24.40%, Upload -- 2.57e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 31.01%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 2.08%, LockUpload -- 2.19e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 15.18%, LockCompute -- 1.60e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 11:25:51.054 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260508 11:25:51.151 | INFO | Starting a round of Baz IO | 20260508 11:25:54.564 | INFO | Populating RTMetrics report | 20260508 11:25:54.564 | INFO | RTMetrics report populated | 20260508 11:25:54.564 | INFO | Opening tmp RTMetrics json file | 20260508 11:25:54.565 | INFO | Serializing RTMetrics to json | 20260508 11:25:54.566 | INFO | Serialization done, swapping tmp file now... | 20260508 11:25:54.566 | INFO | RTMetrics done | 20260508 11:25:54.633 | INFO | Finished a round of Baz IO: Wrote 1.432 GiB in 3.481 seconds | 20260508 11:25:54.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:25:54.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189026% / 0% / 7.54992e-05% | 20260508 11:25:54.633 | INFO | Baz IO overall: | 20260508 11:25:54.633 | INFO | BytesWritten : 3336564376061 | 20260508 11:25:54.633 | INFO | NumEvents : 1602510678912 | 20260508 11:25:56.190 | INFO | Analyzing chunk frames = [7733248,7733760) | 20260508 11:25:56.190 | INFO | MemInfo: { | 20260508 11:25:56.190 | INFO | "gpu" : " 17 GB", | 20260508 11:25:56.190 | INFO | "huge" : "", | 20260508 11:25:56.190 | INFO | "huge_shared" : " 29 GB", | 20260508 11:25:56.190 | INFO | "malloc" : " 14 GB", | 20260508 11:25:56.190 | INFO | "pinned" : " 29 GB" | 20260508 11:25:56.190 | INFO | } | 20260508 11:25:56.190 | INFO | Perf report for the last 5.13679 seconds: | 20260508 11:25:56.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:25:56.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:56.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:56.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:56.191 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 11:25:56.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:25:56.191 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:25:56.191 | INFO | PRE_HQ : 65.5%, 1.6, 53.9ms, 0 | 20260508 11:25:56.191 | INFO | BAZWRITER : 67.1%, 2.5, 85.4ms, 0 | 20260508 11:25:56.191 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 11:25:56.191 | INFO | Uploaded 36GiB to GPU in 2.56988s (14.0085GiB/s) | 20260508 11:25:56.191 | INFO | Downloaded 38GiB to GPU in 2.72426s (13.9487GiB/s) | 20260508 11:26:01.322 | INFO | Analyzing chunk frames = [7733760,7734272) | 20260508 11:26:01.322 | INFO | MemInfo: { | 20260508 11:26:01.322 | INFO | "gpu" : " 17 GB", | 20260508 11:26:01.322 | INFO | "huge" : "", | 20260508 11:26:01.322 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:01.322 | INFO | "malloc" : " 14 GB", | 20260508 11:26:01.322 | INFO | "pinned" : " 29 GB" | 20260508 11:26:01.322 | INFO | } | 20260508 11:26:01.322 | INFO | Perf report for the last 5.13197 seconds: | 20260508 11:26:01.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:01.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.322 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:26:01.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.323 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:26:01.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:01.323 | INFO | Uploaded 36GiB to GPU in 2.46831s (14.5849GiB/s) | 20260508 11:26:01.323 | INFO | Downloaded 34GiB to GPU in 2.46674s (13.7834GiB/s) | 20260508 11:26:06.459 | INFO | Analyzing chunk frames = [7734272,7734784) | 20260508 11:26:06.459 | INFO | MemInfo: { | 20260508 11:26:06.459 | INFO | "gpu" : " 17 GB", | 20260508 11:26:06.459 | INFO | "huge" : "", | 20260508 11:26:06.460 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:06.460 | INFO | "malloc" : " 14 GB", | 20260508 11:26:06.460 | INFO | "pinned" : " 29 GB" | 20260508 11:26:06.460 | INFO | } | 20260508 11:26:06.460 | INFO | Perf report for the last 5.13731 seconds: | 20260508 11:26:06.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:06.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 11:26:06.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:26:06.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:06.460 | INFO | Uploaded 36GiB to GPU in 2.49817s (14.4105GiB/s) | 20260508 11:26:06.460 | INFO | Downloaded 34GiB to GPU in 2.43508s (13.9626GiB/s) | 20260508 11:26:11.601 | INFO | Analyzing chunk frames = [7734784,7735296) | 20260508 11:26:11.601 | INFO | MemInfo: { | 20260508 11:26:11.601 | INFO | "gpu" : " 17 GB", | 20260508 11:26:11.601 | INFO | "huge" : "", | 20260508 11:26:11.601 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:11.601 | INFO | "malloc" : " 14 GB", | 20260508 11:26:11.601 | INFO | "pinned" : " 29 GB" | 20260508 11:26:11.601 | INFO | } | 20260508 11:26:11.601 | INFO | Perf report for the last 5.14132 seconds: | 20260508 11:26:11.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:11.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.601 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:26:11.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.601 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:26:11.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:11.602 | INFO | Uploaded 36GiB to GPU in 2.46702s (14.5925GiB/s) | 20260508 11:26:11.602 | INFO | Downloaded 34GiB to GPU in 2.44959s (13.8799GiB/s) | 20260508 11:26:16.744 | INFO | Analyzing chunk frames = [7735296,7735808) | 20260508 11:26:16.744 | INFO | MemInfo: { | 20260508 11:26:16.744 | INFO | "gpu" : " 17 GB", | 20260508 11:26:16.744 | INFO | "huge" : "", | 20260508 11:26:16.744 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:16.744 | INFO | "malloc" : " 14 GB", | 20260508 11:26:16.744 | INFO | "pinned" : " 29 GB" | 20260508 11:26:16.744 | INFO | } | 20260508 11:26:16.744 | INFO | Perf report for the last 5.14296 seconds: | 20260508 11:26:16.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:16.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.744 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:26:16.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.744 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 11:26:16.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:16.745 | INFO | Uploaded 36GiB to GPU in 2.28411s (15.7611GiB/s) | 20260508 11:26:16.745 | INFO | Downloaded 34GiB to GPU in 2.49972s (13.6015GiB/s) | 20260508 11:26:21.888 | INFO | Analyzing chunk frames = [7735808,7736320) | 20260508 11:26:21.888 | INFO | MemInfo: { | 20260508 11:26:21.888 | INFO | "gpu" : " 17 GB", | 20260508 11:26:21.888 | INFO | "huge" : "", | 20260508 11:26:21.888 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:21.888 | INFO | "malloc" : " 14 GB", | 20260508 11:26:21.888 | INFO | "pinned" : " 29 GB" | 20260508 11:26:21.888 | INFO | } | 20260508 11:26:21.888 | INFO | Perf report for the last 5.14399 seconds: | 20260508 11:26:21.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:21.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:26:21.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:26:21.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:21.888 | INFO | Uploaded 36GiB to GPU in 2.35876s (15.2623GiB/s) | 20260508 11:26:21.888 | INFO | Downloaded 34GiB to GPU in 2.43071s (13.9877GiB/s) | 20260508 11:26:27.024 | INFO | Analyzing chunk frames = [7736320,7736832) | 20260508 11:26:27.024 | INFO | MemInfo: { | 20260508 11:26:27.024 | INFO | "gpu" : " 17 GB", | 20260508 11:26:27.024 | INFO | "huge" : "", | 20260508 11:26:27.024 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:27.024 | INFO | "malloc" : " 14 GB", | 20260508 11:26:27.024 | INFO | "pinned" : " 29 GB" | 20260508 11:26:27.024 | INFO | } | 20260508 11:26:27.025 | INFO | Perf report for the last 5.13657 seconds: | 20260508 11:26:27.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:27.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 11:26:27.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:26:27.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:27.025 | INFO | Uploaded 36GiB to GPU in 2.46762s (14.589GiB/s) | 20260508 11:26:27.025 | INFO | Downloaded 34GiB to GPU in 2.45513s (13.8486GiB/s) | 20260508 11:26:32.157 | INFO | Analyzing chunk frames = [7736832,7737344) | 20260508 11:26:32.157 | INFO | MemInfo: { | 20260508 11:26:32.157 | INFO | "gpu" : " 17 GB", | 20260508 11:26:32.157 | INFO | "huge" : "", | 20260508 11:26:32.157 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:32.157 | INFO | "malloc" : " 14 GB", | 20260508 11:26:32.157 | INFO | "pinned" : " 29 GB" | 20260508 11:26:32.157 | INFO | } | 20260508 11:26:32.157 | INFO | Perf report for the last 5.13238 seconds: | 20260508 11:26:32.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:32.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 11:26:32.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 11:26:32.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:32.157 | INFO | Uploaded 36GiB to GPU in 2.50284s (14.3837GiB/s) | 20260508 11:26:32.157 | INFO | Downloaded 34GiB to GPU in 2.47516s (13.7365GiB/s) | 20260508 11:26:32.266 | INFO | Starting a round of Baz IO | 20260508 11:26:35.660 | INFO | Populating RTMetrics report | 20260508 11:26:35.660 | INFO | RTMetrics report populated | 20260508 11:26:35.660 | INFO | Opening tmp RTMetrics json file | 20260508 11:26:35.668 | INFO | Serializing RTMetrics to json | 20260508 11:26:35.673 | INFO | Serialization done, swapping tmp file now... | 20260508 11:26:35.673 | INFO | RTMetrics done | 20260508 11:26:35.728 | INFO | Finished a round of Baz IO: Wrote 1.432 GiB in 3.462 seconds | 20260508 11:26:35.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.11% / 0.00% / 59.89% | 20260508 11:26:35.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188728% / 0% / 7.94729e-05% | 20260508 11:26:35.728 | INFO | Baz IO overall: | 20260508 11:26:35.728 | INFO | BytesWritten : 3338102132374 | 20260508 11:26:35.728 | INFO | NumEvents : 1603233695327 | 20260508 11:26:37.301 | INFO | Analyzing chunk frames = [7737344,7737856) | 20260508 11:26:37.301 | INFO | MemInfo: { | 20260508 11:26:37.301 | INFO | "gpu" : " 17 GB", | 20260508 11:26:37.301 | INFO | "huge" : "", | 20260508 11:26:37.301 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:37.301 | INFO | "malloc" : " 14 GB", | 20260508 11:26:37.301 | INFO | "pinned" : " 29 GB" | 20260508 11:26:37.301 | INFO | } | 20260508 11:26:37.301 | INFO | Perf report for the last 5.14401 seconds: | 20260508 11:26:37.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:37.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:37.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:37.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:37.301 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:26:37.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:37.301 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 11:26:37.301 | INFO | PRE_HQ : 65.1%, 1.6, 54.0ms, 0 | 20260508 11:26:37.301 | INFO | BAZWRITER : 66.9%, 2.5, 85.1ms, 0 | 20260508 11:26:37.301 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:26:37.301 | INFO | Uploaded 36GiB to GPU in 2.58992s (13.9GiB/s) | 20260508 11:26:37.301 | INFO | Downloaded 38GiB to GPU in 2.71014s (14.0214GiB/s) | 20260508 11:26:42.438 | INFO | Analyzing chunk frames = [7737856,7738368) | 20260508 11:26:42.438 | INFO | MemInfo: { | 20260508 11:26:42.438 | INFO | "gpu" : " 17 GB", | 20260508 11:26:42.438 | INFO | "huge" : "", | 20260508 11:26:42.438 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:42.438 | INFO | "malloc" : " 14 GB", | 20260508 11:26:42.438 | INFO | "pinned" : " 29 GB" | 20260508 11:26:42.438 | INFO | } | 20260508 11:26:42.439 | INFO | Perf report for the last 5.13753 seconds: | 20260508 11:26:42.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:42.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:26:42.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:26:42.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:42.439 | INFO | Uploaded 36GiB to GPU in 2.44093s (14.7485GiB/s) | 20260508 11:26:42.439 | INFO | Downloaded 34GiB to GPU in 2.45935s (13.8248GiB/s) | 20260508 11:26:47.577 | INFO | Analyzing chunk frames = [7738368,7738880) | 20260508 11:26:47.577 | INFO | MemInfo: { | 20260508 11:26:47.577 | INFO | "gpu" : " 17 GB", | 20260508 11:26:47.577 | INFO | "huge" : "", | 20260508 11:26:47.577 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:47.577 | INFO | "malloc" : " 14 GB", | 20260508 11:26:47.577 | INFO | "pinned" : " 29 GB" | 20260508 11:26:47.577 | INFO | } | 20260508 11:26:47.577 | INFO | Perf report for the last 5.13876 seconds: | 20260508 11:26:47.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:47.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.577 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:26:47.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.577 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:26:47.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:47.578 | INFO | Uploaded 36GiB to GPU in 2.47558s (14.5421GiB/s) | 20260508 11:26:47.578 | INFO | Downloaded 34GiB to GPU in 2.42521s (14.0194GiB/s) | 20260508 11:26:52.718 | INFO | Analyzing chunk frames = [7738880,7739392) | 20260508 11:26:52.718 | INFO | MemInfo: { | 20260508 11:26:52.718 | INFO | "gpu" : " 17 GB", | 20260508 11:26:52.718 | INFO | "huge" : "", | 20260508 11:26:52.718 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:52.718 | INFO | "malloc" : " 14 GB", | 20260508 11:26:52.718 | INFO | "pinned" : " 29 GB" | 20260508 11:26:52.718 | INFO | } | 20260508 11:26:52.718 | INFO | Perf report for the last 5.14056 seconds: | 20260508 11:26:52.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:52.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:26:52.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260508 11:26:52.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:52.718 | INFO | Uploaded 36GiB to GPU in 2.48597s (14.4813GiB/s) | 20260508 11:26:52.718 | INFO | Downloaded 34GiB to GPU in 2.46045s (13.8186GiB/s) | 20260508 11:26:57.858 | INFO | Analyzing chunk frames = [7739392,7739904) | 20260508 11:26:57.858 | INFO | MemInfo: { | 20260508 11:26:57.858 | INFO | "gpu" : " 17 GB", | 20260508 11:26:57.858 | INFO | "huge" : "", | 20260508 11:26:57.858 | INFO | "huge_shared" : " 29 GB", | 20260508 11:26:57.858 | INFO | "malloc" : " 14 GB", | 20260508 11:26:57.858 | INFO | "pinned" : " 29 GB" | 20260508 11:26:57.858 | INFO | } | 20260508 11:26:57.858 | INFO | Perf report for the last 5.14033 seconds: | 20260508 11:26:57.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:26:57.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:26:57.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 11:26:57.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:26:57.858 | INFO | Uploaded 36GiB to GPU in 2.31583s (15.5452GiB/s) | 20260508 11:26:57.858 | INFO | Downloaded 34GiB to GPU in 2.49746s (13.6138GiB/s) | 20260508 11:27:02.999 | INFO | Analyzing chunk frames = [7739904,7740416) | 20260508 11:27:02.999 | INFO | MemInfo: { | 20260508 11:27:02.999 | INFO | "gpu" : " 17 GB", | 20260508 11:27:02.999 | INFO | "huge" : "", | 20260508 11:27:02.999 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:02.999 | INFO | "malloc" : " 14 GB", | 20260508 11:27:02.999 | INFO | "pinned" : " 29 GB" | 20260508 11:27:02.999 | INFO | } | 20260508 11:27:02.999 | INFO | Perf report for the last 5.14101 seconds: | 20260508 11:27:02.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:02.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:02.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:02.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:02.999 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:27:02.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:02.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:02.999 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:27:02.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:02.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:03.000 | INFO | Uploaded 36GiB to GPU in 2.37994s (15.1265GiB/s) | 20260508 11:27:03.000 | INFO | Downloaded 34GiB to GPU in 2.41503s (14.0785GiB/s) | 20260508 11:27:08.137 | INFO | Analyzing chunk frames = [7740416,7740928) | 20260508 11:27:08.137 | INFO | MemInfo: { | 20260508 11:27:08.137 | INFO | "gpu" : " 17 GB", | 20260508 11:27:08.137 | INFO | "huge" : "", | 20260508 11:27:08.137 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:08.137 | INFO | "malloc" : " 14 GB", | 20260508 11:27:08.137 | INFO | "pinned" : " 29 GB" | 20260508 11:27:08.137 | INFO | } | 20260508 11:27:08.137 | INFO | Perf report for the last 5.1379 seconds: | 20260508 11:27:08.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:08.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:27:08.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:27:08.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:08.137 | INFO | Uploaded 36GiB to GPU in 2.45595s (14.6583GiB/s) | 20260508 11:27:08.137 | INFO | Downloaded 34GiB to GPU in 2.43313s (13.9738GiB/s) | 20260508 11:27:13.274 | INFO | Analyzing chunk frames = [7740928,7741440) | 20260508 11:27:13.274 | INFO | MemInfo: { | 20260508 11:27:13.274 | INFO | "gpu" : " 17 GB", | 20260508 11:27:13.274 | INFO | "huge" : "", | 20260508 11:27:13.274 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:13.274 | INFO | "malloc" : " 14 GB", | 20260508 11:27:13.274 | INFO | "pinned" : " 29 GB" | 20260508 11:27:13.274 | INFO | } | 20260508 11:27:13.274 | INFO | Perf report for the last 5.13666 seconds: | 20260508 11:27:13.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:13.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:27:13.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:27:13.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:13.274 | INFO | Uploaded 36GiB to GPU in 2.46434s (14.6084GiB/s) | 20260508 11:27:13.274 | INFO | Downloaded 34GiB to GPU in 2.44615s (13.8994GiB/s) | 20260508 11:27:13.274 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:27:13.274 | INFO | Total mem usage: 16.7 GiB | 20260508 11:27:13.274 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:27:13.274 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:27:13.274 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:27:13.275 | INFO | High Water Breakdown: | 20260508 11:27:13.275 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:27:13.275 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:27:13.275 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:27:13.275 | INFO | ------------------------------------------ | 20260508 11:27:13.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:27:13.275 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:27:13.275 | INFO | Total mem usage: 26.5 GiB | 20260508 11:27:13.275 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:27:13.275 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:27:13.275 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:27:13.275 | INFO | High Water Breakdown: | 20260508 11:27:13.275 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:27:13.275 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:27:13.275 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:27:13.275 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:27:13.275 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:27:13.275 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:27:13.275 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:27:13.275 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:27:13.275 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:27:13.275 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:27:13.275 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:27:13.275 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:27:13.275 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:27:13.275 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:27:13.275 | INFO | ------------------------------------------ | 20260508 11:27:13.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:27:13.275 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:27:13.275 | INFO | Total mem usage: 27.8 GiB | 20260508 11:27:13.275 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:27:13.275 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:27:13.275 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:27:13.275 | INFO | High Water Breakdown: | 20260508 11:27:13.275 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:27:13.275 | INFO | ------------------------------------------ | 20260508 11:27:13.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:27:13.275 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:27:13.275 | INFO | Total mem usage: 15.7 GiB | 20260508 11:27:13.275 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:27:13.275 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:27:13.275 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:27:13.275 | INFO | High Water Breakdown: | 20260508 11:27:13.275 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:27:13.275 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:27:13.275 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:27:13.275 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:27:13.275 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:27:13.275 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:27:13.275 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:27:13.275 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:27:13.275 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:27:13.275 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:27:13.275 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:27:13.275 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:27:13.275 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:27:13.275 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:27:13.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:27:13.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:27:13.275 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:27:13.275 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:27:13.275 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:27:13.275 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:27:13.275 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:27:13.275 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:27:13.275 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:27:13.275 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:27:13.275 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:27:13.275 | INFO | ------------------------------------------ | 20260508 11:27:13.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:27:13.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:27:13.275 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:27:13.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:27:13.275 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 6.46e-01 | 20260508 11:27:13.275 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.15e-02 / 4.76e-03 | 20260508 11:27:13.275 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.39e+00, 5.95e-01 / 2.18e-01 | 20260508 11:27:13.276 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 1.05e-01 | 20260508 11:27:13.276 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.56e+01, 3.64e+00 / 1.08e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.75e+00 / 1.18e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.05e-01 / 2.56e-01 | 20260508 11:27:13.276 | INFO | PerfReport: 2.99%, Metrics -- 9.75e-01 / 1.10e+00, 2.39e-01 / 9.28e-02 | 20260508 11:27:13.276 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.26e-01, 4.76e-02 / 2.95e-02 | 20260508 11:27:13.276 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:27:13.276 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:27:13.276 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:27:13.276 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 24.29%, Upload -- 2.55e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 24.63%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 31.02%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 2.17%, LockUpload -- 2.29e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 2.60%, LockDownload -- 2.74e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 15.27%, LockCompute -- 1.61e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:27:13.276 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260508 11:27:13.371 | INFO | Starting a round of Baz IO | 20260508 11:27:16.794 | INFO | Populating RTMetrics report | 20260508 11:27:16.794 | INFO | RTMetrics report populated | 20260508 11:27:16.794 | INFO | Opening tmp RTMetrics json file | 20260508 11:27:16.795 | INFO | Serializing RTMetrics to json | 20260508 11:27:16.803 | INFO | Serialization done, swapping tmp file now... | 20260508 11:27:16.811 | INFO | RTMetrics done | 20260508 11:27:16.859 | INFO | Finished a round of Baz IO: Wrote 1.431 GiB in 3.488 seconds | 20260508 11:27:16.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.89% | 20260508 11:27:16.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188279% / 0% / 5.96046e-05% | 20260508 11:27:16.859 | INFO | Baz IO overall: | 20260508 11:27:16.859 | INFO | BytesWritten : 3339638905191 | 20260508 11:27:16.859 | INFO | NumEvents : 1603955952057 | 20260508 11:27:18.410 | INFO | Analyzing chunk frames = [7741440,7741952) | 20260508 11:27:18.410 | INFO | MemInfo: { | 20260508 11:27:18.410 | INFO | "gpu" : " 17 GB", | 20260508 11:27:18.410 | INFO | "huge" : "", | 20260508 11:27:18.410 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:18.410 | INFO | "malloc" : " 14 GB", | 20260508 11:27:18.410 | INFO | "pinned" : " 29 GB" | 20260508 11:27:18.410 | INFO | } | 20260508 11:27:18.410 | INFO | Perf report for the last 5.13619 seconds: | 20260508 11:27:18.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:18.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:18.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:18.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:18.410 | INFO | ANALYSIS : 68.2%, 3.0, 106.0ms, 0 | 20260508 11:27:18.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:18.410 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 11:27:18.410 | INFO | PRE_HQ : 65.5%, 1.6, 54.1ms, 0 | 20260508 11:27:18.410 | INFO | BAZWRITER : 67.2%, 2.5, 84.7ms, 0 | 20260508 11:27:18.410 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:27:18.410 | INFO | Uploaded 36GiB to GPU in 2.56381s (14.0416GiB/s) | 20260508 11:27:18.411 | INFO | Downloaded 38GiB to GPU in 2.70995s (14.0224GiB/s) | 20260508 11:27:23.555 | INFO | Analyzing chunk frames = [7741952,7742464) | 20260508 11:27:23.555 | INFO | MemInfo: { | 20260508 11:27:23.555 | INFO | "gpu" : " 17 GB", | 20260508 11:27:23.555 | INFO | "huge" : "", | 20260508 11:27:23.555 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:23.555 | INFO | "malloc" : " 14 GB", | 20260508 11:27:23.555 | INFO | "pinned" : " 29 GB" | 20260508 11:27:23.555 | INFO | } | 20260508 11:27:23.555 | INFO | Perf report for the last 5.14521 seconds: | 20260508 11:27:23.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:23.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.555 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:27:23.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.555 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:27:23.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:23.556 | INFO | Uploaded 36GiB to GPU in 2.46073s (14.6298GiB/s) | 20260508 11:27:23.556 | INFO | Downloaded 34GiB to GPU in 2.47604s (13.7316GiB/s) | 20260508 11:27:28.700 | INFO | Analyzing chunk frames = [7742464,7742976) | 20260508 11:27:28.700 | INFO | MemInfo: { | 20260508 11:27:28.700 | INFO | "gpu" : " 17 GB", | 20260508 11:27:28.700 | INFO | "huge" : "", | 20260508 11:27:28.700 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:28.700 | INFO | "malloc" : " 14 GB", | 20260508 11:27:28.700 | INFO | "pinned" : " 29 GB" | 20260508 11:27:28.700 | INFO | } | 20260508 11:27:28.700 | INFO | Perf report for the last 5.14453 seconds: | 20260508 11:27:28.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:28.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:27:28.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:27:28.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:28.700 | INFO | Uploaded 36GiB to GPU in 2.49015s (14.457GiB/s) | 20260508 11:27:28.700 | INFO | Downloaded 34GiB to GPU in 2.45151s (13.869GiB/s) | 20260508 11:27:33.838 | INFO | Analyzing chunk frames = [7742976,7743488) | 20260508 11:27:33.838 | INFO | MemInfo: { | 20260508 11:27:33.838 | INFO | "gpu" : " 17 GB", | 20260508 11:27:33.838 | INFO | "huge" : "", | 20260508 11:27:33.838 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:33.838 | INFO | "malloc" : " 14 GB", | 20260508 11:27:33.838 | INFO | "pinned" : " 29 GB" | 20260508 11:27:33.838 | INFO | } | 20260508 11:27:33.838 | INFO | Perf report for the last 5.13855 seconds: | 20260508 11:27:33.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:33.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.838 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:27:33.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.838 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:27:33.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:33.839 | INFO | Uploaded 36GiB to GPU in 2.52507s (14.2571GiB/s) | 20260508 11:27:33.839 | INFO | Downloaded 34GiB to GPU in 2.46672s (13.7835GiB/s) | 20260508 11:27:38.969 | INFO | Analyzing chunk frames = [7743488,7744000) | 20260508 11:27:38.969 | INFO | MemInfo: { | 20260508 11:27:38.969 | INFO | "gpu" : " 17 GB", | 20260508 11:27:38.969 | INFO | "huge" : "", | 20260508 11:27:38.969 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:38.969 | INFO | "malloc" : " 14 GB", | 20260508 11:27:38.969 | INFO | "pinned" : " 29 GB" | 20260508 11:27:38.969 | INFO | } | 20260508 11:27:38.969 | INFO | Perf report for the last 5.13047 seconds: | 20260508 11:27:38.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:38.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:27:38.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:27:38.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:38.969 | INFO | Uploaded 36GiB to GPU in 2.26741s (15.8771GiB/s) | 20260508 11:27:38.969 | INFO | Downloaded 34GiB to GPU in 2.48504s (13.6819GiB/s) | 20260508 11:27:44.111 | INFO | Analyzing chunk frames = [7744000,7744512) | 20260508 11:27:44.111 | INFO | MemInfo: { | 20260508 11:27:44.111 | INFO | "gpu" : " 17 GB", | 20260508 11:27:44.111 | INFO | "huge" : "", | 20260508 11:27:44.111 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:44.112 | INFO | "malloc" : " 14 GB", | 20260508 11:27:44.112 | INFO | "pinned" : " 29 GB" | 20260508 11:27:44.112 | INFO | } | 20260508 11:27:44.112 | INFO | Perf report for the last 5.14287 seconds: | 20260508 11:27:44.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:44.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:27:44.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:27:44.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:44.112 | INFO | Uploaded 36GiB to GPU in 2.37437s (15.1619GiB/s) | 20260508 11:27:44.112 | INFO | Downloaded 34GiB to GPU in 2.43411s (13.9682GiB/s) | 20260508 11:27:49.252 | INFO | Analyzing chunk frames = [7744512,7745024) | 20260508 11:27:49.252 | INFO | MemInfo: { | 20260508 11:27:49.252 | INFO | "gpu" : " 17 GB", | 20260508 11:27:49.252 | INFO | "huge" : "", | 20260508 11:27:49.252 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:49.252 | INFO | "malloc" : " 14 GB", | 20260508 11:27:49.252 | INFO | "pinned" : " 29 GB" | 20260508 11:27:49.252 | INFO | } | 20260508 11:27:49.252 | INFO | Perf report for the last 5.14005 seconds: | 20260508 11:27:49.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:49.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.252 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 11:27:49.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.252 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:27:49.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:49.253 | INFO | Uploaded 36GiB to GPU in 2.39068s (15.0585GiB/s) | 20260508 11:27:49.253 | INFO | Downloaded 34GiB to GPU in 2.42182s (14.039GiB/s) | 20260508 11:27:54.389 | INFO | Analyzing chunk frames = [7745024,7745536) | 20260508 11:27:54.389 | INFO | MemInfo: { | 20260508 11:27:54.389 | INFO | "gpu" : " 17 GB", | 20260508 11:27:54.389 | INFO | "huge" : "", | 20260508 11:27:54.389 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:54.389 | INFO | "malloc" : " 14 GB", | 20260508 11:27:54.389 | INFO | "pinned" : " 29 GB" | 20260508 11:27:54.389 | INFO | } | 20260508 11:27:54.389 | INFO | Perf report for the last 5.13737 seconds: | 20260508 11:27:54.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:54.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.389 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:27:54.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.389 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:27:54.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:54.390 | INFO | Uploaded 36GiB to GPU in 2.45359s (14.6724GiB/s) | 20260508 11:27:54.390 | INFO | Downloaded 34GiB to GPU in 2.45305s (13.8603GiB/s) | 20260508 11:27:54.501 | INFO | Starting a round of Baz IO | 20260508 11:27:57.910 | INFO | Populating RTMetrics report | 20260508 11:27:57.910 | INFO | RTMetrics report populated | 20260508 11:27:57.910 | INFO | Opening tmp RTMetrics json file | 20260508 11:27:57.917 | INFO | Serializing RTMetrics to json | 20260508 11:27:57.919 | INFO | Serialization done, swapping tmp file now... | 20260508 11:27:57.919 | INFO | RTMetrics done | 20260508 11:27:57.973 | INFO | Finished a round of Baz IO: Wrote 1.43 GiB in 3.472 seconds | 20260508 11:27:57.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:27:57.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188518% / 3.97364e-06% / 4.37101e-05% | 20260508 11:27:57.973 | INFO | Baz IO overall: | 20260508 11:27:57.973 | INFO | BytesWritten : 3341174735064 | 20260508 11:27:57.973 | INFO | NumEvents : 1604677574170 | 20260508 11:27:59.526 | INFO | Analyzing chunk frames = [7745536,7746048) | 20260508 11:27:59.526 | INFO | MemInfo: { | 20260508 11:27:59.526 | INFO | "gpu" : " 17 GB", | 20260508 11:27:59.526 | INFO | "huge" : "", | 20260508 11:27:59.526 | INFO | "huge_shared" : " 29 GB", | 20260508 11:27:59.526 | INFO | "malloc" : " 14 GB", | 20260508 11:27:59.526 | INFO | "pinned" : " 29 GB" | 20260508 11:27:59.527 | INFO | } | 20260508 11:27:59.527 | INFO | Perf report for the last 5.13739 seconds: | 20260508 11:27:59.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:27:59.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:59.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:59.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:59.527 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 11:27:59.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:27:59.527 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260508 11:27:59.527 | INFO | PRE_HQ : 66.1%, 1.7, 58.5ms, 0 | 20260508 11:27:59.527 | INFO | BAZWRITER : 66.8%, 2.6, 89.0ms, 0 | 20260508 11:27:59.527 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:27:59.527 | INFO | Uploaded 36GiB to GPU in 2.61849s (13.7484GiB/s) | 20260508 11:27:59.527 | INFO | Downloaded 38GiB to GPU in 2.70099s (14.0689GiB/s) | 20260508 11:28:04.666 | INFO | Analyzing chunk frames = [7746048,7746560) | 20260508 11:28:04.666 | INFO | MemInfo: { | 20260508 11:28:04.666 | INFO | "gpu" : " 17 GB", | 20260508 11:28:04.666 | INFO | "huge" : "", | 20260508 11:28:04.666 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:04.666 | INFO | "malloc" : " 14 GB", | 20260508 11:28:04.666 | INFO | "pinned" : " 29 GB" | 20260508 11:28:04.666 | INFO | } | 20260508 11:28:04.666 | INFO | Perf report for the last 5.13966 seconds: | 20260508 11:28:04.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:04.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.666 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 11:28:04.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.666 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:28:04.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:04.667 | INFO | Uploaded 36GiB to GPU in 2.48544s (14.4844GiB/s) | 20260508 11:28:04.667 | INFO | Downloaded 34GiB to GPU in 2.46166s (13.8118GiB/s) | 20260508 11:28:09.810 | INFO | Analyzing chunk frames = [7746560,7747072) | 20260508 11:28:09.810 | INFO | MemInfo: { | 20260508 11:28:09.810 | INFO | "gpu" : " 17 GB", | 20260508 11:28:09.810 | INFO | "huge" : "", | 20260508 11:28:09.810 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:09.810 | INFO | "malloc" : " 14 GB", | 20260508 11:28:09.810 | INFO | "pinned" : " 29 GB" | 20260508 11:28:09.810 | INFO | } | 20260508 11:28:09.810 | INFO | Perf report for the last 5.14408 seconds: | 20260508 11:28:09.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:09.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.810 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:28:09.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.810 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:28:09.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:09.811 | INFO | Uploaded 36GiB to GPU in 2.5051s (14.3707GiB/s) | 20260508 11:28:09.811 | INFO | Downloaded 34GiB to GPU in 2.4404s (13.9322GiB/s) | 20260508 11:28:14.948 | INFO | Analyzing chunk frames = [7747072,7747584) | 20260508 11:28:14.948 | INFO | MemInfo: { | 20260508 11:28:14.948 | INFO | "gpu" : " 17 GB", | 20260508 11:28:14.948 | INFO | "huge" : "", | 20260508 11:28:14.949 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:14.949 | INFO | "malloc" : " 14 GB", | 20260508 11:28:14.949 | INFO | "pinned" : " 29 GB" | 20260508 11:28:14.949 | INFO | } | 20260508 11:28:14.949 | INFO | Perf report for the last 5.13836 seconds: | 20260508 11:28:14.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:14.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:28:14.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:28:14.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:14.949 | INFO | Uploaded 36GiB to GPU in 2.51127s (14.3354GiB/s) | 20260508 11:28:14.949 | INFO | Downloaded 34GiB to GPU in 2.45139s (13.8697GiB/s) | 20260508 11:28:20.085 | INFO | Analyzing chunk frames = [7747584,7748096) | 20260508 11:28:20.085 | INFO | MemInfo: { | 20260508 11:28:20.085 | INFO | "gpu" : " 17 GB", | 20260508 11:28:20.085 | INFO | "huge" : "", | 20260508 11:28:20.085 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:20.085 | INFO | "malloc" : " 14 GB", | 20260508 11:28:20.085 | INFO | "pinned" : " 29 GB" | 20260508 11:28:20.085 | INFO | } | 20260508 11:28:20.085 | INFO | Perf report for the last 5.1366 seconds: | 20260508 11:28:20.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:20.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.086 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 11:28:20.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.086 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:28:20.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:20.086 | INFO | Uploaded 36GiB to GPU in 2.44654s (14.7147GiB/s) | 20260508 11:28:20.086 | INFO | Downloaded 34GiB to GPU in 2.52091s (13.4872GiB/s) | 20260508 11:28:25.221 | INFO | Analyzing chunk frames = [7748096,7748608) | 20260508 11:28:25.221 | INFO | MemInfo: { | 20260508 11:28:25.221 | INFO | "gpu" : " 17 GB", | 20260508 11:28:25.221 | INFO | "huge" : "", | 20260508 11:28:25.221 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:25.221 | INFO | "malloc" : " 14 GB", | 20260508 11:28:25.221 | INFO | "pinned" : " 29 GB" | 20260508 11:28:25.221 | INFO | } | 20260508 11:28:25.221 | INFO | Perf report for the last 5.13588 seconds: | 20260508 11:28:25.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:25.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:28:25.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:28:25.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:25.221 | INFO | Uploaded 36GiB to GPU in 2.3985s (15.0094GiB/s) | 20260508 11:28:25.221 | INFO | Downloaded 34GiB to GPU in 2.41638s (14.0706GiB/s) | 20260508 11:28:30.366 | INFO | Analyzing chunk frames = [7748608,7749120) | 20260508 11:28:30.366 | INFO | MemInfo: { | 20260508 11:28:30.366 | INFO | "gpu" : " 17 GB", | 20260508 11:28:30.366 | INFO | "huge" : "", | 20260508 11:28:30.366 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:30.367 | INFO | "malloc" : " 14 GB", | 20260508 11:28:30.367 | INFO | "pinned" : " 29 GB" | 20260508 11:28:30.367 | INFO | } | 20260508 11:28:30.367 | INFO | Perf report for the last 5.14563 seconds: | 20260508 11:28:30.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:30.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 11:28:30.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 11:28:30.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:30.367 | INFO | Uploaded 36GiB to GPU in 2.45845s (14.6433GiB/s) | 20260508 11:28:30.367 | INFO | Downloaded 34GiB to GPU in 2.43773s (13.9474GiB/s) | 20260508 11:28:35.506 | INFO | Analyzing chunk frames = [7749120,7749632) | 20260508 11:28:35.506 | INFO | MemInfo: { | 20260508 11:28:35.506 | INFO | "gpu" : " 17 GB", | 20260508 11:28:35.506 | INFO | "huge" : "", | 20260508 11:28:35.506 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:35.506 | INFO | "malloc" : " 14 GB", | 20260508 11:28:35.506 | INFO | "pinned" : " 29 GB" | 20260508 11:28:35.506 | INFO | } | 20260508 11:28:35.506 | INFO | Perf report for the last 5.13957 seconds: | 20260508 11:28:35.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:35.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.507 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 11:28:35.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.507 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:28:35.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:35.507 | INFO | Uploaded 36GiB to GPU in 2.4743s (14.5496GiB/s) | 20260508 11:28:35.507 | INFO | Downloaded 34GiB to GPU in 2.44406s (13.9113GiB/s) | 20260508 11:28:35.507 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:28:35.507 | INFO | Total mem usage: 16.7 GiB | 20260508 11:28:35.507 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:28:35.507 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:28:35.507 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:28:35.507 | INFO | High Water Breakdown: | 20260508 11:28:35.507 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:28:35.507 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:28:35.507 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:28:35.507 | INFO | ------------------------------------------ | 20260508 11:28:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:28:35.507 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:28:35.507 | INFO | Total mem usage: 26.5 GiB | 20260508 11:28:35.507 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:28:35.507 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:28:35.507 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:28:35.507 | INFO | High Water Breakdown: | 20260508 11:28:35.507 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:28:35.507 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:28:35.507 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:28:35.507 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:28:35.507 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:28:35.507 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:28:35.507 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:28:35.507 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:28:35.507 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:28:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:28:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:28:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:28:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:28:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:28:35.507 | INFO | ------------------------------------------ | 20260508 11:28:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:28:35.507 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:28:35.507 | INFO | Total mem usage: 27.8 GiB | 20260508 11:28:35.507 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:28:35.507 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:28:35.507 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:28:35.507 | INFO | High Water Breakdown: | 20260508 11:28:35.507 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:28:35.507 | INFO | ------------------------------------------ | 20260508 11:28:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:28:35.507 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:28:35.507 | INFO | Total mem usage: 15.7 GiB | 20260508 11:28:35.507 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:28:35.507 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:28:35.507 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:28:35.507 | INFO | High Water Breakdown: | 20260508 11:28:35.507 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:28:35.507 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:28:35.507 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:28:35.507 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:28:35.507 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:28:35.507 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:28:35.507 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:28:35.507 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:28:35.507 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:28:35.507 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:28:35.507 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:28:35.507 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:28:35.507 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:28:35.507 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:28:35.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:28:35.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:28:35.507 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:28:35.507 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:28:35.507 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:28:35.507 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:28:35.507 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:28:35.507 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:28:35.507 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:28:35.507 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:28:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:28:35.507 | INFO | ------------------------------------------ | 20260508 11:28:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:28:35.507 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:28:35.507 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:28:35.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:28:35.508 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 5.69e-01 | 20260508 11:28:35.508 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.15e-02 / 3.58e-03 | 20260508 11:28:35.508 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.46e+00, 5.92e-01 / 1.82e-01 | 20260508 11:28:35.508 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.10e-01 | 20260508 11:28:35.508 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.50e+01, 3.68e+00 / 7.18e-01 | 20260508 11:28:35.508 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.83e+00 / 1.15e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.51e+00, 4.09e-01 / 1.77e-01 | 20260508 11:28:35.508 | INFO | PerfReport: 2.98%, Metrics -- 9.74e-01 / 1.17e+00, 2.48e-01 / 1.03e-01 | 20260508 11:28:35.508 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.28e-01, 4.80e-02 / 1.95e-02 | 20260508 11:28:35.508 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:28:35.508 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:28:35.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:28:35.508 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 24.39%, Upload -- 2.57e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 31.04%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 2.23%, LockUpload -- 2.35e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 2.49%, LockDownload -- 2.62e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 15.23%, LockCompute -- 1.60e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:28:35.508 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260508 11:28:35.604 | INFO | Starting a round of Baz IO | 20260508 11:28:39.017 | INFO | Populating RTMetrics report | 20260508 11:28:39.018 | INFO | RTMetrics report populated | 20260508 11:28:39.018 | INFO | Opening tmp RTMetrics json file | 20260508 11:28:39.027 | INFO | Serializing RTMetrics to json | 20260508 11:28:39.029 | INFO | Serialization done, swapping tmp file now... | 20260508 11:28:39.029 | INFO | RTMetrics done | 20260508 11:28:39.088 | INFO | Finished a round of Baz IO: Wrote 1.433 GiB in 3.484 seconds | 20260508 11:28:39.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:28:39.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18913% / 0% / 5.96046e-05% | 20260508 11:28:39.088 | INFO | Baz IO overall: | 20260508 11:28:39.089 | INFO | BytesWritten : 3342712995563 | 20260508 11:28:39.089 | INFO | NumEvents : 1605401139166 | 20260508 11:28:40.647 | INFO | Analyzing chunk frames = [7749632,7750144) | 20260508 11:28:40.647 | INFO | MemInfo: { | 20260508 11:28:40.647 | INFO | "gpu" : " 17 GB", | 20260508 11:28:40.647 | INFO | "huge" : "", | 20260508 11:28:40.647 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:40.647 | INFO | "malloc" : " 14 GB", | 20260508 11:28:40.647 | INFO | "pinned" : " 29 GB" | 20260508 11:28:40.647 | INFO | } | 20260508 11:28:40.647 | INFO | Perf report for the last 5.14052 seconds: | 20260508 11:28:40.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:40.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:40.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:40.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:40.647 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 11:28:40.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:40.647 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:28:40.647 | INFO | PRE_HQ : 65.4%, 1.6, 53.9ms, 0 | 20260508 11:28:40.647 | INFO | BAZWRITER : 67.1%, 2.5, 84.8ms, 0 | 20260508 11:28:40.647 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:28:40.648 | INFO | Uploaded 36GiB to GPU in 2.63388s (13.6681GiB/s) | 20260508 11:28:40.648 | INFO | Downloaded 38GiB to GPU in 2.70869s (14.0289GiB/s) | 20260508 11:28:45.783 | INFO | Analyzing chunk frames = [7750144,7750656) | 20260508 11:28:45.783 | INFO | MemInfo: { | 20260508 11:28:45.783 | INFO | "gpu" : " 17 GB", | 20260508 11:28:45.783 | INFO | "huge" : "", | 20260508 11:28:45.783 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:45.783 | INFO | "malloc" : " 14 GB", | 20260508 11:28:45.783 | INFO | "pinned" : " 29 GB" | 20260508 11:28:45.783 | INFO | } | 20260508 11:28:45.783 | INFO | Perf report for the last 5.13646 seconds: | 20260508 11:28:45.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:45.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.784 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:28:45.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.784 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:28:45.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:45.784 | INFO | Uploaded 36GiB to GPU in 2.40932s (14.942GiB/s) | 20260508 11:28:45.784 | INFO | Downloaded 34GiB to GPU in 2.46388s (13.7994GiB/s) | 20260508 11:28:50.921 | INFO | Analyzing chunk frames = [7750656,7751168) | 20260508 11:28:50.921 | INFO | MemInfo: { | 20260508 11:28:50.921 | INFO | "gpu" : " 17 GB", | 20260508 11:28:50.921 | INFO | "huge" : "", | 20260508 11:28:50.921 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:50.921 | INFO | "malloc" : " 14 GB", | 20260508 11:28:50.921 | INFO | "pinned" : " 29 GB" | 20260508 11:28:50.921 | INFO | } | 20260508 11:28:50.921 | INFO | Perf report for the last 5.1378 seconds: | 20260508 11:28:50.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:50.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.921 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:28:50.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.921 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:28:50.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:50.922 | INFO | Uploaded 36GiB to GPU in 2.48485s (14.4878GiB/s) | 20260508 11:28:50.922 | INFO | Downloaded 34GiB to GPU in 2.45426s (13.8534GiB/s) | 20260508 11:28:56.058 | INFO | Analyzing chunk frames = [7751168,7751680) | 20260508 11:28:56.058 | INFO | MemInfo: { | 20260508 11:28:56.058 | INFO | "gpu" : " 17 GB", | 20260508 11:28:56.058 | INFO | "huge" : "", | 20260508 11:28:56.059 | INFO | "huge_shared" : " 29 GB", | 20260508 11:28:56.059 | INFO | "malloc" : " 14 GB", | 20260508 11:28:56.059 | INFO | "pinned" : " 29 GB" | 20260508 11:28:56.059 | INFO | } | 20260508 11:28:56.060 | INFO | Perf report for the last 5.13829 seconds: | 20260508 11:28:56.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:28:56.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:28:56.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:28:56.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:28:56.060 | INFO | Uploaded 36GiB to GPU in 2.4568s (14.6532GiB/s) | 20260508 11:28:56.060 | INFO | Downloaded 34GiB to GPU in 2.44882s (13.8842GiB/s) | 20260508 11:29:01.207 | INFO | Analyzing chunk frames = [7751680,7752192) | 20260508 11:29:01.207 | INFO | MemInfo: { | 20260508 11:29:01.207 | INFO | "gpu" : " 17 GB", | 20260508 11:29:01.207 | INFO | "huge" : "", | 20260508 11:29:01.207 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:01.207 | INFO | "malloc" : " 14 GB", | 20260508 11:29:01.207 | INFO | "pinned" : " 29 GB" | 20260508 11:29:01.207 | INFO | } | 20260508 11:29:01.207 | INFO | Perf report for the last 5.14769 seconds: | 20260508 11:29:01.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:01.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.207 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 11:29:01.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.207 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:29:01.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:01.208 | INFO | Uploaded 36GiB to GPU in 2.37866s (15.1346GiB/s) | 20260508 11:29:01.208 | INFO | Downloaded 34GiB to GPU in 2.52521s (13.4642GiB/s) | 20260508 11:29:06.343 | INFO | Analyzing chunk frames = [7752192,7752704) | 20260508 11:29:06.343 | INFO | MemInfo: { | 20260508 11:29:06.343 | INFO | "gpu" : " 17 GB", | 20260508 11:29:06.343 | INFO | "huge" : "", | 20260508 11:29:06.343 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:06.343 | INFO | "malloc" : " 14 GB", | 20260508 11:29:06.343 | INFO | "pinned" : " 29 GB" | 20260508 11:29:06.343 | INFO | } | 20260508 11:29:06.343 | INFO | Perf report for the last 5.13615 seconds: | 20260508 11:29:06.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:06.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.344 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:29:06.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.344 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 11:29:06.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:06.344 | INFO | Uploaded 36GiB to GPU in 2.36753s (15.2057GiB/s) | 20260508 11:29:06.344 | INFO | Downloaded 34GiB to GPU in 2.45285s (13.8614GiB/s) | 20260508 11:29:11.479 | INFO | Analyzing chunk frames = [7752704,7753216) | 20260508 11:29:11.479 | INFO | MemInfo: { | 20260508 11:29:11.479 | INFO | "gpu" : " 17 GB", | 20260508 11:29:11.479 | INFO | "huge" : "", | 20260508 11:29:11.479 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:11.479 | INFO | "malloc" : " 14 GB", | 20260508 11:29:11.479 | INFO | "pinned" : " 29 GB" | 20260508 11:29:11.479 | INFO | } | 20260508 11:29:11.479 | INFO | Perf report for the last 5.13568 seconds: | 20260508 11:29:11.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:11.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:29:11.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:29:11.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:11.479 | INFO | Uploaded 36GiB to GPU in 2.41143s (14.9289GiB/s) | 20260508 11:29:11.479 | INFO | Downloaded 34GiB to GPU in 2.46455s (13.7956GiB/s) | 20260508 11:29:16.618 | INFO | Analyzing chunk frames = [7753216,7753728) | 20260508 11:29:16.618 | INFO | MemInfo: { | 20260508 11:29:16.618 | INFO | "gpu" : " 17 GB", | 20260508 11:29:16.618 | INFO | "huge" : "", | 20260508 11:29:16.618 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:16.618 | INFO | "malloc" : " 14 GB", | 20260508 11:29:16.618 | INFO | "pinned" : " 29 GB" | 20260508 11:29:16.618 | INFO | } | 20260508 11:29:16.618 | INFO | Perf report for the last 5.1392 seconds: | 20260508 11:29:16.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:16.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.618 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 11:29:16.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.618 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:29:16.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:16.619 | INFO | Uploaded 36GiB to GPU in 2.45074s (14.6894GiB/s) | 20260508 11:29:16.619 | INFO | Downloaded 34GiB to GPU in 2.47735s (13.7243GiB/s) | 20260508 11:29:16.716 | INFO | Starting a round of Baz IO | 20260508 11:29:20.118 | INFO | Populating RTMetrics report | 20260508 11:29:20.119 | INFO | RTMetrics report populated | 20260508 11:29:20.119 | INFO | Opening tmp RTMetrics json file | 20260508 11:29:20.128 | INFO | Serializing RTMetrics to json | 20260508 11:29:20.130 | INFO | Serialization done, swapping tmp file now... | 20260508 11:29:20.130 | INFO | RTMetrics done | 20260508 11:29:20.190 | INFO | Finished a round of Baz IO: Wrote 1.432 GiB in 3.474 seconds | 20260508 11:29:20.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:29:20.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188788% / 0% / 4.37101e-05% | 20260508 11:29:20.190 | INFO | Baz IO overall: | 20260508 11:29:20.190 | INFO | BytesWritten : 3344250297214 | 20260508 11:29:20.190 | INFO | NumEvents : 1606124133416 | 20260508 11:29:21.759 | INFO | Analyzing chunk frames = [7753728,7754240) | 20260508 11:29:21.759 | INFO | MemInfo: { | 20260508 11:29:21.759 | INFO | "gpu" : " 17 GB", | 20260508 11:29:21.759 | INFO | "huge" : "", | 20260508 11:29:21.759 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:21.759 | INFO | "malloc" : " 14 GB", | 20260508 11:29:21.759 | INFO | "pinned" : " 29 GB" | 20260508 11:29:21.759 | INFO | } | 20260508 11:29:21.759 | INFO | Perf report for the last 5.1409 seconds: | 20260508 11:29:21.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:21.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:21.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:21.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:21.759 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:29:21.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:21.759 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:29:21.759 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 11:29:21.759 | INFO | BAZWRITER : 66.8%, 2.5, 84.3ms, 0 | 20260508 11:29:21.759 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:29:21.760 | INFO | Uploaded 36GiB to GPU in 2.61945s (13.7434GiB/s) | 20260508 11:29:21.760 | INFO | Downloaded 38GiB to GPU in 2.7191s (13.9752GiB/s) | 20260508 11:29:26.897 | INFO | Analyzing chunk frames = [7754240,7754752) | 20260508 11:29:26.897 | INFO | MemInfo: { | 20260508 11:29:26.897 | INFO | "gpu" : " 17 GB", | 20260508 11:29:26.897 | INFO | "huge" : "", | 20260508 11:29:26.897 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:26.897 | INFO | "malloc" : " 14 GB", | 20260508 11:29:26.897 | INFO | "pinned" : " 29 GB" | 20260508 11:29:26.897 | INFO | } | 20260508 11:29:26.897 | INFO | Perf report for the last 5.13798 seconds: | 20260508 11:29:26.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:26.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.897 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:29:26.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.897 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:29:26.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:26.898 | INFO | Uploaded 36GiB to GPU in 2.45338s (14.6736GiB/s) | 20260508 11:29:26.898 | INFO | Downloaded 34GiB to GPU in 2.46509s (13.7926GiB/s) | 20260508 11:29:32.035 | INFO | Analyzing chunk frames = [7754752,7755264) | 20260508 11:29:32.035 | INFO | MemInfo: { | 20260508 11:29:32.035 | INFO | "gpu" : " 17 GB", | 20260508 11:29:32.035 | INFO | "huge" : "", | 20260508 11:29:32.035 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:32.036 | INFO | "malloc" : " 14 GB", | 20260508 11:29:32.036 | INFO | "pinned" : " 29 GB" | 20260508 11:29:32.036 | INFO | } | 20260508 11:29:32.036 | INFO | Perf report for the last 5.13849 seconds: | 20260508 11:29:32.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:32.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:29:32.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:29:32.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:32.036 | INFO | Uploaded 36GiB to GPU in 2.49362s (14.4369GiB/s) | 20260508 11:29:32.036 | INFO | Downloaded 34GiB to GPU in 2.4458s (13.9014GiB/s) | 20260508 11:29:37.175 | INFO | Analyzing chunk frames = [7755264,7755776) | 20260508 11:29:37.175 | INFO | MemInfo: { | 20260508 11:29:37.175 | INFO | "gpu" : " 17 GB", | 20260508 11:29:37.175 | INFO | "huge" : "", | 20260508 11:29:37.175 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:37.175 | INFO | "malloc" : " 14 GB", | 20260508 11:29:37.175 | INFO | "pinned" : " 29 GB" | 20260508 11:29:37.175 | INFO | } | 20260508 11:29:37.175 | INFO | Perf report for the last 5.13965 seconds: | 20260508 11:29:37.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:37.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.175 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:29:37.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.176 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:29:37.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:37.176 | INFO | Uploaded 36GiB to GPU in 2.47306s (14.5569GiB/s) | 20260508 11:29:37.176 | INFO | Downloaded 34GiB to GPU in 2.42865s (13.9996GiB/s) | 20260508 11:29:42.315 | INFO | Analyzing chunk frames = [7755776,7756288) | 20260508 11:29:42.315 | INFO | MemInfo: { | 20260508 11:29:42.315 | INFO | "gpu" : " 17 GB", | 20260508 11:29:42.315 | INFO | "huge" : "", | 20260508 11:29:42.315 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:42.315 | INFO | "malloc" : " 14 GB", | 20260508 11:29:42.315 | INFO | "pinned" : " 29 GB" | 20260508 11:29:42.315 | INFO | } | 20260508 11:29:42.315 | INFO | Perf report for the last 5.13945 seconds: | 20260508 11:29:42.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:42.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:29:42.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 11:29:42.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:42.315 | INFO | Uploaded 36GiB to GPU in 2.31389s (15.5582GiB/s) | 20260508 11:29:42.315 | INFO | Downloaded 34GiB to GPU in 2.50257s (13.5861GiB/s) | 20260508 11:29:47.462 | INFO | Analyzing chunk frames = [7756288,7756800) | 20260508 11:29:47.462 | INFO | MemInfo: { | 20260508 11:29:47.462 | INFO | "gpu" : " 17 GB", | 20260508 11:29:47.462 | INFO | "huge" : "", | 20260508 11:29:47.462 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:47.462 | INFO | "malloc" : " 14 GB", | 20260508 11:29:47.462 | INFO | "pinned" : " 29 GB" | 20260508 11:29:47.462 | INFO | } | 20260508 11:29:47.463 | INFO | Perf report for the last 5.14761 seconds: | 20260508 11:29:47.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:47.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:29:47.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:29:47.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:47.463 | INFO | Uploaded 36GiB to GPU in 2.38936s (15.0668GiB/s) | 20260508 11:29:47.463 | INFO | Downloaded 34GiB to GPU in 2.41712s (14.0663GiB/s) | 20260508 11:29:52.600 | INFO | Analyzing chunk frames = [7756800,7757312) | 20260508 11:29:52.600 | INFO | MemInfo: { | 20260508 11:29:52.600 | INFO | "gpu" : " 17 GB", | 20260508 11:29:52.600 | INFO | "huge" : "", | 20260508 11:29:52.600 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:52.601 | INFO | "malloc" : " 14 GB", | 20260508 11:29:52.601 | INFO | "pinned" : " 29 GB" | 20260508 11:29:52.601 | INFO | } | 20260508 11:29:52.601 | INFO | Perf report for the last 5.1381 seconds: | 20260508 11:29:52.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:52.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | ANALYSIS : 64.9%, 3.0, 100.9ms, 0 | 20260508 11:29:52.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:29:52.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:52.601 | INFO | Uploaded 36GiB to GPU in 2.37761s (15.1413GiB/s) | 20260508 11:29:52.601 | INFO | Downloaded 34GiB to GPU in 2.41386s (14.0853GiB/s) | 20260508 11:29:57.729 | INFO | Analyzing chunk frames = [7757312,7757824) | 20260508 11:29:57.729 | INFO | MemInfo: { | 20260508 11:29:57.729 | INFO | "gpu" : " 17 GB", | 20260508 11:29:57.729 | INFO | "huge" : "", | 20260508 11:29:57.729 | INFO | "huge_shared" : " 29 GB", | 20260508 11:29:57.730 | INFO | "malloc" : " 14 GB", | 20260508 11:29:57.730 | INFO | "pinned" : " 29 GB" | 20260508 11:29:57.730 | INFO | } | 20260508 11:29:57.730 | INFO | Perf report for the last 5.12895 seconds: | 20260508 11:29:57.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:29:57.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 11:29:57.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 11:29:57.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:29:57.730 | INFO | Uploaded 36GiB to GPU in 2.4779s (14.5284GiB/s) | 20260508 11:29:57.730 | INFO | Downloaded 34GiB to GPU in 2.45309s (13.8601GiB/s) | 20260508 11:29:57.730 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:29:57.730 | INFO | Total mem usage: 16.7 GiB | 20260508 11:29:57.730 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:29:57.730 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:29:57.730 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:29:57.730 | INFO | High Water Breakdown: | 20260508 11:29:57.730 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:29:57.730 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:29:57.730 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:29:57.730 | INFO | ------------------------------------------ | 20260508 11:29:57.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:29:57.730 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:29:57.730 | INFO | Total mem usage: 26.5 GiB | 20260508 11:29:57.730 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:29:57.730 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:29:57.730 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:29:57.730 | INFO | High Water Breakdown: | 20260508 11:29:57.730 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:29:57.730 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:29:57.731 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:29:57.731 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:29:57.731 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:29:57.731 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:29:57.731 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:29:57.731 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:29:57.731 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:29:57.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:29:57.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:29:57.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:29:57.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:29:57.731 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:29:57.731 | INFO | ------------------------------------------ | 20260508 11:29:57.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:29:57.731 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:29:57.731 | INFO | Total mem usage: 27.8 GiB | 20260508 11:29:57.731 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:29:57.731 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:29:57.731 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:29:57.731 | INFO | High Water Breakdown: | 20260508 11:29:57.731 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:29:57.731 | INFO | ------------------------------------------ | 20260508 11:29:57.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:29:57.731 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:29:57.731 | INFO | Total mem usage: 15.7 GiB | 20260508 11:29:57.731 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:29:57.731 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:29:57.731 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:29:57.731 | INFO | High Water Breakdown: | 20260508 11:29:57.731 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:29:57.731 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:29:57.731 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:29:57.731 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:29:57.731 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:29:57.731 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:29:57.731 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:29:57.731 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:29:57.731 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:29:57.731 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:29:57.731 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:29:57.731 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:29:57.731 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:29:57.731 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:29:57.731 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:29:57.731 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:29:57.731 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:29:57.731 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:29:57.731 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:29:57.731 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:29:57.731 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:29:57.731 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:29:57.731 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:29:57.731 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:29:57.731 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:29:57.731 | INFO | ------------------------------------------ | 20260508 11:29:57.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:29:57.731 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:29:57.731 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:29:57.731 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:29:57.731 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 5.41e-01 | 20260508 11:29:57.731 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.10e-02 / 8.42e-03 | 20260508 11:29:57.731 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.41e+00, 5.92e-01 / 2.21e-01 | 20260508 11:29:57.731 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 9.39e-02 | 20260508 11:29:57.731 | INFO | PerfReport: 16.02%, DME -- 4.18e+01 / 4.53e+01, 3.69e+00 / 9.88e-01 | 20260508 11:29:57.731 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.77e+00 / 1.09e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.07e-01 / 2.11e-01 | 20260508 11:29:57.731 | INFO | PerfReport: 2.99%, Metrics -- 9.74e-01 / 1.11e+00, 2.42e-01 / 8.60e-02 | 20260508 11:29:57.731 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.26e-01, 4.72e-02 / 1.99e-02 | 20260508 11:29:57.731 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:29:57.731 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:29:57.731 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:29:57.731 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 24.29%, Upload -- 2.55e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 24.69%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 2.14%, LockUpload -- 2.25e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 2.62%, LockDownload -- 2.76e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 15.19%, LockCompute -- 1.60e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:29:57.731 | INFO | PerfReport: 0.01%, Misc -- 1.19e-02 / 0.00e+00, 8.70e-03 / 0.00e+00 | 20260508 11:29:57.827 | INFO | Starting a round of Baz IO | 20260508 11:30:01.238 | INFO | Populating RTMetrics report | 20260508 11:30:01.239 | INFO | RTMetrics report populated | 20260508 11:30:01.239 | INFO | Opening tmp RTMetrics json file | 20260508 11:30:01.240 | INFO | Serializing RTMetrics to json | 20260508 11:30:01.241 | INFO | Serialization done, swapping tmp file now... | 20260508 11:30:01.241 | INFO | RTMetrics done | 20260508 11:30:01.300 | INFO | Finished a round of Baz IO: Wrote 1.431 GiB in 3.473 seconds | 20260508 11:30:01.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:30:01.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188677% / 0% / 4.76837e-05% | 20260508 11:30:01.300 | INFO | Baz IO overall: | 20260508 11:30:01.300 | INFO | BytesWritten : 3345786907693 | 20260508 11:30:01.300 | INFO | NumEvents : 1606846710955 | 20260508 11:30:02.874 | INFO | Analyzing chunk frames = [7757824,7758336) | 20260508 11:30:02.874 | INFO | MemInfo: { | 20260508 11:30:02.874 | INFO | "gpu" : " 17 GB", | 20260508 11:30:02.874 | INFO | "huge" : "", | 20260508 11:30:02.874 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:02.874 | INFO | "malloc" : " 14 GB", | 20260508 11:30:02.874 | INFO | "pinned" : " 29 GB" | 20260508 11:30:02.874 | INFO | } | 20260508 11:30:02.874 | INFO | Perf report for the last 5.14467 seconds: | 20260508 11:30:02.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:02.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:02.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:02.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:02.874 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 11:30:02.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:02.874 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 11:30:02.874 | INFO | PRE_HQ : 65.7%, 1.6, 54.2ms, 0 | 20260508 11:30:02.874 | INFO | BAZWRITER : 66.9%, 2.5, 85.2ms, 0 | 20260508 11:30:02.875 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 11:30:02.875 | INFO | Uploaded 36GiB to GPU in 2.49514s (14.4281GiB/s) | 20260508 11:30:02.875 | INFO | Downloaded 38GiB to GPU in 2.67954s (14.1816GiB/s) | 20260508 11:30:08.017 | INFO | Analyzing chunk frames = [7758336,7758848) | 20260508 11:30:08.017 | INFO | MemInfo: { | 20260508 11:30:08.017 | INFO | "gpu" : " 17 GB", | 20260508 11:30:08.017 | INFO | "huge" : "", | 20260508 11:30:08.017 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:08.017 | INFO | "malloc" : " 14 GB", | 20260508 11:30:08.017 | INFO | "pinned" : " 29 GB" | 20260508 11:30:08.017 | INFO | } | 20260508 11:30:08.017 | INFO | Perf report for the last 5.14299 seconds: | 20260508 11:30:08.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:08.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.017 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:30:08.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.017 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:30:08.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:08.018 | INFO | Uploaded 36GiB to GPU in 2.45771s (14.6478GiB/s) | 20260508 11:30:08.018 | INFO | Downloaded 34GiB to GPU in 2.46087s (13.8162GiB/s) | 20260508 11:30:13.155 | INFO | Analyzing chunk frames = [7758848,7759360) | 20260508 11:30:13.155 | INFO | MemInfo: { | 20260508 11:30:13.155 | INFO | "gpu" : " 17 GB", | 20260508 11:30:13.155 | INFO | "huge" : "", | 20260508 11:30:13.155 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:13.155 | INFO | "malloc" : " 14 GB", | 20260508 11:30:13.155 | INFO | "pinned" : " 29 GB" | 20260508 11:30:13.155 | INFO | } | 20260508 11:30:13.156 | INFO | Perf report for the last 5.13823 seconds: | 20260508 11:30:13.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:13.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.156 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:30:13.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.156 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 11:30:13.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:13.157 | INFO | Uploaded 36GiB to GPU in 2.51207s (14.3308GiB/s) | 20260508 11:30:13.157 | INFO | Downloaded 34GiB to GPU in 2.44514s (13.9052GiB/s) | 20260508 11:30:18.294 | INFO | Analyzing chunk frames = [7759360,7759872) | 20260508 11:30:18.294 | INFO | MemInfo: { | 20260508 11:30:18.294 | INFO | "gpu" : " 17 GB", | 20260508 11:30:18.294 | INFO | "huge" : "", | 20260508 11:30:18.294 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:18.294 | INFO | "malloc" : " 14 GB", | 20260508 11:30:18.294 | INFO | "pinned" : " 29 GB" | 20260508 11:30:18.294 | INFO | } | 20260508 11:30:18.294 | INFO | Perf report for the last 5.1385 seconds: | 20260508 11:30:18.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:18.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.294 | INFO | ANALYSIS : 66.5%, 3.0, 103.0ms, 0 | 20260508 11:30:18.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.294 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260508 11:30:18.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:18.295 | INFO | Uploaded 36GiB to GPU in 2.54914s (14.1224GiB/s) | 20260508 11:30:18.295 | INFO | Downloaded 34GiB to GPU in 2.4406s (13.931GiB/s) | 20260508 11:30:23.431 | INFO | Analyzing chunk frames = [7759872,7760384) | 20260508 11:30:23.431 | INFO | MemInfo: { | 20260508 11:30:23.431 | INFO | "gpu" : " 17 GB", | 20260508 11:30:23.431 | INFO | "huge" : "", | 20260508 11:30:23.431 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:23.432 | INFO | "malloc" : " 14 GB", | 20260508 11:30:23.432 | INFO | "pinned" : " 29 GB" | 20260508 11:30:23.432 | INFO | } | 20260508 11:30:23.432 | INFO | Perf report for the last 5.13756 seconds: | 20260508 11:30:23.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:23.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:30:23.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 11:30:23.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:23.432 | INFO | Uploaded 36GiB to GPU in 2.33231s (15.4354GiB/s) | 20260508 11:30:23.432 | INFO | Downloaded 34GiB to GPU in 2.49938s (13.6034GiB/s) | 20260508 11:30:28.575 | INFO | Analyzing chunk frames = [7760384,7760896) | 20260508 11:30:28.575 | INFO | MemInfo: { | 20260508 11:30:28.575 | INFO | "gpu" : " 17 GB", | 20260508 11:30:28.575 | INFO | "huge" : "", | 20260508 11:30:28.575 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:28.575 | INFO | "malloc" : " 14 GB", | 20260508 11:30:28.575 | INFO | "pinned" : " 29 GB" | 20260508 11:30:28.575 | INFO | } | 20260508 11:30:28.575 | INFO | Perf report for the last 5.14369 seconds: | 20260508 11:30:28.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:28.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.575 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:30:28.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.575 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:30:28.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:28.576 | INFO | Uploaded 36GiB to GPU in 2.43066s (14.8108GiB/s) | 20260508 11:30:28.576 | INFO | Downloaded 34GiB to GPU in 2.42936s (13.9954GiB/s) | 20260508 11:30:33.711 | INFO | Analyzing chunk frames = [7760896,7761408) | 20260508 11:30:33.711 | INFO | MemInfo: { | 20260508 11:30:33.711 | INFO | "gpu" : " 17 GB", | 20260508 11:30:33.711 | INFO | "huge" : "", | 20260508 11:30:33.711 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:33.711 | INFO | "malloc" : " 14 GB", | 20260508 11:30:33.711 | INFO | "pinned" : " 29 GB" | 20260508 11:30:33.711 | INFO | } | 20260508 11:30:33.711 | INFO | Perf report for the last 5.13556 seconds: | 20260508 11:30:33.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:33.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 11:30:33.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:30:33.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:33.711 | INFO | Uploaded 36GiB to GPU in 2.46879s (14.5821GiB/s) | 20260508 11:30:33.711 | INFO | Downloaded 34GiB to GPU in 2.44059s (13.931GiB/s) | 20260508 11:30:38.846 | INFO | Analyzing chunk frames = [7761408,7761920) | 20260508 11:30:38.846 | INFO | MemInfo: { | 20260508 11:30:38.846 | INFO | "gpu" : " 17 GB", | 20260508 11:30:38.846 | INFO | "huge" : "", | 20260508 11:30:38.846 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:38.846 | INFO | "malloc" : " 14 GB", | 20260508 11:30:38.846 | INFO | "pinned" : " 29 GB" | 20260508 11:30:38.846 | INFO | } | 20260508 11:30:38.846 | INFO | Perf report for the last 5.13545 seconds: | 20260508 11:30:38.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:38.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.846 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 11:30:38.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.846 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:30:38.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:38.847 | INFO | Uploaded 36GiB to GPU in 2.45592s (14.6585GiB/s) | 20260508 11:30:38.847 | INFO | Downloaded 34GiB to GPU in 2.43198s (13.9804GiB/s) | 20260508 11:30:38.944 | INFO | Starting a round of Baz IO | 20260508 11:30:42.363 | INFO | Populating RTMetrics report | 20260508 11:30:42.364 | INFO | RTMetrics report populated | 20260508 11:30:42.364 | INFO | Opening tmp RTMetrics json file | 20260508 11:30:42.365 | INFO | Serializing RTMetrics to json | 20260508 11:30:42.376 | INFO | Serialization done, swapping tmp file now... | 20260508 11:30:42.380 | INFO | RTMetrics done | 20260508 11:30:42.438 | INFO | Finished a round of Baz IO: Wrote 1.432 GiB in 3.493 seconds | 20260508 11:30:42.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:30:42.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188935% / 0% / 3.17891e-05% | 20260508 11:30:42.438 | INFO | Baz IO overall: | 20260508 11:30:42.438 | INFO | BytesWritten : 3347324350828 | 20260508 11:30:42.438 | INFO | NumEvents : 1607569855593 | 20260508 11:30:43.984 | INFO | Analyzing chunk frames = [7761920,7762432) | 20260508 11:30:43.984 | INFO | MemInfo: { | 20260508 11:30:43.984 | INFO | "gpu" : " 17 GB", | 20260508 11:30:43.984 | INFO | "huge" : "", | 20260508 11:30:43.984 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:43.984 | INFO | "malloc" : " 14 GB", | 20260508 11:30:43.984 | INFO | "pinned" : " 29 GB" | 20260508 11:30:43.984 | INFO | } | 20260508 11:30:43.984 | INFO | Perf report for the last 5.13739 seconds: | 20260508 11:30:43.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:43.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:43.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:43.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:43.984 | INFO | ANALYSIS : 68.2%, 3.0, 106.1ms, 0 | 20260508 11:30:43.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:43.984 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:30:43.984 | INFO | PRE_HQ : 65.9%, 1.6, 53.8ms, 0 | 20260508 11:30:43.984 | INFO | BAZWRITER : 67.2%, 2.4, 84.5ms, 0 | 20260508 11:30:43.984 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:30:43.984 | INFO | Uploaded 36GiB to GPU in 2.51093s (14.3373GiB/s) | 20260508 11:30:43.984 | INFO | Downloaded 38GiB to GPU in 2.72526s (13.9436GiB/s) | 20260508 11:30:49.131 | INFO | Analyzing chunk frames = [7762432,7762944) | 20260508 11:30:49.131 | INFO | MemInfo: { | 20260508 11:30:49.131 | INFO | "gpu" : " 17 GB", | 20260508 11:30:49.131 | INFO | "huge" : "", | 20260508 11:30:49.131 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:49.132 | INFO | "malloc" : " 14 GB", | 20260508 11:30:49.132 | INFO | "pinned" : " 29 GB" | 20260508 11:30:49.132 | INFO | } | 20260508 11:30:49.132 | INFO | Perf report for the last 5.14787 seconds: | 20260508 11:30:49.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:49.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 11:30:49.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:30:49.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:49.132 | INFO | Uploaded 36GiB to GPU in 2.50354s (14.3796GiB/s) | 20260508 11:30:49.132 | INFO | Downloaded 34GiB to GPU in 2.46026s (13.8197GiB/s) | 20260508 11:30:54.275 | INFO | Analyzing chunk frames = [7762944,7763456) | 20260508 11:30:54.275 | INFO | MemInfo: { | 20260508 11:30:54.275 | INFO | "gpu" : " 17 GB", | 20260508 11:30:54.275 | INFO | "huge" : "", | 20260508 11:30:54.275 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:54.275 | INFO | "malloc" : " 14 GB", | 20260508 11:30:54.275 | INFO | "pinned" : " 29 GB" | 20260508 11:30:54.275 | INFO | } | 20260508 11:30:54.275 | INFO | Perf report for the last 5.14339 seconds: | 20260508 11:30:54.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:54.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.275 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:30:54.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.275 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:30:54.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:54.276 | INFO | Uploaded 36GiB to GPU in 2.50959s (14.3449GiB/s) | 20260508 11:30:54.276 | INFO | Downloaded 34GiB to GPU in 2.43588s (13.958GiB/s) | 20260508 11:30:59.409 | INFO | Analyzing chunk frames = [7763456,7763968) | 20260508 11:30:59.409 | INFO | MemInfo: { | 20260508 11:30:59.409 | INFO | "gpu" : " 17 GB", | 20260508 11:30:59.409 | INFO | "huge" : "", | 20260508 11:30:59.409 | INFO | "huge_shared" : " 29 GB", | 20260508 11:30:59.409 | INFO | "malloc" : " 14 GB", | 20260508 11:30:59.409 | INFO | "pinned" : " 29 GB" | 20260508 11:30:59.409 | INFO | } | 20260508 11:30:59.409 | INFO | Perf report for the last 5.13409 seconds: | 20260508 11:30:59.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:30:59.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.409 | INFO | ANALYSIS : 65.7%, 3.0, 101.9ms, 0 | 20260508 11:30:59.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.409 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260508 11:30:59.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:30:59.410 | INFO | Uploaded 36GiB to GPU in 2.45304s (14.6757GiB/s) | 20260508 11:30:59.410 | INFO | Downloaded 34GiB to GPU in 2.48128s (13.7026GiB/s) | 20260508 11:31:04.545 | INFO | Analyzing chunk frames = [7763968,7764480) | 20260508 11:31:04.545 | INFO | MemInfo: { | 20260508 11:31:04.545 | INFO | "gpu" : " 17 GB", | 20260508 11:31:04.545 | INFO | "huge" : "", | 20260508 11:31:04.545 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:04.546 | INFO | "malloc" : " 14 GB", | 20260508 11:31:04.546 | INFO | "pinned" : " 29 GB" | 20260508 11:31:04.546 | INFO | } | 20260508 11:31:04.546 | INFO | Perf report for the last 5.13645 seconds: | 20260508 11:31:04.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:04.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:31:04.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 11:31:04.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:04.546 | INFO | Uploaded 36GiB to GPU in 2.37429s (15.1624GiB/s) | 20260508 11:31:04.546 | INFO | Downloaded 34GiB to GPU in 2.50938s (13.5492GiB/s) | 20260508 11:31:09.684 | INFO | Analyzing chunk frames = [7764480,7764992) | 20260508 11:31:09.684 | INFO | MemInfo: { | 20260508 11:31:09.684 | INFO | "gpu" : " 17 GB", | 20260508 11:31:09.684 | INFO | "huge" : "", | 20260508 11:31:09.684 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:09.684 | INFO | "malloc" : " 14 GB", | 20260508 11:31:09.684 | INFO | "pinned" : " 29 GB" | 20260508 11:31:09.684 | INFO | } | 20260508 11:31:09.684 | INFO | Perf report for the last 5.13821 seconds: | 20260508 11:31:09.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:09.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:31:09.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:31:09.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:09.684 | INFO | Uploaded 36GiB to GPU in 2.40868s (14.946GiB/s) | 20260508 11:31:09.684 | INFO | Downloaded 34GiB to GPU in 2.43794s (13.9462GiB/s) | 20260508 11:31:14.821 | INFO | Analyzing chunk frames = [7764992,7765504) | 20260508 11:31:14.821 | INFO | MemInfo: { | 20260508 11:31:14.821 | INFO | "gpu" : " 17 GB", | 20260508 11:31:14.821 | INFO | "huge" : "", | 20260508 11:31:14.821 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:14.821 | INFO | "malloc" : " 14 GB", | 20260508 11:31:14.821 | INFO | "pinned" : " 29 GB" | 20260508 11:31:14.821 | INFO | } | 20260508 11:31:14.822 | INFO | Perf report for the last 5.13767 seconds: | 20260508 11:31:14.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:14.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 11:31:14.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:31:14.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:14.822 | INFO | Uploaded 36GiB to GPU in 2.42787s (14.8278GiB/s) | 20260508 11:31:14.822 | INFO | Downloaded 34GiB to GPU in 2.43935s (13.9381GiB/s) | 20260508 11:31:19.969 | INFO | Analyzing chunk frames = [7765504,7766016) | 20260508 11:31:19.969 | INFO | MemInfo: { | 20260508 11:31:19.969 | INFO | "gpu" : " 17 GB", | 20260508 11:31:19.969 | INFO | "huge" : "", | 20260508 11:31:19.969 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:19.970 | INFO | "malloc" : " 14 GB", | 20260508 11:31:19.970 | INFO | "pinned" : " 29 GB" | 20260508 11:31:19.970 | INFO | } | 20260508 11:31:19.970 | INFO | Perf report for the last 5.14816 seconds: | 20260508 11:31:19.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:19.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | ANALYSIS : 65.3%, 3.0, 102.3ms, 0 | 20260508 11:31:19.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:31:19.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:19.970 | INFO | Uploaded 36GiB to GPU in 2.4112s (14.9304GiB/s) | 20260508 11:31:19.970 | INFO | Downloaded 34GiB to GPU in 2.42763s (14.0054GiB/s) | 20260508 11:31:19.970 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:31:19.970 | INFO | Total mem usage: 16.7 GiB | 20260508 11:31:19.970 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:31:19.970 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:31:19.970 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:31:19.970 | INFO | High Water Breakdown: | 20260508 11:31:19.970 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:31:19.970 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:31:19.970 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:31:19.970 | INFO | ------------------------------------------ | 20260508 11:31:19.970 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:31:19.970 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:31:19.970 | INFO | Total mem usage: 26.5 GiB | 20260508 11:31:19.970 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:31:19.970 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:31:19.970 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:31:19.970 | INFO | High Water Breakdown: | 20260508 11:31:19.971 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:31:19.971 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:31:19.971 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:31:19.971 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:31:19.971 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:31:19.971 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:31:19.971 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:31:19.971 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:31:19.971 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:31:19.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:31:19.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:31:19.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:31:19.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:31:19.971 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:31:19.971 | INFO | ------------------------------------------ | 20260508 11:31:19.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:31:19.971 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:31:19.971 | INFO | Total mem usage: 27.8 GiB | 20260508 11:31:19.971 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:31:19.971 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:31:19.971 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:31:19.971 | INFO | High Water Breakdown: | 20260508 11:31:19.971 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:31:19.971 | INFO | ------------------------------------------ | 20260508 11:31:19.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:31:19.971 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:31:19.971 | INFO | Total mem usage: 15.7 GiB | 20260508 11:31:19.971 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:31:19.971 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:31:19.971 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:31:19.971 | INFO | High Water Breakdown: | 20260508 11:31:19.971 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:31:19.971 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:31:19.971 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:31:19.971 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:31:19.971 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:31:19.971 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:31:19.971 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:31:19.971 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:31:19.971 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:31:19.971 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:31:19.971 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:31:19.971 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:31:19.971 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:31:19.971 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:31:19.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:31:19.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:31:19.971 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:31:19.971 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:31:19.971 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:31:19.971 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:31:19.971 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:31:19.971 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:31:19.971 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:31:19.971 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:31:19.971 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:31:19.971 | INFO | ------------------------------------------ | 20260508 11:31:19.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:31:19.971 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:31:19.971 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:31:19.971 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:31:19.971 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.42e+01 / 7.95e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.05e-01, 1.37e-02 / 3.95e-03 | 20260508 11:31:19.971 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.51e+00, 5.98e-01 / 1.92e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.50e+00, 2.12e-01 / 1.03e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.59e+01, 3.77e+00 / 9.59e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.86e+01 / 1.93e+01, 1.75e+00 / 8.31e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.35e+00 / 2.64e+00, 4.05e-01 / 1.00e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 3.00%, Metrics -- 9.80e-01 / 1.20e+00, 2.43e-01 / 1.15e-01 | 20260508 11:31:19.971 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.98e-01 / 3.35e-01, 4.78e-02 / 1.56e-02 | 20260508 11:31:19.971 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:31:19.971 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:31:19.971 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:31:19.971 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:31:19.971 | INFO | PerfReport: 24.33%, Upload -- 2.56e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 11:31:19.971 | INFO | PerfReport: 24.60%, Download -- 2.59e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 11:31:19.971 | INFO | PerfReport: 31.06%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 11:31:19.971 | INFO | PerfReport: 2.25%, LockUpload -- 2.37e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 11:31:19.972 | INFO | PerfReport: 2.53%, LockDownload -- 2.66e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 11:31:19.972 | INFO | PerfReport: 15.21%, LockCompute -- 1.60e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:31:19.972 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260508 11:31:20.067 | INFO | Starting a round of Baz IO | 20260508 11:31:23.469 | INFO | Populating RTMetrics report | 20260508 11:31:23.469 | INFO | RTMetrics report populated | 20260508 11:31:23.469 | INFO | Opening tmp RTMetrics json file | 20260508 11:31:23.479 | INFO | Serializing RTMetrics to json | 20260508 11:31:23.480 | INFO | Serialization done, swapping tmp file now... | 20260508 11:31:23.480 | INFO | RTMetrics done | 20260508 11:31:23.539 | INFO | Finished a round of Baz IO: Wrote 1.431 GiB in 3.472 seconds | 20260508 11:31:23.539 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:31:23.539 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187866% / 0% / 7.94729e-05% | 20260508 11:31:23.539 | INFO | Baz IO overall: | 20260508 11:31:23.539 | INFO | BytesWritten : 3348861009971 | 20260508 11:31:23.539 | INFO | NumEvents : 1608292334775 | 20260508 11:31:25.107 | INFO | Analyzing chunk frames = [7766016,7766528) | 20260508 11:31:25.107 | INFO | MemInfo: { | 20260508 11:31:25.107 | INFO | "gpu" : " 17 GB", | 20260508 11:31:25.107 | INFO | "huge" : "", | 20260508 11:31:25.107 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:25.107 | INFO | "malloc" : " 14 GB", | 20260508 11:31:25.107 | INFO | "pinned" : " 29 GB" | 20260508 11:31:25.107 | INFO | } | 20260508 11:31:25.107 | INFO | Perf report for the last 5.13709 seconds: | 20260508 11:31:25.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:25.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:25.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:25.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:25.107 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:31:25.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:25.107 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:31:25.107 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 11:31:25.107 | INFO | BAZWRITER : 66.9%, 2.5, 84.8ms, 0 | 20260508 11:31:25.107 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:31:25.107 | INFO | Uploaded 36GiB to GPU in 2.61262s (13.7793GiB/s) | 20260508 11:31:25.107 | INFO | Downloaded 38GiB to GPU in 2.71475s (13.9976GiB/s) | 20260508 11:31:30.245 | INFO | Analyzing chunk frames = [7766528,7767040) | 20260508 11:31:30.245 | INFO | MemInfo: { | 20260508 11:31:30.245 | INFO | "gpu" : " 17 GB", | 20260508 11:31:30.246 | INFO | "huge" : "", | 20260508 11:31:30.246 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:30.246 | INFO | "malloc" : " 14 GB", | 20260508 11:31:30.246 | INFO | "pinned" : " 29 GB" | 20260508 11:31:30.246 | INFO | } | 20260508 11:31:30.246 | INFO | Perf report for the last 5.13893 seconds: | 20260508 11:31:30.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:30.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:31:30.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:31:30.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:30.246 | INFO | Uploaded 36GiB to GPU in 2.46142s (14.6257GiB/s) | 20260508 11:31:30.246 | INFO | Downloaded 34GiB to GPU in 2.46029s (13.8195GiB/s) | 20260508 11:31:35.389 | INFO | Analyzing chunk frames = [7767040,7767552) | 20260508 11:31:35.389 | INFO | MemInfo: { | 20260508 11:31:35.389 | INFO | "gpu" : " 17 GB", | 20260508 11:31:35.389 | INFO | "huge" : "", | 20260508 11:31:35.389 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:35.390 | INFO | "malloc" : " 14 GB", | 20260508 11:31:35.390 | INFO | "pinned" : " 29 GB" | 20260508 11:31:35.390 | INFO | } | 20260508 11:31:35.390 | INFO | Perf report for the last 5.14395 seconds: | 20260508 11:31:35.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:35.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:31:35.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:31:35.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:35.390 | INFO | Uploaded 36GiB to GPU in 2.52123s (14.2788GiB/s) | 20260508 11:31:35.390 | INFO | Downloaded 34GiB to GPU in 2.45312s (13.8599GiB/s) | 20260508 11:31:40.522 | INFO | Analyzing chunk frames = [7767552,7768064) | 20260508 11:31:40.522 | INFO | MemInfo: { | 20260508 11:31:40.522 | INFO | "gpu" : " 17 GB", | 20260508 11:31:40.522 | INFO | "huge" : "", | 20260508 11:31:40.522 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:40.522 | INFO | "malloc" : " 14 GB", | 20260508 11:31:40.522 | INFO | "pinned" : " 29 GB" | 20260508 11:31:40.522 | INFO | } | 20260508 11:31:40.522 | INFO | Perf report for the last 5.13214 seconds: | 20260508 11:31:40.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:40.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.522 | INFO | ANALYSIS : 65.4%, 3.0, 101.3ms, 0 | 20260508 11:31:40.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.522 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 11:31:40.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:40.523 | INFO | Uploaded 36GiB to GPU in 2.38738s (15.0793GiB/s) | 20260508 11:31:40.523 | INFO | Downloaded 34GiB to GPU in 2.44482s (13.907GiB/s) | 20260508 11:31:45.660 | INFO | Analyzing chunk frames = [7768064,7768576) | 20260508 11:31:45.660 | INFO | MemInfo: { | 20260508 11:31:45.660 | INFO | "gpu" : " 17 GB", | 20260508 11:31:45.660 | INFO | "huge" : "", | 20260508 11:31:45.660 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:45.660 | INFO | "malloc" : " 14 GB", | 20260508 11:31:45.660 | INFO | "pinned" : " 29 GB" | 20260508 11:31:45.660 | INFO | } | 20260508 11:31:45.660 | INFO | Perf report for the last 5.13805 seconds: | 20260508 11:31:45.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:45.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:31:45.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:31:45.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:45.660 | INFO | Uploaded 36GiB to GPU in 2.28804s (15.734GiB/s) | 20260508 11:31:45.660 | INFO | Downloaded 34GiB to GPU in 2.48841s (13.6634GiB/s) | 20260508 11:31:50.800 | INFO | Analyzing chunk frames = [7768576,7769088) | 20260508 11:31:50.800 | INFO | MemInfo: { | 20260508 11:31:50.800 | INFO | "gpu" : " 17 GB", | 20260508 11:31:50.800 | INFO | "huge" : "", | 20260508 11:31:50.800 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:50.800 | INFO | "malloc" : " 14 GB", | 20260508 11:31:50.800 | INFO | "pinned" : " 29 GB" | 20260508 11:31:50.800 | INFO | } | 20260508 11:31:50.800 | INFO | Perf report for the last 5.14021 seconds: | 20260508 11:31:50.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:50.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:31:50.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:31:50.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:50.800 | INFO | Uploaded 36GiB to GPU in 2.40125s (14.9922GiB/s) | 20260508 11:31:50.801 | INFO | Downloaded 34GiB to GPU in 2.41821s (14.06GiB/s) | 20260508 11:31:55.937 | INFO | Analyzing chunk frames = [7769088,7769600) | 20260508 11:31:55.937 | INFO | MemInfo: { | 20260508 11:31:55.937 | INFO | "gpu" : " 17 GB", | 20260508 11:31:55.937 | INFO | "huge" : "", | 20260508 11:31:55.937 | INFO | "huge_shared" : " 29 GB", | 20260508 11:31:55.938 | INFO | "malloc" : " 14 GB", | 20260508 11:31:55.938 | INFO | "pinned" : " 29 GB" | 20260508 11:31:55.938 | INFO | } | 20260508 11:31:55.938 | INFO | Perf report for the last 5.13744 seconds: | 20260508 11:31:55.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:31:55.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:31:55.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:31:55.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:31:55.938 | INFO | Uploaded 36GiB to GPU in 2.46017s (14.6331GiB/s) | 20260508 11:31:55.938 | INFO | Downloaded 34GiB to GPU in 2.43814s (13.9451GiB/s) | 20260508 11:32:01.082 | INFO | Analyzing chunk frames = [7769600,7770112) | 20260508 11:32:01.082 | INFO | MemInfo: { | 20260508 11:32:01.082 | INFO | "gpu" : " 17 GB", | 20260508 11:32:01.082 | INFO | "huge" : "", | 20260508 11:32:01.082 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:01.082 | INFO | "malloc" : " 14 GB", | 20260508 11:32:01.082 | INFO | "pinned" : " 29 GB" | 20260508 11:32:01.082 | INFO | } | 20260508 11:32:01.082 | INFO | Perf report for the last 5.1446 seconds: | 20260508 11:32:01.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:01.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.082 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 11:32:01.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.082 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:32:01.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:01.083 | INFO | Uploaded 36GiB to GPU in 2.45787s (14.6468GiB/s) | 20260508 11:32:01.083 | INFO | Downloaded 34GiB to GPU in 2.42424s (14.025GiB/s) | 20260508 11:32:01.182 | INFO | Starting a round of Baz IO | 20260508 11:32:04.590 | INFO | Populating RTMetrics report | 20260508 11:32:04.591 | INFO | RTMetrics report populated | 20260508 11:32:04.591 | INFO | Opening tmp RTMetrics json file | 20260508 11:32:04.598 | INFO | Serializing RTMetrics to json | 20260508 11:32:04.600 | INFO | Serialization done, swapping tmp file now... | 20260508 11:32:04.600 | INFO | RTMetrics done | 20260508 11:32:04.660 | INFO | Finished a round of Baz IO: Wrote 1.433 GiB in 3.478 seconds | 20260508 11:32:04.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:32:04.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189261% / 0% / 6.75519e-05% | 20260508 11:32:04.661 | INFO | Baz IO overall: | 20260508 11:32:04.661 | INFO | BytesWritten : 3350399800537 | 20260508 11:32:04.661 | INFO | NumEvents : 1609016600706 | 20260508 11:32:06.221 | INFO | Analyzing chunk frames = [7770112,7770624) | 20260508 11:32:06.221 | INFO | MemInfo: { | 20260508 11:32:06.221 | INFO | "gpu" : " 17 GB", | 20260508 11:32:06.221 | INFO | "huge" : "", | 20260508 11:32:06.221 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:06.221 | INFO | "malloc" : " 14 GB", | 20260508 11:32:06.221 | INFO | "pinned" : " 29 GB" | 20260508 11:32:06.221 | INFO | } | 20260508 11:32:06.221 | INFO | Perf report for the last 5.13913 seconds: | 20260508 11:32:06.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:06.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:06.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:06.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:06.221 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 11:32:06.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:06.221 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:32:06.221 | INFO | PRE_HQ : 65.4%, 1.6, 54.2ms, 0 | 20260508 11:32:06.221 | INFO | BAZWRITER : 67.0%, 2.5, 85.1ms, 0 | 20260508 11:32:06.222 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:32:06.222 | INFO | Uploaded 36GiB to GPU in 2.6368s (13.6529GiB/s) | 20260508 11:32:06.222 | INFO | Downloaded 38GiB to GPU in 2.72241s (13.9582GiB/s) | 20260508 11:32:11.354 | INFO | Analyzing chunk frames = [7770624,7771136) | 20260508 11:32:11.354 | INFO | MemInfo: { | 20260508 11:32:11.354 | INFO | "gpu" : " 17 GB", | 20260508 11:32:11.354 | INFO | "huge" : "", | 20260508 11:32:11.354 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:11.354 | INFO | "malloc" : " 14 GB", | 20260508 11:32:11.354 | INFO | "pinned" : " 29 GB" | 20260508 11:32:11.354 | INFO | } | 20260508 11:32:11.355 | INFO | Perf report for the last 5.13331 seconds: | 20260508 11:32:11.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:11.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:32:11.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:32:11.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:11.355 | INFO | Uploaded 36GiB to GPU in 2.44314s (14.7351GiB/s) | 20260508 11:32:11.355 | INFO | Downloaded 34GiB to GPU in 2.46657s (13.7843GiB/s) | 20260508 11:32:16.490 | INFO | Analyzing chunk frames = [7771136,7771648) | 20260508 11:32:16.490 | INFO | MemInfo: { | 20260508 11:32:16.490 | INFO | "gpu" : " 17 GB", | 20260508 11:32:16.490 | INFO | "huge" : "", | 20260508 11:32:16.490 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:16.490 | INFO | "malloc" : " 14 GB", | 20260508 11:32:16.490 | INFO | "pinned" : " 29 GB" | 20260508 11:32:16.490 | INFO | } | 20260508 11:32:16.490 | INFO | Perf report for the last 5.13573 seconds: | 20260508 11:32:16.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:16.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.490 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:32:16.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.491 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:32:16.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:16.491 | INFO | Uploaded 36GiB to GPU in 2.47174s (14.5647GiB/s) | 20260508 11:32:16.491 | INFO | Downloaded 34GiB to GPU in 2.44795s (13.8891GiB/s) | 20260508 11:32:21.634 | INFO | Analyzing chunk frames = [7771648,7772160) | 20260508 11:32:21.634 | INFO | MemInfo: { | 20260508 11:32:21.634 | INFO | "gpu" : " 17 GB", | 20260508 11:32:21.634 | INFO | "huge" : "", | 20260508 11:32:21.634 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:21.634 | INFO | "malloc" : " 14 GB", | 20260508 11:32:21.634 | INFO | "pinned" : " 29 GB" | 20260508 11:32:21.634 | INFO | } | 20260508 11:32:21.634 | INFO | Perf report for the last 5.14378 seconds: | 20260508 11:32:21.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:21.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.634 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 11:32:21.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.634 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:32:21.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:21.635 | INFO | Uploaded 36GiB to GPU in 2.45622s (14.6567GiB/s) | 20260508 11:32:21.635 | INFO | Downloaded 34GiB to GPU in 2.44394s (13.912GiB/s) | 20260508 11:32:26.780 | INFO | Analyzing chunk frames = [7772160,7772672) | 20260508 11:32:26.780 | INFO | MemInfo: { | 20260508 11:32:26.780 | INFO | "gpu" : " 17 GB", | 20260508 11:32:26.780 | INFO | "huge" : "", | 20260508 11:32:26.780 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:26.780 | INFO | "malloc" : " 14 GB", | 20260508 11:32:26.780 | INFO | "pinned" : " 29 GB" | 20260508 11:32:26.780 | INFO | } | 20260508 11:32:26.781 | INFO | Perf report for the last 5.14629 seconds: | 20260508 11:32:26.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:26.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:32:26.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:32:26.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:26.781 | INFO | Uploaded 36GiB to GPU in 2.31625s (15.5423GiB/s) | 20260508 11:32:26.781 | INFO | Downloaded 34GiB to GPU in 2.5087s (13.5529GiB/s) | 20260508 11:32:31.922 | INFO | Analyzing chunk frames = [7772672,7773184) | 20260508 11:32:31.922 | INFO | MemInfo: { | 20260508 11:32:31.922 | INFO | "gpu" : " 17 GB", | 20260508 11:32:31.922 | INFO | "huge" : "", | 20260508 11:32:31.922 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:31.922 | INFO | "malloc" : " 14 GB", | 20260508 11:32:31.922 | INFO | "pinned" : " 29 GB" | 20260508 11:32:31.922 | INFO | } | 20260508 11:32:31.922 | INFO | Perf report for the last 5.14187 seconds: | 20260508 11:32:31.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:31.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:32:31.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:32:31.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:31.923 | INFO | Uploaded 36GiB to GPU in 2.37082s (15.1846GiB/s) | 20260508 11:32:31.923 | INFO | Downloaded 34GiB to GPU in 2.4297s (13.9935GiB/s) | 20260508 11:32:37.056 | INFO | Analyzing chunk frames = [7773184,7773696) | 20260508 11:32:37.056 | INFO | MemInfo: { | 20260508 11:32:37.056 | INFO | "gpu" : " 17 GB", | 20260508 11:32:37.056 | INFO | "huge" : "", | 20260508 11:32:37.056 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:37.056 | INFO | "malloc" : " 14 GB", | 20260508 11:32:37.056 | INFO | "pinned" : " 29 GB" | 20260508 11:32:37.056 | INFO | } | 20260508 11:32:37.056 | INFO | Perf report for the last 5.13368 seconds: | 20260508 11:32:37.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:37.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:32:37.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:32:37.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:37.056 | INFO | Uploaded 36GiB to GPU in 2.43878s (14.7615GiB/s) | 20260508 11:32:37.056 | INFO | Downloaded 34GiB to GPU in 2.45114s (13.8711GiB/s) | 20260508 11:32:42.192 | INFO | Analyzing chunk frames = [7773696,7774208) | 20260508 11:32:42.192 | INFO | MemInfo: { | 20260508 11:32:42.192 | INFO | "gpu" : " 17 GB", | 20260508 11:32:42.192 | INFO | "huge" : "", | 20260508 11:32:42.192 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:42.192 | INFO | "malloc" : " 14 GB", | 20260508 11:32:42.192 | INFO | "pinned" : " 29 GB" | 20260508 11:32:42.192 | INFO | } | 20260508 11:32:42.192 | INFO | Perf report for the last 5.13629 seconds: | 20260508 11:32:42.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:42.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.192 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 11:32:42.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.193 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:32:42.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:42.193 | INFO | Uploaded 36GiB to GPU in 2.45542s (14.6614GiB/s) | 20260508 11:32:42.193 | INFO | Downloaded 34GiB to GPU in 2.46101s (13.8155GiB/s) | 20260508 11:32:42.193 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:32:42.193 | INFO | Total mem usage: 16.7 GiB | 20260508 11:32:42.193 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:32:42.193 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:32:42.193 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:32:42.193 | INFO | High Water Breakdown: | 20260508 11:32:42.193 | INFO | 59.5% : 8.47 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:32:42.193 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:32:42.193 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:32:42.193 | INFO | ------------------------------------------ | 20260508 11:32:42.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:32:42.193 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:32:42.193 | INFO | Total mem usage: 26.5 GiB | 20260508 11:32:42.193 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:32:42.193 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:32:42.193 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:32:42.193 | INFO | High Water Breakdown: | 20260508 11:32:42.193 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:32:42.193 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:32:42.193 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:32:42.193 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:32:42.193 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:32:42.193 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:32:42.193 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:32:42.193 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:32:42.193 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:32:42.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:32:42.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:32:42.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:32:42.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:32:42.194 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:32:42.194 | INFO | ------------------------------------------ | 20260508 11:32:42.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:32:42.194 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:32:42.194 | INFO | Total mem usage: 27.8 GiB | 20260508 11:32:42.194 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:32:42.194 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:32:42.194 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:32:42.194 | INFO | High Water Breakdown: | 20260508 11:32:42.194 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:32:42.194 | INFO | ------------------------------------------ | 20260508 11:32:42.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:32:42.194 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:32:42.194 | INFO | Total mem usage: 15.7 GiB | 20260508 11:32:42.194 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:32:42.194 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:32:42.194 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:32:42.194 | INFO | High Water Breakdown: | 20260508 11:32:42.194 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:32:42.194 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:32:42.194 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:32:42.194 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:32:42.194 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:32:42.194 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:32:42.194 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:32:42.194 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:32:42.194 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:32:42.194 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:32:42.194 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:32:42.194 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:32:42.194 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:32:42.194 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:32:42.194 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:32:42.194 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:32:42.194 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:32:42.194 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:32:42.194 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:32:42.194 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:32:42.194 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:32:42.194 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:32:42.194 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:32:42.194 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:32:42.194 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:32:42.194 | INFO | ------------------------------------------ | 20260508 11:32:42.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:32:42.194 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:32:42.194 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:32:42.194 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:32:42.194 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.37e+01 / 4.57e-01 | 20260508 11:32:42.194 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.26e-02 / 6.41e-03 | 20260508 11:32:42.194 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.34e+00, 5.96e-01 / 2.30e-01 | 20260508 11:32:42.194 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 9.18e-02 | 20260508 11:32:42.194 | INFO | PerfReport: 16.05%, DME -- 4.18e+01 / 4.58e+01, 3.79e+00 / 9.29e-01 | 20260508 11:32:42.194 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 1.95e+01, 1.74e+00 / 9.24e-01 | 20260508 11:32:42.194 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.35e+00 / 2.54e+00, 4.07e-01 / 1.87e-01 | 20260508 11:32:42.194 | INFO | PerfReport: 3.00%, Metrics -- 9.78e-01 / 1.09e+00, 2.41e-01 / 7.57e-02 | 20260508 11:32:42.194 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.25e-01, 4.78e-02 / 2.13e-02 | 20260508 11:32:42.194 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:32:42.194 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:32:42.194 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:32:42.194 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 24.29%, Upload -- 2.55e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 24.66%, Download -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 31.04%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 2.62%, LockDownload -- 2.75e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 15.24%, LockCompute -- 1.60e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:32:42.194 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 11:32:42.290 | INFO | Starting a round of Baz IO | 20260508 11:32:45.707 | INFO | Populating RTMetrics report | 20260508 11:32:45.707 | INFO | RTMetrics report populated | 20260508 11:32:45.707 | INFO | Opening tmp RTMetrics json file | 20260508 11:32:45.708 | INFO | Serializing RTMetrics to json | 20260508 11:32:45.709 | INFO | Serialization done, swapping tmp file now... | 20260508 11:32:45.710 | INFO | RTMetrics done | 20260508 11:32:45.774 | INFO | Finished a round of Baz IO: Wrote 1.431 GiB in 3.484 seconds | 20260508 11:32:45.774 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:32:45.774 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188549% / 0% / 5.96046e-05% | 20260508 11:32:45.774 | INFO | Baz IO overall: | 20260508 11:32:45.774 | INFO | BytesWritten : 3351936227884 | 20260508 11:32:45.774 | INFO | NumEvents : 1609739046020 | 20260508 11:32:47.334 | INFO | Analyzing chunk frames = [7774208,7774720) | 20260508 11:32:47.334 | INFO | MemInfo: { | 20260508 11:32:47.334 | INFO | "gpu" : " 17 GB", | 20260508 11:32:47.334 | INFO | "huge" : "", | 20260508 11:32:47.334 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:47.334 | INFO | "malloc" : " 14 GB", | 20260508 11:32:47.334 | INFO | "pinned" : " 29 GB" | 20260508 11:32:47.334 | INFO | } | 20260508 11:32:47.334 | INFO | Perf report for the last 5.14144 seconds: | 20260508 11:32:47.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:47.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:47.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:47.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:47.334 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 11:32:47.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:47.334 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260508 11:32:47.334 | INFO | PRE_HQ : 66.4%, 1.7, 58.9ms, 0 | 20260508 11:32:47.334 | INFO | BAZWRITER : 67.2%, 2.6, 89.0ms, 0 | 20260508 11:32:47.334 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:32:47.335 | INFO | Uploaded 36GiB to GPU in 2.52688s (14.2468GiB/s) | 20260508 11:32:47.335 | INFO | Downloaded 38GiB to GPU in 2.70416s (14.0524GiB/s) | 20260508 11:32:52.470 | INFO | Analyzing chunk frames = [7774720,7775232) | 20260508 11:32:52.470 | INFO | MemInfo: { | 20260508 11:32:52.470 | INFO | "gpu" : " 17 GB", | 20260508 11:32:52.470 | INFO | "huge" : "", | 20260508 11:32:52.470 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:52.470 | INFO | "malloc" : " 14 GB", | 20260508 11:32:52.470 | INFO | "pinned" : " 29 GB" | 20260508 11:32:52.470 | INFO | } | 20260508 11:32:52.470 | INFO | Perf report for the last 5.13631 seconds: | 20260508 11:32:52.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:52.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.470 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:32:52.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.470 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:32:52.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:52.471 | INFO | Uploaded 36GiB to GPU in 2.44515s (14.723GiB/s) | 20260508 11:32:52.471 | INFO | Downloaded 34GiB to GPU in 2.46713s (13.7812GiB/s) | 20260508 11:32:57.612 | INFO | Analyzing chunk frames = [7775232,7775744) | 20260508 11:32:57.612 | INFO | MemInfo: { | 20260508 11:32:57.612 | INFO | "gpu" : " 17 GB", | 20260508 11:32:57.612 | INFO | "huge" : "", | 20260508 11:32:57.612 | INFO | "huge_shared" : " 29 GB", | 20260508 11:32:57.612 | INFO | "malloc" : " 14 GB", | 20260508 11:32:57.612 | INFO | "pinned" : " 29 GB" | 20260508 11:32:57.612 | INFO | } | 20260508 11:32:57.612 | INFO | Perf report for the last 5.14205 seconds: | 20260508 11:32:57.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:32:57.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.612 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:32:57.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.612 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:32:57.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:32:57.613 | INFO | Uploaded 36GiB to GPU in 2.46794s (14.5871GiB/s) | 20260508 11:32:57.613 | INFO | Downloaded 34GiB to GPU in 2.43772s (13.9475GiB/s) | 20260508 11:33:02.747 | INFO | Analyzing chunk frames = [7775744,7776256) | 20260508 11:33:02.747 | INFO | MemInfo: { | 20260508 11:33:02.747 | INFO | "gpu" : " 17 GB", | 20260508 11:33:02.747 | INFO | "huge" : "", | 20260508 11:33:02.747 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:02.747 | INFO | "malloc" : " 14 GB", | 20260508 11:33:02.747 | INFO | "pinned" : " 29 GB" | 20260508 11:33:02.747 | INFO | } | 20260508 11:33:02.747 | INFO | Perf report for the last 5.13479 seconds: | 20260508 11:33:02.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:02.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:33:02.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:33:02.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:02.747 | INFO | Uploaded 36GiB to GPU in 2.52095s (14.2803GiB/s) | 20260508 11:33:02.747 | INFO | Downloaded 34GiB to GPU in 2.47357s (13.7453GiB/s) | 20260508 11:33:07.890 | INFO | Analyzing chunk frames = [7776256,7776768) | 20260508 11:33:07.890 | INFO | MemInfo: { | 20260508 11:33:07.890 | INFO | "gpu" : " 17 GB", | 20260508 11:33:07.890 | INFO | "huge" : "", | 20260508 11:33:07.890 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:07.890 | INFO | "malloc" : " 14 GB", | 20260508 11:33:07.890 | INFO | "pinned" : " 29 GB" | 20260508 11:33:07.891 | INFO | } | 20260508 11:33:07.891 | INFO | Perf report for the last 5.14366 seconds: | 20260508 11:33:07.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:07.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | ANALYSIS : 66.3%, 3.0, 103.7ms, 0 | 20260508 11:33:07.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:33:07.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:07.891 | INFO | Uploaded 36GiB to GPU in 2.43115s (14.8078GiB/s) | 20260508 11:33:07.891 | INFO | Downloaded 34GiB to GPU in 2.55025s (13.332GiB/s) | 20260508 11:33:13.031 | INFO | Analyzing chunk frames = [7776768,7777280) | 20260508 11:33:13.031 | INFO | MemInfo: { | 20260508 11:33:13.031 | INFO | "gpu" : " 17 GB", | 20260508 11:33:13.031 | INFO | "huge" : "", | 20260508 11:33:13.031 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:13.031 | INFO | "malloc" : " 14 GB", | 20260508 11:33:13.031 | INFO | "pinned" : " 29 GB" | 20260508 11:33:13.031 | INFO | } | 20260508 11:33:13.031 | INFO | Perf report for the last 5.14071 seconds: | 20260508 11:33:13.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:13.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.032 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:33:13.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.032 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:33:13.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:13.032 | INFO | Uploaded 36GiB to GPU in 2.38756s (15.0782GiB/s) | 20260508 11:33:13.032 | INFO | Downloaded 34GiB to GPU in 2.43727s (13.9501GiB/s) | 20260508 11:33:18.169 | INFO | Analyzing chunk frames = [7777280,7777792) | 20260508 11:33:18.169 | INFO | MemInfo: { | 20260508 11:33:18.169 | INFO | "gpu" : " 17 GB", | 20260508 11:33:18.169 | INFO | "huge" : "", | 20260508 11:33:18.169 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:18.169 | INFO | "malloc" : " 14 GB", | 20260508 11:33:18.169 | INFO | "pinned" : " 29 GB" | 20260508 11:33:18.169 | INFO | } | 20260508 11:33:18.169 | INFO | Perf report for the last 5.13785 seconds: | 20260508 11:33:18.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:18.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.169 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 11:33:18.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.170 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:33:18.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:18.170 | INFO | Uploaded 36GiB to GPU in 2.43862s (14.7624GiB/s) | 20260508 11:33:18.170 | INFO | Downloaded 34GiB to GPU in 2.45957s (13.8236GiB/s) | 20260508 11:33:23.307 | INFO | Analyzing chunk frames = [7777792,7778304) | 20260508 11:33:23.307 | INFO | MemInfo: { | 20260508 11:33:23.307 | INFO | "gpu" : " 17 GB", | 20260508 11:33:23.307 | INFO | "huge" : "", | 20260508 11:33:23.307 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:23.308 | INFO | "malloc" : " 14 GB", | 20260508 11:33:23.308 | INFO | "pinned" : " 29 GB" | 20260508 11:33:23.308 | INFO | } | 20260508 11:33:23.308 | INFO | Perf report for the last 5.13851 seconds: | 20260508 11:33:23.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:23.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | ANALYSIS : 65.9%, 3.0, 103.1ms, 0 | 20260508 11:33:23.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 11:33:23.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:23.308 | INFO | Uploaded 36GiB to GPU in 2.49906s (14.4054GiB/s) | 20260508 11:33:23.308 | INFO | Downloaded 34GiB to GPU in 2.46741s (13.7797GiB/s) | 20260508 11:33:23.421 | INFO | Starting a round of Baz IO | 20260508 11:33:26.818 | INFO | Populating RTMetrics report | 20260508 11:33:26.818 | INFO | RTMetrics report populated | 20260508 11:33:26.818 | INFO | Opening tmp RTMetrics json file | 20260508 11:33:26.826 | INFO | Serializing RTMetrics to json | 20260508 11:33:26.827 | INFO | Serialization done, swapping tmp file now... | 20260508 11:33:26.828 | INFO | RTMetrics done | 20260508 11:33:26.882 | INFO | Finished a round of Baz IO: Wrote 1.431 GiB in 3.461 seconds | 20260508 11:33:26.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:33:26.882 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188084% / 3.97364e-06% / 5.16574e-05% | 20260508 11:33:26.882 | INFO | Baz IO overall: | 20260508 11:33:26.882 | INFO | BytesWritten : 3353473162791 | 20260508 11:33:26.882 | INFO | NumEvents : 1610461860567 | 20260508 11:33:28.445 | INFO | Analyzing chunk frames = [7778304,7778816) | 20260508 11:33:28.445 | INFO | MemInfo: { | 20260508 11:33:28.445 | INFO | "gpu" : " 17 GB", | 20260508 11:33:28.445 | INFO | "huge" : "", | 20260508 11:33:28.445 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:28.445 | INFO | "malloc" : " 14 GB", | 20260508 11:33:28.445 | INFO | "pinned" : " 29 GB" | 20260508 11:33:28.445 | INFO | } | 20260508 11:33:28.445 | INFO | Perf report for the last 5.13759 seconds: | 20260508 11:33:28.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:28.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:28.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:28.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:28.446 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260508 11:33:28.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:28.446 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260508 11:33:28.446 | INFO | PRE_HQ : 66.6%, 1.8, 60.7ms, 0 | 20260508 11:33:28.446 | INFO | BAZWRITER : 66.7%, 2.6, 90.7ms, 0 | 20260508 11:33:28.446 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 11:33:28.446 | INFO | Uploaded 36GiB to GPU in 2.60093s (13.8412GiB/s) | 20260508 11:33:28.446 | INFO | Downloaded 38GiB to GPU in 2.73176s (13.9104GiB/s) | 20260508 11:33:33.589 | INFO | Analyzing chunk frames = [7778816,7779328) | 20260508 11:33:33.589 | INFO | MemInfo: { | 20260508 11:33:33.589 | INFO | "gpu" : " 17 GB", | 20260508 11:33:33.589 | INFO | "huge" : "", | 20260508 11:33:33.589 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:33.589 | INFO | "malloc" : " 14 GB", | 20260508 11:33:33.590 | INFO | "pinned" : " 29 GB" | 20260508 11:33:33.590 | INFO | } | 20260508 11:33:33.590 | INFO | Perf report for the last 5.14418 seconds: | 20260508 11:33:33.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:33.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:33:33.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:33:33.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:33.590 | INFO | Uploaded 36GiB to GPU in 2.46171s (14.624GiB/s) | 20260508 11:33:33.590 | INFO | Downloaded 34GiB to GPU in 2.46949s (13.768GiB/s) | 20260508 11:33:38.727 | INFO | Analyzing chunk frames = [7779328,7779840) | 20260508 11:33:38.727 | INFO | MemInfo: { | 20260508 11:33:38.727 | INFO | "gpu" : " 17 GB", | 20260508 11:33:38.727 | INFO | "huge" : "", | 20260508 11:33:38.727 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:38.727 | INFO | "malloc" : " 14 GB", | 20260508 11:33:38.727 | INFO | "pinned" : " 29 GB" | 20260508 11:33:38.727 | INFO | } | 20260508 11:33:38.727 | INFO | Perf report for the last 5.13755 seconds: | 20260508 11:33:38.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:38.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:33:38.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:33:38.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:38.727 | INFO | Uploaded 36GiB to GPU in 2.46957s (14.5774GiB/s) | 20260508 11:33:38.727 | INFO | Downloaded 34GiB to GPU in 2.45427s (13.8534GiB/s) | 20260508 11:33:43.869 | INFO | Analyzing chunk frames = [7779840,7780352) | 20260508 11:33:43.869 | INFO | MemInfo: { | 20260508 11:33:43.869 | INFO | "gpu" : " 17 GB", | 20260508 11:33:43.869 | INFO | "huge" : "", | 20260508 11:33:43.869 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:43.869 | INFO | "malloc" : " 14 GB", | 20260508 11:33:43.869 | INFO | "pinned" : " 29 GB" | 20260508 11:33:43.869 | INFO | } | 20260508 11:33:43.869 | INFO | Perf report for the last 5.14227 seconds: | 20260508 11:33:43.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:43.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.870 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 11:33:43.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.870 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:33:43.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:43.870 | INFO | Uploaded 36GiB to GPU in 2.4584s (14.6437GiB/s) | 20260508 11:33:43.870 | INFO | Downloaded 34GiB to GPU in 2.46125s (13.8141GiB/s) | 20260508 11:33:49.005 | INFO | Analyzing chunk frames = [7780352,7780864) | 20260508 11:33:49.005 | INFO | MemInfo: { | 20260508 11:33:49.005 | INFO | "gpu" : " 17 GB", | 20260508 11:33:49.005 | INFO | "huge" : "", | 20260508 11:33:49.005 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:49.005 | INFO | "malloc" : " 14 GB", | 20260508 11:33:49.005 | INFO | "pinned" : " 29 GB" | 20260508 11:33:49.005 | INFO | } | 20260508 11:33:49.005 | INFO | Perf report for the last 5.13572 seconds: | 20260508 11:33:49.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:49.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.005 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:33:49.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.005 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 11:33:49.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:49.006 | INFO | Uploaded 36GiB to GPU in 2.28727s (15.7393GiB/s) | 20260508 11:33:49.006 | INFO | Downloaded 34GiB to GPU in 2.49447s (13.6302GiB/s) | 20260508 11:33:54.142 | INFO | Analyzing chunk frames = [7780864,7781376) | 20260508 11:33:54.142 | INFO | MemInfo: { | 20260508 11:33:54.142 | INFO | "gpu" : " 17 GB", | 20260508 11:33:54.142 | INFO | "huge" : "", | 20260508 11:33:54.142 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:54.142 | INFO | "malloc" : " 14 GB", | 20260508 11:33:54.142 | INFO | "pinned" : " 29 GB" | 20260508 11:33:54.142 | INFO | } | 20260508 11:33:54.142 | INFO | Perf report for the last 5.13673 seconds: | 20260508 11:33:54.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:54.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 11:33:54.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:33:54.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:54.142 | INFO | Uploaded 36GiB to GPU in 2.36757s (15.2055GiB/s) | 20260508 11:33:54.142 | INFO | Downloaded 34GiB to GPU in 2.45235s (13.8643GiB/s) | 20260508 11:33:59.291 | INFO | Analyzing chunk frames = [7781376,7781888) | 20260508 11:33:59.291 | INFO | MemInfo: { | 20260508 11:33:59.291 | INFO | "gpu" : " 17 GB", | 20260508 11:33:59.291 | INFO | "huge" : "", | 20260508 11:33:59.291 | INFO | "huge_shared" : " 29 GB", | 20260508 11:33:59.291 | INFO | "malloc" : " 14 GB", | 20260508 11:33:59.291 | INFO | "pinned" : " 29 GB" | 20260508 11:33:59.291 | INFO | } | 20260508 11:33:59.291 | INFO | Perf report for the last 5.14883 seconds: | 20260508 11:33:59.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:33:59.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.291 | INFO | ANALYSIS : 65.1%, 3.0, 101.7ms, 0 | 20260508 11:33:59.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.291 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:33:59.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:33:59.292 | INFO | Uploaded 36GiB to GPU in 2.39344s (15.0411GiB/s) | 20260508 11:33:59.292 | INFO | Downloaded 34GiB to GPU in 2.4598s (13.8223GiB/s) | 20260508 11:34:04.429 | INFO | Analyzing chunk frames = [7781888,7782400) | 20260508 11:34:04.429 | INFO | MemInfo: { | 20260508 11:34:04.429 | INFO | "gpu" : " 17 GB", | 20260508 11:34:04.429 | INFO | "huge" : "", | 20260508 11:34:04.429 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:04.429 | INFO | "malloc" : " 14 GB", | 20260508 11:34:04.429 | INFO | "pinned" : " 29 GB" | 20260508 11:34:04.429 | INFO | } | 20260508 11:34:04.429 | INFO | Perf report for the last 5.13862 seconds: | 20260508 11:34:04.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:04.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.429 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 11:34:04.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.430 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:34:04.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:04.430 | INFO | Uploaded 36GiB to GPU in 2.42524s (14.8439GiB/s) | 20260508 11:34:04.430 | INFO | Downloaded 34GiB to GPU in 2.4703s (13.7635GiB/s) | 20260508 11:34:04.430 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:34:04.430 | INFO | Total mem usage: 16.7 GiB | 20260508 11:34:04.430 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:34:04.430 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.7%) | 20260508 11:34:04.430 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:34:04.430 | INFO | High Water Breakdown: | 20260508 11:34:04.430 | INFO | 59.6% : 8.52 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:34:04.430 | INFO | 40.3% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:34:04.430 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:34:04.430 | INFO | ------------------------------------------ | 20260508 11:34:04.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:34:04.430 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:34:04.430 | INFO | Total mem usage: 26.5 GiB | 20260508 11:34:04.430 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:34:04.430 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:34:04.430 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:34:04.430 | INFO | High Water Breakdown: | 20260508 11:34:04.430 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:34:04.430 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:34:04.430 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:34:04.430 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:34:04.430 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:34:04.430 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:34:04.430 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:34:04.430 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:34:04.430 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:34:04.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:34:04.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:34:04.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:34:04.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:34:04.430 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:34:04.430 | INFO | ------------------------------------------ | 20260508 11:34:04.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:34:04.430 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:34:04.430 | INFO | Total mem usage: 27.8 GiB | 20260508 11:34:04.430 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:34:04.430 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:34:04.430 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:34:04.430 | INFO | High Water Breakdown: | 20260508 11:34:04.430 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:34:04.430 | INFO | ------------------------------------------ | 20260508 11:34:04.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:34:04.430 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:34:04.431 | INFO | Total mem usage: 15.7 GiB | 20260508 11:34:04.431 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:34:04.431 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:34:04.431 | INFO | Unused Cache: 858 MiB (Waste: 5.34%) | 20260508 11:34:04.431 | INFO | High Water Breakdown: | 20260508 11:34:04.431 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:34:04.431 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:34:04.431 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:34:04.431 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:34:04.431 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:34:04.431 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:34:04.431 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:34:04.431 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:34:04.431 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:34:04.431 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:34:04.431 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:34:04.431 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:34:04.431 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:34:04.431 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:34:04.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:34:04.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:34:04.431 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:34:04.431 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:34:04.431 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:34:04.431 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:34:04.431 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:34:04.431 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:34:04.431 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:34:04.431 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:34:04.431 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:34:04.431 | INFO | ------------------------------------------ | 20260508 11:34:04.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:34:04.431 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:34:04.431 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:34:04.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:34:04.431 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 5.50e-01 | 20260508 11:34:04.431 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.04e-01, 1.16e-02 / 2.45e-03 | 20260508 11:34:04.431 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.48e+00, 5.96e-01 / 1.96e-01 | 20260508 11:34:04.431 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 9.76e-02 | 20260508 11:34:04.431 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.54e+01, 3.71e+00 / 8.92e-01 | 20260508 11:34:04.431 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.83e+00 / 1.02e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.56e+00, 4.05e-01 / 1.73e-01 | 20260508 11:34:04.431 | INFO | PerfReport: 2.98%, Metrics -- 9.74e-01 / 1.16e+00, 2.42e-01 / 1.36e-01 | 20260508 11:34:04.431 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.29e-01, 4.79e-02 / 2.02e-02 | 20260508 11:34:04.431 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:34:04.431 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:34:04.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:34:04.431 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 24.27%, Upload -- 2.55e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 24.77%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 31.08%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 2.06%, LockUpload -- 2.16e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 2.66%, LockDownload -- 2.80e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 15.15%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 11:34:04.431 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 11:34:04.526 | INFO | Starting a round of Baz IO | 20260508 11:34:07.931 | INFO | Populating RTMetrics report | 20260508 11:34:07.931 | INFO | RTMetrics report populated | 20260508 11:34:07.931 | INFO | Opening tmp RTMetrics json file | 20260508 11:34:07.940 | INFO | Serializing RTMetrics to json | 20260508 11:34:07.941 | INFO | Serialization done, swapping tmp file now... | 20260508 11:34:07.941 | INFO | RTMetrics done | 20260508 11:34:07.997 | INFO | Finished a round of Baz IO: Wrote 1.43 GiB in 3.47 seconds | 20260508 11:34:07.997 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.00% / 59.90% | 20260508 11:34:07.997 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187449% / 0% / 4.37101e-05% | 20260508 11:34:07.997 | INFO | Baz IO overall: | 20260508 11:34:07.997 | INFO | BytesWritten : 3355008720905 | 20260508 11:34:07.997 | INFO | NumEvents : 1611184004387 | 20260508 11:34:09.563 | INFO | Analyzing chunk frames = [7782400,7782912) | 20260508 11:34:09.563 | INFO | MemInfo: { | 20260508 11:34:09.563 | INFO | "gpu" : " 17 GB", | 20260508 11:34:09.563 | INFO | "huge" : "", | 20260508 11:34:09.563 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:09.563 | INFO | "malloc" : " 14 GB", | 20260508 11:34:09.563 | INFO | "pinned" : " 29 GB" | 20260508 11:34:09.563 | INFO | } | 20260508 11:34:09.563 | INFO | Perf report for the last 5.13375 seconds: | 20260508 11:34:09.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:09.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:09.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:09.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:09.563 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 11:34:09.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:09.563 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:34:09.563 | INFO | PRE_HQ : 65.5%, 1.6, 54.0ms, 0 | 20260508 11:34:09.563 | INFO | BAZWRITER : 66.9%, 2.5, 84.5ms, 0 | 20260508 11:34:09.563 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:34:09.564 | INFO | Uploaded 36GiB to GPU in 2.53953s (14.1759GiB/s) | 20260508 11:34:09.564 | INFO | Downloaded 38GiB to GPU in 2.72179s (13.9614GiB/s) | 20260508 11:34:14.700 | INFO | Analyzing chunk frames = [7782912,7783424) | 20260508 11:34:14.700 | INFO | MemInfo: { | 20260508 11:34:14.700 | INFO | "gpu" : " 17 GB", | 20260508 11:34:14.700 | INFO | "huge" : "", | 20260508 11:34:14.700 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:14.700 | INFO | "malloc" : " 14 GB", | 20260508 11:34:14.700 | INFO | "pinned" : " 29 GB" | 20260508 11:34:14.700 | INFO | } | 20260508 11:34:14.700 | INFO | Perf report for the last 5.13676 seconds: | 20260508 11:34:14.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:14.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.700 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:34:14.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.700 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:34:14.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:14.701 | INFO | Uploaded 36GiB to GPU in 2.44407s (14.7295GiB/s) | 20260508 11:34:14.701 | INFO | Downloaded 34GiB to GPU in 2.4838s (13.6887GiB/s) | 20260508 11:34:19.839 | INFO | Analyzing chunk frames = [7783424,7783936) | 20260508 11:34:19.839 | INFO | MemInfo: { | 20260508 11:34:19.839 | INFO | "gpu" : " 17 GB", | 20260508 11:34:19.839 | INFO | "huge" : "", | 20260508 11:34:19.839 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:19.839 | INFO | "malloc" : " 14 GB", | 20260508 11:34:19.840 | INFO | "pinned" : " 29 GB" | 20260508 11:34:19.840 | INFO | } | 20260508 11:34:19.840 | INFO | Perf report for the last 5.13964 seconds: | 20260508 11:34:19.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:19.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 11:34:19.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:34:19.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:19.840 | INFO | Uploaded 36GiB to GPU in 2.45011s (14.6932GiB/s) | 20260508 11:34:19.840 | INFO | Downloaded 34GiB to GPU in 2.45784s (13.8333GiB/s) | 20260508 11:34:24.983 | INFO | Analyzing chunk frames = [7783936,7784448) | 20260508 11:34:24.983 | INFO | MemInfo: { | 20260508 11:34:24.983 | INFO | "gpu" : " 17 GB", | 20260508 11:34:24.983 | INFO | "huge" : "", | 20260508 11:34:24.983 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:24.983 | INFO | "malloc" : " 14 GB", | 20260508 11:34:24.983 | INFO | "pinned" : " 29 GB" | 20260508 11:34:24.983 | INFO | } | 20260508 11:34:24.983 | INFO | Perf report for the last 5.14359 seconds: | 20260508 11:34:24.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:24.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 11:34:24.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:34:24.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:24.983 | INFO | Uploaded 36GiB to GPU in 2.44212s (14.7413GiB/s) | 20260508 11:34:24.984 | INFO | Downloaded 34GiB to GPU in 2.46244s (13.8075GiB/s) | 20260508 11:34:30.120 | INFO | Analyzing chunk frames = [7784448,7784960) | 20260508 11:34:30.120 | INFO | MemInfo: { | 20260508 11:34:30.120 | INFO | "gpu" : " 17 GB", | 20260508 11:34:30.120 | INFO | "huge" : "", | 20260508 11:34:30.120 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:30.120 | INFO | "malloc" : " 14 GB", | 20260508 11:34:30.120 | INFO | "pinned" : " 29 GB" | 20260508 11:34:30.120 | INFO | } | 20260508 11:34:30.120 | INFO | Perf report for the last 5.1369 seconds: | 20260508 11:34:30.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:30.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 11:34:30.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:34:30.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:30.120 | INFO | Uploaded 36GiB to GPU in 2.37288s (15.1715GiB/s) | 20260508 11:34:30.120 | INFO | Downloaded 34GiB to GPU in 2.53718s (13.4007GiB/s) | 20260508 11:34:35.252 | INFO | Analyzing chunk frames = [7784960,7785472) | 20260508 11:34:35.252 | INFO | MemInfo: { | 20260508 11:34:35.252 | INFO | "gpu" : " 17 GB", | 20260508 11:34:35.253 | INFO | "huge" : "", | 20260508 11:34:35.253 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:35.253 | INFO | "malloc" : " 14 GB", | 20260508 11:34:35.253 | INFO | "pinned" : " 29 GB" | 20260508 11:34:35.253 | INFO | } | 20260508 11:34:35.253 | INFO | Perf report for the last 5.13257 seconds: | 20260508 11:34:35.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:35.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | ANALYSIS : 65.3%, 3.0, 101.4ms, 0 | 20260508 11:34:35.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260508 11:34:35.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:35.253 | INFO | Uploaded 36GiB to GPU in 2.4028s (14.9825GiB/s) | 20260508 11:34:35.253 | INFO | Downloaded 34GiB to GPU in 2.42757s (14.0058GiB/s) | 20260508 11:34:40.398 | INFO | Analyzing chunk frames = [7785472,7785984) | 20260508 11:34:40.398 | INFO | MemInfo: { | 20260508 11:34:40.398 | INFO | "gpu" : " 17 GB", | 20260508 11:34:40.398 | INFO | "huge" : "", | 20260508 11:34:40.398 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:40.398 | INFO | "malloc" : " 14 GB", | 20260508 11:34:40.398 | INFO | "pinned" : " 29 GB" | 20260508 11:34:40.398 | INFO | } | 20260508 11:34:40.398 | INFO | Perf report for the last 5.14517 seconds: | 20260508 11:34:40.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:40.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:34:40.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:34:40.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:40.398 | INFO | Uploaded 36GiB to GPU in 2.41824s (14.8868GiB/s) | 20260508 11:34:40.398 | INFO | Downloaded 34GiB to GPU in 2.44406s (13.9113GiB/s) | 20260508 11:34:45.545 | INFO | Analyzing chunk frames = [7785984,7786496) | 20260508 11:34:45.545 | INFO | MemInfo: { | 20260508 11:34:45.545 | INFO | "gpu" : " 17 GB", | 20260508 11:34:45.545 | INFO | "huge" : "", | 20260508 11:34:45.545 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:45.545 | INFO | "malloc" : " 14 GB", | 20260508 11:34:45.545 | INFO | "pinned" : " 29 GB" | 20260508 11:34:45.545 | INFO | } | 20260508 11:34:45.545 | INFO | Perf report for the last 5.14704 seconds: | 20260508 11:34:45.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:45.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 11:34:45.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:34:45.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:45.545 | INFO | Uploaded 36GiB to GPU in 2.39701s (15.0187GiB/s) | 20260508 11:34:45.546 | INFO | Downloaded 34GiB to GPU in 2.43937s (13.938GiB/s) | 20260508 11:34:45.642 | INFO | Starting a round of Baz IO | 20260508 11:34:49.047 | INFO | Populating RTMetrics report | 20260508 11:34:49.048 | INFO | RTMetrics report populated | 20260508 11:34:49.048 | INFO | Opening tmp RTMetrics json file | 20260508 11:34:49.055 | INFO | Serializing RTMetrics to json | 20260508 11:34:49.056 | INFO | Serialization done, swapping tmp file now... | 20260508 11:34:49.057 | INFO | RTMetrics done | 20260508 11:34:49.111 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.468 seconds | 20260508 11:34:49.111 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.90% | 20260508 11:34:49.111 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186741% / 0% / 7.15256e-05% | 20260508 11:34:49.111 | INFO | Baz IO overall: | 20260508 11:34:49.111 | INFO | BytesWritten : 3356543249711 | 20260508 11:34:49.111 | INFO | NumEvents : 1611905404395 | 20260508 11:34:50.680 | INFO | Analyzing chunk frames = [7786496,7787008) | 20260508 11:34:50.680 | INFO | MemInfo: { | 20260508 11:34:50.680 | INFO | "gpu" : " 17 GB", | 20260508 11:34:50.680 | INFO | "huge" : "", | 20260508 11:34:50.680 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:50.680 | INFO | "malloc" : " 14 GB", | 20260508 11:34:50.680 | INFO | "pinned" : " 29 GB" | 20260508 11:34:50.680 | INFO | } | 20260508 11:34:50.680 | INFO | Perf report for the last 5.13493 seconds: | 20260508 11:34:50.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:50.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:50.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:50.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:50.680 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 11:34:50.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:50.680 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:34:50.680 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 11:34:50.680 | INFO | BAZWRITER : 66.8%, 2.5, 84.5ms, 0 | 20260508 11:34:50.680 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:34:50.680 | INFO | Uploaded 36GiB to GPU in 2.61267s (13.779GiB/s) | 20260508 11:34:50.680 | INFO | Downloaded 38GiB to GPU in 2.7177s (13.9824GiB/s) | 20260508 11:34:55.814 | INFO | Analyzing chunk frames = [7787008,7787520) | 20260508 11:34:55.814 | INFO | MemInfo: { | 20260508 11:34:55.814 | INFO | "gpu" : " 17 GB", | 20260508 11:34:55.814 | INFO | "huge" : "", | 20260508 11:34:55.814 | INFO | "huge_shared" : " 29 GB", | 20260508 11:34:55.814 | INFO | "malloc" : " 14 GB", | 20260508 11:34:55.814 | INFO | "pinned" : " 29 GB" | 20260508 11:34:55.814 | INFO | } | 20260508 11:34:55.814 | INFO | Perf report for the last 5.13441 seconds: | 20260508 11:34:55.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:34:55.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.814 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:34:55.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.814 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:34:55.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:34:55.815 | INFO | Uploaded 36GiB to GPU in 2.457s (14.652GiB/s) | 20260508 11:34:55.815 | INFO | Downloaded 34GiB to GPU in 2.45601s (13.8436GiB/s) | 20260508 11:35:00.955 | INFO | Analyzing chunk frames = [7787520,7788032) | 20260508 11:35:00.955 | INFO | MemInfo: { | 20260508 11:35:00.955 | INFO | "gpu" : " 17 GB", | 20260508 11:35:00.955 | INFO | "huge" : "", | 20260508 11:35:00.955 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:00.955 | INFO | "malloc" : " 14 GB", | 20260508 11:35:00.955 | INFO | "pinned" : " 29 GB" | 20260508 11:35:00.955 | INFO | } | 20260508 11:35:00.955 | INFO | Perf report for the last 5.14067 seconds: | 20260508 11:35:00.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:00.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:35:00.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:35:00.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:00.955 | INFO | Uploaded 36GiB to GPU in 2.52356s (14.2656GiB/s) | 20260508 11:35:00.956 | INFO | Downloaded 34GiB to GPU in 2.45026s (13.8761GiB/s) | 20260508 11:35:06.096 | INFO | Analyzing chunk frames = [7788032,7788544) | 20260508 11:35:06.096 | INFO | MemInfo: { | 20260508 11:35:06.096 | INFO | "gpu" : " 17 GB", | 20260508 11:35:06.096 | INFO | "huge" : "", | 20260508 11:35:06.096 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:06.096 | INFO | "malloc" : " 14 GB", | 20260508 11:35:06.096 | INFO | "pinned" : " 29 GB" | 20260508 11:35:06.096 | INFO | } | 20260508 11:35:06.096 | INFO | Perf report for the last 5.14139 seconds: | 20260508 11:35:06.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:06.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.096 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:35:06.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.097 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:35:06.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:06.097 | INFO | Uploaded 36GiB to GPU in 2.50225s (14.387GiB/s) | 20260508 11:35:06.097 | INFO | Downloaded 34GiB to GPU in 2.45026s (13.8761GiB/s) | 20260508 11:35:11.238 | INFO | Analyzing chunk frames = [7788544,7789056) | 20260508 11:35:11.239 | INFO | MemInfo: { | 20260508 11:35:11.239 | INFO | "gpu" : " 17 GB", | 20260508 11:35:11.239 | INFO | "huge" : "", | 20260508 11:35:11.239 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:11.239 | INFO | "malloc" : " 14 GB", | 20260508 11:35:11.239 | INFO | "pinned" : " 29 GB" | 20260508 11:35:11.239 | INFO | } | 20260508 11:35:11.239 | INFO | Perf report for the last 5.14237 seconds: | 20260508 11:35:11.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:11.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 11:35:11.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 11:35:11.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:11.239 | INFO | Uploaded 36GiB to GPU in 2.45651s (14.6549GiB/s) | 20260508 11:35:11.239 | INFO | Downloaded 34GiB to GPU in 2.5305s (13.4361GiB/s) | 20260508 11:35:16.376 | INFO | Analyzing chunk frames = [7789056,7789568) | 20260508 11:35:16.376 | INFO | MemInfo: { | 20260508 11:35:16.376 | INFO | "gpu" : " 17 GB", | 20260508 11:35:16.376 | INFO | "huge" : "", | 20260508 11:35:16.376 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:16.376 | INFO | "malloc" : " 14 GB", | 20260508 11:35:16.376 | INFO | "pinned" : " 29 GB" | 20260508 11:35:16.376 | INFO | } | 20260508 11:35:16.376 | INFO | Perf report for the last 5.13725 seconds: | 20260508 11:35:16.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:16.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.376 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:35:16.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.376 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:35:16.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:16.377 | INFO | Uploaded 36GiB to GPU in 2.42473s (14.847GiB/s) | 20260508 11:35:16.377 | INFO | Downloaded 34GiB to GPU in 2.42922s (13.9963GiB/s) | 20260508 11:35:21.513 | INFO | Analyzing chunk frames = [7789568,7790080) | 20260508 11:35:21.513 | INFO | MemInfo: { | 20260508 11:35:21.513 | INFO | "gpu" : " 17 GB", | 20260508 11:35:21.513 | INFO | "huge" : "", | 20260508 11:35:21.513 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:21.513 | INFO | "malloc" : " 14 GB", | 20260508 11:35:21.513 | INFO | "pinned" : " 29 GB" | 20260508 11:35:21.513 | INFO | } | 20260508 11:35:21.513 | INFO | Perf report for the last 5.13682 seconds: | 20260508 11:35:21.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:21.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:35:21.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:35:21.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:21.513 | INFO | Uploaded 36GiB to GPU in 2.4491s (14.6993GiB/s) | 20260508 11:35:21.513 | INFO | Downloaded 34GiB to GPU in 2.4486s (13.8855GiB/s) | 20260508 11:35:26.653 | INFO | Analyzing chunk frames = [7790080,7790592) | 20260508 11:35:26.653 | INFO | MemInfo: { | 20260508 11:35:26.653 | INFO | "gpu" : " 17 GB", | 20260508 11:35:26.653 | INFO | "huge" : "", | 20260508 11:35:26.653 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:26.653 | INFO | "malloc" : " 14 GB", | 20260508 11:35:26.653 | INFO | "pinned" : " 29 GB" | 20260508 11:35:26.653 | INFO | } | 20260508 11:35:26.653 | INFO | Perf report for the last 5.14049 seconds: | 20260508 11:35:26.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:26.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.653 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 11:35:26.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.653 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:35:26.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:26.654 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:35:26.654 | INFO | Total mem usage: 16.7 GiB | 20260508 11:35:26.654 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:35:26.654 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:35:26.654 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:35:26.654 | INFO | High Water Breakdown: | 20260508 11:35:26.654 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:35:26.654 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:35:26.654 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:35:26.654 | INFO | Uploaded 36GiB to GPU in 2.42415s (14.8506GiB/s) | 20260508 11:35:26.654 | INFO | ------------------------------------------ | 20260508 11:35:26.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:35:26.654 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:35:26.654 | INFO | Total mem usage: 26.5 GiB | 20260508 11:35:26.654 | INFO | Downloaded 34GiB to GPU in 2.42623s (14.0135GiB/s) | 20260508 11:35:26.654 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:35:26.654 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:35:26.654 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:35:26.654 | INFO | High Water Breakdown: | 20260508 11:35:26.654 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:35:26.654 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:35:26.654 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:35:26.654 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:35:26.654 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:35:26.654 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:35:26.654 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:35:26.654 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:35:26.654 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:35:26.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:35:26.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:35:26.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:35:26.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:35:26.654 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:35:26.654 | INFO | ------------------------------------------ | 20260508 11:35:26.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:35:26.654 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:35:26.654 | INFO | Total mem usage: 27.8 GiB | 20260508 11:35:26.654 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:35:26.654 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:35:26.654 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:35:26.654 | INFO | High Water Breakdown: | 20260508 11:35:26.654 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:35:26.654 | INFO | ------------------------------------------ | 20260508 11:35:26.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:35:26.655 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:35:26.655 | INFO | Total mem usage: 15.7 GiB | 20260508 11:35:26.655 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:35:26.655 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:35:26.655 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:35:26.655 | INFO | High Water Breakdown: | 20260508 11:35:26.655 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:35:26.655 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:35:26.655 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:35:26.655 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:35:26.655 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:35:26.655 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:35:26.655 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:35:26.655 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:35:26.655 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:35:26.655 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:35:26.655 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:35:26.655 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:35:26.655 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:35:26.655 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:35:26.655 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:35:26.655 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:35:26.655 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:35:26.655 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:35:26.655 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:35:26.655 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:35:26.655 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:35:26.655 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:35:26.655 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:35:26.655 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:35:26.655 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:35:26.655 | INFO | ------------------------------------------ | 20260508 11:35:26.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:35:26.655 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:35:26.655 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:35:26.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:35:26.655 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 6.31e-01 | 20260508 11:35:26.655 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.12e-02 / 6.16e-03 | 20260508 11:35:26.655 | INFO | PerfReport: 12.20%, Baseline -- 3.97e+00 / 4.36e+00, 5.93e-01 / 2.81e-01 | 20260508 11:35:26.655 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.05e-01 | 20260508 11:35:26.655 | INFO | PerfReport: 16.01%, DME -- 4.17e+01 / 4.53e+01, 3.65e+00 / 1.27e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.73e+00 / 1.45e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.45e+00, 4.08e-01 / 3.09e-01 | 20260508 11:35:26.655 | INFO | PerfReport: 2.99%, Metrics -- 9.76e-01 / 1.08e+00, 2.44e-01 / 1.03e-01 | 20260508 11:35:26.655 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.95e-01 / 3.20e-01, 4.80e-02 / 2.89e-02 | 20260508 11:35:26.655 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:35:26.655 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:35:26.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:35:26.655 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 24.37%, Upload -- 2.56e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 24.72%, Download -- 2.60e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 31.03%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 2.12%, LockUpload -- 2.23e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 2.67%, LockDownload -- 2.80e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 15.07%, LockCompute -- 1.58e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:35:26.655 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 11:35:26.751 | INFO | Starting a round of Baz IO | 20260508 11:35:30.172 | INFO | Populating RTMetrics report | 20260508 11:35:30.172 | INFO | RTMetrics report populated | 20260508 11:35:30.172 | INFO | Opening tmp RTMetrics json file | 20260508 11:35:30.174 | INFO | Serializing RTMetrics to json | 20260508 11:35:30.175 | INFO | Serialization done, swapping tmp file now... | 20260508 11:35:30.175 | INFO | RTMetrics done | 20260508 11:35:30.235 | INFO | Finished a round of Baz IO: Wrote 1.43 GiB in 3.484 seconds | 20260508 11:35:30.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:35:30.235 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186753% / 0% / 7.94729e-05% | 20260508 11:35:30.235 | INFO | Baz IO overall: | 20260508 11:35:30.235 | INFO | BytesWritten : 3358078268378 | 20260508 11:35:30.235 | INFO | NumEvents : 1612627314677 | 20260508 11:35:31.793 | INFO | Analyzing chunk frames = [7790592,7791104) | 20260508 11:35:31.793 | INFO | MemInfo: { | 20260508 11:35:31.793 | INFO | "gpu" : " 17 GB", | 20260508 11:35:31.793 | INFO | "huge" : "", | 20260508 11:35:31.793 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:31.793 | INFO | "malloc" : " 14 GB", | 20260508 11:35:31.793 | INFO | "pinned" : " 29 GB" | 20260508 11:35:31.793 | INFO | } | 20260508 11:35:31.793 | INFO | Perf report for the last 5.1394 seconds: | 20260508 11:35:31.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:31.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:31.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:31.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:31.793 | INFO | ANALYSIS : 68.1%, 3.0, 105.9ms, 0 | 20260508 11:35:31.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:31.793 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:35:31.793 | INFO | PRE_HQ : 65.6%, 1.6, 54.2ms, 0 | 20260508 11:35:31.793 | INFO | BAZWRITER : 67.1%, 2.5, 84.9ms, 0 | 20260508 11:35:31.793 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:35:31.793 | INFO | Uploaded 36GiB to GPU in 2.61241s (13.7804GiB/s) | 20260508 11:35:31.793 | INFO | Downloaded 38GiB to GPU in 2.72466s (13.9467GiB/s) | 20260508 11:35:36.930 | INFO | Analyzing chunk frames = [7791104,7791616) | 20260508 11:35:36.930 | INFO | MemInfo: { | 20260508 11:35:36.930 | INFO | "gpu" : " 17 GB", | 20260508 11:35:36.930 | INFO | "huge" : "", | 20260508 11:35:36.930 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:36.930 | INFO | "malloc" : " 14 GB", | 20260508 11:35:36.930 | INFO | "pinned" : " 29 GB" | 20260508 11:35:36.930 | INFO | } | 20260508 11:35:36.930 | INFO | Perf report for the last 5.13709 seconds: | 20260508 11:35:36.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:36.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:35:36.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:35:36.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:36.930 | INFO | Uploaded 36GiB to GPU in 2.4563s (14.6562GiB/s) | 20260508 11:35:36.930 | INFO | Downloaded 34GiB to GPU in 2.46095s (13.8158GiB/s) | 20260508 11:35:42.068 | INFO | Analyzing chunk frames = [7791616,7792128) | 20260508 11:35:42.068 | INFO | MemInfo: { | 20260508 11:35:42.068 | INFO | "gpu" : " 17 GB", | 20260508 11:35:42.068 | INFO | "huge" : "", | 20260508 11:35:42.068 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:42.068 | INFO | "malloc" : " 14 GB", | 20260508 11:35:42.068 | INFO | "pinned" : " 29 GB" | 20260508 11:35:42.068 | INFO | } | 20260508 11:35:42.068 | INFO | Perf report for the last 5.13799 seconds: | 20260508 11:35:42.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:42.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 11:35:42.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:35:42.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:42.068 | INFO | Uploaded 36GiB to GPU in 2.49894s (14.4061GiB/s) | 20260508 11:35:42.069 | INFO | Downloaded 34GiB to GPU in 2.4362s (13.9562GiB/s) | 20260508 11:35:47.209 | INFO | Analyzing chunk frames = [7792128,7792640) | 20260508 11:35:47.209 | INFO | MemInfo: { | 20260508 11:35:47.209 | INFO | "gpu" : " 17 GB", | 20260508 11:35:47.209 | INFO | "huge" : "", | 20260508 11:35:47.209 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:47.209 | INFO | "malloc" : " 14 GB", | 20260508 11:35:47.209 | INFO | "pinned" : " 29 GB" | 20260508 11:35:47.209 | INFO | } | 20260508 11:35:47.210 | INFO | Perf report for the last 5.14166 seconds: | 20260508 11:35:47.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:47.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:35:47.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:35:47.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:47.210 | INFO | Uploaded 36GiB to GPU in 2.48666s (14.4773GiB/s) | 20260508 11:35:47.210 | INFO | Downloaded 34GiB to GPU in 2.44759s (13.8912GiB/s) | 20260508 11:35:52.349 | INFO | Analyzing chunk frames = [7792640,7793152) | 20260508 11:35:52.349 | INFO | MemInfo: { | 20260508 11:35:52.349 | INFO | "gpu" : " 17 GB", | 20260508 11:35:52.349 | INFO | "huge" : "", | 20260508 11:35:52.349 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:52.349 | INFO | "malloc" : " 14 GB", | 20260508 11:35:52.349 | INFO | "pinned" : " 29 GB" | 20260508 11:35:52.349 | INFO | } | 20260508 11:35:52.350 | INFO | Perf report for the last 5.14001 seconds: | 20260508 11:35:52.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:52.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 11:35:52.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:35:52.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:52.350 | INFO | Uploaded 36GiB to GPU in 2.45652s (14.6549GiB/s) | 20260508 11:35:52.350 | INFO | Downloaded 34GiB to GPU in 2.546s (13.3543GiB/s) | 20260508 11:35:57.491 | INFO | Analyzing chunk frames = [7793152,7793664) | 20260508 11:35:57.491 | INFO | MemInfo: { | 20260508 11:35:57.491 | INFO | "gpu" : " 17 GB", | 20260508 11:35:57.491 | INFO | "huge" : "", | 20260508 11:35:57.491 | INFO | "huge_shared" : " 29 GB", | 20260508 11:35:57.491 | INFO | "malloc" : " 14 GB", | 20260508 11:35:57.491 | INFO | "pinned" : " 29 GB" | 20260508 11:35:57.491 | INFO | } | 20260508 11:35:57.491 | INFO | Perf report for the last 5.14141 seconds: | 20260508 11:35:57.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:35:57.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 11:35:57.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:35:57.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:35:57.491 | INFO | Uploaded 36GiB to GPU in 2.37214s (15.1761GiB/s) | 20260508 11:35:57.492 | INFO | Downloaded 34GiB to GPU in 2.43511s (13.9624GiB/s) | 20260508 11:36:02.628 | INFO | Analyzing chunk frames = [7793664,7794176) | 20260508 11:36:02.628 | INFO | MemInfo: { | 20260508 11:36:02.628 | INFO | "gpu" : " 17 GB", | 20260508 11:36:02.628 | INFO | "huge" : "", | 20260508 11:36:02.628 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:02.628 | INFO | "malloc" : " 14 GB", | 20260508 11:36:02.629 | INFO | "pinned" : " 29 GB" | 20260508 11:36:02.629 | INFO | } | 20260508 11:36:02.629 | INFO | Perf report for the last 5.1377 seconds: | 20260508 11:36:02.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:02.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:36:02.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:36:02.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:02.629 | INFO | Uploaded 36GiB to GPU in 2.46338s (14.6141GiB/s) | 20260508 11:36:02.629 | INFO | Downloaded 34GiB to GPU in 2.44645s (13.8977GiB/s) | 20260508 11:36:07.761 | INFO | Analyzing chunk frames = [7794176,7794688) | 20260508 11:36:07.761 | INFO | MemInfo: { | 20260508 11:36:07.761 | INFO | "gpu" : " 17 GB", | 20260508 11:36:07.761 | INFO | "huge" : "", | 20260508 11:36:07.761 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:07.761 | INFO | "malloc" : " 14 GB", | 20260508 11:36:07.761 | INFO | "pinned" : " 29 GB" | 20260508 11:36:07.761 | INFO | } | 20260508 11:36:07.761 | INFO | Perf report for the last 5.13231 seconds: | 20260508 11:36:07.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:07.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.761 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 11:36:07.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.761 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 11:36:07.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:07.762 | INFO | Uploaded 36GiB to GPU in 2.43864s (14.7623GiB/s) | 20260508 11:36:07.762 | INFO | Downloaded 34GiB to GPU in 2.43876s (13.9415GiB/s) | 20260508 11:36:07.858 | INFO | Starting a round of Baz IO | 20260508 11:36:11.263 | INFO | Populating RTMetrics report | 20260508 11:36:11.263 | INFO | RTMetrics report populated | 20260508 11:36:11.263 | INFO | Opening tmp RTMetrics json file | 20260508 11:36:11.264 | INFO | Serializing RTMetrics to json | 20260508 11:36:11.265 | INFO | Serialization done, swapping tmp file now... | 20260508 11:36:11.266 | INFO | RTMetrics done | 20260508 11:36:11.324 | INFO | Finished a round of Baz IO: Wrote 1.43 GiB in 3.465 seconds | 20260508 11:36:11.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:36:11.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186137% / 0% / 9.53674e-05% | 20260508 11:36:11.324 | INFO | Baz IO overall: | 20260508 11:36:11.324 | INFO | BytesWritten : 3359613788273 | 20260508 11:36:11.324 | INFO | NumEvents : 1613349381829 | 20260508 11:36:12.908 | INFO | Analyzing chunk frames = [7794688,7795200) | 20260508 11:36:12.908 | INFO | MemInfo: { | 20260508 11:36:12.908 | INFO | "gpu" : " 17 GB", | 20260508 11:36:12.908 | INFO | "huge" : "", | 20260508 11:36:12.908 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:12.908 | INFO | "malloc" : " 14 GB", | 20260508 11:36:12.908 | INFO | "pinned" : " 29 GB" | 20260508 11:36:12.908 | INFO | } | 20260508 11:36:12.908 | INFO | Perf report for the last 5.14689 seconds: | 20260508 11:36:12.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:12.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:12.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:12.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:12.908 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 11:36:12.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:12.908 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 11:36:12.908 | INFO | PRE_HQ : 65.5%, 1.6, 54.1ms, 0 | 20260508 11:36:12.908 | INFO | BAZWRITER : 66.6%, 2.5, 84.7ms, 0 | 20260508 11:36:12.908 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 11:36:12.909 | INFO | Uploaded 36GiB to GPU in 2.51206s (14.3309GiB/s) | 20260508 11:36:12.909 | INFO | Downloaded 38GiB to GPU in 2.67752s (14.1922GiB/s) | 20260508 11:36:18.047 | INFO | Analyzing chunk frames = [7795200,7795712) | 20260508 11:36:18.047 | INFO | MemInfo: { | 20260508 11:36:18.047 | INFO | "gpu" : " 17 GB", | 20260508 11:36:18.047 | INFO | "huge" : "", | 20260508 11:36:18.047 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:18.047 | INFO | "malloc" : " 14 GB", | 20260508 11:36:18.047 | INFO | "pinned" : " 29 GB" | 20260508 11:36:18.047 | INFO | } | 20260508 11:36:18.047 | INFO | Perf report for the last 5.1393 seconds: | 20260508 11:36:18.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:18.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.047 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:36:18.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.047 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:36:18.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:18.048 | INFO | Uploaded 36GiB to GPU in 2.46623s (14.5972GiB/s) | 20260508 11:36:18.048 | INFO | Downloaded 34GiB to GPU in 2.46873s (13.7723GiB/s) | 20260508 11:36:23.189 | INFO | Analyzing chunk frames = [7795712,7796224) | 20260508 11:36:23.189 | INFO | MemInfo: { | 20260508 11:36:23.189 | INFO | "gpu" : " 17 GB", | 20260508 11:36:23.189 | INFO | "huge" : "", | 20260508 11:36:23.189 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:23.189 | INFO | "malloc" : " 14 GB", | 20260508 11:36:23.189 | INFO | "pinned" : " 29 GB" | 20260508 11:36:23.189 | INFO | } | 20260508 11:36:23.189 | INFO | Perf report for the last 5.14161 seconds: | 20260508 11:36:23.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:23.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:36:23.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:36:23.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:23.189 | INFO | Uploaded 36GiB to GPU in 2.49169s (14.448GiB/s) | 20260508 11:36:23.189 | INFO | Downloaded 34GiB to GPU in 2.44777s (13.8902GiB/s) | 20260508 11:36:28.325 | INFO | Analyzing chunk frames = [7796224,7796736) | 20260508 11:36:28.325 | INFO | MemInfo: { | 20260508 11:36:28.325 | INFO | "gpu" : " 17 GB", | 20260508 11:36:28.325 | INFO | "huge" : "", | 20260508 11:36:28.325 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:28.325 | INFO | "malloc" : " 14 GB", | 20260508 11:36:28.325 | INFO | "pinned" : " 29 GB" | 20260508 11:36:28.326 | INFO | } | 20260508 11:36:28.326 | INFO | Perf report for the last 5.13677 seconds: | 20260508 11:36:28.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:28.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:36:28.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:36:28.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:28.326 | INFO | Uploaded 36GiB to GPU in 2.47638s (14.5374GiB/s) | 20260508 11:36:28.326 | INFO | Downloaded 34GiB to GPU in 2.44439s (13.9094GiB/s) | 20260508 11:36:33.462 | INFO | Analyzing chunk frames = [7796736,7797248) | 20260508 11:36:33.462 | INFO | MemInfo: { | 20260508 11:36:33.462 | INFO | "gpu" : " 17 GB", | 20260508 11:36:33.462 | INFO | "huge" : "", | 20260508 11:36:33.462 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:33.462 | INFO | "malloc" : " 14 GB", | 20260508 11:36:33.462 | INFO | "pinned" : " 29 GB" | 20260508 11:36:33.462 | INFO | } | 20260508 11:36:33.462 | INFO | Perf report for the last 5.13613 seconds: | 20260508 11:36:33.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:33.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:36:33.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:36:33.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:33.462 | INFO | Uploaded 36GiB to GPU in 2.29715s (15.6716GiB/s) | 20260508 11:36:33.462 | INFO | Downloaded 34GiB to GPU in 2.48377s (13.6888GiB/s) | 20260508 11:36:38.604 | INFO | Analyzing chunk frames = [7797248,7797760) | 20260508 11:36:38.604 | INFO | MemInfo: { | 20260508 11:36:38.604 | INFO | "gpu" : " 17 GB", | 20260508 11:36:38.604 | INFO | "huge" : "", | 20260508 11:36:38.604 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:38.604 | INFO | "malloc" : " 14 GB", | 20260508 11:36:38.604 | INFO | "pinned" : " 29 GB" | 20260508 11:36:38.604 | INFO | } | 20260508 11:36:38.604 | INFO | Perf report for the last 5.14239 seconds: | 20260508 11:36:38.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:38.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.604 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:36:38.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.604 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:36:38.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:38.605 | INFO | Uploaded 36GiB to GPU in 2.38667s (15.0838GiB/s) | 20260508 11:36:38.605 | INFO | Downloaded 34GiB to GPU in 2.40904s (14.1135GiB/s) | 20260508 11:36:43.743 | INFO | Analyzing chunk frames = [7797760,7798272) | 20260508 11:36:43.743 | INFO | MemInfo: { | 20260508 11:36:43.743 | INFO | "gpu" : " 17 GB", | 20260508 11:36:43.743 | INFO | "huge" : "", | 20260508 11:36:43.743 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:43.743 | INFO | "malloc" : " 14 GB", | 20260508 11:36:43.743 | INFO | "pinned" : " 29 GB" | 20260508 11:36:43.743 | INFO | } | 20260508 11:36:43.743 | INFO | Perf report for the last 5.13855 seconds: | 20260508 11:36:43.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:43.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:36:43.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:36:43.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:43.743 | INFO | Uploaded 36GiB to GPU in 2.45212s (14.6811GiB/s) | 20260508 11:36:43.743 | INFO | Downloaded 34GiB to GPU in 2.43797s (13.9461GiB/s) | 20260508 11:36:48.883 | INFO | Analyzing chunk frames = [7798272,7798784) | 20260508 11:36:48.883 | INFO | MemInfo: { | 20260508 11:36:48.883 | INFO | "gpu" : " 17 GB", | 20260508 11:36:48.883 | INFO | "huge" : "", | 20260508 11:36:48.883 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:48.884 | INFO | "malloc" : " 14 GB", | 20260508 11:36:48.884 | INFO | "pinned" : " 29 GB" | 20260508 11:36:48.884 | INFO | } | 20260508 11:36:48.884 | INFO | Perf report for the last 5.14091 seconds: | 20260508 11:36:48.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:48.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:36:48.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:36:48.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:48.884 | INFO | Uploaded 36GiB to GPU in 2.45217s (14.6809GiB/s) | 20260508 11:36:48.884 | INFO | Downloaded 34GiB to GPU in 2.42291s (14.0327GiB/s) | 20260508 11:36:48.884 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:36:48.884 | INFO | Total mem usage: 16.7 GiB | 20260508 11:36:48.884 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:36:48.884 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:36:48.884 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:36:48.884 | INFO | High Water Breakdown: | 20260508 11:36:48.884 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:36:48.884 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:36:48.884 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:36:48.884 | INFO | ------------------------------------------ | 20260508 11:36:48.884 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:36:48.885 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:36:48.885 | INFO | Total mem usage: 26.5 GiB | 20260508 11:36:48.885 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:36:48.885 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:36:48.885 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:36:48.885 | INFO | High Water Breakdown: | 20260508 11:36:48.885 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:36:48.885 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:36:48.885 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:36:48.885 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:36:48.885 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:36:48.885 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:36:48.885 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:36:48.885 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:36:48.885 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:36:48.885 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:36:48.885 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:36:48.885 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:36:48.885 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:36:48.885 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:36:48.885 | INFO | ------------------------------------------ | 20260508 11:36:48.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:36:48.885 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:36:48.885 | INFO | Total mem usage: 27.8 GiB | 20260508 11:36:48.885 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:36:48.885 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:36:48.885 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:36:48.885 | INFO | High Water Breakdown: | 20260508 11:36:48.885 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:36:48.885 | INFO | ------------------------------------------ | 20260508 11:36:48.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:36:48.885 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:36:48.885 | INFO | Total mem usage: 15.7 GiB | 20260508 11:36:48.885 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:36:48.885 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:36:48.885 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:36:48.885 | INFO | High Water Breakdown: | 20260508 11:36:48.885 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:36:48.885 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:36:48.885 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:36:48.885 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:36:48.885 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:36:48.885 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:36:48.885 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:36:48.885 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:36:48.885 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:36:48.885 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:36:48.885 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:36:48.885 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:36:48.885 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:36:48.885 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:36:48.885 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:36:48.885 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:36:48.885 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:36:48.885 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:36:48.885 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:36:48.885 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:36:48.886 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:36:48.886 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:36:48.886 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:36:48.886 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:36:48.886 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:36:48.886 | INFO | ------------------------------------------ | 20260508 11:36:48.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:36:48.886 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:36:48.886 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:36:48.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:36:48.886 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.41e+01 / 5.34e-01 | 20260508 11:36:48.886 | INFO | PerfReport: 0.29%, AggregateStats -- 9.60e-02 / 1.03e-01, 1.06e-02 / 4.63e-03 | 20260508 11:36:48.886 | INFO | PerfReport: 12.21%, Baseline -- 3.98e+00 / 4.51e+00, 5.97e-01 / 1.89e-01 | 20260508 11:36:48.886 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.49e+00, 2.12e-01 / 1.22e-01 | 20260508 11:36:48.886 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.52e+01, 3.74e+00 / 7.36e-01 | 20260508 11:36:48.886 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.81e+00 / 1.06e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.59e+00, 4.06e-01 / 2.12e-01 | 20260508 11:36:48.886 | INFO | PerfReport: 2.99%, Metrics -- 9.76e-01 / 1.16e+00, 2.43e-01 / 1.34e-01 | 20260508 11:36:48.886 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.36e-01, 4.76e-02 / 2.64e-02 | 20260508 11:36:48.886 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:36:48.886 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:36:48.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:36:48.886 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 24.33%, Upload -- 2.56e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 31.03%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 2.20%, LockUpload -- 2.31e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 2.61%, LockDownload -- 2.75e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 15.24%, LockCompute -- 1.60e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260508 11:36:48.886 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260508 11:36:48.981 | INFO | Starting a round of Baz IO | 20260508 11:36:52.386 | INFO | Populating RTMetrics report | 20260508 11:36:52.386 | INFO | RTMetrics report populated | 20260508 11:36:52.386 | INFO | Opening tmp RTMetrics json file | 20260508 11:36:52.397 | INFO | Serializing RTMetrics to json | 20260508 11:36:52.398 | INFO | Serialization done, swapping tmp file now... | 20260508 11:36:52.398 | INFO | RTMetrics done | 20260508 11:36:52.458 | INFO | Finished a round of Baz IO: Wrote 1.43 GiB in 3.477 seconds | 20260508 11:36:52.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:36:52.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185815% / 0% / 7.94729e-05% | 20260508 11:36:52.458 | INFO | Baz IO overall: | 20260508 11:36:52.458 | INFO | BytesWritten : 3361148953226 | 20260508 11:36:52.458 | INFO | NumEvents : 1614071436019 | 20260508 11:36:54.018 | INFO | Analyzing chunk frames = [7798784,7799296) | 20260508 11:36:54.018 | INFO | MemInfo: { | 20260508 11:36:54.018 | INFO | "gpu" : " 17 GB", | 20260508 11:36:54.018 | INFO | "huge" : "", | 20260508 11:36:54.018 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:54.018 | INFO | "malloc" : " 14 GB", | 20260508 11:36:54.018 | INFO | "pinned" : " 29 GB" | 20260508 11:36:54.018 | INFO | } | 20260508 11:36:54.018 | INFO | Perf report for the last 5.13444 seconds: | 20260508 11:36:54.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:54.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:54.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:54.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:54.018 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 11:36:54.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:54.018 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:36:54.018 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 11:36:54.018 | INFO | BAZWRITER : 67.0%, 2.5, 86.1ms, 0 | 20260508 11:36:54.018 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:36:54.019 | INFO | Uploaded 36GiB to GPU in 2.64301s (13.6208GiB/s) | 20260508 11:36:54.019 | INFO | Downloaded 38GiB to GPU in 2.71528s (13.9949GiB/s) | 20260508 11:36:59.164 | INFO | Analyzing chunk frames = [7799296,7799808) | 20260508 11:36:59.164 | INFO | MemInfo: { | 20260508 11:36:59.164 | INFO | "gpu" : " 17 GB", | 20260508 11:36:59.164 | INFO | "huge" : "", | 20260508 11:36:59.164 | INFO | "huge_shared" : " 29 GB", | 20260508 11:36:59.164 | INFO | "malloc" : " 14 GB", | 20260508 11:36:59.164 | INFO | "pinned" : " 29 GB" | 20260508 11:36:59.164 | INFO | } | 20260508 11:36:59.164 | INFO | Perf report for the last 5.14585 seconds: | 20260508 11:36:59.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:36:59.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 11:36:59.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 11:36:59.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:36:59.164 | INFO | Uploaded 36GiB to GPU in 2.51254s (14.3281GiB/s) | 20260508 11:36:59.164 | INFO | Downloaded 34GiB to GPU in 2.46763s (13.7784GiB/s) | 20260508 11:37:04.300 | INFO | Analyzing chunk frames = [7799808,7800320) | 20260508 11:37:04.300 | INFO | MemInfo: { | 20260508 11:37:04.300 | INFO | "gpu" : " 17 GB", | 20260508 11:37:04.300 | INFO | "huge" : "", | 20260508 11:37:04.300 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:04.300 | INFO | "malloc" : " 14 GB", | 20260508 11:37:04.300 | INFO | "pinned" : " 29 GB" | 20260508 11:37:04.301 | INFO | } | 20260508 11:37:04.301 | INFO | Perf report for the last 5.13654 seconds: | 20260508 11:37:04.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:04.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.301 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:37:04.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.301 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:37:04.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:04.302 | INFO | Uploaded 36GiB to GPU in 2.51295s (14.3258GiB/s) | 20260508 11:37:04.302 | INFO | Downloaded 34GiB to GPU in 2.43828s (13.9443GiB/s) | 20260508 11:37:09.437 | INFO | Analyzing chunk frames = [7800320,7800832) | 20260508 11:37:09.437 | INFO | MemInfo: { | 20260508 11:37:09.437 | INFO | "gpu" : " 17 GB", | 20260508 11:37:09.437 | INFO | "huge" : "", | 20260508 11:37:09.437 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:09.437 | INFO | "malloc" : " 14 GB", | 20260508 11:37:09.437 | INFO | "pinned" : " 29 GB" | 20260508 11:37:09.437 | INFO | } | 20260508 11:37:09.437 | INFO | Perf report for the last 5.13683 seconds: | 20260508 11:37:09.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:09.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.437 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:37:09.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.438 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:37:09.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:09.438 | INFO | Uploaded 36GiB to GPU in 2.49613s (14.4223GiB/s) | 20260508 11:37:09.438 | INFO | Downloaded 34GiB to GPU in 2.45058s (13.8743GiB/s) | 20260508 11:37:14.578 | INFO | Analyzing chunk frames = [7800832,7801344) | 20260508 11:37:14.578 | INFO | MemInfo: { | 20260508 11:37:14.578 | INFO | "gpu" : " 17 GB", | 20260508 11:37:14.578 | INFO | "huge" : "", | 20260508 11:37:14.578 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:14.578 | INFO | "malloc" : " 14 GB", | 20260508 11:37:14.578 | INFO | "pinned" : " 29 GB" | 20260508 11:37:14.578 | INFO | } | 20260508 11:37:14.578 | INFO | Perf report for the last 5.1409 seconds: | 20260508 11:37:14.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:14.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.578 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:37:14.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.578 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:37:14.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:14.579 | INFO | Uploaded 36GiB to GPU in 2.3107s (15.5797GiB/s) | 20260508 11:37:14.579 | INFO | Downloaded 34GiB to GPU in 2.49494s (13.6276GiB/s) | 20260508 11:37:19.721 | INFO | Analyzing chunk frames = [7801344,7801856) | 20260508 11:37:19.721 | INFO | MemInfo: { | 20260508 11:37:19.721 | INFO | "gpu" : " 17 GB", | 20260508 11:37:19.721 | INFO | "huge" : "", | 20260508 11:37:19.721 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:19.721 | INFO | "malloc" : " 14 GB", | 20260508 11:37:19.721 | INFO | "pinned" : " 29 GB" | 20260508 11:37:19.721 | INFO | } | 20260508 11:37:19.721 | INFO | Perf report for the last 5.14261 seconds: | 20260508 11:37:19.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:19.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:37:19.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:37:19.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:19.721 | INFO | Uploaded 36GiB to GPU in 2.38101s (15.1196GiB/s) | 20260508 11:37:19.721 | INFO | Downloaded 34GiB to GPU in 2.42937s (13.9954GiB/s) | 20260508 11:37:24.862 | INFO | Analyzing chunk frames = [7801856,7802368) | 20260508 11:37:24.862 | INFO | MemInfo: { | 20260508 11:37:24.862 | INFO | "gpu" : " 17 GB", | 20260508 11:37:24.862 | INFO | "huge" : "", | 20260508 11:37:24.862 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:24.863 | INFO | "malloc" : " 14 GB", | 20260508 11:37:24.863 | INFO | "pinned" : " 29 GB" | 20260508 11:37:24.863 | INFO | } | 20260508 11:37:24.863 | INFO | Perf report for the last 5.1419 seconds: | 20260508 11:37:24.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:24.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:37:24.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 11:37:24.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:24.863 | INFO | Uploaded 36GiB to GPU in 2.46354s (14.6131GiB/s) | 20260508 11:37:24.863 | INFO | Downloaded 34GiB to GPU in 2.45542s (13.8469GiB/s) | 20260508 11:37:30.003 | INFO | Analyzing chunk frames = [7802368,7802880) | 20260508 11:37:30.003 | INFO | MemInfo: { | 20260508 11:37:30.003 | INFO | "gpu" : " 17 GB", | 20260508 11:37:30.003 | INFO | "huge" : "", | 20260508 11:37:30.003 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:30.003 | INFO | "malloc" : " 14 GB", | 20260508 11:37:30.003 | INFO | "pinned" : " 29 GB" | 20260508 11:37:30.003 | INFO | } | 20260508 11:37:30.003 | INFO | Perf report for the last 5.14056 seconds: | 20260508 11:37:30.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:30.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.003 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:37:30.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.004 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:37:30.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:30.004 | INFO | Uploaded 36GiB to GPU in 2.45798s (14.6462GiB/s) | 20260508 11:37:30.004 | INFO | Downloaded 34GiB to GPU in 2.45856s (13.8292GiB/s) | 20260508 11:37:30.103 | INFO | Starting a round of Baz IO | 20260508 11:37:33.513 | INFO | Populating RTMetrics report | 20260508 11:37:33.513 | INFO | RTMetrics report populated | 20260508 11:37:33.513 | INFO | Opening tmp RTMetrics json file | 20260508 11:37:33.523 | INFO | Serializing RTMetrics to json | 20260508 11:37:33.524 | INFO | Serialization done, swapping tmp file now... | 20260508 11:37:33.524 | INFO | RTMetrics done | 20260508 11:37:33.578 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.474 seconds | 20260508 11:37:33.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:37:33.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186066% / 0% / 6.75519e-05% | 20260508 11:37:33.578 | INFO | Baz IO overall: | 20260508 11:37:33.578 | INFO | BytesWritten : 3362683614566 | 20260508 11:37:33.578 | INFO | NumEvents : 1614793087249 | 20260508 11:37:35.135 | INFO | Analyzing chunk frames = [7802880,7803392) | 20260508 11:37:35.135 | INFO | MemInfo: { | 20260508 11:37:35.135 | INFO | "gpu" : " 17 GB", | 20260508 11:37:35.135 | INFO | "huge" : "", | 20260508 11:37:35.135 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:35.136 | INFO | "malloc" : " 14 GB", | 20260508 11:37:35.136 | INFO | "pinned" : " 29 GB" | 20260508 11:37:35.136 | INFO | } | 20260508 11:37:35.136 | INFO | Perf report for the last 5.13219 seconds: | 20260508 11:37:35.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:35.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:35.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:35.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:35.136 | INFO | ANALYSIS : 67.8%, 3.0, 105.5ms, 0 | 20260508 11:37:35.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:35.136 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:37:35.136 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 11:37:35.136 | INFO | BAZWRITER : 67.1%, 2.5, 86.2ms, 0 | 20260508 11:37:35.136 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:37:35.136 | INFO | Uploaded 36GiB to GPU in 2.56163s (14.0535GiB/s) | 20260508 11:37:35.136 | INFO | Downloaded 38GiB to GPU in 2.71594s (13.9915GiB/s) | 20260508 11:37:40.273 | INFO | Analyzing chunk frames = [7803392,7803904) | 20260508 11:37:40.273 | INFO | MemInfo: { | 20260508 11:37:40.273 | INFO | "gpu" : " 17 GB", | 20260508 11:37:40.273 | INFO | "huge" : "", | 20260508 11:37:40.273 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:40.273 | INFO | "malloc" : " 14 GB", | 20260508 11:37:40.273 | INFO | "pinned" : " 29 GB" | 20260508 11:37:40.273 | INFO | } | 20260508 11:37:40.273 | INFO | Perf report for the last 5.13774 seconds: | 20260508 11:37:40.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:40.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.273 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:37:40.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.273 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:37:40.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:40.274 | INFO | Uploaded 36GiB to GPU in 2.44412s (14.7292GiB/s) | 20260508 11:37:40.274 | INFO | Downloaded 34GiB to GPU in 2.4634s (13.8021GiB/s) | 20260508 11:37:45.415 | INFO | Analyzing chunk frames = [7803904,7804416) | 20260508 11:37:45.415 | INFO | MemInfo: { | 20260508 11:37:45.415 | INFO | "gpu" : " 17 GB", | 20260508 11:37:45.415 | INFO | "huge" : "", | 20260508 11:37:45.415 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:45.415 | INFO | "malloc" : " 14 GB", | 20260508 11:37:45.415 | INFO | "pinned" : " 29 GB" | 20260508 11:37:45.415 | INFO | } | 20260508 11:37:45.415 | INFO | Perf report for the last 5.14197 seconds: | 20260508 11:37:45.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:45.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.415 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:37:45.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.416 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:37:45.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:45.416 | INFO | Uploaded 36GiB to GPU in 2.49478s (14.4301GiB/s) | 20260508 11:37:45.416 | INFO | Downloaded 34GiB to GPU in 2.44391s (13.9121GiB/s) | 20260508 11:37:50.558 | INFO | Analyzing chunk frames = [7804416,7804928) | 20260508 11:37:50.558 | INFO | MemInfo: { | 20260508 11:37:50.558 | INFO | "gpu" : " 17 GB", | 20260508 11:37:50.558 | INFO | "huge" : "", | 20260508 11:37:50.558 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:50.558 | INFO | "malloc" : " 14 GB", | 20260508 11:37:50.558 | INFO | "pinned" : " 29 GB" | 20260508 11:37:50.558 | INFO | } | 20260508 11:37:50.558 | INFO | Perf report for the last 5.14264 seconds: | 20260508 11:37:50.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:50.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.558 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 11:37:50.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.558 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:37:50.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:50.559 | INFO | Uploaded 36GiB to GPU in 2.48321s (14.4973GiB/s) | 20260508 11:37:50.559 | INFO | Downloaded 34GiB to GPU in 2.45361s (13.8571GiB/s) | 20260508 11:37:55.700 | INFO | Analyzing chunk frames = [7804928,7805440) | 20260508 11:37:55.700 | INFO | MemInfo: { | 20260508 11:37:55.700 | INFO | "gpu" : " 17 GB", | 20260508 11:37:55.700 | INFO | "huge" : "", | 20260508 11:37:55.700 | INFO | "huge_shared" : " 29 GB", | 20260508 11:37:55.700 | INFO | "malloc" : " 14 GB", | 20260508 11:37:55.700 | INFO | "pinned" : " 29 GB" | 20260508 11:37:55.700 | INFO | } | 20260508 11:37:55.700 | INFO | Perf report for the last 5.14234 seconds: | 20260508 11:37:55.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:37:55.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.700 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:37:55.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.700 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:37:55.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:37:55.701 | INFO | Uploaded 36GiB to GPU in 2.31299s (15.5643GiB/s) | 20260508 11:37:55.701 | INFO | Downloaded 34GiB to GPU in 2.49675s (13.6177GiB/s) | 20260508 11:38:00.835 | INFO | Analyzing chunk frames = [7805440,7805952) | 20260508 11:38:00.835 | INFO | MemInfo: { | 20260508 11:38:00.835 | INFO | "gpu" : " 17 GB", | 20260508 11:38:00.835 | INFO | "huge" : "", | 20260508 11:38:00.835 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:00.835 | INFO | "malloc" : " 14 GB", | 20260508 11:38:00.835 | INFO | "pinned" : " 29 GB" | 20260508 11:38:00.835 | INFO | } | 20260508 11:38:00.835 | INFO | Perf report for the last 5.13505 seconds: | 20260508 11:38:00.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:00.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.835 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 11:38:00.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.836 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:38:00.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:00.836 | INFO | Uploaded 36GiB to GPU in 2.44533s (14.7219GiB/s) | 20260508 11:38:00.836 | INFO | Downloaded 34GiB to GPU in 2.43217s (13.9793GiB/s) | 20260508 11:38:05.975 | INFO | Analyzing chunk frames = [7805952,7806464) | 20260508 11:38:05.975 | INFO | MemInfo: { | 20260508 11:38:05.975 | INFO | "gpu" : " 17 GB", | 20260508 11:38:05.975 | INFO | "huge" : "", | 20260508 11:38:05.975 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:05.975 | INFO | "malloc" : " 14 GB", | 20260508 11:38:05.975 | INFO | "pinned" : " 29 GB" | 20260508 11:38:05.975 | INFO | } | 20260508 11:38:05.975 | INFO | Perf report for the last 5.13992 seconds: | 20260508 11:38:05.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:05.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.975 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:38:05.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.975 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:38:05.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:05.976 | INFO | Uploaded 36GiB to GPU in 2.46044s (14.6315GiB/s) | 20260508 11:38:05.976 | INFO | Downloaded 34GiB to GPU in 2.44441s (13.9093GiB/s) | 20260508 11:38:11.108 | INFO | Analyzing chunk frames = [7806464,7806976) | 20260508 11:38:11.108 | INFO | MemInfo: { | 20260508 11:38:11.108 | INFO | "gpu" : " 17 GB", | 20260508 11:38:11.108 | INFO | "huge" : "", | 20260508 11:38:11.109 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:11.109 | INFO | "malloc" : " 14 GB", | 20260508 11:38:11.109 | INFO | "pinned" : " 29 GB" | 20260508 11:38:11.109 | INFO | } | 20260508 11:38:11.109 | INFO | Perf report for the last 5.13328 seconds: | 20260508 11:38:11.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:11.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 11:38:11.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:38:11.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:11.109 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:38:11.109 | INFO | Total mem usage: 16.7 GiB | 20260508 11:38:11.109 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:38:11.109 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:38:11.109 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:38:11.109 | INFO | High Water Breakdown: | 20260508 11:38:11.109 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:38:11.109 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:38:11.109 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:38:11.109 | INFO | ------------------------------------------ | 20260508 11:38:11.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:38:11.109 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:38:11.109 | INFO | Total mem usage: 26.5 GiB | 20260508 11:38:11.109 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:38:11.109 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:38:11.109 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:38:11.109 | INFO | High Water Breakdown: | 20260508 11:38:11.109 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:38:11.109 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:38:11.109 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:38:11.109 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:38:11.109 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:38:11.109 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:38:11.109 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:38:11.109 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:38:11.109 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:38:11.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:38:11.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:38:11.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:38:11.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:38:11.109 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:38:11.109 | INFO | ------------------------------------------ | 20260508 11:38:11.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:38:11.109 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:38:11.109 | INFO | Total mem usage: 27.8 GiB | 20260508 11:38:11.109 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:38:11.109 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:38:11.109 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:38:11.109 | INFO | High Water Breakdown: | 20260508 11:38:11.109 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:38:11.109 | INFO | ------------------------------------------ | 20260508 11:38:11.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:38:11.109 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:38:11.109 | INFO | Total mem usage: 15.7 GiB | 20260508 11:38:11.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:38:11.109 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:38:11.109 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:38:11.109 | INFO | High Water Breakdown: | 20260508 11:38:11.109 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:38:11.109 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:38:11.109 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:38:11.109 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:38:11.109 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:38:11.109 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:38:11.109 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:38:11.109 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:38:11.109 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:38:11.109 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:38:11.109 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:38:11.109 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:38:11.109 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:38:11.109 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:38:11.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:38:11.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:38:11.109 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:38:11.109 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:38:11.109 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:38:11.109 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:38:11.109 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:38:11.109 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:38:11.109 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:38:11.109 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:38:11.109 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:38:11.109 | INFO | ------------------------------------------ | 20260508 11:38:11.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:38:11.109 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:38:11.109 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:38:11.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:38:11.109 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.40e+01 / 7.20e-01 | 20260508 11:38:11.109 | INFO | PerfReport: 0.30%, AggregateStats -- 9.61e-02 / 1.04e-01, 1.12e-02 / 4.82e-03 | 20260508 11:38:11.109 | INFO | PerfReport: 12.18%, Baseline -- 3.96e+00 / 4.33e+00, 5.88e-01 / 1.82e-01 | 20260508 11:38:11.109 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.15e-01 | 20260508 11:38:11.109 | INFO | PerfReport: 16.07%, DME -- 4.19e+01 / 4.59e+01, 3.83e+00 / 8.20e-01 | 20260508 11:38:11.109 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.73e+00 / 7.88e-01 | 20260508 11:38:11.109 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.10e-01 / 1.38e-01 | 20260508 11:38:11.109 | INFO | PerfReport: 2.99%, Metrics -- 9.74e-01 / 1.09e+00, 2.37e-01 / 8.89e-02 | 20260508 11:38:11.109 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.21e-01, 4.78e-02 / 1.86e-02 | 20260508 11:38:11.109 | INFO | Uploaded 36GiB to GPU in 2.48642s (14.4786GiB/s) | 20260508 11:38:11.109 | INFO | Downloaded 34GiB to GPU in 2.453s (13.8606GiB/s) | 20260508 11:38:11.109 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:38:11.109 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:38:11.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:38:11.109 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 24.43%, Upload -- 2.57e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 24.65%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 30.96%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.80e+00 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 2.60%, LockDownload -- 2.74e+00 / 0.00e+00, 4.12e+00 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 15.11%, LockCompute -- 1.59e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:38:11.109 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.21e-02 / 0.00e+00 | 20260508 11:38:11.206 | INFO | Starting a round of Baz IO | 20260508 11:38:14.612 | INFO | Populating RTMetrics report | 20260508 11:38:14.613 | INFO | RTMetrics report populated | 20260508 11:38:14.613 | INFO | Opening tmp RTMetrics json file | 20260508 11:38:14.621 | INFO | Serializing RTMetrics to json | 20260508 11:38:14.623 | INFO | Serialization done, swapping tmp file now... | 20260508 11:38:14.623 | INFO | RTMetrics done | 20260508 11:38:14.682 | INFO | Finished a round of Baz IO: Wrote 1.43 GiB in 3.476 seconds | 20260508 11:38:14.682 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:38:14.682 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185664% / 0% / 7.94729e-05% | 20260508 11:38:14.682 | INFO | Baz IO overall: | 20260508 11:38:14.682 | INFO | BytesWritten : 3364218836890 | 20260508 11:38:14.682 | INFO | NumEvents : 1615515184202 | 20260508 11:38:16.252 | INFO | Analyzing chunk frames = [7806976,7807488) | 20260508 11:38:16.252 | INFO | MemInfo: { | 20260508 11:38:16.252 | INFO | "gpu" : " 17 GB", | 20260508 11:38:16.252 | INFO | "huge" : "", | 20260508 11:38:16.252 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:16.252 | INFO | "malloc" : " 14 GB", | 20260508 11:38:16.252 | INFO | "pinned" : " 29 GB" | 20260508 11:38:16.252 | INFO | } | 20260508 11:38:16.252 | INFO | Perf report for the last 5.14333 seconds: | 20260508 11:38:16.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:16.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:16.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:16.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:16.252 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:38:16.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:16.252 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:38:16.252 | INFO | PRE_HQ : 65.5%, 1.6, 54.0ms, 0 | 20260508 11:38:16.252 | INFO | BAZWRITER : 66.8%, 2.5, 85.0ms, 0 | 20260508 11:38:16.252 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:38:16.253 | INFO | Uploaded 36GiB to GPU in 2.63116s (13.6822GiB/s) | 20260508 11:38:16.253 | INFO | Downloaded 38GiB to GPU in 2.70878s (14.0285GiB/s) | 20260508 11:38:21.392 | INFO | Analyzing chunk frames = [7807488,7808000) | 20260508 11:38:21.392 | INFO | MemInfo: { | 20260508 11:38:21.392 | INFO | "gpu" : " 17 GB", | 20260508 11:38:21.392 | INFO | "huge" : "", | 20260508 11:38:21.392 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:21.392 | INFO | "malloc" : " 14 GB", | 20260508 11:38:21.392 | INFO | "pinned" : " 29 GB" | 20260508 11:38:21.392 | INFO | } | 20260508 11:38:21.392 | INFO | Perf report for the last 5.13983 seconds: | 20260508 11:38:21.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:21.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:38:21.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 11:38:21.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:21.392 | INFO | Uploaded 36GiB to GPU in 2.47938s (14.5198GiB/s) | 20260508 11:38:21.392 | INFO | Downloaded 34GiB to GPU in 2.45544s (13.8468GiB/s) | 20260508 11:38:26.525 | INFO | Analyzing chunk frames = [7808000,7808512) | 20260508 11:38:26.525 | INFO | MemInfo: { | 20260508 11:38:26.525 | INFO | "gpu" : " 17 GB", | 20260508 11:38:26.525 | INFO | "huge" : "", | 20260508 11:38:26.525 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:26.525 | INFO | "malloc" : " 14 GB", | 20260508 11:38:26.525 | INFO | "pinned" : " 29 GB" | 20260508 11:38:26.525 | INFO | } | 20260508 11:38:26.525 | INFO | Perf report for the last 5.13316 seconds: | 20260508 11:38:26.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:26.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:38:26.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:38:26.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:26.525 | INFO | Uploaded 36GiB to GPU in 2.5232s (14.2676GiB/s) | 20260508 11:38:26.525 | INFO | Downloaded 34GiB to GPU in 2.43633s (13.9554GiB/s) | 20260508 11:38:31.667 | INFO | Analyzing chunk frames = [7808512,7809024) | 20260508 11:38:31.667 | INFO | MemInfo: { | 20260508 11:38:31.667 | INFO | "gpu" : " 17 GB", | 20260508 11:38:31.667 | INFO | "huge" : "", | 20260508 11:38:31.667 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:31.667 | INFO | "malloc" : " 14 GB", | 20260508 11:38:31.667 | INFO | "pinned" : " 29 GB" | 20260508 11:38:31.667 | INFO | } | 20260508 11:38:31.667 | INFO | Perf report for the last 5.14223 seconds: | 20260508 11:38:31.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:31.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.667 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 11:38:31.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.667 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 11:38:31.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:31.668 | INFO | Uploaded 36GiB to GPU in 2.49334s (14.4385GiB/s) | 20260508 11:38:31.668 | INFO | Downloaded 34GiB to GPU in 2.45422s (13.8537GiB/s) | 20260508 11:38:36.810 | INFO | Analyzing chunk frames = [7809024,7809536) | 20260508 11:38:36.810 | INFO | MemInfo: { | 20260508 11:38:36.810 | INFO | "gpu" : " 17 GB", | 20260508 11:38:36.810 | INFO | "huge" : "", | 20260508 11:38:36.810 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:36.810 | INFO | "malloc" : " 14 GB", | 20260508 11:38:36.810 | INFO | "pinned" : " 29 GB" | 20260508 11:38:36.810 | INFO | } | 20260508 11:38:36.810 | INFO | Perf report for the last 5.14301 seconds: | 20260508 11:38:36.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:36.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 11:38:36.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:38:36.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:36.811 | INFO | Uploaded 36GiB to GPU in 2.46179s (14.6235GiB/s) | 20260508 11:38:36.811 | INFO | Downloaded 34GiB to GPU in 2.53263s (13.4248GiB/s) | 20260508 11:38:41.947 | INFO | Analyzing chunk frames = [7809536,7810048) | 20260508 11:38:41.947 | INFO | MemInfo: { | 20260508 11:38:41.947 | INFO | "gpu" : " 17 GB", | 20260508 11:38:41.947 | INFO | "huge" : "", | 20260508 11:38:41.947 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:41.947 | INFO | "malloc" : " 14 GB", | 20260508 11:38:41.947 | INFO | "pinned" : " 29 GB" | 20260508 11:38:41.947 | INFO | } | 20260508 11:38:41.947 | INFO | Perf report for the last 5.13687 seconds: | 20260508 11:38:41.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:41.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 11:38:41.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:38:41.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:41.947 | INFO | Uploaded 36GiB to GPU in 2.39333s (15.0418GiB/s) | 20260508 11:38:41.948 | INFO | Downloaded 34GiB to GPU in 2.43112s (13.9853GiB/s) | 20260508 11:38:47.083 | INFO | Analyzing chunk frames = [7810048,7810560) | 20260508 11:38:47.083 | INFO | MemInfo: { | 20260508 11:38:47.083 | INFO | "gpu" : " 17 GB", | 20260508 11:38:47.083 | INFO | "huge" : "", | 20260508 11:38:47.083 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:47.083 | INFO | "malloc" : " 14 GB", | 20260508 11:38:47.083 | INFO | "pinned" : " 29 GB" | 20260508 11:38:47.083 | INFO | } | 20260508 11:38:47.083 | INFO | Perf report for the last 5.13595 seconds: | 20260508 11:38:47.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:47.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 11:38:47.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:38:47.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:47.083 | INFO | Uploaded 36GiB to GPU in 2.46515s (14.6036GiB/s) | 20260508 11:38:47.084 | INFO | Downloaded 34GiB to GPU in 2.45422s (13.8537GiB/s) | 20260508 11:38:52.226 | INFO | Analyzing chunk frames = [7810560,7811072) | 20260508 11:38:52.226 | INFO | MemInfo: { | 20260508 11:38:52.226 | INFO | "gpu" : " 17 GB", | 20260508 11:38:52.226 | INFO | "huge" : "", | 20260508 11:38:52.226 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:52.226 | INFO | "malloc" : " 14 GB", | 20260508 11:38:52.226 | INFO | "pinned" : " 29 GB" | 20260508 11:38:52.226 | INFO | } | 20260508 11:38:52.226 | INFO | Perf report for the last 5.14308 seconds: | 20260508 11:38:52.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:52.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.226 | INFO | ANALYSIS : 65.3%, 3.0, 102.1ms, 0 | 20260508 11:38:52.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.226 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:38:52.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:52.227 | INFO | Uploaded 36GiB to GPU in 2.42902s (14.8208GiB/s) | 20260508 11:38:52.227 | INFO | Downloaded 34GiB to GPU in 2.4273s (14.0073GiB/s) | 20260508 11:38:52.324 | INFO | Starting a round of Baz IO | 20260508 11:38:55.741 | INFO | Populating RTMetrics report | 20260508 11:38:55.742 | INFO | RTMetrics report populated | 20260508 11:38:55.742 | INFO | Opening tmp RTMetrics json file | 20260508 11:38:55.750 | INFO | Serializing RTMetrics to json | 20260508 11:38:55.751 | INFO | Serialization done, swapping tmp file now... | 20260508 11:38:55.751 | INFO | RTMetrics done | 20260508 11:38:55.806 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.481 seconds | 20260508 11:38:55.806 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:38:55.806 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.1858% / 0% / 5.96046e-05% | 20260508 11:38:55.806 | INFO | Baz IO overall: | 20260508 11:38:55.806 | INFO | BytesWritten : 3365751749902 | 20260508 11:38:55.806 | INFO | NumEvents : 1616235768933 | 20260508 11:38:57.373 | INFO | Analyzing chunk frames = [7811072,7811584) | 20260508 11:38:57.373 | INFO | MemInfo: { | 20260508 11:38:57.373 | INFO | "gpu" : " 17 GB", | 20260508 11:38:57.373 | INFO | "huge" : "", | 20260508 11:38:57.373 | INFO | "huge_shared" : " 29 GB", | 20260508 11:38:57.373 | INFO | "malloc" : " 14 GB", | 20260508 11:38:57.373 | INFO | "pinned" : " 29 GB" | 20260508 11:38:57.373 | INFO | } | 20260508 11:38:57.373 | INFO | Perf report for the last 5.14694 seconds: | 20260508 11:38:57.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:38:57.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:57.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:57.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:57.373 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 11:38:57.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:38:57.373 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:38:57.373 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 11:38:57.373 | INFO | BAZWRITER : 66.9%, 2.5, 85.6ms, 0 | 20260508 11:38:57.373 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:38:57.373 | INFO | Uploaded 36GiB to GPU in 2.60333s (13.8284GiB/s) | 20260508 11:38:57.373 | INFO | Downloaded 38GiB to GPU in 2.70777s (14.0337GiB/s) | 20260508 11:39:02.514 | INFO | Analyzing chunk frames = [7811584,7812096) | 20260508 11:39:02.514 | INFO | MemInfo: { | 20260508 11:39:02.514 | INFO | "gpu" : " 17 GB", | 20260508 11:39:02.514 | INFO | "huge" : "", | 20260508 11:39:02.514 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:02.514 | INFO | "malloc" : " 14 GB", | 20260508 11:39:02.514 | INFO | "pinned" : " 29 GB" | 20260508 11:39:02.514 | INFO | } | 20260508 11:39:02.514 | INFO | Perf report for the last 5.14118 seconds: | 20260508 11:39:02.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:02.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.514 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 11:39:02.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.514 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 11:39:02.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:02.515 | INFO | Uploaded 36GiB to GPU in 2.42696s (14.8334GiB/s) | 20260508 11:39:02.515 | INFO | Downloaded 34GiB to GPU in 2.48312s (13.6925GiB/s) | 20260508 11:39:07.647 | INFO | Analyzing chunk frames = [7812096,7812608) | 20260508 11:39:07.647 | INFO | MemInfo: { | 20260508 11:39:07.647 | INFO | "gpu" : " 17 GB", | 20260508 11:39:07.647 | INFO | "huge" : "", | 20260508 11:39:07.647 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:07.647 | INFO | "malloc" : " 14 GB", | 20260508 11:39:07.647 | INFO | "pinned" : " 29 GB" | 20260508 11:39:07.647 | INFO | } | 20260508 11:39:07.647 | INFO | Perf report for the last 5.13241 seconds: | 20260508 11:39:07.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:07.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 11:39:07.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:39:07.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:07.647 | INFO | Uploaded 36GiB to GPU in 2.43898s (14.7602GiB/s) | 20260508 11:39:07.647 | INFO | Downloaded 34GiB to GPU in 2.47924s (13.7139GiB/s) | 20260508 11:39:12.784 | INFO | Analyzing chunk frames = [7812608,7813120) | 20260508 11:39:12.784 | INFO | MemInfo: { | 20260508 11:39:12.784 | INFO | "gpu" : " 17 GB", | 20260508 11:39:12.784 | INFO | "huge" : "", | 20260508 11:39:12.784 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:12.784 | INFO | "malloc" : " 14 GB", | 20260508 11:39:12.784 | INFO | "pinned" : " 29 GB" | 20260508 11:39:12.784 | INFO | } | 20260508 11:39:12.784 | INFO | Perf report for the last 5.13713 seconds: | 20260508 11:39:12.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:12.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.784 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 11:39:12.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.784 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:39:12.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:12.785 | INFO | Uploaded 36GiB to GPU in 2.49685s (14.4182GiB/s) | 20260508 11:39:12.785 | INFO | Downloaded 34GiB to GPU in 2.46612s (13.7868GiB/s) | 20260508 11:39:17.925 | INFO | Analyzing chunk frames = [7813120,7813632) | 20260508 11:39:17.925 | INFO | MemInfo: { | 20260508 11:39:17.925 | INFO | "gpu" : " 17 GB", | 20260508 11:39:17.925 | INFO | "huge" : "", | 20260508 11:39:17.925 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:17.925 | INFO | "malloc" : " 14 GB", | 20260508 11:39:17.925 | INFO | "pinned" : " 29 GB" | 20260508 11:39:17.925 | INFO | } | 20260508 11:39:17.925 | INFO | Perf report for the last 5.14107 seconds: | 20260508 11:39:17.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:17.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 11:39:17.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 11:39:17.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:17.925 | INFO | Uploaded 36GiB to GPU in 2.3991s (15.0056GiB/s) | 20260508 11:39:17.925 | INFO | Downloaded 34GiB to GPU in 2.53726s (13.4003GiB/s) | 20260508 11:39:23.068 | INFO | Analyzing chunk frames = [7813632,7814144) | 20260508 11:39:23.068 | INFO | MemInfo: { | 20260508 11:39:23.068 | INFO | "gpu" : " 17 GB", | 20260508 11:39:23.069 | INFO | "huge" : "", | 20260508 11:39:23.069 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:23.069 | INFO | "malloc" : " 14 GB", | 20260508 11:39:23.069 | INFO | "pinned" : " 29 GB" | 20260508 11:39:23.069 | INFO | } | 20260508 11:39:23.069 | INFO | Perf report for the last 5.14368 seconds: | 20260508 11:39:23.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:23.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 11:39:23.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:39:23.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:23.069 | INFO | Uploaded 36GiB to GPU in 2.35713s (15.2728GiB/s) | 20260508 11:39:23.069 | INFO | Downloaded 34GiB to GPU in 2.43227s (13.9787GiB/s) | 20260508 11:39:28.203 | INFO | Analyzing chunk frames = [7814144,7814656) | 20260508 11:39:28.203 | INFO | MemInfo: { | 20260508 11:39:28.203 | INFO | "gpu" : " 17 GB", | 20260508 11:39:28.203 | INFO | "huge" : "", | 20260508 11:39:28.203 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:28.204 | INFO | "malloc" : " 14 GB", | 20260508 11:39:28.204 | INFO | "pinned" : " 29 GB" | 20260508 11:39:28.204 | INFO | } | 20260508 11:39:28.204 | INFO | Perf report for the last 5.13487 seconds: | 20260508 11:39:28.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:28.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 11:39:28.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:39:28.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:28.204 | INFO | Uploaded 36GiB to GPU in 2.44756s (14.7085GiB/s) | 20260508 11:39:28.204 | INFO | Downloaded 34GiB to GPU in 2.46207s (13.8095GiB/s) | 20260508 11:39:33.338 | INFO | Analyzing chunk frames = [7814656,7815168) | 20260508 11:39:33.338 | INFO | MemInfo: { | 20260508 11:39:33.338 | INFO | "gpu" : " 17 GB", | 20260508 11:39:33.338 | INFO | "huge" : "", | 20260508 11:39:33.338 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:33.338 | INFO | "malloc" : " 14 GB", | 20260508 11:39:33.338 | INFO | "pinned" : " 29 GB" | 20260508 11:39:33.339 | INFO | } | 20260508 11:39:33.339 | INFO | Perf report for the last 5.13496 seconds: | 20260508 11:39:33.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:33.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 11:39:33.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:39:33.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:33.339 | INFO | Uploaded 36GiB to GPU in 2.44436s (14.7278GiB/s) | 20260508 11:39:33.339 | INFO | Downloaded 34GiB to GPU in 2.4547s (13.851GiB/s) | 20260508 11:39:33.339 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:39:33.339 | INFO | Total mem usage: 16.7 GiB | 20260508 11:39:33.339 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:39:33.339 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:39:33.339 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:39:33.339 | INFO | High Water Breakdown: | 20260508 11:39:33.339 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:39:33.339 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:39:33.339 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:39:33.339 | INFO | ------------------------------------------ | 20260508 11:39:33.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:39:33.339 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:39:33.339 | INFO | Total mem usage: 26.5 GiB | 20260508 11:39:33.339 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:39:33.339 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:39:33.339 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:39:33.339 | INFO | High Water Breakdown: | 20260508 11:39:33.339 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:39:33.339 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:39:33.339 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:39:33.339 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:39:33.339 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:39:33.339 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:39:33.339 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:39:33.339 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:39:33.339 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:39:33.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:39:33.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:39:33.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:39:33.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:39:33.339 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:39:33.339 | INFO | ------------------------------------------ | 20260508 11:39:33.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:39:33.339 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:39:33.339 | INFO | Total mem usage: 27.8 GiB | 20260508 11:39:33.339 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:39:33.339 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:39:33.339 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:39:33.339 | INFO | High Water Breakdown: | 20260508 11:39:33.339 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:39:33.339 | INFO | ------------------------------------------ | 20260508 11:39:33.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:39:33.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:39:33.339 | INFO | Total mem usage: 15.7 GiB | 20260508 11:39:33.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:39:33.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:39:33.339 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:39:33.339 | INFO | High Water Breakdown: | 20260508 11:39:33.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:39:33.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:39:33.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:39:33.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:39:33.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:39:33.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:39:33.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:39:33.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:39:33.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:39:33.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:39:33.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:39:33.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:39:33.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:39:33.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:39:33.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:39:33.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:39:33.339 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:39:33.339 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:39:33.339 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:39:33.339 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:39:33.339 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:39:33.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:39:33.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:39:33.339 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:39:33.339 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:39:33.339 | INFO | ------------------------------------------ | 20260508 11:39:33.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:39:33.339 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:39:33.339 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:39:33.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:39:33.339 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.50e+01, 1.37e+01 / 5.70e-01 | 20260508 11:39:33.339 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.11e-02 / 5.52e-03 | 20260508 11:39:33.339 | INFO | PerfReport: 12.16%, Baseline -- 3.96e+00 / 4.41e+00, 5.89e-01 / 2.23e-01 | 20260508 11:39:33.339 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.28e-01 | 20260508 11:39:33.339 | INFO | PerfReport: 16.00%, DME -- 4.17e+01 / 4.47e+01, 3.62e+00 / 9.46e-01 | 20260508 11:39:33.339 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.77e+00 / 1.32e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.33e+00 / 2.49e+00, 4.06e-01 / 1.98e-01 | 20260508 11:39:33.339 | INFO | PerfReport: 2.99%, Metrics -- 9.72e-01 / 1.13e+00, 2.43e-01 / 1.16e-01 | 20260508 11:39:33.339 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.26e-01, 4.80e-02 / 2.31e-02 | 20260508 11:39:33.339 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:39:33.339 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:39:33.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:39:33.339 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 24.47%, Upload -- 2.57e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 31.00%, Compute -- 3.26e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 2.13%, LockUpload -- 2.24e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 2.64%, LockDownload -- 2.77e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 15.02%, LockCompute -- 1.58e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:39:33.339 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 11:39:33.436 | INFO | Starting a round of Baz IO | 20260508 11:39:36.841 | INFO | Populating RTMetrics report | 20260508 11:39:36.841 | INFO | RTMetrics report populated | 20260508 11:39:36.841 | INFO | Opening tmp RTMetrics json file | 20260508 11:39:36.850 | INFO | Serializing RTMetrics to json | 20260508 11:39:36.851 | INFO | Serialization done, swapping tmp file now... | 20260508 11:39:36.851 | INFO | RTMetrics done | 20260508 11:39:36.910 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.474 seconds | 20260508 11:39:36.911 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:39:36.911 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185422% / 0% / 8.74201e-05% | 20260508 11:39:36.911 | INFO | Baz IO overall: | 20260508 11:39:36.911 | INFO | BytesWritten : 3367286229232 | 20260508 11:39:36.911 | INFO | NumEvents : 1616957321094 | 20260508 11:39:38.483 | INFO | Analyzing chunk frames = [7815168,7815680) | 20260508 11:39:38.483 | INFO | MemInfo: { | 20260508 11:39:38.483 | INFO | "gpu" : " 17 GB", | 20260508 11:39:38.483 | INFO | "huge" : "", | 20260508 11:39:38.483 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:38.483 | INFO | "malloc" : " 14 GB", | 20260508 11:39:38.483 | INFO | "pinned" : " 29 GB" | 20260508 11:39:38.483 | INFO | } | 20260508 11:39:38.483 | INFO | Perf report for the last 5.14473 seconds: | 20260508 11:39:38.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:38.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:38.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:38.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:38.483 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 11:39:38.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:38.483 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260508 11:39:38.484 | INFO | PRE_HQ : 65.8%, 1.7, 58.2ms, 0 | 20260508 11:39:38.484 | INFO | BAZWRITER : 66.7%, 2.6, 90.7ms, 0 | 20260508 11:39:38.484 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:39:38.484 | INFO | Uploaded 36GiB to GPU in 2.63182s (13.6787GiB/s) | 20260508 11:39:38.484 | INFO | Downloaded 38GiB to GPU in 2.70433s (14.0515GiB/s) | 20260508 11:39:43.623 | INFO | Analyzing chunk frames = [7815680,7816192) | 20260508 11:39:43.623 | INFO | MemInfo: { | 20260508 11:39:43.623 | INFO | "gpu" : " 17 GB", | 20260508 11:39:43.623 | INFO | "huge" : "", | 20260508 11:39:43.623 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:43.623 | INFO | "malloc" : " 14 GB", | 20260508 11:39:43.623 | INFO | "pinned" : " 29 GB" | 20260508 11:39:43.623 | INFO | } | 20260508 11:39:43.623 | INFO | Perf report for the last 5.13973 seconds: | 20260508 11:39:43.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:43.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:39:43.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:39:43.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:43.623 | INFO | Uploaded 36GiB to GPU in 2.44665s (14.714GiB/s) | 20260508 11:39:43.623 | INFO | Downloaded 34GiB to GPU in 2.47129s (13.758GiB/s) | 20260508 11:39:48.757 | INFO | Analyzing chunk frames = [7816192,7816704) | 20260508 11:39:48.757 | INFO | MemInfo: { | 20260508 11:39:48.757 | INFO | "gpu" : " 17 GB", | 20260508 11:39:48.757 | INFO | "huge" : "", | 20260508 11:39:48.757 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:48.757 | INFO | "malloc" : " 14 GB", | 20260508 11:39:48.757 | INFO | "pinned" : " 29 GB" | 20260508 11:39:48.757 | INFO | } | 20260508 11:39:48.757 | INFO | Perf report for the last 5.13417 seconds: | 20260508 11:39:48.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:48.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.757 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:39:48.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.757 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:39:48.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:48.758 | INFO | Uploaded 36GiB to GPU in 2.49077s (14.4534GiB/s) | 20260508 11:39:48.758 | INFO | Downloaded 34GiB to GPU in 2.46205s (13.8096GiB/s) | 20260508 11:39:53.896 | INFO | Analyzing chunk frames = [7816704,7817216) | 20260508 11:39:53.896 | INFO | MemInfo: { | 20260508 11:39:53.896 | INFO | "gpu" : " 17 GB", | 20260508 11:39:53.896 | INFO | "huge" : "", | 20260508 11:39:53.896 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:53.896 | INFO | "malloc" : " 14 GB", | 20260508 11:39:53.896 | INFO | "pinned" : " 29 GB" | 20260508 11:39:53.896 | INFO | } | 20260508 11:39:53.896 | INFO | Perf report for the last 5.13918 seconds: | 20260508 11:39:53.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:53.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.897 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:39:53.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.897 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:39:53.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:53.897 | INFO | Uploaded 36GiB to GPU in 2.46831s (14.5849GiB/s) | 20260508 11:39:53.897 | INFO | Downloaded 34GiB to GPU in 2.46288s (13.805GiB/s) | 20260508 11:39:59.034 | INFO | Analyzing chunk frames = [7817216,7817728) | 20260508 11:39:59.034 | INFO | MemInfo: { | 20260508 11:39:59.034 | INFO | "gpu" : " 17 GB", | 20260508 11:39:59.034 | INFO | "huge" : "", | 20260508 11:39:59.034 | INFO | "huge_shared" : " 29 GB", | 20260508 11:39:59.034 | INFO | "malloc" : " 14 GB", | 20260508 11:39:59.034 | INFO | "pinned" : " 29 GB" | 20260508 11:39:59.034 | INFO | } | 20260508 11:39:59.035 | INFO | Perf report for the last 5.1381 seconds: | 20260508 11:39:59.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:39:59.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:39:59.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:39:59.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:39:59.035 | INFO | Uploaded 36GiB to GPU in 2.30421s (15.6236GiB/s) | 20260508 11:39:59.035 | INFO | Downloaded 34GiB to GPU in 2.51818s (13.5018GiB/s) | 20260508 11:40:04.181 | INFO | Analyzing chunk frames = [7817728,7818240) | 20260508 11:40:04.181 | INFO | MemInfo: { | 20260508 11:40:04.181 | INFO | "gpu" : " 17 GB", | 20260508 11:40:04.181 | INFO | "huge" : "", | 20260508 11:40:04.181 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:04.181 | INFO | "malloc" : " 14 GB", | 20260508 11:40:04.181 | INFO | "pinned" : " 29 GB" | 20260508 11:40:04.181 | INFO | } | 20260508 11:40:04.181 | INFO | Perf report for the last 5.1464 seconds: | 20260508 11:40:04.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:04.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | ANALYSIS : 65.1%, 3.0, 101.5ms, 0 | 20260508 11:40:04.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:40:04.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:04.181 | INFO | Uploaded 36GiB to GPU in 2.35962s (15.2567GiB/s) | 20260508 11:40:04.181 | INFO | Downloaded 34GiB to GPU in 2.45452s (13.852GiB/s) | 20260508 11:40:09.319 | INFO | Analyzing chunk frames = [7818240,7818752) | 20260508 11:40:09.319 | INFO | MemInfo: { | 20260508 11:40:09.319 | INFO | "gpu" : " 17 GB", | 20260508 11:40:09.319 | INFO | "huge" : "", | 20260508 11:40:09.319 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:09.319 | INFO | "malloc" : " 14 GB", | 20260508 11:40:09.320 | INFO | "pinned" : " 29 GB" | 20260508 11:40:09.320 | INFO | } | 20260508 11:40:09.320 | INFO | Perf report for the last 5.13862 seconds: | 20260508 11:40:09.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:09.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 11:40:09.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 11:40:09.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:09.320 | INFO | Uploaded 36GiB to GPU in 2.40269s (14.9832GiB/s) | 20260508 11:40:09.320 | INFO | Downloaded 34GiB to GPU in 2.46271s (13.8059GiB/s) | 20260508 11:40:14.463 | INFO | Analyzing chunk frames = [7818752,7819264) | 20260508 11:40:14.463 | INFO | MemInfo: { | 20260508 11:40:14.463 | INFO | "gpu" : " 17 GB", | 20260508 11:40:14.463 | INFO | "huge" : "", | 20260508 11:40:14.463 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:14.463 | INFO | "malloc" : " 14 GB", | 20260508 11:40:14.463 | INFO | "pinned" : " 29 GB" | 20260508 11:40:14.463 | INFO | } | 20260508 11:40:14.463 | INFO | Perf report for the last 5.1433 seconds: | 20260508 11:40:14.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:14.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 11:40:14.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:40:14.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:14.463 | INFO | Uploaded 36GiB to GPU in 2.4395s (14.7571GiB/s) | 20260508 11:40:14.464 | INFO | Downloaded 34GiB to GPU in 2.48324s (13.6918GiB/s) | 20260508 11:40:14.560 | INFO | Starting a round of Baz IO | 20260508 11:40:17.965 | INFO | Populating RTMetrics report | 20260508 11:40:17.965 | INFO | RTMetrics report populated | 20260508 11:40:17.965 | INFO | Opening tmp RTMetrics json file | 20260508 11:40:17.966 | INFO | Serializing RTMetrics to json | 20260508 11:40:17.967 | INFO | Serialization done, swapping tmp file now... | 20260508 11:40:17.967 | INFO | RTMetrics done | 20260508 11:40:18.030 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.469 seconds | 20260508 11:40:18.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:40:18.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184937% / 0% / 6.35783e-05% | 20260508 11:40:18.030 | INFO | Baz IO overall: | 20260508 11:40:18.030 | INFO | BytesWritten : 3368819868919 | 20260508 11:40:18.030 | INFO | NumEvents : 1617678598184 | 20260508 11:40:19.595 | INFO | Analyzing chunk frames = [7819264,7819776) | 20260508 11:40:19.595 | INFO | MemInfo: { | 20260508 11:40:19.595 | INFO | "gpu" : " 17 GB", | 20260508 11:40:19.595 | INFO | "huge" : "", | 20260508 11:40:19.596 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:19.596 | INFO | "malloc" : " 14 GB", | 20260508 11:40:19.596 | INFO | "pinned" : " 29 GB" | 20260508 11:40:19.596 | INFO | } | 20260508 11:40:19.596 | INFO | Perf report for the last 5.13274 seconds: | 20260508 11:40:19.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:19.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:19.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:19.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:19.596 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260508 11:40:19.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:19.596 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 11:40:19.596 | INFO | PRE_HQ : 65.6%, 1.6, 53.8ms, 0 | 20260508 11:40:19.596 | INFO | BAZWRITER : 66.8%, 2.5, 85.4ms, 0 | 20260508 11:40:19.596 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:40:19.596 | INFO | Uploaded 36GiB to GPU in 2.57501s (13.9805GiB/s) | 20260508 11:40:19.596 | INFO | Downloaded 38GiB to GPU in 2.74021s (13.8675GiB/s) | 20260508 11:40:24.736 | INFO | Analyzing chunk frames = [7819776,7820288) | 20260508 11:40:24.736 | INFO | MemInfo: { | 20260508 11:40:24.736 | INFO | "gpu" : " 17 GB", | 20260508 11:40:24.736 | INFO | "huge" : "", | 20260508 11:40:24.736 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:24.736 | INFO | "malloc" : " 14 GB", | 20260508 11:40:24.736 | INFO | "pinned" : " 29 GB" | 20260508 11:40:24.736 | INFO | } | 20260508 11:40:24.736 | INFO | Perf report for the last 5.14044 seconds: | 20260508 11:40:24.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:24.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 11:40:24.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:40:24.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:24.736 | INFO | Uploaded 36GiB to GPU in 2.42231s (14.8618GiB/s) | 20260508 11:40:24.736 | INFO | Downloaded 34GiB to GPU in 2.48378s (13.6888GiB/s) | 20260508 11:40:29.874 | INFO | Analyzing chunk frames = [7820288,7820800) | 20260508 11:40:29.874 | INFO | MemInfo: { | 20260508 11:40:29.874 | INFO | "gpu" : " 17 GB", | 20260508 11:40:29.874 | INFO | "huge" : "", | 20260508 11:40:29.874 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:29.875 | INFO | "malloc" : " 14 GB", | 20260508 11:40:29.875 | INFO | "pinned" : " 29 GB" | 20260508 11:40:29.875 | INFO | } | 20260508 11:40:29.875 | INFO | Perf report for the last 5.13868 seconds: | 20260508 11:40:29.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:29.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:40:29.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:40:29.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:29.875 | INFO | Uploaded 36GiB to GPU in 2.47681s (14.5348GiB/s) | 20260508 11:40:29.875 | INFO | Downloaded 34GiB to GPU in 2.46636s (13.7855GiB/s) | 20260508 11:40:35.017 | INFO | Analyzing chunk frames = [7820800,7821312) | 20260508 11:40:35.017 | INFO | MemInfo: { | 20260508 11:40:35.017 | INFO | "gpu" : " 17 GB", | 20260508 11:40:35.017 | INFO | "huge" : "", | 20260508 11:40:35.017 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:35.018 | INFO | "malloc" : " 14 GB", | 20260508 11:40:35.018 | INFO | "pinned" : " 29 GB" | 20260508 11:40:35.018 | INFO | } | 20260508 11:40:35.018 | INFO | Perf report for the last 5.14292 seconds: | 20260508 11:40:35.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:35.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.018 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 11:40:35.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.018 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:40:35.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:35.019 | INFO | Uploaded 36GiB to GPU in 2.4523s (14.6801GiB/s) | 20260508 11:40:35.019 | INFO | Downloaded 34GiB to GPU in 2.4725s (13.7513GiB/s) | 20260508 11:40:40.151 | INFO | Analyzing chunk frames = [7821312,7821824) | 20260508 11:40:40.151 | INFO | MemInfo: { | 20260508 11:40:40.151 | INFO | "gpu" : " 17 GB", | 20260508 11:40:40.151 | INFO | "huge" : "", | 20260508 11:40:40.151 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:40.151 | INFO | "malloc" : " 14 GB", | 20260508 11:40:40.151 | INFO | "pinned" : " 29 GB" | 20260508 11:40:40.151 | INFO | } | 20260508 11:40:40.151 | INFO | Perf report for the last 5.13341 seconds: | 20260508 11:40:40.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:40.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.151 | INFO | ANALYSIS : 66.5%, 3.0, 103.8ms, 0 | 20260508 11:40:40.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.151 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260508 11:40:40.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:40.152 | INFO | Uploaded 36GiB to GPU in 2.43842s (14.7636GiB/s) | 20260508 11:40:40.152 | INFO | Downloaded 34GiB to GPU in 2.54203s (13.3751GiB/s) | 20260508 11:40:45.288 | INFO | Analyzing chunk frames = [7821824,7822336) | 20260508 11:40:45.288 | INFO | MemInfo: { | 20260508 11:40:45.288 | INFO | "gpu" : " 17 GB", | 20260508 11:40:45.288 | INFO | "huge" : "", | 20260508 11:40:45.288 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:45.288 | INFO | "malloc" : " 14 GB", | 20260508 11:40:45.288 | INFO | "pinned" : " 29 GB" | 20260508 11:40:45.288 | INFO | } | 20260508 11:40:45.288 | INFO | Perf report for the last 5.13683 seconds: | 20260508 11:40:45.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:45.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:40:45.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260508 11:40:45.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:45.288 | INFO | Uploaded 36GiB to GPU in 2.38159s (15.116GiB/s) | 20260508 11:40:45.288 | INFO | Downloaded 34GiB to GPU in 2.43523s (13.9617GiB/s) | 20260508 11:40:50.434 | INFO | Analyzing chunk frames = [7822336,7822848) | 20260508 11:40:50.434 | INFO | MemInfo: { | 20260508 11:40:50.434 | INFO | "gpu" : " 17 GB", | 20260508 11:40:50.434 | INFO | "huge" : "", | 20260508 11:40:50.434 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:50.434 | INFO | "malloc" : " 14 GB", | 20260508 11:40:50.434 | INFO | "pinned" : " 29 GB" | 20260508 11:40:50.434 | INFO | } | 20260508 11:40:50.434 | INFO | Perf report for the last 5.14582 seconds: | 20260508 11:40:50.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:50.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | ANALYSIS : 65.1%, 3.0, 101.6ms, 0 | 20260508 11:40:50.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 11:40:50.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:50.434 | INFO | Uploaded 36GiB to GPU in 2.38399s (15.1007GiB/s) | 20260508 11:40:50.434 | INFO | Downloaded 34GiB to GPU in 2.44174s (13.9245GiB/s) | 20260508 11:40:55.573 | INFO | Analyzing chunk frames = [7822848,7823360) | 20260508 11:40:55.573 | INFO | MemInfo: { | 20260508 11:40:55.573 | INFO | "gpu" : " 17 GB", | 20260508 11:40:55.573 | INFO | "huge" : "", | 20260508 11:40:55.573 | INFO | "huge_shared" : " 29 GB", | 20260508 11:40:55.573 | INFO | "malloc" : " 14 GB", | 20260508 11:40:55.573 | INFO | "pinned" : " 29 GB" | 20260508 11:40:55.573 | INFO | } | 20260508 11:40:55.573 | INFO | Perf report for the last 5.13921 seconds: | 20260508 11:40:55.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:40:55.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.573 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 11:40:55.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.573 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:40:55.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:40:55.574 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:40:55.574 | INFO | Total mem usage: 16.7 GiB | 20260508 11:40:55.574 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:40:55.574 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:40:55.574 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:40:55.574 | INFO | High Water Breakdown: | 20260508 11:40:55.574 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:40:55.574 | INFO | Uploaded 36GiB to GPU in 2.44018s (14.753GiB/s) | 20260508 11:40:55.574 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:40:55.574 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:40:55.574 | INFO | ------------------------------------------ | 20260508 11:40:55.574 | INFO | Downloaded 34GiB to GPU in 2.4496s (13.8798GiB/s) | 20260508 11:40:55.574 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:40:55.574 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:40:55.574 | INFO | Total mem usage: 26.5 GiB | 20260508 11:40:55.574 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:40:55.574 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:40:55.574 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:40:55.574 | INFO | High Water Breakdown: | 20260508 11:40:55.574 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:40:55.574 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:40:55.574 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:40:55.574 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:40:55.574 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:40:55.574 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:40:55.574 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:40:55.574 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:40:55.574 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:40:55.574 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:40:55.574 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:40:55.574 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:40:55.574 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:40:55.574 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:40:55.574 | INFO | ------------------------------------------ | 20260508 11:40:55.574 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:40:55.574 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:40:55.574 | INFO | Total mem usage: 27.8 GiB | 20260508 11:40:55.574 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:40:55.574 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:40:55.574 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:40:55.574 | INFO | High Water Breakdown: | 20260508 11:40:55.574 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:40:55.574 | INFO | ------------------------------------------ | 20260508 11:40:55.574 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:40:55.574 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:40:55.574 | INFO | Total mem usage: 15.7 GiB | 20260508 11:40:55.574 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:40:55.574 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:40:55.574 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:40:55.574 | INFO | High Water Breakdown: | 20260508 11:40:55.574 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:40:55.574 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:40:55.574 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:40:55.574 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:40:55.574 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:40:55.574 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:40:55.574 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:40:55.574 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:40:55.574 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:40:55.574 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:40:55.574 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:40:55.574 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:40:55.574 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:40:55.574 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:40:55.574 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:40:55.574 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:40:55.574 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:40:55.574 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:40:55.574 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:40:55.574 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:40:55.574 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:40:55.574 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:40:55.574 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:40:55.575 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:40:55.575 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:40:55.575 | INFO | ------------------------------------------ | 20260508 11:40:55.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:40:55.575 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:40:55.575 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:40:55.575 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:40:55.575 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.38e+01 / 6.91e-01 | 20260508 11:40:55.575 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.13e-02 / 6.03e-03 | 20260508 11:40:55.575 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.45e+00, 5.94e-01 / 2.15e-01 | 20260508 11:40:55.575 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.09e-01 / 1.23e-01 | 20260508 11:40:55.575 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.53e+01, 3.71e+00 / 1.21e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.78e+00 / 1.33e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.08e-01 / 1.81e-01 | 20260508 11:40:55.575 | INFO | PerfReport: 2.99%, Metrics -- 9.75e-01 / 1.11e+00, 2.42e-01 / 9.64e-02 | 20260508 11:40:55.575 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.33e-01, 4.82e-02 / 1.92e-02 | 20260508 11:40:55.575 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:40:55.575 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:40:55.575 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:40:55.575 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 24.24%, Upload -- 2.55e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 24.82%, Download -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 1.99%, LockUpload -- 2.09e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 2.72%, LockDownload -- 2.86e+00 / 0.00e+00, 4.23e+00 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 15.16%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:40:55.575 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260508 11:40:55.671 | INFO | Starting a round of Baz IO | 20260508 11:40:59.089 | INFO | Populating RTMetrics report | 20260508 11:40:59.090 | INFO | RTMetrics report populated | 20260508 11:40:59.090 | INFO | Opening tmp RTMetrics json file | 20260508 11:40:59.101 | INFO | Serializing RTMetrics to json | 20260508 11:40:59.103 | INFO | Serialization done, swapping tmp file now... | 20260508 11:40:59.103 | INFO | RTMetrics done | 20260508 11:40:59.155 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.484 seconds | 20260508 11:40:59.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.09% / 0.00% / 59.91% | 20260508 11:40:59.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184679% / 0% / 6.35783e-05% | 20260508 11:40:59.156 | INFO | Baz IO overall: | 20260508 11:40:59.156 | INFO | BytesWritten : 3370354027007 | 20260508 11:40:59.156 | INFO | NumEvents : 1618399930630 | 20260508 11:41:00.706 | INFO | Analyzing chunk frames = [7823360,7823872) | 20260508 11:41:00.706 | INFO | MemInfo: { | 20260508 11:41:00.706 | INFO | "gpu" : " 17 GB", | 20260508 11:41:00.706 | INFO | "huge" : "", | 20260508 11:41:00.706 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:00.706 | INFO | "malloc" : " 14 GB", | 20260508 11:41:00.706 | INFO | "pinned" : " 29 GB" | 20260508 11:41:00.706 | INFO | } | 20260508 11:41:00.706 | INFO | Perf report for the last 5.13258 seconds: | 20260508 11:41:00.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:00.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:00.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:00.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:00.706 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 11:41:00.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:00.706 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 11:41:00.706 | INFO | PRE_HQ : 65.6%, 1.6, 54.4ms, 0 | 20260508 11:41:00.706 | INFO | BAZWRITER : 67.1%, 2.5, 85.2ms, 0 | 20260508 11:41:00.706 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:41:00.706 | INFO | Uploaded 36GiB to GPU in 2.63083s (13.6839GiB/s) | 20260508 11:41:00.706 | INFO | Downloaded 38GiB to GPU in 2.72211s (13.9597GiB/s) | 20260508 11:41:05.843 | INFO | Analyzing chunk frames = [7823872,7824384) | 20260508 11:41:05.843 | INFO | MemInfo: { | 20260508 11:41:05.843 | INFO | "gpu" : " 17 GB", | 20260508 11:41:05.843 | INFO | "huge" : "", | 20260508 11:41:05.843 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:05.843 | INFO | "malloc" : " 14 GB", | 20260508 11:41:05.843 | INFO | "pinned" : " 29 GB" | 20260508 11:41:05.843 | INFO | } | 20260508 11:41:05.843 | INFO | Perf report for the last 5.13755 seconds: | 20260508 11:41:05.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:05.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.843 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 11:41:05.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.843 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:41:05.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:05.844 | INFO | Uploaded 36GiB to GPU in 2.50097s (14.3944GiB/s) | 20260508 11:41:05.844 | INFO | Downloaded 34GiB to GPU in 2.46113s (13.8148GiB/s) | 20260508 11:41:10.989 | INFO | Analyzing chunk frames = [7824384,7824896) | 20260508 11:41:10.989 | INFO | MemInfo: { | 20260508 11:41:10.989 | INFO | "gpu" : " 17 GB", | 20260508 11:41:10.989 | INFO | "huge" : "", | 20260508 11:41:10.989 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:10.989 | INFO | "malloc" : " 14 GB", | 20260508 11:41:10.989 | INFO | "pinned" : " 29 GB" | 20260508 11:41:10.989 | INFO | } | 20260508 11:41:10.989 | INFO | Perf report for the last 5.14606 seconds: | 20260508 11:41:10.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:10.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.989 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 11:41:10.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.989 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:41:10.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:10.990 | INFO | Uploaded 36GiB to GPU in 2.50171s (14.3902GiB/s) | 20260508 11:41:10.990 | INFO | Downloaded 34GiB to GPU in 2.42395s (14.0267GiB/s) | 20260508 11:41:16.131 | INFO | Analyzing chunk frames = [7824896,7825408) | 20260508 11:41:16.131 | INFO | MemInfo: { | 20260508 11:41:16.131 | INFO | "gpu" : " 17 GB", | 20260508 11:41:16.131 | INFO | "huge" : "", | 20260508 11:41:16.131 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:16.131 | INFO | "malloc" : " 14 GB", | 20260508 11:41:16.131 | INFO | "pinned" : " 29 GB" | 20260508 11:41:16.131 | INFO | } | 20260508 11:41:16.131 | INFO | Perf report for the last 5.14156 seconds: | 20260508 11:41:16.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:16.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:41:16.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:41:16.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:16.131 | INFO | Uploaded 36GiB to GPU in 2.48942s (14.4612GiB/s) | 20260508 11:41:16.131 | INFO | Downloaded 34GiB to GPU in 2.43023s (13.9904GiB/s) | 20260508 11:41:21.275 | INFO | Analyzing chunk frames = [7825408,7825920) | 20260508 11:41:21.275 | INFO | MemInfo: { | 20260508 11:41:21.275 | INFO | "gpu" : " 17 GB", | 20260508 11:41:21.275 | INFO | "huge" : "", | 20260508 11:41:21.275 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:21.275 | INFO | "malloc" : " 14 GB", | 20260508 11:41:21.275 | INFO | "pinned" : " 29 GB" | 20260508 11:41:21.275 | INFO | } | 20260508 11:41:21.275 | INFO | Perf report for the last 5.1445 seconds: | 20260508 11:41:21.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:21.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.275 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:41:21.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.275 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260508 11:41:21.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:21.276 | INFO | Uploaded 36GiB to GPU in 2.40236s (14.9853GiB/s) | 20260508 11:41:21.276 | INFO | Downloaded 34GiB to GPU in 2.51228s (13.5335GiB/s) | 20260508 11:41:26.411 | INFO | Analyzing chunk frames = [7825920,7826432) | 20260508 11:41:26.411 | INFO | MemInfo: { | 20260508 11:41:26.411 | INFO | "gpu" : " 17 GB", | 20260508 11:41:26.411 | INFO | "huge" : "", | 20260508 11:41:26.411 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:26.411 | INFO | "malloc" : " 14 GB", | 20260508 11:41:26.411 | INFO | "pinned" : " 29 GB" | 20260508 11:41:26.411 | INFO | } | 20260508 11:41:26.411 | INFO | Perf report for the last 5.13547 seconds: | 20260508 11:41:26.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:26.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:41:26.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 11:41:26.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:26.411 | INFO | Uploaded 36GiB to GPU in 2.42768s (14.829GiB/s) | 20260508 11:41:26.411 | INFO | Downloaded 34GiB to GPU in 2.4422s (13.9219GiB/s) | 20260508 11:41:31.544 | INFO | Analyzing chunk frames = [7826432,7826944) | 20260508 11:41:31.544 | INFO | MemInfo: { | 20260508 11:41:31.544 | INFO | "gpu" : " 17 GB", | 20260508 11:41:31.544 | INFO | "huge" : "", | 20260508 11:41:31.544 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:31.544 | INFO | "malloc" : " 14 GB", | 20260508 11:41:31.544 | INFO | "pinned" : " 29 GB" | 20260508 11:41:31.544 | INFO | } | 20260508 11:41:31.544 | INFO | Perf report for the last 5.13342 seconds: | 20260508 11:41:31.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:31.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.544 | INFO | ANALYSIS : 65.0%, 3.0, 101.0ms, 0 | 20260508 11:41:31.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.544 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:41:31.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:31.545 | INFO | Uploaded 36GiB to GPU in 2.35991s (15.2548GiB/s) | 20260508 11:41:31.545 | INFO | Downloaded 34GiB to GPU in 2.42445s (14.0238GiB/s) | 20260508 11:41:36.687 | INFO | Analyzing chunk frames = [7826944,7827456) | 20260508 11:41:36.687 | INFO | MemInfo: { | 20260508 11:41:36.687 | INFO | "gpu" : " 17 GB", | 20260508 11:41:36.687 | INFO | "huge" : "", | 20260508 11:41:36.687 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:36.687 | INFO | "malloc" : " 14 GB", | 20260508 11:41:36.687 | INFO | "pinned" : " 29 GB" | 20260508 11:41:36.687 | INFO | } | 20260508 11:41:36.687 | INFO | Perf report for the last 5.14288 seconds: | 20260508 11:41:36.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:36.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.687 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 11:41:36.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.687 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:41:36.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:36.688 | INFO | Uploaded 36GiB to GPU in 2.45819s (14.6449GiB/s) | 20260508 11:41:36.688 | INFO | Downloaded 34GiB to GPU in 2.45217s (13.8652GiB/s) | 20260508 11:41:36.784 | INFO | Starting a round of Baz IO | 20260508 11:41:40.194 | INFO | Populating RTMetrics report | 20260508 11:41:40.194 | INFO | RTMetrics report populated | 20260508 11:41:40.194 | INFO | Opening tmp RTMetrics json file | 20260508 11:41:40.205 | INFO | Serializing RTMetrics to json | 20260508 11:41:40.206 | INFO | Serialization done, swapping tmp file now... | 20260508 11:41:40.207 | INFO | RTMetrics done | 20260508 11:41:40.264 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.479 seconds | 20260508 11:41:40.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.91% | 20260508 11:41:40.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184413% / 7.94729e-06% / 8.34465e-05% | 20260508 11:41:40.264 | INFO | Baz IO overall: | 20260508 11:41:40.264 | INFO | BytesWritten : 3371887946470 | 20260508 11:41:40.264 | INFO | NumEvents : 1619121489967 | 20260508 11:41:41.828 | INFO | Analyzing chunk frames = [7827456,7827968) | 20260508 11:41:41.828 | INFO | MemInfo: { | 20260508 11:41:41.828 | INFO | "gpu" : " 17 GB", | 20260508 11:41:41.828 | INFO | "huge" : "", | 20260508 11:41:41.828 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:41.828 | INFO | "malloc" : " 14 GB", | 20260508 11:41:41.828 | INFO | "pinned" : " 29 GB" | 20260508 11:41:41.828 | INFO | } | 20260508 11:41:41.828 | INFO | Perf report for the last 5.14076 seconds: | 20260508 11:41:41.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:41.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:41.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:41.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:41.828 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 11:41:41.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:41.828 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:41:41.828 | INFO | PRE_HQ : 65.2%, 1.6, 54.1ms, 0 | 20260508 11:41:41.828 | INFO | BAZWRITER : 67.0%, 2.5, 86.4ms, 0 | 20260508 11:41:41.828 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 11:41:41.828 | INFO | Uploaded 36GiB to GPU in 2.59492s (13.8733GiB/s) | 20260508 11:41:41.828 | INFO | Downloaded 38GiB to GPU in 2.71153s (14.0142GiB/s) | 20260508 11:41:46.965 | INFO | Analyzing chunk frames = [7827968,7828480) | 20260508 11:41:46.965 | INFO | MemInfo: { | 20260508 11:41:46.965 | INFO | "gpu" : " 17 GB", | 20260508 11:41:46.965 | INFO | "huge" : "", | 20260508 11:41:46.965 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:46.965 | INFO | "malloc" : " 14 GB", | 20260508 11:41:46.965 | INFO | "pinned" : " 29 GB" | 20260508 11:41:46.965 | INFO | } | 20260508 11:41:46.965 | INFO | Perf report for the last 5.13728 seconds: | 20260508 11:41:46.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:46.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:41:46.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:41:46.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:46.965 | INFO | Uploaded 36GiB to GPU in 2.43808s (14.7657GiB/s) | 20260508 11:41:46.966 | INFO | Downloaded 34GiB to GPU in 2.46397s (13.7989GiB/s) | 20260508 11:41:52.108 | INFO | Analyzing chunk frames = [7828480,7828992) | 20260508 11:41:52.108 | INFO | MemInfo: { | 20260508 11:41:52.108 | INFO | "gpu" : " 17 GB", | 20260508 11:41:52.108 | INFO | "huge" : "", | 20260508 11:41:52.108 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:52.109 | INFO | "malloc" : " 14 GB", | 20260508 11:41:52.109 | INFO | "pinned" : " 29 GB" | 20260508 11:41:52.109 | INFO | } | 20260508 11:41:52.109 | INFO | Perf report for the last 5.1434 seconds: | 20260508 11:41:52.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:52.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:41:52.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:41:52.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:52.109 | INFO | Uploaded 36GiB to GPU in 2.4865s (14.4782GiB/s) | 20260508 11:41:52.109 | INFO | Downloaded 34GiB to GPU in 2.43189s (13.9809GiB/s) | 20260508 11:41:57.244 | INFO | Analyzing chunk frames = [7828992,7829504) | 20260508 11:41:57.244 | INFO | MemInfo: { | 20260508 11:41:57.244 | INFO | "gpu" : " 17 GB", | 20260508 11:41:57.244 | INFO | "huge" : "", | 20260508 11:41:57.245 | INFO | "huge_shared" : " 29 GB", | 20260508 11:41:57.245 | INFO | "malloc" : " 14 GB", | 20260508 11:41:57.245 | INFO | "pinned" : " 29 GB" | 20260508 11:41:57.245 | INFO | } | 20260508 11:41:57.245 | INFO | Perf report for the last 5.13627 seconds: | 20260508 11:41:57.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:41:57.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.245 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 11:41:57.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.245 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 11:41:57.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:41:57.246 | INFO | Uploaded 36GiB to GPU in 2.40521s (14.9675GiB/s) | 20260508 11:41:57.246 | INFO | Downloaded 34GiB to GPU in 2.46785s (13.7772GiB/s) | 20260508 11:42:02.384 | INFO | Analyzing chunk frames = [7829504,7830016) | 20260508 11:42:02.384 | INFO | MemInfo: { | 20260508 11:42:02.384 | INFO | "gpu" : " 17 GB", | 20260508 11:42:02.384 | INFO | "huge" : "", | 20260508 11:42:02.384 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:02.384 | INFO | "malloc" : " 14 GB", | 20260508 11:42:02.384 | INFO | "pinned" : " 29 GB" | 20260508 11:42:02.384 | INFO | } | 20260508 11:42:02.384 | INFO | Perf report for the last 5.13902 seconds: | 20260508 11:42:02.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:02.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:42:02.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260508 11:42:02.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:02.384 | INFO | Uploaded 36GiB to GPU in 2.33125s (15.4423GiB/s) | 20260508 11:42:02.384 | INFO | Downloaded 34GiB to GPU in 2.50195s (13.5894GiB/s) | 20260508 11:42:07.522 | INFO | Analyzing chunk frames = [7830016,7830528) | 20260508 11:42:07.522 | INFO | MemInfo: { | 20260508 11:42:07.522 | INFO | "gpu" : " 17 GB", | 20260508 11:42:07.522 | INFO | "huge" : "", | 20260508 11:42:07.522 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:07.522 | INFO | "malloc" : " 14 GB", | 20260508 11:42:07.522 | INFO | "pinned" : " 29 GB" | 20260508 11:42:07.522 | INFO | } | 20260508 11:42:07.522 | INFO | Perf report for the last 5.13821 seconds: | 20260508 11:42:07.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:07.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.522 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 11:42:07.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.522 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:42:07.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:07.523 | INFO | Uploaded 36GiB to GPU in 2.38419s (15.0995GiB/s) | 20260508 11:42:07.523 | INFO | Downloaded 34GiB to GPU in 2.44657s (13.897GiB/s) | 20260508 11:42:12.659 | INFO | Analyzing chunk frames = [7830528,7831040) | 20260508 11:42:12.659 | INFO | MemInfo: { | 20260508 11:42:12.659 | INFO | "gpu" : " 17 GB", | 20260508 11:42:12.659 | INFO | "huge" : "", | 20260508 11:42:12.659 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:12.660 | INFO | "malloc" : " 14 GB", | 20260508 11:42:12.660 | INFO | "pinned" : " 29 GB" | 20260508 11:42:12.660 | INFO | } | 20260508 11:42:12.660 | INFO | Perf report for the last 5.13747 seconds: | 20260508 11:42:12.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:12.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:42:12.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:42:12.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:12.660 | INFO | Uploaded 36GiB to GPU in 2.4875s (14.4724GiB/s) | 20260508 11:42:12.660 | INFO | Downloaded 34GiB to GPU in 2.45836s (13.8304GiB/s) | 20260508 11:42:17.799 | INFO | Analyzing chunk frames = [7831040,7831552) | 20260508 11:42:17.799 | INFO | MemInfo: { | 20260508 11:42:17.799 | INFO | "gpu" : " 17 GB", | 20260508 11:42:17.799 | INFO | "huge" : "", | 20260508 11:42:17.799 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:17.799 | INFO | "malloc" : " 14 GB", | 20260508 11:42:17.799 | INFO | "pinned" : " 29 GB" | 20260508 11:42:17.799 | INFO | } | 20260508 11:42:17.799 | INFO | Perf report for the last 5.13965 seconds: | 20260508 11:42:17.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:17.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.799 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 11:42:17.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.799 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:42:17.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:17.800 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:42:17.800 | INFO | Total mem usage: 16.7 GiB | 20260508 11:42:17.800 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:42:17.800 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:42:17.800 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:42:17.800 | INFO | High Water Breakdown: | 20260508 11:42:17.800 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:42:17.800 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:42:17.800 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:42:17.800 | INFO | ------------------------------------------ | 20260508 11:42:17.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:42:17.800 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:42:17.800 | INFO | Total mem usage: 26.5 GiB | 20260508 11:42:17.800 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:42:17.800 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:42:17.800 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:42:17.800 | INFO | High Water Breakdown: | 20260508 11:42:17.800 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:42:17.800 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:42:17.800 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:42:17.800 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:42:17.800 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:42:17.800 | INFO | Uploaded 36GiB to GPU in 2.45692s (14.6525GiB/s) | 20260508 11:42:17.800 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:42:17.800 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:42:17.800 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:42:17.800 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:42:17.800 | INFO | Downloaded 34GiB to GPU in 2.44613s (13.8995GiB/s) | 20260508 11:42:17.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:42:17.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:42:17.800 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:42:17.800 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:42:17.800 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:42:17.800 | INFO | ------------------------------------------ | 20260508 11:42:17.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:42:17.800 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:42:17.800 | INFO | Total mem usage: 27.8 GiB | 20260508 11:42:17.800 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:42:17.800 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:42:17.800 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:42:17.800 | INFO | High Water Breakdown: | 20260508 11:42:17.800 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:42:17.800 | INFO | ------------------------------------------ | 20260508 11:42:17.800 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:42:17.800 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:42:17.800 | INFO | Total mem usage: 15.7 GiB | 20260508 11:42:17.800 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:42:17.800 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:42:17.800 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:42:17.800 | INFO | High Water Breakdown: | 20260508 11:42:17.800 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:42:17.800 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:42:17.800 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:42:17.800 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:42:17.800 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:42:17.800 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:42:17.800 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:42:17.800 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:42:17.800 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:42:17.801 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:42:17.801 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:42:17.801 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:42:17.801 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:42:17.801 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:42:17.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:42:17.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:42:17.801 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:42:17.801 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:42:17.801 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:42:17.801 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:42:17.801 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:42:17.801 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:42:17.801 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:42:17.801 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:42:17.801 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:42:17.801 | INFO | ------------------------------------------ | 20260508 11:42:17.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:42:17.801 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:42:17.801 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:42:17.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:42:17.801 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.46e-01 | 20260508 11:42:17.801 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.05e-01, 1.13e-02 / 7.38e-03 | 20260508 11:42:17.801 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.50e+00, 5.98e-01 / 1.68e-01 | 20260508 11:42:17.801 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.49e+00, 2.09e-01 / 1.21e-01 | 20260508 11:42:17.801 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.55e+01, 3.73e+00 / 9.64e-01 | 20260508 11:42:17.801 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.82e+00 / 1.31e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.61e+00, 4.04e-01 / 1.87e-01 | 20260508 11:42:17.801 | INFO | PerfReport: 2.99%, Metrics -- 9.76e-01 / 1.12e+00, 2.39e-01 / 1.01e-01 | 20260508 11:42:17.801 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.33e-01, 4.80e-02 / 2.66e-02 | 20260508 11:42:17.801 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:42:17.801 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:42:17.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:42:17.801 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 24.39%, Upload -- 2.56e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 24.66%, Download -- 2.59e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 2.19%, LockUpload -- 2.30e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 2.54%, LockDownload -- 2.66e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 15.08%, LockCompute -- 1.58e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:42:17.801 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260508 11:42:17.897 | INFO | Starting a round of Baz IO | 20260508 11:42:21.315 | INFO | Populating RTMetrics report | 20260508 11:42:21.316 | INFO | RTMetrics report populated | 20260508 11:42:21.316 | INFO | Opening tmp RTMetrics json file | 20260508 11:42:21.321 | INFO | Serializing RTMetrics to json | 20260508 11:42:21.322 | INFO | Serialization done, swapping tmp file now... | 20260508 11:42:21.323 | INFO | RTMetrics done | 20260508 11:42:21.385 | INFO | Finished a round of Baz IO: Wrote 1.429 GiB in 3.487 seconds | 20260508 11:42:21.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:42:21.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183837% / 0% / 7.94729e-05% | 20260508 11:42:21.385 | INFO | Baz IO overall: | 20260508 11:42:21.385 | INFO | BytesWritten : 3373422079017 | 20260508 11:42:21.385 | INFO | NumEvents : 1619842871087 | 20260508 11:42:22.942 | INFO | Analyzing chunk frames = [7831552,7832064) | 20260508 11:42:22.942 | INFO | MemInfo: { | 20260508 11:42:22.942 | INFO | "gpu" : " 17 GB", | 20260508 11:42:22.942 | INFO | "huge" : "", | 20260508 11:42:22.942 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:22.942 | INFO | "malloc" : " 14 GB", | 20260508 11:42:22.942 | INFO | "pinned" : " 29 GB" | 20260508 11:42:22.942 | INFO | } | 20260508 11:42:22.943 | INFO | Perf report for the last 5.14321 seconds: | 20260508 11:42:22.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:22.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:22.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:22.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:22.943 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 11:42:22.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:22.943 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:42:22.943 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 11:42:22.943 | INFO | BAZWRITER : 67.1%, 2.5, 85.2ms, 0 | 20260508 11:42:22.943 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:42:22.943 | INFO | Uploaded 36GiB to GPU in 2.61242s (13.7803GiB/s) | 20260508 11:42:22.943 | INFO | Downloaded 38GiB to GPU in 2.71683s (13.9869GiB/s) | 20260508 11:42:28.094 | INFO | Analyzing chunk frames = [7832064,7832576) | 20260508 11:42:28.094 | INFO | MemInfo: { | 20260508 11:42:28.094 | INFO | "gpu" : " 17 GB", | 20260508 11:42:28.094 | INFO | "huge" : "", | 20260508 11:42:28.094 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:28.094 | INFO | "malloc" : " 14 GB", | 20260508 11:42:28.094 | INFO | "pinned" : " 29 GB" | 20260508 11:42:28.094 | INFO | } | 20260508 11:42:28.094 | INFO | Perf report for the last 5.15127 seconds: | 20260508 11:42:28.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:28.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 11:42:28.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 11:42:28.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:28.094 | INFO | Uploaded 36GiB to GPU in 2.41735s (14.8924GiB/s) | 20260508 11:42:28.095 | INFO | Downloaded 34GiB to GPU in 2.49061s (13.6513GiB/s) | 20260508 11:42:33.222 | INFO | Analyzing chunk frames = [7832576,7833088) | 20260508 11:42:33.222 | INFO | MemInfo: { | 20260508 11:42:33.222 | INFO | "gpu" : " 17 GB", | 20260508 11:42:33.222 | INFO | "huge" : "", | 20260508 11:42:33.222 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:33.222 | INFO | "malloc" : " 14 GB", | 20260508 11:42:33.222 | INFO | "pinned" : " 29 GB" | 20260508 11:42:33.222 | INFO | } | 20260508 11:42:33.222 | INFO | Perf report for the last 5.12802 seconds: | 20260508 11:42:33.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:33.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 11:42:33.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:42:33.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:33.222 | INFO | Uploaded 36GiB to GPU in 2.42835s (14.8249GiB/s) | 20260508 11:42:33.222 | INFO | Downloaded 34GiB to GPU in 2.46582s (13.7885GiB/s) | 20260508 11:42:38.353 | INFO | Analyzing chunk frames = [7833088,7833600) | 20260508 11:42:38.353 | INFO | MemInfo: { | 20260508 11:42:38.353 | INFO | "gpu" : " 17 GB", | 20260508 11:42:38.353 | INFO | "huge" : "", | 20260508 11:42:38.353 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:38.354 | INFO | "malloc" : " 14 GB", | 20260508 11:42:38.354 | INFO | "pinned" : " 29 GB" | 20260508 11:42:38.354 | INFO | } | 20260508 11:42:38.354 | INFO | Perf report for the last 5.13184 seconds: | 20260508 11:42:38.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:38.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 11:42:38.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:42:38.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:38.354 | INFO | Uploaded 36GiB to GPU in 2.44232s (14.7401GiB/s) | 20260508 11:42:38.354 | INFO | Downloaded 34GiB to GPU in 2.46483s (13.794GiB/s) | 20260508 11:42:43.497 | INFO | Analyzing chunk frames = [7833600,7834112) | 20260508 11:42:43.497 | INFO | MemInfo: { | 20260508 11:42:43.497 | INFO | "gpu" : " 17 GB", | 20260508 11:42:43.497 | INFO | "huge" : "", | 20260508 11:42:43.497 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:43.497 | INFO | "malloc" : " 14 GB", | 20260508 11:42:43.497 | INFO | "pinned" : " 29 GB" | 20260508 11:42:43.497 | INFO | } | 20260508 11:42:43.497 | INFO | Perf report for the last 5.14328 seconds: | 20260508 11:42:43.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:43.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:42:43.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:42:43.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:43.497 | INFO | Uploaded 36GiB to GPU in 2.26693s (15.8805GiB/s) | 20260508 11:42:43.497 | INFO | Downloaded 34GiB to GPU in 2.5109s (13.541GiB/s) | 20260508 11:42:48.636 | INFO | Analyzing chunk frames = [7834112,7834624) | 20260508 11:42:48.636 | INFO | MemInfo: { | 20260508 11:42:48.636 | INFO | "gpu" : " 17 GB", | 20260508 11:42:48.636 | INFO | "huge" : "", | 20260508 11:42:48.636 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:48.636 | INFO | "malloc" : " 14 GB", | 20260508 11:42:48.636 | INFO | "pinned" : " 29 GB" | 20260508 11:42:48.636 | INFO | } | 20260508 11:42:48.636 | INFO | Perf report for the last 5.13942 seconds: | 20260508 11:42:48.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:48.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.637 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:42:48.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.637 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:42:48.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:48.637 | INFO | Uploaded 36GiB to GPU in 2.38588s (15.0888GiB/s) | 20260508 11:42:48.637 | INFO | Downloaded 34GiB to GPU in 2.43839s (13.9436GiB/s) | 20260508 11:42:53.776 | INFO | Analyzing chunk frames = [7834624,7835136) | 20260508 11:42:53.776 | INFO | MemInfo: { | 20260508 11:42:53.776 | INFO | "gpu" : " 17 GB", | 20260508 11:42:53.776 | INFO | "huge" : "", | 20260508 11:42:53.776 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:53.776 | INFO | "malloc" : " 14 GB", | 20260508 11:42:53.776 | INFO | "pinned" : " 29 GB" | 20260508 11:42:53.776 | INFO | } | 20260508 11:42:53.776 | INFO | Perf report for the last 5.14001 seconds: | 20260508 11:42:53.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:53.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:42:53.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:42:53.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:53.777 | INFO | Uploaded 36GiB to GPU in 2.41933s (14.8802GiB/s) | 20260508 11:42:53.777 | INFO | Downloaded 34GiB to GPU in 2.44897s (13.8834GiB/s) | 20260508 11:42:58.916 | INFO | Analyzing chunk frames = [7835136,7835648) | 20260508 11:42:58.916 | INFO | MemInfo: { | 20260508 11:42:58.916 | INFO | "gpu" : " 17 GB", | 20260508 11:42:58.916 | INFO | "huge" : "", | 20260508 11:42:58.916 | INFO | "huge_shared" : " 29 GB", | 20260508 11:42:58.916 | INFO | "malloc" : " 14 GB", | 20260508 11:42:58.916 | INFO | "pinned" : " 29 GB" | 20260508 11:42:58.916 | INFO | } | 20260508 11:42:58.916 | INFO | Perf report for the last 5.13954 seconds: | 20260508 11:42:58.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:42:58.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 11:42:58.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:42:58.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:42:58.916 | INFO | Uploaded 36GiB to GPU in 2.45958s (14.6366GiB/s) | 20260508 11:42:58.916 | INFO | Downloaded 34GiB to GPU in 2.46813s (13.7756GiB/s) | 20260508 11:42:59.013 | INFO | Starting a round of Baz IO | 20260508 11:43:02.414 | INFO | Populating RTMetrics report | 20260508 11:43:02.414 | INFO | RTMetrics report populated | 20260508 11:43:02.414 | INFO | Opening tmp RTMetrics json file | 20260508 11:43:02.424 | INFO | Serializing RTMetrics to json | 20260508 11:43:02.426 | INFO | Serialization done, swapping tmp file now... | 20260508 11:43:02.426 | INFO | RTMetrics done | 20260508 11:43:02.485 | INFO | Finished a round of Baz IO: Wrote 1.427 GiB in 3.471 seconds | 20260508 11:43:02.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:43:02.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183276% / 3.97364e-06% / 7.94729e-05% | 20260508 11:43:02.485 | INFO | Baz IO overall: | 20260508 11:43:02.485 | INFO | BytesWritten : 3374954682243 | 20260508 11:43:02.485 | INFO | NumEvents : 1620563328123 | 20260508 11:43:04.055 | INFO | Analyzing chunk frames = [7835648,7836160) | 20260508 11:43:04.055 | INFO | MemInfo: { | 20260508 11:43:04.055 | INFO | "gpu" : " 17 GB", | 20260508 11:43:04.055 | INFO | "huge" : "", | 20260508 11:43:04.055 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:04.055 | INFO | "malloc" : " 14 GB", | 20260508 11:43:04.055 | INFO | "pinned" : " 29 GB" | 20260508 11:43:04.055 | INFO | } | 20260508 11:43:04.056 | INFO | Perf report for the last 5.13951 seconds: | 20260508 11:43:04.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:04.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:04.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:04.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:04.056 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 11:43:04.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:04.056 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:43:04.056 | INFO | PRE_HQ : 65.2%, 1.6, 53.9ms, 0 | 20260508 11:43:04.056 | INFO | BAZWRITER : 66.8%, 2.5, 84.2ms, 0 | 20260508 11:43:04.056 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:43:04.056 | INFO | Uploaded 36GiB to GPU in 2.63317s (13.6718GiB/s) | 20260508 11:43:04.056 | INFO | Downloaded 38GiB to GPU in 2.72589s (13.9404GiB/s) | 20260508 11:43:09.197 | INFO | Analyzing chunk frames = [7836160,7836672) | 20260508 11:43:09.197 | INFO | MemInfo: { | 20260508 11:43:09.197 | INFO | "gpu" : " 17 GB", | 20260508 11:43:09.197 | INFO | "huge" : "", | 20260508 11:43:09.197 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:09.198 | INFO | "malloc" : " 14 GB", | 20260508 11:43:09.198 | INFO | "pinned" : " 29 GB" | 20260508 11:43:09.198 | INFO | } | 20260508 11:43:09.198 | INFO | Perf report for the last 5.14202 seconds: | 20260508 11:43:09.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:09.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 11:43:09.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 11:43:09.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:09.198 | INFO | Uploaded 36GiB to GPU in 2.4722s (14.5619GiB/s) | 20260508 11:43:09.198 | INFO | Downloaded 34GiB to GPU in 2.48009s (13.7092GiB/s) | 20260508 11:43:14.336 | INFO | Analyzing chunk frames = [7836672,7837184) | 20260508 11:43:14.336 | INFO | MemInfo: { | 20260508 11:43:14.336 | INFO | "gpu" : " 17 GB", | 20260508 11:43:14.336 | INFO | "huge" : "", | 20260508 11:43:14.336 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:14.336 | INFO | "malloc" : " 14 GB", | 20260508 11:43:14.336 | INFO | "pinned" : " 29 GB" | 20260508 11:43:14.337 | INFO | } | 20260508 11:43:14.337 | INFO | Perf report for the last 5.13908 seconds: | 20260508 11:43:14.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:14.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:43:14.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:43:14.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:14.337 | INFO | Uploaded 36GiB to GPU in 2.4715s (14.566GiB/s) | 20260508 11:43:14.337 | INFO | Downloaded 34GiB to GPU in 2.45449s (13.8522GiB/s) | 20260508 11:43:19.473 | INFO | Analyzing chunk frames = [7837184,7837696) | 20260508 11:43:19.473 | INFO | MemInfo: { | 20260508 11:43:19.473 | INFO | "gpu" : " 17 GB", | 20260508 11:43:19.473 | INFO | "huge" : "", | 20260508 11:43:19.473 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:19.473 | INFO | "malloc" : " 14 GB", | 20260508 11:43:19.473 | INFO | "pinned" : " 29 GB" | 20260508 11:43:19.473 | INFO | } | 20260508 11:43:19.473 | INFO | Perf report for the last 5.13678 seconds: | 20260508 11:43:19.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:19.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.474 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:43:19.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.474 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:43:19.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:19.474 | INFO | Uploaded 36GiB to GPU in 2.47989s (14.5168GiB/s) | 20260508 11:43:19.474 | INFO | Downloaded 34GiB to GPU in 2.4556s (13.8459GiB/s) | 20260508 11:43:24.606 | INFO | Analyzing chunk frames = [7837696,7838208) | 20260508 11:43:24.606 | INFO | MemInfo: { | 20260508 11:43:24.606 | INFO | "gpu" : " 17 GB", | 20260508 11:43:24.606 | INFO | "huge" : "", | 20260508 11:43:24.606 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:24.606 | INFO | "malloc" : " 14 GB", | 20260508 11:43:24.606 | INFO | "pinned" : " 29 GB" | 20260508 11:43:24.606 | INFO | } | 20260508 11:43:24.606 | INFO | Perf report for the last 5.1328 seconds: | 20260508 11:43:24.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:24.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.606 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:43:24.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.606 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:43:24.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:24.607 | INFO | Uploaded 36GiB to GPU in 2.28308s (15.7681GiB/s) | 20260508 11:43:24.607 | INFO | Downloaded 34GiB to GPU in 2.49257s (13.6405GiB/s) | 20260508 11:43:29.755 | INFO | Analyzing chunk frames = [7838208,7838720) | 20260508 11:43:29.755 | INFO | MemInfo: { | 20260508 11:43:29.755 | INFO | "gpu" : " 17 GB", | 20260508 11:43:29.755 | INFO | "huge" : "", | 20260508 11:43:29.755 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:29.755 | INFO | "malloc" : " 14 GB", | 20260508 11:43:29.755 | INFO | "pinned" : " 29 GB" | 20260508 11:43:29.755 | INFO | } | 20260508 11:43:29.755 | INFO | Perf report for the last 5.14886 seconds: | 20260508 11:43:29.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:29.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.755 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 11:43:29.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.755 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:43:29.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:29.756 | INFO | Uploaded 36GiB to GPU in 2.35773s (15.2689GiB/s) | 20260508 11:43:29.756 | INFO | Downloaded 34GiB to GPU in 2.43895s (13.9404GiB/s) | 20260508 11:43:34.894 | INFO | Analyzing chunk frames = [7838720,7839232) | 20260508 11:43:34.894 | INFO | MemInfo: { | 20260508 11:43:34.894 | INFO | "gpu" : " 17 GB", | 20260508 11:43:34.894 | INFO | "huge" : "", | 20260508 11:43:34.894 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:34.894 | INFO | "malloc" : " 14 GB", | 20260508 11:43:34.894 | INFO | "pinned" : " 29 GB" | 20260508 11:43:34.894 | INFO | } | 20260508 11:43:34.894 | INFO | Perf report for the last 5.13874 seconds: | 20260508 11:43:34.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:34.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:43:34.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:43:34.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:34.894 | INFO | Uploaded 36GiB to GPU in 2.42116s (14.8689GiB/s) | 20260508 11:43:34.894 | INFO | Downloaded 34GiB to GPU in 2.44922s (13.882GiB/s) | 20260508 11:43:40.036 | INFO | Analyzing chunk frames = [7839232,7839744) | 20260508 11:43:40.036 | INFO | MemInfo: { | 20260508 11:43:40.036 | INFO | "gpu" : " 17 GB", | 20260508 11:43:40.036 | INFO | "huge" : "", | 20260508 11:43:40.036 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:40.036 | INFO | "malloc" : " 14 GB", | 20260508 11:43:40.036 | INFO | "pinned" : " 29 GB" | 20260508 11:43:40.036 | INFO | } | 20260508 11:43:40.036 | INFO | Perf report for the last 5.14246 seconds: | 20260508 11:43:40.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:40.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.036 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 11:43:40.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.037 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:43:40.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:40.037 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:43:40.037 | INFO | Total mem usage: 16.7 GiB | 20260508 11:43:40.037 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:43:40.037 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 11:43:40.037 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:43:40.037 | INFO | High Water Breakdown: | 20260508 11:43:40.037 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:43:40.037 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:43:40.037 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:43:40.037 | INFO | ------------------------------------------ | 20260508 11:43:40.037 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:43:40.037 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:43:40.037 | INFO | Total mem usage: 26.5 GiB | 20260508 11:43:40.037 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:43:40.037 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:43:40.037 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:43:40.037 | INFO | High Water Breakdown: | 20260508 11:43:40.037 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:43:40.037 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:43:40.037 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:43:40.037 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:43:40.037 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:43:40.037 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:43:40.037 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:43:40.037 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:43:40.037 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:43:40.037 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:43:40.037 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:43:40.037 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:43:40.037 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:43:40.037 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:43:40.037 | INFO | ------------------------------------------ | 20260508 11:43:40.037 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:43:40.037 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:43:40.037 | INFO | Total mem usage: 27.8 GiB | 20260508 11:43:40.038 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:43:40.038 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:43:40.037 | INFO | Uploaded 36GiB to GPU in 2.42778s (14.8284GiB/s) | 20260508 11:43:40.038 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:43:40.038 | INFO | High Water Breakdown: | 20260508 11:43:40.038 | INFO | Downloaded 34GiB to GPU in 2.43639s (13.9551GiB/s) | 20260508 11:43:40.038 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:43:40.038 | INFO | ------------------------------------------ | 20260508 11:43:40.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:43:40.038 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:43:40.038 | INFO | Total mem usage: 15.7 GiB | 20260508 11:43:40.038 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:43:40.038 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:43:40.038 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:43:40.038 | INFO | High Water Breakdown: | 20260508 11:43:40.038 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:43:40.038 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:43:40.038 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:43:40.038 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:43:40.038 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:43:40.038 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:43:40.038 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:43:40.038 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:43:40.038 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:43:40.038 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:43:40.038 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:43:40.038 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:43:40.038 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:43:40.038 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:43:40.038 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:43:40.038 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:43:40.038 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:43:40.038 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:43:40.038 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:43:40.038 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:43:40.038 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:43:40.038 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:43:40.038 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:43:40.038 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:43:40.038 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:43:40.038 | INFO | ------------------------------------------ | 20260508 11:43:40.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:43:40.038 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:43:40.038 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:43:40.038 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:43:40.038 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 4.78e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.14e-02 / 3.57e-03 | 20260508 11:43:40.038 | INFO | PerfReport: 12.22%, Baseline -- 3.98e+00 / 4.41e+00, 5.93e-01 / 1.56e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.13e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.58e+01, 3.67e+00 / 6.91e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.72e+00 / 7.75e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.34e+00 / 2.58e+00, 4.05e-01 / 1.20e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 2.99%, Metrics -- 9.75e-01 / 1.08e+00, 2.40e-01 / 1.01e-01 | 20260508 11:43:40.038 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.95e-01 / 3.21e-01, 4.72e-02 / 1.98e-02 | 20260508 11:43:40.038 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:43:40.038 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:43:40.039 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:43:40.039 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 24.18%, Upload -- 2.54e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 24.77%, Download -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 2.05%, LockUpload -- 2.15e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 2.74%, LockDownload -- 2.87e+00 / 0.00e+00, 4.30e+00 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 15.20%, LockCompute -- 1.60e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 11:43:40.039 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260508 11:43:40.149 | INFO | Starting a round of Baz IO | 20260508 11:43:43.552 | INFO | Populating RTMetrics report | 20260508 11:43:43.553 | INFO | RTMetrics report populated | 20260508 11:43:43.553 | INFO | Opening tmp RTMetrics json file | 20260508 11:43:43.553 | INFO | Serializing RTMetrics to json | 20260508 11:43:43.555 | INFO | Serialization done, swapping tmp file now... | 20260508 11:43:43.555 | INFO | RTMetrics done | 20260508 11:43:43.615 | INFO | Finished a round of Baz IO: Wrote 1.427 GiB in 3.466 seconds | 20260508 11:43:43.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:43:43.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183396% / 0% / 8.74201e-05% | 20260508 11:43:43.615 | INFO | Baz IO overall: | 20260508 11:43:43.615 | INFO | BytesWritten : 3376487220939 | 20260508 11:43:43.615 | INFO | NumEvents : 1621283618538 | 20260508 11:43:45.173 | INFO | Analyzing chunk frames = [7839744,7840256) | 20260508 11:43:45.173 | INFO | MemInfo: { | 20260508 11:43:45.173 | INFO | "gpu" : " 17 GB", | 20260508 11:43:45.173 | INFO | "huge" : "", | 20260508 11:43:45.173 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:45.173 | INFO | "malloc" : " 14 GB", | 20260508 11:43:45.173 | INFO | "pinned" : " 29 GB" | 20260508 11:43:45.173 | INFO | } | 20260508 11:43:45.173 | INFO | Perf report for the last 5.13651 seconds: | 20260508 11:43:45.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:45.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:45.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:45.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:45.173 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 11:43:45.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:45.173 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260508 11:43:45.173 | INFO | PRE_HQ : 66.5%, 1.7, 58.7ms, 0 | 20260508 11:43:45.173 | INFO | BAZWRITER : 66.7%, 2.6, 89.9ms, 0 | 20260508 11:43:45.173 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 11:43:45.173 | INFO | Uploaded 36GiB to GPU in 2.58467s (13.9283GiB/s) | 20260508 11:43:45.173 | INFO | Downloaded 38GiB to GPU in 2.71965s (13.9724GiB/s) | 20260508 11:43:50.309 | INFO | Analyzing chunk frames = [7840256,7840768) | 20260508 11:43:50.309 | INFO | MemInfo: { | 20260508 11:43:50.309 | INFO | "gpu" : " 17 GB", | 20260508 11:43:50.309 | INFO | "huge" : "", | 20260508 11:43:50.309 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:50.309 | INFO | "malloc" : " 14 GB", | 20260508 11:43:50.309 | INFO | "pinned" : " 29 GB" | 20260508 11:43:50.309 | INFO | } | 20260508 11:43:50.309 | INFO | Perf report for the last 5.13617 seconds: | 20260508 11:43:50.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:50.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.309 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:43:50.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.309 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:43:50.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:50.310 | INFO | Uploaded 36GiB to GPU in 2.4699s (14.5755GiB/s) | 20260508 11:43:50.310 | INFO | Downloaded 34GiB to GPU in 2.46465s (13.7951GiB/s) | 20260508 11:43:55.448 | INFO | Analyzing chunk frames = [7840768,7841280) | 20260508 11:43:55.448 | INFO | MemInfo: { | 20260508 11:43:55.448 | INFO | "gpu" : " 17 GB", | 20260508 11:43:55.448 | INFO | "huge" : "", | 20260508 11:43:55.448 | INFO | "huge_shared" : " 29 GB", | 20260508 11:43:55.448 | INFO | "malloc" : " 14 GB", | 20260508 11:43:55.448 | INFO | "pinned" : " 29 GB" | 20260508 11:43:55.448 | INFO | } | 20260508 11:43:55.448 | INFO | Perf report for the last 5.13929 seconds: | 20260508 11:43:55.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:43:55.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.448 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:43:55.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.448 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:43:55.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:43:55.449 | INFO | Uploaded 36GiB to GPU in 2.48007s (14.5157GiB/s) | 20260508 11:43:55.449 | INFO | Downloaded 34GiB to GPU in 2.45463s (13.8514GiB/s) | 20260508 11:44:00.586 | INFO | Analyzing chunk frames = [7841280,7841792) | 20260508 11:44:00.586 | INFO | MemInfo: { | 20260508 11:44:00.586 | INFO | "gpu" : " 17 GB", | 20260508 11:44:00.586 | INFO | "huge" : "", | 20260508 11:44:00.586 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:00.586 | INFO | "malloc" : " 14 GB", | 20260508 11:44:00.586 | INFO | "pinned" : " 29 GB" | 20260508 11:44:00.586 | INFO | } | 20260508 11:44:00.586 | INFO | Perf report for the last 5.13758 seconds: | 20260508 11:44:00.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:00.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 11:44:00.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:44:00.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:00.586 | INFO | Uploaded 36GiB to GPU in 2.46332s (14.6144GiB/s) | 20260508 11:44:00.586 | INFO | Downloaded 34GiB to GPU in 2.4615s (13.8127GiB/s) | 20260508 11:44:05.730 | INFO | Analyzing chunk frames = [7841792,7842304) | 20260508 11:44:05.730 | INFO | MemInfo: { | 20260508 11:44:05.730 | INFO | "gpu" : " 17 GB", | 20260508 11:44:05.730 | INFO | "huge" : "", | 20260508 11:44:05.730 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:05.730 | INFO | "malloc" : " 14 GB", | 20260508 11:44:05.730 | INFO | "pinned" : " 29 GB" | 20260508 11:44:05.730 | INFO | } | 20260508 11:44:05.730 | INFO | Perf report for the last 5.14381 seconds: | 20260508 11:44:05.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:05.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:44:05.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:44:05.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:05.730 | INFO | Uploaded 36GiB to GPU in 2.29211s (15.706GiB/s) | 20260508 11:44:05.730 | INFO | Downloaded 34GiB to GPU in 2.48605s (13.6763GiB/s) | 20260508 11:44:10.864 | INFO | Analyzing chunk frames = [7842304,7842816) | 20260508 11:44:10.864 | INFO | MemInfo: { | 20260508 11:44:10.864 | INFO | "gpu" : " 17 GB", | 20260508 11:44:10.864 | INFO | "huge" : "", | 20260508 11:44:10.865 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:10.865 | INFO | "malloc" : " 14 GB", | 20260508 11:44:10.865 | INFO | "pinned" : " 29 GB" | 20260508 11:44:10.865 | INFO | } | 20260508 11:44:10.865 | INFO | Perf report for the last 5.13481 seconds: | 20260508 11:44:10.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:10.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 11:44:10.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:44:10.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:10.865 | INFO | Uploaded 36GiB to GPU in 2.38949s (15.066GiB/s) | 20260508 11:44:10.865 | INFO | Downloaded 34GiB to GPU in 2.4432s (13.9162GiB/s) | 20260508 11:44:16.006 | INFO | Analyzing chunk frames = [7842816,7843328) | 20260508 11:44:16.006 | INFO | MemInfo: { | 20260508 11:44:16.006 | INFO | "gpu" : " 17 GB", | 20260508 11:44:16.006 | INFO | "huge" : "", | 20260508 11:44:16.006 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:16.007 | INFO | "malloc" : " 14 GB", | 20260508 11:44:16.007 | INFO | "pinned" : " 29 GB" | 20260508 11:44:16.007 | INFO | } | 20260508 11:44:16.007 | INFO | Perf report for the last 5.14213 seconds: | 20260508 11:44:16.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:16.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:44:16.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 11:44:16.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:16.007 | INFO | Uploaded 36GiB to GPU in 2.42924s (14.8194GiB/s) | 20260508 11:44:16.007 | INFO | Downloaded 34GiB to GPU in 2.45336s (13.8586GiB/s) | 20260508 11:44:21.155 | INFO | Analyzing chunk frames = [7843328,7843840) | 20260508 11:44:21.155 | INFO | MemInfo: { | 20260508 11:44:21.155 | INFO | "gpu" : " 17 GB", | 20260508 11:44:21.155 | INFO | "huge" : "", | 20260508 11:44:21.156 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:21.156 | INFO | "malloc" : " 14 GB", | 20260508 11:44:21.156 | INFO | "pinned" : " 29 GB" | 20260508 11:44:21.156 | INFO | } | 20260508 11:44:21.156 | INFO | Perf report for the last 5.14899 seconds: | 20260508 11:44:21.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:21.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 11:44:21.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 11:44:21.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:21.156 | INFO | Uploaded 36GiB to GPU in 2.43666s (14.7743GiB/s) | 20260508 11:44:21.156 | INFO | Downloaded 34GiB to GPU in 2.46354s (13.8013GiB/s) | 20260508 11:44:21.270 | INFO | Starting a round of Baz IO | 20260508 11:44:24.653 | INFO | Populating RTMetrics report | 20260508 11:44:24.653 | INFO | RTMetrics report populated | 20260508 11:44:24.653 | INFO | Opening tmp RTMetrics json file | 20260508 11:44:24.654 | INFO | Serializing RTMetrics to json | 20260508 11:44:24.655 | INFO | Serialization done, swapping tmp file now... | 20260508 11:44:24.656 | INFO | RTMetrics done | 20260508 11:44:24.716 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.446 seconds | 20260508 11:44:24.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:44:24.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18328% / 0% / 7.15256e-05% | 20260508 11:44:24.716 | INFO | Baz IO overall: | 20260508 11:44:24.717 | INFO | BytesWritten : 3378020181628 | 20260508 11:44:24.717 | INFO | NumEvents : 1622004211624 | 20260508 11:44:26.286 | INFO | Analyzing chunk frames = [7843840,7844352) | 20260508 11:44:26.286 | INFO | MemInfo: { | 20260508 11:44:26.286 | INFO | "gpu" : " 17 GB", | 20260508 11:44:26.286 | INFO | "huge" : "", | 20260508 11:44:26.286 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:26.286 | INFO | "malloc" : " 14 GB", | 20260508 11:44:26.286 | INFO | "pinned" : " 29 GB" | 20260508 11:44:26.286 | INFO | } | 20260508 11:44:26.286 | INFO | Perf report for the last 5.13068 seconds: | 20260508 11:44:26.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:26.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:26.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:26.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:26.287 | INFO | ANALYSIS : 67.8%, 3.0, 105.1ms, 0 | 20260508 11:44:26.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:26.287 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 11:44:26.287 | INFO | PRE_HQ : 66.6%, 1.7, 60.1ms, 0 | 20260508 11:44:26.287 | INFO | BAZWRITER : 66.7%, 2.6, 90.7ms, 0 | 20260508 11:44:26.287 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 11:44:26.287 | INFO | Uploaded 36GiB to GPU in 2.53481s (14.2023GiB/s) | 20260508 11:44:26.287 | INFO | Downloaded 38GiB to GPU in 2.7007s (14.0704GiB/s) | 20260508 11:44:31.422 | INFO | Analyzing chunk frames = [7844352,7844864) | 20260508 11:44:31.422 | INFO | MemInfo: { | 20260508 11:44:31.422 | INFO | "gpu" : " 17 GB", | 20260508 11:44:31.422 | INFO | "huge" : "", | 20260508 11:44:31.422 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:31.422 | INFO | "malloc" : " 14 GB", | 20260508 11:44:31.422 | INFO | "pinned" : " 29 GB" | 20260508 11:44:31.422 | INFO | } | 20260508 11:44:31.422 | INFO | Perf report for the last 5.13582 seconds: | 20260508 11:44:31.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:31.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.422 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 11:44:31.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.422 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:44:31.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:31.423 | INFO | Uploaded 36GiB to GPU in 2.45935s (14.638GiB/s) | 20260508 11:44:31.423 | INFO | Downloaded 34GiB to GPU in 2.46999s (13.7653GiB/s) | 20260508 11:44:36.558 | INFO | Analyzing chunk frames = [7844864,7845376) | 20260508 11:44:36.558 | INFO | MemInfo: { | 20260508 11:44:36.558 | INFO | "gpu" : " 17 GB", | 20260508 11:44:36.558 | INFO | "huge" : "", | 20260508 11:44:36.558 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:36.558 | INFO | "malloc" : " 14 GB", | 20260508 11:44:36.558 | INFO | "pinned" : " 29 GB" | 20260508 11:44:36.558 | INFO | } | 20260508 11:44:36.558 | INFO | Perf report for the last 5.1361 seconds: | 20260508 11:44:36.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:36.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.558 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:44:36.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.558 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:44:36.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:36.559 | INFO | Uploaded 36GiB to GPU in 2.46344s (14.6137GiB/s) | 20260508 11:44:36.559 | INFO | Downloaded 34GiB to GPU in 2.44909s (13.8827GiB/s) | 20260508 11:44:41.703 | INFO | Analyzing chunk frames = [7845376,7845888) | 20260508 11:44:41.703 | INFO | MemInfo: { | 20260508 11:44:41.703 | INFO | "gpu" : " 17 GB", | 20260508 11:44:41.703 | INFO | "huge" : "", | 20260508 11:44:41.703 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:41.703 | INFO | "malloc" : " 14 GB", | 20260508 11:44:41.703 | INFO | "pinned" : " 29 GB" | 20260508 11:44:41.703 | INFO | } | 20260508 11:44:41.703 | INFO | Perf report for the last 5.14463 seconds: | 20260508 11:44:41.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:41.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.703 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:44:41.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.703 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:44:41.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:41.704 | INFO | Uploaded 36GiB to GPU in 2.47399s (14.5514GiB/s) | 20260508 11:44:41.704 | INFO | Downloaded 34GiB to GPU in 2.44661s (13.8968GiB/s) | 20260508 11:44:46.846 | INFO | Analyzing chunk frames = [7845888,7846400) | 20260508 11:44:46.846 | INFO | MemInfo: { | 20260508 11:44:46.846 | INFO | "gpu" : " 17 GB", | 20260508 11:44:46.846 | INFO | "huge" : "", | 20260508 11:44:46.846 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:46.846 | INFO | "malloc" : " 14 GB", | 20260508 11:44:46.846 | INFO | "pinned" : " 29 GB" | 20260508 11:44:46.846 | INFO | } | 20260508 11:44:46.846 | INFO | Perf report for the last 5.14273 seconds: | 20260508 11:44:46.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:46.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 11:44:46.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:44:46.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:46.846 | INFO | Uploaded 36GiB to GPU in 2.36823s (15.2012GiB/s) | 20260508 11:44:46.846 | INFO | Downloaded 34GiB to GPU in 2.52041s (13.4899GiB/s) | 20260508 11:44:51.988 | INFO | Analyzing chunk frames = [7846400,7846912) | 20260508 11:44:51.988 | INFO | MemInfo: { | 20260508 11:44:51.988 | INFO | "gpu" : " 17 GB", | 20260508 11:44:51.988 | INFO | "huge" : "", | 20260508 11:44:51.988 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:51.988 | INFO | "malloc" : " 14 GB", | 20260508 11:44:51.988 | INFO | "pinned" : " 29 GB" | 20260508 11:44:51.988 | INFO | } | 20260508 11:44:51.988 | INFO | Perf report for the last 5.14231 seconds: | 20260508 11:44:51.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:51.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 11:44:51.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:44:51.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:51.988 | INFO | Uploaded 36GiB to GPU in 2.35941s (15.258GiB/s) | 20260508 11:44:51.988 | INFO | Downloaded 34GiB to GPU in 2.45592s (13.8441GiB/s) | 20260508 11:44:57.118 | INFO | Analyzing chunk frames = [7846912,7847424) | 20260508 11:44:57.118 | INFO | MemInfo: { | 20260508 11:44:57.118 | INFO | "gpu" : " 17 GB", | 20260508 11:44:57.118 | INFO | "huge" : "", | 20260508 11:44:57.118 | INFO | "huge_shared" : " 29 GB", | 20260508 11:44:57.118 | INFO | "malloc" : " 14 GB", | 20260508 11:44:57.118 | INFO | "pinned" : " 29 GB" | 20260508 11:44:57.118 | INFO | } | 20260508 11:44:57.118 | INFO | Perf report for the last 5.13 seconds: | 20260508 11:44:57.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:44:57.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 11:44:57.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:44:57.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:44:57.118 | INFO | Uploaded 36GiB to GPU in 2.43838s (14.7639GiB/s) | 20260508 11:44:57.118 | INFO | Downloaded 34GiB to GPU in 2.4811s (13.7036GiB/s) | 20260508 11:45:02.260 | INFO | Analyzing chunk frames = [7847424,7847936) | 20260508 11:45:02.260 | INFO | MemInfo: { | 20260508 11:45:02.260 | INFO | "gpu" : " 17 GB", | 20260508 11:45:02.261 | INFO | "huge" : "", | 20260508 11:45:02.261 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:02.261 | INFO | "malloc" : " 14 GB", | 20260508 11:45:02.261 | INFO | "pinned" : " 29 GB" | 20260508 11:45:02.261 | INFO | } | 20260508 11:45:02.261 | INFO | Perf report for the last 5.1426 seconds: | 20260508 11:45:02.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:02.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 11:45:02.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 11:45:02.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:02.261 | INFO | Uploaded 36GiB to GPU in 2.43109s (14.8082GiB/s) | 20260508 11:45:02.261 | INFO | Downloaded 34GiB to GPU in 2.46912s (13.7701GiB/s) | 20260508 11:45:02.261 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:45:02.261 | INFO | Total mem usage: 16.7 GiB | 20260508 11:45:02.261 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:45:02.261 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:45:02.261 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:45:02.261 | INFO | High Water Breakdown: | 20260508 11:45:02.261 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:45:02.261 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:45:02.261 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:45:02.261 | INFO | ------------------------------------------ | 20260508 11:45:02.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:45:02.261 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:45:02.261 | INFO | Total mem usage: 26.5 GiB | 20260508 11:45:02.261 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:45:02.261 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:45:02.261 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:45:02.261 | INFO | High Water Breakdown: | 20260508 11:45:02.261 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:45:02.261 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:45:02.261 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:45:02.261 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:45:02.261 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:45:02.261 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:45:02.261 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:45:02.261 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:45:02.261 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:45:02.261 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:45:02.261 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:45:02.261 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:45:02.261 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:45:02.261 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:45:02.261 | INFO | ------------------------------------------ | 20260508 11:45:02.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:45:02.261 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:45:02.261 | INFO | Total mem usage: 27.8 GiB | 20260508 11:45:02.261 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:45:02.261 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:45:02.261 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:45:02.261 | INFO | High Water Breakdown: | 20260508 11:45:02.261 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:45:02.261 | INFO | ------------------------------------------ | 20260508 11:45:02.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:45:02.261 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:45:02.261 | INFO | Total mem usage: 15.7 GiB | 20260508 11:45:02.261 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:45:02.261 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:45:02.261 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:45:02.261 | INFO | High Water Breakdown: | 20260508 11:45:02.261 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:45:02.261 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:45:02.261 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:45:02.261 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:45:02.261 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:45:02.261 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:45:02.261 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:45:02.261 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:45:02.261 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:45:02.261 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:45:02.261 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:45:02.261 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:45:02.261 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:45:02.261 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:45:02.261 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:45:02.261 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:45:02.261 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:45:02.261 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:45:02.261 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:45:02.261 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:45:02.261 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:45:02.261 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:45:02.261 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:45:02.261 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:45:02.262 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:45:02.262 | INFO | ------------------------------------------ | 20260508 11:45:02.262 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:45:02.262 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:45:02.262 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:45:02.262 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:45:02.262 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 5.69e-01 | 20260508 11:45:02.262 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.06e-01, 1.13e-02 / 6.95e-03 | 20260508 11:45:02.262 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.38e+00, 5.98e-01 / 2.61e-01 | 20260508 11:45:02.262 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.49e+00, 2.11e-01 / 1.16e-01 | 20260508 11:45:02.262 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.58e+01, 3.70e+00 / 1.17e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.86e+01 / 1.94e+01, 1.80e+00 / 1.05e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.56e+00, 4.01e-01 / 2.07e-01 | 20260508 11:45:02.262 | INFO | PerfReport: 2.99%, Metrics -- 9.78e-01 / 1.16e+00, 2.44e-01 / 9.08e-02 | 20260508 11:45:02.262 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.31e-01, 4.78e-02 / 1.72e-02 | 20260508 11:45:02.262 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:45:02.262 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:45:02.262 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:45:02.262 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 24.21%, Upload -- 2.54e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 24.75%, Download -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 31.10%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 2.08%, LockUpload -- 2.18e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 2.69%, LockDownload -- 2.83e+00 / 0.00e+00, 4.33e+00 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 15.15%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:45:02.262 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 3.61e-02 / 0.00e+00 | 20260508 11:45:02.358 | INFO | Starting a round of Baz IO | 20260508 11:45:05.764 | INFO | Populating RTMetrics report | 20260508 11:45:05.764 | INFO | RTMetrics report populated | 20260508 11:45:05.764 | INFO | Opening tmp RTMetrics json file | 20260508 11:45:05.775 | INFO | Serializing RTMetrics to json | 20260508 11:45:05.777 | INFO | Serialization done, swapping tmp file now... | 20260508 11:45:05.777 | INFO | RTMetrics done | 20260508 11:45:05.836 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.478 seconds | 20260508 11:45:05.836 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:45:05.836 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183388% / 0% / 5.16574e-05% | 20260508 11:45:05.836 | INFO | Baz IO overall: | 20260508 11:45:05.836 | INFO | BytesWritten : 3379553705858 | 20260508 11:45:05.836 | INFO | NumEvents : 1622725573586 | 20260508 11:45:07.399 | INFO | Analyzing chunk frames = [7847936,7848448) | 20260508 11:45:07.399 | INFO | MemInfo: { | 20260508 11:45:07.399 | INFO | "gpu" : " 17 GB", | 20260508 11:45:07.399 | INFO | "huge" : "", | 20260508 11:45:07.399 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:07.399 | INFO | "malloc" : " 14 GB", | 20260508 11:45:07.399 | INFO | "pinned" : " 29 GB" | 20260508 11:45:07.399 | INFO | } | 20260508 11:45:07.399 | INFO | Perf report for the last 5.13847 seconds: | 20260508 11:45:07.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:07.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:07.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:07.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:07.399 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:45:07.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:07.399 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:45:07.399 | INFO | PRE_HQ : 65.4%, 1.6, 54.4ms, 0 | 20260508 11:45:07.399 | INFO | BAZWRITER : 66.9%, 2.5, 85.6ms, 0 | 20260508 11:45:07.399 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:45:07.400 | INFO | Uploaded 36GiB to GPU in 2.61293s (13.7777GiB/s) | 20260508 11:45:07.400 | INFO | Downloaded 38GiB to GPU in 2.72509s (13.9445GiB/s) | 20260508 11:45:12.540 | INFO | Analyzing chunk frames = [7848448,7848960) | 20260508 11:45:12.540 | INFO | MemInfo: { | 20260508 11:45:12.540 | INFO | "gpu" : " 17 GB", | 20260508 11:45:12.540 | INFO | "huge" : "", | 20260508 11:45:12.540 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:12.540 | INFO | "malloc" : " 14 GB", | 20260508 11:45:12.540 | INFO | "pinned" : " 29 GB" | 20260508 11:45:12.540 | INFO | } | 20260508 11:45:12.540 | INFO | Perf report for the last 5.14075 seconds: | 20260508 11:45:12.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:12.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 11:45:12.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:45:12.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:12.540 | INFO | Uploaded 36GiB to GPU in 2.41155s (14.9282GiB/s) | 20260508 11:45:12.540 | INFO | Downloaded 34GiB to GPU in 2.47924s (13.7139GiB/s) | 20260508 11:45:17.678 | INFO | Analyzing chunk frames = [7848960,7849472) | 20260508 11:45:17.678 | INFO | MemInfo: { | 20260508 11:45:17.678 | INFO | "gpu" : " 17 GB", | 20260508 11:45:17.678 | INFO | "huge" : "", | 20260508 11:45:17.678 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:17.678 | INFO | "malloc" : " 14 GB", | 20260508 11:45:17.678 | INFO | "pinned" : " 29 GB" | 20260508 11:45:17.678 | INFO | } | 20260508 11:45:17.678 | INFO | Perf report for the last 5.13783 seconds: | 20260508 11:45:17.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:17.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:45:17.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:45:17.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:17.678 | INFO | Uploaded 36GiB to GPU in 2.44878s (14.7012GiB/s) | 20260508 11:45:17.678 | INFO | Downloaded 34GiB to GPU in 2.46159s (13.8122GiB/s) | 20260508 11:45:22.820 | INFO | Analyzing chunk frames = [7849472,7849984) | 20260508 11:45:22.820 | INFO | MemInfo: { | 20260508 11:45:22.820 | INFO | "gpu" : " 17 GB", | 20260508 11:45:22.820 | INFO | "huge" : "", | 20260508 11:45:22.820 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:22.820 | INFO | "malloc" : " 14 GB", | 20260508 11:45:22.820 | INFO | "pinned" : " 29 GB" | 20260508 11:45:22.821 | INFO | } | 20260508 11:45:22.821 | INFO | Perf report for the last 5.14276 seconds: | 20260508 11:45:22.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:22.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:45:22.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:45:22.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:22.821 | INFO | Uploaded 36GiB to GPU in 2.46814s (14.5859GiB/s) | 20260508 11:45:22.821 | INFO | Downloaded 34GiB to GPU in 2.46779s (13.7775GiB/s) | 20260508 11:45:27.962 | INFO | Analyzing chunk frames = [7849984,7850496) | 20260508 11:45:27.962 | INFO | MemInfo: { | 20260508 11:45:27.962 | INFO | "gpu" : " 17 GB", | 20260508 11:45:27.962 | INFO | "huge" : "", | 20260508 11:45:27.962 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:27.962 | INFO | "malloc" : " 14 GB", | 20260508 11:45:27.962 | INFO | "pinned" : " 29 GB" | 20260508 11:45:27.962 | INFO | } | 20260508 11:45:27.962 | INFO | Perf report for the last 5.14186 seconds: | 20260508 11:45:27.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:27.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:45:27.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:45:27.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:27.963 | INFO | Uploaded 36GiB to GPU in 2.27699s (15.8103GiB/s) | 20260508 11:45:27.963 | INFO | Downloaded 34GiB to GPU in 2.49423s (13.6314GiB/s) | 20260508 11:45:33.097 | INFO | Analyzing chunk frames = [7850496,7851008) | 20260508 11:45:33.097 | INFO | MemInfo: { | 20260508 11:45:33.097 | INFO | "gpu" : " 17 GB", | 20260508 11:45:33.097 | INFO | "huge" : "", | 20260508 11:45:33.097 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:33.097 | INFO | "malloc" : " 14 GB", | 20260508 11:45:33.097 | INFO | "pinned" : " 29 GB" | 20260508 11:45:33.097 | INFO | } | 20260508 11:45:33.097 | INFO | Perf report for the last 5.13498 seconds: | 20260508 11:45:33.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:33.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.098 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 11:45:33.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.098 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 11:45:33.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:33.098 | INFO | Uploaded 36GiB to GPU in 2.40596s (14.9628GiB/s) | 20260508 11:45:33.098 | INFO | Downloaded 34GiB to GPU in 2.45808s (13.8319GiB/s) | 20260508 11:45:38.234 | INFO | Analyzing chunk frames = [7851008,7851520) | 20260508 11:45:38.234 | INFO | MemInfo: { | 20260508 11:45:38.234 | INFO | "gpu" : " 17 GB", | 20260508 11:45:38.234 | INFO | "huge" : "", | 20260508 11:45:38.234 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:38.234 | INFO | "malloc" : " 14 GB", | 20260508 11:45:38.235 | INFO | "pinned" : " 29 GB" | 20260508 11:45:38.235 | INFO | } | 20260508 11:45:38.235 | INFO | Perf report for the last 5.1372 seconds: | 20260508 11:45:38.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:38.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:45:38.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:45:38.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:38.235 | INFO | Uploaded 36GiB to GPU in 2.42119s (14.8687GiB/s) | 20260508 11:45:38.235 | INFO | Downloaded 34GiB to GPU in 2.46725s (13.7805GiB/s) | 20260508 11:45:43.373 | INFO | Analyzing chunk frames = [7851520,7852032) | 20260508 11:45:43.373 | INFO | MemInfo: { | 20260508 11:45:43.373 | INFO | "gpu" : " 17 GB", | 20260508 11:45:43.373 | INFO | "huge" : "", | 20260508 11:45:43.373 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:43.373 | INFO | "malloc" : " 14 GB", | 20260508 11:45:43.374 | INFO | "pinned" : " 29 GB" | 20260508 11:45:43.374 | INFO | } | 20260508 11:45:43.374 | INFO | Perf report for the last 5.13896 seconds: | 20260508 11:45:43.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:43.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 11:45:43.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:45:43.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:43.374 | INFO | Uploaded 36GiB to GPU in 2.4581s (14.6455GiB/s) | 20260508 11:45:43.374 | INFO | Downloaded 34GiB to GPU in 2.487s (13.6711GiB/s) | 20260508 11:45:43.471 | INFO | Starting a round of Baz IO | 20260508 11:45:46.882 | INFO | Populating RTMetrics report | 20260508 11:45:46.882 | INFO | RTMetrics report populated | 20260508 11:45:46.882 | INFO | Opening tmp RTMetrics json file | 20260508 11:45:46.883 | INFO | Serializing RTMetrics to json | 20260508 11:45:46.884 | INFO | Serialization done, swapping tmp file now... | 20260508 11:45:46.884 | INFO | RTMetrics done | 20260508 11:45:46.945 | INFO | Finished a round of Baz IO: Wrote 1.427 GiB in 3.473 seconds | 20260508 11:45:46.945 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:45:46.945 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182406% / 3.97364e-06% / 5.5631e-05% | 20260508 11:45:46.945 | INFO | Baz IO overall: | 20260508 11:45:46.945 | INFO | BytesWritten : 3381085775628 | 20260508 11:45:46.945 | INFO | NumEvents : 1623445923570 | 20260508 11:45:48.514 | INFO | Analyzing chunk frames = [7852032,7852544) | 20260508 11:45:48.514 | INFO | MemInfo: { | 20260508 11:45:48.514 | INFO | "gpu" : " 17 GB", | 20260508 11:45:48.514 | INFO | "huge" : "", | 20260508 11:45:48.514 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:48.514 | INFO | "malloc" : " 14 GB", | 20260508 11:45:48.514 | INFO | "pinned" : " 29 GB" | 20260508 11:45:48.514 | INFO | } | 20260508 11:45:48.514 | INFO | Perf report for the last 5.14057 seconds: | 20260508 11:45:48.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:48.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:48.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:48.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:48.514 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 11:45:48.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:48.514 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:45:48.514 | INFO | PRE_HQ : 65.6%, 1.6, 54.4ms, 0 | 20260508 11:45:48.514 | INFO | BAZWRITER : 66.8%, 2.5, 85.2ms, 0 | 20260508 11:45:48.514 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 11:45:48.515 | INFO | Uploaded 36GiB to GPU in 2.54267s (14.1583GiB/s) | 20260508 11:45:48.515 | INFO | Downloaded 38GiB to GPU in 2.71791s (13.9813GiB/s) | 20260508 11:45:53.655 | INFO | Analyzing chunk frames = [7852544,7853056) | 20260508 11:45:53.655 | INFO | MemInfo: { | 20260508 11:45:53.655 | INFO | "gpu" : " 17 GB", | 20260508 11:45:53.655 | INFO | "huge" : "", | 20260508 11:45:53.655 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:53.655 | INFO | "malloc" : " 14 GB", | 20260508 11:45:53.655 | INFO | "pinned" : " 29 GB" | 20260508 11:45:53.655 | INFO | } | 20260508 11:45:53.655 | INFO | Perf report for the last 5.14061 seconds: | 20260508 11:45:53.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:53.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.655 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:45:53.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.655 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:45:53.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:53.656 | INFO | Uploaded 36GiB to GPU in 2.45264s (14.6781GiB/s) | 20260508 11:45:53.656 | INFO | Downloaded 34GiB to GPU in 2.4892s (13.659GiB/s) | 20260508 11:45:58.795 | INFO | Analyzing chunk frames = [7853056,7853568) | 20260508 11:45:58.795 | INFO | MemInfo: { | 20260508 11:45:58.795 | INFO | "gpu" : " 17 GB", | 20260508 11:45:58.795 | INFO | "huge" : "", | 20260508 11:45:58.795 | INFO | "huge_shared" : " 29 GB", | 20260508 11:45:58.795 | INFO | "malloc" : " 14 GB", | 20260508 11:45:58.795 | INFO | "pinned" : " 29 GB" | 20260508 11:45:58.795 | INFO | } | 20260508 11:45:58.795 | INFO | Perf report for the last 5.14043 seconds: | 20260508 11:45:58.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:45:58.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.795 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 11:45:58.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.795 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:45:58.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:45:58.796 | INFO | Uploaded 36GiB to GPU in 2.44039s (14.7517GiB/s) | 20260508 11:45:58.796 | INFO | Downloaded 34GiB to GPU in 2.45618s (13.8426GiB/s) | 20260508 11:46:03.932 | INFO | Analyzing chunk frames = [7853568,7854080) | 20260508 11:46:03.932 | INFO | MemInfo: { | 20260508 11:46:03.932 | INFO | "gpu" : " 17 GB", | 20260508 11:46:03.932 | INFO | "huge" : "", | 20260508 11:46:03.932 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:03.932 | INFO | "malloc" : " 14 GB", | 20260508 11:46:03.932 | INFO | "pinned" : " 29 GB" | 20260508 11:46:03.932 | INFO | } | 20260508 11:46:03.932 | INFO | Perf report for the last 5.13704 seconds: | 20260508 11:46:03.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:03.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.932 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:46:03.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.932 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:46:03.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:03.933 | INFO | Uploaded 36GiB to GPU in 2.44872s (14.7016GiB/s) | 20260508 11:46:03.933 | INFO | Downloaded 34GiB to GPU in 2.46214s (13.8091GiB/s) | 20260508 11:46:09.071 | INFO | Analyzing chunk frames = [7854080,7854592) | 20260508 11:46:09.071 | INFO | MemInfo: { | 20260508 11:46:09.071 | INFO | "gpu" : " 17 GB", | 20260508 11:46:09.072 | INFO | "huge" : "", | 20260508 11:46:09.072 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:09.072 | INFO | "malloc" : " 14 GB", | 20260508 11:46:09.072 | INFO | "pinned" : " 29 GB" | 20260508 11:46:09.072 | INFO | } | 20260508 11:46:09.072 | INFO | Perf report for the last 5.13943 seconds: | 20260508 11:46:09.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:09.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:46:09.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 11:46:09.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:09.072 | INFO | Uploaded 36GiB to GPU in 2.25428s (15.9697GiB/s) | 20260508 11:46:09.072 | INFO | Downloaded 34GiB to GPU in 2.49797s (13.611GiB/s) | 20260508 11:46:14.214 | INFO | Analyzing chunk frames = [7854592,7855104) | 20260508 11:46:14.214 | INFO | MemInfo: { | 20260508 11:46:14.214 | INFO | "gpu" : " 17 GB", | 20260508 11:46:14.214 | INFO | "huge" : "", | 20260508 11:46:14.214 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:14.214 | INFO | "malloc" : " 14 GB", | 20260508 11:46:14.214 | INFO | "pinned" : " 29 GB" | 20260508 11:46:14.214 | INFO | } | 20260508 11:46:14.215 | INFO | Perf report for the last 5.1428 seconds: | 20260508 11:46:14.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:14.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:46:14.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:46:14.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:14.215 | INFO | Uploaded 36GiB to GPU in 2.37706s (15.1447GiB/s) | 20260508 11:46:14.215 | INFO | Downloaded 34GiB to GPU in 2.43315s (13.9737GiB/s) | 20260508 11:46:19.351 | INFO | Analyzing chunk frames = [7855104,7855616) | 20260508 11:46:19.351 | INFO | MemInfo: { | 20260508 11:46:19.351 | INFO | "gpu" : " 17 GB", | 20260508 11:46:19.351 | INFO | "huge" : "", | 20260508 11:46:19.351 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:19.351 | INFO | "malloc" : " 14 GB", | 20260508 11:46:19.351 | INFO | "pinned" : " 29 GB" | 20260508 11:46:19.351 | INFO | } | 20260508 11:46:19.351 | INFO | Perf report for the last 5.13669 seconds: | 20260508 11:46:19.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:19.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.351 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 11:46:19.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.351 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:46:19.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:19.352 | INFO | Uploaded 36GiB to GPU in 2.43191s (14.8032GiB/s) | 20260508 11:46:19.352 | INFO | Downloaded 34GiB to GPU in 2.45844s (13.8299GiB/s) | 20260508 11:46:24.492 | INFO | Analyzing chunk frames = [7855616,7856128) | 20260508 11:46:24.492 | INFO | MemInfo: { | 20260508 11:46:24.492 | INFO | "gpu" : " 17 GB", | 20260508 11:46:24.492 | INFO | "huge" : "", | 20260508 11:46:24.492 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:24.493 | INFO | "malloc" : " 14 GB", | 20260508 11:46:24.493 | INFO | "pinned" : " 29 GB" | 20260508 11:46:24.493 | INFO | } | 20260508 11:46:24.493 | INFO | Perf report for the last 5.14138 seconds: | 20260508 11:46:24.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:24.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 11:46:24.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:46:24.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:24.493 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:46:24.493 | INFO | Total mem usage: 16.7 GiB | 20260508 11:46:24.493 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:46:24.493 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.8%) | 20260508 11:46:24.493 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:46:24.493 | INFO | High Water Breakdown: | 20260508 11:46:24.493 | INFO | 59.6% : 8.52 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:46:24.493 | INFO | 40.3% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:46:24.493 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:46:24.493 | INFO | ------------------------------------------ | 20260508 11:46:24.493 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:46:24.493 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:46:24.493 | INFO | Uploaded 36GiB to GPU in 2.43642s (14.7758GiB/s) | 20260508 11:46:24.493 | INFO | Downloaded 34GiB to GPU in 2.44885s (13.8841GiB/s) | 20260508 11:46:24.493 | INFO | Total mem usage: 26.5 GiB | 20260508 11:46:24.493 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:46:24.493 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:46:24.493 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:46:24.494 | INFO | High Water Breakdown: | 20260508 11:46:24.494 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:46:24.494 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:46:24.494 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:46:24.494 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:46:24.494 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:46:24.494 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:46:24.494 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:46:24.494 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:46:24.494 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:46:24.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:46:24.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:46:24.494 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:46:24.494 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:46:24.494 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:46:24.494 | INFO | ------------------------------------------ | 20260508 11:46:24.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:46:24.494 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:46:24.494 | INFO | Total mem usage: 27.8 GiB | 20260508 11:46:24.494 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:46:24.494 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:46:24.494 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:46:24.494 | INFO | High Water Breakdown: | 20260508 11:46:24.494 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:46:24.494 | INFO | ------------------------------------------ | 20260508 11:46:24.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:46:24.494 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:46:24.494 | INFO | Total mem usage: 15.7 GiB | 20260508 11:46:24.494 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:46:24.494 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:46:24.494 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:46:24.494 | INFO | High Water Breakdown: | 20260508 11:46:24.494 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:46:24.494 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:46:24.494 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:46:24.494 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:46:24.494 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:46:24.494 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:46:24.494 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:46:24.494 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:46:24.494 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:46:24.494 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:46:24.494 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:46:24.494 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:46:24.494 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:46:24.494 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:46:24.494 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:46:24.494 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:46:24.494 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:46:24.494 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:46:24.494 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:46:24.494 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:46:24.494 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:46:24.494 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:46:24.494 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:46:24.494 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:46:24.495 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:46:24.495 | INFO | ------------------------------------------ | 20260508 11:46:24.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:46:24.495 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:46:24.495 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:46:24.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:46:24.495 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.41e+01 / 7.26e-01 | 20260508 11:46:24.495 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.08e-02 / 4.14e-03 | 20260508 11:46:24.495 | INFO | PerfReport: 12.22%, Baseline -- 3.98e+00 / 4.45e+00, 5.94e-01 / 1.73e-01 | 20260508 11:46:24.495 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.07e-01 / 1.17e-01 | 20260508 11:46:24.495 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.57e+01, 3.70e+00 / 1.20e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.77e+00 / 1.33e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.34e+00 / 2.55e+00, 4.05e-01 / 2.25e-01 | 20260508 11:46:24.495 | INFO | PerfReport: 2.99%, Metrics -- 9.75e-01 / 1.12e+00, 2.43e-01 / 8.26e-02 | 20260508 11:46:24.495 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.21e-01, 4.86e-02 / 1.59e-02 | 20260508 11:46:24.495 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:46:24.495 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:46:24.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:46:24.495 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 24.11%, Upload -- 2.53e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 24.80%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 1.98%, LockUpload -- 2.08e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 2.79%, LockDownload -- 2.93e+00 / 0.00e+00, 4.42e+00 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 15.25%, LockCompute -- 1.60e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:46:24.495 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 11:46:24.591 | INFO | Starting a round of Baz IO | 20260508 11:46:27.991 | INFO | Populating RTMetrics report | 20260508 11:46:27.991 | INFO | RTMetrics report populated | 20260508 11:46:27.991 | INFO | Opening tmp RTMetrics json file | 20260508 11:46:28.002 | INFO | Serializing RTMetrics to json | 20260508 11:46:28.004 | INFO | Serialization done, swapping tmp file now... | 20260508 11:46:28.004 | INFO | RTMetrics done | 20260508 11:46:28.065 | INFO | Finished a round of Baz IO: Wrote 1.426 GiB in 3.474 seconds | 20260508 11:46:28.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:46:28.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.181925% / 3.97364e-06% / 6.35783e-05% | 20260508 11:46:28.065 | INFO | Baz IO overall: | 20260508 11:46:28.065 | INFO | BytesWritten : 3382617279709 | 20260508 11:46:28.065 | INFO | NumEvents : 1624165792004 | 20260508 11:46:29.627 | INFO | Analyzing chunk frames = [7856128,7856640) | 20260508 11:46:29.627 | INFO | MemInfo: { | 20260508 11:46:29.627 | INFO | "gpu" : " 17 GB", | 20260508 11:46:29.627 | INFO | "huge" : "", | 20260508 11:46:29.627 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:29.627 | INFO | "malloc" : " 14 GB", | 20260508 11:46:29.627 | INFO | "pinned" : " 29 GB" | 20260508 11:46:29.627 | INFO | } | 20260508 11:46:29.627 | INFO | Perf report for the last 5.1348 seconds: | 20260508 11:46:29.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:29.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:29.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:29.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:29.628 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 11:46:29.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:29.628 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:46:29.628 | INFO | PRE_HQ : 65.2%, 1.6, 54.0ms, 0 | 20260508 11:46:29.628 | INFO | BAZWRITER : 66.9%, 2.5, 85.8ms, 0 | 20260508 11:46:29.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:46:29.628 | INFO | Uploaded 36GiB to GPU in 2.62809s (13.6982GiB/s) | 20260508 11:46:29.628 | INFO | Downloaded 38GiB to GPU in 2.71339s (14.0046GiB/s) | 20260508 11:46:34.773 | INFO | Analyzing chunk frames = [7856640,7857152) | 20260508 11:46:34.773 | INFO | MemInfo: { | 20260508 11:46:34.773 | INFO | "gpu" : " 17 GB", | 20260508 11:46:34.773 | INFO | "huge" : "", | 20260508 11:46:34.773 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:34.773 | INFO | "malloc" : " 14 GB", | 20260508 11:46:34.773 | INFO | "pinned" : " 29 GB" | 20260508 11:46:34.773 | INFO | } | 20260508 11:46:34.773 | INFO | Perf report for the last 5.14566 seconds: | 20260508 11:46:34.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:34.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.773 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:46:34.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.773 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:46:34.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:34.774 | INFO | Uploaded 36GiB to GPU in 2.45938s (14.6378GiB/s) | 20260508 11:46:34.774 | INFO | Downloaded 34GiB to GPU in 2.47067s (13.7615GiB/s) | 20260508 11:46:39.915 | INFO | Analyzing chunk frames = [7857152,7857664) | 20260508 11:46:39.915 | INFO | MemInfo: { | 20260508 11:46:39.915 | INFO | "gpu" : " 17 GB", | 20260508 11:46:39.915 | INFO | "huge" : "", | 20260508 11:46:39.915 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:39.915 | INFO | "malloc" : " 14 GB", | 20260508 11:46:39.915 | INFO | "pinned" : " 29 GB" | 20260508 11:46:39.915 | INFO | } | 20260508 11:46:39.915 | INFO | Perf report for the last 5.14217 seconds: | 20260508 11:46:39.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:39.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.915 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 11:46:39.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.916 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 11:46:39.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:39.916 | INFO | Uploaded 36GiB to GPU in 2.49684s (14.4182GiB/s) | 20260508 11:46:39.916 | INFO | Downloaded 34GiB to GPU in 2.46502s (13.793GiB/s) | 20260508 11:46:45.048 | INFO | Analyzing chunk frames = [7857664,7858176) | 20260508 11:46:45.048 | INFO | MemInfo: { | 20260508 11:46:45.048 | INFO | "gpu" : " 17 GB", | 20260508 11:46:45.048 | INFO | "huge" : "", | 20260508 11:46:45.048 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:45.048 | INFO | "malloc" : " 14 GB", | 20260508 11:46:45.048 | INFO | "pinned" : " 29 GB" | 20260508 11:46:45.048 | INFO | } | 20260508 11:46:45.048 | INFO | Perf report for the last 5.13304 seconds: | 20260508 11:46:45.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:45.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.049 | INFO | ANALYSIS : 65.6%, 3.0, 101.7ms, 0 | 20260508 11:46:45.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.049 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260508 11:46:45.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:45.049 | INFO | Uploaded 36GiB to GPU in 2.41509s (14.9063GiB/s) | 20260508 11:46:45.049 | INFO | Downloaded 34GiB to GPU in 2.4753s (13.7357GiB/s) | 20260508 11:46:50.184 | INFO | Analyzing chunk frames = [7858176,7858688) | 20260508 11:46:50.184 | INFO | MemInfo: { | 20260508 11:46:50.184 | INFO | "gpu" : " 17 GB", | 20260508 11:46:50.184 | INFO | "huge" : "", | 20260508 11:46:50.184 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:50.184 | INFO | "malloc" : " 14 GB", | 20260508 11:46:50.184 | INFO | "pinned" : " 29 GB" | 20260508 11:46:50.184 | INFO | } | 20260508 11:46:50.184 | INFO | Perf report for the last 5.13591 seconds: | 20260508 11:46:50.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:50.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.184 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:46:50.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.184 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260508 11:46:50.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:50.185 | INFO | Uploaded 36GiB to GPU in 2.28236s (15.7731GiB/s) | 20260508 11:46:50.185 | INFO | Downloaded 34GiB to GPU in 2.49105s (13.6488GiB/s) | 20260508 11:46:55.323 | INFO | Analyzing chunk frames = [7858688,7859200) | 20260508 11:46:55.323 | INFO | MemInfo: { | 20260508 11:46:55.323 | INFO | "gpu" : " 17 GB", | 20260508 11:46:55.323 | INFO | "huge" : "", | 20260508 11:46:55.323 | INFO | "huge_shared" : " 29 GB", | 20260508 11:46:55.323 | INFO | "malloc" : " 14 GB", | 20260508 11:46:55.323 | INFO | "pinned" : " 29 GB" | 20260508 11:46:55.323 | INFO | } | 20260508 11:46:55.323 | INFO | Perf report for the last 5.13884 seconds: | 20260508 11:46:55.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:46:55.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.323 | INFO | ANALYSIS : 65.1%, 3.0, 101.3ms, 0 | 20260508 11:46:55.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.323 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:46:55.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:46:55.324 | INFO | Uploaded 36GiB to GPU in 2.35294s (15.3GiB/s) | 20260508 11:46:55.324 | INFO | Downloaded 34GiB to GPU in 2.44558s (13.9026GiB/s) | 20260508 11:47:00.464 | INFO | Analyzing chunk frames = [7859200,7859712) | 20260508 11:47:00.464 | INFO | MemInfo: { | 20260508 11:47:00.464 | INFO | "gpu" : " 17 GB", | 20260508 11:47:00.464 | INFO | "huge" : "", | 20260508 11:47:00.464 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:00.464 | INFO | "malloc" : " 14 GB", | 20260508 11:47:00.464 | INFO | "pinned" : " 29 GB" | 20260508 11:47:00.464 | INFO | } | 20260508 11:47:00.464 | INFO | Perf report for the last 5.14063 seconds: | 20260508 11:47:00.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:00.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 11:47:00.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:47:00.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:00.464 | INFO | Uploaded 36GiB to GPU in 2.39127s (15.0547GiB/s) | 20260508 11:47:00.464 | INFO | Downloaded 34GiB to GPU in 2.45924s (13.8254GiB/s) | 20260508 11:47:05.606 | INFO | Analyzing chunk frames = [7859712,7860224) | 20260508 11:47:05.606 | INFO | MemInfo: { | 20260508 11:47:05.606 | INFO | "gpu" : " 17 GB", | 20260508 11:47:05.606 | INFO | "huge" : "", | 20260508 11:47:05.606 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:05.606 | INFO | "malloc" : " 14 GB", | 20260508 11:47:05.606 | INFO | "pinned" : " 29 GB" | 20260508 11:47:05.606 | INFO | } | 20260508 11:47:05.606 | INFO | Perf report for the last 5.14246 seconds: | 20260508 11:47:05.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:05.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.606 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 11:47:05.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.606 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:47:05.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:05.607 | INFO | Uploaded 36GiB to GPU in 2.41976s (14.8775GiB/s) | 20260508 11:47:05.607 | INFO | Downloaded 34GiB to GPU in 2.46133s (13.8137GiB/s) | 20260508 11:47:05.703 | INFO | Starting a round of Baz IO | 20260508 11:47:09.112 | INFO | Populating RTMetrics report | 20260508 11:47:09.113 | INFO | RTMetrics report populated | 20260508 11:47:09.113 | INFO | Opening tmp RTMetrics json file | 20260508 11:47:09.113 | INFO | Serializing RTMetrics to json | 20260508 11:47:09.115 | INFO | Serialization done, swapping tmp file now... | 20260508 11:47:09.115 | INFO | RTMetrics done | 20260508 11:47:09.174 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.47 seconds | 20260508 11:47:09.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:47:09.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182283% / 3.97364e-06% / 5.16574e-05% | 20260508 11:47:09.174 | INFO | Baz IO overall: | 20260508 11:47:09.174 | INFO | BytesWritten : 3384150681164 | 20260508 11:47:09.174 | INFO | NumEvents : 1624886848729 | 20260508 11:47:10.748 | INFO | Analyzing chunk frames = [7860224,7860736) | 20260508 11:47:10.748 | INFO | MemInfo: { | 20260508 11:47:10.748 | INFO | "gpu" : " 17 GB", | 20260508 11:47:10.748 | INFO | "huge" : "", | 20260508 11:47:10.748 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:10.749 | INFO | "malloc" : " 14 GB", | 20260508 11:47:10.749 | INFO | "pinned" : " 29 GB" | 20260508 11:47:10.749 | INFO | } | 20260508 11:47:10.749 | INFO | Perf report for the last 5.14227 seconds: | 20260508 11:47:10.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:10.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:10.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:10.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:10.749 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 11:47:10.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:10.749 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 11:47:10.749 | INFO | PRE_HQ : 65.6%, 1.6, 53.8ms, 0 | 20260508 11:47:10.749 | INFO | BAZWRITER : 66.8%, 2.5, 85.7ms, 0 | 20260508 11:47:10.749 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 11:47:10.749 | INFO | Uploaded 36GiB to GPU in 2.52895s (14.2351GiB/s) | 20260508 11:47:10.749 | INFO | Downloaded 38GiB to GPU in 2.70482s (14.049GiB/s) | 20260508 11:47:15.880 | INFO | Analyzing chunk frames = [7860736,7861248) | 20260508 11:47:15.880 | INFO | MemInfo: { | 20260508 11:47:15.880 | INFO | "gpu" : " 17 GB", | 20260508 11:47:15.880 | INFO | "huge" : "", | 20260508 11:47:15.880 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:15.880 | INFO | "malloc" : " 14 GB", | 20260508 11:47:15.880 | INFO | "pinned" : " 29 GB" | 20260508 11:47:15.880 | INFO | } | 20260508 11:47:15.880 | INFO | Perf report for the last 5.1318 seconds: | 20260508 11:47:15.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:15.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.880 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 11:47:15.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.881 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:47:15.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:15.881 | INFO | Uploaded 36GiB to GPU in 2.4072s (14.9551GiB/s) | 20260508 11:47:15.881 | INFO | Downloaded 34GiB to GPU in 2.47784s (13.7216GiB/s) | 20260508 11:47:21.024 | INFO | Analyzing chunk frames = [7861248,7861760) | 20260508 11:47:21.024 | INFO | MemInfo: { | 20260508 11:47:21.024 | INFO | "gpu" : " 17 GB", | 20260508 11:47:21.024 | INFO | "huge" : "", | 20260508 11:47:21.024 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:21.024 | INFO | "malloc" : " 14 GB", | 20260508 11:47:21.024 | INFO | "pinned" : " 29 GB" | 20260508 11:47:21.024 | INFO | } | 20260508 11:47:21.024 | INFO | Perf report for the last 5.14364 seconds: | 20260508 11:47:21.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:21.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 11:47:21.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:47:21.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:21.024 | INFO | Uploaded 36GiB to GPU in 2.4837s (14.4945GiB/s) | 20260508 11:47:21.024 | INFO | Downloaded 34GiB to GPU in 2.45255s (13.8631GiB/s) | 20260508 11:47:26.162 | INFO | Analyzing chunk frames = [7861760,7862272) | 20260508 11:47:26.162 | INFO | MemInfo: { | 20260508 11:47:26.162 | INFO | "gpu" : " 17 GB", | 20260508 11:47:26.162 | INFO | "huge" : "", | 20260508 11:47:26.162 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:26.162 | INFO | "malloc" : " 14 GB", | 20260508 11:47:26.162 | INFO | "pinned" : " 29 GB" | 20260508 11:47:26.162 | INFO | } | 20260508 11:47:26.162 | INFO | Perf report for the last 5.13781 seconds: | 20260508 11:47:26.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:26.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 11:47:26.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:47:26.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:26.162 | INFO | Uploaded 36GiB to GPU in 2.45777s (14.6474GiB/s) | 20260508 11:47:26.162 | INFO | Downloaded 34GiB to GPU in 2.46937s (13.7687GiB/s) | 20260508 11:47:31.302 | INFO | Analyzing chunk frames = [7862272,7862784) | 20260508 11:47:31.302 | INFO | MemInfo: { | 20260508 11:47:31.302 | INFO | "gpu" : " 17 GB", | 20260508 11:47:31.302 | INFO | "huge" : "", | 20260508 11:47:31.302 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:31.302 | INFO | "malloc" : " 14 GB", | 20260508 11:47:31.302 | INFO | "pinned" : " 29 GB" | 20260508 11:47:31.302 | INFO | } | 20260508 11:47:31.302 | INFO | Perf report for the last 5.14004 seconds: | 20260508 11:47:31.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:31.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 11:47:31.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:47:31.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:31.302 | INFO | Uploaded 36GiB to GPU in 2.40042s (14.9974GiB/s) | 20260508 11:47:31.302 | INFO | Downloaded 34GiB to GPU in 2.55916s (13.2856GiB/s) | 20260508 11:47:36.442 | INFO | Analyzing chunk frames = [7862784,7863296) | 20260508 11:47:36.442 | INFO | MemInfo: { | 20260508 11:47:36.442 | INFO | "gpu" : " 17 GB", | 20260508 11:47:36.442 | INFO | "huge" : "", | 20260508 11:47:36.442 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:36.442 | INFO | "malloc" : " 14 GB", | 20260508 11:47:36.442 | INFO | "pinned" : " 29 GB" | 20260508 11:47:36.442 | INFO | } | 20260508 11:47:36.442 | INFO | Perf report for the last 5.14011 seconds: | 20260508 11:47:36.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:36.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 11:47:36.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:47:36.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:36.442 | INFO | Uploaded 36GiB to GPU in 2.35201s (15.306GiB/s) | 20260508 11:47:36.442 | INFO | Downloaded 34GiB to GPU in 2.44504s (13.9057GiB/s) | 20260508 11:47:41.583 | INFO | Analyzing chunk frames = [7863296,7863808) | 20260508 11:47:41.583 | INFO | MemInfo: { | 20260508 11:47:41.583 | INFO | "gpu" : " 17 GB", | 20260508 11:47:41.583 | INFO | "huge" : "", | 20260508 11:47:41.583 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:41.583 | INFO | "malloc" : " 14 GB", | 20260508 11:47:41.583 | INFO | "pinned" : " 29 GB" | 20260508 11:47:41.583 | INFO | } | 20260508 11:47:41.583 | INFO | Perf report for the last 5.14079 seconds: | 20260508 11:47:41.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:41.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 11:47:41.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:47:41.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:41.583 | INFO | Uploaded 36GiB to GPU in 2.39114s (15.0556GiB/s) | 20260508 11:47:41.583 | INFO | Downloaded 34GiB to GPU in 2.46964s (13.7672GiB/s) | 20260508 11:47:46.719 | INFO | Analyzing chunk frames = [7863808,7864320) | 20260508 11:47:46.719 | INFO | MemInfo: { | 20260508 11:47:46.719 | INFO | "gpu" : " 17 GB", | 20260508 11:47:46.719 | INFO | "huge" : "", | 20260508 11:47:46.719 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:46.719 | INFO | "malloc" : " 14 GB", | 20260508 11:47:46.719 | INFO | "pinned" : " 29 GB" | 20260508 11:47:46.719 | INFO | } | 20260508 11:47:46.719 | INFO | Perf report for the last 5.13626 seconds: | 20260508 11:47:46.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:46.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:47:46.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 11:47:46.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:46.719 | INFO | Uploaded 36GiB to GPU in 2.43547s (14.7815GiB/s) | 20260508 11:47:46.719 | INFO | Downloaded 34GiB to GPU in 2.4739s (13.7435GiB/s) | 20260508 11:47:46.720 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:47:46.720 | INFO | Total mem usage: 16.7 GiB | 20260508 11:47:46.720 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:47:46.720 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 11:47:46.720 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:47:46.720 | INFO | High Water Breakdown: | 20260508 11:47:46.720 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:47:46.720 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:47:46.720 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:47:46.720 | INFO | ------------------------------------------ | 20260508 11:47:46.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:47:46.720 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:47:46.720 | INFO | Total mem usage: 26.5 GiB | 20260508 11:47:46.720 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:47:46.720 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:47:46.720 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:47:46.720 | INFO | High Water Breakdown: | 20260508 11:47:46.720 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:47:46.720 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:47:46.720 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:47:46.720 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:47:46.720 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:47:46.720 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:47:46.720 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:47:46.720 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:47:46.720 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:47:46.720 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:47:46.720 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:47:46.720 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:47:46.720 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:47:46.720 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:47:46.720 | INFO | ------------------------------------------ | 20260508 11:47:46.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:47:46.720 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:47:46.720 | INFO | Total mem usage: 27.8 GiB | 20260508 11:47:46.720 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:47:46.720 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:47:46.720 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:47:46.720 | INFO | High Water Breakdown: | 20260508 11:47:46.720 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:47:46.720 | INFO | ------------------------------------------ | 20260508 11:47:46.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:47:46.720 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:47:46.720 | INFO | Total mem usage: 15.7 GiB | 20260508 11:47:46.720 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:47:46.720 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:47:46.720 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:47:46.720 | INFO | High Water Breakdown: | 20260508 11:47:46.720 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:47:46.720 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:47:46.720 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:47:46.720 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:47:46.720 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:47:46.720 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:47:46.720 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:47:46.720 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:47:46.720 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:47:46.720 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:47:46.720 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:47:46.720 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:47:46.720 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:47:46.720 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:47:46.720 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:47:46.720 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:47:46.720 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:47:46.720 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:47:46.720 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:47:46.720 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:47:46.720 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:47:46.721 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:47:46.721 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:47:46.721 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:47:46.721 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:47:46.721 | INFO | ------------------------------------------ | 20260508 11:47:46.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:47:46.721 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:47:46.721 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:47:46.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:47:46.721 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.39e+01 / 5.48e-01 | 20260508 11:47:46.721 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.04e-01, 1.18e-02 / 4.51e-03 | 20260508 11:47:46.721 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.40e+00, 5.93e-01 / 2.17e-01 | 20260508 11:47:46.721 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.43e+00, 2.10e-01 / 9.91e-02 | 20260508 11:47:46.721 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.54e+01, 3.67e+00 / 9.05e-01 | 20260508 11:47:46.721 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.79e+00 / 1.22e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.07e-01 / 2.06e-01 | 20260508 11:47:46.721 | INFO | PerfReport: 2.99%, Metrics -- 9.75e-01 / 1.12e+00, 2.42e-01 / 9.72e-02 | 20260508 11:47:46.721 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.30e-01, 4.83e-02 / 1.70e-02 | 20260508 11:47:46.721 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:47:46.721 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:47:46.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:47:46.721 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 24.13%, Upload -- 2.53e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 24.84%, Download -- 2.61e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 31.11%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 1.96%, LockUpload -- 2.06e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 2.78%, LockDownload -- 2.92e+00 / 0.00e+00, 4.36e+00 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 15.17%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 11:47:46.721 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260508 11:47:46.816 | INFO | Starting a round of Baz IO | 20260508 11:47:50.227 | INFO | Populating RTMetrics report | 20260508 11:47:50.227 | INFO | RTMetrics report populated | 20260508 11:47:50.227 | INFO | Opening tmp RTMetrics json file | 20260508 11:47:50.234 | INFO | Serializing RTMetrics to json | 20260508 11:47:50.236 | INFO | Serialization done, swapping tmp file now... | 20260508 11:47:50.237 | INFO | RTMetrics done | 20260508 11:47:50.292 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.476 seconds | 20260508 11:47:50.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.00% / 59.92% | 20260508 11:47:50.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182557% / 0% / 5.5631e-05% | 20260508 11:47:50.292 | INFO | Baz IO overall: | 20260508 11:47:50.292 | INFO | BytesWritten : 3385684036279 | 20260508 11:47:50.292 | INFO | NumEvents : 1625607914944 | 20260508 11:47:51.861 | INFO | Analyzing chunk frames = [7864320,7864832) | 20260508 11:47:51.861 | INFO | MemInfo: { | 20260508 11:47:51.861 | INFO | "gpu" : " 17 GB", | 20260508 11:47:51.861 | INFO | "huge" : "", | 20260508 11:47:51.861 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:51.861 | INFO | "malloc" : " 14 GB", | 20260508 11:47:51.861 | INFO | "pinned" : " 29 GB" | 20260508 11:47:51.861 | INFO | } | 20260508 11:47:51.861 | INFO | Perf report for the last 5.14149 seconds: | 20260508 11:47:51.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:51.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:51.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:51.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:51.861 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 11:47:51.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:51.861 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:47:51.861 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 11:47:51.861 | INFO | BAZWRITER : 66.9%, 2.5, 84.9ms, 0 | 20260508 11:47:51.861 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:47:51.861 | INFO | Uploaded 36GiB to GPU in 2.59465s (13.8747GiB/s) | 20260508 11:47:51.861 | INFO | Downloaded 38GiB to GPU in 2.728s (13.9296GiB/s) | 20260508 11:47:56.997 | INFO | Analyzing chunk frames = [7864832,7865344) | 20260508 11:47:56.997 | INFO | MemInfo: { | 20260508 11:47:56.997 | INFO | "gpu" : " 17 GB", | 20260508 11:47:56.997 | INFO | "huge" : "", | 20260508 11:47:56.997 | INFO | "huge_shared" : " 29 GB", | 20260508 11:47:56.997 | INFO | "malloc" : " 14 GB", | 20260508 11:47:56.997 | INFO | "pinned" : " 29 GB" | 20260508 11:47:56.997 | INFO | } | 20260508 11:47:56.998 | INFO | Perf report for the last 5.13686 seconds: | 20260508 11:47:56.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:47:56.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 11:47:56.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 11:47:56.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:47:56.998 | INFO | Uploaded 36GiB to GPU in 2.40976s (14.9393GiB/s) | 20260508 11:47:56.998 | INFO | Downloaded 34GiB to GPU in 2.48554s (13.6791GiB/s) | 20260508 11:48:02.130 | INFO | Analyzing chunk frames = [7865344,7865856) | 20260508 11:48:02.130 | INFO | MemInfo: { | 20260508 11:48:02.130 | INFO | "gpu" : " 17 GB", | 20260508 11:48:02.130 | INFO | "huge" : "", | 20260508 11:48:02.130 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:02.131 | INFO | "malloc" : " 14 GB", | 20260508 11:48:02.131 | INFO | "pinned" : " 29 GB" | 20260508 11:48:02.131 | INFO | } | 20260508 11:48:02.131 | INFO | Perf report for the last 5.13311 seconds: | 20260508 11:48:02.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:02.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:48:02.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:48:02.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:02.131 | INFO | Uploaded 36GiB to GPU in 2.44881s (14.701GiB/s) | 20260508 11:48:02.131 | INFO | Downloaded 34GiB to GPU in 2.45403s (13.8548GiB/s) | 20260508 11:48:07.276 | INFO | Analyzing chunk frames = [7865856,7866368) | 20260508 11:48:07.276 | INFO | MemInfo: { | 20260508 11:48:07.276 | INFO | "gpu" : " 17 GB", | 20260508 11:48:07.276 | INFO | "huge" : "", | 20260508 11:48:07.276 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:07.276 | INFO | "malloc" : " 14 GB", | 20260508 11:48:07.276 | INFO | "pinned" : " 29 GB" | 20260508 11:48:07.276 | INFO | } | 20260508 11:48:07.276 | INFO | Perf report for the last 5.14579 seconds: | 20260508 11:48:07.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:07.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.277 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 11:48:07.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.277 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:48:07.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:07.277 | INFO | Uploaded 36GiB to GPU in 2.43976s (14.7556GiB/s) | 20260508 11:48:07.277 | INFO | Downloaded 34GiB to GPU in 2.42088s (14.0445GiB/s) | 20260508 11:48:12.421 | INFO | Analyzing chunk frames = [7866368,7866880) | 20260508 11:48:12.421 | INFO | MemInfo: { | 20260508 11:48:12.421 | INFO | "gpu" : " 17 GB", | 20260508 11:48:12.421 | INFO | "huge" : "", | 20260508 11:48:12.421 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:12.421 | INFO | "malloc" : " 14 GB", | 20260508 11:48:12.421 | INFO | "pinned" : " 29 GB" | 20260508 11:48:12.421 | INFO | } | 20260508 11:48:12.421 | INFO | Perf report for the last 5.14444 seconds: | 20260508 11:48:12.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:12.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 11:48:12.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:48:12.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:12.421 | INFO | Uploaded 36GiB to GPU in 2.27134s (15.8497GiB/s) | 20260508 11:48:12.422 | INFO | Downloaded 34GiB to GPU in 2.4982s (13.6098GiB/s) | 20260508 11:48:17.560 | INFO | Analyzing chunk frames = [7866880,7867392) | 20260508 11:48:17.560 | INFO | MemInfo: { | 20260508 11:48:17.560 | INFO | "gpu" : " 17 GB", | 20260508 11:48:17.560 | INFO | "huge" : "", | 20260508 11:48:17.560 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:17.560 | INFO | "malloc" : " 14 GB", | 20260508 11:48:17.560 | INFO | "pinned" : " 29 GB" | 20260508 11:48:17.560 | INFO | } | 20260508 11:48:17.560 | INFO | Perf report for the last 5.13896 seconds: | 20260508 11:48:17.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:17.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 11:48:17.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:48:17.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:17.560 | INFO | Uploaded 36GiB to GPU in 2.42842s (14.8245GiB/s) | 20260508 11:48:17.560 | INFO | Downloaded 34GiB to GPU in 2.44961s (13.8798GiB/s) | 20260508 11:48:22.691 | INFO | Analyzing chunk frames = [7867392,7867904) | 20260508 11:48:22.691 | INFO | MemInfo: { | 20260508 11:48:22.691 | INFO | "gpu" : " 17 GB", | 20260508 11:48:22.691 | INFO | "huge" : "", | 20260508 11:48:22.691 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:22.691 | INFO | "malloc" : " 14 GB", | 20260508 11:48:22.691 | INFO | "pinned" : " 29 GB" | 20260508 11:48:22.691 | INFO | } | 20260508 11:48:22.692 | INFO | Perf report for the last 5.13165 seconds: | 20260508 11:48:22.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:22.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 11:48:22.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:48:22.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:22.692 | INFO | Uploaded 36GiB to GPU in 2.44329s (14.7342GiB/s) | 20260508 11:48:22.692 | INFO | Downloaded 34GiB to GPU in 2.45969s (13.8229GiB/s) | 20260508 11:48:27.834 | INFO | Analyzing chunk frames = [7867904,7868416) | 20260508 11:48:27.834 | INFO | MemInfo: { | 20260508 11:48:27.834 | INFO | "gpu" : " 17 GB", | 20260508 11:48:27.834 | INFO | "huge" : "", | 20260508 11:48:27.835 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:27.835 | INFO | "malloc" : " 14 GB", | 20260508 11:48:27.835 | INFO | "pinned" : " 29 GB" | 20260508 11:48:27.835 | INFO | } | 20260508 11:48:27.835 | INFO | Perf report for the last 5.14337 seconds: | 20260508 11:48:27.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:27.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 11:48:27.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:48:27.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:27.835 | INFO | Uploaded 36GiB to GPU in 2.46546s (14.6017GiB/s) | 20260508 11:48:27.835 | INFO | Downloaded 34GiB to GPU in 2.47159s (13.7563GiB/s) | 20260508 11:48:27.947 | INFO | Starting a round of Baz IO | 20260508 11:48:31.350 | INFO | Populating RTMetrics report | 20260508 11:48:31.351 | INFO | RTMetrics report populated | 20260508 11:48:31.351 | INFO | Opening tmp RTMetrics json file | 20260508 11:48:31.352 | INFO | Serializing RTMetrics to json | 20260508 11:48:31.365 | INFO | Serialization done, swapping tmp file now... | 20260508 11:48:31.366 | INFO | RTMetrics done | 20260508 11:48:31.415 | INFO | Finished a round of Baz IO: Wrote 1.428 GiB in 3.467 seconds | 20260508 11:48:31.415 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.92% | 20260508 11:48:31.415 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182112% / 3.97364e-06% / 2.38419e-05% | 20260508 11:48:31.415 | INFO | Baz IO overall: | 20260508 11:48:31.415 | INFO | BytesWritten : 3387217777714 | 20260508 11:48:31.415 | INFO | NumEvents : 1626329105615 | 20260508 11:48:32.970 | INFO | Analyzing chunk frames = [7868416,7868928) | 20260508 11:48:32.970 | INFO | MemInfo: { | 20260508 11:48:32.970 | INFO | "gpu" : " 17 GB", | 20260508 11:48:32.970 | INFO | "huge" : "", | 20260508 11:48:32.970 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:32.970 | INFO | "malloc" : " 14 GB", | 20260508 11:48:32.970 | INFO | "pinned" : " 29 GB" | 20260508 11:48:32.970 | INFO | } | 20260508 11:48:32.970 | INFO | Perf report for the last 5.13539 seconds: | 20260508 11:48:32.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:32.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:32.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:32.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:32.970 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 11:48:32.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:32.970 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260508 11:48:32.970 | INFO | PRE_HQ : 66.6%, 1.8, 61.6ms, 0 | 20260508 11:48:32.970 | INFO | BAZWRITER : 66.8%, 2.7, 92.0ms, 0 | 20260508 11:48:32.970 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 11:48:32.971 | INFO | Uploaded 36GiB to GPU in 2.61643s (13.7592GiB/s) | 20260508 11:48:32.971 | INFO | Downloaded 38GiB to GPU in 2.73655s (13.8861GiB/s) | 20260508 11:48:38.117 | INFO | Analyzing chunk frames = [7868928,7869440) | 20260508 11:48:38.117 | INFO | MemInfo: { | 20260508 11:48:38.117 | INFO | "gpu" : " 17 GB", | 20260508 11:48:38.117 | INFO | "huge" : "", | 20260508 11:48:38.117 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:38.117 | INFO | "malloc" : " 14 GB", | 20260508 11:48:38.117 | INFO | "pinned" : " 29 GB" | 20260508 11:48:38.117 | INFO | } | 20260508 11:48:38.117 | INFO | Perf report for the last 5.14651 seconds: | 20260508 11:48:38.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:38.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:48:38.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:48:38.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:38.117 | INFO | Uploaded 36GiB to GPU in 2.45143s (14.6853GiB/s) | 20260508 11:48:38.117 | INFO | Downloaded 34GiB to GPU in 2.47001s (13.7651GiB/s) | 20260508 11:48:43.255 | INFO | Analyzing chunk frames = [7869440,7869952) | 20260508 11:48:43.255 | INFO | MemInfo: { | 20260508 11:48:43.255 | INFO | "gpu" : " 17 GB", | 20260508 11:48:43.255 | INFO | "huge" : "", | 20260508 11:48:43.255 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:43.255 | INFO | "malloc" : " 14 GB", | 20260508 11:48:43.255 | INFO | "pinned" : " 29 GB" | 20260508 11:48:43.255 | INFO | } | 20260508 11:48:43.255 | INFO | Perf report for the last 5.138 seconds: | 20260508 11:48:43.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:43.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.255 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 11:48:43.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.255 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:48:43.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:43.256 | INFO | Uploaded 36GiB to GPU in 2.49701s (14.4172GiB/s) | 20260508 11:48:43.256 | INFO | Downloaded 34GiB to GPU in 2.45619s (13.8426GiB/s) | 20260508 11:48:48.389 | INFO | Analyzing chunk frames = [7869952,7870464) | 20260508 11:48:48.389 | INFO | MemInfo: { | 20260508 11:48:48.389 | INFO | "gpu" : " 17 GB", | 20260508 11:48:48.389 | INFO | "huge" : "", | 20260508 11:48:48.389 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:48.389 | INFO | "malloc" : " 14 GB", | 20260508 11:48:48.389 | INFO | "pinned" : " 29 GB" | 20260508 11:48:48.389 | INFO | } | 20260508 11:48:48.389 | INFO | Perf report for the last 5.13438 seconds: | 20260508 11:48:48.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:48.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.389 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:48:48.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.389 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:48:48.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:48.390 | INFO | Uploaded 36GiB to GPU in 2.4576s (14.6485GiB/s) | 20260508 11:48:48.390 | INFO | Downloaded 34GiB to GPU in 2.46013s (13.8204GiB/s) | 20260508 11:48:53.539 | INFO | Analyzing chunk frames = [7870464,7870976) | 20260508 11:48:53.539 | INFO | MemInfo: { | 20260508 11:48:53.539 | INFO | "gpu" : " 17 GB", | 20260508 11:48:53.539 | INFO | "huge" : "", | 20260508 11:48:53.539 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:53.539 | INFO | "malloc" : " 14 GB", | 20260508 11:48:53.539 | INFO | "pinned" : " 29 GB" | 20260508 11:48:53.539 | INFO | } | 20260508 11:48:53.540 | INFO | Perf report for the last 5.15031 seconds: | 20260508 11:48:53.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:53.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 11:48:53.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:48:53.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:53.540 | INFO | Uploaded 36GiB to GPU in 2.28598s (15.7482GiB/s) | 20260508 11:48:53.540 | INFO | Downloaded 34GiB to GPU in 2.50559s (13.5696GiB/s) | 20260508 11:48:58.675 | INFO | Analyzing chunk frames = [7870976,7871488) | 20260508 11:48:58.675 | INFO | MemInfo: { | 20260508 11:48:58.675 | INFO | "gpu" : " 17 GB", | 20260508 11:48:58.675 | INFO | "huge" : "", | 20260508 11:48:58.675 | INFO | "huge_shared" : " 29 GB", | 20260508 11:48:58.675 | INFO | "malloc" : " 14 GB", | 20260508 11:48:58.675 | INFO | "pinned" : " 29 GB" | 20260508 11:48:58.675 | INFO | } | 20260508 11:48:58.675 | INFO | Perf report for the last 5.13568 seconds: | 20260508 11:48:58.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:48:58.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.675 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:48:58.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.675 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:48:58.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:48:58.676 | INFO | Uploaded 36GiB to GPU in 2.38312s (15.1062GiB/s) | 20260508 11:48:58.676 | INFO | Downloaded 34GiB to GPU in 2.45802s (13.8323GiB/s) | 20260508 11:49:03.808 | INFO | Analyzing chunk frames = [7871488,7872000) | 20260508 11:49:03.808 | INFO | MemInfo: { | 20260508 11:49:03.808 | INFO | "gpu" : " 17 GB", | 20260508 11:49:03.808 | INFO | "huge" : "", | 20260508 11:49:03.808 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:03.808 | INFO | "malloc" : " 14 GB", | 20260508 11:49:03.808 | INFO | "pinned" : " 29 GB" | 20260508 11:49:03.808 | INFO | } | 20260508 11:49:03.808 | INFO | Perf report for the last 5.13269 seconds: | 20260508 11:49:03.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:03.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 11:49:03.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 11:49:03.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:03.808 | INFO | Uploaded 36GiB to GPU in 2.42334s (14.8555GiB/s) | 20260508 11:49:03.808 | INFO | Downloaded 34GiB to GPU in 2.47799s (13.7208GiB/s) | 20260508 11:49:08.944 | INFO | Analyzing chunk frames = [7872000,7872512) | 20260508 11:49:08.944 | INFO | MemInfo: { | 20260508 11:49:08.944 | INFO | "gpu" : " 17 GB", | 20260508 11:49:08.944 | INFO | "huge" : "", | 20260508 11:49:08.944 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:08.944 | INFO | "malloc" : " 14 GB", | 20260508 11:49:08.944 | INFO | "pinned" : " 29 GB" | 20260508 11:49:08.944 | INFO | } | 20260508 11:49:08.944 | INFO | Perf report for the last 5.13618 seconds: | 20260508 11:49:08.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:08.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 11:49:08.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:49:08.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:08.944 | INFO | Uploaded 36GiB to GPU in 2.48031s (14.5143GiB/s) | 20260508 11:49:08.944 | INFO | Downloaded 34GiB to GPU in 2.48469s (13.6838GiB/s) | 20260508 11:49:08.945 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:49:08.945 | INFO | Total mem usage: 16.7 GiB | 20260508 11:49:08.945 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:49:08.945 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:49:08.945 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:49:08.945 | INFO | High Water Breakdown: | 20260508 11:49:08.945 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:49:08.945 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:49:08.945 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:49:08.945 | INFO | ------------------------------------------ | 20260508 11:49:08.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:49:08.945 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:49:08.945 | INFO | Total mem usage: 26.5 GiB | 20260508 11:49:08.945 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:49:08.945 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:49:08.945 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:49:08.945 | INFO | High Water Breakdown: | 20260508 11:49:08.945 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:49:08.945 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:49:08.945 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:49:08.945 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:49:08.945 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:49:08.945 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:49:08.945 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:49:08.945 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:49:08.945 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:49:08.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:49:08.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:49:08.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:49:08.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:49:08.945 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:49:08.945 | INFO | ------------------------------------------ | 20260508 11:49:08.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:49:08.945 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:49:08.945 | INFO | Total mem usage: 27.8 GiB | 20260508 11:49:08.945 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:49:08.945 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:49:08.945 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:49:08.945 | INFO | High Water Breakdown: | 20260508 11:49:08.945 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:49:08.945 | INFO | ------------------------------------------ | 20260508 11:49:08.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:49:08.945 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:49:08.945 | INFO | Total mem usage: 15.7 GiB | 20260508 11:49:08.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:49:08.945 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:49:08.945 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:49:08.945 | INFO | High Water Breakdown: | 20260508 11:49:08.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:49:08.945 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:49:08.945 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:49:08.945 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:49:08.945 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:49:08.945 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:49:08.945 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:49:08.945 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:49:08.945 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:49:08.945 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:49:08.945 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:49:08.945 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:49:08.945 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:49:08.945 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:49:08.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:49:08.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:49:08.945 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:49:08.945 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:49:08.945 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:49:08.945 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:49:08.945 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:49:08.945 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:49:08.945 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:49:08.945 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:49:08.945 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:49:08.945 | INFO | ------------------------------------------ | 20260508 11:49:08.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:49:08.945 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:49:08.945 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:49:08.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:49:08.945 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 5.46e-01 | 20260508 11:49:08.945 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.08e-02 / 6.59e-03 | 20260508 11:49:08.945 | INFO | PerfReport: 12.20%, Baseline -- 3.97e+00 / 4.46e+00, 5.87e-01 / 1.85e-01 | 20260508 11:49:08.945 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.45e+00, 2.07e-01 / 1.18e-01 | 20260508 11:49:08.945 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.55e+01, 3.49e+00 / 1.04e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.76e+00 / 1.10e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.34e+00 / 2.53e+00, 4.03e-01 / 2.43e-01 | 20260508 11:49:08.945 | INFO | PerfReport: 2.99%, Metrics -- 9.76e-01 / 1.09e+00, 2.40e-01 / 1.15e-01 | 20260508 11:49:08.945 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.26e-01, 4.71e-02 / 2.88e-02 | 20260508 11:49:08.945 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:49:08.945 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:49:08.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:49:08.945 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 24.23%, Upload -- 2.55e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 24.80%, Download -- 2.61e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 31.02%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 2.05%, LockUpload -- 2.16e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 2.76%, LockDownload -- 2.90e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 15.12%, LockCompute -- 1.59e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 11:49:08.945 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 11:49:09.041 | INFO | Starting a round of Baz IO | 20260508 11:49:12.453 | INFO | Populating RTMetrics report | 20260508 11:49:12.453 | INFO | RTMetrics report populated | 20260508 11:49:12.453 | INFO | Opening tmp RTMetrics json file | 20260508 11:49:12.462 | INFO | Serializing RTMetrics to json | 20260508 11:49:12.463 | INFO | Serialization done, swapping tmp file now... | 20260508 11:49:12.463 | INFO | RTMetrics done | 20260508 11:49:12.518 | INFO | Finished a round of Baz IO: Wrote 1.427 GiB in 3.476 seconds | 20260508 11:49:12.518 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:49:12.518 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.181007% / 3.97364e-06% / 6.75519e-05% | 20260508 11:49:12.518 | INFO | Baz IO overall: | 20260508 11:49:12.518 | INFO | BytesWritten : 3388749891828 | 20260508 11:49:12.518 | INFO | NumEvents : 1627049247150 | 20260508 11:49:14.091 | INFO | Analyzing chunk frames = [7872512,7873024) | 20260508 11:49:14.091 | INFO | MemInfo: { | 20260508 11:49:14.091 | INFO | "gpu" : " 17 GB", | 20260508 11:49:14.091 | INFO | "huge" : "", | 20260508 11:49:14.091 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:14.091 | INFO | "malloc" : " 14 GB", | 20260508 11:49:14.091 | INFO | "pinned" : " 29 GB" | 20260508 11:49:14.091 | INFO | } | 20260508 11:49:14.091 | INFO | Perf report for the last 5.14679 seconds: | 20260508 11:49:14.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:14.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:14.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:14.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:14.091 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 11:49:14.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:14.091 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 11:49:14.091 | INFO | PRE_HQ : 65.6%, 1.6, 53.8ms, 0 | 20260508 11:49:14.091 | INFO | BAZWRITER : 66.9%, 2.5, 85.9ms, 0 | 20260508 11:49:14.091 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 11:49:14.091 | INFO | Uploaded 36GiB to GPU in 2.5233s (14.267GiB/s) | 20260508 11:49:14.091 | INFO | Downloaded 38GiB to GPU in 2.73512s (13.8933GiB/s) | 20260508 11:49:19.231 | INFO | Analyzing chunk frames = [7873024,7873536) | 20260508 11:49:19.231 | INFO | MemInfo: { | 20260508 11:49:19.231 | INFO | "gpu" : " 17 GB", | 20260508 11:49:19.231 | INFO | "huge" : "", | 20260508 11:49:19.231 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:19.231 | INFO | "malloc" : " 14 GB", | 20260508 11:49:19.231 | INFO | "pinned" : " 29 GB" | 20260508 11:49:19.231 | INFO | } | 20260508 11:49:19.231 | INFO | Perf report for the last 5.14028 seconds: | 20260508 11:49:19.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:19.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.231 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 11:49:19.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.231 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:49:19.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:19.232 | INFO | Uploaded 36GiB to GPU in 2.40838s (14.9478GiB/s) | 20260508 11:49:19.232 | INFO | Downloaded 34GiB to GPU in 2.47735s (13.7244GiB/s) | 20260508 11:49:24.369 | INFO | Analyzing chunk frames = [7873536,7874048) | 20260508 11:49:24.369 | INFO | MemInfo: { | 20260508 11:49:24.369 | INFO | "gpu" : " 17 GB", | 20260508 11:49:24.369 | INFO | "huge" : "", | 20260508 11:49:24.369 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:24.369 | INFO | "malloc" : " 14 GB", | 20260508 11:49:24.369 | INFO | "pinned" : " 29 GB" | 20260508 11:49:24.369 | INFO | } | 20260508 11:49:24.369 | INFO | Perf report for the last 5.13794 seconds: | 20260508 11:49:24.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:24.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.369 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:49:24.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.369 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:49:24.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:24.370 | INFO | Uploaded 36GiB to GPU in 2.47294s (14.5575GiB/s) | 20260508 11:49:24.370 | INFO | Downloaded 34GiB to GPU in 2.45551s (13.8464GiB/s) | 20260508 11:49:29.509 | INFO | Analyzing chunk frames = [7874048,7874560) | 20260508 11:49:29.509 | INFO | MemInfo: { | 20260508 11:49:29.509 | INFO | "gpu" : " 17 GB", | 20260508 11:49:29.509 | INFO | "huge" : "", | 20260508 11:49:29.509 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:29.509 | INFO | "malloc" : " 14 GB", | 20260508 11:49:29.509 | INFO | "pinned" : " 29 GB" | 20260508 11:49:29.509 | INFO | } | 20260508 11:49:29.509 | INFO | Perf report for the last 5.13978 seconds: | 20260508 11:49:29.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:29.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 11:49:29.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:49:29.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:29.509 | INFO | Uploaded 36GiB to GPU in 2.50165s (14.3905GiB/s) | 20260508 11:49:29.509 | INFO | Downloaded 34GiB to GPU in 2.45766s (13.8343GiB/s) | 20260508 11:49:34.641 | INFO | Analyzing chunk frames = [7874560,7875072) | 20260508 11:49:34.641 | INFO | MemInfo: { | 20260508 11:49:34.641 | INFO | "gpu" : " 17 GB", | 20260508 11:49:34.641 | INFO | "huge" : "", | 20260508 11:49:34.641 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:34.641 | INFO | "malloc" : " 14 GB", | 20260508 11:49:34.641 | INFO | "pinned" : " 29 GB" | 20260508 11:49:34.641 | INFO | } | 20260508 11:49:34.642 | INFO | Perf report for the last 5.13249 seconds: | 20260508 11:49:34.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:34.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 11:49:34.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:49:34.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:34.642 | INFO | Uploaded 36GiB to GPU in 2.36763s (15.2051GiB/s) | 20260508 11:49:34.642 | INFO | Downloaded 34GiB to GPU in 2.51921s (13.4963GiB/s) | 20260508 11:49:39.788 | INFO | Analyzing chunk frames = [7875072,7875584) | 20260508 11:49:39.788 | INFO | MemInfo: { | 20260508 11:49:39.788 | INFO | "gpu" : " 17 GB", | 20260508 11:49:39.788 | INFO | "huge" : "", | 20260508 11:49:39.788 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:39.788 | INFO | "malloc" : " 14 GB", | 20260508 11:49:39.788 | INFO | "pinned" : " 29 GB" | 20260508 11:49:39.788 | INFO | } | 20260508 11:49:39.788 | INFO | Perf report for the last 5.14674 seconds: | 20260508 11:49:39.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:39.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.789 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 11:49:39.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.789 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260508 11:49:39.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:39.789 | INFO | Uploaded 36GiB to GPU in 2.37959s (15.1287GiB/s) | 20260508 11:49:39.789 | INFO | Downloaded 34GiB to GPU in 2.46008s (13.8207GiB/s) | 20260508 11:49:44.925 | INFO | Analyzing chunk frames = [7875584,7876096) | 20260508 11:49:44.925 | INFO | MemInfo: { | 20260508 11:49:44.925 | INFO | "gpu" : " 17 GB", | 20260508 11:49:44.926 | INFO | "huge" : "", | 20260508 11:49:44.926 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:44.926 | INFO | "malloc" : " 14 GB", | 20260508 11:49:44.926 | INFO | "pinned" : " 29 GB" | 20260508 11:49:44.926 | INFO | } | 20260508 11:49:44.926 | INFO | Perf report for the last 5.13749 seconds: | 20260508 11:49:44.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:44.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 11:49:44.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 11:49:44.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:44.926 | INFO | Uploaded 36GiB to GPU in 2.31678s (15.5388GiB/s) | 20260508 11:49:44.926 | INFO | Downloaded 34GiB to GPU in 2.44431s (13.9099GiB/s) | 20260508 11:49:50.068 | INFO | Analyzing chunk frames = [7876096,7876608) | 20260508 11:49:50.068 | INFO | MemInfo: { | 20260508 11:49:50.068 | INFO | "gpu" : " 17 GB", | 20260508 11:49:50.068 | INFO | "huge" : "", | 20260508 11:49:50.068 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:50.068 | INFO | "malloc" : " 14 GB", | 20260508 11:49:50.068 | INFO | "pinned" : " 29 GB" | 20260508 11:49:50.068 | INFO | } | 20260508 11:49:50.068 | INFO | Perf report for the last 5.14227 seconds: | 20260508 11:49:50.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:50.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | ANALYSIS : 64.9%, 3.0, 101.5ms, 0 | 20260508 11:49:50.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260508 11:49:50.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:50.068 | INFO | Uploaded 36GiB to GPU in 2.33125s (15.4423GiB/s) | 20260508 11:49:50.068 | INFO | Downloaded 34GiB to GPU in 2.45758s (13.8347GiB/s) | 20260508 11:49:50.180 | INFO | Starting a round of Baz IO | 20260508 11:49:53.578 | INFO | Populating RTMetrics report | 20260508 11:49:53.578 | INFO | RTMetrics report populated | 20260508 11:49:53.578 | INFO | Opening tmp RTMetrics json file | 20260508 11:49:53.579 | INFO | Serializing RTMetrics to json | 20260508 11:49:53.580 | INFO | Serialization done, swapping tmp file now... | 20260508 11:49:53.580 | INFO | RTMetrics done | 20260508 11:49:53.642 | INFO | Finished a round of Baz IO: Wrote 1.425 GiB in 3.462 seconds | 20260508 11:49:53.642 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:49:53.642 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180988% / 7.94729e-06% / 7.94729e-05% | 20260508 11:49:53.642 | INFO | Baz IO overall: | 20260508 11:49:53.642 | INFO | BytesWritten : 3390279950008 | 20260508 11:49:53.642 | INFO | NumEvents : 1627768289349 | 20260508 11:49:55.207 | INFO | Analyzing chunk frames = [7876608,7877120) | 20260508 11:49:55.207 | INFO | MemInfo: { | 20260508 11:49:55.207 | INFO | "gpu" : " 17 GB", | 20260508 11:49:55.207 | INFO | "huge" : "", | 20260508 11:49:55.207 | INFO | "huge_shared" : " 29 GB", | 20260508 11:49:55.207 | INFO | "malloc" : " 14 GB", | 20260508 11:49:55.207 | INFO | "pinned" : " 29 GB" | 20260508 11:49:55.207 | INFO | } | 20260508 11:49:55.207 | INFO | Perf report for the last 5.13884 seconds: | 20260508 11:49:55.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:49:55.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:55.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:55.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:55.207 | INFO | ANALYSIS : 67.9%, 3.0, 105.5ms, 0 | 20260508 11:49:55.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:49:55.207 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 11:49:55.207 | INFO | PRE_HQ : 66.4%, 1.7, 59.7ms, 0 | 20260508 11:49:55.207 | INFO | BAZWRITER : 66.7%, 2.6, 90.4ms, 0 | 20260508 11:49:55.207 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 11:49:55.208 | INFO | Uploaded 36GiB to GPU in 2.57143s (14GiB/s) | 20260508 11:49:55.208 | INFO | Downloaded 38GiB to GPU in 2.73214s (13.9085GiB/s) | 20260508 11:50:00.342 | INFO | Analyzing chunk frames = [7877120,7877632) | 20260508 11:50:00.342 | INFO | MemInfo: { | 20260508 11:50:00.342 | INFO | "gpu" : " 17 GB", | 20260508 11:50:00.342 | INFO | "huge" : "", | 20260508 11:50:00.342 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:00.343 | INFO | "malloc" : " 14 GB", | 20260508 11:50:00.343 | INFO | "pinned" : " 29 GB" | 20260508 11:50:00.343 | INFO | } | 20260508 11:50:00.343 | INFO | Perf report for the last 5.13569 seconds: | 20260508 11:50:00.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:00.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:50:00.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:50:00.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:00.343 | INFO | Uploaded 36GiB to GPU in 2.42385s (14.8524GiB/s) | 20260508 11:50:00.343 | INFO | Downloaded 34GiB to GPU in 2.47733s (13.7244GiB/s) | 20260508 11:50:05.483 | INFO | Analyzing chunk frames = [7877632,7878144) | 20260508 11:50:05.483 | INFO | MemInfo: { | 20260508 11:50:05.483 | INFO | "gpu" : " 17 GB", | 20260508 11:50:05.483 | INFO | "huge" : "", | 20260508 11:50:05.483 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:05.483 | INFO | "malloc" : " 14 GB", | 20260508 11:50:05.483 | INFO | "pinned" : " 29 GB" | 20260508 11:50:05.483 | INFO | } | 20260508 11:50:05.484 | INFO | Perf report for the last 5.14093 seconds: | 20260508 11:50:05.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:05.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 11:50:05.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:50:05.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:05.484 | INFO | Uploaded 36GiB to GPU in 2.46014s (14.6333GiB/s) | 20260508 11:50:05.484 | INFO | Downloaded 34GiB to GPU in 2.46676s (13.7833GiB/s) | 20260508 11:50:10.621 | INFO | Analyzing chunk frames = [7878144,7878656) | 20260508 11:50:10.621 | INFO | MemInfo: { | 20260508 11:50:10.621 | INFO | "gpu" : " 17 GB", | 20260508 11:50:10.621 | INFO | "huge" : "", | 20260508 11:50:10.621 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:10.621 | INFO | "malloc" : " 14 GB", | 20260508 11:50:10.621 | INFO | "pinned" : " 29 GB" | 20260508 11:50:10.621 | INFO | } | 20260508 11:50:10.621 | INFO | Perf report for the last 5.13778 seconds: | 20260508 11:50:10.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:10.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.621 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 11:50:10.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.622 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:50:10.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:10.622 | INFO | Uploaded 36GiB to GPU in 2.4477s (14.7077GiB/s) | 20260508 11:50:10.622 | INFO | Downloaded 34GiB to GPU in 2.46641s (13.7852GiB/s) | 20260508 11:50:15.759 | INFO | Analyzing chunk frames = [7878656,7879168) | 20260508 11:50:15.759 | INFO | MemInfo: { | 20260508 11:50:15.759 | INFO | "gpu" : " 17 GB", | 20260508 11:50:15.760 | INFO | "huge" : "", | 20260508 11:50:15.760 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:15.760 | INFO | "malloc" : " 14 GB", | 20260508 11:50:15.760 | INFO | "pinned" : " 29 GB" | 20260508 11:50:15.760 | INFO | } | 20260508 11:50:15.760 | INFO | Perf report for the last 5.13838 seconds: | 20260508 11:50:15.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:15.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:50:15.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:50:15.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:15.760 | INFO | Uploaded 36GiB to GPU in 2.28646s (15.7449GiB/s) | 20260508 11:50:15.760 | INFO | Downloaded 34GiB to GPU in 2.49129s (13.6475GiB/s) | 20260508 11:50:20.896 | INFO | Analyzing chunk frames = [7879168,7879680) | 20260508 11:50:20.896 | INFO | MemInfo: { | 20260508 11:50:20.896 | INFO | "gpu" : " 17 GB", | 20260508 11:50:20.896 | INFO | "huge" : "", | 20260508 11:50:20.896 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:20.896 | INFO | "malloc" : " 14 GB", | 20260508 11:50:20.896 | INFO | "pinned" : " 29 GB" | 20260508 11:50:20.896 | INFO | } | 20260508 11:50:20.896 | INFO | Perf report for the last 5.13665 seconds: | 20260508 11:50:20.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:20.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.896 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 11:50:20.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.897 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:50:20.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:20.897 | INFO | Uploaded 36GiB to GPU in 2.31724s (15.5357GiB/s) | 20260508 11:50:20.897 | INFO | Downloaded 34GiB to GPU in 2.44727s (13.893GiB/s) | 20260508 11:50:26.041 | INFO | Analyzing chunk frames = [7879680,7880192) | 20260508 11:50:26.041 | INFO | MemInfo: { | 20260508 11:50:26.041 | INFO | "gpu" : " 17 GB", | 20260508 11:50:26.041 | INFO | "huge" : "", | 20260508 11:50:26.041 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:26.041 | INFO | "malloc" : " 14 GB", | 20260508 11:50:26.042 | INFO | "pinned" : " 29 GB" | 20260508 11:50:26.042 | INFO | } | 20260508 11:50:26.042 | INFO | Perf report for the last 5.14523 seconds: | 20260508 11:50:26.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:26.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 11:50:26.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 11:50:26.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:26.042 | INFO | Uploaded 36GiB to GPU in 2.42993s (14.8152GiB/s) | 20260508 11:50:26.042 | INFO | Downloaded 34GiB to GPU in 2.46341s (13.802GiB/s) | 20260508 11:50:31.184 | INFO | Analyzing chunk frames = [7880192,7880704) | 20260508 11:50:31.184 | INFO | MemInfo: { | 20260508 11:50:31.184 | INFO | "gpu" : " 17 GB", | 20260508 11:50:31.184 | INFO | "huge" : "", | 20260508 11:50:31.184 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:31.184 | INFO | "malloc" : " 14 GB", | 20260508 11:50:31.184 | INFO | "pinned" : " 29 GB" | 20260508 11:50:31.184 | INFO | } | 20260508 11:50:31.184 | INFO | Perf report for the last 5.1424 seconds: | 20260508 11:50:31.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:31.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.184 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 11:50:31.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.184 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260508 11:50:31.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:31.185 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:50:31.185 | INFO | Total mem usage: 16.7 GiB | 20260508 11:50:31.185 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:50:31.185 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:50:31.185 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:50:31.185 | INFO | High Water Breakdown: | 20260508 11:50:31.185 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:50:31.185 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:50:31.185 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:50:31.185 | INFO | ------------------------------------------ | 20260508 11:50:31.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:50:31.185 | INFO | Uploaded 36GiB to GPU in 2.45942s (14.6376GiB/s) | 20260508 11:50:31.185 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:50:31.185 | INFO | Total mem usage: 26.5 GiB | 20260508 11:50:31.185 | INFO | Downloaded 34GiB to GPU in 2.48773s (13.6671GiB/s) | 20260508 11:50:31.185 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:50:31.185 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:50:31.185 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:50:31.185 | INFO | High Water Breakdown: | 20260508 11:50:31.185 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:50:31.185 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:50:31.185 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:50:31.185 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:50:31.185 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:50:31.185 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:50:31.185 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:50:31.185 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:50:31.185 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:50:31.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:50:31.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:50:31.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:50:31.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:50:31.185 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:50:31.185 | INFO | ------------------------------------------ | 20260508 11:50:31.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:50:31.185 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:50:31.185 | INFO | Total mem usage: 27.8 GiB | 20260508 11:50:31.185 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:50:31.185 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:50:31.185 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:50:31.185 | INFO | High Water Breakdown: | 20260508 11:50:31.185 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:50:31.185 | INFO | ------------------------------------------ | 20260508 11:50:31.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:50:31.185 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:50:31.185 | INFO | Total mem usage: 15.7 GiB | 20260508 11:50:31.185 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:50:31.185 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:50:31.186 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:50:31.186 | INFO | High Water Breakdown: | 20260508 11:50:31.186 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:50:31.186 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:50:31.186 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:50:31.186 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:50:31.186 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:50:31.186 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:50:31.186 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:50:31.186 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:50:31.186 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:50:31.186 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:50:31.186 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:50:31.186 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:50:31.186 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:50:31.186 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:50:31.186 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:50:31.186 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:50:31.186 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:50:31.186 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:50:31.186 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:50:31.186 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:50:31.186 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:50:31.186 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:50:31.186 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:50:31.186 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:50:31.186 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:50:31.186 | INFO | ------------------------------------------ | 20260508 11:50:31.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:50:31.186 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:50:31.186 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:50:31.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:50:31.186 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.41e+01 / 5.62e-01 | 20260508 11:50:31.186 | INFO | PerfReport: 0.30%, AggregateStats -- 9.85e-02 / 1.04e-01, 1.13e-02 / 3.71e-03 | 20260508 11:50:31.186 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.41e+00, 5.93e-01 / 2.66e-01 | 20260508 11:50:31.186 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.46e+00, 2.11e-01 / 1.14e-01 | 20260508 11:50:31.186 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.58e+01, 3.83e+00 / 1.06e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.85e+00 / 9.81e-01 | 20260508 11:50:31.186 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.34e+00 / 2.55e+00, 4.04e-01 / 1.93e-01 | 20260508 11:50:31.186 | INFO | PerfReport: 2.98%, Metrics -- 9.76e-01 / 1.11e+00, 2.41e-01 / 1.11e-01 | 20260508 11:50:31.186 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.27e-01, 4.68e-02 / 1.55e-02 | 20260508 11:50:31.186 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:50:31.186 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:50:31.186 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:50:31.186 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 24.02%, Upload -- 2.52e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 24.85%, Download -- 2.61e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 31.21%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 1.98%, LockUpload -- 2.07e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 2.77%, LockDownload -- 2.90e+00 / 0.00e+00, 4.39e+00 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 15.16%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:50:31.186 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260508 11:50:31.302 | INFO | Starting a round of Baz IO | 20260508 11:50:34.694 | INFO | Populating RTMetrics report | 20260508 11:50:34.694 | INFO | RTMetrics report populated | 20260508 11:50:34.694 | INFO | Opening tmp RTMetrics json file | 20260508 11:50:34.695 | INFO | Serializing RTMetrics to json | 20260508 11:50:34.696 | INFO | Serialization done, swapping tmp file now... | 20260508 11:50:34.696 | INFO | RTMetrics done | 20260508 11:50:34.760 | INFO | Finished a round of Baz IO: Wrote 1.426 GiB in 3.458 seconds | 20260508 11:50:34.760 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:50:34.760 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.181003% / 0% / 5.96046e-05% | 20260508 11:50:34.760 | INFO | Baz IO overall: | 20260508 11:50:34.760 | INFO | BytesWritten : 3391811227187 | 20260508 11:50:34.760 | INFO | NumEvents : 1628488414230 | 20260508 11:50:36.321 | INFO | Analyzing chunk frames = [7880704,7881216) | 20260508 11:50:36.321 | INFO | MemInfo: { | 20260508 11:50:36.321 | INFO | "gpu" : " 17 GB", | 20260508 11:50:36.321 | INFO | "huge" : "", | 20260508 11:50:36.321 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:36.321 | INFO | "malloc" : " 14 GB", | 20260508 11:50:36.321 | INFO | "pinned" : " 29 GB" | 20260508 11:50:36.321 | INFO | } | 20260508 11:50:36.321 | INFO | Perf report for the last 5.13712 seconds: | 20260508 11:50:36.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:36.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:36.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:36.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:36.321 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260508 11:50:36.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:36.321 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260508 11:50:36.321 | INFO | PRE_HQ : 66.7%, 1.8, 62.6ms, 0 | 20260508 11:50:36.321 | INFO | BAZWRITER : 66.6%, 2.7, 91.5ms, 0 | 20260508 11:50:36.321 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:50:36.322 | INFO | Uploaded 36GiB to GPU in 2.57654s (13.9722GiB/s) | 20260508 11:50:36.322 | INFO | Downloaded 38GiB to GPU in 2.74035s (13.8668GiB/s) | 20260508 11:50:41.454 | INFO | Analyzing chunk frames = [7881216,7881728) | 20260508 11:50:41.454 | INFO | MemInfo: { | 20260508 11:50:41.454 | INFO | "gpu" : " 17 GB", | 20260508 11:50:41.454 | INFO | "huge" : "", | 20260508 11:50:41.454 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:41.454 | INFO | "malloc" : " 14 GB", | 20260508 11:50:41.454 | INFO | "pinned" : " 29 GB" | 20260508 11:50:41.454 | INFO | } | 20260508 11:50:41.454 | INFO | Perf report for the last 5.13296 seconds: | 20260508 11:50:41.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:41.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.454 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:50:41.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.454 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 11:50:41.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:41.455 | INFO | Uploaded 36GiB to GPU in 2.45678s (14.6533GiB/s) | 20260508 11:50:41.455 | INFO | Downloaded 34GiB to GPU in 2.47838s (13.7186GiB/s) | 20260508 11:50:46.597 | INFO | Analyzing chunk frames = [7881728,7882240) | 20260508 11:50:46.597 | INFO | MemInfo: { | 20260508 11:50:46.597 | INFO | "gpu" : " 17 GB", | 20260508 11:50:46.597 | INFO | "huge" : "", | 20260508 11:50:46.597 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:46.597 | INFO | "malloc" : " 14 GB", | 20260508 11:50:46.597 | INFO | "pinned" : " 29 GB" | 20260508 11:50:46.597 | INFO | } | 20260508 11:50:46.597 | INFO | Perf report for the last 5.14322 seconds: | 20260508 11:50:46.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:46.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.598 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 11:50:46.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.598 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:50:46.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:46.598 | INFO | Uploaded 36GiB to GPU in 2.45713s (14.6512GiB/s) | 20260508 11:50:46.598 | INFO | Downloaded 34GiB to GPU in 2.46303s (13.8041GiB/s) | 20260508 11:50:51.736 | INFO | Analyzing chunk frames = [7882240,7882752) | 20260508 11:50:51.736 | INFO | MemInfo: { | 20260508 11:50:51.736 | INFO | "gpu" : " 17 GB", | 20260508 11:50:51.736 | INFO | "huge" : "", | 20260508 11:50:51.737 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:51.737 | INFO | "malloc" : " 14 GB", | 20260508 11:50:51.737 | INFO | "pinned" : " 29 GB" | 20260508 11:50:51.737 | INFO | } | 20260508 11:50:51.737 | INFO | Perf report for the last 5.13928 seconds: | 20260508 11:50:51.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:51.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:50:51.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:50:51.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:51.737 | INFO | Uploaded 36GiB to GPU in 2.50755s (14.3566GiB/s) | 20260508 11:50:51.737 | INFO | Downloaded 34GiB to GPU in 2.46661s (13.7841GiB/s) | 20260508 11:50:56.877 | INFO | Analyzing chunk frames = [7882752,7883264) | 20260508 11:50:56.877 | INFO | MemInfo: { | 20260508 11:50:56.877 | INFO | "gpu" : " 17 GB", | 20260508 11:50:56.877 | INFO | "huge" : "", | 20260508 11:50:56.877 | INFO | "huge_shared" : " 29 GB", | 20260508 11:50:56.877 | INFO | "malloc" : " 14 GB", | 20260508 11:50:56.877 | INFO | "pinned" : " 29 GB" | 20260508 11:50:56.877 | INFO | } | 20260508 11:50:56.877 | INFO | Perf report for the last 5.14045 seconds: | 20260508 11:50:56.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:50:56.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.877 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:50:56.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.877 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:50:56.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:50:56.878 | INFO | Uploaded 36GiB to GPU in 2.27875s (15.7982GiB/s) | 20260508 11:50:56.878 | INFO | Downloaded 34GiB to GPU in 2.49495s (13.6275GiB/s) | 20260508 11:51:02.014 | INFO | Analyzing chunk frames = [7883264,7883776) | 20260508 11:51:02.014 | INFO | MemInfo: { | 20260508 11:51:02.014 | INFO | "gpu" : " 17 GB", | 20260508 11:51:02.014 | INFO | "huge" : "", | 20260508 11:51:02.015 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:02.015 | INFO | "malloc" : " 14 GB", | 20260508 11:51:02.015 | INFO | "pinned" : " 29 GB" | 20260508 11:51:02.015 | INFO | } | 20260508 11:51:02.015 | INFO | Perf report for the last 5.13755 seconds: | 20260508 11:51:02.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:02.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:51:02.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:51:02.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:02.015 | INFO | Uploaded 36GiB to GPU in 2.38474s (15.096GiB/s) | 20260508 11:51:02.015 | INFO | Downloaded 34GiB to GPU in 2.43243s (13.9778GiB/s) | 20260508 11:51:07.151 | INFO | Analyzing chunk frames = [7883776,7884288) | 20260508 11:51:07.151 | INFO | MemInfo: { | 20260508 11:51:07.151 | INFO | "gpu" : " 17 GB", | 20260508 11:51:07.151 | INFO | "huge" : "", | 20260508 11:51:07.151 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:07.151 | INFO | "malloc" : " 14 GB", | 20260508 11:51:07.151 | INFO | "pinned" : " 29 GB" | 20260508 11:51:07.151 | INFO | } | 20260508 11:51:07.151 | INFO | Perf report for the last 5.13613 seconds: | 20260508 11:51:07.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:07.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:51:07.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 11:51:07.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:07.151 | INFO | Uploaded 36GiB to GPU in 2.42655s (14.8359GiB/s) | 20260508 11:51:07.151 | INFO | Downloaded 34GiB to GPU in 2.44976s (13.8789GiB/s) | 20260508 11:51:12.297 | INFO | Analyzing chunk frames = [7884288,7884800) | 20260508 11:51:12.297 | INFO | MemInfo: { | 20260508 11:51:12.297 | INFO | "gpu" : " 17 GB", | 20260508 11:51:12.297 | INFO | "huge" : "", | 20260508 11:51:12.297 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:12.297 | INFO | "malloc" : " 14 GB", | 20260508 11:51:12.297 | INFO | "pinned" : " 29 GB" | 20260508 11:51:12.297 | INFO | } | 20260508 11:51:12.297 | INFO | Perf report for the last 5.14602 seconds: | 20260508 11:51:12.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:12.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.297 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 11:51:12.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.297 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 11:51:12.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:12.298 | INFO | Uploaded 36GiB to GPU in 2.44685s (14.7128GiB/s) | 20260508 11:51:12.298 | INFO | Downloaded 34GiB to GPU in 2.4467s (13.8963GiB/s) | 20260508 11:51:12.394 | INFO | Starting a round of Baz IO | 20260508 11:51:15.787 | INFO | Populating RTMetrics report | 20260508 11:51:15.787 | INFO | RTMetrics report populated | 20260508 11:51:15.787 | INFO | Opening tmp RTMetrics json file | 20260508 11:51:15.797 | INFO | Serializing RTMetrics to json | 20260508 11:51:15.798 | INFO | Serialization done, swapping tmp file now... | 20260508 11:51:15.799 | INFO | RTMetrics done | 20260508 11:51:15.858 | INFO | Finished a round of Baz IO: Wrote 1.424 GiB in 3.463 seconds | 20260508 11:51:15.858 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:51:15.858 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17918% / 3.97364e-06% / 7.94729e-05% | 20260508 11:51:15.858 | INFO | Baz IO overall: | 20260508 11:51:15.858 | INFO | BytesWritten : 3393340398408 | 20260508 11:51:15.858 | INFO | NumEvents : 1629206808726 | 20260508 11:51:17.431 | INFO | Analyzing chunk frames = [7884800,7885312) | 20260508 11:51:17.431 | INFO | MemInfo: { | 20260508 11:51:17.431 | INFO | "gpu" : " 17 GB", | 20260508 11:51:17.431 | INFO | "huge" : "", | 20260508 11:51:17.431 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:17.431 | INFO | "malloc" : " 14 GB", | 20260508 11:51:17.431 | INFO | "pinned" : " 29 GB" | 20260508 11:51:17.431 | INFO | } | 20260508 11:51:17.431 | INFO | Perf report for the last 5.13382 seconds: | 20260508 11:51:17.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:17.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:17.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:17.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:17.431 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 11:51:17.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:17.431 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 11:51:17.431 | INFO | PRE_HQ : 65.2%, 1.6, 54.1ms, 0 | 20260508 11:51:17.431 | INFO | BAZWRITER : 66.8%, 2.5, 87.1ms, 0 | 20260508 11:51:17.431 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:51:17.432 | INFO | Uploaded 36GiB to GPU in 2.61725s (13.7549GiB/s) | 20260508 11:51:17.432 | INFO | Downloaded 38GiB to GPU in 2.72254s (13.9575GiB/s) | 20260508 11:51:22.571 | INFO | Analyzing chunk frames = [7885312,7885824) | 20260508 11:51:22.571 | INFO | MemInfo: { | 20260508 11:51:22.571 | INFO | "gpu" : " 17 GB", | 20260508 11:51:22.571 | INFO | "huge" : "", | 20260508 11:51:22.571 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:22.571 | INFO | "malloc" : " 14 GB", | 20260508 11:51:22.571 | INFO | "pinned" : " 29 GB" | 20260508 11:51:22.571 | INFO | } | 20260508 11:51:22.571 | INFO | Perf report for the last 5.14011 seconds: | 20260508 11:51:22.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:22.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 11:51:22.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:51:22.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:22.571 | INFO | Uploaded 36GiB to GPU in 2.4855s (14.484GiB/s) | 20260508 11:51:22.571 | INFO | Downloaded 34GiB to GPU in 2.44448s (13.9089GiB/s) | 20260508 11:51:27.709 | INFO | Analyzing chunk frames = [7885824,7886336) | 20260508 11:51:27.709 | INFO | MemInfo: { | 20260508 11:51:27.709 | INFO | "gpu" : " 17 GB", | 20260508 11:51:27.709 | INFO | "huge" : "", | 20260508 11:51:27.709 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:27.709 | INFO | "malloc" : " 14 GB", | 20260508 11:51:27.709 | INFO | "pinned" : " 29 GB" | 20260508 11:51:27.709 | INFO | } | 20260508 11:51:27.710 | INFO | Perf report for the last 5.1387 seconds: | 20260508 11:51:27.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:27.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 11:51:27.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:51:27.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:27.710 | INFO | Uploaded 36GiB to GPU in 2.52593s (14.2522GiB/s) | 20260508 11:51:27.710 | INFO | Downloaded 34GiB to GPU in 2.44988s (13.8782GiB/s) | 20260508 11:51:32.852 | INFO | Analyzing chunk frames = [7886336,7886848) | 20260508 11:51:32.852 | INFO | MemInfo: { | 20260508 11:51:32.852 | INFO | "gpu" : " 17 GB", | 20260508 11:51:32.852 | INFO | "huge" : "", | 20260508 11:51:32.852 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:32.852 | INFO | "malloc" : " 14 GB", | 20260508 11:51:32.852 | INFO | "pinned" : " 29 GB" | 20260508 11:51:32.852 | INFO | } | 20260508 11:51:32.853 | INFO | Perf report for the last 5.14294 seconds: | 20260508 11:51:32.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:32.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 11:51:32.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:51:32.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:32.853 | INFO | Uploaded 36GiB to GPU in 2.50318s (14.3817GiB/s) | 20260508 11:51:32.853 | INFO | Downloaded 34GiB to GPU in 2.44419s (13.9106GiB/s) | 20260508 11:51:37.994 | INFO | Analyzing chunk frames = [7886848,7887360) | 20260508 11:51:37.994 | INFO | MemInfo: { | 20260508 11:51:37.994 | INFO | "gpu" : " 17 GB", | 20260508 11:51:37.994 | INFO | "huge" : "", | 20260508 11:51:37.994 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:37.994 | INFO | "malloc" : " 14 GB", | 20260508 11:51:37.994 | INFO | "pinned" : " 29 GB" | 20260508 11:51:37.994 | INFO | } | 20260508 11:51:37.994 | INFO | Perf report for the last 5.14158 seconds: | 20260508 11:51:37.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:37.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.994 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 11:51:37.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.994 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:51:37.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:37.995 | INFO | Uploaded 36GiB to GPU in 2.40132s (14.9917GiB/s) | 20260508 11:51:37.995 | INFO | Downloaded 34GiB to GPU in 2.50001s (13.6GiB/s) | 20260508 11:51:43.130 | INFO | Analyzing chunk frames = [7887360,7887872) | 20260508 11:51:43.130 | INFO | MemInfo: { | 20260508 11:51:43.130 | INFO | "gpu" : " 17 GB", | 20260508 11:51:43.130 | INFO | "huge" : "", | 20260508 11:51:43.130 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:43.130 | INFO | "malloc" : " 14 GB", | 20260508 11:51:43.130 | INFO | "pinned" : " 29 GB" | 20260508 11:51:43.130 | INFO | } | 20260508 11:51:43.130 | INFO | Perf report for the last 5.13601 seconds: | 20260508 11:51:43.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:43.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.130 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 11:51:43.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.130 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 11:51:43.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:43.131 | INFO | Uploaded 36GiB to GPU in 2.42015s (14.8751GiB/s) | 20260508 11:51:43.131 | INFO | Downloaded 34GiB to GPU in 2.43497s (13.9632GiB/s) | 20260508 11:51:48.274 | INFO | Analyzing chunk frames = [7887872,7888384) | 20260508 11:51:48.274 | INFO | MemInfo: { | 20260508 11:51:48.274 | INFO | "gpu" : " 17 GB", | 20260508 11:51:48.274 | INFO | "huge" : "", | 20260508 11:51:48.274 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:48.274 | INFO | "malloc" : " 14 GB", | 20260508 11:51:48.274 | INFO | "pinned" : " 29 GB" | 20260508 11:51:48.274 | INFO | } | 20260508 11:51:48.274 | INFO | Perf report for the last 5.14378 seconds: | 20260508 11:51:48.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:48.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 11:51:48.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:51:48.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:48.274 | INFO | Uploaded 36GiB to GPU in 2.4602s (14.6329GiB/s) | 20260508 11:51:48.274 | INFO | Downloaded 34GiB to GPU in 2.43323s (13.9732GiB/s) | 20260508 11:51:53.404 | INFO | Analyzing chunk frames = [7888384,7888896) | 20260508 11:51:53.404 | INFO | MemInfo: { | 20260508 11:51:53.404 | INFO | "gpu" : " 17 GB", | 20260508 11:51:53.404 | INFO | "huge" : "", | 20260508 11:51:53.404 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:53.404 | INFO | "malloc" : " 14 GB", | 20260508 11:51:53.405 | INFO | "pinned" : " 29 GB" | 20260508 11:51:53.405 | INFO | } | 20260508 11:51:53.405 | INFO | Perf report for the last 5.13076 seconds: | 20260508 11:51:53.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:53.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | ANALYSIS : 65.7%, 3.0, 102.0ms, 0 | 20260508 11:51:53.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:51:53.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:53.405 | INFO | Uploaded 36GiB to GPU in 2.40757s (14.9528GiB/s) | 20260508 11:51:53.405 | INFO | Downloaded 34GiB to GPU in 2.4349s (13.9636GiB/s) | 20260508 11:51:53.405 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:51:53.405 | INFO | Total mem usage: 16.7 GiB | 20260508 11:51:53.405 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:51:53.405 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:51:53.405 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:51:53.405 | INFO | High Water Breakdown: | 20260508 11:51:53.405 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:51:53.405 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:51:53.405 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:51:53.405 | INFO | ------------------------------------------ | 20260508 11:51:53.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:51:53.405 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:51:53.405 | INFO | Total mem usage: 26.5 GiB | 20260508 11:51:53.405 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:51:53.405 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:51:53.406 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:51:53.406 | INFO | High Water Breakdown: | 20260508 11:51:53.406 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:51:53.406 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:51:53.406 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:51:53.406 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:51:53.406 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:51:53.406 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:51:53.406 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:51:53.406 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:51:53.406 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:51:53.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:51:53.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:51:53.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:51:53.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:51:53.406 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:51:53.406 | INFO | ------------------------------------------ | 20260508 11:51:53.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:51:53.406 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:51:53.406 | INFO | Total mem usage: 27.8 GiB | 20260508 11:51:53.406 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:51:53.406 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:51:53.406 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:51:53.406 | INFO | High Water Breakdown: | 20260508 11:51:53.406 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:51:53.406 | INFO | ------------------------------------------ | 20260508 11:51:53.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:51:53.406 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:51:53.406 | INFO | Total mem usage: 15.7 GiB | 20260508 11:51:53.406 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:51:53.406 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:51:53.406 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:51:53.406 | INFO | High Water Breakdown: | 20260508 11:51:53.406 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:51:53.406 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:51:53.406 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:51:53.406 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:51:53.406 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:51:53.406 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:51:53.406 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:51:53.406 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:51:53.406 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:51:53.406 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:51:53.406 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:51:53.406 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:51:53.406 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:51:53.406 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:51:53.406 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:51:53.406 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:51:53.406 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:51:53.406 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:51:53.406 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:51:53.406 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:51:53.406 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:51:53.406 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:51:53.406 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:51:53.406 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:51:53.406 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:51:53.406 | INFO | ------------------------------------------ | 20260508 11:51:53.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:51:53.406 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:51:53.406 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:51:53.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:51:53.406 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.40e+01 / 5.01e-01 | 20260508 11:51:53.406 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.06e-01, 1.21e-02 / 6.57e-03 | 20260508 11:51:53.406 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.39e+00, 5.94e-01 / 1.43e-01 | 20260508 11:51:53.406 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.18e-01 | 20260508 11:51:53.406 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.53e+01, 3.61e+00 / 9.17e-01 | 20260508 11:51:53.406 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.07e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.05e-01 / 1.29e-01 | 20260508 11:51:53.406 | INFO | PerfReport: 2.99%, Metrics -- 9.74e-01 / 1.10e+00, 2.40e-01 / 8.06e-02 | 20260508 11:51:53.406 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.31e-01, 4.72e-02 / 1.74e-02 | 20260508 11:51:53.406 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:51:53.406 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:51:53.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:51:53.406 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 24.70%, Download -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 31.07%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 2.17%, LockUpload -- 2.27e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 2.57%, LockDownload -- 2.70e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 15.09%, LockCompute -- 1.58e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:51:53.406 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 11:51:53.502 | INFO | Starting a round of Baz IO | 20260508 11:51:56.906 | INFO | Populating RTMetrics report | 20260508 11:51:56.906 | INFO | RTMetrics report populated | 20260508 11:51:56.906 | INFO | Opening tmp RTMetrics json file | 20260508 11:51:56.915 | INFO | Serializing RTMetrics to json | 20260508 11:51:56.916 | INFO | Serialization done, swapping tmp file now... | 20260508 11:51:56.917 | INFO | RTMetrics done | 20260508 11:51:56.974 | INFO | Finished a round of Baz IO: Wrote 1.424 GiB in 3.471 seconds | 20260508 11:51:56.974 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:51:56.974 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179482% / 7.94729e-06% / 5.5631e-05% | 20260508 11:51:56.974 | INFO | Baz IO overall: | 20260508 11:51:56.974 | INFO | BytesWritten : 3394869699113 | 20260508 11:51:56.974 | INFO | NumEvents : 1629925454791 | 20260508 11:51:58.545 | INFO | Analyzing chunk frames = [7888896,7889408) | 20260508 11:51:58.545 | INFO | MemInfo: { | 20260508 11:51:58.545 | INFO | "gpu" : " 17 GB", | 20260508 11:51:58.545 | INFO | "huge" : "", | 20260508 11:51:58.545 | INFO | "huge_shared" : " 29 GB", | 20260508 11:51:58.545 | INFO | "malloc" : " 14 GB", | 20260508 11:51:58.545 | INFO | "pinned" : " 29 GB" | 20260508 11:51:58.545 | INFO | } | 20260508 11:51:58.545 | INFO | Perf report for the last 5.14018 seconds: | 20260508 11:51:58.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:51:58.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:58.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:58.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:58.545 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:51:58.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:51:58.545 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:51:58.545 | INFO | PRE_HQ : 65.3%, 1.6, 54.0ms, 0 | 20260508 11:51:58.545 | INFO | BAZWRITER : 66.9%, 2.5, 84.5ms, 0 | 20260508 11:51:58.545 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:51:58.546 | INFO | Uploaded 36GiB to GPU in 2.61132s (13.7861GiB/s) | 20260508 11:51:58.546 | INFO | Downloaded 38GiB to GPU in 2.71402s (14.0014GiB/s) | 20260508 11:52:03.688 | INFO | Analyzing chunk frames = [7889408,7889920) | 20260508 11:52:03.688 | INFO | MemInfo: { | 20260508 11:52:03.688 | INFO | "gpu" : " 17 GB", | 20260508 11:52:03.688 | INFO | "huge" : "", | 20260508 11:52:03.688 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:03.688 | INFO | "malloc" : " 14 GB", | 20260508 11:52:03.689 | INFO | "pinned" : " 29 GB" | 20260508 11:52:03.689 | INFO | } | 20260508 11:52:03.689 | INFO | Perf report for the last 5.14368 seconds: | 20260508 11:52:03.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:03.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:52:03.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:52:03.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:03.689 | INFO | Uploaded 36GiB to GPU in 2.50218s (14.3875GiB/s) | 20260508 11:52:03.689 | INFO | Downloaded 34GiB to GPU in 2.45782s (13.8334GiB/s) | 20260508 11:52:08.834 | INFO | Analyzing chunk frames = [7889920,7890432) | 20260508 11:52:08.834 | INFO | MemInfo: { | 20260508 11:52:08.834 | INFO | "gpu" : " 17 GB", | 20260508 11:52:08.834 | INFO | "huge" : "", | 20260508 11:52:08.834 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:08.834 | INFO | "malloc" : " 14 GB", | 20260508 11:52:08.834 | INFO | "pinned" : " 29 GB" | 20260508 11:52:08.834 | INFO | } | 20260508 11:52:08.834 | INFO | Perf report for the last 5.14563 seconds: | 20260508 11:52:08.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:08.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.834 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 11:52:08.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.834 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:52:08.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:08.835 | INFO | Uploaded 36GiB to GPU in 2.51619s (14.3074GiB/s) | 20260508 11:52:08.835 | INFO | Downloaded 34GiB to GPU in 2.44368s (13.9134GiB/s) | 20260508 11:52:13.964 | INFO | Analyzing chunk frames = [7890432,7890944) | 20260508 11:52:13.964 | INFO | MemInfo: { | 20260508 11:52:13.964 | INFO | "gpu" : " 17 GB", | 20260508 11:52:13.964 | INFO | "huge" : "", | 20260508 11:52:13.964 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:13.964 | INFO | "malloc" : " 14 GB", | 20260508 11:52:13.964 | INFO | "pinned" : " 29 GB" | 20260508 11:52:13.964 | INFO | } | 20260508 11:52:13.964 | INFO | Perf report for the last 5.12949 seconds: | 20260508 11:52:13.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:13.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | ANALYSIS : 65.6%, 3.0, 101.5ms, 0 | 20260508 11:52:13.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 11:52:13.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:13.964 | INFO | Uploaded 36GiB to GPU in 2.4311s (14.8081GiB/s) | 20260508 11:52:13.964 | INFO | Downloaded 34GiB to GPU in 2.4686s (13.773GiB/s) | 20260508 11:52:19.107 | INFO | Analyzing chunk frames = [7890944,7891456) | 20260508 11:52:19.107 | INFO | MemInfo: { | 20260508 11:52:19.107 | INFO | "gpu" : " 17 GB", | 20260508 11:52:19.107 | INFO | "huge" : "", | 20260508 11:52:19.107 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:19.107 | INFO | "malloc" : " 14 GB", | 20260508 11:52:19.108 | INFO | "pinned" : " 29 GB" | 20260508 11:52:19.108 | INFO | } | 20260508 11:52:19.108 | INFO | Perf report for the last 5.14389 seconds: | 20260508 11:52:19.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:19.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 11:52:19.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260508 11:52:19.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:19.108 | INFO | Uploaded 36GiB to GPU in 2.31541s (15.548GiB/s) | 20260508 11:52:19.108 | INFO | Downloaded 34GiB to GPU in 2.49783s (13.6118GiB/s) | 20260508 11:52:24.247 | INFO | Analyzing chunk frames = [7891456,7891968) | 20260508 11:52:24.247 | INFO | MemInfo: { | 20260508 11:52:24.247 | INFO | "gpu" : " 17 GB", | 20260508 11:52:24.247 | INFO | "huge" : "", | 20260508 11:52:24.247 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:24.247 | INFO | "malloc" : " 14 GB", | 20260508 11:52:24.247 | INFO | "pinned" : " 29 GB" | 20260508 11:52:24.247 | INFO | } | 20260508 11:52:24.247 | INFO | Perf report for the last 5.13934 seconds: | 20260508 11:52:24.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:24.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.247 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 11:52:24.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.247 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:52:24.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:24.248 | INFO | Uploaded 36GiB to GPU in 2.43256s (14.7992GiB/s) | 20260508 11:52:24.248 | INFO | Downloaded 34GiB to GPU in 2.43112s (13.9853GiB/s) | 20260508 11:52:29.381 | INFO | Analyzing chunk frames = [7891968,7892480) | 20260508 11:52:29.381 | INFO | MemInfo: { | 20260508 11:52:29.381 | INFO | "gpu" : " 17 GB", | 20260508 11:52:29.381 | INFO | "huge" : "", | 20260508 11:52:29.381 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:29.381 | INFO | "malloc" : " 14 GB", | 20260508 11:52:29.381 | INFO | "pinned" : " 29 GB" | 20260508 11:52:29.382 | INFO | } | 20260508 11:52:29.382 | INFO | Perf report for the last 5.13455 seconds: | 20260508 11:52:29.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:29.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 11:52:29.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:52:29.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:29.382 | INFO | Uploaded 36GiB to GPU in 2.35881s (15.262GiB/s) | 20260508 11:52:29.382 | INFO | Downloaded 34GiB to GPU in 2.40885s (14.1146GiB/s) | 20260508 11:52:34.520 | INFO | Analyzing chunk frames = [7892480,7892992) | 20260508 11:52:34.520 | INFO | MemInfo: { | 20260508 11:52:34.520 | INFO | "gpu" : " 17 GB", | 20260508 11:52:34.520 | INFO | "huge" : "", | 20260508 11:52:34.520 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:34.520 | INFO | "malloc" : " 14 GB", | 20260508 11:52:34.520 | INFO | "pinned" : " 29 GB" | 20260508 11:52:34.520 | INFO | } | 20260508 11:52:34.521 | INFO | Perf report for the last 5.13898 seconds: | 20260508 11:52:34.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:34.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 11:52:34.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 11:52:34.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:34.521 | INFO | Uploaded 36GiB to GPU in 2.45915s (14.6392GiB/s) | 20260508 11:52:34.521 | INFO | Downloaded 34GiB to GPU in 2.42965s (13.9938GiB/s) | 20260508 11:52:34.618 | INFO | Starting a round of Baz IO | 20260508 11:52:38.021 | INFO | Populating RTMetrics report | 20260508 11:52:38.022 | INFO | RTMetrics report populated | 20260508 11:52:38.022 | INFO | Opening tmp RTMetrics json file | 20260508 11:52:38.031 | INFO | Serializing RTMetrics to json | 20260508 11:52:38.032 | INFO | Serialization done, swapping tmp file now... | 20260508 11:52:38.032 | INFO | RTMetrics done | 20260508 11:52:38.089 | INFO | Finished a round of Baz IO: Wrote 1.425 GiB in 3.47 seconds | 20260508 11:52:38.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:52:38.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179331% / 7.94729e-06% / 5.5631e-05% | 20260508 11:52:38.089 | INFO | Baz IO overall: | 20260508 11:52:38.089 | INFO | BytesWritten : 3396400171915 | 20260508 11:52:38.089 | INFO | NumEvents : 1630645103771 | 20260508 11:52:39.660 | INFO | Analyzing chunk frames = [7892992,7893504) | 20260508 11:52:39.660 | INFO | MemInfo: { | 20260508 11:52:39.660 | INFO | "gpu" : " 17 GB", | 20260508 11:52:39.660 | INFO | "huge" : "", | 20260508 11:52:39.660 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:39.660 | INFO | "malloc" : " 14 GB", | 20260508 11:52:39.660 | INFO | "pinned" : " 29 GB" | 20260508 11:52:39.660 | INFO | } | 20260508 11:52:39.660 | INFO | Perf report for the last 5.13974 seconds: | 20260508 11:52:39.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:39.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:39.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:39.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:39.660 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 11:52:39.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:39.660 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:52:39.660 | INFO | PRE_HQ : 65.4%, 1.6, 54.0ms, 0 | 20260508 11:52:39.660 | INFO | BAZWRITER : 66.8%, 2.5, 84.7ms, 0 | 20260508 11:52:39.660 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:52:39.661 | INFO | Uploaded 36GiB to GPU in 2.62215s (13.7292GiB/s) | 20260508 11:52:39.661 | INFO | Downloaded 38GiB to GPU in 2.70265s (14.0603GiB/s) | 20260508 11:52:44.808 | INFO | Analyzing chunk frames = [7893504,7894016) | 20260508 11:52:44.808 | INFO | MemInfo: { | 20260508 11:52:44.808 | INFO | "gpu" : " 17 GB", | 20260508 11:52:44.808 | INFO | "huge" : "", | 20260508 11:52:44.808 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:44.808 | INFO | "malloc" : " 14 GB", | 20260508 11:52:44.808 | INFO | "pinned" : " 29 GB" | 20260508 11:52:44.808 | INFO | } | 20260508 11:52:44.808 | INFO | Perf report for the last 5.14785 seconds: | 20260508 11:52:44.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:44.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:52:44.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:52:44.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:44.808 | INFO | Uploaded 36GiB to GPU in 2.51028s (14.341GiB/s) | 20260508 11:52:44.808 | INFO | Downloaded 34GiB to GPU in 2.4641s (13.7982GiB/s) | 20260508 11:52:49.948 | INFO | Analyzing chunk frames = [7894016,7894528) | 20260508 11:52:49.948 | INFO | MemInfo: { | 20260508 11:52:49.948 | INFO | "gpu" : " 17 GB", | 20260508 11:52:49.948 | INFO | "huge" : "", | 20260508 11:52:49.948 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:49.949 | INFO | "malloc" : " 14 GB", | 20260508 11:52:49.949 | INFO | "pinned" : " 29 GB" | 20260508 11:52:49.949 | INFO | } | 20260508 11:52:49.949 | INFO | Perf report for the last 5.14043 seconds: | 20260508 11:52:49.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:49.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 11:52:49.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 11:52:49.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:49.949 | INFO | Uploaded 36GiB to GPU in 2.52447s (14.2604GiB/s) | 20260508 11:52:49.949 | INFO | Downloaded 34GiB to GPU in 2.45864s (13.8288GiB/s) | 20260508 11:52:55.081 | INFO | Analyzing chunk frames = [7894528,7895040) | 20260508 11:52:55.081 | INFO | MemInfo: { | 20260508 11:52:55.081 | INFO | "gpu" : " 17 GB", | 20260508 11:52:55.081 | INFO | "huge" : "", | 20260508 11:52:55.082 | INFO | "huge_shared" : " 29 GB", | 20260508 11:52:55.082 | INFO | "malloc" : " 14 GB", | 20260508 11:52:55.082 | INFO | "pinned" : " 29 GB" | 20260508 11:52:55.082 | INFO | } | 20260508 11:52:55.082 | INFO | Perf report for the last 5.13298 seconds: | 20260508 11:52:55.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:52:55.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | ANALYSIS : 65.6%, 3.0, 101.6ms, 0 | 20260508 11:52:55.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:52:55.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:52:55.082 | INFO | Uploaded 36GiB to GPU in 2.42666s (14.8352GiB/s) | 20260508 11:52:55.082 | INFO | Downloaded 34GiB to GPU in 2.46254s (13.8069GiB/s) | 20260508 11:53:00.216 | INFO | Analyzing chunk frames = [7895040,7895552) | 20260508 11:53:00.216 | INFO | MemInfo: { | 20260508 11:53:00.216 | INFO | "gpu" : " 17 GB", | 20260508 11:53:00.216 | INFO | "huge" : "", | 20260508 11:53:00.216 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:00.216 | INFO | "malloc" : " 14 GB", | 20260508 11:53:00.216 | INFO | "pinned" : " 29 GB" | 20260508 11:53:00.216 | INFO | } | 20260508 11:53:00.216 | INFO | Perf report for the last 5.13481 seconds: | 20260508 11:53:00.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:00.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.216 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:53:00.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.217 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 11:53:00.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:00.217 | INFO | Uploaded 36GiB to GPU in 2.31588s (15.5449GiB/s) | 20260508 11:53:00.217 | INFO | Downloaded 34GiB to GPU in 2.49088s (13.6498GiB/s) | 20260508 11:53:05.360 | INFO | Analyzing chunk frames = [7895552,7896064) | 20260508 11:53:05.360 | INFO | MemInfo: { | 20260508 11:53:05.360 | INFO | "gpu" : " 17 GB", | 20260508 11:53:05.360 | INFO | "huge" : "", | 20260508 11:53:05.360 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:05.360 | INFO | "malloc" : " 14 GB", | 20260508 11:53:05.360 | INFO | "pinned" : " 29 GB" | 20260508 11:53:05.360 | INFO | } | 20260508 11:53:05.360 | INFO | Perf report for the last 5.14372 seconds: | 20260508 11:53:05.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:05.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:53:05.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:53:05.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:05.360 | INFO | Uploaded 36GiB to GPU in 2.40003s (14.9998GiB/s) | 20260508 11:53:05.361 | INFO | Downloaded 34GiB to GPU in 2.42068s (14.0456GiB/s) | 20260508 11:53:10.500 | INFO | Analyzing chunk frames = [7896064,7896576) | 20260508 11:53:10.500 | INFO | MemInfo: { | 20260508 11:53:10.500 | INFO | "gpu" : " 17 GB", | 20260508 11:53:10.500 | INFO | "huge" : "", | 20260508 11:53:10.500 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:10.500 | INFO | "malloc" : " 14 GB", | 20260508 11:53:10.500 | INFO | "pinned" : " 29 GB" | 20260508 11:53:10.500 | INFO | } | 20260508 11:53:10.500 | INFO | Perf report for the last 5.13981 seconds: | 20260508 11:53:10.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:10.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.500 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:53:10.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.500 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:53:10.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:10.501 | INFO | Uploaded 36GiB to GPU in 2.46221s (14.621GiB/s) | 20260508 11:53:10.501 | INFO | Downloaded 34GiB to GPU in 2.44669s (13.8963GiB/s) | 20260508 11:53:15.639 | INFO | Analyzing chunk frames = [7896576,7897088) | 20260508 11:53:15.639 | INFO | MemInfo: { | 20260508 11:53:15.639 | INFO | "gpu" : " 17 GB", | 20260508 11:53:15.639 | INFO | "huge" : "", | 20260508 11:53:15.639 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:15.639 | INFO | "malloc" : " 14 GB", | 20260508 11:53:15.639 | INFO | "pinned" : " 29 GB" | 20260508 11:53:15.639 | INFO | } | 20260508 11:53:15.639 | INFO | Perf report for the last 5.13942 seconds: | 20260508 11:53:15.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:15.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.640 | INFO | ANALYSIS : 65.4%, 3.0, 102.3ms, 0 | 20260508 11:53:15.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.640 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:53:15.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:15.640 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:53:15.640 | INFO | Total mem usage: 16.7 GiB | 20260508 11:53:15.640 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:53:15.640 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 11:53:15.640 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:53:15.640 | INFO | High Water Breakdown: | 20260508 11:53:15.640 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:53:15.640 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:53:15.640 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:53:15.640 | INFO | Uploaded 36GiB to GPU in 2.44798s (14.706GiB/s) | 20260508 11:53:15.640 | INFO | ------------------------------------------ | 20260508 11:53:15.640 | INFO | Downloaded 34GiB to GPU in 2.42697s (14.0093GiB/s) | 20260508 11:53:15.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:53:15.640 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:53:15.640 | INFO | Total mem usage: 26.5 GiB | 20260508 11:53:15.640 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:53:15.640 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.14%) | 20260508 11:53:15.640 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260508 11:53:15.640 | INFO | High Water Breakdown: | 20260508 11:53:15.640 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:53:15.640 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:53:15.640 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:53:15.640 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:53:15.640 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:53:15.640 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:53:15.640 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:53:15.640 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:53:15.640 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:53:15.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:53:15.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:53:15.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:53:15.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:53:15.641 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:53:15.641 | INFO | ------------------------------------------ | 20260508 11:53:15.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:53:15.641 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:53:15.641 | INFO | Total mem usage: 27.8 GiB | 20260508 11:53:15.641 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:53:15.641 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:53:15.641 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:53:15.641 | INFO | High Water Breakdown: | 20260508 11:53:15.641 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:53:15.641 | INFO | ------------------------------------------ | 20260508 11:53:15.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:53:15.641 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:53:15.641 | INFO | Total mem usage: 15.7 GiB | 20260508 11:53:15.641 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:53:15.641 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:53:15.641 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:53:15.641 | INFO | High Water Breakdown: | 20260508 11:53:15.641 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:53:15.641 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:53:15.641 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:53:15.641 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:53:15.641 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:53:15.641 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:53:15.641 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:53:15.641 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:53:15.641 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:53:15.641 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:53:15.641 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:53:15.641 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:53:15.641 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:53:15.641 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:53:15.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:53:15.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:53:15.641 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:53:15.641 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:53:15.641 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:53:15.641 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:53:15.641 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:53:15.641 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:53:15.641 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:53:15.641 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:53:15.641 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:53:15.641 | INFO | ------------------------------------------ | 20260508 11:53:15.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:53:15.641 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:53:15.641 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:53:15.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:53:15.641 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.41e+01 / 7.67e-01 | 20260508 11:53:15.641 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.03e-01, 1.28e-02 / 5.55e-03 | 20260508 11:53:15.641 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.47e+00, 6.00e-01 / 1.97e-01 | 20260508 11:53:15.641 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.47e+00, 2.10e-01 / 1.19e-01 | 20260508 11:53:15.641 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.56e+01, 3.56e+00 / 9.62e-01 | 20260508 11:53:15.641 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.78e+00 / 1.09e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.52e+00, 4.05e-01 / 2.13e-01 | 20260508 11:53:15.641 | INFO | PerfReport: 2.98%, Metrics -- 9.73e-01 / 1.10e+00, 2.39e-01 / 9.72e-02 | 20260508 11:53:15.641 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.22e-01, 4.76e-02 / 2.40e-02 | 20260508 11:53:15.641 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:53:15.641 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:53:15.641 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:53:15.641 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 24.64%, Download -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 2.27%, LockUpload -- 2.38e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 2.48%, LockDownload -- 2.60e+00 / 0.00e+00, 3.93e+00 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 15.07%, LockCompute -- 1.58e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 11:53:15.641 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 11:53:15.736 | INFO | Starting a round of Baz IO | 20260508 11:53:19.144 | INFO | Populating RTMetrics report | 20260508 11:53:19.144 | INFO | RTMetrics report populated | 20260508 11:53:19.144 | INFO | Opening tmp RTMetrics json file | 20260508 11:53:19.153 | INFO | Serializing RTMetrics to json | 20260508 11:53:19.154 | INFO | Serialization done, swapping tmp file now... | 20260508 11:53:19.155 | INFO | RTMetrics done | 20260508 11:53:19.209 | INFO | Finished a round of Baz IO: Wrote 1.426 GiB in 3.472 seconds | 20260508 11:53:19.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:53:19.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180002% / 3.97364e-06% / 7.94729e-05% | 20260508 11:53:19.209 | INFO | Baz IO overall: | 20260508 11:53:19.209 | INFO | BytesWritten : 3397930984741 | 20260508 11:53:19.209 | INFO | NumEvents : 1631365281393 | 20260508 11:53:20.777 | INFO | Analyzing chunk frames = [7897088,7897600) | 20260508 11:53:20.777 | INFO | MemInfo: { | 20260508 11:53:20.777 | INFO | "gpu" : " 17 GB", | 20260508 11:53:20.777 | INFO | "huge" : "", | 20260508 11:53:20.777 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:20.777 | INFO | "malloc" : " 14 GB", | 20260508 11:53:20.777 | INFO | "pinned" : " 29 GB" | 20260508 11:53:20.777 | INFO | } | 20260508 11:53:20.777 | INFO | Perf report for the last 5.13782 seconds: | 20260508 11:53:20.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:20.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:20.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:20.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:20.777 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 11:53:20.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:20.777 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:53:20.777 | INFO | PRE_HQ : 65.6%, 1.6, 54.2ms, 0 | 20260508 11:53:20.778 | INFO | BAZWRITER : 66.9%, 2.5, 84.7ms, 0 | 20260508 11:53:20.778 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:53:20.778 | INFO | Uploaded 36GiB to GPU in 2.56843s (14.0164GiB/s) | 20260508 11:53:20.778 | INFO | Downloaded 38GiB to GPU in 2.7156s (13.9932GiB/s) | 20260508 11:53:25.916 | INFO | Analyzing chunk frames = [7897600,7898112) | 20260508 11:53:25.916 | INFO | MemInfo: { | 20260508 11:53:25.916 | INFO | "gpu" : " 17 GB", | 20260508 11:53:25.916 | INFO | "huge" : "", | 20260508 11:53:25.916 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:25.916 | INFO | "malloc" : " 14 GB", | 20260508 11:53:25.916 | INFO | "pinned" : " 29 GB" | 20260508 11:53:25.916 | INFO | } | 20260508 11:53:25.916 | INFO | Perf report for the last 5.1391 seconds: | 20260508 11:53:25.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:25.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.916 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:53:25.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.917 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 11:53:25.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:25.917 | INFO | Uploaded 36GiB to GPU in 2.48585s (14.482GiB/s) | 20260508 11:53:25.917 | INFO | Downloaded 34GiB to GPU in 2.45759s (13.8347GiB/s) | 20260508 11:53:31.058 | INFO | Analyzing chunk frames = [7898112,7898624) | 20260508 11:53:31.058 | INFO | MemInfo: { | 20260508 11:53:31.058 | INFO | "gpu" : " 17 GB", | 20260508 11:53:31.058 | INFO | "huge" : "", | 20260508 11:53:31.058 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:31.058 | INFO | "malloc" : " 14 GB", | 20260508 11:53:31.058 | INFO | "pinned" : " 29 GB" | 20260508 11:53:31.058 | INFO | } | 20260508 11:53:31.058 | INFO | Perf report for the last 5.14193 seconds: | 20260508 11:53:31.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:31.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.058 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:53:31.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.058 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:53:31.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:31.059 | INFO | Uploaded 36GiB to GPU in 2.51809s (14.2966GiB/s) | 20260508 11:53:31.059 | INFO | Downloaded 34GiB to GPU in 2.45077s (13.8732GiB/s) | 20260508 11:53:36.195 | INFO | Analyzing chunk frames = [7898624,7899136) | 20260508 11:53:36.195 | INFO | MemInfo: { | 20260508 11:53:36.195 | INFO | "gpu" : " 17 GB", | 20260508 11:53:36.195 | INFO | "huge" : "", | 20260508 11:53:36.195 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:36.195 | INFO | "malloc" : " 14 GB", | 20260508 11:53:36.195 | INFO | "pinned" : " 29 GB" | 20260508 11:53:36.195 | INFO | } | 20260508 11:53:36.195 | INFO | Perf report for the last 5.13651 seconds: | 20260508 11:53:36.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:36.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:53:36.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:53:36.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:36.195 | INFO | Uploaded 36GiB to GPU in 2.48934s (14.4616GiB/s) | 20260508 11:53:36.195 | INFO | Downloaded 34GiB to GPU in 2.45196s (13.8664GiB/s) | 20260508 11:53:41.334 | INFO | Analyzing chunk frames = [7899136,7899648) | 20260508 11:53:41.334 | INFO | MemInfo: { | 20260508 11:53:41.334 | INFO | "gpu" : " 17 GB", | 20260508 11:53:41.335 | INFO | "huge" : "", | 20260508 11:53:41.335 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:41.335 | INFO | "malloc" : " 14 GB", | 20260508 11:53:41.335 | INFO | "pinned" : " 29 GB" | 20260508 11:53:41.335 | INFO | } | 20260508 11:53:41.335 | INFO | Perf report for the last 5.13985 seconds: | 20260508 11:53:41.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:41.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 11:53:41.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:53:41.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:41.335 | INFO | Uploaded 36GiB to GPU in 2.30762s (15.6005GiB/s) | 20260508 11:53:41.335 | INFO | Downloaded 34GiB to GPU in 2.4785s (13.718GiB/s) | 20260508 11:53:46.473 | INFO | Analyzing chunk frames = [7899648,7900160) | 20260508 11:53:46.473 | INFO | MemInfo: { | 20260508 11:53:46.473 | INFO | "gpu" : " 17 GB", | 20260508 11:53:46.473 | INFO | "huge" : "", | 20260508 11:53:46.473 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:46.473 | INFO | "malloc" : " 14 GB", | 20260508 11:53:46.473 | INFO | "pinned" : " 29 GB" | 20260508 11:53:46.473 | INFO | } | 20260508 11:53:46.473 | INFO | Perf report for the last 5.13832 seconds: | 20260508 11:53:46.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:46.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:53:46.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:53:46.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:46.473 | INFO | Uploaded 36GiB to GPU in 2.40142s (14.9911GiB/s) | 20260508 11:53:46.473 | INFO | Downloaded 34GiB to GPU in 2.43548s (13.9603GiB/s) | 20260508 11:53:51.619 | INFO | Analyzing chunk frames = [7900160,7900672) | 20260508 11:53:51.619 | INFO | MemInfo: { | 20260508 11:53:51.619 | INFO | "gpu" : " 17 GB", | 20260508 11:53:51.619 | INFO | "huge" : "", | 20260508 11:53:51.619 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:51.619 | INFO | "malloc" : " 14 GB", | 20260508 11:53:51.620 | INFO | "pinned" : " 29 GB" | 20260508 11:53:51.620 | INFO | } | 20260508 11:53:51.620 | INFO | Perf report for the last 5.14657 seconds: | 20260508 11:53:51.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:51.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 11:53:51.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:53:51.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:51.620 | INFO | Uploaded 36GiB to GPU in 2.44189s (14.7427GiB/s) | 20260508 11:53:51.620 | INFO | Downloaded 34GiB to GPU in 2.44902s (13.8831GiB/s) | 20260508 11:53:56.755 | INFO | Analyzing chunk frames = [7900672,7901184) | 20260508 11:53:56.755 | INFO | MemInfo: { | 20260508 11:53:56.755 | INFO | "gpu" : " 17 GB", | 20260508 11:53:56.755 | INFO | "huge" : "", | 20260508 11:53:56.755 | INFO | "huge_shared" : " 29 GB", | 20260508 11:53:56.755 | INFO | "malloc" : " 14 GB", | 20260508 11:53:56.755 | INFO | "pinned" : " 29 GB" | 20260508 11:53:56.755 | INFO | } | 20260508 11:53:56.755 | INFO | Perf report for the last 5.1355 seconds: | 20260508 11:53:56.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:53:56.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.755 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 11:53:56.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.755 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:53:56.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:53:56.756 | INFO | Uploaded 36GiB to GPU in 2.47376s (14.5528GiB/s) | 20260508 11:53:56.756 | INFO | Downloaded 34GiB to GPU in 2.46337s (13.8022GiB/s) | 20260508 11:53:56.871 | INFO | Starting a round of Baz IO | 20260508 11:54:00.263 | INFO | Populating RTMetrics report | 20260508 11:54:00.264 | INFO | RTMetrics report populated | 20260508 11:54:00.264 | INFO | Opening tmp RTMetrics json file | 20260508 11:54:00.272 | INFO | Serializing RTMetrics to json | 20260508 11:54:00.273 | INFO | Serialization done, swapping tmp file now... | 20260508 11:54:00.274 | INFO | RTMetrics done | 20260508 11:54:00.330 | INFO | Finished a round of Baz IO: Wrote 1.424 GiB in 3.458 seconds | 20260508 11:54:00.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:54:00.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.178631% / 3.97364e-06% / 5.5631e-05% | 20260508 11:54:00.330 | INFO | Baz IO overall: | 20260508 11:54:00.330 | INFO | BytesWritten : 3399459575925 | 20260508 11:54:00.330 | INFO | NumEvents : 1632083836712 | 20260508 11:54:01.895 | INFO | Analyzing chunk frames = [7901184,7901696) | 20260508 11:54:01.895 | INFO | MemInfo: { | 20260508 11:54:01.895 | INFO | "gpu" : " 17 GB", | 20260508 11:54:01.895 | INFO | "huge" : "", | 20260508 11:54:01.895 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:01.895 | INFO | "malloc" : " 14 GB", | 20260508 11:54:01.895 | INFO | "pinned" : " 29 GB" | 20260508 11:54:01.895 | INFO | } | 20260508 11:54:01.895 | INFO | Perf report for the last 5.13977 seconds: | 20260508 11:54:01.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:01.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:01.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:01.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:01.895 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 11:54:01.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:01.895 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260508 11:54:01.895 | INFO | PRE_HQ : 66.5%, 1.8, 61.5ms, 0 | 20260508 11:54:01.895 | INFO | BAZWRITER : 66.7%, 2.7, 91.1ms, 0 | 20260508 11:54:01.895 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:54:01.895 | INFO | Uploaded 36GiB to GPU in 2.57262s (13.9935GiB/s) | 20260508 11:54:01.895 | INFO | Downloaded 38GiB to GPU in 2.71832s (13.9792GiB/s) | 20260508 11:54:07.027 | INFO | Analyzing chunk frames = [7901696,7902208) | 20260508 11:54:07.027 | INFO | MemInfo: { | 20260508 11:54:07.027 | INFO | "gpu" : " 17 GB", | 20260508 11:54:07.027 | INFO | "huge" : "", | 20260508 11:54:07.027 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:07.027 | INFO | "malloc" : " 14 GB", | 20260508 11:54:07.027 | INFO | "pinned" : " 29 GB" | 20260508 11:54:07.027 | INFO | } | 20260508 11:54:07.028 | INFO | Perf report for the last 5.13262 seconds: | 20260508 11:54:07.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:07.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 11:54:07.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:54:07.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:07.028 | INFO | Uploaded 36GiB to GPU in 2.46066s (14.6302GiB/s) | 20260508 11:54:07.028 | INFO | Downloaded 34GiB to GPU in 2.46591s (13.788GiB/s) | 20260508 11:54:12.166 | INFO | Analyzing chunk frames = [7902208,7902720) | 20260508 11:54:12.166 | INFO | MemInfo: { | 20260508 11:54:12.166 | INFO | "gpu" : " 17 GB", | 20260508 11:54:12.166 | INFO | "huge" : "", | 20260508 11:54:12.166 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:12.166 | INFO | "malloc" : " 14 GB", | 20260508 11:54:12.166 | INFO | "pinned" : " 29 GB" | 20260508 11:54:12.166 | INFO | } | 20260508 11:54:12.166 | INFO | Perf report for the last 5.13858 seconds: | 20260508 11:54:12.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:12.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.166 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:54:12.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.166 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:54:12.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:12.167 | INFO | Uploaded 36GiB to GPU in 2.49431s (14.4329GiB/s) | 20260508 11:54:12.167 | INFO | Downloaded 34GiB to GPU in 2.42893s (13.998GiB/s) | 20260508 11:54:17.313 | INFO | Analyzing chunk frames = [7902720,7903232) | 20260508 11:54:17.313 | INFO | MemInfo: { | 20260508 11:54:17.313 | INFO | "gpu" : " 17 GB", | 20260508 11:54:17.313 | INFO | "huge" : "", | 20260508 11:54:17.313 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:17.313 | INFO | "malloc" : " 14 GB", | 20260508 11:54:17.313 | INFO | "pinned" : " 29 GB" | 20260508 11:54:17.313 | INFO | } | 20260508 11:54:17.313 | INFO | Perf report for the last 5.1471 seconds: | 20260508 11:54:17.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:17.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.313 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 11:54:17.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.313 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:54:17.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:17.314 | INFO | Uploaded 36GiB to GPU in 2.47948s (14.5192GiB/s) | 20260508 11:54:17.314 | INFO | Downloaded 34GiB to GPU in 2.43713s (13.9508GiB/s) | 20260508 11:54:22.450 | INFO | Analyzing chunk frames = [7903232,7903744) | 20260508 11:54:22.450 | INFO | MemInfo: { | 20260508 11:54:22.450 | INFO | "gpu" : " 17 GB", | 20260508 11:54:22.450 | INFO | "huge" : "", | 20260508 11:54:22.450 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:22.451 | INFO | "malloc" : " 14 GB", | 20260508 11:54:22.451 | INFO | "pinned" : " 29 GB" | 20260508 11:54:22.451 | INFO | } | 20260508 11:54:22.451 | INFO | Perf report for the last 5.13733 seconds: | 20260508 11:54:22.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:22.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:54:22.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:54:22.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:22.451 | INFO | Uploaded 36GiB to GPU in 2.28938s (15.7248GiB/s) | 20260508 11:54:22.451 | INFO | Downloaded 34GiB to GPU in 2.50213s (13.5884GiB/s) | 20260508 11:54:27.592 | INFO | Analyzing chunk frames = [7903744,7904256) | 20260508 11:54:27.592 | INFO | MemInfo: { | 20260508 11:54:27.592 | INFO | "gpu" : " 17 GB", | 20260508 11:54:27.592 | INFO | "huge" : "", | 20260508 11:54:27.592 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:27.592 | INFO | "malloc" : " 14 GB", | 20260508 11:54:27.593 | INFO | "pinned" : " 29 GB" | 20260508 11:54:27.593 | INFO | } | 20260508 11:54:27.593 | INFO | Perf report for the last 5.14204 seconds: | 20260508 11:54:27.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:27.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:54:27.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:54:27.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:27.593 | INFO | Uploaded 36GiB to GPU in 2.41574s (14.9023GiB/s) | 20260508 11:54:27.593 | INFO | Downloaded 34GiB to GPU in 2.43299s (13.9746GiB/s) | 20260508 11:54:32.723 | INFO | Analyzing chunk frames = [7904256,7904768) | 20260508 11:54:32.723 | INFO | MemInfo: { | 20260508 11:54:32.723 | INFO | "gpu" : " 17 GB", | 20260508 11:54:32.723 | INFO | "huge" : "", | 20260508 11:54:32.723 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:32.724 | INFO | "malloc" : " 14 GB", | 20260508 11:54:32.724 | INFO | "pinned" : " 29 GB" | 20260508 11:54:32.724 | INFO | } | 20260508 11:54:32.724 | INFO | Perf report for the last 5.13137 seconds: | 20260508 11:54:32.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:32.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 11:54:32.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:54:32.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:32.724 | INFO | Uploaded 36GiB to GPU in 2.47371s (14.553GiB/s) | 20260508 11:54:32.724 | INFO | Downloaded 34GiB to GPU in 2.4455s (13.9031GiB/s) | 20260508 11:54:37.869 | INFO | Analyzing chunk frames = [7904768,7905280) | 20260508 11:54:37.869 | INFO | MemInfo: { | 20260508 11:54:37.869 | INFO | "gpu" : " 17 GB", | 20260508 11:54:37.869 | INFO | "huge" : "", | 20260508 11:54:37.869 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:37.869 | INFO | "malloc" : " 14 GB", | 20260508 11:54:37.869 | INFO | "pinned" : " 29 GB" | 20260508 11:54:37.869 | INFO | } | 20260508 11:54:37.869 | INFO | Perf report for the last 5.14519 seconds: | 20260508 11:54:37.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:37.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 11:54:37.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 11:54:37.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:37.869 | INFO | Uploaded 36GiB to GPU in 2.46503s (14.6043GiB/s) | 20260508 11:54:37.870 | INFO | Downloaded 34GiB to GPU in 2.43474s (13.9645GiB/s) | 20260508 11:54:37.870 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:54:37.870 | INFO | Total mem usage: 16.7 GiB | 20260508 11:54:37.870 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:54:37.870 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:54:37.870 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:54:37.870 | INFO | High Water Breakdown: | 20260508 11:54:37.870 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:54:37.870 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:54:37.870 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:54:37.870 | INFO | ------------------------------------------ | 20260508 11:54:37.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:54:37.870 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:54:37.870 | INFO | Total mem usage: 26.5 GiB | 20260508 11:54:37.870 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:54:37.870 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:54:37.870 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:54:37.870 | INFO | High Water Breakdown: | 20260508 11:54:37.870 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:54:37.870 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:54:37.870 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:54:37.870 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:54:37.870 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:54:37.870 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:54:37.870 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:54:37.870 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:54:37.870 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:54:37.870 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:54:37.870 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:54:37.870 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:54:37.870 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:54:37.870 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:54:37.870 | INFO | ------------------------------------------ | 20260508 11:54:37.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:54:37.870 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:54:37.870 | INFO | Total mem usage: 27.8 GiB | 20260508 11:54:37.870 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:54:37.870 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:54:37.870 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:54:37.870 | INFO | High Water Breakdown: | 20260508 11:54:37.870 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:54:37.870 | INFO | ------------------------------------------ | 20260508 11:54:37.870 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:54:37.870 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:54:37.870 | INFO | Total mem usage: 15.7 GiB | 20260508 11:54:37.870 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:54:37.870 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:54:37.870 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:54:37.870 | INFO | High Water Breakdown: | 20260508 11:54:37.870 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:54:37.870 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:54:37.870 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:54:37.870 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:54:37.870 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:54:37.870 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:54:37.870 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:54:37.870 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:54:37.870 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:54:37.870 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:54:37.870 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:54:37.871 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:54:37.871 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:54:37.871 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:54:37.871 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:54:37.871 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:54:37.871 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:54:37.871 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:54:37.871 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:54:37.871 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:54:37.871 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:54:37.871 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:54:37.871 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:54:37.871 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:54:37.871 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:54:37.871 | INFO | ------------------------------------------ | 20260508 11:54:37.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:54:37.871 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:54:37.871 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:54:37.871 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:54:37.871 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.39e+01 / 6.46e-01 | 20260508 11:54:37.871 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.11e-02 / 2.56e-03 | 20260508 11:54:37.871 | INFO | PerfReport: 12.23%, Baseline -- 3.98e+00 / 4.45e+00, 5.98e-01 / 2.31e-01 | 20260508 11:54:37.871 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.50e+00, 2.10e-01 / 9.47e-02 | 20260508 11:54:37.871 | INFO | PerfReport: 16.07%, DME -- 4.19e+01 / 4.60e+01, 3.70e+00 / 9.23e-01 | 20260508 11:54:37.871 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.85e+01 / 1.92e+01, 1.71e+00 / 6.66e-01 | 20260508 11:54:37.871 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.34e+00 / 2.60e+00, 4.05e-01 / 1.56e-01 | 20260508 11:54:37.871 | INFO | PerfReport: 3.00%, Metrics -- 9.76e-01 / 1.11e+00, 2.43e-01 / 9.15e-02 | 20260508 11:54:37.871 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.31e-01, 4.75e-02 / 2.45e-02 | 20260508 11:54:37.871 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:54:37.871 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:54:37.871 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:54:37.871 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 24.37%, Upload -- 2.56e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 24.64%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 30.98%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 2.23%, LockUpload -- 2.34e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 2.64%, LockDownload -- 2.78e+00 / 0.00e+00, 4.19e+00 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 15.12%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:54:37.871 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260508 11:54:37.966 | INFO | Starting a round of Baz IO | 20260508 11:54:41.365 | INFO | Populating RTMetrics report | 20260508 11:54:41.366 | INFO | RTMetrics report populated | 20260508 11:54:41.366 | INFO | Opening tmp RTMetrics json file | 20260508 11:54:41.375 | INFO | Serializing RTMetrics to json | 20260508 11:54:41.376 | INFO | Serialization done, swapping tmp file now... | 20260508 11:54:41.377 | INFO | RTMetrics done | 20260508 11:54:41.434 | INFO | Finished a round of Baz IO: Wrote 1.424 GiB in 3.467 seconds | 20260508 11:54:41.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:54:41.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.178349% / 0% / 9.13938e-05% | 20260508 11:54:41.434 | INFO | Baz IO overall: | 20260508 11:54:41.434 | INFO | BytesWritten : 3400988367255 | 20260508 11:54:41.434 | INFO | NumEvents : 1632802551010 | 20260508 11:54:43.008 | INFO | Analyzing chunk frames = [7905280,7905792) | 20260508 11:54:43.008 | INFO | MemInfo: { | 20260508 11:54:43.008 | INFO | "gpu" : " 17 GB", | 20260508 11:54:43.008 | INFO | "huge" : "", | 20260508 11:54:43.008 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:43.008 | INFO | "malloc" : " 14 GB", | 20260508 11:54:43.008 | INFO | "pinned" : " 29 GB" | 20260508 11:54:43.008 | INFO | } | 20260508 11:54:43.008 | INFO | Perf report for the last 5.13847 seconds: | 20260508 11:54:43.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:43.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:43.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:43.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:43.008 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 11:54:43.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:43.008 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 11:54:43.008 | INFO | PRE_HQ : 65.4%, 1.6, 54.0ms, 0 | 20260508 11:54:43.008 | INFO | BAZWRITER : 66.8%, 2.5, 84.3ms, 0 | 20260508 11:54:43.008 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:54:43.008 | INFO | Uploaded 36GiB to GPU in 2.62652s (13.7064GiB/s) | 20260508 11:54:43.008 | INFO | Downloaded 38GiB to GPU in 2.72297s (13.9553GiB/s) | 20260508 11:54:48.145 | INFO | Analyzing chunk frames = [7905792,7906304) | 20260508 11:54:48.145 | INFO | MemInfo: { | 20260508 11:54:48.145 | INFO | "gpu" : " 17 GB", | 20260508 11:54:48.145 | INFO | "huge" : "", | 20260508 11:54:48.145 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:48.145 | INFO | "malloc" : " 14 GB", | 20260508 11:54:48.145 | INFO | "pinned" : " 29 GB" | 20260508 11:54:48.145 | INFO | } | 20260508 11:54:48.145 | INFO | Perf report for the last 5.13712 seconds: | 20260508 11:54:48.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:48.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:54:48.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:54:48.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:48.145 | INFO | Uploaded 36GiB to GPU in 2.41685s (14.8954GiB/s) | 20260508 11:54:48.146 | INFO | Downloaded 34GiB to GPU in 2.45316s (13.8597GiB/s) | 20260508 11:54:53.288 | INFO | Analyzing chunk frames = [7906304,7906816) | 20260508 11:54:53.288 | INFO | MemInfo: { | 20260508 11:54:53.288 | INFO | "gpu" : " 17 GB", | 20260508 11:54:53.288 | INFO | "huge" : "", | 20260508 11:54:53.288 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:53.288 | INFO | "malloc" : " 14 GB", | 20260508 11:54:53.288 | INFO | "pinned" : " 29 GB" | 20260508 11:54:53.288 | INFO | } | 20260508 11:54:53.288 | INFO | Perf report for the last 5.1433 seconds: | 20260508 11:54:53.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:53.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 11:54:53.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:54:53.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:53.288 | INFO | Uploaded 36GiB to GPU in 2.46556s (14.6011GiB/s) | 20260508 11:54:53.288 | INFO | Downloaded 34GiB to GPU in 2.45044s (13.8751GiB/s) | 20260508 11:54:58.422 | INFO | Analyzing chunk frames = [7906816,7907328) | 20260508 11:54:58.422 | INFO | MemInfo: { | 20260508 11:54:58.422 | INFO | "gpu" : " 17 GB", | 20260508 11:54:58.422 | INFO | "huge" : "", | 20260508 11:54:58.422 | INFO | "huge_shared" : " 29 GB", | 20260508 11:54:58.422 | INFO | "malloc" : " 14 GB", | 20260508 11:54:58.422 | INFO | "pinned" : " 29 GB" | 20260508 11:54:58.422 | INFO | } | 20260508 11:54:58.422 | INFO | Perf report for the last 5.1341 seconds: | 20260508 11:54:58.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:54:58.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.422 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 11:54:58.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.422 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:54:58.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:54:58.423 | INFO | Uploaded 36GiB to GPU in 2.5159s (14.309GiB/s) | 20260508 11:54:58.423 | INFO | Downloaded 34GiB to GPU in 2.4696s (13.7674GiB/s) | 20260508 11:55:03.569 | INFO | Analyzing chunk frames = [7907328,7907840) | 20260508 11:55:03.569 | INFO | MemInfo: { | 20260508 11:55:03.569 | INFO | "gpu" : " 17 GB", | 20260508 11:55:03.569 | INFO | "huge" : "", | 20260508 11:55:03.569 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:03.569 | INFO | "malloc" : " 14 GB", | 20260508 11:55:03.569 | INFO | "pinned" : " 29 GB" | 20260508 11:55:03.569 | INFO | } | 20260508 11:55:03.569 | INFO | Perf report for the last 5.14704 seconds: | 20260508 11:55:03.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:03.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.569 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 11:55:03.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.569 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 11:55:03.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:03.570 | INFO | Uploaded 36GiB to GPU in 2.27501s (15.8241GiB/s) | 20260508 11:55:03.570 | INFO | Downloaded 34GiB to GPU in 2.50679s (13.5632GiB/s) | 20260508 11:55:08.704 | INFO | Analyzing chunk frames = [7907840,7908352) | 20260508 11:55:08.704 | INFO | MemInfo: { | 20260508 11:55:08.704 | INFO | "gpu" : " 17 GB", | 20260508 11:55:08.704 | INFO | "huge" : "", | 20260508 11:55:08.704 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:08.704 | INFO | "malloc" : " 14 GB", | 20260508 11:55:08.704 | INFO | "pinned" : " 29 GB" | 20260508 11:55:08.704 | INFO | } | 20260508 11:55:08.704 | INFO | Perf report for the last 5.13503 seconds: | 20260508 11:55:08.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:08.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.704 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:55:08.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.705 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 11:55:08.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:08.705 | INFO | Uploaded 36GiB to GPU in 2.3768s (15.1464GiB/s) | 20260508 11:55:08.705 | INFO | Downloaded 34GiB to GPU in 2.45926s (13.8253GiB/s) | 20260508 11:55:13.844 | INFO | Analyzing chunk frames = [7908352,7908864) | 20260508 11:55:13.844 | INFO | MemInfo: { | 20260508 11:55:13.844 | INFO | "gpu" : " 17 GB", | 20260508 11:55:13.844 | INFO | "huge" : "", | 20260508 11:55:13.844 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:13.844 | INFO | "malloc" : " 14 GB", | 20260508 11:55:13.844 | INFO | "pinned" : " 29 GB" | 20260508 11:55:13.844 | INFO | } | 20260508 11:55:13.844 | INFO | Perf report for the last 5.13973 seconds: | 20260508 11:55:13.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:13.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.844 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 11:55:13.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.844 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 11:55:13.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:13.845 | INFO | Uploaded 36GiB to GPU in 2.34313s (15.3641GiB/s) | 20260508 11:55:13.845 | INFO | Downloaded 34GiB to GPU in 2.45358s (13.8573GiB/s) | 20260508 11:55:18.976 | INFO | Analyzing chunk frames = [7908864,7909376) | 20260508 11:55:18.976 | INFO | MemInfo: { | 20260508 11:55:18.976 | INFO | "gpu" : " 17 GB", | 20260508 11:55:18.976 | INFO | "huge" : "", | 20260508 11:55:18.976 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:18.976 | INFO | "malloc" : " 14 GB", | 20260508 11:55:18.976 | INFO | "pinned" : " 29 GB" | 20260508 11:55:18.976 | INFO | } | 20260508 11:55:18.976 | INFO | Perf report for the last 5.13176 seconds: | 20260508 11:55:18.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:18.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.976 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 11:55:18.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.977 | INFO | PRE_HQ : 34.1%, 1.0, 17.7ms, 0 | 20260508 11:55:18.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:18.977 | INFO | Uploaded 36GiB to GPU in 2.361s (15.2477GiB/s) | 20260508 11:55:18.977 | INFO | Downloaded 34GiB to GPU in 2.46659s (13.7842GiB/s) | 20260508 11:55:19.100 | INFO | Starting a round of Baz IO | 20260508 11:55:22.493 | INFO | Populating RTMetrics report | 20260508 11:55:22.493 | INFO | RTMetrics report populated | 20260508 11:55:22.493 | INFO | Opening tmp RTMetrics json file | 20260508 11:55:22.494 | INFO | Serializing RTMetrics to json | 20260508 11:55:22.495 | INFO | Serialization done, swapping tmp file now... | 20260508 11:55:22.495 | INFO | RTMetrics done | 20260508 11:55:22.559 | INFO | Finished a round of Baz IO: Wrote 1.423 GiB in 3.458 seconds | 20260508 11:55:22.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.07% / 0.00% / 59.93% | 20260508 11:55:22.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17753% / 0% / 7.94729e-05% | 20260508 11:55:22.559 | INFO | Baz IO overall: | 20260508 11:55:22.559 | INFO | BytesWritten : 3402515841003 | 20260508 11:55:22.559 | INFO | NumEvents : 1633520359907 | 20260508 11:55:24.125 | INFO | Analyzing chunk frames = [7909376,7909888) | 20260508 11:55:24.125 | INFO | MemInfo: { | 20260508 11:55:24.125 | INFO | "gpu" : " 17 GB", | 20260508 11:55:24.125 | INFO | "huge" : "", | 20260508 11:55:24.125 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:24.125 | INFO | "malloc" : " 14 GB", | 20260508 11:55:24.125 | INFO | "pinned" : " 29 GB" | 20260508 11:55:24.125 | INFO | } | 20260508 11:55:24.125 | INFO | Perf report for the last 5.14911 seconds: | 20260508 11:55:24.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:24.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:24.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:24.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:24.125 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 11:55:24.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:24.125 | INFO | RT_METRICS : 41.8%, 1.0, 22.0ms, 0 | 20260508 11:55:24.125 | INFO | PRE_HQ : 66.5%, 1.7, 59.6ms, 0 | 20260508 11:55:24.125 | INFO | BAZWRITER : 66.8%, 2.6, 90.2ms, 0 | 20260508 11:55:24.125 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:55:24.126 | INFO | Uploaded 36GiB to GPU in 2.60274s (13.8316GiB/s) | 20260508 11:55:24.126 | INFO | Downloaded 38GiB to GPU in 2.72018s (13.9697GiB/s) | 20260508 11:55:29.266 | INFO | Analyzing chunk frames = [7909888,7910400) | 20260508 11:55:29.266 | INFO | MemInfo: { | 20260508 11:55:29.266 | INFO | "gpu" : " 17 GB", | 20260508 11:55:29.266 | INFO | "huge" : "", | 20260508 11:55:29.266 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:29.266 | INFO | "malloc" : " 14 GB", | 20260508 11:55:29.266 | INFO | "pinned" : " 29 GB" | 20260508 11:55:29.266 | INFO | } | 20260508 11:55:29.266 | INFO | Perf report for the last 5.14111 seconds: | 20260508 11:55:29.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:29.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.266 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:55:29.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.266 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:55:29.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:29.267 | INFO | Uploaded 36GiB to GPU in 2.47549s (14.5426GiB/s) | 20260508 11:55:29.267 | INFO | Downloaded 34GiB to GPU in 2.4684s (13.7741GiB/s) | 20260508 11:55:34.406 | INFO | Analyzing chunk frames = [7910400,7910912) | 20260508 11:55:34.406 | INFO | MemInfo: { | 20260508 11:55:34.406 | INFO | "gpu" : " 17 GB", | 20260508 11:55:34.406 | INFO | "huge" : "", | 20260508 11:55:34.406 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:34.406 | INFO | "malloc" : " 14 GB", | 20260508 11:55:34.406 | INFO | "pinned" : " 29 GB" | 20260508 11:55:34.406 | INFO | } | 20260508 11:55:34.406 | INFO | Perf report for the last 5.1396 seconds: | 20260508 11:55:34.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:34.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.406 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:55:34.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.406 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:55:34.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:34.407 | INFO | Uploaded 36GiB to GPU in 2.4845s (14.4899GiB/s) | 20260508 11:55:34.407 | INFO | Downloaded 34GiB to GPU in 2.44553s (13.9029GiB/s) | 20260508 11:55:39.538 | INFO | Analyzing chunk frames = [7910912,7911424) | 20260508 11:55:39.538 | INFO | MemInfo: { | 20260508 11:55:39.538 | INFO | "gpu" : " 17 GB", | 20260508 11:55:39.538 | INFO | "huge" : "", | 20260508 11:55:39.538 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:39.538 | INFO | "malloc" : " 14 GB", | 20260508 11:55:39.538 | INFO | "pinned" : " 29 GB" | 20260508 11:55:39.538 | INFO | } | 20260508 11:55:39.538 | INFO | Perf report for the last 5.13199 seconds: | 20260508 11:55:39.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:39.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:55:39.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:55:39.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:39.538 | INFO | Uploaded 36GiB to GPU in 2.54598s (14.1399GiB/s) | 20260508 11:55:39.538 | INFO | Downloaded 34GiB to GPU in 2.47438s (13.7408GiB/s) | 20260508 11:55:44.680 | INFO | Analyzing chunk frames = [7911424,7911936) | 20260508 11:55:44.680 | INFO | MemInfo: { | 20260508 11:55:44.680 | INFO | "gpu" : " 17 GB", | 20260508 11:55:44.680 | INFO | "huge" : "", | 20260508 11:55:44.680 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:44.680 | INFO | "malloc" : " 14 GB", | 20260508 11:55:44.680 | INFO | "pinned" : " 29 GB" | 20260508 11:55:44.680 | INFO | } | 20260508 11:55:44.680 | INFO | Perf report for the last 5.14258 seconds: | 20260508 11:55:44.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:44.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.680 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:55:44.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.680 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:55:44.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:44.681 | INFO | Uploaded 36GiB to GPU in 2.31448s (15.5543GiB/s) | 20260508 11:55:44.681 | INFO | Downloaded 34GiB to GPU in 2.49221s (13.6425GiB/s) | 20260508 11:55:49.816 | INFO | Analyzing chunk frames = [7911936,7912448) | 20260508 11:55:49.816 | INFO | MemInfo: { | 20260508 11:55:49.816 | INFO | "gpu" : " 17 GB", | 20260508 11:55:49.816 | INFO | "huge" : "", | 20260508 11:55:49.816 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:49.816 | INFO | "malloc" : " 14 GB", | 20260508 11:55:49.816 | INFO | "pinned" : " 29 GB" | 20260508 11:55:49.816 | INFO | } | 20260508 11:55:49.816 | INFO | Perf report for the last 5.13591 seconds: | 20260508 11:55:49.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:49.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.816 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 11:55:49.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.816 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:55:49.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:49.817 | INFO | Uploaded 36GiB to GPU in 2.41539s (14.9044GiB/s) | 20260508 11:55:49.817 | INFO | Downloaded 34GiB to GPU in 2.41681s (14.0681GiB/s) | 20260508 11:55:54.959 | INFO | Analyzing chunk frames = [7912448,7912960) | 20260508 11:55:54.959 | INFO | MemInfo: { | 20260508 11:55:54.959 | INFO | "gpu" : " 17 GB", | 20260508 11:55:54.959 | INFO | "huge" : "", | 20260508 11:55:54.959 | INFO | "huge_shared" : " 29 GB", | 20260508 11:55:54.959 | INFO | "malloc" : " 14 GB", | 20260508 11:55:54.959 | INFO | "pinned" : " 29 GB" | 20260508 11:55:54.959 | INFO | } | 20260508 11:55:54.960 | INFO | Perf report for the last 5.14322 seconds: | 20260508 11:55:54.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:55:54.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 11:55:54.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 11:55:54.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:55:54.960 | INFO | Uploaded 36GiB to GPU in 2.46396s (14.6106GiB/s) | 20260508 11:55:54.960 | INFO | Downloaded 34GiB to GPU in 2.43308s (13.9741GiB/s) | 20260508 11:56:00.098 | INFO | Analyzing chunk frames = [7912960,7913472) | 20260508 11:56:00.098 | INFO | MemInfo: { | 20260508 11:56:00.098 | INFO | "gpu" : " 17 GB", | 20260508 11:56:00.098 | INFO | "huge" : "", | 20260508 11:56:00.098 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:00.098 | INFO | "malloc" : " 14 GB", | 20260508 11:56:00.098 | INFO | "pinned" : " 29 GB" | 20260508 11:56:00.098 | INFO | } | 20260508 11:56:00.098 | INFO | Perf report for the last 5.13854 seconds: | 20260508 11:56:00.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:00.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 11:56:00.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260508 11:56:00.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:00.098 | INFO | Uploaded 36GiB to GPU in 2.49281s (14.4415GiB/s) | 20260508 11:56:00.099 | INFO | Downloaded 34GiB to GPU in 2.45835s (13.8304GiB/s) | 20260508 11:56:00.099 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:56:00.099 | INFO | Total mem usage: 16.7 GiB | 20260508 11:56:00.099 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:56:00.099 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:56:00.099 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:56:00.099 | INFO | High Water Breakdown: | 20260508 11:56:00.099 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:56:00.099 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:56:00.099 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:56:00.099 | INFO | ------------------------------------------ | 20260508 11:56:00.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:56:00.099 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:56:00.099 | INFO | Total mem usage: 26.5 GiB | 20260508 11:56:00.099 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:56:00.099 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:56:00.099 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:56:00.099 | INFO | High Water Breakdown: | 20260508 11:56:00.099 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:56:00.099 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:56:00.099 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:56:00.099 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:56:00.099 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:56:00.099 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:56:00.099 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:56:00.099 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:56:00.099 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:56:00.099 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:56:00.099 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:56:00.099 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:56:00.099 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:56:00.099 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:56:00.099 | INFO | ------------------------------------------ | 20260508 11:56:00.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:56:00.099 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:56:00.099 | INFO | Total mem usage: 27.8 GiB | 20260508 11:56:00.099 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:56:00.099 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:56:00.099 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:56:00.099 | INFO | High Water Breakdown: | 20260508 11:56:00.099 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:56:00.099 | INFO | ------------------------------------------ | 20260508 11:56:00.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:56:00.099 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:56:00.099 | INFO | Total mem usage: 15.7 GiB | 20260508 11:56:00.099 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:56:00.099 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:56:00.099 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:56:00.099 | INFO | High Water Breakdown: | 20260508 11:56:00.099 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:56:00.099 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:56:00.099 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:56:00.100 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:56:00.100 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:56:00.100 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:56:00.100 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:56:00.100 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:56:00.100 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:56:00.100 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:56:00.100 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:56:00.100 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:56:00.100 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:56:00.100 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:56:00.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:56:00.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:56:00.100 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:56:00.100 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:56:00.100 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:56:00.100 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:56:00.100 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:56:00.100 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:56:00.100 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:56:00.100 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:56:00.100 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:56:00.100 | INFO | ------------------------------------------ | 20260508 11:56:00.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:56:00.100 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:56:00.100 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:56:00.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:56:00.100 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.19e-01 | 20260508 11:56:00.100 | INFO | PerfReport: 0.30%, AggregateStats -- 9.87e-02 / 1.07e-01, 1.22e-02 / 6.01e-03 | 20260508 11:56:00.100 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.37e+00, 5.96e-01 / 3.10e-01 | 20260508 11:56:00.100 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.48e+00, 2.12e-01 / 1.19e-01 | 20260508 11:56:00.100 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.59e+01, 3.72e+00 / 1.10e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 1.95e+01, 1.85e+00 / 1.06e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.55e+00, 4.17e-01 / 2.40e-01 | 20260508 11:56:00.100 | INFO | PerfReport: 2.98%, Metrics -- 9.74e-01 / 1.08e+00, 2.44e-01 / 8.10e-02 | 20260508 11:56:00.100 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.97e-01 / 3.27e-01, 4.87e-02 / 2.47e-02 | 20260508 11:56:00.100 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:56:00.100 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:56:00.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:56:00.100 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 24.28%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 31.16%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 2.14%, LockUpload -- 2.25e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 2.55%, LockDownload -- 2.68e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 15.12%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 11:56:00.100 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260508 11:56:00.212 | INFO | Starting a round of Baz IO | 20260508 11:56:03.601 | INFO | Populating RTMetrics report | 20260508 11:56:03.601 | INFO | RTMetrics report populated | 20260508 11:56:03.601 | INFO | Opening tmp RTMetrics json file | 20260508 11:56:03.611 | INFO | Serializing RTMetrics to json | 20260508 11:56:03.613 | INFO | Serialization done, swapping tmp file now... | 20260508 11:56:03.613 | INFO | RTMetrics done | 20260508 11:56:03.672 | INFO | Finished a round of Baz IO: Wrote 1.424 GiB in 3.46 seconds | 20260508 11:56:03.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.93% | 20260508 11:56:03.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177948% / 3.97364e-06% / 7.94729e-05% | 20260508 11:56:03.672 | INFO | Baz IO overall: | 20260508 11:56:03.672 | INFO | BytesWritten : 3404044523039 | 20260508 11:56:03.672 | INFO | NumEvents : 1634239130598 | 20260508 11:56:05.234 | INFO | Analyzing chunk frames = [7913472,7913984) | 20260508 11:56:05.234 | INFO | MemInfo: { | 20260508 11:56:05.234 | INFO | "gpu" : " 17 GB", | 20260508 11:56:05.234 | INFO | "huge" : "", | 20260508 11:56:05.234 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:05.234 | INFO | "malloc" : " 14 GB", | 20260508 11:56:05.234 | INFO | "pinned" : " 29 GB" | 20260508 11:56:05.234 | INFO | } | 20260508 11:56:05.234 | INFO | Perf report for the last 5.13611 seconds: | 20260508 11:56:05.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:05.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:05.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:05.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:05.234 | INFO | ANALYSIS : 67.6%, 3.0, 105.2ms, 0 | 20260508 11:56:05.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:05.234 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260508 11:56:05.234 | INFO | PRE_HQ : 66.2%, 1.8, 62.5ms, 0 | 20260508 11:56:05.234 | INFO | BAZWRITER : 66.7%, 2.7, 93.1ms, 0 | 20260508 11:56:05.234 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:56:05.235 | INFO | Uploaded 36GiB to GPU in 2.62164s (13.7319GiB/s) | 20260508 11:56:05.235 | INFO | Downloaded 38GiB to GPU in 2.71521s (13.9953GiB/s) | 20260508 11:56:10.376 | INFO | Analyzing chunk frames = [7913984,7914496) | 20260508 11:56:10.376 | INFO | MemInfo: { | 20260508 11:56:10.376 | INFO | "gpu" : " 17 GB", | 20260508 11:56:10.376 | INFO | "huge" : "", | 20260508 11:56:10.376 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:10.376 | INFO | "malloc" : " 14 GB", | 20260508 11:56:10.376 | INFO | "pinned" : " 29 GB" | 20260508 11:56:10.376 | INFO | } | 20260508 11:56:10.376 | INFO | Perf report for the last 5.1419 seconds: | 20260508 11:56:10.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:10.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 11:56:10.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:56:10.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:10.376 | INFO | Uploaded 36GiB to GPU in 2.45178s (14.6832GiB/s) | 20260508 11:56:10.376 | INFO | Downloaded 34GiB to GPU in 2.47172s (13.7556GiB/s) | 20260508 11:56:15.521 | INFO | Analyzing chunk frames = [7914496,7915008) | 20260508 11:56:15.521 | INFO | MemInfo: { | 20260508 11:56:15.521 | INFO | "gpu" : " 17 GB", | 20260508 11:56:15.521 | INFO | "huge" : "", | 20260508 11:56:15.521 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:15.521 | INFO | "malloc" : " 14 GB", | 20260508 11:56:15.521 | INFO | "pinned" : " 29 GB" | 20260508 11:56:15.521 | INFO | } | 20260508 11:56:15.521 | INFO | Perf report for the last 5.14463 seconds: | 20260508 11:56:15.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:15.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 11:56:15.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:56:15.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:15.521 | INFO | Uploaded 36GiB to GPU in 2.47173s (14.5647GiB/s) | 20260508 11:56:15.521 | INFO | Downloaded 34GiB to GPU in 2.45011s (13.8769GiB/s) | 20260508 11:56:20.656 | INFO | Analyzing chunk frames = [7915008,7915520) | 20260508 11:56:20.656 | INFO | MemInfo: { | 20260508 11:56:20.656 | INFO | "gpu" : " 17 GB", | 20260508 11:56:20.656 | INFO | "huge" : "", | 20260508 11:56:20.656 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:20.656 | INFO | "malloc" : " 14 GB", | 20260508 11:56:20.656 | INFO | "pinned" : " 29 GB" | 20260508 11:56:20.656 | INFO | } | 20260508 11:56:20.656 | INFO | Perf report for the last 5.1354 seconds: | 20260508 11:56:20.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:20.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.656 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 11:56:20.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.656 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:56:20.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:20.657 | INFO | Uploaded 36GiB to GPU in 2.39395s (15.0379GiB/s) | 20260508 11:56:20.657 | INFO | Downloaded 34GiB to GPU in 2.47499s (13.7374GiB/s) | 20260508 11:56:25.790 | INFO | Analyzing chunk frames = [7915520,7916032) | 20260508 11:56:25.790 | INFO | MemInfo: { | 20260508 11:56:25.790 | INFO | "gpu" : " 17 GB", | 20260508 11:56:25.790 | INFO | "huge" : "", | 20260508 11:56:25.790 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:25.790 | INFO | "malloc" : " 14 GB", | 20260508 11:56:25.790 | INFO | "pinned" : " 29 GB" | 20260508 11:56:25.790 | INFO | } | 20260508 11:56:25.790 | INFO | Perf report for the last 5.13392 seconds: | 20260508 11:56:25.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:25.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.790 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 11:56:25.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.790 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:56:25.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:25.791 | INFO | Uploaded 36GiB to GPU in 2.31968s (15.5194GiB/s) | 20260508 11:56:25.791 | INFO | Downloaded 34GiB to GPU in 2.50014s (13.5992GiB/s) | 20260508 11:56:30.932 | INFO | Analyzing chunk frames = [7916032,7916544) | 20260508 11:56:30.932 | INFO | MemInfo: { | 20260508 11:56:30.932 | INFO | "gpu" : " 17 GB", | 20260508 11:56:30.932 | INFO | "huge" : "", | 20260508 11:56:30.932 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:30.932 | INFO | "malloc" : " 14 GB", | 20260508 11:56:30.932 | INFO | "pinned" : " 29 GB" | 20260508 11:56:30.932 | INFO | } | 20260508 11:56:30.932 | INFO | Perf report for the last 5.14205 seconds: | 20260508 11:56:30.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:30.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.932 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:56:30.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.932 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 11:56:30.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:30.933 | INFO | Uploaded 36GiB to GPU in 2.37694s (15.1456GiB/s) | 20260508 11:56:30.933 | INFO | Downloaded 34GiB to GPU in 2.44676s (13.8959GiB/s) | 20260508 11:56:36.073 | INFO | Analyzing chunk frames = [7916544,7917056) | 20260508 11:56:36.073 | INFO | MemInfo: { | 20260508 11:56:36.073 | INFO | "gpu" : " 17 GB", | 20260508 11:56:36.073 | INFO | "huge" : "", | 20260508 11:56:36.073 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:36.073 | INFO | "malloc" : " 14 GB", | 20260508 11:56:36.073 | INFO | "pinned" : " 29 GB" | 20260508 11:56:36.073 | INFO | } | 20260508 11:56:36.073 | INFO | Perf report for the last 5.14084 seconds: | 20260508 11:56:36.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:36.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.073 | INFO | ANALYSIS : 65.0%, 3.0, 101.1ms, 0 | 20260508 11:56:36.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.073 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 11:56:36.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:36.074 | INFO | Uploaded 36GiB to GPU in 2.35146s (15.3096GiB/s) | 20260508 11:56:36.074 | INFO | Downloaded 34GiB to GPU in 2.42054s (14.0465GiB/s) | 20260508 11:56:41.213 | INFO | Analyzing chunk frames = [7917056,7917568) | 20260508 11:56:41.213 | INFO | MemInfo: { | 20260508 11:56:41.213 | INFO | "gpu" : " 17 GB", | 20260508 11:56:41.213 | INFO | "huge" : "", | 20260508 11:56:41.213 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:41.213 | INFO | "malloc" : " 14 GB", | 20260508 11:56:41.213 | INFO | "pinned" : " 29 GB" | 20260508 11:56:41.213 | INFO | } | 20260508 11:56:41.213 | INFO | Perf report for the last 5.14018 seconds: | 20260508 11:56:41.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:41.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.213 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 11:56:41.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.213 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 11:56:41.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:41.214 | INFO | Uploaded 36GiB to GPU in 2.45932s (14.6382GiB/s) | 20260508 11:56:41.214 | INFO | Downloaded 34GiB to GPU in 2.45686s (13.8388GiB/s) | 20260508 11:56:41.311 | INFO | Starting a round of Baz IO | 20260508 11:56:44.722 | INFO | Populating RTMetrics report | 20260508 11:56:44.722 | INFO | RTMetrics report populated | 20260508 11:56:44.722 | INFO | Opening tmp RTMetrics json file | 20260508 11:56:44.731 | INFO | Serializing RTMetrics to json | 20260508 11:56:44.733 | INFO | Serialization done, swapping tmp file now... | 20260508 11:56:44.733 | INFO | RTMetrics done | 20260508 11:56:44.793 | INFO | Finished a round of Baz IO: Wrote 1.422 GiB in 3.482 seconds | 20260508 11:56:44.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 11:56:44.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176644% / 3.97364e-06% / 5.96046e-05% | 20260508 11:56:44.793 | INFO | Baz IO overall: | 20260508 11:56:44.793 | INFO | BytesWritten : 3405571623123 | 20260508 11:56:44.793 | INFO | NumEvents : 1634956556969 | 20260508 11:56:46.360 | INFO | Analyzing chunk frames = [7917568,7918080) | 20260508 11:56:46.360 | INFO | MemInfo: { | 20260508 11:56:46.360 | INFO | "gpu" : " 17 GB", | 20260508 11:56:46.360 | INFO | "huge" : "", | 20260508 11:56:46.360 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:46.360 | INFO | "malloc" : " 14 GB", | 20260508 11:56:46.360 | INFO | "pinned" : " 29 GB" | 20260508 11:56:46.360 | INFO | } | 20260508 11:56:46.360 | INFO | Perf report for the last 5.1471 seconds: | 20260508 11:56:46.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:46.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:46.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:46.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:46.360 | INFO | ANALYSIS : 67.5%, 3.0, 105.7ms, 0 | 20260508 11:56:46.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:46.360 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 11:56:46.360 | INFO | PRE_HQ : 65.2%, 1.6, 53.9ms, 0 | 20260508 11:56:46.360 | INFO | BAZWRITER : 66.7%, 2.5, 84.9ms, 0 | 20260508 11:56:46.360 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:56:46.361 | INFO | Uploaded 36GiB to GPU in 2.65051s (13.5823GiB/s) | 20260508 11:56:46.361 | INFO | Downloaded 38GiB to GPU in 2.70807s (14.0322GiB/s) | 20260508 11:56:51.490 | INFO | Analyzing chunk frames = [7918080,7918592) | 20260508 11:56:51.490 | INFO | MemInfo: { | 20260508 11:56:51.490 | INFO | "gpu" : " 17 GB", | 20260508 11:56:51.490 | INFO | "huge" : "", | 20260508 11:56:51.490 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:51.490 | INFO | "malloc" : " 14 GB", | 20260508 11:56:51.490 | INFO | "pinned" : " 29 GB" | 20260508 11:56:51.491 | INFO | } | 20260508 11:56:51.491 | INFO | Perf report for the last 5.13036 seconds: | 20260508 11:56:51.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:51.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 11:56:51.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 11:56:51.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:51.491 | INFO | Uploaded 36GiB to GPU in 2.48041s (14.5137GiB/s) | 20260508 11:56:51.491 | INFO | Downloaded 34GiB to GPU in 2.45333s (13.8587GiB/s) | 20260508 11:56:56.633 | INFO | Analyzing chunk frames = [7918592,7919104) | 20260508 11:56:56.633 | INFO | MemInfo: { | 20260508 11:56:56.633 | INFO | "gpu" : " 17 GB", | 20260508 11:56:56.633 | INFO | "huge" : "", | 20260508 11:56:56.633 | INFO | "huge_shared" : " 29 GB", | 20260508 11:56:56.633 | INFO | "malloc" : " 14 GB", | 20260508 11:56:56.633 | INFO | "pinned" : " 29 GB" | 20260508 11:56:56.633 | INFO | } | 20260508 11:56:56.633 | INFO | Perf report for the last 5.14196 seconds: | 20260508 11:56:56.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:56:56.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:56:56.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:56:56.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:56:56.633 | INFO | Uploaded 36GiB to GPU in 2.52238s (14.2722GiB/s) | 20260508 11:56:56.633 | INFO | Downloaded 34GiB to GPU in 2.45361s (13.8572GiB/s) | 20260508 11:57:01.768 | INFO | Analyzing chunk frames = [7919104,7919616) | 20260508 11:57:01.768 | INFO | MemInfo: { | 20260508 11:57:01.768 | INFO | "gpu" : " 17 GB", | 20260508 11:57:01.768 | INFO | "huge" : "", | 20260508 11:57:01.768 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:01.768 | INFO | "malloc" : " 14 GB", | 20260508 11:57:01.768 | INFO | "pinned" : " 29 GB" | 20260508 11:57:01.768 | INFO | } | 20260508 11:57:01.768 | INFO | Perf report for the last 5.13509 seconds: | 20260508 11:57:01.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:01.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 11:57:01.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:57:01.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:01.768 | INFO | Uploaded 36GiB to GPU in 2.49616s (14.4222GiB/s) | 20260508 11:57:01.768 | INFO | Downloaded 34GiB to GPU in 2.44129s (13.9271GiB/s) | 20260508 11:57:06.908 | INFO | Analyzing chunk frames = [7919616,7920128) | 20260508 11:57:06.908 | INFO | MemInfo: { | 20260508 11:57:06.908 | INFO | "gpu" : " 17 GB", | 20260508 11:57:06.908 | INFO | "huge" : "", | 20260508 11:57:06.908 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:06.908 | INFO | "malloc" : " 14 GB", | 20260508 11:57:06.908 | INFO | "pinned" : " 29 GB" | 20260508 11:57:06.908 | INFO | } | 20260508 11:57:06.908 | INFO | Perf report for the last 5.14016 seconds: | 20260508 11:57:06.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:06.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:57:06.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 11:57:06.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:06.908 | INFO | Uploaded 36GiB to GPU in 2.31156s (15.5739GiB/s) | 20260508 11:57:06.908 | INFO | Downloaded 34GiB to GPU in 2.49618s (13.6208GiB/s) | 20260508 11:57:12.045 | INFO | Analyzing chunk frames = [7920128,7920640) | 20260508 11:57:12.045 | INFO | MemInfo: { | 20260508 11:57:12.045 | INFO | "gpu" : " 17 GB", | 20260508 11:57:12.045 | INFO | "huge" : "", | 20260508 11:57:12.045 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:12.045 | INFO | "malloc" : " 14 GB", | 20260508 11:57:12.045 | INFO | "pinned" : " 29 GB" | 20260508 11:57:12.045 | INFO | } | 20260508 11:57:12.045 | INFO | Perf report for the last 5.13752 seconds: | 20260508 11:57:12.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:12.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.046 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 11:57:12.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.046 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 11:57:12.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:12.046 | INFO | Uploaded 36GiB to GPU in 2.38442s (15.098GiB/s) | 20260508 11:57:12.046 | INFO | Downloaded 34GiB to GPU in 2.42643s (14.0123GiB/s) | 20260508 11:57:17.187 | INFO | Analyzing chunk frames = [7920640,7921152) | 20260508 11:57:17.187 | INFO | MemInfo: { | 20260508 11:57:17.187 | INFO | "gpu" : " 17 GB", | 20260508 11:57:17.187 | INFO | "huge" : "", | 20260508 11:57:17.187 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:17.187 | INFO | "malloc" : " 14 GB", | 20260508 11:57:17.187 | INFO | "pinned" : " 29 GB" | 20260508 11:57:17.187 | INFO | } | 20260508 11:57:17.187 | INFO | Perf report for the last 5.142 seconds: | 20260508 11:57:17.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:17.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.188 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:57:17.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.188 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:57:17.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:17.188 | INFO | Uploaded 36GiB to GPU in 2.45462s (14.6662GiB/s) | 20260508 11:57:17.188 | INFO | Downloaded 34GiB to GPU in 2.44944s (13.8807GiB/s) | 20260508 11:57:22.332 | INFO | Analyzing chunk frames = [7921152,7921664) | 20260508 11:57:22.332 | INFO | MemInfo: { | 20260508 11:57:22.332 | INFO | "gpu" : " 17 GB", | 20260508 11:57:22.332 | INFO | "huge" : "", | 20260508 11:57:22.332 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:22.332 | INFO | "malloc" : " 14 GB", | 20260508 11:57:22.332 | INFO | "pinned" : " 29 GB" | 20260508 11:57:22.332 | INFO | } | 20260508 11:57:22.332 | INFO | Perf report for the last 5.14436 seconds: | 20260508 11:57:22.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:22.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 11:57:22.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:57:22.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:22.332 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:57:22.332 | INFO | Total mem usage: 16.7 GiB | 20260508 11:57:22.332 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:57:22.332 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 11:57:22.332 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:57:22.332 | INFO | High Water Breakdown: | 20260508 11:57:22.332 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:57:22.332 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:57:22.332 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:57:22.332 | INFO | ------------------------------------------ | 20260508 11:57:22.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:57:22.332 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:57:22.332 | INFO | Total mem usage: 26.5 GiB | 20260508 11:57:22.332 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:57:22.332 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 11:57:22.332 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 11:57:22.332 | INFO | High Water Breakdown: | 20260508 11:57:22.332 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:57:22.332 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:57:22.332 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:57:22.332 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:57:22.332 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:57:22.332 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:57:22.332 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:57:22.332 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:57:22.332 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:57:22.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:57:22.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:57:22.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:57:22.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:57:22.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:57:22.332 | INFO | ------------------------------------------ | 20260508 11:57:22.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:57:22.332 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:57:22.332 | INFO | Total mem usage: 27.8 GiB | 20260508 11:57:22.332 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:57:22.332 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:57:22.332 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:57:22.332 | INFO | High Water Breakdown: | 20260508 11:57:22.332 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:57:22.332 | INFO | ------------------------------------------ | 20260508 11:57:22.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:57:22.332 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:57:22.332 | INFO | Total mem usage: 15.7 GiB | 20260508 11:57:22.332 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:57:22.332 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:57:22.332 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:57:22.332 | INFO | High Water Breakdown: | 20260508 11:57:22.332 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:57:22.332 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:57:22.332 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:57:22.332 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:57:22.332 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:57:22.332 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:57:22.332 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:57:22.333 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:57:22.333 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:57:22.333 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:57:22.333 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:57:22.333 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:57:22.333 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:57:22.333 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:57:22.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:57:22.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:57:22.333 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:57:22.333 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:57:22.333 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:57:22.333 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:57:22.333 | INFO | Uploaded 36GiB to GPU in 2.4753s (14.5437GiB/s) | 20260508 11:57:22.333 | INFO | Downloaded 34GiB to GPU in 2.46482s (13.7941GiB/s) | 20260508 11:57:22.333 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:57:22.334 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:57:22.334 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:57:22.334 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:57:22.334 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:57:22.334 | INFO | ------------------------------------------ | 20260508 11:57:22.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:57:22.334 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:57:22.334 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:57:22.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:57:22.334 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.40e+01 / 5.13e-01 | 20260508 11:57:22.334 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.06e-01, 1.10e-02 / 5.07e-03 | 20260508 11:57:22.334 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.42e+00, 5.94e-01 / 1.89e-01 | 20260508 11:57:22.334 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.47e+00, 2.08e-01 / 8.11e-02 | 20260508 11:57:22.334 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.55e+01, 3.58e+00 / 7.13e-01 | 20260508 11:57:22.334 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.76e+00 / 9.82e-01 | 20260508 11:57:22.334 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.54e+00, 4.07e-01 / 1.47e-01 | 20260508 11:57:22.334 | INFO | PerfReport: 2.98%, Metrics -- 9.73e-01 / 1.08e+00, 2.40e-01 / 9.50e-02 | 20260508 11:57:22.334 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.30e-01, 4.77e-02 / 2.42e-02 | 20260508 11:57:22.334 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:57:22.334 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:57:22.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:57:22.334 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 24.33%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 24.71%, Download -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 31.10%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 2.15%, LockUpload -- 2.26e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 2.56%, LockDownload -- 2.69e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 15.14%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 11:57:22.334 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260508 11:57:22.430 | INFO | Starting a round of Baz IO | 20260508 11:57:25.841 | INFO | Populating RTMetrics report | 20260508 11:57:25.842 | INFO | RTMetrics report populated | 20260508 11:57:25.842 | INFO | Opening tmp RTMetrics json file | 20260508 11:57:25.843 | INFO | Serializing RTMetrics to json | 20260508 11:57:25.844 | INFO | Serialization done, swapping tmp file now... | 20260508 11:57:25.844 | INFO | RTMetrics done | 20260508 11:57:25.905 | INFO | Finished a round of Baz IO: Wrote 1.422 GiB in 3.474 seconds | 20260508 11:57:25.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 11:57:25.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176529% / 0% / 5.96046e-05% | 20260508 11:57:25.905 | INFO | Baz IO overall: | 20260508 11:57:25.905 | INFO | BytesWritten : 3407098347507 | 20260508 11:57:25.905 | INFO | NumEvents : 1635673883265 | 20260508 11:57:27.467 | INFO | Analyzing chunk frames = [7921664,7922176) | 20260508 11:57:27.467 | INFO | MemInfo: { | 20260508 11:57:27.467 | INFO | "gpu" : " 17 GB", | 20260508 11:57:27.467 | INFO | "huge" : "", | 20260508 11:57:27.467 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:27.467 | INFO | "malloc" : " 14 GB", | 20260508 11:57:27.467 | INFO | "pinned" : " 29 GB" | 20260508 11:57:27.467 | INFO | } | 20260508 11:57:27.467 | INFO | Perf report for the last 5.13485 seconds: | 20260508 11:57:27.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:27.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:27.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:27.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:27.467 | INFO | ANALYSIS : 68.0%, 3.0, 105.7ms, 0 | 20260508 11:57:27.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:27.467 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 11:57:27.467 | INFO | PRE_HQ : 65.5%, 1.6, 53.8ms, 0 | 20260508 11:57:27.467 | INFO | BAZWRITER : 67.0%, 2.5, 84.5ms, 0 | 20260508 11:57:27.467 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 11:57:27.467 | INFO | Uploaded 36GiB to GPU in 2.57228s (13.9953GiB/s) | 20260508 11:57:27.467 | INFO | Downloaded 38GiB to GPU in 2.73415s (13.8983GiB/s) | 20260508 11:57:32.605 | INFO | Analyzing chunk frames = [7922176,7922688) | 20260508 11:57:32.605 | INFO | MemInfo: { | 20260508 11:57:32.605 | INFO | "gpu" : " 17 GB", | 20260508 11:57:32.605 | INFO | "huge" : "", | 20260508 11:57:32.605 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:32.605 | INFO | "malloc" : " 14 GB", | 20260508 11:57:32.605 | INFO | "pinned" : " 29 GB" | 20260508 11:57:32.605 | INFO | } | 20260508 11:57:32.605 | INFO | Perf report for the last 5.13813 seconds: | 20260508 11:57:32.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:32.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.605 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 11:57:32.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.605 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 11:57:32.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:32.606 | INFO | Uploaded 36GiB to GPU in 2.44477s (14.7253GiB/s) | 20260508 11:57:32.606 | INFO | Downloaded 34GiB to GPU in 2.46519s (13.792GiB/s) | 20260508 11:57:37.736 | INFO | Analyzing chunk frames = [7922688,7923200) | 20260508 11:57:37.736 | INFO | MemInfo: { | 20260508 11:57:37.736 | INFO | "gpu" : " 17 GB", | 20260508 11:57:37.736 | INFO | "huge" : "", | 20260508 11:57:37.736 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:37.736 | INFO | "malloc" : " 14 GB", | 20260508 11:57:37.736 | INFO | "pinned" : " 29 GB" | 20260508 11:57:37.736 | INFO | } | 20260508 11:57:37.736 | INFO | Perf report for the last 5.13164 seconds: | 20260508 11:57:37.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:37.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 11:57:37.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:57:37.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:37.737 | INFO | Uploaded 36GiB to GPU in 2.52591s (14.2523GiB/s) | 20260508 11:57:37.737 | INFO | Downloaded 34GiB to GPU in 2.45664s (13.84GiB/s) | 20260508 11:57:42.882 | INFO | Analyzing chunk frames = [7923200,7923712) | 20260508 11:57:42.882 | INFO | MemInfo: { | 20260508 11:57:42.882 | INFO | "gpu" : " 17 GB", | 20260508 11:57:42.882 | INFO | "huge" : "", | 20260508 11:57:42.882 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:42.882 | INFO | "malloc" : " 14 GB", | 20260508 11:57:42.882 | INFO | "pinned" : " 29 GB" | 20260508 11:57:42.882 | INFO | } | 20260508 11:57:42.882 | INFO | Perf report for the last 5.14595 seconds: | 20260508 11:57:42.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:42.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 11:57:42.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 11:57:42.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:42.883 | INFO | Uploaded 36GiB to GPU in 2.4641s (14.6098GiB/s) | 20260508 11:57:42.884 | INFO | Downloaded 34GiB to GPU in 2.42585s (14.0157GiB/s) | 20260508 11:57:48.028 | INFO | Analyzing chunk frames = [7923712,7924224) | 20260508 11:57:48.028 | INFO | MemInfo: { | 20260508 11:57:48.028 | INFO | "gpu" : " 17 GB", | 20260508 11:57:48.028 | INFO | "huge" : "", | 20260508 11:57:48.028 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:48.028 | INFO | "malloc" : " 14 GB", | 20260508 11:57:48.028 | INFO | "pinned" : " 29 GB" | 20260508 11:57:48.028 | INFO | } | 20260508 11:57:48.028 | INFO | Perf report for the last 5.14536 seconds: | 20260508 11:57:48.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:48.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.029 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 11:57:48.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.029 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 11:57:48.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:48.029 | INFO | Uploaded 36GiB to GPU in 2.36876s (15.1978GiB/s) | 20260508 11:57:48.029 | INFO | Downloaded 34GiB to GPU in 2.51358s (13.5265GiB/s) | 20260508 11:57:53.164 | INFO | Analyzing chunk frames = [7924224,7924736) | 20260508 11:57:53.164 | INFO | MemInfo: { | 20260508 11:57:53.164 | INFO | "gpu" : " 17 GB", | 20260508 11:57:53.164 | INFO | "huge" : "", | 20260508 11:57:53.164 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:53.164 | INFO | "malloc" : " 14 GB", | 20260508 11:57:53.164 | INFO | "pinned" : " 29 GB" | 20260508 11:57:53.165 | INFO | } | 20260508 11:57:53.165 | INFO | Perf report for the last 5.13672 seconds: | 20260508 11:57:53.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:53.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 11:57:53.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:57:53.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:53.165 | INFO | Uploaded 36GiB to GPU in 2.44425s (14.7284GiB/s) | 20260508 11:57:53.165 | INFO | Downloaded 34GiB to GPU in 2.43233s (13.9784GiB/s) | 20260508 11:57:58.304 | INFO | Analyzing chunk frames = [7924736,7925248) | 20260508 11:57:58.304 | INFO | MemInfo: { | 20260508 11:57:58.304 | INFO | "gpu" : " 17 GB", | 20260508 11:57:58.304 | INFO | "huge" : "", | 20260508 11:57:58.304 | INFO | "huge_shared" : " 29 GB", | 20260508 11:57:58.304 | INFO | "malloc" : " 14 GB", | 20260508 11:57:58.304 | INFO | "pinned" : " 29 GB" | 20260508 11:57:58.304 | INFO | } | 20260508 11:57:58.304 | INFO | Perf report for the last 5.13918 seconds: | 20260508 11:57:58.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:57:58.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.304 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 11:57:58.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.304 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:57:58.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:57:58.305 | INFO | Uploaded 36GiB to GPU in 2.48308s (14.4981GiB/s) | 20260508 11:57:58.305 | INFO | Downloaded 34GiB to GPU in 2.4507s (13.8736GiB/s) | 20260508 11:58:03.442 | INFO | Analyzing chunk frames = [7925248,7925760) | 20260508 11:58:03.442 | INFO | MemInfo: { | 20260508 11:58:03.442 | INFO | "gpu" : " 17 GB", | 20260508 11:58:03.442 | INFO | "huge" : "", | 20260508 11:58:03.442 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:03.442 | INFO | "malloc" : " 14 GB", | 20260508 11:58:03.442 | INFO | "pinned" : " 29 GB" | 20260508 11:58:03.442 | INFO | } | 20260508 11:58:03.442 | INFO | Perf report for the last 5.13813 seconds: | 20260508 11:58:03.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:03.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 11:58:03.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 11:58:03.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:03.442 | INFO | Uploaded 36GiB to GPU in 2.4562s (14.6568GiB/s) | 20260508 11:58:03.442 | INFO | Downloaded 34GiB to GPU in 2.44309s (13.9168GiB/s) | 20260508 11:58:03.539 | INFO | Starting a round of Baz IO | 20260508 11:58:06.951 | INFO | Populating RTMetrics report | 20260508 11:58:06.951 | INFO | RTMetrics report populated | 20260508 11:58:06.951 | INFO | Opening tmp RTMetrics json file | 20260508 11:58:06.957 | INFO | Serializing RTMetrics to json | 20260508 11:58:06.959 | INFO | Serialization done, swapping tmp file now... | 20260508 11:58:06.959 | INFO | RTMetrics done | 20260508 11:58:07.020 | INFO | Finished a round of Baz IO: Wrote 1.421 GiB in 3.48 seconds | 20260508 11:58:07.020 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 11:58:07.020 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176211% / 3.97364e-06% / 6.35783e-05% | 20260508 11:58:07.020 | INFO | Baz IO overall: | 20260508 11:58:07.020 | INFO | BytesWritten : 3408623950761 | 20260508 11:58:07.020 | INFO | NumEvents : 1636390261610 | 20260508 11:58:08.580 | INFO | Analyzing chunk frames = [7925760,7926272) | 20260508 11:58:08.580 | INFO | MemInfo: { | 20260508 11:58:08.580 | INFO | "gpu" : " 17 GB", | 20260508 11:58:08.580 | INFO | "huge" : "", | 20260508 11:58:08.580 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:08.580 | INFO | "malloc" : " 14 GB", | 20260508 11:58:08.580 | INFO | "pinned" : " 29 GB" | 20260508 11:58:08.580 | INFO | } | 20260508 11:58:08.580 | INFO | Perf report for the last 5.13815 seconds: | 20260508 11:58:08.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:08.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:08.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:08.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:08.580 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 11:58:08.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:08.580 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:58:08.580 | INFO | PRE_HQ : 65.5%, 1.6, 54.1ms, 0 | 20260508 11:58:08.580 | INFO | BAZWRITER : 67.1%, 2.5, 84.6ms, 0 | 20260508 11:58:08.580 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 11:58:08.580 | INFO | Uploaded 36GiB to GPU in 2.62583s (13.71GiB/s) | 20260508 11:58:08.581 | INFO | Downloaded 38GiB to GPU in 2.71994s (13.9709GiB/s) | 20260508 11:58:13.721 | INFO | Analyzing chunk frames = [7926272,7926784) | 20260508 11:58:13.721 | INFO | MemInfo: { | 20260508 11:58:13.721 | INFO | "gpu" : " 17 GB", | 20260508 11:58:13.721 | INFO | "huge" : "", | 20260508 11:58:13.721 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:13.721 | INFO | "malloc" : " 14 GB", | 20260508 11:58:13.721 | INFO | "pinned" : " 29 GB" | 20260508 11:58:13.721 | INFO | } | 20260508 11:58:13.721 | INFO | Perf report for the last 5.14106 seconds: | 20260508 11:58:13.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:13.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:58:13.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 11:58:13.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:13.721 | INFO | Uploaded 36GiB to GPU in 2.43815s (14.7653GiB/s) | 20260508 11:58:13.722 | INFO | Downloaded 34GiB to GPU in 2.47545s (13.7349GiB/s) | 20260508 11:58:18.859 | INFO | Analyzing chunk frames = [7926784,7927296) | 20260508 11:58:18.859 | INFO | MemInfo: { | 20260508 11:58:18.859 | INFO | "gpu" : " 17 GB", | 20260508 11:58:18.859 | INFO | "huge" : "", | 20260508 11:58:18.859 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:18.859 | INFO | "malloc" : " 14 GB", | 20260508 11:58:18.859 | INFO | "pinned" : " 29 GB" | 20260508 11:58:18.859 | INFO | } | 20260508 11:58:18.859 | INFO | Perf report for the last 5.13803 seconds: | 20260508 11:58:18.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:18.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.859 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 11:58:18.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.859 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 11:58:18.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:18.860 | INFO | Uploaded 36GiB to GPU in 2.46305s (14.616GiB/s) | 20260508 11:58:18.860 | INFO | Downloaded 34GiB to GPU in 2.44327s (13.9158GiB/s) | 20260508 11:58:23.994 | INFO | Analyzing chunk frames = [7927296,7927808) | 20260508 11:58:23.994 | INFO | MemInfo: { | 20260508 11:58:23.994 | INFO | "gpu" : " 17 GB", | 20260508 11:58:23.994 | INFO | "huge" : "", | 20260508 11:58:23.994 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:23.994 | INFO | "malloc" : " 14 GB", | 20260508 11:58:23.994 | INFO | "pinned" : " 29 GB" | 20260508 11:58:23.994 | INFO | } | 20260508 11:58:23.994 | INFO | Perf report for the last 5.13489 seconds: | 20260508 11:58:23.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:23.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.994 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 11:58:23.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.994 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:58:23.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:23.995 | INFO | Uploaded 36GiB to GPU in 2.47438s (14.5491GiB/s) | 20260508 11:58:23.995 | INFO | Downloaded 34GiB to GPU in 2.46778s (13.7775GiB/s) | 20260508 11:58:29.139 | INFO | Analyzing chunk frames = [7927808,7928320) | 20260508 11:58:29.139 | INFO | MemInfo: { | 20260508 11:58:29.139 | INFO | "gpu" : " 17 GB", | 20260508 11:58:29.139 | INFO | "huge" : "", | 20260508 11:58:29.139 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:29.139 | INFO | "malloc" : " 14 GB", | 20260508 11:58:29.139 | INFO | "pinned" : " 29 GB" | 20260508 11:58:29.139 | INFO | } | 20260508 11:58:29.139 | INFO | Perf report for the last 5.14455 seconds: | 20260508 11:58:29.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:29.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 11:58:29.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 11:58:29.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:29.140 | INFO | Uploaded 36GiB to GPU in 2.35399s (15.2932GiB/s) | 20260508 11:58:29.140 | INFO | Downloaded 34GiB to GPU in 2.51312s (13.529GiB/s) | 20260508 11:58:34.282 | INFO | Analyzing chunk frames = [7928320,7928832) | 20260508 11:58:34.282 | INFO | MemInfo: { | 20260508 11:58:34.282 | INFO | "gpu" : " 17 GB", | 20260508 11:58:34.282 | INFO | "huge" : "", | 20260508 11:58:34.282 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:34.282 | INFO | "malloc" : " 14 GB", | 20260508 11:58:34.282 | INFO | "pinned" : " 29 GB" | 20260508 11:58:34.282 | INFO | } | 20260508 11:58:34.282 | INFO | Perf report for the last 5.14352 seconds: | 20260508 11:58:34.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:34.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 11:58:34.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 11:58:34.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:34.282 | INFO | Uploaded 36GiB to GPU in 2.3492s (15.3244GiB/s) | 20260508 11:58:34.282 | INFO | Downloaded 34GiB to GPU in 2.45417s (13.854GiB/s) | 20260508 11:58:39.422 | INFO | Analyzing chunk frames = [7928832,7929344) | 20260508 11:58:39.422 | INFO | MemInfo: { | 20260508 11:58:39.422 | INFO | "gpu" : " 17 GB", | 20260508 11:58:39.422 | INFO | "huge" : "", | 20260508 11:58:39.422 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:39.422 | INFO | "malloc" : " 14 GB", | 20260508 11:58:39.422 | INFO | "pinned" : " 29 GB" | 20260508 11:58:39.422 | INFO | } | 20260508 11:58:39.422 | INFO | Perf report for the last 5.14016 seconds: | 20260508 11:58:39.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:39.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.422 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 11:58:39.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.422 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 11:58:39.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:39.423 | INFO | Uploaded 36GiB to GPU in 2.42382s (14.8526GiB/s) | 20260508 11:58:39.423 | INFO | Downloaded 34GiB to GPU in 2.46572s (13.7891GiB/s) | 20260508 11:58:44.554 | INFO | Analyzing chunk frames = [7929344,7929856) | 20260508 11:58:44.554 | INFO | MemInfo: { | 20260508 11:58:44.554 | INFO | "gpu" : " 17 GB", | 20260508 11:58:44.554 | INFO | "huge" : "", | 20260508 11:58:44.554 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:44.554 | INFO | "malloc" : " 14 GB", | 20260508 11:58:44.554 | INFO | "pinned" : " 29 GB" | 20260508 11:58:44.554 | INFO | } | 20260508 11:58:44.554 | INFO | Perf report for the last 5.13181 seconds: | 20260508 11:58:44.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:44.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 11:58:44.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 11:58:44.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:44.554 | INFO | Uploaded 36GiB to GPU in 2.37594s (15.1519GiB/s) | 20260508 11:58:44.554 | INFO | Downloaded 34GiB to GPU in 2.46872s (13.7723GiB/s) | 20260508 11:58:44.555 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 11:58:44.555 | INFO | Total mem usage: 16.7 GiB | 20260508 11:58:44.555 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 11:58:44.555 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 11:58:44.555 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 11:58:44.555 | INFO | High Water Breakdown: | 20260508 11:58:44.555 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 11:58:44.555 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 11:58:44.555 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 11:58:44.555 | INFO | ------------------------------------------ | 20260508 11:58:44.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:58:44.555 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 11:58:44.555 | INFO | Total mem usage: 26.5 GiB | 20260508 11:58:44.555 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 11:58:44.555 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 11:58:44.555 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 11:58:44.555 | INFO | High Water Breakdown: | 20260508 11:58:44.555 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:58:44.555 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:58:44.555 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:58:44.555 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:58:44.555 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:58:44.555 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:58:44.555 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 11:58:44.555 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 11:58:44.555 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:58:44.555 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 11:58:44.555 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 11:58:44.555 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 11:58:44.555 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 11:58:44.555 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 11:58:44.555 | INFO | ------------------------------------------ | 20260508 11:58:44.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:58:44.555 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 11:58:44.555 | INFO | Total mem usage: 27.8 GiB | 20260508 11:58:44.555 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 11:58:44.555 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 11:58:44.555 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 11:58:44.555 | INFO | High Water Breakdown: | 20260508 11:58:44.555 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 11:58:44.555 | INFO | ------------------------------------------ | 20260508 11:58:44.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:58:44.555 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 11:58:44.555 | INFO | Total mem usage: 15.7 GiB | 20260508 11:58:44.555 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 11:58:44.555 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 11:58:44.555 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 11:58:44.555 | INFO | High Water Breakdown: | 20260508 11:58:44.555 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 11:58:44.555 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 11:58:44.555 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 11:58:44.555 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 11:58:44.555 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 11:58:44.555 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 11:58:44.555 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 11:58:44.555 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 11:58:44.555 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 11:58:44.555 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 11:58:44.555 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 11:58:44.555 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 11:58:44.555 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 11:58:44.555 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 11:58:44.555 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 11:58:44.555 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 11:58:44.555 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 11:58:44.555 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 11:58:44.555 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 11:58:44.555 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 11:58:44.555 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 11:58:44.555 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 11:58:44.555 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 11:58:44.555 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 11:58:44.555 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 11:58:44.555 | INFO | ------------------------------------------ | 20260508 11:58:44.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 11:58:44.555 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 11:58:44.555 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:58:44.555 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:58:44.555 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 5.11e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.06e-01, 1.09e-02 / 4.97e-03 | 20260508 11:58:44.555 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.35e+00, 5.94e-01 / 1.93e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.48e+00, 2.08e-01 / 1.00e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 16.02%, DME -- 4.18e+01 / 4.55e+01, 3.67e+00 / 9.26e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.79e+00 / 9.44e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.47e+00, 4.07e-01 / 1.96e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 2.97%, Metrics -- 9.69e-01 / 1.08e+00, 2.42e-01 / 1.02e-01 | 20260508 11:58:44.555 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.23e-01, 4.67e-02 / 2.26e-02 | 20260508 11:58:44.555 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 11:58:44.556 | INFO | PerfReport: Gathered 1536 samples. | 20260508 11:58:44.556 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 11:58:44.556 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 24.33%, Upload -- 2.56e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 31.06%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 2.13%, LockUpload -- 2.24e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 2.62%, LockDownload -- 2.75e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 15.11%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 11:58:44.556 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 5.58e-02 / 0.00e+00 | 20260508 11:58:44.651 | INFO | Starting a round of Baz IO | 20260508 11:58:48.071 | INFO | Populating RTMetrics report | 20260508 11:58:48.071 | INFO | RTMetrics report populated | 20260508 11:58:48.071 | INFO | Opening tmp RTMetrics json file | 20260508 11:58:48.077 | INFO | Serializing RTMetrics to json | 20260508 11:58:48.078 | INFO | Serialization done, swapping tmp file now... | 20260508 11:58:48.079 | INFO | RTMetrics done | 20260508 11:58:48.138 | INFO | Finished a round of Baz IO: Wrote 1.42 GiB in 3.487 seconds | 20260508 11:58:48.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 11:58:48.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17581% / 3.97364e-06% / 7.54992e-05% | 20260508 11:58:48.138 | INFO | Baz IO overall: | 20260508 11:58:48.138 | INFO | BytesWritten : 3410148231121 | 20260508 11:58:48.138 | INFO | NumEvents : 1637105775279 | 20260508 11:58:49.696 | INFO | Analyzing chunk frames = [7929856,7930368) | 20260508 11:58:49.696 | INFO | MemInfo: { | 20260508 11:58:49.696 | INFO | "gpu" : " 17 GB", | 20260508 11:58:49.696 | INFO | "huge" : "", | 20260508 11:58:49.696 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:49.696 | INFO | "malloc" : " 14 GB", | 20260508 11:58:49.696 | INFO | "pinned" : " 29 GB" | 20260508 11:58:49.696 | INFO | } | 20260508 11:58:49.696 | INFO | Perf report for the last 5.14204 seconds: | 20260508 11:58:49.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:49.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:49.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:49.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:49.696 | INFO | ANALYSIS : 67.6%, 3.0, 105.7ms, 0 | 20260508 11:58:49.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:49.696 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 11:58:49.696 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 11:58:49.696 | INFO | BAZWRITER : 66.9%, 2.5, 84.6ms, 0 | 20260508 11:58:49.696 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:58:49.697 | INFO | Uploaded 36GiB to GPU in 2.62049s (13.7379GiB/s) | 20260508 11:58:49.697 | INFO | Downloaded 38GiB to GPU in 2.7103s (14.0206GiB/s) | 20260508 11:58:54.833 | INFO | Analyzing chunk frames = [7930368,7930880) | 20260508 11:58:54.833 | INFO | MemInfo: { | 20260508 11:58:54.833 | INFO | "gpu" : " 17 GB", | 20260508 11:58:54.833 | INFO | "huge" : "", | 20260508 11:58:54.833 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:54.834 | INFO | "malloc" : " 14 GB", | 20260508 11:58:54.834 | INFO | "pinned" : " 29 GB" | 20260508 11:58:54.834 | INFO | } | 20260508 11:58:54.834 | INFO | Perf report for the last 5.13736 seconds: | 20260508 11:58:54.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:54.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 11:58:54.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 11:58:54.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:54.834 | INFO | Uploaded 36GiB to GPU in 2.42612s (14.8385GiB/s) | 20260508 11:58:54.834 | INFO | Downloaded 34GiB to GPU in 2.48287s (13.6938GiB/s) | 20260508 11:58:59.980 | INFO | Analyzing chunk frames = [7930880,7931392) | 20260508 11:58:59.980 | INFO | MemInfo: { | 20260508 11:58:59.980 | INFO | "gpu" : " 17 GB", | 20260508 11:58:59.980 | INFO | "huge" : "", | 20260508 11:58:59.980 | INFO | "huge_shared" : " 29 GB", | 20260508 11:58:59.980 | INFO | "malloc" : " 14 GB", | 20260508 11:58:59.980 | INFO | "pinned" : " 29 GB" | 20260508 11:58:59.980 | INFO | } | 20260508 11:58:59.980 | INFO | Perf report for the last 5.14623 seconds: | 20260508 11:58:59.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:58:59.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 11:58:59.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:58:59.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:58:59.980 | INFO | Uploaded 36GiB to GPU in 2.45269s (14.6778GiB/s) | 20260508 11:58:59.980 | INFO | Downloaded 34GiB to GPU in 2.44106s (13.9284GiB/s) | 20260508 11:59:05.117 | INFO | Analyzing chunk frames = [7931392,7931904) | 20260508 11:59:05.118 | INFO | MemInfo: { | 20260508 11:59:05.118 | INFO | "gpu" : " 17 GB", | 20260508 11:59:05.118 | INFO | "huge" : "", | 20260508 11:59:05.118 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:05.118 | INFO | "malloc" : " 14 GB", | 20260508 11:59:05.118 | INFO | "pinned" : " 29 GB" | 20260508 11:59:05.118 | INFO | } | 20260508 11:59:05.118 | INFO | Perf report for the last 5.13789 seconds: | 20260508 11:59:05.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:05.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 11:59:05.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:59:05.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:05.118 | INFO | Uploaded 36GiB to GPU in 2.51098s (14.337GiB/s) | 20260508 11:59:05.118 | INFO | Downloaded 34GiB to GPU in 2.47833s (13.7189GiB/s) | 20260508 11:59:10.249 | INFO | Analyzing chunk frames = [7931904,7932416) | 20260508 11:59:10.249 | INFO | MemInfo: { | 20260508 11:59:10.249 | INFO | "gpu" : " 17 GB", | 20260508 11:59:10.249 | INFO | "huge" : "", | 20260508 11:59:10.249 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:10.249 | INFO | "malloc" : " 14 GB", | 20260508 11:59:10.249 | INFO | "pinned" : " 29 GB" | 20260508 11:59:10.249 | INFO | } | 20260508 11:59:10.249 | INFO | Perf report for the last 5.13121 seconds: | 20260508 11:59:10.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:10.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 11:59:10.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:59:10.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:10.249 | INFO | Uploaded 36GiB to GPU in 2.28584s (15.7491GiB/s) | 20260508 11:59:10.249 | INFO | Downloaded 34GiB to GPU in 2.51696s (13.5084GiB/s) | 20260508 11:59:15.396 | INFO | Analyzing chunk frames = [7932416,7932928) | 20260508 11:59:15.396 | INFO | MemInfo: { | 20260508 11:59:15.396 | INFO | "gpu" : " 17 GB", | 20260508 11:59:15.396 | INFO | "huge" : "", | 20260508 11:59:15.396 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:15.396 | INFO | "malloc" : " 14 GB", | 20260508 11:59:15.396 | INFO | "pinned" : " 29 GB" | 20260508 11:59:15.396 | INFO | } | 20260508 11:59:15.397 | INFO | Perf report for the last 5.14767 seconds: | 20260508 11:59:15.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:15.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | ANALYSIS : 65.1%, 3.0, 101.5ms, 0 | 20260508 11:59:15.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 11:59:15.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:15.397 | INFO | Uploaded 36GiB to GPU in 2.36075s (15.2494GiB/s) | 20260508 11:59:15.397 | INFO | Downloaded 34GiB to GPU in 2.44653s (13.8972GiB/s) | 20260508 11:59:20.535 | INFO | Analyzing chunk frames = [7932928,7933440) | 20260508 11:59:20.535 | INFO | MemInfo: { | 20260508 11:59:20.535 | INFO | "gpu" : " 17 GB", | 20260508 11:59:20.535 | INFO | "huge" : "", | 20260508 11:59:20.535 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:20.535 | INFO | "malloc" : " 14 GB", | 20260508 11:59:20.535 | INFO | "pinned" : " 29 GB" | 20260508 11:59:20.535 | INFO | } | 20260508 11:59:20.535 | INFO | Perf report for the last 5.13854 seconds: | 20260508 11:59:20.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:20.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 11:59:20.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:59:20.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:20.535 | INFO | Uploaded 36GiB to GPU in 2.44282s (14.737GiB/s) | 20260508 11:59:20.536 | INFO | Downloaded 34GiB to GPU in 2.4746s (13.7396GiB/s) | 20260508 11:59:25.670 | INFO | Analyzing chunk frames = [7933440,7933952) | 20260508 11:59:25.670 | INFO | MemInfo: { | 20260508 11:59:25.670 | INFO | "gpu" : " 17 GB", | 20260508 11:59:25.670 | INFO | "huge" : "", | 20260508 11:59:25.670 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:25.670 | INFO | "malloc" : " 14 GB", | 20260508 11:59:25.670 | INFO | "pinned" : " 29 GB" | 20260508 11:59:25.670 | INFO | } | 20260508 11:59:25.670 | INFO | Perf report for the last 5.13437 seconds: | 20260508 11:59:25.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:25.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 11:59:25.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 11:59:25.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:25.670 | INFO | Uploaded 36GiB to GPU in 2.35553s (15.2832GiB/s) | 20260508 11:59:25.670 | INFO | Downloaded 34GiB to GPU in 2.46994s (13.7655GiB/s) | 20260508 11:59:25.767 | INFO | Starting a round of Baz IO | 20260508 11:59:29.177 | INFO | Populating RTMetrics report | 20260508 11:59:29.177 | INFO | RTMetrics report populated | 20260508 11:59:29.177 | INFO | Opening tmp RTMetrics json file | 20260508 11:59:29.188 | INFO | Serializing RTMetrics to json | 20260508 11:59:29.189 | INFO | Serialization done, swapping tmp file now... | 20260508 11:59:29.189 | INFO | RTMetrics done | 20260508 11:59:29.243 | INFO | Finished a round of Baz IO: Wrote 1.421 GiB in 3.475 seconds | 20260508 11:59:29.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 11:59:29.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.175595% / 0% / 3.97364e-05% | 20260508 11:59:29.243 | INFO | Baz IO overall: | 20260508 11:59:29.243 | INFO | BytesWritten : 3411673523690 | 20260508 11:59:29.243 | INFO | NumEvents : 1637821722636 | 20260508 11:59:30.805 | INFO | Analyzing chunk frames = [7933952,7934464) | 20260508 11:59:30.805 | INFO | MemInfo: { | 20260508 11:59:30.805 | INFO | "gpu" : " 17 GB", | 20260508 11:59:30.805 | INFO | "huge" : "", | 20260508 11:59:30.805 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:30.805 | INFO | "malloc" : " 14 GB", | 20260508 11:59:30.805 | INFO | "pinned" : " 29 GB" | 20260508 11:59:30.805 | INFO | } | 20260508 11:59:30.805 | INFO | Perf report for the last 5.13549 seconds: | 20260508 11:59:30.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:30.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:30.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:30.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:30.805 | INFO | ANALYSIS : 67.7%, 3.0, 105.4ms, 0 | 20260508 11:59:30.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:30.805 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 11:59:30.805 | INFO | PRE_HQ : 65.6%, 1.6, 54.0ms, 0 | 20260508 11:59:30.805 | INFO | BAZWRITER : 67.0%, 2.5, 84.4ms, 0 | 20260508 11:59:30.805 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 11:59:30.805 | INFO | Uploaded 36GiB to GPU in 2.552s (14.1066GiB/s) | 20260508 11:59:30.805 | INFO | Downloaded 38GiB to GPU in 2.72218s (13.9594GiB/s) | 20260508 11:59:35.951 | INFO | Analyzing chunk frames = [7934464,7934976) | 20260508 11:59:35.951 | INFO | MemInfo: { | 20260508 11:59:35.951 | INFO | "gpu" : " 17 GB", | 20260508 11:59:35.951 | INFO | "huge" : "", | 20260508 11:59:35.951 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:35.951 | INFO | "malloc" : " 14 GB", | 20260508 11:59:35.951 | INFO | "pinned" : " 29 GB" | 20260508 11:59:35.951 | INFO | } | 20260508 11:59:35.951 | INFO | Perf report for the last 5.14636 seconds: | 20260508 11:59:35.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:35.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.952 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 11:59:35.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.952 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 11:59:35.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:35.952 | INFO | Uploaded 36GiB to GPU in 2.42059s (14.8724GiB/s) | 20260508 11:59:35.952 | INFO | Downloaded 34GiB to GPU in 2.48157s (13.701GiB/s) | 20260508 11:59:41.091 | INFO | Analyzing chunk frames = [7934976,7935488) | 20260508 11:59:41.091 | INFO | MemInfo: { | 20260508 11:59:41.091 | INFO | "gpu" : " 17 GB", | 20260508 11:59:41.091 | INFO | "huge" : "", | 20260508 11:59:41.091 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:41.091 | INFO | "malloc" : " 14 GB", | 20260508 11:59:41.091 | INFO | "pinned" : " 29 GB" | 20260508 11:59:41.091 | INFO | } | 20260508 11:59:41.092 | INFO | Perf report for the last 5.14003 seconds: | 20260508 11:59:41.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:41.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 11:59:41.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 11:59:41.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:41.092 | INFO | Uploaded 36GiB to GPU in 2.45334s (14.6739GiB/s) | 20260508 11:59:41.092 | INFO | Downloaded 34GiB to GPU in 2.47199s (13.7541GiB/s) | 20260508 11:59:46.231 | INFO | Analyzing chunk frames = [7935488,7936000) | 20260508 11:59:46.231 | INFO | MemInfo: { | 20260508 11:59:46.231 | INFO | "gpu" : " 17 GB", | 20260508 11:59:46.231 | INFO | "huge" : "", | 20260508 11:59:46.231 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:46.231 | INFO | "malloc" : " 14 GB", | 20260508 11:59:46.231 | INFO | "pinned" : " 29 GB" | 20260508 11:59:46.231 | INFO | } | 20260508 11:59:46.231 | INFO | Perf report for the last 5.13951 seconds: | 20260508 11:59:46.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:46.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.231 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 11:59:46.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.231 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 11:59:46.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:46.232 | INFO | Uploaded 36GiB to GPU in 2.44605s (14.7176GiB/s) | 20260508 11:59:46.232 | INFO | Downloaded 34GiB to GPU in 2.46949s (13.768GiB/s) | 20260508 11:59:51.369 | INFO | Analyzing chunk frames = [7936000,7936512) | 20260508 11:59:51.369 | INFO | MemInfo: { | 20260508 11:59:51.369 | INFO | "gpu" : " 17 GB", | 20260508 11:59:51.369 | INFO | "huge" : "", | 20260508 11:59:51.369 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:51.369 | INFO | "malloc" : " 14 GB", | 20260508 11:59:51.369 | INFO | "pinned" : " 29 GB" | 20260508 11:59:51.369 | INFO | } | 20260508 11:59:51.369 | INFO | Perf report for the last 5.13791 seconds: | 20260508 11:59:51.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:51.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 11:59:51.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 11:59:51.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:51.369 | INFO | Uploaded 36GiB to GPU in 2.29501s (15.6862GiB/s) | 20260508 11:59:51.369 | INFO | Downloaded 34GiB to GPU in 2.50193s (13.5895GiB/s) | 20260508 11:59:56.503 | INFO | Analyzing chunk frames = [7936512,7937024) | 20260508 11:59:56.503 | INFO | MemInfo: { | 20260508 11:59:56.503 | INFO | "gpu" : " 17 GB", | 20260508 11:59:56.503 | INFO | "huge" : "", | 20260508 11:59:56.503 | INFO | "huge_shared" : " 29 GB", | 20260508 11:59:56.503 | INFO | "malloc" : " 14 GB", | 20260508 11:59:56.503 | INFO | "pinned" : " 29 GB" | 20260508 11:59:56.503 | INFO | } | 20260508 11:59:56.503 | INFO | Perf report for the last 5.13426 seconds: | 20260508 11:59:56.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 11:59:56.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.503 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 11:59:56.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.503 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 11:59:56.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 11:59:56.504 | INFO | Uploaded 36GiB to GPU in 2.38687s (15.0825GiB/s) | 20260508 11:59:56.504 | INFO | Downloaded 34GiB to GPU in 2.43233s (13.9784GiB/s) | 20260508 12:00:01.647 | INFO | Analyzing chunk frames = [7937024,7937536) | 20260508 12:00:01.647 | INFO | MemInfo: { | 20260508 12:00:01.647 | INFO | "gpu" : " 17 GB", | 20260508 12:00:01.647 | INFO | "huge" : "", | 20260508 12:00:01.647 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:01.647 | INFO | "malloc" : " 14 GB", | 20260508 12:00:01.647 | INFO | "pinned" : " 29 GB" | 20260508 12:00:01.647 | INFO | } | 20260508 12:00:01.647 | INFO | Perf report for the last 5.14349 seconds: | 20260508 12:00:01.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:01.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.647 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 12:00:01.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.647 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 12:00:01.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:01.648 | INFO | Uploaded 36GiB to GPU in 2.44643s (14.7153GiB/s) | 20260508 12:00:01.648 | INFO | Downloaded 34GiB to GPU in 2.45429s (13.8533GiB/s) | 20260508 12:00:06.792 | INFO | Analyzing chunk frames = [7937536,7938048) | 20260508 12:00:06.792 | INFO | MemInfo: { | 20260508 12:00:06.792 | INFO | "gpu" : " 17 GB", | 20260508 12:00:06.792 | INFO | "huge" : "", | 20260508 12:00:06.792 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:06.792 | INFO | "malloc" : " 14 GB", | 20260508 12:00:06.792 | INFO | "pinned" : " 29 GB" | 20260508 12:00:06.792 | INFO | } | 20260508 12:00:06.792 | INFO | Perf report for the last 5.14513 seconds: | 20260508 12:00:06.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:06.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 12:00:06.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:00:06.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:06.792 | INFO | Uploaded 36GiB to GPU in 2.45546s (14.6612GiB/s) | 20260508 12:00:06.792 | INFO | Downloaded 34GiB to GPU in 2.46898s (13.7709GiB/s) | 20260508 12:00:06.792 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:00:06.792 | INFO | Total mem usage: 16.7 GiB | 20260508 12:00:06.792 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:00:06.792 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:00:06.792 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:00:06.792 | INFO | High Water Breakdown: | 20260508 12:00:06.792 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:00:06.792 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:00:06.792 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:00:06.792 | INFO | ------------------------------------------ | 20260508 12:00:06.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:00:06.792 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:00:06.792 | INFO | Total mem usage: 26.5 GiB | 20260508 12:00:06.792 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:00:06.792 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:00:06.792 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:00:06.792 | INFO | High Water Breakdown: | 20260508 12:00:06.792 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:00:06.792 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:00:06.792 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:00:06.792 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:00:06.793 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:00:06.793 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:00:06.793 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:00:06.793 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:00:06.793 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:00:06.793 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:00:06.793 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:00:06.793 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:00:06.793 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:00:06.793 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:00:06.793 | INFO | ------------------------------------------ | 20260508 12:00:06.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:00:06.793 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:00:06.793 | INFO | Total mem usage: 27.8 GiB | 20260508 12:00:06.793 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:00:06.793 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:00:06.793 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:00:06.793 | INFO | High Water Breakdown: | 20260508 12:00:06.793 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:00:06.793 | INFO | ------------------------------------------ | 20260508 12:00:06.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:00:06.793 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:00:06.793 | INFO | Total mem usage: 15.7 GiB | 20260508 12:00:06.793 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:00:06.793 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:00:06.793 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:00:06.793 | INFO | High Water Breakdown: | 20260508 12:00:06.793 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:00:06.793 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:00:06.793 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:00:06.793 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:00:06.793 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:00:06.793 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:00:06.793 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:00:06.793 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:00:06.793 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:00:06.793 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:00:06.793 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:00:06.793 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:00:06.793 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:00:06.793 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:00:06.793 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:00:06.793 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:00:06.793 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:00:06.793 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:00:06.793 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:00:06.793 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:00:06.793 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:00:06.793 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:00:06.793 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:00:06.793 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:00:06.793 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:00:06.793 | INFO | ------------------------------------------ | 20260508 12:00:06.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:00:06.793 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:00:06.793 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:00:06.793 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:00:06.793 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.40e+01 / 5.80e-01 | 20260508 12:00:06.793 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.03e-01, 1.13e-02 / 4.64e-03 | 20260508 12:00:06.793 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.43e+00, 5.93e-01 / 2.55e-01 | 20260508 12:00:06.793 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.32e-01 | 20260508 12:00:06.793 | INFO | PerfReport: 16.05%, DME -- 4.19e+01 / 4.56e+01, 3.77e+00 / 1.01e+00 | 20260508 12:00:06.793 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 1.99e+01, 1.76e+00 / 1.30e+00 | 20260508 12:00:06.793 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.03e-01 / 2.55e-01 | 20260508 12:00:06.793 | INFO | PerfReport: 2.98%, Metrics -- 9.72e-01 / 1.10e+00, 2.36e-01 / 9.55e-02 | 20260508 12:00:06.793 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.26e-01, 4.76e-02 / 3.37e-02 | 20260508 12:00:06.793 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:00:06.794 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:00:06.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:00:06.794 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 24.14%, Upload -- 2.53e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 24.83%, Download -- 2.61e+01 / 0.00e+00, 8.27e+00 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 31.08%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 2.00%, LockUpload -- 2.10e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 2.77%, LockDownload -- 2.91e+00 / 0.00e+00, 4.34e+00 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 15.16%, LockCompute -- 1.59e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260508 12:00:06.794 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 12:00:06.889 | INFO | Starting a round of Baz IO | 20260508 12:00:10.298 | INFO | Populating RTMetrics report | 20260508 12:00:10.298 | INFO | RTMetrics report populated | 20260508 12:00:10.298 | INFO | Opening tmp RTMetrics json file | 20260508 12:00:10.306 | INFO | Serializing RTMetrics to json | 20260508 12:00:10.308 | INFO | Serialization done, swapping tmp file now... | 20260508 12:00:10.309 | INFO | RTMetrics done | 20260508 12:00:10.366 | INFO | Finished a round of Baz IO: Wrote 1.422 GiB in 3.477 seconds | 20260508 12:00:10.366 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 12:00:10.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176199% / 0% / 7.54992e-05% | 20260508 12:00:10.367 | INFO | Baz IO overall: | 20260508 12:00:10.367 | INFO | BytesWritten : 3413199849038 | 20260508 12:00:10.367 | INFO | NumEvents : 1638538657166 | 20260508 12:00:11.926 | INFO | Analyzing chunk frames = [7938048,7938560) | 20260508 12:00:11.926 | INFO | MemInfo: { | 20260508 12:00:11.926 | INFO | "gpu" : " 17 GB", | 20260508 12:00:11.926 | INFO | "huge" : "", | 20260508 12:00:11.926 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:11.926 | INFO | "malloc" : " 14 GB", | 20260508 12:00:11.926 | INFO | "pinned" : " 29 GB" | 20260508 12:00:11.926 | INFO | } | 20260508 12:00:11.926 | INFO | Perf report for the last 5.13382 seconds: | 20260508 12:00:11.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:11.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:11.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:11.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:11.926 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:00:11.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:11.926 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 12:00:11.926 | INFO | PRE_HQ : 65.3%, 1.6, 54.2ms, 0 | 20260508 12:00:11.926 | INFO | BAZWRITER : 67.0%, 2.5, 84.9ms, 0 | 20260508 12:00:11.926 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:00:11.927 | INFO | Uploaded 36GiB to GPU in 2.6303s (13.6866GiB/s) | 20260508 12:00:11.927 | INFO | Downloaded 38GiB to GPU in 2.71506s (13.996GiB/s) | 20260508 12:00:17.067 | INFO | Analyzing chunk frames = [7938560,7939072) | 20260508 12:00:17.067 | INFO | MemInfo: { | 20260508 12:00:17.067 | INFO | "gpu" : " 17 GB", | 20260508 12:00:17.067 | INFO | "huge" : "", | 20260508 12:00:17.067 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:17.067 | INFO | "malloc" : " 14 GB", | 20260508 12:00:17.067 | INFO | "pinned" : " 29 GB" | 20260508 12:00:17.067 | INFO | } | 20260508 12:00:17.067 | INFO | Perf report for the last 5.14153 seconds: | 20260508 12:00:17.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:17.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.067 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:00:17.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.067 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:00:17.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:17.068 | INFO | Uploaded 36GiB to GPU in 2.43591s (14.7789GiB/s) | 20260508 12:00:17.068 | INFO | Downloaded 34GiB to GPU in 2.47669s (13.728GiB/s) | 20260508 12:00:22.207 | INFO | Analyzing chunk frames = [7939072,7939584) | 20260508 12:00:22.207 | INFO | MemInfo: { | 20260508 12:00:22.207 | INFO | "gpu" : " 17 GB", | 20260508 12:00:22.207 | INFO | "huge" : "", | 20260508 12:00:22.207 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:22.207 | INFO | "malloc" : " 14 GB", | 20260508 12:00:22.207 | INFO | "pinned" : " 29 GB" | 20260508 12:00:22.207 | INFO | } | 20260508 12:00:22.207 | INFO | Perf report for the last 5.13997 seconds: | 20260508 12:00:22.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:22.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.207 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 12:00:22.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.207 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:00:22.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:22.208 | INFO | Uploaded 36GiB to GPU in 2.45715s (14.6511GiB/s) | 20260508 12:00:22.208 | INFO | Downloaded 34GiB to GPU in 2.46854s (13.7733GiB/s) | 20260508 12:00:27.346 | INFO | Analyzing chunk frames = [7939584,7940096) | 20260508 12:00:27.346 | INFO | MemInfo: { | 20260508 12:00:27.346 | INFO | "gpu" : " 17 GB", | 20260508 12:00:27.346 | INFO | "huge" : "", | 20260508 12:00:27.346 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:27.346 | INFO | "malloc" : " 14 GB", | 20260508 12:00:27.346 | INFO | "pinned" : " 29 GB" | 20260508 12:00:27.346 | INFO | } | 20260508 12:00:27.346 | INFO | Perf report for the last 5.13888 seconds: | 20260508 12:00:27.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:27.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 12:00:27.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:00:27.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:27.346 | INFO | Uploaded 36GiB to GPU in 2.52952s (14.2319GiB/s) | 20260508 12:00:27.347 | INFO | Downloaded 34GiB to GPU in 2.46511s (13.7925GiB/s) | 20260508 12:00:32.479 | INFO | Analyzing chunk frames = [7940096,7940608) | 20260508 12:00:32.479 | INFO | MemInfo: { | 20260508 12:00:32.479 | INFO | "gpu" : " 17 GB", | 20260508 12:00:32.479 | INFO | "huge" : "", | 20260508 12:00:32.479 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:32.479 | INFO | "malloc" : " 14 GB", | 20260508 12:00:32.479 | INFO | "pinned" : " 29 GB" | 20260508 12:00:32.480 | INFO | } | 20260508 12:00:32.480 | INFO | Perf report for the last 5.13377 seconds: | 20260508 12:00:32.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:32.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 12:00:32.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 12:00:32.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:32.480 | INFO | Uploaded 36GiB to GPU in 2.37907s (15.132GiB/s) | 20260508 12:00:32.480 | INFO | Downloaded 34GiB to GPU in 2.57314s (13.2134GiB/s) | 20260508 12:00:37.619 | INFO | Analyzing chunk frames = [7940608,7941120) | 20260508 12:00:37.619 | INFO | MemInfo: { | 20260508 12:00:37.619 | INFO | "gpu" : " 17 GB", | 20260508 12:00:37.619 | INFO | "huge" : "", | 20260508 12:00:37.619 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:37.619 | INFO | "malloc" : " 14 GB", | 20260508 12:00:37.619 | INFO | "pinned" : " 29 GB" | 20260508 12:00:37.619 | INFO | } | 20260508 12:00:37.619 | INFO | Perf report for the last 5.13926 seconds: | 20260508 12:00:37.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:37.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.619 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 12:00:37.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.619 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:00:37.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:37.620 | INFO | Uploaded 36GiB to GPU in 2.34708s (15.3382GiB/s) | 20260508 12:00:37.620 | INFO | Downloaded 34GiB to GPU in 2.45162s (13.8684GiB/s) | 20260508 12:00:42.760 | INFO | Analyzing chunk frames = [7941120,7941632) | 20260508 12:00:42.760 | INFO | MemInfo: { | 20260508 12:00:42.760 | INFO | "gpu" : " 17 GB", | 20260508 12:00:42.760 | INFO | "huge" : "", | 20260508 12:00:42.760 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:42.760 | INFO | "malloc" : " 14 GB", | 20260508 12:00:42.760 | INFO | "pinned" : " 29 GB" | 20260508 12:00:42.760 | INFO | } | 20260508 12:00:42.760 | INFO | Perf report for the last 5.14041 seconds: | 20260508 12:00:42.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:42.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 12:00:42.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:00:42.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:42.760 | INFO | Uploaded 36GiB to GPU in 2.369s (15.1963GiB/s) | 20260508 12:00:42.760 | INFO | Downloaded 34GiB to GPU in 2.45881s (13.8278GiB/s) | 20260508 12:00:47.903 | INFO | Analyzing chunk frames = [7941632,7942144) | 20260508 12:00:47.903 | INFO | MemInfo: { | 20260508 12:00:47.903 | INFO | "gpu" : " 17 GB", | 20260508 12:00:47.903 | INFO | "huge" : "", | 20260508 12:00:47.903 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:47.903 | INFO | "malloc" : " 14 GB", | 20260508 12:00:47.903 | INFO | "pinned" : " 29 GB" | 20260508 12:00:47.903 | INFO | } | 20260508 12:00:47.903 | INFO | Perf report for the last 5.14304 seconds: | 20260508 12:00:47.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:47.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 12:00:47.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:00:47.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:47.903 | INFO | Uploaded 36GiB to GPU in 2.41095s (14.9319GiB/s) | 20260508 12:00:47.903 | INFO | Downloaded 34GiB to GPU in 2.45916s (13.8258GiB/s) | 20260508 12:00:48.000 | INFO | Starting a round of Baz IO | 20260508 12:00:51.407 | INFO | Populating RTMetrics report | 20260508 12:00:51.407 | INFO | RTMetrics report populated | 20260508 12:00:51.407 | INFO | Opening tmp RTMetrics json file | 20260508 12:00:51.412 | INFO | Serializing RTMetrics to json | 20260508 12:00:51.413 | INFO | Serialization done, swapping tmp file now... | 20260508 12:00:51.414 | INFO | RTMetrics done | 20260508 12:00:51.471 | INFO | Finished a round of Baz IO: Wrote 1.423 GiB in 3.47 seconds | 20260508 12:00:51.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 12:00:51.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176199% / 0% / 7.54992e-05% | 20260508 12:00:51.471 | INFO | Baz IO overall: | 20260508 12:00:51.471 | INFO | BytesWritten : 3414727896559 | 20260508 12:00:51.471 | INFO | NumEvents : 1639257065016 | 20260508 12:00:53.045 | INFO | Analyzing chunk frames = [7942144,7942656) | 20260508 12:00:53.045 | INFO | MemInfo: { | 20260508 12:00:53.045 | INFO | "gpu" : " 17 GB", | 20260508 12:00:53.045 | INFO | "huge" : "", | 20260508 12:00:53.045 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:53.045 | INFO | "malloc" : " 14 GB", | 20260508 12:00:53.045 | INFO | "pinned" : " 29 GB" | 20260508 12:00:53.045 | INFO | } | 20260508 12:00:53.045 | INFO | Perf report for the last 5.14208 seconds: | 20260508 12:00:53.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:53.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:53.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:53.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:53.045 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 12:00:53.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:53.045 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:00:53.045 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:00:53.045 | INFO | BAZWRITER : 66.7%, 2.5, 84.1ms, 0 | 20260508 12:00:53.045 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:00:53.045 | INFO | Uploaded 36GiB to GPU in 2.58096s (13.9483GiB/s) | 20260508 12:00:53.045 | INFO | Downloaded 38GiB to GPU in 2.72417s (13.9492GiB/s) | 20260508 12:00:58.178 | INFO | Analyzing chunk frames = [7942656,7943168) | 20260508 12:00:58.178 | INFO | MemInfo: { | 20260508 12:00:58.178 | INFO | "gpu" : " 17 GB", | 20260508 12:00:58.178 | INFO | "huge" : "", | 20260508 12:00:58.178 | INFO | "huge_shared" : " 29 GB", | 20260508 12:00:58.178 | INFO | "malloc" : " 14 GB", | 20260508 12:00:58.178 | INFO | "pinned" : " 29 GB" | 20260508 12:00:58.178 | INFO | } | 20260508 12:00:58.178 | INFO | Perf report for the last 5.13299 seconds: | 20260508 12:00:58.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:00:58.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.178 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:00:58.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.178 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:00:58.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:00:58.179 | INFO | Uploaded 36GiB to GPU in 2.40172s (14.9893GiB/s) | 20260508 12:00:58.179 | INFO | Downloaded 34GiB to GPU in 2.45643s (13.8413GiB/s) | 20260508 12:01:03.313 | INFO | Analyzing chunk frames = [7943168,7943680) | 20260508 12:01:03.313 | INFO | MemInfo: { | 20260508 12:01:03.313 | INFO | "gpu" : " 17 GB", | 20260508 12:01:03.313 | INFO | "huge" : "", | 20260508 12:01:03.313 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:03.313 | INFO | "malloc" : " 14 GB", | 20260508 12:01:03.313 | INFO | "pinned" : " 29 GB" | 20260508 12:01:03.313 | INFO | } | 20260508 12:01:03.313 | INFO | Perf report for the last 5.13508 seconds: | 20260508 12:01:03.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:03.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 12:01:03.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:01:03.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:03.313 | INFO | Uploaded 36GiB to GPU in 2.47332s (14.5553GiB/s) | 20260508 12:01:03.314 | INFO | Downloaded 34GiB to GPU in 2.44958s (13.8799GiB/s) | 20260508 12:01:08.456 | INFO | Analyzing chunk frames = [7943680,7944192) | 20260508 12:01:08.456 | INFO | MemInfo: { | 20260508 12:01:08.456 | INFO | "gpu" : " 17 GB", | 20260508 12:01:08.456 | INFO | "huge" : "", | 20260508 12:01:08.456 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:08.456 | INFO | "malloc" : " 14 GB", | 20260508 12:01:08.456 | INFO | "pinned" : " 29 GB" | 20260508 12:01:08.456 | INFO | } | 20260508 12:01:08.457 | INFO | Perf report for the last 5.14365 seconds: | 20260508 12:01:08.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:08.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:01:08.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 12:01:08.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:08.457 | INFO | Uploaded 36GiB to GPU in 2.45308s (14.6754GiB/s) | 20260508 12:01:08.457 | INFO | Downloaded 34GiB to GPU in 2.47014s (13.7644GiB/s) | 20260508 12:01:13.599 | INFO | Analyzing chunk frames = [7944192,7944704) | 20260508 12:01:13.599 | INFO | MemInfo: { | 20260508 12:01:13.599 | INFO | "gpu" : " 17 GB", | 20260508 12:01:13.599 | INFO | "huge" : "", | 20260508 12:01:13.599 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:13.599 | INFO | "malloc" : " 14 GB", | 20260508 12:01:13.599 | INFO | "pinned" : " 29 GB" | 20260508 12:01:13.599 | INFO | } | 20260508 12:01:13.599 | INFO | Perf report for the last 5.14241 seconds: | 20260508 12:01:13.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:13.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:01:13.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:01:13.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:13.599 | INFO | Uploaded 36GiB to GPU in 2.38309s (15.1065GiB/s) | 20260508 12:01:13.599 | INFO | Downloaded 34GiB to GPU in 2.51491s (13.5194GiB/s) | 20260508 12:01:18.743 | INFO | Analyzing chunk frames = [7944704,7945216) | 20260508 12:01:18.743 | INFO | MemInfo: { | 20260508 12:01:18.743 | INFO | "gpu" : " 17 GB", | 20260508 12:01:18.743 | INFO | "huge" : "", | 20260508 12:01:18.743 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:18.743 | INFO | "malloc" : " 14 GB", | 20260508 12:01:18.743 | INFO | "pinned" : " 29 GB" | 20260508 12:01:18.743 | INFO | } | 20260508 12:01:18.744 | INFO | Perf report for the last 5.14456 seconds: | 20260508 12:01:18.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:18.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 12:01:18.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 12:01:18.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:18.744 | INFO | Uploaded 36GiB to GPU in 2.3696s (15.1925GiB/s) | 20260508 12:01:18.744 | INFO | Downloaded 34GiB to GPU in 2.46015s (13.8203GiB/s) | 20260508 12:01:23.882 | INFO | Analyzing chunk frames = [7945216,7945728) | 20260508 12:01:23.882 | INFO | MemInfo: { | 20260508 12:01:23.882 | INFO | "gpu" : " 17 GB", | 20260508 12:01:23.882 | INFO | "huge" : "", | 20260508 12:01:23.882 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:23.882 | INFO | "malloc" : " 14 GB", | 20260508 12:01:23.882 | INFO | "pinned" : " 29 GB" | 20260508 12:01:23.882 | INFO | } | 20260508 12:01:23.882 | INFO | Perf report for the last 5.13844 seconds: | 20260508 12:01:23.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:23.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:01:23.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:01:23.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:23.882 | INFO | Uploaded 36GiB to GPU in 2.41653s (14.8974GiB/s) | 20260508 12:01:23.882 | INFO | Downloaded 34GiB to GPU in 2.47916s (13.7143GiB/s) | 20260508 12:01:29.011 | INFO | Analyzing chunk frames = [7945728,7946240) | 20260508 12:01:29.011 | INFO | MemInfo: { | 20260508 12:01:29.011 | INFO | "gpu" : " 17 GB", | 20260508 12:01:29.011 | INFO | "huge" : "", | 20260508 12:01:29.011 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:29.011 | INFO | "malloc" : " 14 GB", | 20260508 12:01:29.011 | INFO | "pinned" : " 29 GB" | 20260508 12:01:29.011 | INFO | } | 20260508 12:01:29.011 | INFO | Perf report for the last 5.12937 seconds: | 20260508 12:01:29.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:29.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 12:01:29.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:01:29.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:29.012 | INFO | Uploaded 36GiB to GPU in 2.35616s (15.2791GiB/s) | 20260508 12:01:29.012 | INFO | Downloaded 34GiB to GPU in 2.45067s (13.8738GiB/s) | 20260508 12:01:29.012 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:01:29.012 | INFO | Total mem usage: 16.7 GiB | 20260508 12:01:29.012 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:01:29.012 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:01:29.012 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:01:29.012 | INFO | High Water Breakdown: | 20260508 12:01:29.012 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:01:29.012 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:01:29.012 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:01:29.012 | INFO | ------------------------------------------ | 20260508 12:01:29.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:01:29.012 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:01:29.012 | INFO | Total mem usage: 26.5 GiB | 20260508 12:01:29.012 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:01:29.012 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:01:29.012 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:01:29.012 | INFO | High Water Breakdown: | 20260508 12:01:29.012 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:01:29.012 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:01:29.012 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:01:29.012 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:01:29.012 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:01:29.012 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:01:29.012 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:01:29.012 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:01:29.012 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:01:29.012 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:01:29.012 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:01:29.012 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:01:29.012 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:01:29.012 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:01:29.012 | INFO | ------------------------------------------ | 20260508 12:01:29.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:01:29.012 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:01:29.012 | INFO | Total mem usage: 27.8 GiB | 20260508 12:01:29.012 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:01:29.012 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:01:29.012 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:01:29.012 | INFO | High Water Breakdown: | 20260508 12:01:29.013 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:01:29.013 | INFO | ------------------------------------------ | 20260508 12:01:29.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:01:29.013 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:01:29.013 | INFO | Total mem usage: 15.7 GiB | 20260508 12:01:29.013 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:01:29.013 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:01:29.013 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:01:29.013 | INFO | High Water Breakdown: | 20260508 12:01:29.013 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:01:29.013 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:01:29.013 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:01:29.013 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:01:29.013 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:01:29.013 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:01:29.013 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:01:29.013 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:01:29.013 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:01:29.013 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:01:29.013 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:01:29.013 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:01:29.013 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:01:29.013 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:01:29.013 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:01:29.013 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:01:29.013 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:01:29.013 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:01:29.013 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:01:29.013 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:01:29.013 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:01:29.013 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:01:29.013 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:01:29.013 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:01:29.013 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:01:29.013 | INFO | ------------------------------------------ | 20260508 12:01:29.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:01:29.013 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:01:29.013 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:01:29.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:01:29.013 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.37e+01 / 1.40e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.05e-01, 1.17e-02 / 4.80e-03 | 20260508 12:01:29.013 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.94e-01 / 2.50e-01 | 20260508 12:01:29.013 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.11e-01 | 20260508 12:01:29.013 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.52e+01, 3.69e+00 / 1.73e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.85e+00 / 1.34e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.32e+00 / 2.42e+00, 4.07e-01 / 2.83e-01 | 20260508 12:01:29.013 | INFO | PerfReport: 2.96%, Metrics -- 9.70e-01 / 1.10e+00, 2.42e-01 / 1.19e-01 | 20260508 12:01:29.013 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.23e-01, 4.76e-02 / 2.83e-02 | 20260508 12:01:29.013 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:01:29.013 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:01:29.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:01:29.013 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 24.18%, Upload -- 2.54e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 24.85%, Download -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 31.17%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 1.98%, LockUpload -- 2.07e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 2.68%, LockDownload -- 2.81e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 15.13%, LockCompute -- 1.59e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:01:29.013 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260508 12:01:29.109 | INFO | Starting a round of Baz IO | 20260508 12:01:32.521 | INFO | Populating RTMetrics report | 20260508 12:01:32.522 | INFO | RTMetrics report populated | 20260508 12:01:32.522 | INFO | Opening tmp RTMetrics json file | 20260508 12:01:32.523 | INFO | Serializing RTMetrics to json | 20260508 12:01:32.524 | INFO | Serialization done, swapping tmp file now... | 20260508 12:01:32.524 | INFO | RTMetrics done | 20260508 12:01:32.593 | INFO | Finished a round of Baz IO: Wrote 1.422 GiB in 3.483 seconds | 20260508 12:01:32.593 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 12:01:32.593 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176831% / 3.97364e-06% / 5.96046e-05% | 20260508 12:01:32.593 | INFO | Baz IO overall: | 20260508 12:01:32.593 | INFO | BytesWritten : 3416254868914 | 20260508 12:01:32.593 | INFO | NumEvents : 1639974822529 | 20260508 12:01:34.155 | INFO | Analyzing chunk frames = [7946240,7946752) | 20260508 12:01:34.155 | INFO | MemInfo: { | 20260508 12:01:34.155 | INFO | "gpu" : " 17 GB", | 20260508 12:01:34.155 | INFO | "huge" : "", | 20260508 12:01:34.155 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:34.155 | INFO | "malloc" : " 14 GB", | 20260508 12:01:34.155 | INFO | "pinned" : " 29 GB" | 20260508 12:01:34.155 | INFO | } | 20260508 12:01:34.155 | INFO | Perf report for the last 5.14415 seconds: | 20260508 12:01:34.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:34.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:34.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:34.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:34.156 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 12:01:34.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:34.156 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:01:34.156 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 12:01:34.156 | INFO | BAZWRITER : 66.9%, 2.4, 83.9ms, 0 | 20260508 12:01:34.156 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 12:01:34.156 | INFO | Uploaded 36GiB to GPU in 2.58266s (13.9391GiB/s) | 20260508 12:01:34.156 | INFO | Downloaded 38GiB to GPU in 2.73164s (13.911GiB/s) | 20260508 12:01:39.296 | INFO | Analyzing chunk frames = [7946752,7947264) | 20260508 12:01:39.296 | INFO | MemInfo: { | 20260508 12:01:39.296 | INFO | "gpu" : " 17 GB", | 20260508 12:01:39.296 | INFO | "huge" : "", | 20260508 12:01:39.296 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:39.296 | INFO | "malloc" : " 14 GB", | 20260508 12:01:39.296 | INFO | "pinned" : " 29 GB" | 20260508 12:01:39.296 | INFO | } | 20260508 12:01:39.296 | INFO | Perf report for the last 5.1409 seconds: | 20260508 12:01:39.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:39.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:01:39.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 12:01:39.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:39.297 | INFO | Uploaded 36GiB to GPU in 2.43635s (14.7762GiB/s) | 20260508 12:01:39.297 | INFO | Downloaded 34GiB to GPU in 2.48838s (13.6635GiB/s) | 20260508 12:01:44.441 | INFO | Analyzing chunk frames = [7947264,7947776) | 20260508 12:01:44.441 | INFO | MemInfo: { | 20260508 12:01:44.441 | INFO | "gpu" : " 17 GB", | 20260508 12:01:44.441 | INFO | "huge" : "", | 20260508 12:01:44.441 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:44.441 | INFO | "malloc" : " 14 GB", | 20260508 12:01:44.441 | INFO | "pinned" : " 29 GB" | 20260508 12:01:44.441 | INFO | } | 20260508 12:01:44.441 | INFO | Perf report for the last 5.14437 seconds: | 20260508 12:01:44.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:44.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 12:01:44.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:01:44.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:44.441 | INFO | Uploaded 36GiB to GPU in 2.46253s (14.6191GiB/s) | 20260508 12:01:44.441 | INFO | Downloaded 34GiB to GPU in 2.46467s (13.795GiB/s) | 20260508 12:01:49.568 | INFO | Analyzing chunk frames = [7947776,7948288) | 20260508 12:01:49.568 | INFO | MemInfo: { | 20260508 12:01:49.568 | INFO | "gpu" : " 17 GB", | 20260508 12:01:49.568 | INFO | "huge" : "", | 20260508 12:01:49.568 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:49.569 | INFO | "malloc" : " 14 GB", | 20260508 12:01:49.569 | INFO | "pinned" : " 29 GB" | 20260508 12:01:49.569 | INFO | } | 20260508 12:01:49.569 | INFO | Perf report for the last 5.12777 seconds: | 20260508 12:01:49.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:49.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | ANALYSIS : 65.5%, 3.0, 101.4ms, 0 | 20260508 12:01:49.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:01:49.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:49.569 | INFO | Uploaded 36GiB to GPU in 2.37671s (15.147GiB/s) | 20260508 12:01:49.569 | INFO | Downloaded 34GiB to GPU in 2.48793s (13.666GiB/s) | 20260508 12:01:54.713 | INFO | Analyzing chunk frames = [7948288,7948800) | 20260508 12:01:54.713 | INFO | MemInfo: { | 20260508 12:01:54.713 | INFO | "gpu" : " 17 GB", | 20260508 12:01:54.713 | INFO | "huge" : "", | 20260508 12:01:54.713 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:54.713 | INFO | "malloc" : " 14 GB", | 20260508 12:01:54.713 | INFO | "pinned" : " 29 GB" | 20260508 12:01:54.713 | INFO | } | 20260508 12:01:54.713 | INFO | Perf report for the last 5.14468 seconds: | 20260508 12:01:54.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:54.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 12:01:54.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:01:54.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:54.713 | INFO | Uploaded 36GiB to GPU in 2.3752s (15.1566GiB/s) | 20260508 12:01:54.714 | INFO | Downloaded 34GiB to GPU in 2.52721s (13.4536GiB/s) | 20260508 12:01:59.851 | INFO | Analyzing chunk frames = [7948800,7949312) | 20260508 12:01:59.851 | INFO | MemInfo: { | 20260508 12:01:59.851 | INFO | "gpu" : " 17 GB", | 20260508 12:01:59.851 | INFO | "huge" : "", | 20260508 12:01:59.851 | INFO | "huge_shared" : " 29 GB", | 20260508 12:01:59.851 | INFO | "malloc" : " 14 GB", | 20260508 12:01:59.851 | INFO | "pinned" : " 29 GB" | 20260508 12:01:59.851 | INFO | } | 20260508 12:01:59.851 | INFO | Perf report for the last 5.13818 seconds: | 20260508 12:01:59.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:01:59.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 12:01:59.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:01:59.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:01:59.852 | INFO | Uploaded 36GiB to GPU in 2.36239s (15.2388GiB/s) | 20260508 12:01:59.852 | INFO | Downloaded 34GiB to GPU in 2.43371s (13.9704GiB/s) | 20260508 12:02:04.991 | INFO | Analyzing chunk frames = [7949312,7949824) | 20260508 12:02:04.991 | INFO | MemInfo: { | 20260508 12:02:04.991 | INFO | "gpu" : " 17 GB", | 20260508 12:02:04.991 | INFO | "huge" : "", | 20260508 12:02:04.991 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:04.991 | INFO | "malloc" : " 14 GB", | 20260508 12:02:04.991 | INFO | "pinned" : " 29 GB" | 20260508 12:02:04.991 | INFO | } | 20260508 12:02:04.991 | INFO | Perf report for the last 5.13979 seconds: | 20260508 12:02:04.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:04.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.991 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:02:04.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.992 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:02:04.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:04.992 | INFO | Uploaded 36GiB to GPU in 2.44099s (14.7481GiB/s) | 20260508 12:02:04.992 | INFO | Downloaded 34GiB to GPU in 2.4628s (13.8054GiB/s) | 20260508 12:02:10.127 | INFO | Analyzing chunk frames = [7949824,7950336) | 20260508 12:02:10.127 | INFO | MemInfo: { | 20260508 12:02:10.127 | INFO | "gpu" : " 17 GB", | 20260508 12:02:10.127 | INFO | "huge" : "", | 20260508 12:02:10.127 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:10.127 | INFO | "malloc" : " 14 GB", | 20260508 12:02:10.127 | INFO | "pinned" : " 29 GB" | 20260508 12:02:10.127 | INFO | } | 20260508 12:02:10.127 | INFO | Perf report for the last 5.1362 seconds: | 20260508 12:02:10.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:10.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 12:02:10.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:02:10.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:10.128 | INFO | Uploaded 36GiB to GPU in 2.46086s (14.629GiB/s) | 20260508 12:02:10.128 | INFO | Downloaded 34GiB to GPU in 2.46342s (13.802GiB/s) | 20260508 12:02:10.224 | INFO | Starting a round of Baz IO | 20260508 12:02:13.636 | INFO | Populating RTMetrics report | 20260508 12:02:13.636 | INFO | RTMetrics report populated | 20260508 12:02:13.636 | INFO | Opening tmp RTMetrics json file | 20260508 12:02:13.646 | INFO | Serializing RTMetrics to json | 20260508 12:02:13.648 | INFO | Serialization done, swapping tmp file now... | 20260508 12:02:13.648 | INFO | RTMetrics done | 20260508 12:02:13.706 | INFO | Finished a round of Baz IO: Wrote 1.42 GiB in 3.481 seconds | 20260508 12:02:13.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.00% / 59.94% | 20260508 12:02:13.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.175393% / 0% / 6.75519e-05% | 20260508 12:02:13.706 | INFO | Baz IO overall: | 20260508 12:02:13.706 | INFO | BytesWritten : 3417780054270 | 20260508 12:02:13.706 | INFO | NumEvents : 1640691359241 | 20260508 12:02:15.264 | INFO | Analyzing chunk frames = [7950336,7950848) | 20260508 12:02:15.264 | INFO | MemInfo: { | 20260508 12:02:15.264 | INFO | "gpu" : " 17 GB", | 20260508 12:02:15.264 | INFO | "huge" : "", | 20260508 12:02:15.264 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:15.264 | INFO | "malloc" : " 14 GB", | 20260508 12:02:15.264 | INFO | "pinned" : " 29 GB" | 20260508 12:02:15.264 | INFO | } | 20260508 12:02:15.264 | INFO | Perf report for the last 5.13665 seconds: | 20260508 12:02:15.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:15.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:15.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:15.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:15.264 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:02:15.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:15.264 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 12:02:15.264 | INFO | PRE_HQ : 65.5%, 1.6, 55.7ms, 0 | 20260508 12:02:15.264 | INFO | BAZWRITER : 67.0%, 2.8, 96.1ms, 0 | 20260508 12:02:15.264 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 12:02:15.264 | INFO | Uploaded 36GiB to GPU in 2.60511s (13.819GiB/s) | 20260508 12:02:15.264 | INFO | Downloaded 38GiB to GPU in 2.69644s (14.0927GiB/s) | 20260508 12:02:20.408 | INFO | Analyzing chunk frames = [7950848,7951360) | 20260508 12:02:20.408 | INFO | MemInfo: { | 20260508 12:02:20.408 | INFO | "gpu" : " 17 GB", | 20260508 12:02:20.408 | INFO | "huge" : "", | 20260508 12:02:20.408 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:20.408 | INFO | "malloc" : " 14 GB", | 20260508 12:02:20.408 | INFO | "pinned" : " 29 GB" | 20260508 12:02:20.409 | INFO | } | 20260508 12:02:20.409 | INFO | Perf report for the last 5.1444 seconds: | 20260508 12:02:20.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:20.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:02:20.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:02:20.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:20.409 | INFO | Uploaded 36GiB to GPU in 2.42936s (14.8187GiB/s) | 20260508 12:02:20.409 | INFO | Downloaded 34GiB to GPU in 2.46109s (13.815GiB/s) | 20260508 12:02:25.551 | INFO | Analyzing chunk frames = [7951360,7951872) | 20260508 12:02:25.551 | INFO | MemInfo: { | 20260508 12:02:25.551 | INFO | "gpu" : " 17 GB", | 20260508 12:02:25.551 | INFO | "huge" : "", | 20260508 12:02:25.551 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:25.551 | INFO | "malloc" : " 14 GB", | 20260508 12:02:25.551 | INFO | "pinned" : " 29 GB" | 20260508 12:02:25.551 | INFO | } | 20260508 12:02:25.551 | INFO | Perf report for the last 5.14249 seconds: | 20260508 12:02:25.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:25.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 12:02:25.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:02:25.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:25.551 | INFO | Uploaded 36GiB to GPU in 2.47507s (14.545GiB/s) | 20260508 12:02:25.551 | INFO | Downloaded 34GiB to GPU in 2.43861s (13.9424GiB/s) | 20260508 12:02:30.691 | INFO | Analyzing chunk frames = [7951872,7952384) | 20260508 12:02:30.691 | INFO | MemInfo: { | 20260508 12:02:30.691 | INFO | "gpu" : " 17 GB", | 20260508 12:02:30.691 | INFO | "huge" : "", | 20260508 12:02:30.691 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:30.691 | INFO | "malloc" : " 14 GB", | 20260508 12:02:30.691 | INFO | "pinned" : " 29 GB" | 20260508 12:02:30.691 | INFO | } | 20260508 12:02:30.691 | INFO | Perf report for the last 5.14032 seconds: | 20260508 12:02:30.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:30.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:02:30.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:02:30.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:30.692 | INFO | Uploaded 36GiB to GPU in 2.46909s (14.5803GiB/s) | 20260508 12:02:30.692 | INFO | Downloaded 34GiB to GPU in 2.46487s (13.7938GiB/s) | 20260508 12:02:35.825 | INFO | Analyzing chunk frames = [7952384,7952896) | 20260508 12:02:35.825 | INFO | MemInfo: { | 20260508 12:02:35.825 | INFO | "gpu" : " 17 GB", | 20260508 12:02:35.825 | INFO | "huge" : "", | 20260508 12:02:35.825 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:35.825 | INFO | "malloc" : " 14 GB", | 20260508 12:02:35.825 | INFO | "pinned" : " 29 GB" | 20260508 12:02:35.825 | INFO | } | 20260508 12:02:35.825 | INFO | Perf report for the last 5.13381 seconds: | 20260508 12:02:35.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:35.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.825 | INFO | ANALYSIS : 66.8%, 3.0, 104.0ms, 0 | 20260508 12:02:35.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.825 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 12:02:35.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:35.826 | INFO | Uploaded 36GiB to GPU in 2.47125s (14.5675GiB/s) | 20260508 12:02:35.826 | INFO | Downloaded 34GiB to GPU in 2.54886s (13.3393GiB/s) | 20260508 12:02:40.970 | INFO | Analyzing chunk frames = [7952896,7953408) | 20260508 12:02:40.970 | INFO | MemInfo: { | 20260508 12:02:40.970 | INFO | "gpu" : " 17 GB", | 20260508 12:02:40.970 | INFO | "huge" : "", | 20260508 12:02:40.970 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:40.970 | INFO | "malloc" : " 14 GB", | 20260508 12:02:40.970 | INFO | "pinned" : " 29 GB" | 20260508 12:02:40.970 | INFO | } | 20260508 12:02:40.970 | INFO | Perf report for the last 5.14481 seconds: | 20260508 12:02:40.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:40.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.970 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:02:40.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.970 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:02:40.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:40.971 | INFO | Uploaded 36GiB to GPU in 2.36935s (15.194GiB/s) | 20260508 12:02:40.971 | INFO | Downloaded 34GiB to GPU in 2.43936s (13.9381GiB/s) | 20260508 12:02:46.105 | INFO | Analyzing chunk frames = [7953408,7953920) | 20260508 12:02:46.105 | INFO | MemInfo: { | 20260508 12:02:46.105 | INFO | "gpu" : " 17 GB", | 20260508 12:02:46.105 | INFO | "huge" : "", | 20260508 12:02:46.105 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:46.105 | INFO | "malloc" : " 14 GB", | 20260508 12:02:46.105 | INFO | "pinned" : " 29 GB" | 20260508 12:02:46.105 | INFO | } | 20260508 12:02:46.105 | INFO | Perf report for the last 5.13509 seconds: | 20260508 12:02:46.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:46.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.105 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 12:02:46.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.105 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 12:02:46.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:46.106 | INFO | Uploaded 36GiB to GPU in 2.4471s (14.7113GiB/s) | 20260508 12:02:46.106 | INFO | Downloaded 34GiB to GPU in 2.4592s (13.8256GiB/s) | 20260508 12:02:51.246 | INFO | Analyzing chunk frames = [7953920,7954432) | 20260508 12:02:51.246 | INFO | MemInfo: { | 20260508 12:02:51.246 | INFO | "gpu" : " 17 GB", | 20260508 12:02:51.246 | INFO | "huge" : "", | 20260508 12:02:51.246 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:51.246 | INFO | "malloc" : " 14 GB", | 20260508 12:02:51.246 | INFO | "pinned" : " 29 GB" | 20260508 12:02:51.246 | INFO | } | 20260508 12:02:51.246 | INFO | Perf report for the last 5.14114 seconds: | 20260508 12:02:51.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:51.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.246 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:02:51.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.246 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 12:02:51.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:51.247 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:02:51.247 | INFO | Total mem usage: 16.7 GiB | 20260508 12:02:51.247 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:02:51.247 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260508 12:02:51.247 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:02:51.247 | INFO | High Water Breakdown: | 20260508 12:02:51.247 | INFO | 60% : 8.77 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:02:51.247 | INFO | 39.9% : 5.83 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:02:51.247 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:02:51.247 | INFO | ------------------------------------------ | 20260508 12:02:51.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:02:51.247 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:02:51.247 | INFO | Total mem usage: 26.5 GiB | 20260508 12:02:51.247 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:02:51.247 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:02:51.247 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:02:51.247 | INFO | High Water Breakdown: | 20260508 12:02:51.247 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:02:51.247 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:02:51.247 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:02:51.247 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:02:51.247 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:02:51.247 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:02:51.247 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:02:51.247 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:02:51.247 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:02:51.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:02:51.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:02:51.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:02:51.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:02:51.247 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:02:51.247 | INFO | ------------------------------------------ | 20260508 12:02:51.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:02:51.247 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:02:51.247 | INFO | Total mem usage: 27.8 GiB | 20260508 12:02:51.247 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:02:51.247 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:02:51.247 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:02:51.247 | INFO | High Water Breakdown: | 20260508 12:02:51.247 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:02:51.247 | INFO | ------------------------------------------ | 20260508 12:02:51.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:02:51.247 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:02:51.247 | INFO | Total mem usage: 15.7 GiB | 20260508 12:02:51.247 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:02:51.247 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:02:51.247 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:02:51.247 | INFO | High Water Breakdown: | 20260508 12:02:51.247 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:02:51.247 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:02:51.247 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:02:51.247 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:02:51.247 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:02:51.247 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:02:51.247 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:02:51.247 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:02:51.247 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:02:51.247 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:02:51.247 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:02:51.247 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:02:51.247 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:02:51.247 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:02:51.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:02:51.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:02:51.247 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:02:51.247 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:02:51.247 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:02:51.247 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:02:51.247 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:02:51.247 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:02:51.247 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:02:51.247 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:02:51.247 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:02:51.247 | INFO | ------------------------------------------ | 20260508 12:02:51.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:02:51.247 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:02:51.247 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:02:51.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:02:51.247 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.38e+01 / 5.80e-01 | 20260508 12:02:51.247 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.15e-02 / 7.51e-03 | 20260508 12:02:51.247 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.41e+00, 5.91e-01 / 2.23e-01 | 20260508 12:02:51.247 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 9.98e-02 | 20260508 12:02:51.247 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.50e+01, 3.78e+00 / 9.28e-01 | 20260508 12:02:51.247 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.83e+00 / 1.33e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.47e+00, 4.07e-01 / 1.85e-01 | 20260508 12:02:51.247 | INFO | PerfReport: 2.97%, Metrics -- 9.69e-01 / 1.13e+00, 2.41e-01 / 1.09e-01 | 20260508 12:02:51.247 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.27e-01, 4.74e-02 / 2.23e-02 | 20260508 12:02:51.247 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:02:51.247 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:02:51.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:02:51.247 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 24.29%, Upload -- 2.55e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 24.79%, Download -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 2.03%, LockUpload -- 2.14e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 2.65%, LockDownload -- 2.78e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 15.17%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 12:02:51.247 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.12e-02 / 0.00e+00 | 20260508 12:02:51.247 | INFO | Uploaded 36GiB to GPU in 2.44744s (14.7092GiB/s) | 20260508 12:02:51.247 | INFO | Downloaded 34GiB to GPU in 2.4488s (13.8843GiB/s) | 20260508 12:02:51.344 | INFO | Starting a round of Baz IO | 20260508 12:02:54.758 | INFO | Populating RTMetrics report | 20260508 12:02:54.759 | INFO | RTMetrics report populated | 20260508 12:02:54.759 | INFO | Opening tmp RTMetrics json file | 20260508 12:02:54.760 | INFO | Serializing RTMetrics to json | 20260508 12:02:54.761 | INFO | Serialization done, swapping tmp file now... | 20260508 12:02:54.761 | INFO | RTMetrics done | 20260508 12:02:54.824 | INFO | Finished a round of Baz IO: Wrote 1.421 GiB in 3.479 seconds | 20260508 12:02:54.824 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.94% | 20260508 12:02:54.824 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.175261% / 3.97364e-06% / 8.74201e-05% | 20260508 12:02:54.824 | INFO | Baz IO overall: | 20260508 12:02:54.824 | INFO | BytesWritten : 3419305780635 | 20260508 12:02:54.824 | INFO | NumEvents : 1641408152446 | 20260508 12:02:56.384 | INFO | Analyzing chunk frames = [7954432,7954944) | 20260508 12:02:56.384 | INFO | MemInfo: { | 20260508 12:02:56.384 | INFO | "gpu" : " 17 GB", | 20260508 12:02:56.384 | INFO | "huge" : "", | 20260508 12:02:56.384 | INFO | "huge_shared" : " 29 GB", | 20260508 12:02:56.384 | INFO | "malloc" : " 14 GB", | 20260508 12:02:56.384 | INFO | "pinned" : " 29 GB" | 20260508 12:02:56.384 | INFO | } | 20260508 12:02:56.384 | INFO | Perf report for the last 5.13769 seconds: | 20260508 12:02:56.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:02:56.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:56.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:56.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:56.384 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 12:02:56.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:02:56.384 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:02:56.384 | INFO | PRE_HQ : 65.6%, 1.6, 54.1ms, 0 | 20260508 12:02:56.384 | INFO | BAZWRITER : 67.0%, 2.5, 84.8ms, 0 | 20260508 12:02:56.384 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260508 12:02:56.385 | INFO | Uploaded 36GiB to GPU in 2.60538s (13.8175GiB/s) | 20260508 12:02:56.385 | INFO | Downloaded 38GiB to GPU in 2.71946s (13.9734GiB/s) | 20260508 12:03:01.523 | INFO | Analyzing chunk frames = [7954944,7955456) | 20260508 12:03:01.523 | INFO | MemInfo: { | 20260508 12:03:01.523 | INFO | "gpu" : " 17 GB", | 20260508 12:03:01.523 | INFO | "huge" : "", | 20260508 12:03:01.523 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:01.523 | INFO | "malloc" : " 14 GB", | 20260508 12:03:01.523 | INFO | "pinned" : " 29 GB" | 20260508 12:03:01.523 | INFO | } | 20260508 12:03:01.523 | INFO | Perf report for the last 5.13912 seconds: | 20260508 12:03:01.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:01.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:03:01.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:03:01.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:01.523 | INFO | Uploaded 36GiB to GPU in 2.50019s (14.3989GiB/s) | 20260508 12:03:01.523 | INFO | Downloaded 34GiB to GPU in 2.46148s (13.8128GiB/s) | 20260508 12:03:06.666 | INFO | Analyzing chunk frames = [7955456,7955968) | 20260508 12:03:06.666 | INFO | MemInfo: { | 20260508 12:03:06.666 | INFO | "gpu" : " 17 GB", | 20260508 12:03:06.666 | INFO | "huge" : "", | 20260508 12:03:06.666 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:06.666 | INFO | "malloc" : " 14 GB", | 20260508 12:03:06.666 | INFO | "pinned" : " 29 GB" | 20260508 12:03:06.666 | INFO | } | 20260508 12:03:06.666 | INFO | Perf report for the last 5.1433 seconds: | 20260508 12:03:06.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:06.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 12:03:06.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:03:06.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:06.667 | INFO | Uploaded 36GiB to GPU in 2.49622s (14.4218GiB/s) | 20260508 12:03:06.667 | INFO | Downloaded 34GiB to GPU in 2.42642s (14.0124GiB/s) | 20260508 12:03:11.807 | INFO | Analyzing chunk frames = [7955968,7956480) | 20260508 12:03:11.807 | INFO | MemInfo: { | 20260508 12:03:11.807 | INFO | "gpu" : " 17 GB", | 20260508 12:03:11.807 | INFO | "huge" : "", | 20260508 12:03:11.807 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:11.807 | INFO | "malloc" : " 14 GB", | 20260508 12:03:11.807 | INFO | "pinned" : " 29 GB" | 20260508 12:03:11.807 | INFO | } | 20260508 12:03:11.807 | INFO | Perf report for the last 5.14028 seconds: | 20260508 12:03:11.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:11.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 12:03:11.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:03:11.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:11.807 | INFO | Uploaded 36GiB to GPU in 2.49821s (14.4103GiB/s) | 20260508 12:03:11.807 | INFO | Downloaded 34GiB to GPU in 2.44561s (13.9025GiB/s) | 20260508 12:03:16.937 | INFO | Analyzing chunk frames = [7956480,7956992) | 20260508 12:03:16.937 | INFO | MemInfo: { | 20260508 12:03:16.937 | INFO | "gpu" : " 17 GB", | 20260508 12:03:16.937 | INFO | "huge" : "", | 20260508 12:03:16.937 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:16.937 | INFO | "malloc" : " 14 GB", | 20260508 12:03:16.938 | INFO | "pinned" : " 29 GB" | 20260508 12:03:16.938 | INFO | } | 20260508 12:03:16.938 | INFO | Perf report for the last 5.13083 seconds: | 20260508 12:03:16.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:16.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:03:16.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 12:03:16.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:16.938 | INFO | Uploaded 36GiB to GPU in 2.36566s (15.2177GiB/s) | 20260508 12:03:16.938 | INFO | Downloaded 34GiB to GPU in 2.49975s (13.6014GiB/s) | 20260508 12:03:22.074 | INFO | Analyzing chunk frames = [7956992,7957504) | 20260508 12:03:22.074 | INFO | MemInfo: { | 20260508 12:03:22.074 | INFO | "gpu" : " 17 GB", | 20260508 12:03:22.074 | INFO | "huge" : "", | 20260508 12:03:22.074 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:22.074 | INFO | "malloc" : " 14 GB", | 20260508 12:03:22.074 | INFO | "pinned" : " 29 GB" | 20260508 12:03:22.074 | INFO | } | 20260508 12:03:22.074 | INFO | Perf report for the last 5.13639 seconds: | 20260508 12:03:22.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:22.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.074 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:03:22.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.074 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 12:03:22.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:22.075 | INFO | Uploaded 36GiB to GPU in 2.39154s (15.0531GiB/s) | 20260508 12:03:22.075 | INFO | Downloaded 34GiB to GPU in 2.4179s (14.0618GiB/s) | 20260508 12:03:27.223 | INFO | Analyzing chunk frames = [7957504,7958016) | 20260508 12:03:27.223 | INFO | MemInfo: { | 20260508 12:03:27.223 | INFO | "gpu" : " 17 GB", | 20260508 12:03:27.223 | INFO | "huge" : "", | 20260508 12:03:27.223 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:27.223 | INFO | "malloc" : " 14 GB", | 20260508 12:03:27.223 | INFO | "pinned" : " 29 GB" | 20260508 12:03:27.223 | INFO | } | 20260508 12:03:27.223 | INFO | Perf report for the last 5.14905 seconds: | 20260508 12:03:27.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:27.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.223 | INFO | ANALYSIS : 65.3%, 3.0, 102.1ms, 0 | 20260508 12:03:27.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.223 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:03:27.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:27.224 | INFO | Uploaded 36GiB to GPU in 2.44367s (14.7319GiB/s) | 20260508 12:03:27.224 | INFO | Downloaded 34GiB to GPU in 2.43737s (13.9495GiB/s) | 20260508 12:03:32.363 | INFO | Analyzing chunk frames = [7958016,7958528) | 20260508 12:03:32.363 | INFO | MemInfo: { | 20260508 12:03:32.363 | INFO | "gpu" : " 17 GB", | 20260508 12:03:32.363 | INFO | "huge" : "", | 20260508 12:03:32.363 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:32.363 | INFO | "malloc" : " 14 GB", | 20260508 12:03:32.363 | INFO | "pinned" : " 29 GB" | 20260508 12:03:32.363 | INFO | } | 20260508 12:03:32.363 | INFO | Perf report for the last 5.13988 seconds: | 20260508 12:03:32.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:32.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.363 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:03:32.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.363 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:03:32.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:32.364 | INFO | Uploaded 36GiB to GPU in 2.47138s (14.5668GiB/s) | 20260508 12:03:32.364 | INFO | Downloaded 34GiB to GPU in 2.44604s (13.9GiB/s) | 20260508 12:03:32.460 | INFO | Starting a round of Baz IO | 20260508 12:03:35.866 | INFO | Populating RTMetrics report | 20260508 12:03:35.867 | INFO | RTMetrics report populated | 20260508 12:03:35.867 | INFO | Opening tmp RTMetrics json file | 20260508 12:03:35.876 | INFO | Serializing RTMetrics to json | 20260508 12:03:35.877 | INFO | Serialization done, swapping tmp file now... | 20260508 12:03:35.877 | INFO | RTMetrics done | 20260508 12:03:35.938 | INFO | Finished a round of Baz IO: Wrote 1.419 GiB in 3.477 seconds | 20260508 12:03:35.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:03:35.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.174232% / 0% / 7.15256e-05% | 20260508 12:03:35.938 | INFO | Baz IO overall: | 20260508 12:03:35.938 | INFO | BytesWritten : 3420829036887 | 20260508 12:03:35.938 | INFO | NumEvents : 1642123178969 | 20260508 12:03:37.500 | INFO | Analyzing chunk frames = [7958528,7959040) | 20260508 12:03:37.500 | INFO | MemInfo: { | 20260508 12:03:37.500 | INFO | "gpu" : " 17 GB", | 20260508 12:03:37.500 | INFO | "huge" : "", | 20260508 12:03:37.500 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:37.500 | INFO | "malloc" : " 14 GB", | 20260508 12:03:37.500 | INFO | "pinned" : " 29 GB" | 20260508 12:03:37.500 | INFO | } | 20260508 12:03:37.500 | INFO | Perf report for the last 5.13734 seconds: | 20260508 12:03:37.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:37.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:37.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:37.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:37.501 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 12:03:37.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:37.501 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 12:03:37.501 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 12:03:37.501 | INFO | BAZWRITER : 67.0%, 2.5, 85.2ms, 0 | 20260508 12:03:37.501 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:03:37.501 | INFO | Uploaded 36GiB to GPU in 2.63415s (13.6667GiB/s) | 20260508 12:03:37.501 | INFO | Downloaded 38GiB to GPU in 2.70941s (14.0252GiB/s) | 20260508 12:03:42.639 | INFO | Analyzing chunk frames = [7959040,7959552) | 20260508 12:03:42.639 | INFO | MemInfo: { | 20260508 12:03:42.639 | INFO | "gpu" : " 17 GB", | 20260508 12:03:42.639 | INFO | "huge" : "", | 20260508 12:03:42.639 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:42.639 | INFO | "malloc" : " 14 GB", | 20260508 12:03:42.639 | INFO | "pinned" : " 29 GB" | 20260508 12:03:42.639 | INFO | } | 20260508 12:03:42.639 | INFO | Perf report for the last 5.13852 seconds: | 20260508 12:03:42.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:42.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:03:42.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:03:42.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:42.639 | INFO | Uploaded 36GiB to GPU in 2.51393s (14.3202GiB/s) | 20260508 12:03:42.639 | INFO | Downloaded 34GiB to GPU in 2.466s (13.7875GiB/s) | 20260508 12:03:47.777 | INFO | Analyzing chunk frames = [7959552,7960064) | 20260508 12:03:47.777 | INFO | MemInfo: { | 20260508 12:03:47.777 | INFO | "gpu" : " 17 GB", | 20260508 12:03:47.777 | INFO | "huge" : "", | 20260508 12:03:47.777 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:47.777 | INFO | "malloc" : " 14 GB", | 20260508 12:03:47.777 | INFO | "pinned" : " 29 GB" | 20260508 12:03:47.777 | INFO | } | 20260508 12:03:47.777 | INFO | Perf report for the last 5.13842 seconds: | 20260508 12:03:47.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:47.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.777 | INFO | ANALYSIS : 66.7%, 3.0, 103.6ms, 0 | 20260508 12:03:47.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.777 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:03:47.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:47.778 | INFO | Uploaded 36GiB to GPU in 2.52793s (14.2409GiB/s) | 20260508 12:03:47.778 | INFO | Downloaded 34GiB to GPU in 2.45223s (13.8649GiB/s) | 20260508 12:03:52.916 | INFO | Analyzing chunk frames = [7960064,7960576) | 20260508 12:03:52.916 | INFO | MemInfo: { | 20260508 12:03:52.916 | INFO | "gpu" : " 17 GB", | 20260508 12:03:52.916 | INFO | "huge" : "", | 20260508 12:03:52.916 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:52.916 | INFO | "malloc" : " 14 GB", | 20260508 12:03:52.916 | INFO | "pinned" : " 29 GB" | 20260508 12:03:52.916 | INFO | } | 20260508 12:03:52.917 | INFO | Perf report for the last 5.13936 seconds: | 20260508 12:03:52.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:52.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 12:03:52.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:03:52.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:52.917 | INFO | Uploaded 36GiB to GPU in 2.53653s (14.1926GiB/s) | 20260508 12:03:52.917 | INFO | Downloaded 34GiB to GPU in 2.44312s (13.9167GiB/s) | 20260508 12:03:58.064 | INFO | Analyzing chunk frames = [7960576,7961088) | 20260508 12:03:58.064 | INFO | MemInfo: { | 20260508 12:03:58.064 | INFO | "gpu" : " 17 GB", | 20260508 12:03:58.064 | INFO | "huge" : "", | 20260508 12:03:58.064 | INFO | "huge_shared" : " 29 GB", | 20260508 12:03:58.064 | INFO | "malloc" : " 14 GB", | 20260508 12:03:58.064 | INFO | "pinned" : " 29 GB" | 20260508 12:03:58.064 | INFO | } | 20260508 12:03:58.064 | INFO | Perf report for the last 5.14735 seconds: | 20260508 12:03:58.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:03:58.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 12:03:58.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 12:03:58.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:03:58.064 | INFO | Uploaded 36GiB to GPU in 2.3102s (15.5831GiB/s) | 20260508 12:03:58.065 | INFO | Downloaded 34GiB to GPU in 2.49725s (13.615GiB/s) | 20260508 12:04:03.198 | INFO | Analyzing chunk frames = [7961088,7961600) | 20260508 12:04:03.198 | INFO | MemInfo: { | 20260508 12:04:03.198 | INFO | "gpu" : " 17 GB", | 20260508 12:04:03.198 | INFO | "huge" : "", | 20260508 12:04:03.198 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:03.198 | INFO | "malloc" : " 14 GB", | 20260508 12:04:03.198 | INFO | "pinned" : " 29 GB" | 20260508 12:04:03.198 | INFO | } | 20260508 12:04:03.198 | INFO | Perf report for the last 5.134 seconds: | 20260508 12:04:03.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:03.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.198 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:04:03.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.198 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:04:03.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:03.199 | INFO | Uploaded 36GiB to GPU in 2.43178s (14.804GiB/s) | 20260508 12:04:03.199 | INFO | Downloaded 34GiB to GPU in 2.42531s (14.0188GiB/s) | 20260508 12:04:08.332 | INFO | Analyzing chunk frames = [7961600,7962112) | 20260508 12:04:08.332 | INFO | MemInfo: { | 20260508 12:04:08.332 | INFO | "gpu" : " 17 GB", | 20260508 12:04:08.332 | INFO | "huge" : "", | 20260508 12:04:08.332 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:08.332 | INFO | "malloc" : " 14 GB", | 20260508 12:04:08.332 | INFO | "pinned" : " 29 GB" | 20260508 12:04:08.332 | INFO | } | 20260508 12:04:08.332 | INFO | Perf report for the last 5.13413 seconds: | 20260508 12:04:08.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:08.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.332 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 12:04:08.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.332 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:04:08.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:08.333 | INFO | Uploaded 36GiB to GPU in 2.4761s (14.539GiB/s) | 20260508 12:04:08.333 | INFO | Downloaded 34GiB to GPU in 2.44225s (13.9216GiB/s) | 20260508 12:04:13.478 | INFO | Analyzing chunk frames = [7962112,7962624) | 20260508 12:04:13.478 | INFO | MemInfo: { | 20260508 12:04:13.478 | INFO | "gpu" : " 17 GB", | 20260508 12:04:13.478 | INFO | "huge" : "", | 20260508 12:04:13.478 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:13.478 | INFO | "malloc" : " 14 GB", | 20260508 12:04:13.478 | INFO | "pinned" : " 29 GB" | 20260508 12:04:13.478 | INFO | } | 20260508 12:04:13.478 | INFO | Perf report for the last 5.14629 seconds: | 20260508 12:04:13.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:13.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | ANALYSIS : 65.6%, 3.0, 102.8ms, 0 | 20260508 12:04:13.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:04:13.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:13.479 | INFO | Uploaded 36GiB to GPU in 2.46784s (14.5877GiB/s) | 20260508 12:04:13.479 | INFO | Downloaded 34GiB to GPU in 2.43927s (13.9386GiB/s) | 20260508 12:04:13.479 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:04:13.479 | INFO | Total mem usage: 16.7 GiB | 20260508 12:04:13.479 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:04:13.479 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 12:04:13.479 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:04:13.479 | INFO | High Water Breakdown: | 20260508 12:04:13.479 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:04:13.479 | INFO | 40.4% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:04:13.479 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:04:13.479 | INFO | ------------------------------------------ | 20260508 12:04:13.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:04:13.479 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:04:13.479 | INFO | Total mem usage: 26.5 GiB | 20260508 12:04:13.479 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:04:13.479 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:04:13.479 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:04:13.479 | INFO | High Water Breakdown: | 20260508 12:04:13.479 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:04:13.479 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:04:13.479 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:04:13.479 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:04:13.479 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:04:13.479 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:04:13.479 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:04:13.479 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:04:13.479 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:04:13.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:04:13.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:04:13.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:04:13.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:04:13.479 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:04:13.479 | INFO | ------------------------------------------ | 20260508 12:04:13.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:04:13.479 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:04:13.479 | INFO | Total mem usage: 27.8 GiB | 20260508 12:04:13.479 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:04:13.479 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:04:13.479 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:04:13.479 | INFO | High Water Breakdown: | 20260508 12:04:13.479 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:04:13.479 | INFO | ------------------------------------------ | 20260508 12:04:13.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:04:13.479 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:04:13.479 | INFO | Total mem usage: 15.7 GiB | 20260508 12:04:13.479 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:04:13.479 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:04:13.479 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:04:13.479 | INFO | High Water Breakdown: | 20260508 12:04:13.479 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:04:13.479 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:04:13.480 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:04:13.480 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:04:13.480 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:04:13.480 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:04:13.480 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:04:13.480 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:04:13.480 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:04:13.480 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:04:13.480 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:04:13.480 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:04:13.480 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:04:13.480 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:04:13.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:04:13.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:04:13.480 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:04:13.480 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:04:13.480 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:04:13.480 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:04:13.480 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:04:13.480 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:04:13.480 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:04:13.480 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:04:13.480 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:04:13.480 | INFO | ------------------------------------------ | 20260508 12:04:13.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:04:13.480 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:04:13.480 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:04:13.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:04:13.480 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.40e+01 / 4.86e-01 | 20260508 12:04:13.480 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.04e-01, 1.19e-02 / 2.06e-03 | 20260508 12:04:13.480 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.43e+00, 5.94e-01 / 1.78e-01 | 20260508 12:04:13.480 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.02e-01 | 20260508 12:04:13.480 | INFO | PerfReport: 16.07%, DME -- 4.19e+01 / 4.56e+01, 3.78e+00 / 1.13e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.77e+00 / 1.23e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.33e+00 / 2.52e+00, 4.02e-01 / 2.23e-01 | 20260508 12:04:13.480 | INFO | PerfReport: 2.98%, Metrics -- 9.72e-01 / 1.10e+00, 2.39e-01 / 1.05e-01 | 20260508 12:04:13.480 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.25e-01, 4.73e-02 / 2.20e-02 | 20260508 12:04:13.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:04:13.480 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:04:13.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:04:13.480 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 24.54%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 24.58%, Download -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 30.96%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 2.35%, LockUpload -- 2.47e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 2.46%, LockDownload -- 2.59e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 15.10%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:04:13.480 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260508 12:04:13.576 | INFO | Starting a round of Baz IO | 20260508 12:04:16.979 | INFO | Populating RTMetrics report | 20260508 12:04:16.979 | INFO | RTMetrics report populated | 20260508 12:04:16.979 | INFO | Opening tmp RTMetrics json file | 20260508 12:04:16.985 | INFO | Serializing RTMetrics to json | 20260508 12:04:16.986 | INFO | Serialization done, swapping tmp file now... | 20260508 12:04:16.987 | INFO | RTMetrics done | 20260508 12:04:17.046 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.47 seconds | 20260508 12:04:17.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:04:17.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.174054% / 0% / 6.75519e-05% | 20260508 12:04:17.046 | INFO | Baz IO overall: | 20260508 12:04:17.046 | INFO | BytesWritten : 3422352033529 | 20260508 12:04:17.046 | INFO | NumEvents : 1642838034027 | 20260508 12:04:18.614 | INFO | Analyzing chunk frames = [7962624,7963136) | 20260508 12:04:18.614 | INFO | MemInfo: { | 20260508 12:04:18.614 | INFO | "gpu" : " 17 GB", | 20260508 12:04:18.614 | INFO | "huge" : "", | 20260508 12:04:18.614 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:18.614 | INFO | "malloc" : " 14 GB", | 20260508 12:04:18.614 | INFO | "pinned" : " 29 GB" | 20260508 12:04:18.614 | INFO | } | 20260508 12:04:18.614 | INFO | Perf report for the last 5.1353 seconds: | 20260508 12:04:18.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:18.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:18.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:18.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:18.614 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:04:18.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:18.614 | INFO | RT_METRICS : 38.6%, 1.0, 20.2ms, 0 | 20260508 12:04:18.614 | INFO | PRE_HQ : 65.5%, 1.6, 54.4ms, 0 | 20260508 12:04:18.614 | INFO | BAZWRITER : 66.9%, 2.5, 84.8ms, 0 | 20260508 12:04:18.614 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:04:18.614 | INFO | Uploaded 36GiB to GPU in 2.60726s (13.8076GiB/s) | 20260508 12:04:18.614 | INFO | Downloaded 38GiB to GPU in 2.71573s (13.9926GiB/s) | 20260508 12:04:23.749 | INFO | Analyzing chunk frames = [7963136,7963648) | 20260508 12:04:23.749 | INFO | MemInfo: { | 20260508 12:04:23.749 | INFO | "gpu" : " 17 GB", | 20260508 12:04:23.749 | INFO | "huge" : "", | 20260508 12:04:23.749 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:23.749 | INFO | "malloc" : " 14 GB", | 20260508 12:04:23.749 | INFO | "pinned" : " 29 GB" | 20260508 12:04:23.749 | INFO | } | 20260508 12:04:23.749 | INFO | Perf report for the last 5.13559 seconds: | 20260508 12:04:23.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:23.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 12:04:23.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:04:23.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:23.750 | INFO | Uploaded 36GiB to GPU in 2.50284s (14.3837GiB/s) | 20260508 12:04:23.750 | INFO | Downloaded 34GiB to GPU in 2.46288s (13.805GiB/s) | 20260508 12:04:28.889 | INFO | Analyzing chunk frames = [7963648,7964160) | 20260508 12:04:28.889 | INFO | MemInfo: { | 20260508 12:04:28.889 | INFO | "gpu" : " 17 GB", | 20260508 12:04:28.889 | INFO | "huge" : "", | 20260508 12:04:28.889 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:28.889 | INFO | "malloc" : " 14 GB", | 20260508 12:04:28.889 | INFO | "pinned" : " 29 GB" | 20260508 12:04:28.889 | INFO | } | 20260508 12:04:28.889 | INFO | Perf report for the last 5.1401 seconds: | 20260508 12:04:28.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:28.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 12:04:28.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:04:28.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:28.890 | INFO | Uploaded 36GiB to GPU in 2.49492s (14.4293GiB/s) | 20260508 12:04:28.890 | INFO | Downloaded 34GiB to GPU in 2.42285s (14.033GiB/s) | 20260508 12:04:34.030 | INFO | Analyzing chunk frames = [7964160,7964672) | 20260508 12:04:34.030 | INFO | MemInfo: { | 20260508 12:04:34.030 | INFO | "gpu" : " 17 GB", | 20260508 12:04:34.030 | INFO | "huge" : "", | 20260508 12:04:34.030 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:34.030 | INFO | "malloc" : " 14 GB", | 20260508 12:04:34.030 | INFO | "pinned" : " 29 GB" | 20260508 12:04:34.030 | INFO | } | 20260508 12:04:34.030 | INFO | Perf report for the last 5.14048 seconds: | 20260508 12:04:34.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:34.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:04:34.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:04:34.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:34.030 | INFO | Uploaded 36GiB to GPU in 2.49998s (14.4001GiB/s) | 20260508 12:04:34.030 | INFO | Downloaded 34GiB to GPU in 2.43986s (13.9352GiB/s) | 20260508 12:04:39.176 | INFO | Analyzing chunk frames = [7964672,7965184) | 20260508 12:04:39.176 | INFO | MemInfo: { | 20260508 12:04:39.176 | INFO | "gpu" : " 17 GB", | 20260508 12:04:39.176 | INFO | "huge" : "", | 20260508 12:04:39.176 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:39.176 | INFO | "malloc" : " 14 GB", | 20260508 12:04:39.176 | INFO | "pinned" : " 29 GB" | 20260508 12:04:39.176 | INFO | } | 20260508 12:04:39.176 | INFO | Perf report for the last 5.14574 seconds: | 20260508 12:04:39.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:39.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | ANALYSIS : 66.2%, 3.0, 103.6ms, 0 | 20260508 12:04:39.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:04:39.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:39.176 | INFO | Uploaded 36GiB to GPU in 2.44473s (14.7256GiB/s) | 20260508 12:04:39.177 | INFO | Downloaded 34GiB to GPU in 2.5222s (13.4803GiB/s) | 20260508 12:04:44.314 | INFO | Analyzing chunk frames = [7965184,7965696) | 20260508 12:04:44.314 | INFO | MemInfo: { | 20260508 12:04:44.314 | INFO | "gpu" : " 17 GB", | 20260508 12:04:44.314 | INFO | "huge" : "", | 20260508 12:04:44.314 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:44.314 | INFO | "malloc" : " 14 GB", | 20260508 12:04:44.314 | INFO | "pinned" : " 29 GB" | 20260508 12:04:44.314 | INFO | } | 20260508 12:04:44.314 | INFO | Perf report for the last 5.13848 seconds: | 20260508 12:04:44.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:44.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.314 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 12:04:44.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.315 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:04:44.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:44.315 | INFO | Uploaded 36GiB to GPU in 2.45935s (14.638GiB/s) | 20260508 12:04:44.315 | INFO | Downloaded 34GiB to GPU in 2.4393s (13.9384GiB/s) | 20260508 12:04:49.453 | INFO | Analyzing chunk frames = [7965696,7966208) | 20260508 12:04:49.453 | INFO | MemInfo: { | 20260508 12:04:49.453 | INFO | "gpu" : " 17 GB", | 20260508 12:04:49.453 | INFO | "huge" : "", | 20260508 12:04:49.453 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:49.453 | INFO | "malloc" : " 14 GB", | 20260508 12:04:49.453 | INFO | "pinned" : " 29 GB" | 20260508 12:04:49.453 | INFO | } | 20260508 12:04:49.453 | INFO | Perf report for the last 5.13875 seconds: | 20260508 12:04:49.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:49.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 12:04:49.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:04:49.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:49.453 | INFO | Uploaded 36GiB to GPU in 2.47241s (14.5607GiB/s) | 20260508 12:04:49.453 | INFO | Downloaded 34GiB to GPU in 2.44969s (13.8793GiB/s) | 20260508 12:04:54.583 | INFO | Analyzing chunk frames = [7966208,7966720) | 20260508 12:04:54.583 | INFO | MemInfo: { | 20260508 12:04:54.583 | INFO | "gpu" : " 17 GB", | 20260508 12:04:54.583 | INFO | "huge" : "", | 20260508 12:04:54.583 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:54.583 | INFO | "malloc" : " 14 GB", | 20260508 12:04:54.583 | INFO | "pinned" : " 29 GB" | 20260508 12:04:54.583 | INFO | } | 20260508 12:04:54.583 | INFO | Perf report for the last 5.13002 seconds: | 20260508 12:04:54.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:54.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 12:04:54.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:04:54.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:54.583 | INFO | Uploaded 36GiB to GPU in 2.48536s (14.4848GiB/s) | 20260508 12:04:54.584 | INFO | Downloaded 34GiB to GPU in 2.44303s (13.9171GiB/s) | 20260508 12:04:54.680 | INFO | Starting a round of Baz IO | 20260508 12:04:58.094 | INFO | Populating RTMetrics report | 20260508 12:04:58.095 | INFO | RTMetrics report populated | 20260508 12:04:58.095 | INFO | Opening tmp RTMetrics json file | 20260508 12:04:58.096 | INFO | Serializing RTMetrics to json | 20260508 12:04:58.110 | INFO | Serialization done, swapping tmp file now... | 20260508 12:04:58.110 | INFO | RTMetrics done | 20260508 12:04:58.160 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.48 seconds | 20260508 12:04:58.160 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:04:58.160 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173668% / 0% / 7.15256e-05% | 20260508 12:04:58.160 | INFO | Baz IO overall: | 20260508 12:04:58.160 | INFO | BytesWritten : 3423875112083 | 20260508 12:04:58.160 | INFO | NumEvents : 1643553070581 | 20260508 12:04:59.728 | INFO | Analyzing chunk frames = [7966720,7967232) | 20260508 12:04:59.728 | INFO | MemInfo: { | 20260508 12:04:59.728 | INFO | "gpu" : " 17 GB", | 20260508 12:04:59.728 | INFO | "huge" : "", | 20260508 12:04:59.728 | INFO | "huge_shared" : " 29 GB", | 20260508 12:04:59.728 | INFO | "malloc" : " 14 GB", | 20260508 12:04:59.728 | INFO | "pinned" : " 29 GB" | 20260508 12:04:59.728 | INFO | } | 20260508 12:04:59.728 | INFO | Perf report for the last 5.14522 seconds: | 20260508 12:04:59.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:04:59.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:59.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:59.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:59.728 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 12:04:59.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:04:59.728 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 12:04:59.728 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:04:59.728 | INFO | BAZWRITER : 66.9%, 2.5, 84.8ms, 0 | 20260508 12:04:59.728 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 12:04:59.729 | INFO | Uploaded 36GiB to GPU in 2.58581s (13.9221GiB/s) | 20260508 12:04:59.729 | INFO | Downloaded 38GiB to GPU in 2.71309s (14.0062GiB/s) | 20260508 12:05:04.870 | INFO | Analyzing chunk frames = [7967232,7967744) | 20260508 12:05:04.870 | INFO | MemInfo: { | 20260508 12:05:04.870 | INFO | "gpu" : " 17 GB", | 20260508 12:05:04.870 | INFO | "huge" : "", | 20260508 12:05:04.870 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:04.870 | INFO | "malloc" : " 14 GB", | 20260508 12:05:04.870 | INFO | "pinned" : " 29 GB" | 20260508 12:05:04.870 | INFO | } | 20260508 12:05:04.870 | INFO | Perf report for the last 5.14173 seconds: | 20260508 12:05:04.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:04.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 12:05:04.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 12:05:04.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:04.870 | INFO | Uploaded 36GiB to GPU in 2.43863s (14.7624GiB/s) | 20260508 12:05:04.871 | INFO | Downloaded 34GiB to GPU in 2.47446s (13.7404GiB/s) | 20260508 12:05:10.010 | INFO | Analyzing chunk frames = [7967744,7968256) | 20260508 12:05:10.010 | INFO | MemInfo: { | 20260508 12:05:10.010 | INFO | "gpu" : " 17 GB", | 20260508 12:05:10.010 | INFO | "huge" : "", | 20260508 12:05:10.010 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:10.010 | INFO | "malloc" : " 14 GB", | 20260508 12:05:10.010 | INFO | "pinned" : " 29 GB" | 20260508 12:05:10.010 | INFO | } | 20260508 12:05:10.010 | INFO | Perf report for the last 5.14005 seconds: | 20260508 12:05:10.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:10.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.010 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 12:05:10.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.010 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:05:10.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:10.011 | INFO | Uploaded 36GiB to GPU in 2.47815s (14.5269GiB/s) | 20260508 12:05:10.011 | INFO | Downloaded 34GiB to GPU in 2.45017s (13.8766GiB/s) | 20260508 12:05:15.151 | INFO | Analyzing chunk frames = [7968256,7968768) | 20260508 12:05:15.151 | INFO | MemInfo: { | 20260508 12:05:15.151 | INFO | "gpu" : " 17 GB", | 20260508 12:05:15.151 | INFO | "huge" : "", | 20260508 12:05:15.151 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:15.151 | INFO | "malloc" : " 14 GB", | 20260508 12:05:15.151 | INFO | "pinned" : " 29 GB" | 20260508 12:05:15.151 | INFO | } | 20260508 12:05:15.151 | INFO | Perf report for the last 5.14094 seconds: | 20260508 12:05:15.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:15.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.151 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:05:15.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.151 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:05:15.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:15.152 | INFO | Uploaded 36GiB to GPU in 2.52232s (14.2726GiB/s) | 20260508 12:05:15.152 | INFO | Downloaded 34GiB to GPU in 2.47455s (13.7399GiB/s) | 20260508 12:05:20.285 | INFO | Analyzing chunk frames = [7968768,7969280) | 20260508 12:05:20.285 | INFO | MemInfo: { | 20260508 12:05:20.285 | INFO | "gpu" : " 17 GB", | 20260508 12:05:20.285 | INFO | "huge" : "", | 20260508 12:05:20.285 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:20.285 | INFO | "malloc" : " 14 GB", | 20260508 12:05:20.285 | INFO | "pinned" : " 29 GB" | 20260508 12:05:20.286 | INFO | } | 20260508 12:05:20.286 | INFO | Perf report for the last 5.13463 seconds: | 20260508 12:05:20.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:20.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:05:20.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:05:20.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:20.286 | INFO | Uploaded 36GiB to GPU in 2.40538s (14.9664GiB/s) | 20260508 12:05:20.287 | INFO | Downloaded 34GiB to GPU in 2.52664s (13.4566GiB/s) | 20260508 12:05:25.428 | INFO | Analyzing chunk frames = [7969280,7969792) | 20260508 12:05:25.428 | INFO | MemInfo: { | 20260508 12:05:25.428 | INFO | "gpu" : " 17 GB", | 20260508 12:05:25.428 | INFO | "huge" : "", | 20260508 12:05:25.428 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:25.428 | INFO | "malloc" : " 14 GB", | 20260508 12:05:25.428 | INFO | "pinned" : " 29 GB" | 20260508 12:05:25.428 | INFO | } | 20260508 12:05:25.429 | INFO | Perf report for the last 5.14284 seconds: | 20260508 12:05:25.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:25.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 12:05:25.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:05:25.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:25.429 | INFO | Uploaded 36GiB to GPU in 2.37341s (15.1681GiB/s) | 20260508 12:05:25.429 | INFO | Downloaded 34GiB to GPU in 2.45323s (13.8593GiB/s) | 20260508 12:05:30.572 | INFO | Analyzing chunk frames = [7969792,7970304) | 20260508 12:05:30.572 | INFO | MemInfo: { | 20260508 12:05:30.572 | INFO | "gpu" : " 17 GB", | 20260508 12:05:30.572 | INFO | "huge" : "", | 20260508 12:05:30.572 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:30.572 | INFO | "malloc" : " 14 GB", | 20260508 12:05:30.572 | INFO | "pinned" : " 29 GB" | 20260508 12:05:30.572 | INFO | } | 20260508 12:05:30.572 | INFO | Perf report for the last 5.14368 seconds: | 20260508 12:05:30.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:30.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.572 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:05:30.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.572 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:05:30.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:30.573 | INFO | Uploaded 36GiB to GPU in 2.4354s (14.782GiB/s) | 20260508 12:05:30.573 | INFO | Downloaded 34GiB to GPU in 2.47388s (13.7436GiB/s) | 20260508 12:05:35.705 | INFO | Analyzing chunk frames = [7970304,7970816) | 20260508 12:05:35.705 | INFO | MemInfo: { | 20260508 12:05:35.705 | INFO | "gpu" : " 17 GB", | 20260508 12:05:35.705 | INFO | "huge" : "", | 20260508 12:05:35.705 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:35.705 | INFO | "malloc" : " 14 GB", | 20260508 12:05:35.705 | INFO | "pinned" : " 29 GB" | 20260508 12:05:35.705 | INFO | } | 20260508 12:05:35.705 | INFO | Perf report for the last 5.13283 seconds: | 20260508 12:05:35.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:35.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 12:05:35.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 12:05:35.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:35.705 | INFO | Uploaded 36GiB to GPU in 2.42213s (14.863GiB/s) | 20260508 12:05:35.705 | INFO | Downloaded 34GiB to GPU in 2.47031s (13.7634GiB/s) | 20260508 12:05:35.706 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:05:35.706 | INFO | Total mem usage: 16.7 GiB | 20260508 12:05:35.706 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:05:35.706 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 12:05:35.706 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:05:35.706 | INFO | High Water Breakdown: | 20260508 12:05:35.706 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:05:35.706 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:05:35.706 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:05:35.706 | INFO | ------------------------------------------ | 20260508 12:05:35.706 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:05:35.706 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:05:35.706 | INFO | Total mem usage: 26.5 GiB | 20260508 12:05:35.706 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:05:35.706 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:05:35.706 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:05:35.706 | INFO | High Water Breakdown: | 20260508 12:05:35.706 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:05:35.706 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:05:35.706 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:05:35.706 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:05:35.706 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:05:35.706 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:05:35.706 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:05:35.706 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:05:35.706 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:05:35.706 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:05:35.706 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:05:35.706 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:05:35.706 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:05:35.706 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:05:35.706 | INFO | ------------------------------------------ | 20260508 12:05:35.706 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:05:35.706 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:05:35.706 | INFO | Total mem usage: 27.8 GiB | 20260508 12:05:35.706 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:05:35.706 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:05:35.706 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:05:35.706 | INFO | High Water Breakdown: | 20260508 12:05:35.706 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:05:35.706 | INFO | ------------------------------------------ | 20260508 12:05:35.706 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:05:35.706 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:05:35.706 | INFO | Total mem usage: 15.7 GiB | 20260508 12:05:35.706 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:05:35.706 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:05:35.706 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:05:35.706 | INFO | High Water Breakdown: | 20260508 12:05:35.706 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:05:35.706 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:05:35.706 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:05:35.706 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:05:35.706 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:05:35.706 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:05:35.706 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:05:35.706 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:05:35.706 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:05:35.706 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:05:35.706 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:05:35.706 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:05:35.707 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:05:35.707 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:05:35.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:05:35.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:05:35.707 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:05:35.707 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:05:35.707 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:05:35.707 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:05:35.707 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:05:35.707 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:05:35.707 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:05:35.707 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:05:35.707 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:05:35.707 | INFO | ------------------------------------------ | 20260508 12:05:35.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:05:35.707 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:05:35.707 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:05:35.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:05:35.707 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.39e+01 / 7.37e-01 | 20260508 12:05:35.707 | INFO | PerfReport: 0.29%, AggregateStats -- 9.62e-02 / 1.04e-01, 1.19e-02 / 6.97e-03 | 20260508 12:05:35.707 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.45e+00, 5.86e-01 / 1.95e-01 | 20260508 12:05:35.707 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.46e+00, 2.07e-01 / 1.13e-01 | 20260508 12:05:35.707 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.52e+01, 3.77e+00 / 1.10e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.80e+00 / 1.31e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.45e+00, 4.04e-01 / 2.30e-01 | 20260508 12:05:35.707 | INFO | PerfReport: 2.97%, Metrics -- 9.70e-01 / 1.12e+00, 2.42e-01 / 1.17e-01 | 20260508 12:05:35.707 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.25e-01, 4.74e-02 / 2.39e-02 | 20260508 12:05:35.707 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:05:35.707 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:05:35.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:05:35.707 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 24.53%, Upload -- 2.58e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 24.72%, Download -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 31.02%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 2.23%, LockUpload -- 2.35e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 2.51%, LockDownload -- 2.65e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 14.97%, LockCompute -- 1.57e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:05:35.707 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260508 12:05:35.802 | INFO | Starting a round of Baz IO | 20260508 12:05:39.217 | INFO | Populating RTMetrics report | 20260508 12:05:39.218 | INFO | RTMetrics report populated | 20260508 12:05:39.218 | INFO | Opening tmp RTMetrics json file | 20260508 12:05:39.219 | INFO | Serializing RTMetrics to json | 20260508 12:05:39.220 | INFO | Serialization done, swapping tmp file now... | 20260508 12:05:39.220 | INFO | RTMetrics done | 20260508 12:05:39.281 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.478 seconds | 20260508 12:05:39.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:05:39.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172997% / 0% / 7.15256e-05% | 20260508 12:05:39.281 | INFO | Baz IO overall: | 20260508 12:05:39.281 | INFO | BytesWritten : 3425398151801 | 20260508 12:05:39.281 | INFO | NumEvents : 1644268025126 | 20260508 12:05:40.844 | INFO | Analyzing chunk frames = [7970816,7971328) | 20260508 12:05:40.844 | INFO | MemInfo: { | 20260508 12:05:40.844 | INFO | "gpu" : " 17 GB", | 20260508 12:05:40.844 | INFO | "huge" : "", | 20260508 12:05:40.844 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:40.844 | INFO | "malloc" : " 14 GB", | 20260508 12:05:40.844 | INFO | "pinned" : " 29 GB" | 20260508 12:05:40.844 | INFO | } | 20260508 12:05:40.844 | INFO | Perf report for the last 5.13932 seconds: | 20260508 12:05:40.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:40.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:40.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:40.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:40.845 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 12:05:40.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:40.845 | INFO | RT_METRICS : 39.7%, 1.0, 20.8ms, 0 | 20260508 12:05:40.845 | INFO | PRE_HQ : 65.9%, 1.6, 55.9ms, 0 | 20260508 12:05:40.845 | INFO | BAZWRITER : 67.2%, 2.5, 87.1ms, 0 | 20260508 12:05:40.845 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 12:05:40.845 | INFO | Uploaded 36GiB to GPU in 2.5797s (13.9551GiB/s) | 20260508 12:05:40.845 | INFO | Downloaded 38GiB to GPU in 2.72509s (13.9445GiB/s) | 20260508 12:05:45.986 | INFO | Analyzing chunk frames = [7971328,7971840) | 20260508 12:05:45.986 | INFO | MemInfo: { | 20260508 12:05:45.986 | INFO | "gpu" : " 17 GB", | 20260508 12:05:45.986 | INFO | "huge" : "", | 20260508 12:05:45.986 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:45.986 | INFO | "malloc" : " 14 GB", | 20260508 12:05:45.986 | INFO | "pinned" : " 29 GB" | 20260508 12:05:45.986 | INFO | } | 20260508 12:05:45.986 | INFO | Perf report for the last 5.14142 seconds: | 20260508 12:05:45.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:45.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:05:45.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:05:45.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:45.986 | INFO | Uploaded 36GiB to GPU in 2.43674s (14.7739GiB/s) | 20260508 12:05:45.987 | INFO | Downloaded 34GiB to GPU in 2.46257s (13.8067GiB/s) | 20260508 12:05:51.121 | INFO | Analyzing chunk frames = [7971840,7972352) | 20260508 12:05:51.121 | INFO | MemInfo: { | 20260508 12:05:51.121 | INFO | "gpu" : " 17 GB", | 20260508 12:05:51.121 | INFO | "huge" : "", | 20260508 12:05:51.121 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:51.121 | INFO | "malloc" : " 14 GB", | 20260508 12:05:51.121 | INFO | "pinned" : " 29 GB" | 20260508 12:05:51.121 | INFO | } | 20260508 12:05:51.121 | INFO | Perf report for the last 5.13564 seconds: | 20260508 12:05:51.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:51.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.121 | INFO | ANALYSIS : 66.5%, 3.0, 103.2ms, 0 | 20260508 12:05:51.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.122 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:05:51.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:51.122 | INFO | Uploaded 36GiB to GPU in 2.50509s (14.3708GiB/s) | 20260508 12:05:51.122 | INFO | Downloaded 34GiB to GPU in 2.45341s (13.8583GiB/s) | 20260508 12:05:56.262 | INFO | Analyzing chunk frames = [7972352,7972864) | 20260508 12:05:56.262 | INFO | MemInfo: { | 20260508 12:05:56.262 | INFO | "gpu" : " 17 GB", | 20260508 12:05:56.262 | INFO | "huge" : "", | 20260508 12:05:56.262 | INFO | "huge_shared" : " 29 GB", | 20260508 12:05:56.262 | INFO | "malloc" : " 14 GB", | 20260508 12:05:56.263 | INFO | "pinned" : " 29 GB" | 20260508 12:05:56.263 | INFO | } | 20260508 12:05:56.263 | INFO | Perf report for the last 5.14104 seconds: | 20260508 12:05:56.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:05:56.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:05:56.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:05:56.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:05:56.263 | INFO | Uploaded 36GiB to GPU in 2.47594s (14.5399GiB/s) | 20260508 12:05:56.263 | INFO | Downloaded 34GiB to GPU in 2.42944s (13.995GiB/s) | 20260508 12:06:01.397 | INFO | Analyzing chunk frames = [7972864,7973376) | 20260508 12:06:01.397 | INFO | MemInfo: { | 20260508 12:06:01.397 | INFO | "gpu" : " 17 GB", | 20260508 12:06:01.397 | INFO | "huge" : "", | 20260508 12:06:01.397 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:01.397 | INFO | "malloc" : " 14 GB", | 20260508 12:06:01.397 | INFO | "pinned" : " 29 GB" | 20260508 12:06:01.397 | INFO | } | 20260508 12:06:01.398 | INFO | Perf report for the last 5.13503 seconds: | 20260508 12:06:01.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:01.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:06:01.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:06:01.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:01.398 | INFO | Uploaded 36GiB to GPU in 2.29485s (15.6873GiB/s) | 20260508 12:06:01.398 | INFO | Downloaded 34GiB to GPU in 2.48439s (13.6855GiB/s) | 20260508 12:06:06.536 | INFO | Analyzing chunk frames = [7973376,7973888) | 20260508 12:06:06.536 | INFO | MemInfo: { | 20260508 12:06:06.536 | INFO | "gpu" : " 17 GB", | 20260508 12:06:06.536 | INFO | "huge" : "", | 20260508 12:06:06.536 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:06.536 | INFO | "malloc" : " 14 GB", | 20260508 12:06:06.536 | INFO | "pinned" : " 29 GB" | 20260508 12:06:06.536 | INFO | } | 20260508 12:06:06.536 | INFO | Perf report for the last 5.13872 seconds: | 20260508 12:06:06.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:06.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.536 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 12:06:06.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.536 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:06:06.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:06.537 | INFO | Uploaded 36GiB to GPU in 2.38109s (15.1191GiB/s) | 20260508 12:06:06.537 | INFO | Downloaded 34GiB to GPU in 2.43262s (13.9767GiB/s) | 20260508 12:06:11.680 | INFO | Analyzing chunk frames = [7973888,7974400) | 20260508 12:06:11.680 | INFO | MemInfo: { | 20260508 12:06:11.680 | INFO | "gpu" : " 17 GB", | 20260508 12:06:11.680 | INFO | "huge" : "", | 20260508 12:06:11.681 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:11.681 | INFO | "malloc" : " 14 GB", | 20260508 12:06:11.681 | INFO | "pinned" : " 29 GB" | 20260508 12:06:11.681 | INFO | } | 20260508 12:06:11.681 | INFO | Perf report for the last 5.14431 seconds: | 20260508 12:06:11.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:11.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:06:11.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:06:11.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:11.681 | INFO | Uploaded 36GiB to GPU in 2.44084s (14.749GiB/s) | 20260508 12:06:11.681 | INFO | Downloaded 34GiB to GPU in 2.44144s (13.9262GiB/s) | 20260508 12:06:16.825 | INFO | Analyzing chunk frames = [7974400,7974912) | 20260508 12:06:16.825 | INFO | MemInfo: { | 20260508 12:06:16.825 | INFO | "gpu" : " 17 GB", | 20260508 12:06:16.825 | INFO | "huge" : "", | 20260508 12:06:16.825 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:16.825 | INFO | "malloc" : " 14 GB", | 20260508 12:06:16.825 | INFO | "pinned" : " 29 GB" | 20260508 12:06:16.825 | INFO | } | 20260508 12:06:16.825 | INFO | Perf report for the last 5.14439 seconds: | 20260508 12:06:16.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:16.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.825 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 12:06:16.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.825 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260508 12:06:16.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:16.826 | INFO | Uploaded 36GiB to GPU in 2.37693s (15.1456GiB/s) | 20260508 12:06:16.826 | INFO | Downloaded 34GiB to GPU in 2.45863s (13.8288GiB/s) | 20260508 12:06:16.939 | INFO | Starting a round of Baz IO | 20260508 12:06:20.337 | INFO | Populating RTMetrics report | 20260508 12:06:20.337 | INFO | RTMetrics report populated | 20260508 12:06:20.337 | INFO | Opening tmp RTMetrics json file | 20260508 12:06:20.346 | INFO | Serializing RTMetrics to json | 20260508 12:06:20.348 | INFO | Serialization done, swapping tmp file now... | 20260508 12:06:20.348 | INFO | RTMetrics done | 20260508 12:06:20.405 | INFO | Finished a round of Baz IO: Wrote 1.417 GiB in 3.466 seconds | 20260508 12:06:20.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:06:20.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172408% / 0% / 5.96046e-05% | 20260508 12:06:20.405 | INFO | Baz IO overall: | 20260508 12:06:20.405 | INFO | BytesWritten : 3426919862910 | 20260508 12:06:20.405 | INFO | NumEvents : 1644982317814 | 20260508 12:06:21.959 | INFO | Analyzing chunk frames = [7974912,7975424) | 20260508 12:06:21.959 | INFO | MemInfo: { | 20260508 12:06:21.959 | INFO | "gpu" : " 17 GB", | 20260508 12:06:21.959 | INFO | "huge" : "", | 20260508 12:06:21.959 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:21.959 | INFO | "malloc" : " 14 GB", | 20260508 12:06:21.959 | INFO | "pinned" : " 29 GB" | 20260508 12:06:21.959 | INFO | } | 20260508 12:06:21.959 | INFO | Perf report for the last 5.13431 seconds: | 20260508 12:06:21.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:21.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:21.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:21.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:21.959 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 12:06:21.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:21.959 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260508 12:06:21.959 | INFO | PRE_HQ : 66.2%, 1.7, 59.6ms, 0 | 20260508 12:06:21.960 | INFO | BAZWRITER : 66.8%, 2.6, 90.4ms, 0 | 20260508 12:06:21.960 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:06:21.960 | INFO | Uploaded 36GiB to GPU in 2.60284s (13.831GiB/s) | 20260508 12:06:21.960 | INFO | Downloaded 38GiB to GPU in 2.71593s (13.9915GiB/s) | 20260508 12:06:27.093 | INFO | Analyzing chunk frames = [7975424,7975936) | 20260508 12:06:27.093 | INFO | MemInfo: { | 20260508 12:06:27.093 | INFO | "gpu" : " 17 GB", | 20260508 12:06:27.093 | INFO | "huge" : "", | 20260508 12:06:27.093 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:27.093 | INFO | "malloc" : " 14 GB", | 20260508 12:06:27.093 | INFO | "pinned" : " 29 GB" | 20260508 12:06:27.093 | INFO | } | 20260508 12:06:27.093 | INFO | Perf report for the last 5.13395 seconds: | 20260508 12:06:27.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:27.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.093 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:06:27.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.093 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:06:27.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:27.094 | INFO | Uploaded 36GiB to GPU in 2.42175s (14.8653GiB/s) | 20260508 12:06:27.094 | INFO | Downloaded 34GiB to GPU in 2.47659s (13.7285GiB/s) | 20260508 12:06:32.241 | INFO | Analyzing chunk frames = [7975936,7976448) | 20260508 12:06:32.241 | INFO | MemInfo: { | 20260508 12:06:32.241 | INFO | "gpu" : " 17 GB", | 20260508 12:06:32.241 | INFO | "huge" : "", | 20260508 12:06:32.241 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:32.241 | INFO | "malloc" : " 14 GB", | 20260508 12:06:32.241 | INFO | "pinned" : " 29 GB" | 20260508 12:06:32.241 | INFO | } | 20260508 12:06:32.241 | INFO | Perf report for the last 5.14775 seconds: | 20260508 12:06:32.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:32.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:06:32.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:06:32.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:32.241 | INFO | Uploaded 36GiB to GPU in 2.45848s (14.6432GiB/s) | 20260508 12:06:32.241 | INFO | Downloaded 34GiB to GPU in 2.45723s (13.8367GiB/s) | 20260508 12:06:37.377 | INFO | Analyzing chunk frames = [7976448,7976960) | 20260508 12:06:37.377 | INFO | MemInfo: { | 20260508 12:06:37.377 | INFO | "gpu" : " 17 GB", | 20260508 12:06:37.377 | INFO | "huge" : "", | 20260508 12:06:37.377 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:37.377 | INFO | "malloc" : " 14 GB", | 20260508 12:06:37.377 | INFO | "pinned" : " 29 GB" | 20260508 12:06:37.377 | INFO | } | 20260508 12:06:37.377 | INFO | Perf report for the last 5.13606 seconds: | 20260508 12:06:37.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:37.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.377 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:06:37.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.377 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:06:37.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:37.378 | INFO | Uploaded 36GiB to GPU in 2.51116s (14.336GiB/s) | 20260508 12:06:37.378 | INFO | Downloaded 34GiB to GPU in 2.46725s (13.7805GiB/s) | 20260508 12:06:42.520 | INFO | Analyzing chunk frames = [7976960,7977472) | 20260508 12:06:42.520 | INFO | MemInfo: { | 20260508 12:06:42.520 | INFO | "gpu" : " 17 GB", | 20260508 12:06:42.520 | INFO | "huge" : "", | 20260508 12:06:42.520 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:42.520 | INFO | "malloc" : " 14 GB", | 20260508 12:06:42.520 | INFO | "pinned" : " 29 GB" | 20260508 12:06:42.520 | INFO | } | 20260508 12:06:42.520 | INFO | Perf report for the last 5.14318 seconds: | 20260508 12:06:42.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:42.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.520 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 12:06:42.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.520 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:06:42.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:42.521 | INFO | Uploaded 36GiB to GPU in 2.29997s (15.6524GiB/s) | 20260508 12:06:42.521 | INFO | Downloaded 34GiB to GPU in 2.48619s (13.6756GiB/s) | 20260508 12:06:47.657 | INFO | Analyzing chunk frames = [7977472,7977984) | 20260508 12:06:47.657 | INFO | MemInfo: { | 20260508 12:06:47.657 | INFO | "gpu" : " 17 GB", | 20260508 12:06:47.657 | INFO | "huge" : "", | 20260508 12:06:47.657 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:47.657 | INFO | "malloc" : " 14 GB", | 20260508 12:06:47.657 | INFO | "pinned" : " 29 GB" | 20260508 12:06:47.657 | INFO | } | 20260508 12:06:47.657 | INFO | Perf report for the last 5.1369 seconds: | 20260508 12:06:47.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:47.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.657 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:06:47.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.657 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:06:47.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:47.658 | INFO | Uploaded 36GiB to GPU in 2.37447s (15.1613GiB/s) | 20260508 12:06:47.658 | INFO | Downloaded 34GiB to GPU in 2.45274s (13.862GiB/s) | 20260508 12:06:52.798 | INFO | Analyzing chunk frames = [7977984,7978496) | 20260508 12:06:52.798 | INFO | MemInfo: { | 20260508 12:06:52.798 | INFO | "gpu" : " 17 GB", | 20260508 12:06:52.798 | INFO | "huge" : "", | 20260508 12:06:52.798 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:52.798 | INFO | "malloc" : " 14 GB", | 20260508 12:06:52.798 | INFO | "pinned" : " 29 GB" | 20260508 12:06:52.798 | INFO | } | 20260508 12:06:52.798 | INFO | Perf report for the last 5.14096 seconds: | 20260508 12:06:52.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:52.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.798 | INFO | ANALYSIS : 65.0%, 3.0, 101.2ms, 0 | 20260508 12:06:52.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.798 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 12:06:52.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:52.799 | INFO | Uploaded 36GiB to GPU in 2.30438s (15.6224GiB/s) | 20260508 12:06:52.799 | INFO | Downloaded 34GiB to GPU in 2.44378s (13.9129GiB/s) | 20260508 12:06:57.933 | INFO | Analyzing chunk frames = [7978496,7979008) | 20260508 12:06:57.933 | INFO | MemInfo: { | 20260508 12:06:57.933 | INFO | "gpu" : " 17 GB", | 20260508 12:06:57.933 | INFO | "huge" : "", | 20260508 12:06:57.933 | INFO | "huge_shared" : " 29 GB", | 20260508 12:06:57.933 | INFO | "malloc" : " 14 GB", | 20260508 12:06:57.933 | INFO | "pinned" : " 29 GB" | 20260508 12:06:57.933 | INFO | } | 20260508 12:06:57.934 | INFO | Perf report for the last 5.13528 seconds: | 20260508 12:06:57.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:06:57.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 12:06:57.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260508 12:06:57.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:06:57.934 | INFO | Uploaded 36GiB to GPU in 2.36608s (15.215GiB/s) | 20260508 12:06:57.934 | INFO | Downloaded 34GiB to GPU in 2.4653s (13.7914GiB/s) | 20260508 12:06:57.934 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:06:57.934 | INFO | Total mem usage: 16.7 GiB | 20260508 12:06:57.934 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:06:57.934 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:06:57.934 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:06:57.934 | INFO | High Water Breakdown: | 20260508 12:06:57.934 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:06:57.934 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:06:57.934 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:06:57.934 | INFO | ------------------------------------------ | 20260508 12:06:57.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:06:57.934 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:06:57.934 | INFO | Total mem usage: 26.5 GiB | 20260508 12:06:57.934 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:06:57.934 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:06:57.934 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:06:57.934 | INFO | High Water Breakdown: | 20260508 12:06:57.934 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:06:57.935 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:06:57.935 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:06:57.935 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:06:57.935 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:06:57.935 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:06:57.935 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:06:57.935 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:06:57.935 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:06:57.935 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:06:57.935 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:06:57.935 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:06:57.935 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:06:57.935 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:06:57.935 | INFO | ------------------------------------------ | 20260508 12:06:57.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:06:57.935 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:06:57.935 | INFO | Total mem usage: 27.8 GiB | 20260508 12:06:57.935 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:06:57.935 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:06:57.935 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:06:57.935 | INFO | High Water Breakdown: | 20260508 12:06:57.935 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:06:57.935 | INFO | ------------------------------------------ | 20260508 12:06:57.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:06:57.935 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:06:57.935 | INFO | Total mem usage: 15.7 GiB | 20260508 12:06:57.935 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:06:57.935 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:06:57.935 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:06:57.935 | INFO | High Water Breakdown: | 20260508 12:06:57.935 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:06:57.935 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:06:57.935 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:06:57.935 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:06:57.935 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:06:57.935 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:06:57.935 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:06:57.935 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:06:57.935 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:06:57.935 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:06:57.935 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:06:57.935 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:06:57.935 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:06:57.935 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:06:57.935 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:06:57.935 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:06:57.935 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:06:57.935 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:06:57.935 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:06:57.935 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:06:57.935 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:06:57.935 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:06:57.935 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:06:57.935 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:06:57.935 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:06:57.935 | INFO | ------------------------------------------ | 20260508 12:06:57.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:06:57.935 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:06:57.935 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:06:57.935 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:06:57.935 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.40e+01 / 5.71e-01 | 20260508 12:06:57.935 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.06e-01, 1.15e-02 / 3.11e-03 | 20260508 12:06:57.935 | INFO | PerfReport: 12.21%, Baseline -- 4.01e+00 / 4.45e+00, 5.98e-01 / 1.96e-01 | 20260508 12:06:57.935 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.50e+00, 2.10e-01 / 9.78e-02 | 20260508 12:06:57.935 | INFO | PerfReport: 15.99%, DME -- 4.20e+01 / 4.55e+01, 3.84e+00 / 9.54e-01 | 20260508 12:06:57.935 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.88e+01 / 1.97e+01, 1.90e+00 / 1.15e+00 | 20260508 12:06:57.935 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.32e+00 / 2.53e+00, 3.99e-01 / 2.15e-01 | 20260508 12:06:57.935 | INFO | PerfReport: 2.96%, Metrics -- 9.73e-01 / 1.12e+00, 2.41e-01 / 1.15e-01 | 20260508 12:06:57.935 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.26e-01, 4.64e-02 / 2.30e-02 | 20260508 12:06:57.935 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:06:57.935 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:06:57.935 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:06:57.935 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:06:57.935 | INFO | PerfReport: 24.11%, Upload -- 2.53e+01 / 0.00e+00, 7.76e+00 / 0.00e+00 | 20260508 12:06:57.935 | INFO | PerfReport: 24.75%, Download -- 2.59e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 12:06:57.935 | INFO | PerfReport: 31.30%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:06:57.935 | INFO | PerfReport: 2.04%, LockUpload -- 2.14e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 12:06:57.935 | INFO | PerfReport: 2.59%, LockDownload -- 2.71e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 12:06:57.936 | INFO | PerfReport: 15.19%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:06:57.936 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 12:06:58.047 | INFO | Starting a round of Baz IO | 20260508 12:07:01.448 | INFO | Populating RTMetrics report | 20260508 12:07:01.449 | INFO | RTMetrics report populated | 20260508 12:07:01.449 | INFO | Opening tmp RTMetrics json file | 20260508 12:07:01.458 | INFO | Serializing RTMetrics to json | 20260508 12:07:01.459 | INFO | Serialization done, swapping tmp file now... | 20260508 12:07:01.459 | INFO | RTMetrics done | 20260508 12:07:01.513 | INFO | Finished a round of Baz IO: Wrote 1.42 GiB in 3.465 seconds | 20260508 12:07:01.513 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:07:01.513 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173577% / 0% / 9.13938e-05% | 20260508 12:07:01.513 | INFO | Baz IO overall: | 20260508 12:07:01.513 | INFO | BytesWritten : 3428444541041 | 20260508 12:07:01.513 | INFO | NumEvents : 1645699055690 | 20260508 12:07:03.075 | INFO | Analyzing chunk frames = [7979008,7979520) | 20260508 12:07:03.075 | INFO | MemInfo: { | 20260508 12:07:03.075 | INFO | "gpu" : " 17 GB", | 20260508 12:07:03.075 | INFO | "huge" : "", | 20260508 12:07:03.075 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:03.075 | INFO | "malloc" : " 14 GB", | 20260508 12:07:03.075 | INFO | "pinned" : " 29 GB" | 20260508 12:07:03.075 | INFO | } | 20260508 12:07:03.075 | INFO | Perf report for the last 5.14119 seconds: | 20260508 12:07:03.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:03.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:03.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:03.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:03.075 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 12:07:03.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:03.075 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 12:07:03.075 | INFO | PRE_HQ : 66.4%, 1.8, 62.0ms, 0 | 20260508 12:07:03.075 | INFO | BAZWRITER : 66.7%, 2.7, 92.0ms, 0 | 20260508 12:07:03.075 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:07:03.075 | INFO | Uploaded 36GiB to GPU in 2.54637s (14.1378GiB/s) | 20260508 12:07:03.075 | INFO | Downloaded 38GiB to GPU in 2.72329s (13.9537GiB/s) | 20260508 12:07:08.208 | INFO | Analyzing chunk frames = [7979520,7980032) | 20260508 12:07:08.208 | INFO | MemInfo: { | 20260508 12:07:08.208 | INFO | "gpu" : " 17 GB", | 20260508 12:07:08.208 | INFO | "huge" : "", | 20260508 12:07:08.208 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:08.208 | INFO | "malloc" : " 14 GB", | 20260508 12:07:08.208 | INFO | "pinned" : " 29 GB" | 20260508 12:07:08.208 | INFO | } | 20260508 12:07:08.208 | INFO | Perf report for the last 5.13368 seconds: | 20260508 12:07:08.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:08.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.209 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 12:07:08.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.209 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:07:08.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:08.209 | INFO | Uploaded 36GiB to GPU in 2.41024s (14.9363GiB/s) | 20260508 12:07:08.209 | INFO | Downloaded 34GiB to GPU in 2.47825s (13.7194GiB/s) | 20260508 12:07:13.347 | INFO | Analyzing chunk frames = [7980032,7980544) | 20260508 12:07:13.347 | INFO | MemInfo: { | 20260508 12:07:13.347 | INFO | "gpu" : " 17 GB", | 20260508 12:07:13.347 | INFO | "huge" : "", | 20260508 12:07:13.347 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:13.347 | INFO | "malloc" : " 14 GB", | 20260508 12:07:13.347 | INFO | "pinned" : " 29 GB" | 20260508 12:07:13.347 | INFO | } | 20260508 12:07:13.347 | INFO | Perf report for the last 5.13901 seconds: | 20260508 12:07:13.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:13.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 12:07:13.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:07:13.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:13.348 | INFO | Uploaded 36GiB to GPU in 2.45422s (14.6686GiB/s) | 20260508 12:07:13.348 | INFO | Downloaded 34GiB to GPU in 2.46291s (13.8048GiB/s) | 20260508 12:07:18.492 | INFO | Analyzing chunk frames = [7980544,7981056) | 20260508 12:07:18.492 | INFO | MemInfo: { | 20260508 12:07:18.492 | INFO | "gpu" : " 17 GB", | 20260508 12:07:18.492 | INFO | "huge" : "", | 20260508 12:07:18.492 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:18.492 | INFO | "malloc" : " 14 GB", | 20260508 12:07:18.492 | INFO | "pinned" : " 29 GB" | 20260508 12:07:18.492 | INFO | } | 20260508 12:07:18.493 | INFO | Perf report for the last 5.14502 seconds: | 20260508 12:07:18.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:18.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 12:07:18.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:07:18.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:18.493 | INFO | Uploaded 36GiB to GPU in 2.44498s (14.724GiB/s) | 20260508 12:07:18.493 | INFO | Downloaded 34GiB to GPU in 2.46587s (13.7883GiB/s) | 20260508 12:07:23.634 | INFO | Analyzing chunk frames = [7981056,7981568) | 20260508 12:07:23.634 | INFO | MemInfo: { | 20260508 12:07:23.634 | INFO | "gpu" : " 17 GB", | 20260508 12:07:23.635 | INFO | "huge" : "", | 20260508 12:07:23.635 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:23.635 | INFO | "malloc" : " 14 GB", | 20260508 12:07:23.635 | INFO | "pinned" : " 29 GB" | 20260508 12:07:23.635 | INFO | } | 20260508 12:07:23.635 | INFO | Perf report for the last 5.14214 seconds: | 20260508 12:07:23.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:23.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 12:07:23.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:07:23.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:23.635 | INFO | Uploaded 36GiB to GPU in 2.35228s (15.3043GiB/s) | 20260508 12:07:23.635 | INFO | Downloaded 34GiB to GPU in 2.52143s (13.4844GiB/s) | 20260508 12:07:28.770 | INFO | Analyzing chunk frames = [7981568,7982080) | 20260508 12:07:28.770 | INFO | MemInfo: { | 20260508 12:07:28.770 | INFO | "gpu" : " 17 GB", | 20260508 12:07:28.770 | INFO | "huge" : "", | 20260508 12:07:28.770 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:28.770 | INFO | "malloc" : " 14 GB", | 20260508 12:07:28.770 | INFO | "pinned" : " 29 GB" | 20260508 12:07:28.770 | INFO | } | 20260508 12:07:28.770 | INFO | Perf report for the last 5.13526 seconds: | 20260508 12:07:28.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:28.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.770 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:07:28.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.770 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:07:28.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:28.771 | INFO | Uploaded 36GiB to GPU in 2.36585s (15.2165GiB/s) | 20260508 12:07:28.771 | INFO | Downloaded 34GiB to GPU in 2.43393s (13.9692GiB/s) | 20260508 12:07:33.904 | INFO | Analyzing chunk frames = [7982080,7982592) | 20260508 12:07:33.904 | INFO | MemInfo: { | 20260508 12:07:33.904 | INFO | "gpu" : " 17 GB", | 20260508 12:07:33.904 | INFO | "huge" : "", | 20260508 12:07:33.904 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:33.904 | INFO | "malloc" : " 14 GB", | 20260508 12:07:33.904 | INFO | "pinned" : " 29 GB" | 20260508 12:07:33.904 | INFO | } | 20260508 12:07:33.904 | INFO | Perf report for the last 5.13404 seconds: | 20260508 12:07:33.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:33.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 12:07:33.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 12:07:33.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:33.904 | INFO | Uploaded 36GiB to GPU in 2.4144s (14.9105GiB/s) | 20260508 12:07:33.904 | INFO | Downloaded 34GiB to GPU in 2.44909s (13.8827GiB/s) | 20260508 12:07:39.046 | INFO | Analyzing chunk frames = [7982592,7983104) | 20260508 12:07:39.046 | INFO | MemInfo: { | 20260508 12:07:39.046 | INFO | "gpu" : " 17 GB", | 20260508 12:07:39.046 | INFO | "huge" : "", | 20260508 12:07:39.046 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:39.046 | INFO | "malloc" : " 14 GB", | 20260508 12:07:39.046 | INFO | "pinned" : " 29 GB" | 20260508 12:07:39.046 | INFO | } | 20260508 12:07:39.047 | INFO | Perf report for the last 5.14251 seconds: | 20260508 12:07:39.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:39.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:07:39.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:07:39.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:39.047 | INFO | Uploaded 36GiB to GPU in 2.41509s (14.9063GiB/s) | 20260508 12:07:39.047 | INFO | Downloaded 34GiB to GPU in 2.44458s (13.9083GiB/s) | 20260508 12:07:39.144 | INFO | Starting a round of Baz IO | 20260508 12:07:42.566 | INFO | Populating RTMetrics report | 20260508 12:07:42.566 | INFO | RTMetrics report populated | 20260508 12:07:42.566 | INFO | Opening tmp RTMetrics json file | 20260508 12:07:42.573 | INFO | Serializing RTMetrics to json | 20260508 12:07:42.574 | INFO | Serialization done, swapping tmp file now... | 20260508 12:07:42.575 | INFO | RTMetrics done | 20260508 12:07:42.627 | INFO | Finished a round of Baz IO: Wrote 1.417 GiB in 3.483 seconds | 20260508 12:07:42.627 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:07:42.627 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171979% / 3.97364e-06% / 6.75519e-05% | 20260508 12:07:42.627 | INFO | Baz IO overall: | 20260508 12:07:42.627 | INFO | BytesWritten : 3429965854785 | 20260508 12:07:42.627 | INFO | NumEvents : 1646413287921 | 20260508 12:07:44.192 | INFO | Analyzing chunk frames = [7983104,7983616) | 20260508 12:07:44.192 | INFO | MemInfo: { | 20260508 12:07:44.192 | INFO | "gpu" : " 17 GB", | 20260508 12:07:44.192 | INFO | "huge" : "", | 20260508 12:07:44.192 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:44.192 | INFO | "malloc" : " 14 GB", | 20260508 12:07:44.192 | INFO | "pinned" : " 29 GB" | 20260508 12:07:44.192 | INFO | } | 20260508 12:07:44.192 | INFO | Perf report for the last 5.14572 seconds: | 20260508 12:07:44.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:44.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:44.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:44.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:44.192 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 12:07:44.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:44.192 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 12:07:44.192 | INFO | PRE_HQ : 65.6%, 1.6, 54.0ms, 0 | 20260508 12:07:44.192 | INFO | BAZWRITER : 67.0%, 2.5, 85.0ms, 0 | 20260508 12:07:44.192 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:07:44.193 | INFO | Uploaded 36GiB to GPU in 2.60903s (13.7982GiB/s) | 20260508 12:07:44.193 | INFO | Downloaded 38GiB to GPU in 2.71767s (13.9825GiB/s) | 20260508 12:07:49.331 | INFO | Analyzing chunk frames = [7983616,7984128) | 20260508 12:07:49.331 | INFO | MemInfo: { | 20260508 12:07:49.331 | INFO | "gpu" : " 17 GB", | 20260508 12:07:49.331 | INFO | "huge" : "", | 20260508 12:07:49.331 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:49.331 | INFO | "malloc" : " 14 GB", | 20260508 12:07:49.331 | INFO | "pinned" : " 29 GB" | 20260508 12:07:49.331 | INFO | } | 20260508 12:07:49.331 | INFO | Perf report for the last 5.13866 seconds: | 20260508 12:07:49.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:49.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 12:07:49.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:07:49.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:49.331 | INFO | Uploaded 36GiB to GPU in 2.48528s (14.4853GiB/s) | 20260508 12:07:49.331 | INFO | Downloaded 34GiB to GPU in 2.46402s (13.7986GiB/s) | 20260508 12:07:54.472 | INFO | Analyzing chunk frames = [7984128,7984640) | 20260508 12:07:54.472 | INFO | MemInfo: { | 20260508 12:07:54.472 | INFO | "gpu" : " 17 GB", | 20260508 12:07:54.472 | INFO | "huge" : "", | 20260508 12:07:54.472 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:54.472 | INFO | "malloc" : " 14 GB", | 20260508 12:07:54.472 | INFO | "pinned" : " 29 GB" | 20260508 12:07:54.472 | INFO | } | 20260508 12:07:54.472 | INFO | Perf report for the last 5.14111 seconds: | 20260508 12:07:54.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:54.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 12:07:54.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:07:54.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:54.472 | INFO | Uploaded 36GiB to GPU in 2.50585s (14.3664GiB/s) | 20260508 12:07:54.472 | INFO | Downloaded 34GiB to GPU in 2.44529s (13.9043GiB/s) | 20260508 12:07:59.602 | INFO | Analyzing chunk frames = [7984640,7985152) | 20260508 12:07:59.602 | INFO | MemInfo: { | 20260508 12:07:59.602 | INFO | "gpu" : " 17 GB", | 20260508 12:07:59.602 | INFO | "huge" : "", | 20260508 12:07:59.602 | INFO | "huge_shared" : " 29 GB", | 20260508 12:07:59.602 | INFO | "malloc" : " 14 GB", | 20260508 12:07:59.602 | INFO | "pinned" : " 29 GB" | 20260508 12:07:59.602 | INFO | } | 20260508 12:07:59.602 | INFO | Perf report for the last 5.12998 seconds: | 20260508 12:07:59.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:07:59.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | ANALYSIS : 66.4%, 3.0, 102.7ms, 0 | 20260508 12:07:59.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:07:59.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:07:59.602 | INFO | Uploaded 36GiB to GPU in 2.53726s (14.1885GiB/s) | 20260508 12:07:59.602 | INFO | Downloaded 34GiB to GPU in 2.45785s (13.8332GiB/s) | 20260508 12:08:04.744 | INFO | Analyzing chunk frames = [7985152,7985664) | 20260508 12:08:04.744 | INFO | MemInfo: { | 20260508 12:08:04.744 | INFO | "gpu" : " 17 GB", | 20260508 12:08:04.744 | INFO | "huge" : "", | 20260508 12:08:04.745 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:04.745 | INFO | "malloc" : " 14 GB", | 20260508 12:08:04.745 | INFO | "pinned" : " 29 GB" | 20260508 12:08:04.745 | INFO | } | 20260508 12:08:04.745 | INFO | Perf report for the last 5.14293 seconds: | 20260508 12:08:04.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:04.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:08:04.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:08:04.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:04.745 | INFO | Uploaded 36GiB to GPU in 2.31984s (15.5183GiB/s) | 20260508 12:08:04.745 | INFO | Downloaded 34GiB to GPU in 2.49897s (13.6056GiB/s) | 20260508 12:08:09.889 | INFO | Analyzing chunk frames = [7985664,7986176) | 20260508 12:08:09.890 | INFO | MemInfo: { | 20260508 12:08:09.890 | INFO | "gpu" : " 17 GB", | 20260508 12:08:09.890 | INFO | "huge" : "", | 20260508 12:08:09.890 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:09.890 | INFO | "malloc" : " 14 GB", | 20260508 12:08:09.890 | INFO | "pinned" : " 29 GB" | 20260508 12:08:09.890 | INFO | } | 20260508 12:08:09.890 | INFO | Perf report for the last 5.14478 seconds: | 20260508 12:08:09.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:09.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 12:08:09.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260508 12:08:09.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:09.890 | INFO | Uploaded 36GiB to GPU in 2.383s (15.107GiB/s) | 20260508 12:08:09.890 | INFO | Downloaded 34GiB to GPU in 2.43017s (13.9908GiB/s) | 20260508 12:08:15.026 | INFO | Analyzing chunk frames = [7986176,7986688) | 20260508 12:08:15.026 | INFO | MemInfo: { | 20260508 12:08:15.026 | INFO | "gpu" : " 17 GB", | 20260508 12:08:15.026 | INFO | "huge" : "", | 20260508 12:08:15.026 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:15.026 | INFO | "malloc" : " 14 GB", | 20260508 12:08:15.026 | INFO | "pinned" : " 29 GB" | 20260508 12:08:15.026 | INFO | } | 20260508 12:08:15.026 | INFO | Perf report for the last 5.13607 seconds: | 20260508 12:08:15.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:15.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 12:08:15.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:08:15.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:15.026 | INFO | Uploaded 36GiB to GPU in 2.33167s (15.4396GiB/s) | 20260508 12:08:15.027 | INFO | Downloaded 34GiB to GPU in 2.43788s (13.9465GiB/s) | 20260508 12:08:20.162 | INFO | Analyzing chunk frames = [7986688,7987200) | 20260508 12:08:20.162 | INFO | MemInfo: { | 20260508 12:08:20.162 | INFO | "gpu" : " 17 GB", | 20260508 12:08:20.162 | INFO | "huge" : "", | 20260508 12:08:20.162 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:20.162 | INFO | "malloc" : " 14 GB", | 20260508 12:08:20.162 | INFO | "pinned" : " 29 GB" | 20260508 12:08:20.162 | INFO | } | 20260508 12:08:20.162 | INFO | Perf report for the last 5.13656 seconds: | 20260508 12:08:20.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:20.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.163 | INFO | ANALYSIS : 65.2%, 3.0, 101.9ms, 0 | 20260508 12:08:20.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.163 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:08:20.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:20.163 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:08:20.163 | INFO | Total mem usage: 16.7 GiB | 20260508 12:08:20.163 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:08:20.163 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.8%) | 20260508 12:08:20.163 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:08:20.163 | INFO | High Water Breakdown: | 20260508 12:08:20.163 | INFO | 59.6% : 8.52 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:08:20.163 | INFO | 40.3% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:08:20.163 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:08:20.163 | INFO | ------------------------------------------ | 20260508 12:08:20.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:08:20.163 | INFO | Uploaded 36GiB to GPU in 2.42642s (14.8367GiB/s) | 20260508 12:08:20.163 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:08:20.163 | INFO | Total mem usage: 26.5 GiB | 20260508 12:08:20.163 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:08:20.163 | INFO | Downloaded 34GiB to GPU in 2.45252s (13.8633GiB/s) | 20260508 12:08:20.163 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:08:20.163 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:08:20.163 | INFO | High Water Breakdown: | 20260508 12:08:20.163 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:08:20.163 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:08:20.163 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:08:20.163 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:08:20.163 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:08:20.163 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:08:20.163 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:08:20.163 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:08:20.163 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:08:20.163 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:08:20.163 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:08:20.163 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:08:20.164 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:08:20.164 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:08:20.164 | INFO | ------------------------------------------ | 20260508 12:08:20.164 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:08:20.164 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:08:20.164 | INFO | Total mem usage: 27.8 GiB | 20260508 12:08:20.164 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:08:20.164 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:08:20.164 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:08:20.164 | INFO | High Water Breakdown: | 20260508 12:08:20.164 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:08:20.164 | INFO | ------------------------------------------ | 20260508 12:08:20.164 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:08:20.164 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:08:20.164 | INFO | Total mem usage: 15.7 GiB | 20260508 12:08:20.164 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:08:20.164 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:08:20.164 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:08:20.164 | INFO | High Water Breakdown: | 20260508 12:08:20.164 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:08:20.164 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:08:20.164 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:08:20.164 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:08:20.164 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:08:20.164 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:08:20.164 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:08:20.164 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:08:20.164 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:08:20.164 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:08:20.164 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:08:20.164 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:08:20.164 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:08:20.164 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:08:20.164 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:08:20.164 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:08:20.164 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:08:20.164 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:08:20.164 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:08:20.164 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:08:20.164 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:08:20.164 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:08:20.164 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:08:20.164 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:08:20.164 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:08:20.164 | INFO | ------------------------------------------ | 20260508 12:08:20.164 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:08:20.164 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:08:20.164 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:08:20.164 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:08:20.164 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.66e-01 | 20260508 12:08:20.164 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.15e-02 / 2.56e-03 | 20260508 12:08:20.164 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.41e+00, 5.90e-01 / 2.53e-01 | 20260508 12:08:20.164 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.49e+00, 2.20e-01 / 9.41e-02 | 20260508 12:08:20.164 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.54e+01, 3.83e+00 / 1.07e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.83e+00 / 1.16e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.52e+00, 4.01e-01 / 2.58e-01 | 20260508 12:08:20.164 | INFO | PerfReport: 2.96%, Metrics -- 9.68e-01 / 1.13e+00, 2.40e-01 / 1.22e-01 | 20260508 12:08:20.164 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.33e-01, 4.82e-02 / 2.83e-02 | 20260508 12:08:20.164 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:08:20.164 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:08:20.164 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:08:20.164 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 24.19%, Upload -- 2.54e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.16e+00 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 31.15%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 2.11%, LockUpload -- 2.22e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 2.66%, LockDownload -- 2.79e+00 / 0.00e+00, 4.27e+00 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 15.13%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:08:20.164 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260508 12:08:20.260 | INFO | Starting a round of Baz IO | 20260508 12:08:23.660 | INFO | Populating RTMetrics report | 20260508 12:08:23.660 | INFO | RTMetrics report populated | 20260508 12:08:23.660 | INFO | Opening tmp RTMetrics json file | 20260508 12:08:23.669 | INFO | Serializing RTMetrics to json | 20260508 12:08:23.670 | INFO | Serialization done, swapping tmp file now... | 20260508 12:08:23.671 | INFO | RTMetrics done | 20260508 12:08:23.729 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.468 seconds | 20260508 12:08:23.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:08:23.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171626% / 3.97364e-06% / 4.37101e-05% | 20260508 12:08:23.729 | INFO | Baz IO overall: | 20260508 12:08:23.729 | INFO | BytesWritten : 3431488297774 | 20260508 12:08:23.729 | INFO | NumEvents : 1647128354038 | 20260508 12:08:25.300 | INFO | Analyzing chunk frames = [7987200,7987712) | 20260508 12:08:25.300 | INFO | MemInfo: { | 20260508 12:08:25.300 | INFO | "gpu" : " 17 GB", | 20260508 12:08:25.300 | INFO | "huge" : "", | 20260508 12:08:25.300 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:25.300 | INFO | "malloc" : " 14 GB", | 20260508 12:08:25.300 | INFO | "pinned" : " 29 GB" | 20260508 12:08:25.300 | INFO | } | 20260508 12:08:25.300 | INFO | Perf report for the last 5.13775 seconds: | 20260508 12:08:25.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:25.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:25.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:25.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:25.300 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 12:08:25.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:25.300 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:08:25.300 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:08:25.300 | INFO | BAZWRITER : 66.8%, 2.5, 84.9ms, 0 | 20260508 12:08:25.300 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:08:25.301 | INFO | Uploaded 36GiB to GPU in 2.6209s (13.7357GiB/s) | 20260508 12:08:25.301 | INFO | Downloaded 38GiB to GPU in 2.71632s (13.9895GiB/s) | 20260508 12:08:30.444 | INFO | Analyzing chunk frames = [7987712,7988224) | 20260508 12:08:30.444 | INFO | MemInfo: { | 20260508 12:08:30.444 | INFO | "gpu" : " 17 GB", | 20260508 12:08:30.444 | INFO | "huge" : "", | 20260508 12:08:30.444 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:30.444 | INFO | "malloc" : " 14 GB", | 20260508 12:08:30.445 | INFO | "pinned" : " 29 GB" | 20260508 12:08:30.445 | INFO | } | 20260508 12:08:30.445 | INFO | Perf report for the last 5.14436 seconds: | 20260508 12:08:30.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:30.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:08:30.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:08:30.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:30.445 | INFO | Uploaded 36GiB to GPU in 2.40614s (14.9617GiB/s) | 20260508 12:08:30.445 | INFO | Downloaded 34GiB to GPU in 2.47532s (13.7356GiB/s) | 20260508 12:08:35.582 | INFO | Analyzing chunk frames = [7988224,7988736) | 20260508 12:08:35.582 | INFO | MemInfo: { | 20260508 12:08:35.582 | INFO | "gpu" : " 17 GB", | 20260508 12:08:35.582 | INFO | "huge" : "", | 20260508 12:08:35.582 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:35.582 | INFO | "malloc" : " 14 GB", | 20260508 12:08:35.582 | INFO | "pinned" : " 29 GB" | 20260508 12:08:35.582 | INFO | } | 20260508 12:08:35.582 | INFO | Perf report for the last 5.13701 seconds: | 20260508 12:08:35.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:35.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 12:08:35.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:08:35.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:35.582 | INFO | Uploaded 36GiB to GPU in 2.47815s (14.527GiB/s) | 20260508 12:08:35.582 | INFO | Downloaded 34GiB to GPU in 2.47677s (13.7276GiB/s) | 20260508 12:08:40.720 | INFO | Analyzing chunk frames = [7988736,7989248) | 20260508 12:08:40.720 | INFO | MemInfo: { | 20260508 12:08:40.720 | INFO | "gpu" : " 17 GB", | 20260508 12:08:40.720 | INFO | "huge" : "", | 20260508 12:08:40.720 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:40.720 | INFO | "malloc" : " 14 GB", | 20260508 12:08:40.720 | INFO | "pinned" : " 29 GB" | 20260508 12:08:40.720 | INFO | } | 20260508 12:08:40.720 | INFO | Perf report for the last 5.13851 seconds: | 20260508 12:08:40.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:40.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:08:40.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:08:40.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:40.720 | INFO | Uploaded 36GiB to GPU in 2.50259s (14.3851GiB/s) | 20260508 12:08:40.720 | INFO | Downloaded 34GiB to GPU in 2.47314s (13.7477GiB/s) | 20260508 12:08:45.855 | INFO | Analyzing chunk frames = [7989248,7989760) | 20260508 12:08:45.855 | INFO | MemInfo: { | 20260508 12:08:45.855 | INFO | "gpu" : " 17 GB", | 20260508 12:08:45.855 | INFO | "huge" : "", | 20260508 12:08:45.855 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:45.855 | INFO | "malloc" : " 14 GB", | 20260508 12:08:45.855 | INFO | "pinned" : " 29 GB" | 20260508 12:08:45.855 | INFO | } | 20260508 12:08:45.855 | INFO | Perf report for the last 5.13497 seconds: | 20260508 12:08:45.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:45.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.855 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 12:08:45.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.855 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 12:08:45.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:45.856 | INFO | Uploaded 36GiB to GPU in 2.38553s (15.091GiB/s) | 20260508 12:08:45.856 | INFO | Downloaded 34GiB to GPU in 2.51629s (13.512GiB/s) | 20260508 12:08:50.999 | INFO | Analyzing chunk frames = [7989760,7990272) | 20260508 12:08:50.999 | INFO | MemInfo: { | 20260508 12:08:50.999 | INFO | "gpu" : " 17 GB", | 20260508 12:08:50.999 | INFO | "huge" : "", | 20260508 12:08:50.999 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:50.999 | INFO | "malloc" : " 14 GB", | 20260508 12:08:50.999 | INFO | "pinned" : " 29 GB" | 20260508 12:08:50.999 | INFO | } | 20260508 12:08:50.999 | INFO | Perf report for the last 5.14363 seconds: | 20260508 12:08:50.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:50.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:08:50.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:08:50.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:50.999 | INFO | Uploaded 36GiB to GPU in 2.38063s (15.122GiB/s) | 20260508 12:08:51.000 | INFO | Downloaded 34GiB to GPU in 2.4351s (13.9625GiB/s) | 20260508 12:08:56.145 | INFO | Analyzing chunk frames = [7990272,7990784) | 20260508 12:08:56.145 | INFO | MemInfo: { | 20260508 12:08:56.145 | INFO | "gpu" : " 17 GB", | 20260508 12:08:56.145 | INFO | "huge" : "", | 20260508 12:08:56.145 | INFO | "huge_shared" : " 29 GB", | 20260508 12:08:56.145 | INFO | "malloc" : " 14 GB", | 20260508 12:08:56.145 | INFO | "pinned" : " 29 GB" | 20260508 12:08:56.145 | INFO | } | 20260508 12:08:56.145 | INFO | Perf report for the last 5.14654 seconds: | 20260508 12:08:56.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:08:56.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.145 | INFO | ANALYSIS : 65.1%, 3.0, 101.7ms, 0 | 20260508 12:08:56.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.145 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:08:56.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:08:56.146 | INFO | Uploaded 36GiB to GPU in 2.42332s (14.8556GiB/s) | 20260508 12:08:56.146 | INFO | Downloaded 34GiB to GPU in 2.44111s (13.9281GiB/s) | 20260508 12:09:01.279 | INFO | Analyzing chunk frames = [7990784,7991296) | 20260508 12:09:01.279 | INFO | MemInfo: { | 20260508 12:09:01.279 | INFO | "gpu" : " 17 GB", | 20260508 12:09:01.279 | INFO | "huge" : "", | 20260508 12:09:01.279 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:01.279 | INFO | "malloc" : " 14 GB", | 20260508 12:09:01.279 | INFO | "pinned" : " 29 GB" | 20260508 12:09:01.279 | INFO | } | 20260508 12:09:01.279 | INFO | Perf report for the last 5.13398 seconds: | 20260508 12:09:01.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:01.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.279 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 12:09:01.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.279 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:09:01.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:01.280 | INFO | Uploaded 36GiB to GPU in 2.49512s (14.4281GiB/s) | 20260508 12:09:01.280 | INFO | Downloaded 34GiB to GPU in 2.47667s (13.7281GiB/s) | 20260508 12:09:01.377 | INFO | Starting a round of Baz IO | 20260508 12:09:04.803 | INFO | Populating RTMetrics report | 20260508 12:09:04.803 | INFO | RTMetrics report populated | 20260508 12:09:04.803 | INFO | Opening tmp RTMetrics json file | 20260508 12:09:04.809 | INFO | Serializing RTMetrics to json | 20260508 12:09:04.818 | INFO | Serialization done, swapping tmp file now... | 20260508 12:09:04.818 | INFO | RTMetrics done | 20260508 12:09:04.877 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.5 seconds | 20260508 12:09:04.877 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.00% / 59.95% | 20260508 12:09:04.877 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172134% / 0% / 7.94729e-05% | 20260508 12:09:04.877 | INFO | Baz IO overall: | 20260508 12:09:04.877 | INFO | BytesWritten : 3433010642140 | 20260508 12:09:04.877 | INFO | NumEvents : 1647843170335 | 20260508 12:09:06.415 | INFO | Analyzing chunk frames = [7991296,7991808) | 20260508 12:09:06.415 | INFO | MemInfo: { | 20260508 12:09:06.415 | INFO | "gpu" : " 17 GB", | 20260508 12:09:06.415 | INFO | "huge" : "", | 20260508 12:09:06.415 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:06.415 | INFO | "malloc" : " 14 GB", | 20260508 12:09:06.415 | INFO | "pinned" : " 29 GB" | 20260508 12:09:06.415 | INFO | } | 20260508 12:09:06.415 | INFO | Perf report for the last 5.13596 seconds: | 20260508 12:09:06.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:06.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:06.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:06.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:06.415 | INFO | ANALYSIS : 68.3%, 3.0, 106.3ms, 0 | 20260508 12:09:06.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:06.415 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:09:06.415 | INFO | PRE_HQ : 65.9%, 1.6, 54.0ms, 0 | 20260508 12:09:06.415 | INFO | BAZWRITER : 67.4%, 2.4, 84.8ms, 0 | 20260508 12:09:06.415 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:09:06.416 | INFO | Uploaded 36GiB to GPU in 2.5716s (13.9991GiB/s) | 20260508 12:09:06.416 | INFO | Downloaded 38GiB to GPU in 2.69371s (14.107GiB/s) | 20260508 12:09:11.558 | INFO | Analyzing chunk frames = [7991808,7992320) | 20260508 12:09:11.558 | INFO | MemInfo: { | 20260508 12:09:11.558 | INFO | "gpu" : " 17 GB", | 20260508 12:09:11.558 | INFO | "huge" : "", | 20260508 12:09:11.558 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:11.558 | INFO | "malloc" : " 14 GB", | 20260508 12:09:11.558 | INFO | "pinned" : " 29 GB" | 20260508 12:09:11.558 | INFO | } | 20260508 12:09:11.558 | INFO | Perf report for the last 5.14302 seconds: | 20260508 12:09:11.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:11.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.558 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:09:11.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.558 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:09:11.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:11.559 | INFO | Uploaded 36GiB to GPU in 2.48552s (14.4839GiB/s) | 20260508 12:09:11.559 | INFO | Downloaded 34GiB to GPU in 2.45892s (13.8272GiB/s) | 20260508 12:09:16.695 | INFO | Analyzing chunk frames = [7992320,7992832) | 20260508 12:09:16.695 | INFO | MemInfo: { | 20260508 12:09:16.695 | INFO | "gpu" : " 17 GB", | 20260508 12:09:16.695 | INFO | "huge" : "", | 20260508 12:09:16.695 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:16.695 | INFO | "malloc" : " 14 GB", | 20260508 12:09:16.695 | INFO | "pinned" : " 29 GB" | 20260508 12:09:16.695 | INFO | } | 20260508 12:09:16.696 | INFO | Perf report for the last 5.13721 seconds: | 20260508 12:09:16.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:16.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 12:09:16.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 12:09:16.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:16.696 | INFO | Uploaded 36GiB to GPU in 2.52113s (14.2793GiB/s) | 20260508 12:09:16.696 | INFO | Downloaded 34GiB to GPU in 2.45826s (13.8309GiB/s) | 20260508 12:09:21.835 | INFO | Analyzing chunk frames = [7992832,7993344) | 20260508 12:09:21.835 | INFO | MemInfo: { | 20260508 12:09:21.835 | INFO | "gpu" : " 17 GB", | 20260508 12:09:21.836 | INFO | "huge" : "", | 20260508 12:09:21.836 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:21.836 | INFO | "malloc" : " 14 GB", | 20260508 12:09:21.836 | INFO | "pinned" : " 29 GB" | 20260508 12:09:21.836 | INFO | } | 20260508 12:09:21.836 | INFO | Perf report for the last 5.14016 seconds: | 20260508 12:09:21.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:21.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 12:09:21.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:09:21.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:21.836 | INFO | Uploaded 36GiB to GPU in 2.50101s (14.3942GiB/s) | 20260508 12:09:21.836 | INFO | Downloaded 34GiB to GPU in 2.44678s (13.8958GiB/s) | 20260508 12:09:26.979 | INFO | Analyzing chunk frames = [7993344,7993856) | 20260508 12:09:26.979 | INFO | MemInfo: { | 20260508 12:09:26.979 | INFO | "gpu" : " 17 GB", | 20260508 12:09:26.979 | INFO | "huge" : "", | 20260508 12:09:26.979 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:26.979 | INFO | "malloc" : " 14 GB", | 20260508 12:09:26.979 | INFO | "pinned" : " 29 GB" | 20260508 12:09:26.979 | INFO | } | 20260508 12:09:26.979 | INFO | Perf report for the last 5.14336 seconds: | 20260508 12:09:26.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:26.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 12:09:26.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:09:26.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:26.979 | INFO | Uploaded 36GiB to GPU in 2.32004s (15.517GiB/s) | 20260508 12:09:26.979 | INFO | Downloaded 34GiB to GPU in 2.50148s (13.5919GiB/s) | 20260508 12:09:32.111 | INFO | Analyzing chunk frames = [7993856,7994368) | 20260508 12:09:32.111 | INFO | MemInfo: { | 20260508 12:09:32.111 | INFO | "gpu" : " 17 GB", | 20260508 12:09:32.111 | INFO | "huge" : "", | 20260508 12:09:32.111 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:32.111 | INFO | "malloc" : " 14 GB", | 20260508 12:09:32.111 | INFO | "pinned" : " 29 GB" | 20260508 12:09:32.111 | INFO | } | 20260508 12:09:32.111 | INFO | Perf report for the last 5.13214 seconds: | 20260508 12:09:32.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:32.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.111 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:09:32.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.111 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:09:32.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:32.112 | INFO | Uploaded 36GiB to GPU in 2.41457s (14.9095GiB/s) | 20260508 12:09:32.112 | INFO | Downloaded 34GiB to GPU in 2.42479s (14.0218GiB/s) | 20260508 12:09:37.258 | INFO | Analyzing chunk frames = [7994368,7994880) | 20260508 12:09:37.258 | INFO | MemInfo: { | 20260508 12:09:37.258 | INFO | "gpu" : " 17 GB", | 20260508 12:09:37.258 | INFO | "huge" : "", | 20260508 12:09:37.258 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:37.258 | INFO | "malloc" : " 14 GB", | 20260508 12:09:37.258 | INFO | "pinned" : " 29 GB" | 20260508 12:09:37.258 | INFO | } | 20260508 12:09:37.258 | INFO | Perf report for the last 5.14697 seconds: | 20260508 12:09:37.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:37.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.258 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 12:09:37.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.258 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:09:37.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:37.259 | INFO | Uploaded 36GiB to GPU in 2.44511s (14.7233GiB/s) | 20260508 12:09:37.259 | INFO | Downloaded 34GiB to GPU in 2.42099s (14.0439GiB/s) | 20260508 12:09:42.397 | INFO | Analyzing chunk frames = [7994880,7995392) | 20260508 12:09:42.397 | INFO | MemInfo: { | 20260508 12:09:42.397 | INFO | "gpu" : " 17 GB", | 20260508 12:09:42.397 | INFO | "huge" : "", | 20260508 12:09:42.397 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:42.397 | INFO | "malloc" : " 14 GB", | 20260508 12:09:42.397 | INFO | "pinned" : " 29 GB" | 20260508 12:09:42.397 | INFO | } | 20260508 12:09:42.398 | INFO | Perf report for the last 5.13937 seconds: | 20260508 12:09:42.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:42.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | ANALYSIS : 65.0%, 3.0, 101.5ms, 0 | 20260508 12:09:42.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:09:42.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:42.398 | INFO | Uploaded 36GiB to GPU in 2.40502s (14.9687GiB/s) | 20260508 12:09:42.398 | INFO | Downloaded 34GiB to GPU in 2.43472s (13.9647GiB/s) | 20260508 12:09:42.398 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:09:42.398 | INFO | Total mem usage: 16.7 GiB | 20260508 12:09:42.398 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:09:42.398 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:09:42.398 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:09:42.398 | INFO | High Water Breakdown: | 20260508 12:09:42.398 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:09:42.398 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:09:42.398 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:09:42.398 | INFO | ------------------------------------------ | 20260508 12:09:42.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:09:42.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:09:42.398 | INFO | Total mem usage: 26.5 GiB | 20260508 12:09:42.398 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:09:42.398 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:09:42.398 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:09:42.398 | INFO | High Water Breakdown: | 20260508 12:09:42.398 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:09:42.398 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:09:42.398 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:09:42.398 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:09:42.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:09:42.398 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:09:42.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:09:42.398 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:09:42.398 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:09:42.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:09:42.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:09:42.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:09:42.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:09:42.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:09:42.398 | INFO | ------------------------------------------ | 20260508 12:09:42.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:09:42.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:09:42.398 | INFO | Total mem usage: 27.8 GiB | 20260508 12:09:42.398 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:09:42.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:09:42.398 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:09:42.398 | INFO | High Water Breakdown: | 20260508 12:09:42.398 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:09:42.398 | INFO | ------------------------------------------ | 20260508 12:09:42.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:09:42.398 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:09:42.398 | INFO | Total mem usage: 15.7 GiB | 20260508 12:09:42.398 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:09:42.398 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:09:42.398 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:09:42.398 | INFO | High Water Breakdown: | 20260508 12:09:42.398 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:09:42.398 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:09:42.398 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:09:42.398 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:09:42.398 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:09:42.398 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:09:42.398 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:09:42.398 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:09:42.398 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:09:42.398 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:09:42.398 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:09:42.398 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:09:42.398 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:09:42.398 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:09:42.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:09:42.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:09:42.398 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:09:42.398 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:09:42.398 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:09:42.398 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:09:42.398 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:09:42.398 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:09:42.398 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:09:42.398 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:09:42.398 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:09:42.398 | INFO | ------------------------------------------ | 20260508 12:09:42.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:09:42.398 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:09:42.398 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:09:42.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:09:42.398 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.39e+01 / 5.88e-01 | 20260508 12:09:42.398 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.04e-01, 1.19e-02 / 3.36e-03 | 20260508 12:09:42.398 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.45e+00, 5.96e-01 / 2.42e-01 | 20260508 12:09:42.398 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 1.22e-01 | 20260508 12:09:42.398 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.55e+01, 3.69e+00 / 1.20e+00 | 20260508 12:09:42.398 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.87e+01 / 2.00e+01, 1.85e+00 / 1.46e+00 | 20260508 12:09:42.398 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.55e+00, 3.98e-01 / 2.22e-01 | 20260508 12:09:42.398 | INFO | PerfReport: 2.96%, Metrics -- 9.68e-01 / 1.13e+00, 2.44e-01 / 1.18e-01 | 20260508 12:09:42.398 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.23e-01, 4.83e-02 / 2.60e-02 | 20260508 12:09:42.398 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:09:42.398 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:09:42.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:09:42.398 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:09:42.398 | INFO | PerfReport: 24.38%, Upload -- 2.56e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 12:09:42.398 | INFO | PerfReport: 24.68%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 12:09:42.398 | INFO | PerfReport: 31.09%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:09:42.399 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 12:09:42.399 | INFO | PerfReport: 2.57%, LockDownload -- 2.70e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 12:09:42.399 | INFO | PerfReport: 15.11%, LockCompute -- 1.59e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 12:09:42.399 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260508 12:09:42.495 | INFO | Starting a round of Baz IO | 20260508 12:09:45.894 | INFO | Populating RTMetrics report | 20260508 12:09:45.894 | INFO | RTMetrics report populated | 20260508 12:09:45.894 | INFO | Opening tmp RTMetrics json file | 20260508 12:09:45.900 | INFO | Serializing RTMetrics to json | 20260508 12:09:45.901 | INFO | Serialization done, swapping tmp file now... | 20260508 12:09:45.902 | INFO | RTMetrics done | 20260508 12:09:45.962 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.467 seconds | 20260508 12:09:45.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.95% | 20260508 12:09:45.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17184% / 0% / 8.34465e-05% | 20260508 12:09:45.962 | INFO | Baz IO overall: | 20260508 12:09:45.962 | INFO | BytesWritten : 3434532914257 | 20260508 12:09:45.962 | INFO | NumEvents : 1648557936769 | 20260508 12:09:47.532 | INFO | Analyzing chunk frames = [7995392,7995904) | 20260508 12:09:47.532 | INFO | MemInfo: { | 20260508 12:09:47.532 | INFO | "gpu" : " 17 GB", | 20260508 12:09:47.532 | INFO | "huge" : "", | 20260508 12:09:47.533 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:47.533 | INFO | "malloc" : " 14 GB", | 20260508 12:09:47.533 | INFO | "pinned" : " 29 GB" | 20260508 12:09:47.533 | INFO | } | 20260508 12:09:47.533 | INFO | Perf report for the last 5.1352 seconds: | 20260508 12:09:47.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:47.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:47.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:47.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:47.533 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:09:47.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:47.533 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 12:09:47.533 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 12:09:47.533 | INFO | BAZWRITER : 66.8%, 2.5, 85.0ms, 0 | 20260508 12:09:47.533 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:09:47.534 | INFO | Uploaded 36GiB to GPU in 2.61974s (13.7418GiB/s) | 20260508 12:09:47.534 | INFO | Downloaded 38GiB to GPU in 2.72745s (13.9324GiB/s) | 20260508 12:09:52.666 | INFO | Analyzing chunk frames = [7995904,7996416) | 20260508 12:09:52.666 | INFO | MemInfo: { | 20260508 12:09:52.666 | INFO | "gpu" : " 17 GB", | 20260508 12:09:52.666 | INFO | "huge" : "", | 20260508 12:09:52.666 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:52.666 | INFO | "malloc" : " 14 GB", | 20260508 12:09:52.666 | INFO | "pinned" : " 29 GB" | 20260508 12:09:52.666 | INFO | } | 20260508 12:09:52.666 | INFO | Perf report for the last 5.13324 seconds: | 20260508 12:09:52.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:52.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.666 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 12:09:52.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.666 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:09:52.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:52.667 | INFO | Uploaded 36GiB to GPU in 2.43659s (14.7747GiB/s) | 20260508 12:09:52.667 | INFO | Downloaded 34GiB to GPU in 2.46979s (13.7663GiB/s) | 20260508 12:09:57.810 | INFO | Analyzing chunk frames = [7996416,7996928) | 20260508 12:09:57.810 | INFO | MemInfo: { | 20260508 12:09:57.810 | INFO | "gpu" : " 17 GB", | 20260508 12:09:57.811 | INFO | "huge" : "", | 20260508 12:09:57.811 | INFO | "huge_shared" : " 29 GB", | 20260508 12:09:57.811 | INFO | "malloc" : " 14 GB", | 20260508 12:09:57.811 | INFO | "pinned" : " 29 GB" | 20260508 12:09:57.811 | INFO | } | 20260508 12:09:57.811 | INFO | Perf report for the last 5.14459 seconds: | 20260508 12:09:57.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:09:57.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 12:09:57.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:09:57.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:09:57.811 | INFO | Uploaded 36GiB to GPU in 2.4215s (14.8668GiB/s) | 20260508 12:09:57.811 | INFO | Downloaded 34GiB to GPU in 2.44224s (13.9217GiB/s) | 20260508 12:10:02.949 | INFO | Analyzing chunk frames = [7996928,7997440) | 20260508 12:10:02.949 | INFO | MemInfo: { | 20260508 12:10:02.949 | INFO | "gpu" : " 17 GB", | 20260508 12:10:02.949 | INFO | "huge" : "", | 20260508 12:10:02.949 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:02.949 | INFO | "malloc" : " 14 GB", | 20260508 12:10:02.949 | INFO | "pinned" : " 29 GB" | 20260508 12:10:02.949 | INFO | } | 20260508 12:10:02.949 | INFO | Perf report for the last 5.13844 seconds: | 20260508 12:10:02.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:02.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:10:02.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:10:02.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:02.949 | INFO | Uploaded 36GiB to GPU in 2.47251s (14.5601GiB/s) | 20260508 12:10:02.950 | INFO | Downloaded 34GiB to GPU in 2.46811s (13.7757GiB/s) | 20260508 12:10:08.095 | INFO | Analyzing chunk frames = [7997440,7997952) | 20260508 12:10:08.095 | INFO | MemInfo: { | 20260508 12:10:08.095 | INFO | "gpu" : " 17 GB", | 20260508 12:10:08.095 | INFO | "huge" : "", | 20260508 12:10:08.095 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:08.095 | INFO | "malloc" : " 14 GB", | 20260508 12:10:08.095 | INFO | "pinned" : " 29 GB" | 20260508 12:10:08.095 | INFO | } | 20260508 12:10:08.095 | INFO | Perf report for the last 5.14606 seconds: | 20260508 12:10:08.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:08.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.095 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 12:10:08.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.095 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:10:08.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:08.096 | INFO | Uploaded 36GiB to GPU in 2.2858s (15.7494GiB/s) | 20260508 12:10:08.096 | INFO | Downloaded 34GiB to GPU in 2.51323s (13.5284GiB/s) | 20260508 12:10:13.236 | INFO | Analyzing chunk frames = [7997952,7998464) | 20260508 12:10:13.236 | INFO | MemInfo: { | 20260508 12:10:13.236 | INFO | "gpu" : " 17 GB", | 20260508 12:10:13.236 | INFO | "huge" : "", | 20260508 12:10:13.236 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:13.236 | INFO | "malloc" : " 14 GB", | 20260508 12:10:13.236 | INFO | "pinned" : " 29 GB" | 20260508 12:10:13.236 | INFO | } | 20260508 12:10:13.236 | INFO | Perf report for the last 5.14084 seconds: | 20260508 12:10:13.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:13.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:10:13.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:10:13.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:13.236 | INFO | Uploaded 36GiB to GPU in 2.38003s (15.1259GiB/s) | 20260508 12:10:13.236 | INFO | Downloaded 34GiB to GPU in 2.43689s (13.9522GiB/s) | 20260508 12:10:18.366 | INFO | Analyzing chunk frames = [7998464,7998976) | 20260508 12:10:18.366 | INFO | MemInfo: { | 20260508 12:10:18.366 | INFO | "gpu" : " 17 GB", | 20260508 12:10:18.366 | INFO | "huge" : "", | 20260508 12:10:18.366 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:18.367 | INFO | "malloc" : " 14 GB", | 20260508 12:10:18.367 | INFO | "pinned" : " 29 GB" | 20260508 12:10:18.367 | INFO | } | 20260508 12:10:18.367 | INFO | Perf report for the last 5.13051 seconds: | 20260508 12:10:18.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:18.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | ANALYSIS : 65.1%, 3.0, 101.1ms, 0 | 20260508 12:10:18.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:10:18.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:18.367 | INFO | Uploaded 36GiB to GPU in 2.3476s (15.3348GiB/s) | 20260508 12:10:18.367 | INFO | Downloaded 34GiB to GPU in 2.42121s (14.0426GiB/s) | 20260508 12:10:23.506 | INFO | Analyzing chunk frames = [7998976,7999488) | 20260508 12:10:23.506 | INFO | MemInfo: { | 20260508 12:10:23.506 | INFO | "gpu" : " 17 GB", | 20260508 12:10:23.506 | INFO | "huge" : "", | 20260508 12:10:23.506 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:23.506 | INFO | "malloc" : " 14 GB", | 20260508 12:10:23.506 | INFO | "pinned" : " 29 GB" | 20260508 12:10:23.506 | INFO | } | 20260508 12:10:23.506 | INFO | Perf report for the last 5.13982 seconds: | 20260508 12:10:23.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:23.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.507 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 12:10:23.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.507 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 12:10:23.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:23.507 | INFO | Uploaded 36GiB to GPU in 2.45524s (14.6625GiB/s) | 20260508 12:10:23.507 | INFO | Downloaded 34GiB to GPU in 2.4595s (13.8239GiB/s) | 20260508 12:10:23.619 | INFO | Starting a round of Baz IO | 20260508 12:10:27.021 | INFO | Populating RTMetrics report | 20260508 12:10:27.021 | INFO | RTMetrics report populated | 20260508 12:10:27.021 | INFO | Opening tmp RTMetrics json file | 20260508 12:10:27.027 | INFO | Serializing RTMetrics to json | 20260508 12:10:27.028 | INFO | Serialization done, swapping tmp file now... | 20260508 12:10:27.029 | INFO | RTMetrics done | 20260508 12:10:27.084 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.465 seconds | 20260508 12:10:27.085 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:10:27.085 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172154% / 0% / 5.5631e-05% | 20260508 12:10:27.085 | INFO | Baz IO overall: | 20260508 12:10:27.085 | INFO | BytesWritten : 3436055697288 | 20260508 12:10:27.085 | INFO | NumEvents : 1649273218220 | 20260508 12:10:28.647 | INFO | Analyzing chunk frames = [7999488,8000000) | 20260508 12:10:28.647 | INFO | MemInfo: { | 20260508 12:10:28.647 | INFO | "gpu" : " 17 GB", | 20260508 12:10:28.647 | INFO | "huge" : "", | 20260508 12:10:28.647 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:28.647 | INFO | "malloc" : " 14 GB", | 20260508 12:10:28.647 | INFO | "pinned" : " 29 GB" | 20260508 12:10:28.647 | INFO | } | 20260508 12:10:28.647 | INFO | Perf report for the last 5.14044 seconds: | 20260508 12:10:28.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:28.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:28.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:28.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:28.647 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:10:28.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:28.647 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260508 12:10:28.647 | INFO | PRE_HQ : 66.1%, 1.8, 60.4ms, 0 | 20260508 12:10:28.647 | INFO | BAZWRITER : 66.8%, 2.6, 90.7ms, 0 | 20260508 12:10:28.647 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:10:28.647 | INFO | Uploaded 36GiB to GPU in 2.61276s (13.7785GiB/s) | 20260508 12:10:28.648 | INFO | Downloaded 38GiB to GPU in 2.71391s (14.0019GiB/s) | 20260508 12:10:33.787 | INFO | Analyzing chunk frames = [8000000,8000512) | 20260508 12:10:33.787 | INFO | MemInfo: { | 20260508 12:10:33.787 | INFO | "gpu" : " 17 GB", | 20260508 12:10:33.787 | INFO | "huge" : "", | 20260508 12:10:33.787 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:33.787 | INFO | "malloc" : " 14 GB", | 20260508 12:10:33.787 | INFO | "pinned" : " 29 GB" | 20260508 12:10:33.787 | INFO | } | 20260508 12:10:33.788 | INFO | Perf report for the last 5.14062 seconds: | 20260508 12:10:33.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:33.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:10:33.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:10:33.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:33.788 | INFO | Uploaded 36GiB to GPU in 2.49725s (14.4159GiB/s) | 20260508 12:10:33.788 | INFO | Downloaded 34GiB to GPU in 2.46634s (13.7856GiB/s) | 20260508 12:10:38.931 | INFO | Analyzing chunk frames = [8000512,8001024) | 20260508 12:10:38.931 | INFO | MemInfo: { | 20260508 12:10:38.931 | INFO | "gpu" : " 17 GB", | 20260508 12:10:38.931 | INFO | "huge" : "", | 20260508 12:10:38.931 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:38.931 | INFO | "malloc" : " 14 GB", | 20260508 12:10:38.931 | INFO | "pinned" : " 29 GB" | 20260508 12:10:38.931 | INFO | } | 20260508 12:10:38.931 | INFO | Perf report for the last 5.14333 seconds: | 20260508 12:10:38.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:38.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.931 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 12:10:38.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.931 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:10:38.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:38.932 | INFO | Uploaded 36GiB to GPU in 2.52217s (14.2734GiB/s) | 20260508 12:10:38.932 | INFO | Downloaded 34GiB to GPU in 2.44726s (13.8931GiB/s) | 20260508 12:10:44.068 | INFO | Analyzing chunk frames = [8001024,8001536) | 20260508 12:10:44.068 | INFO | MemInfo: { | 20260508 12:10:44.068 | INFO | "gpu" : " 17 GB", | 20260508 12:10:44.068 | INFO | "huge" : "", | 20260508 12:10:44.068 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:44.068 | INFO | "malloc" : " 14 GB", | 20260508 12:10:44.068 | INFO | "pinned" : " 29 GB" | 20260508 12:10:44.068 | INFO | } | 20260508 12:10:44.068 | INFO | Perf report for the last 5.1375 seconds: | 20260508 12:10:44.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:44.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.068 | INFO | ANALYSIS : 65.5%, 3.0, 101.5ms, 0 | 20260508 12:10:44.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.068 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:10:44.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:44.069 | INFO | Uploaded 36GiB to GPU in 2.42109s (14.8693GiB/s) | 20260508 12:10:44.069 | INFO | Downloaded 34GiB to GPU in 2.45504s (13.849GiB/s) | 20260508 12:10:49.206 | INFO | Analyzing chunk frames = [8001536,8002048) | 20260508 12:10:49.206 | INFO | MemInfo: { | 20260508 12:10:49.206 | INFO | "gpu" : " 17 GB", | 20260508 12:10:49.206 | INFO | "huge" : "", | 20260508 12:10:49.206 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:49.206 | INFO | "malloc" : " 14 GB", | 20260508 12:10:49.206 | INFO | "pinned" : " 29 GB" | 20260508 12:10:49.206 | INFO | } | 20260508 12:10:49.206 | INFO | Perf report for the last 5.13795 seconds: | 20260508 12:10:49.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:49.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.206 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:10:49.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.206 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:10:49.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:49.207 | INFO | Uploaded 36GiB to GPU in 2.35817s (15.2661GiB/s) | 20260508 12:10:49.207 | INFO | Downloaded 34GiB to GPU in 2.49649s (13.6191GiB/s) | 20260508 12:10:54.346 | INFO | Analyzing chunk frames = [8002048,8002560) | 20260508 12:10:54.346 | INFO | MemInfo: { | 20260508 12:10:54.346 | INFO | "gpu" : " 17 GB", | 20260508 12:10:54.346 | INFO | "huge" : "", | 20260508 12:10:54.346 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:54.346 | INFO | "malloc" : " 14 GB", | 20260508 12:10:54.346 | INFO | "pinned" : " 29 GB" | 20260508 12:10:54.346 | INFO | } | 20260508 12:10:54.346 | INFO | Perf report for the last 5.13972 seconds: | 20260508 12:10:54.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:54.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:10:54.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:10:54.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:54.346 | INFO | Uploaded 36GiB to GPU in 2.40863s (14.9463GiB/s) | 20260508 12:10:54.346 | INFO | Downloaded 34GiB to GPU in 2.42776s (14.0047GiB/s) | 20260508 12:10:59.482 | INFO | Analyzing chunk frames = [8002560,8003072) | 20260508 12:10:59.482 | INFO | MemInfo: { | 20260508 12:10:59.482 | INFO | "gpu" : " 17 GB", | 20260508 12:10:59.482 | INFO | "huge" : "", | 20260508 12:10:59.483 | INFO | "huge_shared" : " 29 GB", | 20260508 12:10:59.483 | INFO | "malloc" : " 14 GB", | 20260508 12:10:59.483 | INFO | "pinned" : " 29 GB" | 20260508 12:10:59.483 | INFO | } | 20260508 12:10:59.483 | INFO | Perf report for the last 5.13661 seconds: | 20260508 12:10:59.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:10:59.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 12:10:59.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 12:10:59.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:10:59.483 | INFO | Uploaded 36GiB to GPU in 2.48242s (14.502GiB/s) | 20260508 12:10:59.483 | INFO | Downloaded 34GiB to GPU in 2.45102s (13.8718GiB/s) | 20260508 12:11:04.616 | INFO | Analyzing chunk frames = [8003072,8003584) | 20260508 12:11:04.616 | INFO | MemInfo: { | 20260508 12:11:04.616 | INFO | "gpu" : " 17 GB", | 20260508 12:11:04.616 | INFO | "huge" : "", | 20260508 12:11:04.616 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:04.616 | INFO | "malloc" : " 14 GB", | 20260508 12:11:04.616 | INFO | "pinned" : " 29 GB" | 20260508 12:11:04.616 | INFO | } | 20260508 12:11:04.616 | INFO | Perf report for the last 5.1337 seconds: | 20260508 12:11:04.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:04.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 12:11:04.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:11:04.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:04.617 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:11:04.617 | INFO | Total mem usage: 16.7 GiB | 20260508 12:11:04.617 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:11:04.617 | INFO | Uploaded 36GiB to GPU in 2.47023s (14.5735GiB/s) | 20260508 12:11:04.617 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:11:04.617 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:11:04.617 | INFO | Downloaded 34GiB to GPU in 2.43538s (13.9609GiB/s) | 20260508 12:11:04.617 | INFO | High Water Breakdown: | 20260508 12:11:04.617 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:11:04.617 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:11:04.617 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:11:04.617 | INFO | ------------------------------------------ | 20260508 12:11:04.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:11:04.617 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:11:04.617 | INFO | Total mem usage: 26.5 GiB | 20260508 12:11:04.617 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:11:04.617 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:11:04.617 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:11:04.617 | INFO | High Water Breakdown: | 20260508 12:11:04.617 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:11:04.617 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:11:04.617 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:11:04.617 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:11:04.617 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:11:04.617 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:11:04.617 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:11:04.617 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:11:04.617 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:11:04.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:11:04.617 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:11:04.617 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:11:04.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:11:04.617 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:11:04.617 | INFO | ------------------------------------------ | 20260508 12:11:04.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:11:04.617 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:11:04.617 | INFO | Total mem usage: 27.8 GiB | 20260508 12:11:04.617 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:11:04.617 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:11:04.618 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:11:04.618 | INFO | High Water Breakdown: | 20260508 12:11:04.618 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:11:04.618 | INFO | ------------------------------------------ | 20260508 12:11:04.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:11:04.618 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:11:04.618 | INFO | Total mem usage: 15.7 GiB | 20260508 12:11:04.618 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:11:04.618 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:11:04.618 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:11:04.618 | INFO | High Water Breakdown: | 20260508 12:11:04.618 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:11:04.618 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:11:04.618 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:11:04.618 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:11:04.618 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:11:04.618 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:11:04.618 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:11:04.618 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:11:04.618 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:11:04.618 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:11:04.618 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:11:04.618 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:11:04.618 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:11:04.618 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:11:04.618 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:11:04.618 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:11:04.618 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:11:04.618 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:11:04.618 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:11:04.618 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:11:04.618 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:11:04.618 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:11:04.618 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:11:04.618 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:11:04.618 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:11:04.618 | INFO | ------------------------------------------ | 20260508 12:11:04.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:11:04.618 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:11:04.618 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:11:04.618 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:11:04.618 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.63e-01 | 20260508 12:11:04.618 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.05e-02 / 5.72e-03 | 20260508 12:11:04.618 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.41e+00, 5.96e-01 / 2.20e-01 | 20260508 12:11:04.618 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.04e-01 | 20260508 12:11:04.618 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.56e+01, 3.81e+00 / 8.23e-01 | 20260508 12:11:04.618 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.76e+00 / 9.75e-01 | 20260508 12:11:04.618 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.51e+00, 4.04e-01 / 2.31e-01 | 20260508 12:11:04.618 | INFO | PerfReport: 2.98%, Metrics -- 9.72e-01 / 1.09e+00, 2.38e-01 / 9.70e-02 | 20260508 12:11:04.618 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.26e-01, 4.69e-02 / 2.52e-02 | 20260508 12:11:04.618 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:11:04.618 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:11:04.618 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:11:04.618 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 24.31%, Upload -- 2.55e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 24.70%, Download -- 2.59e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 2.55%, LockDownload -- 2.67e+00 / 0.00e+00, 4.05e+00 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 15.15%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:11:04.618 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260508 12:11:04.714 | INFO | Starting a round of Baz IO | 20260508 12:11:08.133 | INFO | Populating RTMetrics report | 20260508 12:11:08.133 | INFO | RTMetrics report populated | 20260508 12:11:08.133 | INFO | Opening tmp RTMetrics json file | 20260508 12:11:08.141 | INFO | Serializing RTMetrics to json | 20260508 12:11:08.142 | INFO | Serialization done, swapping tmp file now... | 20260508 12:11:08.143 | INFO | RTMetrics done | 20260508 12:11:08.197 | INFO | Finished a round of Baz IO: Wrote 1.418 GiB in 3.482 seconds | 20260508 12:11:08.197 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:11:08.197 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172047% / 0% / 7.94729e-05% | 20260508 12:11:08.197 | INFO | Baz IO overall: | 20260508 12:11:08.197 | INFO | BytesWritten : 3437578368864 | 20260508 12:11:08.197 | INFO | NumEvents : 1649988269724 | 20260508 12:11:09.766 | INFO | Analyzing chunk frames = [8003584,8004096) | 20260508 12:11:09.766 | INFO | MemInfo: { | 20260508 12:11:09.766 | INFO | "gpu" : " 17 GB", | 20260508 12:11:09.766 | INFO | "huge" : "", | 20260508 12:11:09.766 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:09.766 | INFO | "malloc" : " 14 GB", | 20260508 12:11:09.766 | INFO | "pinned" : " 29 GB" | 20260508 12:11:09.766 | INFO | } | 20260508 12:11:09.766 | INFO | Perf report for the last 5.14972 seconds: | 20260508 12:11:09.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:09.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:09.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:09.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:09.766 | INFO | ANALYSIS : 67.4%, 3.0, 105.6ms, 0 | 20260508 12:11:09.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:09.766 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:11:09.766 | INFO | PRE_HQ : 65.2%, 1.6, 53.9ms, 0 | 20260508 12:11:09.766 | INFO | BAZWRITER : 66.9%, 2.5, 85.2ms, 0 | 20260508 12:11:09.766 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:11:09.767 | INFO | Uploaded 36GiB to GPU in 2.61638s (13.7595GiB/s) | 20260508 12:11:09.767 | INFO | Downloaded 38GiB to GPU in 2.70457s (14.0503GiB/s) | 20260508 12:11:14.907 | INFO | Analyzing chunk frames = [8004096,8004608) | 20260508 12:11:14.907 | INFO | MemInfo: { | 20260508 12:11:14.907 | INFO | "gpu" : " 17 GB", | 20260508 12:11:14.907 | INFO | "huge" : "", | 20260508 12:11:14.907 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:14.907 | INFO | "malloc" : " 14 GB", | 20260508 12:11:14.907 | INFO | "pinned" : " 29 GB" | 20260508 12:11:14.907 | INFO | } | 20260508 12:11:14.907 | INFO | Perf report for the last 5.14067 seconds: | 20260508 12:11:14.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:14.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:11:14.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:11:14.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:14.907 | INFO | Uploaded 36GiB to GPU in 2.47747s (14.531GiB/s) | 20260508 12:11:14.907 | INFO | Downloaded 34GiB to GPU in 2.45765s (13.8344GiB/s) | 20260508 12:11:20.043 | INFO | Analyzing chunk frames = [8004608,8005120) | 20260508 12:11:20.043 | INFO | MemInfo: { | 20260508 12:11:20.043 | INFO | "gpu" : " 17 GB", | 20260508 12:11:20.043 | INFO | "huge" : "", | 20260508 12:11:20.043 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:20.043 | INFO | "malloc" : " 14 GB", | 20260508 12:11:20.043 | INFO | "pinned" : " 29 GB" | 20260508 12:11:20.043 | INFO | } | 20260508 12:11:20.043 | INFO | Perf report for the last 5.13602 seconds: | 20260508 12:11:20.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:20.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 12:11:20.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:11:20.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:20.043 | INFO | Uploaded 36GiB to GPU in 2.48376s (14.4941GiB/s) | 20260508 12:11:20.044 | INFO | Downloaded 34GiB to GPU in 2.45894s (13.8271GiB/s) | 20260508 12:11:25.176 | INFO | Analyzing chunk frames = [8005120,8005632) | 20260508 12:11:25.176 | INFO | MemInfo: { | 20260508 12:11:25.176 | INFO | "gpu" : " 17 GB", | 20260508 12:11:25.176 | INFO | "huge" : "", | 20260508 12:11:25.176 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:25.176 | INFO | "malloc" : " 14 GB", | 20260508 12:11:25.176 | INFO | "pinned" : " 29 GB" | 20260508 12:11:25.176 | INFO | } | 20260508 12:11:25.176 | INFO | Perf report for the last 5.13301 seconds: | 20260508 12:11:25.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:25.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 12:11:25.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:11:25.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:25.176 | INFO | Uploaded 36GiB to GPU in 2.49834s (14.4096GiB/s) | 20260508 12:11:25.176 | INFO | Downloaded 34GiB to GPU in 2.44637s (13.8981GiB/s) | 20260508 12:11:30.318 | INFO | Analyzing chunk frames = [8005632,8006144) | 20260508 12:11:30.318 | INFO | MemInfo: { | 20260508 12:11:30.318 | INFO | "gpu" : " 17 GB", | 20260508 12:11:30.318 | INFO | "huge" : "", | 20260508 12:11:30.318 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:30.318 | INFO | "malloc" : " 14 GB", | 20260508 12:11:30.318 | INFO | "pinned" : " 29 GB" | 20260508 12:11:30.318 | INFO | } | 20260508 12:11:30.318 | INFO | Perf report for the last 5.14235 seconds: | 20260508 12:11:30.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:30.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 12:11:30.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:11:30.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:30.318 | INFO | Uploaded 36GiB to GPU in 2.2837s (15.7639GiB/s) | 20260508 12:11:30.318 | INFO | Downloaded 34GiB to GPU in 2.4869s (13.6716GiB/s) | 20260508 12:11:35.459 | INFO | Analyzing chunk frames = [8006144,8006656) | 20260508 12:11:35.459 | INFO | MemInfo: { | 20260508 12:11:35.459 | INFO | "gpu" : " 17 GB", | 20260508 12:11:35.459 | INFO | "huge" : "", | 20260508 12:11:35.459 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:35.459 | INFO | "malloc" : " 14 GB", | 20260508 12:11:35.459 | INFO | "pinned" : " 29 GB" | 20260508 12:11:35.459 | INFO | } | 20260508 12:11:35.459 | INFO | Perf report for the last 5.14103 seconds: | 20260508 12:11:35.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:35.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.459 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 12:11:35.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.459 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260508 12:11:35.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:35.460 | INFO | Uploaded 36GiB to GPU in 2.43064s (14.8109GiB/s) | 20260508 12:11:35.460 | INFO | Downloaded 34GiB to GPU in 2.42556s (14.0174GiB/s) | 20260508 12:11:40.605 | INFO | Analyzing chunk frames = [8006656,8007168) | 20260508 12:11:40.605 | INFO | MemInfo: { | 20260508 12:11:40.605 | INFO | "gpu" : " 17 GB", | 20260508 12:11:40.605 | INFO | "huge" : "", | 20260508 12:11:40.605 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:40.605 | INFO | "malloc" : " 14 GB", | 20260508 12:11:40.605 | INFO | "pinned" : " 29 GB" | 20260508 12:11:40.605 | INFO | } | 20260508 12:11:40.605 | INFO | Perf report for the last 5.14576 seconds: | 20260508 12:11:40.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:40.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 12:11:40.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:11:40.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:40.605 | INFO | Uploaded 36GiB to GPU in 2.45183s (14.6829GiB/s) | 20260508 12:11:40.606 | INFO | Downloaded 34GiB to GPU in 2.43478s (13.9643GiB/s) | 20260508 12:11:45.743 | INFO | Analyzing chunk frames = [8007168,8007680) | 20260508 12:11:45.743 | INFO | MemInfo: { | 20260508 12:11:45.743 | INFO | "gpu" : " 17 GB", | 20260508 12:11:45.743 | INFO | "huge" : "", | 20260508 12:11:45.743 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:45.743 | INFO | "malloc" : " 14 GB", | 20260508 12:11:45.743 | INFO | "pinned" : " 29 GB" | 20260508 12:11:45.743 | INFO | } | 20260508 12:11:45.743 | INFO | Perf report for the last 5.13813 seconds: | 20260508 12:11:45.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:45.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 12:11:45.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:11:45.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:45.743 | INFO | Uploaded 36GiB to GPU in 2.49663s (14.4194GiB/s) | 20260508 12:11:45.743 | INFO | Downloaded 34GiB to GPU in 2.45429s (13.8533GiB/s) | 20260508 12:11:45.840 | INFO | Starting a round of Baz IO | 20260508 12:11:49.244 | INFO | Populating RTMetrics report | 20260508 12:11:49.244 | INFO | RTMetrics report populated | 20260508 12:11:49.244 | INFO | Opening tmp RTMetrics json file | 20260508 12:11:49.245 | INFO | Serializing RTMetrics to json | 20260508 12:11:49.246 | INFO | Serialization done, swapping tmp file now... | 20260508 12:11:49.246 | INFO | RTMetrics done | 20260508 12:11:49.304 | INFO | Finished a round of Baz IO: Wrote 1.417 GiB in 3.464 seconds | 20260508 12:11:49.304 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:11:49.304 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171045% / 0% / 7.54992e-05% | 20260508 12:11:49.304 | INFO | Baz IO overall: | 20260508 12:11:49.304 | INFO | BytesWritten : 3439099651929 | 20260508 12:11:49.304 | INFO | NumEvents : 1650702618996 | 20260508 12:11:50.876 | INFO | Analyzing chunk frames = [8007680,8008192) | 20260508 12:11:50.876 | INFO | MemInfo: { | 20260508 12:11:50.876 | INFO | "gpu" : " 17 GB", | 20260508 12:11:50.876 | INFO | "huge" : "", | 20260508 12:11:50.876 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:50.876 | INFO | "malloc" : " 14 GB", | 20260508 12:11:50.876 | INFO | "pinned" : " 29 GB" | 20260508 12:11:50.876 | INFO | } | 20260508 12:11:50.876 | INFO | Perf report for the last 5.13243 seconds: | 20260508 12:11:50.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:50.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:50.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:50.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:50.876 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260508 12:11:50.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:50.876 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:11:50.876 | INFO | PRE_HQ : 65.8%, 1.6, 53.9ms, 0 | 20260508 12:11:50.876 | INFO | BAZWRITER : 66.7%, 2.5, 85.3ms, 0 | 20260508 12:11:50.876 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 12:11:50.876 | INFO | Uploaded 36GiB to GPU in 2.54975s (14.119GiB/s) | 20260508 12:11:50.876 | INFO | Downloaded 38GiB to GPU in 2.70145s (14.0665GiB/s) | 20260508 12:11:56.016 | INFO | Analyzing chunk frames = [8008192,8008704) | 20260508 12:11:56.016 | INFO | MemInfo: { | 20260508 12:11:56.016 | INFO | "gpu" : " 17 GB", | 20260508 12:11:56.017 | INFO | "huge" : "", | 20260508 12:11:56.017 | INFO | "huge_shared" : " 29 GB", | 20260508 12:11:56.017 | INFO | "malloc" : " 14 GB", | 20260508 12:11:56.017 | INFO | "pinned" : " 29 GB" | 20260508 12:11:56.017 | INFO | } | 20260508 12:11:56.017 | INFO | Perf report for the last 5.14109 seconds: | 20260508 12:11:56.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:11:56.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:11:56.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:11:56.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:11:56.017 | INFO | Uploaded 36GiB to GPU in 2.43922s (14.7588GiB/s) | 20260508 12:11:56.017 | INFO | Downloaded 34GiB to GPU in 2.47092s (13.76GiB/s) | 20260508 12:12:01.158 | INFO | Analyzing chunk frames = [8008704,8009216) | 20260508 12:12:01.158 | INFO | MemInfo: { | 20260508 12:12:01.158 | INFO | "gpu" : " 17 GB", | 20260508 12:12:01.158 | INFO | "huge" : "", | 20260508 12:12:01.158 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:01.158 | INFO | "malloc" : " 14 GB", | 20260508 12:12:01.158 | INFO | "pinned" : " 29 GB" | 20260508 12:12:01.158 | INFO | } | 20260508 12:12:01.158 | INFO | Perf report for the last 5.14132 seconds: | 20260508 12:12:01.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:01.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.158 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 12:12:01.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.158 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:12:01.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:01.159 | INFO | Uploaded 36GiB to GPU in 2.48383s (14.4937GiB/s) | 20260508 12:12:01.159 | INFO | Downloaded 34GiB to GPU in 2.45975s (13.8225GiB/s) | 20260508 12:12:06.295 | INFO | Analyzing chunk frames = [8009216,8009728) | 20260508 12:12:06.295 | INFO | MemInfo: { | 20260508 12:12:06.295 | INFO | "gpu" : " 17 GB", | 20260508 12:12:06.295 | INFO | "huge" : "", | 20260508 12:12:06.295 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:06.295 | INFO | "malloc" : " 14 GB", | 20260508 12:12:06.295 | INFO | "pinned" : " 29 GB" | 20260508 12:12:06.295 | INFO | } | 20260508 12:12:06.295 | INFO | Perf report for the last 5.13693 seconds: | 20260508 12:12:06.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:06.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:12:06.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:12:06.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:06.295 | INFO | Uploaded 36GiB to GPU in 2.50661s (14.362GiB/s) | 20260508 12:12:06.295 | INFO | Downloaded 34GiB to GPU in 2.44454s (13.9085GiB/s) | 20260508 12:12:11.431 | INFO | Analyzing chunk frames = [8009728,8010240) | 20260508 12:12:11.431 | INFO | MemInfo: { | 20260508 12:12:11.431 | INFO | "gpu" : " 17 GB", | 20260508 12:12:11.431 | INFO | "huge" : "", | 20260508 12:12:11.431 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:11.431 | INFO | "malloc" : " 14 GB", | 20260508 12:12:11.431 | INFO | "pinned" : " 29 GB" | 20260508 12:12:11.431 | INFO | } | 20260508 12:12:11.431 | INFO | Perf report for the last 5.13585 seconds: | 20260508 12:12:11.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:11.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 12:12:11.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:12:11.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:11.431 | INFO | Uploaded 36GiB to GPU in 2.44002s (14.754GiB/s) | 20260508 12:12:11.431 | INFO | Downloaded 34GiB to GPU in 2.55308s (13.3173GiB/s) | 20260508 12:12:16.571 | INFO | Analyzing chunk frames = [8010240,8010752) | 20260508 12:12:16.571 | INFO | MemInfo: { | 20260508 12:12:16.571 | INFO | "gpu" : " 17 GB", | 20260508 12:12:16.571 | INFO | "huge" : "", | 20260508 12:12:16.571 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:16.571 | INFO | "malloc" : " 14 GB", | 20260508 12:12:16.571 | INFO | "pinned" : " 29 GB" | 20260508 12:12:16.571 | INFO | } | 20260508 12:12:16.571 | INFO | Perf report for the last 5.1403 seconds: | 20260508 12:12:16.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:16.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.571 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:12:16.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.571 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:12:16.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:16.572 | INFO | Uploaded 36GiB to GPU in 2.37023s (15.1884GiB/s) | 20260508 12:12:16.572 | INFO | Downloaded 34GiB to GPU in 2.45918s (13.8258GiB/s) | 20260508 12:12:21.714 | INFO | Analyzing chunk frames = [8010752,8011264) | 20260508 12:12:21.714 | INFO | MemInfo: { | 20260508 12:12:21.714 | INFO | "gpu" : " 17 GB", | 20260508 12:12:21.714 | INFO | "huge" : "", | 20260508 12:12:21.714 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:21.714 | INFO | "malloc" : " 14 GB", | 20260508 12:12:21.714 | INFO | "pinned" : " 29 GB" | 20260508 12:12:21.714 | INFO | } | 20260508 12:12:21.714 | INFO | Perf report for the last 5.14296 seconds: | 20260508 12:12:21.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:21.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.714 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:12:21.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.714 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:12:21.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:21.715 | INFO | Uploaded 36GiB to GPU in 2.40487s (14.9696GiB/s) | 20260508 12:12:21.715 | INFO | Downloaded 34GiB to GPU in 2.45462s (13.8515GiB/s) | 20260508 12:12:26.852 | INFO | Analyzing chunk frames = [8011264,8011776) | 20260508 12:12:26.853 | INFO | MemInfo: { | 20260508 12:12:26.853 | INFO | "gpu" : " 17 GB", | 20260508 12:12:26.853 | INFO | "huge" : "", | 20260508 12:12:26.853 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:26.853 | INFO | "malloc" : " 14 GB", | 20260508 12:12:26.853 | INFO | "pinned" : " 29 GB" | 20260508 12:12:26.853 | INFO | } | 20260508 12:12:26.853 | INFO | Perf report for the last 5.1386 seconds: | 20260508 12:12:26.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:26.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 12:12:26.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:12:26.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:26.853 | INFO | Uploaded 36GiB to GPU in 2.45438s (14.6676GiB/s) | 20260508 12:12:26.853 | INFO | Downloaded 34GiB to GPU in 2.4788s (13.7163GiB/s) | 20260508 12:12:26.853 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:12:26.853 | INFO | Total mem usage: 16.7 GiB | 20260508 12:12:26.853 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:12:26.853 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:12:26.853 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:12:26.853 | INFO | High Water Breakdown: | 20260508 12:12:26.853 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:12:26.853 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:12:26.853 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:12:26.853 | INFO | ------------------------------------------ | 20260508 12:12:26.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:12:26.853 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:12:26.853 | INFO | Total mem usage: 26.5 GiB | 20260508 12:12:26.853 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:12:26.853 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:12:26.853 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:12:26.853 | INFO | High Water Breakdown: | 20260508 12:12:26.853 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:12:26.853 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:12:26.853 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:12:26.853 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:12:26.853 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:12:26.853 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:12:26.853 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:12:26.853 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:12:26.853 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:12:26.853 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:12:26.853 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:12:26.853 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:12:26.853 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:12:26.853 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:12:26.853 | INFO | ------------------------------------------ | 20260508 12:12:26.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:12:26.853 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:12:26.853 | INFO | Total mem usage: 27.8 GiB | 20260508 12:12:26.853 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:12:26.853 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:12:26.853 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:12:26.853 | INFO | High Water Breakdown: | 20260508 12:12:26.853 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:12:26.853 | INFO | ------------------------------------------ | 20260508 12:12:26.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:12:26.853 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:12:26.853 | INFO | Total mem usage: 15.7 GiB | 20260508 12:12:26.853 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:12:26.853 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:12:26.853 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:12:26.853 | INFO | High Water Breakdown: | 20260508 12:12:26.853 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:12:26.853 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:12:26.853 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:12:26.853 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:12:26.853 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:12:26.853 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:12:26.853 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:12:26.853 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:12:26.853 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:12:26.853 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:12:26.853 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:12:26.853 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:12:26.853 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:12:26.853 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:12:26.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:12:26.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:12:26.853 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:12:26.853 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:12:26.853 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:12:26.853 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:12:26.853 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:12:26.853 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:12:26.853 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:12:26.853 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:12:26.853 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:12:26.853 | INFO | ------------------------------------------ | 20260508 12:12:26.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:12:26.853 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:12:26.853 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:12:26.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:12:26.853 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 6.95e-01 | 20260508 12:12:26.853 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.12e-02 / 3.14e-03 | 20260508 12:12:26.853 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.48e+00, 6.00e-01 / 2.18e-01 | 20260508 12:12:26.853 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.10e-01 / 1.04e-01 | 20260508 12:12:26.853 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.52e+01, 3.67e+00 / 8.63e-01 | 20260508 12:12:26.853 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.80e+00 / 1.28e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.59e+00, 4.05e-01 / 1.98e-01 | 20260508 12:12:26.853 | INFO | PerfReport: 2.97%, Metrics -- 9.70e-01 / 1.13e+00, 2.40e-01 / 9.60e-02 | 20260508 12:12:26.853 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.29e-01, 4.73e-02 / 2.47e-02 | 20260508 12:12:26.853 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:12:26.853 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:12:26.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:12:26.853 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 24.71%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 2.18%, LockUpload -- 2.29e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 15.05%, LockCompute -- 1.58e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:12:26.853 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260508 12:12:26.950 | INFO | Starting a round of Baz IO | 20260508 12:12:30.342 | INFO | Populating RTMetrics report | 20260508 12:12:30.342 | INFO | RTMetrics report populated | 20260508 12:12:30.342 | INFO | Opening tmp RTMetrics json file | 20260508 12:12:30.352 | INFO | Serializing RTMetrics to json | 20260508 12:12:30.353 | INFO | Serialization done, swapping tmp file now... | 20260508 12:12:30.354 | INFO | RTMetrics done | 20260508 12:12:30.413 | INFO | Finished a round of Baz IO: Wrote 1.417 GiB in 3.463 seconds | 20260508 12:12:30.413 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:12:30.413 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170688% / 3.97364e-06% / 6.35783e-05% | 20260508 12:12:30.414 | INFO | Baz IO overall: | 20260508 12:12:30.414 | INFO | BytesWritten : 3440621242196 | 20260508 12:12:30.414 | INFO | NumEvents : 1651417187033 | 20260508 12:12:31.989 | INFO | Analyzing chunk frames = [8011776,8012288) | 20260508 12:12:31.989 | INFO | MemInfo: { | 20260508 12:12:31.989 | INFO | "gpu" : " 17 GB", | 20260508 12:12:31.989 | INFO | "huge" : "", | 20260508 12:12:31.989 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:31.989 | INFO | "malloc" : " 14 GB", | 20260508 12:12:31.989 | INFO | "pinned" : " 29 GB" | 20260508 12:12:31.989 | INFO | } | 20260508 12:12:31.989 | INFO | Perf report for the last 5.13601 seconds: | 20260508 12:12:31.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:31.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:31.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:31.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:31.989 | INFO | ANALYSIS : 67.3%, 3.0, 105.0ms, 0 | 20260508 12:12:31.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:31.989 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:12:31.989 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 12:12:31.989 | INFO | BAZWRITER : 66.6%, 2.5, 84.3ms, 0 | 20260508 12:12:31.989 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:12:31.989 | INFO | Uploaded 36GiB to GPU in 2.594s (13.8782GiB/s) | 20260508 12:12:31.989 | INFO | Downloaded 38GiB to GPU in 2.72232s (13.9587GiB/s) | 20260508 12:12:37.128 | INFO | Analyzing chunk frames = [8012288,8012800) | 20260508 12:12:37.128 | INFO | MemInfo: { | 20260508 12:12:37.128 | INFO | "gpu" : " 17 GB", | 20260508 12:12:37.128 | INFO | "huge" : "", | 20260508 12:12:37.128 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:37.128 | INFO | "malloc" : " 14 GB", | 20260508 12:12:37.128 | INFO | "pinned" : " 29 GB" | 20260508 12:12:37.128 | INFO | } | 20260508 12:12:37.128 | INFO | Perf report for the last 5.13902 seconds: | 20260508 12:12:37.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:37.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:12:37.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:12:37.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:37.128 | INFO | Uploaded 36GiB to GPU in 2.44644s (14.7153GiB/s) | 20260508 12:12:37.128 | INFO | Downloaded 34GiB to GPU in 2.47811s (13.7201GiB/s) | 20260508 12:12:42.269 | INFO | Analyzing chunk frames = [8012800,8013312) | 20260508 12:12:42.269 | INFO | MemInfo: { | 20260508 12:12:42.269 | INFO | "gpu" : " 17 GB", | 20260508 12:12:42.269 | INFO | "huge" : "", | 20260508 12:12:42.269 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:42.269 | INFO | "malloc" : " 14 GB", | 20260508 12:12:42.269 | INFO | "pinned" : " 29 GB" | 20260508 12:12:42.269 | INFO | } | 20260508 12:12:42.270 | INFO | Perf report for the last 5.1417 seconds: | 20260508 12:12:42.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:42.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 12:12:42.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 12:12:42.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:42.270 | INFO | Uploaded 36GiB to GPU in 2.46482s (14.6055GiB/s) | 20260508 12:12:42.270 | INFO | Downloaded 34GiB to GPU in 2.44309s (13.9168GiB/s) | 20260508 12:12:47.415 | INFO | Analyzing chunk frames = [8013312,8013824) | 20260508 12:12:47.415 | INFO | MemInfo: { | 20260508 12:12:47.415 | INFO | "gpu" : " 17 GB", | 20260508 12:12:47.415 | INFO | "huge" : "", | 20260508 12:12:47.415 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:47.415 | INFO | "malloc" : " 14 GB", | 20260508 12:12:47.415 | INFO | "pinned" : " 29 GB" | 20260508 12:12:47.415 | INFO | } | 20260508 12:12:47.415 | INFO | Perf report for the last 5.14592 seconds: | 20260508 12:12:47.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:47.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.416 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 12:12:47.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.416 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260508 12:12:47.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:47.416 | INFO | Uploaded 36GiB to GPU in 2.50494s (14.3716GiB/s) | 20260508 12:12:47.416 | INFO | Downloaded 34GiB to GPU in 2.45649s (13.8409GiB/s) | 20260508 12:12:52.557 | INFO | Analyzing chunk frames = [8013824,8014336) | 20260508 12:12:52.557 | INFO | MemInfo: { | 20260508 12:12:52.557 | INFO | "gpu" : " 17 GB", | 20260508 12:12:52.557 | INFO | "huge" : "", | 20260508 12:12:52.557 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:52.557 | INFO | "malloc" : " 14 GB", | 20260508 12:12:52.557 | INFO | "pinned" : " 29 GB" | 20260508 12:12:52.557 | INFO | } | 20260508 12:12:52.557 | INFO | Perf report for the last 5.14153 seconds: | 20260508 12:12:52.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:52.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.557 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 12:12:52.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.557 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:12:52.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:52.558 | INFO | Uploaded 36GiB to GPU in 2.27045s (15.8559GiB/s) | 20260508 12:12:52.558 | INFO | Downloaded 34GiB to GPU in 2.49312s (13.6375GiB/s) | 20260508 12:12:57.686 | INFO | Analyzing chunk frames = [8014336,8014848) | 20260508 12:12:57.686 | INFO | MemInfo: { | 20260508 12:12:57.686 | INFO | "gpu" : " 17 GB", | 20260508 12:12:57.686 | INFO | "huge" : "", | 20260508 12:12:57.686 | INFO | "huge_shared" : " 29 GB", | 20260508 12:12:57.686 | INFO | "malloc" : " 14 GB", | 20260508 12:12:57.686 | INFO | "pinned" : " 29 GB" | 20260508 12:12:57.686 | INFO | } | 20260508 12:12:57.686 | INFO | Perf report for the last 5.12893 seconds: | 20260508 12:12:57.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:12:57.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 12:12:57.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | PRE_HQ : 23.1%, 1.0, 11.9ms, 0 | 20260508 12:12:57.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:12:57.686 | INFO | Uploaded 36GiB to GPU in 2.40771s (14.9519GiB/s) | 20260508 12:12:57.686 | INFO | Downloaded 34GiB to GPU in 2.46471s (13.7947GiB/s) | 20260508 12:13:02.830 | INFO | Analyzing chunk frames = [8014848,8015360) | 20260508 12:13:02.830 | INFO | MemInfo: { | 20260508 12:13:02.830 | INFO | "gpu" : " 17 GB", | 20260508 12:13:02.830 | INFO | "huge" : "", | 20260508 12:13:02.830 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:02.830 | INFO | "malloc" : " 14 GB", | 20260508 12:13:02.830 | INFO | "pinned" : " 29 GB" | 20260508 12:13:02.830 | INFO | } | 20260508 12:13:02.830 | INFO | Perf report for the last 5.14385 seconds: | 20260508 12:13:02.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:02.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:13:02.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:13:02.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:02.830 | INFO | Uploaded 36GiB to GPU in 2.40263s (14.9836GiB/s) | 20260508 12:13:02.830 | INFO | Downloaded 34GiB to GPU in 2.4668s (13.783GiB/s) | 20260508 12:13:07.966 | INFO | Analyzing chunk frames = [8015360,8015872) | 20260508 12:13:07.966 | INFO | MemInfo: { | 20260508 12:13:07.966 | INFO | "gpu" : " 17 GB", | 20260508 12:13:07.966 | INFO | "huge" : "", | 20260508 12:13:07.966 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:07.966 | INFO | "malloc" : " 14 GB", | 20260508 12:13:07.966 | INFO | "pinned" : " 29 GB" | 20260508 12:13:07.966 | INFO | } | 20260508 12:13:07.966 | INFO | Perf report for the last 5.13593 seconds: | 20260508 12:13:07.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:07.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | ANALYSIS : 65.1%, 3.0, 101.6ms, 0 | 20260508 12:13:07.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:13:07.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:07.966 | INFO | Uploaded 36GiB to GPU in 2.36639s (15.2131GiB/s) | 20260508 12:13:07.966 | INFO | Downloaded 34GiB to GPU in 2.45995s (13.8214GiB/s) | 20260508 12:13:08.063 | INFO | Starting a round of Baz IO | 20260508 12:13:11.461 | INFO | Populating RTMetrics report | 20260508 12:13:11.461 | INFO | RTMetrics report populated | 20260508 12:13:11.461 | INFO | Opening tmp RTMetrics json file | 20260508 12:13:11.470 | INFO | Serializing RTMetrics to json | 20260508 12:13:11.471 | INFO | Serialization done, swapping tmp file now... | 20260508 12:13:11.471 | INFO | RTMetrics done | 20260508 12:13:11.530 | INFO | Finished a round of Baz IO: Wrote 1.417 GiB in 3.467 seconds | 20260508 12:13:11.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:13:11.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170521% / 0% / 6.75519e-05% | 20260508 12:13:11.530 | INFO | Baz IO overall: | 20260508 12:13:11.530 | INFO | BytesWritten : 3442142625023 | 20260508 12:13:11.530 | INFO | NumEvents : 1652131729331 | 20260508 12:13:13.106 | INFO | Analyzing chunk frames = [8015872,8016384) | 20260508 12:13:13.106 | INFO | MemInfo: { | 20260508 12:13:13.106 | INFO | "gpu" : " 17 GB", | 20260508 12:13:13.106 | INFO | "huge" : "", | 20260508 12:13:13.106 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:13.106 | INFO | "malloc" : " 14 GB", | 20260508 12:13:13.106 | INFO | "pinned" : " 29 GB" | 20260508 12:13:13.106 | INFO | } | 20260508 12:13:13.106 | INFO | Perf report for the last 5.14006 seconds: | 20260508 12:13:13.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:13.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:13.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:13.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:13.106 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:13:13.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:13.106 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 12:13:13.106 | INFO | PRE_HQ : 65.2%, 1.6, 54.2ms, 0 | 20260508 12:13:13.106 | INFO | BAZWRITER : 66.8%, 2.5, 84.6ms, 0 | 20260508 12:13:13.106 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:13:13.106 | INFO | Uploaded 36GiB to GPU in 2.61846s (13.7485GiB/s) | 20260508 12:13:13.107 | INFO | Downloaded 38GiB to GPU in 2.71351s (14.004GiB/s) | 20260508 12:13:18.249 | INFO | Analyzing chunk frames = [8016384,8016896) | 20260508 12:13:18.249 | INFO | MemInfo: { | 20260508 12:13:18.249 | INFO | "gpu" : " 17 GB", | 20260508 12:13:18.249 | INFO | "huge" : "", | 20260508 12:13:18.249 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:18.249 | INFO | "malloc" : " 14 GB", | 20260508 12:13:18.249 | INFO | "pinned" : " 29 GB" | 20260508 12:13:18.249 | INFO | } | 20260508 12:13:18.249 | INFO | Perf report for the last 5.14334 seconds: | 20260508 12:13:18.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:18.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:13:18.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:13:18.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:18.249 | INFO | Uploaded 36GiB to GPU in 2.46303s (14.6161GiB/s) | 20260508 12:13:18.250 | INFO | Downloaded 34GiB to GPU in 2.47184s (13.7549GiB/s) | 20260508 12:13:23.384 | INFO | Analyzing chunk frames = [8016896,8017408) | 20260508 12:13:23.384 | INFO | MemInfo: { | 20260508 12:13:23.384 | INFO | "gpu" : " 17 GB", | 20260508 12:13:23.384 | INFO | "huge" : "", | 20260508 12:13:23.384 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:23.384 | INFO | "malloc" : " 14 GB", | 20260508 12:13:23.384 | INFO | "pinned" : " 29 GB" | 20260508 12:13:23.384 | INFO | } | 20260508 12:13:23.384 | INFO | Perf report for the last 5.13507 seconds: | 20260508 12:13:23.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:23.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.384 | INFO | ANALYSIS : 66.8%, 3.0, 103.7ms, 0 | 20260508 12:13:23.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.384 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 12:13:23.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:23.385 | INFO | Uploaded 36GiB to GPU in 2.52692s (14.2466GiB/s) | 20260508 12:13:23.385 | INFO | Downloaded 34GiB to GPU in 2.45921s (13.8256GiB/s) | 20260508 12:13:28.525 | INFO | Analyzing chunk frames = [8017408,8017920) | 20260508 12:13:28.525 | INFO | MemInfo: { | 20260508 12:13:28.525 | INFO | "gpu" : " 17 GB", | 20260508 12:13:28.525 | INFO | "huge" : "", | 20260508 12:13:28.525 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:28.525 | INFO | "malloc" : " 14 GB", | 20260508 12:13:28.525 | INFO | "pinned" : " 29 GB" | 20260508 12:13:28.525 | INFO | } | 20260508 12:13:28.525 | INFO | Perf report for the last 5.14109 seconds: | 20260508 12:13:28.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:28.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.526 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 12:13:28.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.526 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 12:13:28.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:28.526 | INFO | Uploaded 36GiB to GPU in 2.46158s (14.6248GiB/s) | 20260508 12:13:28.526 | INFO | Downloaded 34GiB to GPU in 2.44727s (13.8931GiB/s) | 20260508 12:13:33.671 | INFO | Analyzing chunk frames = [8017920,8018432) | 20260508 12:13:33.671 | INFO | MemInfo: { | 20260508 12:13:33.671 | INFO | "gpu" : " 17 GB", | 20260508 12:13:33.671 | INFO | "huge" : "", | 20260508 12:13:33.671 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:33.671 | INFO | "malloc" : " 14 GB", | 20260508 12:13:33.671 | INFO | "pinned" : " 29 GB" | 20260508 12:13:33.671 | INFO | } | 20260508 12:13:33.671 | INFO | Perf report for the last 5.146 seconds: | 20260508 12:13:33.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:33.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.671 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 12:13:33.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.671 | INFO | PRE_HQ : 31.5%, 1.0, 16.5ms, 0 | 20260508 12:13:33.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:33.672 | INFO | Uploaded 36GiB to GPU in 2.29457s (15.6892GiB/s) | 20260508 12:13:33.672 | INFO | Downloaded 34GiB to GPU in 2.49268s (13.6399GiB/s) | 20260508 12:13:38.804 | INFO | Analyzing chunk frames = [8018432,8018944) | 20260508 12:13:38.804 | INFO | MemInfo: { | 20260508 12:13:38.804 | INFO | "gpu" : " 17 GB", | 20260508 12:13:38.804 | INFO | "huge" : "", | 20260508 12:13:38.804 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:38.804 | INFO | "malloc" : " 14 GB", | 20260508 12:13:38.804 | INFO | "pinned" : " 29 GB" | 20260508 12:13:38.804 | INFO | } | 20260508 12:13:38.804 | INFO | Perf report for the last 5.13266 seconds: | 20260508 12:13:38.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:38.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 12:13:38.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:13:38.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:38.804 | INFO | Uploaded 36GiB to GPU in 2.39025s (15.0612GiB/s) | 20260508 12:13:38.804 | INFO | Downloaded 34GiB to GPU in 2.43309s (13.974GiB/s) | 20260508 12:13:43.937 | INFO | Analyzing chunk frames = [8018944,8019456) | 20260508 12:13:43.937 | INFO | MemInfo: { | 20260508 12:13:43.937 | INFO | "gpu" : " 17 GB", | 20260508 12:13:43.937 | INFO | "huge" : "", | 20260508 12:13:43.937 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:43.937 | INFO | "malloc" : " 14 GB", | 20260508 12:13:43.937 | INFO | "pinned" : " 29 GB" | 20260508 12:13:43.937 | INFO | } | 20260508 12:13:43.937 | INFO | Perf report for the last 5.13293 seconds: | 20260508 12:13:43.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:43.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 12:13:43.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:13:43.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:43.937 | INFO | Uploaded 36GiB to GPU in 2.47753s (14.5306GiB/s) | 20260508 12:13:43.937 | INFO | Downloaded 34GiB to GPU in 2.45918s (13.8257GiB/s) | 20260508 12:13:49.083 | INFO | Analyzing chunk frames = [8019456,8019968) | 20260508 12:13:49.083 | INFO | MemInfo: { | 20260508 12:13:49.083 | INFO | "gpu" : " 17 GB", | 20260508 12:13:49.083 | INFO | "huge" : "", | 20260508 12:13:49.083 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:49.083 | INFO | "malloc" : " 14 GB", | 20260508 12:13:49.083 | INFO | "pinned" : " 29 GB" | 20260508 12:13:49.083 | INFO | } | 20260508 12:13:49.083 | INFO | Perf report for the last 5.14601 seconds: | 20260508 12:13:49.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:49.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:13:49.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:13:49.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:49.083 | INFO | Uploaded 36GiB to GPU in 2.47717s (14.5327GiB/s) | 20260508 12:13:49.083 | INFO | Downloaded 34GiB to GPU in 2.46001s (13.8211GiB/s) | 20260508 12:13:49.083 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:13:49.083 | INFO | Total mem usage: 16.7 GiB | 20260508 12:13:49.083 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:13:49.083 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:13:49.083 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:13:49.083 | INFO | High Water Breakdown: | 20260508 12:13:49.083 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:13:49.083 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:13:49.083 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:13:49.083 | INFO | ------------------------------------------ | 20260508 12:13:49.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:13:49.083 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:13:49.083 | INFO | Total mem usage: 26.5 GiB | 20260508 12:13:49.083 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:13:49.083 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:13:49.083 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:13:49.083 | INFO | High Water Breakdown: | 20260508 12:13:49.084 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:13:49.084 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:13:49.084 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:13:49.084 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:13:49.084 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:13:49.084 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:13:49.084 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:13:49.084 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:13:49.084 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:13:49.084 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:13:49.084 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:13:49.084 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:13:49.084 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:13:49.084 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:13:49.084 | INFO | ------------------------------------------ | 20260508 12:13:49.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:13:49.084 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:13:49.084 | INFO | Total mem usage: 27.8 GiB | 20260508 12:13:49.084 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:13:49.084 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:13:49.084 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:13:49.084 | INFO | High Water Breakdown: | 20260508 12:13:49.084 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:13:49.084 | INFO | ------------------------------------------ | 20260508 12:13:49.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:13:49.084 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:13:49.084 | INFO | Total mem usage: 15.7 GiB | 20260508 12:13:49.084 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:13:49.084 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:13:49.084 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:13:49.084 | INFO | High Water Breakdown: | 20260508 12:13:49.084 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:13:49.084 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:13:49.084 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:13:49.084 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:13:49.084 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:13:49.084 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:13:49.084 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:13:49.084 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:13:49.084 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:13:49.084 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:13:49.084 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:13:49.084 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:13:49.084 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:13:49.084 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:13:49.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:13:49.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:13:49.084 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:13:49.084 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:13:49.084 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:13:49.084 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:13:49.084 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:13:49.084 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:13:49.084 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:13:49.084 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:13:49.084 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:13:49.084 | INFO | ------------------------------------------ | 20260508 12:13:49.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:13:49.084 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:13:49.084 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:13:49.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:13:49.084 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.42e+01 / 5.72e-01 | 20260508 12:13:49.084 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.07e-01, 1.15e-02 / 8.72e-03 | 20260508 12:13:49.084 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.53e+00, 5.96e-01 / 1.66e-01 | 20260508 12:13:49.084 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.45e+00, 2.08e-01 / 1.24e-01 | 20260508 12:13:49.084 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.54e+01, 3.67e+00 / 9.33e-01 | 20260508 12:13:49.084 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.82e+00 / 1.34e+00 | 20260508 12:13:49.084 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.07e-01 / 2.42e-01 | 20260508 12:13:49.084 | INFO | PerfReport: 2.97%, Metrics -- 9.70e-01 / 1.08e+00, 2.40e-01 / 1.15e-01 | 20260508 12:13:49.084 | INFO | PerfReport: 0.91%, PulseSerializing -- 2.96e-01 / 3.25e-01, 4.79e-02 / 2.61e-02 | 20260508 12:13:49.084 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:13:49.084 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:13:49.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:13:49.085 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 24.29%, Upload -- 2.55e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 31.15%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 2.06%, LockUpload -- 2.16e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 2.64%, LockDownload -- 2.77e+00 / 0.00e+00, 4.17e+00 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 15.10%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:13:49.085 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260508 12:13:49.181 | INFO | Starting a round of Baz IO | 20260508 12:13:52.589 | INFO | Populating RTMetrics report | 20260508 12:13:52.589 | INFO | RTMetrics report populated | 20260508 12:13:52.589 | INFO | Opening tmp RTMetrics json file | 20260508 12:13:52.594 | INFO | Serializing RTMetrics to json | 20260508 12:13:52.596 | INFO | Serialization done, swapping tmp file now... | 20260508 12:13:52.596 | INFO | RTMetrics done | 20260508 12:13:52.656 | INFO | Finished a round of Baz IO: Wrote 1.416 GiB in 3.475 seconds | 20260508 12:13:52.656 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:13:52.656 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170759% / 3.97364e-06% / 7.94729e-05% | 20260508 12:13:52.656 | INFO | Baz IO overall: | 20260508 12:13:52.656 | INFO | BytesWritten : 3443663566440 | 20260508 12:13:52.656 | INFO | NumEvents : 1652846192598 | 20260508 12:13:54.226 | INFO | Analyzing chunk frames = [8019968,8020480) | 20260508 12:13:54.226 | INFO | MemInfo: { | 20260508 12:13:54.226 | INFO | "gpu" : " 17 GB", | 20260508 12:13:54.226 | INFO | "huge" : "", | 20260508 12:13:54.226 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:54.226 | INFO | "malloc" : " 14 GB", | 20260508 12:13:54.226 | INFO | "pinned" : " 29 GB" | 20260508 12:13:54.226 | INFO | } | 20260508 12:13:54.226 | INFO | Perf report for the last 5.14261 seconds: | 20260508 12:13:54.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:54.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:54.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:54.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:54.226 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:13:54.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:54.226 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:13:54.226 | INFO | PRE_HQ : 65.4%, 1.6, 53.8ms, 0 | 20260508 12:13:54.226 | INFO | BAZWRITER : 66.9%, 2.4, 84.0ms, 0 | 20260508 12:13:54.226 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:13:54.226 | INFO | Uploaded 36GiB to GPU in 2.60224s (13.8343GiB/s) | 20260508 12:13:54.226 | INFO | Downloaded 38GiB to GPU in 2.7402s (13.8676GiB/s) | 20260508 12:13:59.363 | INFO | Analyzing chunk frames = [8020480,8020992) | 20260508 12:13:59.363 | INFO | MemInfo: { | 20260508 12:13:59.363 | INFO | "gpu" : " 17 GB", | 20260508 12:13:59.363 | INFO | "huge" : "", | 20260508 12:13:59.363 | INFO | "huge_shared" : " 29 GB", | 20260508 12:13:59.363 | INFO | "malloc" : " 14 GB", | 20260508 12:13:59.363 | INFO | "pinned" : " 29 GB" | 20260508 12:13:59.363 | INFO | } | 20260508 12:13:59.363 | INFO | Perf report for the last 5.13739 seconds: | 20260508 12:13:59.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:13:59.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:13:59.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:13:59.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:13:59.363 | INFO | Uploaded 36GiB to GPU in 2.45535s (14.6619GiB/s) | 20260508 12:13:59.363 | INFO | Downloaded 34GiB to GPU in 2.48352s (13.6903GiB/s) | 20260508 12:14:04.502 | INFO | Analyzing chunk frames = [8020992,8021504) | 20260508 12:14:04.502 | INFO | MemInfo: { | 20260508 12:14:04.502 | INFO | "gpu" : " 17 GB", | 20260508 12:14:04.502 | INFO | "huge" : "", | 20260508 12:14:04.502 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:04.502 | INFO | "malloc" : " 14 GB", | 20260508 12:14:04.502 | INFO | "pinned" : " 29 GB" | 20260508 12:14:04.502 | INFO | } | 20260508 12:14:04.502 | INFO | Perf report for the last 5.13887 seconds: | 20260508 12:14:04.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:04.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 12:14:04.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:14:04.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:04.502 | INFO | Uploaded 36GiB to GPU in 2.48767s (14.4714GiB/s) | 20260508 12:14:04.502 | INFO | Downloaded 34GiB to GPU in 2.47006s (13.7649GiB/s) | 20260508 12:14:09.639 | INFO | Analyzing chunk frames = [8021504,8022016) | 20260508 12:14:09.639 | INFO | MemInfo: { | 20260508 12:14:09.639 | INFO | "gpu" : " 17 GB", | 20260508 12:14:09.639 | INFO | "huge" : "", | 20260508 12:14:09.639 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:09.639 | INFO | "malloc" : " 14 GB", | 20260508 12:14:09.639 | INFO | "pinned" : " 29 GB" | 20260508 12:14:09.639 | INFO | } | 20260508 12:14:09.639 | INFO | Perf report for the last 5.13675 seconds: | 20260508 12:14:09.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:09.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 12:14:09.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:14:09.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:09.639 | INFO | Uploaded 36GiB to GPU in 2.48651s (14.4781GiB/s) | 20260508 12:14:09.639 | INFO | Downloaded 34GiB to GPU in 2.48339s (13.6909GiB/s) | 20260508 12:14:14.776 | INFO | Analyzing chunk frames = [8022016,8022528) | 20260508 12:14:14.776 | INFO | MemInfo: { | 20260508 12:14:14.776 | INFO | "gpu" : " 17 GB", | 20260508 12:14:14.776 | INFO | "huge" : "", | 20260508 12:14:14.776 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:14.776 | INFO | "malloc" : " 14 GB", | 20260508 12:14:14.776 | INFO | "pinned" : " 29 GB" | 20260508 12:14:14.776 | INFO | } | 20260508 12:14:14.776 | INFO | Perf report for the last 5.13738 seconds: | 20260508 12:14:14.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:14.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 12:14:14.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 12:14:14.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:14.776 | INFO | Uploaded 36GiB to GPU in 2.40834s (14.948GiB/s) | 20260508 12:14:14.776 | INFO | Downloaded 34GiB to GPU in 2.55659s (13.299GiB/s) | 20260508 12:14:19.919 | INFO | Analyzing chunk frames = [8022528,8023040) | 20260508 12:14:19.919 | INFO | MemInfo: { | 20260508 12:14:19.919 | INFO | "gpu" : " 17 GB", | 20260508 12:14:19.919 | INFO | "huge" : "", | 20260508 12:14:19.919 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:19.919 | INFO | "malloc" : " 14 GB", | 20260508 12:14:19.919 | INFO | "pinned" : " 29 GB" | 20260508 12:14:19.919 | INFO | } | 20260508 12:14:19.919 | INFO | Perf report for the last 5.14249 seconds: | 20260508 12:14:19.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:19.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:14:19.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:14:19.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:19.919 | INFO | Uploaded 36GiB to GPU in 2.37095s (15.1838GiB/s) | 20260508 12:14:19.920 | INFO | Downloaded 34GiB to GPU in 2.44316s (13.9164GiB/s) | 20260508 12:14:25.059 | INFO | Analyzing chunk frames = [8023040,8023552) | 20260508 12:14:25.059 | INFO | MemInfo: { | 20260508 12:14:25.059 | INFO | "gpu" : " 17 GB", | 20260508 12:14:25.059 | INFO | "huge" : "", | 20260508 12:14:25.059 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:25.059 | INFO | "malloc" : " 14 GB", | 20260508 12:14:25.059 | INFO | "pinned" : " 29 GB" | 20260508 12:14:25.059 | INFO | } | 20260508 12:14:25.059 | INFO | Perf report for the last 5.14076 seconds: | 20260508 12:14:25.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:25.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.059 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:14:25.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.059 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:14:25.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:25.060 | INFO | Uploaded 36GiB to GPU in 2.42648s (14.8363GiB/s) | 20260508 12:14:25.060 | INFO | Downloaded 34GiB to GPU in 2.45055s (13.8745GiB/s) | 20260508 12:14:30.195 | INFO | Analyzing chunk frames = [8023552,8024064) | 20260508 12:14:30.195 | INFO | MemInfo: { | 20260508 12:14:30.195 | INFO | "gpu" : " 17 GB", | 20260508 12:14:30.195 | INFO | "huge" : "", | 20260508 12:14:30.195 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:30.195 | INFO | "malloc" : " 14 GB", | 20260508 12:14:30.195 | INFO | "pinned" : " 29 GB" | 20260508 12:14:30.195 | INFO | } | 20260508 12:14:30.195 | INFO | Perf report for the last 5.13588 seconds: | 20260508 12:14:30.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:30.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.195 | INFO | ANALYSIS : 65.0%, 3.0, 101.5ms, 0 | 20260508 12:14:30.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.195 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:14:30.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:30.196 | INFO | Uploaded 36GiB to GPU in 2.38644s (15.0852GiB/s) | 20260508 12:14:30.196 | INFO | Downloaded 34GiB to GPU in 2.44243s (13.9206GiB/s) | 20260508 12:14:30.293 | INFO | Starting a round of Baz IO | 20260508 12:14:33.691 | INFO | Populating RTMetrics report | 20260508 12:14:33.691 | INFO | RTMetrics report populated | 20260508 12:14:33.691 | INFO | Opening tmp RTMetrics json file | 20260508 12:14:33.700 | INFO | Serializing RTMetrics to json | 20260508 12:14:33.701 | INFO | Serialization done, swapping tmp file now... | 20260508 12:14:33.702 | INFO | RTMetrics done | 20260508 12:14:33.763 | INFO | Finished a round of Baz IO: Wrote 1.416 GiB in 3.469 seconds | 20260508 12:14:33.763 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:14:33.763 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170112% / 0% / 6.35783e-05% | 20260508 12:14:33.763 | INFO | Baz IO overall: | 20260508 12:14:33.763 | INFO | BytesWritten : 3445183741699 | 20260508 12:14:33.763 | INFO | NumEvents : 1653560185490 | 20260508 12:14:35.338 | INFO | Analyzing chunk frames = [8024064,8024576) | 20260508 12:14:35.338 | INFO | MemInfo: { | 20260508 12:14:35.338 | INFO | "gpu" : " 17 GB", | 20260508 12:14:35.338 | INFO | "huge" : "", | 20260508 12:14:35.338 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:35.338 | INFO | "malloc" : " 14 GB", | 20260508 12:14:35.338 | INFO | "pinned" : " 29 GB" | 20260508 12:14:35.338 | INFO | } | 20260508 12:14:35.338 | INFO | Perf report for the last 5.14245 seconds: | 20260508 12:14:35.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:35.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:35.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:35.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:35.338 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 12:14:35.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:35.338 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:14:35.338 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 12:14:35.338 | INFO | BAZWRITER : 66.8%, 2.5, 85.0ms, 0 | 20260508 12:14:35.338 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:14:35.338 | INFO | Uploaded 36GiB to GPU in 2.62155s (13.7323GiB/s) | 20260508 12:14:35.338 | INFO | Downloaded 38GiB to GPU in 2.73505s (13.8937GiB/s) | 20260508 12:14:40.477 | INFO | Analyzing chunk frames = [8024576,8025088) | 20260508 12:14:40.477 | INFO | MemInfo: { | 20260508 12:14:40.477 | INFO | "gpu" : " 17 GB", | 20260508 12:14:40.477 | INFO | "huge" : "", | 20260508 12:14:40.477 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:40.477 | INFO | "malloc" : " 14 GB", | 20260508 12:14:40.477 | INFO | "pinned" : " 29 GB" | 20260508 12:14:40.477 | INFO | } | 20260508 12:14:40.477 | INFO | Perf report for the last 5.1392 seconds: | 20260508 12:14:40.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:40.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:14:40.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 12:14:40.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:40.477 | INFO | Uploaded 36GiB to GPU in 2.43878s (14.7615GiB/s) | 20260508 12:14:40.478 | INFO | Downloaded 34GiB to GPU in 2.46472s (13.7947GiB/s) | 20260508 12:14:45.617 | INFO | Analyzing chunk frames = [8025088,8025600) | 20260508 12:14:45.617 | INFO | MemInfo: { | 20260508 12:14:45.617 | INFO | "gpu" : " 17 GB", | 20260508 12:14:45.617 | INFO | "huge" : "", | 20260508 12:14:45.617 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:45.617 | INFO | "malloc" : " 14 GB", | 20260508 12:14:45.617 | INFO | "pinned" : " 29 GB" | 20260508 12:14:45.617 | INFO | } | 20260508 12:14:45.617 | INFO | Perf report for the last 5.14025 seconds: | 20260508 12:14:45.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:45.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.617 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 12:14:45.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.617 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260508 12:14:45.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:45.618 | INFO | Uploaded 36GiB to GPU in 2.50936s (14.3463GiB/s) | 20260508 12:14:45.618 | INFO | Downloaded 34GiB to GPU in 2.4716s (13.7563GiB/s) | 20260508 12:14:50.752 | INFO | Analyzing chunk frames = [8025600,8026112) | 20260508 12:14:50.752 | INFO | MemInfo: { | 20260508 12:14:50.752 | INFO | "gpu" : " 17 GB", | 20260508 12:14:50.752 | INFO | "huge" : "", | 20260508 12:14:50.752 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:50.752 | INFO | "malloc" : " 14 GB", | 20260508 12:14:50.752 | INFO | "pinned" : " 29 GB" | 20260508 12:14:50.752 | INFO | } | 20260508 12:14:50.752 | INFO | Perf report for the last 5.1348 seconds: | 20260508 12:14:50.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:50.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 12:14:50.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:14:50.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:50.752 | INFO | Uploaded 36GiB to GPU in 2.46069s (14.6301GiB/s) | 20260508 12:14:50.752 | INFO | Downloaded 34GiB to GPU in 2.46166s (13.8118GiB/s) | 20260508 12:14:55.889 | INFO | Analyzing chunk frames = [8026112,8026624) | 20260508 12:14:55.889 | INFO | MemInfo: { | 20260508 12:14:55.889 | INFO | "gpu" : " 17 GB", | 20260508 12:14:55.889 | INFO | "huge" : "", | 20260508 12:14:55.889 | INFO | "huge_shared" : " 29 GB", | 20260508 12:14:55.889 | INFO | "malloc" : " 14 GB", | 20260508 12:14:55.889 | INFO | "pinned" : " 29 GB" | 20260508 12:14:55.889 | INFO | } | 20260508 12:14:55.889 | INFO | Perf report for the last 5.13739 seconds: | 20260508 12:14:55.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:14:55.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.889 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 12:14:55.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.889 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 12:14:55.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:14:55.890 | INFO | Uploaded 36GiB to GPU in 2.42085s (14.8708GiB/s) | 20260508 12:14:55.890 | INFO | Downloaded 34GiB to GPU in 2.53488s (13.4128GiB/s) | 20260508 12:15:01.032 | INFO | Analyzing chunk frames = [8026624,8027136) | 20260508 12:15:01.032 | INFO | MemInfo: { | 20260508 12:15:01.032 | INFO | "gpu" : " 17 GB", | 20260508 12:15:01.032 | INFO | "huge" : "", | 20260508 12:15:01.032 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:01.032 | INFO | "malloc" : " 14 GB", | 20260508 12:15:01.033 | INFO | "pinned" : " 29 GB" | 20260508 12:15:01.033 | INFO | } | 20260508 12:15:01.033 | INFO | Perf report for the last 5.14314 seconds: | 20260508 12:15:01.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:01.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 12:15:01.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:15:01.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:01.033 | INFO | Uploaded 36GiB to GPU in 2.35232s (15.304GiB/s) | 20260508 12:15:01.033 | INFO | Downloaded 34GiB to GPU in 2.46066s (13.8174GiB/s) | 20260508 12:15:06.178 | INFO | Analyzing chunk frames = [8027136,8027648) | 20260508 12:15:06.178 | INFO | MemInfo: { | 20260508 12:15:06.178 | INFO | "gpu" : " 17 GB", | 20260508 12:15:06.178 | INFO | "huge" : "", | 20260508 12:15:06.178 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:06.178 | INFO | "malloc" : " 14 GB", | 20260508 12:15:06.178 | INFO | "pinned" : " 29 GB" | 20260508 12:15:06.178 | INFO | } | 20260508 12:15:06.178 | INFO | Perf report for the last 5.14563 seconds: | 20260508 12:15:06.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:06.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.178 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 12:15:06.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.178 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:15:06.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:06.179 | INFO | Uploaded 36GiB to GPU in 2.41584s (14.9016GiB/s) | 20260508 12:15:06.179 | INFO | Downloaded 34GiB to GPU in 2.46483s (13.794GiB/s) | 20260508 12:15:11.311 | INFO | Analyzing chunk frames = [8027648,8028160) | 20260508 12:15:11.311 | INFO | MemInfo: { | 20260508 12:15:11.311 | INFO | "gpu" : " 17 GB", | 20260508 12:15:11.311 | INFO | "huge" : "", | 20260508 12:15:11.311 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:11.311 | INFO | "malloc" : " 14 GB", | 20260508 12:15:11.311 | INFO | "pinned" : " 29 GB" | 20260508 12:15:11.311 | INFO | } | 20260508 12:15:11.311 | INFO | Perf report for the last 5.13264 seconds: | 20260508 12:15:11.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:11.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | ANALYSIS : 65.1%, 3.0, 101.7ms, 0 | 20260508 12:15:11.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:15:11.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:11.311 | INFO | Uploaded 36GiB to GPU in 2.32593s (15.4777GiB/s) | 20260508 12:15:11.311 | INFO | Downloaded 34GiB to GPU in 2.45309s (13.8601GiB/s) | 20260508 12:15:11.311 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:15:11.311 | INFO | Total mem usage: 16.7 GiB | 20260508 12:15:11.311 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:15:11.311 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:15:11.311 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:15:11.311 | INFO | High Water Breakdown: | 20260508 12:15:11.311 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:15:11.311 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:15:11.311 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:15:11.311 | INFO | ------------------------------------------ | 20260508 12:15:11.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:15:11.311 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:15:11.311 | INFO | Total mem usage: 26.5 GiB | 20260508 12:15:11.311 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:15:11.311 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:15:11.311 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:15:11.311 | INFO | High Water Breakdown: | 20260508 12:15:11.311 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:15:11.311 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:15:11.311 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:15:11.311 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:15:11.311 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:15:11.311 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:15:11.311 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:15:11.311 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:15:11.311 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:15:11.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:15:11.312 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:15:11.312 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:15:11.312 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:15:11.312 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:15:11.312 | INFO | ------------------------------------------ | 20260508 12:15:11.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:15:11.312 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:15:11.312 | INFO | Total mem usage: 27.8 GiB | 20260508 12:15:11.312 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:15:11.312 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:15:11.312 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:15:11.312 | INFO | High Water Breakdown: | 20260508 12:15:11.312 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:15:11.312 | INFO | ------------------------------------------ | 20260508 12:15:11.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:15:11.312 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:15:11.312 | INFO | Total mem usage: 15.7 GiB | 20260508 12:15:11.312 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:15:11.312 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:15:11.312 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:15:11.312 | INFO | High Water Breakdown: | 20260508 12:15:11.312 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:15:11.312 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:15:11.312 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:15:11.312 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:15:11.312 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:15:11.312 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:15:11.312 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:15:11.312 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:15:11.312 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:15:11.312 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:15:11.312 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:15:11.312 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:15:11.312 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:15:11.312 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:15:11.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:15:11.312 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:15:11.312 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:15:11.312 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:15:11.312 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:15:11.312 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:15:11.312 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:15:11.312 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:15:11.312 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:15:11.312 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:15:11.312 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:15:11.312 | INFO | ------------------------------------------ | 20260508 12:15:11.312 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:15:11.312 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:15:11.312 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:15:11.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:15:11.312 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 6.44e-01 | 20260508 12:15:11.312 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.10e-02 / 5.82e-03 | 20260508 12:15:11.312 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.47e+00, 5.90e-01 / 2.48e-01 | 20260508 12:15:11.312 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.12e-01 | 20260508 12:15:11.312 | INFO | PerfReport: 15.99%, DME -- 4.17e+01 / 4.51e+01, 3.57e+00 / 1.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.86e+01 / 2.02e+01, 1.81e+00 / 1.19e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.04e-01 / 2.02e-01 | 20260508 12:15:11.312 | INFO | PerfReport: 2.98%, Metrics -- 9.71e-01 / 1.13e+00, 2.45e-01 / 1.31e-01 | 20260508 12:15:11.312 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.24e-01, 4.70e-02 / 2.03e-02 | 20260508 12:15:11.312 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:15:11.312 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:15:11.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:15:11.312 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 24.30%, Upload -- 2.55e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 24.89%, Download -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 31.10%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 2.01%, LockUpload -- 2.11e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 2.73%, LockDownload -- 2.87e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 14.96%, LockCompute -- 1.57e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:15:11.312 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260508 12:15:11.408 | INFO | Starting a round of Baz IO | 20260508 12:15:14.817 | INFO | Populating RTMetrics report | 20260508 12:15:14.817 | INFO | RTMetrics report populated | 20260508 12:15:14.817 | INFO | Opening tmp RTMetrics json file | 20260508 12:15:14.822 | INFO | Serializing RTMetrics to json | 20260508 12:15:14.823 | INFO | Serialization done, swapping tmp file now... | 20260508 12:15:14.823 | INFO | RTMetrics done | 20260508 12:15:14.885 | INFO | Finished a round of Baz IO: Wrote 1.417 GiB in 3.476 seconds | 20260508 12:15:14.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:15:14.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170032% / 0% / 5.16574e-05% | 20260508 12:15:14.885 | INFO | Baz IO overall: | 20260508 12:15:14.885 | INFO | BytesWritten : 3446705365141 | 20260508 12:15:14.885 | INFO | NumEvents : 1654275225102 | 20260508 12:15:16.443 | INFO | Analyzing chunk frames = [8028160,8028672) | 20260508 12:15:16.443 | INFO | MemInfo: { | 20260508 12:15:16.443 | INFO | "gpu" : " 17 GB", | 20260508 12:15:16.443 | INFO | "huge" : "", | 20260508 12:15:16.443 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:16.443 | INFO | "malloc" : " 14 GB", | 20260508 12:15:16.443 | INFO | "pinned" : " 29 GB" | 20260508 12:15:16.443 | INFO | } | 20260508 12:15:16.443 | INFO | Perf report for the last 5.13179 seconds: | 20260508 12:15:16.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:16.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:16.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:16.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:16.443 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:15:16.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:16.443 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 12:15:16.443 | INFO | PRE_HQ : 65.4%, 1.6, 54.5ms, 0 | 20260508 12:15:16.443 | INFO | BAZWRITER : 67.1%, 2.5, 85.2ms, 0 | 20260508 12:15:16.443 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:15:16.443 | INFO | Uploaded 36GiB to GPU in 2.5931s (13.883GiB/s) | 20260508 12:15:16.443 | INFO | Downloaded 38GiB to GPU in 2.72409s (13.9496GiB/s) | 20260508 12:15:21.590 | INFO | Analyzing chunk frames = [8028672,8029184) | 20260508 12:15:21.590 | INFO | MemInfo: { | 20260508 12:15:21.590 | INFO | "gpu" : " 17 GB", | 20260508 12:15:21.590 | INFO | "huge" : "", | 20260508 12:15:21.590 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:21.590 | INFO | "malloc" : " 14 GB", | 20260508 12:15:21.590 | INFO | "pinned" : " 29 GB" | 20260508 12:15:21.590 | INFO | } | 20260508 12:15:21.590 | INFO | Perf report for the last 5.14776 seconds: | 20260508 12:15:21.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:21.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 12:15:21.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:15:21.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:21.591 | INFO | Uploaded 36GiB to GPU in 2.4328s (14.7978GiB/s) | 20260508 12:15:21.591 | INFO | Downloaded 34GiB to GPU in 2.48565s (13.6785GiB/s) | 20260508 12:15:26.728 | INFO | Analyzing chunk frames = [8029184,8029696) | 20260508 12:15:26.728 | INFO | MemInfo: { | 20260508 12:15:26.728 | INFO | "gpu" : " 17 GB", | 20260508 12:15:26.728 | INFO | "huge" : "", | 20260508 12:15:26.728 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:26.728 | INFO | "malloc" : " 14 GB", | 20260508 12:15:26.728 | INFO | "pinned" : " 29 GB" | 20260508 12:15:26.728 | INFO | } | 20260508 12:15:26.728 | INFO | Perf report for the last 5.13784 seconds: | 20260508 12:15:26.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:26.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.728 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:15:26.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.728 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:15:26.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:26.729 | INFO | Uploaded 36GiB to GPU in 2.48628s (14.4794GiB/s) | 20260508 12:15:26.729 | INFO | Downloaded 34GiB to GPU in 2.46989s (13.7658GiB/s) | 20260508 12:15:31.872 | INFO | Analyzing chunk frames = [8029696,8030208) | 20260508 12:15:31.872 | INFO | MemInfo: { | 20260508 12:15:31.872 | INFO | "gpu" : " 17 GB", | 20260508 12:15:31.872 | INFO | "huge" : "", | 20260508 12:15:31.872 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:31.872 | INFO | "malloc" : " 14 GB", | 20260508 12:15:31.872 | INFO | "pinned" : " 29 GB" | 20260508 12:15:31.872 | INFO | } | 20260508 12:15:31.872 | INFO | Perf report for the last 5.14379 seconds: | 20260508 12:15:31.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:31.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.872 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:15:31.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.872 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:15:31.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:31.873 | INFO | Uploaded 36GiB to GPU in 2.44585s (14.7188GiB/s) | 20260508 12:15:31.873 | INFO | Downloaded 34GiB to GPU in 2.45527s (13.8478GiB/s) | 20260508 12:15:37.010 | INFO | Analyzing chunk frames = [8030208,8030720) | 20260508 12:15:37.010 | INFO | MemInfo: { | 20260508 12:15:37.010 | INFO | "gpu" : " 17 GB", | 20260508 12:15:37.010 | INFO | "huge" : "", | 20260508 12:15:37.010 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:37.010 | INFO | "malloc" : " 14 GB", | 20260508 12:15:37.010 | INFO | "pinned" : " 29 GB" | 20260508 12:15:37.010 | INFO | } | 20260508 12:15:37.010 | INFO | Perf report for the last 5.13769 seconds: | 20260508 12:15:37.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:37.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:15:37.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:15:37.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:37.010 | INFO | Uploaded 36GiB to GPU in 2.38683s (15.0828GiB/s) | 20260508 12:15:37.011 | INFO | Downloaded 34GiB to GPU in 2.51038s (13.5438GiB/s) | 20260508 12:15:42.145 | INFO | Analyzing chunk frames = [8030720,8031232) | 20260508 12:15:42.145 | INFO | MemInfo: { | 20260508 12:15:42.145 | INFO | "gpu" : " 17 GB", | 20260508 12:15:42.145 | INFO | "huge" : "", | 20260508 12:15:42.145 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:42.145 | INFO | "malloc" : " 14 GB", | 20260508 12:15:42.145 | INFO | "pinned" : " 29 GB" | 20260508 12:15:42.145 | INFO | } | 20260508 12:15:42.145 | INFO | Perf report for the last 5.1353 seconds: | 20260508 12:15:42.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:42.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:15:42.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:15:42.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:42.145 | INFO | Uploaded 36GiB to GPU in 2.36242s (15.2386GiB/s) | 20260508 12:15:42.145 | INFO | Downloaded 34GiB to GPU in 2.45568s (13.8454GiB/s) | 20260508 12:15:47.289 | INFO | Analyzing chunk frames = [8031232,8031744) | 20260508 12:15:47.289 | INFO | MemInfo: { | 20260508 12:15:47.289 | INFO | "gpu" : " 17 GB", | 20260508 12:15:47.289 | INFO | "huge" : "", | 20260508 12:15:47.289 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:47.289 | INFO | "malloc" : " 14 GB", | 20260508 12:15:47.289 | INFO | "pinned" : " 29 GB" | 20260508 12:15:47.289 | INFO | } | 20260508 12:15:47.290 | INFO | Perf report for the last 5.14431 seconds: | 20260508 12:15:47.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:47.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 12:15:47.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:15:47.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:47.290 | INFO | Uploaded 36GiB to GPU in 2.42001s (14.876GiB/s) | 20260508 12:15:47.290 | INFO | Downloaded 34GiB to GPU in 2.46197s (13.8101GiB/s) | 20260508 12:15:52.427 | INFO | Analyzing chunk frames = [8031744,8032256) | 20260508 12:15:52.427 | INFO | MemInfo: { | 20260508 12:15:52.427 | INFO | "gpu" : " 17 GB", | 20260508 12:15:52.427 | INFO | "huge" : "", | 20260508 12:15:52.427 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:52.427 | INFO | "malloc" : " 14 GB", | 20260508 12:15:52.427 | INFO | "pinned" : " 29 GB" | 20260508 12:15:52.427 | INFO | } | 20260508 12:15:52.428 | INFO | Perf report for the last 5.13802 seconds: | 20260508 12:15:52.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:52.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 12:15:52.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 12:15:52.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:52.428 | INFO | Uploaded 36GiB to GPU in 2.37688s (15.1459GiB/s) | 20260508 12:15:52.428 | INFO | Downloaded 34GiB to GPU in 2.46317s (13.8033GiB/s) | 20260508 12:15:52.524 | INFO | Starting a round of Baz IO | 20260508 12:15:55.941 | INFO | Populating RTMetrics report | 20260508 12:15:55.942 | INFO | RTMetrics report populated | 20260508 12:15:55.942 | INFO | Opening tmp RTMetrics json file | 20260508 12:15:55.949 | INFO | Serializing RTMetrics to json | 20260508 12:15:55.951 | INFO | Serialization done, swapping tmp file now... | 20260508 12:15:55.951 | INFO | RTMetrics done | 20260508 12:15:56.004 | INFO | Finished a round of Baz IO: Wrote 1.415 GiB in 3.479 seconds | 20260508 12:15:56.004 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:15:56.004 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169659% / 0% / 7.94729e-05% | 20260508 12:15:56.004 | INFO | Baz IO overall: | 20260508 12:15:56.005 | INFO | BytesWritten : 3448224257864 | 20260508 12:15:56.005 | INFO | NumEvents : 1654988429834 | 20260508 12:15:57.569 | INFO | Analyzing chunk frames = [8032256,8032768) | 20260508 12:15:57.569 | INFO | MemInfo: { | 20260508 12:15:57.569 | INFO | "gpu" : " 17 GB", | 20260508 12:15:57.569 | INFO | "huge" : "", | 20260508 12:15:57.569 | INFO | "huge_shared" : " 29 GB", | 20260508 12:15:57.569 | INFO | "malloc" : " 14 GB", | 20260508 12:15:57.569 | INFO | "pinned" : " 29 GB" | 20260508 12:15:57.569 | INFO | } | 20260508 12:15:57.569 | INFO | Perf report for the last 5.14169 seconds: | 20260508 12:15:57.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:15:57.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:57.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:57.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:57.569 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 12:15:57.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:15:57.569 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:15:57.569 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 12:15:57.569 | INFO | BAZWRITER : 66.9%, 2.4, 84.1ms, 0 | 20260508 12:15:57.569 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:15:57.570 | INFO | Uploaded 36GiB to GPU in 2.60842s (13.8015GiB/s) | 20260508 12:15:57.570 | INFO | Downloaded 38GiB to GPU in 2.7148s (13.9973GiB/s) | 20260508 12:16:02.701 | INFO | Analyzing chunk frames = [8032768,8033280) | 20260508 12:16:02.701 | INFO | MemInfo: { | 20260508 12:16:02.701 | INFO | "gpu" : " 17 GB", | 20260508 12:16:02.701 | INFO | "huge" : "", | 20260508 12:16:02.701 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:02.701 | INFO | "malloc" : " 14 GB", | 20260508 12:16:02.701 | INFO | "pinned" : " 29 GB" | 20260508 12:16:02.701 | INFO | } | 20260508 12:16:02.701 | INFO | Perf report for the last 5.13213 seconds: | 20260508 12:16:02.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:02.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.701 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 12:16:02.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.702 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:16:02.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:02.702 | INFO | Uploaded 36GiB to GPU in 2.41564s (14.9029GiB/s) | 20260508 12:16:02.702 | INFO | Downloaded 34GiB to GPU in 2.47005s (13.7649GiB/s) | 20260508 12:16:07.848 | INFO | Analyzing chunk frames = [8033280,8033792) | 20260508 12:16:07.848 | INFO | MemInfo: { | 20260508 12:16:07.848 | INFO | "gpu" : " 17 GB", | 20260508 12:16:07.848 | INFO | "huge" : "", | 20260508 12:16:07.848 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:07.848 | INFO | "malloc" : " 14 GB", | 20260508 12:16:07.848 | INFO | "pinned" : " 29 GB" | 20260508 12:16:07.848 | INFO | } | 20260508 12:16:07.849 | INFO | Perf report for the last 5.14718 seconds: | 20260508 12:16:07.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:07.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 12:16:07.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:16:07.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:07.849 | INFO | Uploaded 36GiB to GPU in 2.45188s (14.6826GiB/s) | 20260508 12:16:07.849 | INFO | Downloaded 34GiB to GPU in 2.45507s (13.8489GiB/s) | 20260508 12:16:12.987 | INFO | Analyzing chunk frames = [8033792,8034304) | 20260508 12:16:12.987 | INFO | MemInfo: { | 20260508 12:16:12.987 | INFO | "gpu" : " 17 GB", | 20260508 12:16:12.987 | INFO | "huge" : "", | 20260508 12:16:12.987 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:12.987 | INFO | "malloc" : " 14 GB", | 20260508 12:16:12.987 | INFO | "pinned" : " 29 GB" | 20260508 12:16:12.987 | INFO | } | 20260508 12:16:12.987 | INFO | Perf report for the last 5.13832 seconds: | 20260508 12:16:12.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:12.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 12:16:12.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:16:12.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:12.987 | INFO | Uploaded 36GiB to GPU in 2.38141s (15.1171GiB/s) | 20260508 12:16:12.987 | INFO | Downloaded 34GiB to GPU in 2.48658s (13.6734GiB/s) | 20260508 12:16:18.119 | INFO | Analyzing chunk frames = [8034304,8034816) | 20260508 12:16:18.119 | INFO | MemInfo: { | 20260508 12:16:18.119 | INFO | "gpu" : " 17 GB", | 20260508 12:16:18.119 | INFO | "huge" : "", | 20260508 12:16:18.119 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:18.119 | INFO | "malloc" : " 14 GB", | 20260508 12:16:18.119 | INFO | "pinned" : " 29 GB" | 20260508 12:16:18.119 | INFO | } | 20260508 12:16:18.119 | INFO | Perf report for the last 5.13218 seconds: | 20260508 12:16:18.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:18.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:16:18.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 12:16:18.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:18.119 | INFO | Uploaded 36GiB to GPU in 2.31307s (15.5637GiB/s) | 20260508 12:16:18.119 | INFO | Downloaded 34GiB to GPU in 2.50782s (13.5576GiB/s) | 20260508 12:16:23.260 | INFO | Analyzing chunk frames = [8034816,8035328) | 20260508 12:16:23.260 | INFO | MemInfo: { | 20260508 12:16:23.260 | INFO | "gpu" : " 17 GB", | 20260508 12:16:23.260 | INFO | "huge" : "", | 20260508 12:16:23.260 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:23.261 | INFO | "malloc" : " 14 GB", | 20260508 12:16:23.261 | INFO | "pinned" : " 29 GB" | 20260508 12:16:23.261 | INFO | } | 20260508 12:16:23.261 | INFO | Perf report for the last 5.14158 seconds: | 20260508 12:16:23.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:23.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:16:23.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:16:23.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:23.261 | INFO | Uploaded 36GiB to GPU in 2.37691s (15.1457GiB/s) | 20260508 12:16:23.261 | INFO | Downloaded 34GiB to GPU in 2.43857s (13.9426GiB/s) | 20260508 12:16:28.396 | INFO | Analyzing chunk frames = [8035328,8035840) | 20260508 12:16:28.397 | INFO | MemInfo: { | 20260508 12:16:28.397 | INFO | "gpu" : " 17 GB", | 20260508 12:16:28.397 | INFO | "huge" : "", | 20260508 12:16:28.397 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:28.397 | INFO | "malloc" : " 14 GB", | 20260508 12:16:28.397 | INFO | "pinned" : " 29 GB" | 20260508 12:16:28.397 | INFO | } | 20260508 12:16:28.397 | INFO | Perf report for the last 5.13602 seconds: | 20260508 12:16:28.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:28.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 12:16:28.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:16:28.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:28.397 | INFO | Uploaded 36GiB to GPU in 2.43747s (14.7694GiB/s) | 20260508 12:16:28.397 | INFO | Downloaded 34GiB to GPU in 2.45466s (13.8512GiB/s) | 20260508 12:16:33.543 | INFO | Analyzing chunk frames = [8035840,8036352) | 20260508 12:16:33.543 | INFO | MemInfo: { | 20260508 12:16:33.543 | INFO | "gpu" : " 17 GB", | 20260508 12:16:33.543 | INFO | "huge" : "", | 20260508 12:16:33.543 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:33.543 | INFO | "malloc" : " 14 GB", | 20260508 12:16:33.543 | INFO | "pinned" : " 29 GB" | 20260508 12:16:33.543 | INFO | } | 20260508 12:16:33.543 | INFO | Perf report for the last 5.14652 seconds: | 20260508 12:16:33.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:33.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.543 | INFO | ANALYSIS : 65.5%, 3.0, 102.6ms, 0 | 20260508 12:16:33.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.543 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:16:33.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:33.544 | INFO | Uploaded 36GiB to GPU in 2.4239s (14.8521GiB/s) | 20260508 12:16:33.544 | INFO | Downloaded 34GiB to GPU in 2.4419s (13.9236GiB/s) | 20260508 12:16:33.544 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:16:33.544 | INFO | Total mem usage: 16.7 GiB | 20260508 12:16:33.544 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:16:33.544 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:16:33.544 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:16:33.544 | INFO | High Water Breakdown: | 20260508 12:16:33.544 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:16:33.544 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:16:33.544 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:16:33.544 | INFO | ------------------------------------------ | 20260508 12:16:33.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:16:33.544 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:16:33.544 | INFO | Total mem usage: 26.5 GiB | 20260508 12:16:33.544 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:16:33.544 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:16:33.544 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:16:33.544 | INFO | High Water Breakdown: | 20260508 12:16:33.544 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:16:33.544 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:16:33.544 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:16:33.544 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:16:33.544 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:16:33.544 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:16:33.544 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:16:33.544 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:16:33.544 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:16:33.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:16:33.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:16:33.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:16:33.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:16:33.544 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:16:33.544 | INFO | ------------------------------------------ | 20260508 12:16:33.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:16:33.544 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:16:33.544 | INFO | Total mem usage: 27.8 GiB | 20260508 12:16:33.544 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:16:33.544 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:16:33.544 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:16:33.544 | INFO | High Water Breakdown: | 20260508 12:16:33.544 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:16:33.544 | INFO | ------------------------------------------ | 20260508 12:16:33.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:16:33.544 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:16:33.544 | INFO | Total mem usage: 15.7 GiB | 20260508 12:16:33.544 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:16:33.544 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:16:33.544 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:16:33.544 | INFO | High Water Breakdown: | 20260508 12:16:33.544 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:16:33.544 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:16:33.544 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:16:33.544 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:16:33.544 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:16:33.544 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:16:33.544 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:16:33.544 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:16:33.544 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:16:33.544 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:16:33.544 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:16:33.545 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:16:33.545 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:16:33.545 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:16:33.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:16:33.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:16:33.545 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:16:33.545 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:16:33.545 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:16:33.545 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:16:33.545 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:16:33.545 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:16:33.545 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:16:33.545 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:16:33.545 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:16:33.545 | INFO | ------------------------------------------ | 20260508 12:16:33.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:16:33.545 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:16:33.545 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:16:33.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:16:33.545 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.42e-01 | 20260508 12:16:33.545 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.10e-02 / 8.26e-03 | 20260508 12:16:33.545 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.44e+00, 5.89e-01 / 2.44e-01 | 20260508 12:16:33.545 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.46e+00, 2.08e-01 / 1.06e-01 | 20260508 12:16:33.545 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.52e+01, 3.69e+00 / 9.52e-01 | 20260508 12:16:33.545 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.27e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.04e-01 / 2.47e-01 | 20260508 12:16:33.545 | INFO | PerfReport: 2.96%, Metrics -- 9.67e-01 / 1.12e+00, 2.40e-01 / 1.51e-01 | 20260508 12:16:33.545 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.22e-01, 4.70e-02 / 2.67e-02 | 20260508 12:16:33.545 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:16:33.545 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:16:33.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:16:33.545 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 24.15%, Upload -- 2.53e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 24.83%, Download -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 31.11%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 1.99%, LockUpload -- 2.08e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 2.75%, LockDownload -- 2.88e+00 / 0.00e+00, 4.32e+00 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 15.16%, LockCompute -- 1.59e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:16:33.545 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260508 12:16:33.641 | INFO | Starting a round of Baz IO | 20260508 12:16:37.045 | INFO | Populating RTMetrics report | 20260508 12:16:37.046 | INFO | RTMetrics report populated | 20260508 12:16:37.046 | INFO | Opening tmp RTMetrics json file | 20260508 12:16:37.057 | INFO | Serializing RTMetrics to json | 20260508 12:16:37.058 | INFO | Serialization done, swapping tmp file now... | 20260508 12:16:37.059 | INFO | RTMetrics done | 20260508 12:16:37.116 | INFO | Finished a round of Baz IO: Wrote 1.414 GiB in 3.474 seconds | 20260508 12:16:37.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.04% / 0.00% / 59.96% | 20260508 12:16:37.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168538% / 3.97364e-06% / 9.53674e-05% | 20260508 12:16:37.116 | INFO | Baz IO overall: | 20260508 12:16:37.116 | INFO | BytesWritten : 3449742290352 | 20260508 12:16:37.116 | INFO | NumEvents : 1655700998967 | 20260508 12:16:38.687 | INFO | Analyzing chunk frames = [8036352,8036864) | 20260508 12:16:38.687 | INFO | MemInfo: { | 20260508 12:16:38.687 | INFO | "gpu" : " 17 GB", | 20260508 12:16:38.687 | INFO | "huge" : "", | 20260508 12:16:38.687 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:38.687 | INFO | "malloc" : " 14 GB", | 20260508 12:16:38.687 | INFO | "pinned" : " 29 GB" | 20260508 12:16:38.687 | INFO | } | 20260508 12:16:38.687 | INFO | Perf report for the last 5.14368 seconds: | 20260508 12:16:38.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:38.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:38.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:38.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:38.687 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:16:38.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:38.687 | INFO | RT_METRICS : 38.4%, 1.0, 20.2ms, 0 | 20260508 12:16:38.687 | INFO | PRE_HQ : 65.2%, 1.6, 54.0ms, 0 | 20260508 12:16:38.687 | INFO | BAZWRITER : 66.9%, 2.4, 84.2ms, 0 | 20260508 12:16:38.687 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:16:38.687 | INFO | Uploaded 36GiB to GPU in 2.63797s (13.6469GiB/s) | 20260508 12:16:38.687 | INFO | Downloaded 38GiB to GPU in 2.71625s (13.9899GiB/s) | 20260508 12:16:43.823 | INFO | Analyzing chunk frames = [8036864,8037376) | 20260508 12:16:43.823 | INFO | MemInfo: { | 20260508 12:16:43.823 | INFO | "gpu" : " 17 GB", | 20260508 12:16:43.823 | INFO | "huge" : "", | 20260508 12:16:43.823 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:43.823 | INFO | "malloc" : " 14 GB", | 20260508 12:16:43.823 | INFO | "pinned" : " 29 GB" | 20260508 12:16:43.823 | INFO | } | 20260508 12:16:43.823 | INFO | Perf report for the last 5.13619 seconds: | 20260508 12:16:43.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:43.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:16:43.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:16:43.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:43.823 | INFO | Uploaded 36GiB to GPU in 2.50982s (14.3437GiB/s) | 20260508 12:16:43.823 | INFO | Downloaded 34GiB to GPU in 2.47555s (13.7343GiB/s) | 20260508 12:16:48.955 | INFO | Analyzing chunk frames = [8037376,8037888) | 20260508 12:16:48.955 | INFO | MemInfo: { | 20260508 12:16:48.955 | INFO | "gpu" : " 17 GB", | 20260508 12:16:48.955 | INFO | "huge" : "", | 20260508 12:16:48.955 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:48.956 | INFO | "malloc" : " 14 GB", | 20260508 12:16:48.956 | INFO | "pinned" : " 29 GB" | 20260508 12:16:48.956 | INFO | } | 20260508 12:16:48.956 | INFO | Perf report for the last 5.1325 seconds: | 20260508 12:16:48.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:48.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 12:16:48.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:16:48.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:48.956 | INFO | Uploaded 36GiB to GPU in 2.52718s (14.2452GiB/s) | 20260508 12:16:48.956 | INFO | Downloaded 34GiB to GPU in 2.45665s (13.84GiB/s) | 20260508 12:16:54.095 | INFO | Analyzing chunk frames = [8037888,8038400) | 20260508 12:16:54.095 | INFO | MemInfo: { | 20260508 12:16:54.095 | INFO | "gpu" : " 17 GB", | 20260508 12:16:54.095 | INFO | "huge" : "", | 20260508 12:16:54.095 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:54.095 | INFO | "malloc" : " 14 GB", | 20260508 12:16:54.095 | INFO | "pinned" : " 29 GB" | 20260508 12:16:54.096 | INFO | } | 20260508 12:16:54.096 | INFO | Perf report for the last 5.13989 seconds: | 20260508 12:16:54.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:54.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:16:54.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:16:54.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:54.096 | INFO | Uploaded 36GiB to GPU in 2.49677s (14.4186GiB/s) | 20260508 12:16:54.096 | INFO | Downloaded 34GiB to GPU in 2.44917s (13.8822GiB/s) | 20260508 12:16:59.244 | INFO | Analyzing chunk frames = [8038400,8038912) | 20260508 12:16:59.244 | INFO | MemInfo: { | 20260508 12:16:59.244 | INFO | "gpu" : " 17 GB", | 20260508 12:16:59.244 | INFO | "huge" : "", | 20260508 12:16:59.244 | INFO | "huge_shared" : " 29 GB", | 20260508 12:16:59.244 | INFO | "malloc" : " 14 GB", | 20260508 12:16:59.244 | INFO | "pinned" : " 29 GB" | 20260508 12:16:59.244 | INFO | } | 20260508 12:16:59.244 | INFO | Perf report for the last 5.14886 seconds: | 20260508 12:16:59.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:16:59.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.245 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 12:16:59.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.245 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 12:16:59.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:16:59.245 | INFO | Uploaded 36GiB to GPU in 2.31876s (15.5256GiB/s) | 20260508 12:16:59.245 | INFO | Downloaded 34GiB to GPU in 2.49998s (13.6001GiB/s) | 20260508 12:17:04.381 | INFO | Analyzing chunk frames = [8038912,8039424) | 20260508 12:17:04.381 | INFO | MemInfo: { | 20260508 12:17:04.381 | INFO | "gpu" : " 17 GB", | 20260508 12:17:04.382 | INFO | "huge" : "", | 20260508 12:17:04.382 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:04.382 | INFO | "malloc" : " 14 GB", | 20260508 12:17:04.382 | INFO | "pinned" : " 29 GB" | 20260508 12:17:04.382 | INFO | } | 20260508 12:17:04.382 | INFO | Perf report for the last 5.13726 seconds: | 20260508 12:17:04.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:04.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:17:04.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:17:04.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:04.382 | INFO | Uploaded 36GiB to GPU in 2.4363s (14.7765GiB/s) | 20260508 12:17:04.382 | INFO | Downloaded 34GiB to GPU in 2.42994s (13.9921GiB/s) | 20260508 12:17:09.516 | INFO | Analyzing chunk frames = [8039424,8039936) | 20260508 12:17:09.516 | INFO | MemInfo: { | 20260508 12:17:09.516 | INFO | "gpu" : " 17 GB", | 20260508 12:17:09.516 | INFO | "huge" : "", | 20260508 12:17:09.516 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:09.516 | INFO | "malloc" : " 14 GB", | 20260508 12:17:09.516 | INFO | "pinned" : " 29 GB" | 20260508 12:17:09.516 | INFO | } | 20260508 12:17:09.517 | INFO | Perf report for the last 5.13483 seconds: | 20260508 12:17:09.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:09.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 12:17:09.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:17:09.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:09.517 | INFO | Uploaded 36GiB to GPU in 2.35364s (15.2955GiB/s) | 20260508 12:17:09.517 | INFO | Downloaded 34GiB to GPU in 2.41308s (14.0899GiB/s) | 20260508 12:17:14.655 | INFO | Analyzing chunk frames = [8039936,8040448) | 20260508 12:17:14.655 | INFO | MemInfo: { | 20260508 12:17:14.655 | INFO | "gpu" : " 17 GB", | 20260508 12:17:14.655 | INFO | "huge" : "", | 20260508 12:17:14.655 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:14.655 | INFO | "malloc" : " 14 GB", | 20260508 12:17:14.655 | INFO | "pinned" : " 29 GB" | 20260508 12:17:14.655 | INFO | } | 20260508 12:17:14.655 | INFO | Perf report for the last 5.13845 seconds: | 20260508 12:17:14.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:14.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 12:17:14.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:17:14.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:14.655 | INFO | Uploaded 36GiB to GPU in 2.49546s (14.4262GiB/s) | 20260508 12:17:14.655 | INFO | Downloaded 34GiB to GPU in 2.45011s (13.877GiB/s) | 20260508 12:17:14.755 | INFO | Starting a round of Baz IO | 20260508 12:17:18.158 | INFO | Populating RTMetrics report | 20260508 12:17:18.158 | INFO | RTMetrics report populated | 20260508 12:17:18.158 | INFO | Opening tmp RTMetrics json file | 20260508 12:17:18.159 | INFO | Serializing RTMetrics to json | 20260508 12:17:18.160 | INFO | Serialization done, swapping tmp file now... | 20260508 12:17:18.160 | INFO | RTMetrics done | 20260508 12:17:18.222 | INFO | Finished a round of Baz IO: Wrote 1.414 GiB in 3.467 seconds | 20260508 12:17:18.222 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.96% | 20260508 12:17:18.222 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.16851% / 0% / 7.15256e-05% | 20260508 12:17:18.222 | INFO | Baz IO overall: | 20260508 12:17:18.222 | INFO | BytesWritten : 3451260061466 | 20260508 12:17:18.222 | INFO | NumEvents : 1656413387972 | 20260508 12:17:19.799 | INFO | Analyzing chunk frames = [8040448,8040960) | 20260508 12:17:19.799 | INFO | MemInfo: { | 20260508 12:17:19.799 | INFO | "gpu" : " 17 GB", | 20260508 12:17:19.799 | INFO | "huge" : "", | 20260508 12:17:19.799 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:19.799 | INFO | "malloc" : " 14 GB", | 20260508 12:17:19.799 | INFO | "pinned" : " 29 GB" | 20260508 12:17:19.799 | INFO | } | 20260508 12:17:19.799 | INFO | Perf report for the last 5.14356 seconds: | 20260508 12:17:19.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:19.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:19.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:19.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:19.799 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 12:17:19.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:19.799 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:17:19.799 | INFO | PRE_HQ : 65.5%, 1.6, 54.0ms, 0 | 20260508 12:17:19.799 | INFO | BAZWRITER : 66.9%, 2.5, 85.3ms, 0 | 20260508 12:17:19.799 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 12:17:19.799 | INFO | Uploaded 36GiB to GPU in 2.54115s (14.1668GiB/s) | 20260508 12:17:19.799 | INFO | Downloaded 38GiB to GPU in 2.69626s (14.0936GiB/s) | 20260508 12:17:24.934 | INFO | Analyzing chunk frames = [8040960,8041472) | 20260508 12:17:24.934 | INFO | MemInfo: { | 20260508 12:17:24.934 | INFO | "gpu" : " 17 GB", | 20260508 12:17:24.934 | INFO | "huge" : "", | 20260508 12:17:24.934 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:24.934 | INFO | "malloc" : " 14 GB", | 20260508 12:17:24.934 | INFO | "pinned" : " 29 GB" | 20260508 12:17:24.934 | INFO | } | 20260508 12:17:24.934 | INFO | Perf report for the last 5.13571 seconds: | 20260508 12:17:24.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:24.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.934 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:17:24.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.934 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:17:24.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:24.935 | INFO | Uploaded 36GiB to GPU in 2.46849s (14.5838GiB/s) | 20260508 12:17:24.935 | INFO | Downloaded 34GiB to GPU in 2.46774s (13.7778GiB/s) | 20260508 12:17:30.071 | INFO | Analyzing chunk frames = [8041472,8041984) | 20260508 12:17:30.071 | INFO | MemInfo: { | 20260508 12:17:30.071 | INFO | "gpu" : " 17 GB", | 20260508 12:17:30.071 | INFO | "huge" : "", | 20260508 12:17:30.071 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:30.071 | INFO | "malloc" : " 14 GB", | 20260508 12:17:30.071 | INFO | "pinned" : " 29 GB" | 20260508 12:17:30.071 | INFO | } | 20260508 12:17:30.071 | INFO | Perf report for the last 5.13676 seconds: | 20260508 12:17:30.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:30.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 12:17:30.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:17:30.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:30.071 | INFO | Uploaded 36GiB to GPU in 2.48556s (14.4836GiB/s) | 20260508 12:17:30.071 | INFO | Downloaded 34GiB to GPU in 2.44057s (13.9312GiB/s) | 20260508 12:17:35.207 | INFO | Analyzing chunk frames = [8041984,8042496) | 20260508 12:17:35.207 | INFO | MemInfo: { | 20260508 12:17:35.207 | INFO | "gpu" : " 17 GB", | 20260508 12:17:35.207 | INFO | "huge" : "", | 20260508 12:17:35.207 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:35.207 | INFO | "malloc" : " 14 GB", | 20260508 12:17:35.207 | INFO | "pinned" : " 29 GB" | 20260508 12:17:35.207 | INFO | } | 20260508 12:17:35.207 | INFO | Perf report for the last 5.13641 seconds: | 20260508 12:17:35.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:35.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:17:35.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:17:35.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:35.208 | INFO | Uploaded 36GiB to GPU in 2.47281s (14.5583GiB/s) | 20260508 12:17:35.208 | INFO | Downloaded 34GiB to GPU in 2.46363s (13.8008GiB/s) | 20260508 12:17:40.358 | INFO | Analyzing chunk frames = [8042496,8043008) | 20260508 12:17:40.358 | INFO | MemInfo: { | 20260508 12:17:40.358 | INFO | "gpu" : " 17 GB", | 20260508 12:17:40.358 | INFO | "huge" : "", | 20260508 12:17:40.358 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:40.358 | INFO | "malloc" : " 14 GB", | 20260508 12:17:40.358 | INFO | "pinned" : " 29 GB" | 20260508 12:17:40.358 | INFO | } | 20260508 12:17:40.358 | INFO | Perf report for the last 5.15049 seconds: | 20260508 12:17:40.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:40.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 12:17:40.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:17:40.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:40.358 | INFO | Uploaded 36GiB to GPU in 2.37491s (15.1584GiB/s) | 20260508 12:17:40.358 | INFO | Downloaded 34GiB to GPU in 2.52918s (13.4431GiB/s) | 20260508 12:17:45.494 | INFO | Analyzing chunk frames = [8043008,8043520) | 20260508 12:17:45.494 | INFO | MemInfo: { | 20260508 12:17:45.494 | INFO | "gpu" : " 17 GB", | 20260508 12:17:45.494 | INFO | "huge" : "", | 20260508 12:17:45.494 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:45.494 | INFO | "malloc" : " 14 GB", | 20260508 12:17:45.495 | INFO | "pinned" : " 29 GB" | 20260508 12:17:45.495 | INFO | } | 20260508 12:17:45.495 | INFO | Perf report for the last 5.13656 seconds: | 20260508 12:17:45.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:45.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:17:45.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 12:17:45.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:45.495 | INFO | Uploaded 36GiB to GPU in 2.40917s (14.9429GiB/s) | 20260508 12:17:45.495 | INFO | Downloaded 34GiB to GPU in 2.44576s (13.9016GiB/s) | 20260508 12:17:50.634 | INFO | Analyzing chunk frames = [8043520,8044032) | 20260508 12:17:50.634 | INFO | MemInfo: { | 20260508 12:17:50.634 | INFO | "gpu" : " 17 GB", | 20260508 12:17:50.634 | INFO | "huge" : "", | 20260508 12:17:50.634 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:50.634 | INFO | "malloc" : " 14 GB", | 20260508 12:17:50.634 | INFO | "pinned" : " 29 GB" | 20260508 12:17:50.634 | INFO | } | 20260508 12:17:50.634 | INFO | Perf report for the last 5.13938 seconds: | 20260508 12:17:50.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:50.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 12:17:50.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:17:50.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:50.634 | INFO | Uploaded 36GiB to GPU in 2.43177s (14.804GiB/s) | 20260508 12:17:50.634 | INFO | Downloaded 34GiB to GPU in 2.45885s (13.8276GiB/s) | 20260508 12:17:55.767 | INFO | Analyzing chunk frames = [8044032,8044544) | 20260508 12:17:55.767 | INFO | MemInfo: { | 20260508 12:17:55.767 | INFO | "gpu" : " 17 GB", | 20260508 12:17:55.767 | INFO | "huge" : "", | 20260508 12:17:55.767 | INFO | "huge_shared" : " 29 GB", | 20260508 12:17:55.767 | INFO | "malloc" : " 14 GB", | 20260508 12:17:55.767 | INFO | "pinned" : " 29 GB" | 20260508 12:17:55.767 | INFO | } | 20260508 12:17:55.767 | INFO | Perf report for the last 5.13284 seconds: | 20260508 12:17:55.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:17:55.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | ANALYSIS : 65.1%, 3.0, 101.6ms, 0 | 20260508 12:17:55.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 12:17:55.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:17:55.767 | INFO | Uploaded 36GiB to GPU in 2.39299s (15.044GiB/s) | 20260508 12:17:55.767 | INFO | Downloaded 34GiB to GPU in 2.45121s (13.8707GiB/s) | 20260508 12:17:55.767 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:17:55.767 | INFO | Total mem usage: 16.7 GiB | 20260508 12:17:55.767 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:17:55.767 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:17:55.767 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:17:55.767 | INFO | High Water Breakdown: | 20260508 12:17:55.767 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:17:55.767 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:17:55.767 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:17:55.767 | INFO | ------------------------------------------ | 20260508 12:17:55.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:17:55.767 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:17:55.767 | INFO | Total mem usage: 26.5 GiB | 20260508 12:17:55.767 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:17:55.767 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:17:55.767 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:17:55.767 | INFO | High Water Breakdown: | 20260508 12:17:55.768 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:17:55.768 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:17:55.768 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:17:55.768 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:17:55.768 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:17:55.768 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:17:55.768 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:17:55.768 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:17:55.768 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:17:55.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:17:55.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:17:55.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:17:55.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:17:55.768 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:17:55.768 | INFO | ------------------------------------------ | 20260508 12:17:55.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:17:55.768 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:17:55.768 | INFO | Total mem usage: 27.8 GiB | 20260508 12:17:55.768 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:17:55.768 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:17:55.768 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:17:55.768 | INFO | High Water Breakdown: | 20260508 12:17:55.768 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:17:55.768 | INFO | ------------------------------------------ | 20260508 12:17:55.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:17:55.768 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:17:55.768 | INFO | Total mem usage: 15.7 GiB | 20260508 12:17:55.768 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:17:55.768 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:17:55.768 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:17:55.768 | INFO | High Water Breakdown: | 20260508 12:17:55.768 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:17:55.768 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:17:55.768 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:17:55.768 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:17:55.768 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:17:55.768 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:17:55.768 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:17:55.768 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:17:55.768 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:17:55.768 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:17:55.768 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:17:55.768 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:17:55.768 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:17:55.769 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:17:55.769 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:17:55.769 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:17:55.769 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:17:55.769 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:17:55.769 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:17:55.769 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:17:55.769 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:17:55.769 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:17:55.769 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:17:55.769 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:17:55.769 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:17:55.769 | INFO | ------------------------------------------ | 20260508 12:17:55.769 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:17:55.769 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:17:55.769 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:17:55.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:17:55.769 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 4.84e-01 | 20260508 12:17:55.769 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.12e-02 / 5.62e-03 | 20260508 12:17:55.769 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.47e+00, 5.96e-01 / 2.15e-01 | 20260508 12:17:55.769 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.10e-01 / 9.49e-02 | 20260508 12:17:55.769 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.57e+01, 3.81e+00 / 8.84e-01 | 20260508 12:17:55.769 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.79e+00 / 1.17e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.54e+00, 4.04e-01 / 2.19e-01 | 20260508 12:17:55.769 | INFO | PerfReport: 2.97%, Metrics -- 9.70e-01 / 1.06e+00, 2.43e-01 / 7.96e-02 | 20260508 12:17:55.769 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.92e-01 / 3.16e-01, 4.65e-02 / 1.97e-02 | 20260508 12:17:55.769 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:17:55.769 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:17:55.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:17:55.769 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 24.70%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 31.08%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 2.23%, LockUpload -- 2.34e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 15.00%, LockCompute -- 1.57e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:17:55.769 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 1.06e-02 / 0.00e+00 | 20260508 12:17:55.865 | INFO | Starting a round of Baz IO | 20260508 12:17:59.260 | INFO | Populating RTMetrics report | 20260508 12:17:59.260 | INFO | RTMetrics report populated | 20260508 12:17:59.260 | INFO | Opening tmp RTMetrics json file | 20260508 12:17:59.261 | INFO | Serializing RTMetrics to json | 20260508 12:17:59.262 | INFO | Serialization done, swapping tmp file now... | 20260508 12:17:59.262 | INFO | RTMetrics done | 20260508 12:17:59.330 | INFO | Finished a round of Baz IO: Wrote 1.413 GiB in 3.465 seconds | 20260508 12:17:59.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:17:59.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.16795% / 0% / 6.75519e-05% | 20260508 12:17:59.330 | INFO | Baz IO overall: | 20260508 12:17:59.330 | INFO | BytesWritten : 3452776925587 | 20260508 12:17:59.330 | INFO | NumEvents : 1657125440605 | 20260508 12:18:00.907 | INFO | Analyzing chunk frames = [8044544,8045056) | 20260508 12:18:00.907 | INFO | MemInfo: { | 20260508 12:18:00.907 | INFO | "gpu" : " 17 GB", | 20260508 12:18:00.907 | INFO | "huge" : "", | 20260508 12:18:00.907 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:00.907 | INFO | "malloc" : " 14 GB", | 20260508 12:18:00.907 | INFO | "pinned" : " 29 GB" | 20260508 12:18:00.907 | INFO | } | 20260508 12:18:00.907 | INFO | Perf report for the last 5.13995 seconds: | 20260508 12:18:00.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:00.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:00.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:00.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:00.907 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 12:18:00.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:00.907 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:18:00.907 | INFO | PRE_HQ : 65.2%, 1.6, 54.0ms, 0 | 20260508 12:18:00.907 | INFO | BAZWRITER : 66.7%, 2.5, 84.5ms, 0 | 20260508 12:18:00.907 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:18:00.907 | INFO | Uploaded 36GiB to GPU in 2.61772s (13.7524GiB/s) | 20260508 12:18:00.907 | INFO | Downloaded 38GiB to GPU in 2.72267s (13.9569GiB/s) | 20260508 12:18:06.051 | INFO | Analyzing chunk frames = [8045056,8045568) | 20260508 12:18:06.051 | INFO | MemInfo: { | 20260508 12:18:06.051 | INFO | "gpu" : " 17 GB", | 20260508 12:18:06.051 | INFO | "huge" : "", | 20260508 12:18:06.051 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:06.051 | INFO | "malloc" : " 14 GB", | 20260508 12:18:06.051 | INFO | "pinned" : " 29 GB" | 20260508 12:18:06.051 | INFO | } | 20260508 12:18:06.051 | INFO | Perf report for the last 5.14437 seconds: | 20260508 12:18:06.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:06.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.051 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 12:18:06.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.051 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260508 12:18:06.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:06.052 | INFO | Uploaded 36GiB to GPU in 2.49501s (14.4288GiB/s) | 20260508 12:18:06.052 | INFO | Downloaded 34GiB to GPU in 2.46355s (13.8012GiB/s) | 20260508 12:18:11.193 | INFO | Analyzing chunk frames = [8045568,8046080) | 20260508 12:18:11.193 | INFO | MemInfo: { | 20260508 12:18:11.193 | INFO | "gpu" : " 17 GB", | 20260508 12:18:11.193 | INFO | "huge" : "", | 20260508 12:18:11.193 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:11.193 | INFO | "malloc" : " 14 GB", | 20260508 12:18:11.193 | INFO | "pinned" : " 29 GB" | 20260508 12:18:11.193 | INFO | } | 20260508 12:18:11.194 | INFO | Perf report for the last 5.14237 seconds: | 20260508 12:18:11.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:11.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 12:18:11.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:18:11.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:11.194 | INFO | Uploaded 36GiB to GPU in 2.51257s (14.328GiB/s) | 20260508 12:18:11.194 | INFO | Downloaded 34GiB to GPU in 2.45138s (13.8698GiB/s) | 20260508 12:18:16.336 | INFO | Analyzing chunk frames = [8046080,8046592) | 20260508 12:18:16.336 | INFO | MemInfo: { | 20260508 12:18:16.336 | INFO | "gpu" : " 17 GB", | 20260508 12:18:16.336 | INFO | "huge" : "", | 20260508 12:18:16.336 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:16.337 | INFO | "malloc" : " 14 GB", | 20260508 12:18:16.337 | INFO | "pinned" : " 29 GB" | 20260508 12:18:16.337 | INFO | } | 20260508 12:18:16.337 | INFO | Perf report for the last 5.14316 seconds: | 20260508 12:18:16.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:16.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 12:18:16.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:18:16.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:16.337 | INFO | Uploaded 36GiB to GPU in 2.53828s (14.1828GiB/s) | 20260508 12:18:16.337 | INFO | Downloaded 34GiB to GPU in 2.4547s (13.851GiB/s) | 20260508 12:18:21.466 | INFO | Analyzing chunk frames = [8046592,8047104) | 20260508 12:18:21.466 | INFO | MemInfo: { | 20260508 12:18:21.466 | INFO | "gpu" : " 17 GB", | 20260508 12:18:21.466 | INFO | "huge" : "", | 20260508 12:18:21.466 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:21.466 | INFO | "malloc" : " 14 GB", | 20260508 12:18:21.466 | INFO | "pinned" : " 29 GB" | 20260508 12:18:21.466 | INFO | } | 20260508 12:18:21.466 | INFO | Perf report for the last 5.12945 seconds: | 20260508 12:18:21.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:21.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 12:18:21.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 12:18:21.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:21.466 | INFO | Uploaded 36GiB to GPU in 2.41838s (14.886GiB/s) | 20260508 12:18:21.466 | INFO | Downloaded 34GiB to GPU in 2.51903s (13.4972GiB/s) | 20260508 12:18:26.609 | INFO | Analyzing chunk frames = [8047104,8047616) | 20260508 12:18:26.609 | INFO | MemInfo: { | 20260508 12:18:26.609 | INFO | "gpu" : " 17 GB", | 20260508 12:18:26.609 | INFO | "huge" : "", | 20260508 12:18:26.609 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:26.609 | INFO | "malloc" : " 14 GB", | 20260508 12:18:26.609 | INFO | "pinned" : " 29 GB" | 20260508 12:18:26.609 | INFO | } | 20260508 12:18:26.609 | INFO | Perf report for the last 5.14299 seconds: | 20260508 12:18:26.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:26.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 12:18:26.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:18:26.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:26.609 | INFO | Uploaded 36GiB to GPU in 2.44139s (14.7457GiB/s) | 20260508 12:18:26.609 | INFO | Downloaded 34GiB to GPU in 2.43304s (13.9743GiB/s) | 20260508 12:18:31.746 | INFO | Analyzing chunk frames = [8047616,8048128) | 20260508 12:18:31.746 | INFO | MemInfo: { | 20260508 12:18:31.746 | INFO | "gpu" : " 17 GB", | 20260508 12:18:31.746 | INFO | "huge" : "", | 20260508 12:18:31.746 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:31.746 | INFO | "malloc" : " 14 GB", | 20260508 12:18:31.746 | INFO | "pinned" : " 29 GB" | 20260508 12:18:31.746 | INFO | } | 20260508 12:18:31.746 | INFO | Perf report for the last 5.1371 seconds: | 20260508 12:18:31.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:31.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.746 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:18:31.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.746 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:18:31.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:31.747 | INFO | Uploaded 36GiB to GPU in 2.46493s (14.6049GiB/s) | 20260508 12:18:31.747 | INFO | Downloaded 34GiB to GPU in 2.44104s (13.9285GiB/s) | 20260508 12:18:36.886 | INFO | Analyzing chunk frames = [8048128,8048640) | 20260508 12:18:36.886 | INFO | MemInfo: { | 20260508 12:18:36.886 | INFO | "gpu" : " 17 GB", | 20260508 12:18:36.886 | INFO | "huge" : "", | 20260508 12:18:36.886 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:36.886 | INFO | "malloc" : " 14 GB", | 20260508 12:18:36.886 | INFO | "pinned" : " 29 GB" | 20260508 12:18:36.886 | INFO | } | 20260508 12:18:36.886 | INFO | Perf report for the last 5.13978 seconds: | 20260508 12:18:36.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:36.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.886 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:18:36.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.886 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 12:18:36.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:36.887 | INFO | Uploaded 36GiB to GPU in 2.45293s (14.6764GiB/s) | 20260508 12:18:36.887 | INFO | Downloaded 34GiB to GPU in 2.42909s (13.997GiB/s) | 20260508 12:18:36.983 | INFO | Starting a round of Baz IO | 20260508 12:18:40.383 | INFO | Populating RTMetrics report | 20260508 12:18:40.383 | INFO | RTMetrics report populated | 20260508 12:18:40.383 | INFO | Opening tmp RTMetrics json file | 20260508 12:18:40.391 | INFO | Serializing RTMetrics to json | 20260508 12:18:40.393 | INFO | Serialization done, swapping tmp file now... | 20260508 12:18:40.393 | INFO | RTMetrics done | 20260508 12:18:40.451 | INFO | Finished a round of Baz IO: Wrote 1.412 GiB in 3.468 seconds | 20260508 12:18:40.452 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:18:40.452 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167835% / 3.97364e-06% / 8.74201e-05% | 20260508 12:18:40.452 | INFO | Baz IO overall: | 20260508 12:18:40.452 | INFO | BytesWritten : 3454293113104 | 20260508 12:18:40.452 | INFO | NumEvents : 1657836834119 | 20260508 12:18:42.023 | INFO | Analyzing chunk frames = [8048640,8049152) | 20260508 12:18:42.023 | INFO | MemInfo: { | 20260508 12:18:42.023 | INFO | "gpu" : " 17 GB", | 20260508 12:18:42.023 | INFO | "huge" : "", | 20260508 12:18:42.023 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:42.023 | INFO | "malloc" : " 14 GB", | 20260508 12:18:42.023 | INFO | "pinned" : " 29 GB" | 20260508 12:18:42.023 | INFO | } | 20260508 12:18:42.023 | INFO | Perf report for the last 5.13667 seconds: | 20260508 12:18:42.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:42.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:42.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:42.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:42.023 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 12:18:42.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:42.023 | INFO | RT_METRICS : 38.8%, 1.0, 20.3ms, 0 | 20260508 12:18:42.023 | INFO | PRE_HQ : 65.6%, 1.6, 54.5ms, 0 | 20260508 12:18:42.023 | INFO | BAZWRITER : 66.7%, 2.5, 84.7ms, 0 | 20260508 12:18:42.023 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:18:42.023 | INFO | Uploaded 36GiB to GPU in 2.62904s (13.6932GiB/s) | 20260508 12:18:42.023 | INFO | Downloaded 38GiB to GPU in 2.71304s (14.0064GiB/s) | 20260508 12:18:47.162 | INFO | Analyzing chunk frames = [8049152,8049664) | 20260508 12:18:47.162 | INFO | MemInfo: { | 20260508 12:18:47.162 | INFO | "gpu" : " 17 GB", | 20260508 12:18:47.163 | INFO | "huge" : "", | 20260508 12:18:47.163 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:47.163 | INFO | "malloc" : " 14 GB", | 20260508 12:18:47.163 | INFO | "pinned" : " 29 GB" | 20260508 12:18:47.163 | INFO | } | 20260508 12:18:47.163 | INFO | Perf report for the last 5.13988 seconds: | 20260508 12:18:47.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:47.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:18:47.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:18:47.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:47.163 | INFO | Uploaded 36GiB to GPU in 2.43035s (14.8127GiB/s) | 20260508 12:18:47.163 | INFO | Downloaded 34GiB to GPU in 2.46374s (13.8001GiB/s) | 20260508 12:18:52.304 | INFO | Analyzing chunk frames = [8049664,8050176) | 20260508 12:18:52.304 | INFO | MemInfo: { | 20260508 12:18:52.304 | INFO | "gpu" : " 17 GB", | 20260508 12:18:52.304 | INFO | "huge" : "", | 20260508 12:18:52.304 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:52.305 | INFO | "malloc" : " 14 GB", | 20260508 12:18:52.305 | INFO | "pinned" : " 29 GB" | 20260508 12:18:52.305 | INFO | } | 20260508 12:18:52.305 | INFO | Perf report for the last 5.14237 seconds: | 20260508 12:18:52.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:52.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 12:18:52.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:18:52.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:52.305 | INFO | Uploaded 36GiB to GPU in 2.47568s (14.5415GiB/s) | 20260508 12:18:52.305 | INFO | Downloaded 34GiB to GPU in 2.45223s (13.8649GiB/s) | 20260508 12:18:57.443 | INFO | Analyzing chunk frames = [8050176,8050688) | 20260508 12:18:57.443 | INFO | MemInfo: { | 20260508 12:18:57.443 | INFO | "gpu" : " 17 GB", | 20260508 12:18:57.443 | INFO | "huge" : "", | 20260508 12:18:57.443 | INFO | "huge_shared" : " 29 GB", | 20260508 12:18:57.443 | INFO | "malloc" : " 14 GB", | 20260508 12:18:57.444 | INFO | "pinned" : " 29 GB" | 20260508 12:18:57.444 | INFO | } | 20260508 12:18:57.444 | INFO | Perf report for the last 5.1386 seconds: | 20260508 12:18:57.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:18:57.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:18:57.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:18:57.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:18:57.444 | INFO | Uploaded 36GiB to GPU in 2.4757s (14.5413GiB/s) | 20260508 12:18:57.444 | INFO | Downloaded 34GiB to GPU in 2.46688s (13.7826GiB/s) | 20260508 12:19:02.579 | INFO | Analyzing chunk frames = [8050688,8051200) | 20260508 12:19:02.579 | INFO | MemInfo: { | 20260508 12:19:02.579 | INFO | "gpu" : " 17 GB", | 20260508 12:19:02.579 | INFO | "huge" : "", | 20260508 12:19:02.579 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:02.579 | INFO | "malloc" : " 14 GB", | 20260508 12:19:02.579 | INFO | "pinned" : " 29 GB" | 20260508 12:19:02.579 | INFO | } | 20260508 12:19:02.579 | INFO | Perf report for the last 5.13562 seconds: | 20260508 12:19:02.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:02.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.579 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 12:19:02.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.579 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:19:02.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:02.580 | INFO | Uploaded 36GiB to GPU in 2.40087s (14.9945GiB/s) | 20260508 12:19:02.580 | INFO | Downloaded 34GiB to GPU in 2.50157s (13.5915GiB/s) | 20260508 12:19:07.716 | INFO | Analyzing chunk frames = [8051200,8051712) | 20260508 12:19:07.716 | INFO | MemInfo: { | 20260508 12:19:07.716 | INFO | "gpu" : " 17 GB", | 20260508 12:19:07.716 | INFO | "huge" : "", | 20260508 12:19:07.716 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:07.716 | INFO | "malloc" : " 14 GB", | 20260508 12:19:07.716 | INFO | "pinned" : " 29 GB" | 20260508 12:19:07.716 | INFO | } | 20260508 12:19:07.716 | INFO | Perf report for the last 5.1371 seconds: | 20260508 12:19:07.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:07.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.716 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:19:07.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.717 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:19:07.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:07.717 | INFO | Uploaded 36GiB to GPU in 2.34337s (15.3625GiB/s) | 20260508 12:19:07.717 | INFO | Downloaded 34GiB to GPU in 2.43945s (13.9376GiB/s) | 20260508 12:19:12.859 | INFO | Analyzing chunk frames = [8051712,8052224) | 20260508 12:19:12.859 | INFO | MemInfo: { | 20260508 12:19:12.859 | INFO | "gpu" : " 17 GB", | 20260508 12:19:12.859 | INFO | "huge" : "", | 20260508 12:19:12.859 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:12.859 | INFO | "malloc" : " 14 GB", | 20260508 12:19:12.860 | INFO | "pinned" : " 29 GB" | 20260508 12:19:12.860 | INFO | } | 20260508 12:19:12.860 | INFO | Perf report for the last 5.14319 seconds: | 20260508 12:19:12.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:12.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:19:12.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 12:19:12.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:12.860 | INFO | Uploaded 36GiB to GPU in 2.42033s (14.874GiB/s) | 20260508 12:19:12.860 | INFO | Downloaded 34GiB to GPU in 2.43737s (13.9495GiB/s) | 20260508 12:19:18.005 | INFO | Analyzing chunk frames = [8052224,8052736) | 20260508 12:19:18.005 | INFO | MemInfo: { | 20260508 12:19:18.005 | INFO | "gpu" : " 17 GB", | 20260508 12:19:18.005 | INFO | "huge" : "", | 20260508 12:19:18.005 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:18.005 | INFO | "malloc" : " 14 GB", | 20260508 12:19:18.005 | INFO | "pinned" : " 29 GB" | 20260508 12:19:18.005 | INFO | } | 20260508 12:19:18.005 | INFO | Perf report for the last 5.14573 seconds: | 20260508 12:19:18.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:18.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.005 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:19:18.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.005 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 12:19:18.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:18.006 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:19:18.006 | INFO | Total mem usage: 16.7 GiB | 20260508 12:19:18.006 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:19:18.006 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:19:18.006 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:19:18.006 | INFO | High Water Breakdown: | 20260508 12:19:18.006 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:19:18.006 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:19:18.006 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:19:18.006 | INFO | ------------------------------------------ | 20260508 12:19:18.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:19:18.006 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:19:18.006 | INFO | Total mem usage: 26.5 GiB | 20260508 12:19:18.006 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:19:18.006 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:19:18.006 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:19:18.006 | INFO | High Water Breakdown: | 20260508 12:19:18.006 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:19:18.006 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:19:18.006 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:19:18.006 | INFO | Uploaded 36GiB to GPU in 2.42641s (14.8368GiB/s) | 20260508 12:19:18.006 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:19:18.006 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:19:18.006 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:19:18.006 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:19:18.006 | INFO | Downloaded 34GiB to GPU in 2.45866s (13.8287GiB/s) | 20260508 12:19:18.006 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:19:18.006 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:19:18.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:19:18.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:19:18.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:19:18.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:19:18.006 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:19:18.006 | INFO | ------------------------------------------ | 20260508 12:19:18.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:19:18.006 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:19:18.006 | INFO | Total mem usage: 27.8 GiB | 20260508 12:19:18.006 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:19:18.006 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:19:18.006 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:19:18.006 | INFO | High Water Breakdown: | 20260508 12:19:18.006 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:19:18.006 | INFO | ------------------------------------------ | 20260508 12:19:18.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:19:18.006 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:19:18.006 | INFO | Total mem usage: 15.7 GiB | 20260508 12:19:18.006 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:19:18.006 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:19:18.006 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:19:18.006 | INFO | High Water Breakdown: | 20260508 12:19:18.006 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:19:18.006 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:19:18.006 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:19:18.006 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:19:18.006 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:19:18.006 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:19:18.007 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:19:18.007 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:19:18.007 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:19:18.007 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:19:18.007 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:19:18.007 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:19:18.007 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:19:18.007 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:19:18.007 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:19:18.007 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:19:18.007 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:19:18.007 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:19:18.007 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:19:18.007 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:19:18.007 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:19:18.007 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:19:18.007 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:19:18.007 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:19:18.007 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:19:18.007 | INFO | ------------------------------------------ | 20260508 12:19:18.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:19:18.007 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:19:18.007 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:19:18.007 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:19:18.007 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.43e-01 | 20260508 12:19:18.007 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.17e-02 / 5.55e-03 | 20260508 12:19:18.007 | INFO | PerfReport: 12.19%, Baseline -- 3.97e+00 / 4.42e+00, 5.93e-01 / 2.03e-01 | 20260508 12:19:18.007 | INFO | PerfReport: 3.36%, Histogram -- 1.09e+00 / 1.42e+00, 2.09e-01 / 1.19e-01 | 20260508 12:19:18.007 | INFO | PerfReport: 16.04%, DME -- 4.18e+01 / 4.52e+01, 3.61e+00 / 1.16e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.76e+00 / 1.34e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.44e+00, 4.05e-01 / 2.63e-01 | 20260508 12:19:18.007 | INFO | PerfReport: 2.97%, Metrics -- 9.68e-01 / 1.11e+00, 2.38e-01 / 1.27e-01 | 20260508 12:19:18.007 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.23e-01, 4.70e-02 / 2.70e-02 | 20260508 12:19:18.007 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:19:18.007 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:19:18.007 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:19:18.007 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 24.49%, Upload -- 2.57e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 24.68%, Download -- 2.59e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 30.99%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 2.22%, LockUpload -- 2.33e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 2.52%, LockDownload -- 2.65e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 15.08%, LockCompute -- 1.58e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:19:18.007 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 12:19:18.103 | INFO | Starting a round of Baz IO | 20260508 12:19:21.495 | INFO | Populating RTMetrics report | 20260508 12:19:21.495 | INFO | RTMetrics report populated | 20260508 12:19:21.495 | INFO | Opening tmp RTMetrics json file | 20260508 12:19:21.503 | INFO | Serializing RTMetrics to json | 20260508 12:19:21.504 | INFO | Serialization done, swapping tmp file now... | 20260508 12:19:21.505 | INFO | RTMetrics done | 20260508 12:19:21.562 | INFO | Finished a round of Baz IO: Wrote 1.412 GiB in 3.459 seconds | 20260508 12:19:21.562 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:19:21.562 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167004% / 0% / 6.35783e-05% | 20260508 12:19:21.562 | INFO | Baz IO overall: | 20260508 12:19:21.562 | INFO | BytesWritten : 3455808957793 | 20260508 12:19:21.563 | INFO | NumEvents : 1658548009649 | 20260508 12:19:23.140 | INFO | Analyzing chunk frames = [8052736,8053248) | 20260508 12:19:23.140 | INFO | MemInfo: { | 20260508 12:19:23.140 | INFO | "gpu" : " 17 GB", | 20260508 12:19:23.140 | INFO | "huge" : "", | 20260508 12:19:23.140 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:23.140 | INFO | "malloc" : " 14 GB", | 20260508 12:19:23.140 | INFO | "pinned" : " 29 GB" | 20260508 12:19:23.140 | INFO | } | 20260508 12:19:23.141 | INFO | Perf report for the last 5.13514 seconds: | 20260508 12:19:23.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:23.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:23.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:23.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:23.141 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 12:19:23.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:23.141 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:19:23.141 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 12:19:23.141 | INFO | BAZWRITER : 66.6%, 2.5, 84.7ms, 0 | 20260508 12:19:23.141 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:19:23.141 | INFO | Uploaded 36GiB to GPU in 2.61858s (13.7479GiB/s) | 20260508 12:19:23.141 | INFO | Downloaded 38GiB to GPU in 2.73993s (13.869GiB/s) | 20260508 12:19:28.281 | INFO | Analyzing chunk frames = [8053248,8053760) | 20260508 12:19:28.281 | INFO | MemInfo: { | 20260508 12:19:28.281 | INFO | "gpu" : " 17 GB", | 20260508 12:19:28.281 | INFO | "huge" : "", | 20260508 12:19:28.281 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:28.281 | INFO | "malloc" : " 14 GB", | 20260508 12:19:28.281 | INFO | "pinned" : " 29 GB" | 20260508 12:19:28.281 | INFO | } | 20260508 12:19:28.281 | INFO | Perf report for the last 5.14036 seconds: | 20260508 12:19:28.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:28.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 12:19:28.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:19:28.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:28.281 | INFO | Uploaded 36GiB to GPU in 2.48587s (14.4819GiB/s) | 20260508 12:19:28.281 | INFO | Downloaded 34GiB to GPU in 2.48076s (13.7055GiB/s) | 20260508 12:19:33.417 | INFO | Analyzing chunk frames = [8053760,8054272) | 20260508 12:19:33.417 | INFO | MemInfo: { | 20260508 12:19:33.417 | INFO | "gpu" : " 17 GB", | 20260508 12:19:33.417 | INFO | "huge" : "", | 20260508 12:19:33.417 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:33.418 | INFO | "malloc" : " 14 GB", | 20260508 12:19:33.418 | INFO | "pinned" : " 29 GB" | 20260508 12:19:33.418 | INFO | } | 20260508 12:19:33.418 | INFO | Perf report for the last 5.13679 seconds: | 20260508 12:19:33.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:33.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 12:19:33.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:19:33.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:33.418 | INFO | Uploaded 36GiB to GPU in 2.47966s (14.5181GiB/s) | 20260508 12:19:33.418 | INFO | Downloaded 34GiB to GPU in 2.45786s (13.8332GiB/s) | 20260508 12:19:38.555 | INFO | Analyzing chunk frames = [8054272,8054784) | 20260508 12:19:38.555 | INFO | MemInfo: { | 20260508 12:19:38.555 | INFO | "gpu" : " 17 GB", | 20260508 12:19:38.555 | INFO | "huge" : "", | 20260508 12:19:38.555 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:38.555 | INFO | "malloc" : " 14 GB", | 20260508 12:19:38.555 | INFO | "pinned" : " 29 GB" | 20260508 12:19:38.555 | INFO | } | 20260508 12:19:38.555 | INFO | Perf report for the last 5.13754 seconds: | 20260508 12:19:38.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:38.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.555 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 12:19:38.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.555 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:19:38.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:38.556 | INFO | Uploaded 36GiB to GPU in 2.52738s (14.244GiB/s) | 20260508 12:19:38.556 | INFO | Downloaded 34GiB to GPU in 2.46967s (13.767GiB/s) | 20260508 12:19:43.699 | INFO | Analyzing chunk frames = [8054784,8055296) | 20260508 12:19:43.699 | INFO | MemInfo: { | 20260508 12:19:43.699 | INFO | "gpu" : " 17 GB", | 20260508 12:19:43.699 | INFO | "huge" : "", | 20260508 12:19:43.699 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:43.699 | INFO | "malloc" : " 14 GB", | 20260508 12:19:43.699 | INFO | "pinned" : " 29 GB" | 20260508 12:19:43.699 | INFO | } | 20260508 12:19:43.699 | INFO | Perf report for the last 5.14356 seconds: | 20260508 12:19:43.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:43.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | ANALYSIS : 66.6%, 3.0, 104.0ms, 0 | 20260508 12:19:43.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:19:43.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:43.699 | INFO | Uploaded 36GiB to GPU in 2.46887s (14.5816GiB/s) | 20260508 12:19:43.699 | INFO | Downloaded 34GiB to GPU in 2.5506s (13.3302GiB/s) | 20260508 12:19:48.836 | INFO | Analyzing chunk frames = [8055296,8055808) | 20260508 12:19:48.836 | INFO | MemInfo: { | 20260508 12:19:48.836 | INFO | "gpu" : " 17 GB", | 20260508 12:19:48.836 | INFO | "huge" : "", | 20260508 12:19:48.836 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:48.836 | INFO | "malloc" : " 14 GB", | 20260508 12:19:48.836 | INFO | "pinned" : " 29 GB" | 20260508 12:19:48.836 | INFO | } | 20260508 12:19:48.836 | INFO | Perf report for the last 5.13742 seconds: | 20260508 12:19:48.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:48.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.836 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 12:19:48.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.836 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:19:48.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:48.837 | INFO | Uploaded 36GiB to GPU in 2.39057s (15.0592GiB/s) | 20260508 12:19:48.837 | INFO | Downloaded 34GiB to GPU in 2.44327s (13.9158GiB/s) | 20260508 12:19:53.971 | INFO | Analyzing chunk frames = [8055808,8056320) | 20260508 12:19:53.971 | INFO | MemInfo: { | 20260508 12:19:53.971 | INFO | "gpu" : " 17 GB", | 20260508 12:19:53.971 | INFO | "huge" : "", | 20260508 12:19:53.971 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:53.971 | INFO | "malloc" : " 14 GB", | 20260508 12:19:53.971 | INFO | "pinned" : " 29 GB" | 20260508 12:19:53.971 | INFO | } | 20260508 12:19:53.971 | INFO | Perf report for the last 5.13474 seconds: | 20260508 12:19:53.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:53.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.971 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:19:53.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.971 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:19:53.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:53.972 | INFO | Uploaded 36GiB to GPU in 2.44661s (14.7143GiB/s) | 20260508 12:19:53.972 | INFO | Downloaded 34GiB to GPU in 2.46234s (13.808GiB/s) | 20260508 12:19:59.116 | INFO | Analyzing chunk frames = [8056320,8056832) | 20260508 12:19:59.116 | INFO | MemInfo: { | 20260508 12:19:59.116 | INFO | "gpu" : " 17 GB", | 20260508 12:19:59.117 | INFO | "huge" : "", | 20260508 12:19:59.117 | INFO | "huge_shared" : " 29 GB", | 20260508 12:19:59.117 | INFO | "malloc" : " 14 GB", | 20260508 12:19:59.117 | INFO | "pinned" : " 29 GB" | 20260508 12:19:59.117 | INFO | } | 20260508 12:19:59.117 | INFO | Perf report for the last 5.14571 seconds: | 20260508 12:19:59.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:19:59.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 12:19:59.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:19:59.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:19:59.117 | INFO | Uploaded 36GiB to GPU in 2.4146s (14.9093GiB/s) | 20260508 12:19:59.117 | INFO | Downloaded 34GiB to GPU in 2.44419s (13.9105GiB/s) | 20260508 12:19:59.214 | INFO | Starting a round of Baz IO | 20260508 12:20:02.608 | INFO | Populating RTMetrics report | 20260508 12:20:02.609 | INFO | RTMetrics report populated | 20260508 12:20:02.609 | INFO | Opening tmp RTMetrics json file | 20260508 12:20:02.617 | INFO | Serializing RTMetrics to json | 20260508 12:20:02.619 | INFO | Serialization done, swapping tmp file now... | 20260508 12:20:02.619 | INFO | RTMetrics done | 20260508 12:20:02.681 | INFO | Finished a round of Baz IO: Wrote 1.415 GiB in 3.467 seconds | 20260508 12:20:02.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:20:02.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168014% / 0% / 7.54992e-05% | 20260508 12:20:02.681 | INFO | Baz IO overall: | 20260508 12:20:02.681 | INFO | BytesWritten : 3457327873542 | 20260508 12:20:02.681 | INFO | NumEvents : 1659261450739 | 20260508 12:20:04.257 | INFO | Analyzing chunk frames = [8056832,8057344) | 20260508 12:20:04.257 | INFO | MemInfo: { | 20260508 12:20:04.257 | INFO | "gpu" : " 17 GB", | 20260508 12:20:04.257 | INFO | "huge" : "", | 20260508 12:20:04.257 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:04.257 | INFO | "malloc" : " 14 GB", | 20260508 12:20:04.257 | INFO | "pinned" : " 29 GB" | 20260508 12:20:04.257 | INFO | } | 20260508 12:20:04.257 | INFO | Perf report for the last 5.1404 seconds: | 20260508 12:20:04.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:04.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:04.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:04.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:04.257 | INFO | ANALYSIS : 67.3%, 3.0, 105.1ms, 0 | 20260508 12:20:04.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:04.257 | INFO | RT_METRICS : 38.4%, 1.0, 20.1ms, 0 | 20260508 12:20:04.257 | INFO | PRE_HQ : 65.2%, 1.6, 53.9ms, 0 | 20260508 12:20:04.257 | INFO | BAZWRITER : 66.7%, 2.5, 85.1ms, 0 | 20260508 12:20:04.257 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:20:04.257 | INFO | Uploaded 36GiB to GPU in 2.61259s (13.7794GiB/s) | 20260508 12:20:04.257 | INFO | Downloaded 38GiB to GPU in 2.72266s (13.9569GiB/s) | 20260508 12:20:09.393 | INFO | Analyzing chunk frames = [8057344,8057856) | 20260508 12:20:09.393 | INFO | MemInfo: { | 20260508 12:20:09.393 | INFO | "gpu" : " 17 GB", | 20260508 12:20:09.393 | INFO | "huge" : "", | 20260508 12:20:09.393 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:09.393 | INFO | "malloc" : " 14 GB", | 20260508 12:20:09.393 | INFO | "pinned" : " 29 GB" | 20260508 12:20:09.393 | INFO | } | 20260508 12:20:09.393 | INFO | Perf report for the last 5.13623 seconds: | 20260508 12:20:09.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:09.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.393 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:20:09.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.393 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:20:09.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:09.394 | INFO | Uploaded 36GiB to GPU in 2.52252s (14.2714GiB/s) | 20260508 12:20:09.394 | INFO | Downloaded 34GiB to GPU in 2.45908s (13.8263GiB/s) | 20260508 12:20:14.528 | INFO | Analyzing chunk frames = [8057856,8058368) | 20260508 12:20:14.528 | INFO | MemInfo: { | 20260508 12:20:14.528 | INFO | "gpu" : " 17 GB", | 20260508 12:20:14.528 | INFO | "huge" : "", | 20260508 12:20:14.528 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:14.528 | INFO | "malloc" : " 14 GB", | 20260508 12:20:14.529 | INFO | "pinned" : " 29 GB" | 20260508 12:20:14.529 | INFO | } | 20260508 12:20:14.529 | INFO | Perf report for the last 5.13523 seconds: | 20260508 12:20:14.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:14.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 12:20:14.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:20:14.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:14.529 | INFO | Uploaded 36GiB to GPU in 2.49886s (14.4066GiB/s) | 20260508 12:20:14.529 | INFO | Downloaded 34GiB to GPU in 2.44967s (13.8794GiB/s) | 20260508 12:20:19.672 | INFO | Analyzing chunk frames = [8058368,8058880) | 20260508 12:20:19.672 | INFO | MemInfo: { | 20260508 12:20:19.672 | INFO | "gpu" : " 17 GB", | 20260508 12:20:19.672 | INFO | "huge" : "", | 20260508 12:20:19.672 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:19.672 | INFO | "malloc" : " 14 GB", | 20260508 12:20:19.672 | INFO | "pinned" : " 29 GB" | 20260508 12:20:19.672 | INFO | } | 20260508 12:20:19.672 | INFO | Perf report for the last 5.14378 seconds: | 20260508 12:20:19.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:19.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.673 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 12:20:19.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.673 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:20:19.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:19.673 | INFO | Uploaded 36GiB to GPU in 2.48817s (14.4684GiB/s) | 20260508 12:20:19.673 | INFO | Downloaded 34GiB to GPU in 2.44684s (13.8955GiB/s) | 20260508 12:20:24.811 | INFO | Analyzing chunk frames = [8058880,8059392) | 20260508 12:20:24.811 | INFO | MemInfo: { | 20260508 12:20:24.811 | INFO | "gpu" : " 17 GB", | 20260508 12:20:24.811 | INFO | "huge" : "", | 20260508 12:20:24.811 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:24.811 | INFO | "malloc" : " 14 GB", | 20260508 12:20:24.811 | INFO | "pinned" : " 29 GB" | 20260508 12:20:24.811 | INFO | } | 20260508 12:20:24.812 | INFO | Perf report for the last 5.13913 seconds: | 20260508 12:20:24.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:24.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 12:20:24.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 12:20:24.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:24.812 | INFO | Uploaded 36GiB to GPU in 2.41965s (14.8782GiB/s) | 20260508 12:20:24.812 | INFO | Downloaded 34GiB to GPU in 2.51289s (13.5302GiB/s) | 20260508 12:20:29.953 | INFO | Analyzing chunk frames = [8059392,8059904) | 20260508 12:20:29.953 | INFO | MemInfo: { | 20260508 12:20:29.953 | INFO | "gpu" : " 17 GB", | 20260508 12:20:29.953 | INFO | "huge" : "", | 20260508 12:20:29.953 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:29.953 | INFO | "malloc" : " 14 GB", | 20260508 12:20:29.953 | INFO | "pinned" : " 29 GB" | 20260508 12:20:29.953 | INFO | } | 20260508 12:20:29.954 | INFO | Perf report for the last 5.14198 seconds: | 20260508 12:20:29.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:29.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:20:29.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:20:29.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:29.954 | INFO | Uploaded 36GiB to GPU in 2.39013s (15.062GiB/s) | 20260508 12:20:29.954 | INFO | Downloaded 34GiB to GPU in 2.41137s (14.0999GiB/s) | 20260508 12:20:35.094 | INFO | Analyzing chunk frames = [8059904,8060416) | 20260508 12:20:35.094 | INFO | MemInfo: { | 20260508 12:20:35.094 | INFO | "gpu" : " 17 GB", | 20260508 12:20:35.094 | INFO | "huge" : "", | 20260508 12:20:35.094 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:35.094 | INFO | "malloc" : " 14 GB", | 20260508 12:20:35.094 | INFO | "pinned" : " 29 GB" | 20260508 12:20:35.094 | INFO | } | 20260508 12:20:35.094 | INFO | Perf report for the last 5.14056 seconds: | 20260508 12:20:35.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:35.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 12:20:35.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:20:35.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:35.094 | INFO | Uploaded 36GiB to GPU in 2.46652s (14.5955GiB/s) | 20260508 12:20:35.094 | INFO | Downloaded 34GiB to GPU in 2.44157s (13.9255GiB/s) | 20260508 12:20:40.225 | INFO | Analyzing chunk frames = [8060416,8060928) | 20260508 12:20:40.225 | INFO | MemInfo: { | 20260508 12:20:40.225 | INFO | "gpu" : " 17 GB", | 20260508 12:20:40.225 | INFO | "huge" : "", | 20260508 12:20:40.225 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:40.225 | INFO | "malloc" : " 14 GB", | 20260508 12:20:40.225 | INFO | "pinned" : " 29 GB" | 20260508 12:20:40.225 | INFO | } | 20260508 12:20:40.225 | INFO | Perf report for the last 5.13077 seconds: | 20260508 12:20:40.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:40.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260508 12:20:40.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:20:40.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:40.225 | INFO | Uploaded 36GiB to GPU in 2.39626s (15.0234GiB/s) | 20260508 12:20:40.225 | INFO | Downloaded 34GiB to GPU in 2.42532s (14.0187GiB/s) | 20260508 12:20:40.226 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:20:40.226 | INFO | Total mem usage: 16.7 GiB | 20260508 12:20:40.226 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:20:40.226 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:20:40.226 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:20:40.226 | INFO | High Water Breakdown: | 20260508 12:20:40.226 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:20:40.226 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:20:40.226 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:20:40.226 | INFO | ------------------------------------------ | 20260508 12:20:40.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:20:40.226 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:20:40.226 | INFO | Total mem usage: 26.5 GiB | 20260508 12:20:40.226 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:20:40.226 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:20:40.226 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:20:40.226 | INFO | High Water Breakdown: | 20260508 12:20:40.226 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:20:40.226 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:20:40.226 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:20:40.226 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:20:40.226 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:20:40.226 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:20:40.226 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:20:40.226 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:20:40.226 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:20:40.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:20:40.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:20:40.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:20:40.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:20:40.226 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:20:40.226 | INFO | ------------------------------------------ | 20260508 12:20:40.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:20:40.226 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:20:40.226 | INFO | Total mem usage: 27.8 GiB | 20260508 12:20:40.226 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:20:40.226 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:20:40.226 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:20:40.226 | INFO | High Water Breakdown: | 20260508 12:20:40.226 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:20:40.226 | INFO | ------------------------------------------ | 20260508 12:20:40.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:20:40.226 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:20:40.226 | INFO | Total mem usage: 15.7 GiB | 20260508 12:20:40.226 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:20:40.226 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:20:40.226 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:20:40.226 | INFO | High Water Breakdown: | 20260508 12:20:40.226 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:20:40.226 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:20:40.226 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:20:40.226 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:20:40.226 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:20:40.226 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:20:40.226 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:20:40.226 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:20:40.226 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:20:40.226 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:20:40.226 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:20:40.226 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:20:40.226 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:20:40.226 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:20:40.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:20:40.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:20:40.226 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:20:40.226 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:20:40.226 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:20:40.226 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:20:40.226 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:20:40.226 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:20:40.227 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:20:40.227 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:20:40.227 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:20:40.227 | INFO | ------------------------------------------ | 20260508 12:20:40.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:20:40.227 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:20:40.227 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:20:40.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:20:40.227 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.39e+01 / 4.68e-01 | 20260508 12:20:40.227 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.06e-01, 1.15e-02 / 4.79e-03 | 20260508 12:20:40.227 | INFO | PerfReport: 12.16%, Baseline -- 3.96e+00 / 4.36e+00, 5.89e-01 / 2.83e-01 | 20260508 12:20:40.227 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.49e+00, 2.10e-01 / 8.58e-02 | 20260508 12:20:40.227 | INFO | PerfReport: 16.00%, DME -- 4.17e+01 / 4.46e+01, 3.52e+00 / 8.22e-01 | 20260508 12:20:40.227 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.77e+00 / 1.36e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.46e+00, 4.07e-01 / 2.48e-01 | 20260508 12:20:40.227 | INFO | PerfReport: 2.97%, Metrics -- 9.68e-01 / 1.16e+00, 2.43e-01 / 1.40e-01 | 20260508 12:20:40.227 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.92e-01 / 3.35e-01, 4.73e-02 / 2.34e-02 | 20260508 12:20:40.227 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:20:40.227 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:20:40.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:20:40.227 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 24.55%, Upload -- 2.58e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 30.99%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 2.19%, LockUpload -- 2.31e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 2.53%, LockDownload -- 2.66e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 14.99%, LockCompute -- 1.58e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:20:40.227 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 12:20:40.323 | INFO | Starting a round of Baz IO | 20260508 12:20:43.724 | INFO | Populating RTMetrics report | 20260508 12:20:43.725 | INFO | RTMetrics report populated | 20260508 12:20:43.725 | INFO | Opening tmp RTMetrics json file | 20260508 12:20:43.737 | INFO | Serializing RTMetrics to json | 20260508 12:20:43.739 | INFO | Serialization done, swapping tmp file now... | 20260508 12:20:43.739 | INFO | RTMetrics done | 20260508 12:20:43.794 | INFO | Finished a round of Baz IO: Wrote 1.412 GiB in 3.471 seconds | 20260508 12:20:43.794 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:20:43.794 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166873% / 0% / 5.96046e-05% | 20260508 12:20:43.794 | INFO | Baz IO overall: | 20260508 12:20:43.794 | INFO | BytesWritten : 3458843882544 | 20260508 12:20:43.794 | INFO | NumEvents : 1659972622721 | 20260508 12:20:45.371 | INFO | Analyzing chunk frames = [8060928,8061440) | 20260508 12:20:45.371 | INFO | MemInfo: { | 20260508 12:20:45.371 | INFO | "gpu" : " 17 GB", | 20260508 12:20:45.371 | INFO | "huge" : "", | 20260508 12:20:45.371 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:45.371 | INFO | "malloc" : " 14 GB", | 20260508 12:20:45.371 | INFO | "pinned" : " 29 GB" | 20260508 12:20:45.371 | INFO | } | 20260508 12:20:45.371 | INFO | Perf report for the last 5.14605 seconds: | 20260508 12:20:45.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:45.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:45.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:45.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:45.371 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:20:45.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:45.371 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:20:45.371 | INFO | PRE_HQ : 65.3%, 1.6, 54.5ms, 0 | 20260508 12:20:45.371 | INFO | BAZWRITER : 66.7%, 2.5, 85.0ms, 0 | 20260508 12:20:45.371 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:20:45.371 | INFO | Uploaded 36GiB to GPU in 2.6073s (13.8074GiB/s) | 20260508 12:20:45.371 | INFO | Downloaded 38GiB to GPU in 2.68536s (14.1508GiB/s) | 20260508 12:20:50.507 | INFO | Analyzing chunk frames = [8061440,8061952) | 20260508 12:20:50.507 | INFO | MemInfo: { | 20260508 12:20:50.507 | INFO | "gpu" : " 17 GB", | 20260508 12:20:50.507 | INFO | "huge" : "", | 20260508 12:20:50.507 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:50.507 | INFO | "malloc" : " 14 GB", | 20260508 12:20:50.507 | INFO | "pinned" : " 29 GB" | 20260508 12:20:50.507 | INFO | } | 20260508 12:20:50.507 | INFO | Perf report for the last 5.13615 seconds: | 20260508 12:20:50.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:50.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:20:50.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:20:50.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:50.507 | INFO | Uploaded 36GiB to GPU in 2.47553s (14.5423GiB/s) | 20260508 12:20:50.507 | INFO | Downloaded 34GiB to GPU in 2.45884s (13.8277GiB/s) | 20260508 12:20:55.648 | INFO | Analyzing chunk frames = [8061952,8062464) | 20260508 12:20:55.648 | INFO | MemInfo: { | 20260508 12:20:55.648 | INFO | "gpu" : " 17 GB", | 20260508 12:20:55.648 | INFO | "huge" : "", | 20260508 12:20:55.648 | INFO | "huge_shared" : " 29 GB", | 20260508 12:20:55.648 | INFO | "malloc" : " 14 GB", | 20260508 12:20:55.648 | INFO | "pinned" : " 29 GB" | 20260508 12:20:55.648 | INFO | } | 20260508 12:20:55.648 | INFO | Perf report for the last 5.14112 seconds: | 20260508 12:20:55.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:20:55.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.648 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 12:20:55.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.648 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:20:55.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:20:55.649 | INFO | Uploaded 36GiB to GPU in 2.54077s (14.1689GiB/s) | 20260508 12:20:55.649 | INFO | Downloaded 34GiB to GPU in 2.45075s (13.8733GiB/s) | 20260508 12:21:00.788 | INFO | Analyzing chunk frames = [8062464,8062976) | 20260508 12:21:00.788 | INFO | MemInfo: { | 20260508 12:21:00.788 | INFO | "gpu" : " 17 GB", | 20260508 12:21:00.788 | INFO | "huge" : "", | 20260508 12:21:00.788 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:00.788 | INFO | "malloc" : " 14 GB", | 20260508 12:21:00.788 | INFO | "pinned" : " 29 GB" | 20260508 12:21:00.788 | INFO | } | 20260508 12:21:00.788 | INFO | Perf report for the last 5.13981 seconds: | 20260508 12:21:00.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:00.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 12:21:00.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:21:00.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:00.788 | INFO | Uploaded 36GiB to GPU in 2.54525s (14.144GiB/s) | 20260508 12:21:00.788 | INFO | Downloaded 34GiB to GPU in 2.44218s (13.922GiB/s) | 20260508 12:21:05.926 | INFO | Analyzing chunk frames = [8062976,8063488) | 20260508 12:21:05.926 | INFO | MemInfo: { | 20260508 12:21:05.926 | INFO | "gpu" : " 17 GB", | 20260508 12:21:05.926 | INFO | "huge" : "", | 20260508 12:21:05.926 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:05.926 | INFO | "malloc" : " 14 GB", | 20260508 12:21:05.926 | INFO | "pinned" : " 29 GB" | 20260508 12:21:05.926 | INFO | } | 20260508 12:21:05.926 | INFO | Perf report for the last 5.13831 seconds: | 20260508 12:21:05.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:05.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.926 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:21:05.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.926 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 12:21:05.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:05.927 | INFO | Uploaded 36GiB to GPU in 2.37092s (15.184GiB/s) | 20260508 12:21:05.927 | INFO | Downloaded 34GiB to GPU in 2.50382s (13.5793GiB/s) | 20260508 12:21:11.067 | INFO | Analyzing chunk frames = [8063488,8064000) | 20260508 12:21:11.067 | INFO | MemInfo: { | 20260508 12:21:11.067 | INFO | "gpu" : " 17 GB", | 20260508 12:21:11.067 | INFO | "huge" : "", | 20260508 12:21:11.067 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:11.067 | INFO | "malloc" : " 14 GB", | 20260508 12:21:11.067 | INFO | "pinned" : " 29 GB" | 20260508 12:21:11.067 | INFO | } | 20260508 12:21:11.067 | INFO | Perf report for the last 5.14103 seconds: | 20260508 12:21:11.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:11.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.067 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 12:21:11.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.068 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:21:11.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:11.068 | INFO | Uploaded 36GiB to GPU in 2.38038s (15.1236GiB/s) | 20260508 12:21:11.068 | INFO | Downloaded 34GiB to GPU in 2.42623s (14.0135GiB/s) | 20260508 12:21:16.204 | INFO | Analyzing chunk frames = [8064000,8064512) | 20260508 12:21:16.204 | INFO | MemInfo: { | 20260508 12:21:16.204 | INFO | "gpu" : " 17 GB", | 20260508 12:21:16.204 | INFO | "huge" : "", | 20260508 12:21:16.204 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:16.204 | INFO | "malloc" : " 14 GB", | 20260508 12:21:16.204 | INFO | "pinned" : " 29 GB" | 20260508 12:21:16.204 | INFO | } | 20260508 12:21:16.204 | INFO | Perf report for the last 5.13679 seconds: | 20260508 12:21:16.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:16.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:21:16.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:21:16.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:16.204 | INFO | Uploaded 36GiB to GPU in 2.47363s (14.5535GiB/s) | 20260508 12:21:16.205 | INFO | Downloaded 34GiB to GPU in 2.4614s (13.8133GiB/s) | 20260508 12:21:21.342 | INFO | Analyzing chunk frames = [8064512,8065024) | 20260508 12:21:21.342 | INFO | MemInfo: { | 20260508 12:21:21.342 | INFO | "gpu" : " 17 GB", | 20260508 12:21:21.342 | INFO | "huge" : "", | 20260508 12:21:21.342 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:21.342 | INFO | "malloc" : " 14 GB", | 20260508 12:21:21.342 | INFO | "pinned" : " 29 GB" | 20260508 12:21:21.342 | INFO | } | 20260508 12:21:21.342 | INFO | Perf report for the last 5.13768 seconds: | 20260508 12:21:21.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:21.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 12:21:21.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:21:21.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:21.342 | INFO | Uploaded 36GiB to GPU in 2.46947s (14.578GiB/s) | 20260508 12:21:21.342 | INFO | Downloaded 34GiB to GPU in 2.4655s (13.7903GiB/s) | 20260508 12:21:21.439 | INFO | Starting a round of Baz IO | 20260508 12:21:24.853 | INFO | Populating RTMetrics report | 20260508 12:21:24.854 | INFO | RTMetrics report populated | 20260508 12:21:24.854 | INFO | Opening tmp RTMetrics json file | 20260508 12:21:24.859 | INFO | Serializing RTMetrics to json | 20260508 12:21:24.860 | INFO | Serialization done, swapping tmp file now... | 20260508 12:21:24.861 | INFO | RTMetrics done | 20260508 12:21:24.919 | INFO | Finished a round of Baz IO: Wrote 1.413 GiB in 3.48 seconds | 20260508 12:21:24.919 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:21:24.919 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166953% / 3.97364e-06% / 5.96046e-05% | 20260508 12:21:24.919 | INFO | Baz IO overall: | 20260508 12:21:24.919 | INFO | BytesWritten : 3460360941821 | 20260508 12:21:24.919 | INFO | NumEvents : 1660684603930 | 20260508 12:21:26.478 | INFO | Analyzing chunk frames = [8065024,8065536) | 20260508 12:21:26.478 | INFO | MemInfo: { | 20260508 12:21:26.478 | INFO | "gpu" : " 17 GB", | 20260508 12:21:26.478 | INFO | "huge" : "", | 20260508 12:21:26.478 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:26.478 | INFO | "malloc" : " 14 GB", | 20260508 12:21:26.478 | INFO | "pinned" : " 29 GB" | 20260508 12:21:26.478 | INFO | } | 20260508 12:21:26.479 | INFO | Perf report for the last 5.13656 seconds: | 20260508 12:21:26.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:26.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:26.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:26.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:26.479 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 12:21:26.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:26.479 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:21:26.479 | INFO | PRE_HQ : 65.4%, 1.6, 53.8ms, 0 | 20260508 12:21:26.479 | INFO | BAZWRITER : 67.0%, 2.5, 84.8ms, 0 | 20260508 12:21:26.479 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:21:26.479 | INFO | Uploaded 36GiB to GPU in 2.60051s (13.8434GiB/s) | 20260508 12:21:26.479 | INFO | Downloaded 38GiB to GPU in 2.70737s (14.0358GiB/s) | 20260508 12:21:31.622 | INFO | Analyzing chunk frames = [8065536,8066048) | 20260508 12:21:31.622 | INFO | MemInfo: { | 20260508 12:21:31.622 | INFO | "gpu" : " 17 GB", | 20260508 12:21:31.622 | INFO | "huge" : "", | 20260508 12:21:31.622 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:31.622 | INFO | "malloc" : " 14 GB", | 20260508 12:21:31.622 | INFO | "pinned" : " 29 GB" | 20260508 12:21:31.622 | INFO | } | 20260508 12:21:31.622 | INFO | Perf report for the last 5.14365 seconds: | 20260508 12:21:31.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:31.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.622 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:21:31.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.622 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 12:21:31.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:31.623 | INFO | Uploaded 36GiB to GPU in 2.46643s (14.596GiB/s) | 20260508 12:21:31.623 | INFO | Downloaded 34GiB to GPU in 2.46871s (13.7724GiB/s) | 20260508 12:21:36.763 | INFO | Analyzing chunk frames = [8066048,8066560) | 20260508 12:21:36.763 | INFO | MemInfo: { | 20260508 12:21:36.763 | INFO | "gpu" : " 17 GB", | 20260508 12:21:36.763 | INFO | "huge" : "", | 20260508 12:21:36.763 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:36.763 | INFO | "malloc" : " 14 GB", | 20260508 12:21:36.763 | INFO | "pinned" : " 29 GB" | 20260508 12:21:36.763 | INFO | } | 20260508 12:21:36.763 | INFO | Perf report for the last 5.14116 seconds: | 20260508 12:21:36.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:36.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.763 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 12:21:36.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.763 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:21:36.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:36.764 | INFO | Uploaded 36GiB to GPU in 2.51322s (14.3243GiB/s) | 20260508 12:21:36.764 | INFO | Downloaded 34GiB to GPU in 2.4398s (13.9356GiB/s) | 20260508 12:21:41.908 | INFO | Analyzing chunk frames = [8066560,8067072) | 20260508 12:21:41.908 | INFO | MemInfo: { | 20260508 12:21:41.908 | INFO | "gpu" : " 17 GB", | 20260508 12:21:41.908 | INFO | "huge" : "", | 20260508 12:21:41.908 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:41.908 | INFO | "malloc" : " 14 GB", | 20260508 12:21:41.908 | INFO | "pinned" : " 29 GB" | 20260508 12:21:41.908 | INFO | } | 20260508 12:21:41.908 | INFO | Perf report for the last 5.14486 seconds: | 20260508 12:21:41.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:41.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:21:41.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:21:41.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:41.908 | INFO | Uploaded 36GiB to GPU in 2.47s (14.5749GiB/s) | 20260508 12:21:41.909 | INFO | Downloaded 34GiB to GPU in 2.45171s (13.8679GiB/s) | 20260508 12:21:47.041 | INFO | Analyzing chunk frames = [8067072,8067584) | 20260508 12:21:47.041 | INFO | MemInfo: { | 20260508 12:21:47.041 | INFO | "gpu" : " 17 GB", | 20260508 12:21:47.041 | INFO | "huge" : "", | 20260508 12:21:47.041 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:47.041 | INFO | "malloc" : " 14 GB", | 20260508 12:21:47.041 | INFO | "pinned" : " 29 GB" | 20260508 12:21:47.041 | INFO | } | 20260508 12:21:47.041 | INFO | Perf report for the last 5.13304 seconds: | 20260508 12:21:47.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:47.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.041 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:21:47.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.041 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:21:47.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:47.042 | INFO | Uploaded 36GiB to GPU in 2.40565s (14.9648GiB/s) | 20260508 12:21:47.042 | INFO | Downloaded 34GiB to GPU in 2.5111s (13.5399GiB/s) | 20260508 12:21:52.180 | INFO | Analyzing chunk frames = [8067584,8068096) | 20260508 12:21:52.180 | INFO | MemInfo: { | 20260508 12:21:52.180 | INFO | "gpu" : " 17 GB", | 20260508 12:21:52.180 | INFO | "huge" : "", | 20260508 12:21:52.180 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:52.180 | INFO | "malloc" : " 14 GB", | 20260508 12:21:52.180 | INFO | "pinned" : " 29 GB" | 20260508 12:21:52.180 | INFO | } | 20260508 12:21:52.180 | INFO | Perf report for the last 5.13889 seconds: | 20260508 12:21:52.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:52.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:21:52.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:21:52.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:52.180 | INFO | Uploaded 36GiB to GPU in 2.41406s (14.9127GiB/s) | 20260508 12:21:52.181 | INFO | Downloaded 34GiB to GPU in 2.42517s (14.0196GiB/s) | 20260508 12:21:57.318 | INFO | Analyzing chunk frames = [8068096,8068608) | 20260508 12:21:57.318 | INFO | MemInfo: { | 20260508 12:21:57.318 | INFO | "gpu" : " 17 GB", | 20260508 12:21:57.318 | INFO | "huge" : "", | 20260508 12:21:57.318 | INFO | "huge_shared" : " 29 GB", | 20260508 12:21:57.318 | INFO | "malloc" : " 14 GB", | 20260508 12:21:57.318 | INFO | "pinned" : " 29 GB" | 20260508 12:21:57.318 | INFO | } | 20260508 12:21:57.318 | INFO | Perf report for the last 5.13772 seconds: | 20260508 12:21:57.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:21:57.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:21:57.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:21:57.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:21:57.318 | INFO | Uploaded 36GiB to GPU in 2.47442s (14.5489GiB/s) | 20260508 12:21:57.318 | INFO | Downloaded 34GiB to GPU in 2.448s (13.8889GiB/s) | 20260508 12:22:02.457 | INFO | Analyzing chunk frames = [8068608,8069120) | 20260508 12:22:02.457 | INFO | MemInfo: { | 20260508 12:22:02.457 | INFO | "gpu" : " 17 GB", | 20260508 12:22:02.457 | INFO | "huge" : "", | 20260508 12:22:02.457 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:02.457 | INFO | "malloc" : " 14 GB", | 20260508 12:22:02.457 | INFO | "pinned" : " 29 GB" | 20260508 12:22:02.457 | INFO | } | 20260508 12:22:02.457 | INFO | Perf report for the last 5.1395 seconds: | 20260508 12:22:02.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:02.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.458 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 12:22:02.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.458 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:22:02.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:02.458 | INFO | Uploaded 36GiB to GPU in 2.47548s (14.5426GiB/s) | 20260508 12:22:02.458 | INFO | Downloaded 34GiB to GPU in 2.44436s (13.9096GiB/s) | 20260508 12:22:02.458 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:22:02.458 | INFO | Total mem usage: 16.7 GiB | 20260508 12:22:02.458 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:22:02.458 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:22:02.458 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:22:02.458 | INFO | High Water Breakdown: | 20260508 12:22:02.458 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:22:02.458 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:22:02.458 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:22:02.458 | INFO | ------------------------------------------ | 20260508 12:22:02.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:22:02.458 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:22:02.458 | INFO | Total mem usage: 26.5 GiB | 20260508 12:22:02.458 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:22:02.458 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:22:02.458 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:22:02.458 | INFO | High Water Breakdown: | 20260508 12:22:02.458 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:22:02.458 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:22:02.458 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:22:02.458 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:22:02.458 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:22:02.458 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:22:02.458 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:22:02.458 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:22:02.458 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:22:02.458 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:22:02.458 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:22:02.458 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:22:02.458 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:22:02.458 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:22:02.458 | INFO | ------------------------------------------ | 20260508 12:22:02.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:22:02.458 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:22:02.458 | INFO | Total mem usage: 27.8 GiB | 20260508 12:22:02.458 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:22:02.458 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:22:02.458 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:22:02.458 | INFO | High Water Breakdown: | 20260508 12:22:02.458 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:22:02.458 | INFO | ------------------------------------------ | 20260508 12:22:02.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:22:02.458 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:22:02.458 | INFO | Total mem usage: 15.7 GiB | 20260508 12:22:02.458 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:22:02.458 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:22:02.458 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:22:02.458 | INFO | High Water Breakdown: | 20260508 12:22:02.458 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:22:02.458 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:22:02.458 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:22:02.458 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:22:02.458 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:22:02.458 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:22:02.458 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:22:02.458 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:22:02.458 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:22:02.458 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:22:02.458 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:22:02.458 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:22:02.458 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:22:02.458 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:22:02.458 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:22:02.458 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:22:02.458 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:22:02.458 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:22:02.458 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:22:02.458 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:22:02.458 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:22:02.458 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:22:02.458 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:22:02.458 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:22:02.459 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:22:02.459 | INFO | ------------------------------------------ | 20260508 12:22:02.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:22:02.459 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:22:02.459 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:22:02.459 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:22:02.459 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.54e+01, 1.42e+01 / 6.37e-01 | 20260508 12:22:02.459 | INFO | PerfReport: 0.30%, AggregateStats -- 9.62e-02 / 1.04e-01, 1.10e-02 / 3.95e-03 | 20260508 12:22:02.459 | INFO | PerfReport: 12.18%, Baseline -- 3.96e+00 / 4.35e+00, 5.91e-01 / 2.82e-01 | 20260508 12:22:02.459 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.42e+00, 2.08e-01 / 1.20e-01 | 20260508 12:22:02.459 | INFO | PerfReport: 16.07%, DME -- 4.18e+01 / 4.55e+01, 3.71e+00 / 1.23e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.71e+00 / 1.33e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.33e+00 / 2.50e+00, 4.06e-01 / 2.42e-01 | 20260508 12:22:02.459 | INFO | PerfReport: 2.98%, Metrics -- 9.69e-01 / 1.09e+00, 2.41e-01 / 1.17e-01 | 20260508 12:22:02.459 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.18e-01, 4.77e-02 / 2.27e-02 | 20260508 12:22:02.459 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:22:02.459 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:22:02.459 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:22:02.459 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.40e+01 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 24.57%, Upload -- 2.58e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 24.63%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 30.93%, Compute -- 3.25e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 2.27%, LockUpload -- 2.39e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 2.54%, LockDownload -- 2.68e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 15.04%, LockCompute -- 1.58e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260508 12:22:02.459 | INFO | PerfReport: 0.01%, Misc -- 1.21e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260508 12:22:02.567 | INFO | Starting a round of Baz IO | 20260508 12:22:05.957 | INFO | Populating RTMetrics report | 20260508 12:22:05.958 | INFO | RTMetrics report populated | 20260508 12:22:05.958 | INFO | Opening tmp RTMetrics json file | 20260508 12:22:05.966 | INFO | Serializing RTMetrics to json | 20260508 12:22:05.967 | INFO | Serialization done, swapping tmp file now... | 20260508 12:22:05.967 | INFO | RTMetrics done | 20260508 12:22:06.027 | INFO | Finished a round of Baz IO: Wrote 1.412 GiB in 3.459 seconds | 20260508 12:22:06.027 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:22:06.027 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167112% / 3.97364e-06% / 8.34465e-05% | 20260508 12:22:06.027 | INFO | Baz IO overall: | 20260508 12:22:06.027 | INFO | BytesWritten : 3461877208666 | 20260508 12:22:06.027 | INFO | NumEvents : 1661396173560 | 20260508 12:22:07.601 | INFO | Analyzing chunk frames = [8069120,8069632) | 20260508 12:22:07.601 | INFO | MemInfo: { | 20260508 12:22:07.601 | INFO | "gpu" : " 17 GB", | 20260508 12:22:07.601 | INFO | "huge" : "", | 20260508 12:22:07.601 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:07.601 | INFO | "malloc" : " 14 GB", | 20260508 12:22:07.601 | INFO | "pinned" : " 29 GB" | 20260508 12:22:07.601 | INFO | } | 20260508 12:22:07.601 | INFO | Perf report for the last 5.14382 seconds: | 20260508 12:22:07.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:07.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:07.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:07.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:07.601 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:22:07.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:07.601 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 12:22:07.601 | INFO | PRE_HQ : 65.4%, 1.6, 55.5ms, 0 | 20260508 12:22:07.601 | INFO | BAZWRITER : 66.6%, 2.5, 85.3ms, 0 | 20260508 12:22:07.601 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:22:07.602 | INFO | Uploaded 36GiB to GPU in 2.64382s (13.6166GiB/s) | 20260508 12:22:07.602 | INFO | Downloaded 38GiB to GPU in 2.71415s (14.0007GiB/s) | 20260508 12:22:12.737 | INFO | Analyzing chunk frames = [8069632,8070144) | 20260508 12:22:12.737 | INFO | MemInfo: { | 20260508 12:22:12.738 | INFO | "gpu" : " 17 GB", | 20260508 12:22:12.738 | INFO | "huge" : "", | 20260508 12:22:12.738 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:12.738 | INFO | "malloc" : " 14 GB", | 20260508 12:22:12.738 | INFO | "pinned" : " 29 GB" | 20260508 12:22:12.738 | INFO | } | 20260508 12:22:12.738 | INFO | Perf report for the last 5.13649 seconds: | 20260508 12:22:12.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:12.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:22:12.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:22:12.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:12.738 | INFO | Uploaded 36GiB to GPU in 2.44708s (14.7114GiB/s) | 20260508 12:22:12.738 | INFO | Downloaded 34GiB to GPU in 2.46752s (13.779GiB/s) | 20260508 12:22:17.882 | INFO | Analyzing chunk frames = [8070144,8070656) | 20260508 12:22:17.882 | INFO | MemInfo: { | 20260508 12:22:17.882 | INFO | "gpu" : " 17 GB", | 20260508 12:22:17.882 | INFO | "huge" : "", | 20260508 12:22:17.883 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:17.883 | INFO | "malloc" : " 14 GB", | 20260508 12:22:17.883 | INFO | "pinned" : " 29 GB" | 20260508 12:22:17.883 | INFO | } | 20260508 12:22:17.883 | INFO | Perf report for the last 5.14492 seconds: | 20260508 12:22:17.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:17.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 12:22:17.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:22:17.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:17.883 | INFO | Uploaded 36GiB to GPU in 2.47919s (14.5209GiB/s) | 20260508 12:22:17.883 | INFO | Downloaded 34GiB to GPU in 2.45708s (13.8376GiB/s) | 20260508 12:22:23.021 | INFO | Analyzing chunk frames = [8070656,8071168) | 20260508 12:22:23.021 | INFO | MemInfo: { | 20260508 12:22:23.021 | INFO | "gpu" : " 17 GB", | 20260508 12:22:23.021 | INFO | "huge" : "", | 20260508 12:22:23.021 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:23.021 | INFO | "malloc" : " 14 GB", | 20260508 12:22:23.021 | INFO | "pinned" : " 29 GB" | 20260508 12:22:23.021 | INFO | } | 20260508 12:22:23.021 | INFO | Perf report for the last 5.13824 seconds: | 20260508 12:22:23.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:23.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:22:23.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:22:23.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:23.021 | INFO | Uploaded 36GiB to GPU in 2.52046s (14.2831GiB/s) | 20260508 12:22:23.021 | INFO | Downloaded 34GiB to GPU in 2.47511s (13.7367GiB/s) | 20260508 12:22:28.154 | INFO | Analyzing chunk frames = [8071168,8071680) | 20260508 12:22:28.154 | INFO | MemInfo: { | 20260508 12:22:28.154 | INFO | "gpu" : " 17 GB", | 20260508 12:22:28.154 | INFO | "huge" : "", | 20260508 12:22:28.154 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:28.154 | INFO | "malloc" : " 14 GB", | 20260508 12:22:28.154 | INFO | "pinned" : " 29 GB" | 20260508 12:22:28.154 | INFO | } | 20260508 12:22:28.154 | INFO | Perf report for the last 5.13331 seconds: | 20260508 12:22:28.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:28.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.154 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 12:22:28.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.154 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:22:28.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:28.155 | INFO | Uploaded 36GiB to GPU in 2.39474s (15.0329GiB/s) | 20260508 12:22:28.155 | INFO | Downloaded 34GiB to GPU in 2.51454s (13.5214GiB/s) | 20260508 12:22:33.290 | INFO | Analyzing chunk frames = [8071680,8072192) | 20260508 12:22:33.290 | INFO | MemInfo: { | 20260508 12:22:33.290 | INFO | "gpu" : " 17 GB", | 20260508 12:22:33.290 | INFO | "huge" : "", | 20260508 12:22:33.290 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:33.290 | INFO | "malloc" : " 14 GB", | 20260508 12:22:33.290 | INFO | "pinned" : " 29 GB" | 20260508 12:22:33.290 | INFO | } | 20260508 12:22:33.290 | INFO | Perf report for the last 5.13559 seconds: | 20260508 12:22:33.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:33.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:22:33.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:22:33.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:33.290 | INFO | Uploaded 36GiB to GPU in 2.36807s (15.2022GiB/s) | 20260508 12:22:33.290 | INFO | Downloaded 34GiB to GPU in 2.45142s (13.8695GiB/s) | 20260508 12:22:38.434 | INFO | Analyzing chunk frames = [8072192,8072704) | 20260508 12:22:38.434 | INFO | MemInfo: { | 20260508 12:22:38.434 | INFO | "gpu" : " 17 GB", | 20260508 12:22:38.434 | INFO | "huge" : "", | 20260508 12:22:38.435 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:38.435 | INFO | "malloc" : " 14 GB", | 20260508 12:22:38.435 | INFO | "pinned" : " 29 GB" | 20260508 12:22:38.435 | INFO | } | 20260508 12:22:38.435 | INFO | Perf report for the last 5.14494 seconds: | 20260508 12:22:38.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:38.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | ANALYSIS : 65.1%, 3.0, 101.6ms, 0 | 20260508 12:22:38.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 12:22:38.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:38.435 | INFO | Uploaded 36GiB to GPU in 2.38822s (15.074GiB/s) | 20260508 12:22:38.435 | INFO | Downloaded 34GiB to GPU in 2.45543s (13.8469GiB/s) | 20260508 12:22:43.575 | INFO | Analyzing chunk frames = [8072704,8073216) | 20260508 12:22:43.575 | INFO | MemInfo: { | 20260508 12:22:43.575 | INFO | "gpu" : " 17 GB", | 20260508 12:22:43.575 | INFO | "huge" : "", | 20260508 12:22:43.575 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:43.575 | INFO | "malloc" : " 14 GB", | 20260508 12:22:43.575 | INFO | "pinned" : " 29 GB" | 20260508 12:22:43.575 | INFO | } | 20260508 12:22:43.575 | INFO | Perf report for the last 5.14058 seconds: | 20260508 12:22:43.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:43.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.576 | INFO | ANALYSIS : 65.7%, 3.0, 102.5ms, 0 | 20260508 12:22:43.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.576 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 12:22:43.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:43.576 | INFO | Uploaded 36GiB to GPU in 2.41572s (14.9024GiB/s) | 20260508 12:22:43.576 | INFO | Downloaded 34GiB to GPU in 2.47337s (13.7464GiB/s) | 20260508 12:22:43.689 | INFO | Starting a round of Baz IO | 20260508 12:22:47.084 | INFO | Populating RTMetrics report | 20260508 12:22:47.084 | INFO | RTMetrics report populated | 20260508 12:22:47.084 | INFO | Opening tmp RTMetrics json file | 20260508 12:22:47.085 | INFO | Serializing RTMetrics to json | 20260508 12:22:47.097 | INFO | Serialization done, swapping tmp file now... | 20260508 12:22:47.100 | INFO | RTMetrics done | 20260508 12:22:47.150 | INFO | Finished a round of Baz IO: Wrote 1.413 GiB in 3.461 seconds | 20260508 12:22:47.150 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:22:47.150 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166686% / 3.97364e-06% / 8.34465e-05% | 20260508 12:22:47.150 | INFO | Baz IO overall: | 20260508 12:22:47.150 | INFO | BytesWritten : 3463394136965 | 20260508 12:22:47.150 | INFO | NumEvents : 1662108173149 | 20260508 12:22:48.715 | INFO | Analyzing chunk frames = [8073216,8073728) | 20260508 12:22:48.715 | INFO | MemInfo: { | 20260508 12:22:48.715 | INFO | "gpu" : " 17 GB", | 20260508 12:22:48.715 | INFO | "huge" : "", | 20260508 12:22:48.716 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:48.716 | INFO | "malloc" : " 14 GB", | 20260508 12:22:48.716 | INFO | "pinned" : " 29 GB" | 20260508 12:22:48.716 | INFO | } | 20260508 12:22:48.716 | INFO | Perf report for the last 5.14019 seconds: | 20260508 12:22:48.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:48.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:48.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:48.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:48.716 | INFO | ANALYSIS : 67.7%, 3.0, 105.2ms, 0 | 20260508 12:22:48.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:48.716 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 12:22:48.716 | INFO | PRE_HQ : 66.2%, 1.8, 61.3ms, 0 | 20260508 12:22:48.716 | INFO | BAZWRITER : 66.6%, 2.7, 92.0ms, 0 | 20260508 12:22:48.716 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 12:22:48.716 | INFO | Uploaded 36GiB to GPU in 2.5904s (13.8975GiB/s) | 20260508 12:22:48.716 | INFO | Downloaded 38GiB to GPU in 2.72643s (13.9377GiB/s) | 20260508 12:22:53.854 | INFO | Analyzing chunk frames = [8073728,8074240) | 20260508 12:22:53.854 | INFO | MemInfo: { | 20260508 12:22:53.854 | INFO | "gpu" : " 17 GB", | 20260508 12:22:53.854 | INFO | "huge" : "", | 20260508 12:22:53.854 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:53.854 | INFO | "malloc" : " 14 GB", | 20260508 12:22:53.854 | INFO | "pinned" : " 29 GB" | 20260508 12:22:53.854 | INFO | } | 20260508 12:22:53.854 | INFO | Perf report for the last 5.13875 seconds: | 20260508 12:22:53.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:53.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.854 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:22:53.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.854 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:22:53.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:53.855 | INFO | Uploaded 36GiB to GPU in 2.45413s (14.6692GiB/s) | 20260508 12:22:53.855 | INFO | Downloaded 34GiB to GPU in 2.46334s (13.8024GiB/s) | 20260508 12:22:58.988 | INFO | Analyzing chunk frames = [8074240,8074752) | 20260508 12:22:58.988 | INFO | MemInfo: { | 20260508 12:22:58.988 | INFO | "gpu" : " 17 GB", | 20260508 12:22:58.988 | INFO | "huge" : "", | 20260508 12:22:58.988 | INFO | "huge_shared" : " 29 GB", | 20260508 12:22:58.988 | INFO | "malloc" : " 14 GB", | 20260508 12:22:58.988 | INFO | "pinned" : " 29 GB" | 20260508 12:22:58.988 | INFO | } | 20260508 12:22:58.988 | INFO | Perf report for the last 5.13393 seconds: | 20260508 12:22:58.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:22:58.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.988 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 12:22:58.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.988 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:22:58.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:22:58.989 | INFO | Uploaded 36GiB to GPU in 2.5379s (14.1849GiB/s) | 20260508 12:22:58.989 | INFO | Downloaded 34GiB to GPU in 2.46375s (13.8001GiB/s) | 20260508 12:23:04.133 | INFO | Analyzing chunk frames = [8074752,8075264) | 20260508 12:23:04.133 | INFO | MemInfo: { | 20260508 12:23:04.133 | INFO | "gpu" : " 17 GB", | 20260508 12:23:04.133 | INFO | "huge" : "", | 20260508 12:23:04.133 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:04.133 | INFO | "malloc" : " 14 GB", | 20260508 12:23:04.133 | INFO | "pinned" : " 29 GB" | 20260508 12:23:04.133 | INFO | } | 20260508 12:23:04.133 | INFO | Perf report for the last 5.1451 seconds: | 20260508 12:23:04.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:04.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.133 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 12:23:04.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.134 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:23:04.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:04.134 | INFO | Uploaded 36GiB to GPU in 2.54658s (14.1366GiB/s) | 20260508 12:23:04.134 | INFO | Downloaded 34GiB to GPU in 2.45334s (13.8587GiB/s) | 20260508 12:23:09.270 | INFO | Analyzing chunk frames = [8075264,8075776) | 20260508 12:23:09.270 | INFO | MemInfo: { | 20260508 12:23:09.270 | INFO | "gpu" : " 17 GB", | 20260508 12:23:09.270 | INFO | "huge" : "", | 20260508 12:23:09.270 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:09.270 | INFO | "malloc" : " 14 GB", | 20260508 12:23:09.270 | INFO | "pinned" : " 29 GB" | 20260508 12:23:09.270 | INFO | } | 20260508 12:23:09.270 | INFO | Perf report for the last 5.13672 seconds: | 20260508 12:23:09.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:09.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.270 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 12:23:09.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.270 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 12:23:09.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:09.271 | INFO | Uploaded 36GiB to GPU in 2.40814s (14.9493GiB/s) | 20260508 12:23:09.271 | INFO | Downloaded 34GiB to GPU in 2.49535s (13.6253GiB/s) | 20260508 12:23:14.412 | INFO | Analyzing chunk frames = [8075776,8076288) | 20260508 12:23:14.412 | INFO | MemInfo: { | 20260508 12:23:14.412 | INFO | "gpu" : " 17 GB", | 20260508 12:23:14.412 | INFO | "huge" : "", | 20260508 12:23:14.412 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:14.412 | INFO | "malloc" : " 14 GB", | 20260508 12:23:14.412 | INFO | "pinned" : " 29 GB" | 20260508 12:23:14.412 | INFO | } | 20260508 12:23:14.412 | INFO | Perf report for the last 5.1416 seconds: | 20260508 12:23:14.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:14.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 12:23:14.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:23:14.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:14.412 | INFO | Uploaded 36GiB to GPU in 2.42481s (14.8465GiB/s) | 20260508 12:23:14.412 | INFO | Downloaded 34GiB to GPU in 2.43134s (13.984GiB/s) | 20260508 12:23:19.550 | INFO | Analyzing chunk frames = [8076288,8076800) | 20260508 12:23:19.550 | INFO | MemInfo: { | 20260508 12:23:19.550 | INFO | "gpu" : " 17 GB", | 20260508 12:23:19.550 | INFO | "huge" : "", | 20260508 12:23:19.550 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:19.550 | INFO | "malloc" : " 14 GB", | 20260508 12:23:19.550 | INFO | "pinned" : " 29 GB" | 20260508 12:23:19.550 | INFO | } | 20260508 12:23:19.550 | INFO | Perf report for the last 5.13824 seconds: | 20260508 12:23:19.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:19.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.550 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 12:23:19.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.550 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 12:23:19.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:19.551 | INFO | Uploaded 36GiB to GPU in 2.4758s (14.5408GiB/s) | 20260508 12:23:19.551 | INFO | Downloaded 34GiB to GPU in 2.44561s (13.9025GiB/s) | 20260508 12:23:24.695 | INFO | Analyzing chunk frames = [8076800,8077312) | 20260508 12:23:24.695 | INFO | MemInfo: { | 20260508 12:23:24.695 | INFO | "gpu" : " 17 GB", | 20260508 12:23:24.695 | INFO | "huge" : "", | 20260508 12:23:24.695 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:24.695 | INFO | "malloc" : " 14 GB", | 20260508 12:23:24.695 | INFO | "pinned" : " 29 GB" | 20260508 12:23:24.695 | INFO | } | 20260508 12:23:24.695 | INFO | Perf report for the last 5.14482 seconds: | 20260508 12:23:24.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:24.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | ANALYSIS : 64.6%, 3.0, 101.1ms, 0 | 20260508 12:23:24.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:23:24.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:24.695 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:23:24.695 | INFO | Total mem usage: 16.7 GiB | 20260508 12:23:24.695 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:23:24.695 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:23:24.695 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:23:24.696 | INFO | Uploaded 36GiB to GPU in 2.38379s (15.102GiB/s) | 20260508 12:23:24.696 | INFO | High Water Breakdown: | 20260508 12:23:24.696 | INFO | Downloaded 34GiB to GPU in 2.42286s (14.033GiB/s) | 20260508 12:23:24.696 | INFO | 59.6% : 8.55 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:23:24.696 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:23:24.696 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:23:24.696 | INFO | ------------------------------------------ | 20260508 12:23:24.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:23:24.696 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:23:24.696 | INFO | Total mem usage: 26.5 GiB | 20260508 12:23:24.696 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:23:24.696 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 12:23:24.696 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 12:23:24.696 | INFO | High Water Breakdown: | 20260508 12:23:24.696 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:23:24.696 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:23:24.696 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:23:24.696 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:23:24.696 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:23:24.696 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:23:24.696 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:23:24.696 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:23:24.696 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:23:24.696 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:23:24.696 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:23:24.696 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:23:24.696 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:23:24.696 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:23:24.696 | INFO | ------------------------------------------ | 20260508 12:23:24.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:23:24.696 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:23:24.696 | INFO | Total mem usage: 27.8 GiB | 20260508 12:23:24.696 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:23:24.696 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:23:24.696 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:23:24.696 | INFO | High Water Breakdown: | 20260508 12:23:24.696 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:23:24.696 | INFO | ------------------------------------------ | 20260508 12:23:24.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:23:24.696 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:23:24.696 | INFO | Total mem usage: 15.7 GiB | 20260508 12:23:24.696 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:23:24.696 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:23:24.696 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:23:24.696 | INFO | High Water Breakdown: | 20260508 12:23:24.696 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:23:24.696 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:23:24.696 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:23:24.696 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:23:24.696 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:23:24.696 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:23:24.696 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:23:24.696 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:23:24.696 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:23:24.696 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:23:24.696 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:23:24.696 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:23:24.696 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:23:24.696 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:23:24.696 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:23:24.696 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:23:24.696 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:23:24.696 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:23:24.696 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:23:24.696 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:23:24.696 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:23:24.696 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:23:24.696 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:23:24.696 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:23:24.696 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:23:24.696 | INFO | ------------------------------------------ | 20260508 12:23:24.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:23:24.696 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:23:24.696 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:23:24.696 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:23:24.696 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.40e+01 / 5.01e-01 | 20260508 12:23:24.696 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.03e-01, 1.12e-02 / 4.93e-03 | 20260508 12:23:24.696 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.46e+00, 5.99e-01 / 1.51e-01 | 20260508 12:23:24.696 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.10e-01 / 1.16e-01 | 20260508 12:23:24.697 | INFO | PerfReport: 15.97%, DME -- 4.17e+01 / 4.51e+01, 3.56e+00 / 9.84e-01 | 20260508 12:23:24.697 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.84e+00 / 1.18e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.46e+00, 4.06e-01 / 2.37e-01 | 20260508 12:23:24.697 | INFO | PerfReport: 2.97%, Metrics -- 9.69e-01 / 1.06e+00, 2.43e-01 / 8.13e-02 | 20260508 12:23:24.697 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.23e-01, 4.79e-02 / 2.79e-02 | 20260508 12:23:24.697 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:23:24.697 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:23:24.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:23:24.697 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 24.49%, Upload -- 2.57e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 31.11%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 2.13%, LockUpload -- 2.24e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 2.55%, LockDownload -- 2.68e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 14.95%, LockCompute -- 1.57e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:23:24.697 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 12:23:24.792 | INFO | Starting a round of Baz IO | 20260508 12:23:28.202 | INFO | Populating RTMetrics report | 20260508 12:23:28.202 | INFO | RTMetrics report populated | 20260508 12:23:28.202 | INFO | Opening tmp RTMetrics json file | 20260508 12:23:28.212 | INFO | Serializing RTMetrics to json | 20260508 12:23:28.213 | INFO | Serialization done, swapping tmp file now... | 20260508 12:23:28.213 | INFO | RTMetrics done | 20260508 12:23:28.270 | INFO | Finished a round of Baz IO: Wrote 1.414 GiB in 3.477 seconds | 20260508 12:23:28.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:23:28.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164739% / 0% / 0.000111262% | 20260508 12:23:28.270 | INFO | Baz IO overall: | 20260508 12:23:28.270 | INFO | BytesWritten : 3464911955096 | 20260508 12:23:28.270 | INFO | NumEvents : 1662820192035 | 20260508 12:23:29.830 | INFO | Analyzing chunk frames = [8077312,8077824) | 20260508 12:23:29.830 | INFO | MemInfo: { | 20260508 12:23:29.830 | INFO | "gpu" : " 17 GB", | 20260508 12:23:29.830 | INFO | "huge" : "", | 20260508 12:23:29.830 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:29.831 | INFO | "malloc" : " 14 GB", | 20260508 12:23:29.831 | INFO | "pinned" : " 29 GB" | 20260508 12:23:29.831 | INFO | } | 20260508 12:23:29.831 | INFO | Perf report for the last 5.13576 seconds: | 20260508 12:23:29.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:29.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:29.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:29.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:29.831 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 12:23:29.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:29.831 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:23:29.831 | INFO | PRE_HQ : 65.4%, 1.6, 53.8ms, 0 | 20260508 12:23:29.831 | INFO | BAZWRITER : 67.0%, 2.5, 85.1ms, 0 | 20260508 12:23:29.831 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:23:29.831 | INFO | Uploaded 36GiB to GPU in 2.58615s (13.9203GiB/s) | 20260508 12:23:29.831 | INFO | Downloaded 38GiB to GPU in 2.72196s (13.9605GiB/s) | 20260508 12:23:34.969 | INFO | Analyzing chunk frames = [8077824,8078336) | 20260508 12:23:34.969 | INFO | MemInfo: { | 20260508 12:23:34.969 | INFO | "gpu" : " 17 GB", | 20260508 12:23:34.969 | INFO | "huge" : "", | 20260508 12:23:34.969 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:34.969 | INFO | "malloc" : " 14 GB", | 20260508 12:23:34.969 | INFO | "pinned" : " 29 GB" | 20260508 12:23:34.969 | INFO | } | 20260508 12:23:34.969 | INFO | Perf report for the last 5.13818 seconds: | 20260508 12:23:34.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:34.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 12:23:34.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:23:34.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:34.969 | INFO | Uploaded 36GiB to GPU in 2.47579s (14.5408GiB/s) | 20260508 12:23:34.970 | INFO | Downloaded 34GiB to GPU in 2.45973s (13.8227GiB/s) | 20260508 12:23:40.106 | INFO | Analyzing chunk frames = [8078336,8078848) | 20260508 12:23:40.106 | INFO | MemInfo: { | 20260508 12:23:40.106 | INFO | "gpu" : " 17 GB", | 20260508 12:23:40.106 | INFO | "huge" : "", | 20260508 12:23:40.106 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:40.106 | INFO | "malloc" : " 14 GB", | 20260508 12:23:40.106 | INFO | "pinned" : " 29 GB" | 20260508 12:23:40.106 | INFO | } | 20260508 12:23:40.106 | INFO | Perf report for the last 5.13728 seconds: | 20260508 12:23:40.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:40.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | ANALYSIS : 66.6%, 3.0, 103.4ms, 0 | 20260508 12:23:40.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:23:40.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:40.106 | INFO | Uploaded 36GiB to GPU in 2.51513s (14.3134GiB/s) | 20260508 12:23:40.106 | INFO | Downloaded 34GiB to GPU in 2.45269s (13.8624GiB/s) | 20260508 12:23:45.242 | INFO | Analyzing chunk frames = [8078848,8079360) | 20260508 12:23:45.242 | INFO | MemInfo: { | 20260508 12:23:45.242 | INFO | "gpu" : " 17 GB", | 20260508 12:23:45.242 | INFO | "huge" : "", | 20260508 12:23:45.242 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:45.242 | INFO | "malloc" : " 14 GB", | 20260508 12:23:45.242 | INFO | "pinned" : " 29 GB" | 20260508 12:23:45.242 | INFO | } | 20260508 12:23:45.243 | INFO | Perf report for the last 5.13634 seconds: | 20260508 12:23:45.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:45.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | ANALYSIS : 66.4%, 3.0, 102.8ms, 0 | 20260508 12:23:45.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:23:45.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:45.243 | INFO | Uploaded 36GiB to GPU in 2.54565s (14.1417GiB/s) | 20260508 12:23:45.243 | INFO | Downloaded 34GiB to GPU in 2.43288s (13.9752GiB/s) | 20260508 12:23:50.386 | INFO | Analyzing chunk frames = [8079360,8079872) | 20260508 12:23:50.386 | INFO | MemInfo: { | 20260508 12:23:50.386 | INFO | "gpu" : " 17 GB", | 20260508 12:23:50.386 | INFO | "huge" : "", | 20260508 12:23:50.386 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:50.386 | INFO | "malloc" : " 14 GB", | 20260508 12:23:50.386 | INFO | "pinned" : " 29 GB" | 20260508 12:23:50.386 | INFO | } | 20260508 12:23:50.386 | INFO | Perf report for the last 5.14333 seconds: | 20260508 12:23:50.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:50.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:23:50.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:23:50.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:50.386 | INFO | Uploaded 36GiB to GPU in 2.3289s (15.4579GiB/s) | 20260508 12:23:50.386 | INFO | Downloaded 34GiB to GPU in 2.5066s (13.5642GiB/s) | 20260508 12:23:55.530 | INFO | Analyzing chunk frames = [8079872,8080384) | 20260508 12:23:55.530 | INFO | MemInfo: { | 20260508 12:23:55.530 | INFO | "gpu" : " 17 GB", | 20260508 12:23:55.530 | INFO | "huge" : "", | 20260508 12:23:55.530 | INFO | "huge_shared" : " 29 GB", | 20260508 12:23:55.530 | INFO | "malloc" : " 14 GB", | 20260508 12:23:55.530 | INFO | "pinned" : " 29 GB" | 20260508 12:23:55.530 | INFO | } | 20260508 12:23:55.530 | INFO | Perf report for the last 5.14403 seconds: | 20260508 12:23:55.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:23:55.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 12:23:55.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 12:23:55.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:23:55.530 | INFO | Uploaded 36GiB to GPU in 2.41175s (14.9269GiB/s) | 20260508 12:23:55.530 | INFO | Downloaded 34GiB to GPU in 2.43956s (13.9369GiB/s) | 20260508 12:24:00.664 | INFO | Analyzing chunk frames = [8080384,8080896) | 20260508 12:24:00.664 | INFO | MemInfo: { | 20260508 12:24:00.664 | INFO | "gpu" : " 17 GB", | 20260508 12:24:00.664 | INFO | "huge" : "", | 20260508 12:24:00.665 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:00.665 | INFO | "malloc" : " 14 GB", | 20260508 12:24:00.665 | INFO | "pinned" : " 29 GB" | 20260508 12:24:00.665 | INFO | } | 20260508 12:24:00.665 | INFO | Perf report for the last 5.13489 seconds: | 20260508 12:24:00.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:00.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | ANALYSIS : 64.9%, 3.0, 101.0ms, 0 | 20260508 12:24:00.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:24:00.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:00.665 | INFO | Uploaded 36GiB to GPU in 2.35177s (15.3076GiB/s) | 20260508 12:24:00.665 | INFO | Downloaded 34GiB to GPU in 2.42668s (14.0109GiB/s) | 20260508 12:24:05.799 | INFO | Analyzing chunk frames = [8080896,8081408) | 20260508 12:24:05.799 | INFO | MemInfo: { | 20260508 12:24:05.799 | INFO | "gpu" : " 17 GB", | 20260508 12:24:05.799 | INFO | "huge" : "", | 20260508 12:24:05.799 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:05.799 | INFO | "malloc" : " 14 GB", | 20260508 12:24:05.799 | INFO | "pinned" : " 29 GB" | 20260508 12:24:05.799 | INFO | } | 20260508 12:24:05.799 | INFO | Perf report for the last 5.13463 seconds: | 20260508 12:24:05.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:05.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.800 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 12:24:05.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.800 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 12:24:05.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:05.800 | INFO | Uploaded 36GiB to GPU in 2.46574s (14.6001GiB/s) | 20260508 12:24:05.800 | INFO | Downloaded 34GiB to GPU in 2.45548s (13.8466GiB/s) | 20260508 12:24:05.903 | INFO | Starting a round of Baz IO | 20260508 12:24:09.306 | INFO | Populating RTMetrics report | 20260508 12:24:09.306 | INFO | RTMetrics report populated | 20260508 12:24:09.306 | INFO | Opening tmp RTMetrics json file | 20260508 12:24:09.314 | INFO | Serializing RTMetrics to json | 20260508 12:24:09.316 | INFO | Serialization done, swapping tmp file now... | 20260508 12:24:09.316 | INFO | RTMetrics done | 20260508 12:24:09.371 | INFO | Finished a round of Baz IO: Wrote 1.411 GiB in 3.467 seconds | 20260508 12:24:09.371 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.00% / 59.97% | 20260508 12:24:09.371 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163805% / 0% / 7.94729e-05% | 20260508 12:24:09.371 | INFO | Baz IO overall: | 20260508 12:24:09.371 | INFO | BytesWritten : 3466427161778 | 20260508 12:24:09.371 | INFO | NumEvents : 1663530521329 | 20260508 12:24:10.943 | INFO | Analyzing chunk frames = [8081408,8081920) | 20260508 12:24:10.943 | INFO | MemInfo: { | 20260508 12:24:10.943 | INFO | "gpu" : " 17 GB", | 20260508 12:24:10.943 | INFO | "huge" : "", | 20260508 12:24:10.943 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:10.943 | INFO | "malloc" : " 14 GB", | 20260508 12:24:10.943 | INFO | "pinned" : " 29 GB" | 20260508 12:24:10.943 | INFO | } | 20260508 12:24:10.944 | INFO | Perf report for the last 5.14409 seconds: | 20260508 12:24:10.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:10.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:10.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:10.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:10.944 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 12:24:10.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:10.944 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:24:10.944 | INFO | PRE_HQ : 65.4%, 1.6, 53.7ms, 0 | 20260508 12:24:10.944 | INFO | BAZWRITER : 67.0%, 2.5, 84.5ms, 0 | 20260508 12:24:10.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:24:10.944 | INFO | Uploaded 36GiB to GPU in 2.56806s (14.0184GiB/s) | 20260508 12:24:10.944 | INFO | Downloaded 38GiB to GPU in 2.71726s (13.9847GiB/s) | 20260508 12:24:16.082 | INFO | Analyzing chunk frames = [8081920,8082432) | 20260508 12:24:16.082 | INFO | MemInfo: { | 20260508 12:24:16.082 | INFO | "gpu" : " 17 GB", | 20260508 12:24:16.082 | INFO | "huge" : "", | 20260508 12:24:16.082 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:16.082 | INFO | "malloc" : " 14 GB", | 20260508 12:24:16.082 | INFO | "pinned" : " 29 GB" | 20260508 12:24:16.082 | INFO | } | 20260508 12:24:16.083 | INFO | Perf report for the last 5.13904 seconds: | 20260508 12:24:16.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:16.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 12:24:16.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:24:16.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:16.083 | INFO | Uploaded 36GiB to GPU in 2.41402s (14.9129GiB/s) | 20260508 12:24:16.083 | INFO | Downloaded 34GiB to GPU in 2.48198s (13.6987GiB/s) | 20260508 12:24:21.224 | INFO | Analyzing chunk frames = [8082432,8082944) | 20260508 12:24:21.224 | INFO | MemInfo: { | 20260508 12:24:21.224 | INFO | "gpu" : " 17 GB", | 20260508 12:24:21.224 | INFO | "huge" : "", | 20260508 12:24:21.224 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:21.225 | INFO | "malloc" : " 14 GB", | 20260508 12:24:21.225 | INFO | "pinned" : " 29 GB" | 20260508 12:24:21.225 | INFO | } | 20260508 12:24:21.225 | INFO | Perf report for the last 5.14205 seconds: | 20260508 12:24:21.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:21.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 12:24:21.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:24:21.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:21.225 | INFO | Uploaded 36GiB to GPU in 2.45595s (14.6583GiB/s) | 20260508 12:24:21.225 | INFO | Downloaded 34GiB to GPU in 2.46116s (13.8146GiB/s) | 20260508 12:24:26.365 | INFO | Analyzing chunk frames = [8082944,8083456) | 20260508 12:24:26.365 | INFO | MemInfo: { | 20260508 12:24:26.365 | INFO | "gpu" : " 17 GB", | 20260508 12:24:26.365 | INFO | "huge" : "", | 20260508 12:24:26.365 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:26.365 | INFO | "malloc" : " 14 GB", | 20260508 12:24:26.365 | INFO | "pinned" : " 29 GB" | 20260508 12:24:26.365 | INFO | } | 20260508 12:24:26.365 | INFO | Perf report for the last 5.14066 seconds: | 20260508 12:24:26.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:26.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.365 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 12:24:26.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.365 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:24:26.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:26.366 | INFO | Uploaded 36GiB to GPU in 2.45098s (14.688GiB/s) | 20260508 12:24:26.366 | INFO | Downloaded 34GiB to GPU in 2.46407s (13.7983GiB/s) | 20260508 12:24:31.499 | INFO | Analyzing chunk frames = [8083456,8083968) | 20260508 12:24:31.499 | INFO | MemInfo: { | 20260508 12:24:31.499 | INFO | "gpu" : " 17 GB", | 20260508 12:24:31.499 | INFO | "huge" : "", | 20260508 12:24:31.500 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:31.500 | INFO | "malloc" : " 14 GB", | 20260508 12:24:31.500 | INFO | "pinned" : " 29 GB" | 20260508 12:24:31.500 | INFO | } | 20260508 12:24:31.500 | INFO | Perf report for the last 5.13442 seconds: | 20260508 12:24:31.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:31.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:24:31.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 12:24:31.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:31.500 | INFO | Uploaded 36GiB to GPU in 2.27468s (15.8264GiB/s) | 20260508 12:24:31.500 | INFO | Downloaded 34GiB to GPU in 2.48831s (13.6639GiB/s) | 20260508 12:24:36.646 | INFO | Analyzing chunk frames = [8083968,8084480) | 20260508 12:24:36.646 | INFO | MemInfo: { | 20260508 12:24:36.646 | INFO | "gpu" : " 17 GB", | 20260508 12:24:36.646 | INFO | "huge" : "", | 20260508 12:24:36.646 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:36.646 | INFO | "malloc" : " 14 GB", | 20260508 12:24:36.646 | INFO | "pinned" : " 29 GB" | 20260508 12:24:36.646 | INFO | } | 20260508 12:24:36.646 | INFO | Perf report for the last 5.14607 seconds: | 20260508 12:24:36.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:36.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 12:24:36.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 12:24:36.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:36.646 | INFO | Uploaded 36GiB to GPU in 2.3646s (15.2245GiB/s) | 20260508 12:24:36.646 | INFO | Downloaded 34GiB to GPU in 2.455s (13.8493GiB/s) | 20260508 12:24:41.784 | INFO | Analyzing chunk frames = [8084480,8084992) | 20260508 12:24:41.784 | INFO | MemInfo: { | 20260508 12:24:41.784 | INFO | "gpu" : " 17 GB", | 20260508 12:24:41.784 | INFO | "huge" : "", | 20260508 12:24:41.784 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:41.785 | INFO | "malloc" : " 14 GB", | 20260508 12:24:41.785 | INFO | "pinned" : " 29 GB" | 20260508 12:24:41.785 | INFO | } | 20260508 12:24:41.785 | INFO | Perf report for the last 5.13871 seconds: | 20260508 12:24:41.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:41.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:24:41.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:24:41.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:41.785 | INFO | Uploaded 36GiB to GPU in 2.33042s (15.4479GiB/s) | 20260508 12:24:41.785 | INFO | Downloaded 34GiB to GPU in 2.44685s (13.8954GiB/s) | 20260508 12:24:46.919 | INFO | Analyzing chunk frames = [8084992,8085504) | 20260508 12:24:46.919 | INFO | MemInfo: { | 20260508 12:24:46.919 | INFO | "gpu" : " 17 GB", | 20260508 12:24:46.919 | INFO | "huge" : "", | 20260508 12:24:46.919 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:46.919 | INFO | "malloc" : " 14 GB", | 20260508 12:24:46.919 | INFO | "pinned" : " 29 GB" | 20260508 12:24:46.919 | INFO | } | 20260508 12:24:46.919 | INFO | Perf report for the last 5.13481 seconds: | 20260508 12:24:46.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:46.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.919 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:24:46.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.920 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 12:24:46.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:46.920 | INFO | Uploaded 36GiB to GPU in 2.40483s (14.9699GiB/s) | 20260508 12:24:46.920 | INFO | Downloaded 34GiB to GPU in 2.46917s (13.7698GiB/s) | 20260508 12:24:46.920 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:24:46.920 | INFO | Total mem usage: 16.7 GiB | 20260508 12:24:46.920 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:24:46.920 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:24:46.920 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:24:46.920 | INFO | High Water Breakdown: | 20260508 12:24:46.920 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:24:46.920 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:24:46.920 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:24:46.920 | INFO | ------------------------------------------ | 20260508 12:24:46.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:24:46.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:24:46.920 | INFO | Total mem usage: 26.5 GiB | 20260508 12:24:46.920 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:24:46.920 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:24:46.920 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:24:46.920 | INFO | High Water Breakdown: | 20260508 12:24:46.920 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:24:46.920 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:24:46.920 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:24:46.920 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:24:46.920 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:24:46.920 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:24:46.920 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:24:46.920 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:24:46.920 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:24:46.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:24:46.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:24:46.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:24:46.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:24:46.920 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:24:46.920 | INFO | ------------------------------------------ | 20260508 12:24:46.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:24:46.920 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:24:46.920 | INFO | Total mem usage: 27.8 GiB | 20260508 12:24:46.921 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:24:46.921 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:24:46.921 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:24:46.921 | INFO | High Water Breakdown: | 20260508 12:24:46.921 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:24:46.921 | INFO | ------------------------------------------ | 20260508 12:24:46.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:24:46.921 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:24:46.921 | INFO | Total mem usage: 15.7 GiB | 20260508 12:24:46.921 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:24:46.921 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:24:46.921 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:24:46.921 | INFO | High Water Breakdown: | 20260508 12:24:46.921 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:24:46.921 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:24:46.921 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:24:46.921 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:24:46.921 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:24:46.921 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:24:46.921 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:24:46.921 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:24:46.921 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:24:46.921 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:24:46.921 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:24:46.921 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:24:46.921 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:24:46.921 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:24:46.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:24:46.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:24:46.921 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:24:46.921 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:24:46.921 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:24:46.921 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:24:46.921 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:24:46.921 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:24:46.921 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:24:46.921 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:24:46.921 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:24:46.921 | INFO | ------------------------------------------ | 20260508 12:24:46.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:24:46.921 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:24:46.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:24:46.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:24:46.921 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.53e+01, 1.39e+01 / 5.11e-01 | 20260508 12:24:46.921 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.09e-02 / 1.90e-03 | 20260508 12:24:46.921 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.50e+00, 5.95e-01 / 1.54e-01 | 20260508 12:24:46.921 | INFO | PerfReport: 3.41%, Histogram -- 1.12e+00 / 1.50e+00, 2.10e-01 / 7.97e-02 | 20260508 12:24:46.921 | INFO | PerfReport: 16.00%, DME -- 4.20e+01 / 4.58e+01, 3.67e+00 / 8.94e-01 | 20260508 12:24:46.921 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.88e+01 / 1.94e+01, 1.89e+00 / 1.11e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.31e+00 / 2.55e+00, 4.03e-01 / 2.22e-01 | 20260508 12:24:46.921 | INFO | PerfReport: 2.95%, Metrics -- 9.66e-01 / 1.08e+00, 2.41e-01 / 1.13e-01 | 20260508 12:24:46.921 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.92e-01 / 3.18e-01, 4.73e-02 / 2.43e-02 | 20260508 12:24:46.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:24:46.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:24:46.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:24:46.921 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.39e+01 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 24.16%, Upload -- 2.54e+01 / 0.00e+00, 7.80e+00 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 31.25%, Compute -- 3.28e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 2.14%, LockUpload -- 2.25e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 2.58%, LockDownload -- 2.71e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 15.11%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:24:46.921 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 12:24:47.017 | INFO | Starting a round of Baz IO | 20260508 12:24:50.426 | INFO | Populating RTMetrics report | 20260508 12:24:50.426 | INFO | RTMetrics report populated | 20260508 12:24:50.426 | INFO | Opening tmp RTMetrics json file | 20260508 12:24:50.435 | INFO | Serializing RTMetrics to json | 20260508 12:24:50.436 | INFO | Serialization done, swapping tmp file now... | 20260508 12:24:50.437 | INFO | RTMetrics done | 20260508 12:24:50.490 | INFO | Finished a round of Baz IO: Wrote 1.41 GiB in 3.473 seconds | 20260508 12:24:50.490 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.97% | 20260508 12:24:50.490 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164111% / 0% / 5.96046e-05% | 20260508 12:24:50.490 | INFO | Baz IO overall: | 20260508 12:24:50.490 | INFO | BytesWritten : 3467941278204 | 20260508 12:24:50.490 | INFO | NumEvents : 1664240421488 | 20260508 12:24:52.058 | INFO | Analyzing chunk frames = [8085504,8086016) | 20260508 12:24:52.058 | INFO | MemInfo: { | 20260508 12:24:52.058 | INFO | "gpu" : " 17 GB", | 20260508 12:24:52.058 | INFO | "huge" : "", | 20260508 12:24:52.058 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:52.058 | INFO | "malloc" : " 14 GB", | 20260508 12:24:52.058 | INFO | "pinned" : " 29 GB" | 20260508 12:24:52.058 | INFO | } | 20260508 12:24:52.058 | INFO | Perf report for the last 5.13826 seconds: | 20260508 12:24:52.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:52.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:52.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:52.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:52.058 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 12:24:52.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:52.058 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:24:52.058 | INFO | PRE_HQ : 65.8%, 1.6, 54.5ms, 0 | 20260508 12:24:52.058 | INFO | BAZWRITER : 66.9%, 2.5, 84.3ms, 0 | 20260508 12:24:52.058 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:24:52.058 | INFO | Uploaded 36GiB to GPU in 2.61619s (13.7605GiB/s) | 20260508 12:24:52.058 | INFO | Downloaded 38GiB to GPU in 2.72038s (13.9686GiB/s) | 20260508 12:24:57.200 | INFO | Analyzing chunk frames = [8086016,8086528) | 20260508 12:24:57.200 | INFO | MemInfo: { | 20260508 12:24:57.200 | INFO | "gpu" : " 17 GB", | 20260508 12:24:57.200 | INFO | "huge" : "", | 20260508 12:24:57.201 | INFO | "huge_shared" : " 29 GB", | 20260508 12:24:57.201 | INFO | "malloc" : " 14 GB", | 20260508 12:24:57.201 | INFO | "pinned" : " 29 GB" | 20260508 12:24:57.201 | INFO | } | 20260508 12:24:57.201 | INFO | Perf report for the last 5.14302 seconds: | 20260508 12:24:57.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:24:57.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:24:57.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:24:57.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:24:57.201 | INFO | Uploaded 36GiB to GPU in 2.43738s (14.77GiB/s) | 20260508 12:24:57.201 | INFO | Downloaded 34GiB to GPU in 2.46692s (13.7823GiB/s) | 20260508 12:25:02.338 | INFO | Analyzing chunk frames = [8086528,8087040) | 20260508 12:25:02.338 | INFO | MemInfo: { | 20260508 12:25:02.338 | INFO | "gpu" : " 17 GB", | 20260508 12:25:02.338 | INFO | "huge" : "", | 20260508 12:25:02.338 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:02.338 | INFO | "malloc" : " 14 GB", | 20260508 12:25:02.338 | INFO | "pinned" : " 29 GB" | 20260508 12:25:02.338 | INFO | } | 20260508 12:25:02.339 | INFO | Perf report for the last 5.13784 seconds: | 20260508 12:25:02.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:02.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:25:02.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:25:02.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:02.339 | INFO | Uploaded 36GiB to GPU in 2.51501s (14.3141GiB/s) | 20260508 12:25:02.339 | INFO | Downloaded 34GiB to GPU in 2.45762s (13.8345GiB/s) | 20260508 12:25:07.477 | INFO | Analyzing chunk frames = [8087040,8087552) | 20260508 12:25:07.477 | INFO | MemInfo: { | 20260508 12:25:07.477 | INFO | "gpu" : " 17 GB", | 20260508 12:25:07.477 | INFO | "huge" : "", | 20260508 12:25:07.477 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:07.477 | INFO | "malloc" : " 14 GB", | 20260508 12:25:07.477 | INFO | "pinned" : " 29 GB" | 20260508 12:25:07.477 | INFO | } | 20260508 12:25:07.477 | INFO | Perf report for the last 5.13841 seconds: | 20260508 12:25:07.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:07.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 12:25:07.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:25:07.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:07.477 | INFO | Uploaded 36GiB to GPU in 2.54932s (14.1214GiB/s) | 20260508 12:25:07.477 | INFO | Downloaded 34GiB to GPU in 2.45295s (13.8608GiB/s) | 20260508 12:25:12.614 | INFO | Analyzing chunk frames = [8087552,8088064) | 20260508 12:25:12.614 | INFO | MemInfo: { | 20260508 12:25:12.614 | INFO | "gpu" : " 17 GB", | 20260508 12:25:12.614 | INFO | "huge" : "", | 20260508 12:25:12.614 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:12.614 | INFO | "malloc" : " 14 GB", | 20260508 12:25:12.614 | INFO | "pinned" : " 29 GB" | 20260508 12:25:12.614 | INFO | } | 20260508 12:25:12.614 | INFO | Perf report for the last 5.13693 seconds: | 20260508 12:25:12.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:12.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.614 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 12:25:12.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.614 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 12:25:12.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:12.615 | INFO | Uploaded 36GiB to GPU in 2.4796s (14.5185GiB/s) | 20260508 12:25:12.615 | INFO | Downloaded 34GiB to GPU in 2.51353s (13.5268GiB/s) | 20260508 12:25:17.751 | INFO | Analyzing chunk frames = [8088064,8088576) | 20260508 12:25:17.751 | INFO | MemInfo: { | 20260508 12:25:17.751 | INFO | "gpu" : " 17 GB", | 20260508 12:25:17.751 | INFO | "huge" : "", | 20260508 12:25:17.751 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:17.751 | INFO | "malloc" : " 14 GB", | 20260508 12:25:17.751 | INFO | "pinned" : " 29 GB" | 20260508 12:25:17.751 | INFO | } | 20260508 12:25:17.752 | INFO | Perf report for the last 5.13764 seconds: | 20260508 12:25:17.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:17.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:25:17.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:25:17.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:17.752 | INFO | Uploaded 36GiB to GPU in 2.42154s (14.8666GiB/s) | 20260508 12:25:17.752 | INFO | Downloaded 34GiB to GPU in 2.4349s (13.9636GiB/s) | 20260508 12:25:22.894 | INFO | Analyzing chunk frames = [8088576,8089088) | 20260508 12:25:22.894 | INFO | MemInfo: { | 20260508 12:25:22.894 | INFO | "gpu" : " 17 GB", | 20260508 12:25:22.894 | INFO | "huge" : "", | 20260508 12:25:22.894 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:22.894 | INFO | "malloc" : " 14 GB", | 20260508 12:25:22.894 | INFO | "pinned" : " 29 GB" | 20260508 12:25:22.894 | INFO | } | 20260508 12:25:22.894 | INFO | Perf report for the last 5.14235 seconds: | 20260508 12:25:22.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:22.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:25:22.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:25:22.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:22.894 | INFO | Uploaded 36GiB to GPU in 2.43828s (14.7645GiB/s) | 20260508 12:25:22.894 | INFO | Downloaded 34GiB to GPU in 2.44147s (13.926GiB/s) | 20260508 12:25:28.038 | INFO | Analyzing chunk frames = [8089088,8089600) | 20260508 12:25:28.038 | INFO | MemInfo: { | 20260508 12:25:28.038 | INFO | "gpu" : " 17 GB", | 20260508 12:25:28.038 | INFO | "huge" : "", | 20260508 12:25:28.038 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:28.038 | INFO | "malloc" : " 14 GB", | 20260508 12:25:28.038 | INFO | "pinned" : " 29 GB" | 20260508 12:25:28.038 | INFO | } | 20260508 12:25:28.038 | INFO | Perf report for the last 5.14446 seconds: | 20260508 12:25:28.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:28.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.038 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:25:28.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.039 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:25:28.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:28.039 | INFO | Uploaded 36GiB to GPU in 2.444s (14.73GiB/s) | 20260508 12:25:28.039 | INFO | Downloaded 34GiB to GPU in 2.44513s (13.9052GiB/s) | 20260508 12:25:28.135 | INFO | Starting a round of Baz IO | 20260508 12:25:31.552 | INFO | Populating RTMetrics report | 20260508 12:25:31.553 | INFO | RTMetrics report populated | 20260508 12:25:31.553 | INFO | Opening tmp RTMetrics json file | 20260508 12:25:31.561 | INFO | Serializing RTMetrics to json | 20260508 12:25:31.562 | INFO | Serialization done, swapping tmp file now... | 20260508 12:25:31.563 | INFO | RTMetrics done | 20260508 12:25:31.617 | INFO | Finished a round of Baz IO: Wrote 1.41 GiB in 3.482 seconds | 20260508 12:25:31.617 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:25:31.617 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163758% / 3.97364e-06% / 6.75519e-05% | 20260508 12:25:31.617 | INFO | Baz IO overall: | 20260508 12:25:31.617 | INFO | BytesWritten : 3469455386350 | 20260508 12:25:31.617 | INFO | NumEvents : 1664950621830 | 20260508 12:25:33.175 | INFO | Analyzing chunk frames = [8089600,8090112) | 20260508 12:25:33.175 | INFO | MemInfo: { | 20260508 12:25:33.175 | INFO | "gpu" : " 17 GB", | 20260508 12:25:33.175 | INFO | "huge" : "", | 20260508 12:25:33.175 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:33.175 | INFO | "malloc" : " 14 GB", | 20260508 12:25:33.175 | INFO | "pinned" : " 29 GB" | 20260508 12:25:33.175 | INFO | } | 20260508 12:25:33.175 | INFO | Perf report for the last 5.13684 seconds: | 20260508 12:25:33.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:33.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:33.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:33.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:33.175 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 12:25:33.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:33.176 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:25:33.176 | INFO | PRE_HQ : 65.7%, 1.6, 54.0ms, 0 | 20260508 12:25:33.176 | INFO | BAZWRITER : 67.0%, 2.5, 84.9ms, 0 | 20260508 12:25:33.176 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:25:33.176 | INFO | Uploaded 36GiB to GPU in 2.6015s (13.8381GiB/s) | 20260508 12:25:33.176 | INFO | Downloaded 38GiB to GPU in 2.73481s (13.8949GiB/s) | 20260508 12:25:38.310 | INFO | Analyzing chunk frames = [8090112,8090624) | 20260508 12:25:38.310 | INFO | MemInfo: { | 20260508 12:25:38.310 | INFO | "gpu" : " 17 GB", | 20260508 12:25:38.310 | INFO | "huge" : "", | 20260508 12:25:38.310 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:38.310 | INFO | "malloc" : " 14 GB", | 20260508 12:25:38.310 | INFO | "pinned" : " 29 GB" | 20260508 12:25:38.310 | INFO | } | 20260508 12:25:38.310 | INFO | Perf report for the last 5.13509 seconds: | 20260508 12:25:38.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:38.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.310 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 12:25:38.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.311 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 12:25:38.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:38.311 | INFO | Uploaded 36GiB to GPU in 2.4225s (14.8607GiB/s) | 20260508 12:25:38.311 | INFO | Downloaded 34GiB to GPU in 2.48917s (13.6591GiB/s) | 20260508 12:25:43.456 | INFO | Analyzing chunk frames = [8090624,8091136) | 20260508 12:25:43.456 | INFO | MemInfo: { | 20260508 12:25:43.456 | INFO | "gpu" : " 17 GB", | 20260508 12:25:43.456 | INFO | "huge" : "", | 20260508 12:25:43.456 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:43.456 | INFO | "malloc" : " 14 GB", | 20260508 12:25:43.456 | INFO | "pinned" : " 29 GB" | 20260508 12:25:43.456 | INFO | } | 20260508 12:25:43.457 | INFO | Perf report for the last 5.14615 seconds: | 20260508 12:25:43.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:43.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:25:43.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:25:43.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:43.457 | INFO | Uploaded 36GiB to GPU in 2.48277s (14.4999GiB/s) | 20260508 12:25:43.457 | INFO | Downloaded 34GiB to GPU in 2.46688s (13.7826GiB/s) | 20260508 12:25:48.593 | INFO | Analyzing chunk frames = [8091136,8091648) | 20260508 12:25:48.593 | INFO | MemInfo: { | 20260508 12:25:48.593 | INFO | "gpu" : " 17 GB", | 20260508 12:25:48.593 | INFO | "huge" : "", | 20260508 12:25:48.593 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:48.593 | INFO | "malloc" : " 14 GB", | 20260508 12:25:48.593 | INFO | "pinned" : " 29 GB" | 20260508 12:25:48.593 | INFO | } | 20260508 12:25:48.594 | INFO | Perf report for the last 5.13698 seconds: | 20260508 12:25:48.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:48.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 12:25:48.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:25:48.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:48.594 | INFO | Uploaded 36GiB to GPU in 2.48925s (14.4622GiB/s) | 20260508 12:25:48.594 | INFO | Downloaded 34GiB to GPU in 2.47588s (13.7325GiB/s) | 20260508 12:25:53.731 | INFO | Analyzing chunk frames = [8091648,8092160) | 20260508 12:25:53.732 | INFO | MemInfo: { | 20260508 12:25:53.732 | INFO | "gpu" : " 17 GB", | 20260508 12:25:53.732 | INFO | "huge" : "", | 20260508 12:25:53.732 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:53.732 | INFO | "malloc" : " 14 GB", | 20260508 12:25:53.732 | INFO | "pinned" : " 29 GB" | 20260508 12:25:53.732 | INFO | } | 20260508 12:25:53.732 | INFO | Perf report for the last 5.13828 seconds: | 20260508 12:25:53.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:53.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:25:53.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:25:53.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:53.732 | INFO | Uploaded 36GiB to GPU in 2.30202s (15.6385GiB/s) | 20260508 12:25:53.732 | INFO | Downloaded 34GiB to GPU in 2.50867s (13.553GiB/s) | 20260508 12:25:58.871 | INFO | Analyzing chunk frames = [8092160,8092672) | 20260508 12:25:58.871 | INFO | MemInfo: { | 20260508 12:25:58.871 | INFO | "gpu" : " 17 GB", | 20260508 12:25:58.871 | INFO | "huge" : "", | 20260508 12:25:58.872 | INFO | "huge_shared" : " 29 GB", | 20260508 12:25:58.872 | INFO | "malloc" : " 14 GB", | 20260508 12:25:58.872 | INFO | "pinned" : " 29 GB" | 20260508 12:25:58.872 | INFO | } | 20260508 12:25:58.872 | INFO | Perf report for the last 5.13981 seconds: | 20260508 12:25:58.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:25:58.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:25:58.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:25:58.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:25:58.872 | INFO | Uploaded 36GiB to GPU in 2.40621s (14.9613GiB/s) | 20260508 12:25:58.872 | INFO | Downloaded 34GiB to GPU in 2.44148s (13.926GiB/s) | 20260508 12:26:04.002 | INFO | Analyzing chunk frames = [8092672,8093184) | 20260508 12:26:04.002 | INFO | MemInfo: { | 20260508 12:26:04.002 | INFO | "gpu" : " 17 GB", | 20260508 12:26:04.002 | INFO | "huge" : "", | 20260508 12:26:04.002 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:04.003 | INFO | "malloc" : " 14 GB", | 20260508 12:26:04.003 | INFO | "pinned" : " 29 GB" | 20260508 12:26:04.003 | INFO | } | 20260508 12:26:04.003 | INFO | Perf report for the last 5.13095 seconds: | 20260508 12:26:04.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:04.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 12:26:04.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:26:04.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:04.003 | INFO | Uploaded 36GiB to GPU in 2.4417s (14.7438GiB/s) | 20260508 12:26:04.003 | INFO | Downloaded 34GiB to GPU in 2.45999s (13.8212GiB/s) | 20260508 12:26:09.150 | INFO | Analyzing chunk frames = [8093184,8093696) | 20260508 12:26:09.150 | INFO | MemInfo: { | 20260508 12:26:09.150 | INFO | "gpu" : " 17 GB", | 20260508 12:26:09.150 | INFO | "huge" : "", | 20260508 12:26:09.151 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:09.151 | INFO | "malloc" : " 14 GB", | 20260508 12:26:09.151 | INFO | "pinned" : " 29 GB" | 20260508 12:26:09.151 | INFO | } | 20260508 12:26:09.151 | INFO | Perf report for the last 5.14824 seconds: | 20260508 12:26:09.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:09.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 12:26:09.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:26:09.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:09.151 | INFO | Uploaded 36GiB to GPU in 2.44798s (14.706GiB/s) | 20260508 12:26:09.151 | INFO | Downloaded 34GiB to GPU in 2.45299s (13.8606GiB/s) | 20260508 12:26:09.152 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:26:09.152 | INFO | Total mem usage: 16.7 GiB | 20260508 12:26:09.152 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:26:09.152 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:26:09.152 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:26:09.152 | INFO | High Water Breakdown: | 20260508 12:26:09.152 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:26:09.152 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:26:09.152 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:26:09.152 | INFO | ------------------------------------------ | 20260508 12:26:09.152 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:26:09.152 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:26:09.152 | INFO | Total mem usage: 26.5 GiB | 20260508 12:26:09.152 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:26:09.152 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:26:09.152 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:26:09.152 | INFO | High Water Breakdown: | 20260508 12:26:09.152 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:26:09.152 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:26:09.152 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:26:09.152 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:26:09.152 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:26:09.152 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:26:09.152 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:26:09.152 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:26:09.152 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:26:09.152 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:26:09.152 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:26:09.152 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:26:09.152 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:26:09.152 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:26:09.152 | INFO | ------------------------------------------ | 20260508 12:26:09.152 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:26:09.152 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:26:09.152 | INFO | Total mem usage: 27.8 GiB | 20260508 12:26:09.152 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:26:09.152 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:26:09.152 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:26:09.152 | INFO | High Water Breakdown: | 20260508 12:26:09.152 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:26:09.152 | INFO | ------------------------------------------ | 20260508 12:26:09.152 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:26:09.152 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:26:09.152 | INFO | Total mem usage: 15.7 GiB | 20260508 12:26:09.152 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:26:09.152 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:26:09.152 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:26:09.152 | INFO | High Water Breakdown: | 20260508 12:26:09.152 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:26:09.152 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:26:09.152 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:26:09.152 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:26:09.152 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:26:09.152 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:26:09.152 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:26:09.152 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:26:09.152 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:26:09.152 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:26:09.152 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:26:09.152 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:26:09.152 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:26:09.152 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:26:09.152 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:26:09.152 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:26:09.152 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:26:09.152 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:26:09.152 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:26:09.153 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:26:09.153 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:26:09.153 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:26:09.153 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:26:09.153 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:26:09.153 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:26:09.153 | INFO | ------------------------------------------ | 20260508 12:26:09.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:26:09.153 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:26:09.153 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:26:09.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:26:09.153 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.41e+01 / 5.12e-01 | 20260508 12:26:09.153 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.12e-02 / 5.36e-03 | 20260508 12:26:09.153 | INFO | PerfReport: 12.15%, Baseline -- 3.96e+00 / 4.38e+00, 5.92e-01 / 1.98e-01 | 20260508 12:26:09.153 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.44e+00, 2.08e-01 / 9.24e-02 | 20260508 12:26:09.153 | INFO | PerfReport: 16.02%, DME -- 4.17e+01 / 4.50e+01, 3.65e+00 / 1.24e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.86e+01 / 2.07e+01, 1.77e+00 / 1.37e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.33e+00 / 2.39e+00, 4.05e-01 / 2.48e-01 | 20260508 12:26:09.153 | INFO | PerfReport: 2.96%, Metrics -- 9.64e-01 / 1.11e+00, 2.39e-01 / 1.17e-01 | 20260508 12:26:09.153 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.92e-01 / 3.16e-01, 4.77e-02 / 2.00e-02 | 20260508 12:26:09.153 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:26:09.153 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:26:09.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:26:09.153 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 24.47%, Upload -- 2.57e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 30.99%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 2.07%, LockUpload -- 2.18e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 2.65%, LockDownload -- 2.78e+00 / 0.00e+00, 4.14e+00 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 15.03%, LockCompute -- 1.58e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:26:09.153 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 12:26:09.259 | INFO | Starting a round of Baz IO | 20260508 12:26:12.669 | INFO | Populating RTMetrics report | 20260508 12:26:12.669 | INFO | RTMetrics report populated | 20260508 12:26:12.669 | INFO | Opening tmp RTMetrics json file | 20260508 12:26:12.684 | INFO | Serializing RTMetrics to json | 20260508 12:26:12.686 | INFO | Serialization done, swapping tmp file now... | 20260508 12:26:12.686 | INFO | RTMetrics done | 20260508 12:26:12.734 | INFO | Finished a round of Baz IO: Wrote 1.409 GiB in 3.475 seconds | 20260508 12:26:12.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:26:12.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163456% / 0% / 6.35783e-05% | 20260508 12:26:12.734 | INFO | Baz IO overall: | 20260508 12:26:12.734 | INFO | BytesWritten : 3470968223231 | 20260508 12:26:12.734 | INFO | NumEvents : 1665659898530 | 20260508 12:26:14.293 | INFO | Analyzing chunk frames = [8093696,8094208) | 20260508 12:26:14.293 | INFO | MemInfo: { | 20260508 12:26:14.293 | INFO | "gpu" : " 17 GB", | 20260508 12:26:14.293 | INFO | "huge" : "", | 20260508 12:26:14.293 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:14.293 | INFO | "malloc" : " 14 GB", | 20260508 12:26:14.293 | INFO | "pinned" : " 29 GB" | 20260508 12:26:14.293 | INFO | } | 20260508 12:26:14.293 | INFO | Perf report for the last 5.14248 seconds: | 20260508 12:26:14.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:14.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:14.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:14.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:14.294 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 12:26:14.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:14.294 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260508 12:26:14.294 | INFO | PRE_HQ : 66.7%, 1.8, 61.3ms, 0 | 20260508 12:26:14.294 | INFO | BAZWRITER : 66.8%, 2.7, 91.1ms, 0 | 20260508 12:26:14.294 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 12:26:14.294 | INFO | Uploaded 36GiB to GPU in 2.58716s (13.9149GiB/s) | 20260508 12:26:14.294 | INFO | Downloaded 38GiB to GPU in 2.74235s (13.8567GiB/s) | 20260508 12:26:19.428 | INFO | Analyzing chunk frames = [8094208,8094720) | 20260508 12:26:19.429 | INFO | MemInfo: { | 20260508 12:26:19.429 | INFO | "gpu" : " 17 GB", | 20260508 12:26:19.429 | INFO | "huge" : "", | 20260508 12:26:19.429 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:19.429 | INFO | "malloc" : " 14 GB", | 20260508 12:26:19.429 | INFO | "pinned" : " 29 GB" | 20260508 12:26:19.429 | INFO | } | 20260508 12:26:19.429 | INFO | Perf report for the last 5.13539 seconds: | 20260508 12:26:19.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:19.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:26:19.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:26:19.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:19.429 | INFO | Uploaded 36GiB to GPU in 2.4839s (14.4933GiB/s) | 20260508 12:26:19.429 | INFO | Downloaded 34GiB to GPU in 2.49057s (13.6515GiB/s) | 20260508 12:26:24.559 | INFO | Analyzing chunk frames = [8094720,8095232) | 20260508 12:26:24.559 | INFO | MemInfo: { | 20260508 12:26:24.559 | INFO | "gpu" : " 17 GB", | 20260508 12:26:24.559 | INFO | "huge" : "", | 20260508 12:26:24.559 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:24.559 | INFO | "malloc" : " 14 GB", | 20260508 12:26:24.559 | INFO | "pinned" : " 29 GB" | 20260508 12:26:24.559 | INFO | } | 20260508 12:26:24.559 | INFO | Perf report for the last 5.13027 seconds: | 20260508 12:26:24.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:24.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 12:26:24.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:26:24.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:24.559 | INFO | Uploaded 36GiB to GPU in 2.46402s (14.6103GiB/s) | 20260508 12:26:24.559 | INFO | Downloaded 34GiB to GPU in 2.46977s (13.7664GiB/s) | 20260508 12:26:29.706 | INFO | Analyzing chunk frames = [8095232,8095744) | 20260508 12:26:29.706 | INFO | MemInfo: { | 20260508 12:26:29.706 | INFO | "gpu" : " 17 GB", | 20260508 12:26:29.706 | INFO | "huge" : "", | 20260508 12:26:29.706 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:29.706 | INFO | "malloc" : " 14 GB", | 20260508 12:26:29.706 | INFO | "pinned" : " 29 GB" | 20260508 12:26:29.706 | INFO | } | 20260508 12:26:29.706 | INFO | Perf report for the last 5.14706 seconds: | 20260508 12:26:29.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:29.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 12:26:29.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:26:29.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:29.706 | INFO | Uploaded 36GiB to GPU in 2.45083s (14.6889GiB/s) | 20260508 12:26:29.706 | INFO | Downloaded 34GiB to GPU in 2.41421s (14.0833GiB/s) | 20260508 12:26:34.846 | INFO | Analyzing chunk frames = [8095744,8096256) | 20260508 12:26:34.846 | INFO | MemInfo: { | 20260508 12:26:34.846 | INFO | "gpu" : " 17 GB", | 20260508 12:26:34.846 | INFO | "huge" : "", | 20260508 12:26:34.846 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:34.846 | INFO | "malloc" : " 14 GB", | 20260508 12:26:34.846 | INFO | "pinned" : " 29 GB" | 20260508 12:26:34.846 | INFO | } | 20260508 12:26:34.846 | INFO | Perf report for the last 5.13984 seconds: | 20260508 12:26:34.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:34.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.846 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:26:34.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.846 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260508 12:26:34.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:34.847 | INFO | Uploaded 36GiB to GPU in 2.37472s (15.1597GiB/s) | 20260508 12:26:34.847 | INFO | Downloaded 34GiB to GPU in 2.51236s (13.5331GiB/s) | 20260508 12:26:39.987 | INFO | Analyzing chunk frames = [8096256,8096768) | 20260508 12:26:39.987 | INFO | MemInfo: { | 20260508 12:26:39.987 | INFO | "gpu" : " 17 GB", | 20260508 12:26:39.987 | INFO | "huge" : "", | 20260508 12:26:39.987 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:39.987 | INFO | "malloc" : " 14 GB", | 20260508 12:26:39.987 | INFO | "pinned" : " 29 GB" | 20260508 12:26:39.987 | INFO | } | 20260508 12:26:39.987 | INFO | Perf report for the last 5.14107 seconds: | 20260508 12:26:39.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:39.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 12:26:39.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:26:39.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:39.987 | INFO | Uploaded 36GiB to GPU in 2.41085s (14.9325GiB/s) | 20260508 12:26:39.987 | INFO | Downloaded 34GiB to GPU in 2.4484s (13.8866GiB/s) | 20260508 12:26:45.126 | INFO | Analyzing chunk frames = [8096768,8097280) | 20260508 12:26:45.126 | INFO | MemInfo: { | 20260508 12:26:45.126 | INFO | "gpu" : " 17 GB", | 20260508 12:26:45.126 | INFO | "huge" : "", | 20260508 12:26:45.127 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:45.127 | INFO | "malloc" : " 14 GB", | 20260508 12:26:45.127 | INFO | "pinned" : " 29 GB" | 20260508 12:26:45.127 | INFO | } | 20260508 12:26:45.127 | INFO | Perf report for the last 5.13969 seconds: | 20260508 12:26:45.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:45.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 12:26:45.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 12:26:45.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:45.127 | INFO | Uploaded 36GiB to GPU in 2.43725s (14.7708GiB/s) | 20260508 12:26:45.127 | INFO | Downloaded 34GiB to GPU in 2.46363s (13.8008GiB/s) | 20260508 12:26:50.262 | INFO | Analyzing chunk frames = [8097280,8097792) | 20260508 12:26:50.262 | INFO | MemInfo: { | 20260508 12:26:50.262 | INFO | "gpu" : " 17 GB", | 20260508 12:26:50.262 | INFO | "huge" : "", | 20260508 12:26:50.262 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:50.262 | INFO | "malloc" : " 14 GB", | 20260508 12:26:50.262 | INFO | "pinned" : " 29 GB" | 20260508 12:26:50.262 | INFO | } | 20260508 12:26:50.262 | INFO | Perf report for the last 5.13517 seconds: | 20260508 12:26:50.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:50.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 12:26:50.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 12:26:50.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:50.262 | INFO | Uploaded 36GiB to GPU in 2.43556s (14.781GiB/s) | 20260508 12:26:50.262 | INFO | Downloaded 34GiB to GPU in 2.46314s (13.8035GiB/s) | 20260508 12:26:50.359 | INFO | Starting a round of Baz IO | 20260508 12:26:53.757 | INFO | Populating RTMetrics report | 20260508 12:26:53.757 | INFO | RTMetrics report populated | 20260508 12:26:53.757 | INFO | Opening tmp RTMetrics json file | 20260508 12:26:53.763 | INFO | Serializing RTMetrics to json | 20260508 12:26:53.764 | INFO | Serialization done, swapping tmp file now... | 20260508 12:26:53.765 | INFO | RTMetrics done | 20260508 12:26:53.825 | INFO | Finished a round of Baz IO: Wrote 1.408 GiB in 3.465 seconds | 20260508 12:26:53.825 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:26:53.825 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163372% / 0% / 8.34465e-05% | 20260508 12:26:53.825 | INFO | Baz IO overall: | 20260508 12:26:53.825 | INFO | BytesWritten : 3472479860589 | 20260508 12:26:53.825 | INFO | NumEvents : 1666368604797 | 20260508 12:26:55.402 | INFO | Analyzing chunk frames = [8097792,8098304) | 20260508 12:26:55.402 | INFO | MemInfo: { | 20260508 12:26:55.402 | INFO | "gpu" : " 17 GB", | 20260508 12:26:55.402 | INFO | "huge" : "", | 20260508 12:26:55.402 | INFO | "huge_shared" : " 29 GB", | 20260508 12:26:55.402 | INFO | "malloc" : " 14 GB", | 20260508 12:26:55.402 | INFO | "pinned" : " 29 GB" | 20260508 12:26:55.402 | INFO | } | 20260508 12:26:55.402 | INFO | Perf report for the last 5.13986 seconds: | 20260508 12:26:55.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:26:55.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:55.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:55.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:55.402 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:26:55.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:26:55.402 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:26:55.402 | INFO | PRE_HQ : 65.2%, 1.6, 54.1ms, 0 | 20260508 12:26:55.402 | INFO | BAZWRITER : 66.7%, 2.5, 84.3ms, 0 | 20260508 12:26:55.402 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:26:55.402 | INFO | Uploaded 36GiB to GPU in 2.61027s (13.7917GiB/s) | 20260508 12:26:55.402 | INFO | Downloaded 38GiB to GPU in 2.72189s (13.9609GiB/s) | 20260508 12:27:00.540 | INFO | Analyzing chunk frames = [8098304,8098816) | 20260508 12:27:00.540 | INFO | MemInfo: { | 20260508 12:27:00.540 | INFO | "gpu" : " 17 GB", | 20260508 12:27:00.540 | INFO | "huge" : "", | 20260508 12:27:00.540 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:00.540 | INFO | "malloc" : " 14 GB", | 20260508 12:27:00.540 | INFO | "pinned" : " 29 GB" | 20260508 12:27:00.540 | INFO | } | 20260508 12:27:00.540 | INFO | Perf report for the last 5.13833 seconds: | 20260508 12:27:00.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:00.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:27:00.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:27:00.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:00.540 | INFO | Uploaded 36GiB to GPU in 2.48802s (14.4693GiB/s) | 20260508 12:27:00.540 | INFO | Downloaded 34GiB to GPU in 2.46206s (13.8096GiB/s) | 20260508 12:27:05.680 | INFO | Analyzing chunk frames = [8098816,8099328) | 20260508 12:27:05.680 | INFO | MemInfo: { | 20260508 12:27:05.680 | INFO | "gpu" : " 17 GB", | 20260508 12:27:05.680 | INFO | "huge" : "", | 20260508 12:27:05.680 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:05.680 | INFO | "malloc" : " 14 GB", | 20260508 12:27:05.680 | INFO | "pinned" : " 29 GB" | 20260508 12:27:05.680 | INFO | } | 20260508 12:27:05.680 | INFO | Perf report for the last 5.1401 seconds: | 20260508 12:27:05.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:05.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.680 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 12:27:05.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.680 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:27:05.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:05.681 | INFO | Uploaded 36GiB to GPU in 2.5055s (14.3684GiB/s) | 20260508 12:27:05.681 | INFO | Downloaded 34GiB to GPU in 2.4298s (13.9929GiB/s) | 20260508 12:27:10.816 | INFO | Analyzing chunk frames = [8099328,8099840) | 20260508 12:27:10.816 | INFO | MemInfo: { | 20260508 12:27:10.816 | INFO | "gpu" : " 17 GB", | 20260508 12:27:10.816 | INFO | "huge" : "", | 20260508 12:27:10.816 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:10.816 | INFO | "malloc" : " 14 GB", | 20260508 12:27:10.817 | INFO | "pinned" : " 29 GB" | 20260508 12:27:10.817 | INFO | } | 20260508 12:27:10.817 | INFO | Perf report for the last 5.1365 seconds: | 20260508 12:27:10.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:10.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 12:27:10.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:27:10.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:10.817 | INFO | Uploaded 36GiB to GPU in 2.54412s (14.1503GiB/s) | 20260508 12:27:10.817 | INFO | Downloaded 34GiB to GPU in 2.46307s (13.8039GiB/s) | 20260508 12:27:15.964 | INFO | Analyzing chunk frames = [8099840,8100352) | 20260508 12:27:15.964 | INFO | MemInfo: { | 20260508 12:27:15.964 | INFO | "gpu" : " 17 GB", | 20260508 12:27:15.964 | INFO | "huge" : "", | 20260508 12:27:15.964 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:15.964 | INFO | "malloc" : " 14 GB", | 20260508 12:27:15.964 | INFO | "pinned" : " 29 GB" | 20260508 12:27:15.964 | INFO | } | 20260508 12:27:15.964 | INFO | Perf report for the last 5.14767 seconds: | 20260508 12:27:15.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:15.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.965 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 12:27:15.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.965 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:27:15.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:15.965 | INFO | Uploaded 36GiB to GPU in 2.43305s (14.7962GiB/s) | 20260508 12:27:15.965 | INFO | Downloaded 34GiB to GPU in 2.52538s (13.4633GiB/s) | 20260508 12:27:21.102 | INFO | Analyzing chunk frames = [8100352,8100864) | 20260508 12:27:21.102 | INFO | MemInfo: { | 20260508 12:27:21.102 | INFO | "gpu" : " 17 GB", | 20260508 12:27:21.102 | INFO | "huge" : "", | 20260508 12:27:21.102 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:21.102 | INFO | "malloc" : " 14 GB", | 20260508 12:27:21.102 | INFO | "pinned" : " 29 GB" | 20260508 12:27:21.102 | INFO | } | 20260508 12:27:21.102 | INFO | Perf report for the last 5.13734 seconds: | 20260508 12:27:21.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:21.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:27:21.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260508 12:27:21.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:21.102 | INFO | Uploaded 36GiB to GPU in 2.42413s (14.8507GiB/s) | 20260508 12:27:21.102 | INFO | Downloaded 34GiB to GPU in 2.44198s (13.9231GiB/s) | 20260508 12:27:26.238 | INFO | Analyzing chunk frames = [8100864,8101376) | 20260508 12:27:26.238 | INFO | MemInfo: { | 20260508 12:27:26.238 | INFO | "gpu" : " 17 GB", | 20260508 12:27:26.238 | INFO | "huge" : "", | 20260508 12:27:26.238 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:26.238 | INFO | "malloc" : " 14 GB", | 20260508 12:27:26.238 | INFO | "pinned" : " 29 GB" | 20260508 12:27:26.238 | INFO | } | 20260508 12:27:26.238 | INFO | Perf report for the last 5.13658 seconds: | 20260508 12:27:26.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:26.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.238 | INFO | ANALYSIS : 65.3%, 3.0, 101.8ms, 0 | 20260508 12:27:26.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.239 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:27:26.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:26.239 | INFO | Uploaded 36GiB to GPU in 2.42555s (14.842GiB/s) | 20260508 12:27:26.239 | INFO | Downloaded 34GiB to GPU in 2.43989s (13.9351GiB/s) | 20260508 12:27:31.373 | INFO | Analyzing chunk frames = [8101376,8101888) | 20260508 12:27:31.373 | INFO | MemInfo: { | 20260508 12:27:31.373 | INFO | "gpu" : " 17 GB", | 20260508 12:27:31.373 | INFO | "huge" : "", | 20260508 12:27:31.373 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:31.373 | INFO | "malloc" : " 14 GB", | 20260508 12:27:31.373 | INFO | "pinned" : " 29 GB" | 20260508 12:27:31.373 | INFO | } | 20260508 12:27:31.373 | INFO | Perf report for the last 5.13442 seconds: | 20260508 12:27:31.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:31.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 12:27:31.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:27:31.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:31.373 | INFO | Uploaded 36GiB to GPU in 2.44351s (14.7329GiB/s) | 20260508 12:27:31.373 | INFO | Downloaded 34GiB to GPU in 2.44996s (13.8778GiB/s) | 20260508 12:27:31.373 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:27:31.373 | INFO | Total mem usage: 16.7 GiB | 20260508 12:27:31.373 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:27:31.373 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:27:31.373 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:27:31.373 | INFO | High Water Breakdown: | 20260508 12:27:31.373 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:27:31.373 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:27:31.373 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:27:31.374 | INFO | ------------------------------------------ | 20260508 12:27:31.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:27:31.374 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:27:31.374 | INFO | Total mem usage: 26.5 GiB | 20260508 12:27:31.374 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:27:31.374 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:27:31.374 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:27:31.374 | INFO | High Water Breakdown: | 20260508 12:27:31.374 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:27:31.374 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:27:31.374 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:27:31.374 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:27:31.374 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:27:31.374 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:27:31.374 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:27:31.374 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:27:31.374 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:27:31.374 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:27:31.374 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:27:31.374 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:27:31.374 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:27:31.374 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:27:31.374 | INFO | ------------------------------------------ | 20260508 12:27:31.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:27:31.374 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:27:31.374 | INFO | Total mem usage: 27.8 GiB | 20260508 12:27:31.374 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:27:31.374 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:27:31.374 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:27:31.374 | INFO | High Water Breakdown: | 20260508 12:27:31.374 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:27:31.374 | INFO | ------------------------------------------ | 20260508 12:27:31.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:27:31.374 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:27:31.374 | INFO | Total mem usage: 15.7 GiB | 20260508 12:27:31.374 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:27:31.374 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:27:31.374 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:27:31.374 | INFO | High Water Breakdown: | 20260508 12:27:31.374 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:27:31.374 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:27:31.374 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:27:31.374 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:27:31.374 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:27:31.374 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:27:31.374 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:27:31.374 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:27:31.374 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:27:31.374 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:27:31.374 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:27:31.374 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:27:31.374 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:27:31.374 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:27:31.374 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:27:31.374 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:27:31.374 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:27:31.374 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:27:31.374 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:27:31.374 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:27:31.374 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:27:31.374 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:27:31.374 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:27:31.374 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:27:31.374 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:27:31.374 | INFO | ------------------------------------------ | 20260508 12:27:31.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:27:31.374 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:27:31.374 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:27:31.374 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:27:31.374 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 6.31e-01 | 20260508 12:27:31.374 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.03e-01, 1.17e-02 / 4.09e-03 | 20260508 12:27:31.374 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.44e+00, 5.97e-01 / 2.07e-01 | 20260508 12:27:31.374 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.01e-01 | 20260508 12:27:31.374 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.50e+01, 3.63e+00 / 1.13e+00 | 20260508 12:27:31.374 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.80e+00 / 1.52e+00 | 20260508 12:27:31.374 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.49e+00, 4.03e-01 / 2.10e-01 | 20260508 12:27:31.374 | INFO | PerfReport: 2.96%, Metrics -- 9.67e-01 / 1.10e+00, 2.43e-01 / 1.19e-01 | 20260508 12:27:31.374 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.20e-01, 4.76e-02 / 2.50e-02 | 20260508 12:27:31.375 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:27:31.375 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:27:31.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:27:31.375 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 24.48%, Upload -- 2.57e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 31.05%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 2.15%, LockUpload -- 2.26e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 14.97%, LockCompute -- 1.57e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 12:27:31.375 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.34e-02 / 0.00e+00 | 20260508 12:27:31.470 | INFO | Starting a round of Baz IO | 20260508 12:27:34.880 | INFO | Populating RTMetrics report | 20260508 12:27:34.881 | INFO | RTMetrics report populated | 20260508 12:27:34.881 | INFO | Opening tmp RTMetrics json file | 20260508 12:27:34.882 | INFO | Serializing RTMetrics to json | 20260508 12:27:34.883 | INFO | Serialization done, swapping tmp file now... | 20260508 12:27:34.883 | INFO | RTMetrics done | 20260508 12:27:34.941 | INFO | Finished a round of Baz IO: Wrote 1.409 GiB in 3.471 seconds | 20260508 12:27:34.942 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:27:34.942 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163754% / 0% / 6.75519e-05% | 20260508 12:27:34.942 | INFO | Baz IO overall: | 20260508 12:27:34.942 | INFO | BytesWritten : 3473993060396 | 20260508 12:27:34.942 | INFO | NumEvents : 1667078544624 | 20260508 12:27:36.512 | INFO | Analyzing chunk frames = [8101888,8102400) | 20260508 12:27:36.512 | INFO | MemInfo: { | 20260508 12:27:36.512 | INFO | "gpu" : " 17 GB", | 20260508 12:27:36.512 | INFO | "huge" : "", | 20260508 12:27:36.512 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:36.512 | INFO | "malloc" : " 14 GB", | 20260508 12:27:36.512 | INFO | "pinned" : " 29 GB" | 20260508 12:27:36.512 | INFO | } | 20260508 12:27:36.512 | INFO | Perf report for the last 5.13964 seconds: | 20260508 12:27:36.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:36.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:36.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:36.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:36.513 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 12:27:36.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:36.513 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 12:27:36.513 | INFO | PRE_HQ : 65.7%, 1.6, 53.8ms, 0 | 20260508 12:27:36.513 | INFO | BAZWRITER : 66.3%, 2.5, 83.6ms, 0 | 20260508 12:27:36.513 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 12:27:36.513 | INFO | Uploaded 36GiB to GPU in 2.5408s (14.1688GiB/s) | 20260508 12:27:36.513 | INFO | Downloaded 38GiB to GPU in 2.71798s (13.9809GiB/s) | 20260508 12:27:41.658 | INFO | Analyzing chunk frames = [8102400,8102912) | 20260508 12:27:41.658 | INFO | MemInfo: { | 20260508 12:27:41.658 | INFO | "gpu" : " 17 GB", | 20260508 12:27:41.658 | INFO | "huge" : "", | 20260508 12:27:41.658 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:41.658 | INFO | "malloc" : " 14 GB", | 20260508 12:27:41.658 | INFO | "pinned" : " 29 GB" | 20260508 12:27:41.658 | INFO | } | 20260508 12:27:41.658 | INFO | Perf report for the last 5.14596 seconds: | 20260508 12:27:41.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:41.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.659 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:27:41.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.659 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:27:41.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:41.659 | INFO | Uploaded 36GiB to GPU in 2.46478s (14.6057GiB/s) | 20260508 12:27:41.659 | INFO | Downloaded 34GiB to GPU in 2.48754s (13.6681GiB/s) | 20260508 12:27:46.802 | INFO | Analyzing chunk frames = [8102912,8103424) | 20260508 12:27:46.802 | INFO | MemInfo: { | 20260508 12:27:46.802 | INFO | "gpu" : " 17 GB", | 20260508 12:27:46.802 | INFO | "huge" : "", | 20260508 12:27:46.802 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:46.802 | INFO | "malloc" : " 14 GB", | 20260508 12:27:46.802 | INFO | "pinned" : " 29 GB" | 20260508 12:27:46.803 | INFO | } | 20260508 12:27:46.803 | INFO | Perf report for the last 5.14419 seconds: | 20260508 12:27:46.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:46.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 12:27:46.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:27:46.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:46.803 | INFO | Uploaded 36GiB to GPU in 2.45471s (14.6657GiB/s) | 20260508 12:27:46.803 | INFO | Downloaded 34GiB to GPU in 2.44996s (13.8778GiB/s) | 20260508 12:27:51.938 | INFO | Analyzing chunk frames = [8103424,8103936) | 20260508 12:27:51.938 | INFO | MemInfo: { | 20260508 12:27:51.938 | INFO | "gpu" : " 17 GB", | 20260508 12:27:51.938 | INFO | "huge" : "", | 20260508 12:27:51.938 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:51.938 | INFO | "malloc" : " 14 GB", | 20260508 12:27:51.938 | INFO | "pinned" : " 29 GB" | 20260508 12:27:51.938 | INFO | } | 20260508 12:27:51.938 | INFO | Perf report for the last 5.1357 seconds: | 20260508 12:27:51.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:51.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.938 | INFO | ANALYSIS : 65.2%, 3.0, 101.1ms, 0 | 20260508 12:27:51.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.939 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:27:51.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:51.939 | INFO | Uploaded 36GiB to GPU in 2.35057s (15.3154GiB/s) | 20260508 12:27:51.939 | INFO | Downloaded 34GiB to GPU in 2.48845s (13.6631GiB/s) | 20260508 12:27:57.072 | INFO | Analyzing chunk frames = [8103936,8104448) | 20260508 12:27:57.072 | INFO | MemInfo: { | 20260508 12:27:57.072 | INFO | "gpu" : " 17 GB", | 20260508 12:27:57.072 | INFO | "huge" : "", | 20260508 12:27:57.072 | INFO | "huge_shared" : " 29 GB", | 20260508 12:27:57.072 | INFO | "malloc" : " 14 GB", | 20260508 12:27:57.072 | INFO | "pinned" : " 29 GB" | 20260508 12:27:57.072 | INFO | } | 20260508 12:27:57.073 | INFO | Perf report for the last 5.13417 seconds: | 20260508 12:27:57.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:27:57.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 12:27:57.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 12:27:57.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:27:57.073 | INFO | Uploaded 36GiB to GPU in 2.40512s (14.9681GiB/s) | 20260508 12:27:57.073 | INFO | Downloaded 34GiB to GPU in 2.52329s (13.4745GiB/s) | 20260508 12:28:02.215 | INFO | Analyzing chunk frames = [8104448,8104960) | 20260508 12:28:02.215 | INFO | MemInfo: { | 20260508 12:28:02.215 | INFO | "gpu" : " 17 GB", | 20260508 12:28:02.215 | INFO | "huge" : "", | 20260508 12:28:02.215 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:02.215 | INFO | "malloc" : " 14 GB", | 20260508 12:28:02.215 | INFO | "pinned" : " 29 GB" | 20260508 12:28:02.215 | INFO | } | 20260508 12:28:02.215 | INFO | Perf report for the last 5.14258 seconds: | 20260508 12:28:02.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:02.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:28:02.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:28:02.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:02.215 | INFO | Uploaded 36GiB to GPU in 2.37952s (15.1291GiB/s) | 20260508 12:28:02.215 | INFO | Downloaded 34GiB to GPU in 2.44324s (13.9159GiB/s) | 20260508 12:28:07.354 | INFO | Analyzing chunk frames = [8104960,8105472) | 20260508 12:28:07.354 | INFO | MemInfo: { | 20260508 12:28:07.354 | INFO | "gpu" : " 17 GB", | 20260508 12:28:07.354 | INFO | "huge" : "", | 20260508 12:28:07.354 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:07.354 | INFO | "malloc" : " 14 GB", | 20260508 12:28:07.354 | INFO | "pinned" : " 29 GB" | 20260508 12:28:07.354 | INFO | } | 20260508 12:28:07.354 | INFO | Perf report for the last 5.13888 seconds: | 20260508 12:28:07.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:07.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.354 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 12:28:07.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.354 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:28:07.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:07.355 | INFO | Uploaded 36GiB to GPU in 2.43894s (14.7605GiB/s) | 20260508 12:28:07.355 | INFO | Downloaded 34GiB to GPU in 2.46092s (13.8159GiB/s) | 20260508 12:28:12.496 | INFO | Analyzing chunk frames = [8105472,8105984) | 20260508 12:28:12.496 | INFO | MemInfo: { | 20260508 12:28:12.496 | INFO | "gpu" : " 17 GB", | 20260508 12:28:12.496 | INFO | "huge" : "", | 20260508 12:28:12.496 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:12.496 | INFO | "malloc" : " 14 GB", | 20260508 12:28:12.496 | INFO | "pinned" : " 29 GB" | 20260508 12:28:12.496 | INFO | } | 20260508 12:28:12.496 | INFO | Perf report for the last 5.14187 seconds: | 20260508 12:28:12.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:12.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:28:12.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:28:12.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:12.496 | INFO | Uploaded 36GiB to GPU in 2.46856s (14.5834GiB/s) | 20260508 12:28:12.496 | INFO | Downloaded 34GiB to GPU in 2.47019s (13.7641GiB/s) | 20260508 12:28:12.593 | INFO | Starting a round of Baz IO | 20260508 12:28:15.989 | INFO | Populating RTMetrics report | 20260508 12:28:15.990 | INFO | RTMetrics report populated | 20260508 12:28:15.990 | INFO | Opening tmp RTMetrics json file | 20260508 12:28:15.998 | INFO | Serializing RTMetrics to json | 20260508 12:28:16.000 | INFO | Serialization done, swapping tmp file now... | 20260508 12:28:16.000 | INFO | RTMetrics done | 20260508 12:28:16.061 | INFO | Finished a round of Baz IO: Wrote 1.409 GiB in 3.467 seconds | 20260508 12:28:16.061 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:28:16.061 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162594% / 3.97364e-06% / 6.75519e-05% | 20260508 12:28:16.061 | INFO | Baz IO overall: | 20260508 12:28:16.061 | INFO | BytesWritten : 3475505462474 | 20260508 12:28:16.061 | INFO | NumEvents : 1667787787364 | 20260508 12:28:17.631 | INFO | Analyzing chunk frames = [8105984,8106496) | 20260508 12:28:17.631 | INFO | MemInfo: { | 20260508 12:28:17.631 | INFO | "gpu" : " 17 GB", | 20260508 12:28:17.632 | INFO | "huge" : "", | 20260508 12:28:17.632 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:17.632 | INFO | "malloc" : " 14 GB", | 20260508 12:28:17.632 | INFO | "pinned" : " 29 GB" | 20260508 12:28:17.632 | INFO | } | 20260508 12:28:17.632 | INFO | Perf report for the last 5.13574 seconds: | 20260508 12:28:17.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:17.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:17.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:17.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:17.632 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:28:17.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:17.632 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:28:17.632 | INFO | PRE_HQ : 65.5%, 1.6, 54.1ms, 0 | 20260508 12:28:17.632 | INFO | BAZWRITER : 66.8%, 2.4, 84.0ms, 0 | 20260508 12:28:17.632 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:28:17.632 | INFO | Uploaded 36GiB to GPU in 2.62985s (13.689GiB/s) | 20260508 12:28:17.632 | INFO | Downloaded 38GiB to GPU in 2.72852s (13.9269GiB/s) | 20260508 12:28:22.773 | INFO | Analyzing chunk frames = [8106496,8107008) | 20260508 12:28:22.773 | INFO | MemInfo: { | 20260508 12:28:22.773 | INFO | "gpu" : " 17 GB", | 20260508 12:28:22.773 | INFO | "huge" : "", | 20260508 12:28:22.773 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:22.773 | INFO | "malloc" : " 14 GB", | 20260508 12:28:22.773 | INFO | "pinned" : " 29 GB" | 20260508 12:28:22.773 | INFO | } | 20260508 12:28:22.773 | INFO | Perf report for the last 5.14157 seconds: | 20260508 12:28:22.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:22.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.773 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:28:22.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.773 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:28:22.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:22.774 | INFO | Uploaded 36GiB to GPU in 2.43878s (14.7615GiB/s) | 20260508 12:28:22.774 | INFO | Downloaded 34GiB to GPU in 2.46766s (13.7782GiB/s) | 20260508 12:28:27.911 | INFO | Analyzing chunk frames = [8107008,8107520) | 20260508 12:28:27.911 | INFO | MemInfo: { | 20260508 12:28:27.911 | INFO | "gpu" : " 17 GB", | 20260508 12:28:27.911 | INFO | "huge" : "", | 20260508 12:28:27.912 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:27.912 | INFO | "malloc" : " 14 GB", | 20260508 12:28:27.912 | INFO | "pinned" : " 29 GB" | 20260508 12:28:27.912 | INFO | } | 20260508 12:28:27.912 | INFO | Perf report for the last 5.13847 seconds: | 20260508 12:28:27.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:27.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260508 12:28:27.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 12:28:27.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:27.912 | INFO | Uploaded 36GiB to GPU in 2.45053s (14.6907GiB/s) | 20260508 12:28:27.913 | INFO | Downloaded 34GiB to GPU in 2.45893s (13.8272GiB/s) | 20260508 12:28:33.051 | INFO | Analyzing chunk frames = [8107520,8108032) | 20260508 12:28:33.051 | INFO | MemInfo: { | 20260508 12:28:33.051 | INFO | "gpu" : " 17 GB", | 20260508 12:28:33.051 | INFO | "huge" : "", | 20260508 12:28:33.051 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:33.051 | INFO | "malloc" : " 14 GB", | 20260508 12:28:33.051 | INFO | "pinned" : " 29 GB" | 20260508 12:28:33.051 | INFO | } | 20260508 12:28:33.052 | INFO | Perf report for the last 5.13977 seconds: | 20260508 12:28:33.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:33.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 12:28:33.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260508 12:28:33.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:33.052 | INFO | Uploaded 36GiB to GPU in 2.40353s (14.9779GiB/s) | 20260508 12:28:33.052 | INFO | Downloaded 34GiB to GPU in 2.47864s (13.7172GiB/s) | 20260508 12:28:38.192 | INFO | Analyzing chunk frames = [8108032,8108544) | 20260508 12:28:38.192 | INFO | MemInfo: { | 20260508 12:28:38.192 | INFO | "gpu" : " 17 GB", | 20260508 12:28:38.192 | INFO | "huge" : "", | 20260508 12:28:38.192 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:38.192 | INFO | "malloc" : " 14 GB", | 20260508 12:28:38.192 | INFO | "pinned" : " 29 GB" | 20260508 12:28:38.192 | INFO | } | 20260508 12:28:38.192 | INFO | Perf report for the last 5.14048 seconds: | 20260508 12:28:38.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:38.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.192 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:28:38.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.192 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260508 12:28:38.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:38.193 | INFO | Uploaded 36GiB to GPU in 2.29338s (15.6974GiB/s) | 20260508 12:28:38.193 | INFO | Downloaded 34GiB to GPU in 2.51473s (13.5203GiB/s) | 20260508 12:28:43.325 | INFO | Analyzing chunk frames = [8108544,8109056) | 20260508 12:28:43.325 | INFO | MemInfo: { | 20260508 12:28:43.325 | INFO | "gpu" : " 17 GB", | 20260508 12:28:43.325 | INFO | "huge" : "", | 20260508 12:28:43.325 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:43.326 | INFO | "malloc" : " 14 GB", | 20260508 12:28:43.326 | INFO | "pinned" : " 29 GB" | 20260508 12:28:43.326 | INFO | } | 20260508 12:28:43.326 | INFO | Perf report for the last 5.13375 seconds: | 20260508 12:28:43.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:43.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 12:28:43.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:28:43.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:43.326 | INFO | Uploaded 36GiB to GPU in 2.36971s (15.1918GiB/s) | 20260508 12:28:43.326 | INFO | Downloaded 34GiB to GPU in 2.46342s (13.802GiB/s) | 20260508 12:28:48.474 | INFO | Analyzing chunk frames = [8109056,8109568) | 20260508 12:28:48.474 | INFO | MemInfo: { | 20260508 12:28:48.474 | INFO | "gpu" : " 17 GB", | 20260508 12:28:48.474 | INFO | "huge" : "", | 20260508 12:28:48.474 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:48.475 | INFO | "malloc" : " 14 GB", | 20260508 12:28:48.475 | INFO | "pinned" : " 29 GB" | 20260508 12:28:48.475 | INFO | } | 20260508 12:28:48.475 | INFO | Perf report for the last 5.14881 seconds: | 20260508 12:28:48.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:48.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | ANALYSIS : 65.2%, 3.0, 101.9ms, 0 | 20260508 12:28:48.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:28:48.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:48.475 | INFO | Uploaded 36GiB to GPU in 2.41117s (14.9305GiB/s) | 20260508 12:28:48.475 | INFO | Downloaded 34GiB to GPU in 2.46917s (13.7698GiB/s) | 20260508 12:28:53.613 | INFO | Analyzing chunk frames = [8109568,8110080) | 20260508 12:28:53.613 | INFO | MemInfo: { | 20260508 12:28:53.613 | INFO | "gpu" : " 17 GB", | 20260508 12:28:53.613 | INFO | "huge" : "", | 20260508 12:28:53.613 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:53.613 | INFO | "malloc" : " 14 GB", | 20260508 12:28:53.613 | INFO | "pinned" : " 29 GB" | 20260508 12:28:53.613 | INFO | } | 20260508 12:28:53.613 | INFO | Perf report for the last 5.13881 seconds: | 20260508 12:28:53.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:53.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.613 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:28:53.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.614 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:28:53.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:53.614 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:28:53.614 | INFO | Total mem usage: 16.7 GiB | 20260508 12:28:53.614 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:28:53.614 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:28:53.614 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:28:53.614 | INFO | High Water Breakdown: | 20260508 12:28:53.614 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:28:53.614 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:28:53.614 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:28:53.614 | INFO | ------------------------------------------ | 20260508 12:28:53.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:28:53.614 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:28:53.614 | INFO | Total mem usage: 26.5 GiB | 20260508 12:28:53.614 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:28:53.614 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:28:53.614 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:28:53.614 | INFO | High Water Breakdown: | 20260508 12:28:53.614 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:28:53.614 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:28:53.614 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:28:53.614 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:28:53.614 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:28:53.614 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:28:53.614 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:28:53.614 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:28:53.614 | INFO | Uploaded 36GiB to GPU in 2.39279s (15.0452GiB/s) | 20260508 12:28:53.614 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:28:53.614 | INFO | Downloaded 34GiB to GPU in 2.47097s (13.7598GiB/s) | 20260508 12:28:53.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:28:53.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:28:53.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:28:53.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:28:53.615 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:28:53.615 | INFO | ------------------------------------------ | 20260508 12:28:53.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:28:53.615 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:28:53.615 | INFO | Total mem usage: 27.8 GiB | 20260508 12:28:53.615 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:28:53.615 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:28:53.615 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:28:53.615 | INFO | High Water Breakdown: | 20260508 12:28:53.615 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:28:53.615 | INFO | ------------------------------------------ | 20260508 12:28:53.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:28:53.615 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:28:53.615 | INFO | Total mem usage: 15.7 GiB | 20260508 12:28:53.615 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:28:53.615 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:28:53.615 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:28:53.615 | INFO | High Water Breakdown: | 20260508 12:28:53.615 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:28:53.615 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:28:53.615 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:28:53.615 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:28:53.615 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:28:53.615 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:28:53.615 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:28:53.615 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:28:53.615 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:28:53.615 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:28:53.615 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:28:53.615 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:28:53.615 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:28:53.615 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:28:53.615 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:28:53.615 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:28:53.615 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:28:53.615 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:28:53.615 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:28:53.615 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:28:53.615 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:28:53.615 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:28:53.615 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:28:53.615 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:28:53.615 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:28:53.615 | INFO | ------------------------------------------ | 20260508 12:28:53.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:28:53.615 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:28:53.615 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:28:53.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:28:53.615 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.42e+01 / 6.40e-01 | 20260508 12:28:53.615 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.05e-01, 1.12e-02 / 7.63e-03 | 20260508 12:28:53.615 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.53e+00, 5.98e-01 / 1.89e-01 | 20260508 12:28:53.615 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.48e+00, 2.10e-01 / 1.27e-01 | 20260508 12:28:53.615 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.54e+01, 3.68e+00 / 9.76e-01 | 20260508 12:28:53.615 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.87e+01 / 1.97e+01, 1.83e+00 / 1.07e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.56e+00, 4.02e-01 / 1.62e-01 | 20260508 12:28:53.615 | INFO | PerfReport: 2.97%, Metrics -- 9.70e-01 / 1.10e+00, 2.41e-01 / 8.33e-02 | 20260508 12:28:53.615 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.31e-01, 4.74e-02 / 2.54e-02 | 20260508 12:28:53.615 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:28:53.615 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:28:53.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:28:53.615 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 24.15%, Upload -- 2.53e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 24.90%, Download -- 2.61e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 31.19%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 1.96%, LockUpload -- 2.06e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 2.74%, LockDownload -- 2.87e+00 / 0.00e+00, 4.31e+00 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 15.05%, LockCompute -- 1.58e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 12:28:53.615 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260508 12:28:53.712 | INFO | Starting a round of Baz IO | 20260508 12:28:57.112 | INFO | Populating RTMetrics report | 20260508 12:28:57.113 | INFO | RTMetrics report populated | 20260508 12:28:57.113 | INFO | Opening tmp RTMetrics json file | 20260508 12:28:57.118 | INFO | Serializing RTMetrics to json | 20260508 12:28:57.120 | INFO | Serialization done, swapping tmp file now... | 20260508 12:28:57.120 | INFO | RTMetrics done | 20260508 12:28:57.178 | INFO | Finished a round of Baz IO: Wrote 1.409 GiB in 3.466 seconds | 20260508 12:28:57.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:28:57.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162721% / 0% / 8.34465e-05% | 20260508 12:28:57.178 | INFO | Baz IO overall: | 20260508 12:28:57.178 | INFO | BytesWritten : 3477018436891 | 20260508 12:28:57.178 | INFO | NumEvents : 1668497605838 | 20260508 12:28:58.746 | INFO | Analyzing chunk frames = [8110080,8110592) | 20260508 12:28:58.746 | INFO | MemInfo: { | 20260508 12:28:58.746 | INFO | "gpu" : " 17 GB", | 20260508 12:28:58.746 | INFO | "huge" : "", | 20260508 12:28:58.746 | INFO | "huge_shared" : " 29 GB", | 20260508 12:28:58.746 | INFO | "malloc" : " 14 GB", | 20260508 12:28:58.746 | INFO | "pinned" : " 29 GB" | 20260508 12:28:58.746 | INFO | } | 20260508 12:28:58.747 | INFO | Perf report for the last 5.13308 seconds: | 20260508 12:28:58.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:28:58.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:58.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:58.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:58.747 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:28:58.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:28:58.747 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:28:58.747 | INFO | PRE_HQ : 65.3%, 1.6, 54.0ms, 0 | 20260508 12:28:58.747 | INFO | BAZWRITER : 66.8%, 2.4, 83.6ms, 0 | 20260508 12:28:58.747 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:28:58.747 | INFO | Uploaded 36GiB to GPU in 2.60891s (13.7989GiB/s) | 20260508 12:28:58.747 | INFO | Downloaded 38GiB to GPU in 2.719s (13.9758GiB/s) | 20260508 12:29:03.883 | INFO | Analyzing chunk frames = [8110592,8111104) | 20260508 12:29:03.883 | INFO | MemInfo: { | 20260508 12:29:03.883 | INFO | "gpu" : " 17 GB", | 20260508 12:29:03.883 | INFO | "huge" : "", | 20260508 12:29:03.883 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:03.883 | INFO | "malloc" : " 14 GB", | 20260508 12:29:03.883 | INFO | "pinned" : " 29 GB" | 20260508 12:29:03.883 | INFO | } | 20260508 12:29:03.883 | INFO | Perf report for the last 5.13634 seconds: | 20260508 12:29:03.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:03.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:29:03.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:29:03.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:03.883 | INFO | Uploaded 36GiB to GPU in 2.45136s (14.6857GiB/s) | 20260508 12:29:03.884 | INFO | Downloaded 34GiB to GPU in 2.45831s (13.8307GiB/s) | 20260508 12:29:09.023 | INFO | Analyzing chunk frames = [8111104,8111616) | 20260508 12:29:09.023 | INFO | MemInfo: { | 20260508 12:29:09.023 | INFO | "gpu" : " 17 GB", | 20260508 12:29:09.023 | INFO | "huge" : "", | 20260508 12:29:09.023 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:09.023 | INFO | "malloc" : " 14 GB", | 20260508 12:29:09.023 | INFO | "pinned" : " 29 GB" | 20260508 12:29:09.023 | INFO | } | 20260508 12:29:09.023 | INFO | Perf report for the last 5.14036 seconds: | 20260508 12:29:09.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:09.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.023 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 12:29:09.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.023 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:29:09.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:09.024 | INFO | Uploaded 36GiB to GPU in 2.51044s (14.3401GiB/s) | 20260508 12:29:09.024 | INFO | Downloaded 34GiB to GPU in 2.45516s (13.8484GiB/s) | 20260508 12:29:14.164 | INFO | Analyzing chunk frames = [8111616,8112128) | 20260508 12:29:14.164 | INFO | MemInfo: { | 20260508 12:29:14.164 | INFO | "gpu" : " 17 GB", | 20260508 12:29:14.164 | INFO | "huge" : "", | 20260508 12:29:14.164 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:14.164 | INFO | "malloc" : " 14 GB", | 20260508 12:29:14.165 | INFO | "pinned" : " 29 GB" | 20260508 12:29:14.165 | INFO | } | 20260508 12:29:14.165 | INFO | Perf report for the last 5.14132 seconds: | 20260508 12:29:14.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:14.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.165 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:29:14.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.165 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:29:14.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:14.166 | INFO | Uploaded 36GiB to GPU in 2.48698s (14.4754GiB/s) | 20260508 12:29:14.166 | INFO | Downloaded 34GiB to GPU in 2.4458s (13.9014GiB/s) | 20260508 12:29:19.309 | INFO | Analyzing chunk frames = [8112128,8112640) | 20260508 12:29:19.309 | INFO | MemInfo: { | 20260508 12:29:19.309 | INFO | "gpu" : " 17 GB", | 20260508 12:29:19.309 | INFO | "huge" : "", | 20260508 12:29:19.309 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:19.309 | INFO | "malloc" : " 14 GB", | 20260508 12:29:19.309 | INFO | "pinned" : " 29 GB" | 20260508 12:29:19.309 | INFO | } | 20260508 12:29:19.309 | INFO | Perf report for the last 5.14463 seconds: | 20260508 12:29:19.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:19.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.309 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 12:29:19.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.309 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:29:19.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:19.310 | INFO | Uploaded 36GiB to GPU in 2.38287s (15.1078GiB/s) | 20260508 12:29:19.310 | INFO | Downloaded 34GiB to GPU in 2.50307s (13.5833GiB/s) | 20260508 12:29:24.444 | INFO | Analyzing chunk frames = [8112640,8113152) | 20260508 12:29:24.444 | INFO | MemInfo: { | 20260508 12:29:24.444 | INFO | "gpu" : " 17 GB", | 20260508 12:29:24.444 | INFO | "huge" : "", | 20260508 12:29:24.444 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:24.444 | INFO | "malloc" : " 14 GB", | 20260508 12:29:24.445 | INFO | "pinned" : " 29 GB" | 20260508 12:29:24.445 | INFO | } | 20260508 12:29:24.445 | INFO | Perf report for the last 5.13534 seconds: | 20260508 12:29:24.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:24.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:29:24.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:29:24.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:24.445 | INFO | Uploaded 36GiB to GPU in 2.47078s (14.5703GiB/s) | 20260508 12:29:24.445 | INFO | Downloaded 34GiB to GPU in 2.43657s (13.954GiB/s) | 20260508 12:29:29.577 | INFO | Analyzing chunk frames = [8113152,8113664) | 20260508 12:29:29.577 | INFO | MemInfo: { | 20260508 12:29:29.577 | INFO | "gpu" : " 17 GB", | 20260508 12:29:29.577 | INFO | "huge" : "", | 20260508 12:29:29.577 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:29.577 | INFO | "malloc" : " 14 GB", | 20260508 12:29:29.577 | INFO | "pinned" : " 29 GB" | 20260508 12:29:29.577 | INFO | } | 20260508 12:29:29.577 | INFO | Perf report for the last 5.13291 seconds: | 20260508 12:29:29.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:29.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:29:29.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | PRE_HQ : 22.9%, 1.0, 11.8ms, 0 | 20260508 12:29:29.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:29.578 | INFO | Uploaded 36GiB to GPU in 2.47481s (14.5466GiB/s) | 20260508 12:29:29.578 | INFO | Downloaded 34GiB to GPU in 2.44892s (13.8837GiB/s) | 20260508 12:29:34.720 | INFO | Analyzing chunk frames = [8113664,8114176) | 20260508 12:29:34.720 | INFO | MemInfo: { | 20260508 12:29:34.720 | INFO | "gpu" : " 17 GB", | 20260508 12:29:34.720 | INFO | "huge" : "", | 20260508 12:29:34.720 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:34.720 | INFO | "malloc" : " 14 GB", | 20260508 12:29:34.721 | INFO | "pinned" : " 29 GB" | 20260508 12:29:34.721 | INFO | } | 20260508 12:29:34.721 | INFO | Perf report for the last 5.14309 seconds: | 20260508 12:29:34.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:34.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:29:34.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:29:34.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:34.721 | INFO | Uploaded 36GiB to GPU in 2.47284s (14.5582GiB/s) | 20260508 12:29:34.721 | INFO | Downloaded 34GiB to GPU in 2.44478s (13.9072GiB/s) | 20260508 12:29:34.818 | INFO | Starting a round of Baz IO | 20260508 12:29:38.226 | INFO | Populating RTMetrics report | 20260508 12:29:38.227 | INFO | RTMetrics report populated | 20260508 12:29:38.227 | INFO | Opening tmp RTMetrics json file | 20260508 12:29:38.235 | INFO | Serializing RTMetrics to json | 20260508 12:29:38.236 | INFO | Serialization done, swapping tmp file now... | 20260508 12:29:38.237 | INFO | RTMetrics done | 20260508 12:29:38.296 | INFO | Finished a round of Baz IO: Wrote 1.408 GiB in 3.478 seconds | 20260508 12:29:38.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:29:38.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162383% / 0% / 9.13938e-05% | 20260508 12:29:38.296 | INFO | Baz IO overall: | 20260508 12:29:38.296 | INFO | BytesWritten : 3478529782683 | 20260508 12:29:38.296 | INFO | NumEvents : 1669206297311 | 20260508 12:29:39.865 | INFO | Analyzing chunk frames = [8114176,8114688) | 20260508 12:29:39.865 | INFO | MemInfo: { | 20260508 12:29:39.865 | INFO | "gpu" : " 17 GB", | 20260508 12:29:39.865 | INFO | "huge" : "", | 20260508 12:29:39.865 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:39.865 | INFO | "malloc" : " 14 GB", | 20260508 12:29:39.865 | INFO | "pinned" : " 29 GB" | 20260508 12:29:39.865 | INFO | } | 20260508 12:29:39.865 | INFO | Perf report for the last 5.14474 seconds: | 20260508 12:29:39.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:39.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:39.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:39.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:39.865 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 12:29:39.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:39.865 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:29:39.865 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:29:39.866 | INFO | BAZWRITER : 66.8%, 2.4, 83.8ms, 0 | 20260508 12:29:39.866 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:29:39.866 | INFO | Uploaded 36GiB to GPU in 2.64639s (13.6034GiB/s) | 20260508 12:29:39.866 | INFO | Downloaded 38GiB to GPU in 2.70129s (14.0674GiB/s) | 20260508 12:29:44.997 | INFO | Analyzing chunk frames = [8114688,8115200) | 20260508 12:29:44.997 | INFO | MemInfo: { | 20260508 12:29:44.997 | INFO | "gpu" : " 17 GB", | 20260508 12:29:44.997 | INFO | "huge" : "", | 20260508 12:29:44.997 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:44.997 | INFO | "malloc" : " 14 GB", | 20260508 12:29:44.997 | INFO | "pinned" : " 29 GB" | 20260508 12:29:44.997 | INFO | } | 20260508 12:29:44.997 | INFO | Perf report for the last 5.13202 seconds: | 20260508 12:29:44.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:44.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.997 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 12:29:44.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.998 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 12:29:44.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:44.998 | INFO | Uploaded 36GiB to GPU in 2.4881s (14.4688GiB/s) | 20260508 12:29:44.998 | INFO | Downloaded 34GiB to GPU in 2.46726s (13.7805GiB/s) | 20260508 12:29:50.144 | INFO | Analyzing chunk frames = [8115200,8115712) | 20260508 12:29:50.144 | INFO | MemInfo: { | 20260508 12:29:50.144 | INFO | "gpu" : " 17 GB", | 20260508 12:29:50.144 | INFO | "huge" : "", | 20260508 12:29:50.144 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:50.144 | INFO | "malloc" : " 14 GB", | 20260508 12:29:50.144 | INFO | "pinned" : " 29 GB" | 20260508 12:29:50.144 | INFO | } | 20260508 12:29:50.144 | INFO | Perf report for the last 5.14675 seconds: | 20260508 12:29:50.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:50.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.144 | INFO | ANALYSIS : 66.7%, 3.0, 103.7ms, 0 | 20260508 12:29:50.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.144 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260508 12:29:50.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:50.145 | INFO | Uploaded 36GiB to GPU in 2.53757s (14.1868GiB/s) | 20260508 12:29:50.145 | INFO | Downloaded 34GiB to GPU in 2.45425s (13.8535GiB/s) | 20260508 12:29:55.281 | INFO | Analyzing chunk frames = [8115712,8116224) | 20260508 12:29:55.281 | INFO | MemInfo: { | 20260508 12:29:55.281 | INFO | "gpu" : " 17 GB", | 20260508 12:29:55.281 | INFO | "huge" : "", | 20260508 12:29:55.281 | INFO | "huge_shared" : " 29 GB", | 20260508 12:29:55.281 | INFO | "malloc" : " 14 GB", | 20260508 12:29:55.281 | INFO | "pinned" : " 29 GB" | 20260508 12:29:55.281 | INFO | } | 20260508 12:29:55.281 | INFO | Perf report for the last 5.13728 seconds: | 20260508 12:29:55.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:29:55.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 12:29:55.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 12:29:55.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:29:55.282 | INFO | Uploaded 36GiB to GPU in 2.44141s (14.7456GiB/s) | 20260508 12:29:55.282 | INFO | Downloaded 34GiB to GPU in 2.47156s (13.7565GiB/s) | 20260508 12:30:00.422 | INFO | Analyzing chunk frames = [8116224,8116736) | 20260508 12:30:00.422 | INFO | MemInfo: { | 20260508 12:30:00.422 | INFO | "gpu" : " 17 GB", | 20260508 12:30:00.422 | INFO | "huge" : "", | 20260508 12:30:00.422 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:00.422 | INFO | "malloc" : " 14 GB", | 20260508 12:30:00.422 | INFO | "pinned" : " 29 GB" | 20260508 12:30:00.422 | INFO | } | 20260508 12:30:00.422 | INFO | Perf report for the last 5.14048 seconds: | 20260508 12:30:00.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:00.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | ANALYSIS : 66.6%, 3.0, 103.9ms, 0 | 20260508 12:30:00.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 12:30:00.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:00.422 | INFO | Uploaded 36GiB to GPU in 2.52201s (14.2743GiB/s) | 20260508 12:30:00.422 | INFO | Downloaded 34GiB to GPU in 2.52897s (13.4442GiB/s) | 20260508 12:30:05.561 | INFO | Analyzing chunk frames = [8116736,8117248) | 20260508 12:30:05.561 | INFO | MemInfo: { | 20260508 12:30:05.561 | INFO | "gpu" : " 17 GB", | 20260508 12:30:05.561 | INFO | "huge" : "", | 20260508 12:30:05.561 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:05.561 | INFO | "malloc" : " 14 GB", | 20260508 12:30:05.561 | INFO | "pinned" : " 29 GB" | 20260508 12:30:05.561 | INFO | } | 20260508 12:30:05.561 | INFO | Perf report for the last 5.13937 seconds: | 20260508 12:30:05.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:05.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.561 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 12:30:05.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.561 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:30:05.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:05.562 | INFO | Uploaded 36GiB to GPU in 2.42222s (14.8624GiB/s) | 20260508 12:30:05.562 | INFO | Downloaded 34GiB to GPU in 2.42941s (13.9952GiB/s) | 20260508 12:30:10.697 | INFO | Analyzing chunk frames = [8117248,8117760) | 20260508 12:30:10.697 | INFO | MemInfo: { | 20260508 12:30:10.697 | INFO | "gpu" : " 17 GB", | 20260508 12:30:10.697 | INFO | "huge" : "", | 20260508 12:30:10.698 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:10.698 | INFO | "malloc" : " 14 GB", | 20260508 12:30:10.698 | INFO | "pinned" : " 29 GB" | 20260508 12:30:10.698 | INFO | } | 20260508 12:30:10.698 | INFO | Perf report for the last 5.13665 seconds: | 20260508 12:30:10.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:10.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:30:10.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:30:10.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:10.698 | INFO | Uploaded 36GiB to GPU in 2.47696s (14.534GiB/s) | 20260508 12:30:10.698 | INFO | Downloaded 34GiB to GPU in 2.44881s (13.8843GiB/s) | 20260508 12:30:15.832 | INFO | Analyzing chunk frames = [8117760,8118272) | 20260508 12:30:15.832 | INFO | MemInfo: { | 20260508 12:30:15.832 | INFO | "gpu" : " 17 GB", | 20260508 12:30:15.832 | INFO | "huge" : "", | 20260508 12:30:15.832 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:15.832 | INFO | "malloc" : " 14 GB", | 20260508 12:30:15.832 | INFO | "pinned" : " 29 GB" | 20260508 12:30:15.832 | INFO | } | 20260508 12:30:15.832 | INFO | Perf report for the last 5.13426 seconds: | 20260508 12:30:15.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:15.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.832 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260508 12:30:15.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.832 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:30:15.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:15.833 | INFO | Uploaded 36GiB to GPU in 2.41532s (14.9048GiB/s) | 20260508 12:30:15.833 | INFO | Downloaded 34GiB to GPU in 2.42329s (14.0305GiB/s) | 20260508 12:30:15.833 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:30:15.833 | INFO | Total mem usage: 16.7 GiB | 20260508 12:30:15.833 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:30:15.833 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:30:15.833 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:30:15.833 | INFO | High Water Breakdown: | 20260508 12:30:15.833 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:30:15.833 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:30:15.833 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:30:15.833 | INFO | ------------------------------------------ | 20260508 12:30:15.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:30:15.833 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:30:15.833 | INFO | Total mem usage: 26.5 GiB | 20260508 12:30:15.833 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:30:15.833 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:30:15.833 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:30:15.833 | INFO | High Water Breakdown: | 20260508 12:30:15.833 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:30:15.833 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:30:15.833 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:30:15.833 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:30:15.833 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:30:15.833 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:30:15.833 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:30:15.833 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:30:15.833 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:30:15.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:30:15.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:30:15.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:30:15.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:30:15.833 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:30:15.833 | INFO | ------------------------------------------ | 20260508 12:30:15.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:30:15.833 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:30:15.833 | INFO | Total mem usage: 27.8 GiB | 20260508 12:30:15.833 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:30:15.833 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:30:15.833 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:30:15.833 | INFO | High Water Breakdown: | 20260508 12:30:15.833 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:30:15.834 | INFO | ------------------------------------------ | 20260508 12:30:15.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:30:15.834 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:30:15.834 | INFO | Total mem usage: 15.7 GiB | 20260508 12:30:15.834 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:30:15.834 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:30:15.834 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:30:15.834 | INFO | High Water Breakdown: | 20260508 12:30:15.834 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:30:15.834 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:30:15.834 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:30:15.834 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:30:15.834 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:30:15.834 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:30:15.834 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:30:15.834 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:30:15.834 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:30:15.834 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:30:15.834 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:30:15.834 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:30:15.834 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:30:15.834 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:30:15.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:30:15.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:30:15.834 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:30:15.834 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:30:15.834 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:30:15.834 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:30:15.834 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:30:15.834 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:30:15.834 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:30:15.834 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:30:15.834 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:30:15.834 | INFO | ------------------------------------------ | 20260508 12:30:15.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:30:15.834 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:30:15.834 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:30:15.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:30:15.834 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.39e+01 / 6.38e-01 | 20260508 12:30:15.834 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.29e-02 / 7.75e-03 | 20260508 12:30:15.834 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.44e+00, 5.94e-01 / 1.55e-01 | 20260508 12:30:15.834 | INFO | PerfReport: 3.36%, Histogram -- 1.10e+00 / 1.43e+00, 2.08e-01 / 1.19e-01 | 20260508 12:30:15.834 | INFO | PerfReport: 15.99%, DME -- 4.17e+01 / 4.49e+01, 3.55e+00 / 1.07e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.78e+00 / 1.33e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.38e+00, 4.06e-01 / 2.70e-01 | 20260508 12:30:15.834 | INFO | PerfReport: 2.97%, Metrics -- 9.69e-01 / 1.09e+00, 2.39e-01 / 1.10e-01 | 20260508 12:30:15.834 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.95e-01 / 3.22e-01, 4.72e-02 / 2.46e-02 | 20260508 12:30:15.834 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:30:15.834 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:30:15.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:30:15.834 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 24.66%, Upload -- 2.59e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 24.68%, Download -- 2.59e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 31.03%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 2.24%, LockUpload -- 2.36e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 2.47%, LockDownload -- 2.60e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 14.90%, LockCompute -- 1.57e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:30:15.834 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260508 12:30:15.930 | INFO | Starting a round of Baz IO | 20260508 12:30:19.331 | INFO | Populating RTMetrics report | 20260508 12:30:19.331 | INFO | RTMetrics report populated | 20260508 12:30:19.331 | INFO | Opening tmp RTMetrics json file | 20260508 12:30:19.340 | INFO | Serializing RTMetrics to json | 20260508 12:30:19.341 | INFO | Serialization done, swapping tmp file now... | 20260508 12:30:19.341 | INFO | RTMetrics done | 20260508 12:30:19.403 | INFO | Finished a round of Baz IO: Wrote 1.409 GiB in 3.472 seconds | 20260508 12:30:19.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:30:19.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162228% / 3.97364e-06% / 6.75519e-05% | 20260508 12:30:19.403 | INFO | Baz IO overall: | 20260508 12:30:19.403 | INFO | BytesWritten : 3480042638450 | 20260508 12:30:19.403 | INFO | NumEvents : 1669916018979 | 20260508 12:30:20.977 | INFO | Analyzing chunk frames = [8118272,8118784) | 20260508 12:30:20.977 | INFO | MemInfo: { | 20260508 12:30:20.977 | INFO | "gpu" : " 17 GB", | 20260508 12:30:20.977 | INFO | "huge" : "", | 20260508 12:30:20.977 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:20.977 | INFO | "malloc" : " 14 GB", | 20260508 12:30:20.977 | INFO | "pinned" : " 29 GB" | 20260508 12:30:20.977 | INFO | } | 20260508 12:30:20.977 | INFO | Perf report for the last 5.14432 seconds: | 20260508 12:30:20.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:20.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:20.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:20.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:20.977 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:30:20.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:20.977 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:30:20.977 | INFO | PRE_HQ : 65.2%, 1.6, 54.1ms, 0 | 20260508 12:30:20.977 | INFO | BAZWRITER : 66.8%, 2.4, 83.9ms, 0 | 20260508 12:30:20.977 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:30:20.977 | INFO | Uploaded 36GiB to GPU in 2.60279s (13.8313GiB/s) | 20260508 12:30:20.977 | INFO | Downloaded 38GiB to GPU in 2.71949s (13.9732GiB/s) | 20260508 12:30:26.117 | INFO | Analyzing chunk frames = [8118784,8119296) | 20260508 12:30:26.117 | INFO | MemInfo: { | 20260508 12:30:26.117 | INFO | "gpu" : " 17 GB", | 20260508 12:30:26.117 | INFO | "huge" : "", | 20260508 12:30:26.117 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:26.117 | INFO | "malloc" : " 14 GB", | 20260508 12:30:26.117 | INFO | "pinned" : " 29 GB" | 20260508 12:30:26.117 | INFO | } | 20260508 12:30:26.117 | INFO | Perf report for the last 5.14062 seconds: | 20260508 12:30:26.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:26.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.117 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:30:26.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.117 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:30:26.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:26.118 | INFO | Uploaded 36GiB to GPU in 2.44832s (14.7039GiB/s) | 20260508 12:30:26.118 | INFO | Downloaded 34GiB to GPU in 2.46944s (13.7683GiB/s) | 20260508 12:30:31.255 | INFO | Analyzing chunk frames = [8119296,8119808) | 20260508 12:30:31.255 | INFO | MemInfo: { | 20260508 12:30:31.255 | INFO | "gpu" : " 17 GB", | 20260508 12:30:31.255 | INFO | "huge" : "", | 20260508 12:30:31.255 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:31.255 | INFO | "malloc" : " 14 GB", | 20260508 12:30:31.255 | INFO | "pinned" : " 29 GB" | 20260508 12:30:31.255 | INFO | } | 20260508 12:30:31.255 | INFO | Perf report for the last 5.13804 seconds: | 20260508 12:30:31.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:31.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.255 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 12:30:31.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.255 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:30:31.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:31.256 | INFO | Uploaded 36GiB to GPU in 2.48234s (14.5024GiB/s) | 20260508 12:30:31.256 | INFO | Downloaded 34GiB to GPU in 2.4477s (13.8906GiB/s) | 20260508 12:30:36.394 | INFO | Analyzing chunk frames = [8119808,8120320) | 20260508 12:30:36.395 | INFO | MemInfo: { | 20260508 12:30:36.395 | INFO | "gpu" : " 17 GB", | 20260508 12:30:36.395 | INFO | "huge" : "", | 20260508 12:30:36.395 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:36.395 | INFO | "malloc" : " 14 GB", | 20260508 12:30:36.395 | INFO | "pinned" : " 29 GB" | 20260508 12:30:36.395 | INFO | } | 20260508 12:30:36.395 | INFO | Perf report for the last 5.13947 seconds: | 20260508 12:30:36.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:36.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:30:36.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:30:36.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:36.395 | INFO | Uploaded 36GiB to GPU in 2.52314s (14.268GiB/s) | 20260508 12:30:36.395 | INFO | Downloaded 34GiB to GPU in 2.47714s (13.7255GiB/s) | 20260508 12:30:41.535 | INFO | Analyzing chunk frames = [8120320,8120832) | 20260508 12:30:41.535 | INFO | MemInfo: { | 20260508 12:30:41.535 | INFO | "gpu" : " 17 GB", | 20260508 12:30:41.535 | INFO | "huge" : "", | 20260508 12:30:41.535 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:41.536 | INFO | "malloc" : " 14 GB", | 20260508 12:30:41.536 | INFO | "pinned" : " 29 GB" | 20260508 12:30:41.536 | INFO | } | 20260508 12:30:41.536 | INFO | Perf report for the last 5.14138 seconds: | 20260508 12:30:41.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:41.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.536 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:30:41.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.536 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:30:41.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:41.537 | INFO | Uploaded 36GiB to GPU in 2.28167s (15.7779GiB/s) | 20260508 12:30:41.537 | INFO | Downloaded 34GiB to GPU in 2.49514s (13.6265GiB/s) | 20260508 12:30:46.674 | INFO | Analyzing chunk frames = [8120832,8121344) | 20260508 12:30:46.674 | INFO | MemInfo: { | 20260508 12:30:46.674 | INFO | "gpu" : " 17 GB", | 20260508 12:30:46.674 | INFO | "huge" : "", | 20260508 12:30:46.674 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:46.674 | INFO | "malloc" : " 14 GB", | 20260508 12:30:46.674 | INFO | "pinned" : " 29 GB" | 20260508 12:30:46.674 | INFO | } | 20260508 12:30:46.674 | INFO | Perf report for the last 5.13826 seconds: | 20260508 12:30:46.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:46.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.675 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:30:46.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.675 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:30:46.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:46.675 | INFO | Uploaded 36GiB to GPU in 2.37539s (15.1554GiB/s) | 20260508 12:30:46.675 | INFO | Downloaded 34GiB to GPU in 2.45513s (13.8486GiB/s) | 20260508 12:30:51.817 | INFO | Analyzing chunk frames = [8121344,8121856) | 20260508 12:30:51.817 | INFO | MemInfo: { | 20260508 12:30:51.817 | INFO | "gpu" : " 17 GB", | 20260508 12:30:51.817 | INFO | "huge" : "", | 20260508 12:30:51.817 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:51.817 | INFO | "malloc" : " 14 GB", | 20260508 12:30:51.817 | INFO | "pinned" : " 29 GB" | 20260508 12:30:51.817 | INFO | } | 20260508 12:30:51.818 | INFO | Perf report for the last 5.14305 seconds: | 20260508 12:30:51.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:51.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:30:51.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:30:51.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:51.818 | INFO | Uploaded 36GiB to GPU in 2.42418s (14.8504GiB/s) | 20260508 12:30:51.818 | INFO | Downloaded 34GiB to GPU in 2.47192s (13.7545GiB/s) | 20260508 12:30:56.956 | INFO | Analyzing chunk frames = [8121856,8122368) | 20260508 12:30:56.956 | INFO | MemInfo: { | 20260508 12:30:56.956 | INFO | "gpu" : " 17 GB", | 20260508 12:30:56.956 | INFO | "huge" : "", | 20260508 12:30:56.956 | INFO | "huge_shared" : " 29 GB", | 20260508 12:30:56.956 | INFO | "malloc" : " 14 GB", | 20260508 12:30:56.956 | INFO | "pinned" : " 29 GB" | 20260508 12:30:56.956 | INFO | } | 20260508 12:30:56.956 | INFO | Perf report for the last 5.13862 seconds: | 20260508 12:30:56.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:30:56.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | ANALYSIS : 65.1%, 3.0, 101.5ms, 0 | 20260508 12:30:56.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:30:56.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:30:56.956 | INFO | Uploaded 36GiB to GPU in 2.36476s (15.2235GiB/s) | 20260508 12:30:56.956 | INFO | Downloaded 34GiB to GPU in 2.46803s (13.7762GiB/s) | 20260508 12:30:57.053 | INFO | Starting a round of Baz IO | 20260508 12:31:00.460 | INFO | Populating RTMetrics report | 20260508 12:31:00.460 | INFO | RTMetrics report populated | 20260508 12:31:00.460 | INFO | Opening tmp RTMetrics json file | 20260508 12:31:00.466 | INFO | Serializing RTMetrics to json | 20260508 12:31:00.467 | INFO | Serialization done, swapping tmp file now... | 20260508 12:31:00.467 | INFO | RTMetrics done | 20260508 12:31:00.523 | INFO | Finished a round of Baz IO: Wrote 1.407 GiB in 3.469 seconds | 20260508 12:31:00.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:31:00.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161886% / 3.97364e-06% / 5.96046e-05% | 20260508 12:31:00.523 | INFO | Baz IO overall: | 20260508 12:31:00.523 | INFO | BytesWritten : 3481553629617 | 20260508 12:31:00.523 | INFO | NumEvents : 1670624467406 | 20260508 12:31:02.088 | INFO | Analyzing chunk frames = [8122368,8122880) | 20260508 12:31:02.088 | INFO | MemInfo: { | 20260508 12:31:02.088 | INFO | "gpu" : " 17 GB", | 20260508 12:31:02.088 | INFO | "huge" : "", | 20260508 12:31:02.088 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:02.088 | INFO | "malloc" : " 14 GB", | 20260508 12:31:02.088 | INFO | "pinned" : " 29 GB" | 20260508 12:31:02.088 | INFO | } | 20260508 12:31:02.088 | INFO | Perf report for the last 5.1322 seconds: | 20260508 12:31:02.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:02.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:02.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:02.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:02.088 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:31:02.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:02.088 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:31:02.088 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:31:02.089 | INFO | BAZWRITER : 66.9%, 2.4, 83.7ms, 0 | 20260508 12:31:02.089 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:31:02.089 | INFO | Uploaded 36GiB to GPU in 2.60676s (13.8103GiB/s) | 20260508 12:31:02.089 | INFO | Downloaded 38GiB to GPU in 2.71016s (14.0213GiB/s) | 20260508 12:31:07.234 | INFO | Analyzing chunk frames = [8122880,8123392) | 20260508 12:31:07.234 | INFO | MemInfo: { | 20260508 12:31:07.234 | INFO | "gpu" : " 17 GB", | 20260508 12:31:07.234 | INFO | "huge" : "", | 20260508 12:31:07.234 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:07.234 | INFO | "malloc" : " 14 GB", | 20260508 12:31:07.234 | INFO | "pinned" : " 29 GB" | 20260508 12:31:07.234 | INFO | } | 20260508 12:31:07.234 | INFO | Perf report for the last 5.14594 seconds: | 20260508 12:31:07.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:07.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.234 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 12:31:07.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.234 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260508 12:31:07.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:07.235 | INFO | Uploaded 36GiB to GPU in 2.45285s (14.6768GiB/s) | 20260508 12:31:07.235 | INFO | Downloaded 34GiB to GPU in 2.46189s (13.8105GiB/s) | 20260508 12:31:12.375 | INFO | Analyzing chunk frames = [8123392,8123904) | 20260508 12:31:12.375 | INFO | MemInfo: { | 20260508 12:31:12.375 | INFO | "gpu" : " 17 GB", | 20260508 12:31:12.375 | INFO | "huge" : "", | 20260508 12:31:12.375 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:12.375 | INFO | "malloc" : " 14 GB", | 20260508 12:31:12.375 | INFO | "pinned" : " 29 GB" | 20260508 12:31:12.375 | INFO | } | 20260508 12:31:12.375 | INFO | Perf report for the last 5.14087 seconds: | 20260508 12:31:12.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:12.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.375 | INFO | ANALYSIS : 66.6%, 3.0, 103.5ms, 0 | 20260508 12:31:12.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.375 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:31:12.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:12.376 | INFO | Uploaded 36GiB to GPU in 2.51131s (14.3352GiB/s) | 20260508 12:31:12.376 | INFO | Downloaded 34GiB to GPU in 2.46702s (13.7818GiB/s) | 20260508 12:31:17.512 | INFO | Analyzing chunk frames = [8123904,8124416) | 20260508 12:31:17.512 | INFO | MemInfo: { | 20260508 12:31:17.512 | INFO | "gpu" : " 17 GB", | 20260508 12:31:17.512 | INFO | "huge" : "", | 20260508 12:31:17.512 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:17.512 | INFO | "malloc" : " 14 GB", | 20260508 12:31:17.512 | INFO | "pinned" : " 29 GB" | 20260508 12:31:17.512 | INFO | } | 20260508 12:31:17.512 | INFO | Perf report for the last 5.1366 seconds: | 20260508 12:31:17.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:17.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | ANALYSIS : 66.3%, 3.0, 102.8ms, 0 | 20260508 12:31:17.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:31:17.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:17.512 | INFO | Uploaded 36GiB to GPU in 2.53586s (14.1964GiB/s) | 20260508 12:31:17.512 | INFO | Downloaded 34GiB to GPU in 2.45586s (13.8445GiB/s) | 20260508 12:31:22.644 | INFO | Analyzing chunk frames = [8124416,8124928) | 20260508 12:31:22.644 | INFO | MemInfo: { | 20260508 12:31:22.644 | INFO | "gpu" : " 17 GB", | 20260508 12:31:22.644 | INFO | "huge" : "", | 20260508 12:31:22.644 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:22.644 | INFO | "malloc" : " 14 GB", | 20260508 12:31:22.644 | INFO | "pinned" : " 29 GB" | 20260508 12:31:22.644 | INFO | } | 20260508 12:31:22.645 | INFO | Perf report for the last 5.13269 seconds: | 20260508 12:31:22.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:22.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:31:22.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:31:22.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:22.645 | INFO | Uploaded 36GiB to GPU in 2.30531s (15.6161GiB/s) | 20260508 12:31:22.645 | INFO | Downloaded 34GiB to GPU in 2.48849s (13.6629GiB/s) | 20260508 12:31:27.786 | INFO | Analyzing chunk frames = [8124928,8125440) | 20260508 12:31:27.786 | INFO | MemInfo: { | 20260508 12:31:27.786 | INFO | "gpu" : " 17 GB", | 20260508 12:31:27.787 | INFO | "huge" : "", | 20260508 12:31:27.787 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:27.787 | INFO | "malloc" : " 14 GB", | 20260508 12:31:27.787 | INFO | "pinned" : " 29 GB" | 20260508 12:31:27.787 | INFO | } | 20260508 12:31:27.787 | INFO | Perf report for the last 5.14219 seconds: | 20260508 12:31:27.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:27.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:31:27.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:31:27.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:27.787 | INFO | Uploaded 36GiB to GPU in 2.3742s (15.163GiB/s) | 20260508 12:31:27.787 | INFO | Downloaded 34GiB to GPU in 2.4245s (14.0235GiB/s) | 20260508 12:31:32.926 | INFO | Analyzing chunk frames = [8125440,8125952) | 20260508 12:31:32.927 | INFO | MemInfo: { | 20260508 12:31:32.927 | INFO | "gpu" : " 17 GB", | 20260508 12:31:32.927 | INFO | "huge" : "", | 20260508 12:31:32.927 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:32.927 | INFO | "malloc" : " 14 GB", | 20260508 12:31:32.927 | INFO | "pinned" : " 29 GB" | 20260508 12:31:32.927 | INFO | } | 20260508 12:31:32.927 | INFO | Perf report for the last 5.14003 seconds: | 20260508 12:31:32.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:32.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:31:32.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:31:32.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:32.927 | INFO | Uploaded 36GiB to GPU in 2.44803s (14.7057GiB/s) | 20260508 12:31:32.928 | INFO | Downloaded 34GiB to GPU in 2.44911s (13.8826GiB/s) | 20260508 12:31:38.071 | INFO | Analyzing chunk frames = [8125952,8126464) | 20260508 12:31:38.071 | INFO | MemInfo: { | 20260508 12:31:38.071 | INFO | "gpu" : " 17 GB", | 20260508 12:31:38.071 | INFO | "huge" : "", | 20260508 12:31:38.071 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:38.071 | INFO | "malloc" : " 14 GB", | 20260508 12:31:38.071 | INFO | "pinned" : " 29 GB" | 20260508 12:31:38.071 | INFO | } | 20260508 12:31:38.071 | INFO | Perf report for the last 5.14449 seconds: | 20260508 12:31:38.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:38.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.071 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 12:31:38.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.071 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:31:38.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:38.072 | INFO | Uploaded 36GiB to GPU in 2.44952s (14.6968GiB/s) | 20260508 12:31:38.072 | INFO | Downloaded 34GiB to GPU in 2.4504s (13.8753GiB/s) | 20260508 12:31:38.072 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:31:38.072 | INFO | Total mem usage: 16.7 GiB | 20260508 12:31:38.072 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:31:38.072 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:31:38.072 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:31:38.072 | INFO | High Water Breakdown: | 20260508 12:31:38.072 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:31:38.072 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:31:38.072 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:31:38.072 | INFO | ------------------------------------------ | 20260508 12:31:38.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:31:38.072 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:31:38.072 | INFO | Total mem usage: 26.5 GiB | 20260508 12:31:38.072 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:31:38.072 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:31:38.072 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:31:38.072 | INFO | High Water Breakdown: | 20260508 12:31:38.072 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:31:38.072 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:31:38.072 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:31:38.072 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:31:38.072 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:31:38.072 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:31:38.072 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:31:38.072 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:31:38.072 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:31:38.072 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:31:38.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:31:38.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:31:38.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:31:38.073 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:31:38.073 | INFO | ------------------------------------------ | 20260508 12:31:38.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:31:38.073 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:31:38.073 | INFO | Total mem usage: 27.8 GiB | 20260508 12:31:38.073 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:31:38.073 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:31:38.073 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:31:38.073 | INFO | High Water Breakdown: | 20260508 12:31:38.073 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:31:38.073 | INFO | ------------------------------------------ | 20260508 12:31:38.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:31:38.073 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:31:38.073 | INFO | Total mem usage: 15.7 GiB | 20260508 12:31:38.073 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:31:38.073 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:31:38.073 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:31:38.073 | INFO | High Water Breakdown: | 20260508 12:31:38.073 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:31:38.073 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:31:38.073 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:31:38.073 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:31:38.073 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:31:38.073 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:31:38.073 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:31:38.073 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:31:38.073 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:31:38.073 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:31:38.073 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:31:38.073 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:31:38.073 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:31:38.073 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:31:38.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:31:38.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:31:38.073 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:31:38.073 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:31:38.073 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:31:38.073 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:31:38.073 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:31:38.073 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:31:38.073 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:31:38.073 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:31:38.073 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:31:38.073 | INFO | ------------------------------------------ | 20260508 12:31:38.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:31:38.073 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:31:38.073 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:31:38.073 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:31:38.073 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 6.21e-01 | 20260508 12:31:38.073 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.04e-01, 1.14e-02 / 3.08e-03 | 20260508 12:31:38.073 | INFO | PerfReport: 12.19%, Baseline -- 3.97e+00 / 4.47e+00, 5.94e-01 / 1.82e-01 | 20260508 12:31:38.073 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.48e+00, 2.11e-01 / 1.04e-01 | 20260508 12:31:38.073 | INFO | PerfReport: 16.04%, DME -- 4.18e+01 / 4.53e+01, 3.67e+00 / 9.58e-01 | 20260508 12:31:38.073 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.77e+00 / 1.31e+00 | 20260508 12:31:38.073 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.48e+00, 4.06e-01 / 2.33e-01 | 20260508 12:31:38.073 | INFO | PerfReport: 2.96%, Metrics -- 9.65e-01 / 1.07e+00, 2.39e-01 / 9.31e-02 | 20260508 12:31:38.073 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.92e-01 / 3.15e-01, 4.64e-02 / 2.12e-02 | 20260508 12:31:38.073 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:31:38.073 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:31:38.073 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:31:38.074 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 24.31%, Upload -- 2.55e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 24.76%, Download -- 2.60e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 31.07%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 2.11%, LockUpload -- 2.22e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 2.64%, LockDownload -- 2.77e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 15.10%, LockCompute -- 1.59e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:31:38.074 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260508 12:31:38.182 | INFO | Starting a round of Baz IO | 20260508 12:31:41.577 | INFO | Populating RTMetrics report | 20260508 12:31:41.577 | INFO | RTMetrics report populated | 20260508 12:31:41.577 | INFO | Opening tmp RTMetrics json file | 20260508 12:31:41.584 | INFO | Serializing RTMetrics to json | 20260508 12:31:41.585 | INFO | Serialization done, swapping tmp file now... | 20260508 12:31:41.586 | INFO | RTMetrics done | 20260508 12:31:41.643 | INFO | Finished a round of Baz IO: Wrote 1.407 GiB in 3.46 seconds | 20260508 12:31:41.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.02% / 0.00% / 59.98% | 20260508 12:31:41.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161644% / 3.97364e-06% / 5.5631e-05% | 20260508 12:31:41.643 | INFO | Baz IO overall: | 20260508 12:31:41.643 | INFO | BytesWritten : 3483063896336 | 20260508 12:31:41.643 | INFO | NumEvents : 1671332290229 | 20260508 12:31:43.208 | INFO | Analyzing chunk frames = [8126464,8126976) | 20260508 12:31:43.208 | INFO | MemInfo: { | 20260508 12:31:43.208 | INFO | "gpu" : " 17 GB", | 20260508 12:31:43.208 | INFO | "huge" : "", | 20260508 12:31:43.208 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:43.208 | INFO | "malloc" : " 14 GB", | 20260508 12:31:43.208 | INFO | "pinned" : " 29 GB" | 20260508 12:31:43.208 | INFO | } | 20260508 12:31:43.208 | INFO | Perf report for the last 5.13657 seconds: | 20260508 12:31:43.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:43.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:43.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:43.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:43.208 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:31:43.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:43.208 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:31:43.208 | INFO | PRE_HQ : 65.2%, 1.6, 54.0ms, 0 | 20260508 12:31:43.208 | INFO | BAZWRITER : 66.7%, 2.4, 83.7ms, 0 | 20260508 12:31:43.208 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:31:43.209 | INFO | Uploaded 36GiB to GPU in 2.60336s (13.8283GiB/s) | 20260508 12:31:43.209 | INFO | Downloaded 38GiB to GPU in 2.7357s (13.8904GiB/s) | 20260508 12:31:48.342 | INFO | Analyzing chunk frames = [8126976,8127488) | 20260508 12:31:48.342 | INFO | MemInfo: { | 20260508 12:31:48.342 | INFO | "gpu" : " 17 GB", | 20260508 12:31:48.342 | INFO | "huge" : "", | 20260508 12:31:48.342 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:48.342 | INFO | "malloc" : " 14 GB", | 20260508 12:31:48.342 | INFO | "pinned" : " 29 GB" | 20260508 12:31:48.342 | INFO | } | 20260508 12:31:48.342 | INFO | Perf report for the last 5.1343 seconds: | 20260508 12:31:48.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:48.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.342 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:31:48.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.342 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:31:48.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:48.343 | INFO | Uploaded 36GiB to GPU in 2.42961s (14.8172GiB/s) | 20260508 12:31:48.343 | INFO | Downloaded 34GiB to GPU in 2.47838s (13.7187GiB/s) | 20260508 12:31:53.490 | INFO | Analyzing chunk frames = [8127488,8128000) | 20260508 12:31:53.490 | INFO | MemInfo: { | 20260508 12:31:53.490 | INFO | "gpu" : " 17 GB", | 20260508 12:31:53.490 | INFO | "huge" : "", | 20260508 12:31:53.490 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:53.490 | INFO | "malloc" : " 14 GB", | 20260508 12:31:53.490 | INFO | "pinned" : " 29 GB" | 20260508 12:31:53.490 | INFO | } | 20260508 12:31:53.490 | INFO | Perf report for the last 5.14819 seconds: | 20260508 12:31:53.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:53.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.490 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:31:53.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.491 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:31:53.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:53.491 | INFO | Uploaded 36GiB to GPU in 2.46077s (14.6296GiB/s) | 20260508 12:31:53.491 | INFO | Downloaded 34GiB to GPU in 2.47347s (13.7459GiB/s) | 20260508 12:31:58.622 | INFO | Analyzing chunk frames = [8128000,8128512) | 20260508 12:31:58.622 | INFO | MemInfo: { | 20260508 12:31:58.622 | INFO | "gpu" : " 17 GB", | 20260508 12:31:58.622 | INFO | "huge" : "", | 20260508 12:31:58.622 | INFO | "huge_shared" : " 29 GB", | 20260508 12:31:58.622 | INFO | "malloc" : " 14 GB", | 20260508 12:31:58.622 | INFO | "pinned" : " 29 GB" | 20260508 12:31:58.622 | INFO | } | 20260508 12:31:58.622 | INFO | Perf report for the last 5.13192 seconds: | 20260508 12:31:58.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:31:58.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.622 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 12:31:58.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.622 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:31:58.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:31:58.623 | INFO | Uploaded 36GiB to GPU in 2.47966s (14.5181GiB/s) | 20260508 12:31:58.623 | INFO | Downloaded 34GiB to GPU in 2.47604s (13.7316GiB/s) | 20260508 12:32:03.765 | INFO | Analyzing chunk frames = [8128512,8129024) | 20260508 12:32:03.765 | INFO | MemInfo: { | 20260508 12:32:03.765 | INFO | "gpu" : " 17 GB", | 20260508 12:32:03.765 | INFO | "huge" : "", | 20260508 12:32:03.765 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:03.765 | INFO | "malloc" : " 14 GB", | 20260508 12:32:03.765 | INFO | "pinned" : " 29 GB" | 20260508 12:32:03.765 | INFO | } | 20260508 12:32:03.765 | INFO | Perf report for the last 5.14283 seconds: | 20260508 12:32:03.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:03.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.765 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:32:03.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.765 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:32:03.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:03.766 | INFO | Uploaded 36GiB to GPU in 2.27681s (15.8116GiB/s) | 20260508 12:32:03.766 | INFO | Downloaded 34GiB to GPU in 2.49725s (13.615GiB/s) | 20260508 12:32:08.901 | INFO | Analyzing chunk frames = [8129024,8129536) | 20260508 12:32:08.901 | INFO | MemInfo: { | 20260508 12:32:08.901 | INFO | "gpu" : " 17 GB", | 20260508 12:32:08.901 | INFO | "huge" : "", | 20260508 12:32:08.901 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:08.901 | INFO | "malloc" : " 14 GB", | 20260508 12:32:08.901 | INFO | "pinned" : " 29 GB" | 20260508 12:32:08.901 | INFO | } | 20260508 12:32:08.901 | INFO | Perf report for the last 5.13617 seconds: | 20260508 12:32:08.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:08.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.901 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:32:08.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.901 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:32:08.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:08.902 | INFO | Uploaded 36GiB to GPU in 2.39916s (15.0053GiB/s) | 20260508 12:32:08.902 | INFO | Downloaded 34GiB to GPU in 2.45178s (13.8675GiB/s) | 20260508 12:32:14.043 | INFO | Analyzing chunk frames = [8129536,8130048) | 20260508 12:32:14.043 | INFO | MemInfo: { | 20260508 12:32:14.043 | INFO | "gpu" : " 17 GB", | 20260508 12:32:14.043 | INFO | "huge" : "", | 20260508 12:32:14.043 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:14.043 | INFO | "malloc" : " 14 GB", | 20260508 12:32:14.043 | INFO | "pinned" : " 29 GB" | 20260508 12:32:14.043 | INFO | } | 20260508 12:32:14.044 | INFO | Perf report for the last 5.14219 seconds: | 20260508 12:32:14.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:14.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 12:32:14.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 12:32:14.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:14.044 | INFO | Uploaded 36GiB to GPU in 2.45732s (14.6501GiB/s) | 20260508 12:32:14.044 | INFO | Downloaded 34GiB to GPU in 2.46706s (13.7816GiB/s) | 20260508 12:32:19.185 | INFO | Analyzing chunk frames = [8130048,8130560) | 20260508 12:32:19.185 | INFO | MemInfo: { | 20260508 12:32:19.185 | INFO | "gpu" : " 17 GB", | 20260508 12:32:19.185 | INFO | "huge" : "", | 20260508 12:32:19.185 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:19.185 | INFO | "malloc" : " 14 GB", | 20260508 12:32:19.185 | INFO | "pinned" : " 29 GB" | 20260508 12:32:19.185 | INFO | } | 20260508 12:32:19.185 | INFO | Perf report for the last 5.14153 seconds: | 20260508 12:32:19.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:19.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.185 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:32:19.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.185 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:32:19.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:19.186 | INFO | Uploaded 36GiB to GPU in 2.47428s (14.5497GiB/s) | 20260508 12:32:19.186 | INFO | Downloaded 34GiB to GPU in 2.47291s (13.749GiB/s) | 20260508 12:32:19.283 | INFO | Starting a round of Baz IO | 20260508 12:32:22.686 | INFO | Populating RTMetrics report | 20260508 12:32:22.686 | INFO | RTMetrics report populated | 20260508 12:32:22.686 | INFO | Opening tmp RTMetrics json file | 20260508 12:32:22.691 | INFO | Serializing RTMetrics to json | 20260508 12:32:22.693 | INFO | Serialization done, swapping tmp file now... | 20260508 12:32:22.693 | INFO | RTMetrics done | 20260508 12:32:22.752 | INFO | Finished a round of Baz IO: Wrote 1.407 GiB in 3.469 seconds | 20260508 12:32:22.752 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.98% | 20260508 12:32:22.752 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162037% / 0% / 6.35783e-05% | 20260508 12:32:22.752 | INFO | Baz IO overall: | 20260508 12:32:22.752 | INFO | BytesWritten : 3484575123966 | 20260508 12:32:22.752 | INFO | NumEvents : 1672041220230 | 20260508 12:32:24.322 | INFO | Analyzing chunk frames = [8130560,8131072) | 20260508 12:32:24.322 | INFO | MemInfo: { | 20260508 12:32:24.322 | INFO | "gpu" : " 17 GB", | 20260508 12:32:24.322 | INFO | "huge" : "", | 20260508 12:32:24.322 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:24.322 | INFO | "malloc" : " 14 GB", | 20260508 12:32:24.322 | INFO | "pinned" : " 29 GB" | 20260508 12:32:24.322 | INFO | } | 20260508 12:32:24.322 | INFO | Perf report for the last 5.1373 seconds: | 20260508 12:32:24.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:24.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:24.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:24.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:24.322 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:32:24.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:24.322 | INFO | RT_METRICS : 38.1%, 1.0, 19.9ms, 0 | 20260508 12:32:24.323 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 12:32:24.323 | INFO | BAZWRITER : 66.7%, 2.4, 83.1ms, 0 | 20260508 12:32:24.323 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:32:24.323 | INFO | Uploaded 36GiB to GPU in 2.63901s (13.6415GiB/s) | 20260508 12:32:24.323 | INFO | Downloaded 38GiB to GPU in 2.70516s (14.0472GiB/s) | 20260508 12:32:29.459 | INFO | Analyzing chunk frames = [8131072,8131584) | 20260508 12:32:29.459 | INFO | MemInfo: { | 20260508 12:32:29.459 | INFO | "gpu" : " 17 GB", | 20260508 12:32:29.459 | INFO | "huge" : "", | 20260508 12:32:29.459 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:29.459 | INFO | "malloc" : " 14 GB", | 20260508 12:32:29.459 | INFO | "pinned" : " 29 GB" | 20260508 12:32:29.459 | INFO | } | 20260508 12:32:29.459 | INFO | Perf report for the last 5.13641 seconds: | 20260508 12:32:29.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:29.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 12:32:29.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:32:29.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:29.459 | INFO | Uploaded 36GiB to GPU in 2.51114s (14.3361GiB/s) | 20260508 12:32:29.459 | INFO | Downloaded 34GiB to GPU in 2.46537s (13.791GiB/s) | 20260508 12:32:34.594 | INFO | Analyzing chunk frames = [8131584,8132096) | 20260508 12:32:34.594 | INFO | MemInfo: { | 20260508 12:32:34.594 | INFO | "gpu" : " 17 GB", | 20260508 12:32:34.594 | INFO | "huge" : "", | 20260508 12:32:34.594 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:34.594 | INFO | "malloc" : " 14 GB", | 20260508 12:32:34.594 | INFO | "pinned" : " 29 GB" | 20260508 12:32:34.594 | INFO | } | 20260508 12:32:34.594 | INFO | Perf report for the last 5.13523 seconds: | 20260508 12:32:34.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:34.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.594 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:32:34.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.594 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:32:34.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:34.595 | INFO | Uploaded 36GiB to GPU in 2.51671s (14.3044GiB/s) | 20260508 12:32:34.595 | INFO | Downloaded 34GiB to GPU in 2.431s (13.986GiB/s) | 20260508 12:32:39.735 | INFO | Analyzing chunk frames = [8132096,8132608) | 20260508 12:32:39.735 | INFO | MemInfo: { | 20260508 12:32:39.735 | INFO | "gpu" : " 17 GB", | 20260508 12:32:39.735 | INFO | "huge" : "", | 20260508 12:32:39.735 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:39.736 | INFO | "malloc" : " 14 GB", | 20260508 12:32:39.736 | INFO | "pinned" : " 29 GB" | 20260508 12:32:39.736 | INFO | } | 20260508 12:32:39.736 | INFO | Perf report for the last 5.14169 seconds: | 20260508 12:32:39.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:39.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:32:39.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:32:39.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:39.736 | INFO | Uploaded 36GiB to GPU in 2.50092s (14.3947GiB/s) | 20260508 12:32:39.736 | INFO | Downloaded 34GiB to GPU in 2.44802s (13.8888GiB/s) | 20260508 12:32:44.881 | INFO | Analyzing chunk frames = [8132608,8133120) | 20260508 12:32:44.881 | INFO | MemInfo: { | 20260508 12:32:44.881 | INFO | "gpu" : " 17 GB", | 20260508 12:32:44.881 | INFO | "huge" : "", | 20260508 12:32:44.881 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:44.881 | INFO | "malloc" : " 14 GB", | 20260508 12:32:44.881 | INFO | "pinned" : " 29 GB" | 20260508 12:32:44.881 | INFO | } | 20260508 12:32:44.881 | INFO | Perf report for the last 5.14547 seconds: | 20260508 12:32:44.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:44.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 12:32:44.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:32:44.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:44.881 | INFO | Uploaded 36GiB to GPU in 2.41634s (14.8986GiB/s) | 20260508 12:32:44.881 | INFO | Downloaded 34GiB to GPU in 2.50748s (13.5594GiB/s) | 20260508 12:32:50.020 | INFO | Analyzing chunk frames = [8133120,8133632) | 20260508 12:32:50.020 | INFO | MemInfo: { | 20260508 12:32:50.020 | INFO | "gpu" : " 17 GB", | 20260508 12:32:50.020 | INFO | "huge" : "", | 20260508 12:32:50.020 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:50.021 | INFO | "malloc" : " 14 GB", | 20260508 12:32:50.021 | INFO | "pinned" : " 29 GB" | 20260508 12:32:50.021 | INFO | } | 20260508 12:32:50.021 | INFO | Perf report for the last 5.13953 seconds: | 20260508 12:32:50.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:50.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | ANALYSIS : 65.6%, 3.0, 102.1ms, 0 | 20260508 12:32:50.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260508 12:32:50.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:50.021 | INFO | Uploaded 36GiB to GPU in 2.47298s (14.5573GiB/s) | 20260508 12:32:50.021 | INFO | Downloaded 34GiB to GPU in 2.45145s (13.8693GiB/s) | 20260508 12:32:55.156 | INFO | Analyzing chunk frames = [8133632,8134144) | 20260508 12:32:55.157 | INFO | MemInfo: { | 20260508 12:32:55.157 | INFO | "gpu" : " 17 GB", | 20260508 12:32:55.157 | INFO | "huge" : "", | 20260508 12:32:55.157 | INFO | "huge_shared" : " 29 GB", | 20260508 12:32:55.157 | INFO | "malloc" : " 14 GB", | 20260508 12:32:55.157 | INFO | "pinned" : " 29 GB" | 20260508 12:32:55.157 | INFO | } | 20260508 12:32:55.157 | INFO | Perf report for the last 5.13589 seconds: | 20260508 12:32:55.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:32:55.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 12:32:55.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260508 12:32:55.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:32:55.157 | INFO | Uploaded 36GiB to GPU in 2.32524s (15.4823GiB/s) | 20260508 12:32:55.157 | INFO | Downloaded 34GiB to GPU in 2.39777s (14.1798GiB/s) | 20260508 12:33:00.295 | INFO | Analyzing chunk frames = [8134144,8134656) | 20260508 12:33:00.295 | INFO | MemInfo: { | 20260508 12:33:00.295 | INFO | "gpu" : " 17 GB", | 20260508 12:33:00.295 | INFO | "huge" : "", | 20260508 12:33:00.295 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:00.295 | INFO | "malloc" : " 14 GB", | 20260508 12:33:00.295 | INFO | "pinned" : " 29 GB" | 20260508 12:33:00.295 | INFO | } | 20260508 12:33:00.295 | INFO | Perf report for the last 5.13845 seconds: | 20260508 12:33:00.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:00.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260508 12:33:00.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260508 12:33:00.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:00.295 | INFO | Uploaded 36GiB to GPU in 2.3526s (15.3022GiB/s) | 20260508 12:33:00.296 | INFO | Downloaded 34GiB to GPU in 2.42479s (14.0219GiB/s) | 20260508 12:33:00.296 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:33:00.296 | INFO | Total mem usage: 16.7 GiB | 20260508 12:33:00.296 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:33:00.296 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:33:00.296 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:33:00.296 | INFO | High Water Breakdown: | 20260508 12:33:00.296 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:33:00.296 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:33:00.296 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:33:00.296 | INFO | ------------------------------------------ | 20260508 12:33:00.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:33:00.296 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:33:00.296 | INFO | Total mem usage: 26.5 GiB | 20260508 12:33:00.296 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:33:00.296 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:33:00.296 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:33:00.296 | INFO | High Water Breakdown: | 20260508 12:33:00.296 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:33:00.296 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:33:00.296 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:33:00.296 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:33:00.296 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:33:00.296 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:33:00.296 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:33:00.296 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:33:00.296 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:33:00.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:33:00.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:33:00.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:33:00.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:33:00.296 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:33:00.296 | INFO | ------------------------------------------ | 20260508 12:33:00.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:33:00.296 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:33:00.296 | INFO | Total mem usage: 27.8 GiB | 20260508 12:33:00.296 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:33:00.296 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:33:00.296 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:33:00.296 | INFO | High Water Breakdown: | 20260508 12:33:00.296 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:33:00.296 | INFO | ------------------------------------------ | 20260508 12:33:00.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:33:00.296 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:33:00.296 | INFO | Total mem usage: 15.7 GiB | 20260508 12:33:00.296 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:33:00.296 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:33:00.296 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:33:00.296 | INFO | High Water Breakdown: | 20260508 12:33:00.296 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:33:00.297 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:33:00.297 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:33:00.297 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:33:00.297 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:33:00.297 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:33:00.297 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:33:00.297 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:33:00.297 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:33:00.297 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:33:00.297 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:33:00.297 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:33:00.297 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:33:00.297 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:33:00.297 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:33:00.297 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:33:00.297 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:33:00.297 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:33:00.297 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:33:00.297 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:33:00.297 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:33:00.297 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:33:00.297 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:33:00.297 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:33:00.297 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:33:00.297 | INFO | ------------------------------------------ | 20260508 12:33:00.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:33:00.297 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:33:00.297 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:33:00.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:33:00.297 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 5.89e-01 | 20260508 12:33:00.297 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.03e-01, 1.11e-02 / 6.63e-03 | 20260508 12:33:00.297 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.37e+00, 5.97e-01 / 2.65e-01 | 20260508 12:33:00.297 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.12e-01 / 1.12e-01 | 20260508 12:33:00.297 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.52e+01, 3.64e+00 / 8.44e-01 | 20260508 12:33:00.297 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.82e+00 / 1.28e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.32e+00 / 2.39e+00, 4.01e-01 / 2.66e-01 | 20260508 12:33:00.297 | INFO | PerfReport: 2.95%, Metrics -- 9.65e-01 / 1.08e+00, 2.36e-01 / 9.44e-02 | 20260508 12:33:00.297 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.22e-01, 4.68e-02 / 2.36e-02 | 20260508 12:33:00.297 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:33:00.297 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:33:00.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:33:00.297 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 24.39%, Upload -- 2.56e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 2.60%, LockDownload -- 2.72e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 14.98%, LockCompute -- 1.57e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:33:00.297 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260508 12:33:00.411 | INFO | Starting a round of Baz IO | 20260508 12:33:03.793 | INFO | Populating RTMetrics report | 20260508 12:33:03.793 | INFO | RTMetrics report populated | 20260508 12:33:03.793 | INFO | Opening tmp RTMetrics json file | 20260508 12:33:03.801 | INFO | Serializing RTMetrics to json | 20260508 12:33:03.802 | INFO | Serialization done, swapping tmp file now... | 20260508 12:33:03.803 | INFO | RTMetrics done | 20260508 12:33:03.860 | INFO | Finished a round of Baz IO: Wrote 1.408 GiB in 3.449 seconds | 20260508 12:33:03.860 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:33:03.860 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162264% / 0% / 5.5631e-05% | 20260508 12:33:03.860 | INFO | Baz IO overall: | 20260508 12:33:03.860 | INFO | BytesWritten : 3486086623390 | 20260508 12:33:03.860 | INFO | NumEvents : 1672750226871 | 20260508 12:33:05.435 | INFO | Analyzing chunk frames = [8134656,8135168) | 20260508 12:33:05.435 | INFO | MemInfo: { | 20260508 12:33:05.435 | INFO | "gpu" : " 17 GB", | 20260508 12:33:05.435 | INFO | "huge" : "", | 20260508 12:33:05.435 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:05.435 | INFO | "malloc" : " 14 GB", | 20260508 12:33:05.435 | INFO | "pinned" : " 29 GB" | 20260508 12:33:05.435 | INFO | } | 20260508 12:33:05.435 | INFO | Perf report for the last 5.13961 seconds: | 20260508 12:33:05.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:05.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:05.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:05.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:05.435 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:33:05.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:05.435 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 12:33:05.435 | INFO | PRE_HQ : 66.2%, 1.8, 61.6ms, 0 | 20260508 12:33:05.435 | INFO | BAZWRITER : 66.4%, 2.6, 90.2ms, 0 | 20260508 12:33:05.435 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:33:05.436 | INFO | Uploaded 36GiB to GPU in 2.60805s (13.8034GiB/s) | 20260508 12:33:05.436 | INFO | Downloaded 38GiB to GPU in 2.7235s (13.9526GiB/s) | 20260508 12:33:10.575 | INFO | Analyzing chunk frames = [8135168,8135680) | 20260508 12:33:10.575 | INFO | MemInfo: { | 20260508 12:33:10.575 | INFO | "gpu" : " 17 GB", | 20260508 12:33:10.575 | INFO | "huge" : "", | 20260508 12:33:10.575 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:10.575 | INFO | "malloc" : " 14 GB", | 20260508 12:33:10.575 | INFO | "pinned" : " 29 GB" | 20260508 12:33:10.575 | INFO | } | 20260508 12:33:10.575 | INFO | Perf report for the last 5.14045 seconds: | 20260508 12:33:10.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:10.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.575 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 12:33:10.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.575 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:33:10.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:10.576 | INFO | Uploaded 36GiB to GPU in 2.48434s (14.4908GiB/s) | 20260508 12:33:10.576 | INFO | Downloaded 34GiB to GPU in 2.47642s (13.7295GiB/s) | 20260508 12:33:15.717 | INFO | Analyzing chunk frames = [8135680,8136192) | 20260508 12:33:15.717 | INFO | MemInfo: { | 20260508 12:33:15.717 | INFO | "gpu" : " 17 GB", | 20260508 12:33:15.717 | INFO | "huge" : "", | 20260508 12:33:15.717 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:15.717 | INFO | "malloc" : " 14 GB", | 20260508 12:33:15.717 | INFO | "pinned" : " 29 GB" | 20260508 12:33:15.717 | INFO | } | 20260508 12:33:15.717 | INFO | Perf report for the last 5.14211 seconds: | 20260508 12:33:15.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:15.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.717 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 12:33:15.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.717 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:33:15.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:15.718 | INFO | Uploaded 36GiB to GPU in 2.48832s (14.4676GiB/s) | 20260508 12:33:15.718 | INFO | Downloaded 34GiB to GPU in 2.43636s (13.9552GiB/s) | 20260508 12:33:20.854 | INFO | Analyzing chunk frames = [8136192,8136704) | 20260508 12:33:20.854 | INFO | MemInfo: { | 20260508 12:33:20.854 | INFO | "gpu" : " 17 GB", | 20260508 12:33:20.854 | INFO | "huge" : "", | 20260508 12:33:20.854 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:20.854 | INFO | "malloc" : " 14 GB", | 20260508 12:33:20.854 | INFO | "pinned" : " 29 GB" | 20260508 12:33:20.854 | INFO | } | 20260508 12:33:20.854 | INFO | Perf report for the last 5.13685 seconds: | 20260508 12:33:20.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:20.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.854 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 12:33:20.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.854 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:33:20.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:20.855 | INFO | Uploaded 36GiB to GPU in 2.47381s (14.5524GiB/s) | 20260508 12:33:20.855 | INFO | Downloaded 34GiB to GPU in 2.47403s (13.7428GiB/s) | 20260508 12:33:25.998 | INFO | Analyzing chunk frames = [8136704,8137216) | 20260508 12:33:25.999 | INFO | MemInfo: { | 20260508 12:33:25.999 | INFO | "gpu" : " 17 GB", | 20260508 12:33:25.999 | INFO | "huge" : "", | 20260508 12:33:25.999 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:25.999 | INFO | "malloc" : " 14 GB", | 20260508 12:33:25.999 | INFO | "pinned" : " 29 GB" | 20260508 12:33:25.999 | INFO | } | 20260508 12:33:25.999 | INFO | Perf report for the last 5.14454 seconds: | 20260508 12:33:25.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:25.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 12:33:25.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:33:25.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:25.999 | INFO | Uploaded 36GiB to GPU in 2.27518s (15.8229GiB/s) | 20260508 12:33:25.999 | INFO | Downloaded 34GiB to GPU in 2.49113s (13.6484GiB/s) | 20260508 12:33:31.137 | INFO | Analyzing chunk frames = [8137216,8137728) | 20260508 12:33:31.137 | INFO | MemInfo: { | 20260508 12:33:31.137 | INFO | "gpu" : " 17 GB", | 20260508 12:33:31.137 | INFO | "huge" : "", | 20260508 12:33:31.137 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:31.137 | INFO | "malloc" : " 14 GB", | 20260508 12:33:31.137 | INFO | "pinned" : " 29 GB" | 20260508 12:33:31.137 | INFO | } | 20260508 12:33:31.137 | INFO | Perf report for the last 5.13845 seconds: | 20260508 12:33:31.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:31.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.137 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:33:31.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.137 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:33:31.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:31.138 | INFO | Uploaded 36GiB to GPU in 2.4088s (14.9452GiB/s) | 20260508 12:33:31.138 | INFO | Downloaded 34GiB to GPU in 2.43463s (13.9652GiB/s) | 20260508 12:33:36.269 | INFO | Analyzing chunk frames = [8137728,8138240) | 20260508 12:33:36.269 | INFO | MemInfo: { | 20260508 12:33:36.269 | INFO | "gpu" : " 17 GB", | 20260508 12:33:36.269 | INFO | "huge" : "", | 20260508 12:33:36.269 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:36.269 | INFO | "malloc" : " 14 GB", | 20260508 12:33:36.270 | INFO | "pinned" : " 29 GB" | 20260508 12:33:36.270 | INFO | } | 20260508 12:33:36.270 | INFO | Perf report for the last 5.13245 seconds: | 20260508 12:33:36.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:36.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | ANALYSIS : 64.8%, 3.0, 100.6ms, 0 | 20260508 12:33:36.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 12:33:36.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:36.270 | INFO | Uploaded 36GiB to GPU in 2.34401s (15.3583GiB/s) | 20260508 12:33:36.270 | INFO | Downloaded 34GiB to GPU in 2.41332s (14.0885GiB/s) | 20260508 12:33:41.407 | INFO | Analyzing chunk frames = [8138240,8138752) | 20260508 12:33:41.407 | INFO | MemInfo: { | 20260508 12:33:41.407 | INFO | "gpu" : " 17 GB", | 20260508 12:33:41.407 | INFO | "huge" : "", | 20260508 12:33:41.407 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:41.407 | INFO | "malloc" : " 14 GB", | 20260508 12:33:41.407 | INFO | "pinned" : " 29 GB" | 20260508 12:33:41.407 | INFO | } | 20260508 12:33:41.407 | INFO | Perf report for the last 5.13733 seconds: | 20260508 12:33:41.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:41.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.407 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:33:41.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.407 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:33:41.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:41.408 | INFO | Uploaded 36GiB to GPU in 2.47358s (14.5538GiB/s) | 20260508 12:33:41.408 | INFO | Downloaded 34GiB to GPU in 2.45961s (13.8233GiB/s) | 20260508 12:33:41.505 | INFO | Starting a round of Baz IO | 20260508 12:33:44.924 | INFO | Populating RTMetrics report | 20260508 12:33:44.924 | INFO | RTMetrics report populated | 20260508 12:33:44.924 | INFO | Opening tmp RTMetrics json file | 20260508 12:33:44.930 | INFO | Serializing RTMetrics to json | 20260508 12:33:44.931 | INFO | Serialization done, swapping tmp file now... | 20260508 12:33:44.932 | INFO | RTMetrics done | 20260508 12:33:44.987 | INFO | Finished a round of Baz IO: Wrote 1.405 GiB in 3.481 seconds | 20260508 12:33:44.987 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:33:44.987 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160662% / 0% / 9.53674e-05% | 20260508 12:33:44.987 | INFO | Baz IO overall: | 20260508 12:33:44.987 | INFO | BytesWritten : 3487595480323 | 20260508 12:33:44.987 | INFO | NumEvents : 1673457502218 | 20260508 12:33:46.555 | INFO | Analyzing chunk frames = [8138752,8139264) | 20260508 12:33:46.555 | INFO | MemInfo: { | 20260508 12:33:46.555 | INFO | "gpu" : " 17 GB", | 20260508 12:33:46.555 | INFO | "huge" : "", | 20260508 12:33:46.555 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:46.555 | INFO | "malloc" : " 14 GB", | 20260508 12:33:46.555 | INFO | "pinned" : " 29 GB" | 20260508 12:33:46.555 | INFO | } | 20260508 12:33:46.555 | INFO | Perf report for the last 5.14803 seconds: | 20260508 12:33:46.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:46.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:46.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:46.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:46.555 | INFO | ANALYSIS : 67.4%, 3.0, 105.7ms, 0 | 20260508 12:33:46.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:46.555 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:33:46.555 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 12:33:46.555 | INFO | BAZWRITER : 66.8%, 2.4, 82.0ms, 0 | 20260508 12:33:46.555 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:33:46.555 | INFO | Uploaded 36GiB to GPU in 2.6132s (13.7762GiB/s) | 20260508 12:33:46.555 | INFO | Downloaded 38GiB to GPU in 2.71952s (13.973GiB/s) | 20260508 12:33:51.694 | INFO | Analyzing chunk frames = [8139264,8139776) | 20260508 12:33:51.695 | INFO | MemInfo: { | 20260508 12:33:51.695 | INFO | "gpu" : " 17 GB", | 20260508 12:33:51.695 | INFO | "huge" : "", | 20260508 12:33:51.695 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:51.695 | INFO | "malloc" : " 14 GB", | 20260508 12:33:51.695 | INFO | "pinned" : " 29 GB" | 20260508 12:33:51.695 | INFO | } | 20260508 12:33:51.695 | INFO | Perf report for the last 5.13969 seconds: | 20260508 12:33:51.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:51.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:33:51.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:33:51.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:51.695 | INFO | Uploaded 36GiB to GPU in 2.45545s (14.6613GiB/s) | 20260508 12:33:51.695 | INFO | Downloaded 34GiB to GPU in 2.48191s (13.6992GiB/s) | 20260508 12:33:56.834 | INFO | Analyzing chunk frames = [8139776,8140288) | 20260508 12:33:56.834 | INFO | MemInfo: { | 20260508 12:33:56.834 | INFO | "gpu" : " 17 GB", | 20260508 12:33:56.834 | INFO | "huge" : "", | 20260508 12:33:56.834 | INFO | "huge_shared" : " 29 GB", | 20260508 12:33:56.834 | INFO | "malloc" : " 14 GB", | 20260508 12:33:56.834 | INFO | "pinned" : " 29 GB" | 20260508 12:33:56.834 | INFO | } | 20260508 12:33:56.834 | INFO | Perf report for the last 5.13963 seconds: | 20260508 12:33:56.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:33:56.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.835 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 12:33:56.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.835 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:33:56.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:33:56.835 | INFO | Uploaded 36GiB to GPU in 2.44562s (14.7202GiB/s) | 20260508 12:33:56.835 | INFO | Downloaded 34GiB to GPU in 2.47323s (13.7472GiB/s) | 20260508 12:34:01.970 | INFO | Analyzing chunk frames = [8140288,8140800) | 20260508 12:34:01.970 | INFO | MemInfo: { | 20260508 12:34:01.970 | INFO | "gpu" : " 17 GB", | 20260508 12:34:01.970 | INFO | "huge" : "", | 20260508 12:34:01.970 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:01.971 | INFO | "malloc" : " 14 GB", | 20260508 12:34:01.971 | INFO | "pinned" : " 29 GB" | 20260508 12:34:01.971 | INFO | } | 20260508 12:34:01.971 | INFO | Perf report for the last 5.13615 seconds: | 20260508 12:34:01.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:01.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | ANALYSIS : 65.7%, 3.0, 101.8ms, 0 | 20260508 12:34:01.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:34:01.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:01.971 | INFO | Uploaded 36GiB to GPU in 2.41063s (14.9339GiB/s) | 20260508 12:34:01.971 | INFO | Downloaded 34GiB to GPU in 2.48649s (13.6739GiB/s) | 20260508 12:34:07.103 | INFO | Analyzing chunk frames = [8140800,8141312) | 20260508 12:34:07.103 | INFO | MemInfo: { | 20260508 12:34:07.103 | INFO | "gpu" : " 17 GB", | 20260508 12:34:07.103 | INFO | "huge" : "", | 20260508 12:34:07.103 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:07.103 | INFO | "malloc" : " 14 GB", | 20260508 12:34:07.103 | INFO | "pinned" : " 29 GB" | 20260508 12:34:07.103 | INFO | } | 20260508 12:34:07.103 | INFO | Perf report for the last 5.13257 seconds: | 20260508 12:34:07.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:07.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 12:34:07.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:34:07.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:07.103 | INFO | Uploaded 36GiB to GPU in 2.31196s (15.5712GiB/s) | 20260508 12:34:07.104 | INFO | Downloaded 34GiB to GPU in 2.50465s (13.5747GiB/s) | 20260508 12:34:12.246 | INFO | Analyzing chunk frames = [8141312,8141824) | 20260508 12:34:12.246 | INFO | MemInfo: { | 20260508 12:34:12.246 | INFO | "gpu" : " 17 GB", | 20260508 12:34:12.246 | INFO | "huge" : "", | 20260508 12:34:12.246 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:12.246 | INFO | "malloc" : " 14 GB", | 20260508 12:34:12.246 | INFO | "pinned" : " 29 GB" | 20260508 12:34:12.246 | INFO | } | 20260508 12:34:12.246 | INFO | Perf report for the last 5.14274 seconds: | 20260508 12:34:12.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:12.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:34:12.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260508 12:34:12.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:12.246 | INFO | Uploaded 36GiB to GPU in 2.39831s (15.0106GiB/s) | 20260508 12:34:12.246 | INFO | Downloaded 34GiB to GPU in 2.46217s (13.8089GiB/s) | 20260508 12:34:17.386 | INFO | Analyzing chunk frames = [8141824,8142336) | 20260508 12:34:17.386 | INFO | MemInfo: { | 20260508 12:34:17.386 | INFO | "gpu" : " 17 GB", | 20260508 12:34:17.386 | INFO | "huge" : "", | 20260508 12:34:17.386 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:17.387 | INFO | "malloc" : " 14 GB", | 20260508 12:34:17.387 | INFO | "pinned" : " 29 GB" | 20260508 12:34:17.387 | INFO | } | 20260508 12:34:17.387 | INFO | Perf report for the last 5.14075 seconds: | 20260508 12:34:17.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:17.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 12:34:17.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:34:17.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:17.387 | INFO | Uploaded 36GiB to GPU in 2.3979s (15.0131GiB/s) | 20260508 12:34:17.387 | INFO | Downloaded 34GiB to GPU in 2.45404s (13.8547GiB/s) | 20260508 12:34:22.529 | INFO | Analyzing chunk frames = [8142336,8142848) | 20260508 12:34:22.529 | INFO | MemInfo: { | 20260508 12:34:22.529 | INFO | "gpu" : " 17 GB", | 20260508 12:34:22.529 | INFO | "huge" : "", | 20260508 12:34:22.529 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:22.529 | INFO | "malloc" : " 14 GB", | 20260508 12:34:22.529 | INFO | "pinned" : " 29 GB" | 20260508 12:34:22.529 | INFO | } | 20260508 12:34:22.529 | INFO | Perf report for the last 5.14239 seconds: | 20260508 12:34:22.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:22.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:34:22.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:34:22.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:22.529 | INFO | Uploaded 36GiB to GPU in 2.45574s (14.6595GiB/s) | 20260508 12:34:22.529 | INFO | Downloaded 34GiB to GPU in 2.4739s (13.7435GiB/s) | 20260508 12:34:22.530 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:34:22.530 | INFO | Total mem usage: 16.7 GiB | 20260508 12:34:22.530 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:34:22.530 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:34:22.530 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:34:22.530 | INFO | High Water Breakdown: | 20260508 12:34:22.530 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:34:22.530 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:34:22.530 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:34:22.530 | INFO | ------------------------------------------ | 20260508 12:34:22.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:34:22.530 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:34:22.530 | INFO | Total mem usage: 26.5 GiB | 20260508 12:34:22.530 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:34:22.530 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:34:22.530 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:34:22.530 | INFO | High Water Breakdown: | 20260508 12:34:22.530 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:34:22.530 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:34:22.530 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:34:22.530 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:34:22.530 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:34:22.530 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:34:22.530 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:34:22.530 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:34:22.530 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:34:22.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:34:22.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:34:22.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:34:22.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:34:22.530 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:34:22.530 | INFO | ------------------------------------------ | 20260508 12:34:22.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:34:22.530 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:34:22.530 | INFO | Total mem usage: 27.8 GiB | 20260508 12:34:22.530 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:34:22.530 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:34:22.530 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:34:22.530 | INFO | High Water Breakdown: | 20260508 12:34:22.530 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:34:22.530 | INFO | ------------------------------------------ | 20260508 12:34:22.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:34:22.530 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:34:22.530 | INFO | Total mem usage: 15.7 GiB | 20260508 12:34:22.530 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:34:22.530 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:34:22.530 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:34:22.530 | INFO | High Water Breakdown: | 20260508 12:34:22.530 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:34:22.530 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:34:22.530 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:34:22.530 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:34:22.530 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:34:22.530 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:34:22.530 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:34:22.531 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:34:22.531 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:34:22.531 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:34:22.531 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:34:22.531 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:34:22.531 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:34:22.531 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:34:22.531 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:34:22.531 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:34:22.531 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:34:22.531 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:34:22.531 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:34:22.531 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:34:22.531 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:34:22.531 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:34:22.531 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:34:22.531 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:34:22.531 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:34:22.531 | INFO | ------------------------------------------ | 20260508 12:34:22.531 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:34:22.531 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:34:22.531 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:34:22.531 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:34:22.531 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.42e+01 / 5.49e-01 | 20260508 12:34:22.531 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.11e-02 / 3.50e-03 | 20260508 12:34:22.531 | INFO | PerfReport: 12.18%, Baseline -- 3.99e+00 / 4.37e+00, 5.92e-01 / 2.23e-01 | 20260508 12:34:22.531 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.45e+00, 2.09e-01 / 1.20e-01 | 20260508 12:34:22.531 | INFO | PerfReport: 15.98%, DME -- 4.19e+01 / 4.53e+01, 3.71e+00 / 1.12e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.90e+00 / 1.44e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.30e+00 / 2.44e+00, 4.06e-01 / 2.77e-01 | 20260508 12:34:22.531 | INFO | PerfReport: 2.94%, Metrics -- 9.62e-01 / 1.10e+00, 2.39e-01 / 1.24e-01 | 20260508 12:34:22.531 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.92e-01 / 3.16e-01, 4.78e-02 / 2.47e-02 | 20260508 12:34:22.531 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:34:22.531 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:34:22.531 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:34:22.531 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 24.24%, Upload -- 2.54e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 24.82%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 31.23%, Compute -- 3.27e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 2.02%, LockUpload -- 2.12e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 2.64%, LockDownload -- 2.77e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 15.04%, LockCompute -- 1.58e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 12:34:22.531 | INFO | PerfReport: 0.01%, Misc -- 1.17e-02 / 0.00e+00, 8.80e-03 / 0.00e+00 | 20260508 12:34:22.626 | INFO | Starting a round of Baz IO | 20260508 12:34:26.038 | INFO | Populating RTMetrics report | 20260508 12:34:26.038 | INFO | RTMetrics report populated | 20260508 12:34:26.038 | INFO | Opening tmp RTMetrics json file | 20260508 12:34:26.044 | INFO | Serializing RTMetrics to json | 20260508 12:34:26.045 | INFO | Serialization done, swapping tmp file now... | 20260508 12:34:26.046 | INFO | RTMetrics done | 20260508 12:34:26.103 | INFO | Finished a round of Baz IO: Wrote 1.406 GiB in 3.476 seconds | 20260508 12:34:26.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:34:26.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160646% / 0% / 5.5631e-05% | 20260508 12:34:26.103 | INFO | Baz IO overall: | 20260508 12:34:26.103 | INFO | BytesWritten : 3489105099774 | 20260508 12:34:26.103 | INFO | NumEvents : 1674165478241 | 20260508 12:34:27.664 | INFO | Analyzing chunk frames = [8142848,8143360) | 20260508 12:34:27.665 | INFO | MemInfo: { | 20260508 12:34:27.665 | INFO | "gpu" : " 17 GB", | 20260508 12:34:27.665 | INFO | "huge" : "", | 20260508 12:34:27.665 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:27.665 | INFO | "malloc" : " 14 GB", | 20260508 12:34:27.665 | INFO | "pinned" : " 29 GB" | 20260508 12:34:27.665 | INFO | } | 20260508 12:34:27.665 | INFO | Perf report for the last 5.13547 seconds: | 20260508 12:34:27.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:27.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:27.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:27.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:27.665 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:34:27.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:27.665 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:34:27.665 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 12:34:27.665 | INFO | BAZWRITER : 67.0%, 2.4, 82.7ms, 0 | 20260508 12:34:27.665 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:34:27.665 | INFO | Uploaded 36GiB to GPU in 2.60838s (13.8017GiB/s) | 20260508 12:34:27.665 | INFO | Downloaded 38GiB to GPU in 2.72465s (13.9467GiB/s) | 20260508 12:34:32.807 | INFO | Analyzing chunk frames = [8143360,8143872) | 20260508 12:34:32.807 | INFO | MemInfo: { | 20260508 12:34:32.807 | INFO | "gpu" : " 17 GB", | 20260508 12:34:32.807 | INFO | "huge" : "", | 20260508 12:34:32.807 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:32.807 | INFO | "malloc" : " 14 GB", | 20260508 12:34:32.807 | INFO | "pinned" : " 29 GB" | 20260508 12:34:32.807 | INFO | } | 20260508 12:34:32.807 | INFO | Perf report for the last 5.14236 seconds: | 20260508 12:34:32.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:32.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:34:32.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:34:32.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:32.807 | INFO | Uploaded 36GiB to GPU in 2.42824s (14.8256GiB/s) | 20260508 12:34:32.807 | INFO | Downloaded 34GiB to GPU in 2.48042s (13.7074GiB/s) | 20260508 12:34:37.942 | INFO | Analyzing chunk frames = [8143872,8144384) | 20260508 12:34:37.942 | INFO | MemInfo: { | 20260508 12:34:37.942 | INFO | "gpu" : " 17 GB", | 20260508 12:34:37.942 | INFO | "huge" : "", | 20260508 12:34:37.942 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:37.942 | INFO | "malloc" : " 14 GB", | 20260508 12:34:37.942 | INFO | "pinned" : " 29 GB" | 20260508 12:34:37.942 | INFO | } | 20260508 12:34:37.943 | INFO | Perf report for the last 5.13555 seconds: | 20260508 12:34:37.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:37.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 12:34:37.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:34:37.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:37.943 | INFO | Uploaded 36GiB to GPU in 2.46052s (14.631GiB/s) | 20260508 12:34:37.943 | INFO | Downloaded 34GiB to GPU in 2.47347s (13.7459GiB/s) | 20260508 12:34:43.082 | INFO | Analyzing chunk frames = [8144384,8144896) | 20260508 12:34:43.083 | INFO | MemInfo: { | 20260508 12:34:43.083 | INFO | "gpu" : " 17 GB", | 20260508 12:34:43.083 | INFO | "huge" : "", | 20260508 12:34:43.083 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:43.083 | INFO | "malloc" : " 14 GB", | 20260508 12:34:43.083 | INFO | "pinned" : " 29 GB" | 20260508 12:34:43.083 | INFO | } | 20260508 12:34:43.083 | INFO | Perf report for the last 5.14048 seconds: | 20260508 12:34:43.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:43.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.083 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 12:34:43.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.083 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:34:43.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:43.084 | INFO | Uploaded 36GiB to GPU in 2.45045s (14.6912GiB/s) | 20260508 12:34:43.084 | INFO | Downloaded 34GiB to GPU in 2.47216s (13.7532GiB/s) | 20260508 12:34:48.226 | INFO | Analyzing chunk frames = [8144896,8145408) | 20260508 12:34:48.226 | INFO | MemInfo: { | 20260508 12:34:48.226 | INFO | "gpu" : " 17 GB", | 20260508 12:34:48.226 | INFO | "huge" : "", | 20260508 12:34:48.226 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:48.226 | INFO | "malloc" : " 14 GB", | 20260508 12:34:48.226 | INFO | "pinned" : " 29 GB" | 20260508 12:34:48.226 | INFO | } | 20260508 12:34:48.226 | INFO | Perf report for the last 5.14277 seconds: | 20260508 12:34:48.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:48.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 12:34:48.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260508 12:34:48.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:48.226 | INFO | Uploaded 36GiB to GPU in 2.40353s (14.978GiB/s) | 20260508 12:34:48.226 | INFO | Downloaded 34GiB to GPU in 2.56213s (13.2702GiB/s) | 20260508 12:34:53.358 | INFO | Analyzing chunk frames = [8145408,8145920) | 20260508 12:34:53.358 | INFO | MemInfo: { | 20260508 12:34:53.358 | INFO | "gpu" : " 17 GB", | 20260508 12:34:53.358 | INFO | "huge" : "", | 20260508 12:34:53.358 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:53.358 | INFO | "malloc" : " 14 GB", | 20260508 12:34:53.358 | INFO | "pinned" : " 29 GB" | 20260508 12:34:53.358 | INFO | } | 20260508 12:34:53.358 | INFO | Perf report for the last 5.13241 seconds: | 20260508 12:34:53.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:53.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.358 | INFO | ANALYSIS : 65.3%, 3.0, 101.4ms, 0 | 20260508 12:34:53.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.358 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:34:53.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:53.359 | INFO | Uploaded 36GiB to GPU in 2.37228s (15.1753GiB/s) | 20260508 12:34:53.359 | INFO | Downloaded 34GiB to GPU in 2.45214s (13.8654GiB/s) | 20260508 12:34:58.503 | INFO | Analyzing chunk frames = [8145920,8146432) | 20260508 12:34:58.503 | INFO | MemInfo: { | 20260508 12:34:58.503 | INFO | "gpu" : " 17 GB", | 20260508 12:34:58.503 | INFO | "huge" : "", | 20260508 12:34:58.503 | INFO | "huge_shared" : " 29 GB", | 20260508 12:34:58.503 | INFO | "malloc" : " 14 GB", | 20260508 12:34:58.503 | INFO | "pinned" : " 29 GB" | 20260508 12:34:58.503 | INFO | } | 20260508 12:34:58.503 | INFO | Perf report for the last 5.14462 seconds: | 20260508 12:34:58.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:34:58.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | ANALYSIS : 65.2%, 3.0, 101.8ms, 0 | 20260508 12:34:58.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:34:58.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:34:58.503 | INFO | Uploaded 36GiB to GPU in 2.41062s (14.9339GiB/s) | 20260508 12:34:58.503 | INFO | Downloaded 34GiB to GPU in 2.45909s (13.8262GiB/s) | 20260508 12:35:03.645 | INFO | Analyzing chunk frames = [8146432,8146944) | 20260508 12:35:03.645 | INFO | MemInfo: { | 20260508 12:35:03.645 | INFO | "gpu" : " 17 GB", | 20260508 12:35:03.645 | INFO | "huge" : "", | 20260508 12:35:03.645 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:03.645 | INFO | "malloc" : " 14 GB", | 20260508 12:35:03.645 | INFO | "pinned" : " 29 GB" | 20260508 12:35:03.645 | INFO | } | 20260508 12:35:03.645 | INFO | Perf report for the last 5.14217 seconds: | 20260508 12:35:03.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:03.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:35:03.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:35:03.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:03.645 | INFO | Uploaded 36GiB to GPU in 2.46007s (14.6337GiB/s) | 20260508 12:35:03.645 | INFO | Downloaded 34GiB to GPU in 2.48078s (13.7054GiB/s) | 20260508 12:35:03.743 | INFO | Starting a round of Baz IO | 20260508 12:35:07.152 | INFO | Populating RTMetrics report | 20260508 12:35:07.152 | INFO | RTMetrics report populated | 20260508 12:35:07.152 | INFO | Opening tmp RTMetrics json file | 20260508 12:35:07.158 | INFO | Serializing RTMetrics to json | 20260508 12:35:07.159 | INFO | Serialization done, swapping tmp file now... | 20260508 12:35:07.159 | INFO | RTMetrics done | 20260508 12:35:07.216 | INFO | Finished a round of Baz IO: Wrote 1.405 GiB in 3.472 seconds | 20260508 12:35:07.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:35:07.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.16017% / 0% / 5.96046e-05% | 20260508 12:35:07.216 | INFO | Baz IO overall: | 20260508 12:35:07.216 | INFO | BytesWritten : 3490614092464 | 20260508 12:35:07.216 | INFO | NumEvents : 1674873188158 | 20260508 12:35:08.781 | INFO | Analyzing chunk frames = [8146944,8147456) | 20260508 12:35:08.781 | INFO | MemInfo: { | 20260508 12:35:08.781 | INFO | "gpu" : " 17 GB", | 20260508 12:35:08.781 | INFO | "huge" : "", | 20260508 12:35:08.781 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:08.781 | INFO | "malloc" : " 14 GB", | 20260508 12:35:08.781 | INFO | "pinned" : " 29 GB" | 20260508 12:35:08.781 | INFO | } | 20260508 12:35:08.781 | INFO | Perf report for the last 5.13639 seconds: | 20260508 12:35:08.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:08.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:08.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:08.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:08.782 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 12:35:08.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:08.782 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:35:08.782 | INFO | PRE_HQ : 65.4%, 1.6, 53.8ms, 0 | 20260508 12:35:08.782 | INFO | BAZWRITER : 66.9%, 2.4, 83.0ms, 0 | 20260508 12:35:08.782 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:35:08.782 | INFO | Uploaded 36GiB to GPU in 2.59398s (13.8783GiB/s) | 20260508 12:35:08.782 | INFO | Downloaded 38GiB to GPU in 2.73244s (13.907GiB/s) | 20260508 12:35:13.915 | INFO | Analyzing chunk frames = [8147456,8147968) | 20260508 12:35:13.915 | INFO | MemInfo: { | 20260508 12:35:13.915 | INFO | "gpu" : " 17 GB", | 20260508 12:35:13.915 | INFO | "huge" : "", | 20260508 12:35:13.915 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:13.915 | INFO | "malloc" : " 14 GB", | 20260508 12:35:13.915 | INFO | "pinned" : " 29 GB" | 20260508 12:35:13.915 | INFO | } | 20260508 12:35:13.915 | INFO | Perf report for the last 5.13362 seconds: | 20260508 12:35:13.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:13.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:35:13.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:35:13.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:13.915 | INFO | Uploaded 36GiB to GPU in 2.42701s (14.833GiB/s) | 20260508 12:35:13.915 | INFO | Downloaded 34GiB to GPU in 2.48055s (13.7066GiB/s) | 20260508 12:35:19.059 | INFO | Analyzing chunk frames = [8147968,8148480) | 20260508 12:35:19.059 | INFO | MemInfo: { | 20260508 12:35:19.059 | INFO | "gpu" : " 17 GB", | 20260508 12:35:19.059 | INFO | "huge" : "", | 20260508 12:35:19.059 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:19.059 | INFO | "malloc" : " 14 GB", | 20260508 12:35:19.059 | INFO | "pinned" : " 29 GB" | 20260508 12:35:19.059 | INFO | } | 20260508 12:35:19.059 | INFO | Perf report for the last 5.14417 seconds: | 20260508 12:35:19.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:19.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.059 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:35:19.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.059 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:35:19.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:19.060 | INFO | Uploaded 36GiB to GPU in 2.46794s (14.5871GiB/s) | 20260508 12:35:19.060 | INFO | Downloaded 34GiB to GPU in 2.46243s (13.8075GiB/s) | 20260508 12:35:24.199 | INFO | Analyzing chunk frames = [8148480,8148992) | 20260508 12:35:24.199 | INFO | MemInfo: { | 20260508 12:35:24.199 | INFO | "gpu" : " 17 GB", | 20260508 12:35:24.199 | INFO | "huge" : "", | 20260508 12:35:24.199 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:24.199 | INFO | "malloc" : " 14 GB", | 20260508 12:35:24.199 | INFO | "pinned" : " 29 GB" | 20260508 12:35:24.199 | INFO | } | 20260508 12:35:24.200 | INFO | Perf report for the last 5.1403 seconds: | 20260508 12:35:24.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:24.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 12:35:24.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 12:35:24.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:24.200 | INFO | Uploaded 36GiB to GPU in 2.39495s (15.0316GiB/s) | 20260508 12:35:24.200 | INFO | Downloaded 34GiB to GPU in 2.4907s (13.6508GiB/s) | 20260508 12:35:29.344 | INFO | Analyzing chunk frames = [8148992,8149504) | 20260508 12:35:29.344 | INFO | MemInfo: { | 20260508 12:35:29.344 | INFO | "gpu" : " 17 GB", | 20260508 12:35:29.344 | INFO | "huge" : "", | 20260508 12:35:29.344 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:29.344 | INFO | "malloc" : " 14 GB", | 20260508 12:35:29.344 | INFO | "pinned" : " 29 GB" | 20260508 12:35:29.344 | INFO | } | 20260508 12:35:29.345 | INFO | Perf report for the last 5.14492 seconds: | 20260508 12:35:29.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:29.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:35:29.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260508 12:35:29.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:29.345 | INFO | Uploaded 36GiB to GPU in 2.37299s (15.1707GiB/s) | 20260508 12:35:29.345 | INFO | Downloaded 34GiB to GPU in 2.53035s (13.4369GiB/s) | 20260508 12:35:34.473 | INFO | Analyzing chunk frames = [8149504,8150016) | 20260508 12:35:34.473 | INFO | MemInfo: { | 20260508 12:35:34.473 | INFO | "gpu" : " 17 GB", | 20260508 12:35:34.473 | INFO | "huge" : "", | 20260508 12:35:34.474 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:34.474 | INFO | "malloc" : " 14 GB", | 20260508 12:35:34.474 | INFO | "pinned" : " 29 GB" | 20260508 12:35:34.474 | INFO | } | 20260508 12:35:34.474 | INFO | Perf report for the last 5.12914 seconds: | 20260508 12:35:34.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:34.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | ANALYSIS : 65.1%, 3.0, 101.1ms, 0 | 20260508 12:35:34.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:35:34.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:34.474 | INFO | Uploaded 36GiB to GPU in 2.31315s (15.5632GiB/s) | 20260508 12:35:34.474 | INFO | Downloaded 34GiB to GPU in 2.4346s (13.9653GiB/s) | 20260508 12:35:39.611 | INFO | Analyzing chunk frames = [8150016,8150528) | 20260508 12:35:39.611 | INFO | MemInfo: { | 20260508 12:35:39.611 | INFO | "gpu" : " 17 GB", | 20260508 12:35:39.611 | INFO | "huge" : "", | 20260508 12:35:39.611 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:39.611 | INFO | "malloc" : " 14 GB", | 20260508 12:35:39.611 | INFO | "pinned" : " 29 GB" | 20260508 12:35:39.611 | INFO | } | 20260508 12:35:39.611 | INFO | Perf report for the last 5.13734 seconds: | 20260508 12:35:39.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:39.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 12:35:39.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:35:39.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:39.611 | INFO | Uploaded 36GiB to GPU in 2.45836s (14.6439GiB/s) | 20260508 12:35:39.611 | INFO | Downloaded 34GiB to GPU in 2.47175s (13.7555GiB/s) | 20260508 12:35:44.752 | INFO | Analyzing chunk frames = [8150528,8151040) | 20260508 12:35:44.752 | INFO | MemInfo: { | 20260508 12:35:44.752 | INFO | "gpu" : " 17 GB", | 20260508 12:35:44.752 | INFO | "huge" : "", | 20260508 12:35:44.752 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:44.752 | INFO | "malloc" : " 14 GB", | 20260508 12:35:44.752 | INFO | "pinned" : " 29 GB" | 20260508 12:35:44.752 | INFO | } | 20260508 12:35:44.753 | INFO | Perf report for the last 5.14154 seconds: | 20260508 12:35:44.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:44.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 12:35:44.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:35:44.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:44.753 | INFO | Uploaded 36GiB to GPU in 2.43776s (14.7676GiB/s) | 20260508 12:35:44.753 | INFO | Downloaded 34GiB to GPU in 2.47039s (13.763GiB/s) | 20260508 12:35:44.753 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:35:44.753 | INFO | Total mem usage: 16.7 GiB | 20260508 12:35:44.753 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:35:44.753 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:35:44.753 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:35:44.753 | INFO | High Water Breakdown: | 20260508 12:35:44.753 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:35:44.753 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:35:44.753 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:35:44.753 | INFO | ------------------------------------------ | 20260508 12:35:44.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:35:44.753 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:35:44.753 | INFO | Total mem usage: 26.5 GiB | 20260508 12:35:44.753 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:35:44.753 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:35:44.753 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:35:44.753 | INFO | High Water Breakdown: | 20260508 12:35:44.753 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:35:44.753 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:35:44.753 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:35:44.753 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:35:44.753 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:35:44.753 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:35:44.753 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:35:44.753 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:35:44.753 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:35:44.753 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:35:44.753 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:35:44.753 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:35:44.753 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:35:44.753 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:35:44.753 | INFO | ------------------------------------------ | 20260508 12:35:44.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:35:44.753 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:35:44.753 | INFO | Total mem usage: 27.8 GiB | 20260508 12:35:44.753 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:35:44.753 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:35:44.753 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:35:44.753 | INFO | High Water Breakdown: | 20260508 12:35:44.753 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:35:44.753 | INFO | ------------------------------------------ | 20260508 12:35:44.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:35:44.753 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:35:44.753 | INFO | Total mem usage: 15.7 GiB | 20260508 12:35:44.753 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:35:44.754 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:35:44.754 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:35:44.754 | INFO | High Water Breakdown: | 20260508 12:35:44.754 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:35:44.754 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:35:44.754 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:35:44.754 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:35:44.754 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:35:44.754 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:35:44.754 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:35:44.754 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:35:44.754 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:35:44.754 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:35:44.754 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:35:44.754 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:35:44.754 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:35:44.754 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:35:44.754 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:35:44.754 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:35:44.754 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:35:44.754 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:35:44.754 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:35:44.754 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:35:44.754 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:35:44.754 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:35:44.754 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:35:44.754 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:35:44.754 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:35:44.754 | INFO | ------------------------------------------ | 20260508 12:35:44.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:35:44.754 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:35:44.754 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:35:44.754 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:35:44.754 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.40e+01 / 5.36e-01 | 20260508 12:35:44.754 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.03e-01, 1.14e-02 / 5.38e-03 | 20260508 12:35:44.754 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.46e+00, 5.95e-01 / 1.81e-01 | 20260508 12:35:44.754 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.06e-01 | 20260508 12:35:44.754 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.51e+01, 3.78e+00 / 1.08e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.82e+00 / 1.28e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.32e+00 / 2.41e+00, 4.06e-01 / 2.59e-01 | 20260508 12:35:44.754 | INFO | PerfReport: 2.95%, Metrics -- 9.62e-01 / 1.10e+00, 2.39e-01 / 1.49e-01 | 20260508 12:35:44.754 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.20e-01, 4.67e-02 / 2.58e-02 | 20260508 12:35:44.754 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:35:44.754 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:35:44.754 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:35:44.754 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 24.24%, Upload -- 2.54e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 24.93%, Download -- 2.62e+01 / 0.00e+00, 8.30e+00 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 31.12%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 1.95%, LockUpload -- 2.04e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 2.76%, LockDownload -- 2.89e+00 / 0.00e+00, 4.25e+00 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 15.00%, LockCompute -- 1.57e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 12:35:44.754 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.60e-02 / 0.00e+00 | 20260508 12:35:44.851 | INFO | Starting a round of Baz IO | 20260508 12:35:48.258 | INFO | Populating RTMetrics report | 20260508 12:35:48.258 | INFO | RTMetrics report populated | 20260508 12:35:48.258 | INFO | Opening tmp RTMetrics json file | 20260508 12:35:48.265 | INFO | Serializing RTMetrics to json | 20260508 12:35:48.266 | INFO | Serialization done, swapping tmp file now... | 20260508 12:35:48.267 | INFO | RTMetrics done | 20260508 12:35:48.323 | INFO | Finished a round of Baz IO: Wrote 1.404 GiB in 3.472 seconds | 20260508 12:35:48.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:35:48.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159399% / 0% / 7.54992e-05% | 20260508 12:35:48.324 | INFO | Baz IO overall: | 20260508 12:35:48.324 | INFO | BytesWritten : 3492121786713 | 20260508 12:35:48.324 | INFO | NumEvents : 1675579834200 | 20260508 12:35:49.895 | INFO | Analyzing chunk frames = [8151040,8151552) | 20260508 12:35:49.895 | INFO | MemInfo: { | 20260508 12:35:49.895 | INFO | "gpu" : " 17 GB", | 20260508 12:35:49.895 | INFO | "huge" : "", | 20260508 12:35:49.895 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:49.895 | INFO | "malloc" : " 14 GB", | 20260508 12:35:49.895 | INFO | "pinned" : " 29 GB" | 20260508 12:35:49.895 | INFO | } | 20260508 12:35:49.895 | INFO | Perf report for the last 5.14257 seconds: | 20260508 12:35:49.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:49.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:49.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:49.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:49.895 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 12:35:49.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:49.895 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:35:49.895 | INFO | PRE_HQ : 65.3%, 1.6, 53.6ms, 0 | 20260508 12:35:49.895 | INFO | BAZWRITER : 67.1%, 2.7, 92.1ms, 0 | 20260508 12:35:49.895 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:35:49.896 | INFO | Uploaded 36GiB to GPU in 2.61881s (13.7467GiB/s) | 20260508 12:35:49.896 | INFO | Downloaded 38GiB to GPU in 2.73392s (13.8995GiB/s) | 20260508 12:35:55.033 | INFO | Analyzing chunk frames = [8151552,8152064) | 20260508 12:35:55.033 | INFO | MemInfo: { | 20260508 12:35:55.033 | INFO | "gpu" : " 17 GB", | 20260508 12:35:55.033 | INFO | "huge" : "", | 20260508 12:35:55.033 | INFO | "huge_shared" : " 29 GB", | 20260508 12:35:55.033 | INFO | "malloc" : " 14 GB", | 20260508 12:35:55.034 | INFO | "pinned" : " 29 GB" | 20260508 12:35:55.034 | INFO | } | 20260508 12:35:55.034 | INFO | Perf report for the last 5.13852 seconds: | 20260508 12:35:55.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:35:55.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:35:55.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:35:55.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:35:55.034 | INFO | Uploaded 36GiB to GPU in 2.41684s (14.8955GiB/s) | 20260508 12:35:55.034 | INFO | Downloaded 34GiB to GPU in 2.48501s (13.6821GiB/s) | 20260508 12:36:00.173 | INFO | Analyzing chunk frames = [8152064,8152576) | 20260508 12:36:00.173 | INFO | MemInfo: { | 20260508 12:36:00.173 | INFO | "gpu" : " 17 GB", | 20260508 12:36:00.173 | INFO | "huge" : "", | 20260508 12:36:00.173 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:00.173 | INFO | "malloc" : " 14 GB", | 20260508 12:36:00.173 | INFO | "pinned" : " 29 GB" | 20260508 12:36:00.173 | INFO | } | 20260508 12:36:00.173 | INFO | Perf report for the last 5.13897 seconds: | 20260508 12:36:00.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:00.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 12:36:00.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:36:00.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:00.173 | INFO | Uploaded 36GiB to GPU in 2.46098s (14.6283GiB/s) | 20260508 12:36:00.173 | INFO | Downloaded 34GiB to GPU in 2.46834s (13.7745GiB/s) | 20260508 12:36:05.316 | INFO | Analyzing chunk frames = [8152576,8153088) | 20260508 12:36:05.316 | INFO | MemInfo: { | 20260508 12:36:05.316 | INFO | "gpu" : " 17 GB", | 20260508 12:36:05.316 | INFO | "huge" : "", | 20260508 12:36:05.316 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:05.316 | INFO | "malloc" : " 14 GB", | 20260508 12:36:05.316 | INFO | "pinned" : " 29 GB" | 20260508 12:36:05.316 | INFO | } | 20260508 12:36:05.316 | INFO | Perf report for the last 5.14332 seconds: | 20260508 12:36:05.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:05.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 12:36:05.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:36:05.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:05.316 | INFO | Uploaded 36GiB to GPU in 2.4499s (14.6945GiB/s) | 20260508 12:36:05.316 | INFO | Downloaded 34GiB to GPU in 2.47308s (13.748GiB/s) | 20260508 12:36:10.454 | INFO | Analyzing chunk frames = [8153088,8153600) | 20260508 12:36:10.454 | INFO | MemInfo: { | 20260508 12:36:10.454 | INFO | "gpu" : " 17 GB", | 20260508 12:36:10.454 | INFO | "huge" : "", | 20260508 12:36:10.454 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:10.454 | INFO | "malloc" : " 14 GB", | 20260508 12:36:10.454 | INFO | "pinned" : " 29 GB" | 20260508 12:36:10.454 | INFO | } | 20260508 12:36:10.454 | INFO | Perf report for the last 5.13839 seconds: | 20260508 12:36:10.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:10.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.454 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:36:10.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.455 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:36:10.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:10.455 | INFO | Uploaded 36GiB to GPU in 2.40134s (14.9916GiB/s) | 20260508 12:36:10.455 | INFO | Downloaded 34GiB to GPU in 2.52655s (13.4571GiB/s) | 20260508 12:36:15.591 | INFO | Analyzing chunk frames = [8153600,8154112) | 20260508 12:36:15.591 | INFO | MemInfo: { | 20260508 12:36:15.591 | INFO | "gpu" : " 17 GB", | 20260508 12:36:15.591 | INFO | "huge" : "", | 20260508 12:36:15.591 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:15.591 | INFO | "malloc" : " 14 GB", | 20260508 12:36:15.591 | INFO | "pinned" : " 29 GB" | 20260508 12:36:15.591 | INFO | } | 20260508 12:36:15.591 | INFO | Perf report for the last 5.13646 seconds: | 20260508 12:36:15.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:15.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:36:15.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:36:15.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:15.591 | INFO | Uploaded 36GiB to GPU in 2.42596s (14.8395GiB/s) | 20260508 12:36:15.592 | INFO | Downloaded 34GiB to GPU in 2.45992s (13.8216GiB/s) | 20260508 12:36:20.730 | INFO | Analyzing chunk frames = [8154112,8154624) | 20260508 12:36:20.730 | INFO | MemInfo: { | 20260508 12:36:20.730 | INFO | "gpu" : " 17 GB", | 20260508 12:36:20.730 | INFO | "huge" : "", | 20260508 12:36:20.730 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:20.730 | INFO | "malloc" : " 14 GB", | 20260508 12:36:20.731 | INFO | "pinned" : " 29 GB" | 20260508 12:36:20.731 | INFO | } | 20260508 12:36:20.731 | INFO | Perf report for the last 5.13967 seconds: | 20260508 12:36:20.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:20.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 12:36:20.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:36:20.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:20.731 | INFO | Uploaded 36GiB to GPU in 2.45454s (14.6667GiB/s) | 20260508 12:36:20.731 | INFO | Downloaded 34GiB to GPU in 2.45984s (13.822GiB/s) | 20260508 12:36:25.869 | INFO | Analyzing chunk frames = [8154624,8155136) | 20260508 12:36:25.869 | INFO | MemInfo: { | 20260508 12:36:25.869 | INFO | "gpu" : " 17 GB", | 20260508 12:36:25.869 | INFO | "huge" : "", | 20260508 12:36:25.869 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:25.869 | INFO | "malloc" : " 14 GB", | 20260508 12:36:25.869 | INFO | "pinned" : " 29 GB" | 20260508 12:36:25.869 | INFO | } | 20260508 12:36:25.869 | INFO | Perf report for the last 5.13824 seconds: | 20260508 12:36:25.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:25.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 12:36:25.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:36:25.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:25.869 | INFO | Uploaded 36GiB to GPU in 2.50035s (14.398GiB/s) | 20260508 12:36:25.869 | INFO | Downloaded 34GiB to GPU in 2.48431s (13.6859GiB/s) | 20260508 12:36:25.966 | INFO | Starting a round of Baz IO | 20260508 12:36:29.383 | INFO | Populating RTMetrics report | 20260508 12:36:29.384 | INFO | RTMetrics report populated | 20260508 12:36:29.384 | INFO | Opening tmp RTMetrics json file | 20260508 12:36:29.384 | INFO | Serializing RTMetrics to json | 20260508 12:36:29.386 | INFO | Serialization done, swapping tmp file now... | 20260508 12:36:29.386 | INFO | RTMetrics done | 20260508 12:36:29.446 | INFO | Finished a round of Baz IO: Wrote 1.405 GiB in 3.48 seconds | 20260508 12:36:29.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:36:29.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160098% / 0% / 6.35783e-05% | 20260508 12:36:29.446 | INFO | Baz IO overall: | 20260508 12:36:29.446 | INFO | BytesWritten : 3493630925811 | 20260508 12:36:29.446 | INFO | NumEvents : 1676287505769 | 20260508 12:36:31.013 | INFO | Analyzing chunk frames = [8155136,8155648) | 20260508 12:36:31.013 | INFO | MemInfo: { | 20260508 12:36:31.013 | INFO | "gpu" : " 17 GB", | 20260508 12:36:31.013 | INFO | "huge" : "", | 20260508 12:36:31.013 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:31.013 | INFO | "malloc" : " 14 GB", | 20260508 12:36:31.013 | INFO | "pinned" : " 29 GB" | 20260508 12:36:31.013 | INFO | } | 20260508 12:36:31.013 | INFO | Perf report for the last 5.14451 seconds: | 20260508 12:36:31.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:31.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:31.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:31.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:31.014 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 12:36:31.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:31.014 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:36:31.014 | INFO | PRE_HQ : 65.6%, 1.6, 53.9ms, 0 | 20260508 12:36:31.014 | INFO | BAZWRITER : 66.9%, 2.4, 83.6ms, 0 | 20260508 12:36:31.014 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 12:36:31.014 | INFO | Uploaded 36GiB to GPU in 2.58143s (13.9458GiB/s) | 20260508 12:36:31.014 | INFO | Downloaded 38GiB to GPU in 2.71268s (14.0083GiB/s) | 20260508 12:36:36.150 | INFO | Analyzing chunk frames = [8155648,8156160) | 20260508 12:36:36.150 | INFO | MemInfo: { | 20260508 12:36:36.150 | INFO | "gpu" : " 17 GB", | 20260508 12:36:36.150 | INFO | "huge" : "", | 20260508 12:36:36.150 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:36.150 | INFO | "malloc" : " 14 GB", | 20260508 12:36:36.150 | INFO | "pinned" : " 29 GB" | 20260508 12:36:36.150 | INFO | } | 20260508 12:36:36.150 | INFO | Perf report for the last 5.13648 seconds: | 20260508 12:36:36.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:36.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:36:36.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 12:36:36.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:36.150 | INFO | Uploaded 36GiB to GPU in 2.43057s (14.8113GiB/s) | 20260508 12:36:36.150 | INFO | Downloaded 34GiB to GPU in 2.47087s (13.7603GiB/s) | 20260508 12:36:41.288 | INFO | Analyzing chunk frames = [8156160,8156672) | 20260508 12:36:41.288 | INFO | MemInfo: { | 20260508 12:36:41.288 | INFO | "gpu" : " 17 GB", | 20260508 12:36:41.288 | INFO | "huge" : "", | 20260508 12:36:41.288 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:41.288 | INFO | "malloc" : " 14 GB", | 20260508 12:36:41.288 | INFO | "pinned" : " 29 GB" | 20260508 12:36:41.288 | INFO | } | 20260508 12:36:41.288 | INFO | Perf report for the last 5.13828 seconds: | 20260508 12:36:41.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:41.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:36:41.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:36:41.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:41.288 | INFO | Uploaded 36GiB to GPU in 2.49364s (14.4367GiB/s) | 20260508 12:36:41.288 | INFO | Downloaded 34GiB to GPU in 2.45307s (13.8602GiB/s) | 20260508 12:36:46.425 | INFO | Analyzing chunk frames = [8156672,8157184) | 20260508 12:36:46.425 | INFO | MemInfo: { | 20260508 12:36:46.425 | INFO | "gpu" : " 17 GB", | 20260508 12:36:46.425 | INFO | "huge" : "", | 20260508 12:36:46.425 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:46.425 | INFO | "malloc" : " 14 GB", | 20260508 12:36:46.425 | INFO | "pinned" : " 29 GB" | 20260508 12:36:46.425 | INFO | } | 20260508 12:36:46.425 | INFO | Perf report for the last 5.13679 seconds: | 20260508 12:36:46.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:46.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 12:36:46.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:36:46.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:46.425 | INFO | Uploaded 36GiB to GPU in 2.50363s (14.3791GiB/s) | 20260508 12:36:46.425 | INFO | Downloaded 34GiB to GPU in 2.46624s (13.7862GiB/s) | 20260508 12:36:51.566 | INFO | Analyzing chunk frames = [8157184,8157696) | 20260508 12:36:51.566 | INFO | MemInfo: { | 20260508 12:36:51.566 | INFO | "gpu" : " 17 GB", | 20260508 12:36:51.566 | INFO | "huge" : "", | 20260508 12:36:51.566 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:51.566 | INFO | "malloc" : " 14 GB", | 20260508 12:36:51.566 | INFO | "pinned" : " 29 GB" | 20260508 12:36:51.566 | INFO | } | 20260508 12:36:51.567 | INFO | Perf report for the last 5.1416 seconds: | 20260508 12:36:51.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:51.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 12:36:51.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:36:51.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:51.567 | INFO | Uploaded 36GiB to GPU in 2.39776s (15.014GiB/s) | 20260508 12:36:51.567 | INFO | Downloaded 34GiB to GPU in 2.53039s (13.4367GiB/s) | 20260508 12:36:56.706 | INFO | Analyzing chunk frames = [8157696,8158208) | 20260508 12:36:56.706 | INFO | MemInfo: { | 20260508 12:36:56.706 | INFO | "gpu" : " 17 GB", | 20260508 12:36:56.706 | INFO | "huge" : "", | 20260508 12:36:56.707 | INFO | "huge_shared" : " 29 GB", | 20260508 12:36:56.707 | INFO | "malloc" : " 14 GB", | 20260508 12:36:56.707 | INFO | "pinned" : " 29 GB" | 20260508 12:36:56.707 | INFO | } | 20260508 12:36:56.707 | INFO | Perf report for the last 5.14006 seconds: | 20260508 12:36:56.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:36:56.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:36:56.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:36:56.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:36:56.707 | INFO | Uploaded 36GiB to GPU in 2.36941s (15.1936GiB/s) | 20260508 12:36:56.707 | INFO | Downloaded 34GiB to GPU in 2.45035s (13.8756GiB/s) | 20260508 12:37:01.851 | INFO | Analyzing chunk frames = [8158208,8158720) | 20260508 12:37:01.851 | INFO | MemInfo: { | 20260508 12:37:01.851 | INFO | "gpu" : " 17 GB", | 20260508 12:37:01.851 | INFO | "huge" : "", | 20260508 12:37:01.852 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:01.852 | INFO | "malloc" : " 14 GB", | 20260508 12:37:01.852 | INFO | "pinned" : " 29 GB" | 20260508 12:37:01.852 | INFO | } | 20260508 12:37:01.852 | INFO | Perf report for the last 5.14531 seconds: | 20260508 12:37:01.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:01.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 12:37:01.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:37:01.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:01.852 | INFO | Uploaded 36GiB to GPU in 2.39162s (15.0525GiB/s) | 20260508 12:37:01.852 | INFO | Downloaded 34GiB to GPU in 2.45602s (13.8436GiB/s) | 20260508 12:37:06.988 | INFO | Analyzing chunk frames = [8158720,8159232) | 20260508 12:37:06.988 | INFO | MemInfo: { | 20260508 12:37:06.988 | INFO | "gpu" : " 17 GB", | 20260508 12:37:06.988 | INFO | "huge" : "", | 20260508 12:37:06.989 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:06.989 | INFO | "malloc" : " 14 GB", | 20260508 12:37:06.989 | INFO | "pinned" : " 29 GB" | 20260508 12:37:06.989 | INFO | } | 20260508 12:37:06.989 | INFO | Perf report for the last 5.13687 seconds: | 20260508 12:37:06.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:06.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260508 12:37:06.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:37:06.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:06.989 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:37:06.989 | INFO | Total mem usage: 16.7 GiB | 20260508 12:37:06.989 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:37:06.989 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.2%) | 20260508 12:37:06.989 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:37:06.989 | INFO | High Water Breakdown: | 20260508 12:37:06.989 | INFO | 59.8% : 8.66 GiB ( 84% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:37:06.989 | INFO | 40.1% : 5.8 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:37:06.989 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:37:06.989 | INFO | ------------------------------------------ | 20260508 12:37:06.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:37:06.989 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:37:06.989 | INFO | Total mem usage: 26.5 GiB | 20260508 12:37:06.989 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:37:06.989 | INFO | Uploaded 36GiB to GPU in 2.34514s (15.3509GiB/s) | 20260508 12:37:06.989 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:37:06.990 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:37:06.989 | INFO | Downloaded 34GiB to GPU in 2.46338s (13.8022GiB/s) | 20260508 12:37:06.990 | INFO | High Water Breakdown: | 20260508 12:37:06.990 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:37:06.990 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:37:06.990 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:37:06.990 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:37:06.990 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:37:06.990 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:37:06.990 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:37:06.990 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:37:06.990 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:37:06.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:37:06.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:37:06.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:37:06.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:37:06.990 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:37:06.990 | INFO | ------------------------------------------ | 20260508 12:37:06.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:37:06.990 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:37:06.990 | INFO | Total mem usage: 27.8 GiB | 20260508 12:37:06.990 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:37:06.990 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:37:06.990 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:37:06.990 | INFO | High Water Breakdown: | 20260508 12:37:06.990 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:37:06.990 | INFO | ------------------------------------------ | 20260508 12:37:06.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:37:06.990 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:37:06.990 | INFO | Total mem usage: 15.7 GiB | 20260508 12:37:06.990 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:37:06.990 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:37:06.990 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:37:06.990 | INFO | High Water Breakdown: | 20260508 12:37:06.990 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:37:06.990 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:37:06.990 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:37:06.990 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:37:06.990 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:37:06.990 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:37:06.990 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:37:06.990 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:37:06.990 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:37:06.990 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:37:06.990 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:37:06.990 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:37:06.990 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:37:06.990 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:37:06.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:37:06.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:37:06.990 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:37:06.991 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:37:06.991 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:37:06.991 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:37:06.991 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:37:06.991 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:37:06.991 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:37:06.991 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:37:06.991 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:37:06.991 | INFO | ------------------------------------------ | 20260508 12:37:06.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:37:06.991 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:37:06.991 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:37:06.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:37:06.991 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 6.50e-01 | 20260508 12:37:06.991 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.06e-01, 1.16e-02 / 8.82e-03 | 20260508 12:37:06.991 | INFO | PerfReport: 12.19%, Baseline -- 3.97e+00 / 4.39e+00, 5.91e-01 / 2.12e-01 | 20260508 12:37:06.991 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.46e+00, 2.06e-01 / 7.18e-02 | 20260508 12:37:06.991 | INFO | PerfReport: 15.99%, DME -- 4.17e+01 / 4.52e+01, 3.61e+00 / 9.84e-01 | 20260508 12:37:06.991 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.86e+01 / 2.03e+01, 1.81e+00 / 1.26e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.32e+00 / 2.43e+00, 4.04e-01 / 2.35e-01 | 20260508 12:37:06.991 | INFO | PerfReport: 2.95%, Metrics -- 9.63e-01 / 1.09e+00, 2.39e-01 / 1.15e-01 | 20260508 12:37:06.991 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.23e-01, 4.62e-02 / 2.19e-02 | 20260508 12:37:06.991 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:37:06.991 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:37:06.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:37:06.991 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 24.35%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 24.88%, Download -- 2.61e+01 / 0.00e+00, 8.29e+00 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 31.08%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 2.01%, LockUpload -- 2.11e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 2.72%, LockDownload -- 2.86e+00 / 0.00e+00, 4.21e+00 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 14.94%, LockCompute -- 1.57e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 12:37:06.991 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260508 12:37:07.101 | INFO | Starting a round of Baz IO | 20260508 12:37:10.495 | INFO | Populating RTMetrics report | 20260508 12:37:10.496 | INFO | RTMetrics report populated | 20260508 12:37:10.496 | INFO | Opening tmp RTMetrics json file | 20260508 12:37:10.500 | INFO | Serializing RTMetrics to json | 20260508 12:37:10.501 | INFO | Serialization done, swapping tmp file now... | 20260508 12:37:10.502 | INFO | RTMetrics done | 20260508 12:37:10.559 | INFO | Finished a round of Baz IO: Wrote 1.405 GiB in 3.458 seconds | 20260508 12:37:10.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:37:10.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159268% / 3.97364e-06% / 6.75519e-05% | 20260508 12:37:10.559 | INFO | Baz IO overall: | 20260508 12:37:10.560 | INFO | BytesWritten : 3495139452307 | 20260508 12:37:10.560 | INFO | NumEvents : 1676994873976 | 20260508 12:37:12.126 | INFO | Analyzing chunk frames = [8159232,8159744) | 20260508 12:37:12.126 | INFO | MemInfo: { | 20260508 12:37:12.126 | INFO | "gpu" : " 17 GB", | 20260508 12:37:12.126 | INFO | "huge" : "", | 20260508 12:37:12.126 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:12.126 | INFO | "malloc" : " 14 GB", | 20260508 12:37:12.126 | INFO | "pinned" : " 29 GB" | 20260508 12:37:12.126 | INFO | } | 20260508 12:37:12.126 | INFO | Perf report for the last 5.13732 seconds: | 20260508 12:37:12.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:12.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:12.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:12.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:12.126 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:37:12.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:12.126 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 12:37:12.126 | INFO | PRE_HQ : 65.6%, 1.7, 56.3ms, 0 | 20260508 12:37:12.126 | INFO | BAZWRITER : 66.5%, 2.5, 85.0ms, 0 | 20260508 12:37:12.126 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:37:12.127 | INFO | Uploaded 36GiB to GPU in 2.62407s (13.7192GiB/s) | 20260508 12:37:12.127 | INFO | Downloaded 38GiB to GPU in 2.71805s (13.9806GiB/s) | 20260508 12:37:17.271 | INFO | Analyzing chunk frames = [8159744,8160256) | 20260508 12:37:17.271 | INFO | MemInfo: { | 20260508 12:37:17.271 | INFO | "gpu" : " 17 GB", | 20260508 12:37:17.271 | INFO | "huge" : "", | 20260508 12:37:17.271 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:17.271 | INFO | "malloc" : " 14 GB", | 20260508 12:37:17.271 | INFO | "pinned" : " 29 GB" | 20260508 12:37:17.271 | INFO | } | 20260508 12:37:17.271 | INFO | Perf report for the last 5.14486 seconds: | 20260508 12:37:17.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:17.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.271 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:37:17.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.271 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:37:17.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:17.272 | INFO | Uploaded 36GiB to GPU in 2.49319s (14.4393GiB/s) | 20260508 12:37:17.272 | INFO | Downloaded 34GiB to GPU in 2.47297s (13.7487GiB/s) | 20260508 12:37:22.403 | INFO | Analyzing chunk frames = [8160256,8160768) | 20260508 12:37:22.403 | INFO | MemInfo: { | 20260508 12:37:22.403 | INFO | "gpu" : " 17 GB", | 20260508 12:37:22.403 | INFO | "huge" : "", | 20260508 12:37:22.403 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:22.403 | INFO | "malloc" : " 14 GB", | 20260508 12:37:22.403 | INFO | "pinned" : " 29 GB" | 20260508 12:37:22.403 | INFO | } | 20260508 12:37:22.403 | INFO | Perf report for the last 5.13197 seconds: | 20260508 12:37:22.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:22.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 12:37:22.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:37:22.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:22.403 | INFO | Uploaded 36GiB to GPU in 2.42948s (14.818GiB/s) | 20260508 12:37:22.403 | INFO | Downloaded 34GiB to GPU in 2.44942s (13.8808GiB/s) | 20260508 12:37:27.542 | INFO | Analyzing chunk frames = [8160768,8161280) | 20260508 12:37:27.542 | INFO | MemInfo: { | 20260508 12:37:27.542 | INFO | "gpu" : " 17 GB", | 20260508 12:37:27.542 | INFO | "huge" : "", | 20260508 12:37:27.542 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:27.543 | INFO | "malloc" : " 14 GB", | 20260508 12:37:27.543 | INFO | "pinned" : " 29 GB" | 20260508 12:37:27.543 | INFO | } | 20260508 12:37:27.543 | INFO | Perf report for the last 5.14006 seconds: | 20260508 12:37:27.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:27.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 12:37:27.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:37:27.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:27.543 | INFO | Uploaded 36GiB to GPU in 2.52466s (14.2594GiB/s) | 20260508 12:37:27.543 | INFO | Downloaded 34GiB to GPU in 2.44884s (13.8841GiB/s) | 20260508 12:37:32.676 | INFO | Analyzing chunk frames = [8161280,8161792) | 20260508 12:37:32.676 | INFO | MemInfo: { | 20260508 12:37:32.676 | INFO | "gpu" : " 17 GB", | 20260508 12:37:32.676 | INFO | "huge" : "", | 20260508 12:37:32.677 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:32.677 | INFO | "malloc" : " 14 GB", | 20260508 12:37:32.677 | INFO | "pinned" : " 29 GB" | 20260508 12:37:32.677 | INFO | } | 20260508 12:37:32.677 | INFO | Perf report for the last 5.13352 seconds: | 20260508 12:37:32.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:32.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 12:37:32.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:37:32.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:32.677 | INFO | Uploaded 36GiB to GPU in 2.39341s (15.0413GiB/s) | 20260508 12:37:32.677 | INFO | Downloaded 34GiB to GPU in 2.50402s (13.5781GiB/s) | 20260508 12:37:37.821 | INFO | Analyzing chunk frames = [8161792,8162304) | 20260508 12:37:37.821 | INFO | MemInfo: { | 20260508 12:37:37.821 | INFO | "gpu" : " 17 GB", | 20260508 12:37:37.821 | INFO | "huge" : "", | 20260508 12:37:37.822 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:37.822 | INFO | "malloc" : " 14 GB", | 20260508 12:37:37.822 | INFO | "pinned" : " 29 GB" | 20260508 12:37:37.822 | INFO | } | 20260508 12:37:37.822 | INFO | Perf report for the last 5.14499 seconds: | 20260508 12:37:37.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:37.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:37:37.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:37:37.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:37.822 | INFO | Uploaded 36GiB to GPU in 2.4049s (14.9694GiB/s) | 20260508 12:37:37.822 | INFO | Downloaded 34GiB to GPU in 2.42698s (14.0092GiB/s) | 20260508 12:37:42.964 | INFO | Analyzing chunk frames = [8162304,8162816) | 20260508 12:37:42.964 | INFO | MemInfo: { | 20260508 12:37:42.964 | INFO | "gpu" : " 17 GB", | 20260508 12:37:42.964 | INFO | "huge" : "", | 20260508 12:37:42.964 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:42.964 | INFO | "malloc" : " 14 GB", | 20260508 12:37:42.964 | INFO | "pinned" : " 29 GB" | 20260508 12:37:42.964 | INFO | } | 20260508 12:37:42.964 | INFO | Perf report for the last 5.14219 seconds: | 20260508 12:37:42.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:42.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:37:42.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 12:37:42.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:42.964 | INFO | Uploaded 36GiB to GPU in 2.4628s (14.6175GiB/s) | 20260508 12:37:42.964 | INFO | Downloaded 34GiB to GPU in 2.4323s (13.9785GiB/s) | 20260508 12:37:48.102 | INFO | Analyzing chunk frames = [8162816,8163328) | 20260508 12:37:48.102 | INFO | MemInfo: { | 20260508 12:37:48.102 | INFO | "gpu" : " 17 GB", | 20260508 12:37:48.102 | INFO | "huge" : "", | 20260508 12:37:48.102 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:48.102 | INFO | "malloc" : " 14 GB", | 20260508 12:37:48.102 | INFO | "pinned" : " 29 GB" | 20260508 12:37:48.102 | INFO | } | 20260508 12:37:48.102 | INFO | Perf report for the last 5.13833 seconds: | 20260508 12:37:48.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:48.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.102 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 12:37:48.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.102 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:37:48.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:48.103 | INFO | Uploaded 36GiB to GPU in 2.48945s (14.461GiB/s) | 20260508 12:37:48.103 | INFO | Downloaded 34GiB to GPU in 2.44499s (13.906GiB/s) | 20260508 12:37:48.200 | INFO | Starting a round of Baz IO | 20260508 12:37:51.606 | INFO | Populating RTMetrics report | 20260508 12:37:51.606 | INFO | RTMetrics report populated | 20260508 12:37:51.606 | INFO | Opening tmp RTMetrics json file | 20260508 12:37:51.613 | INFO | Serializing RTMetrics to json | 20260508 12:37:51.614 | INFO | Serialization done, swapping tmp file now... | 20260508 12:37:51.615 | INFO | RTMetrics done | 20260508 12:37:51.678 | INFO | Finished a round of Baz IO: Wrote 1.405 GiB in 3.477 seconds | 20260508 12:37:51.678 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:37:51.678 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159613% / 3.97364e-06% / 6.35783e-05% | 20260508 12:37:51.678 | INFO | Baz IO overall: | 20260508 12:37:51.678 | INFO | BytesWritten : 3496648230343 | 20260508 12:37:51.678 | INFO | NumEvents : 1677702411684 | 20260508 12:37:53.237 | INFO | Analyzing chunk frames = [8163328,8163840) | 20260508 12:37:53.237 | INFO | MemInfo: { | 20260508 12:37:53.237 | INFO | "gpu" : " 17 GB", | 20260508 12:37:53.237 | INFO | "huge" : "", | 20260508 12:37:53.237 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:53.237 | INFO | "malloc" : " 14 GB", | 20260508 12:37:53.238 | INFO | "pinned" : " 29 GB" | 20260508 12:37:53.238 | INFO | } | 20260508 12:37:53.238 | INFO | Perf report for the last 5.13543 seconds: | 20260508 12:37:53.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:53.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:53.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:53.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:53.238 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 12:37:53.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:53.238 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:37:53.238 | INFO | PRE_HQ : 65.4%, 1.6, 54.1ms, 0 | 20260508 12:37:53.238 | INFO | BAZWRITER : 67.0%, 2.4, 83.0ms, 0 | 20260508 12:37:53.238 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:37:53.238 | INFO | Uploaded 36GiB to GPU in 2.63355s (13.6698GiB/s) | 20260508 12:37:53.238 | INFO | Downloaded 38GiB to GPU in 2.72243s (13.9581GiB/s) | 20260508 12:37:58.373 | INFO | Analyzing chunk frames = [8163840,8164352) | 20260508 12:37:58.373 | INFO | MemInfo: { | 20260508 12:37:58.373 | INFO | "gpu" : " 17 GB", | 20260508 12:37:58.373 | INFO | "huge" : "", | 20260508 12:37:58.373 | INFO | "huge_shared" : " 29 GB", | 20260508 12:37:58.373 | INFO | "malloc" : " 14 GB", | 20260508 12:37:58.374 | INFO | "pinned" : " 29 GB" | 20260508 12:37:58.374 | INFO | } | 20260508 12:37:58.374 | INFO | Perf report for the last 5.13605 seconds: | 20260508 12:37:58.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:37:58.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:37:58.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:37:58.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:37:58.374 | INFO | Uploaded 36GiB to GPU in 2.49125s (14.4506GiB/s) | 20260508 12:37:58.374 | INFO | Downloaded 34GiB to GPU in 2.46406s (13.7983GiB/s) | 20260508 12:38:03.516 | INFO | Analyzing chunk frames = [8164352,8164864) | 20260508 12:38:03.516 | INFO | MemInfo: { | 20260508 12:38:03.516 | INFO | "gpu" : " 17 GB", | 20260508 12:38:03.516 | INFO | "huge" : "", | 20260508 12:38:03.516 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:03.516 | INFO | "malloc" : " 14 GB", | 20260508 12:38:03.516 | INFO | "pinned" : " 29 GB" | 20260508 12:38:03.517 | INFO | } | 20260508 12:38:03.517 | INFO | Perf report for the last 5.14288 seconds: | 20260508 12:38:03.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:03.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 12:38:03.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:38:03.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:03.517 | INFO | Uploaded 36GiB to GPU in 2.51411s (14.3192GiB/s) | 20260508 12:38:03.517 | INFO | Downloaded 34GiB to GPU in 2.4386s (13.9424GiB/s) | 20260508 12:38:08.661 | INFO | Analyzing chunk frames = [8164864,8165376) | 20260508 12:38:08.661 | INFO | MemInfo: { | 20260508 12:38:08.661 | INFO | "gpu" : " 17 GB", | 20260508 12:38:08.661 | INFO | "huge" : "", | 20260508 12:38:08.662 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:08.662 | INFO | "malloc" : " 14 GB", | 20260508 12:38:08.662 | INFO | "pinned" : " 29 GB" | 20260508 12:38:08.662 | INFO | } | 20260508 12:38:08.662 | INFO | Perf report for the last 5.1452 seconds: | 20260508 12:38:08.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:08.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 12:38:08.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:38:08.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:08.662 | INFO | Uploaded 36GiB to GPU in 2.48644s (14.4785GiB/s) | 20260508 12:38:08.662 | INFO | Downloaded 34GiB to GPU in 2.44498s (13.9061GiB/s) | 20260508 12:38:13.799 | INFO | Analyzing chunk frames = [8165376,8165888) | 20260508 12:38:13.799 | INFO | MemInfo: { | 20260508 12:38:13.799 | INFO | "gpu" : " 17 GB", | 20260508 12:38:13.799 | INFO | "huge" : "", | 20260508 12:38:13.799 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:13.799 | INFO | "malloc" : " 14 GB", | 20260508 12:38:13.799 | INFO | "pinned" : " 29 GB" | 20260508 12:38:13.799 | INFO | } | 20260508 12:38:13.799 | INFO | Perf report for the last 5.13742 seconds: | 20260508 12:38:13.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:13.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:38:13.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:38:13.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:13.799 | INFO | Uploaded 36GiB to GPU in 2.36151s (15.2445GiB/s) | 20260508 12:38:13.800 | INFO | Downloaded 34GiB to GPU in 2.51426s (13.5229GiB/s) | 20260508 12:38:18.932 | INFO | Analyzing chunk frames = [8165888,8166400) | 20260508 12:38:18.932 | INFO | MemInfo: { | 20260508 12:38:18.932 | INFO | "gpu" : " 17 GB", | 20260508 12:38:18.932 | INFO | "huge" : "", | 20260508 12:38:18.932 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:18.932 | INFO | "malloc" : " 14 GB", | 20260508 12:38:18.932 | INFO | "pinned" : " 29 GB" | 20260508 12:38:18.932 | INFO | } | 20260508 12:38:18.932 | INFO | Perf report for the last 5.13269 seconds: | 20260508 12:38:18.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:18.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 12:38:18.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 12:38:18.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:18.932 | INFO | Uploaded 36GiB to GPU in 2.43026s (14.8132GiB/s) | 20260508 12:38:18.932 | INFO | Downloaded 34GiB to GPU in 2.43908s (13.9397GiB/s) | 20260508 12:38:24.077 | INFO | Analyzing chunk frames = [8166400,8166912) | 20260508 12:38:24.077 | INFO | MemInfo: { | 20260508 12:38:24.077 | INFO | "gpu" : " 17 GB", | 20260508 12:38:24.077 | INFO | "huge" : "", | 20260508 12:38:24.077 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:24.077 | INFO | "malloc" : " 14 GB", | 20260508 12:38:24.077 | INFO | "pinned" : " 29 GB" | 20260508 12:38:24.077 | INFO | } | 20260508 12:38:24.077 | INFO | Perf report for the last 5.14548 seconds: | 20260508 12:38:24.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:24.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.078 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 12:38:24.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.078 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:38:24.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:24.078 | INFO | Uploaded 36GiB to GPU in 2.46195s (14.6225GiB/s) | 20260508 12:38:24.078 | INFO | Downloaded 34GiB to GPU in 2.44299s (13.9174GiB/s) | 20260508 12:38:29.216 | INFO | Analyzing chunk frames = [8166912,8167424) | 20260508 12:38:29.216 | INFO | MemInfo: { | 20260508 12:38:29.216 | INFO | "gpu" : " 17 GB", | 20260508 12:38:29.216 | INFO | "huge" : "", | 20260508 12:38:29.216 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:29.216 | INFO | "malloc" : " 14 GB", | 20260508 12:38:29.216 | INFO | "pinned" : " 29 GB" | 20260508 12:38:29.216 | INFO | } | 20260508 12:38:29.216 | INFO | Perf report for the last 5.13877 seconds: | 20260508 12:38:29.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:29.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.216 | INFO | ANALYSIS : 64.8%, 3.0, 101.3ms, 0 | 20260508 12:38:29.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.216 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:38:29.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:29.217 | INFO | Uploaded 36GiB to GPU in 2.35369s (15.2951GiB/s) | 20260508 12:38:29.217 | INFO | Downloaded 34GiB to GPU in 2.44145s (13.9262GiB/s) | 20260508 12:38:29.217 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:38:29.217 | INFO | Total mem usage: 16.7 GiB | 20260508 12:38:29.217 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:38:29.217 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:38:29.217 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:38:29.217 | INFO | High Water Breakdown: | 20260508 12:38:29.217 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:38:29.217 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:38:29.217 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:38:29.217 | INFO | ------------------------------------------ | 20260508 12:38:29.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:38:29.217 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:38:29.217 | INFO | Total mem usage: 26.5 GiB | 20260508 12:38:29.217 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:38:29.217 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:38:29.217 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:38:29.217 | INFO | High Water Breakdown: | 20260508 12:38:29.217 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:38:29.217 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:38:29.217 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:38:29.217 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:38:29.217 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:38:29.217 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:38:29.217 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:38:29.217 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:38:29.217 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:38:29.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:38:29.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:38:29.217 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:38:29.217 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:38:29.217 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:38:29.217 | INFO | ------------------------------------------ | 20260508 12:38:29.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:38:29.217 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:38:29.217 | INFO | Total mem usage: 27.8 GiB | 20260508 12:38:29.217 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:38:29.217 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:38:29.217 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:38:29.217 | INFO | High Water Breakdown: | 20260508 12:38:29.217 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:38:29.217 | INFO | ------------------------------------------ | 20260508 12:38:29.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:38:29.217 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:38:29.217 | INFO | Total mem usage: 15.7 GiB | 20260508 12:38:29.217 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:38:29.217 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:38:29.217 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:38:29.217 | INFO | High Water Breakdown: | 20260508 12:38:29.217 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:38:29.217 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:38:29.217 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:38:29.217 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:38:29.217 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:38:29.217 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:38:29.217 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:38:29.217 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:38:29.217 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:38:29.217 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:38:29.217 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:38:29.217 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:38:29.217 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:38:29.217 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:38:29.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:38:29.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:38:29.217 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:38:29.218 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:38:29.218 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:38:29.218 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:38:29.218 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:38:29.218 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:38:29.218 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:38:29.218 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:38:29.218 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:38:29.218 | INFO | ------------------------------------------ | 20260508 12:38:29.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:38:29.218 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:38:29.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:38:29.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:38:29.218 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.41e+01 / 6.03e-01 | 20260508 12:38:29.218 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.05e-01, 1.25e-02 / 3.74e-03 | 20260508 12:38:29.218 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.44e+00, 5.87e-01 / 1.97e-01 | 20260508 12:38:29.218 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.15e-01 | 20260508 12:38:29.218 | INFO | PerfReport: 16.02%, DME -- 4.18e+01 / 4.52e+01, 3.66e+00 / 9.40e-01 | 20260508 12:38:29.218 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.86e+01 / 2.00e+01, 1.79e+00 / 1.24e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.46e+00, 3.98e-01 / 2.52e-01 | 20260508 12:38:29.218 | INFO | PerfReport: 2.96%, Metrics -- 9.66e-01 / 1.06e+00, 2.38e-01 / 6.50e-02 | 20260508 12:38:29.218 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.92e-01 / 3.25e-01, 4.66e-02 / 2.37e-02 | 20260508 12:38:29.218 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:38:29.218 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:38:29.218 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:38:29.218 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 24.53%, Upload -- 2.58e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 24.69%, Download -- 2.59e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 31.06%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 2.22%, LockUpload -- 2.33e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 2.53%, LockDownload -- 2.66e+00 / 0.00e+00, 3.97e+00 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 14.95%, LockCompute -- 1.57e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:38:29.218 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 12:38:29.314 | INFO | Starting a round of Baz IO | 20260508 12:38:32.719 | INFO | Populating RTMetrics report | 20260508 12:38:32.719 | INFO | RTMetrics report populated | 20260508 12:38:32.719 | INFO | Opening tmp RTMetrics json file | 20260508 12:38:32.728 | INFO | Serializing RTMetrics to json | 20260508 12:38:32.729 | INFO | Serialization done, swapping tmp file now... | 20260508 12:38:32.730 | INFO | RTMetrics done | 20260508 12:38:32.785 | INFO | Finished a round of Baz IO: Wrote 1.407 GiB in 3.471 seconds | 20260508 12:38:32.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:38:32.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160388% / 0% / 4.37101e-05% | 20260508 12:38:32.786 | INFO | Baz IO overall: | 20260508 12:38:32.786 | INFO | BytesWritten : 3498158666971 | 20260508 12:38:32.786 | INFO | NumEvents : 1678411253381 | 20260508 12:38:34.353 | INFO | Analyzing chunk frames = [8167424,8167936) | 20260508 12:38:34.353 | INFO | MemInfo: { | 20260508 12:38:34.353 | INFO | "gpu" : " 17 GB", | 20260508 12:38:34.353 | INFO | "huge" : "", | 20260508 12:38:34.353 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:34.353 | INFO | "malloc" : " 14 GB", | 20260508 12:38:34.353 | INFO | "pinned" : " 29 GB" | 20260508 12:38:34.353 | INFO | } | 20260508 12:38:34.354 | INFO | Perf report for the last 5.13729 seconds: | 20260508 12:38:34.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:34.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:34.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:34.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:34.354 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 12:38:34.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:34.354 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:38:34.354 | INFO | PRE_HQ : 65.4%, 1.6, 53.8ms, 0 | 20260508 12:38:34.354 | INFO | BAZWRITER : 66.9%, 2.4, 83.9ms, 0 | 20260508 12:38:34.354 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:38:34.354 | INFO | Uploaded 36GiB to GPU in 2.60695s (13.8093GiB/s) | 20260508 12:38:34.354 | INFO | Downloaded 38GiB to GPU in 2.7174s (13.984GiB/s) | 20260508 12:38:39.493 | INFO | Analyzing chunk frames = [8167936,8168448) | 20260508 12:38:39.493 | INFO | MemInfo: { | 20260508 12:38:39.493 | INFO | "gpu" : " 17 GB", | 20260508 12:38:39.493 | INFO | "huge" : "", | 20260508 12:38:39.493 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:39.493 | INFO | "malloc" : " 14 GB", | 20260508 12:38:39.493 | INFO | "pinned" : " 29 GB" | 20260508 12:38:39.493 | INFO | } | 20260508 12:38:39.493 | INFO | Perf report for the last 5.13969 seconds: | 20260508 12:38:39.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:39.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.493 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:38:39.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.493 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:38:39.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:39.494 | INFO | Uploaded 36GiB to GPU in 2.44368s (14.7319GiB/s) | 20260508 12:38:39.494 | INFO | Downloaded 34GiB to GPU in 2.45937s (13.8247GiB/s) | 20260508 12:38:44.634 | INFO | Analyzing chunk frames = [8168448,8168960) | 20260508 12:38:44.634 | INFO | MemInfo: { | 20260508 12:38:44.634 | INFO | "gpu" : " 17 GB", | 20260508 12:38:44.634 | INFO | "huge" : "", | 20260508 12:38:44.634 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:44.634 | INFO | "malloc" : " 14 GB", | 20260508 12:38:44.634 | INFO | "pinned" : " 29 GB" | 20260508 12:38:44.634 | INFO | } | 20260508 12:38:44.634 | INFO | Perf report for the last 5.14056 seconds: | 20260508 12:38:44.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:44.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 12:38:44.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:38:44.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:44.634 | INFO | Uploaded 36GiB to GPU in 2.50733s (14.3579GiB/s) | 20260508 12:38:44.635 | INFO | Downloaded 34GiB to GPU in 2.46068s (13.8173GiB/s) | 20260508 12:38:49.773 | INFO | Analyzing chunk frames = [8168960,8169472) | 20260508 12:38:49.773 | INFO | MemInfo: { | 20260508 12:38:49.773 | INFO | "gpu" : " 17 GB", | 20260508 12:38:49.773 | INFO | "huge" : "", | 20260508 12:38:49.774 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:49.774 | INFO | "malloc" : " 14 GB", | 20260508 12:38:49.774 | INFO | "pinned" : " 29 GB" | 20260508 12:38:49.774 | INFO | } | 20260508 12:38:49.774 | INFO | Perf report for the last 5.13993 seconds: | 20260508 12:38:49.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:49.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | ANALYSIS : 65.4%, 3.0, 101.5ms, 0 | 20260508 12:38:49.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 12:38:49.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:49.774 | INFO | Uploaded 36GiB to GPU in 2.40906s (14.9436GiB/s) | 20260508 12:38:49.774 | INFO | Downloaded 34GiB to GPU in 2.47235s (13.7521GiB/s) | 20260508 12:38:54.914 | INFO | Analyzing chunk frames = [8169472,8169984) | 20260508 12:38:54.914 | INFO | MemInfo: { | 20260508 12:38:54.914 | INFO | "gpu" : " 17 GB", | 20260508 12:38:54.914 | INFO | "huge" : "", | 20260508 12:38:54.914 | INFO | "huge_shared" : " 29 GB", | 20260508 12:38:54.914 | INFO | "malloc" : " 14 GB", | 20260508 12:38:54.914 | INFO | "pinned" : " 29 GB" | 20260508 12:38:54.914 | INFO | } | 20260508 12:38:54.914 | INFO | Perf report for the last 5.14009 seconds: | 20260508 12:38:54.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:38:54.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:38:54.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:38:54.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:38:54.914 | INFO | Uploaded 36GiB to GPU in 2.3482s (15.3309GiB/s) | 20260508 12:38:54.914 | INFO | Downloaded 34GiB to GPU in 2.49388s (13.6334GiB/s) | 20260508 12:39:00.055 | INFO | Analyzing chunk frames = [8169984,8170496) | 20260508 12:39:00.055 | INFO | MemInfo: { | 20260508 12:39:00.055 | INFO | "gpu" : " 17 GB", | 20260508 12:39:00.055 | INFO | "huge" : "", | 20260508 12:39:00.055 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:00.055 | INFO | "malloc" : " 14 GB", | 20260508 12:39:00.055 | INFO | "pinned" : " 29 GB" | 20260508 12:39:00.055 | INFO | } | 20260508 12:39:00.055 | INFO | Perf report for the last 5.14121 seconds: | 20260508 12:39:00.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:00.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:39:00.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:39:00.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:00.055 | INFO | Uploaded 36GiB to GPU in 2.42139s (14.8675GiB/s) | 20260508 12:39:00.055 | INFO | Downloaded 34GiB to GPU in 2.42785s (14.0042GiB/s) | 20260508 12:39:05.186 | INFO | Analyzing chunk frames = [8170496,8171008) | 20260508 12:39:05.186 | INFO | MemInfo: { | 20260508 12:39:05.186 | INFO | "gpu" : " 17 GB", | 20260508 12:39:05.186 | INFO | "huge" : "", | 20260508 12:39:05.186 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:05.186 | INFO | "malloc" : " 14 GB", | 20260508 12:39:05.186 | INFO | "pinned" : " 29 GB" | 20260508 12:39:05.186 | INFO | } | 20260508 12:39:05.187 | INFO | Perf report for the last 5.13145 seconds: | 20260508 12:39:05.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:05.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 12:39:05.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:39:05.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:05.187 | INFO | Uploaded 36GiB to GPU in 2.34994s (15.3195GiB/s) | 20260508 12:39:05.187 | INFO | Downloaded 34GiB to GPU in 2.39456s (14.1988GiB/s) | 20260508 12:39:10.333 | INFO | Analyzing chunk frames = [8171008,8171520) | 20260508 12:39:10.333 | INFO | MemInfo: { | 20260508 12:39:10.333 | INFO | "gpu" : " 17 GB", | 20260508 12:39:10.334 | INFO | "huge" : "", | 20260508 12:39:10.334 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:10.334 | INFO | "malloc" : " 14 GB", | 20260508 12:39:10.334 | INFO | "pinned" : " 29 GB" | 20260508 12:39:10.334 | INFO | } | 20260508 12:39:10.334 | INFO | Perf report for the last 5.14713 seconds: | 20260508 12:39:10.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:10.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | ANALYSIS : 65.5%, 3.0, 102.6ms, 0 | 20260508 12:39:10.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:39:10.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:10.334 | INFO | Uploaded 36GiB to GPU in 2.45873s (14.6417GiB/s) | 20260508 12:39:10.334 | INFO | Downloaded 34GiB to GPU in 2.42729s (14.0074GiB/s) | 20260508 12:39:10.431 | INFO | Starting a round of Baz IO | 20260508 12:39:13.839 | INFO | Populating RTMetrics report | 20260508 12:39:13.839 | INFO | RTMetrics report populated | 20260508 12:39:13.839 | INFO | Opening tmp RTMetrics json file | 20260508 12:39:13.846 | INFO | Serializing RTMetrics to json | 20260508 12:39:13.848 | INFO | Serialization done, swapping tmp file now... | 20260508 12:39:13.849 | INFO | RTMetrics done | 20260508 12:39:13.912 | INFO | Finished a round of Baz IO: Wrote 1.404 GiB in 3.481 seconds | 20260508 12:39:13.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.00% / 59.99% | 20260508 12:39:13.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159144% / 0% / 6.35783e-05% | 20260508 12:39:13.913 | INFO | Baz IO overall: | 20260508 12:39:13.913 | INFO | BytesWritten : 3499666247637 | 20260508 12:39:13.913 | INFO | NumEvents : 1679117979509 | 20260508 12:39:15.474 | INFO | Analyzing chunk frames = [8171520,8172032) | 20260508 12:39:15.474 | INFO | MemInfo: { | 20260508 12:39:15.474 | INFO | "gpu" : " 17 GB", | 20260508 12:39:15.474 | INFO | "huge" : "", | 20260508 12:39:15.474 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:15.474 | INFO | "malloc" : " 14 GB", | 20260508 12:39:15.474 | INFO | "pinned" : " 29 GB" | 20260508 12:39:15.474 | INFO | } | 20260508 12:39:15.474 | INFO | Perf report for the last 5.14036 seconds: | 20260508 12:39:15.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:15.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:15.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:15.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:15.474 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 12:39:15.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:15.474 | INFO | RT_METRICS : 38.0%, 1.0, 20.0ms, 0 | 20260508 12:39:15.474 | INFO | PRE_HQ : 65.4%, 1.6, 53.7ms, 0 | 20260508 12:39:15.474 | INFO | BAZWRITER : 67.0%, 2.4, 84.4ms, 0 | 20260508 12:39:15.474 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 12:39:15.475 | INFO | Uploaded 36GiB to GPU in 2.64037s (13.6344GiB/s) | 20260508 12:39:15.475 | INFO | Downloaded 38GiB to GPU in 2.71757s (13.9831GiB/s) | 20260508 12:39:20.613 | INFO | Analyzing chunk frames = [8172032,8172544) | 20260508 12:39:20.613 | INFO | MemInfo: { | 20260508 12:39:20.613 | INFO | "gpu" : " 17 GB", | 20260508 12:39:20.613 | INFO | "huge" : "", | 20260508 12:39:20.613 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:20.613 | INFO | "malloc" : " 14 GB", | 20260508 12:39:20.613 | INFO | "pinned" : " 29 GB" | 20260508 12:39:20.613 | INFO | } | 20260508 12:39:20.613 | INFO | Perf report for the last 5.13885 seconds: | 20260508 12:39:20.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:20.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:39:20.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:39:20.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:20.613 | INFO | Uploaded 36GiB to GPU in 2.42041s (14.8735GiB/s) | 20260508 12:39:20.613 | INFO | Downloaded 34GiB to GPU in 2.46947s (13.7682GiB/s) | 20260508 12:39:25.746 | INFO | Analyzing chunk frames = [8172544,8173056) | 20260508 12:39:25.746 | INFO | MemInfo: { | 20260508 12:39:25.746 | INFO | "gpu" : " 17 GB", | 20260508 12:39:25.746 | INFO | "huge" : "", | 20260508 12:39:25.747 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:25.747 | INFO | "malloc" : " 14 GB", | 20260508 12:39:25.747 | INFO | "pinned" : " 29 GB" | 20260508 12:39:25.747 | INFO | } | 20260508 12:39:25.747 | INFO | Perf report for the last 5.13377 seconds: | 20260508 12:39:25.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:25.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 12:39:25.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 12:39:25.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:25.747 | INFO | Uploaded 36GiB to GPU in 2.44559s (14.7204GiB/s) | 20260508 12:39:25.747 | INFO | Downloaded 34GiB to GPU in 2.44346s (13.9147GiB/s) | 20260508 12:39:30.884 | INFO | Analyzing chunk frames = [8173056,8173568) | 20260508 12:39:30.884 | INFO | MemInfo: { | 20260508 12:39:30.884 | INFO | "gpu" : " 17 GB", | 20260508 12:39:30.884 | INFO | "huge" : "", | 20260508 12:39:30.884 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:30.884 | INFO | "malloc" : " 14 GB", | 20260508 12:39:30.884 | INFO | "pinned" : " 29 GB" | 20260508 12:39:30.885 | INFO | } | 20260508 12:39:30.885 | INFO | Perf report for the last 5.13801 seconds: | 20260508 12:39:30.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:30.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | ANALYSIS : 65.6%, 3.0, 101.8ms, 0 | 20260508 12:39:30.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:39:30.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:30.885 | INFO | Uploaded 36GiB to GPU in 2.40353s (14.978GiB/s) | 20260508 12:39:30.885 | INFO | Downloaded 34GiB to GPU in 2.46978s (13.7664GiB/s) | 20260508 12:39:36.022 | INFO | Analyzing chunk frames = [8173568,8174080) | 20260508 12:39:36.022 | INFO | MemInfo: { | 20260508 12:39:36.022 | INFO | "gpu" : " 17 GB", | 20260508 12:39:36.022 | INFO | "huge" : "", | 20260508 12:39:36.023 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:36.023 | INFO | "malloc" : " 14 GB", | 20260508 12:39:36.023 | INFO | "pinned" : " 29 GB" | 20260508 12:39:36.023 | INFO | } | 20260508 12:39:36.023 | INFO | Perf report for the last 5.13801 seconds: | 20260508 12:39:36.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:36.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:39:36.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:39:36.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:36.023 | INFO | Uploaded 36GiB to GPU in 2.28595s (15.7484GiB/s) | 20260508 12:39:36.023 | INFO | Downloaded 34GiB to GPU in 2.49732s (13.6146GiB/s) | 20260508 12:39:41.170 | INFO | Analyzing chunk frames = [8174080,8174592) | 20260508 12:39:41.170 | INFO | MemInfo: { | 20260508 12:39:41.170 | INFO | "gpu" : " 17 GB", | 20260508 12:39:41.170 | INFO | "huge" : "", | 20260508 12:39:41.170 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:41.170 | INFO | "malloc" : " 14 GB", | 20260508 12:39:41.170 | INFO | "pinned" : " 29 GB" | 20260508 12:39:41.171 | INFO | } | 20260508 12:39:41.171 | INFO | Perf report for the last 5.1479 seconds: | 20260508 12:39:41.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:41.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 12:39:41.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:39:41.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:41.171 | INFO | Uploaded 36GiB to GPU in 2.3638s (15.2297GiB/s) | 20260508 12:39:41.171 | INFO | Downloaded 34GiB to GPU in 2.44932s (13.8814GiB/s) | 20260508 12:39:46.308 | INFO | Analyzing chunk frames = [8174592,8175104) | 20260508 12:39:46.308 | INFO | MemInfo: { | 20260508 12:39:46.308 | INFO | "gpu" : " 17 GB", | 20260508 12:39:46.308 | INFO | "huge" : "", | 20260508 12:39:46.308 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:46.309 | INFO | "malloc" : " 14 GB", | 20260508 12:39:46.309 | INFO | "pinned" : " 29 GB" | 20260508 12:39:46.309 | INFO | } | 20260508 12:39:46.309 | INFO | Perf report for the last 5.13801 seconds: | 20260508 12:39:46.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:46.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 12:39:46.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:39:46.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:46.309 | INFO | Uploaded 36GiB to GPU in 2.46725s (14.5912GiB/s) | 20260508 12:39:46.309 | INFO | Downloaded 34GiB to GPU in 2.47034s (13.7633GiB/s) | 20260508 12:39:51.440 | INFO | Analyzing chunk frames = [8175104,8175616) | 20260508 12:39:51.440 | INFO | MemInfo: { | 20260508 12:39:51.440 | INFO | "gpu" : " 17 GB", | 20260508 12:39:51.440 | INFO | "huge" : "", | 20260508 12:39:51.440 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:51.440 | INFO | "malloc" : " 14 GB", | 20260508 12:39:51.440 | INFO | "pinned" : " 29 GB" | 20260508 12:39:51.440 | INFO | } | 20260508 12:39:51.440 | INFO | Perf report for the last 5.13141 seconds: | 20260508 12:39:51.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:51.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260508 12:39:51.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 12:39:51.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:51.440 | INFO | Uploaded 36GiB to GPU in 2.33873s (15.3929GiB/s) | 20260508 12:39:51.441 | INFO | Downloaded 34GiB to GPU in 2.44079s (13.9299GiB/s) | 20260508 12:39:51.441 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:39:51.441 | INFO | Total mem usage: 16.7 GiB | 20260508 12:39:51.441 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:39:51.441 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:39:51.441 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:39:51.441 | INFO | High Water Breakdown: | 20260508 12:39:51.441 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:39:51.441 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:39:51.441 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:39:51.441 | INFO | ------------------------------------------ | 20260508 12:39:51.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:39:51.441 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:39:51.441 | INFO | Total mem usage: 26.5 GiB | 20260508 12:39:51.441 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:39:51.441 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:39:51.441 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:39:51.441 | INFO | High Water Breakdown: | 20260508 12:39:51.441 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:39:51.441 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:39:51.441 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:39:51.441 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:39:51.441 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:39:51.441 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:39:51.441 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:39:51.441 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:39:51.441 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:39:51.441 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:39:51.441 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:39:51.441 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:39:51.441 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:39:51.441 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:39:51.441 | INFO | ------------------------------------------ | 20260508 12:39:51.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:39:51.441 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:39:51.441 | INFO | Total mem usage: 27.8 GiB | 20260508 12:39:51.441 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:39:51.441 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 12:39:51.441 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 12:39:51.441 | INFO | High Water Breakdown: | 20260508 12:39:51.441 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 12:39:51.441 | INFO | ------------------------------------------ | 20260508 12:39:51.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:39:51.441 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:39:51.441 | INFO | Total mem usage: 15.7 GiB | 20260508 12:39:51.441 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:39:51.441 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:39:51.441 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:39:51.441 | INFO | High Water Breakdown: | 20260508 12:39:51.441 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:39:51.441 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:39:51.441 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:39:51.441 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:39:51.441 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:39:51.441 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:39:51.441 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:39:51.441 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:39:51.441 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:39:51.441 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:39:51.441 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:39:51.441 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:39:51.441 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:39:51.441 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:39:51.441 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:39:51.442 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:39:51.442 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:39:51.442 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:39:51.442 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:39:51.442 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:39:51.442 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:39:51.442 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:39:51.442 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:39:51.442 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:39:51.442 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:39:51.442 | INFO | ------------------------------------------ | 20260508 12:39:51.442 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:39:51.442 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:39:51.442 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:39:51.442 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:39:51.442 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.41e+01 / 4.96e-01 | 20260508 12:39:51.442 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.01e-01, 1.27e-02 / 5.85e-03 | 20260508 12:39:51.442 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.45e+00, 6.04e-01 / 1.81e-01 | 20260508 12:39:51.442 | INFO | PerfReport: 3.41%, Histogram -- 1.12e+00 / 1.49e+00, 2.12e-01 / 9.43e-02 | 20260508 12:39:51.442 | INFO | PerfReport: 16.01%, DME -- 4.20e+01 / 4.57e+01, 3.73e+00 / 9.06e-01 | 20260508 12:39:51.442 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.87e+01 / 1.95e+01, 1.85e+00 / 1.10e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.31e+00 / 2.56e+00, 4.06e-01 / 2.31e-01 | 20260508 12:39:51.442 | INFO | PerfReport: 2.95%, Metrics -- 9.65e-01 / 1.10e+00, 2.41e-01 / 7.07e-02 | 20260508 12:39:51.442 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.24e-01, 4.63e-02 / 2.02e-02 | 20260508 12:39:51.442 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:39:51.442 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:39:51.442 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:39:51.442 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 24.19%, Upload -- 2.53e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 24.75%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 31.27%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 2.15%, LockUpload -- 2.25e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 2.57%, LockDownload -- 2.69e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 15.06%, LockCompute -- 1.58e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 12:39:51.442 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260508 12:39:51.537 | INFO | Starting a round of Baz IO | 20260508 12:39:54.954 | INFO | Populating RTMetrics report | 20260508 12:39:54.954 | INFO | RTMetrics report populated | 20260508 12:39:54.954 | INFO | Opening tmp RTMetrics json file | 20260508 12:39:54.955 | INFO | Serializing RTMetrics to json | 20260508 12:39:54.956 | INFO | Serialization done, swapping tmp file now... | 20260508 12:39:54.957 | INFO | RTMetrics done | 20260508 12:39:55.018 | INFO | Finished a round of Baz IO: Wrote 1.403 GiB in 3.48 seconds | 20260508 12:39:55.018 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 59.99% | 20260508 12:39:55.018 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158656% / 0% / 7.54992e-05% | 20260508 12:39:55.018 | INFO | Baz IO overall: | 20260508 12:39:55.018 | INFO | BytesWritten : 3501172659442 | 20260508 12:39:55.018 | INFO | NumEvents : 1679824052436 | 20260508 12:39:56.586 | INFO | Analyzing chunk frames = [8175616,8176128) | 20260508 12:39:56.586 | INFO | MemInfo: { | 20260508 12:39:56.586 | INFO | "gpu" : " 17 GB", | 20260508 12:39:56.586 | INFO | "huge" : "", | 20260508 12:39:56.586 | INFO | "huge_shared" : " 29 GB", | 20260508 12:39:56.586 | INFO | "malloc" : " 14 GB", | 20260508 12:39:56.586 | INFO | "pinned" : " 29 GB" | 20260508 12:39:56.586 | INFO | } | 20260508 12:39:56.586 | INFO | Perf report for the last 5.14601 seconds: | 20260508 12:39:56.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:39:56.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:56.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:56.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:56.586 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 12:39:56.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:39:56.586 | INFO | RT_METRICS : 39.3%, 1.0, 20.6ms, 0 | 20260508 12:39:56.586 | INFO | PRE_HQ : 65.6%, 1.6, 55.9ms, 0 | 20260508 12:39:56.586 | INFO | BAZWRITER : 66.7%, 2.5, 85.4ms, 0 | 20260508 12:39:56.586 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 12:39:56.586 | INFO | Uploaded 36GiB to GPU in 2.5604s (14.0603GiB/s) | 20260508 12:39:56.587 | INFO | Downloaded 38GiB to GPU in 2.70465s (14.0499GiB/s) | 20260508 12:40:01.722 | INFO | Analyzing chunk frames = [8176128,8176640) | 20260508 12:40:01.722 | INFO | MemInfo: { | 20260508 12:40:01.722 | INFO | "gpu" : " 17 GB", | 20260508 12:40:01.722 | INFO | "huge" : "", | 20260508 12:40:01.722 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:01.722 | INFO | "malloc" : " 14 GB", | 20260508 12:40:01.722 | INFO | "pinned" : " 29 GB" | 20260508 12:40:01.722 | INFO | } | 20260508 12:40:01.722 | INFO | Perf report for the last 5.13623 seconds: | 20260508 12:40:01.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:01.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.722 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:40:01.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.722 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:40:01.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:01.723 | INFO | Uploaded 36GiB to GPU in 2.45531s (14.6621GiB/s) | 20260508 12:40:01.723 | INFO | Downloaded 34GiB to GPU in 2.47338s (13.7463GiB/s) | 20260508 12:40:06.857 | INFO | Analyzing chunk frames = [8176640,8177152) | 20260508 12:40:06.857 | INFO | MemInfo: { | 20260508 12:40:06.857 | INFO | "gpu" : " 17 GB", | 20260508 12:40:06.857 | INFO | "huge" : "", | 20260508 12:40:06.857 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:06.857 | INFO | "malloc" : " 14 GB", | 20260508 12:40:06.858 | INFO | "pinned" : " 29 GB" | 20260508 12:40:06.858 | INFO | } | 20260508 12:40:06.858 | INFO | Perf report for the last 5.13523 seconds: | 20260508 12:40:06.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:06.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 12:40:06.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:40:06.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:06.858 | INFO | Uploaded 36GiB to GPU in 2.49153s (14.4489GiB/s) | 20260508 12:40:06.858 | INFO | Downloaded 34GiB to GPU in 2.44783s (13.8899GiB/s) | 20260508 12:40:12.001 | INFO | Analyzing chunk frames = [8177152,8177664) | 20260508 12:40:12.002 | INFO | MemInfo: { | 20260508 12:40:12.002 | INFO | "gpu" : " 17 GB", | 20260508 12:40:12.002 | INFO | "huge" : "", | 20260508 12:40:12.002 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:12.002 | INFO | "malloc" : " 14 GB", | 20260508 12:40:12.002 | INFO | "pinned" : " 29 GB" | 20260508 12:40:12.002 | INFO | } | 20260508 12:40:12.002 | INFO | Perf report for the last 5.14412 seconds: | 20260508 12:40:12.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:12.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 12:40:12.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:40:12.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:12.002 | INFO | Uploaded 36GiB to GPU in 2.45555s (14.6607GiB/s) | 20260508 12:40:12.002 | INFO | Downloaded 34GiB to GPU in 2.45705s (13.8377GiB/s) | 20260508 12:40:17.142 | INFO | Analyzing chunk frames = [8177664,8178176) | 20260508 12:40:17.142 | INFO | MemInfo: { | 20260508 12:40:17.142 | INFO | "gpu" : " 17 GB", | 20260508 12:40:17.142 | INFO | "huge" : "", | 20260508 12:40:17.142 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:17.142 | INFO | "malloc" : " 14 GB", | 20260508 12:40:17.143 | INFO | "pinned" : " 29 GB" | 20260508 12:40:17.143 | INFO | } | 20260508 12:40:17.143 | INFO | Perf report for the last 5.14087 seconds: | 20260508 12:40:17.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:17.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 12:40:17.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:40:17.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:17.143 | INFO | Uploaded 36GiB to GPU in 2.42465s (14.8475GiB/s) | 20260508 12:40:17.143 | INFO | Downloaded 34GiB to GPU in 2.53604s (13.4067GiB/s) | 20260508 12:40:22.284 | INFO | Analyzing chunk frames = [8178176,8178688) | 20260508 12:40:22.284 | INFO | MemInfo: { | 20260508 12:40:22.284 | INFO | "gpu" : " 17 GB", | 20260508 12:40:22.284 | INFO | "huge" : "", | 20260508 12:40:22.284 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:22.284 | INFO | "malloc" : " 14 GB", | 20260508 12:40:22.284 | INFO | "pinned" : " 29 GB" | 20260508 12:40:22.284 | INFO | } | 20260508 12:40:22.285 | INFO | Perf report for the last 5.14192 seconds: | 20260508 12:40:22.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:22.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:40:22.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:40:22.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:22.285 | INFO | Uploaded 36GiB to GPU in 2.38381s (15.1018GiB/s) | 20260508 12:40:22.285 | INFO | Downloaded 34GiB to GPU in 2.44939s (13.881GiB/s) | 20260508 12:40:27.425 | INFO | Analyzing chunk frames = [8178688,8179200) | 20260508 12:40:27.425 | INFO | MemInfo: { | 20260508 12:40:27.425 | INFO | "gpu" : " 17 GB", | 20260508 12:40:27.425 | INFO | "huge" : "", | 20260508 12:40:27.425 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:27.425 | INFO | "malloc" : " 14 GB", | 20260508 12:40:27.425 | INFO | "pinned" : " 29 GB" | 20260508 12:40:27.425 | INFO | } | 20260508 12:40:27.425 | INFO | Perf report for the last 5.14072 seconds: | 20260508 12:40:27.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:27.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.425 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 12:40:27.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.426 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:40:27.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:27.426 | INFO | Uploaded 36GiB to GPU in 2.2982s (15.6645GiB/s) | 20260508 12:40:27.427 | INFO | Downloaded 34GiB to GPU in 2.43433s (13.9669GiB/s) | 20260508 12:40:32.555 | INFO | Analyzing chunk frames = [8179200,8179712) | 20260508 12:40:32.555 | INFO | MemInfo: { | 20260508 12:40:32.555 | INFO | "gpu" : " 17 GB", | 20260508 12:40:32.555 | INFO | "huge" : "", | 20260508 12:40:32.555 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:32.555 | INFO | "malloc" : " 14 GB", | 20260508 12:40:32.555 | INFO | "pinned" : " 29 GB" | 20260508 12:40:32.555 | INFO | } | 20260508 12:40:32.555 | INFO | Perf report for the last 5.12962 seconds: | 20260508 12:40:32.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:32.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | ANALYSIS : 64.6%, 3.0, 100.6ms, 0 | 20260508 12:40:32.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 12:40:32.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:32.555 | INFO | Uploaded 36GiB to GPU in 2.29292s (15.7005GiB/s) | 20260508 12:40:32.555 | INFO | Downloaded 34GiB to GPU in 2.45675s (13.8394GiB/s) | 20260508 12:40:32.655 | INFO | Starting a round of Baz IO | 20260508 12:40:36.069 | INFO | Populating RTMetrics report | 20260508 12:40:36.069 | INFO | RTMetrics report populated | 20260508 12:40:36.069 | INFO | Opening tmp RTMetrics json file | 20260508 12:40:36.075 | INFO | Serializing RTMetrics to json | 20260508 12:40:36.076 | INFO | Serialization done, swapping tmp file now... | 20260508 12:40:36.076 | INFO | RTMetrics done | 20260508 12:40:36.132 | INFO | Finished a round of Baz IO: Wrote 1.404 GiB in 3.476 seconds | 20260508 12:40:36.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:40:36.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158791% / 0% / 6.35783e-05% | 20260508 12:40:36.132 | INFO | Baz IO overall: | 20260508 12:40:36.132 | INFO | BytesWritten : 3502680447085 | 20260508 12:40:36.132 | INFO | NumEvents : 1680531060830 | 20260508 12:40:37.693 | INFO | Analyzing chunk frames = [8179712,8180224) | 20260508 12:40:37.693 | INFO | MemInfo: { | 20260508 12:40:37.693 | INFO | "gpu" : " 17 GB", | 20260508 12:40:37.693 | INFO | "huge" : "", | 20260508 12:40:37.693 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:37.693 | INFO | "malloc" : " 14 GB", | 20260508 12:40:37.693 | INFO | "pinned" : " 29 GB" | 20260508 12:40:37.693 | INFO | } | 20260508 12:40:37.693 | INFO | Perf report for the last 5.13838 seconds: | 20260508 12:40:37.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:37.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:37.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:37.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:37.693 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 12:40:37.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:37.693 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:40:37.693 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:40:37.693 | INFO | BAZWRITER : 67.0%, 2.4, 83.5ms, 0 | 20260508 12:40:37.693 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:40:37.694 | INFO | Uploaded 36GiB to GPU in 2.58722s (13.9146GiB/s) | 20260508 12:40:37.694 | INFO | Downloaded 38GiB to GPU in 2.71211s (14.0112GiB/s) | 20260508 12:40:42.840 | INFO | Analyzing chunk frames = [8180224,8180736) | 20260508 12:40:42.840 | INFO | MemInfo: { | 20260508 12:40:42.840 | INFO | "gpu" : " 17 GB", | 20260508 12:40:42.840 | INFO | "huge" : "", | 20260508 12:40:42.840 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:42.840 | INFO | "malloc" : " 14 GB", | 20260508 12:40:42.840 | INFO | "pinned" : " 29 GB" | 20260508 12:40:42.840 | INFO | } | 20260508 12:40:42.840 | INFO | Perf report for the last 5.14655 seconds: | 20260508 12:40:42.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:42.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:40:42.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:40:42.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:42.840 | INFO | Uploaded 36GiB to GPU in 2.40832s (14.9482GiB/s) | 20260508 12:40:42.840 | INFO | Downloaded 34GiB to GPU in 2.48253s (13.6957GiB/s) | 20260508 12:40:47.982 | INFO | Analyzing chunk frames = [8180736,8181248) | 20260508 12:40:47.982 | INFO | MemInfo: { | 20260508 12:40:47.982 | INFO | "gpu" : " 17 GB", | 20260508 12:40:47.982 | INFO | "huge" : "", | 20260508 12:40:47.982 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:47.982 | INFO | "malloc" : " 14 GB", | 20260508 12:40:47.982 | INFO | "pinned" : " 29 GB" | 20260508 12:40:47.982 | INFO | } | 20260508 12:40:47.982 | INFO | Perf report for the last 5.14248 seconds: | 20260508 12:40:47.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:47.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.982 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 12:40:47.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.982 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:40:47.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:47.983 | INFO | Uploaded 36GiB to GPU in 2.4112s (14.9303GiB/s) | 20260508 12:40:47.983 | INFO | Downloaded 34GiB to GPU in 2.47606s (13.7315GiB/s) | 20260508 12:40:53.115 | INFO | Analyzing chunk frames = [8181248,8181760) | 20260508 12:40:53.115 | INFO | MemInfo: { | 20260508 12:40:53.115 | INFO | "gpu" : " 17 GB", | 20260508 12:40:53.115 | INFO | "huge" : "", | 20260508 12:40:53.115 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:53.115 | INFO | "malloc" : " 14 GB", | 20260508 12:40:53.115 | INFO | "pinned" : " 29 GB" | 20260508 12:40:53.115 | INFO | } | 20260508 12:40:53.115 | INFO | Perf report for the last 5.13255 seconds: | 20260508 12:40:53.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:53.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.115 | INFO | ANALYSIS : 65.9%, 3.0, 101.9ms, 0 | 20260508 12:40:53.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.115 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:40:53.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:53.116 | INFO | Uploaded 36GiB to GPU in 2.48636s (14.479GiB/s) | 20260508 12:40:53.116 | INFO | Downloaded 34GiB to GPU in 2.47044s (13.7627GiB/s) | 20260508 12:40:58.258 | INFO | Analyzing chunk frames = [8181760,8182272) | 20260508 12:40:58.258 | INFO | MemInfo: { | 20260508 12:40:58.258 | INFO | "gpu" : " 17 GB", | 20260508 12:40:58.258 | INFO | "huge" : "", | 20260508 12:40:58.258 | INFO | "huge_shared" : " 29 GB", | 20260508 12:40:58.258 | INFO | "malloc" : " 14 GB", | 20260508 12:40:58.258 | INFO | "pinned" : " 29 GB" | 20260508 12:40:58.258 | INFO | } | 20260508 12:40:58.258 | INFO | Perf report for the last 5.14328 seconds: | 20260508 12:40:58.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:40:58.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 12:40:58.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:40:58.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:40:58.258 | INFO | Uploaded 36GiB to GPU in 2.34737s (15.3363GiB/s) | 20260508 12:40:58.259 | INFO | Downloaded 34GiB to GPU in 2.51442s (13.522GiB/s) | 20260508 12:41:03.394 | INFO | Analyzing chunk frames = [8182272,8182784) | 20260508 12:41:03.394 | INFO | MemInfo: { | 20260508 12:41:03.394 | INFO | "gpu" : " 17 GB", | 20260508 12:41:03.394 | INFO | "huge" : "", | 20260508 12:41:03.395 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:03.395 | INFO | "malloc" : " 14 GB", | 20260508 12:41:03.395 | INFO | "pinned" : " 29 GB" | 20260508 12:41:03.395 | INFO | } | 20260508 12:41:03.395 | INFO | Perf report for the last 5.13643 seconds: | 20260508 12:41:03.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:03.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 12:41:03.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 12:41:03.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:03.395 | INFO | Uploaded 36GiB to GPU in 2.40572s (14.9644GiB/s) | 20260508 12:41:03.395 | INFO | Downloaded 34GiB to GPU in 2.44397s (13.9118GiB/s) | 20260508 12:41:08.537 | INFO | Analyzing chunk frames = [8182784,8183296) | 20260508 12:41:08.537 | INFO | MemInfo: { | 20260508 12:41:08.537 | INFO | "gpu" : " 17 GB", | 20260508 12:41:08.537 | INFO | "huge" : "", | 20260508 12:41:08.537 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:08.537 | INFO | "malloc" : " 14 GB", | 20260508 12:41:08.537 | INFO | "pinned" : " 29 GB" | 20260508 12:41:08.537 | INFO | } | 20260508 12:41:08.537 | INFO | Perf report for the last 5.14257 seconds: | 20260508 12:41:08.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:08.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.537 | INFO | ANALYSIS : 64.6%, 3.0, 100.5ms, 0 | 20260508 12:41:08.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.537 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260508 12:41:08.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:08.538 | INFO | Uploaded 36GiB to GPU in 2.29028s (15.7186GiB/s) | 20260508 12:41:08.538 | INFO | Downloaded 34GiB to GPU in 2.43407s (13.9684GiB/s) | 20260508 12:41:13.678 | INFO | Analyzing chunk frames = [8183296,8183808) | 20260508 12:41:13.678 | INFO | MemInfo: { | 20260508 12:41:13.678 | INFO | "gpu" : " 17 GB", | 20260508 12:41:13.678 | INFO | "huge" : "", | 20260508 12:41:13.678 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:13.678 | INFO | "malloc" : " 14 GB", | 20260508 12:41:13.678 | INFO | "pinned" : " 29 GB" | 20260508 12:41:13.678 | INFO | } | 20260508 12:41:13.678 | INFO | Perf report for the last 5.14057 seconds: | 20260508 12:41:13.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:13.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 12:41:13.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:41:13.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:13.678 | INFO | Uploaded 36GiB to GPU in 2.40882s (14.9451GiB/s) | 20260508 12:41:13.678 | INFO | Downloaded 34GiB to GPU in 2.45166s (13.8682GiB/s) | 20260508 12:41:13.678 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:41:13.678 | INFO | Total mem usage: 16.7 GiB | 20260508 12:41:13.678 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:41:13.678 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:41:13.678 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:41:13.678 | INFO | High Water Breakdown: | 20260508 12:41:13.678 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:41:13.678 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:41:13.678 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:41:13.678 | INFO | ------------------------------------------ | 20260508 12:41:13.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:41:13.678 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:41:13.678 | INFO | Total mem usage: 26.5 GiB | 20260508 12:41:13.678 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:41:13.678 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.14%) | 20260508 12:41:13.678 | INFO | Unused Cache: 1.88 GiB (Waste: 7.08%) | 20260508 12:41:13.678 | INFO | High Water Breakdown: | 20260508 12:41:13.678 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:41:13.678 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:41:13.678 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:41:13.678 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:41:13.678 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:41:13.678 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:41:13.678 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:41:13.679 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:41:13.679 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:41:13.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:41:13.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:41:13.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:41:13.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:41:13.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:41:13.679 | INFO | ------------------------------------------ | 20260508 12:41:13.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:41:13.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:41:13.679 | INFO | Total mem usage: 27.8 GiB | 20260508 12:41:13.679 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:41:13.679 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:41:13.679 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:41:13.679 | INFO | High Water Breakdown: | 20260508 12:41:13.679 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:41:13.679 | INFO | ------------------------------------------ | 20260508 12:41:13.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:41:13.679 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:41:13.679 | INFO | Total mem usage: 15.7 GiB | 20260508 12:41:13.679 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:41:13.679 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:41:13.679 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:41:13.679 | INFO | High Water Breakdown: | 20260508 12:41:13.679 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:41:13.679 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:41:13.679 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:41:13.679 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:41:13.679 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:41:13.679 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:41:13.679 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:41:13.679 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:41:13.679 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:41:13.679 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:41:13.679 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:41:13.679 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:41:13.679 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:41:13.679 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:41:13.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:41:13.679 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:41:13.679 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:41:13.679 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:41:13.679 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:41:13.679 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:41:13.679 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:41:13.679 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:41:13.679 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:41:13.679 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:41:13.679 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:41:13.679 | INFO | ------------------------------------------ | 20260508 12:41:13.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:41:13.679 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:41:13.679 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:41:13.679 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:41:13.679 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.41e+01 / 6.61e-01 | 20260508 12:41:13.679 | INFO | PerfReport: 0.30%, AggregateStats -- 9.75e-02 / 1.04e-01, 1.13e-02 / 4.10e-03 | 20260508 12:41:13.680 | INFO | PerfReport: 12.20%, Baseline -- 4.00e+00 / 4.42e+00, 6.00e-01 / 1.63e-01 | 20260508 12:41:13.680 | INFO | PerfReport: 3.41%, Histogram -- 1.12e+00 / 1.43e+00, 2.11e-01 / 1.13e-01 | 20260508 12:41:13.680 | INFO | PerfReport: 15.94%, DME -- 4.18e+01 / 4.52e+01, 3.67e+00 / 1.16e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.88e+01 / 2.04e+01, 1.93e+00 / 1.32e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.30e+00 / 2.45e+00, 4.02e-01 / 1.86e-01 | 20260508 12:41:13.680 | INFO | PerfReport: 2.93%, Metrics -- 9.61e-01 / 1.14e+00, 2.38e-01 / 1.01e-01 | 20260508 12:41:13.680 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.92e-01 / 3.23e-01, 4.60e-02 / 1.61e-02 | 20260508 12:41:13.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:41:13.680 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:41:13.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:41:13.680 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 24.07%, Upload -- 2.52e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 24.84%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 31.34%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 2.06%, LockUpload -- 2.15e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 2.62%, LockDownload -- 2.74e+00 / 0.00e+00, 4.18e+00 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 15.06%, LockCompute -- 1.58e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:41:13.680 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 12:41:13.775 | INFO | Starting a round of Baz IO | 20260508 12:41:17.188 | INFO | Populating RTMetrics report | 20260508 12:41:17.188 | INFO | RTMetrics report populated | 20260508 12:41:17.188 | INFO | Opening tmp RTMetrics json file | 20260508 12:41:17.196 | INFO | Serializing RTMetrics to json | 20260508 12:41:17.198 | INFO | Serialization done, swapping tmp file now... | 20260508 12:41:17.198 | INFO | RTMetrics done | 20260508 12:41:17.250 | INFO | Finished a round of Baz IO: Wrote 1.403 GiB in 3.474 seconds | 20260508 12:41:17.250 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:41:17.250 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157917% / 3.97364e-06% / 5.16574e-05% | 20260508 12:41:17.250 | INFO | Baz IO overall: | 20260508 12:41:17.250 | INFO | BytesWritten : 3504187368482 | 20260508 12:41:17.250 | INFO | NumEvents : 1681237562386 | 20260508 12:41:18.812 | INFO | Analyzing chunk frames = [8183808,8184320) | 20260508 12:41:18.812 | INFO | MemInfo: { | 20260508 12:41:18.812 | INFO | "gpu" : " 17 GB", | 20260508 12:41:18.812 | INFO | "huge" : "", | 20260508 12:41:18.812 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:18.812 | INFO | "malloc" : " 14 GB", | 20260508 12:41:18.812 | INFO | "pinned" : " 29 GB" | 20260508 12:41:18.812 | INFO | } | 20260508 12:41:18.812 | INFO | Perf report for the last 5.13455 seconds: | 20260508 12:41:18.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:18.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:18.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:18.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:18.813 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:41:18.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:18.813 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:41:18.813 | INFO | PRE_HQ : 65.4%, 1.6, 53.9ms, 0 | 20260508 12:41:18.813 | INFO | BAZWRITER : 66.9%, 2.4, 84.0ms, 0 | 20260508 12:41:18.813 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:41:18.813 | INFO | Uploaded 36GiB to GPU in 2.62297s (13.7249GiB/s) | 20260508 12:41:18.813 | INFO | Downloaded 38GiB to GPU in 2.71053s (14.0194GiB/s) | 20260508 12:41:23.951 | INFO | Analyzing chunk frames = [8184320,8184832) | 20260508 12:41:23.951 | INFO | MemInfo: { | 20260508 12:41:23.951 | INFO | "gpu" : " 17 GB", | 20260508 12:41:23.951 | INFO | "huge" : "", | 20260508 12:41:23.951 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:23.951 | INFO | "malloc" : " 14 GB", | 20260508 12:41:23.951 | INFO | "pinned" : " 29 GB" | 20260508 12:41:23.951 | INFO | } | 20260508 12:41:23.951 | INFO | Perf report for the last 5.13842 seconds: | 20260508 12:41:23.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:23.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:41:23.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:41:23.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:23.951 | INFO | Uploaded 36GiB to GPU in 2.45027s (14.6923GiB/s) | 20260508 12:41:23.951 | INFO | Downloaded 34GiB to GPU in 2.47759s (13.723GiB/s) | 20260508 12:41:29.093 | INFO | Analyzing chunk frames = [8184832,8185344) | 20260508 12:41:29.093 | INFO | MemInfo: { | 20260508 12:41:29.093 | INFO | "gpu" : " 17 GB", | 20260508 12:41:29.093 | INFO | "huge" : "", | 20260508 12:41:29.093 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:29.093 | INFO | "malloc" : " 14 GB", | 20260508 12:41:29.093 | INFO | "pinned" : " 29 GB" | 20260508 12:41:29.093 | INFO | } | 20260508 12:41:29.093 | INFO | Perf report for the last 5.14255 seconds: | 20260508 12:41:29.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:29.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.093 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 12:41:29.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.094 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:41:29.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:29.094 | INFO | Uploaded 36GiB to GPU in 2.4897s (14.4596GiB/s) | 20260508 12:41:29.094 | INFO | Downloaded 34GiB to GPU in 2.4617s (13.8116GiB/s) | 20260508 12:41:34.237 | INFO | Analyzing chunk frames = [8185344,8185856) | 20260508 12:41:34.237 | INFO | MemInfo: { | 20260508 12:41:34.237 | INFO | "gpu" : " 17 GB", | 20260508 12:41:34.237 | INFO | "huge" : "", | 20260508 12:41:34.237 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:34.237 | INFO | "malloc" : " 14 GB", | 20260508 12:41:34.237 | INFO | "pinned" : " 29 GB" | 20260508 12:41:34.237 | INFO | } | 20260508 12:41:34.237 | INFO | Perf report for the last 5.14326 seconds: | 20260508 12:41:34.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:34.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.237 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:41:34.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.237 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:41:34.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:34.238 | INFO | Uploaded 36GiB to GPU in 2.49855s (14.4084GiB/s) | 20260508 12:41:34.238 | INFO | Downloaded 34GiB to GPU in 2.44794s (13.8892GiB/s) | 20260508 12:41:39.374 | INFO | Analyzing chunk frames = [8185856,8186368) | 20260508 12:41:39.374 | INFO | MemInfo: { | 20260508 12:41:39.374 | INFO | "gpu" : " 17 GB", | 20260508 12:41:39.374 | INFO | "huge" : "", | 20260508 12:41:39.374 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:39.374 | INFO | "malloc" : " 14 GB", | 20260508 12:41:39.374 | INFO | "pinned" : " 29 GB" | 20260508 12:41:39.374 | INFO | } | 20260508 12:41:39.374 | INFO | Perf report for the last 5.13753 seconds: | 20260508 12:41:39.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:39.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 12:41:39.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 12:41:39.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:39.374 | INFO | Uploaded 36GiB to GPU in 2.37657s (15.1479GiB/s) | 20260508 12:41:39.375 | INFO | Downloaded 34GiB to GPU in 2.5293s (13.4424GiB/s) | 20260508 12:41:44.507 | INFO | Analyzing chunk frames = [8186368,8186880) | 20260508 12:41:44.507 | INFO | MemInfo: { | 20260508 12:41:44.507 | INFO | "gpu" : " 17 GB", | 20260508 12:41:44.507 | INFO | "huge" : "", | 20260508 12:41:44.507 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:44.507 | INFO | "malloc" : " 14 GB", | 20260508 12:41:44.507 | INFO | "pinned" : " 29 GB" | 20260508 12:41:44.507 | INFO | } | 20260508 12:41:44.507 | INFO | Perf report for the last 5.13279 seconds: | 20260508 12:41:44.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:44.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | ANALYSIS : 65.2%, 3.0, 101.3ms, 0 | 20260508 12:41:44.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:41:44.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:44.507 | INFO | Uploaded 36GiB to GPU in 2.3651s (15.2213GiB/s) | 20260508 12:41:44.507 | INFO | Downloaded 34GiB to GPU in 2.45592s (13.8441GiB/s) | 20260508 12:41:49.647 | INFO | Analyzing chunk frames = [8186880,8187392) | 20260508 12:41:49.647 | INFO | MemInfo: { | 20260508 12:41:49.647 | INFO | "gpu" : " 17 GB", | 20260508 12:41:49.647 | INFO | "huge" : "", | 20260508 12:41:49.647 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:49.647 | INFO | "malloc" : " 14 GB", | 20260508 12:41:49.647 | INFO | "pinned" : " 29 GB" | 20260508 12:41:49.647 | INFO | } | 20260508 12:41:49.647 | INFO | Perf report for the last 5.14017 seconds: | 20260508 12:41:49.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:49.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.647 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 12:41:49.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.647 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:41:49.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:49.648 | INFO | Uploaded 36GiB to GPU in 2.44704s (14.7116GiB/s) | 20260508 12:41:49.648 | INFO | Downloaded 34GiB to GPU in 2.47149s (13.7569GiB/s) | 20260508 12:41:54.788 | INFO | Analyzing chunk frames = [8187392,8187904) | 20260508 12:41:54.788 | INFO | MemInfo: { | 20260508 12:41:54.788 | INFO | "gpu" : " 17 GB", | 20260508 12:41:54.788 | INFO | "huge" : "", | 20260508 12:41:54.788 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:54.788 | INFO | "malloc" : " 14 GB", | 20260508 12:41:54.788 | INFO | "pinned" : " 29 GB" | 20260508 12:41:54.789 | INFO | } | 20260508 12:41:54.789 | INFO | Perf report for the last 5.14146 seconds: | 20260508 12:41:54.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:54.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 12:41:54.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:41:54.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:54.789 | INFO | Uploaded 36GiB to GPU in 2.42495s (14.8457GiB/s) | 20260508 12:41:54.789 | INFO | Downloaded 34GiB to GPU in 2.44803s (13.8887GiB/s) | 20260508 12:41:54.886 | INFO | Starting a round of Baz IO | 20260508 12:41:58.295 | INFO | Populating RTMetrics report | 20260508 12:41:58.295 | INFO | RTMetrics report populated | 20260508 12:41:58.296 | INFO | Opening tmp RTMetrics json file | 20260508 12:41:58.296 | INFO | Serializing RTMetrics to json | 20260508 12:41:58.298 | INFO | Serialization done, swapping tmp file now... | 20260508 12:41:58.298 | INFO | RTMetrics done | 20260508 12:41:58.369 | INFO | Finished a round of Baz IO: Wrote 1.404 GiB in 3.483 seconds | 20260508 12:41:58.369 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:41:58.369 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158175% / 0% / 5.5631e-05% | 20260508 12:41:58.369 | INFO | Baz IO overall: | 20260508 12:41:58.369 | INFO | BytesWritten : 3505694642824 | 20260508 12:41:58.369 | INFO | NumEvents : 1681944386073 | 20260508 12:41:59.929 | INFO | Analyzing chunk frames = [8187904,8188416) | 20260508 12:41:59.929 | INFO | MemInfo: { | 20260508 12:41:59.929 | INFO | "gpu" : " 17 GB", | 20260508 12:41:59.929 | INFO | "huge" : "", | 20260508 12:41:59.929 | INFO | "huge_shared" : " 29 GB", | 20260508 12:41:59.929 | INFO | "malloc" : " 14 GB", | 20260508 12:41:59.929 | INFO | "pinned" : " 29 GB" | 20260508 12:41:59.929 | INFO | } | 20260508 12:41:59.929 | INFO | Perf report for the last 5.14057 seconds: | 20260508 12:41:59.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:41:59.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:59.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:59.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:59.929 | INFO | ANALYSIS : 67.9%, 3.0, 105.8ms, 0 | 20260508 12:41:59.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:41:59.929 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 12:41:59.929 | INFO | PRE_HQ : 65.6%, 1.6, 53.6ms, 0 | 20260508 12:41:59.929 | INFO | BAZWRITER : 67.0%, 2.4, 83.0ms, 0 | 20260508 12:41:59.929 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:41:59.930 | INFO | Uploaded 36GiB to GPU in 2.51661s (14.305GiB/s) | 20260508 12:41:59.930 | INFO | Downloaded 38GiB to GPU in 2.70351s (14.0558GiB/s) | 20260508 12:42:05.071 | INFO | Analyzing chunk frames = [8188416,8188928) | 20260508 12:42:05.071 | INFO | MemInfo: { | 20260508 12:42:05.071 | INFO | "gpu" : " 17 GB", | 20260508 12:42:05.071 | INFO | "huge" : "", | 20260508 12:42:05.071 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:05.071 | INFO | "malloc" : " 14 GB", | 20260508 12:42:05.071 | INFO | "pinned" : " 29 GB" | 20260508 12:42:05.071 | INFO | } | 20260508 12:42:05.071 | INFO | Perf report for the last 5.14164 seconds: | 20260508 12:42:05.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:05.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:42:05.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:42:05.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:05.071 | INFO | Uploaded 36GiB to GPU in 2.4253s (14.8435GiB/s) | 20260508 12:42:05.071 | INFO | Downloaded 34GiB to GPU in 2.4801s (13.7091GiB/s) | 20260508 12:42:10.202 | INFO | Analyzing chunk frames = [8188928,8189440) | 20260508 12:42:10.202 | INFO | MemInfo: { | 20260508 12:42:10.202 | INFO | "gpu" : " 17 GB", | 20260508 12:42:10.202 | INFO | "huge" : "", | 20260508 12:42:10.202 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:10.202 | INFO | "malloc" : " 14 GB", | 20260508 12:42:10.202 | INFO | "pinned" : " 29 GB" | 20260508 12:42:10.202 | INFO | } | 20260508 12:42:10.202 | INFO | Perf report for the last 5.13111 seconds: | 20260508 12:42:10.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:10.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:42:10.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:42:10.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:10.202 | INFO | Uploaded 36GiB to GPU in 2.46789s (14.5873GiB/s) | 20260508 12:42:10.203 | INFO | Downloaded 34GiB to GPU in 2.472s (13.754GiB/s) | 20260508 12:42:15.352 | INFO | Analyzing chunk frames = [8189440,8189952) | 20260508 12:42:15.352 | INFO | MemInfo: { | 20260508 12:42:15.352 | INFO | "gpu" : " 17 GB", | 20260508 12:42:15.352 | INFO | "huge" : "", | 20260508 12:42:15.352 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:15.352 | INFO | "malloc" : " 14 GB", | 20260508 12:42:15.352 | INFO | "pinned" : " 29 GB" | 20260508 12:42:15.352 | INFO | } | 20260508 12:42:15.352 | INFO | Perf report for the last 5.14999 seconds: | 20260508 12:42:15.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:15.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.352 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 12:42:15.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.352 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260508 12:42:15.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:15.353 | INFO | Uploaded 36GiB to GPU in 2.46099s (14.6283GiB/s) | 20260508 12:42:15.353 | INFO | Downloaded 34GiB to GPU in 2.46921s (13.7696GiB/s) | 20260508 12:42:20.483 | INFO | Analyzing chunk frames = [8189952,8190464) | 20260508 12:42:20.483 | INFO | MemInfo: { | 20260508 12:42:20.483 | INFO | "gpu" : " 17 GB", | 20260508 12:42:20.483 | INFO | "huge" : "", | 20260508 12:42:20.483 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:20.483 | INFO | "malloc" : " 14 GB", | 20260508 12:42:20.483 | INFO | "pinned" : " 29 GB" | 20260508 12:42:20.483 | INFO | } | 20260508 12:42:20.483 | INFO | Perf report for the last 5.13122 seconds: | 20260508 12:42:20.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:20.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.483 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 12:42:20.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.483 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 12:42:20.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:20.484 | INFO | Uploaded 36GiB to GPU in 2.41837s (14.8861GiB/s) | 20260508 12:42:20.484 | INFO | Downloaded 34GiB to GPU in 2.52215s (13.4806GiB/s) | 20260508 12:42:25.629 | INFO | Analyzing chunk frames = [8190464,8190976) | 20260508 12:42:25.629 | INFO | MemInfo: { | 20260508 12:42:25.629 | INFO | "gpu" : " 17 GB", | 20260508 12:42:25.629 | INFO | "huge" : "", | 20260508 12:42:25.629 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:25.629 | INFO | "malloc" : " 14 GB", | 20260508 12:42:25.629 | INFO | "pinned" : " 29 GB" | 20260508 12:42:25.629 | INFO | } | 20260508 12:42:25.629 | INFO | Perf report for the last 5.14562 seconds: | 20260508 12:42:25.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:25.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.629 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 12:42:25.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.629 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:42:25.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:25.630 | INFO | Uploaded 36GiB to GPU in 2.37093s (15.1839GiB/s) | 20260508 12:42:25.630 | INFO | Downloaded 34GiB to GPU in 2.45831s (13.8306GiB/s) | 20260508 12:42:30.762 | INFO | Analyzing chunk frames = [8190976,8191488) | 20260508 12:42:30.762 | INFO | MemInfo: { | 20260508 12:42:30.762 | INFO | "gpu" : " 17 GB", | 20260508 12:42:30.762 | INFO | "huge" : "", | 20260508 12:42:30.762 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:30.762 | INFO | "malloc" : " 14 GB", | 20260508 12:42:30.762 | INFO | "pinned" : " 29 GB" | 20260508 12:42:30.762 | INFO | } | 20260508 12:42:30.762 | INFO | Perf report for the last 5.13308 seconds: | 20260508 12:42:30.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:30.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | ANALYSIS : 64.7%, 3.0, 100.5ms, 0 | 20260508 12:42:30.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:42:30.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:30.762 | INFO | Uploaded 36GiB to GPU in 2.26023s (15.9276GiB/s) | 20260508 12:42:30.762 | INFO | Downloaded 34GiB to GPU in 2.43527s (13.9615GiB/s) | 20260508 12:42:35.904 | INFO | Analyzing chunk frames = [8191488,8192000) | 20260508 12:42:35.904 | INFO | MemInfo: { | 20260508 12:42:35.904 | INFO | "gpu" : " 17 GB", | 20260508 12:42:35.904 | INFO | "huge" : "", | 20260508 12:42:35.904 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:35.904 | INFO | "malloc" : " 14 GB", | 20260508 12:42:35.904 | INFO | "pinned" : " 29 GB" | 20260508 12:42:35.904 | INFO | } | 20260508 12:42:35.904 | INFO | Perf report for the last 5.14186 seconds: | 20260508 12:42:35.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:35.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 12:42:35.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:42:35.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:35.904 | INFO | Uploaded 36GiB to GPU in 2.43555s (14.781GiB/s) | 20260508 12:42:35.904 | INFO | Downloaded 34GiB to GPU in 2.46982s (13.7662GiB/s) | 20260508 12:42:35.904 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:42:35.904 | INFO | Total mem usage: 16.7 GiB | 20260508 12:42:35.904 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:42:35.904 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:42:35.904 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:42:35.904 | INFO | High Water Breakdown: | 20260508 12:42:35.904 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:42:35.904 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:42:35.904 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:42:35.904 | INFO | ------------------------------------------ | 20260508 12:42:35.904 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:42:35.904 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:42:35.904 | INFO | Total mem usage: 26.5 GiB | 20260508 12:42:35.905 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:42:35.905 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:42:35.905 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:42:35.905 | INFO | High Water Breakdown: | 20260508 12:42:35.905 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:42:35.905 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:42:35.905 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:42:35.905 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:42:35.905 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:42:35.905 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:42:35.905 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:42:35.905 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:42:35.905 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:42:35.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:42:35.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:42:35.905 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:42:35.905 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:42:35.905 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:42:35.905 | INFO | ------------------------------------------ | 20260508 12:42:35.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:42:35.905 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:42:35.905 | INFO | Total mem usage: 27.8 GiB | 20260508 12:42:35.905 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:42:35.905 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:42:35.905 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:42:35.905 | INFO | High Water Breakdown: | 20260508 12:42:35.905 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:42:35.905 | INFO | ------------------------------------------ | 20260508 12:42:35.905 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:42:35.905 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:42:35.905 | INFO | Total mem usage: 15.7 GiB | 20260508 12:42:35.905 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:42:35.905 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:42:35.905 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:42:35.905 | INFO | High Water Breakdown: | 20260508 12:42:35.905 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:42:35.905 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:42:35.905 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:42:35.905 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:42:35.905 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:42:35.905 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:42:35.905 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:42:35.905 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:42:35.905 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:42:35.905 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:42:35.906 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:42:35.906 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:42:35.906 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:42:35.906 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:42:35.906 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:42:35.906 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:42:35.906 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:42:35.906 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:42:35.906 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:42:35.906 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:42:35.906 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:42:35.906 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:42:35.906 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:42:35.906 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:42:35.906 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:42:35.906 | INFO | ------------------------------------------ | 20260508 12:42:35.906 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:42:35.906 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:42:35.906 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:42:35.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:42:35.906 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 7.23e-01 | 20260508 12:42:35.906 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.05e-01, 1.14e-02 / 4.81e-03 | 20260508 12:42:35.906 | INFO | PerfReport: 12.17%, Baseline -- 3.97e+00 / 4.43e+00, 5.92e-01 / 2.08e-01 | 20260508 12:42:35.906 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.45e+00, 2.09e-01 / 1.30e-01 | 20260508 12:42:35.906 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.47e+01, 3.57e+00 / 1.27e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.87e+01 / 2.09e+01, 1.86e+00 / 1.54e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.31e+00 / 2.32e+00, 4.05e-01 / 3.15e-01 | 20260508 12:42:35.906 | INFO | PerfReport: 2.94%, Metrics -- 9.61e-01 / 1.09e+00, 2.40e-01 / 1.71e-01 | 20260508 12:42:35.906 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.92e-01 / 3.24e-01, 4.69e-02 / 2.92e-02 | 20260508 12:42:35.906 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:42:35.906 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:42:35.906 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:42:35.906 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 24.24%, Upload -- 2.54e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 24.85%, Download -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 31.15%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 2.01%, LockUpload -- 2.10e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 2.71%, LockDownload -- 2.84e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 15.04%, LockCompute -- 1.58e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:42:35.906 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260508 12:42:36.001 | INFO | Starting a round of Baz IO | 20260508 12:42:39.412 | INFO | Populating RTMetrics report | 20260508 12:42:39.412 | INFO | RTMetrics report populated | 20260508 12:42:39.412 | INFO | Opening tmp RTMetrics json file | 20260508 12:42:39.418 | INFO | Serializing RTMetrics to json | 20260508 12:42:39.419 | INFO | Serialization done, swapping tmp file now... | 20260508 12:42:39.420 | INFO | RTMetrics done | 20260508 12:42:39.476 | INFO | Finished a round of Baz IO: Wrote 1.403 GiB in 3.474 seconds | 20260508 12:42:39.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:42:39.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158032% / 0% / 6.35783e-05% | 20260508 12:42:39.476 | INFO | Baz IO overall: | 20260508 12:42:39.476 | INFO | BytesWritten : 3507201142359 | 20260508 12:42:39.476 | INFO | NumEvents : 1682650627835 | 20260508 12:42:41.044 | INFO | Analyzing chunk frames = [8192000,8192512) | 20260508 12:42:41.044 | INFO | MemInfo: { | 20260508 12:42:41.044 | INFO | "gpu" : " 17 GB", | 20260508 12:42:41.044 | INFO | "huge" : "", | 20260508 12:42:41.044 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:41.044 | INFO | "malloc" : " 14 GB", | 20260508 12:42:41.044 | INFO | "pinned" : " 29 GB" | 20260508 12:42:41.044 | INFO | } | 20260508 12:42:41.044 | INFO | Perf report for the last 5.14041 seconds: | 20260508 12:42:41.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:41.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:41.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:41.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:41.044 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:42:41.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:41.044 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 12:42:41.045 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 12:42:41.045 | INFO | BAZWRITER : 66.8%, 2.4, 83.4ms, 0 | 20260508 12:42:41.045 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:42:41.045 | INFO | Uploaded 36GiB to GPU in 2.59914s (13.8507GiB/s) | 20260508 12:42:41.045 | INFO | Downloaded 38GiB to GPU in 2.72184s (13.9612GiB/s) | 20260508 12:42:46.183 | INFO | Analyzing chunk frames = [8192512,8193024) | 20260508 12:42:46.183 | INFO | MemInfo: { | 20260508 12:42:46.183 | INFO | "gpu" : " 17 GB", | 20260508 12:42:46.183 | INFO | "huge" : "", | 20260508 12:42:46.183 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:46.183 | INFO | "malloc" : " 14 GB", | 20260508 12:42:46.183 | INFO | "pinned" : " 29 GB" | 20260508 12:42:46.183 | INFO | } | 20260508 12:42:46.183 | INFO | Perf report for the last 5.13866 seconds: | 20260508 12:42:46.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:46.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.183 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:42:46.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.183 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:42:46.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:46.184 | INFO | Uploaded 36GiB to GPU in 2.41997s (14.8762GiB/s) | 20260508 12:42:46.184 | INFO | Downloaded 34GiB to GPU in 2.48031s (13.708GiB/s) | 20260508 12:42:51.321 | INFO | Analyzing chunk frames = [8193024,8193536) | 20260508 12:42:51.321 | INFO | MemInfo: { | 20260508 12:42:51.321 | INFO | "gpu" : " 17 GB", | 20260508 12:42:51.321 | INFO | "huge" : "", | 20260508 12:42:51.321 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:51.321 | INFO | "malloc" : " 14 GB", | 20260508 12:42:51.321 | INFO | "pinned" : " 29 GB" | 20260508 12:42:51.321 | INFO | } | 20260508 12:42:51.321 | INFO | Perf report for the last 5.13828 seconds: | 20260508 12:42:51.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:51.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.321 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 12:42:51.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.321 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:42:51.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:51.322 | INFO | Uploaded 36GiB to GPU in 2.45066s (14.6899GiB/s) | 20260508 12:42:51.322 | INFO | Downloaded 34GiB to GPU in 2.46747s (13.7793GiB/s) | 20260508 12:42:56.456 | INFO | Analyzing chunk frames = [8193536,8194048) | 20260508 12:42:56.456 | INFO | MemInfo: { | 20260508 12:42:56.456 | INFO | "gpu" : " 17 GB", | 20260508 12:42:56.456 | INFO | "huge" : "", | 20260508 12:42:56.456 | INFO | "huge_shared" : " 29 GB", | 20260508 12:42:56.456 | INFO | "malloc" : " 14 GB", | 20260508 12:42:56.456 | INFO | "pinned" : " 29 GB" | 20260508 12:42:56.456 | INFO | } | 20260508 12:42:56.456 | INFO | Perf report for the last 5.13463 seconds: | 20260508 12:42:56.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:42:56.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 12:42:56.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 12:42:56.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:42:56.456 | INFO | Uploaded 36GiB to GPU in 2.45016s (14.6929GiB/s) | 20260508 12:42:56.456 | INFO | Downloaded 34GiB to GPU in 2.47045s (13.7627GiB/s) | 20260508 12:43:01.599 | INFO | Analyzing chunk frames = [8194048,8194560) | 20260508 12:43:01.599 | INFO | MemInfo: { | 20260508 12:43:01.599 | INFO | "gpu" : " 17 GB", | 20260508 12:43:01.599 | INFO | "huge" : "", | 20260508 12:43:01.599 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:01.599 | INFO | "malloc" : " 14 GB", | 20260508 12:43:01.599 | INFO | "pinned" : " 29 GB" | 20260508 12:43:01.599 | INFO | } | 20260508 12:43:01.599 | INFO | Perf report for the last 5.14289 seconds: | 20260508 12:43:01.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:01.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:43:01.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:43:01.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:01.599 | INFO | Uploaded 36GiB to GPU in 2.25819s (15.942GiB/s) | 20260508 12:43:01.599 | INFO | Downloaded 34GiB to GPU in 2.50705s (13.5618GiB/s) | 20260508 12:43:06.738 | INFO | Analyzing chunk frames = [8194560,8195072) | 20260508 12:43:06.738 | INFO | MemInfo: { | 20260508 12:43:06.738 | INFO | "gpu" : " 17 GB", | 20260508 12:43:06.738 | INFO | "huge" : "", | 20260508 12:43:06.738 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:06.738 | INFO | "malloc" : " 14 GB", | 20260508 12:43:06.738 | INFO | "pinned" : " 29 GB" | 20260508 12:43:06.739 | INFO | } | 20260508 12:43:06.739 | INFO | Perf report for the last 5.13986 seconds: | 20260508 12:43:06.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:06.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 12:43:06.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:43:06.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:06.739 | INFO | Uploaded 36GiB to GPU in 2.37327s (15.1689GiB/s) | 20260508 12:43:06.739 | INFO | Downloaded 34GiB to GPU in 2.45461s (13.8515GiB/s) | 20260508 12:43:11.882 | INFO | Analyzing chunk frames = [8195072,8195584) | 20260508 12:43:11.882 | INFO | MemInfo: { | 20260508 12:43:11.882 | INFO | "gpu" : " 17 GB", | 20260508 12:43:11.882 | INFO | "huge" : "", | 20260508 12:43:11.882 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:11.882 | INFO | "malloc" : " 14 GB", | 20260508 12:43:11.882 | INFO | "pinned" : " 29 GB" | 20260508 12:43:11.882 | INFO | } | 20260508 12:43:11.882 | INFO | Perf report for the last 5.1434 seconds: | 20260508 12:43:11.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:11.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.882 | INFO | ANALYSIS : 65.0%, 3.0, 101.6ms, 0 | 20260508 12:43:11.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.882 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:43:11.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:11.883 | INFO | Uploaded 36GiB to GPU in 2.38317s (15.106GiB/s) | 20260508 12:43:11.883 | INFO | Downloaded 34GiB to GPU in 2.45139s (13.8697GiB/s) | 20260508 12:43:17.017 | INFO | Analyzing chunk frames = [8195584,8196096) | 20260508 12:43:17.017 | INFO | MemInfo: { | 20260508 12:43:17.017 | INFO | "gpu" : " 17 GB", | 20260508 12:43:17.017 | INFO | "huge" : "", | 20260508 12:43:17.017 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:17.017 | INFO | "malloc" : " 14 GB", | 20260508 12:43:17.017 | INFO | "pinned" : " 29 GB" | 20260508 12:43:17.017 | INFO | } | 20260508 12:43:17.017 | INFO | Perf report for the last 5.13469 seconds: | 20260508 12:43:17.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:17.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | ANALYSIS : 64.5%, 3.0, 100.6ms, 0 | 20260508 12:43:17.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:43:17.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:17.017 | INFO | Uploaded 36GiB to GPU in 2.28827s (15.7324GiB/s) | 20260508 12:43:17.017 | INFO | Downloaded 34GiB to GPU in 2.44205s (13.9227GiB/s) | 20260508 12:43:17.115 | INFO | Starting a round of Baz IO | 20260508 12:43:20.512 | INFO | Populating RTMetrics report | 20260508 12:43:20.512 | INFO | RTMetrics report populated | 20260508 12:43:20.512 | INFO | Opening tmp RTMetrics json file | 20260508 12:43:20.521 | INFO | Serializing RTMetrics to json | 20260508 12:43:20.523 | INFO | Serialization done, swapping tmp file now... | 20260508 12:43:20.523 | INFO | RTMetrics done | 20260508 12:43:20.583 | INFO | Finished a round of Baz IO: Wrote 1.403 GiB in 3.468 seconds | 20260508 12:43:20.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:43:20.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158254% / 0% / 8.34465e-05% | 20260508 12:43:20.583 | INFO | Baz IO overall: | 20260508 12:43:20.583 | INFO | BytesWritten : 3508707756142 | 20260508 12:43:20.583 | INFO | NumEvents : 1683357010169 | 20260508 12:43:22.157 | INFO | Analyzing chunk frames = [8196096,8196608) | 20260508 12:43:22.157 | INFO | MemInfo: { | 20260508 12:43:22.157 | INFO | "gpu" : " 17 GB", | 20260508 12:43:22.157 | INFO | "huge" : "", | 20260508 12:43:22.157 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:22.157 | INFO | "malloc" : " 14 GB", | 20260508 12:43:22.157 | INFO | "pinned" : " 29 GB" | 20260508 12:43:22.157 | INFO | } | 20260508 12:43:22.157 | INFO | Perf report for the last 5.1405 seconds: | 20260508 12:43:22.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:22.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:22.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:22.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:22.157 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:43:22.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:22.157 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:43:22.157 | INFO | PRE_HQ : 65.2%, 1.6, 53.6ms, 0 | 20260508 12:43:22.157 | INFO | BAZWRITER : 66.8%, 2.4, 83.5ms, 0 | 20260508 12:43:22.157 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:43:22.158 | INFO | Uploaded 36GiB to GPU in 2.60952s (13.7957GiB/s) | 20260508 12:43:22.158 | INFO | Downloaded 38GiB to GPU in 2.73394s (13.8994GiB/s) | 20260508 12:43:27.297 | INFO | Analyzing chunk frames = [8196608,8197120) | 20260508 12:43:27.297 | INFO | MemInfo: { | 20260508 12:43:27.297 | INFO | "gpu" : " 17 GB", | 20260508 12:43:27.297 | INFO | "huge" : "", | 20260508 12:43:27.297 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:27.297 | INFO | "malloc" : " 14 GB", | 20260508 12:43:27.297 | INFO | "pinned" : " 29 GB" | 20260508 12:43:27.297 | INFO | } | 20260508 12:43:27.297 | INFO | Perf report for the last 5.13994 seconds: | 20260508 12:43:27.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:27.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.297 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 12:43:27.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.298 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 12:43:27.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:27.298 | INFO | Uploaded 36GiB to GPU in 2.47579s (14.5408GiB/s) | 20260508 12:43:27.298 | INFO | Downloaded 34GiB to GPU in 2.46436s (13.7967GiB/s) | 20260508 12:43:32.436 | INFO | Analyzing chunk frames = [8197120,8197632) | 20260508 12:43:32.436 | INFO | MemInfo: { | 20260508 12:43:32.436 | INFO | "gpu" : " 17 GB", | 20260508 12:43:32.436 | INFO | "huge" : "", | 20260508 12:43:32.436 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:32.436 | INFO | "malloc" : " 14 GB", | 20260508 12:43:32.436 | INFO | "pinned" : " 29 GB" | 20260508 12:43:32.436 | INFO | } | 20260508 12:43:32.437 | INFO | Perf report for the last 5.13924 seconds: | 20260508 12:43:32.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:32.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:43:32.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 12:43:32.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:32.437 | INFO | Uploaded 36GiB to GPU in 2.50376s (14.3784GiB/s) | 20260508 12:43:32.437 | INFO | Downloaded 34GiB to GPU in 2.45499s (13.8494GiB/s) | 20260508 12:43:37.573 | INFO | Analyzing chunk frames = [8197632,8198144) | 20260508 12:43:37.573 | INFO | MemInfo: { | 20260508 12:43:37.573 | INFO | "gpu" : " 17 GB", | 20260508 12:43:37.573 | INFO | "huge" : "", | 20260508 12:43:37.573 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:37.573 | INFO | "malloc" : " 14 GB", | 20260508 12:43:37.573 | INFO | "pinned" : " 29 GB" | 20260508 12:43:37.573 | INFO | } | 20260508 12:43:37.574 | INFO | Perf report for the last 5.13689 seconds: | 20260508 12:43:37.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:37.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 12:43:37.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 12:43:37.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:37.574 | INFO | Uploaded 36GiB to GPU in 2.416s (14.9007GiB/s) | 20260508 12:43:37.574 | INFO | Downloaded 34GiB to GPU in 2.46907s (13.7704GiB/s) | 20260508 12:43:42.712 | INFO | Analyzing chunk frames = [8198144,8198656) | 20260508 12:43:42.712 | INFO | MemInfo: { | 20260508 12:43:42.712 | INFO | "gpu" : " 17 GB", | 20260508 12:43:42.712 | INFO | "huge" : "", | 20260508 12:43:42.712 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:42.712 | INFO | "malloc" : " 14 GB", | 20260508 12:43:42.712 | INFO | "pinned" : " 29 GB" | 20260508 12:43:42.713 | INFO | } | 20260508 12:43:42.713 | INFO | Perf report for the last 5.1392 seconds: | 20260508 12:43:42.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:42.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:43:42.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260508 12:43:42.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:42.713 | INFO | Uploaded 36GiB to GPU in 2.3306s (15.4467GiB/s) | 20260508 12:43:42.713 | INFO | Downloaded 34GiB to GPU in 2.50117s (13.5936GiB/s) | 20260508 12:43:47.855 | INFO | Analyzing chunk frames = [8198656,8199168) | 20260508 12:43:47.855 | INFO | MemInfo: { | 20260508 12:43:47.855 | INFO | "gpu" : " 17 GB", | 20260508 12:43:47.855 | INFO | "huge" : "", | 20260508 12:43:47.855 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:47.855 | INFO | "malloc" : " 14 GB", | 20260508 12:43:47.855 | INFO | "pinned" : " 29 GB" | 20260508 12:43:47.855 | INFO | } | 20260508 12:43:47.855 | INFO | Perf report for the last 5.14215 seconds: | 20260508 12:43:47.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:47.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:43:47.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:43:47.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:47.855 | INFO | Uploaded 36GiB to GPU in 2.40582s (14.9637GiB/s) | 20260508 12:43:47.855 | INFO | Downloaded 34GiB to GPU in 2.43846s (13.9432GiB/s) | 20260508 12:43:53.002 | INFO | Analyzing chunk frames = [8199168,8199680) | 20260508 12:43:53.002 | INFO | MemInfo: { | 20260508 12:43:53.002 | INFO | "gpu" : " 17 GB", | 20260508 12:43:53.002 | INFO | "huge" : "", | 20260508 12:43:53.002 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:53.002 | INFO | "malloc" : " 14 GB", | 20260508 12:43:53.002 | INFO | "pinned" : " 29 GB" | 20260508 12:43:53.002 | INFO | } | 20260508 12:43:53.002 | INFO | Perf report for the last 5.14739 seconds: | 20260508 12:43:53.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:53.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.002 | INFO | ANALYSIS : 65.3%, 3.0, 102.1ms, 0 | 20260508 12:43:53.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.002 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:43:53.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:53.003 | INFO | Uploaded 36GiB to GPU in 2.45388s (14.6706GiB/s) | 20260508 12:43:53.003 | INFO | Downloaded 34GiB to GPU in 2.4483s (13.8872GiB/s) | 20260508 12:43:58.130 | INFO | Analyzing chunk frames = [8199680,8200192) | 20260508 12:43:58.130 | INFO | MemInfo: { | 20260508 12:43:58.130 | INFO | "gpu" : " 17 GB", | 20260508 12:43:58.130 | INFO | "huge" : "", | 20260508 12:43:58.130 | INFO | "huge_shared" : " 29 GB", | 20260508 12:43:58.130 | INFO | "malloc" : " 14 GB", | 20260508 12:43:58.130 | INFO | "pinned" : " 29 GB" | 20260508 12:43:58.130 | INFO | } | 20260508 12:43:58.130 | INFO | Perf report for the last 5.12821 seconds: | 20260508 12:43:58.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:43:58.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.131 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 12:43:58.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.131 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:43:58.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:43:58.131 | INFO | Uploaded 36GiB to GPU in 2.34842s (15.3294GiB/s) | 20260508 12:43:58.131 | INFO | Downloaded 34GiB to GPU in 2.41345s (14.0877GiB/s) | 20260508 12:43:58.131 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:43:58.131 | INFO | Total mem usage: 16.7 GiB | 20260508 12:43:58.131 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:43:58.131 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:43:58.131 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:43:58.131 | INFO | High Water Breakdown: | 20260508 12:43:58.131 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:43:58.131 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:43:58.131 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:43:58.131 | INFO | ------------------------------------------ | 20260508 12:43:58.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:43:58.131 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:43:58.131 | INFO | Total mem usage: 26.5 GiB | 20260508 12:43:58.131 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:43:58.131 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:43:58.131 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:43:58.131 | INFO | High Water Breakdown: | 20260508 12:43:58.131 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:43:58.131 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:43:58.131 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:43:58.131 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:43:58.131 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:43:58.131 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:43:58.131 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:43:58.131 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:43:58.131 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:43:58.131 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:43:58.131 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:43:58.131 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:43:58.131 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:43:58.132 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:43:58.132 | INFO | ------------------------------------------ | 20260508 12:43:58.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:43:58.132 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:43:58.132 | INFO | Total mem usage: 27.8 GiB | 20260508 12:43:58.132 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:43:58.132 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:43:58.132 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:43:58.132 | INFO | High Water Breakdown: | 20260508 12:43:58.132 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:43:58.132 | INFO | ------------------------------------------ | 20260508 12:43:58.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:43:58.132 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:43:58.132 | INFO | Total mem usage: 15.7 GiB | 20260508 12:43:58.132 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:43:58.132 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:43:58.132 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:43:58.132 | INFO | High Water Breakdown: | 20260508 12:43:58.132 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:43:58.132 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:43:58.132 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:43:58.132 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:43:58.132 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:43:58.132 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:43:58.132 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:43:58.132 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:43:58.132 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:43:58.132 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:43:58.132 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:43:58.132 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:43:58.132 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:43:58.132 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:43:58.132 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:43:58.132 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:43:58.132 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:43:58.132 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:43:58.132 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:43:58.132 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:43:58.132 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:43:58.132 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:43:58.132 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:43:58.132 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:43:58.132 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:43:58.132 | INFO | ------------------------------------------ | 20260508 12:43:58.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:43:58.132 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:43:58.132 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:43:58.132 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:43:58.132 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.39e+01 / 4.98e-01 | 20260508 12:43:58.132 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.04e-01, 1.12e-02 / 4.97e-03 | 20260508 12:43:58.132 | INFO | PerfReport: 12.29%, Baseline -- 4.01e+00 / 4.38e+00, 6.03e-01 / 2.59e-01 | 20260508 12:43:58.132 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.46e+00, 2.12e-01 / 1.09e-01 | 20260508 12:43:58.132 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.58e+01, 3.60e+00 / 1.07e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.76e+00 / 1.35e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.34e+00 / 2.49e+00, 4.07e-01 / 2.85e-01 | 20260508 12:43:58.132 | INFO | PerfReport: 2.95%, Metrics -- 9.65e-01 / 1.05e+00, 2.37e-01 / 6.81e-02 | 20260508 12:43:58.132 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.21e-01, 4.64e-02 / 2.65e-02 | 20260508 12:43:58.132 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:43:58.132 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:43:58.132 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:43:58.132 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 24.12%, Upload -- 2.52e+01 / 0.00e+00, 7.78e+00 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 24.83%, Download -- 2.60e+01 / 0.00e+00, 8.28e+00 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 31.20%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 2.02%, LockUpload -- 2.12e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 2.73%, LockDownload -- 2.86e+00 / 0.00e+00, 4.26e+00 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 15.08%, LockCompute -- 1.58e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:43:58.132 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.21e-02 / 0.00e+00 | 20260508 12:43:58.229 | INFO | Starting a round of Baz IO | 20260508 12:44:01.643 | INFO | Populating RTMetrics report | 20260508 12:44:01.643 | INFO | RTMetrics report populated | 20260508 12:44:01.643 | INFO | Opening tmp RTMetrics json file | 20260508 12:44:01.644 | INFO | Serializing RTMetrics to json | 20260508 12:44:01.645 | INFO | Serialization done, swapping tmp file now... | 20260508 12:44:01.646 | INFO | RTMetrics done | 20260508 12:44:01.704 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.475 seconds | 20260508 12:44:01.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:44:01.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157293% / 0% / 7.94729e-05% | 20260508 12:44:01.704 | INFO | Baz IO overall: | 20260508 12:44:01.704 | INFO | BytesWritten : 3510213437544 | 20260508 12:44:01.704 | INFO | NumEvents : 1684062687471 | 20260508 12:44:03.267 | INFO | Analyzing chunk frames = [8200192,8200704) | 20260508 12:44:03.267 | INFO | MemInfo: { | 20260508 12:44:03.267 | INFO | "gpu" : " 17 GB", | 20260508 12:44:03.267 | INFO | "huge" : "", | 20260508 12:44:03.267 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:03.267 | INFO | "malloc" : " 14 GB", | 20260508 12:44:03.267 | INFO | "pinned" : " 29 GB" | 20260508 12:44:03.267 | INFO | } | 20260508 12:44:03.267 | INFO | Perf report for the last 5.13635 seconds: | 20260508 12:44:03.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:03.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:03.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:03.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:03.267 | INFO | ANALYSIS : 68.1%, 3.0, 105.8ms, 0 | 20260508 12:44:03.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:03.267 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 12:44:03.267 | INFO | PRE_HQ : 65.7%, 1.6, 54.1ms, 0 | 20260508 12:44:03.267 | INFO | BAZWRITER : 66.9%, 2.5, 84.4ms, 0 | 20260508 12:44:03.267 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 12:44:03.268 | INFO | Uploaded 36GiB to GPU in 2.5735s (13.9887GiB/s) | 20260508 12:44:03.268 | INFO | Downloaded 38GiB to GPU in 2.67943s (14.1821GiB/s) | 20260508 12:44:08.410 | INFO | Analyzing chunk frames = [8200704,8201216) | 20260508 12:44:08.410 | INFO | MemInfo: { | 20260508 12:44:08.410 | INFO | "gpu" : " 17 GB", | 20260508 12:44:08.410 | INFO | "huge" : "", | 20260508 12:44:08.410 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:08.410 | INFO | "malloc" : " 14 GB", | 20260508 12:44:08.410 | INFO | "pinned" : " 29 GB" | 20260508 12:44:08.410 | INFO | } | 20260508 12:44:08.410 | INFO | Perf report for the last 5.14345 seconds: | 20260508 12:44:08.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:08.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.410 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:44:08.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.410 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:44:08.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:08.411 | INFO | Uploaded 36GiB to GPU in 2.467s (14.5926GiB/s) | 20260508 12:44:08.411 | INFO | Downloaded 34GiB to GPU in 2.47223s (13.7528GiB/s) | 20260508 12:44:13.550 | INFO | Analyzing chunk frames = [8201216,8201728) | 20260508 12:44:13.550 | INFO | MemInfo: { | 20260508 12:44:13.550 | INFO | "gpu" : " 17 GB", | 20260508 12:44:13.550 | INFO | "huge" : "", | 20260508 12:44:13.550 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:13.550 | INFO | "malloc" : " 14 GB", | 20260508 12:44:13.550 | INFO | "pinned" : " 29 GB" | 20260508 12:44:13.550 | INFO | } | 20260508 12:44:13.550 | INFO | Perf report for the last 5.13994 seconds: | 20260508 12:44:13.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:13.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.550 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 12:44:13.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.550 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:44:13.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:13.551 | INFO | Uploaded 36GiB to GPU in 2.47905s (14.5217GiB/s) | 20260508 12:44:13.551 | INFO | Downloaded 34GiB to GPU in 2.44324s (13.916GiB/s) | 20260508 12:44:18.691 | INFO | Analyzing chunk frames = [8201728,8202240) | 20260508 12:44:18.691 | INFO | MemInfo: { | 20260508 12:44:18.691 | INFO | "gpu" : " 17 GB", | 20260508 12:44:18.691 | INFO | "huge" : "", | 20260508 12:44:18.692 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:18.692 | INFO | "malloc" : " 14 GB", | 20260508 12:44:18.692 | INFO | "pinned" : " 29 GB" | 20260508 12:44:18.692 | INFO | } | 20260508 12:44:18.692 | INFO | Perf report for the last 5.14136 seconds: | 20260508 12:44:18.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:18.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:44:18.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:44:18.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:18.692 | INFO | Uploaded 36GiB to GPU in 2.47915s (14.5211GiB/s) | 20260508 12:44:18.692 | INFO | Downloaded 34GiB to GPU in 2.4511s (13.8713GiB/s) | 20260508 12:44:23.834 | INFO | Analyzing chunk frames = [8202240,8202752) | 20260508 12:44:23.834 | INFO | MemInfo: { | 20260508 12:44:23.834 | INFO | "gpu" : " 17 GB", | 20260508 12:44:23.834 | INFO | "huge" : "", | 20260508 12:44:23.834 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:23.834 | INFO | "malloc" : " 14 GB", | 20260508 12:44:23.834 | INFO | "pinned" : " 29 GB" | 20260508 12:44:23.834 | INFO | } | 20260508 12:44:23.834 | INFO | Perf report for the last 5.14231 seconds: | 20260508 12:44:23.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:23.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 12:44:23.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:44:23.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:23.834 | INFO | Uploaded 36GiB to GPU in 2.34794s (15.3326GiB/s) | 20260508 12:44:23.835 | INFO | Downloaded 34GiB to GPU in 2.50844s (13.5543GiB/s) | 20260508 12:44:28.967 | INFO | Analyzing chunk frames = [8202752,8203264) | 20260508 12:44:28.967 | INFO | MemInfo: { | 20260508 12:44:28.967 | INFO | "gpu" : " 17 GB", | 20260508 12:44:28.967 | INFO | "huge" : "", | 20260508 12:44:28.967 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:28.967 | INFO | "malloc" : " 14 GB", | 20260508 12:44:28.967 | INFO | "pinned" : " 29 GB" | 20260508 12:44:28.967 | INFO | } | 20260508 12:44:28.967 | INFO | Perf report for the last 5.13316 seconds: | 20260508 12:44:28.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:28.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:44:28.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:44:28.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:28.967 | INFO | Uploaded 36GiB to GPU in 2.43915s (14.7592GiB/s) | 20260508 12:44:28.967 | INFO | Downloaded 34GiB to GPU in 2.4319s (13.9808GiB/s) | 20260508 12:44:34.111 | INFO | Analyzing chunk frames = [8203264,8203776) | 20260508 12:44:34.111 | INFO | MemInfo: { | 20260508 12:44:34.111 | INFO | "gpu" : " 17 GB", | 20260508 12:44:34.111 | INFO | "huge" : "", | 20260508 12:44:34.111 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:34.111 | INFO | "malloc" : " 14 GB", | 20260508 12:44:34.111 | INFO | "pinned" : " 29 GB" | 20260508 12:44:34.111 | INFO | } | 20260508 12:44:34.111 | INFO | Perf report for the last 5.144 seconds: | 20260508 12:44:34.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:34.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 12:44:34.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:44:34.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:34.111 | INFO | Uploaded 36GiB to GPU in 2.49658s (14.4197GiB/s) | 20260508 12:44:34.111 | INFO | Downloaded 34GiB to GPU in 2.4521s (13.8657GiB/s) | 20260508 12:44:39.248 | INFO | Analyzing chunk frames = [8203776,8204288) | 20260508 12:44:39.249 | INFO | MemInfo: { | 20260508 12:44:39.249 | INFO | "gpu" : " 17 GB", | 20260508 12:44:39.249 | INFO | "huge" : "", | 20260508 12:44:39.249 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:39.249 | INFO | "malloc" : " 14 GB", | 20260508 12:44:39.249 | INFO | "pinned" : " 29 GB" | 20260508 12:44:39.249 | INFO | } | 20260508 12:44:39.249 | INFO | Perf report for the last 5.13762 seconds: | 20260508 12:44:39.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:39.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 12:44:39.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:44:39.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:39.249 | INFO | Uploaded 36GiB to GPU in 2.48876s (14.465GiB/s) | 20260508 12:44:39.249 | INFO | Downloaded 34GiB to GPU in 2.45527s (13.8477GiB/s) | 20260508 12:44:39.346 | INFO | Starting a round of Baz IO | 20260508 12:44:42.758 | INFO | Populating RTMetrics report | 20260508 12:44:42.758 | INFO | RTMetrics report populated | 20260508 12:44:42.758 | INFO | Opening tmp RTMetrics json file | 20260508 12:44:42.764 | INFO | Serializing RTMetrics to json | 20260508 12:44:42.765 | INFO | Serialization done, swapping tmp file now... | 20260508 12:44:42.765 | INFO | RTMetrics done | 20260508 12:44:42.826 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.479 seconds | 20260508 12:44:42.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:44:42.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156685% / 3.97364e-06% / 8.74201e-05% | 20260508 12:44:42.826 | INFO | Baz IO overall: | 20260508 12:44:42.826 | INFO | BytesWritten : 3511718980714 | 20260508 12:44:42.826 | INFO | NumEvents : 1684768529857 | 20260508 12:44:44.387 | INFO | Analyzing chunk frames = [8204288,8204800) | 20260508 12:44:44.387 | INFO | MemInfo: { | 20260508 12:44:44.387 | INFO | "gpu" : " 17 GB", | 20260508 12:44:44.387 | INFO | "huge" : "", | 20260508 12:44:44.387 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:44.387 | INFO | "malloc" : " 14 GB", | 20260508 12:44:44.387 | INFO | "pinned" : " 29 GB" | 20260508 12:44:44.387 | INFO | } | 20260508 12:44:44.387 | INFO | Perf report for the last 5.13871 seconds: | 20260508 12:44:44.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:44.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:44.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:44.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:44.388 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 12:44:44.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:44.388 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 12:44:44.388 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 12:44:44.388 | INFO | BAZWRITER : 67.0%, 2.4, 83.1ms, 0 | 20260508 12:44:44.388 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:44:44.388 | INFO | Uploaded 36GiB to GPU in 2.64063s (13.6331GiB/s) | 20260508 12:44:44.388 | INFO | Downloaded 38GiB to GPU in 2.7157s (13.9927GiB/s) | 20260508 12:44:49.523 | INFO | Analyzing chunk frames = [8204800,8205312) | 20260508 12:44:49.523 | INFO | MemInfo: { | 20260508 12:44:49.523 | INFO | "gpu" : " 17 GB", | 20260508 12:44:49.523 | INFO | "huge" : "", | 20260508 12:44:49.523 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:49.523 | INFO | "malloc" : " 14 GB", | 20260508 12:44:49.523 | INFO | "pinned" : " 29 GB" | 20260508 12:44:49.523 | INFO | } | 20260508 12:44:49.523 | INFO | Perf report for the last 5.1358 seconds: | 20260508 12:44:49.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:49.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.523 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 12:44:49.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.523 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260508 12:44:49.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:49.524 | INFO | Uploaded 36GiB to GPU in 2.44529s (14.7222GiB/s) | 20260508 12:44:49.524 | INFO | Downloaded 34GiB to GPU in 2.47178s (13.7553GiB/s) | 20260508 12:44:54.667 | INFO | Analyzing chunk frames = [8205312,8205824) | 20260508 12:44:54.667 | INFO | MemInfo: { | 20260508 12:44:54.667 | INFO | "gpu" : " 17 GB", | 20260508 12:44:54.667 | INFO | "huge" : "", | 20260508 12:44:54.667 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:54.667 | INFO | "malloc" : " 14 GB", | 20260508 12:44:54.667 | INFO | "pinned" : " 29 GB" | 20260508 12:44:54.667 | INFO | } | 20260508 12:44:54.667 | INFO | Perf report for the last 5.14383 seconds: | 20260508 12:44:54.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:54.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.667 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:44:54.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.667 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:44:54.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:54.668 | INFO | Uploaded 36GiB to GPU in 2.48221s (14.5032GiB/s) | 20260508 12:44:54.668 | INFO | Downloaded 34GiB to GPU in 2.45123s (13.8706GiB/s) | 20260508 12:44:59.808 | INFO | Analyzing chunk frames = [8205824,8206336) | 20260508 12:44:59.808 | INFO | MemInfo: { | 20260508 12:44:59.808 | INFO | "gpu" : " 17 GB", | 20260508 12:44:59.808 | INFO | "huge" : "", | 20260508 12:44:59.808 | INFO | "huge_shared" : " 29 GB", | 20260508 12:44:59.808 | INFO | "malloc" : " 14 GB", | 20260508 12:44:59.808 | INFO | "pinned" : " 29 GB" | 20260508 12:44:59.808 | INFO | } | 20260508 12:44:59.808 | INFO | Perf report for the last 5.14133 seconds: | 20260508 12:44:59.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:44:59.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.809 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 12:44:59.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.809 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:44:59.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:44:59.809 | INFO | Uploaded 36GiB to GPU in 2.5129s (14.3261GiB/s) | 20260508 12:44:59.809 | INFO | Downloaded 34GiB to GPU in 2.44149s (13.9259GiB/s) | 20260508 12:45:04.948 | INFO | Analyzing chunk frames = [8206336,8206848) | 20260508 12:45:04.948 | INFO | MemInfo: { | 20260508 12:45:04.948 | INFO | "gpu" : " 17 GB", | 20260508 12:45:04.948 | INFO | "huge" : "", | 20260508 12:45:04.948 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:04.948 | INFO | "malloc" : " 14 GB", | 20260508 12:45:04.948 | INFO | "pinned" : " 29 GB" | 20260508 12:45:04.948 | INFO | } | 20260508 12:45:04.948 | INFO | Perf report for the last 5.13967 seconds: | 20260508 12:45:04.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:04.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.948 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:45:04.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.948 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 12:45:04.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:04.949 | INFO | Uploaded 36GiB to GPU in 2.33206s (15.437GiB/s) | 20260508 12:45:04.949 | INFO | Downloaded 34GiB to GPU in 2.50174s (13.5906GiB/s) | 20260508 12:45:10.084 | INFO | Analyzing chunk frames = [8206848,8207360) | 20260508 12:45:10.084 | INFO | MemInfo: { | 20260508 12:45:10.084 | INFO | "gpu" : " 17 GB", | 20260508 12:45:10.084 | INFO | "huge" : "", | 20260508 12:45:10.084 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:10.084 | INFO | "malloc" : " 14 GB", | 20260508 12:45:10.084 | INFO | "pinned" : " 29 GB" | 20260508 12:45:10.084 | INFO | } | 20260508 12:45:10.084 | INFO | Perf report for the last 5.13571 seconds: | 20260508 12:45:10.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:10.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:45:10.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:45:10.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:10.084 | INFO | Uploaded 36GiB to GPU in 2.37534s (15.1557GiB/s) | 20260508 12:45:10.084 | INFO | Downloaded 34GiB to GPU in 2.44656s (13.8971GiB/s) | 20260508 12:45:15.218 | INFO | Analyzing chunk frames = [8207360,8207872) | 20260508 12:45:15.218 | INFO | MemInfo: { | 20260508 12:45:15.218 | INFO | "gpu" : " 17 GB", | 20260508 12:45:15.218 | INFO | "huge" : "", | 20260508 12:45:15.218 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:15.218 | INFO | "malloc" : " 14 GB", | 20260508 12:45:15.218 | INFO | "pinned" : " 29 GB" | 20260508 12:45:15.218 | INFO | } | 20260508 12:45:15.218 | INFO | Perf report for the last 5.1344 seconds: | 20260508 12:45:15.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:15.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.218 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 12:45:15.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.218 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:45:15.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:15.219 | INFO | Uploaded 36GiB to GPU in 2.43515s (14.7835GiB/s) | 20260508 12:45:15.219 | INFO | Downloaded 34GiB to GPU in 2.47174s (13.7555GiB/s) | 20260508 12:45:20.366 | INFO | Analyzing chunk frames = [8207872,8208384) | 20260508 12:45:20.366 | INFO | MemInfo: { | 20260508 12:45:20.366 | INFO | "gpu" : " 17 GB", | 20260508 12:45:20.366 | INFO | "huge" : "", | 20260508 12:45:20.366 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:20.366 | INFO | "malloc" : " 14 GB", | 20260508 12:45:20.366 | INFO | "pinned" : " 29 GB" | 20260508 12:45:20.366 | INFO | } | 20260508 12:45:20.366 | INFO | Perf report for the last 5.14781 seconds: | 20260508 12:45:20.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:20.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 12:45:20.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:45:20.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:20.366 | INFO | Uploaded 36GiB to GPU in 2.3983s (15.0106GiB/s) | 20260508 12:45:20.366 | INFO | Downloaded 34GiB to GPU in 2.4504s (13.8753GiB/s) | 20260508 12:45:20.367 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:45:20.367 | INFO | Total mem usage: 16.7 GiB | 20260508 12:45:20.367 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:45:20.367 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:45:20.367 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:45:20.367 | INFO | High Water Breakdown: | 20260508 12:45:20.367 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:45:20.367 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:45:20.367 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:45:20.367 | INFO | ------------------------------------------ | 20260508 12:45:20.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:45:20.367 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:45:20.367 | INFO | Total mem usage: 26.5 GiB | 20260508 12:45:20.367 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:45:20.367 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:45:20.367 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:45:20.367 | INFO | High Water Breakdown: | 20260508 12:45:20.367 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:45:20.367 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:45:20.367 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:45:20.367 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:45:20.367 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:45:20.367 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:45:20.367 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:45:20.367 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:45:20.367 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:45:20.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:45:20.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:45:20.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:45:20.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:45:20.367 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:45:20.367 | INFO | ------------------------------------------ | 20260508 12:45:20.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:45:20.367 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:45:20.367 | INFO | Total mem usage: 27.8 GiB | 20260508 12:45:20.367 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:45:20.367 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:45:20.367 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:45:20.367 | INFO | High Water Breakdown: | 20260508 12:45:20.367 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:45:20.367 | INFO | ------------------------------------------ | 20260508 12:45:20.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:45:20.367 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:45:20.367 | INFO | Total mem usage: 15.7 GiB | 20260508 12:45:20.367 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:45:20.367 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:45:20.367 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:45:20.367 | INFO | High Water Breakdown: | 20260508 12:45:20.367 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:45:20.367 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:45:20.367 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:45:20.367 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:45:20.367 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:45:20.367 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:45:20.367 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:45:20.367 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:45:20.367 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:45:20.367 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:45:20.367 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:45:20.367 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:45:20.367 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:45:20.367 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:45:20.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:45:20.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:45:20.367 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:45:20.367 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:45:20.367 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:45:20.367 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:45:20.367 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:45:20.367 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:45:20.367 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:45:20.368 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:45:20.368 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:45:20.368 | INFO | ------------------------------------------ | 20260508 12:45:20.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:45:20.368 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:45:20.368 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:45:20.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:45:20.368 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 4.41e-01 | 20260508 12:45:20.368 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.05e-01, 1.12e-02 / 2.58e-03 | 20260508 12:45:20.368 | INFO | PerfReport: 12.19%, Baseline -- 3.97e+00 / 4.43e+00, 5.93e-01 / 1.88e-01 | 20260508 12:45:20.368 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.47e+00, 2.09e-01 / 9.89e-02 | 20260508 12:45:20.368 | INFO | PerfReport: 16.02%, DME -- 4.18e+01 / 4.53e+01, 3.70e+00 / 1.26e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.79e+00 / 1.47e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.44e+00, 4.06e-01 / 2.29e-01 | 20260508 12:45:20.368 | INFO | PerfReport: 2.95%, Metrics -- 9.62e-01 / 1.09e+00, 2.36e-01 / 8.31e-02 | 20260508 12:45:20.368 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.20e-01, 4.71e-02 / 1.98e-02 | 20260508 12:45:20.368 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:45:20.368 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:45:20.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:45:20.368 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 24.41%, Upload -- 2.56e+01 / 0.00e+00, 8.06e+00 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 24.69%, Download -- 2.59e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 31.01%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 2.15%, LockUpload -- 2.26e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 2.62%, LockDownload -- 2.75e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 15.10%, LockCompute -- 1.59e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:45:20.368 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260508 12:45:20.466 | INFO | Starting a round of Baz IO | 20260508 12:45:23.865 | INFO | Populating RTMetrics report | 20260508 12:45:23.865 | INFO | RTMetrics report populated | 20260508 12:45:23.865 | INFO | Opening tmp RTMetrics json file | 20260508 12:45:23.874 | INFO | Serializing RTMetrics to json | 20260508 12:45:23.875 | INFO | Serialization done, swapping tmp file now... | 20260508 12:45:23.876 | INFO | RTMetrics done | 20260508 12:45:23.933 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.467 seconds | 20260508 12:45:23.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:45:23.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157448% / 0% / 6.35783e-05% | 20260508 12:45:23.933 | INFO | Baz IO overall: | 20260508 12:45:23.933 | INFO | BytesWritten : 3513224296185 | 20260508 12:45:23.933 | INFO | NumEvents : 1685474492501 | 20260508 12:45:25.505 | INFO | Analyzing chunk frames = [8208384,8208896) | 20260508 12:45:25.505 | INFO | MemInfo: { | 20260508 12:45:25.505 | INFO | "gpu" : " 17 GB", | 20260508 12:45:25.505 | INFO | "huge" : "", | 20260508 12:45:25.505 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:25.505 | INFO | "malloc" : " 14 GB", | 20260508 12:45:25.505 | INFO | "pinned" : " 29 GB" | 20260508 12:45:25.505 | INFO | } | 20260508 12:45:25.505 | INFO | Perf report for the last 5.1388 seconds: | 20260508 12:45:25.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:25.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:25.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:25.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:25.505 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:45:25.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:25.505 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:45:25.505 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 12:45:25.505 | INFO | BAZWRITER : 66.8%, 2.4, 83.5ms, 0 | 20260508 12:45:25.505 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:45:25.505 | INFO | Uploaded 36GiB to GPU in 2.62282s (13.7257GiB/s) | 20260508 12:45:25.505 | INFO | Downloaded 38GiB to GPU in 2.7162s (13.9901GiB/s) | 20260508 12:45:30.642 | INFO | Analyzing chunk frames = [8208896,8209408) | 20260508 12:45:30.642 | INFO | MemInfo: { | 20260508 12:45:30.642 | INFO | "gpu" : " 17 GB", | 20260508 12:45:30.642 | INFO | "huge" : "", | 20260508 12:45:30.642 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:30.642 | INFO | "malloc" : " 14 GB", | 20260508 12:45:30.642 | INFO | "pinned" : " 29 GB" | 20260508 12:45:30.642 | INFO | } | 20260508 12:45:30.642 | INFO | Perf report for the last 5.13745 seconds: | 20260508 12:45:30.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:30.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.642 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 12:45:30.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.642 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:45:30.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:30.643 | INFO | Uploaded 36GiB to GPU in 2.49572s (14.4247GiB/s) | 20260508 12:45:30.643 | INFO | Downloaded 34GiB to GPU in 2.4802s (13.7086GiB/s) | 20260508 12:45:35.778 | INFO | Analyzing chunk frames = [8209408,8209920) | 20260508 12:45:35.778 | INFO | MemInfo: { | 20260508 12:45:35.778 | INFO | "gpu" : " 17 GB", | 20260508 12:45:35.778 | INFO | "huge" : "", | 20260508 12:45:35.778 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:35.778 | INFO | "malloc" : " 14 GB", | 20260508 12:45:35.778 | INFO | "pinned" : " 29 GB" | 20260508 12:45:35.778 | INFO | } | 20260508 12:45:35.778 | INFO | Perf report for the last 5.13536 seconds: | 20260508 12:45:35.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:35.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 12:45:35.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:45:35.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:35.778 | INFO | Uploaded 36GiB to GPU in 2.49926s (14.4042GiB/s) | 20260508 12:45:35.778 | INFO | Downloaded 34GiB to GPU in 2.46037s (13.8191GiB/s) | 20260508 12:45:40.921 | INFO | Analyzing chunk frames = [8209920,8210432) | 20260508 12:45:40.921 | INFO | MemInfo: { | 20260508 12:45:40.921 | INFO | "gpu" : " 17 GB", | 20260508 12:45:40.921 | INFO | "huge" : "", | 20260508 12:45:40.921 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:40.921 | INFO | "malloc" : " 14 GB", | 20260508 12:45:40.921 | INFO | "pinned" : " 29 GB" | 20260508 12:45:40.921 | INFO | } | 20260508 12:45:40.921 | INFO | Perf report for the last 5.14343 seconds: | 20260508 12:45:40.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:40.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.921 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:45:40.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.921 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:45:40.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:40.922 | INFO | Uploaded 36GiB to GPU in 2.48376s (14.4942GiB/s) | 20260508 12:45:40.922 | INFO | Downloaded 34GiB to GPU in 2.45175s (13.8677GiB/s) | 20260508 12:45:46.058 | INFO | Analyzing chunk frames = [8210432,8210944) | 20260508 12:45:46.058 | INFO | MemInfo: { | 20260508 12:45:46.058 | INFO | "gpu" : " 17 GB", | 20260508 12:45:46.058 | INFO | "huge" : "", | 20260508 12:45:46.058 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:46.058 | INFO | "malloc" : " 14 GB", | 20260508 12:45:46.058 | INFO | "pinned" : " 29 GB" | 20260508 12:45:46.058 | INFO | } | 20260508 12:45:46.058 | INFO | Perf report for the last 5.13701 seconds: | 20260508 12:45:46.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:46.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.058 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:45:46.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.058 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 12:45:46.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:46.059 | INFO | Uploaded 36GiB to GPU in 2.36934s (15.1941GiB/s) | 20260508 12:45:46.059 | INFO | Downloaded 34GiB to GPU in 2.50626s (13.566GiB/s) | 20260508 12:45:51.197 | INFO | Analyzing chunk frames = [8210944,8211456) | 20260508 12:45:51.197 | INFO | MemInfo: { | 20260508 12:45:51.197 | INFO | "gpu" : " 17 GB", | 20260508 12:45:51.197 | INFO | "huge" : "", | 20260508 12:45:51.197 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:51.197 | INFO | "malloc" : " 14 GB", | 20260508 12:45:51.197 | INFO | "pinned" : " 29 GB" | 20260508 12:45:51.197 | INFO | } | 20260508 12:45:51.197 | INFO | Perf report for the last 5.13923 seconds: | 20260508 12:45:51.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:51.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.197 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:45:51.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.198 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:45:51.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:51.198 | INFO | Uploaded 36GiB to GPU in 2.38994s (15.0631GiB/s) | 20260508 12:45:51.198 | INFO | Downloaded 34GiB to GPU in 2.42198s (14.0381GiB/s) | 20260508 12:45:56.338 | INFO | Analyzing chunk frames = [8211456,8211968) | 20260508 12:45:56.338 | INFO | MemInfo: { | 20260508 12:45:56.338 | INFO | "gpu" : " 17 GB", | 20260508 12:45:56.338 | INFO | "huge" : "", | 20260508 12:45:56.338 | INFO | "huge_shared" : " 29 GB", | 20260508 12:45:56.338 | INFO | "malloc" : " 14 GB", | 20260508 12:45:56.338 | INFO | "pinned" : " 29 GB" | 20260508 12:45:56.338 | INFO | } | 20260508 12:45:56.338 | INFO | Perf report for the last 5.14084 seconds: | 20260508 12:45:56.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:45:56.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.338 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 12:45:56.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.338 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 12:45:56.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:45:56.339 | INFO | Uploaded 36GiB to GPU in 2.47108s (14.5685GiB/s) | 20260508 12:45:56.339 | INFO | Downloaded 34GiB to GPU in 2.44517s (13.9049GiB/s) | 20260508 12:46:01.473 | INFO | Analyzing chunk frames = [8211968,8212480) | 20260508 12:46:01.474 | INFO | MemInfo: { | 20260508 12:46:01.474 | INFO | "gpu" : " 17 GB", | 20260508 12:46:01.474 | INFO | "huge" : "", | 20260508 12:46:01.474 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:01.474 | INFO | "malloc" : " 14 GB", | 20260508 12:46:01.474 | INFO | "pinned" : " 29 GB" | 20260508 12:46:01.474 | INFO | } | 20260508 12:46:01.474 | INFO | Perf report for the last 5.13546 seconds: | 20260508 12:46:01.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:01.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 12:46:01.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:46:01.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:01.474 | INFO | Uploaded 36GiB to GPU in 2.50543s (14.3688GiB/s) | 20260508 12:46:01.474 | INFO | Downloaded 34GiB to GPU in 2.46667s (13.7838GiB/s) | 20260508 12:46:01.571 | INFO | Starting a round of Baz IO | 20260508 12:46:04.983 | INFO | Populating RTMetrics report | 20260508 12:46:04.983 | INFO | RTMetrics report populated | 20260508 12:46:04.983 | INFO | Opening tmp RTMetrics json file | 20260508 12:46:04.990 | INFO | Serializing RTMetrics to json | 20260508 12:46:04.992 | INFO | Serialization done, swapping tmp file now... | 20260508 12:46:04.992 | INFO | RTMetrics done | 20260508 12:46:05.045 | INFO | Finished a round of Baz IO: Wrote 1.403 GiB in 3.473 seconds | 20260508 12:46:05.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:46:05.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156788% / 0% / 5.96046e-05% | 20260508 12:46:05.045 | INFO | Baz IO overall: | 20260508 12:46:05.045 | INFO | BytesWritten : 3514730417996 | 20260508 12:46:05.045 | INFO | NumEvents : 1686180832356 | 20260508 12:46:06.617 | INFO | Analyzing chunk frames = [8212480,8212992) | 20260508 12:46:06.617 | INFO | MemInfo: { | 20260508 12:46:06.617 | INFO | "gpu" : " 17 GB", | 20260508 12:46:06.617 | INFO | "huge" : "", | 20260508 12:46:06.617 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:06.617 | INFO | "malloc" : " 14 GB", | 20260508 12:46:06.617 | INFO | "pinned" : " 29 GB" | 20260508 12:46:06.617 | INFO | } | 20260508 12:46:06.617 | INFO | Perf report for the last 5.14335 seconds: | 20260508 12:46:06.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:06.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:06.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:06.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:06.617 | INFO | ANALYSIS : 67.3%, 3.0, 105.4ms, 0 | 20260508 12:46:06.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:06.617 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:46:06.617 | INFO | PRE_HQ : 65.1%, 1.6, 53.9ms, 0 | 20260508 12:46:06.617 | INFO | BAZWRITER : 66.7%, 2.4, 84.1ms, 0 | 20260508 12:46:06.617 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:46:06.618 | INFO | Uploaded 36GiB to GPU in 2.62177s (13.7312GiB/s) | 20260508 12:46:06.618 | INFO | Downloaded 38GiB to GPU in 2.69926s (14.0779GiB/s) | 20260508 12:46:11.757 | INFO | Analyzing chunk frames = [8212992,8213504) | 20260508 12:46:11.757 | INFO | MemInfo: { | 20260508 12:46:11.757 | INFO | "gpu" : " 17 GB", | 20260508 12:46:11.757 | INFO | "huge" : "", | 20260508 12:46:11.757 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:11.757 | INFO | "malloc" : " 14 GB", | 20260508 12:46:11.758 | INFO | "pinned" : " 29 GB" | 20260508 12:46:11.758 | INFO | } | 20260508 12:46:11.758 | INFO | Perf report for the last 5.14055 seconds: | 20260508 12:46:11.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:11.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:46:11.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:46:11.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:11.758 | INFO | Uploaded 36GiB to GPU in 2.4581s (14.6455GiB/s) | 20260508 12:46:11.758 | INFO | Downloaded 34GiB to GPU in 2.45722s (13.8368GiB/s) | 20260508 12:46:16.896 | INFO | Analyzing chunk frames = [8213504,8214016) | 20260508 12:46:16.896 | INFO | MemInfo: { | 20260508 12:46:16.896 | INFO | "gpu" : " 17 GB", | 20260508 12:46:16.896 | INFO | "huge" : "", | 20260508 12:46:16.896 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:16.896 | INFO | "malloc" : " 14 GB", | 20260508 12:46:16.896 | INFO | "pinned" : " 29 GB" | 20260508 12:46:16.896 | INFO | } | 20260508 12:46:16.896 | INFO | Perf report for the last 5.1379 seconds: | 20260508 12:46:16.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:16.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:46:16.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:46:16.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:16.896 | INFO | Uploaded 36GiB to GPU in 2.51013s (14.3419GiB/s) | 20260508 12:46:16.896 | INFO | Downloaded 34GiB to GPU in 2.45329s (13.8589GiB/s) | 20260508 12:46:22.028 | INFO | Analyzing chunk frames = [8214016,8214528) | 20260508 12:46:22.028 | INFO | MemInfo: { | 20260508 12:46:22.028 | INFO | "gpu" : " 17 GB", | 20260508 12:46:22.028 | INFO | "huge" : "", | 20260508 12:46:22.028 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:22.028 | INFO | "malloc" : " 14 GB", | 20260508 12:46:22.028 | INFO | "pinned" : " 29 GB" | 20260508 12:46:22.028 | INFO | } | 20260508 12:46:22.028 | INFO | Perf report for the last 5.13244 seconds: | 20260508 12:46:22.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:22.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | ANALYSIS : 66.2%, 3.0, 102.4ms, 0 | 20260508 12:46:22.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:46:22.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:22.028 | INFO | Uploaded 36GiB to GPU in 2.50192s (14.389GiB/s) | 20260508 12:46:22.028 | INFO | Downloaded 34GiB to GPU in 2.4472s (13.8934GiB/s) | 20260508 12:46:27.174 | INFO | Analyzing chunk frames = [8214528,8215040) | 20260508 12:46:27.174 | INFO | MemInfo: { | 20260508 12:46:27.174 | INFO | "gpu" : " 17 GB", | 20260508 12:46:27.174 | INFO | "huge" : "", | 20260508 12:46:27.174 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:27.174 | INFO | "malloc" : " 14 GB", | 20260508 12:46:27.174 | INFO | "pinned" : " 29 GB" | 20260508 12:46:27.174 | INFO | } | 20260508 12:46:27.174 | INFO | Perf report for the last 5.14632 seconds: | 20260508 12:46:27.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:27.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.175 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 12:46:27.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.175 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 12:46:27.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:27.175 | INFO | Uploaded 36GiB to GPU in 2.30099s (15.6454GiB/s) | 20260508 12:46:27.175 | INFO | Downloaded 34GiB to GPU in 2.47834s (13.7189GiB/s) | 20260508 12:46:32.317 | INFO | Analyzing chunk frames = [8215040,8215552) | 20260508 12:46:32.317 | INFO | MemInfo: { | 20260508 12:46:32.317 | INFO | "gpu" : " 17 GB", | 20260508 12:46:32.317 | INFO | "huge" : "", | 20260508 12:46:32.317 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:32.317 | INFO | "malloc" : " 14 GB", | 20260508 12:46:32.317 | INFO | "pinned" : " 29 GB" | 20260508 12:46:32.318 | INFO | } | 20260508 12:46:32.318 | INFO | Perf report for the last 5.14315 seconds: | 20260508 12:46:32.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:32.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 12:46:32.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 12:46:32.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:32.318 | INFO | Uploaded 36GiB to GPU in 2.43779s (14.7675GiB/s) | 20260508 12:46:32.318 | INFO | Downloaded 34GiB to GPU in 2.44915s (13.8824GiB/s) | 20260508 12:46:37.460 | INFO | Analyzing chunk frames = [8215552,8216064) | 20260508 12:46:37.460 | INFO | MemInfo: { | 20260508 12:46:37.460 | INFO | "gpu" : " 17 GB", | 20260508 12:46:37.460 | INFO | "huge" : "", | 20260508 12:46:37.460 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:37.460 | INFO | "malloc" : " 14 GB", | 20260508 12:46:37.460 | INFO | "pinned" : " 29 GB" | 20260508 12:46:37.460 | INFO | } | 20260508 12:46:37.461 | INFO | Perf report for the last 5.14289 seconds: | 20260508 12:46:37.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:37.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | ANALYSIS : 64.7%, 3.0, 100.8ms, 0 | 20260508 12:46:37.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 12:46:37.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:37.461 | INFO | Uploaded 36GiB to GPU in 2.33091s (15.4446GiB/s) | 20260508 12:46:37.461 | INFO | Downloaded 34GiB to GPU in 2.42025s (14.0482GiB/s) | 20260508 12:46:42.594 | INFO | Analyzing chunk frames = [8216064,8216576) | 20260508 12:46:42.594 | INFO | MemInfo: { | 20260508 12:46:42.594 | INFO | "gpu" : " 17 GB", | 20260508 12:46:42.594 | INFO | "huge" : "", | 20260508 12:46:42.594 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:42.594 | INFO | "malloc" : " 14 GB", | 20260508 12:46:42.594 | INFO | "pinned" : " 29 GB" | 20260508 12:46:42.594 | INFO | } | 20260508 12:46:42.595 | INFO | Perf report for the last 5.13398 seconds: | 20260508 12:46:42.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:42.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | ANALYSIS : 65.2%, 3.0, 101.3ms, 0 | 20260508 12:46:42.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 12:46:42.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:42.595 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:46:42.595 | INFO | Total mem usage: 16.7 GiB | 20260508 12:46:42.595 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:46:42.595 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:46:42.595 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:46:42.595 | INFO | High Water Breakdown: | 20260508 12:46:42.595 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:46:42.595 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:46:42.595 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:46:42.595 | INFO | ------------------------------------------ | 20260508 12:46:42.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:46:42.595 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:46:42.595 | INFO | Total mem usage: 26.5 GiB | 20260508 12:46:42.595 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:46:42.595 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:46:42.595 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:46:42.595 | INFO | High Water Breakdown: | 20260508 12:46:42.595 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:46:42.595 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:46:42.595 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:46:42.595 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:46:42.595 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:46:42.595 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:46:42.595 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:46:42.595 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:46:42.595 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:46:42.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:46:42.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:46:42.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:46:42.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:46:42.595 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:46:42.595 | INFO | ------------------------------------------ | 20260508 12:46:42.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:46:42.595 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:46:42.595 | INFO | Total mem usage: 27.8 GiB | 20260508 12:46:42.595 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:46:42.595 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:46:42.595 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:46:42.595 | INFO | High Water Breakdown: | 20260508 12:46:42.595 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:46:42.595 | INFO | ------------------------------------------ | 20260508 12:46:42.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:46:42.595 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:46:42.595 | INFO | Total mem usage: 15.7 GiB | 20260508 12:46:42.595 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:46:42.595 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:46:42.595 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:46:42.595 | INFO | High Water Breakdown: | 20260508 12:46:42.595 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:46:42.595 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:46:42.595 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:46:42.596 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:46:42.596 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:46:42.596 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:46:42.596 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:46:42.596 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:46:42.596 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:46:42.596 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:46:42.596 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:46:42.596 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:46:42.596 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:46:42.596 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:46:42.596 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:46:42.596 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:46:42.596 | INFO | Uploaded 36GiB to GPU in 2.31653s (15.5405GiB/s) | 20260508 12:46:42.596 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:46:42.596 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:46:42.596 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:46:42.597 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:46:42.596 | INFO | Downloaded 34GiB to GPU in 2.41842s (14.0588GiB/s) | 20260508 12:46:42.597 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:46:42.597 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:46:42.597 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:46:42.597 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:46:42.597 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:46:42.597 | INFO | ------------------------------------------ | 20260508 12:46:42.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:46:42.597 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:46:42.597 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:46:42.597 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:46:42.597 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.41e+01 / 6.63e-01 | 20260508 12:46:42.597 | INFO | PerfReport: 0.30%, AggregateStats -- 9.80e-02 / 1.06e-01, 1.22e-02 / 4.11e-03 | 20260508 12:46:42.597 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.42e+00, 5.99e-01 / 2.00e-01 | 20260508 12:46:42.597 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.43e+00, 2.13e-01 / 1.15e-01 | 20260508 12:46:42.597 | INFO | PerfReport: 16.04%, DME -- 4.19e+01 / 4.52e+01, 3.77e+00 / 1.20e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.28e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.33e+00 / 2.40e+00, 4.06e-01 / 2.68e-01 | 20260508 12:46:42.597 | INFO | PerfReport: 2.96%, Metrics -- 9.65e-01 / 1.07e+00, 2.34e-01 / 9.61e-02 | 20260508 12:46:42.597 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.94e-01 / 3.16e-01, 4.71e-02 / 3.07e-02 | 20260508 12:46:42.597 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:46:42.597 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:46:42.597 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:46:42.597 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 24.68%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 31.10%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 2.26%, LockUpload -- 2.37e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 2.51%, LockDownload -- 2.63e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 15.03%, LockCompute -- 1.58e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 12:46:42.597 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260508 12:46:42.710 | INFO | Starting a round of Baz IO | 20260508 12:46:46.106 | INFO | Populating RTMetrics report | 20260508 12:46:46.106 | INFO | RTMetrics report populated | 20260508 12:46:46.106 | INFO | Opening tmp RTMetrics json file | 20260508 12:46:46.113 | INFO | Serializing RTMetrics to json | 20260508 12:46:46.114 | INFO | Serialization done, swapping tmp file now... | 20260508 12:46:46.116 | INFO | RTMetrics done | 20260508 12:46:46.172 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.461 seconds | 20260508 12:46:46.172 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:46:46.172 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156383% / 0% / 6.35783e-05% | 20260508 12:46:46.172 | INFO | Baz IO overall: | 20260508 12:46:46.172 | INFO | BytesWritten : 3516235428457 | 20260508 12:46:46.172 | INFO | NumEvents : 1686886470965 | 20260508 12:46:47.727 | INFO | Analyzing chunk frames = [8216576,8217088) | 20260508 12:46:47.727 | INFO | MemInfo: { | 20260508 12:46:47.727 | INFO | "gpu" : " 17 GB", | 20260508 12:46:47.727 | INFO | "huge" : "", | 20260508 12:46:47.727 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:47.727 | INFO | "malloc" : " 14 GB", | 20260508 12:46:47.727 | INFO | "pinned" : " 29 GB" | 20260508 12:46:47.727 | INFO | } | 20260508 12:46:47.727 | INFO | Perf report for the last 5.13222 seconds: | 20260508 12:46:47.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:47.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:47.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:47.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:47.727 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 12:46:47.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:47.727 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260508 12:46:47.727 | INFO | PRE_HQ : 66.1%, 1.8, 62.1ms, 0 | 20260508 12:46:47.727 | INFO | BAZWRITER : 66.8%, 2.7, 92.1ms, 0 | 20260508 12:46:47.727 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:46:47.727 | INFO | Uploaded 36GiB to GPU in 2.60817s (13.8028GiB/s) | 20260508 12:46:47.727 | INFO | Downloaded 38GiB to GPU in 2.71987s (13.9713GiB/s) | 20260508 12:46:52.871 | INFO | Analyzing chunk frames = [8217088,8217600) | 20260508 12:46:52.871 | INFO | MemInfo: { | 20260508 12:46:52.871 | INFO | "gpu" : " 17 GB", | 20260508 12:46:52.871 | INFO | "huge" : "", | 20260508 12:46:52.871 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:52.871 | INFO | "malloc" : " 14 GB", | 20260508 12:46:52.871 | INFO | "pinned" : " 29 GB" | 20260508 12:46:52.871 | INFO | } | 20260508 12:46:52.871 | INFO | Perf report for the last 5.14423 seconds: | 20260508 12:46:52.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:52.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:46:52.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 12:46:52.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:52.871 | INFO | Uploaded 36GiB to GPU in 2.44785s (14.7068GiB/s) | 20260508 12:46:52.871 | INFO | Downloaded 34GiB to GPU in 2.46798s (13.7765GiB/s) | 20260508 12:46:58.008 | INFO | Analyzing chunk frames = [8217600,8218112) | 20260508 12:46:58.008 | INFO | MemInfo: { | 20260508 12:46:58.008 | INFO | "gpu" : " 17 GB", | 20260508 12:46:58.008 | INFO | "huge" : "", | 20260508 12:46:58.008 | INFO | "huge_shared" : " 29 GB", | 20260508 12:46:58.008 | INFO | "malloc" : " 14 GB", | 20260508 12:46:58.008 | INFO | "pinned" : " 29 GB" | 20260508 12:46:58.008 | INFO | } | 20260508 12:46:58.009 | INFO | Perf report for the last 5.13754 seconds: | 20260508 12:46:58.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:46:58.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 12:46:58.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:46:58.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:46:58.009 | INFO | Uploaded 36GiB to GPU in 2.47148s (14.5661GiB/s) | 20260508 12:46:58.009 | INFO | Downloaded 34GiB to GPU in 2.43452s (13.9658GiB/s) | 20260508 12:47:03.149 | INFO | Analyzing chunk frames = [8218112,8218624) | 20260508 12:47:03.149 | INFO | MemInfo: { | 20260508 12:47:03.149 | INFO | "gpu" : " 17 GB", | 20260508 12:47:03.149 | INFO | "huge" : "", | 20260508 12:47:03.149 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:03.149 | INFO | "malloc" : " 14 GB", | 20260508 12:47:03.149 | INFO | "pinned" : " 29 GB" | 20260508 12:47:03.149 | INFO | } | 20260508 12:47:03.149 | INFO | Perf report for the last 5.14071 seconds: | 20260508 12:47:03.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:03.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.149 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 12:47:03.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.149 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 12:47:03.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:03.150 | INFO | Uploaded 36GiB to GPU in 2.51953s (14.2884GiB/s) | 20260508 12:47:03.150 | INFO | Downloaded 34GiB to GPU in 2.43934s (13.9382GiB/s) | 20260508 12:47:08.287 | INFO | Analyzing chunk frames = [8218624,8219136) | 20260508 12:47:08.287 | INFO | MemInfo: { | 20260508 12:47:08.287 | INFO | "gpu" : " 17 GB", | 20260508 12:47:08.287 | INFO | "huge" : "", | 20260508 12:47:08.287 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:08.287 | INFO | "malloc" : " 14 GB", | 20260508 12:47:08.287 | INFO | "pinned" : " 29 GB" | 20260508 12:47:08.287 | INFO | } | 20260508 12:47:08.287 | INFO | Perf report for the last 5.13774 seconds: | 20260508 12:47:08.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:08.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.287 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:47:08.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.287 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:47:08.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:08.288 | INFO | Uploaded 36GiB to GPU in 2.29772s (15.6677GiB/s) | 20260508 12:47:08.288 | INFO | Downloaded 34GiB to GPU in 2.48587s (13.6773GiB/s) | 20260508 12:47:13.435 | INFO | Analyzing chunk frames = [8219136,8219648) | 20260508 12:47:13.435 | INFO | MemInfo: { | 20260508 12:47:13.435 | INFO | "gpu" : " 17 GB", | 20260508 12:47:13.435 | INFO | "huge" : "", | 20260508 12:47:13.435 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:13.435 | INFO | "malloc" : " 14 GB", | 20260508 12:47:13.435 | INFO | "pinned" : " 29 GB" | 20260508 12:47:13.435 | INFO | } | 20260508 12:47:13.435 | INFO | Perf report for the last 5.1481 seconds: | 20260508 12:47:13.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:13.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.435 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:47:13.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.435 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:47:13.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:13.436 | INFO | Uploaded 36GiB to GPU in 2.44974s (14.6954GiB/s) | 20260508 12:47:13.436 | INFO | Downloaded 34GiB to GPU in 2.4317s (13.982GiB/s) | 20260508 12:47:18.568 | INFO | Analyzing chunk frames = [8219648,8220160) | 20260508 12:47:18.568 | INFO | MemInfo: { | 20260508 12:47:18.568 | INFO | "gpu" : " 17 GB", | 20260508 12:47:18.568 | INFO | "huge" : "", | 20260508 12:47:18.568 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:18.568 | INFO | "malloc" : " 14 GB", | 20260508 12:47:18.568 | INFO | "pinned" : " 29 GB" | 20260508 12:47:18.569 | INFO | } | 20260508 12:47:18.569 | INFO | Perf report for the last 5.13347 seconds: | 20260508 12:47:18.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:18.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | ANALYSIS : 64.7%, 3.0, 100.5ms, 0 | 20260508 12:47:18.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 12:47:18.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:18.569 | INFO | Uploaded 36GiB to GPU in 2.32523s (15.4823GiB/s) | 20260508 12:47:18.569 | INFO | Downloaded 34GiB to GPU in 2.38942s (14.2294GiB/s) | 20260508 12:47:23.704 | INFO | Analyzing chunk frames = [8220160,8220672) | 20260508 12:47:23.704 | INFO | MemInfo: { | 20260508 12:47:23.704 | INFO | "gpu" : " 17 GB", | 20260508 12:47:23.704 | INFO | "huge" : "", | 20260508 12:47:23.704 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:23.704 | INFO | "malloc" : " 14 GB", | 20260508 12:47:23.704 | INFO | "pinned" : " 29 GB" | 20260508 12:47:23.704 | INFO | } | 20260508 12:47:23.704 | INFO | Perf report for the last 5.13567 seconds: | 20260508 12:47:23.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:23.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.704 | INFO | ANALYSIS : 65.9%, 3.0, 103.0ms, 0 | 20260508 12:47:23.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.704 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 12:47:23.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:23.705 | INFO | Uploaded 36GiB to GPU in 2.4998s (14.4011GiB/s) | 20260508 12:47:23.705 | INFO | Downloaded 34GiB to GPU in 2.44976s (13.8789GiB/s) | 20260508 12:47:23.802 | INFO | Starting a round of Baz IO | 20260508 12:47:27.203 | INFO | Populating RTMetrics report | 20260508 12:47:27.203 | INFO | RTMetrics report populated | 20260508 12:47:27.203 | INFO | Opening tmp RTMetrics json file | 20260508 12:47:27.212 | INFO | Serializing RTMetrics to json | 20260508 12:47:27.214 | INFO | Serialization done, swapping tmp file now... | 20260508 12:47:27.214 | INFO | RTMetrics done | 20260508 12:47:27.273 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.471 seconds | 20260508 12:47:27.273 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.00% / 0.00% / 60.00% | 20260508 12:47:27.273 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156192% / 0% / 5.5631e-05% | 20260508 12:47:27.273 | INFO | Baz IO overall: | 20260508 12:47:27.273 | INFO | BytesWritten : 3517740567709 | 20260508 12:47:27.273 | INFO | NumEvents : 1687592063243 | 20260508 12:47:28.843 | INFO | Analyzing chunk frames = [8220672,8221184) | 20260508 12:47:28.843 | INFO | MemInfo: { | 20260508 12:47:28.843 | INFO | "gpu" : " 17 GB", | 20260508 12:47:28.843 | INFO | "huge" : "", | 20260508 12:47:28.843 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:28.843 | INFO | "malloc" : " 14 GB", | 20260508 12:47:28.843 | INFO | "pinned" : " 29 GB" | 20260508 12:47:28.843 | INFO | } | 20260508 12:47:28.843 | INFO | Perf report for the last 5.13862 seconds: | 20260508 12:47:28.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:28.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:28.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:28.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:28.843 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 12:47:28.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:28.843 | INFO | RT_METRICS : 38.0%, 1.0, 20.0ms, 0 | 20260508 12:47:28.843 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 12:47:28.843 | INFO | BAZWRITER : 66.8%, 2.4, 83.7ms, 0 | 20260508 12:47:28.843 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:47:28.843 | INFO | Uploaded 36GiB to GPU in 2.6457s (13.607GiB/s) | 20260508 12:47:28.843 | INFO | Downloaded 38GiB to GPU in 2.71197s (14.0119GiB/s) | 20260508 12:47:33.982 | INFO | Analyzing chunk frames = [8221184,8221696) | 20260508 12:47:33.982 | INFO | MemInfo: { | 20260508 12:47:33.982 | INFO | "gpu" : " 17 GB", | 20260508 12:47:33.982 | INFO | "huge" : "", | 20260508 12:47:33.982 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:33.982 | INFO | "malloc" : " 14 GB", | 20260508 12:47:33.982 | INFO | "pinned" : " 29 GB" | 20260508 12:47:33.982 | INFO | } | 20260508 12:47:33.982 | INFO | Perf report for the last 5.13909 seconds: | 20260508 12:47:33.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:33.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:47:33.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:47:33.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:33.982 | INFO | Uploaded 36GiB to GPU in 2.46777s (14.5881GiB/s) | 20260508 12:47:33.982 | INFO | Downloaded 34GiB to GPU in 2.46048s (13.8185GiB/s) | 20260508 12:47:39.123 | INFO | Analyzing chunk frames = [8221696,8222208) | 20260508 12:47:39.123 | INFO | MemInfo: { | 20260508 12:47:39.123 | INFO | "gpu" : " 17 GB", | 20260508 12:47:39.123 | INFO | "huge" : "", | 20260508 12:47:39.123 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:39.124 | INFO | "malloc" : " 14 GB", | 20260508 12:47:39.124 | INFO | "pinned" : " 29 GB" | 20260508 12:47:39.124 | INFO | } | 20260508 12:47:39.124 | INFO | Perf report for the last 5.14159 seconds: | 20260508 12:47:39.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:39.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 12:47:39.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:47:39.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:39.124 | INFO | Uploaded 36GiB to GPU in 2.51333s (14.3236GiB/s) | 20260508 12:47:39.124 | INFO | Downloaded 34GiB to GPU in 2.43495s (13.9633GiB/s) | 20260508 12:47:44.269 | INFO | Analyzing chunk frames = [8222208,8222720) | 20260508 12:47:44.269 | INFO | MemInfo: { | 20260508 12:47:44.269 | INFO | "gpu" : " 17 GB", | 20260508 12:47:44.269 | INFO | "huge" : "", | 20260508 12:47:44.269 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:44.269 | INFO | "malloc" : " 14 GB", | 20260508 12:47:44.269 | INFO | "pinned" : " 29 GB" | 20260508 12:47:44.269 | INFO | } | 20260508 12:47:44.269 | INFO | Perf report for the last 5.14586 seconds: | 20260508 12:47:44.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:44.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.270 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 12:47:44.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.270 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:47:44.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:44.270 | INFO | Uploaded 36GiB to GPU in 2.49639s (14.4208GiB/s) | 20260508 12:47:44.270 | INFO | Downloaded 34GiB to GPU in 2.44777s (13.8902GiB/s) | 20260508 12:47:49.406 | INFO | Analyzing chunk frames = [8222720,8223232) | 20260508 12:47:49.406 | INFO | MemInfo: { | 20260508 12:47:49.406 | INFO | "gpu" : " 17 GB", | 20260508 12:47:49.406 | INFO | "huge" : "", | 20260508 12:47:49.406 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:49.406 | INFO | "malloc" : " 14 GB", | 20260508 12:47:49.406 | INFO | "pinned" : " 29 GB" | 20260508 12:47:49.406 | INFO | } | 20260508 12:47:49.407 | INFO | Perf report for the last 5.137 seconds: | 20260508 12:47:49.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:49.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 12:47:49.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 12:47:49.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:49.407 | INFO | Uploaded 36GiB to GPU in 2.46549s (14.6016GiB/s) | 20260508 12:47:49.407 | INFO | Downloaded 34GiB to GPU in 2.51915s (13.4966GiB/s) | 20260508 12:47:54.544 | INFO | Analyzing chunk frames = [8223232,8223744) | 20260508 12:47:54.544 | INFO | MemInfo: { | 20260508 12:47:54.544 | INFO | "gpu" : " 17 GB", | 20260508 12:47:54.544 | INFO | "huge" : "", | 20260508 12:47:54.544 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:54.544 | INFO | "malloc" : " 14 GB", | 20260508 12:47:54.544 | INFO | "pinned" : " 29 GB" | 20260508 12:47:54.545 | INFO | } | 20260508 12:47:54.545 | INFO | Perf report for the last 5.13804 seconds: | 20260508 12:47:54.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:54.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 12:47:54.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 12:47:54.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:54.545 | INFO | Uploaded 36GiB to GPU in 2.44638s (14.7156GiB/s) | 20260508 12:47:54.545 | INFO | Downloaded 34GiB to GPU in 2.43598s (13.9574GiB/s) | 20260508 12:47:59.686 | INFO | Analyzing chunk frames = [8223744,8224256) | 20260508 12:47:59.686 | INFO | MemInfo: { | 20260508 12:47:59.686 | INFO | "gpu" : " 17 GB", | 20260508 12:47:59.686 | INFO | "huge" : "", | 20260508 12:47:59.686 | INFO | "huge_shared" : " 29 GB", | 20260508 12:47:59.686 | INFO | "malloc" : " 14 GB", | 20260508 12:47:59.686 | INFO | "pinned" : " 29 GB" | 20260508 12:47:59.686 | INFO | } | 20260508 12:47:59.686 | INFO | Perf report for the last 5.14111 seconds: | 20260508 12:47:59.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:47:59.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 12:47:59.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:47:59.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:47:59.686 | INFO | Uploaded 36GiB to GPU in 2.35282s (15.3008GiB/s) | 20260508 12:47:59.686 | INFO | Downloaded 34GiB to GPU in 2.39611s (14.1897GiB/s) | 20260508 12:48:04.821 | INFO | Analyzing chunk frames = [8224256,8224768) | 20260508 12:48:04.821 | INFO | MemInfo: { | 20260508 12:48:04.821 | INFO | "gpu" : " 17 GB", | 20260508 12:48:04.821 | INFO | "huge" : "", | 20260508 12:48:04.821 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:04.821 | INFO | "malloc" : " 14 GB", | 20260508 12:48:04.821 | INFO | "pinned" : " 29 GB" | 20260508 12:48:04.821 | INFO | } | 20260508 12:48:04.821 | INFO | Perf report for the last 5.13503 seconds: | 20260508 12:48:04.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:04.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260508 12:48:04.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:48:04.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:04.821 | INFO | Uploaded 36GiB to GPU in 2.36378s (15.2298GiB/s) | 20260508 12:48:04.821 | INFO | Downloaded 34GiB to GPU in 2.42047s (14.0468GiB/s) | 20260508 12:48:04.821 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:48:04.821 | INFO | Total mem usage: 16.7 GiB | 20260508 12:48:04.821 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:48:04.821 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:48:04.821 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:48:04.821 | INFO | High Water Breakdown: | 20260508 12:48:04.821 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:48:04.821 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:48:04.821 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:48:04.821 | INFO | ------------------------------------------ | 20260508 12:48:04.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:48:04.821 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:48:04.821 | INFO | Total mem usage: 26.5 GiB | 20260508 12:48:04.821 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:48:04.821 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:48:04.821 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:48:04.821 | INFO | High Water Breakdown: | 20260508 12:48:04.821 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:48:04.821 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:48:04.821 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:48:04.821 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:48:04.821 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:48:04.821 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:48:04.821 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:48:04.821 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:48:04.821 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:48:04.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:48:04.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:48:04.821 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:48:04.821 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:48:04.821 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:48:04.821 | INFO | ------------------------------------------ | 20260508 12:48:04.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:48:04.821 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:48:04.821 | INFO | Total mem usage: 27.8 GiB | 20260508 12:48:04.821 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:48:04.821 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:48:04.821 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:48:04.821 | INFO | High Water Breakdown: | 20260508 12:48:04.821 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:48:04.821 | INFO | ------------------------------------------ | 20260508 12:48:04.821 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:48:04.821 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:48:04.821 | INFO | Total mem usage: 15.7 GiB | 20260508 12:48:04.821 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:48:04.821 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:48:04.821 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:48:04.821 | INFO | High Water Breakdown: | 20260508 12:48:04.821 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:48:04.821 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:48:04.822 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:48:04.822 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:48:04.822 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:48:04.822 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:48:04.822 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:48:04.822 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:48:04.822 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:48:04.822 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:48:04.822 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:48:04.822 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:48:04.822 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:48:04.822 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:48:04.822 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:48:04.822 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:48:04.822 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:48:04.822 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:48:04.822 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:48:04.822 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:48:04.822 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:48:04.822 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:48:04.822 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:48:04.822 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:48:04.822 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:48:04.822 | INFO | ------------------------------------------ | 20260508 12:48:04.822 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:48:04.822 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:48:04.822 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:48:04.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:48:04.822 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.38e+01 / 5.17e-01 | 20260508 12:48:04.822 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.15e-02 / 4.89e-03 | 20260508 12:48:04.822 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.47e+00, 5.98e-01 / 1.93e-01 | 20260508 12:48:04.822 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.44e+00, 2.10e-01 / 1.27e-01 | 20260508 12:48:04.822 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.53e+01, 3.71e+00 / 9.67e-01 | 20260508 12:48:04.822 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.85e+00 / 1.31e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.32e+00 / 2.47e+00, 4.03e-01 / 2.66e-01 | 20260508 12:48:04.822 | INFO | PerfReport: 2.94%, Metrics -- 9.62e-01 / 1.09e+00, 2.40e-01 / 1.01e-01 | 20260508 12:48:04.822 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.17e-01, 4.69e-02 / 2.87e-02 | 20260508 12:48:04.822 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:48:04.822 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:48:04.822 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:48:04.822 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 24.44%, Upload -- 2.56e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 24.61%, Download -- 2.58e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 31.18%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 2.27%, LockUpload -- 2.39e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 2.43%, LockDownload -- 2.55e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 15.05%, LockCompute -- 1.58e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:48:04.822 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 5.82e-02 / 0.00e+00 | 20260508 12:48:04.918 | INFO | Starting a round of Baz IO | 20260508 12:48:08.329 | INFO | Populating RTMetrics report | 20260508 12:48:08.330 | INFO | RTMetrics report populated | 20260508 12:48:08.330 | INFO | Opening tmp RTMetrics json file | 20260508 12:48:08.336 | INFO | Serializing RTMetrics to json | 20260508 12:48:08.337 | INFO | Serialization done, swapping tmp file now... | 20260508 12:48:08.338 | INFO | RTMetrics done | 20260508 12:48:08.397 | INFO | Finished a round of Baz IO: Wrote 1.401 GiB in 3.478 seconds | 20260508 12:48:08.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.00% | 20260508 12:48:08.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15556% / 0% / 5.96046e-05% | 20260508 12:48:08.397 | INFO | Baz IO overall: | 20260508 12:48:08.397 | INFO | BytesWritten : 3519245275382 | 20260508 12:48:08.397 | INFO | NumEvents : 1688297525616 | 20260508 12:48:09.962 | INFO | Analyzing chunk frames = [8224768,8225280) | 20260508 12:48:09.962 | INFO | MemInfo: { | 20260508 12:48:09.962 | INFO | "gpu" : " 17 GB", | 20260508 12:48:09.962 | INFO | "huge" : "", | 20260508 12:48:09.962 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:09.962 | INFO | "malloc" : " 14 GB", | 20260508 12:48:09.962 | INFO | "pinned" : " 29 GB" | 20260508 12:48:09.962 | INFO | } | 20260508 12:48:09.962 | INFO | Perf report for the last 5.14093 seconds: | 20260508 12:48:09.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:09.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:09.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:09.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:09.962 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:48:09.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:09.962 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:48:09.962 | INFO | PRE_HQ : 65.2%, 1.6, 53.6ms, 0 | 20260508 12:48:09.962 | INFO | BAZWRITER : 66.9%, 2.4, 83.3ms, 0 | 20260508 12:48:09.962 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:48:09.962 | INFO | Uploaded 36GiB to GPU in 2.61463s (13.7687GiB/s) | 20260508 12:48:09.962 | INFO | Downloaded 38GiB to GPU in 2.72136s (13.9636GiB/s) | 20260508 12:48:15.103 | INFO | Analyzing chunk frames = [8225280,8225792) | 20260508 12:48:15.103 | INFO | MemInfo: { | 20260508 12:48:15.103 | INFO | "gpu" : " 17 GB", | 20260508 12:48:15.103 | INFO | "huge" : "", | 20260508 12:48:15.103 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:15.103 | INFO | "malloc" : " 14 GB", | 20260508 12:48:15.103 | INFO | "pinned" : " 29 GB" | 20260508 12:48:15.103 | INFO | } | 20260508 12:48:15.103 | INFO | Perf report for the last 5.14162 seconds: | 20260508 12:48:15.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:15.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.103 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 12:48:15.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.103 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:48:15.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:15.104 | INFO | Uploaded 36GiB to GPU in 2.44244s (14.7393GiB/s) | 20260508 12:48:15.104 | INFO | Downloaded 34GiB to GPU in 2.47237s (13.752GiB/s) | 20260508 12:48:20.236 | INFO | Analyzing chunk frames = [8225792,8226304) | 20260508 12:48:20.236 | INFO | MemInfo: { | 20260508 12:48:20.236 | INFO | "gpu" : " 17 GB", | 20260508 12:48:20.236 | INFO | "huge" : "", | 20260508 12:48:20.236 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:20.236 | INFO | "malloc" : " 14 GB", | 20260508 12:48:20.236 | INFO | "pinned" : " 29 GB" | 20260508 12:48:20.236 | INFO | } | 20260508 12:48:20.236 | INFO | Perf report for the last 5.13288 seconds: | 20260508 12:48:20.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:20.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 12:48:20.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:48:20.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:20.236 | INFO | Uploaded 36GiB to GPU in 2.48776s (14.4709GiB/s) | 20260508 12:48:20.237 | INFO | Downloaded 34GiB to GPU in 2.46104s (13.8153GiB/s) | 20260508 12:48:25.384 | INFO | Analyzing chunk frames = [8226304,8226816) | 20260508 12:48:25.384 | INFO | MemInfo: { | 20260508 12:48:25.384 | INFO | "gpu" : " 17 GB", | 20260508 12:48:25.384 | INFO | "huge" : "", | 20260508 12:48:25.384 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:25.384 | INFO | "malloc" : " 14 GB", | 20260508 12:48:25.384 | INFO | "pinned" : " 29 GB" | 20260508 12:48:25.384 | INFO | } | 20260508 12:48:25.384 | INFO | Perf report for the last 5.14784 seconds: | 20260508 12:48:25.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:25.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 12:48:25.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:48:25.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:25.384 | INFO | Uploaded 36GiB to GPU in 2.46737s (14.5904GiB/s) | 20260508 12:48:25.384 | INFO | Downloaded 34GiB to GPU in 2.45909s (13.8262GiB/s) | 20260508 12:48:30.524 | INFO | Analyzing chunk frames = [8226816,8227328) | 20260508 12:48:30.524 | INFO | MemInfo: { | 20260508 12:48:30.524 | INFO | "gpu" : " 17 GB", | 20260508 12:48:30.524 | INFO | "huge" : "", | 20260508 12:48:30.524 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:30.524 | INFO | "malloc" : " 14 GB", | 20260508 12:48:30.524 | INFO | "pinned" : " 29 GB" | 20260508 12:48:30.524 | INFO | } | 20260508 12:48:30.524 | INFO | Perf report for the last 5.14019 seconds: | 20260508 12:48:30.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:30.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 12:48:30.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:48:30.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:30.524 | INFO | Uploaded 36GiB to GPU in 2.41134s (14.9294GiB/s) | 20260508 12:48:30.525 | INFO | Downloaded 34GiB to GPU in 2.50992s (13.5463GiB/s) | 20260508 12:48:35.659 | INFO | Analyzing chunk frames = [8227328,8227840) | 20260508 12:48:35.659 | INFO | MemInfo: { | 20260508 12:48:35.659 | INFO | "gpu" : " 17 GB", | 20260508 12:48:35.659 | INFO | "huge" : "", | 20260508 12:48:35.659 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:35.659 | INFO | "malloc" : " 14 GB", | 20260508 12:48:35.659 | INFO | "pinned" : " 29 GB" | 20260508 12:48:35.659 | INFO | } | 20260508 12:48:35.659 | INFO | Perf report for the last 5.13445 seconds: | 20260508 12:48:35.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:35.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 12:48:35.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:48:35.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:35.659 | INFO | Uploaded 36GiB to GPU in 2.2942s (15.6917GiB/s) | 20260508 12:48:35.659 | INFO | Downloaded 34GiB to GPU in 2.44011s (13.9338GiB/s) | 20260508 12:48:40.794 | INFO | Analyzing chunk frames = [8227840,8228352) | 20260508 12:48:40.794 | INFO | MemInfo: { | 20260508 12:48:40.794 | INFO | "gpu" : " 17 GB", | 20260508 12:48:40.794 | INFO | "huge" : "", | 20260508 12:48:40.794 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:40.794 | INFO | "malloc" : " 14 GB", | 20260508 12:48:40.794 | INFO | "pinned" : " 29 GB" | 20260508 12:48:40.794 | INFO | } | 20260508 12:48:40.794 | INFO | Perf report for the last 5.13547 seconds: | 20260508 12:48:40.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:40.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | ANALYSIS : 65.6%, 3.0, 102.2ms, 0 | 20260508 12:48:40.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:48:40.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:40.794 | INFO | Uploaded 36GiB to GPU in 2.439s (14.7601GiB/s) | 20260508 12:48:40.794 | INFO | Downloaded 34GiB to GPU in 2.46832s (13.7746GiB/s) | 20260508 12:48:45.937 | INFO | Analyzing chunk frames = [8228352,8228864) | 20260508 12:48:45.937 | INFO | MemInfo: { | 20260508 12:48:45.937 | INFO | "gpu" : " 17 GB", | 20260508 12:48:45.937 | INFO | "huge" : "", | 20260508 12:48:45.937 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:45.937 | INFO | "malloc" : " 14 GB", | 20260508 12:48:45.937 | INFO | "pinned" : " 29 GB" | 20260508 12:48:45.937 | INFO | } | 20260508 12:48:45.937 | INFO | Perf report for the last 5.14252 seconds: | 20260508 12:48:45.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:45.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:48:45.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:48:45.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:45.937 | INFO | Uploaded 36GiB to GPU in 2.47213s (14.5623GiB/s) | 20260508 12:48:45.937 | INFO | Downloaded 34GiB to GPU in 2.48527s (13.6806GiB/s) | 20260508 12:48:46.034 | INFO | Starting a round of Baz IO | 20260508 12:48:49.431 | INFO | Populating RTMetrics report | 20260508 12:48:49.431 | INFO | RTMetrics report populated | 20260508 12:48:49.431 | INFO | Opening tmp RTMetrics json file | 20260508 12:48:49.441 | INFO | Serializing RTMetrics to json | 20260508 12:48:49.442 | INFO | Serialization done, swapping tmp file now... | 20260508 12:48:49.442 | INFO | RTMetrics done | 20260508 12:48:49.503 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.469 seconds | 20260508 12:48:49.503 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:48:49.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155834% / 0% / 2.78155e-05% | 20260508 12:48:49.503 | INFO | Baz IO overall: | 20260508 12:48:49.503 | INFO | BytesWritten : 3520750579865 | 20260508 12:48:49.503 | INFO | NumEvents : 1689003523431 | 20260508 12:48:51.085 | INFO | Analyzing chunk frames = [8228864,8229376) | 20260508 12:48:51.085 | INFO | MemInfo: { | 20260508 12:48:51.085 | INFO | "gpu" : " 17 GB", | 20260508 12:48:51.085 | INFO | "huge" : "", | 20260508 12:48:51.085 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:51.085 | INFO | "malloc" : " 14 GB", | 20260508 12:48:51.085 | INFO | "pinned" : " 29 GB" | 20260508 12:48:51.085 | INFO | } | 20260508 12:48:51.085 | INFO | Perf report for the last 5.14872 seconds: | 20260508 12:48:51.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:51.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:51.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:51.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:51.085 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 12:48:51.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:51.086 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:48:51.086 | INFO | PRE_HQ : 65.1%, 1.6, 53.8ms, 0 | 20260508 12:48:51.086 | INFO | BAZWRITER : 66.7%, 2.4, 83.6ms, 0 | 20260508 12:48:51.086 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:48:51.086 | INFO | Uploaded 36GiB to GPU in 2.60472s (13.8211GiB/s) | 20260508 12:48:51.086 | INFO | Downloaded 38GiB to GPU in 2.7224s (13.9583GiB/s) | 20260508 12:48:56.216 | INFO | Analyzing chunk frames = [8229376,8229888) | 20260508 12:48:56.216 | INFO | MemInfo: { | 20260508 12:48:56.216 | INFO | "gpu" : " 17 GB", | 20260508 12:48:56.216 | INFO | "huge" : "", | 20260508 12:48:56.216 | INFO | "huge_shared" : " 29 GB", | 20260508 12:48:56.216 | INFO | "malloc" : " 14 GB", | 20260508 12:48:56.216 | INFO | "pinned" : " 29 GB" | 20260508 12:48:56.216 | INFO | } | 20260508 12:48:56.216 | INFO | Perf report for the last 5.13078 seconds: | 20260508 12:48:56.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:48:56.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.216 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 12:48:56.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.216 | INFO | PRE_HQ : 22.5%, 1.0, 11.6ms, 0 | 20260508 12:48:56.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:48:56.217 | INFO | Uploaded 36GiB to GPU in 2.46878s (14.5821GiB/s) | 20260508 12:48:56.217 | INFO | Downloaded 34GiB to GPU in 2.49577s (13.623GiB/s) | 20260508 12:49:01.354 | INFO | Analyzing chunk frames = [8229888,8230400) | 20260508 12:49:01.354 | INFO | MemInfo: { | 20260508 12:49:01.354 | INFO | "gpu" : " 17 GB", | 20260508 12:49:01.354 | INFO | "huge" : "", | 20260508 12:49:01.354 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:01.354 | INFO | "malloc" : " 14 GB", | 20260508 12:49:01.354 | INFO | "pinned" : " 29 GB" | 20260508 12:49:01.354 | INFO | } | 20260508 12:49:01.354 | INFO | Perf report for the last 5.13802 seconds: | 20260508 12:49:01.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:01.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.354 | INFO | ANALYSIS : 66.3%, 3.0, 102.9ms, 0 | 20260508 12:49:01.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.355 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260508 12:49:01.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:01.355 | INFO | Uploaded 36GiB to GPU in 2.49436s (14.4326GiB/s) | 20260508 12:49:01.355 | INFO | Downloaded 34GiB to GPU in 2.47672s (13.7278GiB/s) | 20260508 12:49:06.494 | INFO | Analyzing chunk frames = [8230400,8230912) | 20260508 12:49:06.494 | INFO | MemInfo: { | 20260508 12:49:06.494 | INFO | "gpu" : " 17 GB", | 20260508 12:49:06.494 | INFO | "huge" : "", | 20260508 12:49:06.495 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:06.495 | INFO | "malloc" : " 14 GB", | 20260508 12:49:06.495 | INFO | "pinned" : " 29 GB" | 20260508 12:49:06.495 | INFO | } | 20260508 12:49:06.495 | INFO | Perf report for the last 5.14035 seconds: | 20260508 12:49:06.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:06.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | ANALYSIS : 65.2%, 3.0, 101.2ms, 0 | 20260508 12:49:06.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:49:06.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:06.495 | INFO | Uploaded 36GiB to GPU in 2.33s (15.4506GiB/s) | 20260508 12:49:06.495 | INFO | Downloaded 34GiB to GPU in 2.4776s (13.7229GiB/s) | 20260508 12:49:11.633 | INFO | Analyzing chunk frames = [8230912,8231424) | 20260508 12:49:11.633 | INFO | MemInfo: { | 20260508 12:49:11.633 | INFO | "gpu" : " 17 GB", | 20260508 12:49:11.633 | INFO | "huge" : "", | 20260508 12:49:11.633 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:11.633 | INFO | "malloc" : " 14 GB", | 20260508 12:49:11.633 | INFO | "pinned" : " 29 GB" | 20260508 12:49:11.633 | INFO | } | 20260508 12:49:11.633 | INFO | Perf report for the last 5.13819 seconds: | 20260508 12:49:11.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:11.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:49:11.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 12:49:11.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:11.633 | INFO | Uploaded 36GiB to GPU in 2.29878s (15.6605GiB/s) | 20260508 12:49:11.633 | INFO | Downloaded 34GiB to GPU in 2.51137s (13.5384GiB/s) | 20260508 12:49:16.770 | INFO | Analyzing chunk frames = [8231424,8231936) | 20260508 12:49:16.770 | INFO | MemInfo: { | 20260508 12:49:16.770 | INFO | "gpu" : " 17 GB", | 20260508 12:49:16.770 | INFO | "huge" : "", | 20260508 12:49:16.770 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:16.770 | INFO | "malloc" : " 14 GB", | 20260508 12:49:16.770 | INFO | "pinned" : " 29 GB" | 20260508 12:49:16.770 | INFO | } | 20260508 12:49:16.770 | INFO | Perf report for the last 5.13742 seconds: | 20260508 12:49:16.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:16.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.770 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 12:49:16.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.770 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:49:16.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:16.771 | INFO | Uploaded 36GiB to GPU in 2.34655s (15.3417GiB/s) | 20260508 12:49:16.771 | INFO | Downloaded 34GiB to GPU in 2.45396s (13.8552GiB/s) | 20260508 12:49:21.911 | INFO | Analyzing chunk frames = [8231936,8232448) | 20260508 12:49:21.911 | INFO | MemInfo: { | 20260508 12:49:21.911 | INFO | "gpu" : " 17 GB", | 20260508 12:49:21.911 | INFO | "huge" : "", | 20260508 12:49:21.911 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:21.911 | INFO | "malloc" : " 14 GB", | 20260508 12:49:21.911 | INFO | "pinned" : " 29 GB" | 20260508 12:49:21.911 | INFO | } | 20260508 12:49:21.911 | INFO | Perf report for the last 5.14082 seconds: | 20260508 12:49:21.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:21.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 12:49:21.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:49:21.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:21.911 | INFO | Uploaded 36GiB to GPU in 2.41645s (14.8979GiB/s) | 20260508 12:49:21.911 | INFO | Downloaded 34GiB to GPU in 2.45735s (13.836GiB/s) | 20260508 12:49:27.049 | INFO | Analyzing chunk frames = [8232448,8232960) | 20260508 12:49:27.049 | INFO | MemInfo: { | 20260508 12:49:27.049 | INFO | "gpu" : " 17 GB", | 20260508 12:49:27.049 | INFO | "huge" : "", | 20260508 12:49:27.049 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:27.049 | INFO | "malloc" : " 14 GB", | 20260508 12:49:27.049 | INFO | "pinned" : " 29 GB" | 20260508 12:49:27.049 | INFO | } | 20260508 12:49:27.049 | INFO | Perf report for the last 5.13832 seconds: | 20260508 12:49:27.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:27.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.049 | INFO | ANALYSIS : 65.6%, 3.0, 102.5ms, 0 | 20260508 12:49:27.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.050 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 12:49:27.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:27.050 | INFO | Uploaded 36GiB to GPU in 2.41966s (14.8781GiB/s) | 20260508 12:49:27.050 | INFO | Downloaded 34GiB to GPU in 2.47084s (13.7605GiB/s) | 20260508 12:49:27.050 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:49:27.050 | INFO | Total mem usage: 16.7 GiB | 20260508 12:49:27.050 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:49:27.050 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:49:27.050 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:49:27.050 | INFO | High Water Breakdown: | 20260508 12:49:27.050 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:49:27.050 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:49:27.050 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:49:27.050 | INFO | ------------------------------------------ | 20260508 12:49:27.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:49:27.050 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:49:27.050 | INFO | Total mem usage: 26.5 GiB | 20260508 12:49:27.050 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:49:27.050 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:49:27.050 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:49:27.050 | INFO | High Water Breakdown: | 20260508 12:49:27.050 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:49:27.050 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:49:27.050 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:49:27.050 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:49:27.050 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:49:27.050 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:49:27.050 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:49:27.050 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:49:27.050 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:49:27.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:49:27.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:49:27.050 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:49:27.050 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:49:27.050 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:49:27.050 | INFO | ------------------------------------------ | 20260508 12:49:27.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:49:27.050 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:49:27.050 | INFO | Total mem usage: 27.8 GiB | 20260508 12:49:27.050 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:49:27.050 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:49:27.050 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:49:27.050 | INFO | High Water Breakdown: | 20260508 12:49:27.050 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:49:27.050 | INFO | ------------------------------------------ | 20260508 12:49:27.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:49:27.050 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:49:27.050 | INFO | Total mem usage: 15.7 GiB | 20260508 12:49:27.050 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:49:27.050 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:49:27.050 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:49:27.050 | INFO | High Water Breakdown: | 20260508 12:49:27.050 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:49:27.050 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:49:27.050 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:49:27.050 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:49:27.050 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:49:27.050 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:49:27.050 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:49:27.050 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:49:27.050 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:49:27.050 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:49:27.050 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:49:27.050 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:49:27.050 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:49:27.050 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:49:27.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:49:27.050 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:49:27.050 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:49:27.050 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:49:27.050 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:49:27.050 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:49:27.050 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:49:27.050 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:49:27.050 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:49:27.050 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:49:27.050 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:49:27.050 | INFO | ------------------------------------------ | 20260508 12:49:27.050 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:49:27.050 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:49:27.050 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:49:27.050 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:49:27.050 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.38e+01 / 6.44e-01 | 20260508 12:49:27.050 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.21e-02 / 5.41e-03 | 20260508 12:49:27.050 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.47e+00, 5.99e-01 / 1.91e-01 | 20260508 12:49:27.050 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.43e+00, 2.10e-01 / 1.32e-01 | 20260508 12:49:27.050 | INFO | PerfReport: 16.03%, DME -- 4.19e+01 / 4.54e+01, 3.67e+00 / 1.15e+00 | 20260508 12:49:27.050 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.80e+00 / 1.45e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.32e+00 / 2.44e+00, 4.06e-01 / 2.94e-01 | 20260508 12:49:27.051 | INFO | PerfReport: 2.96%, Metrics -- 9.65e-01 / 1.08e+00, 2.42e-01 / 1.24e-01 | 20260508 12:49:27.051 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.19e-01, 4.66e-02 / 2.75e-02 | 20260508 12:49:27.051 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:49:27.051 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:49:27.051 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:49:27.051 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 24.22%, Upload -- 2.54e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 24.89%, Download -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 31.14%, Compute -- 3.26e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 1.98%, LockUpload -- 2.08e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 2.69%, LockDownload -- 2.82e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 15.06%, LockCompute -- 1.58e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:49:27.051 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260508 12:49:27.147 | INFO | Starting a round of Baz IO | 20260508 12:49:30.556 | INFO | Populating RTMetrics report | 20260508 12:49:30.556 | INFO | RTMetrics report populated | 20260508 12:49:30.556 | INFO | Opening tmp RTMetrics json file | 20260508 12:49:30.562 | INFO | Serializing RTMetrics to json | 20260508 12:49:30.563 | INFO | Serialization done, swapping tmp file now... | 20260508 12:49:30.564 | INFO | RTMetrics done | 20260508 12:49:30.621 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.474 seconds | 20260508 12:49:30.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:49:30.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156081% / 3.97364e-06% / 7.15256e-05% | 20260508 12:49:30.621 | INFO | Baz IO overall: | 20260508 12:49:30.621 | INFO | BytesWritten : 3522255553235 | 20260508 12:49:30.621 | INFO | NumEvents : 1689709387029 | 20260508 12:49:32.194 | INFO | Analyzing chunk frames = [8232960,8233472) | 20260508 12:49:32.194 | INFO | MemInfo: { | 20260508 12:49:32.194 | INFO | "gpu" : " 17 GB", | 20260508 12:49:32.194 | INFO | "huge" : "", | 20260508 12:49:32.194 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:32.194 | INFO | "malloc" : " 14 GB", | 20260508 12:49:32.194 | INFO | "pinned" : " 29 GB" | 20260508 12:49:32.194 | INFO | } | 20260508 12:49:32.194 | INFO | Perf report for the last 5.14437 seconds: | 20260508 12:49:32.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:32.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:32.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:32.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:32.194 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 12:49:32.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:32.194 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:49:32.194 | INFO | PRE_HQ : 65.3%, 1.6, 54.0ms, 0 | 20260508 12:49:32.194 | INFO | BAZWRITER : 66.8%, 2.4, 84.0ms, 0 | 20260508 12:49:32.194 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:49:32.194 | INFO | Uploaded 36GiB to GPU in 2.59635s (13.8656GiB/s) | 20260508 12:49:32.194 | INFO | Downloaded 38GiB to GPU in 2.71662s (13.988GiB/s) | 20260508 12:49:37.332 | INFO | Analyzing chunk frames = [8233472,8233984) | 20260508 12:49:37.332 | INFO | MemInfo: { | 20260508 12:49:37.332 | INFO | "gpu" : " 17 GB", | 20260508 12:49:37.332 | INFO | "huge" : "", | 20260508 12:49:37.332 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:37.332 | INFO | "malloc" : " 14 GB", | 20260508 12:49:37.332 | INFO | "pinned" : " 29 GB" | 20260508 12:49:37.332 | INFO | } | 20260508 12:49:37.332 | INFO | Perf report for the last 5.13786 seconds: | 20260508 12:49:37.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:37.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:49:37.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:49:37.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:37.332 | INFO | Uploaded 36GiB to GPU in 2.444s (14.73GiB/s) | 20260508 12:49:37.332 | INFO | Downloaded 34GiB to GPU in 2.48511s (13.6815GiB/s) | 20260508 12:49:42.469 | INFO | Analyzing chunk frames = [8233984,8234496) | 20260508 12:49:42.469 | INFO | MemInfo: { | 20260508 12:49:42.469 | INFO | "gpu" : " 17 GB", | 20260508 12:49:42.469 | INFO | "huge" : "", | 20260508 12:49:42.469 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:42.469 | INFO | "malloc" : " 14 GB", | 20260508 12:49:42.469 | INFO | "pinned" : " 29 GB" | 20260508 12:49:42.469 | INFO | } | 20260508 12:49:42.469 | INFO | Perf report for the last 5.13705 seconds: | 20260508 12:49:42.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:42.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:49:42.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:49:42.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:42.469 | INFO | Uploaded 36GiB to GPU in 2.46617s (14.5975GiB/s) | 20260508 12:49:42.469 | INFO | Downloaded 34GiB to GPU in 2.47954s (13.7122GiB/s) | 20260508 12:49:47.604 | INFO | Analyzing chunk frames = [8234496,8235008) | 20260508 12:49:47.604 | INFO | MemInfo: { | 20260508 12:49:47.604 | INFO | "gpu" : " 17 GB", | 20260508 12:49:47.605 | INFO | "huge" : "", | 20260508 12:49:47.605 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:47.605 | INFO | "malloc" : " 14 GB", | 20260508 12:49:47.605 | INFO | "pinned" : " 29 GB" | 20260508 12:49:47.605 | INFO | } | 20260508 12:49:47.605 | INFO | Perf report for the last 5.13581 seconds: | 20260508 12:49:47.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:47.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 12:49:47.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:49:47.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:47.605 | INFO | Uploaded 36GiB to GPU in 2.49579s (14.4243GiB/s) | 20260508 12:49:47.605 | INFO | Downloaded 34GiB to GPU in 2.46108s (13.8151GiB/s) | 20260508 12:49:52.745 | INFO | Analyzing chunk frames = [8235008,8235520) | 20260508 12:49:52.745 | INFO | MemInfo: { | 20260508 12:49:52.745 | INFO | "gpu" : " 17 GB", | 20260508 12:49:52.746 | INFO | "huge" : "", | 20260508 12:49:52.746 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:52.746 | INFO | "malloc" : " 14 GB", | 20260508 12:49:52.746 | INFO | "pinned" : " 29 GB" | 20260508 12:49:52.746 | INFO | } | 20260508 12:49:52.746 | INFO | Perf report for the last 5.14098 seconds: | 20260508 12:49:52.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:52.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 12:49:52.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:49:52.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:52.746 | INFO | Uploaded 36GiB to GPU in 2.36696s (15.2094GiB/s) | 20260508 12:49:52.746 | INFO | Downloaded 34GiB to GPU in 2.51239s (13.5329GiB/s) | 20260508 12:49:57.886 | INFO | Analyzing chunk frames = [8235520,8236032) | 20260508 12:49:57.886 | INFO | MemInfo: { | 20260508 12:49:57.886 | INFO | "gpu" : " 17 GB", | 20260508 12:49:57.886 | INFO | "huge" : "", | 20260508 12:49:57.886 | INFO | "huge_shared" : " 29 GB", | 20260508 12:49:57.886 | INFO | "malloc" : " 14 GB", | 20260508 12:49:57.886 | INFO | "pinned" : " 29 GB" | 20260508 12:49:57.886 | INFO | } | 20260508 12:49:57.886 | INFO | Perf report for the last 5.14043 seconds: | 20260508 12:49:57.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:49:57.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 12:49:57.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:49:57.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:49:57.886 | INFO | Uploaded 36GiB to GPU in 2.38642s (15.0854GiB/s) | 20260508 12:49:57.886 | INFO | Downloaded 34GiB to GPU in 2.44314s (13.9165GiB/s) | 20260508 12:50:03.034 | INFO | Analyzing chunk frames = [8236032,8236544) | 20260508 12:50:03.034 | INFO | MemInfo: { | 20260508 12:50:03.034 | INFO | "gpu" : " 17 GB", | 20260508 12:50:03.034 | INFO | "huge" : "", | 20260508 12:50:03.034 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:03.034 | INFO | "malloc" : " 14 GB", | 20260508 12:50:03.034 | INFO | "pinned" : " 29 GB" | 20260508 12:50:03.034 | INFO | } | 20260508 12:50:03.034 | INFO | Perf report for the last 5.14826 seconds: | 20260508 12:50:03.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:03.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.034 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 12:50:03.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.034 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:50:03.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:03.035 | INFO | Uploaded 36GiB to GPU in 2.43017s (14.8138GiB/s) | 20260508 12:50:03.035 | INFO | Downloaded 34GiB to GPU in 2.44844s (13.8864GiB/s) | 20260508 12:50:08.170 | INFO | Analyzing chunk frames = [8236544,8237056) | 20260508 12:50:08.170 | INFO | MemInfo: { | 20260508 12:50:08.170 | INFO | "gpu" : " 17 GB", | 20260508 12:50:08.170 | INFO | "huge" : "", | 20260508 12:50:08.170 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:08.170 | INFO | "malloc" : " 14 GB", | 20260508 12:50:08.170 | INFO | "pinned" : " 29 GB" | 20260508 12:50:08.170 | INFO | } | 20260508 12:50:08.170 | INFO | Perf report for the last 5.13598 seconds: | 20260508 12:50:08.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:08.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.170 | INFO | ANALYSIS : 64.4%, 3.0, 100.6ms, 0 | 20260508 12:50:08.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.170 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:50:08.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:08.171 | INFO | Uploaded 36GiB to GPU in 2.29712s (15.6718GiB/s) | 20260508 12:50:08.171 | INFO | Downloaded 34GiB to GPU in 2.43077s (13.9874GiB/s) | 20260508 12:50:08.281 | INFO | Starting a round of Baz IO | 20260508 12:50:11.679 | INFO | Populating RTMetrics report | 20260508 12:50:11.679 | INFO | RTMetrics report populated | 20260508 12:50:11.679 | INFO | Opening tmp RTMetrics json file | 20260508 12:50:11.680 | INFO | Serializing RTMetrics to json | 20260508 12:50:11.681 | INFO | Serialization done, swapping tmp file now... | 20260508 12:50:11.681 | INFO | RTMetrics done | 20260508 12:50:11.738 | INFO | Finished a round of Baz IO: Wrote 1.401 GiB in 3.457 seconds | 20260508 12:50:11.738 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:50:11.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155071% / 0% / 6.75519e-05% | 20260508 12:50:11.739 | INFO | Baz IO overall: | 20260508 12:50:11.739 | INFO | BytesWritten : 3523759849323 | 20260508 12:50:11.739 | INFO | NumEvents : 1690414618487 | 20260508 12:50:13.299 | INFO | Analyzing chunk frames = [8237056,8237568) | 20260508 12:50:13.299 | INFO | MemInfo: { | 20260508 12:50:13.299 | INFO | "gpu" : " 17 GB", | 20260508 12:50:13.299 | INFO | "huge" : "", | 20260508 12:50:13.299 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:13.299 | INFO | "malloc" : " 14 GB", | 20260508 12:50:13.299 | INFO | "pinned" : " 29 GB" | 20260508 12:50:13.299 | INFO | } | 20260508 12:50:13.299 | INFO | Perf report for the last 5.1288 seconds: | 20260508 12:50:13.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:13.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:13.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:13.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:13.299 | INFO | ANALYSIS : 68.0%, 3.0, 105.5ms, 0 | 20260508 12:50:13.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:13.299 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260508 12:50:13.299 | INFO | PRE_HQ : 66.5%, 1.7, 59.6ms, 0 | 20260508 12:50:13.299 | INFO | BAZWRITER : 66.6%, 2.6, 88.9ms, 0 | 20260508 12:50:13.299 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 12:50:13.299 | INFO | Uploaded 36GiB to GPU in 2.5432s (14.1554GiB/s) | 20260508 12:50:13.299 | INFO | Downloaded 38GiB to GPU in 2.69303s (14.1105GiB/s) | 20260508 12:50:18.446 | INFO | Analyzing chunk frames = [8237568,8238080) | 20260508 12:50:18.446 | INFO | MemInfo: { | 20260508 12:50:18.446 | INFO | "gpu" : " 17 GB", | 20260508 12:50:18.446 | INFO | "huge" : "", | 20260508 12:50:18.446 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:18.446 | INFO | "malloc" : " 14 GB", | 20260508 12:50:18.446 | INFO | "pinned" : " 29 GB" | 20260508 12:50:18.447 | INFO | } | 20260508 12:50:18.447 | INFO | Perf report for the last 5.14741 seconds: | 20260508 12:50:18.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:18.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:50:18.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:50:18.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:18.447 | INFO | Uploaded 36GiB to GPU in 2.46393s (14.6108GiB/s) | 20260508 12:50:18.447 | INFO | Downloaded 34GiB to GPU in 2.47687s (13.727GiB/s) | 20260508 12:50:23.585 | INFO | Analyzing chunk frames = [8238080,8238592) | 20260508 12:50:23.585 | INFO | MemInfo: { | 20260508 12:50:23.585 | INFO | "gpu" : " 17 GB", | 20260508 12:50:23.585 | INFO | "huge" : "", | 20260508 12:50:23.585 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:23.585 | INFO | "malloc" : " 14 GB", | 20260508 12:50:23.585 | INFO | "pinned" : " 29 GB" | 20260508 12:50:23.585 | INFO | } | 20260508 12:50:23.585 | INFO | Perf report for the last 5.13893 seconds: | 20260508 12:50:23.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:23.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 12:50:23.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:50:23.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:23.586 | INFO | Uploaded 36GiB to GPU in 2.47203s (14.5629GiB/s) | 20260508 12:50:23.586 | INFO | Downloaded 34GiB to GPU in 2.45998s (13.8213GiB/s) | 20260508 12:50:28.727 | INFO | Analyzing chunk frames = [8238592,8239104) | 20260508 12:50:28.727 | INFO | MemInfo: { | 20260508 12:50:28.727 | INFO | "gpu" : " 17 GB", | 20260508 12:50:28.727 | INFO | "huge" : "", | 20260508 12:50:28.727 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:28.727 | INFO | "malloc" : " 14 GB", | 20260508 12:50:28.727 | INFO | "pinned" : " 29 GB" | 20260508 12:50:28.727 | INFO | } | 20260508 12:50:28.727 | INFO | Perf report for the last 5.14128 seconds: | 20260508 12:50:28.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:28.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 12:50:28.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:50:28.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:28.727 | INFO | Uploaded 36GiB to GPU in 2.46733s (14.5907GiB/s) | 20260508 12:50:28.727 | INFO | Downloaded 34GiB to GPU in 2.45964s (13.8231GiB/s) | 20260508 12:50:33.866 | INFO | Analyzing chunk frames = [8239104,8239616) | 20260508 12:50:33.866 | INFO | MemInfo: { | 20260508 12:50:33.866 | INFO | "gpu" : " 17 GB", | 20260508 12:50:33.866 | INFO | "huge" : "", | 20260508 12:50:33.866 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:33.866 | INFO | "malloc" : " 14 GB", | 20260508 12:50:33.866 | INFO | "pinned" : " 29 GB" | 20260508 12:50:33.866 | INFO | } | 20260508 12:50:33.866 | INFO | Perf report for the last 5.13927 seconds: | 20260508 12:50:33.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:33.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 12:50:33.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:50:33.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:33.866 | INFO | Uploaded 36GiB to GPU in 2.27608s (15.8167GiB/s) | 20260508 12:50:33.866 | INFO | Downloaded 34GiB to GPU in 2.49611s (13.6212GiB/s) | 20260508 12:50:39.002 | INFO | Analyzing chunk frames = [8239616,8240128) | 20260508 12:50:39.002 | INFO | MemInfo: { | 20260508 12:50:39.002 | INFO | "gpu" : " 17 GB", | 20260508 12:50:39.002 | INFO | "huge" : "", | 20260508 12:50:39.002 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:39.002 | INFO | "malloc" : " 14 GB", | 20260508 12:50:39.002 | INFO | "pinned" : " 29 GB" | 20260508 12:50:39.002 | INFO | } | 20260508 12:50:39.002 | INFO | Perf report for the last 5.13619 seconds: | 20260508 12:50:39.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:39.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.002 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 12:50:39.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.002 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:50:39.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:39.003 | INFO | Uploaded 36GiB to GPU in 2.32894s (15.4577GiB/s) | 20260508 12:50:39.003 | INFO | Downloaded 34GiB to GPU in 2.42882s (13.9986GiB/s) | 20260508 12:50:44.140 | INFO | Analyzing chunk frames = [8240128,8240640) | 20260508 12:50:44.140 | INFO | MemInfo: { | 20260508 12:50:44.140 | INFO | "gpu" : " 17 GB", | 20260508 12:50:44.140 | INFO | "huge" : "", | 20260508 12:50:44.140 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:44.140 | INFO | "malloc" : " 14 GB", | 20260508 12:50:44.140 | INFO | "pinned" : " 29 GB" | 20260508 12:50:44.140 | INFO | } | 20260508 12:50:44.140 | INFO | Perf report for the last 5.13758 seconds: | 20260508 12:50:44.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:44.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 12:50:44.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 12:50:44.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:44.140 | INFO | Uploaded 36GiB to GPU in 2.33129s (15.4421GiB/s) | 20260508 12:50:44.140 | INFO | Downloaded 34GiB to GPU in 2.43313s (13.9738GiB/s) | 20260508 12:50:49.279 | INFO | Analyzing chunk frames = [8240640,8241152) | 20260508 12:50:49.279 | INFO | MemInfo: { | 20260508 12:50:49.279 | INFO | "gpu" : " 17 GB", | 20260508 12:50:49.279 | INFO | "huge" : "", | 20260508 12:50:49.279 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:49.279 | INFO | "malloc" : " 14 GB", | 20260508 12:50:49.279 | INFO | "pinned" : " 29 GB" | 20260508 12:50:49.279 | INFO | } | 20260508 12:50:49.279 | INFO | Perf report for the last 5.13911 seconds: | 20260508 12:50:49.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:49.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 12:50:49.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:50:49.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:49.279 | INFO | Uploaded 36GiB to GPU in 2.48557s (14.4836GiB/s) | 20260508 12:50:49.279 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:50:49.279 | INFO | Downloaded 34GiB to GPU in 2.4722s (13.7529GiB/s) | 20260508 12:50:49.279 | INFO | Total mem usage: 16.7 GiB | 20260508 12:50:49.279 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:50:49.279 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 12:50:49.279 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:50:49.279 | INFO | High Water Breakdown: | 20260508 12:50:49.279 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:50:49.279 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:50:49.279 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:50:49.279 | INFO | ------------------------------------------ | 20260508 12:50:49.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:50:49.279 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:50:49.279 | INFO | Total mem usage: 26.5 GiB | 20260508 12:50:49.279 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:50:49.279 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:50:49.279 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:50:49.279 | INFO | High Water Breakdown: | 20260508 12:50:49.279 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:50:49.279 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:50:49.279 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:50:49.279 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:50:49.279 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:50:49.279 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:50:49.279 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:50:49.280 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:50:49.280 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:50:49.280 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:50:49.280 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:50:49.280 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:50:49.280 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:50:49.280 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:50:49.280 | INFO | ------------------------------------------ | 20260508 12:50:49.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:50:49.280 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:50:49.280 | INFO | Total mem usage: 27.8 GiB | 20260508 12:50:49.280 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:50:49.280 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:50:49.280 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:50:49.280 | INFO | High Water Breakdown: | 20260508 12:50:49.280 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:50:49.280 | INFO | ------------------------------------------ | 20260508 12:50:49.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:50:49.280 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:50:49.280 | INFO | Total mem usage: 15.7 GiB | 20260508 12:50:49.280 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:50:49.280 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:50:49.280 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:50:49.280 | INFO | High Water Breakdown: | 20260508 12:50:49.280 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:50:49.280 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:50:49.280 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:50:49.280 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:50:49.280 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:50:49.280 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:50:49.280 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:50:49.280 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:50:49.280 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:50:49.280 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:50:49.280 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:50:49.280 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:50:49.280 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:50:49.280 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:50:49.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:50:49.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:50:49.280 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:50:49.280 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:50:49.280 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:50:49.280 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:50:49.280 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:50:49.280 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:50:49.280 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:50:49.280 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:50:49.280 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:50:49.280 | INFO | ------------------------------------------ | 20260508 12:50:49.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:50:49.280 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:50:49.280 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:50:49.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:50:49.280 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 6.14e-01 | 20260508 12:50:49.280 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.13e-02 / 3.69e-03 | 20260508 12:50:49.280 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.53e+00, 6.02e-01 / 1.83e-01 | 20260508 12:50:49.280 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.50e+00, 2.12e-01 / 8.89e-02 | 20260508 12:50:49.280 | INFO | PerfReport: 15.99%, DME -- 4.19e+01 / 4.56e+01, 3.69e+00 / 8.45e-01 | 20260508 12:50:49.280 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.86e+00 / 1.30e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.32e+00 / 2.54e+00, 4.04e-01 / 2.67e-01 | 20260508 12:50:49.280 | INFO | PerfReport: 2.95%, Metrics -- 9.65e-01 / 1.12e+00, 2.39e-01 / 1.25e-01 | 20260508 12:50:49.280 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.92e-01 / 3.26e-01, 4.69e-02 / 1.90e-02 | 20260508 12:50:49.280 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:50:49.280 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:50:49.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:50:49.280 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 24.14%, Upload -- 2.53e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 24.79%, Download -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 31.24%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 2.08%, LockUpload -- 2.18e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 2.66%, LockDownload -- 2.79e+00 / 0.00e+00, 4.22e+00 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 15.06%, LockCompute -- 1.58e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:50:49.280 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.31e-02 / 0.00e+00 | 20260508 12:50:49.377 | INFO | Starting a round of Baz IO | 20260508 12:50:52.783 | INFO | Populating RTMetrics report | 20260508 12:50:52.783 | INFO | RTMetrics report populated | 20260508 12:50:52.783 | INFO | Opening tmp RTMetrics json file | 20260508 12:50:52.792 | INFO | Serializing RTMetrics to json | 20260508 12:50:52.793 | INFO | Serialization done, swapping tmp file now... | 20260508 12:50:52.794 | INFO | RTMetrics done | 20260508 12:50:52.845 | INFO | Finished a round of Baz IO: Wrote 1.401 GiB in 3.468 seconds | 20260508 12:50:52.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:50:52.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155425% / 3.97364e-06% / 7.15256e-05% | 20260508 12:50:52.845 | INFO | Baz IO overall: | 20260508 12:50:52.845 | INFO | BytesWritten : 3525264176927 | 20260508 12:50:52.845 | INFO | NumEvents : 1691119686789 | 20260508 12:50:54.420 | INFO | Analyzing chunk frames = [8241152,8241664) | 20260508 12:50:54.420 | INFO | MemInfo: { | 20260508 12:50:54.420 | INFO | "gpu" : " 17 GB", | 20260508 12:50:54.420 | INFO | "huge" : "", | 20260508 12:50:54.420 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:54.420 | INFO | "malloc" : " 14 GB", | 20260508 12:50:54.420 | INFO | "pinned" : " 29 GB" | 20260508 12:50:54.420 | INFO | } | 20260508 12:50:54.420 | INFO | Perf report for the last 5.14126 seconds: | 20260508 12:50:54.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:54.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:54.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:54.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:54.420 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 12:50:54.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:54.420 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:50:54.420 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 12:50:54.420 | INFO | BAZWRITER : 66.7%, 2.4, 83.8ms, 0 | 20260508 12:50:54.420 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:50:54.421 | INFO | Uploaded 36GiB to GPU in 2.61824s (13.7497GiB/s) | 20260508 12:50:54.421 | INFO | Downloaded 38GiB to GPU in 2.70776s (14.0337GiB/s) | 20260508 12:50:59.553 | INFO | Analyzing chunk frames = [8241664,8242176) | 20260508 12:50:59.553 | INFO | MemInfo: { | 20260508 12:50:59.553 | INFO | "gpu" : " 17 GB", | 20260508 12:50:59.553 | INFO | "huge" : "", | 20260508 12:50:59.553 | INFO | "huge_shared" : " 29 GB", | 20260508 12:50:59.553 | INFO | "malloc" : " 14 GB", | 20260508 12:50:59.553 | INFO | "pinned" : " 29 GB" | 20260508 12:50:59.553 | INFO | } | 20260508 12:50:59.553 | INFO | Perf report for the last 5.13257 seconds: | 20260508 12:50:59.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:50:59.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 12:50:59.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:50:59.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:50:59.553 | INFO | Uploaded 36GiB to GPU in 2.47579s (14.5408GiB/s) | 20260508 12:50:59.553 | INFO | Downloaded 34GiB to GPU in 2.4847s (13.6837GiB/s) | 20260508 12:51:04.703 | INFO | Analyzing chunk frames = [8242176,8242688) | 20260508 12:51:04.703 | INFO | MemInfo: { | 20260508 12:51:04.703 | INFO | "gpu" : " 17 GB", | 20260508 12:51:04.703 | INFO | "huge" : "", | 20260508 12:51:04.703 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:04.703 | INFO | "malloc" : " 14 GB", | 20260508 12:51:04.703 | INFO | "pinned" : " 29 GB" | 20260508 12:51:04.703 | INFO | } | 20260508 12:51:04.703 | INFO | Perf report for the last 5.15032 seconds: | 20260508 12:51:04.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:04.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:51:04.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:51:04.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:04.703 | INFO | Uploaded 36GiB to GPU in 2.47662s (14.5359GiB/s) | 20260508 12:51:04.703 | INFO | Downloaded 34GiB to GPU in 2.44355s (13.9142GiB/s) | 20260508 12:51:09.844 | INFO | Analyzing chunk frames = [8242688,8243200) | 20260508 12:51:09.844 | INFO | MemInfo: { | 20260508 12:51:09.844 | INFO | "gpu" : " 17 GB", | 20260508 12:51:09.844 | INFO | "huge" : "", | 20260508 12:51:09.844 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:09.844 | INFO | "malloc" : " 14 GB", | 20260508 12:51:09.844 | INFO | "pinned" : " 29 GB" | 20260508 12:51:09.844 | INFO | } | 20260508 12:51:09.844 | INFO | Perf report for the last 5.14103 seconds: | 20260508 12:51:09.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:09.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 12:51:09.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:51:09.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:09.844 | INFO | Uploaded 36GiB to GPU in 2.51511s (14.3135GiB/s) | 20260508 12:51:09.844 | INFO | Downloaded 34GiB to GPU in 2.45448s (13.8522GiB/s) | 20260508 12:51:14.980 | INFO | Analyzing chunk frames = [8243200,8243712) | 20260508 12:51:14.980 | INFO | MemInfo: { | 20260508 12:51:14.980 | INFO | "gpu" : " 17 GB", | 20260508 12:51:14.980 | INFO | "huge" : "", | 20260508 12:51:14.980 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:14.980 | INFO | "malloc" : " 14 GB", | 20260508 12:51:14.980 | INFO | "pinned" : " 29 GB" | 20260508 12:51:14.980 | INFO | } | 20260508 12:51:14.980 | INFO | Perf report for the last 5.1357 seconds: | 20260508 12:51:14.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:14.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 12:51:14.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:51:14.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:14.980 | INFO | Uploaded 36GiB to GPU in 2.4074s (14.9539GiB/s) | 20260508 12:51:14.980 | INFO | Downloaded 34GiB to GPU in 2.52417s (13.4698GiB/s) | 20260508 12:51:20.112 | INFO | Analyzing chunk frames = [8243712,8244224) | 20260508 12:51:20.112 | INFO | MemInfo: { | 20260508 12:51:20.113 | INFO | "gpu" : " 17 GB", | 20260508 12:51:20.113 | INFO | "huge" : "", | 20260508 12:51:20.113 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:20.113 | INFO | "malloc" : " 14 GB", | 20260508 12:51:20.113 | INFO | "pinned" : " 29 GB" | 20260508 12:51:20.113 | INFO | } | 20260508 12:51:20.113 | INFO | Perf report for the last 5.13275 seconds: | 20260508 12:51:20.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:20.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260508 12:51:20.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260508 12:51:20.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:20.113 | INFO | Uploaded 36GiB to GPU in 2.44245s (14.7393GiB/s) | 20260508 12:51:20.113 | INFO | Downloaded 34GiB to GPU in 2.45676s (13.8394GiB/s) | 20260508 12:51:25.254 | INFO | Analyzing chunk frames = [8244224,8244736) | 20260508 12:51:25.254 | INFO | MemInfo: { | 20260508 12:51:25.254 | INFO | "gpu" : " 17 GB", | 20260508 12:51:25.254 | INFO | "huge" : "", | 20260508 12:51:25.254 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:25.254 | INFO | "malloc" : " 14 GB", | 20260508 12:51:25.254 | INFO | "pinned" : " 29 GB" | 20260508 12:51:25.254 | INFO | } | 20260508 12:51:25.254 | INFO | Perf report for the last 5.14175 seconds: | 20260508 12:51:25.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:25.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.255 | INFO | ANALYSIS : 64.7%, 3.0, 100.8ms, 0 | 20260508 12:51:25.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.255 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 12:51:25.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:25.255 | INFO | Uploaded 36GiB to GPU in 2.29333s (15.6977GiB/s) | 20260508 12:51:25.255 | INFO | Downloaded 34GiB to GPU in 2.40938s (14.1115GiB/s) | 20260508 12:51:30.395 | INFO | Analyzing chunk frames = [8244736,8245248) | 20260508 12:51:30.395 | INFO | MemInfo: { | 20260508 12:51:30.395 | INFO | "gpu" : " 17 GB", | 20260508 12:51:30.395 | INFO | "huge" : "", | 20260508 12:51:30.395 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:30.395 | INFO | "malloc" : " 14 GB", | 20260508 12:51:30.395 | INFO | "pinned" : " 29 GB" | 20260508 12:51:30.395 | INFO | } | 20260508 12:51:30.395 | INFO | Perf report for the last 5.14097 seconds: | 20260508 12:51:30.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:30.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.396 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260508 12:51:30.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.396 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260508 12:51:30.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:30.396 | INFO | Uploaded 36GiB to GPU in 2.33506s (15.4172GiB/s) | 20260508 12:51:30.396 | INFO | Downloaded 34GiB to GPU in 2.43828s (13.9443GiB/s) | 20260508 12:51:30.510 | INFO | Starting a round of Baz IO | 20260508 12:51:33.913 | INFO | Populating RTMetrics report | 20260508 12:51:33.914 | INFO | RTMetrics report populated | 20260508 12:51:33.914 | INFO | Opening tmp RTMetrics json file | 20260508 12:51:33.918 | INFO | Serializing RTMetrics to json | 20260508 12:51:33.920 | INFO | Serialization done, swapping tmp file now... | 20260508 12:51:33.920 | INFO | RTMetrics done | 20260508 12:51:33.978 | INFO | Finished a round of Baz IO: Wrote 1.402 GiB in 3.467 seconds | 20260508 12:51:33.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:51:33.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155358% / 3.97364e-06% / 6.35783e-05% | 20260508 12:51:33.978 | INFO | Baz IO overall: | 20260508 12:51:33.978 | INFO | BytesWritten : 3526769367455 | 20260508 12:51:33.978 | INFO | NumEvents : 1691825442627 | 20260508 12:51:35.539 | INFO | Analyzing chunk frames = [8245248,8245760) | 20260508 12:51:35.539 | INFO | MemInfo: { | 20260508 12:51:35.539 | INFO | "gpu" : " 17 GB", | 20260508 12:51:35.539 | INFO | "huge" : "", | 20260508 12:51:35.539 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:35.539 | INFO | "malloc" : " 14 GB", | 20260508 12:51:35.539 | INFO | "pinned" : " 29 GB" | 20260508 12:51:35.539 | INFO | } | 20260508 12:51:35.539 | INFO | Perf report for the last 5.14395 seconds: | 20260508 12:51:35.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:35.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:35.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:35.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:35.539 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 12:51:35.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:35.539 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260508 12:51:35.539 | INFO | PRE_HQ : 66.0%, 1.8, 61.6ms, 0 | 20260508 12:51:35.540 | INFO | BAZWRITER : 66.7%, 2.9, 101.2ms, 0 | 20260508 12:51:35.540 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 12:51:35.540 | INFO | Uploaded 36GiB to GPU in 2.63712s (13.6512GiB/s) | 20260508 12:51:35.540 | INFO | Downloaded 38GiB to GPU in 2.71438s (13.9995GiB/s) | 20260508 12:51:40.676 | INFO | Analyzing chunk frames = [8245760,8246272) | 20260508 12:51:40.676 | INFO | MemInfo: { | 20260508 12:51:40.676 | INFO | "gpu" : " 17 GB", | 20260508 12:51:40.676 | INFO | "huge" : "", | 20260508 12:51:40.676 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:40.676 | INFO | "malloc" : " 14 GB", | 20260508 12:51:40.676 | INFO | "pinned" : " 29 GB" | 20260508 12:51:40.676 | INFO | } | 20260508 12:51:40.676 | INFO | Perf report for the last 5.13664 seconds: | 20260508 12:51:40.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:40.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:51:40.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 12:51:40.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:40.676 | INFO | Uploaded 36GiB to GPU in 2.50938s (14.3462GiB/s) | 20260508 12:51:40.676 | INFO | Downloaded 34GiB to GPU in 2.46753s (13.779GiB/s) | 20260508 12:51:45.811 | INFO | Analyzing chunk frames = [8246272,8246784) | 20260508 12:51:45.811 | INFO | MemInfo: { | 20260508 12:51:45.811 | INFO | "gpu" : " 17 GB", | 20260508 12:51:45.811 | INFO | "huge" : "", | 20260508 12:51:45.811 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:45.811 | INFO | "malloc" : " 14 GB", | 20260508 12:51:45.811 | INFO | "pinned" : " 29 GB" | 20260508 12:51:45.811 | INFO | } | 20260508 12:51:45.812 | INFO | Perf report for the last 5.13543 seconds: | 20260508 12:51:45.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:45.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:51:45.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:51:45.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:45.812 | INFO | Uploaded 36GiB to GPU in 2.50405s (14.3767GiB/s) | 20260508 12:51:45.812 | INFO | Downloaded 34GiB to GPU in 2.43889s (13.9408GiB/s) | 20260508 12:51:50.957 | INFO | Analyzing chunk frames = [8246784,8247296) | 20260508 12:51:50.957 | INFO | MemInfo: { | 20260508 12:51:50.957 | INFO | "gpu" : " 17 GB", | 20260508 12:51:50.957 | INFO | "huge" : "", | 20260508 12:51:50.957 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:50.957 | INFO | "malloc" : " 14 GB", | 20260508 12:51:50.957 | INFO | "pinned" : " 29 GB" | 20260508 12:51:50.957 | INFO | } | 20260508 12:51:50.958 | INFO | Perf report for the last 5.14599 seconds: | 20260508 12:51:50.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:50.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 12:51:50.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:51:50.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:50.958 | INFO | Uploaded 36GiB to GPU in 2.49302s (14.4403GiB/s) | 20260508 12:51:50.958 | INFO | Downloaded 34GiB to GPU in 2.45017s (13.8766GiB/s) | 20260508 12:51:56.091 | INFO | Analyzing chunk frames = [8247296,8247808) | 20260508 12:51:56.091 | INFO | MemInfo: { | 20260508 12:51:56.091 | INFO | "gpu" : " 17 GB", | 20260508 12:51:56.091 | INFO | "huge" : "", | 20260508 12:51:56.091 | INFO | "huge_shared" : " 29 GB", | 20260508 12:51:56.091 | INFO | "malloc" : " 14 GB", | 20260508 12:51:56.091 | INFO | "pinned" : " 29 GB" | 20260508 12:51:56.091 | INFO | } | 20260508 12:51:56.091 | INFO | Perf report for the last 5.13374 seconds: | 20260508 12:51:56.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:51:56.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | ANALYSIS : 66.6%, 3.0, 103.7ms, 0 | 20260508 12:51:56.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:51:56.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:51:56.091 | INFO | Uploaded 36GiB to GPU in 2.51044s (14.3401GiB/s) | 20260508 12:51:56.092 | INFO | Downloaded 34GiB to GPU in 2.51618s (13.5126GiB/s) | 20260508 12:52:01.229 | INFO | Analyzing chunk frames = [8247808,8248320) | 20260508 12:52:01.229 | INFO | MemInfo: { | 20260508 12:52:01.229 | INFO | "gpu" : " 17 GB", | 20260508 12:52:01.229 | INFO | "huge" : "", | 20260508 12:52:01.229 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:01.229 | INFO | "malloc" : " 14 GB", | 20260508 12:52:01.229 | INFO | "pinned" : " 29 GB" | 20260508 12:52:01.229 | INFO | } | 20260508 12:52:01.229 | INFO | Perf report for the last 5.13742 seconds: | 20260508 12:52:01.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:01.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 12:52:01.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:52:01.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:01.229 | INFO | Uploaded 36GiB to GPU in 2.41762s (14.8907GiB/s) | 20260508 12:52:01.229 | INFO | Downloaded 34GiB to GPU in 2.44136s (13.9267GiB/s) | 20260508 12:52:06.370 | INFO | Analyzing chunk frames = [8248320,8248832) | 20260508 12:52:06.370 | INFO | MemInfo: { | 20260508 12:52:06.370 | INFO | "gpu" : " 17 GB", | 20260508 12:52:06.370 | INFO | "huge" : "", | 20260508 12:52:06.370 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:06.370 | INFO | "malloc" : " 14 GB", | 20260508 12:52:06.370 | INFO | "pinned" : " 29 GB" | 20260508 12:52:06.370 | INFO | } | 20260508 12:52:06.370 | INFO | Perf report for the last 5.14175 seconds: | 20260508 12:52:06.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:06.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.370 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 12:52:06.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.371 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 12:52:06.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:06.371 | INFO | Uploaded 36GiB to GPU in 2.44181s (14.7432GiB/s) | 20260508 12:52:06.371 | INFO | Downloaded 34GiB to GPU in 2.44948s (13.8805GiB/s) | 20260508 12:52:11.510 | INFO | Analyzing chunk frames = [8248832,8249344) | 20260508 12:52:11.510 | INFO | MemInfo: { | 20260508 12:52:11.510 | INFO | "gpu" : " 17 GB", | 20260508 12:52:11.510 | INFO | "huge" : "", | 20260508 12:52:11.510 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:11.510 | INFO | "malloc" : " 14 GB", | 20260508 12:52:11.510 | INFO | "pinned" : " 29 GB" | 20260508 12:52:11.510 | INFO | } | 20260508 12:52:11.510 | INFO | Perf report for the last 5.13969 seconds: | 20260508 12:52:11.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:11.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:52:11.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:52:11.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:11.510 | INFO | Uploaded 36GiB to GPU in 2.50765s (14.3561GiB/s) | 20260508 12:52:11.510 | INFO | Downloaded 34GiB to GPU in 2.47776s (13.7221GiB/s) | 20260508 12:52:11.510 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:52:11.510 | INFO | Total mem usage: 16.7 GiB | 20260508 12:52:11.510 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:52:11.510 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14%) | 20260508 12:52:11.510 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:52:11.511 | INFO | High Water Breakdown: | 20260508 12:52:11.511 | INFO | 59.9% : 8.76 GiB ( 85% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:52:11.511 | INFO | 40% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:52:11.511 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:52:11.511 | INFO | ------------------------------------------ | 20260508 12:52:11.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:52:11.511 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:52:11.511 | INFO | Total mem usage: 26.5 GiB | 20260508 12:52:11.511 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:52:11.511 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:52:11.511 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:52:11.511 | INFO | High Water Breakdown: | 20260508 12:52:11.511 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:52:11.511 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:52:11.511 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:52:11.511 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:52:11.511 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:52:11.511 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:52:11.511 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:52:11.511 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:52:11.511 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:52:11.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:52:11.511 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:52:11.511 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:52:11.511 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:52:11.511 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:52:11.511 | INFO | ------------------------------------------ | 20260508 12:52:11.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:52:11.511 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:52:11.511 | INFO | Total mem usage: 27.8 GiB | 20260508 12:52:11.511 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:52:11.511 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:52:11.511 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:52:11.511 | INFO | High Water Breakdown: | 20260508 12:52:11.511 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:52:11.511 | INFO | ------------------------------------------ | 20260508 12:52:11.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:52:11.511 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:52:11.511 | INFO | Total mem usage: 15.7 GiB | 20260508 12:52:11.511 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:52:11.511 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:52:11.511 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:52:11.511 | INFO | High Water Breakdown: | 20260508 12:52:11.511 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:52:11.511 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:52:11.511 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:52:11.511 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:52:11.511 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:52:11.511 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:52:11.511 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:52:11.511 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:52:11.511 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:52:11.511 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:52:11.511 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:52:11.511 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:52:11.511 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:52:11.511 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:52:11.511 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:52:11.511 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:52:11.511 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:52:11.511 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:52:11.511 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:52:11.511 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:52:11.511 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:52:11.511 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:52:11.511 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:52:11.511 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:52:11.511 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:52:11.511 | INFO | ------------------------------------------ | 20260508 12:52:11.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:52:11.511 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:52:11.511 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:52:11.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:52:11.511 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.39e+01 / 4.36e-01 | 20260508 12:52:11.511 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.14e-02 / 7.87e-03 | 20260508 12:52:11.511 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.33e+00, 5.90e-01 / 2.40e-01 | 20260508 12:52:11.511 | INFO | PerfReport: 3.39%, Histogram -- 1.10e+00 / 1.46e+00, 2.10e-01 / 1.20e-01 | 20260508 12:52:11.511 | INFO | PerfReport: 15.97%, DME -- 4.17e+01 / 4.48e+01, 3.75e+00 / 9.03e-01 | 20260508 12:52:11.511 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.84e+00 / 1.42e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.32e+00 / 2.38e+00, 4.05e-01 / 2.78e-01 | 20260508 12:52:11.511 | INFO | PerfReport: 2.94%, Metrics -- 9.60e-01 / 1.08e+00, 2.38e-01 / 1.05e-01 | 20260508 12:52:11.511 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.19e-01, 4.69e-02 / 2.91e-02 | 20260508 12:52:11.511 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:52:11.511 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:52:11.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:52:11.511 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 24.56%, Upload -- 2.58e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 31.06%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 2.21%, LockUpload -- 2.32e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 2.50%, LockDownload -- 2.63e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 14.91%, LockCompute -- 1.56e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 12:52:11.511 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 2.83e-02 / 0.00e+00 | 20260508 12:52:11.607 | INFO | Starting a round of Baz IO | 20260508 12:52:15.008 | INFO | Populating RTMetrics report | 20260508 12:52:15.009 | INFO | RTMetrics report populated | 20260508 12:52:15.009 | INFO | Opening tmp RTMetrics json file | 20260508 12:52:15.018 | INFO | Serializing RTMetrics to json | 20260508 12:52:15.019 | INFO | Serialization done, swapping tmp file now... | 20260508 12:52:15.020 | INFO | RTMetrics done | 20260508 12:52:15.080 | INFO | Finished a round of Baz IO: Wrote 1.399 GiB in 3.473 seconds | 20260508 12:52:15.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:52:15.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154452% / 0% / 7.54992e-05% | 20260508 12:52:15.080 | INFO | Baz IO overall: | 20260508 12:52:15.080 | INFO | BytesWritten : 3528271807371 | 20260508 12:52:15.080 | INFO | NumEvents : 1692529514745 | 20260508 12:52:16.650 | INFO | Analyzing chunk frames = [8249344,8249856) | 20260508 12:52:16.650 | INFO | MemInfo: { | 20260508 12:52:16.650 | INFO | "gpu" : " 17 GB", | 20260508 12:52:16.650 | INFO | "huge" : "", | 20260508 12:52:16.650 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:16.650 | INFO | "malloc" : " 14 GB", | 20260508 12:52:16.650 | INFO | "pinned" : " 29 GB" | 20260508 12:52:16.650 | INFO | } | 20260508 12:52:16.650 | INFO | Perf report for the last 5.13978 seconds: | 20260508 12:52:16.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:16.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:16.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:16.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:16.650 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 12:52:16.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:16.650 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 12:52:16.650 | INFO | PRE_HQ : 65.4%, 1.6, 53.9ms, 0 | 20260508 12:52:16.650 | INFO | BAZWRITER : 66.9%, 2.4, 84.1ms, 0 | 20260508 12:52:16.650 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:52:16.650 | INFO | Uploaded 36GiB to GPU in 2.6041s (13.8244GiB/s) | 20260508 12:52:16.650 | INFO | Downloaded 38GiB to GPU in 2.72457s (13.9471GiB/s) | 20260508 12:52:21.795 | INFO | Analyzing chunk frames = [8249856,8250368) | 20260508 12:52:21.795 | INFO | MemInfo: { | 20260508 12:52:21.795 | INFO | "gpu" : " 17 GB", | 20260508 12:52:21.795 | INFO | "huge" : "", | 20260508 12:52:21.795 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:21.795 | INFO | "malloc" : " 14 GB", | 20260508 12:52:21.795 | INFO | "pinned" : " 29 GB" | 20260508 12:52:21.795 | INFO | } | 20260508 12:52:21.795 | INFO | Perf report for the last 5.14552 seconds: | 20260508 12:52:21.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:21.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.796 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 12:52:21.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.796 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:52:21.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:21.796 | INFO | Uploaded 36GiB to GPU in 2.44684s (14.7128GiB/s) | 20260508 12:52:21.796 | INFO | Downloaded 34GiB to GPU in 2.46379s (13.7999GiB/s) | 20260508 12:52:26.926 | INFO | Analyzing chunk frames = [8250368,8250880) | 20260508 12:52:26.926 | INFO | MemInfo: { | 20260508 12:52:26.926 | INFO | "gpu" : " 17 GB", | 20260508 12:52:26.926 | INFO | "huge" : "", | 20260508 12:52:26.926 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:26.926 | INFO | "malloc" : " 14 GB", | 20260508 12:52:26.926 | INFO | "pinned" : " 29 GB" | 20260508 12:52:26.926 | INFO | } | 20260508 12:52:26.926 | INFO | Perf report for the last 5.13025 seconds: | 20260508 12:52:26.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:26.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 12:52:26.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:52:26.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:26.926 | INFO | Uploaded 36GiB to GPU in 2.46601s (14.5985GiB/s) | 20260508 12:52:26.926 | INFO | Downloaded 34GiB to GPU in 2.46099s (13.8156GiB/s) | 20260508 12:52:32.063 | INFO | Analyzing chunk frames = [8250880,8251392) | 20260508 12:52:32.063 | INFO | MemInfo: { | 20260508 12:52:32.063 | INFO | "gpu" : " 17 GB", | 20260508 12:52:32.063 | INFO | "huge" : "", | 20260508 12:52:32.063 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:32.063 | INFO | "malloc" : " 14 GB", | 20260508 12:52:32.063 | INFO | "pinned" : " 29 GB" | 20260508 12:52:32.063 | INFO | } | 20260508 12:52:32.063 | INFO | Perf report for the last 5.13718 seconds: | 20260508 12:52:32.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:32.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 12:52:32.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:52:32.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:32.063 | INFO | Uploaded 36GiB to GPU in 2.46405s (14.6101GiB/s) | 20260508 12:52:32.063 | INFO | Downloaded 34GiB to GPU in 2.43166s (13.9822GiB/s) | 20260508 12:52:37.208 | INFO | Analyzing chunk frames = [8251392,8251904) | 20260508 12:52:37.208 | INFO | MemInfo: { | 20260508 12:52:37.208 | INFO | "gpu" : " 17 GB", | 20260508 12:52:37.208 | INFO | "huge" : "", | 20260508 12:52:37.208 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:37.208 | INFO | "malloc" : " 14 GB", | 20260508 12:52:37.208 | INFO | "pinned" : " 29 GB" | 20260508 12:52:37.208 | INFO | } | 20260508 12:52:37.208 | INFO | Perf report for the last 5.14544 seconds: | 20260508 12:52:37.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:37.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.208 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 12:52:37.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.209 | INFO | PRE_HQ : 24.1%, 1.0, 12.6ms, 0 | 20260508 12:52:37.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:37.209 | INFO | Uploaded 36GiB to GPU in 2.30209s (15.6379GiB/s) | 20260508 12:52:37.209 | INFO | Downloaded 34GiB to GPU in 2.4913s (13.6475GiB/s) | 20260508 12:52:42.352 | INFO | Analyzing chunk frames = [8251904,8252416) | 20260508 12:52:42.352 | INFO | MemInfo: { | 20260508 12:52:42.352 | INFO | "gpu" : " 17 GB", | 20260508 12:52:42.352 | INFO | "huge" : "", | 20260508 12:52:42.352 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:42.352 | INFO | "malloc" : " 14 GB", | 20260508 12:52:42.352 | INFO | "pinned" : " 29 GB" | 20260508 12:52:42.352 | INFO | } | 20260508 12:52:42.352 | INFO | Perf report for the last 5.14371 seconds: | 20260508 12:52:42.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:42.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.352 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:52:42.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.352 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 12:52:42.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:42.353 | INFO | Uploaded 36GiB to GPU in 2.38619s (15.0868GiB/s) | 20260508 12:52:42.353 | INFO | Downloaded 34GiB to GPU in 2.43418s (13.9678GiB/s) | 20260508 12:52:47.488 | INFO | Analyzing chunk frames = [8252416,8252928) | 20260508 12:52:47.488 | INFO | MemInfo: { | 20260508 12:52:47.488 | INFO | "gpu" : " 17 GB", | 20260508 12:52:47.488 | INFO | "huge" : "", | 20260508 12:52:47.488 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:47.488 | INFO | "malloc" : " 14 GB", | 20260508 12:52:47.488 | INFO | "pinned" : " 29 GB" | 20260508 12:52:47.488 | INFO | } | 20260508 12:52:47.488 | INFO | Perf report for the last 5.13594 seconds: | 20260508 12:52:47.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:47.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | ANALYSIS : 64.5%, 3.0, 100.3ms, 0 | 20260508 12:52:47.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:52:47.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:47.488 | INFO | Uploaded 36GiB to GPU in 2.29779s (15.6672GiB/s) | 20260508 12:52:47.488 | INFO | Downloaded 34GiB to GPU in 2.41389s (14.0852GiB/s) | 20260508 12:52:52.623 | INFO | Analyzing chunk frames = [8252928,8253440) | 20260508 12:52:52.623 | INFO | MemInfo: { | 20260508 12:52:52.623 | INFO | "gpu" : " 17 GB", | 20260508 12:52:52.623 | INFO | "huge" : "", | 20260508 12:52:52.623 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:52.623 | INFO | "malloc" : " 14 GB", | 20260508 12:52:52.623 | INFO | "pinned" : " 29 GB" | 20260508 12:52:52.623 | INFO | } | 20260508 12:52:52.623 | INFO | Perf report for the last 5.13492 seconds: | 20260508 12:52:52.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:52.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260508 12:52:52.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 12:52:52.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:52.623 | INFO | Uploaded 36GiB to GPU in 2.35066s (15.3149GiB/s) | 20260508 12:52:52.623 | INFO | Downloaded 34GiB to GPU in 2.43266s (13.9765GiB/s) | 20260508 12:52:52.720 | INFO | Starting a round of Baz IO | 20260508 12:52:56.133 | INFO | Populating RTMetrics report | 20260508 12:52:56.133 | INFO | RTMetrics report populated | 20260508 12:52:56.133 | INFO | Opening tmp RTMetrics json file | 20260508 12:52:56.144 | INFO | Serializing RTMetrics to json | 20260508 12:52:56.146 | INFO | Serialization done, swapping tmp file now... | 20260508 12:52:56.147 | INFO | RTMetrics done | 20260508 12:52:56.197 | INFO | Finished a round of Baz IO: Wrote 1.399 GiB in 3.476 seconds | 20260508 12:52:56.197 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:52:56.197 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154106% / 3.97364e-06% / 7.15256e-05% | 20260508 12:52:56.197 | INFO | Baz IO overall: | 20260508 12:52:56.197 | INFO | BytesWritten : 3529773825039 | 20260508 12:52:56.197 | INFO | NumEvents : 1693232915625 | 20260508 12:52:57.764 | INFO | Analyzing chunk frames = [8253440,8253952) | 20260508 12:52:57.764 | INFO | MemInfo: { | 20260508 12:52:57.764 | INFO | "gpu" : " 17 GB", | 20260508 12:52:57.764 | INFO | "huge" : "", | 20260508 12:52:57.764 | INFO | "huge_shared" : " 29 GB", | 20260508 12:52:57.764 | INFO | "malloc" : " 14 GB", | 20260508 12:52:57.764 | INFO | "pinned" : " 29 GB" | 20260508 12:52:57.764 | INFO | } | 20260508 12:52:57.764 | INFO | Perf report for the last 5.1411 seconds: | 20260508 12:52:57.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:52:57.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:57.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:57.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:57.764 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:52:57.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:52:57.764 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:52:57.764 | INFO | PRE_HQ : 65.3%, 1.6, 53.9ms, 0 | 20260508 12:52:57.764 | INFO | BAZWRITER : 66.9%, 2.5, 84.5ms, 0 | 20260508 12:52:57.764 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:52:57.764 | INFO | Uploaded 36GiB to GPU in 2.60921s (13.7973GiB/s) | 20260508 12:52:57.764 | INFO | Downloaded 38GiB to GPU in 2.7212s (13.9644GiB/s) | 20260508 12:53:02.905 | INFO | Analyzing chunk frames = [8253952,8254464) | 20260508 12:53:02.905 | INFO | MemInfo: { | 20260508 12:53:02.905 | INFO | "gpu" : " 17 GB", | 20260508 12:53:02.905 | INFO | "huge" : "", | 20260508 12:53:02.905 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:02.905 | INFO | "malloc" : " 14 GB", | 20260508 12:53:02.905 | INFO | "pinned" : " 29 GB" | 20260508 12:53:02.905 | INFO | } | 20260508 12:53:02.905 | INFO | Perf report for the last 5.14102 seconds: | 20260508 12:53:02.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:02.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:53:02.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:53:02.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:02.905 | INFO | Uploaded 36GiB to GPU in 2.45422s (14.6686GiB/s) | 20260508 12:53:02.905 | INFO | Downloaded 34GiB to GPU in 2.46654s (13.7845GiB/s) | 20260508 12:53:08.045 | INFO | Analyzing chunk frames = [8254464,8254976) | 20260508 12:53:08.045 | INFO | MemInfo: { | 20260508 12:53:08.045 | INFO | "gpu" : " 17 GB", | 20260508 12:53:08.045 | INFO | "huge" : "", | 20260508 12:53:08.045 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:08.045 | INFO | "malloc" : " 14 GB", | 20260508 12:53:08.045 | INFO | "pinned" : " 29 GB" | 20260508 12:53:08.045 | INFO | } | 20260508 12:53:08.045 | INFO | Perf report for the last 5.13952 seconds: | 20260508 12:53:08.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:08.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 12:53:08.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:53:08.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:08.045 | INFO | Uploaded 36GiB to GPU in 2.49147s (14.4493GiB/s) | 20260508 12:53:08.045 | INFO | Downloaded 34GiB to GPU in 2.45141s (13.8696GiB/s) | 20260508 12:53:13.183 | INFO | Analyzing chunk frames = [8254976,8255488) | 20260508 12:53:13.183 | INFO | MemInfo: { | 20260508 12:53:13.183 | INFO | "gpu" : " 17 GB", | 20260508 12:53:13.184 | INFO | "huge" : "", | 20260508 12:53:13.184 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:13.184 | INFO | "malloc" : " 14 GB", | 20260508 12:53:13.184 | INFO | "pinned" : " 29 GB" | 20260508 12:53:13.184 | INFO | } | 20260508 12:53:13.184 | INFO | Perf report for the last 5.13894 seconds: | 20260508 12:53:13.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:13.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 12:53:13.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:53:13.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:13.184 | INFO | Uploaded 36GiB to GPU in 2.52591s (14.2523GiB/s) | 20260508 12:53:13.184 | INFO | Downloaded 34GiB to GPU in 2.47009s (13.7647GiB/s) | 20260508 12:53:18.320 | INFO | Analyzing chunk frames = [8255488,8256000) | 20260508 12:53:18.320 | INFO | MemInfo: { | 20260508 12:53:18.320 | INFO | "gpu" : " 17 GB", | 20260508 12:53:18.321 | INFO | "huge" : "", | 20260508 12:53:18.321 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:18.321 | INFO | "malloc" : " 14 GB", | 20260508 12:53:18.321 | INFO | "pinned" : " 29 GB" | 20260508 12:53:18.321 | INFO | } | 20260508 12:53:18.321 | INFO | Perf report for the last 5.13699 seconds: | 20260508 12:53:18.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:18.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:53:18.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260508 12:53:18.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:18.321 | INFO | Uploaded 36GiB to GPU in 2.31398s (15.5576GiB/s) | 20260508 12:53:18.321 | INFO | Downloaded 34GiB to GPU in 2.49592s (13.6222GiB/s) | 20260508 12:53:23.466 | INFO | Analyzing chunk frames = [8256000,8256512) | 20260508 12:53:23.466 | INFO | MemInfo: { | 20260508 12:53:23.466 | INFO | "gpu" : " 17 GB", | 20260508 12:53:23.466 | INFO | "huge" : "", | 20260508 12:53:23.466 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:23.466 | INFO | "malloc" : " 14 GB", | 20260508 12:53:23.466 | INFO | "pinned" : " 29 GB" | 20260508 12:53:23.466 | INFO | } | 20260508 12:53:23.466 | INFO | Perf report for the last 5.14528 seconds: | 20260508 12:53:23.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:23.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.466 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 12:53:23.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.466 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:53:23.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:23.467 | INFO | Uploaded 36GiB to GPU in 2.44795s (14.7062GiB/s) | 20260508 12:53:23.467 | INFO | Downloaded 34GiB to GPU in 2.434s (13.9688GiB/s) | 20260508 12:53:28.599 | INFO | Analyzing chunk frames = [8256512,8257024) | 20260508 12:53:28.599 | INFO | MemInfo: { | 20260508 12:53:28.599 | INFO | "gpu" : " 17 GB", | 20260508 12:53:28.599 | INFO | "huge" : "", | 20260508 12:53:28.599 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:28.599 | INFO | "malloc" : " 14 GB", | 20260508 12:53:28.599 | INFO | "pinned" : " 29 GB" | 20260508 12:53:28.599 | INFO | } | 20260508 12:53:28.599 | INFO | Perf report for the last 5.13283 seconds: | 20260508 12:53:28.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:28.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 12:53:28.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:53:28.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:28.599 | INFO | Uploaded 36GiB to GPU in 2.50104s (14.394GiB/s) | 20260508 12:53:28.599 | INFO | Downloaded 34GiB to GPU in 2.45355s (13.8575GiB/s) | 20260508 12:53:33.739 | INFO | Analyzing chunk frames = [8257024,8257536) | 20260508 12:53:33.739 | INFO | MemInfo: { | 20260508 12:53:33.739 | INFO | "gpu" : " 17 GB", | 20260508 12:53:33.739 | INFO | "huge" : "", | 20260508 12:53:33.739 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:33.739 | INFO | "malloc" : " 14 GB", | 20260508 12:53:33.739 | INFO | "pinned" : " 29 GB" | 20260508 12:53:33.739 | INFO | } | 20260508 12:53:33.739 | INFO | Perf report for the last 5.13995 seconds: | 20260508 12:53:33.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:33.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 12:53:33.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:53:33.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:33.739 | INFO | Uploaded 36GiB to GPU in 2.50281s (14.3838GiB/s) | 20260508 12:53:33.739 | INFO | Downloaded 34GiB to GPU in 2.4529s (13.8612GiB/s) | 20260508 12:53:33.739 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:53:33.739 | INFO | Total mem usage: 16.7 GiB | 20260508 12:53:33.739 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:53:33.739 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:53:33.739 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:53:33.739 | INFO | High Water Breakdown: | 20260508 12:53:33.739 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:53:33.739 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:53:33.739 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:53:33.739 | INFO | ------------------------------------------ | 20260508 12:53:33.739 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:53:33.739 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:53:33.739 | INFO | Total mem usage: 26.5 GiB | 20260508 12:53:33.739 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:53:33.739 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:53:33.739 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:53:33.739 | INFO | High Water Breakdown: | 20260508 12:53:33.739 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:53:33.739 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:53:33.739 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:53:33.739 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:53:33.739 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:53:33.739 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:53:33.739 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:53:33.739 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:53:33.739 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:53:33.739 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:53:33.739 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:53:33.739 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:53:33.739 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:53:33.739 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:53:33.739 | INFO | ------------------------------------------ | 20260508 12:53:33.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:53:33.740 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:53:33.740 | INFO | Total mem usage: 27.8 GiB | 20260508 12:53:33.740 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:53:33.740 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:53:33.740 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:53:33.740 | INFO | High Water Breakdown: | 20260508 12:53:33.740 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:53:33.740 | INFO | ------------------------------------------ | 20260508 12:53:33.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:53:33.740 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:53:33.740 | INFO | Total mem usage: 15.7 GiB | 20260508 12:53:33.740 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:53:33.740 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:53:33.740 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:53:33.740 | INFO | High Water Breakdown: | 20260508 12:53:33.740 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:53:33.740 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:53:33.740 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:53:33.740 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:53:33.740 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:53:33.740 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:53:33.740 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:53:33.740 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:53:33.740 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:53:33.740 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:53:33.740 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:53:33.740 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:53:33.740 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:53:33.740 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:53:33.740 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:53:33.740 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:53:33.740 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:53:33.740 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:53:33.740 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:53:33.740 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:53:33.740 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:53:33.740 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:53:33.740 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:53:33.740 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:53:33.740 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:53:33.740 | INFO | ------------------------------------------ | 20260508 12:53:33.740 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:53:33.740 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:53:33.740 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:53:33.740 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:53:33.740 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.40e+01 / 5.83e-01 | 20260508 12:53:33.740 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.04e-01, 1.11e-02 / 2.57e-03 | 20260508 12:53:33.740 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.38e+00, 5.91e-01 / 2.13e-01 | 20260508 12:53:33.740 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.47e+00, 2.13e-01 / 1.10e-01 | 20260508 12:53:33.740 | INFO | PerfReport: 16.05%, DME -- 4.20e+01 / 4.58e+01, 3.78e+00 / 1.05e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.86e+01 / 1.97e+01, 1.82e+00 / 9.43e-01 | 20260508 12:53:33.740 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.32e+00 / 2.52e+00, 4.05e-01 / 1.90e-01 | 20260508 12:53:33.740 | INFO | PerfReport: 2.94%, Metrics -- 9.60e-01 / 1.10e+00, 2.36e-01 / 8.89e-02 | 20260508 12:53:33.740 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.25e-01, 4.64e-02 / 1.95e-02 | 20260508 12:53:33.740 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:53:33.740 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:53:33.740 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:53:33.740 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 24.32%, Upload -- 2.55e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 24.71%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 31.16%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 2.17%, LockUpload -- 2.27e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 2.59%, LockDownload -- 2.71e+00 / 0.00e+00, 4.10e+00 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 15.04%, LockCompute -- 1.58e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 12:53:33.740 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.35e-02 / 0.00e+00 | 20260508 12:53:33.836 | INFO | Starting a round of Baz IO | 20260508 12:53:37.252 | INFO | Populating RTMetrics report | 20260508 12:53:37.252 | INFO | RTMetrics report populated | 20260508 12:53:37.252 | INFO | Opening tmp RTMetrics json file | 20260508 12:53:37.267 | INFO | Serializing RTMetrics to json | 20260508 12:53:37.268 | INFO | Serialization done, swapping tmp file now... | 20260508 12:53:37.269 | INFO | RTMetrics done | 20260508 12:53:37.334 | INFO | Finished a round of Baz IO: Wrote 1.398 GiB in 3.498 seconds | 20260508 12:53:37.334 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:53:37.334 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152755% / 3.97364e-06% / 7.94729e-05% | 20260508 12:53:37.334 | INFO | Baz IO overall: | 20260508 12:53:37.335 | INFO | BytesWritten : 3531275399455 | 20260508 12:53:37.335 | INFO | NumEvents : 1693936186169 | 20260508 12:53:38.877 | INFO | Analyzing chunk frames = [8257536,8258048) | 20260508 12:53:38.877 | INFO | MemInfo: { | 20260508 12:53:38.877 | INFO | "gpu" : " 17 GB", | 20260508 12:53:38.877 | INFO | "huge" : "", | 20260508 12:53:38.877 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:38.877 | INFO | "malloc" : " 14 GB", | 20260508 12:53:38.877 | INFO | "pinned" : " 29 GB" | 20260508 12:53:38.877 | INFO | } | 20260508 12:53:38.877 | INFO | Perf report for the last 5.13862 seconds: | 20260508 12:53:38.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:38.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:38.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:38.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:38.877 | INFO | ANALYSIS : 67.8%, 3.0, 105.7ms, 0 | 20260508 12:53:38.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:38.877 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 12:53:38.877 | INFO | PRE_HQ : 65.4%, 1.6, 53.7ms, 0 | 20260508 12:53:38.877 | INFO | BAZWRITER : 67.4%, 2.5, 87.8ms, 0 | 20260508 12:53:38.877 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260508 12:53:38.878 | INFO | Uploaded 36GiB to GPU in 2.59182s (13.8898GiB/s) | 20260508 12:53:38.878 | INFO | Downloaded 38GiB to GPU in 2.71493s (13.9967GiB/s) | 20260508 12:53:44.015 | INFO | Analyzing chunk frames = [8258048,8258560) | 20260508 12:53:44.015 | INFO | MemInfo: { | 20260508 12:53:44.015 | INFO | "gpu" : " 17 GB", | 20260508 12:53:44.015 | INFO | "huge" : "", | 20260508 12:53:44.015 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:44.015 | INFO | "malloc" : " 14 GB", | 20260508 12:53:44.015 | INFO | "pinned" : " 29 GB" | 20260508 12:53:44.015 | INFO | } | 20260508 12:53:44.015 | INFO | Perf report for the last 5.13756 seconds: | 20260508 12:53:44.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:44.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 12:53:44.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:53:44.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:44.015 | INFO | Uploaded 36GiB to GPU in 2.45142s (14.6853GiB/s) | 20260508 12:53:44.015 | INFO | Downloaded 34GiB to GPU in 2.47272s (13.7501GiB/s) | 20260508 12:53:49.160 | INFO | Analyzing chunk frames = [8258560,8259072) | 20260508 12:53:49.160 | INFO | MemInfo: { | 20260508 12:53:49.160 | INFO | "gpu" : " 17 GB", | 20260508 12:53:49.160 | INFO | "huge" : "", | 20260508 12:53:49.160 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:49.160 | INFO | "malloc" : " 14 GB", | 20260508 12:53:49.160 | INFO | "pinned" : " 29 GB" | 20260508 12:53:49.160 | INFO | } | 20260508 12:53:49.160 | INFO | Perf report for the last 5.1453 seconds: | 20260508 12:53:49.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:49.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.160 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 12:53:49.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.160 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:53:49.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:49.161 | INFO | Uploaded 36GiB to GPU in 2.46204s (14.6221GiB/s) | 20260508 12:53:49.161 | INFO | Downloaded 34GiB to GPU in 2.43953s (13.9371GiB/s) | 20260508 12:53:54.304 | INFO | Analyzing chunk frames = [8259072,8259584) | 20260508 12:53:54.304 | INFO | MemInfo: { | 20260508 12:53:54.304 | INFO | "gpu" : " 17 GB", | 20260508 12:53:54.304 | INFO | "huge" : "", | 20260508 12:53:54.304 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:54.304 | INFO | "malloc" : " 14 GB", | 20260508 12:53:54.304 | INFO | "pinned" : " 29 GB" | 20260508 12:53:54.304 | INFO | } | 20260508 12:53:54.304 | INFO | Perf report for the last 5.14393 seconds: | 20260508 12:53:54.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:54.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.304 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 12:53:54.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.304 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:53:54.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:54.305 | INFO | Uploaded 36GiB to GPU in 2.51563s (14.3105GiB/s) | 20260508 12:53:54.305 | INFO | Downloaded 34GiB to GPU in 2.45903s (13.8266GiB/s) | 20260508 12:53:59.436 | INFO | Analyzing chunk frames = [8259584,8260096) | 20260508 12:53:59.436 | INFO | MemInfo: { | 20260508 12:53:59.436 | INFO | "gpu" : " 17 GB", | 20260508 12:53:59.436 | INFO | "huge" : "", | 20260508 12:53:59.436 | INFO | "huge_shared" : " 29 GB", | 20260508 12:53:59.436 | INFO | "malloc" : " 14 GB", | 20260508 12:53:59.436 | INFO | "pinned" : " 29 GB" | 20260508 12:53:59.436 | INFO | } | 20260508 12:53:59.436 | INFO | Perf report for the last 5.13176 seconds: | 20260508 12:53:59.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:53:59.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 12:53:59.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:53:59.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:53:59.436 | INFO | Uploaded 36GiB to GPU in 2.40626s (14.961GiB/s) | 20260508 12:53:59.436 | INFO | Downloaded 34GiB to GPU in 2.53272s (13.4243GiB/s) | 20260508 12:54:04.570 | INFO | Analyzing chunk frames = [8260096,8260608) | 20260508 12:54:04.570 | INFO | MemInfo: { | 20260508 12:54:04.570 | INFO | "gpu" : " 17 GB", | 20260508 12:54:04.570 | INFO | "huge" : "", | 20260508 12:54:04.570 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:04.570 | INFO | "malloc" : " 14 GB", | 20260508 12:54:04.570 | INFO | "pinned" : " 29 GB" | 20260508 12:54:04.570 | INFO | } | 20260508 12:54:04.570 | INFO | Perf report for the last 5.13434 seconds: | 20260508 12:54:04.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:04.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.570 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 12:54:04.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.570 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:54:04.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:04.571 | INFO | Uploaded 36GiB to GPU in 2.39466s (15.0334GiB/s) | 20260508 12:54:04.571 | INFO | Downloaded 34GiB to GPU in 2.44699s (13.8946GiB/s) | 20260508 12:54:09.714 | INFO | Analyzing chunk frames = [8260608,8261120) | 20260508 12:54:09.714 | INFO | MemInfo: { | 20260508 12:54:09.714 | INFO | "gpu" : " 17 GB", | 20260508 12:54:09.714 | INFO | "huge" : "", | 20260508 12:54:09.714 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:09.714 | INFO | "malloc" : " 14 GB", | 20260508 12:54:09.714 | INFO | "pinned" : " 29 GB" | 20260508 12:54:09.714 | INFO | } | 20260508 12:54:09.714 | INFO | Perf report for the last 5.1437 seconds: | 20260508 12:54:09.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:09.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 12:54:09.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:54:09.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:09.714 | INFO | Uploaded 36GiB to GPU in 2.46978s (14.5762GiB/s) | 20260508 12:54:09.714 | INFO | Downloaded 34GiB to GPU in 2.46925s (13.7693GiB/s) | 20260508 12:54:14.866 | INFO | Analyzing chunk frames = [8261120,8261632) | 20260508 12:54:14.866 | INFO | MemInfo: { | 20260508 12:54:14.866 | INFO | "gpu" : " 17 GB", | 20260508 12:54:14.866 | INFO | "huge" : "", | 20260508 12:54:14.866 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:14.866 | INFO | "malloc" : " 14 GB", | 20260508 12:54:14.866 | INFO | "pinned" : " 29 GB" | 20260508 12:54:14.866 | INFO | } | 20260508 12:54:14.866 | INFO | Perf report for the last 5.15202 seconds: | 20260508 12:54:14.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:14.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | ANALYSIS : 65.3%, 3.0, 102.3ms, 0 | 20260508 12:54:14.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:54:14.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:14.866 | INFO | Uploaded 36GiB to GPU in 2.41695s (14.8948GiB/s) | 20260508 12:54:14.866 | INFO | Downloaded 34GiB to GPU in 2.44623s (13.8989GiB/s) | 20260508 12:54:14.963 | INFO | Starting a round of Baz IO | 20260508 12:54:18.367 | INFO | Populating RTMetrics report | 20260508 12:54:18.367 | INFO | RTMetrics report populated | 20260508 12:54:18.367 | INFO | Opening tmp RTMetrics json file | 20260508 12:54:18.368 | INFO | Serializing RTMetrics to json | 20260508 12:54:18.370 | INFO | Serialization done, swapping tmp file now... | 20260508 12:54:18.370 | INFO | RTMetrics done | 20260508 12:54:18.429 | INFO | Finished a round of Baz IO: Wrote 1.399 GiB in 3.465 seconds | 20260508 12:54:18.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:54:18.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154197% / 7.94729e-06% / 5.16574e-05% | 20260508 12:54:18.429 | INFO | Baz IO overall: | 20260508 12:54:18.429 | INFO | BytesWritten : 3532777794838 | 20260508 12:54:18.429 | INFO | NumEvents : 1694640310002 | 20260508 12:54:19.997 | INFO | Analyzing chunk frames = [8261632,8262144) | 20260508 12:54:19.997 | INFO | MemInfo: { | 20260508 12:54:19.997 | INFO | "gpu" : " 17 GB", | 20260508 12:54:19.997 | INFO | "huge" : "", | 20260508 12:54:19.997 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:19.997 | INFO | "malloc" : " 14 GB", | 20260508 12:54:19.997 | INFO | "pinned" : " 29 GB" | 20260508 12:54:19.997 | INFO | } | 20260508 12:54:19.997 | INFO | Perf report for the last 5.13074 seconds: | 20260508 12:54:19.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:19.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:19.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:19.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:19.997 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260508 12:54:19.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:19.997 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 12:54:19.997 | INFO | PRE_HQ : 65.7%, 1.6, 53.9ms, 0 | 20260508 12:54:19.997 | INFO | BAZWRITER : 66.8%, 2.5, 86.6ms, 0 | 20260508 12:54:19.997 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 12:54:19.997 | INFO | Uploaded 36GiB to GPU in 2.56092s (14.0574GiB/s) | 20260508 12:54:19.997 | INFO | Downloaded 38GiB to GPU in 2.70675s (14.039GiB/s) | 20260508 12:54:25.132 | INFO | Analyzing chunk frames = [8262144,8262656) | 20260508 12:54:25.132 | INFO | MemInfo: { | 20260508 12:54:25.132 | INFO | "gpu" : " 17 GB", | 20260508 12:54:25.132 | INFO | "huge" : "", | 20260508 12:54:25.132 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:25.132 | INFO | "malloc" : " 14 GB", | 20260508 12:54:25.132 | INFO | "pinned" : " 29 GB" | 20260508 12:54:25.132 | INFO | } | 20260508 12:54:25.132 | INFO | Perf report for the last 5.13525 seconds: | 20260508 12:54:25.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:25.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.132 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 12:54:25.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.132 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 12:54:25.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:25.133 | INFO | Uploaded 36GiB to GPU in 2.47279s (14.5585GiB/s) | 20260508 12:54:25.133 | INFO | Downloaded 34GiB to GPU in 2.47965s (13.7116GiB/s) | 20260508 12:54:30.274 | INFO | Analyzing chunk frames = [8262656,8263168) | 20260508 12:54:30.274 | INFO | MemInfo: { | 20260508 12:54:30.274 | INFO | "gpu" : " 17 GB", | 20260508 12:54:30.274 | INFO | "huge" : "", | 20260508 12:54:30.274 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:30.274 | INFO | "malloc" : " 14 GB", | 20260508 12:54:30.274 | INFO | "pinned" : " 29 GB" | 20260508 12:54:30.274 | INFO | } | 20260508 12:54:30.274 | INFO | Perf report for the last 5.14206 seconds: | 20260508 12:54:30.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:30.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.274 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 12:54:30.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.274 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 12:54:30.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:30.275 | INFO | Uploaded 36GiB to GPU in 2.44232s (14.7401GiB/s) | 20260508 12:54:30.275 | INFO | Downloaded 34GiB to GPU in 2.45355s (13.8575GiB/s) | 20260508 12:54:35.415 | INFO | Analyzing chunk frames = [8263168,8263680) | 20260508 12:54:35.415 | INFO | MemInfo: { | 20260508 12:54:35.415 | INFO | "gpu" : " 17 GB", | 20260508 12:54:35.415 | INFO | "huge" : "", | 20260508 12:54:35.415 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:35.415 | INFO | "malloc" : " 14 GB", | 20260508 12:54:35.415 | INFO | "pinned" : " 29 GB" | 20260508 12:54:35.415 | INFO | } | 20260508 12:54:35.415 | INFO | Perf report for the last 5.14102 seconds: | 20260508 12:54:35.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:35.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.415 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 12:54:35.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.415 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:54:35.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:35.416 | INFO | Uploaded 36GiB to GPU in 2.39922s (15.0049GiB/s) | 20260508 12:54:35.416 | INFO | Downloaded 34GiB to GPU in 2.48638s (13.6745GiB/s) | 20260508 12:54:40.549 | INFO | Analyzing chunk frames = [8263680,8264192) | 20260508 12:54:40.549 | INFO | MemInfo: { | 20260508 12:54:40.549 | INFO | "gpu" : " 17 GB", | 20260508 12:54:40.549 | INFO | "huge" : "", | 20260508 12:54:40.549 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:40.549 | INFO | "malloc" : " 14 GB", | 20260508 12:54:40.549 | INFO | "pinned" : " 29 GB" | 20260508 12:54:40.549 | INFO | } | 20260508 12:54:40.549 | INFO | Perf report for the last 5.1337 seconds: | 20260508 12:54:40.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:40.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.549 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 12:54:40.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.549 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:54:40.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:40.550 | INFO | Uploaded 36GiB to GPU in 2.3891s (15.0684GiB/s) | 20260508 12:54:40.550 | INFO | Downloaded 34GiB to GPU in 2.50306s (13.5834GiB/s) | 20260508 12:54:45.692 | INFO | Analyzing chunk frames = [8264192,8264704) | 20260508 12:54:45.692 | INFO | MemInfo: { | 20260508 12:54:45.692 | INFO | "gpu" : " 17 GB", | 20260508 12:54:45.692 | INFO | "huge" : "", | 20260508 12:54:45.692 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:45.692 | INFO | "malloc" : " 14 GB", | 20260508 12:54:45.692 | INFO | "pinned" : " 29 GB" | 20260508 12:54:45.692 | INFO | } | 20260508 12:54:45.692 | INFO | Perf report for the last 5.14303 seconds: | 20260508 12:54:45.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:45.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.692 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 12:54:45.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.692 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 12:54:45.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:45.693 | INFO | Uploaded 36GiB to GPU in 2.37965s (15.1283GiB/s) | 20260508 12:54:45.693 | INFO | Downloaded 34GiB to GPU in 2.43928s (13.9386GiB/s) | 20260508 12:54:50.825 | INFO | Analyzing chunk frames = [8264704,8265216) | 20260508 12:54:50.825 | INFO | MemInfo: { | 20260508 12:54:50.825 | INFO | "gpu" : " 17 GB", | 20260508 12:54:50.825 | INFO | "huge" : "", | 20260508 12:54:50.825 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:50.825 | INFO | "malloc" : " 14 GB", | 20260508 12:54:50.825 | INFO | "pinned" : " 29 GB" | 20260508 12:54:50.825 | INFO | } | 20260508 12:54:50.825 | INFO | Perf report for the last 5.13314 seconds: | 20260508 12:54:50.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:50.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 12:54:50.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 12:54:50.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:50.825 | INFO | Uploaded 36GiB to GPU in 2.29096s (15.7139GiB/s) | 20260508 12:54:50.826 | INFO | Downloaded 34GiB to GPU in 2.41891s (14.0559GiB/s) | 20260508 12:54:55.972 | INFO | Analyzing chunk frames = [8265216,8265728) | 20260508 12:54:55.972 | INFO | MemInfo: { | 20260508 12:54:55.972 | INFO | "gpu" : " 17 GB", | 20260508 12:54:55.972 | INFO | "huge" : "", | 20260508 12:54:55.972 | INFO | "huge_shared" : " 29 GB", | 20260508 12:54:55.972 | INFO | "malloc" : " 14 GB", | 20260508 12:54:55.972 | INFO | "pinned" : " 29 GB" | 20260508 12:54:55.972 | INFO | } | 20260508 12:54:55.972 | INFO | Perf report for the last 5.14707 seconds: | 20260508 12:54:55.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:54:55.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.972 | INFO | ANALYSIS : 65.4%, 3.0, 102.4ms, 0 | 20260508 12:54:55.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.972 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:54:55.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:54:55.973 | INFO | Uploaded 36GiB to GPU in 2.43827s (14.7646GiB/s) | 20260508 12:54:55.973 | INFO | Downloaded 34GiB to GPU in 2.45088s (13.8726GiB/s) | 20260508 12:54:55.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:54:55.973 | INFO | Total mem usage: 16.7 GiB | 20260508 12:54:55.973 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:54:55.973 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:54:55.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:54:55.973 | INFO | High Water Breakdown: | 20260508 12:54:55.973 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:54:55.973 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:54:55.973 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:54:55.973 | INFO | ------------------------------------------ | 20260508 12:54:55.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:54:55.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:54:55.973 | INFO | Total mem usage: 26.5 GiB | 20260508 12:54:55.973 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:54:55.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:54:55.973 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:54:55.973 | INFO | High Water Breakdown: | 20260508 12:54:55.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:54:55.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:54:55.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:54:55.973 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:54:55.973 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:54:55.973 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:54:55.973 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:54:55.973 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:54:55.973 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:54:55.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:54:55.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:54:55.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:54:55.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:54:55.973 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:54:55.973 | INFO | ------------------------------------------ | 20260508 12:54:55.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:54:55.973 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:54:55.973 | INFO | Total mem usage: 27.8 GiB | 20260508 12:54:55.973 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:54:55.973 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:54:55.973 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:54:55.973 | INFO | High Water Breakdown: | 20260508 12:54:55.973 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:54:55.973 | INFO | ------------------------------------------ | 20260508 12:54:55.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:54:55.973 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:54:55.973 | INFO | Total mem usage: 15.7 GiB | 20260508 12:54:55.973 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:54:55.973 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:54:55.973 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:54:55.973 | INFO | High Water Breakdown: | 20260508 12:54:55.973 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:54:55.973 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:54:55.973 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:54:55.973 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:54:55.973 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:54:55.973 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:54:55.973 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:54:55.973 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:54:55.973 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:54:55.973 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:54:55.973 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:54:55.973 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:54:55.973 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:54:55.973 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:54:55.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:54:55.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:54:55.973 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:54:55.973 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:54:55.973 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:54:55.973 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:54:55.973 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:54:55.973 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:54:55.973 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:54:55.973 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:54:55.973 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:54:55.973 | INFO | ------------------------------------------ | 20260508 12:54:55.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:54:55.974 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:54:55.974 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:54:55.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:54:55.974 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.45e-01 | 20260508 12:54:55.974 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.05e-01, 1.08e-02 / 3.49e-03 | 20260508 12:54:55.974 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.40e+00, 6.00e-01 / 2.17e-01 | 20260508 12:54:55.974 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.44e+00, 2.11e-01 / 1.02e-01 | 20260508 12:54:55.974 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.51e+01, 3.61e+00 / 1.08e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.86e+00 / 1.11e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.31e+00 / 2.48e+00, 4.04e-01 / 1.90e-01 | 20260508 12:54:55.974 | INFO | PerfReport: 2.93%, Metrics -- 9.59e-01 / 1.08e+00, 2.38e-01 / 8.55e-02 | 20260508 12:54:55.974 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.22e-01, 4.65e-02 / 1.68e-02 | 20260508 12:54:55.974 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:54:55.974 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:54:55.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:54:55.974 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 24.27%, Upload -- 2.54e+01 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 24.79%, Download -- 2.60e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 31.16%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 2.12%, LockUpload -- 2.22e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 2.61%, LockDownload -- 2.74e+00 / 0.00e+00, 4.13e+00 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 15.05%, LockCompute -- 1.58e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 12:54:55.974 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260508 12:54:56.069 | INFO | Starting a round of Baz IO | 20260508 12:54:59.479 | INFO | Populating RTMetrics report | 20260508 12:54:59.480 | INFO | RTMetrics report populated | 20260508 12:54:59.480 | INFO | Opening tmp RTMetrics json file | 20260508 12:54:59.605 | INFO | Serializing RTMetrics to json | 20260508 12:54:59.621 | INFO | Serialization done, swapping tmp file now... | 20260508 12:54:59.626 | INFO | RTMetrics done | 20260508 12:54:59.636 | INFO | Finished a round of Baz IO: Wrote 1.398 GiB in 3.566 seconds | 20260508 12:54:59.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.00% / 60.01% | 20260508 12:54:59.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15254% / 3.97364e-06% / 9.13938e-05% | 20260508 12:54:59.636 | INFO | Baz IO overall: | 20260508 12:54:59.636 | INFO | BytesWritten : 3534278549764 | 20260508 12:54:59.636 | INFO | NumEvents : 1695342997458 | 20260508 12:55:01.110 | INFO | Analyzing chunk frames = [8265728,8266240) | 20260508 12:55:01.110 | INFO | MemInfo: { | 20260508 12:55:01.110 | INFO | "gpu" : " 17 GB", | 20260508 12:55:01.110 | INFO | "huge" : "", | 20260508 12:55:01.110 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:01.110 | INFO | "malloc" : " 14 GB", | 20260508 12:55:01.110 | INFO | "pinned" : " 29 GB" | 20260508 12:55:01.110 | INFO | } | 20260508 12:55:01.110 | INFO | Perf report for the last 5.13827 seconds: | 20260508 12:55:01.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:01.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:01.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:01.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:01.111 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 12:55:01.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:01.111 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260508 12:55:01.111 | INFO | PRE_HQ : 65.2%, 1.6, 54.9ms, 0 | 20260508 12:55:01.111 | INFO | BAZWRITER : 68.7%, 2.8, 98.2ms, 0 | 20260508 12:55:01.111 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260508 12:55:01.111 | INFO | Uploaded 36GiB to GPU in 2.62676s (13.7051GiB/s) | 20260508 12:55:01.111 | INFO | Downloaded 38GiB to GPU in 2.71022s (14.021GiB/s) | 20260508 12:55:06.250 | INFO | Analyzing chunk frames = [8266240,8266752) | 20260508 12:55:06.250 | INFO | MemInfo: { | 20260508 12:55:06.250 | INFO | "gpu" : " 17 GB", | 20260508 12:55:06.250 | INFO | "huge" : "", | 20260508 12:55:06.250 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:06.250 | INFO | "malloc" : " 14 GB", | 20260508 12:55:06.250 | INFO | "pinned" : " 29 GB" | 20260508 12:55:06.250 | INFO | } | 20260508 12:55:06.250 | INFO | Perf report for the last 5.13985 seconds: | 20260508 12:55:06.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:06.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.250 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:55:06.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.250 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 12:55:06.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:06.251 | INFO | Uploaded 36GiB to GPU in 2.47716s (14.5327GiB/s) | 20260508 12:55:06.251 | INFO | Downloaded 34GiB to GPU in 2.49027s (13.6531GiB/s) | 20260508 12:55:11.382 | INFO | Analyzing chunk frames = [8266752,8267264) | 20260508 12:55:11.382 | INFO | MemInfo: { | 20260508 12:55:11.382 | INFO | "gpu" : " 17 GB", | 20260508 12:55:11.382 | INFO | "huge" : "", | 20260508 12:55:11.382 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:11.382 | INFO | "malloc" : " 14 GB", | 20260508 12:55:11.382 | INFO | "pinned" : " 29 GB" | 20260508 12:55:11.382 | INFO | } | 20260508 12:55:11.382 | INFO | Perf report for the last 5.132 seconds: | 20260508 12:55:11.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:11.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.382 | INFO | ANALYSIS : 66.5%, 3.0, 103.1ms, 0 | 20260508 12:55:11.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.382 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:55:11.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:11.383 | INFO | Uploaded 36GiB to GPU in 2.48954s (14.4605GiB/s) | 20260508 12:55:11.383 | INFO | Downloaded 34GiB to GPU in 2.46152s (13.8126GiB/s) | 20260508 12:55:16.528 | INFO | Analyzing chunk frames = [8267264,8267776) | 20260508 12:55:16.528 | INFO | MemInfo: { | 20260508 12:55:16.528 | INFO | "gpu" : " 17 GB", | 20260508 12:55:16.528 | INFO | "huge" : "", | 20260508 12:55:16.528 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:16.528 | INFO | "malloc" : " 14 GB", | 20260508 12:55:16.528 | INFO | "pinned" : " 29 GB" | 20260508 12:55:16.528 | INFO | } | 20260508 12:55:16.528 | INFO | Perf report for the last 5.14602 seconds: | 20260508 12:55:16.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:16.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.528 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 12:55:16.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.528 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:55:16.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:16.529 | INFO | Uploaded 36GiB to GPU in 2.48385s (14.4936GiB/s) | 20260508 12:55:16.529 | INFO | Downloaded 34GiB to GPU in 2.45683s (13.839GiB/s) | 20260508 12:55:21.666 | INFO | Analyzing chunk frames = [8267776,8268288) | 20260508 12:55:21.666 | INFO | MemInfo: { | 20260508 12:55:21.666 | INFO | "gpu" : " 17 GB", | 20260508 12:55:21.666 | INFO | "huge" : "", | 20260508 12:55:21.666 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:21.666 | INFO | "malloc" : " 14 GB", | 20260508 12:55:21.666 | INFO | "pinned" : " 29 GB" | 20260508 12:55:21.666 | INFO | } | 20260508 12:55:21.667 | INFO | Perf report for the last 5.13817 seconds: | 20260508 12:55:21.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:21.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 12:55:21.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:55:21.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:21.667 | INFO | Uploaded 36GiB to GPU in 2.32509s (15.4832GiB/s) | 20260508 12:55:21.667 | INFO | Downloaded 34GiB to GPU in 2.50453s (13.5754GiB/s) | 20260508 12:55:26.811 | INFO | Analyzing chunk frames = [8268288,8268800) | 20260508 12:55:26.811 | INFO | MemInfo: { | 20260508 12:55:26.811 | INFO | "gpu" : " 17 GB", | 20260508 12:55:26.811 | INFO | "huge" : "", | 20260508 12:55:26.811 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:26.811 | INFO | "malloc" : " 14 GB", | 20260508 12:55:26.811 | INFO | "pinned" : " 29 GB" | 20260508 12:55:26.811 | INFO | } | 20260508 12:55:26.811 | INFO | Perf report for the last 5.14465 seconds: | 20260508 12:55:26.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:26.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.811 | INFO | ANALYSIS : 65.1%, 3.0, 101.4ms, 0 | 20260508 12:55:26.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.811 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:55:26.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:26.812 | INFO | Uploaded 36GiB to GPU in 2.36915s (15.1954GiB/s) | 20260508 12:55:26.812 | INFO | Downloaded 34GiB to GPU in 2.45779s (13.8336GiB/s) | 20260508 12:55:31.952 | INFO | Analyzing chunk frames = [8268800,8269312) | 20260508 12:55:31.952 | INFO | MemInfo: { | 20260508 12:55:31.952 | INFO | "gpu" : " 17 GB", | 20260508 12:55:31.952 | INFO | "huge" : "", | 20260508 12:55:31.952 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:31.952 | INFO | "malloc" : " 14 GB", | 20260508 12:55:31.952 | INFO | "pinned" : " 29 GB" | 20260508 12:55:31.952 | INFO | } | 20260508 12:55:31.952 | INFO | Perf report for the last 5.14083 seconds: | 20260508 12:55:31.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:31.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | ANALYSIS : 64.5%, 3.0, 100.4ms, 0 | 20260508 12:55:31.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:55:31.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:31.952 | INFO | Uploaded 36GiB to GPU in 2.29069s (15.7158GiB/s) | 20260508 12:55:31.952 | INFO | Downloaded 34GiB to GPU in 2.4389s (13.9407GiB/s) | 20260508 12:55:37.080 | INFO | Analyzing chunk frames = [8269312,8269824) | 20260508 12:55:37.080 | INFO | MemInfo: { | 20260508 12:55:37.080 | INFO | "gpu" : " 17 GB", | 20260508 12:55:37.080 | INFO | "huge" : "", | 20260508 12:55:37.080 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:37.080 | INFO | "malloc" : " 14 GB", | 20260508 12:55:37.080 | INFO | "pinned" : " 29 GB" | 20260508 12:55:37.080 | INFO | } | 20260508 12:55:37.080 | INFO | Perf report for the last 5.12795 seconds: | 20260508 12:55:37.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:37.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | ANALYSIS : 65.0%, 3.0, 101.1ms, 0 | 20260508 12:55:37.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:55:37.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:37.080 | INFO | Uploaded 36GiB to GPU in 2.32323s (15.4957GiB/s) | 20260508 12:55:37.080 | INFO | Downloaded 34GiB to GPU in 2.46254s (13.8069GiB/s) | 20260508 12:55:37.179 | INFO | Starting a round of Baz IO | 20260508 12:55:40.585 | INFO | Populating RTMetrics report | 20260508 12:55:40.586 | INFO | RTMetrics report populated | 20260508 12:55:40.586 | INFO | Opening tmp RTMetrics json file | 20260508 12:55:40.591 | INFO | Serializing RTMetrics to json | 20260508 12:55:40.592 | INFO | Serialization done, swapping tmp file now... | 20260508 12:55:40.593 | INFO | RTMetrics done | 20260508 12:55:40.650 | INFO | Finished a round of Baz IO: Wrote 1.399 GiB in 3.47 seconds | 20260508 12:55:40.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.01% | 20260508 12:55:40.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153136% / 0% / 9.13938e-05% | 20260508 12:55:40.650 | INFO | Baz IO overall: | 20260508 12:55:40.650 | INFO | BytesWritten : 3535780591492 | 20260508 12:55:40.650 | INFO | NumEvents : 1696047076079 | 20260508 12:55:42.227 | INFO | Analyzing chunk frames = [8269824,8270336) | 20260508 12:55:42.227 | INFO | MemInfo: { | 20260508 12:55:42.227 | INFO | "gpu" : " 17 GB", | 20260508 12:55:42.227 | INFO | "huge" : "", | 20260508 12:55:42.227 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:42.227 | INFO | "malloc" : " 14 GB", | 20260508 12:55:42.227 | INFO | "pinned" : " 29 GB" | 20260508 12:55:42.227 | INFO | } | 20260508 12:55:42.227 | INFO | Perf report for the last 5.14703 seconds: | 20260508 12:55:42.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:42.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:42.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:42.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:42.227 | INFO | ANALYSIS : 67.2%, 3.0, 105.3ms, 0 | 20260508 12:55:42.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:42.227 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 12:55:42.227 | INFO | PRE_HQ : 65.0%, 1.6, 53.5ms, 0 | 20260508 12:55:42.227 | INFO | BAZWRITER : 66.7%, 2.5, 84.6ms, 0 | 20260508 12:55:42.227 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:55:42.227 | INFO | Uploaded 36GiB to GPU in 2.59758s (13.8591GiB/s) | 20260508 12:55:42.227 | INFO | Downloaded 38GiB to GPU in 2.71451s (13.9988GiB/s) | 20260508 12:55:47.361 | INFO | Analyzing chunk frames = [8270336,8270848) | 20260508 12:55:47.361 | INFO | MemInfo: { | 20260508 12:55:47.361 | INFO | "gpu" : " 17 GB", | 20260508 12:55:47.361 | INFO | "huge" : "", | 20260508 12:55:47.361 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:47.361 | INFO | "malloc" : " 14 GB", | 20260508 12:55:47.361 | INFO | "pinned" : " 29 GB" | 20260508 12:55:47.361 | INFO | } | 20260508 12:55:47.361 | INFO | Perf report for the last 5.13415 seconds: | 20260508 12:55:47.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:47.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.361 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:55:47.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.362 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:55:47.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:47.362 | INFO | Uploaded 36GiB to GPU in 2.46842s (14.5843GiB/s) | 20260508 12:55:47.362 | INFO | Downloaded 34GiB to GPU in 2.46987s (13.7659GiB/s) | 20260508 12:55:52.501 | INFO | Analyzing chunk frames = [8270848,8271360) | 20260508 12:55:52.501 | INFO | MemInfo: { | 20260508 12:55:52.501 | INFO | "gpu" : " 17 GB", | 20260508 12:55:52.501 | INFO | "huge" : "", | 20260508 12:55:52.501 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:52.501 | INFO | "malloc" : " 14 GB", | 20260508 12:55:52.501 | INFO | "pinned" : " 29 GB" | 20260508 12:55:52.501 | INFO | } | 20260508 12:55:52.501 | INFO | Perf report for the last 5.1399 seconds: | 20260508 12:55:52.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:52.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.501 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 12:55:52.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.501 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:55:52.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:52.502 | INFO | Uploaded 36GiB to GPU in 2.47602s (14.5394GiB/s) | 20260508 12:55:52.502 | INFO | Downloaded 34GiB to GPU in 2.43789s (13.9465GiB/s) | 20260508 12:55:57.635 | INFO | Analyzing chunk frames = [8271360,8271872) | 20260508 12:55:57.635 | INFO | MemInfo: { | 20260508 12:55:57.635 | INFO | "gpu" : " 17 GB", | 20260508 12:55:57.635 | INFO | "huge" : "", | 20260508 12:55:57.635 | INFO | "huge_shared" : " 29 GB", | 20260508 12:55:57.636 | INFO | "malloc" : " 14 GB", | 20260508 12:55:57.636 | INFO | "pinned" : " 29 GB" | 20260508 12:55:57.636 | INFO | } | 20260508 12:55:57.636 | INFO | Perf report for the last 5.13441 seconds: | 20260508 12:55:57.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:55:57.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | ANALYSIS : 66.1%, 3.0, 102.3ms, 0 | 20260508 12:55:57.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:55:57.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:55:57.636 | INFO | Uploaded 36GiB to GPU in 2.46606s (14.5982GiB/s) | 20260508 12:55:57.636 | INFO | Downloaded 34GiB to GPU in 2.45049s (13.8748GiB/s) | 20260508 12:56:02.780 | INFO | Analyzing chunk frames = [8271872,8272384) | 20260508 12:56:02.780 | INFO | MemInfo: { | 20260508 12:56:02.780 | INFO | "gpu" : " 17 GB", | 20260508 12:56:02.780 | INFO | "huge" : "", | 20260508 12:56:02.780 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:02.780 | INFO | "malloc" : " 14 GB", | 20260508 12:56:02.780 | INFO | "pinned" : " 29 GB" | 20260508 12:56:02.780 | INFO | } | 20260508 12:56:02.780 | INFO | Perf report for the last 5.14475 seconds: | 20260508 12:56:02.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:02.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.780 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:56:02.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.780 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 12:56:02.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:02.781 | INFO | Uploaded 36GiB to GPU in 2.30786s (15.5989GiB/s) | 20260508 12:56:02.781 | INFO | Downloaded 34GiB to GPU in 2.50809s (13.5561GiB/s) | 20260508 12:56:07.926 | INFO | Analyzing chunk frames = [8272384,8272896) | 20260508 12:56:07.926 | INFO | MemInfo: { | 20260508 12:56:07.926 | INFO | "gpu" : " 17 GB", | 20260508 12:56:07.926 | INFO | "huge" : "", | 20260508 12:56:07.926 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:07.926 | INFO | "malloc" : " 14 GB", | 20260508 12:56:07.926 | INFO | "pinned" : " 29 GB" | 20260508 12:56:07.926 | INFO | } | 20260508 12:56:07.926 | INFO | Perf report for the last 5.14572 seconds: | 20260508 12:56:07.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:07.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 12:56:07.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 12:56:07.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:07.926 | INFO | Uploaded 36GiB to GPU in 2.44076s (14.7495GiB/s) | 20260508 12:56:07.927 | INFO | Downloaded 34GiB to GPU in 2.43511s (13.9624GiB/s) | 20260508 12:56:13.064 | INFO | Analyzing chunk frames = [8272896,8273408) | 20260508 12:56:13.064 | INFO | MemInfo: { | 20260508 12:56:13.064 | INFO | "gpu" : " 17 GB", | 20260508 12:56:13.064 | INFO | "huge" : "", | 20260508 12:56:13.064 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:13.064 | INFO | "malloc" : " 14 GB", | 20260508 12:56:13.064 | INFO | "pinned" : " 29 GB" | 20260508 12:56:13.064 | INFO | } | 20260508 12:56:13.064 | INFO | Perf report for the last 5.13801 seconds: | 20260508 12:56:13.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:13.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.064 | INFO | ANALYSIS : 64.6%, 3.0, 100.5ms, 0 | 20260508 12:56:13.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.064 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260508 12:56:13.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:13.065 | INFO | Uploaded 36GiB to GPU in 2.31062s (15.5802GiB/s) | 20260508 12:56:13.065 | INFO | Downloaded 34GiB to GPU in 2.38773s (14.2395GiB/s) | 20260508 12:56:18.196 | INFO | Analyzing chunk frames = [8273408,8273920) | 20260508 12:56:18.196 | INFO | MemInfo: { | 20260508 12:56:18.196 | INFO | "gpu" : " 17 GB", | 20260508 12:56:18.196 | INFO | "huge" : "", | 20260508 12:56:18.196 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:18.196 | INFO | "malloc" : " 14 GB", | 20260508 12:56:18.196 | INFO | "pinned" : " 29 GB" | 20260508 12:56:18.196 | INFO | } | 20260508 12:56:18.196 | INFO | Perf report for the last 5.1321 seconds: | 20260508 12:56:18.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:18.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.196 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260508 12:56:18.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.196 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:56:18.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:18.197 | INFO | Uploaded 36GiB to GPU in 2.37814s (15.1379GiB/s) | 20260508 12:56:18.197 | INFO | Downloaded 34GiB to GPU in 2.39787s (14.1792GiB/s) | 20260508 12:56:18.197 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:56:18.197 | INFO | Total mem usage: 16.7 GiB | 20260508 12:56:18.197 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:56:18.197 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15%) | 20260508 12:56:18.197 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:56:18.197 | INFO | High Water Breakdown: | 20260508 12:56:18.197 | INFO | 59.7% : 8.67 GiB ( 84.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:56:18.197 | INFO | 40.1% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:56:18.197 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:56:18.197 | INFO | ------------------------------------------ | 20260508 12:56:18.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:56:18.197 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:56:18.197 | INFO | Total mem usage: 26.5 GiB | 20260508 12:56:18.197 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:56:18.197 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:56:18.197 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:56:18.197 | INFO | High Water Breakdown: | 20260508 12:56:18.197 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:56:18.197 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:56:18.197 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:56:18.197 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:56:18.197 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:56:18.197 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:56:18.197 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:56:18.197 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:56:18.197 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:56:18.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:56:18.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:56:18.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:56:18.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:56:18.197 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:56:18.197 | INFO | ------------------------------------------ | 20260508 12:56:18.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:56:18.197 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:56:18.197 | INFO | Total mem usage: 27.8 GiB | 20260508 12:56:18.197 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:56:18.197 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:56:18.197 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:56:18.197 | INFO | High Water Breakdown: | 20260508 12:56:18.197 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:56:18.197 | INFO | ------------------------------------------ | 20260508 12:56:18.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:56:18.197 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:56:18.197 | INFO | Total mem usage: 15.7 GiB | 20260508 12:56:18.197 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:56:18.197 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:56:18.197 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:56:18.197 | INFO | High Water Breakdown: | 20260508 12:56:18.197 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:56:18.197 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:56:18.197 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:56:18.197 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:56:18.197 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:56:18.197 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:56:18.197 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:56:18.197 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:56:18.197 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:56:18.197 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:56:18.197 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:56:18.197 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:56:18.197 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:56:18.197 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:56:18.197 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:56:18.197 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:56:18.197 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:56:18.197 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:56:18.197 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:56:18.197 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:56:18.197 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:56:18.197 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:56:18.197 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:56:18.197 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:56:18.197 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:56:18.197 | INFO | ------------------------------------------ | 20260508 12:56:18.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:56:18.197 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:56:18.197 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:56:18.197 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:56:18.197 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.56e+01, 1.39e+01 / 5.57e-01 | 20260508 12:56:18.197 | INFO | PerfReport: 0.30%, AggregateStats -- 9.72e-02 / 1.04e-01, 1.14e-02 / 5.11e-03 | 20260508 12:56:18.197 | INFO | PerfReport: 12.25%, Baseline -- 4.00e+00 / 4.33e+00, 5.96e-01 / 2.59e-01 | 20260508 12:56:18.197 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 8.62e-02 | 20260508 12:56:18.197 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.60e+01, 3.82e+00 / 1.14e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.85e+00 / 1.23e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.32e+00 / 2.47e+00, 4.03e-01 / 2.26e-01 | 20260508 12:56:18.197 | INFO | PerfReport: 2.94%, Metrics -- 9.60e-01 / 1.09e+00, 2.38e-01 / 7.76e-02 | 20260508 12:56:18.197 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.11e-01, 4.71e-02 / 1.79e-02 | 20260508 12:56:18.197 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:56:18.197 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:56:18.197 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:56:18.197 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 24.17%, Upload -- 2.53e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 24.76%, Download -- 2.59e+01 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 31.27%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 2.13%, LockUpload -- 2.22e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 2.60%, LockDownload -- 2.72e+00 / 0.00e+00, 4.11e+00 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 15.06%, LockCompute -- 1.58e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 12:56:18.197 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.45e-02 / 0.00e+00 | 20260508 12:56:18.293 | INFO | Starting a round of Baz IO | 20260508 12:56:21.690 | INFO | Populating RTMetrics report | 20260508 12:56:21.691 | INFO | RTMetrics report populated | 20260508 12:56:21.691 | INFO | Opening tmp RTMetrics json file | 20260508 12:56:21.701 | INFO | Serializing RTMetrics to json | 20260508 12:56:21.702 | INFO | Serialization done, swapping tmp file now... | 20260508 12:56:21.703 | INFO | RTMetrics done | 20260508 12:56:21.764 | INFO | Finished a round of Baz IO: Wrote 1.398 GiB in 3.47 seconds | 20260508 12:56:21.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 12:56:21.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152814% / 3.97364e-06% / 9.93411e-05% | 20260508 12:56:21.764 | INFO | Baz IO overall: | 20260508 12:56:21.764 | INFO | BytesWritten : 3537281303710 | 20260508 12:56:21.764 | INFO | NumEvents : 1696750230422 | 20260508 12:56:23.334 | INFO | Analyzing chunk frames = [8273920,8274432) | 20260508 12:56:23.334 | INFO | MemInfo: { | 20260508 12:56:23.334 | INFO | "gpu" : " 17 GB", | 20260508 12:56:23.334 | INFO | "huge" : "", | 20260508 12:56:23.334 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:23.334 | INFO | "malloc" : " 14 GB", | 20260508 12:56:23.334 | INFO | "pinned" : " 29 GB" | 20260508 12:56:23.334 | INFO | } | 20260508 12:56:23.334 | INFO | Perf report for the last 5.13739 seconds: | 20260508 12:56:23.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:23.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:23.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:23.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:23.334 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 12:56:23.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:23.334 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:56:23.334 | INFO | PRE_HQ : 65.1%, 1.6, 53.4ms, 0 | 20260508 12:56:23.334 | INFO | BAZWRITER : 66.9%, 2.5, 86.2ms, 0 | 20260508 12:56:23.334 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:56:23.334 | INFO | Uploaded 36GiB to GPU in 2.62488s (13.7149GiB/s) | 20260508 12:56:23.334 | INFO | Downloaded 38GiB to GPU in 2.72112s (13.9648GiB/s) | 20260508 12:56:28.477 | INFO | Analyzing chunk frames = [8274432,8274944) | 20260508 12:56:28.477 | INFO | MemInfo: { | 20260508 12:56:28.477 | INFO | "gpu" : " 17 GB", | 20260508 12:56:28.477 | INFO | "huge" : "", | 20260508 12:56:28.477 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:28.477 | INFO | "malloc" : " 14 GB", | 20260508 12:56:28.477 | INFO | "pinned" : " 29 GB" | 20260508 12:56:28.477 | INFO | } | 20260508 12:56:28.477 | INFO | Perf report for the last 5.14319 seconds: | 20260508 12:56:28.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:28.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 12:56:28.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 12:56:28.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:28.477 | INFO | Uploaded 36GiB to GPU in 2.47005s (14.5746GiB/s) | 20260508 12:56:28.477 | INFO | Downloaded 34GiB to GPU in 2.46664s (13.7839GiB/s) | 20260508 12:56:33.627 | INFO | Analyzing chunk frames = [8274944,8275456) | 20260508 12:56:33.627 | INFO | MemInfo: { | 20260508 12:56:33.627 | INFO | "gpu" : " 17 GB", | 20260508 12:56:33.627 | INFO | "huge" : "", | 20260508 12:56:33.627 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:33.627 | INFO | "malloc" : " 14 GB", | 20260508 12:56:33.627 | INFO | "pinned" : " 29 GB" | 20260508 12:56:33.627 | INFO | } | 20260508 12:56:33.627 | INFO | Perf report for the last 5.15016 seconds: | 20260508 12:56:33.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:33.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:56:33.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 12:56:33.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:33.627 | INFO | Uploaded 36GiB to GPU in 2.51519s (14.313GiB/s) | 20260508 12:56:33.627 | INFO | Downloaded 34GiB to GPU in 2.44896s (13.8834GiB/s) | 20260508 12:56:38.759 | INFO | Analyzing chunk frames = [8275456,8275968) | 20260508 12:56:38.759 | INFO | MemInfo: { | 20260508 12:56:38.759 | INFO | "gpu" : " 17 GB", | 20260508 12:56:38.759 | INFO | "huge" : "", | 20260508 12:56:38.759 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:38.759 | INFO | "malloc" : " 14 GB", | 20260508 12:56:38.759 | INFO | "pinned" : " 29 GB" | 20260508 12:56:38.759 | INFO | } | 20260508 12:56:38.759 | INFO | Perf report for the last 5.13209 seconds: | 20260508 12:56:38.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:38.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.759 | INFO | ANALYSIS : 65.5%, 3.0, 101.5ms, 0 | 20260508 12:56:38.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.759 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 12:56:38.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:38.760 | INFO | Uploaded 36GiB to GPU in 2.42984s (14.8158GiB/s) | 20260508 12:56:38.760 | INFO | Downloaded 34GiB to GPU in 2.47073s (13.7611GiB/s) | 20260508 12:56:43.898 | INFO | Analyzing chunk frames = [8275968,8276480) | 20260508 12:56:43.898 | INFO | MemInfo: { | 20260508 12:56:43.898 | INFO | "gpu" : " 17 GB", | 20260508 12:56:43.898 | INFO | "huge" : "", | 20260508 12:56:43.898 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:43.898 | INFO | "malloc" : " 14 GB", | 20260508 12:56:43.899 | INFO | "pinned" : " 29 GB" | 20260508 12:56:43.899 | INFO | } | 20260508 12:56:43.899 | INFO | Perf report for the last 5.13949 seconds: | 20260508 12:56:43.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:43.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | ANALYSIS : 66.7%, 3.0, 103.9ms, 0 | 20260508 12:56:43.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 12:56:43.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:43.899 | INFO | Uploaded 36GiB to GPU in 2.51459s (14.3165GiB/s) | 20260508 12:56:43.899 | INFO | Downloaded 34GiB to GPU in 2.52414s (13.4699GiB/s) | 20260508 12:56:49.038 | INFO | Analyzing chunk frames = [8276480,8276992) | 20260508 12:56:49.038 | INFO | MemInfo: { | 20260508 12:56:49.038 | INFO | "gpu" : " 17 GB", | 20260508 12:56:49.038 | INFO | "huge" : "", | 20260508 12:56:49.038 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:49.038 | INFO | "malloc" : " 14 GB", | 20260508 12:56:49.039 | INFO | "pinned" : " 29 GB" | 20260508 12:56:49.039 | INFO | } | 20260508 12:56:49.039 | INFO | Perf report for the last 5.1401 seconds: | 20260508 12:56:49.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:49.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 12:56:49.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:56:49.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:49.039 | INFO | Uploaded 36GiB to GPU in 2.45319s (14.6747GiB/s) | 20260508 12:56:49.039 | INFO | Downloaded 34GiB to GPU in 2.43716s (13.9507GiB/s) | 20260508 12:56:54.173 | INFO | Analyzing chunk frames = [8276992,8277504) | 20260508 12:56:54.173 | INFO | MemInfo: { | 20260508 12:56:54.173 | INFO | "gpu" : " 17 GB", | 20260508 12:56:54.173 | INFO | "huge" : "", | 20260508 12:56:54.173 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:54.173 | INFO | "malloc" : " 14 GB", | 20260508 12:56:54.173 | INFO | "pinned" : " 29 GB" | 20260508 12:56:54.173 | INFO | } | 20260508 12:56:54.173 | INFO | Perf report for the last 5.13463 seconds: | 20260508 12:56:54.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:54.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.173 | INFO | ANALYSIS : 64.6%, 3.0, 100.5ms, 0 | 20260508 12:56:54.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.173 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:56:54.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:54.174 | INFO | Uploaded 36GiB to GPU in 2.3461s (15.3446GiB/s) | 20260508 12:56:54.174 | INFO | Downloaded 34GiB to GPU in 2.39604s (14.1901GiB/s) | 20260508 12:56:59.310 | INFO | Analyzing chunk frames = [8277504,8278016) | 20260508 12:56:59.310 | INFO | MemInfo: { | 20260508 12:56:59.310 | INFO | "gpu" : " 17 GB", | 20260508 12:56:59.310 | INFO | "huge" : "", | 20260508 12:56:59.310 | INFO | "huge_shared" : " 29 GB", | 20260508 12:56:59.311 | INFO | "malloc" : " 14 GB", | 20260508 12:56:59.311 | INFO | "pinned" : " 29 GB" | 20260508 12:56:59.311 | INFO | } | 20260508 12:56:59.311 | INFO | Perf report for the last 5.13727 seconds: | 20260508 12:56:59.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:56:59.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 12:56:59.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 12:56:59.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:56:59.311 | INFO | Uploaded 36GiB to GPU in 2.47798s (14.5279GiB/s) | 20260508 12:56:59.311 | INFO | Downloaded 34GiB to GPU in 2.44844s (13.8864GiB/s) | 20260508 12:56:59.408 | INFO | Starting a round of Baz IO | 20260508 12:57:02.814 | INFO | Populating RTMetrics report | 20260508 12:57:02.814 | INFO | RTMetrics report populated | 20260508 12:57:02.814 | INFO | Opening tmp RTMetrics json file | 20260508 12:57:02.827 | INFO | Serializing RTMetrics to json | 20260508 12:57:02.828 | INFO | Serialization done, swapping tmp file now... | 20260508 12:57:02.828 | INFO | RTMetrics done | 20260508 12:57:02.892 | INFO | Finished a round of Baz IO: Wrote 1.397 GiB in 3.483 seconds | 20260508 12:57:02.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 12:57:02.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152552% / 3.97364e-06% / 7.94729e-05% | 20260508 12:57:02.892 | INFO | Baz IO overall: | 20260508 12:57:02.892 | INFO | BytesWritten : 3538781211373 | 20260508 12:57:02.892 | INFO | NumEvents : 1697453151389 | 20260508 12:57:04.452 | INFO | Analyzing chunk frames = [8278016,8278528) | 20260508 12:57:04.452 | INFO | MemInfo: { | 20260508 12:57:04.452 | INFO | "gpu" : " 17 GB", | 20260508 12:57:04.452 | INFO | "huge" : "", | 20260508 12:57:04.452 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:04.452 | INFO | "malloc" : " 14 GB", | 20260508 12:57:04.452 | INFO | "pinned" : " 29 GB" | 20260508 12:57:04.452 | INFO | } | 20260508 12:57:04.452 | INFO | Perf report for the last 5.14124 seconds: | 20260508 12:57:04.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:04.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:04.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:04.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:04.452 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 12:57:04.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:04.452 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:57:04.452 | INFO | PRE_HQ : 65.5%, 1.6, 54.0ms, 0 | 20260508 12:57:04.452 | INFO | BAZWRITER : 67.2%, 2.5, 86.2ms, 0 | 20260508 12:57:04.452 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:57:04.452 | INFO | Uploaded 36GiB to GPU in 2.64361s (13.6177GiB/s) | 20260508 12:57:04.452 | INFO | Downloaded 38GiB to GPU in 2.71797s (13.981GiB/s) | 20260508 12:57:09.590 | INFO | Analyzing chunk frames = [8278528,8279040) | 20260508 12:57:09.590 | INFO | MemInfo: { | 20260508 12:57:09.590 | INFO | "gpu" : " 17 GB", | 20260508 12:57:09.590 | INFO | "huge" : "", | 20260508 12:57:09.590 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:09.590 | INFO | "malloc" : " 14 GB", | 20260508 12:57:09.590 | INFO | "pinned" : " 29 GB" | 20260508 12:57:09.590 | INFO | } | 20260508 12:57:09.590 | INFO | Perf report for the last 5.1384 seconds: | 20260508 12:57:09.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:09.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.591 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:57:09.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.591 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:57:09.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:09.591 | INFO | Uploaded 36GiB to GPU in 2.46694s (14.593GiB/s) | 20260508 12:57:09.591 | INFO | Downloaded 34GiB to GPU in 2.46423s (13.7974GiB/s) | 20260508 12:57:14.735 | INFO | Analyzing chunk frames = [8279040,8279552) | 20260508 12:57:14.735 | INFO | MemInfo: { | 20260508 12:57:14.735 | INFO | "gpu" : " 17 GB", | 20260508 12:57:14.735 | INFO | "huge" : "", | 20260508 12:57:14.735 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:14.735 | INFO | "malloc" : " 14 GB", | 20260508 12:57:14.735 | INFO | "pinned" : " 29 GB" | 20260508 12:57:14.735 | INFO | } | 20260508 12:57:14.735 | INFO | Perf report for the last 5.14444 seconds: | 20260508 12:57:14.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:14.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 12:57:14.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 12:57:14.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:14.735 | INFO | Uploaded 36GiB to GPU in 2.5274s (14.2439GiB/s) | 20260508 12:57:14.735 | INFO | Downloaded 34GiB to GPU in 2.44196s (13.9232GiB/s) | 20260508 12:57:19.876 | INFO | Analyzing chunk frames = [8279552,8280064) | 20260508 12:57:19.876 | INFO | MemInfo: { | 20260508 12:57:19.876 | INFO | "gpu" : " 17 GB", | 20260508 12:57:19.876 | INFO | "huge" : "", | 20260508 12:57:19.876 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:19.876 | INFO | "malloc" : " 14 GB", | 20260508 12:57:19.876 | INFO | "pinned" : " 29 GB" | 20260508 12:57:19.876 | INFO | } | 20260508 12:57:19.876 | INFO | Perf report for the last 5.1415 seconds: | 20260508 12:57:19.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:19.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.876 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 12:57:19.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.877 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 12:57:19.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:19.877 | INFO | Uploaded 36GiB to GPU in 2.54245s (14.1596GiB/s) | 20260508 12:57:19.877 | INFO | Downloaded 34GiB to GPU in 2.46128s (13.8139GiB/s) | 20260508 12:57:25.010 | INFO | Analyzing chunk frames = [8280064,8280576) | 20260508 12:57:25.010 | INFO | MemInfo: { | 20260508 12:57:25.010 | INFO | "gpu" : " 17 GB", | 20260508 12:57:25.010 | INFO | "huge" : "", | 20260508 12:57:25.010 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:25.010 | INFO | "malloc" : " 14 GB", | 20260508 12:57:25.010 | INFO | "pinned" : " 29 GB" | 20260508 12:57:25.010 | INFO | } | 20260508 12:57:25.010 | INFO | Perf report for the last 5.13351 seconds: | 20260508 12:57:25.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:25.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 12:57:25.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 12:57:25.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:25.010 | INFO | Uploaded 36GiB to GPU in 2.48712s (14.4746GiB/s) | 20260508 12:57:25.010 | INFO | Downloaded 34GiB to GPU in 2.5211s (13.4862GiB/s) | 20260508 12:57:30.147 | INFO | Analyzing chunk frames = [8280576,8281088) | 20260508 12:57:30.147 | INFO | MemInfo: { | 20260508 12:57:30.147 | INFO | "gpu" : " 17 GB", | 20260508 12:57:30.147 | INFO | "huge" : "", | 20260508 12:57:30.147 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:30.147 | INFO | "malloc" : " 14 GB", | 20260508 12:57:30.147 | INFO | "pinned" : " 29 GB" | 20260508 12:57:30.147 | INFO | } | 20260508 12:57:30.147 | INFO | Perf report for the last 5.13721 seconds: | 20260508 12:57:30.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:30.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 12:57:30.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:57:30.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:30.147 | INFO | Uploaded 36GiB to GPU in 2.41023s (14.9363GiB/s) | 20260508 12:57:30.147 | INFO | Downloaded 34GiB to GPU in 2.43562s (13.9595GiB/s) | 20260508 12:57:35.293 | INFO | Analyzing chunk frames = [8281088,8281600) | 20260508 12:57:35.293 | INFO | MemInfo: { | 20260508 12:57:35.293 | INFO | "gpu" : " 17 GB", | 20260508 12:57:35.293 | INFO | "huge" : "", | 20260508 12:57:35.293 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:35.293 | INFO | "malloc" : " 14 GB", | 20260508 12:57:35.293 | INFO | "pinned" : " 29 GB" | 20260508 12:57:35.293 | INFO | } | 20260508 12:57:35.293 | INFO | Perf report for the last 5.14613 seconds: | 20260508 12:57:35.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:35.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.293 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 12:57:35.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.293 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:57:35.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:35.294 | INFO | Uploaded 36GiB to GPU in 2.46022s (14.6328GiB/s) | 20260508 12:57:35.294 | INFO | Downloaded 34GiB to GPU in 2.45615s (13.8428GiB/s) | 20260508 12:57:40.428 | INFO | Analyzing chunk frames = [8281600,8282112) | 20260508 12:57:40.428 | INFO | MemInfo: { | 20260508 12:57:40.428 | INFO | "gpu" : " 17 GB", | 20260508 12:57:40.428 | INFO | "huge" : "", | 20260508 12:57:40.428 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:40.428 | INFO | "malloc" : " 14 GB", | 20260508 12:57:40.428 | INFO | "pinned" : " 29 GB" | 20260508 12:57:40.428 | INFO | } | 20260508 12:57:40.428 | INFO | Perf report for the last 5.13499 seconds: | 20260508 12:57:40.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:40.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260508 12:57:40.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:57:40.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:40.428 | INFO | Uploaded 36GiB to GPU in 2.42582s (14.8403GiB/s) | 20260508 12:57:40.429 | INFO | Downloaded 34GiB to GPU in 2.43141s (13.9837GiB/s) | 20260508 12:57:40.429 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:57:40.429 | INFO | Total mem usage: 16.7 GiB | 20260508 12:57:40.429 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:57:40.429 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:57:40.429 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:57:40.429 | INFO | High Water Breakdown: | 20260508 12:57:40.429 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:57:40.429 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:57:40.429 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:57:40.429 | INFO | ------------------------------------------ | 20260508 12:57:40.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:57:40.429 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:57:40.429 | INFO | Total mem usage: 26.5 GiB | 20260508 12:57:40.429 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:57:40.429 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 12:57:40.429 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 12:57:40.429 | INFO | High Water Breakdown: | 20260508 12:57:40.429 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:57:40.429 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:57:40.429 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:57:40.429 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:57:40.429 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:57:40.429 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:57:40.429 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:57:40.429 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:57:40.429 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:57:40.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:57:40.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:57:40.429 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:57:40.429 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:57:40.429 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:57:40.429 | INFO | ------------------------------------------ | 20260508 12:57:40.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:57:40.429 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:57:40.429 | INFO | Total mem usage: 27.8 GiB | 20260508 12:57:40.429 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:57:40.429 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:57:40.429 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:57:40.429 | INFO | High Water Breakdown: | 20260508 12:57:40.429 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:57:40.429 | INFO | ------------------------------------------ | 20260508 12:57:40.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:57:40.429 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:57:40.429 | INFO | Total mem usage: 15.7 GiB | 20260508 12:57:40.429 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:57:40.429 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:57:40.429 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:57:40.429 | INFO | High Water Breakdown: | 20260508 12:57:40.429 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:57:40.429 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:57:40.429 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:57:40.429 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:57:40.429 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:57:40.429 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:57:40.429 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:57:40.429 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:57:40.429 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:57:40.429 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:57:40.429 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:57:40.429 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:57:40.429 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:57:40.429 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:57:40.429 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:57:40.429 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:57:40.429 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:57:40.429 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:57:40.429 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:57:40.429 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:57:40.429 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:57:40.429 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:57:40.429 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:57:40.429 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:57:40.429 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:57:40.429 | INFO | ------------------------------------------ | 20260508 12:57:40.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:57:40.429 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:57:40.429 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:57:40.429 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:57:40.429 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 4.46e-01 | 20260508 12:57:40.429 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.02e-01, 1.12e-02 / 5.86e-03 | 20260508 12:57:40.430 | INFO | PerfReport: 12.19%, Baseline -- 3.97e+00 / 4.43e+00, 5.90e-01 / 1.91e-01 | 20260508 12:57:40.430 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.42e+00, 2.08e-01 / 1.11e-01 | 20260508 12:57:40.430 | INFO | PerfReport: 15.99%, DME -- 4.16e+01 / 4.48e+01, 3.57e+00 / 7.29e-01 | 20260508 12:57:40.430 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.86e+01 / 2.08e+01, 1.80e+00 / 1.17e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.32e+00 / 2.39e+00, 4.02e-01 / 2.51e-01 | 20260508 12:57:40.430 | INFO | PerfReport: 2.95%, Metrics -- 9.60e-01 / 1.07e+00, 2.36e-01 / 8.50e-02 | 20260508 12:57:40.430 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.14e-01, 4.64e-02 / 2.22e-02 | 20260508 12:57:40.430 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:57:40.430 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:57:40.430 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:57:40.430 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 24.69%, Upload -- 2.59e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 24.72%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 31.03%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 2.28%, LockUpload -- 2.39e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 2.47%, LockDownload -- 2.60e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 14.79%, LockCompute -- 1.55e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 12:57:40.430 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260508 12:57:40.525 | INFO | Starting a round of Baz IO | 20260508 12:57:43.944 | INFO | Populating RTMetrics report | 20260508 12:57:43.944 | INFO | RTMetrics report populated | 20260508 12:57:43.944 | INFO | Opening tmp RTMetrics json file | 20260508 12:57:43.959 | INFO | Serializing RTMetrics to json | 20260508 12:57:43.960 | INFO | Serialization done, swapping tmp file now... | 20260508 12:57:43.961 | INFO | RTMetrics done | 20260508 12:57:44.008 | INFO | Finished a round of Baz IO: Wrote 1.397 GiB in 3.482 seconds | 20260508 12:57:44.008 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 12:57:44.008 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152437% / 0% / 7.54992e-05% | 20260508 12:57:44.008 | INFO | Baz IO overall: | 20260508 12:57:44.008 | INFO | BytesWritten : 3540281630645 | 20260508 12:57:44.008 | INFO | NumEvents : 1698156389897 | 20260508 12:57:45.569 | INFO | Analyzing chunk frames = [8282112,8282624) | 20260508 12:57:45.569 | INFO | MemInfo: { | 20260508 12:57:45.569 | INFO | "gpu" : " 17 GB", | 20260508 12:57:45.569 | INFO | "huge" : "", | 20260508 12:57:45.569 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:45.569 | INFO | "malloc" : " 14 GB", | 20260508 12:57:45.569 | INFO | "pinned" : " 29 GB" | 20260508 12:57:45.569 | INFO | } | 20260508 12:57:45.569 | INFO | Perf report for the last 5.14084 seconds: | 20260508 12:57:45.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:45.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:45.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:45.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:45.569 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 12:57:45.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:45.569 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 12:57:45.569 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 12:57:45.569 | INFO | BAZWRITER : 67.0%, 2.4, 81.7ms, 0 | 20260508 12:57:45.569 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260508 12:57:45.570 | INFO | Uploaded 36GiB to GPU in 2.61436s (13.7701GiB/s) | 20260508 12:57:45.570 | INFO | Downloaded 38GiB to GPU in 2.73013s (13.9188GiB/s) | 20260508 12:57:50.709 | INFO | Analyzing chunk frames = [8282624,8283136) | 20260508 12:57:50.709 | INFO | MemInfo: { | 20260508 12:57:50.709 | INFO | "gpu" : " 17 GB", | 20260508 12:57:50.709 | INFO | "huge" : "", | 20260508 12:57:50.709 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:50.709 | INFO | "malloc" : " 14 GB", | 20260508 12:57:50.709 | INFO | "pinned" : " 29 GB" | 20260508 12:57:50.709 | INFO | } | 20260508 12:57:50.709 | INFO | Perf report for the last 5.13976 seconds: | 20260508 12:57:50.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:50.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 12:57:50.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:57:50.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:50.709 | INFO | Uploaded 36GiB to GPU in 2.44281s (14.7371GiB/s) | 20260508 12:57:50.709 | INFO | Downloaded 34GiB to GPU in 2.46623s (13.7862GiB/s) | 20260508 12:57:55.843 | INFO | Analyzing chunk frames = [8283136,8283648) | 20260508 12:57:55.843 | INFO | MemInfo: { | 20260508 12:57:55.843 | INFO | "gpu" : " 17 GB", | 20260508 12:57:55.843 | INFO | "huge" : "", | 20260508 12:57:55.843 | INFO | "huge_shared" : " 29 GB", | 20260508 12:57:55.843 | INFO | "malloc" : " 14 GB", | 20260508 12:57:55.843 | INFO | "pinned" : " 29 GB" | 20260508 12:57:55.843 | INFO | } | 20260508 12:57:55.843 | INFO | Perf report for the last 5.13436 seconds: | 20260508 12:57:55.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:57:55.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 12:57:55.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:57:55.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:57:55.843 | INFO | Uploaded 36GiB to GPU in 2.48438s (14.4906GiB/s) | 20260508 12:57:55.843 | INFO | Downloaded 34GiB to GPU in 2.46963s (13.7672GiB/s) | 20260508 12:58:00.992 | INFO | Analyzing chunk frames = [8283648,8284160) | 20260508 12:58:00.992 | INFO | MemInfo: { | 20260508 12:58:00.992 | INFO | "gpu" : " 17 GB", | 20260508 12:58:00.992 | INFO | "huge" : "", | 20260508 12:58:00.992 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:00.992 | INFO | "malloc" : " 14 GB", | 20260508 12:58:00.992 | INFO | "pinned" : " 29 GB" | 20260508 12:58:00.992 | INFO | } | 20260508 12:58:00.992 | INFO | Perf report for the last 5.14877 seconds: | 20260508 12:58:00.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:00.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 12:58:00.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:58:00.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:00.992 | INFO | Uploaded 36GiB to GPU in 2.45857s (14.6426GiB/s) | 20260508 12:58:00.992 | INFO | Downloaded 34GiB to GPU in 2.43344s (13.972GiB/s) | 20260508 12:58:06.124 | INFO | Analyzing chunk frames = [8284160,8284672) | 20260508 12:58:06.124 | INFO | MemInfo: { | 20260508 12:58:06.124 | INFO | "gpu" : " 17 GB", | 20260508 12:58:06.124 | INFO | "huge" : "", | 20260508 12:58:06.124 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:06.124 | INFO | "malloc" : " 14 GB", | 20260508 12:58:06.125 | INFO | "pinned" : " 29 GB" | 20260508 12:58:06.125 | INFO | } | 20260508 12:58:06.125 | INFO | Perf report for the last 5.13267 seconds: | 20260508 12:58:06.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:06.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 12:58:06.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 12:58:06.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:06.125 | INFO | Uploaded 36GiB to GPU in 2.48155s (14.5071GiB/s) | 20260508 12:58:06.125 | INFO | Downloaded 34GiB to GPU in 2.53993s (13.3862GiB/s) | 20260508 12:58:11.262 | INFO | Analyzing chunk frames = [8284672,8285184) | 20260508 12:58:11.262 | INFO | MemInfo: { | 20260508 12:58:11.262 | INFO | "gpu" : " 17 GB", | 20260508 12:58:11.262 | INFO | "huge" : "", | 20260508 12:58:11.262 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:11.262 | INFO | "malloc" : " 14 GB", | 20260508 12:58:11.262 | INFO | "pinned" : " 29 GB" | 20260508 12:58:11.262 | INFO | } | 20260508 12:58:11.262 | INFO | Perf report for the last 5.13742 seconds: | 20260508 12:58:11.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:11.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 12:58:11.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:58:11.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:11.262 | INFO | Uploaded 36GiB to GPU in 2.39574s (15.0267GiB/s) | 20260508 12:58:11.262 | INFO | Downloaded 34GiB to GPU in 2.43428s (13.9672GiB/s) | 20260508 12:58:16.398 | INFO | Analyzing chunk frames = [8285184,8285696) | 20260508 12:58:16.398 | INFO | MemInfo: { | 20260508 12:58:16.398 | INFO | "gpu" : " 17 GB", | 20260508 12:58:16.398 | INFO | "huge" : "", | 20260508 12:58:16.398 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:16.398 | INFO | "malloc" : " 14 GB", | 20260508 12:58:16.398 | INFO | "pinned" : " 29 GB" | 20260508 12:58:16.398 | INFO | } | 20260508 12:58:16.398 | INFO | Perf report for the last 5.13639 seconds: | 20260508 12:58:16.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:16.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.398 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 12:58:16.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.399 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:58:16.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:16.399 | INFO | Uploaded 36GiB to GPU in 2.4561s (14.6574GiB/s) | 20260508 12:58:16.399 | INFO | Downloaded 34GiB to GPU in 2.43238s (13.9781GiB/s) | 20260508 12:58:21.540 | INFO | Analyzing chunk frames = [8285696,8286208) | 20260508 12:58:21.540 | INFO | MemInfo: { | 20260508 12:58:21.540 | INFO | "gpu" : " 17 GB", | 20260508 12:58:21.540 | INFO | "huge" : "", | 20260508 12:58:21.540 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:21.540 | INFO | "malloc" : " 14 GB", | 20260508 12:58:21.541 | INFO | "pinned" : " 29 GB" | 20260508 12:58:21.541 | INFO | } | 20260508 12:58:21.541 | INFO | Perf report for the last 5.14212 seconds: | 20260508 12:58:21.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:21.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 12:58:21.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 12:58:21.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:21.541 | INFO | Uploaded 36GiB to GPU in 2.43704s (14.772GiB/s) | 20260508 12:58:21.541 | INFO | Downloaded 34GiB to GPU in 2.44868s (13.885GiB/s) | 20260508 12:58:21.638 | INFO | Starting a round of Baz IO | 20260508 12:58:25.053 | INFO | Populating RTMetrics report | 20260508 12:58:25.053 | INFO | RTMetrics report populated | 20260508 12:58:25.053 | INFO | Opening tmp RTMetrics json file | 20260508 12:58:25.054 | INFO | Serializing RTMetrics to json | 20260508 12:58:25.056 | INFO | Serialization done, swapping tmp file now... | 20260508 12:58:25.056 | INFO | RTMetrics done | 20260508 12:58:25.126 | INFO | Finished a round of Baz IO: Wrote 1.396 GiB in 3.488 seconds | 20260508 12:58:25.126 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 12:58:25.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151642% / 0% / 7.54992e-05% | 20260508 12:58:25.126 | INFO | Baz IO overall: | 20260508 12:58:25.126 | INFO | BytesWritten : 3541780315850 | 20260508 12:58:25.126 | INFO | NumEvents : 1698858523675 | 20260508 12:58:26.684 | INFO | Analyzing chunk frames = [8286208,8286720) | 20260508 12:58:26.684 | INFO | MemInfo: { | 20260508 12:58:26.684 | INFO | "gpu" : " 17 GB", | 20260508 12:58:26.684 | INFO | "huge" : "", | 20260508 12:58:26.684 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:26.684 | INFO | "malloc" : " 14 GB", | 20260508 12:58:26.684 | INFO | "pinned" : " 29 GB" | 20260508 12:58:26.684 | INFO | } | 20260508 12:58:26.684 | INFO | Perf report for the last 5.14328 seconds: | 20260508 12:58:26.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:26.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:26.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:26.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:26.684 | INFO | ANALYSIS : 67.9%, 3.0, 105.7ms, 0 | 20260508 12:58:26.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:26.684 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 12:58:26.684 | INFO | PRE_HQ : 65.4%, 1.6, 53.4ms, 0 | 20260508 12:58:26.684 | INFO | BAZWRITER : 67.1%, 2.5, 85.3ms, 0 | 20260508 12:58:26.684 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260508 12:58:26.685 | INFO | Uploaded 36GiB to GPU in 2.57022s (14.0066GiB/s) | 20260508 12:58:26.685 | INFO | Downloaded 38GiB to GPU in 2.69767s (14.0862GiB/s) | 20260508 12:58:31.826 | INFO | Analyzing chunk frames = [8286720,8287232) | 20260508 12:58:31.826 | INFO | MemInfo: { | 20260508 12:58:31.826 | INFO | "gpu" : " 17 GB", | 20260508 12:58:31.826 | INFO | "huge" : "", | 20260508 12:58:31.826 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:31.826 | INFO | "malloc" : " 14 GB", | 20260508 12:58:31.826 | INFO | "pinned" : " 29 GB" | 20260508 12:58:31.826 | INFO | } | 20260508 12:58:31.827 | INFO | Perf report for the last 5.14273 seconds: | 20260508 12:58:31.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:31.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 12:58:31.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 12:58:31.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:31.827 | INFO | Uploaded 36GiB to GPU in 2.46244s (14.6197GiB/s) | 20260508 12:58:31.827 | INFO | Downloaded 34GiB to GPU in 2.46438s (13.7966GiB/s) | 20260508 12:58:36.962 | INFO | Analyzing chunk frames = [8287232,8287744) | 20260508 12:58:36.962 | INFO | MemInfo: { | 20260508 12:58:36.962 | INFO | "gpu" : " 17 GB", | 20260508 12:58:36.962 | INFO | "huge" : "", | 20260508 12:58:36.962 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:36.962 | INFO | "malloc" : " 14 GB", | 20260508 12:58:36.962 | INFO | "pinned" : " 29 GB" | 20260508 12:58:36.962 | INFO | } | 20260508 12:58:36.962 | INFO | Perf report for the last 5.13516 seconds: | 20260508 12:58:36.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:36.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | ANALYSIS : 65.9%, 3.0, 102.5ms, 0 | 20260508 12:58:36.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:58:36.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:36.962 | INFO | Uploaded 36GiB to GPU in 2.45788s (14.6468GiB/s) | 20260508 12:58:36.962 | INFO | Downloaded 34GiB to GPU in 2.46624s (13.7861GiB/s) | 20260508 12:58:42.098 | INFO | Analyzing chunk frames = [8287744,8288256) | 20260508 12:58:42.098 | INFO | MemInfo: { | 20260508 12:58:42.098 | INFO | "gpu" : " 17 GB", | 20260508 12:58:42.098 | INFO | "huge" : "", | 20260508 12:58:42.098 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:42.098 | INFO | "malloc" : " 14 GB", | 20260508 12:58:42.098 | INFO | "pinned" : " 29 GB" | 20260508 12:58:42.098 | INFO | } | 20260508 12:58:42.098 | INFO | Perf report for the last 5.13589 seconds: | 20260508 12:58:42.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:42.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 12:58:42.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 12:58:42.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:42.098 | INFO | Uploaded 36GiB to GPU in 2.38846s (15.0724GiB/s) | 20260508 12:58:42.098 | INFO | Downloaded 34GiB to GPU in 2.46565s (13.7895GiB/s) | 20260508 12:58:47.236 | INFO | Analyzing chunk frames = [8288256,8288768) | 20260508 12:58:47.236 | INFO | MemInfo: { | 20260508 12:58:47.236 | INFO | "gpu" : " 17 GB", | 20260508 12:58:47.236 | INFO | "huge" : "", | 20260508 12:58:47.236 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:47.236 | INFO | "malloc" : " 14 GB", | 20260508 12:58:47.236 | INFO | "pinned" : " 29 GB" | 20260508 12:58:47.236 | INFO | } | 20260508 12:58:47.237 | INFO | Perf report for the last 5.13879 seconds: | 20260508 12:58:47.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:47.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 12:58:47.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 12:58:47.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:47.237 | INFO | Uploaded 36GiB to GPU in 2.34151s (15.3747GiB/s) | 20260508 12:58:47.237 | INFO | Downloaded 34GiB to GPU in 2.48714s (13.6703GiB/s) | 20260508 12:58:52.384 | INFO | Analyzing chunk frames = [8288768,8289280) | 20260508 12:58:52.384 | INFO | MemInfo: { | 20260508 12:58:52.384 | INFO | "gpu" : " 17 GB", | 20260508 12:58:52.384 | INFO | "huge" : "", | 20260508 12:58:52.384 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:52.384 | INFO | "malloc" : " 14 GB", | 20260508 12:58:52.384 | INFO | "pinned" : " 29 GB" | 20260508 12:58:52.384 | INFO | } | 20260508 12:58:52.385 | INFO | Perf report for the last 5.14811 seconds: | 20260508 12:58:52.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:52.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 12:58:52.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 12:58:52.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:52.385 | INFO | Uploaded 36GiB to GPU in 2.38883s (15.0702GiB/s) | 20260508 12:58:52.385 | INFO | Downloaded 34GiB to GPU in 2.41712s (14.0663GiB/s) | 20260508 12:58:57.519 | INFO | Analyzing chunk frames = [8289280,8289792) | 20260508 12:58:57.520 | INFO | MemInfo: { | 20260508 12:58:57.520 | INFO | "gpu" : " 17 GB", | 20260508 12:58:57.520 | INFO | "huge" : "", | 20260508 12:58:57.520 | INFO | "huge_shared" : " 29 GB", | 20260508 12:58:57.520 | INFO | "malloc" : " 14 GB", | 20260508 12:58:57.520 | INFO | "pinned" : " 29 GB" | 20260508 12:58:57.520 | INFO | } | 20260508 12:58:57.520 | INFO | Perf report for the last 5.13499 seconds: | 20260508 12:58:57.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:58:57.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.520 | INFO | ANALYSIS : 64.5%, 3.0, 100.3ms, 0 | 20260508 12:58:57.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.520 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:58:57.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:58:57.521 | INFO | Uploaded 36GiB to GPU in 2.29417s (15.6919GiB/s) | 20260508 12:58:57.521 | INFO | Downloaded 34GiB to GPU in 2.39135s (14.2179GiB/s) | 20260508 12:59:02.657 | INFO | Analyzing chunk frames = [8289792,8290304) | 20260508 12:59:02.657 | INFO | MemInfo: { | 20260508 12:59:02.657 | INFO | "gpu" : " 17 GB", | 20260508 12:59:02.657 | INFO | "huge" : "", | 20260508 12:59:02.657 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:02.657 | INFO | "malloc" : " 14 GB", | 20260508 12:59:02.657 | INFO | "pinned" : " 29 GB" | 20260508 12:59:02.657 | INFO | } | 20260508 12:59:02.657 | INFO | Perf report for the last 5.13717 seconds: | 20260508 12:59:02.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:02.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 12:59:02.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 12:59:02.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:02.657 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 12:59:02.657 | INFO | Total mem usage: 16.7 GiB | 20260508 12:59:02.657 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 12:59:02.657 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 12:59:02.657 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 12:59:02.657 | INFO | High Water Breakdown: | 20260508 12:59:02.657 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 12:59:02.657 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 12:59:02.657 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 12:59:02.657 | INFO | ------------------------------------------ | 20260508 12:59:02.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:59:02.657 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 12:59:02.657 | INFO | Total mem usage: 26.5 GiB | 20260508 12:59:02.657 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 12:59:02.657 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 12:59:02.657 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 12:59:02.657 | INFO | High Water Breakdown: | 20260508 12:59:02.657 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:59:02.657 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:59:02.657 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:59:02.657 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:59:02.657 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:59:02.657 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:59:02.657 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 12:59:02.657 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 12:59:02.657 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:59:02.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 12:59:02.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 12:59:02.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 12:59:02.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 12:59:02.657 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 12:59:02.657 | INFO | ------------------------------------------ | 20260508 12:59:02.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:59:02.657 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 12:59:02.657 | INFO | Total mem usage: 27.8 GiB | 20260508 12:59:02.657 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 12:59:02.657 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 12:59:02.657 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 12:59:02.657 | INFO | High Water Breakdown: | 20260508 12:59:02.657 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 12:59:02.657 | INFO | ------------------------------------------ | 20260508 12:59:02.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:59:02.657 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 12:59:02.657 | INFO | Total mem usage: 15.7 GiB | 20260508 12:59:02.657 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 12:59:02.657 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 12:59:02.657 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 12:59:02.657 | INFO | Uploaded 36GiB to GPU in 2.49065s (14.4541GiB/s) | 20260508 12:59:02.658 | INFO | Downloaded 34GiB to GPU in 2.44995s (13.8778GiB/s) | 20260508 12:59:02.657 | INFO | High Water Breakdown: | 20260508 12:59:02.658 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 12:59:02.658 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 12:59:02.658 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 12:59:02.658 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 12:59:02.658 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 12:59:02.658 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 12:59:02.658 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 12:59:02.658 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 12:59:02.658 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 12:59:02.658 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 12:59:02.658 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 12:59:02.658 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 12:59:02.658 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 12:59:02.658 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 12:59:02.658 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 12:59:02.658 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 12:59:02.658 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 12:59:02.658 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 12:59:02.658 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 12:59:02.658 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 12:59:02.658 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 12:59:02.658 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 12:59:02.658 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 12:59:02.658 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 12:59:02.658 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 12:59:02.658 | INFO | ------------------------------------------ | 20260508 12:59:02.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 12:59:02.658 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 12:59:02.658 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:59:02.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:59:02.658 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 5.09e-01 | 20260508 12:59:02.658 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.07e-01, 1.14e-02 / 9.44e-03 | 20260508 12:59:02.658 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.45e+00, 6.02e-01 / 2.02e-01 | 20260508 12:59:02.658 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.49e+00, 2.11e-01 / 9.35e-02 | 20260508 12:59:02.658 | INFO | PerfReport: 16.00%, DME -- 4.19e+01 / 4.55e+01, 3.73e+00 / 6.37e-01 | 20260508 12:59:02.658 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.87e+01 / 1.99e+01, 1.87e+00 / 1.20e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.31e+00 / 2.53e+00, 4.01e-01 / 1.85e-01 | 20260508 12:59:02.658 | INFO | PerfReport: 2.93%, Metrics -- 9.59e-01 / 1.11e+00, 2.39e-01 / 8.67e-02 | 20260508 12:59:02.658 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.27e-01, 4.66e-02 / 2.40e-02 | 20260508 12:59:02.658 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 12:59:02.658 | INFO | PerfReport: Gathered 1536 samples. | 20260508 12:59:02.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 12:59:02.658 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 24.33%, Upload -- 2.55e+01 / 0.00e+00, 7.85e+00 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 24.72%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 31.21%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 2.17%, LockUpload -- 2.27e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 2.53%, LockDownload -- 2.65e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 15.03%, LockCompute -- 1.58e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 12:59:02.658 | INFO | PerfReport: 0.01%, Misc -- 1.16e-02 / 0.00e+00, 8.22e-03 / 0.00e+00 | 20260508 12:59:02.755 | INFO | Starting a round of Baz IO | 20260508 12:59:06.162 | INFO | Populating RTMetrics report | 20260508 12:59:06.162 | INFO | RTMetrics report populated | 20260508 12:59:06.162 | INFO | Opening tmp RTMetrics json file | 20260508 12:59:06.171 | INFO | Serializing RTMetrics to json | 20260508 12:59:06.172 | INFO | Serialization done, swapping tmp file now... | 20260508 12:59:06.173 | INFO | RTMetrics done | 20260508 12:59:06.225 | INFO | Finished a round of Baz IO: Wrote 1.396 GiB in 3.47 seconds | 20260508 12:59:06.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 12:59:06.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151722% / 3.97364e-06% / 8.74201e-05% | 20260508 12:59:06.225 | INFO | Baz IO overall: | 20260508 12:59:06.225 | INFO | BytesWritten : 3543279728161 | 20260508 12:59:06.225 | INFO | NumEvents : 1699561230863 | 20260508 12:59:07.800 | INFO | Analyzing chunk frames = [8290304,8290816) | 20260508 12:59:07.800 | INFO | MemInfo: { | 20260508 12:59:07.800 | INFO | "gpu" : " 17 GB", | 20260508 12:59:07.800 | INFO | "huge" : "", | 20260508 12:59:07.800 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:07.800 | INFO | "malloc" : " 14 GB", | 20260508 12:59:07.800 | INFO | "pinned" : " 29 GB" | 20260508 12:59:07.800 | INFO | } | 20260508 12:59:07.800 | INFO | Perf report for the last 5.14271 seconds: | 20260508 12:59:07.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:07.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:07.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:07.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:07.800 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 12:59:07.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:07.800 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 12:59:07.800 | INFO | PRE_HQ : 65.1%, 1.6, 53.7ms, 0 | 20260508 12:59:07.800 | INFO | BAZWRITER : 66.8%, 2.5, 85.2ms, 0 | 20260508 12:59:07.800 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:59:07.800 | INFO | Uploaded 36GiB to GPU in 2.62557s (13.7113GiB/s) | 20260508 12:59:07.800 | INFO | Downloaded 38GiB to GPU in 2.70354s (14.0556GiB/s) | 20260508 12:59:12.939 | INFO | Analyzing chunk frames = [8290816,8291328) | 20260508 12:59:12.939 | INFO | MemInfo: { | 20260508 12:59:12.939 | INFO | "gpu" : " 17 GB", | 20260508 12:59:12.939 | INFO | "huge" : "", | 20260508 12:59:12.939 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:12.939 | INFO | "malloc" : " 14 GB", | 20260508 12:59:12.939 | INFO | "pinned" : " 29 GB" | 20260508 12:59:12.939 | INFO | } | 20260508 12:59:12.939 | INFO | Perf report for the last 5.13922 seconds: | 20260508 12:59:12.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:12.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 12:59:12.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:59:12.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:12.939 | INFO | Uploaded 36GiB to GPU in 2.43719s (14.7711GiB/s) | 20260508 12:59:12.939 | INFO | Downloaded 34GiB to GPU in 2.47079s (13.7608GiB/s) | 20260508 12:59:18.074 | INFO | Analyzing chunk frames = [8291328,8291840) | 20260508 12:59:18.074 | INFO | MemInfo: { | 20260508 12:59:18.074 | INFO | "gpu" : " 17 GB", | 20260508 12:59:18.074 | INFO | "huge" : "", | 20260508 12:59:18.074 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:18.074 | INFO | "malloc" : " 14 GB", | 20260508 12:59:18.074 | INFO | "pinned" : " 29 GB" | 20260508 12:59:18.074 | INFO | } | 20260508 12:59:18.074 | INFO | Perf report for the last 5.13558 seconds: | 20260508 12:59:18.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:18.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.074 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:59:18.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.075 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:59:18.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:18.075 | INFO | Uploaded 36GiB to GPU in 2.48713s (14.4745GiB/s) | 20260508 12:59:18.075 | INFO | Downloaded 34GiB to GPU in 2.46575s (13.7889GiB/s) | 20260508 12:59:23.211 | INFO | Analyzing chunk frames = [8291840,8292352) | 20260508 12:59:23.211 | INFO | MemInfo: { | 20260508 12:59:23.211 | INFO | "gpu" : " 17 GB", | 20260508 12:59:23.211 | INFO | "huge" : "", | 20260508 12:59:23.211 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:23.211 | INFO | "malloc" : " 14 GB", | 20260508 12:59:23.211 | INFO | "pinned" : " 29 GB" | 20260508 12:59:23.211 | INFO | } | 20260508 12:59:23.211 | INFO | Perf report for the last 5.13707 seconds: | 20260508 12:59:23.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:23.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.212 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 12:59:23.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.212 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 12:59:23.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:23.212 | INFO | Uploaded 36GiB to GPU in 2.45764s (14.6482GiB/s) | 20260508 12:59:23.212 | INFO | Downloaded 34GiB to GPU in 2.44834s (13.887GiB/s) | 20260508 12:59:28.349 | INFO | Analyzing chunk frames = [8292352,8292864) | 20260508 12:59:28.349 | INFO | MemInfo: { | 20260508 12:59:28.349 | INFO | "gpu" : " 17 GB", | 20260508 12:59:28.349 | INFO | "huge" : "", | 20260508 12:59:28.349 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:28.349 | INFO | "malloc" : " 14 GB", | 20260508 12:59:28.349 | INFO | "pinned" : " 29 GB" | 20260508 12:59:28.349 | INFO | } | 20260508 12:59:28.349 | INFO | Perf report for the last 5.13776 seconds: | 20260508 12:59:28.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:28.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.349 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 12:59:28.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.349 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 12:59:28.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:28.350 | INFO | Uploaded 36GiB to GPU in 2.44575s (14.7194GiB/s) | 20260508 12:59:28.350 | INFO | Downloaded 34GiB to GPU in 2.5227s (13.4776GiB/s) | 20260508 12:59:33.495 | INFO | Analyzing chunk frames = [8292864,8293376) | 20260508 12:59:33.495 | INFO | MemInfo: { | 20260508 12:59:33.495 | INFO | "gpu" : " 17 GB", | 20260508 12:59:33.495 | INFO | "huge" : "", | 20260508 12:59:33.495 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:33.495 | INFO | "malloc" : " 14 GB", | 20260508 12:59:33.495 | INFO | "pinned" : " 29 GB" | 20260508 12:59:33.495 | INFO | } | 20260508 12:59:33.495 | INFO | Perf report for the last 5.14553 seconds: | 20260508 12:59:33.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:33.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 12:59:33.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 12:59:33.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:33.495 | INFO | Uploaded 36GiB to GPU in 2.38224s (15.1118GiB/s) | 20260508 12:59:33.495 | INFO | Downloaded 34GiB to GPU in 2.46321s (13.8031GiB/s) | 20260508 12:59:38.640 | INFO | Analyzing chunk frames = [8293376,8293888) | 20260508 12:59:38.640 | INFO | MemInfo: { | 20260508 12:59:38.640 | INFO | "gpu" : " 17 GB", | 20260508 12:59:38.640 | INFO | "huge" : "", | 20260508 12:59:38.640 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:38.640 | INFO | "malloc" : " 14 GB", | 20260508 12:59:38.640 | INFO | "pinned" : " 29 GB" | 20260508 12:59:38.640 | INFO | } | 20260508 12:59:38.640 | INFO | Perf report for the last 5.14568 seconds: | 20260508 12:59:38.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:38.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 12:59:38.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 12:59:38.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:38.641 | INFO | Uploaded 36GiB to GPU in 2.44571s (14.7197GiB/s) | 20260508 12:59:38.641 | INFO | Downloaded 34GiB to GPU in 2.47018s (13.7642GiB/s) | 20260508 12:59:43.772 | INFO | Analyzing chunk frames = [8293888,8294400) | 20260508 12:59:43.772 | INFO | MemInfo: { | 20260508 12:59:43.772 | INFO | "gpu" : " 17 GB", | 20260508 12:59:43.772 | INFO | "huge" : "", | 20260508 12:59:43.772 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:43.772 | INFO | "malloc" : " 14 GB", | 20260508 12:59:43.772 | INFO | "pinned" : " 29 GB" | 20260508 12:59:43.773 | INFO | } | 20260508 12:59:43.773 | INFO | Perf report for the last 5.13219 seconds: | 20260508 12:59:43.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:43.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 12:59:43.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 12:59:43.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:43.773 | INFO | Uploaded 36GiB to GPU in 2.33955s (15.3876GiB/s) | 20260508 12:59:43.773 | INFO | Downloaded 34GiB to GPU in 2.4472s (13.8934GiB/s) | 20260508 12:59:43.871 | INFO | Starting a round of Baz IO | 20260508 12:59:47.264 | INFO | Populating RTMetrics report | 20260508 12:59:47.264 | INFO | RTMetrics report populated | 20260508 12:59:47.265 | INFO | Opening tmp RTMetrics json file | 20260508 12:59:47.275 | INFO | Serializing RTMetrics to json | 20260508 12:59:47.276 | INFO | Serialization done, swapping tmp file now... | 20260508 12:59:47.277 | INFO | RTMetrics done | 20260508 12:59:47.336 | INFO | Finished a round of Baz IO: Wrote 1.397 GiB in 3.465 seconds | 20260508 12:59:47.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 12:59:47.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152171% / 3.97364e-06% / 7.15256e-05% | 20260508 12:59:47.336 | INFO | Baz IO overall: | 20260508 12:59:47.336 | INFO | BytesWritten : 3544780178662 | 20260508 12:59:47.336 | INFO | NumEvents : 1700264484158 | 20260508 12:59:48.908 | INFO | Analyzing chunk frames = [8294400,8294912) | 20260508 12:59:48.908 | INFO | MemInfo: { | 20260508 12:59:48.908 | INFO | "gpu" : " 17 GB", | 20260508 12:59:48.908 | INFO | "huge" : "", | 20260508 12:59:48.908 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:48.908 | INFO | "malloc" : " 14 GB", | 20260508 12:59:48.908 | INFO | "pinned" : " 29 GB" | 20260508 12:59:48.908 | INFO | } | 20260508 12:59:48.909 | INFO | Perf report for the last 5.13581 seconds: | 20260508 12:59:48.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:48.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:48.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:48.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:48.909 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 12:59:48.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:48.909 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 12:59:48.909 | INFO | PRE_HQ : 65.3%, 1.6, 54.4ms, 0 | 20260508 12:59:48.909 | INFO | BAZWRITER : 66.7%, 2.5, 84.5ms, 0 | 20260508 12:59:48.909 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 12:59:48.909 | INFO | Uploaded 36GiB to GPU in 2.60537s (13.8176GiB/s) | 20260508 12:59:48.909 | INFO | Downloaded 38GiB to GPU in 2.71946s (13.9734GiB/s) | 20260508 12:59:54.053 | INFO | Analyzing chunk frames = [8294912,8295424) | 20260508 12:59:54.053 | INFO | MemInfo: { | 20260508 12:59:54.053 | INFO | "gpu" : " 17 GB", | 20260508 12:59:54.053 | INFO | "huge" : "", | 20260508 12:59:54.053 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:54.053 | INFO | "malloc" : " 14 GB", | 20260508 12:59:54.053 | INFO | "pinned" : " 29 GB" | 20260508 12:59:54.053 | INFO | } | 20260508 12:59:54.053 | INFO | Perf report for the last 5.14445 seconds: | 20260508 12:59:54.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:54.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 12:59:54.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 12:59:54.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:54.053 | INFO | Uploaded 36GiB to GPU in 2.40966s (14.9398GiB/s) | 20260508 12:59:54.053 | INFO | Downloaded 34GiB to GPU in 2.47686s (13.727GiB/s) | 20260508 12:59:59.191 | INFO | Analyzing chunk frames = [8295424,8295936) | 20260508 12:59:59.191 | INFO | MemInfo: { | 20260508 12:59:59.191 | INFO | "gpu" : " 17 GB", | 20260508 12:59:59.191 | INFO | "huge" : "", | 20260508 12:59:59.191 | INFO | "huge_shared" : " 29 GB", | 20260508 12:59:59.191 | INFO | "malloc" : " 14 GB", | 20260508 12:59:59.191 | INFO | "pinned" : " 29 GB" | 20260508 12:59:59.191 | INFO | } | 20260508 12:59:59.191 | INFO | Perf report for the last 5.13795 seconds: | 20260508 12:59:59.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 12:59:59.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 12:59:59.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 12:59:59.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 12:59:59.191 | INFO | Uploaded 36GiB to GPU in 2.47734s (14.5317GiB/s) | 20260508 12:59:59.191 | INFO | Downloaded 34GiB to GPU in 2.47424s (13.7416GiB/s) | 20260508 13:00:04.332 | INFO | Analyzing chunk frames = [8295936,8296448) | 20260508 13:00:04.332 | INFO | MemInfo: { | 20260508 13:00:04.332 | INFO | "gpu" : " 17 GB", | 20260508 13:00:04.332 | INFO | "huge" : "", | 20260508 13:00:04.332 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:04.332 | INFO | "malloc" : " 14 GB", | 20260508 13:00:04.332 | INFO | "pinned" : " 29 GB" | 20260508 13:00:04.332 | INFO | } | 20260508 13:00:04.332 | INFO | Perf report for the last 5.14067 seconds: | 20260508 13:00:04.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:04.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 13:00:04.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:00:04.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:04.332 | INFO | Uploaded 36GiB to GPU in 2.47929s (14.5203GiB/s) | 20260508 13:00:04.332 | INFO | Downloaded 34GiB to GPU in 2.45875s (13.8282GiB/s) | 20260508 13:00:09.470 | INFO | Analyzing chunk frames = [8296448,8296960) | 20260508 13:00:09.470 | INFO | MemInfo: { | 20260508 13:00:09.470 | INFO | "gpu" : " 17 GB", | 20260508 13:00:09.470 | INFO | "huge" : "", | 20260508 13:00:09.470 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:09.470 | INFO | "malloc" : " 14 GB", | 20260508 13:00:09.470 | INFO | "pinned" : " 29 GB" | 20260508 13:00:09.470 | INFO | } | 20260508 13:00:09.470 | INFO | Perf report for the last 5.13816 seconds: | 20260508 13:00:09.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:09.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:00:09.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:00:09.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:09.470 | INFO | Uploaded 36GiB to GPU in 2.32743s (15.4677GiB/s) | 20260508 13:00:09.470 | INFO | Downloaded 34GiB to GPU in 2.51437s (13.5223GiB/s) | 20260508 13:00:14.608 | INFO | Analyzing chunk frames = [8296960,8297472) | 20260508 13:00:14.608 | INFO | MemInfo: { | 20260508 13:00:14.608 | INFO | "gpu" : " 17 GB", | 20260508 13:00:14.608 | INFO | "huge" : "", | 20260508 13:00:14.608 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:14.608 | INFO | "malloc" : " 14 GB", | 20260508 13:00:14.608 | INFO | "pinned" : " 29 GB" | 20260508 13:00:14.608 | INFO | } | 20260508 13:00:14.608 | INFO | Perf report for the last 5.13848 seconds: | 20260508 13:00:14.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:14.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.608 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 13:00:14.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.608 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260508 13:00:14.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:14.609 | INFO | Uploaded 36GiB to GPU in 2.33762s (15.4003GiB/s) | 20260508 13:00:14.609 | INFO | Downloaded 34GiB to GPU in 2.42378s (14.0277GiB/s) | 20260508 13:00:19.755 | INFO | Analyzing chunk frames = [8297472,8297984) | 20260508 13:00:19.755 | INFO | MemInfo: { | 20260508 13:00:19.755 | INFO | "gpu" : " 17 GB", | 20260508 13:00:19.755 | INFO | "huge" : "", | 20260508 13:00:19.755 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:19.755 | INFO | "malloc" : " 14 GB", | 20260508 13:00:19.755 | INFO | "pinned" : " 29 GB" | 20260508 13:00:19.755 | INFO | } | 20260508 13:00:19.755 | INFO | Perf report for the last 5.14667 seconds: | 20260508 13:00:19.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:19.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.755 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 13:00:19.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.755 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:00:19.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:19.756 | INFO | Uploaded 36GiB to GPU in 2.45064s (14.69GiB/s) | 20260508 13:00:19.756 | INFO | Downloaded 34GiB to GPU in 2.45739s (13.8358GiB/s) | 20260508 13:00:24.887 | INFO | Analyzing chunk frames = [8297984,8298496) | 20260508 13:00:24.887 | INFO | MemInfo: { | 20260508 13:00:24.887 | INFO | "gpu" : " 17 GB", | 20260508 13:00:24.887 | INFO | "huge" : "", | 20260508 13:00:24.887 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:24.887 | INFO | "malloc" : " 14 GB", | 20260508 13:00:24.887 | INFO | "pinned" : " 29 GB" | 20260508 13:00:24.887 | INFO | } | 20260508 13:00:24.887 | INFO | Perf report for the last 5.13248 seconds: | 20260508 13:00:24.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:24.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.887 | INFO | ANALYSIS : 64.5%, 3.0, 100.7ms, 0 | 20260508 13:00:24.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.888 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:00:24.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:24.888 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:00:24.888 | INFO | Total mem usage: 16.7 GiB | 20260508 13:00:24.888 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:00:24.888 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:00:24.888 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:00:24.888 | INFO | High Water Breakdown: | 20260508 13:00:24.888 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:00:24.888 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:00:24.888 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:00:24.888 | INFO | ------------------------------------------ | 20260508 13:00:24.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:00:24.888 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:00:24.888 | INFO | Total mem usage: 26.5 GiB | 20260508 13:00:24.888 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:00:24.888 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:00:24.888 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:00:24.888 | INFO | High Water Breakdown: | 20260508 13:00:24.888 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:00:24.888 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:00:24.888 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:00:24.888 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:00:24.888 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:00:24.888 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:00:24.888 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:00:24.888 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:00:24.888 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:00:24.888 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:00:24.888 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:00:24.888 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:00:24.888 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:00:24.888 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:00:24.888 | INFO | ------------------------------------------ | 20260508 13:00:24.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:00:24.888 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:00:24.888 | INFO | Total mem usage: 27.8 GiB | 20260508 13:00:24.888 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:00:24.888 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:00:24.888 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:00:24.888 | INFO | High Water Breakdown: | 20260508 13:00:24.888 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:00:24.888 | INFO | ------------------------------------------ | 20260508 13:00:24.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:00:24.888 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:00:24.888 | INFO | Total mem usage: 15.7 GiB | 20260508 13:00:24.888 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:00:24.888 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:00:24.888 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:00:24.888 | INFO | High Water Breakdown: | 20260508 13:00:24.888 | INFO | Uploaded 36GiB to GPU in 2.3217s (15.5059GiB/s) | 20260508 13:00:24.888 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:00:24.888 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:00:24.888 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:00:24.889 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:00:24.888 | INFO | Downloaded 34GiB to GPU in 2.42761s (14.0055GiB/s) | 20260508 13:00:24.889 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:00:24.889 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:00:24.889 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:00:24.889 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:00:24.889 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:00:24.889 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:00:24.889 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:00:24.889 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:00:24.889 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:00:24.889 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:00:24.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:00:24.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:00:24.889 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:00:24.889 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:00:24.889 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:00:24.889 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:00:24.889 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:00:24.889 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:00:24.889 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:00:24.889 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:00:24.889 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:00:24.889 | INFO | ------------------------------------------ | 20260508 13:00:24.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:00:24.889 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:00:24.889 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:00:24.889 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:00:24.889 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.55e+01, 1.40e+01 / 6.52e-01 | 20260508 13:00:24.889 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.19e-02 / 6.15e-03 | 20260508 13:00:24.889 | INFO | PerfReport: 12.17%, Baseline -- 3.98e+00 / 4.34e+00, 5.95e-01 / 2.52e-01 | 20260508 13:00:24.889 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.11e-01 / 1.20e-01 | 20260508 13:00:24.889 | INFO | PerfReport: 16.01%, DME -- 4.19e+01 / 4.53e+01, 3.77e+00 / 1.06e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.87e+00 / 1.25e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.31e+00 / 2.36e+00, 4.04e-01 / 2.79e-01 | 20260508 13:00:24.889 | INFO | PerfReport: 2.93%, Metrics -- 9.57e-01 / 1.05e+00, 2.34e-01 / 1.16e-01 | 20260508 13:00:24.889 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.10e-01, 4.63e-02 / 2.74e-02 | 20260508 13:00:24.889 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:00:24.889 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:00:24.889 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:00:24.889 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 24.26%, Upload -- 2.54e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 24.83%, Download -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 31.22%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 2.06%, LockUpload -- 2.16e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 2.61%, LockDownload -- 2.74e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 15.01%, LockCompute -- 1.57e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260508 13:00:24.889 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260508 13:00:24.985 | INFO | Starting a round of Baz IO | 20260508 13:00:28.390 | INFO | Populating RTMetrics report | 20260508 13:00:28.390 | INFO | RTMetrics report populated | 20260508 13:00:28.390 | INFO | Opening tmp RTMetrics json file | 20260508 13:00:28.404 | INFO | Serializing RTMetrics to json | 20260508 13:00:28.406 | INFO | Serialization done, swapping tmp file now... | 20260508 13:00:28.406 | INFO | RTMetrics done | 20260508 13:00:28.458 | INFO | Finished a round of Baz IO: Wrote 1.397 GiB in 3.472 seconds | 20260508 13:00:28.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 13:00:28.459 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151602% / 0% / 5.16574e-05% | 20260508 13:00:28.459 | INFO | Baz IO overall: | 20260508 13:00:28.459 | INFO | BytesWritten : 3546280527813 | 20260508 13:00:28.459 | INFO | NumEvents : 1700967880029 | 20260508 13:00:30.030 | INFO | Analyzing chunk frames = [8298496,8299008) | 20260508 13:00:30.030 | INFO | MemInfo: { | 20260508 13:00:30.030 | INFO | "gpu" : " 17 GB", | 20260508 13:00:30.030 | INFO | "huge" : "", | 20260508 13:00:30.030 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:30.030 | INFO | "malloc" : " 14 GB", | 20260508 13:00:30.030 | INFO | "pinned" : " 29 GB" | 20260508 13:00:30.030 | INFO | } | 20260508 13:00:30.030 | INFO | Perf report for the last 5.14216 seconds: | 20260508 13:00:30.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:30.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:30.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:30.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:30.030 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 13:00:30.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:30.030 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:00:30.030 | INFO | PRE_HQ : 65.4%, 1.6, 53.7ms, 0 | 20260508 13:00:30.030 | INFO | BAZWRITER : 66.9%, 2.5, 84.8ms, 0 | 20260508 13:00:30.030 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 13:00:30.030 | INFO | Uploaded 36GiB to GPU in 2.58886s (13.9057GiB/s) | 20260508 13:00:30.030 | INFO | Downloaded 38GiB to GPU in 2.72173s (13.9617GiB/s) | 20260508 13:00:35.160 | INFO | Analyzing chunk frames = [8299008,8299520) | 20260508 13:00:35.160 | INFO | MemInfo: { | 20260508 13:00:35.160 | INFO | "gpu" : " 17 GB", | 20260508 13:00:35.160 | INFO | "huge" : "", | 20260508 13:00:35.160 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:35.160 | INFO | "malloc" : " 14 GB", | 20260508 13:00:35.160 | INFO | "pinned" : " 29 GB" | 20260508 13:00:35.160 | INFO | } | 20260508 13:00:35.161 | INFO | Perf report for the last 5.13087 seconds: | 20260508 13:00:35.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:35.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 13:00:35.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:00:35.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:35.161 | INFO | Uploaded 36GiB to GPU in 2.48119s (14.5092GiB/s) | 20260508 13:00:35.161 | INFO | Downloaded 34GiB to GPU in 2.47522s (13.7362GiB/s) | 20260508 13:00:40.303 | INFO | Analyzing chunk frames = [8299520,8300032) | 20260508 13:00:40.303 | INFO | MemInfo: { | 20260508 13:00:40.303 | INFO | "gpu" : " 17 GB", | 20260508 13:00:40.303 | INFO | "huge" : "", | 20260508 13:00:40.303 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:40.303 | INFO | "malloc" : " 14 GB", | 20260508 13:00:40.303 | INFO | "pinned" : " 29 GB" | 20260508 13:00:40.303 | INFO | } | 20260508 13:00:40.303 | INFO | Perf report for the last 5.14251 seconds: | 20260508 13:00:40.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:40.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.303 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 13:00:40.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.303 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:00:40.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:40.304 | INFO | Uploaded 36GiB to GPU in 2.47047s (14.5721GiB/s) | 20260508 13:00:40.304 | INFO | Downloaded 34GiB to GPU in 2.45239s (13.864GiB/s) | 20260508 13:00:45.447 | INFO | Analyzing chunk frames = [8300032,8300544) | 20260508 13:00:45.447 | INFO | MemInfo: { | 20260508 13:00:45.447 | INFO | "gpu" : " 17 GB", | 20260508 13:00:45.447 | INFO | "huge" : "", | 20260508 13:00:45.447 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:45.447 | INFO | "malloc" : " 14 GB", | 20260508 13:00:45.447 | INFO | "pinned" : " 29 GB" | 20260508 13:00:45.447 | INFO | } | 20260508 13:00:45.447 | INFO | Perf report for the last 5.14403 seconds: | 20260508 13:00:45.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:45.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.447 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:00:45.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.447 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:00:45.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:45.448 | INFO | Uploaded 36GiB to GPU in 2.46466s (14.6065GiB/s) | 20260508 13:00:45.448 | INFO | Downloaded 34GiB to GPU in 2.45681s (13.8391GiB/s) | 20260508 13:00:50.588 | INFO | Analyzing chunk frames = [8300544,8301056) | 20260508 13:00:50.588 | INFO | MemInfo: { | 20260508 13:00:50.588 | INFO | "gpu" : " 17 GB", | 20260508 13:00:50.588 | INFO | "huge" : "", | 20260508 13:00:50.588 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:50.588 | INFO | "malloc" : " 14 GB", | 20260508 13:00:50.588 | INFO | "pinned" : " 29 GB" | 20260508 13:00:50.588 | INFO | } | 20260508 13:00:50.588 | INFO | Perf report for the last 5.14078 seconds: | 20260508 13:00:50.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:50.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.588 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:00:50.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.588 | INFO | PRE_HQ : 31.2%, 1.0, 16.3ms, 0 | 20260508 13:00:50.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:50.589 | INFO | Uploaded 36GiB to GPU in 2.27639s (15.8145GiB/s) | 20260508 13:00:50.589 | INFO | Downloaded 34GiB to GPU in 2.51758s (13.505GiB/s) | 20260508 13:00:55.720 | INFO | Analyzing chunk frames = [8301056,8301568) | 20260508 13:00:55.720 | INFO | MemInfo: { | 20260508 13:00:55.720 | INFO | "gpu" : " 17 GB", | 20260508 13:00:55.720 | INFO | "huge" : "", | 20260508 13:00:55.720 | INFO | "huge_shared" : " 29 GB", | 20260508 13:00:55.720 | INFO | "malloc" : " 14 GB", | 20260508 13:00:55.720 | INFO | "pinned" : " 29 GB" | 20260508 13:00:55.720 | INFO | } | 20260508 13:00:55.720 | INFO | Perf report for the last 5.13243 seconds: | 20260508 13:00:55.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:00:55.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:00:55.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:00:55.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:00:55.721 | INFO | Uploaded 36GiB to GPU in 2.40072s (14.9955GiB/s) | 20260508 13:00:55.721 | INFO | Downloaded 34GiB to GPU in 2.4577s (13.834GiB/s) | 20260508 13:01:00.861 | INFO | Analyzing chunk frames = [8301568,8302080) | 20260508 13:01:00.861 | INFO | MemInfo: { | 20260508 13:01:00.861 | INFO | "gpu" : " 17 GB", | 20260508 13:01:00.861 | INFO | "huge" : "", | 20260508 13:01:00.861 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:00.861 | INFO | "malloc" : " 14 GB", | 20260508 13:01:00.861 | INFO | "pinned" : " 29 GB" | 20260508 13:01:00.861 | INFO | } | 20260508 13:01:00.861 | INFO | Perf report for the last 5.14045 seconds: | 20260508 13:01:00.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:00.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | ANALYSIS : 65.3%, 3.0, 101.9ms, 0 | 20260508 13:01:00.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:01:00.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:00.861 | INFO | Uploaded 36GiB to GPU in 2.43408s (14.79GiB/s) | 20260508 13:01:00.861 | INFO | Downloaded 34GiB to GPU in 2.46886s (13.7715GiB/s) | 20260508 13:01:05.999 | INFO | Analyzing chunk frames = [8302080,8302592) | 20260508 13:01:05.999 | INFO | MemInfo: { | 20260508 13:01:05.999 | INFO | "gpu" : " 17 GB", | 20260508 13:01:05.999 | INFO | "huge" : "", | 20260508 13:01:05.999 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:05.999 | INFO | "malloc" : " 14 GB", | 20260508 13:01:05.999 | INFO | "pinned" : " 29 GB" | 20260508 13:01:05.999 | INFO | } | 20260508 13:01:05.999 | INFO | Perf report for the last 5.13861 seconds: | 20260508 13:01:06.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:06.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | ANALYSIS : 65.0%, 3.0, 101.5ms, 0 | 20260508 13:01:06.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:01:06.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:06.000 | INFO | Uploaded 36GiB to GPU in 2.36453s (15.225GiB/s) | 20260508 13:01:06.000 | INFO | Downloaded 34GiB to GPU in 2.46556s (13.7899GiB/s) | 20260508 13:01:06.097 | INFO | Starting a round of Baz IO | 20260508 13:01:09.506 | INFO | Populating RTMetrics report | 20260508 13:01:09.507 | INFO | RTMetrics report populated | 20260508 13:01:09.507 | INFO | Opening tmp RTMetrics json file | 20260508 13:01:09.519 | INFO | Serializing RTMetrics to json | 20260508 13:01:09.520 | INFO | Serialization done, swapping tmp file now... | 20260508 13:01:09.521 | INFO | RTMetrics done | 20260508 13:01:09.578 | INFO | Finished a round of Baz IO: Wrote 1.396 GiB in 3.481 seconds | 20260508 13:01:09.579 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 13:01:09.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150609% / 0% / 5.5631e-05% | 20260508 13:01:09.579 | INFO | Baz IO overall: | 20260508 13:01:09.579 | INFO | BytesWritten : 3547779202816 | 20260508 13:01:09.579 | INFO | NumEvents : 1701669912117 | 20260508 13:01:11.141 | INFO | Analyzing chunk frames = [8302592,8303104) | 20260508 13:01:11.141 | INFO | MemInfo: { | 20260508 13:01:11.141 | INFO | "gpu" : " 17 GB", | 20260508 13:01:11.141 | INFO | "huge" : "", | 20260508 13:01:11.141 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:11.141 | INFO | "malloc" : " 14 GB", | 20260508 13:01:11.141 | INFO | "pinned" : " 29 GB" | 20260508 13:01:11.141 | INFO | } | 20260508 13:01:11.141 | INFO | Perf report for the last 5.14179 seconds: | 20260508 13:01:11.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:11.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:11.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:11.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:11.141 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:01:11.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:11.141 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 13:01:11.141 | INFO | PRE_HQ : 65.5%, 1.6, 53.9ms, 0 | 20260508 13:01:11.141 | INFO | BAZWRITER : 67.0%, 2.4, 84.2ms, 0 | 20260508 13:01:11.141 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260508 13:01:11.142 | INFO | Uploaded 36GiB to GPU in 2.60046s (13.8437GiB/s) | 20260508 13:01:11.142 | INFO | Downloaded 38GiB to GPU in 2.71738s (13.984GiB/s) | 20260508 13:01:16.285 | INFO | Analyzing chunk frames = [8303104,8303616) | 20260508 13:01:16.285 | INFO | MemInfo: { | 20260508 13:01:16.285 | INFO | "gpu" : " 17 GB", | 20260508 13:01:16.285 | INFO | "huge" : "", | 20260508 13:01:16.285 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:16.285 | INFO | "malloc" : " 14 GB", | 20260508 13:01:16.285 | INFO | "pinned" : " 29 GB" | 20260508 13:01:16.285 | INFO | } | 20260508 13:01:16.285 | INFO | Perf report for the last 5.14408 seconds: | 20260508 13:01:16.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:16.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.285 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 13:01:16.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.285 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:01:16.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:16.286 | INFO | Uploaded 36GiB to GPU in 2.48718s (14.4742GiB/s) | 20260508 13:01:16.286 | INFO | Downloaded 34GiB to GPU in 2.46853s (13.7734GiB/s) | 20260508 13:01:21.419 | INFO | Analyzing chunk frames = [8303616,8304128) | 20260508 13:01:21.419 | INFO | MemInfo: { | 20260508 13:01:21.419 | INFO | "gpu" : " 17 GB", | 20260508 13:01:21.419 | INFO | "huge" : "", | 20260508 13:01:21.419 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:21.419 | INFO | "malloc" : " 14 GB", | 20260508 13:01:21.419 | INFO | "pinned" : " 29 GB" | 20260508 13:01:21.419 | INFO | } | 20260508 13:01:21.419 | INFO | Perf report for the last 5.13389 seconds: | 20260508 13:01:21.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:21.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.419 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:01:21.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.419 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:01:21.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:21.420 | INFO | Uploaded 36GiB to GPU in 2.4683s (14.585GiB/s) | 20260508 13:01:21.420 | INFO | Downloaded 34GiB to GPU in 2.46854s (13.7733GiB/s) | 20260508 13:01:26.558 | INFO | Analyzing chunk frames = [8304128,8304640) | 20260508 13:01:26.558 | INFO | MemInfo: { | 20260508 13:01:26.558 | INFO | "gpu" : " 17 GB", | 20260508 13:01:26.558 | INFO | "huge" : "", | 20260508 13:01:26.558 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:26.558 | INFO | "malloc" : " 14 GB", | 20260508 13:01:26.558 | INFO | "pinned" : " 29 GB" | 20260508 13:01:26.558 | INFO | } | 20260508 13:01:26.559 | INFO | Perf report for the last 5.13927 seconds: | 20260508 13:01:26.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:26.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | ANALYSIS : 65.6%, 3.0, 101.8ms, 0 | 20260508 13:01:26.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:01:26.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:26.559 | INFO | Uploaded 36GiB to GPU in 2.44911s (14.6992GiB/s) | 20260508 13:01:26.559 | INFO | Downloaded 34GiB to GPU in 2.45681s (13.8391GiB/s) | 20260508 13:01:31.705 | INFO | Analyzing chunk frames = [8304640,8305152) | 20260508 13:01:31.705 | INFO | MemInfo: { | 20260508 13:01:31.705 | INFO | "gpu" : " 17 GB", | 20260508 13:01:31.705 | INFO | "huge" : "", | 20260508 13:01:31.705 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:31.705 | INFO | "malloc" : " 14 GB", | 20260508 13:01:31.705 | INFO | "pinned" : " 29 GB" | 20260508 13:01:31.705 | INFO | } | 20260508 13:01:31.705 | INFO | Perf report for the last 5.1466 seconds: | 20260508 13:01:31.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:31.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | ANALYSIS : 66.5%, 3.0, 104.0ms, 0 | 20260508 13:01:31.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:01:31.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:31.705 | INFO | Uploaded 36GiB to GPU in 2.4758s (14.5407GiB/s) | 20260508 13:01:31.705 | INFO | Downloaded 34GiB to GPU in 2.54255s (13.3724GiB/s) | 20260508 13:01:36.841 | INFO | Analyzing chunk frames = [8305152,8305664) | 20260508 13:01:36.841 | INFO | MemInfo: { | 20260508 13:01:36.841 | INFO | "gpu" : " 17 GB", | 20260508 13:01:36.841 | INFO | "huge" : "", | 20260508 13:01:36.841 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:36.841 | INFO | "malloc" : " 14 GB", | 20260508 13:01:36.841 | INFO | "pinned" : " 29 GB" | 20260508 13:01:36.841 | INFO | } | 20260508 13:01:36.841 | INFO | Perf report for the last 5.13559 seconds: | 20260508 13:01:36.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:36.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:01:36.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:01:36.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:36.841 | INFO | Uploaded 36GiB to GPU in 2.46603s (14.5984GiB/s) | 20260508 13:01:36.841 | INFO | Downloaded 34GiB to GPU in 2.43597s (13.9575GiB/s) | 20260508 13:01:41.976 | INFO | Analyzing chunk frames = [8305664,8306176) | 20260508 13:01:41.976 | INFO | MemInfo: { | 20260508 13:01:41.976 | INFO | "gpu" : " 17 GB", | 20260508 13:01:41.976 | INFO | "huge" : "", | 20260508 13:01:41.976 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:41.976 | INFO | "malloc" : " 14 GB", | 20260508 13:01:41.976 | INFO | "pinned" : " 29 GB" | 20260508 13:01:41.976 | INFO | } | 20260508 13:01:41.976 | INFO | Perf report for the last 5.13552 seconds: | 20260508 13:01:41.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:41.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.976 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 13:01:41.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.976 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:01:41.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:41.977 | INFO | Uploaded 36GiB to GPU in 2.49126s (14.4505GiB/s) | 20260508 13:01:41.977 | INFO | Downloaded 34GiB to GPU in 2.45643s (13.8412GiB/s) | 20260508 13:01:47.113 | INFO | Analyzing chunk frames = [8306176,8306688) | 20260508 13:01:47.113 | INFO | MemInfo: { | 20260508 13:01:47.113 | INFO | "gpu" : " 17 GB", | 20260508 13:01:47.113 | INFO | "huge" : "", | 20260508 13:01:47.113 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:47.113 | INFO | "malloc" : " 14 GB", | 20260508 13:01:47.113 | INFO | "pinned" : " 29 GB" | 20260508 13:01:47.114 | INFO | } | 20260508 13:01:47.114 | INFO | Perf report for the last 5.13733 seconds: | 20260508 13:01:47.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:47.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 13:01:47.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:01:47.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:47.114 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:01:47.114 | INFO | Total mem usage: 16.7 GiB | 20260508 13:01:47.114 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:01:47.114 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:01:47.114 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:01:47.114 | INFO | High Water Breakdown: | 20260508 13:01:47.114 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:01:47.114 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:01:47.114 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:01:47.114 | INFO | ------------------------------------------ | 20260508 13:01:47.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:01:47.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:01:47.114 | INFO | Total mem usage: 26.5 GiB | 20260508 13:01:47.114 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:01:47.114 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:01:47.114 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:01:47.114 | INFO | High Water Breakdown: | 20260508 13:01:47.114 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:01:47.114 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:01:47.114 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:01:47.114 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:01:47.114 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:01:47.114 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:01:47.114 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:01:47.114 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:01:47.114 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:01:47.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:01:47.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:01:47.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:01:47.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:01:47.114 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:01:47.114 | INFO | ------------------------------------------ | 20260508 13:01:47.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:01:47.114 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:01:47.114 | INFO | Total mem usage: 27.8 GiB | 20260508 13:01:47.114 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:01:47.114 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:01:47.114 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:01:47.114 | INFO | High Water Breakdown: | 20260508 13:01:47.114 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:01:47.114 | INFO | ------------------------------------------ | 20260508 13:01:47.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:01:47.114 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:01:47.114 | INFO | Total mem usage: 15.7 GiB | 20260508 13:01:47.114 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:01:47.114 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:01:47.114 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:01:47.114 | INFO | High Water Breakdown: | 20260508 13:01:47.114 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:01:47.114 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:01:47.114 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:01:47.114 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:01:47.114 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:01:47.114 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:01:47.114 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:01:47.114 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:01:47.114 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:01:47.114 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:01:47.114 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:01:47.114 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:01:47.114 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:01:47.114 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:01:47.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:01:47.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:01:47.114 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:01:47.114 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:01:47.114 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:01:47.114 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:01:47.114 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:01:47.114 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:01:47.114 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:01:47.114 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:01:47.114 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:01:47.114 | INFO | ------------------------------------------ | 20260508 13:01:47.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:01:47.114 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:01:47.114 | INFO | Uploaded 36GiB to GPU in 2.47863s (14.5242GiB/s) | 20260508 13:01:47.114 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:01:47.114 | INFO | Downloaded 34GiB to GPU in 2.44696s (13.8948GiB/s) | 20260508 13:01:47.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:01:47.115 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 6.18e-01 | 20260508 13:01:47.115 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.17e-02 / 4.35e-03 | 20260508 13:01:47.115 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.43e+00, 5.94e-01 / 2.19e-01 | 20260508 13:01:47.115 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.47e+00, 2.08e-01 / 9.47e-02 | 20260508 13:01:47.115 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.51e+01, 3.68e+00 / 9.10e-01 | 20260508 13:01:47.115 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.89e+00 / 1.23e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.30e+00 / 2.43e+00, 4.03e-01 / 2.12e-01 | 20260508 13:01:47.115 | INFO | PerfReport: 2.93%, Metrics -- 9.58e-01 / 1.09e+00, 2.40e-01 / 1.40e-01 | 20260508 13:01:47.115 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.13e-01, 4.68e-02 / 2.30e-02 | 20260508 13:01:47.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:01:47.115 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:01:47.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:01:47.115 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 24.43%, Upload -- 2.56e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 24.81%, Download -- 2.60e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 31.14%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 2.11%, LockUpload -- 2.22e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 2.60%, LockDownload -- 2.73e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 14.89%, LockCompute -- 1.56e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 13:01:47.115 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 3.31e-02 / 0.00e+00 | 20260508 13:01:47.211 | INFO | Starting a round of Baz IO | 20260508 13:01:50.612 | INFO | Populating RTMetrics report | 20260508 13:01:50.612 | INFO | RTMetrics report populated | 20260508 13:01:50.612 | INFO | Opening tmp RTMetrics json file | 20260508 13:01:50.626 | INFO | Serializing RTMetrics to json | 20260508 13:01:50.628 | INFO | Serialization done, swapping tmp file now... | 20260508 13:01:50.628 | INFO | RTMetrics done | 20260508 13:01:50.686 | INFO | Finished a round of Baz IO: Wrote 1.395 GiB in 3.474 seconds | 20260508 13:01:50.686 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 13:01:50.686 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15047% / 3.97364e-06% / 6.35783e-05% | 20260508 13:01:50.686 | INFO | Baz IO overall: | 20260508 13:01:50.686 | INFO | BytesWritten : 3549276965242 | 20260508 13:01:50.686 | INFO | NumEvents : 1702371546100 | 20260508 13:01:52.258 | INFO | Analyzing chunk frames = [8306688,8307200) | 20260508 13:01:52.258 | INFO | MemInfo: { | 20260508 13:01:52.258 | INFO | "gpu" : " 17 GB", | 20260508 13:01:52.258 | INFO | "huge" : "", | 20260508 13:01:52.258 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:52.258 | INFO | "malloc" : " 14 GB", | 20260508 13:01:52.258 | INFO | "pinned" : " 29 GB" | 20260508 13:01:52.258 | INFO | } | 20260508 13:01:52.258 | INFO | Perf report for the last 5.14441 seconds: | 20260508 13:01:52.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:52.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:52.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:52.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:52.258 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:01:52.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:52.258 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 13:01:52.258 | INFO | PRE_HQ : 65.1%, 1.6, 53.9ms, 0 | 20260508 13:01:52.258 | INFO | BAZWRITER : 66.9%, 2.4, 83.6ms, 0 | 20260508 13:01:52.258 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:01:52.259 | INFO | Uploaded 36GiB to GPU in 2.63974s (13.6377GiB/s) | 20260508 13:01:52.259 | INFO | Downloaded 38GiB to GPU in 2.71412s (14.0008GiB/s) | 20260508 13:01:57.402 | INFO | Analyzing chunk frames = [8307200,8307712) | 20260508 13:01:57.402 | INFO | MemInfo: { | 20260508 13:01:57.402 | INFO | "gpu" : " 17 GB", | 20260508 13:01:57.402 | INFO | "huge" : "", | 20260508 13:01:57.402 | INFO | "huge_shared" : " 29 GB", | 20260508 13:01:57.402 | INFO | "malloc" : " 14 GB", | 20260508 13:01:57.402 | INFO | "pinned" : " 29 GB" | 20260508 13:01:57.402 | INFO | } | 20260508 13:01:57.402 | INFO | Perf report for the last 5.14413 seconds: | 20260508 13:01:57.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:01:57.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:01:57.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 13:01:57.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:01:57.402 | INFO | Uploaded 36GiB to GPU in 2.48462s (14.4892GiB/s) | 20260508 13:01:57.403 | INFO | Downloaded 34GiB to GPU in 2.45936s (13.8247GiB/s) | 20260508 13:02:02.538 | INFO | Analyzing chunk frames = [8307712,8308224) | 20260508 13:02:02.538 | INFO | MemInfo: { | 20260508 13:02:02.538 | INFO | "gpu" : " 17 GB", | 20260508 13:02:02.538 | INFO | "huge" : "", | 20260508 13:02:02.538 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:02.538 | INFO | "malloc" : " 14 GB", | 20260508 13:02:02.539 | INFO | "pinned" : " 29 GB" | 20260508 13:02:02.539 | INFO | } | 20260508 13:02:02.539 | INFO | Perf report for the last 5.13637 seconds: | 20260508 13:02:02.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:02.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 13:02:02.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:02:02.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:02.539 | INFO | Uploaded 36GiB to GPU in 2.47061s (14.5713GiB/s) | 20260508 13:02:02.539 | INFO | Downloaded 34GiB to GPU in 2.43983s (13.9354GiB/s) | 20260508 13:02:07.671 | INFO | Analyzing chunk frames = [8308224,8308736) | 20260508 13:02:07.671 | INFO | MemInfo: { | 20260508 13:02:07.671 | INFO | "gpu" : " 17 GB", | 20260508 13:02:07.671 | INFO | "huge" : "", | 20260508 13:02:07.671 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:07.672 | INFO | "malloc" : " 14 GB", | 20260508 13:02:07.672 | INFO | "pinned" : " 29 GB" | 20260508 13:02:07.672 | INFO | } | 20260508 13:02:07.672 | INFO | Perf report for the last 5.13307 seconds: | 20260508 13:02:07.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:07.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | ANALYSIS : 65.6%, 3.0, 101.6ms, 0 | 20260508 13:02:07.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:02:07.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:07.672 | INFO | Uploaded 36GiB to GPU in 2.43321s (14.7953GiB/s) | 20260508 13:02:07.672 | INFO | Downloaded 34GiB to GPU in 2.46802s (13.7762GiB/s) | 20260508 13:02:12.816 | INFO | Analyzing chunk frames = [8308736,8309248) | 20260508 13:02:12.816 | INFO | MemInfo: { | 20260508 13:02:12.816 | INFO | "gpu" : " 17 GB", | 20260508 13:02:12.816 | INFO | "huge" : "", | 20260508 13:02:12.816 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:12.816 | INFO | "malloc" : " 14 GB", | 20260508 13:02:12.816 | INFO | "pinned" : " 29 GB" | 20260508 13:02:12.816 | INFO | } | 20260508 13:02:12.816 | INFO | Perf report for the last 5.14433 seconds: | 20260508 13:02:12.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:12.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:02:12.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:02:12.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:12.816 | INFO | Uploaded 36GiB to GPU in 2.35315s (15.2986GiB/s) | 20260508 13:02:12.816 | INFO | Downloaded 34GiB to GPU in 2.49705s (13.6161GiB/s) | 20260508 13:02:17.952 | INFO | Analyzing chunk frames = [8309248,8309760) | 20260508 13:02:17.952 | INFO | MemInfo: { | 20260508 13:02:17.952 | INFO | "gpu" : " 17 GB", | 20260508 13:02:17.952 | INFO | "huge" : "", | 20260508 13:02:17.952 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:17.952 | INFO | "malloc" : " 14 GB", | 20260508 13:02:17.952 | INFO | "pinned" : " 29 GB" | 20260508 13:02:17.952 | INFO | } | 20260508 13:02:17.952 | INFO | Perf report for the last 5.1363 seconds: | 20260508 13:02:17.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:17.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.952 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:02:17.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.952 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:02:17.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:17.953 | INFO | Uploaded 36GiB to GPU in 2.42817s (14.826GiB/s) | 20260508 13:02:17.953 | INFO | Downloaded 34GiB to GPU in 2.43062s (13.9882GiB/s) | 20260508 13:02:23.090 | INFO | Analyzing chunk frames = [8309760,8310272) | 20260508 13:02:23.090 | INFO | MemInfo: { | 20260508 13:02:23.090 | INFO | "gpu" : " 17 GB", | 20260508 13:02:23.090 | INFO | "huge" : "", | 20260508 13:02:23.090 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:23.090 | INFO | "malloc" : " 14 GB", | 20260508 13:02:23.090 | INFO | "pinned" : " 29 GB" | 20260508 13:02:23.090 | INFO | } | 20260508 13:02:23.090 | INFO | Perf report for the last 5.13786 seconds: | 20260508 13:02:23.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:23.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.090 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 13:02:23.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.090 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:02:23.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:23.091 | INFO | Uploaded 36GiB to GPU in 2.38659s (15.0843GiB/s) | 20260508 13:02:23.091 | INFO | Downloaded 34GiB to GPU in 2.41103s (14.1018GiB/s) | 20260508 13:02:28.234 | INFO | Analyzing chunk frames = [8310272,8310784) | 20260508 13:02:28.234 | INFO | MemInfo: { | 20260508 13:02:28.234 | INFO | "gpu" : " 17 GB", | 20260508 13:02:28.234 | INFO | "huge" : "", | 20260508 13:02:28.234 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:28.234 | INFO | "malloc" : " 14 GB", | 20260508 13:02:28.234 | INFO | "pinned" : " 29 GB" | 20260508 13:02:28.234 | INFO | } | 20260508 13:02:28.234 | INFO | Perf report for the last 5.14395 seconds: | 20260508 13:02:28.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:28.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:02:28.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:02:28.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:28.234 | INFO | Uploaded 36GiB to GPU in 2.48956s (14.4604GiB/s) | 20260508 13:02:28.234 | INFO | Downloaded 34GiB to GPU in 2.45093s (13.8723GiB/s) | 20260508 13:02:28.331 | INFO | Starting a round of Baz IO | 20260508 13:02:31.743 | INFO | Populating RTMetrics report | 20260508 13:02:31.744 | INFO | RTMetrics report populated | 20260508 13:02:31.744 | INFO | Opening tmp RTMetrics json file | 20260508 13:02:31.754 | INFO | Serializing RTMetrics to json | 20260508 13:02:31.756 | INFO | Serialization done, swapping tmp file now... | 20260508 13:02:31.756 | INFO | RTMetrics done | 20260508 13:02:31.811 | INFO | Finished a round of Baz IO: Wrote 1.395 GiB in 3.48 seconds | 20260508 13:02:31.811 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 13:02:31.811 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149747% / 0% / 6.35783e-05% | 20260508 13:02:31.811 | INFO | Baz IO overall: | 20260508 13:02:31.811 | INFO | BytesWritten : 3550775153211 | 20260508 13:02:31.811 | INFO | NumEvents : 1703073352261 | 20260508 13:02:33.374 | INFO | Analyzing chunk frames = [8310784,8311296) | 20260508 13:02:33.374 | INFO | MemInfo: { | 20260508 13:02:33.374 | INFO | "gpu" : " 17 GB", | 20260508 13:02:33.374 | INFO | "huge" : "", | 20260508 13:02:33.374 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:33.374 | INFO | "malloc" : " 14 GB", | 20260508 13:02:33.374 | INFO | "pinned" : " 29 GB" | 20260508 13:02:33.374 | INFO | } | 20260508 13:02:33.374 | INFO | Perf report for the last 5.13983 seconds: | 20260508 13:02:33.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:33.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:33.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:33.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:33.374 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 13:02:33.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:33.374 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 13:02:33.374 | INFO | PRE_HQ : 65.6%, 1.6, 54.0ms, 0 | 20260508 13:02:33.374 | INFO | BAZWRITER : 67.3%, 2.7, 93.2ms, 0 | 20260508 13:02:33.374 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260508 13:02:33.374 | INFO | Uploaded 36GiB to GPU in 2.6155s (13.7641GiB/s) | 20260508 13:02:33.374 | INFO | Downloaded 38GiB to GPU in 2.71168s (14.0134GiB/s) | 20260508 13:02:38.513 | INFO | Analyzing chunk frames = [8311296,8311808) | 20260508 13:02:38.513 | INFO | MemInfo: { | 20260508 13:02:38.513 | INFO | "gpu" : " 17 GB", | 20260508 13:02:38.513 | INFO | "huge" : "", | 20260508 13:02:38.513 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:38.513 | INFO | "malloc" : " 14 GB", | 20260508 13:02:38.513 | INFO | "pinned" : " 29 GB" | 20260508 13:02:38.513 | INFO | } | 20260508 13:02:38.513 | INFO | Perf report for the last 5.13921 seconds: | 20260508 13:02:38.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:38.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:02:38.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:02:38.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:38.513 | INFO | Uploaded 36GiB to GPU in 2.51358s (14.3222GiB/s) | 20260508 13:02:38.514 | INFO | Downloaded 34GiB to GPU in 2.45585s (13.8445GiB/s) | 20260508 13:02:43.654 | INFO | Analyzing chunk frames = [8311808,8312320) | 20260508 13:02:43.654 | INFO | MemInfo: { | 20260508 13:02:43.654 | INFO | "gpu" : " 17 GB", | 20260508 13:02:43.654 | INFO | "huge" : "", | 20260508 13:02:43.655 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:43.655 | INFO | "malloc" : " 14 GB", | 20260508 13:02:43.655 | INFO | "pinned" : " 29 GB" | 20260508 13:02:43.655 | INFO | } | 20260508 13:02:43.655 | INFO | Perf report for the last 5.1415 seconds: | 20260508 13:02:43.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:43.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:02:43.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:02:43.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:43.655 | INFO | Uploaded 36GiB to GPU in 2.5495s (14.1204GiB/s) | 20260508 13:02:43.655 | INFO | Downloaded 34GiB to GPU in 2.46131s (13.8138GiB/s) | 20260508 13:02:48.787 | INFO | Analyzing chunk frames = [8312320,8312832) | 20260508 13:02:48.787 | INFO | MemInfo: { | 20260508 13:02:48.787 | INFO | "gpu" : " 17 GB", | 20260508 13:02:48.787 | INFO | "huge" : "", | 20260508 13:02:48.787 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:48.787 | INFO | "malloc" : " 14 GB", | 20260508 13:02:48.787 | INFO | "pinned" : " 29 GB" | 20260508 13:02:48.788 | INFO | } | 20260508 13:02:48.788 | INFO | Perf report for the last 5.13284 seconds: | 20260508 13:02:48.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:48.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:02:48.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:02:48.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:48.788 | INFO | Uploaded 36GiB to GPU in 2.4209s (14.8705GiB/s) | 20260508 13:02:48.788 | INFO | Downloaded 34GiB to GPU in 2.46983s (13.7661GiB/s) | 20260508 13:02:53.921 | INFO | Analyzing chunk frames = [8312832,8313344) | 20260508 13:02:53.921 | INFO | MemInfo: { | 20260508 13:02:53.921 | INFO | "gpu" : " 17 GB", | 20260508 13:02:53.921 | INFO | "huge" : "", | 20260508 13:02:53.921 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:53.921 | INFO | "malloc" : " 14 GB", | 20260508 13:02:53.921 | INFO | "pinned" : " 29 GB" | 20260508 13:02:53.921 | INFO | } | 20260508 13:02:53.921 | INFO | Perf report for the last 5.1336 seconds: | 20260508 13:02:53.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:53.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.921 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 13:02:53.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.921 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 13:02:53.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:53.922 | INFO | Uploaded 36GiB to GPU in 2.45864s (14.6423GiB/s) | 20260508 13:02:53.922 | INFO | Downloaded 34GiB to GPU in 2.51315s (13.5288GiB/s) | 20260508 13:02:59.069 | INFO | Analyzing chunk frames = [8313344,8313856) | 20260508 13:02:59.069 | INFO | MemInfo: { | 20260508 13:02:59.069 | INFO | "gpu" : " 17 GB", | 20260508 13:02:59.069 | INFO | "huge" : "", | 20260508 13:02:59.069 | INFO | "huge_shared" : " 29 GB", | 20260508 13:02:59.069 | INFO | "malloc" : " 14 GB", | 20260508 13:02:59.069 | INFO | "pinned" : " 29 GB" | 20260508 13:02:59.069 | INFO | } | 20260508 13:02:59.069 | INFO | Perf report for the last 5.14828 seconds: | 20260508 13:02:59.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:02:59.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.070 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 13:02:59.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.070 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:02:59.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:02:59.070 | INFO | Uploaded 36GiB to GPU in 2.38338s (15.1046GiB/s) | 20260508 13:02:59.070 | INFO | Downloaded 34GiB to GPU in 2.43676s (13.9529GiB/s) | 20260508 13:03:04.210 | INFO | Analyzing chunk frames = [8313856,8314368) | 20260508 13:03:04.210 | INFO | MemInfo: { | 20260508 13:03:04.210 | INFO | "gpu" : " 17 GB", | 20260508 13:03:04.210 | INFO | "huge" : "", | 20260508 13:03:04.210 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:04.210 | INFO | "malloc" : " 14 GB", | 20260508 13:03:04.210 | INFO | "pinned" : " 29 GB" | 20260508 13:03:04.210 | INFO | } | 20260508 13:03:04.210 | INFO | Perf report for the last 5.14029 seconds: | 20260508 13:03:04.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:04.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.210 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 13:03:04.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.210 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 13:03:04.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:04.211 | INFO | Uploaded 36GiB to GPU in 2.48238s (14.5022GiB/s) | 20260508 13:03:04.211 | INFO | Downloaded 34GiB to GPU in 2.458s (13.8324GiB/s) | 20260508 13:03:09.346 | INFO | Analyzing chunk frames = [8314368,8314880) | 20260508 13:03:09.346 | INFO | MemInfo: { | 20260508 13:03:09.346 | INFO | "gpu" : " 17 GB", | 20260508 13:03:09.346 | INFO | "huge" : "", | 20260508 13:03:09.346 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:09.346 | INFO | "malloc" : " 14 GB", | 20260508 13:03:09.346 | INFO | "pinned" : " 29 GB" | 20260508 13:03:09.346 | INFO | } | 20260508 13:03:09.347 | INFO | Perf report for the last 5.13674 seconds: | 20260508 13:03:09.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:09.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260508 13:03:09.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260508 13:03:09.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:09.347 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:03:09.347 | INFO | Total mem usage: 16.7 GiB | 20260508 13:03:09.347 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:03:09.347 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -13.9%) | 20260508 13:03:09.347 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:03:09.347 | INFO | High Water Breakdown: | 20260508 13:03:09.347 | INFO | 59.9% : 8.78 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:03:09.347 | INFO | 39.9% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:03:09.347 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:03:09.347 | INFO | ------------------------------------------ | 20260508 13:03:09.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:03:09.347 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:03:09.347 | INFO | Total mem usage: 26.5 GiB | 20260508 13:03:09.347 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:03:09.347 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:03:09.347 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:03:09.347 | INFO | High Water Breakdown: | 20260508 13:03:09.347 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:03:09.347 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:03:09.347 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:03:09.347 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:03:09.347 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:03:09.347 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:03:09.347 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:03:09.347 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:03:09.347 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:03:09.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:03:09.348 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:03:09.348 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:03:09.348 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:03:09.348 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:03:09.348 | INFO | ------------------------------------------ | 20260508 13:03:09.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:03:09.348 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:03:09.348 | INFO | Total mem usage: 27.8 GiB | 20260508 13:03:09.348 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:03:09.348 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:03:09.348 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:03:09.348 | INFO | High Water Breakdown: | 20260508 13:03:09.348 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:03:09.348 | INFO | ------------------------------------------ | 20260508 13:03:09.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:03:09.348 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:03:09.348 | INFO | Total mem usage: 15.7 GiB | 20260508 13:03:09.348 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:03:09.348 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:03:09.348 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:03:09.348 | INFO | High Water Breakdown: | 20260508 13:03:09.348 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:03:09.348 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:03:09.348 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:03:09.348 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:03:09.348 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:03:09.348 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:03:09.348 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:03:09.348 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:03:09.348 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:03:09.348 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:03:09.348 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:03:09.348 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:03:09.348 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:03:09.348 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:03:09.348 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:03:09.348 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:03:09.348 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:03:09.348 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:03:09.348 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:03:09.348 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:03:09.348 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:03:09.348 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:03:09.348 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:03:09.348 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:03:09.348 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:03:09.348 | INFO | ------------------------------------------ | 20260508 13:03:09.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:03:09.348 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:03:09.348 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:03:09.348 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:03:09.348 | INFO | Uploaded 36GiB to GPU in 2.33887s (15.392GiB/s) | 20260508 13:03:09.348 | INFO | Downloaded 34GiB to GPU in 2.44002s (13.9343GiB/s) | 20260508 13:03:09.348 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 4.84e-01 | 20260508 13:03:09.348 | INFO | PerfReport: 0.30%, AggregateStats -- 9.73e-02 / 1.03e-01, 1.15e-02 / 4.45e-03 | 20260508 13:03:09.348 | INFO | PerfReport: 12.23%, Baseline -- 4.00e+00 / 4.40e+00, 5.99e-01 / 2.25e-01 | 20260508 13:03:09.348 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.47e+00, 2.11e-01 / 1.18e-01 | 20260508 13:03:09.348 | INFO | PerfReport: 15.98%, DME -- 4.18e+01 / 4.53e+01, 3.56e+00 / 1.03e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.87e+01 / 2.02e+01, 1.85e+00 / 1.42e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.47e+00, 4.03e-01 / 2.71e-01 | 20260508 13:03:09.348 | INFO | PerfReport: 2.94%, Metrics -- 9.61e-01 / 1.09e+00, 2.39e-01 / 1.09e-01 | 20260508 13:03:09.348 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.17e-01, 4.64e-02 / 2.34e-02 | 20260508 13:03:09.348 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:03:09.348 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:03:09.348 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:03:09.348 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 24.51%, Upload -- 2.57e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 24.74%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 31.19%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 2.23%, LockUpload -- 2.34e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 2.47%, LockDownload -- 2.59e+00 / 0.00e+00, 3.89e+00 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 14.85%, LockCompute -- 1.56e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:03:09.348 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260508 13:03:09.459 | INFO | Starting a round of Baz IO | 20260508 13:03:12.890 | INFO | Populating RTMetrics report | 20260508 13:03:12.890 | INFO | RTMetrics report populated | 20260508 13:03:12.890 | INFO | Opening tmp RTMetrics json file | 20260508 13:03:12.897 | INFO | Serializing RTMetrics to json | 20260508 13:03:12.898 | INFO | Serialization done, swapping tmp file now... | 20260508 13:03:12.899 | INFO | RTMetrics done | 20260508 13:03:12.962 | INFO | Finished a round of Baz IO: Wrote 1.396 GiB in 3.503 seconds | 20260508 13:03:12.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.98% / 0.00% / 60.02% | 20260508 13:03:12.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149858% / 0% / 7.15256e-05% | 20260508 13:03:12.962 | INFO | Baz IO overall: | 20260508 13:03:12.962 | INFO | BytesWritten : 3552273757850 | 20260508 13:03:12.962 | INFO | NumEvents : 1703775493714 | 20260508 13:03:14.484 | INFO | Analyzing chunk frames = [8314880,8315392) | 20260508 13:03:14.484 | INFO | MemInfo: { | 20260508 13:03:14.484 | INFO | "gpu" : " 17 GB", | 20260508 13:03:14.484 | INFO | "huge" : "", | 20260508 13:03:14.484 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:14.484 | INFO | "malloc" : " 14 GB", | 20260508 13:03:14.484 | INFO | "pinned" : " 29 GB" | 20260508 13:03:14.484 | INFO | } | 20260508 13:03:14.484 | INFO | Perf report for the last 5.13768 seconds: | 20260508 13:03:14.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:14.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:14.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:14.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:14.484 | INFO | ANALYSIS : 68.7%, 3.0, 106.6ms, 0 | 20260508 13:03:14.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:14.484 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260508 13:03:14.484 | INFO | PRE_HQ : 67.2%, 1.7, 61.0ms, 0 | 20260508 13:03:14.484 | INFO | BAZWRITER : 67.7%, 3.0, 103.9ms, 0 | 20260508 13:03:14.484 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:03:14.485 | INFO | Uploaded 36GiB to GPU in 2.55501s (14.09GiB/s) | 20260508 13:03:14.485 | INFO | Downloaded 38GiB to GPU in 2.71291s (14.0071GiB/s) | 20260508 13:03:19.624 | INFO | Analyzing chunk frames = [8315392,8315904) | 20260508 13:03:19.624 | INFO | MemInfo: { | 20260508 13:03:19.624 | INFO | "gpu" : " 17 GB", | 20260508 13:03:19.624 | INFO | "huge" : "", | 20260508 13:03:19.624 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:19.624 | INFO | "malloc" : " 14 GB", | 20260508 13:03:19.624 | INFO | "pinned" : " 29 GB" | 20260508 13:03:19.624 | INFO | } | 20260508 13:03:19.624 | INFO | Perf report for the last 5.14001 seconds: | 20260508 13:03:19.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:19.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.624 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:03:19.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.624 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:03:19.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:19.625 | INFO | Uploaded 36GiB to GPU in 2.47158s (14.5656GiB/s) | 20260508 13:03:19.625 | INFO | Downloaded 34GiB to GPU in 2.46903s (13.7706GiB/s) | 20260508 13:03:24.761 | INFO | Analyzing chunk frames = [8315904,8316416) | 20260508 13:03:24.761 | INFO | MemInfo: { | 20260508 13:03:24.761 | INFO | "gpu" : " 17 GB", | 20260508 13:03:24.761 | INFO | "huge" : "", | 20260508 13:03:24.761 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:24.761 | INFO | "malloc" : " 14 GB", | 20260508 13:03:24.761 | INFO | "pinned" : " 29 GB" | 20260508 13:03:24.761 | INFO | } | 20260508 13:03:24.761 | INFO | Perf report for the last 5.13678 seconds: | 20260508 13:03:24.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:24.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.761 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 13:03:24.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.761 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:03:24.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:24.762 | INFO | Uploaded 36GiB to GPU in 2.50218s (14.3874GiB/s) | 20260508 13:03:24.762 | INFO | Downloaded 34GiB to GPU in 2.45217s (13.8652GiB/s) | 20260508 13:03:29.906 | INFO | Analyzing chunk frames = [8316416,8316928) | 20260508 13:03:29.906 | INFO | MemInfo: { | 20260508 13:03:29.906 | INFO | "gpu" : " 17 GB", | 20260508 13:03:29.906 | INFO | "huge" : "", | 20260508 13:03:29.906 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:29.906 | INFO | "malloc" : " 14 GB", | 20260508 13:03:29.906 | INFO | "pinned" : " 29 GB" | 20260508 13:03:29.906 | INFO | } | 20260508 13:03:29.906 | INFO | Perf report for the last 5.1448 seconds: | 20260508 13:03:29.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:29.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.906 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 13:03:29.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.906 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:03:29.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:29.907 | INFO | Uploaded 36GiB to GPU in 2.46767s (14.5886GiB/s) | 20260508 13:03:29.907 | INFO | Downloaded 34GiB to GPU in 2.44232s (13.9212GiB/s) | 20260508 13:03:35.046 | INFO | Analyzing chunk frames = [8316928,8317440) | 20260508 13:03:35.046 | INFO | MemInfo: { | 20260508 13:03:35.046 | INFO | "gpu" : " 17 GB", | 20260508 13:03:35.046 | INFO | "huge" : "", | 20260508 13:03:35.046 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:35.046 | INFO | "malloc" : " 14 GB", | 20260508 13:03:35.046 | INFO | "pinned" : " 29 GB" | 20260508 13:03:35.046 | INFO | } | 20260508 13:03:35.046 | INFO | Perf report for the last 5.1406 seconds: | 20260508 13:03:35.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:35.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.047 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 13:03:35.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.047 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:03:35.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:35.047 | INFO | Uploaded 36GiB to GPU in 2.44767s (14.7079GiB/s) | 20260508 13:03:35.047 | INFO | Downloaded 34GiB to GPU in 2.51173s (13.5365GiB/s) | 20260508 13:03:40.179 | INFO | Analyzing chunk frames = [8317440,8317952) | 20260508 13:03:40.179 | INFO | MemInfo: { | 20260508 13:03:40.179 | INFO | "gpu" : " 17 GB", | 20260508 13:03:40.179 | INFO | "huge" : "", | 20260508 13:03:40.179 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:40.179 | INFO | "malloc" : " 14 GB", | 20260508 13:03:40.179 | INFO | "pinned" : " 29 GB" | 20260508 13:03:40.179 | INFO | } | 20260508 13:03:40.179 | INFO | Perf report for the last 5.13258 seconds: | 20260508 13:03:40.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:40.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 13:03:40.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:03:40.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:40.179 | INFO | Uploaded 36GiB to GPU in 2.41027s (14.9361GiB/s) | 20260508 13:03:40.179 | INFO | Downloaded 34GiB to GPU in 2.43971s (13.9361GiB/s) | 20260508 13:03:45.330 | INFO | Analyzing chunk frames = [8317952,8318464) | 20260508 13:03:45.330 | INFO | MemInfo: { | 20260508 13:03:45.330 | INFO | "gpu" : " 17 GB", | 20260508 13:03:45.330 | INFO | "huge" : "", | 20260508 13:03:45.330 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:45.330 | INFO | "malloc" : " 14 GB", | 20260508 13:03:45.330 | INFO | "pinned" : " 29 GB" | 20260508 13:03:45.330 | INFO | } | 20260508 13:03:45.330 | INFO | Perf report for the last 5.15077 seconds: | 20260508 13:03:45.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:45.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | ANALYSIS : 65.2%, 3.0, 101.9ms, 0 | 20260508 13:03:45.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | PRE_HQ : 24.3%, 1.0, 12.7ms, 0 | 20260508 13:03:45.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:45.330 | INFO | Uploaded 36GiB to GPU in 2.46384s (14.6113GiB/s) | 20260508 13:03:45.330 | INFO | Downloaded 34GiB to GPU in 2.43365s (13.9708GiB/s) | 20260508 13:03:50.467 | INFO | Analyzing chunk frames = [8318464,8318976) | 20260508 13:03:50.467 | INFO | MemInfo: { | 20260508 13:03:50.467 | INFO | "gpu" : " 17 GB", | 20260508 13:03:50.467 | INFO | "huge" : "", | 20260508 13:03:50.467 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:50.467 | INFO | "malloc" : " 14 GB", | 20260508 13:03:50.467 | INFO | "pinned" : " 29 GB" | 20260508 13:03:50.467 | INFO | } | 20260508 13:03:50.467 | INFO | Perf report for the last 5.13759 seconds: | 20260508 13:03:50.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:50.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.467 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260508 13:03:50.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.467 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:03:50.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:50.468 | INFO | Uploaded 36GiB to GPU in 2.36117s (15.2467GiB/s) | 20260508 13:03:50.468 | INFO | Downloaded 34GiB to GPU in 2.43909s (13.9396GiB/s) | 20260508 13:03:50.565 | INFO | Starting a round of Baz IO | 20260508 13:03:53.979 | INFO | Populating RTMetrics report | 20260508 13:03:53.979 | INFO | RTMetrics report populated | 20260508 13:03:53.979 | INFO | Opening tmp RTMetrics json file | 20260508 13:03:53.990 | INFO | Serializing RTMetrics to json | 20260508 13:03:53.991 | INFO | Serialization done, swapping tmp file now... | 20260508 13:03:53.992 | INFO | RTMetrics done | 20260508 13:03:54.052 | INFO | Finished a round of Baz IO: Wrote 1.396 GiB in 3.486 seconds | 20260508 13:03:54.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.02% | 20260508 13:03:54.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150319% / 0% / 7.15256e-05% | 20260508 13:03:54.052 | INFO | Baz IO overall: | 20260508 13:03:54.052 | INFO | BytesWritten : 3553772556944 | 20260508 13:03:54.052 | INFO | NumEvents : 1704478071001 | 20260508 13:03:55.597 | INFO | Analyzing chunk frames = [8318976,8319488) | 20260508 13:03:55.597 | INFO | MemInfo: { | 20260508 13:03:55.597 | INFO | "gpu" : " 17 GB", | 20260508 13:03:55.597 | INFO | "huge" : "", | 20260508 13:03:55.597 | INFO | "huge_shared" : " 29 GB", | 20260508 13:03:55.597 | INFO | "malloc" : " 14 GB", | 20260508 13:03:55.597 | INFO | "pinned" : " 29 GB" | 20260508 13:03:55.597 | INFO | } | 20260508 13:03:55.597 | INFO | Perf report for the last 5.12996 seconds: | 20260508 13:03:55.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:03:55.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:55.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:55.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:55.598 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:03:55.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:03:55.598 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:03:55.598 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 13:03:55.598 | INFO | BAZWRITER : 67.5%, 3.0, 102.4ms, 0 | 20260508 13:03:55.598 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:03:55.598 | INFO | Uploaded 36GiB to GPU in 2.64078s (13.6323GiB/s) | 20260508 13:03:55.598 | INFO | Downloaded 38GiB to GPU in 2.71421s (14.0004GiB/s) | 20260508 13:04:00.740 | INFO | Analyzing chunk frames = [8319488,8320000) | 20260508 13:04:00.740 | INFO | MemInfo: { | 20260508 13:04:00.740 | INFO | "gpu" : " 17 GB", | 20260508 13:04:00.740 | INFO | "huge" : "", | 20260508 13:04:00.740 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:00.740 | INFO | "malloc" : " 14 GB", | 20260508 13:04:00.740 | INFO | "pinned" : " 29 GB" | 20260508 13:04:00.740 | INFO | } | 20260508 13:04:00.740 | INFO | Perf report for the last 5.14249 seconds: | 20260508 13:04:00.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:00.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:04:00.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:04:00.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:00.740 | INFO | Uploaded 36GiB to GPU in 2.46195s (14.6226GiB/s) | 20260508 13:04:00.740 | INFO | Downloaded 34GiB to GPU in 2.47759s (13.723GiB/s) | 20260508 13:04:05.880 | INFO | Analyzing chunk frames = [8320000,8320512) | 20260508 13:04:05.880 | INFO | MemInfo: { | 20260508 13:04:05.880 | INFO | "gpu" : " 17 GB", | 20260508 13:04:05.880 | INFO | "huge" : "", | 20260508 13:04:05.880 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:05.880 | INFO | "malloc" : " 14 GB", | 20260508 13:04:05.880 | INFO | "pinned" : " 29 GB" | 20260508 13:04:05.880 | INFO | } | 20260508 13:04:05.881 | INFO | Perf report for the last 5.14057 seconds: | 20260508 13:04:05.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:05.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 13:04:05.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:04:05.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:05.881 | INFO | Uploaded 36GiB to GPU in 2.5088s (14.3495GiB/s) | 20260508 13:04:05.881 | INFO | Downloaded 34GiB to GPU in 2.44824s (13.8875GiB/s) | 20260508 13:04:11.021 | INFO | Analyzing chunk frames = [8320512,8321024) | 20260508 13:04:11.021 | INFO | MemInfo: { | 20260508 13:04:11.021 | INFO | "gpu" : " 17 GB", | 20260508 13:04:11.021 | INFO | "huge" : "", | 20260508 13:04:11.021 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:11.021 | INFO | "malloc" : " 14 GB", | 20260508 13:04:11.021 | INFO | "pinned" : " 29 GB" | 20260508 13:04:11.021 | INFO | } | 20260508 13:04:11.022 | INFO | Perf report for the last 5.14112 seconds: | 20260508 13:04:11.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:11.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:04:11.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:04:11.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:11.022 | INFO | Uploaded 36GiB to GPU in 2.52583s (14.2528GiB/s) | 20260508 13:04:11.022 | INFO | Downloaded 34GiB to GPU in 2.46888s (13.7714GiB/s) | 20260508 13:04:16.163 | INFO | Analyzing chunk frames = [8321024,8321536) | 20260508 13:04:16.163 | INFO | MemInfo: { | 20260508 13:04:16.163 | INFO | "gpu" : " 17 GB", | 20260508 13:04:16.163 | INFO | "huge" : "", | 20260508 13:04:16.163 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:16.163 | INFO | "malloc" : " 14 GB", | 20260508 13:04:16.163 | INFO | "pinned" : " 29 GB" | 20260508 13:04:16.163 | INFO | } | 20260508 13:04:16.163 | INFO | Perf report for the last 5.1417 seconds: | 20260508 13:04:16.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:16.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.163 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 13:04:16.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.164 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 13:04:16.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:16.164 | INFO | Uploaded 36GiB to GPU in 2.45482s (14.665GiB/s) | 20260508 13:04:16.164 | INFO | Downloaded 34GiB to GPU in 2.51555s (13.5159GiB/s) | 20260508 13:04:21.297 | INFO | Analyzing chunk frames = [8321536,8322048) | 20260508 13:04:21.297 | INFO | MemInfo: { | 20260508 13:04:21.297 | INFO | "gpu" : " 17 GB", | 20260508 13:04:21.297 | INFO | "huge" : "", | 20260508 13:04:21.297 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:21.297 | INFO | "malloc" : " 14 GB", | 20260508 13:04:21.297 | INFO | "pinned" : " 29 GB" | 20260508 13:04:21.297 | INFO | } | 20260508 13:04:21.297 | INFO | Perf report for the last 5.13397 seconds: | 20260508 13:04:21.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:21.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.297 | INFO | ANALYSIS : 64.7%, 3.0, 100.5ms, 0 | 20260508 13:04:21.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.297 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 13:04:21.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:21.298 | INFO | Uploaded 36GiB to GPU in 2.3286s (15.46GiB/s) | 20260508 13:04:21.298 | INFO | Downloaded 34GiB to GPU in 2.39856s (14.1752GiB/s) | 20260508 13:04:26.429 | INFO | Analyzing chunk frames = [8322048,8322560) | 20260508 13:04:26.429 | INFO | MemInfo: { | 20260508 13:04:26.429 | INFO | "gpu" : " 17 GB", | 20260508 13:04:26.429 | INFO | "huge" : "", | 20260508 13:04:26.430 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:26.430 | INFO | "malloc" : " 14 GB", | 20260508 13:04:26.430 | INFO | "pinned" : " 29 GB" | 20260508 13:04:26.430 | INFO | } | 20260508 13:04:26.430 | INFO | Perf report for the last 5.13235 seconds: | 20260508 13:04:26.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:26.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 13:04:26.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:04:26.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:26.430 | INFO | Uploaded 36GiB to GPU in 2.51327s (14.324GiB/s) | 20260508 13:04:26.430 | INFO | Downloaded 34GiB to GPU in 2.45713s (13.8373GiB/s) | 20260508 13:04:31.580 | INFO | Analyzing chunk frames = [8322560,8323072) | 20260508 13:04:31.580 | INFO | MemInfo: { | 20260508 13:04:31.580 | INFO | "gpu" : " 17 GB", | 20260508 13:04:31.580 | INFO | "huge" : "", | 20260508 13:04:31.580 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:31.580 | INFO | "malloc" : " 14 GB", | 20260508 13:04:31.580 | INFO | "pinned" : " 29 GB" | 20260508 13:04:31.580 | INFO | } | 20260508 13:04:31.580 | INFO | Perf report for the last 5.15062 seconds: | 20260508 13:04:31.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:31.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.580 | INFO | ANALYSIS : 65.6%, 3.0, 102.8ms, 0 | 20260508 13:04:31.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.580 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:04:31.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:31.581 | INFO | Uploaded 36GiB to GPU in 2.48042s (14.5137GiB/s) | 20260508 13:04:31.581 | INFO | Downloaded 34GiB to GPU in 2.44558s (13.9027GiB/s) | 20260508 13:04:31.581 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:04:31.581 | INFO | Total mem usage: 16.7 GiB | 20260508 13:04:31.581 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:04:31.581 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260508 13:04:31.581 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:04:31.581 | INFO | High Water Breakdown: | 20260508 13:04:31.581 | INFO | 60.2% : 8.96 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:04:31.581 | INFO | 39.7% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:04:31.581 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:04:31.581 | INFO | ------------------------------------------ | 20260508 13:04:31.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:04:31.581 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:04:31.581 | INFO | Total mem usage: 26.5 GiB | 20260508 13:04:31.581 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:04:31.581 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 13:04:31.581 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 13:04:31.581 | INFO | High Water Breakdown: | 20260508 13:04:31.581 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:04:31.581 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:04:31.581 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:04:31.581 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:04:31.581 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:04:31.581 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:04:31.581 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:04:31.581 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:04:31.581 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:04:31.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:04:31.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:04:31.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:04:31.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:04:31.581 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:04:31.581 | INFO | ------------------------------------------ | 20260508 13:04:31.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:04:31.582 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:04:31.582 | INFO | Total mem usage: 27.8 GiB | 20260508 13:04:31.582 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:04:31.582 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:04:31.582 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:04:31.582 | INFO | High Water Breakdown: | 20260508 13:04:31.582 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:04:31.582 | INFO | ------------------------------------------ | 20260508 13:04:31.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:04:31.582 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:04:31.582 | INFO | Total mem usage: 15.7 GiB | 20260508 13:04:31.582 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:04:31.582 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:04:31.582 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:04:31.582 | INFO | High Water Breakdown: | 20260508 13:04:31.582 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:04:31.582 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:04:31.582 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:04:31.582 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:04:31.582 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:04:31.582 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:04:31.582 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:04:31.582 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:04:31.582 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:04:31.582 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:04:31.582 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:04:31.582 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:04:31.582 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:04:31.582 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:04:31.582 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:04:31.582 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:04:31.582 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:04:31.582 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:04:31.582 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:04:31.582 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:04:31.582 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:04:31.582 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:04:31.582 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:04:31.582 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:04:31.582 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:04:31.582 | INFO | ------------------------------------------ | 20260508 13:04:31.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:04:31.582 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:04:31.582 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:04:31.582 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:04:31.582 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.58e+01, 1.41e+01 / 6.18e-01 | 20260508 13:04:31.582 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.10e-02 / 4.14e-03 | 20260508 13:04:31.582 | INFO | PerfReport: 12.18%, Baseline -- 3.97e+00 / 4.46e+00, 5.95e-01 / 2.21e-01 | 20260508 13:04:31.582 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.37e+00, 2.09e-01 / 9.27e-02 | 20260508 13:04:31.582 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.52e+01, 3.58e+00 / 1.15e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.86e+00 / 1.08e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.31e+00 / 2.34e+00, 4.00e-01 / 1.68e-01 | 20260508 13:04:31.582 | INFO | PerfReport: 2.94%, Metrics -- 9.61e-01 / 1.19e+00, 2.40e-01 / 1.09e-01 | 20260508 13:04:31.582 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.30e-01, 4.72e-02 / 2.87e-02 | 20260508 13:04:31.582 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:04:31.582 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:04:31.582 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:04:31.582 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.38e+01 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 24.55%, Upload -- 2.58e+01 / 0.00e+00, 7.96e+00 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 24.69%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 31.09%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 2.16%, LockUpload -- 2.27e+00 / 0.00e+00, 5.64e+00 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 2.56%, LockDownload -- 2.69e+00 / 0.00e+00, 4.03e+00 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 14.94%, LockCompute -- 1.57e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 13:04:31.582 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260508 13:04:31.677 | INFO | Starting a round of Baz IO | 20260508 13:04:35.078 | INFO | Populating RTMetrics report | 20260508 13:04:35.078 | INFO | RTMetrics report populated | 20260508 13:04:35.078 | INFO | Opening tmp RTMetrics json file | 20260508 13:04:35.090 | INFO | Serializing RTMetrics to json | 20260508 13:04:35.091 | INFO | Serialization done, swapping tmp file now... | 20260508 13:04:35.092 | INFO | RTMetrics done | 20260508 13:04:35.149 | INFO | Finished a round of Baz IO: Wrote 1.394 GiB in 3.471 seconds | 20260508 13:04:35.149 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:04:35.149 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149282% / 0% / 7.94729e-05% | 20260508 13:04:35.149 | INFO | Baz IO overall: | 20260508 13:04:35.149 | INFO | BytesWritten : 3555269715022 | 20260508 13:04:35.149 | INFO | NumEvents : 1705179314762 | 20260508 13:04:36.715 | INFO | Analyzing chunk frames = [8323072,8323584) | 20260508 13:04:36.715 | INFO | MemInfo: { | 20260508 13:04:36.715 | INFO | "gpu" : " 17 GB", | 20260508 13:04:36.715 | INFO | "huge" : "", | 20260508 13:04:36.715 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:36.715 | INFO | "malloc" : " 14 GB", | 20260508 13:04:36.715 | INFO | "pinned" : " 29 GB" | 20260508 13:04:36.715 | INFO | } | 20260508 13:04:36.715 | INFO | Perf report for the last 5.13446 seconds: | 20260508 13:04:36.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:36.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:36.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:36.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:36.715 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 13:04:36.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:36.715 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:04:36.715 | INFO | PRE_HQ : 65.2%, 1.6, 53.5ms, 0 | 20260508 13:04:36.715 | INFO | BAZWRITER : 67.0%, 2.5, 86.0ms, 0 | 20260508 13:04:36.715 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:04:36.716 | INFO | Uploaded 36GiB to GPU in 2.64165s (13.6278GiB/s) | 20260508 13:04:36.716 | INFO | Downloaded 38GiB to GPU in 2.70736s (14.0358GiB/s) | 20260508 13:04:41.857 | INFO | Analyzing chunk frames = [8323584,8324096) | 20260508 13:04:41.857 | INFO | MemInfo: { | 20260508 13:04:41.857 | INFO | "gpu" : " 17 GB", | 20260508 13:04:41.857 | INFO | "huge" : "", | 20260508 13:04:41.857 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:41.857 | INFO | "malloc" : " 14 GB", | 20260508 13:04:41.857 | INFO | "pinned" : " 29 GB" | 20260508 13:04:41.858 | INFO | } | 20260508 13:04:41.858 | INFO | Perf report for the last 5.14269 seconds: | 20260508 13:04:41.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:41.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:04:41.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:04:41.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:41.858 | INFO | Uploaded 36GiB to GPU in 2.51135s (14.3349GiB/s) | 20260508 13:04:41.858 | INFO | Downloaded 34GiB to GPU in 2.47254s (13.7511GiB/s) | 20260508 13:04:46.995 | INFO | Analyzing chunk frames = [8324096,8324608) | 20260508 13:04:46.995 | INFO | MemInfo: { | 20260508 13:04:46.995 | INFO | "gpu" : " 17 GB", | 20260508 13:04:46.995 | INFO | "huge" : "", | 20260508 13:04:46.995 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:46.995 | INFO | "malloc" : " 14 GB", | 20260508 13:04:46.995 | INFO | "pinned" : " 29 GB" | 20260508 13:04:46.995 | INFO | } | 20260508 13:04:46.995 | INFO | Perf report for the last 5.13777 seconds: | 20260508 13:04:46.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:46.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.995 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 13:04:46.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.995 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:04:46.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:46.996 | INFO | Uploaded 36GiB to GPU in 2.50359s (14.3794GiB/s) | 20260508 13:04:46.996 | INFO | Downloaded 34GiB to GPU in 2.46264s (13.8063GiB/s) | 20260508 13:04:52.133 | INFO | Analyzing chunk frames = [8324608,8325120) | 20260508 13:04:52.133 | INFO | MemInfo: { | 20260508 13:04:52.133 | INFO | "gpu" : " 17 GB", | 20260508 13:04:52.133 | INFO | "huge" : "", | 20260508 13:04:52.133 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:52.133 | INFO | "malloc" : " 14 GB", | 20260508 13:04:52.133 | INFO | "pinned" : " 29 GB" | 20260508 13:04:52.133 | INFO | } | 20260508 13:04:52.133 | INFO | Perf report for the last 5.13788 seconds: | 20260508 13:04:52.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:52.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | ANALYSIS : 66.2%, 3.0, 102.6ms, 0 | 20260508 13:04:52.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:04:52.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:52.133 | INFO | Uploaded 36GiB to GPU in 2.54347s (14.1539GiB/s) | 20260508 13:04:52.133 | INFO | Downloaded 34GiB to GPU in 2.45878s (13.828GiB/s) | 20260508 13:04:57.275 | INFO | Analyzing chunk frames = [8325120,8325632) | 20260508 13:04:57.275 | INFO | MemInfo: { | 20260508 13:04:57.275 | INFO | "gpu" : " 17 GB", | 20260508 13:04:57.275 | INFO | "huge" : "", | 20260508 13:04:57.275 | INFO | "huge_shared" : " 29 GB", | 20260508 13:04:57.275 | INFO | "malloc" : " 14 GB", | 20260508 13:04:57.275 | INFO | "pinned" : " 29 GB" | 20260508 13:04:57.275 | INFO | } | 20260508 13:04:57.275 | INFO | Perf report for the last 5.14179 seconds: | 20260508 13:04:57.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:04:57.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.275 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 13:04:57.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.275 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:04:57.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:04:57.276 | INFO | Uploaded 36GiB to GPU in 2.34784s (15.3333GiB/s) | 20260508 13:04:57.276 | INFO | Downloaded 34GiB to GPU in 2.50357s (13.5806GiB/s) | 20260508 13:05:02.415 | INFO | Analyzing chunk frames = [8325632,8326144) | 20260508 13:05:02.415 | INFO | MemInfo: { | 20260508 13:05:02.415 | INFO | "gpu" : " 17 GB", | 20260508 13:05:02.415 | INFO | "huge" : "", | 20260508 13:05:02.415 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:02.415 | INFO | "malloc" : " 14 GB", | 20260508 13:05:02.415 | INFO | "pinned" : " 29 GB" | 20260508 13:05:02.415 | INFO | } | 20260508 13:05:02.415 | INFO | Perf report for the last 5.14007 seconds: | 20260508 13:05:02.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:02.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.415 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 13:05:02.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.415 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:05:02.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:02.416 | INFO | Uploaded 36GiB to GPU in 2.44479s (14.7252GiB/s) | 20260508 13:05:02.416 | INFO | Downloaded 34GiB to GPU in 2.44347s (13.9147GiB/s) | 20260508 13:05:07.547 | INFO | Analyzing chunk frames = [8326144,8326656) | 20260508 13:05:07.547 | INFO | MemInfo: { | 20260508 13:05:07.547 | INFO | "gpu" : " 17 GB", | 20260508 13:05:07.547 | INFO | "huge" : "", | 20260508 13:05:07.547 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:07.547 | INFO | "malloc" : " 14 GB", | 20260508 13:05:07.547 | INFO | "pinned" : " 29 GB" | 20260508 13:05:07.547 | INFO | } | 20260508 13:05:07.547 | INFO | Perf report for the last 5.13222 seconds: | 20260508 13:05:07.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:07.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.547 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 13:05:07.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.547 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:05:07.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:07.548 | INFO | Uploaded 36GiB to GPU in 2.33932s (15.3891GiB/s) | 20260508 13:05:07.548 | INFO | Downloaded 34GiB to GPU in 2.40671s (14.1272GiB/s) | 20260508 13:05:12.688 | INFO | Analyzing chunk frames = [8326656,8327168) | 20260508 13:05:12.688 | INFO | MemInfo: { | 20260508 13:05:12.688 | INFO | "gpu" : " 17 GB", | 20260508 13:05:12.688 | INFO | "huge" : "", | 20260508 13:05:12.688 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:12.688 | INFO | "malloc" : " 14 GB", | 20260508 13:05:12.688 | INFO | "pinned" : " 29 GB" | 20260508 13:05:12.688 | INFO | } | 20260508 13:05:12.688 | INFO | Perf report for the last 5.14062 seconds: | 20260508 13:05:12.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:12.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 13:05:12.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:05:12.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:12.688 | INFO | Uploaded 36GiB to GPU in 2.48692s (14.4757GiB/s) | 20260508 13:05:12.688 | INFO | Downloaded 34GiB to GPU in 2.45749s (13.8353GiB/s) | 20260508 13:05:12.785 | INFO | Starting a round of Baz IO | 20260508 13:05:16.198 | INFO | Populating RTMetrics report | 20260508 13:05:16.198 | INFO | RTMetrics report populated | 20260508 13:05:16.198 | INFO | Opening tmp RTMetrics json file | 20260508 13:05:16.206 | INFO | Serializing RTMetrics to json | 20260508 13:05:16.208 | INFO | Serialization done, swapping tmp file now... | 20260508 13:05:16.208 | INFO | RTMetrics done | 20260508 13:05:16.268 | INFO | Finished a round of Baz IO: Wrote 1.395 GiB in 3.482 seconds | 20260508 13:05:16.268 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:05:16.268 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14952% / 3.97364e-06% / 6.75519e-05% | 20260508 13:05:16.268 | INFO | Baz IO overall: | 20260508 13:05:16.268 | INFO | BytesWritten : 3556767622751 | 20260508 13:05:16.268 | INFO | NumEvents : 1705881154467 | 20260508 13:05:17.832 | INFO | Analyzing chunk frames = [8327168,8327680) | 20260508 13:05:17.832 | INFO | MemInfo: { | 20260508 13:05:17.832 | INFO | "gpu" : " 17 GB", | 20260508 13:05:17.832 | INFO | "huge" : "", | 20260508 13:05:17.832 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:17.832 | INFO | "malloc" : " 14 GB", | 20260508 13:05:17.832 | INFO | "pinned" : " 29 GB" | 20260508 13:05:17.832 | INFO | } | 20260508 13:05:17.832 | INFO | Perf report for the last 5.14415 seconds: | 20260508 13:05:17.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:17.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:17.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:17.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:17.832 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:05:17.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:17.832 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 13:05:17.832 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 13:05:17.832 | INFO | BAZWRITER : 66.9%, 2.9, 98.1ms, 0 | 20260508 13:05:17.832 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:05:17.832 | INFO | Uploaded 36GiB to GPU in 2.64663s (13.6022GiB/s) | 20260508 13:05:17.832 | INFO | Downloaded 38GiB to GPU in 2.71325s (14.0053GiB/s) | 20260508 13:05:22.977 | INFO | Analyzing chunk frames = [8327680,8328192) | 20260508 13:05:22.977 | INFO | MemInfo: { | 20260508 13:05:22.977 | INFO | "gpu" : " 17 GB", | 20260508 13:05:22.977 | INFO | "huge" : "", | 20260508 13:05:22.977 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:22.977 | INFO | "malloc" : " 14 GB", | 20260508 13:05:22.977 | INFO | "pinned" : " 29 GB" | 20260508 13:05:22.977 | INFO | } | 20260508 13:05:22.977 | INFO | Perf report for the last 5.14511 seconds: | 20260508 13:05:22.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:22.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.977 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:05:22.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.978 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:05:22.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:22.978 | INFO | Uploaded 36GiB to GPU in 2.50872s (14.3499GiB/s) | 20260508 13:05:22.978 | INFO | Downloaded 34GiB to GPU in 2.47323s (13.7472GiB/s) | 20260508 13:05:28.111 | INFO | Analyzing chunk frames = [8328192,8328704) | 20260508 13:05:28.111 | INFO | MemInfo: { | 20260508 13:05:28.111 | INFO | "gpu" : " 17 GB", | 20260508 13:05:28.111 | INFO | "huge" : "", | 20260508 13:05:28.111 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:28.111 | INFO | "malloc" : " 14 GB", | 20260508 13:05:28.111 | INFO | "pinned" : " 29 GB" | 20260508 13:05:28.111 | INFO | } | 20260508 13:05:28.111 | INFO | Perf report for the last 5.13359 seconds: | 20260508 13:05:28.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:28.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:05:28.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:05:28.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:28.111 | INFO | Uploaded 36GiB to GPU in 2.51274s (14.327GiB/s) | 20260508 13:05:28.112 | INFO | Downloaded 34GiB to GPU in 2.45067s (13.8738GiB/s) | 20260508 13:05:33.245 | INFO | Analyzing chunk frames = [8328704,8329216) | 20260508 13:05:33.245 | INFO | MemInfo: { | 20260508 13:05:33.245 | INFO | "gpu" : " 17 GB", | 20260508 13:05:33.245 | INFO | "huge" : "", | 20260508 13:05:33.245 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:33.245 | INFO | "malloc" : " 14 GB", | 20260508 13:05:33.245 | INFO | "pinned" : " 29 GB" | 20260508 13:05:33.245 | INFO | } | 20260508 13:05:33.245 | INFO | Perf report for the last 5.13431 seconds: | 20260508 13:05:33.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:33.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.245 | INFO | ANALYSIS : 66.3%, 3.0, 102.6ms, 0 | 20260508 13:05:33.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.245 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:05:33.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:33.246 | INFO | Uploaded 36GiB to GPU in 2.54309s (14.156GiB/s) | 20260508 13:05:33.246 | INFO | Downloaded 34GiB to GPU in 2.46522s (13.7919GiB/s) | 20260508 13:05:38.388 | INFO | Analyzing chunk frames = [8329216,8329728) | 20260508 13:05:38.388 | INFO | MemInfo: { | 20260508 13:05:38.388 | INFO | "gpu" : " 17 GB", | 20260508 13:05:38.388 | INFO | "huge" : "", | 20260508 13:05:38.388 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:38.388 | INFO | "malloc" : " 14 GB", | 20260508 13:05:38.388 | INFO | "pinned" : " 29 GB" | 20260508 13:05:38.388 | INFO | } | 20260508 13:05:38.388 | INFO | Perf report for the last 5.1428 seconds: | 20260508 13:05:38.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:38.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.388 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:05:38.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.388 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:05:38.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:38.389 | INFO | Uploaded 36GiB to GPU in 2.39872s (15.008GiB/s) | 20260508 13:05:38.389 | INFO | Downloaded 34GiB to GPU in 2.49455s (13.6297GiB/s) | 20260508 13:05:43.525 | INFO | Analyzing chunk frames = [8329728,8330240) | 20260508 13:05:43.525 | INFO | MemInfo: { | 20260508 13:05:43.525 | INFO | "gpu" : " 17 GB", | 20260508 13:05:43.525 | INFO | "huge" : "", | 20260508 13:05:43.525 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:43.526 | INFO | "malloc" : " 14 GB", | 20260508 13:05:43.526 | INFO | "pinned" : " 29 GB" | 20260508 13:05:43.526 | INFO | } | 20260508 13:05:43.526 | INFO | Perf report for the last 5.1377 seconds: | 20260508 13:05:43.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:43.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:05:43.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:05:43.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:43.526 | INFO | Uploaded 36GiB to GPU in 2.42709s (14.8326GiB/s) | 20260508 13:05:43.526 | INFO | Downloaded 34GiB to GPU in 2.43141s (13.9837GiB/s) | 20260508 13:05:48.669 | INFO | Analyzing chunk frames = [8330240,8330752) | 20260508 13:05:48.669 | INFO | MemInfo: { | 20260508 13:05:48.669 | INFO | "gpu" : " 17 GB", | 20260508 13:05:48.670 | INFO | "huge" : "", | 20260508 13:05:48.670 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:48.670 | INFO | "malloc" : " 14 GB", | 20260508 13:05:48.670 | INFO | "pinned" : " 29 GB" | 20260508 13:05:48.670 | INFO | } | 20260508 13:05:48.670 | INFO | Perf report for the last 5.14395 seconds: | 20260508 13:05:48.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:48.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:05:48.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:05:48.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:48.670 | INFO | Uploaded 36GiB to GPU in 2.47421s (14.5501GiB/s) | 20260508 13:05:48.670 | INFO | Downloaded 34GiB to GPU in 2.45291s (13.8611GiB/s) | 20260508 13:05:53.804 | INFO | Analyzing chunk frames = [8330752,8331264) | 20260508 13:05:53.804 | INFO | MemInfo: { | 20260508 13:05:53.804 | INFO | "gpu" : " 17 GB", | 20260508 13:05:53.804 | INFO | "huge" : "", | 20260508 13:05:53.804 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:53.804 | INFO | "malloc" : " 14 GB", | 20260508 13:05:53.804 | INFO | "pinned" : " 29 GB" | 20260508 13:05:53.804 | INFO | } | 20260508 13:05:53.804 | INFO | Perf report for the last 5.13456 seconds: | 20260508 13:05:53.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:53.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.804 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260508 13:05:53.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.804 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:05:53.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:53.805 | INFO | Uploaded 36GiB to GPU in 2.37901s (15.1323GiB/s) | 20260508 13:05:53.805 | INFO | Downloaded 34GiB to GPU in 2.41502s (14.0785GiB/s) | 20260508 13:05:53.805 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:05:53.805 | INFO | Total mem usage: 16.7 GiB | 20260508 13:05:53.805 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:05:53.805 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -12.1%) | 20260508 13:05:53.805 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:05:53.805 | INFO | High Water Breakdown: | 20260508 13:05:53.805 | INFO | 60.2% : 8.96 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:05:53.805 | INFO | 39.6% : 5.9 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:05:53.805 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:05:53.805 | INFO | ------------------------------------------ | 20260508 13:05:53.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:05:53.805 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:05:53.805 | INFO | Total mem usage: 26.5 GiB | 20260508 13:05:53.805 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:05:53.805 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:05:53.805 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:05:53.805 | INFO | High Water Breakdown: | 20260508 13:05:53.805 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:05:53.805 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:05:53.805 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:05:53.805 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:05:53.805 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:05:53.805 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:05:53.805 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:05:53.805 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:05:53.805 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:05:53.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:05:53.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:05:53.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:05:53.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:05:53.805 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:05:53.805 | INFO | ------------------------------------------ | 20260508 13:05:53.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:05:53.805 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:05:53.805 | INFO | Total mem usage: 27.8 GiB | 20260508 13:05:53.805 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:05:53.805 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:05:53.805 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:05:53.805 | INFO | High Water Breakdown: | 20260508 13:05:53.805 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:05:53.805 | INFO | ------------------------------------------ | 20260508 13:05:53.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:05:53.805 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:05:53.805 | INFO | Total mem usage: 15.7 GiB | 20260508 13:05:53.806 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:05:53.806 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:05:53.806 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:05:53.806 | INFO | High Water Breakdown: | 20260508 13:05:53.806 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:05:53.806 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:05:53.806 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:05:53.806 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:05:53.806 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:05:53.806 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:05:53.806 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:05:53.806 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:05:53.806 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:05:53.806 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:05:53.806 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:05:53.806 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:05:53.806 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:05:53.806 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:05:53.806 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:05:53.806 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:05:53.806 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:05:53.806 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:05:53.806 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:05:53.806 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:05:53.806 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:05:53.806 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:05:53.806 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:05:53.806 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:05:53.806 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:05:53.806 | INFO | ------------------------------------------ | 20260508 13:05:53.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:05:53.806 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:05:53.806 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:05:53.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:05:53.806 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 5.52e-01 | 20260508 13:05:53.806 | INFO | PerfReport: 0.29%, AggregateStats -- 9.58e-02 / 1.04e-01, 1.12e-02 / 3.69e-03 | 20260508 13:05:53.806 | INFO | PerfReport: 12.17%, Baseline -- 3.96e+00 / 4.42e+00, 5.89e-01 / 2.09e-01 | 20260508 13:05:53.806 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 9.61e-02 | 20260508 13:05:53.806 | INFO | PerfReport: 16.01%, DME -- 4.17e+01 / 4.51e+01, 3.53e+00 / 1.03e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.79e+00 / 1.20e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.42e+00, 4.00e-01 / 2.87e-01 | 20260508 13:05:53.806 | INFO | PerfReport: 2.94%, Metrics -- 9.59e-01 / 1.08e+00, 2.39e-01 / 9.53e-02 | 20260508 13:05:53.806 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.18e-01, 4.67e-02 / 2.83e-02 | 20260508 13:05:53.806 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:05:53.806 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:05:53.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:05:53.806 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 24.64%, Upload -- 2.59e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 24.71%, Download -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 31.04%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 2.31%, LockUpload -- 2.42e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 2.48%, LockDownload -- 2.60e+00 / 0.00e+00, 3.84e+00 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 14.81%, LockCompute -- 1.55e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:05:53.806 | INFO | PerfReport: 0.01%, Misc -- 1.20e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260508 13:05:53.902 | INFO | Starting a round of Baz IO | 20260508 13:05:57.298 | INFO | Populating RTMetrics report | 20260508 13:05:57.298 | INFO | RTMetrics report populated | 20260508 13:05:57.298 | INFO | Opening tmp RTMetrics json file | 20260508 13:05:57.311 | INFO | Serializing RTMetrics to json | 20260508 13:05:57.312 | INFO | Serialization done, swapping tmp file now... | 20260508 13:05:57.313 | INFO | RTMetrics done | 20260508 13:05:57.370 | INFO | Finished a round of Baz IO: Wrote 1.394 GiB in 3.468 seconds | 20260508 13:05:57.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:05:57.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148674% / 7.94729e-06% / 8.74201e-05% | 20260508 13:05:57.370 | INFO | Baz IO overall: | 20260508 13:05:57.370 | INFO | BytesWritten : 3558264295896 | 20260508 13:05:57.370 | INFO | NumEvents : 1706581972055 | 20260508 13:05:58.943 | INFO | Analyzing chunk frames = [8331264,8331776) | 20260508 13:05:58.943 | INFO | MemInfo: { | 20260508 13:05:58.943 | INFO | "gpu" : " 17 GB", | 20260508 13:05:58.943 | INFO | "huge" : "", | 20260508 13:05:58.943 | INFO | "huge_shared" : " 29 GB", | 20260508 13:05:58.943 | INFO | "malloc" : " 14 GB", | 20260508 13:05:58.943 | INFO | "pinned" : " 29 GB" | 20260508 13:05:58.943 | INFO | } | 20260508 13:05:58.944 | INFO | Perf report for the last 5.13924 seconds: | 20260508 13:05:58.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:05:58.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:58.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:58.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:58.944 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 13:05:58.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:05:58.944 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260508 13:05:58.944 | INFO | PRE_HQ : 65.6%, 1.7, 57.5ms, 0 | 20260508 13:05:58.944 | INFO | BAZWRITER : 67.1%, 2.6, 88.5ms, 0 | 20260508 13:05:58.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:05:58.944 | INFO | Uploaded 36GiB to GPU in 2.63516s (13.6614GiB/s) | 20260508 13:05:58.944 | INFO | Downloaded 38GiB to GPU in 2.70273s (14.0599GiB/s) | 20260508 13:06:04.091 | INFO | Analyzing chunk frames = [8331776,8332288) | 20260508 13:06:04.091 | INFO | MemInfo: { | 20260508 13:06:04.091 | INFO | "gpu" : " 17 GB", | 20260508 13:06:04.091 | INFO | "huge" : "", | 20260508 13:06:04.091 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:04.091 | INFO | "malloc" : " 14 GB", | 20260508 13:06:04.091 | INFO | "pinned" : " 29 GB" | 20260508 13:06:04.091 | INFO | } | 20260508 13:06:04.091 | INFO | Perf report for the last 5.14774 seconds: | 20260508 13:06:04.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:04.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.091 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:06:04.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.091 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:06:04.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:04.092 | INFO | Uploaded 36GiB to GPU in 2.47971s (14.5178GiB/s) | 20260508 13:06:04.092 | INFO | Downloaded 34GiB to GPU in 2.46038s (13.819GiB/s) | 20260508 13:06:09.223 | INFO | Analyzing chunk frames = [8332288,8332800) | 20260508 13:06:09.223 | INFO | MemInfo: { | 20260508 13:06:09.223 | INFO | "gpu" : " 17 GB", | 20260508 13:06:09.223 | INFO | "huge" : "", | 20260508 13:06:09.223 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:09.223 | INFO | "malloc" : " 14 GB", | 20260508 13:06:09.223 | INFO | "pinned" : " 29 GB" | 20260508 13:06:09.223 | INFO | } | 20260508 13:06:09.223 | INFO | Perf report for the last 5.13179 seconds: | 20260508 13:06:09.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:09.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:06:09.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:06:09.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:09.223 | INFO | Uploaded 36GiB to GPU in 2.49567s (14.425GiB/s) | 20260508 13:06:09.223 | INFO | Downloaded 34GiB to GPU in 2.44578s (13.9015GiB/s) | 20260508 13:06:14.362 | INFO | Analyzing chunk frames = [8332800,8333312) | 20260508 13:06:14.362 | INFO | MemInfo: { | 20260508 13:06:14.362 | INFO | "gpu" : " 17 GB", | 20260508 13:06:14.362 | INFO | "huge" : "", | 20260508 13:06:14.362 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:14.362 | INFO | "malloc" : " 14 GB", | 20260508 13:06:14.363 | INFO | "pinned" : " 29 GB" | 20260508 13:06:14.363 | INFO | } | 20260508 13:06:14.363 | INFO | Perf report for the last 5.13952 seconds: | 20260508 13:06:14.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:14.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.363 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 13:06:14.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.363 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:06:14.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:14.364 | INFO | Uploaded 36GiB to GPU in 2.54806s (14.1284GiB/s) | 20260508 13:06:14.364 | INFO | Downloaded 34GiB to GPU in 2.46453s (13.7957GiB/s) | 20260508 13:06:19.502 | INFO | Analyzing chunk frames = [8333312,8333824) | 20260508 13:06:19.502 | INFO | MemInfo: { | 20260508 13:06:19.502 | INFO | "gpu" : " 17 GB", | 20260508 13:06:19.502 | INFO | "huge" : "", | 20260508 13:06:19.502 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:19.502 | INFO | "malloc" : " 14 GB", | 20260508 13:06:19.502 | INFO | "pinned" : " 29 GB" | 20260508 13:06:19.502 | INFO | } | 20260508 13:06:19.503 | INFO | Perf report for the last 5.13993 seconds: | 20260508 13:06:19.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:19.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 13:06:19.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:06:19.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:19.503 | INFO | Uploaded 36GiB to GPU in 2.48967s (14.4598GiB/s) | 20260508 13:06:19.503 | INFO | Downloaded 34GiB to GPU in 2.51649s (13.5109GiB/s) | 20260508 13:06:24.644 | INFO | Analyzing chunk frames = [8333824,8334336) | 20260508 13:06:24.644 | INFO | MemInfo: { | 20260508 13:06:24.644 | INFO | "gpu" : " 17 GB", | 20260508 13:06:24.644 | INFO | "huge" : "", | 20260508 13:06:24.644 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:24.644 | INFO | "malloc" : " 14 GB", | 20260508 13:06:24.644 | INFO | "pinned" : " 29 GB" | 20260508 13:06:24.644 | INFO | } | 20260508 13:06:24.644 | INFO | Perf report for the last 5.14168 seconds: | 20260508 13:06:24.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:24.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.644 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:06:24.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.644 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:06:24.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:24.645 | INFO | Uploaded 36GiB to GPU in 2.41759s (14.8909GiB/s) | 20260508 13:06:24.645 | INFO | Downloaded 34GiB to GPU in 2.43918s (13.9391GiB/s) | 20260508 13:06:29.782 | INFO | Analyzing chunk frames = [8334336,8334848) | 20260508 13:06:29.782 | INFO | MemInfo: { | 20260508 13:06:29.782 | INFO | "gpu" : " 17 GB", | 20260508 13:06:29.782 | INFO | "huge" : "", | 20260508 13:06:29.782 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:29.782 | INFO | "malloc" : " 14 GB", | 20260508 13:06:29.782 | INFO | "pinned" : " 29 GB" | 20260508 13:06:29.782 | INFO | } | 20260508 13:06:29.782 | INFO | Perf report for the last 5.13806 seconds: | 20260508 13:06:29.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:29.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.782 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:06:29.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.782 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:06:29.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:29.783 | INFO | Uploaded 36GiB to GPU in 2.47093s (14.5694GiB/s) | 20260508 13:06:29.783 | INFO | Downloaded 34GiB to GPU in 2.456s (13.8436GiB/s) | 20260508 13:06:34.920 | INFO | Analyzing chunk frames = [8334848,8335360) | 20260508 13:06:34.921 | INFO | MemInfo: { | 20260508 13:06:34.921 | INFO | "gpu" : " 17 GB", | 20260508 13:06:34.921 | INFO | "huge" : "", | 20260508 13:06:34.921 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:34.921 | INFO | "malloc" : " 14 GB", | 20260508 13:06:34.921 | INFO | "pinned" : " 29 GB" | 20260508 13:06:34.921 | INFO | } | 20260508 13:06:34.921 | INFO | Perf report for the last 5.13861 seconds: | 20260508 13:06:34.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:34.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:06:34.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:06:34.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:34.921 | INFO | Uploaded 36GiB to GPU in 2.40644s (14.9598GiB/s) | 20260508 13:06:34.921 | INFO | Downloaded 34GiB to GPU in 2.45307s (13.8602GiB/s) | 20260508 13:06:35.017 | INFO | Starting a round of Baz IO | 20260508 13:06:38.433 | INFO | Populating RTMetrics report | 20260508 13:06:38.433 | INFO | RTMetrics report populated | 20260508 13:06:38.433 | INFO | Opening tmp RTMetrics json file | 20260508 13:06:38.442 | INFO | Serializing RTMetrics to json | 20260508 13:06:38.443 | INFO | Serialization done, swapping tmp file now... | 20260508 13:06:38.444 | INFO | RTMetrics done | 20260508 13:06:38.499 | INFO | Finished a round of Baz IO: Wrote 1.394 GiB in 3.482 seconds | 20260508 13:06:38.499 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:06:38.499 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148884% / 3.97364e-06% / 7.94729e-05% | 20260508 13:06:38.500 | INFO | Baz IO overall: | 20260508 13:06:38.500 | INFO | BytesWritten : 3559761507453 | 20260508 13:06:38.500 | INFO | NumEvents : 1707283338952 | 20260508 13:06:40.058 | INFO | Analyzing chunk frames = [8335360,8335872) | 20260508 13:06:40.058 | INFO | MemInfo: { | 20260508 13:06:40.059 | INFO | "gpu" : " 17 GB", | 20260508 13:06:40.059 | INFO | "huge" : "", | 20260508 13:06:40.059 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:40.059 | INFO | "malloc" : " 14 GB", | 20260508 13:06:40.059 | INFO | "pinned" : " 29 GB" | 20260508 13:06:40.059 | INFO | } | 20260508 13:06:40.059 | INFO | Perf report for the last 5.13777 seconds: | 20260508 13:06:40.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:40.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:40.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:40.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:40.059 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 13:06:40.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:40.059 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:06:40.059 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 13:06:40.059 | INFO | BAZWRITER : 67.0%, 2.5, 84.8ms, 0 | 20260508 13:06:40.059 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:06:40.059 | INFO | Uploaded 36GiB to GPU in 2.61817s (13.75GiB/s) | 20260508 13:06:40.059 | INFO | Downloaded 38GiB to GPU in 2.70994s (14.0224GiB/s) | 20260508 13:06:45.193 | INFO | Analyzing chunk frames = [8335872,8336384) | 20260508 13:06:45.193 | INFO | MemInfo: { | 20260508 13:06:45.193 | INFO | "gpu" : " 17 GB", | 20260508 13:06:45.193 | INFO | "huge" : "", | 20260508 13:06:45.193 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:45.193 | INFO | "malloc" : " 14 GB", | 20260508 13:06:45.193 | INFO | "pinned" : " 29 GB" | 20260508 13:06:45.193 | INFO | } | 20260508 13:06:45.193 | INFO | Perf report for the last 5.13433 seconds: | 20260508 13:06:45.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:45.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.193 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 13:06:45.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.193 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:06:45.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:45.194 | INFO | Uploaded 36GiB to GPU in 2.50365s (14.379GiB/s) | 20260508 13:06:45.194 | INFO | Downloaded 34GiB to GPU in 2.47692s (13.7267GiB/s) | 20260508 13:06:50.339 | INFO | Analyzing chunk frames = [8336384,8336896) | 20260508 13:06:50.339 | INFO | MemInfo: { | 20260508 13:06:50.339 | INFO | "gpu" : " 17 GB", | 20260508 13:06:50.339 | INFO | "huge" : "", | 20260508 13:06:50.339 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:50.339 | INFO | "malloc" : " 14 GB", | 20260508 13:06:50.339 | INFO | "pinned" : " 29 GB" | 20260508 13:06:50.339 | INFO | } | 20260508 13:06:50.339 | INFO | Perf report for the last 5.14626 seconds: | 20260508 13:06:50.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:50.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.339 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:06:50.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.339 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 13:06:50.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:50.340 | INFO | Uploaded 36GiB to GPU in 2.47103s (14.5688GiB/s) | 20260508 13:06:50.340 | INFO | Downloaded 34GiB to GPU in 2.44422s (13.9104GiB/s) | 20260508 13:06:55.482 | INFO | Analyzing chunk frames = [8336896,8337408) | 20260508 13:06:55.482 | INFO | MemInfo: { | 20260508 13:06:55.482 | INFO | "gpu" : " 17 GB", | 20260508 13:06:55.482 | INFO | "huge" : "", | 20260508 13:06:55.482 | INFO | "huge_shared" : " 29 GB", | 20260508 13:06:55.483 | INFO | "malloc" : " 14 GB", | 20260508 13:06:55.483 | INFO | "pinned" : " 29 GB" | 20260508 13:06:55.483 | INFO | } | 20260508 13:06:55.483 | INFO | Perf report for the last 5.14332 seconds: | 20260508 13:06:55.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:06:55.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 13:06:55.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:06:55.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:06:55.483 | INFO | Uploaded 36GiB to GPU in 2.52885s (14.2357GiB/s) | 20260508 13:06:55.483 | INFO | Downloaded 34GiB to GPU in 2.44722s (13.8933GiB/s) | 20260508 13:07:00.619 | INFO | Analyzing chunk frames = [8337408,8337920) | 20260508 13:07:00.619 | INFO | MemInfo: { | 20260508 13:07:00.619 | INFO | "gpu" : " 17 GB", | 20260508 13:07:00.619 | INFO | "huge" : "", | 20260508 13:07:00.619 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:00.619 | INFO | "malloc" : " 14 GB", | 20260508 13:07:00.619 | INFO | "pinned" : " 29 GB" | 20260508 13:07:00.619 | INFO | } | 20260508 13:07:00.619 | INFO | Perf report for the last 5.13625 seconds: | 20260508 13:07:00.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:00.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 13:07:00.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:07:00.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:00.619 | INFO | Uploaded 36GiB to GPU in 2.41978s (14.8774GiB/s) | 20260508 13:07:00.619 | INFO | Downloaded 34GiB to GPU in 2.51471s (13.5205GiB/s) | 20260508 13:07:05.754 | INFO | Analyzing chunk frames = [8337920,8338432) | 20260508 13:07:05.754 | INFO | MemInfo: { | 20260508 13:07:05.754 | INFO | "gpu" : " 17 GB", | 20260508 13:07:05.754 | INFO | "huge" : "", | 20260508 13:07:05.754 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:05.754 | INFO | "malloc" : " 14 GB", | 20260508 13:07:05.754 | INFO | "pinned" : " 29 GB" | 20260508 13:07:05.754 | INFO | } | 20260508 13:07:05.754 | INFO | Perf report for the last 5.13535 seconds: | 20260508 13:07:05.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:05.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.754 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:07:05.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.754 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 13:07:05.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:05.755 | INFO | Uploaded 36GiB to GPU in 2.42138s (14.8676GiB/s) | 20260508 13:07:05.755 | INFO | Downloaded 34GiB to GPU in 2.43537s (13.9609GiB/s) | 20260508 13:07:10.897 | INFO | Analyzing chunk frames = [8338432,8338944) | 20260508 13:07:10.897 | INFO | MemInfo: { | 20260508 13:07:10.897 | INFO | "gpu" : " 17 GB", | 20260508 13:07:10.897 | INFO | "huge" : "", | 20260508 13:07:10.897 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:10.897 | INFO | "malloc" : " 14 GB", | 20260508 13:07:10.897 | INFO | "pinned" : " 29 GB" | 20260508 13:07:10.897 | INFO | } | 20260508 13:07:10.897 | INFO | Perf report for the last 5.14273 seconds: | 20260508 13:07:10.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:10.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.897 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 13:07:10.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.897 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:07:10.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:10.898 | INFO | Uploaded 36GiB to GPU in 2.34781s (15.3334GiB/s) | 20260508 13:07:10.898 | INFO | Downloaded 34GiB to GPU in 2.39668s (14.1863GiB/s) | 20260508 13:07:16.036 | INFO | Analyzing chunk frames = [8338944,8339456) | 20260508 13:07:16.036 | INFO | MemInfo: { | 20260508 13:07:16.036 | INFO | "gpu" : " 17 GB", | 20260508 13:07:16.036 | INFO | "huge" : "", | 20260508 13:07:16.036 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:16.036 | INFO | "malloc" : " 14 GB", | 20260508 13:07:16.036 | INFO | "pinned" : " 29 GB" | 20260508 13:07:16.036 | INFO | } | 20260508 13:07:16.036 | INFO | Perf report for the last 5.13876 seconds: | 20260508 13:07:16.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:16.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | ANALYSIS : 64.7%, 3.0, 101.2ms, 0 | 20260508 13:07:16.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:07:16.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:16.036 | INFO | Uploaded 36GiB to GPU in 2.36847s (15.1997GiB/s) | 20260508 13:07:16.036 | INFO | Downloaded 34GiB to GPU in 2.42256s (14.0347GiB/s) | 20260508 13:07:16.036 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:07:16.036 | INFO | Total mem usage: 16.7 GiB | 20260508 13:07:16.036 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:07:16.036 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:07:16.036 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:07:16.036 | INFO | High Water Breakdown: | 20260508 13:07:16.036 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:07:16.036 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:07:16.036 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:07:16.036 | INFO | ------------------------------------------ | 20260508 13:07:16.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:07:16.036 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:07:16.036 | INFO | Total mem usage: 26.5 GiB | 20260508 13:07:16.036 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:07:16.036 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:07:16.036 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:07:16.036 | INFO | High Water Breakdown: | 20260508 13:07:16.036 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:07:16.036 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:07:16.036 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:07:16.036 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:07:16.036 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:07:16.036 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:07:16.036 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:07:16.036 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:07:16.036 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:07:16.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:07:16.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:07:16.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:07:16.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:07:16.036 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:07:16.036 | INFO | ------------------------------------------ | 20260508 13:07:16.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:07:16.036 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:07:16.036 | INFO | Total mem usage: 27.8 GiB | 20260508 13:07:16.036 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:07:16.036 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:07:16.036 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:07:16.036 | INFO | High Water Breakdown: | 20260508 13:07:16.036 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:07:16.036 | INFO | ------------------------------------------ | 20260508 13:07:16.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:07:16.036 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:07:16.036 | INFO | Total mem usage: 15.7 GiB | 20260508 13:07:16.036 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:07:16.036 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:07:16.036 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:07:16.036 | INFO | High Water Breakdown: | 20260508 13:07:16.036 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:07:16.036 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:07:16.036 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:07:16.036 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:07:16.036 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:07:16.036 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:07:16.036 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:07:16.037 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:07:16.037 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:07:16.037 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:07:16.037 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:07:16.037 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:07:16.037 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:07:16.037 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:07:16.037 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:07:16.037 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:07:16.037 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:07:16.037 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:07:16.037 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:07:16.037 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:07:16.037 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:07:16.037 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:07:16.037 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:07:16.037 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:07:16.037 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:07:16.037 | INFO | ------------------------------------------ | 20260508 13:07:16.037 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:07:16.038 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:07:16.038 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:07:16.038 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:07:16.038 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 6.73e-01 | 20260508 13:07:16.038 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.06e-01, 1.15e-02 / 6.17e-03 | 20260508 13:07:16.038 | INFO | PerfReport: 12.16%, Baseline -- 3.98e+00 / 4.34e+00, 5.97e-01 / 3.03e-01 | 20260508 13:07:16.038 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.40e+00, 2.08e-01 / 1.43e-01 | 20260508 13:07:16.038 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.49e+01, 3.75e+00 / 1.36e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.88e+01 / 2.07e+01, 1.92e+00 / 1.47e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.30e+00 / 2.35e+00, 4.06e-01 / 2.70e-01 | 20260508 13:07:16.038 | INFO | PerfReport: 2.92%, Metrics -- 9.54e-01 / 1.10e+00, 2.38e-01 / 8.88e-02 | 20260508 13:07:16.038 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.23e-01, 4.73e-02 / 2.39e-02 | 20260508 13:07:16.038 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:07:16.038 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:07:16.038 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:07:16.038 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 24.61%, Upload -- 2.58e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 24.71%, Download -- 2.59e+01 / 0.00e+00, 8.25e+00 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 31.21%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 2.23%, LockUpload -- 2.33e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 2.41%, LockDownload -- 2.53e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 14.82%, LockCompute -- 1.55e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 13:07:16.038 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260508 13:07:16.133 | INFO | Starting a round of Baz IO | 20260508 13:07:19.533 | INFO | Populating RTMetrics report | 20260508 13:07:19.533 | INFO | RTMetrics report populated | 20260508 13:07:19.533 | INFO | Opening tmp RTMetrics json file | 20260508 13:07:19.547 | INFO | Serializing RTMetrics to json | 20260508 13:07:19.548 | INFO | Serialization done, swapping tmp file now... | 20260508 13:07:19.549 | INFO | RTMetrics done | 20260508 13:07:19.609 | INFO | Finished a round of Baz IO: Wrote 1.394 GiB in 3.475 seconds | 20260508 13:07:19.609 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:07:19.609 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149095% / 0% / 7.15256e-05% | 20260508 13:07:19.609 | INFO | Baz IO overall: | 20260508 13:07:19.609 | INFO | BytesWritten : 3561258030237 | 20260508 13:07:19.609 | INFO | NumEvents : 1707984377647 | 20260508 13:07:21.176 | INFO | Analyzing chunk frames = [8339456,8339968) | 20260508 13:07:21.176 | INFO | MemInfo: { | 20260508 13:07:21.176 | INFO | "gpu" : " 17 GB", | 20260508 13:07:21.176 | INFO | "huge" : "", | 20260508 13:07:21.176 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:21.176 | INFO | "malloc" : " 14 GB", | 20260508 13:07:21.176 | INFO | "pinned" : " 29 GB" | 20260508 13:07:21.176 | INFO | } | 20260508 13:07:21.176 | INFO | Perf report for the last 5.14013 seconds: | 20260508 13:07:21.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:21.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:21.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:21.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:21.176 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:07:21.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:21.176 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 13:07:21.176 | INFO | PRE_HQ : 65.1%, 1.6, 53.8ms, 0 | 20260508 13:07:21.176 | INFO | BAZWRITER : 66.9%, 2.5, 85.6ms, 0 | 20260508 13:07:21.176 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:07:21.176 | INFO | Uploaded 36GiB to GPU in 2.63159s (13.68GiB/s) | 20260508 13:07:21.176 | INFO | Downloaded 38GiB to GPU in 2.70743s (14.0355GiB/s) | 20260508 13:07:26.314 | INFO | Analyzing chunk frames = [8339968,8340480) | 20260508 13:07:26.314 | INFO | MemInfo: { | 20260508 13:07:26.314 | INFO | "gpu" : " 17 GB", | 20260508 13:07:26.314 | INFO | "huge" : "", | 20260508 13:07:26.314 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:26.314 | INFO | "malloc" : " 14 GB", | 20260508 13:07:26.314 | INFO | "pinned" : " 29 GB" | 20260508 13:07:26.315 | INFO | } | 20260508 13:07:26.315 | INFO | Perf report for the last 5.13868 seconds: | 20260508 13:07:26.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:26.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:07:26.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:07:26.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:26.315 | INFO | Uploaded 36GiB to GPU in 2.47214s (14.5623GiB/s) | 20260508 13:07:26.315 | INFO | Downloaded 34GiB to GPU in 2.46961s (13.7673GiB/s) | 20260508 13:07:31.451 | INFO | Analyzing chunk frames = [8340480,8340992) | 20260508 13:07:31.451 | INFO | MemInfo: { | 20260508 13:07:31.451 | INFO | "gpu" : " 17 GB", | 20260508 13:07:31.451 | INFO | "huge" : "", | 20260508 13:07:31.451 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:31.451 | INFO | "malloc" : " 14 GB", | 20260508 13:07:31.451 | INFO | "pinned" : " 29 GB" | 20260508 13:07:31.451 | INFO | } | 20260508 13:07:31.451 | INFO | Perf report for the last 5.13639 seconds: | 20260508 13:07:31.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:31.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.451 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 13:07:31.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.451 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:07:31.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:31.452 | INFO | Uploaded 36GiB to GPU in 2.524s (14.2631GiB/s) | 20260508 13:07:31.452 | INFO | Downloaded 34GiB to GPU in 2.45592s (13.8441GiB/s) | 20260508 13:07:36.599 | INFO | Analyzing chunk frames = [8340992,8341504) | 20260508 13:07:36.599 | INFO | MemInfo: { | 20260508 13:07:36.599 | INFO | "gpu" : " 17 GB", | 20260508 13:07:36.599 | INFO | "huge" : "", | 20260508 13:07:36.599 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:36.599 | INFO | "malloc" : " 14 GB", | 20260508 13:07:36.599 | INFO | "pinned" : " 29 GB" | 20260508 13:07:36.599 | INFO | } | 20260508 13:07:36.599 | INFO | Perf report for the last 5.14837 seconds: | 20260508 13:07:36.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:36.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.600 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 13:07:36.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.600 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:07:36.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:36.600 | INFO | Uploaded 36GiB to GPU in 2.40638s (14.9602GiB/s) | 20260508 13:07:36.600 | INFO | Downloaded 34GiB to GPU in 2.46216s (13.809GiB/s) | 20260508 13:07:41.737 | INFO | Analyzing chunk frames = [8341504,8342016) | 20260508 13:07:41.737 | INFO | MemInfo: { | 20260508 13:07:41.737 | INFO | "gpu" : " 17 GB", | 20260508 13:07:41.737 | INFO | "huge" : "", | 20260508 13:07:41.737 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:41.737 | INFO | "malloc" : " 14 GB", | 20260508 13:07:41.737 | INFO | "pinned" : " 29 GB" | 20260508 13:07:41.737 | INFO | } | 20260508 13:07:41.737 | INFO | Perf report for the last 5.13743 seconds: | 20260508 13:07:41.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:41.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 13:07:41.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 13:07:41.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:41.737 | INFO | Uploaded 36GiB to GPU in 2.53857s (14.1812GiB/s) | 20260508 13:07:41.738 | INFO | Downloaded 34GiB to GPU in 2.53214s (13.4274GiB/s) | 20260508 13:07:46.869 | INFO | Analyzing chunk frames = [8342016,8342528) | 20260508 13:07:46.869 | INFO | MemInfo: { | 20260508 13:07:46.869 | INFO | "gpu" : " 17 GB", | 20260508 13:07:46.869 | INFO | "huge" : "", | 20260508 13:07:46.869 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:46.869 | INFO | "malloc" : " 14 GB", | 20260508 13:07:46.869 | INFO | "pinned" : " 29 GB" | 20260508 13:07:46.869 | INFO | } | 20260508 13:07:46.869 | INFO | Perf report for the last 5.13211 seconds: | 20260508 13:07:46.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:46.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.869 | INFO | ANALYSIS : 64.7%, 3.0, 100.5ms, 0 | 20260508 13:07:46.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.869 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:07:46.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:46.870 | INFO | Uploaded 36GiB to GPU in 2.30924s (15.5895GiB/s) | 20260508 13:07:46.870 | INFO | Downloaded 34GiB to GPU in 2.39729s (14.1827GiB/s) | 20260508 13:07:52.007 | INFO | Analyzing chunk frames = [8342528,8343040) | 20260508 13:07:52.007 | INFO | MemInfo: { | 20260508 13:07:52.007 | INFO | "gpu" : " 17 GB", | 20260508 13:07:52.007 | INFO | "huge" : "", | 20260508 13:07:52.007 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:52.007 | INFO | "malloc" : " 14 GB", | 20260508 13:07:52.007 | INFO | "pinned" : " 29 GB" | 20260508 13:07:52.007 | INFO | } | 20260508 13:07:52.007 | INFO | Perf report for the last 5.13823 seconds: | 20260508 13:07:52.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:52.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:07:52.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:07:52.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:52.007 | INFO | Uploaded 36GiB to GPU in 2.46322s (14.615GiB/s) | 20260508 13:07:52.008 | INFO | Downloaded 34GiB to GPU in 2.45092s (13.8724GiB/s) | 20260508 13:07:57.147 | INFO | Analyzing chunk frames = [8343040,8343552) | 20260508 13:07:57.148 | INFO | MemInfo: { | 20260508 13:07:57.148 | INFO | "gpu" : " 17 GB", | 20260508 13:07:57.148 | INFO | "huge" : "", | 20260508 13:07:57.148 | INFO | "huge_shared" : " 29 GB", | 20260508 13:07:57.148 | INFO | "malloc" : " 14 GB", | 20260508 13:07:57.148 | INFO | "pinned" : " 29 GB" | 20260508 13:07:57.148 | INFO | } | 20260508 13:07:57.148 | INFO | Perf report for the last 5.14055 seconds: | 20260508 13:07:57.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:07:57.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | ANALYSIS : 65.8%, 3.0, 102.9ms, 0 | 20260508 13:07:57.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:07:57.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:07:57.148 | INFO | Uploaded 36GiB to GPU in 2.49003s (14.4577GiB/s) | 20260508 13:07:57.148 | INFO | Downloaded 34GiB to GPU in 2.45373s (13.8565GiB/s) | 20260508 13:07:57.245 | INFO | Starting a round of Baz IO | 20260508 13:08:00.656 | INFO | Populating RTMetrics report | 20260508 13:08:00.656 | INFO | RTMetrics report populated | 20260508 13:08:00.657 | INFO | Opening tmp RTMetrics json file | 20260508 13:08:00.665 | INFO | Serializing RTMetrics to json | 20260508 13:08:00.666 | INFO | Serialization done, swapping tmp file now... | 20260508 13:08:00.667 | INFO | RTMetrics done | 20260508 13:08:00.727 | INFO | Finished a round of Baz IO: Wrote 1.392 GiB in 3.482 seconds | 20260508 13:08:00.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:08:00.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148161% / 3.97364e-06% / 6.35783e-05% | 20260508 13:08:00.727 | INFO | Baz IO overall: | 20260508 13:08:00.727 | INFO | BytesWritten : 3562753192602 | 20260508 13:08:00.727 | INFO | NumEvents : 1708684520522 | 20260508 13:08:02.293 | INFO | Analyzing chunk frames = [8343552,8344064) | 20260508 13:08:02.293 | INFO | MemInfo: { | 20260508 13:08:02.293 | INFO | "gpu" : " 17 GB", | 20260508 13:08:02.293 | INFO | "huge" : "", | 20260508 13:08:02.293 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:02.293 | INFO | "malloc" : " 14 GB", | 20260508 13:08:02.293 | INFO | "pinned" : " 29 GB" | 20260508 13:08:02.293 | INFO | } | 20260508 13:08:02.293 | INFO | Perf report for the last 5.14544 seconds: | 20260508 13:08:02.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:02.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:02.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:02.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:02.293 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 13:08:02.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:02.293 | INFO | RT_METRICS : 38.0%, 1.0, 20.0ms, 0 | 20260508 13:08:02.293 | INFO | PRE_HQ : 65.0%, 1.6, 53.5ms, 0 | 20260508 13:08:02.293 | INFO | BAZWRITER : 67.0%, 2.4, 84.2ms, 0 | 20260508 13:08:02.293 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:08:02.294 | INFO | Uploaded 36GiB to GPU in 2.66206s (13.5233GiB/s) | 20260508 13:08:02.294 | INFO | Downloaded 38GiB to GPU in 2.7115s (14.0144GiB/s) | 20260508 13:08:07.433 | INFO | Analyzing chunk frames = [8344064,8344576) | 20260508 13:08:07.433 | INFO | MemInfo: { | 20260508 13:08:07.433 | INFO | "gpu" : " 17 GB", | 20260508 13:08:07.433 | INFO | "huge" : "", | 20260508 13:08:07.433 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:07.433 | INFO | "malloc" : " 14 GB", | 20260508 13:08:07.434 | INFO | "pinned" : " 29 GB" | 20260508 13:08:07.434 | INFO | } | 20260508 13:08:07.434 | INFO | Perf report for the last 5.14033 seconds: | 20260508 13:08:07.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:07.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:08:07.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:08:07.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:07.434 | INFO | Uploaded 36GiB to GPU in 2.51592s (14.3089GiB/s) | 20260508 13:08:07.434 | INFO | Downloaded 34GiB to GPU in 2.44781s (13.89GiB/s) | 20260508 13:08:12.569 | INFO | Analyzing chunk frames = [8344576,8345088) | 20260508 13:08:12.569 | INFO | MemInfo: { | 20260508 13:08:12.569 | INFO | "gpu" : " 17 GB", | 20260508 13:08:12.569 | INFO | "huge" : "", | 20260508 13:08:12.569 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:12.569 | INFO | "malloc" : " 14 GB", | 20260508 13:08:12.569 | INFO | "pinned" : " 29 GB" | 20260508 13:08:12.569 | INFO | } | 20260508 13:08:12.569 | INFO | Perf report for the last 5.13551 seconds: | 20260508 13:08:12.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:12.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:08:12.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 13:08:12.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:12.569 | INFO | Uploaded 36GiB to GPU in 2.47832s (14.526GiB/s) | 20260508 13:08:12.570 | INFO | Downloaded 34GiB to GPU in 2.45523s (13.848GiB/s) | 20260508 13:08:17.700 | INFO | Analyzing chunk frames = [8345088,8345600) | 20260508 13:08:17.700 | INFO | MemInfo: { | 20260508 13:08:17.700 | INFO | "gpu" : " 17 GB", | 20260508 13:08:17.700 | INFO | "huge" : "", | 20260508 13:08:17.700 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:17.700 | INFO | "malloc" : " 14 GB", | 20260508 13:08:17.700 | INFO | "pinned" : " 29 GB" | 20260508 13:08:17.700 | INFO | } | 20260508 13:08:17.700 | INFO | Perf report for the last 5.1313 seconds: | 20260508 13:08:17.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:17.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.701 | INFO | ANALYSIS : 65.6%, 3.0, 101.6ms, 0 | 20260508 13:08:17.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.701 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:08:17.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:17.701 | INFO | Uploaded 36GiB to GPU in 2.44098s (14.7482GiB/s) | 20260508 13:08:17.701 | INFO | Downloaded 34GiB to GPU in 2.46768s (13.7781GiB/s) | 20260508 13:08:22.848 | INFO | Analyzing chunk frames = [8345600,8346112) | 20260508 13:08:22.848 | INFO | MemInfo: { | 20260508 13:08:22.848 | INFO | "gpu" : " 17 GB", | 20260508 13:08:22.848 | INFO | "huge" : "", | 20260508 13:08:22.848 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:22.848 | INFO | "malloc" : " 14 GB", | 20260508 13:08:22.848 | INFO | "pinned" : " 29 GB" | 20260508 13:08:22.848 | INFO | } | 20260508 13:08:22.848 | INFO | Perf report for the last 5.14758 seconds: | 20260508 13:08:22.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:22.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 13:08:22.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 13:08:22.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:22.848 | INFO | Uploaded 36GiB to GPU in 2.43608s (14.7778GiB/s) | 20260508 13:08:22.848 | INFO | Downloaded 34GiB to GPU in 2.51678s (13.5093GiB/s) | 20260508 13:08:27.988 | INFO | Analyzing chunk frames = [8346112,8346624) | 20260508 13:08:27.988 | INFO | MemInfo: { | 20260508 13:08:27.988 | INFO | "gpu" : " 17 GB", | 20260508 13:08:27.988 | INFO | "huge" : "", | 20260508 13:08:27.988 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:27.988 | INFO | "malloc" : " 14 GB", | 20260508 13:08:27.988 | INFO | "pinned" : " 29 GB" | 20260508 13:08:27.988 | INFO | } | 20260508 13:08:27.988 | INFO | Perf report for the last 5.13983 seconds: | 20260508 13:08:27.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:27.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:08:27.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:08:27.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:27.988 | INFO | Uploaded 36GiB to GPU in 2.4407s (14.7499GiB/s) | 20260508 13:08:27.988 | INFO | Downloaded 34GiB to GPU in 2.44624s (13.8989GiB/s) | 20260508 13:08:33.129 | INFO | Analyzing chunk frames = [8346624,8347136) | 20260508 13:08:33.129 | INFO | MemInfo: { | 20260508 13:08:33.129 | INFO | "gpu" : " 17 GB", | 20260508 13:08:33.129 | INFO | "huge" : "", | 20260508 13:08:33.129 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:33.129 | INFO | "malloc" : " 14 GB", | 20260508 13:08:33.129 | INFO | "pinned" : " 29 GB" | 20260508 13:08:33.129 | INFO | } | 20260508 13:08:33.129 | INFO | Perf report for the last 5.1414 seconds: | 20260508 13:08:33.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:33.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.129 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:08:33.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.129 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:08:33.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:33.130 | INFO | Uploaded 36GiB to GPU in 2.47157s (14.5657GiB/s) | 20260508 13:08:33.130 | INFO | Downloaded 34GiB to GPU in 2.45934s (13.8249GiB/s) | 20260508 13:08:38.266 | INFO | Analyzing chunk frames = [8347136,8347648) | 20260508 13:08:38.266 | INFO | MemInfo: { | 20260508 13:08:38.266 | INFO | "gpu" : " 17 GB", | 20260508 13:08:38.266 | INFO | "huge" : "", | 20260508 13:08:38.266 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:38.266 | INFO | "malloc" : " 14 GB", | 20260508 13:08:38.266 | INFO | "pinned" : " 29 GB" | 20260508 13:08:38.266 | INFO | } | 20260508 13:08:38.266 | INFO | Perf report for the last 5.13656 seconds: | 20260508 13:08:38.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:38.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260508 13:08:38.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:08:38.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:38.266 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:08:38.266 | INFO | Total mem usage: 16.7 GiB | 20260508 13:08:38.266 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:08:38.266 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:08:38.266 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:08:38.266 | INFO | High Water Breakdown: | 20260508 13:08:38.266 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:08:38.266 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:08:38.266 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:08:38.266 | INFO | ------------------------------------------ | 20260508 13:08:38.267 | INFO | Uploaded 36GiB to GPU in 2.39411s (15.0369GiB/s) | 20260508 13:08:38.267 | INFO | Downloaded 34GiB to GPU in 2.42774s (14.0048GiB/s) | 20260508 13:08:38.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:08:38.267 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:08:38.267 | INFO | Total mem usage: 26.5 GiB | 20260508 13:08:38.267 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:08:38.267 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:08:38.267 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:08:38.267 | INFO | High Water Breakdown: | 20260508 13:08:38.267 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:08:38.267 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:08:38.267 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:08:38.267 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:08:38.267 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:08:38.267 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:08:38.267 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:08:38.267 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:08:38.267 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:08:38.267 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:08:38.267 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:08:38.267 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:08:38.267 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:08:38.267 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:08:38.267 | INFO | ------------------------------------------ | 20260508 13:08:38.267 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:08:38.267 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:08:38.267 | INFO | Total mem usage: 27.8 GiB | 20260508 13:08:38.267 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:08:38.267 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:08:38.267 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:08:38.267 | INFO | High Water Breakdown: | 20260508 13:08:38.267 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:08:38.267 | INFO | ------------------------------------------ | 20260508 13:08:38.267 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:08:38.267 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:08:38.267 | INFO | Total mem usage: 15.7 GiB | 20260508 13:08:38.267 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:08:38.267 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:08:38.267 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:08:38.267 | INFO | High Water Breakdown: | 20260508 13:08:38.267 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:08:38.267 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:08:38.267 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:08:38.267 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:08:38.267 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:08:38.267 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:08:38.267 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:08:38.267 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:08:38.267 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:08:38.267 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:08:38.267 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:08:38.267 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:08:38.267 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:08:38.267 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:08:38.267 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:08:38.267 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:08:38.267 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:08:38.267 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:08:38.267 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:08:38.267 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:08:38.267 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:08:38.267 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:08:38.267 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:08:38.267 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:08:38.267 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:08:38.267 | INFO | ------------------------------------------ | 20260508 13:08:38.267 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:08:38.267 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:08:38.267 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:08:38.267 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:08:38.267 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.38e+01 / 6.80e-01 | 20260508 13:08:38.267 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.04e-01, 1.08e-02 / 4.30e-03 | 20260508 13:08:38.267 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.40e+00, 5.94e-01 / 1.58e-01 | 20260508 13:08:38.267 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.46e+00, 2.10e-01 / 1.00e-01 | 20260508 13:08:38.267 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.51e+01, 3.52e+00 / 1.13e+00 | 20260508 13:08:38.267 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.83e+00 / 1.15e+00 | 20260508 13:08:38.267 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.31e+00 / 2.45e+00, 4.02e-01 / 1.26e-01 | 20260508 13:08:38.267 | INFO | PerfReport: 2.94%, Metrics -- 9.61e-01 / 1.06e+00, 2.42e-01 / 8.71e-02 | 20260508 13:08:38.267 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.89e-01 / 3.20e-01, 4.63e-02 / 2.03e-02 | 20260508 13:08:38.268 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:08:38.268 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:08:38.268 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:08:38.268 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 24.62%, Upload -- 2.58e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 24.74%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 2.25%, LockUpload -- 2.36e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 2.45%, LockDownload -- 2.57e+00 / 0.00e+00, 3.81e+00 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 14.80%, LockCompute -- 1.55e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260508 13:08:38.268 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260508 13:08:38.363 | INFO | Starting a round of Baz IO | 20260508 13:08:41.764 | INFO | Populating RTMetrics report | 20260508 13:08:41.765 | INFO | RTMetrics report populated | 20260508 13:08:41.765 | INFO | Opening tmp RTMetrics json file | 20260508 13:08:41.776 | INFO | Serializing RTMetrics to json | 20260508 13:08:41.778 | INFO | Serialization done, swapping tmp file now... | 20260508 13:08:41.778 | INFO | RTMetrics done | 20260508 13:08:41.838 | INFO | Finished a round of Baz IO: Wrote 1.393 GiB in 3.474 seconds | 20260508 13:08:41.838 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:08:41.838 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148344% / 0% / 6.35783e-05% | 20260508 13:08:41.838 | INFO | Baz IO overall: | 20260508 13:08:41.838 | INFO | BytesWritten : 3564248673394 | 20260508 13:08:41.838 | INFO | NumEvents : 1709384643877 | 20260508 13:08:43.408 | INFO | Analyzing chunk frames = [8347648,8348160) | 20260508 13:08:43.408 | INFO | MemInfo: { | 20260508 13:08:43.408 | INFO | "gpu" : " 17 GB", | 20260508 13:08:43.408 | INFO | "huge" : "", | 20260508 13:08:43.408 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:43.408 | INFO | "malloc" : " 14 GB", | 20260508 13:08:43.408 | INFO | "pinned" : " 29 GB" | 20260508 13:08:43.408 | INFO | } | 20260508 13:08:43.408 | INFO | Perf report for the last 5.14233 seconds: | 20260508 13:08:43.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:43.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:43.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:43.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:43.408 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:08:43.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:43.408 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 13:08:43.408 | INFO | PRE_HQ : 65.2%, 1.6, 53.6ms, 0 | 20260508 13:08:43.408 | INFO | BAZWRITER : 66.9%, 2.5, 84.6ms, 0 | 20260508 13:08:43.408 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:08:43.409 | INFO | Uploaded 36GiB to GPU in 2.62707s (13.7035GiB/s) | 20260508 13:08:43.409 | INFO | Downloaded 38GiB to GPU in 2.7201s (13.9701GiB/s) | 20260508 13:08:48.543 | INFO | Analyzing chunk frames = [8348160,8348672) | 20260508 13:08:48.543 | INFO | MemInfo: { | 20260508 13:08:48.543 | INFO | "gpu" : " 17 GB", | 20260508 13:08:48.543 | INFO | "huge" : "", | 20260508 13:08:48.543 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:48.543 | INFO | "malloc" : " 14 GB", | 20260508 13:08:48.543 | INFO | "pinned" : " 29 GB" | 20260508 13:08:48.543 | INFO | } | 20260508 13:08:48.543 | INFO | Perf report for the last 5.13496 seconds: | 20260508 13:08:48.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:48.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.543 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 13:08:48.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.543 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260508 13:08:48.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:48.544 | INFO | Uploaded 36GiB to GPU in 2.5034s (14.3805GiB/s) | 20260508 13:08:48.544 | INFO | Downloaded 34GiB to GPU in 2.48622s (13.6754GiB/s) | 20260508 13:08:53.680 | INFO | Analyzing chunk frames = [8348672,8349184) | 20260508 13:08:53.680 | INFO | MemInfo: { | 20260508 13:08:53.680 | INFO | "gpu" : " 17 GB", | 20260508 13:08:53.680 | INFO | "huge" : "", | 20260508 13:08:53.680 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:53.680 | INFO | "malloc" : " 14 GB", | 20260508 13:08:53.680 | INFO | "pinned" : " 29 GB" | 20260508 13:08:53.680 | INFO | } | 20260508 13:08:53.680 | INFO | Perf report for the last 5.13709 seconds: | 20260508 13:08:53.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:53.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.680 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:08:53.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.680 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:08:53.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:53.681 | INFO | Uploaded 36GiB to GPU in 2.50573s (14.3671GiB/s) | 20260508 13:08:53.681 | INFO | Downloaded 34GiB to GPU in 2.44666s (13.8965GiB/s) | 20260508 13:08:58.820 | INFO | Analyzing chunk frames = [8349184,8349696) | 20260508 13:08:58.820 | INFO | MemInfo: { | 20260508 13:08:58.820 | INFO | "gpu" : " 17 GB", | 20260508 13:08:58.820 | INFO | "huge" : "", | 20260508 13:08:58.820 | INFO | "huge_shared" : " 29 GB", | 20260508 13:08:58.820 | INFO | "malloc" : " 14 GB", | 20260508 13:08:58.821 | INFO | "pinned" : " 29 GB" | 20260508 13:08:58.821 | INFO | } | 20260508 13:08:58.821 | INFO | Perf report for the last 5.14038 seconds: | 20260508 13:08:58.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:08:58.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 13:08:58.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:08:58.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:08:58.821 | INFO | Uploaded 36GiB to GPU in 2.50925s (14.3469GiB/s) | 20260508 13:08:58.821 | INFO | Downloaded 34GiB to GPU in 2.44571s (13.9019GiB/s) | 20260508 13:09:03.957 | INFO | Analyzing chunk frames = [8349696,8350208) | 20260508 13:09:03.957 | INFO | MemInfo: { | 20260508 13:09:03.957 | INFO | "gpu" : " 17 GB", | 20260508 13:09:03.957 | INFO | "huge" : "", | 20260508 13:09:03.957 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:03.957 | INFO | "malloc" : " 14 GB", | 20260508 13:09:03.958 | INFO | "pinned" : " 29 GB" | 20260508 13:09:03.958 | INFO | } | 20260508 13:09:03.958 | INFO | Perf report for the last 5.13701 seconds: | 20260508 13:09:03.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:03.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:09:03.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:09:03.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:03.958 | INFO | Uploaded 36GiB to GPU in 2.41003s (14.9376GiB/s) | 20260508 13:09:03.958 | INFO | Downloaded 34GiB to GPU in 2.50039s (13.5979GiB/s) | 20260508 13:09:09.105 | INFO | Analyzing chunk frames = [8350208,8350720) | 20260508 13:09:09.105 | INFO | MemInfo: { | 20260508 13:09:09.105 | INFO | "gpu" : " 17 GB", | 20260508 13:09:09.105 | INFO | "huge" : "", | 20260508 13:09:09.106 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:09.106 | INFO | "malloc" : " 14 GB", | 20260508 13:09:09.106 | INFO | "pinned" : " 29 GB" | 20260508 13:09:09.106 | INFO | } | 20260508 13:09:09.106 | INFO | Perf report for the last 5.14793 seconds: | 20260508 13:09:09.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:09.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 13:09:09.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:09:09.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:09.106 | INFO | Uploaded 36GiB to GPU in 2.434s (14.7905GiB/s) | 20260508 13:09:09.106 | INFO | Downloaded 34GiB to GPU in 2.42408s (14.026GiB/s) | 20260508 13:09:14.244 | INFO | Analyzing chunk frames = [8350720,8351232) | 20260508 13:09:14.244 | INFO | MemInfo: { | 20260508 13:09:14.244 | INFO | "gpu" : " 17 GB", | 20260508 13:09:14.244 | INFO | "huge" : "", | 20260508 13:09:14.244 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:14.244 | INFO | "malloc" : " 14 GB", | 20260508 13:09:14.244 | INFO | "pinned" : " 29 GB" | 20260508 13:09:14.244 | INFO | } | 20260508 13:09:14.244 | INFO | Perf report for the last 5.13867 seconds: | 20260508 13:09:14.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:14.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.244 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 13:09:14.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.244 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:09:14.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:14.245 | INFO | Uploaded 36GiB to GPU in 2.36864s (15.1986GiB/s) | 20260508 13:09:14.245 | INFO | Downloaded 34GiB to GPU in 2.40402s (14.143GiB/s) | 20260508 13:09:19.382 | INFO | Analyzing chunk frames = [8351232,8351744) | 20260508 13:09:19.382 | INFO | MemInfo: { | 20260508 13:09:19.382 | INFO | "gpu" : " 17 GB", | 20260508 13:09:19.382 | INFO | "huge" : "", | 20260508 13:09:19.382 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:19.383 | INFO | "malloc" : " 14 GB", | 20260508 13:09:19.383 | INFO | "pinned" : " 29 GB" | 20260508 13:09:19.383 | INFO | } | 20260508 13:09:19.383 | INFO | Perf report for the last 5.13838 seconds: | 20260508 13:09:19.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:19.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.383 | INFO | ANALYSIS : 64.7%, 3.0, 101.2ms, 0 | 20260508 13:09:19.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.383 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:09:19.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:19.384 | INFO | Uploaded 36GiB to GPU in 2.3733s (15.1688GiB/s) | 20260508 13:09:19.384 | INFO | Downloaded 34GiB to GPU in 2.4199s (14.0502GiB/s) | 20260508 13:09:19.493 | INFO | Starting a round of Baz IO | 20260508 13:09:22.884 | INFO | Populating RTMetrics report | 20260508 13:09:22.885 | INFO | RTMetrics report populated | 20260508 13:09:22.885 | INFO | Opening tmp RTMetrics json file | 20260508 13:09:22.898 | INFO | Serializing RTMetrics to json | 20260508 13:09:22.899 | INFO | Serialization done, swapping tmp file now... | 20260508 13:09:22.899 | INFO | RTMetrics done | 20260508 13:09:22.959 | INFO | Finished a round of Baz IO: Wrote 1.394 GiB in 3.465 seconds | 20260508 13:09:22.959 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:09:22.959 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148499% / 0% / 7.15256e-05% | 20260508 13:09:22.959 | INFO | Baz IO overall: | 20260508 13:09:22.959 | INFO | BytesWritten : 3565745407937 | 20260508 13:09:22.959 | INFO | NumEvents : 1710085732966 | 20260508 13:09:24.513 | INFO | Analyzing chunk frames = [8351744,8352256) | 20260508 13:09:24.513 | INFO | MemInfo: { | 20260508 13:09:24.513 | INFO | "gpu" : " 17 GB", | 20260508 13:09:24.513 | INFO | "huge" : "", | 20260508 13:09:24.513 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:24.513 | INFO | "malloc" : " 14 GB", | 20260508 13:09:24.513 | INFO | "pinned" : " 29 GB" | 20260508 13:09:24.513 | INFO | } | 20260508 13:09:24.513 | INFO | Perf report for the last 5.13067 seconds: | 20260508 13:09:24.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:24.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:24.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:24.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:24.513 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 13:09:24.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:24.514 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260508 13:09:24.514 | INFO | PRE_HQ : 66.2%, 1.8, 60.2ms, 0 | 20260508 13:09:24.514 | INFO | BAZWRITER : 66.7%, 2.7, 91.5ms, 0 | 20260508 13:09:24.514 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:09:24.514 | INFO | Uploaded 36GiB to GPU in 2.64219s (13.6251GiB/s) | 20260508 13:09:24.514 | INFO | Downloaded 38GiB to GPU in 2.71854s (13.9781GiB/s) | 20260508 13:09:29.661 | INFO | Analyzing chunk frames = [8352256,8352768) | 20260508 13:09:29.661 | INFO | MemInfo: { | 20260508 13:09:29.661 | INFO | "gpu" : " 17 GB", | 20260508 13:09:29.661 | INFO | "huge" : "", | 20260508 13:09:29.661 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:29.661 | INFO | "malloc" : " 14 GB", | 20260508 13:09:29.661 | INFO | "pinned" : " 29 GB" | 20260508 13:09:29.661 | INFO | } | 20260508 13:09:29.661 | INFO | Perf report for the last 5.14763 seconds: | 20260508 13:09:29.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:29.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 13:09:29.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | PRE_HQ : 22.2%, 1.0, 11.6ms, 0 | 20260508 13:09:29.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:29.661 | INFO | Uploaded 36GiB to GPU in 2.45444s (14.6673GiB/s) | 20260508 13:09:29.661 | INFO | Downloaded 34GiB to GPU in 2.46849s (13.7736GiB/s) | 20260508 13:09:34.798 | INFO | Analyzing chunk frames = [8352768,8353280) | 20260508 13:09:34.798 | INFO | MemInfo: { | 20260508 13:09:34.798 | INFO | "gpu" : " 17 GB", | 20260508 13:09:34.798 | INFO | "huge" : "", | 20260508 13:09:34.798 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:34.798 | INFO | "malloc" : " 14 GB", | 20260508 13:09:34.798 | INFO | "pinned" : " 29 GB" | 20260508 13:09:34.798 | INFO | } | 20260508 13:09:34.798 | INFO | Perf report for the last 5.13729 seconds: | 20260508 13:09:34.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:34.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.798 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:09:34.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.798 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:09:34.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:34.799 | INFO | Uploaded 36GiB to GPU in 2.52328s (14.2671GiB/s) | 20260508 13:09:34.799 | INFO | Downloaded 34GiB to GPU in 2.47427s (13.7414GiB/s) | 20260508 13:09:39.939 | INFO | Analyzing chunk frames = [8353280,8353792) | 20260508 13:09:39.939 | INFO | MemInfo: { | 20260508 13:09:39.939 | INFO | "gpu" : " 17 GB", | 20260508 13:09:39.939 | INFO | "huge" : "", | 20260508 13:09:39.939 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:39.939 | INFO | "malloc" : " 14 GB", | 20260508 13:09:39.939 | INFO | "pinned" : " 29 GB" | 20260508 13:09:39.939 | INFO | } | 20260508 13:09:39.939 | INFO | Perf report for the last 5.14092 seconds: | 20260508 13:09:39.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:39.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.939 | INFO | ANALYSIS : 66.2%, 3.0, 102.5ms, 0 | 20260508 13:09:39.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.939 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:09:39.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:39.940 | INFO | Uploaded 36GiB to GPU in 2.51053s (14.3396GiB/s) | 20260508 13:09:39.940 | INFO | Downloaded 34GiB to GPU in 2.45s (13.8776GiB/s) | 20260508 13:09:45.080 | INFO | Analyzing chunk frames = [8353792,8354304) | 20260508 13:09:45.080 | INFO | MemInfo: { | 20260508 13:09:45.080 | INFO | "gpu" : " 17 GB", | 20260508 13:09:45.080 | INFO | "huge" : "", | 20260508 13:09:45.080 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:45.080 | INFO | "malloc" : " 14 GB", | 20260508 13:09:45.080 | INFO | "pinned" : " 29 GB" | 20260508 13:09:45.080 | INFO | } | 20260508 13:09:45.080 | INFO | Perf report for the last 5.14106 seconds: | 20260508 13:09:45.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:45.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.080 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 13:09:45.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.080 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:09:45.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:45.081 | INFO | Uploaded 36GiB to GPU in 2.42856s (14.8236GiB/s) | 20260508 13:09:45.081 | INFO | Downloaded 34GiB to GPU in 2.52613s (13.4593GiB/s) | 20260508 13:09:50.212 | INFO | Analyzing chunk frames = [8354304,8354816) | 20260508 13:09:50.212 | INFO | MemInfo: { | 20260508 13:09:50.212 | INFO | "gpu" : " 17 GB", | 20260508 13:09:50.212 | INFO | "huge" : "", | 20260508 13:09:50.212 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:50.212 | INFO | "malloc" : " 14 GB", | 20260508 13:09:50.213 | INFO | "pinned" : " 29 GB" | 20260508 13:09:50.213 | INFO | } | 20260508 13:09:50.213 | INFO | Perf report for the last 5.13233 seconds: | 20260508 13:09:50.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:50.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | ANALYSIS : 65.4%, 3.0, 101.5ms, 0 | 20260508 13:09:50.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:09:50.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:50.213 | INFO | Uploaded 36GiB to GPU in 2.40156s (14.9903GiB/s) | 20260508 13:09:50.213 | INFO | Downloaded 34GiB to GPU in 2.45707s (13.8376GiB/s) | 20260508 13:09:55.354 | INFO | Analyzing chunk frames = [8354816,8355328) | 20260508 13:09:55.354 | INFO | MemInfo: { | 20260508 13:09:55.354 | INFO | "gpu" : " 17 GB", | 20260508 13:09:55.354 | INFO | "huge" : "", | 20260508 13:09:55.354 | INFO | "huge_shared" : " 29 GB", | 20260508 13:09:55.354 | INFO | "malloc" : " 14 GB", | 20260508 13:09:55.354 | INFO | "pinned" : " 29 GB" | 20260508 13:09:55.354 | INFO | } | 20260508 13:09:55.355 | INFO | Perf report for the last 5.14184 seconds: | 20260508 13:09:55.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:09:55.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 13:09:55.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 13:09:55.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:09:55.355 | INFO | Uploaded 36GiB to GPU in 2.43448s (14.7876GiB/s) | 20260508 13:09:55.355 | INFO | Downloaded 34GiB to GPU in 2.47072s (13.7612GiB/s) | 20260508 13:10:00.494 | INFO | Analyzing chunk frames = [8355328,8355840) | 20260508 13:10:00.494 | INFO | MemInfo: { | 20260508 13:10:00.494 | INFO | "gpu" : " 17 GB", | 20260508 13:10:00.494 | INFO | "huge" : "", | 20260508 13:10:00.494 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:00.494 | INFO | "malloc" : " 14 GB", | 20260508 13:10:00.494 | INFO | "pinned" : " 29 GB" | 20260508 13:10:00.494 | INFO | } | 20260508 13:10:00.494 | INFO | Perf report for the last 5.13941 seconds: | 20260508 13:10:00.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:00.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | ANALYSIS : 64.8%, 3.0, 101.3ms, 0 | 20260508 13:10:00.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:10:00.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:00.494 | INFO | Uploaded 36GiB to GPU in 2.34805s (15.3318GiB/s) | 20260508 13:10:00.494 | INFO | Downloaded 34GiB to GPU in 2.46147s (13.8129GiB/s) | 20260508 13:10:00.494 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:10:00.494 | INFO | Total mem usage: 16.7 GiB | 20260508 13:10:00.494 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:10:00.494 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 13:10:00.494 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:10:00.494 | INFO | High Water Breakdown: | 20260508 13:10:00.494 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:10:00.494 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:10:00.494 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:10:00.494 | INFO | ------------------------------------------ | 20260508 13:10:00.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:10:00.494 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:10:00.494 | INFO | Total mem usage: 26.5 GiB | 20260508 13:10:00.494 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:10:00.494 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 13:10:00.495 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 13:10:00.495 | INFO | High Water Breakdown: | 20260508 13:10:00.495 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:10:00.495 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:10:00.495 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:10:00.495 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:10:00.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:10:00.495 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:10:00.495 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:10:00.495 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:10:00.495 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:10:00.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:10:00.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:10:00.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:10:00.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:10:00.495 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:10:00.495 | INFO | ------------------------------------------ | 20260508 13:10:00.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:10:00.495 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:10:00.495 | INFO | Total mem usage: 27.8 GiB | 20260508 13:10:00.495 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:10:00.495 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:10:00.495 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:10:00.495 | INFO | High Water Breakdown: | 20260508 13:10:00.495 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:10:00.495 | INFO | ------------------------------------------ | 20260508 13:10:00.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:10:00.495 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:10:00.495 | INFO | Total mem usage: 15.7 GiB | 20260508 13:10:00.495 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:10:00.495 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:10:00.495 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:10:00.495 | INFO | High Water Breakdown: | 20260508 13:10:00.495 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:10:00.495 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:10:00.495 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:10:00.495 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:10:00.495 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:10:00.495 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:10:00.495 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:10:00.495 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:10:00.495 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:10:00.495 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:10:00.495 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:10:00.495 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:10:00.495 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:10:00.495 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:10:00.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:10:00.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:10:00.495 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:10:00.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:10:00.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:10:00.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:10:00.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:10:00.495 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:10:00.496 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:10:00.496 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:10:00.496 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:10:00.496 | INFO | ------------------------------------------ | 20260508 13:10:00.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:10:00.496 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:10:00.496 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:10:00.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:10:00.496 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.39e+01 / 5.11e-01 | 20260508 13:10:00.496 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.06e-01, 1.13e-02 / 9.02e-03 | 20260508 13:10:00.496 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.42e+00, 5.92e-01 / 2.61e-01 | 20260508 13:10:00.496 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.38e+00, 2.07e-01 / 1.06e-01 | 20260508 13:10:00.496 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.46e+01, 3.50e+00 / 1.13e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.87e+01 / 2.10e+01, 1.90e+00 / 1.42e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.30e+00 / 2.32e+00, 4.02e-01 / 2.85e-01 | 20260508 13:10:00.496 | INFO | PerfReport: 2.93%, Metrics -- 9.56e-01 / 1.06e+00, 2.42e-01 / 1.19e-01 | 20260508 13:10:00.496 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.13e-01, 4.64e-02 / 2.42e-02 | 20260508 13:10:00.496 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:10:00.496 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:10:00.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:10:00.496 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 24.52%, Upload -- 2.57e+01 / 0.00e+00, 8.04e+00 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 24.77%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 31.17%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 2.15%, LockUpload -- 2.25e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 2.51%, LockDownload -- 2.63e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 14.87%, LockCompute -- 1.56e+01 / 0.00e+00, 1.78e+01 / 0.00e+00 | 20260508 13:10:00.496 | INFO | PerfReport: 0.01%, Misc -- 1.19e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260508 13:10:00.592 | INFO | Starting a round of Baz IO | 20260508 13:10:03.994 | INFO | Populating RTMetrics report | 20260508 13:10:03.994 | INFO | RTMetrics report populated | 20260508 13:10:03.994 | INFO | Opening tmp RTMetrics json file | 20260508 13:10:04.004 | INFO | Serializing RTMetrics to json | 20260508 13:10:04.005 | INFO | Serialization done, swapping tmp file now... | 20260508 13:10:04.005 | INFO | RTMetrics done | 20260508 13:10:04.066 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.474 seconds | 20260508 13:10:04.066 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:10:04.066 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147438% / 0% / 7.15256e-05% | 20260508 13:10:04.066 | INFO | Baz IO overall: | 20260508 13:10:04.066 | INFO | BytesWritten : 3567239049225 | 20260508 13:10:04.067 | INFO | NumEvents : 1710784756685 | 20260508 13:10:05.633 | INFO | Analyzing chunk frames = [8355840,8356352) | 20260508 13:10:05.633 | INFO | MemInfo: { | 20260508 13:10:05.633 | INFO | "gpu" : " 17 GB", | 20260508 13:10:05.633 | INFO | "huge" : "", | 20260508 13:10:05.633 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:05.633 | INFO | "malloc" : " 14 GB", | 20260508 13:10:05.633 | INFO | "pinned" : " 29 GB" | 20260508 13:10:05.633 | INFO | } | 20260508 13:10:05.633 | INFO | Perf report for the last 5.13913 seconds: | 20260508 13:10:05.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:05.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:05.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:05.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:05.633 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:10:05.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:05.633 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:10:05.633 | INFO | PRE_HQ : 65.3%, 1.6, 53.7ms, 0 | 20260508 13:10:05.633 | INFO | BAZWRITER : 66.9%, 2.5, 84.9ms, 0 | 20260508 13:10:05.633 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:10:05.634 | INFO | Uploaded 36GiB to GPU in 2.60505s (13.8193GiB/s) | 20260508 13:10:05.634 | INFO | Downloaded 38GiB to GPU in 2.72331s (13.9536GiB/s) | 20260508 13:10:10.774 | INFO | Analyzing chunk frames = [8356352,8356864) | 20260508 13:10:10.774 | INFO | MemInfo: { | 20260508 13:10:10.774 | INFO | "gpu" : " 17 GB", | 20260508 13:10:10.774 | INFO | "huge" : "", | 20260508 13:10:10.775 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:10.775 | INFO | "malloc" : " 14 GB", | 20260508 13:10:10.775 | INFO | "pinned" : " 29 GB" | 20260508 13:10:10.775 | INFO | } | 20260508 13:10:10.775 | INFO | Perf report for the last 5.14158 seconds: | 20260508 13:10:10.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:10.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 13:10:10.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:10:10.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:10.775 | INFO | Uploaded 36GiB to GPU in 2.41097s (14.9318GiB/s) | 20260508 13:10:10.775 | INFO | Downloaded 34GiB to GPU in 2.47612s (13.7311GiB/s) | 20260508 13:10:15.916 | INFO | Analyzing chunk frames = [8356864,8357376) | 20260508 13:10:15.916 | INFO | MemInfo: { | 20260508 13:10:15.916 | INFO | "gpu" : " 17 GB", | 20260508 13:10:15.916 | INFO | "huge" : "", | 20260508 13:10:15.916 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:15.917 | INFO | "malloc" : " 14 GB", | 20260508 13:10:15.917 | INFO | "pinned" : " 29 GB" | 20260508 13:10:15.917 | INFO | } | 20260508 13:10:15.917 | INFO | Perf report for the last 5.14199 seconds: | 20260508 13:10:15.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:15.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 13:10:15.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:10:15.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:15.917 | INFO | Uploaded 36GiB to GPU in 2.49635s (14.421GiB/s) | 20260508 13:10:15.917 | INFO | Downloaded 34GiB to GPU in 2.48703s (13.6709GiB/s) | 20260508 13:10:21.053 | INFO | Analyzing chunk frames = [8357376,8357888) | 20260508 13:10:21.053 | INFO | MemInfo: { | 20260508 13:10:21.053 | INFO | "gpu" : " 17 GB", | 20260508 13:10:21.053 | INFO | "huge" : "", | 20260508 13:10:21.053 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:21.053 | INFO | "malloc" : " 14 GB", | 20260508 13:10:21.053 | INFO | "pinned" : " 29 GB" | 20260508 13:10:21.053 | INFO | } | 20260508 13:10:21.053 | INFO | Perf report for the last 5.13649 seconds: | 20260508 13:10:21.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:21.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.053 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 13:10:21.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.053 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:10:21.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:21.054 | INFO | Uploaded 36GiB to GPU in 2.40248s (14.9845GiB/s) | 20260508 13:10:21.054 | INFO | Downloaded 34GiB to GPU in 2.49414s (13.632GiB/s) | 20260508 13:10:26.193 | INFO | Analyzing chunk frames = [8357888,8358400) | 20260508 13:10:26.194 | INFO | MemInfo: { | 20260508 13:10:26.194 | INFO | "gpu" : " 17 GB", | 20260508 13:10:26.194 | INFO | "huge" : "", | 20260508 13:10:26.194 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:26.194 | INFO | "malloc" : " 14 GB", | 20260508 13:10:26.194 | INFO | "pinned" : " 29 GB" | 20260508 13:10:26.194 | INFO | } | 20260508 13:10:26.194 | INFO | Perf report for the last 5.14063 seconds: | 20260508 13:10:26.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:26.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 13:10:26.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 13:10:26.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:26.194 | INFO | Uploaded 36GiB to GPU in 2.33404s (15.4239GiB/s) | 20260508 13:10:26.194 | INFO | Downloaded 34GiB to GPU in 2.5112s (13.5394GiB/s) | 20260508 13:10:31.329 | INFO | Analyzing chunk frames = [8358400,8358912) | 20260508 13:10:31.329 | INFO | MemInfo: { | 20260508 13:10:31.329 | INFO | "gpu" : " 17 GB", | 20260508 13:10:31.330 | INFO | "huge" : "", | 20260508 13:10:31.330 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:31.330 | INFO | "malloc" : " 14 GB", | 20260508 13:10:31.330 | INFO | "pinned" : " 29 GB" | 20260508 13:10:31.330 | INFO | } | 20260508 13:10:31.330 | INFO | Perf report for the last 5.1359 seconds: | 20260508 13:10:31.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:31.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.330 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:10:31.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.330 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260508 13:10:31.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:31.331 | INFO | Uploaded 36GiB to GPU in 2.39808s (15.012GiB/s) | 20260508 13:10:31.331 | INFO | Downloaded 34GiB to GPU in 2.44375s (13.913GiB/s) | 20260508 13:10:36.467 | INFO | Analyzing chunk frames = [8358912,8359424) | 20260508 13:10:36.467 | INFO | MemInfo: { | 20260508 13:10:36.467 | INFO | "gpu" : " 17 GB", | 20260508 13:10:36.468 | INFO | "huge" : "", | 20260508 13:10:36.468 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:36.468 | INFO | "malloc" : " 14 GB", | 20260508 13:10:36.468 | INFO | "pinned" : " 29 GB" | 20260508 13:10:36.468 | INFO | } | 20260508 13:10:36.468 | INFO | Perf report for the last 5.13796 seconds: | 20260508 13:10:36.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:36.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.468 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 13:10:36.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.473 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260508 13:10:36.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:36.473 | INFO | Uploaded 36GiB to GPU in 2.45193s (14.6823GiB/s) | 20260508 13:10:36.473 | INFO | Downloaded 34GiB to GPU in 2.47261s (13.7507GiB/s) | 20260508 13:10:41.607 | INFO | Analyzing chunk frames = [8359424,8359936) | 20260508 13:10:41.607 | INFO | MemInfo: { | 20260508 13:10:41.607 | INFO | "gpu" : " 17 GB", | 20260508 13:10:41.607 | INFO | "huge" : "", | 20260508 13:10:41.607 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:41.607 | INFO | "malloc" : " 14 GB", | 20260508 13:10:41.607 | INFO | "pinned" : " 29 GB" | 20260508 13:10:41.607 | INFO | } | 20260508 13:10:41.607 | INFO | Perf report for the last 5.13944 seconds: | 20260508 13:10:41.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:41.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.608 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260508 13:10:41.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.608 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 13:10:41.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:41.608 | INFO | Uploaded 36GiB to GPU in 2.30729s (15.6027GiB/s) | 20260508 13:10:41.608 | INFO | Downloaded 34GiB to GPU in 2.43007s (13.9913GiB/s) | 20260508 13:10:41.723 | INFO | Starting a round of Baz IO | 20260508 13:10:45.118 | INFO | Populating RTMetrics report | 20260508 13:10:45.118 | INFO | RTMetrics report populated | 20260508 13:10:45.118 | INFO | Opening tmp RTMetrics json file | 20260508 13:10:45.128 | INFO | Serializing RTMetrics to json | 20260508 13:10:45.129 | INFO | Serialization done, swapping tmp file now... | 20260508 13:10:45.130 | INFO | RTMetrics done | 20260508 13:10:45.193 | INFO | Finished a round of Baz IO: Wrote 1.392 GiB in 3.469 seconds | 20260508 13:10:45.193 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.00% / 60.03% | 20260508 13:10:45.193 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147136% / 0% / 7.15256e-05% | 20260508 13:10:45.193 | INFO | Baz IO overall: | 20260508 13:10:45.193 | INFO | BytesWritten : 3568733995888 | 20260508 13:10:45.193 | INFO | NumEvents : 1711484910928 | 20260508 13:10:46.749 | INFO | Analyzing chunk frames = [8359936,8360448) | 20260508 13:10:46.749 | INFO | MemInfo: { | 20260508 13:10:46.749 | INFO | "gpu" : " 17 GB", | 20260508 13:10:46.749 | INFO | "huge" : "", | 20260508 13:10:46.749 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:46.749 | INFO | "malloc" : " 14 GB", | 20260508 13:10:46.749 | INFO | "pinned" : " 29 GB" | 20260508 13:10:46.749 | INFO | } | 20260508 13:10:46.749 | INFO | Perf report for the last 5.14212 seconds: | 20260508 13:10:46.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:46.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:46.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:46.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:46.749 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:10:46.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:46.749 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260508 13:10:46.749 | INFO | PRE_HQ : 66.2%, 1.8, 60.6ms, 0 | 20260508 13:10:46.749 | INFO | BAZWRITER : 66.8%, 2.7, 91.1ms, 0 | 20260508 13:10:46.749 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:10:46.750 | INFO | Uploaded 36GiB to GPU in 2.64772s (13.5966GiB/s) | 20260508 13:10:46.750 | INFO | Downloaded 38GiB to GPU in 2.71302s (14.0065GiB/s) | 20260508 13:10:51.885 | INFO | Analyzing chunk frames = [8360448,8360960) | 20260508 13:10:51.885 | INFO | MemInfo: { | 20260508 13:10:51.885 | INFO | "gpu" : " 17 GB", | 20260508 13:10:51.885 | INFO | "huge" : "", | 20260508 13:10:51.885 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:51.885 | INFO | "malloc" : " 14 GB", | 20260508 13:10:51.885 | INFO | "pinned" : " 29 GB" | 20260508 13:10:51.885 | INFO | } | 20260508 13:10:51.885 | INFO | Perf report for the last 5.13602 seconds: | 20260508 13:10:51.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:51.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.885 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 13:10:51.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.885 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:10:51.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:51.886 | INFO | Uploaded 36GiB to GPU in 2.47952s (14.5189GiB/s) | 20260508 13:10:51.886 | INFO | Downloaded 34GiB to GPU in 2.45682s (13.839GiB/s) | 20260508 13:10:57.023 | INFO | Analyzing chunk frames = [8360960,8361472) | 20260508 13:10:57.023 | INFO | MemInfo: { | 20260508 13:10:57.023 | INFO | "gpu" : " 17 GB", | 20260508 13:10:57.023 | INFO | "huge" : "", | 20260508 13:10:57.023 | INFO | "huge_shared" : " 29 GB", | 20260508 13:10:57.023 | INFO | "malloc" : " 14 GB", | 20260508 13:10:57.023 | INFO | "pinned" : " 29 GB" | 20260508 13:10:57.023 | INFO | } | 20260508 13:10:57.024 | INFO | Perf report for the last 5.13813 seconds: | 20260508 13:10:57.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:10:57.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 13:10:57.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:10:57.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:10:57.024 | INFO | Uploaded 36GiB to GPU in 2.54209s (14.1616GiB/s) | 20260508 13:10:57.024 | INFO | Downloaded 34GiB to GPU in 2.46547s (13.7905GiB/s) | 20260508 13:11:02.168 | INFO | Analyzing chunk frames = [8361472,8361984) | 20260508 13:11:02.168 | INFO | MemInfo: { | 20260508 13:11:02.168 | INFO | "gpu" : " 17 GB", | 20260508 13:11:02.168 | INFO | "huge" : "", | 20260508 13:11:02.168 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:02.168 | INFO | "malloc" : " 14 GB", | 20260508 13:11:02.168 | INFO | "pinned" : " 29 GB" | 20260508 13:11:02.168 | INFO | } | 20260508 13:11:02.168 | INFO | Perf report for the last 5.14488 seconds: | 20260508 13:11:02.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:02.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.168 | INFO | ANALYSIS : 66.2%, 3.0, 102.7ms, 0 | 20260508 13:11:02.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.169 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:11:02.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:02.169 | INFO | Uploaded 36GiB to GPU in 2.54389s (14.1516GiB/s) | 20260508 13:11:02.169 | INFO | Downloaded 34GiB to GPU in 2.4578s (13.8335GiB/s) | 20260508 13:11:07.306 | INFO | Analyzing chunk frames = [8361984,8362496) | 20260508 13:11:07.306 | INFO | MemInfo: { | 20260508 13:11:07.306 | INFO | "gpu" : " 17 GB", | 20260508 13:11:07.306 | INFO | "huge" : "", | 20260508 13:11:07.306 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:07.306 | INFO | "malloc" : " 14 GB", | 20260508 13:11:07.307 | INFO | "pinned" : " 29 GB" | 20260508 13:11:07.307 | INFO | } | 20260508 13:11:07.307 | INFO | Perf report for the last 5.13815 seconds: | 20260508 13:11:07.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:07.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 13:11:07.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:11:07.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:07.307 | INFO | Uploaded 36GiB to GPU in 2.42963s (14.8171GiB/s) | 20260508 13:11:07.307 | INFO | Downloaded 34GiB to GPU in 2.51645s (13.5111GiB/s) | 20260508 13:11:12.445 | INFO | Analyzing chunk frames = [8362496,8363008) | 20260508 13:11:12.445 | INFO | MemInfo: { | 20260508 13:11:12.445 | INFO | "gpu" : " 17 GB", | 20260508 13:11:12.445 | INFO | "huge" : "", | 20260508 13:11:12.445 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:12.445 | INFO | "malloc" : " 14 GB", | 20260508 13:11:12.445 | INFO | "pinned" : " 29 GB" | 20260508 13:11:12.445 | INFO | } | 20260508 13:11:12.445 | INFO | Perf report for the last 5.13834 seconds: | 20260508 13:11:12.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:12.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:11:12.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:11:12.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:12.445 | INFO | Uploaded 36GiB to GPU in 2.42327s (14.856GiB/s) | 20260508 13:11:12.445 | INFO | Downloaded 34GiB to GPU in 2.43244s (13.9777GiB/s) | 20260508 13:11:17.583 | INFO | Analyzing chunk frames = [8363008,8363520) | 20260508 13:11:17.583 | INFO | MemInfo: { | 20260508 13:11:17.583 | INFO | "gpu" : " 17 GB", | 20260508 13:11:17.583 | INFO | "huge" : "", | 20260508 13:11:17.583 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:17.583 | INFO | "malloc" : " 14 GB", | 20260508 13:11:17.583 | INFO | "pinned" : " 29 GB" | 20260508 13:11:17.584 | INFO | } | 20260508 13:11:17.584 | INFO | Perf report for the last 5.13866 seconds: | 20260508 13:11:17.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:17.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:11:17.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:11:17.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:17.584 | INFO | Uploaded 36GiB to GPU in 2.485s (14.4869GiB/s) | 20260508 13:11:17.584 | INFO | Downloaded 34GiB to GPU in 2.44943s (13.8808GiB/s) | 20260508 13:11:22.721 | INFO | Analyzing chunk frames = [8363520,8364032) | 20260508 13:11:22.721 | INFO | MemInfo: { | 20260508 13:11:22.721 | INFO | "gpu" : " 17 GB", | 20260508 13:11:22.721 | INFO | "huge" : "", | 20260508 13:11:22.721 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:22.721 | INFO | "malloc" : " 14 GB", | 20260508 13:11:22.721 | INFO | "pinned" : " 29 GB" | 20260508 13:11:22.721 | INFO | } | 20260508 13:11:22.721 | INFO | Perf report for the last 5.13753 seconds: | 20260508 13:11:22.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:22.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.721 | INFO | ANALYSIS : 64.7%, 3.0, 101.2ms, 0 | 20260508 13:11:22.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.721 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:11:22.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:22.722 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:11:22.722 | INFO | Total mem usage: 16.7 GiB | 20260508 13:11:22.722 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:11:22.722 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 13:11:22.722 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:11:22.722 | INFO | High Water Breakdown: | 20260508 13:11:22.722 | INFO | 59.6% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:11:22.722 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:11:22.722 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:11:22.722 | INFO | ------------------------------------------ | 20260508 13:11:22.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:11:22.722 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:11:22.722 | INFO | Total mem usage: 26.5 GiB | 20260508 13:11:22.722 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:11:22.722 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:11:22.722 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:11:22.722 | INFO | High Water Breakdown: | 20260508 13:11:22.722 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:11:22.722 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:11:22.722 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:11:22.722 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:11:22.722 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:11:22.722 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:11:22.722 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:11:22.722 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:11:22.722 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:11:22.722 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:11:22.722 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:11:22.722 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:11:22.722 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:11:22.722 | INFO | Uploaded 36GiB to GPU in 2.37932s (15.1304GiB/s) | 20260508 13:11:22.722 | INFO | Downloaded 34GiB to GPU in 2.42493s (14.021GiB/s) | 20260508 13:11:22.722 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:11:22.722 | INFO | ------------------------------------------ | 20260508 13:11:22.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:11:22.722 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:11:22.722 | INFO | Total mem usage: 27.8 GiB | 20260508 13:11:22.722 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:11:22.722 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:11:22.722 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:11:22.722 | INFO | High Water Breakdown: | 20260508 13:11:22.722 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:11:22.722 | INFO | ------------------------------------------ | 20260508 13:11:22.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:11:22.722 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:11:22.722 | INFO | Total mem usage: 15.7 GiB | 20260508 13:11:22.722 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:11:22.722 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:11:22.722 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:11:22.722 | INFO | High Water Breakdown: | 20260508 13:11:22.722 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:11:22.722 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:11:22.722 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:11:22.722 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:11:22.722 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:11:22.722 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:11:22.722 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:11:22.722 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:11:22.723 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:11:22.723 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:11:22.723 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:11:22.723 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:11:22.723 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:11:22.723 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:11:22.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:11:22.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:11:22.723 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:11:22.723 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:11:22.723 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:11:22.723 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:11:22.723 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:11:22.723 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:11:22.723 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:11:22.723 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:11:22.723 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:11:22.723 | INFO | ------------------------------------------ | 20260508 13:11:22.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:11:22.723 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:11:22.723 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:11:22.723 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:11:22.723 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.41e+01 / 6.46e-01 | 20260508 13:11:22.723 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.20e-02 / 9.54e-03 | 20260508 13:11:22.723 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.39e+00, 5.94e-01 / 2.55e-01 | 20260508 13:11:22.723 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.39e+00, 2.08e-01 / 1.32e-01 | 20260508 13:11:22.723 | INFO | PerfReport: 16.00%, DME -- 4.18e+01 / 4.50e+01, 3.66e+00 / 1.24e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.82e+00 / 1.44e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.32e+00 / 2.37e+00, 4.05e-01 / 2.79e-01 | 20260508 13:11:22.723 | INFO | PerfReport: 2.94%, Metrics -- 9.59e-01 / 1.07e+00, 2.36e-01 / 1.44e-01 | 20260508 13:11:22.723 | INFO | PerfReport: 0.90%, PulseSerializing -- 2.93e-01 / 3.16e-01, 6.00e-02 / 2.95e-02 | 20260508 13:11:22.723 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:11:22.723 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:11:22.723 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:11:22.723 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 24.42%, Upload -- 2.56e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 24.81%, Download -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 31.13%, Compute -- 3.26e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 2.14%, LockUpload -- 2.24e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 2.55%, LockDownload -- 2.68e+00 / 0.00e+00, 3.99e+00 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 14.93%, LockCompute -- 1.57e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 13:11:22.723 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260508 13:11:22.819 | INFO | Starting a round of Baz IO | 20260508 13:11:26.227 | INFO | Populating RTMetrics report | 20260508 13:11:26.227 | INFO | RTMetrics report populated | 20260508 13:11:26.227 | INFO | Opening tmp RTMetrics json file | 20260508 13:11:26.236 | INFO | Serializing RTMetrics to json | 20260508 13:11:26.237 | INFO | Serialization done, swapping tmp file now... | 20260508 13:11:26.238 | INFO | RTMetrics done | 20260508 13:11:26.292 | INFO | Finished a round of Baz IO: Wrote 1.393 GiB in 3.473 seconds | 20260508 13:11:26.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.03% | 20260508 13:11:26.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147541% / 0% / 7.94729e-05% | 20260508 13:11:26.292 | INFO | Baz IO overall: | 20260508 13:11:26.292 | INFO | BytesWritten : 3570229556456 | 20260508 13:11:26.292 | INFO | NumEvents : 1712185538533 | 20260508 13:11:27.868 | INFO | Analyzing chunk frames = [8364032,8364544) | 20260508 13:11:27.868 | INFO | MemInfo: { | 20260508 13:11:27.868 | INFO | "gpu" : " 17 GB", | 20260508 13:11:27.868 | INFO | "huge" : "", | 20260508 13:11:27.868 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:27.868 | INFO | "malloc" : " 14 GB", | 20260508 13:11:27.868 | INFO | "pinned" : " 29 GB" | 20260508 13:11:27.868 | INFO | } | 20260508 13:11:27.868 | INFO | Perf report for the last 5.14645 seconds: | 20260508 13:11:27.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:27.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:27.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:27.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:27.868 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:11:27.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:27.868 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:11:27.868 | INFO | PRE_HQ : 65.2%, 1.6, 53.5ms, 0 | 20260508 13:11:27.868 | INFO | BAZWRITER : 66.8%, 2.5, 84.4ms, 0 | 20260508 13:11:27.868 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:11:27.868 | INFO | Uploaded 36GiB to GPU in 2.61027s (13.7917GiB/s) | 20260508 13:11:27.868 | INFO | Downloaded 38GiB to GPU in 2.71154s (14.0142GiB/s) | 20260508 13:11:33.004 | INFO | Analyzing chunk frames = [8364544,8365056) | 20260508 13:11:33.004 | INFO | MemInfo: { | 20260508 13:11:33.004 | INFO | "gpu" : " 17 GB", | 20260508 13:11:33.004 | INFO | "huge" : "", | 20260508 13:11:33.004 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:33.004 | INFO | "malloc" : " 14 GB", | 20260508 13:11:33.004 | INFO | "pinned" : " 29 GB" | 20260508 13:11:33.004 | INFO | } | 20260508 13:11:33.004 | INFO | Perf report for the last 5.13632 seconds: | 20260508 13:11:33.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:33.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:11:33.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:11:33.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:33.004 | INFO | Uploaded 36GiB to GPU in 2.46168s (14.6242GiB/s) | 20260508 13:11:33.004 | INFO | Downloaded 34GiB to GPU in 2.47255s (13.751GiB/s) | 20260508 13:11:38.144 | INFO | Analyzing chunk frames = [8365056,8365568) | 20260508 13:11:38.144 | INFO | MemInfo: { | 20260508 13:11:38.144 | INFO | "gpu" : " 17 GB", | 20260508 13:11:38.144 | INFO | "huge" : "", | 20260508 13:11:38.144 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:38.144 | INFO | "malloc" : " 14 GB", | 20260508 13:11:38.144 | INFO | "pinned" : " 29 GB" | 20260508 13:11:38.145 | INFO | } | 20260508 13:11:38.145 | INFO | Perf report for the last 5.14062 seconds: | 20260508 13:11:38.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:38.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 13:11:38.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:11:38.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:38.145 | INFO | Uploaded 36GiB to GPU in 2.47682s (14.5348GiB/s) | 20260508 13:11:38.145 | INFO | Downloaded 34GiB to GPU in 2.46391s (13.7992GiB/s) | 20260508 13:11:43.272 | INFO | Analyzing chunk frames = [8365568,8366080) | 20260508 13:11:43.272 | INFO | MemInfo: { | 20260508 13:11:43.272 | INFO | "gpu" : " 17 GB", | 20260508 13:11:43.272 | INFO | "huge" : "", | 20260508 13:11:43.273 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:43.273 | INFO | "malloc" : " 14 GB", | 20260508 13:11:43.273 | INFO | "pinned" : " 29 GB" | 20260508 13:11:43.273 | INFO | } | 20260508 13:11:43.273 | INFO | Perf report for the last 5.12829 seconds: | 20260508 13:11:43.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:43.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | ANALYSIS : 65.5%, 3.0, 101.3ms, 0 | 20260508 13:11:43.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:11:43.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:43.273 | INFO | Uploaded 36GiB to GPU in 2.40009s (14.9994GiB/s) | 20260508 13:11:43.273 | INFO | Downloaded 34GiB to GPU in 2.47921s (13.7141GiB/s) | 20260508 13:11:48.419 | INFO | Analyzing chunk frames = [8366080,8366592) | 20260508 13:11:48.419 | INFO | MemInfo: { | 20260508 13:11:48.419 | INFO | "gpu" : " 17 GB", | 20260508 13:11:48.419 | INFO | "huge" : "", | 20260508 13:11:48.419 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:48.419 | INFO | "malloc" : " 14 GB", | 20260508 13:11:48.419 | INFO | "pinned" : " 29 GB" | 20260508 13:11:48.419 | INFO | } | 20260508 13:11:48.419 | INFO | Perf report for the last 5.14654 seconds: | 20260508 13:11:48.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:48.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.419 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 13:11:48.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.420 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:11:48.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:48.420 | INFO | Uploaded 36GiB to GPU in 2.40424s (14.9735GiB/s) | 20260508 13:11:48.420 | INFO | Downloaded 34GiB to GPU in 2.54651s (13.3516GiB/s) | 20260508 13:11:53.559 | INFO | Analyzing chunk frames = [8366592,8367104) | 20260508 13:11:53.559 | INFO | MemInfo: { | 20260508 13:11:53.559 | INFO | "gpu" : " 17 GB", | 20260508 13:11:53.559 | INFO | "huge" : "", | 20260508 13:11:53.559 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:53.559 | INFO | "malloc" : " 14 GB", | 20260508 13:11:53.559 | INFO | "pinned" : " 29 GB" | 20260508 13:11:53.559 | INFO | } | 20260508 13:11:53.559 | INFO | Perf report for the last 5.13984 seconds: | 20260508 13:11:53.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:53.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.559 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:11:53.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.560 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260508 13:11:53.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:53.560 | INFO | Uploaded 36GiB to GPU in 2.4007s (14.9956GiB/s) | 20260508 13:11:53.560 | INFO | Downloaded 34GiB to GPU in 2.44597s (13.9004GiB/s) | 20260508 13:11:58.698 | INFO | Analyzing chunk frames = [8367104,8367616) | 20260508 13:11:58.698 | INFO | MemInfo: { | 20260508 13:11:58.698 | INFO | "gpu" : " 17 GB", | 20260508 13:11:58.698 | INFO | "huge" : "", | 20260508 13:11:58.698 | INFO | "huge_shared" : " 29 GB", | 20260508 13:11:58.698 | INFO | "malloc" : " 14 GB", | 20260508 13:11:58.698 | INFO | "pinned" : " 29 GB" | 20260508 13:11:58.698 | INFO | } | 20260508 13:11:58.698 | INFO | Perf report for the last 5.13858 seconds: | 20260508 13:11:58.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:11:58.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:11:58.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:11:58.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:11:58.698 | INFO | Uploaded 36GiB to GPU in 2.4567s (14.6538GiB/s) | 20260508 13:11:58.698 | INFO | Downloaded 34GiB to GPU in 2.4641s (13.7981GiB/s) | 20260508 13:12:03.839 | INFO | Analyzing chunk frames = [8367616,8368128) | 20260508 13:12:03.839 | INFO | MemInfo: { | 20260508 13:12:03.839 | INFO | "gpu" : " 17 GB", | 20260508 13:12:03.839 | INFO | "huge" : "", | 20260508 13:12:03.839 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:03.839 | INFO | "malloc" : " 14 GB", | 20260508 13:12:03.839 | INFO | "pinned" : " 29 GB" | 20260508 13:12:03.839 | INFO | } | 20260508 13:12:03.839 | INFO | Perf report for the last 5.14151 seconds: | 20260508 13:12:03.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:03.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.839 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 13:12:03.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.839 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:12:03.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:03.840 | INFO | Uploaded 36GiB to GPU in 2.47107s (14.5686GiB/s) | 20260508 13:12:03.840 | INFO | Downloaded 34GiB to GPU in 2.46938s (13.7687GiB/s) | 20260508 13:12:03.936 | INFO | Starting a round of Baz IO | 20260508 13:12:07.347 | INFO | Populating RTMetrics report | 20260508 13:12:07.348 | INFO | RTMetrics report populated | 20260508 13:12:07.348 | INFO | Opening tmp RTMetrics json file | 20260508 13:12:07.356 | INFO | Serializing RTMetrics to json | 20260508 13:12:07.357 | INFO | Serialization done, swapping tmp file now... | 20260508 13:12:07.357 | INFO | RTMetrics done | 20260508 13:12:07.411 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.474 seconds | 20260508 13:12:07.411 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:12:07.411 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146027% / 0% / 6.35783e-05% | 20260508 13:12:07.411 | INFO | Baz IO overall: | 20260508 13:12:07.411 | INFO | BytesWritten : 3571722781179 | 20260508 13:12:07.411 | INFO | NumEvents : 1712884463492 | 20260508 13:12:08.975 | INFO | Analyzing chunk frames = [8368128,8368640) | 20260508 13:12:08.975 | INFO | MemInfo: { | 20260508 13:12:08.975 | INFO | "gpu" : " 17 GB", | 20260508 13:12:08.975 | INFO | "huge" : "", | 20260508 13:12:08.975 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:08.975 | INFO | "malloc" : " 14 GB", | 20260508 13:12:08.975 | INFO | "pinned" : " 29 GB" | 20260508 13:12:08.975 | INFO | } | 20260508 13:12:08.975 | INFO | Perf report for the last 5.13581 seconds: | 20260508 13:12:08.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:08.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:08.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:08.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:08.975 | INFO | ANALYSIS : 67.7%, 3.0, 105.5ms, 0 | 20260508 13:12:08.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:08.975 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:12:08.975 | INFO | PRE_HQ : 65.6%, 1.6, 53.6ms, 0 | 20260508 13:12:08.975 | INFO | BAZWRITER : 66.9%, 2.4, 83.9ms, 0 | 20260508 13:12:08.975 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:12:08.975 | INFO | Uploaded 36GiB to GPU in 2.6163s (13.7599GiB/s) | 20260508 13:12:08.975 | INFO | Downloaded 38GiB to GPU in 2.73251s (13.9066GiB/s) | 20260508 13:12:14.115 | INFO | Analyzing chunk frames = [8368640,8369152) | 20260508 13:12:14.115 | INFO | MemInfo: { | 20260508 13:12:14.115 | INFO | "gpu" : " 17 GB", | 20260508 13:12:14.115 | INFO | "huge" : "", | 20260508 13:12:14.115 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:14.115 | INFO | "malloc" : " 14 GB", | 20260508 13:12:14.115 | INFO | "pinned" : " 29 GB" | 20260508 13:12:14.115 | INFO | } | 20260508 13:12:14.115 | INFO | Perf report for the last 5.13974 seconds: | 20260508 13:12:14.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:14.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 13:12:14.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:12:14.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:14.115 | INFO | Uploaded 36GiB to GPU in 2.47558s (14.5421GiB/s) | 20260508 13:12:14.116 | INFO | Downloaded 34GiB to GPU in 2.47821s (13.7196GiB/s) | 20260508 13:12:19.254 | INFO | Analyzing chunk frames = [8369152,8369664) | 20260508 13:12:19.254 | INFO | MemInfo: { | 20260508 13:12:19.254 | INFO | "gpu" : " 17 GB", | 20260508 13:12:19.254 | INFO | "huge" : "", | 20260508 13:12:19.254 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:19.254 | INFO | "malloc" : " 14 GB", | 20260508 13:12:19.254 | INFO | "pinned" : " 29 GB" | 20260508 13:12:19.254 | INFO | } | 20260508 13:12:19.254 | INFO | Perf report for the last 5.1388 seconds: | 20260508 13:12:19.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:19.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 13:12:19.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:12:19.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:19.254 | INFO | Uploaded 36GiB to GPU in 2.48725s (14.4738GiB/s) | 20260508 13:12:19.254 | INFO | Downloaded 34GiB to GPU in 2.45546s (13.8467GiB/s) | 20260508 13:12:24.395 | INFO | Analyzing chunk frames = [8369664,8370176) | 20260508 13:12:24.395 | INFO | MemInfo: { | 20260508 13:12:24.395 | INFO | "gpu" : " 17 GB", | 20260508 13:12:24.395 | INFO | "huge" : "", | 20260508 13:12:24.395 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:24.395 | INFO | "malloc" : " 14 GB", | 20260508 13:12:24.395 | INFO | "pinned" : " 29 GB" | 20260508 13:12:24.395 | INFO | } | 20260508 13:12:24.395 | INFO | Perf report for the last 5.14157 seconds: | 20260508 13:12:24.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:24.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.395 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 13:12:24.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.395 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:12:24.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:24.396 | INFO | Uploaded 36GiB to GPU in 2.46545s (14.6018GiB/s) | 20260508 13:12:24.396 | INFO | Downloaded 34GiB to GPU in 2.44869s (13.885GiB/s) | 20260508 13:12:29.530 | INFO | Analyzing chunk frames = [8370176,8370688) | 20260508 13:12:29.530 | INFO | MemInfo: { | 20260508 13:12:29.530 | INFO | "gpu" : " 17 GB", | 20260508 13:12:29.530 | INFO | "huge" : "", | 20260508 13:12:29.530 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:29.530 | INFO | "malloc" : " 14 GB", | 20260508 13:12:29.530 | INFO | "pinned" : " 29 GB" | 20260508 13:12:29.530 | INFO | } | 20260508 13:12:29.531 | INFO | Perf report for the last 5.13516 seconds: | 20260508 13:12:29.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:29.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 13:12:29.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 13:12:29.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:29.531 | INFO | Uploaded 36GiB to GPU in 2.37188s (15.1778GiB/s) | 20260508 13:12:29.531 | INFO | Downloaded 34GiB to GPU in 2.51523s (13.5176GiB/s) | 20260508 13:12:34.674 | INFO | Analyzing chunk frames = [8370688,8371200) | 20260508 13:12:34.674 | INFO | MemInfo: { | 20260508 13:12:34.674 | INFO | "gpu" : " 17 GB", | 20260508 13:12:34.674 | INFO | "huge" : "", | 20260508 13:12:34.674 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:34.674 | INFO | "malloc" : " 14 GB", | 20260508 13:12:34.674 | INFO | "pinned" : " 29 GB" | 20260508 13:12:34.674 | INFO | } | 20260508 13:12:34.674 | INFO | Perf report for the last 5.14377 seconds: | 20260508 13:12:34.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:34.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.674 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 13:12:34.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.674 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:12:34.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:34.675 | INFO | Uploaded 36GiB to GPU in 2.37659s (15.1477GiB/s) | 20260508 13:12:34.675 | INFO | Downloaded 34GiB to GPU in 2.45511s (13.8487GiB/s) | 20260508 13:12:39.820 | INFO | Analyzing chunk frames = [8371200,8371712) | 20260508 13:12:39.820 | INFO | MemInfo: { | 20260508 13:12:39.820 | INFO | "gpu" : " 17 GB", | 20260508 13:12:39.820 | INFO | "huge" : "", | 20260508 13:12:39.821 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:39.821 | INFO | "malloc" : " 14 GB", | 20260508 13:12:39.821 | INFO | "pinned" : " 29 GB" | 20260508 13:12:39.821 | INFO | } | 20260508 13:12:39.821 | INFO | Perf report for the last 5.14634 seconds: | 20260508 13:12:39.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:39.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | ANALYSIS : 64.8%, 3.0, 100.9ms, 0 | 20260508 13:12:39.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:12:39.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:39.821 | INFO | Uploaded 36GiB to GPU in 2.3087s (15.5932GiB/s) | 20260508 13:12:39.821 | INFO | Downloaded 34GiB to GPU in 2.43485s (13.9639GiB/s) | 20260508 13:12:44.955 | INFO | Analyzing chunk frames = [8371712,8372224) | 20260508 13:12:44.955 | INFO | MemInfo: { | 20260508 13:12:44.955 | INFO | "gpu" : " 17 GB", | 20260508 13:12:44.955 | INFO | "huge" : "", | 20260508 13:12:44.955 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:44.955 | INFO | "malloc" : " 14 GB", | 20260508 13:12:44.955 | INFO | "pinned" : " 29 GB" | 20260508 13:12:44.955 | INFO | } | 20260508 13:12:44.955 | INFO | Perf report for the last 5.13412 seconds: | 20260508 13:12:44.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:44.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 13:12:44.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:12:44.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:44.955 | INFO | Uploaded 36GiB to GPU in 2.30819s (15.5966GiB/s) | 20260508 13:12:44.955 | INFO | Downloaded 34GiB to GPU in 2.45416s (13.854GiB/s) | 20260508 13:12:44.955 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:12:44.955 | INFO | Total mem usage: 16.7 GiB | 20260508 13:12:44.955 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:12:44.955 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:12:44.955 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:12:44.955 | INFO | High Water Breakdown: | 20260508 13:12:44.955 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:12:44.955 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:12:44.955 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:12:44.955 | INFO | ------------------------------------------ | 20260508 13:12:44.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:12:44.955 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:12:44.955 | INFO | Total mem usage: 26.5 GiB | 20260508 13:12:44.955 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:12:44.955 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:12:44.955 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:12:44.955 | INFO | High Water Breakdown: | 20260508 13:12:44.955 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:12:44.955 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:12:44.955 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:12:44.955 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:12:44.955 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:12:44.955 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:12:44.955 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:12:44.955 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:12:44.955 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:12:44.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:12:44.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:12:44.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:12:44.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:12:44.955 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:12:44.955 | INFO | ------------------------------------------ | 20260508 13:12:44.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:12:44.955 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:12:44.955 | INFO | Total mem usage: 27.8 GiB | 20260508 13:12:44.955 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:12:44.955 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:12:44.955 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:12:44.955 | INFO | High Water Breakdown: | 20260508 13:12:44.955 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:12:44.955 | INFO | ------------------------------------------ | 20260508 13:12:44.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:12:44.955 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:12:44.955 | INFO | Total mem usage: 15.7 GiB | 20260508 13:12:44.955 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:12:44.955 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:12:44.955 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:12:44.955 | INFO | High Water Breakdown: | 20260508 13:12:44.955 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:12:44.955 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:12:44.956 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:12:44.956 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:12:44.956 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:12:44.956 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:12:44.956 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:12:44.956 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:12:44.956 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:12:44.956 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:12:44.956 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:12:44.956 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:12:44.956 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:12:44.956 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:12:44.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:12:44.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:12:44.956 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:12:44.956 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:12:44.956 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:12:44.956 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:12:44.956 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:12:44.956 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:12:44.956 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:12:44.956 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:12:44.956 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:12:44.956 | INFO | ------------------------------------------ | 20260508 13:12:44.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:12:44.956 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:12:44.956 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:12:44.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:12:44.956 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 4.79e-01 | 20260508 13:12:44.956 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.01e-01, 1.26e-02 / 6.13e-03 | 20260508 13:12:44.956 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.47e+00, 6.03e-01 / 1.89e-01 | 20260508 13:12:44.956 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.42e+00, 2.13e-01 / 1.31e-01 | 20260508 13:12:44.956 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.50e+01, 3.61e+00 / 1.09e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.83e+00 / 1.39e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.41e+00, 4.02e-01 / 2.71e-01 | 20260508 13:12:44.956 | INFO | PerfReport: 2.93%, Metrics -- 9.58e-01 / 1.09e+00, 2.40e-01 / 9.51e-02 | 20260508 13:12:44.956 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.16e-01, 4.61e-02 / 2.59e-02 | 20260508 13:12:44.956 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:12:44.956 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:12:44.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:12:44.956 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 24.28%, Upload -- 2.55e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 24.86%, Download -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 31.15%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 2.05%, LockUpload -- 2.15e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 2.71%, LockDownload -- 2.84e+00 / 0.00e+00, 4.20e+00 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 14.93%, LockCompute -- 1.57e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 13:12:44.956 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260508 13:12:45.052 | INFO | Starting a round of Baz IO | 20260508 13:12:48.456 | INFO | Populating RTMetrics report | 20260508 13:12:48.456 | INFO | RTMetrics report populated | 20260508 13:12:48.456 | INFO | Opening tmp RTMetrics json file | 20260508 13:12:48.468 | INFO | Serializing RTMetrics to json | 20260508 13:12:48.469 | INFO | Serialization done, swapping tmp file now... | 20260508 13:12:48.469 | INFO | RTMetrics done | 20260508 13:12:48.526 | INFO | Finished a round of Baz IO: Wrote 1.39 GiB in 3.474 seconds | 20260508 13:12:48.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:12:48.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146163% / 0% / 6.35783e-05% | 20260508 13:12:48.526 | INFO | Baz IO overall: | 20260508 13:12:48.526 | INFO | BytesWritten : 3573215283540 | 20260508 13:12:48.526 | INFO | NumEvents : 1713582653136 | 20260508 13:12:50.087 | INFO | Analyzing chunk frames = [8372224,8372736) | 20260508 13:12:50.087 | INFO | MemInfo: { | 20260508 13:12:50.087 | INFO | "gpu" : " 17 GB", | 20260508 13:12:50.087 | INFO | "huge" : "", | 20260508 13:12:50.087 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:50.087 | INFO | "malloc" : " 14 GB", | 20260508 13:12:50.088 | INFO | "pinned" : " 29 GB" | 20260508 13:12:50.088 | INFO | } | 20260508 13:12:50.088 | INFO | Perf report for the last 5.1328 seconds: | 20260508 13:12:50.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:50.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:50.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:50.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:50.088 | INFO | ANALYSIS : 67.6%, 3.0, 105.4ms, 0 | 20260508 13:12:50.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:50.088 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:12:50.088 | INFO | PRE_HQ : 65.4%, 1.6, 53.6ms, 0 | 20260508 13:12:50.088 | INFO | BAZWRITER : 67.0%, 2.4, 84.1ms, 0 | 20260508 13:12:50.088 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:12:50.088 | INFO | Uploaded 36GiB to GPU in 2.62469s (13.7159GiB/s) | 20260508 13:12:50.088 | INFO | Downloaded 38GiB to GPU in 2.72718s (13.9338GiB/s) | 20260508 13:12:55.232 | INFO | Analyzing chunk frames = [8372736,8373248) | 20260508 13:12:55.232 | INFO | MemInfo: { | 20260508 13:12:55.232 | INFO | "gpu" : " 17 GB", | 20260508 13:12:55.232 | INFO | "huge" : "", | 20260508 13:12:55.232 | INFO | "huge_shared" : " 29 GB", | 20260508 13:12:55.232 | INFO | "malloc" : " 14 GB", | 20260508 13:12:55.232 | INFO | "pinned" : " 29 GB" | 20260508 13:12:55.232 | INFO | } | 20260508 13:12:55.232 | INFO | Perf report for the last 5.14415 seconds: | 20260508 13:12:55.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:12:55.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:12:55.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:12:55.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:12:55.232 | INFO | Uploaded 36GiB to GPU in 2.44405s (14.7296GiB/s) | 20260508 13:12:55.232 | INFO | Downloaded 34GiB to GPU in 2.47691s (13.7268GiB/s) | 20260508 13:13:00.373 | INFO | Analyzing chunk frames = [8373248,8373760) | 20260508 13:13:00.373 | INFO | MemInfo: { | 20260508 13:13:00.373 | INFO | "gpu" : " 17 GB", | 20260508 13:13:00.373 | INFO | "huge" : "", | 20260508 13:13:00.373 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:00.373 | INFO | "malloc" : " 14 GB", | 20260508 13:13:00.373 | INFO | "pinned" : " 29 GB" | 20260508 13:13:00.373 | INFO | } | 20260508 13:13:00.373 | INFO | Perf report for the last 5.14122 seconds: | 20260508 13:13:00.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:00.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 13:13:00.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:13:00.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:00.373 | INFO | Uploaded 36GiB to GPU in 2.47628s (14.5379GiB/s) | 20260508 13:13:00.374 | INFO | Downloaded 34GiB to GPU in 2.48434s (13.6857GiB/s) | 20260508 13:13:05.523 | INFO | Analyzing chunk frames = [8373760,8374272) | 20260508 13:13:05.523 | INFO | MemInfo: { | 20260508 13:13:05.523 | INFO | "gpu" : " 17 GB", | 20260508 13:13:05.523 | INFO | "huge" : "", | 20260508 13:13:05.523 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:05.523 | INFO | "malloc" : " 14 GB", | 20260508 13:13:05.523 | INFO | "pinned" : " 29 GB" | 20260508 13:13:05.523 | INFO | } | 20260508 13:13:05.523 | INFO | Perf report for the last 5.14984 seconds: | 20260508 13:13:05.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:05.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | ANALYSIS : 65.7%, 3.0, 102.1ms, 0 | 20260508 13:13:05.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260508 13:13:05.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:05.523 | INFO | Uploaded 36GiB to GPU in 2.43045s (14.812GiB/s) | 20260508 13:13:05.523 | INFO | Downloaded 34GiB to GPU in 2.49154s (13.6462GiB/s) | 20260508 13:13:10.654 | INFO | Analyzing chunk frames = [8374272,8374784) | 20260508 13:13:10.654 | INFO | MemInfo: { | 20260508 13:13:10.654 | INFO | "gpu" : " 17 GB", | 20260508 13:13:10.654 | INFO | "huge" : "", | 20260508 13:13:10.654 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:10.654 | INFO | "malloc" : " 14 GB", | 20260508 13:13:10.654 | INFO | "pinned" : " 29 GB" | 20260508 13:13:10.654 | INFO | } | 20260508 13:13:10.655 | INFO | Perf report for the last 5.13163 seconds: | 20260508 13:13:10.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:10.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 13:13:10.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260508 13:13:10.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:10.655 | INFO | Uploaded 36GiB to GPU in 2.4172s (14.8932GiB/s) | 20260508 13:13:10.655 | INFO | Downloaded 34GiB to GPU in 2.51407s (13.5239GiB/s) | 20260508 13:13:15.784 | INFO | Analyzing chunk frames = [8374784,8375296) | 20260508 13:13:15.784 | INFO | MemInfo: { | 20260508 13:13:15.784 | INFO | "gpu" : " 17 GB", | 20260508 13:13:15.785 | INFO | "huge" : "", | 20260508 13:13:15.785 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:15.785 | INFO | "malloc" : " 14 GB", | 20260508 13:13:15.785 | INFO | "pinned" : " 29 GB" | 20260508 13:13:15.785 | INFO | } | 20260508 13:13:15.785 | INFO | Perf report for the last 5.13009 seconds: | 20260508 13:13:15.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:15.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | ANALYSIS : 65.2%, 3.0, 101.3ms, 0 | 20260508 13:13:15.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:13:15.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:15.785 | INFO | Uploaded 36GiB to GPU in 2.30831s (15.5958GiB/s) | 20260508 13:13:15.785 | INFO | Downloaded 34GiB to GPU in 2.44937s (13.8811GiB/s) | 20260508 13:13:20.931 | INFO | Analyzing chunk frames = [8375296,8375808) | 20260508 13:13:20.931 | INFO | MemInfo: { | 20260508 13:13:20.931 | INFO | "gpu" : " 17 GB", | 20260508 13:13:20.931 | INFO | "huge" : "", | 20260508 13:13:20.931 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:20.931 | INFO | "malloc" : " 14 GB", | 20260508 13:13:20.931 | INFO | "pinned" : " 29 GB" | 20260508 13:13:20.931 | INFO | } | 20260508 13:13:20.931 | INFO | Perf report for the last 5.14608 seconds: | 20260508 13:13:20.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:20.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 13:13:20.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:13:20.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:20.931 | INFO | Uploaded 36GiB to GPU in 2.46436s (14.6083GiB/s) | 20260508 13:13:20.931 | INFO | Downloaded 34GiB to GPU in 2.4769s (13.7268GiB/s) | 20260508 13:13:26.069 | INFO | Analyzing chunk frames = [8375808,8376320) | 20260508 13:13:26.069 | INFO | MemInfo: { | 20260508 13:13:26.069 | INFO | "gpu" : " 17 GB", | 20260508 13:13:26.069 | INFO | "huge" : "", | 20260508 13:13:26.069 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:26.069 | INFO | "malloc" : " 14 GB", | 20260508 13:13:26.069 | INFO | "pinned" : " 29 GB" | 20260508 13:13:26.069 | INFO | } | 20260508 13:13:26.069 | INFO | Perf report for the last 5.1382 seconds: | 20260508 13:13:26.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:26.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.069 | INFO | ANALYSIS : 65.1%, 3.0, 101.5ms, 0 | 20260508 13:13:26.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.069 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:13:26.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:26.070 | INFO | Uploaded 36GiB to GPU in 2.36496s (15.2223GiB/s) | 20260508 13:13:26.070 | INFO | Downloaded 34GiB to GPU in 2.46093s (13.8159GiB/s) | 20260508 13:13:26.166 | INFO | Starting a round of Baz IO | 20260508 13:13:29.570 | INFO | Populating RTMetrics report | 20260508 13:13:29.571 | INFO | RTMetrics report populated | 20260508 13:13:29.571 | INFO | Opening tmp RTMetrics json file | 20260508 13:13:29.571 | INFO | Serializing RTMetrics to json | 20260508 13:13:29.573 | INFO | Serialization done, swapping tmp file now... | 20260508 13:13:29.573 | INFO | RTMetrics done | 20260508 13:13:29.633 | INFO | Finished a round of Baz IO: Wrote 1.39 GiB in 3.466 seconds | 20260508 13:13:29.633 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:13:29.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146155% / 0% / 8.74201e-05% | 20260508 13:13:29.633 | INFO | Baz IO overall: | 20260508 13:13:29.633 | INFO | BytesWritten : 3574708126164 | 20260508 13:13:29.633 | INFO | NumEvents : 1714281052568 | 20260508 13:13:31.207 | INFO | Analyzing chunk frames = [8376320,8376832) | 20260508 13:13:31.207 | INFO | MemInfo: { | 20260508 13:13:31.207 | INFO | "gpu" : " 17 GB", | 20260508 13:13:31.207 | INFO | "huge" : "", | 20260508 13:13:31.207 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:31.208 | INFO | "malloc" : " 14 GB", | 20260508 13:13:31.208 | INFO | "pinned" : " 29 GB" | 20260508 13:13:31.208 | INFO | } | 20260508 13:13:31.208 | INFO | Perf report for the last 5.13867 seconds: | 20260508 13:13:31.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:31.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:31.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:31.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:31.208 | INFO | ANALYSIS : 67.8%, 3.0, 105.4ms, 0 | 20260508 13:13:31.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:31.208 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:13:31.208 | INFO | PRE_HQ : 65.7%, 1.6, 53.8ms, 0 | 20260508 13:13:31.208 | INFO | BAZWRITER : 66.6%, 2.5, 84.0ms, 0 | 20260508 13:13:31.208 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:13:31.208 | INFO | Uploaded 36GiB to GPU in 2.53601s (14.1955GiB/s) | 20260508 13:13:31.208 | INFO | Downloaded 38GiB to GPU in 2.7277s (13.9312GiB/s) | 20260508 13:13:36.347 | INFO | Analyzing chunk frames = [8376832,8377344) | 20260508 13:13:36.347 | INFO | MemInfo: { | 20260508 13:13:36.347 | INFO | "gpu" : " 17 GB", | 20260508 13:13:36.347 | INFO | "huge" : "", | 20260508 13:13:36.347 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:36.347 | INFO | "malloc" : " 14 GB", | 20260508 13:13:36.347 | INFO | "pinned" : " 29 GB" | 20260508 13:13:36.347 | INFO | } | 20260508 13:13:36.347 | INFO | Perf report for the last 5.13974 seconds: | 20260508 13:13:36.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:36.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.347 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:13:36.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.347 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:13:36.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:36.348 | INFO | Uploaded 36GiB to GPU in 2.46417s (14.6094GiB/s) | 20260508 13:13:36.348 | INFO | Downloaded 34GiB to GPU in 2.46196s (13.8101GiB/s) | 20260508 13:13:41.483 | INFO | Analyzing chunk frames = [8377344,8377856) | 20260508 13:13:41.483 | INFO | MemInfo: { | 20260508 13:13:41.483 | INFO | "gpu" : " 17 GB", | 20260508 13:13:41.483 | INFO | "huge" : "", | 20260508 13:13:41.483 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:41.484 | INFO | "malloc" : " 14 GB", | 20260508 13:13:41.484 | INFO | "pinned" : " 29 GB" | 20260508 13:13:41.484 | INFO | } | 20260508 13:13:41.484 | INFO | Perf report for the last 5.13626 seconds: | 20260508 13:13:41.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:41.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 13:13:41.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:13:41.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:41.484 | INFO | Uploaded 36GiB to GPU in 2.47863s (14.5242GiB/s) | 20260508 13:13:41.484 | INFO | Downloaded 34GiB to GPU in 2.47201s (13.754GiB/s) | 20260508 13:13:46.624 | INFO | Analyzing chunk frames = [8377856,8378368) | 20260508 13:13:46.624 | INFO | MemInfo: { | 20260508 13:13:46.624 | INFO | "gpu" : " 17 GB", | 20260508 13:13:46.624 | INFO | "huge" : "", | 20260508 13:13:46.624 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:46.624 | INFO | "malloc" : " 14 GB", | 20260508 13:13:46.624 | INFO | "pinned" : " 29 GB" | 20260508 13:13:46.624 | INFO | } | 20260508 13:13:46.624 | INFO | Perf report for the last 5.14081 seconds: | 20260508 13:13:46.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:46.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.625 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 13:13:46.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.625 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:13:46.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:46.625 | INFO | Uploaded 36GiB to GPU in 2.47218s (14.5621GiB/s) | 20260508 13:13:46.625 | INFO | Downloaded 34GiB to GPU in 2.45338s (13.8584GiB/s) | 20260508 13:13:51.766 | INFO | Analyzing chunk frames = [8378368,8378880) | 20260508 13:13:51.766 | INFO | MemInfo: { | 20260508 13:13:51.766 | INFO | "gpu" : " 17 GB", | 20260508 13:13:51.766 | INFO | "huge" : "", | 20260508 13:13:51.766 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:51.766 | INFO | "malloc" : " 14 GB", | 20260508 13:13:51.766 | INFO | "pinned" : " 29 GB" | 20260508 13:13:51.766 | INFO | } | 20260508 13:13:51.766 | INFO | Perf report for the last 5.14193 seconds: | 20260508 13:13:51.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:51.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.766 | INFO | ANALYSIS : 66.3%, 3.0, 103.5ms, 0 | 20260508 13:13:51.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.767 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:13:51.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:51.767 | INFO | Uploaded 36GiB to GPU in 2.42097s (14.8701GiB/s) | 20260508 13:13:51.767 | INFO | Downloaded 34GiB to GPU in 2.50573s (13.5689GiB/s) | 20260508 13:13:56.906 | INFO | Analyzing chunk frames = [8378880,8379392) | 20260508 13:13:56.906 | INFO | MemInfo: { | 20260508 13:13:56.906 | INFO | "gpu" : " 17 GB", | 20260508 13:13:56.906 | INFO | "huge" : "", | 20260508 13:13:56.907 | INFO | "huge_shared" : " 29 GB", | 20260508 13:13:56.907 | INFO | "malloc" : " 14 GB", | 20260508 13:13:56.907 | INFO | "pinned" : " 29 GB" | 20260508 13:13:56.907 | INFO | } | 20260508 13:13:56.907 | INFO | Perf report for the last 5.1403 seconds: | 20260508 13:13:56.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:13:56.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 13:13:56.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:13:56.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:13:56.907 | INFO | Uploaded 36GiB to GPU in 2.43768s (14.7682GiB/s) | 20260508 13:13:56.907 | INFO | Downloaded 34GiB to GPU in 2.43759s (13.9482GiB/s) | 20260508 13:14:02.038 | INFO | Analyzing chunk frames = [8379392,8379904) | 20260508 13:14:02.038 | INFO | MemInfo: { | 20260508 13:14:02.038 | INFO | "gpu" : " 17 GB", | 20260508 13:14:02.038 | INFO | "huge" : "", | 20260508 13:14:02.038 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:02.039 | INFO | "malloc" : " 14 GB", | 20260508 13:14:02.039 | INFO | "pinned" : " 29 GB" | 20260508 13:14:02.039 | INFO | } | 20260508 13:14:02.039 | INFO | Perf report for the last 5.13192 seconds: | 20260508 13:14:02.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:02.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | ANALYSIS : 64.8%, 3.0, 100.6ms, 0 | 20260508 13:14:02.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:14:02.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:02.039 | INFO | Uploaded 36GiB to GPU in 2.3595s (15.2575GiB/s) | 20260508 13:14:02.039 | INFO | Downloaded 34GiB to GPU in 2.39943s (14.17GiB/s) | 20260508 13:14:07.189 | INFO | Analyzing chunk frames = [8379904,8380416) | 20260508 13:14:07.189 | INFO | MemInfo: { | 20260508 13:14:07.189 | INFO | "gpu" : " 17 GB", | 20260508 13:14:07.189 | INFO | "huge" : "", | 20260508 13:14:07.189 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:07.189 | INFO | "malloc" : " 14 GB", | 20260508 13:14:07.189 | INFO | "pinned" : " 29 GB" | 20260508 13:14:07.189 | INFO | } | 20260508 13:14:07.189 | INFO | Perf report for the last 5.15029 seconds: | 20260508 13:14:07.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:07.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | ANALYSIS : 65.6%, 3.0, 102.8ms, 0 | 20260508 13:14:07.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:14:07.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:07.189 | INFO | Uploaded 36GiB to GPU in 2.49399s (14.4347GiB/s) | 20260508 13:14:07.189 | INFO | Downloaded 34GiB to GPU in 2.45453s (13.8519GiB/s) | 20260508 13:14:07.189 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:14:07.189 | INFO | Total mem usage: 16.7 GiB | 20260508 13:14:07.189 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:14:07.189 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:14:07.189 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:14:07.189 | INFO | High Water Breakdown: | 20260508 13:14:07.189 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:14:07.189 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:14:07.189 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:14:07.189 | INFO | ------------------------------------------ | 20260508 13:14:07.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:14:07.189 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:14:07.189 | INFO | Total mem usage: 26.5 GiB | 20260508 13:14:07.189 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:14:07.189 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:14:07.189 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:14:07.189 | INFO | High Water Breakdown: | 20260508 13:14:07.189 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:14:07.189 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:14:07.190 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:14:07.190 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:14:07.190 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:14:07.190 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:14:07.190 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:14:07.190 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:14:07.190 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:14:07.190 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:14:07.190 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:14:07.190 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:14:07.190 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:14:07.190 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:14:07.190 | INFO | ------------------------------------------ | 20260508 13:14:07.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:14:07.190 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:14:07.190 | INFO | Total mem usage: 27.8 GiB | 20260508 13:14:07.190 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:14:07.190 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:14:07.190 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:14:07.190 | INFO | High Water Breakdown: | 20260508 13:14:07.190 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:14:07.190 | INFO | ------------------------------------------ | 20260508 13:14:07.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:14:07.190 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:14:07.190 | INFO | Total mem usage: 15.7 GiB | 20260508 13:14:07.190 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:14:07.190 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:14:07.190 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:14:07.190 | INFO | High Water Breakdown: | 20260508 13:14:07.190 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:14:07.190 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:14:07.190 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:14:07.190 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:14:07.190 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:14:07.190 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:14:07.190 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:14:07.190 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:14:07.190 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:14:07.190 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:14:07.190 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:14:07.190 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:14:07.190 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:14:07.190 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:14:07.190 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:14:07.190 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:14:07.190 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:14:07.190 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:14:07.190 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:14:07.190 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:14:07.190 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:14:07.190 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:14:07.190 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:14:07.190 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:14:07.190 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:14:07.190 | INFO | ------------------------------------------ | 20260508 13:14:07.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:14:07.190 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:14:07.190 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:14:07.190 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:14:07.190 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.54e+01, 1.41e+01 / 5.76e-01 | 20260508 13:14:07.190 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.08e-02 / 4.99e-03 | 20260508 13:14:07.190 | INFO | PerfReport: 12.18%, Baseline -- 4.00e+00 / 4.49e+00, 6.01e-01 / 1.81e-01 | 20260508 13:14:07.190 | INFO | PerfReport: 3.40%, Histogram -- 1.12e+00 / 1.49e+00, 2.12e-01 / 1.00e-01 | 20260508 13:14:07.190 | INFO | PerfReport: 15.96%, DME -- 4.19e+01 / 4.54e+01, 3.68e+00 / 1.06e+00 | 20260508 13:14:07.190 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.88e+01 / 2.01e+01, 1.96e+00 / 1.26e+00 | 20260508 13:14:07.190 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.29e+00 / 2.42e+00, 4.01e-01 / 2.33e-01 | 20260508 13:14:07.190 | INFO | PerfReport: 2.92%, Metrics -- 9.57e-01 / 1.08e+00, 2.40e-01 / 1.15e-01 | 20260508 13:14:07.190 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.91e-01 / 3.24e-01, 4.72e-02 / 3.05e-02 | 20260508 13:14:07.190 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:14:07.190 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:14:07.190 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:14:07.190 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:14:07.190 | INFO | PerfReport: 24.33%, Upload -- 2.55e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260508 13:14:07.190 | INFO | PerfReport: 24.83%, Download -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 13:14:07.190 | INFO | PerfReport: 31.31%, Compute -- 3.28e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 13:14:07.190 | INFO | PerfReport: 2.08%, LockUpload -- 2.18e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 13:14:07.191 | INFO | PerfReport: 2.53%, LockDownload -- 2.65e+00 / 0.00e+00, 4.07e+00 / 0.00e+00 | 20260508 13:14:07.191 | INFO | PerfReport: 14.90%, LockCompute -- 1.56e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:14:07.191 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.22e-02 / 0.00e+00 | 20260508 13:14:07.286 | INFO | Starting a round of Baz IO | 20260508 13:14:10.692 | INFO | Populating RTMetrics report | 20260508 13:14:10.693 | INFO | RTMetrics report populated | 20260508 13:14:10.693 | INFO | Opening tmp RTMetrics json file | 20260508 13:14:10.703 | INFO | Serializing RTMetrics to json | 20260508 13:14:10.704 | INFO | Serialization done, swapping tmp file now... | 20260508 13:14:10.704 | INFO | RTMetrics done | 20260508 13:14:10.763 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.477 seconds | 20260508 13:14:10.763 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:14:10.763 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146353% / 0% / 5.96046e-05% | 20260508 13:14:10.763 | INFO | Baz IO overall: | 20260508 13:14:10.763 | INFO | BytesWritten : 3576201337883 | 20260508 13:14:10.763 | INFO | NumEvents : 1714980028427 | 20260508 13:14:12.328 | INFO | Analyzing chunk frames = [8380416,8380928) | 20260508 13:14:12.328 | INFO | MemInfo: { | 20260508 13:14:12.328 | INFO | "gpu" : " 17 GB", | 20260508 13:14:12.328 | INFO | "huge" : "", | 20260508 13:14:12.328 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:12.328 | INFO | "malloc" : " 14 GB", | 20260508 13:14:12.328 | INFO | "pinned" : " 29 GB" | 20260508 13:14:12.328 | INFO | } | 20260508 13:14:12.328 | INFO | Perf report for the last 5.1394 seconds: | 20260508 13:14:12.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:12.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:12.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:12.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:12.329 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 13:14:12.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:12.329 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:14:12.329 | INFO | PRE_HQ : 65.4%, 1.6, 54.0ms, 0 | 20260508 13:14:12.329 | INFO | BAZWRITER : 66.9%, 2.5, 84.8ms, 0 | 20260508 13:14:12.329 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:14:12.330 | INFO | Uploaded 36GiB to GPU in 2.64582s (13.6064GiB/s) | 20260508 13:14:12.330 | INFO | Downloaded 38GiB to GPU in 2.72187s (13.961GiB/s) | 20260508 13:14:17.465 | INFO | Analyzing chunk frames = [8380928,8381440) | 20260508 13:14:17.465 | INFO | MemInfo: { | 20260508 13:14:17.465 | INFO | "gpu" : " 17 GB", | 20260508 13:14:17.465 | INFO | "huge" : "", | 20260508 13:14:17.465 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:17.465 | INFO | "malloc" : " 14 GB", | 20260508 13:14:17.465 | INFO | "pinned" : " 29 GB" | 20260508 13:14:17.465 | INFO | } | 20260508 13:14:17.466 | INFO | Perf report for the last 5.13709 seconds: | 20260508 13:14:17.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:17.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 13:14:17.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260508 13:14:17.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:17.466 | INFO | Uploaded 36GiB to GPU in 2.49503s (14.4287GiB/s) | 20260508 13:14:17.466 | INFO | Downloaded 34GiB to GPU in 2.49299s (13.6383GiB/s) | 20260508 13:14:22.600 | INFO | Analyzing chunk frames = [8381440,8381952) | 20260508 13:14:22.600 | INFO | MemInfo: { | 20260508 13:14:22.600 | INFO | "gpu" : " 17 GB", | 20260508 13:14:22.600 | INFO | "huge" : "", | 20260508 13:14:22.600 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:22.600 | INFO | "malloc" : " 14 GB", | 20260508 13:14:22.600 | INFO | "pinned" : " 29 GB" | 20260508 13:14:22.600 | INFO | } | 20260508 13:14:22.600 | INFO | Perf report for the last 5.13453 seconds: | 20260508 13:14:22.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:22.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | ANALYSIS : 65.6%, 3.0, 102.0ms, 0 | 20260508 13:14:22.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260508 13:14:22.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:22.600 | INFO | Uploaded 36GiB to GPU in 2.42205s (14.8634GiB/s) | 20260508 13:14:22.601 | INFO | Downloaded 34GiB to GPU in 2.46673s (13.7834GiB/s) | 20260508 13:14:27.741 | INFO | Analyzing chunk frames = [8381952,8382464) | 20260508 13:14:27.741 | INFO | MemInfo: { | 20260508 13:14:27.741 | INFO | "gpu" : " 17 GB", | 20260508 13:14:27.741 | INFO | "huge" : "", | 20260508 13:14:27.741 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:27.741 | INFO | "malloc" : " 14 GB", | 20260508 13:14:27.741 | INFO | "pinned" : " 29 GB" | 20260508 13:14:27.741 | INFO | } | 20260508 13:14:27.742 | INFO | Perf report for the last 5.14152 seconds: | 20260508 13:14:27.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:27.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:14:27.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260508 13:14:27.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:27.742 | INFO | Uploaded 36GiB to GPU in 2.41478s (14.9082GiB/s) | 20260508 13:14:27.742 | INFO | Downloaded 34GiB to GPU in 2.48956s (13.657GiB/s) | 20260508 13:14:32.878 | INFO | Analyzing chunk frames = [8382464,8382976) | 20260508 13:14:32.878 | INFO | MemInfo: { | 20260508 13:14:32.878 | INFO | "gpu" : " 17 GB", | 20260508 13:14:32.878 | INFO | "huge" : "", | 20260508 13:14:32.878 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:32.878 | INFO | "malloc" : " 14 GB", | 20260508 13:14:32.878 | INFO | "pinned" : " 29 GB" | 20260508 13:14:32.878 | INFO | } | 20260508 13:14:32.878 | INFO | Perf report for the last 5.13674 seconds: | 20260508 13:14:32.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:32.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.878 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:14:32.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.878 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 13:14:32.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:32.879 | INFO | Uploaded 36GiB to GPU in 2.32827s (15.4621GiB/s) | 20260508 13:14:32.879 | INFO | Downloaded 34GiB to GPU in 2.51799s (13.5028GiB/s) | 20260508 13:14:38.016 | INFO | Analyzing chunk frames = [8382976,8383488) | 20260508 13:14:38.016 | INFO | MemInfo: { | 20260508 13:14:38.016 | INFO | "gpu" : " 17 GB", | 20260508 13:14:38.016 | INFO | "huge" : "", | 20260508 13:14:38.016 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:38.016 | INFO | "malloc" : " 14 GB", | 20260508 13:14:38.016 | INFO | "pinned" : " 29 GB" | 20260508 13:14:38.016 | INFO | } | 20260508 13:14:38.016 | INFO | Perf report for the last 5.13801 seconds: | 20260508 13:14:38.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:38.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.016 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:14:38.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.016 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:14:38.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:38.017 | INFO | Uploaded 36GiB to GPU in 2.3795s (15.1292GiB/s) | 20260508 13:14:38.017 | INFO | Downloaded 34GiB to GPU in 2.45893s (13.8272GiB/s) | 20260508 13:14:43.156 | INFO | Analyzing chunk frames = [8383488,8384000) | 20260508 13:14:43.156 | INFO | MemInfo: { | 20260508 13:14:43.156 | INFO | "gpu" : " 17 GB", | 20260508 13:14:43.156 | INFO | "huge" : "", | 20260508 13:14:43.156 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:43.156 | INFO | "malloc" : " 14 GB", | 20260508 13:14:43.156 | INFO | "pinned" : " 29 GB" | 20260508 13:14:43.157 | INFO | } | 20260508 13:14:43.157 | INFO | Perf report for the last 5.14022 seconds: | 20260508 13:14:43.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:43.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:14:43.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:14:43.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:43.157 | INFO | Uploaded 36GiB to GPU in 2.43377s (14.7919GiB/s) | 20260508 13:14:43.157 | INFO | Downloaded 34GiB to GPU in 2.4742s (13.7418GiB/s) | 20260508 13:14:48.291 | INFO | Analyzing chunk frames = [8384000,8384512) | 20260508 13:14:48.291 | INFO | MemInfo: { | 20260508 13:14:48.291 | INFO | "gpu" : " 17 GB", | 20260508 13:14:48.291 | INFO | "huge" : "", | 20260508 13:14:48.291 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:48.291 | INFO | "malloc" : " 14 GB", | 20260508 13:14:48.291 | INFO | "pinned" : " 29 GB" | 20260508 13:14:48.291 | INFO | } | 20260508 13:14:48.291 | INFO | Perf report for the last 5.13464 seconds: | 20260508 13:14:48.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:48.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.291 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 13:14:48.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.292 | INFO | PRE_HQ : 23.4%, 1.0, 12.2ms, 0 | 20260508 13:14:48.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:48.292 | INFO | Uploaded 36GiB to GPU in 2.38721s (15.0804GiB/s) | 20260508 13:14:48.292 | INFO | Downloaded 34GiB to GPU in 2.44273s (13.9189GiB/s) | 20260508 13:14:48.389 | INFO | Starting a round of Baz IO | 20260508 13:14:51.793 | INFO | Populating RTMetrics report | 20260508 13:14:51.793 | INFO | RTMetrics report populated | 20260508 13:14:51.793 | INFO | Opening tmp RTMetrics json file | 20260508 13:14:51.800 | INFO | Serializing RTMetrics to json | 20260508 13:14:51.801 | INFO | Serialization done, swapping tmp file now... | 20260508 13:14:51.801 | INFO | RTMetrics done | 20260508 13:14:51.862 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.474 seconds | 20260508 13:14:51.863 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:14:51.863 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145737% / 3.97364e-06% / 5.96046e-05% | 20260508 13:14:51.863 | INFO | Baz IO overall: | 20260508 13:14:51.863 | INFO | BytesWritten : 3577694552645 | 20260508 13:14:51.863 | INFO | NumEvents : 1715678933411 | 20260508 13:14:53.438 | INFO | Analyzing chunk frames = [8384512,8385024) | 20260508 13:14:53.438 | INFO | MemInfo: { | 20260508 13:14:53.438 | INFO | "gpu" : " 17 GB", | 20260508 13:14:53.438 | INFO | "huge" : "", | 20260508 13:14:53.438 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:53.438 | INFO | "malloc" : " 14 GB", | 20260508 13:14:53.438 | INFO | "pinned" : " 29 GB" | 20260508 13:14:53.439 | INFO | } | 20260508 13:14:53.439 | INFO | Perf report for the last 5.14739 seconds: | 20260508 13:14:53.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:53.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:53.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:53.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:53.439 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:14:53.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:53.439 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:14:53.439 | INFO | PRE_HQ : 65.4%, 1.6, 53.9ms, 0 | 20260508 13:14:53.439 | INFO | BAZWRITER : 66.6%, 2.5, 84.3ms, 0 | 20260508 13:14:53.439 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:14:53.439 | INFO | Uploaded 36GiB to GPU in 2.62228s (13.7285GiB/s) | 20260508 13:14:53.439 | INFO | Downloaded 38GiB to GPU in 2.71298s (14.0067GiB/s) | 20260508 13:14:58.579 | INFO | Analyzing chunk frames = [8385024,8385536) | 20260508 13:14:58.579 | INFO | MemInfo: { | 20260508 13:14:58.579 | INFO | "gpu" : " 17 GB", | 20260508 13:14:58.579 | INFO | "huge" : "", | 20260508 13:14:58.579 | INFO | "huge_shared" : " 29 GB", | 20260508 13:14:58.579 | INFO | "malloc" : " 14 GB", | 20260508 13:14:58.579 | INFO | "pinned" : " 29 GB" | 20260508 13:14:58.579 | INFO | } | 20260508 13:14:58.579 | INFO | Perf report for the last 5.14067 seconds: | 20260508 13:14:58.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:14:58.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.579 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:14:58.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.579 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:14:58.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:14:58.580 | INFO | Uploaded 36GiB to GPU in 2.45438s (14.6676GiB/s) | 20260508 13:14:58.580 | INFO | Downloaded 34GiB to GPU in 2.46339s (13.8021GiB/s) | 20260508 13:15:03.717 | INFO | Analyzing chunk frames = [8385536,8386048) | 20260508 13:15:03.718 | INFO | MemInfo: { | 20260508 13:15:03.718 | INFO | "gpu" : " 17 GB", | 20260508 13:15:03.718 | INFO | "huge" : "", | 20260508 13:15:03.718 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:03.718 | INFO | "malloc" : " 14 GB", | 20260508 13:15:03.718 | INFO | "pinned" : " 29 GB" | 20260508 13:15:03.718 | INFO | } | 20260508 13:15:03.718 | INFO | Perf report for the last 5.13846 seconds: | 20260508 13:15:03.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:03.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 13:15:03.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:15:03.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:03.718 | INFO | Uploaded 36GiB to GPU in 2.44349s (14.733GiB/s) | 20260508 13:15:03.718 | INFO | Downloaded 34GiB to GPU in 2.45864s (13.8288GiB/s) | 20260508 13:15:08.853 | INFO | Analyzing chunk frames = [8386048,8386560) | 20260508 13:15:08.853 | INFO | MemInfo: { | 20260508 13:15:08.853 | INFO | "gpu" : " 17 GB", | 20260508 13:15:08.854 | INFO | "huge" : "", | 20260508 13:15:08.854 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:08.854 | INFO | "malloc" : " 14 GB", | 20260508 13:15:08.854 | INFO | "pinned" : " 29 GB" | 20260508 13:15:08.854 | INFO | } | 20260508 13:15:08.854 | INFO | Perf report for the last 5.1359 seconds: | 20260508 13:15:08.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:08.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 13:15:08.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:15:08.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:08.854 | INFO | Uploaded 36GiB to GPU in 2.5279s (14.2411GiB/s) | 20260508 13:15:08.855 | INFO | Downloaded 34GiB to GPU in 2.44514s (13.9051GiB/s) | 20260508 13:15:13.995 | INFO | Analyzing chunk frames = [8386560,8387072) | 20260508 13:15:13.995 | INFO | MemInfo: { | 20260508 13:15:13.995 | INFO | "gpu" : " 17 GB", | 20260508 13:15:13.995 | INFO | "huge" : "", | 20260508 13:15:13.995 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:13.995 | INFO | "malloc" : " 14 GB", | 20260508 13:15:13.995 | INFO | "pinned" : " 29 GB" | 20260508 13:15:13.995 | INFO | } | 20260508 13:15:13.995 | INFO | Perf report for the last 5.14136 seconds: | 20260508 13:15:13.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:13.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:15:13.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:15:13.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:13.995 | INFO | Uploaded 36GiB to GPU in 2.29492s (15.6869GiB/s) | 20260508 13:15:13.995 | INFO | Downloaded 34GiB to GPU in 2.49205s (13.6434GiB/s) | 20260508 13:15:19.134 | INFO | Analyzing chunk frames = [8387072,8387584) | 20260508 13:15:19.134 | INFO | MemInfo: { | 20260508 13:15:19.134 | INFO | "gpu" : " 17 GB", | 20260508 13:15:19.134 | INFO | "huge" : "", | 20260508 13:15:19.134 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:19.134 | INFO | "malloc" : " 14 GB", | 20260508 13:15:19.134 | INFO | "pinned" : " 29 GB" | 20260508 13:15:19.134 | INFO | } | 20260508 13:15:19.134 | INFO | Perf report for the last 5.13905 seconds: | 20260508 13:15:19.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:19.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.134 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:15:19.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.134 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:15:19.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:19.135 | INFO | Uploaded 36GiB to GPU in 2.45852s (14.643GiB/s) | 20260508 13:15:19.135 | INFO | Downloaded 34GiB to GPU in 2.43709s (13.951GiB/s) | 20260508 13:15:24.277 | INFO | Analyzing chunk frames = [8387584,8388096) | 20260508 13:15:24.277 | INFO | MemInfo: { | 20260508 13:15:24.277 | INFO | "gpu" : " 17 GB", | 20260508 13:15:24.277 | INFO | "huge" : "", | 20260508 13:15:24.277 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:24.277 | INFO | "malloc" : " 14 GB", | 20260508 13:15:24.277 | INFO | "pinned" : " 29 GB" | 20260508 13:15:24.277 | INFO | } | 20260508 13:15:24.278 | INFO | Perf report for the last 5.1436 seconds: | 20260508 13:15:24.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:24.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 13:15:24.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:15:24.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:24.278 | INFO | Uploaded 36GiB to GPU in 2.49409s (14.4341GiB/s) | 20260508 13:15:24.278 | INFO | Downloaded 34GiB to GPU in 2.44902s (13.8831GiB/s) | 20260508 13:15:29.414 | INFO | Analyzing chunk frames = [8388096,8388608) | 20260508 13:15:29.414 | INFO | MemInfo: { | 20260508 13:15:29.414 | INFO | "gpu" : " 17 GB", | 20260508 13:15:29.414 | INFO | "huge" : "", | 20260508 13:15:29.414 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:29.414 | INFO | "malloc" : " 14 GB", | 20260508 13:15:29.414 | INFO | "pinned" : " 29 GB" | 20260508 13:15:29.414 | INFO | } | 20260508 13:15:29.414 | INFO | Perf report for the last 5.13645 seconds: | 20260508 13:15:29.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:29.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | ANALYSIS : 64.9%, 3.0, 101.3ms, 0 | 20260508 13:15:29.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:15:29.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:29.414 | INFO | Uploaded 36GiB to GPU in 2.38894s (15.0694GiB/s) | 20260508 13:15:29.415 | INFO | Downloaded 34GiB to GPU in 2.42415s (14.0255GiB/s) | 20260508 13:15:29.415 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:15:29.415 | INFO | Total mem usage: 16.7 GiB | 20260508 13:15:29.415 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:15:29.415 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:15:29.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:15:29.415 | INFO | High Water Breakdown: | 20260508 13:15:29.415 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:15:29.415 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:15:29.415 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:15:29.415 | INFO | ------------------------------------------ | 20260508 13:15:29.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:15:29.415 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:15:29.415 | INFO | Total mem usage: 26.5 GiB | 20260508 13:15:29.415 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:15:29.415 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:15:29.415 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:15:29.415 | INFO | High Water Breakdown: | 20260508 13:15:29.415 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:15:29.415 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:15:29.415 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:15:29.415 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:15:29.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:15:29.415 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:15:29.415 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:15:29.415 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:15:29.415 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:15:29.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:15:29.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:15:29.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:15:29.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:15:29.415 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:15:29.415 | INFO | ------------------------------------------ | 20260508 13:15:29.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:15:29.415 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:15:29.415 | INFO | Total mem usage: 27.8 GiB | 20260508 13:15:29.415 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:15:29.415 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:15:29.415 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:15:29.415 | INFO | High Water Breakdown: | 20260508 13:15:29.415 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:15:29.415 | INFO | ------------------------------------------ | 20260508 13:15:29.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:15:29.415 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:15:29.415 | INFO | Total mem usage: 15.7 GiB | 20260508 13:15:29.415 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:15:29.415 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:15:29.415 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:15:29.415 | INFO | High Water Breakdown: | 20260508 13:15:29.415 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:15:29.415 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:15:29.415 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:15:29.415 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:15:29.415 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:15:29.415 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:15:29.415 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:15:29.415 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:15:29.415 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:15:29.416 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:15:29.416 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:15:29.416 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:15:29.416 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:15:29.416 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:15:29.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:15:29.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:15:29.416 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:15:29.416 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:15:29.416 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:15:29.416 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:15:29.416 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:15:29.416 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:15:29.416 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:15:29.416 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:15:29.416 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:15:29.416 | INFO | ------------------------------------------ | 20260508 13:15:29.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:15:29.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:15:29.416 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:15:29.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:15:29.416 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.38e+01 / 6.25e-01 | 20260508 13:15:29.416 | INFO | PerfReport: 0.30%, AggregateStats -- 9.83e-02 / 1.05e-01, 1.22e-02 / 6.78e-03 | 20260508 13:15:29.416 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.34e+00, 5.96e-01 / 2.38e-01 | 20260508 13:15:29.416 | INFO | PerfReport: 3.39%, Histogram -- 1.10e+00 / 1.44e+00, 2.10e-01 / 1.10e-01 | 20260508 13:15:29.416 | INFO | PerfReport: 16.06%, DME -- 4.19e+01 / 4.60e+01, 3.69e+00 / 1.04e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.86e+01 / 1.96e+01, 1.76e+00 / 1.08e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.32e+00 / 2.51e+00, 4.05e-01 / 2.17e-01 | 20260508 13:15:29.416 | INFO | PerfReport: 2.94%, Metrics -- 9.58e-01 / 1.07e+00, 2.37e-01 / 8.74e-02 | 20260508 13:15:29.416 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.22e-01, 4.68e-02 / 1.62e-02 | 20260508 13:15:29.416 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:15:29.416 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:15:29.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:15:29.416 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 24.35%, Upload -- 2.55e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 24.82%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 31.13%, Compute -- 3.26e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 2.09%, LockUpload -- 2.19e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 2.67%, LockDownload -- 2.80e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 14.92%, LockCompute -- 1.56e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:15:29.416 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260508 13:15:29.511 | INFO | Starting a round of Baz IO | 20260508 13:15:32.919 | INFO | Populating RTMetrics report | 20260508 13:15:32.919 | INFO | RTMetrics report populated | 20260508 13:15:32.919 | INFO | Opening tmp RTMetrics json file | 20260508 13:15:32.936 | INFO | Serializing RTMetrics to json | 20260508 13:15:32.937 | INFO | Serialization done, swapping tmp file now... | 20260508 13:15:32.937 | INFO | RTMetrics done | 20260508 13:15:32.988 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.476 seconds | 20260508 13:15:32.988 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:15:32.988 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14567% / 0% / 5.5631e-05% | 20260508 13:15:32.988 | INFO | Baz IO overall: | 20260508 13:15:32.988 | INFO | BytesWritten : 3579188312407 | 20260508 13:15:32.988 | INFO | NumEvents : 1716378448444 | 20260508 13:15:34.547 | INFO | Analyzing chunk frames = [8388608,8389120) | 20260508 13:15:34.547 | INFO | MemInfo: { | 20260508 13:15:34.547 | INFO | "gpu" : " 17 GB", | 20260508 13:15:34.547 | INFO | "huge" : "", | 20260508 13:15:34.548 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:34.548 | INFO | "malloc" : " 14 GB", | 20260508 13:15:34.548 | INFO | "pinned" : " 29 GB" | 20260508 13:15:34.548 | INFO | } | 20260508 13:15:34.548 | INFO | Perf report for the last 5.13335 seconds: | 20260508 13:15:34.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:34.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:34.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:34.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:34.548 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:15:34.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:34.548 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 13:15:34.548 | INFO | PRE_HQ : 65.3%, 1.6, 53.6ms, 0 | 20260508 13:15:34.548 | INFO | BAZWRITER : 67.0%, 2.8, 95.8ms, 0 | 20260508 13:15:34.548 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:15:34.548 | INFO | Uploaded 36GiB to GPU in 2.637s (13.6519GiB/s) | 20260508 13:15:34.548 | INFO | Downloaded 38GiB to GPU in 2.71013s (14.0215GiB/s) | 20260508 13:15:39.692 | INFO | Analyzing chunk frames = [8389120,8389632) | 20260508 13:15:39.692 | INFO | MemInfo: { | 20260508 13:15:39.692 | INFO | "gpu" : " 17 GB", | 20260508 13:15:39.692 | INFO | "huge" : "", | 20260508 13:15:39.692 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:39.692 | INFO | "malloc" : " 14 GB", | 20260508 13:15:39.692 | INFO | "pinned" : " 29 GB" | 20260508 13:15:39.692 | INFO | } | 20260508 13:15:39.693 | INFO | Perf report for the last 5.14495 seconds: | 20260508 13:15:39.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:39.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 13:15:39.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260508 13:15:39.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:39.693 | INFO | Uploaded 36GiB to GPU in 2.44134s (14.746GiB/s) | 20260508 13:15:39.693 | INFO | Downloaded 34GiB to GPU in 2.47679s (13.7275GiB/s) | 20260508 13:15:44.831 | INFO | Analyzing chunk frames = [8389632,8390144) | 20260508 13:15:44.831 | INFO | MemInfo: { | 20260508 13:15:44.831 | INFO | "gpu" : " 17 GB", | 20260508 13:15:44.831 | INFO | "huge" : "", | 20260508 13:15:44.831 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:44.831 | INFO | "malloc" : " 14 GB", | 20260508 13:15:44.831 | INFO | "pinned" : " 29 GB" | 20260508 13:15:44.831 | INFO | } | 20260508 13:15:44.832 | INFO | Perf report for the last 5.13914 seconds: | 20260508 13:15:44.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:44.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 13:15:44.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:15:44.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:44.832 | INFO | Uploaded 36GiB to GPU in 2.47324s (14.5558GiB/s) | 20260508 13:15:44.832 | INFO | Downloaded 34GiB to GPU in 2.46127s (13.814GiB/s) | 20260508 13:15:49.968 | INFO | Analyzing chunk frames = [8390144,8390656) | 20260508 13:15:49.968 | INFO | MemInfo: { | 20260508 13:15:49.968 | INFO | "gpu" : " 17 GB", | 20260508 13:15:49.968 | INFO | "huge" : "", | 20260508 13:15:49.969 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:49.969 | INFO | "malloc" : " 14 GB", | 20260508 13:15:49.969 | INFO | "pinned" : " 29 GB" | 20260508 13:15:49.969 | INFO | } | 20260508 13:15:49.969 | INFO | Perf report for the last 5.13699 seconds: | 20260508 13:15:49.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:49.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 13:15:49.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:15:49.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:49.969 | INFO | Uploaded 36GiB to GPU in 2.50439s (14.3748GiB/s) | 20260508 13:15:49.969 | INFO | Downloaded 34GiB to GPU in 2.46259s (13.8066GiB/s) | 20260508 13:15:55.105 | INFO | Analyzing chunk frames = [8390656,8391168) | 20260508 13:15:55.105 | INFO | MemInfo: { | 20260508 13:15:55.105 | INFO | "gpu" : " 17 GB", | 20260508 13:15:55.105 | INFO | "huge" : "", | 20260508 13:15:55.105 | INFO | "huge_shared" : " 29 GB", | 20260508 13:15:55.105 | INFO | "malloc" : " 14 GB", | 20260508 13:15:55.105 | INFO | "pinned" : " 29 GB" | 20260508 13:15:55.105 | INFO | } | 20260508 13:15:55.105 | INFO | Perf report for the last 5.13655 seconds: | 20260508 13:15:55.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:15:55.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 13:15:55.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:15:55.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:15:55.105 | INFO | Uploaded 36GiB to GPU in 2.40965s (14.9399GiB/s) | 20260508 13:15:55.106 | INFO | Downloaded 34GiB to GPU in 2.51642s (13.5112GiB/s) | 20260508 13:16:00.247 | INFO | Analyzing chunk frames = [8391168,8391680) | 20260508 13:16:00.247 | INFO | MemInfo: { | 20260508 13:16:00.247 | INFO | "gpu" : " 17 GB", | 20260508 13:16:00.247 | INFO | "huge" : "", | 20260508 13:16:00.247 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:00.247 | INFO | "malloc" : " 14 GB", | 20260508 13:16:00.248 | INFO | "pinned" : " 29 GB" | 20260508 13:16:00.248 | INFO | } | 20260508 13:16:00.248 | INFO | Perf report for the last 5.14238 seconds: | 20260508 13:16:00.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:00.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 13:16:00.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:16:00.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:00.248 | INFO | Uploaded 36GiB to GPU in 2.38825s (15.0738GiB/s) | 20260508 13:16:00.248 | INFO | Downloaded 34GiB to GPU in 2.44093s (13.9291GiB/s) | 20260508 13:16:05.394 | INFO | Analyzing chunk frames = [8391680,8392192) | 20260508 13:16:05.394 | INFO | MemInfo: { | 20260508 13:16:05.394 | INFO | "gpu" : " 17 GB", | 20260508 13:16:05.394 | INFO | "huge" : "", | 20260508 13:16:05.394 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:05.394 | INFO | "malloc" : " 14 GB", | 20260508 13:16:05.394 | INFO | "pinned" : " 29 GB" | 20260508 13:16:05.394 | INFO | } | 20260508 13:16:05.394 | INFO | Perf report for the last 5.14682 seconds: | 20260508 13:16:05.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:05.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.395 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 13:16:05.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.395 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:16:05.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:05.395 | INFO | Uploaded 36GiB to GPU in 2.47018s (14.5738GiB/s) | 20260508 13:16:05.395 | INFO | Downloaded 34GiB to GPU in 2.46553s (13.7901GiB/s) | 20260508 13:16:10.534 | INFO | Analyzing chunk frames = [8392192,8392704) | 20260508 13:16:10.534 | INFO | MemInfo: { | 20260508 13:16:10.534 | INFO | "gpu" : " 17 GB", | 20260508 13:16:10.534 | INFO | "huge" : "", | 20260508 13:16:10.534 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:10.534 | INFO | "malloc" : " 14 GB", | 20260508 13:16:10.534 | INFO | "pinned" : " 29 GB" | 20260508 13:16:10.534 | INFO | } | 20260508 13:16:10.534 | INFO | Perf report for the last 5.13981 seconds: | 20260508 13:16:10.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:10.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.534 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260508 13:16:10.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.534 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:16:10.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:10.535 | INFO | Uploaded 36GiB to GPU in 2.2966s (15.6754GiB/s) | 20260508 13:16:10.535 | INFO | Downloaded 34GiB to GPU in 2.43421s (13.9676GiB/s) | 20260508 13:16:10.632 | INFO | Starting a round of Baz IO | 20260508 13:16:14.046 | INFO | Populating RTMetrics report | 20260508 13:16:14.047 | INFO | RTMetrics report populated | 20260508 13:16:14.047 | INFO | Opening tmp RTMetrics json file | 20260508 13:16:14.051 | INFO | Serializing RTMetrics to json | 20260508 13:16:14.053 | INFO | Serialization done, swapping tmp file now... | 20260508 13:16:14.054 | INFO | RTMetrics done | 20260508 13:16:14.108 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.476 seconds | 20260508 13:16:14.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:16:14.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145876% / 3.97364e-06% / 3.97364e-05% | 20260508 13:16:14.108 | INFO | Baz IO overall: | 20260508 13:16:14.108 | INFO | BytesWritten : 3580682304390 | 20260508 13:16:14.108 | INFO | NumEvents : 1717078034392 | 20260508 13:16:15.665 | INFO | Analyzing chunk frames = [8392704,8393216) | 20260508 13:16:15.665 | INFO | MemInfo: { | 20260508 13:16:15.665 | INFO | "gpu" : " 17 GB", | 20260508 13:16:15.665 | INFO | "huge" : "", | 20260508 13:16:15.665 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:15.665 | INFO | "malloc" : " 14 GB", | 20260508 13:16:15.665 | INFO | "pinned" : " 29 GB" | 20260508 13:16:15.665 | INFO | } | 20260508 13:16:15.665 | INFO | Perf report for the last 5.13051 seconds: | 20260508 13:16:15.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:15.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:15.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:15.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:15.665 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:16:15.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:15.665 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 13:16:15.665 | INFO | PRE_HQ : 65.7%, 1.7, 56.3ms, 0 | 20260508 13:16:15.665 | INFO | BAZWRITER : 67.2%, 2.5, 86.4ms, 0 | 20260508 13:16:15.665 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:16:15.665 | INFO | Uploaded 36GiB to GPU in 2.64413s (13.6151GiB/s) | 20260508 13:16:15.665 | INFO | Downloaded 38GiB to GPU in 2.71521s (13.9952GiB/s) | 20260508 13:16:20.800 | INFO | Analyzing chunk frames = [8393216,8393728) | 20260508 13:16:20.800 | INFO | MemInfo: { | 20260508 13:16:20.800 | INFO | "gpu" : " 17 GB", | 20260508 13:16:20.800 | INFO | "huge" : "", | 20260508 13:16:20.800 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:20.800 | INFO | "malloc" : " 14 GB", | 20260508 13:16:20.800 | INFO | "pinned" : " 29 GB" | 20260508 13:16:20.800 | INFO | } | 20260508 13:16:20.800 | INFO | Perf report for the last 5.13553 seconds: | 20260508 13:16:20.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:20.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.800 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:16:20.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.800 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:16:20.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:20.801 | INFO | Uploaded 36GiB to GPU in 2.48113s (14.5095GiB/s) | 20260508 13:16:20.801 | INFO | Downloaded 34GiB to GPU in 2.47845s (13.7182GiB/s) | 20260508 13:16:25.944 | INFO | Analyzing chunk frames = [8393728,8394240) | 20260508 13:16:25.944 | INFO | MemInfo: { | 20260508 13:16:25.944 | INFO | "gpu" : " 17 GB", | 20260508 13:16:25.944 | INFO | "huge" : "", | 20260508 13:16:25.944 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:25.944 | INFO | "malloc" : " 14 GB", | 20260508 13:16:25.944 | INFO | "pinned" : " 29 GB" | 20260508 13:16:25.944 | INFO | } | 20260508 13:16:25.944 | INFO | Perf report for the last 5.1439 seconds: | 20260508 13:16:25.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:25.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.944 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:16:25.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.944 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:16:25.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:25.945 | INFO | Uploaded 36GiB to GPU in 2.47873s (14.5236GiB/s) | 20260508 13:16:25.945 | INFO | Downloaded 34GiB to GPU in 2.46093s (13.8159GiB/s) | 20260508 13:16:31.088 | INFO | Analyzing chunk frames = [8394240,8394752) | 20260508 13:16:31.088 | INFO | MemInfo: { | 20260508 13:16:31.088 | INFO | "gpu" : " 17 GB", | 20260508 13:16:31.088 | INFO | "huge" : "", | 20260508 13:16:31.088 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:31.088 | INFO | "malloc" : " 14 GB", | 20260508 13:16:31.088 | INFO | "pinned" : " 29 GB" | 20260508 13:16:31.088 | INFO | } | 20260508 13:16:31.088 | INFO | Perf report for the last 5.14387 seconds: | 20260508 13:16:31.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:31.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.088 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 13:16:31.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.088 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:16:31.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:31.089 | INFO | Uploaded 36GiB to GPU in 2.45963s (14.6364GiB/s) | 20260508 13:16:31.089 | INFO | Downloaded 34GiB to GPU in 2.46089s (13.8161GiB/s) | 20260508 13:16:36.224 | INFO | Analyzing chunk frames = [8394752,8395264) | 20260508 13:16:36.224 | INFO | MemInfo: { | 20260508 13:16:36.224 | INFO | "gpu" : " 17 GB", | 20260508 13:16:36.224 | INFO | "huge" : "", | 20260508 13:16:36.224 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:36.224 | INFO | "malloc" : " 14 GB", | 20260508 13:16:36.224 | INFO | "pinned" : " 29 GB" | 20260508 13:16:36.224 | INFO | } | 20260508 13:16:36.225 | INFO | Perf report for the last 5.13638 seconds: | 20260508 13:16:36.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:36.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | ANALYSIS : 66.0%, 3.0, 103.1ms, 0 | 20260508 13:16:36.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:16:36.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:36.225 | INFO | Uploaded 36GiB to GPU in 2.40748s (14.9534GiB/s) | 20260508 13:16:36.225 | INFO | Downloaded 34GiB to GPU in 2.52868s (13.4458GiB/s) | 20260508 13:16:41.364 | INFO | Analyzing chunk frames = [8395264,8395776) | 20260508 13:16:41.364 | INFO | MemInfo: { | 20260508 13:16:41.364 | INFO | "gpu" : " 17 GB", | 20260508 13:16:41.364 | INFO | "huge" : "", | 20260508 13:16:41.364 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:41.364 | INFO | "malloc" : " 14 GB", | 20260508 13:16:41.364 | INFO | "pinned" : " 29 GB" | 20260508 13:16:41.364 | INFO | } | 20260508 13:16:41.364 | INFO | Perf report for the last 5.13981 seconds: | 20260508 13:16:41.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:41.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.364 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:16:41.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.364 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:16:41.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:41.365 | INFO | Uploaded 36GiB to GPU in 2.4043s (14.9732GiB/s) | 20260508 13:16:41.365 | INFO | Downloaded 34GiB to GPU in 2.44335s (13.9153GiB/s) | 20260508 13:16:46.505 | INFO | Analyzing chunk frames = [8395776,8396288) | 20260508 13:16:46.505 | INFO | MemInfo: { | 20260508 13:16:46.505 | INFO | "gpu" : " 17 GB", | 20260508 13:16:46.505 | INFO | "huge" : "", | 20260508 13:16:46.505 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:46.505 | INFO | "malloc" : " 14 GB", | 20260508 13:16:46.505 | INFO | "pinned" : " 29 GB" | 20260508 13:16:46.505 | INFO | } | 20260508 13:16:46.505 | INFO | Perf report for the last 5.14101 seconds: | 20260508 13:16:46.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:46.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.505 | INFO | ANALYSIS : 64.6%, 3.0, 100.6ms, 0 | 20260508 13:16:46.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.506 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:16:46.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:46.506 | INFO | Uploaded 36GiB to GPU in 2.33725s (15.4027GiB/s) | 20260508 13:16:46.506 | INFO | Downloaded 34GiB to GPU in 2.41813s (14.0605GiB/s) | 20260508 13:16:51.640 | INFO | Analyzing chunk frames = [8396288,8396800) | 20260508 13:16:51.640 | INFO | MemInfo: { | 20260508 13:16:51.640 | INFO | "gpu" : " 17 GB", | 20260508 13:16:51.640 | INFO | "huge" : "", | 20260508 13:16:51.640 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:51.640 | INFO | "malloc" : " 14 GB", | 20260508 13:16:51.640 | INFO | "pinned" : " 29 GB" | 20260508 13:16:51.640 | INFO | } | 20260508 13:16:51.640 | INFO | Perf report for the last 5.1348 seconds: | 20260508 13:16:51.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:51.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260508 13:16:51.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:16:51.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:51.640 | INFO | Uploaded 36GiB to GPU in 2.3719s (15.1777GiB/s) | 20260508 13:16:51.641 | INFO | Downloaded 34GiB to GPU in 2.45094s (13.8722GiB/s) | 20260508 13:16:51.641 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:16:51.641 | INFO | Total mem usage: 16.7 GiB | 20260508 13:16:51.641 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:16:51.641 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -14.1%) | 20260508 13:16:51.641 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:16:51.641 | INFO | High Water Breakdown: | 20260508 13:16:51.641 | INFO | 59.9% : 8.76 GiB ( 85% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:16:51.641 | INFO | 40% : 5.85 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:16:51.641 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:16:51.641 | INFO | ------------------------------------------ | 20260508 13:16:51.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:16:51.641 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:16:51.641 | INFO | Total mem usage: 26.5 GiB | 20260508 13:16:51.641 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:16:51.641 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:16:51.641 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:16:51.641 | INFO | High Water Breakdown: | 20260508 13:16:51.641 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:16:51.641 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:16:51.641 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:16:51.641 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:16:51.641 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:16:51.641 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:16:51.641 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:16:51.641 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:16:51.641 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:16:51.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:16:51.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:16:51.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:16:51.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:16:51.641 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:16:51.641 | INFO | ------------------------------------------ | 20260508 13:16:51.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:16:51.641 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:16:51.641 | INFO | Total mem usage: 27.8 GiB | 20260508 13:16:51.641 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:16:51.641 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:16:51.641 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:16:51.641 | INFO | High Water Breakdown: | 20260508 13:16:51.641 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:16:51.641 | INFO | ------------------------------------------ | 20260508 13:16:51.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:16:51.641 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:16:51.641 | INFO | Total mem usage: 15.7 GiB | 20260508 13:16:51.641 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:16:51.641 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:16:51.641 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:16:51.641 | INFO | High Water Breakdown: | 20260508 13:16:51.641 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:16:51.641 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:16:51.641 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:16:51.641 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:16:51.641 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:16:51.641 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:16:51.641 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:16:51.641 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:16:51.641 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:16:51.641 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:16:51.641 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:16:51.641 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:16:51.641 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:16:51.641 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:16:51.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:16:51.641 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:16:51.641 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:16:51.641 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:16:51.641 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:16:51.641 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:16:51.641 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:16:51.641 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:16:51.641 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:16:51.641 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:16:51.641 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:16:51.642 | INFO | ------------------------------------------ | 20260508 13:16:51.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:16:51.642 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:16:51.642 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:16:51.642 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:16:51.642 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 4.81e-01 | 20260508 13:16:51.642 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.05e-01, 1.10e-02 / 6.53e-03 | 20260508 13:16:51.642 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.40e+00, 5.95e-01 / 2.12e-01 | 20260508 13:16:51.642 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.11e-01 / 1.33e-01 | 20260508 13:16:51.642 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.48e+01, 3.69e+00 / 1.18e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.86e+00 / 1.55e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.33e+00, 4.06e-01 / 3.24e-01 | 20260508 13:16:51.642 | INFO | PerfReport: 2.93%, Metrics -- 9.56e-01 / 1.09e+00, 2.38e-01 / 1.09e-01 | 20260508 13:16:51.642 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.89e-01 / 3.11e-01, 4.55e-02 / 1.94e-02 | 20260508 13:16:51.642 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:16:51.642 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:16:51.642 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:16:51.642 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 24.38%, Upload -- 2.55e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 24.83%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 31.18%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 2.06%, LockUpload -- 2.16e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 2.61%, LockDownload -- 2.74e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 14.92%, LockCompute -- 1.56e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:16:51.642 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260508 13:16:51.737 | INFO | Starting a round of Baz IO | 20260508 13:16:55.140 | INFO | Populating RTMetrics report | 20260508 13:16:55.140 | INFO | RTMetrics report populated | 20260508 13:16:55.140 | INFO | Opening tmp RTMetrics json file | 20260508 13:16:55.149 | INFO | Serializing RTMetrics to json | 20260508 13:16:55.151 | INFO | Serialization done, swapping tmp file now... | 20260508 13:16:55.151 | INFO | RTMetrics done | 20260508 13:16:55.209 | INFO | Finished a round of Baz IO: Wrote 1.39 GiB in 3.471 seconds | 20260508 13:16:55.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:16:55.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145801% / 0% / 5.5631e-05% | 20260508 13:16:55.209 | INFO | Baz IO overall: | 20260508 13:16:55.209 | INFO | BytesWritten : 3582175293526 | 20260508 13:16:55.209 | INFO | NumEvents : 1717777104021 | 20260508 13:16:56.782 | INFO | Analyzing chunk frames = [8396800,8397312) | 20260508 13:16:56.782 | INFO | MemInfo: { | 20260508 13:16:56.782 | INFO | "gpu" : " 17 GB", | 20260508 13:16:56.782 | INFO | "huge" : "", | 20260508 13:16:56.782 | INFO | "huge_shared" : " 29 GB", | 20260508 13:16:56.782 | INFO | "malloc" : " 14 GB", | 20260508 13:16:56.782 | INFO | "pinned" : " 29 GB" | 20260508 13:16:56.782 | INFO | } | 20260508 13:16:56.782 | INFO | Perf report for the last 5.14151 seconds: | 20260508 13:16:56.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:16:56.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:56.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:56.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:56.782 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:16:56.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:16:56.782 | INFO | RT_METRICS : 38.2%, 1.0, 20.1ms, 0 | 20260508 13:16:56.782 | INFO | PRE_HQ : 65.3%, 1.6, 54.0ms, 0 | 20260508 13:16:56.782 | INFO | BAZWRITER : 66.9%, 2.4, 84.2ms, 0 | 20260508 13:16:56.782 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:16:56.782 | INFO | Uploaded 36GiB to GPU in 2.64171s (13.6275GiB/s) | 20260508 13:16:56.782 | INFO | Downloaded 38GiB to GPU in 2.71786s (13.9816GiB/s) | 20260508 13:17:01.918 | INFO | Analyzing chunk frames = [8397312,8397824) | 20260508 13:17:01.918 | INFO | MemInfo: { | 20260508 13:17:01.918 | INFO | "gpu" : " 17 GB", | 20260508 13:17:01.918 | INFO | "huge" : "", | 20260508 13:17:01.918 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:01.918 | INFO | "malloc" : " 14 GB", | 20260508 13:17:01.918 | INFO | "pinned" : " 29 GB" | 20260508 13:17:01.918 | INFO | } | 20260508 13:17:01.918 | INFO | Perf report for the last 5.13643 seconds: | 20260508 13:17:01.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:01.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.918 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:17:01.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.918 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:17:01.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:01.919 | INFO | Uploaded 36GiB to GPU in 2.50399s (14.3771GiB/s) | 20260508 13:17:01.919 | INFO | Downloaded 34GiB to GPU in 2.46358s (13.801GiB/s) | 20260508 13:17:07.056 | INFO | Analyzing chunk frames = [8397824,8398336) | 20260508 13:17:07.056 | INFO | MemInfo: { | 20260508 13:17:07.056 | INFO | "gpu" : " 17 GB", | 20260508 13:17:07.056 | INFO | "huge" : "", | 20260508 13:17:07.056 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:07.056 | INFO | "malloc" : " 14 GB", | 20260508 13:17:07.056 | INFO | "pinned" : " 29 GB" | 20260508 13:17:07.056 | INFO | } | 20260508 13:17:07.057 | INFO | Perf report for the last 5.13835 seconds: | 20260508 13:17:07.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:07.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:17:07.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 13:17:07.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:07.057 | INFO | Uploaded 36GiB to GPU in 2.51752s (14.2998GiB/s) | 20260508 13:17:07.057 | INFO | Downloaded 34GiB to GPU in 2.44941s (13.8809GiB/s) | 20260508 13:17:12.203 | INFO | Analyzing chunk frames = [8398336,8398848) | 20260508 13:17:12.203 | INFO | MemInfo: { | 20260508 13:17:12.203 | INFO | "gpu" : " 17 GB", | 20260508 13:17:12.203 | INFO | "huge" : "", | 20260508 13:17:12.203 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:12.203 | INFO | "malloc" : " 14 GB", | 20260508 13:17:12.203 | INFO | "pinned" : " 29 GB" | 20260508 13:17:12.203 | INFO | } | 20260508 13:17:12.203 | INFO | Perf report for the last 5.1462 seconds: | 20260508 13:17:12.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:12.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:17:12.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:17:12.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:12.203 | INFO | Uploaded 36GiB to GPU in 2.49166s (14.4482GiB/s) | 20260508 13:17:12.203 | INFO | Downloaded 34GiB to GPU in 2.44978s (13.8788GiB/s) | 20260508 13:17:17.339 | INFO | Analyzing chunk frames = [8398848,8399360) | 20260508 13:17:17.339 | INFO | MemInfo: { | 20260508 13:17:17.339 | INFO | "gpu" : " 17 GB", | 20260508 13:17:17.339 | INFO | "huge" : "", | 20260508 13:17:17.339 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:17.339 | INFO | "malloc" : " 14 GB", | 20260508 13:17:17.339 | INFO | "pinned" : " 29 GB" | 20260508 13:17:17.339 | INFO | } | 20260508 13:17:17.339 | INFO | Perf report for the last 5.1364 seconds: | 20260508 13:17:17.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:17.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.339 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 13:17:17.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.339 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:17:17.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:17.340 | INFO | Uploaded 36GiB to GPU in 2.50242s (14.3861GiB/s) | 20260508 13:17:17.340 | INFO | Downloaded 34GiB to GPU in 2.50474s (13.5743GiB/s) | 20260508 13:17:22.480 | INFO | Analyzing chunk frames = [8399360,8399872) | 20260508 13:17:22.480 | INFO | MemInfo: { | 20260508 13:17:22.480 | INFO | "gpu" : " 17 GB", | 20260508 13:17:22.480 | INFO | "huge" : "", | 20260508 13:17:22.480 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:22.480 | INFO | "malloc" : " 14 GB", | 20260508 13:17:22.480 | INFO | "pinned" : " 29 GB" | 20260508 13:17:22.480 | INFO | } | 20260508 13:17:22.480 | INFO | Perf report for the last 5.14106 seconds: | 20260508 13:17:22.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:22.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.480 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:17:22.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.480 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260508 13:17:22.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:22.481 | INFO | Uploaded 36GiB to GPU in 2.43633s (14.7763GiB/s) | 20260508 13:17:22.481 | INFO | Downloaded 34GiB to GPU in 2.44356s (13.9141GiB/s) | 20260508 13:17:27.613 | INFO | Analyzing chunk frames = [8399872,8400384) | 20260508 13:17:27.613 | INFO | MemInfo: { | 20260508 13:17:27.613 | INFO | "gpu" : " 17 GB", | 20260508 13:17:27.613 | INFO | "huge" : "", | 20260508 13:17:27.613 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:27.613 | INFO | "malloc" : " 14 GB", | 20260508 13:17:27.613 | INFO | "pinned" : " 29 GB" | 20260508 13:17:27.613 | INFO | } | 20260508 13:17:27.614 | INFO | Perf report for the last 5.13324 seconds: | 20260508 13:17:27.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:27.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:17:27.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:17:27.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:27.614 | INFO | Uploaded 36GiB to GPU in 2.31834s (15.5284GiB/s) | 20260508 13:17:27.614 | INFO | Downloaded 34GiB to GPU in 2.40643s (14.1288GiB/s) | 20260508 13:17:32.756 | INFO | Analyzing chunk frames = [8400384,8400896) | 20260508 13:17:32.756 | INFO | MemInfo: { | 20260508 13:17:32.756 | INFO | "gpu" : " 17 GB", | 20260508 13:17:32.756 | INFO | "huge" : "", | 20260508 13:17:32.756 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:32.756 | INFO | "malloc" : " 14 GB", | 20260508 13:17:32.756 | INFO | "pinned" : " 29 GB" | 20260508 13:17:32.756 | INFO | } | 20260508 13:17:32.756 | INFO | Perf report for the last 5.14243 seconds: | 20260508 13:17:32.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:32.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 13:17:32.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:17:32.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:32.756 | INFO | Uploaded 36GiB to GPU in 2.50412s (14.3763GiB/s) | 20260508 13:17:32.756 | INFO | Downloaded 34GiB to GPU in 2.47141s (13.7573GiB/s) | 20260508 13:17:32.853 | INFO | Starting a round of Baz IO | 20260508 13:17:36.249 | INFO | Populating RTMetrics report | 20260508 13:17:36.249 | INFO | RTMetrics report populated | 20260508 13:17:36.249 | INFO | Opening tmp RTMetrics json file | 20260508 13:17:36.257 | INFO | Serializing RTMetrics to json | 20260508 13:17:36.258 | INFO | Serialization done, swapping tmp file now... | 20260508 13:17:36.258 | INFO | RTMetrics done | 20260508 13:17:36.319 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.466 seconds | 20260508 13:17:36.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:17:36.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145511% / 0% / 8.34465e-05% | 20260508 13:17:36.319 | INFO | Baz IO overall: | 20260508 13:17:36.319 | INFO | BytesWritten : 3583668828417 | 20260508 13:17:36.319 | INFO | NumEvents : 1718476612626 | 20260508 13:17:37.900 | INFO | Analyzing chunk frames = [8400896,8401408) | 20260508 13:17:37.900 | INFO | MemInfo: { | 20260508 13:17:37.900 | INFO | "gpu" : " 17 GB", | 20260508 13:17:37.900 | INFO | "huge" : "", | 20260508 13:17:37.900 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:37.900 | INFO | "malloc" : " 14 GB", | 20260508 13:17:37.900 | INFO | "pinned" : " 29 GB" | 20260508 13:17:37.900 | INFO | } | 20260508 13:17:37.900 | INFO | Perf report for the last 5.14405 seconds: | 20260508 13:17:37.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:37.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:37.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:37.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:37.900 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 13:17:37.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:37.900 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:17:37.900 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 13:17:37.900 | INFO | BAZWRITER : 66.7%, 2.5, 84.1ms, 0 | 20260508 13:17:37.900 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:17:37.900 | INFO | Uploaded 36GiB to GPU in 2.60608s (13.8138GiB/s) | 20260508 13:17:37.900 | INFO | Downloaded 38GiB to GPU in 2.731s (13.9143GiB/s) | 20260508 13:17:43.042 | INFO | Analyzing chunk frames = [8401408,8401920) | 20260508 13:17:43.042 | INFO | MemInfo: { | 20260508 13:17:43.042 | INFO | "gpu" : " 17 GB", | 20260508 13:17:43.042 | INFO | "huge" : "", | 20260508 13:17:43.042 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:43.042 | INFO | "malloc" : " 14 GB", | 20260508 13:17:43.042 | INFO | "pinned" : " 29 GB" | 20260508 13:17:43.042 | INFO | } | 20260508 13:17:43.042 | INFO | Perf report for the last 5.14215 seconds: | 20260508 13:17:43.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:43.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:17:43.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:17:43.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:43.042 | INFO | Uploaded 36GiB to GPU in 2.4692s (14.5796GiB/s) | 20260508 13:17:43.042 | INFO | Downloaded 34GiB to GPU in 2.46775s (13.7777GiB/s) | 20260508 13:17:48.175 | INFO | Analyzing chunk frames = [8401920,8402432) | 20260508 13:17:48.175 | INFO | MemInfo: { | 20260508 13:17:48.175 | INFO | "gpu" : " 17 GB", | 20260508 13:17:48.175 | INFO | "huge" : "", | 20260508 13:17:48.175 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:48.175 | INFO | "malloc" : " 14 GB", | 20260508 13:17:48.175 | INFO | "pinned" : " 29 GB" | 20260508 13:17:48.175 | INFO | } | 20260508 13:17:48.175 | INFO | Perf report for the last 5.13284 seconds: | 20260508 13:17:48.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:48.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:17:48.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:17:48.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:48.175 | INFO | Uploaded 36GiB to GPU in 2.46231s (14.6204GiB/s) | 20260508 13:17:48.175 | INFO | Downloaded 34GiB to GPU in 2.45949s (13.824GiB/s) | 20260508 13:17:53.309 | INFO | Analyzing chunk frames = [8402432,8402944) | 20260508 13:17:53.309 | INFO | MemInfo: { | 20260508 13:17:53.309 | INFO | "gpu" : " 17 GB", | 20260508 13:17:53.309 | INFO | "huge" : "", | 20260508 13:17:53.309 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:53.309 | INFO | "malloc" : " 14 GB", | 20260508 13:17:53.309 | INFO | "pinned" : " 29 GB" | 20260508 13:17:53.309 | INFO | } | 20260508 13:17:53.309 | INFO | Perf report for the last 5.13418 seconds: | 20260508 13:17:53.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:53.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | ANALYSIS : 66.1%, 3.0, 102.2ms, 0 | 20260508 13:17:53.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:17:53.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:53.309 | INFO | Uploaded 36GiB to GPU in 2.52145s (14.2775GiB/s) | 20260508 13:17:53.309 | INFO | Downloaded 34GiB to GPU in 2.45181s (13.8673GiB/s) | 20260508 13:17:58.451 | INFO | Analyzing chunk frames = [8402944,8403456) | 20260508 13:17:58.451 | INFO | MemInfo: { | 20260508 13:17:58.451 | INFO | "gpu" : " 17 GB", | 20260508 13:17:58.451 | INFO | "huge" : "", | 20260508 13:17:58.451 | INFO | "huge_shared" : " 29 GB", | 20260508 13:17:58.451 | INFO | "malloc" : " 14 GB", | 20260508 13:17:58.451 | INFO | "pinned" : " 29 GB" | 20260508 13:17:58.452 | INFO | } | 20260508 13:17:58.452 | INFO | Perf report for the last 5.14231 seconds: | 20260508 13:17:58.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:17:58.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 13:17:58.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 13:17:58.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:17:58.452 | INFO | Uploaded 36GiB to GPU in 2.46887s (14.5816GiB/s) | 20260508 13:17:58.452 | INFO | Downloaded 34GiB to GPU in 2.53298s (13.4229GiB/s) | 20260508 13:18:03.600 | INFO | Analyzing chunk frames = [8403456,8403968) | 20260508 13:18:03.600 | INFO | MemInfo: { | 20260508 13:18:03.600 | INFO | "gpu" : " 17 GB", | 20260508 13:18:03.600 | INFO | "huge" : "", | 20260508 13:18:03.600 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:03.600 | INFO | "malloc" : " 14 GB", | 20260508 13:18:03.600 | INFO | "pinned" : " 29 GB" | 20260508 13:18:03.600 | INFO | } | 20260508 13:18:03.600 | INFO | Perf report for the last 5.14847 seconds: | 20260508 13:18:03.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:03.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.600 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:18:03.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.600 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:18:03.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:03.601 | INFO | Uploaded 36GiB to GPU in 2.42454s (14.8482GiB/s) | 20260508 13:18:03.601 | INFO | Downloaded 34GiB to GPU in 2.43129s (13.9843GiB/s) | 20260508 13:18:08.729 | INFO | Analyzing chunk frames = [8403968,8404480) | 20260508 13:18:08.729 | INFO | MemInfo: { | 20260508 13:18:08.729 | INFO | "gpu" : " 17 GB", | 20260508 13:18:08.729 | INFO | "huge" : "", | 20260508 13:18:08.729 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:08.729 | INFO | "malloc" : " 14 GB", | 20260508 13:18:08.729 | INFO | "pinned" : " 29 GB" | 20260508 13:18:08.729 | INFO | } | 20260508 13:18:08.729 | INFO | Perf report for the last 5.12926 seconds: | 20260508 13:18:08.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:08.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.729 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 13:18:08.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.729 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:18:08.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:08.730 | INFO | Uploaded 36GiB to GPU in 2.34888s (15.3264GiB/s) | 20260508 13:18:08.730 | INFO | Downloaded 34GiB to GPU in 2.40185s (14.1558GiB/s) | 20260508 13:18:13.867 | INFO | Analyzing chunk frames = [8404480,8404992) | 20260508 13:18:13.867 | INFO | MemInfo: { | 20260508 13:18:13.867 | INFO | "gpu" : " 17 GB", | 20260508 13:18:13.867 | INFO | "huge" : "", | 20260508 13:18:13.867 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:13.867 | INFO | "malloc" : " 14 GB", | 20260508 13:18:13.867 | INFO | "pinned" : " 29 GB" | 20260508 13:18:13.867 | INFO | } | 20260508 13:18:13.867 | INFO | Perf report for the last 5.13738 seconds: | 20260508 13:18:13.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:13.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 13:18:13.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:18:13.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:13.867 | INFO | Uploaded 36GiB to GPU in 2.48957s (14.4604GiB/s) | 20260508 13:18:13.867 | INFO | Downloaded 34GiB to GPU in 2.44908s (13.8828GiB/s) | 20260508 13:18:13.867 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:18:13.867 | INFO | Total mem usage: 16.7 GiB | 20260508 13:18:13.867 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:18:13.867 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:18:13.867 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:18:13.867 | INFO | High Water Breakdown: | 20260508 13:18:13.867 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:18:13.867 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:18:13.867 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:18:13.867 | INFO | ------------------------------------------ | 20260508 13:18:13.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:18:13.867 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:18:13.867 | INFO | Total mem usage: 26.5 GiB | 20260508 13:18:13.867 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:18:13.867 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:18:13.867 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:18:13.867 | INFO | High Water Breakdown: | 20260508 13:18:13.867 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:18:13.867 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:18:13.867 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:18:13.867 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:18:13.867 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:18:13.867 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:18:13.867 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:18:13.867 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:18:13.867 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:18:13.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:18:13.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:18:13.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:18:13.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:18:13.867 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:18:13.867 | INFO | ------------------------------------------ | 20260508 13:18:13.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:18:13.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:18:13.867 | INFO | Total mem usage: 27.8 GiB | 20260508 13:18:13.867 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:18:13.867 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:18:13.867 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:18:13.867 | INFO | High Water Breakdown: | 20260508 13:18:13.867 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:18:13.867 | INFO | ------------------------------------------ | 20260508 13:18:13.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:18:13.867 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:18:13.867 | INFO | Total mem usage: 15.7 GiB | 20260508 13:18:13.867 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:18:13.867 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:18:13.868 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:18:13.868 | INFO | High Water Breakdown: | 20260508 13:18:13.868 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:18:13.868 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:18:13.868 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:18:13.868 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:18:13.868 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:18:13.868 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:18:13.868 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:18:13.868 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:18:13.868 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:18:13.868 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:18:13.868 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:18:13.868 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:18:13.868 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:18:13.868 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:18:13.868 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:18:13.868 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:18:13.868 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:18:13.868 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:18:13.868 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:18:13.868 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:18:13.868 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:18:13.868 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:18:13.868 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:18:13.868 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:18:13.868 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:18:13.868 | INFO | ------------------------------------------ | 20260508 13:18:13.868 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:18:13.868 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:18:13.868 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:18:13.868 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:18:13.868 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.51e+01, 1.39e+01 / 5.63e-01 | 20260508 13:18:13.868 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.06e-01, 1.24e-02 / 7.08e-03 | 20260508 13:18:13.868 | INFO | PerfReport: 12.19%, Baseline -- 3.98e+00 / 4.42e+00, 5.93e-01 / 2.27e-01 | 20260508 13:18:13.868 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.42e+00, 2.09e-01 / 1.21e-01 | 20260508 13:18:13.868 | INFO | PerfReport: 15.95%, DME -- 4.16e+01 / 4.45e+01, 3.60e+00 / 8.10e-01 | 20260508 13:18:13.868 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.87e+01 / 2.09e+01, 1.82e+00 / 1.21e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.31e+00 / 2.39e+00, 4.03e-01 / 2.75e-01 | 20260508 13:18:13.868 | INFO | PerfReport: 2.94%, Metrics -- 9.58e-01 / 1.08e+00, 2.41e-01 / 7.12e-02 | 20260508 13:18:13.868 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.89e-01 / 3.21e-01, 4.57e-02 / 2.81e-02 | 20260508 13:18:13.868 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:18:13.868 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:18:13.868 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:18:13.868 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 24.64%, Upload -- 2.59e+01 / 0.00e+00, 8.05e+00 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 24.72%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 31.08%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 2.22%, LockUpload -- 2.33e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 2.47%, LockDownload -- 2.59e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 14.85%, LockCompute -- 1.56e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 13:18:13.868 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260508 13:18:13.964 | INFO | Starting a round of Baz IO | 20260508 13:18:17.379 | INFO | Populating RTMetrics report | 20260508 13:18:17.379 | INFO | RTMetrics report populated | 20260508 13:18:17.379 | INFO | Opening tmp RTMetrics json file | 20260508 13:18:17.387 | INFO | Serializing RTMetrics to json | 20260508 13:18:17.388 | INFO | Serialization done, swapping tmp file now... | 20260508 13:18:17.389 | INFO | RTMetrics done | 20260508 13:18:17.446 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.481 seconds | 20260508 13:18:17.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:18:17.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145268% / 0% / 4.37101e-05% | 20260508 13:18:17.446 | INFO | Baz IO overall: | 20260508 13:18:17.446 | INFO | BytesWritten : 3585162287457 | 20260508 13:18:17.446 | INFO | NumEvents : 1719176209027 | 20260508 13:18:19.011 | INFO | Analyzing chunk frames = [8404992,8405504) | 20260508 13:18:19.011 | INFO | MemInfo: { | 20260508 13:18:19.011 | INFO | "gpu" : " 17 GB", | 20260508 13:18:19.011 | INFO | "huge" : "", | 20260508 13:18:19.011 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:19.011 | INFO | "malloc" : " 14 GB", | 20260508 13:18:19.011 | INFO | "pinned" : " 29 GB" | 20260508 13:18:19.011 | INFO | } | 20260508 13:18:19.011 | INFO | Perf report for the last 5.1446 seconds: | 20260508 13:18:19.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:19.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:19.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:19.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:19.011 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 13:18:19.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:19.011 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:18:19.011 | INFO | PRE_HQ : 65.3%, 1.6, 53.6ms, 0 | 20260508 13:18:19.011 | INFO | BAZWRITER : 66.9%, 2.5, 84.9ms, 0 | 20260508 13:18:19.011 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:18:19.011 | INFO | Uploaded 36GiB to GPU in 2.57446s (13.9835GiB/s) | 20260508 13:18:19.011 | INFO | Downloaded 38GiB to GPU in 2.71751s (13.9834GiB/s) | 20260508 13:18:24.151 | INFO | Analyzing chunk frames = [8405504,8406016) | 20260508 13:18:24.151 | INFO | MemInfo: { | 20260508 13:18:24.151 | INFO | "gpu" : " 17 GB", | 20260508 13:18:24.151 | INFO | "huge" : "", | 20260508 13:18:24.151 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:24.151 | INFO | "malloc" : " 14 GB", | 20260508 13:18:24.151 | INFO | "pinned" : " 29 GB" | 20260508 13:18:24.151 | INFO | } | 20260508 13:18:24.151 | INFO | Perf report for the last 5.13997 seconds: | 20260508 13:18:24.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:24.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:18:24.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:18:24.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:24.151 | INFO | Uploaded 36GiB to GPU in 2.47299s (14.5573GiB/s) | 20260508 13:18:24.151 | INFO | Downloaded 34GiB to GPU in 2.46184s (13.8108GiB/s) | 20260508 13:18:29.289 | INFO | Analyzing chunk frames = [8406016,8406528) | 20260508 13:18:29.289 | INFO | MemInfo: { | 20260508 13:18:29.289 | INFO | "gpu" : " 17 GB", | 20260508 13:18:29.289 | INFO | "huge" : "", | 20260508 13:18:29.289 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:29.289 | INFO | "malloc" : " 14 GB", | 20260508 13:18:29.289 | INFO | "pinned" : " 29 GB" | 20260508 13:18:29.289 | INFO | } | 20260508 13:18:29.289 | INFO | Perf report for the last 5.13748 seconds: | 20260508 13:18:29.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:29.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 13:18:29.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260508 13:18:29.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:29.289 | INFO | Uploaded 36GiB to GPU in 2.47614s (14.5388GiB/s) | 20260508 13:18:29.289 | INFO | Downloaded 34GiB to GPU in 2.46738s (13.7798GiB/s) | 20260508 13:18:34.426 | INFO | Analyzing chunk frames = [8406528,8407040) | 20260508 13:18:34.426 | INFO | MemInfo: { | 20260508 13:18:34.426 | INFO | "gpu" : " 17 GB", | 20260508 13:18:34.426 | INFO | "huge" : "", | 20260508 13:18:34.426 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:34.427 | INFO | "malloc" : " 14 GB", | 20260508 13:18:34.427 | INFO | "pinned" : " 29 GB" | 20260508 13:18:34.427 | INFO | } | 20260508 13:18:34.427 | INFO | Perf report for the last 5.13787 seconds: | 20260508 13:18:34.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:34.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | ANALYSIS : 66.1%, 3.0, 102.4ms, 0 | 20260508 13:18:34.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:18:34.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:34.427 | INFO | Uploaded 36GiB to GPU in 2.5222s (14.2733GiB/s) | 20260508 13:18:34.427 | INFO | Downloaded 34GiB to GPU in 2.4566s (13.8402GiB/s) | 20260508 13:18:39.566 | INFO | Analyzing chunk frames = [8407040,8407552) | 20260508 13:18:39.566 | INFO | MemInfo: { | 20260508 13:18:39.566 | INFO | "gpu" : " 17 GB", | 20260508 13:18:39.566 | INFO | "huge" : "", | 20260508 13:18:39.566 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:39.566 | INFO | "malloc" : " 14 GB", | 20260508 13:18:39.566 | INFO | "pinned" : " 29 GB" | 20260508 13:18:39.566 | INFO | } | 20260508 13:18:39.566 | INFO | Perf report for the last 5.13953 seconds: | 20260508 13:18:39.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:39.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.566 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:18:39.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.566 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:18:39.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:39.567 | INFO | Uploaded 36GiB to GPU in 2.40013s (14.9992GiB/s) | 20260508 13:18:39.567 | INFO | Downloaded 34GiB to GPU in 2.51026s (13.5444GiB/s) | 20260508 13:18:44.710 | INFO | Analyzing chunk frames = [8407552,8408064) | 20260508 13:18:44.710 | INFO | MemInfo: { | 20260508 13:18:44.710 | INFO | "gpu" : " 17 GB", | 20260508 13:18:44.710 | INFO | "huge" : "", | 20260508 13:18:44.710 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:44.710 | INFO | "malloc" : " 14 GB", | 20260508 13:18:44.710 | INFO | "pinned" : " 29 GB" | 20260508 13:18:44.710 | INFO | } | 20260508 13:18:44.710 | INFO | Perf report for the last 5.14394 seconds: | 20260508 13:18:44.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:44.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.710 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:18:44.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.710 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:18:44.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:44.711 | INFO | Uploaded 36GiB to GPU in 2.40062s (14.9961GiB/s) | 20260508 13:18:44.711 | INFO | Downloaded 34GiB to GPU in 2.46557s (13.7899GiB/s) | 20260508 13:18:49.849 | INFO | Analyzing chunk frames = [8408064,8408576) | 20260508 13:18:49.849 | INFO | MemInfo: { | 20260508 13:18:49.849 | INFO | "gpu" : " 17 GB", | 20260508 13:18:49.849 | INFO | "huge" : "", | 20260508 13:18:49.849 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:49.849 | INFO | "malloc" : " 14 GB", | 20260508 13:18:49.849 | INFO | "pinned" : " 29 GB" | 20260508 13:18:49.849 | INFO | } | 20260508 13:18:49.849 | INFO | Perf report for the last 5.13893 seconds: | 20260508 13:18:49.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:49.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | ANALYSIS : 65.1%, 3.0, 101.3ms, 0 | 20260508 13:18:49.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:18:49.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:49.849 | INFO | Uploaded 36GiB to GPU in 2.36643s (15.2128GiB/s) | 20260508 13:18:49.849 | INFO | Downloaded 34GiB to GPU in 2.44835s (13.8869GiB/s) | 20260508 13:18:54.986 | INFO | Analyzing chunk frames = [8408576,8409088) | 20260508 13:18:54.986 | INFO | MemInfo: { | 20260508 13:18:54.986 | INFO | "gpu" : " 17 GB", | 20260508 13:18:54.986 | INFO | "huge" : "", | 20260508 13:18:54.986 | INFO | "huge_shared" : " 29 GB", | 20260508 13:18:54.986 | INFO | "malloc" : " 14 GB", | 20260508 13:18:54.986 | INFO | "pinned" : " 29 GB" | 20260508 13:18:54.986 | INFO | } | 20260508 13:18:54.986 | INFO | Perf report for the last 5.13733 seconds: | 20260508 13:18:54.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:18:54.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 13:18:54.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 13:18:54.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:18:54.987 | INFO | Uploaded 36GiB to GPU in 2.49s (14.4579GiB/s) | 20260508 13:18:54.987 | INFO | Downloaded 34GiB to GPU in 2.48709s (13.6706GiB/s) | 20260508 13:18:55.084 | INFO | Starting a round of Baz IO | 20260508 13:18:58.498 | INFO | Populating RTMetrics report | 20260508 13:18:58.499 | INFO | RTMetrics report populated | 20260508 13:18:58.499 | INFO | Opening tmp RTMetrics json file | 20260508 13:18:58.507 | INFO | Serializing RTMetrics to json | 20260508 13:18:58.509 | INFO | Serialization done, swapping tmp file now... | 20260508 13:18:58.509 | INFO | RTMetrics done | 20260508 13:18:58.564 | INFO | Finished a round of Baz IO: Wrote 1.39 GiB in 3.48 seconds | 20260508 13:18:58.564 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.96% / 0.00% / 60.04% | 20260508 13:18:58.564 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144267% / 0% / 7.15256e-05% | 20260508 13:18:58.564 | INFO | Baz IO overall: | 20260508 13:18:58.564 | INFO | BytesWritten : 3586654455911 | 20260508 13:18:58.564 | INFO | NumEvents : 1719874856767 | 20260508 13:19:00.124 | INFO | Analyzing chunk frames = [8409088,8409600) | 20260508 13:19:00.124 | INFO | MemInfo: { | 20260508 13:19:00.124 | INFO | "gpu" : " 17 GB", | 20260508 13:19:00.124 | INFO | "huge" : "", | 20260508 13:19:00.124 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:00.124 | INFO | "malloc" : " 14 GB", | 20260508 13:19:00.124 | INFO | "pinned" : " 29 GB" | 20260508 13:19:00.124 | INFO | } | 20260508 13:19:00.124 | INFO | Perf report for the last 5.13774 seconds: | 20260508 13:19:00.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:00.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:00.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:00.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:00.124 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 13:19:00.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:00.124 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:19:00.124 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 13:19:00.124 | INFO | BAZWRITER : 67.0%, 2.4, 84.1ms, 0 | 20260508 13:19:00.124 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:19:00.124 | INFO | Uploaded 36GiB to GPU in 2.62976s (13.6894GiB/s) | 20260508 13:19:00.124 | INFO | Downloaded 38GiB to GPU in 2.72799s (13.9297GiB/s) | 20260508 13:19:05.268 | INFO | Analyzing chunk frames = [8409600,8410112) | 20260508 13:19:05.268 | INFO | MemInfo: { | 20260508 13:19:05.268 | INFO | "gpu" : " 17 GB", | 20260508 13:19:05.268 | INFO | "huge" : "", | 20260508 13:19:05.268 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:05.268 | INFO | "malloc" : " 14 GB", | 20260508 13:19:05.269 | INFO | "pinned" : " 29 GB" | 20260508 13:19:05.269 | INFO | } | 20260508 13:19:05.269 | INFO | Perf report for the last 5.14434 seconds: | 20260508 13:19:05.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:05.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:19:05.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260508 13:19:05.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:05.269 | INFO | Uploaded 36GiB to GPU in 2.47358s (14.5538GiB/s) | 20260508 13:19:05.269 | INFO | Downloaded 34GiB to GPU in 2.50006s (13.5997GiB/s) | 20260508 13:19:10.403 | INFO | Analyzing chunk frames = [8410112,8410624) | 20260508 13:19:10.403 | INFO | MemInfo: { | 20260508 13:19:10.403 | INFO | "gpu" : " 17 GB", | 20260508 13:19:10.403 | INFO | "huge" : "", | 20260508 13:19:10.403 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:10.403 | INFO | "malloc" : " 14 GB", | 20260508 13:19:10.403 | INFO | "pinned" : " 29 GB" | 20260508 13:19:10.403 | INFO | } | 20260508 13:19:10.403 | INFO | Perf report for the last 5.13458 seconds: | 20260508 13:19:10.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:10.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.403 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:19:10.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.403 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:19:10.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:10.404 | INFO | Uploaded 36GiB to GPU in 2.47379s (14.5526GiB/s) | 20260508 13:19:10.404 | INFO | Downloaded 34GiB to GPU in 2.4776s (13.723GiB/s) | 20260508 13:19:15.546 | INFO | Analyzing chunk frames = [8410624,8411136) | 20260508 13:19:15.546 | INFO | MemInfo: { | 20260508 13:19:15.546 | INFO | "gpu" : " 17 GB", | 20260508 13:19:15.546 | INFO | "huge" : "", | 20260508 13:19:15.546 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:15.546 | INFO | "malloc" : " 14 GB", | 20260508 13:19:15.546 | INFO | "pinned" : " 29 GB" | 20260508 13:19:15.546 | INFO | } | 20260508 13:19:15.546 | INFO | Perf report for the last 5.143 seconds: | 20260508 13:19:15.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:15.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 13:19:15.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:19:15.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:15.546 | INFO | Uploaded 36GiB to GPU in 2.47159s (14.5655GiB/s) | 20260508 13:19:15.547 | INFO | Downloaded 34GiB to GPU in 2.4782s (13.7196GiB/s) | 20260508 13:19:20.686 | INFO | Analyzing chunk frames = [8411136,8411648) | 20260508 13:19:20.686 | INFO | MemInfo: { | 20260508 13:19:20.686 | INFO | "gpu" : " 17 GB", | 20260508 13:19:20.686 | INFO | "huge" : "", | 20260508 13:19:20.686 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:20.686 | INFO | "malloc" : " 14 GB", | 20260508 13:19:20.686 | INFO | "pinned" : " 29 GB" | 20260508 13:19:20.686 | INFO | } | 20260508 13:19:20.686 | INFO | Perf report for the last 5.13975 seconds: | 20260508 13:19:20.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:20.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 13:19:20.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:19:20.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:20.686 | INFO | Uploaded 36GiB to GPU in 2.30838s (15.5954GiB/s) | 20260508 13:19:20.686 | INFO | Downloaded 34GiB to GPU in 2.50089s (13.5951GiB/s) | 20260508 13:19:25.818 | INFO | Analyzing chunk frames = [8411648,8412160) | 20260508 13:19:25.818 | INFO | MemInfo: { | 20260508 13:19:25.818 | INFO | "gpu" : " 17 GB", | 20260508 13:19:25.818 | INFO | "huge" : "", | 20260508 13:19:25.818 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:25.818 | INFO | "malloc" : " 14 GB", | 20260508 13:19:25.818 | INFO | "pinned" : " 29 GB" | 20260508 13:19:25.818 | INFO | } | 20260508 13:19:25.819 | INFO | Perf report for the last 5.13254 seconds: | 20260508 13:19:25.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:25.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:19:25.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:19:25.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:25.819 | INFO | Uploaded 36GiB to GPU in 2.43651s (14.7752GiB/s) | 20260508 13:19:25.819 | INFO | Downloaded 34GiB to GPU in 2.44965s (13.8795GiB/s) | 20260508 13:19:30.965 | INFO | Analyzing chunk frames = [8412160,8412672) | 20260508 13:19:30.965 | INFO | MemInfo: { | 20260508 13:19:30.965 | INFO | "gpu" : " 17 GB", | 20260508 13:19:30.965 | INFO | "huge" : "", | 20260508 13:19:30.965 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:30.966 | INFO | "malloc" : " 14 GB", | 20260508 13:19:30.966 | INFO | "pinned" : " 29 GB" | 20260508 13:19:30.966 | INFO | } | 20260508 13:19:30.966 | INFO | Perf report for the last 5.14713 seconds: | 20260508 13:19:30.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:30.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | ANALYSIS : 65.4%, 3.0, 102.2ms, 0 | 20260508 13:19:30.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:19:30.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:30.966 | INFO | Uploaded 36GiB to GPU in 2.47149s (14.5661GiB/s) | 20260508 13:19:30.966 | INFO | Downloaded 34GiB to GPU in 2.47441s (13.7406GiB/s) | 20260508 13:19:36.099 | INFO | Analyzing chunk frames = [8412672,8413184) | 20260508 13:19:36.099 | INFO | MemInfo: { | 20260508 13:19:36.099 | INFO | "gpu" : " 17 GB", | 20260508 13:19:36.099 | INFO | "huge" : "", | 20260508 13:19:36.099 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:36.099 | INFO | "malloc" : " 14 GB", | 20260508 13:19:36.099 | INFO | "pinned" : " 29 GB" | 20260508 13:19:36.099 | INFO | } | 20260508 13:19:36.099 | INFO | Perf report for the last 5.13376 seconds: | 20260508 13:19:36.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:36.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.099 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:19:36.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.099 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:19:36.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:36.100 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:19:36.100 | INFO | Total mem usage: 16.7 GiB | 20260508 13:19:36.100 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:19:36.100 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:19:36.100 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:19:36.100 | INFO | High Water Breakdown: | 20260508 13:19:36.100 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:19:36.100 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:19:36.100 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:19:36.100 | INFO | ------------------------------------------ | 20260508 13:19:36.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:19:36.100 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:19:36.100 | INFO | Total mem usage: 26.5 GiB | 20260508 13:19:36.100 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:19:36.100 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:19:36.100 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:19:36.100 | INFO | High Water Breakdown: | 20260508 13:19:36.100 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:19:36.100 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:19:36.100 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:19:36.100 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:19:36.100 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:19:36.100 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:19:36.100 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:19:36.100 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:19:36.100 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:19:36.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:19:36.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:19:36.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:19:36.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:19:36.100 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:19:36.100 | INFO | ------------------------------------------ | 20260508 13:19:36.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:19:36.100 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:19:36.100 | INFO | Total mem usage: 27.8 GiB | 20260508 13:19:36.100 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:19:36.100 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:19:36.100 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:19:36.100 | INFO | High Water Breakdown: | 20260508 13:19:36.100 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:19:36.100 | INFO | ------------------------------------------ | 20260508 13:19:36.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:19:36.100 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:19:36.100 | INFO | Total mem usage: 15.7 GiB | 20260508 13:19:36.100 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:19:36.100 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:19:36.100 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:19:36.100 | INFO | High Water Breakdown: | 20260508 13:19:36.100 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:19:36.100 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:19:36.100 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:19:36.100 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:19:36.100 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:19:36.100 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:19:36.100 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:19:36.100 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:19:36.100 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:19:36.100 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:19:36.100 | INFO | Uploaded 36GiB to GPU in 2.3432s (15.3636GiB/s) | 20260508 13:19:36.100 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:19:36.100 | INFO | Downloaded 34GiB to GPU in 2.44192s (13.9234GiB/s) | 20260508 13:19:36.100 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:19:36.100 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:19:36.100 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:19:36.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:19:36.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:19:36.100 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:19:36.100 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:19:36.100 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:19:36.100 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:19:36.100 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:19:36.100 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:19:36.100 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:19:36.100 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:19:36.100 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:19:36.100 | INFO | ------------------------------------------ | 20260508 13:19:36.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:19:36.101 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:19:36.101 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:19:36.101 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:19:36.101 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.38e+01 / 5.82e-01 | 20260508 13:19:36.101 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.03e-01, 1.20e-02 / 5.59e-03 | 20260508 13:19:36.101 | INFO | PerfReport: 12.16%, Baseline -- 3.97e+00 / 4.42e+00, 5.89e-01 / 1.76e-01 | 20260508 13:19:36.101 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.09e-01 / 1.05e-01 | 20260508 13:19:36.101 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.51e+01, 3.61e+00 / 1.24e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.87e+00 / 1.39e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.30e+00 / 2.45e+00, 4.03e-01 / 2.66e-01 | 20260508 13:19:36.101 | INFO | PerfReport: 2.92%, Metrics -- 9.54e-01 / 1.09e+00, 2.39e-01 / 1.14e-01 | 20260508 13:19:36.101 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.26e-01, 4.67e-02 / 2.10e-02 | 20260508 13:19:36.101 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:19:36.101 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:19:36.101 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:19:36.101 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.37e+01 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.97e+00 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 24.87%, Download -- 2.61e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 31.12%, Compute -- 3.26e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 2.06%, LockUpload -- 2.16e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 2.67%, LockDownload -- 2.80e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 14.87%, LockCompute -- 1.56e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 13:19:36.101 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260508 13:19:36.210 | INFO | Starting a round of Baz IO | 20260508 13:19:39.612 | INFO | Populating RTMetrics report | 20260508 13:19:39.612 | INFO | RTMetrics report populated | 20260508 13:19:39.612 | INFO | Opening tmp RTMetrics json file | 20260508 13:19:39.621 | INFO | Serializing RTMetrics to json | 20260508 13:19:39.622 | INFO | Serialization done, swapping tmp file now... | 20260508 13:19:39.622 | INFO | RTMetrics done | 20260508 13:19:39.673 | INFO | Finished a round of Baz IO: Wrote 1.389 GiB in 3.463 seconds | 20260508 13:19:39.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.04% | 20260508 13:19:39.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144915% / 3.97364e-06% / 5.96046e-05% | 20260508 13:19:39.673 | INFO | Baz IO overall: | 20260508 13:19:39.673 | INFO | BytesWritten : 3588146315530 | 20260508 13:19:39.673 | INFO | NumEvents : 1720573391409 | 20260508 13:19:41.241 | INFO | Analyzing chunk frames = [8413184,8413696) | 20260508 13:19:41.241 | INFO | MemInfo: { | 20260508 13:19:41.242 | INFO | "gpu" : " 17 GB", | 20260508 13:19:41.242 | INFO | "huge" : "", | 20260508 13:19:41.242 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:41.242 | INFO | "malloc" : " 14 GB", | 20260508 13:19:41.242 | INFO | "pinned" : " 29 GB" | 20260508 13:19:41.242 | INFO | } | 20260508 13:19:41.242 | INFO | Perf report for the last 5.14216 seconds: | 20260508 13:19:41.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:41.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:41.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:41.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:41.242 | INFO | ANALYSIS : 67.8%, 3.0, 105.6ms, 0 | 20260508 13:19:41.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:41.242 | INFO | RT_METRICS : 40.5%, 1.0, 21.2ms, 0 | 20260508 13:19:41.242 | INFO | PRE_HQ : 66.0%, 1.7, 57.5ms, 0 | 20260508 13:19:41.242 | INFO | BAZWRITER : 66.5%, 2.6, 87.8ms, 0 | 20260508 13:19:41.242 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260508 13:19:41.242 | INFO | Uploaded 36GiB to GPU in 2.61677s (13.7574GiB/s) | 20260508 13:19:41.242 | INFO | Downloaded 38GiB to GPU in 2.71441s (13.9993GiB/s) | 20260508 13:19:46.373 | INFO | Analyzing chunk frames = [8413696,8414208) | 20260508 13:19:46.373 | INFO | MemInfo: { | 20260508 13:19:46.373 | INFO | "gpu" : " 17 GB", | 20260508 13:19:46.373 | INFO | "huge" : "", | 20260508 13:19:46.373 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:46.373 | INFO | "malloc" : " 14 GB", | 20260508 13:19:46.373 | INFO | "pinned" : " 29 GB" | 20260508 13:19:46.373 | INFO | } | 20260508 13:19:46.373 | INFO | Perf report for the last 5.1317 seconds: | 20260508 13:19:46.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:46.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.373 | INFO | ANALYSIS : 66.6%, 3.0, 103.6ms, 0 | 20260508 13:19:46.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.373 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:19:46.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:46.374 | INFO | Uploaded 36GiB to GPU in 2.52843s (14.2381GiB/s) | 20260508 13:19:46.374 | INFO | Downloaded 34GiB to GPU in 2.46016s (13.8203GiB/s) | 20260508 13:19:51.519 | INFO | Analyzing chunk frames = [8414208,8414720) | 20260508 13:19:51.519 | INFO | MemInfo: { | 20260508 13:19:51.519 | INFO | "gpu" : " 17 GB", | 20260508 13:19:51.519 | INFO | "huge" : "", | 20260508 13:19:51.519 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:51.519 | INFO | "malloc" : " 14 GB", | 20260508 13:19:51.519 | INFO | "pinned" : " 29 GB" | 20260508 13:19:51.519 | INFO | } | 20260508 13:19:51.519 | INFO | Perf report for the last 5.14575 seconds: | 20260508 13:19:51.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:51.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.519 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:19:51.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.519 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:19:51.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:51.520 | INFO | Uploaded 36GiB to GPU in 2.50472s (14.3729GiB/s) | 20260508 13:19:51.520 | INFO | Downloaded 34GiB to GPU in 2.44676s (13.8959GiB/s) | 20260508 13:19:56.662 | INFO | Analyzing chunk frames = [8414720,8415232) | 20260508 13:19:56.662 | INFO | MemInfo: { | 20260508 13:19:56.662 | INFO | "gpu" : " 17 GB", | 20260508 13:19:56.662 | INFO | "huge" : "", | 20260508 13:19:56.662 | INFO | "huge_shared" : " 29 GB", | 20260508 13:19:56.662 | INFO | "malloc" : " 14 GB", | 20260508 13:19:56.662 | INFO | "pinned" : " 29 GB" | 20260508 13:19:56.662 | INFO | } | 20260508 13:19:56.662 | INFO | Perf report for the last 5.14338 seconds: | 20260508 13:19:56.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:19:56.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.662 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:19:56.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.663 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:19:56.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:19:56.663 | INFO | Uploaded 36GiB to GPU in 2.49773s (14.4131GiB/s) | 20260508 13:19:56.663 | INFO | Downloaded 34GiB to GPU in 2.45463s (13.8514GiB/s) | 20260508 13:20:01.800 | INFO | Analyzing chunk frames = [8415232,8415744) | 20260508 13:20:01.800 | INFO | MemInfo: { | 20260508 13:20:01.800 | INFO | "gpu" : " 17 GB", | 20260508 13:20:01.800 | INFO | "huge" : "", | 20260508 13:20:01.800 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:01.800 | INFO | "malloc" : " 14 GB", | 20260508 13:20:01.800 | INFO | "pinned" : " 29 GB" | 20260508 13:20:01.800 | INFO | } | 20260508 13:20:01.800 | INFO | Perf report for the last 5.1374 seconds: | 20260508 13:20:01.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:01.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.800 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 13:20:01.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.800 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:20:01.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:01.801 | INFO | Uploaded 36GiB to GPU in 2.44503s (14.7237GiB/s) | 20260508 13:20:01.801 | INFO | Downloaded 34GiB to GPU in 2.50862s (13.5533GiB/s) | 20260508 13:20:06.939 | INFO | Analyzing chunk frames = [8415744,8416256) | 20260508 13:20:06.939 | INFO | MemInfo: { | 20260508 13:20:06.939 | INFO | "gpu" : " 17 GB", | 20260508 13:20:06.939 | INFO | "huge" : "", | 20260508 13:20:06.939 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:06.940 | INFO | "malloc" : " 14 GB", | 20260508 13:20:06.940 | INFO | "pinned" : " 29 GB" | 20260508 13:20:06.940 | INFO | } | 20260508 13:20:06.940 | INFO | Perf report for the last 5.13971 seconds: | 20260508 13:20:06.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:06.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:20:06.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:20:06.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:06.940 | INFO | Uploaded 36GiB to GPU in 2.42005s (14.8757GiB/s) | 20260508 13:20:06.940 | INFO | Downloaded 34GiB to GPU in 2.44174s (13.9245GiB/s) | 20260508 13:20:12.072 | INFO | Analyzing chunk frames = [8416256,8416768) | 20260508 13:20:12.072 | INFO | MemInfo: { | 20260508 13:20:12.072 | INFO | "gpu" : " 17 GB", | 20260508 13:20:12.072 | INFO | "huge" : "", | 20260508 13:20:12.072 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:12.072 | INFO | "malloc" : " 14 GB", | 20260508 13:20:12.072 | INFO | "pinned" : " 29 GB" | 20260508 13:20:12.072 | INFO | } | 20260508 13:20:12.073 | INFO | Perf report for the last 5.13288 seconds: | 20260508 13:20:12.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:12.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 13:20:12.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 13:20:12.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:12.073 | INFO | Uploaded 36GiB to GPU in 2.33789s (15.3985GiB/s) | 20260508 13:20:12.073 | INFO | Downloaded 34GiB to GPU in 2.40957s (14.1104GiB/s) | 20260508 13:20:17.216 | INFO | Analyzing chunk frames = [8416768,8417280) | 20260508 13:20:17.216 | INFO | MemInfo: { | 20260508 13:20:17.216 | INFO | "gpu" : " 17 GB", | 20260508 13:20:17.216 | INFO | "huge" : "", | 20260508 13:20:17.216 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:17.216 | INFO | "malloc" : " 14 GB", | 20260508 13:20:17.216 | INFO | "pinned" : " 29 GB" | 20260508 13:20:17.217 | INFO | } | 20260508 13:20:17.217 | INFO | Perf report for the last 5.14404 seconds: | 20260508 13:20:17.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:17.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | ANALYSIS : 64.6%, 3.0, 101.2ms, 0 | 20260508 13:20:17.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260508 13:20:17.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:17.217 | INFO | Uploaded 36GiB to GPU in 2.34959s (15.3218GiB/s) | 20260508 13:20:17.217 | INFO | Downloaded 34GiB to GPU in 2.44391s (13.9121GiB/s) | 20260508 13:20:17.314 | INFO | Starting a round of Baz IO | 20260508 13:20:20.724 | INFO | Populating RTMetrics report | 20260508 13:20:20.724 | INFO | RTMetrics report populated | 20260508 13:20:20.724 | INFO | Opening tmp RTMetrics json file | 20260508 13:20:20.732 | INFO | Serializing RTMetrics to json | 20260508 13:20:20.733 | INFO | Serialization done, swapping tmp file now... | 20260508 13:20:20.733 | INFO | RTMetrics done | 20260508 13:20:20.788 | INFO | Finished a round of Baz IO: Wrote 1.389 GiB in 3.473 seconds | 20260508 13:20:20.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:20:20.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14472% / 0% / 6.75519e-05% | 20260508 13:20:20.788 | INFO | Baz IO overall: | 20260508 13:20:20.788 | INFO | BytesWritten : 3589638279196 | 20260508 13:20:20.788 | INFO | NumEvents : 1721272242338 | 20260508 13:20:22.355 | INFO | Analyzing chunk frames = [8417280,8417792) | 20260508 13:20:22.355 | INFO | MemInfo: { | 20260508 13:20:22.355 | INFO | "gpu" : " 17 GB", | 20260508 13:20:22.355 | INFO | "huge" : "", | 20260508 13:20:22.355 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:22.355 | INFO | "malloc" : " 14 GB", | 20260508 13:20:22.355 | INFO | "pinned" : " 29 GB" | 20260508 13:20:22.355 | INFO | } | 20260508 13:20:22.355 | INFO | Perf report for the last 5.13874 seconds: | 20260508 13:20:22.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:22.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:22.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:22.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:22.355 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:20:22.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:22.355 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:20:22.355 | INFO | PRE_HQ : 65.5%, 1.6, 53.7ms, 0 | 20260508 13:20:22.355 | INFO | BAZWRITER : 66.9%, 2.7, 93.4ms, 0 | 20260508 13:20:22.355 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:20:22.356 | INFO | Uploaded 36GiB to GPU in 2.6053s (13.818GiB/s) | 20260508 13:20:22.356 | INFO | Downloaded 38GiB to GPU in 2.7163s (13.9896GiB/s) | 20260508 13:20:27.495 | INFO | Analyzing chunk frames = [8417792,8418304) | 20260508 13:20:27.495 | INFO | MemInfo: { | 20260508 13:20:27.495 | INFO | "gpu" : " 17 GB", | 20260508 13:20:27.495 | INFO | "huge" : "", | 20260508 13:20:27.495 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:27.495 | INFO | "malloc" : " 14 GB", | 20260508 13:20:27.495 | INFO | "pinned" : " 29 GB" | 20260508 13:20:27.495 | INFO | } | 20260508 13:20:27.495 | INFO | Perf report for the last 5.13988 seconds: | 20260508 13:20:27.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:27.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.495 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:20:27.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.495 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:20:27.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:27.496 | INFO | Uploaded 36GiB to GPU in 2.49602s (14.423GiB/s) | 20260508 13:20:27.496 | INFO | Downloaded 34GiB to GPU in 2.47074s (13.7611GiB/s) | 20260508 13:20:32.634 | INFO | Analyzing chunk frames = [8418304,8418816) | 20260508 13:20:32.634 | INFO | MemInfo: { | 20260508 13:20:32.634 | INFO | "gpu" : " 17 GB", | 20260508 13:20:32.634 | INFO | "huge" : "", | 20260508 13:20:32.634 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:32.634 | INFO | "malloc" : " 14 GB", | 20260508 13:20:32.634 | INFO | "pinned" : " 29 GB" | 20260508 13:20:32.634 | INFO | } | 20260508 13:20:32.634 | INFO | Perf report for the last 5.139 seconds: | 20260508 13:20:32.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:32.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.634 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 13:20:32.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.634 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:20:32.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:32.635 | INFO | Uploaded 36GiB to GPU in 2.51102s (14.3368GiB/s) | 20260508 13:20:32.635 | INFO | Downloaded 34GiB to GPU in 2.44986s (13.8783GiB/s) | 20260508 13:20:37.775 | INFO | Analyzing chunk frames = [8418816,8419328) | 20260508 13:20:37.775 | INFO | MemInfo: { | 20260508 13:20:37.775 | INFO | "gpu" : " 17 GB", | 20260508 13:20:37.775 | INFO | "huge" : "", | 20260508 13:20:37.776 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:37.776 | INFO | "malloc" : " 14 GB", | 20260508 13:20:37.776 | INFO | "pinned" : " 29 GB" | 20260508 13:20:37.776 | INFO | } | 20260508 13:20:37.776 | INFO | Perf report for the last 5.1414 seconds: | 20260508 13:20:37.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:37.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | ANALYSIS : 65.2%, 3.0, 101.2ms, 0 | 20260508 13:20:37.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:20:37.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:37.776 | INFO | Uploaded 36GiB to GPU in 2.39389s (15.0383GiB/s) | 20260508 13:20:37.776 | INFO | Downloaded 34GiB to GPU in 2.45814s (13.8316GiB/s) | 20260508 13:20:42.911 | INFO | Analyzing chunk frames = [8419328,8419840) | 20260508 13:20:42.911 | INFO | MemInfo: { | 20260508 13:20:42.911 | INFO | "gpu" : " 17 GB", | 20260508 13:20:42.911 | INFO | "huge" : "", | 20260508 13:20:42.911 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:42.911 | INFO | "malloc" : " 14 GB", | 20260508 13:20:42.911 | INFO | "pinned" : " 29 GB" | 20260508 13:20:42.911 | INFO | } | 20260508 13:20:42.911 | INFO | Perf report for the last 5.13548 seconds: | 20260508 13:20:42.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:42.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.911 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 13:20:42.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.911 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:20:42.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:42.912 | INFO | Uploaded 36GiB to GPU in 2.44192s (14.7425GiB/s) | 20260508 13:20:42.912 | INFO | Downloaded 34GiB to GPU in 2.52398s (13.4708GiB/s) | 20260508 13:20:48.050 | INFO | Analyzing chunk frames = [8419840,8420352) | 20260508 13:20:48.050 | INFO | MemInfo: { | 20260508 13:20:48.050 | INFO | "gpu" : " 17 GB", | 20260508 13:20:48.050 | INFO | "huge" : "", | 20260508 13:20:48.050 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:48.050 | INFO | "malloc" : " 14 GB", | 20260508 13:20:48.050 | INFO | "pinned" : " 29 GB" | 20260508 13:20:48.050 | INFO | } | 20260508 13:20:48.050 | INFO | Perf report for the last 5.13916 seconds: | 20260508 13:20:48.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:48.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.050 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:20:48.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.050 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:20:48.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:48.051 | INFO | Uploaded 36GiB to GPU in 2.37341s (15.168GiB/s) | 20260508 13:20:48.051 | INFO | Downloaded 34GiB to GPU in 2.43677s (13.9529GiB/s) | 20260508 13:20:53.189 | INFO | Analyzing chunk frames = [8420352,8420864) | 20260508 13:20:53.189 | INFO | MemInfo: { | 20260508 13:20:53.189 | INFO | "gpu" : " 17 GB", | 20260508 13:20:53.189 | INFO | "huge" : "", | 20260508 13:20:53.189 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:53.189 | INFO | "malloc" : " 14 GB", | 20260508 13:20:53.189 | INFO | "pinned" : " 29 GB" | 20260508 13:20:53.190 | INFO | } | 20260508 13:20:53.190 | INFO | Perf report for the last 5.13924 seconds: | 20260508 13:20:53.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:53.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 13:20:53.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:20:53.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:53.190 | INFO | Uploaded 36GiB to GPU in 2.48638s (14.4789GiB/s) | 20260508 13:20:53.190 | INFO | Downloaded 34GiB to GPU in 2.46612s (13.7869GiB/s) | 20260508 13:20:58.330 | INFO | Analyzing chunk frames = [8420864,8421376) | 20260508 13:20:58.330 | INFO | MemInfo: { | 20260508 13:20:58.330 | INFO | "gpu" : " 17 GB", | 20260508 13:20:58.330 | INFO | "huge" : "", | 20260508 13:20:58.330 | INFO | "huge_shared" : " 29 GB", | 20260508 13:20:58.330 | INFO | "malloc" : " 14 GB", | 20260508 13:20:58.330 | INFO | "pinned" : " 29 GB" | 20260508 13:20:58.330 | INFO | } | 20260508 13:20:58.330 | INFO | Perf report for the last 5.14014 seconds: | 20260508 13:20:58.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:20:58.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260508 13:20:58.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 13:20:58.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:20:58.330 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:20:58.330 | INFO | Total mem usage: 16.7 GiB | 20260508 13:20:58.330 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:20:58.330 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.8%) | 20260508 13:20:58.330 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:20:58.330 | INFO | High Water Breakdown: | 20260508 13:20:58.330 | INFO | 59.7% : 8.68 GiB ( 84.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:20:58.330 | INFO | 40.1% : 5.83 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:20:58.330 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:20:58.330 | INFO | ------------------------------------------ | 20260508 13:20:58.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:20:58.330 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:20:58.330 | INFO | Total mem usage: 26.5 GiB | 20260508 13:20:58.330 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:20:58.330 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:20:58.330 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:20:58.330 | INFO | High Water Breakdown: | 20260508 13:20:58.330 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:20:58.330 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:20:58.330 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:20:58.330 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:20:58.330 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:20:58.330 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:20:58.330 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:20:58.330 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:20:58.330 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:20:58.330 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:20:58.330 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:20:58.330 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:20:58.330 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:20:58.330 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:20:58.330 | INFO | ------------------------------------------ | 20260508 13:20:58.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:20:58.330 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:20:58.330 | INFO | Total mem usage: 27.8 GiB | 20260508 13:20:58.330 | INFO | Uploaded 36GiB to GPU in 2.36679s (15.2105GiB/s) | 20260508 13:20:58.330 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:20:58.330 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:20:58.330 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:20:58.330 | INFO | High Water Breakdown: | 20260508 13:20:58.330 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:20:58.330 | INFO | ------------------------------------------ | 20260508 13:20:58.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:20:58.330 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:20:58.330 | INFO | Total mem usage: 15.7 GiB | 20260508 13:20:58.330 | INFO | Downloaded 34GiB to GPU in 2.44497s (13.9061GiB/s) | 20260508 13:20:58.330 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:20:58.330 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:20:58.330 | INFO | Unused Cache: 862 MiB (Waste: 5.37%) | 20260508 13:20:58.330 | INFO | High Water Breakdown: | 20260508 13:20:58.330 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:20:58.330 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:20:58.330 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:20:58.330 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:20:58.330 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:20:58.330 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:20:58.330 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:20:58.330 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:20:58.330 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:20:58.330 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:20:58.330 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:20:58.330 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:20:58.330 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:20:58.330 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:20:58.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:20:58.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:20:58.330 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:20:58.330 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:20:58.330 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:20:58.330 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:20:58.330 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:20:58.330 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:20:58.330 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:20:58.330 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:20:58.330 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:20:58.330 | INFO | ------------------------------------------ | 20260508 13:20:58.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:20:58.330 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:20:58.331 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:20:58.331 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:20:58.331 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.38e+01 / 4.35e-01 | 20260508 13:20:58.331 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.20e-02 / 4.60e-03 | 20260508 13:20:58.331 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.46e+00, 5.99e-01 / 1.99e-01 | 20260508 13:20:58.331 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.43e+00, 2.11e-01 / 1.22e-01 | 20260508 13:20:58.331 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.51e+01, 3.56e+00 / 9.86e-01 | 20260508 13:20:58.331 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.85e+00 / 1.48e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.31e+00 / 2.41e+00, 4.02e-01 / 2.72e-01 | 20260508 13:20:58.331 | INFO | PerfReport: 2.93%, Metrics -- 9.58e-01 / 1.11e+00, 2.37e-01 / 1.21e-01 | 20260508 13:20:58.331 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.22e-01, 4.56e-02 / 2.62e-02 | 20260508 13:20:58.331 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:20:58.331 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:20:58.331 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:20:58.331 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 24.46%, Upload -- 2.56e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 24.75%, Download -- 2.59e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 31.15%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 2.23%, LockUpload -- 2.34e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 2.52%, LockDownload -- 2.65e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 14.88%, LockCompute -- 1.56e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:20:58.331 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.69e-02 / 0.00e+00 | 20260508 13:20:58.427 | INFO | Starting a round of Baz IO | 20260508 13:21:01.839 | INFO | Populating RTMetrics report | 20260508 13:21:01.839 | INFO | RTMetrics report populated | 20260508 13:21:01.839 | INFO | Opening tmp RTMetrics json file | 20260508 13:21:01.844 | INFO | Serializing RTMetrics to json | 20260508 13:21:01.846 | INFO | Serialization done, swapping tmp file now... | 20260508 13:21:01.847 | INFO | RTMetrics done | 20260508 13:21:01.903 | INFO | Finished a round of Baz IO: Wrote 1.388 GiB in 3.476 seconds | 20260508 13:21:01.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:21:01.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14315% / 3.97364e-06% / 9.13938e-05% | 20260508 13:21:01.903 | INFO | Baz IO overall: | 20260508 13:21:01.903 | INFO | BytesWritten : 3591128689514 | 20260508 13:21:01.903 | INFO | NumEvents : 1721969930401 | 20260508 13:21:03.468 | INFO | Analyzing chunk frames = [8421376,8421888) | 20260508 13:21:03.468 | INFO | MemInfo: { | 20260508 13:21:03.468 | INFO | "gpu" : " 17 GB", | 20260508 13:21:03.468 | INFO | "huge" : "", | 20260508 13:21:03.468 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:03.468 | INFO | "malloc" : " 14 GB", | 20260508 13:21:03.468 | INFO | "pinned" : " 29 GB" | 20260508 13:21:03.468 | INFO | } | 20260508 13:21:03.468 | INFO | Perf report for the last 5.13798 seconds: | 20260508 13:21:03.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:03.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:03.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:03.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:03.468 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:21:03.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:03.468 | INFO | RT_METRICS : 38.1%, 1.0, 19.9ms, 0 | 20260508 13:21:03.468 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 13:21:03.468 | INFO | BAZWRITER : 66.9%, 2.4, 84.0ms, 0 | 20260508 13:21:03.468 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:21:03.468 | INFO | Uploaded 36GiB to GPU in 2.61414s (13.7712GiB/s) | 20260508 13:21:03.468 | INFO | Downloaded 38GiB to GPU in 2.7209s (13.966GiB/s) | 20260508 13:21:08.612 | INFO | Analyzing chunk frames = [8421888,8422400) | 20260508 13:21:08.612 | INFO | MemInfo: { | 20260508 13:21:08.612 | INFO | "gpu" : " 17 GB", | 20260508 13:21:08.612 | INFO | "huge" : "", | 20260508 13:21:08.612 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:08.612 | INFO | "malloc" : " 14 GB", | 20260508 13:21:08.612 | INFO | "pinned" : " 29 GB" | 20260508 13:21:08.612 | INFO | } | 20260508 13:21:08.612 | INFO | Perf report for the last 5.14463 seconds: | 20260508 13:21:08.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:08.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.612 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:21:08.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.612 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:21:08.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:08.613 | INFO | Uploaded 36GiB to GPU in 2.47166s (14.5651GiB/s) | 20260508 13:21:08.613 | INFO | Downloaded 34GiB to GPU in 2.46641s (13.7852GiB/s) | 20260508 13:21:13.746 | INFO | Analyzing chunk frames = [8422400,8422912) | 20260508 13:21:13.746 | INFO | MemInfo: { | 20260508 13:21:13.746 | INFO | "gpu" : " 17 GB", | 20260508 13:21:13.746 | INFO | "huge" : "", | 20260508 13:21:13.746 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:13.746 | INFO | "malloc" : " 14 GB", | 20260508 13:21:13.746 | INFO | "pinned" : " 29 GB" | 20260508 13:21:13.746 | INFO | } | 20260508 13:21:13.746 | INFO | Perf report for the last 5.13381 seconds: | 20260508 13:21:13.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:13.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | ANALYSIS : 66.2%, 3.0, 102.8ms, 0 | 20260508 13:21:13.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:21:13.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:13.746 | INFO | Uploaded 36GiB to GPU in 2.51674s (14.3042GiB/s) | 20260508 13:21:13.746 | INFO | Downloaded 34GiB to GPU in 2.46319s (13.8033GiB/s) | 20260508 13:21:18.883 | INFO | Analyzing chunk frames = [8422912,8423424) | 20260508 13:21:18.883 | INFO | MemInfo: { | 20260508 13:21:18.883 | INFO | "gpu" : " 17 GB", | 20260508 13:21:18.883 | INFO | "huge" : "", | 20260508 13:21:18.883 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:18.883 | INFO | "malloc" : " 14 GB", | 20260508 13:21:18.883 | INFO | "pinned" : " 29 GB" | 20260508 13:21:18.883 | INFO | } | 20260508 13:21:18.883 | INFO | Perf report for the last 5.13691 seconds: | 20260508 13:21:18.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:18.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 13:21:18.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:21:18.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:18.883 | INFO | Uploaded 36GiB to GPU in 2.51769s (14.2988GiB/s) | 20260508 13:21:18.884 | INFO | Downloaded 34GiB to GPU in 2.43575s (13.9588GiB/s) | 20260508 13:21:24.028 | INFO | Analyzing chunk frames = [8423424,8423936) | 20260508 13:21:24.028 | INFO | MemInfo: { | 20260508 13:21:24.028 | INFO | "gpu" : " 17 GB", | 20260508 13:21:24.028 | INFO | "huge" : "", | 20260508 13:21:24.028 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:24.028 | INFO | "malloc" : " 14 GB", | 20260508 13:21:24.028 | INFO | "pinned" : " 29 GB" | 20260508 13:21:24.028 | INFO | } | 20260508 13:21:24.028 | INFO | Perf report for the last 5.14509 seconds: | 20260508 13:21:24.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:24.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.028 | INFO | ANALYSIS : 66.2%, 3.0, 103.5ms, 0 | 20260508 13:21:24.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.028 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260508 13:21:24.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:24.029 | INFO | Uploaded 36GiB to GPU in 2.47482s (14.5465GiB/s) | 20260508 13:21:24.029 | INFO | Downloaded 34GiB to GPU in 2.50956s (13.5482GiB/s) | 20260508 13:21:29.170 | INFO | Analyzing chunk frames = [8423936,8424448) | 20260508 13:21:29.170 | INFO | MemInfo: { | 20260508 13:21:29.170 | INFO | "gpu" : " 17 GB", | 20260508 13:21:29.170 | INFO | "huge" : "", | 20260508 13:21:29.170 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:29.170 | INFO | "malloc" : " 14 GB", | 20260508 13:21:29.170 | INFO | "pinned" : " 29 GB" | 20260508 13:21:29.170 | INFO | } | 20260508 13:21:29.170 | INFO | Perf report for the last 5.14156 seconds: | 20260508 13:21:29.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:29.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:21:29.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:21:29.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:29.170 | INFO | Uploaded 36GiB to GPU in 2.43686s (14.7731GiB/s) | 20260508 13:21:29.170 | INFO | Downloaded 34GiB to GPU in 2.43301s (13.9744GiB/s) | 20260508 13:21:34.316 | INFO | Analyzing chunk frames = [8424448,8424960) | 20260508 13:21:34.316 | INFO | MemInfo: { | 20260508 13:21:34.316 | INFO | "gpu" : " 17 GB", | 20260508 13:21:34.316 | INFO | "huge" : "", | 20260508 13:21:34.316 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:34.316 | INFO | "malloc" : " 14 GB", | 20260508 13:21:34.316 | INFO | "pinned" : " 29 GB" | 20260508 13:21:34.316 | INFO | } | 20260508 13:21:34.316 | INFO | Perf report for the last 5.14594 seconds: | 20260508 13:21:34.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:34.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | ANALYSIS : 65.5%, 3.0, 102.3ms, 0 | 20260508 13:21:34.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:21:34.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:34.316 | INFO | Uploaded 36GiB to GPU in 2.48175s (14.5059GiB/s) | 20260508 13:21:34.316 | INFO | Downloaded 34GiB to GPU in 2.44867s (13.8851GiB/s) | 20260508 13:21:39.447 | INFO | Analyzing chunk frames = [8424960,8425472) | 20260508 13:21:39.447 | INFO | MemInfo: { | 20260508 13:21:39.447 | INFO | "gpu" : " 17 GB", | 20260508 13:21:39.447 | INFO | "huge" : "", | 20260508 13:21:39.447 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:39.447 | INFO | "malloc" : " 14 GB", | 20260508 13:21:39.447 | INFO | "pinned" : " 29 GB" | 20260508 13:21:39.447 | INFO | } | 20260508 13:21:39.447 | INFO | Perf report for the last 5.13121 seconds: | 20260508 13:21:39.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:39.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.447 | INFO | ANALYSIS : 64.6%, 3.0, 100.7ms, 0 | 20260508 13:21:39.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.447 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:21:39.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:39.448 | INFO | Uploaded 36GiB to GPU in 2.35405s (15.2928GiB/s) | 20260508 13:21:39.448 | INFO | Downloaded 34GiB to GPU in 2.38508s (14.2553GiB/s) | 20260508 13:21:39.544 | INFO | Starting a round of Baz IO | 20260508 13:21:42.946 | INFO | Populating RTMetrics report | 20260508 13:21:42.946 | INFO | RTMetrics report populated | 20260508 13:21:42.946 | INFO | Opening tmp RTMetrics json file | 20260508 13:21:42.954 | INFO | Serializing RTMetrics to json | 20260508 13:21:42.955 | INFO | Serialization done, swapping tmp file now... | 20260508 13:21:42.955 | INFO | RTMetrics done | 20260508 13:21:43.014 | INFO | Finished a round of Baz IO: Wrote 1.389 GiB in 3.47 seconds | 20260508 13:21:43.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:21:43.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144204% / 0% / 6.75519e-05% | 20260508 13:21:43.014 | INFO | Baz IO overall: | 20260508 13:21:43.014 | INFO | BytesWritten : 3592619592914 | 20260508 13:21:43.014 | INFO | NumEvents : 1722667914508 | 20260508 13:21:44.580 | INFO | Analyzing chunk frames = [8425472,8425984) | 20260508 13:21:44.580 | INFO | MemInfo: { | 20260508 13:21:44.580 | INFO | "gpu" : " 17 GB", | 20260508 13:21:44.580 | INFO | "huge" : "", | 20260508 13:21:44.580 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:44.581 | INFO | "malloc" : " 14 GB", | 20260508 13:21:44.581 | INFO | "pinned" : " 29 GB" | 20260508 13:21:44.581 | INFO | } | 20260508 13:21:44.581 | INFO | Perf report for the last 5.13366 seconds: | 20260508 13:21:44.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:44.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:44.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:44.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:44.581 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:21:44.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:44.581 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:21:44.581 | INFO | PRE_HQ : 65.2%, 1.6, 53.5ms, 0 | 20260508 13:21:44.581 | INFO | BAZWRITER : 66.8%, 2.4, 83.0ms, 0 | 20260508 13:21:44.581 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:21:44.581 | INFO | Uploaded 36GiB to GPU in 2.64002s (13.6362GiB/s) | 20260508 13:21:44.581 | INFO | Downloaded 38GiB to GPU in 2.71663s (13.9879GiB/s) | 20260508 13:21:49.725 | INFO | Analyzing chunk frames = [8425984,8426496) | 20260508 13:21:49.725 | INFO | MemInfo: { | 20260508 13:21:49.725 | INFO | "gpu" : " 17 GB", | 20260508 13:21:49.725 | INFO | "huge" : "", | 20260508 13:21:49.725 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:49.725 | INFO | "malloc" : " 14 GB", | 20260508 13:21:49.725 | INFO | "pinned" : " 29 GB" | 20260508 13:21:49.725 | INFO | } | 20260508 13:21:49.725 | INFO | Perf report for the last 5.14483 seconds: | 20260508 13:21:49.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:49.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.726 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:21:49.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.726 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:21:49.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:49.726 | INFO | Uploaded 36GiB to GPU in 2.45195s (14.6822GiB/s) | 20260508 13:21:49.726 | INFO | Downloaded 34GiB to GPU in 2.4688s (13.7719GiB/s) | 20260508 13:21:54.866 | INFO | Analyzing chunk frames = [8426496,8427008) | 20260508 13:21:54.866 | INFO | MemInfo: { | 20260508 13:21:54.866 | INFO | "gpu" : " 17 GB", | 20260508 13:21:54.866 | INFO | "huge" : "", | 20260508 13:21:54.866 | INFO | "huge_shared" : " 29 GB", | 20260508 13:21:54.866 | INFO | "malloc" : " 14 GB", | 20260508 13:21:54.866 | INFO | "pinned" : " 29 GB" | 20260508 13:21:54.866 | INFO | } | 20260508 13:21:54.866 | INFO | Perf report for the last 5.14064 seconds: | 20260508 13:21:54.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:21:54.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.866 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:21:54.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.866 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:21:54.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:21:54.867 | INFO | Uploaded 36GiB to GPU in 2.45406s (14.6696GiB/s) | 20260508 13:21:54.867 | INFO | Downloaded 34GiB to GPU in 2.45137s (13.8698GiB/s) | 20260508 13:22:00.004 | INFO | Analyzing chunk frames = [8427008,8427520) | 20260508 13:22:00.004 | INFO | MemInfo: { | 20260508 13:22:00.005 | INFO | "gpu" : " 17 GB", | 20260508 13:22:00.005 | INFO | "huge" : "", | 20260508 13:22:00.005 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:00.005 | INFO | "malloc" : " 14 GB", | 20260508 13:22:00.005 | INFO | "pinned" : " 29 GB" | 20260508 13:22:00.005 | INFO | } | 20260508 13:22:00.005 | INFO | Perf report for the last 5.13863 seconds: | 20260508 13:22:00.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:00.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:22:00.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 13:22:00.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:00.005 | INFO | Uploaded 36GiB to GPU in 2.4209s (14.8705GiB/s) | 20260508 13:22:00.005 | INFO | Downloaded 34GiB to GPU in 2.49789s (13.6115GiB/s) | 20260508 13:22:05.137 | INFO | Analyzing chunk frames = [8427520,8428032) | 20260508 13:22:05.137 | INFO | MemInfo: { | 20260508 13:22:05.137 | INFO | "gpu" : " 17 GB", | 20260508 13:22:05.137 | INFO | "huge" : "", | 20260508 13:22:05.137 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:05.137 | INFO | "malloc" : " 14 GB", | 20260508 13:22:05.137 | INFO | "pinned" : " 29 GB" | 20260508 13:22:05.137 | INFO | } | 20260508 13:22:05.137 | INFO | Perf report for the last 5.1321 seconds: | 20260508 13:22:05.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:05.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:22:05.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260508 13:22:05.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:05.137 | INFO | Uploaded 36GiB to GPU in 2.43283s (14.7976GiB/s) | 20260508 13:22:05.137 | INFO | Downloaded 34GiB to GPU in 2.52997s (13.4389GiB/s) | 20260508 13:22:10.283 | INFO | Analyzing chunk frames = [8428032,8428544) | 20260508 13:22:10.283 | INFO | MemInfo: { | 20260508 13:22:10.283 | INFO | "gpu" : " 17 GB", | 20260508 13:22:10.283 | INFO | "huge" : "", | 20260508 13:22:10.283 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:10.283 | INFO | "malloc" : " 14 GB", | 20260508 13:22:10.283 | INFO | "pinned" : " 29 GB" | 20260508 13:22:10.283 | INFO | } | 20260508 13:22:10.283 | INFO | Perf report for the last 5.14602 seconds: | 20260508 13:22:10.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:10.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | ANALYSIS : 65.0%, 3.0, 101.2ms, 0 | 20260508 13:22:10.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:22:10.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:10.283 | INFO | Uploaded 36GiB to GPU in 2.34763s (15.3346GiB/s) | 20260508 13:22:10.283 | INFO | Downloaded 34GiB to GPU in 2.45043s (13.8751GiB/s) | 20260508 13:22:15.421 | INFO | Analyzing chunk frames = [8428544,8429056) | 20260508 13:22:15.421 | INFO | MemInfo: { | 20260508 13:22:15.421 | INFO | "gpu" : " 17 GB", | 20260508 13:22:15.421 | INFO | "huge" : "", | 20260508 13:22:15.421 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:15.421 | INFO | "malloc" : " 14 GB", | 20260508 13:22:15.421 | INFO | "pinned" : " 29 GB" | 20260508 13:22:15.421 | INFO | } | 20260508 13:22:15.421 | INFO | Perf report for the last 5.13793 seconds: | 20260508 13:22:15.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:15.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 13:22:15.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:22:15.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:15.421 | INFO | Uploaded 36GiB to GPU in 2.30499s (15.6183GiB/s) | 20260508 13:22:15.422 | INFO | Downloaded 34GiB to GPU in 2.43257s (13.977GiB/s) | 20260508 13:22:20.556 | INFO | Analyzing chunk frames = [8429056,8429568) | 20260508 13:22:20.556 | INFO | MemInfo: { | 20260508 13:22:20.556 | INFO | "gpu" : " 17 GB", | 20260508 13:22:20.556 | INFO | "huge" : "", | 20260508 13:22:20.556 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:20.556 | INFO | "malloc" : " 14 GB", | 20260508 13:22:20.556 | INFO | "pinned" : " 29 GB" | 20260508 13:22:20.556 | INFO | } | 20260508 13:22:20.556 | INFO | Perf report for the last 5.13542 seconds: | 20260508 13:22:20.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:20.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260508 13:22:20.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:22:20.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:20.556 | INFO | Uploaded 36GiB to GPU in 2.32978s (15.4521GiB/s) | 20260508 13:22:20.556 | INFO | Downloaded 34GiB to GPU in 2.45993s (13.8215GiB/s) | 20260508 13:22:20.557 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:22:20.557 | INFO | Total mem usage: 16.7 GiB | 20260508 13:22:20.557 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:22:20.557 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:22:20.557 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:22:20.557 | INFO | High Water Breakdown: | 20260508 13:22:20.557 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:22:20.557 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:22:20.557 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:22:20.557 | INFO | ------------------------------------------ | 20260508 13:22:20.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:22:20.557 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:22:20.557 | INFO | Total mem usage: 26.5 GiB | 20260508 13:22:20.557 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:22:20.557 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:22:20.557 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:22:20.557 | INFO | High Water Breakdown: | 20260508 13:22:20.557 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:22:20.557 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:22:20.557 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:22:20.557 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:22:20.557 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:22:20.557 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:22:20.557 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:22:20.557 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:22:20.557 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:22:20.557 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:22:20.557 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:22:20.557 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:22:20.557 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:22:20.557 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:22:20.557 | INFO | ------------------------------------------ | 20260508 13:22:20.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:22:20.557 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:22:20.557 | INFO | Total mem usage: 27.8 GiB | 20260508 13:22:20.557 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:22:20.557 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:22:20.557 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:22:20.557 | INFO | High Water Breakdown: | 20260508 13:22:20.557 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:22:20.557 | INFO | ------------------------------------------ | 20260508 13:22:20.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:22:20.557 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:22:20.557 | INFO | Total mem usage: 15.7 GiB | 20260508 13:22:20.557 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:22:20.557 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:22:20.557 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:22:20.557 | INFO | High Water Breakdown: | 20260508 13:22:20.557 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:22:20.557 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:22:20.557 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:22:20.557 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:22:20.557 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:22:20.557 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:22:20.557 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:22:20.557 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:22:20.557 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:22:20.557 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:22:20.557 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:22:20.557 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:22:20.557 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:22:20.557 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:22:20.557 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:22:20.557 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:22:20.557 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:22:20.557 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:22:20.557 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:22:20.557 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:22:20.557 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:22:20.557 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:22:20.557 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:22:20.557 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:22:20.557 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:22:20.557 | INFO | ------------------------------------------ | 20260508 13:22:20.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:22:20.557 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:22:20.557 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:22:20.557 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:22:20.557 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 4.37e-01 | 20260508 13:22:20.557 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.02e-01, 1.09e-02 / 5.01e-03 | 20260508 13:22:20.557 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.45e+00, 5.92e-01 / 1.88e-01 | 20260508 13:22:20.557 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.42e+00, 2.10e-01 / 1.01e-01 | 20260508 13:22:20.557 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.49e+01, 3.64e+00 / 8.80e-01 | 20260508 13:22:20.557 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.83e+00 / 1.20e+00 | 20260508 13:22:20.557 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.31e+00 / 2.36e+00, 4.03e-01 / 2.61e-01 | 20260508 13:22:20.557 | INFO | PerfReport: 2.93%, Metrics -- 9.57e-01 / 1.06e+00, 2.33e-01 / 9.30e-02 | 20260508 13:22:20.557 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.13e-01, 4.58e-02 / 2.48e-02 | 20260508 13:22:20.557 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:22:20.557 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:22:20.557 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:22:20.557 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:22:20.557 | INFO | PerfReport: 24.41%, Upload -- 2.56e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 13:22:20.557 | INFO | PerfReport: 24.80%, Download -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:22:20.557 | INFO | PerfReport: 31.20%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:22:20.557 | INFO | PerfReport: 2.11%, LockUpload -- 2.20e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260508 13:22:20.557 | INFO | PerfReport: 2.59%, LockDownload -- 2.72e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 13:22:20.558 | INFO | PerfReport: 14.88%, LockCompute -- 1.56e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260508 13:22:20.558 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260508 13:22:20.653 | INFO | Starting a round of Baz IO | 20260508 13:22:24.053 | INFO | Populating RTMetrics report | 20260508 13:22:24.054 | INFO | RTMetrics report populated | 20260508 13:22:24.054 | INFO | Opening tmp RTMetrics json file | 20260508 13:22:24.065 | INFO | Serializing RTMetrics to json | 20260508 13:22:24.066 | INFO | Serialization done, swapping tmp file now... | 20260508 13:22:24.066 | INFO | RTMetrics done | 20260508 13:22:24.124 | INFO | Finished a round of Baz IO: Wrote 1.388 GiB in 3.471 seconds | 20260508 13:22:24.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:22:24.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143047% / 0% / 5.96046e-05% | 20260508 13:22:24.124 | INFO | Baz IO overall: | 20260508 13:22:24.124 | INFO | BytesWritten : 3594110152785 | 20260508 13:22:24.124 | INFO | NumEvents : 1723365687374 | 20260508 13:22:25.695 | INFO | Analyzing chunk frames = [8429568,8430080) | 20260508 13:22:25.695 | INFO | MemInfo: { | 20260508 13:22:25.695 | INFO | "gpu" : " 17 GB", | 20260508 13:22:25.695 | INFO | "huge" : "", | 20260508 13:22:25.695 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:25.695 | INFO | "malloc" : " 14 GB", | 20260508 13:22:25.695 | INFO | "pinned" : " 29 GB" | 20260508 13:22:25.695 | INFO | } | 20260508 13:22:25.695 | INFO | Perf report for the last 5.13893 seconds: | 20260508 13:22:25.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:25.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:25.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:25.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:25.695 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:22:25.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:25.695 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:22:25.695 | INFO | PRE_HQ : 65.3%, 1.6, 53.4ms, 0 | 20260508 13:22:25.695 | INFO | BAZWRITER : 66.8%, 2.5, 84.2ms, 0 | 20260508 13:22:25.695 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:22:25.696 | INFO | Uploaded 36GiB to GPU in 2.62149s (13.7327GiB/s) | 20260508 13:22:25.696 | INFO | Downloaded 38GiB to GPU in 2.71432s (13.9998GiB/s) | 20260508 13:22:30.834 | INFO | Analyzing chunk frames = [8430080,8430592) | 20260508 13:22:30.834 | INFO | MemInfo: { | 20260508 13:22:30.834 | INFO | "gpu" : " 17 GB", | 20260508 13:22:30.834 | INFO | "huge" : "", | 20260508 13:22:30.834 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:30.834 | INFO | "malloc" : " 14 GB", | 20260508 13:22:30.835 | INFO | "pinned" : " 29 GB" | 20260508 13:22:30.835 | INFO | } | 20260508 13:22:30.835 | INFO | Perf report for the last 5.13935 seconds: | 20260508 13:22:30.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:30.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:22:30.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:22:30.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:30.835 | INFO | Uploaded 36GiB to GPU in 2.4769s (14.5343GiB/s) | 20260508 13:22:30.835 | INFO | Downloaded 34GiB to GPU in 2.47744s (13.7239GiB/s) | 20260508 13:22:35.983 | INFO | Analyzing chunk frames = [8430592,8431104) | 20260508 13:22:35.983 | INFO | MemInfo: { | 20260508 13:22:35.983 | INFO | "gpu" : " 17 GB", | 20260508 13:22:35.983 | INFO | "huge" : "", | 20260508 13:22:35.983 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:35.983 | INFO | "malloc" : " 14 GB", | 20260508 13:22:35.984 | INFO | "pinned" : " 29 GB" | 20260508 13:22:35.984 | INFO | } | 20260508 13:22:35.984 | INFO | Perf report for the last 5.14904 seconds: | 20260508 13:22:35.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:35.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:22:35.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 13:22:35.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:35.984 | INFO | Uploaded 36GiB to GPU in 2.49475s (14.4303GiB/s) | 20260508 13:22:35.984 | INFO | Downloaded 34GiB to GPU in 2.44549s (13.9031GiB/s) | 20260508 13:22:41.124 | INFO | Analyzing chunk frames = [8431104,8431616) | 20260508 13:22:41.124 | INFO | MemInfo: { | 20260508 13:22:41.124 | INFO | "gpu" : " 17 GB", | 20260508 13:22:41.124 | INFO | "huge" : "", | 20260508 13:22:41.124 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:41.124 | INFO | "malloc" : " 14 GB", | 20260508 13:22:41.124 | INFO | "pinned" : " 29 GB" | 20260508 13:22:41.124 | INFO | } | 20260508 13:22:41.125 | INFO | Perf report for the last 5.14093 seconds: | 20260508 13:22:41.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:41.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 13:22:41.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:22:41.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:41.125 | INFO | Uploaded 36GiB to GPU in 2.38789s (15.0761GiB/s) | 20260508 13:22:41.125 | INFO | Downloaded 34GiB to GPU in 2.46985s (13.766GiB/s) | 20260508 13:22:46.260 | INFO | Analyzing chunk frames = [8431616,8432128) | 20260508 13:22:46.260 | INFO | MemInfo: { | 20260508 13:22:46.260 | INFO | "gpu" : " 17 GB", | 20260508 13:22:46.260 | INFO | "huge" : "", | 20260508 13:22:46.261 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:46.261 | INFO | "malloc" : " 14 GB", | 20260508 13:22:46.261 | INFO | "pinned" : " 29 GB" | 20260508 13:22:46.261 | INFO | } | 20260508 13:22:46.261 | INFO | Perf report for the last 5.13628 seconds: | 20260508 13:22:46.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:46.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:22:46.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:22:46.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:46.261 | INFO | Uploaded 36GiB to GPU in 2.40054s (14.9966GiB/s) | 20260508 13:22:46.261 | INFO | Downloaded 34GiB to GPU in 2.51137s (13.5384GiB/s) | 20260508 13:22:51.391 | INFO | Analyzing chunk frames = [8432128,8432640) | 20260508 13:22:51.391 | INFO | MemInfo: { | 20260508 13:22:51.391 | INFO | "gpu" : " 17 GB", | 20260508 13:22:51.391 | INFO | "huge" : "", | 20260508 13:22:51.391 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:51.391 | INFO | "malloc" : " 14 GB", | 20260508 13:22:51.391 | INFO | "pinned" : " 29 GB" | 20260508 13:22:51.391 | INFO | } | 20260508 13:22:51.392 | INFO | Perf report for the last 5.13074 seconds: | 20260508 13:22:51.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:51.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 13:22:51.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260508 13:22:51.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:51.392 | INFO | Uploaded 36GiB to GPU in 2.47176s (14.5645GiB/s) | 20260508 13:22:51.392 | INFO | Downloaded 34GiB to GPU in 2.44258s (13.9197GiB/s) | 20260508 13:22:56.533 | INFO | Analyzing chunk frames = [8432640,8433152) | 20260508 13:22:56.533 | INFO | MemInfo: { | 20260508 13:22:56.533 | INFO | "gpu" : " 17 GB", | 20260508 13:22:56.533 | INFO | "huge" : "", | 20260508 13:22:56.534 | INFO | "huge_shared" : " 29 GB", | 20260508 13:22:56.534 | INFO | "malloc" : " 14 GB", | 20260508 13:22:56.534 | INFO | "pinned" : " 29 GB" | 20260508 13:22:56.534 | INFO | } | 20260508 13:22:56.534 | INFO | Perf report for the last 5.14227 seconds: | 20260508 13:22:56.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:22:56.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.534 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:22:56.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.534 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260508 13:22:56.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:22:56.535 | INFO | Uploaded 36GiB to GPU in 2.34899s (15.3257GiB/s) | 20260508 13:22:56.535 | INFO | Downloaded 34GiB to GPU in 2.40683s (14.1265GiB/s) | 20260508 13:23:01.677 | INFO | Analyzing chunk frames = [8433152,8433664) | 20260508 13:23:01.677 | INFO | MemInfo: { | 20260508 13:23:01.677 | INFO | "gpu" : " 17 GB", | 20260508 13:23:01.677 | INFO | "huge" : "", | 20260508 13:23:01.678 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:01.678 | INFO | "malloc" : " 14 GB", | 20260508 13:23:01.678 | INFO | "pinned" : " 29 GB" | 20260508 13:23:01.678 | INFO | } | 20260508 13:23:01.678 | INFO | Perf report for the last 5.14437 seconds: | 20260508 13:23:01.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:01.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.678 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 13:23:01.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.678 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260508 13:23:01.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:01.679 | INFO | Uploaded 36GiB to GPU in 2.40555s (14.9654GiB/s) | 20260508 13:23:01.679 | INFO | Downloaded 34GiB to GPU in 2.43384s (13.9697GiB/s) | 20260508 13:23:01.792 | INFO | Starting a round of Baz IO | 20260508 13:23:05.201 | INFO | Populating RTMetrics report | 20260508 13:23:05.201 | INFO | RTMetrics report populated | 20260508 13:23:05.201 | INFO | Opening tmp RTMetrics json file | 20260508 13:23:05.209 | INFO | Serializing RTMetrics to json | 20260508 13:23:05.210 | INFO | Serialization done, swapping tmp file now... | 20260508 13:23:05.210 | INFO | RTMetrics done | 20260508 13:23:05.266 | INFO | Finished a round of Baz IO: Wrote 1.391 GiB in 3.474 seconds | 20260508 13:23:05.266 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:23:05.266 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143488% / 0% / 9.53674e-05% | 20260508 13:23:05.266 | INFO | Baz IO overall: | 20260508 13:23:05.266 | INFO | BytesWritten : 3595604073184 | 20260508 13:23:05.266 | INFO | NumEvents : 1724064696881 | 20260508 13:23:06.815 | INFO | Analyzing chunk frames = [8433664,8434176) | 20260508 13:23:06.815 | INFO | MemInfo: { | 20260508 13:23:06.815 | INFO | "gpu" : " 17 GB", | 20260508 13:23:06.815 | INFO | "huge" : "", | 20260508 13:23:06.815 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:06.815 | INFO | "malloc" : " 14 GB", | 20260508 13:23:06.815 | INFO | "pinned" : " 29 GB" | 20260508 13:23:06.815 | INFO | } | 20260508 13:23:06.815 | INFO | Perf report for the last 5.13699 seconds: | 20260508 13:23:06.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:06.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:06.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:06.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:06.815 | INFO | ANALYSIS : 68.0%, 3.0, 105.8ms, 0 | 20260508 13:23:06.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:06.815 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260508 13:23:06.815 | INFO | PRE_HQ : 66.5%, 1.8, 60.6ms, 0 | 20260508 13:23:06.815 | INFO | BAZWRITER : 67.0%, 2.7, 91.4ms, 0 | 20260508 13:23:06.815 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:23:06.816 | INFO | Uploaded 36GiB to GPU in 2.58576s (13.9224GiB/s) | 20260508 13:23:06.816 | INFO | Downloaded 38GiB to GPU in 2.71924s (13.9745GiB/s) | 20260508 13:23:11.954 | INFO | Analyzing chunk frames = [8434176,8434688) | 20260508 13:23:11.954 | INFO | MemInfo: { | 20260508 13:23:11.954 | INFO | "gpu" : " 17 GB", | 20260508 13:23:11.954 | INFO | "huge" : "", | 20260508 13:23:11.954 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:11.954 | INFO | "malloc" : " 14 GB", | 20260508 13:23:11.954 | INFO | "pinned" : " 29 GB" | 20260508 13:23:11.954 | INFO | } | 20260508 13:23:11.954 | INFO | Perf report for the last 5.13908 seconds: | 20260508 13:23:11.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:11.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.955 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:23:11.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.955 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:23:11.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:11.955 | INFO | Uploaded 36GiB to GPU in 2.49085s (14.4529GiB/s) | 20260508 13:23:11.955 | INFO | Downloaded 34GiB to GPU in 2.46349s (13.8016GiB/s) | 20260508 13:23:17.089 | INFO | Analyzing chunk frames = [8434688,8435200) | 20260508 13:23:17.089 | INFO | MemInfo: { | 20260508 13:23:17.089 | INFO | "gpu" : " 17 GB", | 20260508 13:23:17.089 | INFO | "huge" : "", | 20260508 13:23:17.089 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:17.089 | INFO | "malloc" : " 14 GB", | 20260508 13:23:17.089 | INFO | "pinned" : " 29 GB" | 20260508 13:23:17.089 | INFO | } | 20260508 13:23:17.089 | INFO | Perf report for the last 5.13437 seconds: | 20260508 13:23:17.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:17.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 13:23:17.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:23:17.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:17.089 | INFO | Uploaded 36GiB to GPU in 2.4768s (14.5349GiB/s) | 20260508 13:23:17.089 | INFO | Downloaded 34GiB to GPU in 2.44505s (13.9056GiB/s) | 20260508 13:23:22.230 | INFO | Analyzing chunk frames = [8435200,8435712) | 20260508 13:23:22.230 | INFO | MemInfo: { | 20260508 13:23:22.230 | INFO | "gpu" : " 17 GB", | 20260508 13:23:22.230 | INFO | "huge" : "", | 20260508 13:23:22.230 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:22.230 | INFO | "malloc" : " 14 GB", | 20260508 13:23:22.230 | INFO | "pinned" : " 29 GB" | 20260508 13:23:22.230 | INFO | } | 20260508 13:23:22.230 | INFO | Perf report for the last 5.14154 seconds: | 20260508 13:23:22.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:22.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:23:22.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:23:22.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:22.231 | INFO | Uploaded 36GiB to GPU in 2.51335s (14.3235GiB/s) | 20260508 13:23:22.231 | INFO | Downloaded 34GiB to GPU in 2.44545s (13.9034GiB/s) | 20260508 13:23:27.369 | INFO | Analyzing chunk frames = [8435712,8436224) | 20260508 13:23:27.369 | INFO | MemInfo: { | 20260508 13:23:27.369 | INFO | "gpu" : " 17 GB", | 20260508 13:23:27.369 | INFO | "huge" : "", | 20260508 13:23:27.369 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:27.370 | INFO | "malloc" : " 14 GB", | 20260508 13:23:27.370 | INFO | "pinned" : " 29 GB" | 20260508 13:23:27.370 | INFO | } | 20260508 13:23:27.370 | INFO | Perf report for the last 5.13942 seconds: | 20260508 13:23:27.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:27.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | ANALYSIS : 66.3%, 3.0, 103.6ms, 0 | 20260508 13:23:27.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:23:27.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:27.370 | INFO | Uploaded 36GiB to GPU in 2.44176s (14.7435GiB/s) | 20260508 13:23:27.370 | INFO | Downloaded 34GiB to GPU in 2.52425s (13.4694GiB/s) | 20260508 13:23:32.508 | INFO | Analyzing chunk frames = [8436224,8436736) | 20260508 13:23:32.508 | INFO | MemInfo: { | 20260508 13:23:32.508 | INFO | "gpu" : " 17 GB", | 20260508 13:23:32.508 | INFO | "huge" : "", | 20260508 13:23:32.508 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:32.508 | INFO | "malloc" : " 14 GB", | 20260508 13:23:32.508 | INFO | "pinned" : " 29 GB" | 20260508 13:23:32.508 | INFO | } | 20260508 13:23:32.508 | INFO | Perf report for the last 5.13864 seconds: | 20260508 13:23:32.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:32.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.509 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 13:23:32.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.509 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:23:32.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:32.509 | INFO | Uploaded 36GiB to GPU in 2.38558s (15.0907GiB/s) | 20260508 13:23:32.509 | INFO | Downloaded 34GiB to GPU in 2.4401s (13.9338GiB/s) | 20260508 13:23:37.646 | INFO | Analyzing chunk frames = [8436736,8437248) | 20260508 13:23:37.646 | INFO | MemInfo: { | 20260508 13:23:37.646 | INFO | "gpu" : " 17 GB", | 20260508 13:23:37.646 | INFO | "huge" : "", | 20260508 13:23:37.646 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:37.646 | INFO | "malloc" : " 14 GB", | 20260508 13:23:37.646 | INFO | "pinned" : " 29 GB" | 20260508 13:23:37.646 | INFO | } | 20260508 13:23:37.646 | INFO | Perf report for the last 5.13788 seconds: | 20260508 13:23:37.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:37.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.646 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:23:37.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.646 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 13:23:37.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:37.647 | INFO | Uploaded 36GiB to GPU in 2.46913s (14.58GiB/s) | 20260508 13:23:37.647 | INFO | Downloaded 34GiB to GPU in 2.45628s (13.8421GiB/s) | 20260508 13:23:42.789 | INFO | Analyzing chunk frames = [8437248,8437760) | 20260508 13:23:42.789 | INFO | MemInfo: { | 20260508 13:23:42.789 | INFO | "gpu" : " 17 GB", | 20260508 13:23:42.789 | INFO | "huge" : "", | 20260508 13:23:42.789 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:42.789 | INFO | "malloc" : " 14 GB", | 20260508 13:23:42.789 | INFO | "pinned" : " 29 GB" | 20260508 13:23:42.789 | INFO | } | 20260508 13:23:42.789 | INFO | Perf report for the last 5.14306 seconds: | 20260508 13:23:42.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:42.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.789 | INFO | ANALYSIS : 65.1%, 3.0, 101.8ms, 0 | 20260508 13:23:42.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.790 | INFO | PRE_HQ : 23.6%, 1.0, 12.2ms, 0 | 20260508 13:23:42.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:42.790 | INFO | Uploaded 36GiB to GPU in 2.3989s (15.0069GiB/s) | 20260508 13:23:42.790 | INFO | Downloaded 34GiB to GPU in 2.44719s (13.8935GiB/s) | 20260508 13:23:42.790 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:23:42.790 | INFO | Total mem usage: 16.7 GiB | 20260508 13:23:42.790 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:23:42.790 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.5%) | 20260508 13:23:42.790 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:23:42.790 | INFO | High Water Breakdown: | 20260508 13:23:42.790 | INFO | 59.7% : 8.54 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:23:42.790 | INFO | 40.2% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:23:42.790 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:23:42.790 | INFO | ------------------------------------------ | 20260508 13:23:42.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:23:42.790 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:23:42.790 | INFO | Total mem usage: 26.5 GiB | 20260508 13:23:42.790 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:23:42.790 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:23:42.790 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:23:42.790 | INFO | High Water Breakdown: | 20260508 13:23:42.790 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:23:42.790 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:23:42.790 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:23:42.790 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:23:42.790 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:23:42.790 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:23:42.790 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:23:42.790 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:23:42.790 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:23:42.790 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:23:42.790 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:23:42.790 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:23:42.790 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:23:42.790 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:23:42.790 | INFO | ------------------------------------------ | 20260508 13:23:42.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:23:42.790 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:23:42.790 | INFO | Total mem usage: 27.8 GiB | 20260508 13:23:42.790 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:23:42.790 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 13:23:42.790 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 13:23:42.790 | INFO | High Water Breakdown: | 20260508 13:23:42.790 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 13:23:42.790 | INFO | ------------------------------------------ | 20260508 13:23:42.790 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:23:42.790 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:23:42.790 | INFO | Total mem usage: 15.7 GiB | 20260508 13:23:42.790 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:23:42.790 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:23:42.790 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:23:42.790 | INFO | High Water Breakdown: | 20260508 13:23:42.790 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:23:42.790 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:23:42.790 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:23:42.790 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:23:42.790 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:23:42.790 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:23:42.790 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:23:42.790 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:23:42.790 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:23:42.790 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:23:42.790 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:23:42.790 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:23:42.790 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:23:42.790 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:23:42.790 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:23:42.791 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:23:42.791 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:23:42.791 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:23:42.791 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:23:42.791 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:23:42.791 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:23:42.791 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:23:42.791 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:23:42.791 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:23:42.791 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:23:42.791 | INFO | ------------------------------------------ | 20260508 13:23:42.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:23:42.791 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:23:42.791 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:23:42.791 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:23:42.791 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.40e+01 / 5.61e-01 | 20260508 13:23:42.791 | INFO | PerfReport: 0.30%, AggregateStats -- 9.81e-02 / 1.06e-01, 1.22e-02 / 4.67e-03 | 20260508 13:23:42.791 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.42e+00, 5.96e-01 / 2.28e-01 | 20260508 13:23:42.791 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.45e+00, 2.10e-01 / 1.24e-01 | 20260508 13:23:42.791 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.50e+01, 3.69e+00 / 1.20e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.87e+00 / 1.48e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.31e+00 / 2.41e+00, 3.98e-01 / 2.54e-01 | 20260508 13:23:42.791 | INFO | PerfReport: 2.93%, Metrics -- 9.58e-01 / 1.06e+00, 2.37e-01 / 1.27e-01 | 20260508 13:23:42.791 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.92e-01 / 3.12e-01, 4.71e-02 / 2.11e-02 | 20260508 13:23:42.791 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:23:42.791 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:23:42.791 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:23:42.791 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 24.45%, Upload -- 2.56e+01 / 0.00e+00, 7.84e+00 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 24.74%, Download -- 2.59e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 31.19%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 2.19%, LockUpload -- 2.29e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 2.53%, LockDownload -- 2.66e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 14.88%, LockCompute -- 1.56e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:23:42.791 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260508 13:23:42.887 | INFO | Starting a round of Baz IO | 20260508 13:23:46.293 | INFO | Populating RTMetrics report | 20260508 13:23:46.293 | INFO | RTMetrics report populated | 20260508 13:23:46.293 | INFO | Opening tmp RTMetrics json file | 20260508 13:23:46.304 | INFO | Serializing RTMetrics to json | 20260508 13:23:46.305 | INFO | Serialization done, swapping tmp file now... | 20260508 13:23:46.306 | INFO | RTMetrics done | 20260508 13:23:46.357 | INFO | Finished a round of Baz IO: Wrote 1.39 GiB in 3.47 seconds | 20260508 13:23:46.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:23:46.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143119% / 0% / 5.96046e-05% | 20260508 13:23:46.357 | INFO | Baz IO overall: | 20260508 13:23:46.357 | INFO | BytesWritten : 3597096820405 | 20260508 13:23:46.357 | INFO | NumEvents : 1724762732315 | 20260508 13:23:47.935 | INFO | Analyzing chunk frames = [8437760,8438272) | 20260508 13:23:47.935 | INFO | MemInfo: { | 20260508 13:23:47.935 | INFO | "gpu" : " 17 GB", | 20260508 13:23:47.935 | INFO | "huge" : "", | 20260508 13:23:47.935 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:47.935 | INFO | "malloc" : " 14 GB", | 20260508 13:23:47.935 | INFO | "pinned" : " 29 GB" | 20260508 13:23:47.935 | INFO | } | 20260508 13:23:47.935 | INFO | Perf report for the last 5.14561 seconds: | 20260508 13:23:47.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:47.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:47.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:47.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:47.935 | INFO | ANALYSIS : 67.2%, 3.0, 105.3ms, 0 | 20260508 13:23:47.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:47.935 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:23:47.935 | INFO | PRE_HQ : 65.1%, 1.6, 53.8ms, 0 | 20260508 13:23:47.935 | INFO | BAZWRITER : 66.8%, 2.5, 84.9ms, 0 | 20260508 13:23:47.935 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:23:47.935 | INFO | Uploaded 36GiB to GPU in 2.60927s (13.7969GiB/s) | 20260508 13:23:47.935 | INFO | Downloaded 38GiB to GPU in 2.71727s (13.9846GiB/s) | 20260508 13:23:53.071 | INFO | Analyzing chunk frames = [8438272,8438784) | 20260508 13:23:53.071 | INFO | MemInfo: { | 20260508 13:23:53.071 | INFO | "gpu" : " 17 GB", | 20260508 13:23:53.071 | INFO | "huge" : "", | 20260508 13:23:53.071 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:53.071 | INFO | "malloc" : " 14 GB", | 20260508 13:23:53.071 | INFO | "pinned" : " 29 GB" | 20260508 13:23:53.072 | INFO | } | 20260508 13:23:53.072 | INFO | Perf report for the last 5.13661 seconds: | 20260508 13:23:53.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:53.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 13:23:53.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:23:53.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:53.072 | INFO | Uploaded 36GiB to GPU in 2.44489s (14.7246GiB/s) | 20260508 13:23:53.072 | INFO | Downloaded 34GiB to GPU in 2.48218s (13.6976GiB/s) | 20260508 13:23:58.207 | INFO | Analyzing chunk frames = [8438784,8439296) | 20260508 13:23:58.207 | INFO | MemInfo: { | 20260508 13:23:58.207 | INFO | "gpu" : " 17 GB", | 20260508 13:23:58.207 | INFO | "huge" : "", | 20260508 13:23:58.207 | INFO | "huge_shared" : " 29 GB", | 20260508 13:23:58.207 | INFO | "malloc" : " 14 GB", | 20260508 13:23:58.207 | INFO | "pinned" : " 29 GB" | 20260508 13:23:58.207 | INFO | } | 20260508 13:23:58.207 | INFO | Perf report for the last 5.13522 seconds: | 20260508 13:23:58.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:23:58.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:23:58.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:23:58.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:23:58.207 | INFO | Uploaded 36GiB to GPU in 2.49926s (14.4042GiB/s) | 20260508 13:23:58.207 | INFO | Downloaded 34GiB to GPU in 2.48045s (13.7072GiB/s) | 20260508 13:24:03.347 | INFO | Analyzing chunk frames = [8439296,8439808) | 20260508 13:24:03.347 | INFO | MemInfo: { | 20260508 13:24:03.347 | INFO | "gpu" : " 17 GB", | 20260508 13:24:03.347 | INFO | "huge" : "", | 20260508 13:24:03.347 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:03.347 | INFO | "malloc" : " 14 GB", | 20260508 13:24:03.347 | INFO | "pinned" : " 29 GB" | 20260508 13:24:03.347 | INFO | } | 20260508 13:24:03.347 | INFO | Perf report for the last 5.14032 seconds: | 20260508 13:24:03.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:03.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.347 | INFO | ANALYSIS : 65.7%, 3.0, 101.9ms, 0 | 20260508 13:24:03.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.347 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:24:03.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:03.348 | INFO | Uploaded 36GiB to GPU in 2.41923s (14.8807GiB/s) | 20260508 13:24:03.348 | INFO | Downloaded 34GiB to GPU in 2.48609s (13.6761GiB/s) | 20260508 13:24:08.485 | INFO | Analyzing chunk frames = [8439808,8440320) | 20260508 13:24:08.485 | INFO | MemInfo: { | 20260508 13:24:08.485 | INFO | "gpu" : " 17 GB", | 20260508 13:24:08.485 | INFO | "huge" : "", | 20260508 13:24:08.485 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:08.485 | INFO | "malloc" : " 14 GB", | 20260508 13:24:08.485 | INFO | "pinned" : " 29 GB" | 20260508 13:24:08.485 | INFO | } | 20260508 13:24:08.485 | INFO | Perf report for the last 5.13798 seconds: | 20260508 13:24:08.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:08.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:24:08.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 13:24:08.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:08.485 | INFO | Uploaded 36GiB to GPU in 2.3421s (15.3708GiB/s) | 20260508 13:24:08.486 | INFO | Downloaded 34GiB to GPU in 2.52352s (13.4733GiB/s) | 20260508 13:24:13.628 | INFO | Analyzing chunk frames = [8440320,8440832) | 20260508 13:24:13.628 | INFO | MemInfo: { | 20260508 13:24:13.628 | INFO | "gpu" : " 17 GB", | 20260508 13:24:13.628 | INFO | "huge" : "", | 20260508 13:24:13.628 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:13.628 | INFO | "malloc" : " 14 GB", | 20260508 13:24:13.628 | INFO | "pinned" : " 29 GB" | 20260508 13:24:13.628 | INFO | } | 20260508 13:24:13.628 | INFO | Perf report for the last 5.143 seconds: | 20260508 13:24:13.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:13.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.628 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 13:24:13.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.628 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260508 13:24:13.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:13.629 | INFO | Uploaded 36GiB to GPU in 2.39483s (15.0324GiB/s) | 20260508 13:24:13.629 | INFO | Downloaded 34GiB to GPU in 2.45654s (13.8406GiB/s) | 20260508 13:24:18.768 | INFO | Analyzing chunk frames = [8440832,8441344) | 20260508 13:24:18.768 | INFO | MemInfo: { | 20260508 13:24:18.768 | INFO | "gpu" : " 17 GB", | 20260508 13:24:18.768 | INFO | "huge" : "", | 20260508 13:24:18.768 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:18.768 | INFO | "malloc" : " 14 GB", | 20260508 13:24:18.768 | INFO | "pinned" : " 29 GB" | 20260508 13:24:18.768 | INFO | } | 20260508 13:24:18.768 | INFO | Perf report for the last 5.14023 seconds: | 20260508 13:24:18.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:18.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:24:18.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260508 13:24:18.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:18.769 | INFO | Uploaded 36GiB to GPU in 2.25822s (15.9418GiB/s) | 20260508 13:24:18.769 | INFO | Downloaded 34GiB to GPU in 2.43666s (13.9535GiB/s) | 20260508 13:24:23.904 | INFO | Analyzing chunk frames = [8441344,8441856) | 20260508 13:24:23.904 | INFO | MemInfo: { | 20260508 13:24:23.904 | INFO | "gpu" : " 17 GB", | 20260508 13:24:23.904 | INFO | "huge" : "", | 20260508 13:24:23.904 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:23.904 | INFO | "malloc" : " 14 GB", | 20260508 13:24:23.904 | INFO | "pinned" : " 29 GB" | 20260508 13:24:23.904 | INFO | } | 20260508 13:24:23.904 | INFO | Perf report for the last 5.13576 seconds: | 20260508 13:24:23.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:23.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.904 | INFO | ANALYSIS : 64.4%, 3.0, 100.4ms, 0 | 20260508 13:24:23.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.904 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 13:24:23.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:23.905 | INFO | Uploaded 36GiB to GPU in 2.28966s (15.7229GiB/s) | 20260508 13:24:23.905 | INFO | Downloaded 34GiB to GPU in 2.422s (14.038GiB/s) | 20260508 13:24:24.017 | INFO | Starting a round of Baz IO | 20260508 13:24:27.416 | INFO | Populating RTMetrics report | 20260508 13:24:27.417 | INFO | RTMetrics report populated | 20260508 13:24:27.417 | INFO | Opening tmp RTMetrics json file | 20260508 13:24:27.422 | INFO | Serializing RTMetrics to json | 20260508 13:24:27.423 | INFO | Serialization done, swapping tmp file now... | 20260508 13:24:27.424 | INFO | RTMetrics done | 20260508 13:24:27.484 | INFO | Finished a round of Baz IO: Wrote 1.39 GiB in 3.466 seconds | 20260508 13:24:27.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:24:27.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14261% / 0% / 4.37101e-05% | 20260508 13:24:27.484 | INFO | Baz IO overall: | 20260508 13:24:27.484 | INFO | BytesWritten : 3598588878429 | 20260508 13:24:27.484 | INFO | NumEvents : 1725460777859 | 20260508 13:24:29.047 | INFO | Analyzing chunk frames = [8441856,8442368) | 20260508 13:24:29.047 | INFO | MemInfo: { | 20260508 13:24:29.047 | INFO | "gpu" : " 17 GB", | 20260508 13:24:29.047 | INFO | "huge" : "", | 20260508 13:24:29.047 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:29.047 | INFO | "malloc" : " 14 GB", | 20260508 13:24:29.047 | INFO | "pinned" : " 29 GB" | 20260508 13:24:29.047 | INFO | } | 20260508 13:24:29.047 | INFO | Perf report for the last 5.14303 seconds: | 20260508 13:24:29.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:29.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:29.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:29.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:29.047 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:24:29.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:29.047 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260508 13:24:29.047 | INFO | PRE_HQ : 66.1%, 1.7, 59.5ms, 0 | 20260508 13:24:29.047 | INFO | BAZWRITER : 66.8%, 2.6, 89.1ms, 0 | 20260508 13:24:29.047 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:24:29.048 | INFO | Uploaded 36GiB to GPU in 2.62323s (13.7236GiB/s) | 20260508 13:24:29.048 | INFO | Downloaded 38GiB to GPU in 2.71767s (13.9826GiB/s) | 20260508 13:24:34.184 | INFO | Analyzing chunk frames = [8442368,8442880) | 20260508 13:24:34.184 | INFO | MemInfo: { | 20260508 13:24:34.184 | INFO | "gpu" : " 17 GB", | 20260508 13:24:34.184 | INFO | "huge" : "", | 20260508 13:24:34.184 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:34.184 | INFO | "malloc" : " 14 GB", | 20260508 13:24:34.184 | INFO | "pinned" : " 29 GB" | 20260508 13:24:34.184 | INFO | } | 20260508 13:24:34.184 | INFO | Perf report for the last 5.13681 seconds: | 20260508 13:24:34.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:34.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 13:24:34.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:24:34.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:34.184 | INFO | Uploaded 36GiB to GPU in 2.47898s (14.5221GiB/s) | 20260508 13:24:34.184 | INFO | Downloaded 34GiB to GPU in 2.48033s (13.7078GiB/s) | 20260508 13:24:39.320 | INFO | Analyzing chunk frames = [8442880,8443392) | 20260508 13:24:39.320 | INFO | MemInfo: { | 20260508 13:24:39.320 | INFO | "gpu" : " 17 GB", | 20260508 13:24:39.320 | INFO | "huge" : "", | 20260508 13:24:39.320 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:39.320 | INFO | "malloc" : " 14 GB", | 20260508 13:24:39.320 | INFO | "pinned" : " 29 GB" | 20260508 13:24:39.320 | INFO | } | 20260508 13:24:39.320 | INFO | Perf report for the last 5.13551 seconds: | 20260508 13:24:39.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:39.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 13:24:39.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:24:39.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:39.320 | INFO | Uploaded 36GiB to GPU in 2.48703s (14.4751GiB/s) | 20260508 13:24:39.320 | INFO | Downloaded 34GiB to GPU in 2.48457s (13.6844GiB/s) | 20260508 13:24:44.457 | INFO | Analyzing chunk frames = [8443392,8443904) | 20260508 13:24:44.457 | INFO | MemInfo: { | 20260508 13:24:44.457 | INFO | "gpu" : " 17 GB", | 20260508 13:24:44.457 | INFO | "huge" : "", | 20260508 13:24:44.457 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:44.457 | INFO | "malloc" : " 14 GB", | 20260508 13:24:44.457 | INFO | "pinned" : " 29 GB" | 20260508 13:24:44.457 | INFO | } | 20260508 13:24:44.457 | INFO | Perf report for the last 5.1378 seconds: | 20260508 13:24:44.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:44.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.458 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 13:24:44.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.458 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:24:44.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:44.458 | INFO | Uploaded 36GiB to GPU in 2.48721s (14.4741GiB/s) | 20260508 13:24:44.458 | INFO | Downloaded 34GiB to GPU in 2.48064s (13.7061GiB/s) | 20260508 13:24:49.599 | INFO | Analyzing chunk frames = [8443904,8444416) | 20260508 13:24:49.599 | INFO | MemInfo: { | 20260508 13:24:49.599 | INFO | "gpu" : " 17 GB", | 20260508 13:24:49.599 | INFO | "huge" : "", | 20260508 13:24:49.599 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:49.599 | INFO | "malloc" : " 14 GB", | 20260508 13:24:49.600 | INFO | "pinned" : " 29 GB" | 20260508 13:24:49.600 | INFO | } | 20260508 13:24:49.600 | INFO | Perf report for the last 5.14216 seconds: | 20260508 13:24:49.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:49.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | ANALYSIS : 66.4%, 3.0, 103.7ms, 0 | 20260508 13:24:49.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:24:49.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:49.600 | INFO | Uploaded 36GiB to GPU in 2.39664s (15.021GiB/s) | 20260508 13:24:49.600 | INFO | Downloaded 34GiB to GPU in 2.53636s (13.4051GiB/s) | 20260508 13:24:54.741 | INFO | Analyzing chunk frames = [8444416,8444928) | 20260508 13:24:54.741 | INFO | MemInfo: { | 20260508 13:24:54.741 | INFO | "gpu" : " 17 GB", | 20260508 13:24:54.741 | INFO | "huge" : "", | 20260508 13:24:54.741 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:54.741 | INFO | "malloc" : " 14 GB", | 20260508 13:24:54.741 | INFO | "pinned" : " 29 GB" | 20260508 13:24:54.741 | INFO | } | 20260508 13:24:54.741 | INFO | Perf report for the last 5.14125 seconds: | 20260508 13:24:54.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:54.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:24:54.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:24:54.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:54.741 | INFO | Uploaded 36GiB to GPU in 2.39937s (15.0039GiB/s) | 20260508 13:24:54.741 | INFO | Downloaded 34GiB to GPU in 2.44161s (13.9252GiB/s) | 20260508 13:24:59.886 | INFO | Analyzing chunk frames = [8444928,8445440) | 20260508 13:24:59.886 | INFO | MemInfo: { | 20260508 13:24:59.886 | INFO | "gpu" : " 17 GB", | 20260508 13:24:59.886 | INFO | "huge" : "", | 20260508 13:24:59.886 | INFO | "huge_shared" : " 29 GB", | 20260508 13:24:59.886 | INFO | "malloc" : " 14 GB", | 20260508 13:24:59.886 | INFO | "pinned" : " 29 GB" | 20260508 13:24:59.886 | INFO | } | 20260508 13:24:59.886 | INFO | Perf report for the last 5.14516 seconds: | 20260508 13:24:59.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:24:59.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.886 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 13:24:59.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.886 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 13:24:59.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:24:59.887 | INFO | Uploaded 36GiB to GPU in 2.44873s (14.7015GiB/s) | 20260508 13:24:59.887 | INFO | Downloaded 34GiB to GPU in 2.44388s (13.9123GiB/s) | 20260508 13:25:05.020 | INFO | Analyzing chunk frames = [8445440,8445952) | 20260508 13:25:05.020 | INFO | MemInfo: { | 20260508 13:25:05.020 | INFO | "gpu" : " 17 GB", | 20260508 13:25:05.020 | INFO | "huge" : "", | 20260508 13:25:05.020 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:05.020 | INFO | "malloc" : " 14 GB", | 20260508 13:25:05.020 | INFO | "pinned" : " 29 GB" | 20260508 13:25:05.020 | INFO | } | 20260508 13:25:05.020 | INFO | Perf report for the last 5.13379 seconds: | 20260508 13:25:05.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:05.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | ANALYSIS : 64.4%, 3.0, 100.5ms, 0 | 20260508 13:25:05.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:25:05.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:05.020 | INFO | Uploaded 36GiB to GPU in 2.29483s (15.6874GiB/s) | 20260508 13:25:05.020 | INFO | Downloaded 34GiB to GPU in 2.42113s (14.043GiB/s) | 20260508 13:25:05.020 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:25:05.020 | INFO | Total mem usage: 16.7 GiB | 20260508 13:25:05.020 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:25:05.020 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.8%) | 20260508 13:25:05.020 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:25:05.020 | INFO | High Water Breakdown: | 20260508 13:25:05.020 | INFO | 59.6% : 8.51 GiB ( 82.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:25:05.020 | INFO | 40.3% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:25:05.020 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:25:05.020 | INFO | ------------------------------------------ | 20260508 13:25:05.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:25:05.020 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:25:05.020 | INFO | Total mem usage: 26.5 GiB | 20260508 13:25:05.020 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:25:05.020 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:25:05.020 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:25:05.020 | INFO | High Water Breakdown: | 20260508 13:25:05.020 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:25:05.020 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:25:05.020 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:25:05.020 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:25:05.020 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:25:05.020 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:25:05.020 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:25:05.020 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:25:05.021 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:25:05.021 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:25:05.021 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:25:05.021 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:25:05.021 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:25:05.021 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:25:05.021 | INFO | ------------------------------------------ | 20260508 13:25:05.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:25:05.021 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:25:05.021 | INFO | Total mem usage: 27.8 GiB | 20260508 13:25:05.021 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:25:05.021 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:25:05.021 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:25:05.021 | INFO | High Water Breakdown: | 20260508 13:25:05.021 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:25:05.021 | INFO | ------------------------------------------ | 20260508 13:25:05.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:25:05.021 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:25:05.021 | INFO | Total mem usage: 15.7 GiB | 20260508 13:25:05.021 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:25:05.021 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:25:05.021 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:25:05.021 | INFO | High Water Breakdown: | 20260508 13:25:05.021 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:25:05.021 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:25:05.021 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:25:05.021 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:25:05.021 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:25:05.021 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:25:05.021 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:25:05.021 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:25:05.021 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:25:05.021 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:25:05.021 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:25:05.021 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:25:05.021 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:25:05.021 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:25:05.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:25:05.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:25:05.021 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:25:05.021 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:25:05.021 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:25:05.021 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:25:05.021 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:25:05.021 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:25:05.021 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:25:05.021 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:25:05.021 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:25:05.021 | INFO | ------------------------------------------ | 20260508 13:25:05.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:25:05.021 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:25:05.021 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:25:05.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:25:05.021 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.41e-01 | 20260508 13:25:05.021 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.05e-01, 1.16e-02 / 5.43e-03 | 20260508 13:25:05.021 | INFO | PerfReport: 12.22%, Baseline -- 4.00e+00 / 4.42e+00, 5.95e-01 / 2.51e-01 | 20260508 13:25:05.022 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.38e+00, 2.09e-01 / 1.30e-01 | 20260508 13:25:05.022 | INFO | PerfReport: 15.95%, DME -- 4.17e+01 / 4.49e+01, 3.63e+00 / 1.01e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.87e+01 / 2.07e+01, 1.88e+00 / 1.33e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.31e+00 / 2.36e+00, 4.02e-01 / 2.85e-01 | 20260508 13:25:05.022 | INFO | PerfReport: 2.92%, Metrics -- 9.56e-01 / 1.06e+00, 2.39e-01 / 1.43e-01 | 20260508 13:25:05.022 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.13e-01, 4.77e-02 / 2.87e-02 | 20260508 13:25:05.022 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:25:05.022 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:25:05.022 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:25:05.022 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 24.21%, Upload -- 2.53e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 24.92%, Download -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 31.30%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 1.99%, LockUpload -- 2.08e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 2.68%, LockDownload -- 2.80e+00 / 0.00e+00, 4.15e+00 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 14.89%, LockCompute -- 1.56e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260508 13:25:05.022 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.92e-02 / 0.00e+00 | 20260508 13:25:05.117 | INFO | Starting a round of Baz IO | 20260508 13:25:08.512 | INFO | Populating RTMetrics report | 20260508 13:25:08.512 | INFO | RTMetrics report populated | 20260508 13:25:08.512 | INFO | Opening tmp RTMetrics json file | 20260508 13:25:08.523 | INFO | Serializing RTMetrics to json | 20260508 13:25:08.524 | INFO | Serialization done, swapping tmp file now... | 20260508 13:25:08.524 | INFO | RTMetrics done | 20260508 13:25:08.585 | INFO | Finished a round of Baz IO: Wrote 1.389 GiB in 3.468 seconds | 20260508 13:25:08.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:25:08.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142109% / 0% / 7.94729e-05% | 20260508 13:25:08.585 | INFO | Baz IO overall: | 20260508 13:25:08.585 | INFO | BytesWritten : 3600079826266 | 20260508 13:25:08.585 | INFO | NumEvents : 1726158340111 | 20260508 13:25:10.150 | INFO | Analyzing chunk frames = [8445952,8446464) | 20260508 13:25:10.150 | INFO | MemInfo: { | 20260508 13:25:10.150 | INFO | "gpu" : " 17 GB", | 20260508 13:25:10.150 | INFO | "huge" : "", | 20260508 13:25:10.150 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:10.150 | INFO | "malloc" : " 14 GB", | 20260508 13:25:10.150 | INFO | "pinned" : " 29 GB" | 20260508 13:25:10.150 | INFO | } | 20260508 13:25:10.150 | INFO | Perf report for the last 5.13024 seconds: | 20260508 13:25:10.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:10.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:10.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:10.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:10.150 | INFO | ANALYSIS : 67.5%, 3.0, 105.2ms, 0 | 20260508 13:25:10.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:10.150 | INFO | RT_METRICS : 38.1%, 1.0, 19.9ms, 0 | 20260508 13:25:10.150 | INFO | PRE_HQ : 65.2%, 1.6, 53.4ms, 0 | 20260508 13:25:10.150 | INFO | BAZWRITER : 66.9%, 2.5, 84.2ms, 0 | 20260508 13:25:10.150 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:25:10.150 | INFO | Uploaded 36GiB to GPU in 2.63893s (13.6419GiB/s) | 20260508 13:25:10.150 | INFO | Downloaded 38GiB to GPU in 2.71404s (14.0013GiB/s) | 20260508 13:25:15.293 | INFO | Analyzing chunk frames = [8446464,8446976) | 20260508 13:25:15.293 | INFO | MemInfo: { | 20260508 13:25:15.293 | INFO | "gpu" : " 17 GB", | 20260508 13:25:15.293 | INFO | "huge" : "", | 20260508 13:25:15.293 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:15.293 | INFO | "malloc" : " 14 GB", | 20260508 13:25:15.293 | INFO | "pinned" : " 29 GB" | 20260508 13:25:15.294 | INFO | } | 20260508 13:25:15.294 | INFO | Perf report for the last 5.14356 seconds: | 20260508 13:25:15.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:15.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 13:25:15.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | PRE_HQ : 22.1%, 1.0, 11.5ms, 0 | 20260508 13:25:15.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:15.294 | INFO | Uploaded 36GiB to GPU in 2.46635s (14.5964GiB/s) | 20260508 13:25:15.294 | INFO | Downloaded 34GiB to GPU in 2.46532s (13.7913GiB/s) | 20260508 13:25:20.439 | INFO | Analyzing chunk frames = [8446976,8447488) | 20260508 13:25:20.439 | INFO | MemInfo: { | 20260508 13:25:20.439 | INFO | "gpu" : " 17 GB", | 20260508 13:25:20.439 | INFO | "huge" : "", | 20260508 13:25:20.439 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:20.439 | INFO | "malloc" : " 14 GB", | 20260508 13:25:20.439 | INFO | "pinned" : " 29 GB" | 20260508 13:25:20.439 | INFO | } | 20260508 13:25:20.439 | INFO | Perf report for the last 5.14527 seconds: | 20260508 13:25:20.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:20.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.439 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:25:20.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.439 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:25:20.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:20.440 | INFO | Uploaded 36GiB to GPU in 2.47653s (14.5365GiB/s) | 20260508 13:25:20.440 | INFO | Downloaded 34GiB to GPU in 2.45019s (13.8765GiB/s) | 20260508 13:25:25.581 | INFO | Analyzing chunk frames = [8447488,8448000) | 20260508 13:25:25.581 | INFO | MemInfo: { | 20260508 13:25:25.581 | INFO | "gpu" : " 17 GB", | 20260508 13:25:25.581 | INFO | "huge" : "", | 20260508 13:25:25.581 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:25.581 | INFO | "malloc" : " 14 GB", | 20260508 13:25:25.581 | INFO | "pinned" : " 29 GB" | 20260508 13:25:25.581 | INFO | } | 20260508 13:25:25.581 | INFO | Perf report for the last 5.14209 seconds: | 20260508 13:25:25.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:25.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.581 | INFO | ANALYSIS : 65.2%, 3.0, 101.3ms, 0 | 20260508 13:25:25.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.581 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:25:25.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:25.582 | INFO | Uploaded 36GiB to GPU in 2.42273s (14.8593GiB/s) | 20260508 13:25:25.582 | INFO | Downloaded 34GiB to GPU in 2.44868s (13.885GiB/s) | 20260508 13:25:30.717 | INFO | Analyzing chunk frames = [8448000,8448512) | 20260508 13:25:30.717 | INFO | MemInfo: { | 20260508 13:25:30.717 | INFO | "gpu" : " 17 GB", | 20260508 13:25:30.717 | INFO | "huge" : "", | 20260508 13:25:30.717 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:30.717 | INFO | "malloc" : " 14 GB", | 20260508 13:25:30.717 | INFO | "pinned" : " 29 GB" | 20260508 13:25:30.717 | INFO | } | 20260508 13:25:30.717 | INFO | Perf report for the last 5.1363 seconds: | 20260508 13:25:30.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:30.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.717 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 13:25:30.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.717 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:25:30.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:30.718 | INFO | Uploaded 36GiB to GPU in 2.46204s (14.622GiB/s) | 20260508 13:25:30.718 | INFO | Downloaded 34GiB to GPU in 2.51408s (13.5238GiB/s) | 20260508 13:25:35.856 | INFO | Analyzing chunk frames = [8448512,8449024) | 20260508 13:25:35.856 | INFO | MemInfo: { | 20260508 13:25:35.856 | INFO | "gpu" : " 17 GB", | 20260508 13:25:35.856 | INFO | "huge" : "", | 20260508 13:25:35.856 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:35.856 | INFO | "malloc" : " 14 GB", | 20260508 13:25:35.856 | INFO | "pinned" : " 29 GB" | 20260508 13:25:35.856 | INFO | } | 20260508 13:25:35.856 | INFO | Perf report for the last 5.13852 seconds: | 20260508 13:25:35.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:35.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.856 | INFO | ANALYSIS : 65.0%, 3.0, 101.1ms, 0 | 20260508 13:25:35.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.856 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260508 13:25:35.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:35.857 | INFO | Uploaded 36GiB to GPU in 2.35853s (15.2637GiB/s) | 20260508 13:25:35.857 | INFO | Downloaded 34GiB to GPU in 2.42286s (14.033GiB/s) | 20260508 13:25:40.994 | INFO | Analyzing chunk frames = [8449024,8449536) | 20260508 13:25:40.994 | INFO | MemInfo: { | 20260508 13:25:40.994 | INFO | "gpu" : " 17 GB", | 20260508 13:25:40.994 | INFO | "huge" : "", | 20260508 13:25:40.994 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:40.994 | INFO | "malloc" : " 14 GB", | 20260508 13:25:40.994 | INFO | "pinned" : " 29 GB" | 20260508 13:25:40.994 | INFO | } | 20260508 13:25:40.995 | INFO | Perf report for the last 5.13858 seconds: | 20260508 13:25:40.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:40.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:25:40.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:25:40.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:40.995 | INFO | Uploaded 36GiB to GPU in 2.473s (14.5572GiB/s) | 20260508 13:25:40.995 | INFO | Downloaded 34GiB to GPU in 2.46058s (13.8179GiB/s) | 20260508 13:25:46.137 | INFO | Analyzing chunk frames = [8449536,8450048) | 20260508 13:25:46.137 | INFO | MemInfo: { | 20260508 13:25:46.137 | INFO | "gpu" : " 17 GB", | 20260508 13:25:46.137 | INFO | "huge" : "", | 20260508 13:25:46.137 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:46.137 | INFO | "malloc" : " 14 GB", | 20260508 13:25:46.137 | INFO | "pinned" : " 29 GB" | 20260508 13:25:46.137 | INFO | } | 20260508 13:25:46.137 | INFO | Perf report for the last 5.14264 seconds: | 20260508 13:25:46.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:46.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.137 | INFO | ANALYSIS : 65.5%, 3.0, 102.4ms, 0 | 20260508 13:25:46.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.137 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:25:46.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:46.138 | INFO | Uploaded 36GiB to GPU in 2.43857s (14.7627GiB/s) | 20260508 13:25:46.138 | INFO | Downloaded 34GiB to GPU in 2.44219s (13.9219GiB/s) | 20260508 13:25:46.235 | INFO | Starting a round of Baz IO | 20260508 13:25:49.647 | INFO | Populating RTMetrics report | 20260508 13:25:49.647 | INFO | RTMetrics report populated | 20260508 13:25:49.647 | INFO | Opening tmp RTMetrics json file | 20260508 13:25:49.652 | INFO | Serializing RTMetrics to json | 20260508 13:25:49.654 | INFO | Serialization done, swapping tmp file now... | 20260508 13:25:49.654 | INFO | RTMetrics done | 20260508 13:25:49.714 | INFO | Finished a round of Baz IO: Wrote 1.388 GiB in 3.479 seconds | 20260508 13:25:49.714 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:25:49.714 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142173% / 3.97364e-06% / 5.16574e-05% | 20260508 13:25:49.714 | INFO | Baz IO overall: | 20260508 13:25:49.714 | INFO | BytesWritten : 3601570060472 | 20260508 13:25:49.714 | INFO | NumEvents : 1726855719526 | 20260508 13:25:51.270 | INFO | Analyzing chunk frames = [8450048,8450560) | 20260508 13:25:51.270 | INFO | MemInfo: { | 20260508 13:25:51.270 | INFO | "gpu" : " 17 GB", | 20260508 13:25:51.270 | INFO | "huge" : "", | 20260508 13:25:51.270 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:51.270 | INFO | "malloc" : " 14 GB", | 20260508 13:25:51.270 | INFO | "pinned" : " 29 GB" | 20260508 13:25:51.270 | INFO | } | 20260508 13:25:51.270 | INFO | Perf report for the last 5.13296 seconds: | 20260508 13:25:51.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:51.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:51.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:51.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:51.270 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:25:51.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:51.270 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:25:51.270 | INFO | PRE_HQ : 65.4%, 1.6, 53.5ms, 0 | 20260508 13:25:51.270 | INFO | BAZWRITER : 67.0%, 2.4, 83.9ms, 0 | 20260508 13:25:51.270 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:25:51.271 | INFO | Uploaded 36GiB to GPU in 2.61536s (13.7649GiB/s) | 20260508 13:25:51.271 | INFO | Downloaded 38GiB to GPU in 2.71845s (13.9785GiB/s) | 20260508 13:25:56.406 | INFO | Analyzing chunk frames = [8450560,8451072) | 20260508 13:25:56.406 | INFO | MemInfo: { | 20260508 13:25:56.406 | INFO | "gpu" : " 17 GB", | 20260508 13:25:56.406 | INFO | "huge" : "", | 20260508 13:25:56.406 | INFO | "huge_shared" : " 29 GB", | 20260508 13:25:56.406 | INFO | "malloc" : " 14 GB", | 20260508 13:25:56.406 | INFO | "pinned" : " 29 GB" | 20260508 13:25:56.406 | INFO | } | 20260508 13:25:56.406 | INFO | Perf report for the last 5.13616 seconds: | 20260508 13:25:56.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:25:56.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.407 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 13:25:56.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.407 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:25:56.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:25:56.407 | INFO | Uploaded 36GiB to GPU in 2.40364s (14.9773GiB/s) | 20260508 13:25:56.407 | INFO | Downloaded 34GiB to GPU in 2.46629s (13.7859GiB/s) | 20260508 13:26:01.554 | INFO | Analyzing chunk frames = [8451072,8451584) | 20260508 13:26:01.554 | INFO | MemInfo: { | 20260508 13:26:01.554 | INFO | "gpu" : " 17 GB", | 20260508 13:26:01.554 | INFO | "huge" : "", | 20260508 13:26:01.554 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:01.554 | INFO | "malloc" : " 14 GB", | 20260508 13:26:01.554 | INFO | "pinned" : " 29 GB" | 20260508 13:26:01.554 | INFO | } | 20260508 13:26:01.554 | INFO | Perf report for the last 5.14774 seconds: | 20260508 13:26:01.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:01.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.554 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:26:01.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.554 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:26:01.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:01.555 | INFO | Uploaded 36GiB to GPU in 2.46473s (14.6061GiB/s) | 20260508 13:26:01.555 | INFO | Downloaded 34GiB to GPU in 2.45987s (13.8219GiB/s) | 20260508 13:26:06.697 | INFO | Analyzing chunk frames = [8451584,8452096) | 20260508 13:26:06.697 | INFO | MemInfo: { | 20260508 13:26:06.697 | INFO | "gpu" : " 17 GB", | 20260508 13:26:06.697 | INFO | "huge" : "", | 20260508 13:26:06.697 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:06.697 | INFO | "malloc" : " 14 GB", | 20260508 13:26:06.697 | INFO | "pinned" : " 29 GB" | 20260508 13:26:06.697 | INFO | } | 20260508 13:26:06.697 | INFO | Perf report for the last 5.14314 seconds: | 20260508 13:26:06.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:06.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.697 | INFO | ANALYSIS : 65.2%, 3.0, 101.3ms, 0 | 20260508 13:26:06.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.697 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:26:06.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:06.698 | INFO | Uploaded 36GiB to GPU in 2.37782s (15.1399GiB/s) | 20260508 13:26:06.698 | INFO | Downloaded 34GiB to GPU in 2.49199s (13.6437GiB/s) | 20260508 13:26:11.830 | INFO | Analyzing chunk frames = [8452096,8452608) | 20260508 13:26:11.830 | INFO | MemInfo: { | 20260508 13:26:11.830 | INFO | "gpu" : " 17 GB", | 20260508 13:26:11.830 | INFO | "huge" : "", | 20260508 13:26:11.830 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:11.830 | INFO | "malloc" : " 14 GB", | 20260508 13:26:11.830 | INFO | "pinned" : " 29 GB" | 20260508 13:26:11.830 | INFO | } | 20260508 13:26:11.830 | INFO | Perf report for the last 5.13276 seconds: | 20260508 13:26:11.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:11.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.830 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:26:11.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.830 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:26:11.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:11.831 | INFO | Uploaded 36GiB to GPU in 2.43025s (14.8133GiB/s) | 20260508 13:26:11.831 | INFO | Downloaded 34GiB to GPU in 2.53395s (13.4178GiB/s) | 20260508 13:26:16.966 | INFO | Analyzing chunk frames = [8452608,8453120) | 20260508 13:26:16.966 | INFO | MemInfo: { | 20260508 13:26:16.966 | INFO | "gpu" : " 17 GB", | 20260508 13:26:16.966 | INFO | "huge" : "", | 20260508 13:26:16.966 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:16.966 | INFO | "malloc" : " 14 GB", | 20260508 13:26:16.966 | INFO | "pinned" : " 29 GB" | 20260508 13:26:16.966 | INFO | } | 20260508 13:26:16.966 | INFO | Perf report for the last 5.13574 seconds: | 20260508 13:26:16.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:16.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | ANALYSIS : 65.2%, 3.0, 101.4ms, 0 | 20260508 13:26:16.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:26:16.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:16.966 | INFO | Uploaded 36GiB to GPU in 2.37824s (15.1372GiB/s) | 20260508 13:26:16.966 | INFO | Downloaded 34GiB to GPU in 2.45728s (13.8365GiB/s) | 20260508 13:26:22.106 | INFO | Analyzing chunk frames = [8453120,8453632) | 20260508 13:26:22.106 | INFO | MemInfo: { | 20260508 13:26:22.106 | INFO | "gpu" : " 17 GB", | 20260508 13:26:22.106 | INFO | "huge" : "", | 20260508 13:26:22.106 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:22.106 | INFO | "malloc" : " 14 GB", | 20260508 13:26:22.106 | INFO | "pinned" : " 29 GB" | 20260508 13:26:22.106 | INFO | } | 20260508 13:26:22.106 | INFO | Perf report for the last 5.14054 seconds: | 20260508 13:26:22.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:22.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.107 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 13:26:22.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.107 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:26:22.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:22.107 | INFO | Uploaded 36GiB to GPU in 2.29513s (15.6854GiB/s) | 20260508 13:26:22.107 | INFO | Downloaded 34GiB to GPU in 2.44182s (13.924GiB/s) | 20260508 13:26:27.248 | INFO | Analyzing chunk frames = [8453632,8454144) | 20260508 13:26:27.248 | INFO | MemInfo: { | 20260508 13:26:27.248 | INFO | "gpu" : " 17 GB", | 20260508 13:26:27.248 | INFO | "huge" : "", | 20260508 13:26:27.248 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:27.248 | INFO | "malloc" : " 14 GB", | 20260508 13:26:27.248 | INFO | "pinned" : " 29 GB" | 20260508 13:26:27.248 | INFO | } | 20260508 13:26:27.248 | INFO | Perf report for the last 5.14181 seconds: | 20260508 13:26:27.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:27.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.248 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260508 13:26:27.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.248 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:26:27.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:27.249 | INFO | Uploaded 36GiB to GPU in 2.35849s (15.264GiB/s) | 20260508 13:26:27.249 | INFO | Downloaded 34GiB to GPU in 2.46646s (13.7849GiB/s) | 20260508 13:26:27.249 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:26:27.249 | INFO | Total mem usage: 16.7 GiB | 20260508 13:26:27.249 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:26:27.249 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:26:27.249 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:26:27.249 | INFO | High Water Breakdown: | 20260508 13:26:27.249 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:26:27.249 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:26:27.249 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:26:27.249 | INFO | ------------------------------------------ | 20260508 13:26:27.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:26:27.249 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:26:27.249 | INFO | Total mem usage: 26.5 GiB | 20260508 13:26:27.249 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:26:27.249 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:26:27.249 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:26:27.249 | INFO | High Water Breakdown: | 20260508 13:26:27.249 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:26:27.249 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:26:27.249 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:26:27.249 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:26:27.249 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:26:27.249 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:26:27.249 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:26:27.249 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:26:27.249 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:26:27.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:26:27.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:26:27.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:26:27.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:26:27.249 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:26:27.249 | INFO | ------------------------------------------ | 20260508 13:26:27.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:26:27.249 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:26:27.249 | INFO | Total mem usage: 27.8 GiB | 20260508 13:26:27.249 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:26:27.249 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:26:27.249 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:26:27.249 | INFO | High Water Breakdown: | 20260508 13:26:27.249 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:26:27.249 | INFO | ------------------------------------------ | 20260508 13:26:27.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:26:27.249 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:26:27.249 | INFO | Total mem usage: 15.7 GiB | 20260508 13:26:27.249 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:26:27.249 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:26:27.249 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:26:27.249 | INFO | High Water Breakdown: | 20260508 13:26:27.249 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:26:27.249 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:26:27.249 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:26:27.249 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:26:27.249 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:26:27.249 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:26:27.249 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:26:27.249 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:26:27.249 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:26:27.249 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:26:27.249 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:26:27.249 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:26:27.249 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:26:27.249 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:26:27.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:26:27.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:26:27.249 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:26:27.249 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:26:27.249 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:26:27.249 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:26:27.249 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:26:27.249 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:26:27.249 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:26:27.249 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:26:27.249 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:26:27.249 | INFO | ------------------------------------------ | 20260508 13:26:27.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:26:27.249 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:26:27.249 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:26:27.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:26:27.249 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.41e+01 / 6.35e-01 | 20260508 13:26:27.249 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.03e-01, 1.06e-02 / 3.75e-03 | 20260508 13:26:27.249 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.45e+00, 5.97e-01 / 2.12e-01 | 20260508 13:26:27.249 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.43e+00, 2.10e-01 / 1.03e-01 | 20260508 13:26:27.249 | INFO | PerfReport: 15.96%, DME -- 4.18e+01 / 4.50e+01, 3.60e+00 / 1.27e+00 | 20260508 13:26:27.249 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.88e+01 / 2.06e+01, 1.89e+00 / 1.40e+00 | 20260508 13:26:27.249 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.29e+00 / 2.37e+00, 4.02e-01 / 2.68e-01 | 20260508 13:26:27.249 | INFO | PerfReport: 2.91%, Metrics -- 9.53e-01 / 1.09e+00, 2.37e-01 / 1.05e-01 | 20260508 13:26:27.249 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.89e-01 / 3.10e-01, 4.66e-02 / 2.82e-02 | 20260508 13:26:27.249 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:26:27.249 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:26:27.249 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:26:27.249 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 24.30%, Upload -- 2.54e+01 / 0.00e+00, 7.77e+00 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 24.85%, Download -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 31.27%, Compute -- 3.27e+01 / 0.00e+00, 1.41e+01 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 2.07%, LockUpload -- 2.17e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 2.61%, LockDownload -- 2.73e+00 / 0.00e+00, 4.06e+00 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 14.89%, LockCompute -- 1.56e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:26:27.250 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260508 13:26:27.345 | INFO | Starting a round of Baz IO | 20260508 13:26:30.764 | INFO | Populating RTMetrics report | 20260508 13:26:30.764 | INFO | RTMetrics report populated | 20260508 13:26:30.764 | INFO | Opening tmp RTMetrics json file | 20260508 13:26:30.771 | INFO | Serializing RTMetrics to json | 20260508 13:26:30.772 | INFO | Serialization done, swapping tmp file now... | 20260508 13:26:30.773 | INFO | RTMetrics done | 20260508 13:26:30.832 | INFO | Finished a round of Baz IO: Wrote 1.388 GiB in 3.486 seconds | 20260508 13:26:30.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:26:30.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141923% / 0% / 5.16574e-05% | 20260508 13:26:30.832 | INFO | Baz IO overall: | 20260508 13:26:30.832 | INFO | BytesWritten : 3603059881995 | 20260508 13:26:30.832 | INFO | NumEvents : 1727553006833 | 20260508 13:26:32.389 | INFO | Analyzing chunk frames = [8454144,8454656) | 20260508 13:26:32.389 | INFO | MemInfo: { | 20260508 13:26:32.389 | INFO | "gpu" : " 17 GB", | 20260508 13:26:32.389 | INFO | "huge" : "", | 20260508 13:26:32.389 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:32.389 | INFO | "malloc" : " 14 GB", | 20260508 13:26:32.389 | INFO | "pinned" : " 29 GB" | 20260508 13:26:32.389 | INFO | } | 20260508 13:26:32.390 | INFO | Perf report for the last 5.1414 seconds: | 20260508 13:26:32.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:32.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:32.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:32.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:32.390 | INFO | ANALYSIS : 67.6%, 3.0, 105.6ms, 0 | 20260508 13:26:32.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:32.390 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:26:32.390 | INFO | PRE_HQ : 65.4%, 1.6, 53.8ms, 0 | 20260508 13:26:32.390 | INFO | BAZWRITER : 67.2%, 2.4, 84.5ms, 0 | 20260508 13:26:32.390 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:26:32.390 | INFO | Uploaded 36GiB to GPU in 2.62156s (13.7323GiB/s) | 20260508 13:26:32.390 | INFO | Downloaded 38GiB to GPU in 2.72269s (13.9568GiB/s) | 20260508 13:26:37.527 | INFO | Analyzing chunk frames = [8454656,8455168) | 20260508 13:26:37.527 | INFO | MemInfo: { | 20260508 13:26:37.527 | INFO | "gpu" : " 17 GB", | 20260508 13:26:37.527 | INFO | "huge" : "", | 20260508 13:26:37.527 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:37.527 | INFO | "malloc" : " 14 GB", | 20260508 13:26:37.527 | INFO | "pinned" : " 29 GB" | 20260508 13:26:37.527 | INFO | } | 20260508 13:26:37.527 | INFO | Perf report for the last 5.13775 seconds: | 20260508 13:26:37.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:37.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.527 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:26:37.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.527 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:26:37.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:37.528 | INFO | Uploaded 36GiB to GPU in 2.49648s (14.4203GiB/s) | 20260508 13:26:37.528 | INFO | Downloaded 34GiB to GPU in 2.4707s (13.7613GiB/s) | 20260508 13:26:42.664 | INFO | Analyzing chunk frames = [8455168,8455680) | 20260508 13:26:42.664 | INFO | MemInfo: { | 20260508 13:26:42.664 | INFO | "gpu" : " 17 GB", | 20260508 13:26:42.664 | INFO | "huge" : "", | 20260508 13:26:42.664 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:42.664 | INFO | "malloc" : " 14 GB", | 20260508 13:26:42.665 | INFO | "pinned" : " 29 GB" | 20260508 13:26:42.665 | INFO | } | 20260508 13:26:42.665 | INFO | Perf report for the last 5.13735 seconds: | 20260508 13:26:42.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:42.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | ANALYSIS : 66.5%, 3.0, 103.3ms, 0 | 20260508 13:26:42.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260508 13:26:42.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:42.665 | INFO | Uploaded 36GiB to GPU in 2.52833s (14.2387GiB/s) | 20260508 13:26:42.665 | INFO | Downloaded 34GiB to GPU in 2.46758s (13.7787GiB/s) | 20260508 13:26:47.807 | INFO | Analyzing chunk frames = [8455680,8456192) | 20260508 13:26:47.807 | INFO | MemInfo: { | 20260508 13:26:47.807 | INFO | "gpu" : " 17 GB", | 20260508 13:26:47.807 | INFO | "huge" : "", | 20260508 13:26:47.807 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:47.807 | INFO | "malloc" : " 14 GB", | 20260508 13:26:47.807 | INFO | "pinned" : " 29 GB" | 20260508 13:26:47.807 | INFO | } | 20260508 13:26:47.807 | INFO | Perf report for the last 5.14259 seconds: | 20260508 13:26:47.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:47.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.808 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 13:26:47.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.808 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:26:47.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:47.808 | INFO | Uploaded 36GiB to GPU in 2.51865s (14.2934GiB/s) | 20260508 13:26:47.808 | INFO | Downloaded 34GiB to GPU in 2.47155s (13.7566GiB/s) | 20260508 13:26:52.944 | INFO | Analyzing chunk frames = [8456192,8456704) | 20260508 13:26:52.944 | INFO | MemInfo: { | 20260508 13:26:52.944 | INFO | "gpu" : " 17 GB", | 20260508 13:26:52.944 | INFO | "huge" : "", | 20260508 13:26:52.944 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:52.945 | INFO | "malloc" : " 14 GB", | 20260508 13:26:52.945 | INFO | "pinned" : " 29 GB" | 20260508 13:26:52.945 | INFO | } | 20260508 13:26:52.945 | INFO | Perf report for the last 5.13739 seconds: | 20260508 13:26:52.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:52.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | ANALYSIS : 66.5%, 3.0, 103.7ms, 0 | 20260508 13:26:52.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:26:52.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:52.945 | INFO | Uploaded 36GiB to GPU in 2.48963s (14.46GiB/s) | 20260508 13:26:52.945 | INFO | Downloaded 34GiB to GPU in 2.52414s (13.4699GiB/s) | 20260508 13:26:58.084 | INFO | Analyzing chunk frames = [8456704,8457216) | 20260508 13:26:58.084 | INFO | MemInfo: { | 20260508 13:26:58.084 | INFO | "gpu" : " 17 GB", | 20260508 13:26:58.084 | INFO | "huge" : "", | 20260508 13:26:58.084 | INFO | "huge_shared" : " 29 GB", | 20260508 13:26:58.084 | INFO | "malloc" : " 14 GB", | 20260508 13:26:58.084 | INFO | "pinned" : " 29 GB" | 20260508 13:26:58.084 | INFO | } | 20260508 13:26:58.084 | INFO | Perf report for the last 5.13911 seconds: | 20260508 13:26:58.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:26:58.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:26:58.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:26:58.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:26:58.084 | INFO | Uploaded 36GiB to GPU in 2.43692s (14.7728GiB/s) | 20260508 13:26:58.084 | INFO | Downloaded 34GiB to GPU in 2.44318s (13.9163GiB/s) | 20260508 13:27:03.218 | INFO | Analyzing chunk frames = [8457216,8457728) | 20260508 13:27:03.218 | INFO | MemInfo: { | 20260508 13:27:03.218 | INFO | "gpu" : " 17 GB", | 20260508 13:27:03.218 | INFO | "huge" : "", | 20260508 13:27:03.218 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:03.218 | INFO | "malloc" : " 14 GB", | 20260508 13:27:03.218 | INFO | "pinned" : " 29 GB" | 20260508 13:27:03.218 | INFO | } | 20260508 13:27:03.218 | INFO | Perf report for the last 5.13457 seconds: | 20260508 13:27:03.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:03.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.218 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:27:03.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.218 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:27:03.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:03.219 | INFO | Uploaded 36GiB to GPU in 2.4637s (14.6122GiB/s) | 20260508 13:27:03.219 | INFO | Downloaded 34GiB to GPU in 2.45886s (13.8276GiB/s) | 20260508 13:27:08.360 | INFO | Analyzing chunk frames = [8457728,8458240) | 20260508 13:27:08.360 | INFO | MemInfo: { | 20260508 13:27:08.360 | INFO | "gpu" : " 17 GB", | 20260508 13:27:08.360 | INFO | "huge" : "", | 20260508 13:27:08.360 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:08.360 | INFO | "malloc" : " 14 GB", | 20260508 13:27:08.360 | INFO | "pinned" : " 29 GB" | 20260508 13:27:08.360 | INFO | } | 20260508 13:27:08.360 | INFO | Perf report for the last 5.14169 seconds: | 20260508 13:27:08.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:08.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.360 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 13:27:08.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.360 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:27:08.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:08.361 | INFO | Uploaded 36GiB to GPU in 2.4509s (14.6885GiB/s) | 20260508 13:27:08.361 | INFO | Downloaded 34GiB to GPU in 2.44688s (13.8952GiB/s) | 20260508 13:27:08.457 | INFO | Starting a round of Baz IO | 20260508 13:27:11.869 | INFO | Populating RTMetrics report | 20260508 13:27:11.870 | INFO | RTMetrics report populated | 20260508 13:27:11.870 | INFO | Opening tmp RTMetrics json file | 20260508 13:27:11.879 | INFO | Serializing RTMetrics to json | 20260508 13:27:11.880 | INFO | Serialization done, swapping tmp file now... | 20260508 13:27:11.881 | INFO | RTMetrics done | 20260508 13:27:11.937 | INFO | Finished a round of Baz IO: Wrote 1.387 GiB in 3.479 seconds | 20260508 13:27:11.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.00% / 60.05% | 20260508 13:27:11.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141076% / 3.97364e-06% / 5.5631e-05% | 20260508 13:27:11.937 | INFO | Baz IO overall: | 20260508 13:27:11.937 | INFO | BytesWritten : 3604549500982 | 20260508 13:27:11.937 | INFO | NumEvents : 1728250176742 | 20260508 13:27:13.503 | INFO | Analyzing chunk frames = [8458240,8458752) | 20260508 13:27:13.503 | INFO | MemInfo: { | 20260508 13:27:13.503 | INFO | "gpu" : " 17 GB", | 20260508 13:27:13.503 | INFO | "huge" : "", | 20260508 13:27:13.504 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:13.504 | INFO | "malloc" : " 14 GB", | 20260508 13:27:13.504 | INFO | "pinned" : " 29 GB" | 20260508 13:27:13.504 | INFO | } | 20260508 13:27:13.504 | INFO | Perf report for the last 5.14355 seconds: | 20260508 13:27:13.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:13.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:13.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:13.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:13.504 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:27:13.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:13.504 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:27:13.504 | INFO | PRE_HQ : 65.3%, 1.6, 53.5ms, 0 | 20260508 13:27:13.504 | INFO | BAZWRITER : 67.0%, 2.5, 84.9ms, 0 | 20260508 13:27:13.504 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:27:13.504 | INFO | Uploaded 36GiB to GPU in 2.61183s (13.7834GiB/s) | 20260508 13:27:13.504 | INFO | Downloaded 38GiB to GPU in 2.72049s (13.9681GiB/s) | 20260508 13:27:18.644 | INFO | Analyzing chunk frames = [8458752,8459264) | 20260508 13:27:18.644 | INFO | MemInfo: { | 20260508 13:27:18.644 | INFO | "gpu" : " 17 GB", | 20260508 13:27:18.644 | INFO | "huge" : "", | 20260508 13:27:18.644 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:18.644 | INFO | "malloc" : " 14 GB", | 20260508 13:27:18.644 | INFO | "pinned" : " 29 GB" | 20260508 13:27:18.644 | INFO | } | 20260508 13:27:18.644 | INFO | Perf report for the last 5.14001 seconds: | 20260508 13:27:18.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:18.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:27:18.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260508 13:27:18.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:18.644 | INFO | Uploaded 36GiB to GPU in 2.44462s (14.7262GiB/s) | 20260508 13:27:18.644 | INFO | Downloaded 34GiB to GPU in 2.48308s (13.6927GiB/s) | 20260508 13:27:23.782 | INFO | Analyzing chunk frames = [8459264,8459776) | 20260508 13:27:23.782 | INFO | MemInfo: { | 20260508 13:27:23.782 | INFO | "gpu" : " 17 GB", | 20260508 13:27:23.782 | INFO | "huge" : "", | 20260508 13:27:23.782 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:23.782 | INFO | "malloc" : " 14 GB", | 20260508 13:27:23.782 | INFO | "pinned" : " 29 GB" | 20260508 13:27:23.782 | INFO | } | 20260508 13:27:23.782 | INFO | Perf report for the last 5.1384 seconds: | 20260508 13:27:23.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:23.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.782 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 13:27:23.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.782 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:27:23.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:23.783 | INFO | Uploaded 36GiB to GPU in 2.42483s (14.8464GiB/s) | 20260508 13:27:23.783 | INFO | Downloaded 34GiB to GPU in 2.47014s (13.7644GiB/s) | 20260508 13:27:28.917 | INFO | Analyzing chunk frames = [8459776,8460288) | 20260508 13:27:28.917 | INFO | MemInfo: { | 20260508 13:27:28.917 | INFO | "gpu" : " 17 GB", | 20260508 13:27:28.917 | INFO | "huge" : "", | 20260508 13:27:28.917 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:28.917 | INFO | "malloc" : " 14 GB", | 20260508 13:27:28.919 | INFO | "pinned" : " 29 GB" | 20260508 13:27:28.919 | INFO | } | 20260508 13:27:28.919 | INFO | Perf report for the last 5.13652 seconds: | 20260508 13:27:28.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:28.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | ANALYSIS : 65.3%, 3.0, 101.3ms, 0 | 20260508 13:27:28.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:27:28.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:28.919 | INFO | Uploaded 36GiB to GPU in 2.3836s (15.1032GiB/s) | 20260508 13:27:28.919 | INFO | Downloaded 34GiB to GPU in 2.48429s (13.686GiB/s) | 20260508 13:27:34.061 | INFO | Analyzing chunk frames = [8460288,8460800) | 20260508 13:27:34.061 | INFO | MemInfo: { | 20260508 13:27:34.061 | INFO | "gpu" : " 17 GB", | 20260508 13:27:34.061 | INFO | "huge" : "", | 20260508 13:27:34.061 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:34.061 | INFO | "malloc" : " 14 GB", | 20260508 13:27:34.061 | INFO | "pinned" : " 29 GB" | 20260508 13:27:34.061 | INFO | } | 20260508 13:27:34.061 | INFO | Perf report for the last 5.14239 seconds: | 20260508 13:27:34.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:34.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.061 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:27:34.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.061 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:27:34.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:34.062 | INFO | Uploaded 36GiB to GPU in 2.35005s (15.3188GiB/s) | 20260508 13:27:34.062 | INFO | Downloaded 34GiB to GPU in 2.52164s (13.4833GiB/s) | 20260508 13:27:39.200 | INFO | Analyzing chunk frames = [8460800,8461312) | 20260508 13:27:39.200 | INFO | MemInfo: { | 20260508 13:27:39.200 | INFO | "gpu" : " 17 GB", | 20260508 13:27:39.200 | INFO | "huge" : "", | 20260508 13:27:39.200 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:39.201 | INFO | "malloc" : " 14 GB", | 20260508 13:27:39.201 | INFO | "pinned" : " 29 GB" | 20260508 13:27:39.201 | INFO | } | 20260508 13:27:39.201 | INFO | Perf report for the last 5.13958 seconds: | 20260508 13:27:39.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:39.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:27:39.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:27:39.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:39.201 | INFO | Uploaded 36GiB to GPU in 2.42052s (14.8728GiB/s) | 20260508 13:27:39.201 | INFO | Downloaded 34GiB to GPU in 2.45309s (13.8601GiB/s) | 20260508 13:27:44.342 | INFO | Analyzing chunk frames = [8461312,8461824) | 20260508 13:27:44.342 | INFO | MemInfo: { | 20260508 13:27:44.342 | INFO | "gpu" : " 17 GB", | 20260508 13:27:44.342 | INFO | "huge" : "", | 20260508 13:27:44.342 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:44.342 | INFO | "malloc" : " 14 GB", | 20260508 13:27:44.342 | INFO | "pinned" : " 29 GB" | 20260508 13:27:44.342 | INFO | } | 20260508 13:27:44.342 | INFO | Perf report for the last 5.14122 seconds: | 20260508 13:27:44.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:44.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | ANALYSIS : 64.5%, 3.0, 100.4ms, 0 | 20260508 13:27:44.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:27:44.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:44.342 | INFO | Uploaded 36GiB to GPU in 2.3086s (15.5938GiB/s) | 20260508 13:27:44.342 | INFO | Downloaded 34GiB to GPU in 2.4248s (14.0218GiB/s) | 20260508 13:27:49.478 | INFO | Analyzing chunk frames = [8461824,8462336) | 20260508 13:27:49.478 | INFO | MemInfo: { | 20260508 13:27:49.478 | INFO | "gpu" : " 17 GB", | 20260508 13:27:49.478 | INFO | "huge" : "", | 20260508 13:27:49.478 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:49.478 | INFO | "malloc" : " 14 GB", | 20260508 13:27:49.478 | INFO | "pinned" : " 29 GB" | 20260508 13:27:49.478 | INFO | } | 20260508 13:27:49.478 | INFO | Perf report for the last 5.13665 seconds: | 20260508 13:27:49.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:49.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | ANALYSIS : 64.5%, 3.0, 100.8ms, 0 | 20260508 13:27:49.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:27:49.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:49.479 | INFO | Uploaded 36GiB to GPU in 2.31711s (15.5366GiB/s) | 20260508 13:27:49.479 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:27:49.479 | INFO | Total mem usage: 16.7 GiB | 20260508 13:27:49.479 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:27:49.479 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:27:49.479 | INFO | Downloaded 34GiB to GPU in 2.43691s (13.9521GiB/s) | 20260508 13:27:49.479 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:27:49.479 | INFO | High Water Breakdown: | 20260508 13:27:49.479 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:27:49.479 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:27:49.479 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:27:49.479 | INFO | ------------------------------------------ | 20260508 13:27:49.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:27:49.479 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:27:49.479 | INFO | Total mem usage: 26.5 GiB | 20260508 13:27:49.479 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:27:49.479 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:27:49.479 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:27:49.479 | INFO | High Water Breakdown: | 20260508 13:27:49.479 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:27:49.479 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:27:49.479 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:27:49.479 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:27:49.479 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:27:49.479 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:27:49.479 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:27:49.479 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:27:49.479 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:27:49.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:27:49.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:27:49.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:27:49.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:27:49.479 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:27:49.479 | INFO | ------------------------------------------ | 20260508 13:27:49.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:27:49.479 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:27:49.479 | INFO | Total mem usage: 27.8 GiB | 20260508 13:27:49.479 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:27:49.479 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:27:49.479 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:27:49.479 | INFO | High Water Breakdown: | 20260508 13:27:49.479 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:27:49.479 | INFO | ------------------------------------------ | 20260508 13:27:49.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:27:49.479 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:27:49.479 | INFO | Total mem usage: 15.7 GiB | 20260508 13:27:49.479 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:27:49.479 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:27:49.479 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:27:49.479 | INFO | High Water Breakdown: | 20260508 13:27:49.479 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:27:49.479 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:27:49.479 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:27:49.479 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:27:49.479 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:27:49.479 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:27:49.479 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:27:49.479 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:27:49.479 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:27:49.479 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:27:49.479 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:27:49.479 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:27:49.479 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:27:49.479 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:27:49.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:27:49.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:27:49.479 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:27:49.479 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:27:49.479 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:27:49.479 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:27:49.479 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:27:49.479 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:27:49.479 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:27:49.479 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:27:49.479 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:27:49.479 | INFO | ------------------------------------------ | 20260508 13:27:49.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:27:49.479 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:27:49.479 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:27:49.479 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:27:49.479 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.31e-01 | 20260508 13:27:49.479 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.08e-01, 1.59e-02 / 7.90e-03 | 20260508 13:27:49.479 | INFO | PerfReport: 12.21%, Baseline -- 3.99e+00 / 4.38e+00, 5.91e-01 / 2.34e-01 | 20260508 13:27:49.479 | INFO | PerfReport: 3.38%, Histogram -- 1.11e+00 / 1.42e+00, 2.10e-01 / 1.14e-01 | 20260508 13:27:49.479 | INFO | PerfReport: 15.97%, DME -- 4.17e+01 / 4.50e+01, 3.70e+00 / 1.36e+00 | 20260508 13:27:49.479 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.87e+00 / 1.35e+00 | 20260508 13:27:49.479 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.31e+00 / 2.40e+00, 4.05e-01 / 2.44e-01 | 20260508 13:27:49.479 | INFO | PerfReport: 2.91%, Metrics -- 9.52e-01 / 1.06e+00, 2.37e-01 / 1.05e-01 | 20260508 13:27:49.479 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.89e-01 / 3.19e-01, 4.69e-02 / 2.27e-02 | 20260508 13:27:49.479 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:27:49.480 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:27:49.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:27:49.480 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 24.40%, Upload -- 2.56e+01 / 0.00e+00, 7.94e+00 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 24.86%, Download -- 2.60e+01 / 0.00e+00, 8.23e+00 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 31.19%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 2.08%, LockUpload -- 2.17e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 2.60%, LockDownload -- 2.72e+00 / 0.00e+00, 4.01e+00 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 14.86%, LockCompute -- 1.56e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:27:49.480 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260508 13:27:49.575 | INFO | Starting a round of Baz IO | 20260508 13:27:52.974 | INFO | Populating RTMetrics report | 20260508 13:27:52.975 | INFO | RTMetrics report populated | 20260508 13:27:52.975 | INFO | Opening tmp RTMetrics json file | 20260508 13:27:52.985 | INFO | Serializing RTMetrics to json | 20260508 13:27:52.986 | INFO | Serialization done, swapping tmp file now... | 20260508 13:27:52.986 | INFO | RTMetrics done | 20260508 13:27:53.046 | INFO | Finished a round of Baz IO: Wrote 1.387 GiB in 3.471 seconds | 20260508 13:27:53.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.05% | 20260508 13:27:53.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141339% / 0% / 7.15256e-05% | 20260508 13:27:53.046 | INFO | Baz IO overall: | 20260508 13:27:53.046 | INFO | BytesWritten : 3606039167110 | 20260508 13:27:53.046 | INFO | NumEvents : 1728947598507 | 20260508 13:27:54.620 | INFO | Analyzing chunk frames = [8462336,8462848) | 20260508 13:27:54.620 | INFO | MemInfo: { | 20260508 13:27:54.620 | INFO | "gpu" : " 17 GB", | 20260508 13:27:54.620 | INFO | "huge" : "", | 20260508 13:27:54.620 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:54.620 | INFO | "malloc" : " 14 GB", | 20260508 13:27:54.620 | INFO | "pinned" : " 29 GB" | 20260508 13:27:54.620 | INFO | } | 20260508 13:27:54.620 | INFO | Perf report for the last 5.14148 seconds: | 20260508 13:27:54.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:54.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:54.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:54.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:54.620 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 13:27:54.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:54.620 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:27:54.620 | INFO | PRE_HQ : 65.1%, 1.6, 53.6ms, 0 | 20260508 13:27:54.620 | INFO | BAZWRITER : 66.7%, 2.5, 84.7ms, 0 | 20260508 13:27:54.620 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:27:54.620 | INFO | Uploaded 36GiB to GPU in 2.63511s (13.6616GiB/s) | 20260508 13:27:54.620 | INFO | Downloaded 38GiB to GPU in 2.72595s (13.9401GiB/s) | 20260508 13:27:59.758 | INFO | Analyzing chunk frames = [8462848,8463360) | 20260508 13:27:59.758 | INFO | MemInfo: { | 20260508 13:27:59.758 | INFO | "gpu" : " 17 GB", | 20260508 13:27:59.758 | INFO | "huge" : "", | 20260508 13:27:59.758 | INFO | "huge_shared" : " 29 GB", | 20260508 13:27:59.758 | INFO | "malloc" : " 14 GB", | 20260508 13:27:59.758 | INFO | "pinned" : " 29 GB" | 20260508 13:27:59.758 | INFO | } | 20260508 13:27:59.758 | INFO | Perf report for the last 5.13775 seconds: | 20260508 13:27:59.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:27:59.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:27:59.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:27:59.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:27:59.758 | INFO | Uploaded 36GiB to GPU in 2.47452s (14.5483GiB/s) | 20260508 13:27:59.758 | INFO | Downloaded 34GiB to GPU in 2.4847s (13.6837GiB/s) | 20260508 13:28:04.895 | INFO | Analyzing chunk frames = [8463360,8463872) | 20260508 13:28:04.895 | INFO | MemInfo: { | 20260508 13:28:04.895 | INFO | "gpu" : " 17 GB", | 20260508 13:28:04.895 | INFO | "huge" : "", | 20260508 13:28:04.895 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:04.895 | INFO | "malloc" : " 14 GB", | 20260508 13:28:04.895 | INFO | "pinned" : " 29 GB" | 20260508 13:28:04.895 | INFO | } | 20260508 13:28:04.895 | INFO | Perf report for the last 5.13709 seconds: | 20260508 13:28:04.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:04.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:28:04.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:28:04.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:04.895 | INFO | Uploaded 36GiB to GPU in 2.46203s (14.6221GiB/s) | 20260508 13:28:04.895 | INFO | Downloaded 34GiB to GPU in 2.47197s (13.7542GiB/s) | 20260508 13:28:10.032 | INFO | Analyzing chunk frames = [8463872,8464384) | 20260508 13:28:10.032 | INFO | MemInfo: { | 20260508 13:28:10.032 | INFO | "gpu" : " 17 GB", | 20260508 13:28:10.032 | INFO | "huge" : "", | 20260508 13:28:10.032 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:10.032 | INFO | "malloc" : " 14 GB", | 20260508 13:28:10.032 | INFO | "pinned" : " 29 GB" | 20260508 13:28:10.032 | INFO | } | 20260508 13:28:10.032 | INFO | Perf report for the last 5.13757 seconds: | 20260508 13:28:10.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:10.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | ANALYSIS : 66.0%, 3.0, 102.2ms, 0 | 20260508 13:28:10.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:28:10.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:10.033 | INFO | Uploaded 36GiB to GPU in 2.46472s (14.6061GiB/s) | 20260508 13:28:10.033 | INFO | Downloaded 34GiB to GPU in 2.45662s (13.8402GiB/s) | 20260508 13:28:15.167 | INFO | Analyzing chunk frames = [8464384,8464896) | 20260508 13:28:15.167 | INFO | MemInfo: { | 20260508 13:28:15.167 | INFO | "gpu" : " 17 GB", | 20260508 13:28:15.167 | INFO | "huge" : "", | 20260508 13:28:15.167 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:15.167 | INFO | "malloc" : " 14 GB", | 20260508 13:28:15.167 | INFO | "pinned" : " 29 GB" | 20260508 13:28:15.167 | INFO | } | 20260508 13:28:15.167 | INFO | Perf report for the last 5.13491 seconds: | 20260508 13:28:15.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:15.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 13:28:15.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:28:15.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:15.168 | INFO | Uploaded 36GiB to GPU in 2.40969s (14.9397GiB/s) | 20260508 13:28:15.168 | INFO | Downloaded 34GiB to GPU in 2.5146s (13.521GiB/s) | 20260508 13:28:20.316 | INFO | Analyzing chunk frames = [8464896,8465408) | 20260508 13:28:20.316 | INFO | MemInfo: { | 20260508 13:28:20.316 | INFO | "gpu" : " 17 GB", | 20260508 13:28:20.316 | INFO | "huge" : "", | 20260508 13:28:20.316 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:20.316 | INFO | "malloc" : " 14 GB", | 20260508 13:28:20.316 | INFO | "pinned" : " 29 GB" | 20260508 13:28:20.316 | INFO | } | 20260508 13:28:20.316 | INFO | Perf report for the last 5.14821 seconds: | 20260508 13:28:20.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:20.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | ANALYSIS : 65.0%, 3.0, 101.4ms, 0 | 20260508 13:28:20.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:28:20.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:20.316 | INFO | Uploaded 36GiB to GPU in 2.36895s (15.1966GiB/s) | 20260508 13:28:20.316 | INFO | Downloaded 34GiB to GPU in 2.45652s (13.8407GiB/s) | 20260508 13:28:25.461 | INFO | Analyzing chunk frames = [8465408,8465920) | 20260508 13:28:25.461 | INFO | MemInfo: { | 20260508 13:28:25.461 | INFO | "gpu" : " 17 GB", | 20260508 13:28:25.461 | INFO | "huge" : "", | 20260508 13:28:25.461 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:25.461 | INFO | "malloc" : " 14 GB", | 20260508 13:28:25.461 | INFO | "pinned" : " 29 GB" | 20260508 13:28:25.461 | INFO | } | 20260508 13:28:25.461 | INFO | Perf report for the last 5.14532 seconds: | 20260508 13:28:25.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:25.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | ANALYSIS : 64.8%, 3.0, 100.8ms, 0 | 20260508 13:28:25.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 13:28:25.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:25.461 | INFO | Uploaded 36GiB to GPU in 2.2778s (15.8047GiB/s) | 20260508 13:28:25.461 | INFO | Downloaded 34GiB to GPU in 2.44007s (13.934GiB/s) | 20260508 13:28:30.589 | INFO | Analyzing chunk frames = [8465920,8466432) | 20260508 13:28:30.589 | INFO | MemInfo: { | 20260508 13:28:30.589 | INFO | "gpu" : " 17 GB", | 20260508 13:28:30.589 | INFO | "huge" : "", | 20260508 13:28:30.589 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:30.589 | INFO | "malloc" : " 14 GB", | 20260508 13:28:30.589 | INFO | "pinned" : " 29 GB" | 20260508 13:28:30.589 | INFO | } | 20260508 13:28:30.589 | INFO | Perf report for the last 5.12843 seconds: | 20260508 13:28:30.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:30.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | ANALYSIS : 64.5%, 3.0, 100.5ms, 0 | 20260508 13:28:30.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:28:30.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:30.590 | INFO | Uploaded 36GiB to GPU in 2.30866s (15.5935GiB/s) | 20260508 13:28:30.590 | INFO | Downloaded 34GiB to GPU in 2.44652s (13.8973GiB/s) | 20260508 13:28:30.686 | INFO | Starting a round of Baz IO | 20260508 13:28:34.090 | INFO | Populating RTMetrics report | 20260508 13:28:34.091 | INFO | RTMetrics report populated | 20260508 13:28:34.091 | INFO | Opening tmp RTMetrics json file | 20260508 13:28:34.096 | INFO | Serializing RTMetrics to json | 20260508 13:28:34.098 | INFO | Serialization done, swapping tmp file now... | 20260508 13:28:34.098 | INFO | RTMetrics done | 20260508 13:28:34.155 | INFO | Finished a round of Baz IO: Wrote 1.387 GiB in 3.468 seconds | 20260508 13:28:34.155 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:28:34.155 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140921% / 3.97364e-06% / 5.96046e-05% | 20260508 13:28:34.155 | INFO | Baz IO overall: | 20260508 13:28:34.155 | INFO | BytesWritten : 3607528480522 | 20260508 13:28:34.155 | INFO | NumEvents : 1729644881765 | 20260508 13:28:35.727 | INFO | Analyzing chunk frames = [8466432,8466944) | 20260508 13:28:35.727 | INFO | MemInfo: { | 20260508 13:28:35.727 | INFO | "gpu" : " 17 GB", | 20260508 13:28:35.727 | INFO | "huge" : "", | 20260508 13:28:35.727 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:35.727 | INFO | "malloc" : " 14 GB", | 20260508 13:28:35.727 | INFO | "pinned" : " 29 GB" | 20260508 13:28:35.727 | INFO | } | 20260508 13:28:35.727 | INFO | Perf report for the last 5.13808 seconds: | 20260508 13:28:35.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:35.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:35.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:35.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:35.728 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 13:28:35.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:35.728 | INFO | RT_METRICS : 38.5%, 1.0, 20.2ms, 0 | 20260508 13:28:35.728 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 13:28:35.728 | INFO | BAZWRITER : 66.8%, 2.5, 84.4ms, 0 | 20260508 13:28:35.728 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:28:35.728 | INFO | Uploaded 36GiB to GPU in 2.60507s (13.8192GiB/s) | 20260508 13:28:35.728 | INFO | Downloaded 38GiB to GPU in 2.72932s (13.9229GiB/s) | 20260508 13:28:40.870 | INFO | Analyzing chunk frames = [8466944,8467456) | 20260508 13:28:40.870 | INFO | MemInfo: { | 20260508 13:28:40.870 | INFO | "gpu" : " 17 GB", | 20260508 13:28:40.870 | INFO | "huge" : "", | 20260508 13:28:40.870 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:40.870 | INFO | "malloc" : " 14 GB", | 20260508 13:28:40.870 | INFO | "pinned" : " 29 GB" | 20260508 13:28:40.870 | INFO | } | 20260508 13:28:40.870 | INFO | Perf report for the last 5.14287 seconds: | 20260508 13:28:40.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:40.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | ANALYSIS : 65.8%, 3.0, 102.6ms, 0 | 20260508 13:28:40.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:28:40.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:40.871 | INFO | Uploaded 36GiB to GPU in 2.42257s (14.8603GiB/s) | 20260508 13:28:40.871 | INFO | Downloaded 34GiB to GPU in 2.47945s (13.7127GiB/s) | 20260508 13:28:46.009 | INFO | Analyzing chunk frames = [8467456,8467968) | 20260508 13:28:46.009 | INFO | MemInfo: { | 20260508 13:28:46.009 | INFO | "gpu" : " 17 GB", | 20260508 13:28:46.009 | INFO | "huge" : "", | 20260508 13:28:46.009 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:46.009 | INFO | "malloc" : " 14 GB", | 20260508 13:28:46.009 | INFO | "pinned" : " 29 GB" | 20260508 13:28:46.009 | INFO | } | 20260508 13:28:46.009 | INFO | Perf report for the last 5.13865 seconds: | 20260508 13:28:46.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:46.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:28:46.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:28:46.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:46.009 | INFO | Uploaded 36GiB to GPU in 2.46885s (14.5817GiB/s) | 20260508 13:28:46.009 | INFO | Downloaded 34GiB to GPU in 2.47236s (13.752GiB/s) | 20260508 13:28:51.156 | INFO | Analyzing chunk frames = [8467968,8468480) | 20260508 13:28:51.156 | INFO | MemInfo: { | 20260508 13:28:51.156 | INFO | "gpu" : " 17 GB", | 20260508 13:28:51.156 | INFO | "huge" : "", | 20260508 13:28:51.156 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:51.156 | INFO | "malloc" : " 14 GB", | 20260508 13:28:51.156 | INFO | "pinned" : " 29 GB" | 20260508 13:28:51.156 | INFO | } | 20260508 13:28:51.156 | INFO | Perf report for the last 5.1467 seconds: | 20260508 13:28:51.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:51.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.156 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 13:28:51.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.156 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:28:51.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:51.157 | INFO | Uploaded 36GiB to GPU in 2.43397s (14.7906GiB/s) | 20260508 13:28:51.157 | INFO | Downloaded 34GiB to GPU in 2.4761s (13.7313GiB/s) | 20260508 13:28:56.292 | INFO | Analyzing chunk frames = [8468480,8468992) | 20260508 13:28:56.292 | INFO | MemInfo: { | 20260508 13:28:56.292 | INFO | "gpu" : " 17 GB", | 20260508 13:28:56.292 | INFO | "huge" : "", | 20260508 13:28:56.292 | INFO | "huge_shared" : " 29 GB", | 20260508 13:28:56.292 | INFO | "malloc" : " 14 GB", | 20260508 13:28:56.292 | INFO | "pinned" : " 29 GB" | 20260508 13:28:56.292 | INFO | } | 20260508 13:28:56.292 | INFO | Perf report for the last 5.13657 seconds: | 20260508 13:28:56.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:28:56.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.292 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:28:56.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.292 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:28:56.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:28:56.293 | INFO | Uploaded 36GiB to GPU in 2.40627s (14.9609GiB/s) | 20260508 13:28:56.293 | INFO | Downloaded 34GiB to GPU in 2.52347s (13.4735GiB/s) | 20260508 13:29:01.426 | INFO | Analyzing chunk frames = [8468992,8469504) | 20260508 13:29:01.426 | INFO | MemInfo: { | 20260508 13:29:01.426 | INFO | "gpu" : " 17 GB", | 20260508 13:29:01.426 | INFO | "huge" : "", | 20260508 13:29:01.426 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:01.426 | INFO | "malloc" : " 14 GB", | 20260508 13:29:01.426 | INFO | "pinned" : " 29 GB" | 20260508 13:29:01.427 | INFO | } | 20260508 13:29:01.427 | INFO | Perf report for the last 5.13439 seconds: | 20260508 13:29:01.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:01.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.427 | INFO | ANALYSIS : 64.9%, 3.0, 100.9ms, 0 | 20260508 13:29:01.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.427 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 13:29:01.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:01.428 | INFO | Uploaded 36GiB to GPU in 2.33225s (15.4357GiB/s) | 20260508 13:29:01.428 | INFO | Downloaded 34GiB to GPU in 2.42403s (14.0262GiB/s) | 20260508 13:29:06.569 | INFO | Analyzing chunk frames = [8469504,8470016) | 20260508 13:29:06.569 | INFO | MemInfo: { | 20260508 13:29:06.569 | INFO | "gpu" : " 17 GB", | 20260508 13:29:06.569 | INFO | "huge" : "", | 20260508 13:29:06.569 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:06.569 | INFO | "malloc" : " 14 GB", | 20260508 13:29:06.569 | INFO | "pinned" : " 29 GB" | 20260508 13:29:06.569 | INFO | } | 20260508 13:29:06.569 | INFO | Perf report for the last 5.14226 seconds: | 20260508 13:29:06.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:06.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.569 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:29:06.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.569 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:29:06.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:06.570 | INFO | Uploaded 36GiB to GPU in 2.47909s (14.5215GiB/s) | 20260508 13:29:06.570 | INFO | Downloaded 34GiB to GPU in 2.46328s (13.8027GiB/s) | 20260508 13:29:11.704 | INFO | Analyzing chunk frames = [8470016,8470528) | 20260508 13:29:11.704 | INFO | MemInfo: { | 20260508 13:29:11.704 | INFO | "gpu" : " 17 GB", | 20260508 13:29:11.704 | INFO | "huge" : "", | 20260508 13:29:11.704 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:11.704 | INFO | "malloc" : " 14 GB", | 20260508 13:29:11.704 | INFO | "pinned" : " 29 GB" | 20260508 13:29:11.704 | INFO | } | 20260508 13:29:11.704 | INFO | Perf report for the last 5.1349 seconds: | 20260508 13:29:11.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:11.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | ANALYSIS : 64.7%, 3.0, 101.2ms, 0 | 20260508 13:29:11.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:29:11.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:11.704 | INFO | Uploaded 36GiB to GPU in 2.3621s (15.2407GiB/s) | 20260508 13:29:11.704 | INFO | Downloaded 34GiB to GPU in 2.43638s (13.9551GiB/s) | 20260508 13:29:11.704 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:29:11.704 | INFO | Total mem usage: 16.7 GiB | 20260508 13:29:11.704 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:29:11.704 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:29:11.704 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:29:11.704 | INFO | High Water Breakdown: | 20260508 13:29:11.704 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:29:11.704 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:29:11.704 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:29:11.704 | INFO | ------------------------------------------ | 20260508 13:29:11.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:29:11.704 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:29:11.704 | INFO | Total mem usage: 26.5 GiB | 20260508 13:29:11.704 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:29:11.704 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:29:11.704 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:29:11.704 | INFO | High Water Breakdown: | 20260508 13:29:11.704 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:29:11.704 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:29:11.704 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:29:11.704 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:29:11.704 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:29:11.704 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:29:11.704 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:29:11.705 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:29:11.705 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:29:11.705 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:29:11.705 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:29:11.705 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:29:11.705 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:29:11.705 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:29:11.705 | INFO | ------------------------------------------ | 20260508 13:29:11.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:29:11.705 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:29:11.705 | INFO | Total mem usage: 27.8 GiB | 20260508 13:29:11.705 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:29:11.705 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:29:11.705 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:29:11.705 | INFO | High Water Breakdown: | 20260508 13:29:11.705 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:29:11.705 | INFO | ------------------------------------------ | 20260508 13:29:11.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:29:11.705 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:29:11.705 | INFO | Total mem usage: 15.7 GiB | 20260508 13:29:11.705 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:29:11.705 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:29:11.705 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:29:11.705 | INFO | High Water Breakdown: | 20260508 13:29:11.705 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:29:11.705 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:29:11.705 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:29:11.705 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:29:11.705 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:29:11.705 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:29:11.705 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:29:11.705 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:29:11.705 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:29:11.705 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:29:11.705 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:29:11.705 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:29:11.705 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:29:11.705 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:29:11.705 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:29:11.705 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:29:11.705 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:29:11.705 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:29:11.705 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:29:11.705 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:29:11.705 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:29:11.705 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:29:11.705 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:29:11.705 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:29:11.705 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:29:11.705 | INFO | ------------------------------------------ | 20260508 13:29:11.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:29:11.705 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:29:11.705 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:29:11.705 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:29:11.705 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 3.51e-01 | 20260508 13:29:11.705 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.01e-01, 1.06e-02 / 5.93e-03 | 20260508 13:29:11.705 | INFO | PerfReport: 12.22%, Baseline -- 4.00e+00 / 4.47e+00, 6.02e-01 / 1.59e-01 | 20260508 13:29:11.705 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.43e+00, 2.11e-01 / 1.11e-01 | 20260508 13:29:11.705 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.49e+01, 3.71e+00 / 9.23e-01 | 20260508 13:29:11.705 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.88e+00 / 1.14e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.30e+00 / 2.33e+00, 4.04e-01 / 2.29e-01 | 20260508 13:29:11.705 | INFO | PerfReport: 2.92%, Metrics -- 9.56e-01 / 1.06e+00, 2.38e-01 / 9.29e-02 | 20260508 13:29:11.705 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.19e-01, 4.63e-02 / 3.69e-02 | 20260508 13:29:11.705 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:29:11.705 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:29:11.705 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:29:11.705 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 24.23%, Upload -- 2.53e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 24.91%, Download -- 2.60e+01 / 0.00e+00, 8.24e+00 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 31.29%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 2.03%, LockUpload -- 2.13e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 2.65%, LockDownload -- 2.77e+00 / 0.00e+00, 4.16e+00 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 14.88%, LockCompute -- 1.56e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:29:11.705 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260508 13:29:11.801 | INFO | Starting a round of Baz IO | 20260508 13:29:15.209 | INFO | Populating RTMetrics report | 20260508 13:29:15.210 | INFO | RTMetrics report populated | 20260508 13:29:15.210 | INFO | Opening tmp RTMetrics json file | 20260508 13:29:15.216 | INFO | Serializing RTMetrics to json | 20260508 13:29:15.217 | INFO | Serialization done, swapping tmp file now... | 20260508 13:29:15.218 | INFO | RTMetrics done | 20260508 13:29:15.282 | INFO | Finished a round of Baz IO: Wrote 1.386 GiB in 3.481 seconds | 20260508 13:29:15.282 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:29:15.282 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140691% / 0% / 8.34465e-05% | 20260508 13:29:15.282 | INFO | Baz IO overall: | 20260508 13:29:15.282 | INFO | BytesWritten : 3609016772024 | 20260508 13:29:15.282 | INFO | NumEvents : 1730341455788 | 20260508 13:29:16.843 | INFO | Analyzing chunk frames = [8470528,8471040) | 20260508 13:29:16.843 | INFO | MemInfo: { | 20260508 13:29:16.843 | INFO | "gpu" : " 17 GB", | 20260508 13:29:16.843 | INFO | "huge" : "", | 20260508 13:29:16.843 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:16.843 | INFO | "malloc" : " 14 GB", | 20260508 13:29:16.843 | INFO | "pinned" : " 29 GB" | 20260508 13:29:16.843 | INFO | } | 20260508 13:29:16.843 | INFO | Perf report for the last 5.13934 seconds: | 20260508 13:29:16.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:16.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:16.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:16.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:16.843 | INFO | ANALYSIS : 67.6%, 3.0, 105.5ms, 0 | 20260508 13:29:16.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:16.844 | INFO | RT_METRICS : 38.0%, 1.0, 20.0ms, 0 | 20260508 13:29:16.844 | INFO | PRE_HQ : 65.2%, 1.6, 53.5ms, 0 | 20260508 13:29:16.844 | INFO | BAZWRITER : 67.0%, 2.4, 83.8ms, 0 | 20260508 13:29:16.844 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:29:16.844 | INFO | Uploaded 36GiB to GPU in 2.64692s (13.6007GiB/s) | 20260508 13:29:16.844 | INFO | Downloaded 38GiB to GPU in 2.71611s (13.9906GiB/s) | 20260508 13:29:21.986 | INFO | Analyzing chunk frames = [8471040,8471552) | 20260508 13:29:21.986 | INFO | MemInfo: { | 20260508 13:29:21.986 | INFO | "gpu" : " 17 GB", | 20260508 13:29:21.986 | INFO | "huge" : "", | 20260508 13:29:21.986 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:21.986 | INFO | "malloc" : " 14 GB", | 20260508 13:29:21.986 | INFO | "pinned" : " 29 GB" | 20260508 13:29:21.986 | INFO | } | 20260508 13:29:21.986 | INFO | Perf report for the last 5.14255 seconds: | 20260508 13:29:21.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:21.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:29:21.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:29:21.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:21.986 | INFO | Uploaded 36GiB to GPU in 2.49018s (14.4568GiB/s) | 20260508 13:29:21.986 | INFO | Downloaded 34GiB to GPU in 2.46319s (13.8033GiB/s) | 20260508 13:29:27.128 | INFO | Analyzing chunk frames = [8471552,8472064) | 20260508 13:29:27.128 | INFO | MemInfo: { | 20260508 13:29:27.128 | INFO | "gpu" : " 17 GB", | 20260508 13:29:27.128 | INFO | "huge" : "", | 20260508 13:29:27.128 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:27.128 | INFO | "malloc" : " 14 GB", | 20260508 13:29:27.128 | INFO | "pinned" : " 29 GB" | 20260508 13:29:27.128 | INFO | } | 20260508 13:29:27.129 | INFO | Perf report for the last 5.1427 seconds: | 20260508 13:29:27.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:27.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:29:27.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 13:29:27.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:27.129 | INFO | Uploaded 36GiB to GPU in 2.52035s (14.2837GiB/s) | 20260508 13:29:27.129 | INFO | Downloaded 34GiB to GPU in 2.45623s (13.8424GiB/s) | 20260508 13:29:32.267 | INFO | Analyzing chunk frames = [8472064,8472576) | 20260508 13:29:32.267 | INFO | MemInfo: { | 20260508 13:29:32.267 | INFO | "gpu" : " 17 GB", | 20260508 13:29:32.267 | INFO | "huge" : "", | 20260508 13:29:32.267 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:32.268 | INFO | "malloc" : " 14 GB", | 20260508 13:29:32.268 | INFO | "pinned" : " 29 GB" | 20260508 13:29:32.268 | INFO | } | 20260508 13:29:32.268 | INFO | Perf report for the last 5.1393 seconds: | 20260508 13:29:32.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:32.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.268 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:29:32.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.268 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 13:29:32.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:32.269 | INFO | Uploaded 36GiB to GPU in 2.45309s (14.6754GiB/s) | 20260508 13:29:32.269 | INFO | Downloaded 34GiB to GPU in 2.47305s (13.7482GiB/s) | 20260508 13:29:37.402 | INFO | Analyzing chunk frames = [8472576,8473088) | 20260508 13:29:37.402 | INFO | MemInfo: { | 20260508 13:29:37.402 | INFO | "gpu" : " 17 GB", | 20260508 13:29:37.402 | INFO | "huge" : "", | 20260508 13:29:37.402 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:37.402 | INFO | "malloc" : " 14 GB", | 20260508 13:29:37.402 | INFO | "pinned" : " 29 GB" | 20260508 13:29:37.402 | INFO | } | 20260508 13:29:37.402 | INFO | Perf report for the last 5.13447 seconds: | 20260508 13:29:37.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:37.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.402 | INFO | ANALYSIS : 66.8%, 3.0, 104.1ms, 0 | 20260508 13:29:37.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.402 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260508 13:29:37.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:37.403 | INFO | Uploaded 36GiB to GPU in 2.55665s (14.0809GiB/s) | 20260508 13:29:37.403 | INFO | Downloaded 34GiB to GPU in 2.54571s (13.3558GiB/s) | 20260508 13:29:42.543 | INFO | Analyzing chunk frames = [8473088,8473600) | 20260508 13:29:42.543 | INFO | MemInfo: { | 20260508 13:29:42.543 | INFO | "gpu" : " 17 GB", | 20260508 13:29:42.543 | INFO | "huge" : "", | 20260508 13:29:42.543 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:42.543 | INFO | "malloc" : " 14 GB", | 20260508 13:29:42.543 | INFO | "pinned" : " 29 GB" | 20260508 13:29:42.543 | INFO | } | 20260508 13:29:42.543 | INFO | Perf report for the last 5.14084 seconds: | 20260508 13:29:42.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:42.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.543 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:29:42.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.543 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:29:42.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:42.544 | INFO | Uploaded 36GiB to GPU in 2.42448s (14.8486GiB/s) | 20260508 13:29:42.544 | INFO | Downloaded 34GiB to GPU in 2.43306s (13.9742GiB/s) | 20260508 13:29:47.680 | INFO | Analyzing chunk frames = [8473600,8474112) | 20260508 13:29:47.680 | INFO | MemInfo: { | 20260508 13:29:47.680 | INFO | "gpu" : " 17 GB", | 20260508 13:29:47.680 | INFO | "huge" : "", | 20260508 13:29:47.680 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:47.680 | INFO | "malloc" : " 14 GB", | 20260508 13:29:47.680 | INFO | "pinned" : " 29 GB" | 20260508 13:29:47.680 | INFO | } | 20260508 13:29:47.680 | INFO | Perf report for the last 5.137 seconds: | 20260508 13:29:47.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:47.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.680 | INFO | ANALYSIS : 64.6%, 3.0, 100.5ms, 0 | 20260508 13:29:47.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.680 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:29:47.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:47.681 | INFO | Uploaded 36GiB to GPU in 2.34285s (15.3659GiB/s) | 20260508 13:29:47.681 | INFO | Downloaded 34GiB to GPU in 2.39055s (14.2227GiB/s) | 20260508 13:29:52.822 | INFO | Analyzing chunk frames = [8474112,8474624) | 20260508 13:29:52.822 | INFO | MemInfo: { | 20260508 13:29:52.822 | INFO | "gpu" : " 17 GB", | 20260508 13:29:52.822 | INFO | "huge" : "", | 20260508 13:29:52.822 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:52.822 | INFO | "malloc" : " 14 GB", | 20260508 13:29:52.822 | INFO | "pinned" : " 29 GB" | 20260508 13:29:52.822 | INFO | } | 20260508 13:29:52.822 | INFO | Perf report for the last 5.14201 seconds: | 20260508 13:29:52.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:52.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.822 | INFO | ANALYSIS : 65.7%, 3.0, 102.8ms, 0 | 20260508 13:29:52.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.822 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:29:52.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:52.823 | INFO | Uploaded 36GiB to GPU in 2.49649s (14.4202GiB/s) | 20260508 13:29:52.823 | INFO | Downloaded 34GiB to GPU in 2.45504s (13.8491GiB/s) | 20260508 13:29:52.926 | INFO | Starting a round of Baz IO | 20260508 13:29:56.323 | INFO | Populating RTMetrics report | 20260508 13:29:56.324 | INFO | RTMetrics report populated | 20260508 13:29:56.324 | INFO | Opening tmp RTMetrics json file | 20260508 13:29:56.332 | INFO | Serializing RTMetrics to json | 20260508 13:29:56.333 | INFO | Serialization done, swapping tmp file now... | 20260508 13:29:56.334 | INFO | RTMetrics done | 20260508 13:29:56.393 | INFO | Finished a round of Baz IO: Wrote 1.386 GiB in 3.466 seconds | 20260508 13:29:56.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:29:56.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141184% / 0% / 3.97364e-05% | 20260508 13:29:56.393 | INFO | Baz IO overall: | 20260508 13:29:56.393 | INFO | BytesWritten : 3610504634331 | 20260508 13:29:56.393 | INFO | NumEvents : 1731037661694 | 20260508 13:29:57.965 | INFO | Analyzing chunk frames = [8474624,8475136) | 20260508 13:29:57.965 | INFO | MemInfo: { | 20260508 13:29:57.965 | INFO | "gpu" : " 17 GB", | 20260508 13:29:57.965 | INFO | "huge" : "", | 20260508 13:29:57.965 | INFO | "huge_shared" : " 29 GB", | 20260508 13:29:57.965 | INFO | "malloc" : " 14 GB", | 20260508 13:29:57.966 | INFO | "pinned" : " 29 GB" | 20260508 13:29:57.966 | INFO | } | 20260508 13:29:57.966 | INFO | Perf report for the last 5.14336 seconds: | 20260508 13:29:57.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:29:57.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:57.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:57.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:57.966 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:29:57.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:29:57.966 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:29:57.966 | INFO | PRE_HQ : 65.1%, 1.6, 53.5ms, 0 | 20260508 13:29:57.966 | INFO | BAZWRITER : 66.9%, 2.4, 82.9ms, 0 | 20260508 13:29:57.966 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:29:57.966 | INFO | Uploaded 36GiB to GPU in 2.63214s (13.6771GiB/s) | 20260508 13:29:57.966 | INFO | Downloaded 38GiB to GPU in 2.71924s (13.9745GiB/s) | 20260508 13:30:03.107 | INFO | Analyzing chunk frames = [8475136,8475648) | 20260508 13:30:03.107 | INFO | MemInfo: { | 20260508 13:30:03.107 | INFO | "gpu" : " 17 GB", | 20260508 13:30:03.107 | INFO | "huge" : "", | 20260508 13:30:03.107 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:03.107 | INFO | "malloc" : " 14 GB", | 20260508 13:30:03.107 | INFO | "pinned" : " 29 GB" | 20260508 13:30:03.107 | INFO | } | 20260508 13:30:03.107 | INFO | Perf report for the last 5.14133 seconds: | 20260508 13:30:03.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:03.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:30:03.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:30:03.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:03.107 | INFO | Uploaded 36GiB to GPU in 2.5169s (14.3033GiB/s) | 20260508 13:30:03.108 | INFO | Downloaded 34GiB to GPU in 2.47011s (13.7646GiB/s) | 20260508 13:30:08.238 | INFO | Analyzing chunk frames = [8475648,8476160) | 20260508 13:30:08.238 | INFO | MemInfo: { | 20260508 13:30:08.238 | INFO | "gpu" : " 17 GB", | 20260508 13:30:08.238 | INFO | "huge" : "", | 20260508 13:30:08.238 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:08.238 | INFO | "malloc" : " 14 GB", | 20260508 13:30:08.238 | INFO | "pinned" : " 29 GB" | 20260508 13:30:08.238 | INFO | } | 20260508 13:30:08.238 | INFO | Perf report for the last 5.13095 seconds: | 20260508 13:30:08.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:08.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | ANALYSIS : 65.7%, 3.0, 101.9ms, 0 | 20260508 13:30:08.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | PRE_HQ : 22.5%, 1.0, 11.6ms, 0 | 20260508 13:30:08.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:08.238 | INFO | Uploaded 36GiB to GPU in 2.45523s (14.6626GiB/s) | 20260508 13:30:08.238 | INFO | Downloaded 34GiB to GPU in 2.42675s (14.0105GiB/s) | 20260508 13:30:13.380 | INFO | Analyzing chunk frames = [8476160,8476672) | 20260508 13:30:13.380 | INFO | MemInfo: { | 20260508 13:30:13.380 | INFO | "gpu" : " 17 GB", | 20260508 13:30:13.380 | INFO | "huge" : "", | 20260508 13:30:13.380 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:13.380 | INFO | "malloc" : " 14 GB", | 20260508 13:30:13.380 | INFO | "pinned" : " 29 GB" | 20260508 13:30:13.380 | INFO | } | 20260508 13:30:13.380 | INFO | Perf report for the last 5.14259 seconds: | 20260508 13:30:13.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:13.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.380 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 13:30:13.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.381 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:30:13.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:13.381 | INFO | Uploaded 36GiB to GPU in 2.55033s (14.1158GiB/s) | 20260508 13:30:13.381 | INFO | Downloaded 34GiB to GPU in 2.44726s (13.8931GiB/s) | 20260508 13:30:18.516 | INFO | Analyzing chunk frames = [8476672,8477184) | 20260508 13:30:18.516 | INFO | MemInfo: { | 20260508 13:30:18.516 | INFO | "gpu" : " 17 GB", | 20260508 13:30:18.516 | INFO | "huge" : "", | 20260508 13:30:18.516 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:18.516 | INFO | "malloc" : " 14 GB", | 20260508 13:30:18.516 | INFO | "pinned" : " 29 GB" | 20260508 13:30:18.516 | INFO | } | 20260508 13:30:18.516 | INFO | Perf report for the last 5.1359 seconds: | 20260508 13:30:18.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:18.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:30:18.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:30:18.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:18.517 | INFO | Uploaded 36GiB to GPU in 2.33345s (15.4278GiB/s) | 20260508 13:30:18.517 | INFO | Downloaded 34GiB to GPU in 2.50001s (13.6GiB/s) | 20260508 13:30:23.657 | INFO | Analyzing chunk frames = [8477184,8477696) | 20260508 13:30:23.657 | INFO | MemInfo: { | 20260508 13:30:23.657 | INFO | "gpu" : " 17 GB", | 20260508 13:30:23.657 | INFO | "huge" : "", | 20260508 13:30:23.657 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:23.657 | INFO | "malloc" : " 14 GB", | 20260508 13:30:23.657 | INFO | "pinned" : " 29 GB" | 20260508 13:30:23.657 | INFO | } | 20260508 13:30:23.657 | INFO | Perf report for the last 5.14098 seconds: | 20260508 13:30:23.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:23.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.657 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:30:23.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.657 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:30:23.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:23.658 | INFO | Uploaded 36GiB to GPU in 2.42705s (14.8328GiB/s) | 20260508 13:30:23.658 | INFO | Downloaded 34GiB to GPU in 2.42653s (14.0118GiB/s) | 20260508 13:30:28.799 | INFO | Analyzing chunk frames = [8477696,8478208) | 20260508 13:30:28.799 | INFO | MemInfo: { | 20260508 13:30:28.799 | INFO | "gpu" : " 17 GB", | 20260508 13:30:28.799 | INFO | "huge" : "", | 20260508 13:30:28.799 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:28.799 | INFO | "malloc" : " 14 GB", | 20260508 13:30:28.799 | INFO | "pinned" : " 29 GB" | 20260508 13:30:28.800 | INFO | } | 20260508 13:30:28.800 | INFO | Perf report for the last 5.14215 seconds: | 20260508 13:30:28.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:28.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:30:28.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:30:28.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:28.800 | INFO | Uploaded 36GiB to GPU in 2.48511s (14.4863GiB/s) | 20260508 13:30:28.800 | INFO | Downloaded 34GiB to GPU in 2.45267s (13.8625GiB/s) | 20260508 13:30:33.932 | INFO | Analyzing chunk frames = [8478208,8478720) | 20260508 13:30:33.932 | INFO | MemInfo: { | 20260508 13:30:33.932 | INFO | "gpu" : " 17 GB", | 20260508 13:30:33.932 | INFO | "huge" : "", | 20260508 13:30:33.932 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:33.932 | INFO | "malloc" : " 14 GB", | 20260508 13:30:33.932 | INFO | "pinned" : " 29 GB" | 20260508 13:30:33.932 | INFO | } | 20260508 13:30:33.932 | INFO | Perf report for the last 5.13268 seconds: | 20260508 13:30:33.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:33.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.932 | INFO | ANALYSIS : 65.0%, 3.0, 101.4ms, 0 | 20260508 13:30:33.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.932 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 13:30:33.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:33.933 | INFO | Uploaded 36GiB to GPU in 2.42384s (14.8525GiB/s) | 20260508 13:30:33.933 | INFO | Downloaded 34GiB to GPU in 2.42146s (14.0411GiB/s) | 20260508 13:30:33.933 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:30:33.933 | INFO | Total mem usage: 16.7 GiB | 20260508 13:30:33.933 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:30:33.933 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:30:33.933 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:30:33.933 | INFO | High Water Breakdown: | 20260508 13:30:33.933 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:30:33.933 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:30:33.933 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:30:33.933 | INFO | ------------------------------------------ | 20260508 13:30:33.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:30:33.933 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:30:33.933 | INFO | Total mem usage: 26.5 GiB | 20260508 13:30:33.933 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:30:33.933 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:30:33.933 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:30:33.933 | INFO | High Water Breakdown: | 20260508 13:30:33.933 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:30:33.933 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:30:33.933 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:30:33.933 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:30:33.933 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:30:33.933 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:30:33.933 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:30:33.933 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:30:33.933 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:30:33.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:30:33.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:30:33.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:30:33.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:30:33.933 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:30:33.933 | INFO | ------------------------------------------ | 20260508 13:30:33.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:30:33.933 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:30:33.933 | INFO | Total mem usage: 27.8 GiB | 20260508 13:30:33.933 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:30:33.934 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -14.4%) | 20260508 13:30:33.934 | INFO | Unused Cache: 3.5 GiB (Waste: 12.6%) | 20260508 13:30:33.934 | INFO | High Water Breakdown: | 20260508 13:30:33.934 | INFO | 100% : 24.2 GiB ( 87.4% Of Peak) : WXIPCDataSourceConfig | 20260508 13:30:33.934 | INFO | ------------------------------------------ | 20260508 13:30:33.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:30:33.934 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:30:33.934 | INFO | Total mem usage: 15.7 GiB | 20260508 13:30:33.934 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:30:33.934 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:30:33.934 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:30:33.934 | INFO | High Water Breakdown: | 20260508 13:30:33.934 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:30:33.934 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:30:33.934 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:30:33.934 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:30:33.934 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:30:33.934 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:30:33.934 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:30:33.934 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:30:33.934 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:30:33.934 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:30:33.934 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:30:33.934 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:30:33.934 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:30:33.934 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:30:33.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:30:33.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:30:33.934 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:30:33.934 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:30:33.934 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:30:33.934 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:30:33.934 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:30:33.934 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:30:33.934 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:30:33.934 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:30:33.934 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:30:33.934 | INFO | ------------------------------------------ | 20260508 13:30:33.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:30:33.934 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:30:33.934 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:30:33.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:30:33.934 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.39e+01 / 6.22e-01 | 20260508 13:30:33.934 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.05e-01, 1.21e-02 / 7.20e-03 | 20260508 13:30:33.934 | INFO | PerfReport: 12.21%, Baseline -- 3.98e+00 / 4.42e+00, 5.92e-01 / 2.08e-01 | 20260508 13:30:33.934 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.13e-01 | 20260508 13:30:33.934 | INFO | PerfReport: 15.99%, DME -- 4.17e+01 / 4.50e+01, 3.67e+00 / 1.03e+00 | 20260508 13:30:33.934 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.80e+00 / 1.24e+00 | 20260508 13:30:33.934 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.32e+00 / 2.38e+00, 4.03e-01 / 2.33e-01 | 20260508 13:30:33.934 | INFO | PerfReport: 2.93%, Metrics -- 9.55e-01 / 1.11e+00, 2.39e-01 / 1.14e-01 | 20260508 13:30:33.934 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.89e-01 / 3.18e-01, 4.65e-02 / 2.67e-02 | 20260508 13:30:33.934 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:30:33.934 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:30:33.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:30:33.935 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 24.68%, Upload -- 2.59e+01 / 0.00e+00, 8.03e+00 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 24.71%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 31.08%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 2.30%, LockUpload -- 2.41e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 2.47%, LockDownload -- 2.59e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 14.75%, LockCompute -- 1.55e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:30:33.935 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260508 13:30:34.030 | INFO | Starting a round of Baz IO | 20260508 13:30:37.441 | INFO | Populating RTMetrics report | 20260508 13:30:37.441 | INFO | RTMetrics report populated | 20260508 13:30:37.441 | INFO | Opening tmp RTMetrics json file | 20260508 13:30:37.447 | INFO | Serializing RTMetrics to json | 20260508 13:30:37.449 | INFO | Serialization done, swapping tmp file now... | 20260508 13:30:37.449 | INFO | RTMetrics done | 20260508 13:30:37.506 | INFO | Finished a round of Baz IO: Wrote 1.386 GiB in 3.476 seconds | 20260508 13:30:37.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:30:37.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141136% / 3.97364e-06% / 7.94729e-05% | 20260508 13:30:37.506 | INFO | Baz IO overall: | 20260508 13:30:37.506 | INFO | BytesWritten : 3611992558728 | 20260508 13:30:37.506 | INFO | NumEvents : 1731734005413 | 20260508 13:30:39.080 | INFO | Analyzing chunk frames = [8478720,8479232) | 20260508 13:30:39.080 | INFO | MemInfo: { | 20260508 13:30:39.080 | INFO | "gpu" : " 17 GB", | 20260508 13:30:39.080 | INFO | "huge" : "", | 20260508 13:30:39.080 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:39.080 | INFO | "malloc" : " 14 GB", | 20260508 13:30:39.080 | INFO | "pinned" : " 29 GB" | 20260508 13:30:39.080 | INFO | } | 20260508 13:30:39.080 | INFO | Perf report for the last 5.14751 seconds: | 20260508 13:30:39.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:39.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:39.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:39.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:39.080 | INFO | ANALYSIS : 67.3%, 3.0, 105.4ms, 0 | 20260508 13:30:39.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:39.080 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:30:39.080 | INFO | PRE_HQ : 65.2%, 1.6, 53.9ms, 0 | 20260508 13:30:39.080 | INFO | BAZWRITER : 66.8%, 2.5, 84.3ms, 0 | 20260508 13:30:39.080 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:30:39.080 | INFO | Uploaded 36GiB to GPU in 2.62811s (13.6981GiB/s) | 20260508 13:30:39.080 | INFO | Downloaded 38GiB to GPU in 2.71321s (14.0056GiB/s) | 20260508 13:30:44.216 | INFO | Analyzing chunk frames = [8479232,8479744) | 20260508 13:30:44.216 | INFO | MemInfo: { | 20260508 13:30:44.216 | INFO | "gpu" : " 17 GB", | 20260508 13:30:44.216 | INFO | "huge" : "", | 20260508 13:30:44.216 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:44.217 | INFO | "malloc" : " 14 GB", | 20260508 13:30:44.217 | INFO | "pinned" : " 29 GB" | 20260508 13:30:44.217 | INFO | } | 20260508 13:30:44.217 | INFO | Perf report for the last 5.13701 seconds: | 20260508 13:30:44.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:44.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 13:30:44.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260508 13:30:44.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:44.217 | INFO | Uploaded 36GiB to GPU in 2.42975s (14.8164GiB/s) | 20260508 13:30:44.217 | INFO | Downloaded 34GiB to GPU in 2.47603s (13.7317GiB/s) | 20260508 13:30:49.352 | INFO | Analyzing chunk frames = [8479744,8480256) | 20260508 13:30:49.352 | INFO | MemInfo: { | 20260508 13:30:49.352 | INFO | "gpu" : " 17 GB", | 20260508 13:30:49.352 | INFO | "huge" : "", | 20260508 13:30:49.352 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:49.352 | INFO | "malloc" : " 14 GB", | 20260508 13:30:49.352 | INFO | "pinned" : " 29 GB" | 20260508 13:30:49.352 | INFO | } | 20260508 13:30:49.352 | INFO | Perf report for the last 5.13523 seconds: | 20260508 13:30:49.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:49.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 13:30:49.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260508 13:30:49.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:49.352 | INFO | Uploaded 36GiB to GPU in 2.46185s (14.6232GiB/s) | 20260508 13:30:49.353 | INFO | Downloaded 34GiB to GPU in 2.47334s (13.7466GiB/s) | 20260508 13:30:54.489 | INFO | Analyzing chunk frames = [8480256,8480768) | 20260508 13:30:54.489 | INFO | MemInfo: { | 20260508 13:30:54.489 | INFO | "gpu" : " 17 GB", | 20260508 13:30:54.490 | INFO | "huge" : "", | 20260508 13:30:54.490 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:54.490 | INFO | "malloc" : " 14 GB", | 20260508 13:30:54.490 | INFO | "pinned" : " 29 GB" | 20260508 13:30:54.490 | INFO | } | 20260508 13:30:54.490 | INFO | Perf report for the last 5.13785 seconds: | 20260508 13:30:54.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:54.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.490 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 13:30:54.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.490 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 13:30:54.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:54.491 | INFO | Uploaded 36GiB to GPU in 2.39393s (15.0381GiB/s) | 20260508 13:30:54.491 | INFO | Downloaded 34GiB to GPU in 2.47007s (13.7648GiB/s) | 20260508 13:30:59.635 | INFO | Analyzing chunk frames = [8480768,8481280) | 20260508 13:30:59.635 | INFO | MemInfo: { | 20260508 13:30:59.635 | INFO | "gpu" : " 17 GB", | 20260508 13:30:59.635 | INFO | "huge" : "", | 20260508 13:30:59.635 | INFO | "huge_shared" : " 29 GB", | 20260508 13:30:59.635 | INFO | "malloc" : " 14 GB", | 20260508 13:30:59.635 | INFO | "pinned" : " 29 GB" | 20260508 13:30:59.635 | INFO | } | 20260508 13:30:59.635 | INFO | Perf report for the last 5.1453 seconds: | 20260508 13:30:59.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:30:59.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.635 | INFO | ANALYSIS : 66.0%, 3.0, 103.0ms, 0 | 20260508 13:30:59.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.635 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260508 13:30:59.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:30:59.636 | INFO | Uploaded 36GiB to GPU in 2.34287s (15.3658GiB/s) | 20260508 13:30:59.636 | INFO | Downloaded 34GiB to GPU in 2.51799s (13.5029GiB/s) | 20260508 13:31:04.773 | INFO | Analyzing chunk frames = [8481280,8481792) | 20260508 13:31:04.773 | INFO | MemInfo: { | 20260508 13:31:04.773 | INFO | "gpu" : " 17 GB", | 20260508 13:31:04.773 | INFO | "huge" : "", | 20260508 13:31:04.773 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:04.773 | INFO | "malloc" : " 14 GB", | 20260508 13:31:04.773 | INFO | "pinned" : " 29 GB" | 20260508 13:31:04.774 | INFO | } | 20260508 13:31:04.774 | INFO | Perf report for the last 5.13839 seconds: | 20260508 13:31:04.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:04.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:31:04.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:31:04.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:04.774 | INFO | Uploaded 36GiB to GPU in 2.42339s (14.8552GiB/s) | 20260508 13:31:04.774 | INFO | Downloaded 34GiB to GPU in 2.44285s (13.9182GiB/s) | 20260508 13:31:09.913 | INFO | Analyzing chunk frames = [8481792,8482304) | 20260508 13:31:09.913 | INFO | MemInfo: { | 20260508 13:31:09.913 | INFO | "gpu" : " 17 GB", | 20260508 13:31:09.913 | INFO | "huge" : "", | 20260508 13:31:09.913 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:09.913 | INFO | "malloc" : " 14 GB", | 20260508 13:31:09.913 | INFO | "pinned" : " 29 GB" | 20260508 13:31:09.913 | INFO | } | 20260508 13:31:09.913 | INFO | Perf report for the last 5.13955 seconds: | 20260508 13:31:09.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:09.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.913 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 13:31:09.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.913 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:31:09.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:09.914 | INFO | Uploaded 36GiB to GPU in 2.46568s (14.6005GiB/s) | 20260508 13:31:09.914 | INFO | Downloaded 34GiB to GPU in 2.46802s (13.7762GiB/s) | 20260508 13:31:15.051 | INFO | Analyzing chunk frames = [8482304,8482816) | 20260508 13:31:15.051 | INFO | MemInfo: { | 20260508 13:31:15.051 | INFO | "gpu" : " 17 GB", | 20260508 13:31:15.051 | INFO | "huge" : "", | 20260508 13:31:15.051 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:15.051 | INFO | "malloc" : " 14 GB", | 20260508 13:31:15.051 | INFO | "pinned" : " 29 GB" | 20260508 13:31:15.051 | INFO | } | 20260508 13:31:15.051 | INFO | Perf report for the last 5.13828 seconds: | 20260508 13:31:15.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:15.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | ANALYSIS : 64.8%, 3.0, 101.3ms, 0 | 20260508 13:31:15.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:31:15.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:15.052 | INFO | Uploaded 36GiB to GPU in 2.41538s (14.9045GiB/s) | 20260508 13:31:15.052 | INFO | Downloaded 34GiB to GPU in 2.42226s (14.0365GiB/s) | 20260508 13:31:15.149 | INFO | Starting a round of Baz IO | 20260508 13:31:18.543 | INFO | Populating RTMetrics report | 20260508 13:31:18.543 | INFO | RTMetrics report populated | 20260508 13:31:18.543 | INFO | Opening tmp RTMetrics json file | 20260508 13:31:18.555 | INFO | Serializing RTMetrics to json | 20260508 13:31:18.556 | INFO | Serialization done, swapping tmp file now... | 20260508 13:31:18.557 | INFO | RTMetrics done | 20260508 13:31:18.616 | INFO | Finished a round of Baz IO: Wrote 1.385 GiB in 3.467 seconds | 20260508 13:31:18.616 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:31:18.616 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139999% / 0% / 5.5631e-05% | 20260508 13:31:18.616 | INFO | Baz IO overall: | 20260508 13:31:18.616 | INFO | BytesWritten : 3613479703703 | 20260508 13:31:18.616 | INFO | NumEvents : 1732430043578 | 20260508 13:31:20.188 | INFO | Analyzing chunk frames = [8482816,8483328) | 20260508 13:31:20.188 | INFO | MemInfo: { | 20260508 13:31:20.188 | INFO | "gpu" : " 17 GB", | 20260508 13:31:20.188 | INFO | "huge" : "", | 20260508 13:31:20.188 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:20.188 | INFO | "malloc" : " 14 GB", | 20260508 13:31:20.188 | INFO | "pinned" : " 29 GB" | 20260508 13:31:20.188 | INFO | } | 20260508 13:31:20.188 | INFO | Perf report for the last 5.13649 seconds: | 20260508 13:31:20.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:20.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:20.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:20.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:20.188 | INFO | ANALYSIS : 67.4%, 3.0, 105.1ms, 0 | 20260508 13:31:20.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:20.188 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:31:20.188 | INFO | PRE_HQ : 65.2%, 1.6, 53.5ms, 0 | 20260508 13:31:20.188 | INFO | BAZWRITER : 66.7%, 2.5, 84.7ms, 0 | 20260508 13:31:20.188 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:31:20.188 | INFO | Uploaded 36GiB to GPU in 2.62455s (13.7166GiB/s) | 20260508 13:31:20.188 | INFO | Downloaded 38GiB to GPU in 2.72521s (13.9439GiB/s) | 20260508 13:31:25.331 | INFO | Analyzing chunk frames = [8483328,8483840) | 20260508 13:31:25.331 | INFO | MemInfo: { | 20260508 13:31:25.331 | INFO | "gpu" : " 17 GB", | 20260508 13:31:25.331 | INFO | "huge" : "", | 20260508 13:31:25.331 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:25.331 | INFO | "malloc" : " 14 GB", | 20260508 13:31:25.331 | INFO | "pinned" : " 29 GB" | 20260508 13:31:25.331 | INFO | } | 20260508 13:31:25.331 | INFO | Perf report for the last 5.14304 seconds: | 20260508 13:31:25.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:25.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:31:25.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:31:25.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:25.331 | INFO | Uploaded 36GiB to GPU in 2.44116s (14.7471GiB/s) | 20260508 13:31:25.332 | INFO | Downloaded 34GiB to GPU in 2.47696s (13.7265GiB/s) | 20260508 13:31:30.473 | INFO | Analyzing chunk frames = [8483840,8484352) | 20260508 13:31:30.473 | INFO | MemInfo: { | 20260508 13:31:30.473 | INFO | "gpu" : " 17 GB", | 20260508 13:31:30.473 | INFO | "huge" : "", | 20260508 13:31:30.473 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:30.473 | INFO | "malloc" : " 14 GB", | 20260508 13:31:30.473 | INFO | "pinned" : " 29 GB" | 20260508 13:31:30.473 | INFO | } | 20260508 13:31:30.473 | INFO | Perf report for the last 5.14211 seconds: | 20260508 13:31:30.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:30.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.473 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 13:31:30.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.473 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:31:30.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:30.474 | INFO | Uploaded 36GiB to GPU in 2.514s (14.3198GiB/s) | 20260508 13:31:30.474 | INFO | Downloaded 34GiB to GPU in 2.4689s (13.7713GiB/s) | 20260508 13:31:35.609 | INFO | Analyzing chunk frames = [8484352,8484864) | 20260508 13:31:35.609 | INFO | MemInfo: { | 20260508 13:31:35.609 | INFO | "gpu" : " 17 GB", | 20260508 13:31:35.609 | INFO | "huge" : "", | 20260508 13:31:35.609 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:35.609 | INFO | "malloc" : " 14 GB", | 20260508 13:31:35.610 | INFO | "pinned" : " 29 GB" | 20260508 13:31:35.610 | INFO | } | 20260508 13:31:35.610 | INFO | Perf report for the last 5.13659 seconds: | 20260508 13:31:35.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:35.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | ANALYSIS : 65.3%, 3.0, 101.3ms, 0 | 20260508 13:31:35.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:31:35.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:35.610 | INFO | Uploaded 36GiB to GPU in 2.38164s (15.1156GiB/s) | 20260508 13:31:35.610 | INFO | Downloaded 34GiB to GPU in 2.46444s (13.7962GiB/s) | 20260508 13:31:40.748 | INFO | Analyzing chunk frames = [8484864,8485376) | 20260508 13:31:40.748 | INFO | MemInfo: { | 20260508 13:31:40.748 | INFO | "gpu" : " 17 GB", | 20260508 13:31:40.748 | INFO | "huge" : "", | 20260508 13:31:40.748 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:40.748 | INFO | "malloc" : " 14 GB", | 20260508 13:31:40.748 | INFO | "pinned" : " 29 GB" | 20260508 13:31:40.748 | INFO | } | 20260508 13:31:40.749 | INFO | Perf report for the last 5.13876 seconds: | 20260508 13:31:40.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:40.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:31:40.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:31:40.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:40.749 | INFO | Uploaded 36GiB to GPU in 2.36097s (15.2479GiB/s) | 20260508 13:31:40.749 | INFO | Downloaded 34GiB to GPU in 2.51517s (13.518GiB/s) | 20260508 13:31:45.889 | INFO | Analyzing chunk frames = [8485376,8485888) | 20260508 13:31:45.889 | INFO | MemInfo: { | 20260508 13:31:45.889 | INFO | "gpu" : " 17 GB", | 20260508 13:31:45.889 | INFO | "huge" : "", | 20260508 13:31:45.889 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:45.890 | INFO | "malloc" : " 14 GB", | 20260508 13:31:45.890 | INFO | "pinned" : " 29 GB" | 20260508 13:31:45.890 | INFO | } | 20260508 13:31:45.890 | INFO | Perf report for the last 5.14101 seconds: | 20260508 13:31:45.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:45.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.890 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:31:45.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.890 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:31:45.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:45.891 | INFO | Uploaded 36GiB to GPU in 2.42328s (14.8559GiB/s) | 20260508 13:31:45.891 | INFO | Downloaded 34GiB to GPU in 2.44525s (13.9045GiB/s) | 20260508 13:31:51.024 | INFO | Analyzing chunk frames = [8485888,8486400) | 20260508 13:31:51.024 | INFO | MemInfo: { | 20260508 13:31:51.024 | INFO | "gpu" : " 17 GB", | 20260508 13:31:51.024 | INFO | "huge" : "", | 20260508 13:31:51.024 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:51.024 | INFO | "malloc" : " 14 GB", | 20260508 13:31:51.024 | INFO | "pinned" : " 29 GB" | 20260508 13:31:51.024 | INFO | } | 20260508 13:31:51.024 | INFO | Perf report for the last 5.13491 seconds: | 20260508 13:31:51.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:51.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | ANALYSIS : 64.4%, 3.0, 100.1ms, 0 | 20260508 13:31:51.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:31:51.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:51.025 | INFO | Uploaded 36GiB to GPU in 2.25617s (15.9563GiB/s) | 20260508 13:31:51.025 | INFO | Downloaded 34GiB to GPU in 2.41441s (14.0821GiB/s) | 20260508 13:31:56.170 | INFO | Analyzing chunk frames = [8486400,8486912) | 20260508 13:31:56.170 | INFO | MemInfo: { | 20260508 13:31:56.170 | INFO | "gpu" : " 17 GB", | 20260508 13:31:56.170 | INFO | "huge" : "", | 20260508 13:31:56.170 | INFO | "huge_shared" : " 29 GB", | 20260508 13:31:56.170 | INFO | "malloc" : " 14 GB", | 20260508 13:31:56.170 | INFO | "pinned" : " 29 GB" | 20260508 13:31:56.170 | INFO | } | 20260508 13:31:56.170 | INFO | Perf report for the last 5.14591 seconds: | 20260508 13:31:56.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:31:56.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | ANALYSIS : 65.6%, 3.0, 102.7ms, 0 | 20260508 13:31:56.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:31:56.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:31:56.171 | INFO | Uploaded 36GiB to GPU in 2.46003s (14.634GiB/s) | 20260508 13:31:56.171 | INFO | Downloaded 34GiB to GPU in 2.47013s (13.7644GiB/s) | 20260508 13:31:56.171 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:31:56.171 | INFO | Total mem usage: 16.7 GiB | 20260508 13:31:56.171 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:31:56.171 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:31:56.171 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:31:56.171 | INFO | High Water Breakdown: | 20260508 13:31:56.171 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:31:56.171 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:31:56.171 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:31:56.171 | INFO | ------------------------------------------ | 20260508 13:31:56.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:31:56.171 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:31:56.171 | INFO | Total mem usage: 26.5 GiB | 20260508 13:31:56.171 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:31:56.171 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:31:56.171 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:31:56.171 | INFO | High Water Breakdown: | 20260508 13:31:56.171 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:31:56.171 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:31:56.171 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:31:56.171 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:31:56.171 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:31:56.171 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:31:56.171 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:31:56.171 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:31:56.171 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:31:56.171 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:31:56.171 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:31:56.171 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:31:56.171 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:31:56.171 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:31:56.171 | INFO | ------------------------------------------ | 20260508 13:31:56.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:31:56.171 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:31:56.171 | INFO | Total mem usage: 27.8 GiB | 20260508 13:31:56.171 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:31:56.171 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:31:56.171 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:31:56.171 | INFO | High Water Breakdown: | 20260508 13:31:56.171 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:31:56.171 | INFO | ------------------------------------------ | 20260508 13:31:56.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:31:56.171 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:31:56.171 | INFO | Total mem usage: 15.7 GiB | 20260508 13:31:56.171 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:31:56.171 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:31:56.171 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:31:56.171 | INFO | High Water Breakdown: | 20260508 13:31:56.171 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:31:56.171 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:31:56.171 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:31:56.171 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:31:56.171 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:31:56.171 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:31:56.171 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:31:56.171 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:31:56.171 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:31:56.171 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:31:56.171 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:31:56.171 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:31:56.171 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:31:56.171 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:31:56.171 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:31:56.171 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:31:56.171 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:31:56.171 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:31:56.171 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:31:56.171 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:31:56.171 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:31:56.171 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:31:56.171 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:31:56.171 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:31:56.171 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:31:56.171 | INFO | ------------------------------------------ | 20260508 13:31:56.171 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:31:56.171 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:31:56.171 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:31:56.171 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:31:56.171 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.39e+01 / 6.49e-01 | 20260508 13:31:56.171 | INFO | PerfReport: 0.30%, AggregateStats -- 9.78e-02 / 1.05e-01, 1.09e-02 / 3.92e-03 | 20260508 13:31:56.171 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.49e+00, 5.98e-01 / 1.39e-01 | 20260508 13:31:56.171 | INFO | PerfReport: 3.41%, Histogram -- 1.11e+00 / 1.44e+00, 2.11e-01 / 1.34e-01 | 20260508 13:31:56.171 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.57e+01, 3.70e+00 / 1.14e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.82e+00 / 1.15e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.50e+00, 4.04e-01 / 2.59e-01 | 20260508 13:31:56.171 | INFO | PerfReport: 2.92%, Metrics -- 9.56e-01 / 1.06e+00, 2.39e-01 / 9.32e-02 | 20260508 13:31:56.171 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.91e-01 / 3.18e-01, 4.73e-02 / 2.12e-02 | 20260508 13:31:56.171 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:31:56.171 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:31:56.171 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:31:56.171 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 24.28%, Upload -- 2.54e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 24.87%, Download -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 31.25%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 2.06%, LockUpload -- 2.15e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 2.62%, LockDownload -- 2.74e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 14.90%, LockCompute -- 1.56e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260508 13:31:56.171 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260508 13:31:56.267 | INFO | Starting a round of Baz IO | 20260508 13:31:59.663 | INFO | Populating RTMetrics report | 20260508 13:31:59.664 | INFO | RTMetrics report populated | 20260508 13:31:59.664 | INFO | Opening tmp RTMetrics json file | 20260508 13:31:59.675 | INFO | Serializing RTMetrics to json | 20260508 13:31:59.676 | INFO | Serialization done, swapping tmp file now... | 20260508 13:31:59.677 | INFO | RTMetrics done | 20260508 13:31:59.737 | INFO | Finished a round of Baz IO: Wrote 1.385 GiB in 3.469 seconds | 20260508 13:31:59.737 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:31:59.737 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139793% / 0% / 7.54992e-05% | 20260508 13:31:59.737 | INFO | Baz IO overall: | 20260508 13:31:59.737 | INFO | BytesWritten : 3614966440549 | 20260508 13:31:59.737 | INFO | NumEvents : 1733125679819 | 20260508 13:32:01.306 | INFO | Analyzing chunk frames = [8486912,8487424) | 20260508 13:32:01.306 | INFO | MemInfo: { | 20260508 13:32:01.306 | INFO | "gpu" : " 17 GB", | 20260508 13:32:01.306 | INFO | "huge" : "", | 20260508 13:32:01.306 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:01.306 | INFO | "malloc" : " 14 GB", | 20260508 13:32:01.306 | INFO | "pinned" : " 29 GB" | 20260508 13:32:01.306 | INFO | } | 20260508 13:32:01.306 | INFO | Perf report for the last 5.13533 seconds: | 20260508 13:32:01.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:01.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:01.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:01.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:01.306 | INFO | ANALYSIS : 67.4%, 3.0, 105.2ms, 0 | 20260508 13:32:01.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:01.306 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 13:32:01.306 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 13:32:01.306 | INFO | BAZWRITER : 66.8%, 2.5, 85.1ms, 0 | 20260508 13:32:01.306 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:32:01.306 | INFO | Uploaded 36GiB to GPU in 2.65214s (13.574GiB/s) | 20260508 13:32:01.306 | INFO | Downloaded 38GiB to GPU in 2.71218s (14.0109GiB/s) | 20260508 13:32:06.439 | INFO | Analyzing chunk frames = [8487424,8487936) | 20260508 13:32:06.439 | INFO | MemInfo: { | 20260508 13:32:06.439 | INFO | "gpu" : " 17 GB", | 20260508 13:32:06.439 | INFO | "huge" : "", | 20260508 13:32:06.439 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:06.440 | INFO | "malloc" : " 14 GB", | 20260508 13:32:06.440 | INFO | "pinned" : " 29 GB" | 20260508 13:32:06.440 | INFO | } | 20260508 13:32:06.440 | INFO | Perf report for the last 5.13409 seconds: | 20260508 13:32:06.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:06.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 13:32:06.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260508 13:32:06.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:06.440 | INFO | Uploaded 36GiB to GPU in 2.49206s (14.4459GiB/s) | 20260508 13:32:06.440 | INFO | Downloaded 34GiB to GPU in 2.45705s (13.8377GiB/s) | 20260508 13:32:11.581 | INFO | Analyzing chunk frames = [8487936,8488448) | 20260508 13:32:11.581 | INFO | MemInfo: { | 20260508 13:32:11.581 | INFO | "gpu" : " 17 GB", | 20260508 13:32:11.581 | INFO | "huge" : "", | 20260508 13:32:11.581 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:11.582 | INFO | "malloc" : " 14 GB", | 20260508 13:32:11.582 | INFO | "pinned" : " 29 GB" | 20260508 13:32:11.582 | INFO | } | 20260508 13:32:11.582 | INFO | Perf report for the last 5.14201 seconds: | 20260508 13:32:11.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:11.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 13:32:11.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260508 13:32:11.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:11.582 | INFO | Uploaded 36GiB to GPU in 2.51593s (14.3088GiB/s) | 20260508 13:32:11.582 | INFO | Downloaded 34GiB to GPU in 2.45297s (13.8608GiB/s) | 20260508 13:32:16.728 | INFO | Analyzing chunk frames = [8488448,8488960) | 20260508 13:32:16.728 | INFO | MemInfo: { | 20260508 13:32:16.728 | INFO | "gpu" : " 17 GB", | 20260508 13:32:16.728 | INFO | "huge" : "", | 20260508 13:32:16.728 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:16.728 | INFO | "malloc" : " 14 GB", | 20260508 13:32:16.728 | INFO | "pinned" : " 29 GB" | 20260508 13:32:16.729 | INFO | } | 20260508 13:32:16.729 | INFO | Perf report for the last 5.14666 seconds: | 20260508 13:32:16.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:16.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 13:32:16.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260508 13:32:16.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:16.729 | INFO | Uploaded 36GiB to GPU in 2.5542s (14.0944GiB/s) | 20260508 13:32:16.729 | INFO | Downloaded 34GiB to GPU in 2.46398s (13.7988GiB/s) | 20260508 13:32:21.861 | INFO | Analyzing chunk frames = [8488960,8489472) | 20260508 13:32:21.861 | INFO | MemInfo: { | 20260508 13:32:21.861 | INFO | "gpu" : " 17 GB", | 20260508 13:32:21.861 | INFO | "huge" : "", | 20260508 13:32:21.861 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:21.861 | INFO | "malloc" : " 14 GB", | 20260508 13:32:21.861 | INFO | "pinned" : " 29 GB" | 20260508 13:32:21.861 | INFO | } | 20260508 13:32:21.861 | INFO | Perf report for the last 5.13242 seconds: | 20260508 13:32:21.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:21.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.861 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:32:21.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.861 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260508 13:32:21.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:21.862 | INFO | Uploaded 36GiB to GPU in 2.5015s (14.3914GiB/s) | 20260508 13:32:21.862 | INFO | Downloaded 34GiB to GPU in 2.52071s (13.4883GiB/s) | 20260508 13:32:26.995 | INFO | Analyzing chunk frames = [8489472,8489984) | 20260508 13:32:26.995 | INFO | MemInfo: { | 20260508 13:32:26.995 | INFO | "gpu" : " 17 GB", | 20260508 13:32:26.995 | INFO | "huge" : "", | 20260508 13:32:26.995 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:26.995 | INFO | "malloc" : " 14 GB", | 20260508 13:32:26.996 | INFO | "pinned" : " 29 GB" | 20260508 13:32:26.996 | INFO | } | 20260508 13:32:26.996 | INFO | Perf report for the last 5.13455 seconds: | 20260508 13:32:26.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:26.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | ANALYSIS : 65.6%, 3.0, 101.9ms, 0 | 20260508 13:32:26.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:32:26.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:26.996 | INFO | Uploaded 36GiB to GPU in 2.44491s (14.7245GiB/s) | 20260508 13:32:26.996 | INFO | Downloaded 34GiB to GPU in 2.44062s (13.9309GiB/s) | 20260508 13:32:32.145 | INFO | Analyzing chunk frames = [8489984,8490496) | 20260508 13:32:32.145 | INFO | MemInfo: { | 20260508 13:32:32.145 | INFO | "gpu" : " 17 GB", | 20260508 13:32:32.145 | INFO | "huge" : "", | 20260508 13:32:32.145 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:32.145 | INFO | "malloc" : " 14 GB", | 20260508 13:32:32.145 | INFO | "pinned" : " 29 GB" | 20260508 13:32:32.145 | INFO | } | 20260508 13:32:32.145 | INFO | Perf report for the last 5.14921 seconds: | 20260508 13:32:32.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:32.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | ANALYSIS : 65.3%, 3.0, 102.1ms, 0 | 20260508 13:32:32.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:32:32.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:32.145 | INFO | Uploaded 36GiB to GPU in 2.4705s (14.572GiB/s) | 20260508 13:32:32.145 | INFO | Downloaded 34GiB to GPU in 2.44547s (13.9033GiB/s) | 20260508 13:32:37.282 | INFO | Analyzing chunk frames = [8490496,8491008) | 20260508 13:32:37.282 | INFO | MemInfo: { | 20260508 13:32:37.282 | INFO | "gpu" : " 17 GB", | 20260508 13:32:37.282 | INFO | "huge" : "", | 20260508 13:32:37.282 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:37.282 | INFO | "malloc" : " 14 GB", | 20260508 13:32:37.282 | INFO | "pinned" : " 29 GB" | 20260508 13:32:37.282 | INFO | } | 20260508 13:32:37.282 | INFO | Perf report for the last 5.13709 seconds: | 20260508 13:32:37.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:37.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | ANALYSIS : 64.6%, 3.0, 101.0ms, 0 | 20260508 13:32:37.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | PRE_HQ : 23.2%, 1.0, 12.1ms, 0 | 20260508 13:32:37.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:37.282 | INFO | Uploaded 36GiB to GPU in 2.37936s (15.1301GiB/s) | 20260508 13:32:37.282 | INFO | Downloaded 34GiB to GPU in 2.41315s (14.0895GiB/s) | 20260508 13:32:37.381 | INFO | Starting a round of Baz IO | 20260508 13:32:40.785 | INFO | Populating RTMetrics report | 20260508 13:32:40.785 | INFO | RTMetrics report populated | 20260508 13:32:40.785 | INFO | Opening tmp RTMetrics json file | 20260508 13:32:40.789 | INFO | Serializing RTMetrics to json | 20260508 13:32:40.791 | INFO | Serialization done, swapping tmp file now... | 20260508 13:32:40.791 | INFO | RTMetrics done | 20260508 13:32:40.851 | INFO | Finished a round of Baz IO: Wrote 1.384 GiB in 3.469 seconds | 20260508 13:32:40.851 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:32:40.851 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139864% / 0% / 7.54992e-05% | 20260508 13:32:40.851 | INFO | Baz IO overall: | 20260508 13:32:40.851 | INFO | BytesWritten : 3616452767093 | 20260508 13:32:40.851 | INFO | NumEvents : 1733820998012 | 20260508 13:32:42.426 | INFO | Analyzing chunk frames = [8491008,8491520) | 20260508 13:32:42.426 | INFO | MemInfo: { | 20260508 13:32:42.426 | INFO | "gpu" : " 17 GB", | 20260508 13:32:42.426 | INFO | "huge" : "", | 20260508 13:32:42.427 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:42.427 | INFO | "malloc" : " 14 GB", | 20260508 13:32:42.427 | INFO | "pinned" : " 29 GB" | 20260508 13:32:42.427 | INFO | } | 20260508 13:32:42.427 | INFO | Perf report for the last 5.14468 seconds: | 20260508 13:32:42.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:42.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:42.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:42.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:42.427 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 13:32:42.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:42.427 | INFO | RT_METRICS : 38.0%, 1.0, 20.0ms, 0 | 20260508 13:32:42.427 | INFO | PRE_HQ : 65.0%, 1.6, 54.1ms, 0 | 20260508 13:32:42.427 | INFO | BAZWRITER : 66.8%, 2.5, 85.2ms, 0 | 20260508 13:32:42.427 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:32:42.427 | INFO | Uploaded 36GiB to GPU in 2.61942s (13.7435GiB/s) | 20260508 13:32:42.427 | INFO | Downloaded 38GiB to GPU in 2.71895s (13.976GiB/s) | 20260508 13:32:47.559 | INFO | Analyzing chunk frames = [8491520,8492032) | 20260508 13:32:47.559 | INFO | MemInfo: { | 20260508 13:32:47.559 | INFO | "gpu" : " 17 GB", | 20260508 13:32:47.559 | INFO | "huge" : "", | 20260508 13:32:47.559 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:47.559 | INFO | "malloc" : " 14 GB", | 20260508 13:32:47.559 | INFO | "pinned" : " 29 GB" | 20260508 13:32:47.559 | INFO | } | 20260508 13:32:47.559 | INFO | Perf report for the last 5.13276 seconds: | 20260508 13:32:47.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:47.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 13:32:47.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:32:47.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:47.560 | INFO | Uploaded 36GiB to GPU in 2.51783s (14.298GiB/s) | 20260508 13:32:47.560 | INFO | Downloaded 34GiB to GPU in 2.46206s (13.8096GiB/s) | 20260508 13:32:52.701 | INFO | Analyzing chunk frames = [8492032,8492544) | 20260508 13:32:52.701 | INFO | MemInfo: { | 20260508 13:32:52.701 | INFO | "gpu" : " 17 GB", | 20260508 13:32:52.701 | INFO | "huge" : "", | 20260508 13:32:52.701 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:52.701 | INFO | "malloc" : " 14 GB", | 20260508 13:32:52.701 | INFO | "pinned" : " 29 GB" | 20260508 13:32:52.701 | INFO | } | 20260508 13:32:52.701 | INFO | Perf report for the last 5.14149 seconds: | 20260508 13:32:52.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:52.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:32:52.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:32:52.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:52.701 | INFO | Uploaded 36GiB to GPU in 2.52061s (14.2823GiB/s) | 20260508 13:32:52.701 | INFO | Downloaded 34GiB to GPU in 2.45669s (13.8397GiB/s) | 20260508 13:32:57.843 | INFO | Analyzing chunk frames = [8492544,8493056) | 20260508 13:32:57.843 | INFO | MemInfo: { | 20260508 13:32:57.843 | INFO | "gpu" : " 17 GB", | 20260508 13:32:57.843 | INFO | "huge" : "", | 20260508 13:32:57.843 | INFO | "huge_shared" : " 29 GB", | 20260508 13:32:57.843 | INFO | "malloc" : " 14 GB", | 20260508 13:32:57.843 | INFO | "pinned" : " 29 GB" | 20260508 13:32:57.843 | INFO | } | 20260508 13:32:57.843 | INFO | Perf report for the last 5.14197 seconds: | 20260508 13:32:57.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:32:57.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 13:32:57.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:32:57.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:32:57.843 | INFO | Uploaded 36GiB to GPU in 2.44719s (14.7107GiB/s) | 20260508 13:32:57.843 | INFO | Downloaded 34GiB to GPU in 2.47053s (13.7622GiB/s) | 20260508 13:33:02.979 | INFO | Analyzing chunk frames = [8493056,8493568) | 20260508 13:33:02.979 | INFO | MemInfo: { | 20260508 13:33:02.979 | INFO | "gpu" : " 17 GB", | 20260508 13:33:02.979 | INFO | "huge" : "", | 20260508 13:33:02.979 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:02.979 | INFO | "malloc" : " 14 GB", | 20260508 13:33:02.979 | INFO | "pinned" : " 29 GB" | 20260508 13:33:02.980 | INFO | } | 20260508 13:33:02.980 | INFO | Perf report for the last 5.13668 seconds: | 20260508 13:33:02.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:02.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:33:02.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:33:02.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:02.980 | INFO | Uploaded 36GiB to GPU in 2.34063s (15.3805GiB/s) | 20260508 13:33:02.980 | INFO | Downloaded 34GiB to GPU in 2.49583s (13.6227GiB/s) | 20260508 13:33:08.120 | INFO | Analyzing chunk frames = [8493568,8494080) | 20260508 13:33:08.120 | INFO | MemInfo: { | 20260508 13:33:08.120 | INFO | "gpu" : " 17 GB", | 20260508 13:33:08.120 | INFO | "huge" : "", | 20260508 13:33:08.120 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:08.120 | INFO | "malloc" : " 14 GB", | 20260508 13:33:08.120 | INFO | "pinned" : " 29 GB" | 20260508 13:33:08.121 | INFO | } | 20260508 13:33:08.121 | INFO | Perf report for the last 5.14105 seconds: | 20260508 13:33:08.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:08.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:33:08.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:33:08.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:08.121 | INFO | Uploaded 36GiB to GPU in 2.43685s (14.7732GiB/s) | 20260508 13:33:08.121 | INFO | Downloaded 34GiB to GPU in 2.44393s (13.912GiB/s) | 20260508 13:33:13.256 | INFO | Analyzing chunk frames = [8494080,8494592) | 20260508 13:33:13.256 | INFO | MemInfo: { | 20260508 13:33:13.256 | INFO | "gpu" : " 17 GB", | 20260508 13:33:13.256 | INFO | "huge" : "", | 20260508 13:33:13.256 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:13.256 | INFO | "malloc" : " 14 GB", | 20260508 13:33:13.256 | INFO | "pinned" : " 29 GB" | 20260508 13:33:13.256 | INFO | } | 20260508 13:33:13.256 | INFO | Perf report for the last 5.13546 seconds: | 20260508 13:33:13.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:13.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:33:13.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:33:13.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:13.256 | INFO | Uploaded 36GiB to GPU in 2.32501s (15.4838GiB/s) | 20260508 13:33:13.257 | INFO | Downloaded 34GiB to GPU in 2.41207s (14.0958GiB/s) | 20260508 13:33:18.401 | INFO | Analyzing chunk frames = [8494592,8495104) | 20260508 13:33:18.401 | INFO | MemInfo: { | 20260508 13:33:18.401 | INFO | "gpu" : " 17 GB", | 20260508 13:33:18.401 | INFO | "huge" : "", | 20260508 13:33:18.401 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:18.401 | INFO | "malloc" : " 14 GB", | 20260508 13:33:18.401 | INFO | "pinned" : " 29 GB" | 20260508 13:33:18.401 | INFO | } | 20260508 13:33:18.401 | INFO | Perf report for the last 5.14484 seconds: | 20260508 13:33:18.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:18.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | ANALYSIS : 64.8%, 3.0, 101.4ms, 0 | 20260508 13:33:18.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:33:18.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:18.401 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:33:18.401 | INFO | Total mem usage: 16.7 GiB | 20260508 13:33:18.401 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:33:18.401 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:33:18.401 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:33:18.401 | INFO | High Water Breakdown: | 20260508 13:33:18.401 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:33:18.401 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:33:18.401 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:33:18.401 | INFO | ------------------------------------------ | 20260508 13:33:18.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:33:18.401 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:33:18.401 | INFO | Total mem usage: 26.5 GiB | 20260508 13:33:18.401 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:33:18.401 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:33:18.401 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:33:18.401 | INFO | High Water Breakdown: | 20260508 13:33:18.401 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:33:18.401 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:33:18.401 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:33:18.401 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:33:18.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:33:18.402 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:33:18.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:33:18.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:33:18.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:33:18.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:33:18.402 | INFO | Uploaded 36GiB to GPU in 2.38756s (15.0782GiB/s) | 20260508 13:33:18.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:33:18.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:33:18.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:33:18.402 | INFO | Downloaded 34GiB to GPU in 2.43963s (13.9365GiB/s) | 20260508 13:33:18.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:33:18.402 | INFO | ------------------------------------------ | 20260508 13:33:18.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:33:18.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:33:18.402 | INFO | Total mem usage: 27.8 GiB | 20260508 13:33:18.402 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:33:18.402 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:33:18.402 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:33:18.402 | INFO | High Water Breakdown: | 20260508 13:33:18.402 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:33:18.402 | INFO | ------------------------------------------ | 20260508 13:33:18.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:33:18.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:33:18.402 | INFO | Total mem usage: 15.7 GiB | 20260508 13:33:18.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:33:18.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:33:18.402 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:33:18.402 | INFO | High Water Breakdown: | 20260508 13:33:18.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:33:18.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:33:18.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:33:18.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:33:18.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:33:18.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:33:18.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:33:18.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:33:18.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:33:18.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:33:18.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:33:18.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:33:18.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:33:18.402 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:33:18.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:33:18.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:33:18.402 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:33:18.402 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:33:18.402 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:33:18.402 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:33:18.402 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:33:18.402 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:33:18.402 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:33:18.402 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:33:18.402 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:33:18.402 | INFO | ------------------------------------------ | 20260508 13:33:18.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:33:18.402 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:33:18.402 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:33:18.402 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:33:18.402 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.39e+01 / 7.16e-01 | 20260508 13:33:18.402 | INFO | PerfReport: 0.30%, AggregateStats -- 9.76e-02 / 1.07e-01, 1.17e-02 / 3.40e-03 | 20260508 13:33:18.402 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.34e+00, 5.96e-01 / 1.77e-01 | 20260508 13:33:18.402 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.42e+00, 2.11e-01 / 1.17e-01 | 20260508 13:33:18.402 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.53e+01, 3.58e+00 / 1.08e+00 | 20260508 13:33:18.402 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.87e+01 / 2.06e+01, 1.83e+00 / 1.13e+00 | 20260508 13:33:18.402 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.27e+00, 4.03e-01 / 2.41e-01 | 20260508 13:33:18.402 | INFO | PerfReport: 2.93%, Metrics -- 9.56e-01 / 1.08e+00, 2.37e-01 / 9.70e-02 | 20260508 13:33:18.403 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.13e-01, 4.70e-02 / 2.54e-02 | 20260508 13:33:18.403 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:33:18.403 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:33:18.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:33:18.403 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 24.61%, Upload -- 2.58e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 24.74%, Download -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 31.16%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 2.26%, LockUpload -- 2.37e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 2.43%, LockDownload -- 2.55e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 14.78%, LockCompute -- 1.55e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 13:33:18.403 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.05e-02 / 0.00e+00 | 20260508 13:33:18.498 | INFO | Starting a round of Baz IO | 20260508 13:33:21.921 | INFO | Populating RTMetrics report | 20260508 13:33:21.922 | INFO | RTMetrics report populated | 20260508 13:33:21.922 | INFO | Opening tmp RTMetrics json file | 20260508 13:33:21.936 | INFO | Serializing RTMetrics to json | 20260508 13:33:21.937 | INFO | Serialization done, swapping tmp file now... | 20260508 13:33:21.937 | INFO | RTMetrics done | 20260508 13:33:21.996 | INFO | Finished a round of Baz IO: Wrote 1.383 GiB in 3.497 seconds | 20260508 13:33:21.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:33:21.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139825% / 0% / 4.76837e-05% | 20260508 13:33:21.996 | INFO | Baz IO overall: | 20260508 13:33:21.996 | INFO | BytesWritten : 3617937828848 | 20260508 13:33:21.996 | INFO | NumEvents : 1734515333928 | 20260508 13:33:23.539 | INFO | Analyzing chunk frames = [8495104,8495616) | 20260508 13:33:23.539 | INFO | MemInfo: { | 20260508 13:33:23.539 | INFO | "gpu" : " 17 GB", | 20260508 13:33:23.539 | INFO | "huge" : "", | 20260508 13:33:23.539 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:23.539 | INFO | "malloc" : " 14 GB", | 20260508 13:33:23.539 | INFO | "pinned" : " 29 GB" | 20260508 13:33:23.539 | INFO | } | 20260508 13:33:23.539 | INFO | Perf report for the last 5.13841 seconds: | 20260508 13:33:23.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:23.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:23.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:23.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:23.540 | INFO | ANALYSIS : 68.2%, 3.0, 106.2ms, 0 | 20260508 13:33:23.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:23.540 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:33:23.540 | INFO | PRE_HQ : 66.0%, 1.6, 53.8ms, 0 | 20260508 13:33:23.540 | INFO | BAZWRITER : 67.3%, 2.4, 84.3ms, 0 | 20260508 13:33:23.540 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:33:23.540 | INFO | Uploaded 36GiB to GPU in 2.5881s (13.9098GiB/s) | 20260508 13:33:23.540 | INFO | Downloaded 38GiB to GPU in 2.73561s (13.8908GiB/s) | 20260508 13:33:28.674 | INFO | Analyzing chunk frames = [8495616,8496128) | 20260508 13:33:28.674 | INFO | MemInfo: { | 20260508 13:33:28.674 | INFO | "gpu" : " 17 GB", | 20260508 13:33:28.674 | INFO | "huge" : "", | 20260508 13:33:28.674 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:28.674 | INFO | "malloc" : " 14 GB", | 20260508 13:33:28.675 | INFO | "pinned" : " 29 GB" | 20260508 13:33:28.675 | INFO | } | 20260508 13:33:28.675 | INFO | Perf report for the last 5.13528 seconds: | 20260508 13:33:28.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:28.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:33:28.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260508 13:33:28.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:28.675 | INFO | Uploaded 36GiB to GPU in 2.50299s (14.3828GiB/s) | 20260508 13:33:28.675 | INFO | Downloaded 34GiB to GPU in 2.48067s (13.706GiB/s) | 20260508 13:33:33.811 | INFO | Analyzing chunk frames = [8496128,8496640) | 20260508 13:33:33.811 | INFO | MemInfo: { | 20260508 13:33:33.811 | INFO | "gpu" : " 17 GB", | 20260508 13:33:33.811 | INFO | "huge" : "", | 20260508 13:33:33.811 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:33.811 | INFO | "malloc" : " 14 GB", | 20260508 13:33:33.811 | INFO | "pinned" : " 29 GB" | 20260508 13:33:33.811 | INFO | } | 20260508 13:33:33.811 | INFO | Perf report for the last 5.13638 seconds: | 20260508 13:33:33.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:33.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.811 | INFO | ANALYSIS : 66.0%, 3.0, 102.6ms, 0 | 20260508 13:33:33.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.811 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260508 13:33:33.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:33.812 | INFO | Uploaded 36GiB to GPU in 2.48549s (14.4841GiB/s) | 20260508 13:33:33.812 | INFO | Downloaded 34GiB to GPU in 2.46678s (13.7831GiB/s) | 20260508 13:33:38.951 | INFO | Analyzing chunk frames = [8496640,8497152) | 20260508 13:33:38.951 | INFO | MemInfo: { | 20260508 13:33:38.951 | INFO | "gpu" : " 17 GB", | 20260508 13:33:38.951 | INFO | "huge" : "", | 20260508 13:33:38.951 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:38.951 | INFO | "malloc" : " 14 GB", | 20260508 13:33:38.951 | INFO | "pinned" : " 29 GB" | 20260508 13:33:38.951 | INFO | } | 20260508 13:33:38.951 | INFO | Perf report for the last 5.13997 seconds: | 20260508 13:33:38.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:38.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.951 | INFO | ANALYSIS : 65.5%, 3.0, 101.7ms, 0 | 20260508 13:33:38.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.951 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260508 13:33:38.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:38.952 | INFO | Uploaded 36GiB to GPU in 2.4123s (14.9235GiB/s) | 20260508 13:33:38.952 | INFO | Downloaded 34GiB to GPU in 2.47885s (13.716GiB/s) | 20260508 13:33:44.093 | INFO | Analyzing chunk frames = [8497152,8497664) | 20260508 13:33:44.093 | INFO | MemInfo: { | 20260508 13:33:44.093 | INFO | "gpu" : " 17 GB", | 20260508 13:33:44.093 | INFO | "huge" : "", | 20260508 13:33:44.093 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:44.093 | INFO | "malloc" : " 14 GB", | 20260508 13:33:44.093 | INFO | "pinned" : " 29 GB" | 20260508 13:33:44.093 | INFO | } | 20260508 13:33:44.093 | INFO | Perf report for the last 5.14228 seconds: | 20260508 13:33:44.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:44.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.093 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:33:44.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.093 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:33:44.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:44.094 | INFO | Uploaded 36GiB to GPU in 2.3915s (15.0533GiB/s) | 20260508 13:33:44.094 | INFO | Downloaded 34GiB to GPU in 2.51423s (13.523GiB/s) | 20260508 13:33:49.233 | INFO | Analyzing chunk frames = [8497664,8498176) | 20260508 13:33:49.233 | INFO | MemInfo: { | 20260508 13:33:49.233 | INFO | "gpu" : " 17 GB", | 20260508 13:33:49.233 | INFO | "huge" : "", | 20260508 13:33:49.233 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:49.233 | INFO | "malloc" : " 14 GB", | 20260508 13:33:49.233 | INFO | "pinned" : " 29 GB" | 20260508 13:33:49.233 | INFO | } | 20260508 13:33:49.233 | INFO | Perf report for the last 5.13931 seconds: | 20260508 13:33:49.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:49.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:33:49.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:33:49.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:49.233 | INFO | Uploaded 36GiB to GPU in 2.40685s (14.9573GiB/s) | 20260508 13:33:49.233 | INFO | Downloaded 34GiB to GPU in 2.44374s (13.9131GiB/s) | 20260508 13:33:54.374 | INFO | Analyzing chunk frames = [8498176,8498688) | 20260508 13:33:54.374 | INFO | MemInfo: { | 20260508 13:33:54.374 | INFO | "gpu" : " 17 GB", | 20260508 13:33:54.374 | INFO | "huge" : "", | 20260508 13:33:54.374 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:54.374 | INFO | "malloc" : " 14 GB", | 20260508 13:33:54.374 | INFO | "pinned" : " 29 GB" | 20260508 13:33:54.374 | INFO | } | 20260508 13:33:54.374 | INFO | Perf report for the last 5.14162 seconds: | 20260508 13:33:54.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:54.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.374 | INFO | ANALYSIS : 64.6%, 3.0, 100.5ms, 0 | 20260508 13:33:54.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.375 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:33:54.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:54.375 | INFO | Uploaded 36GiB to GPU in 2.32982s (15.4518GiB/s) | 20260508 13:33:54.375 | INFO | Downloaded 34GiB to GPU in 2.42941s (13.9952GiB/s) | 20260508 13:33:59.509 | INFO | Analyzing chunk frames = [8498688,8499200) | 20260508 13:33:59.509 | INFO | MemInfo: { | 20260508 13:33:59.509 | INFO | "gpu" : " 17 GB", | 20260508 13:33:59.509 | INFO | "huge" : "", | 20260508 13:33:59.509 | INFO | "huge_shared" : " 29 GB", | 20260508 13:33:59.509 | INFO | "malloc" : " 14 GB", | 20260508 13:33:59.509 | INFO | "pinned" : " 29 GB" | 20260508 13:33:59.509 | INFO | } | 20260508 13:33:59.509 | INFO | Perf report for the last 5.13442 seconds: | 20260508 13:33:59.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:33:59.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260508 13:33:59.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:33:59.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:33:59.509 | INFO | Uploaded 36GiB to GPU in 2.36576s (15.2171GiB/s) | 20260508 13:33:59.509 | INFO | Downloaded 34GiB to GPU in 2.43754s (13.9485GiB/s) | 20260508 13:33:59.606 | INFO | Starting a round of Baz IO | 20260508 13:34:03.013 | INFO | Populating RTMetrics report | 20260508 13:34:03.013 | INFO | RTMetrics report populated | 20260508 13:34:03.013 | INFO | Opening tmp RTMetrics json file | 20260508 13:34:03.022 | INFO | Serializing RTMetrics to json | 20260508 13:34:03.023 | INFO | Serialization done, swapping tmp file now... | 20260508 13:34:03.023 | INFO | RTMetrics done | 20260508 13:34:03.079 | INFO | Finished a round of Baz IO: Wrote 1.383 GiB in 3.473 seconds | 20260508 13:34:03.079 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:34:03.079 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.13928% / 3.97364e-06% / 5.96046e-05% | 20260508 13:34:03.079 | INFO | Baz IO overall: | 20260508 13:34:03.079 | INFO | BytesWritten : 3619423297712 | 20260508 13:34:03.079 | INFO | NumEvents : 1735209833692 | 20260508 13:34:04.652 | INFO | Analyzing chunk frames = [8499200,8499712) | 20260508 13:34:04.652 | INFO | MemInfo: { | 20260508 13:34:04.652 | INFO | "gpu" : " 17 GB", | 20260508 13:34:04.652 | INFO | "huge" : "", | 20260508 13:34:04.652 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:04.652 | INFO | "malloc" : " 14 GB", | 20260508 13:34:04.652 | INFO | "pinned" : " 29 GB" | 20260508 13:34:04.652 | INFO | } | 20260508 13:34:04.652 | INFO | Perf report for the last 5.14315 seconds: | 20260508 13:34:04.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:04.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:04.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:04.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:04.652 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:34:04.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:04.652 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:34:04.652 | INFO | PRE_HQ : 65.3%, 1.6, 53.6ms, 0 | 20260508 13:34:04.652 | INFO | BAZWRITER : 66.8%, 2.4, 84.1ms, 0 | 20260508 13:34:04.652 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:34:04.652 | INFO | Uploaded 36GiB to GPU in 2.60479s (13.8207GiB/s) | 20260508 13:34:04.652 | INFO | Downloaded 38GiB to GPU in 2.73462s (13.8959GiB/s) | 20260508 13:34:09.792 | INFO | Analyzing chunk frames = [8499712,8500224) | 20260508 13:34:09.792 | INFO | MemInfo: { | 20260508 13:34:09.792 | INFO | "gpu" : " 17 GB", | 20260508 13:34:09.792 | INFO | "huge" : "", | 20260508 13:34:09.792 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:09.792 | INFO | "malloc" : " 14 GB", | 20260508 13:34:09.792 | INFO | "pinned" : " 29 GB" | 20260508 13:34:09.792 | INFO | } | 20260508 13:34:09.792 | INFO | Perf report for the last 5.14036 seconds: | 20260508 13:34:09.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:09.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.792 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:34:09.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.793 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:34:09.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:09.793 | INFO | Uploaded 36GiB to GPU in 2.45613s (14.6572GiB/s) | 20260508 13:34:09.793 | INFO | Downloaded 34GiB to GPU in 2.48995s (13.6549GiB/s) | 20260508 13:34:14.929 | INFO | Analyzing chunk frames = [8500224,8500736) | 20260508 13:34:14.929 | INFO | MemInfo: { | 20260508 13:34:14.929 | INFO | "gpu" : " 17 GB", | 20260508 13:34:14.929 | INFO | "huge" : "", | 20260508 13:34:14.929 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:14.929 | INFO | "malloc" : " 14 GB", | 20260508 13:34:14.929 | INFO | "pinned" : " 29 GB" | 20260508 13:34:14.929 | INFO | } | 20260508 13:34:14.930 | INFO | Perf report for the last 5.1374 seconds: | 20260508 13:34:14.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:14.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | ANALYSIS : 66.4%, 3.0, 103.2ms, 0 | 20260508 13:34:14.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:34:14.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:14.930 | INFO | Uploaded 36GiB to GPU in 2.49611s (14.4224GiB/s) | 20260508 13:34:14.930 | INFO | Downloaded 34GiB to GPU in 2.48394s (13.6879GiB/s) | 20260508 13:34:20.067 | INFO | Analyzing chunk frames = [8500736,8501248) | 20260508 13:34:20.067 | INFO | MemInfo: { | 20260508 13:34:20.067 | INFO | "gpu" : " 17 GB", | 20260508 13:34:20.067 | INFO | "huge" : "", | 20260508 13:34:20.067 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:20.067 | INFO | "malloc" : " 14 GB", | 20260508 13:34:20.067 | INFO | "pinned" : " 29 GB" | 20260508 13:34:20.067 | INFO | } | 20260508 13:34:20.067 | INFO | Perf report for the last 5.1377 seconds: | 20260508 13:34:20.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:20.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | ANALYSIS : 65.2%, 3.0, 101.1ms, 0 | 20260508 13:34:20.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:34:20.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:20.068 | INFO | Uploaded 36GiB to GPU in 2.36197s (15.2415GiB/s) | 20260508 13:34:20.068 | INFO | Downloaded 34GiB to GPU in 2.47953s (13.7123GiB/s) | 20260508 13:34:25.207 | INFO | Analyzing chunk frames = [8501248,8501760) | 20260508 13:34:25.207 | INFO | MemInfo: { | 20260508 13:34:25.207 | INFO | "gpu" : " 17 GB", | 20260508 13:34:25.207 | INFO | "huge" : "", | 20260508 13:34:25.207 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:25.207 | INFO | "malloc" : " 14 GB", | 20260508 13:34:25.207 | INFO | "pinned" : " 29 GB" | 20260508 13:34:25.207 | INFO | } | 20260508 13:34:25.207 | INFO | Perf report for the last 5.13999 seconds: | 20260508 13:34:25.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:25.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.207 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:34:25.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.207 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:34:25.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:25.208 | INFO | Uploaded 36GiB to GPU in 2.30046s (15.6491GiB/s) | 20260508 13:34:25.208 | INFO | Downloaded 34GiB to GPU in 2.50054s (13.5971GiB/s) | 20260508 13:34:30.345 | INFO | Analyzing chunk frames = [8501760,8502272) | 20260508 13:34:30.345 | INFO | MemInfo: { | 20260508 13:34:30.345 | INFO | "gpu" : " 17 GB", | 20260508 13:34:30.345 | INFO | "huge" : "", | 20260508 13:34:30.345 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:30.345 | INFO | "malloc" : " 14 GB", | 20260508 13:34:30.345 | INFO | "pinned" : " 29 GB" | 20260508 13:34:30.345 | INFO | } | 20260508 13:34:30.346 | INFO | Perf report for the last 5.13827 seconds: | 20260508 13:34:30.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:30.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:34:30.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:34:30.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:30.346 | INFO | Uploaded 36GiB to GPU in 2.40067s (14.9958GiB/s) | 20260508 13:34:30.346 | INFO | Downloaded 34GiB to GPU in 2.45474s (13.8508GiB/s) | 20260508 13:34:35.488 | INFO | Analyzing chunk frames = [8502272,8502784) | 20260508 13:34:35.488 | INFO | MemInfo: { | 20260508 13:34:35.488 | INFO | "gpu" : " 17 GB", | 20260508 13:34:35.488 | INFO | "huge" : "", | 20260508 13:34:35.488 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:35.488 | INFO | "malloc" : " 14 GB", | 20260508 13:34:35.488 | INFO | "pinned" : " 29 GB" | 20260508 13:34:35.488 | INFO | } | 20260508 13:34:35.488 | INFO | Perf report for the last 5.14246 seconds: | 20260508 13:34:35.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:35.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 13:34:35.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:34:35.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:35.488 | INFO | Uploaded 36GiB to GPU in 2.38875s (15.0706GiB/s) | 20260508 13:34:35.488 | INFO | Downloaded 34GiB to GPU in 2.45652s (13.8407GiB/s) | 20260508 13:34:40.623 | INFO | Analyzing chunk frames = [8502784,8503296) | 20260508 13:34:40.623 | INFO | MemInfo: { | 20260508 13:34:40.623 | INFO | "gpu" : " 17 GB", | 20260508 13:34:40.623 | INFO | "huge" : "", | 20260508 13:34:40.623 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:40.623 | INFO | "malloc" : " 14 GB", | 20260508 13:34:40.623 | INFO | "pinned" : " 29 GB" | 20260508 13:34:40.623 | INFO | } | 20260508 13:34:40.623 | INFO | Perf report for the last 5.13487 seconds: | 20260508 13:34:40.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:40.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:34:40.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:34:40.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:40.623 | INFO | Uploaded 36GiB to GPU in 2.44499s (14.724GiB/s) | 20260508 13:34:40.623 | INFO | Downloaded 34GiB to GPU in 2.48412s (13.6869GiB/s) | 20260508 13:34:40.623 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:34:40.623 | INFO | Total mem usage: 16.7 GiB | 20260508 13:34:40.623 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:34:40.623 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:34:40.623 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:34:40.623 | INFO | High Water Breakdown: | 20260508 13:34:40.623 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:34:40.623 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:34:40.623 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:34:40.623 | INFO | ------------------------------------------ | 20260508 13:34:40.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:34:40.623 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:34:40.623 | INFO | Total mem usage: 26.5 GiB | 20260508 13:34:40.623 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:34:40.624 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:34:40.624 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:34:40.624 | INFO | High Water Breakdown: | 20260508 13:34:40.624 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:34:40.624 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:34:40.624 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:34:40.624 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:34:40.624 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:34:40.624 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:34:40.624 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:34:40.624 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:34:40.624 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:34:40.624 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:34:40.624 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:34:40.624 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:34:40.624 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:34:40.624 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:34:40.624 | INFO | ------------------------------------------ | 20260508 13:34:40.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:34:40.624 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:34:40.624 | INFO | Total mem usage: 27.8 GiB | 20260508 13:34:40.624 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:34:40.624 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:34:40.624 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:34:40.624 | INFO | High Water Breakdown: | 20260508 13:34:40.624 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:34:40.624 | INFO | ------------------------------------------ | 20260508 13:34:40.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:34:40.624 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:34:40.624 | INFO | Total mem usage: 15.7 GiB | 20260508 13:34:40.624 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:34:40.624 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:34:40.624 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:34:40.624 | INFO | High Water Breakdown: | 20260508 13:34:40.624 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:34:40.624 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:34:40.624 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:34:40.624 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:34:40.624 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:34:40.624 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:34:40.624 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:34:40.624 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:34:40.624 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:34:40.624 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:34:40.624 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:34:40.624 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:34:40.624 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:34:40.624 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:34:40.624 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:34:40.624 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:34:40.624 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:34:40.624 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:34:40.624 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:34:40.624 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:34:40.624 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:34:40.624 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:34:40.624 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:34:40.624 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:34:40.624 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:34:40.624 | INFO | ------------------------------------------ | 20260508 13:34:40.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:34:40.624 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:34:40.624 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:34:40.624 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:34:40.624 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.77e-01 | 20260508 13:34:40.624 | INFO | PerfReport: 0.30%, AggregateStats -- 9.77e-02 / 1.05e-01, 1.22e-02 / 8.02e-03 | 20260508 13:34:40.624 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.40e+00, 5.96e-01 / 2.08e-01 | 20260508 13:34:40.624 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.45e+00, 2.10e-01 / 1.26e-01 | 20260508 13:34:40.624 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.56e+01, 3.73e+00 / 1.06e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.81e+00 / 1.21e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.51e+00, 4.02e-01 / 2.53e-01 | 20260508 13:34:40.624 | INFO | PerfReport: 2.92%, Metrics -- 9.55e-01 / 1.09e+00, 2.40e-01 / 1.28e-01 | 20260508 13:34:40.624 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.90e-01 / 3.20e-01, 4.68e-02 / 2.03e-02 | 20260508 13:34:40.624 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:34:40.624 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:34:40.624 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:34:40.624 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 24.22%, Upload -- 2.53e+01 / 0.00e+00, 7.79e+00 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 24.92%, Download -- 2.61e+01 / 0.00e+00, 8.26e+00 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 31.23%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 1.99%, LockUpload -- 2.08e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 2.70%, LockDownload -- 2.83e+00 / 0.00e+00, 4.24e+00 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 14.92%, LockCompute -- 1.56e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:34:40.624 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260508 13:34:40.720 | INFO | Starting a round of Baz IO | 20260508 13:34:44.151 | INFO | Populating RTMetrics report | 20260508 13:34:44.152 | INFO | RTMetrics report populated | 20260508 13:34:44.152 | INFO | Opening tmp RTMetrics json file | 20260508 13:34:44.153 | INFO | Serializing RTMetrics to json | 20260508 13:34:44.154 | INFO | Serialization done, swapping tmp file now... | 20260508 13:34:44.155 | INFO | RTMetrics done | 20260508 13:34:44.226 | INFO | Finished a round of Baz IO: Wrote 1.383 GiB in 3.506 seconds | 20260508 13:34:44.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:34:44.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139165% / 0% / 7.15256e-05% | 20260508 13:34:44.227 | INFO | Baz IO overall: | 20260508 13:34:44.227 | INFO | BytesWritten : 3620907875205 | 20260508 13:34:44.227 | INFO | NumEvents : 1735903872247 | 20260508 13:34:45.756 | INFO | Analyzing chunk frames = [8503296,8503808) | 20260508 13:34:45.756 | INFO | MemInfo: { | 20260508 13:34:45.756 | INFO | "gpu" : " 17 GB", | 20260508 13:34:45.756 | INFO | "huge" : "", | 20260508 13:34:45.756 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:45.756 | INFO | "malloc" : " 14 GB", | 20260508 13:34:45.756 | INFO | "pinned" : " 29 GB" | 20260508 13:34:45.756 | INFO | } | 20260508 13:34:45.756 | INFO | Perf report for the last 5.13317 seconds: | 20260508 13:34:45.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:45.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:45.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:45.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:45.756 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 13:34:45.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:45.756 | INFO | RT_METRICS : 38.7%, 1.0, 20.3ms, 0 | 20260508 13:34:45.756 | INFO | PRE_HQ : 65.9%, 1.6, 54.8ms, 0 | 20260508 13:34:45.756 | INFO | BAZWRITER : 67.6%, 2.7, 93.9ms, 0 | 20260508 13:34:45.756 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260508 13:34:45.756 | INFO | Uploaded 36GiB to GPU in 2.59254s (13.886GiB/s) | 20260508 13:34:45.756 | INFO | Downloaded 38GiB to GPU in 2.71755s (13.9832GiB/s) | 20260508 13:34:50.906 | INFO | Analyzing chunk frames = [8503808,8504320) | 20260508 13:34:50.906 | INFO | MemInfo: { | 20260508 13:34:50.906 | INFO | "gpu" : " 17 GB", | 20260508 13:34:50.906 | INFO | "huge" : "", | 20260508 13:34:50.906 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:50.906 | INFO | "malloc" : " 14 GB", | 20260508 13:34:50.906 | INFO | "pinned" : " 29 GB" | 20260508 13:34:50.906 | INFO | } | 20260508 13:34:50.906 | INFO | Perf report for the last 5.14998 seconds: | 20260508 13:34:50.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:50.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.906 | INFO | ANALYSIS : 66.4%, 3.0, 103.8ms, 0 | 20260508 13:34:50.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.906 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:34:50.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:50.907 | INFO | Uploaded 36GiB to GPU in 2.4914s (14.4497GiB/s) | 20260508 13:34:50.907 | INFO | Downloaded 34GiB to GPU in 2.48304s (13.6929GiB/s) | 20260508 13:34:56.045 | INFO | Analyzing chunk frames = [8504320,8504832) | 20260508 13:34:56.045 | INFO | MemInfo: { | 20260508 13:34:56.045 | INFO | "gpu" : " 17 GB", | 20260508 13:34:56.045 | INFO | "huge" : "", | 20260508 13:34:56.045 | INFO | "huge_shared" : " 29 GB", | 20260508 13:34:56.045 | INFO | "malloc" : " 14 GB", | 20260508 13:34:56.045 | INFO | "pinned" : " 29 GB" | 20260508 13:34:56.045 | INFO | } | 20260508 13:34:56.045 | INFO | Perf report for the last 5.13922 seconds: | 20260508 13:34:56.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:34:56.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | ANALYSIS : 65.7%, 3.0, 102.2ms, 0 | 20260508 13:34:56.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:34:56.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:34:56.046 | INFO | Uploaded 36GiB to GPU in 2.43295s (14.7968GiB/s) | 20260508 13:34:56.046 | INFO | Downloaded 34GiB to GPU in 2.47217s (13.7531GiB/s) | 20260508 13:35:01.185 | INFO | Analyzing chunk frames = [8504832,8505344) | 20260508 13:35:01.185 | INFO | MemInfo: { | 20260508 13:35:01.185 | INFO | "gpu" : " 17 GB", | 20260508 13:35:01.185 | INFO | "huge" : "", | 20260508 13:35:01.185 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:01.185 | INFO | "malloc" : " 14 GB", | 20260508 13:35:01.185 | INFO | "pinned" : " 29 GB" | 20260508 13:35:01.185 | INFO | } | 20260508 13:35:01.185 | INFO | Perf report for the last 5.13957 seconds: | 20260508 13:35:01.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:01.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | ANALYSIS : 65.8%, 3.0, 102.0ms, 0 | 20260508 13:35:01.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:35:01.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:01.185 | INFO | Uploaded 36GiB to GPU in 2.48383s (14.4938GiB/s) | 20260508 13:35:01.185 | INFO | Downloaded 34GiB to GPU in 2.47375s (13.7443GiB/s) | 20260508 13:35:06.322 | INFO | Analyzing chunk frames = [8505344,8505856) | 20260508 13:35:06.322 | INFO | MemInfo: { | 20260508 13:35:06.322 | INFO | "gpu" : " 17 GB", | 20260508 13:35:06.322 | INFO | "huge" : "", | 20260508 13:35:06.322 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:06.322 | INFO | "malloc" : " 14 GB", | 20260508 13:35:06.322 | INFO | "pinned" : " 29 GB" | 20260508 13:35:06.322 | INFO | } | 20260508 13:35:06.322 | INFO | Perf report for the last 5.1371 seconds: | 20260508 13:35:06.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:06.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.322 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 13:35:06.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.322 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:35:06.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:06.323 | INFO | Uploaded 36GiB to GPU in 2.40614s (14.9617GiB/s) | 20260508 13:35:06.323 | INFO | Downloaded 34GiB to GPU in 2.54062s (13.3825GiB/s) | 20260508 13:35:11.460 | INFO | Analyzing chunk frames = [8505856,8506368) | 20260508 13:35:11.460 | INFO | MemInfo: { | 20260508 13:35:11.460 | INFO | "gpu" : " 17 GB", | 20260508 13:35:11.460 | INFO | "huge" : "", | 20260508 13:35:11.460 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:11.460 | INFO | "malloc" : " 14 GB", | 20260508 13:35:11.460 | INFO | "pinned" : " 29 GB" | 20260508 13:35:11.460 | INFO | } | 20260508 13:35:11.460 | INFO | Perf report for the last 5.13794 seconds: | 20260508 13:35:11.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:11.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:35:11.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:35:11.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:11.460 | INFO | Uploaded 36GiB to GPU in 2.4302s (14.8136GiB/s) | 20260508 13:35:11.460 | INFO | Downloaded 34GiB to GPU in 2.45675s (13.8394GiB/s) | 20260508 13:35:16.598 | INFO | Analyzing chunk frames = [8506368,8506880) | 20260508 13:35:16.598 | INFO | MemInfo: { | 20260508 13:35:16.598 | INFO | "gpu" : " 17 GB", | 20260508 13:35:16.598 | INFO | "huge" : "", | 20260508 13:35:16.598 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:16.598 | INFO | "malloc" : " 14 GB", | 20260508 13:35:16.598 | INFO | "pinned" : " 29 GB" | 20260508 13:35:16.598 | INFO | } | 20260508 13:35:16.598 | INFO | Perf report for the last 5.13838 seconds: | 20260508 13:35:16.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:16.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:35:16.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:35:16.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:16.599 | INFO | Uploaded 36GiB to GPU in 2.31173s (15.5728GiB/s) | 20260508 13:35:16.599 | INFO | Downloaded 34GiB to GPU in 2.42869s (13.9993GiB/s) | 20260508 13:35:21.743 | INFO | Analyzing chunk frames = [8506880,8507392) | 20260508 13:35:21.743 | INFO | MemInfo: { | 20260508 13:35:21.743 | INFO | "gpu" : " 17 GB", | 20260508 13:35:21.743 | INFO | "huge" : "", | 20260508 13:35:21.743 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:21.743 | INFO | "malloc" : " 14 GB", | 20260508 13:35:21.743 | INFO | "pinned" : " 29 GB" | 20260508 13:35:21.744 | INFO | } | 20260508 13:35:21.744 | INFO | Perf report for the last 5.1453 seconds: | 20260508 13:35:21.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:21.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.744 | INFO | ANALYSIS : 65.6%, 3.0, 102.6ms, 0 | 20260508 13:35:21.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.744 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 13:35:21.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:21.745 | INFO | Uploaded 36GiB to GPU in 2.4785s (14.5249GiB/s) | 20260508 13:35:21.745 | INFO | Downloaded 34GiB to GPU in 2.47816s (13.7199GiB/s) | 20260508 13:35:21.841 | INFO | Starting a round of Baz IO | 20260508 13:35:25.250 | INFO | Populating RTMetrics report | 20260508 13:35:25.250 | INFO | RTMetrics report populated | 20260508 13:35:25.250 | INFO | Opening tmp RTMetrics json file | 20260508 13:35:25.262 | INFO | Serializing RTMetrics to json | 20260508 13:35:25.264 | INFO | Serialization done, swapping tmp file now... | 20260508 13:35:25.265 | INFO | RTMetrics done | 20260508 13:35:25.320 | INFO | Finished a round of Baz IO: Wrote 1.384 GiB in 3.479 seconds | 20260508 13:35:25.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.94% / 0.00% / 60.06% | 20260508 13:35:25.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138879% / 0% / 7.94729e-05% | 20260508 13:35:25.320 | INFO | Baz IO overall: | 20260508 13:35:25.320 | INFO | BytesWritten : 3622393615697 | 20260508 13:35:25.321 | INFO | NumEvents : 1736598733185 | 20260508 13:35:26.881 | INFO | Analyzing chunk frames = [8507392,8507904) | 20260508 13:35:26.881 | INFO | MemInfo: { | 20260508 13:35:26.881 | INFO | "gpu" : " 17 GB", | 20260508 13:35:26.881 | INFO | "huge" : "", | 20260508 13:35:26.881 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:26.881 | INFO | "malloc" : " 14 GB", | 20260508 13:35:26.881 | INFO | "pinned" : " 29 GB" | 20260508 13:35:26.881 | INFO | } | 20260508 13:35:26.881 | INFO | Perf report for the last 5.13711 seconds: | 20260508 13:35:26.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:26.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:26.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:26.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:26.881 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:35:26.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:26.881 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:35:26.881 | INFO | PRE_HQ : 65.2%, 1.6, 53.5ms, 0 | 20260508 13:35:26.881 | INFO | BAZWRITER : 67.0%, 2.4, 83.5ms, 0 | 20260508 13:35:26.881 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:35:26.881 | INFO | Uploaded 36GiB to GPU in 2.64009s (13.6359GiB/s) | 20260508 13:35:26.881 | INFO | Downloaded 38GiB to GPU in 2.71816s (13.9801GiB/s) | 20260508 13:35:32.017 | INFO | Analyzing chunk frames = [8507904,8508416) | 20260508 13:35:32.017 | INFO | MemInfo: { | 20260508 13:35:32.017 | INFO | "gpu" : " 17 GB", | 20260508 13:35:32.017 | INFO | "huge" : "", | 20260508 13:35:32.017 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:32.017 | INFO | "malloc" : " 14 GB", | 20260508 13:35:32.017 | INFO | "pinned" : " 29 GB" | 20260508 13:35:32.017 | INFO | } | 20260508 13:35:32.017 | INFO | Perf report for the last 5.13629 seconds: | 20260508 13:35:32.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:32.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.017 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:35:32.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.017 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:35:32.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:32.018 | INFO | Uploaded 36GiB to GPU in 2.47331s (14.5554GiB/s) | 20260508 13:35:32.018 | INFO | Downloaded 34GiB to GPU in 2.49066s (13.651GiB/s) | 20260508 13:35:37.160 | INFO | Analyzing chunk frames = [8508416,8508928) | 20260508 13:35:37.160 | INFO | MemInfo: { | 20260508 13:35:37.160 | INFO | "gpu" : " 17 GB", | 20260508 13:35:37.160 | INFO | "huge" : "", | 20260508 13:35:37.160 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:37.160 | INFO | "malloc" : " 14 GB", | 20260508 13:35:37.160 | INFO | "pinned" : " 29 GB" | 20260508 13:35:37.160 | INFO | } | 20260508 13:35:37.160 | INFO | Perf report for the last 5.14305 seconds: | 20260508 13:35:37.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:37.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.160 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 13:35:37.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.160 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:35:37.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:37.161 | INFO | Uploaded 36GiB to GPU in 2.48843s (14.4669GiB/s) | 20260508 13:35:37.161 | INFO | Downloaded 34GiB to GPU in 2.46434s (13.7968GiB/s) | 20260508 13:35:42.296 | INFO | Analyzing chunk frames = [8508928,8509440) | 20260508 13:35:42.296 | INFO | MemInfo: { | 20260508 13:35:42.296 | INFO | "gpu" : " 17 GB", | 20260508 13:35:42.296 | INFO | "huge" : "", | 20260508 13:35:42.296 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:42.296 | INFO | "malloc" : " 14 GB", | 20260508 13:35:42.296 | INFO | "pinned" : " 29 GB" | 20260508 13:35:42.296 | INFO | } | 20260508 13:35:42.296 | INFO | Perf report for the last 5.13605 seconds: | 20260508 13:35:42.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:42.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.296 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 13:35:42.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.296 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:35:42.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:42.297 | INFO | Uploaded 36GiB to GPU in 2.39463s (15.0337GiB/s) | 20260508 13:35:42.297 | INFO | Downloaded 34GiB to GPU in 2.47017s (13.7642GiB/s) | 20260508 13:35:47.437 | INFO | Analyzing chunk frames = [8509440,8509952) | 20260508 13:35:47.437 | INFO | MemInfo: { | 20260508 13:35:47.437 | INFO | "gpu" : " 17 GB", | 20260508 13:35:47.437 | INFO | "huge" : "", | 20260508 13:35:47.437 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:47.437 | INFO | "malloc" : " 14 GB", | 20260508 13:35:47.437 | INFO | "pinned" : " 29 GB" | 20260508 13:35:47.438 | INFO | } | 20260508 13:35:47.438 | INFO | Perf report for the last 5.14121 seconds: | 20260508 13:35:47.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:47.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 13:35:47.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:35:47.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:47.438 | INFO | Uploaded 36GiB to GPU in 2.4372s (14.7711GiB/s) | 20260508 13:35:47.438 | INFO | Downloaded 34GiB to GPU in 2.52466s (13.4671GiB/s) | 20260508 13:35:52.573 | INFO | Analyzing chunk frames = [8509952,8510464) | 20260508 13:35:52.573 | INFO | MemInfo: { | 20260508 13:35:52.573 | INFO | "gpu" : " 17 GB", | 20260508 13:35:52.573 | INFO | "huge" : "", | 20260508 13:35:52.573 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:52.573 | INFO | "malloc" : " 14 GB", | 20260508 13:35:52.573 | INFO | "pinned" : " 29 GB" | 20260508 13:35:52.573 | INFO | } | 20260508 13:35:52.573 | INFO | Perf report for the last 5.13561 seconds: | 20260508 13:35:52.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:52.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:35:52.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:35:52.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:52.573 | INFO | Uploaded 36GiB to GPU in 2.3981s (15.0119GiB/s) | 20260508 13:35:52.573 | INFO | Downloaded 34GiB to GPU in 2.44667s (13.8964GiB/s) | 20260508 13:35:57.714 | INFO | Analyzing chunk frames = [8510464,8510976) | 20260508 13:35:57.714 | INFO | MemInfo: { | 20260508 13:35:57.714 | INFO | "gpu" : " 17 GB", | 20260508 13:35:57.714 | INFO | "huge" : "", | 20260508 13:35:57.714 | INFO | "huge_shared" : " 29 GB", | 20260508 13:35:57.714 | INFO | "malloc" : " 14 GB", | 20260508 13:35:57.714 | INFO | "pinned" : " 29 GB" | 20260508 13:35:57.714 | INFO | } | 20260508 13:35:57.714 | INFO | Perf report for the last 5.14065 seconds: | 20260508 13:35:57.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:35:57.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:35:57.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260508 13:35:57.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:35:57.714 | INFO | Uploaded 36GiB to GPU in 2.46872s (14.5824GiB/s) | 20260508 13:35:57.714 | INFO | Downloaded 34GiB to GPU in 2.46987s (13.7659GiB/s) | 20260508 13:36:02.853 | INFO | Analyzing chunk frames = [8510976,8511488) | 20260508 13:36:02.853 | INFO | MemInfo: { | 20260508 13:36:02.853 | INFO | "gpu" : " 17 GB", | 20260508 13:36:02.853 | INFO | "huge" : "", | 20260508 13:36:02.853 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:02.853 | INFO | "malloc" : " 14 GB", | 20260508 13:36:02.853 | INFO | "pinned" : " 29 GB" | 20260508 13:36:02.853 | INFO | } | 20260508 13:36:02.853 | INFO | Perf report for the last 5.13871 seconds: | 20260508 13:36:02.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:02.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | ANALYSIS : 64.6%, 3.0, 100.9ms, 0 | 20260508 13:36:02.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:36:02.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:02.853 | INFO | Uploaded 36GiB to GPU in 2.35971s (15.2561GiB/s) | 20260508 13:36:02.853 | INFO | Downloaded 34GiB to GPU in 2.43367s (13.9707GiB/s) | 20260508 13:36:02.853 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:36:02.853 | INFO | Total mem usage: 16.7 GiB | 20260508 13:36:02.853 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:36:02.853 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -14.8%) | 20260508 13:36:02.853 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:36:02.853 | INFO | High Water Breakdown: | 20260508 13:36:02.853 | INFO | 59.8% : 8.69 GiB ( 84.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:36:02.853 | INFO | 40.1% : 5.82 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:36:02.853 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:36:02.853 | INFO | ------------------------------------------ | 20260508 13:36:02.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:36:02.853 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:36:02.853 | INFO | Total mem usage: 26.5 GiB | 20260508 13:36:02.853 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:36:02.853 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:36:02.853 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:36:02.853 | INFO | High Water Breakdown: | 20260508 13:36:02.853 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:36:02.853 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:36:02.853 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:36:02.853 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:36:02.853 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:36:02.853 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:36:02.853 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:36:02.853 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:36:02.853 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:36:02.853 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:36:02.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:36:02.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:36:02.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:36:02.854 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:36:02.854 | INFO | ------------------------------------------ | 20260508 13:36:02.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:36:02.854 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:36:02.854 | INFO | Total mem usage: 27.8 GiB | 20260508 13:36:02.854 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:36:02.854 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:36:02.854 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:36:02.854 | INFO | High Water Breakdown: | 20260508 13:36:02.854 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:36:02.854 | INFO | ------------------------------------------ | 20260508 13:36:02.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:36:02.854 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:36:02.854 | INFO | Total mem usage: 15.7 GiB | 20260508 13:36:02.854 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:36:02.854 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:36:02.854 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:36:02.854 | INFO | High Water Breakdown: | 20260508 13:36:02.854 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:36:02.854 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:36:02.854 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:36:02.854 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:36:02.854 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:36:02.854 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:36:02.854 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:36:02.854 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:36:02.854 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:36:02.854 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:36:02.854 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:36:02.854 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:36:02.854 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:36:02.854 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:36:02.854 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:36:02.854 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:36:02.854 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:36:02.854 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:36:02.855 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:36:02.855 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:36:02.855 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:36:02.855 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:36:02.855 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:36:02.855 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:36:02.855 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:36:02.855 | INFO | ------------------------------------------ | 20260508 13:36:02.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:36:02.855 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:36:02.855 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:36:02.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:36:02.855 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 4.83e-01 | 20260508 13:36:02.855 | INFO | PerfReport: 0.30%, AggregateStats -- 9.62e-02 / 1.07e-01, 1.10e-02 / 1.25e-02 | 20260508 13:36:02.855 | INFO | PerfReport: 12.20%, Baseline -- 3.97e+00 / 4.44e+00, 5.96e-01 / 2.14e-01 | 20260508 13:36:02.855 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.21e-01 | 20260508 13:36:02.855 | INFO | PerfReport: 16.00%, DME -- 4.17e+01 / 4.50e+01, 3.59e+00 / 9.57e-01 | 20260508 13:36:02.855 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.86e+01 / 2.04e+01, 1.81e+00 / 1.31e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.31e+00 / 2.43e+00, 4.03e-01 / 2.38e-01 | 20260508 13:36:02.855 | INFO | PerfReport: 2.92%, Metrics -- 9.53e-01 / 1.06e+00, 2.38e-01 / 1.05e-01 | 20260508 13:36:02.855 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.10e-01, 4.55e-02 / 2.27e-02 | 20260508 13:36:02.855 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:36:02.855 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:36:02.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:36:02.855 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 24.41%, Upload -- 2.56e+01 / 0.00e+00, 8.02e+00 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 24.90%, Download -- 2.61e+01 / 0.00e+00, 8.22e+00 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 31.10%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 2.04%, LockUpload -- 2.14e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 2.70%, LockDownload -- 2.83e+00 / 0.00e+00, 4.08e+00 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 14.84%, LockCompute -- 1.55e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 13:36:02.855 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 13:36:02.953 | INFO | Starting a round of Baz IO | 20260508 13:36:06.359 | INFO | Populating RTMetrics report | 20260508 13:36:06.359 | INFO | RTMetrics report populated | 20260508 13:36:06.359 | INFO | Opening tmp RTMetrics json file | 20260508 13:36:06.363 | INFO | Serializing RTMetrics to json | 20260508 13:36:06.364 | INFO | Serialization done, swapping tmp file now... | 20260508 13:36:06.365 | INFO | RTMetrics done | 20260508 13:36:06.426 | INFO | Finished a round of Baz IO: Wrote 1.384 GiB in 3.473 seconds | 20260508 13:36:06.426 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.06% | 20260508 13:36:06.426 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138994% / 7.94729e-06% / 5.5631e-05% | 20260508 13:36:06.426 | INFO | Baz IO overall: | 20260508 13:36:06.426 | INFO | BytesWritten : 3623879693160 | 20260508 13:36:06.426 | INFO | NumEvents : 1737294116108 | 20260508 13:36:07.995 | INFO | Analyzing chunk frames = [8511488,8512000) | 20260508 13:36:07.995 | INFO | MemInfo: { | 20260508 13:36:07.995 | INFO | "gpu" : " 17 GB", | 20260508 13:36:07.995 | INFO | "huge" : "", | 20260508 13:36:07.995 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:07.995 | INFO | "malloc" : " 14 GB", | 20260508 13:36:07.995 | INFO | "pinned" : " 29 GB" | 20260508 13:36:07.995 | INFO | } | 20260508 13:36:07.995 | INFO | Perf report for the last 5.14245 seconds: | 20260508 13:36:07.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:07.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:07.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:07.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:07.995 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:36:07.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:07.995 | INFO | RT_METRICS : 38.3%, 1.0, 20.1ms, 0 | 20260508 13:36:07.995 | INFO | PRE_HQ : 65.2%, 1.6, 54.3ms, 0 | 20260508 13:36:07.995 | INFO | BAZWRITER : 66.9%, 2.5, 84.9ms, 0 | 20260508 13:36:07.995 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:36:07.996 | INFO | Uploaded 36GiB to GPU in 2.6313s (13.6815GiB/s) | 20260508 13:36:07.996 | INFO | Downloaded 38GiB to GPU in 2.7124s (14.0097GiB/s) | 20260508 13:36:13.133 | INFO | Analyzing chunk frames = [8512000,8512512) | 20260508 13:36:13.133 | INFO | MemInfo: { | 20260508 13:36:13.133 | INFO | "gpu" : " 17 GB", | 20260508 13:36:13.133 | INFO | "huge" : "", | 20260508 13:36:13.133 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:13.133 | INFO | "malloc" : " 14 GB", | 20260508 13:36:13.133 | INFO | "pinned" : " 29 GB" | 20260508 13:36:13.133 | INFO | } | 20260508 13:36:13.133 | INFO | Perf report for the last 5.13818 seconds: | 20260508 13:36:13.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:13.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.133 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:36:13.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.134 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:36:13.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:13.134 | INFO | Uploaded 36GiB to GPU in 2.51088s (14.3376GiB/s) | 20260508 13:36:13.134 | INFO | Downloaded 34GiB to GPU in 2.4712s (13.7585GiB/s) | 20260508 13:36:18.266 | INFO | Analyzing chunk frames = [8512512,8513024) | 20260508 13:36:18.266 | INFO | MemInfo: { | 20260508 13:36:18.266 | INFO | "gpu" : " 17 GB", | 20260508 13:36:18.266 | INFO | "huge" : "", | 20260508 13:36:18.266 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:18.266 | INFO | "malloc" : " 14 GB", | 20260508 13:36:18.266 | INFO | "pinned" : " 29 GB" | 20260508 13:36:18.266 | INFO | } | 20260508 13:36:18.266 | INFO | Perf report for the last 5.1328 seconds: | 20260508 13:36:18.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:18.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 13:36:18.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:36:18.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:18.266 | INFO | Uploaded 36GiB to GPU in 2.46646s (14.5958GiB/s) | 20260508 13:36:18.266 | INFO | Downloaded 34GiB to GPU in 2.44058s (13.9311GiB/s) | 20260508 13:36:23.413 | INFO | Analyzing chunk frames = [8513024,8513536) | 20260508 13:36:23.413 | INFO | MemInfo: { | 20260508 13:36:23.413 | INFO | "gpu" : " 17 GB", | 20260508 13:36:23.413 | INFO | "huge" : "", | 20260508 13:36:23.413 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:23.413 | INFO | "malloc" : " 14 GB", | 20260508 13:36:23.413 | INFO | "pinned" : " 29 GB" | 20260508 13:36:23.413 | INFO | } | 20260508 13:36:23.414 | INFO | Perf report for the last 5.14761 seconds: | 20260508 13:36:23.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:23.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:36:23.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:36:23.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:23.414 | INFO | Uploaded 36GiB to GPU in 2.51394s (14.3201GiB/s) | 20260508 13:36:23.414 | INFO | Downloaded 34GiB to GPU in 2.43978s (13.9357GiB/s) | 20260508 13:36:28.555 | INFO | Analyzing chunk frames = [8513536,8514048) | 20260508 13:36:28.556 | INFO | MemInfo: { | 20260508 13:36:28.556 | INFO | "gpu" : " 17 GB", | 20260508 13:36:28.556 | INFO | "huge" : "", | 20260508 13:36:28.556 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:28.556 | INFO | "malloc" : " 14 GB", | 20260508 13:36:28.556 | INFO | "pinned" : " 29 GB" | 20260508 13:36:28.556 | INFO | } | 20260508 13:36:28.556 | INFO | Perf report for the last 5.14187 seconds: | 20260508 13:36:28.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:28.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 13:36:28.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 13:36:28.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:28.556 | INFO | Uploaded 36GiB to GPU in 2.3069s (15.6053GiB/s) | 20260508 13:36:28.556 | INFO | Downloaded 34GiB to GPU in 2.48258s (13.6954GiB/s) | 20260508 13:36:33.695 | INFO | Analyzing chunk frames = [8514048,8514560) | 20260508 13:36:33.695 | INFO | MemInfo: { | 20260508 13:36:33.695 | INFO | "gpu" : " 17 GB", | 20260508 13:36:33.695 | INFO | "huge" : "", | 20260508 13:36:33.695 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:33.695 | INFO | "malloc" : " 14 GB", | 20260508 13:36:33.695 | INFO | "pinned" : " 29 GB" | 20260508 13:36:33.695 | INFO | } | 20260508 13:36:33.695 | INFO | Perf report for the last 5.13967 seconds: | 20260508 13:36:33.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:33.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.695 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:36:33.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.695 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:36:33.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:33.696 | INFO | Uploaded 36GiB to GPU in 2.34468s (15.3539GiB/s) | 20260508 13:36:33.696 | INFO | Downloaded 34GiB to GPU in 2.4032s (14.1478GiB/s) | 20260508 13:36:38.833 | INFO | Analyzing chunk frames = [8514560,8515072) | 20260508 13:36:38.833 | INFO | MemInfo: { | 20260508 13:36:38.833 | INFO | "gpu" : " 17 GB", | 20260508 13:36:38.833 | INFO | "huge" : "", | 20260508 13:36:38.833 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:38.833 | INFO | "malloc" : " 14 GB", | 20260508 13:36:38.833 | INFO | "pinned" : " 29 GB" | 20260508 13:36:38.833 | INFO | } | 20260508 13:36:38.833 | INFO | Perf report for the last 5.13791 seconds: | 20260508 13:36:38.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:38.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | ANALYSIS : 64.4%, 3.0, 100.2ms, 0 | 20260508 13:36:38.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:36:38.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:38.833 | INFO | Uploaded 36GiB to GPU in 2.33814s (15.3968GiB/s) | 20260508 13:36:38.834 | INFO | Downloaded 34GiB to GPU in 2.37988s (14.2865GiB/s) | 20260508 13:36:43.970 | INFO | Analyzing chunk frames = [8515072,8515584) | 20260508 13:36:43.970 | INFO | MemInfo: { | 20260508 13:36:43.970 | INFO | "gpu" : " 17 GB", | 20260508 13:36:43.970 | INFO | "huge" : "", | 20260508 13:36:43.970 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:43.970 | INFO | "malloc" : " 14 GB", | 20260508 13:36:43.970 | INFO | "pinned" : " 29 GB" | 20260508 13:36:43.970 | INFO | } | 20260508 13:36:43.970 | INFO | Perf report for the last 5.13702 seconds: | 20260508 13:36:43.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:43.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260508 13:36:43.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:36:43.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:43.970 | INFO | Uploaded 36GiB to GPU in 2.33512s (15.4167GiB/s) | 20260508 13:36:43.970 | INFO | Downloaded 34GiB to GPU in 2.39789s (14.1791GiB/s) | 20260508 13:36:44.068 | INFO | Starting a round of Baz IO | 20260508 13:36:47.466 | INFO | Populating RTMetrics report | 20260508 13:36:47.466 | INFO | RTMetrics report populated | 20260508 13:36:47.466 | INFO | Opening tmp RTMetrics json file | 20260508 13:36:47.474 | INFO | Serializing RTMetrics to json | 20260508 13:36:47.475 | INFO | Serialization done, swapping tmp file now... | 20260508 13:36:47.476 | INFO | RTMetrics done | 20260508 13:36:47.537 | INFO | Finished a round of Baz IO: Wrote 1.384 GiB in 3.469 seconds | 20260508 13:36:47.537 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:36:47.537 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138779% / 0% / 6.75519e-05% | 20260508 13:36:47.537 | INFO | Baz IO overall: | 20260508 13:36:47.537 | INFO | BytesWritten : 3625366235376 | 20260508 13:36:47.537 | INFO | NumEvents : 1737990033916 | 20260508 13:36:49.108 | INFO | Analyzing chunk frames = [8515584,8516096) | 20260508 13:36:49.108 | INFO | MemInfo: { | 20260508 13:36:49.108 | INFO | "gpu" : " 17 GB", | 20260508 13:36:49.108 | INFO | "huge" : "", | 20260508 13:36:49.109 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:49.109 | INFO | "malloc" : " 14 GB", | 20260508 13:36:49.109 | INFO | "pinned" : " 29 GB" | 20260508 13:36:49.109 | INFO | } | 20260508 13:36:49.109 | INFO | Perf report for the last 5.13841 seconds: | 20260508 13:36:49.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:49.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:49.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:49.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:49.109 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:36:49.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:49.109 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:36:49.109 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 13:36:49.109 | INFO | BAZWRITER : 66.7%, 2.4, 83.2ms, 0 | 20260508 13:36:49.109 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:36:49.109 | INFO | Uploaded 36GiB to GPU in 2.63039s (13.6862GiB/s) | 20260508 13:36:49.109 | INFO | Downloaded 38GiB to GPU in 2.72442s (13.9479GiB/s) | 20260508 13:36:54.249 | INFO | Analyzing chunk frames = [8516096,8516608) | 20260508 13:36:54.249 | INFO | MemInfo: { | 20260508 13:36:54.249 | INFO | "gpu" : " 17 GB", | 20260508 13:36:54.249 | INFO | "huge" : "", | 20260508 13:36:54.249 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:54.249 | INFO | "malloc" : " 14 GB", | 20260508 13:36:54.249 | INFO | "pinned" : " 29 GB" | 20260508 13:36:54.249 | INFO | } | 20260508 13:36:54.249 | INFO | Perf report for the last 5.14055 seconds: | 20260508 13:36:54.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:54.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.249 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:36:54.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.249 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:36:54.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:54.250 | INFO | Uploaded 36GiB to GPU in 2.47828s (14.5262GiB/s) | 20260508 13:36:54.250 | INFO | Downloaded 34GiB to GPU in 2.48231s (13.6969GiB/s) | 20260508 13:36:59.387 | INFO | Analyzing chunk frames = [8516608,8517120) | 20260508 13:36:59.388 | INFO | MemInfo: { | 20260508 13:36:59.388 | INFO | "gpu" : " 17 GB", | 20260508 13:36:59.388 | INFO | "huge" : "", | 20260508 13:36:59.388 | INFO | "huge_shared" : " 29 GB", | 20260508 13:36:59.388 | INFO | "malloc" : " 14 GB", | 20260508 13:36:59.388 | INFO | "pinned" : " 29 GB" | 20260508 13:36:59.388 | INFO | } | 20260508 13:36:59.388 | INFO | Perf report for the last 5.13853 seconds: | 20260508 13:36:59.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:36:59.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 13:36:59.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:36:59.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:36:59.388 | INFO | Uploaded 36GiB to GPU in 2.44744s (14.7093GiB/s) | 20260508 13:36:59.388 | INFO | Downloaded 34GiB to GPU in 2.46536s (13.7911GiB/s) | 20260508 13:37:04.530 | INFO | Analyzing chunk frames = [8517120,8517632) | 20260508 13:37:04.530 | INFO | MemInfo: { | 20260508 13:37:04.530 | INFO | "gpu" : " 17 GB", | 20260508 13:37:04.530 | INFO | "huge" : "", | 20260508 13:37:04.530 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:04.530 | INFO | "malloc" : " 14 GB", | 20260508 13:37:04.530 | INFO | "pinned" : " 29 GB" | 20260508 13:37:04.530 | INFO | } | 20260508 13:37:04.530 | INFO | Perf report for the last 5.14199 seconds: | 20260508 13:37:04.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:04.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | ANALYSIS : 65.1%, 3.0, 101.0ms, 0 | 20260508 13:37:04.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260508 13:37:04.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:04.530 | INFO | Uploaded 36GiB to GPU in 2.34512s (15.351GiB/s) | 20260508 13:37:04.530 | INFO | Downloaded 34GiB to GPU in 2.45535s (13.8473GiB/s) | 20260508 13:37:09.668 | INFO | Analyzing chunk frames = [8517632,8518144) | 20260508 13:37:09.668 | INFO | MemInfo: { | 20260508 13:37:09.668 | INFO | "gpu" : " 17 GB", | 20260508 13:37:09.668 | INFO | "huge" : "", | 20260508 13:37:09.668 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:09.668 | INFO | "malloc" : " 14 GB", | 20260508 13:37:09.668 | INFO | "pinned" : " 29 GB" | 20260508 13:37:09.668 | INFO | } | 20260508 13:37:09.668 | INFO | Perf report for the last 5.13846 seconds: | 20260508 13:37:09.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:09.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.668 | INFO | ANALYSIS : 66.0%, 3.0, 103.2ms, 0 | 20260508 13:37:09.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.668 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:37:09.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:09.669 | INFO | Uploaded 36GiB to GPU in 2.41854s (14.885GiB/s) | 20260508 13:37:09.669 | INFO | Downloaded 34GiB to GPU in 2.52025s (13.4907GiB/s) | 20260508 13:37:14.807 | INFO | Analyzing chunk frames = [8518144,8518656) | 20260508 13:37:14.807 | INFO | MemInfo: { | 20260508 13:37:14.807 | INFO | "gpu" : " 17 GB", | 20260508 13:37:14.807 | INFO | "huge" : "", | 20260508 13:37:14.807 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:14.807 | INFO | "malloc" : " 14 GB", | 20260508 13:37:14.807 | INFO | "pinned" : " 29 GB" | 20260508 13:37:14.807 | INFO | } | 20260508 13:37:14.807 | INFO | Perf report for the last 5.13847 seconds: | 20260508 13:37:14.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:14.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:37:14.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:37:14.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:14.807 | INFO | Uploaded 36GiB to GPU in 2.41776s (14.8898GiB/s) | 20260508 13:37:14.807 | INFO | Downloaded 34GiB to GPU in 2.44668s (13.8964GiB/s) | 20260508 13:37:19.945 | INFO | Analyzing chunk frames = [8518656,8519168) | 20260508 13:37:19.945 | INFO | MemInfo: { | 20260508 13:37:19.945 | INFO | "gpu" : " 17 GB", | 20260508 13:37:19.945 | INFO | "huge" : "", | 20260508 13:37:19.945 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:19.945 | INFO | "malloc" : " 14 GB", | 20260508 13:37:19.945 | INFO | "pinned" : " 29 GB" | 20260508 13:37:19.945 | INFO | } | 20260508 13:37:19.945 | INFO | Perf report for the last 5.13808 seconds: | 20260508 13:37:19.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:19.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 13:37:19.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:37:19.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:19.945 | INFO | Uploaded 36GiB to GPU in 2.48344s (14.496GiB/s) | 20260508 13:37:19.945 | INFO | Downloaded 34GiB to GPU in 2.47536s (13.7354GiB/s) | 20260508 13:37:25.083 | INFO | Analyzing chunk frames = [8519168,8519680) | 20260508 13:37:25.083 | INFO | MemInfo: { | 20260508 13:37:25.083 | INFO | "gpu" : " 17 GB", | 20260508 13:37:25.083 | INFO | "huge" : "", | 20260508 13:37:25.083 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:25.083 | INFO | "malloc" : " 14 GB", | 20260508 13:37:25.083 | INFO | "pinned" : " 29 GB" | 20260508 13:37:25.083 | INFO | } | 20260508 13:37:25.083 | INFO | Perf report for the last 5.13825 seconds: | 20260508 13:37:25.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:25.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260508 13:37:25.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | PRE_HQ : 23.4%, 1.0, 12.1ms, 0 | 20260508 13:37:25.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:25.083 | INFO | Uploaded 36GiB to GPU in 2.33183s (15.4385GiB/s) | 20260508 13:37:25.083 | INFO | Downloaded 34GiB to GPU in 2.43401s (13.9687GiB/s) | 20260508 13:37:25.084 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:37:25.084 | INFO | Total mem usage: 16.7 GiB | 20260508 13:37:25.084 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:37:25.084 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:37:25.084 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:37:25.084 | INFO | High Water Breakdown: | 20260508 13:37:25.084 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:37:25.084 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:37:25.084 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:37:25.084 | INFO | ------------------------------------------ | 20260508 13:37:25.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:37:25.084 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:37:25.084 | INFO | Total mem usage: 26.5 GiB | 20260508 13:37:25.084 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:37:25.084 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:37:25.084 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:37:25.084 | INFO | High Water Breakdown: | 20260508 13:37:25.084 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:37:25.084 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:37:25.084 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:37:25.084 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:37:25.084 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:37:25.084 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:37:25.084 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:37:25.084 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:37:25.084 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:37:25.084 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:37:25.084 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:37:25.084 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:37:25.084 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:37:25.084 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:37:25.084 | INFO | ------------------------------------------ | 20260508 13:37:25.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:37:25.084 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:37:25.084 | INFO | Total mem usage: 27.8 GiB | 20260508 13:37:25.084 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:37:25.084 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:37:25.084 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:37:25.084 | INFO | High Water Breakdown: | 20260508 13:37:25.084 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:37:25.084 | INFO | ------------------------------------------ | 20260508 13:37:25.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:37:25.084 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:37:25.084 | INFO | Total mem usage: 15.7 GiB | 20260508 13:37:25.084 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:37:25.084 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:37:25.084 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:37:25.084 | INFO | High Water Breakdown: | 20260508 13:37:25.084 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:37:25.084 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:37:25.084 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:37:25.084 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:37:25.084 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:37:25.084 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:37:25.084 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:37:25.084 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:37:25.084 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:37:25.084 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:37:25.084 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:37:25.084 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:37:25.084 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:37:25.084 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:37:25.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:37:25.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:37:25.084 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:37:25.084 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:37:25.084 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:37:25.084 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:37:25.084 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:37:25.085 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:37:25.085 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:37:25.085 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:37:25.085 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:37:25.085 | INFO | ------------------------------------------ | 20260508 13:37:25.085 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:37:25.085 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:37:25.085 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:37:25.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:37:25.085 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.52e+01, 1.40e+01 / 5.63e-01 | 20260508 13:37:25.085 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.03e-01, 1.09e-02 / 5.06e-03 | 20260508 13:37:25.085 | INFO | PerfReport: 12.25%, Baseline -- 4.01e+00 / 4.42e+00, 6.02e-01 / 2.04e-01 | 20260508 13:37:25.085 | INFO | PerfReport: 3.42%, Histogram -- 1.12e+00 / 1.42e+00, 2.12e-01 / 1.46e-01 | 20260508 13:37:25.085 | INFO | PerfReport: 15.97%, DME -- 4.19e+01 / 4.53e+01, 3.73e+00 / 1.16e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.88e+01 / 2.01e+01, 1.89e+00 / 1.37e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.30e+00 / 2.41e+00, 4.02e-01 / 2.70e-01 | 20260508 13:37:25.085 | INFO | PerfReport: 2.91%, Metrics -- 9.53e-01 / 1.04e+00, 2.35e-01 / 9.56e-02 | 20260508 13:37:25.085 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.17e-01, 4.63e-02 / 2.70e-02 | 20260508 13:37:25.085 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:37:25.085 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:37:25.085 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:37:25.085 | INFO | PerfReport: Full Execution -- 1.04e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 24.31%, Upload -- 2.54e+01 / 0.00e+00, 7.81e+00 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 24.76%, Download -- 2.59e+01 / 0.00e+00, 8.11e+00 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 31.37%, Compute -- 3.28e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 2.19%, LockUpload -- 2.29e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 2.50%, LockDownload -- 2.61e+00 / 0.00e+00, 3.98e+00 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 14.86%, LockCompute -- 1.55e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260508 13:37:25.085 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260508 13:37:25.181 | INFO | Starting a round of Baz IO | 20260508 13:37:28.598 | INFO | Populating RTMetrics report | 20260508 13:37:28.598 | INFO | RTMetrics report populated | 20260508 13:37:28.598 | INFO | Opening tmp RTMetrics json file | 20260508 13:37:28.608 | INFO | Serializing RTMetrics to json | 20260508 13:37:28.610 | INFO | Serialization done, swapping tmp file now... | 20260508 13:37:28.610 | INFO | RTMetrics done | 20260508 13:37:28.666 | INFO | Finished a round of Baz IO: Wrote 1.383 GiB in 3.485 seconds | 20260508 13:37:28.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:37:28.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138299% / 0% / 7.15256e-05% | 20260508 13:37:28.666 | INFO | Baz IO overall: | 20260508 13:37:28.666 | INFO | BytesWritten : 3626851576557 | 20260508 13:37:28.666 | INFO | NumEvents : 1738684807410 | 20260508 13:37:30.222 | INFO | Analyzing chunk frames = [8519680,8520192) | 20260508 13:37:30.222 | INFO | MemInfo: { | 20260508 13:37:30.222 | INFO | "gpu" : " 17 GB", | 20260508 13:37:30.222 | INFO | "huge" : "", | 20260508 13:37:30.222 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:30.222 | INFO | "malloc" : " 14 GB", | 20260508 13:37:30.222 | INFO | "pinned" : " 29 GB" | 20260508 13:37:30.222 | INFO | } | 20260508 13:37:30.222 | INFO | Perf report for the last 5.13914 seconds: | 20260508 13:37:30.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:30.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:30.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:30.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:30.222 | INFO | ANALYSIS : 67.5%, 3.0, 105.6ms, 0 | 20260508 13:37:30.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:30.222 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:37:30.222 | INFO | PRE_HQ : 65.3%, 1.6, 53.7ms, 0 | 20260508 13:37:30.222 | INFO | BAZWRITER : 67.1%, 2.4, 82.9ms, 0 | 20260508 13:37:30.222 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:37:30.223 | INFO | Uploaded 36GiB to GPU in 2.63506s (13.662GiB/s) | 20260508 13:37:30.223 | INFO | Downloaded 38GiB to GPU in 2.7204s (13.9685GiB/s) | 20260508 13:37:35.361 | INFO | Analyzing chunk frames = [8520192,8520704) | 20260508 13:37:35.361 | INFO | MemInfo: { | 20260508 13:37:35.361 | INFO | "gpu" : " 17 GB", | 20260508 13:37:35.361 | INFO | "huge" : "", | 20260508 13:37:35.361 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:35.361 | INFO | "malloc" : " 14 GB", | 20260508 13:37:35.361 | INFO | "pinned" : " 29 GB" | 20260508 13:37:35.361 | INFO | } | 20260508 13:37:35.361 | INFO | Perf report for the last 5.13896 seconds: | 20260508 13:37:35.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:35.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.361 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:37:35.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.361 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260508 13:37:35.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:35.362 | INFO | Uploaded 36GiB to GPU in 2.47454s (14.5482GiB/s) | 20260508 13:37:35.362 | INFO | Downloaded 34GiB to GPU in 2.46041s (13.8189GiB/s) | 20260508 13:37:40.508 | INFO | Analyzing chunk frames = [8520704,8521216) | 20260508 13:37:40.508 | INFO | MemInfo: { | 20260508 13:37:40.508 | INFO | "gpu" : " 17 GB", | 20260508 13:37:40.508 | INFO | "huge" : "", | 20260508 13:37:40.508 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:40.508 | INFO | "malloc" : " 14 GB", | 20260508 13:37:40.508 | INFO | "pinned" : " 29 GB" | 20260508 13:37:40.509 | INFO | } | 20260508 13:37:40.509 | INFO | Perf report for the last 5.14736 seconds: | 20260508 13:37:40.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:40.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:37:40.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:37:40.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:40.509 | INFO | Uploaded 36GiB to GPU in 2.51458s (14.3165GiB/s) | 20260508 13:37:40.509 | INFO | Downloaded 34GiB to GPU in 2.44321s (13.9161GiB/s) | 20260508 13:37:45.643 | INFO | Analyzing chunk frames = [8521216,8521728) | 20260508 13:37:45.643 | INFO | MemInfo: { | 20260508 13:37:45.643 | INFO | "gpu" : " 17 GB", | 20260508 13:37:45.643 | INFO | "huge" : "", | 20260508 13:37:45.643 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:45.643 | INFO | "malloc" : " 14 GB", | 20260508 13:37:45.643 | INFO | "pinned" : " 29 GB" | 20260508 13:37:45.643 | INFO | } | 20260508 13:37:45.643 | INFO | Perf report for the last 5.13474 seconds: | 20260508 13:37:45.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:45.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | ANALYSIS : 65.5%, 3.0, 101.5ms, 0 | 20260508 13:37:45.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:37:45.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:45.644 | INFO | Uploaded 36GiB to GPU in 2.42954s (14.8176GiB/s) | 20260508 13:37:45.644 | INFO | Downloaded 34GiB to GPU in 2.46611s (13.7869GiB/s) | 20260508 13:37:50.776 | INFO | Analyzing chunk frames = [8521728,8522240) | 20260508 13:37:50.776 | INFO | MemInfo: { | 20260508 13:37:50.776 | INFO | "gpu" : " 17 GB", | 20260508 13:37:50.776 | INFO | "huge" : "", | 20260508 13:37:50.776 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:50.776 | INFO | "malloc" : " 14 GB", | 20260508 13:37:50.776 | INFO | "pinned" : " 29 GB" | 20260508 13:37:50.776 | INFO | } | 20260508 13:37:50.777 | INFO | Perf report for the last 5.13371 seconds: | 20260508 13:37:50.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:50.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | ANALYSIS : 66.6%, 3.0, 103.8ms, 0 | 20260508 13:37:50.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:37:50.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:50.777 | INFO | Uploaded 36GiB to GPU in 2.50968s (14.3445GiB/s) | 20260508 13:37:50.777 | INFO | Downloaded 34GiB to GPU in 2.51828s (13.5013GiB/s) | 20260508 13:37:55.923 | INFO | Analyzing chunk frames = [8522240,8522752) | 20260508 13:37:55.923 | INFO | MemInfo: { | 20260508 13:37:55.923 | INFO | "gpu" : " 17 GB", | 20260508 13:37:55.923 | INFO | "huge" : "", | 20260508 13:37:55.923 | INFO | "huge_shared" : " 29 GB", | 20260508 13:37:55.924 | INFO | "malloc" : " 14 GB", | 20260508 13:37:55.924 | INFO | "pinned" : " 29 GB" | 20260508 13:37:55.924 | INFO | } | 20260508 13:37:55.924 | INFO | Perf report for the last 5.14651 seconds: | 20260508 13:37:55.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:37:55.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:37:55.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:37:55.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:37:55.924 | INFO | Uploaded 36GiB to GPU in 2.42654s (14.8359GiB/s) | 20260508 13:37:55.924 | INFO | Downloaded 34GiB to GPU in 2.43081s (13.9871GiB/s) | 20260508 13:38:01.061 | INFO | Analyzing chunk frames = [8522752,8523264) | 20260508 13:38:01.061 | INFO | MemInfo: { | 20260508 13:38:01.061 | INFO | "gpu" : " 17 GB", | 20260508 13:38:01.061 | INFO | "huge" : "", | 20260508 13:38:01.061 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:01.061 | INFO | "malloc" : " 14 GB", | 20260508 13:38:01.061 | INFO | "pinned" : " 29 GB" | 20260508 13:38:01.061 | INFO | } | 20260508 13:38:01.061 | INFO | Perf report for the last 5.13755 seconds: | 20260508 13:38:01.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:01.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 13:38:01.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:38:01.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:01.061 | INFO | Uploaded 36GiB to GPU in 2.36058s (15.2505GiB/s) | 20260508 13:38:01.061 | INFO | Downloaded 34GiB to GPU in 2.40441s (14.1407GiB/s) | 20260508 13:38:06.198 | INFO | Analyzing chunk frames = [8523264,8523776) | 20260508 13:38:06.198 | INFO | MemInfo: { | 20260508 13:38:06.198 | INFO | "gpu" : " 17 GB", | 20260508 13:38:06.198 | INFO | "huge" : "", | 20260508 13:38:06.198 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:06.198 | INFO | "malloc" : " 14 GB", | 20260508 13:38:06.198 | INFO | "pinned" : " 29 GB" | 20260508 13:38:06.198 | INFO | } | 20260508 13:38:06.198 | INFO | Perf report for the last 5.1369 seconds: | 20260508 13:38:06.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:06.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.198 | INFO | ANALYSIS : 65.7%, 3.0, 102.6ms, 0 | 20260508 13:38:06.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.198 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:38:06.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:06.199 | INFO | Uploaded 36GiB to GPU in 2.50953s (14.3453GiB/s) | 20260508 13:38:06.199 | INFO | Downloaded 34GiB to GPU in 2.47357s (13.7453GiB/s) | 20260508 13:38:06.295 | INFO | Starting a round of Baz IO | 20260508 13:38:09.697 | INFO | Populating RTMetrics report | 20260508 13:38:09.697 | INFO | RTMetrics report populated | 20260508 13:38:09.697 | INFO | Opening tmp RTMetrics json file | 20260508 13:38:09.705 | INFO | Serializing RTMetrics to json | 20260508 13:38:09.706 | INFO | Serialization done, swapping tmp file now... | 20260508 13:38:09.707 | INFO | RTMetrics done | 20260508 13:38:09.765 | INFO | Finished a round of Baz IO: Wrote 1.382 GiB in 3.469 seconds | 20260508 13:38:09.765 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:38:09.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138017% / 0% / 7.54992e-05% | 20260508 13:38:09.765 | INFO | Baz IO overall: | 20260508 13:38:09.765 | INFO | BytesWritten : 3628335868825 | 20260508 13:38:09.765 | INFO | NumEvents : 1739379028564 | 20260508 13:38:11.338 | INFO | Analyzing chunk frames = [8523776,8524288) | 20260508 13:38:11.338 | INFO | MemInfo: { | 20260508 13:38:11.338 | INFO | "gpu" : " 17 GB", | 20260508 13:38:11.338 | INFO | "huge" : "", | 20260508 13:38:11.338 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:11.338 | INFO | "malloc" : " 14 GB", | 20260508 13:38:11.338 | INFO | "pinned" : " 29 GB" | 20260508 13:38:11.338 | INFO | } | 20260508 13:38:11.338 | INFO | Perf report for the last 5.13989 seconds: | 20260508 13:38:11.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:11.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:11.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:11.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:11.338 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:38:11.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:11.338 | INFO | RT_METRICS : 40.2%, 1.0, 21.1ms, 0 | 20260508 13:38:11.338 | INFO | PRE_HQ : 65.7%, 1.7, 57.5ms, 0 | 20260508 13:38:11.338 | INFO | BAZWRITER : 66.6%, 2.5, 86.5ms, 0 | 20260508 13:38:11.338 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:38:11.338 | INFO | Uploaded 36GiB to GPU in 2.63179s (13.6789GiB/s) | 20260508 13:38:11.339 | INFO | Downloaded 38GiB to GPU in 2.72284s (13.956GiB/s) | 20260508 13:38:16.479 | INFO | Analyzing chunk frames = [8524288,8524800) | 20260508 13:38:16.479 | INFO | MemInfo: { | 20260508 13:38:16.479 | INFO | "gpu" : " 17 GB", | 20260508 13:38:16.479 | INFO | "huge" : "", | 20260508 13:38:16.479 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:16.479 | INFO | "malloc" : " 14 GB", | 20260508 13:38:16.479 | INFO | "pinned" : " 29 GB" | 20260508 13:38:16.479 | INFO | } | 20260508 13:38:16.479 | INFO | Perf report for the last 5.14121 seconds: | 20260508 13:38:16.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:16.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.479 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:38:16.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.479 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:38:16.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:16.480 | INFO | Uploaded 36GiB to GPU in 2.49211s (14.4456GiB/s) | 20260508 13:38:16.480 | INFO | Downloaded 34GiB to GPU in 2.46501s (13.7931GiB/s) | 20260508 13:38:21.620 | INFO | Analyzing chunk frames = [8524800,8525312) | 20260508 13:38:21.620 | INFO | MemInfo: { | 20260508 13:38:21.620 | INFO | "gpu" : " 17 GB", | 20260508 13:38:21.620 | INFO | "huge" : "", | 20260508 13:38:21.620 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:21.620 | INFO | "malloc" : " 14 GB", | 20260508 13:38:21.620 | INFO | "pinned" : " 29 GB" | 20260508 13:38:21.620 | INFO | } | 20260508 13:38:21.620 | INFO | Perf report for the last 5.14085 seconds: | 20260508 13:38:21.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:21.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.620 | INFO | ANALYSIS : 65.8%, 3.0, 102.3ms, 0 | 20260508 13:38:21.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.620 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:38:21.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:21.621 | INFO | Uploaded 36GiB to GPU in 2.47199s (14.5632GiB/s) | 20260508 13:38:21.621 | INFO | Downloaded 34GiB to GPU in 2.44756s (13.8914GiB/s) | 20260508 13:38:26.759 | INFO | Analyzing chunk frames = [8525312,8525824) | 20260508 13:38:26.759 | INFO | MemInfo: { | 20260508 13:38:26.759 | INFO | "gpu" : " 17 GB", | 20260508 13:38:26.759 | INFO | "huge" : "", | 20260508 13:38:26.759 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:26.759 | INFO | "malloc" : " 14 GB", | 20260508 13:38:26.759 | INFO | "pinned" : " 29 GB" | 20260508 13:38:26.759 | INFO | } | 20260508 13:38:26.759 | INFO | Perf report for the last 5.13933 seconds: | 20260508 13:38:26.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:26.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.759 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 13:38:26.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.760 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:38:26.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:26.760 | INFO | Uploaded 36GiB to GPU in 2.54033s (14.1714GiB/s) | 20260508 13:38:26.760 | INFO | Downloaded 34GiB to GPU in 2.45327s (13.8591GiB/s) | 20260508 13:38:31.896 | INFO | Analyzing chunk frames = [8525824,8526336) | 20260508 13:38:31.896 | INFO | MemInfo: { | 20260508 13:38:31.896 | INFO | "gpu" : " 17 GB", | 20260508 13:38:31.896 | INFO | "huge" : "", | 20260508 13:38:31.896 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:31.896 | INFO | "malloc" : " 14 GB", | 20260508 13:38:31.896 | INFO | "pinned" : " 29 GB" | 20260508 13:38:31.896 | INFO | } | 20260508 13:38:31.897 | INFO | Perf report for the last 5.13709 seconds: | 20260508 13:38:31.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:31.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 13:38:31.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:38:31.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:31.897 | INFO | Uploaded 36GiB to GPU in 2.51797s (14.2973GiB/s) | 20260508 13:38:31.897 | INFO | Downloaded 34GiB to GPU in 2.53318s (13.4219GiB/s) | 20260508 13:38:37.028 | INFO | Analyzing chunk frames = [8526336,8526848) | 20260508 13:38:37.028 | INFO | MemInfo: { | 20260508 13:38:37.028 | INFO | "gpu" : " 17 GB", | 20260508 13:38:37.028 | INFO | "huge" : "", | 20260508 13:38:37.028 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:37.028 | INFO | "malloc" : " 14 GB", | 20260508 13:38:37.028 | INFO | "pinned" : " 29 GB" | 20260508 13:38:37.028 | INFO | } | 20260508 13:38:37.028 | INFO | Perf report for the last 5.13151 seconds: | 20260508 13:38:37.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:37.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.028 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:38:37.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.028 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:38:37.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:37.029 | INFO | Uploaded 36GiB to GPU in 2.44444s (14.7273GiB/s) | 20260508 13:38:37.029 | INFO | Downloaded 34GiB to GPU in 2.44054s (13.9313GiB/s) | 20260508 13:38:42.178 | INFO | Analyzing chunk frames = [8526848,8527360) | 20260508 13:38:42.178 | INFO | MemInfo: { | 20260508 13:38:42.178 | INFO | "gpu" : " 17 GB", | 20260508 13:38:42.178 | INFO | "huge" : "", | 20260508 13:38:42.178 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:42.178 | INFO | "malloc" : " 14 GB", | 20260508 13:38:42.178 | INFO | "pinned" : " 29 GB" | 20260508 13:38:42.178 | INFO | } | 20260508 13:38:42.178 | INFO | Perf report for the last 5.14985 seconds: | 20260508 13:38:42.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:42.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.178 | INFO | ANALYSIS : 65.3%, 3.0, 102.0ms, 0 | 20260508 13:38:42.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.178 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:38:42.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:42.179 | INFO | Uploaded 36GiB to GPU in 2.48352s (14.4955GiB/s) | 20260508 13:38:42.179 | INFO | Downloaded 34GiB to GPU in 2.43594s (13.9577GiB/s) | 20260508 13:38:47.312 | INFO | Analyzing chunk frames = [8527360,8527872) | 20260508 13:38:47.312 | INFO | MemInfo: { | 20260508 13:38:47.312 | INFO | "gpu" : " 17 GB", | 20260508 13:38:47.312 | INFO | "huge" : "", | 20260508 13:38:47.312 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:47.312 | INFO | "malloc" : " 14 GB", | 20260508 13:38:47.312 | INFO | "pinned" : " 29 GB" | 20260508 13:38:47.312 | INFO | } | 20260508 13:38:47.312 | INFO | Perf report for the last 5.13433 seconds: | 20260508 13:38:47.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:47.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260508 13:38:47.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260508 13:38:47.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:47.312 | INFO | Uploaded 36GiB to GPU in 2.33525s (15.4159GiB/s) | 20260508 13:38:47.313 | INFO | Downloaded 34GiB to GPU in 2.40735s (14.1234GiB/s) | 20260508 13:38:47.313 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:38:47.313 | INFO | Total mem usage: 16.7 GiB | 20260508 13:38:47.313 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:38:47.313 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:38:47.313 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:38:47.313 | INFO | High Water Breakdown: | 20260508 13:38:47.313 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:38:47.313 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:38:47.313 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:38:47.313 | INFO | ------------------------------------------ | 20260508 13:38:47.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:38:47.313 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:38:47.313 | INFO | Total mem usage: 26.5 GiB | 20260508 13:38:47.313 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:38:47.313 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:38:47.313 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:38:47.313 | INFO | High Water Breakdown: | 20260508 13:38:47.313 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:38:47.313 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:38:47.313 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:38:47.313 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:38:47.313 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:38:47.313 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:38:47.313 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:38:47.313 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:38:47.313 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:38:47.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:38:47.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:38:47.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:38:47.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:38:47.313 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:38:47.313 | INFO | ------------------------------------------ | 20260508 13:38:47.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:38:47.313 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:38:47.313 | INFO | Total mem usage: 27.8 GiB | 20260508 13:38:47.313 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:38:47.313 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:38:47.313 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:38:47.313 | INFO | High Water Breakdown: | 20260508 13:38:47.313 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:38:47.313 | INFO | ------------------------------------------ | 20260508 13:38:47.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:38:47.313 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:38:47.313 | INFO | Total mem usage: 15.7 GiB | 20260508 13:38:47.313 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:38:47.313 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:38:47.313 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:38:47.313 | INFO | High Water Breakdown: | 20260508 13:38:47.313 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:38:47.313 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:38:47.313 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:38:47.313 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:38:47.313 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:38:47.313 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:38:47.313 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:38:47.313 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:38:47.313 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:38:47.313 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:38:47.313 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:38:47.313 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:38:47.313 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:38:47.313 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:38:47.314 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:38:47.314 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:38:47.314 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:38:47.314 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:38:47.314 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:38:47.314 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:38:47.314 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:38:47.314 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:38:47.314 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:38:47.314 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:38:47.314 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:38:47.314 | INFO | ------------------------------------------ | 20260508 13:38:47.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:38:47.314 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:38:47.314 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:38:47.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:38:47.314 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.39e+01 / 5.61e-01 | 20260508 13:38:47.314 | INFO | PerfReport: 0.30%, AggregateStats -- 9.71e-02 / 1.04e-01, 1.17e-02 / 6.04e-03 | 20260508 13:38:47.314 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.46e+00, 5.97e-01 / 2.05e-01 | 20260508 13:38:47.314 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.41e+00, 2.09e-01 / 1.30e-01 | 20260508 13:38:47.314 | INFO | PerfReport: 15.99%, DME -- 4.17e+01 / 4.47e+01, 3.56e+00 / 8.35e-01 | 20260508 13:38:47.314 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.86e+01 / 2.09e+01, 1.81e+00 / 1.12e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.31e+00 / 2.39e+00, 4.06e-01 / 2.77e-01 | 20260508 13:38:47.314 | INFO | PerfReport: 2.92%, Metrics -- 9.52e-01 / 1.12e+00, 2.38e-01 / 1.23e-01 | 20260508 13:38:47.314 | INFO | PerfReport: 0.89%, PulseSerializing -- 2.88e-01 / 3.14e-01, 4.72e-02 / 2.74e-02 | 20260508 13:38:47.314 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:38:47.314 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:38:47.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:38:47.314 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 24.69%, Upload -- 2.59e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 24.72%, Download -- 2.59e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 31.07%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 2.30%, LockUpload -- 2.41e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 2.42%, LockDownload -- 2.54e+00 / 0.00e+00, 3.77e+00 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 14.77%, LockCompute -- 1.55e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:38:47.314 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260508 13:38:47.409 | INFO | Starting a round of Baz IO | 20260508 13:38:50.825 | INFO | Populating RTMetrics report | 20260508 13:38:50.826 | INFO | RTMetrics report populated | 20260508 13:38:50.826 | INFO | Opening tmp RTMetrics json file | 20260508 13:38:50.835 | INFO | Serializing RTMetrics to json | 20260508 13:38:50.836 | INFO | Serialization done, swapping tmp file now... | 20260508 13:38:50.836 | INFO | RTMetrics done | 20260508 13:38:50.890 | INFO | Finished a round of Baz IO: Wrote 1.381 GiB in 3.481 seconds | 20260508 13:38:50.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:38:50.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136753% / 0% / 5.5631e-05% | 20260508 13:38:50.890 | INFO | Baz IO overall: | 20260508 13:38:50.891 | INFO | BytesWritten : 3629819222773 | 20260508 13:38:50.891 | INFO | NumEvents : 1740072767362 | 20260508 13:38:52.454 | INFO | Analyzing chunk frames = [8527872,8528384) | 20260508 13:38:52.454 | INFO | MemInfo: { | 20260508 13:38:52.454 | INFO | "gpu" : " 17 GB", | 20260508 13:38:52.454 | INFO | "huge" : "", | 20260508 13:38:52.454 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:52.454 | INFO | "malloc" : " 14 GB", | 20260508 13:38:52.454 | INFO | "pinned" : " 29 GB" | 20260508 13:38:52.454 | INFO | } | 20260508 13:38:52.454 | INFO | Perf report for the last 5.14164 seconds: | 20260508 13:38:52.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:52.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:52.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:52.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:52.454 | INFO | ANALYSIS : 67.7%, 3.0, 105.7ms, 0 | 20260508 13:38:52.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:52.454 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:38:52.454 | INFO | PRE_HQ : 65.5%, 1.6, 53.6ms, 0 | 20260508 13:38:52.454 | INFO | BAZWRITER : 67.0%, 2.4, 83.6ms, 0 | 20260508 13:38:52.454 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:38:52.455 | INFO | Uploaded 36GiB to GPU in 2.62617s (13.7082GiB/s) | 20260508 13:38:52.455 | INFO | Downloaded 38GiB to GPU in 2.72089s (13.966GiB/s) | 20260508 13:38:57.594 | INFO | Analyzing chunk frames = [8528384,8528896) | 20260508 13:38:57.594 | INFO | MemInfo: { | 20260508 13:38:57.594 | INFO | "gpu" : " 17 GB", | 20260508 13:38:57.594 | INFO | "huge" : "", | 20260508 13:38:57.594 | INFO | "huge_shared" : " 29 GB", | 20260508 13:38:57.594 | INFO | "malloc" : " 14 GB", | 20260508 13:38:57.594 | INFO | "pinned" : " 29 GB" | 20260508 13:38:57.594 | INFO | } | 20260508 13:38:57.594 | INFO | Perf report for the last 5.14003 seconds: | 20260508 13:38:57.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:38:57.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:38:57.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:38:57.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:38:57.594 | INFO | Uploaded 36GiB to GPU in 2.51475s (14.3156GiB/s) | 20260508 13:38:57.594 | INFO | Downloaded 34GiB to GPU in 2.47845s (13.7182GiB/s) | 20260508 13:39:02.734 | INFO | Analyzing chunk frames = [8528896,8529408) | 20260508 13:39:02.734 | INFO | MemInfo: { | 20260508 13:39:02.734 | INFO | "gpu" : " 17 GB", | 20260508 13:39:02.734 | INFO | "huge" : "", | 20260508 13:39:02.734 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:02.734 | INFO | "malloc" : " 14 GB", | 20260508 13:39:02.734 | INFO | "pinned" : " 29 GB" | 20260508 13:39:02.734 | INFO | } | 20260508 13:39:02.734 | INFO | Perf report for the last 5.1405 seconds: | 20260508 13:39:02.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:02.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.735 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 13:39:02.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.735 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:39:02.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:02.735 | INFO | Uploaded 36GiB to GPU in 2.49879s (14.407GiB/s) | 20260508 13:39:02.735 | INFO | Downloaded 34GiB to GPU in 2.44682s (13.8956GiB/s) | 20260508 13:39:07.870 | INFO | Analyzing chunk frames = [8529408,8529920) | 20260508 13:39:07.870 | INFO | MemInfo: { | 20260508 13:39:07.870 | INFO | "gpu" : " 17 GB", | 20260508 13:39:07.870 | INFO | "huge" : "", | 20260508 13:39:07.870 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:07.870 | INFO | "malloc" : " 14 GB", | 20260508 13:39:07.870 | INFO | "pinned" : " 29 GB" | 20260508 13:39:07.870 | INFO | } | 20260508 13:39:07.870 | INFO | Perf report for the last 5.13586 seconds: | 20260508 13:39:07.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:07.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.870 | INFO | ANALYSIS : 65.5%, 3.0, 101.5ms, 0 | 20260508 13:39:07.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.870 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:39:07.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:07.871 | INFO | Uploaded 36GiB to GPU in 2.43312s (14.7958GiB/s) | 20260508 13:39:07.871 | INFO | Downloaded 34GiB to GPU in 2.46771s (13.778GiB/s) | 20260508 13:39:13.011 | INFO | Analyzing chunk frames = [8529920,8530432) | 20260508 13:39:13.011 | INFO | MemInfo: { | 20260508 13:39:13.011 | INFO | "gpu" : " 17 GB", | 20260508 13:39:13.011 | INFO | "huge" : "", | 20260508 13:39:13.011 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:13.011 | INFO | "malloc" : " 14 GB", | 20260508 13:39:13.011 | INFO | "pinned" : " 29 GB" | 20260508 13:39:13.011 | INFO | } | 20260508 13:39:13.011 | INFO | Perf report for the last 5.14079 seconds: | 20260508 13:39:13.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:13.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.011 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 13:39:13.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.011 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:39:13.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:13.012 | INFO | Uploaded 36GiB to GPU in 2.43984s (14.7551GiB/s) | 20260508 13:39:13.012 | INFO | Downloaded 34GiB to GPU in 2.51272s (13.5311GiB/s) | 20260508 13:39:18.154 | INFO | Analyzing chunk frames = [8530432,8530944) | 20260508 13:39:18.154 | INFO | MemInfo: { | 20260508 13:39:18.154 | INFO | "gpu" : " 17 GB", | 20260508 13:39:18.155 | INFO | "huge" : "", | 20260508 13:39:18.155 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:18.155 | INFO | "malloc" : " 14 GB", | 20260508 13:39:18.155 | INFO | "pinned" : " 29 GB" | 20260508 13:39:18.155 | INFO | } | 20260508 13:39:18.155 | INFO | Perf report for the last 5.14368 seconds: | 20260508 13:39:18.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:18.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | ANALYSIS : 65.4%, 3.0, 102.0ms, 0 | 20260508 13:39:18.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:39:18.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:18.155 | INFO | Uploaded 36GiB to GPU in 2.46987s (14.5757GiB/s) | 20260508 13:39:18.155 | INFO | Downloaded 34GiB to GPU in 2.44852s (13.8859GiB/s) | 20260508 13:39:23.287 | INFO | Analyzing chunk frames = [8530944,8531456) | 20260508 13:39:23.287 | INFO | MemInfo: { | 20260508 13:39:23.287 | INFO | "gpu" : " 17 GB", | 20260508 13:39:23.287 | INFO | "huge" : "", | 20260508 13:39:23.287 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:23.287 | INFO | "malloc" : " 14 GB", | 20260508 13:39:23.287 | INFO | "pinned" : " 29 GB" | 20260508 13:39:23.287 | INFO | } | 20260508 13:39:23.287 | INFO | Perf report for the last 5.13259 seconds: | 20260508 13:39:23.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:23.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.287 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:39:23.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.288 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:39:23.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:23.288 | INFO | Uploaded 36GiB to GPU in 2.3152s (15.5494GiB/s) | 20260508 13:39:23.288 | INFO | Downloaded 34GiB to GPU in 2.39839s (14.1762GiB/s) | 20260508 13:39:28.431 | INFO | Analyzing chunk frames = [8531456,8531968) | 20260508 13:39:28.431 | INFO | MemInfo: { | 20260508 13:39:28.431 | INFO | "gpu" : " 17 GB", | 20260508 13:39:28.431 | INFO | "huge" : "", | 20260508 13:39:28.431 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:28.431 | INFO | "malloc" : " 14 GB", | 20260508 13:39:28.431 | INFO | "pinned" : " 29 GB" | 20260508 13:39:28.431 | INFO | } | 20260508 13:39:28.431 | INFO | Perf report for the last 5.14376 seconds: | 20260508 13:39:28.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:28.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 13:39:28.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:39:28.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:28.431 | INFO | Uploaded 36GiB to GPU in 2.46701s (14.5925GiB/s) | 20260508 13:39:28.431 | INFO | Downloaded 34GiB to GPU in 2.45154s (13.8689GiB/s) | 20260508 13:39:28.528 | INFO | Starting a round of Baz IO | 20260508 13:39:31.921 | INFO | Populating RTMetrics report | 20260508 13:39:31.921 | INFO | RTMetrics report populated | 20260508 13:39:31.921 | INFO | Opening tmp RTMetrics json file | 20260508 13:39:31.930 | INFO | Serializing RTMetrics to json | 20260508 13:39:31.931 | INFO | Serialization done, swapping tmp file now... | 20260508 13:39:31.931 | INFO | RTMetrics done | 20260508 13:39:31.989 | INFO | Finished a round of Baz IO: Wrote 1.383 GiB in 3.46 seconds | 20260508 13:39:31.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:39:31.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138048% / 3.97364e-06% / 8.74201e-05% | 20260508 13:39:31.989 | INFO | Baz IO overall: | 20260508 13:39:31.989 | INFO | BytesWritten : 3631304441057 | 20260508 13:39:31.989 | INFO | NumEvents : 1740768094342 | 20260508 13:39:33.573 | INFO | Analyzing chunk frames = [8531968,8532480) | 20260508 13:39:33.573 | INFO | MemInfo: { | 20260508 13:39:33.573 | INFO | "gpu" : " 17 GB", | 20260508 13:39:33.573 | INFO | "huge" : "", | 20260508 13:39:33.573 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:33.573 | INFO | "malloc" : " 14 GB", | 20260508 13:39:33.573 | INFO | "pinned" : " 29 GB" | 20260508 13:39:33.573 | INFO | } | 20260508 13:39:33.573 | INFO | Perf report for the last 5.14198 seconds: | 20260508 13:39:33.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:33.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:33.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:33.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:33.573 | INFO | ANALYSIS : 67.2%, 3.0, 105.1ms, 0 | 20260508 13:39:33.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:33.573 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:39:33.573 | INFO | PRE_HQ : 65.1%, 1.6, 53.5ms, 0 | 20260508 13:39:33.573 | INFO | BAZWRITER : 66.5%, 2.4, 82.9ms, 0 | 20260508 13:39:33.573 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:39:33.574 | INFO | Uploaded 36GiB to GPU in 2.63934s (13.6398GiB/s) | 20260508 13:39:33.574 | INFO | Downloaded 38GiB to GPU in 2.71057s (14.0192GiB/s) | 20260508 13:39:38.706 | INFO | Analyzing chunk frames = [8532480,8532992) | 20260508 13:39:38.706 | INFO | MemInfo: { | 20260508 13:39:38.706 | INFO | "gpu" : " 17 GB", | 20260508 13:39:38.706 | INFO | "huge" : "", | 20260508 13:39:38.706 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:38.706 | INFO | "malloc" : " 14 GB", | 20260508 13:39:38.706 | INFO | "pinned" : " 29 GB" | 20260508 13:39:38.706 | INFO | } | 20260508 13:39:38.707 | INFO | Perf report for the last 5.13337 seconds: | 20260508 13:39:38.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:38.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | ANALYSIS : 66.5%, 3.0, 103.5ms, 0 | 20260508 13:39:38.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:39:38.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:38.707 | INFO | Uploaded 36GiB to GPU in 2.51416s (14.3189GiB/s) | 20260508 13:39:38.707 | INFO | Downloaded 34GiB to GPU in 2.47667s (13.7281GiB/s) | 20260508 13:39:43.841 | INFO | Analyzing chunk frames = [8532992,8533504) | 20260508 13:39:43.841 | INFO | MemInfo: { | 20260508 13:39:43.841 | INFO | "gpu" : " 17 GB", | 20260508 13:39:43.841 | INFO | "huge" : "", | 20260508 13:39:43.841 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:43.841 | INFO | "malloc" : " 14 GB", | 20260508 13:39:43.841 | INFO | "pinned" : " 29 GB" | 20260508 13:39:43.841 | INFO | } | 20260508 13:39:43.841 | INFO | Perf report for the last 5.1346 seconds: | 20260508 13:39:43.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:43.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 13:39:43.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:39:43.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:43.841 | INFO | Uploaded 36GiB to GPU in 2.51866s (14.2933GiB/s) | 20260508 13:39:43.841 | INFO | Downloaded 34GiB to GPU in 2.44927s (13.8817GiB/s) | 20260508 13:39:48.984 | INFO | Analyzing chunk frames = [8533504,8534016) | 20260508 13:39:48.985 | INFO | MemInfo: { | 20260508 13:39:48.985 | INFO | "gpu" : " 17 GB", | 20260508 13:39:48.985 | INFO | "huge" : "", | 20260508 13:39:48.985 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:48.985 | INFO | "malloc" : " 14 GB", | 20260508 13:39:48.985 | INFO | "pinned" : " 29 GB" | 20260508 13:39:48.985 | INFO | } | 20260508 13:39:48.985 | INFO | Perf report for the last 5.14354 seconds: | 20260508 13:39:48.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:48.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:39:48.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:39:48.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:48.985 | INFO | Uploaded 36GiB to GPU in 2.50848s (14.3513GiB/s) | 20260508 13:39:48.985 | INFO | Downloaded 34GiB to GPU in 2.44653s (13.8972GiB/s) | 20260508 13:39:54.125 | INFO | Analyzing chunk frames = [8534016,8534528) | 20260508 13:39:54.125 | INFO | MemInfo: { | 20260508 13:39:54.125 | INFO | "gpu" : " 17 GB", | 20260508 13:39:54.125 | INFO | "huge" : "", | 20260508 13:39:54.125 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:54.125 | INFO | "malloc" : " 14 GB", | 20260508 13:39:54.125 | INFO | "pinned" : " 29 GB" | 20260508 13:39:54.125 | INFO | } | 20260508 13:39:54.125 | INFO | Perf report for the last 5.14067 seconds: | 20260508 13:39:54.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:54.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.125 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 13:39:54.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.125 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260508 13:39:54.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:54.126 | INFO | Uploaded 36GiB to GPU in 2.42832s (14.8251GiB/s) | 20260508 13:39:54.126 | INFO | Downloaded 34GiB to GPU in 2.5175s (13.5054GiB/s) | 20260508 13:39:59.271 | INFO | Analyzing chunk frames = [8534528,8535040) | 20260508 13:39:59.271 | INFO | MemInfo: { | 20260508 13:39:59.271 | INFO | "gpu" : " 17 GB", | 20260508 13:39:59.271 | INFO | "huge" : "", | 20260508 13:39:59.272 | INFO | "huge_shared" : " 29 GB", | 20260508 13:39:59.272 | INFO | "malloc" : " 14 GB", | 20260508 13:39:59.272 | INFO | "pinned" : " 29 GB" | 20260508 13:39:59.272 | INFO | } | 20260508 13:39:59.272 | INFO | Perf report for the last 5.14629 seconds: | 20260508 13:39:59.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:39:59.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | ANALYSIS : 65.2%, 3.0, 101.7ms, 0 | 20260508 13:39:59.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:39:59.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:39:59.272 | INFO | Uploaded 36GiB to GPU in 2.43785s (14.7671GiB/s) | 20260508 13:39:59.272 | INFO | Downloaded 34GiB to GPU in 2.43611s (13.9567GiB/s) | 20260508 13:40:04.402 | INFO | Analyzing chunk frames = [8535040,8535552) | 20260508 13:40:04.402 | INFO | MemInfo: { | 20260508 13:40:04.402 | INFO | "gpu" : " 17 GB", | 20260508 13:40:04.403 | INFO | "huge" : "", | 20260508 13:40:04.403 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:04.403 | INFO | "malloc" : " 14 GB", | 20260508 13:40:04.403 | INFO | "pinned" : " 29 GB" | 20260508 13:40:04.403 | INFO | } | 20260508 13:40:04.403 | INFO | Perf report for the last 5.13098 seconds: | 20260508 13:40:04.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:04.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | ANALYSIS : 64.7%, 3.0, 100.5ms, 0 | 20260508 13:40:04.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:40:04.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:04.403 | INFO | Uploaded 36GiB to GPU in 2.34786s (15.3331GiB/s) | 20260508 13:40:04.403 | INFO | Downloaded 34GiB to GPU in 2.39672s (14.1861GiB/s) | 20260508 13:40:09.538 | INFO | Analyzing chunk frames = [8535552,8536064) | 20260508 13:40:09.538 | INFO | MemInfo: { | 20260508 13:40:09.538 | INFO | "gpu" : " 17 GB", | 20260508 13:40:09.538 | INFO | "huge" : "", | 20260508 13:40:09.538 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:09.539 | INFO | "malloc" : " 14 GB", | 20260508 13:40:09.539 | INFO | "pinned" : " 29 GB" | 20260508 13:40:09.539 | INFO | } | 20260508 13:40:09.539 | INFO | Perf report for the last 5.13595 seconds: | 20260508 13:40:09.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:09.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | ANALYSIS : 65.0%, 3.0, 101.5ms, 0 | 20260508 13:40:09.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:40:09.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:09.539 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:40:09.539 | INFO | Total mem usage: 16.7 GiB | 20260508 13:40:09.539 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:40:09.539 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:40:09.539 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:40:09.539 | INFO | High Water Breakdown: | 20260508 13:40:09.539 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:40:09.539 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:40:09.539 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:40:09.539 | INFO | ------------------------------------------ | 20260508 13:40:09.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:40:09.539 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:40:09.539 | INFO | Total mem usage: 26.5 GiB | 20260508 13:40:09.539 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:40:09.539 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:40:09.539 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:40:09.539 | INFO | High Water Breakdown: | 20260508 13:40:09.539 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:40:09.539 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:40:09.539 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:40:09.539 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:40:09.539 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:40:09.539 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:40:09.539 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:40:09.539 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:40:09.539 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:40:09.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:40:09.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:40:09.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:40:09.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:40:09.539 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:40:09.539 | INFO | ------------------------------------------ | 20260508 13:40:09.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:40:09.539 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:40:09.539 | INFO | Total mem usage: 27.8 GiB | 20260508 13:40:09.539 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:40:09.539 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:40:09.539 | INFO | Uploaded 36GiB to GPU in 2.37878s (15.1338GiB/s) | 20260508 13:40:09.539 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:40:09.540 | INFO | Downloaded 34GiB to GPU in 2.42366s (14.0284GiB/s) | 20260508 13:40:09.540 | INFO | High Water Breakdown: | 20260508 13:40:09.540 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:40:09.540 | INFO | ------------------------------------------ | 20260508 13:40:09.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:40:09.540 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:40:09.540 | INFO | Total mem usage: 15.7 GiB | 20260508 13:40:09.540 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:40:09.540 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:40:09.540 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:40:09.540 | INFO | High Water Breakdown: | 20260508 13:40:09.540 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:40:09.540 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:40:09.540 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:40:09.540 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:40:09.540 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:40:09.540 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:40:09.540 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:40:09.540 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:40:09.540 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:40:09.540 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:40:09.540 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:40:09.540 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:40:09.540 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:40:09.540 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:40:09.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:40:09.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:40:09.540 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:40:09.540 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:40:09.540 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:40:09.540 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:40:09.540 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:40:09.540 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:40:09.540 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:40:09.540 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:40:09.540 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:40:09.540 | INFO | ------------------------------------------ | 20260508 13:40:09.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:40:09.540 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:40:09.540 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:40:09.540 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:40:09.540 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.55e+01, 1.39e+01 / 4.58e-01 | 20260508 13:40:09.540 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.02e-01, 1.12e-02 / 4.27e-03 | 20260508 13:40:09.540 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.29e+00, 5.91e-01 / 2.09e-01 | 20260508 13:40:09.540 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.37e+00, 2.08e-01 / 1.19e-01 | 20260508 13:40:09.540 | INFO | PerfReport: 15.98%, DME -- 4.17e+01 / 4.54e+01, 3.65e+00 / 1.32e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.82e+00 / 1.17e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.30e+00 / 2.25e+00, 4.03e-01 / 2.46e-01 | 20260508 13:40:09.540 | INFO | PerfReport: 2.92%, Metrics -- 9.51e-01 / 1.04e+00, 2.36e-01 / 6.95e-02 | 20260508 13:40:09.540 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.03e-01, 4.57e-02 / 1.85e-02 | 20260508 13:40:09.540 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:40:09.540 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:40:09.540 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:40:09.540 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 24.57%, Upload -- 2.57e+01 / 0.00e+00, 7.91e+00 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 24.72%, Download -- 2.59e+01 / 0.00e+00, 8.12e+00 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 31.14%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 2.30%, LockUpload -- 2.41e+00 / 0.00e+00, 5.71e+00 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 2.47%, LockDownload -- 2.59e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 14.78%, LockCompute -- 1.55e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:40:09.540 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260508 13:40:09.636 | INFO | Starting a round of Baz IO | 20260508 13:40:13.043 | INFO | Populating RTMetrics report | 20260508 13:40:13.043 | INFO | RTMetrics report populated | 20260508 13:40:13.043 | INFO | Opening tmp RTMetrics json file | 20260508 13:40:13.051 | INFO | Serializing RTMetrics to json | 20260508 13:40:13.052 | INFO | Serialization done, swapping tmp file now... | 20260508 13:40:13.053 | INFO | RTMetrics done | 20260508 13:40:13.107 | INFO | Finished a round of Baz IO: Wrote 1.381 GiB in 3.47 seconds | 20260508 13:40:13.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:40:13.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136852% / 0% / 7.94729e-05% | 20260508 13:40:13.107 | INFO | Baz IO overall: | 20260508 13:40:13.107 | INFO | BytesWritten : 3632787353365 | 20260508 13:40:13.107 | INFO | NumEvents : 1741461727839 | 20260508 13:40:14.683 | INFO | Analyzing chunk frames = [8536064,8536576) | 20260508 13:40:14.683 | INFO | MemInfo: { | 20260508 13:40:14.683 | INFO | "gpu" : " 17 GB", | 20260508 13:40:14.683 | INFO | "huge" : "", | 20260508 13:40:14.683 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:14.683 | INFO | "malloc" : " 14 GB", | 20260508 13:40:14.683 | INFO | "pinned" : " 29 GB" | 20260508 13:40:14.683 | INFO | } | 20260508 13:40:14.683 | INFO | Perf report for the last 5.14444 seconds: | 20260508 13:40:14.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:14.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:14.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:14.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:14.683 | INFO | ANALYSIS : 67.2%, 3.0, 105.3ms, 0 | 20260508 13:40:14.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:14.683 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:40:14.683 | INFO | PRE_HQ : 65.0%, 1.6, 53.4ms, 0 | 20260508 13:40:14.683 | INFO | BAZWRITER : 66.7%, 2.4, 83.4ms, 0 | 20260508 13:40:14.683 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:40:14.684 | INFO | Uploaded 36GiB to GPU in 2.6334s (13.6705GiB/s) | 20260508 13:40:14.684 | INFO | Downloaded 38GiB to GPU in 2.7078s (14.0336GiB/s) | 20260508 13:40:19.826 | INFO | Analyzing chunk frames = [8536576,8537088) | 20260508 13:40:19.826 | INFO | MemInfo: { | 20260508 13:40:19.826 | INFO | "gpu" : " 17 GB", | 20260508 13:40:19.826 | INFO | "huge" : "", | 20260508 13:40:19.826 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:19.826 | INFO | "malloc" : " 14 GB", | 20260508 13:40:19.826 | INFO | "pinned" : " 29 GB" | 20260508 13:40:19.826 | INFO | } | 20260508 13:40:19.826 | INFO | Perf report for the last 5.14285 seconds: | 20260508 13:40:19.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:19.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 13:40:19.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:40:19.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:19.826 | INFO | Uploaded 36GiB to GPU in 2.44053s (14.7509GiB/s) | 20260508 13:40:19.826 | INFO | Downloaded 34GiB to GPU in 2.47105s (13.7593GiB/s) | 20260508 13:40:24.962 | INFO | Analyzing chunk frames = [8537088,8537600) | 20260508 13:40:24.962 | INFO | MemInfo: { | 20260508 13:40:24.962 | INFO | "gpu" : " 17 GB", | 20260508 13:40:24.962 | INFO | "huge" : "", | 20260508 13:40:24.962 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:24.962 | INFO | "malloc" : " 14 GB", | 20260508 13:40:24.962 | INFO | "pinned" : " 29 GB" | 20260508 13:40:24.962 | INFO | } | 20260508 13:40:24.963 | INFO | Perf report for the last 5.13651 seconds: | 20260508 13:40:24.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:24.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | ANALYSIS : 65.9%, 3.0, 102.3ms, 0 | 20260508 13:40:24.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:40:24.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:24.963 | INFO | Uploaded 36GiB to GPU in 2.46317s (14.6153GiB/s) | 20260508 13:40:24.963 | INFO | Downloaded 34GiB to GPU in 2.45682s (13.8391GiB/s) | 20260508 13:40:30.099 | INFO | Analyzing chunk frames = [8537600,8538112) | 20260508 13:40:30.099 | INFO | MemInfo: { | 20260508 13:40:30.099 | INFO | "gpu" : " 17 GB", | 20260508 13:40:30.099 | INFO | "huge" : "", | 20260508 13:40:30.099 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:30.099 | INFO | "malloc" : " 14 GB", | 20260508 13:40:30.099 | INFO | "pinned" : " 29 GB" | 20260508 13:40:30.099 | INFO | } | 20260508 13:40:30.099 | INFO | Perf report for the last 5.13642 seconds: | 20260508 13:40:30.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:30.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.099 | INFO | ANALYSIS : 65.3%, 3.0, 101.2ms, 0 | 20260508 13:40:30.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.099 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:40:30.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:30.100 | INFO | Uploaded 36GiB to GPU in 2.40202s (14.9874GiB/s) | 20260508 13:40:30.100 | INFO | Downloaded 34GiB to GPU in 2.48023s (13.7084GiB/s) | 20260508 13:40:35.244 | INFO | Analyzing chunk frames = [8538112,8538624) | 20260508 13:40:35.244 | INFO | MemInfo: { | 20260508 13:40:35.244 | INFO | "gpu" : " 17 GB", | 20260508 13:40:35.244 | INFO | "huge" : "", | 20260508 13:40:35.244 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:35.244 | INFO | "malloc" : " 14 GB", | 20260508 13:40:35.244 | INFO | "pinned" : " 29 GB" | 20260508 13:40:35.244 | INFO | } | 20260508 13:40:35.245 | INFO | Perf report for the last 5.14558 seconds: | 20260508 13:40:35.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:35.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | ANALYSIS : 66.1%, 3.0, 103.4ms, 0 | 20260508 13:40:35.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:40:35.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:35.245 | INFO | Uploaded 36GiB to GPU in 2.45549s (14.661GiB/s) | 20260508 13:40:35.245 | INFO | Downloaded 34GiB to GPU in 2.53486s (13.413GiB/s) | 20260508 13:40:40.383 | INFO | Analyzing chunk frames = [8538624,8539136) | 20260508 13:40:40.383 | INFO | MemInfo: { | 20260508 13:40:40.383 | INFO | "gpu" : " 17 GB", | 20260508 13:40:40.383 | INFO | "huge" : "", | 20260508 13:40:40.383 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:40.384 | INFO | "malloc" : " 14 GB", | 20260508 13:40:40.384 | INFO | "pinned" : " 29 GB" | 20260508 13:40:40.384 | INFO | } | 20260508 13:40:40.384 | INFO | Perf report for the last 5.1391 seconds: | 20260508 13:40:40.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:40.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:40:40.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:40:40.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:40.384 | INFO | Uploaded 36GiB to GPU in 2.43089s (14.8094GiB/s) | 20260508 13:40:40.384 | INFO | Downloaded 34GiB to GPU in 2.45244s (13.8638GiB/s) | 20260508 13:40:45.518 | INFO | Analyzing chunk frames = [8539136,8539648) | 20260508 13:40:45.518 | INFO | MemInfo: { | 20260508 13:40:45.518 | INFO | "gpu" : " 17 GB", | 20260508 13:40:45.518 | INFO | "huge" : "", | 20260508 13:40:45.518 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:45.518 | INFO | "malloc" : " 14 GB", | 20260508 13:40:45.518 | INFO | "pinned" : " 29 GB" | 20260508 13:40:45.518 | INFO | } | 20260508 13:40:45.519 | INFO | Perf report for the last 5.13482 seconds: | 20260508 13:40:45.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:45.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:40:45.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:40:45.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:45.519 | INFO | Uploaded 36GiB to GPU in 2.29468s (15.6885GiB/s) | 20260508 13:40:45.519 | INFO | Downloaded 34GiB to GPU in 2.42639s (14.0126GiB/s) | 20260508 13:40:50.656 | INFO | Analyzing chunk frames = [8539648,8540160) | 20260508 13:40:50.656 | INFO | MemInfo: { | 20260508 13:40:50.656 | INFO | "gpu" : " 17 GB", | 20260508 13:40:50.656 | INFO | "huge" : "", | 20260508 13:40:50.657 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:50.657 | INFO | "malloc" : " 14 GB", | 20260508 13:40:50.657 | INFO | "pinned" : " 29 GB" | 20260508 13:40:50.657 | INFO | } | 20260508 13:40:50.657 | INFO | Perf report for the last 5.13809 seconds: | 20260508 13:40:50.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:50.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260508 13:40:50.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:40:50.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:50.657 | INFO | Uploaded 36GiB to GPU in 2.37701s (15.1451GiB/s) | 20260508 13:40:50.657 | INFO | Downloaded 34GiB to GPU in 2.43816s (13.9449GiB/s) | 20260508 13:40:50.754 | INFO | Starting a round of Baz IO | 20260508 13:40:54.170 | INFO | Populating RTMetrics report | 20260508 13:40:54.170 | INFO | RTMetrics report populated | 20260508 13:40:54.170 | INFO | Opening tmp RTMetrics json file | 20260508 13:40:54.177 | INFO | Serializing RTMetrics to json | 20260508 13:40:54.179 | INFO | Serialization done, swapping tmp file now... | 20260508 13:40:54.179 | INFO | RTMetrics done | 20260508 13:40:54.232 | INFO | Finished a round of Baz IO: Wrote 1.384 GiB in 3.478 seconds | 20260508 13:40:54.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:40:54.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136737% / 0% / 6.35783e-05% | 20260508 13:40:54.232 | INFO | Baz IO overall: | 20260508 13:40:54.232 | INFO | BytesWritten : 3634272888513 | 20260508 13:40:54.232 | INFO | NumEvents : 1742157309122 | 20260508 13:40:55.793 | INFO | Analyzing chunk frames = [8540160,8540672) | 20260508 13:40:55.793 | INFO | MemInfo: { | 20260508 13:40:55.793 | INFO | "gpu" : " 17 GB", | 20260508 13:40:55.793 | INFO | "huge" : "", | 20260508 13:40:55.793 | INFO | "huge_shared" : " 29 GB", | 20260508 13:40:55.793 | INFO | "malloc" : " 14 GB", | 20260508 13:40:55.793 | INFO | "pinned" : " 29 GB" | 20260508 13:40:55.793 | INFO | } | 20260508 13:40:55.793 | INFO | Perf report for the last 5.13635 seconds: | 20260508 13:40:55.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:40:55.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:55.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:55.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:55.793 | INFO | ANALYSIS : 67.7%, 3.0, 105.6ms, 0 | 20260508 13:40:55.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:40:55.793 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:40:55.793 | INFO | PRE_HQ : 65.3%, 1.6, 53.5ms, 0 | 20260508 13:40:55.793 | INFO | BAZWRITER : 67.0%, 2.5, 84.7ms, 0 | 20260508 13:40:55.793 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:40:55.793 | INFO | Uploaded 36GiB to GPU in 2.61414s (13.7713GiB/s) | 20260508 13:40:55.793 | INFO | Downloaded 38GiB to GPU in 2.70581s (14.0439GiB/s) | 20260508 13:41:00.942 | INFO | Analyzing chunk frames = [8540672,8541184) | 20260508 13:41:00.942 | INFO | MemInfo: { | 20260508 13:41:00.942 | INFO | "gpu" : " 17 GB", | 20260508 13:41:00.942 | INFO | "huge" : "", | 20260508 13:41:00.942 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:00.942 | INFO | "malloc" : " 14 GB", | 20260508 13:41:00.942 | INFO | "pinned" : " 29 GB" | 20260508 13:41:00.942 | INFO | } | 20260508 13:41:00.942 | INFO | Perf report for the last 5.1489 seconds: | 20260508 13:41:00.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:00.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.942 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:41:00.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.942 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:41:00.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:00.943 | INFO | Uploaded 36GiB to GPU in 2.49844s (14.409GiB/s) | 20260508 13:41:00.943 | INFO | Downloaded 34GiB to GPU in 2.45518s (13.8483GiB/s) | 20260508 13:41:06.076 | INFO | Analyzing chunk frames = [8541184,8541696) | 20260508 13:41:06.076 | INFO | MemInfo: { | 20260508 13:41:06.076 | INFO | "gpu" : " 17 GB", | 20260508 13:41:06.076 | INFO | "huge" : "", | 20260508 13:41:06.076 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:06.076 | INFO | "malloc" : " 14 GB", | 20260508 13:41:06.076 | INFO | "pinned" : " 29 GB" | 20260508 13:41:06.076 | INFO | } | 20260508 13:41:06.076 | INFO | Perf report for the last 5.13444 seconds: | 20260508 13:41:06.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:06.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.076 | INFO | ANALYSIS : 66.6%, 3.0, 103.3ms, 0 | 20260508 13:41:06.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.076 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:41:06.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:06.077 | INFO | Uploaded 36GiB to GPU in 2.52512s (14.2567GiB/s) | 20260508 13:41:06.077 | INFO | Downloaded 34GiB to GPU in 2.46137s (13.8134GiB/s) | 20260508 13:41:11.216 | INFO | Analyzing chunk frames = [8541696,8542208) | 20260508 13:41:11.216 | INFO | MemInfo: { | 20260508 13:41:11.216 | INFO | "gpu" : " 17 GB", | 20260508 13:41:11.216 | INFO | "huge" : "", | 20260508 13:41:11.216 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:11.216 | INFO | "malloc" : " 14 GB", | 20260508 13:41:11.216 | INFO | "pinned" : " 29 GB" | 20260508 13:41:11.216 | INFO | } | 20260508 13:41:11.216 | INFO | Perf report for the last 5.13954 seconds: | 20260508 13:41:11.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:11.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | ANALYSIS : 66.0%, 3.0, 102.4ms, 0 | 20260508 13:41:11.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:41:11.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:11.216 | INFO | Uploaded 36GiB to GPU in 2.52539s (14.2552GiB/s) | 20260508 13:41:11.217 | INFO | Downloaded 34GiB to GPU in 2.44602s (13.9001GiB/s) | 20260508 13:41:16.353 | INFO | Analyzing chunk frames = [8542208,8542720) | 20260508 13:41:16.353 | INFO | MemInfo: { | 20260508 13:41:16.353 | INFO | "gpu" : " 17 GB", | 20260508 13:41:16.353 | INFO | "huge" : "", | 20260508 13:41:16.353 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:16.353 | INFO | "malloc" : " 14 GB", | 20260508 13:41:16.353 | INFO | "pinned" : " 29 GB" | 20260508 13:41:16.353 | INFO | } | 20260508 13:41:16.353 | INFO | Perf report for the last 5.13705 seconds: | 20260508 13:41:16.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:16.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:41:16.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:41:16.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:16.353 | INFO | Uploaded 36GiB to GPU in 2.43925s (14.7586GiB/s) | 20260508 13:41:16.353 | INFO | Downloaded 34GiB to GPU in 2.51047s (13.5433GiB/s) | 20260508 13:41:21.496 | INFO | Analyzing chunk frames = [8542720,8543232) | 20260508 13:41:21.497 | INFO | MemInfo: { | 20260508 13:41:21.497 | INFO | "gpu" : " 17 GB", | 20260508 13:41:21.497 | INFO | "huge" : "", | 20260508 13:41:21.497 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:21.497 | INFO | "malloc" : " 14 GB", | 20260508 13:41:21.497 | INFO | "pinned" : " 29 GB" | 20260508 13:41:21.497 | INFO | } | 20260508 13:41:21.497 | INFO | Perf report for the last 5.14385 seconds: | 20260508 13:41:21.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:21.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:41:21.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:41:21.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:21.497 | INFO | Uploaded 36GiB to GPU in 2.44904s (14.6997GiB/s) | 20260508 13:41:21.497 | INFO | Downloaded 34GiB to GPU in 2.43558s (13.9597GiB/s) | 20260508 13:41:26.634 | INFO | Analyzing chunk frames = [8543232,8543744) | 20260508 13:41:26.634 | INFO | MemInfo: { | 20260508 13:41:26.634 | INFO | "gpu" : " 17 GB", | 20260508 13:41:26.634 | INFO | "huge" : "", | 20260508 13:41:26.634 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:26.634 | INFO | "malloc" : " 14 GB", | 20260508 13:41:26.634 | INFO | "pinned" : " 29 GB" | 20260508 13:41:26.635 | INFO | } | 20260508 13:41:26.635 | INFO | Perf report for the last 5.13776 seconds: | 20260508 13:41:26.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:26.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 13:41:26.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:41:26.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:26.635 | INFO | Uploaded 36GiB to GPU in 2.39644s (15.0223GiB/s) | 20260508 13:41:26.635 | INFO | Downloaded 34GiB to GPU in 2.40339s (14.1467GiB/s) | 20260508 13:41:31.774 | INFO | Analyzing chunk frames = [8543744,8544256) | 20260508 13:41:31.774 | INFO | MemInfo: { | 20260508 13:41:31.774 | INFO | "gpu" : " 17 GB", | 20260508 13:41:31.774 | INFO | "huge" : "", | 20260508 13:41:31.774 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:31.774 | INFO | "malloc" : " 14 GB", | 20260508 13:41:31.774 | INFO | "pinned" : " 29 GB" | 20260508 13:41:31.774 | INFO | } | 20260508 13:41:31.775 | INFO | Perf report for the last 5.13999 seconds: | 20260508 13:41:31.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:31.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | ANALYSIS : 64.8%, 3.0, 101.3ms, 0 | 20260508 13:41:31.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:41:31.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:31.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:41:31.775 | INFO | Total mem usage: 16.7 GiB | 20260508 13:41:31.775 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:41:31.775 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:41:31.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:41:31.775 | INFO | High Water Breakdown: | 20260508 13:41:31.775 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:41:31.775 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:41:31.775 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:41:31.775 | INFO | ------------------------------------------ | 20260508 13:41:31.775 | INFO | Uploaded 36GiB to GPU in 2.42028s (14.8743GiB/s) | 20260508 13:41:31.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:41:31.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:41:31.775 | INFO | Total mem usage: 26.5 GiB | 20260508 13:41:31.775 | INFO | Downloaded 34GiB to GPU in 2.42241s (14.0356GiB/s) | 20260508 13:41:31.775 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:41:31.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:41:31.775 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:41:31.775 | INFO | High Water Breakdown: | 20260508 13:41:31.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:41:31.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:41:31.775 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:41:31.775 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:41:31.775 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:41:31.775 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:41:31.776 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:41:31.776 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:41:31.776 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:41:31.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:41:31.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:41:31.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:41:31.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:41:31.776 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:41:31.776 | INFO | ------------------------------------------ | 20260508 13:41:31.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:41:31.776 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:41:31.776 | INFO | Total mem usage: 27.8 GiB | 20260508 13:41:31.776 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:41:31.776 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:41:31.776 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:41:31.776 | INFO | High Water Breakdown: | 20260508 13:41:31.776 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:41:31.776 | INFO | ------------------------------------------ | 20260508 13:41:31.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:41:31.776 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:41:31.776 | INFO | Total mem usage: 15.7 GiB | 20260508 13:41:31.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:41:31.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:41:31.776 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:41:31.776 | INFO | High Water Breakdown: | 20260508 13:41:31.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:41:31.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:41:31.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:41:31.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:41:31.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:41:31.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:41:31.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:41:31.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:41:31.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:41:31.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:41:31.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:41:31.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:41:31.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:41:31.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:41:31.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:41:31.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:41:31.776 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:41:31.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:41:31.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:41:31.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:41:31.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:41:31.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:41:31.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:41:31.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:41:31.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:41:31.776 | INFO | ------------------------------------------ | 20260508 13:41:31.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:41:31.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:41:31.776 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:41:31.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:41:31.776 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.38e+01 / 3.74e-01 | 20260508 13:41:31.776 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.24e-02 / 4.20e-03 | 20260508 13:41:31.776 | INFO | PerfReport: 12.22%, Baseline -- 4.00e+00 / 4.36e+00, 6.06e-01 / 1.83e-01 | 20260508 13:41:31.776 | INFO | PerfReport: 3.41%, Histogram -- 1.12e+00 / 1.44e+00, 2.10e-01 / 8.89e-02 | 20260508 13:41:31.776 | INFO | PerfReport: 15.94%, DME -- 4.17e+01 / 4.52e+01, 3.69e+00 / 1.07e+00 | 20260508 13:41:31.776 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.88e+01 / 2.03e+01, 1.90e+00 / 1.04e+00 | 20260508 13:41:31.776 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.30e+00 / 2.38e+00, 3.97e-01 / 2.20e-01 | 20260508 13:41:31.776 | INFO | PerfReport: 2.90%, Metrics -- 9.51e-01 / 1.07e+00, 2.39e-01 / 7.34e-02 | 20260508 13:41:31.776 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.87e-01 / 3.18e-01, 4.65e-02 / 2.54e-02 | 20260508 13:41:31.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:41:31.776 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:41:31.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:41:31.777 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 24.49%, Upload -- 2.56e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 24.77%, Download -- 2.59e+01 / 0.00e+00, 8.15e+00 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 31.29%, Compute -- 3.27e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 2.21%, LockUpload -- 2.31e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 2.45%, LockDownload -- 2.56e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 14.78%, LockCompute -- 1.55e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:41:31.777 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.77e-02 / 0.00e+00 | 20260508 13:41:31.886 | INFO | Starting a round of Baz IO | 20260508 13:41:35.291 | INFO | Populating RTMetrics report | 20260508 13:41:35.291 | INFO | RTMetrics report populated | 20260508 13:41:35.291 | INFO | Opening tmp RTMetrics json file | 20260508 13:41:35.296 | INFO | Serializing RTMetrics to json | 20260508 13:41:35.297 | INFO | Serialization done, swapping tmp file now... | 20260508 13:41:35.297 | INFO | RTMetrics done | 20260508 13:41:35.358 | INFO | Finished a round of Baz IO: Wrote 1.38 GiB in 3.471 seconds | 20260508 13:41:35.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:41:35.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136169% / 3.97364e-06% / 5.16574e-05% | 20260508 13:41:35.358 | INFO | Baz IO overall: | 20260508 13:41:35.358 | INFO | BytesWritten : 3635754704667 | 20260508 13:41:35.358 | INFO | NumEvents : 1742850257211 | 20260508 13:41:36.920 | INFO | Analyzing chunk frames = [8544256,8544768) | 20260508 13:41:36.920 | INFO | MemInfo: { | 20260508 13:41:36.920 | INFO | "gpu" : " 17 GB", | 20260508 13:41:36.920 | INFO | "huge" : "", | 20260508 13:41:36.920 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:36.920 | INFO | "malloc" : " 14 GB", | 20260508 13:41:36.920 | INFO | "pinned" : " 29 GB" | 20260508 13:41:36.920 | INFO | } | 20260508 13:41:36.920 | INFO | Perf report for the last 5.14565 seconds: | 20260508 13:41:36.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:36.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:36.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:36.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:36.920 | INFO | ANALYSIS : 67.5%, 3.0, 105.7ms, 0 | 20260508 13:41:36.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:36.920 | INFO | RT_METRICS : 39.9%, 1.0, 20.9ms, 0 | 20260508 13:41:36.920 | INFO | PRE_HQ : 65.7%, 1.7, 56.9ms, 0 | 20260508 13:41:36.920 | INFO | BAZWRITER : 66.8%, 2.9, 100.3ms, 0 | 20260508 13:41:36.920 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:41:36.921 | INFO | Uploaded 36GiB to GPU in 2.64178s (13.6272GiB/s) | 20260508 13:41:36.921 | INFO | Downloaded 38GiB to GPU in 2.71852s (13.9782GiB/s) | 20260508 13:41:42.050 | INFO | Analyzing chunk frames = [8544768,8545280) | 20260508 13:41:42.050 | INFO | MemInfo: { | 20260508 13:41:42.050 | INFO | "gpu" : " 17 GB", | 20260508 13:41:42.050 | INFO | "huge" : "", | 20260508 13:41:42.050 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:42.050 | INFO | "malloc" : " 14 GB", | 20260508 13:41:42.050 | INFO | "pinned" : " 29 GB" | 20260508 13:41:42.050 | INFO | } | 20260508 13:41:42.050 | INFO | Perf report for the last 5.13007 seconds: | 20260508 13:41:42.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:42.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.050 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:41:42.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.050 | INFO | PRE_HQ : 22.3%, 1.0, 11.5ms, 0 | 20260508 13:41:42.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:42.051 | INFO | Uploaded 36GiB to GPU in 2.47077s (14.5704GiB/s) | 20260508 13:41:42.051 | INFO | Downloaded 34GiB to GPU in 2.49472s (13.6288GiB/s) | 20260508 13:41:47.194 | INFO | Analyzing chunk frames = [8545280,8545792) | 20260508 13:41:47.194 | INFO | MemInfo: { | 20260508 13:41:47.194 | INFO | "gpu" : " 17 GB", | 20260508 13:41:47.194 | INFO | "huge" : "", | 20260508 13:41:47.194 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:47.194 | INFO | "malloc" : " 14 GB", | 20260508 13:41:47.194 | INFO | "pinned" : " 29 GB" | 20260508 13:41:47.194 | INFO | } | 20260508 13:41:47.194 | INFO | Perf report for the last 5.14401 seconds: | 20260508 13:41:47.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:47.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.194 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 13:41:47.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.194 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:41:47.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:47.195 | INFO | Uploaded 36GiB to GPU in 2.45868s (14.642GiB/s) | 20260508 13:41:47.195 | INFO | Downloaded 34GiB to GPU in 2.46251s (13.8071GiB/s) | 20260508 13:41:52.335 | INFO | Analyzing chunk frames = [8545792,8546304) | 20260508 13:41:52.335 | INFO | MemInfo: { | 20260508 13:41:52.335 | INFO | "gpu" : " 17 GB", | 20260508 13:41:52.335 | INFO | "huge" : "", | 20260508 13:41:52.335 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:52.335 | INFO | "malloc" : " 14 GB", | 20260508 13:41:52.335 | INFO | "pinned" : " 29 GB" | 20260508 13:41:52.336 | INFO | } | 20260508 13:41:52.336 | INFO | Perf report for the last 5.14122 seconds: | 20260508 13:41:52.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:52.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | ANALYSIS : 65.3%, 3.0, 101.3ms, 0 | 20260508 13:41:52.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:41:52.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:52.336 | INFO | Uploaded 36GiB to GPU in 2.38885s (15.07GiB/s) | 20260508 13:41:52.336 | INFO | Downloaded 34GiB to GPU in 2.47026s (13.7637GiB/s) | 20260508 13:41:57.468 | INFO | Analyzing chunk frames = [8546304,8546816) | 20260508 13:41:57.468 | INFO | MemInfo: { | 20260508 13:41:57.468 | INFO | "gpu" : " 17 GB", | 20260508 13:41:57.468 | INFO | "huge" : "", | 20260508 13:41:57.468 | INFO | "huge_shared" : " 29 GB", | 20260508 13:41:57.469 | INFO | "malloc" : " 14 GB", | 20260508 13:41:57.469 | INFO | "pinned" : " 29 GB" | 20260508 13:41:57.469 | INFO | } | 20260508 13:41:57.469 | INFO | Perf report for the last 5.13305 seconds: | 20260508 13:41:57.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:41:57.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 13:41:57.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:41:57.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:41:57.469 | INFO | Uploaded 36GiB to GPU in 2.34981s (15.3204GiB/s) | 20260508 13:41:57.469 | INFO | Downloaded 34GiB to GPU in 2.50833s (13.5548GiB/s) | 20260508 13:42:02.606 | INFO | Analyzing chunk frames = [8546816,8547328) | 20260508 13:42:02.606 | INFO | MemInfo: { | 20260508 13:42:02.606 | INFO | "gpu" : " 17 GB", | 20260508 13:42:02.606 | INFO | "huge" : "", | 20260508 13:42:02.606 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:02.606 | INFO | "malloc" : " 14 GB", | 20260508 13:42:02.606 | INFO | "pinned" : " 29 GB" | 20260508 13:42:02.606 | INFO | } | 20260508 13:42:02.606 | INFO | Perf report for the last 5.1378 seconds: | 20260508 13:42:02.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:02.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.607 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:42:02.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.607 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:42:02.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:02.607 | INFO | Uploaded 36GiB to GPU in 2.40004s (14.9998GiB/s) | 20260508 13:42:02.607 | INFO | Downloaded 34GiB to GPU in 2.44973s (13.8791GiB/s) | 20260508 13:42:07.747 | INFO | Analyzing chunk frames = [8547328,8547840) | 20260508 13:42:07.747 | INFO | MemInfo: { | 20260508 13:42:07.747 | INFO | "gpu" : " 17 GB", | 20260508 13:42:07.747 | INFO | "huge" : "", | 20260508 13:42:07.747 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:07.747 | INFO | "malloc" : " 14 GB", | 20260508 13:42:07.747 | INFO | "pinned" : " 29 GB" | 20260508 13:42:07.747 | INFO | } | 20260508 13:42:07.747 | INFO | Perf report for the last 5.1409 seconds: | 20260508 13:42:07.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:07.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.747 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 13:42:07.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.747 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 13:42:07.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:07.748 | INFO | Uploaded 36GiB to GPU in 2.47125s (14.5676GiB/s) | 20260508 13:42:07.748 | INFO | Downloaded 34GiB to GPU in 2.4678s (13.7775GiB/s) | 20260508 13:42:12.889 | INFO | Analyzing chunk frames = [8547840,8548352) | 20260508 13:42:12.889 | INFO | MemInfo: { | 20260508 13:42:12.889 | INFO | "gpu" : " 17 GB", | 20260508 13:42:12.889 | INFO | "huge" : "", | 20260508 13:42:12.889 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:12.889 | INFO | "malloc" : " 14 GB", | 20260508 13:42:12.889 | INFO | "pinned" : " 29 GB" | 20260508 13:42:12.889 | INFO | } | 20260508 13:42:12.889 | INFO | Perf report for the last 5.14202 seconds: | 20260508 13:42:12.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:12.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.889 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 13:42:12.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.889 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:42:12.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:12.890 | INFO | Uploaded 36GiB to GPU in 2.47512s (14.5447GiB/s) | 20260508 13:42:12.890 | INFO | Downloaded 34GiB to GPU in 2.46789s (13.7769GiB/s) | 20260508 13:42:12.987 | INFO | Starting a round of Baz IO | 20260508 13:42:16.387 | INFO | Populating RTMetrics report | 20260508 13:42:16.387 | INFO | RTMetrics report populated | 20260508 13:42:16.387 | INFO | Opening tmp RTMetrics json file | 20260508 13:42:16.394 | INFO | Serializing RTMetrics to json | 20260508 13:42:16.396 | INFO | Serialization done, swapping tmp file now... | 20260508 13:42:16.396 | INFO | RTMetrics done | 20260508 13:42:16.457 | INFO | Finished a round of Baz IO: Wrote 1.38 GiB in 3.469 seconds | 20260508 13:42:16.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:42:16.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136344% / 3.97364e-06% / 5.16574e-05% | 20260508 13:42:16.457 | INFO | Baz IO overall: | 20260508 13:42:16.457 | INFO | BytesWritten : 3637236917115 | 20260508 13:42:16.457 | INFO | NumEvents : 1743543713023 | 20260508 13:42:18.030 | INFO | Analyzing chunk frames = [8548352,8548864) | 20260508 13:42:18.030 | INFO | MemInfo: { | 20260508 13:42:18.030 | INFO | "gpu" : " 17 GB", | 20260508 13:42:18.030 | INFO | "huge" : "", | 20260508 13:42:18.030 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:18.031 | INFO | "malloc" : " 14 GB", | 20260508 13:42:18.031 | INFO | "pinned" : " 29 GB" | 20260508 13:42:18.031 | INFO | } | 20260508 13:42:18.031 | INFO | Perf report for the last 5.1412 seconds: | 20260508 13:42:18.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:18.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:18.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:18.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:18.031 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:42:18.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:18.031 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:42:18.031 | INFO | PRE_HQ : 65.3%, 1.6, 54.1ms, 0 | 20260508 13:42:18.031 | INFO | BAZWRITER : 66.7%, 2.4, 83.8ms, 0 | 20260508 13:42:18.031 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:42:18.031 | INFO | Uploaded 36GiB to GPU in 2.64249s (13.6235GiB/s) | 20260508 13:42:18.031 | INFO | Downloaded 38GiB to GPU in 2.72619s (13.9389GiB/s) | 20260508 13:42:23.163 | INFO | Analyzing chunk frames = [8548864,8549376) | 20260508 13:42:23.163 | INFO | MemInfo: { | 20260508 13:42:23.163 | INFO | "gpu" : " 17 GB", | 20260508 13:42:23.163 | INFO | "huge" : "", | 20260508 13:42:23.163 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:23.163 | INFO | "malloc" : " 14 GB", | 20260508 13:42:23.163 | INFO | "pinned" : " 29 GB" | 20260508 13:42:23.163 | INFO | } | 20260508 13:42:23.163 | INFO | Perf report for the last 5.13245 seconds: | 20260508 13:42:23.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:23.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | ANALYSIS : 66.1%, 3.0, 102.8ms, 0 | 20260508 13:42:23.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:42:23.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:23.163 | INFO | Uploaded 36GiB to GPU in 2.47409s (14.5508GiB/s) | 20260508 13:42:23.163 | INFO | Downloaded 34GiB to GPU in 2.48508s (13.6816GiB/s) | 20260508 13:42:28.304 | INFO | Analyzing chunk frames = [8549376,8549888) | 20260508 13:42:28.304 | INFO | MemInfo: { | 20260508 13:42:28.304 | INFO | "gpu" : " 17 GB", | 20260508 13:42:28.304 | INFO | "huge" : "", | 20260508 13:42:28.304 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:28.304 | INFO | "malloc" : " 14 GB", | 20260508 13:42:28.304 | INFO | "pinned" : " 29 GB" | 20260508 13:42:28.304 | INFO | } | 20260508 13:42:28.304 | INFO | Perf report for the last 5.14078 seconds: | 20260508 13:42:28.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:28.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | ANALYSIS : 66.1%, 3.0, 102.7ms, 0 | 20260508 13:42:28.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:42:28.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:28.304 | INFO | Uploaded 36GiB to GPU in 2.48235s (14.5024GiB/s) | 20260508 13:42:28.304 | INFO | Downloaded 34GiB to GPU in 2.46465s (13.7951GiB/s) | 20260508 13:42:33.446 | INFO | Analyzing chunk frames = [8549888,8550400) | 20260508 13:42:33.446 | INFO | MemInfo: { | 20260508 13:42:33.446 | INFO | "gpu" : " 17 GB", | 20260508 13:42:33.446 | INFO | "huge" : "", | 20260508 13:42:33.446 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:33.446 | INFO | "malloc" : " 14 GB", | 20260508 13:42:33.446 | INFO | "pinned" : " 29 GB" | 20260508 13:42:33.446 | INFO | } | 20260508 13:42:33.446 | INFO | Perf report for the last 5.14253 seconds: | 20260508 13:42:33.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:33.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.446 | INFO | ANALYSIS : 65.3%, 3.0, 101.4ms, 0 | 20260508 13:42:33.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.447 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:42:33.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:33.447 | INFO | Uploaded 36GiB to GPU in 2.38122s (15.1183GiB/s) | 20260508 13:42:33.447 | INFO | Downloaded 34GiB to GPU in 2.46808s (13.7759GiB/s) | 20260508 13:42:38.587 | INFO | Analyzing chunk frames = [8550400,8550912) | 20260508 13:42:38.587 | INFO | MemInfo: { | 20260508 13:42:38.587 | INFO | "gpu" : " 17 GB", | 20260508 13:42:38.587 | INFO | "huge" : "", | 20260508 13:42:38.587 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:38.587 | INFO | "malloc" : " 14 GB", | 20260508 13:42:38.587 | INFO | "pinned" : " 29 GB" | 20260508 13:42:38.587 | INFO | } | 20260508 13:42:38.587 | INFO | Perf report for the last 5.14055 seconds: | 20260508 13:42:38.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:38.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 13:42:38.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 13:42:38.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:38.587 | INFO | Uploaded 36GiB to GPU in 2.44142s (14.7455GiB/s) | 20260508 13:42:38.587 | INFO | Downloaded 34GiB to GPU in 2.53279s (13.4239GiB/s) | 20260508 13:42:43.723 | INFO | Analyzing chunk frames = [8550912,8551424) | 20260508 13:42:43.723 | INFO | MemInfo: { | 20260508 13:42:43.723 | INFO | "gpu" : " 17 GB", | 20260508 13:42:43.723 | INFO | "huge" : "", | 20260508 13:42:43.723 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:43.723 | INFO | "malloc" : " 14 GB", | 20260508 13:42:43.723 | INFO | "pinned" : " 29 GB" | 20260508 13:42:43.723 | INFO | } | 20260508 13:42:43.723 | INFO | Perf report for the last 5.13647 seconds: | 20260508 13:42:43.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:43.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.724 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 13:42:43.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.724 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:42:43.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:43.724 | INFO | Uploaded 36GiB to GPU in 2.4098s (14.939GiB/s) | 20260508 13:42:43.724 | INFO | Downloaded 34GiB to GPU in 2.44648s (13.8975GiB/s) | 20260508 13:42:48.861 | INFO | Analyzing chunk frames = [8551424,8551936) | 20260508 13:42:48.861 | INFO | MemInfo: { | 20260508 13:42:48.861 | INFO | "gpu" : " 17 GB", | 20260508 13:42:48.861 | INFO | "huge" : "", | 20260508 13:42:48.861 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:48.861 | INFO | "malloc" : " 14 GB", | 20260508 13:42:48.861 | INFO | "pinned" : " 29 GB" | 20260508 13:42:48.861 | INFO | } | 20260508 13:42:48.862 | INFO | Perf report for the last 5.13805 seconds: | 20260508 13:42:48.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:48.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:42:48.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:42:48.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:48.862 | INFO | Uploaded 36GiB to GPU in 2.45933s (14.6382GiB/s) | 20260508 13:42:48.862 | INFO | Downloaded 34GiB to GPU in 2.4656s (13.7898GiB/s) | 20260508 13:42:54.006 | INFO | Analyzing chunk frames = [8551936,8552448) | 20260508 13:42:54.006 | INFO | MemInfo: { | 20260508 13:42:54.006 | INFO | "gpu" : " 17 GB", | 20260508 13:42:54.006 | INFO | "huge" : "", | 20260508 13:42:54.006 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:54.007 | INFO | "malloc" : " 14 GB", | 20260508 13:42:54.007 | INFO | "pinned" : " 29 GB" | 20260508 13:42:54.007 | INFO | } | 20260508 13:42:54.007 | INFO | Perf report for the last 5.14521 seconds: | 20260508 13:42:54.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:54.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | ANALYSIS : 65.0%, 3.0, 101.5ms, 0 | 20260508 13:42:54.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:42:54.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:54.007 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:42:54.007 | INFO | Total mem usage: 16.7 GiB | 20260508 13:42:54.007 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:42:54.007 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13%) | 20260508 13:42:54.007 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:42:54.007 | INFO | High Water Breakdown: | 20260508 13:42:54.007 | INFO | 60.1% : 8.87 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:42:54.007 | INFO | 39.8% : 5.87 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:42:54.007 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:42:54.007 | INFO | ------------------------------------------ | 20260508 13:42:54.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:42:54.007 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:42:54.007 | INFO | Total mem usage: 26.5 GiB | 20260508 13:42:54.007 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:42:54.007 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:42:54.007 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:42:54.007 | INFO | High Water Breakdown: | 20260508 13:42:54.007 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:42:54.007 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:42:54.007 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:42:54.007 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:42:54.007 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:42:54.007 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:42:54.007 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:42:54.007 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:42:54.007 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:42:54.007 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:42:54.007 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:42:54.007 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:42:54.007 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:42:54.007 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:42:54.007 | INFO | ------------------------------------------ | 20260508 13:42:54.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:42:54.007 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:42:54.007 | INFO | Total mem usage: 27.8 GiB | 20260508 13:42:54.007 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:42:54.007 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:42:54.007 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:42:54.007 | INFO | High Water Breakdown: | 20260508 13:42:54.007 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:42:54.007 | INFO | ------------------------------------------ | 20260508 13:42:54.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:42:54.007 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:42:54.007 | INFO | Total mem usage: 15.7 GiB | 20260508 13:42:54.007 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:42:54.007 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:42:54.007 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:42:54.007 | INFO | High Water Breakdown: | 20260508 13:42:54.007 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:42:54.007 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:42:54.007 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:42:54.007 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:42:54.007 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:42:54.007 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:42:54.007 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:42:54.007 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:42:54.007 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:42:54.007 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:42:54.007 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:42:54.007 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:42:54.007 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:42:54.007 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:42:54.007 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:42:54.007 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:42:54.007 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:42:54.007 | INFO | Uploaded 36GiB to GPU in 2.42043s (14.8734GiB/s) | 20260508 13:42:54.007 | INFO | Downloaded 34GiB to GPU in 2.44266s (13.9193GiB/s) | 20260508 13:42:54.007 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:42:54.008 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:42:54.008 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:42:54.008 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:42:54.008 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:42:54.008 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:42:54.008 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:42:54.008 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:42:54.008 | INFO | ------------------------------------------ | 20260508 13:42:54.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:42:54.008 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:42:54.008 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:42:54.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:42:54.008 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 6.20e-01 | 20260508 13:42:54.008 | INFO | PerfReport: 0.30%, AggregateStats -- 9.70e-02 / 1.04e-01, 1.13e-02 / 5.26e-03 | 20260508 13:42:54.008 | INFO | PerfReport: 12.22%, Baseline -- 3.98e+00 / 4.50e+00, 5.94e-01 / 2.21e-01 | 20260508 13:42:54.008 | INFO | PerfReport: 3.38%, Histogram -- 1.10e+00 / 1.45e+00, 2.06e-01 / 1.46e-01 | 20260508 13:42:54.008 | INFO | PerfReport: 16.03%, DME -- 4.18e+01 / 4.55e+01, 3.67e+00 / 1.16e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.86e+01 / 1.98e+01, 1.77e+00 / 1.26e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.31e+00 / 2.48e+00, 4.02e-01 / 2.84e-01 | 20260508 13:42:54.008 | INFO | PerfReport: 2.93%, Metrics -- 9.53e-01 / 1.06e+00, 2.38e-01 / 1.07e-01 | 20260508 13:42:54.008 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.17e-01, 4.65e-02 / 2.13e-02 | 20260508 13:42:54.008 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:42:54.008 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:42:54.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:42:54.008 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.33e+01 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 24.46%, Upload -- 2.56e+01 / 0.00e+00, 8.01e+00 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 24.90%, Download -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 31.07%, Compute -- 3.26e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 2.03%, LockUpload -- 2.13e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 2.69%, LockDownload -- 2.82e+00 / 0.00e+00, 4.09e+00 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 14.84%, LockCompute -- 1.56e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:42:54.008 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260508 13:42:54.104 | INFO | Starting a round of Baz IO | 20260508 13:42:57.503 | INFO | Populating RTMetrics report | 20260508 13:42:57.504 | INFO | RTMetrics report populated | 20260508 13:42:57.504 | INFO | Opening tmp RTMetrics json file | 20260508 13:42:57.513 | INFO | Serializing RTMetrics to json | 20260508 13:42:57.514 | INFO | Serialization done, swapping tmp file now... | 20260508 13:42:57.514 | INFO | RTMetrics done | 20260508 13:42:57.577 | INFO | Finished a round of Baz IO: Wrote 1.379 GiB in 3.472 seconds | 20260508 13:42:57.577 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:42:57.577 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135803% / 3.97364e-06% / 7.94729e-05% | 20260508 13:42:57.577 | INFO | Baz IO overall: | 20260508 13:42:57.577 | INFO | BytesWritten : 3638717716241 | 20260508 13:42:57.577 | INFO | NumEvents : 1744236305543 | 20260508 13:42:59.142 | INFO | Analyzing chunk frames = [8552448,8552960) | 20260508 13:42:59.142 | INFO | MemInfo: { | 20260508 13:42:59.142 | INFO | "gpu" : " 17 GB", | 20260508 13:42:59.142 | INFO | "huge" : "", | 20260508 13:42:59.142 | INFO | "huge_shared" : " 29 GB", | 20260508 13:42:59.142 | INFO | "malloc" : " 14 GB", | 20260508 13:42:59.142 | INFO | "pinned" : " 29 GB" | 20260508 13:42:59.142 | INFO | } | 20260508 13:42:59.142 | INFO | Perf report for the last 5.13495 seconds: | 20260508 13:42:59.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:42:59.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:59.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:59.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:59.142 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 13:42:59.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:42:59.142 | INFO | RT_METRICS : 38.1%, 1.0, 19.9ms, 0 | 20260508 13:42:59.142 | INFO | PRE_HQ : 65.2%, 1.6, 53.8ms, 0 | 20260508 13:42:59.142 | INFO | BAZWRITER : 66.8%, 2.5, 84.4ms, 0 | 20260508 13:42:59.142 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:42:59.142 | INFO | Uploaded 36GiB to GPU in 2.63951s (13.6389GiB/s) | 20260508 13:42:59.142 | INFO | Downloaded 38GiB to GPU in 2.71107s (14.0166GiB/s) | 20260508 13:43:04.281 | INFO | Analyzing chunk frames = [8552960,8553472) | 20260508 13:43:04.281 | INFO | MemInfo: { | 20260508 13:43:04.281 | INFO | "gpu" : " 17 GB", | 20260508 13:43:04.281 | INFO | "huge" : "", | 20260508 13:43:04.281 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:04.281 | INFO | "malloc" : " 14 GB", | 20260508 13:43:04.281 | INFO | "pinned" : " 29 GB" | 20260508 13:43:04.281 | INFO | } | 20260508 13:43:04.281 | INFO | Perf report for the last 5.13967 seconds: | 20260508 13:43:04.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:04.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.281 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:43:04.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.281 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:43:04.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:04.282 | INFO | Uploaded 36GiB to GPU in 2.46761s (14.589GiB/s) | 20260508 13:43:04.282 | INFO | Downloaded 34GiB to GPU in 2.47353s (13.7456GiB/s) | 20260508 13:43:09.416 | INFO | Analyzing chunk frames = [8553472,8553984) | 20260508 13:43:09.416 | INFO | MemInfo: { | 20260508 13:43:09.416 | INFO | "gpu" : " 17 GB", | 20260508 13:43:09.416 | INFO | "huge" : "", | 20260508 13:43:09.416 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:09.416 | INFO | "malloc" : " 14 GB", | 20260508 13:43:09.416 | INFO | "pinned" : " 29 GB" | 20260508 13:43:09.416 | INFO | } | 20260508 13:43:09.416 | INFO | Perf report for the last 5.13476 seconds: | 20260508 13:43:09.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:09.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | ANALYSIS : 66.1%, 3.0, 102.6ms, 0 | 20260508 13:43:09.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:43:09.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:09.416 | INFO | Uploaded 36GiB to GPU in 2.47979s (14.5173GiB/s) | 20260508 13:43:09.416 | INFO | Downloaded 34GiB to GPU in 2.46158s (13.8122GiB/s) | 20260508 13:43:14.558 | INFO | Analyzing chunk frames = [8553984,8554496) | 20260508 13:43:14.558 | INFO | MemInfo: { | 20260508 13:43:14.558 | INFO | "gpu" : " 17 GB", | 20260508 13:43:14.558 | INFO | "huge" : "", | 20260508 13:43:14.558 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:14.558 | INFO | "malloc" : " 14 GB", | 20260508 13:43:14.558 | INFO | "pinned" : " 29 GB" | 20260508 13:43:14.558 | INFO | } | 20260508 13:43:14.559 | INFO | Perf report for the last 5.14237 seconds: | 20260508 13:43:14.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:14.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | ANALYSIS : 65.9%, 3.0, 102.1ms, 0 | 20260508 13:43:14.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260508 13:43:14.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:14.559 | INFO | Uploaded 36GiB to GPU in 2.50666s (14.3617GiB/s) | 20260508 13:43:14.559 | INFO | Downloaded 34GiB to GPU in 2.46852s (13.7734GiB/s) | 20260508 13:43:19.703 | INFO | Analyzing chunk frames = [8554496,8555008) | 20260508 13:43:19.703 | INFO | MemInfo: { | 20260508 13:43:19.703 | INFO | "gpu" : " 17 GB", | 20260508 13:43:19.703 | INFO | "huge" : "", | 20260508 13:43:19.703 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:19.703 | INFO | "malloc" : " 14 GB", | 20260508 13:43:19.703 | INFO | "pinned" : " 29 GB" | 20260508 13:43:19.703 | INFO | } | 20260508 13:43:19.703 | INFO | Perf report for the last 5.1449 seconds: | 20260508 13:43:19.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:19.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.704 | INFO | ANALYSIS : 66.1%, 3.0, 103.2ms, 0 | 20260508 13:43:19.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.704 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:43:19.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:19.704 | INFO | Uploaded 36GiB to GPU in 2.37345s (15.1678GiB/s) | 20260508 13:43:19.704 | INFO | Downloaded 34GiB to GPU in 2.52237s (13.4794GiB/s) | 20260508 13:43:24.840 | INFO | Analyzing chunk frames = [8555008,8555520) | 20260508 13:43:24.840 | INFO | MemInfo: { | 20260508 13:43:24.840 | INFO | "gpu" : " 17 GB", | 20260508 13:43:24.840 | INFO | "huge" : "", | 20260508 13:43:24.841 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:24.841 | INFO | "malloc" : " 14 GB", | 20260508 13:43:24.841 | INFO | "pinned" : " 29 GB" | 20260508 13:43:24.841 | INFO | } | 20260508 13:43:24.841 | INFO | Perf report for the last 5.13721 seconds: | 20260508 13:43:24.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:24.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | ANALYSIS : 65.4%, 3.0, 101.7ms, 0 | 20260508 13:43:24.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:43:24.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:24.841 | INFO | Uploaded 36GiB to GPU in 2.43018s (14.8137GiB/s) | 20260508 13:43:24.841 | INFO | Downloaded 34GiB to GPU in 2.45447s (13.8523GiB/s) | 20260508 13:43:29.980 | INFO | Analyzing chunk frames = [8555520,8556032) | 20260508 13:43:29.980 | INFO | MemInfo: { | 20260508 13:43:29.980 | INFO | "gpu" : " 17 GB", | 20260508 13:43:29.980 | INFO | "huge" : "", | 20260508 13:43:29.980 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:29.980 | INFO | "malloc" : " 14 GB", | 20260508 13:43:29.980 | INFO | "pinned" : " 29 GB" | 20260508 13:43:29.980 | INFO | } | 20260508 13:43:29.980 | INFO | Perf report for the last 5.13913 seconds: | 20260508 13:43:29.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:29.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:43:29.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:43:29.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:29.980 | INFO | Uploaded 36GiB to GPU in 2.33693s (15.4048GiB/s) | 20260508 13:43:29.980 | INFO | Downloaded 34GiB to GPU in 2.40091s (14.1613GiB/s) | 20260508 13:43:35.111 | INFO | Analyzing chunk frames = [8556032,8556544) | 20260508 13:43:35.111 | INFO | MemInfo: { | 20260508 13:43:35.111 | INFO | "gpu" : " 17 GB", | 20260508 13:43:35.111 | INFO | "huge" : "", | 20260508 13:43:35.111 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:35.111 | INFO | "malloc" : " 14 GB", | 20260508 13:43:35.111 | INFO | "pinned" : " 29 GB" | 20260508 13:43:35.111 | INFO | } | 20260508 13:43:35.111 | INFO | Perf report for the last 5.13126 seconds: | 20260508 13:43:35.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:35.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | ANALYSIS : 64.7%, 3.0, 100.9ms, 0 | 20260508 13:43:35.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | PRE_HQ : 23.3%, 1.0, 12.1ms, 0 | 20260508 13:43:35.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:35.111 | INFO | Uploaded 36GiB to GPU in 2.32782s (15.4651GiB/s) | 20260508 13:43:35.112 | INFO | Downloaded 34GiB to GPU in 2.4535s (13.8578GiB/s) | 20260508 13:43:35.209 | INFO | Starting a round of Baz IO | 20260508 13:43:38.607 | INFO | Populating RTMetrics report | 20260508 13:43:38.607 | INFO | RTMetrics report populated | 20260508 13:43:38.607 | INFO | Opening tmp RTMetrics json file | 20260508 13:43:38.618 | INFO | Serializing RTMetrics to json | 20260508 13:43:38.619 | INFO | Serialization done, swapping tmp file now... | 20260508 13:43:38.620 | INFO | RTMetrics done | 20260508 13:43:38.679 | INFO | Finished a round of Baz IO: Wrote 1.381 GiB in 3.47 seconds | 20260508 13:43:38.679 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.00% / 60.07% | 20260508 13:43:38.679 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136006% / 0% / 7.15256e-05% | 20260508 13:43:38.679 | INFO | Baz IO overall: | 20260508 13:43:38.680 | INFO | BytesWritten : 3640200543422 | 20260508 13:43:38.680 | INFO | NumEvents : 1744930039004 | 20260508 13:43:40.256 | INFO | Analyzing chunk frames = [8556544,8557056) | 20260508 13:43:40.256 | INFO | MemInfo: { | 20260508 13:43:40.256 | INFO | "gpu" : " 17 GB", | 20260508 13:43:40.256 | INFO | "huge" : "", | 20260508 13:43:40.256 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:40.256 | INFO | "malloc" : " 14 GB", | 20260508 13:43:40.256 | INFO | "pinned" : " 29 GB" | 20260508 13:43:40.256 | INFO | } | 20260508 13:43:40.256 | INFO | Perf report for the last 5.14484 seconds: | 20260508 13:43:40.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:40.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:40.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:40.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:40.256 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 13:43:40.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:40.256 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:43:40.256 | INFO | PRE_HQ : 65.1%, 1.6, 53.5ms, 0 | 20260508 13:43:40.256 | INFO | BAZWRITER : 66.8%, 2.5, 85.1ms, 0 | 20260508 13:43:40.256 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:43:40.256 | INFO | Uploaded 36GiB to GPU in 2.63175s (13.6791GiB/s) | 20260508 13:43:40.256 | INFO | Downloaded 38GiB to GPU in 2.7207s (13.967GiB/s) | 20260508 13:43:45.396 | INFO | Analyzing chunk frames = [8557056,8557568) | 20260508 13:43:45.396 | INFO | MemInfo: { | 20260508 13:43:45.396 | INFO | "gpu" : " 17 GB", | 20260508 13:43:45.396 | INFO | "huge" : "", | 20260508 13:43:45.396 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:45.396 | INFO | "malloc" : " 14 GB", | 20260508 13:43:45.396 | INFO | "pinned" : " 29 GB" | 20260508 13:43:45.396 | INFO | } | 20260508 13:43:45.396 | INFO | Perf report for the last 5.14044 seconds: | 20260508 13:43:45.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:45.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.396 | INFO | ANALYSIS : 66.4%, 3.0, 103.5ms, 0 | 20260508 13:43:45.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.397 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:43:45.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:45.397 | INFO | Uploaded 36GiB to GPU in 2.48246s (14.5017GiB/s) | 20260508 13:43:45.397 | INFO | Downloaded 34GiB to GPU in 2.45929s (13.8252GiB/s) | 20260508 13:43:50.540 | INFO | Analyzing chunk frames = [8557568,8558080) | 20260508 13:43:50.540 | INFO | MemInfo: { | 20260508 13:43:50.540 | INFO | "gpu" : " 17 GB", | 20260508 13:43:50.540 | INFO | "huge" : "", | 20260508 13:43:50.540 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:50.540 | INFO | "malloc" : " 14 GB", | 20260508 13:43:50.540 | INFO | "pinned" : " 29 GB" | 20260508 13:43:50.540 | INFO | } | 20260508 13:43:50.540 | INFO | Perf report for the last 5.14396 seconds: | 20260508 13:43:50.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:50.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.540 | INFO | ANALYSIS : 66.5%, 3.0, 103.4ms, 0 | 20260508 13:43:50.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.540 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:43:50.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:50.541 | INFO | Uploaded 36GiB to GPU in 2.52051s (14.2828GiB/s) | 20260508 13:43:50.541 | INFO | Downloaded 34GiB to GPU in 2.45367s (13.8568GiB/s) | 20260508 13:43:55.677 | INFO | Analyzing chunk frames = [8558080,8558592) | 20260508 13:43:55.677 | INFO | MemInfo: { | 20260508 13:43:55.677 | INFO | "gpu" : " 17 GB", | 20260508 13:43:55.677 | INFO | "huge" : "", | 20260508 13:43:55.677 | INFO | "huge_shared" : " 29 GB", | 20260508 13:43:55.677 | INFO | "malloc" : " 14 GB", | 20260508 13:43:55.677 | INFO | "pinned" : " 29 GB" | 20260508 13:43:55.677 | INFO | } | 20260508 13:43:55.677 | INFO | Perf report for the last 5.13674 seconds: | 20260508 13:43:55.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:43:55.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.677 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:43:55.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.677 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:43:55.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:43:55.678 | INFO | Uploaded 36GiB to GPU in 2.45247s (14.6791GiB/s) | 20260508 13:43:55.678 | INFO | Downloaded 34GiB to GPU in 2.47038s (13.763GiB/s) | 20260508 13:44:00.814 | INFO | Analyzing chunk frames = [8558592,8559104) | 20260508 13:44:00.814 | INFO | MemInfo: { | 20260508 13:44:00.814 | INFO | "gpu" : " 17 GB", | 20260508 13:44:00.814 | INFO | "huge" : "", | 20260508 13:44:00.814 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:00.814 | INFO | "malloc" : " 14 GB", | 20260508 13:44:00.814 | INFO | "pinned" : " 29 GB" | 20260508 13:44:00.814 | INFO | } | 20260508 13:44:00.814 | INFO | Perf report for the last 5.13687 seconds: | 20260508 13:44:00.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:00.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:44:00.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:44:00.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:00.814 | INFO | Uploaded 36GiB to GPU in 2.39044s (15.06GiB/s) | 20260508 13:44:00.814 | INFO | Downloaded 34GiB to GPU in 2.49904s (13.6052GiB/s) | 20260508 13:44:05.954 | INFO | Analyzing chunk frames = [8559104,8559616) | 20260508 13:44:05.954 | INFO | MemInfo: { | 20260508 13:44:05.954 | INFO | "gpu" : " 17 GB", | 20260508 13:44:05.954 | INFO | "huge" : "", | 20260508 13:44:05.954 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:05.954 | INFO | "malloc" : " 14 GB", | 20260508 13:44:05.954 | INFO | "pinned" : " 29 GB" | 20260508 13:44:05.954 | INFO | } | 20260508 13:44:05.954 | INFO | Perf report for the last 5.14001 seconds: | 20260508 13:44:05.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:05.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.954 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:44:05.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.954 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:44:05.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:05.955 | INFO | Uploaded 36GiB to GPU in 2.47887s (14.5227GiB/s) | 20260508 13:44:05.955 | INFO | Downloaded 34GiB to GPU in 2.44807s (13.8885GiB/s) | 20260508 13:44:11.090 | INFO | Analyzing chunk frames = [8559616,8560128) | 20260508 13:44:11.090 | INFO | MemInfo: { | 20260508 13:44:11.090 | INFO | "gpu" : " 17 GB", | 20260508 13:44:11.090 | INFO | "huge" : "", | 20260508 13:44:11.090 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:11.090 | INFO | "malloc" : " 14 GB", | 20260508 13:44:11.090 | INFO | "pinned" : " 29 GB" | 20260508 13:44:11.090 | INFO | } | 20260508 13:44:11.090 | INFO | Perf report for the last 5.13582 seconds: | 20260508 13:44:11.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:11.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | ANALYSIS : 65.7%, 3.0, 102.3ms, 0 | 20260508 13:44:11.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:44:11.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:11.090 | INFO | Uploaded 36GiB to GPU in 2.5012s (14.3931GiB/s) | 20260508 13:44:11.090 | INFO | Downloaded 34GiB to GPU in 2.4555s (13.8465GiB/s) | 20260508 13:44:16.234 | INFO | Analyzing chunk frames = [8560128,8560640) | 20260508 13:44:16.234 | INFO | MemInfo: { | 20260508 13:44:16.234 | INFO | "gpu" : " 17 GB", | 20260508 13:44:16.234 | INFO | "huge" : "", | 20260508 13:44:16.235 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:16.235 | INFO | "malloc" : " 14 GB", | 20260508 13:44:16.235 | INFO | "pinned" : " 29 GB" | 20260508 13:44:16.235 | INFO | } | 20260508 13:44:16.235 | INFO | Perf report for the last 5.14486 seconds: | 20260508 13:44:16.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:16.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | ANALYSIS : 65.4%, 3.0, 101.9ms, 0 | 20260508 13:44:16.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:44:16.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:16.235 | INFO | Uploaded 36GiB to GPU in 2.40874s (14.9455GiB/s) | 20260508 13:44:16.235 | INFO | Downloaded 34GiB to GPU in 2.42528s (14.019GiB/s) | 20260508 13:44:16.235 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:44:16.235 | INFO | Total mem usage: 16.7 GiB | 20260508 13:44:16.235 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:44:16.235 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:44:16.235 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:44:16.235 | INFO | High Water Breakdown: | 20260508 13:44:16.235 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:44:16.235 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:44:16.235 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:44:16.235 | INFO | ------------------------------------------ | 20260508 13:44:16.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:44:16.235 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:44:16.235 | INFO | Total mem usage: 26.5 GiB | 20260508 13:44:16.235 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:44:16.235 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:44:16.235 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:44:16.235 | INFO | High Water Breakdown: | 20260508 13:44:16.235 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:44:16.235 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:44:16.235 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:44:16.235 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:44:16.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:44:16.236 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:44:16.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:44:16.236 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:44:16.236 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:44:16.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:44:16.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:44:16.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:44:16.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:44:16.236 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:44:16.236 | INFO | ------------------------------------------ | 20260508 13:44:16.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:44:16.236 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:44:16.236 | INFO | Total mem usage: 27.8 GiB | 20260508 13:44:16.236 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:44:16.236 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:44:16.236 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:44:16.236 | INFO | High Water Breakdown: | 20260508 13:44:16.236 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:44:16.236 | INFO | ------------------------------------------ | 20260508 13:44:16.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:44:16.236 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:44:16.236 | INFO | Total mem usage: 15.7 GiB | 20260508 13:44:16.236 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:44:16.236 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:44:16.236 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:44:16.236 | INFO | High Water Breakdown: | 20260508 13:44:16.236 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:44:16.236 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:44:16.236 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:44:16.236 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:44:16.236 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:44:16.236 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:44:16.236 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:44:16.236 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:44:16.236 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:44:16.236 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:44:16.236 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:44:16.236 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:44:16.236 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:44:16.236 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:44:16.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:44:16.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:44:16.236 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:44:16.236 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:44:16.236 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:44:16.236 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:44:16.236 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:44:16.236 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:44:16.236 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:44:16.236 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:44:16.236 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:44:16.236 | INFO | ------------------------------------------ | 20260508 13:44:16.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:44:16.236 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:44:16.236 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:44:16.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:44:16.236 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.52e+01, 1.39e+01 / 5.47e-01 | 20260508 13:44:16.236 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.02e-01, 1.12e-02 / 5.30e-03 | 20260508 13:44:16.236 | INFO | PerfReport: 12.20%, Baseline -- 3.98e+00 / 4.37e+00, 5.95e-01 / 1.74e-01 | 20260508 13:44:16.236 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.46e+00, 2.09e-01 / 9.59e-02 | 20260508 13:44:16.236 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.53e+01, 3.71e+00 / 1.18e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.87e+01 / 2.03e+01, 1.81e+00 / 1.20e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.30e+00 / 2.36e+00, 3.98e-01 / 2.78e-01 | 20260508 13:44:16.236 | INFO | PerfReport: 2.92%, Metrics -- 9.53e-01 / 1.06e+00, 2.37e-01 / 6.77e-02 | 20260508 13:44:16.236 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.17e-01, 4.74e-02 / 2.09e-02 | 20260508 13:44:16.236 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:44:16.236 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:44:16.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:44:16.236 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 24.50%, Upload -- 2.57e+01 / 0.00e+00, 7.98e+00 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 24.78%, Download -- 2.60e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 31.14%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 2.21%, LockUpload -- 2.31e+00 / 0.00e+00, 5.57e+00 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 2.54%, LockDownload -- 2.66e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 14.82%, LockCompute -- 1.55e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:44:16.236 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260508 13:44:16.332 | INFO | Starting a round of Baz IO | 20260508 13:44:19.739 | INFO | Populating RTMetrics report | 20260508 13:44:19.739 | INFO | RTMetrics report populated | 20260508 13:44:19.739 | INFO | Opening tmp RTMetrics json file | 20260508 13:44:19.740 | INFO | Serializing RTMetrics to json | 20260508 13:44:19.741 | INFO | Serialization done, swapping tmp file now... | 20260508 13:44:19.742 | INFO | RTMetrics done | 20260508 13:44:19.800 | INFO | Finished a round of Baz IO: Wrote 1.38 GiB in 3.468 seconds | 20260508 13:44:19.800 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.07% | 20260508 13:44:19.800 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135839% / 7.94729e-06% / 5.5631e-05% | 20260508 13:44:19.800 | INFO | Baz IO overall: | 20260508 13:44:19.800 | INFO | BytesWritten : 3641682479485 | 20260508 13:44:19.800 | INFO | NumEvents : 1745623419211 | 20260508 13:44:21.371 | INFO | Analyzing chunk frames = [8560640,8561152) | 20260508 13:44:21.371 | INFO | MemInfo: { | 20260508 13:44:21.371 | INFO | "gpu" : " 17 GB", | 20260508 13:44:21.371 | INFO | "huge" : "", | 20260508 13:44:21.371 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:21.371 | INFO | "malloc" : " 14 GB", | 20260508 13:44:21.371 | INFO | "pinned" : " 29 GB" | 20260508 13:44:21.371 | INFO | } | 20260508 13:44:21.371 | INFO | Perf report for the last 5.13607 seconds: | 20260508 13:44:21.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:21.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:21.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:21.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:21.371 | INFO | ANALYSIS : 67.9%, 3.0, 105.4ms, 0 | 20260508 13:44:21.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:21.371 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 13:44:21.371 | INFO | PRE_HQ : 66.0%, 1.6, 56.4ms, 0 | 20260508 13:44:21.371 | INFO | BAZWRITER : 66.9%, 2.5, 87.5ms, 0 | 20260508 13:44:21.371 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:44:21.372 | INFO | Uploaded 36GiB to GPU in 2.59297s (13.8837GiB/s) | 20260508 13:44:21.372 | INFO | Downloaded 38GiB to GPU in 2.68742s (14.14GiB/s) | 20260508 13:44:26.509 | INFO | Analyzing chunk frames = [8561152,8561664) | 20260508 13:44:26.509 | INFO | MemInfo: { | 20260508 13:44:26.509 | INFO | "gpu" : " 17 GB", | 20260508 13:44:26.509 | INFO | "huge" : "", | 20260508 13:44:26.509 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:26.509 | INFO | "malloc" : " 14 GB", | 20260508 13:44:26.509 | INFO | "pinned" : " 29 GB" | 20260508 13:44:26.509 | INFO | } | 20260508 13:44:26.509 | INFO | Perf report for the last 5.13857 seconds: | 20260508 13:44:26.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:26.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.509 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:44:26.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.510 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:44:26.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:26.510 | INFO | Uploaded 36GiB to GPU in 2.49068s (14.4539GiB/s) | 20260508 13:44:26.510 | INFO | Downloaded 34GiB to GPU in 2.46521s (13.7919GiB/s) | 20260508 13:44:31.655 | INFO | Analyzing chunk frames = [8561664,8562176) | 20260508 13:44:31.655 | INFO | MemInfo: { | 20260508 13:44:31.655 | INFO | "gpu" : " 17 GB", | 20260508 13:44:31.655 | INFO | "huge" : "", | 20260508 13:44:31.655 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:31.656 | INFO | "malloc" : " 14 GB", | 20260508 13:44:31.656 | INFO | "pinned" : " 29 GB" | 20260508 13:44:31.656 | INFO | } | 20260508 13:44:31.656 | INFO | Perf report for the last 5.14632 seconds: | 20260508 13:44:31.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:31.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 13:44:31.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:44:31.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:31.656 | INFO | Uploaded 36GiB to GPU in 2.51525s (14.3127GiB/s) | 20260508 13:44:31.656 | INFO | Downloaded 34GiB to GPU in 2.44709s (13.8941GiB/s) | 20260508 13:44:36.787 | INFO | Analyzing chunk frames = [8562176,8562688) | 20260508 13:44:36.787 | INFO | MemInfo: { | 20260508 13:44:36.787 | INFO | "gpu" : " 17 GB", | 20260508 13:44:36.787 | INFO | "huge" : "", | 20260508 13:44:36.787 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:36.787 | INFO | "malloc" : " 14 GB", | 20260508 13:44:36.787 | INFO | "pinned" : " 29 GB" | 20260508 13:44:36.787 | INFO | } | 20260508 13:44:36.787 | INFO | Perf report for the last 5.13142 seconds: | 20260508 13:44:36.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:36.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.787 | INFO | ANALYSIS : 65.5%, 3.0, 101.5ms, 0 | 20260508 13:44:36.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.787 | INFO | PRE_HQ : 22.7%, 1.0, 11.7ms, 0 | 20260508 13:44:36.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:36.788 | INFO | Uploaded 36GiB to GPU in 2.44089s (14.7487GiB/s) | 20260508 13:44:36.788 | INFO | Downloaded 34GiB to GPU in 2.47699s (13.7263GiB/s) | 20260508 13:44:41.925 | INFO | Analyzing chunk frames = [8562688,8563200) | 20260508 13:44:41.925 | INFO | MemInfo: { | 20260508 13:44:41.925 | INFO | "gpu" : " 17 GB", | 20260508 13:44:41.925 | INFO | "huge" : "", | 20260508 13:44:41.925 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:41.925 | INFO | "malloc" : " 14 GB", | 20260508 13:44:41.925 | INFO | "pinned" : " 29 GB" | 20260508 13:44:41.925 | INFO | } | 20260508 13:44:41.925 | INFO | Perf report for the last 5.13805 seconds: | 20260508 13:44:41.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:41.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.925 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:44:41.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.925 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 13:44:41.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:41.926 | INFO | Uploaded 36GiB to GPU in 2.45141s (14.6854GiB/s) | 20260508 13:44:41.926 | INFO | Downloaded 34GiB to GPU in 2.51428s (13.5228GiB/s) | 20260508 13:44:47.063 | INFO | Analyzing chunk frames = [8563200,8563712) | 20260508 13:44:47.063 | INFO | MemInfo: { | 20260508 13:44:47.063 | INFO | "gpu" : " 17 GB", | 20260508 13:44:47.063 | INFO | "huge" : "", | 20260508 13:44:47.063 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:47.063 | INFO | "malloc" : " 14 GB", | 20260508 13:44:47.063 | INFO | "pinned" : " 29 GB" | 20260508 13:44:47.063 | INFO | } | 20260508 13:44:47.063 | INFO | Perf report for the last 5.13766 seconds: | 20260508 13:44:47.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:47.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:44:47.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:44:47.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:47.063 | INFO | Uploaded 36GiB to GPU in 2.41387s (14.9138GiB/s) | 20260508 13:44:47.063 | INFO | Downloaded 34GiB to GPU in 2.42676s (14.0104GiB/s) | 20260508 13:44:52.209 | INFO | Analyzing chunk frames = [8563712,8564224) | 20260508 13:44:52.209 | INFO | MemInfo: { | 20260508 13:44:52.209 | INFO | "gpu" : " 17 GB", | 20260508 13:44:52.209 | INFO | "huge" : "", | 20260508 13:44:52.209 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:52.209 | INFO | "malloc" : " 14 GB", | 20260508 13:44:52.209 | INFO | "pinned" : " 29 GB" | 20260508 13:44:52.209 | INFO | } | 20260508 13:44:52.209 | INFO | Perf report for the last 5.14639 seconds: | 20260508 13:44:52.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:52.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.209 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:44:52.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.209 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260508 13:44:52.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:52.210 | INFO | Uploaded 36GiB to GPU in 2.48232s (14.5026GiB/s) | 20260508 13:44:52.210 | INFO | Downloaded 34GiB to GPU in 2.44593s (13.9007GiB/s) | 20260508 13:44:57.350 | INFO | Analyzing chunk frames = [8564224,8564736) | 20260508 13:44:57.350 | INFO | MemInfo: { | 20260508 13:44:57.350 | INFO | "gpu" : " 17 GB", | 20260508 13:44:57.350 | INFO | "huge" : "", | 20260508 13:44:57.350 | INFO | "huge_shared" : " 29 GB", | 20260508 13:44:57.350 | INFO | "malloc" : " 14 GB", | 20260508 13:44:57.350 | INFO | "pinned" : " 29 GB" | 20260508 13:44:57.350 | INFO | } | 20260508 13:44:57.350 | INFO | Perf report for the last 5.14057 seconds: | 20260508 13:44:57.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:44:57.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | ANALYSIS : 64.5%, 3.0, 100.9ms, 0 | 20260508 13:44:57.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 13:44:57.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:44:57.350 | INFO | Uploaded 36GiB to GPU in 2.38083s (15.1207GiB/s) | 20260508 13:44:57.350 | INFO | Downloaded 34GiB to GPU in 2.41342s (14.0879GiB/s) | 20260508 13:44:57.447 | INFO | Starting a round of Baz IO | 20260508 13:45:00.848 | INFO | Populating RTMetrics report | 20260508 13:45:00.849 | INFO | RTMetrics report populated | 20260508 13:45:00.849 | INFO | Opening tmp RTMetrics json file | 20260508 13:45:00.862 | INFO | Serializing RTMetrics to json | 20260508 13:45:00.863 | INFO | Serialization done, swapping tmp file now... | 20260508 13:45:00.864 | INFO | RTMetrics done | 20260508 13:45:00.922 | INFO | Finished a round of Baz IO: Wrote 1.38 GiB in 3.475 seconds | 20260508 13:45:00.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:45:00.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135966% / 3.97364e-06% / 5.16574e-05% | 20260508 13:45:00.922 | INFO | Baz IO overall: | 20260508 13:45:00.922 | INFO | BytesWritten : 3643163814047 | 20260508 13:45:00.922 | INFO | NumEvents : 1746316452116 | 20260508 13:45:02.484 | INFO | Analyzing chunk frames = [8564736,8565248) | 20260508 13:45:02.484 | INFO | MemInfo: { | 20260508 13:45:02.484 | INFO | "gpu" : " 17 GB", | 20260508 13:45:02.484 | INFO | "huge" : "", | 20260508 13:45:02.484 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:02.484 | INFO | "malloc" : " 14 GB", | 20260508 13:45:02.484 | INFO | "pinned" : " 29 GB" | 20260508 13:45:02.484 | INFO | } | 20260508 13:45:02.484 | INFO | Perf report for the last 5.13445 seconds: | 20260508 13:45:02.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:02.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:02.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:02.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:02.484 | INFO | ANALYSIS : 67.5%, 3.0, 105.4ms, 0 | 20260508 13:45:02.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:02.484 | INFO | RT_METRICS : 38.2%, 1.0, 20.0ms, 0 | 20260508 13:45:02.484 | INFO | PRE_HQ : 65.3%, 1.6, 53.8ms, 0 | 20260508 13:45:02.484 | INFO | BAZWRITER : 67.1%, 2.4, 84.2ms, 0 | 20260508 13:45:02.485 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:45:02.485 | INFO | Uploaded 36GiB to GPU in 2.62736s (13.702GiB/s) | 20260508 13:45:02.485 | INFO | Downloaded 38GiB to GPU in 2.72767s (13.9313GiB/s) | 20260508 13:45:07.619 | INFO | Analyzing chunk frames = [8565248,8565760) | 20260508 13:45:07.619 | INFO | MemInfo: { | 20260508 13:45:07.619 | INFO | "gpu" : " 17 GB", | 20260508 13:45:07.619 | INFO | "huge" : "", | 20260508 13:45:07.619 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:07.620 | INFO | "malloc" : " 14 GB", | 20260508 13:45:07.620 | INFO | "pinned" : " 29 GB" | 20260508 13:45:07.620 | INFO | } | 20260508 13:45:07.620 | INFO | Perf report for the last 5.13534 seconds: | 20260508 13:45:07.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:07.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:45:07.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:45:07.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:07.620 | INFO | Uploaded 36GiB to GPU in 2.48951s (14.4607GiB/s) | 20260508 13:45:07.620 | INFO | Downloaded 34GiB to GPU in 2.46348s (13.8016GiB/s) | 20260508 13:45:12.766 | INFO | Analyzing chunk frames = [8565760,8566272) | 20260508 13:45:12.766 | INFO | MemInfo: { | 20260508 13:45:12.766 | INFO | "gpu" : " 17 GB", | 20260508 13:45:12.766 | INFO | "huge" : "", | 20260508 13:45:12.766 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:12.766 | INFO | "malloc" : " 14 GB", | 20260508 13:45:12.766 | INFO | "pinned" : " 29 GB" | 20260508 13:45:12.766 | INFO | } | 20260508 13:45:12.766 | INFO | Perf report for the last 5.14628 seconds: | 20260508 13:45:12.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:12.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.766 | INFO | ANALYSIS : 66.4%, 3.0, 103.3ms, 0 | 20260508 13:45:12.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.766 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:45:12.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:12.767 | INFO | Uploaded 36GiB to GPU in 2.52123s (14.2788GiB/s) | 20260508 13:45:12.767 | INFO | Downloaded 34GiB to GPU in 2.44978s (13.8788GiB/s) | 20260508 13:45:17.905 | INFO | Analyzing chunk frames = [8566272,8566784) | 20260508 13:45:17.905 | INFO | MemInfo: { | 20260508 13:45:17.905 | INFO | "gpu" : " 17 GB", | 20260508 13:45:17.905 | INFO | "huge" : "", | 20260508 13:45:17.905 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:17.905 | INFO | "malloc" : " 14 GB", | 20260508 13:45:17.905 | INFO | "pinned" : " 29 GB" | 20260508 13:45:17.905 | INFO | } | 20260508 13:45:17.905 | INFO | Perf report for the last 5.13937 seconds: | 20260508 13:45:17.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:17.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.905 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 13:45:17.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.906 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:45:17.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:17.906 | INFO | Uploaded 36GiB to GPU in 2.54107s (14.1673GiB/s) | 20260508 13:45:17.906 | INFO | Downloaded 34GiB to GPU in 2.46434s (13.7968GiB/s) | 20260508 13:45:23.044 | INFO | Analyzing chunk frames = [8566784,8567296) | 20260508 13:45:23.044 | INFO | MemInfo: { | 20260508 13:45:23.044 | INFO | "gpu" : " 17 GB", | 20260508 13:45:23.044 | INFO | "huge" : "", | 20260508 13:45:23.044 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:23.044 | INFO | "malloc" : " 14 GB", | 20260508 13:45:23.044 | INFO | "pinned" : " 29 GB" | 20260508 13:45:23.044 | INFO | } | 20260508 13:45:23.045 | INFO | Perf report for the last 5.13906 seconds: | 20260508 13:45:23.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:23.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:45:23.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:45:23.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:23.045 | INFO | Uploaded 36GiB to GPU in 2.36861s (15.1988GiB/s) | 20260508 13:45:23.045 | INFO | Downloaded 34GiB to GPU in 2.49216s (13.6428GiB/s) | 20260508 13:45:28.183 | INFO | Analyzing chunk frames = [8567296,8567808) | 20260508 13:45:28.183 | INFO | MemInfo: { | 20260508 13:45:28.183 | INFO | "gpu" : " 17 GB", | 20260508 13:45:28.183 | INFO | "huge" : "", | 20260508 13:45:28.183 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:28.183 | INFO | "malloc" : " 14 GB", | 20260508 13:45:28.183 | INFO | "pinned" : " 29 GB" | 20260508 13:45:28.183 | INFO | } | 20260508 13:45:28.183 | INFO | Perf report for the last 5.13868 seconds: | 20260508 13:45:28.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:28.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:45:28.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260508 13:45:28.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:28.183 | INFO | Uploaded 36GiB to GPU in 2.45331s (14.6741GiB/s) | 20260508 13:45:28.183 | INFO | Downloaded 34GiB to GPU in 2.43727s (13.95GiB/s) | 20260508 13:45:33.322 | INFO | Analyzing chunk frames = [8567808,8568320) | 20260508 13:45:33.322 | INFO | MemInfo: { | 20260508 13:45:33.322 | INFO | "gpu" : " 17 GB", | 20260508 13:45:33.322 | INFO | "huge" : "", | 20260508 13:45:33.322 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:33.322 | INFO | "malloc" : " 14 GB", | 20260508 13:45:33.322 | INFO | "pinned" : " 29 GB" | 20260508 13:45:33.322 | INFO | } | 20260508 13:45:33.322 | INFO | Perf report for the last 5.13871 seconds: | 20260508 13:45:33.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:33.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:45:33.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:45:33.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:33.322 | INFO | Uploaded 36GiB to GPU in 2.32571s (15.4792GiB/s) | 20260508 13:45:33.322 | INFO | Downloaded 34GiB to GPU in 2.39516s (14.1953GiB/s) | 20260508 13:45:38.462 | INFO | Analyzing chunk frames = [8568320,8568832) | 20260508 13:45:38.462 | INFO | MemInfo: { | 20260508 13:45:38.462 | INFO | "gpu" : " 17 GB", | 20260508 13:45:38.462 | INFO | "huge" : "", | 20260508 13:45:38.462 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:38.462 | INFO | "malloc" : " 14 GB", | 20260508 13:45:38.462 | INFO | "pinned" : " 29 GB" | 20260508 13:45:38.462 | INFO | } | 20260508 13:45:38.462 | INFO | Perf report for the last 5.14042 seconds: | 20260508 13:45:38.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:38.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.462 | INFO | ANALYSIS : 64.8%, 3.0, 101.4ms, 0 | 20260508 13:45:38.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.462 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:45:38.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:38.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:45:38.463 | INFO | Total mem usage: 16.7 GiB | 20260508 13:45:38.463 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:45:38.463 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:45:38.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:45:38.463 | INFO | High Water Breakdown: | 20260508 13:45:38.463 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:45:38.463 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:45:38.463 | INFO | Uploaded 36GiB to GPU in 2.39782s (15.0136GiB/s) | 20260508 13:45:38.463 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:45:38.463 | INFO | ------------------------------------------ | 20260508 13:45:38.463 | INFO | Downloaded 34GiB to GPU in 2.43243s (13.9778GiB/s) | 20260508 13:45:38.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:45:38.463 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:45:38.463 | INFO | Total mem usage: 26.5 GiB | 20260508 13:45:38.463 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:45:38.463 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:45:38.463 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:45:38.463 | INFO | High Water Breakdown: | 20260508 13:45:38.463 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:45:38.463 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:45:38.463 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:45:38.463 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:45:38.463 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:45:38.463 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:45:38.463 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:45:38.463 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:45:38.463 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:45:38.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:45:38.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:45:38.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:45:38.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:45:38.463 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:45:38.463 | INFO | ------------------------------------------ | 20260508 13:45:38.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:45:38.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:45:38.463 | INFO | Total mem usage: 27.8 GiB | 20260508 13:45:38.463 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:45:38.463 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:45:38.463 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:45:38.463 | INFO | High Water Breakdown: | 20260508 13:45:38.463 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:45:38.463 | INFO | ------------------------------------------ | 20260508 13:45:38.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:45:38.463 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:45:38.463 | INFO | Total mem usage: 15.7 GiB | 20260508 13:45:38.463 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:45:38.463 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:45:38.463 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:45:38.464 | INFO | High Water Breakdown: | 20260508 13:45:38.464 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:45:38.464 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:45:38.464 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:45:38.464 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:45:38.464 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:45:38.464 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:45:38.464 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:45:38.464 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:45:38.464 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:45:38.464 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:45:38.464 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:45:38.464 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:45:38.464 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:45:38.464 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:45:38.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:45:38.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:45:38.464 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:45:38.464 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:45:38.464 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:45:38.464 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:45:38.464 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:45:38.464 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:45:38.464 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:45:38.464 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:45:38.464 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:45:38.464 | INFO | ------------------------------------------ | 20260508 13:45:38.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:45:38.464 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:45:38.464 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:45:38.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:45:38.464 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.53e+01, 1.40e+01 / 5.34e-01 | 20260508 13:45:38.464 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.02e-01, 1.09e-02 / 4.58e-03 | 20260508 13:45:38.464 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.33e+00, 6.00e-01 / 1.77e-01 | 20260508 13:45:38.464 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.45e+00, 2.11e-01 / 8.00e-02 | 20260508 13:45:38.464 | INFO | PerfReport: 16.01%, DME -- 4.18e+01 / 4.55e+01, 3.60e+00 / 1.02e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.86e+01 / 2.01e+01, 1.82e+00 / 1.11e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.31e+00 / 2.50e+00, 3.99e-01 / 1.52e-01 | 20260508 13:45:38.464 | INFO | PerfReport: 2.93%, Metrics -- 9.54e-01 / 1.08e+00, 2.39e-01 / 9.81e-02 | 20260508 13:45:38.464 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.15e-01, 4.64e-02 / 1.69e-02 | 20260508 13:45:38.464 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:45:38.464 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:45:38.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:45:38.464 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.34e+01 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 24.55%, Upload -- 2.57e+01 / 0.00e+00, 7.95e+00 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 24.70%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 31.13%, Compute -- 3.26e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 2.29%, LockUpload -- 2.40e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 2.49%, LockDownload -- 2.61e+00 / 0.00e+00, 3.95e+00 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 14.83%, LockCompute -- 1.55e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 13:45:38.464 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260508 13:45:38.559 | INFO | Starting a round of Baz IO | 20260508 13:45:41.960 | INFO | Populating RTMetrics report | 20260508 13:45:41.960 | INFO | RTMetrics report populated | 20260508 13:45:41.960 | INFO | Opening tmp RTMetrics json file | 20260508 13:45:41.969 | INFO | Serializing RTMetrics to json | 20260508 13:45:41.970 | INFO | Serialization done, swapping tmp file now... | 20260508 13:45:41.971 | INFO | RTMetrics done | 20260508 13:45:42.031 | INFO | Finished a round of Baz IO: Wrote 1.38 GiB in 3.471 seconds | 20260508 13:45:42.031 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:45:42.031 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135648% / 3.97364e-06% / 7.54992e-05% | 20260508 13:45:42.031 | INFO | Baz IO overall: | 20260508 13:45:42.031 | INFO | BytesWritten : 3644645051212 | 20260508 13:45:42.031 | INFO | NumEvents : 1747009505177 | 20260508 13:45:43.601 | INFO | Analyzing chunk frames = [8568832,8569344) | 20260508 13:45:43.601 | INFO | MemInfo: { | 20260508 13:45:43.601 | INFO | "gpu" : " 17 GB", | 20260508 13:45:43.601 | INFO | "huge" : "", | 20260508 13:45:43.601 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:43.601 | INFO | "malloc" : " 14 GB", | 20260508 13:45:43.601 | INFO | "pinned" : " 29 GB" | 20260508 13:45:43.601 | INFO | } | 20260508 13:45:43.601 | INFO | Perf report for the last 5.13875 seconds: | 20260508 13:45:43.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:43.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:43.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:43.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:43.601 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 13:45:43.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:43.601 | INFO | RT_METRICS : 38.0%, 1.0, 19.9ms, 0 | 20260508 13:45:43.601 | INFO | PRE_HQ : 65.1%, 1.6, 53.4ms, 0 | 20260508 13:45:43.601 | INFO | BAZWRITER : 66.8%, 2.4, 83.0ms, 0 | 20260508 13:45:43.601 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:45:43.602 | INFO | Uploaded 36GiB to GPU in 2.63739s (13.6499GiB/s) | 20260508 13:45:43.602 | INFO | Downloaded 38GiB to GPU in 2.72114s (13.9647GiB/s) | 20260508 13:45:48.743 | INFO | Analyzing chunk frames = [8569344,8569856) | 20260508 13:45:48.743 | INFO | MemInfo: { | 20260508 13:45:48.743 | INFO | "gpu" : " 17 GB", | 20260508 13:45:48.743 | INFO | "huge" : "", | 20260508 13:45:48.743 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:48.743 | INFO | "malloc" : " 14 GB", | 20260508 13:45:48.743 | INFO | "pinned" : " 29 GB" | 20260508 13:45:48.743 | INFO | } | 20260508 13:45:48.743 | INFO | Perf report for the last 5.14233 seconds: | 20260508 13:45:48.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:48.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.743 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:45:48.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.744 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:45:48.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:48.744 | INFO | Uploaded 36GiB to GPU in 2.47331s (14.5554GiB/s) | 20260508 13:45:48.744 | INFO | Downloaded 34GiB to GPU in 2.49005s (13.6544GiB/s) | 20260508 13:45:53.874 | INFO | Analyzing chunk frames = [8569856,8570368) | 20260508 13:45:53.874 | INFO | MemInfo: { | 20260508 13:45:53.874 | INFO | "gpu" : " 17 GB", | 20260508 13:45:53.874 | INFO | "huge" : "", | 20260508 13:45:53.874 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:53.874 | INFO | "malloc" : " 14 GB", | 20260508 13:45:53.875 | INFO | "pinned" : " 29 GB" | 20260508 13:45:53.875 | INFO | } | 20260508 13:45:53.875 | INFO | Perf report for the last 5.13113 seconds: | 20260508 13:45:53.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:53.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | ANALYSIS : 65.4%, 3.0, 101.5ms, 0 | 20260508 13:45:53.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:45:53.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:53.875 | INFO | Uploaded 36GiB to GPU in 2.41226s (14.9238GiB/s) | 20260508 13:45:53.875 | INFO | Downloaded 34GiB to GPU in 2.43454s (13.9657GiB/s) | 20260508 13:45:59.019 | INFO | Analyzing chunk frames = [8570368,8570880) | 20260508 13:45:59.019 | INFO | MemInfo: { | 20260508 13:45:59.019 | INFO | "gpu" : " 17 GB", | 20260508 13:45:59.019 | INFO | "huge" : "", | 20260508 13:45:59.019 | INFO | "huge_shared" : " 29 GB", | 20260508 13:45:59.019 | INFO | "malloc" : " 14 GB", | 20260508 13:45:59.019 | INFO | "pinned" : " 29 GB" | 20260508 13:45:59.019 | INFO | } | 20260508 13:45:59.019 | INFO | Perf report for the last 5.14433 seconds: | 20260508 13:45:59.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:45:59.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | ANALYSIS : 65.8%, 3.0, 102.1ms, 0 | 20260508 13:45:59.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | PRE_HQ : 23.5%, 1.0, 12.2ms, 0 | 20260508 13:45:59.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:45:59.019 | INFO | Uploaded 36GiB to GPU in 2.51131s (14.3351GiB/s) | 20260508 13:45:59.019 | INFO | Downloaded 34GiB to GPU in 2.46786s (13.7771GiB/s) | 20260508 13:46:04.161 | INFO | Analyzing chunk frames = [8570880,8571392) | 20260508 13:46:04.161 | INFO | MemInfo: { | 20260508 13:46:04.161 | INFO | "gpu" : " 17 GB", | 20260508 13:46:04.161 | INFO | "huge" : "", | 20260508 13:46:04.161 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:04.161 | INFO | "malloc" : " 14 GB", | 20260508 13:46:04.161 | INFO | "pinned" : " 29 GB" | 20260508 13:46:04.161 | INFO | } | 20260508 13:46:04.161 | INFO | Perf report for the last 5.14234 seconds: | 20260508 13:46:04.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:04.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.161 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 13:46:04.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.161 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:46:04.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:04.162 | INFO | Uploaded 36GiB to GPU in 2.31365s (15.5598GiB/s) | 20260508 13:46:04.162 | INFO | Downloaded 34GiB to GPU in 2.50582s (13.5684GiB/s) | 20260508 13:46:09.297 | INFO | Analyzing chunk frames = [8571392,8571904) | 20260508 13:46:09.297 | INFO | MemInfo: { | 20260508 13:46:09.297 | INFO | "gpu" : " 17 GB", | 20260508 13:46:09.297 | INFO | "huge" : "", | 20260508 13:46:09.297 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:09.297 | INFO | "malloc" : " 14 GB", | 20260508 13:46:09.297 | INFO | "pinned" : " 29 GB" | 20260508 13:46:09.297 | INFO | } | 20260508 13:46:09.298 | INFO | Perf report for the last 5.13625 seconds: | 20260508 13:46:09.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:09.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:46:09.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:46:09.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:09.298 | INFO | Uploaded 36GiB to GPU in 2.42372s (14.8532GiB/s) | 20260508 13:46:09.298 | INFO | Downloaded 34GiB to GPU in 2.45855s (13.8293GiB/s) | 20260508 13:46:14.435 | INFO | Analyzing chunk frames = [8571904,8572416) | 20260508 13:46:14.435 | INFO | MemInfo: { | 20260508 13:46:14.435 | INFO | "gpu" : " 17 GB", | 20260508 13:46:14.435 | INFO | "huge" : "", | 20260508 13:46:14.435 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:14.435 | INFO | "malloc" : " 14 GB", | 20260508 13:46:14.435 | INFO | "pinned" : " 29 GB" | 20260508 13:46:14.435 | INFO | } | 20260508 13:46:14.435 | INFO | Perf report for the last 5.13747 seconds: | 20260508 13:46:14.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:14.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.435 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:46:14.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.435 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:46:14.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:14.436 | INFO | Uploaded 36GiB to GPU in 2.31294s (15.5646GiB/s) | 20260508 13:46:14.436 | INFO | Downloaded 34GiB to GPU in 2.42673s (14.0106GiB/s) | 20260508 13:46:19.572 | INFO | Analyzing chunk frames = [8572416,8572928) | 20260508 13:46:19.572 | INFO | MemInfo: { | 20260508 13:46:19.572 | INFO | "gpu" : " 17 GB", | 20260508 13:46:19.572 | INFO | "huge" : "", | 20260508 13:46:19.572 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:19.572 | INFO | "malloc" : " 14 GB", | 20260508 13:46:19.572 | INFO | "pinned" : " 29 GB" | 20260508 13:46:19.572 | INFO | } | 20260508 13:46:19.572 | INFO | Perf report for the last 5.13731 seconds: | 20260508 13:46:19.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:19.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.572 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260508 13:46:19.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.572 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:46:19.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:19.573 | INFO | Uploaded 36GiB to GPU in 2.38552s (15.0911GiB/s) | 20260508 13:46:19.573 | INFO | Downloaded 34GiB to GPU in 2.43278s (13.9758GiB/s) | 20260508 13:46:19.670 | INFO | Starting a round of Baz IO | 20260508 13:46:23.068 | INFO | Populating RTMetrics report | 20260508 13:46:23.068 | INFO | RTMetrics report populated | 20260508 13:46:23.068 | INFO | Opening tmp RTMetrics json file | 20260508 13:46:23.076 | INFO | Serializing RTMetrics to json | 20260508 13:46:23.077 | INFO | Serialization done, swapping tmp file now... | 20260508 13:46:23.078 | INFO | RTMetrics done | 20260508 13:46:23.137 | INFO | Finished a round of Baz IO: Wrote 1.378 GiB in 3.467 seconds | 20260508 13:46:23.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:46:23.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135159% / 0% / 0.000103315% | 20260508 13:46:23.137 | INFO | Baz IO overall: | 20260508 13:46:23.137 | INFO | BytesWritten : 3646124863732 | 20260508 13:46:23.137 | INFO | NumEvents : 1747701177095 | 20260508 13:46:24.716 | INFO | Analyzing chunk frames = [8572928,8573440) | 20260508 13:46:24.716 | INFO | MemInfo: { | 20260508 13:46:24.716 | INFO | "gpu" : " 17 GB", | 20260508 13:46:24.716 | INFO | "huge" : "", | 20260508 13:46:24.716 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:24.716 | INFO | "malloc" : " 14 GB", | 20260508 13:46:24.716 | INFO | "pinned" : " 29 GB" | 20260508 13:46:24.716 | INFO | } | 20260508 13:46:24.716 | INFO | Perf report for the last 5.14353 seconds: | 20260508 13:46:24.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:24.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:24.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:24.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:24.716 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:46:24.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:24.716 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:46:24.716 | INFO | PRE_HQ : 65.0%, 1.6, 53.8ms, 0 | 20260508 13:46:24.716 | INFO | BAZWRITER : 66.7%, 2.4, 82.8ms, 0 | 20260508 13:46:24.716 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:46:24.716 | INFO | Uploaded 36GiB to GPU in 2.62718s (13.7029GiB/s) | 20260508 13:46:24.716 | INFO | Downloaded 38GiB to GPU in 2.71913s (13.9751GiB/s) | 20260508 13:46:29.858 | INFO | Analyzing chunk frames = [8573440,8573952) | 20260508 13:46:29.858 | INFO | MemInfo: { | 20260508 13:46:29.858 | INFO | "gpu" : " 17 GB", | 20260508 13:46:29.858 | INFO | "huge" : "", | 20260508 13:46:29.858 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:29.858 | INFO | "malloc" : " 14 GB", | 20260508 13:46:29.858 | INFO | "pinned" : " 29 GB" | 20260508 13:46:29.858 | INFO | } | 20260508 13:46:29.858 | INFO | Perf report for the last 5.14211 seconds: | 20260508 13:46:29.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:29.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | ANALYSIS : 65.9%, 3.0, 102.8ms, 0 | 20260508 13:46:29.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | PRE_HQ : 22.2%, 1.0, 11.6ms, 0 | 20260508 13:46:29.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:29.858 | INFO | Uploaded 36GiB to GPU in 2.45674s (14.6535GiB/s) | 20260508 13:46:29.858 | INFO | Downloaded 34GiB to GPU in 2.47713s (13.7256GiB/s) | 20260508 13:46:34.996 | INFO | Analyzing chunk frames = [8573952,8574464) | 20260508 13:46:34.996 | INFO | MemInfo: { | 20260508 13:46:34.996 | INFO | "gpu" : " 17 GB", | 20260508 13:46:34.996 | INFO | "huge" : "", | 20260508 13:46:34.996 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:34.996 | INFO | "malloc" : " 14 GB", | 20260508 13:46:34.997 | INFO | "pinned" : " 29 GB" | 20260508 13:46:34.997 | INFO | } | 20260508 13:46:34.997 | INFO | Perf report for the last 5.13863 seconds: | 20260508 13:46:34.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:34.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | ANALYSIS : 65.7%, 3.0, 102.0ms, 0 | 20260508 13:46:34.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260508 13:46:34.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:34.997 | INFO | Uploaded 36GiB to GPU in 2.44261s (14.7383GiB/s) | 20260508 13:46:34.997 | INFO | Downloaded 34GiB to GPU in 2.45247s (13.8636GiB/s) | 20260508 13:46:40.127 | INFO | Analyzing chunk frames = [8574464,8574976) | 20260508 13:46:40.127 | INFO | MemInfo: { | 20260508 13:46:40.127 | INFO | "gpu" : " 17 GB", | 20260508 13:46:40.127 | INFO | "huge" : "", | 20260508 13:46:40.127 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:40.127 | INFO | "malloc" : " 14 GB", | 20260508 13:46:40.127 | INFO | "pinned" : " 29 GB" | 20260508 13:46:40.127 | INFO | } | 20260508 13:46:40.127 | INFO | Perf report for the last 5.13039 seconds: | 20260508 13:46:40.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:40.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | ANALYSIS : 65.5%, 3.0, 101.5ms, 0 | 20260508 13:46:40.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:46:40.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:40.127 | INFO | Uploaded 36GiB to GPU in 2.39987s (15.0008GiB/s) | 20260508 13:46:40.127 | INFO | Downloaded 34GiB to GPU in 2.48443s (13.6852GiB/s) | 20260508 13:46:45.270 | INFO | Analyzing chunk frames = [8574976,8575488) | 20260508 13:46:45.270 | INFO | MemInfo: { | 20260508 13:46:45.270 | INFO | "gpu" : " 17 GB", | 20260508 13:46:45.270 | INFO | "huge" : "", | 20260508 13:46:45.270 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:45.270 | INFO | "malloc" : " 14 GB", | 20260508 13:46:45.270 | INFO | "pinned" : " 29 GB" | 20260508 13:46:45.270 | INFO | } | 20260508 13:46:45.270 | INFO | Perf report for the last 5.14288 seconds: | 20260508 13:46:45.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:45.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:46:45.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:46:45.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:45.270 | INFO | Uploaded 36GiB to GPU in 2.37622s (15.1501GiB/s) | 20260508 13:46:45.270 | INFO | Downloaded 34GiB to GPU in 2.50964s (13.5477GiB/s) | 20260508 13:46:50.416 | INFO | Analyzing chunk frames = [8575488,8576000) | 20260508 13:46:50.416 | INFO | MemInfo: { | 20260508 13:46:50.416 | INFO | "gpu" : " 17 GB", | 20260508 13:46:50.416 | INFO | "huge" : "", | 20260508 13:46:50.416 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:50.416 | INFO | "malloc" : " 14 GB", | 20260508 13:46:50.416 | INFO | "pinned" : " 29 GB" | 20260508 13:46:50.416 | INFO | } | 20260508 13:46:50.416 | INFO | Perf report for the last 5.14641 seconds: | 20260508 13:46:50.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:50.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.416 | INFO | ANALYSIS : 65.2%, 3.0, 101.6ms, 0 | 20260508 13:46:50.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.416 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:46:50.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:50.417 | INFO | Uploaded 36GiB to GPU in 2.40828s (14.9484GiB/s) | 20260508 13:46:50.417 | INFO | Downloaded 34GiB to GPU in 2.44926s (13.8817GiB/s) | 20260508 13:46:55.550 | INFO | Analyzing chunk frames = [8576000,8576512) | 20260508 13:46:55.550 | INFO | MemInfo: { | 20260508 13:46:55.550 | INFO | "gpu" : " 17 GB", | 20260508 13:46:55.550 | INFO | "huge" : "", | 20260508 13:46:55.550 | INFO | "huge_shared" : " 29 GB", | 20260508 13:46:55.550 | INFO | "malloc" : " 14 GB", | 20260508 13:46:55.550 | INFO | "pinned" : " 29 GB" | 20260508 13:46:55.551 | INFO | } | 20260508 13:46:55.551 | INFO | Perf report for the last 5.13421 seconds: | 20260508 13:46:55.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:46:55.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | ANALYSIS : 64.4%, 3.0, 100.1ms, 0 | 20260508 13:46:55.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:46:55.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:46:55.551 | INFO | Uploaded 36GiB to GPU in 2.28122s (15.781GiB/s) | 20260508 13:46:55.551 | INFO | Downloaded 34GiB to GPU in 2.40313s (14.1482GiB/s) | 20260508 13:47:00.691 | INFO | Analyzing chunk frames = [8576512,8577024) | 20260508 13:47:00.691 | INFO | MemInfo: { | 20260508 13:47:00.691 | INFO | "gpu" : " 17 GB", | 20260508 13:47:00.691 | INFO | "huge" : "", | 20260508 13:47:00.691 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:00.691 | INFO | "malloc" : " 14 GB", | 20260508 13:47:00.691 | INFO | "pinned" : " 29 GB" | 20260508 13:47:00.691 | INFO | } | 20260508 13:47:00.691 | INFO | Perf report for the last 5.14051 seconds: | 20260508 13:47:00.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:00.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | ANALYSIS : 65.7%, 3.0, 102.7ms, 0 | 20260508 13:47:00.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | PRE_HQ : 23.1%, 1.0, 12.0ms, 0 | 20260508 13:47:00.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:00.691 | INFO | Uploaded 36GiB to GPU in 2.47757s (14.5304GiB/s) | 20260508 13:47:00.691 | INFO | Downloaded 34GiB to GPU in 2.46641s (13.7852GiB/s) | 20260508 13:47:00.692 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:47:00.692 | INFO | Total mem usage: 16.7 GiB | 20260508 13:47:00.692 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:47:00.692 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.3%) | 20260508 13:47:00.692 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:47:00.692 | INFO | High Water Breakdown: | 20260508 13:47:00.692 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:47:00.692 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:47:00.692 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:47:00.692 | INFO | ------------------------------------------ | 20260508 13:47:00.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:47:00.692 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:47:00.692 | INFO | Total mem usage: 26.5 GiB | 20260508 13:47:00.692 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:47:00.692 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:47:00.692 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:47:00.692 | INFO | High Water Breakdown: | 20260508 13:47:00.692 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:47:00.692 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:47:00.692 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:47:00.692 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:47:00.692 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:47:00.692 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:47:00.692 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:47:00.692 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:47:00.692 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:47:00.692 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:47:00.692 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:47:00.692 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:47:00.692 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:47:00.692 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:47:00.692 | INFO | ------------------------------------------ | 20260508 13:47:00.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:47:00.692 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:47:00.692 | INFO | Total mem usage: 27.8 GiB | 20260508 13:47:00.692 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:47:00.692 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:47:00.692 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:47:00.692 | INFO | High Water Breakdown: | 20260508 13:47:00.692 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:47:00.692 | INFO | ------------------------------------------ | 20260508 13:47:00.692 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:47:00.692 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:47:00.692 | INFO | Total mem usage: 15.7 GiB | 20260508 13:47:00.692 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:47:00.692 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:47:00.692 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:47:00.692 | INFO | High Water Breakdown: | 20260508 13:47:00.692 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:47:00.692 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:47:00.692 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:47:00.692 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:47:00.692 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:47:00.692 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:47:00.692 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:47:00.692 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:47:00.692 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:47:00.692 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:47:00.692 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:47:00.693 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:47:00.693 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:47:00.693 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:47:00.693 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:47:00.693 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:47:00.693 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:47:00.693 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:47:00.693 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:47:00.693 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:47:00.693 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:47:00.693 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:47:00.693 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:47:00.693 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:47:00.693 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:47:00.693 | INFO | ------------------------------------------ | 20260508 13:47:00.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:47:00.693 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:47:00.693 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:47:00.693 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:47:00.693 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.39e+01 / 5.57e-01 | 20260508 13:47:00.693 | INFO | PerfReport: 0.30%, AggregateStats -- 9.69e-02 / 1.05e-01, 1.15e-02 / 5.22e-03 | 20260508 13:47:00.693 | INFO | PerfReport: 12.25%, Baseline -- 4.01e+00 / 4.39e+00, 5.93e-01 / 1.98e-01 | 20260508 13:47:00.693 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.42e+00, 2.08e-01 / 1.11e-01 | 20260508 13:47:00.693 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.58e+01, 3.73e+00 / 1.07e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.87e+01 / 1.98e+01, 1.87e+00 / 1.05e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.30e+00 / 2.43e+00, 4.01e-01 / 2.40e-01 | 20260508 13:47:00.693 | INFO | PerfReport: 2.90%, Metrics -- 9.50e-01 / 1.05e+00, 2.37e-01 / 5.47e-02 | 20260508 13:47:00.693 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.87e-01 / 3.13e-01, 4.60e-02 / 2.06e-02 | 20260508 13:47:00.693 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:47:00.693 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:47:00.693 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:47:00.693 | INFO | PerfReport: Full Execution -- 1.04e+02 / -nan, 2.31e+01 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 24.27%, Upload -- 2.54e+01 / 0.00e+00, 7.87e+00 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 24.87%, Download -- 2.60e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 31.32%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 2.05%, LockUpload -- 2.15e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 2.59%, LockDownload -- 2.71e+00 / 0.00e+00, 4.04e+00 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 14.89%, LockCompute -- 1.56e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260508 13:47:00.693 | INFO | PerfReport: 0.01%, Misc -- 1.23e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260508 13:47:00.788 | INFO | Starting a round of Baz IO | 20260508 13:47:04.195 | INFO | Populating RTMetrics report | 20260508 13:47:04.195 | INFO | RTMetrics report populated | 20260508 13:47:04.195 | INFO | Opening tmp RTMetrics json file | 20260508 13:47:04.204 | INFO | Serializing RTMetrics to json | 20260508 13:47:04.205 | INFO | Serialization done, swapping tmp file now... | 20260508 13:47:04.206 | INFO | RTMetrics done | 20260508 13:47:04.263 | INFO | Finished a round of Baz IO: Wrote 1.379 GiB in 3.474 seconds | 20260508 13:47:04.263 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:47:04.263 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135012% / 0% / 8.34465e-05% | 20260508 13:47:04.263 | INFO | Baz IO overall: | 20260508 13:47:04.263 | INFO | BytesWritten : 3647606018894 | 20260508 13:47:04.263 | INFO | NumEvents : 1748393825292 | 20260508 13:47:05.833 | INFO | Analyzing chunk frames = [8577024,8577536) | 20260508 13:47:05.833 | INFO | MemInfo: { | 20260508 13:47:05.833 | INFO | "gpu" : " 17 GB", | 20260508 13:47:05.833 | INFO | "huge" : "", | 20260508 13:47:05.833 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:05.833 | INFO | "malloc" : " 14 GB", | 20260508 13:47:05.833 | INFO | "pinned" : " 29 GB" | 20260508 13:47:05.833 | INFO | } | 20260508 13:47:05.833 | INFO | Perf report for the last 5.14159 seconds: | 20260508 13:47:05.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:05.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:05.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:05.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:05.833 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:47:05.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:05.833 | INFO | RT_METRICS : 39.6%, 1.0, 20.8ms, 0 | 20260508 13:47:05.833 | INFO | PRE_HQ : 65.4%, 1.7, 56.2ms, 0 | 20260508 13:47:05.833 | INFO | BAZWRITER : 66.8%, 2.5, 85.6ms, 0 | 20260508 13:47:05.833 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:47:05.833 | INFO | Uploaded 36GiB to GPU in 2.64351s (13.6182GiB/s) | 20260508 13:47:05.833 | INFO | Downloaded 38GiB to GPU in 2.71912s (13.9751GiB/s) | 20260508 13:47:10.973 | INFO | Analyzing chunk frames = [8577536,8578048) | 20260508 13:47:10.973 | INFO | MemInfo: { | 20260508 13:47:10.973 | INFO | "gpu" : " 17 GB", | 20260508 13:47:10.973 | INFO | "huge" : "", | 20260508 13:47:10.973 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:10.973 | INFO | "malloc" : " 14 GB", | 20260508 13:47:10.973 | INFO | "pinned" : " 29 GB" | 20260508 13:47:10.973 | INFO | } | 20260508 13:47:10.973 | INFO | Perf report for the last 5.1404 seconds: | 20260508 13:47:10.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:10.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.973 | INFO | ANALYSIS : 66.1%, 3.0, 103.0ms, 0 | 20260508 13:47:10.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.973 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:47:10.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:10.974 | INFO | Uploaded 36GiB to GPU in 2.50275s (14.3842GiB/s) | 20260508 13:47:10.974 | INFO | Downloaded 34GiB to GPU in 2.47514s (13.7366GiB/s) | 20260508 13:47:16.113 | INFO | Analyzing chunk frames = [8578048,8578560) | 20260508 13:47:16.113 | INFO | MemInfo: { | 20260508 13:47:16.113 | INFO | "gpu" : " 17 GB", | 20260508 13:47:16.113 | INFO | "huge" : "", | 20260508 13:47:16.113 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:16.113 | INFO | "malloc" : " 14 GB", | 20260508 13:47:16.114 | INFO | "pinned" : " 29 GB" | 20260508 13:47:16.114 | INFO | } | 20260508 13:47:16.114 | INFO | Perf report for the last 5.14044 seconds: | 20260508 13:47:16.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:16.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | ANALYSIS : 66.3%, 3.0, 103.1ms, 0 | 20260508 13:47:16.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | PRE_HQ : 22.4%, 1.0, 11.7ms, 0 | 20260508 13:47:16.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:16.114 | INFO | Uploaded 36GiB to GPU in 2.53292s (14.2129GiB/s) | 20260508 13:47:16.114 | INFO | Downloaded 34GiB to GPU in 2.44827s (13.8874GiB/s) | 20260508 13:47:21.249 | INFO | Analyzing chunk frames = [8578560,8579072) | 20260508 13:47:21.249 | INFO | MemInfo: { | 20260508 13:47:21.249 | INFO | "gpu" : " 17 GB", | 20260508 13:47:21.249 | INFO | "huge" : "", | 20260508 13:47:21.249 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:21.250 | INFO | "malloc" : " 14 GB", | 20260508 13:47:21.250 | INFO | "pinned" : " 29 GB" | 20260508 13:47:21.250 | INFO | } | 20260508 13:47:21.250 | INFO | Perf report for the last 5.13604 seconds: | 20260508 13:47:21.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:21.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:47:21.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:47:21.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:21.250 | INFO | Uploaded 36GiB to GPU in 2.43238s (14.8003GiB/s) | 20260508 13:47:21.250 | INFO | Downloaded 34GiB to GPU in 2.47057s (13.762GiB/s) | 20260508 13:47:26.387 | INFO | Analyzing chunk frames = [8579072,8579584) | 20260508 13:47:26.387 | INFO | MemInfo: { | 20260508 13:47:26.387 | INFO | "gpu" : " 17 GB", | 20260508 13:47:26.387 | INFO | "huge" : "", | 20260508 13:47:26.387 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:26.387 | INFO | "malloc" : " 14 GB", | 20260508 13:47:26.387 | INFO | "pinned" : " 29 GB" | 20260508 13:47:26.387 | INFO | } | 20260508 13:47:26.387 | INFO | Perf report for the last 5.1378 seconds: | 20260508 13:47:26.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:26.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.387 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:47:26.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.388 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:47:26.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:26.388 | INFO | Uploaded 36GiB to GPU in 2.41821s (14.8871GiB/s) | 20260508 13:47:26.388 | INFO | Downloaded 34GiB to GPU in 2.50444s (13.5759GiB/s) | 20260508 13:47:31.530 | INFO | Analyzing chunk frames = [8579584,8580096) | 20260508 13:47:31.530 | INFO | MemInfo: { | 20260508 13:47:31.530 | INFO | "gpu" : " 17 GB", | 20260508 13:47:31.530 | INFO | "huge" : "", | 20260508 13:47:31.530 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:31.530 | INFO | "malloc" : " 14 GB", | 20260508 13:47:31.530 | INFO | "pinned" : " 29 GB" | 20260508 13:47:31.530 | INFO | } | 20260508 13:47:31.530 | INFO | Perf report for the last 5.14276 seconds: | 20260508 13:47:31.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:31.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.530 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:47:31.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.530 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:47:31.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:31.531 | INFO | Uploaded 36GiB to GPU in 2.46644s (14.5959GiB/s) | 20260508 13:47:31.531 | INFO | Downloaded 34GiB to GPU in 2.44406s (13.9113GiB/s) | 20260508 13:47:36.667 | INFO | Analyzing chunk frames = [8580096,8580608) | 20260508 13:47:36.667 | INFO | MemInfo: { | 20260508 13:47:36.667 | INFO | "gpu" : " 17 GB", | 20260508 13:47:36.667 | INFO | "huge" : "", | 20260508 13:47:36.667 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:36.667 | INFO | "malloc" : " 14 GB", | 20260508 13:47:36.667 | INFO | "pinned" : " 29 GB" | 20260508 13:47:36.667 | INFO | } | 20260508 13:47:36.668 | INFO | Perf report for the last 5.13737 seconds: | 20260508 13:47:36.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:36.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 13:47:36.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:47:36.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:36.668 | INFO | Uploaded 36GiB to GPU in 2.38461s (15.0968GiB/s) | 20260508 13:47:36.668 | INFO | Downloaded 34GiB to GPU in 2.40012s (14.1659GiB/s) | 20260508 13:47:41.803 | INFO | Analyzing chunk frames = [8580608,8581120) | 20260508 13:47:41.803 | INFO | MemInfo: { | 20260508 13:47:41.803 | INFO | "gpu" : " 17 GB", | 20260508 13:47:41.803 | INFO | "huge" : "", | 20260508 13:47:41.803 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:41.803 | INFO | "malloc" : " 14 GB", | 20260508 13:47:41.803 | INFO | "pinned" : " 29 GB" | 20260508 13:47:41.803 | INFO | } | 20260508 13:47:41.803 | INFO | Perf report for the last 5.13527 seconds: | 20260508 13:47:41.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:41.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | ANALYSIS : 65.8%, 3.0, 102.7ms, 0 | 20260508 13:47:41.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:47:41.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:41.803 | INFO | Uploaded 36GiB to GPU in 2.49883s (14.4067GiB/s) | 20260508 13:47:41.804 | INFO | Downloaded 34GiB to GPU in 2.44972s (13.8792GiB/s) | 20260508 13:47:41.900 | INFO | Starting a round of Baz IO | 20260508 13:47:45.312 | INFO | Populating RTMetrics report | 20260508 13:47:45.313 | INFO | RTMetrics report populated | 20260508 13:47:45.313 | INFO | Opening tmp RTMetrics json file | 20260508 13:47:45.317 | INFO | Serializing RTMetrics to json | 20260508 13:47:45.318 | INFO | Serialization done, swapping tmp file now... | 20260508 13:47:45.319 | INFO | RTMetrics done | 20260508 13:47:45.379 | INFO | Finished a round of Baz IO: Wrote 1.378 GiB in 3.478 seconds | 20260508 13:47:45.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:47:45.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134178% / 0% / 6.35783e-05% | 20260508 13:47:45.379 | INFO | Baz IO overall: | 20260508 13:47:45.379 | INFO | BytesWritten : 3649086089879 | 20260508 13:47:45.379 | INFO | NumEvents : 1749085606106 | 20260508 13:47:46.944 | INFO | Analyzing chunk frames = [8581120,8581632) | 20260508 13:47:46.944 | INFO | MemInfo: { | 20260508 13:47:46.944 | INFO | "gpu" : " 17 GB", | 20260508 13:47:46.944 | INFO | "huge" : "", | 20260508 13:47:46.944 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:46.944 | INFO | "malloc" : " 14 GB", | 20260508 13:47:46.944 | INFO | "pinned" : " 29 GB" | 20260508 13:47:46.944 | INFO | } | 20260508 13:47:46.944 | INFO | Perf report for the last 5.14109 seconds: | 20260508 13:47:46.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:46.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:46.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:46.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:46.944 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 13:47:46.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:46.944 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:47:46.944 | INFO | PRE_HQ : 65.2%, 1.6, 53.6ms, 0 | 20260508 13:47:46.944 | INFO | BAZWRITER : 66.8%, 2.4, 83.1ms, 0 | 20260508 13:47:46.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:47:46.944 | INFO | Uploaded 36GiB to GPU in 2.63082s (13.6839GiB/s) | 20260508 13:47:46.944 | INFO | Downloaded 38GiB to GPU in 2.72365s (13.9519GiB/s) | 20260508 13:47:52.082 | INFO | Analyzing chunk frames = [8581632,8582144) | 20260508 13:47:52.082 | INFO | MemInfo: { | 20260508 13:47:52.082 | INFO | "gpu" : " 17 GB", | 20260508 13:47:52.082 | INFO | "huge" : "", | 20260508 13:47:52.082 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:52.082 | INFO | "malloc" : " 14 GB", | 20260508 13:47:52.082 | INFO | "pinned" : " 29 GB" | 20260508 13:47:52.082 | INFO | } | 20260508 13:47:52.082 | INFO | Perf report for the last 5.13834 seconds: | 20260508 13:47:52.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:52.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.082 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:47:52.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.082 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:47:52.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:52.083 | INFO | Uploaded 36GiB to GPU in 2.47394s (14.5517GiB/s) | 20260508 13:47:52.083 | INFO | Downloaded 34GiB to GPU in 2.49203s (13.6435GiB/s) | 20260508 13:47:57.221 | INFO | Analyzing chunk frames = [8582144,8582656) | 20260508 13:47:57.221 | INFO | MemInfo: { | 20260508 13:47:57.221 | INFO | "gpu" : " 17 GB", | 20260508 13:47:57.221 | INFO | "huge" : "", | 20260508 13:47:57.221 | INFO | "huge_shared" : " 29 GB", | 20260508 13:47:57.221 | INFO | "malloc" : " 14 GB", | 20260508 13:47:57.221 | INFO | "pinned" : " 29 GB" | 20260508 13:47:57.221 | INFO | } | 20260508 13:47:57.221 | INFO | Perf report for the last 5.13869 seconds: | 20260508 13:47:57.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:47:57.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:47:57.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:47:57.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:47:57.221 | INFO | Uploaded 36GiB to GPU in 2.50157s (14.391GiB/s) | 20260508 13:47:57.222 | INFO | Downloaded 34GiB to GPU in 2.46535s (13.7911GiB/s) | 20260508 13:48:02.365 | INFO | Analyzing chunk frames = [8582656,8583168) | 20260508 13:48:02.365 | INFO | MemInfo: { | 20260508 13:48:02.365 | INFO | "gpu" : " 17 GB", | 20260508 13:48:02.365 | INFO | "huge" : "", | 20260508 13:48:02.365 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:02.365 | INFO | "malloc" : " 14 GB", | 20260508 13:48:02.365 | INFO | "pinned" : " 29 GB" | 20260508 13:48:02.365 | INFO | } | 20260508 13:48:02.365 | INFO | Perf report for the last 5.14396 seconds: | 20260508 13:48:02.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:02.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | ANALYSIS : 66.0%, 3.0, 102.3ms, 0 | 20260508 13:48:02.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:48:02.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:02.365 | INFO | Uploaded 36GiB to GPU in 2.51707s (14.3023GiB/s) | 20260508 13:48:02.365 | INFO | Downloaded 34GiB to GPU in 2.44819s (13.8878GiB/s) | 20260508 13:48:07.505 | INFO | Analyzing chunk frames = [8583168,8583680) | 20260508 13:48:07.505 | INFO | MemInfo: { | 20260508 13:48:07.505 | INFO | "gpu" : " 17 GB", | 20260508 13:48:07.505 | INFO | "huge" : "", | 20260508 13:48:07.505 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:07.505 | INFO | "malloc" : " 14 GB", | 20260508 13:48:07.505 | INFO | "pinned" : " 29 GB" | 20260508 13:48:07.505 | INFO | } | 20260508 13:48:07.505 | INFO | Perf report for the last 5.14024 seconds: | 20260508 13:48:07.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:07.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.505 | INFO | ANALYSIS : 66.2%, 3.0, 103.3ms, 0 | 20260508 13:48:07.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.505 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:48:07.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:07.506 | INFO | Uploaded 36GiB to GPU in 2.41505s (14.9065GiB/s) | 20260508 13:48:07.506 | INFO | Downloaded 34GiB to GPU in 2.5289s (13.4446GiB/s) | 20260508 13:48:12.639 | INFO | Analyzing chunk frames = [8583680,8584192) | 20260508 13:48:12.639 | INFO | MemInfo: { | 20260508 13:48:12.639 | INFO | "gpu" : " 17 GB", | 20260508 13:48:12.639 | INFO | "huge" : "", | 20260508 13:48:12.639 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:12.639 | INFO | "malloc" : " 14 GB", | 20260508 13:48:12.639 | INFO | "pinned" : " 29 GB" | 20260508 13:48:12.639 | INFO | } | 20260508 13:48:12.639 | INFO | Perf report for the last 5.13403 seconds: | 20260508 13:48:12.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:12.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.639 | INFO | ANALYSIS : 65.5%, 3.0, 101.8ms, 0 | 20260508 13:48:12.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.639 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260508 13:48:12.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:12.640 | INFO | Uploaded 36GiB to GPU in 2.43382s (14.7916GiB/s) | 20260508 13:48:12.640 | INFO | Downloaded 34GiB to GPU in 2.45397s (13.8551GiB/s) | 20260508 13:48:17.785 | INFO | Analyzing chunk frames = [8584192,8584704) | 20260508 13:48:17.785 | INFO | MemInfo: { | 20260508 13:48:17.785 | INFO | "gpu" : " 17 GB", | 20260508 13:48:17.785 | INFO | "huge" : "", | 20260508 13:48:17.785 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:17.785 | INFO | "malloc" : " 14 GB", | 20260508 13:48:17.785 | INFO | "pinned" : " 29 GB" | 20260508 13:48:17.785 | INFO | } | 20260508 13:48:17.785 | INFO | Perf report for the last 5.14607 seconds: | 20260508 13:48:17.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:17.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.785 | INFO | ANALYSIS : 64.5%, 3.0, 100.4ms, 0 | 20260508 13:48:17.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.785 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:48:17.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:17.786 | INFO | Uploaded 36GiB to GPU in 2.32761s (15.4665GiB/s) | 20260508 13:48:17.786 | INFO | Downloaded 34GiB to GPU in 2.42357s (14.0289GiB/s) | 20260508 13:48:22.919 | INFO | Analyzing chunk frames = [8584704,8585216) | 20260508 13:48:22.919 | INFO | MemInfo: { | 20260508 13:48:22.919 | INFO | "gpu" : " 17 GB", | 20260508 13:48:22.919 | INFO | "huge" : "", | 20260508 13:48:22.919 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:22.919 | INFO | "malloc" : " 14 GB", | 20260508 13:48:22.919 | INFO | "pinned" : " 29 GB" | 20260508 13:48:22.919 | INFO | } | 20260508 13:48:22.919 | INFO | Perf report for the last 5.13384 seconds: | 20260508 13:48:22.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:22.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260508 13:48:22.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:48:22.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:22.919 | INFO | Uploaded 36GiB to GPU in 2.36526s (15.2203GiB/s) | 20260508 13:48:22.920 | INFO | Downloaded 34GiB to GPU in 2.45154s (13.8688GiB/s) | 20260508 13:48:22.920 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:48:22.920 | INFO | Total mem usage: 16.7 GiB | 20260508 13:48:22.920 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:48:22.920 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:48:22.920 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:48:22.920 | INFO | High Water Breakdown: | 20260508 13:48:22.920 | INFO | 59.5% : 8.46 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:48:22.920 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:48:22.920 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:48:22.920 | INFO | ------------------------------------------ | 20260508 13:48:22.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:48:22.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:48:22.920 | INFO | Total mem usage: 26.5 GiB | 20260508 13:48:22.920 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:48:22.920 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:48:22.920 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:48:22.920 | INFO | High Water Breakdown: | 20260508 13:48:22.920 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:48:22.920 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:48:22.920 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:48:22.920 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:48:22.920 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:48:22.920 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:48:22.920 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:48:22.920 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:48:22.920 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:48:22.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:48:22.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:48:22.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:48:22.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:48:22.920 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:48:22.920 | INFO | ------------------------------------------ | 20260508 13:48:22.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:48:22.920 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:48:22.920 | INFO | Total mem usage: 27.8 GiB | 20260508 13:48:22.920 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:48:22.920 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:48:22.920 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:48:22.920 | INFO | High Water Breakdown: | 20260508 13:48:22.920 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:48:22.920 | INFO | ------------------------------------------ | 20260508 13:48:22.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:48:22.920 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:48:22.920 | INFO | Total mem usage: 15.7 GiB | 20260508 13:48:22.920 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:48:22.920 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:48:22.920 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:48:22.920 | INFO | High Water Breakdown: | 20260508 13:48:22.920 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:48:22.920 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:48:22.920 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:48:22.920 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:48:22.920 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:48:22.920 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:48:22.921 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:48:22.921 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:48:22.921 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:48:22.921 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:48:22.921 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:48:22.921 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:48:22.921 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:48:22.921 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:48:22.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:48:22.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:48:22.921 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:48:22.921 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:48:22.921 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:48:22.921 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:48:22.921 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:48:22.921 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:48:22.921 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:48:22.921 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:48:22.921 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:48:22.921 | INFO | ------------------------------------------ | 20260508 13:48:22.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:48:22.921 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:48:22.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:48:22.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:48:22.921 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.54e+01, 1.39e+01 / 5.46e-01 | 20260508 13:48:22.921 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.13e-02 / 7.81e-03 | 20260508 13:48:22.921 | INFO | PerfReport: 12.20%, Baseline -- 3.97e+00 / 4.38e+00, 5.91e-01 / 2.67e-01 | 20260508 13:48:22.921 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.39e+00, 2.09e-01 / 1.26e-01 | 20260508 13:48:22.921 | INFO | PerfReport: 15.98%, DME -- 4.16e+01 / 4.52e+01, 3.73e+00 / 1.19e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.86e+01 / 2.06e+01, 1.81e+00 / 1.32e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.31e+00 / 2.38e+00, 3.99e-01 / 2.39e-01 | 20260508 13:48:22.921 | INFO | PerfReport: 2.93%, Metrics -- 9.53e-01 / 1.05e+00, 2.38e-01 / 7.19e-02 | 20260508 13:48:22.921 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.14e-01, 4.72e-02 / 1.96e-02 | 20260508 13:48:22.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:48:22.921 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:48:22.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:48:22.921 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 24.58%, Upload -- 2.57e+01 / 0.00e+00, 8.00e+00 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 24.80%, Download -- 2.60e+01 / 0.00e+00, 8.21e+00 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 31.11%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 2.23%, LockUpload -- 2.33e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 2.53%, LockDownload -- 2.65e+00 / 0.00e+00, 3.92e+00 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 14.75%, LockCompute -- 1.54e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:48:22.921 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260508 13:48:23.016 | INFO | Starting a round of Baz IO | 20260508 13:48:26.413 | INFO | Populating RTMetrics report | 20260508 13:48:26.414 | INFO | RTMetrics report populated | 20260508 13:48:26.414 | INFO | Opening tmp RTMetrics json file | 20260508 13:48:26.423 | INFO | Serializing RTMetrics to json | 20260508 13:48:26.424 | INFO | Serialization done, swapping tmp file now... | 20260508 13:48:26.424 | INFO | RTMetrics done | 20260508 13:48:26.482 | INFO | Finished a round of Baz IO: Wrote 1.378 GiB in 3.465 seconds | 20260508 13:48:26.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:48:26.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134365% / 0% / 8.74201e-05% | 20260508 13:48:26.482 | INFO | Baz IO overall: | 20260508 13:48:26.482 | INFO | BytesWritten : 3650565593647 | 20260508 13:48:26.482 | INFO | NumEvents : 1749777455871 | 20260508 13:48:28.065 | INFO | Analyzing chunk frames = [8585216,8585728) | 20260508 13:48:28.065 | INFO | MemInfo: { | 20260508 13:48:28.065 | INFO | "gpu" : " 17 GB", | 20260508 13:48:28.065 | INFO | "huge" : "", | 20260508 13:48:28.065 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:28.065 | INFO | "malloc" : " 14 GB", | 20260508 13:48:28.065 | INFO | "pinned" : " 29 GB" | 20260508 13:48:28.065 | INFO | } | 20260508 13:48:28.065 | INFO | Perf report for the last 5.14604 seconds: | 20260508 13:48:28.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:28.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:28.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:28.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:28.065 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 13:48:28.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:28.065 | INFO | RT_METRICS : 37.8%, 1.0, 19.9ms, 0 | 20260508 13:48:28.066 | INFO | PRE_HQ : 65.0%, 1.6, 53.4ms, 0 | 20260508 13:48:28.066 | INFO | BAZWRITER : 66.5%, 2.4, 82.8ms, 0 | 20260508 13:48:28.066 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:48:28.066 | INFO | Uploaded 36GiB to GPU in 2.63188s (13.6785GiB/s) | 20260508 13:48:28.066 | INFO | Downloaded 38GiB to GPU in 2.71753s (13.9833GiB/s) | 20260508 13:48:33.199 | INFO | Analyzing chunk frames = [8585728,8586240) | 20260508 13:48:33.199 | INFO | MemInfo: { | 20260508 13:48:33.199 | INFO | "gpu" : " 17 GB", | 20260508 13:48:33.199 | INFO | "huge" : "", | 20260508 13:48:33.199 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:33.199 | INFO | "malloc" : " 14 GB", | 20260508 13:48:33.199 | INFO | "pinned" : " 29 GB" | 20260508 13:48:33.199 | INFO | } | 20260508 13:48:33.199 | INFO | Perf report for the last 5.13398 seconds: | 20260508 13:48:33.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:33.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.199 | INFO | ANALYSIS : 66.1%, 3.0, 102.9ms, 0 | 20260508 13:48:33.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.199 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:48:33.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:33.200 | INFO | Uploaded 36GiB to GPU in 2.48578s (14.4824GiB/s) | 20260508 13:48:33.200 | INFO | Downloaded 34GiB to GPU in 2.49286s (13.639GiB/s) | 20260508 13:48:38.338 | INFO | Analyzing chunk frames = [8586240,8586752) | 20260508 13:48:38.338 | INFO | MemInfo: { | 20260508 13:48:38.338 | INFO | "gpu" : " 17 GB", | 20260508 13:48:38.338 | INFO | "huge" : "", | 20260508 13:48:38.338 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:38.338 | INFO | "malloc" : " 14 GB", | 20260508 13:48:38.338 | INFO | "pinned" : " 29 GB" | 20260508 13:48:38.338 | INFO | } | 20260508 13:48:38.338 | INFO | Perf report for the last 5.1387 seconds: | 20260508 13:48:38.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:38.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:48:38.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:48:38.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:38.338 | INFO | Uploaded 36GiB to GPU in 2.50008s (14.3995GiB/s) | 20260508 13:48:38.338 | INFO | Downloaded 34GiB to GPU in 2.46973s (13.7667GiB/s) | 20260508 13:48:43.479 | INFO | Analyzing chunk frames = [8586752,8587264) | 20260508 13:48:43.479 | INFO | MemInfo: { | 20260508 13:48:43.479 | INFO | "gpu" : " 17 GB", | 20260508 13:48:43.479 | INFO | "huge" : "", | 20260508 13:48:43.479 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:43.479 | INFO | "malloc" : " 14 GB", | 20260508 13:48:43.479 | INFO | "pinned" : " 29 GB" | 20260508 13:48:43.479 | INFO | } | 20260508 13:48:43.479 | INFO | Perf report for the last 5.14151 seconds: | 20260508 13:48:43.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:43.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.480 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 13:48:43.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.480 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:48:43.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:43.480 | INFO | Uploaded 36GiB to GPU in 2.51289s (14.3261GiB/s) | 20260508 13:48:43.480 | INFO | Downloaded 34GiB to GPU in 2.43877s (13.9415GiB/s) | 20260508 13:48:48.619 | INFO | Analyzing chunk frames = [8587264,8587776) | 20260508 13:48:48.619 | INFO | MemInfo: { | 20260508 13:48:48.619 | INFO | "gpu" : " 17 GB", | 20260508 13:48:48.619 | INFO | "huge" : "", | 20260508 13:48:48.619 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:48.619 | INFO | "malloc" : " 14 GB", | 20260508 13:48:48.619 | INFO | "pinned" : " 29 GB" | 20260508 13:48:48.619 | INFO | } | 20260508 13:48:48.619 | INFO | Perf report for the last 5.13977 seconds: | 20260508 13:48:48.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:48.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.619 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 13:48:48.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.619 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260508 13:48:48.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:48.620 | INFO | Uploaded 36GiB to GPU in 2.45912s (14.6394GiB/s) | 20260508 13:48:48.620 | INFO | Downloaded 34GiB to GPU in 2.52225s (13.48GiB/s) | 20260508 13:48:53.755 | INFO | Analyzing chunk frames = [8587776,8588288) | 20260508 13:48:53.755 | INFO | MemInfo: { | 20260508 13:48:53.755 | INFO | "gpu" : " 17 GB", | 20260508 13:48:53.755 | INFO | "huge" : "", | 20260508 13:48:53.755 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:53.755 | INFO | "malloc" : " 14 GB", | 20260508 13:48:53.755 | INFO | "pinned" : " 29 GB" | 20260508 13:48:53.755 | INFO | } | 20260508 13:48:53.755 | INFO | Perf report for the last 5.13558 seconds: | 20260508 13:48:53.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:53.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 13:48:53.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:48:53.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:53.755 | INFO | Uploaded 36GiB to GPU in 2.43009s (14.8142GiB/s) | 20260508 13:48:53.755 | INFO | Downloaded 34GiB to GPU in 2.45298s (13.8607GiB/s) | 20260508 13:48:58.890 | INFO | Analyzing chunk frames = [8588288,8588800) | 20260508 13:48:58.890 | INFO | MemInfo: { | 20260508 13:48:58.890 | INFO | "gpu" : " 17 GB", | 20260508 13:48:58.890 | INFO | "huge" : "", | 20260508 13:48:58.890 | INFO | "huge_shared" : " 29 GB", | 20260508 13:48:58.890 | INFO | "malloc" : " 14 GB", | 20260508 13:48:58.890 | INFO | "pinned" : " 29 GB" | 20260508 13:48:58.890 | INFO | } | 20260508 13:48:58.890 | INFO | Perf report for the last 5.13497 seconds: | 20260508 13:48:58.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:48:58.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | ANALYSIS : 65.5%, 3.0, 102.1ms, 0 | 20260508 13:48:58.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:48:58.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:48:58.890 | INFO | Uploaded 36GiB to GPU in 2.46049s (14.6312GiB/s) | 20260508 13:48:58.890 | INFO | Downloaded 34GiB to GPU in 2.45978s (13.8224GiB/s) | 20260508 13:49:04.033 | INFO | Analyzing chunk frames = [8588800,8589312) | 20260508 13:49:04.033 | INFO | MemInfo: { | 20260508 13:49:04.033 | INFO | "gpu" : " 17 GB", | 20260508 13:49:04.033 | INFO | "huge" : "", | 20260508 13:49:04.033 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:04.033 | INFO | "malloc" : " 14 GB", | 20260508 13:49:04.033 | INFO | "pinned" : " 29 GB" | 20260508 13:49:04.033 | INFO | } | 20260508 13:49:04.033 | INFO | Perf report for the last 5.1431 seconds: | 20260508 13:49:04.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:04.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.033 | INFO | ANALYSIS : 65.5%, 3.0, 102.5ms, 0 | 20260508 13:49:04.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.033 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:49:04.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:04.034 | INFO | Uploaded 36GiB to GPU in 2.44592s (14.7184GiB/s) | 20260508 13:49:04.034 | INFO | Downloaded 34GiB to GPU in 2.44825s (13.8875GiB/s) | 20260508 13:49:04.130 | INFO | Starting a round of Baz IO | 20260508 13:49:07.529 | INFO | Populating RTMetrics report | 20260508 13:49:07.529 | INFO | RTMetrics report populated | 20260508 13:49:07.529 | INFO | Opening tmp RTMetrics json file | 20260508 13:49:07.538 | INFO | Serializing RTMetrics to json | 20260508 13:49:07.539 | INFO | Serialization done, swapping tmp file now... | 20260508 13:49:07.540 | INFO | RTMetrics done | 20260508 13:49:07.602 | INFO | Finished a round of Baz IO: Wrote 1.376 GiB in 3.471 seconds | 20260508 13:49:07.602 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:49:07.602 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133634% / 0% / 7.54992e-05% | 20260508 13:49:07.602 | INFO | Baz IO overall: | 20260508 13:49:07.602 | INFO | BytesWritten : 3652043339690 | 20260508 13:49:07.602 | INFO | NumEvents : 1750467789336 | 20260508 13:49:09.180 | INFO | Analyzing chunk frames = [8589312,8589824) | 20260508 13:49:09.180 | INFO | MemInfo: { | 20260508 13:49:09.180 | INFO | "gpu" : " 17 GB", | 20260508 13:49:09.180 | INFO | "huge" : "", | 20260508 13:49:09.180 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:09.180 | INFO | "malloc" : " 14 GB", | 20260508 13:49:09.180 | INFO | "pinned" : " 29 GB" | 20260508 13:49:09.180 | INFO | } | 20260508 13:49:09.180 | INFO | Perf report for the last 5.14694 seconds: | 20260508 13:49:09.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:09.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:09.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:09.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:09.180 | INFO | ANALYSIS : 67.3%, 3.0, 105.3ms, 0 | 20260508 13:49:09.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:09.180 | INFO | RT_METRICS : 37.8%, 1.0, 19.9ms, 0 | 20260508 13:49:09.180 | INFO | PRE_HQ : 65.1%, 1.6, 53.6ms, 0 | 20260508 13:49:09.180 | INFO | BAZWRITER : 66.7%, 2.7, 92.9ms, 0 | 20260508 13:49:09.180 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:49:09.180 | INFO | Uploaded 36GiB to GPU in 2.64028s (13.6349GiB/s) | 20260508 13:49:09.180 | INFO | Downloaded 38GiB to GPU in 2.72433s (13.9484GiB/s) | 20260508 13:49:14.316 | INFO | Analyzing chunk frames = [8589824,8590336) | 20260508 13:49:14.316 | INFO | MemInfo: { | 20260508 13:49:14.316 | INFO | "gpu" : " 17 GB", | 20260508 13:49:14.316 | INFO | "huge" : "", | 20260508 13:49:14.316 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:14.316 | INFO | "malloc" : " 14 GB", | 20260508 13:49:14.316 | INFO | "pinned" : " 29 GB" | 20260508 13:49:14.316 | INFO | } | 20260508 13:49:14.317 | INFO | Perf report for the last 5.13661 seconds: | 20260508 13:49:14.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:14.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:49:14.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:49:14.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:14.317 | INFO | Uploaded 36GiB to GPU in 2.48239s (14.5022GiB/s) | 20260508 13:49:14.317 | INFO | Downloaded 34GiB to GPU in 2.49402s (13.6326GiB/s) | 20260508 13:49:19.450 | INFO | Analyzing chunk frames = [8590336,8590848) | 20260508 13:49:19.450 | INFO | MemInfo: { | 20260508 13:49:19.450 | INFO | "gpu" : " 17 GB", | 20260508 13:49:19.450 | INFO | "huge" : "", | 20260508 13:49:19.450 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:19.450 | INFO | "malloc" : " 14 GB", | 20260508 13:49:19.450 | INFO | "pinned" : " 29 GB" | 20260508 13:49:19.450 | INFO | } | 20260508 13:49:19.450 | INFO | Perf report for the last 5.1337 seconds: | 20260508 13:49:19.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:19.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.450 | INFO | ANALYSIS : 66.4%, 3.0, 103.1ms, 0 | 20260508 13:49:19.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.450 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:49:19.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:19.451 | INFO | Uploaded 36GiB to GPU in 2.50971s (14.3443GiB/s) | 20260508 13:49:19.451 | INFO | Downloaded 34GiB to GPU in 2.47074s (13.761GiB/s) | 20260508 13:49:24.592 | INFO | Analyzing chunk frames = [8590848,8591360) | 20260508 13:49:24.592 | INFO | MemInfo: { | 20260508 13:49:24.592 | INFO | "gpu" : " 17 GB", | 20260508 13:49:24.592 | INFO | "huge" : "", | 20260508 13:49:24.592 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:24.592 | INFO | "malloc" : " 14 GB", | 20260508 13:49:24.592 | INFO | "pinned" : " 29 GB" | 20260508 13:49:24.592 | INFO | } | 20260508 13:49:24.592 | INFO | Perf report for the last 5.14155 seconds: | 20260508 13:49:24.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:24.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:49:24.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:49:24.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:24.592 | INFO | Uploaded 36GiB to GPU in 2.41034s (14.9356GiB/s) | 20260508 13:49:24.592 | INFO | Downloaded 34GiB to GPU in 2.48488s (13.6827GiB/s) | 20260508 13:49:29.731 | INFO | Analyzing chunk frames = [8591360,8591872) | 20260508 13:49:29.731 | INFO | MemInfo: { | 20260508 13:49:29.731 | INFO | "gpu" : " 17 GB", | 20260508 13:49:29.731 | INFO | "huge" : "", | 20260508 13:49:29.731 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:29.731 | INFO | "malloc" : " 14 GB", | 20260508 13:49:29.731 | INFO | "pinned" : " 29 GB" | 20260508 13:49:29.731 | INFO | } | 20260508 13:49:29.731 | INFO | Perf report for the last 5.13935 seconds: | 20260508 13:49:29.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:29.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.731 | INFO | ANALYSIS : 66.1%, 3.0, 103.3ms, 0 | 20260508 13:49:29.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.731 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260508 13:49:29.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:29.732 | INFO | Uploaded 36GiB to GPU in 2.42547s (14.8425GiB/s) | 20260508 13:49:29.732 | INFO | Downloaded 34GiB to GPU in 2.52782s (13.4503GiB/s) | 20260508 13:49:34.872 | INFO | Analyzing chunk frames = [8591872,8592384) | 20260508 13:49:34.872 | INFO | MemInfo: { | 20260508 13:49:34.872 | INFO | "gpu" : " 17 GB", | 20260508 13:49:34.872 | INFO | "huge" : "", | 20260508 13:49:34.872 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:34.872 | INFO | "malloc" : " 14 GB", | 20260508 13:49:34.872 | INFO | "pinned" : " 29 GB" | 20260508 13:49:34.872 | INFO | } | 20260508 13:49:34.872 | INFO | Perf report for the last 5.14081 seconds: | 20260508 13:49:34.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:34.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | ANALYSIS : 65.2%, 3.0, 101.5ms, 0 | 20260508 13:49:34.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:49:34.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:34.872 | INFO | Uploaded 36GiB to GPU in 2.3827s (15.1089GiB/s) | 20260508 13:49:34.872 | INFO | Downloaded 34GiB to GPU in 2.44584s (13.9011GiB/s) | 20260508 13:49:40.011 | INFO | Analyzing chunk frames = [8592384,8592896) | 20260508 13:49:40.011 | INFO | MemInfo: { | 20260508 13:49:40.011 | INFO | "gpu" : " 17 GB", | 20260508 13:49:40.011 | INFO | "huge" : "", | 20260508 13:49:40.011 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:40.011 | INFO | "malloc" : " 14 GB", | 20260508 13:49:40.011 | INFO | "pinned" : " 29 GB" | 20260508 13:49:40.011 | INFO | } | 20260508 13:49:40.011 | INFO | Perf report for the last 5.13888 seconds: | 20260508 13:49:40.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:40.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | ANALYSIS : 65.5%, 3.0, 102.2ms, 0 | 20260508 13:49:40.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:49:40.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:40.011 | INFO | Uploaded 36GiB to GPU in 2.47477s (14.5468GiB/s) | 20260508 13:49:40.011 | INFO | Downloaded 34GiB to GPU in 2.46677s (13.7832GiB/s) | 20260508 13:49:45.148 | INFO | Analyzing chunk frames = [8592896,8593408) | 20260508 13:49:45.148 | INFO | MemInfo: { | 20260508 13:49:45.148 | INFO | "gpu" : " 17 GB", | 20260508 13:49:45.148 | INFO | "huge" : "", | 20260508 13:49:45.148 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:45.148 | INFO | "malloc" : " 14 GB", | 20260508 13:49:45.148 | INFO | "pinned" : " 29 GB" | 20260508 13:49:45.148 | INFO | } | 20260508 13:49:45.148 | INFO | Perf report for the last 5.13699 seconds: | 20260508 13:49:45.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:45.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | ANALYSIS : 64.3%, 3.0, 100.5ms, 0 | 20260508 13:49:45.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:49:45.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:45.148 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:49:45.148 | INFO | Total mem usage: 16.7 GiB | 20260508 13:49:45.148 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:49:45.148 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -12.9%) | 20260508 13:49:45.148 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:49:45.148 | INFO | High Water Breakdown: | 20260508 13:49:45.148 | INFO | 60.1% : 8.88 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:49:45.148 | INFO | 39.7% : 5.87 GiB ( 87.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:49:45.148 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:49:45.148 | INFO | ------------------------------------------ | 20260508 13:49:45.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:49:45.148 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:49:45.148 | INFO | Total mem usage: 26.5 GiB | 20260508 13:49:45.148 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:49:45.148 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:49:45.148 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:49:45.148 | INFO | High Water Breakdown: | 20260508 13:49:45.148 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:49:45.148 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:49:45.148 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:49:45.148 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:49:45.148 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:49:45.148 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:49:45.148 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:49:45.148 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:49:45.148 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:49:45.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:49:45.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:49:45.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:49:45.148 | INFO | Uploaded 36GiB to GPU in 2.28824s (15.7326GiB/s) | 20260508 13:49:45.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:49:45.149 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:49:45.149 | INFO | Downloaded 34GiB to GPU in 2.43677s (13.9529GiB/s) | 20260508 13:49:45.149 | INFO | ------------------------------------------ | 20260508 13:49:45.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:49:45.149 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:49:45.149 | INFO | Total mem usage: 27.8 GiB | 20260508 13:49:45.149 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:49:45.149 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:49:45.149 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:49:45.149 | INFO | High Water Breakdown: | 20260508 13:49:45.149 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:49:45.149 | INFO | ------------------------------------------ | 20260508 13:49:45.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:49:45.149 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:49:45.149 | INFO | Total mem usage: 15.7 GiB | 20260508 13:49:45.149 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:49:45.149 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:49:45.149 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:49:45.149 | INFO | High Water Breakdown: | 20260508 13:49:45.149 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:49:45.149 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:49:45.149 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:49:45.149 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:49:45.149 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:49:45.149 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:49:45.149 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:49:45.149 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:49:45.149 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:49:45.149 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:49:45.149 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:49:45.149 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:49:45.149 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:49:45.149 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:49:45.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:49:45.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:49:45.149 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:49:45.149 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:49:45.149 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:49:45.149 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:49:45.149 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:49:45.149 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:49:45.149 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:49:45.149 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:49:45.149 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:49:45.149 | INFO | ------------------------------------------ | 20260508 13:49:45.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:49:45.149 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:49:45.149 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:49:45.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:49:45.149 | INFO | PerfReport: Full Execution -- 3.25e+01 / 7.54e+01, 1.42e+01 / 6.86e-01 | 20260508 13:49:45.149 | INFO | PerfReport: 0.30%, AggregateStats -- 9.64e-02 / 1.05e-01, 1.07e-02 / 6.13e-03 | 20260508 13:49:45.150 | INFO | PerfReport: 12.18%, Baseline -- 3.96e+00 / 4.38e+00, 5.84e-01 / 2.28e-01 | 20260508 13:49:45.150 | INFO | PerfReport: 3.37%, Histogram -- 1.10e+00 / 1.44e+00, 2.08e-01 / 1.31e-01 | 20260508 13:49:45.150 | INFO | PerfReport: 16.01%, DME -- 4.17e+01 / 4.50e+01, 3.70e+00 / 1.18e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.86e+01 / 2.08e+01, 1.77e+00 / 1.23e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.31e+00 / 2.38e+00, 4.01e-01 / 2.27e-01 | 20260508 13:49:45.150 | INFO | PerfReport: 2.92%, Metrics -- 9.50e-01 / 1.06e+00, 2.36e-01 / 8.13e-02 | 20260508 13:49:45.150 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.87e-01 / 3.10e-01, 4.63e-02 / 2.15e-02 | 20260508 13:49:45.150 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:49:45.150 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:49:45.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:49:45.150 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 24.55%, Upload -- 2.57e+01 / 0.00e+00, 8.09e+00 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 24.87%, Download -- 2.61e+01 / 0.00e+00, 8.20e+00 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 31.03%, Compute -- 3.25e+01 / 0.00e+00, 1.42e+01 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 2.09%, LockUpload -- 2.19e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 2.63%, LockDownload -- 2.76e+00 / 0.00e+00, 3.96e+00 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 14.83%, LockCompute -- 1.55e+01 / 0.00e+00, 1.75e+01 / 0.00e+00 | 20260508 13:49:45.150 | INFO | PerfReport: 0.01%, Misc -- 1.17e-02 / 0.00e+00, 1.05e-02 / 0.00e+00 | 20260508 13:49:45.246 | INFO | Starting a round of Baz IO | 20260508 13:49:48.646 | INFO | Populating RTMetrics report | 20260508 13:49:48.647 | INFO | RTMetrics report populated | 20260508 13:49:48.647 | INFO | Opening tmp RTMetrics json file | 20260508 13:49:48.656 | INFO | Serializing RTMetrics to json | 20260508 13:49:48.657 | INFO | Serialization done, swapping tmp file now... | 20260508 13:49:48.657 | INFO | RTMetrics done | 20260508 13:49:48.718 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.472 seconds | 20260508 13:49:48.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:49:48.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133832% / 3.97364e-06% / 7.15256e-05% | 20260508 13:49:48.719 | INFO | Baz IO overall: | 20260508 13:49:48.719 | INFO | BytesWritten : 3653522355777 | 20260508 13:49:48.719 | INFO | NumEvents : 1751159200037 | 20260508 13:49:50.294 | INFO | Analyzing chunk frames = [8593408,8593920) | 20260508 13:49:50.294 | INFO | MemInfo: { | 20260508 13:49:50.294 | INFO | "gpu" : " 17 GB", | 20260508 13:49:50.294 | INFO | "huge" : "", | 20260508 13:49:50.294 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:50.294 | INFO | "malloc" : " 14 GB", | 20260508 13:49:50.294 | INFO | "pinned" : " 29 GB" | 20260508 13:49:50.294 | INFO | } | 20260508 13:49:50.294 | INFO | Perf report for the last 5.14643 seconds: | 20260508 13:49:50.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:50.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:50.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:50.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:50.294 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:49:50.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:50.294 | INFO | RT_METRICS : 38.1%, 1.0, 20.0ms, 0 | 20260508 13:49:50.294 | INFO | PRE_HQ : 65.1%, 1.6, 53.7ms, 0 | 20260508 13:49:50.294 | INFO | BAZWRITER : 66.7%, 2.4, 83.6ms, 0 | 20260508 13:49:50.294 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:49:50.295 | INFO | Uploaded 36GiB to GPU in 2.66007s (13.5335GiB/s) | 20260508 13:49:50.295 | INFO | Downloaded 38GiB to GPU in 2.71724s (13.9848GiB/s) | 20260508 13:49:55.431 | INFO | Analyzing chunk frames = [8593920,8594432) | 20260508 13:49:55.431 | INFO | MemInfo: { | 20260508 13:49:55.431 | INFO | "gpu" : " 17 GB", | 20260508 13:49:55.431 | INFO | "huge" : "", | 20260508 13:49:55.432 | INFO | "huge_shared" : " 29 GB", | 20260508 13:49:55.432 | INFO | "malloc" : " 14 GB", | 20260508 13:49:55.432 | INFO | "pinned" : " 29 GB" | 20260508 13:49:55.432 | INFO | } | 20260508 13:49:55.432 | INFO | Perf report for the last 5.13734 seconds: | 20260508 13:49:55.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:49:55.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 13:49:55.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260508 13:49:55.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:49:55.432 | INFO | Uploaded 36GiB to GPU in 2.51034s (14.3407GiB/s) | 20260508 13:49:55.432 | INFO | Downloaded 34GiB to GPU in 2.47481s (13.7384GiB/s) | 20260508 13:50:00.565 | INFO | Analyzing chunk frames = [8594432,8594944) | 20260508 13:50:00.565 | INFO | MemInfo: { | 20260508 13:50:00.565 | INFO | "gpu" : " 17 GB", | 20260508 13:50:00.565 | INFO | "huge" : "", | 20260508 13:50:00.565 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:00.565 | INFO | "malloc" : " 14 GB", | 20260508 13:50:00.565 | INFO | "pinned" : " 29 GB" | 20260508 13:50:00.566 | INFO | } | 20260508 13:50:00.566 | INFO | Perf report for the last 5.13405 seconds: | 20260508 13:50:00.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:00.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | ANALYSIS : 66.4%, 3.0, 103.0ms, 0 | 20260508 13:50:00.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:50:00.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:00.566 | INFO | Uploaded 36GiB to GPU in 2.5212s (14.2789GiB/s) | 20260508 13:50:00.566 | INFO | Downloaded 34GiB to GPU in 2.455s (13.8493GiB/s) | 20260508 13:50:05.706 | INFO | Analyzing chunk frames = [8594944,8595456) | 20260508 13:50:05.706 | INFO | MemInfo: { | 20260508 13:50:05.706 | INFO | "gpu" : " 17 GB", | 20260508 13:50:05.706 | INFO | "huge" : "", | 20260508 13:50:05.706 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:05.706 | INFO | "malloc" : " 14 GB", | 20260508 13:50:05.706 | INFO | "pinned" : " 29 GB" | 20260508 13:50:05.706 | INFO | } | 20260508 13:50:05.707 | INFO | Perf report for the last 5.14077 seconds: | 20260508 13:50:05.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:05.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | ANALYSIS : 65.3%, 3.0, 101.3ms, 0 | 20260508 13:50:05.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:50:05.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:05.707 | INFO | Uploaded 36GiB to GPU in 2.41053s (14.9345GiB/s) | 20260508 13:50:05.707 | INFO | Downloaded 34GiB to GPU in 2.44505s (13.9056GiB/s) | 20260508 13:50:10.845 | INFO | Analyzing chunk frames = [8595456,8595968) | 20260508 13:50:10.845 | INFO | MemInfo: { | 20260508 13:50:10.845 | INFO | "gpu" : " 17 GB", | 20260508 13:50:10.845 | INFO | "huge" : "", | 20260508 13:50:10.845 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:10.845 | INFO | "malloc" : " 14 GB", | 20260508 13:50:10.845 | INFO | "pinned" : " 29 GB" | 20260508 13:50:10.845 | INFO | } | 20260508 13:50:10.845 | INFO | Perf report for the last 5.13869 seconds: | 20260508 13:50:10.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:10.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.845 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:50:10.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.845 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260508 13:50:10.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:10.846 | INFO | Uploaded 36GiB to GPU in 2.3361s (15.4103GiB/s) | 20260508 13:50:10.846 | INFO | Downloaded 34GiB to GPU in 2.49835s (13.609GiB/s) | 20260508 13:50:15.986 | INFO | Analyzing chunk frames = [8595968,8596480) | 20260508 13:50:15.986 | INFO | MemInfo: { | 20260508 13:50:15.986 | INFO | "gpu" : " 17 GB", | 20260508 13:50:15.986 | INFO | "huge" : "", | 20260508 13:50:15.986 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:15.986 | INFO | "malloc" : " 14 GB", | 20260508 13:50:15.986 | INFO | "pinned" : " 29 GB" | 20260508 13:50:15.986 | INFO | } | 20260508 13:50:15.986 | INFO | Perf report for the last 5.1411 seconds: | 20260508 13:50:15.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:15.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.986 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:50:15.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.986 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:50:15.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:15.987 | INFO | Uploaded 36GiB to GPU in 2.45354s (14.6727GiB/s) | 20260508 13:50:15.987 | INFO | Downloaded 34GiB to GPU in 2.43661s (13.9538GiB/s) | 20260508 13:50:21.127 | INFO | Analyzing chunk frames = [8596480,8596992) | 20260508 13:50:21.127 | INFO | MemInfo: { | 20260508 13:50:21.127 | INFO | "gpu" : " 17 GB", | 20260508 13:50:21.127 | INFO | "huge" : "", | 20260508 13:50:21.127 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:21.127 | INFO | "malloc" : " 14 GB", | 20260508 13:50:21.127 | INFO | "pinned" : " 29 GB" | 20260508 13:50:21.127 | INFO | } | 20260508 13:50:21.128 | INFO | Perf report for the last 5.14118 seconds: | 20260508 13:50:21.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:21.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | ANALYSIS : 65.6%, 3.0, 102.3ms, 0 | 20260508 13:50:21.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | PRE_HQ : 23.0%, 1.0, 12.0ms, 0 | 20260508 13:50:21.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:21.128 | INFO | Uploaded 36GiB to GPU in 2.4969s (14.4179GiB/s) | 20260508 13:50:21.128 | INFO | Downloaded 34GiB to GPU in 2.46096s (13.8157GiB/s) | 20260508 13:50:26.264 | INFO | Analyzing chunk frames = [8596992,8597504) | 20260508 13:50:26.264 | INFO | MemInfo: { | 20260508 13:50:26.264 | INFO | "gpu" : " 17 GB", | 20260508 13:50:26.264 | INFO | "huge" : "", | 20260508 13:50:26.264 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:26.264 | INFO | "malloc" : " 14 GB", | 20260508 13:50:26.264 | INFO | "pinned" : " 29 GB" | 20260508 13:50:26.264 | INFO | } | 20260508 13:50:26.264 | INFO | Perf report for the last 5.13669 seconds: | 20260508 13:50:26.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:26.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.264 | INFO | ANALYSIS : 64.8%, 3.0, 101.1ms, 0 | 20260508 13:50:26.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.264 | INFO | PRE_HQ : 23.2%, 1.0, 12.0ms, 0 | 20260508 13:50:26.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:26.265 | INFO | Uploaded 36GiB to GPU in 2.38936s (15.0668GiB/s) | 20260508 13:50:26.265 | INFO | Downloaded 34GiB to GPU in 2.42557s (14.0173GiB/s) | 20260508 13:50:26.362 | INFO | Starting a round of Baz IO | 20260508 13:50:29.762 | INFO | Populating RTMetrics report | 20260508 13:50:29.762 | INFO | RTMetrics report populated | 20260508 13:50:29.762 | INFO | Opening tmp RTMetrics json file | 20260508 13:50:29.775 | INFO | Serializing RTMetrics to json | 20260508 13:50:29.776 | INFO | Serialization done, swapping tmp file now... | 20260508 13:50:29.777 | INFO | RTMetrics done | 20260508 13:50:29.837 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.474 seconds | 20260508 13:50:29.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:50:29.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134079% / 0% / 3.57628e-05% | 20260508 13:50:29.837 | INFO | Baz IO overall: | 20260508 13:50:29.837 | INFO | BytesWritten : 3655001092435 | 20260508 13:50:29.837 | INFO | NumEvents : 1751850532816 | 20260508 13:50:31.399 | INFO | Analyzing chunk frames = [8597504,8598016) | 20260508 13:50:31.399 | INFO | MemInfo: { | 20260508 13:50:31.399 | INFO | "gpu" : " 17 GB", | 20260508 13:50:31.399 | INFO | "huge" : "", | 20260508 13:50:31.399 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:31.399 | INFO | "malloc" : " 14 GB", | 20260508 13:50:31.399 | INFO | "pinned" : " 29 GB" | 20260508 13:50:31.399 | INFO | } | 20260508 13:50:31.399 | INFO | Perf report for the last 5.13499 seconds: | 20260508 13:50:31.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:31.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:31.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:31.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:31.399 | INFO | ANALYSIS : 67.5%, 3.0, 105.3ms, 0 | 20260508 13:50:31.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:31.399 | INFO | RT_METRICS : 39.5%, 1.0, 20.7ms, 0 | 20260508 13:50:31.399 | INFO | PRE_HQ : 65.6%, 1.7, 56.2ms, 0 | 20260508 13:50:31.399 | INFO | BAZWRITER : 67.0%, 2.5, 87.0ms, 0 | 20260508 13:50:31.399 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:50:31.400 | INFO | Uploaded 36GiB to GPU in 2.63541s (13.6601GiB/s) | 20260508 13:50:31.400 | INFO | Downloaded 38GiB to GPU in 2.71613s (13.9905GiB/s) | 20260508 13:50:36.542 | INFO | Analyzing chunk frames = [8598016,8598528) | 20260508 13:50:36.542 | INFO | MemInfo: { | 20260508 13:50:36.542 | INFO | "gpu" : " 17 GB", | 20260508 13:50:36.542 | INFO | "huge" : "", | 20260508 13:50:36.542 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:36.542 | INFO | "malloc" : " 14 GB", | 20260508 13:50:36.542 | INFO | "pinned" : " 29 GB" | 20260508 13:50:36.542 | INFO | } | 20260508 13:50:36.542 | INFO | Perf report for the last 5.14311 seconds: | 20260508 13:50:36.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:36.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.542 | INFO | ANALYSIS : 66.2%, 3.0, 103.2ms, 0 | 20260508 13:50:36.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.542 | INFO | PRE_HQ : 22.1%, 1.0, 11.5ms, 0 | 20260508 13:50:36.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:36.543 | INFO | Uploaded 36GiB to GPU in 2.47981s (14.5173GiB/s) | 20260508 13:50:36.543 | INFO | Downloaded 34GiB to GPU in 2.46376s (13.8GiB/s) | 20260508 13:50:41.686 | INFO | Analyzing chunk frames = [8598528,8599040) | 20260508 13:50:41.686 | INFO | MemInfo: { | 20260508 13:50:41.686 | INFO | "gpu" : " 17 GB", | 20260508 13:50:41.686 | INFO | "huge" : "", | 20260508 13:50:41.686 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:41.686 | INFO | "malloc" : " 14 GB", | 20260508 13:50:41.686 | INFO | "pinned" : " 29 GB" | 20260508 13:50:41.686 | INFO | } | 20260508 13:50:41.686 | INFO | Perf report for the last 5.14363 seconds: | 20260508 13:50:41.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:41.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.686 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:50:41.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.686 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:50:41.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:41.687 | INFO | Uploaded 36GiB to GPU in 2.51383s (14.3208GiB/s) | 20260508 13:50:41.687 | INFO | Downloaded 34GiB to GPU in 2.45309s (13.86GiB/s) | 20260508 13:50:46.829 | INFO | Analyzing chunk frames = [8599040,8599552) | 20260508 13:50:46.829 | INFO | MemInfo: { | 20260508 13:50:46.829 | INFO | "gpu" : " 17 GB", | 20260508 13:50:46.829 | INFO | "huge" : "", | 20260508 13:50:46.829 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:46.830 | INFO | "malloc" : " 14 GB", | 20260508 13:50:46.830 | INFO | "pinned" : " 29 GB" | 20260508 13:50:46.830 | INFO | } | 20260508 13:50:46.830 | INFO | Perf report for the last 5.1436 seconds: | 20260508 13:50:46.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:46.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 13:50:46.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:50:46.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:46.830 | INFO | Uploaded 36GiB to GPU in 2.43918s (14.7591GiB/s) | 20260508 13:50:46.830 | INFO | Downloaded 34GiB to GPU in 2.46744s (13.7795GiB/s) | 20260508 13:50:51.962 | INFO | Analyzing chunk frames = [8599552,8600064) | 20260508 13:50:51.962 | INFO | MemInfo: { | 20260508 13:50:51.962 | INFO | "gpu" : " 17 GB", | 20260508 13:50:51.962 | INFO | "huge" : "", | 20260508 13:50:51.962 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:51.962 | INFO | "malloc" : " 14 GB", | 20260508 13:50:51.962 | INFO | "pinned" : " 29 GB" | 20260508 13:50:51.962 | INFO | } | 20260508 13:50:51.962 | INFO | Perf report for the last 5.13253 seconds: | 20260508 13:50:51.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:51.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 13:50:51.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:50:51.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:51.962 | INFO | Uploaded 36GiB to GPU in 2.51074s (14.3384GiB/s) | 20260508 13:50:51.962 | INFO | Downloaded 34GiB to GPU in 2.51623s (13.5123GiB/s) | 20260508 13:50:57.103 | INFO | Analyzing chunk frames = [8600064,8600576) | 20260508 13:50:57.103 | INFO | MemInfo: { | 20260508 13:50:57.103 | INFO | "gpu" : " 17 GB", | 20260508 13:50:57.103 | INFO | "huge" : "", | 20260508 13:50:57.103 | INFO | "huge_shared" : " 29 GB", | 20260508 13:50:57.103 | INFO | "malloc" : " 14 GB", | 20260508 13:50:57.103 | INFO | "pinned" : " 29 GB" | 20260508 13:50:57.103 | INFO | } | 20260508 13:50:57.103 | INFO | Perf report for the last 5.14102 seconds: | 20260508 13:50:57.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:50:57.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 13:50:57.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:50:57.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:50:57.103 | INFO | Uploaded 36GiB to GPU in 2.34812s (15.3314GiB/s) | 20260508 13:50:57.104 | INFO | Downloaded 34GiB to GPU in 2.41404s (14.0843GiB/s) | 20260508 13:51:02.239 | INFO | Analyzing chunk frames = [8600576,8601088) | 20260508 13:51:02.239 | INFO | MemInfo: { | 20260508 13:51:02.239 | INFO | "gpu" : " 17 GB", | 20260508 13:51:02.239 | INFO | "huge" : "", | 20260508 13:51:02.240 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:02.240 | INFO | "malloc" : " 14 GB", | 20260508 13:51:02.240 | INFO | "pinned" : " 29 GB" | 20260508 13:51:02.240 | INFO | } | 20260508 13:51:02.240 | INFO | Perf report for the last 5.13661 seconds: | 20260508 13:51:02.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:02.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:51:02.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 13:51:02.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:02.240 | INFO | Uploaded 36GiB to GPU in 2.33119s (15.4427GiB/s) | 20260508 13:51:02.240 | INFO | Downloaded 34GiB to GPU in 2.39669s (14.1862GiB/s) | 20260508 13:51:07.378 | INFO | Analyzing chunk frames = [8601088,8601600) | 20260508 13:51:07.378 | INFO | MemInfo: { | 20260508 13:51:07.378 | INFO | "gpu" : " 17 GB", | 20260508 13:51:07.378 | INFO | "huge" : "", | 20260508 13:51:07.378 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:07.378 | INFO | "malloc" : " 14 GB", | 20260508 13:51:07.378 | INFO | "pinned" : " 29 GB" | 20260508 13:51:07.379 | INFO | } | 20260508 13:51:07.379 | INFO | Perf report for the last 5.13874 seconds: | 20260508 13:51:07.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:07.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260508 13:51:07.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260508 13:51:07.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:07.379 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:51:07.379 | INFO | Total mem usage: 16.7 GiB | 20260508 13:51:07.379 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:51:07.379 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.4%) | 20260508 13:51:07.379 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:51:07.379 | INFO | High Water Breakdown: | 20260508 13:51:07.379 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:51:07.379 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:51:07.379 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:51:07.379 | INFO | ------------------------------------------ | 20260508 13:51:07.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:51:07.379 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:51:07.379 | INFO | Total mem usage: 26.5 GiB | 20260508 13:51:07.379 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:51:07.379 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:51:07.379 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:51:07.379 | INFO | High Water Breakdown: | 20260508 13:51:07.379 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:51:07.379 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:51:07.379 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:51:07.379 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:51:07.379 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:51:07.379 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:51:07.379 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:51:07.379 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:51:07.379 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:51:07.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:51:07.379 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:51:07.379 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:51:07.379 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:51:07.379 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:51:07.379 | INFO | ------------------------------------------ | 20260508 13:51:07.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:51:07.379 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:51:07.379 | INFO | Total mem usage: 27.8 GiB | 20260508 13:51:07.379 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:51:07.379 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:51:07.379 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:51:07.379 | INFO | High Water Breakdown: | 20260508 13:51:07.379 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:51:07.379 | INFO | ------------------------------------------ | 20260508 13:51:07.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:51:07.379 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:51:07.379 | INFO | Total mem usage: 15.7 GiB | 20260508 13:51:07.379 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:51:07.379 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:51:07.379 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:51:07.379 | INFO | High Water Breakdown: | 20260508 13:51:07.379 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:51:07.379 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:51:07.379 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:51:07.379 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:51:07.379 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:51:07.379 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:51:07.379 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:51:07.379 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:51:07.379 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:51:07.379 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:51:07.379 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:51:07.379 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:51:07.379 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:51:07.379 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:51:07.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:51:07.379 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:51:07.379 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:51:07.379 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:51:07.379 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:51:07.379 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:51:07.379 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:51:07.379 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:51:07.379 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:51:07.379 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:51:07.379 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:51:07.379 | INFO | ------------------------------------------ | 20260508 13:51:07.379 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:51:07.379 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:51:07.379 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:51:07.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:51:07.379 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 4.14e-01 | 20260508 13:51:07.379 | INFO | PerfReport: 0.30%, AggregateStats -- 9.74e-02 / 1.06e-01, 1.12e-02 / 6.64e-03 | 20260508 13:51:07.379 | INFO | PerfReport: 12.24%, Baseline -- 4.00e+00 / 4.37e+00, 6.04e-01 / 2.22e-01 | 20260508 13:51:07.379 | INFO | PerfReport: 3.41%, Histogram -- 1.11e+00 / 1.39e+00, 2.12e-01 / 1.28e-01 | 20260508 13:51:07.379 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.50e+01, 3.76e+00 / 9.33e-01 | 20260508 13:51:07.379 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.87e+01 / 2.08e+01, 1.87e+00 / 1.23e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.31e+00 / 2.33e+00, 4.00e-01 / 2.56e-01 | 20260508 13:51:07.379 | INFO | PerfReport: 2.91%, Metrics -- 9.51e-01 / 1.07e+00, 2.37e-01 / 1.17e-01 | 20260508 13:51:07.379 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.88e-01 / 3.15e-01, 4.71e-02 / 2.63e-02 | 20260508 13:51:07.379 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:51:07.379 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:51:07.379 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:51:07.379 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 24.50%, Upload -- 2.56e+01 / 0.00e+00, 7.88e+00 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 24.73%, Download -- 2.59e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 31.26%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 2.27%, LockUpload -- 2.37e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 2.43%, LockDownload -- 2.54e+00 / 0.00e+00, 3.82e+00 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 14.79%, LockCompute -- 1.55e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:51:07.379 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260508 13:51:07.379 | INFO | Uploaded 36GiB to GPU in 2.32184s (15.5049GiB/s) | 20260508 13:51:07.380 | INFO | Downloaded 34GiB to GPU in 2.39852s (14.1754GiB/s) | 20260508 13:51:07.477 | INFO | Starting a round of Baz IO | 20260508 13:51:10.891 | INFO | Populating RTMetrics report | 20260508 13:51:10.892 | INFO | RTMetrics report populated | 20260508 13:51:10.892 | INFO | Opening tmp RTMetrics json file | 20260508 13:51:10.896 | INFO | Serializing RTMetrics to json | 20260508 13:51:10.898 | INFO | Serialization done, swapping tmp file now... | 20260508 13:51:10.899 | INFO | RTMetrics done | 20260508 13:51:10.954 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.477 seconds | 20260508 13:51:10.954 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:51:10.954 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133324% / 7.94729e-06% / 6.35783e-05% | 20260508 13:51:10.954 | INFO | Baz IO overall: | 20260508 13:51:10.954 | INFO | BytesWritten : 3656479701197 | 20260508 13:51:10.954 | INFO | NumEvents : 1752541863918 | 20260508 13:51:12.520 | INFO | Analyzing chunk frames = [8601600,8602112) | 20260508 13:51:12.520 | INFO | MemInfo: { | 20260508 13:51:12.520 | INFO | "gpu" : " 17 GB", | 20260508 13:51:12.520 | INFO | "huge" : "", | 20260508 13:51:12.520 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:12.520 | INFO | "malloc" : " 14 GB", | 20260508 13:51:12.520 | INFO | "pinned" : " 29 GB" | 20260508 13:51:12.520 | INFO | } | 20260508 13:51:12.520 | INFO | Perf report for the last 5.14128 seconds: | 20260508 13:51:12.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:12.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:12.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:12.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:12.520 | INFO | ANALYSIS : 67.5%, 3.0, 105.5ms, 0 | 20260508 13:51:12.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:12.520 | INFO | RT_METRICS : 37.8%, 1.0, 19.8ms, 0 | 20260508 13:51:12.520 | INFO | PRE_HQ : 65.1%, 1.6, 53.3ms, 0 | 20260508 13:51:12.520 | INFO | BAZWRITER : 66.9%, 2.5, 84.5ms, 0 | 20260508 13:51:12.520 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:51:12.520 | INFO | Uploaded 36GiB to GPU in 2.63749s (13.6494GiB/s) | 20260508 13:51:12.520 | INFO | Downloaded 38GiB to GPU in 2.71337s (14.0047GiB/s) | 20260508 13:51:17.653 | INFO | Analyzing chunk frames = [8602112,8602624) | 20260508 13:51:17.653 | INFO | MemInfo: { | 20260508 13:51:17.653 | INFO | "gpu" : " 17 GB", | 20260508 13:51:17.653 | INFO | "huge" : "", | 20260508 13:51:17.653 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:17.654 | INFO | "malloc" : " 14 GB", | 20260508 13:51:17.654 | INFO | "pinned" : " 29 GB" | 20260508 13:51:17.654 | INFO | } | 20260508 13:51:17.654 | INFO | Perf report for the last 5.13389 seconds: | 20260508 13:51:17.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:17.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:51:17.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:51:17.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:17.654 | INFO | Uploaded 36GiB to GPU in 2.49317s (14.4394GiB/s) | 20260508 13:51:17.654 | INFO | Downloaded 34GiB to GPU in 2.49457s (13.6296GiB/s) | 20260508 13:51:22.804 | INFO | Analyzing chunk frames = [8602624,8603136) | 20260508 13:51:22.804 | INFO | MemInfo: { | 20260508 13:51:22.804 | INFO | "gpu" : " 17 GB", | 20260508 13:51:22.805 | INFO | "huge" : "", | 20260508 13:51:22.805 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:22.805 | INFO | "malloc" : " 14 GB", | 20260508 13:51:22.805 | INFO | "pinned" : " 29 GB" | 20260508 13:51:22.805 | INFO | } | 20260508 13:51:22.805 | INFO | Perf report for the last 5.15082 seconds: | 20260508 13:51:22.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:22.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 13:51:22.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | PRE_HQ : 22.2%, 1.0, 11.6ms, 0 | 20260508 13:51:22.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:22.805 | INFO | Uploaded 36GiB to GPU in 2.47359s (14.5538GiB/s) | 20260508 13:51:22.805 | INFO | Downloaded 34GiB to GPU in 2.45229s (13.8646GiB/s) | 20260508 13:51:27.942 | INFO | Analyzing chunk frames = [8603136,8603648) | 20260508 13:51:27.942 | INFO | MemInfo: { | 20260508 13:51:27.942 | INFO | "gpu" : " 17 GB", | 20260508 13:51:27.942 | INFO | "huge" : "", | 20260508 13:51:27.942 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:27.942 | INFO | "malloc" : " 14 GB", | 20260508 13:51:27.942 | INFO | "pinned" : " 29 GB" | 20260508 13:51:27.942 | INFO | } | 20260508 13:51:27.942 | INFO | Perf report for the last 5.13742 seconds: | 20260508 13:51:27.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:27.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | ANALYSIS : 65.4%, 3.0, 101.4ms, 0 | 20260508 13:51:27.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:51:27.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:27.942 | INFO | Uploaded 36GiB to GPU in 2.39458s (15.034GiB/s) | 20260508 13:51:27.942 | INFO | Downloaded 34GiB to GPU in 2.489s (13.6601GiB/s) | 20260508 13:51:33.073 | INFO | Analyzing chunk frames = [8603648,8604160) | 20260508 13:51:33.073 | INFO | MemInfo: { | 20260508 13:51:33.073 | INFO | "gpu" : " 17 GB", | 20260508 13:51:33.073 | INFO | "huge" : "", | 20260508 13:51:33.074 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:33.074 | INFO | "malloc" : " 14 GB", | 20260508 13:51:33.074 | INFO | "pinned" : " 29 GB" | 20260508 13:51:33.074 | INFO | } | 20260508 13:51:33.074 | INFO | Perf report for the last 5.13166 seconds: | 20260508 13:51:33.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:33.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | ANALYSIS : 66.0%, 3.0, 102.7ms, 0 | 20260508 13:51:33.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | PRE_HQ : 23.7%, 1.0, 12.3ms, 0 | 20260508 13:51:33.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:33.074 | INFO | Uploaded 36GiB to GPU in 2.32146s (15.5075GiB/s) | 20260508 13:51:33.074 | INFO | Downloaded 34GiB to GPU in 2.49479s (13.6284GiB/s) | 20260508 13:51:38.207 | INFO | Analyzing chunk frames = [8604160,8604672) | 20260508 13:51:38.207 | INFO | MemInfo: { | 20260508 13:51:38.207 | INFO | "gpu" : " 17 GB", | 20260508 13:51:38.207 | INFO | "huge" : "", | 20260508 13:51:38.207 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:38.207 | INFO | "malloc" : " 14 GB", | 20260508 13:51:38.207 | INFO | "pinned" : " 29 GB" | 20260508 13:51:38.207 | INFO | } | 20260508 13:51:38.207 | INFO | Perf report for the last 5.13356 seconds: | 20260508 13:51:38.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:38.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.207 | INFO | ANALYSIS : 65.4%, 3.0, 101.6ms, 0 | 20260508 13:51:38.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.207 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:51:38.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:38.208 | INFO | Uploaded 36GiB to GPU in 2.39585s (15.026GiB/s) | 20260508 13:51:38.208 | INFO | Downloaded 34GiB to GPU in 2.44235s (13.921GiB/s) | 20260508 13:51:43.355 | INFO | Analyzing chunk frames = [8604672,8605184) | 20260508 13:51:43.355 | INFO | MemInfo: { | 20260508 13:51:43.355 | INFO | "gpu" : " 17 GB", | 20260508 13:51:43.355 | INFO | "huge" : "", | 20260508 13:51:43.355 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:43.355 | INFO | "malloc" : " 14 GB", | 20260508 13:51:43.355 | INFO | "pinned" : " 29 GB" | 20260508 13:51:43.355 | INFO | } | 20260508 13:51:43.356 | INFO | Perf report for the last 5.14824 seconds: | 20260508 13:51:43.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:43.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | ANALYSIS : 65.4%, 3.0, 102.1ms, 0 | 20260508 13:51:43.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 13:51:43.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:43.356 | INFO | Uploaded 36GiB to GPU in 2.45542s (14.6614GiB/s) | 20260508 13:51:43.356 | INFO | Downloaded 34GiB to GPU in 2.4693s (13.7691GiB/s) | 20260508 13:51:48.496 | INFO | Analyzing chunk frames = [8605184,8605696) | 20260508 13:51:48.496 | INFO | MemInfo: { | 20260508 13:51:48.496 | INFO | "gpu" : " 17 GB", | 20260508 13:51:48.496 | INFO | "huge" : "", | 20260508 13:51:48.496 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:48.496 | INFO | "malloc" : " 14 GB", | 20260508 13:51:48.496 | INFO | "pinned" : " 29 GB" | 20260508 13:51:48.496 | INFO | } | 20260508 13:51:48.496 | INFO | Perf report for the last 5.14037 seconds: | 20260508 13:51:48.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:48.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | ANALYSIS : 64.4%, 3.0, 100.7ms, 0 | 20260508 13:51:48.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:51:48.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:48.496 | INFO | Uploaded 36GiB to GPU in 2.33658s (15.4072GiB/s) | 20260508 13:51:48.497 | INFO | Downloaded 34GiB to GPU in 2.41252s (14.0932GiB/s) | 20260508 13:51:48.594 | INFO | Starting a round of Baz IO | 20260508 13:51:51.995 | INFO | Populating RTMetrics report | 20260508 13:51:51.996 | INFO | RTMetrics report populated | 20260508 13:51:51.996 | INFO | Opening tmp RTMetrics json file | 20260508 13:51:52.001 | INFO | Serializing RTMetrics to json | 20260508 13:51:52.002 | INFO | Serialization done, swapping tmp file now... | 20260508 13:51:52.002 | INFO | RTMetrics done | 20260508 13:51:52.064 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.469 seconds | 20260508 13:51:52.064 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:51:52.064 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132914% / 0% / 7.15256e-05% | 20260508 13:51:52.064 | INFO | Baz IO overall: | 20260508 13:51:52.064 | INFO | BytesWritten : 3657957849758 | 20260508 13:51:52.064 | INFO | NumEvents : 1753232785117 | 20260508 13:51:53.636 | INFO | Analyzing chunk frames = [8605696,8606208) | 20260508 13:51:53.636 | INFO | MemInfo: { | 20260508 13:51:53.636 | INFO | "gpu" : " 17 GB", | 20260508 13:51:53.636 | INFO | "huge" : "", | 20260508 13:51:53.636 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:53.636 | INFO | "malloc" : " 14 GB", | 20260508 13:51:53.636 | INFO | "pinned" : " 29 GB" | 20260508 13:51:53.636 | INFO | } | 20260508 13:51:53.636 | INFO | Perf report for the last 5.14056 seconds: | 20260508 13:51:53.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:53.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:53.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:53.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:53.636 | INFO | ANALYSIS : 67.3%, 3.0, 105.2ms, 0 | 20260508 13:51:53.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:53.637 | INFO | RT_METRICS : 37.8%, 1.0, 19.8ms, 0 | 20260508 13:51:53.637 | INFO | PRE_HQ : 65.1%, 1.6, 53.6ms, 0 | 20260508 13:51:53.637 | INFO | BAZWRITER : 66.6%, 2.4, 83.1ms, 0 | 20260508 13:51:53.637 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:51:53.637 | INFO | Uploaded 36GiB to GPU in 2.61573s (13.7629GiB/s) | 20260508 13:51:53.637 | INFO | Downloaded 38GiB to GPU in 2.71565s (13.993GiB/s) | 20260508 13:51:58.774 | INFO | Analyzing chunk frames = [8606208,8606720) | 20260508 13:51:58.774 | INFO | MemInfo: { | 20260508 13:51:58.774 | INFO | "gpu" : " 17 GB", | 20260508 13:51:58.774 | INFO | "huge" : "", | 20260508 13:51:58.774 | INFO | "huge_shared" : " 29 GB", | 20260508 13:51:58.774 | INFO | "malloc" : " 14 GB", | 20260508 13:51:58.774 | INFO | "pinned" : " 29 GB" | 20260508 13:51:58.774 | INFO | } | 20260508 13:51:58.774 | INFO | Perf report for the last 5.13794 seconds: | 20260508 13:51:58.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:51:58.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.774 | INFO | ANALYSIS : 66.3%, 3.0, 103.2ms, 0 | 20260508 13:51:58.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.775 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:51:58.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:51:58.775 | INFO | Uploaded 36GiB to GPU in 2.51547s (14.3114GiB/s) | 20260508 13:51:58.775 | INFO | Downloaded 34GiB to GPU in 2.46881s (13.7718GiB/s) | 20260508 13:52:03.909 | INFO | Analyzing chunk frames = [8606720,8607232) | 20260508 13:52:03.909 | INFO | MemInfo: { | 20260508 13:52:03.909 | INFO | "gpu" : " 17 GB", | 20260508 13:52:03.909 | INFO | "huge" : "", | 20260508 13:52:03.909 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:03.909 | INFO | "malloc" : " 14 GB", | 20260508 13:52:03.909 | INFO | "pinned" : " 29 GB" | 20260508 13:52:03.910 | INFO | } | 20260508 13:52:03.910 | INFO | Perf report for the last 5.13514 seconds: | 20260508 13:52:03.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:03.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | ANALYSIS : 66.0%, 3.0, 102.5ms, 0 | 20260508 13:52:03.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:52:03.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:03.910 | INFO | Uploaded 36GiB to GPU in 2.48503s (14.4867GiB/s) | 20260508 13:52:03.910 | INFO | Downloaded 34GiB to GPU in 2.45502s (13.8492GiB/s) | 20260508 13:52:09.052 | INFO | Analyzing chunk frames = [8607232,8607744) | 20260508 13:52:09.052 | INFO | MemInfo: { | 20260508 13:52:09.052 | INFO | "gpu" : " 17 GB", | 20260508 13:52:09.052 | INFO | "huge" : "", | 20260508 13:52:09.052 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:09.052 | INFO | "malloc" : " 14 GB", | 20260508 13:52:09.052 | INFO | "pinned" : " 29 GB" | 20260508 13:52:09.052 | INFO | } | 20260508 13:52:09.052 | INFO | Perf report for the last 5.14277 seconds: | 20260508 13:52:09.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:09.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.052 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 13:52:09.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.052 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:52:09.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:09.053 | INFO | Uploaded 36GiB to GPU in 2.54143s (14.1653GiB/s) | 20260508 13:52:09.053 | INFO | Downloaded 34GiB to GPU in 2.45698s (13.8381GiB/s) | 20260508 13:52:14.189 | INFO | Analyzing chunk frames = [8607744,8608256) | 20260508 13:52:14.189 | INFO | MemInfo: { | 20260508 13:52:14.189 | INFO | "gpu" : " 17 GB", | 20260508 13:52:14.189 | INFO | "huge" : "", | 20260508 13:52:14.189 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:14.189 | INFO | "malloc" : " 14 GB", | 20260508 13:52:14.189 | INFO | "pinned" : " 29 GB" | 20260508 13:52:14.189 | INFO | } | 20260508 13:52:14.190 | INFO | Perf report for the last 5.13716 seconds: | 20260508 13:52:14.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:14.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 13:52:14.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260508 13:52:14.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:14.190 | INFO | Uploaded 36GiB to GPU in 2.54031s (14.1715GiB/s) | 20260508 13:52:14.190 | INFO | Downloaded 34GiB to GPU in 2.52988s (13.4394GiB/s) | 20260508 13:52:19.334 | INFO | Analyzing chunk frames = [8608256,8608768) | 20260508 13:52:19.334 | INFO | MemInfo: { | 20260508 13:52:19.334 | INFO | "gpu" : " 17 GB", | 20260508 13:52:19.334 | INFO | "huge" : "", | 20260508 13:52:19.334 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:19.334 | INFO | "malloc" : " 14 GB", | 20260508 13:52:19.334 | INFO | "pinned" : " 29 GB" | 20260508 13:52:19.334 | INFO | } | 20260508 13:52:19.334 | INFO | Perf report for the last 5.14439 seconds: | 20260508 13:52:19.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:19.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.334 | INFO | ANALYSIS : 65.3%, 3.0, 101.7ms, 0 | 20260508 13:52:19.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.334 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:52:19.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:19.335 | INFO | Uploaded 36GiB to GPU in 2.42938s (14.8186GiB/s) | 20260508 13:52:19.335 | INFO | Downloaded 34GiB to GPU in 2.43176s (13.9816GiB/s) | 20260508 13:52:24.471 | INFO | Analyzing chunk frames = [8608768,8609280) | 20260508 13:52:24.471 | INFO | MemInfo: { | 20260508 13:52:24.471 | INFO | "gpu" : " 17 GB", | 20260508 13:52:24.471 | INFO | "huge" : "", | 20260508 13:52:24.471 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:24.471 | INFO | "malloc" : " 14 GB", | 20260508 13:52:24.472 | INFO | "pinned" : " 29 GB" | 20260508 13:52:24.472 | INFO | } | 20260508 13:52:24.472 | INFO | Perf report for the last 5.13769 seconds: | 20260508 13:52:24.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:24.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | ANALYSIS : 64.8%, 3.0, 100.7ms, 0 | 20260508 13:52:24.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | PRE_HQ : 22.8%, 1.0, 11.9ms, 0 | 20260508 13:52:24.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:24.472 | INFO | Uploaded 36GiB to GPU in 2.3761s (15.1509GiB/s) | 20260508 13:52:24.472 | INFO | Downloaded 34GiB to GPU in 2.40397s (14.1433GiB/s) | 20260508 13:52:29.613 | INFO | Analyzing chunk frames = [8609280,8609792) | 20260508 13:52:29.613 | INFO | MemInfo: { | 20260508 13:52:29.613 | INFO | "gpu" : " 17 GB", | 20260508 13:52:29.613 | INFO | "huge" : "", | 20260508 13:52:29.613 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:29.613 | INFO | "malloc" : " 14 GB", | 20260508 13:52:29.613 | INFO | "pinned" : " 29 GB" | 20260508 13:52:29.613 | INFO | } | 20260508 13:52:29.613 | INFO | Perf report for the last 5.14125 seconds: | 20260508 13:52:29.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:29.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | ANALYSIS : 64.8%, 3.0, 101.2ms, 0 | 20260508 13:52:29.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | PRE_HQ : 22.9%, 1.0, 11.9ms, 0 | 20260508 13:52:29.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:29.613 | INFO | Uploaded 36GiB to GPU in 2.40736s (14.9542GiB/s) | 20260508 13:52:29.613 | INFO | Downloaded 34GiB to GPU in 2.40893s (14.1142GiB/s) | 20260508 13:52:29.613 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:52:29.613 | INFO | Total mem usage: 16.7 GiB | 20260508 13:52:29.613 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:52:29.614 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:52:29.614 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:52:29.614 | INFO | High Water Breakdown: | 20260508 13:52:29.614 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:52:29.614 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:52:29.614 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:52:29.614 | INFO | ------------------------------------------ | 20260508 13:52:29.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:52:29.614 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:52:29.614 | INFO | Total mem usage: 26.5 GiB | 20260508 13:52:29.614 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:52:29.614 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:52:29.614 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:52:29.614 | INFO | High Water Breakdown: | 20260508 13:52:29.614 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:52:29.614 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:52:29.614 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:52:29.614 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:52:29.614 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:52:29.614 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:52:29.614 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:52:29.614 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:52:29.614 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:52:29.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:52:29.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:52:29.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:52:29.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:52:29.614 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:52:29.614 | INFO | ------------------------------------------ | 20260508 13:52:29.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:52:29.614 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:52:29.614 | INFO | Total mem usage: 27.8 GiB | 20260508 13:52:29.614 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:52:29.614 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:52:29.614 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:52:29.614 | INFO | High Water Breakdown: | 20260508 13:52:29.614 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:52:29.614 | INFO | ------------------------------------------ | 20260508 13:52:29.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:52:29.614 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:52:29.614 | INFO | Total mem usage: 15.7 GiB | 20260508 13:52:29.614 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:52:29.614 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:52:29.614 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:52:29.614 | INFO | High Water Breakdown: | 20260508 13:52:29.614 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:52:29.614 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:52:29.614 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:52:29.614 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:52:29.614 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:52:29.614 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:52:29.614 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:52:29.614 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:52:29.614 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:52:29.614 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:52:29.614 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:52:29.614 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:52:29.614 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:52:29.614 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:52:29.614 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:52:29.614 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:52:29.614 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:52:29.614 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:52:29.614 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:52:29.614 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:52:29.614 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:52:29.614 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:52:29.614 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:52:29.614 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:52:29.614 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:52:29.614 | INFO | ------------------------------------------ | 20260508 13:52:29.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:52:29.615 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:52:29.615 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:52:29.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:52:29.615 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.53e+01, 1.40e+01 / 5.32e-01 | 20260508 13:52:29.615 | INFO | PerfReport: 0.30%, AggregateStats -- 9.68e-02 / 1.05e-01, 1.12e-02 / 6.52e-03 | 20260508 13:52:29.615 | INFO | PerfReport: 12.23%, Baseline -- 3.99e+00 / 4.39e+00, 5.97e-01 / 1.77e-01 | 20260508 13:52:29.615 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.43e+00, 2.10e-01 / 1.26e-01 | 20260508 13:52:29.615 | INFO | PerfReport: 15.99%, DME -- 4.18e+01 / 4.51e+01, 3.65e+00 / 9.10e-01 | 20260508 13:52:29.615 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.87e+01 / 2.05e+01, 1.86e+00 / 1.27e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.31e+00 / 2.41e+00, 4.01e-01 / 2.58e-01 | 20260508 13:52:29.615 | INFO | PerfReport: 2.90%, Metrics -- 9.49e-01 / 1.04e+00, 2.36e-01 / 9.00e-02 | 20260508 13:52:29.615 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.87e-01 / 3.12e-01, 4.62e-02 / 2.64e-02 | 20260508 13:52:29.615 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:52:29.615 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:52:29.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:52:29.615 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 24.52%, Upload -- 2.57e+01 / 0.00e+00, 7.90e+00 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 24.78%, Download -- 2.59e+01 / 0.00e+00, 8.13e+00 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 31.21%, Compute -- 3.27e+01 / 0.00e+00, 1.40e+01 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 2.17%, LockUpload -- 2.27e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 2.49%, LockDownload -- 2.60e+00 / 0.00e+00, 3.88e+00 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 14.83%, LockCompute -- 1.55e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260508 13:52:29.615 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260508 13:52:29.710 | INFO | Starting a round of Baz IO | 20260508 13:52:33.116 | INFO | Populating RTMetrics report | 20260508 13:52:33.116 | INFO | RTMetrics report populated | 20260508 13:52:33.116 | INFO | Opening tmp RTMetrics json file | 20260508 13:52:33.117 | INFO | Serializing RTMetrics to json | 20260508 13:52:33.118 | INFO | Serialization done, swapping tmp file now... | 20260508 13:52:33.119 | INFO | RTMetrics done | 20260508 13:52:33.176 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.466 seconds | 20260508 13:52:33.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.92% / 0.00% / 60.08% | 20260508 13:52:33.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133542% / 0% / 7.15256e-05% | 20260508 13:52:33.176 | INFO | Baz IO overall: | 20260508 13:52:33.176 | INFO | BytesWritten : 3659436564224 | 20260508 13:52:33.176 | INFO | NumEvents : 1753924075146 | 20260508 13:52:34.749 | INFO | Analyzing chunk frames = [8609792,8610304) | 20260508 13:52:34.749 | INFO | MemInfo: { | 20260508 13:52:34.749 | INFO | "gpu" : " 17 GB", | 20260508 13:52:34.749 | INFO | "huge" : "", | 20260508 13:52:34.749 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:34.749 | INFO | "malloc" : " 14 GB", | 20260508 13:52:34.749 | INFO | "pinned" : " 29 GB" | 20260508 13:52:34.749 | INFO | } | 20260508 13:52:34.749 | INFO | Perf report for the last 5.1361 seconds: | 20260508 13:52:34.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:34.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:34.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:34.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:34.749 | INFO | ANALYSIS : 67.9%, 3.0, 105.5ms, 0 | 20260508 13:52:34.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:34.749 | INFO | RT_METRICS : 37.7%, 1.0, 19.8ms, 0 | 20260508 13:52:34.749 | INFO | PRE_HQ : 65.5%, 1.6, 53.4ms, 0 | 20260508 13:52:34.749 | INFO | BAZWRITER : 66.7%, 2.4, 83.5ms, 0 | 20260508 13:52:34.749 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:52:34.750 | INFO | Uploaded 36GiB to GPU in 2.54857s (14.1256GiB/s) | 20260508 13:52:34.750 | INFO | Downloaded 38GiB to GPU in 2.71095s (14.0172GiB/s) | 20260508 13:52:39.891 | INFO | Analyzing chunk frames = [8610304,8610816) | 20260508 13:52:39.891 | INFO | MemInfo: { | 20260508 13:52:39.891 | INFO | "gpu" : " 17 GB", | 20260508 13:52:39.891 | INFO | "huge" : "", | 20260508 13:52:39.891 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:39.891 | INFO | "malloc" : " 14 GB", | 20260508 13:52:39.891 | INFO | "pinned" : " 29 GB" | 20260508 13:52:39.891 | INFO | } | 20260508 13:52:39.892 | INFO | Perf report for the last 5.14268 seconds: | 20260508 13:52:39.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:39.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:52:39.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:52:39.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:39.892 | INFO | Uploaded 36GiB to GPU in 2.48542s (14.4845GiB/s) | 20260508 13:52:39.892 | INFO | Downloaded 34GiB to GPU in 2.45749s (13.8353GiB/s) | 20260508 13:52:45.024 | INFO | Analyzing chunk frames = [8610816,8611328) | 20260508 13:52:45.024 | INFO | MemInfo: { | 20260508 13:52:45.024 | INFO | "gpu" : " 17 GB", | 20260508 13:52:45.024 | INFO | "huge" : "", | 20260508 13:52:45.024 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:45.024 | INFO | "malloc" : " 14 GB", | 20260508 13:52:45.024 | INFO | "pinned" : " 29 GB" | 20260508 13:52:45.024 | INFO | } | 20260508 13:52:45.024 | INFO | Perf report for the last 5.13259 seconds: | 20260508 13:52:45.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:45.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.024 | INFO | ANALYSIS : 65.9%, 3.0, 102.4ms, 0 | 20260508 13:52:45.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.025 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:52:45.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:45.025 | INFO | Uploaded 36GiB to GPU in 2.48901s (14.4636GiB/s) | 20260508 13:52:45.025 | INFO | Downloaded 34GiB to GPU in 2.46109s (13.815GiB/s) | 20260508 13:52:50.162 | INFO | Analyzing chunk frames = [8611328,8611840) | 20260508 13:52:50.162 | INFO | MemInfo: { | 20260508 13:52:50.162 | INFO | "gpu" : " 17 GB", | 20260508 13:52:50.162 | INFO | "huge" : "", | 20260508 13:52:50.162 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:50.162 | INFO | "malloc" : " 14 GB", | 20260508 13:52:50.162 | INFO | "pinned" : " 29 GB" | 20260508 13:52:50.162 | INFO | } | 20260508 13:52:50.162 | INFO | Perf report for the last 5.13766 seconds: | 20260508 13:52:50.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:50.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | ANALYSIS : 66.3%, 3.0, 102.7ms, 0 | 20260508 13:52:50.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260508 13:52:50.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:50.162 | INFO | Uploaded 36GiB to GPU in 2.54761s (14.1309GiB/s) | 20260508 13:52:50.162 | INFO | Downloaded 34GiB to GPU in 2.46628s (13.7859GiB/s) | 20260508 13:52:55.302 | INFO | Analyzing chunk frames = [8611840,8612352) | 20260508 13:52:55.302 | INFO | MemInfo: { | 20260508 13:52:55.302 | INFO | "gpu" : " 17 GB", | 20260508 13:52:55.303 | INFO | "huge" : "", | 20260508 13:52:55.303 | INFO | "huge_shared" : " 29 GB", | 20260508 13:52:55.303 | INFO | "malloc" : " 14 GB", | 20260508 13:52:55.303 | INFO | "pinned" : " 29 GB" | 20260508 13:52:55.303 | INFO | } | 20260508 13:52:55.303 | INFO | Perf report for the last 5.14077 seconds: | 20260508 13:52:55.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:52:55.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | ANALYSIS : 66.4%, 3.0, 103.6ms, 0 | 20260508 13:52:55.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:52:55.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:52:55.303 | INFO | Uploaded 36GiB to GPU in 2.44482s (14.725GiB/s) | 20260508 13:52:55.303 | INFO | Downloaded 34GiB to GPU in 2.5126s (13.5318GiB/s) | 20260508 13:53:00.446 | INFO | Analyzing chunk frames = [8612352,8612864) | 20260508 13:53:00.446 | INFO | MemInfo: { | 20260508 13:53:00.446 | INFO | "gpu" : " 17 GB", | 20260508 13:53:00.446 | INFO | "huge" : "", | 20260508 13:53:00.446 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:00.446 | INFO | "malloc" : " 14 GB", | 20260508 13:53:00.446 | INFO | "pinned" : " 29 GB" | 20260508 13:53:00.446 | INFO | } | 20260508 13:53:00.446 | INFO | Perf report for the last 5.14369 seconds: | 20260508 13:53:00.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:00.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.446 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:53:00.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.447 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:53:00.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:00.447 | INFO | Uploaded 36GiB to GPU in 2.41886s (14.883GiB/s) | 20260508 13:53:00.447 | INFO | Downloaded 34GiB to GPU in 2.4194s (14.0531GiB/s) | 20260508 13:53:05.584 | INFO | Analyzing chunk frames = [8612864,8613376) | 20260508 13:53:05.584 | INFO | MemInfo: { | 20260508 13:53:05.584 | INFO | "gpu" : " 17 GB", | 20260508 13:53:05.584 | INFO | "huge" : "", | 20260508 13:53:05.584 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:05.584 | INFO | "malloc" : " 14 GB", | 20260508 13:53:05.584 | INFO | "pinned" : " 29 GB" | 20260508 13:53:05.584 | INFO | } | 20260508 13:53:05.585 | INFO | Perf report for the last 5.13805 seconds: | 20260508 13:53:05.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:05.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | ANALYSIS : 65.7%, 3.0, 102.4ms, 0 | 20260508 13:53:05.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:53:05.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:05.585 | INFO | Uploaded 36GiB to GPU in 2.50059s (14.3966GiB/s) | 20260508 13:53:05.586 | INFO | Downloaded 34GiB to GPU in 2.45723s (13.8367GiB/s) | 20260508 13:53:10.720 | INFO | Analyzing chunk frames = [8613376,8613888) | 20260508 13:53:10.720 | INFO | MemInfo: { | 20260508 13:53:10.720 | INFO | "gpu" : " 17 GB", | 20260508 13:53:10.720 | INFO | "huge" : "", | 20260508 13:53:10.720 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:10.720 | INFO | "malloc" : " 14 GB", | 20260508 13:53:10.720 | INFO | "pinned" : " 29 GB" | 20260508 13:53:10.720 | INFO | } | 20260508 13:53:10.720 | INFO | Perf report for the last 5.13563 seconds: | 20260508 13:53:10.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:10.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | ANALYSIS : 64.9%, 3.0, 101.4ms, 0 | 20260508 13:53:10.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:53:10.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:10.720 | INFO | Uploaded 36GiB to GPU in 2.39639s (15.0226GiB/s) | 20260508 13:53:10.720 | INFO | Downloaded 34GiB to GPU in 2.42887s (13.9983GiB/s) | 20260508 13:53:10.817 | INFO | Starting a round of Baz IO | 20260508 13:53:14.227 | INFO | Populating RTMetrics report | 20260508 13:53:14.228 | INFO | RTMetrics report populated | 20260508 13:53:14.228 | INFO | Opening tmp RTMetrics json file | 20260508 13:53:14.232 | INFO | Serializing RTMetrics to json | 20260508 13:53:14.233 | INFO | Serialization done, swapping tmp file now... | 20260508 13:53:14.234 | INFO | RTMetrics done | 20260508 13:53:14.293 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.475 seconds | 20260508 13:53:14.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.08% | 20260508 13:53:14.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133173% / 7.94729e-06% / 6.75519e-05% | 20260508 13:53:14.293 | INFO | Baz IO overall: | 20260508 13:53:14.293 | INFO | BytesWritten : 3660914679443 | 20260508 13:53:14.293 | INFO | NumEvents : 1754615072987 | 20260508 13:53:15.865 | INFO | Analyzing chunk frames = [8613888,8614400) | 20260508 13:53:15.865 | INFO | MemInfo: { | 20260508 13:53:15.865 | INFO | "gpu" : " 17 GB", | 20260508 13:53:15.865 | INFO | "huge" : "", | 20260508 13:53:15.865 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:15.865 | INFO | "malloc" : " 14 GB", | 20260508 13:53:15.865 | INFO | "pinned" : " 29 GB" | 20260508 13:53:15.865 | INFO | } | 20260508 13:53:15.865 | INFO | Perf report for the last 5.14509 seconds: | 20260508 13:53:15.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:15.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:15.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:15.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:15.865 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:53:15.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:15.865 | INFO | RT_METRICS : 37.8%, 1.0, 19.9ms, 0 | 20260508 13:53:15.865 | INFO | PRE_HQ : 65.1%, 1.6, 53.4ms, 0 | 20260508 13:53:15.865 | INFO | BAZWRITER : 66.8%, 2.4, 83.7ms, 0 | 20260508 13:53:15.865 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:53:15.865 | INFO | Uploaded 36GiB to GPU in 2.62184s (13.7308GiB/s) | 20260508 13:53:15.865 | INFO | Downloaded 38GiB to GPU in 2.73117s (13.9135GiB/s) | 20260508 13:53:21.001 | INFO | Analyzing chunk frames = [8614400,8614912) | 20260508 13:53:21.001 | INFO | MemInfo: { | 20260508 13:53:21.001 | INFO | "gpu" : " 17 GB", | 20260508 13:53:21.001 | INFO | "huge" : "", | 20260508 13:53:21.001 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:21.001 | INFO | "malloc" : " 14 GB", | 20260508 13:53:21.001 | INFO | "pinned" : " 29 GB" | 20260508 13:53:21.001 | INFO | } | 20260508 13:53:21.001 | INFO | Perf report for the last 5.136 seconds: | 20260508 13:53:21.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:21.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.001 | INFO | ANALYSIS : 66.5%, 3.0, 103.6ms, 0 | 20260508 13:53:21.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.001 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:53:21.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:21.002 | INFO | Uploaded 36GiB to GPU in 2.52596s (14.252GiB/s) | 20260508 13:53:21.002 | INFO | Downloaded 34GiB to GPU in 2.46151s (13.8127GiB/s) | 20260508 13:53:26.145 | INFO | Analyzing chunk frames = [8614912,8615424) | 20260508 13:53:26.145 | INFO | MemInfo: { | 20260508 13:53:26.145 | INFO | "gpu" : " 17 GB", | 20260508 13:53:26.145 | INFO | "huge" : "", | 20260508 13:53:26.145 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:26.145 | INFO | "malloc" : " 14 GB", | 20260508 13:53:26.145 | INFO | "pinned" : " 29 GB" | 20260508 13:53:26.145 | INFO | } | 20260508 13:53:26.146 | INFO | Perf report for the last 5.14423 seconds: | 20260508 13:53:26.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:26.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | ANALYSIS : 65.9%, 3.0, 102.6ms, 0 | 20260508 13:53:26.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:53:26.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:26.146 | INFO | Uploaded 36GiB to GPU in 2.4685s (14.5837GiB/s) | 20260508 13:53:26.146 | INFO | Downloaded 34GiB to GPU in 2.44737s (13.8925GiB/s) | 20260508 13:53:31.282 | INFO | Analyzing chunk frames = [8615424,8615936) | 20260508 13:53:31.282 | INFO | MemInfo: { | 20260508 13:53:31.282 | INFO | "gpu" : " 17 GB", | 20260508 13:53:31.282 | INFO | "huge" : "", | 20260508 13:53:31.282 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:31.282 | INFO | "malloc" : " 14 GB", | 20260508 13:53:31.282 | INFO | "pinned" : " 29 GB" | 20260508 13:53:31.282 | INFO | } | 20260508 13:53:31.282 | INFO | Perf report for the last 5.13639 seconds: | 20260508 13:53:31.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:31.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:53:31.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:53:31.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:31.282 | INFO | Uploaded 36GiB to GPU in 2.41815s (14.8874GiB/s) | 20260508 13:53:31.282 | INFO | Downloaded 34GiB to GPU in 2.46778s (13.7776GiB/s) | 20260508 13:53:36.420 | INFO | Analyzing chunk frames = [8615936,8616448) | 20260508 13:53:36.420 | INFO | MemInfo: { | 20260508 13:53:36.420 | INFO | "gpu" : " 17 GB", | 20260508 13:53:36.420 | INFO | "huge" : "", | 20260508 13:53:36.420 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:36.420 | INFO | "malloc" : " 14 GB", | 20260508 13:53:36.420 | INFO | "pinned" : " 29 GB" | 20260508 13:53:36.420 | INFO | } | 20260508 13:53:36.420 | INFO | Perf report for the last 5.13824 seconds: | 20260508 13:53:36.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:36.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.420 | INFO | ANALYSIS : 66.7%, 3.0, 104.0ms, 0 | 20260508 13:53:36.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.420 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:53:36.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:36.421 | INFO | Uploaded 36GiB to GPU in 2.55371s (14.0972GiB/s) | 20260508 13:53:36.421 | INFO | Downloaded 34GiB to GPU in 2.53318s (13.4219GiB/s) | 20260508 13:53:41.565 | INFO | Analyzing chunk frames = [8616448,8616960) | 20260508 13:53:41.565 | INFO | MemInfo: { | 20260508 13:53:41.565 | INFO | "gpu" : " 17 GB", | 20260508 13:53:41.565 | INFO | "huge" : "", | 20260508 13:53:41.565 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:41.565 | INFO | "malloc" : " 14 GB", | 20260508 13:53:41.565 | INFO | "pinned" : " 29 GB" | 20260508 13:53:41.565 | INFO | } | 20260508 13:53:41.565 | INFO | Perf report for the last 5.14513 seconds: | 20260508 13:53:41.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:41.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.565 | INFO | ANALYSIS : 65.5%, 3.0, 102.0ms, 0 | 20260508 13:53:41.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.565 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:53:41.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:41.566 | INFO | Uploaded 36GiB to GPU in 2.46223s (14.6209GiB/s) | 20260508 13:53:41.566 | INFO | Downloaded 34GiB to GPU in 2.44042s (13.9321GiB/s) | 20260508 13:53:46.699 | INFO | Analyzing chunk frames = [8616960,8617472) | 20260508 13:53:46.699 | INFO | MemInfo: { | 20260508 13:53:46.699 | INFO | "gpu" : " 17 GB", | 20260508 13:53:46.699 | INFO | "huge" : "", | 20260508 13:53:46.699 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:46.699 | INFO | "malloc" : " 14 GB", | 20260508 13:53:46.699 | INFO | "pinned" : " 29 GB" | 20260508 13:53:46.699 | INFO | } | 20260508 13:53:46.699 | INFO | Perf report for the last 5.13359 seconds: | 20260508 13:53:46.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:46.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | ANALYSIS : 64.6%, 3.0, 100.4ms, 0 | 20260508 13:53:46.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:53:46.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:46.699 | INFO | Uploaded 36GiB to GPU in 2.33274s (15.4325GiB/s) | 20260508 13:53:46.699 | INFO | Downloaded 34GiB to GPU in 2.397s (14.1844GiB/s) | 20260508 13:53:51.837 | INFO | Analyzing chunk frames = [8617472,8617984) | 20260508 13:53:51.837 | INFO | MemInfo: { | 20260508 13:53:51.837 | INFO | "gpu" : " 17 GB", | 20260508 13:53:51.838 | INFO | "huge" : "", | 20260508 13:53:51.838 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:51.838 | INFO | "malloc" : " 14 GB", | 20260508 13:53:51.838 | INFO | "pinned" : " 29 GB" | 20260508 13:53:51.838 | INFO | } | 20260508 13:53:51.838 | INFO | Perf report for the last 5.1389 seconds: | 20260508 13:53:51.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:51.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 13:53:51.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:53:51.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:51.838 | INFO | Uploaded 36GiB to GPU in 2.49295s (14.4407GiB/s) | 20260508 13:53:51.838 | INFO | Downloaded 34GiB to GPU in 2.45399s (13.855GiB/s) | 20260508 13:53:51.838 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:53:51.838 | INFO | Total mem usage: 16.7 GiB | 20260508 13:53:51.838 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:53:51.838 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -17.2%) | 20260508 13:53:51.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:53:51.838 | INFO | High Water Breakdown: | 20260508 13:53:51.838 | INFO | 59.5% : 8.46 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:53:51.839 | INFO | 40.4% : 5.75 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:53:51.839 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:53:51.839 | INFO | ------------------------------------------ | 20260508 13:53:51.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:53:51.839 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:53:51.839 | INFO | Total mem usage: 26.5 GiB | 20260508 13:53:51.839 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:53:51.839 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4.15%) | 20260508 13:53:51.839 | INFO | Unused Cache: 1.88 GiB (Waste: 7.09%) | 20260508 13:53:51.839 | INFO | High Water Breakdown: | 20260508 13:53:51.839 | INFO | 56.4% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:53:51.839 | INFO | 36.5% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:53:51.839 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:53:51.839 | INFO | 1.6% : 418 MiB ( 66.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:53:51.839 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:53:51.839 | INFO | 0.42% : 110 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:53:51.839 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:53:51.839 | INFO | 9.4e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:53:51.839 | INFO | 8.78e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:53:51.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:53:51.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:53:51.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:53:51.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:53:51.839 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:53:51.839 | INFO | ------------------------------------------ | 20260508 13:53:51.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:53:51.839 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:53:51.839 | INFO | Total mem usage: 27.8 GiB | 20260508 13:53:51.839 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:53:51.839 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:53:51.839 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:53:51.839 | INFO | High Water Breakdown: | 20260508 13:53:51.839 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:53:51.839 | INFO | ------------------------------------------ | 20260508 13:53:51.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:53:51.839 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:53:51.839 | INFO | Total mem usage: 15.7 GiB | 20260508 13:53:51.839 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:53:51.839 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:53:51.839 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:53:51.839 | INFO | High Water Breakdown: | 20260508 13:53:51.839 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:53:51.839 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:53:51.839 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:53:51.839 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:53:51.839 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:53:51.839 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:53:51.839 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:53:51.839 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:53:51.839 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:53:51.839 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:53:51.839 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:53:51.839 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:53:51.839 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:53:51.839 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:53:51.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:53:51.840 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:53:51.840 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:53:51.840 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:53:51.840 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:53:51.840 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:53:51.840 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:53:51.840 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:53:51.840 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:53:51.840 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:53:51.840 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:53:51.840 | INFO | ------------------------------------------ | 20260508 13:53:51.840 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:53:51.840 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:53:51.840 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:53:51.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:53:51.840 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.39e+01 / 5.95e-01 | 20260508 13:53:51.840 | INFO | PerfReport: 0.30%, AggregateStats -- 9.65e-02 / 1.03e-01, 1.12e-02 / 5.05e-03 | 20260508 13:53:51.840 | INFO | PerfReport: 12.20%, Baseline -- 3.99e+00 / 4.46e+00, 5.95e-01 / 1.95e-01 | 20260508 13:53:51.840 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.43e+00, 2.08e-01 / 1.43e-01 | 20260508 13:53:51.840 | INFO | PerfReport: 15.97%, DME -- 4.18e+01 / 4.50e+01, 3.58e+00 / 8.23e-01 | 20260508 13:53:51.840 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.88e+00 / 1.40e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.30e+00 / 2.44e+00, 3.99e-01 / 2.36e-01 | 20260508 13:53:51.840 | INFO | PerfReport: 2.91%, Metrics -- 9.49e-01 / 1.09e+00, 2.41e-01 / 1.01e-01 | 20260508 13:53:51.840 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.87e-01 / 3.24e-01, 4.62e-02 / 2.55e-02 | 20260508 13:53:51.840 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:53:51.840 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:53:51.840 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:53:51.840 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.35e+01 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 24.63%, Upload -- 2.59e+01 / 0.00e+00, 8.08e+00 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 24.71%, Download -- 2.59e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 31.13%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 2.28%, LockUpload -- 2.40e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 2.43%, LockDownload -- 2.56e+00 / 0.00e+00, 3.87e+00 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 14.80%, LockCompute -- 1.55e+01 / 0.00e+00, 1.74e+01 / 0.00e+00 | 20260508 13:53:51.840 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260508 13:53:51.952 | INFO | Starting a round of Baz IO | 20260508 13:53:55.337 | INFO | Populating RTMetrics report | 20260508 13:53:55.338 | INFO | RTMetrics report populated | 20260508 13:53:55.338 | INFO | Opening tmp RTMetrics json file | 20260508 13:53:55.348 | INFO | Serializing RTMetrics to json | 20260508 13:53:55.349 | INFO | Serialization done, swapping tmp file now... | 20260508 13:53:55.350 | INFO | RTMetrics done | 20260508 13:53:55.406 | INFO | Finished a round of Baz IO: Wrote 1.375 GiB in 3.453 seconds | 20260508 13:53:55.406 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:53:55.406 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131834% / 0% / 7.15256e-05% | 20260508 13:53:55.406 | INFO | Baz IO overall: | 20260508 13:53:55.406 | INFO | BytesWritten : 3662390765402 | 20260508 13:53:55.406 | INFO | NumEvents : 1755304582251 | 20260508 13:53:56.972 | INFO | Analyzing chunk frames = [8617984,8618496) | 20260508 13:53:56.972 | INFO | MemInfo: { | 20260508 13:53:56.972 | INFO | "gpu" : " 17 GB", | 20260508 13:53:56.972 | INFO | "huge" : "", | 20260508 13:53:56.972 | INFO | "huge_shared" : " 29 GB", | 20260508 13:53:56.972 | INFO | "malloc" : " 14 GB", | 20260508 13:53:56.972 | INFO | "pinned" : " 29 GB" | 20260508 13:53:56.972 | INFO | } | 20260508 13:53:56.972 | INFO | Perf report for the last 5.13403 seconds: | 20260508 13:53:56.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:53:56.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:56.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:56.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:56.972 | INFO | ANALYSIS : 67.6%, 3.0, 105.3ms, 0 | 20260508 13:53:56.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:53:56.972 | INFO | RT_METRICS : 40.6%, 1.0, 21.3ms, 0 | 20260508 13:53:56.972 | INFO | PRE_HQ : 65.9%, 1.7, 57.6ms, 0 | 20260508 13:53:56.972 | INFO | BAZWRITER : 66.6%, 2.5, 86.9ms, 0 | 20260508 13:53:56.972 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:53:56.972 | INFO | Uploaded 36GiB to GPU in 2.6226s (13.7269GiB/s) | 20260508 13:53:56.973 | INFO | Downloaded 38GiB to GPU in 2.71656s (13.9883GiB/s) | 20260508 13:54:02.116 | INFO | Analyzing chunk frames = [8618496,8619008) | 20260508 13:54:02.116 | INFO | MemInfo: { | 20260508 13:54:02.116 | INFO | "gpu" : " 17 GB", | 20260508 13:54:02.116 | INFO | "huge" : "", | 20260508 13:54:02.116 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:02.116 | INFO | "malloc" : " 14 GB", | 20260508 13:54:02.116 | INFO | "pinned" : " 29 GB" | 20260508 13:54:02.116 | INFO | } | 20260508 13:54:02.117 | INFO | Perf report for the last 5.14463 seconds: | 20260508 13:54:02.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:02.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | ANALYSIS : 66.1%, 3.0, 103.1ms, 0 | 20260508 13:54:02.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | PRE_HQ : 22.3%, 1.0, 11.6ms, 0 | 20260508 13:54:02.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:02.117 | INFO | Uploaded 36GiB to GPU in 2.4447s (14.7257GiB/s) | 20260508 13:54:02.117 | INFO | Downloaded 34GiB to GPU in 2.48166s (13.7005GiB/s) | 20260508 13:54:07.257 | INFO | Analyzing chunk frames = [8619008,8619520) | 20260508 13:54:07.257 | INFO | MemInfo: { | 20260508 13:54:07.257 | INFO | "gpu" : " 17 GB", | 20260508 13:54:07.257 | INFO | "huge" : "", | 20260508 13:54:07.257 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:07.257 | INFO | "malloc" : " 14 GB", | 20260508 13:54:07.257 | INFO | "pinned" : " 29 GB" | 20260508 13:54:07.257 | INFO | } | 20260508 13:54:07.257 | INFO | Perf report for the last 5.14091 seconds: | 20260508 13:54:07.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:07.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.257 | INFO | ANALYSIS : 66.3%, 3.0, 103.0ms, 0 | 20260508 13:54:07.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.258 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:54:07.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:07.258 | INFO | Uploaded 36GiB to GPU in 2.48094s (14.5106GiB/s) | 20260508 13:54:07.258 | INFO | Downloaded 34GiB to GPU in 2.45264s (13.8626GiB/s) | 20260508 13:54:12.399 | INFO | Analyzing chunk frames = [8619520,8620032) | 20260508 13:54:12.399 | INFO | MemInfo: { | 20260508 13:54:12.399 | INFO | "gpu" : " 17 GB", | 20260508 13:54:12.399 | INFO | "huge" : "", | 20260508 13:54:12.399 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:12.399 | INFO | "malloc" : " 14 GB", | 20260508 13:54:12.399 | INFO | "pinned" : " 29 GB" | 20260508 13:54:12.399 | INFO | } | 20260508 13:54:12.399 | INFO | Perf report for the last 5.14198 seconds: | 20260508 13:54:12.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:12.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.400 | INFO | ANALYSIS : 65.3%, 3.0, 101.3ms, 0 | 20260508 13:54:12.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.400 | INFO | PRE_HQ : 22.6%, 1.0, 11.8ms, 0 | 20260508 13:54:12.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:12.400 | INFO | Uploaded 36GiB to GPU in 2.3834s (15.1045GiB/s) | 20260508 13:54:12.400 | INFO | Downloaded 34GiB to GPU in 2.46345s (13.8018GiB/s) | 20260508 13:54:17.533 | INFO | Analyzing chunk frames = [8620032,8620544) | 20260508 13:54:17.533 | INFO | MemInfo: { | 20260508 13:54:17.533 | INFO | "gpu" : " 17 GB", | 20260508 13:54:17.533 | INFO | "huge" : "", | 20260508 13:54:17.533 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:17.533 | INFO | "malloc" : " 14 GB", | 20260508 13:54:17.533 | INFO | "pinned" : " 29 GB" | 20260508 13:54:17.533 | INFO | } | 20260508 13:54:17.533 | INFO | Perf report for the last 5.13377 seconds: | 20260508 13:54:17.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:17.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.533 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:54:17.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.533 | INFO | PRE_HQ : 23.8%, 1.0, 12.4ms, 0 | 20260508 13:54:17.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:17.534 | INFO | Uploaded 36GiB to GPU in 2.42414s (14.8506GiB/s) | 20260508 13:54:17.534 | INFO | Downloaded 34GiB to GPU in 2.53069s (13.4351GiB/s) | 20260508 13:54:22.672 | INFO | Analyzing chunk frames = [8620544,8621056) | 20260508 13:54:22.672 | INFO | MemInfo: { | 20260508 13:54:22.672 | INFO | "gpu" : " 17 GB", | 20260508 13:54:22.672 | INFO | "huge" : "", | 20260508 13:54:22.672 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:22.672 | INFO | "malloc" : " 14 GB", | 20260508 13:54:22.672 | INFO | "pinned" : " 29 GB" | 20260508 13:54:22.672 | INFO | } | 20260508 13:54:22.672 | INFO | Perf report for the last 5.1386 seconds: | 20260508 13:54:22.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:22.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | ANALYSIS : 65.5%, 3.0, 101.9ms, 0 | 20260508 13:54:22.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | PRE_HQ : 22.8%, 1.0, 11.8ms, 0 | 20260508 13:54:22.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:22.672 | INFO | Uploaded 36GiB to GPU in 2.44326s (14.7344GiB/s) | 20260508 13:54:22.672 | INFO | Downloaded 34GiB to GPU in 2.45295s (13.8609GiB/s) | 20260508 13:54:27.811 | INFO | Analyzing chunk frames = [8621056,8621568) | 20260508 13:54:27.811 | INFO | MemInfo: { | 20260508 13:54:27.811 | INFO | "gpu" : " 17 GB", | 20260508 13:54:27.811 | INFO | "huge" : "", | 20260508 13:54:27.811 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:27.811 | INFO | "malloc" : " 14 GB", | 20260508 13:54:27.811 | INFO | "pinned" : " 29 GB" | 20260508 13:54:27.811 | INFO | } | 20260508 13:54:27.811 | INFO | Perf report for the last 5.13948 seconds: | 20260508 13:54:27.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:27.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.811 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 13:54:27.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.811 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260508 13:54:27.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:27.812 | INFO | Uploaded 36GiB to GPU in 2.33913s (15.3903GiB/s) | 20260508 13:54:27.812 | INFO | Downloaded 34GiB to GPU in 2.41053s (14.1048GiB/s) | 20260508 13:54:32.950 | INFO | Analyzing chunk frames = [8621568,8622080) | 20260508 13:54:32.950 | INFO | MemInfo: { | 20260508 13:54:32.950 | INFO | "gpu" : " 17 GB", | 20260508 13:54:32.950 | INFO | "huge" : "", | 20260508 13:54:32.950 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:32.950 | INFO | "malloc" : " 14 GB", | 20260508 13:54:32.950 | INFO | "pinned" : " 29 GB" | 20260508 13:54:32.950 | INFO | } | 20260508 13:54:32.950 | INFO | Perf report for the last 5.13875 seconds: | 20260508 13:54:32.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:32.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.950 | INFO | ANALYSIS : 65.8%, 3.0, 102.8ms, 0 | 20260508 13:54:32.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.950 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:54:32.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:32.951 | INFO | Uploaded 36GiB to GPU in 2.46262s (14.6186GiB/s) | 20260508 13:54:32.951 | INFO | Downloaded 34GiB to GPU in 2.46907s (13.7704GiB/s) | 20260508 13:54:33.047 | INFO | Starting a round of Baz IO | 20260508 13:54:36.449 | INFO | Populating RTMetrics report | 20260508 13:54:36.449 | INFO | RTMetrics report populated | 20260508 13:54:36.449 | INFO | Opening tmp RTMetrics json file | 20260508 13:54:36.457 | INFO | Serializing RTMetrics to json | 20260508 13:54:36.458 | INFO | Serialization done, swapping tmp file now... | 20260508 13:54:36.459 | INFO | RTMetrics done | 20260508 13:54:36.521 | INFO | Finished a round of Baz IO: Wrote 1.375 GiB in 3.473 seconds | 20260508 13:54:36.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:54:36.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.13206% / 0% / 7.15256e-05% | 20260508 13:54:36.521 | INFO | Baz IO overall: | 20260508 13:54:36.521 | INFO | BytesWritten : 3663866715010 | 20260508 13:54:36.521 | INFO | NumEvents : 1755994265154 | 20260508 13:54:38.095 | INFO | Analyzing chunk frames = [8622080,8622592) | 20260508 13:54:38.095 | INFO | MemInfo: { | 20260508 13:54:38.095 | INFO | "gpu" : " 17 GB", | 20260508 13:54:38.095 | INFO | "huge" : "", | 20260508 13:54:38.095 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:38.095 | INFO | "malloc" : " 14 GB", | 20260508 13:54:38.095 | INFO | "pinned" : " 29 GB" | 20260508 13:54:38.095 | INFO | } | 20260508 13:54:38.095 | INFO | Perf report for the last 5.14513 seconds: | 20260508 13:54:38.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:38.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:38.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:38.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:38.095 | INFO | ANALYSIS : 67.4%, 3.0, 105.4ms, 0 | 20260508 13:54:38.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:38.095 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:54:38.095 | INFO | PRE_HQ : 65.2%, 1.6, 53.7ms, 0 | 20260508 13:54:38.095 | INFO | BAZWRITER : 66.8%, 2.4, 83.9ms, 0 | 20260508 13:54:38.095 | INFO | BAZ_SAMPLING : 0.3%, 1.0, 0.2ms, 0 | 20260508 13:54:38.096 | INFO | Uploaded 36GiB to GPU in 2.63616s (13.6562GiB/s) | 20260508 13:54:38.096 | INFO | Downloaded 38GiB to GPU in 2.7183s (13.9793GiB/s) | 20260508 13:54:43.232 | INFO | Analyzing chunk frames = [8622592,8623104) | 20260508 13:54:43.232 | INFO | MemInfo: { | 20260508 13:54:43.232 | INFO | "gpu" : " 17 GB", | 20260508 13:54:43.232 | INFO | "huge" : "", | 20260508 13:54:43.232 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:43.232 | INFO | "malloc" : " 14 GB", | 20260508 13:54:43.232 | INFO | "pinned" : " 29 GB" | 20260508 13:54:43.232 | INFO | } | 20260508 13:54:43.232 | INFO | Perf report for the last 5.13661 seconds: | 20260508 13:54:43.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:43.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | ANALYSIS : 66.0%, 3.0, 102.8ms, 0 | 20260508 13:54:43.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:54:43.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:43.232 | INFO | Uploaded 36GiB to GPU in 2.42773s (14.8287GiB/s) | 20260508 13:54:43.232 | INFO | Downloaded 34GiB to GPU in 2.47004s (13.7649GiB/s) | 20260508 13:54:48.372 | INFO | Analyzing chunk frames = [8623104,8623616) | 20260508 13:54:48.372 | INFO | MemInfo: { | 20260508 13:54:48.372 | INFO | "gpu" : " 17 GB", | 20260508 13:54:48.372 | INFO | "huge" : "", | 20260508 13:54:48.372 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:48.372 | INFO | "malloc" : " 14 GB", | 20260508 13:54:48.372 | INFO | "pinned" : " 29 GB" | 20260508 13:54:48.372 | INFO | } | 20260508 13:54:48.372 | INFO | Perf report for the last 5.14017 seconds: | 20260508 13:54:48.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:48.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.372 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:54:48.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.372 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:54:48.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:48.373 | INFO | Uploaded 36GiB to GPU in 2.51123s (14.3356GiB/s) | 20260508 13:54:48.373 | INFO | Downloaded 34GiB to GPU in 2.47058s (13.762GiB/s) | 20260508 13:54:53.513 | INFO | Analyzing chunk frames = [8623616,8624128) | 20260508 13:54:53.513 | INFO | MemInfo: { | 20260508 13:54:53.513 | INFO | "gpu" : " 17 GB", | 20260508 13:54:53.513 | INFO | "huge" : "", | 20260508 13:54:53.513 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:53.513 | INFO | "malloc" : " 14 GB", | 20260508 13:54:53.513 | INFO | "pinned" : " 29 GB" | 20260508 13:54:53.513 | INFO | } | 20260508 13:54:53.513 | INFO | Perf report for the last 5.14116 seconds: | 20260508 13:54:53.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:53.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.513 | INFO | ANALYSIS : 65.9%, 3.0, 102.2ms, 0 | 20260508 13:54:53.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.513 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:54:53.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:53.514 | INFO | Uploaded 36GiB to GPU in 2.50371s (14.3787GiB/s) | 20260508 13:54:53.514 | INFO | Downloaded 34GiB to GPU in 2.46324s (13.8029GiB/s) | 20260508 13:54:58.646 | INFO | Analyzing chunk frames = [8624128,8624640) | 20260508 13:54:58.646 | INFO | MemInfo: { | 20260508 13:54:58.646 | INFO | "gpu" : " 17 GB", | 20260508 13:54:58.646 | INFO | "huge" : "", | 20260508 13:54:58.646 | INFO | "huge_shared" : " 29 GB", | 20260508 13:54:58.646 | INFO | "malloc" : " 14 GB", | 20260508 13:54:58.646 | INFO | "pinned" : " 29 GB" | 20260508 13:54:58.646 | INFO | } | 20260508 13:54:58.647 | INFO | Perf report for the last 5.13326 seconds: | 20260508 13:54:58.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:54:58.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | ANALYSIS : 66.4%, 3.0, 103.4ms, 0 | 20260508 13:54:58.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260508 13:54:58.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:54:58.647 | INFO | Uploaded 36GiB to GPU in 2.41506s (14.9065GiB/s) | 20260508 13:54:58.647 | INFO | Downloaded 34GiB to GPU in 2.53083s (13.4343GiB/s) | 20260508 13:55:03.786 | INFO | Analyzing chunk frames = [8624640,8625152) | 20260508 13:55:03.786 | INFO | MemInfo: { | 20260508 13:55:03.786 | INFO | "gpu" : " 17 GB", | 20260508 13:55:03.786 | INFO | "huge" : "", | 20260508 13:55:03.786 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:03.786 | INFO | "malloc" : " 14 GB", | 20260508 13:55:03.787 | INFO | "pinned" : " 29 GB" | 20260508 13:55:03.787 | INFO | } | 20260508 13:55:03.787 | INFO | Perf report for the last 5.14028 seconds: | 20260508 13:55:03.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:03.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.787 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:55:03.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.787 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 13:55:03.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:03.788 | INFO | Uploaded 36GiB to GPU in 2.39069s (15.0584GiB/s) | 20260508 13:55:03.788 | INFO | Downloaded 34GiB to GPU in 2.44371s (13.9133GiB/s) | 20260508 13:55:08.925 | INFO | Analyzing chunk frames = [8625152,8625664) | 20260508 13:55:08.926 | INFO | MemInfo: { | 20260508 13:55:08.926 | INFO | "gpu" : " 17 GB", | 20260508 13:55:08.926 | INFO | "huge" : "", | 20260508 13:55:08.926 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:08.926 | INFO | "malloc" : " 14 GB", | 20260508 13:55:08.926 | INFO | "pinned" : " 29 GB" | 20260508 13:55:08.926 | INFO | } | 20260508 13:55:08.926 | INFO | Perf report for the last 5.13906 seconds: | 20260508 13:55:08.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:08.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | ANALYSIS : 64.7%, 3.0, 100.6ms, 0 | 20260508 13:55:08.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260508 13:55:08.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:08.926 | INFO | Uploaded 36GiB to GPU in 2.31179s (15.5723GiB/s) | 20260508 13:55:08.926 | INFO | Downloaded 34GiB to GPU in 2.42472s (14.0222GiB/s) | 20260508 13:55:14.071 | INFO | Analyzing chunk frames = [8625664,8626176) | 20260508 13:55:14.071 | INFO | MemInfo: { | 20260508 13:55:14.071 | INFO | "gpu" : " 17 GB", | 20260508 13:55:14.072 | INFO | "huge" : "", | 20260508 13:55:14.072 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:14.072 | INFO | "malloc" : " 14 GB", | 20260508 13:55:14.072 | INFO | "pinned" : " 29 GB" | 20260508 13:55:14.072 | INFO | } | 20260508 13:55:14.072 | INFO | Perf report for the last 5.14592 seconds: | 20260508 13:55:14.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:14.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | ANALYSIS : 64.6%, 3.0, 101.1ms, 0 | 20260508 13:55:14.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260508 13:55:14.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:14.072 | INFO | Uploaded 36GiB to GPU in 2.37367s (15.1664GiB/s) | 20260508 13:55:14.072 | INFO | Downloaded 34GiB to GPU in 2.43784s (13.9467GiB/s) | 20260508 13:55:14.073 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:55:14.073 | INFO | Total mem usage: 16.7 GiB | 20260508 13:55:14.073 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:55:14.073 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.8%) | 20260508 13:55:14.073 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:55:14.073 | INFO | High Water Breakdown: | 20260508 13:55:14.073 | INFO | 59.6% : 8.51 GiB ( 82.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:55:14.073 | INFO | 40.3% : 5.75 GiB ( 85.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:55:14.073 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:55:14.073 | INFO | ------------------------------------------ | 20260508 13:55:14.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:55:14.073 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:55:14.073 | INFO | Total mem usage: 26.5 GiB | 20260508 13:55:14.073 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:55:14.073 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:55:14.073 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:55:14.073 | INFO | High Water Breakdown: | 20260508 13:55:14.073 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:55:14.073 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:55:14.073 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:55:14.073 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:55:14.073 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:55:14.073 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:55:14.073 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:55:14.073 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:55:14.073 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:55:14.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:55:14.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:55:14.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:55:14.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:55:14.073 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:55:14.073 | INFO | ------------------------------------------ | 20260508 13:55:14.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:55:14.073 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:55:14.073 | INFO | Total mem usage: 27.8 GiB | 20260508 13:55:14.073 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:55:14.073 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:55:14.073 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:55:14.073 | INFO | High Water Breakdown: | 20260508 13:55:14.073 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:55:14.073 | INFO | ------------------------------------------ | 20260508 13:55:14.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:55:14.073 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:55:14.073 | INFO | Total mem usage: 15.7 GiB | 20260508 13:55:14.073 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:55:14.073 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:55:14.073 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:55:14.073 | INFO | High Water Breakdown: | 20260508 13:55:14.073 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:55:14.073 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:55:14.073 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:55:14.073 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:55:14.073 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:55:14.073 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:55:14.073 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:55:14.073 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:55:14.073 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:55:14.073 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:55:14.073 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:55:14.073 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:55:14.074 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:55:14.074 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:55:14.074 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:55:14.074 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:55:14.074 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:55:14.074 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:55:14.074 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:55:14.074 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:55:14.074 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:55:14.074 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:55:14.074 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:55:14.074 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:55:14.074 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:55:14.074 | INFO | ------------------------------------------ | 20260508 13:55:14.074 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:55:14.074 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:55:14.074 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:55:14.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:55:14.074 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.52e+01, 1.39e+01 / 5.57e-01 | 20260508 13:55:14.074 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.05e-01, 1.13e-02 / 4.32e-03 | 20260508 13:55:14.074 | INFO | PerfReport: 12.22%, Baseline -- 3.99e+00 / 4.44e+00, 5.91e-01 / 1.58e-01 | 20260508 13:55:14.074 | INFO | PerfReport: 3.40%, Histogram -- 1.11e+00 / 1.43e+00, 2.09e-01 / 1.13e-01 | 20260508 13:55:14.074 | INFO | PerfReport: 15.96%, DME -- 4.17e+01 / 4.50e+01, 3.65e+00 / 1.05e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.87e+01 / 2.04e+01, 1.88e+00 / 1.36e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.30e+00 / 2.38e+00, 3.99e-01 / 2.57e-01 | 20260508 13:55:14.074 | INFO | PerfReport: 2.91%, Metrics -- 9.51e-01 / 1.10e+00, 2.37e-01 / 1.28e-01 | 20260508 13:55:14.074 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.89e-01 / 3.20e-01, 4.60e-02 / 2.85e-02 | 20260508 13:55:14.074 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:55:14.074 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:55:14.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:55:14.074 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.36e+01 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 24.35%, Upload -- 2.55e+01 / 0.00e+00, 7.93e+00 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 24.83%, Download -- 2.60e+01 / 0.00e+00, 8.14e+00 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 31.21%, Compute -- 3.27e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 2.10%, LockUpload -- 2.19e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 2.55%, LockDownload -- 2.67e+00 / 0.00e+00, 4.00e+00 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 14.94%, LockCompute -- 1.56e+01 / 0.00e+00, 1.76e+01 / 0.00e+00 | 20260508 13:55:14.074 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260508 13:55:14.184 | INFO | Starting a round of Baz IO | 20260508 13:55:17.599 | INFO | Populating RTMetrics report | 20260508 13:55:17.599 | INFO | RTMetrics report populated | 20260508 13:55:17.599 | INFO | Opening tmp RTMetrics json file | 20260508 13:55:17.607 | INFO | Serializing RTMetrics to json | 20260508 13:55:17.608 | INFO | Serialization done, swapping tmp file now... | 20260508 13:55:17.608 | INFO | RTMetrics done | 20260508 13:55:17.664 | INFO | Finished a round of Baz IO: Wrote 1.377 GiB in 3.479 seconds | 20260508 13:55:17.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:55:17.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132517% / 0% / 6.75519e-05% | 20260508 13:55:17.664 | INFO | Baz IO overall: | 20260508 13:55:17.664 | INFO | BytesWritten : 3665345184112 | 20260508 13:55:17.664 | INFO | NumEvents : 1756685777442 | 20260508 13:55:19.212 | INFO | Analyzing chunk frames = [8626176,8626688) | 20260508 13:55:19.212 | INFO | MemInfo: { | 20260508 13:55:19.212 | INFO | "gpu" : " 17 GB", | 20260508 13:55:19.212 | INFO | "huge" : "", | 20260508 13:55:19.212 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:19.212 | INFO | "malloc" : " 14 GB", | 20260508 13:55:19.212 | INFO | "pinned" : " 29 GB" | 20260508 13:55:19.212 | INFO | } | 20260508 13:55:19.212 | INFO | Perf report for the last 5.14047 seconds: | 20260508 13:55:19.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:19.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:19.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:19.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:19.212 | INFO | ANALYSIS : 68.0%, 3.0, 106.0ms, 0 | 20260508 13:55:19.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:19.212 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260508 13:55:19.212 | INFO | PRE_HQ : 66.4%, 1.8, 61.0ms, 0 | 20260508 13:55:19.212 | INFO | BAZWRITER : 67.0%, 2.6, 89.5ms, 0 | 20260508 13:55:19.212 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:55:19.213 | INFO | Uploaded 36GiB to GPU in 2.61332s (13.7756GiB/s) | 20260508 13:55:19.213 | INFO | Downloaded 38GiB to GPU in 2.71702s (13.9859GiB/s) | 20260508 13:55:24.348 | INFO | Analyzing chunk frames = [8626688,8627200) | 20260508 13:55:24.348 | INFO | MemInfo: { | 20260508 13:55:24.348 | INFO | "gpu" : " 17 GB", | 20260508 13:55:24.348 | INFO | "huge" : "", | 20260508 13:55:24.348 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:24.348 | INFO | "malloc" : " 14 GB", | 20260508 13:55:24.348 | INFO | "pinned" : " 29 GB" | 20260508 13:55:24.348 | INFO | } | 20260508 13:55:24.348 | INFO | Perf report for the last 5.13617 seconds: | 20260508 13:55:24.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:24.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.348 | INFO | ANALYSIS : 66.2%, 3.0, 103.1ms, 0 | 20260508 13:55:24.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.349 | INFO | PRE_HQ : 22.5%, 1.0, 11.6ms, 0 | 20260508 13:55:24.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:24.349 | INFO | Uploaded 36GiB to GPU in 2.51523s (14.3128GiB/s) | 20260508 13:55:24.349 | INFO | Downloaded 34GiB to GPU in 2.48416s (13.6867GiB/s) | 20260508 13:55:29.483 | INFO | Analyzing chunk frames = [8627200,8627712) | 20260508 13:55:29.483 | INFO | MemInfo: { | 20260508 13:55:29.483 | INFO | "gpu" : " 17 GB", | 20260508 13:55:29.483 | INFO | "huge" : "", | 20260508 13:55:29.483 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:29.483 | INFO | "malloc" : " 14 GB", | 20260508 13:55:29.483 | INFO | "pinned" : " 29 GB" | 20260508 13:55:29.483 | INFO | } | 20260508 13:55:29.483 | INFO | Perf report for the last 5.13472 seconds: | 20260508 13:55:29.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:29.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.483 | INFO | ANALYSIS : 65.8%, 3.0, 102.2ms, 0 | 20260508 13:55:29.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.483 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:55:29.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:29.484 | INFO | Uploaded 36GiB to GPU in 2.47438s (14.5491GiB/s) | 20260508 13:55:29.484 | INFO | Downloaded 34GiB to GPU in 2.4449s (13.9065GiB/s) | 20260508 13:55:34.621 | INFO | Analyzing chunk frames = [8627712,8628224) | 20260508 13:55:34.621 | INFO | MemInfo: { | 20260508 13:55:34.621 | INFO | "gpu" : " 17 GB", | 20260508 13:55:34.621 | INFO | "huge" : "", | 20260508 13:55:34.622 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:34.622 | INFO | "malloc" : " 14 GB", | 20260508 13:55:34.622 | INFO | "pinned" : " 29 GB" | 20260508 13:55:34.622 | INFO | } | 20260508 13:55:34.622 | INFO | Perf report for the last 5.13847 seconds: | 20260508 13:55:34.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:34.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | ANALYSIS : 66.1%, 3.0, 102.5ms, 0 | 20260508 13:55:34.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:55:34.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:34.622 | INFO | Uploaded 36GiB to GPU in 2.52625s (14.2504GiB/s) | 20260508 13:55:34.622 | INFO | Downloaded 34GiB to GPU in 2.46263s (13.8064GiB/s) | 20260508 13:55:39.766 | INFO | Analyzing chunk frames = [8628224,8628736) | 20260508 13:55:39.766 | INFO | MemInfo: { | 20260508 13:55:39.766 | INFO | "gpu" : " 17 GB", | 20260508 13:55:39.766 | INFO | "huge" : "", | 20260508 13:55:39.766 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:39.766 | INFO | "malloc" : " 14 GB", | 20260508 13:55:39.766 | INFO | "pinned" : " 29 GB" | 20260508 13:55:39.766 | INFO | } | 20260508 13:55:39.766 | INFO | Perf report for the last 5.14458 seconds: | 20260508 13:55:39.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:39.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.766 | INFO | ANALYSIS : 65.9%, 3.0, 102.9ms, 0 | 20260508 13:55:39.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.766 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:55:39.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:39.767 | INFO | Uploaded 36GiB to GPU in 2.3285s (15.4606GiB/s) | 20260508 13:55:39.767 | INFO | Downloaded 34GiB to GPU in 2.48184s (13.6995GiB/s) | 20260508 13:55:44.908 | INFO | Analyzing chunk frames = [8628736,8629248) | 20260508 13:55:44.908 | INFO | MemInfo: { | 20260508 13:55:44.908 | INFO | "gpu" : " 17 GB", | 20260508 13:55:44.908 | INFO | "huge" : "", | 20260508 13:55:44.908 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:44.908 | INFO | "malloc" : " 14 GB", | 20260508 13:55:44.908 | INFO | "pinned" : " 29 GB" | 20260508 13:55:44.908 | INFO | } | 20260508 13:55:44.908 | INFO | Perf report for the last 5.14153 seconds: | 20260508 13:55:44.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:44.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | ANALYSIS : 65.3%, 3.0, 101.6ms, 0 | 20260508 13:55:44.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:55:44.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:44.908 | INFO | Uploaded 36GiB to GPU in 2.43112s (14.808GiB/s) | 20260508 13:55:44.908 | INFO | Downloaded 34GiB to GPU in 2.43291s (13.975GiB/s) | 20260508 13:55:50.046 | INFO | Analyzing chunk frames = [8629248,8629760) | 20260508 13:55:50.046 | INFO | MemInfo: { | 20260508 13:55:50.046 | INFO | "gpu" : " 17 GB", | 20260508 13:55:50.046 | INFO | "huge" : "", | 20260508 13:55:50.046 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:50.046 | INFO | "malloc" : " 14 GB", | 20260508 13:55:50.046 | INFO | "pinned" : " 29 GB" | 20260508 13:55:50.046 | INFO | } | 20260508 13:55:50.046 | INFO | Perf report for the last 5.13818 seconds: | 20260508 13:55:50.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:50.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | ANALYSIS : 64.4%, 3.0, 100.2ms, 0 | 20260508 13:55:50.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260508 13:55:50.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:50.046 | INFO | Uploaded 36GiB to GPU in 2.29274s (15.7017GiB/s) | 20260508 13:55:50.046 | INFO | Downloaded 34GiB to GPU in 2.40209s (14.1544GiB/s) | 20260508 13:55:55.182 | INFO | Analyzing chunk frames = [8629760,8630272) | 20260508 13:55:55.182 | INFO | MemInfo: { | 20260508 13:55:55.182 | INFO | "gpu" : " 17 GB", | 20260508 13:55:55.182 | INFO | "huge" : "", | 20260508 13:55:55.182 | INFO | "huge_shared" : " 29 GB", | 20260508 13:55:55.182 | INFO | "malloc" : " 14 GB", | 20260508 13:55:55.182 | INFO | "pinned" : " 29 GB" | 20260508 13:55:55.182 | INFO | } | 20260508 13:55:55.182 | INFO | Perf report for the last 5.13567 seconds: | 20260508 13:55:55.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:55:55.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | ANALYSIS : 64.7%, 3.0, 101.1ms, 0 | 20260508 13:55:55.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:55:55.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:55:55.182 | INFO | Uploaded 36GiB to GPU in 2.3607s (15.2497GiB/s) | 20260508 13:55:55.182 | INFO | Downloaded 34GiB to GPU in 2.41844s (14.0586GiB/s) | 20260508 13:55:55.279 | INFO | Starting a round of Baz IO | 20260508 13:55:58.677 | INFO | Populating RTMetrics report | 20260508 13:55:58.677 | INFO | RTMetrics report populated | 20260508 13:55:58.677 | INFO | Opening tmp RTMetrics json file | 20260508 13:55:58.688 | INFO | Serializing RTMetrics to json | 20260508 13:55:58.689 | INFO | Serialization done, swapping tmp file now... | 20260508 13:55:58.690 | INFO | RTMetrics done | 20260508 13:55:58.749 | INFO | Finished a round of Baz IO: Wrote 1.374 GiB in 3.47 seconds | 20260508 13:55:58.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:55:58.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131877% / 0% / 6.35783e-05% | 20260508 13:55:58.749 | INFO | Baz IO overall: | 20260508 13:55:58.749 | INFO | BytesWritten : 3666820843846 | 20260508 13:55:58.749 | INFO | NumEvents : 1757375047984 | 20260508 13:56:00.322 | INFO | Analyzing chunk frames = [8630272,8630784) | 20260508 13:56:00.322 | INFO | MemInfo: { | 20260508 13:56:00.322 | INFO | "gpu" : " 17 GB", | 20260508 13:56:00.322 | INFO | "huge" : "", | 20260508 13:56:00.322 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:00.322 | INFO | "malloc" : " 14 GB", | 20260508 13:56:00.322 | INFO | "pinned" : " 29 GB" | 20260508 13:56:00.322 | INFO | } | 20260508 13:56:00.322 | INFO | Perf report for the last 5.14073 seconds: | 20260508 13:56:00.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:00.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:00.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:00.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:00.323 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:56:00.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:00.323 | INFO | RT_METRICS : 37.9%, 1.0, 19.9ms, 0 | 20260508 13:56:00.323 | INFO | PRE_HQ : 65.2%, 1.6, 53.4ms, 0 | 20260508 13:56:00.323 | INFO | BAZWRITER : 66.8%, 2.4, 84.1ms, 0 | 20260508 13:56:00.323 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:56:00.323 | INFO | Uploaded 36GiB to GPU in 2.65227s (13.5733GiB/s) | 20260508 13:56:00.323 | INFO | Downloaded 38GiB to GPU in 2.71119s (14.016GiB/s) | 20260508 13:56:05.461 | INFO | Analyzing chunk frames = [8630784,8631296) | 20260508 13:56:05.461 | INFO | MemInfo: { | 20260508 13:56:05.461 | INFO | "gpu" : " 17 GB", | 20260508 13:56:05.461 | INFO | "huge" : "", | 20260508 13:56:05.461 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:05.461 | INFO | "malloc" : " 14 GB", | 20260508 13:56:05.461 | INFO | "pinned" : " 29 GB" | 20260508 13:56:05.461 | INFO | } | 20260508 13:56:05.461 | INFO | Perf report for the last 5.13837 seconds: | 20260508 13:56:05.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:05.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | ANALYSIS : 65.9%, 3.0, 102.7ms, 0 | 20260508 13:56:05.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | PRE_HQ : 22.1%, 1.0, 11.4ms, 0 | 20260508 13:56:05.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:05.461 | INFO | Uploaded 36GiB to GPU in 2.48229s (14.5027GiB/s) | 20260508 13:56:05.461 | INFO | Downloaded 34GiB to GPU in 2.46016s (13.8202GiB/s) | 20260508 13:56:10.602 | INFO | Analyzing chunk frames = [8631296,8631808) | 20260508 13:56:10.602 | INFO | MemInfo: { | 20260508 13:56:10.602 | INFO | "gpu" : " 17 GB", | 20260508 13:56:10.602 | INFO | "huge" : "", | 20260508 13:56:10.602 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:10.602 | INFO | "malloc" : " 14 GB", | 20260508 13:56:10.602 | INFO | "pinned" : " 29 GB" | 20260508 13:56:10.602 | INFO | } | 20260508 13:56:10.602 | INFO | Perf report for the last 5.14121 seconds: | 20260508 13:56:10.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:10.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.602 | INFO | ANALYSIS : 66.2%, 3.0, 102.9ms, 0 | 20260508 13:56:10.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.602 | INFO | PRE_HQ : 22.4%, 1.0, 11.6ms, 0 | 20260508 13:56:10.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:10.603 | INFO | Uploaded 36GiB to GPU in 2.50147s (14.3916GiB/s) | 20260508 13:56:10.603 | INFO | Downloaded 34GiB to GPU in 2.45191s (13.8667GiB/s) | 20260508 13:56:15.734 | INFO | Analyzing chunk frames = [8631808,8632320) | 20260508 13:56:15.734 | INFO | MemInfo: { | 20260508 13:56:15.734 | INFO | "gpu" : " 17 GB", | 20260508 13:56:15.734 | INFO | "huge" : "", | 20260508 13:56:15.734 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:15.734 | INFO | "malloc" : " 14 GB", | 20260508 13:56:15.734 | INFO | "pinned" : " 29 GB" | 20260508 13:56:15.734 | INFO | } | 20260508 13:56:15.735 | INFO | Perf report for the last 5.13252 seconds: | 20260508 13:56:15.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:15.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | ANALYSIS : 65.6%, 3.0, 101.6ms, 0 | 20260508 13:56:15.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:56:15.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:15.735 | INFO | Uploaded 36GiB to GPU in 2.44184s (14.743GiB/s) | 20260508 13:56:15.735 | INFO | Downloaded 34GiB to GPU in 2.46451s (13.7958GiB/s) | 20260508 13:56:20.883 | INFO | Analyzing chunk frames = [8632320,8632832) | 20260508 13:56:20.883 | INFO | MemInfo: { | 20260508 13:56:20.883 | INFO | "gpu" : " 17 GB", | 20260508 13:56:20.883 | INFO | "huge" : "", | 20260508 13:56:20.884 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:20.884 | INFO | "malloc" : " 14 GB", | 20260508 13:56:20.884 | INFO | "pinned" : " 29 GB" | 20260508 13:56:20.884 | INFO | } | 20260508 13:56:20.884 | INFO | Perf report for the last 5.14915 seconds: | 20260508 13:56:20.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:20.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | ANALYSIS : 66.2%, 3.0, 103.4ms, 0 | 20260508 13:56:20.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | PRE_HQ : 23.6%, 1.0, 12.3ms, 0 | 20260508 13:56:20.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:20.884 | INFO | Uploaded 36GiB to GPU in 2.43658s (14.7748GiB/s) | 20260508 13:56:20.884 | INFO | Downloaded 34GiB to GPU in 2.5078s (13.5577GiB/s) | 20260508 13:56:26.025 | INFO | Analyzing chunk frames = [8632832,8633344) | 20260508 13:56:26.025 | INFO | MemInfo: { | 20260508 13:56:26.025 | INFO | "gpu" : " 17 GB", | 20260508 13:56:26.025 | INFO | "huge" : "", | 20260508 13:56:26.025 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:26.025 | INFO | "malloc" : " 14 GB", | 20260508 13:56:26.025 | INFO | "pinned" : " 29 GB" | 20260508 13:56:26.025 | INFO | } | 20260508 13:56:26.025 | INFO | Perf report for the last 5.14127 seconds: | 20260508 13:56:26.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:26.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.025 | INFO | ANALYSIS : 64.7%, 3.0, 100.7ms, 0 | 20260508 13:56:26.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.025 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260508 13:56:26.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:26.026 | INFO | Uploaded 36GiB to GPU in 2.34904s (15.3254GiB/s) | 20260508 13:56:26.026 | INFO | Downloaded 34GiB to GPU in 2.41163s (14.0983GiB/s) | 20260508 13:56:31.158 | INFO | Analyzing chunk frames = [8633344,8633856) | 20260508 13:56:31.158 | INFO | MemInfo: { | 20260508 13:56:31.158 | INFO | "gpu" : " 17 GB", | 20260508 13:56:31.158 | INFO | "huge" : "", | 20260508 13:56:31.158 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:31.159 | INFO | "malloc" : " 14 GB", | 20260508 13:56:31.159 | INFO | "pinned" : " 29 GB" | 20260508 13:56:31.159 | INFO | } | 20260508 13:56:31.159 | INFO | Perf report for the last 5.13368 seconds: | 20260508 13:56:31.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:31.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | ANALYSIS : 64.5%, 3.0, 100.3ms, 0 | 20260508 13:56:31.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260508 13:56:31.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:31.159 | INFO | Uploaded 36GiB to GPU in 2.28644s (15.745GiB/s) | 20260508 13:56:31.159 | INFO | Downloaded 34GiB to GPU in 2.38459s (14.2582GiB/s) | 20260508 13:56:36.294 | INFO | Analyzing chunk frames = [8633856,8634368) | 20260508 13:56:36.294 | INFO | MemInfo: { | 20260508 13:56:36.294 | INFO | "gpu" : " 17 GB", | 20260508 13:56:36.294 | INFO | "huge" : "", | 20260508 13:56:36.294 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:36.294 | INFO | "malloc" : " 14 GB", | 20260508 13:56:36.295 | INFO | "pinned" : " 29 GB" | 20260508 13:56:36.295 | INFO | } | 20260508 13:56:36.295 | INFO | Perf report for the last 5.13586 seconds: | 20260508 13:56:36.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:36.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | ANALYSIS : 64.7%, 3.0, 101.0ms, 0 | 20260508 13:56:36.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260508 13:56:36.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:36.295 | INFO | Uploaded 36GiB to GPU in 2.34755s (15.3352GiB/s) | 20260508 13:56:36.295 | INFO | Downloaded 34GiB to GPU in 2.41523s (14.0773GiB/s) | 20260508 13:56:36.295 | INFO | ----Memory usage report for Standard Host Memory----- | 20260508 13:56:36.295 | INFO | Total mem usage: 16.7 GiB | 20260508 13:56:36.295 | INFO | All Time High Water Marks sum: 17 GiB (Savings: 2.11%) | 20260508 13:56:36.295 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -16.3%) | 20260508 13:56:36.295 | INFO | Unused Cache: 0B (Waste: 0%) | 20260508 13:56:36.295 | INFO | High Water Breakdown: | 20260508 13:56:36.295 | INFO | 59.6% : 8.55 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260508 13:56:36.295 | INFO | 40.3% : 5.77 GiB ( 85.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260508 13:56:36.295 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260508 13:56:36.295 | INFO | ------------------------------------------ | 20260508 13:56:36.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:56:36.295 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:56:36.296 | INFO | Total mem usage: 26.5 GiB | 20260508 13:56:36.296 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:56:36.296 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260508 13:56:36.296 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260508 13:56:36.296 | INFO | High Water Breakdown: | 20260508 13:56:36.296 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:56:36.296 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:56:36.296 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:56:36.296 | INFO | 2% : 523 MiB ( 83.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:56:36.296 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:56:36.296 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:56:36.296 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:56:36.296 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:56:36.296 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:56:36.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:56:36.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:56:36.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:56:36.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:56:36.296 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:56:36.296 | INFO | ------------------------------------------ | 20260508 13:56:36.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:56:36.296 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:56:36.296 | INFO | Total mem usage: 27.8 GiB | 20260508 13:56:36.296 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:56:36.296 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:56:36.296 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:56:36.296 | INFO | High Water Breakdown: | 20260508 13:56:36.296 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:56:36.296 | INFO | ------------------------------------------ | 20260508 13:56:36.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:56:36.296 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:56:36.296 | INFO | Total mem usage: 15.7 GiB | 20260508 13:56:36.296 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:56:36.296 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:56:36.296 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:56:36.296 | INFO | High Water Breakdown: | 20260508 13:56:36.296 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:56:36.296 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:56:36.296 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:56:36.296 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:56:36.296 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:56:36.296 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:56:36.296 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:56:36.296 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:56:36.296 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:56:36.296 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:56:36.296 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:56:36.296 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:56:36.296 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:56:36.296 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:56:36.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:56:36.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:56:36.296 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:56:36.296 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:56:36.296 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:56:36.296 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:56:36.296 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:56:36.296 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:56:36.296 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:56:36.297 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:56:36.297 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:56:36.297 | INFO | ------------------------------------------ | 20260508 13:56:36.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:56:36.297 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:56:36.297 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:56:36.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:56:36.297 | INFO | PerfReport: Full Execution -- 3.28e+01 / 7.55e+01, 1.38e+01 / 5.87e-01 | 20260508 13:56:36.297 | INFO | PerfReport: 0.30%, AggregateStats -- 9.84e-02 / 1.04e-01, 1.15e-02 / 5.42e-03 | 20260508 13:56:36.297 | INFO | PerfReport: 12.24%, Baseline -- 4.02e+00 / 4.33e+00, 6.02e-01 / 2.68e-01 | 20260508 13:56:36.297 | INFO | PerfReport: 3.43%, Histogram -- 1.12e+00 / 1.41e+00, 2.11e-01 / 1.34e-01 | 20260508 13:56:36.297 | INFO | PerfReport: 15.98%, DME -- 4.19e+01 / 4.58e+01, 3.69e+00 / 9.59e-01 | 20260508 13:56:36.297 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.88e+01 / 2.00e+01, 1.93e+00 / 1.19e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.30e+00 / 2.45e+00, 4.01e-01 / 2.63e-01 | 20260508 13:56:36.297 | INFO | PerfReport: 2.90%, Metrics -- 9.50e-01 / 1.08e+00, 2.39e-01 / 1.07e-01 | 20260508 13:56:36.297 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.89e-01 / 3.15e-01, 4.70e-02 / 2.46e-02 | 20260508 13:56:36.297 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:56:36.297 | INFO | PerfReport: Gathered 1536 samples. | 20260508 13:56:36.297 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:56:36.297 | INFO | PerfReport: Full Execution -- 1.04e+02 / -nan, 2.32e+01 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 24.33%, Upload -- 2.54e+01 / 0.00e+00, 7.82e+00 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 24.71%, Download -- 2.58e+01 / 0.00e+00, 8.17e+00 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 31.41%, Compute -- 3.28e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 2.26%, LockUpload -- 2.36e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 2.43%, LockDownload -- 2.54e+00 / 0.00e+00, 3.90e+00 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 14.84%, LockCompute -- 1.55e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260508 13:56:36.297 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260508 13:56:36.406 | INFO | Starting a round of Baz IO | 20260508 13:56:39.793 | INFO | Populating RTMetrics report | 20260508 13:56:39.793 | INFO | RTMetrics report populated | 20260508 13:56:39.793 | INFO | Opening tmp RTMetrics json file | 20260508 13:56:39.804 | INFO | Serializing RTMetrics to json | 20260508 13:56:39.805 | INFO | Serialization done, swapping tmp file now... | 20260508 13:56:39.806 | INFO | RTMetrics done | 20260508 13:56:39.868 | INFO | Finished a round of Baz IO: Wrote 1.374 GiB in 3.461 seconds | 20260508 13:56:39.868 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:56:39.868 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131897% / 3.97364e-06% / 7.94729e-05% | 20260508 13:56:39.868 | INFO | Baz IO overall: | 20260508 13:56:39.868 | INFO | BytesWritten : 3668296338585 | 20260508 13:56:39.868 | INFO | NumEvents : 1758064284519 | 20260508 13:56:41.436 | INFO | Analyzing chunk frames = [8634368,8634880) | 20260508 13:56:41.436 | INFO | MemInfo: { | 20260508 13:56:41.436 | INFO | "gpu" : " 17 GB", | 20260508 13:56:41.436 | INFO | "huge" : "", | 20260508 13:56:41.436 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:41.436 | INFO | "malloc" : " 14 GB", | 20260508 13:56:41.436 | INFO | "pinned" : " 29 GB" | 20260508 13:56:41.436 | INFO | } | 20260508 13:56:41.436 | INFO | Perf report for the last 5.14158 seconds: | 20260508 13:56:41.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:41.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:41.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:41.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:41.436 | INFO | ANALYSIS : 67.4%, 3.0, 105.3ms, 0 | 20260508 13:56:41.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:41.437 | INFO | RT_METRICS : 41.8%, 1.0, 22.0ms, 0 | 20260508 13:56:41.437 | INFO | PRE_HQ : 66.0%, 1.7, 59.3ms, 0 | 20260508 13:56:41.437 | INFO | BAZWRITER : 66.6%, 2.6, 89.3ms, 0 | 20260508 13:56:41.437 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260508 13:56:41.437 | INFO | Uploaded 36GiB to GPU in 2.65607s (13.5538GiB/s) | 20260508 13:56:41.437 | INFO | Downloaded 38GiB to GPU in 2.71579s (13.9923GiB/s) | 20260508 13:56:46.574 | INFO | Analyzing chunk frames = [8634880,8635392) | 20260508 13:56:46.574 | INFO | MemInfo: { | 20260508 13:56:46.574 | INFO | "gpu" : " 17 GB", | 20260508 13:56:46.574 | INFO | "huge" : "", | 20260508 13:56:46.574 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:46.574 | INFO | "malloc" : " 14 GB", | 20260508 13:56:46.574 | INFO | "pinned" : " 29 GB" | 20260508 13:56:46.574 | INFO | } | 20260508 13:56:46.574 | INFO | Perf report for the last 5.13781 seconds: | 20260508 13:56:46.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:46.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | ANALYSIS : 66.3%, 3.0, 103.3ms, 0 | 20260508 13:56:46.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:56:46.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:46.574 | INFO | Uploaded 36GiB to GPU in 2.51375s (14.3213GiB/s) | 20260508 13:56:46.574 | INFO | Downloaded 34GiB to GPU in 2.4748s (13.7385GiB/s) | 20260508 13:56:51.716 | INFO | Analyzing chunk frames = [8635392,8635904) | 20260508 13:56:51.716 | INFO | MemInfo: { | 20260508 13:56:51.716 | INFO | "gpu" : " 17 GB", | 20260508 13:56:51.716 | INFO | "huge" : "", | 20260508 13:56:51.716 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:51.716 | INFO | "malloc" : " 14 GB", | 20260508 13:56:51.716 | INFO | "pinned" : " 29 GB" | 20260508 13:56:51.716 | INFO | } | 20260508 13:56:51.716 | INFO | Perf report for the last 5.14167 seconds: | 20260508 13:56:51.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:51.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.716 | INFO | ANALYSIS : 66.2%, 3.0, 103.0ms, 0 | 20260508 13:56:51.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.716 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:56:51.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:51.717 | INFO | Uploaded 36GiB to GPU in 2.5225s (14.2716GiB/s) | 20260508 13:56:51.717 | INFO | Downloaded 34GiB to GPU in 2.45505s (13.849GiB/s) | 20260508 13:56:56.856 | INFO | Analyzing chunk frames = [8635904,8636416) | 20260508 13:56:56.856 | INFO | MemInfo: { | 20260508 13:56:56.856 | INFO | "gpu" : " 17 GB", | 20260508 13:56:56.856 | INFO | "huge" : "", | 20260508 13:56:56.856 | INFO | "huge_shared" : " 29 GB", | 20260508 13:56:56.856 | INFO | "malloc" : " 14 GB", | 20260508 13:56:56.856 | INFO | "pinned" : " 29 GB" | 20260508 13:56:56.856 | INFO | } | 20260508 13:56:56.857 | INFO | Perf report for the last 5.14077 seconds: | 20260508 13:56:56.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:56:56.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | ANALYSIS : 65.5%, 3.0, 101.6ms, 0 | 20260508 13:56:56.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | PRE_HQ : 22.6%, 1.0, 11.7ms, 0 | 20260508 13:56:56.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:56:56.857 | INFO | Uploaded 36GiB to GPU in 2.42866s (14.823GiB/s) | 20260508 13:56:56.857 | INFO | Downloaded 34GiB to GPU in 2.46944s (13.7683GiB/s) | 20260508 13:57:01.990 | INFO | Analyzing chunk frames = [8636416,8636928) | 20260508 13:57:01.990 | INFO | MemInfo: { | 20260508 13:57:01.990 | INFO | "gpu" : " 17 GB", | 20260508 13:57:01.990 | INFO | "huge" : "", | 20260508 13:57:01.990 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:01.990 | INFO | "malloc" : " 14 GB", | 20260508 13:57:01.990 | INFO | "pinned" : " 29 GB" | 20260508 13:57:01.990 | INFO | } | 20260508 13:57:01.990 | INFO | Perf report for the last 5.13334 seconds: | 20260508 13:57:01.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:01.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | ANALYSIS : 66.3%, 3.0, 103.4ms, 0 | 20260508 13:57:01.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | PRE_HQ : 23.8%, 1.0, 12.3ms, 0 | 20260508 13:57:01.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:01.990 | INFO | Uploaded 36GiB to GPU in 2.43314s (14.7957GiB/s) | 20260508 13:57:01.991 | INFO | Downloaded 34GiB to GPU in 2.51252s (13.5322GiB/s) | 20260508 13:57:07.131 | INFO | Analyzing chunk frames = [8636928,8637440) | 20260508 13:57:07.131 | INFO | MemInfo: { | 20260508 13:57:07.131 | INFO | "gpu" : " 17 GB", | 20260508 13:57:07.131 | INFO | "huge" : "", | 20260508 13:57:07.131 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:07.132 | INFO | "malloc" : " 14 GB", | 20260508 13:57:07.132 | INFO | "pinned" : " 29 GB" | 20260508 13:57:07.132 | INFO | } | 20260508 13:57:07.132 | INFO | Perf report for the last 5.14187 seconds: | 20260508 13:57:07.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:07.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | ANALYSIS : 65.4%, 3.0, 101.8ms, 0 | 20260508 13:57:07.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | PRE_HQ : 22.7%, 1.0, 11.8ms, 0 | 20260508 13:57:07.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:07.132 | INFO | Uploaded 36GiB to GPU in 2.42196s (14.864GiB/s) | 20260508 13:57:07.132 | INFO | Downloaded 34GiB to GPU in 2.43982s (13.9354GiB/s) | 20260508 13:57:12.274 | INFO | Analyzing chunk frames = [8637440,8637952) | 20260508 13:57:12.274 | INFO | MemInfo: { | 20260508 13:57:12.274 | INFO | "gpu" : " 17 GB", | 20260508 13:57:12.274 | INFO | "huge" : "", | 20260508 13:57:12.275 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:12.275 | INFO | "malloc" : " 14 GB", | 20260508 13:57:12.275 | INFO | "pinned" : " 29 GB" | 20260508 13:57:12.275 | INFO | } | 20260508 13:57:12.275 | INFO | Perf report for the last 5.14293 seconds: | 20260508 13:57:12.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:12.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | ANALYSIS : 65.6%, 3.0, 102.4ms, 0 | 20260508 13:57:12.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260508 13:57:12.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:12.275 | INFO | Uploaded 36GiB to GPU in 2.49975s (14.4014GiB/s) | 20260508 13:57:12.275 | INFO | Downloaded 34GiB to GPU in 2.45963s (13.8232GiB/s) | 20260508 13:57:17.408 | INFO | Analyzing chunk frames = [8637952,8638464) | 20260508 13:57:17.408 | INFO | MemInfo: { | 20260508 13:57:17.408 | INFO | "gpu" : " 17 GB", | 20260508 13:57:17.408 | INFO | "huge" : "", | 20260508 13:57:17.408 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:17.408 | INFO | "malloc" : " 14 GB", | 20260508 13:57:17.408 | INFO | "pinned" : " 29 GB" | 20260508 13:57:17.408 | INFO | } | 20260508 13:57:17.408 | INFO | Perf report for the last 5.13328 seconds: | 20260508 13:57:17.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:17.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | ANALYSIS : 65.3%, 3.0, 101.5ms, 0 | 20260508 13:57:17.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | PRE_HQ : 23.0%, 1.0, 11.9ms, 0 | 20260508 13:57:17.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:17.408 | INFO | Uploaded 36GiB to GPU in 2.36805s (15.2024GiB/s) | 20260508 13:57:17.408 | INFO | Downloaded 34GiB to GPU in 2.41347s (14.0876GiB/s) | 20260508 13:57:17.505 | INFO | Starting a round of Baz IO | 20260508 13:57:20.915 | INFO | Populating RTMetrics report | 20260508 13:57:20.915 | INFO | RTMetrics report populated | 20260508 13:57:20.915 | INFO | Opening tmp RTMetrics json file | 20260508 13:57:20.916 | INFO | Serializing RTMetrics to json | 20260508 13:57:20.917 | INFO | Serialization done, swapping tmp file now... | 20260508 13:57:20.917 | INFO | RTMetrics done | 20260508 13:57:20.978 | INFO | Finished a round of Baz IO: Wrote 1.375 GiB in 3.472 seconds | 20260508 13:57:20.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:57:20.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131726% / 0% / 6.35783e-05% | 20260508 13:57:20.978 | INFO | Baz IO overall: | 20260508 13:57:20.978 | INFO | BytesWritten : 3669772685264 | 20260508 13:57:20.978 | INFO | NumEvents : 1758754317502 | 20260508 13:57:22.549 | INFO | Analyzing chunk frames = [8638464,8638976) | 20260508 13:57:22.549 | INFO | MemInfo: { | 20260508 13:57:22.549 | INFO | "gpu" : " 17 GB", | 20260508 13:57:22.549 | INFO | "huge" : "", | 20260508 13:57:22.549 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:22.549 | INFO | "malloc" : " 14 GB", | 20260508 13:57:22.549 | INFO | "pinned" : " 29 GB" | 20260508 13:57:22.549 | INFO | } | 20260508 13:57:22.549 | INFO | Perf report for the last 5.14144 seconds: | 20260508 13:57:22.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:22.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:22.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:22.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:22.550 | INFO | ANALYSIS : 67.9%, 3.0, 105.6ms, 0 | 20260508 13:57:22.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:22.550 | INFO | RT_METRICS : 37.7%, 1.0, 19.8ms, 0 | 20260508 13:57:22.550 | INFO | PRE_HQ : 65.3%, 1.6, 53.4ms, 0 | 20260508 13:57:22.550 | INFO | BAZWRITER : 66.7%, 2.4, 82.9ms, 0 | 20260508 13:57:22.550 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260508 13:57:22.550 | INFO | Uploaded 36GiB to GPU in 2.58075s (13.9494GiB/s) | 20260508 13:57:22.550 | INFO | Downloaded 38GiB to GPU in 2.67058s (14.2291GiB/s) | 20260508 13:57:27.690 | INFO | Analyzing chunk frames = [8638976,8639488) | 20260508 13:57:27.690 | INFO | MemInfo: { | 20260508 13:57:27.690 | INFO | "gpu" : " 17 GB", | 20260508 13:57:27.690 | INFO | "huge" : "", | 20260508 13:57:27.690 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:27.690 | INFO | "malloc" : " 14 GB", | 20260508 13:57:27.690 | INFO | "pinned" : " 29 GB" | 20260508 13:57:27.690 | INFO | } | 20260508 13:57:27.690 | INFO | Perf report for the last 5.14101 seconds: | 20260508 13:57:27.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:27.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.690 | INFO | ANALYSIS : 66.0%, 3.0, 102.9ms, 0 | 20260508 13:57:27.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.691 | INFO | PRE_HQ : 22.2%, 1.0, 11.5ms, 0 | 20260508 13:57:27.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:27.691 | INFO | Uploaded 36GiB to GPU in 2.48357s (14.4952GiB/s) | 20260508 13:57:27.691 | INFO | Downloaded 34GiB to GPU in 2.46024s (13.8198GiB/s) | 20260508 13:57:32.830 | INFO | Analyzing chunk frames = [8639488,8640000) | 20260508 13:57:32.830 | INFO | MemInfo: { | 20260508 13:57:32.830 | INFO | Reached end of movie, endFrame:8640000 expectedEndFrame:8640000 | 20260508 13:57:32.830 | INFO | DataSourceRunner thread exiting | 20260508 13:57:32.830 | INFO | "gpu" : " 17 GB", | 20260508 13:57:32.830 | INFO | "huge" : "", | 20260508 13:57:32.830 | INFO | "huge_shared" : " 29 GB", | 20260508 13:57:32.830 | INFO | "malloc" : " 14 GB", | 20260508 13:57:32.830 | INFO | "pinned" : " 29 GB" | 20260508 13:57:32.830 | INFO | } | 20260508 13:57:32.830 | INFO | Perf report for the last 5.13948 seconds: | 20260508 13:57:32.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260508 13:57:32.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | ANALYSIS : 65.8%, 3.0, 102.4ms, 0 | 20260508 13:57:32.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | PRE_HQ : 22.5%, 1.0, 11.7ms, 0 | 20260508 13:57:32.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260508 13:57:32.830 | INFO | Uploaded 36GiB to GPU in 2.48755s (14.4721GiB/s) | 20260508 13:57:32.830 | INFO | Downloaded 34GiB to GPU in 2.45703s (13.8379GiB/s) | 20260508 13:57:32.835 | INFO | Flushing graph node REPACKER with 0 jobs. | 20260508 13:57:36.268 | INFO | Flushing graph node PREP_TRACE with 0 jobs. | 20260508 13:57:36.268 | INFO | Flushing graph node SAVE_TRACE with 0 jobs. | 20260508 13:57:36.268 | INFO | Flushing graph node SAVE_TRACE done. | 20260508 13:57:36.268 | INFO | Flushing graph node PREP_TRACE done. | 20260508 13:57:36.268 | INFO | Flushing graph node ANALYSIS with 0 jobs. | 20260508 13:57:36.268 | INFO | Flushing graph node PREP_RT_METRICS with 0 jobs. | 20260508 13:57:36.269 | INFO | Flushing graph node RT_METRICS with 0 jobs. | 20260508 13:57:36.269 | INFO | Flushing graph node RT_METRICS done. | 20260508 13:57:36.269 | INFO | Flushing graph node PREP_RT_METRICS done. | 20260508 13:57:36.269 | INFO | Flushing graph node PRE_HQ with 96 jobs. | 20260508 13:57:36.281 | INFO | Starting a round of Baz IO | 20260508 13:57:40.045 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.764 seconds | 20260508 13:57:40.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.00% / 60.09% | 20260508 13:57:40.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.75519e-05% / 0% / 3.17891e-05% | 20260508 13:57:40.045 | INFO | Baz IO overall: | 20260508 13:57:40.045 | INFO | BytesWritten : 3671749353197 | 20260508 13:57:40.045 | INFO | NumEvents : 1759012710199 | 20260508 13:57:40.046 | INFO | Flushing graph node BAZWRITER with 0 jobs. | 20260508 13:57:40.046 | INFO | Flushing graph node BAZWRITER done. | 20260508 13:57:40.046 | INFO | Flushing graph node BAZ_SAMPLING with 0 jobs. | 20260508 13:57:40.046 | INFO | Flushing graph node BAZ_SAMPLING done. | 20260508 13:57:40.046 | INFO | Flushing graph node PRE_HQ done. | 20260508 13:57:40.046 | INFO | Flushing graph node ANALYSIS done. | 20260508 13:57:40.046 | INFO | Flushing graph node REPACKER done. | 20260508 13:57:40.046 | INFO | Exited chunk analysis loop. | 20260508 13:57:40.046 | INFO | framesAnalyzed:8640000 framesRequested:8640000 | 20260508 13:57:40.046 | INFO | All frames analyzed = 8640000 | 20260508 13:57:40.046 | INFO | Analyzed 16875 chunks at 0.194557 chunks/sec (4.89618e+06 zmws/sec) | 20260508 13:57:40.238 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:57:40.238 | INFO | PerfReport: Gathered 1152 samples. | 20260508 13:57:40.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:57:40.238 | INFO | PerfReport: Full Execution -- 3.27e+01 / 7.54e+01, 1.37e+01 / 3.17e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 0.30%, AggregateStats -- 9.67e-02 / 1.03e-01, 1.07e-02 / 3.60e-03 | 20260508 13:57:40.238 | INFO | PerfReport: 12.18%, Baseline -- 3.98e+00 / 4.45e+00, 5.91e-01 / 1.92e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 3.39%, Histogram -- 1.11e+00 / 1.43e+00, 2.09e-01 / 1.14e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 16.02%, DME -- 4.19e+01 / 4.55e+01, 3.55e+00 / 1.13e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.87e+01 / 2.01e+01, 1.82e+00 / 1.24e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.29e+00 / 2.45e+00, 3.96e-01 / 1.75e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 2.98%, Metrics -- 9.75e-01 / 1.11e+00, 2.50e-01 / 1.49e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 0.88%, PulseSerializing -- 2.87e-01 / 3.17e-01, 4.64e-02 / 1.82e-02 | 20260508 13:57:40.238 | INFO | Final Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260508 13:57:40.238 | INFO | PerfReport: Gathered 1615776 samples. | 20260508 13:57:40.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:57:40.238 | INFO | PerfReport: Full Execution -- 3.26e+01 / 7.59e+01, 1.39e+01 / 1.71e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 0.30%, AggregateStats -- 9.66e-02 / 1.03e-01, 1.34e-02 / 6.41e-03 | 20260508 13:57:40.238 | INFO | PerfReport: 12.04%, Baseline -- 3.92e+00 / 4.47e+00, 5.71e-01 / 2.24e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 3.35%, Histogram -- 1.09e+00 / 1.48e+00, 1.99e-01 / 1.13e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 15.95%, DME -- 4.16e+01 / 4.54e+01, 3.81e+00 / 1.75e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.85e+01 / 2.01e+01, 1.68e+00 / 1.64e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.39e+00 / 2.71e+00, 4.01e-01 / 2.33e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 3.33%, Metrics -- 1.08e+00 / 1.31e+00, 2.65e-01 / 1.53e-01 | 20260508 13:57:40.238 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.41e-01 / 3.80e-01, 5.47e-02 / 3.03e-02 | 20260508 13:57:40.238 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:57:40.238 | INFO | PerfReport: Gathered 1152 samples. | 20260508 13:57:40.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:57:40.238 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.30e+01 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 24.68%, Upload -- 2.60e+01 / 0.00e+00, 8.07e+00 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 24.73%, Download -- 2.60e+01 / 0.00e+00, 8.10e+00 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 31.09%, Compute -- 3.27e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 2.27%, LockUpload -- 2.38e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 2.40%, LockDownload -- 2.53e+00 / 0.00e+00, 3.86e+00 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 14.82%, LockCompute -- 1.56e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260508 13:57:40.238 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260508 13:57:40.238 | INFO | Final Performance Profiling Results for IOStages: (reported in milliseconds) | 20260508 13:57:40.238 | INFO | PerfReport: Gathered 1620000 samples. | 20260508 13:57:40.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260508 13:57:40.238 | INFO | PerfReport: Full Execution -- 1.05e+02 / -nan, 2.41e+01 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 23.81%, Upload -- 2.49e+01 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 23.78%, Download -- 2.49e+01 / 0.00e+00, 7.89e+00 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 31.16%, Compute -- 3.26e+01 / 0.00e+00, 1.39e+01 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 1.99%, LockUpload -- 2.08e+00 / 0.00e+00, 5.29e+00 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 2.33%, LockDownload -- 2.44e+00 / 0.00e+00, 3.83e+00 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 16.91%, LockCompute -- 1.77e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260508 13:57:40.239 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260508 13:57:40.308 | INFO | Closing BAZ file: /data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1.bazfilelist.txt | 20260508 13:57:40.311 | INFO | 90 baz files left to close | 20260508 13:57:40.311 | INFO | 80 baz files left to close | 20260508 13:57:40.312 | INFO | 70 baz files left to close | 20260508 13:57:40.313 | INFO | 60 baz files left to close | 20260508 13:57:40.314 | INFO | 50 baz files left to close | 20260508 13:57:40.315 | INFO | 40 baz files left to close | 20260508 13:57:40.316 | INFO | 30 baz files left to close | 20260508 13:57:40.317 | INFO | 20 baz files left to close | 20260508 13:57:40.318 | INFO | 10 baz files left to close | 20260508 13:57:40.318 | INFO | 0 baz files left to close | 20260508 13:57:40.319 | INFO | Header/Footer Bytes : 2.395 GiB | 20260508 13:57:40.319 | INFO | OverheadBytes : 106.3 GiB | 20260508 13:57:40.319 | INFO | EventBytes : 3.104 TiB | 20260508 13:57:40.319 | INFO | MetricsBytes : 132.6 GiB | 20260508 13:57:40.319 | INFO | NumEvents : 1759012710199 | 20260508 13:57:40.319 | INFO | NumMetrics : 1723157400 | 20260508 13:57:40.319 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | Total BytesWritten : 3.339 TiB | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | UncompressedEventBytes : 3.514 TiB | 20260508 13:57:40.320 | INFO | UncompressedMetricsBytes : 150.9 GiB | 20260508 13:57:40.320 | INFO | Data compression ratio : 1.13 | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | 0.000266% of recorded slices had data loss, with 0.000171% from the start of a preliminary HQ regions | 20260508 13:57:40.320 | INFO | Closing baz file "/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1_basic_preview.baz": | 20260508 13:57:40.320 | INFO | Header/Footer Bytes : 2.464 MiB | 20260508 13:57:40.320 | INFO | OverheadBytes : 107.7 MiB | 20260508 13:57:40.320 | INFO | EventBytes : 3.11 GiB | 20260508 13:57:40.320 | INFO | MetricsBytes : 132.4 MiB | 20260508 13:57:40.320 | INFO | NumEvents : 1720616942 | 20260508 13:57:40.320 | INFO | NumMetrics : 1679476 | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | Total BytesWritten : 3.346 GiB | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | UncompressedEventBytes : 3.52 GiB | 20260508 13:57:40.320 | INFO | UncompressedMetricsBytes : 150.6 MiB | 20260508 13:57:40.320 | INFO | Data compression ratio : 1.13 | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | 0.000143% of recorded slices had data loss, with 0% from the start of a preliminary HQ regions | 20260508 13:57:40.320 | INFO | Closing baz file "/data/nrta/2/m84031_260507_084733_s1/pb_internal/baz/m84031_260507_084733_s1_full_preview.baz": | 20260508 13:57:40.320 | INFO | Header/Footer Bytes : 305.9 KiB | 20260508 13:57:40.320 | INFO | OverheadBytes : 11.93 MiB | 20260508 13:57:40.320 | INFO | EventBytes : 351.1 MiB | 20260508 13:57:40.320 | INFO | MetricsBytes : 14.42 MiB | 20260508 13:57:40.320 | INFO | NumEvents : 189282346 | 20260508 13:57:40.320 | INFO | NumMetrics : 182501 | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | Total BytesWritten : 377.7 MiB | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | UncompressedEventBytes : 396.9 MiB | 20260508 13:57:40.320 | INFO | UncompressedMetricsBytes : 16.36 MiB | 20260508 13:57:40.320 | INFO | Data compression ratio : 1.13 | 20260508 13:57:40.320 | INFO | ------------------------------------------ | 20260508 13:57:40.320 | INFO | 9.26e-05% of recorded slices had data loss, with 0% from the start of a preliminary HQ regions | 20260508 13:57:40.325 | NOTICE | ~WXIPCDataSource | 20260508 13:57:40.327 | NOTICE | Packet History start: now:325421.610 1970-01-04T18:23:41.610Z | 20260508 13:57:40.327 | NOTICE | packetHistory[0] 325402.489 2026-05-08T13:57:21.206Z 0x6e0000000 67093946721 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[1] 325402.489 2026-05-08T13:57:21.206Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[2] 325402.533 2026-05-08T13:57:21.250Z 0x680000000 67228164449 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[3] 325402.533 2026-05-08T13:57:21.250Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[4] 325402.575 2026-05-08T13:57:21.292Z 0xab8000000 67362382177 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[5] 325402.575 2026-05-08T13:57:21.292Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[6] 325402.617 2026-05-08T13:57:21.334Z 0xac0000000 67496599905 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[7] 325402.617 2026-05-08T13:57:21.334Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[8] 325402.661 2026-05-08T13:57:21.378Z 0x620000000 67630817633 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[9] 325402.661 2026-05-08T13:57:21.378Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[10] 325402.703 2026-05-08T13:57:21.420Z 0xaf8000000 67765035361 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[11] 325402.703 2026-05-08T13:57:21.420Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[12] 325402.745 2026-05-08T13:57:21.462Z 0xb60000000 67899253089 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[13] 325402.745 2026-05-08T13:57:21.462Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[14] 325402.788 2026-05-08T13:57:21.505Z 0x558000000 68033470817 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[15] 325402.788 2026-05-08T13:57:21.505Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[16] 325403.876 2026-05-08T13:57:22.593Z 0x9a0000000 68275691873 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[17] 325403.876 2026-05-08T13:57:22.593Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[18] 325403.920 2026-05-08T13:57:22.637Z 0x598000000 68143571301 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[19] 325403.920 2026-05-08T13:57:22.637Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[20] 325403.962 2026-05-08T13:57:22.679Z 0x890000000 68009353573 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[21] 325403.962 2026-05-08T13:57:22.679Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[22] 325404.004 2026-05-08T13:57:22.721Z 0x7d8000000 67899253113 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[23] 325404.004 2026-05-08T13:57:22.721Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[24] 325404.047 2026-05-08T13:57:22.764Z 0xa20000000 68143571301 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[25] 325404.047 2026-05-08T13:57:22.764Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[26] 325404.090 2026-05-08T13:57:22.807Z 0xa28000000 68009353573 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[27] 325404.090 2026-05-08T13:57:22.807Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[28] 325404.133 2026-05-08T13:57:22.850Z 0x898000000 67983139173 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[29] 325404.133 2026-05-08T13:57:22.850Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[30] 325404.175 2026-05-08T13:57:22.892Z 0x8b8000000 67740918117 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[31] 325404.175 2026-05-08T13:57:22.892Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[32] 325404.218 2026-05-08T13:57:22.935Z 0xba8000000 67740918141 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[33] 325404.218 2026-05-08T13:57:22.935Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[34] 325404.261 2026-05-08T13:57:22.978Z 0xb28000000 67630817633 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[35] 325404.261 2026-05-08T13:57:22.978Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[36] 325404.303 2026-05-08T13:57:23.020Z 0xbb0000000 67740918117 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[37] 325404.303 2026-05-08T13:57:23.020Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[38] 325404.346 2026-05-08T13:57:23.063Z 0x6f0000000 67714703717 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[39] 325404.346 2026-05-08T13:57:23.063Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[40] 325404.389 2026-05-08T13:57:23.106Z 0xb48000000 67472482661 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[41] 325404.389 2026-05-08T13:57:23.106Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[42] 325404.431 2026-05-08T13:57:23.148Z 0x748000000 67472482661 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[43] 325404.431 2026-05-08T13:57:23.148Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[44] 325404.474 2026-05-08T13:57:23.191Z 0xbd8000000 67228164473 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[45] 325404.474 2026-05-08T13:57:23.191Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[46] 325404.516 2026-05-08T13:57:23.233Z 0x7e8000000 67472482661 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[47] 325404.516 2026-05-08T13:57:23.233Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[48] 325404.560 2026-05-08T13:57:23.277Z 0x978000000 67338264933 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[49] 325404.560 2026-05-08T13:57:23.277Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[50] 325404.602 2026-05-08T13:57:23.319Z 0x7a8000000 67204047205 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[51] 325404.602 2026-05-08T13:57:23.319Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[52] 325404.644 2026-05-08T13:57:23.361Z 0x9c0000000 67069829477 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[53] 325404.644 2026-05-08T13:57:23.361Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[54] 325404.687 2026-05-08T13:57:23.404Z 0x630000000 67069829501 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[55] 325404.687 2026-05-08T13:57:23.404Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[56] 325404.730 2026-05-08T13:57:23.447Z 0x758000000 66959728993 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[57] 325404.730 2026-05-08T13:57:23.447Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[58] 325404.772 2026-05-08T13:57:23.489Z 0x998000000 67069829477 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[59] 325404.772 2026-05-08T13:57:23.489Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[60] 325404.815 2026-05-08T13:57:23.532Z 0x800000000 66935611749 free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[61] 325404.815 2026-05-08T13:57:23.532Z BURST 1 pointers, free:0 | 20260508 13:57:40.327 | NOTICE | packetHistory[62] 325404.858 2026-05-08T13:57:23.575Z 0x950000000 66935611749 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[63] 325404.858 2026-05-08T13:57:23.575Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[64] 325404.900 2026-05-08T13:57:23.617Z 0x548000000 66935611749 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[65] 325404.900 2026-05-08T13:57:23.617Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[66] 325404.943 2026-05-08T13:57:23.660Z 0x878000000 66935611749 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[67] 325404.943 2026-05-08T13:57:23.660Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[68] 325404.986 2026-05-08T13:57:23.703Z 0x660000000 66801394045 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[69] 325404.986 2026-05-08T13:57:23.703Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[70] 325405.028 2026-05-08T13:57:23.745Z 0x858000000 66935611749 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[71] 325405.028 2026-05-08T13:57:23.745Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[72] 325405.072 2026-05-08T13:57:23.789Z 0x5a8000000 66935611749 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[73] 325405.072 2026-05-08T13:57:23.789Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[74] 325405.114 2026-05-08T13:57:23.831Z 0x5b8000000 66935611749 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[75] 325405.114 2026-05-08T13:57:23.831Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[76] 325405.156 2026-05-08T13:57:23.873Z 0xac8000000 66909397349 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[77] 325405.156 2026-05-08T13:57:23.873Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[78] 325405.200 2026-05-08T13:57:23.917Z 0x740000000 66801394021 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[79] 325405.200 2026-05-08T13:57:23.917Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[80] 325405.242 2026-05-08T13:57:23.959Z 0x820000000 66801394021 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[81] 325405.242 2026-05-08T13:57:23.959Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[82] 325405.284 2026-05-08T13:57:24.001Z 0xb38000000 66667176317 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[83] 325405.284 2026-05-08T13:57:24.001Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[84] 325405.327 2026-05-08T13:57:24.044Z 0x8d0000000 66801394021 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[85] 325405.327 2026-05-08T13:57:24.044Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[86] 325405.370 2026-05-08T13:57:24.087Z 0xbc8000000 66801394021 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[87] 325405.370 2026-05-08T13:57:24.087Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[88] 325405.413 2026-05-08T13:57:24.130Z 0x500000000 66801394021 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[89] 325405.413 2026-05-08T13:57:24.130Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[90] 325405.455 2026-05-08T13:57:24.172Z 0x9d0000000 66665079137 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[91] 325405.455 2026-05-08T13:57:24.172Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[92] 325405.498 2026-05-08T13:57:24.215Z 0x910000000 66667176293 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[93] 325405.498 2026-05-08T13:57:24.215Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[94] 325405.541 2026-05-08T13:57:24.258Z 0x818000000 66667176293 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[95] 325405.541 2026-05-08T13:57:24.258Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[96] 325405.583 2026-05-08T13:57:24.300Z 0x788000000 66532958589 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[97] 325405.583 2026-05-08T13:57:24.300Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[98] 325405.626 2026-05-08T13:57:24.343Z 0xa70000000 66667176293 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[99] 325405.626 2026-05-08T13:57:24.343Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[100] 325405.669 2026-05-08T13:57:24.386Z 0x5c0000000 66667176293 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[101] 325405.669 2026-05-08T13:57:24.386Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[102] 325405.711 2026-05-08T13:57:24.428Z 0xaa8000000 66667176293 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[103] 325405.711 2026-05-08T13:57:24.428Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[104] 325405.753 2026-05-08T13:57:24.470Z 0x968000000 66640961893 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[105] 325405.753 2026-05-08T13:57:24.470Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[106] 325405.797 2026-05-08T13:57:24.514Z 0x778000000 66532958565 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[107] 325405.797 2026-05-08T13:57:24.514Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[108] 325405.839 2026-05-08T13:57:24.556Z 0x710000000 66532958565 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[109] 325405.839 2026-05-08T13:57:24.556Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[110] 325405.882 2026-05-08T13:57:24.599Z 0x570000000 66532958589 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[111] 325405.882 2026-05-08T13:57:24.599Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[112] 325405.924 2026-05-08T13:57:24.641Z 0x540000000 66530861409 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[113] 325405.924 2026-05-08T13:57:24.641Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[114] 325405.967 2026-05-08T13:57:24.684Z 0x618000000 66532958565 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[115] 325405.967 2026-05-08T13:57:24.684Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[116] 325406.010 2026-05-08T13:57:24.727Z 0xb58000000 66532958565 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[117] 325406.010 2026-05-08T13:57:24.727Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[118] 325406.053 2026-05-08T13:57:24.770Z 0x9d8000000 66532958565 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[119] 325406.053 2026-05-08T13:57:24.770Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[120] 325406.095 2026-05-08T13:57:24.812Z 0x880000000 66396643681 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[121] 325406.095 2026-05-08T13:57:24.812Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[122] 325406.138 2026-05-08T13:57:24.855Z 0x6a0000000 66398740837 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[123] 325406.138 2026-05-08T13:57:24.855Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[124] 325406.181 2026-05-08T13:57:24.898Z 0x5b0000000 66398740861 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[125] 325406.181 2026-05-08T13:57:24.898Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[126] 325406.223 2026-05-08T13:57:24.940Z 0x730000000 66288640353 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[127] 325406.223 2026-05-08T13:57:24.940Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[128] 325406.265 2026-05-08T13:57:24.982Z 0x988000000 66508841321 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[129] 325406.265 2026-05-08T13:57:24.982Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[130] 325406.309 2026-05-08T13:57:25.026Z 0x518000000 66398740837 free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[131] 325406.309 2026-05-08T13:57:25.026Z BURST 1 pointers, free:0 | 20260508 13:57:40.328 | NOTICE | packetHistory[132] 325406.351 2026-05-08T13:57:25.068Z 0x940000000 66398740837 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[133] 325406.351 2026-05-08T13:57:25.068Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[134] 325406.394 2026-05-08T13:57:25.111Z 0x810000000 66154422625 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[135] 325406.394 2026-05-08T13:57:25.111Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[136] 325406.437 2026-05-08T13:57:25.154Z 0x508000000 66264523109 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[137] 325406.437 2026-05-08T13:57:25.154Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[138] 325406.479 2026-05-08T13:57:25.196Z 0xa30000000 66264523133 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[139] 325406.479 2026-05-08T13:57:25.196Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[140] 325406.522 2026-05-08T13:57:25.239Z 0x5d0000000 66154422625 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[141] 325406.522 2026-05-08T13:57:25.239Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[142] 325406.565 2026-05-08T13:57:25.282Z 0x8e0000000 66372526437 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[143] 325406.565 2026-05-08T13:57:25.282Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[144] 325406.607 2026-05-08T13:57:25.324Z 0x6b0000000 66374623593 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[145] 325406.607 2026-05-08T13:57:25.324Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[146] 325406.650 2026-05-08T13:57:25.367Z 0x9f0000000 66264523109 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[147] 325406.650 2026-05-08T13:57:25.367Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[148] 325406.692 2026-05-08T13:57:25.409Z 0xa10000000 66130305381 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[149] 325406.692 2026-05-08T13:57:25.409Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[150] 325406.736 2026-05-08T13:57:25.453Z 0x610000000 66240405865 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[151] 325406.736 2026-05-08T13:57:25.453Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[152] 325406.778 2026-05-08T13:57:25.495Z 0x8c0000000 66130305405 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[153] 325406.778 2026-05-08T13:57:25.495Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[154] 325406.820 2026-05-08T13:57:25.537Z 0xa68000000 66020204897 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[155] 325406.820 2026-05-08T13:57:25.537Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[156] 325406.863 2026-05-08T13:57:25.580Z 0x948000000 66264523109 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[157] 325406.863 2026-05-08T13:57:25.580Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[158] 325406.906 2026-05-08T13:57:25.623Z 0xb70000000 66240405865 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[159] 325406.906 2026-05-08T13:57:25.623Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[160] 325406.949 2026-05-08T13:57:25.666Z 0xb68000000 66130305381 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[161] 325406.949 2026-05-08T13:57:25.666Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[162] 325406.991 2026-05-08T13:57:25.708Z 0x7c0000000 65996087653 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[163] 325406.991 2026-05-08T13:57:25.708Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[164] 325407.034 2026-05-08T13:57:25.751Z 0xb88000000 66104090981 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[165] 325407.034 2026-05-08T13:57:25.751Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[166] 325407.077 2026-05-08T13:57:25.794Z 0x578000000 66106188161 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[167] 325407.077 2026-05-08T13:57:25.794Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[168] 325407.119 2026-05-08T13:57:25.836Z 0xae0000000 65885987169 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[169] 325407.119 2026-05-08T13:57:25.836Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[170] 325407.162 2026-05-08T13:57:25.879Z 0x5f0000000 66130305381 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[171] 325407.162 2026-05-08T13:57:25.879Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[172] 325407.204 2026-05-08T13:57:25.921Z 0x9e8000000 66106188137 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[173] 325407.204 2026-05-08T13:57:25.921Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[174] 325407.247 2026-05-08T13:57:25.965Z 0x5c8000000 65885987169 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[175] 325407.248 2026-05-08T13:57:25.965Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[176] 325407.290 2026-05-08T13:57:26.007Z 0x7d0000000 66020204897 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[177] 325407.290 2026-05-08T13:57:26.007Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[178] 325407.332 2026-05-08T13:57:26.049Z 0x8c8000000 66154422625 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[179] 325407.332 2026-05-08T13:57:26.049Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[180] 325407.376 2026-05-08T13:57:26.093Z 0xae8000000 66288640353 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[181] 325407.376 2026-05-08T13:57:26.093Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[182] 325407.418 2026-05-08T13:57:26.135Z 0xab0000000 66422858081 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[183] 325407.418 2026-05-08T13:57:26.135Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[184] 325407.460 2026-05-08T13:57:26.177Z 0xaf0000000 66557075809 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[185] 325407.460 2026-05-08T13:57:26.177Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[186] 325407.503 2026-05-08T13:57:26.220Z 0xa40000000 66691293537 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[187] 325407.503 2026-05-08T13:57:26.220Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[188] 325407.546 2026-05-08T13:57:26.263Z 0xb00000000 66825511265 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[189] 325407.546 2026-05-08T13:57:26.263Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[190] 325407.589 2026-05-08T13:57:26.306Z 0xbe8000000 66959728993 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[191] 325407.589 2026-05-08T13:57:26.306Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[192] 325407.631 2026-05-08T13:57:26.348Z 0x808000000 67093946721 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[193] 325407.631 2026-05-08T13:57:26.348Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[194] 325407.674 2026-05-08T13:57:26.391Z 0x650000000 67228164449 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[195] 325407.674 2026-05-08T13:57:26.391Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[196] 325407.716 2026-05-08T13:57:26.433Z 0xba0000000 67362382177 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[197] 325407.716 2026-05-08T13:57:26.433Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[198] 325407.760 2026-05-08T13:57:26.477Z 0x930000000 67496599905 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[199] 325407.760 2026-05-08T13:57:26.477Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[200] 325407.802 2026-05-08T13:57:26.519Z 0x5e8000000 67630817633 free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[201] 325407.802 2026-05-08T13:57:26.519Z BURST 1 pointers, free:0 | 20260508 13:57:40.329 | NOTICE | packetHistory[202] 325407.845 2026-05-08T13:57:26.562Z 0x830000000 67765035361 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[203] 325407.845 2026-05-08T13:57:26.562Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[204] 325407.887 2026-05-08T13:57:26.604Z 0x8a0000000 67899253089 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[205] 325407.887 2026-05-08T13:57:26.604Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[206] 325407.930 2026-05-08T13:57:26.647Z 0x920000000 68033470817 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[207] 325407.930 2026-05-08T13:57:26.647Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[208] 325409.018 2026-05-08T13:57:27.735Z 0x990000000 68275691873 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[209] 325409.018 2026-05-08T13:57:27.735Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[210] 325409.061 2026-05-08T13:57:27.778Z 0x550000000 68143571301 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[211] 325409.061 2026-05-08T13:57:27.778Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[212] 325409.103 2026-05-08T13:57:27.820Z 0x670000000 68009353573 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[213] 325409.103 2026-05-08T13:57:27.820Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[214] 325409.145 2026-05-08T13:57:27.862Z 0x870000000 68009353597 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[215] 325409.146 2026-05-08T13:57:27.863Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[216] 325409.188 2026-05-08T13:57:27.905Z 0x7b8000000 67899253089 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[217] 325409.188 2026-05-08T13:57:27.905Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[218] 325409.231 2026-05-08T13:57:27.948Z 0xb78000000 68009353573 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[219] 325409.231 2026-05-08T13:57:27.948Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[220] 325409.274 2026-05-08T13:57:27.991Z 0x908000000 67983139173 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[221] 325409.274 2026-05-08T13:57:27.991Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[222] 325409.316 2026-05-08T13:57:28.033Z 0x9c8000000 67740918117 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[223] 325409.316 2026-05-08T13:57:28.033Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[224] 325409.359 2026-05-08T13:57:28.076Z 0xb18000000 67606700389 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[225] 325409.359 2026-05-08T13:57:28.076Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[226] 325409.402 2026-05-08T13:57:28.119Z 0x700000000 67496599929 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[227] 325409.402 2026-05-08T13:57:28.119Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[228] 325409.444 2026-05-08T13:57:28.161Z 0xa60000000 67740918117 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[229] 325409.444 2026-05-08T13:57:28.161Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[230] 325409.487 2026-05-08T13:57:28.204Z 0x530000000 67606700389 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[231] 325409.487 2026-05-08T13:57:28.204Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[232] 325409.530 2026-05-08T13:57:28.247Z 0xa78000000 67580485989 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[233] 325409.530 2026-05-08T13:57:28.247Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[234] 325409.572 2026-05-08T13:57:28.289Z 0xb98000000 67338264933 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[235] 325409.572 2026-05-08T13:57:28.289Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[236] 325409.614 2026-05-08T13:57:28.331Z 0x8b0000000 67338264957 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[237] 325409.614 2026-05-08T13:57:28.331Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[238] 325409.658 2026-05-08T13:57:28.375Z 0x538000000 67228164449 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[239] 325409.658 2026-05-08T13:57:28.375Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[240] 325409.700 2026-05-08T13:57:28.417Z 0x848000000 67338264933 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[241] 325409.700 2026-05-08T13:57:28.417Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[242] 325409.743 2026-05-08T13:57:28.460Z 0x8e8000000 67204047205 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[243] 325409.743 2026-05-08T13:57:28.460Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[244] 325409.786 2026-05-08T13:57:28.503Z 0x738000000 67177832805 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[245] 325409.786 2026-05-08T13:57:28.503Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[246] 325409.828 2026-05-08T13:57:28.545Z 0x520000000 66935611749 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[247] 325409.828 2026-05-08T13:57:28.545Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[248] 325409.871 2026-05-08T13:57:28.588Z 0xb30000000 66825511289 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[249] 325409.871 2026-05-08T13:57:28.588Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[250] 325409.914 2026-05-08T13:57:28.631Z 0xb80000000 66935611749 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[251] 325409.914 2026-05-08T13:57:28.631Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[252] 325409.956 2026-05-08T13:57:28.673Z 0x938000000 66935611749 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[253] 325409.956 2026-05-08T13:57:28.673Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[254] 325409.999 2026-05-08T13:57:28.716Z 0x760000000 66935611749 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[255] 325409.999 2026-05-08T13:57:28.716Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[256] 325410.041 2026-05-08T13:57:28.758Z 0x718000000 66801394021 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[257] 325410.041 2026-05-08T13:57:28.758Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[258] 325410.085 2026-05-08T13:57:28.802Z 0xb50000000 66799296865 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[259] 325410.085 2026-05-08T13:57:28.802Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[260] 325410.127 2026-05-08T13:57:28.844Z 0xbc0000000 66911494529 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[261] 325410.127 2026-05-08T13:57:28.844Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[262] 325410.170 2026-05-08T13:57:28.887Z 0x560000000 66691293537 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[263] 325410.170 2026-05-08T13:57:28.887Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[264] 325410.212 2026-05-08T13:57:28.929Z 0x840000000 66935611749 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[265] 325410.212 2026-05-08T13:57:28.929Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[266] 325410.255 2026-05-08T13:57:28.972Z 0x7f8000000 66935611749 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[267] 325410.255 2026-05-08T13:57:28.972Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[268] 325410.297 2026-05-08T13:57:29.014Z 0x7a0000000 66909397349 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[269] 325410.297 2026-05-08T13:57:29.014Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[270] 325410.341 2026-05-08T13:57:29.058Z 0x850000000 66801394021 free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[271] 325410.341 2026-05-08T13:57:29.058Z BURST 1 pointers, free:0 | 20260508 13:57:40.330 | NOTICE | packetHistory[272] 325410.383 2026-05-08T13:57:29.100Z 0x750000000 66667176293 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[273] 325410.383 2026-05-08T13:57:29.100Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[274] 325410.426 2026-05-08T13:57:29.143Z 0x728000000 66801394021 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[275] 325410.426 2026-05-08T13:57:29.143Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[276] 325410.468 2026-05-08T13:57:29.185Z 0x5e0000000 66667176317 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[277] 325410.468 2026-05-08T13:57:29.185Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[278] 325410.511 2026-05-08T13:57:29.228Z 0x928000000 66801394021 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[279] 325410.511 2026-05-08T13:57:29.228Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[280] 325410.554 2026-05-08T13:57:29.271Z 0x7f0000000 66801394021 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[281] 325410.554 2026-05-08T13:57:29.271Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[282] 325410.596 2026-05-08T13:57:29.313Z 0x608000000 66775179621 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[283] 325410.596 2026-05-08T13:57:29.313Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[284] 325410.639 2026-05-08T13:57:29.356Z 0x970000000 66667176293 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[285] 325410.639 2026-05-08T13:57:29.356Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[286] 325410.682 2026-05-08T13:57:29.399Z 0x790000000 66532958565 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[287] 325410.682 2026-05-08T13:57:29.399Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[288] 325410.724 2026-05-08T13:57:29.441Z 0x6c0000000 66667176293 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[289] 325410.724 2026-05-08T13:57:29.441Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[290] 325410.767 2026-05-08T13:57:29.484Z 0xa58000000 66532958589 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[291] 325410.767 2026-05-08T13:57:29.484Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[292] 325410.810 2026-05-08T13:57:29.527Z 0x510000000 66665079137 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[293] 325410.810 2026-05-08T13:57:29.527Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[294] 325410.852 2026-05-08T13:57:29.569Z 0xbd0000000 66667176293 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[295] 325410.852 2026-05-08T13:57:29.569Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[296] 325410.895 2026-05-08T13:57:29.612Z 0x690000000 66667176293 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[297] 325410.895 2026-05-08T13:57:29.612Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[298] 325410.937 2026-05-08T13:57:29.654Z 0xb10000000 66640961893 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[299] 325410.937 2026-05-08T13:57:29.654Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[300] 325410.980 2026-05-08T13:57:29.697Z 0xa18000000 66532958565 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[301] 325410.980 2026-05-08T13:57:29.697Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[302] 325411.023 2026-05-08T13:57:29.740Z 0x7b0000000 66532958565 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[303] 325411.023 2026-05-08T13:57:29.740Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[304] 325411.066 2026-05-08T13:57:29.783Z 0xa80000000 66532958589 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[305] 325411.066 2026-05-08T13:57:29.783Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[306] 325411.108 2026-05-08T13:57:29.825Z 0x5d8000000 66530861409 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[307] 325411.108 2026-05-08T13:57:29.825Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[308] 325411.150 2026-05-08T13:57:29.867Z 0x580000000 66532958565 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[309] 325411.150 2026-05-08T13:57:29.867Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[310] 325411.194 2026-05-08T13:57:29.911Z 0xaa0000000 66532958565 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[311] 325411.194 2026-05-08T13:57:29.911Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[312] 325411.236 2026-05-08T13:57:29.953Z 0xa50000000 66532958565 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[313] 325411.236 2026-05-08T13:57:29.953Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[314] 325411.279 2026-05-08T13:57:29.996Z 0x5f8000000 66398740837 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[315] 325411.279 2026-05-08T13:57:29.996Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[316] 325411.321 2026-05-08T13:57:30.038Z 0x8f0000000 66398740837 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[317] 325411.321 2026-05-08T13:57:30.038Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[318] 325411.365 2026-05-08T13:57:30.082Z 0xa98000000 66398740861 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[319] 325411.365 2026-05-08T13:57:30.082Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[320] 325411.407 2026-05-08T13:57:30.124Z 0x600000000 66396643681 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[321] 325411.407 2026-05-08T13:57:30.124Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[322] 325411.449 2026-05-08T13:57:30.166Z 0x6d0000000 66398740837 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[323] 325411.449 2026-05-08T13:57:30.166Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[324] 325411.492 2026-05-08T13:57:30.209Z 0x588000000 66398740837 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[325] 325411.492 2026-05-08T13:57:30.209Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[326] 325411.535 2026-05-08T13:57:30.252Z 0x980000000 66398740837 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[327] 325411.535 2026-05-08T13:57:30.252Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[328] 325411.578 2026-05-08T13:57:30.295Z 0x8f8000000 66262425953 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[329] 325411.578 2026-05-08T13:57:30.295Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[330] 325411.620 2026-05-08T13:57:30.337Z 0xad0000000 66264523109 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[331] 325411.620 2026-05-08T13:57:30.337Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[332] 325411.662 2026-05-08T13:57:30.379Z 0x640000000 66264523133 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[333] 325411.662 2026-05-08T13:57:30.379Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[334] 325411.706 2026-05-08T13:57:30.423Z 0x768000000 66154422625 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[335] 325411.706 2026-05-08T13:57:30.423Z BURST 1 pointers, free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[336] 325411.748 2026-05-08T13:57:30.465Z 0xbb8000000 66374623593 free:0 | 20260508 13:57:40.331 | NOTICE | packetHistory[337] 325411.748 2026-05-08T13:57:30.465Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[338] 325411.791 2026-05-08T13:57:30.508Z 0x9e0000000 66264523109 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[339] 325411.791 2026-05-08T13:57:30.508Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[340] 325411.833 2026-05-08T13:57:30.550Z 0x888000000 66264523109 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[341] 325411.833 2026-05-08T13:57:30.550Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[342] 325411.876 2026-05-08T13:57:30.593Z 0x838000000 66128208225 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[343] 325411.876 2026-05-08T13:57:30.593Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[344] 325411.919 2026-05-08T13:57:30.636Z 0xb40000000 66130305381 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[345] 325411.919 2026-05-08T13:57:30.636Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[346] 325411.961 2026-05-08T13:57:30.678Z 0x7e0000000 66130305405 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[347] 325411.961 2026-05-08T13:57:30.678Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[348] 325412.005 2026-05-08T13:57:30.722Z 0x860000000 66020204897 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[349] 325412.005 2026-05-08T13:57:30.722Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[350] 325412.047 2026-05-08T13:57:30.764Z 0xb08000000 66240405865 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[351] 325412.047 2026-05-08T13:57:30.764Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[352] 325412.089 2026-05-08T13:57:30.806Z 0xa48000000 66130305381 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[353] 325412.089 2026-05-08T13:57:30.806Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[354] 325412.133 2026-05-08T13:57:30.850Z 0x6e0000000 66130305381 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[355] 325412.133 2026-05-08T13:57:30.850Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[356] 325412.175 2026-05-08T13:57:30.892Z 0xab8000000 65885987169 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[357] 325412.175 2026-05-08T13:57:30.892Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[358] 325412.218 2026-05-08T13:57:30.935Z 0xac0000000 65996087653 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[359] 325412.218 2026-05-08T13:57:30.935Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[360] 325412.260 2026-05-08T13:57:30.977Z 0x620000000 65996087677 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[361] 325412.260 2026-05-08T13:57:30.977Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[362] 325412.302 2026-05-08T13:57:31.019Z 0xaf8000000 65993990497 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[363] 325412.302 2026-05-08T13:57:31.019Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[364] 325412.346 2026-05-08T13:57:31.063Z 0xb60000000 65996087653 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[365] 325412.346 2026-05-08T13:57:31.063Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[366] 325412.388 2026-05-08T13:57:31.105Z 0x558000000 65885987169 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[367] 325412.388 2026-05-08T13:57:31.105Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[368] 325412.431 2026-05-08T13:57:31.148Z 0x680000000 66020204897 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[369] 325412.431 2026-05-08T13:57:31.148Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[370] 325412.473 2026-05-08T13:57:31.190Z 0x5a0000000 66154422625 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[371] 325412.473 2026-05-08T13:57:31.190Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[372] 325412.517 2026-05-08T13:57:31.234Z 0x780000000 66288640353 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[373] 325412.517 2026-05-08T13:57:31.234Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[374] 325412.559 2026-05-08T13:57:31.276Z 0x528000000 66422858081 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[375] 325412.559 2026-05-08T13:57:31.276Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[376] 325412.601 2026-05-08T13:57:31.318Z 0xa08000000 66557075809 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[377] 325412.601 2026-05-08T13:57:31.318Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[378] 325412.645 2026-05-08T13:57:31.362Z 0x590000000 66691293537 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[379] 325412.645 2026-05-08T13:57:31.362Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[380] 325412.687 2026-05-08T13:57:31.404Z 0x9b8000000 66825511265 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[381] 325412.687 2026-05-08T13:57:31.404Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[382] 325412.729 2026-05-08T13:57:31.446Z 0xbe0000000 66959728993 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[383] 325412.729 2026-05-08T13:57:31.446Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[384] 325412.772 2026-05-08T13:57:31.489Z 0x798000000 67093946721 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[385] 325412.772 2026-05-08T13:57:31.489Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[386] 325412.815 2026-05-08T13:57:31.532Z 0x770000000 67228164449 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[387] 325412.815 2026-05-08T13:57:31.532Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[388] 325412.858 2026-05-08T13:57:31.575Z 0xa90000000 67362382177 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[389] 325412.858 2026-05-08T13:57:31.575Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[390] 325412.900 2026-05-08T13:57:31.617Z 0xb90000000 67496599905 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[391] 325412.900 2026-05-08T13:57:31.617Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[392] 325412.943 2026-05-08T13:57:31.660Z 0x720000000 67630817633 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[393] 325412.943 2026-05-08T13:57:31.660Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[394] 325412.986 2026-05-08T13:57:31.703Z 0xa00000000 67765035361 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[395] 325412.986 2026-05-08T13:57:31.703Z BURST 1 pointers, free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[396] 325413.028 2026-05-08T13:57:31.745Z 0x9b0000000 67899253089 free:0 | 20260508 13:57:40.332 | NOTICE | packetHistory[397] 325413.028 2026-05-08T13:57:31.745Z BURST 1 pointers, free:0 | 20260508 13:57:40.333 | NOTICE | packetHistory[398] 325413.070 2026-05-08T13:57:31.787Z 0x7c8000000 68033470817 free:0 | 20260508 13:57:40.333 | NOTICE | packetHistory[399] 325413.070 2026-05-08T13:57:31.787Z BURST 1 pointers, free:0 | 20260508 13:57:40.333 | NOTICE | std::size(inFlight):96 freequeue:0 | 20260508 13:57:40.333 | NOTICE | Packet History end | 20260508 13:57:40.333 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260508 13:57:40.333 | INFO | Total mem usage: 27.8 GiB | 20260508 13:57:40.333 | INFO | All Time High Water Marks sum: 27.8 GiB (Savings: 0%) | 20260508 13:57:40.333 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -15%) | 20260508 13:57:40.333 | INFO | Unused Cache: 3.62 GiB (Waste: 13.1%) | 20260508 13:57:40.333 | INFO | High Water Breakdown: | 20260508 13:57:40.333 | INFO | 100% : 24.1 GiB ( 86.9% Of Peak) : WXIPCDataSourceConfig | 20260508 13:57:40.333 | INFO | ------------------------------------------ | 20260508 13:57:40.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:57:40.428 | INFO | CpuSchedulerCanary: done | 20260508 13:57:44.101 | INFO | ----Memory usage report for Gpu Memory----- | 20260508 13:57:44.101 | INFO | Total mem usage: 15.7 GiB | 20260508 13:57:44.101 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260508 13:57:44.101 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.74%) | 20260508 13:57:44.101 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260508 13:57:44.101 | INFO | High Water Breakdown: | 20260508 13:57:44.101 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260508 13:57:44.101 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260508 13:57:44.101 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260508 13:57:44.101 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260508 13:57:44.101 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260508 13:57:44.101 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260508 13:57:44.101 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260508 13:57:44.101 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260508 13:57:44.101 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260508 13:57:44.101 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:57:44.101 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260508 13:57:44.101 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:57:44.101 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260508 13:57:44.101 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260508 13:57:44.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260508 13:57:44.101 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260508 13:57:44.101 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:57:44.101 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260508 13:57:44.101 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:57:44.101 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:57:44.101 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:57:44.102 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260508 13:57:44.102 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260508 13:57:44.102 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:57:44.102 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260508 13:57:44.102 | INFO | ------------------------------------------ | 20260508 13:57:44.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:57:51.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260508 13:57:51.775 | INFO | Total mem usage: 26.5 GiB | 20260508 13:57:51.775 | INFO | All Time High Water Marks sum: 27.3 GiB (Savings: 3.03%) | 20260508 13:57:51.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260508 13:57:51.775 | INFO | Unused Cache: 1.84 GiB (Waste: 6.95%) | 20260508 13:57:51.775 | INFO | High Water Breakdown: | 20260508 13:57:51.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260508 13:57:51.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260508 13:57:51.775 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260508 13:57:51.775 | INFO | 1.61% : 420 MiB ( 66.9% Of Peak) : /PulseSerializerDevice.cu:107 | 20260508 13:57:51.775 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260508 13:57:51.775 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260508 13:57:51.775 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260508 13:57:51.775 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260508 13:57:51.775 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260508 13:57:51.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260508 13:57:51.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260508 13:57:51.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260508 13:57:51.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260508 13:57:51.775 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260508 13:57:51.775 | INFO | ------------------------------------------ | 20260508 13:57:51.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260508 13:57:51.775 | NOTICE | Because ALLOCATION_HISTORY is defined, dumping CUDA allocation history for IA-849 | 20260508 13:57:51.775 | NOTICE | ----- PBCudaRuntime DumpAllocationHistoryToLog ---- (begin) | 20260508 13:57:51.775 | NOTICE | [0]: 1970-01-03T18:17:03.150Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [1]: 1970-01-03T18:17:03.157Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [2]: 1970-01-03T18:17:03.163Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.775 | NOTICE | [3]: 1970-01-03T18:17:03.168Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.775 | NOTICE | [4]: 1970-01-03T18:17:03.172Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.775 | NOTICE | [5]: 1970-01-03T18:17:03.175Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.775 | NOTICE | [6]: 1970-01-03T18:17:03.178Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.775 | NOTICE | [7]: 1970-01-03T18:17:03.178Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.775 | NOTICE | [8]: 1970-01-03T18:17:03.178Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.775 | NOTICE | [9]: 1970-01-03T18:17:03.178Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.775 | NOTICE | [10]: 1970-01-03T18:17:03.178Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.775 | NOTICE | [11]: 1970-01-03T18:17:03.280Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.775 | NOTICE | [12]: 1970-01-03T18:17:03.302Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.775 | NOTICE | [13]: 1970-01-03T18:17:03.324Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.775 | NOTICE | [14]: 1970-01-03T18:17:03.331Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [15]: 1970-01-03T18:17:03.337Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [16]: 1970-01-03T18:17:03.344Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.775 | NOTICE | [17]: 1970-01-03T18:17:03.348Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.775 | NOTICE | [18]: 1970-01-03T18:17:03.352Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.775 | NOTICE | [19]: 1970-01-03T18:17:03.355Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.775 | NOTICE | [20]: 1970-01-03T18:17:03.358Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.775 | NOTICE | [21]: 1970-01-03T18:17:03.358Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.775 | NOTICE | [22]: 1970-01-03T18:17:03.358Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.775 | NOTICE | [23]: 1970-01-03T18:17:03.358Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.775 | NOTICE | [24]: 1970-01-03T18:17:03.358Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.775 | NOTICE | [25]: 1970-01-03T18:17:03.461Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.775 | NOTICE | [26]: 1970-01-03T18:17:03.482Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.775 | NOTICE | [27]: 1970-01-03T18:17:03.504Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.775 | NOTICE | [28]: 1970-01-03T18:17:03.511Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [29]: 1970-01-03T18:17:03.517Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [30]: 1970-01-03T18:17:03.524Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.775 | NOTICE | [31]: 1970-01-03T18:17:03.528Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.775 | NOTICE | [32]: 1970-01-03T18:17:03.533Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.775 | NOTICE | [33]: 1970-01-03T18:17:03.536Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.775 | NOTICE | [34]: 1970-01-03T18:17:03.538Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.775 | NOTICE | [35]: 1970-01-03T18:17:03.539Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.775 | NOTICE | [36]: 1970-01-03T18:17:03.539Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.775 | NOTICE | [37]: 1970-01-03T18:17:03.539Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.775 | NOTICE | [38]: 1970-01-03T18:17:03.539Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.775 | NOTICE | [39]: 1970-01-03T18:17:03.641Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.775 | NOTICE | [40]: 1970-01-03T18:17:03.662Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.775 | NOTICE | [41]: 1970-01-03T18:17:03.684Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.775 | NOTICE | [42]: 1970-01-03T18:17:03.691Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [43]: 1970-01-03T18:17:03.697Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.775 | NOTICE | [44]: 1970-01-03T18:17:03.704Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [45]: 1970-01-03T18:17:03.708Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [46]: 1970-01-03T18:17:03.713Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [47]: 1970-01-03T18:17:03.716Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [48]: 1970-01-03T18:17:03.718Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.776 | NOTICE | [49]: 1970-01-03T18:17:03.719Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.776 | NOTICE | [50]: 1970-01-03T18:17:03.719Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.776 | NOTICE | [51]: 1970-01-03T18:17:03.719Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.776 | NOTICE | [52]: 1970-01-03T18:17:03.719Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.776 | NOTICE | [53]: 1970-01-03T18:17:03.820Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [54]: 1970-01-03T18:17:03.842Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [55]: 1970-01-03T18:17:03.864Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.776 | NOTICE | [56]: 1970-01-03T18:17:03.870Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [57]: 1970-01-03T18:17:03.877Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [58]: 1970-01-03T18:17:03.883Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [59]: 1970-01-03T18:17:03.888Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [60]: 1970-01-03T18:17:03.892Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [61]: 1970-01-03T18:17:03.895Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [62]: 1970-01-03T18:17:03.898Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.776 | NOTICE | [63]: 1970-01-03T18:17:03.898Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.776 | NOTICE | [64]: 1970-01-03T18:17:03.898Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.776 | NOTICE | [65]: 1970-01-03T18:17:03.898Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.776 | NOTICE | [66]: 1970-01-03T18:17:03.898Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.776 | NOTICE | [67]: 1970-01-03T18:17:04.001Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [68]: 1970-01-03T18:17:04.022Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [69]: 1970-01-03T18:17:04.044Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.776 | NOTICE | [70]: 1970-01-03T18:17:04.050Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [71]: 1970-01-03T18:17:04.057Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [72]: 1970-01-03T18:17:04.063Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [73]: 1970-01-03T18:17:04.068Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [74]: 1970-01-03T18:17:04.072Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [75]: 1970-01-03T18:17:04.075Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [76]: 1970-01-03T18:17:04.078Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.776 | NOTICE | [77]: 1970-01-03T18:17:04.078Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.776 | NOTICE | [78]: 1970-01-03T18:17:04.078Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.776 | NOTICE | [79]: 1970-01-03T18:17:04.078Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.776 | NOTICE | [80]: 1970-01-03T18:17:04.078Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.776 | NOTICE | [81]: 1970-01-03T18:17:04.180Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [82]: 1970-01-03T18:17:04.202Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [83]: 1970-01-03T18:17:04.224Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.776 | NOTICE | [84]: 1970-01-03T18:17:04.231Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [85]: 1970-01-03T18:17:04.237Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [86]: 1970-01-03T18:17:04.244Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [87]: 1970-01-03T18:17:04.248Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [88]: 1970-01-03T18:17:04.252Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [89]: 1970-01-03T18:17:04.255Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [90]: 1970-01-03T18:17:04.258Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.776 | NOTICE | [91]: 1970-01-03T18:17:04.258Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.776 | NOTICE | [92]: 1970-01-03T18:17:04.258Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.776 | NOTICE | [93]: 1970-01-03T18:17:04.258Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.776 | NOTICE | [94]: 1970-01-03T18:17:04.258Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.776 | NOTICE | [95]: 1970-01-03T18:17:04.361Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [96]: 1970-01-03T18:17:04.382Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [97]: 1970-01-03T18:17:04.404Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.776 | NOTICE | [98]: 1970-01-03T18:17:04.411Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [99]: 1970-01-03T18:17:04.417Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.776 | NOTICE | [100]: 1970-01-03T18:17:04.424Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [101]: 1970-01-03T18:17:04.428Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.776 | NOTICE | [102]: 1970-01-03T18:17:04.432Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [103]: 1970-01-03T18:17:04.435Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.776 | NOTICE | [104]: 1970-01-03T18:17:04.438Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.776 | NOTICE | [105]: 1970-01-03T18:17:04.438Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.776 | NOTICE | [106]: 1970-01-03T18:17:04.438Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.776 | NOTICE | [107]: 1970-01-03T18:17:04.438Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.776 | NOTICE | [108]: 1970-01-03T18:17:04.438Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.776 | NOTICE | [109]: 1970-01-03T18:17:04.540Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [110]: 1970-01-03T18:17:04.562Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.776 | NOTICE | [111]: 1970-01-03T18:17:04.584Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.776 | NOTICE | [112]: 1970-01-03T18:17:04.590Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [113]: 1970-01-03T18:17:04.597Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [114]: 1970-01-03T18:17:04.603Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [115]: 1970-01-03T18:17:04.608Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [116]: 1970-01-03T18:17:04.612Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [117]: 1970-01-03T18:17:04.615Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [118]: 1970-01-03T18:17:04.618Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.777 | NOTICE | [119]: 1970-01-03T18:17:04.618Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.777 | NOTICE | [120]: 1970-01-03T18:17:04.618Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.777 | NOTICE | [121]: 1970-01-03T18:17:04.618Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.777 | NOTICE | [122]: 1970-01-03T18:17:04.618Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.777 | NOTICE | [123]: 1970-01-03T18:17:04.720Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [124]: 1970-01-03T18:17:04.742Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [125]: 1970-01-03T18:17:04.764Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.777 | NOTICE | [126]: 1970-01-03T18:17:04.770Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [127]: 1970-01-03T18:17:04.777Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [128]: 1970-01-03T18:17:04.783Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [129]: 1970-01-03T18:17:04.788Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [130]: 1970-01-03T18:17:04.792Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [131]: 1970-01-03T18:17:04.795Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [132]: 1970-01-03T18:17:04.798Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.777 | NOTICE | [133]: 1970-01-03T18:17:04.798Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.777 | NOTICE | [134]: 1970-01-03T18:17:04.798Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.777 | NOTICE | [135]: 1970-01-03T18:17:04.798Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.777 | NOTICE | [136]: 1970-01-03T18:17:04.798Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.777 | NOTICE | [137]: 1970-01-03T18:17:04.900Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [138]: 1970-01-03T18:17:04.922Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [139]: 1970-01-03T18:17:04.944Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.777 | NOTICE | [140]: 1970-01-03T18:17:04.950Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [141]: 1970-01-03T18:17:04.957Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [142]: 1970-01-03T18:17:04.963Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [143]: 1970-01-03T18:17:04.968Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [144]: 1970-01-03T18:17:04.972Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [145]: 1970-01-03T18:17:04.975Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [146]: 1970-01-03T18:17:04.978Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.777 | NOTICE | [147]: 1970-01-03T18:17:04.978Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.777 | NOTICE | [148]: 1970-01-03T18:17:04.978Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.777 | NOTICE | [149]: 1970-01-03T18:17:04.978Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.777 | NOTICE | [150]: 1970-01-03T18:17:04.978Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.777 | NOTICE | [151]: 1970-01-03T18:17:05.080Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [152]: 1970-01-03T18:17:05.101Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [153]: 1970-01-03T18:17:05.123Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.777 | NOTICE | [154]: 1970-01-03T18:17:05.130Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [155]: 1970-01-03T18:17:05.136Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.777 | NOTICE | [156]: 1970-01-03T18:17:05.143Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [157]: 1970-01-03T18:17:05.147Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.777 | NOTICE | [158]: 1970-01-03T18:17:05.152Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [159]: 1970-01-03T18:17:05.154Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.777 | NOTICE | [160]: 1970-01-03T18:17:05.157Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.777 | NOTICE | [161]: 1970-01-03T18:17:05.157Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.777 | NOTICE | [162]: 1970-01-03T18:17:05.157Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.777 | NOTICE | [163]: 1970-01-03T18:17:05.157Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.777 | NOTICE | [164]: 1970-01-03T18:17:05.157Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.777 | NOTICE | [165]: 1970-01-03T18:17:05.259Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.777 | NOTICE | [166]: 1970-01-03T18:17:05.281Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [167]: 1970-01-03T18:17:05.303Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.778 | NOTICE | [168]: 1970-01-03T18:17:05.309Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [169]: 1970-01-03T18:17:05.316Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [170]: 1970-01-03T18:17:05.322Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [171]: 1970-01-03T18:17:05.327Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [172]: 1970-01-03T18:17:05.331Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [173]: 1970-01-03T18:17:05.334Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [174]: 1970-01-03T18:17:05.337Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.778 | NOTICE | [175]: 1970-01-03T18:17:05.337Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.778 | NOTICE | [176]: 1970-01-03T18:17:05.337Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.778 | NOTICE | [177]: 1970-01-03T18:17:05.337Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.778 | NOTICE | [178]: 1970-01-03T18:17:05.337Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.778 | NOTICE | [179]: 1970-01-03T18:17:05.439Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [180]: 1970-01-03T18:17:05.461Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [181]: 1970-01-03T18:17:05.483Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.778 | NOTICE | [182]: 1970-01-03T18:17:05.490Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [183]: 1970-01-03T18:17:05.496Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [184]: 1970-01-03T18:17:05.503Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [185]: 1970-01-03T18:17:05.507Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [186]: 1970-01-03T18:17:05.511Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [187]: 1970-01-03T18:17:05.514Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [188]: 1970-01-03T18:17:05.517Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.778 | NOTICE | [189]: 1970-01-03T18:17:05.517Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.778 | NOTICE | [190]: 1970-01-03T18:17:05.517Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.778 | NOTICE | [191]: 1970-01-03T18:17:05.517Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.778 | NOTICE | [192]: 1970-01-03T18:17:05.517Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.778 | NOTICE | [193]: 1970-01-03T18:17:05.620Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [194]: 1970-01-03T18:17:05.641Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [195]: 1970-01-03T18:17:05.663Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.778 | NOTICE | [196]: 1970-01-03T18:17:05.670Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [197]: 1970-01-03T18:17:05.676Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [198]: 1970-01-03T18:17:05.683Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [199]: 1970-01-03T18:17:05.687Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [200]: 1970-01-03T18:17:05.692Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [201]: 1970-01-03T18:17:05.694Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [202]: 1970-01-03T18:17:05.697Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.778 | NOTICE | [203]: 1970-01-03T18:17:05.697Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.778 | NOTICE | [204]: 1970-01-03T18:17:05.697Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.778 | NOTICE | [205]: 1970-01-03T18:17:05.697Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.778 | NOTICE | [206]: 1970-01-03T18:17:05.698Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.778 | NOTICE | [207]: 1970-01-03T18:17:05.799Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [208]: 1970-01-03T18:17:05.821Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [209]: 1970-01-03T18:17:05.843Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.778 | NOTICE | [210]: 1970-01-03T18:17:05.849Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [211]: 1970-01-03T18:17:05.856Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [212]: 1970-01-03T18:17:05.862Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [213]: 1970-01-03T18:17:05.867Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [214]: 1970-01-03T18:17:05.871Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [215]: 1970-01-03T18:17:05.874Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [216]: 1970-01-03T18:17:05.877Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.778 | NOTICE | [217]: 1970-01-03T18:17:05.877Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.778 | NOTICE | [218]: 1970-01-03T18:17:05.877Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.778 | NOTICE | [219]: 1970-01-03T18:17:05.877Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.778 | NOTICE | [220]: 1970-01-03T18:17:05.877Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.778 | NOTICE | [221]: 1970-01-03T18:17:05.979Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [222]: 1970-01-03T18:17:06.001Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [223]: 1970-01-03T18:17:06.023Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.778 | NOTICE | [224]: 1970-01-03T18:17:06.029Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [225]: 1970-01-03T18:17:06.036Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [226]: 1970-01-03T18:17:06.042Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [227]: 1970-01-03T18:17:06.047Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [228]: 1970-01-03T18:17:06.051Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [229]: 1970-01-03T18:17:06.054Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [230]: 1970-01-03T18:17:06.057Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.778 | NOTICE | [231]: 1970-01-03T18:17:06.057Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.778 | NOTICE | [232]: 1970-01-03T18:17:06.057Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.778 | NOTICE | [233]: 1970-01-03T18:17:06.057Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.778 | NOTICE | [234]: 1970-01-03T18:17:06.057Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.778 | NOTICE | [235]: 1970-01-03T18:17:06.159Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [236]: 1970-01-03T18:17:06.181Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.778 | NOTICE | [237]: 1970-01-03T18:17:06.203Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.778 | NOTICE | [238]: 1970-01-03T18:17:06.209Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [239]: 1970-01-03T18:17:06.216Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.778 | NOTICE | [240]: 1970-01-03T18:17:06.222Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [241]: 1970-01-03T18:17:06.227Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.778 | NOTICE | [242]: 1970-01-03T18:17:06.231Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.778 | NOTICE | [243]: 1970-01-03T18:17:06.234Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [244]: 1970-01-03T18:17:06.237Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.779 | NOTICE | [245]: 1970-01-03T18:17:06.237Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.779 | NOTICE | [246]: 1970-01-03T18:17:06.237Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.779 | NOTICE | [247]: 1970-01-03T18:17:06.237Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.779 | NOTICE | [248]: 1970-01-03T18:17:06.237Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.779 | NOTICE | [249]: 1970-01-03T18:17:06.371Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [250]: 1970-01-03T18:17:06.402Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [251]: 1970-01-03T18:17:06.436Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.779 | NOTICE | [252]: 1970-01-03T18:17:06.446Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [253]: 1970-01-03T18:17:06.455Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [254]: 1970-01-03T18:17:06.465Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [255]: 1970-01-03T18:17:06.473Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [256]: 1970-01-03T18:17:06.480Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [257]: 1970-01-03T18:17:06.484Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [258]: 1970-01-03T18:17:06.489Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.779 | NOTICE | [259]: 1970-01-03T18:17:06.489Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.779 | NOTICE | [260]: 1970-01-03T18:17:06.489Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.779 | NOTICE | [261]: 1970-01-03T18:17:06.489Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.779 | NOTICE | [262]: 1970-01-03T18:17:06.489Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.779 | NOTICE | [263]: 1970-01-03T18:17:06.650Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [264]: 1970-01-03T18:17:06.684Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [265]: 1970-01-03T18:17:06.716Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.779 | NOTICE | [266]: 1970-01-03T18:17:06.727Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [267]: 1970-01-03T18:17:06.737Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [268]: 1970-01-03T18:17:06.748Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [269]: 1970-01-03T18:17:06.753Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [270]: 1970-01-03T18:17:06.759Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [271]: 1970-01-03T18:17:06.764Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [272]: 1970-01-03T18:17:06.768Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.779 | NOTICE | [273]: 1970-01-03T18:17:06.769Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.779 | NOTICE | [274]: 1970-01-03T18:17:06.769Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.779 | NOTICE | [275]: 1970-01-03T18:17:06.769Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.779 | NOTICE | [276]: 1970-01-03T18:17:06.769Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.779 | NOTICE | [277]: 1970-01-03T18:17:06.925Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [278]: 1970-01-03T18:17:06.959Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [279]: 1970-01-03T18:17:06.993Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.779 | NOTICE | [280]: 1970-01-03T18:17:07.004Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [281]: 1970-01-03T18:17:07.015Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [282]: 1970-01-03T18:17:07.024Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [283]: 1970-01-03T18:17:07.031Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [284]: 1970-01-03T18:17:07.038Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [285]: 1970-01-03T18:17:07.043Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [286]: 1970-01-03T18:17:07.047Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.779 | NOTICE | [287]: 1970-01-03T18:17:07.048Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.779 | NOTICE | [288]: 1970-01-03T18:17:07.048Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.779 | NOTICE | [289]: 1970-01-03T18:17:07.048Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.779 | NOTICE | [290]: 1970-01-03T18:17:07.048Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.779 | NOTICE | [291]: 1970-01-03T18:17:07.205Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [292]: 1970-01-03T18:17:07.239Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [293]: 1970-01-03T18:17:07.270Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.779 | NOTICE | [294]: 1970-01-03T18:17:07.280Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [295]: 1970-01-03T18:17:07.291Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [296]: 1970-01-03T18:17:07.301Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [297]: 1970-01-03T18:17:07.308Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [298]: 1970-01-03T18:17:07.315Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [299]: 1970-01-03T18:17:07.320Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [300]: 1970-01-03T18:17:07.324Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.779 | NOTICE | [301]: 1970-01-03T18:17:07.325Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.779 | NOTICE | [302]: 1970-01-03T18:17:07.325Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.779 | NOTICE | [303]: 1970-01-03T18:17:07.325Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.779 | NOTICE | [304]: 1970-01-03T18:17:07.325Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.779 | NOTICE | [305]: 1970-01-03T18:17:07.482Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [306]: 1970-01-03T18:17:07.511Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [307]: 1970-01-03T18:17:07.542Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.779 | NOTICE | [308]: 1970-01-03T18:17:07.553Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [309]: 1970-01-03T18:17:07.563Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.779 | NOTICE | [310]: 1970-01-03T18:17:07.574Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [311]: 1970-01-03T18:17:07.581Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.779 | NOTICE | [312]: 1970-01-03T18:17:07.588Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [313]: 1970-01-03T18:17:07.591Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.779 | NOTICE | [314]: 1970-01-03T18:17:07.595Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.779 | NOTICE | [315]: 1970-01-03T18:17:07.596Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.779 | NOTICE | [316]: 1970-01-03T18:17:07.596Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.779 | NOTICE | [317]: 1970-01-03T18:17:07.596Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.779 | NOTICE | [318]: 1970-01-03T18:17:07.596Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.779 | NOTICE | [319]: 1970-01-03T18:17:07.745Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.779 | NOTICE | [320]: 1970-01-03T18:17:07.770Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [321]: 1970-01-03T18:17:07.802Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.780 | NOTICE | [322]: 1970-01-03T18:17:07.813Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [323]: 1970-01-03T18:17:07.821Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [324]: 1970-01-03T18:17:07.831Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [325]: 1970-01-03T18:17:07.837Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [326]: 1970-01-03T18:17:07.843Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [327]: 1970-01-03T18:17:07.848Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [328]: 1970-01-03T18:17:07.852Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.780 | NOTICE | [329]: 1970-01-03T18:17:07.853Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.780 | NOTICE | [330]: 1970-01-03T18:17:07.853Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.780 | NOTICE | [331]: 1970-01-03T18:17:07.853Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.780 | NOTICE | [332]: 1970-01-03T18:17:07.853Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.780 | NOTICE | [333]: 1970-01-03T18:17:08.014Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [334]: 1970-01-03T18:17:08.046Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [335]: 1970-01-03T18:17:08.077Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.780 | NOTICE | [336]: 1970-01-03T18:17:08.087Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [337]: 1970-01-03T18:17:08.097Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [338]: 1970-01-03T18:17:08.106Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [339]: 1970-01-03T18:17:08.113Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [340]: 1970-01-03T18:17:08.119Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [341]: 1970-01-03T18:17:08.124Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [342]: 1970-01-03T18:17:08.128Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.780 | NOTICE | [343]: 1970-01-03T18:17:08.128Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.780 | NOTICE | [344]: 1970-01-03T18:17:08.128Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.780 | NOTICE | [345]: 1970-01-03T18:17:08.128Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.780 | NOTICE | [346]: 1970-01-03T18:17:08.128Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.780 | NOTICE | [347]: 1970-01-03T18:17:08.277Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [348]: 1970-01-03T18:17:08.308Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [349]: 1970-01-03T18:17:08.340Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.780 | NOTICE | [350]: 1970-01-03T18:17:08.349Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [351]: 1970-01-03T18:17:08.359Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [352]: 1970-01-03T18:17:08.368Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [353]: 1970-01-03T18:17:08.375Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [354]: 1970-01-03T18:17:08.381Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [355]: 1970-01-03T18:17:08.386Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [356]: 1970-01-03T18:17:08.390Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.780 | NOTICE | [357]: 1970-01-03T18:17:08.390Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.780 | NOTICE | [358]: 1970-01-03T18:17:08.390Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.780 | NOTICE | [359]: 1970-01-03T18:17:08.390Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.780 | NOTICE | [360]: 1970-01-03T18:17:08.390Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.780 | NOTICE | [361]: 1970-01-03T18:17:08.542Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [362]: 1970-01-03T18:17:08.568Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [363]: 1970-01-03T18:17:08.602Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.780 | NOTICE | [364]: 1970-01-03T18:17:08.613Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [365]: 1970-01-03T18:17:08.622Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [366]: 1970-01-03T18:17:08.628Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [367]: 1970-01-03T18:17:08.633Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [368]: 1970-01-03T18:17:08.638Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [369]: 1970-01-03T18:17:08.641Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [370]: 1970-01-03T18:17:08.643Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.780 | NOTICE | [371]: 1970-01-03T18:17:08.644Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.780 | NOTICE | [372]: 1970-01-03T18:17:08.644Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.780 | NOTICE | [373]: 1970-01-03T18:17:08.644Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.780 | NOTICE | [374]: 1970-01-03T18:17:08.644Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.780 | NOTICE | [375]: 1970-01-03T18:17:08.773Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [376]: 1970-01-03T18:17:08.796Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [377]: 1970-01-03T18:17:08.827Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.780 | NOTICE | [378]: 1970-01-03T18:17:08.835Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [379]: 1970-01-03T18:17:08.842Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [380]: 1970-01-03T18:17:08.849Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [381]: 1970-01-03T18:17:08.854Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [382]: 1970-01-03T18:17:08.858Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [383]: 1970-01-03T18:17:08.861Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.780 | NOTICE | [384]: 1970-01-03T18:17:08.864Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.780 | NOTICE | [385]: 1970-01-03T18:17:08.864Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.780 | NOTICE | [386]: 1970-01-03T18:17:08.864Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.780 | NOTICE | [387]: 1970-01-03T18:17:08.864Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.780 | NOTICE | [388]: 1970-01-03T18:17:08.864Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.780 | NOTICE | [389]: 1970-01-03T18:17:08.978Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [390]: 1970-01-03T18:17:09.010Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.780 | NOTICE | [391]: 1970-01-03T18:17:09.033Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.780 | NOTICE | [392]: 1970-01-03T18:17:09.040Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [393]: 1970-01-03T18:17:09.050Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.780 | NOTICE | [394]: 1970-01-03T18:17:09.061Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [395]: 1970-01-03T18:17:09.067Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.780 | NOTICE | [396]: 1970-01-03T18:17:09.071Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [397]: 1970-01-03T18:17:09.074Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [398]: 1970-01-03T18:17:09.077Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.781 | NOTICE | [399]: 1970-01-03T18:17:09.077Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.781 | NOTICE | [400]: 1970-01-03T18:17:09.077Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.781 | NOTICE | [401]: 1970-01-03T18:17:09.077Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.781 | NOTICE | [402]: 1970-01-03T18:17:09.077Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.781 | NOTICE | [403]: 1970-01-03T18:17:09.205Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [404]: 1970-01-03T18:17:09.229Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [405]: 1970-01-03T18:17:09.261Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.781 | NOTICE | [406]: 1970-01-03T18:17:09.268Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [407]: 1970-01-03T18:17:09.274Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [408]: 1970-01-03T18:17:09.281Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [409]: 1970-01-03T18:17:09.286Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [410]: 1970-01-03T18:17:09.291Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [411]: 1970-01-03T18:17:09.293Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [412]: 1970-01-03T18:17:09.296Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.781 | NOTICE | [413]: 1970-01-03T18:17:09.297Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.781 | NOTICE | [414]: 1970-01-03T18:17:09.297Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.781 | NOTICE | [415]: 1970-01-03T18:17:09.297Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.781 | NOTICE | [416]: 1970-01-03T18:17:09.297Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.781 | NOTICE | [417]: 1970-01-03T18:17:09.413Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [418]: 1970-01-03T18:17:09.442Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [419]: 1970-01-03T18:17:09.468Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.781 | NOTICE | [420]: 1970-01-03T18:17:09.476Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [421]: 1970-01-03T18:17:09.487Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [422]: 1970-01-03T18:17:09.497Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [423]: 1970-01-03T18:17:09.503Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [424]: 1970-01-03T18:17:09.508Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [425]: 1970-01-03T18:17:09.511Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [426]: 1970-01-03T18:17:09.514Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.781 | NOTICE | [427]: 1970-01-03T18:17:09.514Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.781 | NOTICE | [428]: 1970-01-03T18:17:09.514Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.781 | NOTICE | [429]: 1970-01-03T18:17:09.514Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.781 | NOTICE | [430]: 1970-01-03T18:17:09.514Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.781 | NOTICE | [431]: 1970-01-03T18:17:09.632Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [432]: 1970-01-03T18:17:09.655Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [433]: 1970-01-03T18:17:09.680Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.781 | NOTICE | [434]: 1970-01-03T18:17:09.688Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [435]: 1970-01-03T18:17:09.695Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [436]: 1970-01-03T18:17:09.706Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [437]: 1970-01-03T18:17:09.711Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [438]: 1970-01-03T18:17:09.716Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [439]: 1970-01-03T18:17:09.719Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [440]: 1970-01-03T18:17:09.722Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.781 | NOTICE | [441]: 1970-01-03T18:17:09.722Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.781 | NOTICE | [442]: 1970-01-03T18:17:09.722Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.781 | NOTICE | [443]: 1970-01-03T18:17:09.722Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.781 | NOTICE | [444]: 1970-01-03T18:17:09.722Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.781 | NOTICE | [445]: 1970-01-03T18:17:09.858Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [446]: 1970-01-03T18:17:09.892Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [447]: 1970-01-03T18:17:09.927Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.781 | NOTICE | [448]: 1970-01-03T18:17:09.938Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [449]: 1970-01-03T18:17:09.948Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [450]: 1970-01-03T18:17:09.959Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [451]: 1970-01-03T18:17:09.966Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [452]: 1970-01-03T18:17:09.973Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [453]: 1970-01-03T18:17:09.977Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [454]: 1970-01-03T18:17:09.981Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.781 | NOTICE | [455]: 1970-01-03T18:17:09.981Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.781 | NOTICE | [456]: 1970-01-03T18:17:09.981Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.781 | NOTICE | [457]: 1970-01-03T18:17:09.982Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.781 | NOTICE | [458]: 1970-01-03T18:17:09.982Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.781 | NOTICE | [459]: 1970-01-03T18:17:10.139Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [460]: 1970-01-03T18:17:10.174Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.781 | NOTICE | [461]: 1970-01-03T18:17:10.208Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.781 | NOTICE | [462]: 1970-01-03T18:17:10.217Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [463]: 1970-01-03T18:17:10.227Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.781 | NOTICE | [464]: 1970-01-03T18:17:10.238Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [465]: 1970-01-03T18:17:10.244Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.781 | NOTICE | [466]: 1970-01-03T18:17:10.251Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [467]: 1970-01-03T18:17:10.255Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.781 | NOTICE | [468]: 1970-01-03T18:17:10.260Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.781 | NOTICE | [469]: 1970-01-03T18:17:10.260Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.781 | NOTICE | [470]: 1970-01-03T18:17:10.260Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.781 | NOTICE | [471]: 1970-01-03T18:17:10.260Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.781 | NOTICE | [472]: 1970-01-03T18:17:10.260Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.782 | NOTICE | [473]: 1970-01-03T18:17:10.411Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [474]: 1970-01-03T18:17:10.441Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [475]: 1970-01-03T18:17:10.471Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.782 | NOTICE | [476]: 1970-01-03T18:17:10.481Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [477]: 1970-01-03T18:17:10.491Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [478]: 1970-01-03T18:17:10.499Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [479]: 1970-01-03T18:17:10.506Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [480]: 1970-01-03T18:17:10.513Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [481]: 1970-01-03T18:17:10.517Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [482]: 1970-01-03T18:17:10.521Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.782 | NOTICE | [483]: 1970-01-03T18:17:10.521Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.782 | NOTICE | [484]: 1970-01-03T18:17:10.521Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.782 | NOTICE | [485]: 1970-01-03T18:17:10.521Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.782 | NOTICE | [486]: 1970-01-03T18:17:10.521Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.782 | NOTICE | [487]: 1970-01-03T18:17:10.663Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [488]: 1970-01-03T18:17:10.691Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [489]: 1970-01-03T18:17:10.727Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.782 | NOTICE | [490]: 1970-01-03T18:17:10.740Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [491]: 1970-01-03T18:17:10.752Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [492]: 1970-01-03T18:17:10.763Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [493]: 1970-01-03T18:17:10.771Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [494]: 1970-01-03T18:17:10.779Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [495]: 1970-01-03T18:17:10.784Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [496]: 1970-01-03T18:17:10.787Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.782 | NOTICE | [497]: 1970-01-03T18:17:10.788Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.782 | NOTICE | [498]: 1970-01-03T18:17:10.788Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.782 | NOTICE | [499]: 1970-01-03T18:17:10.788Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.782 | NOTICE | [500]: 1970-01-03T18:17:10.788Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.782 | NOTICE | [501]: 1970-01-03T18:17:10.891Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [502]: 1970-01-03T18:17:10.913Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [503]: 1970-01-03T18:17:10.935Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.782 | NOTICE | [504]: 1970-01-03T18:17:10.941Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [505]: 1970-01-03T18:17:10.948Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [506]: 1970-01-03T18:17:10.954Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [507]: 1970-01-03T18:17:10.959Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [508]: 1970-01-03T18:17:10.963Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [509]: 1970-01-03T18:17:10.966Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [510]: 1970-01-03T18:17:10.969Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.782 | NOTICE | [511]: 1970-01-03T18:17:10.969Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.782 | NOTICE | [512]: 1970-01-03T18:17:10.969Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.782 | NOTICE | [513]: 1970-01-03T18:17:10.969Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.782 | NOTICE | [514]: 1970-01-03T18:17:10.969Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.782 | NOTICE | [515]: 1970-01-03T18:17:11.072Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [516]: 1970-01-03T18:17:11.094Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [517]: 1970-01-03T18:17:11.116Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.782 | NOTICE | [518]: 1970-01-03T18:17:11.123Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [519]: 1970-01-03T18:17:11.129Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [520]: 1970-01-03T18:17:11.136Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [521]: 1970-01-03T18:17:11.140Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [522]: 1970-01-03T18:17:11.145Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [523]: 1970-01-03T18:17:11.148Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [524]: 1970-01-03T18:17:11.150Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.782 | NOTICE | [525]: 1970-01-03T18:17:11.151Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.782 | NOTICE | [526]: 1970-01-03T18:17:11.151Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.782 | NOTICE | [527]: 1970-01-03T18:17:11.151Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.782 | NOTICE | [528]: 1970-01-03T18:17:11.151Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.782 | NOTICE | [529]: 1970-01-03T18:17:11.255Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [530]: 1970-01-03T18:17:11.277Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [531]: 1970-01-03T18:17:11.299Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.782 | NOTICE | [532]: 1970-01-03T18:17:11.306Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [533]: 1970-01-03T18:17:11.312Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [534]: 1970-01-03T18:17:11.319Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [535]: 1970-01-03T18:17:11.323Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.782 | NOTICE | [536]: 1970-01-03T18:17:11.328Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [537]: 1970-01-03T18:17:11.331Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.782 | NOTICE | [538]: 1970-01-03T18:17:11.334Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.782 | NOTICE | [539]: 1970-01-03T18:17:11.334Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.782 | NOTICE | [540]: 1970-01-03T18:17:11.334Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.782 | NOTICE | [541]: 1970-01-03T18:17:11.334Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.782 | NOTICE | [542]: 1970-01-03T18:17:11.334Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.782 | NOTICE | [543]: 1970-01-03T18:17:11.438Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [544]: 1970-01-03T18:17:11.460Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.782 | NOTICE | [545]: 1970-01-03T18:17:11.481Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.782 | NOTICE | [546]: 1970-01-03T18:17:11.488Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [547]: 1970-01-03T18:17:11.495Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.782 | NOTICE | [548]: 1970-01-03T18:17:11.501Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [549]: 1970-01-03T18:17:11.506Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [550]: 1970-01-03T18:17:11.511Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [551]: 1970-01-03T18:17:11.513Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [552]: 1970-01-03T18:17:11.516Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.783 | NOTICE | [553]: 1970-01-03T18:17:11.516Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.783 | NOTICE | [554]: 1970-01-03T18:17:11.516Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.783 | NOTICE | [555]: 1970-01-03T18:17:11.516Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.783 | NOTICE | [556]: 1970-01-03T18:17:11.516Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.783 | NOTICE | [557]: 1970-01-03T18:17:11.619Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [558]: 1970-01-03T18:17:11.641Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [559]: 1970-01-03T18:17:11.663Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.783 | NOTICE | [560]: 1970-01-03T18:17:11.670Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [561]: 1970-01-03T18:17:11.676Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [562]: 1970-01-03T18:17:11.683Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [563]: 1970-01-03T18:17:11.687Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [564]: 1970-01-03T18:17:11.692Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [565]: 1970-01-03T18:17:11.695Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [566]: 1970-01-03T18:17:11.697Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.783 | NOTICE | [567]: 1970-01-03T18:17:11.698Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.783 | NOTICE | [568]: 1970-01-03T18:17:11.698Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.783 | NOTICE | [569]: 1970-01-03T18:17:11.698Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.783 | NOTICE | [570]: 1970-01-03T18:17:11.698Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.783 | NOTICE | [571]: 1970-01-03T18:17:11.802Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [572]: 1970-01-03T18:17:11.824Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [573]: 1970-01-03T18:17:11.846Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.783 | NOTICE | [574]: 1970-01-03T18:17:11.852Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [575]: 1970-01-03T18:17:11.859Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [576]: 1970-01-03T18:17:11.866Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [577]: 1970-01-03T18:17:11.870Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [578]: 1970-01-03T18:17:11.874Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [579]: 1970-01-03T18:17:11.877Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [580]: 1970-01-03T18:17:11.880Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.783 | NOTICE | [581]: 1970-01-03T18:17:11.880Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.783 | NOTICE | [582]: 1970-01-03T18:17:11.880Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.783 | NOTICE | [583]: 1970-01-03T18:17:11.880Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.783 | NOTICE | [584]: 1970-01-03T18:17:11.880Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.783 | NOTICE | [585]: 1970-01-03T18:17:11.984Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [586]: 1970-01-03T18:17:12.006Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [587]: 1970-01-03T18:17:12.028Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.783 | NOTICE | [588]: 1970-01-03T18:17:12.035Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [589]: 1970-01-03T18:17:12.041Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [590]: 1970-01-03T18:17:12.048Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [591]: 1970-01-03T18:17:12.052Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [592]: 1970-01-03T18:17:12.057Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [593]: 1970-01-03T18:17:12.060Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [594]: 1970-01-03T18:17:12.063Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.783 | NOTICE | [595]: 1970-01-03T18:17:12.063Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.783 | NOTICE | [596]: 1970-01-03T18:17:12.063Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.783 | NOTICE | [597]: 1970-01-03T18:17:12.063Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.783 | NOTICE | [598]: 1970-01-03T18:17:12.063Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.783 | NOTICE | [599]: 1970-01-03T18:17:12.166Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [600]: 1970-01-03T18:17:12.188Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [601]: 1970-01-03T18:17:12.210Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.783 | NOTICE | [602]: 1970-01-03T18:17:12.217Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [603]: 1970-01-03T18:17:12.223Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [604]: 1970-01-03T18:17:12.230Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [605]: 1970-01-03T18:17:12.234Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [606]: 1970-01-03T18:17:12.239Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [607]: 1970-01-03T18:17:12.242Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [608]: 1970-01-03T18:17:12.245Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.783 | NOTICE | [609]: 1970-01-03T18:17:12.245Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.783 | NOTICE | [610]: 1970-01-03T18:17:12.245Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.783 | NOTICE | [611]: 1970-01-03T18:17:12.245Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.783 | NOTICE | [612]: 1970-01-03T18:17:12.245Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.783 | NOTICE | [613]: 1970-01-03T18:17:12.349Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [614]: 1970-01-03T18:17:12.372Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.783 | NOTICE | [615]: 1970-01-03T18:17:12.394Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.783 | NOTICE | [616]: 1970-01-03T18:17:12.401Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [617]: 1970-01-03T18:17:12.408Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.783 | NOTICE | [618]: 1970-01-03T18:17:12.415Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [619]: 1970-01-03T18:17:12.419Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.783 | NOTICE | [620]: 1970-01-03T18:17:12.424Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [621]: 1970-01-03T18:17:12.427Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.783 | NOTICE | [622]: 1970-01-03T18:17:12.429Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.784 | NOTICE | [623]: 1970-01-03T18:17:12.430Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.784 | NOTICE | [624]: 1970-01-03T18:17:12.430Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.784 | NOTICE | [625]: 1970-01-03T18:17:12.430Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.784 | NOTICE | [626]: 1970-01-03T18:17:12.430Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.784 | NOTICE | [627]: 1970-01-03T18:17:12.535Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [628]: 1970-01-03T18:17:12.556Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [629]: 1970-01-03T18:17:12.578Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.784 | NOTICE | [630]: 1970-01-03T18:17:12.585Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [631]: 1970-01-03T18:17:12.592Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [632]: 1970-01-03T18:17:12.599Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [633]: 1970-01-03T18:17:12.603Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [634]: 1970-01-03T18:17:12.608Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [635]: 1970-01-03T18:17:12.610Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [636]: 1970-01-03T18:17:12.613Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.784 | NOTICE | [637]: 1970-01-03T18:17:12.613Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.784 | NOTICE | [638]: 1970-01-03T18:17:12.613Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.784 | NOTICE | [639]: 1970-01-03T18:17:12.613Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.784 | NOTICE | [640]: 1970-01-03T18:17:12.614Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.784 | NOTICE | [641]: 1970-01-03T18:17:12.718Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [642]: 1970-01-03T18:17:12.740Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [643]: 1970-01-03T18:17:12.762Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.784 | NOTICE | [644]: 1970-01-03T18:17:12.769Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [645]: 1970-01-03T18:17:12.776Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [646]: 1970-01-03T18:17:12.782Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [647]: 1970-01-03T18:17:12.787Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [648]: 1970-01-03T18:17:12.792Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [649]: 1970-01-03T18:17:12.794Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [650]: 1970-01-03T18:17:12.797Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.784 | NOTICE | [651]: 1970-01-03T18:17:12.797Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.784 | NOTICE | [652]: 1970-01-03T18:17:12.797Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.784 | NOTICE | [653]: 1970-01-03T18:17:12.797Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.784 | NOTICE | [654]: 1970-01-03T18:17:12.797Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.784 | NOTICE | [655]: 1970-01-03T18:17:12.901Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [656]: 1970-01-03T18:17:12.924Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [657]: 1970-01-03T18:17:12.948Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.784 | NOTICE | [658]: 1970-01-03T18:17:12.955Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [659]: 1970-01-03T18:17:12.963Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [660]: 1970-01-03T18:17:12.970Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [661]: 1970-01-03T18:17:12.974Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [662]: 1970-01-03T18:17:12.979Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [663]: 1970-01-03T18:17:12.982Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [664]: 1970-01-03T18:17:12.985Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.784 | NOTICE | [665]: 1970-01-03T18:17:12.986Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.784 | NOTICE | [666]: 1970-01-03T18:17:12.986Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.784 | NOTICE | [667]: 1970-01-03T18:17:12.986Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.784 | NOTICE | [668]: 1970-01-03T18:17:12.986Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.784 | NOTICE | [669]: 1970-01-03T18:17:13.100Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [670]: 1970-01-03T18:17:13.124Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [671]: 1970-01-03T18:17:13.148Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.784 | NOTICE | [672]: 1970-01-03T18:17:13.155Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [673]: 1970-01-03T18:17:13.163Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [674]: 1970-01-03T18:17:13.170Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [675]: 1970-01-03T18:17:13.175Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [676]: 1970-01-03T18:17:13.180Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [677]: 1970-01-03T18:17:13.183Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [678]: 1970-01-03T18:17:13.186Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.784 | NOTICE | [679]: 1970-01-03T18:17:13.186Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.784 | NOTICE | [680]: 1970-01-03T18:17:13.186Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.784 | NOTICE | [681]: 1970-01-03T18:17:13.186Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.784 | NOTICE | [682]: 1970-01-03T18:17:13.186Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.784 | NOTICE | [683]: 1970-01-03T18:17:13.296Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [684]: 1970-01-03T18:17:13.320Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [685]: 1970-01-03T18:17:13.344Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.784 | NOTICE | [686]: 1970-01-03T18:17:13.352Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [687]: 1970-01-03T18:17:13.359Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.784 | NOTICE | [688]: 1970-01-03T18:17:13.366Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [689]: 1970-01-03T18:17:13.371Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.784 | NOTICE | [690]: 1970-01-03T18:17:13.376Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [691]: 1970-01-03T18:17:13.379Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.784 | NOTICE | [692]: 1970-01-03T18:17:13.382Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.784 | NOTICE | [693]: 1970-01-03T18:17:13.382Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.784 | NOTICE | [694]: 1970-01-03T18:17:13.382Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.784 | NOTICE | [695]: 1970-01-03T18:17:13.382Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.784 | NOTICE | [696]: 1970-01-03T18:17:13.382Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.784 | NOTICE | [697]: 1970-01-03T18:17:13.490Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.784 | NOTICE | [698]: 1970-01-03T18:17:13.513Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [699]: 1970-01-03T18:17:13.536Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.785 | NOTICE | [700]: 1970-01-03T18:17:13.543Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [701]: 1970-01-03T18:17:13.550Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [702]: 1970-01-03T18:17:13.556Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [703]: 1970-01-03T18:17:13.561Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [704]: 1970-01-03T18:17:13.565Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [705]: 1970-01-03T18:17:13.568Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [706]: 1970-01-03T18:17:13.571Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.785 | NOTICE | [707]: 1970-01-03T18:17:13.572Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.785 | NOTICE | [708]: 1970-01-03T18:17:13.572Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.785 | NOTICE | [709]: 1970-01-03T18:17:13.572Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.785 | NOTICE | [710]: 1970-01-03T18:17:13.572Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.785 | NOTICE | [711]: 1970-01-03T18:17:13.695Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [712]: 1970-01-03T18:17:13.717Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [713]: 1970-01-03T18:17:13.740Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.785 | NOTICE | [714]: 1970-01-03T18:17:13.747Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [715]: 1970-01-03T18:17:13.754Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [716]: 1970-01-03T18:17:13.760Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [717]: 1970-01-03T18:17:13.765Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [718]: 1970-01-03T18:17:13.770Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [719]: 1970-01-03T18:17:13.772Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [720]: 1970-01-03T18:17:13.775Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.785 | NOTICE | [721]: 1970-01-03T18:17:13.776Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.785 | NOTICE | [722]: 1970-01-03T18:17:13.776Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.785 | NOTICE | [723]: 1970-01-03T18:17:13.776Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.785 | NOTICE | [724]: 1970-01-03T18:17:13.776Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.785 | NOTICE | [725]: 1970-01-03T18:17:13.880Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [726]: 1970-01-03T18:17:13.902Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [727]: 1970-01-03T18:17:13.925Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.785 | NOTICE | [728]: 1970-01-03T18:17:13.932Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [729]: 1970-01-03T18:17:13.938Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [730]: 1970-01-03T18:17:13.945Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [731]: 1970-01-03T18:17:13.950Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [732]: 1970-01-03T18:17:13.954Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [733]: 1970-01-03T18:17:13.957Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [734]: 1970-01-03T18:17:13.960Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.785 | NOTICE | [735]: 1970-01-03T18:17:13.960Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.785 | NOTICE | [736]: 1970-01-03T18:17:13.960Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.785 | NOTICE | [737]: 1970-01-03T18:17:13.960Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.785 | NOTICE | [738]: 1970-01-03T18:17:13.960Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.785 | NOTICE | [739]: 1970-01-03T18:17:14.065Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [740]: 1970-01-03T18:17:14.087Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [741]: 1970-01-03T18:17:14.111Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.785 | NOTICE | [742]: 1970-01-03T18:17:14.118Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [743]: 1970-01-03T18:17:14.125Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [744]: 1970-01-03T18:17:14.132Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [745]: 1970-01-03T18:17:14.137Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [746]: 1970-01-03T18:17:14.142Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [747]: 1970-01-03T18:17:14.145Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [748]: 1970-01-03T18:17:14.148Z GPU thread:139873799102464 size:60293120 | 20260508 13:57:51.785 | NOTICE | [749]: 1970-01-03T18:17:14.149Z GPU thread:139873799102464 size:15728640 | 20260508 13:57:51.785 | NOTICE | [750]: 1970-01-03T18:17:14.149Z GPU thread:139873799102464 size:6291456 | 20260508 13:57:51.785 | NOTICE | [751]: 1970-01-03T18:17:14.149Z GPU thread:139873799102464 size:1048576 | 20260508 13:57:51.785 | NOTICE | [752]: 1970-01-03T18:17:14.149Z HOST thread:139873799102464 size:160432128 | 20260508 13:57:51.785 | NOTICE | [753]: 1970-01-03T18:17:14.257Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [754]: 1970-01-03T18:17:14.279Z HOST thread:139873799102464 size:33030144 | 20260508 13:57:51.785 | NOTICE | [755]: 1970-01-03T18:17:14.301Z HOST thread:139873799102464 size:10485760 | 20260508 13:57:51.785 | NOTICE | [756]: 1970-01-03T18:17:14.308Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [757]: 1970-01-03T18:17:14.315Z HOST thread:139873799102464 size:9961472 | 20260508 13:57:51.785 | NOTICE | [758]: 1970-01-03T18:17:14.322Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [759]: 1970-01-03T18:17:14.326Z HOST thread:139873799102464 size:6815744 | 20260508 13:57:51.785 | NOTICE | [760]: 1970-01-03T18:17:14.331Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [761]: 1970-01-03T18:17:14.334Z HOST thread:139873799102464 size:4194304 | 20260508 13:57:51.785 | NOTICE | [762]: 1970-01-03T18:18:12.647Z GPU thread:139832345665536 size:134217728 | 20260508 13:57:51.785 | NOTICE | [763]: 1970-01-03T18:18:12.665Z HOST thread:139832345665536 size:1 | 20260508 13:57:51.785 | NOTICE | [764]: 1970-01-03T18:18:12.665Z GPU thread:139832313131008 size:134217728 | 20260508 13:57:51.785 | NOTICE | [765]: 1970-01-03T18:18:12.665Z HOST thread:139832345665536 size:256 | 20260508 13:57:51.785 | NOTICE | [766]: 1970-01-03T18:18:12.666Z GPU thread:139832345665536 size:268435456 | 20260508 13:57:51.785 | NOTICE | [767]: 1970-01-03T18:18:12.671Z GPU thread:139832345665536 size:24641536 | 20260508 13:57:51.785 | NOTICE | [768]: 1970-01-03T18:18:12.671Z GPU thread:139832313131008 size:160432128 | 20260508 13:57:51.785 | NOTICE | [769]: 1970-01-03T18:18:12.672Z GPU thread:139832345665536 size:134217728 | 20260508 13:57:51.785 | NOTICE | [770]: 1970-01-03T18:18:12.676Z GPU thread:139832345665536 size:134217728 | 20260508 13:57:51.785 | NOTICE | [771]: 1970-01-03T18:18:12.678Z GPU thread:139832313131008 size:33030144 | 20260508 13:57:51.785 | NOTICE | [772]: 1970-01-03T18:18:12.679Z GPU thread:139832313131008 size:33030144 | 20260508 13:57:51.785 | NOTICE | [773]: 1970-01-03T18:18:12.681Z GPU thread:139832313131008 size:10485760 | 20260508 13:57:51.786 | NOTICE | [774]: 1970-01-03T18:18:12.682Z GPU thread:139832313131008 size:9961472 | 20260508 13:57:51.786 | NOTICE | [775]: 1970-01-03T18:18:12.682Z GPU thread:139832313131008 size:9961472 | 20260508 13:57:51.786 | NOTICE | [776]: 1970-01-03T18:18:12.683Z GPU thread:139832313131008 size:6815744 | 20260508 13:57:51.786 | NOTICE | [777]: 1970-01-03T18:18:12.683Z GPU thread:139832313131008 size:6815744 | 20260508 13:57:51.786 | NOTICE | [778]: 1970-01-03T18:18:12.684Z GPU thread:139832313131008 size:4194304 | 20260508 13:57:51.786 | NOTICE | [779]: 1970-01-03T18:18:12.686Z GPU thread:139832313131008 size:4194304 | 20260508 13:57:51.786 | NOTICE | [780]: 1970-01-03T18:18:12.688Z GPU thread:139832324673536 size:134217728 | 20260508 13:57:51.786 | NOTICE | [781]: 1970-01-03T18:18:12.704Z HOST thread:139832345665536 size:268435456 | 20260508 13:57:51.786 | NOTICE | [782]: 1970-01-03T18:18:12.707Z GPU thread:139832324673536 size:160432128 | 20260508 13:57:51.786 | NOTICE | [783]: 1970-01-03T18:18:12.897Z HOST thread:139832345665536 size:12582912 | 20260508 13:57:51.786 | NOTICE | [784]: 1970-01-03T18:18:12.913Z GPU thread:139832324673536 size:33030144 | 20260508 13:57:51.786 | NOTICE | [785]: 1970-01-03T18:18:12.914Z GPU thread:139832324673536 size:33030144 | 20260508 13:57:51.786 | NOTICE | [786]: 1970-01-03T18:18:12.915Z GPU thread:139832324673536 size:10485760 | 20260508 13:57:51.786 | NOTICE | [787]: 1970-01-03T18:18:12.916Z GPU thread:139832324673536 size:9961472 | 20260508 13:57:51.786 | NOTICE | [788]: 1970-01-03T18:18:12.916Z GPU thread:139832324673536 size:9961472 | 20260508 13:57:51.786 | NOTICE | [789]: 1970-01-03T18:18:12.917Z GPU thread:139832324673536 size:6815744 | 20260508 13:57:51.786 | NOTICE | [790]: 1970-01-03T18:18:12.917Z GPU thread:139832324673536 size:6815744 | 20260508 13:57:51.786 | NOTICE | [791]: 1970-01-03T18:18:12.917Z GPU thread:139832324673536 size:4194304 | 20260508 13:57:51.786 | NOTICE | [792]: 1970-01-03T18:18:12.917Z GPU thread:139832324673536 size:4194304 | 20260508 13:57:51.786 | NOTICE | [793]: 1970-01-03T18:18:12.924Z HOST thread:139832345665536 size:540016640 | 20260508 13:57:51.786 | NOTICE | [794]: 1970-01-03T18:18:13.255Z HOST thread:139832345665536 size:1048576 | 20260508 13:57:51.786 | NOTICE | [795]: 1970-01-03T18:18:13.255Z HOST thread:139832345665536 size:1048576 | 20260508 13:57:51.786 | NOTICE | [796]: 1970-01-03T18:18:13.256Z HOST thread:139832345665536 size:4194304 | 20260508 13:57:51.786 | NOTICE | [797]: 1970-01-03T18:18:13.259Z HOST thread:139832345665536 size:1 | 20260508 13:57:51.786 | NOTICE | [798]: 1970-01-03T18:18:13.259Z HOST thread:139832345665536 size:2097156 | 20260508 13:57:51.786 | NOTICE | [799]: 1970-01-03T18:18:48.705Z HOST thread:139832341467136 size:1 | 20260508 13:57:51.786 | NOTICE | [800]: 1970-01-03T18:18:48.705Z HOST thread:139832341467136 size:2097156 | 20260508 13:57:51.786 | NOTICE | [801]: 1970-01-03T18:19:45.165Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [802]: 1970-01-03T18:19:45.445Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [803]: 1970-01-03T18:19:45.713Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [804]: 1970-01-03T18:19:45.982Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [805]: 1970-01-03T18:19:46.253Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [806]: 1970-01-03T18:19:46.522Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [807]: 1970-01-03T18:19:46.790Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [808]: 1970-01-03T18:19:47.056Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [809]: 1970-01-03T18:19:47.320Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [810]: 1970-01-03T18:19:47.587Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [811]: 1970-01-03T18:19:47.853Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [812]: 1970-01-03T18:19:48.114Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [813]: 1970-01-03T18:19:50.303Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [814]: 1970-01-03T18:19:50.534Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [815]: 1970-01-03T18:19:50.747Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [816]: 1970-01-03T18:19:50.989Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [817]: 1970-01-03T18:19:51.218Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [818]: 1970-01-03T18:19:51.461Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [819]: 1970-01-03T18:19:51.693Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [820]: 1970-01-03T18:19:51.929Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [821]: 1970-01-03T18:19:52.162Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [822]: 1970-01-03T18:19:52.405Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [823]: 1970-01-03T18:19:52.638Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [824]: 1970-01-03T18:19:52.877Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.786 | NOTICE | [825]: 1970-01-03T18:19:55.445Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [826]: 1970-01-03T18:19:55.702Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [827]: 1970-01-03T18:19:55.960Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [828]: 1970-01-03T18:19:56.220Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [829]: 1970-01-03T18:19:56.480Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [830]: 1970-01-03T18:19:56.741Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [831]: 1970-01-03T18:19:57.002Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [832]: 1970-01-03T18:19:57.269Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [833]: 1970-01-03T18:19:57.533Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [834]: 1970-01-03T18:19:57.799Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [835]: 1970-01-03T18:19:58.061Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [836]: 1970-01-03T18:19:58.322Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [837]: 1970-01-03T18:20:00.583Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [838]: 1970-01-03T18:20:00.849Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [839]: 1970-01-03T18:20:01.108Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [840]: 1970-01-03T18:20:01.364Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [841]: 1970-01-03T18:20:01.636Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [842]: 1970-01-03T18:20:01.907Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [843]: 1970-01-03T18:20:02.173Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [844]: 1970-01-03T18:20:02.447Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [845]: 1970-01-03T18:20:02.719Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.786 | NOTICE | [846]: 1970-01-03T18:20:02.992Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [847]: 1970-01-03T18:20:03.268Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [848]: 1970-01-03T18:20:03.543Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [849]: 1970-01-03T18:20:05.722Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [850]: 1970-01-03T18:20:05.990Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [851]: 1970-01-03T18:20:06.263Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [852]: 1970-01-03T18:20:06.532Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [853]: 1970-01-03T18:20:06.805Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [854]: 1970-01-03T18:20:07.078Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [855]: 1970-01-03T18:20:07.353Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [856]: 1970-01-03T18:20:07.624Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [857]: 1970-01-03T18:20:07.883Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [858]: 1970-01-03T18:20:08.115Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [859]: 1970-01-03T18:20:08.339Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [860]: 1970-01-03T18:20:08.567Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [861]: 1970-01-03T18:20:10.862Z GPU thread:139832307884032 size:8912896 | 20260508 13:57:51.787 | NOTICE | [862]: 1970-01-03T18:20:11.045Z GPU thread:139832303685632 size:8912896 | 20260508 13:57:51.787 | NOTICE | [863]: 1970-01-03T18:20:11.229Z GPU thread:139832307884032 size:8912896 | 20260508 13:57:51.787 | NOTICE | [864]: 1970-01-03T18:20:11.413Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [865]: 1970-01-03T18:20:11.598Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [866]: 1970-01-03T18:20:11.786Z GPU thread:139832349863936 size:8912896 | 20260508 13:57:51.787 | NOTICE | [867]: 1970-01-03T18:20:11.972Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [868]: 1970-01-03T18:20:12.157Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [869]: 1970-01-03T18:20:12.341Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [870]: 1970-01-03T18:20:12.524Z GPU thread:139832341467136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [871]: 1970-01-03T18:20:12.709Z GPU thread:139832328871936 size:8912896 | 20260508 13:57:51.787 | NOTICE | [872]: 1970-01-03T18:20:12.894Z GPU thread:139832333070336 size:8912896 | 20260508 13:57:51.787 | NOTICE | [873]: 1970-01-03T18:20:16.001Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [874]: 1970-01-03T18:20:16.188Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [875]: 1970-01-03T18:20:16.374Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [876]: 1970-01-03T18:20:16.560Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [877]: 1970-01-03T18:20:16.748Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [878]: 1970-01-03T18:20:16.935Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [879]: 1970-01-03T18:20:17.122Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [880]: 1970-01-03T18:20:17.309Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [881]: 1970-01-03T18:20:17.494Z GPU thread:139832324673536 size:8912896 | 20260508 13:57:51.787 | NOTICE | [882]: 1970-01-03T18:20:17.680Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [883]: 1970-01-03T18:20:17.866Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [884]: 1970-01-03T18:20:18.052Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [885]: 1970-01-03T18:20:21.140Z GPU thread:139832337268736 size:8912896 | 20260508 13:57:51.787 | NOTICE | [886]: 1970-01-03T18:20:21.329Z GPU thread:139832337268736 size:8912896 | 20260508 13:57:51.787 | NOTICE | [887]: 1970-01-03T18:20:21.516Z GPU thread:139832337268736 size:8912896 | 20260508 13:57:51.787 | NOTICE | [888]: 1970-01-03T18:20:21.703Z GPU thread:139832337268736 size:8912896 | 20260508 13:57:51.787 | NOTICE | [889]: 1970-01-03T18:20:21.892Z GPU thread:139832337268736 size:8912896 | 20260508 13:57:51.787 | NOTICE | [890]: 1970-01-03T18:20:22.083Z GPU thread:139832337268736 size:8912896 | 20260508 13:57:51.787 | NOTICE | [891]: 1970-01-03T18:20:22.271Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [892]: 1970-01-03T18:20:22.460Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [893]: 1970-01-03T18:20:22.647Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [894]: 1970-01-03T18:20:22.834Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [895]: 1970-01-03T18:20:23.021Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [896]: 1970-01-03T18:20:23.208Z GPU thread:139832320475136 size:8912896 | 20260508 13:57:51.787 | NOTICE | [897]: 1970-01-03T18:20:26.279Z GPU thread:139832313131008 size:8912896 | 20260508 13:57:51.787 | NOTICE | [898]: 1970-01-03T18:20:31.457Z GPU thread:139832303685632 size:268435456 | 20260508 13:57:51.787 | NOTICE | [899]: 1970-01-03T18:20:31.458Z GPU thread:139832303685632 size:12582912 | 20260508 13:57:51.787 | NOTICE | [900]: 1970-01-03T18:20:31.458Z GPU thread:139832303685632 size:1048576 | 20260508 13:57:51.787 | NOTICE | [901]: 1970-01-03T18:20:31.458Z GPU thread:139832303685632 size:1107296256 | 20260508 13:57:51.787 | NOTICE | [902]: 1970-01-03T18:20:31.460Z GPU thread:139832303685632 size:279969792 | 20260508 13:57:51.787 | NOTICE | [903]: 1970-01-03T18:20:31.480Z GPU thread:139832303685632 size:540016640 | 20260508 13:57:51.787 | NOTICE | [904]: 1970-01-03T18:20:31.481Z GPU thread:139832303685632 size:1048576 | 20260508 13:57:51.787 | NOTICE | [905]: 1970-01-03T18:20:31.481Z GPU thread:139832303685632 size:1048576 | 20260508 13:57:51.787 | NOTICE | [906]: 1970-01-03T18:20:31.481Z GPU thread:139832303685632 size:4194304 | 20260508 13:57:51.787 | NOTICE | [907]: 1970-01-03T18:20:31.487Z GPU thread:139832303685632 size:108003328 | 20260508 13:57:51.787 | NOTICE | [908]: 1970-01-03T18:20:31.487Z GPU thread:139832303685632 size:2097156 | 20260508 13:57:51.787 | NOTICE | [909]: 1970-01-03T18:20:31.489Z HOST thread:139832303685632 size:108003328 | 20260508 13:57:51.787 | NOTICE | [910]: 1970-01-03T18:20:41.834Z HOST thread:139832303685632 size:108003328 | 20260508 13:57:51.787 | NOTICE | [911]: 1970-01-03T18:20:51.994Z GPU thread:139832341467136 size:38273024 | 20260508 13:57:51.787 | NOTICE | [912]: 1970-01-03T18:20:51.996Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.787 | NOTICE | [913]: 1970-01-03T18:20:52.004Z HOST thread:139832341467136 size:38273024 | 20260508 13:57:51.787 | NOTICE | [914]: 1970-01-03T18:20:52.029Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.787 | NOTICE | [915]: 1970-01-03T18:20:52.044Z HOST thread:139832333070336 size:38273024 | 20260508 13:57:51.787 | NOTICE | [916]: 1970-01-03T18:20:52.070Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.787 | NOTICE | [917]: 1970-01-03T18:20:52.104Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.787 | NOTICE | [918]: 1970-01-03T18:20:52.106Z HOST thread:139832328871936 size:108003328 | 20260508 13:57:51.787 | NOTICE | [919]: 1970-01-03T18:20:52.263Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.787 | NOTICE | [920]: 1970-01-03T18:20:52.313Z HOST thread:139832337268736 size:2097156 | 20260508 13:57:51.787 | NOTICE | [921]: 1970-01-03T18:20:52.362Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [922]: 1970-01-03T18:20:52.388Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.788 | NOTICE | [923]: 1970-01-03T18:20:52.414Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [924]: 1970-01-03T18:20:52.421Z HOST thread:139832337268736 size:38273024 | 20260508 13:57:51.788 | NOTICE | [925]: 1970-01-03T18:20:52.446Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [926]: 1970-01-03T18:20:52.541Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [927]: 1970-01-03T18:20:52.638Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [928]: 1970-01-03T18:20:52.664Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [929]: 1970-01-03T18:20:52.689Z GPU thread:139832303685632 size:11272192 | 20260508 13:57:51.788 | NOTICE | [930]: 1970-01-03T18:20:52.716Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [931]: 1970-01-03T18:20:52.810Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.788 | NOTICE | [932]: 1970-01-03T18:20:52.907Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [933]: 1970-01-03T18:20:52.933Z GPU thread:139832303685632 size:11272192 | 20260508 13:57:51.788 | NOTICE | [934]: 1970-01-03T18:20:52.959Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [935]: 1970-01-03T18:20:52.984Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [936]: 1970-01-03T18:20:53.077Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [937]: 1970-01-03T18:20:53.174Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [938]: 1970-01-03T18:20:53.200Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.788 | NOTICE | [939]: 1970-01-03T18:20:53.226Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [940]: 1970-01-03T18:20:53.251Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [941]: 1970-01-03T18:20:53.344Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [942]: 1970-01-03T18:20:53.441Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [943]: 1970-01-03T18:20:53.467Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [944]: 1970-01-03T18:20:53.493Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [945]: 1970-01-03T18:20:53.519Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [946]: 1970-01-03T18:20:53.611Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.788 | NOTICE | [947]: 1970-01-03T18:20:53.707Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.788 | NOTICE | [948]: 1970-01-03T18:20:53.731Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [949]: 1970-01-03T18:20:53.756Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [950]: 1970-01-03T18:20:53.781Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.788 | NOTICE | [951]: 1970-01-03T18:20:53.875Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [952]: 1970-01-03T18:20:53.971Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [953]: 1970-01-03T18:20:53.997Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [954]: 1970-01-03T18:20:54.023Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [955]: 1970-01-03T18:20:54.048Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [956]: 1970-01-03T18:20:54.140Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [957]: 1970-01-03T18:20:54.238Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [958]: 1970-01-03T18:20:54.263Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [959]: 1970-01-03T18:20:54.289Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [960]: 1970-01-03T18:20:54.314Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.788 | NOTICE | [961]: 1970-01-03T18:20:54.408Z GPU thread:139832303685632 size:11272192 | 20260508 13:57:51.788 | NOTICE | [962]: 1970-01-03T18:20:54.505Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [963]: 1970-01-03T18:20:54.530Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [964]: 1970-01-03T18:20:54.556Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.788 | NOTICE | [965]: 1970-01-03T18:20:54.583Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [966]: 1970-01-03T18:20:54.673Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [967]: 1970-01-03T18:20:54.770Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [968]: 1970-01-03T18:20:54.795Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [969]: 1970-01-03T18:20:54.821Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [970]: 1970-01-03T18:20:54.846Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [971]: 1970-01-03T18:20:54.938Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [972]: 1970-01-03T18:20:55.034Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.788 | NOTICE | [973]: 1970-01-03T18:20:55.059Z GPU thread:139832303685632 size:11272192 | 20260508 13:57:51.788 | NOTICE | [974]: 1970-01-03T18:20:55.085Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [975]: 1970-01-03T18:20:55.111Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [976]: 1970-01-03T18:20:55.199Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [977]: 1970-01-03T18:21:12.533Z HOST thread:139832337268736 size:24 | 20260508 13:57:51.788 | NOTICE | [978]: 1970-01-03T18:21:12.533Z GPU thread:139832337268736 size:24 | 20260508 13:57:51.788 | NOTICE | [979]: 1970-01-03T18:21:33.373Z HOST thread:139832313131008 size:108003328 | 20260508 13:57:51.788 | NOTICE | [980]: 1970-01-03T18:21:34.981Z GPU thread:139832303685632 size:11272192 | 20260508 13:57:51.788 | NOTICE | [981]: 1970-01-03T18:21:35.006Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [982]: 1970-01-03T18:21:35.032Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [983]: 1970-01-03T18:21:35.058Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [984]: 1970-01-03T18:21:35.084Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [985]: 1970-01-03T18:21:35.192Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.788 | NOTICE | [986]: 1970-01-03T18:21:35.218Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [987]: 1970-01-03T18:21:35.243Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [988]: 1970-01-03T18:21:35.269Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.788 | NOTICE | [989]: 1970-01-03T18:21:35.295Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [990]: 1970-01-03T18:21:35.320Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.788 | NOTICE | [991]: 1970-01-03T18:21:35.346Z GPU thread:139832333070336 size:11272192 | 20260508 13:57:51.788 | NOTICE | [992]: 1970-01-03T18:21:35.454Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.788 | NOTICE | [993]: 1970-01-03T18:21:35.480Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [994]: 1970-01-03T18:21:35.505Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.788 | NOTICE | [995]: 1970-01-03T18:21:35.532Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.788 | NOTICE | [996]: 1970-01-03T18:21:35.557Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.789 | NOTICE | [997]: 1970-01-03T18:21:35.583Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.789 | NOTICE | [998]: 1970-01-03T18:21:35.609Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.789 | NOTICE | [999]: 1970-01-03T18:21:35.718Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1000]: 1970-01-03T18:21:35.744Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1001]: 1970-01-03T18:21:35.770Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1002]: 1970-01-03T18:21:35.796Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1003]: 1970-01-03T18:21:35.822Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1004]: 1970-01-03T18:21:35.848Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1005]: 1970-01-03T18:21:35.874Z GPU thread:139832307884032 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1006]: 1970-01-03T18:21:35.982Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1007]: 1970-01-03T18:21:36.008Z GPU thread:139832345665536 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1008]: 1970-01-03T18:21:36.033Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1009]: 1970-01-03T18:21:36.059Z GPU thread:139832337268736 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1010]: 1970-01-03T18:21:36.085Z GPU thread:139832313131008 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1011]: 1970-01-03T18:21:36.110Z GPU thread:139832324673536 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1012]: 1970-01-03T18:21:36.136Z GPU thread:139832341467136 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1013]: 1970-01-03T18:21:36.243Z GPU thread:139832320475136 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1014]: 1970-01-03T18:21:36.268Z GPU thread:139832349863936 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1015]: 1970-01-03T18:21:36.292Z GPU thread:139832328871936 size:11272192 | 20260508 13:57:51.789 | NOTICE | [1016]: 1970-01-03T18:22:16.498Z HOST thread:139832307884032 size:38273024 | 20260508 13:57:51.789 | NOTICE | [1017]: 1970-01-03T18:25:00.156Z HOST thread:139832333070336 size:2097156 | 20260508 13:57:51.789 | NOTICE | [1018]: 1970-01-03T18:55:50.096Z HOST thread:139832313131008 size:108003328 | 20260508 13:57:51.789 | NOTICE | [1019]: 1970-01-03T18:56:31.243Z HOST thread:139832333070336 size:2097156 | 20260508 13:57:51.789 | NOTICE | [1020]: 1970-01-03T18:56:31.245Z HOST thread:139832333070336 size:38273024 | 20260508 13:57:51.789 | NOTICE | [1021]: 1970-01-03T18:56:31.278Z GPU thread:139832328871936 size:38273024 | 20260508 13:57:51.789 | NOTICE | [1022]: 1970-01-03T19:26:41.349Z HOST thread:139832320475136 size:38273024 | 20260508 13:57:51.789 | NOTICE | [1023]: 1970-01-03T20:49:36.633Z HOST thread:139832303685632 size:108003328 | 20260508 13:57:51.789 | NOTICE | ----- PBCudaRuntime DumpAllocationHistoryToLog ---- (end) | 20260508 13:57:51.789 | INFO | Application done | 20260508 13:57:51.789 | INFO | Run Time : 1d 10s | 20260508 13:57:51.789 | INFO | CPU Time : 2d 3h | 20260508 13:57:51.789 | INFO | Peak RSS : 58.860 GB | 20260508 13:57:51.789 | INFO | Exit code : 0